diff --git a/Android.bp b/Android.bp index 76a43caf97230b28df94ced64b2db6f5403ad31b..14290b9e1c9d0926add9eb57a6c0054187e7c025 100644 --- a/Android.bp +++ b/Android.bp @@ -51,6 +51,7 @@ opencl_srcs = [ "src/core/CL/cl_kernels/common/instance_normalization.cl", "src/core/CL/cl_kernels/common/l2_normalize.cl", "src/core/CL/cl_kernels/common/mat_mul.cl", + "src/core/CL/cl_kernels/common/mat_mul_mmul.cl", "src/core/CL/cl_kernels/common/mat_mul_quantized.cl", "src/core/CL/cl_kernels/common/mean_stddev_normalization.cl", "src/core/CL/cl_kernels/common/memset.cl", @@ -165,7 +166,9 @@ arm_compute_library_defaults { "-DARM_COMPUTE_ENABLE_NEON", "-Wno-unused-parameter", "-DNO_DOT_IN_TOOLCHAIN", - "-Wno-implicit-fallthrough" + "-Wno-implicit-fallthrough", + "-fPIC", + "-DACL_INTERNAL_TEST_CKW_IN_DF" ], rtti: true, } @@ -176,6 +179,8 @@ cc_library_static { proprietary: true, local_include_dirs: ["build/android-arm64v8a/src/core", "build/android-arm64v8a/src/core/CL", + "compute_kernel_writer/prototype/include", + "compute_kernel_writer/prototype", "src/core/common", "src/core/helpers", "src/core/NEON/kernels/arm_gemm", @@ -185,6 +190,15 @@ cc_library_static { "src/cpu/kernels/assembly"], export_include_dirs: [".", "./include"], srcs: [ + "compute_kernel_writer/prototype/src/Kernel.cpp", + "compute_kernel_writer/prototype/src/KernelArgument.cpp", + "compute_kernel_writer/prototype/src/KernelWriter.cpp", + "compute_kernel_writer/prototype/src/OperandBase.cpp", + "compute_kernel_writer/prototype/src/TensorInfo.cpp", + "compute_kernel_writer/prototype/src/TensorOperand.cpp", + "compute_kernel_writer/prototype/src/TensorTileSampler.cpp", + "compute_kernel_writer/prototype/src/TileInfo.cpp", + "compute_kernel_writer/prototype/src/TileOperand.cpp", "src/c/AclContext.cpp", "src/c/AclOperator.cpp", "src/c/AclQueue.cpp", @@ -313,9 +327,9 @@ cc_library_static { "src/core/NEON/kernels/arm_conv/depthwise/depthwise_strategies_common.cpp", "src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8q.cpp", "src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8s8u8q.cpp", - "src/core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.cpp", "src/core/NEON/kernels/arm_conv/depthwise/interleaves/generic.cpp", "src/core/NEON/kernels/arm_conv/depthwise/interleaves/generic_quantized_dot_product.cpp", + "src/core/NEON/kernels/arm_conv/depthwise/premultiply.cpp", "src/core/NEON/kernels/arm_conv/pooling/kernels/cpp_nhwc_1x1_stride_any_depthfirst/generic.cpp", "src/core/NEON/kernels/arm_conv/pooling/pooling_fp16.cpp", "src/core/NEON/kernels/arm_conv/pooling/pooling_fp32.cpp", @@ -363,7 +377,6 @@ cc_library_static { "src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_4x4_3x3.cpp", "src/core/NEON/kernels/convolution/winograd/output_transforms_fp16.cpp", "src/core/NEON/kernels/convolution/winograd/output_transforms_fp32.cpp", - "src/core/NEON/kernels/convolution/winograd/padding.cpp", "src/core/NEON/kernels/convolution/winograd/weight_transforms/arm_fp32_2x2_3x3.cpp", "src/core/NEON/kernels/convolution/winograd/weight_transforms/arm_fp32_2x2_5x5.cpp", "src/core/NEON/kernels/convolution/winograd/weight_transforms/arm_fp32_4x4_3x3.cpp", @@ -384,8 +397,14 @@ cc_library_static { "src/core/Version.cpp", "src/core/helpers/SoftmaxHelpers.cpp", "src/core/helpers/WindowHelpers.cpp", + "src/core/utils/ActivationFunctionUtils.cpp", "src/core/utils/AssemblyUtils.cpp", + "src/core/utils/DataLayoutUtils.cpp", + "src/core/utils/DataTypeUtils.cpp", + "src/core/utils/FormatUtils.cpp", + "src/core/utils/InterpolationPolicyUtils.cpp", "src/core/utils/ScaleUtils.cpp", + "src/core/utils/StringUtils.cpp", "src/core/utils/helpers/fft.cpp", "src/core/utils/helpers/tensor_transform.cpp", "src/core/utils/io/FileHandler.cpp", @@ -604,6 +623,7 @@ cc_library_static { "src/cpu/operators/internal/CpuGemmAssemblyDispatch.cpp", "src/dynamic_fusion/runtime/gpu/cl/ClKernelRuntime.cpp", "src/dynamic_fusion/runtime/gpu/cl/ClWorkloadRuntime.cpp", + "src/dynamic_fusion/runtime/gpu/cl/ckw_driver/GpuCkwKernelArgumentsHelpers.cpp", "src/dynamic_fusion/sketch/attributes/CastAttributes.cpp", "src/dynamic_fusion/sketch/attributes/ClampAttributes.cpp", "src/dynamic_fusion/sketch/attributes/Conv2dAttributes.cpp", @@ -620,6 +640,16 @@ cc_library_static { "src/dynamic_fusion/sketch/gpu/GpuOperatorGroup.cpp", "src/dynamic_fusion/sketch/gpu/GpuWorkloadContext.cpp", "src/dynamic_fusion/sketch/gpu/GpuWorkloadSketch.cpp", + "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwComponentArgument.cpp", + "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwDriver.cpp", + "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwKernelWriter.cpp", + "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwScopedKernelWriter.cpp", + "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwVariableTable.cpp", + "src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwActivation.cpp", + "src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwCast.cpp", + "src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwDirectConv2d.cpp", + "src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwElementwiseBinary.cpp", + "src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwStore.cpp", "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentActivation.cpp", "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentCast.cpp", "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentDepthwiseConv2d.cpp", @@ -700,6 +730,7 @@ cc_library_static { "src/gpu/cl/kernels/ClIndirectConv2dKernel.cpp", "src/gpu/cl/kernels/ClMatMulLowpNativeKernel.cpp", "src/gpu/cl/kernels/ClMatMulNativeKernel.cpp", + "src/gpu/cl/kernels/ClMatMulNativeMMULKernel.cpp", "src/gpu/cl/kernels/ClMulKernel.cpp", "src/gpu/cl/kernels/ClPermuteKernel.cpp", "src/gpu/cl/kernels/ClPool2dKernel.cpp", @@ -1020,7 +1051,6 @@ cc_library_static { srcs: [ "src/core/NEON/kernels/arm_conv/depthwise/interleaves/a64_s8q_3x3_dot.cpp", "src/core/NEON/kernels/arm_conv/depthwise/interleaves/a64_u8q_3x3_dot.cpp", - "src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_8b_mla.cpp", "src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_s8q_3x3_dot.cpp", "src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_u8q_3x3_dot.cpp", "src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp", diff --git a/BUILD.bazel b/BUILD.bazel index f1766d9582ca433953cd780649d8662027cb3eda..3a7d941a0e74811dd66794f95788ab9bd6f3c942 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -382,6 +382,7 @@ cc_library( }), includes = [ "arm_compute/runtime", + "src/core/NEON/kernels/arm_gemm", "src/core/NEON/kernels/assembly", "src/core/NEON/kernels/convolution/common", "src/core/NEON/kernels/convolution/winograd", diff --git a/CMakeLists.txt b/CMakeLists.txt index 8ab69f73115bc27bda220d6b77582a3d0a73f042..bf029a7e9e15d032c4f6da46f74ccd9f34b0783c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,7 +28,7 @@ cmake_minimum_required(VERSION 3.13 FATAL_ERROR) list(APPEND CMAKE_MESSAGE_CONTEXT ArmCompute) project( ArmCompute - VERSION 31.0.1 + VERSION 32.0.0 DESCRIPTION "The Arm Compute Library is a collection of low-level machine learning functions optimized for Arm® Cortex®-A CPU and Arm® Mali™ GPU architectures" LANGUAGES C CXX ASM) @@ -197,6 +197,7 @@ target_include_directories( ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE src src/cpu/kernels/assembly + src/core/NEON/kernels/arm_gemm src/core/NEON/kernels/assembly src/core/NEON/kernels/convolution/common src/core/NEON/kernels/arm_conv/depthwise @@ -223,6 +224,7 @@ target_include_directories( ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE src src/cpu/kernels/assembly + src/core/NEON/kernels/arm_gemm src/core/NEON/kernels/assembly src/core/NEON/kernels/convolution/common src/core/NEON/kernels/arm_conv/depthwise diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b11239035a13658c58b48e745baf7f62976313f7..171d101bd11dffb48b592db3b3ae8361ef2d2684 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -Please read https://arm-software.github.io/ComputeLibrary/v23.02.1/contribution_guidelines.xhtml +Please read https://arm-software.github.io/ComputeLibrary/latest/contribution_guidelines.xhtml Here on github we only publish a snapshot of the main development branch for each release, that's the reason why we don't accept pull requests. diff --git a/README.md b/README.md index 8585ddd94fef5c5881b0922b3084b2e80e607086..a8f0def7a18b9ae2a940dcda026142f00c86b9a9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ > **⚠ Important** -> From release 22.05: 'master' branch has been replaced with 'main' following our inclusive language update, more information [here](https://arm-software.github.io/ComputeLibrary/v23.05.1/contribution_guidelines.xhtml#S5_0_inc_lang). +> From release 22.05: 'master' branch has been replaced with 'main' following our inclusive language update, more information [here](https://arm-software.github.io/ComputeLibrary/latest/contribution_guidelines.xhtml#S5_0_inc_lang). > **⚠ Important** > From release 22.08: armv7a with Android build will no longer be tested or maintained. @@ -16,7 +16,7 @@

-# Compute Library ![](https://img.shields.io/badge/latest_release-23.05.1-green) +# Compute Library ![](https://img.shields.io/badge/latest_release-23.08-green) The Compute Library is a collection of low-level machine learning functions optimized for Arm® Cortex®-A, Arm® Neoverse® and Arm® Mali™ GPUs architectures.
@@ -44,7 +44,7 @@ Key Features:
## Documentation -[![Documentation](https://img.shields.io/badge/documentation-23.05.1-green)](https://arm-software.github.io/ComputeLibrary/v23.05.1) +[![Documentation](https://img.shields.io/badge/documentation-23.08-green)](https://arm-software.github.io/ComputeLibrary/latest) > Note: The documentation includes the reference API, changelogs, build guide, contribution guide, errata, etc. @@ -57,24 +57,24 @@ All the binaries can be downloaded from [here](https://github.com/ARM-software/C | Platform | Operating System | Release archive (Download) | | -------------- | ---------------- | -------------------------- | -| Raspberry Pi 4 | Linux 32bit | [![](https://img.shields.io/badge/build-neon-orange)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.05.1/arm_compute-v23.05.1-bin-linux-armv7a-neon.tar.gz) | -| Raspberry Pi 4 | Linux 64bit | [![](https://img.shields.io/badge/build-neon-orange)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.05.1/arm_compute-v23.05.1-bin-linux-arm64-v8a-neon.tar.gz) | -| Odroid N2 | Linux 64bit | [![](https://img.shields.io/badge/build-neon-orange)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.05.1/arm_compute-v23.05.1-bin-linux-arm64-v8a-neon.tar.gz) [![](https://img.shields.io/badge/build-opencl-blue)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.05.1/arm_compute-v23.05.1-bin-linux-arm64-v8a-cl.tar.gz) [![](https://img.shields.io/badge/build-neon+cl-yellowgreen)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.05.1/arm_compute-v23.05.1-bin-linux-arm64-v8a-neon-cl.tar.gz) | -| HiKey960 | Linux 64bit | [![](https://img.shields.io/badge/build-neon-orange)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.05.1/arm_compute-v23.05.1-bin-linux-arm64-v8a-neon.tar.gz) [![](https://img.shields.io/badge/build-opencl-blue)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.05.1/arm_compute-v23.05.1-bin-linux-arm64-v8a-cl.tar.gz) [![](https://img.shields.io/badge/build-neon+cl-yellowgreen)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.05.1/arm_compute-v23.05.1-bin-linux-arm64-v8a-neon-cl.tar.gz) | +| Raspberry Pi 4 | Linux® 32bit | [![](https://img.shields.io/badge/build-neon-orange)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.08/arm_compute-v23.08-bin-linux-armv7a-neon.tar.gz) | +| Raspberry Pi 4 | Linux® 64bit | [![](https://img.shields.io/badge/build-neon-orange)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.08/arm_compute-v23.08-bin-linux-arm64-v8a-neon.tar.gz) | +| Odroid N2 | Linux® 64bit | [![](https://img.shields.io/badge/build-neon-orange)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.08/arm_compute-v23.08-bin-linux-arm64-v8a-neon.tar.gz) [![](https://img.shields.io/badge/build-opencl-blue)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.08/arm_compute-v23.08-bin-linux-arm64-v8a-cl.tar.gz) [![](https://img.shields.io/badge/build-neon+cl-yellowgreen)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.08/arm_compute-v23.08-bin-linux-arm64-v8a-neon-cl.tar.gz) | +| HiKey960 | Linux® 64bit | [![](https://img.shields.io/badge/build-neon-orange)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.08/arm_compute-v23.08-bin-linux-arm64-v8a-neon.tar.gz) [![](https://img.shields.io/badge/build-opencl-blue)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.08/arm_compute-v23.08-bin-linux-arm64-v8a-cl.tar.gz) [![](https://img.shields.io/badge/build-neon+cl-yellowgreen)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.08/arm_compute-v23.08-bin-linux-arm64-v8a-neon-cl.tar.gz) |
| Architecture | Operating System | Release archive (Download) | | ------------ | ---------------- | -------------------------- | -| armv7 | Linux | [![](https://img.shields.io/badge/build-neon-orange)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.05.1/arm_compute-v23.05.1-bin-linux-armv7a-neon.tar.gz) [![](https://img.shields.io/badge/build-opencl-blue)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.05.1/arm_compute-v23.05.1-bin-linux-armv7a-cl.tar.gz) [![](https://img.shields.io/badge/build-neon+cl-yellowgreen)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.05.1/arm_compute-v23.05.1-bin-linux-armv7a-neon-cl.tar.gz) | -| arm64-v8a | Android | [![](https://img.shields.io/badge/build-neon-orange)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.05.1/arm_compute-v23.05.1-bin-android-arm64-v8a-neon.tar.gz) [![](https://img.shields.io/badge/build-opencl-blue)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.05.1/arm_compute-v23.05.1-bin-android-arm64-v8a-cl.tar.gz) [![](https://img.shields.io/badge/build-neon+cl-yellowgreen)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.05.1/arm_compute-v23.05.1-bin-android-arm64-v8a-neon-cl.tar.gz) | -| arm64-v8a | Linux | [![](https://img.shields.io/badge/build-neon-orange)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.05.1/arm_compute-v23.05.1-bin-linux-arm64-v8a-neon.tar.gz) [![](https://img.shields.io/badge/build-opencl-blue)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.05.1/arm_compute-v23.05.1-bin-linux-arm64-v8a-cl.tar.gz) [![](https://img.shields.io/badge/build-neon+cl-yellowgreen)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.05.1/arm_compute-v23.05.1-bin-linux-arm64-v8a-neon-cl.tar.gz) | -| arm64-v8.2-a | Android | [![](https://img.shields.io/badge/build-neon-orange)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.05.1/arm_compute-v23.05.1-bin-android-arm64-v8.2-a-neon.tar.gz) [![](https://img.shields.io/badge/build-opencl-blue)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.05.1/arm_compute-v23.05.1-bin-android-arm64-v8.2-a-cl.tar.gz) [![](https://img.shields.io/badge/build-neon+cl-yellowgreen)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.05.1/arm_compute-v23.05.1-bin-android-arm64-v8.2-a-neon-cl.tar.gz) | -| arm64-v8.2-a | Linux | [![](https://img.shields.io/badge/build-neon-orange)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.05.1/arm_compute-v23.05.1-bin-linux-arm64-v8.2-a-neon.tar.gz) [![](https://img.shields.io/badge/build-opencl-blue)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.05.1/arm_compute-v23.05.1-bin-linux-arm64-v8.2-a-cl.tar.gz) [![](https://img.shields.io/badge/build-neon+cl-yellowgreen)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.05.1/arm_compute-v23.05.1-bin-linux-arm64-v8.2-a-neon-cl.tar.gz) | +| armv7 | Linux® | [![](https://img.shields.io/badge/build-neon-orange)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.08/arm_compute-v23.08-bin-linux-armv7a-neon.tar.gz) [![](https://img.shields.io/badge/build-opencl-blue)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.08/arm_compute-v23.08-bin-linux-armv7a-cl.tar.gz) [![](https://img.shields.io/badge/build-neon+cl-yellowgreen)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.08/arm_compute-v23.08-bin-linux-armv7a-neon-cl.tar.gz) | +| arm64-v8a | Android™ | [![](https://img.shields.io/badge/build-neon-orange)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.08/arm_compute-v23.08-bin-android-arm64-v8a-neon.tar.gz) [![](https://img.shields.io/badge/build-opencl-blue)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.08/arm_compute-v23.08-bin-android-arm64-v8a-cl.tar.gz) [![](https://img.shields.io/badge/build-neon+cl-yellowgreen)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.08/arm_compute-v23.08-bin-android-arm64-v8a-neon-cl.tar.gz) | +| arm64-v8a | Linux® | [![](https://img.shields.io/badge/build-neon-orange)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.08/arm_compute-v23.08-bin-linux-arm64-v8a-neon.tar.gz) [![](https://img.shields.io/badge/build-opencl-blue)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.08/arm_compute-v23.08-bin-linux-arm64-v8a-cl.tar.gz) [![](https://img.shields.io/badge/build-neon+cl-yellowgreen)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.08/arm_compute-v23.08-bin-linux-arm64-v8a-neon-cl.tar.gz) | +| arm64-v8.2-a | Android™ | [![](https://img.shields.io/badge/build-neon-orange)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.08/arm_compute-v23.08-bin-android-arm64-v8.2-a-neon.tar.gz) [![](https://img.shields.io/badge/build-opencl-blue)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.08/arm_compute-v23.08-bin-android-arm64-v8.2-a-cl.tar.gz) [![](https://img.shields.io/badge/build-neon+cl-yellowgreen)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.08/arm_compute-v23.08-bin-android-arm64-v8.2-a-neon-cl.tar.gz) | +| arm64-v8.2-a | Linux® | [![](https://img.shields.io/badge/build-neon-orange)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.08/arm_compute-v23.08-bin-linux-arm64-v8.2-a-neon.tar.gz) [![](https://img.shields.io/badge/build-opencl-blue)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.08/arm_compute-v23.08-bin-linux-arm64-v8.2-a-cl.tar.gz) [![](https://img.shields.io/badge/build-neon+cl-yellowgreen)](https://github.com/ARM-software/ComputeLibrary/releases/download/v23.08/arm_compute-v23.08-bin-linux-arm64-v8.2-a-neon-cl.tar.gz) |
-Please refer to the following link for more pre-built binaries: [![](https://img.shields.io/badge/v23.05.1-bins-yellowgreen)](https://github.com/ARM-software/ComputeLibrary/releases/tag/v23.05.1) +Please refer to the following link for more pre-built binaries: [![](https://img.shields.io/badge/v23.08-bins-yellowgreen)](https://github.com/ARM-software/ComputeLibrary/releases/tag/v23.08) Pre-build binaries are generated with the following security / good coding practices related flags: > -Wall, -Wextra, -Wformat=2, -Winit-self, -Wstrict-overflow=2, -Wswitch-default, -Woverloaded-virtual, -Wformat-security, -Wctor-dtor-privacy, -Wsign-promo, -Weffc++, -pedantic, -fstack-protector-strong @@ -116,13 +116,13 @@ Pre-build binaries are generated with the following security / good coding pract ## Experimental builds -**⚠ Important** Bazel and CMake builds are experimental CPU only builds, please see the [documentation](https://arm-software.github.io/ComputeLibrary/v23.05.1/how_to_build.xhtml) for more details. +**⚠ Important** Bazel and CMake builds are experimental CPU only builds, please see the [documentation](https://arm-software.github.io/ComputeLibrary/latest/how_to_build.xhtml) for more details.
## How to contribute -Contributions to the Compute Library are more than welcome. If you are interested on contributing, please have a look at our [how to contribute guidelines](https://arm-software.github.io/ComputeLibrary/v23.05.1/contribution_guidelines.xhtml). +Contributions to the Compute Library are more than welcome. If you are interested on contributing, please have a look at our [how to contribute guidelines](https://arm-software.github.io/ComputeLibrary/latest/contribution_guidelines.xhtml). ### Developer Certificate of Origin (DCO) Before the Compute Library accepts your contribution, you need to certify its origin and give us your permission. To manage this process we use the Developer Certificate of Origin (DCO) V1.1 (https://developercertificate.org/) diff --git a/SConscript b/SConscript index 75aedbf1cf692b4c29f462eab778f11947949251..467f84cb55ae335738975332b2859aa56c10be97 100644 --- a/SConscript +++ b/SConscript @@ -31,10 +31,17 @@ import zlib import json import codecs -VERSION = "v23.05.1" -LIBRARY_VERSION_MAJOR = 31 +from SCons.Warnings import warn, DeprecatedWarning + +warn(DeprecatedWarning, + "DEPRECATION NOTICE: Legacy libarm_compute_core has been deprecated and is scheduled for removal in 24.02 release." + " Link your application only to libarm_compute for core library functionality" + ) + +VERSION = "v23.08" +LIBRARY_VERSION_MAJOR = 32 LIBRARY_VERSION_MINOR = 0 -LIBRARY_VERSION_PATCH = 1 +LIBRARY_VERSION_PATCH = 0 SONAME_VERSION = str(LIBRARY_VERSION_MAJOR) + "." + str(LIBRARY_VERSION_MINOR) + "." + str(LIBRARY_VERSION_PATCH) Import('env') @@ -49,8 +56,6 @@ def build_bootcode_objs(sources): return obj - - # @brief Create a list of object from a given file list. # # @param arch_info A dictionary represents the architecture info such as the @@ -113,6 +118,24 @@ def build_lib_objects(): return lib_static_objs, lib_shared_objs +# The built-in SCons Glob() method does not support recursive searching of directories, thus we implement our own: +def recursive_glob(root_dir, pattern): + files = [] + regex = re.compile(pattern) + + for dirpath, _, filenames in os.walk(root_dir): + for f in filenames: + f = os.path.join(dirpath, f) + if regex.match(f): + files.append(f) + + return files + + +def get_ckw_obj_list(): + cmake_obj_dir = os.path.abspath("prototype/CMakeFiles/ckw_prototype.dir/src") + return recursive_glob(root_dir=cmake_obj_dir, pattern=".*.o$") + def build_library(name, build_env, sources, static=False, libs=[]): cloned_build_env = build_env.Clone() @@ -120,9 +143,25 @@ def build_library(name, build_env, sources, static=False, libs=[]): cloned_build_env["LINKFLAGS"].remove('-pie') cloned_build_env["LINKFLAGS"].remove('-static-libstdc++') + # -- Static Library -- if static: - obj = cloned_build_env.StaticLibrary(name, source=sources, LIBS = arm_compute_env["LIBS"] + libs) + # Recreate the list to avoid mutating the original + static_sources = list(sources) + + # Dynamic Fusion has a direct dependency on the Compute Kernel Writer (CKW) subproject, therefore we collect the + # built CKW objects to pack into the Compute Library archive. + if env['experimental_dynamic_fusion'] and name == "arm_compute-static": + static_sources += get_ckw_obj_list() + + obj = cloned_build_env.StaticLibrary(name, source=static_sources, LIBS=arm_compute_env["LIBS"] + libs) + + # -- Shared Library -- else: + # Always statically link Compute Library against CKW + if env['experimental_dynamic_fusion'] and name == "arm_compute": + libs.append('libckw_prototype.a') + + # Add shared library versioning if env['set_soname']: obj = cloned_build_env.SharedLibrary(name, source=sources, SHLIBVERSION = SONAME_VERSION, LIBS = arm_compute_env["LIBS"] + libs) else: @@ -220,7 +259,8 @@ def create_version_file(target, source, env): except (OSError, subprocess.CalledProcessError): git_hash="unknown" - build_info = "\"arm_compute_version=%s Build options: %s Git hash=%s\"" % (VERSION, vars.args, git_hash.strip()) + build_options = str(vars.args).replace('"', '\\"') + build_info = "\"arm_compute_version=%s Build options: %s Git hash=%s\"" % (VERSION,build_options, git_hash.strip()) with open(target[0].get_path(), "w") as fd: fd.write(build_info) @@ -394,6 +434,7 @@ if env['opencl'] and env['embed_kernels']: 'src/core/CL/cl_kernels/common/instance_normalization.cl', 'src/core/CL/cl_kernels/common/l2_normalize.cl', 'src/core/CL/cl_kernels/common/mat_mul.cl', + 'src/core/CL/cl_kernels/common/mat_mul_mmul.cl', 'src/core/CL/cl_kernels/common/mat_mul_quantized.cl', 'src/core/CL/cl_kernels/common/mean_stddev_normalization.cl', 'src/core/CL/cl_kernels/common/memset.cl', @@ -493,7 +534,8 @@ arm_compute_env.Append(CPPDEFINES = [('ARM_COMPUTE_VERSION_MAJOR', LIBRARY_VERSI # Don't allow undefined references in the libraries: undefined_flag = '-Wl,-undefined,error' if 'macos' in arm_compute_env["os"] else '-Wl,--no-undefined' -arm_compute_env.Append(LINKFLAGS=[undefined_flag]) +if not env['thread_sanitizer']: + arm_compute_env.Append(LINKFLAGS=[undefined_flag]) arm_compute_env.Append(CPPPATH =[Dir("./src/core/").path] ) if env['os'] != 'openbsd': @@ -522,7 +564,14 @@ if env['fixed_format_kernels']: # Experimental files # Dynamic fusion if env['experimental_dynamic_fusion']: - lib_files += filelist['experimental']['dynamic_fusion'] + lib_files += filelist['experimental']['dynamic_fusion']['common'] + lib_files += filelist['experimental']['dynamic_fusion']['template_writer'] + +if "ACL_INTERNAL_TEST_CKW_IN_DF" in env["extra_cxx_flags"]: + if not env["experimental_dynamic_fusion"]: + print("To use ACL_INTERNAL_TEST_CKW_IN_DF experimental_dynamic_fusion must be set to 1") + Exit(1) + lib_files += filelist['experimental']['dynamic_fusion']['ckw_driver'] # Logging files if env["logging"]: @@ -548,7 +597,7 @@ custom_operators = [] custom_types = [] custom_layouts = [] -use_custom_ops = env['high_priority'] or env['build_config']; +use_custom_ops = env['high_priority'] or env['build_config'] if env['high_priority']: custom_operators = filelist['high_priority'] @@ -574,7 +623,8 @@ lib_files_sve2 = [] if env['neon']: # build winograd/depthwise sources for either v7a / v8a - arm_compute_env.Append(CPPPATH = ["src/core/NEON/kernels/convolution/common/", + arm_compute_env.Append(CPPPATH = ["src/core/NEON/kernels/arm_gemm", + "src/core/NEON/kernels/convolution/common/", "src/core/NEON/kernels/convolution/winograd/", "src/core/NEON/kernels/arm_conv/depthwise/", "src/core/NEON/kernels/arm_conv/pooling/", @@ -682,7 +732,7 @@ arm_compute_graph_env = arm_compute_env.Clone() # Build graph libraries arm_compute_graph_env.Append(CXXFLAGS = ['-Wno-redundant-move', '-Wno-pessimizing-move']) -arm_compute_graph_a = build_library('arm_compute_graph-static', arm_compute_graph_env, graph_files, static=True, libs = [ arm_compute_a ]) +arm_compute_graph_a = build_library('arm_compute_graph-static', arm_compute_graph_env, graph_files, static=True) Export('arm_compute_graph_a') if env['os'] != 'bare_metal' and not env['standalone']: diff --git a/SConstruct b/SConstruct index f6c90c3098d2f4793ccef605779800927f74d7d6..68c518a4a0af20a978d9ef0487d3f6e7e41d3282 100644 --- a/SConstruct +++ b/SConstruct @@ -25,7 +25,7 @@ import SCons import json import os -from subprocess import check_output +import subprocess def version_at_least(version, required): @@ -125,7 +125,7 @@ vars.AddVariables(     ├── datasets     ├── fixtures     └── Neon\n""", "", PathVariable.PathAccept), - BoolVariable("experimental_dynamic_fusion", "Build the experimental dynamic fusion files", False), + BoolVariable("experimental_dynamic_fusion", "Build the experimental dynamic fusion files. This option also enables opencl=1 on which it has a direct dependency.", False), BoolVariable("fixed_format_kernels", "Enable fixed format kernels for GEMM", False), BoolVariable("mapfile", "Generate a map file", False), ListVariable("custom_options", "Custom options that can be used to turn on/off features", "none", ["disable_mmla_fp"]), @@ -133,6 +133,7 @@ vars.AddVariables( ListVariable("data_layout_support", "Enable a list of data layout to support", "all", ["nhwc", "nchw"]), ("toolchain_prefix", "Override the toolchain prefix; used by all toolchain components: compilers, linker, assembler etc. If unspecified, use default(auto) prefixes; if passed an empty string '' prefixes would be disabled", "auto"), ("compiler_prefix", "Override the compiler prefix; used by just compilers (CC,CXX); further overrides toolchain_prefix for compilers; this is for when the compiler prefixes are different from that of the linkers, archivers etc. If unspecified, this is the same as toolchain_prefix; if passed an empty string '' prefixes would be disabled", "auto"), + BoolVariable("thread_sanitizer", "Enable ThreadSanitizer", False), ("extra_cxx_flags", "Extra CXX flags to be appended to the build command", ""), ("extra_link_flags", "Extra LD flags to be appended to the build command", ""), ("compiler_cache", "Command to prefix to the C and C++ compiler (e.g ccache)", ""), @@ -160,7 +161,7 @@ install_path = env['install_dir'] if not env['install_dir'].startswith('/') and install_path != "": install_path = "%s/%s" % (build_path, install_path) -env.Append(LIBPATH = [build_path]) +env.Append(LIBPATH = [build_path, os.path.join(build_path, "prototype")]) Export('env') Export('vars') @@ -214,6 +215,10 @@ if env['os'] == 'bare_metal': print("ERROR: OpenMP and C++11 threads not supported in bare_metal. Use cppthreads=0 openmp=0") Exit(1) +if env['experimental_dynamic_fusion']: + # Dynamic Fusion on GPU has a direct dependency on OpenCL and Compute Kernel Writer + env['opencl'] = 1 + if env['opencl'] and env['embed_kernels'] and env['compress_kernels'] and env['os'] not in ['android']: print("Compressed kernels are supported only for android builds") Exit(1) @@ -415,12 +420,57 @@ print("Using compilers:") print("CC", env['CC']) print("CXX", env['CXX']) +"""Build the Compute Kernel Writer subproject""" +if env['experimental_dynamic_fusion']: + # Strip ccache prefix from CC and CXX to obtain only the target triple + CKW_CC = env['CC'].replace(env['compiler_cache'] + " ", "") + CKW_CXX = env['CXX'].replace(env['compiler_cache'] + " ", "") + CKW_CCACHE = 1 if env['compiler_cache'] else 0 + + CKW_BUILD_TYPE = "Debug" if env['debug'] else "Release" + + CKW_ENABLE_OPENCL = env['opencl'] + CKW_ENABLE_ASSERTS = env['debug'] or env['asserts'] + + CKW_PROJECT_DIR = Dir('.').path + "/compute_kernel_writer" + CKW_INCLUDE_DIR = CKW_PROJECT_DIR + "/prototype/include" + CKW_BUILD_DIR = build_path.replace("#", "") + + CKW_CMAKE_CMD = "CC={CKW_CC} CXX={CKW_CXX} cmake -G \"Unix Makefiles\" " \ + "-S {CKW_PROJECT_DIR} -B {CKW_BUILD_DIR} " \ + "-DCMAKE_BUILD_TYPE={CKW_BUILD_TYPE} " \ + "-DCKW_ENABLE_OPENCL={CKW_ENABLE_OPENCL} " \ + "-DCKW_ENABLE_ASSERTS={CKW_ENABLE_ASSERTS} " \ + "-DCKW_BUILD_PROTOTYPE=ON " \ + "-DCKW_CCACHE={CKW_CCACHE} ".format(CKW_CC=CKW_CC, + CKW_CXX=CKW_CXX, + CKW_PROJECT_DIR=CKW_PROJECT_DIR, + CKW_BUILD_DIR=CKW_BUILD_DIR, + CKW_BUILD_TYPE=CKW_BUILD_TYPE, + CKW_ENABLE_OPENCL=CKW_ENABLE_OPENCL, + CKW_ENABLE_ASSERTS=CKW_ENABLE_ASSERTS, + CKW_CCACHE=CKW_CCACHE + ) + + # Configure CKW static objects with -fPIC (CMAKE_POSITION_INDEPENDENT_CODE) option to enable linking statically to ACL + CKW_CMAKE_CONFIGURE_STATIC = CKW_CMAKE_CMD + "-DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON" + CKW_CMAKE_BUILD = "cmake --build {CKW_BUILD_DIR} --target ckw_prototype -j{NUM_JOBS}".format(CKW_BUILD_DIR=CKW_BUILD_DIR, + NUM_JOBS=GetOption('num_jobs') + ) + + # Build Compute Kernel Writer Static Library + subprocess.check_call(CKW_CMAKE_CONFIGURE_STATIC, stderr=subprocess.STDOUT, shell=True) + subprocess.check_call(CKW_CMAKE_BUILD, stderr=subprocess.STDOUT, shell=True) + + # Let ACL know where to find CKW headers + env.Append(CPPPATH = CKW_INCLUDE_DIR) + if not GetOption("help"): try: if env['os'] == 'windows': - compiler_ver = check_output("clang++ -dumpversion").decode().strip() + compiler_ver = subprocess.check_output("clang++ -dumpversion").decode().strip() else: - compiler_ver = check_output(env['CXX'].split() + ["-dumpversion"]).decode().strip() + compiler_ver = subprocess.check_output(env['CXX'].split() + ["-dumpversion"]).decode().strip() except OSError: print("ERROR: Compiler '%s' not found" % env['CXX']) Exit(1) @@ -564,6 +614,10 @@ if env['asserts']: if env['logging']: env.Append(CPPDEFINES = ['ARM_COMPUTE_LOGGING_ENABLED']) +if env['thread_sanitizer']: + env.Append(CXXFLAGS = ['-fsanitize=thread']) + env.Append(LINKFLAGS = ['-fsanitize=thread']) + env.Append(CPPPATH = ['#/include', "#"]) env.Append(CXXFLAGS = env['extra_cxx_flags']) env.Append(LINKFLAGS = env['extra_link_flags']) diff --git a/arm_compute/BUILD.bazel b/arm_compute/BUILD.bazel index 641a327d515431d4ab78797781d79424e943f522..d1219015b7599a7d4f28a2c9d32de00b876be2f8 100644 --- a/arm_compute/BUILD.bazel +++ b/arm_compute/BUILD.bazel @@ -28,6 +28,7 @@ cc_library( "*.h", "*.hpp", "dynamic_fusion/**/*.h", + "function_info/**/*.h", ]), visibility = ["//visibility:public"], ) diff --git a/arm_compute/core/CoreTypes.h b/arm_compute/core/CoreTypes.h new file mode 100644 index 0000000000000000000000000000000000000000..4a48a366518b99cc82893a102ee62360a8f3da10 --- /dev/null +++ b/arm_compute/core/CoreTypes.h @@ -0,0 +1,346 @@ +/* + * Copyright (c) 2016-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_ARM_COMPUTE_CORE_CORETYPES +#define ACL_ARM_COMPUTE_CORE_CORETYPES + +#include "arm_compute/core/Strides.h" +#include "support/Half.h" + +/** CoreTypes.h groups together essential small types that are used across functions */ + +namespace arm_compute +{ +/** 16-bit floating point type */ +using half = half_float::half; +/** Permutation vector */ +using PermutationVector = Strides; + +/** Available channels */ +enum class Channel +{ + UNKNOWN, /** Unknown channel format */ + C0, /**< First channel (used by formats with unknown channel types). */ + C1, /**< Second channel (used by formats with unknown channel types). */ + C2, /**< Third channel (used by formats with unknown channel types). */ + C3, /**< Fourth channel (used by formats with unknown channel types). */ + R, /**< Red channel. */ + G, /**< Green channel. */ + B, /**< Blue channel. */ + A, /**< Alpha channel. */ + Y, /**< Luma channel. */ + U, /**< Cb/U channel. */ + V /**< Cr/V/Value channel. */ +}; + +/** Image colour formats */ +enum class Format +{ + UNKNOWN, /**< Unknown image format */ + U8, /**< 1 channel, 1 U8 per channel */ + S16, /**< 1 channel, 1 S16 per channel */ + U16, /**< 1 channel, 1 U16 per channel */ + S32, /**< 1 channel, 1 S32 per channel */ + U32, /**< 1 channel, 1 U32 per channel */ + S64, /**< 1 channel, 1 S64 per channel */ + U64, /**< 1 channel, 1 U64 per channel */ + BFLOAT16, /**< 16-bit brain floating-point number */ + F16, /**< 1 channel, 1 F16 per channel */ + F32, /**< 1 channel, 1 F32 per channel */ + UV88, /**< 2 channel, 1 U8 per channel */ + RGB888, /**< 3 channels, 1 U8 per channel */ + RGBA8888, /**< 4 channels, 1 U8 per channel */ + YUV444, /**< A 3 plane of 8 bit 4:4:4 sampled Y, U, V planes */ + YUYV422, /**< A single plane of 32-bit macro pixel of Y0, U0, Y1, V0 bytes */ + NV12, /**< A 2 plane YUV format of Luma (Y) and interleaved UV data at 4:2:0 sampling */ + NV21, /**< A 2 plane YUV format of Luma (Y) and interleaved VU data at 4:2:0 sampling */ + IYUV, /**< A 3 plane of 8-bit 4:2:0 sampled Y, U, V planes */ + UYVY422 /**< A single plane of 32-bit macro pixel of U0, Y0, V0, Y1 byte */ +}; + +/** Available data types */ +enum class DataType +{ + UNKNOWN, /**< Unknown data type */ + U8, /**< unsigned 8-bit number */ + S8, /**< signed 8-bit number */ + QSYMM8, /**< quantized, symmetric fixed-point 8-bit number */ + QASYMM8, /**< quantized, asymmetric fixed-point 8-bit number unsigned */ + QASYMM8_SIGNED, /**< quantized, asymmetric fixed-point 8-bit number signed */ + QSYMM8_PER_CHANNEL, /**< quantized, symmetric per channel fixed-point 8-bit number */ + U16, /**< unsigned 16-bit number */ + S16, /**< signed 16-bit number */ + QSYMM16, /**< quantized, symmetric fixed-point 16-bit number */ + QASYMM16, /**< quantized, asymmetric fixed-point 16-bit number */ + U32, /**< unsigned 32-bit number */ + S32, /**< signed 32-bit number */ + U64, /**< unsigned 64-bit number */ + S64, /**< signed 64-bit number */ + BFLOAT16, /**< 16-bit brain floating-point number */ + F16, /**< 16-bit floating-point number */ + F32, /**< 32-bit floating-point number */ + F64, /**< 64-bit floating-point number */ + SIZET /**< size_t */ +}; + +/** [DataLayout enum definition] **/ + +/** Supported tensor data layouts */ +enum class DataLayout +{ + UNKNOWN, /**< Unknown data layout */ + NCHW, /**< Num samples, channels, height, width */ + NHWC, /**< Num samples, height, width, channels */ + NCDHW, /**< Num samples, channels, depth, height, width */ + NDHWC /**< Num samples, depth, height, width, channels */ +}; +/** [DataLayout enum definition] **/ + +/** Supported tensor data layout dimensions */ +enum class DataLayoutDimension +{ + CHANNEL, /**< channel */ + HEIGHT, /**< height */ + WIDTH, /**< width */ + DEPTH, /**< depth */ + BATCHES /**< batches */ +}; + +/** Dimension rounding type when down-scaling on CNNs + * @note Used in pooling and convolution layer + */ +enum class DimensionRoundingType +{ + FLOOR, /**< Floor rounding */ + CEIL /**< Ceil rounding */ +}; + +class PadStrideInfo +{ +public: + /** Constructor + * + * @param[in] stride_x (Optional) Stride, in elements, across x. Defaults to 1. + * @param[in] stride_y (Optional) Stride, in elements, across y. Defaults to 1. + * @param[in] pad_x (Optional) Padding, in elements, across x. Defaults to 0. + * @param[in] pad_y (Optional) Padding, in elements, across y. Defaults to 0. + * @param[in] round (Optional) Dimensions rounding. Defaults to @ref DimensionRoundingType::FLOOR. + */ + PadStrideInfo(unsigned int stride_x = 1, unsigned int stride_y = 1, + unsigned int pad_x = 0, unsigned int pad_y = 0, + DimensionRoundingType round = DimensionRoundingType::FLOOR) + : _stride(std::make_pair(stride_x, stride_y)), + _pad_left(pad_x), + _pad_top(pad_y), + _pad_right(pad_x), + _pad_bottom(pad_y), + _round_type(round) + { + } + /** Constructor + * + * @param[in] stride_x Stride, in elements, across x. + * @param[in] stride_y Stride, in elements, across y. + * @param[in] pad_left Padding across x on the left, in elements. + * @param[in] pad_right Padding across x on the right, in elements. + * @param[in] pad_top Padding across y on the top, in elements. + * @param[in] pad_bottom Padding across y on the bottom, in elements. + * @param[in] round Dimensions rounding. + */ + PadStrideInfo(unsigned int stride_x, unsigned int stride_y, + unsigned int pad_left, unsigned int pad_right, + unsigned int pad_top, unsigned int pad_bottom, + DimensionRoundingType round) + : _stride(std::make_pair(stride_x, stride_y)), + _pad_left(pad_left), + _pad_top(pad_top), + _pad_right(pad_right), + _pad_bottom(pad_bottom), + _round_type(round) + { + } + /** Get the stride. + * + * @return a pair: stride x, stride y. + */ + std::pair stride() const + { + return _stride; + } + /** Check whether the padding is symmetric. + * + * @return True if the padding is symmetric. + */ + bool padding_is_symmetric() const + { + return (_pad_left == _pad_right) && (_pad_top == _pad_bottom); + } + /** Get the padding. + * + * @note This should only be used when the padding is symmetric. + * + * @return a pair: padding left/right, padding top/bottom + */ + std::pair pad() const + { + //this accessor should be used only when padding is symmetric + ARM_COMPUTE_ERROR_ON(!padding_is_symmetric()); + return std::make_pair(_pad_left, _pad_top); + } + + /** Get the left padding */ + unsigned int pad_left() const + { + return _pad_left; + } + /** Get the right padding */ + unsigned int pad_right() const + { + return _pad_right; + } + /** Get the top padding */ + unsigned int pad_top() const + { + return _pad_top; + } + /** Get the bottom padding */ + unsigned int pad_bottom() const + { + return _pad_bottom; + } + + /** Get the rounding type */ + DimensionRoundingType round() const + { + return _round_type; + } + + /** Check whether this has any padding */ + bool has_padding() const + { + return (_pad_left != 0 || _pad_top != 0 || _pad_right != 0 || _pad_bottom != 0); + } + +private: + std::pair _stride; + unsigned int _pad_left; + unsigned int _pad_top; + unsigned int _pad_right; + unsigned int _pad_bottom; + + DimensionRoundingType _round_type; +}; + +/** Memory layouts for the weights tensor. + * + * * UNSPECIFIED is used to select kernels that do not run in + * variable weights mode. + * + * * ANY is used to query the kernel database to retrieve any of the + * kernels that runs in variable weights mode. Once a kernel is + * found, the specific format expected by the kernel can be + * retrieved by the user for reordering the weights tensor + * accordingly. + * + * The other values OHWIo{interleave_by}i{block_by} describe the + * memory layout of a 4D tensor with layout OHWI that has been + * transformed into a 4D tensor with dimensions O'HWI' where: + * + * O' = first multiple of {interleave_by} s.t. O<=O' + * I' = first multiple of {block_by} s.t. I<=I' + * + * The total size of the dst tensor is O' x H x W x I' + * + * The access function of the tensor with layout + * OHWIo{interleave_by}i{block_by} and size O'HWI' is a 6-parameter + * access function, where the 6 parameters are computed as follows: + * + * x5 = floor(o/{interleave_by}) RANGE [0, O'/{interleave_by} -1] SIZE: O'/{interleave_by} + * + * x4 = h RANGE [0, H-1] SIZE: H + * x3 = w RANGE [0, W-1] SIZE: W + * x2 = floor(i/{block_by}) RANGE [0, I'/{block_by} -1] SIZE: I'/{block_by} + * x1 = o%{interleave_by} RANGE [0, {interleave_by} -1] SIZE: {interleave_by} + * x0 = i%{block_by} RANGE [0, {block_by} -1] SIZE: {block_by} + * TOTAL SIZE: O' * H * W * I' + * + * 4D 6D + * ----------------- ----------------------------------- + * value(o, h, w, i) = x5 * H * W * I' * {interleave_by} + * + x4 * W * I' * {interleave_by} + * + x3 * I' * {interleave_by} + * + x2 * {interleave_by} * {block_by} + * + x1 * {block_by} + * + x0 + * + * Notice that in arm_gemm the 4D tensor of dimension O'HWI' created + * for the OHWIo{interleave_by}i{block_by} format is in reality seen + * as a 2D tensor, where the number of rows is O'/{interleave_by} + * and the number of columns is {interleave_by} * H * W * I'. + * + * The postfix *_bf16 is for the memory layout needed for the + * fast-mode kernels, in which the weights are passed in bfloat16 + * format. + */ +enum class WeightFormat +{ + UNSPECIFIED = 0x1, + ANY = 0x2, + OHWI = 0x100100, + OHWIo2 = 0x100200, + OHWIo4 = 0x100400, + OHWIo8 = 0x100800, + OHWIo16 = 0x101000, + OHWIo32 = 0x102000, + OHWIo64 = 0x104000, + OHWIo128 = 0x108000, + OHWIo4i2 = 0x200400, + OHWIo4i2_bf16 = 0x200410, + OHWIo8i2 = 0x200800, + OHWIo8i2_bf16 = 0x200810, + OHWIo16i2 = 0x201000, + OHWIo16i2_bf16 = 0x201010, + OHWIo32i2 = 0x202000, + OHWIo32i2_bf16 = 0x202010, + OHWIo64i2 = 0x204000, + OHWIo64i2_bf16 = 0x204010, + OHWIo4i4 = 0x400400, + OHWIo4i4_bf16 = 0x400410, + OHWIo8i4 = 0x400800, + OHWIo8i4_bf16 = 0x400810, + OHWIo16i4 = 0x401000, + OHWIo16i4_bf16 = 0x401010, + OHWIo32i4 = 0x402000, + OHWIo32i4_bf16 = 0x402010, + OHWIo64i4 = 0x404000, + OHWIo64i4_bf16 = 0x404010, + OHWIo2i8 = 0x800200, + OHWIo4i8 = 0x800400, + OHWIo8i8 = 0x800800, + OHWIo16i8 = 0x801000, + OHWIo32i8 = 0x802000, + OHWIo64i8 = 0x804000 +}; + +} // namespace arm_compute +#endif /* ACL_ARM_COMPUTE_CORE_CORETYPES */ diff --git a/arm_compute/core/Error.h b/arm_compute/core/Error.h index c9a0d85f0c24aa2f6582ccfa74edfc1386cdd7e1..0854f2c5274eddb3a5ac5052c4732548719d6d95 100644 --- a/arm_compute/core/Error.h +++ b/arm_compute/core/Error.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2019, 2021 Arm Limited. + * Copyright (c) 2016-2019, 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -204,9 +204,10 @@ Status create_error_msg(ErrorCode error_code, const char *func, const char *file #define ARM_COMPUTE_RETURN_ON_ERROR(status) \ do \ { \ - if(!bool(status)) \ + const auto s = status; \ + if(!bool(s)) \ { \ - return status; \ + return s; \ } \ } while(false) diff --git a/arm_compute/core/ITensorInfo.h b/arm_compute/core/ITensorInfo.h index 7b0fd1c2b70a539a724f5a362bb6b8967f1af3a3..e7c0b182c69e5f192dafb2d33f8498a815f33d38 100644 --- a/arm_compute/core/ITensorInfo.h +++ b/arm_compute/core/ITensorInfo.h @@ -28,7 +28,6 @@ #include "arm_compute/core/Strides.h" #include "arm_compute/core/TensorShape.h" #include "arm_compute/core/Types.h" -#include "arm_compute/core/Utils.h" #include "arm_compute/core/utils/misc/Utility.h" #include "support/ICloneable.h" @@ -36,6 +35,7 @@ namespace arm_compute { +class QuantizationInfo; // Note: Any changes to the fields of the class below that have setters should be mirrored // (if possible) in the auto_init_if_empty function in AutoConfiguration.h diff --git a/arm_compute/core/KernelDescriptors.h b/arm_compute/core/KernelDescriptors.h index f637351e265f0aec559d90a588649ba9578ff1b0..305766e8251721935efcd581388951674921ffc7 100644 --- a/arm_compute/core/KernelDescriptors.h +++ b/arm_compute/core/KernelDescriptors.h @@ -27,6 +27,7 @@ #include "arm_compute/core/PixelValue.h" #include "arm_compute/core/Types.h" #include "arm_compute/core/experimental/IPostOp.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" namespace arm_compute { diff --git a/arm_compute/core/PixelValue.h b/arm_compute/core/PixelValue.h index 0e3d26c515b0f05a4e9ee7c778aca30b3a3a6fd9..790f58a7935ab3f210a1038695506c177e5dbd6f 100644 --- a/arm_compute/core/PixelValue.h +++ b/arm_compute/core/PixelValue.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021 Arm Limited. + * Copyright (c) 2016-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #define ARM_COMPUTE_PIXELVALUE_H #include "arm_compute/core/Types.h" +#include "arm_compute/core/QuantizationInfo.h" #include diff --git a/arm_compute/core/QuantizationInfo.h b/arm_compute/core/QuantizationInfo.h index ddf1342e3227ced70c10a862459edb92987d7bc0..8fa513eee1e61e442eae70589bcbe6166da27fd3 100644 --- a/arm_compute/core/QuantizationInfo.h +++ b/arm_compute/core/QuantizationInfo.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022 Arm Limited. + * Copyright (c) 2019-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,13 +24,10 @@ #ifndef ARM_COMPUTE_QUANTIZATION_INFO_H #define ARM_COMPUTE_QUANTIZATION_INFO_H -#include "arm_compute/core/Error.h" #include "arm_compute/core/Rounding.h" +#include "arm_compute/core/utils/misc/Utility.h" #include "support/ToolchainSupport.h" -#include "utils/misc/Utility.h" -#include -#include #include namespace arm_compute @@ -399,57 +396,6 @@ inline float dequantize_qsymm8(int8_t value, const UniformQuantizationInfo &qinf return value * qinfo.scale; } -inline qasymm8_t qasymm8_hard_swish(qasymm8_t in, - const UniformQuantizationInfo &qi_in, - const UniformQuantizationInfo &qi_out) -{ - float tmp_f = dequantize_qasymm8(in, qi_in); - tmp_f = tmp_f * ((std::min(std::max((tmp_f + 3), 0.0f), 6.0f)) * 0.166666667f); - const qasymm8_t tmp = quantize_qasymm8(tmp_f, qi_out); - return tmp; -} - -inline qasymm8_signed_t qasymm8_signed_hard_swish(qasymm8_signed_t in, - const UniformQuantizationInfo &qi_in, - const UniformQuantizationInfo &qi_out) -{ - float tmp_f = dequantize_qasymm8_signed(in, qi_in); - tmp_f = tmp_f * ((std::min(std::max((tmp_f + 3), 0.0f), 6.0f)) * 0.166666667f); - const qasymm8_t tmp = quantize_qasymm8_signed(tmp_f, qi_out); - return tmp; -} - -inline qasymm8_t qasymm8_leaky_relu(qasymm8_t in, - const UniformQuantizationInfo &qi_in, - const UniformQuantizationInfo &qi_out, - float alpha) -{ - float tmp_f = dequantize_qasymm8(in, qi_in); - tmp_f = tmp_f > 0 ? tmp_f : tmp_f * alpha; - const qasymm8_t tmp = quantize_qasymm8(tmp_f, qi_out); - return tmp; -} - -inline qasymm8_t qasymm8_logistic(qasymm8_t in, - const UniformQuantizationInfo &qi_in, - const UniformQuantizationInfo &qi_out) -{ - float tmp_f = dequantize_qasymm8(in, qi_in); - tmp_f = 1.f / (1.f + std::exp(-tmp_f)); - const qasymm8_t tmp = quantize_qasymm8(tmp_f, qi_out); - return tmp; -} - -inline qasymm8_signed_t qasymm8_signed_logistic(qasymm8_signed_t in, - const UniformQuantizationInfo &qi_in, - const UniformQuantizationInfo &qi_out) -{ - float tmp_f = dequantize_qasymm8_signed(in, qi_in); - tmp_f = 1.f / (1.f + std::exp(-tmp_f)); - const qasymm8_signed_t tmp = quantize_qasymm8_signed(tmp_f, qi_out); - return tmp; -} - /** Dequantize a value given a 8-bit symmetric quantization scheme * * @param[in] value Value to dequantize diff --git a/arm_compute/core/TensorInfo.h b/arm_compute/core/TensorInfo.h index 8436407a7521f975ed681fcf9d62fda2eb705178..e738a797b298ff3d468a8e55f74fa627350e5ccf 100644 --- a/arm_compute/core/TensorInfo.h +++ b/arm_compute/core/TensorInfo.h @@ -32,7 +32,6 @@ #include "arm_compute/core/Strides.h" #include "arm_compute/core/TensorShape.h" #include "arm_compute/core/Types.h" -#include "arm_compute/core/Utils.h" #include #include diff --git a/arm_compute/core/TensorShape.h b/arm_compute/core/TensorShape.h index b6ab9dc75a7392d1efa3ec1ed16fae1971456d86..4c9186ac64cfb5c847692781de6ab0d7f2faed6a 100644 --- a/arm_compute/core/TensorShape.h +++ b/arm_compute/core/TensorShape.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021 Arm Limited. + * Copyright (c) 2016-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -106,9 +106,10 @@ public: * * @note The upper dimensions of the tensor shape will be shifted down by 1 * - * @param[in] n Dimension to remove + * @param[in] n Dimension to remove + * @param[in] apply_dim_correction (Optional) Flag to state whether apply dimension correction (removing trailing dimensions with size of 1) after removing a dimension. */ - void remove_dimension(size_t n) + void remove_dimension(size_t n, bool apply_dim_correction = true) { ARM_COMPUTE_ERROR_ON(_num_dimensions < 1); ARM_COMPUTE_ERROR_ON(n >= _num_dimensions); @@ -122,7 +123,10 @@ public: std::fill(_id.begin() + _num_dimensions, _id.end(), 1); // Correct number dimensions to ignore trailing dimensions of size 1 - apply_dimension_correction(); + if(apply_dim_correction) + { + apply_dimension_correction(); + } } /** Collapse the first n dimensions. diff --git a/arm_compute/core/Types.h b/arm_compute/core/Types.h index 8a1d189e8ea6d28040ed4084ba03b589de864012..12d860205ead9c91301fc4af498d248e75e0556b 100644 --- a/arm_compute/core/Types.h +++ b/arm_compute/core/Types.h @@ -21,19 +21,53 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifndef ARM_COMPUTE_TYPES_H -#define ARM_COMPUTE_TYPES_H +#ifndef ACL_ARM_COMPUTE_CORE_TYPES +#define ACL_ARM_COMPUTE_CORE_TYPES + +/** The following symbols have been moved to: + * half + * PermutationVector + * Format + * DataType + * DataLayout + * DataLayoutDimension + * PadStrideInfo + * WeightFormat + * Channel + * DimensionRoundingType + */ +#include "arm_compute/core/CoreTypes.h" +/** The following symbols have been moved to: + * ActivationFunction + * ActivationLayerInfo + */ +#include "arm_compute/function_info/ActivationLayerInfo.h" +/** The following symbols have been moved to: + * ConvolutionInfo + */ +#include "arm_compute/function_info/ConvolutionInfo.h" +/** The following symbols have been moved to: + * FullyConnectedLayerInfo + */ +#include "arm_compute/function_info/FullyConnectedLayerInfo.h" +/** The following symbols have been moved to: + * GEMMLowpOutputStageType + * GEMMLowpOutputStageInfo + * GEMMInfo + */ +#include "arm_compute/function_info/GEMMInfo.h" +/** The following symbols have been moved to: + * MatMulInfo + */ +#include "arm_compute/function_info/MatMulInfo.h" #include "arm_compute/core/Coordinates.h" -#include "arm_compute/core/QuantizationInfo.h" #include "arm_compute/core/Size2D.h" #include "arm_compute/core/Size3D.h" -#include "arm_compute/core/Strides.h" #include "arm_compute/core/TensorShape.h" #include "arm_compute/core/experimental/IPostOp.h" #include "arm_compute/core/utils/misc/Macros.h" #include "support/Bfloat16.h" -#include "support/Half.h" #include #include @@ -44,62 +78,9 @@ namespace arm_compute { -/** 16-bit floating point type */ -using half = half_float::half; - -/** Permutation vector */ -using PermutationVector = Strides; /** Bidirectional strides */ using BiStrides = Coordinates; -/** Image colour formats */ -enum class Format -{ - UNKNOWN, /**< Unknown image format */ - U8, /**< 1 channel, 1 U8 per channel */ - S16, /**< 1 channel, 1 S16 per channel */ - U16, /**< 1 channel, 1 U16 per channel */ - S32, /**< 1 channel, 1 S32 per channel */ - U32, /**< 1 channel, 1 U32 per channel */ - BFLOAT16, /**< 16-bit brain floating-point number */ - F16, /**< 1 channel, 1 F16 per channel */ - F32, /**< 1 channel, 1 F32 per channel */ - UV88, /**< 2 channel, 1 U8 per channel */ - RGB888, /**< 3 channels, 1 U8 per channel */ - RGBA8888, /**< 4 channels, 1 U8 per channel */ - YUV444, /**< A 3 plane of 8 bit 4:4:4 sampled Y, U, V planes */ - YUYV422, /**< A single plane of 32-bit macro pixel of Y0, U0, Y1, V0 bytes */ - NV12, /**< A 2 plane YUV format of Luma (Y) and interleaved UV data at 4:2:0 sampling */ - NV21, /**< A 2 plane YUV format of Luma (Y) and interleaved VU data at 4:2:0 sampling */ - IYUV, /**< A 3 plane of 8-bit 4:2:0 sampled Y, U, V planes */ - UYVY422 /**< A single plane of 32-bit macro pixel of U0, Y0, V0, Y1 byte */ -}; - -/** Available data types */ -enum class DataType -{ - UNKNOWN, /**< Unknown data type */ - U8, /**< unsigned 8-bit number */ - S8, /**< signed 8-bit number */ - QSYMM8, /**< quantized, symmetric fixed-point 8-bit number */ - QASYMM8, /**< quantized, asymmetric fixed-point 8-bit number unsigned */ - QASYMM8_SIGNED, /**< quantized, asymmetric fixed-point 8-bit number signed */ - QSYMM8_PER_CHANNEL, /**< quantized, symmetric per channel fixed-point 8-bit number */ - U16, /**< unsigned 16-bit number */ - S16, /**< signed 16-bit number */ - QSYMM16, /**< quantized, symmetric fixed-point 16-bit number */ - QASYMM16, /**< quantized, asymmetric fixed-point 16-bit number */ - U32, /**< unsigned 32-bit number */ - S32, /**< signed 32-bit number */ - U64, /**< unsigned 64-bit number */ - S64, /**< signed 64-bit number */ - BFLOAT16, /**< 16-bit brain floating-point number */ - F16, /**< 16-bit floating-point number */ - F32, /**< 32-bit floating-point number */ - F64, /**< 64-bit floating-point number */ - SIZET /**< size_t */ -}; - /** Available Sampling Policies */ enum class SamplingPolicy { @@ -107,29 +88,6 @@ enum class SamplingPolicy TOP_LEFT /**< Samples are taken at pixel top left corner */ }; -/** [DataLayout enum definition] **/ - -/** Supported tensor data layouts */ -enum class DataLayout -{ - UNKNOWN, /**< Unknown data layout */ - NCHW, /**< Num samples, channels, height, width */ - NHWC, /**< Num samples, height, width, channels */ - NCDHW, /**< Num samples, channels, depth, height, width */ - NDHWC /**< Num samples, depth, height, width, channels */ -}; -/** [DataLayout enum definition] **/ - -/** Supported tensor data layout dimensions */ -enum class DataLayoutDimension -{ - CHANNEL, /**< channel */ - HEIGHT, /**< height */ - WIDTH, /**< width */ - DEPTH, /**< depth */ - BATCHES /**< batches */ -}; - /** Available ConvolutionMethod*/ enum class ConvolutionMethod { @@ -457,23 +415,6 @@ using PaddingList = std::vector; /** Information to produce a tiled version of a Tensor */ using Multiples = std::vector; -/** Available channels */ -enum class Channel -{ - UNKNOWN, /** Unknown channel format */ - C0, /**< First channel (used by formats with unknown channel types). */ - C1, /**< Second channel (used by formats with unknown channel types). */ - C2, /**< Third channel (used by formats with unknown channel types). */ - C3, /**< Fourth channel (used by formats with unknown channel types). */ - R, /**< Red channel. */ - G, /**< Green channel. */ - B, /**< Blue channel. */ - A, /**< Alpha channel. */ - Y, /**< Luma channel. */ - U, /**< Cb/U channel. */ - V /**< Cr/V/Value channel. */ -}; - /** Available reduction operations */ enum class ReductionOperation { @@ -546,15 +487,6 @@ struct DetectionWindow float score{ 0.f }; /**< Confidence value for the detection window */ }; -/** Dimension rounding type when down-scaling on CNNs - * @note Used in pooling and convolution layer - */ -enum class DimensionRoundingType -{ - FLOOR, /**< Floor rounding */ - CEIL /**< Ceil rounding */ -}; - /** Available pooling types */ enum class PoolingType { @@ -668,122 +600,6 @@ private: }; /** Padding and stride information class */ -class PadStrideInfo -{ -public: - /** Constructor - * - * @param[in] stride_x (Optional) Stride, in elements, across x. Defaults to 1. - * @param[in] stride_y (Optional) Stride, in elements, across y. Defaults to 1. - * @param[in] pad_x (Optional) Padding, in elements, across x. Defaults to 0. - * @param[in] pad_y (Optional) Padding, in elements, across y. Defaults to 0. - * @param[in] round (Optional) Dimensions rounding. Defaults to @ref DimensionRoundingType::FLOOR. - */ - PadStrideInfo(unsigned int stride_x = 1, unsigned int stride_y = 1, - unsigned int pad_x = 0, unsigned int pad_y = 0, - DimensionRoundingType round = DimensionRoundingType::FLOOR) - : _stride(std::make_pair(stride_x, stride_y)), - _pad_left(pad_x), - _pad_top(pad_y), - _pad_right(pad_x), - _pad_bottom(pad_y), - _round_type(round) - { - } - /** Constructor - * - * @param[in] stride_x Stride, in elements, across x. - * @param[in] stride_y Stride, in elements, across y. - * @param[in] pad_left Padding across x on the left, in elements. - * @param[in] pad_right Padding across x on the right, in elements. - * @param[in] pad_top Padding across y on the top, in elements. - * @param[in] pad_bottom Padding across y on the bottom, in elements. - * @param[in] round Dimensions rounding. - */ - PadStrideInfo(unsigned int stride_x, unsigned int stride_y, - unsigned int pad_left, unsigned int pad_right, - unsigned int pad_top, unsigned int pad_bottom, - DimensionRoundingType round) - : _stride(std::make_pair(stride_x, stride_y)), - _pad_left(pad_left), - _pad_top(pad_top), - _pad_right(pad_right), - _pad_bottom(pad_bottom), - _round_type(round) - { - } - /** Get the stride. - * - * @return a pair: stride x, stride y. - */ - std::pair stride() const - { - return _stride; - } - /** Check whether the padding is symmetric. - * - * @return True if the padding is symmetric. - */ - bool padding_is_symmetric() const - { - return (_pad_left == _pad_right) && (_pad_top == _pad_bottom); - } - /** Get the padding. - * - * @note This should only be used when the padding is symmetric. - * - * @return a pair: padding left/right, padding top/bottom - */ - std::pair pad() const - { - //this accessor should be used only when padding is symmetric - ARM_COMPUTE_ERROR_ON(!padding_is_symmetric()); - return std::make_pair(_pad_left, _pad_top); - } - - /** Get the left padding */ - unsigned int pad_left() const - { - return _pad_left; - } - /** Get the right padding */ - unsigned int pad_right() const - { - return _pad_right; - } - /** Get the top padding */ - unsigned int pad_top() const - { - return _pad_top; - } - /** Get the bottom padding */ - unsigned int pad_bottom() const - { - return _pad_bottom; - } - - /** Get the rounding type */ - DimensionRoundingType round() const - { - return _round_type; - } - - /** Check whether this has any padding */ - bool has_padding() const - { - return (_pad_left != 0 || _pad_top != 0 || _pad_right != 0 || _pad_bottom != 0); - } - -private: - std::pair _stride; - unsigned int _pad_left; - unsigned int _pad_top; - unsigned int _pad_right; - unsigned int _pad_bottom; - - DimensionRoundingType _round_type; -}; - /** Padding information for 2D operations like Conv2d */ struct Padding2D { @@ -1655,214 +1471,6 @@ private: float _bbox_xform_clip; }; -/** Activation Layer Information class */ -class ActivationLayerInfo -{ -public: - /** Available activation functions */ - enum class ActivationFunction - { - LOGISTIC, /**< Logistic ( \f$ f(x) = \frac{1}{1 + e^{-x}} \f$ ) */ - TANH, /**< Hyperbolic tangent ( \f$ f(x) = a \cdot tanh(b \cdot x) \f$ ) */ - RELU, /**< Rectifier ( \f$ f(x) = max(0,x) \f$ ) */ - BOUNDED_RELU, /**< Upper Bounded Rectifier ( \f$ f(x) = min(a, max(0,x)) \f$ ) */ - LU_BOUNDED_RELU, /**< Lower and Upper Bounded Rectifier ( \f$ f(x) = min(a, max(b,x)) \f$ ) */ - LEAKY_RELU, /**< Leaky Rectifier ( \f$ f(x) = \begin{cases} \alpha x & \quad \text{if } x \text{ < 0}\\ x & \quad \text{if } x \geq \text{ 0 } \end{cases} \f$ ) */ - SOFT_RELU, /**< Soft Rectifier ( \f$ f(x)= log(1+e^x) \f$ ) */ - ELU, /**< Exponential Linear Unit ( \f$ f(x) = \begin{cases} \alpha (exp(x) - 1) & \quad \text{if } x \text{ < 0}\\ x & \quad \text{if } x \geq \text{ 0 } \end{cases} \f$ ) */ - ABS, /**< Absolute ( \f$ f(x)= |x| \f$ ) */ - SQUARE, /**< Square ( \f$ f(x)= x^2 \f$ )*/ - SQRT, /**< Square root ( \f$ f(x) = \sqrt{x} \f$ )*/ - LINEAR, /**< Linear ( \f$ f(x)= ax + b \f$ ) */ - IDENTITY, /**< Identity ( \f$ f(x)= x \f$ ) */ - HARD_SWISH, /**< Hard-swish ( \f$ f(x) = (x \text{ReLU6}(x+3))/6 = x \min(\max(0,x+3),6)/6 \f$ ) */ - SWISH, /**< Swish ( \f$ f(x) = \frac{x}{1 + e^{-ax}} = x \text{logistic}(ax) \f$ ) */ - GELU /**< GELU ( \f$ f(x) = x * 1/2 * 1 + erf(x / \sqrt{2}) \f$ ) */ - }; - - /** Lookup table */ - using LookupTable256 = std::array; - - ActivationLayerInfo() = default; - /** Default Constructor - * - * @param[in] f The activation function to use. - * @param[in] a (Optional) The alpha parameter used by some activation functions - * (@ref ActivationFunction::BOUNDED_RELU, @ref ActivationFunction::LU_BOUNDED_RELU, @ref ActivationFunction::LINEAR, @ref ActivationFunction::TANH). - * @param[in] b (Optional) The beta parameter used by some activation functions (@ref ActivationFunction::LINEAR, @ref ActivationFunction::LU_BOUNDED_RELU, @ref ActivationFunction::TANH). - */ - ActivationLayerInfo(ActivationFunction f, float a = 0.0f, float b = 0.0f) - : _act(f), _a(a), _b(b), _enabled(true) - { - } - /** Get the type of activation function */ - ActivationFunction activation() const - { - return _act; - } - /** Get the alpha value */ - float a() const - { - return _a; - } - /** Get the beta value */ - float b() const - { - return _b; - } - /** Check if initialised */ - bool enabled() const - { - return _enabled; - } - -#ifdef __aarch64__ - const LookupTable256 &lut() const - { - return _lut; - } - - void init_lut(DataType data_type, const UniformQuantizationInfo &qi_in, const UniformQuantizationInfo &qi_out) - { - if(_act == ActivationFunction::HARD_SWISH) - { - if(data_type == DataType::QASYMM8) - { - qasymm8_hard_swish_populate_table(_lut, qi_in, qi_out); - } - else - { - qasymm8_signed_hard_swish_populate_table(_lut, qi_in, qi_out); - } - } - else if(_act == ActivationFunction::LEAKY_RELU) - { - qasymm8_leaky_relu_populate_table(_lut, qi_in, qi_out, _a); - } - else if(_act == ActivationFunction::LOGISTIC) - { - if(data_type == DataType::QASYMM8) - { - qasymm8_logistic_populate_table(_lut, qi_in, qi_out); - } - else - { - qasymm8_signed_logistic_populate_table(_lut, qi_in, qi_out); - } - } - } -#endif // __aarch64__ - - static inline bool is_lut_supported(ActivationFunction act_func, DataType data_type) - { -#ifdef __aarch64__ - switch(act_func) - { - case ActivationFunction::HARD_SWISH: - return data_type == DataType::QASYMM8 || data_type == DataType::QASYMM8_SIGNED; - case ActivationFunction::LEAKY_RELU: - return data_type == DataType::QASYMM8; - case ActivationFunction::LOGISTIC: - return data_type == DataType::QASYMM8 || data_type == DataType::QASYMM8_SIGNED; - default: - return false; - } -#else // __aarch64__ - ARM_COMPUTE_UNUSED(act_func); - ARM_COMPUTE_UNUSED(data_type); - return false; -#endif // __aarch64__ - } - -private: - ActivationFunction _act = { ActivationLayerInfo::ActivationFunction::IDENTITY }; - float _a = {}; - float _b = {}; - bool _enabled = { false }; - -#ifdef __aarch64__ - LookupTable256 _lut = {}; - - static inline void qasymm8_hard_swish_populate_table(LookupTable256 &lut, const UniformQuantizationInfo &qi_in, const UniformQuantizationInfo &qi_out) - { - for(size_t i = 0; i < lut.size(); ++i) - { - lut[i] = qasymm8_hard_swish(i, qi_in, qi_out); - } - } - - static inline void qasymm8_signed_hard_swish_populate_table(LookupTable256 &lut, const UniformQuantizationInfo &qi_in, const UniformQuantizationInfo &qi_out) - { - for(size_t i = 0; i < lut.size(); ++i) - { - lut[i] = qasymm8_signed_hard_swish(i, qi_in, qi_out); - } - } - - static inline void qasymm8_leaky_relu_populate_table(LookupTable256 &lut, const UniformQuantizationInfo &qi_in, const UniformQuantizationInfo &qi_out, float alpha) - { - for(size_t i = 0; i < lut.size(); ++i) - { - lut[i] = qasymm8_leaky_relu(i, qi_in, qi_out, alpha); - } - } - - static inline void qasymm8_logistic_populate_table(LookupTable256 &lut, const UniformQuantizationInfo &qi_in, const UniformQuantizationInfo &qi_out) - { - for(size_t i = 0; i < lut.size(); ++i) - { - lut[i] = qasymm8_logistic(i, qi_in, qi_out); - } - } - - static inline void qasymm8_signed_logistic_populate_table(LookupTable256 &lut, const UniformQuantizationInfo &qi_in, const UniformQuantizationInfo &qi_out) - { - for(size_t i = 0; i < lut.size(); ++i) - { - lut[i] = qasymm8_signed_logistic(static_cast(i), qi_in, qi_out); - } - } -#endif // __aarch64__ -}; - -/** Fully connected layer info */ -struct FullyConnectedLayerInfo -{ - /* Fused-activation parameters */ - ActivationLayerInfo activation_info{}; /**< Fused activation to apply after the matrix multiplication. */ - /* Information about weights */ - DataLayout weights_trained_layout{ DataLayout::NCHW }; /**< Layout that the weights have been trained with. */ - bool transpose_weights{ true }; /**< Transpose weights if true. */ - bool are_weights_reshaped{ false }; /**< @deprecated Reshape the weights tensor if false. */ - bool retain_internal_weights{ false }; /**< Retain internal reshaped weights. */ - bool enable_fast_math{ false }; /**< Enable fast math computation. */ - /* Other parameters */ - bool fp_mixed_precision{ false }; /**< Use wider accumulators (32 bit instead of 16 for FP16) to improve accuracy. */ - - /** Sets the weights trained data layout - * - * @param[in] layout Data layout that the weights were trained with - * - * @return Updated object - */ - FullyConnectedLayerInfo &set_weights_trained_layout(DataLayout layout) - { - weights_trained_layout = layout; - return *this; - } - /** Sets the transpose weights flag - * - * @param[in] should_transpose_weights Boolean flag indicating if weights should be transposed - * - * @return Updated object - */ - FullyConnectedLayerInfo &set_transpose_weights(bool should_transpose_weights) - { - transpose_weights = should_transpose_weights; - return *this; - } -}; - /** Normalization Layer Information class */ class NormalizationLayerInfo { @@ -1981,96 +1589,6 @@ private: int32_t _shrink_axis_mask; }; -/** Memory layouts for the weights tensor. - * - * * UNSPECIFIED is used to select kernels that do not run in - * variable weights mode. - * - * * ANY is used to query the kernel database to retrieve any of the - * kernels that runs in variable weights mode. Once a kernel is - * found, the specific format expected by the kernel can be - * retrieved by the user for reordering the weights tensor - * accordingly. - * - * The other values OHWIo{interleave_by}i{block_by} describe the - * memory layout of a 4D tensor with layout OHWI that has been - * transformed into a 4D tensor with dimensions O'HWI' where: - * - * O' = first multiple of {interleave_by} s.t. O<=O' - * I' = first multiple of {block_by} s.t. I<=I' - * - * The total size of the dst tensor is O' x H x W x I' - * - * The access function of the tensor with layout - * OHWIo{interleave_by}i{block_by} and size O'HWI' is a 6-parameter - * access function, where the 6 parameters are computed as follows: - * - * x5 = floor(o/{interleave_by}) RANGE [0, O'/{interleave_by} -1] SIZE: O'/{interleave_by} - * - * x4 = h RANGE [0, H-1] SIZE: H - * x3 = w RANGE [0, W-1] SIZE: W - * x2 = floor(i/{block_by}) RANGE [0, I'/{block_by} -1] SIZE: I'/{block_by} - * x1 = o%{interleave_by} RANGE [0, {interleave_by} -1] SIZE: {interleave_by} - * x0 = i%{block_by} RANGE [0, {block_by} -1] SIZE: {block_by} - * TOTAL SIZE: O' * H * W * I' - * - * 4D 6D - * ----------------- ----------------------------------- - * value(o, h, w, i) = x5 * H * W * I' * {interleave_by} - * + x4 * W * I' * {interleave_by} - * + x3 * I' * {interleave_by} - * + x2 * {interleave_by} * {block_by} - * + x1 * {block_by} - * + x0 - * - * Notice that in arm_gemm the 4D tensor of dimension O'HWI' created - * for the OHWIo{interleave_by}i{block_by} format is in reality seen - * as a 2D tensor, where the number of rows is O'/{interleave_by} - * and the number of columns is {interleave_by} * H * W * I'. - * - * The postfix *_bf16 is for the memory layout needed for the - * fast-mode kernels, in which the weights are passed in bfloat16 - * format. - */ -enum class WeightFormat -{ - UNSPECIFIED = 0x1, - ANY = 0x2, - OHWI = 0x100100, - OHWIo2 = 0x100200, - OHWIo4 = 0x100400, - OHWIo8 = 0x100800, - OHWIo16 = 0x101000, - OHWIo32 = 0x102000, - OHWIo64 = 0x104000, - OHWIo128 = 0x108000, - OHWIo4i2 = 0x200400, - OHWIo4i2_bf16 = 0x200410, - OHWIo8i2 = 0x200800, - OHWIo8i2_bf16 = 0x200810, - OHWIo16i2 = 0x201000, - OHWIo16i2_bf16 = 0x201010, - OHWIo32i2 = 0x202000, - OHWIo32i2_bf16 = 0x202010, - OHWIo64i2 = 0x204000, - OHWIo64i2_bf16 = 0x204010, - OHWIo4i4 = 0x400400, - OHWIo4i4_bf16 = 0x400410, - OHWIo8i4 = 0x400800, - OHWIo8i4_bf16 = 0x400810, - OHWIo16i4 = 0x401000, - OHWIo16i4_bf16 = 0x401010, - OHWIo32i4 = 0x402000, - OHWIo32i4_bf16 = 0x402010, - OHWIo64i4 = 0x404000, - OHWIo64i4_bf16 = 0x404010, - OHWIo2i8 = 0x800200, - OHWIo4i8 = 0x800400, - OHWIo8i8 = 0x800800, - OHWIo16i8 = 0x801000, - OHWIo32i8 = 0x802000, - OHWIo64i8 = 0x804000 -}; // OHWIoi inline int interleave_by(const WeightFormat wf) { @@ -2281,44 +1799,6 @@ private: bool _broadcast_bias; }; -struct ConvolutionInfo -{ - ConvolutionInfo() = default; - ConvolutionInfo(const PadStrideInfo &pad_stride_info, unsigned int depth_multiplier, const ActivationLayerInfo &act_info, const Size2D &dilation) - : pad_stride_info(pad_stride_info), depth_multiplier(depth_multiplier), act_info(act_info), dilation(dilation) - { - } - PadStrideInfo pad_stride_info{}; /**< Convolution info (Pads, strides,...) */ - unsigned int depth_multiplier{ 1 }; /**< Multiplier to apply to input's depth to retrieve the output depth. Defaults to 1 */ - ActivationLayerInfo act_info{}; /**< Fused activation to apply after convolution. */ - Size2D dilation{ Size2D(1, 1) }; /**< Dilation, in elements, across x and y. Defaults to (1, 1). */ -}; - -/** GEMMLowp output stage type */ -enum class GEMMLowpOutputStageType -{ - NONE, /**< No quantization */ - QUANTIZE_DOWN, /**< Quantize using an integer multiplication */ - QUANTIZE_DOWN_FIXEDPOINT, /**< Quantize using a fixed point multiplication */ - QUANTIZE_DOWN_FLOAT /**< Quantize using a floating point multiplication */ -}; - -/** GEMMLowp output stage info */ -struct GEMMLowpOutputStageInfo -{ - GEMMLowpOutputStageType type{ GEMMLowpOutputStageType::NONE }; /**< GEMMLowp output stage type */ - int32_t gemmlowp_offset{ 0 }; /**< GEMMLowp output stage offset used for quantizing to QASYMM8 */ - int32_t gemmlowp_multiplier{ 0 }; /**< GEMMLowp output stage multiplier used for quantizing to QASYMM8 */ - int32_t gemmlowp_shift{ 0 }; /**< GEMMLowp output stage shift used for quantizing to uint8 */ - int32_t gemmlowp_min_bound{ std::numeric_limits::lowest() }; /**< GEMMLowp min value used to saturate down the output result before converting back to QASYMM8 */ - int32_t gemmlowp_max_bound{ std::numeric_limits::max() }; /**< GEMMLowp max value used to saturate down the output result before converting back to QASYMM8 */ - std::vector gemmlowp_multipliers{}; /**< GEMMLowp output stage multiplier used for quantizing to QASYMM8 */ - std::vector gemmlowp_shifts{}; /**< GEMMLowp output stage multiplier used for quantizing to QASYMM8 */ - float gemmlowp_real_multiplier{ 0 }; /**< GEMMLowp output stage real multiplier used for quantizing to QASYMM8 */ - bool is_quantized_per_channel{ false }; /**< GEMMLowp quantized per-channel flag */ - DataType output_data_type{ DataType::UNKNOWN }; /**< Output tensor data type to use if the output is not initialized */ -}; - /** GEMM LHS (Left Hand Side) matrix information */ struct GEMMLHSMatrixInfo { @@ -2351,287 +1831,6 @@ struct GEMMRHSMatrixInfo }; class ITensorInfo; -/** GEMM information class. This class stores the necessary information to compute GEMM functions - * - * This object also contains the information about how matrix A and matrix B have been reshaped - * - */ -class GEMMInfo -{ -public: - /** Default constructor */ - GEMMInfo() noexcept - : _is_a_reshaped(false), - _is_b_reshaped(false), - _reshape_b_only_on_first_run(true), - _depth_output_gemm3d(0), - _reinterpret_input_as_3d(false), - _retain_internal_weights(false), - _gemmlowp_output_stage(), - _fast_math(false), - _fp_mixed_precision(false), - _broadcast_bias(false), - _pretranspose_A(false), - _pretranspose_B(false), - _activation_info(), - _post_ops(), - _fixed_format(false), - _weight_format(arm_compute::WeightFormat::UNSPECIFIED) - { - } - /** Constructor - * - * @param[in] is_a_reshaped True if the matrix A has been reshaped - * @param[in] is_b_reshaped True if the matrix B has been reshaped - * @param[in] reshape_b_only_on_first_run Reshape matrix B only for the first run - * @param[in] depth_output_gemm3d (Optional) Depth (third dimension) of the output tensor to be used with the GEMM3D kernel - * If 0 the output will not be reinterpreted as 3D. Default 0 - * @param[in] reinterpret_input_as_3d (Optional) Reinterpret the input as 3D tensor. (i.e. this flag should be set to true when GEMM is used - * to perform 1x1 convolutions with the NHWC data layout) - * @param[in] retain_internal_weights (Optional) Retain the weights tensor from previous run - * @param[in] gemmlowp_output_stage (Optional) GEMMLowp Output stage info - * @param[in] fp_mixed_precision (Optional) Use wider accumulators (32 bit instead of 16 for FP16) to improve accuracy. - * @param[in] fast_math (Optional) Use a data type of shorter width to improve performance - * @param[in] broadcast_bias (Optional) Broadcast the shape of the bias tensor from a vector to a matrix. - * @param[in] activation_info (Optional) Activation to apply after the matrix multiplication - * @param[in] post_ops (Optional) A sequence of post operations that are performed after the main operation. - * @param[in] fixed_format (Optional) Specify the selection of fixed format kernels for variable weights support in GEMM. These kernels expect the weights tensor to be in amemory format that is fixed by the kernel itself. For more information, see arm_compute::WeightFormat. - * @param[in] weight_format (Optional) arm_gemm:WeightFormat enumeration requested by the user. Default is arm_compute::WeightFormat::UNSPECIFIED. - */ - GEMMInfo(bool is_a_reshaped, bool is_b_reshaped, bool reshape_b_only_on_first_run, int depth_output_gemm3d = 0, bool reinterpret_input_as_3d = false, bool retain_internal_weights = false, - GEMMLowpOutputStageInfo gemmlowp_output_stage = GEMMLowpOutputStageInfo(), bool fp_mixed_precision = false, bool fast_math = false, bool broadcast_bias = false, - const ActivationLayerInfo &activation_info = ActivationLayerInfo(), const experimental::PostOpList &post_ops = experimental::PostOpList(), - bool fixed_format = false, arm_compute::WeightFormat weight_format = arm_compute::WeightFormat::UNSPECIFIED) noexcept - : _is_a_reshaped(is_a_reshaped), - _is_b_reshaped(is_b_reshaped), - _reshape_b_only_on_first_run(reshape_b_only_on_first_run), - _depth_output_gemm3d(depth_output_gemm3d), - _reinterpret_input_as_3d(reinterpret_input_as_3d), - _retain_internal_weights(retain_internal_weights), - _gemmlowp_output_stage(gemmlowp_output_stage), - _fast_math(fast_math), - _fp_mixed_precision(fp_mixed_precision), - _broadcast_bias(broadcast_bias), - _pretranspose_A(false), - _pretranspose_B(false), - _activation_info(activation_info), - _post_ops(post_ops), - _fixed_format(fixed_format), - _weight_format(weight_format) - { - } - /** Flag which specifies if the matrix A has been reshaped - * - * @return True if the matrix A has been reshaped - */ - bool is_a_reshaped() const - { - return _is_a_reshaped; - }; - /** Flag which specifies if the matrix B has been reshaped - * - * @return True if the matrix B has been reshaped - */ - bool is_b_reshaped() const - { - return _is_b_reshaped; - }; - /** Flag which specifies if the reshape of matrix B should executed only for the first - * - * @note This flag could be set to TRUE when GEMM is used to accelerate convolution layer - * - * @return True if the reshaped of matrix B happens only for the first run - */ - bool reshape_b_only_on_first_run() const - { - return _reshape_b_only_on_first_run; - }; - /** Depth of the output when GEMM output is reinterpreted as 3D tensor - * - * @return the depth of the output tensor - */ - int depth_output_gemm3d() const - { - return _depth_output_gemm3d; - }; - /** Flag which specifies if the input tensor has to be reinterpreted as 3D - * - * @return True if the input tensor has to be reinterpreted as 3D tensor - */ - bool reinterpret_input_as_3d() const - { - return _reinterpret_input_as_3d; - }; - /** Flag which specifies if the weights tensor has to be retained from previous run - * - * @return True if the weights tensor has to be retained - */ - bool retain_internal_weights() const - { - return _retain_internal_weights; - }; - /** GEMMLowp output stage - * - * @return the GEMMLowp output stage info - */ - GEMMLowpOutputStageInfo gemmlowp_output_stage() const - { - return _gemmlowp_output_stage; - }; - /** Sets GEMMLowp output stage - * - * @param[in] output_stage Output stage to set - */ - void set_gemmlowp_output_stage(GEMMLowpOutputStageInfo &output_stage) - { - _gemmlowp_output_stage = output_stage; - }; - /** Flag which specifies if a wider accumulator should be used. - * - * @return True if a wider accumulator has to be used - */ - bool fp_mixed_precision() const - { - return _fp_mixed_precision; - }; - /** Flag which specifies if a shorter accumulator to be used. - * - * @return True if a shorter accumulator has to be used - */ - bool fast_math() const - { - return _fast_math; - }; - /** Set fast math flag - * - * @param[in] fast_math Flag to set - */ - void set_fast_math(bool fast_math) - { - _fast_math = fast_math; - } - /** Flag which specifies whether to broadcast the shape of the bias tensor. - * - * @return True if the shape of the bias tensor is to be broadcasted. - */ - bool broadcast_bias() const - { - return _broadcast_bias; - }; - /** Flag which specifies whether A should be pre-transposed if supported. - * - * @return True if A should be pre-transposed else false. - */ - bool pretranspose_A() const - { - return _pretranspose_A; - }; - /** Set pre-transpose A flag - * - * @param[in] flag Flag to set - */ - void set_pretranspose_A(bool flag) - { - _pretranspose_A = flag; - } - /** Flag which specifies whether b should be pre-transposed if supported. - * - * @return True if b should be pre-transposed else false. - */ - bool pretranspose_B() const - { - return _pretranspose_B; - }; - /** Set pre-transpose b flag - * - * @param[in] flag Flag to set - */ - void set_pretranspose_B(bool flag) - { - _pretranspose_B = flag; - } - /** Activation layer to apply after the matrix multiplication - * - * @return ActivationLayerInfo object - */ - ActivationLayerInfo activation_info() const - { - return _activation_info; - } - /** Set activation layer info - * - * @param[in] activation_info ActivationLayerInfo object to set - */ - void set_activation_info(const ActivationLayerInfo &activation_info) - { - _activation_info = activation_info; - } - /** Post operations to apply after the matrix multiplication - * - * @return experimental::PostOpList object - */ - const experimental::PostOpList &post_ops() const - { - return _post_ops; - } - /** Set post ops - * - * @param[in] post_ops experimental::PostOpList object to set - */ - void set_post_ops(const experimental::PostOpList &post_ops) - { - _post_ops = post_ops; - } - /** Flag which specifies if the GEMM operation is running fixed-format kernels. - * - * @return True if the GEMM operation is running fixed-format kernel else false. - */ - bool fixed_format() const - { - return _fixed_format; - } - - /** Set fixed-format flag - * - * @param[in] fixed_format sets whether or not to use fixed-format kernels - */ - void set_fixed_format(bool fixed_format) - { - _fixed_format = fixed_format; - } - - arm_compute::WeightFormat weight_format() const - { - return _weight_format; - } - - /** Set weight format to be used - * - * @param[in] weight_format arm_compute::WeightFormat enumeration - */ - void set_weight_format(arm_compute::WeightFormat weight_format) - { - _weight_format = weight_format; - } - -private: - bool _is_a_reshaped; - bool _is_b_reshaped; - bool _reshape_b_only_on_first_run; - int _depth_output_gemm3d; - bool _reinterpret_input_as_3d; - bool _retain_internal_weights; - GEMMLowpOutputStageInfo _gemmlowp_output_stage; - bool _fast_math; - bool _fp_mixed_precision; - bool _broadcast_bias; - bool _pretranspose_A; - bool _pretranspose_B; - ActivationLayerInfo _activation_info; - experimental::PostOpList _post_ops; - bool _fixed_format; - arm_compute::WeightFormat _weight_format; -}; /** Winograd information */ struct WinogradInfo @@ -2713,52 +1912,7 @@ struct IOFormatInfo bool align_columns; }; -/** Class for holding information related to matrix multiplication function - */ -class MatMulInfo -{ -public: - /* Get Adjoint LHS flag value */ - bool adj_lhs() const - { - return _adj_lhs; - } - /* Get Adjoint RHS flag value */ - bool adj_rhs() const - { - return _adj_rhs; - } - /* Get Fused Activation Layer Info */ - ActivationLayerInfo fused_activation() const - { - return _fused_act; - } - /* Set Adjoint LHS flag */ - MatMulInfo &adj_lhs(bool adj_lhs) - { - _adj_lhs = adj_lhs; - return *this; - } - /* Set Adjoint RHS flag */ - MatMulInfo &adj_rhs(bool adj_rhs) - { - _adj_rhs = adj_rhs; - return *this; - } - /* Set Fused Activation Layer Info */ - MatMulInfo &fused_activation(const ActivationLayerInfo &act_info) - { - _fused_act = act_info; - return *this; - } - -private: - bool _adj_lhs{ false }; - bool _adj_rhs{ false }; - ActivationLayerInfo _fused_act{}; // disabled by default -}; - /** Class for holding information related to cropping */ using CropInfo = Padding2D; } // namespace arm_compute -#endif /* ARM_COMPUTE_TYPES_H */ +#endif /* ACL_ARM_COMPUTE_CORE_TYPES */ diff --git a/arm_compute/core/Utils.h b/arm_compute/core/Utils.h index a47cfbdec6e19aa8be7038d0c60ccb8ddc3eae59..c5b50167bf6eaecd4b3d2f471dc6ac3877adc3c2 100644 --- a/arm_compute/core/Utils.h +++ b/arm_compute/core/Utils.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2022 Arm Limited. + * Copyright (c) 2016-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,67 +26,29 @@ #include "arm_compute/core/Error.h" #include "arm_compute/core/PixelValue.h" -#include "arm_compute/core/Rounding.h" #include "arm_compute/core/Types.h" -#include "arm_compute/core/Version.h" -#include -#include -#include -#include +#include #include #include #include #include #include #include -#include + +/* Convenience / backwards compatibility includes */ +#include "arm_compute/core/utils/ActivationFunctionUtils.h" +#include "arm_compute/core/utils/DataLayoutUtils.h" +#include "arm_compute/core/utils/DataTypeUtils.h" +#include "arm_compute/core/utils/FormatUtils.h" +#include "arm_compute/core/utils/InterpolationPolicyUtils.h" +#include "arm_compute/core/utils/StringUtils.h" namespace arm_compute { class ITensor; class ITensorInfo; - -/** Calculate the rounded up quotient of val / m. - * - * @param[in] val Value to divide and round up. - * @param[in] m Value to divide by. - * - * @return the result. - */ -template -constexpr auto DIV_CEIL(S val, T m) -> decltype((val + m - 1) / m) -{ - return (val + m - 1) / m; -} - -/** Computes the smallest number larger or equal to value that is a multiple of divisor. - * - * @param[in] value Lower bound value - * @param[in] divisor Value to compute multiple of. - * - * @return the result. - */ -template -inline auto ceil_to_multiple(S value, T divisor) -> decltype(((value + divisor - 1) / divisor) * divisor) -{ - ARM_COMPUTE_ERROR_ON(value < 0 || divisor <= 0); - return DIV_CEIL(value, divisor) * divisor; -} - -/** Computes the largest number smaller or equal to value that is a multiple of divisor. - * - * @param[in] value Upper bound value - * @param[in] divisor Value to compute multiple of. - * - * @return the result. - */ -template -inline auto floor_to_multiple(S value, T divisor) -> decltype((value / divisor) * divisor) -{ - ARM_COMPUTE_ERROR_ON(value < 0 || divisor <= 0); - return (value / divisor) * divisor; -} +class ActivationLayerInfo; /** Load an entire file in memory * @@ -97,627 +59,6 @@ inline auto floor_to_multiple(S value, T divisor) -> decltype((value / divisor) */ std::string read_file(const std::string &filename, bool binary); -/** The size in bytes of the data type - * - * @param[in] data_type Input data type - * - * @return The size in bytes of the data type - */ -inline size_t data_size_from_type(DataType data_type) -{ - switch(data_type) - { - case DataType::U8: - case DataType::S8: - case DataType::QSYMM8: - case DataType::QASYMM8: - case DataType::QASYMM8_SIGNED: - case DataType::QSYMM8_PER_CHANNEL: - return 1; - case DataType::U16: - case DataType::S16: - case DataType::QSYMM16: - case DataType::QASYMM16: - case DataType::BFLOAT16: - case DataType::F16: - return 2; - case DataType::F32: - case DataType::U32: - case DataType::S32: - return 4; - case DataType::F64: - case DataType::U64: - case DataType::S64: - return 8; - case DataType::SIZET: - return sizeof(size_t); - default: - ARM_COMPUTE_ERROR("Invalid data type"); - return 0; - } -} - -/** The size in bytes of the pixel format - * - * @param[in] format Input format - * - * @return The size in bytes of the pixel format - */ -inline size_t pixel_size_from_format(Format format) -{ - switch(format) - { - case Format::U8: - return 1; - case Format::U16: - case Format::S16: - case Format::BFLOAT16: - case Format::F16: - case Format::UV88: - case Format::YUYV422: - case Format::UYVY422: - return 2; - case Format::RGB888: - return 3; - case Format::RGBA8888: - return 4; - case Format::U32: - case Format::S32: - case Format::F32: - return 4; - //Doesn't make sense for planar formats: - case Format::NV12: - case Format::NV21: - case Format::IYUV: - case Format::YUV444: - default: - ARM_COMPUTE_ERROR("Undefined pixel size for given format"); - return 0; - } -} - -/** The size in bytes of the data type - * - * @param[in] dt Input data type - * - * @return The size in bytes of the data type - */ -inline size_t element_size_from_data_type(DataType dt) -{ - switch(dt) - { - case DataType::S8: - case DataType::U8: - case DataType::QSYMM8: - case DataType::QASYMM8: - case DataType::QASYMM8_SIGNED: - case DataType::QSYMM8_PER_CHANNEL: - return 1; - case DataType::U16: - case DataType::S16: - case DataType::QSYMM16: - case DataType::QASYMM16: - case DataType::BFLOAT16: - case DataType::F16: - return 2; - case DataType::U32: - case DataType::S32: - case DataType::F32: - return 4; - default: - ARM_COMPUTE_ERROR("Undefined element size for given data type"); - return 0; - } -} - -/** Return the data type used by a given single-planar pixel format - * - * @param[in] format Input format - * - * @return The size in bytes of the pixel format - */ -inline DataType data_type_from_format(Format format) -{ - switch(format) - { - case Format::U8: - case Format::UV88: - case Format::RGB888: - case Format::RGBA8888: - case Format::YUYV422: - case Format::UYVY422: - return DataType::U8; - case Format::U16: - return DataType::U16; - case Format::S16: - return DataType::S16; - case Format::U32: - return DataType::U32; - case Format::S32: - return DataType::S32; - case Format::BFLOAT16: - return DataType::BFLOAT16; - case Format::F16: - return DataType::F16; - case Format::F32: - return DataType::F32; - //Doesn't make sense for planar formats: - case Format::NV12: - case Format::NV21: - case Format::IYUV: - case Format::YUV444: - default: - ARM_COMPUTE_ERROR("Not supported data_type for given format"); - return DataType::UNKNOWN; - } -} - -/** Return the plane index of a given channel given an input format. - * - * @param[in] format Input format - * @param[in] channel Input channel - * - * @return The plane index of the specific channel of the specific format - */ -inline int plane_idx_from_channel(Format format, Channel channel) -{ - switch(format) - { - // Single planar formats have a single plane - case Format::U8: - case Format::U16: - case Format::S16: - case Format::U32: - case Format::S32: - case Format::BFLOAT16: - case Format::F16: - case Format::F32: - case Format::UV88: - case Format::RGB888: - case Format::RGBA8888: - case Format::YUYV422: - case Format::UYVY422: - return 0; - // Multi planar formats - case Format::NV12: - case Format::NV21: - { - // Channel U and V share the same plane of format UV88 - switch(channel) - { - case Channel::Y: - return 0; - case Channel::U: - case Channel::V: - return 1; - default: - ARM_COMPUTE_ERROR("Not supported channel"); - return 0; - } - } - case Format::IYUV: - case Format::YUV444: - { - switch(channel) - { - case Channel::Y: - return 0; - case Channel::U: - return 1; - case Channel::V: - return 2; - default: - ARM_COMPUTE_ERROR("Not supported channel"); - return 0; - } - } - default: - ARM_COMPUTE_ERROR("Not supported format"); - return 0; - } -} - -/** Return the channel index of a given channel given an input format. - * - * @param[in] format Input format - * @param[in] channel Input channel - * - * @return The channel index of the specific channel of the specific format - */ -inline int channel_idx_from_format(Format format, Channel channel) -{ - switch(format) - { - case Format::RGB888: - { - switch(channel) - { - case Channel::R: - return 0; - case Channel::G: - return 1; - case Channel::B: - return 2; - default: - ARM_COMPUTE_ERROR("Not supported channel"); - return 0; - } - } - case Format::RGBA8888: - { - switch(channel) - { - case Channel::R: - return 0; - case Channel::G: - return 1; - case Channel::B: - return 2; - case Channel::A: - return 3; - default: - ARM_COMPUTE_ERROR("Not supported channel"); - return 0; - } - } - case Format::YUYV422: - { - switch(channel) - { - case Channel::Y: - return 0; - case Channel::U: - return 1; - case Channel::V: - return 3; - default: - ARM_COMPUTE_ERROR("Not supported channel"); - return 0; - } - } - case Format::UYVY422: - { - switch(channel) - { - case Channel::Y: - return 1; - case Channel::U: - return 0; - case Channel::V: - return 2; - default: - ARM_COMPUTE_ERROR("Not supported channel"); - return 0; - } - } - case Format::NV12: - { - switch(channel) - { - case Channel::Y: - return 0; - case Channel::U: - return 0; - case Channel::V: - return 1; - default: - ARM_COMPUTE_ERROR("Not supported channel"); - return 0; - } - } - case Format::NV21: - { - switch(channel) - { - case Channel::Y: - return 0; - case Channel::U: - return 1; - case Channel::V: - return 0; - default: - ARM_COMPUTE_ERROR("Not supported channel"); - return 0; - } - } - case Format::YUV444: - case Format::IYUV: - { - switch(channel) - { - case Channel::Y: - return 0; - case Channel::U: - return 0; - case Channel::V: - return 0; - default: - ARM_COMPUTE_ERROR("Not supported channel"); - return 0; - } - } - default: - ARM_COMPUTE_ERROR("Not supported format"); - return 0; - } -} - -/** Return the number of planes for a given format - * - * @param[in] format Input format - * - * @return The number of planes for a given image format. - */ -inline size_t num_planes_from_format(Format format) -{ - switch(format) - { - case Format::U8: - case Format::S16: - case Format::U16: - case Format::S32: - case Format::U32: - case Format::BFLOAT16: - case Format::F16: - case Format::F32: - case Format::RGB888: - case Format::RGBA8888: - case Format::YUYV422: - case Format::UYVY422: - return 1; - case Format::NV12: - case Format::NV21: - return 2; - case Format::IYUV: - case Format::YUV444: - return 3; - default: - ARM_COMPUTE_ERROR("Not supported format"); - return 0; - } -} - -/** Return the number of channels for a given single-planar pixel format - * - * @param[in] format Input format - * - * @return The number of channels for a given image format. - */ -inline size_t num_channels_from_format(Format format) -{ - switch(format) - { - case Format::U8: - case Format::U16: - case Format::S16: - case Format::U32: - case Format::S32: - case Format::BFLOAT16: - case Format::F16: - case Format::F32: - return 1; - // Because the U and V channels are subsampled - // these formats appear like having only 2 channels: - case Format::YUYV422: - case Format::UYVY422: - return 2; - case Format::UV88: - return 2; - case Format::RGB888: - return 3; - case Format::RGBA8888: - return 4; - //Doesn't make sense for planar formats: - case Format::NV12: - case Format::NV21: - case Format::IYUV: - case Format::YUV444: - default: - return 0; - } -} - -/** Return the promoted data type of a given data type. - * - * @note If promoted data type is not supported an error will be thrown - * - * @param[in] dt Data type to get the promoted type of. - * - * @return Promoted data type - */ -inline DataType get_promoted_data_type(DataType dt) -{ - switch(dt) - { - case DataType::U8: - return DataType::U16; - case DataType::S8: - return DataType::S16; - case DataType::U16: - return DataType::U32; - case DataType::S16: - return DataType::S32; - case DataType::QSYMM8: - case DataType::QASYMM8: - case DataType::QASYMM8_SIGNED: - case DataType::QSYMM8_PER_CHANNEL: - case DataType::QSYMM16: - case DataType::QASYMM16: - case DataType::BFLOAT16: - case DataType::F16: - case DataType::U32: - case DataType::S32: - case DataType::F32: - ARM_COMPUTE_ERROR("Unsupported data type promotions!"); - default: - ARM_COMPUTE_ERROR("Undefined data type!"); - } - return DataType::UNKNOWN; -} - -/** Compute the mininum and maximum values a data type can take - * - * @param[in] dt Data type to get the min/max bounds of - * - * @return A tuple (min,max) with the minimum and maximum values respectively wrapped in PixelValue. - */ -inline std::tuple get_min_max(DataType dt) -{ - PixelValue min{}; - PixelValue max{}; - switch(dt) - { - case DataType::U8: - case DataType::QASYMM8: - { - min = PixelValue(static_cast(std::numeric_limits::lowest())); - max = PixelValue(static_cast(std::numeric_limits::max())); - break; - } - case DataType::S8: - case DataType::QSYMM8: - case DataType::QASYMM8_SIGNED: - case DataType::QSYMM8_PER_CHANNEL: - { - min = PixelValue(static_cast(std::numeric_limits::lowest())); - max = PixelValue(static_cast(std::numeric_limits::max())); - break; - } - case DataType::U16: - case DataType::QASYMM16: - { - min = PixelValue(static_cast(std::numeric_limits::lowest())); - max = PixelValue(static_cast(std::numeric_limits::max())); - break; - } - case DataType::S16: - case DataType::QSYMM16: - { - min = PixelValue(static_cast(std::numeric_limits::lowest())); - max = PixelValue(static_cast(std::numeric_limits::max())); - break; - } - case DataType::U32: - { - min = PixelValue(std::numeric_limits::lowest()); - max = PixelValue(std::numeric_limits::max()); - break; - } - case DataType::S32: - { - min = PixelValue(std::numeric_limits::lowest()); - max = PixelValue(std::numeric_limits::max()); - break; - } - case DataType::BFLOAT16: - { - min = PixelValue(bfloat16::lowest()); - max = PixelValue(bfloat16::max()); - break; - } - case DataType::F16: - { - min = PixelValue(std::numeric_limits::lowest()); - max = PixelValue(std::numeric_limits::max()); - break; - } - case DataType::F32: - { - min = PixelValue(std::numeric_limits::lowest()); - max = PixelValue(std::numeric_limits::max()); - break; - } - default: - ARM_COMPUTE_ERROR("Undefined data type!"); - } - return std::make_tuple(min, max); -} - -/** Return true if the given format has horizontal subsampling. - * - * @param[in] format Format to determine subsampling. - * - * @return True if the format can be subsampled horizontaly. - */ -inline bool has_format_horizontal_subsampling(Format format) -{ - return (format == Format::YUYV422 || format == Format::UYVY422 || format == Format::NV12 || format == Format::NV21 || format == Format::IYUV || format == Format::UV88) ? true : false; -} - -/** Return true if the given format has vertical subsampling. - * - * @param[in] format Format to determine subsampling. - * - * @return True if the format can be subsampled verticaly. - */ -inline bool has_format_vertical_subsampling(Format format) -{ - return (format == Format::NV12 || format == Format::NV21 || format == Format::IYUV || format == Format::UV88) ? true : false; -} - -/** Adjust tensor shape size if width or height are odd for a given multi-planar format. No modification is done for other formats. - * - * @note Adding here a few links discussing the issue of odd size and sharing the same solution: - * Android Source - * WebM - * libYUV - * YUVPlayer * - * - * @param[in, out] shape Tensor shape of 2D size - * @param[in] format Format of the tensor - * - * @return The adjusted tensor shape. - */ -inline TensorShape adjust_odd_shape(const TensorShape &shape, Format format) -{ - TensorShape output{ shape }; - - // Force width to be even for formats which require subsampling of the U and V channels - if(has_format_horizontal_subsampling(format)) - { - output.set(0, (output.x() + 1) & ~1U); - } - - // Force height to be even for formats which require subsampling of the U and V channels - if(has_format_vertical_subsampling(format)) - { - output.set(1, (output.y() + 1) & ~1U); - } - - return output; -} - -/** Calculate subsampled shape for a given format and channel - * - * @param[in] shape Shape of the tensor to calculate the extracted channel. - * @param[in] format Format of the tensor. - * @param[in] channel Channel to create tensor shape to be extracted. - * - * @return The subsampled tensor shape. - */ -inline TensorShape calculate_subsampled_shape(const TensorShape &shape, Format format, Channel channel = Channel::UNKNOWN) -{ - TensorShape output{ shape }; - - // Subsample shape only for U or V channel - if(Channel::U == channel || Channel::V == channel || Channel::UNKNOWN == channel) - { - // Subsample width for the tensor shape when channel is U or V - if(has_format_horizontal_subsampling(format)) - { - output.set(0, output.x() / 2U); - } - - // Subsample height for the tensor shape when channel is U or V - if(has_format_vertical_subsampling(format)) - { - output.set(1, output.y() / 2U); - } - } - - return output; -} - /** Permutes the given dimensions according the permutation vector * * @param[in,out] dimensions Dimensions to be permuted. @@ -837,15 +178,7 @@ QuantizationInfo get_softmax_output_quantization_info(DataType input_type, bool * * @return The pair with minimum and maximum values */ -std::pair get_quantized_activation_min_max(ActivationLayerInfo act_info, DataType data_type, UniformQuantizationInfo oq_info); - -/** Convert a tensor format into a string. - * - * @param[in] format @ref Format to be translated to string. - * - * @return The string describing the format. - */ -const std::string &string_from_format(Format format); +std::pair get_quantized_activation_min_max(const ActivationLayerInfo &act_info, DataType data_type, UniformQuantizationInfo oq_info); /** Convert a channel identity into a string. * @@ -854,34 +187,7 @@ const std::string &string_from_format(Format format); * @return The string describing the channel. */ const std::string &string_from_channel(Channel channel); -/** Convert a data layout identity into a string. - * - * @param[in] dl @ref DataLayout to be translated to string. - * - * @return The string describing the data layout. - */ -const std::string &string_from_data_layout(DataLayout dl); -/** Convert a data type identity into a string. - * - * @param[in] dt @ref DataType to be translated to string. - * - * @return The string describing the data type. - */ -const std::string &string_from_data_type(DataType dt); -/** Translates a given activation function to a string. - * - * @param[in] act @ref ActivationLayerInfo::ActivationFunction to be translated to string. - * - * @return The string describing the activation function. - */ -const std::string &string_from_activation_func(ActivationLayerInfo::ActivationFunction act); -/** Translates a given interpolation policy to a string. - * - * @param[in] policy @ref InterpolationPolicy to be translated to string. - * - * @return The string describing the interpolation policy. - */ -const std::string &string_from_interpolation_policy(InterpolationPolicy policy); + /** Translates a given border mode policy to a string. * * @param[in] border_mode @ref BorderMode to be translated to string. @@ -923,7 +229,7 @@ bool is_pool_3d_region_entirely_outside_input(const Pooling3dLayerInfo &info); * * @return True if padding is symmetric */ -inline bool is_symmetric(const Padding3D& info) +inline bool is_symmetric(const Padding3D &info) { return ((info.left == info.right) && (info.top == info.bottom) && (info.front == info.back)); } @@ -942,13 +248,7 @@ const std::string &string_from_gemmlowp_output_stage(GEMMLowpOutputStageType out * @return String representation of the PixelValue through the given data type. */ std::string string_from_pixel_value(const PixelValue &value, const DataType data_type); -/** Convert a string to DataType - * - * @param[in] name The name of the data type - * - * @return DataType - */ -DataType data_type_from_name(const std::string &name); + /** Stores padding information before configuring a kernel * * @param[in] infos list of tensor infos to store the padding info for @@ -971,170 +271,6 @@ std::unordered_map get_padding_info(std::initi */ bool has_padding_changed(const std::unordered_map &padding_map); -/** Input Stream operator for @ref DataType - * - * @param[in] stream Stream to parse - * @param[out] data_type Output data type - * - * @return Updated stream - */ -inline ::std::istream &operator>>(::std::istream &stream, DataType &data_type) -{ - std::string value; - stream >> value; - data_type = data_type_from_name(value); - return stream; -} -/** Lower a given string. - * - * @param[in] val Given string to lower. - * - * @return The lowered string - */ -std::string lower_string(const std::string &val); - -/** Raise a given string to upper case - * - * @param[in] val Given string to lower. - * - * @return The upper case string - */ -std::string upper_string(const std::string &val); - -/** Check if a given data type is of floating point type - * - * @param[in] dt Input data type. - * - * @return True if data type is of floating point type, else false. - */ -inline bool is_data_type_float(DataType dt) -{ - switch(dt) - { - case DataType::F16: - case DataType::F32: - return true; - default: - return false; - } -} - -/** Check if a given data type is of quantized type - * - * @note Quantized is considered a super-set of fixed-point and asymmetric data types. - * - * @param[in] dt Input data type. - * - * @return True if data type is of quantized type, else false. - */ -inline bool is_data_type_quantized(DataType dt) -{ - switch(dt) - { - case DataType::QSYMM8: - case DataType::QASYMM8: - case DataType::QASYMM8_SIGNED: - case DataType::QSYMM8_PER_CHANNEL: - case DataType::QSYMM16: - case DataType::QASYMM16: - return true; - default: - return false; - } -} - -/** Check if a given data type is of asymmetric quantized type - * - * @param[in] dt Input data type. - * - * @return True if data type is of asymmetric quantized type, else false. - */ -inline bool is_data_type_quantized_asymmetric(DataType dt) -{ - switch(dt) - { - case DataType::QASYMM8: - case DataType::QASYMM8_SIGNED: - case DataType::QASYMM16: - return true; - default: - return false; - } -} - -/** Check if a given data type is of asymmetric quantized signed type - * - * @param[in] dt Input data type. - * - * @return True if data type is of asymmetric quantized signed type, else false. - */ -inline bool is_data_type_quantized_asymmetric_signed(DataType dt) -{ - switch(dt) - { - case DataType::QASYMM8_SIGNED: - return true; - default: - return false; - } -} - -/** Check if a given data type is of symmetric quantized type - * - * @param[in] dt Input data type. - * - * @return True if data type is of symmetric quantized type, else false. - */ -inline bool is_data_type_quantized_symmetric(DataType dt) -{ - switch(dt) - { - case DataType::QSYMM8: - case DataType::QSYMM8_PER_CHANNEL: - case DataType::QSYMM16: - return true; - default: - return false; - } -} - -/** Check if a given data type is of per channel type - * - * @param[in] dt Input data type. - * - * @return True if data type is of per channel type, else false. - */ -inline bool is_data_type_quantized_per_channel(DataType dt) -{ - switch(dt) - { - case DataType::QSYMM8_PER_CHANNEL: - return true; - default: - return false; - } -} - -/** Create a string with the float in full precision. - * - * @param val Floating point value - * - * @return String with the floating point value. - */ -inline std::string float_to_string_with_full_precision(float val) -{ - std::stringstream ss; - ss.precision(std::numeric_limits::max_digits10); - ss << val; - - if(val != static_cast(val)) - { - ss << "f"; - } - - return ss.str(); -} - /** Returns the number of elements required to go from start to end with the wanted step * * @param[in] start start value @@ -1149,142 +285,6 @@ inline size_t num_of_elements_in_range(const float start, const float end, const return size_t(std::ceil((end - start) / step)); } -/** Returns true if the value can be represented by the given data type - * - * @param[in] val value to be checked - * @param[in] dt data type that is checked - * @param[in] qinfo (Optional) quantization info if the data type is QASYMM8 - * - * @return true if the data type can hold the value. - */ -template -bool check_value_range(T val, DataType dt, QuantizationInfo qinfo = QuantizationInfo()) -{ - switch(dt) - { - case DataType::U8: - { - const auto val_u8 = static_cast(val); - return ((val_u8 == val) && val >= std::numeric_limits::lowest() && val <= std::numeric_limits::max()); - } - case DataType::QASYMM8: - { - double min = static_cast(dequantize_qasymm8(0, qinfo)); - double max = static_cast(dequantize_qasymm8(std::numeric_limits::max(), qinfo)); - return ((double)val >= min && (double)val <= max); - } - case DataType::S8: - { - const auto val_s8 = static_cast(val); - return ((val_s8 == val) && val >= std::numeric_limits::lowest() && val <= std::numeric_limits::max()); - } - case DataType::U16: - { - const auto val_u16 = static_cast(val); - return ((val_u16 == val) && val >= std::numeric_limits::lowest() && val <= std::numeric_limits::max()); - } - case DataType::S16: - { - const auto val_s16 = static_cast(val); - return ((val_s16 == val) && val >= std::numeric_limits::lowest() && val <= std::numeric_limits::max()); - } - case DataType::U32: - { - const auto val_d64 = static_cast(val); - const auto val_u32 = static_cast(val); - return ((val_u32 == val_d64) && val_d64 >= std::numeric_limits::lowest() && val_d64 <= std::numeric_limits::max()); - } - case DataType::S32: - { - const auto val_d64 = static_cast(val); - const auto val_s32 = static_cast(val); - return ((val_s32 == val_d64) && val_d64 >= std::numeric_limits::lowest() && val_d64 <= std::numeric_limits::max()); - } - case DataType::BFLOAT16: - return (val >= bfloat16::lowest() && val <= bfloat16::max()); - case DataType::F16: - return (val >= std::numeric_limits::lowest() && val <= std::numeric_limits::max()); - case DataType::F32: - return (val >= std::numeric_limits::lowest() && val <= std::numeric_limits::max()); - default: - ARM_COMPUTE_ERROR("Data type not supported"); - return false; - } -} - -/** Returns the adjusted vector size in case it is less than the input's first dimension, getting rounded down to its closest valid vector size - * - * @param[in] vec_size vector size to be adjusted - * @param[in] dim0 size of the first dimension - * - * @return the number of element processed along the X axis per thread - */ -inline unsigned int adjust_vec_size(unsigned int vec_size, size_t dim0) -{ - ARM_COMPUTE_ERROR_ON(vec_size > 16); - - if((vec_size >= dim0) && (dim0 == 3)) - { - return dim0; - } - - while(vec_size > dim0) - { - vec_size >>= 1; - } - - return vec_size; -} - -/** Returns the suffix string of CPU kernel implementation names based on the given data type - * - * @param[in] data_type The data type the CPU kernel implemetation uses - * - * @return the suffix string of CPU kernel implementations - */ -inline std::string cpu_impl_dt(const DataType &data_type) -{ - std::string ret = ""; - - switch(data_type) - { - case DataType::F32: - ret = "fp32"; - break; - case DataType::F16: - ret = "fp16"; - break; - case DataType::U8: - ret = "u8"; - break; - case DataType::S16: - ret = "s16"; - break; - case DataType::S32: - ret = "s32"; - break; - case DataType::QASYMM8: - ret = "qu8"; - break; - case DataType::QASYMM8_SIGNED: - ret = "qs8"; - break; - case DataType::QSYMM16: - ret = "qs16"; - break; - case DataType::QSYMM8_PER_CHANNEL: - ret = "qp8"; - break; - case DataType::BFLOAT16: - ret = "bf16"; - break; - default: - ARM_COMPUTE_ERROR("Unsupported."); - } - - return ret; -} - #ifdef ARM_COMPUTE_ASSERTS_ENABLED /** Print consecutive elements to an output stream. * diff --git a/arm_compute/core/Validate.h b/arm_compute/core/Validate.h index e755cacae60f305cf1a94be6880682419ee2cf48..5bffc16f3b56831fed298cc8ac39c582df4e2b5a 100644 --- a/arm_compute/core/Validate.h +++ b/arm_compute/core/Validate.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021 Arm Limited. + * Copyright (c) 2016-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,9 +24,13 @@ #ifndef ARM_COMPUTE_VALIDATE_H #define ARM_COMPUTE_VALIDATE_H +#include "arm_compute/core/utils/DataLayoutUtils.h" +#include "arm_compute/core/utils/DataTypeUtils.h" #include "arm_compute/core/Error.h" +#include "arm_compute/core/utils/FormatUtils.h" #include "arm_compute/core/IKernel.h" #include "arm_compute/core/ITensor.h" +#include "arm_compute/core/QuantizationInfo.h" #include "arm_compute/core/Window.h" #include @@ -284,6 +288,60 @@ arm_compute::Status error_on_mismatching_dimensions(const char *function, const #define ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DIMENSIONS(...) \ ARM_COMPUTE_RETURN_ON_ERROR(::arm_compute::error_on_mismatching_dimensions(__func__, __FILE__, __LINE__, __VA_ARGS__)) +/** Return true if the given format has horizontal subsampling. + * + * @param[in] format Format to determine subsampling. + * + * @return True if the format can be subsampled horizontaly. + */ +inline bool has_format_horizontal_subsampling(Format format) +{ + return (format == Format::YUYV422 || format == Format::UYVY422 || format == Format::NV12 || format == Format::NV21 || format == Format::IYUV || format == Format::UV88) ? true : false; +} + +/** Return true if the given format has vertical subsampling. + * + * @param[in] format Format to determine subsampling. + * + * @return True if the format can be subsampled verticaly. + */ +inline bool has_format_vertical_subsampling(Format format) +{ + return (format == Format::NV12 || format == Format::NV21 || format == Format::IYUV || format == Format::UV88) ? true : false; +} + +/** Adjust tensor shape size if width or height are odd for a given multi-planar format. No modification is done for other formats. + * + * @note Adding here a few links discussing the issue of odd size and sharing the same solution: + * Android Source + * WebM + * libYUV + * YUVPlayer * + * + * @param[in, out] shape Tensor shape of 2D size + * @param[in] format Format of the tensor + * + * @return The adjusted tensor shape. + */ +inline TensorShape adjust_odd_shape(const TensorShape &shape, Format format) +{ + TensorShape output{ shape }; + + // Force width to be even for formats which require subsampling of the U and V channels + if(has_format_horizontal_subsampling(format)) + { + output.set(0, (output.x() + 1) & ~1U); + } + + // Force height to be even for formats which require subsampling of the U and V channels + if(has_format_vertical_subsampling(format)) + { + output.set(1, (output.y() + 1) & ~1U); + } + + return output; +} + /** Return an error if the passed tensor objects are not even. * * @param[in] function Function in which the error occurred. @@ -316,6 +374,37 @@ arm_compute::Status error_on_tensors_not_even(const char *function, const char * #define ARM_COMPUTE_RETURN_ERROR_ON_TENSORS_NOT_EVEN(...) \ ARM_COMPUTE_RETURN_ON_ERROR(::arm_compute::error_on_tensors_not_even(__func__, __FILE__, __LINE__, __VA_ARGS__)) +/** Calculate subsampled shape for a given format and channel + * + * @param[in] shape Shape of the tensor to calculate the extracted channel. + * @param[in] format Format of the tensor. + * @param[in] channel Channel to create tensor shape to be extracted. + * + * @return The subsampled tensor shape. + */ +inline TensorShape calculate_subsampled_shape(const TensorShape &shape, Format format, Channel channel = Channel::UNKNOWN) +{ + TensorShape output{ shape }; + + // Subsample shape only for U or V channel + if(Channel::U == channel || Channel::V == channel || Channel::UNKNOWN == channel) + { + // Subsample width for the tensor shape when channel is U or V + if(has_format_horizontal_subsampling(format)) + { + output.set(0, output.x() / 2U); + } + + // Subsample height for the tensor shape when channel is U or V + if(has_format_vertical_subsampling(format)) + { + output.set(1, output.y() / 2U); + } + } + + return output; +} + /** Return an error if the passed tensor objects are not sub-sampled. * * @param[in] function Function in which the error occurred. @@ -401,9 +490,9 @@ inline arm_compute::Status error_on_mismatching_shapes(const char *function, con { ARM_COMPUTE_RETURN_ERROR_ON_LOC(tensor_info_1 == nullptr, function, file, line); ARM_COMPUTE_RETURN_ERROR_ON_LOC(tensor_info_2 == nullptr, function, file, line); - ARM_COMPUTE_RETURN_ON_ERROR(::arm_compute::error_on_nullptr(function, file, line, std::forward(tensor_infos)...)); + ARM_COMPUTE_RETURN_ON_ERROR(::arm_compute::error_on_nullptr(function, file, line, tensor_infos...)); - const std::array < const ITensorInfo *, 2 + sizeof...(Ts) > tensors_info_array{ { tensor_info_1, tensor_info_2, std::forward(tensor_infos)... } }; + const std::array < const ITensorInfo *, 2 + sizeof...(Ts) > tensors_info_array{ { tensor_info_1, tensor_info_2, tensor_infos... } }; ARM_COMPUTE_RETURN_ERROR_ON_LOC_MSG(std::any_of(std::next(tensors_info_array.cbegin()), tensors_info_array.cend(), [&](const ITensorInfo * tensor_info) { return detail::have_different_dimensions((*tensors_info_array.cbegin())->tensor_shape(), tensor_info->tensor_shape(), upper_dim); @@ -429,7 +518,7 @@ inline arm_compute::Status error_on_mismatching_shapes(const char *function, con { ARM_COMPUTE_RETURN_ERROR_ON_LOC(tensor_1 == nullptr, function, file, line); ARM_COMPUTE_RETURN_ERROR_ON_LOC(tensor_2 == nullptr, function, file, line); - ARM_COMPUTE_RETURN_ON_ERROR(::arm_compute::error_on_nullptr(function, file, line, std::forward(tensors)...)); + ARM_COMPUTE_RETURN_ON_ERROR(::arm_compute::error_on_nullptr(function, file, line, tensors...)); ARM_COMPUTE_RETURN_ON_ERROR(::arm_compute::error_on_mismatching_shapes(function, file, line, upper_dim, tensor_1->info(), tensor_2->info(), detail::get_tensor_info_t()(tensors)...)); return arm_compute::Status{}; @@ -454,10 +543,10 @@ inline arm_compute::Status error_on_mismatching_data_layouts(const char *functio const ITensorInfo *tensor_info, Ts... tensor_infos) { ARM_COMPUTE_RETURN_ERROR_ON_LOC(tensor_info == nullptr, function, file, line); - ARM_COMPUTE_RETURN_ON_ERROR(::arm_compute::error_on_nullptr(function, file, line, std::forward(tensor_infos)...)); + ARM_COMPUTE_RETURN_ON_ERROR(::arm_compute::error_on_nullptr(function, file, line, tensor_infos...)); DataLayout &&tensor_data_layout = tensor_info->data_layout(); - const std::array tensors_infos_array{ { std::forward(tensor_infos)... } }; + const std::array tensors_infos_array{ { tensor_infos... } }; ARM_COMPUTE_RETURN_ERROR_ON_LOC_MSG(std::any_of(tensors_infos_array.begin(), tensors_infos_array.end(), [&](const ITensorInfo * tensor_info_obj) { return tensor_info_obj->data_layout() != tensor_data_layout; @@ -505,10 +594,10 @@ inline arm_compute::Status error_on_mismatching_data_types(const char *function, const ITensorInfo *tensor_info, Ts... tensor_infos) { ARM_COMPUTE_RETURN_ERROR_ON_LOC(tensor_info == nullptr, function, file, line); - ARM_COMPUTE_RETURN_ON_ERROR(::arm_compute::error_on_nullptr(function, file, line, std::forward(tensor_infos)...)); + ARM_COMPUTE_RETURN_ON_ERROR(::arm_compute::error_on_nullptr(function, file, line, tensor_infos...)); DataType &&tensor_data_type = tensor_info->data_type(); - const std::array tensors_infos_array{ { std::forward(tensor_infos)... } }; + const std::array tensors_infos_array{ { tensor_infos... } }; ARM_COMPUTE_RETURN_ERROR_ON_LOC_MSG(std::any_of(tensors_infos_array.begin(), tensors_infos_array.end(), [&](const ITensorInfo * tensor_info_obj) { return tensor_info_obj->data_type() != tensor_data_type; @@ -531,7 +620,7 @@ inline arm_compute::Status error_on_mismatching_data_types(const char *function, const ITensor *tensor, Ts... tensors) { ARM_COMPUTE_RETURN_ERROR_ON_LOC(tensor == nullptr, function, file, line); - ARM_COMPUTE_RETURN_ON_ERROR(::arm_compute::error_on_nullptr(function, file, line, std::forward(tensors)...)); + ARM_COMPUTE_RETURN_ON_ERROR(::arm_compute::error_on_nullptr(function, file, line, tensors...)); ARM_COMPUTE_RETURN_ON_ERROR(::arm_compute::error_on_mismatching_data_types(function, file, line, tensor->info(), detail::get_tensor_info_t()(tensors)...)); return arm_compute::Status{}; diff --git a/arm_compute/core/Window.h b/arm_compute/core/Window.h index 440b942dcfc95115ac64aa7ca15c80ee78ad680d..8ae859f4b325902744fa67a40228035e6383e775 100644 --- a/arm_compute/core/Window.h +++ b/arm_compute/core/Window.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2020, 2022 Arm Limited. + * Copyright (c) 2016-2020, 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -31,7 +31,7 @@ #include "arm_compute/core/Coordinates.h" #include "arm_compute/core/Error.h" #include "arm_compute/core/ITensorInfo.h" -#include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/math/Math.h" namespace arm_compute { diff --git a/arm_compute/core/experimental/PostOps.h b/arm_compute/core/experimental/PostOps.h index 4ea90fc348931907d58d849ea7e777498d15b580..a5585bab5d6ea2e777d4e4703a6c30dc53b87adc 100644 --- a/arm_compute/core/experimental/PostOps.h +++ b/arm_compute/core/experimental/PostOps.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,6 +27,7 @@ #include "arm_compute/core/experimental/IPostOp.h" #include "arm_compute/core/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include @@ -159,4 +160,4 @@ public: }; } // namespace experimental } // namespace arm_compute -#endif //ARM_COMPUTE_EXPERIMENTAL_POSTOPS \ No newline at end of file +#endif //ARM_COMPUTE_EXPERIMENTAL_POSTOPS diff --git a/arm_compute/core/utils/ActivationFunctionUtils.h b/arm_compute/core/utils/ActivationFunctionUtils.h new file mode 100644 index 0000000000000000000000000000000000000000..1cb66da13de6c17d9413268ca848498141260a12 --- /dev/null +++ b/arm_compute/core/utils/ActivationFunctionUtils.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2016-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_CORE_UTILS_ACTIVATIONFUNCTIONUTILS_H +#define ARM_COMPUTE_CORE_UTILS_ACTIVATIONFUNCTIONUTILS_H + +#include "arm_compute/core/Types.h" + +#include + +namespace arm_compute +{ +/** Translates a given activation function to a string. + * + * @param[in] act @ref ActivationLayerInfo::ActivationFunction to be translated to string. + * + * @return The string describing the activation function. + */ +const std::string &string_from_activation_func(const ActivationFunction &act); +} +#endif /*ARM_COMPUTE_CORE_UTILS_ACTIVATIONFUNCTIONUTILS_H */ diff --git a/arm_compute/core/utils/DataLayoutUtils.h b/arm_compute/core/utils/DataLayoutUtils.h new file mode 100644 index 0000000000000000000000000000000000000000..399f55c63f6ad423ef5cac7d26c4c6202ef4629b --- /dev/null +++ b/arm_compute/core/utils/DataLayoutUtils.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2016-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_CORE_UTILS_DATALAYOUTUTILS_H +#define ARM_COMPUTE_CORE_UTILS_DATALAYOUTUTILS_H +#include "arm_compute/core/Types.h" + +#include + +namespace arm_compute +{ +/** Convert a data layout identity into a string. + * + * @param[in] dl @ref DataLayout to be translated to string. + * + * @return The string describing the data layout. + */ +const std::string &string_from_data_layout(DataLayout dl); +} +#endif /*ARM_COMPUTE_CORE_UTILS_DATALAYOUTUTILS_H */ diff --git a/arm_compute/core/utils/DataTypeUtils.h b/arm_compute/core/utils/DataTypeUtils.h new file mode 100644 index 0000000000000000000000000000000000000000..cbb409c8a10c7ca10caf28a935fbcba9ce85936c --- /dev/null +++ b/arm_compute/core/utils/DataTypeUtils.h @@ -0,0 +1,525 @@ +/* + * Copyright (c) 2016-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_CORE_UTILS_DATATYPEUTILS_H +#define ARM_COMPUTE_CORE_UTILS_DATATYPEUTILS_H + +#include "arm_compute/core/PixelValue.h" +#include "arm_compute/core/Types.h" + +namespace arm_compute +{ +/** The size in bytes of the data type + * + * @param[in] data_type Input data type + * + * @return The size in bytes of the data type + */ +inline size_t data_size_from_type(DataType data_type) +{ + switch(data_type) + { + case DataType::U8: + case DataType::S8: + case DataType::QSYMM8: + case DataType::QASYMM8: + case DataType::QASYMM8_SIGNED: + case DataType::QSYMM8_PER_CHANNEL: + return 1; + case DataType::U16: + case DataType::S16: + case DataType::QSYMM16: + case DataType::QASYMM16: + case DataType::BFLOAT16: + case DataType::F16: + return 2; + case DataType::F32: + case DataType::U32: + case DataType::S32: + return 4; + case DataType::F64: + case DataType::U64: + case DataType::S64: + return 8; + case DataType::SIZET: + return sizeof(size_t); + default: + ARM_COMPUTE_ERROR("Invalid data type"); + return 0; + } +} + +/** The size in bytes of the data type + * + * @param[in] dt Input data type + * + * @return The size in bytes of the data type + */ +inline size_t element_size_from_data_type(DataType dt) +{ + switch(dt) + { + case DataType::S8: + case DataType::U8: + case DataType::QSYMM8: + case DataType::QASYMM8: + case DataType::QASYMM8_SIGNED: + case DataType::QSYMM8_PER_CHANNEL: + return 1; + case DataType::U16: + case DataType::S16: + case DataType::QSYMM16: + case DataType::QASYMM16: + case DataType::BFLOAT16: + case DataType::F16: + return 2; + case DataType::U32: + case DataType::S32: + case DataType::F32: + return 4; + case DataType::U64: + case DataType::S64: + return 8; + default: + ARM_COMPUTE_ERROR("Undefined element size for given data type"); + return 0; + } +} + +/** Return the data type used by a given single-planar pixel format + * + * @param[in] format Input format + * + * @return The size in bytes of the pixel format + */ +inline DataType data_type_from_format(Format format) +{ + switch(format) + { + case Format::U8: + case Format::UV88: + case Format::RGB888: + case Format::RGBA8888: + case Format::YUYV422: + case Format::UYVY422: + return DataType::U8; + case Format::U16: + return DataType::U16; + case Format::S16: + return DataType::S16; + case Format::U32: + return DataType::U32; + case Format::S32: + return DataType::S32; + case Format::BFLOAT16: + return DataType::BFLOAT16; + case Format::F16: + return DataType::F16; + case Format::F32: + return DataType::F32; + //Doesn't make sense for planar formats: + case Format::NV12: + case Format::NV21: + case Format::IYUV: + case Format::YUV444: + default: + ARM_COMPUTE_ERROR("Not supported data_type for given format"); + return DataType::UNKNOWN; + } +} + +/** Return the promoted data type of a given data type. + * + * @note If promoted data type is not supported an error will be thrown + * + * @param[in] dt Data type to get the promoted type of. + * + * @return Promoted data type + */ +inline DataType get_promoted_data_type(DataType dt) +{ + switch(dt) + { + case DataType::U8: + return DataType::U16; + case DataType::S8: + return DataType::S16; + case DataType::U16: + return DataType::U32; + case DataType::S16: + return DataType::S32; + case DataType::QSYMM8: + case DataType::QASYMM8: + case DataType::QASYMM8_SIGNED: + case DataType::QSYMM8_PER_CHANNEL: + case DataType::QSYMM16: + case DataType::QASYMM16: + case DataType::BFLOAT16: + case DataType::F16: + case DataType::U32: + case DataType::S32: + case DataType::F32: + ARM_COMPUTE_ERROR("Unsupported data type promotions!"); + default: + ARM_COMPUTE_ERROR("Undefined data type!"); + } + return DataType::UNKNOWN; +} + +/** Compute the mininum and maximum values a data type can take + * + * @param[in] dt Data type to get the min/max bounds of + * + * @return A tuple (min,max) with the minimum and maximum values respectively wrapped in PixelValue. + */ +inline std::tuple get_min_max(DataType dt) +{ + PixelValue min{}; + PixelValue max{}; + switch(dt) + { + case DataType::U8: + case DataType::QASYMM8: + { + min = PixelValue(static_cast(std::numeric_limits::lowest())); + max = PixelValue(static_cast(std::numeric_limits::max())); + break; + } + case DataType::S8: + case DataType::QSYMM8: + case DataType::QASYMM8_SIGNED: + case DataType::QSYMM8_PER_CHANNEL: + { + min = PixelValue(static_cast(std::numeric_limits::lowest())); + max = PixelValue(static_cast(std::numeric_limits::max())); + break; + } + case DataType::U16: + case DataType::QASYMM16: + { + min = PixelValue(static_cast(std::numeric_limits::lowest())); + max = PixelValue(static_cast(std::numeric_limits::max())); + break; + } + case DataType::S16: + case DataType::QSYMM16: + { + min = PixelValue(static_cast(std::numeric_limits::lowest())); + max = PixelValue(static_cast(std::numeric_limits::max())); + break; + } + case DataType::U32: + { + min = PixelValue(std::numeric_limits::lowest()); + max = PixelValue(std::numeric_limits::max()); + break; + } + case DataType::S32: + { + min = PixelValue(std::numeric_limits::lowest()); + max = PixelValue(std::numeric_limits::max()); + break; + } + case DataType::BFLOAT16: + { + min = PixelValue(bfloat16::lowest()); + max = PixelValue(bfloat16::max()); + break; + } + case DataType::F16: + { + min = PixelValue(std::numeric_limits::lowest()); + max = PixelValue(std::numeric_limits::max()); + break; + } + case DataType::F32: + { + min = PixelValue(std::numeric_limits::lowest()); + max = PixelValue(std::numeric_limits::max()); + break; + } + default: + ARM_COMPUTE_ERROR("Undefined data type!"); + } + return std::make_tuple(min, max); +} + +/** Convert a data type identity into a string. + * + * @param[in] dt @ref DataType to be translated to string. + * + * @return The string describing the data type. + */ +const std::string &string_from_data_type(DataType dt); + +/** Convert a string to DataType + * + * @param[in] name The name of the data type + * + * @return DataType + */ +DataType data_type_from_name(const std::string &name); + +/** Input Stream operator for @ref DataType + * + * @param[in] stream Stream to parse + * @param[out] data_type Output data type + * + * @return Updated stream + */ +inline ::std::istream &operator>>(::std::istream &stream, DataType &data_type) +{ + std::string value; + stream >> value; + data_type = data_type_from_name(value); + return stream; +} + +/** Check if a given data type is of floating point type + * + * @param[in] dt Input data type. + * + * @return True if data type is of floating point type, else false. + */ +inline bool is_data_type_float(DataType dt) +{ + switch(dt) + { + case DataType::F16: + case DataType::F32: + return true; + default: + return false; + } +} + +/** Check if a given data type is of quantized type + * + * @note Quantized is considered a super-set of fixed-point and asymmetric data types. + * + * @param[in] dt Input data type. + * + * @return True if data type is of quantized type, else false. + */ +inline bool is_data_type_quantized(DataType dt) +{ + switch(dt) + { + case DataType::QSYMM8: + case DataType::QASYMM8: + case DataType::QASYMM8_SIGNED: + case DataType::QSYMM8_PER_CHANNEL: + case DataType::QSYMM16: + case DataType::QASYMM16: + return true; + default: + return false; + } +} + +/** Check if a given data type is of asymmetric quantized type + * + * @param[in] dt Input data type. + * + * @return True if data type is of asymmetric quantized type, else false. + */ +inline bool is_data_type_quantized_asymmetric(DataType dt) +{ + switch(dt) + { + case DataType::QASYMM8: + case DataType::QASYMM8_SIGNED: + case DataType::QASYMM16: + return true; + default: + return false; + } +} + +/** Check if a given data type is of asymmetric quantized signed type + * + * @param[in] dt Input data type. + * + * @return True if data type is of asymmetric quantized signed type, else false. + */ +inline bool is_data_type_quantized_asymmetric_signed(DataType dt) +{ + switch(dt) + { + case DataType::QASYMM8_SIGNED: + return true; + default: + return false; + } +} + +/** Check if a given data type is of symmetric quantized type + * + * @param[in] dt Input data type. + * + * @return True if data type is of symmetric quantized type, else false. + */ +inline bool is_data_type_quantized_symmetric(DataType dt) +{ + switch(dt) + { + case DataType::QSYMM8: + case DataType::QSYMM8_PER_CHANNEL: + case DataType::QSYMM16: + return true; + default: + return false; + } +} + +/** Check if a given data type is of per channel type + * + * @param[in] dt Input data type. + * + * @return True if data type is of per channel type, else false. + */ +inline bool is_data_type_quantized_per_channel(DataType dt) +{ + switch(dt) + { + case DataType::QSYMM8_PER_CHANNEL: + return true; + default: + return false; + } +} + +/** Returns true if the value can be represented by the given data type + * + * @param[in] val value to be checked + * @param[in] dt data type that is checked + * @param[in] qinfo (Optional) quantization info if the data type is QASYMM8 + * + * @return true if the data type can hold the value. + */ +template +bool check_value_range(T val, DataType dt, QuantizationInfo qinfo = QuantizationInfo()) +{ + switch(dt) + { + case DataType::U8: + { + const auto val_u8 = static_cast(val); + return ((val_u8 == val) && val >= std::numeric_limits::lowest() && val <= std::numeric_limits::max()); + } + case DataType::QASYMM8: + { + double min = static_cast(dequantize_qasymm8(0, qinfo)); + double max = static_cast(dequantize_qasymm8(std::numeric_limits::max(), qinfo)); + return ((double)val >= min && (double)val <= max); + } + case DataType::S8: + { + const auto val_s8 = static_cast(val); + return ((val_s8 == val) && val >= std::numeric_limits::lowest() && val <= std::numeric_limits::max()); + } + case DataType::U16: + { + const auto val_u16 = static_cast(val); + return ((val_u16 == val) && val >= std::numeric_limits::lowest() && val <= std::numeric_limits::max()); + } + case DataType::S16: + { + const auto val_s16 = static_cast(val); + return ((val_s16 == val) && val >= std::numeric_limits::lowest() && val <= std::numeric_limits::max()); + } + case DataType::U32: + { + const auto val_d64 = static_cast(val); + const auto val_u32 = static_cast(val); + return ((val_u32 == val_d64) && val_d64 >= std::numeric_limits::lowest() && val_d64 <= std::numeric_limits::max()); + } + case DataType::S32: + { + const auto val_d64 = static_cast(val); + const auto val_s32 = static_cast(val); + return ((val_s32 == val_d64) && val_d64 >= std::numeric_limits::lowest() && val_d64 <= std::numeric_limits::max()); + } + case DataType::BFLOAT16: + return (val >= bfloat16::lowest() && val <= bfloat16::max()); + case DataType::F16: + return (val >= std::numeric_limits::lowest() && val <= std::numeric_limits::max()); + case DataType::F32: + return (val >= std::numeric_limits::lowest() && val <= std::numeric_limits::max()); + default: + ARM_COMPUTE_ERROR("Data type not supported"); + return false; + } +} + +/** Returns the suffix string of CPU kernel implementation names based on the given data type + * + * @param[in] data_type The data type the CPU kernel implemetation uses + * + * @return the suffix string of CPU kernel implementations + */ +inline std::string cpu_impl_dt(const DataType &data_type) +{ + std::string ret = ""; + + switch(data_type) + { + case DataType::F32: + ret = "fp32"; + break; + case DataType::F16: + ret = "fp16"; + break; + case DataType::U8: + ret = "u8"; + break; + case DataType::S16: + ret = "s16"; + break; + case DataType::S32: + ret = "s32"; + break; + case DataType::QASYMM8: + ret = "qu8"; + break; + case DataType::QASYMM8_SIGNED: + ret = "qs8"; + break; + case DataType::QSYMM16: + ret = "qs16"; + break; + case DataType::QSYMM8_PER_CHANNEL: + ret = "qp8"; + break; + case DataType::BFLOAT16: + ret = "bf16"; + break; + default: + ARM_COMPUTE_ERROR("Unsupported."); + } + + return ret; +} + +} +#endif /*ARM_COMPUTE_CORE_UTILS_DATATYPEUTILS_H */ diff --git a/arm_compute/core/utils/FormatUtils.h b/arm_compute/core/utils/FormatUtils.h new file mode 100644 index 0000000000000000000000000000000000000000..afb0f78255cf806e2ee57ecec3d473906b392529 --- /dev/null +++ b/arm_compute/core/utils/FormatUtils.h @@ -0,0 +1,344 @@ +/* + * Copyright (c) 2016-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_CORE_UTILS_FORMATUTILS_H +#define ARM_COMPUTE_CORE_UTILS_FORMATUTILS_H + +#include "arm_compute/core/CoreTypes.h" +#include "arm_compute/core/Error.h" + +namespace arm_compute +{ +/** The size in bytes of the pixel format + * + * @param[in] format Input format + * + * @return The size in bytes of the pixel format + */ +inline size_t pixel_size_from_format(Format format) +{ + switch(format) + { + case Format::U8: + return 1; + case Format::U16: + case Format::S16: + case Format::BFLOAT16: + case Format::F16: + case Format::UV88: + case Format::YUYV422: + case Format::UYVY422: + return 2; + case Format::RGB888: + return 3; + case Format::RGBA8888: + return 4; + case Format::U32: + case Format::S32: + case Format::F32: + return 4; + //Doesn't make sense for planar formats: + case Format::NV12: + case Format::NV21: + case Format::IYUV: + case Format::YUV444: + default: + ARM_COMPUTE_ERROR("Undefined pixel size for given format"); + return 0; + } +} + +/** Return the plane index of a given channel given an input format. + * + * @param[in] format Input format + * @param[in] channel Input channel + * + * @return The plane index of the specific channel of the specific format + */ +inline int plane_idx_from_channel(Format format, Channel channel) +{ + switch(format) + { + // Single planar formats have a single plane + case Format::U8: + case Format::U16: + case Format::S16: + case Format::U32: + case Format::S32: + case Format::BFLOAT16: + case Format::F16: + case Format::F32: + case Format::UV88: + case Format::RGB888: + case Format::RGBA8888: + case Format::YUYV422: + case Format::UYVY422: + return 0; + // Multi planar formats + case Format::NV12: + case Format::NV21: + { + // Channel U and V share the same plane of format UV88 + switch(channel) + { + case Channel::Y: + return 0; + case Channel::U: + case Channel::V: + return 1; + default: + ARM_COMPUTE_ERROR("Not supported channel"); + return 0; + } + } + case Format::IYUV: + case Format::YUV444: + { + switch(channel) + { + case Channel::Y: + return 0; + case Channel::U: + return 1; + case Channel::V: + return 2; + default: + ARM_COMPUTE_ERROR("Not supported channel"); + return 0; + } + } + default: + ARM_COMPUTE_ERROR("Not supported format"); + return 0; + } +} + +/** Return the channel index of a given channel given an input format. + * + * @param[in] format Input format + * @param[in] channel Input channel + * + * @return The channel index of the specific channel of the specific format + */ +inline int channel_idx_from_format(Format format, Channel channel) +{ + switch(format) + { + case Format::RGB888: + { + switch(channel) + { + case Channel::R: + return 0; + case Channel::G: + return 1; + case Channel::B: + return 2; + default: + ARM_COMPUTE_ERROR("Not supported channel"); + return 0; + } + } + case Format::RGBA8888: + { + switch(channel) + { + case Channel::R: + return 0; + case Channel::G: + return 1; + case Channel::B: + return 2; + case Channel::A: + return 3; + default: + ARM_COMPUTE_ERROR("Not supported channel"); + return 0; + } + } + case Format::YUYV422: + { + switch(channel) + { + case Channel::Y: + return 0; + case Channel::U: + return 1; + case Channel::V: + return 3; + default: + ARM_COMPUTE_ERROR("Not supported channel"); + return 0; + } + } + case Format::UYVY422: + { + switch(channel) + { + case Channel::Y: + return 1; + case Channel::U: + return 0; + case Channel::V: + return 2; + default: + ARM_COMPUTE_ERROR("Not supported channel"); + return 0; + } + } + case Format::NV12: + { + switch(channel) + { + case Channel::Y: + return 0; + case Channel::U: + return 0; + case Channel::V: + return 1; + default: + ARM_COMPUTE_ERROR("Not supported channel"); + return 0; + } + } + case Format::NV21: + { + switch(channel) + { + case Channel::Y: + return 0; + case Channel::U: + return 1; + case Channel::V: + return 0; + default: + ARM_COMPUTE_ERROR("Not supported channel"); + return 0; + } + } + case Format::YUV444: + case Format::IYUV: + { + switch(channel) + { + case Channel::Y: + return 0; + case Channel::U: + return 0; + case Channel::V: + return 0; + default: + ARM_COMPUTE_ERROR("Not supported channel"); + return 0; + } + } + default: + ARM_COMPUTE_ERROR("Not supported format"); + return 0; + } +} + +/** Return the number of planes for a given format + * + * @param[in] format Input format + * + * @return The number of planes for a given image format. + */ +inline size_t num_planes_from_format(Format format) +{ + switch(format) + { + case Format::U8: + case Format::S16: + case Format::U16: + case Format::S32: + case Format::U32: + case Format::BFLOAT16: + case Format::F16: + case Format::F32: + case Format::RGB888: + case Format::RGBA8888: + case Format::YUYV422: + case Format::UYVY422: + return 1; + case Format::NV12: + case Format::NV21: + return 2; + case Format::IYUV: + case Format::YUV444: + return 3; + default: + ARM_COMPUTE_ERROR("Not supported format"); + return 0; + } +} + +/** Return the number of channels for a given single-planar pixel format + * + * @param[in] format Input format + * + * @return The number of channels for a given image format. + */ +inline size_t num_channels_from_format(Format format) +{ + switch(format) + { + case Format::U8: + case Format::U16: + case Format::S16: + case Format::U32: + case Format::S32: + case Format::BFLOAT16: + case Format::F16: + case Format::F32: + return 1; + // Because the U and V channels are subsampled + // these formats appear like having only 2 channels: + case Format::YUYV422: + case Format::UYVY422: + return 2; + case Format::UV88: + return 2; + case Format::RGB888: + return 3; + case Format::RGBA8888: + return 4; + //Doesn't make sense for planar formats: + case Format::NV12: + case Format::NV21: + case Format::IYUV: + case Format::YUV444: + default: + return 0; + } +} + +/** Convert a tensor format into a string. + * + * @param[in] format @ref Format to be translated to string. + * + * @return The string describing the format. + */ +const std::string &string_from_format(Format format); +} +#endif /*ARM_COMPUTE_CORE_UTILS_FORMATUTILS_H */ diff --git a/arm_compute/core/utils/InterpolationPolicyUtils.h b/arm_compute/core/utils/InterpolationPolicyUtils.h new file mode 100644 index 0000000000000000000000000000000000000000..79f6e3aa5f14ac174d09ff63d1405cd006b74fea --- /dev/null +++ b/arm_compute/core/utils/InterpolationPolicyUtils.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2016-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_CORE_UTILS_INTERPOLATIONPOLICYUTILS_H +#define ARM_COMPUTE_CORE_UTILS_INTERPOLATIONPOLICYUTILS_H + +#include "arm_compute/core/Types.h" + +#include + +namespace arm_compute +{ +/** Translates a given interpolation policy to a string. + * + * @param[in] policy @ref InterpolationPolicy to be translated to string. + * + * @return The string describing the interpolation policy. + */ +const std::string &string_from_interpolation_policy(InterpolationPolicy policy); +} +#endif /*ARM_COMPUTE_CORE_UTILS_INTERPOLATIONPOLICYUTILS_H */ diff --git a/arm_compute/core/utils/StringUtils.h b/arm_compute/core/utils/StringUtils.h new file mode 100644 index 0000000000000000000000000000000000000000..41f29b0901686592ea8e817adb012d917ebda91b --- /dev/null +++ b/arm_compute/core/utils/StringUtils.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2016-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_CORE_UTILS_STRINGUTILS_H +#define ARM_COMPUTE_CORE_UTILS_STRINGUTILS_H + +#include +#include + +namespace arm_compute +{ +/** Lower a given string. + * + * @param[in] val Given string to lower. + * + * @return The lowered string + */ +std::string lower_string(const std::string &val); + +/** Raise a given string to upper case + * + * @param[in] val Given string to lower. + * + * @return The upper case string + */ +std::string upper_string(const std::string &val); + +/** Create a string with the float in full precision. + * + * @param val Floating point value + * + * @return String with the floating point value. + */ +std::string float_to_string_with_full_precision(float val); + +/** Join a sequence of strings with separator @p sep + * + * @param[in] strings Strings to join + * @param[in] sep Separator to join consecutive strings in the sequence + * + * @return std::string + */ +std::string join(const std::vector strings, const std::string &sep); +} +#endif /*ARM_COMPUTE_CORE_UTILS_STRINGUTILS_H */ diff --git a/arm_compute/core/utils/helpers/AdjustVecSize.h b/arm_compute/core/utils/helpers/AdjustVecSize.h new file mode 100644 index 0000000000000000000000000000000000000000..bbb3048b84c14fdf99427338a30ba0da2a56988a --- /dev/null +++ b/arm_compute/core/utils/helpers/AdjustVecSize.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2016-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_UTILS_ADJUSTVECSIZE_H +#define ARM_COMPUTE_UTILS_ADJUSTVECSIZE_H + +#include "arm_compute/core/Error.h" + +namespace arm_compute +{ +/** Returns the adjusted vector size in case it is less than the input's first dimension, getting rounded down to its closest valid vector size + * + * @param[in] vec_size vector size to be adjusted + * @param[in] dim0 size of the first dimension + * + * @return the number of element processed along the X axis per thread + */ +inline unsigned int adjust_vec_size(unsigned int vec_size, size_t dim0) +{ + ARM_COMPUTE_ERROR_ON(vec_size > 16); + + if((vec_size >= dim0) && (dim0 == 3)) + { + return dim0; + } + + while(vec_size > dim0) + { + vec_size >>= 1; + } + + return vec_size; +} +} +#endif /*ARM_COMPUTE_UTILS_H */ diff --git a/arm_compute/core/utils/math/Math.h b/arm_compute/core/utils/math/Math.h new file mode 100644 index 0000000000000000000000000000000000000000..c1dce7ff087a01a9abf6ad6aca2baa291956a030 --- /dev/null +++ b/arm_compute/core/utils/math/Math.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2017-2018, 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_UTILS_MATH_H +#define ARM_COMPUTE_UTILS_MATH_H + +namespace arm_compute +{ +/** Calculate the rounded up quotient of val / m. + * + * @param[in] val Value to divide and round up. + * @param[in] m Value to divide by. + * + * @return the result. + */ +template +constexpr auto DIV_CEIL(S val, T m) -> decltype((val + m - 1) / m) +{ + return (val + m - 1) / m; +} + +/** Computes the smallest number larger or equal to value that is a multiple of divisor. + * + * @param[in] value Lower bound value + * @param[in] divisor Value to compute multiple of. + * + * @return the result. + */ +template +inline auto ceil_to_multiple(S value, T divisor) -> decltype(((value + divisor - 1) / divisor) * divisor) +{ + ARM_COMPUTE_ERROR_ON(value < 0 || divisor <= 0); + return DIV_CEIL(value, divisor) * divisor; +} + +/** Computes the largest number smaller or equal to value that is a multiple of divisor. + * + * @param[in] value Upper bound value + * @param[in] divisor Value to compute multiple of. + * + * @return the result. + */ +template +inline auto floor_to_multiple(S value, T divisor) -> decltype((value / divisor) * divisor) +{ + ARM_COMPUTE_ERROR_ON(value < 0 || divisor <= 0); + return (value / divisor) * divisor; +} + +} +#endif /*ARM_COMPUTE_UTILS_MATH_H */ diff --git a/arm_compute/core/utils/misc/ShapeCalculator.h b/arm_compute/core/utils/misc/ShapeCalculator.h index f9352650b62cce52eeb328b221a51361b123af45..4c2037ab8d2be7b83735bf4a49196ef9c2f67bf9 100644 --- a/arm_compute/core/utils/misc/ShapeCalculator.h +++ b/arm_compute/core/utils/misc/ShapeCalculator.h @@ -28,6 +28,7 @@ #include "arm_compute/core/ITensorInfo.h" #include "arm_compute/core/KernelDescriptors.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/function_info/ConvolutionInfo.h" #include "arm_compute/runtime/FunctionDescriptors.h" #include "arm_compute/core/utils/helpers/tensor_transform.h" @@ -63,7 +64,7 @@ inline TensorShape calculate_reduce_mean_shape(ITensorInfo *input, const Coordin std::sort(axis_local.begin(), axis_local.begin() + reduction_ops); for(int i = 0; i < reduction_ops; ++i) { - out_shape.remove_dimension(axis_local[i] - i); + out_shape.remove_dimension(axis_local[i] - i, false); } return out_shape; } @@ -404,8 +405,8 @@ inline TensorShape compute_transposed_shape(const ITensorInfo &input) { TensorShape shape_transposed{ input.tensor_shape() }; - shape_transposed.set(0, input.dimension(1)); - shape_transposed.set(1, input.dimension(0)); + shape_transposed.set(0, input.dimension(1), false); + shape_transposed.set(1, input.dimension(0), false); return shape_transposed; } @@ -432,8 +433,8 @@ inline TensorShape compute_depthwise_convolution_shape(const ITensorInfo &input, const int weights_width_idx = get_data_layout_dimension_index(weights_data_layout, DataLayoutDimension::WIDTH); const int weights_height_idx = get_data_layout_dimension_index(weights_data_layout, DataLayoutDimension::HEIGHT); - unsigned int output_width = 0; - unsigned int output_height = 0; + unsigned int output_width = 0; + unsigned int output_height = 0; std::tie(output_width, output_height) = scaled_dimensions(input_shape[width_idx], input_shape[height_idx], weights_shape[weights_width_idx], weights_shape[weights_height_idx], info.pad_stride_info, info.dilation); @@ -683,8 +684,8 @@ inline TensorShape compute_winograd_output_transform_shape(const ITensorInfo &in const DataLayout data_layout = winograd_info.output_data_layout; // Compute output shape - unsigned int output_width = 0; - unsigned int output_height = 0; + unsigned int output_width = 0; + unsigned int output_height = 0; std::tie(output_width, output_height) = scaled_dimensions(input_dimensions.width, input_dimensions.height, kernel_size.width, kernel_size.height, conv_info); @@ -724,7 +725,7 @@ inline TensorShape compute_deep_convolution_shape(const TensorShape &input_shape const unsigned int weights_out_channel = weights_shape[3]; unsigned int output_width = 0; unsigned int output_height = 0; - std::tie(output_width, output_height) = scaled_dimensions(input_width, input_height, weights_width, weights_height, conv_info); + std::tie(output_width, output_height) = scaled_dimensions(input_width, input_height, weights_width, weights_height, conv_info); TensorShape output_shape{ input_shape }; output_shape.set(idx_width, output_width); diff --git a/arm_compute/dynamic_fusion/sketch/gpu/GpuWorkloadContext.h b/arm_compute/dynamic_fusion/sketch/gpu/GpuWorkloadContext.h index 1ee3c7e3ec25ac34fa459f644b04e71175fe32cd..0b6089973448002da8b0985ddc8867e345af43e5 100644 --- a/arm_compute/dynamic_fusion/sketch/gpu/GpuWorkloadContext.h +++ b/arm_compute/dynamic_fusion/sketch/gpu/GpuWorkloadContext.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #define ARM_COMPUTE_DYNAMIC_FUSION_SKETCH_GPU_GPUWORKLOADCONTEXT #include "arm_compute/core/GPUTarget.h" +#include "arm_compute/core/TensorInfo.h" #include @@ -56,16 +57,20 @@ enum class GpuLanguage class GpuWorkloadContext { public: + class Impl; + /** Constructor */ GpuWorkloadContext(CLCompileContext *cl_compile_context); - /** Allow instances of this class to be copy constructed */ - GpuWorkloadContext(const GpuWorkloadContext &config) = default; - /** Allow instances of this class to be copied */ - GpuWorkloadContext &operator=(const GpuWorkloadContext &config) = default; + /** Destructor */ + ~GpuWorkloadContext(); + /** Prohibit instances of this class to be copy constructed */ + GpuWorkloadContext(const GpuWorkloadContext &config) = delete; + /** Prohibit instances of this class to be copied */ + GpuWorkloadContext &operator=(const GpuWorkloadContext &config) = delete; /** Allow instances of this class to be move constructed */ - GpuWorkloadContext(GpuWorkloadContext &&config) = default; + GpuWorkloadContext(GpuWorkloadContext &&config); /** Allow instances of this class to be moved */ - GpuWorkloadContext &operator=(GpuWorkloadContext &&config) = default; + GpuWorkloadContext &operator=(GpuWorkloadContext &&config); /** Get @ref GpuLanguage of the context */ GpuLanguage gpu_language() const; /** Get @ref GpuTarget of the context */ @@ -75,9 +80,33 @@ public: */ const CLCompileContext *cl_compile_context() const; + /** Create a @ref TensorInfo associated with the workload context. + * + * @return TensorInfo Newly created tensor info + */ + template + TensorInfo create_tensor_info(TArgs &&... args) + { + auto tensor_info = TensorInfo(std::forward(args)...); + register_user_tensor(tensor_info); + return tensor_info; + } + + /** Get the internal implementation */ + Impl &implementation(); + + /** Get the internal implementation */ + const Impl &implementation() const; + private: - GpuLanguage _gpu_language{ GpuLanguage::Unknown }; - CLCompileContext *_cl_compile_ctx{ nullptr }; + /** Set a new ID to the tensor info and register its memory descriptor to the context. + * + * @param[in,out] tensor_info @ref ITensorInfo to be registered. + */ + void register_user_tensor(ITensorInfo &tensor_info); + + /** Internal implementation */ + std::unique_ptr _impl; }; } // namespace dynamic_fusion diff --git a/arm_compute/dynamic_fusion/sketch/gpu/GpuWorkloadSketch.h b/arm_compute/dynamic_fusion/sketch/gpu/GpuWorkloadSketch.h index 155df293bf3789195959066c86100cd1f49f1e6b..75c2b1f5289e8f065e7186a3a94523e78f31d8ba 100644 --- a/arm_compute/dynamic_fusion/sketch/gpu/GpuWorkloadSketch.h +++ b/arm_compute/dynamic_fusion/sketch/gpu/GpuWorkloadSketch.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,7 +24,6 @@ #ifndef ARM_COMPUTE_DYNAMIC_FUSION_SKETCH_GPU_GPUWORKLOADSKETCH #define ARM_COMPUTE_DYNAMIC_FUSION_SKETCH_GPU_GPUWORKLOADSKETCH -#include "arm_compute/core/TensorInfo.h" #include "arm_compute/dynamic_fusion/sketch/gpu/GpuWorkloadContext.h" #include @@ -62,30 +61,8 @@ public: const Implementation &implementation() const; /** Get the gpu workload context of this sketch */ const GpuWorkloadContext *gpu_context() const; - /** Create a @ref TensorInfo associated with the workload sketch. - * - * @return TensorInfo Newly created tensor info - */ - template - TensorInfo create_tensor_info(Args &&... args) - { - auto tensor_info = TensorInfo(std::forward(args)...); - register_new_tensor(tensor_info); - return tensor_info; - } - /** Create a default @ref TensorInfo associated with the workload sketch - * It is usually used by user input or output tensors - * - * @return TensorInfo Newly created tensor info - */ - TensorInfo create_tensor_info(); private: - /** Register a new tensor by setting a new id to it and register its memory descriptor in the sketch - * - * @param[in,out] tensor_info @ref ITensorInfo that will be registered - */ - void register_new_tensor(ITensorInfo &tensor_info); std::unique_ptr _impl; /**< Internal opaque implementation*/ }; diff --git a/arm_compute/function_info/ActivationLayerInfo.h b/arm_compute/function_info/ActivationLayerInfo.h new file mode 100644 index 0000000000000000000000000000000000000000..84e962cb3a229f76a33e829b9848ab51a30a22d5 --- /dev/null +++ b/arm_compute/function_info/ActivationLayerInfo.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2016-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_ARM_COMPUTE_FUNCTION_INFO_ACTIVATIONLAYERINFO +#define ACL_ARM_COMPUTE_FUNCTION_INFO_ACTIVATIONLAYERINFO + +#include "arm_compute/core/CoreTypes.h" +#include "arm_compute/core/QuantizationInfo.h" + +#include + +namespace arm_compute +{ +/** Available activation functions */ +enum class ActivationFunction +{ + LOGISTIC, /**< Logistic ( \f$ f(x) = \frac{1}{1 + e^{-x}} \f$ ) */ + TANH, /**< Hyperbolic tangent ( \f$ f(x) = a \cdot tanh(b \cdot x) \f$ ) */ + RELU, /**< Rectifier ( \f$ f(x) = max(0,x) \f$ ) */ + BOUNDED_RELU, /**< Upper Bounded Rectifier ( \f$ f(x) = min(a, max(0,x)) \f$ ) */ + LU_BOUNDED_RELU, /**< Lower and Upper Bounded Rectifier ( \f$ f(x) = min(a, max(b,x)) \f$ ) */ + LEAKY_RELU, /**< Leaky Rectifier ( \f$ f(x) = \begin{cases} \alpha x & \quad \text{if } x \text{ < 0}\\ x & \quad \text{if } x \geq \text{ 0 } \end{cases} \f$ ) */ + SOFT_RELU, /**< Soft Rectifier ( \f$ f(x)= log(1+e^x) \f$ ) */ + ELU, /**< Exponential Linear Unit ( \f$ f(x) = \begin{cases} \alpha (exp(x) - 1) & \quad \text{if } x \text{ < 0}\\ x & \quad \text{if } x \geq \text{ 0 } \end{cases} \f$ ) */ + ABS, /**< Absolute ( \f$ f(x)= |x| \f$ ) */ + SQUARE, /**< Square ( \f$ f(x)= x^2 \f$ )*/ + SQRT, /**< Square root ( \f$ f(x) = \sqrt{x} \f$ )*/ + LINEAR, /**< Linear ( \f$ f(x)= ax + b \f$ ) */ + IDENTITY, /**< Identity ( \f$ f(x)= x \f$ ) */ + HARD_SWISH, /**< Hard-swish ( \f$ f(x) = (x \text{ReLU6}(x+3))/6 = x \min(\max(0,x+3),6)/6 \f$ ) */ + SWISH, /**< Swish ( \f$ f(x) = \frac{x}{1 + e^{-ax}} = x \text{logistic}(ax) \f$ ) */ + GELU /**< GELU ( \f$ f(x) = x * 1/2 * 1 + erf(x / \sqrt{2}) \f$ ) */ +}; +/** Activation Layer Information class */ +class ActivationLayerInfo +{ +public: + typedef arm_compute::ActivationFunction ActivationFunction; + + /** Lookup table */ + using LookupTable256 = std::array; + + ActivationLayerInfo() = default; + /** Default Constructor + * + * @param[in] f The activation function to use. + * @param[in] a (Optional) The alpha parameter used by some activation functions + * (@ref ActivationFunction::BOUNDED_RELU, @ref ActivationFunction::LU_BOUNDED_RELU, @ref ActivationFunction::LINEAR, @ref ActivationFunction::TANH). + * @param[in] b (Optional) The beta parameter used by some activation functions (@ref ActivationFunction::LINEAR, @ref ActivationFunction::LU_BOUNDED_RELU, @ref ActivationFunction::TANH). + */ + ActivationLayerInfo(ActivationFunction f, float a = 0.0f, float b = 0.0f) + : _act(f), _a(a), _b(b), _enabled(true) + { + } + /** Get the type of activation function */ + ActivationFunction activation() const + { + return _act; + } + /** Get the alpha value */ + float a() const + { + return _a; + } + /** Get the beta value */ + float b() const + { + return _b; + } + /** Check if initialised */ + bool enabled() const + { + return _enabled; + } + +#ifdef __aarch64__ + const LookupTable256 &lut() const + { + return _lut; + } + void setLookupTable256(LookupTable256 &lut) + { + _lut = std::move(lut); + } +#endif // __aarch64__ +private: + ActivationFunction _act = { ActivationLayerInfo::ActivationFunction::IDENTITY }; + float _a = {}; + float _b = {}; + bool _enabled = { false }; + +#ifdef __aarch64__ + LookupTable256 _lut = {}; +#endif // __aarch64__ +}; +} // namespace arm_compute +#endif /* ACL_ARM_COMPUTE_FUNCTION_INFO_ACTIVATIONLAYERINFO */ diff --git a/arm_compute/function_info/ConvolutionInfo.h b/arm_compute/function_info/ConvolutionInfo.h new file mode 100644 index 0000000000000000000000000000000000000000..c27dc523c868ddf4abb83eb471ab12a9431e6115 --- /dev/null +++ b/arm_compute/function_info/ConvolutionInfo.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2016-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_ARM_COMPUTE_FUNCTION_INFO_CONVOLUTIONINFO +#define ACL_ARM_COMPUTE_FUNCTION_INFO_CONVOLUTIONINFO + +#include "arm_compute/core/CoreTypes.h" +#include "arm_compute/core/Size2D.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" + +namespace arm_compute +{ +struct ConvolutionInfo +{ + ConvolutionInfo() = default; + ConvolutionInfo(const PadStrideInfo &pad_stride_info, unsigned int depth_multiplier, const ActivationLayerInfo &act_info, const Size2D &dilation) + : pad_stride_info(pad_stride_info), depth_multiplier(depth_multiplier), act_info(act_info), dilation(dilation) + { + } + PadStrideInfo pad_stride_info{}; /**< Convolution info (Pads, strides,...) */ + unsigned int depth_multiplier{ 1 }; /**< Multiplier to apply to input's depth to retrieve the output depth. Defaults to 1 */ + ActivationLayerInfo act_info{}; /**< Fused activation to apply after convolution. */ + Size2D dilation{ Size2D(1, 1) }; /**< Dilation, in elements, across x and y. Defaults to (1, 1). */ +}; +} // namespace arm_compute +#endif /* ACL_ARM_COMPUTE_FUNCTION_INFO_CONVOLUTIONINFO */ diff --git a/arm_compute/function_info/FullyConnectedLayerInfo.h b/arm_compute/function_info/FullyConnectedLayerInfo.h new file mode 100644 index 0000000000000000000000000000000000000000..5f5578eaddb01e65e06fceaff2ade332db51dcda --- /dev/null +++ b/arm_compute/function_info/FullyConnectedLayerInfo.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2016-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_ARM_COMPUTE_FUNCTION_INFO_FULLYCONNECTEDLAYERINFO +#define ACL_ARM_COMPUTE_FUNCTION_INFO_FULLYCONNECTEDLAYERINFO + +#include "arm_compute/core/CoreTypes.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" + +namespace arm_compute +{ +/** Fully connected layer info */ +struct FullyConnectedLayerInfo +{ + /* Fused-activation parameters */ + ActivationLayerInfo activation_info{}; /**< Fused activation to apply after the matrix multiplication. */ + /* Information about weights */ + DataLayout weights_trained_layout{ DataLayout::NCHW }; /**< Layout that the weights have been trained with. */ + bool transpose_weights{ true }; /**< Transpose weights if true. */ + bool are_weights_reshaped{ false }; /**< @deprecated Reshape the weights tensor if false. */ + bool retain_internal_weights{ false }; /**< Retain internal reshaped weights. */ + bool enable_fast_math{ false }; /**< Enable fast math computation. */ + /* Other parameters */ + bool fp_mixed_precision{ false }; /**< Use wider accumulators (32 bit instead of 16 for FP16) to improve accuracy. */ + + /** Sets the weights trained data layout + * + * @param[in] layout Data layout that the weights were trained with + * + * @return Updated object + */ + FullyConnectedLayerInfo &set_weights_trained_layout(DataLayout layout) + { + weights_trained_layout = layout; + return *this; + } + /** Sets the transpose weights flag + * + * @param[in] should_transpose_weights Boolean flag indicating if weights should be transposed + * + * @return Updated object + */ + FullyConnectedLayerInfo &set_transpose_weights(bool should_transpose_weights) + { + transpose_weights = should_transpose_weights; + return *this; + } +}; + +} // namespace arm_compute +#endif /* ACL_ARM_COMPUTE_FUNCTION_INFO_FULLYCONNECTEDLAYERINFO */ diff --git a/arm_compute/function_info/GEMMInfo.h b/arm_compute/function_info/GEMMInfo.h new file mode 100644 index 0000000000000000000000000000000000000000..daaf86243afe190b2a50f5b69580e7ab7be82a60 --- /dev/null +++ b/arm_compute/function_info/GEMMInfo.h @@ -0,0 +1,341 @@ +/* + * Copyright (c) 2016-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_ARM_COMPUTE_FUNCTION_INFO_GEMMINFO +#define ACL_ARM_COMPUTE_FUNCTION_INFO_GEMMINFO + +#include "arm_compute/core/CoreTypes.h" +#include "arm_compute/core/experimental/IPostOp.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" +#include + +namespace arm_compute +{ +class ITensorInfo; +/** GEMMLowp output stage type */ +enum class GEMMLowpOutputStageType +{ + NONE, /**< No quantization */ + QUANTIZE_DOWN, /**< Quantize using an integer multiplication */ + QUANTIZE_DOWN_FIXEDPOINT, /**< Quantize using a fixed point multiplication */ + QUANTIZE_DOWN_FLOAT /**< Quantize using a floating point multiplication */ +}; + +/** GEMMLowp output stage info */ +struct GEMMLowpOutputStageInfo +{ + GEMMLowpOutputStageType type{ GEMMLowpOutputStageType::NONE }; /**< GEMMLowp output stage type */ + int32_t gemmlowp_offset{ 0 }; /**< GEMMLowp output stage offset used for quantizing to QASYMM8 */ + int32_t gemmlowp_multiplier{ 0 }; /**< GEMMLowp output stage multiplier used for quantizing to QASYMM8 */ + int32_t gemmlowp_shift{ 0 }; /**< GEMMLowp output stage shift used for quantizing to uint8 */ + int32_t gemmlowp_min_bound{ std::numeric_limits::lowest() }; /**< GEMMLowp min value used to saturate down the output result before converting back to QASYMM8 */ + int32_t gemmlowp_max_bound{ std::numeric_limits::max() }; /**< GEMMLowp max value used to saturate down the output result before converting back to QASYMM8 */ + std::vector gemmlowp_multipliers{}; /**< GEMMLowp output stage multiplier used for quantizing to QASYMM8 */ + std::vector gemmlowp_shifts{}; /**< GEMMLowp output stage multiplier used for quantizing to QASYMM8 */ + float gemmlowp_real_multiplier{ 0 }; /**< GEMMLowp output stage real multiplier used for quantizing to QASYMM8 */ + bool is_quantized_per_channel{ false }; /**< GEMMLowp quantized per-channel flag */ + DataType output_data_type{ DataType::UNKNOWN }; /**< Output tensor data type to use if the output is not initialized */ +}; +/** GEMM information class. This class stores the necessary information to compute GEMM functions + * + * This object also contains the information about how matrix A and matrix B have been reshaped + * + */ +class GEMMInfo +{ +public: + /** Default constructor */ + GEMMInfo() noexcept + : _is_a_reshaped(false), + _is_b_reshaped(false), + _reshape_b_only_on_first_run(true), + _depth_output_gemm3d(0), + _reinterpret_input_as_3d(false), + _retain_internal_weights(false), + _gemmlowp_output_stage(), + _fast_math(false), + _fp_mixed_precision(false), + _broadcast_bias(false), + _pretranspose_A(false), + _pretranspose_B(false), + _activation_info(), + _post_ops(), + _fixed_format(false), + _weight_format(arm_compute::WeightFormat::UNSPECIFIED) + { + } + /** Constructor + * + * @param[in] is_a_reshaped True if the matrix A has been reshaped + * @param[in] is_b_reshaped True if the matrix B has been reshaped + * @param[in] reshape_b_only_on_first_run Reshape matrix B only for the first run + * @param[in] depth_output_gemm3d (Optional) Depth (third dimension) of the output tensor to be used with the GEMM3D kernel + * If 0 the output will not be reinterpreted as 3D. Default 0 + * @param[in] reinterpret_input_as_3d (Optional) Reinterpret the input as 3D tensor. (i.e. this flag should be set to true when GEMM is used + * to perform 1x1 convolutions with the NHWC data layout) + * @param[in] retain_internal_weights (Optional) Retain the weights tensor from previous run + * @param[in] gemmlowp_output_stage (Optional) GEMMLowp Output stage info + * @param[in] fp_mixed_precision (Optional) Use wider accumulators (32 bit instead of 16 for FP16) to improve accuracy. + * @param[in] fast_math (Optional) Use a data type of shorter width to improve performance + * @param[in] broadcast_bias (Optional) Broadcast the shape of the bias tensor from a vector to a matrix. + * @param[in] activation_info (Optional) Activation to apply after the matrix multiplication + * @param[in] post_ops (Optional) A sequence of post operations that are performed after the main operation. + * @param[in] fixed_format (Optional) Specify the selection of fixed format kernels for variable weights support in GEMM. These kernels expect the weights tensor to be in amemory format that is fixed by the kernel itself. For more information, see arm_compute::WeightFormat. + * @param[in] weight_format (Optional) arm_gemm:WeightFormat enumeration requested by the user. Default is arm_compute::WeightFormat::UNSPECIFIED. + */ + GEMMInfo(bool is_a_reshaped, bool is_b_reshaped, bool reshape_b_only_on_first_run, int depth_output_gemm3d = 0, bool reinterpret_input_as_3d = false, bool retain_internal_weights = false, + GEMMLowpOutputStageInfo gemmlowp_output_stage = GEMMLowpOutputStageInfo(), bool fp_mixed_precision = false, bool fast_math = false, bool broadcast_bias = false, + const ActivationLayerInfo &activation_info = ActivationLayerInfo(), const experimental::PostOpList &post_ops = experimental::PostOpList(), + bool fixed_format = false, arm_compute::WeightFormat weight_format = arm_compute::WeightFormat::UNSPECIFIED) noexcept + : _is_a_reshaped(is_a_reshaped), + _is_b_reshaped(is_b_reshaped), + _reshape_b_only_on_first_run(reshape_b_only_on_first_run), + _depth_output_gemm3d(depth_output_gemm3d), + _reinterpret_input_as_3d(reinterpret_input_as_3d), + _retain_internal_weights(retain_internal_weights), + _gemmlowp_output_stage(gemmlowp_output_stage), + _fast_math(fast_math), + _fp_mixed_precision(fp_mixed_precision), + _broadcast_bias(broadcast_bias), + _pretranspose_A(false), + _pretranspose_B(false), + _activation_info(activation_info), + _post_ops(post_ops), + _fixed_format(fixed_format), + _weight_format(weight_format) + { + } + /** Flag which specifies if the matrix A has been reshaped + * + * @return True if the matrix A has been reshaped + */ + bool is_a_reshaped() const + { + return _is_a_reshaped; + }; + /** Flag which specifies if the matrix B has been reshaped + * + * @return True if the matrix B has been reshaped + */ + bool is_b_reshaped() const + { + return _is_b_reshaped; + }; + /** Flag which specifies if the reshape of matrix B should executed only for the first + * + * @note This flag could be set to TRUE when GEMM is used to accelerate convolution layer + * + * @return True if the reshaped of matrix B happens only for the first run + */ + bool reshape_b_only_on_first_run() const + { + return _reshape_b_only_on_first_run; + }; + /** Depth of the output when GEMM output is reinterpreted as 3D tensor + * + * @return the depth of the output tensor + */ + int depth_output_gemm3d() const + { + return _depth_output_gemm3d; + }; + /** Flag which specifies if the input tensor has to be reinterpreted as 3D + * + * @return True if the input tensor has to be reinterpreted as 3D tensor + */ + bool reinterpret_input_as_3d() const + { + return _reinterpret_input_as_3d; + }; + /** Flag which specifies if the weights tensor has to be retained from previous run + * + * @return True if the weights tensor has to be retained + */ + bool retain_internal_weights() const + { + return _retain_internal_weights; + }; + /** GEMMLowp output stage + * + * @return the GEMMLowp output stage info + */ + GEMMLowpOutputStageInfo gemmlowp_output_stage() const + { + return _gemmlowp_output_stage; + }; + /** Sets GEMMLowp output stage + * + * @param[in] output_stage Output stage to set + */ + void set_gemmlowp_output_stage(GEMMLowpOutputStageInfo &output_stage) + { + _gemmlowp_output_stage = output_stage; + }; + /** Flag which specifies if a wider accumulator should be used. + * + * @return True if a wider accumulator has to be used + */ + bool fp_mixed_precision() const + { + return _fp_mixed_precision; + }; + /** Flag which specifies if a shorter accumulator to be used. + * + * @return True if a shorter accumulator has to be used + */ + bool fast_math() const + { + return _fast_math; + }; + /** Set fast math flag + * + * @param[in] fast_math Flag to set + */ + void set_fast_math(bool fast_math) + { + _fast_math = fast_math; + } + /** Flag which specifies whether to broadcast the shape of the bias tensor. + * + * @return True if the shape of the bias tensor is to be broadcasted. + */ + bool broadcast_bias() const + { + return _broadcast_bias; + }; + /** Flag which specifies whether A should be pre-transposed if supported. + * + * @return True if A should be pre-transposed else false. + */ + bool pretranspose_A() const + { + return _pretranspose_A; + }; + /** Set pre-transpose A flag + * + * @param[in] flag Flag to set + */ + void set_pretranspose_A(bool flag) + { + _pretranspose_A = flag; + } + /** Flag which specifies whether b should be pre-transposed if supported. + * + * @return True if b should be pre-transposed else false. + */ + bool pretranspose_B() const + { + return _pretranspose_B; + }; + /** Set pre-transpose b flag + * + * @param[in] flag Flag to set + */ + void set_pretranspose_B(bool flag) + { + _pretranspose_B = flag; + } + /** Activation layer to apply after the matrix multiplication + * + * @return ActivationLayerInfo object + */ + ActivationLayerInfo activation_info() const + { + return _activation_info; + } + /** Set activation layer info + * + * @param[in] activation_info ActivationLayerInfo object to set + */ + void set_activation_info(const ActivationLayerInfo &activation_info) + { + _activation_info = activation_info; + } + /** Post operations to apply after the matrix multiplication + * + * @return experimental::PostOpList object + */ + const experimental::PostOpList &post_ops() const + { + return _post_ops; + } + /** Set post ops + * + * @param[in] post_ops experimental::PostOpList object to set + */ + void set_post_ops(const experimental::PostOpList &post_ops) + { + _post_ops = post_ops; + } + /** Flag which specifies if the GEMM operation is running fixed-format kernels. + * + * @return True if the GEMM operation is running fixed-format kernel else false. + */ + bool fixed_format() const + { + return _fixed_format; + } + + /** Set fixed-format flag + * + * @param[in] fixed_format sets whether or not to use fixed-format kernels + */ + void set_fixed_format(bool fixed_format) + { + _fixed_format = fixed_format; + } + + arm_compute::WeightFormat weight_format() const + { + return _weight_format; + } + + /** Set weight format to be used + * + * @param[in] weight_format arm_compute::WeightFormat enumeration + */ + void set_weight_format(arm_compute::WeightFormat weight_format) + { + _weight_format = weight_format; + } + +private: + bool _is_a_reshaped; + bool _is_b_reshaped; + bool _reshape_b_only_on_first_run; + int _depth_output_gemm3d; + bool _reinterpret_input_as_3d; + bool _retain_internal_weights; + GEMMLowpOutputStageInfo _gemmlowp_output_stage; + bool _fast_math; + bool _fp_mixed_precision; + bool _broadcast_bias; + bool _pretranspose_A; + bool _pretranspose_B; + ActivationLayerInfo _activation_info; + experimental::PostOpList _post_ops; + bool _fixed_format; + arm_compute::WeightFormat _weight_format; +}; +} //namespace arm_compute +#endif /* ACL_ARM_COMPUTE_FUNCTION_INFO_GEMMINFO */ diff --git a/arm_compute/function_info/MatMulInfo.h b/arm_compute/function_info/MatMulInfo.h new file mode 100644 index 0000000000000000000000000000000000000000..cd9ef1f4d957766c9c9a87450a27403210b0ef56 --- /dev/null +++ b/arm_compute/function_info/MatMulInfo.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2016-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_ARM_COMPUTE_FUNCTION_INFO_MATMULINFO +#define ACL_ARM_COMPUTE_FUNCTION_INFO_MATMULINFO + +namespace arm_compute +{ +/** Class for holding information related to matrix multiplication function + */ +class MatMulInfo +{ +public: + /* Get Adjoint LHS flag value */ + bool adj_lhs() const + { + return _adj_lhs; + } + /* Get Adjoint RHS flag value */ + bool adj_rhs() const + { + return _adj_rhs; + } + /* Set Adjoint LHS flag */ + MatMulInfo &adj_lhs(bool adj_lhs) + { + _adj_lhs = adj_lhs; + return *this; + } + /* Set Adjoint RHS flag */ + MatMulInfo &adj_rhs(bool adj_rhs) + { + _adj_rhs = adj_rhs; + return *this; + } + +private: + bool _adj_lhs{ false }; + bool _adj_rhs{ false }; +}; +} // namespace arm_compute +#endif /* ACL_ARM_COMPUTE_FUNCTION_INFO_MATMULINFO */ diff --git a/arm_compute/graph/Graph.h b/arm_compute/graph/Graph.h index d8d3feb1f7f657c6e9a0a09eab6abebc4e1208e1..806d84c3fdb9daa4a50abc272a3928546ea4ee64 100644 --- a/arm_compute/graph/Graph.h +++ b/arm_compute/graph/Graph.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2020 Arm Limited. + * Copyright (c) 2018-2020,2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -30,7 +30,6 @@ #include "arm_compute/graph/Types.h" #include "support/Mutex.h" -#include "support/ToolchainSupport.h" #include #include diff --git a/arm_compute/graph/Types.h b/arm_compute/graph/Types.h index ff33d5037b624e1db530aa1a617dfaa0a977d81a..167f7388d4e132572ca8579965b38c192c9abe66 100644 --- a/arm_compute/graph/Types.h +++ b/arm_compute/graph/Types.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,6 +27,10 @@ #include "arm_compute/core/Error.h" #include "arm_compute/core/PixelValue.h" #include "arm_compute/core/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" +#include "arm_compute/function_info/ConvolutionInfo.h" +#include "arm_compute/function_info/FullyConnectedLayerInfo.h" +#include "arm_compute/function_info/GEMMInfo.h" #include "arm_compute/runtime/CL/CLTunerTypes.h" #include "arm_compute/runtime/CL/CLTypes.h" diff --git a/arm_compute/runtime/CL/CLMemoryRegion.h b/arm_compute/runtime/CL/CLMemoryRegion.h index 1fd8fdb79ec7982b988a9e9c47a64feda1b78bd9..66a30fa56bb96f8c8de54cac617483f755184c62 100644 --- a/arm_compute/runtime/CL/CLMemoryRegion.h +++ b/arm_compute/runtime/CL/CLMemoryRegion.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -85,10 +85,9 @@ public: std::unique_ptr extract_subregion(size_t offset, size_t size) override; protected: - cl::CommandQueue _queue; - cl::Context _ctx; - void *_mapping; - cl::Buffer _mem; + cl::Context _ctx; + void *_mapping; + cl::Buffer _mem; }; /** OpenCL buffer memory region implementation */ @@ -106,6 +105,7 @@ public: * @param[in] buffer Buffer to be used as a memory region */ CLBufferMemoryRegion(const cl::Buffer &buffer); + virtual ~CLBufferMemoryRegion() override; // Inherited methods overridden : void *ptr() final; diff --git a/arm_compute/runtime/CL/functions/CLActivationLayer.h b/arm_compute/runtime/CL/functions/CLActivationLayer.h index e6bb192532151b917f4b922d9a928bb4cd174241..4a718ab4b6c2162fdedbc0e74a9baf3bd59483ca 100644 --- a/arm_compute/runtime/CL/functions/CLActivationLayer.h +++ b/arm_compute/runtime/CL/functions/CLActivationLayer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021 Arm Limited. + * Copyright (c) 2016-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,6 +27,7 @@ #include "arm_compute/runtime/IFunction.h" #include "arm_compute/core/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "arm_compute/runtime/CL/CLRuntimeContext.h" namespace arm_compute diff --git a/arm_compute/runtime/CL/functions/CLArgMinMaxLayer.h b/arm_compute/runtime/CL/functions/CLArgMinMaxLayer.h index a971163c45d2aba7d032777f350e62cd8f5656ce..ce5bee8d9528893741289433af77d97fd5b75788 100644 --- a/arm_compute/runtime/CL/functions/CLArgMinMaxLayer.h +++ b/arm_compute/runtime/CL/functions/CLArgMinMaxLayer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -107,13 +107,11 @@ public: void run() override; private: - MemoryGroup _memory_group; - std::vector _results_vector; - CLTensor _not_reshaped_output; - std::vector> _reduction_kernels_vector; - CLReshapeLayer _reshape; - unsigned int _num_of_stages; - unsigned int _reduction_axis; + MemoryGroup _memory_group; + CLTensor _not_reshaped_output; + std::unique_ptr _arg_min_max_kernel; + CLReshapeLayer _reshape; + unsigned int _reduction_axis; }; } // namespace arm_compute #endif /* ARM_COMPUTE_CLARGMINMAXLAYER_H */ diff --git a/arm_compute/runtime/CL/functions/CLBatchNormalizationLayer.h b/arm_compute/runtime/CL/functions/CLBatchNormalizationLayer.h index fcfeb5ea3b3eefca1bc714fec4d6d21e72f413c8..37a0680709984012f4672f0028572c67fb0c3118 100644 --- a/arm_compute/runtime/CL/functions/CLBatchNormalizationLayer.h +++ b/arm_compute/runtime/CL/functions/CLBatchNormalizationLayer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,6 +27,7 @@ #include "arm_compute/runtime/IFunction.h" #include "arm_compute/core/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include diff --git a/arm_compute/runtime/CL/functions/CLCast.h b/arm_compute/runtime/CL/functions/CLCast.h index d2cea7a8a2828df835a77cf1393099a23ec56e5a..650cd11b9b9db2abbd9a020f474a0f22e11fb06a 100644 --- a/arm_compute/runtime/CL/functions/CLCast.h +++ b/arm_compute/runtime/CL/functions/CLCast.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -61,43 +61,28 @@ public: * |src |dst | * |:--------------|:--------------------------------------| * |U8 | S8, U16, S16, U32, S32, F16, F32 | + * |S8 | U8, U16, S16, U32, S32, F16, F32 | * |U16 | U8, S8, S16, U32, S32, F16, F32 | * |S16 | U8, S8, U16, U32, S32, F16, F32 | * |U32 | U8, S8, U16, S16, S32, F16, F32 | * |S32 | U8, S8, U16, S16, U32, F16, F32 | - * |F16 | U8, S8, U16, S16, U32, F32 | - * |F32 | U8, S8, U16, S16, U32, F16 | + * |U64 | U8, S8, U16, S16, U32, S32, F16, F32 | + * |S64 | U8, S8, U16, S16, U32, S32, F16, F32 | + * |F16 | U8, S8, U16, S16, S32, U32, F32 | + * |F32 | U8, S8, U16, S16, S32, U32, F16 | * * Input data type must be different than output data type. * - * @param[in] input The input tensor to convert. Data types supported: U8/S8/U16/S16/U32/S32/F16/F32. + * @param[in] input The input tensor to convert. Data types supported: U8/S8/U16/S16/U32/S32/U64/S64/F16/F32. * @param[out] output The output tensor. Data types supported: U8/S8/U16/S16/U32/S32/F16/F32. * @param[in] policy Conversion policy. */ void configure(const ICLTensor *input, ICLTensor *output, ConvertPolicy policy); - /** Initialize the function's source, destination - * - * Input data type must be different than output data type. - * - * Valid conversions Input -> Output : - * - * - U8 -> S8, U16, S16, U32, S32, F16, F32 - * - U16 -> U8, S8, S16, U32, S32, F16, F32 - * - S16 -> U8, S8, U16, U32, S32, F16, F32 - * - U32 -> U8, S8, U16, S16, S32, F16, F32 - * - S32 -> U8, S8, U16, S16, U32, F16, F32 - * - F16 -> U8, S8, U16, S16, U32, F32 - * - F32 -> U8, S8, U16, S16, U32, F16 - * - * @param[in] compile_context The compile context to be used. - * @param[in] input The input tensor to convert. Data types supported: U8/S8/U16/S16/U32/S32/F16/F32. - * @param[out] output The output tensor. Data types supported: U8/S8/U16/S16/U32/S32/F16/F32. - * @param[in] policy Conversion policy. - */ + // Initialize the function's source, destination void configure(const CLCompileContext &compile_context, const ICLTensor *input, ICLTensor *output, ConvertPolicy policy); /** Static function to check if given info will lead to a valid configuration of @ref CLCast * - * @param[in] input Source tensor info. Data types supported: U8/S8/U16/S16/U32/S32/F16/F32. + * @param[in] input Source tensor info. Data types supported: U8/S8/U16/S16/U32/S32/U64/S64/F16/F32. * @param[in] output Destination tensor info. Data type supported: U8/S8/U16/S16/U32/S32/F16/F32. * @param[in] policy Conversion policy. * diff --git a/arm_compute/runtime/CL/functions/CLConvolutionLayer.h b/arm_compute/runtime/CL/functions/CLConvolutionLayer.h index 38a4019609ed1c98d49c3fe94c0164bc54646a71..8c9e45d753dbbf508a7d127f156e5d0f398d4188 100644 --- a/arm_compute/runtime/CL/functions/CLConvolutionLayer.h +++ b/arm_compute/runtime/CL/functions/CLConvolutionLayer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,6 +27,7 @@ #include "arm_compute/core/CL/CLCompileContext.h" #include "arm_compute/core/Types.h" #include "arm_compute/core/experimental/IPostOp.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "arm_compute/runtime/CL/CLTensor.h" #include "arm_compute/runtime/IFunction.h" #include "arm_compute/runtime/IMemoryManager.h" diff --git a/arm_compute/runtime/CL/functions/CLDepthwiseConvolutionLayer.h b/arm_compute/runtime/CL/functions/CLDepthwiseConvolutionLayer.h index 01ddae12bbff5d9229b83ec681a63647ec8cf78a..27984491005d044cdd43dd6b8fa6eade8aa235c5 100644 --- a/arm_compute/runtime/CL/functions/CLDepthwiseConvolutionLayer.h +++ b/arm_compute/runtime/CL/functions/CLDepthwiseConvolutionLayer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #define ARM_COMPUTE_CLDEPTHWISECONVOLUTION_H #include "arm_compute/core/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "arm_compute/runtime/CL/CLTensor.h" #include "arm_compute/runtime/CL/functions/CLPermute.h" #include "arm_compute/runtime/IFunction.h" diff --git a/arm_compute/runtime/CL/functions/CLDirectConvolutionLayer.h b/arm_compute/runtime/CL/functions/CLDirectConvolutionLayer.h index 3fd0c637821aa4249b806c2a25deb7e9f857fb55..462a3ac07e6f6418fa67ac15274508d4dddd4742 100644 --- a/arm_compute/runtime/CL/functions/CLDirectConvolutionLayer.h +++ b/arm_compute/runtime/CL/functions/CLDirectConvolutionLayer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #define ARM_COMPUTE_CLDIRECTCONVOLUTIONLAYER_H #include "arm_compute/core/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "arm_compute/runtime/CL/functions/CLActivationLayer.h" #include "arm_compute/runtime/IFunction.h" diff --git a/arm_compute/runtime/CL/functions/CLElementwiseOperations.h b/arm_compute/runtime/CL/functions/CLElementwiseOperations.h index 555e84a2510ac66799e662b13ef9a10ba4371744..9de362d2b2673bd5a28378bb5b3a459c549e38d2 100644 --- a/arm_compute/runtime/CL/functions/CLElementwiseOperations.h +++ b/arm_compute/runtime/CL/functions/CLElementwiseOperations.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CLELEMENTWISEOPERATIONS_H #define ARM_COMPUTE_CLELEMENTWISEOPERATIONS_H +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "arm_compute/runtime/CL/ICLOperator.h" #include "arm_compute/runtime/IFunction.h" diff --git a/arm_compute/runtime/CL/functions/CLFill.h b/arm_compute/runtime/CL/functions/CLFill.h index a01e0c3188ebc67adceffba931e879b9d2744381..341d93a9f6b58ac57286ca5ed734e13d84a52ceb 100644 --- a/arm_compute/runtime/CL/functions/CLFill.h +++ b/arm_compute/runtime/CL/functions/CLFill.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CLFILL_H #define ARM_COMPUTE_CLFILL_H +#include "arm_compute/core/PixelValue.h" #include "arm_compute/core/Types.h" #include "arm_compute/core/Window.h" #include "arm_compute/runtime/IFunction.h" diff --git a/arm_compute/runtime/CL/functions/CLFullyConnectedLayer.h b/arm_compute/runtime/CL/functions/CLFullyConnectedLayer.h index 2947b4890cce543f309e7e2a99e6297eaa49ab74..b784226a2f83e55c3602c9f3f1017f9302d5cdfa 100644 --- a/arm_compute/runtime/CL/functions/CLFullyConnectedLayer.h +++ b/arm_compute/runtime/CL/functions/CLFullyConnectedLayer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CLFULLYCONNECTEDLAYER_H #define ARM_COMPUTE_CLFULLYCONNECTEDLAYER_H +#include "arm_compute/function_info/FullyConnectedLayerInfo.h" #include "arm_compute/runtime/IFunction.h" #include "arm_compute/runtime/CL/CLTensor.h" diff --git a/arm_compute/runtime/CL/functions/CLGEMM.h b/arm_compute/runtime/CL/functions/CLGEMM.h index b267bf1cf2230abbab14cc4c3cbe039facc6eeaf..3a39aca6924b33738499e411ef0d5128af559110 100644 --- a/arm_compute/runtime/CL/functions/CLGEMM.h +++ b/arm_compute/runtime/CL/functions/CLGEMM.h @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CLGEMM_H #define ARM_COMPUTE_CLGEMM_H +#include "arm_compute/function_info/GEMMInfo.h" #include "arm_compute/runtime/CL/CLTensor.h" #include "arm_compute/runtime/CL/CLTypes.h" #include "arm_compute/runtime/IFunction.h" diff --git a/arm_compute/runtime/CL/functions/CLGEMMConvolutionLayer.h b/arm_compute/runtime/CL/functions/CLGEMMConvolutionLayer.h index 9918a61cab268835e8743b8e26f9bcd6418341b7..98273403828c63c26d16b713cc7e3bc63ec58701 100644 --- a/arm_compute/runtime/CL/functions/CLGEMMConvolutionLayer.h +++ b/arm_compute/runtime/CL/functions/CLGEMMConvolutionLayer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #define ARM_COMPUTE_CLGEMMCONVOLUTIONLAYER_H #include "arm_compute/core/experimental/IPostOp.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "arm_compute/runtime/CL/CLTensor.h" #include "arm_compute/runtime/CL/CLTypes.h" #include "arm_compute/runtime/IFunction.h" diff --git a/arm_compute/runtime/CL/functions/CLGEMMLowpMatrixMultiplyCore.h b/arm_compute/runtime/CL/functions/CLGEMMLowpMatrixMultiplyCore.h index a8ee9e5b560c62c18e9c5684024929a3c022c570..8b8d9f235f6482e621702b4b9ce815fa0f867747 100644 --- a/arm_compute/runtime/CL/functions/CLGEMMLowpMatrixMultiplyCore.h +++ b/arm_compute/runtime/CL/functions/CLGEMMLowpMatrixMultiplyCore.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CLGEMMLOWPMATRIXMULTIPLYCORE_H #define ARM_COMPUTE_CLGEMMLOWPMATRIXMULTIPLYCORE_H +#include "arm_compute/function_info/GEMMInfo.h" #include "arm_compute/runtime/CL/CLTensor.h" #include "arm_compute/runtime/IFunction.h" #include "arm_compute/runtime/MemoryGroup.h" @@ -132,4 +133,4 @@ private: std::unique_ptr _impl; }; } // namespace arm_compute -#endif /*ARM_COMPUTE_CLGEMMLOWPMATRIXMULTIPLYCORE_H */ \ No newline at end of file +#endif /*ARM_COMPUTE_CLGEMMLOWPMATRIXMULTIPLYCORE_H */ diff --git a/arm_compute/runtime/CL/functions/CLIndirectConvolutionLayer.h b/arm_compute/runtime/CL/functions/CLIndirectConvolutionLayer.h index 8185f8df78360dcd217afb58e4f96398006a26e8..12b83ea25b0c4bdb7e21d831b1e9ddb68e74c427 100644 --- a/arm_compute/runtime/CL/functions/CLIndirectConvolutionLayer.h +++ b/arm_compute/runtime/CL/functions/CLIndirectConvolutionLayer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #define ARM_COMPUTE_CLINDIRECTCONVOLUTIONLAYER_H #include "arm_compute/core/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "arm_compute/runtime/IFunction.h" #include diff --git a/arm_compute/runtime/CL/functions/CLMatMul.h b/arm_compute/runtime/CL/functions/CLMatMul.h index 2af9a4a9a690f420be98d68e0aebee9a77bbdbf3..9d54bab8683d44f7afd865634e9b822488d7f835 100644 --- a/arm_compute/runtime/CL/functions/CLMatMul.h +++ b/arm_compute/runtime/CL/functions/CLMatMul.h @@ -24,6 +24,8 @@ #ifndef ACL_ARM_COMPUTE_RUNTIME_CL_FUNCTIONS_CLMATMUL #define ACL_ARM_COMPUTE_RUNTIME_CL_FUNCTIONS_CLMATMUL +#include "arm_compute/core/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "arm_compute/runtime/IFunction.h" #include @@ -83,21 +85,29 @@ public: * @param[in] rhs Right-hand side tensor info containing the input weights as Matrix B. Data types supported: same as @p lhs. * @param[out] dst Output tensor to store the result of the batched matrix multiplication. Data types supported: same as @p lhs. * @param[in] matmul_info Contains MatMul operation information described in @ref MatMulInfo. - * @param[in] settings Class containing flags for function level settings + * @param[in] settings Contains flags for function level settings + * @param[in] act_info (Optional) Contains activation function and lower and upper bound values for bounded activation functions. */ - void configure(const CLCompileContext &compile_context, ICLTensor *rhs, ICLTensor *lhs, ICLTensor *dst, const MatMulInfo &matmul_info, const GpuMatMulSettings &settings = GpuMatMulSettings{}); + void configure(const CLCompileContext &compile_context, ICLTensor *rhs, ICLTensor *lhs, ICLTensor *dst, const MatMulInfo &matmul_info, const GpuMatMulSettings &settings = GpuMatMulSettings{}, const + ActivationLayerInfo &act_info = ActivationLayerInfo{}); /** Initialise the kernel's inputs and output * * Similar to @ref CLMatMul::configure() */ - void configure(ICLTensor *lhs, ICLTensor *rhs, ICLTensor *dst, const MatMulInfo &matmul_info, const GpuMatMulSettings &settings = GpuMatMulSettings{}); + void configure(ICLTensor *lhs, ICLTensor *rhs, ICLTensor *dst, const MatMulInfo &matmul_info, const GpuMatMulSettings &settings = GpuMatMulSettings{}, const ActivationLayerInfo &act_info = + ActivationLayerInfo{}); /** Static function to check if given info will lead to a valid configuration of @ref CLMatMul. * - * Similar to @ref CLMatMul::configure() * - * @return a status + * @note All tensors must have the same data type. + * + * @param[in] lhs Left-hand side (Matrix A) tensor info. Data types supported: F16/F32/QASYMM8_SIGNED/QASYMM8. + * @param[in] rhs Right-hand side (Matrix B) tensor info. Data types supported: same as @p lhs. + * @param[out] output Output tensor info to store the result of the batched matrix multiplication. Data types supported: same as @p lhs. + * @param[in] matmul_info Contains MatMul operation information described in @ref MatMulInfo. + * @param[in] act_info (Optional) Contains activation function and lower and upper bound values for bounded activation functions. */ - static Status validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *output, const MatMulInfo &matmul_info); + static Status validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *output, const MatMulInfo &matmul_info, const ActivationLayerInfo &act_info = ActivationLayerInfo{}); // Inherited methods overridden: void run() override; diff --git a/arm_compute/runtime/CL/functions/CLPixelWiseMultiplication.h b/arm_compute/runtime/CL/functions/CLPixelWiseMultiplication.h index d352c6e2828ca06560d13cca6f383580b178bb42..62b6d96ad530a6bd8cd9660d91a6ad83b953aa4e 100644 --- a/arm_compute/runtime/CL/functions/CLPixelWiseMultiplication.h +++ b/arm_compute/runtime/CL/functions/CLPixelWiseMultiplication.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021 Arm Limited. + * Copyright (c) 2016-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,8 @@ #ifndef ARM_COMPUTE_CLPIXELWISEMULTIPLICATION_H #define ARM_COMPUTE_CLPIXELWISEMULTIPLICATION_H +#include "arm_compute/core/Rounding.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "arm_compute/runtime/CL/ICLOperator.h" #include "arm_compute/runtime/IFunction.h" diff --git a/arm_compute/runtime/CL/functions/CLWinogradConvolutionLayer.h b/arm_compute/runtime/CL/functions/CLWinogradConvolutionLayer.h index 4b351267e31ef742daf0b348abb532a2d1f9b5af..adf5f18626d8e4327c050930a70f7c2415fd8ee1 100644 --- a/arm_compute/runtime/CL/functions/CLWinogradConvolutionLayer.h +++ b/arm_compute/runtime/CL/functions/CLWinogradConvolutionLayer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #define ARM_COMPUTE_CLWINOGRADCONVOLUTIONLAYER_H #include "arm_compute/core/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "arm_compute/runtime/IFunction.h" #include "arm_compute/runtime/IMemoryManager.h" diff --git a/arm_compute/runtime/CPP/functions/CPPSplit.h b/arm_compute/runtime/CPP/functions/CPPSplit.h index e2e5add2645e70a0ca2adbf673366ed0bbe5c27f..56aad2db4b1792a40fb0ce4dfa7e60a9a57b98cb 100644 --- a/arm_compute/runtime/CPP/functions/CPPSplit.h +++ b/arm_compute/runtime/CPP/functions/CPPSplit.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021 Arm Limited. + * Copyright (c) 2020-2021,2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -30,8 +30,6 @@ #include "arm_compute/core/Types.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" -#include "support/ToolchainSupport.h" - #include "arm_compute/runtime/IFunction.h" namespace arm_compute diff --git a/arm_compute/runtime/FunctionDescriptors.h b/arm_compute/runtime/FunctionDescriptors.h index af79820bc3a0302653baaa5d2be14295fe93fd8f..630f53324403c41686b72ab4e681b9835b2f60e6 100644 --- a/arm_compute/runtime/FunctionDescriptors.h +++ b/arm_compute/runtime/FunctionDescriptors.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022 Arm Limited. + * Copyright (c) 2019-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #define ARM_COMPUTE_RUNTIME_FUNCTION_DESCRIPTORS_H #include "arm_compute/core/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include diff --git a/arm_compute/runtime/NEON/functions/NEActivationLayer.h b/arm_compute/runtime/NEON/functions/NEActivationLayer.h index b39a8d7701319c11873c4ab7bd03e3b6d2c10816..9992de2af80deee16754c401a41007ad90eb216d 100644 --- a/arm_compute/runtime/NEON/functions/NEActivationLayer.h +++ b/arm_compute/runtime/NEON/functions/NEActivationLayer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,6 +27,7 @@ #include "arm_compute/runtime/IFunction.h" #include "arm_compute/core/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "arm_compute/runtime/IRuntimeContext.h" #include diff --git a/arm_compute/runtime/NEON/functions/NEAddMulAdd.h b/arm_compute/runtime/NEON/functions/NEAddMulAdd.h index 28185f338f58e59133fcd006f6e5a76d460fffcc..e5e85542f8526ba45058c25c7874ef862f85399f 100644 --- a/arm_compute/runtime/NEON/functions/NEAddMulAdd.h +++ b/arm_compute/runtime/NEON/functions/NEAddMulAdd.h @@ -34,6 +34,7 @@ namespace arm_compute { class ITensor; class ITensorInfo; +class ActivationLayerInfo; /** Function to compute Add+Mul+Add fused operation */ class NEAddMulAdd : public IFunction diff --git a/arm_compute/runtime/NEON/functions/NEArithmeticAddition.h b/arm_compute/runtime/NEON/functions/NEArithmeticAddition.h index b9012b02a95356a9be2811f30b823dc251969954..b0d710d51702aa82e264bd7628e4284403284e59 100644 --- a/arm_compute/runtime/NEON/functions/NEArithmeticAddition.h +++ b/arm_compute/runtime/NEON/functions/NEArithmeticAddition.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021 Arm Limited. + * Copyright (c) 2016-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #define ARM_COMPUTE_NEARITHMETICADDITION_H #include "arm_compute/core/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "arm_compute/runtime/IFunction.h" #include diff --git a/arm_compute/runtime/NEON/functions/NEArithmeticSubtraction.h b/arm_compute/runtime/NEON/functions/NEArithmeticSubtraction.h index 0b4db61d291d103f43766fe2f2002fd021f6c0c5..6fbe9ad45053e57a37d15d3676588de4a51b834b 100644 --- a/arm_compute/runtime/NEON/functions/NEArithmeticSubtraction.h +++ b/arm_compute/runtime/NEON/functions/NEArithmeticSubtraction.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021 Arm Limited. + * Copyright (c) 2016-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #define ARM_COMPUTE_NEARITHMETICSUBTRACTION_H #include "arm_compute/core/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "arm_compute/runtime/IFunction.h" #include "arm_compute/runtime/NEON/INEOperator.h" diff --git a/arm_compute/runtime/NEON/functions/NEConvolutionLayer.h b/arm_compute/runtime/NEON/functions/NEConvolutionLayer.h index 974b320043489154ab28407f03f9f81924c64d2d..4dd76d082bb6af20c4f16c2af6f710d497424319 100644 --- a/arm_compute/runtime/NEON/functions/NEConvolutionLayer.h +++ b/arm_compute/runtime/NEON/functions/NEConvolutionLayer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -28,6 +28,7 @@ #include "arm_compute/core/ITensorInfo.h" #include "arm_compute/core/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "arm_compute/runtime/MemoryGroup.h" #include diff --git a/arm_compute/runtime/NEON/functions/NEDirectConvolutionLayer.h b/arm_compute/runtime/NEON/functions/NEDirectConvolutionLayer.h index 70352fdfaa33f99ee0e81af9af11724e3034b051..8db7e6596b818d991f478e0b31520a2ec4db64e7 100644 --- a/arm_compute/runtime/NEON/functions/NEDirectConvolutionLayer.h +++ b/arm_compute/runtime/NEON/functions/NEDirectConvolutionLayer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #define ARM_COMPUTE_NEDIRECTCONVOLUTIONLAYER_H #include "arm_compute/core/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "arm_compute/runtime/IFunction.h" #include "arm_compute/runtime/IMemoryManager.h" #include "arm_compute/runtime/MemoryGroup.h" diff --git a/arm_compute/runtime/NEON/functions/NEElementwiseOperations.h b/arm_compute/runtime/NEON/functions/NEElementwiseOperations.h index 95274bdb0c56ac1dafa63d61362152f1a4fb6012..bfcd221e17fc388b4053e382405b4717d1e51771 100644 --- a/arm_compute/runtime/NEON/functions/NEElementwiseOperations.h +++ b/arm_compute/runtime/NEON/functions/NEElementwiseOperations.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #define ARM_COMPUTE_NEELEMENTWISEOPERATIONS_H #include "arm_compute/core/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "arm_compute/runtime/IFunction.h" #include "arm_compute/runtime/NEON/INEOperator.h" diff --git a/arm_compute/runtime/NEON/functions/NEFullyConnectedLayer.h b/arm_compute/runtime/NEON/functions/NEFullyConnectedLayer.h index 6a4de2e311f4012cd503c020a66e4a9103c5ccce..05b7ce3735452ccec71d4ee82013f29c9fec43dc 100644 --- a/arm_compute/runtime/NEON/functions/NEFullyConnectedLayer.h +++ b/arm_compute/runtime/NEON/functions/NEFullyConnectedLayer.h @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_NEFULLYCONNECTEDLAYER_H #define ARM_COMPUTE_NEFULLYCONNECTEDLAYER_H +#include "arm_compute/function_info/FullyConnectedLayerInfo.h" #include "arm_compute/runtime/IFunction.h" #include "arm_compute/runtime/IMemoryManager.h" #include "arm_compute/runtime/IWeightsManager.h" diff --git a/arm_compute/runtime/NEON/functions/NEGEMM.h b/arm_compute/runtime/NEON/functions/NEGEMM.h index db1592316526330c1923ebdfbf1997523cb91070..c6ff2dfb92b4671ceb5fc6db3eb6fb93bfcf6546 100644 --- a/arm_compute/runtime/NEON/functions/NEGEMM.h +++ b/arm_compute/runtime/NEON/functions/NEGEMM.h @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_NEGEMM_H #define ARM_COMPUTE_NEGEMM_H +#include "arm_compute/function_info/GEMMInfo.h" #include "arm_compute/runtime/IFunction.h" #include "arm_compute/runtime/IMemoryManager.h" #include "arm_compute/runtime/IWeightsManager.h" diff --git a/arm_compute/runtime/NEON/functions/NEGEMMConvolutionLayer.h b/arm_compute/runtime/NEON/functions/NEGEMMConvolutionLayer.h index a28266265dc83f4d5b44fae09aa2c21b608630ea..72309e464ef18703789da07ad269efa1b2c3d793 100644 --- a/arm_compute/runtime/NEON/functions/NEGEMMConvolutionLayer.h +++ b/arm_compute/runtime/NEON/functions/NEGEMMConvolutionLayer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2022 Arm Limited. + * Copyright (c) 2017-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,6 +27,7 @@ #include "arm_compute/runtime/IFunction.h" #include "arm_compute/core/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "arm_compute/runtime/IFunction.h" #include "arm_compute/runtime/IMemoryManager.h" #include "arm_compute/runtime/IWeightsManager.h" diff --git a/arm_compute/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.h b/arm_compute/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.h index 896ef60d6fa7206f60d3f4f34cbe0e55d12a3dd1..addb13cdfa6a267b02bbaed2fc45d14433962802 100644 --- a/arm_compute/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.h +++ b/arm_compute/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #define ARM_COMPUTE_NEGEMMLOWPMATRIXMULTIPLYCORE_H #include "arm_compute/core/Types.h" +#include "arm_compute/function_info/GEMMInfo.h" #include "arm_compute/runtime/IFunction.h" #include "arm_compute/runtime/IMemoryManager.h" #include "arm_compute/runtime/IWeightsManager.h" diff --git a/arm_compute/runtime/NEON/functions/NEMatMul.h b/arm_compute/runtime/NEON/functions/NEMatMul.h index a331c55a98865c193fa93fceb27ae83c9b354799..e961f860c1c55c12d8d9cc23cfd11485e357980d 100644 --- a/arm_compute/runtime/NEON/functions/NEMatMul.h +++ b/arm_compute/runtime/NEON/functions/NEMatMul.h @@ -24,6 +24,8 @@ #ifndef ACL_ARM_COMPUTE_RUNTIME_NEON_FUNCTIONS_NEMATMUL #define ACL_ARM_COMPUTE_RUNTIME_NEON_FUNCTIONS_NEMATMUL +#include "arm_compute/core/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "arm_compute/runtime/IFunction.h" #include @@ -91,16 +93,23 @@ public: * @param[in] rhs Right-hand side tensor info. Data types supported: same as @p lhs. * @param[out] dst Output tensor to store the result of the batched matrix multiplication. Data types supported: same as @p lhs / @p rhs. * @param[in] info Contains MatMul operation information described in @ref MatMulInfo. - * @param[in] settings Class containing flags for function level settings i.e fast math + * @param[in] settings Contains flags for function level settings i.e fast math + * @param[in] act_info (Optional) Contains activation function and lower and upper bound values for bounded activation functions. */ - void configure(ITensor *lhs, ITensor *rhs, ITensor *dst, const MatMulInfo &info, const CpuMatMulSettings &settings); + void configure(ITensor *lhs, ITensor *rhs, ITensor *dst, const MatMulInfo &info, const CpuMatMulSettings &settings, const ActivationLayerInfo &act_info = ActivationLayerInfo()); /** Static function to check if given info will lead to a valid configuration of @ref NEMatMul * - * Parameters are similar to @ref NEMatMul::configure() + * @param[in] lhs Left-hand side tensor info. Data types supported: F16/F32/QASYMM8_SIGNED/QASYMM8. + * @param[in] rhs Right-hand side tensor info. Data types supported: same as @p lhs. + * @param[out] dst Output tensor info to store the result of the batched matrix multiplication. Data types supported: same as @p lhs / @p rhs. + * @param[in] info Contains MatMul operation information described in @ref MatMulInfo. + * @param[in] settings Contains flags for function level settings i.e fast math + * @param[in] act_info (Optional) Contains activation function and lower and upper bound values for bounded activation functions. * * @return Status */ - static Status validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *dst, const MatMulInfo &info, const CpuMatMulSettings &settings); + static Status validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *dst, const MatMulInfo &info, const CpuMatMulSettings &settings, + const ActivationLayerInfo &act_info = ActivationLayerInfo()); // Inherited methods overridden void run() override; diff --git a/arm_compute/runtime/NEON/functions/NEPixelWiseMultiplication.h b/arm_compute/runtime/NEON/functions/NEPixelWiseMultiplication.h index 4684c2d4b807bf1894274fb6c26f1c1963e590c6..634e8e0c39ebd5abb782e65a507acfa5154b1f6b 100644 --- a/arm_compute/runtime/NEON/functions/NEPixelWiseMultiplication.h +++ b/arm_compute/runtime/NEON/functions/NEPixelWiseMultiplication.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021 Arm Limited. + * Copyright (c) 2016-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,7 +24,9 @@ #ifndef ARM_COMPUTE_NEPIXELWISEMULTIPLICATION_H #define ARM_COMPUTE_NEPIXELWISEMULTIPLICATION_H +#include "arm_compute/core/Rounding.h" #include "arm_compute/core/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "arm_compute/runtime/IFunction.h" #include diff --git a/arm_compute/runtime/NEON/functions/NEWinogradConvolutionLayer.h b/arm_compute/runtime/NEON/functions/NEWinogradConvolutionLayer.h index 85b4d047ef2e6cd792a2b15acf43211880a600fa..f6f0185e7de9e22c16374881ba8f8ef24335b405 100644 --- a/arm_compute/runtime/NEON/functions/NEWinogradConvolutionLayer.h +++ b/arm_compute/runtime/NEON/functions/NEWinogradConvolutionLayer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2022 Arm Limited. + * Copyright (c) 2017-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,6 +27,7 @@ #include "arm_compute/runtime/IFunction.h" #include "arm_compute/core/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "arm_compute/runtime/Tensor.h" #include diff --git a/compute_kernel_writer/CMakeLists.txt b/compute_kernel_writer/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..9a975630254bc96fc7eace671e1f7c17c30c0958 --- /dev/null +++ b/compute_kernel_writer/CMakeLists.txt @@ -0,0 +1,181 @@ +# Copyright (c) 2023 Arm Limited. +# +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +cmake_minimum_required(VERSION 3.14 FATAL_ERROR) + +#--------------------------------------------------------------------- +# Compute Kernel Writer Project + +project(ComputeKernelWriter + VERSION 1.0.0 + LANGUAGES CXX +) + +set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +include(GNUInstallDirs) + +message(STATUS "${CMAKE_PROJECT_NAME} ${CMAKE_PROJECT_VERSION}") + +#--------------------------------------------------------------------- +# Options + +option(CKW_ENABLE_OPENCL "Enable OpenCL code generation" OFF) +option(CKW_ENABLE_ASSERTS "Enable assertions. Always enabled in Debug builds" OFF) +option(CKW_BUILD_TESTING "Build the Compute Kernel Writer validation test suite" OFF) +option(CKW_BUILD_PROTOTYPE "Build the prototype implementation of kernel writer." OFF) +option(CKW_CCACHE "Use compiler cache for faster recompilation" OFF) + +#--------------------------------------------------------------------- +# Build configuration + +get_property(CKW_IS_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) + +# Allow only Release or Debug builds +if(NOT CKW_IS_MULTI_CONFIG) # Single-config generators + if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release CACHE STRING "Options: Release (default) or Debug" FORCE) + endif() +else() # Multi-config generators + list(REMOVE_ITEM CMAKE_CONFIGURATION_TYPES RelWithDebInfo MinSizeRel) +endif() + +# Simplistic CCache setup +if(CKW_CCACHE) + find_program(CCACHE_FOUND ccache) + if(CCACHE_FOUND) + set(CMAKE_C_COMPILER_LAUNCHER ${CACHE_FOUND}) + set(CMAKE_CXX_COMPILER_LAUNCHER ${CACHE_FOUND}) + endif() +endif() + +#--------------------------------------------------------------------- +# Library targets + +set(CKW_CXX_FLAGS + -Wall + -Werror + -Wextra + -Wdisabled-optimization + -Wformat=2 + -Winit-self + -Wstrict-overflow=2 + -Wswitch-default + -Woverloaded-virtual + -Wformat-security + -Wctor-dtor-privacy + -Wsign-promo + -Weffc++ + -pedantic +) +set(GNU_WARNINGS + -Wlogical-op + -Wstrict-null-sentinel +) +set(CKW_ASSERTS_OPTS + -fstack-protector-strong +) + +add_library(ckw) +target_compile_options(ckw + PUBLIC + ${CKW_CXX_FLAGS} + "$<$:${GNU_WARNINGS}>" + "$<$:${CKW_ASSERTS_OPTS}>" + "$<$:${CKW_ASSERTS_OPTS}>" + # Set CMAKE_CXX_FLAGS last so user can overwrite options + ${CMAKE_CXX_FLAGS} + PRIVATE + # Always optimize for binary size + $<$:-Os> +) + +target_compile_definitions(ckw PUBLIC + $<$:COMPUTE_KERNEL_WRITER_DEBUG_ENABLED> + $<$:COMPUTE_KERNEL_WRITER_ASSERTS_ENABLED> + $<$:COMPUTE_KERNEL_WRITER_ASSERTS_ENABLED> + $<$:COMPUTE_KERNEL_WRITER_OPENCL_ENABLED> +) + +target_sources(ckw PRIVATE + src/Error.cpp + src/Helpers.cpp + src/Kernel.cpp + src/KernelWriter.cpp + src/TensorInfo.cpp + src/TensorOperand.cpp + src/TensorSampler.cpp + src/TensorUtils.cpp + src/TileInfo.cpp + src/TileOperand.cpp +) + +if(CKW_ENABLE_OPENCL) + target_sources(ckw PRIVATE + src/cl/CLTensorArgument.cpp + src/cl/CLTensorComponent.cpp + src/cl/CLHelpers.cpp + src/cl/CLTile.cpp + src/cl/CLKernelWriter.cpp + ) +endif() + +target_include_directories(ckw + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include + PRIVATE ${CMAKE_CURRENT_LIST_DIR} +) + +#--------------------------------------------------------------------- +# Validation tests + +if(CKW_BUILD_TESTING) + add_executable(ckw_validation + validation/Validation.cpp + ) + + target_link_libraries(ckw_validation PRIVATE ckw) + target_include_directories(ckw_validation + PRIVATE ${CMAKE_CURRENT_LIST_DIR} + ) +endif() + +#--------------------------------------------------------------------- +# Prototype + +if(CKW_BUILD_PROTOTYPE) + add_subdirectory(prototype) +endif() + +#--------------------------------------------------------------------- +# Installing + +install(TARGETS ckw + CONFIGURATIONS Release + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} +) + +install(DIRECTORY include/ckw + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} +) diff --git a/compute_kernel_writer/README.md b/compute_kernel_writer/README.md new file mode 100644 index 0000000000000000000000000000000000000000..2c7636ca9ffbe716827f9015411bae7ed48b1982 --- /dev/null +++ b/compute_kernel_writer/README.md @@ -0,0 +1,97 @@ +# Compute Kernel Writer + +Compute Kernel Writer is a tile-based, just-in-time code writer for deep learning and computer vision applications. +This tool offers a C++ interface to allow developers to write functions without a return type (called "kernels") +using their preferred programming language (at the moment, only OpenCL is supported). +The library is specifically designed to be lightweight and to offer an intuitive API for efficient code writing. + +## Getting started + +The fastest way to get started with Compute Kernel Writer is to build and run the test suite. +The following subsections show you how to do this. + +### Dependencies + +This project requires the following dependencies, obtainable via your preferred package manager, to be installed and +available on your system. + +* `build-essential` +* `cmake >= 3.14` +* (Optional) `ninja-build` + +In addition, the guide makes use of the following toolchains: + +* (Optional) `Arm GNU toolchain` available to download from + the [Arm Developer](https://developer.arm.com/downloads/-/arm-gnu-toolchain-downloads) website +* (Optional) `Android NDK toolset` available to download from + the [Android Developer](https://developer.android.com/ndk/downloads/index.html) website + +### Building and running tests + +#### Native compilation + +You can quickly compile the library on your computer by using the following commands: + +```shell +mkdir -p build && cd build +CXX=g++ cmake -G Ninja -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCKW_ENABLE_OPENCL=ON -DCKW_ENABLE_ASSERTS=ON -DCKW_BUILD_TESTING=ON -S .. +cmake --build . +``` + +The preceding commands build the library in release mode (`-DCMAKE_BUILD_TYPE=Release`) and targets OpenCL code +generation (`-DCKW_ENABLE_OPENCL=ON`). +In addition, code assertions are enabled (`-DCKW_ENABLE_ASSERTS=ON`) and the test suite is +built (`-DCKW_BUILD_TESTING=ON`). +Alternatively, choose to build a static instead of a shared library by setting `-DBUILD_SHARED_LIBS=OFF`. + +#### Cross-compile to Linux AArch64 + +The Arm GNU toolchain can be used to cross-compile the project to a Linux system with an AArch64 processor, like a +Raspberry Pi, using an x86_64 Linux host machine. + +```shell +mkdir -p build && cd build +CXX=aarch64-none-linux-gnu-g++ cmake -G Ninja -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCKW_ENABLE_OPENCL=ON -DCKW_ENABLE_ASSERTS=ON -DCKW_BUILD_TESTING=ON -S .. +cmake --build . +``` + +The build configuration is identical to the previous step but now requires specifying the target triple in the CXX +compiler (`CXX=aarch64-none-linux-gnu-g++`) to generate binaries for the target platform. + +#### Cross-compile to Android AArch64 + +Cross-compiling for Android systems requires the Android NDK toolset. The downloaded NDK contains the toolchain file +necessary for cross-compiling the project. + +```shell +mkdir -p build && cd build +cmake -G Ninja -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCKW_ENABLE_OPENCL=ON -DCKW_ENABLE_ASSERTS=ON -DCKW_BUILD_TESTING=ON -DCMAKE_TOOLCHAIN_FILE=/build/cmake/android.toolchain.cmake -S .. +cmake --build . +``` + +This build re-uses the same build configuration as before, but this time does not require specifying the CXX compiler as +this (and other target-specific information) is handled by the toolchain file (`-DCMAKE_TOOLCHAIN_FILE`). + +#### Run the validation test suite + +Confirm the project has been built successfully by running the validation test suite. + +```shell +./ckw_validation +``` + +### List of build options + +This project can be configured with the following build options. Enable options by passing them to the CMake command, +preceded with `-D`. + +| Option | Description | +|:---------------------|:------------------------------------------------------------------------------------------------------------------------------------------| +| BUILD_SHARED_LIBS | Controls whether to build static or shared libraries. | +| CMAKE_BUILD_TYPE | The project build type or configuration. Choose from Release or Debug.
The release build will always build for smallest binary size. | +| CKW_ENABLE_OPENCL | Enable OpenCL code generation. | +| CKW_ENABLE_ASSERTS | Enable assertions. Always enabled for Debug builds. | +| CKW_BUILD_TESTING | Build the validation test suite. | +| CKW_BUILD_PROTOTYPE | Build the prototype implementation. | +| CKW_CCACHE | Use compiler cache for faster recompilation. | +| CMAKE_TOOLCHAIN_FILE | When cross-compiling, set this variable to the path of the CMake toolchain file. | diff --git a/compute_kernel_writer/include/ckw/Error.h b/compute_kernel_writer/include/ckw/Error.h new file mode 100644 index 0000000000000000000000000000000000000000..eaf3f10c05ea1b51202e5f44661ff7ff7d70558f --- /dev/null +++ b/compute_kernel_writer/include/ckw/Error.h @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef CKW_INCLUDE_CKW_ERROR_H +#define CKW_INCLUDE_CKW_ERROR_H + +#include +#include + +namespace ckw +{ +/** Creates the error message + * + * @param[in] file File in which the error occurred. + * @param[in] func Function in which the error occurred. + * @param[in] line Line in which the error occurred. + * @param[in] msg Message to display before abandoning. + * + * @return status containing the error + */ +std::string +create_error_msg(const std::string &file, const std::string &func, const std::string &line, const std::string &msg); + +/** Print the given message then throw an std::runtime_error. + * + * @param[in] msg Message to display. + */ +#define COMPUTE_KERNEL_WRITER_ERROR_ON_MSG(msg) \ + do \ + { \ + const std::string arg0(__FILE__); \ + const std::string arg1(__func__); \ + const std::string arg2(std::to_string(__LINE__)); \ + const std::string arg3(msg); \ + std::runtime_error(create_error_msg(arg0, arg1, arg2, arg3)); \ + } while(false) + +/** Mark the variables as unused. + * + * @param[in] ... Variables which are unused. + */ +#define CKW_UNUSED(...) ckw::ignore_unused(__VA_ARGS__) // NOLINT + +/** Mark the variables as unused. + * + * @param[in] ... Variables which are unused. + */ +template +inline void ignore_unused(T &&...) +{ +} + +/** Throw an std::runtime_error with the specified message. + * + * @param[in] msg The error message. + */ +#define CKW_THROW_MSG(msg) \ + do \ + { \ + const std::string file(__FILE__); \ + const std::string func(__func__); \ + const std::string line(std::to_string(__LINE__)); \ + const std::string message(msg); \ + \ + throw std::runtime_error(ckw::create_error_msg(file, func, line, message)); \ + } while(false) + +#ifdef COMPUTE_KERNEL_WRITER_ASSERTS_ENABLED + +/** If the condition is not met, throw an std::runtime_error with the specified message if assertion is enabled. + * + * @param[in] cond The condition that is expected to be true. + * @param[in] msg The error message when the condition is not met. + */ +#define CKW_ASSERT_MSG(cond, msg) \ + do \ + { \ + if(!(cond)) \ + { \ + CKW_THROW_MSG(msg); \ + } \ + } while(false) + +#else // COMPUTE_KERNEL_WRITER_ASSERTS_ENABLED + +#define CKW_ASSERT_MSG(cond, msg) + +#endif // COMPUTE_KERNEL_WRITER_ASSERTS_ENABLED + +/** If the condition is not met, throw an std::runtime_error if assertion is enabled. + * + * @param[in] cond The condition that is expected to be true. + */ +#define CKW_ASSERT(cond) CKW_ASSERT_MSG(cond, #cond) + +/** Throw an std::runtime_error with the specified message if assertion is enabled. + * + * @param[in] msg The error message when the condition is not met. + */ +#define CKW_ASSERT_FAILED_MSG(msg) CKW_ASSERT_MSG(false, msg) + +} // namespace ckw + +#endif // CKW_INCLUDE_CKW_ERROR_H diff --git a/compute_kernel_writer/include/ckw/Kernel.h b/compute_kernel_writer/include/ckw/Kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..d93ed6f1d365fd55cfd8dad144abfbc7c82e46c6 --- /dev/null +++ b/compute_kernel_writer/include/ckw/Kernel.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_INCLUDE_CKW_KERNEL_H +#define CKW_INCLUDE_CKW_KERNEL_H + +#include + +namespace ckw +{ + +// Forward Declerations +class TileInfo; +class TileOperand; + +enum class TargetLanguage; + +/** The kernel that has been emitted by the kernel writer. + * + * It contains all the necessary information to compile and execute the kernel. + */ +class Kernel +{ +public: + virtual ~Kernel(); + + /** Initialize a new instance of @ref Kernel class with all emitted kernel information. + * + * @param[in] language The target language of the kernel. + * @param[in] source_code The source code of the kernel. + */ + Kernel(TargetLanguage language, const std::string &source_code); + + /** Get the target language. */ + TargetLanguage target_language() const; + + /** Get the source code. */ + const std::string &source_code() const; + + /** Add a tile operand */ + virtual TileOperand &add_operand(const std::string &name, const TileInfo &tile_info) = 0; + +private: + TargetLanguage _language; + std::string _source_code; +}; + +} // namespace ckw + +#endif // CKW_INCLUDE_CKW_KERNEL_H diff --git a/compute_kernel_writer/include/ckw/KernelWriter.h b/compute_kernel_writer/include/ckw/KernelWriter.h new file mode 100644 index 0000000000000000000000000000000000000000..2a347e9ae0e5877ada58d2a751c0a19b73b42301 --- /dev/null +++ b/compute_kernel_writer/include/ckw/KernelWriter.h @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_INCLUDE_CKW_KERNELWRITER_H +#define CKW_INCLUDE_CKW_KERNELWRITER_H + +#include "ckw/TensorOperand.h" +#include "ckw/TileOperand.h" + +#include +#include + +namespace ckw +{ + +class Kernel; + +/** Forward Declerations */ +class TensorInfo; +class TileInfo; +enum class TargetArchitecture; +enum class TargetLanguage; + +/** A kernel writer. + * + * This class is used to construct a new kernel by defining arguments, declaring variable and writing code. + * + * Use @ref KernelWriter::create_instance method to create the kernel writer for the specific target architecture and language. + * + * After having finished constructing the kernel, call @ref KernelWriter::emit_kernel to get the kernel object. + */ +class KernelWriter +{ +public: + // ============================================================================================= + // Construtors and destructor + // ============================================================================================= + + /** Initialize a new instance of @ref KernelWriter class for the specific architecture and language. + * + * Supported target architectures and languages: + * + * Architecture | Languages | + * ------------------------------|------------------------------| + * GpuArmMaliValhall | OpenCL | + * + * @param[in] architecture The architecture on which the kernel is executed. + * @param[in] language The language to write the kernel. + */ + static std::unique_ptr create_instance(TargetArchitecture architecture, TargetLanguage language); + + /** Destructor */ + virtual ~KernelWriter(); + + // ============================================================================================= + // Misc + // ============================================================================================= + + /** Write the line comment in debug build. + * + * This function does not take effect on release build. + * + * The comment must only contain one line (i.e. no newline character is allowed). + * + * @param[in] text The comment to be written. + */ + virtual void comment(const std::string &text) = 0; + + // ============================================================================================= + // Code generation + // ============================================================================================= + + /** Emit the kernel object. + * + * @param[in] name The name of the kernel object to be generated. + */ + virtual std::unique_ptr emit_kernel(const std::string &name) = 0; + + // ============================================================================================= + // Tensor and tile declaration + // ============================================================================================= + + /** Declare a tensor argument. + * + * @param[in] name The name of the tensor. + * @param[in] info The tensor info. + * + * @return The @ref TensorOperand object. + */ + virtual TensorOperand declare_tensor_argument(const std::string &name, const TensorInfo &info) = 0; + + /** Declare a tile given its name and tile info + * + * @param[in] name Name of the tile + * @param[in] tile_info Shape and data type of the tile + * + * @returns The created tile operand + */ + virtual TileOperand declare_tile(const std::string &name, const TileInfo &tile_info) = 0; + + /** Write the given raw code to kernel source code + * It's used to address the cases where the user needs to + * explicitly add a code where it's not (yet) supported by + * the kernel writer utility calls. + * + * @param[in] raw_code raw code to write as string + */ + virtual void op_write_raw_code(const std::string &raw_code) = 0; + +protected: + int32_t id_space() const; + + /** Generate full variable name by prefixing it with id space */ + std::string generate_full_name(const std::string &name) const; + + /** Create a new tile operand referring to the specified tile object. */ + static TileOperand create_tile_operand(ITile &tile); + + /** Get the reference to tile object from the tile operand. */ + static ITile &get_tile(const TileOperand &operand); + + /** Create a new tensor operand from a tensor object. */ + static TensorOperand create_tensor_operand(ITensor &tensor); + + /** Get the reference to tensor object from the tensor operand. */ + static ITensor &get_tensor(const TensorOperand &operand); + +private: + int32_t _id_space{ 0 }; +}; + +} // namespace ckw + +#endif // CKW_INCLUDE_CKW_KERNELWRITER_H diff --git a/compute_kernel_writer/include/ckw/TensorInfo.h b/compute_kernel_writer/include/ckw/TensorInfo.h new file mode 100644 index 0000000000000000000000000000000000000000..87cf7c142653e991f4a928078d60ea4fce08fe09 --- /dev/null +++ b/compute_kernel_writer/include/ckw/TensorInfo.h @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef COMPUTE_KERNEL_WRITER_INCLUDE_CKW_TENSORINFO_H +#define COMPUTE_KERNEL_WRITER_INCLUDE_CKW_TENSORINFO_H + +#include "ckw/types/DataType.h" +#include "ckw/types/TensorDataLayout.h" +#include +#include + +namespace ckw +{ + +/** Compute Kernel Writer tensor shape + * The value -1 for the tensor dimension is reserved to dynamic dimensions. + */ +using TensorShape = std::array; + +/** Tensor dimension value reserved to dynamic dimensions */ +constexpr int32_t kDynamicTensorDimensionValue = -1; + +/** Compute Kernel Writer tensor info */ +class TensorInfo +{ +public: + /** Default constructor */ + TensorInfo() = default; + /** Constructor + * + * @param[in] dt Tensor data type + * @param[in] shape Tensor shape + * @param[in] dl Tensor data layout + * @param[in] id Tensor id. The id is used to keep track of the user tensor binded. Through the id, + * the user can know what tensor has been used by the Compute Kernel Writer. + * Possible id values: + * - greater than or equal to 0: bind a user specific tensors + * - less than 0: bind a virtual tensor (tile) + */ + TensorInfo(DataType dt, const TensorShape &shape, TensorDataLayout dl, int32_t id); + + /** Set shape */ + TensorInfo &shape(const TensorShape &shape); + + /** Get shape */ + TensorShape shape() const; + + /** Set data type */ + TensorInfo &data_type(DataType dt); + + /** Get data type */ + DataType data_type() const; + + /** Set data layout */ + TensorInfo &data_layout(TensorDataLayout dl); + + /** Get data layout */ + TensorDataLayout data_layout() const; + + /** Set id */ + TensorInfo &id(int32_t id); + + /** Get layout */ + int32_t id() const; + +private: + TensorShape _shape{ { 0 } }; + DataType _dt{ DataType::Unknown }; + TensorDataLayout _dl{ TensorDataLayout::Unknown }; + int32_t _id{ -1 }; +}; +} // namespace ckw + +#endif /* COMPUTE_KERNEL_WRITER_INCLUDE_CKW_TENSORINFO_H */ diff --git a/compute_kernel_writer/include/ckw/TensorOperand.h b/compute_kernel_writer/include/ckw/TensorOperand.h new file mode 100644 index 0000000000000000000000000000000000000000..2672cd53346997becf758ed9c3f4892fb7b2d9af --- /dev/null +++ b/compute_kernel_writer/include/ckw/TensorOperand.h @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_INCLUDE_CKW_TENSOROPERAND_H +#define CKW_INCLUDE_CKW_TENSOROPERAND_H + +#include "ckw/TileOperand.h" + +namespace ckw +{ + +class ITensor; +class TensorInfo; + +/** A tensor operand provides access to the tensor info, tensor storages for load/store operations + * and tensor components (e.g. shape, strides, etc.) in the form of @ref TileOperand objects. + */ +class TensorOperand +{ +public: + // _tensor field is completely hidden from the public API to avoid any misuse. + // Only kernel writer class interacts with tensor operand hence we allow it to access this field. + friend class KernelWriter; + + /** Get the tensor info. */ + const TensorInfo &info() const; + + /** Get the operand that contains the stride in dimension 0 of the tensor. */ + TileOperand stride0(); + + /** Get the operand that contains the stride in dimension 1 of the tensor. */ + TileOperand stride1(); + + /** Get the operand that contains the stride in dimension 2 of the tensor. */ + TileOperand stride2(); + + /** Get the operand that contains the stride in dimension 3 of the tensor. */ + TileOperand stride3(); + + /** Get the operand that contains the stride in dimension 4 of the tensor. */ + TileOperand stride4(); + + /** Get the operand that contains the size of dimension 0 of the tensor. */ + TileOperand dim0(); + + /** Get the operand that contains the size of dimension 1 of the tensor. */ + TileOperand dim1(); + + /** Get the operand that contains the size of dimension 2 of the tensor. */ + TileOperand dim2(); + + /** Get the operand that contains the size of dimension 3 of the tensor. */ + TileOperand dim3(); + + /** Get the operand that contains the size of dimension 4 of the tensor. */ + TileOperand dim4(); + + /** Get the operand that contains the size of dimensions 1 and 2 collapsed. */ + TileOperand dim1_dim2(); + + /** Get the operand that contains the size of dimensions 1, 2 and 3 collapsed. */ + TileOperand dim1_dim2_dim3(); + + /** Get the operand that contains the size of dimensions 2 and 3 collapsed. */ + TileOperand dim2_dim3(); + + /** Get the operand that contains the offset in bytes to the first element. */ + TileOperand offset_first_element_in_bytes(); + +private: + /** Initialize a new instance of @ref TensorOperand class for a tensor. */ + TensorOperand(ITensor &tensor); + + ITensor &_tensor; +}; + +} // namespace ckw + +#endif // CKW_INCLUDE_CKW_TENSOROPERAND_H diff --git a/compute_kernel_writer/include/ckw/TensorSampler.h b/compute_kernel_writer/include/ckw/TensorSampler.h new file mode 100644 index 0000000000000000000000000000000000000000..1b51636edbb4c7f15bdfa45ec0efa29ccc1e6183 --- /dev/null +++ b/compute_kernel_writer/include/ckw/TensorSampler.h @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_INCLUDE_CKW_TENSORSAMPLER_H +#define CKW_INCLUDE_CKW_TENSORSAMPLER_H + +#include "ckw/types/TensorStorageType.h" +#include "ckw/types/TensorSamplerTypes.h" + +namespace ckw +{ + +/** Tensor sampler + * + * It contains information about how the tensor is sampled. It can be used to + * tell how a tile should be stored to tensor memory, and how a tensor should be + * sampled to get the values stored in a tile. Where to sample the tensor is + * defined with the coordinates respecting the addressing modes, storage type and + * the tensor format defined in this class. + */ +class TensorSampler +{ +public: + /** Initialize a new instance of @ref TensorSampler class. */ + TensorSampler(); + + /** Initialize a new instance of @ref TensorSampler class. + * + * @param[in] storage Tensor storage to load/store the tensor from/to + * @param[in] format The tensor data format. + * @param[in] address_mode_x The address mode of the x dimension. + * @param[in] address_mode_y The address mode of the y dimension. + * @param[in] address_mode_z The address mode of the z dimension. + */ + TensorSampler( + TensorStorageType storage, + TensorSamplerFormat format, + TensorSamplerAddressModeX address_mode_x, + TensorSamplerAddressModeY address_mode_y, + TensorSamplerAddressModeZ address_mode_z); + + /** Get the storage for the tensor */ + TensorStorageType storage() const; + + /** Set the storage for the tensor */ + TensorSampler &storage(TensorStorageType storage); + + /** Get the format of the tensor. */ + TensorSamplerFormat format() const; + + /** Set the format of the tensor. */ + TensorSampler &format(TensorSamplerFormat format); + + /** Get the address mode of the x dimension. */ + TensorSamplerAddressModeX address_mode_x() const; + + /** Set the address mode of the x dimension. */ + TensorSampler &address_mode_x(TensorSamplerAddressModeX address_mode_x); + + /** Get the address mode of the y dimension. */ + TensorSamplerAddressModeY address_mode_y() const; + + /** Set the address mode of the y dimension. */ + TensorSampler &address_mode_y(TensorSamplerAddressModeY address_mode_y); + + /** Get the address mode of the z dimension. */ + TensorSamplerAddressModeZ address_mode_z() const; + + /** Set the address mode of the z dimension. */ + TensorSampler &address_mode_z(TensorSamplerAddressModeZ address_mode_z); + +private: + TensorStorageType _storage { TensorStorageType::BufferUint8Ptr }; + TensorSamplerFormat _format { TensorSamplerFormat::Unknown }; + TensorSamplerAddressModeX _address_mode_x { TensorSamplerAddressModeX::Unknown }; + TensorSamplerAddressModeY _address_mode_y { TensorSamplerAddressModeY::Unknown }; + TensorSamplerAddressModeZ _address_mode_z { TensorSamplerAddressModeZ::Unknown }; +}; + +} // namespace ckw + +#endif // CKW_PROTOTYPE_INCLUDE_CKW_TENSORSAMPLER_H diff --git a/compute_kernel_writer/include/ckw/TileInfo.h b/compute_kernel_writer/include/ckw/TileInfo.h new file mode 100644 index 0000000000000000000000000000000000000000..b8094f79bffffefb9a6f5dbbe3268bb6f3ec70fd --- /dev/null +++ b/compute_kernel_writer/include/ckw/TileInfo.h @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef COMPUTE_KERNEL_WRITER_INCLUDE_CKW_TILEINFO +#define COMPUTE_KERNEL_WRITER_INCLUDE_CKW_TILEINFO + +#include "ckw/types/DataType.h" + +#include +#include + +namespace ckw +{ +// Constants to access the tile width and height in the TileShape +constexpr int32_t kTileWidthIdx = 0; +constexpr int32_t kTileHeightIdx = 1; + +/** Compute Kernel Writer tile shape. It is used to define the shape of the tile */ +using TileShape = std::array; + +/** Compute Kernel Writer tile info */ +class TileInfo +{ +public: + /** Constructor used to initialize a scalar variable with a given data type + * + * @param[in] dt Tile data type + */ + TileInfo(DataType dt); + + /** Constructor used to initialize a vector with a given data type and vector length. + * + * @param[in] dt Tile data type + * @param[in] w Tile width (or vector length) + */ + TileInfo(DataType dt, int32_t w); + + /** Constructor used to initialize a tile with a given data type and tile sizes. + * + * @param[in] dt Tile data type + * @param[in] h Tile height + * @param[in] w Tile width + */ + TileInfo(DataType dt, int32_t h, int32_t w); + + /** Set width */ + TileInfo &width(int32_t w); + + /** Get width */ + int32_t width() const; + + /** Set height */ + TileInfo &height(int32_t h); + + /** Get height */ + int32_t height() const; + + /** Set data type */ + TileInfo &data_type(DataType dt); + + /** Get data type */ + DataType data_type() const; + +private: + DataType _dt{ DataType::Unknown }; + TileShape _shape{}; +}; + +} // namespace ckw + +#endif /* COMPUTE_KERNEL_WRITER_INCLUDE_CKW_TILEINFO */ diff --git a/compute_kernel_writer/include/ckw/TileOperand.h b/compute_kernel_writer/include/ckw/TileOperand.h new file mode 100644 index 0000000000000000000000000000000000000000..873a9825f3631e2855dbce26208ff41a346fd2d1 --- /dev/null +++ b/compute_kernel_writer/include/ckw/TileOperand.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_INCLUDE_CKW_TILEOPERAND_H +#define CKW_INCLUDE_CKW_TILEOPERAND_H + +namespace ckw +{ + +class KernelWriter; +class TensorOperand; +class ITile; + +/** A tile operand refers to a tile object that can be used for kernel writing. */ +class TileOperand +{ +public: + // The constructor and _tile field is completely hidden from the public API to avoid any misuse. + // Only kernel writer and tensor operand classes create and interact with tile operand hence we allow them to access this field. + friend class KernelWriter; + friend class TensorOperand; + +private: + // These are hidden from the public API to avoid any misuse. + + /** Initialize a new instance of @ref TileOperand class for the given tile. */ + TileOperand(ITile &tile); + + ITile &_tile; +}; + +} // namespace ckw + +#endif // CKW_INCLUDE_CKW_TILEOPERAND_H diff --git a/compute_kernel_writer/include/ckw/types/DataType.h b/compute_kernel_writer/include/ckw/types/DataType.h new file mode 100644 index 0000000000000000000000000000000000000000..3447dd61d68f6f33a267f83f3ba424bf4520bc4a --- /dev/null +++ b/compute_kernel_writer/include/ckw/types/DataType.h @@ -0,0 +1,50 @@ +/* +* Copyright (c) 2023 Arm Limited. +* +* SPDX-License-Identifier: MIT +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to +* deal in the Software without restriction, including without limitation the +* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +* sell copies of the Software, and to permit persons to whom the Software is +* furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +* SOFTWARE. +*/ + +#ifndef CKW_INCLUDE_CKW_DATATYPE_H +#define CKW_INCLUDE_CKW_DATATYPE_H + +#include + +namespace ckw +{ + +/** Compute Kernel Writer data types. This data type is used by the code variables and tensor arguments. */ +enum class DataType : int32_t +{ + Unknown = 0x00, + Fp32 = 0x11, + Fp16 = 0x12, + Int32 = 0x21, + Int16 = 0x22, + Int8 = 0x24, + Uint32 = 0x31, + Uint16 = 0x32, + Uint8 = 0x34, + Bool = 0x41 +}; + +} // namespace ckw + +#endif //CKW_INCLUDE_CKW_DATATYPE_H diff --git a/compute_kernel_writer/include/ckw/types/TargetArchitecture.h b/compute_kernel_writer/include/ckw/types/TargetArchitecture.h new file mode 100644 index 0000000000000000000000000000000000000000..25662a01f0f175d2a4e03efd35cdfa9f19a2ba7b --- /dev/null +++ b/compute_kernel_writer/include/ckw/types/TargetArchitecture.h @@ -0,0 +1,40 @@ +/* +* Copyright (c) 2023 Arm Limited. +* +* SPDX-License-Identifier: MIT +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to +* deal in the Software without restriction, including without limitation the +* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +* sell copies of the Software, and to permit persons to whom the Software is +* furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +* SOFTWARE. +*/ + +#ifndef CKW_INCLUDE_CKW_TYPES_TARGETARCHITECTURE_H +#define CKW_INCLUDE_CKW_TYPES_TARGETARCHITECTURE_H + +namespace ckw +{ + +/** Target platform architecture. */ +enum class TargetArchitecture +{ + Unknown, + GpuArmMaliValhall, +}; + +} // namespace ckw + +#endif // CKW_INCLUDE_CKW_TYPES_TARGETARCHITECTURE_H diff --git a/compute_kernel_writer/include/ckw/types/TargetLanguage.h b/compute_kernel_writer/include/ckw/types/TargetLanguage.h new file mode 100644 index 0000000000000000000000000000000000000000..1f507573dd114f2b5352d2fe6082ec1183deb137 --- /dev/null +++ b/compute_kernel_writer/include/ckw/types/TargetLanguage.h @@ -0,0 +1,40 @@ +/* +* Copyright (c) 2023 Arm Limited. +* +* SPDX-License-Identifier: MIT +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to +* deal in the Software without restriction, including without limitation the +* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +* sell copies of the Software, and to permit persons to whom the Software is +* furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +* SOFTWARE. +*/ + +#ifndef CKW_INCLUDE_CKW_TYPES_TARGETLANGUAGE_H +#define CKW_INCLUDE_CKW_TYPES_TARGETLANGUAGE_H + +namespace ckw +{ + +/** Target language. */ +enum class TargetLanguage +{ + Unknown, + OpenCL +}; + +} // namespace ckw + +#endif // CKW_INCLUDE_CKW_TYPES_TARGETLANGUAGE_H diff --git a/compute_kernel_writer/include/ckw/types/TensorComponentType.h b/compute_kernel_writer/include/ckw/types/TensorComponentType.h new file mode 100644 index 0000000000000000000000000000000000000000..7a5031d8c06f0531b17c040b592bcd89d96622a2 --- /dev/null +++ b/compute_kernel_writer/include/ckw/types/TensorComponentType.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_INCLUDE_CKW_TYPES_TENSORCOMPONENTTYPE_H +#define CKW_INCLUDE_CKW_TYPES_TENSORCOMPONENTTYPE_H + +#include + +namespace ckw +{ + +/** Compute Kernel Writer tensor component. + * + * The tensor components are used to access specific backend-agnostic tensor arguments, + * such as the tensor dimensions and tensor strides. + * The tensor component is represented as an unsigned integer. The value of the integer value + * is assigned to retrieve the information through the @ref TensorComponentBitmask. + */ +enum class TensorComponentType : uint32_t +{ + Unknown = 0x00000000, + OffsetFirstElement = 0x01000000, + Stride0 = 0x02000001, + Stride1 = 0x02000002, + Stride2 = 0x02000003, + Stride3 = 0x02000004, + Stride4 = 0x02000005, + Dim0 = 0x04000001, + Dim1 = 0x04000002, + Dim2 = 0x04000003, + Dim3 = 0x04000004, + Dim4 = 0x04000005, + Dim1xDim2 = 0x08000032, + Dim2xDim3 = 0x08000043, + Dim1xDim2xDim3 = 0x08000432 +}; + +} // namespace ckw + +#endif // CKW_INCLUDE_CKW_TYPES_TENSORCOMPONENTTYPE_H diff --git a/compute_kernel_writer/include/ckw/types/TensorDataLayout.h b/compute_kernel_writer/include/ckw/types/TensorDataLayout.h new file mode 100644 index 0000000000000000000000000000000000000000..532b299910234fb8f963cb9a6b5d524a324b1470 --- /dev/null +++ b/compute_kernel_writer/include/ckw/types/TensorDataLayout.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_INCLUDE_CKW_TYPES_TENSORDATALAYOUT_H +#define CKW_INCLUDE_CKW_TYPES_TENSORDATALAYOUT_H + +namespace ckw +{ + +/** Compute Kernel Writer tensor data layout (or memory format) */ +enum class TensorDataLayout +{ + Unknown, + Nhwc, + Ndhwc +}; + +/** Compute Kernel Writer tensor data layout component */ +enum class TensorDataLayoutComponent +{ + Unknown, + N, + D, + H, + W, + C, +}; + +} // namespace ckw + +#endif // CKW_INCLUDE_CKW_TYPES_TENSORDATALAYOUT_H diff --git a/compute_kernel_writer/include/ckw/types/TensorSamplerTypes.h b/compute_kernel_writer/include/ckw/types/TensorSamplerTypes.h new file mode 100644 index 0000000000000000000000000000000000000000..347536512e8f61f17534aad8c7714a923d622301 --- /dev/null +++ b/compute_kernel_writer/include/ckw/types/TensorSamplerTypes.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_INCLUDE_CKW_TENSORSAMPLERTYPES_H +#define CKW_INCLUDE_CKW_TENSORSAMPLERTYPES_H + +#include + +namespace ckw +{ + +// This enum class defines how the dimensions of a 3d tensor is mapped into x,y and z coordianates. +enum class TensorSamplerFormat : int32_t +{ + Unknown = 0, + D0_D1xD2_1 = 1, // Original dimensions 1 and 2 are collapsed onto y-axis + D0_D1_D2 = 2 // Original dimensions stays as they're defined. No collapsing. +}; + +/** Tensor sampler address mode enum class for X dimension + * + * The following address modes are available in total: + * Unknown + * None : The user guarantees that the coordinate is always in-bound + * OverlappingMin : (FIXED shapes only) Reduce the load/store length when x == 0 (MIN). The load length will be width % original length + * Leftover elements can be handled using overlapping. This involves processing some of the elements in the array twice. + * ClampToBorderMaxOnly : Clamp to max value allowed in the corresponding dimension, and construct an if/else guard to prevent out of bound access, + * e.g. if( y < size-of-dimension-y ){ } + * + * Individual dimensions choose which adddress mode to implement in their respective enum classes. + */ +enum class TensorSamplerAddressModeX : int32_t +{ + Unknown = 0, + None = 1, // The user guarantees that the coordinate is always in-bound + OverlappingMin = 2 // (FIXED shapes only) Reduce the load/store length when x == 0 (MIN). The load length will be width % original length + // Leftover elements can be handled using overlapping. This involves processing some of the elements in the array twice. +}; + +/** + * Similar to @ref TensorSamplerAddressModeX + */ +enum class TensorSamplerAddressModeY : int32_t +{ + Unknown = 0, + None = 1, + OverlappingMin = 2, + ClampToBorderMaxOnly = 3 +}; + +/** + * Similar to @ref TensorSamplerAddressModeX + */ +enum class TensorSamplerAddressModeZ : int32_t +{ + Unknown = 0, + None = 1, +}; + +} // namespace ckw + +#endif //CKW_INCLUDE_CKW_TENSORSAMPLERTYPES_H diff --git a/compute_kernel_writer/include/ckw/types/TensorStorageType.h b/compute_kernel_writer/include/ckw/types/TensorStorageType.h new file mode 100644 index 0000000000000000000000000000000000000000..5a2f17d520d1bc42deec8af34fd7db9292692304 --- /dev/null +++ b/compute_kernel_writer/include/ckw/types/TensorStorageType.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_INCLUDE_CKW_TYPES_TENSORSTORAGETYPE_H +#define CKW_INCLUDE_CKW_TYPES_TENSORSTORAGETYPE_H + +#include + +namespace ckw +{ + +/** Compute Kernel Writer tensor storage. + * The tensor storage represents the type of tensor memory object. + */ +enum class TensorStorageType : uint32_t +{ + Unknown = 0x00000000, + BufferUint8Ptr = 0x01000000, + Texture2dReadOnly = 0x02000001, + Texture2dWriteOnly = 0x02000010, +}; + +} // namespace ckw + +#endif // CKW_INCLUDE_CKW_TYPES_TENSORSTORAGETYPE_H diff --git a/compute_kernel_writer/prototype/CMakeLists.txt b/compute_kernel_writer/prototype/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..13d1ae8fc44eb83caece3a1c7b95484ef063ca7a --- /dev/null +++ b/compute_kernel_writer/prototype/CMakeLists.txt @@ -0,0 +1,78 @@ +# Copyright (c) 2023 Arm Limited. +# +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +cmake_minimum_required(VERSION 3.14 FATAL_ERROR) + +#--------------------------------------------------------------------- +# Prototype + +add_library(ckw_prototype + src/TileInfo.cpp + src/TensorInfo.cpp + src/Kernel.cpp + src/KernelWriter.cpp + src/OperandBase.cpp + src/TileOperand.cpp + src/TensorOperand.cpp + src/TensorTileSampler.cpp + src/KernelArgument.cpp +) + +target_compile_options(ckw_prototype + PUBLIC + ${CKW_CXX_FLAGS} + "$<$:${GNU_WARNINGS}>" + "$<$:${CKW_ASSERTS_OPTS}>" + "$<$:${CKW_ASSERTS_OPTS}>" + ${CMAKE_CXX_FLAGS} + PRIVATE + $<$:-Os> +) + +target_compile_definitions(ckw_prototype PUBLIC + $<$:COMPUTE_KERNEL_WRITER_DEBUG_ENABLED> + $<$:COMPUTE_KERNEL_WRITER_ASSERTS_ENABLED> + $<$:COMPUTE_KERNEL_WRITER_ASSERTS_ENABLED> + $<$:COMPUTE_KERNEL_WRITER_OPENCL_ENABLED> +) + +target_include_directories(ckw_prototype + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include + PRIVATE ${CMAKE_CURRENT_LIST_DIR} +) + +#--------------------------------------------------------------------- +# Examples + +add_library(ckw_prototype_examples_common + examples/common/ExampleKernelWriter.cpp + examples/common/ExampleScopedKernelWriter.cpp + examples/common/ExampleComponentArgument.cpp +) + +target_link_libraries(ckw_prototype_examples_common PUBLIC ckw_prototype) + +add_executable(ckw_prototype_examples_add_exp_store examples/add_exp_store.cpp) +target_link_libraries(ckw_prototype_examples_add_exp_store PUBLIC ckw_prototype_examples_common) + +add_executable(writer_helper examples/writer_helper.cpp) +target_link_libraries(writer_helper PUBLIC ckw_prototype) diff --git a/compute_kernel_writer/prototype/examples/add_exp_store.cpp b/compute_kernel_writer/prototype/examples/add_exp_store.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6a9884543c5f3ed499aab1365c50468183a88ac9 --- /dev/null +++ b/compute_kernel_writer/prototype/examples/add_exp_store.cpp @@ -0,0 +1,203 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "ckw/Error.h" +#include "ckw/KernelArgument.h" +#include "ckw/KernelWriter.h" +#include "ckw/TensorOperand.h" +#include "ckw/TensorTileSampler.h" +#include "ckw/TileOperand.h" + +#include "common/ExampleComponentArgument.h" +#include "common/ExampleKernelWriter.h" +#include "common/ExampleScopedKernelWriter.h" + +#include +#include + +using namespace ckw; + +TensorTileSampler create_simple_sampler(ExampleScopedKernelWriter writer) +{ + TensorTileSampler sampler; + + constexpr int32_t m0 = 4; + constexpr int32_t n0 = 4; + + auto &gid_0 = writer->declare_tile("gid_0", DataType::Int32); + auto &gid_1 = writer->declare_tile("gid_1", DataType::Int32); + auto &gid_2 = writer->declare_tile("gid_2", DataType::Int32); + + auto &const_0 = writer->declare_tile("0", 0); + + writer->op_get_global_id(gid_0, 0); + writer->op_get_global_id(gid_1, 1); + writer->op_get_global_id(gid_2, 2); + + sampler.x(gid_0); + sampler.y(gid_1); + sampler.z(const_0); + sampler.b(gid_2); + + sampler.width(n0); + sampler.height(m0); + + sampler.format(TensorSamplerFormat::C_WH_1); + sampler.address_mode_x(TensorSamplerAddressModeX::None); + sampler.address_mode_y(TensorSamplerAddressModeY::ClampToBorder); + sampler.address_mode_z(TensorSamplerAddressModeZ::Skip); + + return sampler; +} + +void op_binary_elementwise(ExampleScopedKernelWriter writer, std::vector operands) +{ + auto lhs = operands.at(0); + auto rhs = operands.at(1); + auto dst = operands.at(2); + + // Load the LHS and RHS tile and prepare the tensor sampler. + if(!lhs->has_tile() && !rhs->has_tile()) + { + const auto sampler = create_simple_sampler(writer); + + writer->op_load_once(lhs, sampler); + writer->op_load_once(rhs, sampler); + } + else if(lhs->has_tile()) + { + const auto &sampler = lhs->tile_sampler(); + writer->op_load_once(rhs, sampler); + } + else + { + const auto &sampler = rhs->tile_sampler(); + writer->op_load_once(lhs, sampler); + } + + auto &lhs_tile = lhs->tile(); + auto &rhs_tile = rhs->tile(); + const auto &sampler = lhs->tile_sampler(); + + // Prepare the output tile. + if(!dst->has_tile()) + { + auto &tile = writer->declare_tile("dst_tile", lhs_tile.tile_info()); + dst->init_virtual_tensor(tile, sampler); + } + + auto &dst_tile = dst->tile(); + + // Perform the operation. + writer->op_binary_expression(dst_tile, lhs_tile, BinaryOp::Add, rhs_tile); +} + +void op_exp(ExampleScopedKernelWriter writer, std::vector operands) +{ + auto src = operands.at(0); + auto dst = operands.at(1); + + // Load the source tile and prepare the sampler. + if(!src->has_tile()) + { + const auto sampler = create_simple_sampler(writer); + writer->op_load_once(src, sampler); + } + + auto &src_tile = src->tile(); + const auto &sampler = src->tile_sampler(); + + // Prepare the output tile. + if(!dst->has_tile()) + { + auto &tile = writer->declare_tile("dst_tile", src_tile.tile_info()); + dst->init_virtual_tensor(tile, sampler); + } + + auto &dst_tile = dst->tile(); + + // Perform the operation. + writer->op_unary_elementwise_function(dst_tile, UnaryFunction::Exp, src_tile); +} + +void op_store(ExampleScopedKernelWriter writer, std::vector operands) +{ + auto src = operands.at(0); + auto dst = operands.at(1); + + auto &src_tile = src->tile(); + const auto &sampler = src->tile_sampler(); + auto &dst_tensor = dst->tensor(); + + writer->op_store(dst_tensor, src_tile, sampler); +} + +int main() +{ + Kernel kernel("example", GpuTargetLanguage::OpenCL); + ExampleKernelWriter root_writer(kernel); + + ExampleScopedKernelWriter writer(&root_writer); + + const TensorInfo src0_info(DataType::Fp32, TensorShape({ 3, 10, 20, 1, 1 }), TensorDataLayout::Nhwc, 0); + const TensorInfo src1_info(DataType::Fp32, TensorShape({ 3, 10, 20, 1, 1 }), TensorDataLayout::Nhwc, 1); + const TensorInfo dst_info(DataType::Fp32, TensorShape({ 3, 10, 20, 1, 1 }), TensorDataLayout::Nhwc, 2); + + ExampleComponentArgument src0(writer->declare_tensor_argument("src0", src0_info, TensorStorageType::BufferUint8Ptr)); + ExampleComponentArgument src1(writer->declare_tensor_argument("src1", src1_info, TensorStorageType::BufferUint8Ptr)); + ExampleComponentArgument dst(writer->declare_tensor_argument("dst", dst_info, TensorStorageType::BufferUint8Ptr)); + + ExampleComponentArgument ans; + + op_binary_elementwise(writer, { &src0, &src1, &ans }); + op_exp(writer, { &ans, &ans }); + op_store(writer, { &ans, &dst }); + + const auto arguments = kernel.arguments(); + + std::cout << "\n====================\nArguments:\n====================\n"; + + for(auto &arg : arguments) + { + switch(arg.type()) + { + case ckw::KernelArgument::Type::TensorStorage: + std::cout << "* Tensor storage: ID = " << arg.id() << ", type = " << std::hex << "0x" << static_cast(arg.tensor_storage_type()) << std::dec << "\n"; + break; + + case ckw::KernelArgument::Type::TensorComponent: + std::cout << "* Tensor component: ID = " << arg.id() << ", type = " << std::hex << "0x" << static_cast(arg.tensor_component_type()) << std::dec << "\n"; + break; + + default: + CKW_ASSERT(false); + } + } + + std::cout << "\n====================\nCode:\n====================\n"; + const auto code = root_writer.generate_code(); + std::cout << code; + + return 0; +} diff --git a/compute_kernel_writer/prototype/examples/common/ExampleComponentArgument.cpp b/compute_kernel_writer/prototype/examples/common/ExampleComponentArgument.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5a2ec526cc75d006cb44346189c5981e2244194d --- /dev/null +++ b/compute_kernel_writer/prototype/examples/common/ExampleComponentArgument.cpp @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "ExampleComponentArgument.h" +#include "ckw/Error.h" + +ExampleComponentArgument::ExampleComponentArgument() +{ +} + +ExampleComponentArgument::ExampleComponentArgument(ckw::TensorOperand &tensor) + : _tensor(&tensor) +{ +} + +ExampleComponentArgument & +ExampleComponentArgument::init_virtual_tensor(ckw::TileOperand &tile, const ckw::TensorTileSampler &tile_sampler) +{ + CKW_ASSERT(_tile == nullptr); + + _tile = &tile; + _tile_sampler = tile_sampler; + + return *this; +} + +bool ExampleComponentArgument::has_tensor() const +{ + return _tensor != nullptr; +} + +ckw::TensorOperand &ExampleComponentArgument::tensor() +{ + CKW_ASSERT(_tensor != nullptr); + + return *_tensor; +} + +const ckw::TensorOperand &ExampleComponentArgument::tensor() const +{ + CKW_ASSERT(_tensor != nullptr); + + return *_tensor; +} + +bool ExampleComponentArgument::has_tile() const +{ + return _tile != nullptr; +} + +ckw::TileOperand &ExampleComponentArgument::tile() +{ + CKW_ASSERT(_tile != nullptr); + + return *_tile; +} + +const ckw::TileOperand &ExampleComponentArgument::tile() const +{ + CKW_ASSERT(_tile != nullptr); + + return *_tile; +} + +ckw::TensorTileSampler &ExampleComponentArgument::tile_sampler() +{ + CKW_ASSERT(_tile != nullptr); + + return _tile_sampler; +} + +const ckw::TensorTileSampler &ExampleComponentArgument::tile_sampler() const +{ + CKW_ASSERT(_tile != nullptr); + + return _tile_sampler; +} diff --git a/compute_kernel_writer/prototype/examples/common/ExampleComponentArgument.h b/compute_kernel_writer/prototype/examples/common/ExampleComponentArgument.h new file mode 100644 index 0000000000000000000000000000000000000000..9fdc50ba086ab82f870b51e6ce928ec97d2b2bbe --- /dev/null +++ b/compute_kernel_writer/prototype/examples/common/ExampleComponentArgument.h @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_PROTOTYPE_EXAMPLES_COMMON_EXAMPLECOMPONENTARGUMENT_H +#define CKW_PROTOTYPE_EXAMPLES_COMMON_EXAMPLECOMPONENTARGUMENT_H + +#include "ckw/TensorTileSampler.h" + +namespace ckw +{ +class TensorOperand; + +class TileOperand; +} // namespace ckw + +/** The argument of a dynamic fusion component which can be either user tensor or virtual tensor. */ +class ExampleComponentArgument +{ +public: + /** Initialize a new instance of @ref ExampleComponentArgument class for empty virtual tensor. */ + ExampleComponentArgument(); + + /** Initialize a new instance of @ref ExampleComponentArgument class for user tensor. + * + * @param[in] tensor The user tensor. + */ + explicit ExampleComponentArgument(ckw::TensorOperand &tensor); + + /** Set virtual tensor information (tile, sampler) for the argument. + * + * If the component is a user tensor, it can be treated as virtual tensor as well + * and won't be loaded again using @ref ExampleKernelWriter::op_load_once method. + * + * @param[in] tile The tile that has been loaded. + * @param[in] sampler The tensor sampling information that has been used to load the tile. + */ + ExampleComponentArgument &init_virtual_tensor(ckw::TileOperand &tile, const ckw::TensorTileSampler &sampler); + + /** Get whether the argument is a user tensor. */ + bool has_tensor() const; + + /** Get the tensor operand. + * + * If the tensor is not available, throw an error. + */ + ckw::TensorOperand &tensor(); + + /** Get the tensor operand. + * + * If the tensor is not available, throw an error. + */ + const ckw::TensorOperand &tensor() const; + + /** Get whether the argument contains a tile. + * + * The argument can be either a user tensor that has been loaded, + * or a virtual tensor (i.e. a tile with tensor sampling information). + */ + bool has_tile() const; + + /** Get the tile operand. + * + * If the tile is not available, throw an error. + */ + ckw::TileOperand &tile(); + + /** Get the tile operand. + * + * If the tile is not available, throw an error. + */ + const ckw::TileOperand &tile() const; + + /** Get the tensor sampling information for the tile. + * + * If the tile is not available, throw an error. + */ + ckw::TensorTileSampler &tile_sampler(); + + /** Get the tensor sampling information for the tile. + * + * If the tile is not available, throw an error. + */ + const ckw::TensorTileSampler &tile_sampler() const; + +private: + ckw::TensorOperand *_tensor{ nullptr }; + ckw::TileOperand *_tile{ nullptr }; + ckw::TensorTileSampler _tile_sampler{}; +}; + +#endif // CKW_PROTOTYPE_EXAMPLES_COMMON_EXAMPLECOMPONENTARGUMENT_H diff --git a/compute_kernel_writer/prototype/examples/common/ExampleKernelWriter.cpp b/compute_kernel_writer/prototype/examples/common/ExampleKernelWriter.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6b9f244735d2677f3f6af089ddfdba818447e4c6 --- /dev/null +++ b/compute_kernel_writer/prototype/examples/common/ExampleKernelWriter.cpp @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "ExampleKernelWriter.h" +#include "ExampleComponentArgument.h" +#include "ckw/Error.h" +#include "ckw/TileInfo.h" + +ExampleKernelWriter::ExampleKernelWriter(ckw::Kernel &kernel) + : KernelWriter(kernel) +{ +} + +void ExampleKernelWriter::op_load_once(ExampleComponentArgument *tensor_or_tile, const ckw::TensorTileSampler &sampler) +{ + if(!tensor_or_tile->has_tile()) + { + CKW_ASSERT(tensor_or_tile->has_tensor()); + + auto &tensor = tensor_or_tile->tensor(); + + const auto tile_name = tensor.name() + "_tile"; + auto &tile = declare_tile(tile_name.c_str(), + ckw::TileInfo(tensor.data_type(), sampler.height(), sampler.width())); + + op_load(tile, tensor, sampler); + + tensor_or_tile->init_virtual_tensor(tile, sampler); + } +} diff --git a/compute_kernel_writer/prototype/examples/common/ExampleKernelWriter.h b/compute_kernel_writer/prototype/examples/common/ExampleKernelWriter.h new file mode 100644 index 0000000000000000000000000000000000000000..1528c3d933dcbec66c92a2fdccea92134ac56316 --- /dev/null +++ b/compute_kernel_writer/prototype/examples/common/ExampleKernelWriter.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_PROTOTYPE_EXAMPLES_COMMON_EXAMPLEKERNELWRITER_H +#define CKW_PROTOTYPE_EXAMPLES_COMMON_EXAMPLEKERNELWRITER_H + +#include "ckw/KernelWriter.h" +#include "ckw/TensorTileSampler.h" + +class ExampleComponentArgument; + +namespace ckw +{ +class Kernel; +} // namespace ckw + +/** Extended implementation of kernel writer for dynamic fusion. */ +class ExampleKernelWriter : public ckw::KernelWriter +{ +public: + /** Initialize a new instance of @ref ExampleKernelWriter class. + * + * @param[in] kernel The kernel to be generated. + */ + explicit ExampleKernelWriter(ckw::Kernel &kernel); + + /** Load the user tensor to the tile in the same component argument if it hasn't been loaded. + * + * @param[in] tensor_or_tile The component argument that is either a user tensor or a virtual tensor. + * @param[in] sampler The tensor sampling information to load the tile. + */ + void op_load_once(ExampleComponentArgument *tensor_or_tile, const ckw::TensorTileSampler &sampler); +}; + +#endif // CKW_PROTOTYPE_EXAMPLES_COMMON_EXAMPLEKERNELWRITER_H diff --git a/compute_kernel_writer/prototype/examples/common/ExampleScopedKernelWriter.cpp b/compute_kernel_writer/prototype/examples/common/ExampleScopedKernelWriter.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7c44fa8749c50f44c9cb48fd3d1366accd72da4a --- /dev/null +++ b/compute_kernel_writer/prototype/examples/common/ExampleScopedKernelWriter.cpp @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "ExampleScopedKernelWriter.h" +#include "ExampleKernelWriter.h" + +ExampleScopedKernelWriter::ExampleScopedKernelWriter(ExampleKernelWriter *writer) + : _writer(writer), _parent_id_space(writer->id_space()) +{ + _writer->next_id_space(); +} + +ExampleScopedKernelWriter::ExampleScopedKernelWriter(const ExampleScopedKernelWriter &other) + : _writer(other._writer), _parent_id_space(other._writer->id_space()) +{ + _writer->next_id_space(); +} + +ExampleKernelWriter *ExampleScopedKernelWriter::operator->() +{ + return _writer; +} + +const ExampleKernelWriter *ExampleScopedKernelWriter::operator->() const +{ + return _writer; +} + +ExampleKernelWriter *ExampleScopedKernelWriter::writer() +{ + return _writer; +} + +const ExampleKernelWriter *ExampleScopedKernelWriter::writer() const +{ + return _writer; +} diff --git a/compute_kernel_writer/prototype/examples/common/ExampleScopedKernelWriter.h b/compute_kernel_writer/prototype/examples/common/ExampleScopedKernelWriter.h new file mode 100644 index 0000000000000000000000000000000000000000..4655b1897e86138ce72ad88b8967c25c45de15b9 --- /dev/null +++ b/compute_kernel_writer/prototype/examples/common/ExampleScopedKernelWriter.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_PROTOTYPE_EXAMPLES_COMMON_EXAMPLESCOPEDKERNELWRITER_H +#define CKW_PROTOTYPE_EXAMPLES_COMMON_EXAMPLESCOPEDKERNELWRITER_H + +#include + +class ExampleKernelWriter; + +/** Helper to automatically manage kernel writer ID space. */ +class ExampleScopedKernelWriter +{ +public: + /** Initialize a new instance of @ref ExampleScopedKernelWriter class. */ + explicit ExampleScopedKernelWriter(ExampleKernelWriter *writer); + + /** Create a new scope from the specified scoped kernel writer. */ + ExampleScopedKernelWriter(const ExampleScopedKernelWriter &other); + + /** Assignment is disallowed. */ + ExampleScopedKernelWriter &operator=(const ExampleScopedKernelWriter &) = delete; + + /** Access the underlying kernel writer. */ + ExampleKernelWriter *operator->(); + + /** Access the underlying kernel writer. */ + const ExampleKernelWriter *operator->() const; + + /** Get the kernel writer. */ + ExampleKernelWriter *writer(); + + /** Get the kernel writer. */ + const ExampleKernelWriter *writer() const; + +private: + ExampleKernelWriter *_writer; + int32_t _parent_id_space; +}; + +#endif // CKW_PROTOTYPE_EXAMPLES_COMMON_EXAMPLESCOPEDKERNELWRITER_H diff --git a/compute_kernel_writer/prototype/examples/writer_helper.cpp b/compute_kernel_writer/prototype/examples/writer_helper.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ccef92dcdfc717a003bbaa49bbe9526014f73889 --- /dev/null +++ b/compute_kernel_writer/prototype/examples/writer_helper.cpp @@ -0,0 +1,118 @@ +/* +* Copyright (c) 2023 Arm Limited. +* +* SPDX-License-Identifier: MIT +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to +* deal in the Software without restriction, including without limitation the +* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +* sell copies of the Software, and to permit persons to whom the Software is +* furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +* SOFTWARE. +*/ + +#include "ckw/KernelWriter.h" +#include "../include/ckw/KernelWriterHelper.h" +#include "ckw/TensorTileSampler.h" + +#include + +using namespace ckw; + +TensorTileSampler create_simple_sampler(KernelWriter& writer) +{ + TensorTileSampler sampler; + + constexpr int32_t m0 = 1; + constexpr int32_t n0 = 1; + + auto &gid_0 = writer.declare_tile("gid_0", DataType::Int32); + auto &gid_1 = writer.declare_tile("gid_1", DataType::Int32); + auto &gid_2 = writer.declare_tile("gid_2", DataType::Int32); + + auto &const_0 = writer.declare_tile("0", 0); + + writer.op_get_global_id(gid_0, 0); + writer.op_get_global_id(gid_1, 1); + writer.op_get_global_id(gid_2, 2); + + sampler.x(gid_0); + sampler.y(gid_1); + sampler.z(gid_2); + sampler.b(const_0); + + sampler.width(n0); + sampler.height(m0); + + sampler.format(TensorSamplerFormat::C_WH_1); + sampler.address_mode_x(TensorSamplerAddressModeX::None); + sampler.address_mode_y(TensorSamplerAddressModeY::ClampToBorder); + sampler.address_mode_z(TensorSamplerAddressModeZ::Skip); + + return sampler; +} + +int main() +{ + Kernel kernel("test", GpuTargetLanguage::OpenCL); + KernelWriterHelper writer(kernel); + + const TensorInfo src_info(DataType::Fp32, TensorShape({ 1, 1, 1, 1, 1 }), TensorDataLayout::Nhwc, 0); + const TensorInfo dst_info(DataType::Fp32, TensorShape({ 1, 1, 1, 1, 1 }), TensorDataLayout::Nhwc, 1); + + auto &src_tensor = writer.declare_tensor_argument("src", src_info); + auto &dst_tensor = writer.declare_tensor_argument("dst", dst_info); + + const auto sampler = create_simple_sampler(writer); + + auto &src = writer.declare_tile("src_tile", TileInfo(src_tensor.data_type(), sampler.height(), sampler.width())); + auto &other = writer.declare_tile("other_tile", TileInfo(src_tensor.data_type(), sampler.height(), sampler.width())); + auto &dst = writer.declare_tile("dst_tile", TileInfo(src_tensor.data_type(), sampler.height(), sampler.width())); + + writer.op_load(src, src_tensor, sampler); + writer.op_load(other, src_tensor, sampler); + writer.op_load(dst, dst_tensor, sampler); + + auto test = dst ^ src ^ other; + auto other_test = logical_and(dst, src, other); + writer.op_assign(dst, logical_and(dst, src, other)); + writer.op_assign(dst, test); + writer.op_assign(dst, other_test); + writer.op_assign(dst, operator^(operator^(dst, src), other)); + + writer.op_if(exp(src) == dst, [&]{ + writer.op_binary_expression(dst, src, BinaryOp::Add, src); + }).op_else_if(exp(src) > dst, [&]{ + writer.op_binary_expression(dst, src, BinaryOp::Add, src); + }).op_else([&] { + writer.op_assign(dst, src); + }); + + writer.op_assign(dst, src + src * src); + writer.op_assign(dst, src * max(src, dst) + src); + writer.op_assign(dst, src * select(src, dst, src) + src); + + writer.op_assign(dst, src ^ dst); + writer.op_assign(dst, ~src); + + writer.op_for_loop(dst < src, dst += src, [&]{ + writer.op_assign(dst, src + dst); + }); + + writer.op_assign(dst += src); + writer.op_assign(dst += exp(src)); + + std::cout << "======== KERNEL ========" << std::endl; + std::cout << writer.generate_code() << std::endl; +} \ No newline at end of file diff --git a/compute_kernel_writer/prototype/include/ckw/Error.h b/compute_kernel_writer/prototype/include/ckw/Error.h new file mode 100644 index 0000000000000000000000000000000000000000..b18944eac5ccd6e49eda7db1a1245d06dcdc4ad0 --- /dev/null +++ b/compute_kernel_writer/prototype/include/ckw/Error.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_PROTOTYPE_INCLUDE_CKW_ERROR_H +#define CKW_PROTOTYPE_INCLUDE_CKW_ERROR_H + +#include +#include + +namespace ckw +{ + +/** If the condition is not met, throw an std::runtime_error with the specified message. + * + * @param[in] cond The condition that is expected to be true. + * @param[in] msg The error message when the condition is not met. + */ +#define CKW_ASSERT_MSG(cond, msg) \ + do \ + { \ + if(!(cond)) \ + { \ + throw ::std::runtime_error(msg); \ + } \ + } while(false) + +/** If the condition is not met, throw an std::runtime_error. + * + * @param[in] cond The condition that is expected to be true. + */ +#define CKW_ASSERT(cond) CKW_ASSERT_MSG(cond, #cond) + +/** If the precondition is met but the consequence is not met, throw an std::runtime_error. + * + * @param[in] precond The condition if is met requires the consequence must also be met. + * @param[in] cond The condition that is expected to be true if the precondition is true. + */ +#define CKW_ASSERT_IF(precond, cond) \ + CKW_ASSERT_MSG(!(precond) || ((precond) && (cond)), #precond " |-> " #cond) + +/** Mark the variables as unused. + * + * @param[in] ... Variables which are unused. + */ +#define CKW_UNUSED(...) ::ckw::ignore_unused(__VA_ARGS__) // NOLINT + +/** Mark the variables as unused. + * + * @param[in] ... Variables which are unused. + */ +template +inline void ignore_unused(T &&...) +{ +} + +} // namespace ckw + +#endif // CKW_INCLUDE_CKW_ERROR_H diff --git a/compute_kernel_writer/prototype/include/ckw/Kernel.h b/compute_kernel_writer/prototype/include/ckw/Kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..ba31a29ba7d882c0e2c1597a763aeb2df861951b --- /dev/null +++ b/compute_kernel_writer/prototype/include/ckw/Kernel.h @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_PROTOTYPE_INCLUDE_CKW_KERNEL_H +#define CKW_PROTOTYPE_INCLUDE_CKW_KERNEL_H + +#include "ckw/KernelArgument.h" +#include "ckw/OperandBase.h" +#include "ckw/types/GpuTargetLanguage.h" + +#include +#include +#include +#include + +namespace ckw +{ + +class TileOperand; + +namespace prototype +{ +class GpuKernelWriterDataHolder; +} // namespace prototype + +/** The target for kernel writer to write into. */ +class Kernel +{ +public: + /** Constructor + * + * @param[in] language The programming language to write the kernel. + */ + Kernel(GpuTargetLanguage language); + /** Constructor + * + * @param[in] name The name of the kernel function. + * @param[in] language The programming language to write the kernel. + */ + Kernel(const char *name, GpuTargetLanguage language); + + /** Destructor */ + ~Kernel(); + + /** Get the name of the kernel function. */ + const std::string &name() const; + + /** Set the name of the kernel function. + * + * @param[in] name The name of the kernel function. + */ + void name(const std::string &name); + + /** Get the list of kernel arguments. */ + ::std::vector arguments() const; + + /** (Internal use only) Register the tile operand. + * + * @param operand The tile operand to be registered. + */ + TileOperand ®ister_operand(::std::unique_ptr operand); + + /** (Internal use only) Register the tensor operand. + * + * @param operand The tensor operand to be registered. + */ + TensorOperand ®ister_operand(::std::unique_ptr operand); + + /** (Internal use only) Get the implementation data. */ + prototype::GpuKernelWriterDataHolder *impl(); + +private: + ::std::string _name; + ::std::unique_ptr _kernel; + ::std::map<::std::string, ::std::unique_ptr> _operands; + ::std::map _tensor_id_operands; +}; + +} // namespace ckw + +#endif // CKW_PROTOTYPE_INCLUDE_CKW_KERNEL_H diff --git a/compute_kernel_writer/prototype/include/ckw/KernelArgument.h b/compute_kernel_writer/prototype/include/ckw/KernelArgument.h new file mode 100644 index 0000000000000000000000000000000000000000..af8bcde6340b62382293c283bba5818e1a309d35 --- /dev/null +++ b/compute_kernel_writer/prototype/include/ckw/KernelArgument.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_PROTOTYPE_INCLUDE_CKW_KERNELARGUMENT_H +#define CKW_PROTOTYPE_INCLUDE_CKW_KERNELARGUMENT_H + +#include "ckw/TensorInfo.h" +#include + +namespace ckw +{ + +class TensorOperand; +class TensorComponentOperand; + +/** A kernel argument which can be either a tensor storage or a tensor component. */ +class KernelArgument +{ +public: + /** The type of kernel argument. */ + enum class Type : int32_t + { + /** The argument that provides the read and/or write access to the tensor data. + * + * See @ref ckw::TensorStorage to see the list of supported storage type. + */ + TensorStorage, + + /** The argument that provides extra information about the tensor. + * + * See @ref ckw::TensorComponent to see the list of supported component. + */ + TensorComponent, + }; + + /** Initialize a new instance of kernel argument class for a tensor storage argument. + * + * @param[in] tensor The tensor whose storage is exposed to kernel arguments. + */ + KernelArgument(TensorOperand &tensor); + + /** Initialize a new instance of kernel argument class for a tensor component argument. + * + * @param[in] tensor_component The tensor component to be exposed to kernel arguments. + */ + KernelArgument(TensorComponentOperand &tensor_component); + + /** Get the type of kernel argument. */ + Type type() const; + + /** Get the argument ID. + * + * This method can be used to get the tensor info ID of both tensor storage and tensor component arguments. + */ + int32_t id() const; + + /** Get the type of tensor storage. + * + * This method can only be used for tensor storage argument. + */ + TensorStorageType tensor_storage_type() const; + + /** Get the tensor component type. + * + * This method can only be used for tensor component argument. + */ + TensorComponentType tensor_component_type() const; + +private: + Type _type; + int32_t _id; + + union SubId + { + int32_t unknown; + TensorStorageType tensor_storage_type; + TensorComponentType tensor_component_type; + }; + + SubId _sub_id{ 0 }; +}; + +} // namespace ckw + +#endif // CKW_PROTOTYPE_INCLUDE_CKW_KERNELARGUMENT_H diff --git a/compute_kernel_writer/prototype/include/ckw/KernelWriter.h b/compute_kernel_writer/prototype/include/ckw/KernelWriter.h new file mode 100644 index 0000000000000000000000000000000000000000..72f85c78aa234b42d5949ade092ce5a121d69b51 --- /dev/null +++ b/compute_kernel_writer/prototype/include/ckw/KernelWriter.h @@ -0,0 +1,320 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_PROTOTYPE_INCLUDE_CKW_KERNELWRITER_H +#define CKW_PROTOTYPE_INCLUDE_CKW_KERNELWRITER_H + +#include "ckw/Kernel.h" +#include "ckw/TensorInfo.h" +#include "ckw/TensorOperand.h" +#include "ckw/TileInfo.h" +#include "ckw/TileOperand.h" +#include "ckw/types/ConvertPolicy.h" +#include "ckw/types/Functions.h" +#include "ckw/types/Operators.h" + +#include + +namespace ckw +{ + +namespace prototype +{ +struct GpuKernelWriterAttribute; + +class IGpuKernelWriter; +} // namespace prototype + +/** Kernel writer. */ +class KernelWriter +{ +public: + // ============================================================================================= + // Constructors and destructor + // ============================================================================================= + + /** Initialize a new instance of kernel writer. + * + * @param[in] kernel The kernel to be written to. + */ + explicit KernelWriter(Kernel &kernel); + + /** Destructor */ + ~KernelWriter(); + + /** No copy constructor. */ + KernelWriter(const KernelWriter &) = delete; + + /** No copy assignment. */ + KernelWriter &operator=(const KernelWriter &) = delete; + + // ============================================================================================= + // Scope management + // ============================================================================================= + + /** Get the current ID space. */ + int32_t id_space() const; + + /** Set the current ID space. */ + KernelWriter &id_space(int32_t id_space); + + /** Switch to and return a new ID space. */ + int32_t next_id_space(); + + // ============================================================================================= + // Tensor and tile declaration + // ============================================================================================= + + /** Declare a tensor argument. + * + * @param[in] name The name of the tensor. + * @param[in] info The tensor info. + * @param[in] storage_type The tensor storage type. + * + * @return The @ref TensorOperand object. + */ + TensorOperand &declare_tensor_argument(const std::string &name, const TensorInfo &info, TensorStorageType storage_type = TensorStorageType::BufferUint8Ptr); + + /** Declare a compile-time constant scalar argument. + * + * @param[in] name The name of the tile. + * @param[in] value The value of the tile. + * + * @return The @ref TileOperand object. + */ + TileOperand &declare_tile_argument(const std::string &name, int32_t value); + + /** Declare a new tile. + * + * The name of the tile must be unique in the current ID space. + * + * @param[in] name The name of the tile. + * @param[in] ... The necessary arguments to create a new @ref TileOperand. + * + * @return The @ref TileOperand object. + */ + template + TileOperand &declare_tile(const std::string &name, TArgs &&...args) + { + const auto var_name = generate_variable_name(name); + auto operand = std::make_unique(var_name, ::std::forward(args)...); + + return declare_tile_operand(std::move(operand)); + } + + // ============================================================================================= + // Load and store + // ============================================================================================= + + /** Load the data from the tensor memory to the tile using the sampling information. + * + * @param[out] tile The tile to be loaded. + * @param[in] tensor The tensor to be read. + * @param[in] sampler The tensor sampling information. + * @param[in] dilation_y Dilation in the Y dimension. + */ + void op_load(TileOperand &tile, const TensorOperand &tensor, const TensorTileSampler &sampler, const TileOperand &dilation_y = TileOperand("dil_y", 1)); + + /** Load the data from the tensor memory to the tile using the indirect buffer approach and respective of the sampling information. + * + * @param[out] tile The tile to be loaded. + * @param[in] tensor The tensor to be read. + * @param[in] sampler The tensor sampling information. + */ + void op_load_indirect(TileOperand &tile, const TensorOperand &tensor, const TensorTileSampler &sampler); + + /** Construct an indirection buffer in @p tile containing the precalculated addresses of elements in the source tensor. + * + * @param[out] tile The tile to be loaded. + * @param[in] tensor The tensor the be read. + * @param[in] sampler The tensor sampling information. + * @param[in] x The X coordinate. + * @param[in] y The Y coordinate. + * @param[in] x_off Offset in the X dimension. + * @param[in] y_off Offset in the Y dimension. + */ + void util_get_indirect_buffer(TileOperand &tile, + const TensorOperand &tensor, + const TensorTileSampler &sampler, + const TileOperand &x, + const TileOperand &y, + const TileOperand &x_off, + const TileOperand &y_off); + + /** Store the tile to the tensor using the specified sampling information. + * + * @param[out] dst The tensor that the tile is written to. + * @param[in] src The tile to be stored. + * @param[in] sampler The tensor sampling information. + */ + void op_store(TensorOperand &tensor, const TileOperand &tile, const TensorTileSampler &sampler); + + // ============================================================================================= + // Data processing + // ============================================================================================= + + /** Write assignment: ` = ;`. + * + * @param[out] dst The destination tile. + * @param[in] src The source tile. + */ + void op_assign(const TileOperand &dst, const TileOperand &src); + + /** Write the cast: ` = convert_<_sat>();`. + * + * @param[out] dst The destination tile. + * @param[in] src The source tile. + * @param[in] policy The policy governing the behavior of the cast. + */ + void op_cast_expression(const TileOperand &dst, const TileOperand &src, ConvertPolicy policy); + + /** Write the unary expression: ` = `. + * + * @param[out] dst The destination tile. + * @param[in] op The unary operator. + * @param[in] src The source tile. + */ + void op_unary_expression(const TileOperand &dst, UnaryOp op, const TileOperand &src); + + /** Write binary expression: ` = ;`. + * + * @param[out] dst The destination tile. + * @param[in] lhs The LHS tile. + * @param[in] op The binary operator. + * @param[in] rhs The RHS tile. + */ + void op_binary_expression(const TileOperand &dst, const TileOperand &lhs, BinaryOp op, const TileOperand &rhs); + + /** Write function applied to scalar value: ` = ();`. + * + * @param[out] dst The destination tile. + * @param[in] func The function to be applied to the source tile. + * @param[in] src The source tile. + */ + void op_unary_elementwise_function(const TileOperand &dst, UnaryFunction func, const TileOperand &src); + + /** Write function applied to scalar value: ` = (, );`. + * + * @param[out] dst The destination tile. + * @param[in] func The function to be applied to the source tiles. + * @param[in] first The first argument tile. + * @param[in] second The second argument tile. + */ + void op_binary_elementwise_function(const TileOperand &dst, BinaryFunction func, const TileOperand &first, const TileOperand &second); + + /** Write function applied to scalar value: ` = (, , );`. + * + * @param[out] dst The destination tile. + * @param[in] func The function to be applied to the source tiles. + * @param[in] first The first argument tile. + * @param[in] second The second argument tile. + * @param[in] third The third argument tile. + */ + void op_ternary_elementwise_function(const TileOperand &dst, TernaryFunction func, const TileOperand &first, const TileOperand &second, const TileOperand &third); + + /** Write if-statement: `if( ) { }`. + * + * @param[in] lhs The LHS tile of the condition. + * @param[in] op The relational binary operator. + * @param[in] rhs The RHS tile of the condition. + * @param[in] body The body of the if-statement. + */ + void op_if(const TileOperand &lhs, BinaryOp op, const TileOperand &rhs, const std::function &body); + + /** Write else-if-statement: `else if( ) { }`. + * + * @param[in] lhs The LHS tile of the condition. + * @param[in] op The relational binary operator. + * @param[in] rhs The RHS tile of the condition. + * @param[in] body The body of the else-if-statement. + */ + void op_else_if(const TileOperand &lhs, BinaryOp op, const TileOperand &rhs, const std::function &body); + + /** Write an else-statement: `else { }`. + * + * @param[in] body The body of the else-statement. + */ + void op_else(const std::function &body); + + /** Write for-loops: `for(; ; ) { body }`. + * + * @param[in] var_name The name of the variable used in condition. + * @param[in] cond_op The relational binary operator used in condition. + * @param[in] cond_value_name The value which the variable is compared against. + * @param[in] update_var_name The name of the variable which is updated. + * @param[in] update_op The assignment operator used for updating the update value. + * @param[in, out] update_value The value which is updated at every iteration. + * @param[in] body The body of the for-loop. + */ + void op_for_loop(const TileOperand &var_name, BinaryOp cond_op, const TileOperand &cond_value_name, const TileOperand &update_var_name, AssignmentOp update_op, const TileOperand &update_value_name, const std::function &body); + + /** Write the return statement: `return;` + */ + void op_return(); + + // ============================================================================================= + // Misc + // ============================================================================================= + + /** Set `dst` the global ID of dimension `dim`. + * + * @param[out] dst The tile to be written to. + * @param[in] dim The global ID dimension. + */ + void op_get_global_id(TileOperand &dst, int32_t dim); + + // ============================================================================================= + // Code generation + // ============================================================================================= + + /** Generate the source code of the kernel. */ + ::std::string generate_code(); + +private: + /** Generate the full variable name based on the original name and the ID space. + * + * @param[in] name The name of the variable. + * + * @return The full variable name. + */ + ::std::string generate_variable_name(const std::string &name) const; + + /** Declare the tile operand. + * + * @param[in] operand The tile operand to be declared. + */ + TileOperand &declare_tile_operand(std::unique_ptr operand); + +private: + Kernel *_kernel; + ::std::unique_ptr _impl_attr; + ::std::unique_ptr _impl; + + int32_t _id_space{ 0 }; + int32_t _max_id_space{ 0 }; +}; + +} // namespace ckw + +#endif // CKW_PROTOTYPE_INCLUDE_CKW_KERNELWRITER_H diff --git a/compute_kernel_writer/prototype/include/ckw/KernelWriterHelper.h b/compute_kernel_writer/prototype/include/ckw/KernelWriterHelper.h new file mode 100644 index 0000000000000000000000000000000000000000..a8be859680ca03885c1c428cda1df1515678a65b --- /dev/null +++ b/compute_kernel_writer/prototype/include/ckw/KernelWriterHelper.h @@ -0,0 +1,1268 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_INCLUDE_CKW_KERNELWRITERHELPER_H +#define CKW_INCLUDE_CKW_KERNELWRITERHELPER_H + +#include "ckw/KernelWriter.h" +#include "ckw/TensorOperand.h" +#include "ckw/TileOperand.h" + +#include +#include + +#include + +/* + * By including this header file you will be able to supplement the default + * Compute Kernel Writer API with additional syntax to help ease the use of CKW. + * + * To use the KernelWriterHelper you need to wrap your instance of KernelWriter + * (or any class deriving from KernelWriter): + * KernelWriterHelper writer; + * The resulting writer object comprises the original KernelWriter + * functionality (drop-in replacement), but extends the syntax as follows. + * + * Common functions/operators have natural syntax: + * 1. Unary expressions: + * writer.op_assign(dst, !src); // Logical NOT + * writer.op_assign(dst, ~src); // Bitwise NOT + * + * 2. Binary expressions: + * writer.op_assign(dst, lhs + rhs); // Addition + * writer.op_assign(dst, lhs - rhs); // Subtraction + * writer.op_assign(dst, lhs * rhs); // Multiplication + * writer.op_assign(dst, lhs / rhs); // Division + * writer.op_assign(dst, lhs % rhs); // Modulo + * writer.op_assign(dst, lhs == rhs); // Equality + * writer.op_assign(dst, lhs < rhs); // Less-than + * writer.op_assign(dst, lhs <= rhs); // Less-than-or-equal + * writer.op_assign(dst, lhs > rhs); // Greater-than + * writer.op_assign(dst, lhs >= rhs); // Greater-than-or-equal + * writer.op_assign(dst, lhs ^ rhs); // Bitwise XOR + * writer.op_assign(dst, logical_and(lhs, rhs)); // Logical AND + * writer.op_assign(dst, logical_or(lhs, rhs)); // Logical OR + * + * 3. Unary elementwise functions: + * writer.op_assign(dst, exp(src)); // Exponent + * writer.op_assign(dst, tanh(src)); // Hyperbolic tangent + * writer.op_assign(dst, sqrt(src)); // Square root + * writer.op_assign(dst, erf(src)); // Error function + * writer.op_assign(dst, fabs(src)); // Absolute of floating-point number + * writer.op_assign(dst, log(src)); // Natural logarithm + * writer.op_assign(dst, round(src)); // Round + * writer.op_assign(dst, sizeOf(src)); // sizeof + * + * 4. Binary elementwise functions: + * writer.op_assign(dst, max(first, second)); // Max + * writer.op_assign(dst, min(first, second)); // Min + * + * 5. Ternary elementwise functions: + * writer.op_assign(dst, select(first, second, third)); // Select + * + * NOTE: All the above examples support nesting, so you could write + * something like: writer.op_assign(dst, src * (log(arg) + sqrt(abs(arg))); + * + * + * 6. If-statements. The preceding syntax also allows easier writing of if-statements: + * writer.op_if(, ); + * + * For example: + * writer.op_if(exp(first_arg) == dst, [&]{ + * //... + * }).op_else_if(exp(first_arg) > dst, [&]{ + * //... + * }).op_else([&] { + * //... + * }); + * + * 7. For-loops. A similar syntax exists for for-loops: + * writer.op_for_loop(, , ); + * + * For example: + * writer.op_for_loop(index < limit, index += step, [&]{ + * //... + * }); + * + * NOTE: There are limitations on the for-loop and parameters. + * In neither the (Binary expression) or (Increment/Decrement) + * is it allowed to use nesting. For example, `(index + other) < limit` and + * `index < round(limit)` are invalid parameters. This is because the + * semantics of for-loops rely on the condition being evaluated at every iteration, + * but as temporary variables might be defined for nested expressions the semantics + * cannot be guaranteed. + */ + +namespace ckw +{ + +// ================================================== +// Type traits +// ================================================== + +/** Specifies if the type can be used as an operand for functions (e.g. max), operations (e.g. *), or assignments. */ +template +struct can_be_operand : ::std::false_type +{ +}; + +/** Specifies if the type can be assigned/written to. */ +template +struct can_be_assigned : ::std::false_type +{ +}; + +template <> +struct can_be_operand : ::std::true_type +{ +}; + +template <> +struct can_be_assigned : ::std::true_type +{ +}; + +// ================================================== +// Assignment +// ================================================== + +/** AST node for assignments. + * + * Note that \p TRight must be an operand, and \p TLeft must be assignable. + * + * @tparam TLeft The type of the destination of the assignment. + * @tparam TRight The type of the source assigned to the destination. + */ +template ::value && can_be_assigned::value>> +struct Assignment +{ + TLeft lhs; + TRight rhs; + AssignmentOp opcode; +}; + +/** Represents the expression: `\p lhs += \p rhs`. + * + * @tparam TLeft The type of the LHS of the assignment. + * @tparam TRight The type of the RHS of the assignment. + * @param[in] lhs The LHS of the assignment. + * @param[in] rhs The RHS of the assignment. + * @return The resulting AST node. + */ +template +inline Assignment operator+=(TLeft &&lhs, TRight &&rhs) +{ + return Assignment{ std::forward(lhs), std::forward(rhs), AssignmentOp::Increment }; +} + +/** Represents the expression: `\p lhs -= \p rhs`. + * + * @tparam TLeft The type of the LHS of the assignment. + * @tparam TRight The type of the RHS of the assignment. + * @param[in] lhs The LHS of the assignment. + * @param[in] rhs The RHS of the assignment. + * @return The resulting AST node. + */ +template +inline Assignment operator-=(TLeft &&lhs, TRight &&rhs) +{ + return Assignment{ std::forward(lhs), std::forward(rhs), AssignmentOp::Decrement }; +} + +// ================================================== +// Unary expression +// ================================================== + +/** AST node for unary expressions. + * + * Note that \p TSrc must be an operand. + * + * @tparam TSrc The type of the argument to the expression. + */ +template ::value>> +struct UnaryExpression +{ + TSrc src; + UnaryOp opcode; +}; + +template +struct can_be_operand> : ::std::true_type +{ +}; + +/** Represents the expression: `!\p src`. + * + * @tparam TSrc The type of the argument. + * @param[in] src The argument. + * @return The resulting AST node. + */ +template +inline UnaryExpression operator!(TSrc &&src) +{ + return UnaryExpression{ std::forward(src), UnaryOp::LogicalNot }; +} + +/** Represents the expression: `~\p src`. + * + * @tparam TSrc The type of the argument. + * @param[in] src The argument. + * @return The resulting AST node. + */ +template +inline UnaryExpression operator~(TSrc &&src) +{ + return UnaryExpression{ std::forward(src), UnaryOp::BitwiseNot }; +} + +// ================================================== +// Binary expressions +// ================================================== + +/** AST node for binary expressions. + * + * Note that both \p TLeft and \p TRight must be operands. + * + * @tparam TLeft The type of the left argument of the expression. + * @tparam TRight The type of the right argument of the expression. + */ +template ::value && can_be_operand::value>> +struct BinaryExpression +{ + TLeft lhs; + TRight rhs; + BinaryOp opcode; +}; + +template +struct can_be_operand> : ::std::true_type +{ +}; + +/** Represents the expression: `\p lhs + \p rhs`. + * + * @tparam TLeft The type of the LHS of the expression. + * @tparam TRight The type of the RHS of the expression. + * @param[in] lhs The LHS of the expression. + * @param[in] rhs The RHS of the expression. + * @return The resulting AST node. + */ +template +inline BinaryExpression operator+(TLeft &&lhs, TRight &&rhs) +{ + return BinaryExpression{ std::forward(lhs), std::forward(rhs), BinaryOp::Add }; +} + +/** Represents the expression: `\p lhs - \p rhs`. + * + * @tparam TLeft The type of the LHS of the expression. + * @tparam TRight The type of the RHS of the expression. + * @param[in] lhs The LHS of the expression. + * @param[in] rhs The RHS of the expression. + * @return The resulting AST node. + */ +template +inline BinaryExpression operator-(TLeft &&lhs, TRight &&rhs) +{ + return BinaryExpression{ std::forward(lhs), std::forward(rhs), BinaryOp::Sub }; +} + +/** Represents the expression: `\p lhs * \p rhs`. + * + * @tparam TLeft The type of the LHS of the expression. + * @tparam TRight The type of the RHS of the expression. + * @param[in] lhs The LHS of the expression. + * @param[in] rhs The RHS of the expression. + * @return The resulting AST node. + */ +template +inline BinaryExpression operator*(TLeft &&lhs, TRight &&rhs) +{ + return BinaryExpression{ std::forward(lhs), std::forward(rhs), BinaryOp::Mul }; +} + +/** Represents the expression: `\p lhs / \p rhs`. + * + * @tparam TLeft The type of the LHS of the expression. + * @tparam TRight The type of the RHS of the expression. + * @param[in] lhs The LHS of the expression. + * @param[in] rhs The RHS of the expression. + * @return The resulting AST node. + */ +template +inline BinaryExpression operator/(TLeft &&lhs, TRight &&rhs) +{ + return BinaryExpression{ std::forward(lhs), std::forward(rhs), BinaryOp::Div }; +} + +/** Represents the expression: `\p lhs % \p rhs`. + * + * @tparam TLeft The type of the LHS of the expression. + * @tparam TRight The type of the RHS of the expression. + * @param[in] lhs The LHS of the expression. + * @param[in] rhs The RHS of the expression. + * @return The resulting AST node. + */ +template +inline BinaryExpression operator%(TLeft &&lhs, TRight &&rhs) +{ + return BinaryExpression{ std::forward(lhs), std::forward(rhs), BinaryOp::Mod }; +} + +/** Represents the expression: `\p lhs == \p rhs`. + * + * @tparam TLeft The type of the LHS of the expression. + * @tparam TRight The type of the RHS of the expression. + * @param[in] lhs The LHS of the expression. + * @param[in] rhs The RHS of the expression. + * @return The resulting AST node. + */ +template +inline BinaryExpression operator==(TLeft &&lhs, TRight &&rhs) +{ + return BinaryExpression{ std::forward(lhs), std::forward(rhs), BinaryOp::Equal }; +} + +/** Represents the expression: `\p lhs < \p rhs`. + * + * @tparam TLeft The type of the LHS of the expression. + * @tparam TRight The type of the RHS of the expression. + * @param[in] lhs The LHS of the expression. + * @param[in] rhs The RHS of the expression. + * @return The resulting AST node. + */ +template +inline BinaryExpression operator<(TLeft &&lhs, TRight &&rhs) +{ + return BinaryExpression{ std::forward(lhs), std::forward(rhs), BinaryOp::Less }; +} + +/** Represents the expression: `\p lhs <= \p rhs`. + * + * @tparam TLeft The type of the LHS of the expression. + * @tparam TRight The type of the RHS of the expression. + * @param[in] lhs The LHS of the expression. + * @param[in] rhs The RHS of the expression. + * @return The resulting AST node. + */ +template +inline BinaryExpression operator<=(TLeft &&lhs, TRight &&rhs) +{ + return BinaryExpression{ std::forward(lhs), std::forward(rhs), BinaryOp::LessEqual }; +} + +/** Represents the expression: `\p lhs > \p rhs`. + * + * @tparam TLeft The type of the LHS of the expression. + * @tparam TRight The type of the RHS of the expression. + * @param[in] lhs The LHS of the expression. + * @param[in] rhs The RHS of the expression. + * @return The resulting AST node. + */ +template +inline BinaryExpression operator>(TLeft &&lhs, TRight &&rhs) +{ + return BinaryExpression{ std::forward(lhs), std::forward(rhs), BinaryOp::Greater }; +} + +/** Represents the expression: `\p lhs >= \p rhs`. + * + * @tparam TLeft The type of the LHS of the expression. + * @tparam TRight The type of the RHS of the expression. + * @param[in] lhs The LHS of the expression. + * @param[in] rhs The RHS of the expression. + * @return The resulting AST node. + */ +template +inline BinaryExpression operator>=(TLeft &&lhs, TRight &&rhs) +{ + return BinaryExpression{ std::forward(lhs), std::forward(rhs), BinaryOp::GreaterEqual }; +} + +/** Represents the expression: `\p lhs ^ \p rhs`. + * + * @tparam TLeft The type of the LHS of the expression. + * @tparam TRight The type of the RHS of the expression. + * @param[in] lhs The LHS of the expression. + * @param[in] rhs The RHS of the expression. + * @return The resulting AST node. + */ +template +inline BinaryExpression operator^(TLeft &&lhs, TRight &&rhs) +{ + return BinaryExpression{ std::forward(lhs), std::forward(rhs), BinaryOp::BitwiseXOR }; +} + +/** Represents the expression: `\p lhs && \p rhs`. + * + * @tparam TLeft The type of the LHS of the expression. + * @tparam TRight The type of the RHS of the expression. + * @param[in] lhs The LHS of the expression. + * @param[in] rhs The RHS of the expression. + * @return The resulting AST node. + */ +template +inline BinaryExpression logical_and(TLeft &&lhs, TRight &&rhs) +{ + return BinaryExpression{ std::forward(lhs), std::forward(rhs), BinaryOp::LogicalAnd }; +} + +/** Represents the expression: `\p lhs && \p rhs`. + * + * @tparam TLeft The type of the LHS of the expression. + * @tparam TRight The type of the RHS of the expression. + * @param[in] lhs The LHS of the expression. + * @param[in] rhs The RHS of the expression. + * @return The resulting AST node. + */ +template +inline BinaryExpression, TOps...> logical_and(TLeft &&lhs, TRight &&rhs, TOps &&...ops) +{ + return logical_and( + BinaryExpression{ std::forward(lhs), std::forward(rhs), BinaryOp::LogicalAnd }, + std::forward(ops)...); +} + +/** Represents the expression: `\p lhs || \p rhs`. + * + * @tparam TLeft The type of the LHS of the expression. + * @tparam TRight The type of the RHS of the expression. + * @param[in] lhs The LHS of the expression. + * @param[in] rhs The RHS of the expression. + * @return The resulting AST node. + */ +template +inline BinaryExpression logical_or(TLeft &&lhs, TRight &&rhs) +{ + return BinaryExpression{ std::forward(lhs), std::forward(rhs), BinaryOp::LogicalOr }; +} + +/** Represents the expression: `\p lhs || \p rhs`. + * + * @tparam TLeft The type of the LHS of the expression. + * @tparam TRight The type of the RHS of the expression. + * @param[in] lhs The LHS of the expression. + * @param[in] rhs The RHS of the expression. + * @return The resulting AST node. + */ +template +inline BinaryExpression, TOps...> logical_or(TLeft &&lhs, TRight &&rhs, TOps &&...ops) +{ + return logical_or( + BinaryExpression{ std::forward(lhs), std::forward(rhs), BinaryOp::LogicalOr }, + std::forward(ops)...); +} + +// ================================================== +// Unary elementwise functions +// ================================================== + +/** AST node for unary elementwise functions. + * + * Note that \p TSrc must be an operand. + * + * @tparam TSrc The type of the argument to the function. + */ +template ::value>> +struct UnaryElementwiseFunction +{ + TSrc src; + UnaryFunction opcode; +}; + +template +struct can_be_operand> : ::std::true_type +{ +}; + +/** Represents the expression: `exp(\p src)`. + * + * @tparam TSrc The type of the argument. + * @param[in] src The argument. + * @return The resulting AST node. + */ +template +UnaryElementwiseFunction exp(TSrc &&src) +{ + return UnaryElementwiseFunction{ std::forward(src), UnaryFunction::Exp }; +} + +/** Represents the expression: `tanh(\p src)`. + * + * @tparam TSrc The type of the argument. + * @param[in] src The argument. + * @return The resulting AST node. + */ +template +UnaryElementwiseFunction tanh(TSrc &&src) +{ + return UnaryElementwiseFunction{ std::forward(src), UnaryFunction::Tanh }; +} + +/** Represents the expression: `sqrt(\p src)`. + * + * @tparam TSrc The type of the argument. + * @param[in] src The argument. + * @return The resulting AST node. + */ +template +UnaryElementwiseFunction sqrt(TSrc &&src) +{ + return UnaryElementwiseFunction{ std::forward(src), UnaryFunction::Sqrt }; +} + +/** Represents the expression: `erf(\p src)`. + * + * @tparam TSrc The type of the argument. + * @param[in] src The argument. + * @return The resulting AST node. + */ +template +UnaryElementwiseFunction erf(TSrc &&src) +{ + return UnaryElementwiseFunction{ std::forward(src), UnaryFunction::Erf }; +} + +/** Represents the expression: `fabs(\p src)`. + * + * @tparam TSrc The type of the argument. + * @param[in] src The argument. + * @return The resulting AST node. + */ +template +UnaryElementwiseFunction fabs(TSrc &&src) +{ + return UnaryElementwiseFunction{ std::forward(src), UnaryFunction::Fabs }; +} + +/** Represents the expression: `log(\p src)`. + * + * @tparam TSrc The type of the argument. + * @param[in] src The argument. + * @return The resulting AST node. + */ +template +UnaryElementwiseFunction log(TSrc &&src) +{ + return UnaryElementwiseFunction{ std::forward(src), UnaryFunction::Log }; +} + +/** Represents the expression: `round(\p src)`. + * + * @tparam TSrc The type of the argument. + * @param[in] src The argument. + * @return The resulting AST node. + */ +template +UnaryElementwiseFunction round(TSrc &&src) +{ + return UnaryElementwiseFunction{ std::forward(src), UnaryFunction::Round }; +} + +/** Represents the expression: `sizeof(\p src)`. + * + * @tparam TSrc The type of the argument. + * @param[in] src The argument. + * @return The resulting AST node. + */ +template +UnaryElementwiseFunction sizeOf(TSrc &&src) +{ + return UnaryElementwiseFunction{ std::forward(src), UnaryFunction::SizeOf }; +} + +// ================================================== +// Binary elementwise functions +// ================================================== + +/** AST node for binary elementwise functions. + * + * Note that both \p TFirst and \p TSecond must be operands. + * + * @tparam TFirst The type of the left argument of the function. + * @tparam TSecond The type of the right argument of the function. + */ +template ::value && can_be_operand::value>> +struct BinaryElementwiseFunction +{ + TFirst first; + TSecond second; + BinaryFunction opcode; +}; + +template +struct can_be_operand> : ::std::true_type +{ +}; + +/** Represents the function call: `max(\p first, \p second)`. + * + * @tparam TFirst The type of the first argument. + * @tparam TSecond The type of the second argument. + * @param[in] first The first argument. + * @param[in] second The second argument. + * @return The resulting AST node. + */ +template +BinaryElementwiseFunction max(TFirst &&first, TSecond &&second) +{ + return BinaryElementwiseFunction{ std::forward(first), std::forward(second), BinaryFunction::Max }; +} + +/** Represents the function call: `min(\p first, \p second)`. + * + * @tparam TFirst The type of the first argument. + * @tparam TSecond The type of the second argument. + * @param[in] first The first argument. + * @param[in] second The second argument. + * @return The resulting AST node. + */ +template +BinaryElementwiseFunction min(TFirst &&first, TSecond &&second) +{ + return BinaryElementwiseFunction{ std::forward(first), std::forward(second), BinaryFunction::Min }; +} + +// ================================================== +// Ternary elementwise functions +// ================================================== + +/** AST node for ternary elementwise functions. + * + * Note that \p TFirst, \p TSecond, and \p TThird all must be operands. + * + * @tparam TFirst The type of the first argument to the function. + * @tparam TSecond The type of the second argument to the function. + * @tparam TThird The type of the third argument to the function. + */ +template ::value && can_be_operand::value && can_be_operand::value>> +struct TernaryElementwiseFunction +{ + TFirst first; + TSecond second; + TThird third; + TernaryFunction opcode; +}; + +template +struct can_be_operand> : ::std::true_type +{ +}; + +/** Represents the function call: `select(\p first, \p second, \p third)`. + * + * @tparam TFirst The type of the first argument. + * @tparam TSecond The type of the second argument. + * @tparam TThird The type of the third argument. + * @param[in] first The first argument. + * @param[in] second The second argument. + * @param[in] third The third argument. + * @return The resulting AST node. + */ +template +TernaryElementwiseFunction select(TFirst &&first, TSecond &&second, TThird &&third) +{ + return TernaryElementwiseFunction{ std::forward(first), std::forward(second), std::forward(third), TernaryFunction::Select }; +} + +/** Helper class used to extend a KernelWriter with additional functionality + * in order to make writing easier. + * + * This extension automatically handles creation of temporary variables, and + * allows nested function calls and operations. + * + * @tparam TWriter The type of KernelWriter to be overloaded. This must inherit from KernelWriter. + */ +template ::value>> +class KernelWriterHelper : public TWriter +{ +public: + using TWriter::TWriter; + + // ================================================== + // If-statements + // ================================================== + + // Un-hide original implementation, in case the original implementation is required. + using TWriter::op_if; + + /** Represents the if-statement: `if(\p cond) { \p body }`. + * + * The BinaryExpression is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] cond The BinaryExpression representing the condition. + * @param[in] body The body of the if-statement. + */ + KernelWriterHelper &op_if(const BinaryExpression &cond, const std::function &body) + { + TWriter::op_if(cond.lhs, cond.opcode, cond.rhs, body); + return *this; + } + + /** Represents the if-statement: `if(\p cond) { \p body }`. + * + * The BinaryExpression is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] cond The BinaryExpression representing the condition. + * @param[in] body The body of the if-statement. + */ + template + KernelWriterHelper &op_if(const BinaryExpression &cond, const std::function &body) + { + auto &tmp1 = declare_temp_tile(cond.lhs.tile_info()); + op_assign(tmp1, cond.rhs); + TWriter::op_if(cond.lhs, cond.opcode, tmp1, body); + return *this; + } + + /** Represents the if-statement: `if(\p cond) { \p body }`. + * + * The BinaryExpression is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] cond The BinaryExpression representing the condition. + * @param[in] body The body of the if-statement. + */ + template + KernelWriterHelper &op_if(const BinaryExpression &cond, const std::function &body) + { + auto &tmp1 = declare_temp_tile(cond.rhs.tile_info()); + op_assign(tmp1, cond.lhs); + TWriter::op_if(tmp1, cond.opcode, cond.rhs, body); + return *this; + } + + // Un-hide original implementation, in case the original implementation is required. + using TWriter::op_else_if; + + /** Represents the else-if-statement: `else if(\p cond) { \p body }`. + * + * The BinaryExpression is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] cond The BinaryExpression representing the condition. + * @param[in] body The body of the else-if-statement. + */ + KernelWriterHelper &op_else_if(const BinaryExpression &cond, const std::function &body) + { + TWriter::op_else_if(cond.lhs, cond.opcode, cond.rhs, body); + return *this; + } + + /** Represents the else-if-statement: `else if(\p cond) { \p body }`. + * + * The BinaryExpression is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] cond The BinaryExpression representing the condition. + * @param[in] body The body of the else-if-statement. + */ + template + KernelWriterHelper &op_else_if(const BinaryExpression &cond, const std::function &body) + { + auto &tmp1 = declare_temp_tile(cond.lhs.tile_info()); + op_assign(tmp1, cond.rhs); + TWriter::op_else_if(cond.lhs, cond.opcode, tmp1, body); + return *this; + } + + /** Represents the else-if-statement: `else if(\p cond) { \p body }`. + * + * The BinaryExpression is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] cond The BinaryExpression representing the condition. + * @param[in] body The body of the else-if-statement. + */ + template + KernelWriterHelper &op_else_if(const BinaryExpression &cond, const std::function &body) + { + auto &tmp1 = declare_temp_tile(cond.rhs.tile_info()); + op_assign(tmp1, cond.lhs); + TWriter::op_else_if(tmp1, cond.opcode, cond.rhs, body); + return *this; + } + + // ================================================== + // For-loops + // ================================================== + + // Un-hide original implementation, in case the original implementation is required. + using TWriter::op_for_loop; + + /** Represents the for-loop: `for(;\p cond; \p updater) { \p body }`. + * + * The BinaryExpression for the condition and the Assignment + * for the updater are unpacked and their components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] cond The BinaryExpression representing the condition. + * @param[in] updater The Assignment representing the updater. + * @param[in] body The body of the for-loop. + */ + void op_for_loop(const BinaryExpression &cond, const Assignment &updater, const std::function &body) + { + TWriter::op_for_loop(cond.lhs, cond.opcode, cond.rhs, updater.lhs, updater.opcode, updater.rhs, body); + } + + // ================================================== + // Unary expressions + // ================================================== + + // Un-hide original implementation, in case the original implementation is required. + using TWriter::op_assign; + + /** Represents the assignment: `\p dst = \p exp`. + * + * The UnaryExpression is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] dst The tile which is assigned to. + * @param[in] exp The UnaryExpression representing the expression to be evaluated and assigned. + */ + void op_assign(const TileOperand &dst, const UnaryExpression &exp) + { + TWriter::op_unary_expression(dst, exp.opcode, exp.src); + } + + // ================================================== + // Binary expressions + // ================================================== + + /** Represents the assignment: `\p dst = \p exp`. + * + * The BinaryExpression is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] dst The tile which is assigned to. + * @param[in] exp The BinaryExpression representing the expression to be evaluated and assigned. + */ + void op_assign(const TileOperand &dst, const BinaryExpression &exp) + { + TWriter::op_binary_expression(dst, exp.lhs, exp.opcode, exp.rhs); + } + + /** Represents the assignment: `\p dst = \p exp`. + * + * The BinaryExpression is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] dst The tile which is assigned to. + * @param[in] exp The BinaryExpression representing the expression to be evaluated and assigned. + */ + template + void op_assign(const TileOperand &dst, const BinaryExpression &exp) + { + std::cout << "Beginning assignment!" << std::endl; + auto &tmp1 = declare_temp_tile(dst.tile_info()); + op_assign(tmp1, exp.rhs); + TWriter::op_binary_expression(dst, exp.lhs, exp.opcode, tmp1); + } + + /** Represents the assignment: `\p dst = \p exp`. + * + * The BinaryExpression is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] dst The tile which is assigned to. + * @param[in] exp The BinaryExpression representing the expression to be evaluated and assigned. + */ + template + void op_assign(const TileOperand &dst, const BinaryExpression &exp) + { + std::cout << "Beginning assignment!" << std::endl; + auto &tmp1 = declare_temp_tile(dst.tile_info()); + op_assign(tmp1, exp.lhs); + TWriter::op_binary_expression(dst, tmp1, exp.opcode, exp.rhs); + } + + /** Represents the assignment: `\p dst = \p exp`. + * + * The BinaryExpression is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] dst The tile which is assigned to. + * @param[in] exp The BinaryExpression representing the expression to be evaluated and assigned. + */ + template + void op_assign(const TileOperand &dst, const BinaryExpression &exp) + { + auto &tmp1 = declare_temp_tile(dst.tile_info()); + auto &tmp2 = declare_temp_tile(dst.tile_info()); + op_assign(tmp1, exp.lhs); + op_assign(tmp2, exp.rhs); + TWriter::op_binary_expression(dst, tmp1, exp.opcode, tmp2); + } + + // ================================================== + // Unary elementwise functions + // ================================================== + + /** Represents the assignment: `\p dst = \p exp`. + * + * The UnaryElementwiseFunction is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] dst The tile which is assigned to. + * @param[in] exp The UnaryElementwiseFunction representing the expression to be evaluated and assigned. + */ + void op_assign(const TileOperand &dst, const UnaryElementwiseFunction &exp) + { + TWriter::op_unary_elementwise_function(dst, exp.opcode, exp.src); + } + + /** Represents the assignment: `\p dst = \p exp`. + * + * The UnaryElementwiseFunction is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] dst The tile which is assigned to. + * @param[in] exp The UnaryElementwiseFunction representing the expression to be evaluated and assigned. + */ + template + void op_assign(const TileOperand &dst, const UnaryElementwiseFunction &exp) + { + auto &tmp1 = declare_temp_tile(dst.tile_info()); + op_assign(tmp1, exp.lhs); + TWriter::op_unary_elementwise_function(dst, exp.opcode, tmp1); + } + + // ================================================== + // Binary elementwise functions + // ================================================== + + /** Represents the assignment: `\p dst = \p exp`. + * + * The BinaryElementwiseFunction is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] dst The tile which is assigned to. + * @param[in] exp The BinaryElementwiseFunction representing the expression to be evaluated and assigned. + */ + void op_assign(const TileOperand &dst, const BinaryElementwiseFunction &exp) + { + TWriter::op_binary_elementwise_function(dst, exp.opcode, exp.first, exp.second); + } + + /** Represents the assignment: `\p dst = \p exp`. + * + * The BinaryElementwiseFunction is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] dst The tile which is assigned to. + * @param[in] exp The BinaryElementwiseFunction representing the expression to be evaluated and assigned. + */ + template + void op_assign(const TileOperand &dst, const BinaryElementwiseFunction &exp) + { + auto &tmp1 = declare_temp_tile(dst.tile_info()); + op_assign(tmp1, exp.second); + TWriter::op_binary_elementwise_function(dst, exp.opcode, exp.first, tmp1); + } + + /** Represents the assignment: `\p dst = \p exp`. + * + * The BinaryElementwiseFunction is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] dst The tile which is assigned to. + * @param[in] exp The BinaryElementwiseFunction representing the expression to be evaluated and assigned. + */ + template + void op_assign(const TileOperand &dst, const BinaryElementwiseFunction &exp) + { + auto &tmp1 = declare_temp_tile(dst.tile_info()); + op_assign(tmp1, exp.first); + TWriter::op_binary_elementwise_function(dst, exp.opcode, tmp1, exp.second); + } + + /** Represents the assignment: `\p dst = \p exp`. + * + * The BinaryElementwiseFunction is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] dst The tile which is assigned to. + * @param[in] exp The BinaryElementwiseFunction representing the expression to be evaluated and assigned. + */ + template + void op_assign(const TileOperand &dst, const BinaryElementwiseFunction &exp) + { + auto &tmp1 = declare_temp_tile(dst.tile_info()); + auto &tmp2 = declare_temp_tile(dst.tile_info()); + op_assign(tmp1, exp.first); + op_assign(tmp2, exp.second); + TWriter::op_binary_elementwise_function(dst, exp.opcode, tmp1, tmp2); + } + + // ================================================== + // Ternary elementwise functions + // ================================================== + + /** Represents the assignment: `\p dst = \p exp`. + * + * The TernaryElementwiseFunction is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] dst The tile which is assigned to. + * @param[in] exp The TernaryElementwiseFunction representing the expression to be evaluated and assigned. + */ + void op_assign(const TileOperand &dst, const TernaryElementwiseFunction &exp) + { + TWriter::op_ternary_elementwise_function(dst, exp.opcode, exp.first, exp.second, exp.third); + } + + /** Represents the assignment: `\p dst = \p exp`. + * + * The TernaryElementwiseFunction is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] dst The tile which is assigned to. + * @param[in] exp The TernaryElementwiseFunction representing the expression to be evaluated and assigned. + */ + template + void op_assign(const TileOperand &dst, const TernaryElementwiseFunction &exp) + { + auto &tmp1 = declare_temp_tile(dst.tile_info()); + op_assign(tmp1, exp.first); + TWriter::op_ternary_elementwise_function(dst, exp.opcode, tmp1, exp.second, exp.third); + } + + /** Represents the assignment: `\p dst = \p exp`. + * + * The TernaryElementwiseFunction is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] dst The tile which is assigned to. + * @param[in] exp The TernaryElementwiseFunction representing the expression to be evaluated and assigned. + */ + template + void op_assign(const TileOperand &dst, const TernaryElementwiseFunction &exp) + { + auto &tmp1 = declare_temp_tile(dst.tile_info()); + op_assign(tmp1, exp.second); + TWriter::op_ternary_elementwise_function(dst, exp.opcode, exp.first, tmp1, exp.third); + } + + /** Represents the assignment: `\p dst = \p exp`. + * + * The TernaryElementwiseFunction is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] dst The tile which is assigned to. + * @param[in] exp The TernaryElementwiseFunction representing the expression to be evaluated and assigned. + */ + template + void op_assign(const TileOperand &dst, const TernaryElementwiseFunction &exp) + { + auto &tmp1 = declare_temp_tile(dst.tile_info()); + op_assign(tmp1, exp.third); + TWriter::op_ternary_elementwise_function(dst, exp.opcode, exp.first, exp.second, tmp1); + } + + /** Represents the assignment: `\p dst = \p exp`. + * + * The TernaryElementwiseFunction is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] dst The tile which is assigned to. + * @param[in] exp The TernaryElementwiseFunction representing the expression to be evaluated and assigned. + */ + template + void op_assign(const TileOperand &dst, const TernaryElementwiseFunction &exp) + { + auto &tmp1 = declare_temp_tile(dst.tile_info()); + auto &tmp2 = declare_temp_tile(dst.tile_info()); + op_assign(tmp1, exp.first); + op_assign(tmp2, exp.second); + TWriter::op_ternary_elementwise_function(dst, exp.opcode, tmp1, tmp2, exp.third); + } + + /** Represents the assignment: `\p dst = \p exp`. + * + * The TernaryElementwiseFunction is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] dst The tile which is assigned to. + * @param[in] exp The TernaryElementwiseFunction representing the expression to be evaluated and assigned. + */ + template + void op_assign(const TileOperand &dst, const TernaryElementwiseFunction &exp) + { + auto &tmp1 = declare_temp_tile(dst.tile_info()); + auto &tmp2 = declare_temp_tile(dst.tile_info()); + op_assign(tmp1, exp.first); + op_assign(tmp2, exp.third); + TWriter::op_ternary_elementwise_function(dst, exp.opcode, tmp1, exp.second, tmp2); + } + + /** Represents the assignment: `\p dst = \p exp`. + * + * The TernaryElementwiseFunction is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] dst The tile which is assigned to. + * @param[in] exp The TernaryElementwiseFunction representing the expression to be evaluated and assigned. + */ + template + void op_assign(const TileOperand &dst, const TernaryElementwiseFunction &exp) + { + auto &tmp1 = declare_temp_tile(dst.tile_info()); + auto &tmp2 = declare_temp_tile(dst.tile_info()); + op_assign(tmp1, exp.second); + op_assign(tmp2, exp.third); + TWriter::op_ternary_elementwise_function(dst, exp.opcode, exp.first, tmp1, tmp2); + } + + /** Represents the assignment: `\p dst = \p exp`. + * + * The TernaryElementwiseFunction is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] dst The tile which is assigned to. + * @param[in] exp The TernaryElementwiseFunction representing the expression to be evaluated and assigned. + */ + template + void op_assign(const TileOperand &dst, const TernaryElementwiseFunction &exp) + { + auto &tmp1 = declare_temp_tile(dst.tile_info(), dst.tile_info(), dst.tile_info()); + auto &tmp2 = declare_temp_tile(dst.tile_info()); + auto &tmp3 = declare_temp_tile(dst.tile_info()); + op_assign(tmp1, exp.first); + op_assign(tmp2, exp.second); + op_assign(tmp3, exp.third); + TWriter::op_ternary_elementwise_function(dst, exp.opcode, tmp1, tmp2, tmp3); + } + + // ================================================== + // Assignments + // ================================================== + + /** Represents the assignment: `\p lhs += \p rhs` or `\p lhs -= \p rhs`. + * + * The Assignment is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @param[in] exp The Assignment representing the expression to be evaluated. + */ + void op_assign(const Assignment &exp) + { + if(exp.opcode == AssignmentOp::Increment) + { + TWriter::op_binary_expression(exp.lhs, exp.lhs, BinaryOp::Add, exp.rhs); + } + else if(exp.opcode == AssignmentOp::Decrement) + { + TWriter::op_binary_expression(exp.lhs, exp.lhs, BinaryOp::Sub, exp.rhs); + } + } + + /** Represents the assignment: `\p lhs += \p rhs` or `\p lhs -= \p rhs`. + * + * The Assignment is unpacked and its components are forwarded to + * the underlying KernelWriter's implementation. + * + * @tparam TRight The type of the RHS of the assignment. + * @param[in] exp The Assignment representing the expression to be evaluated. + */ + template + void op_assign(const Assignment &exp) + { + auto &tmp1 = declare_temp_tile(exp.lhs.tile_info()); + op_assign(tmp1, exp.rhs); + op_assign(Assignment{ exp.lhs, tmp1, exp.opcode }); + } + +private: + unsigned int temp_var_counter = 0; + + /** Return the current counter value, then increment it. + * + * @return The current counter value. + */ + int next_ctr() + { + return temp_var_counter++; + } + + /** Gets the next temporary variable counter value, + * and returns a suitable temporary variable name. + * + * @return A temporary variable name. + */ + std::string next_tmp_var_name() + { + return "tmp_" + std::to_string(next_ctr()); + } + + /** Returns the argument. + * + * Used for recursion with the variadic function version of this function. + * + * @param[in] arg The TileInfo to return. + * @return The \p arg. + */ + TileInfo get_largest_size(const TileInfo &arg) + { + return arg; + } + + /** Returns a TileInfo object where the size in each dimension (width, height) is the largest + * of either TileInfo argument in the corresponding dimension. + * + * @tparam TOps Must be of TileInfo type. + * @param[in] first A TileInfo object. + * @param[in] second A TileInfo object. + * @param[in] ops A number of TileInfo objects. + * @return A TileInfo object which represents the largest shape in each dimension across the arguments. + */ + template ::value>> + TileInfo get_largest_size(const TileInfo &first, const TileInfo &second, const TOps &...ops) + { + TileInfo largest = { + first.data_type(), + std::max(first.width(), second.width()), + std::max(first.height(), second.height()) + }; + return get_largest_size(largest, ops...); + } + + /** Helper function to define a suitable TileOperand with appropriate TileInfo + * such that broadcasting is taken into account, based on the arguments provided. + * + * @tparam TArgs Must be of TileInfo type. + * @param[in] args A number of TileInfo which determine the shape of the TileOperand to declare. + * @return A newly created TileOperand. + */ + template ::value>> + TileOperand &declare_temp_tile(const TArgs &...args) + { + return TWriter::declare_tile(next_tmp_var_name().c_str(), get_largest_size(args...)); + } +}; + +} // namespace ckw + +#endif // CKW_INCLUDE_CKW_KERNELWRITERHELPER_H diff --git a/compute_kernel_writer/prototype/include/ckw/OperandBase.h b/compute_kernel_writer/prototype/include/ckw/OperandBase.h new file mode 100644 index 0000000000000000000000000000000000000000..06d9f82756d430e74ac387adfc64048c901a9904 --- /dev/null +++ b/compute_kernel_writer/prototype/include/ckw/OperandBase.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_PROTOTYPE_INCLUDE_CKW_OPERANDBASE_H +#define CKW_PROTOTYPE_INCLUDE_CKW_OPERANDBASE_H + +#include "ckw/types/DataType.h" +#include + +namespace ckw +{ +namespace prototype +{ +class IGpuKernelWriter; + +class Operand; +} // namespace prototype + +/** The base class for all operands. */ +class OperandBase +{ +public: + /** Constructor + * + * @param[in] name The name of the operand. + */ + explicit OperandBase(const ::std::string &name); + + /** Destructor */ + virtual ~OperandBase(); + + /** (Internal use only) Create the implementation operand. + * + * @param[in] writer The implementation kernel writer. + */ + virtual prototype::Operand create_impl_operand(prototype::IGpuKernelWriter *writer) const = 0; + + /** Get the name of the operand. */ + const ::std::string &name() const; + + /** Set the name of the operand. */ + OperandBase &name(const ::std::string &name); + + /** Get the data type of the operand. */ + virtual DataType data_type() const = 0; + + /** Get whether the operand is compile-time constant. */ + virtual bool is_constant() const = 0; + +private: + ::std::string _name; +}; + +} // namespace ckw + +#endif // CKW_PROTOTYPE_INCLUDE_CKW_OPERANDBASE_H diff --git a/compute_kernel_writer/prototype/include/ckw/ScalarValue.h b/compute_kernel_writer/prototype/include/ckw/ScalarValue.h new file mode 100644 index 0000000000000000000000000000000000000000..16c3f6d441dc8676bafd2940380b3009f59d58e8 --- /dev/null +++ b/compute_kernel_writer/prototype/include/ckw/ScalarValue.h @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_PROTOTYPE_INCLUDE_CKW_SCALARVALUE_H +#define CKW_PROTOTYPE_INCLUDE_CKW_SCALARVALUE_H + +#include "ckw/Error.h" + +#include + +namespace ckw +{ + +/** The scalar value known at compile-time. */ +class ScalarValue +{ +public: + /** Initialize a new instance of @ref ScalarValue class with integer value 0. */ + ScalarValue() + { + _type = Type::INT; + _value.i64 = 0; + } + + /** Initialize a new instance of @ref ScalarValue class with the specified value. */ + template + ScalarValue(T value) + { + set(value); + } + + /** Set the value. */ + template + void set(T value) + { + CKW_ASSERT(::std::is_integral::value || ::std::is_floating_point::value); + CKW_ASSERT(sizeof(T) <= 8); + + _size = sizeof(T); + + if(::std::is_integral::value) + { + if(::std::is_signed::value) + { + _type = Type::INT; + _value.i64 = value; + } + else + { + _type = Type::UINT; + _value.u64 = value; + } + } + else + { + _type = Type::FLOAT; + _value.f64 = value; + } + } + + /** Get the value. + * + * The caller must make sure that what has been stored in the object must fit + * the output data type without data corruption or loss of accuracy. + */ + template + T get() const + { + CKW_ASSERT(::std::is_integral::value || ::std::is_floating_point::value); + CKW_ASSERT(sizeof(T) >= _size); + + if(::std::is_integral::value) + { + if(::std::is_signed::value) + { + CKW_ASSERT(_type == Type::INT || _type == Type::UINT); + CKW_ASSERT_IF(_type == Type::UINT, sizeof(T) > _size); + + return _value.i64; + } + else + { + CKW_ASSERT(_type == Type::INT); + + return _value.u64; + } + } + else + { + return _value.f64; + } + } + +private: + union Value + { + int64_t i64; + uint64_t u64; + double f64; + }; + + enum class Type : int32_t + { + UINT, + INT, + FLOAT, + }; + + Value _value{}; + Type _type{}; + uint32_t _size{}; +}; + +} // namespace ckw + +#endif // CKW_PROTOTYPE_INCLUDE_CKW_SCALARVALUE_H diff --git a/compute_kernel_writer/prototype/include/ckw/TensorInfo.h b/compute_kernel_writer/prototype/include/ckw/TensorInfo.h new file mode 100644 index 0000000000000000000000000000000000000000..55f8101a53c68d09703c848fdc4214ea3ea7b4fd --- /dev/null +++ b/compute_kernel_writer/prototype/include/ckw/TensorInfo.h @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_PROTOTYPE_INCLUDE_CKW_TENSORINFO_H +#define CKW_PROTOTYPE_INCLUDE_CKW_TENSORINFO_H + +#include "ckw/types/DataType.h" + +#include +#include + +namespace ckw +{ +/** Compute Kernel Writer tensor data layout (or memory format) */ +enum class TensorDataLayout +{ + Unknown, + Nhwc, + Ndhwc +}; + +/** Compute Kernel Writer tensor data layout component */ +enum class TensorDataLayoutComponent +{ + Unknown, + N, + D, + H, + W, + C, +}; + +/** Compute Kernel Writer tensor component bitmask. The bitmask can be used to retrieve + * the info from @ref TensorComponent. + */ +enum class TensorComponentBitmask : uint32_t +{ + OffsetFirstElement = 0x01000000, // For example, OffsetFirstElement in @ref TensorComponent + Stride = 0x02000000, // For example, stride0 in @ref TensorComponent + Dimension = 0x04000000, // For example, Dim0 in @ref TensorComponent + FoldedDimensions = 0x08000000, // For example, Dim0xDim1 in @ref TensorComponent +}; + +/** Compute Kernel Writer tensor component. The tensor components are used to access specific backend-agnostic tensor arguments, + * such as the tensor dimensions and tensor strides. + * The data type is represented as an integer. The value of the integer value + * is assigned to retrieve the information through the @ref TensorComponentBitmask. + */ +enum class TensorComponentType : uint32_t +{ + Unknown = 0x00000000, + OffsetFirstElement = 0x01000000, + Stride0 = 0x02000001, + Stride1 = 0x02000010, + Stride2 = 0x02000100, + Stride3 = 0x02001000, + Stride4 = 0x02010000, + Dim0 = 0x04000001, + Dim1 = 0x04000010, + Dim2 = 0x04000100, + Dim3 = 0x04001000, + Dim4 = 0x04010000, + Dim1xDim2 = 0x08000110, + Dim2xDim3 = 0x08001100, + Dim1xDim2xDim3 = 0x08001110 +}; + +/** Compute Kernel Writer tensor storage. The tensor storage represents the type of tensor memory object. + */ +enum class TensorStorageType : uint32_t +{ + Unknown = 0x00000000, + BufferUint8Ptr = 0x01000000, + Texture2dReadOnly = 0x02000001, + Texture2dWriteOnly = 0x02000010, +}; + +/** Compute Kernel Writer tensor shape + * Negative dimensions can be interpreted as dynamic dimensions by the Compute Kernel Writer + */ +using TensorShape = std::array; + +/** Compute Kernel Writer tensor info */ +class TensorInfo +{ +public: + /** Constructor + * + * @param[in] dt Tensor data type + * @param[in] shape Tensor shape + * @param[in] dl Tensor data layout + * @param[in] id Tensor id. The id is used to keep track of the bound user tensor. Through the id, + * the user can know what tensor has been used by the Compute Kernel Writer. + * Possible id values: + * - greater than or equal to 0: bind a user specific tensors + * - less than 0: bind a virtual tensor (tile) + */ + TensorInfo(DataType dt, const TensorShape &shape, TensorDataLayout dl, int32_t id); + + /** Set shape */ + TensorInfo &shape(const TensorShape &shape); + + /** Get shape */ + TensorShape shape() const; + + /** Set data type */ + TensorInfo &data_type(DataType dt); + + /** Get data type */ + DataType data_type() const; + + /** Set data layout */ + TensorInfo &data_layout(TensorDataLayout dl); + + /** Get data layout */ + TensorDataLayout data_layout() const; + + /** Set id */ + TensorInfo &id(int32_t id); + + /** Get layout */ + int32_t id() const; + +private: + TensorShape _shape{ { 0 } }; + DataType _dt{ DataType::Unknown }; + TensorDataLayout _dl{ TensorDataLayout::Unknown }; + int32_t _id{ -1 }; +}; +} // namespace ckw + +#endif /* CKW_PROTOTYPE_INCLUDE_CKW_TENSORINFO_H */ diff --git a/compute_kernel_writer/prototype/include/ckw/TensorOperand.h b/compute_kernel_writer/prototype/include/ckw/TensorOperand.h new file mode 100644 index 0000000000000000000000000000000000000000..6d88932c6635cd7e0211ab0002ef75ab52922c42 --- /dev/null +++ b/compute_kernel_writer/prototype/include/ckw/TensorOperand.h @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_PROTOTYPE_INCLUDE_CKW_TENSOROPERAND_H +#define CKW_PROTOTYPE_INCLUDE_CKW_TENSOROPERAND_H + +#include "ckw/OperandBase.h" +#include "ckw/TensorInfo.h" +#include "ckw/TensorTileSampler.h" +#include "ckw/TileOperand.h" +#include "ckw/types/DataType.h" + +#include + +namespace ckw +{ + +class TensorComponentOperand; + +// ================================================================================================= +// TensorOperand +// ================================================================================================= + +/** Tensor operand */ +class TensorOperand : public OperandBase +{ +public: + /** Initialize a new instance of @ref TensorOperand class. + * + * @param[in] name The name of the tensor. + * @param[in] info The tensor info. + * @param[in] storage_type The tensor storage type. + */ + TensorOperand(const ::std::string &name, const TensorInfo &info, TensorStorageType storage_type); + + /** No copy constructor. */ + TensorOperand(const TensorOperand &other) = delete; + + /** No copy assignment. */ + TensorOperand &operator=(const TensorOperand &other) = delete; + + /** (Internal use only) Create the implementation operand. + * + * @param[in] writer The implementation kernel writer. + */ + virtual prototype::Operand create_impl_operand(prototype::IGpuKernelWriter *writer) const override; + + /** Get the tensor info. */ + const TensorInfo &info() const; + + /** Get the tensor info. */ + TensorInfo &info(); + + /** Get the tensor storage type. */ + TensorStorageType storage_type() const; + + /** Get the data type. */ + virtual DataType data_type() const override; + + /** Get whether the tensor is compile-time constant. */ + virtual bool is_constant() const override; + + /** Get the default tile attached to the tensor. */ + const TileOperand &tile() const; + + /** Get the default tile attached to the tensor. */ + TileOperand &tile(); + + /** Set the default tile attached to the tensor. */ + TensorOperand &tile(TileOperand &tile); + + /** Get the tensor sampler of the default tile. */ + const TensorTileSampler &tile_sampler() const; + + /** Get the tensor sampler of the default tile. */ + TensorTileSampler &tile_sampler(); + + /** Set the tensor sampler of the default tile. */ + TensorOperand &tile_sampler(const TensorTileSampler &value); + + /** Get the operand that contains the stride in y dimension of the tensor. */ + TensorComponentOperand &stride1(); + + /** Get the operand that contains the stride in z dimension of the tensor. */ + TensorComponentOperand &stride2(); + + /** Get the operand that contains the stride in w dimension of the tensor. */ + TensorComponentOperand &stride3(); + + /** Get the operand that contains the stride in w dimension of the tensor. */ + TensorComponentOperand &stride4(); + + /** Get the operand that contains the size of dimension 0 of the tensor. */ + TensorComponentOperand &dim0(); + + /** Get the operand that contains the size of dimension 1 of the tensor. */ + TensorComponentOperand &dim1(); + + /** Get the operand that contains the size of dimension 2 of the tensor. */ + TensorComponentOperand &dim2(); + + /** Get the operand that contains the size of dimension 3 of the tensor. */ + TensorComponentOperand &dim3(); + + /** Get the operand that contains the size of dimension 4 of the tensor. */ + TensorComponentOperand &dim4(); + + /** Get the operand that contains the size of dimensions 1 and 2 collapsed. */ + TensorComponentOperand &dim1_dim2(); + + /** Get the operand that contains the size of dimensions 1, 2 and 3 collapsed. */ + TensorComponentOperand &dim1_dim2_dim3(); + + /** Get the operand that contains the offset in bytes to the first element. */ + TensorComponentOperand &offset_first_element_in_bytes(); + +private: + TensorInfo _info; + TensorStorageType _storage_type; + + TileOperand *_tile{ nullptr }; + TensorTileSampler _tile_sampler{}; + + ::std::unique_ptr _stride1{ nullptr }; + ::std::unique_ptr _stride2{ nullptr }; + ::std::unique_ptr _stride3{ nullptr }; + ::std::unique_ptr _stride4{ nullptr }; + ::std::unique_ptr _dim0{ nullptr }; + ::std::unique_ptr _dim1{ nullptr }; + ::std::unique_ptr _dim2{ nullptr }; + ::std::unique_ptr _dim3{ nullptr }; + ::std::unique_ptr _dim4{ nullptr }; + ::std::unique_ptr _dim1_dim2{ nullptr }; + ::std::unique_ptr _dim1_dim2_dim3{ nullptr }; + ::std::unique_ptr _offset_first_element_in_bytes{ nullptr }; +}; + +// ================================================================================================= +// TensorComponentOperand +// ================================================================================================= + +/** Tile operand that contains tensor information. */ +class TensorComponentOperand : public TileOperand +{ +public: + /** Initialize a new instance of @ref TensorComponentOperand class. + * + * @param[in] tensor The tensor operand. + * @param[in] component The tensor info component. + */ + TensorComponentOperand(TensorOperand &tensor, TensorComponentType component); + + /** Get the tensor operand. */ + TensorOperand &tensor(); + + /** Get the tensor operand. */ + const TensorOperand &tensor() const; + + /** Get the tensor component. */ + TensorComponentType component_type() const; + + /** (Internal use only) Create the implementation operand. + * + * @param[in] writer The implementation kernel writer. + */ + virtual prototype::Operand create_impl_operand(prototype::IGpuKernelWriter *writer) const override; + +private: + TensorOperand &_tensor; + TensorComponentType _component; +}; + +} // namespace ckw + +#endif // CKW_PROTOTYPE_INCLUDE_CKW_TENSOROPERAND_H diff --git a/compute_kernel_writer/prototype/include/ckw/TensorTileSampler.h b/compute_kernel_writer/prototype/include/ckw/TensorTileSampler.h new file mode 100644 index 0000000000000000000000000000000000000000..e1bf0c52b8639e0fe2ba6820f6e5c8568a071d4a --- /dev/null +++ b/compute_kernel_writer/prototype/include/ckw/TensorTileSampler.h @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_PROTOTYPE_INCLUDE_CKW_TENSORTILESAMPLER_H +#define CKW_PROTOTYPE_INCLUDE_CKW_TENSORTILESAMPLER_H + +#include "ckw/types/TensorSamplerTypes.h" +#include + +namespace ckw +{ + +class TileOperand; + +/** Tensor sampler + * + * It contains information about how the result tile should be stored to tensor memory. + * It can also be used to dictate how the subsequent operators fetch the input tensor. + */ +class TensorTileSampler +{ +public: + /** Initialize a new instance of @ref TensorSampler class. */ + TensorTileSampler(); + + /** Initialize a new instance of @ref TensorSampler class. + * + * @param[in] x The coordinate in the x dimension. + * @param[in] y The coordinate in the y dimension. + * @param[in] z The coordinate in the z dimension. + * @param[in] b The coordinate in the batch dimension. + * @param[in] format The tensor data format. + * @param[in] address_mode_x The address mode of the x dimension. + * @param[in] address_mode_y The address mode of the y dimension. + * @param[in] address_mode_z The address mode of the z dimension. + */ + TensorTileSampler( + TileOperand &x, TileOperand &y, TileOperand &z, TileOperand &b, + TensorSamplerFormat format, + TensorSamplerAddressModeX address_mode_x, + TensorSamplerAddressModeY address_mode_y, + TensorSamplerAddressModeZ address_mode_z); + + /** Initialize a new instance of @ref TensorSampler class. + * + * @param[in] x The coordinate in the x dimension. + * @param[in] y The coordinate in the y dimension. + * @param[in] z The coordinate in the z dimension. + * @param[in] b The coordinate in the batch dimension. + * @param[in] height The height of the tile. + * @param[in] width The width of the tile. + * @param[in] format The tensor data format. + * @param[in] address_mode_x The address mode of the x dimension. + * @param[in] address_mode_y The address mode of the y dimension. + * @param[in] address_mode_z The address mode of the z dimension. + */ + TensorTileSampler( + TileOperand &x, TileOperand &y, TileOperand &z, TileOperand &b, + int32_t height, int32_t width, + TensorSamplerFormat format, + TensorSamplerAddressModeX address_mode_x, + TensorSamplerAddressModeY address_mode_y, + TensorSamplerAddressModeZ address_mode_z); + + /** Get the coordinate in the x dimension. */ + const TileOperand &x() const; + + /** Set the coordinate in the x dimension. */ + TensorTileSampler &x(TileOperand &x); + + /** Get the coordinate in the y dimension. */ + const TileOperand &y() const; + + /** Set the coordinate in the y dimension. */ + TensorTileSampler &y(TileOperand &y); + + /** Get the coordinate in the z dimension. */ + const TileOperand &z() const; + + /** Set the coordinate in the z dimension. */ + TensorTileSampler &z(TileOperand &z); + + /** Get the coordinate in the batch dimension. */ + const TileOperand &b() const; + + /** Set the coordinate in the batch dimension. */ + TensorTileSampler &b(TileOperand &b); + + /** Get the width of the tile. */ + int32_t width() const; + + /** Set the width of the tile. */ + TensorTileSampler &width(int32_t width); + + /** Get the height of the tile. */ + int32_t height() const; + + /** Set the height of the tile. */ + TensorTileSampler &height(int32_t height); + + /** Get the format of the tensor. */ + TensorSamplerFormat format() const; + + /** Set the format of the tensor. */ + TensorTileSampler &format(TensorSamplerFormat format); + + /** Get the address mode of the x dimension. */ + TensorSamplerAddressModeX address_mode_x() const; + + /** Set the address mode of the x-dimension. */ + TensorTileSampler &address_mode_x(TensorSamplerAddressModeX address_mode_x); + + /** Get the address mode of the y dimension. */ + TensorSamplerAddressModeY address_mode_y() const; + + /** Set the address mode of the y dimension. */ + TensorTileSampler &address_mode_y(TensorSamplerAddressModeY address_mode_y); + + /** Get the address mode of the z dimension. */ + TensorSamplerAddressModeZ address_mode_z() const; + + /** Set the address mode of the z dimension. */ + TensorTileSampler &address_mode_z(TensorSamplerAddressModeZ address_mode_z); + +private: + TileOperand *_x{ nullptr }; + TileOperand *_y{ nullptr }; + TileOperand *_z{ nullptr }; + TileOperand *_b{ nullptr }; + + int32_t _height{ 0 }; + int32_t _width{ 0 }; + + TensorSamplerFormat _format{ TensorSamplerFormat::Unknown }; + TensorSamplerAddressModeX _address_mode_x{ TensorSamplerAddressModeX::Unknown }; + TensorSamplerAddressModeY _address_mode_y{ TensorSamplerAddressModeY::Unknown }; + TensorSamplerAddressModeZ _address_mode_z{ TensorSamplerAddressModeZ::Unknown }; +}; + +} // namespace ckw + +#endif // CKW_PROTOTYPE_INCLUDE_CKW_TENSORTILESAMPLER_H diff --git a/compute_kernel_writer/prototype/include/ckw/TileInfo.h b/compute_kernel_writer/prototype/include/ckw/TileInfo.h new file mode 100644 index 0000000000000000000000000000000000000000..de9e47af2b1d3b0571f720ebbd92455460ea4345 --- /dev/null +++ b/compute_kernel_writer/prototype/include/ckw/TileInfo.h @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_PROTOTYPE_INCLUDE_CKW_TILEINFO_H +#define CKW_PROTOTYPE_INCLUDE_CKW_TILEINFO_H + +#include "ckw/types/DataType.h" + +#include +#include + +namespace ckw +{ +// Constants to access the tile width and height in the TileShape +constexpr int32_t kTileWidthIdx = 0; +constexpr int32_t kTileHeightIdx = 1; + +/** Compute Kernel Writer tile shape. It is used to define the shape of the tile */ +using TileShape = std::array; + +/** Compute Kernel Writer tile info */ +class TileInfo +{ +public: + /** Constructor used to initialize a scalar variable with a given data type + * + * @param[in] dt Tile data type + */ + TileInfo(DataType dt); + + /** Constructor used to initialize a vector with a given data type and vector length. + * + * @param[in] dt Tile data type + * @param[in] w Tile width (or vector length) + */ + TileInfo(DataType dt, int32_t w); + + /** Constructor used to initialize a tile with a given data type and tile sizes. + * + * @param[in] dt Tile data type + * @param[in] h Tile height + * @param[in] w Tile width + */ + TileInfo(DataType dt, int32_t h, int32_t w); + + /** Set width */ + TileInfo &width(int32_t w); + + /** Get width */ + int32_t width() const; + + /** Set height */ + TileInfo &height(int32_t h); + + /** Get height */ + int32_t height() const; + + /** Set data type */ + TileInfo &data_type(DataType dt); + + /** Get data type */ + DataType data_type() const; + +private: + DataType _dt{ DataType::Unknown }; + TileShape _shape{}; +}; + +} // namespace ckw + +#endif /* COMPUTE_KERNEL_WRITER_INCLUDE_CKW_TILEINFO_H */ diff --git a/compute_kernel_writer/prototype/include/ckw/TileOperand.h b/compute_kernel_writer/prototype/include/ckw/TileOperand.h new file mode 100644 index 0000000000000000000000000000000000000000..24ee373a2408d930bb9144570076a2613acbb62a --- /dev/null +++ b/compute_kernel_writer/prototype/include/ckw/TileOperand.h @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_PROTOTYPE_INCLUDE_CKW_TILEOPERAND_H +#define CKW_PROTOTYPE_INCLUDE_CKW_TILEOPERAND_H + +#include "ckw/Error.h" +#include "ckw/OperandBase.h" +#include "ckw/ScalarValue.h" +#include "ckw/TileInfo.h" + +#include + +namespace ckw +{ + +class Kernel; + +using TileContainer = std::vector>; + +/** Tile operand which can be either scalar, vector or 2D tile. */ +class TileOperand : public OperandBase +{ +public: + /** Initialize a new instance of @ref TileOperand class with the tile information. + * + * @param[in] name The name of the tile. + * @param[in] tile_info The tile info. + */ + TileOperand(const ::std::string &name, const TileInfo &tile_info); + + /** Initialize a new instance of @ref TileOperand for scalar variable. + * + * @param[in] name The name of the tile. + * @param[in] data_type The data type of the tile. + */ + TileOperand(const ::std::string &name, DataType data_type); + + /** Initialize a new instance of @ref TileOperand for compile-time constant scalar variable. + * + * @param[in] name The name of the tile. + * @param[in] value The value of the tile. + */ + TileOperand(const ::std::string &name, int32_t value); + + /** Initialize a new instance of @ref TileOperand for compile-time constant scalar variable. + * + * @param[in] name The name of the tile. + * @param[in] value The value of the tile. + */ + TileOperand(const ::std::string &name, float value); + + /** Initialize a new instance of @ref TileOperand for compile-time constant variable. + * + * @param[in] name The name of the tile. + * @param[in] value The value of the tile. + */ + TileOperand(const ::std::string &name, const ::std::vector> &value, DataType dt); + + /** Prohibit copy of tile operand. */ + TileOperand(const TileOperand &) = delete; + + /** Prohibit copy of tile operand. */ + TileOperand &operator=(const TileOperand &) = delete; + + /** (Internal use only) Create the implementation operand. + * + * @param[in] writer The implementation kernel writer. + */ + virtual prototype::Operand create_impl_operand(prototype::IGpuKernelWriter *writer) const override; + + /** Get the tile info. */ + const TileInfo &tile_info() const; + + /** Get the data type of the tile. */ + virtual DataType data_type() const override; + + /** Get whether the tile is compile-time constant. */ + virtual bool is_constant() const override; + + /** Get whether the tile is a scalar value. */ + bool is_scalar() const; + + /** Get the scalar value of the tile. + * + * The tile must have the shape of 1, 1 (i.e. scalar). + * + * @return Scalar value as a string. + */ + std::string scalar_value() const; + + /** Get the values of the tile. + * + * @return 2D container of values. + */ + const TileContainer &value() const; + +private: + TileInfo _info; + TileContainer _value{}; + bool _constant; +}; + +} // namespace ckw + +#endif // CKW_PROTOTYPE_INCLUDE_CKW_TILEOPERAND_H diff --git a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_8b_mla.cpp b/compute_kernel_writer/prototype/include/ckw/types/ConvertPolicy.h similarity index 72% rename from src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_8b_mla.cpp rename to compute_kernel_writer/prototype/include/ckw/types/ConvertPolicy.h index de74ca5f4399979ba9586e4ebddb19b7203830a2..2a198507eb649059694dac201a4d8a0cbd1c1878 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_8b_mla.cpp +++ b/compute_kernel_writer/prototype/include/ckw/types/ConvertPolicy.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,19 +22,20 @@ * SOFTWARE. */ -#include "8b_mla.hpp" +#ifndef CKW_INCLUDE_CKW_CONVERTPOLICY_H +#define CKW_INCLUDE_CKW_CONVERTPOLICY_H -namespace arm_conv { -namespace depthwise { +#include -#if defined(__ARM_FEATURE_SVE) +namespace ckw +{ -ADD_IMPLEMENTATION(sve, s8q, int8_t, SVE, 2, 3, 3) -ADD_IMPLEMENTATION(sve, s8q, int8_t, SVE, 2, 5, 5) -ADD_IMPLEMENTATION(sve, u8q, uint8_t, SVE, 2, 3, 3) -ADD_IMPLEMENTATION(sve, u8q, uint8_t, SVE, 2, 5, 5) +enum class ConvertPolicy : int32_t +{ + None = 0, // No policy specified. + Saturate = 1, // Saturated. +}; -#endif // defined(__ARM_FEATURE_SVE) +} // namespace ckw -} // namespace depthwise -} // namespace arm_conv +#endif //CKW_INCLUDE_CKW_CONVERTPOLICY_H diff --git a/compute_kernel_writer/prototype/include/ckw/types/DataType.h b/compute_kernel_writer/prototype/include/ckw/types/DataType.h new file mode 100644 index 0000000000000000000000000000000000000000..3447dd61d68f6f33a267f83f3ba424bf4520bc4a --- /dev/null +++ b/compute_kernel_writer/prototype/include/ckw/types/DataType.h @@ -0,0 +1,50 @@ +/* +* Copyright (c) 2023 Arm Limited. +* +* SPDX-License-Identifier: MIT +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to +* deal in the Software without restriction, including without limitation the +* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +* sell copies of the Software, and to permit persons to whom the Software is +* furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +* SOFTWARE. +*/ + +#ifndef CKW_INCLUDE_CKW_DATATYPE_H +#define CKW_INCLUDE_CKW_DATATYPE_H + +#include + +namespace ckw +{ + +/** Compute Kernel Writer data types. This data type is used by the code variables and tensor arguments. */ +enum class DataType : int32_t +{ + Unknown = 0x00, + Fp32 = 0x11, + Fp16 = 0x12, + Int32 = 0x21, + Int16 = 0x22, + Int8 = 0x24, + Uint32 = 0x31, + Uint16 = 0x32, + Uint8 = 0x34, + Bool = 0x41 +}; + +} // namespace ckw + +#endif //CKW_INCLUDE_CKW_DATATYPE_H diff --git a/compute_kernel_writer/prototype/include/ckw/types/Functions.h b/compute_kernel_writer/prototype/include/ckw/types/Functions.h new file mode 100644 index 0000000000000000000000000000000000000000..2dd5ed0b3d3cee1e6e0a4df6b78418c18b322126 --- /dev/null +++ b/compute_kernel_writer/prototype/include/ckw/types/Functions.h @@ -0,0 +1,60 @@ +/* +* Copyright (c) 2023 Arm Limited. +* +* SPDX-License-Identifier: MIT +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to +* deal in the Software without restriction, including without limitation the +* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +* sell copies of the Software, and to permit persons to whom the Software is +* furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +* SOFTWARE. +*/ + +#ifndef CKW_INCLUDE_CKW_FUNCTIONS_H +#define CKW_INCLUDE_CKW_FUNCTIONS_H + +#include + +namespace ckw +{ + +enum class UnaryFunction : int32_t +{ + Exp = 0x0000, + Tanh = 0x0001, + Sqrt = 0x0002, + Erf = 0x0003, + Fabs = 0x0004, + Log = 0x0006, + Round = 0x0007, + + // Misc + SizeOf = 0x0008, +}; + +enum class BinaryFunction : int32_t +{ + Min = 0x0000, + Max = 0x0001, +}; + +enum class TernaryFunction : int32_t +{ + Select = 0x0000, +}; + +} // namespace ckw + +#endif //CKW_INCLUDE_CKW_FUNCTIONS_H diff --git a/compute_kernel_writer/prototype/include/ckw/types/GpuTargetLanguage.h b/compute_kernel_writer/prototype/include/ckw/types/GpuTargetLanguage.h new file mode 100644 index 0000000000000000000000000000000000000000..6c08617949fef1f9d681fd2d11f090362ebbb9c6 --- /dev/null +++ b/compute_kernel_writer/prototype/include/ckw/types/GpuTargetLanguage.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_INCLUDE_CKW_GPUTARGETLANGUAGE_H +#define CKW_INCLUDE_CKW_GPUTARGETLANGUAGE_H + +#include + +namespace ckw +{ + +enum class GpuTargetLanguage : int32_t +{ + Unknown, + OpenCL +}; + +} // namespace ckw + +#endif //CKW_INCLUDE_CKW_GPUTARGETLANGUAGE_H diff --git a/compute_kernel_writer/prototype/include/ckw/types/Operators.h b/compute_kernel_writer/prototype/include/ckw/types/Operators.h new file mode 100644 index 0000000000000000000000000000000000000000..14a88c91b48f1ee976deec9a30a81d9000e3cfac --- /dev/null +++ b/compute_kernel_writer/prototype/include/ckw/types/Operators.h @@ -0,0 +1,77 @@ +/* +* Copyright (c) 2023 Arm Limited. +* +* SPDX-License-Identifier: MIT +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to +* deal in the Software without restriction, including without limitation the +* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +* sell copies of the Software, and to permit persons to whom the Software is +* furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +* SOFTWARE. +*/ + +#ifndef CKW_INCLUDE_CKW_OPERATORS_H +#define CKW_INCLUDE_CKW_OPERATORS_H + +#include + +namespace ckw +{ + +enum class UnaryOp : int32_t +{ + LogicalNot = 0x0000, // ! + BitwiseNot = 0x0001, // ~ +}; + +/* Binary operations +*/ +enum class BinaryOp : int32_t +{ + // Elementwise + Add = 0x0000, // + + Sub = 0x0001, // - + Mul = 0x0002, // * + Div = 0x0003, // / + Mod = 0x0004, // % + // Relational + Equal = 0x1000, // == + Less = 0x1001, // < + LessEqual = 0x1002, // <= + Greater = 0x1003, // > + GreaterEqual = 0x1004, // >= + // Algebra + MatMul_Nt_Nt = 0x2000, // X + MatMul_Nt_T = 0x2001, // X + MatMul_T_Nt = 0x2002, // X + MatMul_T_T = 0x2003, // X + Dot = 0x2004, // . + // Logical + LogicalAnd = 0x3000, // && + LogicalOr = 0x3001, // || + // Bitwise + BitwiseXOR = 0x4000, // ^ +}; + +enum class AssignmentOp : int32_t +{ + // Unary + Increment = 0x0000, // += + Decrement = 0x0001, // -= +}; + +} // namespace ckw + +#endif //CKW_INCLUDE_CKW_OPERATORS_H diff --git a/compute_kernel_writer/prototype/include/ckw/types/TensorSamplerTypes.h b/compute_kernel_writer/prototype/include/ckw/types/TensorSamplerTypes.h new file mode 100644 index 0000000000000000000000000000000000000000..836bd13c95b17e69854713a75eadfe7f29278785 --- /dev/null +++ b/compute_kernel_writer/prototype/include/ckw/types/TensorSamplerTypes.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_INCLUDE_CKW_TENSORSAMPLERTYPES_H +#define CKW_INCLUDE_CKW_TENSORSAMPLERTYPES_H + +#include + +namespace ckw +{ + +enum class TensorSamplerFormat : int32_t +{ + Unknown = 0, + C_WH_1 = 1, + C_W_H = 2 +}; + +enum class TensorSamplerAddressModeX : int32_t +{ + Unknown = 0, + None = 1, // The user guarantees that the X coordinate is always in-bound + OverlappingMin = 2 // (FIXED shapes only) Reduce the load/store length when x == 0 (MIN). The load length will be width % original length + // Leftover elements can be handled using overlapping. This involves processing some of the elements in the array twice. +}; + +enum class TensorSamplerAddressModeY : int32_t +{ + Unknown = 0, + None = 1, // The user guarantees that the Y coordinate is always in-bound + OverlappingMin = 2, // (FIXED shapes only) Reduce the load/store length when x == 0 (MIN). The load length will be width % original length + Skip = 3, // Skip the read/write + SkipMinEdgeOnly = 4, // Skip greater than or equal to max only. The user guarantees that the Y coordinate is always >= 0 + SkipMaxEdgeOnly = 5, // Skip less than 0 only + ClampToNearest = 6, // Clamp the coordinate to nearest edge (0 or max value allowed on Y) + ClampToMinEdgeOnly = 7, // Clamp the negative coordinate to 0 only. Therefore, we expect Y to be always < MAX + ClampToMaxEdgeOnly = 8, // Clamp the coordinate to the max value allowed on Y only. We expect Y to be always >= 0 + ClampToBorder = 9, // Clamp to border which always has 0 value + ClampToBorderMinEdgeOnly = 10, + ClampToBorderMaxEdgeOnly = 11 +}; + +enum class TensorSamplerAddressModeZ : int32_t +{ + Unknown = 0, + None = 1, // The user guarantees that the Y coordinate is always in-bound + Skip = 3, // Skip the read/write + SkipMinEdgeOnly = 4, // Skip greater than or equal to max only. The user guarantees that the Y coordinate is always >= 0 + SkipMaxEdgeOnly = 5, // Skip less than 0 only + ClampToNearest = 6, // Clamp the coordinate to nearest edge (0 or max value allowed on Y) + ClampToMinEdgeOnly = 7, // Clamp the negative coordinate to 0 only. Therefore, we expect Y to be always < MAX + ClampToMaxEdgeOnly = 8, // Clamp the coordinate to the max value allowed on Y only. We expect Y to be always >= 0 +}; + +} // namespace ckw + +#endif //CKW_INCLUDE_CKW_TENSORSAMPLERTYPES_H diff --git a/compute_kernel_writer/prototype/src/Kernel.cpp b/compute_kernel_writer/prototype/src/Kernel.cpp new file mode 100644 index 0000000000000000000000000000000000000000..095ac879f18a8eec2831688b00b9e207e7fb9fd9 --- /dev/null +++ b/compute_kernel_writer/prototype/src/Kernel.cpp @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "ckw/Kernel.h" +#include "ckw/TensorOperand.h" +#include "ckw/types/GpuTargetLanguage.h" +#include "src/Prototype.h" + +namespace ckw +{ + +Kernel::Kernel(GpuTargetLanguage language) + : Kernel{"unnamed", language} +{ +} + +Kernel::Kernel(const char *name, GpuTargetLanguage language) + : _name(name), _kernel(std::make_unique(language)), _operands{}, _tensor_id_operands{} +{ +} + + +Kernel::~Kernel() +{ +} + +const std::string &Kernel::name() const +{ + return _name; +} + +void Kernel::name(const std::string& name) +{ + _name = name; +} +std::vector Kernel::arguments() const +{ + std::vector arguments; + + const auto impl_args = _kernel->arguments.tensor_argument_declarations(); + + for(auto tensor_arg : impl_args) + { + auto tensor = _tensor_id_operands.at(tensor_arg->format().id); + arguments.push_back(*tensor); + + for(auto component_arg : tensor_arg->component_declarations()) + { + switch(component_arg) + { + case TensorComponentType::OffsetFirstElement: + arguments.push_back(tensor->offset_first_element_in_bytes()); + break; + + case TensorComponentType::Stride1: + arguments.push_back(tensor->stride1()); + break; + + case TensorComponentType::Stride2: + arguments.push_back(tensor->stride2()); + break; + + case TensorComponentType::Stride3: + arguments.push_back(tensor->stride3()); + break; + + case TensorComponentType::Stride4: + arguments.push_back(tensor->stride4()); + break; + + case TensorComponentType::Dim0: + arguments.push_back(tensor->dim0()); + break; + + case TensorComponentType::Dim1: + arguments.push_back(tensor->dim1()); + break; + + case TensorComponentType::Dim2: + arguments.push_back(tensor->dim2()); + break; + + case TensorComponentType::Dim3: + arguments.push_back(tensor->dim3()); + break; + + case TensorComponentType::Dim4: + arguments.push_back(tensor->dim4()); + break; + + case TensorComponentType::Dim1xDim2: + arguments.push_back(tensor->dim1_dim2()); + break; + + case TensorComponentType::Dim1xDim2xDim3: + arguments.push_back(tensor->dim1_dim2_dim3()); + break; + + default: + CKW_ASSERT(false); + } + } + } + + return arguments; +} + +TileOperand &Kernel::register_operand(std::unique_ptr operand) +{ + const auto &name = operand->name(); + auto ptr = operand.get(); + + CKW_ASSERT(_operands.find(name) == _operands.end()); + _operands[name] = std::move(operand); + + return *ptr; +} + +TensorOperand &Kernel::register_operand(std::unique_ptr operand) +{ + const auto id = operand->info().id(); + const auto &name = operand->name(); + auto ptr = operand.get(); + + CKW_ASSERT(_tensor_id_operands.find(id) == _tensor_id_operands.end()); + CKW_ASSERT(_operands.find(name) == _operands.end()); + + _tensor_id_operands[id] = operand.get(); + _operands[name] = std::move(operand); + + return *ptr; +} + +prototype::GpuKernelWriterDataHolder *Kernel::impl() +{ + return _kernel.get(); +} + +} // namespace ckw diff --git a/compute_kernel_writer/prototype/src/KernelArgument.cpp b/compute_kernel_writer/prototype/src/KernelArgument.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2b4d7c8cee5f93c587e0e48f35c777899abd0759 --- /dev/null +++ b/compute_kernel_writer/prototype/src/KernelArgument.cpp @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "ckw/KernelArgument.h" +#include "ckw/Error.h" +#include "ckw/TensorOperand.h" + +namespace ckw +{ + +KernelArgument::KernelArgument(TensorOperand &tensor) + : _type(Type::TensorStorage), _id(tensor.info().id()) +{ + _sub_id.tensor_storage_type = tensor.storage_type(); +} + +KernelArgument::KernelArgument(TensorComponentOperand &tensor_component) + : _type(Type::TensorComponent), _id(tensor_component.tensor().info().id()) +{ + _sub_id.tensor_component_type = tensor_component.component_type(); +} + +KernelArgument::Type KernelArgument::type() const +{ + return _type; +} + +int32_t KernelArgument::id() const +{ + return _id; +} + +TensorStorageType KernelArgument::tensor_storage_type() const +{ + CKW_ASSERT(_type == Type::TensorStorage); + return _sub_id.tensor_storage_type; +} + +TensorComponentType KernelArgument::tensor_component_type() const +{ + CKW_ASSERT(_type == Type::TensorComponent); + return _sub_id.tensor_component_type; +} + +} // namespace ckw diff --git a/compute_kernel_writer/prototype/src/KernelWriter.cpp b/compute_kernel_writer/prototype/src/KernelWriter.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f29cf1280215487bf0afa0a2e52c16d9ee95f3e1 --- /dev/null +++ b/compute_kernel_writer/prototype/src/KernelWriter.cpp @@ -0,0 +1,363 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "ckw/KernelWriter.h" +#include "ckw/Error.h" +#include "ckw/TensorInfo.h" +#include "ckw/TensorOperand.h" +#include "src/Prototype.h" + +#include + +namespace ckw +{ + +namespace +{ + +inline prototype::TensorInfo create_impl_tensor_info(const TensorInfo &info) +{ + return prototype::TensorInfo{ info.shape(), info.data_type(), info.data_layout(), info.id() }; +} + +} // namespace + +// ================================================================================================= +// Constructors and destructor +// ================================================================================================= + +KernelWriter::KernelWriter(Kernel &kernel) + : _kernel(&kernel), + _impl_attr(std::make_unique()), + _impl(prototype::GpuKernelWriterFactory::create(_impl_attr.get(), kernel.impl())) +{ + _impl->set_IdSpace(1); +} + +KernelWriter::~KernelWriter() +{ +} + +// ================================================================================================= +// Scope management +// ================================================================================================= + +int32_t KernelWriter::id_space() const +{ + return _id_space; +} + +KernelWriter &KernelWriter::id_space(int32_t id_space) +{ + CKW_ASSERT(id_space <= _max_id_space); + + _id_space = id_space; + return *this; +} + +int32_t KernelWriter::next_id_space() +{ + id_space(++_max_id_space); + return _id_space; +} + +// ================================================================================================= +// Tensor and tile declaration +// ================================================================================================= + +TensorOperand &KernelWriter::declare_tensor_argument(const std::string &name, const TensorInfo &info, TensorStorageType storage_type) +{ + const auto var_name = generate_variable_name(name); + + _impl->declare_argument(var_name, create_impl_tensor_info(info)); + + auto &operand = _kernel->register_operand(std::make_unique(var_name, info, storage_type)); + + return operand; +} + +TileOperand &KernelWriter::declare_tile_argument(const std::string &name, int32_t value) +{ + const auto var_name = generate_variable_name(name); + + auto &operand = _kernel->register_operand(std::make_unique(var_name, value)); + + return operand; +} + +std::string KernelWriter::generate_variable_name(const std::string &name) const +{ + std::stringstream var_name; + + var_name << "_" << _id_space << "_" << name; + + return var_name.str(); +} + +TileOperand &KernelWriter::declare_tile_operand(std::unique_ptr operand_ptr) +{ + auto &operand = _kernel->register_operand(std::move(operand_ptr)); + const auto &name = operand.name(); + + if(!operand.is_constant()) + { + const auto &info = operand.tile_info(); + + _impl->declare_tile( + name, + prototype::TileInfo(info.data_type(), info.width(), info.height())); + } + else + { + _impl->declare_const_tile(name, operand.value(), operand.data_type()); + } + + return operand; +} + +// ================================================================================================= +// Load and store +// ================================================================================================= + +void KernelWriter::op_load(TileOperand &tile, const TensorOperand &tensor, const TensorTileSampler &sampler, const TileOperand &dilation_y) +{ + prototype::TensorOperand impl_tensor( + tensor.name(), + prototype::GpuSampler{ + sampler.format(), + prototype::to_gpu_tensor_storage(tensor.storage_type()), + sampler.address_mode_x(), + sampler.address_mode_y(), + sampler.address_mode_z() }); + + auto impl_x = sampler.x().create_impl_operand(_impl.get()); + auto impl_y = sampler.y().create_impl_operand(_impl.get()); + auto impl_z = sampler.z().create_impl_operand(_impl.get()); + auto impl_b = sampler.b().create_impl_operand(_impl.get()); + + auto impl_dilation_y = dilation_y.create_impl_operand(_impl.get()); + + auto impl_dst = tile.create_impl_operand(_impl.get()); + + _impl->op_load_immediate(impl_tensor, impl_dst, impl_x, impl_y, impl_z, impl_b, impl_dilation_y); +} + +void KernelWriter::op_load_indirect(TileOperand &tile, const TensorOperand &tensor, const TensorTileSampler &sampler) +{ + prototype::TensorOperand impl_tensor( + tensor.name(), + prototype::GpuSampler{ + sampler.format(), + prototype::to_gpu_tensor_storage(tensor.storage_type()), + sampler.address_mode_x(), + sampler.address_mode_y(), + sampler.address_mode_z() }); + + auto impl_x = sampler.x().create_impl_operand(_impl.get()); + auto impl_y = sampler.y().create_impl_operand(_impl.get()); + auto impl_z = sampler.z().create_impl_operand(_impl.get()); + auto impl_b = sampler.b().create_impl_operand(_impl.get()); + + auto impl_dst = tile.create_impl_operand(_impl.get()); + + _impl->op_load_indirect(impl_tensor, impl_dst, impl_x, impl_y, impl_z, impl_b); +} + +void KernelWriter::util_get_indirect_buffer(TileOperand &tile, + const TensorOperand &tensor, + const TensorTileSampler &sampler, + const TileOperand &x, + const TileOperand &y, + const TileOperand &x_off, + const TileOperand &y_off) +{ + prototype::TensorOperand impl_tensor( + tensor.name(), + prototype::GpuSampler{ + sampler.format(), + prototype::to_gpu_tensor_storage(tensor.storage_type()), + sampler.address_mode_x(), + sampler.address_mode_y(), + sampler.address_mode_z() }); + + auto impl_x = x.create_impl_operand(_impl.get()); + auto impl_y = y.create_impl_operand(_impl.get()); + auto impl_x_off = x_off.create_impl_operand(_impl.get()); + auto impl_y_off = y_off.create_impl_operand(_impl.get()); + + auto impl_dst = tile.create_impl_operand(_impl.get()); + + _impl->util_get_indirect_buffer(impl_dst, impl_tensor, impl_x, impl_y, impl_x_off, impl_y_off); +} + +void KernelWriter::op_store(TensorOperand &tensor, const TileOperand &tile, const TensorTileSampler &sampler) +{ + prototype::TensorOperand impl_tensor( + tensor.name(), + prototype::GpuSampler{ + sampler.format(), + prototype::to_gpu_tensor_storage(tensor.storage_type()), + sampler.address_mode_x(), + sampler.address_mode_y(), + sampler.address_mode_z() }); + auto impl_src = tile.create_impl_operand(_impl.get()); + auto impl_x = sampler.x().create_impl_operand(_impl.get()); + auto impl_y = sampler.y().create_impl_operand(_impl.get()); + auto impl_z = sampler.z().create_impl_operand(_impl.get()); + auto impl_b = sampler.b().create_impl_operand(_impl.get()); + + _impl->op_store_immediate(impl_tensor, impl_src, impl_x, impl_y, impl_z, impl_b); +} + +// ================================================================================================= +// Data processing +// ================================================================================================= + +void KernelWriter::op_assign(const TileOperand &dst, const TileOperand &src) +{ + auto impl_dst = dst.create_impl_operand(_impl.get()); + auto impl_src = src.create_impl_operand(_impl.get()); + + _impl->op_assign(impl_dst, impl_src); +} + +void KernelWriter::op_cast_expression(const TileOperand &dst, const TileOperand &src, const ConvertPolicy policy) +{ + auto impl_dst = dst.create_impl_operand(_impl.get()); + auto impl_src = src.create_impl_operand(_impl.get()); + + _impl->op_cast_expression(impl_dst, impl_src, policy); +} + +void KernelWriter::op_binary_expression(const TileOperand &dst, const TileOperand &lhs, BinaryOp op, const TileOperand &rhs) +{ + auto impl_lhs = lhs.create_impl_operand(_impl.get()); + auto impl_rhs = rhs.create_impl_operand(_impl.get()); + auto impl_dst = dst.create_impl_operand(_impl.get()); + + _impl->op_binary_expression(impl_dst, impl_lhs, op, impl_rhs); +} + +void KernelWriter::op_unary_expression(const TileOperand &dst, UnaryOp op, const TileOperand &src) +{ + auto impl_dst = dst.create_impl_operand(_impl.get()); + auto impl_src = src.create_impl_operand(_impl.get()); + + _impl->op_unary_expression(impl_dst, op, impl_src); +} + +void KernelWriter::op_unary_elementwise_function(const TileOperand &dst, UnaryFunction opcode, const TileOperand &src) +{ + auto impl_dst = dst.create_impl_operand(_impl.get()); + auto impl_src = src.create_impl_operand(_impl.get()); + + _impl->op_unary_elementwise_function(impl_dst, opcode, impl_src); +} + +void KernelWriter::op_binary_elementwise_function(const TileOperand &dst, BinaryFunction opcode, const TileOperand &first, const TileOperand &second) +{ + auto impl_dst = dst.create_impl_operand(_impl.get()); + auto impl_first = first.create_impl_operand(_impl.get()); + auto impl_second = second.create_impl_operand(_impl.get()); + + _impl->op_binary_elementwise_function(impl_dst, opcode, impl_first, impl_second); +} + +void KernelWriter::op_ternary_elementwise_function(const TileOperand &dst, TernaryFunction opcode, const TileOperand &first, const TileOperand &second, const TileOperand &third) +{ + auto impl_dst = dst.create_impl_operand(_impl.get()); + auto impl_first = first.create_impl_operand(_impl.get()); + auto impl_second = second.create_impl_operand(_impl.get()); + auto impl_third = third.create_impl_operand(_impl.get()); + + _impl->op_ternary_elementwise_function(impl_dst, opcode, impl_first, impl_second, impl_third); +} + +void KernelWriter::op_if(const TileOperand &lhs, BinaryOp op, const TileOperand &rhs, const std::function &body) +{ + auto impl_lhs = lhs.create_impl_operand(_impl.get()); + auto impl_rhs = rhs.create_impl_operand(_impl.get()); + + _impl->op_if_header(impl_lhs, op, impl_rhs); + _impl->compound_statement_begin(); + body(); + _impl->compound_statement_end(); +} + +void KernelWriter::op_else_if(const TileOperand &lhs, BinaryOp op, const TileOperand &rhs, const std::function &body) +{ + auto impl_lhs = lhs.create_impl_operand(_impl.get()); + auto impl_rhs = rhs.create_impl_operand(_impl.get()); + + _impl->op_else_if_header(impl_lhs, op, impl_rhs); + _impl->compound_statement_begin(); + body(); + _impl->compound_statement_end(); +} + +void KernelWriter::op_else(const std::function &body) +{ + _impl->op_else_header(); + _impl->compound_statement_begin(); + body(); + _impl->compound_statement_end(); +} + +void KernelWriter::op_for_loop(const TileOperand &var_name, BinaryOp cond_op, const TileOperand &cond_value_name, const TileOperand &update_var_name, AssignmentOp update_op, const TileOperand &update_value_name, const std::function &body) +{ + auto impl_var_name = var_name.create_impl_operand(_impl.get()); + auto impl_cond_value_name = cond_value_name.create_impl_operand(_impl.get()); + auto impl_update_var_name = update_var_name.create_impl_operand(_impl.get()); + auto impl_update_value_name = update_value_name.create_impl_operand(_impl.get()); + + _impl->op_for_loop_header(impl_var_name, cond_op, impl_cond_value_name, impl_update_var_name, update_op, impl_update_value_name); + _impl->compound_statement_begin(); + body(); + _impl->compound_statement_end(); +} + +// ================================================================================================= +// Misc +// ================================================================================================= + +void KernelWriter::op_get_global_id(TileOperand &dst, int32_t dim) +{ + _impl->op_get_global_id(prototype::Operand(dst.name()), dim); +} + +void KernelWriter::op_return() +{ + _impl->op_return(); +} + +// ================================================================================================= +// Code generation +// ================================================================================================= + +std::string KernelWriter::generate_code() +{ + return prototype::generate_code(*_kernel->impl(), _kernel->name()); +} + +} // namespace ckw diff --git a/compute_kernel_writer/prototype/src/OperandBase.cpp b/compute_kernel_writer/prototype/src/OperandBase.cpp new file mode 100644 index 0000000000000000000000000000000000000000..59cf846cc70247b779ac0eaa02d07fdd77425eb0 --- /dev/null +++ b/compute_kernel_writer/prototype/src/OperandBase.cpp @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "ckw/OperandBase.h" + +namespace ckw +{ + +OperandBase::OperandBase(const std::string &name) + : _name(name) +{ +} + +OperandBase::~OperandBase() +{ +} + +const std::string &OperandBase::name() const +{ + return _name; +} + +OperandBase &OperandBase::name(const std::string &name) +{ + _name = name; + return *this; +} + +} // namespace ckw diff --git a/compute_kernel_writer/prototype/src/Prototype.h b/compute_kernel_writer/prototype/src/Prototype.h new file mode 100644 index 0000000000000000000000000000000000000000..2b519471ac6d519d29d6f0ccf256dd7cd6f4bcc4 --- /dev/null +++ b/compute_kernel_writer/prototype/src/Prototype.h @@ -0,0 +1,4115 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_PROTOTYPE_SRC_PROTOTYPE_H +#define CKW_PROTOTYPE_SRC_PROTOTYPE_H + +#include +#include +#include // assert (to be removed) +#include +#include +#include // int32_t +#include +#include // cout (to be removed) +#include +#include +#include +#include +#include +#include + +#include "ckw/Error.h" +#include "ckw/TensorInfo.h" +#include "ckw/types/ConvertPolicy.h" +#include "ckw/types/DataType.h" +#include "ckw/types/Functions.h" +#include "ckw/types/GpuTargetLanguage.h" +#include "ckw/types/Operators.h" +#include "ckw/types/TensorSamplerTypes.h" + +namespace ckw +{ +namespace prototype +{ + +// Dummy data structure for Size2D +using Size2D = std::vector; + +// Dummy Status +using Status = void; + +enum class ComponentType : int32_t +{ + Complex = 0, + Simple = 1, + Unfusable = 2 +}; + +enum class GpuCompilationSpeed +{ + Fast = 0x00, // fast compilation may increase the latency of the network + Slow = 0x01 // slow compilation may decrease the latency of the network +}; + +enum class GpuExtensions +{ + Fp16, + Dot8, + Mmul, + FastMath +}; + +struct TensorInfo +{ + TensorShape shape{ { 0 } }; + DataType data_type{ DataType::Unknown }; + TensorDataLayout data_layout{ TensorDataLayout::Nhwc }; + int32_t id{ -1 }; +}; + +struct ComponentAttribute +{ + GpuCompilationSpeed compilation_speed{ GpuCompilationSpeed::Fast }; + bool overwrite_tile{ true }; +}; + +inline std::string data_type_to_cl_type(DataType dt) +{ + switch(dt) + { + case DataType::Fp32: + return "float"; + case DataType::Fp16: + return "half"; + case DataType::Int8: + return "char"; + case DataType::Uint8: + return "uchar"; + case DataType::Uint16: + return "ushort"; + case DataType::Int16: + return "short"; + case DataType::Uint32: + return "uint"; + case DataType::Int32: + return "int"; + case DataType::Bool: + return "bool"; + default: + assert(false); + return ""; + } +} + +inline int32_t width_to_cl_vector_size(int32_t width) +{ + switch(width) + { + case 1: + return 1; + case 2: + return 2; + case 3: + return 3; + case 4: + return 4; + case 5: + case 6: + case 7: + case 8: + return 8; + case 9: + case 10: + case 11: + case 12: + case 13: + case 14: + case 15: + case 16: + return 16; + default: + assert(false); + return 0; + } +} + +inline std::string get_cl_data_type(DataType dt, int32_t width) +{ + std::string data_type; + int32_t w = width_to_cl_vector_size(width); + data_type += data_type_to_cl_type(dt); + if(w != 1) + { + data_type += std::to_string(w); + } + return data_type; +} + +inline std::string to_opencl_store(int32_t vector_length) +{ + if(vector_length != 1) + { + return "vstore" + std::to_string(vector_length) + "("; + } + else + { + return "*("; + } +} + +struct TileInfo +{ + TileInfo() + { + } + + TileInfo(DataType dt) + : dt(dt), w(1), h(1) + { + } + + TileInfo(DataType dt, int32_t width) + : dt(dt), w(width), h(1) + { + } + + TileInfo(DataType dt, int32_t width, int32_t height) + : dt(dt), w(width), h(height) + { + } + + DataType dt{ DataType::Unknown }; // Data type of the tile + int32_t w{ 0 }; // Width (i.e. c0 - portion of the channels) + int32_t h{ 0 }; // Height (i.e. s0 - portion of the spatial dimensions) +}; + +inline std::ostream &operator<<(std::ostream &o, const TileInfo &a) +{ + o << a.w << " x " << a.h; + return o; +} + +struct DataTypeAsString +{ + std::string str{ "" }; + DataType dt{ DataType::Unknown }; + int32_t size{ 1 }; +}; + +struct ValueAsString +{ + std::string str{ "" }; + DataTypeAsString type{}; +}; + +// https://stackoverflow.com/questions/51515378/storing-and-accessing-tile-properties-in-c +// A Tile is a collection of variables used to express a 2D data. +class IScalarTile +{ +public: + virtual ~IScalarTile() = default; + + /** Method to get the scalar variable from a tile + * @param[in] x X coordinate on the width of the tile. If out-of-bound, the coordinate is clamped to the nearest valid edge + * @param[in] y Y coordinate on the height of the tile. If out-of-bound, the coordinate is clamped to the nearest valid edge + * + * @return the scalar variable as a string + */ + virtual ValueAsString scalar(int32_t x, int32_t y) const = 0; + + /** Method to get the list of underlying variable names used by the tile + * + * @return the list of variable names + */ + virtual std::vector underlying_source_variables() const = 0; + + /** Method to get the name of the tile. + * + * @return the name of the tile + */ + std::string name() const + { + return _basename; + } + + /** Method to get the tile format + * + * @return the format + */ + TileInfo format() const + { + return _format; + } + + /** Method to know whether the tile is assignable or not (constant) + * + * @return true if the tile is assignable + */ + virtual bool is_assignable() const = 0; + + /** Method to know whether the tile needs to be declared + * + * @return true if the tile needs to be declared in the code before being used + */ + virtual bool need_declaration() const = 0; + +protected: + TileInfo _format{}; // Tile format + std::string _basename{ "" }; // Tile name +}; + +// A tile is a collection of variables used to express a 2D data. The variables are vectors in the GPU context. +// The vector size is given by the width of the tile. The number of vectors height by depth defines the number of vectors +class IVectorTile : public IScalarTile +{ +public: + virtual ~IVectorTile() = default; + + /** Method to get the vector variable from a tile. A vector is an ordered homogeneous collection of two or more scalars. + * The user can query the list of supported width for the vectors through preferred_vector_sizes(). + * + * @param[in] y Y coordinate on the height of the tile. If out-of-bound, the coordinate is clamped to the nearest valid edge + * + * @return the vector variable as a string + */ + virtual ValueAsString vector(int32_t y) const = 0; + + /** Method to get a vector variable from a tile. A vector is an ordered homogeneous collection of two or more scalars. + * + * @return the vector variable as a string + */ + virtual ValueAsString vector(int32_t x_start, int32_t width, int32_t y) const = 0; + /** Method to get the preferred vector sizes. + * + * @return a vector with the preferred vector sizes + */ + //virtual std::vector preferred_vector_sizes() const = 0; +}; + +class ClTile : public IVectorTile +{ +public: + ClTile(const std::string &name, TileInfo format) + { + _format = format; + _basename = name; + } + + ValueAsString scalar(int32_t x, int32_t y) const override + { + x = std::max(std::min(x, _format.w - 1), static_cast(0)); + y = std::max(std::min(y, _format.h - 1), static_cast(0)); + + ValueAsString t; + t.str = build_variable_name(y); + t.type.str = get_cl_data_type(_format.dt, 1); + t.type.dt = _format.dt; + t.type.size = 1; + + // Check required because if the width has only one element, we cannot use .s0 + if(_format.w != 1) + { + // Automatic broadcasting + t.str += ".s" + std::to_string(x); + } + + return t; + } + + ValueAsString vector(int32_t y) const override + { + y = std::max(std::min(y, _format.h - 1), static_cast(0)); + + ValueAsString t; + t.str = build_variable_name(y); + t.type.str = get_cl_data_type(_format.dt, _format.w); + t.type.dt = _format.dt; + t.type.size = _format.w; + return t; + } + + ValueAsString vector(int32_t x_start, int32_t width, int32_t y) const override + { + y = std::max(std::min(y, _format.h - 1), static_cast(0)); + + ValueAsString t; + t.str = build_variable_name(y); + t.type.str = get_cl_data_type(_format.dt, width); + t.type.dt = _format.dt; + t.type.size = width; + + if(_format.w != 1) + { + t.str += ".s"; + for(int i = 0; i < width; ++i) + { + t.str += to_scalar_hex(x_start + i); + } + } + return t; + } + + std::vector underlying_source_variables() const override + { + std::vector vars; + for(int32_t y = 0; y < _format.h; ++y) + { + ValueAsString t; + t.str = build_variable_name(y); + t.type.str = get_cl_data_type(_format.dt, _format.w); + t.type.dt = _format.dt; + t.type.size = _format.w; + vars.push_back(t); + } + return vars; + } + + bool is_assignable() const override + { + return true; + } + + bool need_declaration() const override + { + return true; + } + +private: + std::string build_variable_name(int32_t y) const + { + std::string var_name = _basename; + + if(_format.h == 1) + { + return var_name; + } + else + { + var_name += "_"; + var_name += std::to_string(y); + } + + return var_name; + } + + std::string to_scalar_hex(int32_t x) const + { + switch(x) + { + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + case 8: + case 9: + return std::to_string(x); + case 10: + return "A"; + case 11: + return "B"; + case 12: + return "C"; + case 13: + return "D"; + case 14: + return "E"; + case 15: + return "F"; + default: + std::cout << "Unsupported hexadecimal value" << std::endl; + assert(false); + return ""; + } + } +}; + +// Unique features: It contains values in the form of string. The name used for this object is misleading since the variables can change the value over time. +class ClConstantTile : public IVectorTile +{ +public: + ClConstantTile(const std::vector> &in, DataType dt) + { + _format.w = in[0].size(); + _format.h = in.size(); + _format.dt = dt; + + _data = std::vector>(_format.h, std::vector(_format.w)); + + for(int32_t y = 0; y < _format.h; ++y) + { + for(int32_t x = 0; x < _format.w; ++x) + { + _data[y][x] = in[y][x]; + } + } + } + + ValueAsString scalar(int32_t x, int32_t y) const override + { + x = std::max(std::min(x, _format.w - 1), static_cast(0)); + y = std::max(std::min(y, _format.h - 1), static_cast(0)); + + ValueAsString t; + t.str = _data[y][x]; + t.type.str = get_cl_data_type(_format.dt, 1); + t.type.dt = _format.dt; + t.type.size = 1; + + return t; + } + + ValueAsString vector(int32_t y) const override + { + y = std::max(std::min(y, _format.h - 1), static_cast(0)); + + return vector(0, _format.w, y); + } + + ValueAsString vector(int32_t x_start, int32_t width, int32_t y) const override + { + y = std::max(std::min(y, _format.h - 1), static_cast(0)); + + ValueAsString t; + t.str = ""; + t.type.str = get_cl_data_type(_format.dt, width); + t.type.dt = _format.dt; + t.type.size = width; + + if(width > 1) + { + t.str += "((" + get_cl_data_type(_format.dt, width) + ")("; + } + + int32_t x = x_start; + for(; x < width - 1; ++x) + { + t.str += scalar(x, y).str; + t.str += ", "; + } + t.str += scalar(x, y).str; + + if(width > 1) + { + t.str += "))"; + } + + return t; + } + + std::vector underlying_source_variables() const override + { + std::vector vars; + + for(int32_t y = 0; y < _format.h; ++y) + { + for(int32_t x = 0; x < _format.w; ++x) + { + ValueAsString t; + t.str = _data[y][x]; + t.type.str = get_cl_data_type(_format.dt, 1); + t.type.dt = _format.dt; + t.type.size = 1; + vars.push_back(t); + } + } + + return vars; + } + + bool is_assignable() const override + { + return false; + } + + bool need_declaration() const override + { + return false; + } + +private: + std::vector> _data{}; +}; + +enum class TensorComponentIndex : int32_t +{ + IndexMask = 0x0000000f, +}; + +enum class TensorComponentGroup : int32_t +{ + OffsetFirstElement = 0x00000100, + Stride = 0x00001000, + Dimension = 0x00010000, + FoldedDimension = 0x00100000, + Constant = 0x01000000 +}; + +inline std::string to_string(TensorComponentType x) +{ + switch(x) + { + case TensorComponentType::Unknown: + return "Unknown"; + case TensorComponentType::OffsetFirstElement: + return "OffsetFirstElement"; + case TensorComponentType::Stride1: + return "Stride1"; + case TensorComponentType::Stride2: + return "Stride2"; + case TensorComponentType::Stride3: + return "Stride3"; + case TensorComponentType::Stride4: + return "Stride4"; + case TensorComponentType::Dim0: + return "Dim0"; + case TensorComponentType::Dim1: + return "Dim1"; + case TensorComponentType::Dim2: + return "Dim2"; + case TensorComponentType::Dim3: + return "Dim3"; + case TensorComponentType::Dim4: + return "Dim4"; + case TensorComponentType::Dim1xDim2: + return "Dim1xDim2"; + case TensorComponentType::Dim1xDim2xDim3: + return "Dim1xDim2xDim3"; + default: + assert(false); + return ""; + } +} + +class ITensorArgument +{ +public: + virtual ~ITensorArgument() = default; + + /** Method to get the tensor component as a string + * + * @param[in] x tensor component to query + * + * @return the tensor component as a string + */ + virtual std::string component(TensorComponentType x) = 0; + + /** Method to get the tensor component type declaration as a string + * + * @return the tensor component type declaration as a string + */ + virtual std::string component_type_declaration() const = 0; + + /** Method to get the tensor component data type + * + * @return the tensor component data type + */ + virtual DataType component_data_type() const = 0; + + /** Method to get the tensor component declarations + * + * @return a vector containing the tensor component declarations + */ + virtual std::vector component_declarations() const = 0; + + /** Method to get the name of the tensor argument. + * + * @return the name of the tensor argument + */ + std::string name() const + { + return _basename; + } + + /** Method to get the tensor format + * + * @return the format + */ + TensorInfo format() const + { + return _format; + } + +protected: + TensorInfo _format{}; + std::string _basename{}; +}; + +enum class GpuTensorStorage : int32_t +{ + Unknown = 0x0000, + BufferUint8Ptr = 0x0012, + Image2dReadOnly = 0x0020, + Image2dWriteOnly = 0x0021, + Image3dReadOnly = 0x0030, + Image3dWriteOnly = 0x0031 +}; + +inline GpuTensorStorage to_gpu_tensor_storage(TensorStorageType s) +{ + switch(s) + { + case TensorStorageType::Unknown: + return GpuTensorStorage::Unknown; + + case TensorStorageType::BufferUint8Ptr: + return GpuTensorStorage::BufferUint8Ptr; + + case TensorStorageType::Texture2dReadOnly: + return GpuTensorStorage::Image2dReadOnly; + + case TensorStorageType::Texture2dWriteOnly: + return GpuTensorStorage::Image2dWriteOnly; + + default: + assert(false); + return GpuTensorStorage::Unknown; + } +} + +inline TensorStorageType to_tensor_storage(GpuTensorStorage s) +{ + switch(s) + { + case GpuTensorStorage::Unknown: + return TensorStorageType::Unknown; + + case GpuTensorStorage::BufferUint8Ptr: + return TensorStorageType::BufferUint8Ptr; + + case GpuTensorStorage::Image2dReadOnly: + return TensorStorageType::Texture2dReadOnly; + + case GpuTensorStorage::Image2dWriteOnly: + return TensorStorageType::Texture2dWriteOnly; + + default: + assert(false); + return TensorStorageType::Unknown; + } +} + +class IGpuTensorArgument : public ITensorArgument +{ +public: + virtual ~IGpuTensorArgument() = default; + + /** Method to get the tensor storage, which is the underlying storage used to keep the data memory + * + * @param[in] x tensor storage to query + * + * @return the tensor storage as a string + */ + virtual std::string storage(GpuTensorStorage x) = 0; + + /** Method to get the tensor storage type declaration as a string + * + * @param[in] x tensor component to query + * + * @return the tensor storage type declaration as a string + */ + virtual std::string storage_type_declaration(GpuTensorStorage x) const = 0; + + /** Method to get the tensor storage declarations + * + * @return a vector containing the tensor storage declarations + */ + virtual std::vector storage_declarations() const = 0; +}; + +class ClTensorArgument : public IGpuTensorArgument +{ +public: + ClTensorArgument(const std::string &name, const TensorInfo &x, bool return_by_value_when_possible) + { + _basename = name; + _format = x; + _return_by_value_when_possible = return_by_value_when_possible; + } + + // Methods to override + std::string component(TensorComponentType x) override + { + if((static_cast(x) & static_cast(TensorComponentGroup::Constant))) + { + int32_t idx = static_cast(x) & static_cast(TensorComponentIndex::IndexMask); + return std::to_string(idx - 1); + } + + if(_return_by_value_when_possible) + { + if((static_cast(x) & static_cast(TensorComponentGroup::Dimension))) + { + int32_t idx = static_cast(x) & static_cast(TensorComponentIndex::IndexMask); + return std::to_string(_format.shape[idx]); + } + + if((static_cast(x) & static_cast(TensorComponentGroup::FoldedDimension))) + { + switch(x) + { + case TensorComponentType::Dim1xDim2: + return std::to_string(_format.shape[1] * _format.shape[2]); + case TensorComponentType::Dim1xDim2xDim3: + return std::to_string(_format.shape[1] * _format.shape[2] * _format.shape[2]); + default: + std::cout << "Unsupported folded dimension" << std::endl; + assert(false); + } + } + } + + if(std::find(_components_required.begin(), _components_required.end(), x) == _components_required.end()) + { + _components_required.push_back(x); + } + + return build_component_name(x); + } + + std::string component_type_declaration() const override + { + return "int"; + }; + + DataType component_data_type() const override + { + return DataType::Int32; + } + + std::string storage(GpuTensorStorage x) override + { + if(std::find(_storage_required.begin(), _storage_required.end(), x) == _storage_required.end()) + { + _storage_required.push_back(x); + } + + return build_storage_name(x); + } + + std::string storage_type_declaration(GpuTensorStorage x) const override + { + switch(x) + { + case GpuTensorStorage::BufferUint8Ptr: + return "__global uchar*"; + case GpuTensorStorage::Image2dReadOnly: + return "__read_only image2d_t"; + case GpuTensorStorage::Image2dWriteOnly: + return "__write_only image2d_t"; + case GpuTensorStorage::Image3dReadOnly: + return "__read_only image3d_t "; + case GpuTensorStorage::Image3dWriteOnly: + return "__write_only image3d_t "; + default: + std::cout << "Unsupported storage" << std::endl; + assert(false); + return ""; + } + }; + + std::vector storage_declarations() const override + { + return _storage_required; + } + + std::vector component_declarations() const override + { + return _components_required; + } + +private: + std::string build_storage_name(GpuTensorStorage x) const + { + std::string var_name = _basename; + + switch(x) + { + case GpuTensorStorage::BufferUint8Ptr: + return var_name + "_ptr"; + case GpuTensorStorage::Image2dReadOnly: + case GpuTensorStorage::Image2dWriteOnly: + return var_name + "_img2d"; + case GpuTensorStorage::Image3dReadOnly: + case GpuTensorStorage::Image3dWriteOnly: + return var_name + "_img3d"; + default: + std::cout << "Unsupported storage" << std::endl; + assert(false); + } + + return var_name; + } + + std::string build_component_name(TensorComponentType x) const + { + std::string var_name = _basename; + + switch(x) + { + case TensorComponentType::OffsetFirstElement: + return var_name + "_offset_first_element"; + case TensorComponentType::Stride1: + return var_name + "_stride1"; + case TensorComponentType::Stride2: + return var_name + "_stride2"; + case TensorComponentType::Stride3: + return var_name + "_stride3"; + case TensorComponentType::Dim0: + return var_name + "_dim0"; + case TensorComponentType::Dim1: + return var_name + "_dim1"; + case TensorComponentType::Dim2: + return var_name + "_dim2"; + case TensorComponentType::Dim3: + return var_name + "_dim3"; + case TensorComponentType::Dim1xDim2: + return var_name + "_dim1xdim2"; + case TensorComponentType::Dim1xDim2xDim3: + return var_name + "_dim1xdim2xdim3"; + default: + std::cout << "Unsupported component" << std::endl; + assert(false); + } + + return var_name; + } + + bool _return_by_value_when_possible{ false }; + std::vector _storage_required{}; + std::vector _components_required{}; +}; + +/** + * @brief Data structure that contains the declared tiles by the components. + * The registry is a linear data structure that follows the similar principle of the stack. The user can use the @p increment_registry_level() method to + * increase the level of the stack (0 when it starts). When the user uses the @p decrement_registry_level() method, the registry decreases the level of the stack + * and remove (pop) all the tiles from the level above. + * When a tile is declared on the level 0, it is a global tile. A global tile is visible in all parts of the code. + * Since different components may use the same name to define a tile, the registry adopts the IdSpace concept, an @p id to prevent name collisions + * when declaring tiles among different components. + * + */ +class GpuTileRegistry +{ +public: + enum class RegistryTileType + { + Tile, + Link + }; + + using RegistryIdSpace = int32_t; + using RegistryLevel = int32_t; + using RegistryTileName = std::string; + + struct RegistryTileTableEntry + { + RegistryLevel registry_level{ 0 }; + std::unique_ptr tile_object{ nullptr }; + }; + + struct RegistryTileTypeTableEntry + { + RegistryTileType tile_type{ RegistryTileType::Tile }; + RegistryTileName tile_name{}; + RegistryIdSpace registry_idspace{ 0 }; + RegistryLevel registry_level{ 0 }; + }; + + using RegistryTileTable = std::map>; + using RegistryTileTypeTable = std::map>; + + /** + * @brief Construct a new Gpu Tile Registry object + * + */ + GpuTileRegistry() + { + _language = GpuTargetLanguage::Unknown; + } + + /** + * @brief Construct a new Gpu Tile Registry object providing the Gpu programming language + * + * @param[in] language Gpu programming language to use + */ + GpuTileRegistry(GpuTargetLanguage language) + { + _language = language; + } + + /** + * @brief Default destructor. Destroy the Gpu Tile Registry object + * + */ + ~GpuTileRegistry() = default; + + /** + * @brief Set the working IdSpace for the tile registry. IdSpace is used to prevent name collisions when declaring tiles. + * Therefore, the IdSpace should be set before declaring any tiles. + * + * @param[in] id The IdSpace id + */ + void set_IdSpace(int32_t id) + { + _IdSpace = id; + } + + /** + * @brief Get the current working IdSpace for the tile registry. IdSpace is used to prevent name collisions when declaring tiles + * + * @return The IdSpace id + */ + int32_t IdSpace() const + { + return _IdSpace; + } + + /** + * @brief Gets all the IdSpace declarations defined in the tile registry. + * + * @return all the IdSpace declarations defined in the tile registry as std::vector. It returns an empty vector if there are no IdSpace declarations. + */ + std::vector IdSpace_declarations() const + { + std::vector x; + + auto it = _frags.begin(); + + while(it != _frags.end()) + { + x.push_back(it->first); + + it++; + } + + return x; + } + + /** + * @brief Declare a tile from a previously created tile + */ + void insert(const std::string &name, const IVectorTile *frag) + { + assert(_language == GpuTargetLanguage::OpenCL); + const int32_t key_IdSpace = _IdSpace; + const std::string key_var_name = name; + const std::string var_name = frag->name(); + TileInfo format = frag->format(); + + // First check whether a tile with the same name exists + IVectorTile *result = (*this)[key_var_name]; + assert(result == nullptr); + if(result == nullptr) + { + std::unique_ptr tile = std::make_unique(var_name, format); + + _frags[key_IdSpace][key_var_name].tile_object = std::move(tile); + _frags[key_IdSpace][key_var_name].registry_level = _registry_level; + + _frag_types[key_IdSpace][key_var_name].tile_type = RegistryTileType::Link; + _frag_types[key_IdSpace][key_var_name].tile_name = key_var_name; + _frag_types[key_IdSpace][key_var_name].registry_idspace = _IdSpace; + _frag_types[key_IdSpace][key_var_name].registry_level = _registry_level; + } + } + + /** + * @brief Declare a tile with TileInfo. The tile will be stored in the IdSpace set with @p set_IdSpace() + * + * @note The reference name used for declaring the tile should not be previously used in the IdSpace + * + * @param[in] name Reference name for the tile. The reference name can be used to retrieve the tile stored in the registry. + * @param[in] format Tile format use to use + */ + void insert(const std::string &name, const TileInfo &format) + { + assert(_language == GpuTargetLanguage::OpenCL); + const int32_t key_IdSpace = _IdSpace; + const std::string key_var_name = name; + const std::string var_name = generate_tile_name(name); + + // First check whether a tile with the same name exists + IVectorTile *result = (*this)[key_var_name]; + assert(result == nullptr); + if(result == nullptr) + { + std::unique_ptr tile = std::make_unique(var_name, format); + _frags[key_IdSpace][key_var_name].tile_object = std::move(tile); + _frags[key_IdSpace][key_var_name].registry_level = _registry_level; + + _frag_types[key_IdSpace][key_var_name].tile_type = RegistryTileType::Tile; + _frag_types[key_IdSpace][key_var_name].tile_name = key_var_name; + _frag_types[key_IdSpace][key_var_name].registry_idspace = _IdSpace; + _frag_types[key_IdSpace][key_var_name].registry_level = _registry_level; + } + } + + /** + * @brief Declare a constant tile. The content of the tile is passed as a vector of std::string + * + * @note The reference name used for declaring the tile should not be previously used in the IdSpace + * + * @param[in] name Reference name for the tile. The reference name can be used to retrieve the tile stored in the registry. + * @param[in] in A 3D std::vector of std::string. From the 3D std::vector we can know the dimensions for the tile + * @param[in] dt The data type for the elements stored in the 3D std::vector as std::string. It is user's responsibilty to ensure + * that the data type is aligned with the content of the std::string. + */ + void insert(const std::string &name, const std::vector> &in, DataType dt) + { + assert(_language == GpuTargetLanguage::OpenCL); + const int32_t key_IdSpace = _IdSpace; + const std::string key_var_name = name; + + // First check whether a tile with the same name exists + IVectorTile *result = (*this)[key_var_name]; + assert(result == nullptr); + if(result == nullptr) + { + std::unique_ptr tile = std::make_unique(in, dt); + _frags[key_IdSpace][key_var_name].tile_object = std::move(tile); + _frags[key_IdSpace][key_var_name].registry_level = _registry_level; + + _frag_types[key_IdSpace][key_var_name].tile_type = RegistryTileType::Tile; + _frag_types[key_IdSpace][key_var_name].tile_name = key_var_name; + _frag_types[key_IdSpace][key_var_name].registry_idspace = _IdSpace; + _frag_types[key_IdSpace][key_var_name].registry_level = _registry_level; + } + } + + /** + * @brief Declare an anonymous constant tile. The content of the tile is passed as a vector of std::string + * + * @note This method can be used to declare temporary tiles that need to be accessed only once. + * + * @param[in] in A 3D std::vector of std::string. From the 3D std::vector we can know the dimensions for the tile + * @param[in] dt The data type for the elements stored in the 3D std::vector as std::string. It is user responsibilty to ensure + * that the data type is aligned with what passed with the std::string. + * + * @return IVectorTile* the anonymous constant tile + */ + IVectorTile *insert(const std::vector> &in, DataType dt) + { + assert(_language == GpuTargetLanguage::OpenCL); + const int32_t key_IdSpace = _IdSpace; + const std::string key_var_name = "_" + std::to_string(_anonymous_frag_count++); + + // First check whether a tile with the same name exists + IVectorTile *result = (*this)[key_var_name]; + assert(result == nullptr); + if(result == nullptr) + { + std::unique_ptr tile = std::make_unique(in, dt); + _frags[key_IdSpace][key_var_name].tile_object = std::move(tile); + _frags[key_IdSpace][key_var_name].registry_level = _registry_level; + + _frag_types[key_IdSpace][key_var_name].tile_type = RegistryTileType::Tile; + _frag_types[key_IdSpace][key_var_name].tile_name = key_var_name; + _frag_types[key_IdSpace][key_var_name].registry_idspace = _IdSpace; + _frag_types[key_IdSpace][key_var_name].registry_level = _registry_level; + } + + return (*this)[key_var_name]; + } + + /** + * @brief Get the tile from the registry. This method searches the tile in the IdSpace provided by the user + * + * @param[in] name The name of the tile to retrieve + * @param[in] IdSpace The IdSpace id where to search the tile + * + * @return IVectorTile* The tile + */ + IVectorTile *get(const std::string &name, int32_t IdSpace) + { + const int32_t key_IdSpace = IdSpace; + const std::string key_var_name = name; + + IVectorTile *result = nullptr; + auto search_IdSpace = _frags.find(key_IdSpace); + if(search_IdSpace != _frags.end()) + { + auto search_tile = _frags[key_IdSpace].find(key_var_name); + if(search_tile != _frags[key_IdSpace].end()) + { + result = search_tile->second.tile_object.get(); + assert(result != nullptr); + } + } + + return result; + } + + /** + * @brief Get the tile from the registry. This method searches the tile in the IdSpace set with @p set_IdSpace() + * + * @param[in] name The name of the tile to retrieve + * + * @return IVectorTile* The tile + */ + IVectorTile *operator[](const std::string &name) + { + return get(name, _IdSpace); + } + + /** + * @brief Check whether the tile in the in the IdSpace provided by the user exists + * + * @param[in] name Name of the tile to search for + * @param[in] IdSpace The IdSpace id where to search the tile + * + * @return true if the tile exists + * @return false if the tile does not exist + */ + bool has_tile(const std::string &name, int32_t IdSpace) const + { + const int32_t key_IdSpace = IdSpace; + const std::string key_var_name = name; + + // IVectorTile* result = nullptr; + auto search_IdSpace = _frags.find(key_IdSpace); + + return search_IdSpace != _frags.end(); + } + + /** + * @brief Check whether the tile within the current IdSpace exists + * + * @param[in] name Name of the tile to search for + * + * @return true if the tile exists + * @return false if the tile does not exist + */ + bool has_tile(const std::string &name) const + { + return has_tile(name, _IdSpace); + } + + /** + * @brief Get all the tiles declared within the IdSpace provided by the user + * + * @param[in] IdSpace IdSpace where to retrieve all the declared tiles + * + * @return std::vector A vector with all the declared tiles in the IdSpace provided by the user + */ + std::vector tile_declarations(int32_t IdSpace) + { + std::vector tiles; + + std::map::iterator it = _frag_types[IdSpace].begin(); + + while(it != _frag_types[IdSpace].end()) + { + // The following line should be enabled. However, we cannot at this stage + // because it used to retrieve the output tile produced by each component. + // However, this method should NOT be used to retrieve the output tile + //if(it->second.tile_type == RegistryTileType::Tile) + { + tiles.push_back(get(it->second.tile_name, it->second.registry_idspace)); + } + it++; + } + + return tiles; + } + + /** + * @brief Increase the level of stack. + * + */ + void increment_registry_level() + { + _registry_level++; + } + + /** + * @brief Remove all the tiles declared at the current stack level and decrease the level of the stack. + * + */ + void decrement_registry_level() + { + assert(_registry_level >= 0); + + // Remove all variables in the local scope + std::map::iterator it = _frags[_IdSpace].begin(); + + while(it != _frags[_IdSpace].end()) + { + if(it->second.registry_level == _registry_level) + { + it = _frags[_IdSpace].erase(it); + } + else + { + it++; + } + } + + std::map::iterator it_type = _frag_types[_IdSpace].begin(); + + while(it_type != _frag_types[_IdSpace].end()) + { + if(it_type->second.registry_level == _registry_level) + { + it_type = _frag_types[_IdSpace].erase(it_type); + } + else + { + it_type++; + } + } + + _registry_level--; + } + + /** + * @brief Get the level of the stack + * + */ + int32_t level() const + { + return _registry_level; + } + +private: + // This method ensures that the key is unique among different components + std::string generate_tile_name(const std::string &name) + { + assert(_IdSpace >= 0); + if(_registry_level == 0) + { + return "_G" + std::to_string(_IdSpace) + "_" + name; + } + else + { + return name; + } + } + + RegistryTileTable _frags{}; + RegistryTileTypeTable _frag_types{}; + RegistryLevel _registry_level{ 0 }; + RegistryIdSpace _IdSpace{ -1 }; + int32_t _anonymous_frag_count{ 0 }; // Counter used to create the anonymous tiles + GpuTargetLanguage _language{ GpuTargetLanguage::Unknown }; // Gpu programming language +}; + +using TensorEntry = std::unique_ptr; + +/** + * @brief Data structure that contains the tensors consumed by the components. + * Since different components may use the same name as reference for a tensor, the registry adopts the IdSpace concept, an @p id to prevent name collisions + * when declaring tensors among different components. + * + */ +class GpuTensorArgumentRegistry +{ +public: + /** + * @brief Construct a new Gpu Tensor Registry object + * + */ + GpuTensorArgumentRegistry() + { + _language = GpuTargetLanguage::Unknown; + } + + /** + * @brief Construct a new Gpu Tensor Registry object + * + * @param[in] language Gpu programming language to use + */ + GpuTensorArgumentRegistry(GpuTargetLanguage language) + { + _language = language; + } + + /** + * @brief Default destructor. Destroy the Gpu Tensor Registry object + * + */ + ~GpuTensorArgumentRegistry() = default; + + /** + * @brief Set the working IdSpace for the tensor registry. IdSpace is used to prevent name collisions when declaring tensors. + * Therefore, the IdSpace should be set before declaring any tensors. + * + * @param[in] id The IdSpace id + */ + void set_IdSpace(int32_t id) + { + _IdSpace = id; + } + + /** + * @brief Get the current working IdSpace for the tensor registry. IdSpace is used to prevent name collisions when declaring tensors + * + * @return The IdSpace id + */ + int32_t IdSpace() const + { + return _IdSpace; + } + + /** + * @brief Gets all the IdSpace declarations defined in the tensor registry. + * + * @return all the IdSpace declarations defined in the tensor registry as std::vector. It returns an empty vector if there are no IdSpace declarations. + */ + std::vector IdSpace_declarations() const + { + std::vector x; + + auto it = _refs.begin(); + + while(it != _refs.end()) + { + x.push_back(it->first); + + it++; + } + + return x; + } + + /** + * @brief Declare a tensor with TensorInfo. The tensor will be stored in the IdSpace set with @p set_IdSpace() + * + * @note The reference name used for declaring the tensor should not be previously used in the IdSpace + * + * @param[in] name Reference name for the tensor. The reference name can be used to retrieve the tensor stored in the registry. + * @param[in] x Pair of tensor info and tensor id + * @param[in] return_by_value_when_possible True if we want the value stored in the tensor components + */ + void insert(const std::string &name, const TensorInfo &x, bool return_by_value_when_possible) + { + assert(_language == GpuTargetLanguage::OpenCL); + const int32_t key_IdSpace = _IdSpace; + const int32_t tensor_id = x.id; + const std::string key_var_name = name; + const std::string var_name = generate_tensor_name(name, tensor_id); + + // First, check whether the tensor has already a reference. If so, trigger an assert + assert(!has_tensor_argument(name)); + + // Check whether a tensor with that tensorID exists + auto result = _tensor_arguments.find(tensor_id); + if(result == _tensor_arguments.end()) + { + // It means that we haven't added a tensor with that tensor_id yet. Create a IGpuTensorArgument before creating the reference + std::unique_ptr arg = std::make_unique(var_name, x, + return_by_value_when_possible); + _tensor_arguments[tensor_id] = std::move(arg); + } + + _refs[key_IdSpace][key_var_name] = tensor_id; + } + + /** + * @brief Get the tensor from the registry. This method searches the tensor in the IdSpace set with @p set_IdSpace() + * + * @param[in] name The name of the tensor to retrieve + * + * @return IGpuTensor* The tensor + */ + IGpuTensorArgument *operator[](const std::string &name) + { + const int32_t key_IdSpace = _IdSpace; + const std::string key_var_name = name; + + IGpuTensorArgument *result = nullptr; + auto search_IdSpace = _refs.find(key_IdSpace); + if(search_IdSpace != _refs.end()) + { + auto search_tensor_id = _refs[key_IdSpace].find(key_var_name); + + if(search_tensor_id != _refs[key_IdSpace].end()) + { + const int32_t tensor_id = search_tensor_id->second; + auto search_tensor_argument = _tensor_arguments.find(tensor_id); + if(search_tensor_argument != _tensor_arguments.end()) + { + result = search_tensor_argument->second.get(); + } + assert(result != nullptr); + } + } + + return result; + } + + /** + * @brief Get all the tensors declared in the IdSpace provided by the user + * + * @return std::vector A vector with all the declared tensors + */ + std::vector tensor_argument_declarations() + { + std::vector args; + + auto it = _tensor_arguments.begin(); + + while(it != _tensor_arguments.end()) + { + args.push_back(it->second.get()); + it++; + } + + return args; + } + + /** + * @brief Check whether the tensor argument in the IdSpace set with @p set_IdSpace() exists + * + * @param[in] name Name of the tensor argument to search for + * + * @return true if the tensor argument exists + * @return false if the tensor argument does not exist + */ + bool has_tensor_argument(const std::string &name) + { + const int32_t key_IdSpace = _IdSpace; + const std::string key_var_name = name; + + auto search_IdSpace = _refs.find(key_IdSpace); + + if(search_IdSpace != _refs.end()) + { + auto search_tensor_id = _refs[key_IdSpace].find(key_var_name); + + return search_tensor_id != _refs[key_IdSpace].end(); + } + else + { + return false; + } + } + + /** + * @brief Check whether the tensor argument is in the the IdSpace provided by the user + * + * @param[in] name Name of the tensor argument to search for + * @param[in] IdSpace The IdSpace id where to search the tensor argument + * + * @return true if the tile exists + * @return false if the tile does not exist + */ + bool has_tensor_argument(const std::string &name, int32_t IdSpace) + { + const int32_t key_IdSpace = IdSpace; + const std::string key_var_name = name; + + auto search_IdSpace = _refs.find(key_IdSpace); + + if(search_IdSpace != _refs.end()) + { + auto search_tensor_id = _refs[key_IdSpace].find(key_var_name); + + return search_tensor_id != _refs[key_IdSpace].end(); + } + else + { + return false; + } + } + +private: + // This method ensures that the key is unique among different components + std::string generate_tensor_name(const std::string &name, int32_t tensor_id) + { + assert(tensor_id >= 0); + + return name + std::to_string(tensor_id); + } + + std::map _tensor_arguments{}; + std::map> _refs{}; + int32_t _IdSpace{ -1 }; + GpuTargetLanguage _language{ GpuTargetLanguage::Unknown }; // Gpu programming language +}; + +enum class OpType : int32_t +{ + Elementwise = 0x0000, + Relational = 0x1000, + Algebra = 0x2000 +}; + +inline std::string to_string(AssignmentOp op) +{ + switch(op) + { + case AssignmentOp::Decrement: + return "-="; + case AssignmentOp::Increment: + return "+="; + default: + assert(false); + return ""; + } +} + +inline std::string to_string(UnaryOp op) +{ + switch(op) + { + case UnaryOp::LogicalNot: + return "!"; + case UnaryOp::BitwiseNot: + return "~"; + default: + assert(false); + return ""; + } +} + +inline std::string to_string(BinaryOp op) +{ + switch(op) + { + case BinaryOp::Add: + return "+"; + case BinaryOp::Sub: + return "-"; + case BinaryOp::Mul: + return "*"; + case BinaryOp::Div: + return "/"; + case BinaryOp::Mod: + return "%"; + case BinaryOp::Equal: + return "=="; + case BinaryOp::Less: + return "<"; + case BinaryOp::LessEqual: + return "<="; + case BinaryOp::Greater: + return ">"; + case BinaryOp::GreaterEqual: + return ">="; + case BinaryOp::LogicalAnd: + return "&&"; + case BinaryOp::LogicalOr: + return "||"; + case BinaryOp::BitwiseXOR: + return "^"; + default: + assert(false); + return ""; + } +} + +inline std::string binary_op_string(BinaryOp op) +{ + switch(op) + { + case BinaryOp::Add: + return "add"; + case BinaryOp::Sub: + return "sub"; + case BinaryOp::Mul: + return "mul"; + case BinaryOp::Div: + return "div"; + case BinaryOp::Mod: + return "mod"; + case BinaryOp::Equal: + return "eq"; + case BinaryOp::Less: + return "gt"; + case BinaryOp::LessEqual: + return "gteq"; + case BinaryOp::Greater: + return "lt"; + case BinaryOp::GreaterEqual: + return "lte"; + default: + assert(false); + return ""; + } +} + +enum class OperandType : int32_t +{ + Unknown = 0x00000000, + ScalarFp32 = 0x00001011, // Immediate scalar tile + ScalarFp16 = 0x00001012, // Immediate scalar tile + ScalarInt32 = 0x00001021, // Immediate scalar tile + ScalarInt16 = 0x00001022, // Immediate scalar tile + ScalarInt8 = 0x00001024, // Immediate scalar tile + ScalarUInt32 = 0x00001031, // Immediate scalar tile + ScalarUInt16 = 0x00001032, // Immediate scalar tile + ScalarUInt8 = 0x00001034, // Immediate scalar tile + ScalarBool = 0x00001041, // Immediate scalar tile + ScalarTile = 0x00001050, // Scalar from a tile + Tile = 0x00010000, // Tile + TensorStride1 = 0x00100001, // Tensor component + TensorStride2 = 0x00100002, // Tensor component + TensorStride3 = 0x00100003, // Tensor component + TensorStride4 = 0x00100004, // Tensor component + TensorDim0 = 0x00100010, // Tensor component + TensorDim1 = 0x00100020, // Tensor component + TensorDim2 = 0x00100030, // Tensor component + TensorDim3 = 0x00100040, // Tensor component + TensorDim4 = 0x00100050, // Tensor component + TensorC = 0x00100010, // Tensor component + TensorW = 0x00100020, // Tensor component + TensorH = 0x00100030, // Tensor component + TensorD = 0x00100040, // Tensor component + TensorN = 0x00100050, // Tensor component + TensorDim1xDim2 = 0x00100100, // Tensor component + TensorDim1xDim2xDim3 = 0x00100200, // Tensor component + TensorWxH = 0x00100300, // Tensor component + TensorWxHxD = 0x00100400, // Tensor component + TensorDataOffset = 0x00100500, // Tensor component +}; + +struct ScalarTileCoord +{ + ScalarTileCoord() + { + } + + ScalarTileCoord(int32_t x0, int32_t y0) + : x(x0), y(y0) + { + } + + int32_t x{ -1 }; + int32_t y{ -1 }; +}; + +/** + * @brief Operand class. This object is used to pass the operands to the operations performed by the writer. + * Operand can be of three types: + * -# Scalar immediate: constant expression + * -# Tile: A tile + * -# Tensor component: A component (scalar) of a tensor + * + */ +class Operand +{ +public: + Operand(const std::string &val) + { + _str = val; + _type = OperandType::Tile; + } + + Operand(const std::string &val, const ScalarTileCoord &coord) + { + _str = val; + _type = OperandType::ScalarTile; + _coord = coord; + } + + Operand(const std::string &val, OperandType type) + { + _str = val; + _type = type; + } + + Operand(const Operand &t) + { + _str = t.value(); + _type = t.type(); + } + + Operand &operator=(const Operand &t) + { + _str = t.value(); + _type = t.type(); + _coord = t.scalar_tile_coordinate(); + return *this; + } + + std::string value() const + { + return _str; + } + + OperandType type() const + { + return _type; + } + + ScalarTileCoord scalar_tile_coordinate() const + { + return _coord; + } + +private: + std::string _str{}; + OperandType _type{ OperandType::Unknown }; + ScalarTileCoord _coord{}; +}; + +using GpuSamplerTensorStorage = GpuTensorStorage; + +struct GpuSampler +{ + GpuSampler() = default; + + TensorSamplerFormat format{ TensorSamplerFormat::Unknown }; + GpuSamplerTensorStorage storage{ GpuSamplerTensorStorage::Unknown }; + TensorSamplerAddressModeX address_mode_x{ TensorSamplerAddressModeX::Unknown }; + TensorSamplerAddressModeY address_mode_y{ TensorSamplerAddressModeY::Unknown }; + TensorSamplerAddressModeZ address_mode_z{ TensorSamplerAddressModeZ::Unknown }; +}; + +inline GpuSampler +create_simple_sampler(const TensorInfo *tensor_info_id, GpuSampler sampler, int32_t step_x, int32_t step_y, + int32_t step_z) +{ + CKW_UNUSED(step_x, step_y, step_z); + + auto tensor = tensor_info_id->shape; + + GpuSampler dst_sampler; + dst_sampler.format = sampler.format; + dst_sampler.storage = GpuSamplerTensorStorage::BufferUint8Ptr; + dst_sampler.address_mode_x = sampler.address_mode_x; + dst_sampler.address_mode_y = sampler.address_mode_y; + dst_sampler.address_mode_z = sampler.address_mode_z; + + int32_t dim_x = 0; + int32_t dim_y = 0; + int32_t dim_z = 0; + + switch(sampler.format) + { + case TensorSamplerFormat::C_W_H: + dim_x = tensor[0]; + dim_y = tensor[1]; + dim_z = tensor[2]; + break; + case TensorSamplerFormat::C_WH_1: + dim_x = tensor[0]; + dim_y = tensor[1] * tensor[2]; + dim_z = 1; + break; + default: + std::cout << "Unsupported tensor format" << std::endl; + assert(false); + break; + } + + if(dim_x == 1) + { + assert(step_x == 1); + dst_sampler.address_mode_x = TensorSamplerAddressModeX::None; + } + + if(dim_y == 1) + { + assert(step_y == 1); + dst_sampler.address_mode_y = TensorSamplerAddressModeY::None; + } + + if(dim_z == 1) + { + assert(step_z == 1); + dst_sampler.address_mode_z = TensorSamplerAddressModeZ::None; + } + + return dst_sampler; +} + +class GpuOutputSampler +{ +public: + GpuOutputSampler() = default; + + /** + * @brief Method used to initialize the GpuOutputSampler. The GpuOutputSampler can be initialized only once + * by the root component. Once initialized, all simpler components will need to used this sampler + * or a broadcasted version of it + * + * @param[in] sampler GpuSampler + * @param[in] step_x Increment step in the X direction. Not necessarily it is the same of n0 of tile! + * @param[in] step_y Increment step in the Y direction. Not necessarily it is the same of m0 of tile! + * @param[in] step_z Increment step in the Z direction. Not necessarily it is the same of d0 of tile! + */ + void initialize(const TensorInfo *tensor_info_id, GpuSamplerTensorStorage tensor_storage, + TensorSamplerFormat tensor_format, int32_t step_x, int32_t step_y, int32_t step_z) + { + assert(_is_initialized == false); + + _step_x = step_x; + _step_y = step_y; + _step_z = step_z; + _tensor_info_id = tensor_info_id; + _sampler = create_sampler(tensor_storage, tensor_format); + _is_initialized = true; + }; + + GpuSampler sampler() const + { + return _sampler; + }; + + int32_t step_x() const + { + return _step_x; + }; + + int32_t step_y() const + { + return _step_y; + }; + + int32_t step_z() const + { + return _step_z; + }; + +private: + GpuSampler create_sampler(GpuSamplerTensorStorage tensor_storage, TensorSamplerFormat tensor_format) + { + // Output can only be in output mode + assert(tensor_storage != GpuSamplerTensorStorage::Image2dReadOnly); + assert(tensor_storage != GpuSamplerTensorStorage::Image3dReadOnly); + + auto tensor = _tensor_info_id->shape; + + GpuSampler sampler; + sampler.format = tensor_format; + sampler.storage = tensor_storage; + sampler.address_mode_x = TensorSamplerAddressModeX::None; + sampler.address_mode_y = TensorSamplerAddressModeY::None; + sampler.address_mode_z = TensorSamplerAddressModeZ::None; + + // In the case of texture, we do not need any special checks at the border + if(tensor_storage == GpuSamplerTensorStorage::BufferUint8Ptr) + { + int32_t dim_x = 0; + int32_t dim_y = 0; + int32_t dim_z = 0; + + switch(tensor_format) + { + case TensorSamplerFormat::C_W_H: + dim_x = tensor[0]; + dim_y = tensor[1]; + dim_z = tensor[2]; + break; + case TensorSamplerFormat::C_WH_1: + dim_x = tensor[0]; + dim_y = tensor[1] * tensor[2]; + dim_z = 1; + break; + default: + std::cout << "Unsupported tensor format" << std::endl; + assert(false); + break; + } + + if((dim_x % _step_x) != 0 && dim_x != 1) + { + sampler.address_mode_x = TensorSamplerAddressModeX::OverlappingMin; + } + + if((dim_y % _step_y) != 0 && dim_y != 1) + { + sampler.address_mode_y = TensorSamplerAddressModeY::ClampToMaxEdgeOnly; + } + + if((dim_z % _step_z) != 0 && dim_z != 1) + { + sampler.address_mode_z = TensorSamplerAddressModeZ::ClampToMaxEdgeOnly; + } + } + + return sampler; + } + + GpuSampler _sampler{}; // GpuSampler + int32_t _step_x{ 1 }; + int32_t _step_y{ 1 }; + int32_t _step_z{ 1 }; + const TensorInfo *_tensor_info_id{ nullptr }; + bool _is_initialized{ false }; +}; + +/** + * @brief Tensor operand class. This object is used to pass the operands as tensor to the operations performed by the writer. + */ +class TensorOperand +{ +public: + TensorOperand(const std::string &val, GpuSampler sampler) + : _str(val), _sampler(sampler) + { + } + + TensorOperand &operator=(const TensorOperand &t) + { + _str = t.value(); + _sampler = t.sampler(); + return *this; + } + + std::string value() const + { + return _str; + } + + GpuSampler sampler() const + { + return _sampler; + } + +private: + std::string _str{}; + GpuSampler _sampler{}; +}; + +/** + * @brief Data structure that contains all the necessary information to write the Gpu kernel with the Gpu kernel Writer + * This data structure must be initialized before being passed to the Gpu Kernel Writer + * + */ +class GpuKernelWriterDataHolder +{ +public: + /** + * @brief Construct a new Gpu Kernel Data object. In this phase, we should also store + * the GPU target and target specific capabilities (extensions). For now, we just initialize the + * programming language + * + * @param[in] language Gpu programming language to use + */ + GpuKernelWriterDataHolder(GpuTargetLanguage language) + : tiles(language), arguments(language), code(""), _language(language) + { + } + + /** + * @brief Get the Gpu programming language used + * + * @return GpuTargetLanguage the Gpu programming language + */ + GpuTargetLanguage programming_language() const + { + return _language; + } + + /** + * @brief @ref GpuTileRegistry + * + */ + GpuTileRegistry tiles{}; + /** + * @brief @ref GpuTensorArgumentRegistry + * + */ + GpuTensorArgumentRegistry arguments{}; + /** + * @brief @ref GpuOutputSampler. + * + */ + GpuOutputSampler output_sampler{}; + /** + * @brief Source code + * + */ + std::string code{}; + + // GpuExtensionRegistry extensions{}; +private: + GpuTargetLanguage _language; +}; + +struct LWS +{ + int32_t x{ 1 }; + int32_t y{ 1 }; + int32_t z{ 1 }; +}; + +/** + * @brief Utility class used to get the tile from the operand. If the operand is not a tile, @ref OperandUnpacker + * declare an anonymous tile in the tile registry. + */ +class OperandUnpacker +{ +public: + OperandUnpacker(GpuTileRegistry &tiles, GpuTensorArgumentRegistry &arguments) + : _tiles(tiles), _arguments(arguments) + { + // Increase the level of the stack to allocate possible temporary tiles + _tiles.increment_registry_level(); + }; + + ~OperandUnpacker() + { + // Decrease the level of the stack to deallocate any temporary tiles + _tiles.decrement_registry_level(); + } + + IVectorTile *unpack(const Operand &src) + { + // Get the tile + if(src.type() == OperandType::Tile) + { + assert(_tiles.has_tile(src.value())); + return _tiles[src.value()]; + } + // Create an anonymous tile with a constant + else if(static_cast(src.type()) & 0x00001000) + { + if(src.type() == OperandType::ScalarTile) + { + ScalarTileCoord coord = src.scalar_tile_coordinate(); + assert(_tiles.has_tile(src.value())); + assert(coord.x >= 0); + assert(coord.y >= 0); + auto val = _tiles[src.value()]->scalar(coord.x, coord.y); + return _tiles.insert({ { { val.str } } }, val.type.dt); + } + else + { + return _tiles.insert({ { { src.value() } } }, to_tile_data_type(src.type())); + } + } + // Create an anonymous tile with the tensor component + else + { + assert(_arguments.has_tensor_argument(src.value())); + auto x = _arguments[src.value()]; + const std::string val = x->component(to_tensor_component(src.type())); + const DataType dt = x->component_data_type(); + return _tiles.insert({ { { val } } }, dt); + } + } + +private: + DataType to_tile_data_type(OperandType x) + { + return static_cast(static_cast(x) & 0x00ff); + } + + TensorComponentType to_tensor_component(OperandType x) + { + switch(x) + { + case OperandType::TensorDim0: + return TensorComponentType::Dim0; + case OperandType::TensorDim1: + return TensorComponentType::Dim1; + case OperandType::TensorDim2: + return TensorComponentType::Dim2; + case OperandType::TensorDim3: + return TensorComponentType::Dim3; + case OperandType::TensorDim4: + return TensorComponentType::Dim4; + case OperandType::TensorStride1: + return TensorComponentType::Stride1; + case OperandType::TensorStride2: + return TensorComponentType::Stride2; + case OperandType::TensorStride3: + return TensorComponentType::Stride3; + case OperandType::TensorStride4: + return TensorComponentType::Stride4; + case OperandType::TensorDim1xDim2: + return TensorComponentType::Dim1xDim2; + case OperandType::TensorDim1xDim2xDim3: + return TensorComponentType::Dim1xDim2xDim3; + case OperandType::TensorDataOffset: + return TensorComponentType::OffsetFirstElement; + default: + assert(false); + return TensorComponentType::Unknown; + } + } + + GpuTileRegistry &_tiles; + GpuTensorArgumentRegistry &_arguments; +}; + +/** + * @brief Utility class used to get the tensor argument from the operand. If the operand is not a tile, @ref OperandUnpacker + * declare an anonymous tile in the tile registry. + * Tensor dimension reduction aims for reducing the tensor data dimension while keeping data's tensor structure. + */ +class TensorOperandUnpacker +{ +public: + TensorOperandUnpacker(GpuTensorArgumentRegistry &arguments) + : _arguments(arguments){}; + + IGpuTensorArgument *unpack(const TensorOperand &src) + { + assert(_arguments.has_tensor_argument(src.value())); + return _arguments[src.value()]; + } + +private: + GpuTensorArgumentRegistry &_arguments; +}; + +/** + * @brief The GpuKernel will be used in three occasions (stages): + * #- Compilation stage + * #- Tuning stage + * #- Dispatch stage + */ +struct GpuKernel +{ + // Compilation stage + std::string code{}; // Source code, required for the compilation stage + std::vector list_extensions{}; // Extensions, required for the compilation stage + // Tuning stage + std::string config_id{}; // Unique id, required for the tuning stage + std::vector list_lws{}; // LWS to test, required for the tuning stage + // Dispatch stage + GpuOutputSampler output_sampler{}; // GpuOutputSampler, required for the dispatch stage + std::vector> list_tensor_storages; // List of tensor storages, required for the dispatch stage + std::vector> list_tensor_components; // List of tensor components (width, stride,..), required for the dispatch stage) +}; + +// Generate all extension pragmas (hardcoded for now) +inline std::string generate_extensions() +{ + std::string ext = R"( +#if defined(cl_khr_fp16) +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#endif // defined(cl_khr_fp16) + +#if defined(cl_arm_integer_dot_product_int8) +#pragma OPENCL EXTENSION cl_arm_integer_dot_product_int8 : enable +#endif // defined(cl_arm_integer_dot_product_int8) + +#if defined(cl_arm_integer_dot_product_accumulate_int8) +#pragma OPENCL EXTENSION cl_arm_integer_dot_product_accumulate_int8 : enable +#endif // defined(cl_arm_integer_dot_product_accumulate_int8) + +#if defined(cl_arm_printf) +#pragma OPENCL EXTENSION cl_arm_printf : enable +#endif // defined(cl_arm_printf); +)"; + return ext; +} + +// This function should produce an object with the source +inline std::string generate_code(GpuKernelWriterDataHolder &in, const std::string &name) +{ + std::string code; + code += generate_extensions(); + code += "__kernel void "; + code += name; + code += "(\n"; + + auto IdSpaces = in.arguments.IdSpace_declarations(); + + std::vector arg_str; + + auto tensor_args = in.arguments.tensor_argument_declarations(); + + for(auto &i : tensor_args) + { + // For each tensor used, get the storage and tensor components + auto storages = i->storage_declarations(); + auto components = i->component_declarations(); + + for(auto &y : storages) + { + std::string str; + str += i->storage_type_declaration(y); + str += " "; + str += i->storage(y); + arg_str.push_back(str); + } + + for(auto &y : components) + { + std::string str; + str += i->component_type_declaration(); + str += " "; + str += i->component(y); + arg_str.push_back(str); + } + } + + for(size_t i = 0; i < arg_str.size(); ++i) + { + code += arg_str[i]; + if(i + 1 < arg_str.size()) + { + code += ",\n"; + } + } + + code += ")\n"; + code += "{\n"; + code += in.code; + code += "}\n"; + + return code; +} + +/** + * @brief This class is responsible to map a N-Tensor to a 3d tensor. The mapper needs the GpuSampler to know + * how to reduce the dimensionality of a tensor + * + */ +class GpuTensor3dMapper +{ +public: + GpuTensor3dMapper(IGpuTensorArgument *tensor, GpuSampler sampler) + : _sampler(sampler), _tensor(tensor){}; + + std::string tensor_component_x() const + { + const auto format = _sampler.format; + switch(format) + { + case TensorSamplerFormat::C_WH_1: + case TensorSamplerFormat::C_W_H: + return _tensor->component(TensorComponentType::Dim0); + default: + std::cout << "Unsupported tensor format" << std::endl; + assert(false); + return ""; + } + } + + std::string tensor_component_y() const + { + const auto format = _sampler.format; + switch(format) + { + case TensorSamplerFormat::C_WH_1: + return _tensor->component(TensorComponentType::Dim1xDim2); + case TensorSamplerFormat::C_W_H: + return _tensor->component(TensorComponentType::Dim1); + default: + std::cout << "Unsupported tensor format" << std::endl; + assert(false); + return ""; + } + } + + std::string tensor_component_z() const + { + const auto format = _sampler.format; + switch(format) + { + case TensorSamplerFormat::C_WH_1: + return "1"; + case TensorSamplerFormat::C_W_H: + return _tensor->component(TensorComponentType::Dim2); + default: + std::cout << "Unsupported tensor format" << std::endl; + assert(false); + return ""; + } + } + + std::string tensor_component_stride_y() const + { + const auto format = _sampler.format; + switch(format) + { + case TensorSamplerFormat::C_WH_1: + case TensorSamplerFormat::C_W_H: + return _tensor->component(TensorComponentType::Stride1); + default: + std::cout << "Unsupported tensor format" << std::endl; + assert(false); + return ""; + } + } + + std::string tensor_component_stride_z() const + { + const auto format = _sampler.format; + switch(format) + { + case TensorSamplerFormat::C_WH_1: + return "0"; + case TensorSamplerFormat::C_W_H: + return _tensor->component(TensorComponentType::Stride2); + default: + std::cout << "Unsupported tensor format" << std::endl; + assert(false); + return ""; + } + } + + std::string tensor_component_stride_batch() const + { + const auto format = _sampler.format; + switch(format) + { + case TensorSamplerFormat::C_WH_1: + case TensorSamplerFormat::C_W_H: + return _tensor->component(TensorComponentType::Stride3); + default: + std::cout << "Unsupported tensor format" << std::endl; + assert(false); + return ""; + } + } + + bool is_one_component_x() const + { + auto t = _tensor->format(); + const auto format = _sampler.format; + switch(format) + { + case TensorSamplerFormat::C_WH_1: + case TensorSamplerFormat::C_W_H: + return t.shape[0] == 1; + default: + std::cout << "Unsupported tensor format" << std::endl; + assert(false); + return ""; + } + } + + bool is_one_component_y() const + { + auto t = _tensor->format(); + const auto format = _sampler.format; + switch(format) + { + case TensorSamplerFormat::C_WH_1: + return (t.shape[1] * t.shape[2]) == 1; + case TensorSamplerFormat::C_W_H: + return t.shape[1] == 1; + default: + std::cout << "Unsupported tensor format" << std::endl; + assert(false); + return ""; + } + } + + bool is_one_component_z() const + { + auto t = _tensor->format(); + const auto format = _sampler.format; + switch(format) + { + case TensorSamplerFormat::C_WH_1: + return true; + case TensorSamplerFormat::C_W_H: + return t.shape[2] == 1; + default: + std::cout << "Unsupported tensor format" << std::endl; + assert(false); + return ""; + } + } + + bool is_one_component_batch() const + { + auto t = _tensor->format(); + const auto format = _sampler.format; + switch(format) + { + case TensorSamplerFormat::C_WH_1: + case TensorSamplerFormat::C_W_H: + return t.shape[3] == 1; + default: + std::cout << "Unsupported tensor format" << std::endl; + assert(false); + return ""; + } + } + + GpuSampler gpu_sampler() const + { + return _sampler; + } + + IGpuTensorArgument *tensor_argument() const + { + return _tensor; + } + +private: + GpuSampler _sampler; + IGpuTensorArgument *_tensor; +}; + +struct GpuKernelWriterAttribute +{ + bool return_tensor_component_by_value{ false }; +}; + +enum class RoundingMode +{ + None, + Rte, + Rtz, + Rtp, + Rtn +}; + +// https://llvm.org/docs/tutorial/MyFirstLanguageFrontend/LangImpl05.html +class IGpuKernelWriter +{ +public: + virtual ~IGpuKernelWriter() = default; + + virtual void set_IdSpace(int32_t id) = 0; + + virtual void import_tile(const std::string &dst, const IVectorTile *src) = 0; + + virtual void declare_argument(const std::string &name, const TensorInfo &tensor) = 0; + + virtual void declare_tile(const std::string &name, const TileInfo &info) = 0; + + virtual void declare_const_tile(const std::string &name, const std::vector> &in, DataType dt) = 0; + + virtual void write_text(const std::string &x) = 0; + + virtual void compound_statement_begin() = 0; + + virtual void compound_statement_end() = 0; + + // Operations + virtual void op_get_global_id(const Operand &dst_var, int32_t dim) = 0; + + virtual void op_get_global_coord(const Operand &dst, const Operand &step, const TensorOperand &tensor, int32_t dim) = 0; + + virtual void op_get_global_batch(const Operand &dst, const TensorOperand &tensor) = 0; + + virtual void op_get_global_size(const Operand &dst_var, int32_t dim) = 0; + + virtual void op_unary_expression(const Operand &dst, UnaryOp op, const Operand &src) = 0; + + virtual void op_binary_expression(const Operand &dst, const Operand &lhs, BinaryOp op, const Operand &rhs) = 0; + + virtual void op_assign(const Operand &dst_name, const Operand &src_name) = 0; + + virtual void op_unary_elementwise_function(const Operand &dst_name, UnaryFunction func, const Operand &src_name) = 0; + + virtual void op_binary_elementwise_function(const Operand &dst_name, BinaryFunction func, const Operand &first_name, const Operand &second_name) = 0; + + virtual void op_ternary_elementwise_function(const Operand &dst_name, TernaryFunction func, const Operand &first_name, const Operand &second_name, const Operand &third_name) = 0; + + virtual void op_if_header(const Operand &lhs, BinaryOp op, const Operand &rhs) = 0; + + virtual void op_else_if_header(const Operand &lhs, BinaryOp op, const Operand &rhs) = 0; + + virtual void op_else_header() = 0; + + virtual void op_for_loop_header(const Operand &var_name, BinaryOp cond_op, const Operand &cond_value, const Operand &update_var, AssignmentOp update_op, const Operand &update_value) = 0; + + virtual void op_load_indirect(const TensorOperand &tensor, const Operand &dst, const Operand &x, const Operand &y_indirect, const Operand &z, const Operand &b = Operand("0", OperandType::ScalarInt32)) = 0; + + virtual void op_load_immediate(const TensorOperand &tensor, const Operand &dst, const Operand &x, const Operand &y, const Operand &z, const Operand &b = Operand("0", OperandType::ScalarInt32), const Operand &dilation_y = Operand("1", OperandType::ScalarInt32)) = 0; + + virtual void op_store_immediate(const TensorOperand &tensor, const Operand &src, const Operand &x, const Operand &y, const Operand &z, const Operand &b = Operand("0", OperandType::ScalarInt32)) = 0; + + virtual void op_cast_expression(const Operand &dst, const Operand &src, ConvertPolicy policy) = 0; + + virtual void op_return() = 0; + + // Utils + // It is the process of converting + virtual void util_get_indirect_buffer(const Operand &dst, const TensorOperand &tensor, const Operand &x, + const Operand &y, const Operand &x_off, const Operand &y_off) = 0; +}; + +enum class GpuLoadStoreType +{ + Load = 1, + Store = 2 +}; + +class IGpuLoadStoreHelperWriter +{ +public: + IGpuLoadStoreHelperWriter(IGpuKernelWriter *x, GpuTensor3dMapper mapper, GpuLoadStoreType type) + : _writer(x), _mapper(mapper), _type(type) + { + } + + IGpuLoadStoreHelperWriter(const IGpuLoadStoreHelperWriter &) = default; + + IGpuLoadStoreHelperWriter &operator=(const IGpuLoadStoreHelperWriter &) = default; + + virtual ~IGpuLoadStoreHelperWriter() = default; + + virtual void initialize(IVectorTile *dst, IVectorTile *x, IVectorTile *z, IVectorTile *b) = 0; + + virtual void write(const std::pair &y) = 0; + + virtual void finalize() = 0; + +protected: + IGpuKernelWriter *_writer; + GpuTensor3dMapper _mapper; + GpuLoadStoreType _type; +}; + +class ClLoadStoreBufferHelperWriter : public IGpuLoadStoreHelperWriter +{ +public: + ClLoadStoreBufferHelperWriter(IGpuKernelWriter *x, const GpuTensor3dMapper &mapper, GpuLoadStoreType type) + : IGpuLoadStoreHelperWriter(x, mapper, type) + { + } + + ClLoadStoreBufferHelperWriter(const ClLoadStoreBufferHelperWriter &) = default; + + ClLoadStoreBufferHelperWriter &operator=(const ClLoadStoreBufferHelperWriter &) = default; + + static bool + validate(IGpuKernelWriter *x, GpuTensor3dMapper mapper, GpuLoadStoreType type, IVectorTile *dst) + { + CKW_UNUSED(x, type, dst); + + if(mapper.gpu_sampler().storage != GpuSamplerTensorStorage::BufferUint8Ptr) + { + return false; + } + return true; + } + + void initialize(IVectorTile *dst, IVectorTile *x, IVectorTile *z, IVectorTile *b) override + { + assert(validate(_writer, _mapper, _type, dst)); + + _dst = dst; + _ls_width_full = dst->format().w; + + _coord_x = x->scalar(0, 0).str; + _coord_z = z->scalar(0, 0).str; + _coord_b = b->scalar(0, 0).str; + _coord_orig_z = _coord_z; + + out_of_bound_initialize_x(_coord_x); + out_of_bound_initialize_z(_coord_z); + + /* + meaning of else: + - x: partial load/store + - y: no load/store operation + - z: no load/store operation + if(x) + { + if(z) + { + if(y) + { + // full load/store width + } + else + { + // no load/store + } + } + else + { + // no load/store + } + } + else + { + if(z) + { + if(y) + { + // partial load/store width + } + else + { + // no load/store + } + } + else + { + // no load/store + } + } + */ + } + + void write(const std::pair &y) override + { + int32_t idx_y = y.first; + std::string coord_y = y.second; + + // The only check required is on Y. + out_of_bound_initialize_y(coord_y); + + const std::string dst = _dst->vector(idx_y).str; + const std::string address = to_ls_buffer_address(_coord_x, coord_y, _coord_z, _coord_b); + const std::string ls_buf = to_ls_buffer(_type, _ls_width_full, dst, address); + + _writer->write_text(ls_buf); + _writer->write_text(";\n"); + + out_of_bound_finalize_y(dst); + + // The left over load/store will be written in the finalize stage + if(_ls_width_part.size() != 0) + { + int32_t w = 0; + for(auto &p : _ls_width_part) + { + const std::string dst0 = _dst->vector(w, p, idx_y).str; + const std::string coord_x = _coord_x + " + " + std::to_string(w); + const std::string address = to_ls_buffer_address(coord_x, coord_y, _coord_z, _coord_b); + const std::string ls_buf0 = to_ls_buffer(_type, p, dst0, address); + _leftovers_x.push_back(std::make_pair(std::make_pair(dst0, coord_y), ls_buf0)); + + w += p; + } + } + } + + void finalize() override + { + out_of_bound_finalize_z(); + out_of_bound_finalize_x(); + } + +private: + IVectorTile *_dst{ nullptr }; + int32_t _ls_width_full{ 0 }; + std::vector _ls_width_part{}; + std::vector, std::string>> _leftovers_x{}; + std::string _coord_x{}; + std::string _coord_z{}; + std::string _coord_orig_z{}; + std::string _coord_b{}; + + void out_of_bound_initialize_x(std::string &coord) + { + if(_mapper.gpu_sampler().address_mode_x == TensorSamplerAddressModeX::OverlappingMin) + { + auto tensor_format = _mapper.tensor_argument()->format(); + auto shape = tensor_format.shape; + + _ls_width_part = decompose_leftover_ls_vector_width(shape[0] % _ls_width_full); + if(_ls_width_part.size() != 0) + { + _writer->write_text("if(" + coord + " > 0)\n"); + _writer->compound_statement_begin(); + } + } + }; + + void out_of_bound_finalize_x() + { + if(_mapper.gpu_sampler().address_mode_x == TensorSamplerAddressModeX::OverlappingMin) + { + if(_ls_width_part.size() != 0) + { + _writer->compound_statement_end(); + _writer->write_text("else\n"); + _writer->compound_statement_begin(); + + out_of_bound_initialize_z(_coord_orig_z); + for(auto &i : _leftovers_x) + { + out_of_bound_initialize_y(i.first.second); + _writer->write_text(i.second); + _writer->write_text(";\n"); + out_of_bound_finalize_y(i.first.first); + } + out_of_bound_finalize_z(); + _writer->compound_statement_end(); + } + } + }; + + void out_of_bound_initialize_y(std::string &coord) + { + std::string max = ""; + + const auto address_mode_y = _mapper.gpu_sampler().address_mode_y; + + switch(address_mode_y) + { + case TensorSamplerAddressModeY::Skip: + case TensorSamplerAddressModeY::ClampToBorder: + // NOTE: This line should not be moved outside of the switch statement. + // The reason for that is because when we query the component, the component is marked as used + // and added to the list of arguments of the kernel. Since, not in all cases this component is required, + // we should request the component only when used + max = _mapper.tensor_component_y(); + _writer->write_text("if((" + coord + " >= 0) && (" + coord + " < " + max + "))\n"); + _writer->compound_statement_begin(); + break; + case TensorSamplerAddressModeY::SkipMinEdgeOnly: + case TensorSamplerAddressModeY::ClampToBorderMinEdgeOnly: + _writer->write_text("if(" + coord + " >= 0)\n"); + _writer->compound_statement_begin(); + break; + case TensorSamplerAddressModeY::SkipMaxEdgeOnly: + case TensorSamplerAddressModeY::ClampToBorderMaxEdgeOnly: + max = _mapper.tensor_component_y(); + _writer->write_text("if(" + coord + " < " + max + ")\n"); + _writer->compound_statement_begin(); + break; + case TensorSamplerAddressModeY::ClampToNearest: + max = _mapper.tensor_component_y(); + coord = "clamp(" + coord + ", 0, " + max + " - 1)"; + break; + case TensorSamplerAddressModeY::ClampToMaxEdgeOnly: + max = _mapper.tensor_component_y(); + coord = "min(" + coord + ", " + max + " - 1)"; + break; + case TensorSamplerAddressModeY::ClampToMinEdgeOnly: + coord = "max(" + coord + ", 0)"; + break; + case TensorSamplerAddressModeY::None: + break; + default: + std::cout << "Unsupported address mode for write_out_of_bound_check_yz" << std::endl; + assert(false); + } + }; + + void out_of_bound_finalize_y(const std::string &dst) + { + const auto address_mode_y = _mapper.gpu_sampler().address_mode_y; + + switch(address_mode_y) + { + case TensorSamplerAddressModeY::ClampToBorder: + case TensorSamplerAddressModeY::ClampToBorderMaxEdgeOnly: + case TensorSamplerAddressModeY::ClampToBorderMinEdgeOnly: + case TensorSamplerAddressModeY::Skip: + case TensorSamplerAddressModeY::SkipMaxEdgeOnly: + case TensorSamplerAddressModeY::SkipMinEdgeOnly: + _writer->compound_statement_end(); + break; + case TensorSamplerAddressModeY::None: + break; + + default: + assert(false); + } + + switch(address_mode_y) + { + case TensorSamplerAddressModeY::ClampToBorder: + case TensorSamplerAddressModeY::ClampToBorderMinEdgeOnly: + case TensorSamplerAddressModeY::ClampToBorderMaxEdgeOnly: + _writer->write_text("else\n"); + _writer->compound_statement_begin(); + _writer->write_text(dst); + _writer->write_text(" = 0.0f;\n"); + _writer->compound_statement_end(); + break; + case TensorSamplerAddressModeY::None: + break; + + default: + assert(false); + } + }; + + void out_of_bound_initialize_z(std::string &coord) + { + std::string max = ""; + + const auto address_mode_z = _mapper.gpu_sampler().address_mode_z; + + switch(address_mode_z) + { + case TensorSamplerAddressModeZ::Skip: + max = _mapper.tensor_component_z(); + _writer->write_text("if((" + coord + " >= 0) && (" + coord + " < " + max + "))\n"); + _writer->compound_statement_begin(); + break; + case TensorSamplerAddressModeZ::SkipMinEdgeOnly: + _writer->write_text("if(" + coord + " >= 0)\n"); + _writer->compound_statement_begin(); + break; + case TensorSamplerAddressModeZ::SkipMaxEdgeOnly: + max = _mapper.tensor_component_z(); + _writer->write_text("if(" + coord + " < " + max + ")\n"); + _writer->compound_statement_begin(); + break; + case TensorSamplerAddressModeZ::ClampToNearest: + max = _mapper.tensor_component_z(); + coord = "clamp(" + coord + ", 0, " + max + " - 1)"; + break; + case TensorSamplerAddressModeZ::ClampToMaxEdgeOnly: + max = _mapper.tensor_component_z(); + coord = "min(" + coord + ", " + max + " - 1)"; + break; + case TensorSamplerAddressModeZ::ClampToMinEdgeOnly: + coord = "max(" + coord + ", 0)"; + break; + case TensorSamplerAddressModeZ::None: + break; + default: + std::cout << "Unsupported address mode for write_out_of_bound_check_yz" << std::endl; + assert(false); + } + }; + + void out_of_bound_finalize_z() + { + const auto address_mode_z = _mapper.gpu_sampler().address_mode_z; + + switch(address_mode_z) + { + case TensorSamplerAddressModeZ::Skip: + case TensorSamplerAddressModeZ::SkipMinEdgeOnly: + case TensorSamplerAddressModeZ::SkipMaxEdgeOnly: + _writer->compound_statement_end(); + break; + case TensorSamplerAddressModeZ::None: + break; + + default: + assert(false); + } + }; + + std::vector decompose_leftover_ls_vector_width(int32_t ls_leftover_vector_width) const + { + std::vector x; + + switch(ls_leftover_vector_width) + { + case 0: + break; + case 1: + case 2: + case 3: + case 4: + case 8: + case 16: + x.push_back(ls_leftover_vector_width); + break; + case 5: + x.push_back(4); + x.push_back(1); + break; + case 6: + x.push_back(4); + x.push_back(2); + break; + case 7: + x.push_back(4); + x.push_back(3); + break; + case 9: + x.push_back(8); + x.push_back(1); + break; + case 10: + x.push_back(8); + x.push_back(2); + break; + case 11: + x.push_back(8); + x.push_back(3); + break; + case 12: + x.push_back(8); + x.push_back(4); + break; + case 13: + x.push_back(8); + x.push_back(4); + x.push_back(1); + break; + case 14: + x.push_back(8); + x.push_back(4); + x.push_back(2); + break; + case 15: + x.push_back(8); + x.push_back(4); + x.push_back(3); + break; + + default: + assert(false); + } + return x; + } + + std::string to_ls_buffer(GpuLoadStoreType type, int32_t vector_width, const std::string &data, + const std::string &address) + { + switch(type) + { + case GpuLoadStoreType::Load: + if(vector_width != 1) + { + return data + " = vload" + std::to_string(vector_width) + "(0, " + address + ")"; + } + else + { + return data + " = *(" + address + ")"; + } + break; + case GpuLoadStoreType::Store: + if(vector_width != 1) + { + return "vstore" + std::to_string(vector_width) + "(" + data + ", 0, " + address + ")"; + } + else + { + return "*(" + address + ") = " + data; + } + break; + default: + std::cout << "Unsupported GpuLoadStoreType" << std::endl; + assert(false); + return ""; + } + } + + std::string to_ls_buffer_address(const std::string &x, const std::string &y, const std::string &z, + const std::string &b) const + { + auto tensor_storage = static_cast(_mapper.gpu_sampler().storage); + assert(tensor_storage == GpuTensorStorage::BufferUint8Ptr); + const std::string ptr_buf = _mapper.tensor_argument()->storage(tensor_storage); + const std::string dst_type = get_cl_data_type(_dst->format().dt, 1); + + std::string address; + address += "(__global "; + address += dst_type; + address += "*)("; + address += ptr_buf; + if(x != "0" && (_mapper.is_one_component_x() != true)) + { + address += " + ("; + address += x + ") * sizeof(" + dst_type + ")"; + } + if(y != "0") + { + const std::string stride_y = _mapper.tensor_component_stride_y(); + address += " + ("; + address += y + ")"; + address += " * "; + address += stride_y; + } + if(z != "0" && (_mapper.is_one_component_z() != true)) + { + const std::string stride_z = _mapper.tensor_component_stride_z(); + address += " + ("; + address += z + ")"; + address += " * "; + address += stride_z; + } + if(b != "0" && (_mapper.is_one_component_batch() != true)) + { + const std::string stride_b = _mapper.tensor_component_stride_batch(); + address += " + ("; + address += b + ")"; + address += " * "; + address += stride_b; + } + address += ")"; + return address; + } +}; + +class ClLoadStoreImage2dHelperWriter : public IGpuLoadStoreHelperWriter +{ +public: + static bool + validate(IGpuKernelWriter *x, const GpuTensor3dMapper &mapper, GpuLoadStoreType type, IVectorTile *dst) + { + CKW_UNUSED(x); + + if(dst->format().w != 4) + { + return false; + } + if(mapper.gpu_sampler().address_mode_x != TensorSamplerAddressModeX::None) + { + return false; + } + if(mapper.gpu_sampler().address_mode_z != TensorSamplerAddressModeZ::None) + { + return false; + } + if(mapper.gpu_sampler().storage != GpuSamplerTensorStorage::Image2dReadOnly && type == GpuLoadStoreType::Load) + { + return false; + } + if(mapper.gpu_sampler().storage != GpuSamplerTensorStorage::Image2dWriteOnly && type == GpuLoadStoreType::Store) + { + return false; + } + if((dst->format().dt != DataType::Fp32) && (dst->format().dt != DataType::Fp16)) + { + return false; + } + return true; + /* + - x: Only GpuSamplerAddressModeX::None is supported and vector length = 4 + - z: Only GpuSamplerAddressModeZ::None is supported + */ + } + + ClLoadStoreImage2dHelperWriter(IGpuKernelWriter *x, const GpuTensor3dMapper &mapper, GpuLoadStoreType type) + : IGpuLoadStoreHelperWriter(x, mapper, type) + { + } + + ClLoadStoreImage2dHelperWriter(const ClLoadStoreImage2dHelperWriter &) = default; + + ClLoadStoreImage2dHelperWriter &operator=(const ClLoadStoreImage2dHelperWriter &) = default; + + void initialize(IVectorTile *dst, IVectorTile *x, IVectorTile *z, IVectorTile *b) override + { + assert(validate(_writer, _mapper, _type, dst)); + + _dst = dst; + _ls_width_full = dst->format().w; + _coord_x = x->scalar(0, 0).str; + _coord_z = z->scalar(0, 0).str; + _coord_b = b->scalar(0, 0).str; + + /* + if(y) + { + // full load/store width + } + else + { + // no load/store + } + */ + } + + void write(const std::pair &y) override + { + int32_t idx_y = y.first; + std::string coord_y = y.second; + + // The only check required is on Y. + out_of_bound_initialize_y(coord_y); + + const std::string dst = _dst->vector(idx_y).str; + const std::string sampler = to_ls_image2d_sampler(); + const std::string coord = to_ls_image2d_coord(_coord_x, coord_y, _coord_z, _coord_b); + const std::string ls_buf = to_ls_image2d(_type, _ls_width_full, dst, sampler, coord); + + _writer->write_text(ls_buf); + _writer->write_text(";\n"); + + out_of_bound_finalize_y(dst); + } + + void finalize() override + { + } + +private: + IVectorTile *_dst{ nullptr }; + int32_t _ls_width_full{ 0 }; + std::string _coord_x{}; + std::string _coord_z{}; + std::string _coord_b{}; + + void out_of_bound_initialize_y(std::string &coord) + { + std::string max = ""; + + const auto address_mode_y = _mapper.gpu_sampler().address_mode_y; + + switch(address_mode_y) + { + case TensorSamplerAddressModeY::Skip: + max = _mapper.tensor_component_y(); + _writer->write_text("if((" + coord + " >= 0) && (" + coord + " < " + max + "))\n"); + _writer->compound_statement_begin(); + break; + case TensorSamplerAddressModeY::SkipMinEdgeOnly: + _writer->write_text("if(" + coord + " >= 0)\n"); + _writer->compound_statement_begin(); + break; + case TensorSamplerAddressModeY::SkipMaxEdgeOnly: + max = _mapper.tensor_component_y(); + _writer->write_text("if(" + coord + " < " + max + ")\n"); + _writer->compound_statement_begin(); + break; + case TensorSamplerAddressModeY::ClampToBorder: + case TensorSamplerAddressModeY::ClampToBorderMinEdgeOnly: + case TensorSamplerAddressModeY::ClampToBorderMaxEdgeOnly: + case TensorSamplerAddressModeY::ClampToNearest: + case TensorSamplerAddressModeY::ClampToMaxEdgeOnly: + case TensorSamplerAddressModeY::ClampToMinEdgeOnly: + case TensorSamplerAddressModeY::None: + break; + default: + std::cout << "Unsupported address mode for write_out_of_bound_check_y" << std::endl; + assert(false); + } + }; + + void out_of_bound_finalize_y(const std::string &dst) + { + CKW_UNUSED(dst); + + const auto address_mode_y = _mapper.gpu_sampler().address_mode_y; + + switch(address_mode_y) + { + case TensorSamplerAddressModeY::Skip: + case TensorSamplerAddressModeY::SkipMinEdgeOnly: + case TensorSamplerAddressModeY::SkipMaxEdgeOnly: + _writer->compound_statement_end(); + break; + + default: + assert(false); + } + }; + + std::string to_ls_image2d(GpuLoadStoreType type, int32_t vector_width, const std::string &data, + const std::string &sampler, const std::string &coord) + { + CKW_UNUSED(vector_width); + + auto tensor_storage = static_cast(_mapper.gpu_sampler().storage); + const std::string image2d_obj = _mapper.tensor_argument()->storage(tensor_storage); + const std::string post_fix = _dst->format().dt == DataType::Fp32 ? "f" : "h"; + + switch(type) + { + case GpuLoadStoreType::Load: + return data + " = read_image" + post_fix + "(" + image2d_obj + ", " + sampler + ", " + coord + ")"; + break; + case GpuLoadStoreType::Store: + return "write_image" + post_fix + "(" + image2d_obj + ", " + coord + ", " + data + ")"; + default: + assert(false); + std::cout << "Unsupported GpuLoadStoreType" << std::endl; + assert(false); + return ""; + } + } + + std::string to_ls_image2d_sampler() const + { + const auto address_mode_y = _mapper.gpu_sampler().address_mode_y; + + switch(address_mode_y) + { + case TensorSamplerAddressModeY::None: + return "CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_NONE | CLK_FILTER_NEAREST"; + case TensorSamplerAddressModeY::Skip: + case TensorSamplerAddressModeY::SkipMinEdgeOnly: + case TensorSamplerAddressModeY::SkipMaxEdgeOnly: + case TensorSamplerAddressModeY::ClampToBorder: + case TensorSamplerAddressModeY::ClampToBorderMinEdgeOnly: + case TensorSamplerAddressModeY::ClampToBorderMaxEdgeOnly: + return "CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP | CLK_FILTER_NEAREST"; + case TensorSamplerAddressModeY::ClampToNearest: + case TensorSamplerAddressModeY::ClampToMaxEdgeOnly: + case TensorSamplerAddressModeY::ClampToMinEdgeOnly: + return "CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_NEAREST"; + default: + std::cout << "Unsupported address_mode_coord" << std::endl; + assert(false); + return ""; + } + } + + std::string to_ls_image2d_coord(const std::string &x, const std::string &y, const std::string &z, + const std::string &b) const + { + std::string coord_x = "(" + x + ") >> 2"; + std::string coord_y = "("; + + if(y != "0") + { + coord_y += y; + } + if(z != "0" && (_mapper.is_one_component_z() != true)) + { + const std::string dim = _mapper.tensor_component_y(); + coord_y += " + ("; + coord_y += z + ")"; + coord_y += " * "; + coord_y += dim; + } + if(b != "0" && (_mapper.is_one_component_batch() != true)) + { + const std::string dim0 = _mapper.tensor_component_y(); + const std::string dim1 = _mapper.tensor_component_z(); + coord_y += " + ("; + coord_y += b + ")"; + coord_y += " * "; + coord_y += dim0; + coord_y += " * "; + coord_y += dim1; + } + coord_y += ")"; + return "(int2)(" + coord_x + ", " + coord_y + ")"; + } +}; + +/** IGpuLoadStoreHelperWriter factory class */ +class ClLoadStoreHelperWriterFactory final +{ +public: + /** Static method to call the IGpuLoadStoreHelperWriter class accordingly with the tensor storage set in the mapper + * + * + * @return IGpuLoadStoreHelperWriter + */ + static std::unique_ptr + create(IGpuKernelWriter *x, const GpuTensor3dMapper &mapper, GpuLoadStoreType type) + { + const auto tensor_storage = mapper.gpu_sampler().storage; + switch(tensor_storage) + { + case GpuSamplerTensorStorage::BufferUint8Ptr: + return std::make_unique(x, mapper, type); + case GpuSamplerTensorStorage::Image2dReadOnly: + case GpuSamplerTensorStorage::Image2dWriteOnly: + return std::make_unique(x, mapper, type); + default: + std::cout << "Unsupported Gpu tensor storage" << std::endl; + assert(false); + return nullptr; + } + } +}; + +// This utility method needs to go in utils.h +inline bool is_tile_scalar(const IVectorTile *x) +{ + return x->format().w == 1 && x->format().h == 1; +} + +class ClKernelWriter : public IGpuKernelWriter +{ +public: + ClKernelWriter(GpuKernelWriterAttribute *attr, GpuKernelWriterDataHolder *x) + { + _data = x; + _attr = attr; + } + + ClKernelWriter(const ClKernelWriter &) = default; + + ClKernelWriter &operator=(const ClKernelWriter &) = default; + + // A IdSpaced ID is a term used to describe a fragment that is registered in ICode to ensure + // there are no conflicts or ambiguity in the code + void set_IdSpace(int32_t id) override + { + _data->tiles.set_IdSpace(id); + _data->arguments.set_IdSpace(id); + } + + void import_tile(const std::string &dst_name, const IVectorTile *src) override + { + _data->tiles.insert(dst_name, src); + } + + void declare_argument(const std::string &name, const TensorInfo &tensor) override + { + assert(_data->arguments[name] == nullptr); + _data->arguments.insert(name, tensor, _attr->return_tensor_component_by_value); + } + + void declare_tile(const std::string &name, const TileInfo &format) override + { + assert(_data->tiles[name] == nullptr); + _data->tiles.insert(name, format); + + IVectorTile *x = _data->tiles[name]; + + for(auto &t : x->underlying_source_variables()) + { + _data->code += t.type.str + " " + t.str + ";\n"; + } + } + + void declare_const_tile(const std::string &name, const std::vector> &in, + DataType dt) override + { + assert(_data->tiles[name] == nullptr); + _data->tiles.insert(name, in, dt); + // Note: A constant does not need to be declared in the code + } + + void write_text(const std::string &x) override + { + _data->code += x; + } + + void compound_statement_begin() override + { + _data->tiles.increment_registry_level(); + _data->code += "{\n"; + } + + void compound_statement_end() override + { + _data->tiles.decrement_registry_level(); + _data->code += "}\n"; + } + + void op_get_global_id(const Operand &dst_var, int32_t dim) override + { + assert(dst_var.type() == OperandType::Tile); + assert(_data->tiles.has_tile(dst_var.value())); + assert(_data->tiles[dst_var.value()]->format().w == 1 && _data->tiles[dst_var.value()]->format().h == 1); // It must be a scalar variable + + auto var = _data->tiles[dst_var.value()]; + + _data->code += var->scalar(0, 0).str; + _data->code += " = get_global_id("; + _data->code += std::to_string(dim); + _data->code += ");\n"; + }; + + void op_get_global_coord(const Operand &o_dst, const Operand &o_step, const TensorOperand &o_tensor, + int32_t dim) override + { + OperandUnpacker operands(_data->tiles, _data->arguments); + auto dst = operands.unpack(o_dst); + auto step = operands.unpack(o_step); + + // Validation: Check that x, y and z are scalar + + TensorOperandUnpacker tensor_operands(_data->arguments); + auto tensor = tensor_operands.unpack(o_tensor); + auto gpu_sampler = o_tensor.sampler(); + + GpuTensor3dMapper mapper(tensor, gpu_sampler); + + switch(dim) + { + case 0: + if(mapper.is_one_component_x()) + { + _data->code += dst->scalar(0, 0).str; + _data->code += " = 0;\n"; + } + else + { + if(mapper.gpu_sampler().address_mode_x == TensorSamplerAddressModeX::OverlappingMin) + { + // Validation: Check: fixed tensor shape + // TO BE CHANGED + _data->code += dst->scalar(0, 0).str; + _data->code += " = get_global_id(0) * "; + _data->code += step->scalar(0, 0).str; + _data->code += ";\n"; + } + else + { + _data->code += dst->scalar(0, 0).str; + _data->code += " = get_global_id(0) * "; + _data->code += step->scalar(0, 0).str; + _data->code += ";\n"; + } + } + break; + case 1: + if(mapper.is_one_component_y()) + { + _data->code += dst->scalar(0, 0).str; + _data->code += " = 0;\n"; + } + else + { + if(mapper.gpu_sampler().address_mode_y == TensorSamplerAddressModeY::OverlappingMin) + { + } + else + { + _data->code += dst->scalar(0, 0).str; + _data->code += " = get_global_id(1) * "; + _data->code += step->scalar(0, 0).str; + _data->code += ";\n"; + } + } + break; + case 2: + if(mapper.is_one_component_z()) + { + _data->code += dst->scalar(0, 0).str; + _data->code += " = 0;\n"; + } + else + { + _data->code += dst->scalar(0, 0).str; + _data->code += " = get_global_id(2) * "; + _data->code += step->scalar(0, 0).str; + _data->code += ";\n"; + } + break; + default: + break; + } + }; + + void op_get_global_batch(const Operand &o_dst, const TensorOperand &o_tensor) override + { + OperandUnpacker operands(_data->tiles, _data->arguments); + const IVectorTile *dst = operands.unpack(o_dst); + + TensorOperandUnpacker tensor_operands(_data->arguments); + IGpuTensorArgument *tensor = tensor_operands.unpack(o_tensor); + auto gpu_sampler = o_tensor.sampler(); + + GpuTensor3dMapper mapper(tensor, gpu_sampler); + + if(mapper.is_one_component_batch()) + { + _data->code += dst->scalar(0, 0).str; + _data->code += " = 0;\n"; + } + else + { + std::cout << "Unsupported batched computation" << std::endl; + assert(false); + } + }; + + void op_get_global_size(const Operand &dst_var, int32_t dim) override + { + assert(dst_var.type() == OperandType::Tile); + assert(_data->tiles.has_tile(dst_var.value())); + assert(_data->tiles[dst_var.value()]->format().w == 1 && _data->tiles[dst_var.value()]->format().h == 1); // It must be a scalar variable + + auto var = _data->tiles[dst_var.value()]; + + _data->code += var->scalar(0, 0).str; + _data->code += " = get_global_size("; + _data->code += std::to_string(dim); + _data->code += ");\n"; + } + + void op_unary_expression(const Operand &dst_name, UnaryOp op, const Operand &src_name) override + { + OperandUnpacker operands(_data->tiles, _data->arguments); + const IVectorTile *src = operands.unpack(src_name); + const IVectorTile *dst = operands.unpack(dst_name); + + const int32_t dst_w = dst->format().w; + const int32_t dst_h = dst->format().h; + const int32_t src_w = src->format().w; + const std::string dt = dst->underlying_source_variables()[0].type.str; + + const bool broadcast_src_x = dst_w != 1 && src_w == 1; + + const std::string src_prefix = broadcast_src_x ? "(" + dt + ")" : ""; + + // Broadcasting on Y is automatic + for(int32_t y = 0; y < dst_h; ++y) + { + _data->code += dst->vector(y).str; + _data->code += " = "; + _data->code += to_string(op); + _data->code += src_prefix + src->vector(y).str; + _data->code += ";\n"; + } + } + + void op_binary_expression(const Operand &dst_name, const Operand &lhs_name, BinaryOp op, + const Operand &rhs_name) override + { + OperandUnpacker operands(_data->tiles, _data->arguments); + const IVectorTile *lhs = operands.unpack(lhs_name); + const IVectorTile *rhs = operands.unpack(rhs_name); + const IVectorTile *dst = operands.unpack(dst_name); + + const int32_t dst_w = dst->format().w; + const int32_t dst_h = dst->format().h; + assert(lhs != nullptr); + const int32_t lhs_w = lhs->format().w; + const int32_t rhs_w = rhs->format().w; + + if(op == BinaryOp::MatMul_Nt_T) + { + assert((dst->format().dt == DataType::Fp32) || (dst->format().dt == DataType::Fp16)); + for(int32_t y = 0; y < dst_h; ++y) + { + for(int32_t x = 0; x < dst_w; ++x) + { + for(int32_t k = 0; k < lhs_w; ++k) + { + _data->code += dst->scalar(x, y).str; + _data->code += " = fma("; + _data->code += lhs->scalar(k, y).str; + _data->code += ", "; + _data->code += rhs->scalar(k, x).str; + _data->code += ", "; + _data->code += dst->scalar(x, y).str; + _data->code += ");\n"; + } + } + } + + return; + } + + const bool broadcast_lhs_x = dst_w != 1 && lhs_w == 1; + const bool broadcast_rhs_x = dst_w != 1 && rhs_w == 1; + + const std::string lhs_prefix = broadcast_lhs_x ? "(" + dst->underlying_source_variables()[0].type.str + ")" : ""; + const std::string rhs_prefix = broadcast_rhs_x ? "(" + dst->underlying_source_variables()[0].type.str + ")" : ""; + const std::string op_str = to_string(op); + + // Broadcasting on Y is automatic + for(int32_t y = 0; y < dst_h; ++y) + { + _data->code += dst->vector(y).str; + _data->code += " = "; + _data->code += lhs_prefix + lhs->vector(y).str; + _data->code += " "; + _data->code += op_str; + _data->code += " "; + _data->code += rhs_prefix + rhs->vector(y).str; + _data->code += ";\n"; + } + }; + + void op_cast_expression(const Operand &o_dst, const Operand &o_src, ConvertPolicy policy) override + { + OperandUnpacker operands(_data->tiles, _data->arguments); + const IVectorTile *src = operands.unpack(o_src); + const IVectorTile *dst = operands.unpack(o_dst); + // const int32_t dst_w = dst->format().w; + const int32_t dst_h = dst->format().h; + const std::string dt = dst->underlying_source_variables()[0].type.str; + const bool is_float = (dst->format().dt == DataType::Fp32) || (dst->format().dt == DataType::Fp16); + const std::string sat = ((policy == ConvertPolicy::Saturate && !is_float) ? "_sat" : ""); + + // Broadcasting on Y is automatic + for(int32_t y = 0; y < dst_h; ++y) + { + _data->code += dst->vector(y).str; + _data->code += " = convert_" + dt + sat + "("; + _data->code += src->vector(y).str; + _data->code += ");\n"; + } + }; + + void op_assign(const Operand &dst_name, const Operand &src_name) override + { + OperandUnpacker operands(_data->tiles, _data->arguments); + const IVectorTile *src = operands.unpack(src_name); + const IVectorTile *dst = operands.unpack(dst_name); + + const int32_t dst_w = dst->format().w; + const int32_t dst_h = dst->format().h; + const int32_t src_w = src->format().w; + const std::string dt = dst->underlying_source_variables()[0].type.str; + + const bool broadcast_src_x = dst_w != 1 && src_w == 1; + + const std::string src_prefix = broadcast_src_x ? "(" + dt + ")" : ""; + + // Broadcasting on Y is automatic + for(int32_t y = 0; y < dst_h; ++y) + { + _data->code += dst->vector(y).str; + _data->code += " = "; + _data->code += src_prefix + src->vector(y).str; + _data->code += ";\n"; + } + } + + void + op_unary_elementwise_function(const Operand &dst_name, UnaryFunction func, const Operand &src_name) override + { + OperandUnpacker operands(_data->tiles, _data->arguments); + const IVectorTile *src = operands.unpack(src_name); + const IVectorTile *dst = operands.unpack(dst_name); + + const int32_t dst_h = dst->format().h; + const std::string dt = dst->underlying_source_variables()[0].type.str; + + // Always perform an explicit cast. This automatically covers at least the 2 scenarios: + // 1. Widen a scalar into a vector type. This enables scalar-vector broadcasting + // 2. Ensure non-ambiguity over function overloads. + // E.g. a constant tile may be accidentally initialized with a double literal. By casting it to single float, + // it avoids ambiguous function calls + const std::string src_prefix = "(" + dt + ")"; + + // Broadcasting on Y is automatic + for(int32_t y = 0; y < dst_h; ++y) + { + _data->code += dst->vector(y).str; + _data->code += " = "; + + switch(func) + { + case UnaryFunction::Exp: + _data->code += "exp("; + break; + case UnaryFunction::Tanh: + _data->code += "tanh("; + break; + case UnaryFunction::Sqrt: + _data->code += "sqrt("; + break; + case UnaryFunction::Erf: + _data->code += "erf("; + break; + case UnaryFunction::Fabs: + _data->code += "fabs("; + break; + case UnaryFunction::Log: + _data->code += "log("; + break; + case UnaryFunction::SizeOf: + _data->code += "sizeof("; + break; + case UnaryFunction::Round: + _data->code += "round("; + break; + default: + CKW_ASSERT_MSG(false, "Unexpected UnaryFunction used."); + } + + _data->code += src_prefix + src->vector(y).str; + _data->code += ");\n"; + } + } + + void op_binary_elementwise_function(const Operand &dst_name, BinaryFunction func, const Operand &first_name, const Operand &second_name) override + { + OperandUnpacker operands(_data->tiles, _data->arguments); + const IVectorTile *first = operands.unpack(first_name); + const IVectorTile *second = operands.unpack(second_name); + const IVectorTile *dst = operands.unpack(dst_name); + + const int32_t dst_h = dst->format().h; + const auto datatype = dst->underlying_source_variables()[0].type; + const std::string datatype_str = datatype.str; + + // Always perform an explicit cast. See similar comments in op_unary_elementwise_function + const std::string first_prefix = "(" + datatype_str + ")"; + const std::string second_prefix = "(" + datatype_str + ")"; + + const bool is_float = (datatype.dt == DataType::Fp32 || datatype.dt == DataType::Fp16); + + // Broadcasting on Y is automatic + for(int32_t y = 0; y < dst_h; ++y) + { + _data->code += dst->vector(y).str; + _data->code += " = "; + + switch(func) + { + case BinaryFunction::Min: + _data->code += is_float ? "fmin(" : "min("; + break; + case BinaryFunction::Max: + _data->code += is_float ? "fmax(" : "max("; + break; + default: + CKW_ASSERT_MSG(false, "Unexpected BinaryFunction used."); + } + + _data->code += first_prefix + first->vector(y).str; + _data->code += ", "; + _data->code += second_prefix + second->vector(y).str; + _data->code += ");\n"; + } + } + + void op_ternary_elementwise_function(const Operand &dst_name, TernaryFunction func, const Operand &first_name, const Operand &second_name, const Operand &third_name) override + { + OperandUnpacker operands(_data->tiles, _data->arguments); + const IVectorTile *first = operands.unpack(first_name); + const IVectorTile *second = operands.unpack(second_name); + const IVectorTile *third = operands.unpack(third_name); + const IVectorTile *dst = operands.unpack(dst_name); + + const int32_t dst_h = dst->format().h; + const std::string dt = dst->underlying_source_variables()[0].type.str; + + // Always perform an explicit cast. See similar comments in op_unary_elementwise_function + const std::string first_prefix = "(" + dt + ")"; + const std::string second_prefix = "(" + dt + ")"; + const std::string third_prefix = "(" + dt + ")"; + + // Broadcasting on Y is automatic + for(int32_t y = 0; y < dst_h; ++y) + { + _data->code += dst->vector(y).str; + _data->code += " = "; + + switch(func) + { + case TernaryFunction::Select: + _data->code += "select("; + break; + default: + CKW_ASSERT_MSG(false, "Unexpected TernaryFunction used."); + } + + _data->code += first_prefix + first->vector(y).str; + _data->code += ", "; + _data->code += second_prefix + second->vector(y).str; + _data->code += ", "; + _data->code += third_prefix + third->vector(y).str; + _data->code += ");\n"; + } + } + + void op_if_header(const Operand &o_lhs, BinaryOp op, const Operand &o_rhs) override + { + OperandUnpacker operands(_data->tiles, _data->arguments); + const IVectorTile *lhs = operands.unpack(o_lhs); + const IVectorTile *rhs = operands.unpack(o_rhs); + + assert(is_tile_scalar(lhs)); + assert(is_tile_scalar(rhs)); + + _data->code += "if("; + _data->code += lhs->scalar(0, 0).str; + _data->code += " "; + _data->code += to_string(op); + _data->code += " "; + _data->code += rhs->scalar(0, 0).str; + _data->code += ")\n"; + } + + void op_else_if_header(const Operand &o_lhs, BinaryOp op, const Operand &o_rhs) override + { + _data->code += "else "; + op_if_header(o_lhs, op, o_rhs); + } + + void op_else_header() override + { + _data->code += "else\n"; + } + + void op_for_loop_header(const Operand& var_name, BinaryOp cond_op, const Operand& cond_value_name, const Operand &update_var_name, AssignmentOp update_op, const Operand& update_value_name) override + { + OperandUnpacker operands(_data->tiles, _data->arguments); + const IVectorTile *var = operands.unpack(var_name); + const IVectorTile *cond_value = operands.unpack(cond_value_name); + const IVectorTile *update_var = operands.unpack(update_var_name); + const IVectorTile *update_value = operands.unpack(update_value_name); + + const int32_t dst_w = var->format().w; + const int32_t dst_h = var->format().h; + + // It must be a scalar variable + CKW_UNUSED(dst_w, dst_h); + assert(dst_w == 1); + assert(dst_h == 1); + + _data->code += "for(; "; + _data->code += var->scalar(0, 0).str; + _data->code += " "; + _data->code += to_string(cond_op); + _data->code += " " + cond_value->scalar(0, 0).str + "; "; + _data->code += update_var->scalar(0, 0).str; + _data->code += " "; + _data->code += to_string(update_op); + _data->code += " " + update_value->scalar(0, 0).str + ")"; + _data->code += "\n"; + } + + void op_load_immediate(const TensorOperand &o_tensor, const Operand &o_dst, const Operand &o_x, + const Operand &o_y, const Operand &o_z, const Operand &o_batch_idx, + const Operand &dilation_y) override + { + OperandUnpacker operands(_data->tiles, _data->arguments); + + // Not const as it requires changes to 'load_writer'. + IVectorTile *dst = operands.unpack(o_dst); + IVectorTile *x = operands.unpack(o_x); + IVectorTile *y = operands.unpack(o_y); + IVectorTile *z = operands.unpack(o_z); + IVectorTile *dil_y = operands.unpack(dilation_y); + IVectorTile *b = operands.unpack(o_batch_idx); + + TensorOperandUnpacker tensor_operands(_data->arguments); + IGpuTensorArgument *tensor = tensor_operands.unpack(o_tensor); + auto gpu_sampler = o_tensor.sampler(); + + GpuTensor3dMapper mapper(tensor, gpu_sampler); + + auto load_writer = ClLoadStoreHelperWriterFactory::create(this, mapper, GpuLoadStoreType::Load); + + // Initialize the constant part + load_writer->initialize(dst, x, z, b); + + for(int i = 0; i < dst->format().h; ++i) + { + std::string coord_y = y->scalar(0, 0).str + " + " + std::to_string(i); + if(dil_y->scalar(0, 0).str != "1") + { + coord_y += " * " + dil_y->scalar(0, 0).str; + } + load_writer->write(std::make_pair(i, coord_y)); + } + + load_writer->finalize(); + } + + void op_load_indirect(const TensorOperand &o_tensor, const Operand &o_dst, const Operand &o_x, + const Operand &o_indirect_h, const Operand &o_z, + const Operand &o_batch_idx) override + { + OperandUnpacker operands(_data->tiles, _data->arguments); + + // Not const as it requires changes to 'load_writer'. + IVectorTile *dst = operands.unpack(o_dst); + IVectorTile *x = operands.unpack(o_x); + IVectorTile *y_ind = operands.unpack(o_indirect_h); + IVectorTile *z = operands.unpack(o_z); + IVectorTile *b = operands.unpack(o_batch_idx); + + TensorOperandUnpacker tensor_operands(_data->arguments); + IGpuTensorArgument *tensor = tensor_operands.unpack(o_tensor); + auto gpu_sampler = o_tensor.sampler(); + + GpuTensor3dMapper mapper(tensor, gpu_sampler); + + auto load_writer = ClLoadStoreHelperWriterFactory::create(this, mapper, GpuLoadStoreType::Load); + + // Initialize the constant part + load_writer->initialize(dst, x, z, b); + + for(int i = 0; i < dst->format().h; ++i) + { + load_writer->write(std::make_pair(i, y_ind->scalar(0, i).str)); + } + + load_writer->finalize(); + } + + void op_store_immediate(const TensorOperand &tensor_name, const Operand &src_name, const Operand &x_name, + const Operand &y_name, const Operand &z_name, + const Operand &batch_index_name) override + { + OperandUnpacker operands(_data->tiles, _data->arguments); + + // Not const as it requires changes to 'load_writer'. + IVectorTile *src = operands.unpack(src_name); + IVectorTile *x = operands.unpack(x_name); + IVectorTile *y = operands.unpack(y_name); + IVectorTile *z = operands.unpack(z_name); + IVectorTile *b = operands.unpack(batch_index_name); + + TensorOperandUnpacker tensor_operands(_data->arguments); + IGpuTensorArgument *tensor = tensor_operands.unpack(tensor_name); + auto gpu_sampler = tensor_name.sampler(); + + GpuTensor3dMapper mapper(tensor, gpu_sampler); + + auto store_writer = ClLoadStoreHelperWriterFactory::create(this, mapper, GpuLoadStoreType::Store); + + // Initialize the constant part + store_writer->initialize(src, x, z, b); + + int32_t tile_h = src->format().h; + + for(int m0 = tile_h - 1; m0 >= 0; m0--) + { + store_writer->write(std::make_pair(m0, y->scalar(0, 0).str + " + " + std::to_string(m0))); + } + + store_writer->finalize(); + } + + void op_return() override + { + _data->code += "return;\n"; + } + + void util_get_indirect_buffer(const Operand &o_dst, const TensorOperand &o_tensor, const Operand &o_x, + const Operand &o_y, const Operand &o_x_off, const Operand &o_y_off) override + { + OperandUnpacker operands(_data->tiles, _data->arguments); + const IVectorTile *dst = operands.unpack(o_dst); + const IVectorTile *x = operands.unpack(o_x); + const IVectorTile *y = operands.unpack(o_y); + const IVectorTile *x_off = operands.unpack(o_x_off); + const IVectorTile *y_off = operands.unpack(o_y_off); + + TensorOperandUnpacker tensor_operands(_data->arguments); + IGpuTensorArgument *tensor = tensor_operands.unpack(o_tensor); + + assert(dst->format().w == 1); + assert(x->format().w == 1); + assert(y->format().w == 1); + assert(x_off->format().w == 1); + assert(y_off->format().w == 1); + assert(dst->format().dt == DataType::Int32); + assert(x->format().dt == DataType::Int32); + assert(y->format().dt == DataType::Int32); + assert(x_off->format().dt == DataType::Int32); + assert(y_off->format().dt == DataType::Int32); + + const std::string width = tensor->component(TensorComponentType::Dim1); + const std::string height = tensor->component(TensorComponentType::Dim2); + const std::string wxh = tensor->component(TensorComponentType::Dim1xDim2); + /* + int x_s; + int y_s; + x_s = (xi_0 + x_k); + y_s = (yi_0 + y_k); + mi_0 = x_s + y_s * width + b * widthxheight; + mi_0 = select(-1, mi_0, x_s >= 0); + mi_0 = select(-1, mi_0, y_s >= 0); + mi_0 = select(-1, mi_0, x_s < 128); + mi_0 = select(-1, mi_0, y_s < 128); + */ + compound_statement_begin(); + declare_tile("_x_s", TileInfo(DataType::Int32)); + declare_tile("_y_s", TileInfo(DataType::Int32)); + auto x_s = operands.unpack(Operand("_x_s")); + auto y_s = operands.unpack(Operand("_y_s")); + for(int i = 0; i < dst->format().h; ++i) + { + // x_s = (xi_0 + x_k); + // y_s = (yi_0 + y_k); + _data->code += x_s->scalar(0, i).str; + _data->code += " = ("; + _data->code += x->scalar(0, i).str; + _data->code += " + "; + _data->code += x_off->scalar(0, i).str; + _data->code += ");\n"; + _data->code += y_s->scalar(0, i).str; + _data->code += " = ("; + _data->code += y->scalar(0, i).str; + _data->code += " + "; + _data->code += y_off->scalar(0, i).str; + _data->code += ");\n"; + // mi_0 = x_s + y_s * width; + _data->code += dst->scalar(0, i).str; + _data->code += " = "; + _data->code += x_s->scalar(0, i).str; + _data->code += " + "; + _data->code += y_s->scalar(0, i).str; + _data->code += " * " + width + ";\n"; + // mi_0 = select(wxh, mi_0, x_s >= 0); + _data->code += dst->scalar(0, i).str; + _data->code += " = select(-1, "; + _data->code += dst->scalar(0, i).str; + _data->code += ", "; + _data->code += x_s->scalar(0, i).str; + _data->code += " >= 0);\n"; + // mi_0 = select(wxh, mi_0, x_s < width); + _data->code += dst->scalar(0, i).str; + _data->code += " = select(-1, "; + _data->code += dst->scalar(0, i).str; + _data->code += ", "; + _data->code += x_s->scalar(0, i).str; + _data->code += " < "; + _data->code += width + ");\n"; + // mi_0 = select(wxh, mi_0, y_s >= 0); + _data->code += dst->scalar(0, i).str; + _data->code += " = select(-1, "; + _data->code += dst->scalar(0, i).str; + _data->code += ", "; + _data->code += y_s->scalar(0, i).str; + _data->code += " >= 0);\n"; + // mi_0 = select(wxh, mi_0, y_s < height); + _data->code += dst->scalar(0, i).str; + _data->code += " = select(-1, "; + _data->code += dst->scalar(0, i).str; + _data->code += ", "; + _data->code += y_s->scalar(0, i).str; + _data->code += " < "; + _data->code += height + ");\n"; + } + compound_statement_end(); + } + +private: + GpuKernelWriterDataHolder *_data{ nullptr }; + GpuKernelWriterAttribute *_attr{ nullptr }; +}; + +/** IGpuKernelWriter factory class */ +class GpuKernelWriterFactory final +{ +public: + /** Static method to call the IGpuKernelWriter class accordingly with the Gpu programming language + * + * @param[in] gpu GPU target + * + * @return IGpuKernelWriter + */ + static std::unique_ptr + create(GpuKernelWriterAttribute *attr, GpuKernelWriterDataHolder *x) + { + switch(x->programming_language()) + { + case GpuTargetLanguage::OpenCL: + return std::make_unique(attr, x); + default: + std::cout << "Unsupported Gpu programming language" << std::endl; + assert(false); + return nullptr; + } + } +}; + +inline int32_t +adjust_step(TensorSamplerFormat tensor_format, int32_t step, const TensorInfo *tensor_info_id, int32_t idx) +{ + auto tensor = tensor_info_id->shape; + + int32_t dim[3] = { 0 }; + + switch(tensor_format) + { + case TensorSamplerFormat::C_W_H: + dim[0] = tensor[0]; + dim[1] = tensor[1]; + dim[2] = tensor[2]; + break; + case TensorSamplerFormat::C_WH_1: + dim[0] = tensor[0]; + dim[1] = tensor[1] * tensor[2]; + dim[2] = 1; + break; + default: + std::cout << "Unsupported tensor format" << std::endl; + assert(false); + break; + } + + return std::min(step, dim[idx]); +} + +} // namespace prototype +} // namespace ckw + +#endif // CKW_PROTOTYPE_SRC_PROTOTYPE_H diff --git a/compute_kernel_writer/prototype/src/TensorInfo.cpp b/compute_kernel_writer/prototype/src/TensorInfo.cpp new file mode 100644 index 0000000000000000000000000000000000000000..561c126469efdeaebc12ae74019d7548a35c13de --- /dev/null +++ b/compute_kernel_writer/prototype/src/TensorInfo.cpp @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "ckw/TensorInfo.h" + +namespace ckw +{ +TensorInfo::TensorInfo(DataType dt, const TensorShape &shape, TensorDataLayout dl, int32_t id) + : _shape(shape), _dt(dt), _dl(dl), _id(id) +{ +} + +TensorInfo &TensorInfo::shape(const TensorShape &shape) +{ + _shape = shape; + return *this; +} + +TensorShape TensorInfo::shape() const +{ + return _shape; +} + +TensorInfo &TensorInfo::data_type(DataType dt) +{ + _dt = dt; + return *this; +} + +DataType TensorInfo::data_type() const +{ + return _dt; +} + +TensorInfo &TensorInfo::data_layout(TensorDataLayout dl) +{ + _dl = dl; + return *this; +} + +TensorDataLayout TensorInfo::data_layout() const +{ + return _dl; +} + +TensorInfo &TensorInfo::id(int32_t id) +{ + _id = id; + return *this; +} + +int32_t TensorInfo::id() const +{ + return _id; +} +} // namespace ckw diff --git a/compute_kernel_writer/prototype/src/TensorOperand.cpp b/compute_kernel_writer/prototype/src/TensorOperand.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c6725d3b26c2f5c4d398fa2cfc7e16785a3bd107 --- /dev/null +++ b/compute_kernel_writer/prototype/src/TensorOperand.cpp @@ -0,0 +1,268 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "ckw/TensorOperand.h" +#include "ckw/Error.h" +#include "ckw/Kernel.h" +#include "ckw/TensorInfo.h" +#include "ckw/TileOperand.h" +#include "src/Prototype.h" + +namespace ckw +{ + +namespace +{ + +TensorComponentOperand &get_or_create_component(TensorOperand &tensor, std::unique_ptr &ptr, TensorComponentType component) +{ + if(ptr == nullptr) + { + ptr = std::make_unique(tensor, component); + } + + return *ptr; +} + +} // namespace + +// ================================================================================================= +// TensorOperand +// ================================================================================================= + +TensorOperand::TensorOperand(const std::string &name, const TensorInfo &info, TensorStorageType storage_type) + : OperandBase(name), _info(info), _storage_type(storage_type) +{ +} + +prototype::Operand TensorOperand::create_impl_operand(prototype::IGpuKernelWriter *writer) const +{ + CKW_UNUSED(writer); + return { name() }; +} + +const TensorInfo &TensorOperand::info() const +{ + return _info; +} + +TensorInfo &TensorOperand::info() +{ + return _info; +} + +TensorStorageType TensorOperand::storage_type() const +{ + return _storage_type; +} + +DataType TensorOperand::data_type() const +{ + return _info.data_type(); +} + +bool TensorOperand::is_constant() const +{ + return false; +} + +const TileOperand &TensorOperand::tile() const +{ + return *_tile; +} + +TileOperand &TensorOperand::tile() +{ + return *_tile; +} + +TensorOperand &TensorOperand::tile(TileOperand &tile) +{ + _tile = &tile; + return *this; +} + +const TensorTileSampler &TensorOperand::tile_sampler() const +{ + return _tile_sampler; +} + +TensorTileSampler &TensorOperand::tile_sampler() +{ + return _tile_sampler; +} + +TensorOperand &TensorOperand::tile_sampler(const TensorTileSampler &value) +{ + _tile_sampler = value; + return *this; +} + +TensorComponentOperand &TensorOperand::stride1() +{ + return get_or_create_component(*this, _stride1, TensorComponentType::Stride1); +} + +TensorComponentOperand &TensorOperand::stride2() +{ + return get_or_create_component(*this, _stride2, TensorComponentType::Stride2); +} + +TensorComponentOperand &TensorOperand::stride3() +{ + return get_or_create_component(*this, _stride3, TensorComponentType::Stride3); +} + +TensorComponentOperand &TensorOperand::stride4() +{ + return get_or_create_component(*this, _stride4, TensorComponentType::Stride4); +} + +TensorComponentOperand &TensorOperand::dim0() +{ + return get_or_create_component(*this, _dim0, TensorComponentType::Dim0); +} + +TensorComponentOperand &TensorOperand::dim1() +{ + return get_or_create_component(*this, _dim1, TensorComponentType::Dim1); +} + +TensorComponentOperand &TensorOperand::dim2() +{ + return get_or_create_component(*this, _dim2, TensorComponentType::Dim2); +} + +TensorComponentOperand &TensorOperand::dim3() +{ + return get_or_create_component(*this, _dim3, TensorComponentType::Dim3); +} + +TensorComponentOperand &TensorOperand::dim4() +{ + return get_or_create_component(*this, _dim4, TensorComponentType::Dim4); +} + +TensorComponentOperand &TensorOperand::dim1_dim2() +{ + return get_or_create_component(*this, _dim1_dim2, TensorComponentType::Dim1xDim2); +} + +TensorComponentOperand &TensorOperand::dim1_dim2_dim3() +{ + return get_or_create_component(*this, _dim1_dim2_dim3, TensorComponentType::Dim1xDim2xDim3); +} + +TensorComponentOperand &TensorOperand::offset_first_element_in_bytes() +{ + return get_or_create_component(*this, _offset_first_element_in_bytes, TensorComponentType::OffsetFirstElement); +} + +// ================================================================================================= +// TensorComponentOperand +// ================================================================================================= + +TensorComponentOperand::TensorComponentOperand(TensorOperand &tensor, TensorComponentType component) + : TileOperand(tensor.name(), DataType::Int32), _tensor(tensor), _component(component) +{ +} + +TensorOperand &TensorComponentOperand::tensor() +{ + return _tensor; +} + +const TensorOperand &TensorComponentOperand::tensor() const +{ + return _tensor; +} + +TensorComponentType TensorComponentOperand::component_type() const +{ + return _component; +} + +prototype::Operand TensorComponentOperand::create_impl_operand(prototype::IGpuKernelWriter *writer) const +{ + CKW_UNUSED(writer); + prototype::OperandType type{ prototype::OperandType::Unknown }; + + switch(_component) + { + case TensorComponentType::OffsetFirstElement: + type = prototype::OperandType::TensorDataOffset; + break; + + case TensorComponentType::Stride1: + type = prototype::OperandType::TensorStride1; + break; + + case TensorComponentType::Stride2: + type = prototype::OperandType::TensorStride2; + break; + + case TensorComponentType::Stride3: + type = prototype::OperandType::TensorStride3; + break; + + case TensorComponentType::Stride4: + type = prototype::OperandType::TensorStride4; + break; + + case TensorComponentType::Dim0: + type = prototype::OperandType::TensorDim0; + break; + + case TensorComponentType::Dim1: + type = prototype::OperandType::TensorDim1; + break; + + case TensorComponentType::Dim2: + type = prototype::OperandType::TensorDim2; + break; + + case TensorComponentType::Dim3: + type = prototype::OperandType::TensorDim3; + break; + + case TensorComponentType::Dim4: + type = prototype::OperandType::TensorDim4; + break; + + case TensorComponentType::Dim1xDim2: + type = prototype::OperandType::TensorDim1xDim2; + break; + + case TensorComponentType::Dim1xDim2xDim3: + type = prototype::OperandType::TensorDim1xDim2xDim3; + break; + + default: + CKW_ASSERT(false); + } + + return prototype::Operand(name(), type); +} + +} // namespace ckw diff --git a/compute_kernel_writer/prototype/src/TensorTileSampler.cpp b/compute_kernel_writer/prototype/src/TensorTileSampler.cpp new file mode 100644 index 0000000000000000000000000000000000000000..28e54df3a570a1bec16e23179ef0b6da24b7b6d1 --- /dev/null +++ b/compute_kernel_writer/prototype/src/TensorTileSampler.cpp @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "ckw/TensorTileSampler.h" +#include "ckw/TileOperand.h" +#include "ckw/types/TensorSamplerTypes.h" + +namespace ckw +{ + +TensorTileSampler::TensorTileSampler() +{ +} + +TensorTileSampler::TensorTileSampler( + TileOperand &x, TileOperand &y, TileOperand &z, TileOperand &b, + TensorSamplerFormat format, + TensorSamplerAddressModeX address_mode_x, + TensorSamplerAddressModeY address_mode_y, + TensorSamplerAddressModeZ address_mode_z) + : _x(&x), _y(&y), _z(&z), _b(&b), _height(0), _width(0), _format(format), _address_mode_x(address_mode_x), _address_mode_y(address_mode_y), _address_mode_z(address_mode_z) +{ +} + +TensorTileSampler::TensorTileSampler( + TileOperand &x, TileOperand &y, TileOperand &z, TileOperand &b, + int32_t height, int32_t width, + TensorSamplerFormat format, + TensorSamplerAddressModeX address_mode_x, + TensorSamplerAddressModeY address_mode_y, + TensorSamplerAddressModeZ address_mode_z) + : _x(&x), _y(&y), _z(&z), _b(&b), _height(height), _width(width), _format(format), _address_mode_x(address_mode_x), _address_mode_y(address_mode_y), _address_mode_z(address_mode_z) +{ +} + +const TileOperand &TensorTileSampler::x() const +{ + return *_x; +} + +TensorTileSampler &TensorTileSampler::x(TileOperand &x) +{ + _x = &x; + return *this; +} + +const TileOperand &TensorTileSampler::y() const +{ + return *_y; +} + +TensorTileSampler &TensorTileSampler::y(TileOperand &y) +{ + _y = &y; + return *this; +} + +const TileOperand &TensorTileSampler::z() const +{ + return *_z; +} + +TensorTileSampler &TensorTileSampler::z(TileOperand &z) +{ + _z = &z; + return *this; +} + +const TileOperand &TensorTileSampler::b() const +{ + return *_b; +} + +TensorTileSampler &TensorTileSampler::b(TileOperand &b) +{ + _b = &b; + return *this; +} + +int32_t TensorTileSampler::width() const +{ + return _width; +} + +TensorTileSampler &TensorTileSampler::width(int32_t width) +{ + _width = width; + return *this; +} + +int32_t TensorTileSampler::height() const +{ + return _height; +} + +TensorTileSampler &TensorTileSampler::height(int32_t height) +{ + _height = height; + return *this; +} + +TensorSamplerFormat TensorTileSampler::format() const +{ + return _format; +} + +TensorTileSampler &TensorTileSampler::format(TensorSamplerFormat format) +{ + _format = format; + return *this; +} + +TensorSamplerAddressModeX TensorTileSampler::address_mode_x() const +{ + return _address_mode_x; +} + +TensorTileSampler &TensorTileSampler::address_mode_x(TensorSamplerAddressModeX address_mode_x) +{ + _address_mode_x = address_mode_x; + return *this; +} + +TensorSamplerAddressModeY TensorTileSampler::address_mode_y() const +{ + return _address_mode_y; +} + +TensorTileSampler &TensorTileSampler::address_mode_y(TensorSamplerAddressModeY address_mode_y) +{ + _address_mode_y = address_mode_y; + return *this; +} + +TensorSamplerAddressModeZ TensorTileSampler::address_mode_z() const +{ + return _address_mode_z; +} + +TensorTileSampler &TensorTileSampler::address_mode_z(TensorSamplerAddressModeZ address_mode_z) +{ + _address_mode_z = address_mode_z; + return *this; +} + +} // namespace ckw diff --git a/compute_kernel_writer/prototype/src/TileInfo.cpp b/compute_kernel_writer/prototype/src/TileInfo.cpp new file mode 100644 index 0000000000000000000000000000000000000000..66d8cb1620732f26914cc812c58da5eea4e3e0f9 --- /dev/null +++ b/compute_kernel_writer/prototype/src/TileInfo.cpp @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "ckw/TileInfo.h" + +namespace ckw +{ +TileInfo::TileInfo(DataType dt) + : _dt(dt), _shape({ { 1, 1 } }) +{ +} + +TileInfo::TileInfo(DataType dt, int32_t w) + : _dt(dt), _shape({ { w, 1 } }) +{ +} + +TileInfo::TileInfo(DataType dt, int32_t h, int32_t w) + : _dt(dt), _shape({ { w, h } }) +{ +} + +TileInfo &TileInfo::width(int32_t w) +{ + _shape[kTileWidthIdx] = w; + return *this; +} + +int32_t TileInfo::width() const +{ + return _shape[kTileWidthIdx]; +} + +TileInfo &TileInfo::height(int32_t h) +{ + _shape[kTileHeightIdx] = h; + return *this; +} + +int32_t TileInfo::height() const +{ + return _shape[kTileHeightIdx]; +} + +TileInfo &TileInfo::data_type(DataType dt) +{ + _dt = dt; + return *this; +} + +DataType TileInfo::data_type() const +{ + return _dt; +} +} // namespace ckw diff --git a/compute_kernel_writer/prototype/src/TileOperand.cpp b/compute_kernel_writer/prototype/src/TileOperand.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bf6a15b9df69e13482046f47e8a6724c041755ae --- /dev/null +++ b/compute_kernel_writer/prototype/src/TileOperand.cpp @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "ckw/TileOperand.h" +#include "ckw/Error.h" +#include "src/Prototype.h" + +namespace ckw +{ + +TileOperand::TileOperand(const std::string &name, const TileInfo &info) + : OperandBase(name), + _info(info), + _value{ std::vector{ "0" } }, + _constant(false) +{ +} + +TileOperand::TileOperand(const std::string &name, DataType data_type) + : OperandBase(name), + _info(TileInfo{ data_type }), + _value{ std::vector{ "0" } }, + _constant(false) +{ +} + +TileOperand::TileOperand(const std::string &name, int32_t value) + : OperandBase(name), + _info(TileInfo{ DataType::Int32 }), + _value{ std::vector{ std::to_string(value) } }, + _constant(true) +{ +} + +TileOperand::TileOperand(const std::string &name, float value) + : OperandBase(name), + _info(TileInfo{ DataType::Fp32 }), + _value{ std::vector{ std::to_string(value) } }, + _constant(true) +{ +} + +TileOperand::TileOperand(const std::string &name, const TileContainer &vals, DataType dt) + : OperandBase(name), + _info(TileInfo{ dt, static_cast(vals.size()), static_cast(vals[0].size()) }), + _value(vals), + _constant(true) +{ +} + +prototype::Operand TileOperand::create_impl_operand(prototype::IGpuKernelWriter *writer) const +{ + CKW_UNUSED(writer); + + if(_constant) + { + if(is_scalar()) + { + switch(_info.data_type()) + { + case DataType::Int32: + return prototype::Operand(_value[0][0], prototype::OperandType::ScalarInt32); + + case DataType::Fp32: + return prototype::Operand(_value[0][0], prototype::OperandType::ScalarFp32); + + default: + CKW_ASSERT(false); + } + } + else + { + return prototype::Operand(name()); + } + } + else + { + return prototype::Operand(name(), prototype::OperandType::Tile); + } +} + +const TileInfo &TileOperand::tile_info() const +{ + return _info; +} + +DataType TileOperand::data_type() const +{ + return _info.data_type(); +} + +bool TileOperand::is_constant() const +{ + return _constant; +} + +bool TileOperand::is_scalar() const +{ + return _info.width() == 1 && _info.height() == 1; +} + +std::string TileOperand::scalar_value() const +{ + CKW_ASSERT(is_scalar()); + CKW_ASSERT(is_constant()); + + return _value[0][0]; +} + +const TileContainer &TileOperand::value() const +{ + return _value; +} + +} // namespace ckw diff --git a/compute_kernel_writer/src/Error.cpp b/compute_kernel_writer/src/Error.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c5dae2eb75768ebcc02c09999497b74351c1eea5 --- /dev/null +++ b/compute_kernel_writer/src/Error.cpp @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "ckw/Error.h" + +#include + +namespace ckw +{ +std::string create_error_msg(const std::string &file, const std::string &func, const std::string &line, + const std::string &msg) +{ + std::string err; + err += "[COMPUTE_KERNEL_WRITER][ERROR]:"; + err += " " + file + ":" + line; + err += " " + func; + err += " " + msg; + return err; +} +} // namespace ckw \ No newline at end of file diff --git a/compute_kernel_writer/src/Helpers.cpp b/compute_kernel_writer/src/Helpers.cpp new file mode 100644 index 0000000000000000000000000000000000000000..799f79a18712ab40cc3d14ccdadaad75bef8d302 --- /dev/null +++ b/compute_kernel_writer/src/Helpers.cpp @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "ckw/Error.h" + +#include "src/Helpers.h" + +namespace ckw +{ +std::string dec_to_hex_as_string(int32_t dec) +{ + switch(dec) + { + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + case 8: + case 9: + return std::to_string(dec); + case 10: + return "A"; + case 11: + return "B"; + case 12: + return "C"; + case 13: + return "D"; + case 14: + return "E"; + case 15: + return "F"; + default: + COMPUTE_KERNEL_WRITER_ERROR_ON_MSG("Unsupported decimal number"); + return ""; + } +} +} // namespace ckw diff --git a/compute_kernel_writer/src/Helpers.h b/compute_kernel_writer/src/Helpers.h new file mode 100644 index 0000000000000000000000000000000000000000..16c06d60e7971ecec41c66cb13ffe7ec24f21b21 --- /dev/null +++ b/compute_kernel_writer/src/Helpers.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef COMPUTE_KERNEL_WRITER_SRC_HELPERS_H +#define COMPUTE_KERNEL_WRITER_SRC_HELPERS_H + +#include +#include + +/** Generic helper functions */ +namespace ckw +{ +/** Helper function to convert a decimal number passed as int32_t variable to hexadecimal number as string + * + * @param[in] dec Decimal number. It must be >= 0 and < 16 + * + * @return the OpenCL datatype as a string + */ +std::string dec_to_hex_as_string(int32_t dec); + +/** Helper function to clamp a value between min_val and max_val + * + * @param[in] val Value to clamp + * @param[in] min_val Lower value + * @param[in] max_val Upper value + * + * @return the clamped value + */ +template +T clamp(const T &val, const T &min_val, const T &max_val) +{ + return std::max(min_val, std::min(val, max_val)); +} +} // namespace ckw +#endif /* COMPUTE_KERNEL_WRITER_SRC_HELPERS_H */ diff --git a/compute_kernel_writer/src/ITensor.h b/compute_kernel_writer/src/ITensor.h new file mode 100644 index 0000000000000000000000000000000000000000..4c1c56fd35a8af4f21ad4dbb5900484470ed52f8 --- /dev/null +++ b/compute_kernel_writer/src/ITensor.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_SRC_ITENSOR_H +#define CKW_SRC_ITENSOR_H + +#include "src/ITensorArgument.h" + +namespace ckw +{ + +/** The generic class for all tensor objects in CKW. + * + * Tensors in CKW are always kernel arguments consisting of: + * - Essential information such as name, tensor info, etc. + * - Tensor storage access: allowing load/store operation to perform. + * - Tensor component access: allowing interaction with tensor information such as shape, strides, etc. in the form of tile objects. + */ +class ITensor : public ITensorArgument, public ITensorStorageAccess, public ITensorComponentAccess +{ +}; + +} // namespace ckw + +#endif // CKW_SRC_ITENSOR_H \ No newline at end of file diff --git a/compute_kernel_writer/src/ITensorArgument.h b/compute_kernel_writer/src/ITensorArgument.h new file mode 100644 index 0000000000000000000000000000000000000000..838bd40f85b5d812df07072e4d659f8ee9da7342 --- /dev/null +++ b/compute_kernel_writer/src/ITensorArgument.h @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_SRC_ITENSORARGUMENT_H +#define CKW_SRC_ITENSORARGUMENT_H + +#include "ckw/TensorInfo.h" +#include "ckw/types/TensorComponentType.h" +#include "ckw/types/TensorStorageType.h" +#include "src/ITile.h" + +#include +#include + +namespace ckw +{ + +class ITensorComponent; + +/** Tensor storage variable */ +struct TensorStorageVariable +{ + std::string val{ "" }; /** Tensor storage as a string */ + TensorStorageType type{ TensorStorageType::Unknown }; /** Tensor storage type */ +}; + +/** Tensor argument base class. + * A tensor is a multidimensional array used to store data. To access an element (or multiple elements) from a tensor, + * the following information are required: + * -# The data memory object. For example, the pointer to the array + * -# The tensor components, such as the size of each tensor dimension, or the number of elements in bytes contained in each dimension (also known as the "stride") + */ +class ITensorArgument +{ +public: + virtual ~ITensorArgument() = default; + /** Method to get the name of the tensor argument. + * + * @return the name of the tensor argument + */ + std::string name() const + { + return _basename; + } + + /** Method to get the tensor info + * + * @return the @ref TensorInfo + */ + TensorInfo &info() + { + return _info; + } + + /** Method to get the tensor info + * + * @return the @ref TensorInfo + */ + const TensorInfo &info() const + { + return _info; + } + +protected: + TensorInfo _info{}; // Tensor info + std::string _basename{ "" }; // Tensor name +}; + +/** Tensor component argument base class */ +class ITensorComponentAccess +{ +public: + virtual ~ITensorComponentAccess() = default; + /** Method to get the tensor component variable as a tile. + * + * @param[in] x The tensor component to query + * + * @return the tensor component variable as a @ref ITile. + */ + virtual ITile &component(TensorComponentType x) = 0; + /** Method to get all tensor components needed to access the data in the tensor + * + * The tensor components returned by this method must be all passed as kernel argument + * + * @return a vector containing all the tensor components as pointers to @ref ITensorComponent objects. + */ + virtual std::vector components() const = 0; +}; + +/** Tensor storage argument base class */ +class ITensorStorageAccess +{ +public: + virtual ~ITensorStorageAccess() = default; + /** Method to get the tensor storage as a string + * + * @param[in] x The tensor storage to query + * + * @return the tensor storage as a @ref TensorStorageVariable + */ + virtual TensorStorageVariable &storage(TensorStorageType x) = 0; + /** Method to get all tensor storages needed to access the data in the tensor + * + * The tensor storages returned by this method must be all passed as kernel argument + * + * @return a vector containing all the tensor storages as @ref TensorStorageVariable objects + */ + virtual std::vector storages() const = 0; +}; + +} // namespace ckw + +#endif // CKW_SRC_ITENSORARGUMENT_H diff --git a/compute_kernel_writer/src/ITensorComponent.h b/compute_kernel_writer/src/ITensorComponent.h new file mode 100644 index 0000000000000000000000000000000000000000..e2775b62b0dc494f9581eb3e43e1ab3e6157fd26 --- /dev/null +++ b/compute_kernel_writer/src/ITensorComponent.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_SRC_ITENSORCOMPONENT_H +#define CKW_SRC_ITENSORCOMPONENT_H + +#include "ckw/types/TensorComponentType.h" +#include "src/ITile.h" + +namespace ckw +{ + +/** A tensor component provides access to tensor information such as shape, strides, etc. in the form of @ref ITile objects. */ +class ITensorComponent +{ +public: + /** Destructor. */ + virtual ~ITensorComponent() = default; + + /** Get the tile variable for the component. */ + virtual ITile &tile() = 0; + + /** Get the const tile variable for the component. */ + virtual const ITile &tile() const = 0; + + /** Get the component type. */ + virtual TensorComponentType component_type() const = 0; +}; + +} // namespace ckw + +#endif // CKW_SRC_ITENSORCOMPONENT_H diff --git a/compute_kernel_writer/src/ITile.h b/compute_kernel_writer/src/ITile.h new file mode 100644 index 0000000000000000000000000000000000000000..a54fd9b7943bed9acee08b597ceff62c8761db4d --- /dev/null +++ b/compute_kernel_writer/src/ITile.h @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef COMPUTE_KERNEL_WRITER_SRC_ITILE +#define COMPUTE_KERNEL_WRITER_SRC_ITILE + +#include "ckw/TileInfo.h" + +#include +#include + +namespace ckw +{ +/** Compute Kernel Writer tile container. It contains the variables stored in the tile as a string */ +using TileContainer = std::vector>; + +/** Tile descriptor which reports the underlying datatype and vector length */ +struct TileVariableDescriptor +{ + DataType dt{ DataType::Unknown }; /** Data type */ + int32_t len{ 1 }; /** Number of elements in a single variable. For example, 1 for scalar */ +}; + +/** Tile variable */ +struct TileVariable +{ + std::string str{ "" }; /** Tile variable as a string */ + TileVariableDescriptor desc{}; /** Tile value descriptor which reports the datatype and vector length */ +}; + +/** Tile base class. + * A Tile is a collection of variables (either program variables or constants) used to express a 2D data. + */ +class ITile +{ +public: + virtual ~ITile() = default; + + /** Method to get all TileVariable objects + * + * @return a vector containing all @ref TileVariable objects + */ + virtual std::vector all() const = 0; + + /** Method to get the name of the tile. + * + * @return the name of the tile + */ + virtual const std::string &name() const = 0; + + /** Method to get the tile info + * + * @return the @ref TileInfo + */ + virtual const TileInfo &info() const = 0; + + /** Method to know whether the tile is assignable or not. + * For example, a constant tile is not assignable. + * + * @return true if the tile is assignable + */ + virtual bool is_assignable() const = 0; +}; + +/** Interface to provide support for scalar access for a Tile. + */ +class IScalarAccess +{ +public: + virtual ~IScalarAccess() = default; + + /** Method to get the scalar variable from a tile as a string + * @param[in] row Tile row. If out-of-bound, the row is clamped to the nearest valid edge + * @param[in] col Tile column. If out-of-bound, the column is clamped to the nearest valid edge + * + * @return the @ref TileVariable + */ + virtual TileVariable scalar(int32_t row, int32_t col) const = 0; +}; + +/** Interface to provide support for vector access for a tile. + */ +class IVectorAccess +{ +public: + virtual ~IVectorAccess() = default; + + /** Method to get the vector variable from a tile. + * The user can query the list of supported vector lengths through the supported_vector_lengths() method. + * + * @param[in] row Tile row. If out-of-bound, the row is clamped to the nearest valid edge + * + * @return the vector variable as a @ref TileVariable + */ + virtual TileVariable vector(int32_t row) const = 0; + + /** Method to get a sub-vector variable. The length of the sub-vector must be supported by the derived IVectorAccess class + * + * @param[in] row Tile row. If out-of-bound, the row is clamped to the nearest valid edge + * @param[in] col_start Tile starting column to get the sub-vector. If out-of-bound, the derived IVectorAccess class may throw an assert. + * @param[in] width The width of the sub-vector. The width must be supported by the derived IVectorAccess class and the last element must be in-bound. + * + * @return the vector variable as a @ref TileVariable + */ + virtual TileVariable vector(int32_t row, int32_t col_start, int32_t width) const = 0; + + /** Method to get the supported vector length. + * + * @return a vector containing the supported vector lengths + */ + virtual std::vector supported_vector_lengths() const = 0; +}; +} // namespace ckw + +#endif /* COMPUTE_KERNEL_WRITER_SRC_ITILE */ diff --git a/compute_kernel_writer/src/Kernel.cpp b/compute_kernel_writer/src/Kernel.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5eea1aa5486c9357dbca282383eec7c7d01783e4 --- /dev/null +++ b/compute_kernel_writer/src/Kernel.cpp @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "ckw/Kernel.h" +#include "ckw/types/TargetLanguage.h" + +namespace ckw +{ + +Kernel::~Kernel() = default; + +Kernel::Kernel(TargetLanguage language, const std::string &source_code) + : _language(language), _source_code(source_code) +{ +} + +TargetLanguage Kernel::target_language() const +{ + return _language; +} + +const std::string &Kernel::source_code() const +{ + return _source_code; +} + +} // namespace ckw diff --git a/compute_kernel_writer/src/KernelWriter.cpp b/compute_kernel_writer/src/KernelWriter.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ce34a1c2d66803f9887d8c90331c6da4386a3211 --- /dev/null +++ b/compute_kernel_writer/src/KernelWriter.cpp @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "ckw/KernelWriter.h" +#include "ckw/Error.h" +#include "ckw/TileOperand.h" +#include "ckw/types/TargetArchitecture.h" +#include "ckw/types/TargetLanguage.h" +#include "src/cl/CLKernelWriter.h" +#include "src/cl/CLTensorArgument.h" +#include "src/cl/CLTile.h" + +namespace ckw +{ + +KernelWriter::~KernelWriter() = default; + +std::unique_ptr KernelWriter::create_instance(TargetArchitecture architecture, TargetLanguage language) +{ + CKW_UNUSED(architecture); + switch(language) + { + case TargetLanguage::OpenCL: + // Currently this is the oldest and the only supported GPU architecture. + CKW_ASSERT(architecture == TargetArchitecture::GpuArmMaliValhall); + return std::make_unique(); + + default: + CKW_THROW_MSG("Language not supported!"); + } +} + +int32_t KernelWriter::id_space() const +{ + return _id_space; +} + +std::string KernelWriter::generate_full_name(const std::string &name) const +{ + return "G" + std::to_string(id_space()) + "__" + name; +} + +TileOperand KernelWriter::create_tile_operand(ITile &tile) +{ + return TileOperand(tile); +} + +ITile &KernelWriter::get_tile(const TileOperand &operand) +{ + return operand._tile; +} + +TensorOperand KernelWriter::create_tensor_operand(ITensor &tensor) +{ + return TensorOperand(tensor); +} + +ITensor &KernelWriter::get_tensor(const TensorOperand &operand) +{ + return operand._tensor; +} + +} // namespace ckw diff --git a/compute_kernel_writer/src/TensorInfo.cpp b/compute_kernel_writer/src/TensorInfo.cpp new file mode 100644 index 0000000000000000000000000000000000000000..561c126469efdeaebc12ae74019d7548a35c13de --- /dev/null +++ b/compute_kernel_writer/src/TensorInfo.cpp @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "ckw/TensorInfo.h" + +namespace ckw +{ +TensorInfo::TensorInfo(DataType dt, const TensorShape &shape, TensorDataLayout dl, int32_t id) + : _shape(shape), _dt(dt), _dl(dl), _id(id) +{ +} + +TensorInfo &TensorInfo::shape(const TensorShape &shape) +{ + _shape = shape; + return *this; +} + +TensorShape TensorInfo::shape() const +{ + return _shape; +} + +TensorInfo &TensorInfo::data_type(DataType dt) +{ + _dt = dt; + return *this; +} + +DataType TensorInfo::data_type() const +{ + return _dt; +} + +TensorInfo &TensorInfo::data_layout(TensorDataLayout dl) +{ + _dl = dl; + return *this; +} + +TensorDataLayout TensorInfo::data_layout() const +{ + return _dl; +} + +TensorInfo &TensorInfo::id(int32_t id) +{ + _id = id; + return *this; +} + +int32_t TensorInfo::id() const +{ + return _id; +} +} // namespace ckw diff --git a/compute_kernel_writer/src/TensorOperand.cpp b/compute_kernel_writer/src/TensorOperand.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5ad24c62768af8ad1444737c7548d412a63596ed --- /dev/null +++ b/compute_kernel_writer/src/TensorOperand.cpp @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "ckw/TensorOperand.h" +#include "src/ITensor.h" + +namespace ckw +{ + +TensorOperand::TensorOperand(ITensor &tensor) + : _tensor(tensor) +{ +} + +const TensorInfo &TensorOperand::info() const +{ + return _tensor.info(); +} + +TileOperand TensorOperand::stride0() +{ + return TileOperand(_tensor.component(TensorComponentType::Stride0)); +} + +TileOperand TensorOperand::stride1() +{ + return TileOperand(_tensor.component(TensorComponentType::Stride1)); +} + +TileOperand TensorOperand::stride2() +{ + return TileOperand(_tensor.component(TensorComponentType::Stride2)); +} + +TileOperand TensorOperand::stride3() +{ + return TileOperand(_tensor.component(TensorComponentType::Stride3)); +} + +TileOperand TensorOperand::stride4() +{ + return TileOperand(_tensor.component(TensorComponentType::Stride4)); +} + +TileOperand TensorOperand::dim0() +{ + return TileOperand(_tensor.component(TensorComponentType::Dim0)); +} + +TileOperand TensorOperand::dim1() +{ + return TileOperand(_tensor.component(TensorComponentType::Dim1)); +} + +TileOperand TensorOperand::dim2() +{ + return TileOperand(_tensor.component(TensorComponentType::Dim2)); +} + +TileOperand TensorOperand::dim3() +{ + return TileOperand(_tensor.component(TensorComponentType::Dim3)); +} + +TileOperand TensorOperand::dim4() +{ + return TileOperand(_tensor.component(TensorComponentType::Dim4)); +} + +TileOperand TensorOperand::dim1_dim2() +{ + return TileOperand(_tensor.component(TensorComponentType::Dim1xDim2)); +} + +TileOperand TensorOperand::dim1_dim2_dim3() +{ + return TileOperand(_tensor.component(TensorComponentType::Dim1xDim2xDim3)); +} + +TileOperand TensorOperand::dim2_dim3() +{ + return TileOperand(_tensor.component(TensorComponentType::Dim2xDim3)); +} + +TileOperand TensorOperand::offset_first_element_in_bytes() +{ + return TileOperand(_tensor.component(TensorComponentType::OffsetFirstElement)); +} + +} // namespace ckw \ No newline at end of file diff --git a/compute_kernel_writer/src/TensorSampler.cpp b/compute_kernel_writer/src/TensorSampler.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2ee8df4bca33b860c68e9252737c5d29af18ab26 --- /dev/null +++ b/compute_kernel_writer/src/TensorSampler.cpp @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "ckw/TensorSampler.h" + +namespace ckw +{ + +TensorSampler::TensorSampler(TensorStorageType storage, + TensorSamplerFormat format, + TensorSamplerAddressModeX address_mode_x, + TensorSamplerAddressModeY address_mode_y, + TensorSamplerAddressModeZ address_mode_z) + : _storage(storage), _format(format), _address_mode_x(address_mode_x), _address_mode_y(address_mode_y), _address_mode_z(address_mode_z) +{ +} + +TensorStorageType TensorSampler::storage() const +{ + return _storage; +} + +TensorSampler &TensorSampler::storage(TensorStorageType storage) +{ + _storage = storage; + return *this; +} + +/** Get the format of the tensor. */ +TensorSamplerFormat TensorSampler::format() const +{ + return _format; +} + +/** Set the format of the tensor. */ +TensorSampler &TensorSampler::format(TensorSamplerFormat format) +{ + _format = format; + return *this; +} + +/** Get the address mode of the x dimension. */ +TensorSamplerAddressModeX TensorSampler::address_mode_x() const +{ + return _address_mode_x; +} + +/** Set the address mode of the x-dimension. */ +TensorSampler &TensorSampler::address_mode_x(TensorSamplerAddressModeX address_mode_x) +{ + _address_mode_x = address_mode_x; + return *this; +} + +/** Get the address mode of the y dimension. */ +TensorSamplerAddressModeY TensorSampler::address_mode_y() const +{ + return _address_mode_y; +} + +/** Set the address mode of the y dimension. */ +TensorSampler &TensorSampler::address_mode_y(TensorSamplerAddressModeY address_mode_y) +{ + _address_mode_y = address_mode_y; + return *this; +} + +/** Get the address mode of the z dimension. */ +TensorSamplerAddressModeZ TensorSampler::address_mode_z() const +{ + return _address_mode_z; +} + +/** Set the address mode of the z dimension. */ +TensorSampler &TensorSampler::address_mode_z(TensorSamplerAddressModeZ address_mode_z) +{ + _address_mode_z = address_mode_z; + return *this; +} + +} // namespace ckw diff --git a/compute_kernel_writer/src/TensorUtils.cpp b/compute_kernel_writer/src/TensorUtils.cpp new file mode 100644 index 0000000000000000000000000000000000000000..24836092d4b65ff7f2afd94e5695fe4a6026000b --- /dev/null +++ b/compute_kernel_writer/src/TensorUtils.cpp @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "src/TensorUtils.h" +#include "ckw/Error.h" +#include "ckw/TensorInfo.h" +#include "ckw/types/TensorComponentType.h" + +namespace ckw +{ +TensorComponentType get_tensor_dimension(TensorDataLayout layout, TensorDataLayoutComponent component) +{ + switch(layout) + { + case TensorDataLayout::Nhwc: + switch(component) + { + case TensorDataLayoutComponent::C: + return TensorComponentType::Dim0; + case TensorDataLayoutComponent::W: + return TensorComponentType::Dim1; + case TensorDataLayoutComponent::H: + return TensorComponentType::Dim2; + case TensorDataLayoutComponent::N: + return TensorComponentType::Dim3; + default: + COMPUTE_KERNEL_WRITER_ERROR_ON_MSG("Unsupported tensor component for NHWC"); + return TensorComponentType::Unknown; + } + case TensorDataLayout::Ndhwc: + switch(component) + { + case TensorDataLayoutComponent::C: + return TensorComponentType::Dim0; + case TensorDataLayoutComponent::W: + return TensorComponentType::Dim1; + case TensorDataLayoutComponent::H: + return TensorComponentType::Dim2; + case TensorDataLayoutComponent::D: + return TensorComponentType::Dim3; + case TensorDataLayoutComponent::N: + return TensorComponentType::Dim4; + default: + COMPUTE_KERNEL_WRITER_ERROR_ON_MSG("Unsupported tensor component for NDHWC"); + return TensorComponentType::Unknown; + } + default: + COMPUTE_KERNEL_WRITER_ERROR_ON_MSG("Unsupported tensor data layout"); + return TensorComponentType::Unknown; + } +} + +TensorComponentType get_tensor_stride(TensorDataLayout layout, TensorDataLayoutComponent component) +{ + switch(layout) + { + case TensorDataLayout::Nhwc: + switch(component) + { + case TensorDataLayoutComponent::C: + return TensorComponentType::Stride0; + case TensorDataLayoutComponent::W: + return TensorComponentType::Stride1; + case TensorDataLayoutComponent::H: + return TensorComponentType::Stride2; + case TensorDataLayoutComponent::N: + return TensorComponentType::Stride3; + default: + COMPUTE_KERNEL_WRITER_ERROR_ON_MSG("Unsupported tensor component for NHWC"); + return TensorComponentType::Unknown; + } + case TensorDataLayout::Ndhwc: + switch(component) + { + case TensorDataLayoutComponent::C: + return TensorComponentType::Stride0; + case TensorDataLayoutComponent::W: + return TensorComponentType::Stride1; + case TensorDataLayoutComponent::H: + return TensorComponentType::Stride2; + case TensorDataLayoutComponent::D: + return TensorComponentType::Stride3; + case TensorDataLayoutComponent::N: + return TensorComponentType::Stride4; + default: + COMPUTE_KERNEL_WRITER_ERROR_ON_MSG("Unsupported tensor component for NDHWC"); + return TensorComponentType::Unknown; + } + default: + COMPUTE_KERNEL_WRITER_ERROR_ON_MSG("Unsupported tensor data layout"); + return TensorComponentType::Unknown; + } +} +} // namespace ckw diff --git a/compute_kernel_writer/src/TensorUtils.h b/compute_kernel_writer/src/TensorUtils.h new file mode 100644 index 0000000000000000000000000000000000000000..bb0af5c0b93325670cbce27b8ade79892bb510fd --- /dev/null +++ b/compute_kernel_writer/src/TensorUtils.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_SRC_TENSORUTILS_H +#define CKW_SRC_TENSORUTILS_H + +#include + +/** Tensor specific utility functions */ +namespace ckw +{ +// Forward declarations +enum class TensorDataLayout; +enum class TensorDataLayoutComponent; +enum class TensorComponentType : uint32_t; + +/** Get tensor dimension from a given data layout and data layout component + * + * @param[in] layout Layout of the tensor + * @param[in] component Data layout component + * + * @return the @ref TensorComponent + */ +TensorComponentType get_tensor_dimension(TensorDataLayout layout, TensorDataLayoutComponent component); + +/** Get tensor stride from a given data layout and data layout component + * + * @param[in] layout Layout of the tensor + * @param[in] component Data layout component + * + * @return the @ref TensorComponent + */ +TensorComponentType get_tensor_stride(TensorDataLayout layout, TensorDataLayoutComponent component); +} // namespace ckw + +#endif // CKW_SRC_TENSORUTILS_H diff --git a/compute_kernel_writer/src/TileInfo.cpp b/compute_kernel_writer/src/TileInfo.cpp new file mode 100644 index 0000000000000000000000000000000000000000..66d8cb1620732f26914cc812c58da5eea4e3e0f9 --- /dev/null +++ b/compute_kernel_writer/src/TileInfo.cpp @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "ckw/TileInfo.h" + +namespace ckw +{ +TileInfo::TileInfo(DataType dt) + : _dt(dt), _shape({ { 1, 1 } }) +{ +} + +TileInfo::TileInfo(DataType dt, int32_t w) + : _dt(dt), _shape({ { w, 1 } }) +{ +} + +TileInfo::TileInfo(DataType dt, int32_t h, int32_t w) + : _dt(dt), _shape({ { w, h } }) +{ +} + +TileInfo &TileInfo::width(int32_t w) +{ + _shape[kTileWidthIdx] = w; + return *this; +} + +int32_t TileInfo::width() const +{ + return _shape[kTileWidthIdx]; +} + +TileInfo &TileInfo::height(int32_t h) +{ + _shape[kTileHeightIdx] = h; + return *this; +} + +int32_t TileInfo::height() const +{ + return _shape[kTileHeightIdx]; +} + +TileInfo &TileInfo::data_type(DataType dt) +{ + _dt = dt; + return *this; +} + +DataType TileInfo::data_type() const +{ + return _dt; +} +} // namespace ckw diff --git a/compute_kernel_writer/src/TileOperand.cpp b/compute_kernel_writer/src/TileOperand.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7d180feec826bf47859455de5150bfed2d43d22c --- /dev/null +++ b/compute_kernel_writer/src/TileOperand.cpp @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "ckw/TileOperand.h" + +namespace ckw +{ + +TileOperand::TileOperand(ITile &tile) + : _tile(tile) +{ +} + +} // namespace ckw diff --git a/compute_kernel_writer/src/cl/CLHelpers.cpp b/compute_kernel_writer/src/cl/CLHelpers.cpp new file mode 100644 index 0000000000000000000000000000000000000000..08108e383fe87997aa3bd62b8483289137d547a2 --- /dev/null +++ b/compute_kernel_writer/src/cl/CLHelpers.cpp @@ -0,0 +1,223 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "src/cl/CLHelpers.h" +#include "ckw/Error.h" +#include "ckw/types/DataType.h" +#include "ckw/types/TensorStorageType.h" + +namespace ckw +{ +bool cl_validate_vector_length(int32_t len) +{ + bool valid_vector_length = true; + if(len < 1 || len > 16 || (len > 4 && len < 8) || (len > 8 && len < 16)) + { + valid_vector_length = false; + } + return valid_vector_length; +} + +std::string cl_get_variable_datatype_as_string(DataType dt, int32_t len) +{ + if(cl_validate_vector_length(len) == false) + { + CKW_THROW_MSG("Unsupported vector length"); + return ""; + } + + std::string res; + switch(dt) + { + case DataType::Fp32: + res += "float"; + break; + case DataType::Fp16: + res += "half"; + break; + case DataType::Int8: + res += "char"; + break; + case DataType::Uint8: + res += "uchar"; + break; + case DataType::Uint16: + res += "ushort"; + break; + case DataType::Int16: + res += "short"; + break; + case DataType::Uint32: + res += "uint"; + break; + case DataType::Int32: + res += "int"; + break; + case DataType::Bool: + res += "bool"; + break; + default: + CKW_THROW_MSG("Unsupported datatype"); + return ""; + } + + if(len > 1) + { + res += std::to_string(len); + } + + return res; +} + +int32_t cl_round_up_to_nearest_valid_vector_width(int32_t width) +{ + switch(width) + { + case 1: + return 1; + case 2: + return 2; + case 3: + return 3; + case 4: + return 4; + case 5: + case 6: + case 7: + case 8: + return 8; + case 9: + case 10: + case 11: + case 12: + case 13: + case 14: + case 15: + case 16: + return 16; + default: + CKW_THROW_MSG("Unsupported width to convert to OpenCL vector"); + return 0; + } +} + +std::string cl_get_variable_storagetype_as_string(TensorStorageType storage) +{ + std::string res; + switch(storage) + { + case TensorStorageType::BufferUint8Ptr: + res += "__global uchar*"; + break; + case TensorStorageType::Texture2dReadOnly: + res += "__read_only image2d_t"; + break; + case TensorStorageType::Texture2dWriteOnly: + res += "__write_only image2d_t"; + break; + default: + CKW_THROW_MSG("Unsupported storage type"); + } + + return res; +} + +std::string cl_data_type_rounded_up_to_valid_vector_width(DataType dt, int32_t width) +{ + std::string data_type; + const int32_t w = cl_round_up_to_nearest_valid_vector_width(width); + data_type += cl_get_variable_datatype_as_string(dt, 1); + if(w != 1) + { + data_type += std::to_string(w); + } + return data_type; +} + +std::vector cl_decompose_vector_width(int32_t vector_width) +{ + std::vector x; + + switch(vector_width) + { + case 0: + break; + case 1: + case 2: + case 3: + case 4: + case 8: + case 16: + x.push_back(vector_width); + break; + case 5: + x.push_back(4); + x.push_back(1); + break; + case 6: + x.push_back(4); + x.push_back(2); + break; + case 7: + x.push_back(4); + x.push_back(3); + break; + case 9: + x.push_back(8); + x.push_back(1); + break; + case 10: + x.push_back(8); + x.push_back(2); + break; + case 11: + x.push_back(8); + x.push_back(3); + break; + case 12: + x.push_back(8); + x.push_back(4); + break; + case 13: + x.push_back(8); + x.push_back(4); + x.push_back(1); + break; + case 14: + x.push_back(8); + x.push_back(4); + x.push_back(2); + break; + case 15: + x.push_back(8); + x.push_back(4); + x.push_back(3); + break; + + default: + CKW_THROW_MSG("Vector width is too large"); + } + return x; +} + +} // namespace ckw diff --git a/compute_kernel_writer/src/cl/CLHelpers.h b/compute_kernel_writer/src/cl/CLHelpers.h new file mode 100644 index 0000000000000000000000000000000000000000..669424088effc97fb618e0cfdb52d7828f6b2675 --- /dev/null +++ b/compute_kernel_writer/src/cl/CLHelpers.h @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef CKW_SRC_CL_CLHELPERS_H +#define CKW_SRC_CL_CLHELPERS_H + +#include +#include +#include + +/** OpenCL specific helper functions */ +namespace ckw +{ +// Forward declarations +enum class DataType; +enum class TensorStorageType : uint32_t; + +/** Helper function to validate the vector length of OpenCL vector data types + * + * @param[in] len Vector length + * + * @return true if the vector lenght is valid. It returns false, otherwise. + */ +bool cl_validate_vector_length(int32_t len); + +/** Helper function to return the OpenCL datatype as a string from a @ref DataType and vector length as int32_t variable + * + * @param[in] dt Datatype + * @param[in] len Vector length + * + * @return the OpenCL datatype as a string + */ +std::string cl_get_variable_datatype_as_string(DataType dt, int32_t len); + +/** Helper function to return the OpenCL vector size that accommodate the the desired width + * + * @param[in] width The desired width + * + * @return the OpenCL vector size +*/ +int32_t cl_round_up_to_nearest_valid_vector_width(int32_t width); + +/** Helper function to return the OpenCL storage type as a string from a @ref TensorStorage + * + * @param[in] storage Storage type + * + * @return the OpenCL storage type as a string + */ +std::string cl_get_variable_storagetype_as_string(TensorStorageType storage); + +/** Helper function to decompose a vector width into a summation of valid OpenCL vector widths. + * + * @param[in] vector_width Vector width to be decomposed + * + * @return a vector of OpenCL vector widths + */ +std::vector cl_decompose_vector_width(int32_t vector_width); + +/** Helper function to get OpenCL data type from the data type enum and width + * It'll round up the given vector width to the nearest valid OpenCL vector width. + * + * @param[in] dt data type enum + * @param[in] width vector width + * + * @return a string representation of the data type + */ +std::string cl_data_type_rounded_up_to_valid_vector_width(DataType dt, int32_t width); +} // namespace ckw + +#endif /* CKW_SRC_CL_CLHELPERS_H */ diff --git a/compute_kernel_writer/src/cl/CLKernelWriter.cpp b/compute_kernel_writer/src/cl/CLKernelWriter.cpp new file mode 100644 index 0000000000000000000000000000000000000000..93630769019962af19fef06073d509ffd251827d --- /dev/null +++ b/compute_kernel_writer/src/cl/CLKernelWriter.cpp @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "src/cl/CLKernelWriter.h" +#include "ckw/Error.h" +#include "ckw/Kernel.h" +#include "ckw/TileOperand.h" +#include "src/cl/CLHelpers.h" +#include "src/cl/CLTensorArgument.h" +#include "src/cl/CLTile.h" +#include + +namespace ckw +{ + +CLKernelWriter::CLKernelWriter() = default; +CLKernelWriter::~CLKernelWriter() = default; + +std::unique_ptr CLKernelWriter::emit_kernel(const std::string &name) +{ + CKW_UNUSED(name); + CKW_THROW_MSG("Not implemented!"); +} + +void CLKernelWriter::comment(const std::string &text) +{ +#ifdef COMPUTE_KERNEL_WRITER_DEBUG_ENABLED + + CKW_ASSERT(text.find("\n") == text.npos); + CKW_ASSERT(text.find("\r") == text.npos); + + append_code("// ", text, "\n"); + +#else // COMPUTE_KERNEL_WRITER_DEBUG_ENABLED + + CKW_UNUSED(text); + +#endif // COMPUTE_KERNEL_WRITER_DEBUG_ENABLED +} + +const std::string &CLKernelWriter::body_source_code() const +{ + return _body_source_code; +} + +TensorOperand CLKernelWriter::declare_tensor_argument(const std::string &name, const TensorInfo &info) +{ + const auto fullname = generate_full_name(name); + + auto tensor = std::make_unique(fullname, info, false /* return_dims_by_value */); + const auto operand = create_tensor_operand(*tensor); + + _tensors.insert(std::move(tensor)); + + return operand; +} + +TileOperand CLKernelWriter::declare_tile(const std::string &name, const TileInfo &tile_info) +{ + const std::string fullname = generate_full_name(name); + + const int32_t height = tile_info.height(); + const int32_t width = tile_info.width(); + const DataType data_type = tile_info.data_type(); + + for(int32_t row = 0; row < height; ++row) + { + const std::string cl_type = cl_get_variable_datatype_as_string(data_type, width); + append_code(cl_type, " ", fullname, std::to_string(row), ";\n"); + } + + auto tile = std::make_unique(name, tile_info); + const auto operand = create_tile_operand(*tile); + + _tiles.insert(std::move(tile)); + + return operand; +} + +void CLKernelWriter::op_write_raw_code(const std::string &raw_code) +{ + append_code(raw_code); +} + +} // namespace ckw diff --git a/compute_kernel_writer/src/cl/CLKernelWriter.h b/compute_kernel_writer/src/cl/CLKernelWriter.h new file mode 100644 index 0000000000000000000000000000000000000000..5df148da7b48aa70f5371a5b95aea7b73eb1d2bd --- /dev/null +++ b/compute_kernel_writer/src/cl/CLKernelWriter.h @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_SRC_CL_CLKERNELWRITER_H +#define CKW_SRC_CL_CLKERNELWRITER_H + +#include "ckw/KernelWriter.h" + +#include +#include +#include + +namespace ckw +{ + +class CLTile; +class CLTensorArgument; + +/** OpenCL kernel writer. */ +class CLKernelWriter : public KernelWriter +{ +public: + // ============================================================================================= + // Construtors and destructor + // ============================================================================================= + + /** Initialize a new instance of @ref CLKernelWriter class. */ + CLKernelWriter(); + + /** Destructor */ + ~CLKernelWriter(); + + // ============================================================================================= + // Misc + // ============================================================================================= + + /** Similar to @ref KernelWriter::comment() */ + void comment(const std::string &text) override; + + /** Similar to @ref KernelWriter::op_write_raw_code() */ + void op_write_raw_code(const std::string &raw_code) override; + + // ============================================================================================= + // Code generation + // ============================================================================================= + + std::unique_ptr emit_kernel(const std::string &name) override; + + // ============================================================================================= + // Tensor and tile declaration + // ============================================================================================= + + TensorOperand declare_tensor_argument(const std::string &name, const TensorInfo &info) override; + + /** Declare a tile given name and tile information + * + * Similar to @ref KernelWriter::declare_tile() + */ + TileOperand declare_tile(const std::string &name, const TileInfo &tile_info) override; + +protected: + /** Append the specified code to the kernel body source code. */ + template + void append_code(T &&code, TArgs &&...args) + { + append_code(std::forward(code)); + append_code(std::forward(args)...); + } + + /** Append the specified code to the kernel body source code. */ + template + void append_code(T &&code) + { + _body_source_code += std::forward(code); + } + + /** Get the current kernel body source code. */ + const std::string &body_source_code() const; + +private: + /** This string contains the kernel body source code, not the full CL source code. + * The full source code will only be generated when the user calls @ref KernelWriter::emit_kernel. + * + * In order to add code to this, use @ref CLKernelWriter::append_code. + * Do not attempt to concatenate and alter this string directly. + */ + std::string _body_source_code{}; + + std::set> _tensors{}; + std::set> _tiles{}; +}; + +} // namespace ckw + +#endif // CKW_SRC_CL_CLKERNELWRITER_H diff --git a/compute_kernel_writer/src/cl/CLTensorArgument.cpp b/compute_kernel_writer/src/cl/CLTensorArgument.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7d4dc958df6d82477e696458addc1d19f1bd13ff --- /dev/null +++ b/compute_kernel_writer/src/cl/CLTensorArgument.cpp @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "src/cl/CLTensorArgument.h" +#include "ckw/Error.h" +#include "src/ITensorArgument.h" +#include "src/ITensorComponent.h" +#include "src/cl/CLHelpers.h" +#include "src/cl/CLTensorComponent.h" +#include "src/types/TensorComponentType.h" + +#include +#include + +namespace ckw +{ +CLTensorArgument::CLTensorArgument(const std::string &name, const TensorInfo &info, bool return_dims_by_value) +{ + _return_dims_by_value = return_dims_by_value; + _basename = name; + _info = info; +} + +CLTensorArgument::~CLTensorArgument() = default; + +CLTensorComponent &CLTensorArgument::cl_component(TensorComponentType x) +{ + // Return the component if it has already been created. + { + const auto it = std::find_if( + _components_used.begin(), _components_used.end(), + [=](const std::unique_ptr &item) + { + return item->component_type() == x; + }); + + if(it != _components_used.end()) + { + return **it; + } + } + + if(_return_dims_by_value) + { + uint32_t component_type = static_cast(x); + + const bool is_dimension = (component_type & static_cast(TensorComponentBitmask::Dimension)) != 0; + const bool is_folded_dimensions = (component_type & static_cast(TensorComponentBitmask::FoldedDimensions)) != 0; + + constexpr auto bitmask_all = static_cast(TensorComponentIndexBitmask::All); + constexpr auto bitmask_index_0 = static_cast(TensorComponentIndexBitmask::Index0); +#ifdef COMPUTE_KERNEL_WRITER_ASSERTS_ENABLED + constexpr auto bitmask_index_1 = static_cast(TensorComponentIndexBitmask::Index1); + constexpr auto bitmask_index_2 = static_cast(TensorComponentIndexBitmask::Index2); + constexpr auto bitmask_index_3 = static_cast(TensorComponentIndexBitmask::Index3); +#endif // COMPUTE_KERNEL_WRITER_ASSERTS_ENABLED + + // Make sure that the encoding of component type hasn't changed and each nibble is 4 bits apart. + CKW_ASSERT(bitmask_all == (bitmask_index_0 | bitmask_index_1 | bitmask_index_2 | bitmask_index_3)); + CKW_ASSERT(bitmask_index_0 == bitmask_index_1 >> 4); + CKW_ASSERT(bitmask_index_1 == bitmask_index_2 >> 4); + CKW_ASSERT(bitmask_index_2 == bitmask_index_3 >> 4); + + // If we have a dimension or folded dimensions, we can return the corresponding value if it is not dynamic (not equal to -1) + if(is_dimension == true || is_folded_dimensions == true) + { + component_type = component_type & bitmask_all; + + int32_t idx = 1; + for(int32_t i = 0; i < tensor_component_index_max_count; ++i) + { + uint32_t dim_idx = component_type & bitmask_index_0; + + if(dim_idx == 0) + { + // Stop at the first nibble containing 0 + break; + } + + // Subtract - 1. Please refer to the TensorComponentIndexBitmask documentation + dim_idx -= 1; + + // Get the dimension value + const int32_t dim_val = _info.shape()[dim_idx]; + + if(dim_val == kDynamicTensorDimensionValue) + { + // We cannot return the dimension by value if it is dynamic. + // Therefore, force the idx variable to kDynamicTensorDimensionValue and break the loop. + idx = kDynamicTensorDimensionValue; + break; + } + + idx *= dim_val; + + // Go to the next nibble + component_type >>= 4; + } + + if(idx != kDynamicTensorDimensionValue) + { + _components_used.emplace_back(std::make_unique(*this, x, idx)); + + return *_components_used.back(); + } + } + } + + _components_used.emplace_back(std::make_unique(*this, x)); + + return *_components_used.back(); +} + +ITile &CLTensorArgument::component(TensorComponentType x) +{ + return cl_component(x); +} + +TensorStorageVariable &CLTensorArgument::storage(TensorStorageType x) +{ + // Return the storage if it has already been created. + { + const auto it = std::find_if( + _storages_used.begin(), _storages_used.end(), + [=](const TensorStorageVariable &item) + { + return item.type == x; + }); + + if(it != _storages_used.end()) + { + return *it; + } + } + + TensorStorageVariable t; + t.val = create_storage_name(x); + t.type = x; + + _storages_used.emplace_back(t); + + return _storages_used.back(); +} + +std::string CLTensorArgument::create_storage_name(TensorStorageType x) const +{ + std::string var_name = _basename; + + switch(x) + { + case TensorStorageType::BufferUint8Ptr: + var_name += "_ptr"; + break; + case TensorStorageType::Texture2dReadOnly: + case TensorStorageType::Texture2dWriteOnly: + var_name += "_img2d"; + break; + default: + CKW_ASSERT_FAILED_MSG("Unsupported tensor storage"); + return ""; + } + + return var_name; +} + +std::vector CLTensorArgument::storages() const +{ + std::vector storages; + storages.reserve(_storages_used.size()); + + std::copy(_storages_used.begin(), _storages_used.end(), std::back_inserter(storages)); + + return storages; +} + +std::vector CLTensorArgument::components() const +{ + std::vector components; + + for(const auto &component : _components_used) + { + if(component->is_assignable()) + { + components.push_back(component.get()); + } + } + + return components; +} +} // namespace ckw diff --git a/compute_kernel_writer/src/cl/CLTensorArgument.h b/compute_kernel_writer/src/cl/CLTensorArgument.h new file mode 100644 index 0000000000000000000000000000000000000000..4cbbee21ee107d3ad902fe2aede0b27f213e7997 --- /dev/null +++ b/compute_kernel_writer/src/cl/CLTensorArgument.h @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef CKW_SRC_CL_CLTENSORARGUMENT_H +#define CKW_SRC_CL_CLTENSORARGUMENT_H + +#include "ckw/types/TensorComponentType.h" +#include "ckw/types/TensorStorageType.h" +#include "src/ITensor.h" +#include +#include +#include + +namespace ckw +{ +// Forward declarations +class TensorInfo; + +class ITensorComponent; +class CLTensorComponent; +class CLTensorStorage; + +/** OpenCL specific tensor argument + * Internally, the object keeps track of the components and storages used to minimize the number + * of kernel arguments required. Therefore, if we create this object but we do not access any components + * or storages, the storages() and components() method will return an empty list. +*/ +class CLTensorArgument : public ITensor +{ +public: + /** Constructor + * + * @param[in] name Tensor name + * @param[in] info Tensor info + * @param[in] return_dims_by_value Flag to return the dimensions by value whenever it is possible. + * True, if the dimensions should be returned as value instead as variable. + */ + CLTensorArgument(const std::string &name, const TensorInfo &info, bool return_dims_by_value); + + /** Destructor. */ + ~CLTensorArgument(); + + /** Get a tensor component of the given type. + * + * This function is for internal use as it returns a reference to @ref CLTensorComponent object. + * It provides rich functionalities and doesn't require unnecessary casting + * unlike @ref CLTensorComponent::component which is for the public API and only returns + * a reference to a generic @ref ITile object. + */ + CLTensorComponent& cl_component(TensorComponentType component_type); + + // Inherited method overridden + TensorStorageVariable &storage(TensorStorageType x) override; + ITile &component(TensorComponentType x) override; + std::vector storages() const override; + std::vector components() const override; + +private: + std::string create_storage_name(TensorStorageType x) const; + + bool _return_dims_by_value{ false }; + std::vector _storages_used{}; + std::vector> _components_used{}; +}; + +} // namespace ckw + +#endif // CKW_SRC_CL_CLTENSORARGUMENT_H diff --git a/compute_kernel_writer/src/cl/CLTensorComponent.cpp b/compute_kernel_writer/src/cl/CLTensorComponent.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c29b307748e3c2554bef71416b4d486b60198687 --- /dev/null +++ b/compute_kernel_writer/src/cl/CLTensorComponent.cpp @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "src/cl/CLTensorComponent.h" +#include "ckw/Error.h" +#include "ckw/types/TensorComponentType.h" +#include "src/cl/CLTensorArgument.h" +#include "src/cl/CLTile.h" + +namespace ckw +{ + +namespace +{ + +std::string create_component_name(const std::string &name, TensorComponentType x) +{ + std::string var_name(name); + + switch(x) + { + case TensorComponentType::OffsetFirstElement: + var_name += "_offset_first_element"; + break; + case TensorComponentType::Stride0: + var_name += "_stride0"; + break; + case TensorComponentType::Stride1: + var_name += "_stride1"; + break; + case TensorComponentType::Stride2: + var_name += "_stride2"; + break; + case TensorComponentType::Stride3: + var_name += "_stride3"; + break; + case TensorComponentType::Stride4: + var_name += "_stride4"; + break; + case TensorComponentType::Dim0: + var_name += "_dim0"; + break; + case TensorComponentType::Dim1: + var_name += "_dim1"; + break; + case TensorComponentType::Dim2: + var_name += "_dim2"; + break; + case TensorComponentType::Dim3: + var_name += "_dim3"; + break; + case TensorComponentType::Dim4: + var_name += "_dim4"; + break; + case TensorComponentType::Dim1xDim2: + var_name += "_dim1xdim2"; + break; + case TensorComponentType::Dim2xDim3: + var_name += "_dim2xdim3"; + break; + case TensorComponentType::Dim1xDim2xDim3: + var_name += "_dim1xdim2xdim3"; + break; + default: + CKW_THROW_MSG("Unsupported tensor component"); + return ""; + } + + return var_name; +} + +} // namespace + +CLTensorComponent::CLTensorComponent(const CLTensorArgument &tensor, TensorComponentType component_type) + : CLTile(create_component_name(tensor.name(), component_type), TileInfo(DataType::Int32)), _component_type(component_type) +{ +} + +CLTensorComponent::CLTensorComponent(const CLTensorArgument &tensor, TensorComponentType component_type, int32_t value) + : CLTile({ { std::to_string(value) } }, DataType::Int32), _component_type(component_type) +{ + CKW_UNUSED(tensor); +} + +CLTensorComponent::~CLTensorComponent() = default; + +ITile &CLTensorComponent::tile() +{ + return *this; +} + +const ITile &CLTensorComponent::tile() const +{ + return *this; +} + +TensorComponentType CLTensorComponent::component_type() const +{ + return _component_type; +} + +} // namespace ckw diff --git a/compute_kernel_writer/src/cl/CLTensorComponent.h b/compute_kernel_writer/src/cl/CLTensorComponent.h new file mode 100644 index 0000000000000000000000000000000000000000..42a42666dc37666628c4b1ee5523b1b96af4f219 --- /dev/null +++ b/compute_kernel_writer/src/cl/CLTensorComponent.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_SRC_CL_CLTENSORCOMPONENT_H +#define CKW_SRC_CL_CLTENSORCOMPONENT_H + +#include "ckw/types/TensorComponentType.h" +#include "src/ITensorComponent.h" +#include "src/cl/CLTile.h" + +namespace ckw +{ + +class CLTensorArgument; + +/** A tensor component object that can be used as a tile. + * + * The tensor component is created by @ref CLTensorArgument object when it is used + * either by the user or internally by a kernel writer operation. + * It allows the user to perform operation on tensor component just like any other tile. + * + * Because of the nature of tensor component, it's always a scalar tile of 32-bit integer. + * + * To find the list of all tensor components, see @ref TensorComponentType. + */ +class CLTensorComponent : public CLTile, public ITensorComponent +{ +public: + /** Initialize a new instance of @ref CLTensorComponent class for dynamic component. + * + * @param[in] tensor The tensor to which this component belongs. + * @param[in] component_type The tensor component type. + */ + CLTensorComponent(const CLTensorArgument &tensor, TensorComponentType component_type); + + /** Initialize a new instance of @ref CLTensorComponent class for compile-time constant component. + * + * @param[in] tensor The tensor to which this component belongs. + * @param[in] component_type The tensor component type. + * @param[in] value The value of the component. + */ + CLTensorComponent(const CLTensorArgument &tensor, TensorComponentType component_type, int32_t value); + + /** Destructor. */ + virtual ~CLTensorComponent(); + + ITile &tile() override; + + const ITile &tile() const override; + + TensorComponentType component_type() const override; + +private: + TensorComponentType _component_type{ TensorComponentType::Unknown }; +}; + +} // namespace ckw + +#endif // CKW_SRC_CL_CLTENSORCOMPONENT_H diff --git a/compute_kernel_writer/src/cl/CLTile.cpp b/compute_kernel_writer/src/cl/CLTile.cpp new file mode 100644 index 0000000000000000000000000000000000000000..013ac4c2768b940cf20a8f864fc7808fea9cd5c7 --- /dev/null +++ b/compute_kernel_writer/src/cl/CLTile.cpp @@ -0,0 +1,232 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "ckw/Error.h" +#include "ckw/TileInfo.h" + +#include "src/Helpers.h" +#include "src/cl/CLHelpers.h" +#include "src/cl/CLTile.h" + +#include +#include + +namespace ckw +{ +CLTile::CLTile(const std::string &name, const TileInfo &info) + : _is_constant(false) +{ + validate_tile_info(info); + + _basename = name; + _info = info; +} + +CLTile::CLTile(const TileContainer &vals, DataType dt) + : _is_constant(true) +{ + const int32_t w = vals[0].size(); + const int32_t h = vals.size(); + + _info.width(w); + _info.height(h); + _info.data_type(dt); + + validate_tile_info(_info); + + _vals = TileContainer(h, std::vector(w)); + + for(int32_t y = 0; y < h; ++y) + { + for(int32_t x = 0; x < w; ++x) + { + _vals[y][x] = vals[y][x]; + } + } +} + +const std::string &CLTile::name() const +{ + return _basename; +} + +const TileInfo &CLTile::info() const +{ + return _info; +} + +TileVariable CLTile::scalar(int32_t row, int32_t col) const +{ + // Clamp to nearest valid edge + col = clamp(col, static_cast(0), _info.width() - 1); + row = clamp(row, static_cast(0), _info.height() - 1); + + if(_is_constant) + { + // We can use the vector method to retrieve the scalar variable stored in the constant tile + return vector(row, col, 1); + } + else + { + TileVariable t; + t.str = create_var_name(row); + t.desc.dt = _info.data_type(); + t.desc.len = 1; + + // This check is required because if the width has only one element, we cannot use .s0 + if(_info.width() != 1) + { + // Automatic broadcasting + t.str += ".s" + dec_to_hex_as_string(col); + } + + return t; + } +} + +TileVariable CLTile::vector(int32_t row) const +{ + // Clamp to nearest valid edge + row = clamp(row, static_cast(0), _info.height() - 1); + + if(_is_constant) + { + return vector(row, 0, _info.width()); + } + else + { + TileVariable t; + t.str = create_var_name(row); + t.desc.dt = _info.data_type(); + t.desc.len = _info.width(); + return t; + } +} + +TileVariable CLTile::vector(int32_t row, int32_t col_start, int32_t width) const +{ + // Validate the new vector length + cl_validate_vector_length(width); + + // Clamp to nearest valid edge + row = clamp(row, static_cast(0), _info.height() - 1); + + TileVariable t; + t.desc.dt = _info.data_type(); + t.desc.len = width; + + if(_is_constant) + { + // The vector has the following form: ((data_typeN)(val0, val1,..., ValN-1)) + t.str = "((" + cl_get_variable_datatype_as_string(t.desc.dt, width) + ")"; + t.str += "("; + + int32_t col = col_start; + for(; col < width - 1; ++col) + { + t.str += _vals[row][col]; + t.str += ", "; + } + t.str += _vals[row][col]; + t.str += "))"; + } + else + { + t.str = create_var_name(row); + + if(_info.width() != 1) + { + t.str += ".s"; + for(int i = 0; i < width; ++i) + { + t.str += dec_to_hex_as_string(col_start + i); + } + } + } + + return t; +} + +std::vector CLTile::all() const +{ + std::vector vars; + + if(_is_constant) + { + for(int32_t y = 0; y < _info.height(); ++y) + { + for(int32_t x = 0; x < _info.width(); ++x) + { + // We can use the vector method to retrieve all the scalar variables stored in the constant tile + TileVariable t = vector(y, x, 1); + vars.push_back(t); + } + } + } + else + { + for(int32_t y = 0; y < _info.height(); ++y) + { + TileVariable t; + t.str = create_var_name(y); + t.desc.dt = _info.data_type(); + t.desc.len = _info.width(); + vars.push_back(t); + } + } + + return vars; +} + +bool CLTile::is_assignable() const +{ + return !_is_constant; +} + +std::string CLTile::create_var_name(int32_t row) const +{ + std::string var_name = _basename; + + // If a scalar variable, we do not append the row index + if(_info.height() > 1) + { + var_name += "_"; + var_name += std::to_string(row); + } + + return var_name; +} + +std::vector CLTile::supported_vector_lengths() const +{ + return std::vector{ 1, 2, 3, 4, 8, 16 }; +} + +void CLTile::validate_tile_info(const TileInfo &info) const +{ + CKW_UNUSED(info); + CKW_ASSERT_MSG(cl_validate_vector_length(info.width()), "Unsupported TileInfo width"); + CKW_ASSERT_MSG(info.data_type() != DataType::Unknown, "DataType::Unknown is not supported"); +} + +} // namespace ckw \ No newline at end of file diff --git a/compute_kernel_writer/src/cl/CLTile.h b/compute_kernel_writer/src/cl/CLTile.h new file mode 100644 index 0000000000000000000000000000000000000000..46af4de364979f70303f1a20d24a1a3105c3168a --- /dev/null +++ b/compute_kernel_writer/src/cl/CLTile.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef COMPUTE_KERNEL_WRITER_SRC_CL_CLTILE_H +#define COMPUTE_KERNEL_WRITER_SRC_CL_CLTILE_H + +#include "src/ITile.h" +#include + +namespace ckw +{ +// Forward declarations +class TileInfo; + +/** OpenCL specific tile */ +class CLTile : public ITile, public IVectorAccess, public IScalarAccess +{ +public: + /** Initialize a new instance of @ref CLTile class for variable tile. + * + * @param[in] name Tile name + * @param[in] info Tile info + */ + CLTile(const std::string &name, const TileInfo &info); + + /** Initialize a new instane of @ref CLTile class for compile-time constant tile. + * + * @note A constant tile does not need a name since this object does not return variable's name but rather + * values stored as string type + * + * @param[in] vals The tile container with the constant values as std::string + * @param[in] dt Datatype of the values stored in the tile container + */ + CLTile(const TileContainer &vals, DataType dt); + + // Inherited method overridden + const std::string &name() const override; + + const TileInfo &info() const override; + + TileVariable scalar(int32_t row, int32_t col) const override; + + TileVariable vector(int32_t row) const override; + + TileVariable vector(int32_t row, int32_t col_start, int32_t width) const override; + + std::vector all() const override; + + bool is_assignable() const override; + + std::vector supported_vector_lengths() const override; + +private: + void validate_tile_info(const TileInfo &info) const; + + std::string create_var_name(int32_t row) const; + + TileInfo _info{ DataType::Unknown }; + std::string _basename{ "" }; + bool _is_constant{ false }; + TileContainer _vals{}; +}; +} // namespace ckw + +#endif /* COMPUTE_KERNEL_WRITER_SRC_CL_CLTILE_H */ diff --git a/compute_kernel_writer/src/types/TensorComponentType.h b/compute_kernel_writer/src/types/TensorComponentType.h new file mode 100644 index 0000000000000000000000000000000000000000..03f4f4f5c83ae46bd80af38783432b1bef576a3b --- /dev/null +++ b/compute_kernel_writer/src/types/TensorComponentType.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_SRC_TYPES_TENSORCOMPONENTTYPE_H +#define CKW_SRC_TYPES_TENSORCOMPONENTTYPE_H + +#include + +namespace ckw +{ + +/** Compute Kernel Writer tensor component bitmask. + * + * The bitmask can be used to retrieve the info from @ref TensorComponent. + */ +enum class TensorComponentBitmask : uint32_t +{ + OffsetFirstElement = 0x01000000, // For example, OffsetFirstElement in TensorComponent + Stride = 0x02000000, // For example, stride0 in TensorComponent + Dimension = 0x04000000, // For example, Dim0 in TensorComponent + FoldedDimensions = 0x08000000, // For example, Dim0xDim1 in TensorComponent +}; + +/** Mask to retrieve the component index (for example, 1 for stride1, 2 for stride2, or 1 and 2 for Dim1xDim2). + * + * The 4 least significant half-bytes (nibbles) of the @ref TensorComponent are used to retrieve the specific component index. + * TensorComponent = | i7 | i6 | i5 | i4 | i3 | i2 | i1 | i0 |, where i7,...i0 are the nibbles + * of the TensorComponent hexadecimal number. i0, i1, i2 and i3 are reserved to the component index. + * + * In particular: + * + * -# i0: reserved to the first folded dimension component index + * -# i1: reserved to the second folded dimension component index + * -# i2: reserved to the third folded dimension component index + * -# i3: reserved to the fourth folded dimension component index + * + * Therefore, if there are no folded dimensions (dimensions and strides), only i0 is used. + * Instead, if there are two folded dimensions, only i0 and i1 are used. + * + * The component index is stored with the corresponding hexadecimal number + 1, + * hence the component index 0 is represented as 1, while the component index 3 is represented as 4. + */ +enum class TensorComponentIndexBitmask : uint32_t +{ + All = 0x0000ffff, // All nibbles reserved to the tensor component index + Index0 = 0x0000000f, // Folded dimension 0 + Index1 = 0x000000f0, // Folded dimension 1 + Index2 = 0x00000f00, // Folded dimension 2 + Index3 = 0x0000f000 // Folded dimension 3 +}; + +/** The maximum number of folded dimensions. */ +constexpr int tensor_component_index_max_count = 4; + +} // namespace ckw + +#endif // CKW_SRC_TYPES_TENSORCOMPONENTTYPE_H diff --git a/compute_kernel_writer/validation/Validation.cpp b/compute_kernel_writer/validation/Validation.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5d53a16effbd643231d53fcab0e91f19e857e8cc --- /dev/null +++ b/compute_kernel_writer/validation/Validation.cpp @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "tests/CLConstantTileTest.hpp" +#include "tests/CLKernelWriterCommentTest.h" +#include "tests/CLKernelWriterDeclareTileTest.h" +#include "tests/CLTensorArgumentTest.h" +#include "tests/CLTileTest.hpp" +#include "tests/TensorBitMaskTest.h" +#include "tests/UtilsTest.h" + +#include +#include + +using namespace ckw; + +/** Main test program + */ +int32_t main() +{ + std::vector tests; + + // Add your test here + const auto test0 = std::make_unique(); + const auto test1 = std::make_unique(); + const auto test2 = std::make_unique(); + tests.push_back(test0.get()); + tests.push_back(test1.get()); + tests.push_back(test2.get()); + +#ifdef COMPUTE_KERNEL_WRITER_OPENCL_ENABLED + const auto test3 = std::make_unique(); + const auto test4 = std::make_unique(); + const auto test5 = std::make_unique(); + const auto test6 = std::make_unique(); + const auto test7 = std::make_unique(); + const auto test8 = std::make_unique(); + const auto test9 = std::make_unique(); + const auto test10 = std::make_unique(); + const auto test11 = std::make_unique(); + const auto test12 = std::make_unique(); + const auto test13 = std::make_unique(); + const auto test14 = std::make_unique(); +#ifdef COMPUTE_KERNEL_WRITER_DEBUG_ENABLED + const auto test15 = std::make_unique(); +#endif /* COMPUTE_KERNEL_WRITER_DEBUG_ENABLED */ + const auto test16 = std::make_unique(); + const auto test17 = std::make_unique(); + const auto test18 = std::make_unique(); + const auto test19 = std::make_unique(); + const auto test20 = std::make_unique(); + const auto test21 = std::make_unique(); + const auto test22 = std::make_unique(); + const auto test23 = std::make_unique(); + + tests.push_back(test3.get()); + tests.push_back(test4.get()); + tests.push_back(test5.get()); + tests.push_back(test6.get()); + tests.push_back(test7.get()); + tests.push_back(test8.get()); + tests.push_back(test9.get()); + tests.push_back(test10.get()); + tests.push_back(test11.get()); + tests.push_back(test12.get()); + tests.push_back(test13.get()); + tests.push_back(test14.get()); +#ifdef COMPUTE_KERNEL_WRITER_DEBUG_ENABLED + tests.push_back(test15.get()); +#endif /* COMPUTE_KERNEL_WRITER_DEBUG_ENABLED */ + tests.push_back(test16.get()); + tests.push_back(test17.get()); + tests.push_back(test18.get()); + tests.push_back(test19.get()); + tests.push_back(test20.get()); + tests.push_back(test21.get()); + tests.push_back(test22.get()); + tests.push_back(test23.get()); +#endif /* COMPUTE_KERNEL_WRITER_OPENCL_ENABLED */ + + bool all_test_passed = true; + + for(auto &x : tests) + { + std::cout << x->name() << std::endl; + all_test_passed &= x->run(); + } + + if(all_test_passed == true) + { + std::cout << "All tests passed" << std::endl; + } + else + { + throw std::runtime_error("One or more tests failed"); + } + + return 0; +} diff --git a/compute_kernel_writer/validation/tests/CLConstantTileTest.hpp b/compute_kernel_writer/validation/tests/CLConstantTileTest.hpp new file mode 100644 index 0000000000000000000000000000000000000000..f10ad10146bdb7b9001579a52a0188ba762aa769 --- /dev/null +++ b/compute_kernel_writer/validation/tests/CLConstantTileTest.hpp @@ -0,0 +1,371 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_TESTS_CLCONSTANTTILETEST_HPP +#define CKW_TESTS_CLCONSTANTTILETEST_HPP + +#include "common/Common.h" +#include "src/Helpers.h" +#include "src/cl/CLHelpers.h" +#include "src/cl/CLTile.h" + +#include +#include +#include + +namespace ckw +{ +class CLConstantTileInternalValuesTest : public ITest +{ +public: + CLConstantTileInternalValuesTest() + { + _values.push_back({ { "1.2", "3.5" }, + { "4.2", "1.3" } }); + _values.push_back({ { "1.2" } }); + _values.push_back({ { "1.2", "6.9" } }); + } + + bool run() override + { + // The status of this variable can change in VALIDATE_TEST() + bool all_tests_passed = true; + + int32_t test_idx = 0; + for(const auto &test : _values) + { + const CLTile tile(test, DataType::Fp16); + const auto vars = tile.all(); + const int32_t num_vars = vars.size(); + const int32_t width = tile.info().width(); + + for(int32_t y = 0; y < num_vars; ++y) + { + const int32_t col = y % width; + const int32_t row = y / width; + const std::string expected_var_name = "((half)(" + test[row][col] + "))"; + const std::string actual_var_name = vars[y].str; + VALIDATE_TEST(actual_var_name.compare(expected_var_name) == 0, all_tests_passed, test_idx++); + } + } + return all_tests_passed; + } + + std::string name() override + { + return "CLConstantTileInternalValuesTest"; + } + +private: + std::vector _values{}; +}; + +class CLConstantTileAccessScalarVariableBroadcastXTest : public ITest +{ +public: + const std::string tile_name = "src"; + const int32_t height = 8; + const DataType dt = DataType::Fp16; + + CLConstantTileAccessScalarVariableBroadcastXTest() + { + _width.push_back(1); + _width.push_back(2); + _width.push_back(3); + + _x_coord.push_back(4); + _x_coord.push_back(5); + _x_coord.push_back(6); + + _y_coord.push_back(1); + _y_coord.push_back(3); + _y_coord.push_back(2); + } + + bool run() override + { + VALIDATE_ON_MSG(_width.size() == _y_coord.size(), "The number of widths and y-coords does not match"); + VALIDATE_ON_MSG(_x_coord.size() == _y_coord.size(), "The number of x-coords and y-coords does not match"); + + // The status of this variable can change in VALIDATE_TEST() + bool all_tests_passed = true; + + const size_t num_coords = _x_coord.size(); + + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_real_distribution<> dist(-1, 1); + + int32_t test_idx = 0; + for(size_t i = 0; i < num_coords; ++i) + { + const int32_t width = _width[i]; + const int32_t x_coord = _x_coord[i]; + const int32_t y_coord = _y_coord[i]; + + const int32_t x_coord_clamped = clamp(x_coord, static_cast(0), width - 1); + + TileContainer container = TileContainer(height, std::vector(width)); + + for(int32_t row = 0; row < height; ++row) + { + for(int32_t col = 0; col < width; ++col) + { + container[row][col] = std::to_string(dist(gen)); + } + } + + const CLTile tile(container, dt); + + const TileVariable var = tile.scalar(y_coord, x_coord); + + const std::string actual_var_name = var.str; + const std::string expected_var_name = "((half)(" + container[y_coord][x_coord_clamped] + "))"; + + VALIDATE_TEST(actual_var_name.compare(expected_var_name) == 0, all_tests_passed, test_idx++); + } + return all_tests_passed; + } + + std::string name() override + { + return "CLConstantTileAccessScalarVariableBroadcastXTest"; + } + +private: + std::vector _width{}; + std::vector _x_coord{}; + std::vector _y_coord{}; +}; + +class CLConstantTileAccessScalarVariableBroadcastYTest : public ITest +{ +public: + const std::string tile_name = "src"; + const int32_t width = 8; + const DataType dt = DataType::Fp16; + + CLConstantTileAccessScalarVariableBroadcastYTest() + { + _height.push_back(1); + _height.push_back(2); + _height.push_back(3); + + _x_coord.push_back(4); + _x_coord.push_back(5); + _x_coord.push_back(6); + + _y_coord.push_back(3); + _y_coord.push_back(4); + _y_coord.push_back(5); + } + + bool run() override + { + VALIDATE_ON_MSG(_height.size() == _y_coord.size(), "The number of widths and y-coords does not match"); + VALIDATE_ON_MSG(_x_coord.size() == _y_coord.size(), "The number of x-coords and y-coords does not match"); + + // The status of this variable can change in VALIDATE_TEST() + bool all_tests_passed = true; + + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_real_distribution<> dist(-1, 1); + + const size_t num_coords = _x_coord.size(); + + int32_t test_idx = 0; + for(size_t i = 0; i < num_coords; ++i) + { + const int32_t height = _height[i]; + const int32_t x_coord = _x_coord[i]; + const int32_t y_coord = _y_coord[i]; + + const int32_t y_coord_clamped = clamp(y_coord, static_cast(0), height - 1); + + TileContainer container = TileContainer(height, std::vector(width)); + + for(int32_t row = 0; row < height; ++row) + { + for(int32_t col = 0; col < width; ++col) + { + container[row][col] = std::to_string(dist(gen)); + } + } + + const CLTile tile(container, dt); + + const TileVariable var = tile.scalar(y_coord, x_coord); + + const std::string actual_var_name = var.str; + const std::string expected_var_name = "((half)(" + container[y_coord_clamped][x_coord] + "))"; + + VALIDATE_TEST(actual_var_name.compare(expected_var_name) == 0, all_tests_passed, test_idx++); + } + return all_tests_passed; + } + + std::string name() override + { + return "CLConstantTileAccessScalarVariableBroadcastYTest"; + } + +private: + std::vector _height{}; + std::vector _x_coord{}; + std::vector _y_coord{}; +}; + +class CLConstantTileAccessVectorVariablesTest : public ITest +{ +public: + const DataType dt = DataType::Fp16; + + CLConstantTileAccessVectorVariablesTest() + { + _values.push_back({ { "1.2", "3.5" }, + { "4.2", "1.3" } }); + _values.push_back({ { "1.2" } }); + // Mix variable names and values + _values.push_back({ { "1.2", "acc", "8.7", "9.3", "ratio", "2.9", "1.7", "0.3" } }); + } + + bool run() override + { + // The status of this variable can change in VALIDATE_TEST() + bool all_tests_passed = true; + + int32_t test_idx = 0; + + for(const auto &test : _values) + { + const CLTile tile(test, dt); + const int32_t width = tile.info().width(); + const int32_t height = tile.info().height(); + + for(int32_t row = 0; row < height; ++row) + { + std::string expected_var_name = "(("; + expected_var_name += cl_get_variable_datatype_as_string(dt, width); + expected_var_name += ")("; + + int32_t col = 0; + for(; col < width - 1; ++col) + { + expected_var_name += test[row][col]; + expected_var_name += ", "; + } + + expected_var_name += test[row][col]; + expected_var_name += "))"; + + const std::string actual_var_name = tile.vector(row).str; + VALIDATE_TEST(actual_var_name.compare(expected_var_name) == 0, all_tests_passed, test_idx++); + } + } + return all_tests_passed; + } + + std::string name() override + { + return "CLConstantTileAccessVectorVariablesTest"; + } + +private: + std::vector _values{}; +}; + +class CLConstantTileAccessSubVectorVariablesTest : public ITest +{ +public: + const DataType dt = DataType::Fp16; + + CLConstantTileAccessSubVectorVariablesTest() + { + _values.push_back({ { "1.2", "acc", "8.7", "9.3", "ratio", "2.9", "1.7", "0.3" } }); + _subwidths.push_back(1); + _subwidths.push_back(2); + _subwidths.push_back(3); + _subwidths.push_back(4); + _offsets.push_back(1); + _offsets.push_back(3); + _offsets.push_back(4); + } + + bool run() override + { + // The status of this variable can change in VALIDATE_TEST() + bool all_tests_passed = true; + + size_t test_idx = 0; + + for(auto &test : _values) + { + for(auto &col_start : _offsets) + { + for(auto &subwidth : _subwidths) + { + const CLTile tile(test, dt); + const int32_t height = tile.info().height(); + + for(int32_t row = 0; row < height; ++row) + { + std::string expected_var_name = "(("; + expected_var_name += cl_get_variable_datatype_as_string(dt, subwidth); + expected_var_name += ")("; + + int32_t col = col_start; + for(; col < subwidth - 1; ++col) + { + expected_var_name += test[row][col]; + expected_var_name += ", "; + } + + expected_var_name += test[row][col]; + expected_var_name += "))"; + + const std::string actual_var_name = tile.vector(row, col_start, subwidth).str; + VALIDATE_TEST(actual_var_name.compare(expected_var_name) == 0, all_tests_passed, + test_idx++); + } + } + } + } + return all_tests_passed; + } + + std::string name() override + { + return "CLConstantTileAccessSubVectorVariablesTest"; + } + +private: + std::vector _values{}; + std::vector _subwidths{}; + std::vector _offsets{}; +}; + +} // namespace ckw + +#endif // CKW_TESTS_CLCONSTANTTILETEST_HPP diff --git a/compute_kernel_writer/validation/tests/CLKernelWriterCommentTest.h b/compute_kernel_writer/validation/tests/CLKernelWriterCommentTest.h new file mode 100644 index 0000000000000000000000000000000000000000..ff09ea80737e4461c93b907c6142402b9a4e912b --- /dev/null +++ b/compute_kernel_writer/validation/tests/CLKernelWriterCommentTest.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_VALIDATION_TESTS_CLKERNELTEST_H +#define CKW_VALIDATION_TESTS_CLKERNELTEST_H + +#include "src/cl/CLKernelWriter.h" +#include "validation/tests/common/Common.h" +#include "validation/tests/common/KernelWriterInterceptor.h" + +namespace ckw +{ + +class CLKernelWriterCommentTest : public ITest +{ +public: + CLKernelWriterCommentTest() + { + } + + bool run() override + { + bool all_tests_passed = true; + + KernelWriterInterceptor writer; + + writer.comment("previous code"); + + writer.start_capture_code(); + + writer.comment("code under test 0"); + writer.comment("code under test 1"); + + constexpr auto expected_code = "// code under test 0\n// code under test 1\n"; + + VALIDATE_TEST(writer.check_added_code(expected_code), all_tests_passed, 0); + + return all_tests_passed; + } + + std::string name() override + { + return "CLKernelWriterCommentTest"; + } +}; + +} // namespace ckw + +#endif // CKW_VALIDATION_TESTS_CLKERNELTEST_H diff --git a/compute_kernel_writer/validation/tests/CLKernelWriterDeclareTileTest.h b/compute_kernel_writer/validation/tests/CLKernelWriterDeclareTileTest.h new file mode 100644 index 0000000000000000000000000000000000000000..5e00084aaac55c36e1d929a479307692b060bf92 --- /dev/null +++ b/compute_kernel_writer/validation/tests/CLKernelWriterDeclareTileTest.h @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_VALIDATION_TESTS_CLKERNELWRITER_H +#define CKW_VALIDATION_TESTS_CLKERNELWRITER_H + +#include "ckw/TileInfo.h" +#include "ckw/types/DataType.h" +#include "src/cl/CLKernelWriter.h" +#include "validation/tests/common/KernelWriterInterceptor.h" +#include "validation/tests/common/Common.h" + +#include + +namespace ckw +{ + +using CLKernelWriterDeclareTileConfig = std::tuple; + +class CLKernelWriterDeclareTileTest : public ITest +{ +public: + CLKernelWriterDeclareTileTest() + { + _configs = { + {DataType::Fp32, 4, 4, "float4 G0__a_tile"}, + {DataType::Uint8, 4, 1, "uchar G0__a_tile"}, + {DataType::Int8, 4, 2, "char2 G0__a_tile"}, + {DataType::Bool, 9, 3, "bool3 G0__a_tile"}, + {DataType::Fp16, 4, 16, "half16 G0__a_tile"}, + {DataType::Uint32, 1, 8, "uint8 G0__a_tile"}, + {DataType::Uint16, 2, 3, "ushort3 G0__a_tile"}, + }; + } + + bool run() override + { + bool all_tests_passed = true; + int32_t test_idx = 0; + + for(auto _config: _configs) + { + KernelWriterInterceptor writer; + writer.start_capture_code(); + + const DataType data_type = std::get<0>(_config); + const int32_t height = std::get<1>(_config); + const int32_t width = std::get<2>(_config); + const std::string prefix = std::get<3>(_config); + + // expected output + std::string expected_code = ""; + for(int32_t row = 0; row < height; ++row) + { + expected_code += prefix + std::to_string(row) + ";\n"; + } + + TileInfo tile_info(data_type, height, width); + writer.declare_tile("a_tile", tile_info); + + VALIDATE_TEST(writer.check_added_code(expected_code), all_tests_passed, test_idx++); + } + + return all_tests_passed; + } + + std::string name() override + { + return "CLKernelWriterDeclareTileTest"; + } + +private: + std::vector _configs {}; +}; + +} // namespace ckw + +#endif /* CKW_VALIDATION_TESTS_CLKERNELWRITER_H */ diff --git a/compute_kernel_writer/validation/tests/CLTensorArgumentTest.h b/compute_kernel_writer/validation/tests/CLTensorArgumentTest.h new file mode 100644 index 0000000000000000000000000000000000000000..d3e455cb83009ced2f0cc5551f138c0e7e3ce71c --- /dev/null +++ b/compute_kernel_writer/validation/tests/CLTensorArgumentTest.h @@ -0,0 +1,540 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_VALIDATION_TESTS_CLTENSORARGUMENTTEST_H +#define CKW_VALIDATION_TESTS_CLTENSORARGUMENTTEST_H + +#include "common/Common.h" +#include "src/cl/CLHelpers.h" +#include "src/cl/CLTensorArgument.h" +#include "src/cl/CLTensorComponent.h" + +#include +#include + +namespace ckw +{ +class CLTensorArgumentComponentNamesTest : public ITest +{ +public: + const DataType dt = DataType::Fp32; + const TensorShape shape = TensorShape({ { 12, 14, 3, 1, 2 } }); + const std::string tensor_name = "src"; + + CLTensorArgumentComponentNamesTest() + { + _components.push_back(TensorComponentType::Dim0); + _components.push_back(TensorComponentType::Dim1); + _components.push_back(TensorComponentType::Dim2); + _components.push_back(TensorComponentType::Dim3); + _components.push_back(TensorComponentType::Dim4); + _components.push_back(TensorComponentType::Dim1xDim2); + _components.push_back(TensorComponentType::Dim2xDim3); + _components.push_back(TensorComponentType::OffsetFirstElement); + _components.push_back(TensorComponentType::Stride0); + _components.push_back(TensorComponentType::Stride1); + _components.push_back(TensorComponentType::Stride2); + _components.push_back(TensorComponentType::Stride3); + _components.push_back(TensorComponentType::Stride4); + + _expected_vars.push_back("src_dim0"); + _expected_vars.push_back("src_dim1"); + _expected_vars.push_back("src_dim2"); + _expected_vars.push_back("src_dim3"); + _expected_vars.push_back("src_dim4"); + _expected_vars.push_back("src_dim1xdim2"); + _expected_vars.push_back("src_dim2xdim3"); + _expected_vars.push_back("src_offset_first_element"); + _expected_vars.push_back("src_stride0"); + _expected_vars.push_back("src_stride1"); + _expected_vars.push_back("src_stride2"); + _expected_vars.push_back("src_stride3"); + _expected_vars.push_back("src_stride4"); + } + + bool run() override + { + VALIDATE_ON_MSG(_components.size() == _expected_vars.size(), "The number of components and variables does not match"); + + // The status of this variable can change in VALIDATE_TEST() + bool all_tests_passed = true; + + const TensorInfo info(dt, shape, TensorDataLayout::Nhwc, 1); + + const size_t num_tests = _expected_vars.size(); + + int32_t test_idx = 0; + for(size_t i = 0; i < num_tests; ++i) + { + CLTensorArgument arg(tensor_name, info, false /* return_dims_by_value */); + + const std::string expected_var_name = _expected_vars[i]; + const std::string actual_var_name = arg.component(_components[i]).name(); + + VALIDATE_TEST(actual_var_name.compare(expected_var_name) == 0, all_tests_passed, test_idx++); + } + return all_tests_passed; + } + + std::string name() override + { + return "CLTensorArgumentVariableNamesTest"; + } + +private: + std::vector _components{}; + std::vector _expected_vars{}; +}; + +class CLTensorArgumentStorageNamesTest : public ITest +{ +public: + const DataType dt = DataType::Fp32; + const TensorShape shape = TensorShape({ { 12, 14, 3, 1, 2 } }); + const std::string tensor_name = "src"; + + CLTensorArgumentStorageNamesTest() + { + _storages.push_back(TensorStorageType::BufferUint8Ptr); + _storages.push_back(TensorStorageType::Texture2dReadOnly); + _storages.push_back(TensorStorageType::Texture2dWriteOnly); + + _expected_vars.push_back("src_ptr"); + _expected_vars.push_back("src_img2d"); + _expected_vars.push_back("src_img2d"); + } + + bool run() override + { + VALIDATE_ON_MSG(_storages.size() == _expected_vars.size(), "The number of storages and variables does not match"); + + // The status of this variable can change in VALIDATE_TEST() + bool all_tests_passed = true; + + const TensorInfo info(dt, shape, TensorDataLayout::Nhwc, 1); + + const size_t num_tests = _expected_vars.size(); + + int32_t test_idx = 0; + for(size_t i = 0; i < num_tests; ++i) + { + CLTensorArgument arg(tensor_name, info, false /* return_dims_by_value */); + + const std::string expected_var_name = _expected_vars[i]; + const std::string actual_var_name = arg.storage(_storages[i]).val; + + VALIDATE_TEST(actual_var_name.compare(expected_var_name) == 0, all_tests_passed, test_idx++); + } + return all_tests_passed; + } + + std::string name() override + { + return "CLTensorArgumentStorageNamesTest"; + } + +private: + std::vector _storages{}; + std::vector _expected_vars{}; +}; + +class CLTensorArgumentComponentValuesTest : public ITest +{ +public: + const DataType dt = DataType::Fp32; + const TensorShape shape = TensorShape({ { 12, 14, 3, 1, 2 } }); + const std::string tensor_name = "src"; + + CLTensorArgumentComponentValuesTest() + { + _components.push_back(TensorComponentType::Dim0); + _components.push_back(TensorComponentType::Dim1); + _components.push_back(TensorComponentType::Dim2); + _components.push_back(TensorComponentType::Dim3); + _components.push_back(TensorComponentType::Dim4); + _components.push_back(TensorComponentType::Dim1xDim2); + _components.push_back(TensorComponentType::Dim2xDim3); + + _expected_vals.push_back(std::to_string(shape[0])); + _expected_vals.push_back(std::to_string(shape[1])); + _expected_vals.push_back(std::to_string(shape[2])); + _expected_vals.push_back(std::to_string(shape[3])); + _expected_vals.push_back(std::to_string(shape[4])); + _expected_vals.push_back(std::to_string(shape[1] * shape[2])); + _expected_vals.push_back(std::to_string(shape[2] * shape[3])); + } + + bool run() override + { + VALIDATE_ON_MSG(_components.size() == _expected_vals.size(), "The number of components and values does not match"); + + // The status of this variable can change in VALIDATE_TEST() + bool all_tests_passed = true; + + const TensorInfo info(dt, shape, TensorDataLayout::Nhwc, 1); + + const size_t num_tests = _expected_vals.size(); + + int32_t test_idx = 0; + for(size_t i = 0; i < num_tests; ++i) + { + CLTensorArgument arg(tensor_name, info, true /* return_dims_by_value */); + + const std::string expected_var_val = std::string("((int)(") + _expected_vals[i] + "))"; + const std::string actual_var_val = arg.cl_component(_components[i]).scalar(0, 0).str; + + VALIDATE_TEST(actual_var_val.compare(expected_var_val) == 0, all_tests_passed, test_idx++); + } + return all_tests_passed; + } + + std::string name() override + { + return "CLTensorArgumentComponentValuesTest"; + } + +private: + std::vector _components{}; + std::vector _expected_vals{}; +}; + +class CLTensorArgumentComponentsUsedPassByValueFalseTest : public ITest +{ +public: + const DataType dt = DataType::Fp32; + const TensorShape shape = TensorShape({ { 12, 14, 3, 1, 2 } }); + const std::string tensor_name = "src"; + + CLTensorArgumentComponentsUsedPassByValueFalseTest() + { + _components.push_back(TensorComponentType::Dim0); + _components.push_back(TensorComponentType::Dim2); + _components.push_back(TensorComponentType::Dim3); + _components.push_back(TensorComponentType::Dim1xDim2); + _components.push_back(TensorComponentType::OffsetFirstElement); + _components.push_back(TensorComponentType::Stride1); + _components.push_back(TensorComponentType::Stride2); + _components.push_back(TensorComponentType::Stride3); + _components.push_back(TensorComponentType::Dim0); // Repeat the query. The TensorArgument should not create a new variable + _components.push_back(TensorComponentType::Dim2); // Repeat the query. The TensorArgument should not create a new variable + _components.push_back(TensorComponentType::Dim3); // Repeat the query. The TensorArgument should not create a new variable + + _expected_vars.push_back("src_dim0"); + _expected_vars.push_back("src_dim2"); + _expected_vars.push_back("src_dim3"); + _expected_vars.push_back("src_dim1xdim2"); + _expected_vars.push_back("src_offset_first_element"); + _expected_vars.push_back("src_stride1"); + _expected_vars.push_back("src_stride2"); + _expected_vars.push_back("src_stride3"); + } + + bool run() override + { + // The status of this variable can change in VALIDATE_TEST() + bool all_tests_passed = true; + + const TensorInfo info(dt, shape, TensorDataLayout::Nhwc, 1); + + const size_t num_components = _components.size(); + + int32_t test_idx = 0; + + CLTensorArgument arg(tensor_name, info, false /* return_dims_by_value */); + for(size_t i = 0; i < num_components; ++i) + { + arg.component(_components[i]); + } + + const auto actual_vars = arg.components(); + + const size_t num_vars = _expected_vars.size(); + + VALIDATE_ON_MSG(actual_vars.size() == num_vars, "The number of variables must match the number of expected variables"); + + for(size_t i = 0; i < num_vars; ++i) + { + // Validate variable name + const std::string expected_var_name = _expected_vars[i]; + const std::string actual_var_name = actual_vars[i]->tile().name(); + VALIDATE_TEST(actual_var_name.compare(expected_var_name) == 0, all_tests_passed, test_idx++); + + // Validate data type + const DataType expected_var_type = DataType::Int32; + const DataType actual_var_type = actual_vars[i]->tile().info().data_type(); + VALIDATE_TEST(actual_var_type == expected_var_type, all_tests_passed, test_idx++); + + // Validate tile shape + const int32_t actual_var_width = actual_vars[i]->tile().info().width(); + const int32_t actual_var_height = actual_vars[i]->tile().info().height(); + + VALIDATE_TEST(actual_var_height == 1, all_tests_passed, test_idx++); + VALIDATE_TEST(actual_var_width == 1, all_tests_passed, test_idx++); + } + return all_tests_passed; + } + + std::string name() override + { + return "CLTensorArgumentComponentsUsedPassByValueFalseTest"; + } + +private: + std::vector _components{}; + std::vector _expected_vars{}; +}; + +class CLTensorArgumentComponentsUsedPassByValueTrueTest : public ITest +{ +public: + const DataType dt = DataType::Fp32; + const TensorShape shape = TensorShape({ { 12, 14, 3, 1, 2 } }); + const std::string tensor_name = "src"; + + CLTensorArgumentComponentsUsedPassByValueTrueTest() + { + _components.push_back(TensorComponentType::Dim0); + _components.push_back(TensorComponentType::Dim2); + _components.push_back(TensorComponentType::Dim3); + _components.push_back(TensorComponentType::Dim1xDim2); + _components.push_back(TensorComponentType::OffsetFirstElement); + _components.push_back(TensorComponentType::Stride1); + _components.push_back(TensorComponentType::Stride2); + _components.push_back(TensorComponentType::Stride3); + _components.push_back(TensorComponentType::OffsetFirstElement); // Repeat the query. The TensorArgument should not create a new variable + _components.push_back(TensorComponentType::Stride1); // Repeat the query. The TensorArgument should not create a new variable + + _expected_vars.push_back("src_offset_first_element"); + _expected_vars.push_back("src_stride1"); + _expected_vars.push_back("src_stride2"); + _expected_vars.push_back("src_stride3"); + } + + bool run() override + { + // The status of this variable can change in VALIDATE_TEST() + bool all_tests_passed = true; + + const TensorInfo info(dt, shape, TensorDataLayout::Nhwc, 1); + + const size_t num_components = _components.size(); + + int32_t test_idx = 0; + + CLTensorArgument arg(tensor_name, info, true /* return_dims_by_value */); + for(size_t i = 0; i < num_components; ++i) + { + arg.component(_components[i]); + } + + const auto actual_vars = arg.components(); + + const size_t num_vars = _expected_vars.size(); + + VALIDATE_ON_MSG(actual_vars.size() == num_vars, "The number of variables must match the number of expected variables"); + + // Since the dimensions are passed by value, we expect only the variables for the strides + for(size_t i = 0; i < num_vars; ++i) + { + // Validate variable name + const std::string expected_var_name = _expected_vars[i]; + const std::string actual_var_name = actual_vars[i]->tile().name(); + VALIDATE_TEST(actual_var_name.compare(expected_var_name) == 0, all_tests_passed, test_idx++); + + // Validate data type + const DataType expected_var_type = DataType::Int32; + const DataType actual_var_type = actual_vars[i]->tile().info().data_type(); + VALIDATE_TEST(actual_var_type == expected_var_type, all_tests_passed, test_idx++); + + // Validate tile shape + const int32_t actual_var_width = actual_vars[i]->tile().info().width(); + const int32_t actual_var_height = actual_vars[i]->tile().info().height(); + + VALIDATE_TEST(actual_var_height == 1, all_tests_passed, test_idx++); + VALIDATE_TEST(actual_var_width == 1, all_tests_passed, test_idx++); + } + return all_tests_passed; + } + + std::string name() override + { + return "CLTensorArgumentComponentsUsedPassByValueTrueTest"; + } + +private: + std::vector _components{}; + std::vector _expected_vars{}; +}; + +class CLTensorArgumentStoragesUsedTest : public ITest +{ +public: + const DataType dt = DataType::Fp32; + const TensorShape shape = TensorShape({ { 12, 14, 3, 1, 2 } }); + const std::string tensor_name = "src"; + + CLTensorArgumentStoragesUsedTest() + { + _storages.push_back(TensorStorageType::BufferUint8Ptr); + _storages.push_back(TensorStorageType::Texture2dReadOnly); + _storages.push_back(TensorStorageType::BufferUint8Ptr); // Repeat the query. The TensorArgument should not create a new variable + + _expected_vars.push_back("src_ptr"); + _expected_vars.push_back("src_img2d"); + } + + bool run() override + { + // The status of this variable can change in VALIDATE_TEST() + bool all_tests_passed = true; + + const TensorInfo info(dt, shape, TensorDataLayout::Nhwc, 1); + + const size_t num_storages = _storages.size(); + + int32_t test_idx = 0; + + CLTensorArgument arg(tensor_name, info, true /* return_dims_by_value */); + for(size_t i = 0; i < num_storages; ++i) + { + arg.storage(_storages[i]); + } + + const auto actual_vars = arg.storages(); + + const size_t num_vars = _expected_vars.size(); + + VALIDATE_ON_MSG(actual_vars.size() == num_vars, "The number of variables must match the number of expected variables"); + + for(size_t i = 0; i < num_vars; ++i) + { + // Validate variable name + const std::string expected_var_name = _expected_vars[i]; + const std::string actual_var_name = actual_vars[i].val; + VALIDATE_TEST(actual_var_name.compare(expected_var_name) == 0, all_tests_passed, test_idx++); + + // Validate storage type + const auto expected_var_type = _storages[i]; + const auto actual_var_type = actual_vars[i].type; + VALIDATE_TEST(actual_var_type == expected_var_type, all_tests_passed, test_idx++); + } + return all_tests_passed; + } + + std::string name() override + { + return "CLTensorArgumentStoragesUsedTest"; + } + +private: + std::vector _storages{}; + std::vector _expected_vars{}; +}; + +class CLTensorArgumentComponentsUsedPassByValueTrueDynamicDimTrueTest : public ITest +{ +public: + const DataType dt = DataType::Fp32; + const TensorShape shape = TensorShape({ { -1, -1, 3, 1, 2 } }); + const std::string tensor_name = "src"; + + CLTensorArgumentComponentsUsedPassByValueTrueDynamicDimTrueTest() + { + _components.push_back(TensorComponentType::Dim0); + _components.push_back(TensorComponentType::Dim2); + _components.push_back(TensorComponentType::Dim3); + _components.push_back(TensorComponentType::Dim1xDim2); + _components.push_back(TensorComponentType::OffsetFirstElement); + _components.push_back(TensorComponentType::Stride1); + _components.push_back(TensorComponentType::Stride2); + _components.push_back(TensorComponentType::Stride3); + _components.push_back(TensorComponentType::OffsetFirstElement); // Repeat the query. The TensorArgument should not create a new variable + _components.push_back(TensorComponentType::Stride1); // Repeat the query. The TensorArgument should not create a new variable + + _expected_vars.push_back("src_dim0"); + _expected_vars.push_back("src_dim1xdim2"); + _expected_vars.push_back("src_offset_first_element"); + _expected_vars.push_back("src_stride1"); + _expected_vars.push_back("src_stride2"); + _expected_vars.push_back("src_stride3"); + } + + bool run() override + { + // The status of this variable can change in VALIDATE_TEST() + bool all_tests_passed = true; + + const TensorInfo info(dt, shape, TensorDataLayout::Nhwc, 1); + + const size_t num_components = _components.size(); + + int32_t test_idx = 0; + + CLTensorArgument arg(tensor_name, info, true /* return_dims_by_value */); + for(size_t i = 0; i < num_components; ++i) + { + arg.component(_components[i]); + } + + const auto actual_vars = arg.components(); + + const size_t num_vars = _expected_vars.size(); + + VALIDATE_ON_MSG(actual_vars.size() == num_vars, "The number of variables must match the number of expected variables"); + + // Since the dimensions are passed by value, we expect only the variables for the strides + for(size_t i = 0; i < num_vars; ++i) + { + // Validate variable name + const std::string expected_var_name = _expected_vars[i]; + const std::string actual_var_name = actual_vars[i]->tile().name(); + VALIDATE_TEST(actual_var_name.compare(expected_var_name) == 0, all_tests_passed, test_idx++); + + // Validate data type + const DataType expected_var_type = DataType::Int32; + const DataType actual_var_type = actual_vars[i]->tile().info().data_type(); + VALIDATE_TEST(actual_var_type == expected_var_type, all_tests_passed, test_idx++); + + // Validate tile shape + const int32_t actual_var_width = actual_vars[i]->tile().info().width(); + const int32_t actual_var_height = actual_vars[i]->tile().info().height(); + + VALIDATE_TEST(actual_var_height == 1, all_tests_passed, test_idx++); + VALIDATE_TEST(actual_var_width == 1, all_tests_passed, test_idx++); + } + return all_tests_passed; + } + + std::string name() override + { + return "CLTensorArgumentComponentsUsedPassByValueTrueDynamicDimTrueTest"; + } + +private: + std::vector _components{}; + std::vector _expected_vars{}; +}; +} // namespace ckw + +#endif // CKW_VALIDATION_TESTS_CLTENSORARGUMENTTEST_H diff --git a/compute_kernel_writer/validation/tests/CLTileTest.hpp b/compute_kernel_writer/validation/tests/CLTileTest.hpp new file mode 100644 index 0000000000000000000000000000000000000000..ecfe811267a069eec73563cf2ea135f6afe316e9 --- /dev/null +++ b/compute_kernel_writer/validation/tests/CLTileTest.hpp @@ -0,0 +1,467 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef COMPUTE_KERNEL_WRITER_TESTS_CLTILETEST_HPP +#define COMPUTE_KERNEL_WRITER_TESTS_CLTILETEST_HPP + +#include "common/Common.h" +#include "src/Helpers.h" +#include "src/cl/CLTile.h" + +#include +#include + +namespace ckw +{ +class CLTileInternalVariableNamesTest : public ITest +{ +public: + const int32_t width = 4; + const int32_t height = 4; + const DataType dt = DataType::Fp32; + + CLTileInternalVariableNamesTest() + { + _tile_name.push_back("dst"); + _tile_name.push_back("_G0_dst"); + _tile_name.push_back("_SRC"); + } + + bool run() override + { + // The status of this variable can change in VALIDATE_TEST() + bool all_tests_passed = true; + + const TileInfo info(dt, height, width); + + int32_t test_idx = 0; + for(const auto &tile_name : _tile_name) + { + const CLTile tile(tile_name, info); + const auto vars = tile.all(); + + for(int32_t y = 0; y < height; ++y) + { + const std::string expected_var_name = tile_name + "_" + std::to_string(y); + const std::string actual_var_name = vars[y].str; + VALIDATE_TEST(actual_var_name.compare(expected_var_name) == 0, all_tests_passed, test_idx++); + } + } + return all_tests_passed; + } + + std::string name() override + { + return "CLTileInternalVariableNamesTest"; + } + +private: + std::vector _tile_name{}; +}; + +class CLTileInternalNumVariablesTest : public ITest +{ +public: + CLTileInternalNumVariablesTest() + { + _width.push_back(4); + _width.push_back(1); + _width.push_back(16); + + _height.push_back(1); + _height.push_back(5); + _height.push_back(3); + } + + bool run() override + { + VALIDATE_ON_MSG(_width.size() == _height.size(), "The number of widths and heights does not match"); + + // The status of this variable can change in VALIDATE_TEST() + bool all_tests_passed = true; + + const size_t num_dims = _width.size(); + + int32_t test_idx = 0; + for(size_t i = 0; i < num_dims; ++i) + { + const int32_t width = _width[i]; + const int32_t height = _height[i]; + const TileInfo info(DataType::Fp32, height, width); + const CLTile tile("src", info); + const auto vars = tile.all(); + const int32_t num_vars = vars.size(); + + // We expect the number of variables to match the heigth of the tile + VALIDATE_TEST(num_vars == height, all_tests_passed, test_idx++); + } + return all_tests_passed; + } + + std::string name() override + { + return "CLTileInternalNumVariablesTest"; + } + +private: + std::vector _width{}; + std::vector _height{}; +}; + +class CLTileAccessScalarVariableTest : public ITest +{ +public: + const std::string tile_name = "src"; + const int32_t width = 16; + const int32_t height = 8; + const DataType dt = DataType::Fp32; + + CLTileAccessScalarVariableTest() + { + _x_coord.push_back(4); + _x_coord.push_back(1); + _x_coord.push_back(15); + _x_coord.push_back(10); + + _y_coord.push_back(1); + _y_coord.push_back(5); + _y_coord.push_back(3); + _y_coord.push_back(4); + } + + bool run() override + { + const TileInfo info(dt, height, width); + const CLTile tile(tile_name, info); + + VALIDATE_ON_MSG(_x_coord.size() == _y_coord.size(), "The number of x-coords and y-coords does not match"); + + // The status of this variable can change in VALIDATE_TEST() + bool all_tests_passed = true; + + const size_t num_coords = _x_coord.size(); + + int32_t test_idx = 0; + for(size_t i = 0; i < num_coords; ++i) + { + const int32_t x_coord = _x_coord[i]; + const int32_t y_coord = _y_coord[i]; + + const TileVariable var = tile.scalar(y_coord, x_coord); + + const std::string actual_var_name = var.str; + std::string expected_var_name = tile_name; + expected_var_name += "_" + std::to_string(y_coord); + expected_var_name += ".s" + dec_to_hex_as_string(x_coord); + + VALIDATE_TEST(actual_var_name.compare(expected_var_name) == 0, all_tests_passed, test_idx++); + } + return all_tests_passed; + } + + std::string name() override + { + return "CLTileAccessScalarVariableTest"; + } + +private: + std::vector _x_coord{}; + std::vector _y_coord{}; +}; + +class CLTileAccessScalarVariableBroadcastXTest : public ITest +{ +public: + const std::string tile_name = "src"; + const int32_t height = 8; + const DataType dt = DataType::Fp32; + + CLTileAccessScalarVariableBroadcastXTest() + { + _width.push_back(1); + _width.push_back(2); + _width.push_back(3); + + _x_coord.push_back(4); + _x_coord.push_back(5); + _x_coord.push_back(6); + + _y_coord.push_back(1); + _y_coord.push_back(3); + _y_coord.push_back(2); + } + + bool run() override + { + VALIDATE_ON_MSG(_width.size() == _y_coord.size(), "The number of widths and y-coords does not match"); + VALIDATE_ON_MSG(_x_coord.size() == _y_coord.size(), "The number of x-coords and y-coords does not match"); + + // The status of this variable can change in VALIDATE_TEST() + bool all_tests_passed = true; + + const size_t num_coords = _x_coord.size(); + + int32_t test_idx = 0; + for(size_t i = 0; i < num_coords; ++i) + { + const int32_t width = _width[i]; + const int32_t x_coord = _x_coord[i]; + const int32_t y_coord = _y_coord[i]; + + const int32_t x_coord_clamped = clamp(x_coord, static_cast(0), width - 1); + + const TileInfo info(dt, height, width); + const CLTile tile(tile_name, info); + + const TileVariable var = tile.scalar(y_coord, x_coord); + + const std::string actual_var_name = var.str; + std::string expected_var_name = tile_name; + expected_var_name += "_" + std::to_string(y_coord); + if(width != 1) + { + expected_var_name += ".s" + dec_to_hex_as_string(x_coord_clamped); + } + + VALIDATE_TEST(actual_var_name.compare(expected_var_name) == 0, all_tests_passed, test_idx++); + } + return all_tests_passed; + } + + std::string name() override + { + return "CLTileAccessScalarVariableBroadcastXTest"; + } + +private: + std::vector _width{}; + std::vector _x_coord{}; + std::vector _y_coord{}; +}; + +class CLTileAccessScalarVariableBroadcastYTest : public ITest +{ +public: + const std::string tile_name = "src"; + const int32_t width = 8; + const DataType dt = DataType::Fp32; + + CLTileAccessScalarVariableBroadcastYTest() + { + _height.push_back(1); + _height.push_back(2); + _height.push_back(3); + + _x_coord.push_back(4); + _x_coord.push_back(5); + _x_coord.push_back(6); + + _y_coord.push_back(3); + _y_coord.push_back(4); + _y_coord.push_back(5); + } + + bool run() override + { + VALIDATE_ON_MSG(_height.size() == _y_coord.size(), "The number of widths and y-coords does not match"); + VALIDATE_ON_MSG(_x_coord.size() == _y_coord.size(), "The number of x-coords and y-coords does not match"); + + // The status of this variable can change in VALIDATE_TEST() + bool all_tests_passed = true; + + const size_t num_coords = _x_coord.size(); + + int32_t test_idx = 0; + for(size_t i = 0; i < num_coords; ++i) + { + const int32_t height = _height[i]; + const int32_t x_coord = _x_coord[i]; + const int32_t y_coord = _y_coord[i]; + + const int32_t y_coord_clamped = clamp(y_coord, static_cast(0), height - 1); + + const TileInfo info(dt, height, width); + const CLTile tile(tile_name, info); + + const TileVariable var = tile.scalar(y_coord, x_coord); + + const std::string actual_var_name = var.str; + std::string expected_var_name = tile_name; + if(height != 1) + { + expected_var_name += "_" + std::to_string(y_coord_clamped); + } + + if(width != 1) + { + expected_var_name += ".s" + dec_to_hex_as_string(x_coord); + } + + VALIDATE_TEST(actual_var_name.compare(expected_var_name) == 0, all_tests_passed, test_idx++); + } + return all_tests_passed; + } + + std::string name() override + { + return "CLTileAccessScalarVariableBroadcastYTest"; + } + +private: + std::vector _height{}; + std::vector _x_coord{}; + std::vector _y_coord{}; +}; + +class CLTileAccessVectorVariablesTest : public ITest +{ +public: + const std::string tile_name = "src"; + const int32_t width = 8; + const DataType dt = DataType::Fp32; + + CLTileAccessVectorVariablesTest() + { + _heights.push_back(1); + _heights.push_back(2); + _heights.push_back(3); + } + + bool run() override + { + // The status of this variable can change in VALIDATE_TEST() + bool all_tests_passed = true; + + int32_t test_idx = 0; + for(const auto &height : _heights) + { + const TileInfo info(dt, height, width); + const CLTile tile(tile_name, info); + + for(int32_t row = 0; row < height; ++row) + { + const TileVariable var = tile.vector(row); + + const std::string actual_var_name = var.str; + std::string expected_var_name = tile_name; + if(height != 1) + { + expected_var_name += "_" + std::to_string(row); + } + + VALIDATE_TEST(actual_var_name.compare(expected_var_name) == 0, all_tests_passed, test_idx++); + } + } + return all_tests_passed; + } + + std::string name() override + { + return "CLTileAccessVectorVariablesTest"; + } + +private: + std::vector _heights{}; +}; + +class CLTileAccessSubVectorVariablesTest : public ITest +{ +public: + const std::string tile_name = "src"; + const int32_t width = 8; + const int32_t height = 3; + const DataType dt = DataType::Fp32; + + CLTileAccessSubVectorVariablesTest() + { + _subwidths.push_back(1); + _subwidths.push_back(2); + _subwidths.push_back(3); + _subwidths.push_back(4); + _offsets.push_back(1); + _offsets.push_back(3); + _offsets.push_back(4); + } + + bool run() override + { + // The status of this variable can change in VALIDATE_TEST() + bool all_tests_passed = true; + + size_t test_idx = 0; + + for(auto &col_start : _offsets) + { + for(const auto &subwidth : _subwidths) + { + const TileInfo info(dt, height, width); + const CLTile tile(tile_name, info); + + for(int32_t row = 0; row < height; ++row) + { + std::string expected_var_name = tile_name; + if(height != 1) + { + expected_var_name += "_" + std::to_string(row); + } + + if(width != 1) + { + expected_var_name += ".s"; + } + + int32_t col = col_start; + for(; col < col_start + subwidth - 1; ++col) + { + if(width != 1) + { + expected_var_name += dec_to_hex_as_string(col); + } + } + + if(width != 1) + { + expected_var_name += dec_to_hex_as_string(col); + } + + const std::string actual_var_name = tile.vector(row, col_start, subwidth).str; + VALIDATE_TEST(actual_var_name.compare(expected_var_name) == 0, all_tests_passed, test_idx++); + } + } + } + return all_tests_passed; + } + + std::string name() override + { + return "CLTileAccessSubVectorVariablesTest"; + } + +private: + std::vector _subwidths{}; + std::vector _offsets{}; +}; +} // namespace ckw + +#endif /* COMPUTE_KERNEL_WRITER_TESTS_CLTILETEST_HPP */ diff --git a/compute_kernel_writer/validation/tests/TensorBitMaskTest.h b/compute_kernel_writer/validation/tests/TensorBitMaskTest.h new file mode 100644 index 0000000000000000000000000000000000000000..759d926d1824253ac8867f756676946d174ef518 --- /dev/null +++ b/compute_kernel_writer/validation/tests/TensorBitMaskTest.h @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef CKW_TESTS_TENSORBITMASKTEST_H +#define CKW_TESTS_TENSORBITMASKTEST_H + +#include "ckw/TensorInfo.h" +#include "ckw/types/TensorComponentType.h" +#include "common/Common.h" +#include "src/types/TensorComponentType.h" + +#include + +namespace ckw +{ +class TensorBitMaskTrueTest : public ITest +{ +public: + TensorBitMaskTrueTest() + { + _component.push_back(TensorComponentType::Dim0); + _component.push_back(TensorComponentType::Dim1); + _component.push_back(TensorComponentType::Dim2); + _component.push_back(TensorComponentType::Dim3); + _component.push_back(TensorComponentType::Dim4); + _component.push_back(TensorComponentType::Stride0); + _component.push_back(TensorComponentType::Stride1); + _component.push_back(TensorComponentType::Stride2); + _component.push_back(TensorComponentType::Stride3); + _component.push_back(TensorComponentType::Stride4); + _component.push_back(TensorComponentType::Dim1xDim2); + _component.push_back(TensorComponentType::Dim1xDim2xDim3); + _component.push_back(TensorComponentType::Dim2xDim3); + _component.push_back(TensorComponentType::OffsetFirstElement); + + _bitmask.push_back(TensorComponentBitmask::Dimension); + _bitmask.push_back(TensorComponentBitmask::Dimension); + _bitmask.push_back(TensorComponentBitmask::Dimension); + _bitmask.push_back(TensorComponentBitmask::Dimension); + _bitmask.push_back(TensorComponentBitmask::Dimension); + _bitmask.push_back(TensorComponentBitmask::Stride); + _bitmask.push_back(TensorComponentBitmask::Stride); + _bitmask.push_back(TensorComponentBitmask::Stride); + _bitmask.push_back(TensorComponentBitmask::Stride); + _bitmask.push_back(TensorComponentBitmask::Stride); + _bitmask.push_back(TensorComponentBitmask::FoldedDimensions); + _bitmask.push_back(TensorComponentBitmask::FoldedDimensions); + _bitmask.push_back(TensorComponentBitmask::FoldedDimensions); + _bitmask.push_back(TensorComponentBitmask::OffsetFirstElement); + } + + bool run() override + { + // The status of this variable can change in VALIDATE_TEST() + bool all_tests_passed = true; + + VALIDATE_ON_MSG(_component.size() == _bitmask.size(), + "The number of layouts and components does not match"); + const size_t num_tests = _component.size(); + for(size_t i = 0; i < num_tests; ++i) + { + const TensorComponentType component = _component[i]; + const TensorComponentBitmask bitmask = _bitmask[i]; + const bool out = static_cast(component) & static_cast(bitmask); + VALIDATE_TEST(out == true, all_tests_passed, i); + } + return all_tests_passed; + } + + std::string name() override + { + return "TensorBitMaskTrueTest"; + } + +private: + std::vector _component{}; + std::vector _bitmask{}; +}; + +class TensorBitMaskFalseTest : public ITest +{ +public: + TensorBitMaskFalseTest() + { + _component.push_back(TensorComponentType::Dim0); + _component.push_back(TensorComponentType::Dim1); + _component.push_back(TensorComponentType::Dim2); + _component.push_back(TensorComponentType::Dim3); + _component.push_back(TensorComponentType::Dim4); + _component.push_back(TensorComponentType::Dim0); + _component.push_back(TensorComponentType::Dim1); + _component.push_back(TensorComponentType::Dim2); + _component.push_back(TensorComponentType::Dim3); + _component.push_back(TensorComponentType::Dim4); + _component.push_back(TensorComponentType::Dim0); + _component.push_back(TensorComponentType::Dim1); + _component.push_back(TensorComponentType::Dim2); + _component.push_back(TensorComponentType::Dim3); + _component.push_back(TensorComponentType::Dim4); + _component.push_back(TensorComponentType::Stride0); + _component.push_back(TensorComponentType::Stride1); + _component.push_back(TensorComponentType::Stride2); + _component.push_back(TensorComponentType::Stride3); + _component.push_back(TensorComponentType::Stride4); + _component.push_back(TensorComponentType::Stride0); + _component.push_back(TensorComponentType::Stride1); + _component.push_back(TensorComponentType::Stride2); + _component.push_back(TensorComponentType::Stride3); + _component.push_back(TensorComponentType::Stride4); + _component.push_back(TensorComponentType::Stride0); + _component.push_back(TensorComponentType::Stride1); + _component.push_back(TensorComponentType::Stride2); + _component.push_back(TensorComponentType::Stride3); + _component.push_back(TensorComponentType::Stride4); + _component.push_back(TensorComponentType::Dim1xDim2); + _component.push_back(TensorComponentType::Dim1xDim2xDim3); + _component.push_back(TensorComponentType::Dim2xDim3); + _component.push_back(TensorComponentType::Dim1xDim2); + _component.push_back(TensorComponentType::Dim1xDim2xDim3); + _component.push_back(TensorComponentType::Dim2xDim3); + _component.push_back(TensorComponentType::Dim1xDim2); + _component.push_back(TensorComponentType::Dim1xDim2xDim3); + _component.push_back(TensorComponentType::Dim2xDim3); + _component.push_back(TensorComponentType::OffsetFirstElement); + _component.push_back(TensorComponentType::OffsetFirstElement); + _component.push_back(TensorComponentType::OffsetFirstElement); + + _bitmask.push_back(TensorComponentBitmask::Stride); + _bitmask.push_back(TensorComponentBitmask::Stride); + _bitmask.push_back(TensorComponentBitmask::Stride); + _bitmask.push_back(TensorComponentBitmask::Stride); + _bitmask.push_back(TensorComponentBitmask::Stride); + _bitmask.push_back(TensorComponentBitmask::FoldedDimensions); + _bitmask.push_back(TensorComponentBitmask::FoldedDimensions); + _bitmask.push_back(TensorComponentBitmask::FoldedDimensions); + _bitmask.push_back(TensorComponentBitmask::FoldedDimensions); + _bitmask.push_back(TensorComponentBitmask::FoldedDimensions); + _bitmask.push_back(TensorComponentBitmask::OffsetFirstElement); + _bitmask.push_back(TensorComponentBitmask::OffsetFirstElement); + _bitmask.push_back(TensorComponentBitmask::OffsetFirstElement); + _bitmask.push_back(TensorComponentBitmask::OffsetFirstElement); + _bitmask.push_back(TensorComponentBitmask::OffsetFirstElement); + _bitmask.push_back(TensorComponentBitmask::Dimension); + _bitmask.push_back(TensorComponentBitmask::Dimension); + _bitmask.push_back(TensorComponentBitmask::Dimension); + _bitmask.push_back(TensorComponentBitmask::Dimension); + _bitmask.push_back(TensorComponentBitmask::Dimension); + _bitmask.push_back(TensorComponentBitmask::FoldedDimensions); + _bitmask.push_back(TensorComponentBitmask::FoldedDimensions); + _bitmask.push_back(TensorComponentBitmask::FoldedDimensions); + _bitmask.push_back(TensorComponentBitmask::FoldedDimensions); + _bitmask.push_back(TensorComponentBitmask::FoldedDimensions); + _bitmask.push_back(TensorComponentBitmask::OffsetFirstElement); + _bitmask.push_back(TensorComponentBitmask::OffsetFirstElement); + _bitmask.push_back(TensorComponentBitmask::OffsetFirstElement); + _bitmask.push_back(TensorComponentBitmask::OffsetFirstElement); + _bitmask.push_back(TensorComponentBitmask::OffsetFirstElement); + _bitmask.push_back(TensorComponentBitmask::Dimension); + _bitmask.push_back(TensorComponentBitmask::Dimension); + _bitmask.push_back(TensorComponentBitmask::Dimension); + _bitmask.push_back(TensorComponentBitmask::Stride); + _bitmask.push_back(TensorComponentBitmask::Stride); + _bitmask.push_back(TensorComponentBitmask::Stride); + _bitmask.push_back(TensorComponentBitmask::OffsetFirstElement); + _bitmask.push_back(TensorComponentBitmask::OffsetFirstElement); + _bitmask.push_back(TensorComponentBitmask::OffsetFirstElement); + _bitmask.push_back(TensorComponentBitmask::Dimension); + _bitmask.push_back(TensorComponentBitmask::Stride); + _bitmask.push_back(TensorComponentBitmask::FoldedDimensions); + } + + bool run() override + { + // The status of this variable can change in VALIDATE_TEST() + bool all_tests_passed = true; + + VALIDATE_ON_MSG(_component.size() == _bitmask.size(), + "The number of layouts and components does not match"); + const size_t num_tests = _component.size(); + for(size_t i = 0; i < num_tests; ++i) + { + const TensorComponentType component = _component[i]; + const TensorComponentBitmask bitmask = _bitmask[i]; + const bool out = static_cast(component) & static_cast(bitmask); + VALIDATE_TEST(out == false, all_tests_passed, i); + } + return all_tests_passed; + } + + std::string name() override + { + return "TensorBitMaskFalseTest"; + } + +private: + std::vector _component{}; + std::vector _bitmask{}; +}; +} // namespace ckw + +#endif // CKW_TESTS_TENSORBITMASKTEST_H diff --git a/compute_kernel_writer/validation/tests/UtilsTest.h b/compute_kernel_writer/validation/tests/UtilsTest.h new file mode 100644 index 0000000000000000000000000000000000000000..a335a48f818d79290a711363896180ea8b413cc3 --- /dev/null +++ b/compute_kernel_writer/validation/tests/UtilsTest.h @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef CKW_TESTS_UTILSTEST_H +#define CKW_TESTS_UTILSTEST_H + +#include "ckw/TensorInfo.h" +#include "ckw/types/TensorDataLayout.h" +#include "common/Common.h" +#include "src/TensorUtils.h" + +#include + +namespace ckw +{ +class UtilsTest : public ITest +{ +public: + UtilsTest() + { + _layout.push_back(TensorDataLayout::Nhwc); + _layout.push_back(TensorDataLayout::Nhwc); + _layout.push_back(TensorDataLayout::Nhwc); + _layout.push_back(TensorDataLayout::Nhwc); + _layout.push_back(TensorDataLayout::Ndhwc); + _layout.push_back(TensorDataLayout::Ndhwc); + _layout.push_back(TensorDataLayout::Ndhwc); + _layout.push_back(TensorDataLayout::Ndhwc); + _layout.push_back(TensorDataLayout::Ndhwc); + + _component.push_back(TensorDataLayoutComponent::N); + _component.push_back(TensorDataLayoutComponent::H); + _component.push_back(TensorDataLayoutComponent::W); + _component.push_back(TensorDataLayoutComponent::C); + _component.push_back(TensorDataLayoutComponent::N); + _component.push_back(TensorDataLayoutComponent::D); + _component.push_back(TensorDataLayoutComponent::H); + _component.push_back(TensorDataLayoutComponent::W); + _component.push_back(TensorDataLayoutComponent::C); + + _expected.push_back(TensorComponentType::Dim3); + _expected.push_back(TensorComponentType::Dim2); + _expected.push_back(TensorComponentType::Dim1); + _expected.push_back(TensorComponentType::Dim0); + _expected.push_back(TensorComponentType::Dim4); + _expected.push_back(TensorComponentType::Dim3); + _expected.push_back(TensorComponentType::Dim2); + _expected.push_back(TensorComponentType::Dim1); + _expected.push_back(TensorComponentType::Dim0); + } + + bool run() override + { + // The status of this variable can change in VALIDATE_TEST() + bool all_tests_passed = true; + + VALIDATE_ON_MSG(_layout.size() == _component.size(), "The number of layouts and components does not match"); + VALIDATE_ON_MSG(_layout.size() == _expected.size(), + "The number of layouts and expected outputs does not match"); + const size_t num_tests = _layout.size(); + for(size_t i = 0; i < num_tests; ++i) + { + const TensorDataLayout layout = _layout[i]; + const TensorDataLayoutComponent component = _component[i]; + const TensorComponentType expected = _expected[i]; + const TensorComponentType out = get_tensor_dimension(layout, component); + VALIDATE_TEST(out == expected, all_tests_passed, i); + } + return all_tests_passed; + } + + std::string name() override + { + return "UtilsTest"; + } + +private: + std::vector _layout{}; + std::vector _component{}; + std::vector _expected{}; +}; +} // namespace ckw + +#endif // CKW_TESTS_UTILSTEST_H diff --git a/compute_kernel_writer/validation/tests/common/Common.h b/compute_kernel_writer/validation/tests/common/Common.h new file mode 100644 index 0000000000000000000000000000000000000000..8573c42b882a683732af6303c81ad2a5247fc820 --- /dev/null +++ b/compute_kernel_writer/validation/tests/common/Common.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef COMPUTE_KERNEL_WRITER_TEST_COMMON_COMMON_H +#define COMPUTE_KERNEL_WRITER_TEST_COMMON_COMMON_H + +#include +#include +#include + +namespace ckw +{ +#define VALIDATE_ON_MSG(exp, msg) assert(((void)msg, exp)) + +#define VALIDATE_TEST(exp, all_tests_passed, id_test) \ + do \ + { \ + if((exp) == true) \ + { \ + all_tests_passed &= true; \ + const std::string msg = "TEST " + std::to_string((id_test)) + ": [PASSED]"; \ + std::cout << msg << std::endl; \ + } \ + else \ + { \ + all_tests_passed &= false; \ + const std::string msg = "TEST " + std::to_string((id_test)) + ": [FAILED]"; \ + std::cout << msg << std::endl; \ + } \ + } while(false) + +class ITest +{ +public: + virtual ~ITest() = default; + + /** Method to run the test + * + * @return it returns true if all tests passed + */ + virtual bool run() = 0; + + /** Name of the test + * + * @return it returns the name of the test + */ + virtual std::string name() = 0; +}; +} // namespace ckw + +#endif /* COMPUTE_KERNEL_WRITER_TEST_COMMON_COMMON_H */ diff --git a/compute_kernel_writer/validation/tests/common/KernelWriterInterceptor.h b/compute_kernel_writer/validation/tests/common/KernelWriterInterceptor.h new file mode 100644 index 0000000000000000000000000000000000000000..89bb76e37ff73c0f2fbaeaff238cfde79ade2c45 --- /dev/null +++ b/compute_kernel_writer/validation/tests/common/KernelWriterInterceptor.h @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef CKW_VALIDATION_TESTS_COMMON_KERNELWRITERINTERCEPTOR_H +#define CKW_VALIDATION_TESTS_COMMON_KERNELWRITERINTERCEPTOR_H + +#include +#include + +namespace ckw +{ + +/** This class provides the ability to capture only the code changed after a point + * and compares that part of code with the expected value. + * + * It is useful for testing purpose when a particular sequence of instructions is interested + * while the rest of the initialization code is out of scope. + */ +template +class KernelWriterInterceptor : public T +{ +public: + template + KernelWriterInterceptor(const TArgs &&...args) + : T(std::forward(args)...) + { + } + + /** Mark this point in the source code as the start position to capture. + * Only source code added after this function is considered when check_add_code is called. + */ + void start_capture_code() + { + _start_code = this->body_source_code(); + } + + /** Compare the source code added after start_capture_code is called the the specified expected code. */ + bool check_added_code(const std::string &expected_added_code) + { + const auto &end_code = this->body_source_code(); + + // Code can only grow over time. + if(end_code.length() < _start_code.length()) + { + return false; + } + + // New code must be added to the source code without changing the already existed code. + if(end_code.substr(0, _start_code.length()) != _start_code) + { + return false; + } + + // The newly added code must match the expected value. + if(end_code.substr(_start_code.length(), end_code.length() - _start_code.length()) != expected_added_code) + { + return false; + } + + return true; + } + +private: + std::string _start_code{}; +}; + +} // namespace ckw + +#endif // CKW_VALIDATION_TESTS_COMMON_KERNELWRITERINTERCEPTOR_H diff --git a/docs/Doxyfile b/docs/Doxyfile index a6735ebdb6a27a7e438557bd9069988ff753e275..1654d16fc410940d7381ea99305a7ee3e954967c 100644 --- a/docs/Doxyfile +++ b/docs/Doxyfile @@ -38,7 +38,7 @@ PROJECT_NAME = "Compute Library" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 23.05.1 +PROJECT_NUMBER = 23.08 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -773,6 +773,7 @@ INPUT = ./docs/user_guide/introduction.dox \ ./docs/user_guide/library.dox \ ./docs/user_guide/data_type.dox \ ./docs/user_guide/data_layout.dox \ + ./docs/user_guide/conv2d_heuristic.dox \ ./docs/user_guide/operator_list.dox \ ./docs/user_guide/tests.dox \ ./docs/user_guide/advanced.dox \ diff --git a/docs/contributor_guide/adding_operator.dox b/docs/contributor_guide/adding_operator.dox index b9bc7e09ce4c043fbec737cb8b6ec13e2da586a6..559e8e2e7613014c79cb7ba1aacbe0155e5ea382 100644 --- a/docs/contributor_guide/adding_operator.dox +++ b/docs/contributor_guide/adding_operator.dox @@ -55,13 +55,13 @@ Following are the steps involved in adding support for a new operator in Compute @subsection S4_1_1_add_datatypes Adding new data types Compute Library declares a few new datatypes related to its domain, kernels, and functions in the library process Tensors and Images (Computer Vision functions). Tensors are multi-dimensional arrays with a maximum of Coordinates::num_max_dimensions dimensions; depending on the number of dimensions tensors can be interpreted as various objects. A scalar can be represented as a zero-dimensional tensor and a vector of numbers can be represented as a one-dimensional tensor. Furthermore, an image is just a 2D tensor, a 3D tensor can be seen as an array of images and a 4D tensor as a 2D array of images, etc. -All the datatype classes or structures are grouped in the core library folder arm_compute/core like the @ref ITensor, @ref ITensorInfo (all the information of a tensor), TensorShape and simpler types are in arm_compute/core/Types.h. +All the datatype classes or structures are grouped in the core library folder arm_compute/core like the @ref ITensor, @ref ITensorInfo (all the information of a tensor), TensorShape and simpler types are in arm_compute/core/CoreTypes.h. If an operator handles a new datatype, it must be added to the library. While adding a new data type to the library, it's necessary to implement the function to enable printing, the to_string() method and the output stream insertion (<<) operator. Every datatype implements these two functions in utils/TypePrinter.h -A quick example, in Types.h we add: +A quick example, in CoreTypes.h we add: -@snippet arm_compute/core/Types.h DataLayout enum definition +@snippet arm_compute/core/CoreTypes.h DataLayout enum definition And for printing: diff --git a/docs/user_guide/conv2d_heuristic.dox b/docs/user_guide/conv2d_heuristic.dox new file mode 100644 index 0000000000000000000000000000000000000000..edd24a3d36012422949c04fcfbaafda70f03dab3 --- /dev/null +++ b/docs/user_guide/conv2d_heuristic.dox @@ -0,0 +1,89 @@ +/// +/// Copyright (c) 2023 Arm Limited. +/// +/// SPDX-License-Identifier: MIT +/// +/// Permission is hereby granted, free of charge, to any person obtaining a copy +/// of this software and associated documentation files (the "Software"), to +/// deal in the Software without restriction, including without limitation the +/// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +/// sell copies of the Software, and to permit persons to whom the Software is +/// furnished to do so, subject to the following conditions: +/// +/// The above copyright notice and this permission notice shall be included in all +/// copies or substantial portions of the Software. +/// +/// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +/// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +/// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +/// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +/// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +/// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +/// SOFTWARE. +/// + +namespace arm_compute +{ +/** +@page conv2d_heuristic Convolution 2D heuristic + +@section conv2d_heuristic_algorithms_used Convolution 2D heuristic: algorithm selection + +The convolution 2D (in short, conv2D) is certainly one of the most compute intensive and performance critical operators in ML workloads. +This operator can be implemented with different algorithms, which differ in terms of accuracy, kernel size support, and additional memory required. +Unfortunately, it does not exist a single algorithm that can be used in all scenarios to achieve the best performance. +Therefore, the Arm Compute Library integrates an heuristic within the conv2d operators to select the most efficient algorithm, depending on input and kernel shapes and desired level of accuracy. +The heuristic depends on the target backend (either NEON™ for Arm® CPUs or OpenCL for Arm® GPUs) and the following subsections will provide the main details behind the selection of the algorithm. + +⚠ Attention: The heuristics presented in the following subsections will only refer to the NHWC data layout, which is the optimal and recommended layout for the Arm Compute Library. + +@subsection conv2d_heuristic_on_cpu Convolution 2D heuristic: Arm® Cortex®-based CPUs + +The conv2d heuristic for Arm® Cortex®-based CPUs is inside the get_convolution_method() method in the CpuConv2d function. +The algorithms used in the get_convolution_method() function are the following: +- Direct-Conv2D +- Im2Col+GeMM-based +- Indirect-GeMM (a.k.a. GEMMCONV2D) +- GeMM +- Winograd + +⚠ Attention: Winograd only works with floating-point data types (F32, F16) + +The heuristic first checks less frequent cases that we may have in ML workloads for edge devices. These cases are the following: +-# Non unit dilation: We call Im2Col+GeMM +-# Large input and kernel shapes: We call Direct-Conv2D because it is the only algorithm that does not extra additionally temporary memory +-# Small Input-Feature-Maps (IFM): In this scenario, we have found that the GeMM implementation is generally the most efficient algorithm compared to Winograd and Indirect-GeMM + +If we have a most frequent case, such as unit dilations, of larger IFM, we evaluate the following conditions instead: +-# Unit kernel size (1x1): In this scenario, the conv2d operations corresponds to a matrix multiplication and we call GeMM. +-# Winograd. Winograd only works with unit strides and supports a limited number of kernel sizes, such as 3x3, 3x1, 1x3, 5x1, 1x5 and 5x5 +-# Indirect-GeMM: It should be used in all cases expect when the kernel size is 1x1 or when the IFM is small + +If the preceding cases are not met, we will fall-back to the Im2Col+GeMM-based algorithm. + +@subsection conv2d_heuristic_on_gpu Convolution 2D heuristic: Arm® Mali™-based GPUs + +The conv2d heuristic for Arm® Mali™-based GPUs is inside the get_convolution_method() method in the ClConv2d function. + +The algorithms used in the get_convolution_method() function are the following: +- Direct-Conv2D +- Im2Col+GeMM-based +- Indirect-GeMM +- GeMM +- Winograd + +⚠ Attention: Winograd only works with floating-point data types (F32, F16) + +The heuristic first checks less frequent cases that we may have in ML workloads for edge devices. These cases are the following: +-# Non unit dilation: We call Im2Col+GeMM +-# Large input and kernel shapes: We call Direct-Conv2D because it is the only algorithm that does not extra additionally temporary memory + +In all the other cases, the GPU heuristic evaluates the suitability of Winograd and Direct-Conv2D/Indirect-Conv2D. +In particular, Winograd is adopted when the convolution parameters (kernel size and strides) are supported by the algorithm and when the IFM is not small (for example, greater than 8). +The conditions for using the Direct-Conv2D algorithms are several and we recommend you look at the heuristic directly. +In general, the Direct-Conv2D operators is used in almost all cases where kernel size is not 1x1. +The Indirect-GeMM algorithm is used in alternative to Direct-Conv2D only for Arm® Mali™-G77 GPU. +If neither Winograd nor Direct-Conv2D can be used, we will fall-back to either GeMM (when the kernel size is 1x1) or the Im2Col+GeMM-based algorithm. + +*/ +} // namespace diff --git a/docs/user_guide/errata.dox b/docs/user_guide/errata.dox index 78dd232c3f9e9d384c35e55c7ba7b1579de6953e..525ad3e396a61252e70c99438c6e19744a6a9f68 100644 --- a/docs/user_guide/errata.dox +++ b/docs/user_guide/errata.dox @@ -30,8 +30,17 @@ namespace arm_compute @section S7_1_errata Errata +- (COMPMID-6404) Under certain conditions, CLTile may produce incorrect result. + - Versions: >= v19.02 && < v23.08 + - OSs: Linux, Android. + - Conditions: + - The size of the lowest dimension of the input tensor is greater than 16 bytes. + - The size of the lowest dimension of the input tensor is not a multiple of 16. + - Result: + - Incorrect result is produced. + - (COMPMID-6271) Under certain conditions, CLArgMinMaxLayer validation tests may fail - - Versions Affected: >= v20.02 + - Versions Affected: >= v20.02 && < v23.08 - OSs Affected: Linux - Conditions: - Backend: OpenCL diff --git a/docs/user_guide/introduction.dox b/docs/user_guide/introduction.dox index be4f5c8605da4096c09c393fdb6d17d7932073cf..094a4480a04382b4e036b3bc9d7d5d0adc41aea6 100644 --- a/docs/user_guide/introduction.dox +++ b/docs/user_guide/introduction.dox @@ -34,7 +34,7 @@ namespace arm_compute The Compute Library is a collection of low-level machine learning functions optimized for both Arm CPUs and GPUs using SIMD technologies. Several builds of the library are available using various configurations: - - OS: Linux, Android, macOS or bare metal. + - OS: Linux®, Android™, macOS or bare metal. - Architecture: armv7a (32bit) or armv8a (64bit). - Technology: Arm® Neon™ / OpenCL / Arm® Neon™ and OpenCL. - Debug / Asserts / Release: Use a build with asserts enabled to debug your application and enable extra validation. Once you are sure your application works as expected you can switch to a release build of the library for maximum performance. @@ -49,7 +49,7 @@ Several builds of the library are available using various configurations: Architecture Minimum Toolchain - Linux + Linux® armv7a gcc-linaro-6.3.1-2017.05-x86_64_arm-linux-gnueabihf @@ -61,11 +61,14 @@ Several builds of the library are available using various configurations: armv8.2-a-sve gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu - Android + Android™ armv8a NDK r20b armv8.2-a + + armv8.2-a-sve + NDK r23b macOS armv8.2-a @@ -86,13 +89,14 @@ In order to facilitate the work of the support team please provide the build inf For each release we provide some pre-built binaries of the library [here](https://github.com/ARM-software/ComputeLibrary/releases). These binaries have been built using the following toolchains: - - Linux armv7a: gcc-linaro-7.2.1-2017.11-x86_64_arm-linux-gnueabihf - - Linux armv8a: gcc-linaro-7.2.1-2017.11-x86_64_aarch64-linux-gnu - - Linux armv8.2-a: gcc-linaro-7.2.1-2017.11-x86_64_aarch64-linux-gnu - - Linux armv8.2-a (multi-ISA binary): gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu - - Linux armv8.2-a-sve: gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu - - Android armv8a: clang++ / libc++ NDK r20b - - Android armv8.2-a: clang++ / libc++ NDK r20b + - Linux® armv7a: gcc-linaro-7.2.1-2017.11-x86_64_arm-linux-gnueabihf + - Linux® armv8a: gcc-linaro-7.2.1-2017.11-x86_64_aarch64-linux-gnu + - Linux® armv8.2-a: gcc-linaro-7.2.1-2017.11-x86_64_aarch64-linux-gnu + - Linux® armv8.2-a (multi-ISA binary): gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu + - Linux® armv8.2-a-sve: gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu + - Android™ armv8a: clang++ / libc++ NDK r20b + - Android™ armv8.2-a: clang++ / libc++ NDK r20b + - Android™ armv8.2-a-sve: clang++ / libc++ NDK r23b @warning Make sure to use a compatible toolchain to build your application or you will get some std::bad_alloc errors at runtime. diff --git a/docs/user_guide/library.dox b/docs/user_guide/library.dox index 11948d7136a2f240317b137ec89f312d94296517..4b54abed1f64c94ffbf2c3a37e07ae81c7dbcfb2 100644 --- a/docs/user_guide/library.dox +++ b/docs/user_guide/library.dox @@ -1,5 +1,5 @@ /// -/// Copyright (c) 2017-2021 Arm Limited. +/// Copyright (c) 2017-2021, 2023 Arm Limited. /// /// SPDX-License-Identifier: MIT /// @@ -28,20 +28,18 @@ namespace arm_compute @tableofcontents -@section architecture_core_vs_runtime Core vs Runtime libraries +@section architecture_compute_library Compute Library architecture -The Core library is a low level collection of algorithms implementations, it is designed to be embedded in existing projects and applications: +The Compute Library is a collection of low level algorithm implementations known as kernels @ref IKernel. +These kernels are implemented as operators @ref IOperator that do not allocate any memory (i.e. all the memory allocations/mappings have to be handled by the caller) +and are are designed to be embedded in existing projects and applications. -- It doesn't allocate any memory (All the memory allocations/mappings have to be handled by the caller). -- It doesn't perform any kind of multi-threading (but provide information to the caller about how the workload can be split). +A higher-level interface wraps the operators into functions @ref IFunction that: +- Performs memory allocation of images and tensors through the use of standard malloc(). +- Enables multi-threading of Arm® Neon™ code in a very basic way using a very simple pool of threads. +- For OpenCL, uses the default CLScheduler command queue for all mapping operations and kernels. -The Runtime library is a very basic wrapper around the Core library which can be used for quick prototyping, it is basic in the sense that: - -- It allocates images and tensors by using standard malloc(). -- It multi-threads Arm® Neon™ code in a very basic way using a very simple pool of threads. -- For OpenCL it uses the default CLScheduler command queue for all mapping operations and kernels. - -For maximum performance, it is expected that the users would re-implement an equivalent to the runtime library which suits better their needs (With a more clever multi-threading strategy, load-balancing between Arm® Neon™ and OpenCL, etc.) +For maximum performance, it is expected that the users would re-implement an equivalent to the function interface which suits better their needs (With a more clever multi-threading strategy, load-balancing between Arm® Neon™ and OpenCL, etc.) @section architecture_fast_math Fast-math support diff --git a/docs/user_guide/operator_list.dox b/docs/user_guide/operator_list.dox index 8d34a763a5bea78d47386106f28da913e4e372b5..66b8988d29528c669acdb00cd9fd2520eecf8b7d 100644 --- a/docs/user_guide/operator_list.dox +++ b/docs/user_guide/operator_list.dox @@ -437,12 +437,15 @@ where N = batches, C = channels, H = height, W = width, D = depth
srcdst
U8S8, U16, S16, U32, S32, F16, F32 +
S8U8, U16, S16, U32, S32, F16, F32
U16U8, S8, S16, U32, S32, F16, F32
S16U8, S8, U16, U32, S32, F16, F32
U32U8, S8, U16, S16, S32, F16, F32
S32U8, S8, U16, S16, U32, F16, F32 -
F16U8, S8, U16, S16, U32, F32 -
F32U8, S8, U16, S16, U32, F16 +
U64U8, S8, U16, S16, U32, S32, F16, F32 +
S64U8, S8, U16, S16, U32, S32, F16, F32 +
F16U8, S8, U16, S16, S32, U32, F32 +
F32U8, S8, U16, S16, S32, U32, F16
ChannelShuffleLayer diff --git a/docs/user_guide/release_version_and_change_log.dox b/docs/user_guide/release_version_and_change_log.dox index 362049d8237a1998f4e36bbea930cc0c81573090..348fc1778c599ccd2620ec57a0b13eab3c8a782b 100644 --- a/docs/user_guide/release_version_and_change_log.dox +++ b/docs/user_guide/release_version_and_change_log.dox @@ -40,6 +40,32 @@ If there is more than one release in a month then an extra sequential number is @note Starting from release 22.05, 'master' branch is no longer being used, it has been replaced by 'main'. Please update your clone jobs accordingly. @section S2_2_changelog Changelog + +v23.08 Public major release + - Deprecate the legacy 'libarm_compute_core' library. This library is an artifact of Compute Library's legacy library architecture and no longer serves any purpose. + Users must no longer link their applications to this library and instead link only to the main `libarm_compute` library for core functionality. + - New features + - Rewrite CLArgMinMaxLayer for axis 0 and enable S64 output. + - Add multi-sketch support for dynamic fusion. + - Break up arm_compute/core/Types.h and utils/Utils.h a bit to reduce unused code in each inclusion of these headers. + - Add Fused Activation to CLMatMul. + - Implement FP32/FP16 @ref opencl::kernels::ClMatMulNativeMMULKernel using the MMUL extension. + - Use MatMul in fully connected layer with dynamic weights when supported. + - Optimize CPU depthwise convolution with channel multiplier. + - Add support in CpuCastKernel for conversion of S64/U64 to F32. + - Add new OpenCL™ kernels: + - @ref opencl::kernels::ClMatMulNativeMMULKernel support for FP32 and FP16, with batch support + - Enable transposed convolution with non-square kernels on CPU and GPU. + - Add support for input data type U64/S64 in CLCast. + - Add new Compute Kernel Writer (CKW) subproject that offers a C++ interface to generate tile-based OpenCL code in just-in-time fashion. + - Port the following kernels in the experimental Dynamic Fusion interface to use the new Compute Kernel Writer interface with support for FP16/FP32 only: + - @ref experimental::dynamic_fusion::GpuCkwActivation + - @ref experimental::dynamic_fusion::GpuCkwCast + - @ref experimental::dynamic_fusion::GpuCkwDirectConv2d + - @ref experimental::dynamic_fusion::GpuCkwElementwiseBinary + - @ref experimental::dynamic_fusion::GpuCkwStore + - Various optimizations and bug fixes. + v23.05.1 Public patch release - Enable CMake and Bazel option to build multi_isa without FP16 support. - Fix compilation error in NEReorderLayer (aarch64 only). diff --git a/filelist.json b/filelist.json index c12873ad63c5defe614808161eb8e3b6e6072767..953b81de5a1c32631b346466af7574390bcfa477 100644 --- a/filelist.json +++ b/filelist.json @@ -27,8 +27,14 @@ "src/core/Version.cpp", "src/core/helpers/SoftmaxHelpers.cpp", "src/core/helpers/WindowHelpers.cpp", + "src/core/utils/ActivationFunctionUtils.cpp", "src/core/utils/AssemblyUtils.cpp", + "src/core/utils/DataTypeUtils.cpp", + "src/core/utils/DataLayoutUtils.cpp", + "src/core/utils/InterpolationPolicyUtils.cpp", + "src/core/utils/FormatUtils.cpp", "src/core/utils/ScaleUtils.cpp", + "src/core/utils/StringUtils.cpp", "src/core/utils/helpers/fft.cpp", "src/core/utils/helpers/tensor_transform.cpp", "src/core/utils/io/FileHandler.cpp", @@ -506,9 +512,7 @@ "src/runtime/heuristics/dwc_native/ClDWCNativeDefaultConfigBifrost.cpp", "src/runtime/heuristics/dwc_native/ClDWCNativeDefaultConfigValhall.cpp", "src/runtime/heuristics/dwc_native/ClDWCNativeHeuristicsHelpers.cpp", - "src/runtime/heuristics/indirect_conv/ClIndirectConvDefaultConfigValhall.cpp", - "src/runtime/heuristics/matmul_native/ClMatMulNativeDefaultConfigValhall.cpp", - "src/runtime/heuristics/matmul_native/ClMatMulNativeHelpers.cpp" + "src/runtime/heuristics/indirect_conv/ClIndirectConvDefaultConfigValhall.cpp" ] } }, @@ -517,8 +521,11 @@ "common": [ "src/gpu/cl/kernels/ClMatMulLowpNativeKernel.cpp", "src/gpu/cl/kernels/ClMatMulNativeKernel.cpp", + "src/gpu/cl/kernels/ClMatMulNativeMMULKernel.cpp", "src/gpu/cl/operators/ClMatMul.cpp", - "src/runtime/CL/functions/CLMatMul.cpp" + "src/runtime/CL/functions/CLMatMul.cpp", + "src/runtime/heuristics/matmul_native/ClMatMulNativeDefaultConfigValhall.cpp", + "src/runtime/heuristics/matmul_native/ClMatMulNativeHelpers.cpp" ] } }, @@ -1223,7 +1230,6 @@ "src/core/NEON/kernels/arm_conv/depthwise/depthwise_strategies_common.cpp", "src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8q.cpp", "src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8s8u8q.cpp", - "src/core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.cpp", "src/core/NEON/kernels/arm_conv/depthwise/depthwise_s8q.cpp", "src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp32.cpp", "src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp", @@ -1282,6 +1288,7 @@ "src/core/NEON/kernels/arm_conv/depthwise/interleaves/a64_u8q_3x3_dot.cpp", "src/core/NEON/kernels/arm_conv/depthwise/interleaves/generic.cpp", "src/core/NEON/kernels/arm_conv/depthwise/interleaves/generic_quantized_dot_product.cpp", + "src/core/NEON/kernels/arm_conv/depthwise/premultiply.cpp", "src/cpu/kernels/depthwiseconv2d/generic/neon/impl.cpp" ], "fp16":["src/cpu/kernels/depthwiseconv2d/generic/neon/fp16.cpp"], @@ -1291,7 +1298,6 @@ }, "sve": { "common": [ - "src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_8b_mla.cpp", "src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_s8q_3x3_dot.cpp", "src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_u8q_3x3_dot.cpp", "src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp", @@ -2280,64 +2286,81 @@ } }, "experimental": { - "dynamic_fusion": [ - "src/dynamic_fusion/runtime/gpu/cl/ClKernelRuntime.cpp", - "src/dynamic_fusion/runtime/gpu/cl/ClWorkloadRuntime.cpp", - "src/dynamic_fusion/sketch/attributes/CastAttributes.cpp", - "src/dynamic_fusion/sketch/attributes/ClampAttributes.cpp", - "src/dynamic_fusion/sketch/attributes/Conv2dAttributes.cpp", - "src/dynamic_fusion/sketch/attributes/DepthwiseConv2dAttributes.cpp", - "src/dynamic_fusion/sketch/attributes/Pool2dAttributes.cpp", - "src/dynamic_fusion/sketch/attributes/ResizeAttributes.cpp", - "src/dynamic_fusion/sketch/attributes/SoftmaxAttributes.cpp", - "src/dynamic_fusion/sketch/attributes/ReshapeAttributes.cpp", - "src/dynamic_fusion/sketch/gpu/GpuKernelArgument.cpp", - "src/dynamic_fusion/sketch/gpu/GpuKernelComponentGraph.cpp", - "src/dynamic_fusion/sketch/gpu/GpuKernelComponentGroup.cpp", - "src/dynamic_fusion/sketch/gpu/GpuKernelComponentStream.cpp", - "src/dynamic_fusion/sketch/gpu/GpuLogicalKernel.cpp", - "src/dynamic_fusion/sketch/gpu/GpuOperatorGroup.cpp", - "src/dynamic_fusion/sketch/gpu/GpuWorkloadContext.cpp", - "src/dynamic_fusion/sketch/gpu/GpuWorkloadSketch.cpp", - "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentActivation.cpp", - "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentCast.cpp", - "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentDepthwiseConv2d.cpp", - "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentDirectConv2d.cpp", - "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentPool2d.cpp", - "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentElementwiseBinary.cpp", - "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentLogits1DMaxShiftExpSum.cpp", - "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentLogits1DNorm.cpp", - "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentReshape.cpp", - "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentResize.cpp", - "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentStore.cpp", - "src/dynamic_fusion/sketch/gpu/operators/GpuAdd.cpp", - "src/dynamic_fusion/sketch/gpu/operators/GpuCast.cpp", - "src/dynamic_fusion/sketch/gpu/operators/GpuClamp.cpp", - "src/dynamic_fusion/sketch/gpu/operators/GpuConv2d.cpp", - "src/dynamic_fusion/sketch/gpu/operators/GpuDepthwiseConv2d.cpp", - "src/dynamic_fusion/sketch/gpu/operators/GpuMul.cpp", - "src/dynamic_fusion/sketch/gpu/operators/GpuReshape.cpp", - "src/dynamic_fusion/sketch/gpu/operators/GpuPool2d.cpp", - "src/dynamic_fusion/sketch/gpu/operators/GpuOutput.cpp", - "src/dynamic_fusion/sketch/gpu/operators/GpuResize.cpp", - "src/dynamic_fusion/sketch/gpu/operators/GpuSigmoid.cpp", - "src/dynamic_fusion/sketch/gpu/operators/GpuSoftmax.cpp", - "src/dynamic_fusion/sketch/gpu/operators/GpuSub.cpp", - "src/dynamic_fusion/sketch/gpu/operators/GpuTanh.cpp", - "src/dynamic_fusion/sketch/gpu/operators/internal/GpuElementwiseBinaryCommon.cpp", - "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateActivation.cpp", - "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateCast.cpp", - "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateDepthwiseConv2d.cpp", - "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateDirectConv2d.cpp", - "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplatePool2d.cpp", - "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateElementwiseBinary.cpp", - "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateLogits1DMaxShiftExpSum.cpp", - "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateLogits1DNorm.cpp", - "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateReshape.cpp", - "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateResize.cpp", - "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateStore.cpp", - "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateWriter.cpp", - "src/dynamic_fusion/sketch/gpu/template_writer/GpuKernelVariableTable.cpp" - ] + "dynamic_fusion": { + "common": [ + "src/dynamic_fusion/runtime/gpu/cl/ClKernelRuntime.cpp", + "src/dynamic_fusion/runtime/gpu/cl/ClWorkloadRuntime.cpp", + "src/dynamic_fusion/sketch/attributes/CastAttributes.cpp", + "src/dynamic_fusion/sketch/attributes/ClampAttributes.cpp", + "src/dynamic_fusion/sketch/attributes/Conv2dAttributes.cpp", + "src/dynamic_fusion/sketch/attributes/DepthwiseConv2dAttributes.cpp", + "src/dynamic_fusion/sketch/attributes/Pool2dAttributes.cpp", + "src/dynamic_fusion/sketch/attributes/ResizeAttributes.cpp", + "src/dynamic_fusion/sketch/attributes/SoftmaxAttributes.cpp", + "src/dynamic_fusion/sketch/attributes/ReshapeAttributes.cpp", + "src/dynamic_fusion/sketch/gpu/GpuKernelArgument.cpp", + "src/dynamic_fusion/sketch/gpu/GpuKernelComponentGraph.cpp", + "src/dynamic_fusion/sketch/gpu/GpuKernelComponentGroup.cpp", + "src/dynamic_fusion/sketch/gpu/GpuKernelComponentStream.cpp", + "src/dynamic_fusion/sketch/gpu/GpuLogicalKernel.cpp", + "src/dynamic_fusion/sketch/gpu/GpuOperatorGroup.cpp", + "src/dynamic_fusion/sketch/gpu/GpuWorkloadContext.cpp", + "src/dynamic_fusion/sketch/gpu/GpuWorkloadSketch.cpp", + "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentActivation.cpp", + "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentCast.cpp", + "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentDepthwiseConv2d.cpp", + "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentDirectConv2d.cpp", + "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentPool2d.cpp", + "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentElementwiseBinary.cpp", + "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentLogits1DMaxShiftExpSum.cpp", + "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentLogits1DNorm.cpp", + "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentReshape.cpp", + "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentResize.cpp", + "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentStore.cpp", + "src/dynamic_fusion/sketch/gpu/operators/GpuAdd.cpp", + "src/dynamic_fusion/sketch/gpu/operators/GpuCast.cpp", + "src/dynamic_fusion/sketch/gpu/operators/GpuClamp.cpp", + "src/dynamic_fusion/sketch/gpu/operators/GpuConv2d.cpp", + "src/dynamic_fusion/sketch/gpu/operators/GpuDepthwiseConv2d.cpp", + "src/dynamic_fusion/sketch/gpu/operators/GpuMul.cpp", + "src/dynamic_fusion/sketch/gpu/operators/GpuReshape.cpp", + "src/dynamic_fusion/sketch/gpu/operators/GpuPool2d.cpp", + "src/dynamic_fusion/sketch/gpu/operators/GpuOutput.cpp", + "src/dynamic_fusion/sketch/gpu/operators/GpuResize.cpp", + "src/dynamic_fusion/sketch/gpu/operators/GpuSigmoid.cpp", + "src/dynamic_fusion/sketch/gpu/operators/GpuSoftmax.cpp", + "src/dynamic_fusion/sketch/gpu/operators/GpuSub.cpp", + "src/dynamic_fusion/sketch/gpu/operators/GpuTanh.cpp", + "src/dynamic_fusion/sketch/gpu/operators/internal/GpuElementwiseBinaryCommon.cpp" + ], + "template_writer": [ + "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateActivation.cpp", + "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateCast.cpp", + "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateDepthwiseConv2d.cpp", + "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateDirectConv2d.cpp", + "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplatePool2d.cpp", + "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateElementwiseBinary.cpp", + "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateLogits1DMaxShiftExpSum.cpp", + "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateLogits1DNorm.cpp", + "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateReshape.cpp", + "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateResize.cpp", + "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateStore.cpp", + "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateWriter.cpp", + "src/dynamic_fusion/sketch/gpu/template_writer/GpuKernelVariableTable.cpp" + ], + "ckw_driver": [ + "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwComponentArgument.cpp", + "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwDriver.cpp", + "src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwActivation.cpp", + "src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwCast.cpp", + "src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwDirectConv2d.cpp", + "src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwElementwiseBinary.cpp", + "src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwStore.cpp", + "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwKernelWriter.cpp", + "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwScopedKernelWriter.cpp", + "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwVariableTable.cpp", + "src/dynamic_fusion/runtime/gpu/cl/ckw_driver/GpuCkwKernelArgumentsHelpers.cpp" + ] + } } } diff --git a/scripts/arm_compute_library_nn_driver.go b/scripts/arm_compute_library_nn_driver.go index 73a5ce43aa4ac0e2df1c018604347485a1a5e41d..dda77b55df75fff8791d8297d17514d5ba738e78 100644 --- a/scripts/arm_compute_library_nn_driver.go +++ b/scripts/arm_compute_library_nn_driver.go @@ -1,5 +1,5 @@ // -// Copyright © 2020-2021 Arm Ltd. All rights reserved. +// Copyright © 2020-2023 Arm Ltd. All rights reserved. // SPDX-License-Identifier: MIT // @@ -22,6 +22,7 @@ func isVersionAtLeast(version_name string, target_version int) bool { "R": 11, "11": 11, "S": 12, "12": 12, "T": 13, "13": 13, + "U": 14, "14": 14, } if _, ok := name_map[version_name]; ok { return name_map[version_name] >= target_version diff --git a/scripts/clang_tidy_rules.py b/scripts/clang_tidy_rules.py index 2e0b32e6ccddce3d00d40ed1057f0bc07eaa057a..1e1ab7f545783de5e321af9e029de6d462592f00 100755 --- a/scripts/clang_tidy_rules.py +++ b/scripts/clang_tidy_rules.py @@ -28,7 +28,8 @@ import re import sys def get_list_includes(): - return "src/cpu/kernels/assembly " \ + return "compute_kernel_writer/prototype/include " \ + "src/cpu/kernels/assembly " \ "src/core/NEON/kernels/assembly " \ "src/core/NEON/kernels/convolution/winograd " \ "include/linux include " \ @@ -42,6 +43,9 @@ def get_list_flags( filename, arch): flags.append("-DARM_COMPUTE_OPENCL_ENABLED") if arch == "aarch64": flags.append("-DARM_COMPUTE_AARCH64_V8_2") + if "ckw_driver" in filename: + flags.append("-DACL_INTERNAL_TEST_CKW_IN_DF") + return flags def filter_files( list_files ): @@ -70,6 +74,9 @@ def filter_clang_tidy_lines( lines ): if "/arm_gemm/" in line: continue + if "compute_kernel_writer/" in line: + continue + if "/convolution/" in line: continue diff --git a/src/BUILD.bazel b/src/BUILD.bazel index bf71e534e260816e422f6a1cb3cd945317e7cfb1..ca90107a5c99a1d88c1b6f0e8d0224522ac96a47 100644 --- a/src/BUILD.bazel +++ b/src/BUILD.bazel @@ -129,8 +129,7 @@ filegroup( filegroup( name = "arm_compute_sve_srcs", - srcs = ["core/NEON/kernels/arm_conv/depthwise/interleaves/sve_8b_mla.cpp", - "core/NEON/kernels/arm_conv/depthwise/interleaves/sve_s8q_3x3_dot.cpp", + srcs = ["core/NEON/kernels/arm_conv/depthwise/interleaves/sve_s8q_3x3_dot.cpp", "core/NEON/kernels/arm_conv/depthwise/interleaves/sve_u8q_3x3_dot.cpp", "core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp", "core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp", @@ -433,7 +432,6 @@ filegroup( "core/NEON/kernels/arm_conv/depthwise/depthwise_strategies_common.cpp", "core/NEON/kernels/arm_conv/depthwise/depthwise_u8q.cpp", "core/NEON/kernels/arm_conv/depthwise/depthwise_u8s8u8q.cpp", - "core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.cpp", "core/NEON/kernels/arm_conv/depthwise/interleaves/a64_s8q_3x3_dot.cpp", "core/NEON/kernels/arm_conv/depthwise/interleaves/a64_u8q_3x3_dot.cpp", "core/NEON/kernels/arm_conv/depthwise/interleaves/generic.cpp", @@ -489,6 +487,7 @@ filegroup( "core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp", "core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst/generic.cpp", "core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp", + "core/NEON/kernels/arm_conv/depthwise/premultiply.cpp", "core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp", "core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_generic_depthfirst/generic.cpp", "core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp", @@ -646,7 +645,12 @@ filegroup( "core/Version.cpp", "core/helpers/SoftmaxHelpers.cpp", "core/helpers/WindowHelpers.cpp", + "core/utils/ActivationFunctionUtils.cpp", "core/utils/AssemblyUtils.cpp", + "core/utils/DataLayoutUtils.cpp", + "core/utils/DataTypeUtils.cpp", + "core/utils/FormatUtils.cpp", + "core/utils/InterpolationPolicyUtils.cpp", "core/utils/ScaleUtils.cpp", "core/utils/helpers/fft.cpp", "core/utils/helpers/tensor_transform.cpp", @@ -657,6 +661,7 @@ filegroup( "core/utils/logging/LoggerRegistry.cpp", "core/utils/misc/MMappedFile.cpp", "core/utils/quantization/AsymmHelpers.cpp", + "core/utils/StringUtils.cpp", "cpu/CpuContext.cpp", "cpu/CpuQueue.cpp", "cpu/CpuTensor.cpp", diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ed660e8aac4e1c45c88b4aacfb3be3aff696e673..20379dd5f484fd5f4c6d1c3293990f9ed7aeba8a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -105,8 +105,7 @@ target_sources( target_sources( arm_compute_sve PRIVATE - core/NEON/kernels/arm_conv/depthwise/interleaves/sve_8b_mla.cpp - core/NEON/kernels/arm_conv/depthwise/interleaves/sve_s8q_3x3_dot.cpp + core/NEON/kernels/arm_conv/depthwise/interleaves/sve_s8q_3x3_dot.cpp core/NEON/kernels/arm_conv/depthwise/interleaves/sve_u8q_3x3_dot.cpp core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -425,7 +424,6 @@ target_sources( core/NEON/kernels/arm_conv/depthwise/depthwise_strategies_common.cpp core/NEON/kernels/arm_conv/depthwise/depthwise_u8q.cpp core/NEON/kernels/arm_conv/depthwise/depthwise_u8s8u8q.cpp - core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.cpp core/NEON/kernels/arm_conv/depthwise/interleaves/a64_s8q_3x3_dot.cpp core/NEON/kernels/arm_conv/depthwise/interleaves/a64_u8q_3x3_dot.cpp core/NEON/kernels/arm_conv/depthwise/interleaves/generic.cpp @@ -481,6 +479,7 @@ target_sources( core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst/generic.cpp core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp + core/NEON/kernels/arm_conv/depthwise/premultiply.cpp core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_generic_depthfirst/generic.cpp core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp @@ -638,8 +637,14 @@ target_sources( core/Version.cpp core/helpers/SoftmaxHelpers.cpp core/helpers/WindowHelpers.cpp + core/utils/ActivationFunctionUtils.cpp core/utils/AssemblyUtils.cpp + core/utils/DataLayoutUtils.cpp + core/utils/DataTypeUtils.cpp + core/utils/FormatUtils.cpp + core/utils/InterpolationPolicyUtils.cpp core/utils/ScaleUtils.cpp + core/utils/StringUtils.cpp core/utils/helpers/fft.cpp core/utils/helpers/tensor_transform.cpp core/utils/io/FileHandler.cpp diff --git a/src/c/AclContext.cpp b/src/c/AclContext.cpp index dbf2a3df88757e11abea577b04023a74e0699cf3..9b8ffea6198ee520d4617c0588fdf3b1ff28be78 100644 --- a/src/c/AclContext.cpp +++ b/src/c/AclContext.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021,2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,6 +23,8 @@ */ #include "arm_compute/AclEntrypoints.h" +#include "arm_compute/core/Error.h" + #include "src/common/IContext.h" #include "src/common/utils/Macros.h" #include "src/common/utils/Validate.h" diff --git a/src/c/AclTensor.cpp b/src/c/AclTensor.cpp index 8f6ce45628f81371d7d9385967e86c1964772eea..5b184697aa983e2dcf2e5ba16ee522e8f1804f3a 100644 --- a/src/c/AclTensor.cpp +++ b/src/c/AclTensor.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021,2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,6 +23,7 @@ */ #include "arm_compute/AclEntrypoints.h" #include "arm_compute/AclUtils.h" +#include "arm_compute/core/Error.h" #include "src/common/ITensorV2.h" #include "src/common/utils/Macros.h" diff --git a/src/common/IContext.h b/src/common/IContext.h index 1ae46c57de25785678247eb2eb1bcd78828875a2..65bb76744d800ba62fa5192c6702bcca04ef263a 100644 --- a/src/common/IContext.h +++ b/src/common/IContext.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021,2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -29,6 +29,7 @@ #include "src/common/utils/Object.h" #include +#include struct AclContext_ { diff --git a/src/common/utils/LegacySupport.cpp b/src/common/utils/LegacySupport.cpp index 662382512402483c434b97928f9c85ad59240c61..06b1693bd1cbd23b9cb9ba5730ffbda3db11978b 100644 --- a/src/common/utils/LegacySupport.cpp +++ b/src/common/utils/LegacySupport.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,6 +23,8 @@ */ #include "src/common/utils/LegacySupport.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" + namespace arm_compute { namespace detail diff --git a/src/common/utils/LegacySupport.h b/src/common/utils/LegacySupport.h index 3c3b09fce68052483e51673e794a61ac89eb9050..05a70fc2c6c1ddf8741e4749c678781f1859c45e 100644 --- a/src/common/utils/LegacySupport.h +++ b/src/common/utils/LegacySupport.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,6 +27,7 @@ #include "arm_compute/Acl.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" namespace arm_compute { diff --git a/src/common/utils/Log.h b/src/common/utils/Log.h index f3ae38a57c89b886127f1f6703850865cff215b5..bbfe1ce1b3def8ebfc03ceadfd11486460d554cb 100644 --- a/src/common/utils/Log.h +++ b/src/common/utils/Log.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021,2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,11 +24,22 @@ #ifndef SRC_COMMON_LOG_H #define SRC_COMMON_LOG_H +#ifndef ARM_COMPUTE_LOGGING_ENABLED + +#define ARM_COMPUTE_CREATE_ACL_LOGGER() +#define ARM_COMPUTE_LOG_MSG_ACL(log_level, msg) +#define ARM_COMPUTE_LOG_MSG_WITH_FORMAT_ACL(log_level, fmt, ...) +#define ARM_COMPUTE_LOG_ERROR_ACL(msg) +#define ARM_COMPUTE_LOG_ERROR_WITH_FUNCNAME_ACL(msg) +#define ARM_COMPUTE_LOG_INFO_WITH_FUNCNAME_ACL(msg) +#define ARM_COMPUTE_LOG_PARAMS(...) + +#else /* ARM_COMPUTE_LOGGING_ENABLED */ + #include "arm_compute/core/Error.h" #include "arm_compute/core/utils/logging/Macros.h" #include "utils/TypePrinter.h" -#ifdef ARM_COMPUTE_LOGGING_ENABLED /** Create a logger * * @note It will eventually create all default loggers in don't exist @@ -41,9 +52,6 @@ arm_compute::logging::LoggerRegistry::get().create_logger("ComputeLibrary", arm_compute::logging::LogLevel::INFO); \ } \ } while(false) -#else /* ARM_COMPUTE_LOGGING_ENABLED */ -#define ARM_COMPUTE_CREATE_ACL_LOGGER() -#endif /* ARM_COMPUTE_LOGGING_ENABLED */ /** Log a message to the logger * @@ -218,4 +226,5 @@ inline const std::string constructDataLog(const std::vector ¶ms ARM_COMPUTE_LOG_INFO_WITH_FUNCNAME_ACL(constructDataLog(getParamsNames(#__VA_ARGS__), \ logParams(__VA_ARGS__))); \ } while(false) +#endif /* ARM_COMPUTE_LOGGING_ENABLED */ #endif /* SRC_COMMON_LOG_H */ diff --git a/src/common/utils/Utils.h b/src/common/utils/Utils.h index 79f4f39c47ac3c131591bdec826c31f4cd437d6e..1bd1c7ec5726525334007fb5df243ab2eca0f675 100644 --- a/src/common/utils/Utils.h +++ b/src/common/utils/Utils.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021,2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,8 @@ #ifndef SRC_COMMON_UTILS_H #define SRC_COMMON_UTILS_H +#include +#include #include namespace arm_compute @@ -72,7 +74,7 @@ constexpr SE as_enum(const E val) noexcept template bool is_in(E check, std::initializer_list list) { - return std::any_of(std::cbegin(list), std::cend(list), [&check](E e) + return std::any_of(list.begin(), list.end(), [&check](E e) { return check == e; }); diff --git a/src/core/CL/CLCompileContext.cpp b/src/core/CL/CLCompileContext.cpp index ea03d59fc2ad53f876b5f44f09d3aa9f4a1b944c..2d024f9c2f76c114473e0c88711a18622d69a4cd 100644 --- a/src/core/CL/CLCompileContext.cpp +++ b/src/core/CL/CLCompileContext.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022 Arm Limited. + * Copyright (c) 2020-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -331,8 +331,8 @@ const cl::Device &CLCompileContext::get_device() const void CLCompileContext::set_device(cl::Device device) { - _device = std::move(device); _is_wbsm_supported = get_wbsm_support_info(device); + _device = std::move(device); } cl::NDRange CLCompileContext::default_ndrange() const diff --git a/src/core/CL/CLHelpers.cpp b/src/core/CL/CLHelpers.cpp index 6b011f1f7c8921a52c5dda7a6d7c336d9be22f93..a32bcca65522f336685d5ef4a70ef1e8f6619a7a 100644 --- a/src/core/CL/CLHelpers.cpp +++ b/src/core/CL/CLHelpers.cpp @@ -24,6 +24,7 @@ #include "arm_compute/core/CL/CLHelpers.h" #include "arm_compute/core/CL/CLKernelLibrary.h" #include "arm_compute/core/CL/CLTypes.h" +#include "arm_compute/core/utils/DataTypeUtils.h" #include "arm_compute/core/Error.h" #include "arm_compute/core/Log.h" #include "arm_compute/core/Types.h" @@ -144,7 +145,6 @@ std::string get_cl_select_type_from_data_type(const DataType &dt) { case DataType::U8: case DataType::QASYMM8: - return "uchar"; case DataType::S8: case DataType::QASYMM8_SIGNED: case DataType::QSYMM8: diff --git a/src/core/CL/CLUtils.cpp b/src/core/CL/CLUtils.cpp index 84cf88e09964f4589744153770e1266548566fe0..03f78697bc9cab661e9b1559dc6341c975f5160e 100644 --- a/src/core/CL/CLUtils.cpp +++ b/src/core/CL/CLUtils.cpp @@ -21,19 +21,54 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ +#include "src/core/CL/CLUtils.h" + +#include "arm_compute/core/utils/ActivationFunctionUtils.h" #include "arm_compute/core/CL/CLCompileContext.h" -#include "arm_compute/core/Error.h" -#include "arm_compute/core/Types.h" +#include "arm_compute/core/CL/CLKernelLibrary.h" +#include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/Validate.h" +#include "arm_compute/core/utils/StringUtils.h" #include "support/StringSupport.h" -#include "src/core/CL/CLUtils.h" #include "src/core/experimental/PostOpUtils.h" namespace arm_compute { -cl::Image2D create_image2d_from_buffer(const cl::Context &ctx, const cl::Buffer &buffer, const TensorShape &shape2d, DataType data_type, size_t image_row_pitch, CLImage2DType type) +cl::Image2D create_image2d_from_tensor(const ICLTensor *tensor, CLImage2DType image_type) +{ + ARM_COMPUTE_ERROR_ON_NULLPTR(tensor); + + const cl::Context &ctx = CLKernelLibrary::get().context(); + const cl::Buffer &buffer = tensor->cl_buffer(); + const ITensorInfo *info = tensor->info(); + ARM_COMPUTE_ERROR_ON_MSG(info->lock_paddings(), + "Tensor paddings must not be locked to allow extending paddings to satisfy cl_image pitch alignment requirement"); + + const size_t image_w{ info->dimension(0) / 4 }; + const size_t image_h{ info->tensor_shape().total_size() / info->dimension(0) }; + const size_t max_image_w{ CLKernelLibrary::get().get_device().getInfo() }; + const size_t max_image_h{ CLKernelLibrary::get().get_device().getInfo() }; + + ARM_COMPUTE_UNUSED(max_image_w, max_image_h); + ARM_COMPUTE_ERROR_ON_MSG(image_w > max_image_w, "Image width exceeds maximum width for exporting to cl_image"); + ARM_COMPUTE_ERROR_ON_MSG(image_h > max_image_h, "Image height exceeds maximum height for exporting to cl_image"); + + const TensorShape shape2d(image_w, image_h); + const size_t image_row_pitch = info->strides_in_bytes()[1]; + + return create_image2d_from_buffer(ctx, buffer, shape2d, info->data_type(), image_row_pitch, image_type); +} + +cl::Image2D create_image2d_from_buffer(const cl::Context &ctx, const cl::Buffer &buffer, const TensorShape &shape2d, DataType data_type, size_t image_row_pitch, CLImage2DType image_type) { + ARM_COMPUTE_ERROR_ON_MSG(!image2d_from_buffer_supported(CLKernelLibrary::get().get_device()), + "The extension cl_khr_image2d_from_buffer is not supported on the target platform"); + ARM_COMPUTE_ERROR_ON_MSG(get_cl_image_pitch_alignment(CLKernelLibrary::get().get_device()) == 0, + "Impossible to retrieve the cl_image pitch alignment"); + ARM_COMPUTE_ERROR_ON_MSG(buffer.get() == nullptr, + "Cannot create cl_image from empty cl_buffer"); + cl_channel_type cl_data_type; switch(data_type) @@ -61,7 +96,7 @@ cl::Image2D create_image2d_from_buffer(const cl::Context &ctx, const cl::Buffer desc.image_width = shape2d[0]; desc.image_height = shape2d[1]; - switch(type) + switch(image_type) { case CLImage2DType::ReadOnly: cl_image = clCreateImage(ctx(), CL_MEM_READ_ONLY, &format, &desc, nullptr, &err); diff --git a/src/core/CL/CLUtils.h b/src/core/CL/CLUtils.h index b31944c72f522cde294fffb944c74f50ce38d5ca..e3f12d4b53b88dffafa6fd8105d48ae6c6160a78 100644 --- a/src/core/CL/CLUtils.h +++ b/src/core/CL/CLUtils.h @@ -28,11 +28,15 @@ #include "arm_compute/core/CL/OpenCL.h" #include "arm_compute/core/experimental/IPostOp.h" +#include + namespace arm_compute { class TensorShape; class CLBuildOptions; class ITensorInfo; +class ICLTensor; +enum class DataType; /** OpenCL Image2D types */ enum class CLImage2DType @@ -41,6 +45,15 @@ enum class CLImage2DType WriteOnly }; +/** Create a cl::Image2D object from a tensor + * + * @param[in] tensor Tensor from which to construct Image 2D object + * @param[in] image_type Image 2D type (@ref CLImage2DType) + * + * @return cl::Image2D object + */ +cl::Image2D create_image2d_from_tensor(const ICLTensor *tensor, CLImage2DType image_type); + /** Create a cl::Image2D object from an OpenCL buffer * * @note The following conditions are required to create a OpenCL image object from OpenCL buffer, diff --git a/src/core/CL/cl_kernels/common/arg_min_max.cl b/src/core/CL/cl_kernels/common/arg_min_max.cl index 6e57ed0af1f4dbf5153cb9325a12e18c970041c1..413fcf53334acf782d013ef515a53628f5c85a09 100644 --- a/src/core/CL/cl_kernels/common/arg_min_max.cl +++ b/src/core/CL/cl_kernels/common/arg_min_max.cl @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,7 @@ * SOFTWARE. */ #include "helpers.h" +#include "tile_helpers.h" #if defined(VEC_SIZE) && defined(DATA_TYPE) && defined(DATA_TYPE_OUTPUT) @@ -52,246 +53,183 @@ #endif // defined(ARG_MAX) #if defined(WIDTH) -#if defined(ARG_MIN) -#if defined(PREV_OUTPUT) -/** Find index minimum value of a vector - * - * @param[in] input Pointer to the first value. - * - * @return index of the vector. - */ -inline DATA_TYPE_OUTPUT arg_idx_min_prev_out(__global const DATA_TYPE *input, __global const DATA_TYPE_OUTPUT *prev_res, const int x_idx) + +#if defined(ARG_MAX) +#define VECTOR_PREDICATE_EQ(x, y) ((x) >= (y)) +#define VECTOR_PREDICATE(x, y) ((x) > (y)) +#define SCALAR_SELECT_OP(x, y) ((x) > (y)) ? (x) : (y); +#elif defined(ARG_MIN) +#define VECTOR_PREDICATE_EQ(x, y) ((x) <= (y)) +#define VECTOR_PREDICATE(x, y) ((x) < (y)) +#define SCALAR_SELECT_OP(x, y) ((x) < (y)) ? (x) : (y); +#else // !(defined(ARG_MAX) || defined(ARG_MIN)) +#error "Unsupported reduction operation!" +#endif // defined(ARG_MAX) + +inline DATA_TYPE_OUTPUT vectorized_compute_arg_min_max_2(DATA_TYPE *min_max_val, DATA_TYPE_OUTPUT *min_max_idx, VEC_DATA_TYPE(DATA_TYPE, 2) in, VEC_DATA_TYPE(DATA_TYPE_OUTPUT, 2) res) { - int end_elem = (x_idx + 1) * 16; - if(end_elem > WIDTH) + if( VECTOR_PREDICATE_EQ(in.s0,in.s1) ) { - end_elem = WIDTH - x_idx * 16; + *min_max_val = in.s0; + *min_max_idx = res.s0; } - DATA_TYPE_OUTPUT res = prev_res[0]; - for(int x_v = 1; x_v < end_elem; ++x_v) + else { - res = select(res, prev_res[x_v], *(input + prev_res[x_v]) < * (input + res)); + *min_max_val = in.s1; + *min_max_idx = res.s1; } - return res; } -#else // !defined(PREV_OUTPUT) -/** Find index minimum value of a vector - * - * @param[in] input Pointer to the first value. - * - * @return index of the vector. - */ -inline DATA_TYPE_OUTPUT arg_idx_min(__global const DATA_TYPE *input, const int x_idx) + +inline DATA_TYPE_OUTPUT vectorized_compute_arg_min_max_4(DATA_TYPE *min_max_val, DATA_TYPE_OUTPUT *min_max_idx, VEC_DATA_TYPE(DATA_TYPE, 4) in, VEC_DATA_TYPE(DATA_TYPE_OUTPUT, 4) res) { -#if WIDTH < 16 - DATA_TYPE_OUTPUT res = 0; - for(DATA_TYPE_OUTPUT x_v = res + 1; x_v < WIDTH; ++x_v) - { - res = select(res, x_v, *(input + x_v) < * (input + res)); - } - return res; -#else // WIDTH >= 16 - int x_elem = x_idx * 16; - const int x_goback = select(0, 16 - WIDTH % 16, x_elem + 16 > WIDTH); - x_elem -= x_goback; - - VEC_DATA_TYPE(DATA_TYPE, 16) - in = vload16(0, input - x_goback); - VEC_DATA_TYPE(DATA_TYPE_OUTPUT, 16) - res = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }; - - SIGNED_INT_VEC_DATA_TYPE(DATA_TYPE, 8) - idx_sel = (in.s01234567 <= in.s89abcdef); - in.s01234567 = select(in.s89abcdef, in.s01234567, idx_sel); - res.s01234567 = select(res.s89abcdef, res.s01234567, CONVERT(idx_sel, int8)); + VEC_DATA_TYPE(COND_DATA_TYPE, 2) + idx_sel = VECTOR_PREDICATE_EQ(in.s01, in.s23); + in.s01 = select(in.s23, in.s01, idx_sel); + res.s01 = select(res.s23, res.s01, CONVERT(idx_sel, VEC_DATA_TYPE(DATA_TYPE_OUTPUT, 2) )); + idx_sel.s0 = VECTOR_PREDICATE(in.s0, in.s1) || (in.s0 == in.s1 && CONVERT((res.s0 < res.s1), COND_DATA_TYPE)); + res.s0 = select(res.s1, res.s0, CONVERT(idx_sel.s0, DATA_TYPE_OUTPUT)); + *min_max_val = SCALAR_SELECT_OP(in.s0, in.s1); + *min_max_idx = res.s0; +} - idx_sel.s0123 = (in.s0123 < in.s4567) || (in.s0123 == in.s4567 && CONVERT((res.s0123 < res.s4567), SIGNED_INT_VEC_DATA_TYPE(DATA_TYPE, 4))); +inline DATA_TYPE_OUTPUT vectorized_compute_arg_min_max_8(DATA_TYPE *min_max_val, DATA_TYPE_OUTPUT *min_max_idx, VEC_DATA_TYPE(DATA_TYPE, 8) in, VEC_DATA_TYPE(DATA_TYPE_OUTPUT, 8) res) +{ + VEC_DATA_TYPE(COND_DATA_TYPE, 4) + idx_sel = VECTOR_PREDICATE_EQ(in.s0123, in.s4567); + in.s0123 = select(in.s4567, in.s0123, idx_sel); + res.s0123 = select(res.s4567, res.s0123, CONVERT(idx_sel, VEC_DATA_TYPE(DATA_TYPE_OUTPUT, 4) )); + idx_sel.s01 = (VECTOR_PREDICATE(in.s01, in.s23)) || (in.s01 == in.s23 && CONVERT(((res.s01 < res.s23)), VEC_DATA_TYPE(COND_DATA_TYPE, 2))); + in.s01 = select(in.s23, in.s01, idx_sel.s01); + res.s01 = select(res.s23, res.s01, CONVERT(idx_sel.s01, VEC_DATA_TYPE(DATA_TYPE_OUTPUT, 2) )); + idx_sel.s0 = VECTOR_PREDICATE(in.s0, in.s1) || (in.s0 == in.s1 && CONVERT((res.s0 < res.s1), COND_DATA_TYPE)); + res.s0 = select(res.s1, res.s0, CONVERT(idx_sel.s0, DATA_TYPE_OUTPUT)); + *min_max_val = SCALAR_SELECT_OP(in.s0, in.s1); + *min_max_idx = res.s0; +} + +inline DATA_TYPE_OUTPUT vectorized_compute_arg_min_max_16(DATA_TYPE *min_max_val, DATA_TYPE_OUTPUT *min_max_idx, VEC_DATA_TYPE(DATA_TYPE, 16) in, VEC_DATA_TYPE(DATA_TYPE_OUTPUT, 16) res) +{ + VEC_DATA_TYPE(COND_DATA_TYPE, 8) + idx_sel = VECTOR_PREDICATE_EQ(in.s01234567, in.s89abcdef); + in.s01234567 = select(in.s89abcdef, in.s01234567, idx_sel); + res.s01234567 = select(res.s89abcdef, res.s01234567, CONVERT(idx_sel, VEC_DATA_TYPE(DATA_TYPE_OUTPUT, 8) )); + idx_sel.s0123 = VECTOR_PREDICATE(in.s0123, in.s4567) || (in.s0123 == in.s4567 && CONVERT(((res.s0123 < res.s4567)), VEC_DATA_TYPE(COND_DATA_TYPE, 4))); in.s0123 = select(in.s4567, in.s0123, idx_sel.s0123); - res.s0123 = select(res.s4567, res.s0123, CONVERT(idx_sel.s0123, int4)); + res.s0123 = select(res.s4567, res.s0123, CONVERT(idx_sel.s0123, VEC_DATA_TYPE(DATA_TYPE_OUTPUT, 4) )); + idx_sel.s01 = (VECTOR_PREDICATE(in.s01, in.s23)) || (in.s01 == in.s23 && CONVERT(((res.s01 < res.s23)), VEC_DATA_TYPE(COND_DATA_TYPE, 2))); + in.s01 = select(in.s23, in.s01, idx_sel.s01); + res.s01 = select(res.s23, res.s01, CONVERT(idx_sel.s01, VEC_DATA_TYPE(DATA_TYPE_OUTPUT, 2) )); + idx_sel.s0 = VECTOR_PREDICATE(in.s0, in.s1) || (in.s0 == in.s1 && CONVERT((res.s0 < res.s1), COND_DATA_TYPE)); + res.s0 = select(res.s1, res.s0, CONVERT(idx_sel.s0, DATA_TYPE_OUTPUT)); + *min_max_val = SCALAR_SELECT_OP(in.s0, in.s1); + *min_max_idx = res.s0; +} - idx_sel.s01 = (in.s01 < in.s23) || (in.s01 == in.s23 && CONVERT((res.s01 < res.s23), SIGNED_INT_VEC_DATA_TYPE(DATA_TYPE, 2))); - in.s01 = select(in.s23, in.s01, idx_sel.s01); - res.s01 = select(res.s23, res.s01, CONVERT(idx_sel.s01, int2)); - idx_sel.s0 = (in.s0 < in.s1) || (in.s0 == in.s1 && CONVERT((res.s0 < res.s1), SIGNED_INT_DATA_TYPE(DATA_TYPE))); - res.s0 = select(res.s1, res.s0, CONVERT(idx_sel.s0, int)); - return res.s0 + x_elem; -#endif // WIDTH < 16 -} -#endif // defined(PREV_OUTPUT) -#endif // defined(ARG_MIN) -#if defined(ARG_MAX) -#if defined(PREV_OUTPUT) -/** Find index maximum value of a vector - * - * @param[in] input Pointer to the first value. - * - * @return index of the vector. - */ -inline DATA_TYPE_OUTPUT arg_idx_max_prev_out(__global const DATA_TYPE *input, __global const DATA_TYPE_OUTPUT *prev_res, const int x_idx) +inline void scalar_compute_global_min_max(DATA_TYPE in_val, int idx, DATA_TYPE *out_min_max_val, DATA_TYPE_OUTPUT *out_idx) { - int end_elem = (x_idx + 1) * 16; - if(end_elem > WIDTH) - { - end_elem = WIDTH - x_idx * 16; - } - DATA_TYPE_OUTPUT res = prev_res[0]; - for(int x_v = 1; x_v < end_elem; ++x_v) +#if defined(ARG_MAX) + if(in_val > *out_min_max_val) +#else // defined(ARG_MAX) + if(in_val < *out_min_max_val) +#endif // defined(ARG_MAX) { - res = select(res, prev_res[x_v], *(input + prev_res[x_v]) > *(input + res)); + *out_min_max_val = in_val; + *out_idx = idx; } - return res; } -#else // !defined(PREV_OUTPUT) -/** Find index maximum value of a vector - * - * @param[in] input Pointer to the first value. - * - * @return index of the vector. - */ -inline DATA_TYPE_OUTPUT arg_idx_max(__global const DATA_TYPE *input, const int x_idx) -{ -#if WIDTH < 16 - DATA_TYPE_OUTPUT res = 0; - for(DATA_TYPE_OUTPUT x_v = res + 1; x_v < WIDTH; ++x_v) - { - res = select(res, x_v, *(input + x_v) > *(input + res)); - } - return res; -#else // WIDTH >= 16 - int x_elem = x_idx * 16; - const int x_goback = select(0, 16 - WIDTH % 16, x_elem + 16 > WIDTH); - x_elem -= x_goback; - - VEC_DATA_TYPE(DATA_TYPE, 16) - in = vload16(0, input - x_goback); - VEC_DATA_TYPE(DATA_TYPE_OUTPUT, 16) - res = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }; - - SIGNED_INT_VEC_DATA_TYPE(DATA_TYPE, 8) - idx_sel = (in.s01234567 >= in.s89abcdef); - in.s01234567 = select(in.s89abcdef, in.s01234567, idx_sel); - res.s01234567 = select(res.s89abcdef, res.s01234567, CONVERT(idx_sel, int8)); - - idx_sel.s0123 = (in.s0123 > in.s4567) || (in.s0123 == in.s4567 && CONVERT((res.s0123 < res.s4567), SIGNED_INT_VEC_DATA_TYPE(DATA_TYPE, 4))); - in.s0123 = select(in.s4567, in.s0123, idx_sel.s0123); - res.s0123 = select(res.s4567, res.s0123, CONVERT(idx_sel.s0123, int4)); - idx_sel.s01 = (in.s01 > in.s23) || (in.s01 == in.s23 && CONVERT((res.s01 < res.s23), SIGNED_INT_VEC_DATA_TYPE(DATA_TYPE, 2))); - in.s01 = select(in.s23, in.s01, idx_sel.s01); - res.s01 = select(res.s23, res.s01, CONVERT(idx_sel.s01, int2)); - - idx_sel.s0 = (in.s0 > in.s1) || (in.s0 == in.s1 && CONVERT((res.s0 < res.s1), SIGNED_INT_DATA_TYPE(DATA_TYPE))); - res.s0 = select(res.s1, res.s0, CONVERT(idx_sel.s0, int)); - - return res.s0 + x_elem; -#endif // WIDTH < 16 +#if VEC_SIZE > 1 +#if VEC_SIZE == 16 + #define VECTORIZED_OP(min_max_val,min_max_idx,in,res) vectorized_compute_arg_min_max_16(min_max_val,min_max_idx,in,res) +#elif VEC_SIZE == 8 // #if VEC_SIZE == 16 + #define VECTORIZED_OP(min_max_val,min_max_idx,in,res) vectorized_compute_arg_min_max_8(min_max_val,min_max_idx,in,res) +#elif VEC_SIZE == 4 // # elif VEC_SIZE == 8 + #define VECTORIZED_OP(min_max_val,min_max_idx,in,res) vectorized_compute_arg_min_max_4(min_max_val,min_max_idx,in,res) +#elif VEC_SIZE == 2 // elif VEC_SIZE == 4 + #define VECTORIZED_OP(min_max_val,min_max_idx,in,res) vectorized_compute_arg_min_max_2(min_max_val,min_max_idx,in,res) +#else // elif VEC_SIZE == 2 + #error "Not supported" +#endif // #if VEC_SIZE == 16 + +inline VEC_DATA_TYPE(DATA_TYPE_OUTPUT, VEC_SIZE) init_idx_vector() +{ +#if VEC_SIZE == 16 + VEC_DATA_TYPE(DATA_TYPE_OUTPUT, VEC_SIZE) + vidx = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }; +#elif VEC_SIZE == 8 // #if VEC_SIZE == 16 + VEC_DATA_TYPE(DATA_TYPE_OUTPUT, VEC_SIZE) + vidx = { 0, 1, 2, 3, 4, 5, 6, 7 }; +#elif VEC_SIZE == 4 // elif VEC_SIZE == 8 + VEC_DATA_TYPE(DATA_TYPE_OUTPUT, VEC_SIZE) + vidx = { 0, 1, 2, 3 }; +#elif VEC_SIZE == 2 // elif VEC_SIZE == 4 + VEC_DATA_TYPE(DATA_TYPE_OUTPUT, VEC_SIZE) + vidx = { 0, 1 }; +#else // elif VEC_SIZE == 2 +#error "Not supported" +#endif // #if VEC_SIZE == 16 + return vidx; } -#endif // defined(PREV_OUTPUT) -#endif // defined(ARG_MAX) +#endif // VEC_SIZE > 1 -/** This kernel performs parallel reduction given an operation on x-axis. +/** This kernel performs reduction on x-axis. * - * @note In case the results of previous stages are passed the flag PREV_OUTPUT has to be passed using -DPREV_OUTPUT - * @note The data type must be passed at compile time using -DDATA_TYPE: e.g. -DDATA_TYPE=float + * @note The input data type must be passed at compile time using -DDATA_TYPE: e.g. -DDATA_TYPE=float * @note The data type of the output must be passed at compile time using -DDATA_TYPE_OUTPUT: e.g. -DDATA_TYPE_OUTPUT=uint - * @note The arg_max flag must be passed at compile time using -DARG_MAX if we want to compute the ArgMax - * @note The arg_min flag must be passed at compile time using -DARG_MIN if we want to compute the ArgMin + * @note The data type used for the comparing indexe must be passed at compile type using -DCOND_DATA_TYPE: e.g -DCOND_DATA_TYPE=uint + * @note The height size must be passed at compile time using -DHEIGHT e.g. -DHEIGHT=128 * - * @param[in] src_ptr Pointer to the source tensor. Supported data types: QASYMM8/QASYMM8_SIGNED/S32/F16/F32 - * @param[in] src_stride_x Stride of the source tensor in X dimension (in bytes) - * @param[in] src_step_x src_stride_x * number of elements along X processed per workitem(in bytes) - * @param[in] src_stride_y Stride of the source tensor in Y dimension (in bytes) - * @param[in] src_step_y src_stride_y * number of elements along Y processed per workitem(in bytes) - * @param[in] src_offset_first_element_in_bytes The offset of the first element in the source tensor - * @param[in] prev_res_ptr (Optional) Pointer to previous results tensor. Supported data types: U32/S32 - * @param[in] prev_res_stride_x (Optional) Stride of the output tensor in X dimension (in bytes) - * @param[in] prev_res_step_x (Optional) prev_res_stride_x * number of elements along X processed per workitem(in bytes) - * @param[in] prev_res_stride_y (Optional) Stride of the output tensor in Y dimension (in bytes) - * @param[in] prev_res_step_y (Optional) prev_res_stride_y * number of elements along Y processed per workitem(in bytes) - * @param[in] prev_res_offset_first_element_in_bytes (Optional) The offset of the first element in the previous results tensor - * @param[in] partial_res_ptr The local buffer to hold partial result values. Supported data types: U32/S32 - * @param[in] partial_res_stride_x Stride of the output tensor in X dimension (in bytes) - * @param[in] partial_res_step_x partial_res_stride_x * number of elements along X processed per workitem(in bytes) - * @param[in] partial_res_stride_y Stride of the output tensor in Y dimension (in bytes) - * @param[in] partial_res_step_y partial_res_stride_y * number of elements along Y processed per workitem(in bytes) - * @param[in] partial_res_offset_first_element_in_bytes The offset of the first element in the source tensor - * @param[in] local_results Local buffer for storing the partial result + * @param[in] input_ptr Pointer to the source tensor. Supported data types: QASYMM8/QASYMM8_SIGNED/S32/F16/F32 + * @param[in] input_stride_x Stride of the source tensor in X dimension (in bytes) + * @param[in] input_step_x input_stride_x * number of elements along X processed per workitem(in bytes) + * @param[in] input_stride_y Stride of the source tensor in Y dimension (in bytes) + * @param[in] input_step_y input_stride_y * number of elements along Y processed per workitem(in bytes) + * @param[in] input_offset_first_element_in_bytes The offset of the first element in the source tensor + * @param[in] output_ptr The local buffer to hold sumed values. Supported data types: U32/S32 + * @param[in] output_stride_x Stride of the output tensor in X dimension (in bytes) + * @param[in] output_step_x output_stride_x * number of elements along X processed per workitem(in bytes) + * @param[in] output_stride_y Stride of the output tensor in Y dimension (in bytes) + * @param[in] output_step_y output_stride_y * number of elements along Y processed per workitem(in bytes) + * @param[in] output_offset_first_element_in_bytes The offset of the first element in the source tensor */ __kernel void arg_min_max_x( - IMAGE_DECLARATION(src), -#if defined(PREV_OUTPUT) - IMAGE_DECLARATION(prev_res), -#endif // defined(PREV_OUTPUT) - IMAGE_DECLARATION(partial_res), - __local DATA_TYPE_OUTPUT *local_results) + IMAGE_DECLARATION(input), + IMAGE_DECLARATION(output)) { -#if defined(PREV_OUTPUT) - Image src = CONVERT_TO_IMAGE_STRUCT_NO_STEP(src); - Image prev_res = CONVERT_TO_IMAGE_STRUCT(prev_res); -#else // !defined(PREV_OUTPUT) - Image src = CONVERT_TO_IMAGE_STRUCT(src); -#endif // defined(PREV_OUTPUT) - Image partial_res = CONVERT_TO_IMAGE_STRUCT(partial_res); - - unsigned int lsize = get_local_size(0); - unsigned int lid = get_local_id(0); - - const uint x_idx = get_global_id(0); - const uint y_idx = get_global_id(1); - const __global DATA_TYPE *src_in_row = (const __global DATA_TYPE *)(src_ptr + src_offset_first_element_in_bytes + y_idx * src_step_y); - - for(unsigned int y = 0; y < get_local_size(1); ++y) + __global DATA_TYPE *input_addr = (__global DATA_TYPE *)(input_ptr + input_offset_first_element_in_bytes + get_global_id(1) * input_stride_y); + __global DATA_TYPE_OUTPUT *output_addr = (__global DATA_TYPE_OUTPUT *)(output_ptr + output_offset_first_element_in_bytes + get_global_id(1) * output_stride_y); + + DATA_TYPE final_value = input_addr[0]; + DATA_TYPE_OUTPUT final_idx = 0; + +#if VEC_SIZE > 1 + VEC_DATA_TYPE(DATA_TYPE_OUTPUT, VEC_SIZE) + vidx = init_idx_vector(); + + int x = 0; + for(; x <= (WIDTH - VEC_SIZE); x += VEC_SIZE) { -#if defined(ARG_MAX) -#if defined(PREV_OUTPUT) - local_results[lid] = arg_idx_max_prev_out(src_in_row, (__global DATA_TYPE_OUTPUT *)offset(&prev_res, 0, y), x_idx); -#else // !defined(PREV_OUTPUT) - local_results[lid] = arg_idx_max((__global DATA_TYPE *)offset(&src, 0, y), x_idx); -#endif // defined(PREV_OUTPUT) -#else // defined(ARG_MIN) -#if defined(PREV_OUTPUT) - local_results[lid] = arg_idx_min_prev_out(src_in_row, (__global DATA_TYPE_OUTPUT *)offset(&prev_res, 0, y), x_idx); -#else // !defined(PREV_OUTPUT) - local_results[lid] = arg_idx_min((__global DATA_TYPE *)offset(&src, 0, y), x_idx); -#endif // defined(PREV_OUTPUT) -#endif // defined(ARG_MAX) || defined(ARG_MIN) - - barrier(CLK_LOCAL_MEM_FENCE); - - // Looking for the next highest power of 2 (maximum value of lsize is 8) - unsigned int middle = lsize - 1; - middle |= middle >> 1; - middle |= middle >> 2; - middle += 1; - // Perform parallel reduction - for(unsigned int i = middle; i > 0; i >>= 1) - { - if(lid < i && lid + i < lsize) - { - DATA_TYPE tmp0 = *(src_in_row + local_results[lid]); - DATA_TYPE tmp1 = *(src_in_row + local_results[lid + i]); -#if defined(ARG_MAX) - local_results[lid] = select( - local_results[lid], - local_results[lid + i], - ((tmp0 == tmp1) && (local_results[lid + i] < local_results[lid])) || (tmp0 < tmp1)); -#else // defined(ARG_MIN) - local_results[lid] = select( - local_results[lid], - local_results[lid + i], - ((tmp0 == tmp1) && (local_results[lid + i] < local_results[lid])) || (tmp0 > tmp1)); -#endif // defined(ARG_MAX) || defined(ARG_MIN) - } - barrier(CLK_LOCAL_MEM_FENCE); - } - - if(lid == 0) - { - ((__global DATA_TYPE_OUTPUT *)offset(&partial_res, get_group_id(0), y))[0] = local_results[0]; - } + VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE) + vals = VLOAD(VEC_SIZE)(0, (input_addr + x)); + DATA_TYPE local_min_max_value; + DATA_TYPE_OUTPUT local_min_max_idx; + + VECTORIZED_OP(&local_min_max_value, &local_min_max_idx, vals, vidx); + local_min_max_idx += x; + scalar_compute_global_min_max(local_min_max_value, local_min_max_idx, &final_value, &final_idx); } +#endif // VEC_SIZE > 1 + +#if(WIDTH % VEC_SIZE) + LOOP_UNROLLING(int, j, 0, 1, WIDTH % VEC_SIZE, + { + scalar_compute_global_min_max(*(input_addr + j + x), j + x, &final_value, &final_idx); + }) +#endif // (WIDTH % VEC_SIZE) + + output_addr[0] = final_idx; } #endif // defined(WIDTH) @@ -320,8 +258,7 @@ __kernel void arg_min_max_y( IMAGE_DECLARATION(input), IMAGE_DECLARATION(output)) { - const int x_offs = max((int)(get_global_id(0) * VEC_SIZE - (VEC_SIZE - VEC_SIZE_LEFTOVER) % VEC_SIZE), 0); - + const int x_offs = max((int)(get_global_id(0) * VEC_SIZE - (VEC_SIZE - VEC_SIZE_LEFTOVER) % VEC_SIZE), 0); __global uchar *input_addr = input_ptr + input_offset_first_element_in_bytes + x_offs * sizeof(DATA_TYPE) + get_global_id(1) * input_stride_y; __global uchar *output_addr = output_ptr + output_offset_first_element_in_bytes + x_offs * sizeof(DATA_TYPE_OUTPUT) + get_global_id(1) * output_stride_y; @@ -448,4 +385,4 @@ __kernel void arg_min_max_w( STORE_VECTOR_SELECT(indx, DATA_TYPE_OUTPUT, output_addr, VEC_SIZE, VEC_SIZE_LEFTOVER, VEC_SIZE_LEFTOVER != 0 && get_global_id(0) == 0); } #endif /* defined(BATCH) && defined(DEPTH) */ -#endif // defined(VEC_SIZE) && defined(DATA_TYPE) && defined(DATA_TYPE_OUTPUT) \ No newline at end of file +#endif // defined(VEC_SIZE) && defined(DATA_TYPE) && defined(DATA_TYPE_OUTPUT) diff --git a/src/core/CL/cl_kernels/common/mat_mul.cl b/src/core/CL/cl_kernels/common/mat_mul.cl index 90d485e8159ba7ffe047200d3b1ec4b714d9fa08..c7ef8ae52bb90e7595df4c413e096b305f0a06b3 100644 --- a/src/core/CL/cl_kernels/common/mat_mul.cl +++ b/src/core/CL/cl_kernels/common/mat_mul.cl @@ -21,9 +21,25 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ +#include "activation_float_helpers.h" #include "helpers.h" #include "tile_helpers.h" +#ifdef BIAS +// This function performs in-place bias addition for float/half datatype when bias is enabled. +// Note The tile's dimensions used for the LHS and RHS matrices (M0, N0 and K0) must be passed at compile time using -DN0, -DM0 (e.g. -DN0=8, -DM0=4). +inline void perform_bias_addition(uchar *bias_ptr, uint bias_offset_first_element_in_bytes, TILE(DATA_TYPE, M0, N0, acc), uint x) +{ + TILE(DATA_TYPE, 1, N0, bias_tile); + + // below expands to use bias_ptr and bias_offset_first_element_in_bytes + T_LOAD(DATA_TYPE, 1, N0, BUFFER, bias, x, 0, 1, 0, bias_tile); + + // c = c + bias[broadcasted] + T_ELTWISE_BROADCAST_ADD_X(DATA_TYPE, M0, N0, acc, bias_tile, acc); +} +#endif // defined(BIAS) + #if defined(MAT_MUL_NATIVE_NT_NT) /** This OpenCL kernel performs the batch matrix multiplication (BatchMatMul): LHS non-transposed, RHS non-transposed - buffer only * @@ -31,6 +47,7 @@ * should NOT be confused with the batch size of the model. For NHWC the "batch" is the "H" dimension * @note The data type must be passed at compile time using -DDATA_TYPE (e.g. -DDATA_TYPE=float) * @note The block's dimensions used for the LHS and RHS matrices (M0, N0 and K0) must be passed at compile time using -DN0, -DM0 and -DK0 (e.g. -DN0=8, -DM0=4, -DK0=4). + * @note The fused activation function used should be passed with -DACTIVATION_TYPE, -DA_VAL and -DB_VAL are used for min and max output bounded activation functions. * @note The number of leftover outputs rows/columns must be passed using -DPARTIAL_STORE_N0 and -DPARTIAL_STORE_M0 (e.g. -DPARTIAL_STORE_N0=2, -DPARTIAL_STORE_M0=3) * @note The dimension K must be passed at compile time using -DK (e.g. -DK=6) * @note The tensor type ("BUFFER" or "IMAGE") of the rhs tensor must be passed at compile time using -DRHS_TENSOR_TYPE (e.g. -DRHS_TENSOR_TYPE=BUFFER) @@ -41,32 +58,42 @@ * - K0 = 1, 2, 3, 4, 8, 16 * @note Values > 8 for M0 are not expected to be efficient * - * @param[in] lhs_ptr Pointer to the lhs matrix. Supported data types: F32/F16 - * @param[in] lhs_stride_y Stride of the lhs matrix in Y (2nd) dimension (in bytes) - * @param[in] lhs_stride_z Stride of the lhs tensor in Z (3rd) dimension (in bytes) - * @param[in] lhs_w The width of the lhs tensor - * @param[in] lhs_h The height of the lhs tensor - * @param[in] lhs_n Number of the matrices (buffers) in the batch - * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the lhs matrix - * @param[in] rhs_img (Optional) Read only cl_image object for the rhs tensor. Included when RHS_TENSOR_TYPE=IMAGE - * @param[in] rhs_ptr Pointer to the rhs matrix. Supported data types: same as @p lhs_ptr - * @param[in] rhs_stride_y Stride of the rhs matrix in Y (2nd) dimension (in bytes) - * @param[in] rhs_stride_z Stride of the rhs tensor in Z (3rd) dimension (in bytes) - * @param[in] rhs_w The width of the rhs tensor - * @param[in] rhs_h The height of the rhs tensor - * @param[in] rhs_n Number of the matrices (buffers) in the batch - * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the rhs matrix - * @param[out] dst_ptr Pointer to the dst matrix. Supported data types: same as @p lhs_ptr - * @param[in] dst_stride_y Stride of the dst matrix in Y (2nd) dimension (in bytes) - * @param[in] dst_stride_z Stride of the dst tensor in Z (3rd) dimension (in bytes) - * @param[in] dst_w The width of the dst tensor - * @param[in] dst_h The height of the dst tensor - * @param[in] dst_n Number of the matrices (buffers) in the batch - * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the dst matrix + * @param[in] lhs_ptr Pointer to the lhs matrix. Supported data types: F32/F16 + * @param[in] lhs_stride_y Stride of the lhs matrix in Y (2nd) dimension (in bytes) + * @param[in] lhs_stride_z Stride of the lhs tensor in Z (3rd) dimension (in bytes) + * @param[in] lhs_w The width of the lhs tensor + * @param[in] lhs_h The height of the lhs tensor + * @param[in] lhs_n Number of the matrices (buffers) in the batch + * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the lhs matrix + * @param[in] rhs_img (Optional) Read only cl_image object for the rhs tensor. Included when RHS_TENSOR_TYPE=IMAGE + * @param[in] rhs_ptr Pointer to the rhs matrix. Supported data types: same as @p lhs_ptr + * @param[in] rhs_stride_y Stride of the rhs matrix in Y (2nd) dimension (in bytes) + * @param[in] rhs_stride_z Stride of the rhs tensor in Z (3rd) dimension (in bytes) + * @param[in] rhs_w The width of the rhs tensor + * @param[in] rhs_h The height of the rhs tensor + * @param[in] rhs_n Number of the matrices (buffers) in the batch + * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the rhs matrix + * @param[in] bias_ptr (Optional) Pointer to the bias tensor. Supported data type: same as @p lhs_ptr + * @param[in] bias_stride_y (Optional) Stride of the bias tensor in Y dimension (in bytes) + * @param[in] bias_stride_z (Optional) Stride of the bias tensor in Z dimension (in bytes) + * @param[in] bias_w (Optional) The size of the width dimension of the bias tensor + * @param[in] bias_h (Optional) The size of the height dimension of the bias tensor + * @param[in] bias_n (Optional) The size of the depth dimension of the bias tensor + * @param[in] bias_offset_first_element_in_bytes (Optional) The offset of the first element in the bias tensor + * @param[out] dst_ptr Pointer to the dst matrix. Supported data types: same as @p lhs_ptr + * @param[in] dst_stride_y Stride of the dst matrix in Y (2nd) dimension (in bytes) + * @param[in] dst_stride_z Stride of the dst tensor in Z (3rd) dimension (in bytes) + * @param[in] dst_w The width of the dst tensor + * @param[in] dst_h The height of the dst tensor + * @param[in] dst_n Number of the matrices (buffers) in the batch + * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the dst matrix */ __kernel void mat_mul_native_nt_nt( TENSOR3D_T(lhs, BUFFER), TENSOR3D_T(rhs, RHS_TENSOR_TYPE), +#ifdef BIAS + TENSOR3D_T(bias, BUFFER), +#endif // defined(BIAS) TENSOR3D_T(dst, BUFFER)) { const uint x = GET_SPATIAL_IDX(0, N0, PARTIAL_STORE_N0); @@ -86,7 +113,7 @@ __kernel void mat_mul_native_nt_nt( }) const int rhs_z = z * rhs_h; - int k; + int k; for(k = 0; k <= K - K0; k += K0) { TILE(DATA_TYPE, M0, K0, a); @@ -111,7 +138,7 @@ __kernel void mat_mul_native_nt_nt( lhs_offset_first_element_in_bytes += K0 * sizeof(DATA_TYPE); } -#ifdef K % K0 != 0 +#if K % K0 != 0 /* Leftover Loop */ for(; k < K; ++k) { @@ -147,6 +174,12 @@ __kernel void mat_mul_native_nt_nt( indirect_buffer[_i].v = min(_i, select(M0 - 1, PARTIAL_STORE_M0 - 1, y_cond)); }); +#ifdef BIAS + perform_bias_addition(bias_ptr, bias_offset_first_element_in_bytes, acc, x); +#endif // defined(BIAS) + + T_ACTIVATION(DATA_TYPE, M0, N0, ACTIVATION_TYPE, A_VAL, B_VAL, acc, acc); + T_STORE_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, dst, 0, dst_stride_y, x_cond, acc, indirect_buffer); } #endif // defined(MAT_MUL_NATIVE_NT_NT) @@ -158,6 +191,7 @@ __kernel void mat_mul_native_nt_nt( * should NOT be confused with the batch size of the model. For NHWC the "batch" is the "H" dimension * @note The data type must be passed at compile time using -DDATA_TYPE (e.g. -DDATA_TYPE=float) * @note The block's dimensions used for the LHS and RHS matrices (M0, N0 and K0) must be passed at compile time using -DN0, -DM0 and -DK0 (e.g. -DN0=8, -DM0=4, -DK0=4). + * @note The fused activation function used should be passed with -DACTIVATION_TYPE, -DA_VAL and -DB_VAL are used for min and max output bounded activation functions. * @note The number of leftover outputs rows/columns must be passed using -DPARTIAL_STORE_N0 and -DPARTIAL_STORE_M0 (e.g. -DPARTIAL_STORE_N0=2, -DPARTIAL_STORE_M0=3) * @note The dimension K must be passed at compile time using -DK (e.g. -DK=6) * @note The tensor type ("BUFFER" or "IMAGE") of the rhs tensor must be passed at compile time using -DRHS_TENSOR_TYPE (e.g. -DRHS_TENSOR_TYPE=BUFFER) @@ -168,31 +202,41 @@ __kernel void mat_mul_native_nt_nt( * - K0 = 1, 2, 3, 4, 8, 16 (only 4, 8, 16 if RHS_TENSOR_TYPE=IMAGE) * @note Values > 8 for M0, N0 and K0 are not expected to be efficient * - * @param[in] lhs_ptr Pointer to the lhs matrix. Supported data types: F32/F16 - * @param[in] lhs_stride_y Stride of the lhs matrix in Y (2nd) dimension (in bytes) - * @param[in] lhs_stride_z Stride of the lhs tensor in Z (3rd) dimension (in bytes) - * @param[in] lhs_w The width of the lhs tensor - * @param[in] lhs_h The height of the lhs tensor - * @param[in] lhs_n Number of the matrices (buffers) in the batch - * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the lhs matrix - * @param[in] rhs_img (Optional) Read only cl_image object for the rhs tensor. Included when RHS_TENSOR_TYPE=IMAGE - * @param[in] rhs_ptr Pointer to the rhs matrix. Supported data types: same as @p lhs_ptr - * @param[in] rhs_stride_y Stride of the rhs matrix in Y (2nd) dimension (in bytes) - * @param[in] rhs_stride_z Stride of the rhs tensor in Z (3rd) dimension (in bytes) - * @param[in] rhs_w The width of the rhs tensor - * @param[in] rhs_h The height of the rhs tensor - * @param[in] rhs_n Number of the matrices (buffers) in the batch - * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the rhs matrix - * @param[out] dst_ptr Pointer to the dst matrix. Supported data types: same as @p lhs_ptr - * @param[in] dst_stride_y Stride of the dst matrix in Y (2nd) dimension (in bytes) - * @param[in] dst_stride_z Stride of the dst tensor in Z (3rd) dimension (in bytes) - * @param[in] dst_w The width of the dst tensor - * @param[in] dst_h The height of the dst tensor - * @param[in] dst_n Number of the matrices (buffers) in the batch - * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the dst matrix + * @param[in] lhs_ptr Pointer to the lhs matrix. Supported data types: F32/F16 + * @param[in] lhs_stride_y Stride of the lhs matrix in Y (2nd) dimension (in bytes) + * @param[in] lhs_stride_z Stride of the lhs tensor in Z (3rd) dimension (in bytes) + * @param[in] lhs_w The width of the lhs tensor + * @param[in] lhs_h The height of the lhs tensor + * @param[in] lhs_n Number of the matrices (buffers) in the batch + * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the lhs matrix + * @param[in] rhs_img (Optional) Read only cl_image object for the rhs tensor. Included when RHS_TENSOR_TYPE=IMAGE + * @param[in] rhs_ptr Pointer to the rhs matrix. Supported data types: same as @p lhs_ptr + * @param[in] rhs_stride_y Stride of the rhs matrix in Y (2nd) dimension (in bytes) + * @param[in] rhs_stride_z Stride of the rhs tensor in Z (3rd) dimension (in bytes) + * @param[in] rhs_w The width of the rhs tensor + * @param[in] rhs_h The height of the rhs tensor + * @param[in] rhs_n Number of the matrices (buffers) in the batch + * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the rhs matrix + * @param[in] bias_ptr (Optional) Pointer to the bias tensor. Supported data type: same as @p lhs_ptr + * @param[in] bias_stride_y (Optional) Stride of the bias tensor in Y dimension (in bytes) + * @param[in] bias_stride_z (Optional) Stride of the bias tensor in Z dimension (in bytes) + * @param[in] bias_w (Optional) The size of the width dimension of the bias tensor + * @param[in] bias_h (Optional) The size of the height dimension of the bias tensor + * @param[in] bias_n (Optional) The size of the depth dimension of the bias tensor + * @param[in] bias_offset_first_element_in_bytes (Optional) The offset of the first element in the bias tensor + * @param[out] dst_ptr Pointer to the dst matrix. Supported data types: same as @p lhs_ptr + * @param[in] dst_stride_y Stride of the dst matrix in Y (2nd) dimension (in bytes) + * @param[in] dst_stride_z Stride of the dst tensor in Z (3rd) dimension (in bytes) + * @param[in] dst_w The width of the dst tensor + * @param[in] dst_h The height of the dst tensor + * @param[in] dst_n Number of the matrices (buffers) in the batch + * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the dst matrix */ __kernel void mat_mul_native_nt_t(TENSOR3D_T(lhs, BUFFER), TENSOR3D_T(rhs, RHS_TENSOR_TYPE), +#ifdef BIAS + TENSOR3D_T(bias, BUFFER), +#endif // defined(BIAS) TENSOR3D_T(dst, BUFFER)) { @@ -213,7 +257,7 @@ __kernel void mat_mul_native_nt_t(TENSOR3D_T(lhs, BUFFER), }) const int rhs_z = z * rhs_h; - int k; + int k; for(k = 0; k <= K - K0; k += K0) { TILE(DATA_TYPE, M0, K0, a); @@ -301,6 +345,12 @@ __kernel void mat_mul_native_nt_t(TENSOR3D_T(lhs, BUFFER), indirect_buffer[_i].v = min(_i, select(M0 - 1, PARTIAL_STORE_M0 - 1, y_cond)); }); +#ifdef BIAS + perform_bias_addition(bias_ptr, bias_offset_first_element_in_bytes, acc, x); +#endif // defined(BIAS) + + T_ACTIVATION(DATA_TYPE, M0, N0, ACTIVATION_TYPE, A_VAL, B_VAL, acc, acc); + T_STORE_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, dst, 0, dst_stride_y, x_cond, acc, indirect_buffer); } #endif // defined(MAT_MUL_NATIVE_NT_T) @@ -312,6 +362,7 @@ __kernel void mat_mul_native_nt_t(TENSOR3D_T(lhs, BUFFER), * should NOT be confused with the batch size of the model. For NHWC the "batch" is the "H" dimension * @note The data type must be passed at compile time using -DDATA_TYPE (e.g. -DDATA_TYPE=float) * @note The block's dimensions used for the LHS and RHS matrices (M0, N0 and K0) must be passed at compile time using -DN0, -DM0 and -DK0 (e.g. -DN0=8, -DM0=4, -DK0=4). + * @note The fused activation function used should be passed with -DACTIVATION_TYPE, -DA_VAL and -DB_VAL are used for min and max output bounded activation functions. * @note The number of leftover outputs rows/columns must be passed using -DPARTIAL_STORE_N0 and -DPARTIAL_STORE_M0 (e.g. -DPARTIAL_STORE_N0=2, -DPARTIAL_STORE_M0=3) * @note The dimension K must be passed at compile time using -DK (e.g. -DK=6) * @note The tensor type ("BUFFER" or "IMAGE") of the rhs tensor must be passed at compile time using -DRHS_TENSOR_TYPE (e.g. -DRHS_TENSOR_TYPE=BUFFER) @@ -322,32 +373,42 @@ __kernel void mat_mul_native_nt_t(TENSOR3D_T(lhs, BUFFER), * - K0 > 0 * * @note Values > 8 for M0, and K0 are not expected to be efficient * - * @param[in] lhs_ptr Pointer to the lhs matrix. Supported data types: F32/F16 - * @param[in] lhs_stride_y Stride of the lhs matrix in Y (2nd) dimension (in bytes) - * @param[in] lhs_stride_z Stride of the lhs tensor in Z (3rd) dimension (in bytes) - * @param[in] lhs_w The width of the lhs tensor - * @param[in] lhs_h The height of the lhs tensor - * @param[in] lhs_n Number of the matrices (buffers) in the batch - * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the lhs matrix - * @param[in] rhs_img (Optional) Read only cl_image object for the rhs tensor. Included when RHS_TENSOR_TYPE=IMAGE - * @param[in] rhs_ptr Pointer to the rhs matrix. Supported data types: same as @p lhs_ptr - * @param[in] rhs_stride_y Stride of the rhs matrix in Y (2nd) dimension (in bytes) - * @param[in] rhs_stride_z Stride of the rhs tensor in Z (3rd) dimension (in bytes) - * @param[in] rhs_w The width of the rhs tensor - * @param[in] rhs_h The height of the rhs tensor - * @param[in] rhs_n Number of the matrices (buffers) in the batch - * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the rhs matrix - * @param[out] dst_ptr Pointer to the dst matrix. Supported data types: same as @p lhs_ptr - * @param[in] dst_stride_y Stride of the dst matrix in Y (2nd) dimension (in bytes) - * @param[in] dst_stride_z Stride of the dst tensor in Z (3rd) dimension (in bytes) - * @param[in] dst_w The width of the dst tensor - * @param[in] dst_h The height of the dst tensor - * @param[in] dst_n Number of the matrices (buffers) in the batch - * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the dst matrix + * @param[in] lhs_ptr Pointer to the lhs matrix. Supported data types: F32/F16 + * @param[in] lhs_stride_y Stride of the lhs matrix in Y (2nd) dimension (in bytes) + * @param[in] lhs_stride_z Stride of the lhs tensor in Z (3rd) dimension (in bytes) + * @param[in] lhs_w The width of the lhs tensor + * @param[in] lhs_h The height of the lhs tensor + * @param[in] lhs_n Number of the matrices (buffers) in the batch + * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the lhs matrix + * @param[in] rhs_img (Optional) Read only cl_image object for the rhs tensor. Included when RHS_TENSOR_TYPE=IMAGE + * @param[in] rhs_ptr Pointer to the rhs matrix. Supported data types: same as @p lhs_ptr + * @param[in] rhs_stride_y Stride of the rhs matrix in Y (2nd) dimension (in bytes) + * @param[in] rhs_stride_z Stride of the rhs tensor in Z (3rd) dimension (in bytes) + * @param[in] rhs_w The width of the rhs tensor + * @param[in] rhs_h The height of the rhs tensor + * @param[in] rhs_n Number of the matrices (buffers) in the batch + * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the rhs matrix + * @param[in] bias_ptr (Optional) Pointer to the bias tensor. Supported data type: same as @p lhs_ptr + * @param[in] bias_stride_y (Optional) Stride of the bias tensor in Y dimension (in bytes) + * @param[in] bias_stride_z (Optional) Stride of the bias tensor in Z dimension (in bytes) + * @param[in] bias_w (Optional) The size of the width dimension of the bias tensor + * @param[in] bias_h (Optional) The size of the height dimension of the bias tensor + * @param[in] bias_n (Optional) The size of the depth dimension of the bias tensor + * @param[in] bias_offset_first_element_in_bytes (Optional) The offset of the first element in the bias tensor + * @param[out] dst_ptr Pointer to the dst matrix. Supported data types: same as @p lhs_ptr + * @param[in] dst_stride_y Stride of the dst matrix in Y (2nd) dimension (in bytes) + * @param[in] dst_stride_z Stride of the dst tensor in Z (3rd) dimension (in bytes) + * @param[in] dst_w The width of the dst tensor + * @param[in] dst_h The height of the dst tensor + * @param[in] dst_n Number of the matrices (buffers) in the batch + * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the dst matrix */ __kernel void mat_mul_native_t_nt( TENSOR3D_T(lhs, BUFFER), TENSOR3D_T(rhs, RHS_TENSOR_TYPE), +#ifdef BIAS + TENSOR3D_T(bias, BUFFER), +#endif // defined(BIAS) TENSOR3D_T(dst, BUFFER)) { const uint x = GET_SPATIAL_IDX(0, N0, PARTIAL_STORE_N0); @@ -367,7 +428,7 @@ __kernel void mat_mul_native_t_nt( }) const int rhs_z = z * rhs_h; - int k; + int k; for(k = 0; k <= K - K0; k += K0) { TILE(DATA_TYPE, K0, M0, a); @@ -405,7 +466,7 @@ __kernel void mat_mul_native_t_nt( lhs_offset_first_element_in_bytes += K0 * lhs_stride_y; } -#ifdef K % K0 != 0 +#if K % K0 != 0 /* Leftover Loop */ for(; k < K; ++k) { @@ -451,6 +512,12 @@ __kernel void mat_mul_native_t_nt( indirect_buffer[_i].v = min(_i, select(M0 - 1, PARTIAL_STORE_M0 - 1, y_cond)); }); +#ifdef BIAS + perform_bias_addition(bias_ptr, bias_offset_first_element_in_bytes, acc, x); +#endif // defined(BIAS) + + T_ACTIVATION(DATA_TYPE, M0, N0, ACTIVATION_TYPE, A_VAL, B_VAL, acc, acc); + T_STORE_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, dst, 0, dst_stride_y, x_cond, acc, indirect_buffer); } #endif // defined(MAT_MUL_NATIVE_T_NT) @@ -462,6 +529,7 @@ __kernel void mat_mul_native_t_nt( * should NOT be confused with the batch size of the model. For NHWC the "batch" is the "H" dimension * @note The data type must be passed at compile time using -DDATA_TYPE (e.g. -DDATA_TYPE=float) * @note The block's dimensions used for the LHS and RHS matrices (M0, N0 and K0) must be passed at compile time using -DN0, -DM0 and -DK0 (e.g. -DN0=8, -DM0=4, -DK0=4). + * @note The fused activation function used should be passed with -DACTIVATION_TYPE, -DA_VAL and -DB_VAL are used for min and max output bounded activation functions. * @note The number of leftover outputs rows/columns must be passed using -DPARTIAL_STORE_N0 and -DPARTIAL_STORE_M0 (e.g. -DPARTIAL_STORE_N0=2, -DPARTIAL_STORE_M0=3) * @note The dimension K must be passed at compile time using -DK (e.g. -DK=6) * @note The tensor type ("BUFFER" or "IMAGE") of the rhs tensor must be passed at compile time using -DRHS_TENSOR_TYPE (e.g. -DRHS_TENSOR_TYPE=BUFFER) @@ -472,32 +540,42 @@ __kernel void mat_mul_native_t_nt( * - K0 = 1, 2, 3, 4, 8, 16 (only 4, 8, 16 if RHS_TENSOR_TYPE=IMAGE) * @note Values > 8 for M0, N0 and K0 are not expected to be efficient * - * @param[in] lhs_ptr Pointer to the lhs matrix. Supported data types: F32/F16 - * @param[in] lhs_stride_y Stride of the lhs matrix in Y (2nd) dimension (in bytes) - * @param[in] lhs_stride_z Stride of the lhs tensor in Z (3rd) dimension (in bytes) - * @param[in] lhs_w The width of the lhs tensor - * @param[in] lhs_h The height of the lhs tensor - * @param[in] lhs_n Number of the matrices (buffers) in the batch - * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the lhs matrix - * @param[in] rhs_img (Optional) Read only cl_image object for the rhs tensor. Included when RHS_TENSOR_TYPE=IMAGE - * @param[in] rhs_ptr Pointer to the rhs matrix. Supported data types: same as @p lhs_ptr - * @param[in] rhs_stride_y Stride of the rhs matrix in Y (2nd) dimension (in bytes) - * @param[in] rhs_stride_z Stride of the rhs tensor in Z (3rd) dimension (in bytes) - * @param[in] rhs_w The width of the rhs tensor - * @param[in] rhs_h The height of the rhs tensor - * @param[in] rhs_n Number of the matrices (buffers) in the batch - * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the rhs matrix - * @param[out] dst_ptr Pointer to the dst matrix. Supported data types: same as @p lhs_ptr - * @param[in] dst_stride_y Stride of the dst matrix in Y (2nd) dimension (in bytes) - * @param[in] dst_stride_z Stride of the dst tensor in Z (3rd) dimension (in bytes) - * @param[in] dst_w The width of the dst tensor - * @param[in] dst_h The height of the dst tensor - * @param[in] dst_n Number of the matrices (buffers) in the batch - * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the dst matrix + * @param[in] lhs_ptr Pointer to the lhs matrix. Supported data types: F32/F16 + * @param[in] lhs_stride_y Stride of the lhs matrix in Y (2nd) dimension (in bytes) + * @param[in] lhs_stride_z Stride of the lhs tensor in Z (3rd) dimension (in bytes) + * @param[in] lhs_w The width of the lhs tensor + * @param[in] lhs_h The height of the lhs tensor + * @param[in] lhs_n Number of the matrices (buffers) in the batch + * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the lhs matrix + * @param[in] rhs_img (Optional) Read only cl_image object for the rhs tensor. Included when RHS_TENSOR_TYPE=IMAGE + * @param[in] rhs_ptr Pointer to the rhs matrix. Supported data types: same as @p lhs_ptr + * @param[in] rhs_stride_y Stride of the rhs matrix in Y (2nd) dimension (in bytes) + * @param[in] rhs_stride_z Stride of the rhs tensor in Z (3rd) dimension (in bytes) + * @param[in] rhs_w The width of the rhs tensor + * @param[in] rhs_h The height of the rhs tensor + * @param[in] rhs_n Number of the matrices (buffers) in the batch + * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the rhs matrix + * @param[in] bias_ptr (Optional) Pointer to the bias tensor. Supported data type: same as @p lhs_ptr + * @param[in] bias_stride_y (Optional) Stride of the bias tensor in Y dimension (in bytes) + * @param[in] bias_stride_z (Optional) Stride of the bias tensor in Z dimension (in bytes) + * @param[in] bias_w (Optional) The size of the width dimension of the bias tensor + * @param[in] bias_h (Optional) The size of the height dimension of the bias tensor + * @param[in] bias_n (Optional) The size of the depth dimension of the bias tensor + * @param[in] bias_offset_first_element_in_bytes (Optional) The offset of the first element in the bias tensor + * @param[out] dst_ptr Pointer to the dst matrix. Supported data types: same as @p lhs_ptr, + * @param[in] dst_stride_y Stride of the dst matrix in Y (2nd) dimension (in bytes) + * @param[in] dst_stride_z Stride of the dst tensor in Z (3rd) dimension (in bytes) + * @param[in] dst_w The width of the dst tensor + * @param[in] dst_h The height of the dst tensor + * @param[in] dst_n Number of the matrices (buffers) in the batch + * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the dst matrix */ __kernel void mat_mul_native_t_t( TENSOR3D_T(lhs, BUFFER), TENSOR3D_T(rhs, RHS_TENSOR_TYPE), +#ifdef BIAS + TENSOR3D_T(bias, BUFFER), +#endif // defined(BIAS) TENSOR3D_T(dst, BUFFER)) { const uint x = GET_SPATIAL_IDX(0, N0, PARTIAL_STORE_N0); @@ -517,7 +595,7 @@ __kernel void mat_mul_native_t_t( }) const int rhs_z = z * rhs_h; - int k; + int k; for(k = 0; k <= K - K0; k += K0) { TILE(DATA_TYPE, K0, M0, a); @@ -565,7 +643,7 @@ __kernel void mat_mul_native_t_t( lhs_offset_first_element_in_bytes += K0 * lhs_stride_y; } -#ifdef K % K0 != 0 +#if K % K0 != 0 /* Leftover Loop */ for(; k < K; ++k) { @@ -619,6 +697,12 @@ __kernel void mat_mul_native_t_t( indirect_buffer[_i].v = min(_i, select(M0 - 1, PARTIAL_STORE_M0 - 1, y_cond)); }); +#ifdef BIAS + perform_bias_addition(bias_ptr, bias_offset_first_element_in_bytes, acc, x); +#endif // defined(BIAS) + + T_ACTIVATION(DATA_TYPE, M0, N0, ACTIVATION_TYPE, A_VAL, B_VAL, acc, acc); + T_STORE_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, dst, 0, dst_stride_y, x_cond, acc, indirect_buffer); } #endif // defined(MAT_MUL_NATIVE_T_T) diff --git a/src/core/CL/cl_kernels/common/mat_mul_mmul.cl b/src/core/CL/cl_kernels/common/mat_mul_mmul.cl new file mode 100644 index 0000000000000000000000000000000000000000..e549da86d449c5e6f739884959e119b9c686e83a --- /dev/null +++ b/src/core/CL/cl_kernels/common/mat_mul_mmul.cl @@ -0,0 +1,946 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "helpers.h" +#include "tile_helpers.h" + +#ifdef BIAS +// This function performs in-place bias addition for float and half datatypes when bias is enabled. +// Note The tile's dimensions used for the LHS and RHS matrices (M0, N0) must be passed at compile time using -DN0, -DM0 (e.g. -DN0=8, -DM0=4). +inline void perform_bias_addition(uchar *bias_ptr, uint bias_offset_first_element_in_bytes, TILE(DATA_TYPE, M0, N0, acc), uint x) +{ + TILE(DATA_TYPE, 1, N0, bias_tile); + + // below expands to use bias_ptr and bias_offset_first_element_in_bytes + T_LOAD(DATA_TYPE, 1, N0, BUFFER, bias, x, 0, 1, 0, bias_tile); + + // c = c + bias[broadcasted] + T_ELTWISE_BROADCAST_ADD_X(DATA_TYPE, M0, N0, acc, bias_tile, acc); +} +#endif // defined(BIAS) + +#if defined(MAT_MUL_NATIVE_MMUL_NT_NT) +/** This OpenCL kernel performs the batch matrix multiplication (BatchMatMul) using MMUL: LHS non-transposed, RHS non-transposed - buffer only + * + * @note the "batch" here expresses the number of matrix multiplications to run in parallel. However, it + * should NOT be confused with the batch size of the model. For NHWC the "batch" is the "H" dimension + * @note The data type must be passed at compile time using -DDATA_TYPE (e.g. -DDATA_TYPE=float) + * @note The tile's dimensions used for the LHS and RHS matrices (M0, N0 and K0) must be passed at compile time using -DN0, -DM0 and -DK0 (e.g. -DN0=8, -DM0=4, -DK0=1). + * @note The number of leftover outputs rows/columns must be passed using -DN0_LEFTOVER and -DM0_LEFTOVER (e.g. -DN0_LEFTOVER=2, -DM0_LEFTOVER=3) + * @note The MMUL block dimension (MMUL_M0, MMUL_N0, MMUL_K0) must be passed at compile time using -DMMUL_M0, -DMMUL_N0 and -DMMUL_K0 (e.g. -DMMUL_M0=4, -DMMUL_N0=4, -DMMUL_K0=4). + * @note The kernel name in uppercase must be passed at compile time (e.g. -DMAT_MUL_NATIVE_MMUL_NT_NT) + * @note Only the following configurations of M0, N0 and K0 are currently supported: + * - M0 > 0 + * - N0 = 1, 2, 3, 4, 8, 16 + * - K0 = 1 + * @note Values > 8 for M0 are not expected to be efficient + * + * @param[in] lhs_ptr Pointer to the lhs matrix. Supported data types: F32/F16 + * @param[in] lhs_stride_y Stride of the lhs matrix in Y (2nd) dimension (in bytes) + * @param[in] lhs_stride_z Stride of the lhs tensor in Z (3rd) dimension (in bytes) + * @param[in] lhs_w The width of the lhs tensor + * @param[in] lhs_h The height of the lhs tensor + * @param[in] lhs_n Number of the matrices (buffers) in the batch + * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the lhs matrix + * @param[in] rhs_ptr Pointer to the rhs matrix. Supported data types: same as @p lhs_ptr + * @param[in] rhs_stride_y Stride of the rhs matrix in Y (2nd) dimension (in bytes) + * @param[in] rhs_stride_z Stride of the rhs tensor in Z (3rd) dimension (in bytes) + * @param[in] rhs_w The width of the rhs tensor + * @param[in] rhs_h The height of the rhs tensor + * @param[in] rhs_n Number of the matrices (buffers) in the batch + * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the rhs matrix + * @param[in] bias_ptr (Optional) Pointer to the bias tensor. Supported data type: same as @p lhs_ptr + * @param[in] bias_stride_y (Optional) Stride of the bias tensor in Y dimension (in bytes) + * @param[in] bias_stride_z (Optional) Stride of the bias tensor in Z dimension (in bytes) + * @param[in] bias_w (Optional) The size of the width dimension of the bias tensor + * @param[in] bias_h (Optional) The size of the height dimension of the bias tensor + * @param[in] bias_n (Optional) The size of the depth dimension of the bias tensor + * @param[in] bias_offset_first_element_in_bytes (Optional) The offset of the first element in the bias tensor + * @param[out] dst_ptr Pointer to the dst matrix. Supported data types: same as @p lhs_ptr + * @param[in] dst_stride_y Stride of the dst matrix in Y (2nd) dimension (in bytes) + * @param[in] dst_stride_z Stride of the dst tensor in Z (3rd) dimension (in bytes) + * @param[in] dst_w The width of the dst tensor + * @param[in] dst_h The height of the dst tensor + * @param[in] dst_n Number of the matrices (buffers) in the batch + * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the dst matrix + * @param[in] M Number of rows in LHS matrix + * @param[in] N Number of columns in RHS matrix + * @param[in] K Number of columns in LHS matrix and rows in RHS matrix, which is multiple of MMUL_K0. + */ +__kernel void mat_mul_native_mmul_nt_nt( + TENSOR3D_T(lhs, BUFFER), + TENSOR3D_T(rhs, BUFFER), +#ifdef BIAS + TENSOR3D_T(bias, BUFFER), +#endif // defined(BIAS) + TENSOR3D_T(dst, BUFFER), + const int M, + const int N, + const int K) +{ +#define MMUL_BLOCK_SIZE (MMUL_M0 * MMUL_N0) // MMUL block size for the output matrix + + // The output/destination matrix is divided into "sections". Each section is filled by a group of + // threads of size MMUL_BLOCK_SIZE, bundled together according to GWS_x. + // Each thread writes to a tile of M0 x N0 (the usual output block size for a thread) in the output matrix. + // Therefore, the section dimensions are (MMUL_M0 x M0) x (MMUL_N0 x N0). + + // The GWS is constructed in such a way that the y global id is the y section coordinate, + // and the x global id is a transformed thread id: MMUL_BLOCK_SIZE number of consecutive threads + // in the x dimension corresponding to a section. + // This can be visualized as first obtaining the coordinates of all the sections: + // x = [0, (N / N0) / MMUL_N0) --> (N / N0) / MMUL_N0 is the number of sections in x dimension + // y = [0, (M / M0) / MMUL_M0) --> (M / M0) / MMUL_M0 is the number of sections in y dimension + // Then multiply the x coordinates with MMUL_SECTION_NUM_THREADS to get the consecutive thread ids in the x dimension + // x = [0, ((N / N0) / MMUL_N0) * MMUL_N0 * MMUL_M0) + // x = [0, (N / N0) * MMUL_MO) + const uint x0 = get_global_id(0); // [0, (N / N0) * MMUL_M0) + // The upper limit is a simplified version of (N / N0) / MMUL_N0) * MMUL_BLOCK_SIZE) + const uint y0 = get_global_id(1); // [0, (M / M0) / MMUL_M0) + const uint z = get_global_id(2); // Batch + + // Get section coordinates + const uint section_x = (x0 / MMUL_BLOCK_SIZE); + const uint section_y = y0; + + // Within these sections, each thread writes onto a small output block of size M0 x N0 + // in row major order. A section divided into tiles can be visualized as below. + // + // (Figure 1) + // A Section in the Output Matrix + // + // _____N0__________N0____________________N0____ + // | | | | | + // | | | | | + // M0 | Thread 1 | Thread 2 | ... | Thread | + // | | | | MMUL_N0 | + // |___________|__________|_________|___________| + // | | | | + // | | | | + // M0 | Thread | . | | + // | MMUL_N0+1 | . | | (M0 x MMUL_M0) + // |___________| . | | + // | . | | + // | . | | + // | . | | + // | |___________| + // | | | + // | | Thread | + // M0 | | MMUL_N0 x | + // | | MMUL_M0 | + // |________________________________|___________| + // N0 x MMUL_N0 + // + // The output matrix has several of these sections. As shown above, each section + // will be filled by a separate thread group of size MMUL_BLOCK_SIZE. The overall + // section layout of the output matrix is as below. For instance, S(1,1) will be filled + // by MMUL_BLOCK_SIZE (possibly equal to 16) threads, so as S(0,1) and others. + // + // (Figure 2) + // DST Matrix + // ____________________________________ + // | | | | | + // | S(0,0) | S(0,1) | ... | S(0, X) | + // |________|________|_______|_________| + // | | | | | + // | S(1,0) | S(1,1) | ... | S(1, X) | + // |________|________|_______|_________| + // | . | | | + // | . | | | Y = (M / M0) / MMUL_M0 - 1 (Max possible section y coordinate) + // | . | | | X = (N / N0) / MMUL_N0 - 1 (Max possible section x coordinate) + // |________|________|_________________| + // | | | | | S(y, x) denotes the section, and y and x are computed in + // | S(Y,0) | S(Y,1) | | S(Y, X) | section_y, section_x respectively. + // |________|________|_______|_________| + // + // + // + // + // A complete view involving the three matrices is given below. It examplifies how the section S(0,0) is computed. + // + // (Figure 3) + // Complete View + // + // LHS Matrix RHS Matrix DST Matrix + // + // ___MMUL_K0___________ __MMUL_N0 x N0____________ ___MMUL_N0 x N0____________________ + // /|xxxxxxxxxx| | /|xxxxxxxxxxxxxxx| | /|xxxxxxxxxxxxxxxxxxx| | + // / |xxxxxxxxxx| | MMUK_K0 ||xxxxxxxxxxxxxxx| | / |xxxxxxxxxxxxxxxxxxx| | + // MMUL_M0 | |xxxxxxxxxx| ---> | ||xxxxxxxxxxxxxxx| . . . | MMUL_M0 | |xxxxxxxxxxxxxxxxxxx| | + // x M0 | |xxxxxxxxxx| | \|_______________|_________| x M0 | |xxxxxxxxxxxxxxxxxxx| ... | + // | |xxxxxxxxxx| | | | | |xxxxxxxxxxxxxxxxxxx| | + // | |xxxxxxxxxx| | x | | | = \ |xxxxxxxxxxxxxxxxxxx| | + // \|__________|_________| | | | \|___________________| | + // | | | \/ | | | + // | , | |_________________________| | . | + // | , | | . | + // | , | | . | + // |____________________| |_________________________________| + // + // Horizontal and vertical arrows show the direction of K loop (main loop in the kernel). + // Each output section shown above is a zooomed out version of Figure 1. + // + // In each iteration of the main loop, LHS matrix traverses towards rightward, and RHS matrix traverses towards downward, + // the LHS section of (MMUL_M0 x M0) x MMUL_K0 and RHS section of MMUL_K0 x (MMUL_N0 x N0) is multiplied + // "cooperatively" using arm_matrix_multiply calls, and the result is accummulated over the output (DST) section + // of size (MMUL_M0 x M0) x (MMUL_N0 x N0) shown with 'x' signs. + // + // If it was a single thread, this multiplication would have been straightforward with a T_MMUL call. + // However, since it involves multiple threads working together using the aforementioned extension, it + // works slightly differently. + // + // Here is how threads access the LHS and RHS matrices: + // (Assume MMUL_K0 = MMUL_N0 = MMUL_M0 = 4 because the following diagram is heavily dependent on this) + // + // (Figure 4) + // Thread Access Layouts in LHS & RHS matrices + // + // LHS matrix RHS Matrix + // ___________________________ __________N0 times______N0 times____________________N0 times_______ + // |__T0__|__T1__|__T2__|__T3__| |__T0__| ... |__T0__|__T1__| ... |__T1__| ... |__T3__| ... |__T3__| + // |__T0__| ... | |__T4__| ... |__T4__|__T5__| ... |__T5__| ... |__T7__| ... |__T7__| + // M0 | . . | |__T8__| ... |__T8__|__T9__| ... |__T9__| ... |__T11_| ... |__T11_| + // Times | . . | |__T12_|_____|__T12_|__T13_|______|__T13_|_____|__T15_|_____|__T15_| + // | . . | X + // |__T0__|__T1__|__T2__|__T3__| + // |__T4__|__T5__|__T6__|__T7__| + // |__T4__|__T5__|__T6__|__T7__| + // M0 | . . | + // Times | . . | + // | . . | + // |__T4__|__T5__|__T6__|__T7__| + // |__T8__|__T9__|__T10_|__T11_| + // M0 | . | + // Times | . | + // | . | + // |__T12_|__T13_|__T14_|__T15_| + // M0 | . | + // Times | . | + // | . | + // |__T12_|__T13_|__T14_|__T15_| + // + // + // This access layout is designed such that the threads access continuous elements of each matrix (in terms of row/column). + // To multiply these large sections, the arm_matrix_multiply call is made for each of the M0xN0 elements. So, for each + // combination of m0 and n0 (iterators of M0 and N0 from 0 to M0-1 and N0-1 respectively), one arm_matrix_multiply call is + // made, and MMUL_BLOCK_SIZE number of threads compute the result. + // + // The matrix multiplication taking place in this extension + // is an "interleaved" one, because, for example, if m0=0 and n0=0, i.e. the first iteration, we would use the first, + // M0-th, 2M0-th and 3M0-th rows of the LHS matrix. Similarly, we jump N0 steps in the RHS matrix. This is how we access + // one element for each thread in a single (m0, n0) loop. + // + // For example, if we have + // - a 8 x 4 LHS section + // - 4 x 8 RHS section + // - Each vector variable ai, bj represent a 4x1 vector + // - ^T (superscript T) denotes transpose + // - M0 = N0 = 2 + // - MMUL_N0 = MMUL_M0 = MMUL_K0 = 4 + // + // (Figure 5) + // Mathematical view of the Matrix Multiplication + // + // LHS RHS DST + // [ a1^T ] [ b1 b2 b3 b4 b5 b6 b7 ] [ a1^Tb1 a1^Tb2 a1^Tb3 ... a1^Tb7 ] + // [ a2^T ] 4 x 8 [ a2^Tb1 a2^Tb2 a2^Tb3 ... a2^Tb7 ] + // [ a3^T ] [ ] + // [ a4^T ] = [ . . ] + // [ a5^T ] X [ . . ] + // [ a6^T ] [ . . ] + // [ a7^T ] [ ] + // [ a8^T ] [ a7^Tb1 a7^Tb2 a7^Tb3 ... a7^Tb7 ] + // 8 x 4 8 x 8 + // + // + // For the first iteration, i.e. (m0, n0) = (0, 0), the arm_matrix_multiply would multiply the following matrices: + // + // [ a1^T ] [ b1 b3 b5 b7 ] [ a1^Tb1 a1^Tb3 a1^Tb5 a1^Tb7 ] + // [ a3^T ] x 4 x 4 = [ a3^Tb1 a1^Tb3 a1^Tb5 a1^Tb7 ] + // [ a5^T ] [ a5^Tb1 a1^Tb3 a1^Tb5 a1^Tb7 ] + // [ a7^T ] [ a7^Tb1 a7^Tb3 a7^Tb5 a7^Tb7 ] + // 4 x 4 4 x 4 + // The elements calculated in the 4x4 output block are the "interleaved" elements in the DST above. + // When we follow for each combination of (m0, n0), every element of the DST matrix "section" is filled. + // + + // Get thread coordinates within an mmul block (of size MMUL_BLOCK_SIZE) + // Since threads are grouped in x dimension, the modular of x-dim global id + // wrt the MMUL_BLOCK_SIZE is the thread id in the group, ranging from 0 to + // MMUL_BLOCK_SIZE-1. Because the thread numbering is in row-major order. + const uint thread_id = (x0 % MMUL_BLOCK_SIZE); + const uint thread_x = thread_id % MMUL_N0; + const uint thread_y = (thread_id / MMUL_N0); + + // Starting destination coordinates + // Note: We need to clamp dst_x and dst_y because we always need to execute a complete MMUL block! Only after the matrix multiplication + // part can we exit the kernel if it is out-of-bound. Remember, we have a cooperative matrix multiplication. Therefore, we need a full block to get the correct results + // Although we will never write out-of-bound, we still need this clamp to ensure that we do not read out-of-bound either. + // The unclamped dst coordinates can be calculated easily from the output section coordinates and the thread coordinates (see above figure). + + // See Figure 1 & 2. Thread step size is N0 and M0, + // Section step size is N0 x MMUL_N0 and M0 x MMUL_M0 + // respectively for x, y dimensions. + const uint dst_x_unclamped = thread_x * N0 + section_x * N0 * MMUL_N0; + const uint dst_y_unclamped = thread_y * M0 + section_y * M0 * MMUL_M0; + const uint dst_x = min(dst_x_unclamped, (uint)(N - N0)); + const uint dst_y = min(dst_y_unclamped, (uint)(M - M0)); + + // Starting LHS coordinates + const uint lhs_x = thread_x; + const uint lhs_y = dst_y; + + // Starting RHS coordinates + const uint rhs_x = dst_x; + const uint rhs_y = thread_y; + + // Compute LHS/RHS/DST matrix address + lhs_offset_first_element_in_bytes += lhs_x * sizeof(DATA_TYPE) + lhs_y * lhs_stride_y + z * lhs_stride_z; + rhs_offset_first_element_in_bytes += rhs_x * sizeof(DATA_TYPE) + rhs_y * rhs_stride_y + z * rhs_stride_z; + dst_offset_first_element_in_bytes += dst_x * sizeof(DATA_TYPE) + dst_y * dst_stride_y + z * dst_stride_z; + + // Initialize the accumulators + // MMUL extension accumulate the result in F32 for both F32 and F16 + TILE(float, M0, N0, c_f32); + + LOOP_UNROLLING(int, i, 0, 1, M0, + { + c_f32[i].v = 0; + }) + + for(int k = 0; k < K; k += MMUL_K0) + { + // A tile of M0xK0 but K0 must be set to 1 + TILE(DATA_TYPE, M0, 1, a); + // A tile of K0xN0 but K0 must be set to 1 + TILE(DATA_TYPE, 1, N0, b); + + // Load tile from the lhs/rhs tensors + T_LOAD(DATA_TYPE, M0, 1, BUFFER, lhs, 0, 0, 1, lhs_stride_y, a); + T_LOAD(DATA_TYPE, 1, N0, BUFFER, rhs, 0, 0, 1, rhs_stride_y, b); + + LOOP_UNROLLING(int, m0, 0, 1, M0, + { + LOOP_UNROLLING(int, n0, 0, 1, N0, + { + c_f32[m0].s[n0] = arm_matrix_multiply(a[m0].s[0], b[0].s[n0], c_f32[m0].s[n0]); + }) + }) + + lhs_offset_first_element_in_bytes += MMUL_K0 * sizeof(DATA_TYPE); + rhs_offset_first_element_in_bytes += MMUL_K0 * rhs_stride_y; + } + + // For threads "outside" of the dst bound, we do not write but we have to "read" (arm_matrix_multiply). That's why this needs to happen after arm_matrix_multiply + if(dst_x_unclamped >= N || dst_y_unclamped >= M) + { + return; + } + +#if defined(HALF_PRECISION) + TILE(DATA_TYPE, M0, N0, c); + + // Conversion required for the half precision + LOOP_UNROLLING(int, m0, 0, 1, M0, + { + LOOP_UNROLLING(int, n0, 0, 1, N0, + { + c[m0].s[n0] = c_f32[m0].s[n0]; + }) + }) +#else // defined(HALF_PRECISION) +#define c c_f32 +#endif // defined(HALF_PRECISION) + +#ifdef BIAS + perform_bias_addition(bias_ptr, bias_offset_first_element_in_bytes, c, dst_x); +#endif // defined(BIAS) + + if(dst_x + N0 <= N || N0_LEFTOVER == 0) + { + LOOP_UNROLLING(int, m0, 0, 1, M0, + { + if(dst_y + m0 < M || M0_LEFTOVER == 0) + { + VSTORE(N0) + (c[m0].v, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_first_element_in_bytes + m0 * dst_stride_y)); + } + }) + } + else + { + LOOP_UNROLLING(int, m0, 0, 1, M0, + { + if(dst_y + m0 < M || M0_LEFTOVER == 0) + { + VSTORE_PARTIAL(N0, N0_LEFTOVER) + (c[m0].v, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_first_element_in_bytes + m0 * dst_stride_y)); + } + }) + } + +#undef MMUL_BLOCK_SIZE +} +#endif // defined(MAT_MUL_NATIVE_MMUL_NT_NT) + +#if defined(MAT_MUL_NATIVE_MMUL_T_NT) +/** This OpenCL kernel performs the batch matrix multiplication (BatchMatMul) using MMUL: LHS transposed, RHS non-transposed - buffer only + * + * @note the "batch" here expresses the number of matrix multiplications to run in parallel. However, it + * should NOT be confused with the batch size of the model. For NHWC the "batch" is the "H" dimension + * @note The data type must be passed at compile time using -DDATA_TYPE (e.g. -DDATA_TYPE=float) + * @note The tile's dimensions used for the LHS and RHS matrices (M0, N0 and K0) must be passed at compile time using -DN0, -DM0 and -DK0 (e.g. -DN0=8, -DM0=4, -DK0=1). + * @note The number of leftover outputs rows/columns must be passed using -DN0_LEFTOVER and -DM0_LEFTOVER (e.g. -DN0_LEFTOVER=2, -DM0_LEFTOVER=3) + * @note The MMUL block dimension (MMUL_M0, MMUL_N0, MMUL_K0) must be passed at compile time using -DMMUL_M0, -DMMUL_N0 and -DMMUL_K0 (e.g. -DMMUL_M0=4, -DMMUL_N0=4, -DMMUL_K0=4). + * @note The dimension K must be passed at compile time using -DK (e.g. -DK=4). K must be a multiple of MMUL_K0 + * @note The kernel name in uppercase must be passed at compile time (e.g. -DMAT_MUL_NATIVE_MMUL_T_NT) + * @note Only the following configurations of M0, N0 and K0 are currently supported: + * - M0 = 1, 2, 3, 4, 8, 16 + * - N0 = 1, 2, 3, 4, 8, 16 + * - K0 = 1 + * @note Values > 8 for M0 are not expected to be efficient + * + * @param[in] lhs_ptr Pointer to the lhs matrix. Supported data types: F32/F16 + * @param[in] lhs_stride_y Stride of the lhs matrix in Y (2nd) dimension (in bytes) + * @param[in] lhs_stride_z Stride of the lhs tensor in Z (3rd) dimension (in bytes) + * @param[in] lhs_w The width of the lhs tensor + * @param[in] lhs_h The height of the lhs tensor + * @param[in] lhs_n Number of the matrices (buffers) in the batch + * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the lhs matrix + * @param[in] rhs_ptr Pointer to the rhs matrix. Supported data types: same as @p lhs_ptr + * @param[in] rhs_stride_y Stride of the rhs matrix in Y (2nd) dimension (in bytes) + * @param[in] rhs_stride_z Stride of the rhs tensor in Z (3rd) dimension (in bytes) + * @param[in] rhs_w The width of the rhs tensor + * @param[in] rhs_h The height of the rhs tensor + * @param[in] rhs_n Number of the matrices (buffers) in the batch + * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the rhs matrix + * @param[in] bias_ptr (Optional) Pointer to the bias tensor. Supported data type: same as @p lhs_ptr + * @param[in] bias_stride_y (Optional) Stride of the bias tensor in Y dimension (in bytes) + * @param[in] bias_stride_z (Optional) Stride of the bias tensor in Z dimension (in bytes) + * @param[in] bias_w (Optional) The size of the width dimension of the bias tensor + * @param[in] bias_h (Optional) The size of the height dimension of the bias tensor + * @param[in] bias_n (Optional) The size of the depth dimension of the bias tensor + * @param[in] bias_offset_first_element_in_bytes (Optional) The offset of the first element in the bias tensor + * @param[out] dst_ptr Pointer to the dst matrix. Supported data types: same as @p lhs_ptr + * @param[in] dst_stride_y Stride of the dst matrix in Y (2nd) dimension (in bytes) + * @param[in] dst_stride_z Stride of the dst tensor in Z (3rd) dimension (in bytes) + * @param[in] dst_w The width of the dst tensor + * @param[in] dst_h The height of the dst tensor + * @param[in] dst_n Number of the matrices (buffers) in the batch + * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the dst matrix + * @param[in] M Number of rows in DST matrix + * @param[in] N Number of columns in DST matrix + * @param[in] K Number of rows in LHS and RHS matrices, which is multiple of MMUL_K0. + */ +__kernel void mat_mul_native_mmul_t_nt( + TENSOR3D_T(lhs, BUFFER), + TENSOR3D_T(rhs, BUFFER), +#ifdef BIAS + TENSOR3D_T(bias, BUFFER), +#endif // defined(BIAS) + TENSOR3D_T(dst, BUFFER), + const int M, + const int N, + const int K) +{ +#define MMUL_BLOCK_SIZE (MMUL_M0 * MMUL_N0) + // For explanations on how this kernel works, please refer to NT/NT kernel. This kernel makes little modifications to it. + + const uint x0 = get_global_id(0); // [0, (N / N0) * MMUL_M0) + // The upper limit is a simplified version of (N / N0) / MMUL_N0) * MMUL_BLOCK_SIZE) + const uint y0 = get_global_id(1); // [0, (M / M0) / MMUL_M0) + const uint z = get_global_id(2); // Batch + + // Get section coordinates + const uint section_x = (x0 / MMUL_BLOCK_SIZE); + const uint section_y = y0; + + // Get thread coordinates + uint thread_id = (x0 % MMUL_BLOCK_SIZE); + uint thread_x = thread_id % MMUL_N0; + uint thread_y = (thread_id / MMUL_N0); + + // See Nt/Nt kernel for explanations + const uint dst_x_unclamped = thread_x * N0 + section_x * N0 * MMUL_N0; + const uint dst_y_unclamped = thread_y * M0 + section_y * M0 * MMUL_M0; + const uint dst_x = min(dst_x_unclamped, (uint)(N - N0)); + const uint dst_y = min(dst_y_unclamped, (uint)(M - M0)); + + // Starting LHS coordinates + uint lhs_x = dst_y; + uint lhs_y = thread_x; + + // Starting RHS coordinates + uint rhs_x = dst_x; + uint rhs_y = thread_y; + + // Compute LHS/RHS/DST matrix address + lhs_offset_first_element_in_bytes += lhs_x * sizeof(DATA_TYPE) + lhs_y * lhs_stride_y + z * lhs_stride_z; + rhs_offset_first_element_in_bytes += rhs_x * sizeof(DATA_TYPE) + rhs_y * rhs_stride_y + z * rhs_stride_z; + dst_offset_first_element_in_bytes += dst_x * sizeof(DATA_TYPE) + dst_y * dst_stride_y + z * dst_stride_z; + + // Initialize the accumulators + // MMUL extension accumulate the result in F32 for both F32 and F16 + TILE(float, M0, N0, c_f32); + + LOOP_UNROLLING(int, i, 0, 1, M0, + { + c_f32[i].v = 0; + }) + + for(int k = 0; k < K; k += MMUL_K0) + { + TILE(DATA_TYPE, 1, M0, a); + TILE(DATA_TYPE, 1, N0, b); + + // Load tile from the lhs/rhs tensors + T_LOAD(DATA_TYPE, 1, M0, BUFFER, lhs, 0, 0, 1, lhs_stride_y, a); + T_LOAD(DATA_TYPE, 1, N0, BUFFER, rhs, 0, 0, 1, rhs_stride_y, b); + + LOOP_UNROLLING(int, m0, 0, 1, M0, + { + LOOP_UNROLLING(int, n0, 0, 1, N0, + { + c_f32[m0].s[n0] = arm_matrix_multiply(a[0].s[m0], b[0].s[n0], c_f32[m0].s[n0]); + }) + }) + + lhs_offset_first_element_in_bytes += MMUL_K0 * lhs_stride_y; + rhs_offset_first_element_in_bytes += MMUL_K0 * rhs_stride_y; + } + + // For threads "outside" of the dst bound, we do not write but we have to "read" (arm_matrix_multiply). That's why this needs to happen after arm_matrix_multiply + if(dst_x_unclamped >= N || dst_y_unclamped >= M) + { + return; + } + +#if defined(HALF_PRECISION) + TILE(DATA_TYPE, M0, N0, c); + + // Conversion required for the half precision + LOOP_UNROLLING(int, m0, 0, 1, M0, + { + LOOP_UNROLLING(int, n0, 0, 1, N0, + { + c[m0].s[n0] = c_f32[m0].s[n0]; + }) + }) +#else // defined(HALF_PRECISION) +#define c c_f32 +#endif // defined(HALF_PRECISION) + +#ifdef BIAS + perform_bias_addition(bias_ptr, bias_offset_first_element_in_bytes, c, dst_x); +#endif // defined(BIAS) + + if(dst_x + N0 <= N || N0_LEFTOVER == 0) + { + LOOP_UNROLLING(int, m0, 0, 1, M0, + { + if(dst_y + m0 < M || M0_LEFTOVER == 0) + { + VSTORE(N0) + (c[m0].v, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_first_element_in_bytes + m0 * dst_stride_y)); + } + }) + } + else + { + LOOP_UNROLLING(int, m0, 0, 1, M0, + { + if(dst_y + m0 < M || M0_LEFTOVER == 0) + { + VSTORE_PARTIAL(N0, N0_LEFTOVER) + (c[m0].v, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_first_element_in_bytes + m0 * dst_stride_y)); + } + }) + } + +#undef MMUL_BLOCK_SIZE +} +#endif // defined(MAT_MUL_NATIVE_MMUL_T_NT) + +#if defined(MAT_MUL_NATIVE_MMUL_NT_T) +/** This OpenCL kernel performs the batch matrix multiplication (BatchMatMul) using MMUL: LHS non-transposed, RHS transposed - buffer only + * + * @note the "batch" here expresses the number of matrix multiplications to run in parallel. However, it + * should NOT be confused with the batch size of the model. For NHWC the "batch" is the "H" dimension + * @note The data type must be passed at compile time using -DDATA_TYPE (e.g. -DDATA_TYPE=float) + * @note The tile's dimensions used for the LHS and RHS matrices (M0, N0 and K0) must be passed at compile time using -DN0, -DM0 and -DK0 (e.g. -DN0=8, -DM0=4, -DK0=1). + * @note The number of leftover outputs rows/columns must be passed using -DN0_LEFTOVER and -DM0_LEFTOVER (e.g. -DN0_LEFTOVER=2, -DM0_LEFTOVER=3) + * @note The MMUL block dimension (MMUL_M0, MMUL_N0, MMUL_K0) must be passed at compile time using -DMMUL_M0, -DMMUL_N0 and -DMMUL_K0 (e.g. -DMMUL_M0=4, -DMMUL_N0=4, -DMMUL_K0=4). + * @note The kernel name in uppercase must be passed at compile time (e.g. -DMAT_MUL_NATIVE_MMUL_NT_T) + * @note Only the following configurations of M0, N0 and K0 are currently supported: + * - M0 > 0 + * - N0 = 1, 2, 3, 4, 8, 16 + * - K0 = 1 + * @note Values > 8 for M0 are not expected to be efficient + * + * @param[in] lhs_ptr Pointer to the lhs matrix. Supported data types: F32/F16 + * @param[in] lhs_stride_y Stride of the lhs matrix in Y (2nd) dimension (in bytes) + * @param[in] lhs_stride_z Stride of the lhs tensor in Z (3rd) dimension (in bytes) + * @param[in] lhs_w The width of the lhs tensor + * @param[in] lhs_h The height of the lhs tensor + * @param[in] lhs_n Number of the matrices (buffers) in the batch + * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the lhs matrix + * @param[in] rhs_ptr Pointer to the rhs matrix. Supported data types: same as @p lhs_ptr + * @param[in] rhs_stride_y Stride of the rhs matrix in Y (2nd) dimension (in bytes) + * @param[in] rhs_stride_z Stride of the rhs tensor in Z (3rd) dimension (in bytes) + * @param[in] rhs_w The width of the rhs tensor + * @param[in] rhs_h The height of the rhs tensor + * @param[in] rhs_n Number of the matrices (buffers) in the batch + * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the rhs matrix + * @param[in] bias_ptr (Optional) Pointer to the bias tensor. Supported data type: same as @p lhs_ptr + * @param[in] bias_stride_y (Optional) Stride of the bias tensor in Y dimension (in bytes) + * @param[in] bias_stride_z (Optional) Stride of the bias tensor in Z dimension (in bytes) + * @param[in] bias_w (Optional) The size of the width dimension of the bias tensor + * @param[in] bias_h (Optional) The size of the height dimension of the bias tensor + * @param[in] bias_n (Optional) The size of the depth dimension of the bias tensor + * @param[in] bias_offset_first_element_in_bytes (Optional) The offset of the first element in the bias tensor + * @param[out] dst_ptr Pointer to the dst matrix. Supported data types: same as @p lhs_ptr + * @param[in] dst_stride_y Stride of the dst matrix in Y (2nd) dimension (in bytes) + * @param[in] dst_stride_z Stride of the dst tensor in Z (3rd) dimension (in bytes) + * @param[in] dst_w The width of the dst tensor + * @param[in] dst_h The height of the dst tensor + * @param[in] dst_n Number of the matrices (buffers) in the batch + * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the dst matrix + * @param[in] M Number of rows in LHS matrix + * @param[in] N Number of columns in RHS matrix + * @param[in] K Number of columns in LHS matrix and columns in RHS matrix, which is multiple of MMUL_K0. + */ +__kernel void mat_mul_native_mmul_nt_t( + TENSOR3D_T(lhs, BUFFER), + TENSOR3D_T(rhs, BUFFER), +#ifdef BIAS + TENSOR3D_T(bias, BUFFER), +#endif // defined(BIAS) + TENSOR3D_T(dst, BUFFER), + const int M, + const int N, + const int K) +{ +#define MMUL_BLOCK_SIZE (MMUL_M0 * MMUL_N0) + // For explanations on how this kernel works, please refer to NT/NT kernel. This kernel makes little modifications to it. + + const uint x0 = get_global_id(0); // [0, (N / N0) * MMUL_M0) + // The upper limit is a simplified version of (N / N0) / MMUL_N0) * MMUL_BLOCK_SIZE) + const uint y0 = get_global_id(1); // [0, (M / M0) / MMUL_M0) + const uint z = get_global_id(2); // Batch + + // Get block coordinates + const uint section_x = (x0 / MMUL_BLOCK_SIZE); + const uint section_y = y0; + + // Get thread coordinates within a block + const uint thread_id = (x0 % MMUL_BLOCK_SIZE); + const uint thread_x = thread_id % MMUL_N0; + const uint thread_y = (thread_id / MMUL_N0); + + // Starting destination coordinates + // Note: We need to clamp dst_x and dst_y because we always need to execute a complete MMUL block! Only after the matrix multiplication + // part can we exit the kernel if it is out-of-bound. Remember, we have a cooperative matrix multiplication. Therefore, we need a full block to get the correct results + // Although we will never write out-of-bound, we still need this clamp to ensure that we do not read out-of-bound either. + const uint dst_x_unclamped = thread_x * N0 + section_x * N0 * MMUL_N0; + const uint dst_y_unclamped = thread_y * M0 + section_y * M0 * MMUL_M0; + const uint dst_x = min(dst_x_unclamped, (uint)(N - N0)); + const uint dst_y = min(dst_y_unclamped, (uint)(M - M0)); + + // Starting LHS coordinates + const uint lhs_x = thread_x; + const uint lhs_y = dst_y; + + // Starting RHS coordinates + const uint rhs_x = thread_y; + const uint rhs_y = dst_x; + + // Compute LHS/RHS/DST matrix address + lhs_offset_first_element_in_bytes += lhs_x * sizeof(DATA_TYPE) + lhs_y * lhs_stride_y + z * lhs_stride_z; + rhs_offset_first_element_in_bytes += rhs_x * sizeof(DATA_TYPE) + rhs_y * rhs_stride_y + z * rhs_stride_z; + dst_offset_first_element_in_bytes += dst_x * sizeof(DATA_TYPE) + dst_y * dst_stride_y + z * dst_stride_z; + + // Initialize the accumulators + // MMUL extension accumulate the result in F32 for both F32 and F16 + TILE(float, M0, N0, c_f32); + + LOOP_UNROLLING(int, i, 0, 1, M0, + { + c_f32[i].v = 0; + }) + + for(int k = 0; k < K; k += MMUL_K0) + { + // A tile of M0xK0 but K0 must be set to 1 + TILE(DATA_TYPE, M0, 1, a); + // A tile of N0xK0 but K0 must be set to 1 + TILE(DATA_TYPE, N0, 1, b); + + // Load tile from the lhs/rhs tensors + T_LOAD(DATA_TYPE, M0, 1, BUFFER, lhs, 0, 0, 1, lhs_stride_y, a); + T_LOAD(DATA_TYPE, N0, 1, BUFFER, rhs, 0, 0, 1, rhs_stride_y, b); + + LOOP_UNROLLING(int, m0, 0, 1, M0, + { + LOOP_UNROLLING(int, n0, 0, 1, N0, + { + c_f32[m0].s[n0] = arm_matrix_multiply(a[m0].s[0], b[n0].s[0], c_f32[m0].s[n0]); + }) + }) + + lhs_offset_first_element_in_bytes += MMUL_K0 * sizeof(DATA_TYPE); + rhs_offset_first_element_in_bytes += MMUL_N0 * sizeof(DATA_TYPE); + } + + // For threads "outside" of the dst bound, we do not write but we have to "read" (arm_matrix_multiply). That's why this needs to happen after arm_matrix_multiply + if(dst_x_unclamped >= N || dst_y_unclamped >= M) + { + return; + } + +#if defined(HALF_PRECISION) + TILE(DATA_TYPE, M0, N0, c); + + // Conversion required for the half precision + LOOP_UNROLLING(int, m0, 0, 1, M0, + { + LOOP_UNROLLING(int, n0, 0, 1, N0, + { + c[m0].s[n0] = c_f32[m0].s[n0]; + }) + }) +#else // defined(HALF_PRECISION) +#define c c_f32 +#endif // defined(HALF_PRECISION) + +#ifdef BIAS + perform_bias_addition(bias_ptr, bias_offset_first_element_in_bytes, c, dst_x); +#endif // defined(BIAS) + + if(dst_x + N0 <= N || N0_LEFTOVER == 0) + { + LOOP_UNROLLING(int, m0, 0, 1, M0, + { + if(dst_y + m0 < M || M0_LEFTOVER == 0) + { + VSTORE(N0) + (c[m0].v, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_first_element_in_bytes + m0 * dst_stride_y)); + } + }) + } + else + { + LOOP_UNROLLING(int, m0, 0, 1, M0, + { + if(dst_y + m0 < M || M0_LEFTOVER == 0) + { + VSTORE_PARTIAL(N0, N0_LEFTOVER) + (c[m0].v, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_first_element_in_bytes + m0 * dst_stride_y)); + } + }) + } + +#undef MMUL_BLOCK_SIZE +} +#endif // defined(MAT_MUL_NATIVE_MMUL_NT_T) + +#if defined(MAT_MUL_NATIVE_MMUL_T_T) +/** This OpenCL kernel performs the batch matrix multiplication (BatchMatMul) using MMUL: LHS non-transposed, RHS transposed - buffer only + * + * @note the "batch" here expresses the number of matrix multiplications to run in parallel. However, it + * should NOT be confused with the batch size of the model. For NHWC the "batch" is the "H" dimension + * @note The data type must be passed at compile time using -DDATA_TYPE (e.g. -DDATA_TYPE=float) + * @note The tile's dimensions used for the LHS and RHS matrices (M0, N0 and K0) must be passed at compile time using -DN0, -DM0 and -DK0 (e.g. -DN0=8, -DM0=4, -DK0=1). + * @note The number of leftover outputs rows/columns must be passed using -DN0_LEFTOVER and -DM0_LEFTOVER (e.g. -DN0_LEFTOVER=2, -DM0_LEFTOVER=3) + * @note The MMUL block dimension (MMUL_M0, MMUL_N0, MMUL_K0) must be passed at compile time using -DMMUL_M0, -DMMUL_N0 and -DMMUL_K0 (e.g. -DMMUL_M0=4, -DMMUL_N0=4, -DMMUL_K0=4). + * @note The kernel name in uppercase must be passed at compile time (e.g. -DMAT_MUL_NATIVE_MMUL_NT_T) + * @note Only the following configurations of M0, N0 and K0 are currently supported: + * - M0 = 1, 2, 3, 4, 8, 16 + * - N0 = 1, 2, 3, 4, 8, 16 + * - K0 = 1 + * @note Values > 8 for M0 are not expected to be efficient + * + * @param[in] lhs_ptr Pointer to the lhs matrix. Supported data types: F32/F16 + * @param[in] lhs_stride_y Stride of the lhs matrix in Y (2nd) dimension (in bytes) + * @param[in] lhs_stride_z Stride of the lhs tensor in Z (3rd) dimension (in bytes) + * @param[in] lhs_w The width of the lhs tensor + * @param[in] lhs_h The height of the lhs tensor + * @param[in] lhs_n Number of the matrices (buffers) in the batch + * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the lhs matrix + * @param[in] rhs_ptr Pointer to the rhs matrix. Supported data types: same as @p lhs_ptr + * @param[in] rhs_stride_y Stride of the rhs matrix in Y (2nd) dimension (in bytes) + * @param[in] rhs_stride_z Stride of the rhs tensor in Z (3rd) dimension (in bytes) + * @param[in] rhs_w The width of the rhs tensor + * @param[in] rhs_h The height of the rhs tensor + * @param[in] rhs_n Number of the matrices (buffers) in the batch + * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the rhs matrix + * @param[in] bias_ptr (Optional) Pointer to the bias tensor. Supported data type: same as @p lhs_ptr + * @param[in] bias_stride_y (Optional) Stride of the bias tensor in Y dimension (in bytes) + * @param[in] bias_stride_z (Optional) Stride of the bias tensor in Z dimension (in bytes) + * @param[in] bias_w (Optional) The size of the width dimension of the bias tensor + * @param[in] bias_h (Optional) The size of the height dimension of the bias tensor + * @param[in] bias_n (Optional) The size of the depth dimension of the bias tensor + * @param[in] bias_offset_first_element_in_bytes (Optional) The offset of the first element in the bias tensor + * @param[out] dst_ptr Pointer to the dst matrix. Supported data types: same as @p lhs_ptr + * @param[in] dst_stride_y Stride of the dst matrix in Y (2nd) dimension (in bytes) + * @param[in] dst_stride_z Stride of the dst tensor in Z (3rd) dimension (in bytes) + * @param[in] dst_w The width of the dst tensor + * @param[in] dst_h The height of the dst tensor + * @param[in] dst_n Number of the matrices (buffers) in the batch + * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the dst matrix + * @param[in] M Number of rows in LHS matrix + * @param[in] N Number of columns in RHS matrix + * @param[in] K Number of rows in LHS matrix and columns in RHS matrix, which is multiple of MMUL_K0. + */ +__kernel void mat_mul_native_mmul_t_t( + TENSOR3D_T(lhs, BUFFER), + TENSOR3D_T(rhs, BUFFER), +#ifdef BIAS + TENSOR3D_T(bias, BUFFER), +#endif // defined(BIAS) + TENSOR3D_T(dst, BUFFER), + const int M, + const int N, + const int K) +{ +#define MMUL_BLOCK_SIZE (MMUL_M0 * MMUL_N0) + // For explanations on how this kernel works, please refer to NT/NT kernel. This kernel makes little modifications to it. + + const uint x0 = get_global_id(0); // [0, (N / N0) * MMUL_M0) + // The upper limit is a simplified version of (N / N0) / MMUL_N0) * MMUL_BLOCK_SIZE) + const uint y0 = get_global_id(1); // [0, (M / M0) / MMUL_M0) + const uint z = get_global_id(2); // Batch + + // Get block coordinates + const uint section_x = (x0 / MMUL_BLOCK_SIZE); + const uint section_y = y0; + + // Get thread coordinates within a block + const uint thread_id = (x0 % MMUL_BLOCK_SIZE); + const uint thread_x = thread_id % MMUL_N0; + const uint thread_y = (thread_id / MMUL_N0); + + // Starting destination coordinates + // Note: We need to clamp dst_x and dst_y because we always need to execute a complete MMUL block! Only after the matrix multiplication + // part can we exit the kernel if it is out-of-bound. Remember, we have a cooperative matrix multiplication. Therefore, we need a full block to get the correct results + // Although we will never write out-of-bound, we still need this clamp to ensure that we do not read out-of-bound either. + const uint dst_x_unclamped = thread_x * N0 + section_x * N0 * MMUL_N0; + const uint dst_y_unclamped = thread_y * M0 + section_y * M0 * MMUL_M0; + const uint dst_x = min(dst_x_unclamped, (uint)(N - N0)); + const uint dst_y = min(dst_y_unclamped, (uint)(M - M0)); + + // Starting LHS coordinates + const uint lhs_x = dst_y; + const uint lhs_y = thread_x; + + // Starting RHS coordinates + const uint rhs_x = thread_y; + const uint rhs_y = dst_x; + + // Compute LHS/RHS/DST matrix address + lhs_offset_first_element_in_bytes += lhs_x * sizeof(DATA_TYPE) + lhs_y * lhs_stride_y + z * lhs_stride_z; + rhs_offset_first_element_in_bytes += rhs_x * sizeof(DATA_TYPE) + rhs_y * rhs_stride_y + z * rhs_stride_z; + dst_offset_first_element_in_bytes += dst_x * sizeof(DATA_TYPE) + dst_y * dst_stride_y + z * dst_stride_z; + + // Initialize the accumulators + // MMUL extension accumulate the result in F32 for both F32 and F16 + TILE(float, M0, N0, c_f32); + + LOOP_UNROLLING(int, i, 0, 1, M0, + { + c_f32[i].v = 0; + }) + + for(int k = 0; k < K; k += MMUL_K0) + { + // A tile of K0xM0 but K0 must be set to 1 + TILE(DATA_TYPE, 1, M0, a); + // A tile of N0xK0 but K0 must be set to 1 + TILE(DATA_TYPE, N0, 1, b); + + // Load tile from the lhs/rhs tensors + T_LOAD(DATA_TYPE, 1, M0, BUFFER, lhs, 0, 0, 1, lhs_stride_y, a); + T_LOAD(DATA_TYPE, N0, 1, BUFFER, rhs, 0, 0, 1, rhs_stride_y, b); + + LOOP_UNROLLING(int, m0, 0, 1, M0, + { + LOOP_UNROLLING(int, n0, 0, 1, N0, + { + c_f32[m0].s[n0] = arm_matrix_multiply(a[0].s[m0], b[n0].s[0], c_f32[m0].s[n0]); + }) + }) + + lhs_offset_first_element_in_bytes += MMUL_K0 * lhs_stride_y; + rhs_offset_first_element_in_bytes += MMUL_N0 * sizeof(DATA_TYPE); + } + + // For threads "outside" of the dst bound, we do not write but we have to "read" (arm_matrix_multiply). That's why this needs to happen after arm_matrix_multiply + if(dst_x_unclamped >= N || dst_y_unclamped >= M) + { + return; + } + +#if defined(HALF_PRECISION) + TILE(DATA_TYPE, M0, N0, c); + + // Conversion required for the half precision + LOOP_UNROLLING(int, m0, 0, 1, M0, + { + LOOP_UNROLLING(int, n0, 0, 1, N0, + { + c[m0].s[n0] = c_f32[m0].s[n0]; + }) + }) +#else // defined(HALF_PRECISION) +#define c c_f32 +#endif // defined(HALF_PRECISION) + +#ifdef BIAS + perform_bias_addition(bias_ptr, bias_offset_first_element_in_bytes, c, dst_x); +#endif // defined(BIAS) + + if(dst_x + N0 <= N || N0_LEFTOVER == 0) + { + LOOP_UNROLLING(int, m0, 0, 1, M0, + { + if(dst_y + m0 < M || M0_LEFTOVER == 0) + { + VSTORE(N0) + (c[m0].v, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_first_element_in_bytes + m0 * dst_stride_y)); + } + }) + } + else + { + LOOP_UNROLLING(int, m0, 0, 1, M0, + { + if(dst_y + m0 < M || M0_LEFTOVER == 0) + { + VSTORE_PARTIAL(N0, N0_LEFTOVER) + (c[m0].v, 0, (__global DATA_TYPE *)(dst_ptr + dst_offset_first_element_in_bytes + m0 * dst_stride_y)); + } + }) + } + +#undef MMUL_BLOCK_SIZE +} +#endif // defined(MAT_MUL_NATIVE_MMUL_T_T) diff --git a/src/core/CL/cl_kernels/common/mat_mul_quantized.cl b/src/core/CL/cl_kernels/common/mat_mul_quantized.cl index 0c3cbca9a62b35fed0df389d3cf11063fcc6cc25..7f81ac45490e1accfb23eead59756a49645b3cb8 100644 --- a/src/core/CL/cl_kernels/common/mat_mul_quantized.cl +++ b/src/core/CL/cl_kernels/common/mat_mul_quantized.cl @@ -21,9 +21,25 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ +#include "activation_float_helpers.h" #include "helpers.h" #include "tile_helpers.h" +#ifdef BIAS +// This function performs in-place bias addition for integer datatype when bias is enabled. +// Note The tile's dimensions used for the LHS and RHS matrices (M0, N0) must be passed at compile time using -DN0, -DM0 (e.g. -DN0=8, -DM0=4). +inline void perform_bias_addition(uchar *bias_ptr, uint bias_offset_first_element_in_bytes, TILE(int, M0, N0, acc), uint x) +{ + TILE(int, 1, N0, bias_tile); + + // below expands to use bias_ptr and bias_offset_first_element_in_bytes + T_LOAD(int, 1, N0, BUFFER, bias, x, 0, 1, 0, bias_tile); + + // c = c + bias[broadcasted] + T_ELTWISE_BROADCAST_ADD_X(int, M0, N0, acc, bias_tile, acc); +} +#endif // defined(BIAS) + #if defined(MAT_MUL_NATIVE_QUANTIZED_NT_NT) /** This OpenCL kernel performs the batch matrix multiplication (BatchMatMul): LHS non-transposed, RHS non-transposed - buffer only * @@ -32,6 +48,8 @@ * @note The data type must be passed at compile time using -DDATA_TYPE (e.g. -DDATA_TYPE=uchar) * @note The block's dimensions used for the LHS and RHS matrices (M0, N0 and K0) must be passed at compile time using -DN0, -DM0 and -DK0 (e.g. -DN0=8, -DM0=4, -DK0=4). * @note The number of leftover outputs rows/columns must be passed using -DPARTIAL_STORE_N0 and -DPARTIAL_STORE_M0 (e.g. -DPARTIAL_STORE_N0=2, -DPARTIAL_STORE_M0=3) + * @note The fused activation function used should be passed with -DACTIVATION_TYPE, -DA_VAL and -DB_VAL are used for min and max output with the relu and bounded relu operations. + * @note The value of 0 in quantized format is equivalent to the quantization offset of the output data. This should be passed with -DZERO_POINT * @note The dimension K must be passed at compile time using -DK (e.g. -DK=6) * @note The kernel name in uppercase must be passed at compile time (e.g. -DMAT_MUL_NATIVE_QUANTIZED_NT_NT) * @note Only the following configurations of M0, N0 and K0 are currently supported: @@ -40,31 +58,41 @@ * - K0 = 1, 2, 3, 4, 8, 16 * @note Values > 8 for M0 are not expected to be efficient * - * @param[in] lhs_ptr Pointer to the lhs matrix. Supported data types: QASYMM8_SIGNED/QASYMM8 - * @param[in] lhs_stride_y Stride of the lhs matrix in Y (2nd) dimension (in bytes) - * @param[in] lhs_stride_z Stride of the lhs tensor in Z (3rd) dimension (in bytes) - * @param[in] lhs_w The width of the lhs tensor - * @param[in] lhs_h The height of the lhs tensor - * @param[in] lhs_n Number of the matrices (buffers) in the batch - * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the lhs matrix - * @param[in] rhs_ptr Pointer to the rhs matrix. Supported data types: same as @p lhs_ptr - * @param[in] rhs_stride_y Stride of the rhs matrix in Y (2nd) dimension (in bytes) - * @param[in] rhs_stride_z Stride of the rhs tensor in Z (3rd) dimension (in bytes) - * @param[in] rhs_w The width of the rhs tensor - * @param[in] rhs_h The height of the rhs tensor - * @param[in] rhs_n Number of the matrices (buffers) in the batch - * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the rhs matrix - * @param[out] dst_ptr Pointer to the dst matrix. Supported data types: same as @p lhs_ptr - * @param[in] dst_stride_y Stride of the dst matrix in Y (2nd) dimension (in bytes) - * @param[in] dst_stride_z Stride of the dst tensor in Z (3rd) dimension (in bytes) - * @param[in] dst_w The width of the dst tensor - * @param[in] dst_h The height of the dst tensor - * @param[in] dst_n Number of the matrices (buffers) in the batch - * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the dst matrix + * @param[in] lhs_ptr Pointer to the lhs matrix. Supported data types: QASYMM8_SIGNED/QASYMM8 + * @param[in] lhs_stride_y Stride of the lhs matrix in Y (2nd) dimension (in bytes) + * @param[in] lhs_stride_z Stride of the lhs tensor in Z (3rd) dimension (in bytes) + * @param[in] lhs_w The width of the lhs tensor + * @param[in] lhs_h The height of the lhs tensor + * @param[in] lhs_n Number of the matrices (buffers) in the batch + * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the lhs matrix + * @param[in] rhs_ptr Pointer to the rhs matrix. Supported data types: same as @p lhs_ptr + * @param[in] rhs_stride_y Stride of the rhs matrix in Y (2nd) dimension (in bytes) + * @param[in] rhs_stride_z Stride of the rhs tensor in Z (3rd) dimension (in bytes) + * @param[in] rhs_w The width of the rhs tensor + * @param[in] rhs_h The height of the rhs tensor + * @param[in] rhs_n Number of the matrices (buffers) in the batch + * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the rhs matrix + * @param[in] bias_ptr (Optional) Pointer to the bias tensor. Supported data type: same as @p lhs_ptr + * @param[in] bias_stride_y (Optional) Stride of the bias tensor in Y dimension (in bytes) + * @param[in] bias_stride_z (Optional) Stride of the bias tensor in Z dimension (in bytes) + * @param[in] bias_w (Optional) The size of the width dimension of the bias tensor + * @param[in] bias_h (Optional) The size of the height dimension of the bias tensor + * @param[in] bias_n (Optional) The size of the depth dimension of the bias tensor + * @param[in] bias_offset_first_element_in_bytes (Optional) The offset of the first element in the bias tensor + * @param[out] dst_ptr Pointer to the dst matrix. Supported data types: same as @p lhs_ptr + * @param[in] dst_stride_y Stride of the dst matrix in Y (2nd) dimension (in bytes) + * @param[in] dst_stride_z Stride of the dst tensor in Z (3rd) dimension (in bytes) + * @param[in] dst_w The width of the dst tensor + * @param[in] dst_h The height of the dst tensor + * @param[in] dst_n Number of the matrices (buffers) in the batch + * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the dst matrix */ __kernel void mat_mul_native_quantized_nt_nt( TENSOR3D_T(lhs, BUFFER), TENSOR3D_T(rhs, BUFFER), +#ifdef BIAS + TENSOR3D_T(bias, BUFFER), +#endif // defined(BIAS) TENSOR3D_T(dst, BUFFER)) { const uint x = GET_SPATIAL_IDX(0, N0, PARTIAL_STORE_N0); @@ -187,17 +215,23 @@ __kernel void mat_mul_native_quantized_nt_nt( { LOOP_UNROLLING(int, j, 0, 1, N0, { - acc[i].s[j] += ((int)RHS_OFFSET) * a_sum[0].s[i] + ((int)(LHS_OFFSET)) * b_sum[0].s[j]; + acc[i].s[j] -= ((int)RHS_OFFSET) * a_sum[0].s[i] + ((int)(LHS_OFFSET)) * b_sum[0].s[j]; }) }) const bool x_cond = PARTIAL_STORE_N0 != 0 && get_global_id(0) == 0; const bool y_cond = PARTIAL_STORE_M0 != 0 && get_global_id(1) == 0; +#ifdef BIAS + perform_bias_addition(bias_ptr, bias_offset_first_element_in_bytes, acc, x); +#endif // defined(BIAS) + // Quantize the tile TILE(DATA_TYPE, M0, N0, accq); T_QUANTIZE8_ASYMMETRIC(int, DATA_TYPE, M0, N0, DST_OFFSET, DST_SHIFT, DST_MULTIPLIER, acc, accq); + T_ACTIVATION_QUANTIZED(DATA_TYPE, M0, N0, ACTIVATION_TYPE, ZERO_POINT, A_VAL, B_VAL, accq, accq); + TILE(int, M0, 1, indirect_buffer); LOOP_UNROLLING(int, _i, 0, 1, M0, { @@ -216,6 +250,8 @@ __kernel void mat_mul_native_quantized_nt_nt( * @note The data type must be passed at compile time using -DDATA_TYPE (e.g. -DDATA_TYPE=uchar) * @note The block's dimensions used for the LHS and RHS matrices (M0, N0 and K0) must be passed at compile time using -DN0, -DM0 and -DK0 (e.g. -DN0=8, -DM0=4, -DK0=4). * @note The number of leftover outputs rows/columns must be passed using -DPARTIAL_STORE_N0 and -DPARTIAL_STORE_M0 (e.g. -DPARTIAL_STORE_N0=2, -DPARTIAL_STORE_M0=3) + * @note The fused activation function used should be passed with -DACTIVATION_TYPE, -DA_VAL and -DB_VAL are used for min and max output bounded activation functions. + * @note The value of 0 in quantized format is equivalent to the quantization offset of the output data. This should be passed with -DZERO_POINT * @note The dimension K must be passed at compile time using -DK (e.g. -DK=6) * @note The kernel name in uppercase must be passed at compile time (e.g. -DMAT_MUL_NATIVE_QUANTIZED_NT_T) * @note Only the following configurations of M0, N0 and K0 are currently supported: @@ -224,31 +260,41 @@ __kernel void mat_mul_native_quantized_nt_nt( * - K0 = 1, 2, 3, 4, 8, 16 * @note Values > 8 for M0, N0, K0 are not expected to be efficient * - * @param[in] lhs_ptr Pointer to the lhs matrix. Supported data types: QASYMM8/QASYMM8_SIGNED - * @param[in] lhs_stride_y Stride of the lhs matrix in Y (2nd) dimension (in bytes) - * @param[in] lhs_stride_z Stride of the lhs tensor in Z (3rd) dimension (in bytes) - * @param[in] lhs_w The width of the lhs tensor - * @param[in] lhs_h The height of the lhs tensor - * @param[in] lhs_n Number of the matrices (buffers) in the batch - * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the lhs matrix - * @param[in] rhs_ptr Pointer to the rhs matrix. Supported data types: same as @p lhs_ptr - * @param[in] rhs_stride_y Stride of the rhs matrix in Y (2nd) dimension (in bytes) - * @param[in] rhs_stride_z Stride of the rhs tensor in Z (3rd) dimension (in bytes) - * @param[in] rhs_w The width of the rhs tensor - * @param[in] rhs_h The height of the rhs tensor - * @param[in] rhs_n Number of the matrices (buffers) in the batch - * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the rhs matrix - * @param[out] dst_ptr Pointer to the dst matrix. Supported data types: same as @p lhs_ptr - * @param[in] dst_stride_y Stride of the dst matrix in Y (2nd) dimension (in bytes) - * @param[in] dst_stride_z Stride of the dst tensor in Z (3rd) dimension (in bytes) - * @param[in] dst_w The width of the dst tensor - * @param[in] dst_h The height of the dst tensor - * @param[in] dst_n Number of the matrices (buffers) in the batch - * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the dst matrix + * @param[in] lhs_ptr Pointer to the lhs matrix. Supported data types: QASYMM8/QASYMM8_SIGNED + * @param[in] lhs_stride_y Stride of the lhs matrix in Y (2nd) dimension (in bytes) + * @param[in] lhs_stride_z Stride of the lhs tensor in Z (3rd) dimension (in bytes) + * @param[in] lhs_w The width of the lhs tensor + * @param[in] lhs_h The height of the lhs tensor + * @param[in] lhs_n Number of the matrices (buffers) in the batch + * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the lhs matrix + * @param[in] rhs_ptr Pointer to the rhs matrix. Supported data types: same as @p lhs_ptr + * @param[in] rhs_stride_y Stride of the rhs matrix in Y (2nd) dimension (in bytes) + * @param[in] rhs_stride_z Stride of the rhs tensor in Z (3rd) dimension (in bytes) + * @param[in] rhs_w The width of the rhs tensor + * @param[in] rhs_h The height of the rhs tensor + * @param[in] rhs_n Number of the matrices (buffers) in the batch + * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the rhs matrix + * @param[in] bias_ptr (Optional) Pointer to the bias tensor. Supported data type: same as @p lhs_ptr + * @param[in] bias_stride_y (Optional) Stride of the bias tensor in Y dimension (in bytes) + * @param[in] bias_stride_z (Optional) Stride of the bias tensor in Z dimension (in bytes) + * @param[in] bias_w (Optional) The size of the width dimension of the bias tensor + * @param[in] bias_h (Optional) The size of the height dimension of the bias tensor + * @param[in] bias_n (Optional) The size of the depth dimension of the bias tensor + * @param[in] bias_offset_first_element_in_bytes (Optional) The offset of the first element in the bias tensor + * @param[out] dst_ptr Pointer to the dst matrix. Supported data types: same as @p lhs_ptr + * @param[in] dst_stride_y Stride of the dst matrix in Y (2nd) dimension (in bytes) + * @param[in] dst_stride_z Stride of the dst tensor in Z (3rd) dimension (in bytes) + * @param[in] dst_w The width of the dst tensor + * @param[in] dst_h The height of the dst tensor + * @param[in] dst_n Number of the matrices (buffers) in the batch + * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the dst matrix */ __kernel void mat_mul_native_quantized_nt_t( TENSOR3D_T(lhs, BUFFER), TENSOR3D_T(rhs, BUFFER), +#ifdef BIAS + TENSOR3D_T(bias, BUFFER), +#endif // defined(BIAS) TENSOR3D_T(dst, BUFFER)) { const uint x = GET_SPATIAL_IDX(0, N0, PARTIAL_STORE_N0); @@ -315,7 +361,7 @@ __kernel void mat_mul_native_quantized_nt_t( rhs_offset_first_element_in_bytes += K0 * sizeof(DATA_TYPE); } -#if ((K % K0) != 0) +#if((K % K0) != 0) // Leftover loop for(; k < K; ++k) { @@ -363,17 +409,23 @@ __kernel void mat_mul_native_quantized_nt_t( { LOOP_UNROLLING(int, j, 0, 1, N0, { - acc[i].s[j] += ((int)(RHS_OFFSET)) * a_sum[0].s[i] + ((int)(LHS_OFFSET)) * b_sum[0].s[j]; + acc[i].s[j] -= ((int)(RHS_OFFSET)) * a_sum[0].s[i] + ((int)(LHS_OFFSET)) * b_sum[0].s[j]; }) }) const bool x_cond = PARTIAL_STORE_N0 != 0 && get_global_id(0) == 0; const bool y_cond = PARTIAL_STORE_M0 != 0 && get_global_id(1) == 0; +#ifdef BIAS + perform_bias_addition(bias_ptr, bias_offset_first_element_in_bytes, acc, x); +#endif // defined(BIAS) + // Quantize the tile TILE(DATA_TYPE, M0, N0, accq); T_QUANTIZE8_ASYMMETRIC(int, DATA_TYPE, M0, N0, DST_OFFSET, DST_SHIFT, DST_MULTIPLIER, acc, accq); + T_ACTIVATION_QUANTIZED(DATA_TYPE, M0, N0, ACTIVATION_TYPE, ZERO_POINT, A_VAL, B_VAL, accq, accq); + TILE(int, M0, 1, indirect_buffer); LOOP_UNROLLING(int, _i, 0, 1, M0, { @@ -392,6 +444,8 @@ __kernel void mat_mul_native_quantized_nt_t( * @note The data type must be passed at compile time using -DDATA_TYPE (e.g. -DDATA_TYPE=uchar) * @note The block's dimensions used for the LHS and RHS matrices (M0, N0 and K0) must be passed at compile time using -DN0, -DM0 and -DK0 (e.g. -DN0=8, -DM0=4, -DK0=4). * @note The number of leftover outputs rows/columns must be passed using -DPARTIAL_STORE_N0 and -DPARTIAL_STORE_M0 (e.g. -DPARTIAL_STORE_N0=2, -DPARTIAL_STORE_M0=3) + * @note The fused activation function used should be passed with -DACTIVATION_TYPE, -DA_VAL and -DB_VAL are used for min and max output with the relu and bounded relu operations. + * @note The value of 0 in quantized format is equivalent to the quantization offset of the output data. This should be passed with -DZERO_POINT * @note The dimension K must be passed at compile time using -DK (e.g. -DK=6) * @note The kernel name in uppercase must be passed at compile time (e.g. -DMAT_MUL_NATIVE_QUANTIZED_T_NT) * @note Only the following configurations of M0, N0 and K0 are currently supported: @@ -400,31 +454,41 @@ __kernel void mat_mul_native_quantized_nt_t( * - K0 = 1, 2, 3, 4, 8, 16 * @note Values > 8 for M0, N0 and K0 are not expected to be efficient * - * @param[in] lhs_ptr Pointer to the lhs matrix. Supported data types: QASYMM8/QASYMM8_SIGNED - * @param[in] lhs_stride_y Stride of the lhs matrix in Y (2nd) dimension (in bytes) - * @param[in] lhs_stride_z Stride of the lhs tensor in Z (3rd) dimension (in bytes) - * @param[in] lhs_w The width of the lhs tensor - * @param[in] lhs_h The height of the lhs tensor - * @param[in] lhs_n Number of the matrices (buffers) in the batch - * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the lhs matrix - * @param[in] rhs_ptr Pointer to the rhs matrix. Supported data types: same as @p lhs_ptr - * @param[in] rhs_stride_y Stride of the rhs matrix in Y (2nd) dimension (in bytes) - * @param[in] rhs_stride_z Stride of the rhs tensor in Z (3rd) dimension (in bytes) - * @param[in] rhs_w The width of the rhs tensor - * @param[in] rhs_h The height of the rhs tensor - * @param[in] rhs_n Number of the matrices (buffers) in the batch - * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the rhs matrix - * @param[out] dst_ptr Pointer to the dst matrix. Supported data types: same as @p lhs_ptr - * @param[in] dst_stride_y Stride of the dst matrix in Y (2nd) dimension (in bytes) - * @param[in] dst_stride_z Stride of the dst tensor in Z (3rd) dimension (in bytes) - * @param[in] dst_w The width of the dst tensor - * @param[in] dst_h The height of the dst tensor - * @param[in] dst_n Number of the matrices (buffers) in the batch - * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the dst matrix + * @param[in] lhs_ptr Pointer to the lhs matrix. Supported data types: QASYMM8/QASYMM8_SIGNED + * @param[in] lhs_stride_y Stride of the lhs matrix in Y (2nd) dimension (in bytes) + * @param[in] lhs_stride_z Stride of the lhs tensor in Z (3rd) dimension (in bytes) + * @param[in] lhs_w The width of the lhs tensor + * @param[in] lhs_h The height of the lhs tensor + * @param[in] lhs_n Number of the matrices (buffers) in the batch + * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the lhs matrix + * @param[in] rhs_ptr Pointer to the rhs matrix. Supported data types: same as @p lhs_ptr + * @param[in] rhs_stride_y Stride of the rhs matrix in Y (2nd) dimension (in bytes) + * @param[in] rhs_stride_z Stride of the rhs tensor in Z (3rd) dimension (in bytes) + * @param[in] rhs_w The width of the rhs tensor + * @param[in] rhs_h The height of the rhs tensor + * @param[in] rhs_n Number of the matrices (buffers) in the batch + * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the rhs matrix + * @param[in] bias_ptr (Optional) Pointer to the bias tensor. Supported data type: same as @p lhs_ptr + * @param[in] bias_stride_y (Optional) Stride of the bias tensor in Y dimension (in bytes) + * @param[in] bias_stride_z (Optional) Stride of the bias tensor in Z dimension (in bytes) + * @param[in] bias_w (Optional) The size of the width dimension of the bias tensor + * @param[in] bias_h (Optional) The size of the height dimension of the bias tensor + * @param[in] bias_n (Optional) The size of the depth dimension of the bias tensor + * @param[in] bias_offset_first_element_in_bytes (Optional) The offset of the first element in the bias tensor + * @param[out] dst_ptr Pointer to the dst matrix. Supported data types: same as @p lhs_ptr + * @param[in] dst_stride_y Stride of the dst matrix in Y (2nd) dimension (in bytes) + * @param[in] dst_stride_z Stride of the dst tensor in Z (3rd) dimension (in bytes) + * @param[in] dst_w The width of the dst tensor + * @param[in] dst_h The height of the dst tensor + * @param[in] dst_n Number of the matrices (buffers) in the batch + * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the dst matrix */ __kernel void mat_mul_native_quantized_t_nt( TENSOR3D_T(lhs, BUFFER), TENSOR3D_T(rhs, BUFFER), +#ifdef BIAS + TENSOR3D_T(bias, BUFFER), +#endif // defined(BIAS) TENSOR3D_T(dst, BUFFER)) { const uint x = GET_SPATIAL_IDX(0, N0, PARTIAL_STORE_N0); @@ -541,17 +605,23 @@ __kernel void mat_mul_native_quantized_t_nt( { LOOP_UNROLLING(int, j, 0, 1, N0, { - acc[i].s[j] += ((int)(RHS_OFFSET)) * a_sum[0].s[i] + ((int)(LHS_OFFSET)) * b_sum[0].s[j]; + acc[i].s[j] -= ((int)(RHS_OFFSET)) * a_sum[0].s[i] + ((int)(LHS_OFFSET)) * b_sum[0].s[j]; }) }) const bool x_cond = PARTIAL_STORE_N0 != 0 && get_global_id(0) == 0; const bool y_cond = PARTIAL_STORE_M0 != 0 && get_global_id(1) == 0; +#ifdef BIAS + perform_bias_addition(bias_ptr, bias_offset_first_element_in_bytes, acc, x); +#endif // defined(BIAS) + // Quantize the tile TILE(DATA_TYPE, M0, N0, accq); T_QUANTIZE8_ASYMMETRIC(int, DATA_TYPE, M0, N0, DST_OFFSET, DST_SHIFT, DST_MULTIPLIER, acc, accq); + T_ACTIVATION_QUANTIZED(DATA_TYPE, M0, N0, ACTIVATION_TYPE, ZERO_POINT, A_VAL, B_VAL, accq, accq); + TILE(int, M0, 1, indirect_buffer); LOOP_UNROLLING(int, _i, 0, 1, M0, { @@ -570,6 +640,8 @@ __kernel void mat_mul_native_quantized_t_nt( * @note The data type must be passed at compile time using -DDATA_TYPE (e.g. -DDATA_TYPE=uchar) * @note The block's dimensions used for the LHS and RHS matrices (M0, N0 and K0) must be passed at compile time using -DN0, -DM0 and -DK0 (e.g. -DN0=8, -DM0=4, -DK0=4). * @note The number of leftover outputs rows/columns must be passed using -DPARTIAL_STORE_N0 and -DPARTIAL_STORE_M0 (e.g. -DPARTIAL_STORE_N0=2, -DPARTIAL_STORE_M0=3) + * @note The fused activation function used should be passed with -DACTIVATION_TYPE, -DA_VAL and -DB_VAL are used for min and max output with the relu and bounded relu operations. + * @note The value of 0 in quantized format is equivalent to the quantization offset of the output data. This should be passed with -DZERO_POINT * @note The dimension K must be passed at compile time using -DK (e.g. -DK=6) * @note The kernel name in uppercase must be passed at compile time (e.g. -DMAT_MUL_NATIVE_QUANTIZED_T_T) * @note Only the following configurations of M0, N0 and K0 are currently supported: @@ -578,31 +650,41 @@ __kernel void mat_mul_native_quantized_t_nt( * - K0 = 1, 2, 3, 4, 8, 16 * @note Values > 8 for M0, N0 and K0 are not expected to be efficient * - * @param[in] lhs_ptr Pointer to the lhs matrix. Supported data types: QASYMM8/QASYMM8_SIGNED - * @param[in] lhs_stride_y Stride of the lhs matrix in Y (2nd) dimension (in bytes) - * @param[in] lhs_stride_z Stride of the lhs tensor in Z (3rd) dimension (in bytes) - * @param[in] lhs_w The width of the lhs tensor - * @param[in] lhs_h The height of the lhs tensor - * @param[in] lhs_n Number of the matrices (buffers) in the batch - * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the lhs matrix - * @param[in] rhs_ptr Pointer to the rhs matrix. Supported data types: same as @p lhs_ptr - * @param[in] rhs_stride_y Stride of the rhs matrix in Y (2nd) dimension (in bytes) - * @param[in] rhs_stride_z Stride of the rhs tensor in Z (3rd) dimension (in bytes) - * @param[in] rhs_w The width of the rhs tensor - * @param[in] rhs_h The height of the rhs tensor - * @param[in] rhs_n Number of the matrices (buffers) in the batch - * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the rhs matrix - * @param[out] dst_ptr Pointer to the dst matrix. Supported data types: same as @p lhs_ptr - * @param[in] dst_stride_y Stride of the dst matrix in Y (2nd) dimension (in bytes) - * @param[in] dst_stride_z Stride of the dst tensor in Z (3rd) dimension (in bytes) - * @param[in] dst_w The width of the dst tensor - * @param[in] dst_h The height of the dst tensor - * @param[in] dst_n Number of the matrices (buffers) in the batch - * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the dst matrix + * @param[in] lhs_ptr Pointer to the lhs matrix. Supported data types: QASYMM8/QASYMM8_SIGNED + * @param[in] lhs_stride_y Stride of the lhs matrix in Y (2nd) dimension (in bytes) + * @param[in] lhs_stride_z Stride of the lhs tensor in Z (3rd) dimension (in bytes) + * @param[in] lhs_w The width of the lhs tensor + * @param[in] lhs_h The height of the lhs tensor + * @param[in] lhs_n Number of the matrices (buffers) in the batch + * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the lhs matrix + * @param[in] rhs_ptr Pointer to the rhs matrix. Supported data types: same as @p lhs_ptr + * @param[in] rhs_stride_y Stride of the rhs matrix in Y (2nd) dimension (in bytes) + * @param[in] rhs_stride_z Stride of the rhs tensor in Z (3rd) dimension (in bytes) + * @param[in] rhs_w The width of the rhs tensor + * @param[in] rhs_h The height of the rhs tensor + * @param[in] rhs_n Number of the matrices (buffers) in the batch + * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the rhs matrix + * @param[in] bias_ptr (Optional) Pointer to the bias tensor. Supported data type: same as @p lhs_ptr + * @param[in] bias_stride_y (Optional) Stride of the bias tensor in Y dimension (in bytes) + * @param[in] bias_stride_z (Optional) Stride of the bias tensor in Z dimension (in bytes) + * @param[in] bias_w (Optional) The size of the width dimension of the bias tensor + * @param[in] bias_h (Optional) The size of the height dimension of the bias tensor + * @param[in] bias_n (Optional) The size of the depth dimension of the bias tensor + * @param[in] bias_offset_first_element_in_bytes (Optional) The offset of the first element in the bias tensor + * @param[out] dst_ptr Pointer to the dst matrix. Supported data types: same as @p lhs_ptr + * @param[in] dst_stride_y Stride of the dst matrix in Y (2nd) dimension (in bytes) + * @param[in] dst_stride_z Stride of the dst tensor in Z (3rd) dimension (in bytes) + * @param[in] dst_w The width of the dst tensor + * @param[in] dst_h The height of the dst tensor + * @param[in] dst_n Number of the matrices (buffers) in the batch + * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the dst matrix */ __kernel void mat_mul_native_quantized_t_t( TENSOR3D_T(lhs, BUFFER), TENSOR3D_T(rhs, BUFFER), +#ifdef BIAS + TENSOR3D_T(bias, BUFFER), +#endif // defined(BIAS) TENSOR3D_T(dst, BUFFER)) { const uint x = GET_SPATIAL_IDX(0, N0, PARTIAL_STORE_N0); @@ -723,17 +805,23 @@ __kernel void mat_mul_native_quantized_t_t( { LOOP_UNROLLING(int, j, 0, 1, N0, { - acc[i].s[j] += ((int)RHS_OFFSET) * a_sum[0].s[i] + ((int)(LHS_OFFSET)) * b_sum[0].s[j]; + acc[i].s[j] -= ((int)RHS_OFFSET) * a_sum[0].s[i] + ((int)(LHS_OFFSET)) * b_sum[0].s[j]; }) }) const bool x_cond = PARTIAL_STORE_N0 != 0 && get_global_id(0) == 0; const bool y_cond = PARTIAL_STORE_M0 != 0 && get_global_id(1) == 0; +#ifdef BIAS + perform_bias_addition(bias_ptr, bias_offset_first_element_in_bytes, acc, x); +#endif // defined(BIAS) + // Quantize the tile TILE(DATA_TYPE, M0, N0, accq); T_QUANTIZE8_ASYMMETRIC(int, DATA_TYPE, M0, N0, DST_OFFSET, DST_SHIFT, DST_MULTIPLIER, acc, accq); + T_ACTIVATION_QUANTIZED(DATA_TYPE, M0, N0, ACTIVATION_TYPE, ZERO_POINT, A_VAL, B_VAL, accq, accq); + TILE(int, M0, 1, indirect_buffer); LOOP_UNROLLING(int, _i, 0, 1, M0, { diff --git a/src/core/CL/cl_kernels/common/tile.cl b/src/core/CL/cl_kernels/common/tile.cl index 4332411688a212ba7b57866eb7fb24b345ddf0a6..971750b7b2ba3eb370fe3578b245e35936759aeb 100644 --- a/src/core/CL/cl_kernels/common/tile.cl +++ b/src/core/CL/cl_kernels/common/tile.cl @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -50,7 +50,7 @@ __kernel void tile( TENSOR4D_DECLARATION(input), TENSOR4D_DECLARATION(output)) { - Tensor4D output = CONVERT_TO_TENSOR4D_STRUCT(output, DST_DEPTH); + Tensor4D output = CONVERT_TO_TENSOR4D_STRUCT_NO_STEP(output, DST_DEPTH); Tensor4D input = CONVERT_TO_TENSOR4D_STRUCT_NO_STEP(input, SRC_DEPTH); // For all coordinates but x, each tile copies from the input @@ -62,22 +62,18 @@ __kernel void tile( // If we are loading/storing multiple elements at time, we need to // not exceed the input boundaries. The last threads need to backtrack // of OFFSET elements. Those elements cumulates for previous tiles - const int id = (int)(get_global_id(0)); - int x = id * VEC_SIZE; - // Shift x based on the previous offsets - const int tile_number = x / SRC_WIDTH; - x -= (tile_number) * OFFSET; - int x_input = x % SRC_WIDTH; + const int id = (int)(get_global_id(0)); + const int multiple_no = id / SRC_WIDTH_TILES; + const int tile_no = id % SRC_WIDTH_TILES; + const int last_tile = (int)(tile_no == SRC_WIDTH_TILES - 1); - // Shift x based on being the last tile - const int last_tile = (int)(x_input + VEC_SIZE > SRC_WIDTH); - x -= last_tile * OFFSET; - x_input = x % SRC_WIDTH; - output.ptr -= (tile_number + last_tile) * OFFSET * output_stride_x; + const int x_input = tile_no * VEC_SIZE - last_tile * OFFSET; + const int x_output = multiple_no * SRC_WIDTH + x_input; - // Update the input pointer - input.ptr = tensor4D_offset(&input, x_input, y % SRC_HEIGHT, z % SRC_DEPTH, batch % SRC_BATCHES); + // Update the input and output pointers. + input.ptr = tensor4D_offset(&input, x_input, y % SRC_HEIGHT, z % SRC_DEPTH, batch % SRC_BATCHES); + output.ptr = tensor4D_offset(&output, x_output, y, z, batch); // Copy the data VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE) @@ -88,8 +84,9 @@ __kernel void tile( #else // !defined(VEC_SIZE) || !defined(OFFSET) const int x = get_global_id(0); - // Update the input pointer - input.ptr = tensor4D_offset(&input, x % SRC_WIDTH, y % SRC_HEIGHT, z % SRC_DEPTH, batch % SRC_BATCHES); + // Update the input and output pointers. + input.ptr = tensor4D_offset(&input, x % SRC_WIDTH, y % SRC_HEIGHT, z % SRC_DEPTH, batch % SRC_BATCHES); + output.ptr = tensor4D_offset(&output, x, y, z, batch); *((__global DATA_TYPE *)(output.ptr)) = *((__global DATA_TYPE *)(input.ptr)); #endif // defined(VEC_SIZE) && defined(OFFSET) diff --git a/src/core/CL/cl_kernels/tile_helpers.h b/src/core/CL/cl_kernels/tile_helpers.h index 85bd59afd481672d753d79d89c449405a664ca77..812960627712fa15356130873fbff6ed5c1f8391 100644 --- a/src/core/CL/cl_kernels/tile_helpers.h +++ b/src/core/CL/cl_kernels/tile_helpers.h @@ -1144,19 +1144,21 @@ }) \ }) + +// NOTE : A_VAL and B_VAL should be quantized values (using same quantization info as x) // RELU Activation -#define relu_op_quantized(DATA_TYPE, VEC_SIZE, ZERO_VALUE, A_VAL, B_VAL, x) (max((DATA_TYPE)ZERO_VALUE, x)) +#define relu_op_quantized(DATA_TYPE, VEC_SIZE, ZERO_POINT, A_VAL, B_VAL, x) (max((DATA_TYPE)ZERO_POINT, x)) // Bounded RELU Activation -#define brelu_op_quantized(DATA_TYPE, VEC_SIZE, ZERO_VALUE, A_VAL, B_VAL, x) (min((DATA_TYPE)A_VAL, max((DATA_TYPE)ZERO_VALUE, x))) +#define brelu_op_quantized(DATA_TYPE, VEC_SIZE, ZERO_POINT, A_VAL, B_VAL, x) (min((DATA_TYPE)A_VAL, max((DATA_TYPE)ZERO_POINT, x))) // Lower Upper Bounded RELU Activation -#define lu_brelu_op_quantized(DATA_TYPE, VEC_SIZE, ZERO_VALUE, A_VAL, B_VAL, x) (min(max(x, (DATA_TYPE)B_VAL), (DATA_TYPE)A_VAL)) +#define lu_brelu_op_quantized(DATA_TYPE, VEC_SIZE, ZERO_POINT, A_VAL, B_VAL, x) (min(max(x, (DATA_TYPE)B_VAL), (DATA_TYPE)A_VAL)) // Hard Swish Activation -#define hard_swish_op_quantized(DATA_TYPE, VEC_SIZE, ZERO_VALUE, A_VAL, B_VAL, x) (x * ((min(max((DATA_TYPE)(x + (DATA_TYPE)3.f), (DATA_TYPE)0.f), (DATA_TYPE)6.f)) * (DATA_TYPE)0.166666667f)) +#define hard_swish_op_quantized(DATA_TYPE, VEC_SIZE, ZERO_POINT, A_VAL, B_VAL, x) (x * ((min(max((DATA_TYPE)(x + (DATA_TYPE)3.f), (DATA_TYPE)0.f), (DATA_TYPE)6.f)) * (DATA_TYPE)0.166666667f)) // Identity Activation -#define identity_op_quantized(DATA_TYPE, VEC_SIZE, ZERO_VALUE, A_VAL, B_VAL, x) (x) +#define identity_op_quantized(DATA_TYPE, VEC_SIZE, ZERO_POINT, A_VAL, B_VAL, x) (x) -#define ACT_OP_QUANTIZED(op, DATA_TYPE, VEC_SIZE, ZERO_VALUE, A_VAL, B_VAL, x) op##_op_quantized(DATA_TYPE, VEC_SIZE, ZERO_VALUE, A_VAL, B_VAL, x) -#define ACTIVATION_QUANTIZED(op, DATA_TYPE, VEC_SIZE, ZERO_VALUE, A_VAL, B_VAL, x) ACT_OP_QUANTIZED(op, DATA_TYPE, VEC_SIZE, ZERO_VALUE, A_VAL, B_VAL, x) +#define ACT_OP_QUANTIZED(op, DATA_TYPE, VEC_SIZE, ZERO_POINT, A_VAL, B_VAL, x) op##_op_quantized(DATA_TYPE, VEC_SIZE, ZERO_POINT, A_VAL, B_VAL, x) +#define ACTIVATION_QUANTIZED(op, DATA_TYPE, VEC_SIZE, ZERO_POINT, A_VAL, B_VAL, x) ACT_OP_QUANTIZED(op, DATA_TYPE, VEC_SIZE, ZERO_POINT, A_VAL, B_VAL, x) #define V_ADD(A_VAL, B_VAL) ((A_VAL) + (B_VAL)) #define V_SUB(A_VAL, B_VAL) ((A_VAL) - (B_VAL)) @@ -1171,17 +1173,17 @@ * @param[in] M0 Number of SRC/DST rows * @param[in] N0 Number of SRC/DST columns * @param[in] ACTIVATION_TYPE Activation type - * @param[in] ZERO_VALUE The zero value to consider in the computation - * @param[in] A_VAL A value used for the activation (e.g. tanh_op, brelu,..) - * @param[in] B_VAL B value used for the activation (e.g. tanh_op, brelu,..) + * @param[in] ZERO_POINT The zero value to consider in the computation + * @param[in] A_VAL Quantized A value used for the activation (e.g. tanh_op, brelu,..) + * @param[in] B_VAL Quantized B value used for the activation (e.g. tanh_op, brelu,..) * @param[out] src SRC tile * @param[out] dst DST tile */ -#define T_ACTIVATION_QUANTIZED(DATA_TYPE, M0, N0, ACTIVATION_TYPE, ZERO_VALUE, A_VAL, B_VAL, src, dst) \ +#define T_ACTIVATION_QUANTIZED(DATA_TYPE, M0, N0, ACTIVATION_TYPE, ZERO_POINT, A_VAL, B_VAL, src, dst) \ ({ \ LOOP_UNROLLING(int, _m0, 0, 1, M0, \ { \ - dst[_m0].v = ACTIVATION_QUANTIZED(ACTIVATION_TYPE, DATA_TYPE, N0, ZERO_VALUE, A_VAL, B_VAL, src[_m0].v); \ + dst[_m0].v = ACTIVATION_QUANTIZED(ACTIVATION_TYPE, DATA_TYPE, N0, ZERO_POINT, A_VAL, B_VAL, src[_m0].v); \ }) \ }) diff --git a/src/core/CL/kernels/CLArgMinMaxLayerKernel.cpp b/src/core/CL/kernels/CLArgMinMaxLayerKernel.cpp index 7af2fa1e64b3d3e941eb72c971ffd6e042dd1f3e..2728958addd55abf3efe94b2cdd6fd41c458da97 100644 --- a/src/core/CL/kernels/CLArgMinMaxLayerKernel.cpp +++ b/src/core/CL/kernels/CLArgMinMaxLayerKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -29,37 +29,31 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/Validate.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" - #include "support/StringSupport.h" namespace arm_compute { namespace { -Status validate_arguments(const ITensorInfo *input, const ITensorInfo *prev_output, const ITensorInfo *output, unsigned int axis, ReductionOperation op) +Status validate_arguments(const ITensorInfo *input, const ITensorInfo *output, unsigned int axis, ReductionOperation op) { ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, output); ARM_COMPUTE_RETURN_ERROR_ON_F16_UNSUPPORTED(input); ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QASYMM8, DataType::QASYMM8_SIGNED, DataType::S32, DataType::F16, DataType::F32); + ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(output, 1, DataType::S32, DataType::S64); + ARM_COMPUTE_RETURN_ERROR_ON_MSG(op != ReductionOperation::ARG_IDX_MAX && op != ReductionOperation::ARG_IDX_MIN, "Only ARG_IDX_MAX and ARG_IDX_MIN are supported"); ARM_COMPUTE_RETURN_ERROR_ON_MSG(axis >= TensorShape::num_max_dimensions, "Reduction axis greater than max number of dimensions"); ARM_COMPUTE_RETURN_ERROR_ON_MSG(axis > 3, "Unsupported reduction axis"); if(output->total_size() != 0) { - ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(output, 1, DataType::U32, DataType::S32); - } - if(prev_output != nullptr && prev_output->total_size() != 0) - { - ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(prev_output, 1, DataType::U32, DataType::S32); - if(output->total_size() != 0) - { - ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(prev_output, output); - } + ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(output, 1, DataType::U32, DataType::S32, DataType::S64, DataType::U64); } return Status{}; @@ -67,17 +61,17 @@ Status validate_arguments(const ITensorInfo *input, const ITensorInfo *prev_outp } // namespace CLArgMinMaxLayerKernel::CLArgMinMaxLayerKernel() - : _input(nullptr), _prev_output(nullptr), _output(nullptr), _reduction_axis(0), _op(ReductionOperation::ARG_IDX_MAX) + : _input(nullptr), _output(nullptr), _reduction_axis(0), _op(ReductionOperation::ARG_IDX_MAX) { _type = CLKernelType::ELEMENTWISE; } -void CLArgMinMaxLayerKernel::configure(const ICLTensor *input, const ICLTensor *prev_output, ICLTensor *output, unsigned int axis, ReductionOperation op) +void CLArgMinMaxLayerKernel::configure(const ICLTensor *input, ICLTensor *output, unsigned int axis, ReductionOperation op) { - configure(CLKernelLibrary::get().get_compile_context(), input, prev_output, output, axis, op); + configure(CLKernelLibrary::get().get_compile_context(), input, output, axis, op); } -void CLArgMinMaxLayerKernel::configure(const CLCompileContext &compile_context, const ICLTensor *input, const ICLTensor *prev_output, ICLTensor *output, unsigned int axis, ReductionOperation op) +void CLArgMinMaxLayerKernel::configure(const CLCompileContext &compile_context, const ICLTensor *input, ICLTensor *output, unsigned int axis, ReductionOperation op) { ARM_COMPUTE_ERROR_ON_NULLPTR(input, output); @@ -85,42 +79,37 @@ void CLArgMinMaxLayerKernel::configure(const CLCompileContext &compile_context, output_shape.set(axis, 1); auto_init_if_empty(*output->info(), input->info()->clone()->set_tensor_shape(output_shape).set_data_type(DataType::S32).reset_padding().set_is_resizable(true)); - ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(input->info(), (prev_output != nullptr) ? prev_output->info() : nullptr, output->info(), axis, op)); + ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(input->info(), output->info(), axis, op)); - auto padding_info = get_padding_info({ input, prev_output, output }); + auto padding_info = get_padding_info({ input, output }); _input = input; - _prev_output = prev_output; _output = output; _reduction_axis = axis; _op = op; // Set build options - const auto vector_size = (axis == 0) ? 16U : adjust_vec_size(16U, input->info()->dimension(0)); + const auto adjusted_vector_size = adjust_vec_size(16U, input->info()->dimension(0)); + const auto vector_size = (adjusted_vector_size == 3U && axis == 0U) ? 2U : adjusted_vector_size; // the opencl kernel only supports sizes 2, 4, 8 and 16. CLBuildOptions build_opts; - build_opts.add_option_if(_prev_output != nullptr, "-DPREV_OUTPUT"); build_opts.add_option("-DDATA_TYPE=" + get_cl_type_from_data_type(input->info()->data_type())); build_opts.add_option("-DVEC_SIZE_LEFTOVER=" + support::cpp11::to_string(input->info()->dimension(0) % vector_size)); build_opts.add_option("-DVEC_SIZE=" + support::cpp11::to_string(vector_size)); build_opts.add_option_if(is_data_type_float(input->info()->data_type()), "-DFLOAT_DATA_TYPE"); build_opts.add_option_if_else(op == ReductionOperation::ARG_IDX_MAX, "-DARG_MAX", "-DARG_MIN"); build_opts.add_option("-DDATA_TYPE_OUTPUT=" + get_cl_type_from_data_type(output->info()->data_type())); + build_opts.add_option("-DCOND_DATA_TYPE=" + get_cl_select_type_from_data_type(input->info()->data_type())); + build_opts.add_option("-DUNROLL_WITH_PRAGMA=1"); // Create kernel - cl::NDRange lws_hint = CLKernelLibrary::get().default_ndrange(); std::string kernel_axis_name; switch(axis) { case 0: - { - const ICLTensor *input_for_width = prev_output != nullptr ? _prev_output : _input; - build_opts.add_option("-DWIDTH=" + support::cpp11::to_string(input_for_width->info()->dimension(0))); - + build_opts.add_option("-DWIDTH=" + support::cpp11::to_string(input->info()->dimension(0))); kernel_axis_name = "x"; - lws_hint = create_lws_hint_parallel_implementations(input_for_width->info()->dimension(0), vector_size); - } - break; + break; case 1: build_opts.add_option("-DHEIGHT=" + support::cpp11::to_string(input->info()->dimension(1))); kernel_axis_name = "y"; @@ -140,15 +129,15 @@ void CLArgMinMaxLayerKernel::configure(const CLCompileContext &compile_context, _kernel = create_kernel(compile_context, "arg_min_max_" + kernel_axis_name, build_opts.options()); // Configure kernel window - Window win = calculate_max_window((prev_output != nullptr) ? (*prev_output->info()) : (*input->info()), Steps(vector_size)); - ICLKernel::configure_internal(win, lws_hint); + Window win = calculate_max_window(*input->info(), Steps(vector_size)); + ICLKernel::configure_internal(win); ARM_COMPUTE_ERROR_ON(has_padding_changed(padding_info)); } -Status CLArgMinMaxLayerKernel::validate(const ITensorInfo *input, const ITensorInfo *prev_output, const ITensorInfo *output, unsigned int axis, ReductionOperation op) +Status CLArgMinMaxLayerKernel::validate(const ITensorInfo *input, const ITensorInfo *output, unsigned int axis, ReductionOperation op) { - ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(input, prev_output, output, axis, op)); + ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(input, output, axis, op)); return Status{}; } @@ -163,30 +152,22 @@ void CLArgMinMaxLayerKernel::run(const Window &window, cl::CommandQueue &queue) { // Set out window Window out_window(window); + Window in_window(window); out_window.set(Window::DimX, Window::Dimension(0, 0, 0)); + in_window.set(Window::DimX, Window::Dimension(0, _input->info()->dimension(0), _input->info()->dimension(0))); + in_window.set(Window::DimY, Window::Dimension(0, _input->info()->dimension(1), 1u)); // Get first input and output slices - Window in_slice = window.first_slice_window_2D(); + Window in_slice = in_window.first_slice_window_2D(); Window out_slice = out_window.first_slice_window_2D(); - - // Reshape window - const unsigned int num_tensors = _prev_output != nullptr ? 3 : 2; - - // Set local sums buffer - unsigned int local_res_size = lws_hint()[0] * _output->info()->element_size(); - _kernel.setArg(num_arguments_per_2D_tensor() * num_tensors, local_res_size, nullptr); do { unsigned int idx = 0; add_2D_tensor_argument(idx, _input, in_slice); - if(_prev_output != nullptr) - { - add_2D_tensor_argument(idx, _prev_output, in_slice); - } add_2D_tensor_argument(idx, _output, out_slice); enqueue(queue, *this, in_slice, lws_hint()); } - while(window.slide_window_slice_2D(in_slice) && window.slide_window_slice_2D(out_slice)); + while(in_window.slide_window_slice_2D(in_slice) && out_window.slide_window_slice_2D(out_slice)); } break; case 1: diff --git a/src/core/CL/kernels/CLArgMinMaxLayerKernel.h b/src/core/CL/kernels/CLArgMinMaxLayerKernel.h index 929677f9054ddae9008a159a1d6267f5eb9f313b..5f36bdf113249b4ec1616840a56b98616675a716 100644 --- a/src/core/CL/kernels/CLArgMinMaxLayerKernel.h +++ b/src/core/CL/kernels/CLArgMinMaxLayerKernel.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2020 Arm Limited. + * Copyright (c) 2019-2020, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -56,48 +56,41 @@ public: /** Set the input and output tensors. * - * @param[in] input Source tensor. Data types supported: QASYMM8/QASYMM8_SIGNED/S32/F16/F32. - * @param[in] prev_output Destination tensor of the previous iterations of @ref CLArgMinMaxLayerKernel. Data types supported: U32/S32 - * Has to be nullptr for the first iteration - * @param[out] output Destination tensor. Data types supported: U32/S32 - * Output will have the same number of dimensions as input. - * @param[in] axis Axis along which to reduce. Supported reduction axis : 0,1,2,3 - * @param[in] op Reduction operation to perform. Only ArgMin and ArgMax are supported. + * @param[in] input Source tensor. Data types supported: QASYMM8/QASYMM8_SIGNED/S32/F16/F32. + * @param[out] output Destination tensor. Data types supported: U32/S32 + * Output will have the same number of dimensions as input. + * @param[in] axis Axis along which to reduce. Supported reduction axis : 0,1,2,3 + * @param[in] op Reduction operation to perform. Only ArgMin and ArgMax are supported. */ - void configure(const ICLTensor *input, const ICLTensor *prev_output, ICLTensor *output, unsigned int axis, ReductionOperation op); + void configure(const ICLTensor *input, ICLTensor *output, unsigned int axis, ReductionOperation op); /** Set the input and output tensors. * * @param[in] compile_context The compile context to be used. * @param[in] input Source tensor. Data types supported: QASYMM8/QASYMM8_SIGNED/S32/F16/F32. - * @param[in] prev_output Destination tensor of the previous iterations of @ref CLArgMinMaxLayerKernel. Data types supported: U32/S32 - * Has to be nullptr for the first iteration * @param[out] output Destination tensor. Data types supported: U32/S32 * Output will have the same number of dimensions as input. * @param[in] axis Axis along which to reduce. Supported reduction axis : 0,1,2,3 * @param[in] op Reduction operation to perform. Only ArgMin and ArgMax are supported. */ - void configure(const CLCompileContext &compile_context, const ICLTensor *input, const ICLTensor *prev_output, ICLTensor *output, unsigned int axis, ReductionOperation op); + void configure(const CLCompileContext &compile_context, const ICLTensor *input, ICLTensor *output, unsigned int axis, ReductionOperation op); /** Static function to check if given info will lead to a valid configuration of @ref CLArgMinMaxLayerKernel. * - * @param[in] input Source tensor info. Data types supported: QASYMM8/QASYMM8_SIGNED/S32/F16/F32. - * @param[in] prev_output Destination tensor info of the previous iterations. Data types supported: U32/S32 - * Has to be nullptr for the first iteration - * @param[in] output Destination tensor info. Data types supported: U32/S32 - * Output will have the same number of dimensions as input. - * @param[in] axis Axis along which to reduce. Supported reduction axis : 0,1,2,3 - * @param[in] op Reduction operation to perform. Only ArgMin and ArgMax are supported. + * @param[in] input Source tensor info. Data types supported: QASYMM8/QASYMM8_SIGNED/S32/F16/F32. + * @param[in] output Destination tensor info. Data types supported: U32/S32 + * Output will have the same number of dimensions as input. + * @param[in] axis Axis along which to reduce. Supported reduction axis : 0,1,2,3 + * @param[in] op Reduction operation to perform. Only ArgMin and ArgMax are supported. * * @return a status */ - static Status validate(const ITensorInfo *input, const ITensorInfo *prev_output, const ITensorInfo *output, unsigned int axis, ReductionOperation op); + static Status validate(const ITensorInfo *input, const ITensorInfo *output, unsigned int axis, ReductionOperation op); // Inherited methods overridden: void run(const Window &window, cl::CommandQueue &queue) override; private: const ICLTensor *_input; - const ICLTensor *_prev_output; ICLTensor *_output; unsigned int _reduction_axis; ReductionOperation _op; diff --git a/src/core/CL/kernels/CLBatchNormalizationLayerKernel.cpp b/src/core/CL/kernels/CLBatchNormalizationLayerKernel.cpp index 24c8b4da5b76588f99af01caa56c326b89557f3e..3fa8a8edaa133e080c444b834e05cff5e8819e08 100644 --- a/src/core/CL/kernels/CLBatchNormalizationLayerKernel.cpp +++ b/src/core/CL/kernels/CLBatchNormalizationLayerKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,12 +23,15 @@ */ #include "src/core/CL/kernels/CLBatchNormalizationLayerKernel.h" +#include "arm_compute/core/utils/ActivationFunctionUtils.h" #include "arm_compute/core/CL/CLHelpers.h" #include "arm_compute/core/CL/CLKernelLibrary.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/core/CL/kernels/CLBatchNormalizationLayerKernel.h b/src/core/CL/kernels/CLBatchNormalizationLayerKernel.h index 743f4a95948fa9ca1e90c4d71837eadd81f92839..acbe0f2a26f49de04a40e616833585a317dd0a9a 100644 --- a/src/core/CL/kernels/CLBatchNormalizationLayerKernel.h +++ b/src/core/CL/kernels/CLBatchNormalizationLayerKernel.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2020 Arm Limited. + * Copyright (c) 2017-2020, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CLBATCHNORMALIZATIONLAYERKERNEL_H #define ARM_COMPUTE_CLBATCHNORMALIZATIONLAYERKERNEL_H +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/core/CL/ICLKernel.h" namespace arm_compute diff --git a/src/core/CL/kernels/CLBatchToSpaceLayerKernel.cpp b/src/core/CL/kernels/CLBatchToSpaceLayerKernel.cpp index c3dc91b8f67c3859fd8ff99613ee9c47154a0388..143a842d02d8959e80f20f880d2514b2a43f4ffd 100644 --- a/src/core/CL/kernels/CLBatchToSpaceLayerKernel.cpp +++ b/src/core/CL/kernels/CLBatchToSpaceLayerKernel.cpp @@ -26,6 +26,7 @@ #include "arm_compute/core/CL/CLHelpers.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/core/CL/kernels/CLBitwiseKernel.cpp b/src/core/CL/kernels/CLBitwiseKernel.cpp index 4c3ea80895db1461272c9b2e44850fe51d4dc520..11e6d021a5ff4d03d0d76333c453cba26cd1b2de 100644 --- a/src/core/CL/kernels/CLBitwiseKernel.cpp +++ b/src/core/CL/kernels/CLBitwiseKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021 Arm Limited. + * Copyright (c) 2020-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,7 +27,9 @@ #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/CL/OpenCL.h" #include "arm_compute/core/Helpers.h" +#include "arm_compute/core/Utils.h" #include "arm_compute/core/Validate.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" #include "support/StringSupport.h" diff --git a/src/core/CL/kernels/CLBoundingBoxTransformKernel.cpp b/src/core/CL/kernels/CLBoundingBoxTransformKernel.cpp index f57221d16b3798e53d69f625f14eaaebbbca28a2..72de854afb139463f75ad302c6665376f0227042 100644 --- a/src/core/CL/kernels/CLBoundingBoxTransformKernel.cpp +++ b/src/core/CL/kernels/CLBoundingBoxTransformKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -30,6 +30,7 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/core/CL/kernels/CLChannelShuffleLayerKernel.cpp b/src/core/CL/kernels/CLChannelShuffleLayerKernel.cpp index 7c8a7ce15096c6f61e4352a3d90c5389073e06e0..a2a0bc4fb4a97c346b78faa6e26b1d36839f3e90 100644 --- a/src/core/CL/kernels/CLChannelShuffleLayerKernel.cpp +++ b/src/core/CL/kernels/CLChannelShuffleLayerKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -28,6 +28,9 @@ #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" +#include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/core/CL/kernels/CLComparisonKernel.cpp b/src/core/CL/kernels/CLComparisonKernel.cpp index 21f98349a07f265b9b809c96070810d7aa5d68ab..f4d6316517e5db8843412a46ec64aecec6e85927 100644 --- a/src/core/CL/kernels/CLComparisonKernel.cpp +++ b/src/core/CL/kernels/CLComparisonKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #include "arm_compute/core/CL/CLHelpers.h" #include "arm_compute/core/CL/ICLTensor.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/core/CL/kernels/CLDeconvolutionLayerUpsampleKernel.cpp b/src/core/CL/kernels/CLDeconvolutionLayerUpsampleKernel.cpp index 505a93761d677fd8f44040bdb40bdbbfdeb4c06a..76af5d564a61cb037f5796bc0d06b9c0f4e22559 100644 --- a/src/core/CL/kernels/CLDeconvolutionLayerUpsampleKernel.cpp +++ b/src/core/CL/kernels/CLDeconvolutionLayerUpsampleKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,6 +27,7 @@ #include "arm_compute/core/CL/CLKernelLibrary.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/Helpers.h" +#include "arm_compute/core/Utils.h" #include "arm_compute/core/Validate.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/core/CL/kernels/CLDeconvolutionReshapeOutputKernel.cpp b/src/core/CL/kernels/CLDeconvolutionReshapeOutputKernel.cpp index 8863de5c576e44c000f68e5878c379a946d8a6ac..0fc0ff816811b36f784b67d2403bcdc2429feff2 100644 --- a/src/core/CL/kernels/CLDeconvolutionReshapeOutputKernel.cpp +++ b/src/core/CL/kernels/CLDeconvolutionReshapeOutputKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,9 +27,9 @@ #include "arm_compute/core/CL/CLKernelLibrary.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/Helpers.h" -#include "arm_compute/core/Utils.h" #include "arm_compute/core/Validate.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" #include "support/StringSupport.h" diff --git a/src/core/CL/kernels/CLDepthToSpaceLayerKernel.cpp b/src/core/CL/kernels/CLDepthToSpaceLayerKernel.cpp index efc6f820f2369dbaf7b9f899503c1d294436f9c1..5c1dc4fbf6bb1bb0cd22cbd9aa49ad3035359ed1 100644 --- a/src/core/CL/kernels/CLDepthToSpaceLayerKernel.cpp +++ b/src/core/CL/kernels/CLDepthToSpaceLayerKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,7 @@ #include "arm_compute/core/CL/CLHelpers.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/core/CL/kernels/CLDepthwiseConvolutionLayerNativeKernel.cpp b/src/core/CL/kernels/CLDepthwiseConvolutionLayerNativeKernel.cpp index 2d21a6eff05024b04f7c58715ab682fa1d667810..e34b6929e7f49df6b8e4299678dfe866ea923343 100644 --- a/src/core/CL/kernels/CLDepthwiseConvolutionLayerNativeKernel.cpp +++ b/src/core/CL/kernels/CLDepthwiseConvolutionLayerNativeKernel.cpp @@ -23,14 +23,16 @@ */ #include "src/core/CL/kernels/CLDepthwiseConvolutionLayerNativeKernel.h" +#include "arm_compute/core/utils/ActivationFunctionUtils.h" #include "arm_compute/core/CL/CLHelpers.h" #include "arm_compute/core/CL/CLKernelLibrary.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" -#include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" #include "arm_compute/core/utils/quantization/AsymmHelpers.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/CL/ICLKernel.h" diff --git a/src/core/CL/kernels/CLDepthwiseConvolutionLayerNativeKernel.h b/src/core/CL/kernels/CLDepthwiseConvolutionLayerNativeKernel.h index 5352f685ea79f916ca4a76ad596c9645092d58a6..8eee7b25004757f6c9e5f33dfc39b114dbda4c1d 100644 --- a/src/core/CL/kernels/CLDepthwiseConvolutionLayerNativeKernel.h +++ b/src/core/CL/kernels/CLDepthwiseConvolutionLayerNativeKernel.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022 Arm Limited. + * Copyright (c) 2019-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,6 +27,7 @@ #include "src/core/CL/ICLKernel.h" #include "arm_compute/core/KernelDescriptors.h" +#include "arm_compute/function_info/ConvolutionInfo.h" namespace arm_compute { diff --git a/src/core/CL/kernels/CLFFTDigitReverseKernel.cpp b/src/core/CL/kernels/CLFFTDigitReverseKernel.cpp index bbf4e554332a769e50fb78545347e89edffeed71..9b514ed705f915239dbc948ff2569b095da762cf 100644 --- a/src/core/CL/kernels/CLFFTDigitReverseKernel.cpp +++ b/src/core/CL/kernels/CLFFTDigitReverseKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,8 @@ #include "arm_compute/core/CL/CLKernelLibrary.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/TensorInfo.h" +#include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/core/CL/kernels/CLFFTRadixStageKernel.cpp b/src/core/CL/kernels/CLFFTRadixStageKernel.cpp index 0ee247fecd48a1d3d44b9c49a057fb4aa8d708ea..95f4b640bd4877fc7b9f8d271a21272b1194bd90 100644 --- a/src/core/CL/kernels/CLFFTRadixStageKernel.cpp +++ b/src/core/CL/kernels/CLFFTRadixStageKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -28,6 +28,7 @@ #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/core/CL/kernels/CLFFTScaleKernel.cpp b/src/core/CL/kernels/CLFFTScaleKernel.cpp index 8901345738a7f6a99132f6ef3be07c367e1ed57e..8a714d71bf93069552136baf80d0bee57bcc219d 100644 --- a/src/core/CL/kernels/CLFFTScaleKernel.cpp +++ b/src/core/CL/kernels/CLFFTScaleKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,8 @@ #include "arm_compute/core/CL/CLKernelLibrary.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/TensorInfo.h" +#include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/core/CL/kernels/CLFillBorderKernel.cpp b/src/core/CL/kernels/CLFillBorderKernel.cpp index ded707e9d66c23d641a45e1a2578035261006779..fcd99a4ed9180026b1421374aaea8f67fec721d3 100644 --- a/src/core/CL/kernels/CLFillBorderKernel.cpp +++ b/src/core/CL/kernels/CLFillBorderKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021 Arm Limited. + * Copyright (c) 2016-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -29,6 +29,7 @@ #include "arm_compute/core/CL/OpenCL.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/StringUtils.h" #include "arm_compute/core/Validate.h" #include "src/core/helpers/WindowHelpers.h" #include "support/Cast.h" diff --git a/src/core/CL/kernels/CLFuseBatchNormalizationKernel.cpp b/src/core/CL/kernels/CLFuseBatchNormalizationKernel.cpp index 0695ff9935a1218c181002aab0119bfb65b2dbbd..68fe324df6c3dfcdd4478619b4360054e21ac382 100644 --- a/src/core/CL/kernels/CLFuseBatchNormalizationKernel.cpp +++ b/src/core/CL/kernels/CLFuseBatchNormalizationKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -29,6 +29,7 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/core/CL/kernels/CLGenerateProposalsLayerKernel.cpp b/src/core/CL/kernels/CLGenerateProposalsLayerKernel.cpp index 8b008c306b1c354bea247f96499f848c6d0a5979..088c454f3c4b1e6974a99c4f199686e740708410 100644 --- a/src/core/CL/kernels/CLGenerateProposalsLayerKernel.cpp +++ b/src/core/CL/kernels/CLGenerateProposalsLayerKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -30,6 +30,7 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/core/CL/kernels/CLInstanceNormalizationLayerKernel.cpp b/src/core/CL/kernels/CLInstanceNormalizationLayerKernel.cpp index 74cbef151b3fdd31de4e22e0c7043fbe4c2dbd15..7ed323c95087228aef45e8d5da651ab7251b481d 100644 --- a/src/core/CL/kernels/CLInstanceNormalizationLayerKernel.cpp +++ b/src/core/CL/kernels/CLInstanceNormalizationLayerKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -29,6 +29,7 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/core/CL/kernels/CLL2NormalizeLayerKernel.cpp b/src/core/CL/kernels/CLL2NormalizeLayerKernel.cpp index 46c0747cf5e989758585f4e66d2f7b21ad19633c..542d380e4a8d674ebf45a44887df2f898bd474db 100644 --- a/src/core/CL/kernels/CLL2NormalizeLayerKernel.cpp +++ b/src/core/CL/kernels/CLL2NormalizeLayerKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -29,6 +29,7 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/Validate.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" diff --git a/src/core/CL/kernels/CLMaxUnpoolingLayerKernel.cpp b/src/core/CL/kernels/CLMaxUnpoolingLayerKernel.cpp index 89a6d829475f82a21e4c91808b010d3fc0cb838d..dc9d68626db1c3ab5b8ea06f9393afe0b3908256 100644 --- a/src/core/CL/kernels/CLMaxUnpoolingLayerKernel.cpp +++ b/src/core/CL/kernels/CLMaxUnpoolingLayerKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021 Arm Limited. + * Copyright (c) 2020-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -29,8 +29,8 @@ #include "arm_compute/core/CL/OpenCL.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" -#include "arm_compute/core/Utils.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/core/CL/kernels/CLMeanStdDevNormalizationKernel.cpp b/src/core/CL/kernels/CLMeanStdDevNormalizationKernel.cpp index b94593943c7e7df6d796d9ba54bfa92501e6efff..ac33468ad8e068eb57529a282226df4307667c9b 100644 --- a/src/core/CL/kernels/CLMeanStdDevNormalizationKernel.cpp +++ b/src/core/CL/kernels/CLMeanStdDevNormalizationKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022 Arm Limited. + * Copyright (c) 2019-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -29,6 +29,8 @@ #include "arm_compute/core/CL/OpenCL.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/core/CL/kernels/CLNormalizationLayerKernel.cpp b/src/core/CL/kernels/CLNormalizationLayerKernel.cpp index 2765300925f36dce39af98690758e8c27ddf9bc1..c6c4229c0062123711b329624ad99b298c6fc4e2 100644 --- a/src/core/CL/kernels/CLNormalizationLayerKernel.cpp +++ b/src/core/CL/kernels/CLNormalizationLayerKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -29,6 +29,8 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/StringUtils.h" #include "arm_compute/core/Window.h" #include "src/core/AccessWindowStatic.h" #include "src/core/CL/CLValidate.h" diff --git a/src/core/CL/kernels/CLNormalizePlanarYUVLayerKernel.cpp b/src/core/CL/kernels/CLNormalizePlanarYUVLayerKernel.cpp index 6c23b18e62f4b0c8a8b49b1ef698c7c957d9be6f..6b0400d50e9ddeeaaea1f311df7a4afff1c0fa2d 100644 --- a/src/core/CL/kernels/CLNormalizePlanarYUVLayerKernel.cpp +++ b/src/core/CL/kernels/CLNormalizePlanarYUVLayerKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -29,6 +29,8 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/AccessWindowStatic.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" diff --git a/src/core/CL/kernels/CLPadLayerKernel.cpp b/src/core/CL/kernels/CLPadLayerKernel.cpp index eaab99284032a5197a606237e065d9e490c86a51..53f313c0d39bc8c569230f9196e688ffcd5e69e5 100644 --- a/src/core/CL/kernels/CLPadLayerKernel.cpp +++ b/src/core/CL/kernels/CLPadLayerKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #include "arm_compute/core/CL/CLHelpers.h" #include "arm_compute/core/CL/ICLTensor.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/core/CL/kernels/CLQLSTMLayerNormalizationKernel.cpp b/src/core/CL/kernels/CLQLSTMLayerNormalizationKernel.cpp index 5ad43552025006612bd4aa261af4a225c0d8ec80..bd573e54c8eeff775555b482af552c2a032556ee 100644 --- a/src/core/CL/kernels/CLQLSTMLayerNormalizationKernel.cpp +++ b/src/core/CL/kernels/CLQLSTMLayerNormalizationKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021 Arm Limited. + * Copyright (c) 2020-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,8 @@ #include "src/core/CL/kernels/CLQLSTMLayerNormalizationKernel.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/utils/quantization/AsymmHelpers.h" +#include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" #include "support/StringSupport.h" diff --git a/src/core/CL/kernels/CLROIAlignLayerKernel.cpp b/src/core/CL/kernels/CLROIAlignLayerKernel.cpp index 34d3e70c25f2010ed8c4580b7bfdc38d90c28e5f..69a6fa5fa01838dc2183a4ba565af154ab8ed0c3 100644 --- a/src/core/CL/kernels/CLROIAlignLayerKernel.cpp +++ b/src/core/CL/kernels/CLROIAlignLayerKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -29,8 +29,8 @@ #include "arm_compute/core/CL/OpenCL.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" -#include "arm_compute/core/Utils.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/core/CL/kernels/CLROIPoolingLayerKernel.cpp b/src/core/CL/kernels/CLROIPoolingLayerKernel.cpp index 663da0467af01a561df37bb3757727e4b3e80f43..f6933c6cfd241dbfa2e08756b0043cbb484f268a 100644 --- a/src/core/CL/kernels/CLROIPoolingLayerKernel.cpp +++ b/src/core/CL/kernels/CLROIPoolingLayerKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -30,6 +30,7 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/core/CL/kernels/CLRangeKernel.cpp b/src/core/CL/kernels/CLRangeKernel.cpp index b245e62bc0393eae04dee9256abdf5ca9d873ca3..a06c2eed75f23525fda4b35aa129bad9cd4ea9ad 100644 --- a/src/core/CL/kernels/CLRangeKernel.cpp +++ b/src/core/CL/kernels/CLRangeKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,8 @@ #include "arm_compute/core/CL/CLHelpers.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" @@ -136,4 +138,4 @@ void CLRangeKernel::run(const Window &window, cl::CommandQueue &queue) enqueue(queue, *this, window, lws_hint()); } -} // namespace arm_compute \ No newline at end of file +} // namespace arm_compute diff --git a/src/core/CL/kernels/CLReductionOperationKernel.cpp b/src/core/CL/kernels/CLReductionOperationKernel.cpp index 3b3b6c0364b5cc1c82be5150f739f083580958ac..ee60b8e1df4c14035d8ddabd9372383368be05d6 100644 --- a/src/core/CL/kernels/CLReductionOperationKernel.cpp +++ b/src/core/CL/kernels/CLReductionOperationKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -28,9 +28,10 @@ #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" -#include "arm_compute/core/Utils.h" #include "arm_compute/core/Validate.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/AccessWindowStatic.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" diff --git a/src/core/CL/kernels/CLReorgLayerKernel.cpp b/src/core/CL/kernels/CLReorgLayerKernel.cpp index aa5f16fbc1d581ae818da365e10e0985ffc3542a..3c74e80d33fc7b61518baf9170e45c87adb8abb3 100644 --- a/src/core/CL/kernels/CLReorgLayerKernel.cpp +++ b/src/core/CL/kernels/CLReorgLayerKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -30,6 +30,7 @@ #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Validate.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" #include "support/StringSupport.h" diff --git a/src/core/CL/kernels/CLReverseKernel.cpp b/src/core/CL/kernels/CLReverseKernel.cpp index 7e9431e23079826e5eb88f1aca775d4d5226f234..0d70ff4f3c8646169bf853e1347e7300a1236da4 100644 --- a/src/core/CL/kernels/CLReverseKernel.cpp +++ b/src/core/CL/kernels/CLReverseKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -28,6 +28,8 @@ #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" +#include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/core/CL/kernels/CLSelectKernel.cpp b/src/core/CL/kernels/CLSelectKernel.cpp index 43b958a8dca6aad394949b97d4fc55e257bed184..c0e014e8b88dd83fb338ea872d8115b6cb9cc17e 100644 --- a/src/core/CL/kernels/CLSelectKernel.cpp +++ b/src/core/CL/kernels/CLSelectKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -29,6 +29,7 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/core/CL/kernels/CLSpaceToBatchLayerKernel.cpp b/src/core/CL/kernels/CLSpaceToBatchLayerKernel.cpp index 6533731571eca450ab747fdcba08a9d58e541941..3632ae2b03d45ad32396e7ca82521affce790559 100644 --- a/src/core/CL/kernels/CLSpaceToBatchLayerKernel.cpp +++ b/src/core/CL/kernels/CLSpaceToBatchLayerKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,7 @@ #include "arm_compute/core/CL/CLHelpers.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/core/CL/kernels/CLSpaceToDepthLayerKernel.cpp b/src/core/CL/kernels/CLSpaceToDepthLayerKernel.cpp index e7656b805c8b11d489bb472dab620f82291740db..c5ffdb588ba2b2525783431f237fb04ebea1d0a6 100644 --- a/src/core/CL/kernels/CLSpaceToDepthLayerKernel.cpp +++ b/src/core/CL/kernels/CLSpaceToDepthLayerKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,7 @@ #include "arm_compute/core/CL/CLHelpers.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/core/CL/kernels/CLStridedSliceKernel.cpp b/src/core/CL/kernels/CLStridedSliceKernel.cpp index 464f74c9ebc2d74fbf5166970e678e13dff997aa..9acbafdb190a6b33eeb1b0104a0b8c42a1767961 100644 --- a/src/core/CL/kernels/CLStridedSliceKernel.cpp +++ b/src/core/CL/kernels/CLStridedSliceKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,7 @@ #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/utils/helpers/tensor_transform.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" #include "src/core/utils/helpers/bit_ops.h" diff --git a/src/core/CL/kernels/CLTileKernel.cpp b/src/core/CL/kernels/CLTileKernel.cpp index e4eed68c84d7a5322da9de7a691b610738294d4c..3e7015cfd22492c94b242a61989dffa32057c545 100644 --- a/src/core/CL/kernels/CLTileKernel.cpp +++ b/src/core/CL/kernels/CLTileKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #include "src/core/CL/kernels/CLTileKernel.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" #include "support/StringSupport.h" @@ -79,11 +80,13 @@ void CLTileKernel::configure(const CLCompileContext &compile_context, const ICLT _input = input; _output = output; - const DataType data_type = input->info()->data_type(); - const int vec_size_x = 16 / input->info()->element_size(); - const int input_width_x = input->info()->tensor_shape().x(); - const unsigned int offset = ceil_to_multiple(input_width_x, vec_size_x) - input_width_x; - const bool multi_access_x = (input_width_x / vec_size_x > 0); + const DataType data_type = input->info()->data_type(); + const int vec_size_x = 16 / input->info()->element_size(); + const int input_width_x = input->info()->tensor_shape().x(); + const unsigned int input_width_ceil = ceil_to_multiple(input_width_x, vec_size_x); + const unsigned int input_width_tiles = input_width_ceil / vec_size_x; + const unsigned int offset = input_width_ceil - input_width_x; + const bool multi_access_x = (input_width_x / vec_size_x > 0); // Create kernel CLBuildOptions build_opts; @@ -95,6 +98,7 @@ void CLTileKernel::configure(const CLCompileContext &compile_context, const ICLT build_opts.add_option("-DDST_DEPTH=" + support::cpp11::to_string(output->info()->dimension(2))); build_opts.add_option_if(multi_access_x, "-DOFFSET=" + support::cpp11::to_string(offset)); build_opts.add_option_if(multi_access_x, "-DVEC_SIZE=" + support::cpp11::to_string(vec_size_x)); + build_opts.add_option_if(multi_access_x, "-DSRC_WIDTH_TILES=" + support::cpp11::to_string(input_width_tiles)); _kernel = create_kernel(compile_context, "tile", build_opts.options()); // Configure window without padding diff --git a/src/core/ITensor.cpp b/src/core/ITensor.cpp index 9ed9b3f2e97edd4ee46ab7a9b83bc2f9232ba832..2f4354cc6f1938f8f0b8c9281b7c5789cdb9d58f 100644 --- a/src/core/ITensor.cpp +++ b/src/core/ITensor.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021 Arm Limited. + * Copyright (c) 2016-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #include "arm_compute/core/Error.h" #include "arm_compute/core/Helpers.h" +#include "arm_compute/core/Utils.h" #include "arm_compute/core/Window.h" #include @@ -174,4 +175,4 @@ void ITensor::mark_as_used() const { _is_used = true; } -} // namespace arm_compute \ No newline at end of file +} // namespace arm_compute diff --git a/src/core/NEON/NEMath.inl b/src/core/NEON/NEMath.inl index 6198a257fc323b82b4b32175187b7d722ff916c2..1cbe66937304a338d58148f8f24c6bd5217c9e8c 100644 --- a/src/core/NEON/NEMath.inl +++ b/src/core/NEON/NEMath.inl @@ -52,11 +52,11 @@ constexpr float te_sin_coeff5 = 0.013888888889f; // 1/(8*9) #ifndef DOXYGEN_SKIP_THIS inline float32x4_t prefer_vfmaq_f32(float32x4_t a, float32x4_t b, float32x4_t c) { -#ifdef __aarch64__ +#if __ARM_FEATURE_FMA return vfmaq_f32(a, b, c); -#else // __aarch64__ +#else // __ARM_FEATURE_FMA return vmlaq_f32(a, b, c); -#endif // __aarch64__ +#endif // __ARM_FEATURE_FMA } inline float32x4_t vfloorq_f32(float32x4_t val) diff --git a/src/core/NEON/kernels/NEBatchNormalizationLayerKernel.h b/src/core/NEON/kernels/NEBatchNormalizationLayerKernel.h index 9312073ce803bce3545276488b3e95ad40e8ddc7..0551ace30cebb5778fcb58b225b10298031fac0b 100644 --- a/src/core/NEON/kernels/NEBatchNormalizationLayerKernel.h +++ b/src/core/NEON/kernels/NEBatchNormalizationLayerKernel.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2020 Arm Limited. + * Copyright (c) 2017-2020, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_NEBATCHNORMALIZATIONLAYERKERNEL_H #define ARM_COMPUTE_NEBATCHNORMALIZATIONLAYERKERNEL_H +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/core/NEON/INEKernel.h" namespace arm_compute diff --git a/src/core/NEON/kernels/NEPadLayerKernel.h b/src/core/NEON/kernels/NEPadLayerKernel.h index b3b0725af817b786cbd533d5fe17a8c52626ab30..f82af1558a57872e312ca026e1b73a5fe2c5c6a5 100644 --- a/src/core/NEON/kernels/NEPadLayerKernel.h +++ b/src/core/NEON/kernels/NEPadLayerKernel.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_NEPADLAYERKERNEL_H #define ARM_COMPUTE_NEPADLAYERKERNEL_H +#include "arm_compute/core/PixelValue.h" #include "src/core/NEON/INEKernel.h" namespace arm_compute diff --git a/src/core/NEON/kernels/NERangeKernel.cpp b/src/core/NEON/kernels/NERangeKernel.cpp index 82d1403c56aa2b80431c0d16b613da28b8b27693..ec63a35de9b86bd2191d0bc13fd335beca7cd332 100644 --- a/src/core/NEON/kernels/NERangeKernel.cpp +++ b/src/core/NEON/kernels/NERangeKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,6 +27,7 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/ITensor.h" #include "arm_compute/core/TensorInfo.h" +#include "arm_compute/core/Utils.h" #include "arm_compute/core/Validate.h" #include "src/core/NEON/NEAsymm.h" #include "src/core/NEON/wrapper/wrapper.h" diff --git a/src/core/NEON/kernels/arm_conv/addressing.cpp b/src/core/NEON/kernels/arm_conv/addressing.cpp index d01627bc5adca93dc2a56009f2c6b7bd027f90d2..246039888028ac7335909bfc7a21d5d865c6701a 100644 --- a/src/core/NEON/kernels/arm_conv/addressing.cpp +++ b/src/core/NEON/kernels/arm_conv/addressing.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,7 +23,8 @@ */ #include "addressing.hpp" -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" +#include #include namespace arm_conv { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthfirst_driver.hpp b/src/core/NEON/kernels/arm_conv/depthwise/depthfirst_driver.hpp index c3058351070e4486abbb33aeb923c815ff8307f2..592ee7282006c2ca40034754067091c6d2f46505 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthfirst_driver.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthfirst_driver.hpp @@ -24,8 +24,8 @@ #pragma once -#include "src/core/NEON/kernels/assembly/depthwise.hpp" -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "depthwise.hpp" +#include "utils.hpp" namespace arm_conv { namespace depthwise { @@ -72,10 +72,10 @@ class DepthfirstDriver : public DepthwiseCommon std::unique_ptr m_strat; /* Compute the amount of working space required for a single thread. */ - virtual size_t get_working_size_per_thread(unsigned int n_input_channels) const = 0; + virtual size_t get_working_size_per_thread() const = 0; /* Initialise the working space for a thread. */ - virtual void initialise_working_space(void *, unsigned int n_input_channels) const = 0; + virtual void initialise_working_space(void *) const = 0; /* Compute a portion of the output tensor with padding. */ virtual void compute_tile_padded( @@ -164,8 +164,8 @@ class DepthfirstDriver : public DepthwiseCommon { // Get and initialise the working space for this thread. void *thread_working_space = - static_cast(working_space) + thread_id * this->get_working_size_per_thread(args.input_channels); - this->initialise_working_space(thread_working_space, args.input_channels); + static_cast(working_space) + thread_id * this->get_working_size_per_thread(); + this->initialise_working_space(thread_working_space); // Construct convenient representations of the input/output tensors. TensorSpec input_tensor(reinterpret_cast(input), ld_input_row, ld_input_col); @@ -189,7 +189,9 @@ class DepthfirstDriver : public DepthwiseCommon const bool pad_input_top = start_input_i < 0; const int end_input_i = start_input_i + m_strat->get_input_rows(); const bool pad_input_bottom = static_cast(args.input_rows) < end_input_i; - const bool pad_row = pad_input_top || pad_input_bottom || pad_output_bottom; + // We only need to account for input padding if direct padding is not supported. + const bool pad_row = ((pad_input_top || pad_input_bottom) && !this->supports_direct_padding()) + || pad_output_bottom; // Iterate over the columns of the output tensor; we attempt to grab as // much as possible of the unpadded regions, so the loop structure is a @@ -202,7 +204,7 @@ class DepthfirstDriver : public DepthwiseCommon // Determine if we can process a number of unpadded tiles in one go. int n_unpadded_tiles = 0; - if (!pad_input_left) + if ((!pad_input_left) || this->supports_direct_padding()) { // Determine the maximum number of tiles we could handle. n_unpadded_tiles = (args.output_cols - start_output_j) / m_strat->get_output_cols(); @@ -273,9 +275,14 @@ class DepthfirstDriver : public DepthwiseCommon { } - size_t get_working_size(unsigned int n_threads, unsigned int n_input_channels) const override final + size_t get_working_size(unsigned int n_threads) const override final { - return n_threads * this->get_working_size_per_thread(n_input_channels); + return n_threads * this->get_working_size_per_thread(); + } + + virtual bool supports_direct_padding() const + { + return false; } }; diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_common.cpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_common.cpp index c2b861000cf2496e27214b926d577402874bb4fd..2950d5e95737f1971b43691f8782311e26b2e250 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_common.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_common.cpp @@ -10,8 +10,8 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, @@ -22,9 +22,10 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" #include "depthwise_common.hpp" +#include "utils.hpp" + using arm_gemm::iceildiv; namespace arm_conv { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst.hpp index 2620b48e17f7f289bfab072a90978b20dced6100..7b00c9a7af9bfb7d36dcfc5ea08a00902226a4a8 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst.hpp @@ -115,7 +115,7 @@ class DepthwiseDepthfirstStrategy { return interleaves::PackingArguments( this->get_kernel_rows(), this->get_kernel_cols(), sizeof(TWeight), - false, sizeof(int32_t), // Don't pack the bias + false, sizeof(int32_t), this->uses_premultiply(), // Don't pack the bias this->get_vl_type(), sizeof(int32_t), this->get_accumulator_depth_vl(), [this] (unsigned int idx, unsigned int &x, unsigned int &y) -> bool { return this->get_kernel_packing_point(idx, x, y); } @@ -162,6 +162,64 @@ class DepthwiseDepthfirstCommon : public DepthfirstDriverm_args.channel_multiplier != 1 && this->uses_premultiply(); + } + + virtual void fill_inptr_array(const DepthwiseArgs &args, + const TensorSpec &input, + const TInput **inptr_array, TInput *input_buffer, + const unsigned int input_i, const unsigned int input_j, + const unsigned int input_pad_top, const unsigned int input_pad_left) const = 0; + + void initialise_inptr_array(const DepthwiseArgs &args, + unsigned int output_channel_start, unsigned int output_channel_end, + const TensorSpec &input, + const TInput **inptr_array, TInput *input_buffer, TInput *intermediate_buffer, + const unsigned int input_i, const unsigned int input_j, + const unsigned int input_pad_top, const unsigned int input_pad_left, + Tile &multiplied_input + ) const + { + // Compute the input pointer array + const auto input_channel_start = output_channel_start / args.channel_multiplier; + + const auto last_valid_row = std::min(input_pad_top + args.input_rows - input_i, this->m_strat->get_input_rows()); + const auto last_valid_col = std::min(input_pad_left + args.input_cols - input_j, this->m_strat->get_input_cols()); + + const auto tile_rows = last_valid_row - input_pad_top; + const auto tile_cols = last_valid_col - input_pad_left; + + const auto tile_channels = output_channel_end - output_channel_start; + + TensorSpec tile_tensor(0, 0, 0); + if (this->uses_intermediate_array()) { + multiplied_input = Tile(intermediate_buffer, tile_rows, tile_cols, tile_channels); + multiplied_input.load_from(input.base, input.ld_row, input.ld_col, + args.input_rows, args.input_cols, + input_i, input_j, args.channel_multiplier); + + tile_tensor = TensorSpec( + multiplied_input.array, + tile_cols * tile_channels, tile_channels + ); + } else { + tile_tensor = TensorSpec( + input.base + input_i*input.ld_row + input_j*input.ld_col + input_channel_start, + input.ld_row, input.ld_col + ); + } + + fill_inptr_array(args, + tile_tensor, + inptr_array, input_buffer, + input_i, input_j, + input_pad_top, + input_pad_left + ); + } + public: DepthwiseDepthfirstCommon(StratType *const strat, const DepthwiseArgs &args, const OutputStage &os) : DepthfirstDriver(strat, args), m_os(os) @@ -321,6 +379,7 @@ class DepthwiseDepthfirst OutputArrayElement, depthwise_depthfirst::InputArrayElement, InputBufferElement, + IntermediateBufferElement, typename depthwise_depthfirst::WorkspaceFinalElement::Element >; using WorkingSpace = typename WorkspaceManager::WorkspaceType; @@ -347,25 +406,46 @@ class DepthwiseDepthfirst depthwise_depthfirst::stash_bias(this->get_output_stage(), biases); } - size_t get_working_size_per_thread(const unsigned int n_input_channels) const override + size_t get_working_size_per_thread() const override { DepthwiseArgs args(this->m_args); - args.input_channels = n_input_channels; return WorkspaceManager::get_sizeof_workspace( WorkspaceArgs(this->m_strat.get(), args, this->get_output_stage()) ); } - void initialise_working_space(void *buffer, unsigned int n_input_channels) const override + void initialise_working_space(void *buffer) const override { DepthwiseArgs args(this->m_args); - args.input_channels = n_input_channels; WorkspaceManager::initialise( buffer, WorkspaceArgs(this->m_strat.get(), args, this->get_output_stage()) ); } + virtual bool supports_direct_padding() const override + { + using Invoker = depthwise_depthfirst::Invoke; + return Invoker::supports_direct_kernel && this->uses_intermediate_array(); + } + protected: + + void fill_inptr_array(const DepthwiseArgs &args, + const TensorSpec &input, + const TInput **inptr_array, TInput *input_buffer, + const unsigned int input_i, const unsigned int input_j, + const unsigned int input_pad_top, const unsigned int input_pad_left) const override + { + fill_pointer_array( + inptr_array, this->m_strat->get_input_rows(), this->m_strat->get_input_cols(), + input.base, + input.ld_row, input.ld_col, + input_buffer, + input_pad_top, args.input_rows - input_i, + input_pad_left, args.input_cols - input_j + ); + } + void compute_tile_padded( const DepthwiseArgs &args, unsigned int output_i, unsigned int output_j, @@ -380,8 +460,6 @@ class DepthwiseDepthfirst auto ws = reinterpret_cast(working_space_raw); // Compute the input pointer array - const auto input_channel_start = output_channel_start / args.channel_multiplier; - const int ii = static_cast(output_i * args.stride_rows) - args.padding.top; const auto input_pad_top = static_cast(ii < 0 ? -ii : 0); const auto input_i = static_cast(ii < 0 ? 0 : ii); @@ -390,14 +468,10 @@ class DepthwiseDepthfirst const auto input_pad_left = static_cast(ij < 0 ? -ij : 0); const auto input_j = static_cast(ij < 0 ? 0 : ij); - fill_pointer_array( - ws->inptr_array, this->m_strat->get_input_rows(), this->m_strat->get_input_cols(), - input.base + input_i*input.ld_row + input_j*input.ld_col + input_channel_start, - input.ld_row, input.ld_col, - ws->input_buffer, - input_pad_top, args.input_rows - input_i, - input_pad_left, args.input_cols - input_j - ); + Tile multiplied_input; + this->initialise_inptr_array(args, output_channel_start, output_channel_end, input, + ws->inptr_array, ws->input_buffer, ws->intermediate_buffer, + input_i, input_j, input_pad_top, input_pad_left, multiplied_input); // Compute the output pointer array fill_pointer_array( @@ -432,12 +506,11 @@ class DepthwiseDepthfirst const auto os = this->get_output_stage(); // Compute top and bottom padding; hence fill in the initial pointer arrays. - const auto input_channel_start = output_channel_start / args.channel_multiplier; const int ii = static_cast(output_i * args.stride_rows) - args.padding.top; const auto input_pad_top = static_cast(ii < 0 ? -ii : 0); const auto input_i = static_cast(ii < 0 ? 0 : ii); - const auto input_j = output_j * args.stride_cols - args.padding.left; + auto input_j = output_j * args.stride_cols - args.padding.left; // Valid input rows is the smallest of the input rows that aren't padding for this tile, and the number of rows // available. @@ -447,14 +520,10 @@ class DepthwiseDepthfirst const auto input_point_stride = input.ld_col * this->m_strat->get_output_cols() * args.stride_cols; const auto output_point_stride = output.ld_col * this->m_strat->get_output_cols(); - fill_pointer_array( - ws->inptr_array, this->m_strat->get_input_rows(), this->m_strat->get_input_cols(), - input.base + input_i*input.ld_row + input_j*input.ld_col + input_channel_start, - input.ld_row, input.ld_col, - ws->input_buffer, - input_pad_top, args.input_rows - input_i, - 0, args.input_cols - input_j // No left padding - ); + Tile multiplied_input; + this->initialise_inptr_array(args, output_channel_start, output_channel_end, input, + ws->inptr_array, ws->input_buffer, ws->intermediate_buffer, + input_i, input_j, input_pad_top, 0, multiplied_input); fill_pointer_array( ws->outptr_array, this->m_strat->get_output_rows(), this->m_strat->get_output_cols(), @@ -473,16 +542,25 @@ class DepthwiseDepthfirst ); // Update all unpadded pointers - { - auto ptr = ws->inptr_array + strat->get_input_cols() * input_pad_top; - for (auto n = input_pad_top; n < (valid_input_rows + input_pad_top); n++) + if (this->uses_intermediate_array()) { + input_j += input_point_stride / input.ld_col; + multiplied_input.load_from(input.base, + input.ld_row, input.ld_col, + args.input_rows, args.input_cols, + input_i, input_j, args.channel_multiplier); + } else { { - for (auto m = 0u; m < strat->get_input_cols(); m++) + auto ptr = ws->inptr_array + strat->get_input_cols() * input_pad_top; + for (auto n = input_pad_top; n < (valid_input_rows + input_pad_top); n++) { - *(ptr++) += input_point_stride; + for (auto m = 0u; m < strat->get_input_cols(); m++) + { + *(ptr++) += input_point_stride; + } } } } + { auto ptr = ws->outptr_array; for (auto n = 0u; n < valid_output_rows * strat->get_output_cols(); n++) @@ -511,6 +589,13 @@ class DepthwiseDepthfirst if (Invoker::supports_direct_kernel) { + PaddingValues tile_padding = { + args.kernel_cols / 2, + args.kernel_rows / 2, + args.kernel_cols / 2, + args.kernel_rows / 2 + }; + // If the direct kernel is supported, then use it. // Compute the base pointers we'll use in the tile. auto outptr = output.base + output_channel_start + output_i * output.ld_row + output_j * output.ld_col; @@ -518,11 +603,31 @@ class DepthwiseDepthfirst const int start_input_j = output_j * args.stride_cols - args.padding.left; auto inptr = input.base + output_channel_start + start_input_i * input.ld_row + start_input_j * input.ld_col; + auto ld_row = input.ld_row; + auto ld_col = input.ld_col; + + const auto tile_rows = this->m_strat->get_output_rows() * args.stride_rows * n_tile_rows + tile_padding.top + tile_padding.bottom; + const auto tile_cols = this->m_strat->get_output_cols() * args.stride_cols * n_tile_cols + tile_padding.left + tile_padding.right; + const auto tile_channels = output_channel_end - output_channel_start; + + Tile multiplied_input; + if (this->uses_intermediate_array()) { + multiplied_input = Tile(ws->intermediate_buffer, tile_rows, tile_cols, tile_channels); + multiplied_input.load_from(input.base, + input.ld_row, input.ld_col, + args.input_rows, args.input_cols, + start_input_i, start_input_j, args.channel_multiplier); + + ld_row = tile_cols * tile_channels; + ld_col = tile_channels; + inptr = multiplied_input.array; + } + // Execute the kernel Invoker::direct( strat, ws, os, n_tile_rows, n_tile_cols, - inptr, input.ld_row, input.ld_col, + inptr, ld_row, ld_col, outptr, output.ld_row, output.ld_col, parameters, output_channel_end - output_channel_start ); @@ -531,7 +636,6 @@ class DepthwiseDepthfirst { // Otherwise, we repeatedly call the padded kernel but use our knowledge // of the tensor structure to avoid recomputing the pointer array. - const auto input_channel_start = output_channel_start / args.channel_multiplier; const auto n_input_pointers = this->m_strat->get_input_rows() * this->m_strat->get_input_cols(); const auto input_point_stride = input.ld_col * this->m_strat->get_output_cols() * args.stride_cols; @@ -543,16 +647,12 @@ class DepthwiseDepthfirst for (unsigned int tile_i = 0; tile_i < n_tile_rows; tile_i++) { const int input_i = static_cast(output_i * args.stride_rows) - args.padding.top; - const int input_j = static_cast(output_j * args.stride_cols) - args.padding.left; + int input_j = static_cast(output_j * args.stride_cols) - args.padding.left; - fill_pointer_array( - ws->inptr_array, this->m_strat->get_input_rows(), this->m_strat->get_input_cols(), - input.base + input_i*input.ld_row + input_j*input.ld_col + input_channel_start, - input.ld_row, input.ld_col, - ws->input_buffer, - 0, args.input_rows, - 0, args.input_cols - ); + Tile multiplied_input; + this->initialise_inptr_array(args, output_channel_start, output_channel_end, input, + ws->inptr_array, ws->input_buffer, ws->intermediate_buffer, + input_i, input_j, 0, 0, multiplied_input); // Compute the output pointer array fill_pointer_array( @@ -572,10 +672,18 @@ class DepthwiseDepthfirst ); // Progress the pointers - for (auto i = 0u; i < n_input_pointers; i++) - { - ws->inptr_array[i] += input_point_stride; + if (this->uses_intermediate_array()) { + input_j += input_point_stride / input.ld_col; + multiplied_input.load_from(input.base, + input.ld_row, input.ld_col, + args.input_rows, args.input_cols, input_i, input_j, args.channel_multiplier); + } else { + for (auto i = 0u; i < n_input_pointers; i++) + { + ws->inptr_array[i] += input_point_stride; + } } + for (auto i = 0u; i < n_output_pointers; i++) { ws->outptr_array[i] += output_point_stride; diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst_generic.hpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst_generic.hpp index b058ce26f2ff0001db5a9bdf3f789a74c76f8d52..e2d05560a1ce3e61d040d753389dee23e8e710a5 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst_generic.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst_generic.hpp @@ -99,7 +99,7 @@ class GenericDepthfirstStrategy : public DepthwiseDepthfirstStrategyCommonget_kernel_rows(), this->get_kernel_cols(), sizeof(TWeight), - false, sizeof(TAccum), // Don't pack the bias + false, sizeof(TAccum), this->uses_premultiply(), // Don't pack the bias this->get_vl_type(), sizeof(TAccum), this->get_accumulator_depth_vl(), [this] (unsigned int idx, unsigned int &x, unsigned int &y) -> bool { return this->get_kernel_packing_point(idx, x, y); } @@ -115,7 +115,7 @@ class GenericDepthfirstStrategy : public DepthwiseDepthfirstStrategyCommonget_kernel_rows(), this->get_kernel_cols(), sizeof(TWeight), - false, sizeof(TAccum), // Don't pack the bias + false, sizeof(TAccum), this->uses_premultiply(), // Don't pack the bias this->get_vl_type(), sizeof(TAccum), this->get_accumulator_depth_vl(), [this] (unsigned int idx, unsigned int &x, unsigned int &y) -> bool { return this->get_kernel_packing_point(idx, x, y); } @@ -186,7 +186,7 @@ class GenericInputArrayElement static size_t get_element_size(const WorkspaceArgs &args) { const auto kernel_points = args.depthwise_args.kernel_rows * args.depthwise_args.kernel_cols; - return sizeof(T **) * args.strategy->get_input_rows() * args.strategy->get_input_cols() * kernel_points; + return sizeof(T **) * args.strategy->get_output_rows() * args.strategy->get_output_cols() * kernel_points; } template @@ -208,6 +208,7 @@ class DepthwiseDepthfirstGeneric : public DepthwiseDepthfirstCommon, GenericInputArrayElement, InputBufferElement, + IntermediateBufferElement, ActivationsElement >; using WorkingSpace = typename WorkspaceManager::WorkspaceType; @@ -232,21 +233,38 @@ class DepthwiseDepthfirstGeneric : public DepthwiseDepthfirstCommonget_output_stage(), m_bias); } - size_t get_working_size_per_thread(const unsigned int n_input_channels) const override + size_t get_working_size_per_thread() const override { DepthwiseArgs args(this->m_args); - args.input_channels = n_input_channels; return WorkspaceManager::get_sizeof_workspace(WorkspaceArgs(this->m_strat.get(), args, this->get_output_stage())); } - void initialise_working_space(void *buffer, unsigned int n_input_channels) const override + void initialise_working_space(void *buffer) const override { DepthwiseArgs args(this->m_args); - args.input_channels = n_input_channels; return WorkspaceManager::initialise(buffer, WorkspaceArgs(this->m_strat.get(), args, this->get_output_stage())); } protected: + void fill_inptr_array(const DepthwiseArgs &args, + const TensorSpec &input, + const TInput **inptr_array, TInput *input_buffer, + const unsigned int input_i, const unsigned int input_j, + const unsigned int input_pad_top, const unsigned int input_pad_left) const override + { + fill_pointer_array_generic_kernel( + inptr_array, + this->m_strat->get_output_rows(), this->m_strat->get_output_cols(), + args.kernel_rows, args.kernel_cols, + args.stride_rows, args.stride_cols, + input.base, + input.ld_row, input.ld_col, + input_buffer, + input_pad_top, args.input_rows - input_i, + input_pad_left, args.input_cols - input_j + ); + } + void compute_tile_padded( const DepthwiseArgs &args, unsigned int output_i, unsigned int output_j, @@ -268,17 +286,10 @@ class DepthwiseDepthfirstGeneric : public DepthwiseDepthfirstCommon(ij < 0 ? -ij : 0); const auto input_j = static_cast(ij < 0 ? 0 : ij); - fill_pointer_array_generic_kernel( - ws->inptr_array, - this->m_strat->get_output_rows(), this->m_strat->get_output_cols(), - args.kernel_rows, args.kernel_cols, - args.stride_rows, args.stride_cols, - input.base + input_i*input.ld_row + input_j*input.ld_col + channel_start, - input.ld_row, input.ld_col, - ws->input_buffer, - input_pad_top, args.input_rows - input_i, - input_pad_left, args.input_cols - input_j - ); + Tile multiplied_input; + this->initialise_inptr_array(args, channel_start, channel_end, input, + ws->inptr_array, ws->input_buffer, ws->intermediate_buffer, + input_i, input_j, input_pad_top, input_pad_left, multiplied_input); // Compute the output pointer array fill_pointer_array( diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst_multiplier.hpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst_multiplier.hpp index cef568fadd278358bc06f7f12008efaad9facc6b..b93caa2aaab45ed2a17e62f4dfce7230e36dd8ea 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst_multiplier.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst_multiplier.hpp @@ -27,10 +27,6 @@ #include "depthwise_depthfirst.hpp" #include "interleaves/generic_quantized_dot_product.hpp" -#ifdef CYCLE_PROFILING -#include "profiler.hpp" -#endif - #include namespace arm_conv { @@ -46,7 +42,7 @@ class DepthfirstMultiplierStrategy : public DepthwiseDepthfirstStrategyCommonuses_premultiply(), this->get_vl_type(), sizeof(TAccum), 1, [args] (unsigned int pos, unsigned int &x, unsigned int &y) -> bool @@ -61,6 +57,10 @@ class DepthfirstMultiplierStrategy : public DepthwiseDepthfirstStrategyCommonuses_premultiply(), this->get_vl_type(), sizeof(TAccum), 1, [args] (unsigned int pos, unsigned int &x, unsigned int &y) -> bool @@ -211,6 +211,10 @@ class GenericDepthfirstMultiplierStrategy : public DepthwiseDepthfirstStrategyCo } ); } + + bool uses_premultiply() const override { + return false; + } public: GenericDepthfirstMultiplierStrategy(KernelStrategyType *kern, const DepthwiseArgs &args) @@ -487,6 +491,10 @@ class DepthwiseDepthfirstMultiplier : public DepthfirstDriver(strat, args), m_os(os) @@ -510,17 +518,15 @@ class DepthwiseDepthfirstMultiplier : public DepthfirstDriverm_args); - args.input_channels = n_input_channels; return WorkspaceManager::get_sizeof_workspace(WorkspaceArgs(this->m_strat.get(), args, m_os)); } - void initialise_working_space(void *buffer, unsigned int n_input_channels) const override + void initialise_working_space(void *buffer) const override { DepthwiseArgs args(this->m_args); - args.input_channels = n_input_channels; return WorkspaceManager::initialise(buffer, WorkspaceArgs(this->m_strat.get(), args, m_os)); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp16.cpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp16.cpp index 350e93b87424f9109fd40b39abff3bc4846ef766..8fef6f8ae090dafa3872636c96a9e6acdf10a062 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp16.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp16.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -28,6 +28,7 @@ #include "depthwise_depthfirst.hpp" #include "depthwise_depthfirst_generic.hpp" #include "depthwise_depthfirst_multiplier.hpp" +#include "depthwise_planar.hpp" #include "depthwise_implementation_constraints.hpp" @@ -35,14 +36,14 @@ #if defined(__ARM_FP16_ARGS) #if defined(__aarch64__) -#if defined(ARM_COMPUTE_ENABLE_SVE) #if defined(ARM_COMPUTE_ENABLE_SME2) #include "kernels/sme2_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp" #include "kernels/sme2_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp" #include "kernels/sme2_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp" #include "kernels/sme2_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp" #include "kernels/sme2_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp" -#endif // defined(ARM_COMPUTE_ENABLE_SME2) +#endif // defined(ARM_COMPUTE_ENABLE_SME2) +#if defined(ARM_COMPUTE_ENABLE_SVE) #include "kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp" #include "kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp" #include "kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp" @@ -65,9 +66,47 @@ namespace depthwise { namespace { +#if defined(__aarch64__) +#if defined(ENABLE_FP16_KERNELS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) + bool prefer_premultiply(const DepthwiseArgs &args) { + if ((args.stride_rows != args.stride_cols) || (args.kernel_rows != args.kernel_cols)) + { + return false; + } + + unsigned int threshold; + + if (args.stride_rows == 1 && args.kernel_rows == 3) + { + threshold = 30; + } + else if (args.stride_rows == 1 && args.kernel_rows == 5) + { + threshold = 31; + } + else if (args.stride_rows == 2 && args.kernel_rows == 3) + { + threshold = 11; + } + else if (args.stride_rows == 2 && args.kernel_rows == 5) + { + threshold = 19; + } else + { + return false; + } + + return args.channel_multiplier <= threshold; + } + template unsigned int cycle_estimate(const DepthwiseArgs &args, const Nothing &) { + if (args.channel_multiplier > 1 && !prefer_premultiply(args)) + { + return std::numeric_limits::max(); + } + // First-pass: compute the number of output pixels which will be computed. return arm_gemm::roundup(args.output_rows, Strategy::output_rows) * arm_gemm::roundup(args.output_cols, Strategy::output_cols) * @@ -77,24 +116,41 @@ namespace ); } -#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) + template + unsigned int planar_cycle_estimate(const DepthwiseArgs &args, const Nothing &) + { + // First-pass: compute the number of output pixels which will be computed. + return arm_gemm::roundup(args.output_rows, Strategy::output_rows) * + args.output_cols * + arm_gemm::iceildiv( + (long unsigned) args.input_channels * args.channel_multiplier, + arm_gemm::utils::get_vector_length(Strategy::vl_type) + ); + } + + unsigned int multiplier_cycle_estimate(const DepthwiseArgs &args, const Nothing &) + { + return prefer_premultiply(args)? std::numeric_limits::max() : 0; + } + unsigned int not_preferred(const DepthwiseArgs &, const Nothing &) __attribute__ ((unused)); unsigned int not_preferred(const DepthwiseArgs &, const Nothing &) { return std::numeric_limits::max(); } -#endif // defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(ENABLE_FP16_KERNELS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) } static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = { #if defined(__aarch64__) +#if defined(ENABLE_FP16_KERNELS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) #if defined(ARM_COMPUTE_ENABLE_SVE) #if defined(ARM_COMPUTE_ENABLE_SME2) { DepthwiseMethod::DEPTHFIRST, "sme2_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_sme2), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -106,7 +162,6 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "sme2_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_sme2), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -118,7 +173,6 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "sme2_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_sme2), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -130,7 +184,6 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "sme2_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_sme2), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -142,7 +195,6 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "sme2_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_sme2), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -150,12 +202,11 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = return new DepthwiseDepthfirst<__fp16>(strat, args); }, }, -#endif // defined(ARM_COMPUTE_ENABLE_SME2) +#endif // defined(ARM_COMPUTE_ENABLE_SME2) { DepthwiseMethod::DEPTHFIRST, "sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_sve), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -167,7 +218,6 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_sve), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -179,7 +229,6 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_sve), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -191,7 +240,6 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_sve), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -203,7 +251,6 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_sve), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -212,12 +259,10 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = }, }, #endif // defined(ARM_COMPUTE_ENABLE_SVE) -#if defined(ENABLE_FP16_KERNELS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) { DepthwiseMethod::DEPTHFIRST, "a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_fp16), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -229,7 +274,6 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_fp16), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -241,7 +285,6 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_fp16), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -253,7 +296,6 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_fp16), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -265,7 +307,6 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_fp16), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -276,7 +317,7 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = { DepthwiseMethod::DEPTHFIRST, "a64_fp16_nhwc_generic_output3x3_mla_depthfirst", - constraint(has_no_channel_multiplier, cpu_has_fp16), + constraint(cpu_has_fp16), not_preferred, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { auto kern = new a64_fp16_nhwc_generic_output9_mla_depthfirst(args.cpu_info); @@ -288,7 +329,7 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "a64_fp16_nhwc_generic_with_multiplier_output2x8_mla_depthfirst", constraint(cpu_has_fp16, has_channel_multiplier), - nullptr, + multiplier_cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { auto kern = new a64_fp16_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst(args.cpu_info); auto strat = new GenericDepthfirstMultiplierStrategy<__fp16>(kern, args); diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp32.cpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp32.cpp index 09ee9839070b94e691a41a24ee36299c504c7396..760328f3babc5486036e223fdd11c0b9069c1c06 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp32.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp32.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -79,9 +79,46 @@ namespace depthwise { namespace { +#if defined(__aarch64__) + bool prefer_premultiply(const DepthwiseArgs &args) { + if ((args.stride_rows != args.stride_cols) || (args.kernel_rows != args.kernel_cols)) + { + return false; + } + + unsigned int threshold; + + if (args.stride_rows == 1 && args.kernel_rows == 3) + { + threshold = 18; + } + else if (args.stride_rows == 1 && args.kernel_rows == 5) + { + threshold = 5; + } + else if (args.stride_rows == 2 && args.kernel_rows == 3) + { + threshold = 5; + } + else if (args.stride_rows == 2 && args.kernel_rows == 5) + { + threshold = 12; + } else + { + return false; + } + + return args.channel_multiplier <= threshold; + } + template unsigned int cycle_estimate(const DepthwiseArgs &args, const Nothing &) { + if (args.channel_multiplier > 1 && !prefer_premultiply(args)) + { + return std::numeric_limits::max(); + } + // First-pass: compute the number of output pixels which will be computed. return arm_gemm::roundup(args.output_rows, Strategy::output_rows) * arm_gemm::roundup(args.output_cols, Strategy::output_cols) * @@ -103,7 +140,23 @@ namespace ); } -#if defined(__aarch64__) + template + unsigned int fast_mode_cycle_estimate(const DepthwiseArgs &args, const Nothing &) + { + // First-pass: compute the number of output pixels which will be computed. + return arm_gemm::roundup(args.output_rows, Strategy::output_rows) * + arm_gemm::roundup(args.output_cols, Strategy::output_cols) * + arm_gemm::iceildiv( + (long unsigned) args.input_channels * args.channel_multiplier, + arm_gemm::utils::get_vector_length(Strategy::vl_type) + ) * 2 / 3; + } + + unsigned int multiplier_cycle_estimate(const DepthwiseArgs &args, const Nothing &) + { + return prefer_premultiply(args)? std::numeric_limits::max() : 0; + } + unsigned int not_preferred(const DepthwiseArgs &, const Nothing &) { return std::numeric_limits::max(); @@ -234,8 +287,7 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { DepthwiseMethod::DEPTHFIRST, "sme2_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst", constraint(cpu_has_sme, cpu_has_sme2, - is_supported, - has_no_channel_multiplier), + is_supported), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { auto strat = new sme2_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst(args.cpu_info); @@ -246,8 +298,7 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { DepthwiseMethod::DEPTHFIRST, "sme2_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst", constraint(cpu_has_sme, cpu_has_sme2, - is_supported, - has_no_channel_multiplier), + is_supported), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { auto strat = new sme2_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst(args.cpu_info); @@ -258,8 +309,7 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { DepthwiseMethod::DEPTHFIRST, "sme2_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint(cpu_has_sme, cpu_has_sme2, - is_supported, - has_no_channel_multiplier), + is_supported), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { auto strat = new sme2_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst(args.cpu_info); @@ -270,8 +320,7 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { DepthwiseMethod::DEPTHFIRST, "sme2_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint(cpu_has_sme, cpu_has_sme2, - is_supported, - has_no_channel_multiplier), + is_supported), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { auto strat = new sme2_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst(args.cpu_info); @@ -283,7 +332,6 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { DepthwiseMethod::DEPTHFIRST, "sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_sve), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { @@ -295,7 +343,6 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { DepthwiseMethod::DEPTHFIRST, "sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_sve), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { @@ -307,7 +354,6 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { DepthwiseMethod::DEPTHFIRST, "sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_sve), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { @@ -319,7 +365,6 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { DepthwiseMethod::DEPTHFIRST, "sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_sve), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { @@ -331,7 +376,6 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { DepthwiseMethod::DEPTHFIRST, "sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_sve), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { @@ -342,7 +386,7 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { { DepthwiseMethod::DEPTHFIRST, "sve_fp32_nhwc_generic_output3x3_mla_depthfirst", - constraint(has_no_channel_multiplier, cpu_has_sve), + constraint(cpu_has_sve), not_preferred, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { auto kern = new sve_fp32_nhwc_generic_output9_mla_depthfirst(args.cpu_info); @@ -355,7 +399,7 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { "sve_fp32_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst", constraint(is_supported, cpu_has_sve, has_channel_multiplier), - nullptr, + multiplier_cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { auto strat = new sve_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier(strat, args); @@ -366,7 +410,7 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { "sve_fp32_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst", constraint(is_supported, cpu_has_sve, has_channel_multiplier), - nullptr, + multiplier_cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { auto strat = new sve_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier(strat, args); @@ -376,7 +420,7 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { DepthwiseMethod::DEPTHFIRST, "sve_fp32_nhwc_generic_with_multiplier_output2x8_mla_depthfirst", constraint(cpu_has_sve, has_channel_multiplier), - nullptr, + multiplier_cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { auto kern = new sve_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst(args.cpu_info); auto strat = new GenericDepthfirstMultiplierStrategy(kern, args); @@ -387,8 +431,7 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { { DepthwiseMethod::DEPTHFIRST, "a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst", - constraint(is_supported, - has_no_channel_multiplier), + constraint(is_supported), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { auto strat = new a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst(args.cpu_info); @@ -398,8 +441,7 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { { DepthwiseMethod::DEPTHFIRST, "a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst", - constraint(is_supported, - has_no_channel_multiplier), + constraint(is_supported), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { auto strat = new a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst(args.cpu_info); @@ -409,8 +451,7 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { { DepthwiseMethod::DEPTHFIRST, "a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst", - constraint(is_supported, - has_no_channel_multiplier), + constraint(is_supported), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { auto strat = new a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst(args.cpu_info); @@ -420,8 +461,7 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { { DepthwiseMethod::DEPTHFIRST, "a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst", - constraint(is_supported, - has_no_channel_multiplier), + constraint(is_supported), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { auto strat = new a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst(args.cpu_info); @@ -431,8 +471,7 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { { DepthwiseMethod::DEPTHFIRST, "a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst", - constraint(is_supported, - has_no_channel_multiplier), + constraint(is_supported), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { auto strat = new a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst(args.cpu_info); @@ -442,7 +481,7 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { { DepthwiseMethod::DEPTHFIRST, "a64_fp32_nhwc_generic_output3x3_mla_depthfirst", - constraint(has_no_channel_multiplier), + nullptr, not_preferred, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { auto kern = new a64_fp32_nhwc_generic_output9_mla_depthfirst(args.cpu_info); @@ -455,7 +494,7 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { "a64_fp32_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst", constraint(is_supported, has_channel_multiplier), - nullptr, + multiplier_cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { auto strat = new a64_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier(strat, args); @@ -466,7 +505,7 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { "a64_fp32_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst", constraint(is_supported, has_channel_multiplier), - nullptr, + multiplier_cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { auto strat = new a64_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier(strat, args); @@ -476,7 +515,7 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { DepthwiseMethod::DEPTHFIRST, "a64_fp32_nhwc_generic_with_multiplier_output2x8_mla_depthfirst", constraint(has_channel_multiplier), - nullptr, + multiplier_cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { auto kern = new a64_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst(args.cpu_info); auto strat = new GenericDepthfirstMultiplierStrategy(kern, args); diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_implementation_constraints.hpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_implementation_constraints.hpp index 1ba7694f1e66118dc1f186ce7966d2a5ac85c213..15064aeedc2dfb98dcbdbb4d0b643a05733b78de 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_implementation_constraints.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_implementation_constraints.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -33,7 +33,7 @@ #pragma once #include "arm_gemm.hpp" -#include "src/core/NEON/kernels/assembly/depthwise.hpp" +#include "depthwise.hpp" namespace arm_conv { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp index 2b2e6f355505a02e9a72ff92141458fe86b4a0f2..c3daaf04fea052d664f97f698b45583fc6117533 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp @@ -22,6 +22,8 @@ * SOFTWARE. */ +#pragma once + #include "depthfirst_driver.hpp" #include "interleaves/generic.hpp" @@ -52,7 +54,7 @@ struct PlanarKernelType; template struct PlanarKernelType { - using Type = std::function TOutput **, const size_t *, const size_t *, unsigned int output_cols, unsigned int start_channels, unsigned int valid_channels, TAccum act_min, TAccum act_max - )>; + ); template static inline void execute( @@ -89,7 +91,7 @@ struct PlanarKernelType template struct PlanarKernelType { - using Type = std::function; + ); template static inline void execute( @@ -151,7 +153,7 @@ class PlanarStrategy : public IPlanarStrategy { return interleaves::PackingArguments( m_kernel_rows, m_kernel_cols, sizeof(TWeight), - false, sizeof(TAccum), // Don't pack the bias + false, sizeof(TAccum), true, // Don't pack the bias m_vl_type, sizeof(TAccum), 1, // Accumulator depth of 1 TODO [this] (unsigned int idx, unsigned int &x, unsigned int &y) -> bool { return this->get_kernel_packing_point(idx, x, y); } @@ -274,7 +276,7 @@ class DepthwisePlanar : public DepthwiseCommon depthwise_depthfirst::stash_bias(this->m_os, biases); } - size_t get_working_size(unsigned int n_threads, unsigned int) const override + size_t get_working_size(unsigned int n_threads) const override { return this->get_working_size_per_thread() * n_threads; } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_s8q.cpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_s8q.cpp index 2d03183c5986385011972f1376a7f0994f61daa5..6ecdc36bf0ea0b92fa5c0415ba33541734c149e7 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_s8q.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_s8q.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -75,6 +75,11 @@ bool qp_weights_are_symmetric(const DepthwiseArgs &, const void *_qp) const auto qp = static_cast(_qp); return qp->b_offset == 0; } + +uint64_t not_preferred(const DepthwiseArgs &, const Requantize32 &) +{ + return std::numeric_limits::max(); +} #endif // defined(__aarch64__) } @@ -139,7 +144,6 @@ static const DepthwiseImplementation depth DepthwiseMethod::DEPTHFIRST, "sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, qp_weights_are_symmetric, cpu_has_sve2), @@ -153,7 +157,6 @@ static const DepthwiseImplementation depth DepthwiseMethod::DEPTHFIRST, "sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -166,7 +169,6 @@ static const DepthwiseImplementation depth DepthwiseMethod::DEPTHFIRST, "sve_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -179,7 +181,6 @@ static const DepthwiseImplementation depth DepthwiseMethod::DEPTHFIRST, "sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -192,7 +193,6 @@ static const DepthwiseImplementation depth DepthwiseMethod::DEPTHFIRST, "sve_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -208,7 +208,7 @@ static const DepthwiseImplementation depth qp_has_no_left_shift, has_channel_multiplier, cpu_has_sve2), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto strat = new sve_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier(strat, args, qp); @@ -221,7 +221,7 @@ static const DepthwiseImplementation depth qp_has_no_left_shift, has_channel_multiplier, cpu_has_sve2), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto strat = new sve_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier(strat, args, qp); @@ -232,7 +232,6 @@ static const DepthwiseImplementation depth DepthwiseMethod::DEPTHFIRST, "a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_weights_are_symmetric, qp_has_no_left_shift, cpu_has_dot_product), @@ -246,7 +245,6 @@ static const DepthwiseImplementation depth DepthwiseMethod::DEPTHFIRST, "a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_dot_product), nullptr, @@ -259,7 +257,6 @@ static const DepthwiseImplementation depth DepthwiseMethod::DEPTHFIRST, "a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { @@ -271,7 +268,6 @@ static const DepthwiseImplementation depth DepthwiseMethod::DEPTHFIRST, "a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { @@ -283,7 +279,6 @@ static const DepthwiseImplementation depth DepthwiseMethod::DEPTHFIRST, "a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { @@ -294,7 +289,7 @@ static const DepthwiseImplementation depth { DepthwiseMethod::DEPTHFIRST, "a64_s8q_nhwc_generic_output3x3_mla_depthfirst", - constraint(has_no_channel_multiplier), + nullptr, nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto kernel = new a64_s8q_nhwc_generic_output9_mla_depthfirst(args.cpu_info); @@ -309,7 +304,7 @@ static const DepthwiseImplementation depth qp_has_no_left_shift, has_channel_multiplier, cpu_has_dot_product), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto strat = new a64_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier(strat, args, qp); @@ -322,7 +317,7 @@ static const DepthwiseImplementation depth qp_has_no_left_shift, has_channel_multiplier, cpu_has_dot_product), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto strat = new a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier(strat, args, qp); @@ -332,7 +327,7 @@ static const DepthwiseImplementation depth DepthwiseMethod::DEPTHFIRST, "a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst", constraint(has_channel_multiplier), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto kern = new a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst(args.cpu_info); auto strat = new GenericDepthfirstMultiplierStrategy(kern, args); diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_strategies_common.cpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_strategies_common.cpp index 33f2177efe84115636b7d448448b5cc6d2a25889..37892b696338fc7c0c6d177caad172a063de11dc 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_strategies_common.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_strategies_common.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -41,6 +41,8 @@ unsigned int DepthfirstStrategyUntyped::get_n_input_points() const { return this unsigned int DepthfirstStrategyUntyped::get_n_output_points() const { return this->get_output_rows() * this->get_output_cols(); } unsigned int DepthfirstStrategyUntyped::get_n_kernel_points() const { return this->get_kernel_rows() * this->get_kernel_cols(); } +bool DepthfirstStrategyUntyped::uses_premultiply() const { return true; } + unsigned int DepthfirstStrategyUntyped::get_accumulator_depth_vl() const { return 1; } bool DepthfirstStrategyUntyped::get_kernel_packing_point(const unsigned int index, unsigned int &x, unsigned int &y) const diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_strategies_common.hpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_strategies_common.hpp index 99b91fb83391c6ac3d40abe526744caec3152c4d..19cf26dd2fcfbbb4a6ee0176757e4dc0256a5821 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_strategies_common.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_strategies_common.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,7 +24,7 @@ #pragma once -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "interleaves/generic.hpp" #include "depthfirst_driver.hpp" @@ -49,6 +49,8 @@ class DepthfirstStrategyUntyped : public IDepthfirstStrategy virtual unsigned int get_n_output_points() const; virtual unsigned int get_n_kernel_points() const; + virtual bool uses_premultiply() const; + // Get the number of VLs used in the accumulator, this defaults to 1. virtual unsigned int get_accumulator_depth_vl() const; @@ -65,7 +67,7 @@ class DepthfirstStrategy : public DepthfirstStrategyUntyped { interleaves::PackingArguments packing_args( this->get_kernel_rows(), this->get_kernel_cols(), sizeof(TWeight), - true, sizeof(TAccum), + true, sizeof(TAccum), this->uses_premultiply(), this->get_vl_type(), sizeof(TAccum), this->get_accumulator_depth_vl(), [this] (unsigned int idx, unsigned int &x, unsigned int &y) -> bool { return this->get_kernel_packing_point(idx, x, y); } @@ -81,7 +83,7 @@ class DepthfirstStrategy : public DepthfirstStrategyUntyped { interleaves::PackingArguments packing_args( this->get_kernel_rows(), this->get_kernel_cols(), sizeof(TWeight), - true, sizeof(TAccum), + true, sizeof(TAccum), this->uses_premultiply(), this->get_vl_type(), sizeof(TAccum), this->get_accumulator_depth_vl(), [this] (unsigned int idx, unsigned int &x, unsigned int &y) -> bool { return this->get_kernel_packing_point(idx, x, y); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8q.cpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8q.cpp index 9dbd89fb5249e2e471d387154d866b5b0b1c229c..236930ee265649636d8d7dcbfba1ef53511ea3be 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8q.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8q.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -71,6 +71,16 @@ using arm_gemm::Requantize32; namespace arm_conv { namespace depthwise { +namespace +{ +#if defined(__aarch64__) +uint64_t not_preferred(const DepthwiseArgs &, const Requantize32 &) +{ + return std::numeric_limits::max(); +} +#endif // defined(__aarch64__) +} + static const DepthwiseImplementation depthwise_u8q_methods[] = { #if defined(__aarch64__) #if defined(ARM_COMPUTE_ENABLE_SVE) @@ -132,7 +142,6 @@ static const DepthwiseImplementation de DepthwiseMethod::DEPTHFIRST, "sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -145,7 +154,6 @@ static const DepthwiseImplementation de DepthwiseMethod::DEPTHFIRST, "sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -158,7 +166,6 @@ static const DepthwiseImplementation de DepthwiseMethod::DEPTHFIRST, "sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -171,7 +178,6 @@ static const DepthwiseImplementation de DepthwiseMethod::DEPTHFIRST, "sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -187,7 +193,7 @@ static const DepthwiseImplementation de qp_has_no_left_shift, has_channel_multiplier, cpu_has_sve2), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto strat = new sve_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier(strat, args, qp); @@ -200,7 +206,7 @@ static const DepthwiseImplementation de qp_has_no_left_shift, has_channel_multiplier, cpu_has_sve2), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto strat = new sve_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier(strat, args, qp); @@ -212,7 +218,6 @@ static const DepthwiseImplementation de "a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst", constraint(is_supported, cpu_has_dot_product, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { @@ -225,7 +230,6 @@ static const DepthwiseImplementation de DepthwiseMethod::DEPTHFIRST, "a64_u8qa_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_zero_a_offset, qp_has_no_left_shift), nullptr, @@ -238,7 +242,6 @@ static const DepthwiseImplementation de DepthwiseMethod::DEPTHFIRST, "a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_zero_a_offset, qp_has_no_left_shift), nullptr, @@ -251,7 +254,6 @@ static const DepthwiseImplementation de DepthwiseMethod::DEPTHFIRST, "a64_u8qa_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_zero_a_offset, qp_has_no_left_shift), nullptr, @@ -265,7 +267,6 @@ static const DepthwiseImplementation de DepthwiseMethod::DEPTHFIRST, "a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { @@ -277,7 +278,6 @@ static const DepthwiseImplementation de DepthwiseMethod::DEPTHFIRST, "a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { @@ -289,7 +289,6 @@ static const DepthwiseImplementation de DepthwiseMethod::DEPTHFIRST, "a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { @@ -300,7 +299,7 @@ static const DepthwiseImplementation de { DepthwiseMethod::DEPTHFIRST, "a64_u8q_nhwc_generic_output3x3_mla_depthfirst", - constraint(has_no_channel_multiplier), + nullptr, nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto kernel = new a64_u8q_nhwc_generic_output9_mla_depthfirst(args.cpu_info); @@ -315,7 +314,7 @@ static const DepthwiseImplementation de cpu_has_dot_product, has_channel_multiplier, qp_has_no_left_shift), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto strat = new a64_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier(strat, args, qp); @@ -328,7 +327,7 @@ static const DepthwiseImplementation de cpu_has_dot_product, has_channel_multiplier, qp_has_no_left_shift), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto strat = new a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier(strat, args, qp); @@ -338,7 +337,7 @@ static const DepthwiseImplementation de DepthwiseMethod::DEPTHFIRST, "a64_u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst", constraint(has_channel_multiplier), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto kern = new a64_u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst(args.cpu_info); auto strat = new GenericDepthfirstMultiplierStrategy(kern, args); diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8s8u8q.cpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8s8u8q.cpp index 0665c67fbb9f23ebee71dac3e22a783df1f2b85b..a888958b7662acb66cfc3c427593a7e38f3cc4d6 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8s8u8q.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8s8u8q.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -58,6 +58,16 @@ using arm_gemm::Requantize32; namespace arm_conv { namespace depthwise { +namespace +{ +#if defined(__aarch64__) +uint64_t not_preferred(const DepthwiseArgs &, const Requantize32 &) +{ + return std::numeric_limits::max(); +} +#endif // defined(__aarch64__) +} + static const DepthwiseImplementation depthwise_u8q_methods[] = { #if defined(__aarch64__) #if defined(ARM_COMPUTE_ENABLE_SVE) @@ -119,7 +129,6 @@ static const DepthwiseImplementation dep DepthwiseMethod::DEPTHFIRST, "sve_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -132,7 +141,6 @@ static const DepthwiseImplementation dep DepthwiseMethod::DEPTHFIRST, "sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -145,7 +153,6 @@ static const DepthwiseImplementation dep DepthwiseMethod::DEPTHFIRST, "sve_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -159,7 +166,6 @@ static const DepthwiseImplementation dep DepthwiseMethod::DEPTHFIRST, "a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { @@ -171,7 +177,6 @@ static const DepthwiseImplementation dep DepthwiseMethod::DEPTHFIRST, "a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { @@ -183,7 +188,6 @@ static const DepthwiseImplementation dep DepthwiseMethod::DEPTHFIRST, "a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { @@ -194,7 +198,7 @@ static const DepthwiseImplementation dep { DepthwiseMethod::DEPTHFIRST, "a64_u8s8u8q_nhwc_generic_output3x3_mla_depthfirst", - constraint(has_no_channel_multiplier), + nullptr, nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto kernel = new a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst(args.cpu_info); @@ -206,7 +210,7 @@ static const DepthwiseImplementation dep DepthwiseMethod::DEPTHFIRST, "a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst", constraint(has_channel_multiplier), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto kern = new a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst(args.cpu_info); auto strat = new GenericDepthfirstMultiplierStrategy(kern, args); diff --git a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.cpp b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.cpp deleted file mode 100644 index d59d6b7e3583125165110c07d6dc33a01b46e774..0000000000000000000000000000000000000000 --- a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.cpp +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2021 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include "8b_mla.hpp" - -size_t generic_get_packed_size( - const VLType vec_type, - const unsigned int acc_depth, - const unsigned int kernel_rows, - const unsigned int kernel_cols, - const unsigned int n_input_channels -) -{ - const auto per_iter = acc_depth * arm_gemm::utils::get_vector_length(vec_type); - return arm_gemm::roundup((long unsigned int) n_input_channels, per_iter) * kernel_rows * kernel_cols * sizeof(int8_t); -} - -void generic_pack( - const VLType vec_type, - const unsigned int acc_depth, - const unsigned int kernel_rows, - const unsigned int kernel_cols, - const unsigned int n_channels, - void *_outptr, - const void *_weights, - size_t ld_weight_col, - size_t ld_weight_row -) -{ - int8_t *outptr = reinterpret_cast(_outptr); - const int8_t *weights = reinterpret_cast(_weights); - - // Get the strides - ld_weight_col = (ld_weight_col == 0) ? n_channels * sizeof(int8_t) : ld_weight_col; - ld_weight_row = (ld_weight_row == 0) ? kernel_cols * ld_weight_col : ld_weight_row; - - // Pack into per-iter chunks. - const auto per_iter = acc_depth * arm_gemm::utils::get_vector_length(vec_type); - for (unsigned int c = 0; c < n_channels; c += per_iter) - { - auto weight_row = weights + c; - const auto to_copy = std::min(per_iter, n_channels - c); - - for (unsigned int i = 0; i < kernel_rows; i++) - { - auto weight_col = weight_row; - - for (unsigned int j = 0; j < kernel_cols; j++) - { - memcpy(outptr, weight_col, to_copy); - outptr += per_iter; - weight_col += ld_weight_col; - } - - weight_row += ld_weight_row; - } - } -} - -namespace arm_conv { -namespace depthwise { - -ADD_IMPLEMENTATION(a64, s8q, int8_t, None, 2, 3, 3) -ADD_IMPLEMENTATION(a64, s8q, int8_t, None, 2, 5, 5) -ADD_IMPLEMENTATION(a64, u8q, uint8_t, None, 2, 3, 3) -ADD_IMPLEMENTATION(a64, u8q, uint8_t, None, 2, 5, 5) - -} // namespace depthwise -} // namespace arm_conv diff --git a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.hpp b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.hpp deleted file mode 100644 index 3176d1deddd66a828e993973ebfe9c929f9e1889..0000000000000000000000000000000000000000 --- a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.hpp +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2021 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include "arm_gemm.hpp" -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" -#include "src/core/NEON/kernels/assembly/depthwise.hpp" -#include -#include - -using namespace arm_gemm; - -size_t generic_get_packed_size( - const VLType vec_type, - const unsigned int acc_depth, - const unsigned int kernel_rows, - const unsigned int kernel_cols, - const unsigned int n_input_channels -); - -void generic_pack( - const VLType vec_type, - const unsigned int acc_depth, - const unsigned int kernel_rows, - const unsigned int kernel_cols, - const unsigned int n_channels, - void *_outptr, - const void *_weights, - size_t ld_weight_col, - size_t ld_weight_row -); - -#define ADD_IMPLEMENTATION(ARCH, TYPENAME, TYPE, VEC_TYPE, ACC_DEPTH, KERN_ROWS, KERN_COLS) \ -struct interleave_ ## ARCH ## _ ## TYPENAME ## _ ## KERN_ROWS ## x ## KERN_COLS ## _mla \ -{ \ - static size_t get_packed_size(const DepthwiseArgs &args); \ - static void pack_parameters( \ - unsigned int n_channels, void *outptr, \ - const TYPE *weights, size_t ld_weight_col, size_t ld_weight_row \ - ); \ -}; \ -\ -size_t interleave_ ## ARCH ## _ ## TYPENAME ## _ ## KERN_ROWS ## x ## KERN_COLS ## _mla::get_packed_size(const DepthwiseArgs &args) \ -{ \ - return generic_get_packed_size(VLType::VEC_TYPE, ACC_DEPTH, KERN_ROWS, KERN_COLS, args.input_channels); \ -} \ -\ -void interleave_ ## ARCH ## _ ## TYPENAME ## _ ## KERN_ROWS ## x ## KERN_COLS ## _mla::pack_parameters(unsigned int n_channels, void *outptr, \ - const TYPE *weights, size_t ld_weight_col, size_t ld_weight_row) \ -{ \ - generic_pack(VLType::VEC_TYPE, ACC_DEPTH, KERN_ROWS, KERN_COLS, n_channels, outptr, weights, ld_weight_col, ld_weight_row); \ -} diff --git a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/a64_s8q_3x3_dot.cpp b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/a64_s8q_3x3_dot.cpp index adda78f1640dd41a6ecb16d2acf992b581fe4de2..3de4bdc1fba73cbdd08fe625173b24e9caf9bd20 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/a64_s8q_3x3_dot.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/a64_s8q_3x3_dot.cpp @@ -25,8 +25,8 @@ #if defined(__aarch64__) #include "arm_gemm.hpp" -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" -#include "src/core/NEON/kernels/assembly/depthwise.hpp" +#include "utils.hpp" +#include "depthwise.hpp" #include namespace arm_conv { @@ -42,7 +42,7 @@ size_t interleave_a64_s8q_3x3_dot::get_packed_size(const DepthwiseArgs &args) { // We store 7 vectors for every of channels. const unsigned int n = arm_gemm::roundup( - arm_gemm::iceildiv((long unsigned int) args.input_channels, + arm_gemm::iceildiv((long unsigned int) args.input_channels * args.channel_multiplier, get_vector_length(arm_gemm::VLType::None)), 4lu ); return n * 7 * get_vector_length(arm_gemm::VLType::None); @@ -54,162 +54,162 @@ void interleave_a64_s8q_3x3_dot::pack_parameters(unsigned int n_channels, void * "cmp %x[ld_weight_col], XZR\n" "csel %x[ld_weight_col], %x[ld_weight_col], %x[n_channels], NE\n" "movi v16.4s, #0x9\n" - "movi v0.16b, #0x0\n" + "movi v31.16b, #0x0\n" "mov x21, #0x3\n" "mul x21, %x[ld_weight_col], x21\n" "add x20, %x[qp], %[offsetof_input_offset]\n" - "ld1r { v31.4s }, [x20]\n" - "add x20, %x[qp], %[offsetof_weights_offset]\n" "ld1r { v30.4s }, [x20]\n" + "add x20, %x[qp], %[offsetof_weights_offset]\n" + "ld1r { v29.4s }, [x20]\n" "cmp %x[ld_weight_row], XZR\n" - "mul v30.4s, v30.4s, v31.4s\n" + "mul v29.4s, v29.4s, v30.4s\n" "csel %x[ld_weight_row], %x[ld_weight_row], x21, NE\n" "lsr x21, %x[n_channels], #0x2\n" - "movi v29.16b, #0x1\n" - "mul v30.4s, v30.4s, v16.4s\n" + "movi v28.16b, #0x1\n" + "mul v29.4s, v29.4s, v16.4s\n" "add x25, %x[weights], %x[ld_weight_row]\n" "add x20, %x[qp], %[offsetof_per_layer_mul]\n" - "ld1r { v28.4s }, [x20]\n" - "add x20, %x[qp], %[offsetof_per_layer_right_shift]\n" "ld1r { v27.4s }, [x20]\n" + "add x20, %x[qp], %[offsetof_per_layer_right_shift]\n" + "ld1r { v26.4s }, [x20]\n" "add x24, x25, %x[ld_weight_row]\n" "add x23, %x[ld_weight_col], %x[ld_weight_col]\n" "mov x22, #0x0\n" "cbz x21, 4f\n" "1:" // Loop - "movi v26.4s, #0x0\n" + "movi v25.4s, #0x0\n" "cbz %x[bias], 2f\n" - "ldr q26, [%x[bias], x22]\n" + "ldr q25, [%x[bias], x22]\n" "2:" // Loop: Skip bias load - "ldr s25, [%x[weights], #0x0]\n" - "ldr s22, [%x[weights], %x[ld_weight_col]]\n" - "zip1 v22.16b, v22.16b, v0.16b\n" - "movi v24.4s, #0x0\n" - "ldr s20, [%x[weights], x23]\n" - "ldr s23, [x25, #0x0]\n" - "zip1 v20.16b, v25.16b, v20.16b\n" - "zip1 v22.16b, v20.16b, v22.16b\n" - "ldr s21, [x25, %x[ld_weight_col]]\n" - "ldr s18, [x25, x23]\n" - "zip1 v20.16b, v23.16b, v18.16b\n" - "zip1 v18.16b, v21.16b, v0.16b\n" + "ldr s19, [%x[weights], #0x0]\n" + "ldr s16, [%x[weights], %x[ld_weight_col]]\n" + "zip1 v17.16b, v16.16b, v31.16b\n" + "movi v21.4s, #0x0\n" + "ldr s16, [%x[weights], x23]\n" + "ldr s18, [x25, #0x0]\n" + "zip1 v16.16b, v19.16b, v16.16b\n" + "zip1 v20.16b, v16.16b, v17.16b\n" + "ldr s17, [x25, %x[ld_weight_col]]\n" + "ldr s16, [x25, x23]\n" + "zip1 v18.16b, v18.16b, v16.16b\n" + "zip1 v16.16b, v17.16b, v31.16b\n" "ldr s17, [x24, #0x0]\n" "ldr s19, [x24, %x[ld_weight_col]]\n" - ".inst 0x4e9697b8 // sdot v24.4s, v29.16b, v22.16b\n" - "zip1 v18.16b, v20.16b, v18.16b\n" + ".inst 0x4e949795 // sdot v21.4s, v28.16b, v20.16b\n" + "zip1 v18.16b, v18.16b, v16.16b\n" "ldr s16, [x24, x23]\n" "zip1 v17.16b, v17.16b, v16.16b\n" - "zip1 v16.16b, v19.16b, v0.16b\n" - ".inst 0x4e9297b8 // sdot v24.4s, v29.16b, v18.16b\n" + "zip1 v16.16b, v19.16b, v31.16b\n" + ".inst 0x4e929795 // sdot v21.4s, v28.16b, v18.16b\n" "zip1 v16.16b, v17.16b, v16.16b\n" - ".inst 0x4e9097b8 // sdot v24.4s, v29.16b, v16.16b\n" + ".inst 0x4e909795 // sdot v21.4s, v28.16b, v16.16b\n" "add %x[weights], %x[weights], #0x4\n" "add x25, x25, #0x4\n" - "mls v26.4s, v24.4s, v31.4s\n" + "mls v25.4s, v21.4s, v30.4s\n" "add x24, x24, #0x4\n" - "add v26.4s, v26.4s, v30.4s\n" - "str q26, [%x[outptr], #0x0]\n" - "str q22, [%x[outptr], #0x10]\n" + "add v25.4s, v25.4s, v29.4s\n" + "str q25, [%x[outptr], #0x0]\n" + "str q20, [%x[outptr], #0x10]\n" "str q18, [%x[outptr], #0x20]\n" "str q16, [%x[outptr], #0x30]\n" "add %x[outptr], %x[outptr], #0x40\n" "cbz %x[rq_mul_perchannel], 3f\n" - "ldr q28, [%x[rq_mul_perchannel], x22]\n" - "ldr q27, [%x[rq_shift_perchannel], x22]\n" + "ldr q27, [%x[rq_mul_perchannel], x22]\n" + "ldr q26, [%x[rq_shift_perchannel], x22]\n" "3:" // Loop: Quantisation parameters: Store "subs x21, x21, #0x1\n" - "str q28, [%x[outptr], #0x0]\n" + "str q27, [%x[outptr], #0x0]\n" "add x22, x22, #0x10\n" - "str q27, [%x[outptr], #0x10]\n" + "str q26, [%x[outptr], #0x10]\n" "add %x[outptr], %x[outptr], #0x20\n" "bgt 1b\n" "tst %x[n_channels], #0x3\n" "beq 13f\n" "4:" // Oddments - "movi v26.4s, #0x0\n" + "movi v25.4s, #0x0\n" "cbz %x[bias], 7f\n" "add %x[bias], %x[bias], x22\n" "tbz %x[n_channels], #1, 5f\n" - "ld1 { v26.d }[0], [%x[bias]], #0x8\n" + "ld1 { v25.d }[0], [%x[bias]], #0x8\n" "tbz %x[n_channels], #0, 6f\n" - "ld1 { v26.s }[2], [%x[bias]], #0x4\n" + "ld1 { v25.s }[2], [%x[bias]], #0x4\n" "b 6f\n" "5:" // Oddments: Load bias: Bit 1: Unset - "ld1 { v26.s }[0], [%x[bias]], #0x4\n" + "ld1 { v25.s }[0], [%x[bias]], #0x4\n" "6:" // Oddments: Load bias: Bit 1: End "7:" // Oddments: Skip bias load "tbz %x[n_channels], #1, 8f\n" - "ld1 { v25.h }[0], [%x[weights]]\n" - "ld1 { v23.h }[0], [x25]\n" + "ld1 { v17.h }[0], [%x[weights]]\n" + "ld1 { v24.h }[0], [x25]\n" "add x21, %x[weights], %x[ld_weight_col]\n" "add x20, %x[weights], x23\n" - "ld1 { v22.h }[0], [x21]\n" - "ld1 { v20.h }[0], [x20]\n" + "ld1 { v20.h }[0], [x21]\n" + "ld1 { v16.h }[0], [x20]\n" "add x21, x25, %x[ld_weight_col]\n" "add x20, x25, x23\n" - "ld1 { v21.h }[0], [x21]\n" + "ld1 { v19.h }[0], [x21]\n" "ld1 { v18.h }[0], [x20]\n" "add x21, x24, %x[ld_weight_col]\n" "add x20, x24, x23\n" - "ld1 { v17.h }[0], [x24]\n" - "ld1 { v19.h }[0], [x21]\n" + "ld1 { v23.h }[0], [x24]\n" + "ld1 { v22.h }[0], [x21]\n" "add %x[weights], %x[weights], #0x2\n" "add x25, x25, #0x2\n" - "ld1 { v16.h }[0], [x20]\n" + "ld1 { v21.h }[0], [x20]\n" "add x24, x24, #0x2\n" "tbz %x[n_channels], #0, 9f\n" - "ld1 { v25.b }[2], [%x[weights]]\n" - "ld1 { v23.b }[2], [x25]\n" + "ld1 { v17.b }[2], [%x[weights]]\n" + "ld1 { v24.b }[2], [x25]\n" "add x21, %x[weights], %x[ld_weight_col]\n" "add x20, %x[weights], x23\n" - "ld1 { v22.b }[2], [x21]\n" - "ld1 { v20.b }[2], [x20]\n" + "ld1 { v20.b }[2], [x21]\n" + "ld1 { v16.b }[2], [x20]\n" "add x21, x25, %x[ld_weight_col]\n" "add x20, x25, x23\n" - "ld1 { v21.b }[2], [x21]\n" + "ld1 { v19.b }[2], [x21]\n" "ld1 { v18.b }[2], [x20]\n" "add x21, x24, %x[ld_weight_col]\n" "add x20, x24, x23\n" - "ld1 { v17.b }[2], [x24]\n" - "ld1 { v19.b }[2], [x21]\n" + "ld1 { v23.b }[2], [x24]\n" + "ld1 { v22.b }[2], [x21]\n" "add %x[weights], %x[weights], #0x1\n" - "ld1 { v16.b }[2], [x20]\n" + "ld1 { v21.b }[2], [x20]\n" "b 9f\n" "8:" // Oddments: Load weights: Bit 1: Unset - "ld1 { v25.b }[0], [%x[weights]]\n" - "ld1 { v23.b }[0], [x25]\n" + "ld1 { v17.b }[0], [%x[weights]]\n" + "ld1 { v24.b }[0], [x25]\n" "add x21, %x[weights], %x[ld_weight_col]\n" "add x20, %x[weights], x23\n" - "ld1 { v22.b }[0], [x21]\n" - "ld1 { v20.b }[0], [x20]\n" + "ld1 { v20.b }[0], [x21]\n" + "ld1 { v16.b }[0], [x20]\n" "add x21, x25, %x[ld_weight_col]\n" "add x20, x25, x23\n" - "ld1 { v21.b }[0], [x21]\n" + "ld1 { v19.b }[0], [x21]\n" "ld1 { v18.b }[0], [x20]\n" "add x21, x24, %x[ld_weight_col]\n" "add x20, x24, x23\n" - "ld1 { v17.b }[0], [x24]\n" - "ld1 { v19.b }[0], [x21]\n" + "ld1 { v23.b }[0], [x24]\n" + "ld1 { v22.b }[0], [x21]\n" "add %x[weights], %x[weights], #0x1\n" - "ld1 { v16.b }[0], [x20]\n" + "ld1 { v21.b }[0], [x20]\n" "9:" // Oddments: Load weights: Bit 1: End - "zip1 v20.16b, v25.16b, v20.16b\n" - "zip1 v22.16b, v22.16b, v0.16b\n" - "zip1 v22.16b, v20.16b, v22.16b\n" - "zip1 v20.16b, v23.16b, v18.16b\n" - "zip1 v18.16b, v21.16b, v0.16b\n" - "movi v24.4s, #0x0\n" - ".inst 0x4e9697b8 // sdot v24.4s, v29.16b, v22.16b\n" - "zip1 v18.16b, v20.16b, v18.16b\n" "zip1 v17.16b, v17.16b, v16.16b\n" - ".inst 0x4e9297b8 // sdot v24.4s, v29.16b, v18.16b\n" - "zip1 v16.16b, v19.16b, v0.16b\n" + "zip1 v16.16b, v20.16b, v31.16b\n" + "zip1 v20.16b, v17.16b, v16.16b\n" + "zip1 v17.16b, v24.16b, v18.16b\n" + "zip1 v16.16b, v19.16b, v31.16b\n" + "movi v19.4s, #0x0\n" + ".inst 0x4e949793 // sdot v19.4s, v28.16b, v20.16b\n" + "zip1 v18.16b, v17.16b, v16.16b\n" + "zip1 v17.16b, v23.16b, v21.16b\n" + ".inst 0x4e929793 // sdot v19.4s, v28.16b, v18.16b\n" + "zip1 v16.16b, v22.16b, v31.16b\n" "zip1 v16.16b, v17.16b, v16.16b\n" - ".inst 0x4e9097b8 // sdot v24.4s, v29.16b, v16.16b\n" - "mls v26.4s, v24.4s, v31.4s\n" - "add v26.4s, v26.4s, v30.4s\n" - "str q26, [%x[outptr], #0x0]\n" - "str q22, [%x[outptr], #0x10]\n" + ".inst 0x4e909793 // sdot v19.4s, v28.16b, v16.16b\n" + "mls v25.4s, v19.4s, v30.4s\n" + "add v25.4s, v25.4s, v29.4s\n" + "str q25, [%x[outptr], #0x0]\n" + "str q20, [%x[outptr], #0x10]\n" "str q18, [%x[outptr], #0x20]\n" "str q16, [%x[outptr], #0x30]\n" "add %x[outptr], %x[outptr], #0x40\n" @@ -217,24 +217,24 @@ void interleave_a64_s8q_3x3_dot::pack_parameters(unsigned int n_channels, void * "add x21, %x[rq_mul_perchannel], x22\n" "add x20, %x[rq_shift_perchannel], x22\n" "tbz %x[n_channels], #1, 10f\n" - "ld1 { v28.d }[0], [x21], #0x8\n" - "ld1 { v27.d }[0], [x20], #0x8\n" + "ld1 { v27.d }[0], [x21], #0x8\n" + "ld1 { v26.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #0, 11f\n" - "ld1 { v28.s }[2], [x21], #0x4\n" - "ld1 { v27.s }[2], [x20], #0x4\n" + "ld1 { v27.s }[2], [x21], #0x4\n" + "ld1 { v26.s }[2], [x20], #0x4\n" "b 11f\n" "10:" // Oddments: Quantisation parameters: Load quant params: Bit 1: Unset - "ld1 { v28.s }[0], [x21], #0x4\n" - "ld1 { v27.s }[0], [x20], #0x4\n" + "ld1 { v27.s }[0], [x21], #0x4\n" + "ld1 { v26.s }[0], [x20], #0x4\n" "11:" // Oddments: Quantisation parameters: Load quant params: Bit 1: End "12:" // Oddments: Quantisation parameters: Store - "str q28, [%x[outptr], #0x0]\n" - "str q27, [%x[outptr], #0x10]\n" + "str q27, [%x[outptr], #0x0]\n" + "str q26, [%x[outptr], #0x10]\n" "add %x[outptr], %x[outptr], #0x20\n" "13:" // End : [bias] "+&r" (bias), [ld_weight_col] "+&r" (ld_weight_col), [ld_weight_row] "+&r" (ld_weight_row), [outptr] "+&r" (outptr), [weights] "+&r" (weights) : [n_channels] "r" (n_channels), [offsetof_input_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_per_layer_mul] "I" (offsetof(arm_gemm::Requantize32, per_layer_mul)), [offsetof_per_layer_right_shift] "I" (offsetof(arm_gemm::Requantize32, per_layer_right_shift)), [offsetof_weights_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [qp] "r" (&qp), [rq_mul_perchannel] "r" (qp.per_channel_muls), [rq_shift_perchannel] "r" (qp.per_channel_right_shifts) - : "cc", "memory", "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25" + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/a64_u8q_3x3_dot.cpp b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/a64_u8q_3x3_dot.cpp index b89886ae0c2f9825584ea199a4822bf07ae25012..19264c9fce91a82ea9d061ca479c22011e5d7247 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/a64_u8q_3x3_dot.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/a64_u8q_3x3_dot.cpp @@ -25,8 +25,8 @@ #if defined(__aarch64__) #include "arm_gemm.hpp" -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" -#include "src/core/NEON/kernels/assembly/depthwise.hpp" +#include "utils.hpp" +#include "depthwise.hpp" #include namespace arm_conv { @@ -42,7 +42,7 @@ size_t interleave_a64_u8q_3x3_dot::get_packed_size(const DepthwiseArgs &args) { // We store 7 vectors for every of channels. const unsigned int n = arm_gemm::roundup( - arm_gemm::iceildiv((long unsigned int) args.input_channels, + arm_gemm::iceildiv((long unsigned int) args.input_channels * args.channel_multiplier, get_vector_length(arm_gemm::VLType::None)), 4lu ); return n * 7 * get_vector_length(arm_gemm::VLType::None); @@ -54,162 +54,162 @@ void interleave_a64_u8q_3x3_dot::pack_parameters(unsigned int n_channels, void * "cmp %x[ld_weight_col], XZR\n" "csel %x[ld_weight_col], %x[ld_weight_col], %x[n_channels], NE\n" "movi v16.4s, #0x9\n" - "movi v0.16b, #0x0\n" + "movi v31.16b, #0x0\n" "mov x21, #0x3\n" "mul x21, %x[ld_weight_col], x21\n" "add x20, %x[qp], %[offsetof_input_offset]\n" - "ld1r { v31.4s }, [x20]\n" - "add x20, %x[qp], %[offsetof_weights_offset]\n" "ld1r { v30.4s }, [x20]\n" + "add x20, %x[qp], %[offsetof_weights_offset]\n" + "ld1r { v29.4s }, [x20]\n" "cmp %x[ld_weight_row], XZR\n" - "mul v30.4s, v30.4s, v31.4s\n" + "mul v29.4s, v29.4s, v30.4s\n" "csel %x[ld_weight_row], %x[ld_weight_row], x21, NE\n" "lsr x21, %x[n_channels], #0x2\n" - "movi v29.16b, #0x1\n" - "mul v30.4s, v30.4s, v16.4s\n" + "movi v28.16b, #0x1\n" + "mul v29.4s, v29.4s, v16.4s\n" "add x25, %x[weights], %x[ld_weight_row]\n" "add x20, %x[qp], %[offsetof_per_layer_mul]\n" - "ld1r { v28.4s }, [x20]\n" - "add x20, %x[qp], %[offsetof_per_layer_right_shift]\n" "ld1r { v27.4s }, [x20]\n" + "add x20, %x[qp], %[offsetof_per_layer_right_shift]\n" + "ld1r { v26.4s }, [x20]\n" "add x24, x25, %x[ld_weight_row]\n" "add x23, %x[ld_weight_col], %x[ld_weight_col]\n" "mov x22, #0x0\n" "cbz x21, 4f\n" "1:" // Loop - "movi v26.4s, #0x0\n" + "movi v25.4s, #0x0\n" "cbz %x[bias], 2f\n" - "ldr q26, [%x[bias], x22]\n" + "ldr q25, [%x[bias], x22]\n" "2:" // Loop: Skip bias load - "ldr s25, [%x[weights], #0x0]\n" - "ldr s22, [%x[weights], %x[ld_weight_col]]\n" - "zip1 v22.16b, v22.16b, v0.16b\n" - "movi v24.4s, #0x0\n" - "ldr s20, [%x[weights], x23]\n" - "ldr s23, [x25, #0x0]\n" - "zip1 v20.16b, v25.16b, v20.16b\n" - "zip1 v22.16b, v20.16b, v22.16b\n" - "ldr s21, [x25, %x[ld_weight_col]]\n" - "ldr s18, [x25, x23]\n" - "zip1 v20.16b, v23.16b, v18.16b\n" - "zip1 v18.16b, v21.16b, v0.16b\n" + "ldr s19, [%x[weights], #0x0]\n" + "ldr s16, [%x[weights], %x[ld_weight_col]]\n" + "zip1 v17.16b, v16.16b, v31.16b\n" + "movi v21.4s, #0x0\n" + "ldr s16, [%x[weights], x23]\n" + "ldr s18, [x25, #0x0]\n" + "zip1 v16.16b, v19.16b, v16.16b\n" + "zip1 v20.16b, v16.16b, v17.16b\n" + "ldr s17, [x25, %x[ld_weight_col]]\n" + "ldr s16, [x25, x23]\n" + "zip1 v18.16b, v18.16b, v16.16b\n" + "zip1 v16.16b, v17.16b, v31.16b\n" "ldr s17, [x24, #0x0]\n" "ldr s19, [x24, %x[ld_weight_col]]\n" - ".inst 0x6e9697b8 // udot v24.4s, v29.16b, v22.16b\n" - "zip1 v18.16b, v20.16b, v18.16b\n" + ".inst 0x6e949795 // udot v21.4s, v28.16b, v20.16b\n" + "zip1 v18.16b, v18.16b, v16.16b\n" "ldr s16, [x24, x23]\n" "zip1 v17.16b, v17.16b, v16.16b\n" - "zip1 v16.16b, v19.16b, v0.16b\n" - ".inst 0x6e9297b8 // udot v24.4s, v29.16b, v18.16b\n" + "zip1 v16.16b, v19.16b, v31.16b\n" + ".inst 0x6e929795 // udot v21.4s, v28.16b, v18.16b\n" "zip1 v16.16b, v17.16b, v16.16b\n" - ".inst 0x6e9097b8 // udot v24.4s, v29.16b, v16.16b\n" + ".inst 0x6e909795 // udot v21.4s, v28.16b, v16.16b\n" "add %x[weights], %x[weights], #0x4\n" "add x25, x25, #0x4\n" - "mls v26.4s, v24.4s, v31.4s\n" + "mls v25.4s, v21.4s, v30.4s\n" "add x24, x24, #0x4\n" - "add v26.4s, v26.4s, v30.4s\n" - "str q26, [%x[outptr], #0x0]\n" - "str q22, [%x[outptr], #0x10]\n" + "add v25.4s, v25.4s, v29.4s\n" + "str q25, [%x[outptr], #0x0]\n" + "str q20, [%x[outptr], #0x10]\n" "str q18, [%x[outptr], #0x20]\n" "str q16, [%x[outptr], #0x30]\n" "add %x[outptr], %x[outptr], #0x40\n" "cbz %x[rq_mul_perchannel], 3f\n" - "ldr q28, [%x[rq_mul_perchannel], x22]\n" - "ldr q27, [%x[rq_shift_perchannel], x22]\n" + "ldr q27, [%x[rq_mul_perchannel], x22]\n" + "ldr q26, [%x[rq_shift_perchannel], x22]\n" "3:" // Loop: Quantisation parameters: Store "subs x21, x21, #0x1\n" - "str q28, [%x[outptr], #0x0]\n" + "str q27, [%x[outptr], #0x0]\n" "add x22, x22, #0x10\n" - "str q27, [%x[outptr], #0x10]\n" + "str q26, [%x[outptr], #0x10]\n" "add %x[outptr], %x[outptr], #0x20\n" "bgt 1b\n" "tst %x[n_channels], #0x3\n" "beq 13f\n" "4:" // Oddments - "movi v26.4s, #0x0\n" + "movi v25.4s, #0x0\n" "cbz %x[bias], 7f\n" "add %x[bias], %x[bias], x22\n" "tbz %x[n_channels], #1, 5f\n" - "ld1 { v26.d }[0], [%x[bias]], #0x8\n" + "ld1 { v25.d }[0], [%x[bias]], #0x8\n" "tbz %x[n_channels], #0, 6f\n" - "ld1 { v26.s }[2], [%x[bias]], #0x4\n" + "ld1 { v25.s }[2], [%x[bias]], #0x4\n" "b 6f\n" "5:" // Oddments: Load bias: Bit 1: Unset - "ld1 { v26.s }[0], [%x[bias]], #0x4\n" + "ld1 { v25.s }[0], [%x[bias]], #0x4\n" "6:" // Oddments: Load bias: Bit 1: End "7:" // Oddments: Skip bias load "tbz %x[n_channels], #1, 8f\n" - "ld1 { v25.h }[0], [%x[weights]]\n" - "ld1 { v23.h }[0], [x25]\n" + "ld1 { v17.h }[0], [%x[weights]]\n" + "ld1 { v24.h }[0], [x25]\n" "add x21, %x[weights], %x[ld_weight_col]\n" "add x20, %x[weights], x23\n" - "ld1 { v22.h }[0], [x21]\n" - "ld1 { v20.h }[0], [x20]\n" + "ld1 { v20.h }[0], [x21]\n" + "ld1 { v16.h }[0], [x20]\n" "add x21, x25, %x[ld_weight_col]\n" "add x20, x25, x23\n" - "ld1 { v21.h }[0], [x21]\n" + "ld1 { v19.h }[0], [x21]\n" "ld1 { v18.h }[0], [x20]\n" "add x21, x24, %x[ld_weight_col]\n" "add x20, x24, x23\n" - "ld1 { v17.h }[0], [x24]\n" - "ld1 { v19.h }[0], [x21]\n" + "ld1 { v23.h }[0], [x24]\n" + "ld1 { v22.h }[0], [x21]\n" "add %x[weights], %x[weights], #0x2\n" "add x25, x25, #0x2\n" - "ld1 { v16.h }[0], [x20]\n" + "ld1 { v21.h }[0], [x20]\n" "add x24, x24, #0x2\n" "tbz %x[n_channels], #0, 9f\n" - "ld1 { v25.b }[2], [%x[weights]]\n" - "ld1 { v23.b }[2], [x25]\n" + "ld1 { v17.b }[2], [%x[weights]]\n" + "ld1 { v24.b }[2], [x25]\n" "add x21, %x[weights], %x[ld_weight_col]\n" "add x20, %x[weights], x23\n" - "ld1 { v22.b }[2], [x21]\n" - "ld1 { v20.b }[2], [x20]\n" + "ld1 { v20.b }[2], [x21]\n" + "ld1 { v16.b }[2], [x20]\n" "add x21, x25, %x[ld_weight_col]\n" "add x20, x25, x23\n" - "ld1 { v21.b }[2], [x21]\n" + "ld1 { v19.b }[2], [x21]\n" "ld1 { v18.b }[2], [x20]\n" "add x21, x24, %x[ld_weight_col]\n" "add x20, x24, x23\n" - "ld1 { v17.b }[2], [x24]\n" - "ld1 { v19.b }[2], [x21]\n" + "ld1 { v23.b }[2], [x24]\n" + "ld1 { v22.b }[2], [x21]\n" "add %x[weights], %x[weights], #0x1\n" - "ld1 { v16.b }[2], [x20]\n" + "ld1 { v21.b }[2], [x20]\n" "b 9f\n" "8:" // Oddments: Load weights: Bit 1: Unset - "ld1 { v25.b }[0], [%x[weights]]\n" - "ld1 { v23.b }[0], [x25]\n" + "ld1 { v17.b }[0], [%x[weights]]\n" + "ld1 { v24.b }[0], [x25]\n" "add x21, %x[weights], %x[ld_weight_col]\n" "add x20, %x[weights], x23\n" - "ld1 { v22.b }[0], [x21]\n" - "ld1 { v20.b }[0], [x20]\n" + "ld1 { v20.b }[0], [x21]\n" + "ld1 { v16.b }[0], [x20]\n" "add x21, x25, %x[ld_weight_col]\n" "add x20, x25, x23\n" - "ld1 { v21.b }[0], [x21]\n" + "ld1 { v19.b }[0], [x21]\n" "ld1 { v18.b }[0], [x20]\n" "add x21, x24, %x[ld_weight_col]\n" "add x20, x24, x23\n" - "ld1 { v17.b }[0], [x24]\n" - "ld1 { v19.b }[0], [x21]\n" + "ld1 { v23.b }[0], [x24]\n" + "ld1 { v22.b }[0], [x21]\n" "add %x[weights], %x[weights], #0x1\n" - "ld1 { v16.b }[0], [x20]\n" + "ld1 { v21.b }[0], [x20]\n" "9:" // Oddments: Load weights: Bit 1: End - "zip1 v20.16b, v25.16b, v20.16b\n" - "zip1 v22.16b, v22.16b, v0.16b\n" - "zip1 v22.16b, v20.16b, v22.16b\n" - "zip1 v20.16b, v23.16b, v18.16b\n" - "zip1 v18.16b, v21.16b, v0.16b\n" - "movi v24.4s, #0x0\n" - ".inst 0x6e9697b8 // udot v24.4s, v29.16b, v22.16b\n" - "zip1 v18.16b, v20.16b, v18.16b\n" "zip1 v17.16b, v17.16b, v16.16b\n" - ".inst 0x6e9297b8 // udot v24.4s, v29.16b, v18.16b\n" - "zip1 v16.16b, v19.16b, v0.16b\n" + "zip1 v16.16b, v20.16b, v31.16b\n" + "zip1 v20.16b, v17.16b, v16.16b\n" + "zip1 v17.16b, v24.16b, v18.16b\n" + "zip1 v16.16b, v19.16b, v31.16b\n" + "movi v19.4s, #0x0\n" + ".inst 0x6e949793 // udot v19.4s, v28.16b, v20.16b\n" + "zip1 v18.16b, v17.16b, v16.16b\n" + "zip1 v17.16b, v23.16b, v21.16b\n" + ".inst 0x6e929793 // udot v19.4s, v28.16b, v18.16b\n" + "zip1 v16.16b, v22.16b, v31.16b\n" "zip1 v16.16b, v17.16b, v16.16b\n" - ".inst 0x6e9097b8 // udot v24.4s, v29.16b, v16.16b\n" - "mls v26.4s, v24.4s, v31.4s\n" - "add v26.4s, v26.4s, v30.4s\n" - "str q26, [%x[outptr], #0x0]\n" - "str q22, [%x[outptr], #0x10]\n" + ".inst 0x6e909793 // udot v19.4s, v28.16b, v16.16b\n" + "mls v25.4s, v19.4s, v30.4s\n" + "add v25.4s, v25.4s, v29.4s\n" + "str q25, [%x[outptr], #0x0]\n" + "str q20, [%x[outptr], #0x10]\n" "str q18, [%x[outptr], #0x20]\n" "str q16, [%x[outptr], #0x30]\n" "add %x[outptr], %x[outptr], #0x40\n" @@ -217,24 +217,24 @@ void interleave_a64_u8q_3x3_dot::pack_parameters(unsigned int n_channels, void * "add x21, %x[rq_mul_perchannel], x22\n" "add x20, %x[rq_shift_perchannel], x22\n" "tbz %x[n_channels], #1, 10f\n" - "ld1 { v28.d }[0], [x21], #0x8\n" - "ld1 { v27.d }[0], [x20], #0x8\n" + "ld1 { v27.d }[0], [x21], #0x8\n" + "ld1 { v26.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #0, 11f\n" - "ld1 { v28.s }[2], [x21], #0x4\n" - "ld1 { v27.s }[2], [x20], #0x4\n" + "ld1 { v27.s }[2], [x21], #0x4\n" + "ld1 { v26.s }[2], [x20], #0x4\n" "b 11f\n" "10:" // Oddments: Quantisation parameters: Load quant params: Bit 1: Unset - "ld1 { v28.s }[0], [x21], #0x4\n" - "ld1 { v27.s }[0], [x20], #0x4\n" + "ld1 { v27.s }[0], [x21], #0x4\n" + "ld1 { v26.s }[0], [x20], #0x4\n" "11:" // Oddments: Quantisation parameters: Load quant params: Bit 1: End "12:" // Oddments: Quantisation parameters: Store - "str q28, [%x[outptr], #0x0]\n" - "str q27, [%x[outptr], #0x10]\n" + "str q27, [%x[outptr], #0x0]\n" + "str q26, [%x[outptr], #0x10]\n" "add %x[outptr], %x[outptr], #0x20\n" "13:" // End : [bias] "+&r" (bias), [ld_weight_col] "+&r" (ld_weight_col), [ld_weight_row] "+&r" (ld_weight_row), [outptr] "+&r" (outptr), [weights] "+&r" (weights) : [n_channels] "r" (n_channels), [offsetof_input_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_per_layer_mul] "I" (offsetof(arm_gemm::Requantize32, per_layer_mul)), [offsetof_per_layer_right_shift] "I" (offsetof(arm_gemm::Requantize32, per_layer_right_shift)), [offsetof_weights_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [qp] "r" (&qp), [rq_mul_perchannel] "r" (qp.per_channel_muls), [rq_shift_perchannel] "r" (qp.per_channel_right_shifts) - : "cc", "memory", "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25" + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/generic.cpp index 056f08d037006566f77694e05f2b7aadb09cf5fe..dc505a013d46e78466802817f18c11ea25f753cb 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,11 +32,11 @@ namespace interleaves { PackingArguments::PackingArguments( unsigned int kernel_rows, unsigned int kernel_cols, size_t weight_element_size, - bool include_bias, size_t bias_element_size, + bool include_bias, size_t bias_element_size, bool premultiply, arm_gemm::VLType vl_type, size_t accumulator_element_size, unsigned int accumulator_depth_vl, std::function get_weight_pos ) : kernel_rows(kernel_rows), kernel_cols(kernel_cols), weight_element_size(weight_element_size), - include_bias(include_bias), bias_element_size(bias_element_size), + include_bias(include_bias), bias_element_size(bias_element_size), premultiply(premultiply), vl_type(vl_type), accumulator_element_size(accumulator_element_size), accumulator_depth_vl(accumulator_depth_vl), get_weight_pos(get_weight_pos) { @@ -46,7 +46,7 @@ size_t get_storage_size_generic(const PackingArguments &packing_args, const Dept { // If the channel multiplier is greater than one, then we treat this as a // repeated packing of `channel_multiplier`-sized problems. - if (args.channel_multiplier > 1) + if (args.channel_multiplier > 1 && !packing_args.premultiply) { DepthwiseArgs args_per_input_channel(args); args_per_input_channel.input_channels = args.channel_multiplier; @@ -58,7 +58,7 @@ size_t get_storage_size_generic(const PackingArguments &packing_args, const Dept const unsigned int vl = packing_args.accumulator_depth_vl * arm_gemm::utils::get_vector_length(packing_args.vl_type) / packing_args.accumulator_element_size; - const unsigned int n_packs = arm_gemm::iceildiv(args.input_channels, vl); + const unsigned int n_packs = arm_gemm::iceildiv(args.input_channels * args.channel_multiplier, vl); const auto pack_size = (packing_args.include_bias ? packing_args.bias_element_size : 0) + packing_args.kernel_points() * packing_args.weight_element_size; return n_packs * pack_size * vl; @@ -81,7 +81,7 @@ void pack_parameters_generic( // If the channel multiplier is greater than one, then we treat this as a // repeated packing of `channel_multiplier`-sized problems. - if (args.channel_multiplier > 1) + if (args.channel_multiplier > 1 && !packing_args.premultiply) { // Get a modified copy of the depthwise arguments DepthwiseArgs args_per_input_channel(args); @@ -107,17 +107,19 @@ void pack_parameters_generic( return; } + auto input_channels = args.input_channels * args.channel_multiplier; + // Finalise the weight strides - ld_weight_col = (ld_weight_col == 0) ? args.input_channels : ld_weight_col; + ld_weight_col = (ld_weight_col == 0) ? input_channels : ld_weight_col; ld_weight_row = (ld_weight_row == 0) ? packing_args.kernel_cols * ld_weight_col : ld_weight_row; const unsigned int vl = packing_args.accumulator_depth_vl * arm_gemm::utils::get_vector_length(packing_args.vl_type) / packing_args.accumulator_element_size; - for (unsigned int n = 0; n < args.input_channels; n += vl) + for (unsigned int n = 0; n < input_channels; n += vl) { - const unsigned int todo = std::min(vl, args.input_channels - n); + const unsigned int todo = std::min(vl, input_channels - n); if (packing_args.include_bias) { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/generic.hpp b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/generic.hpp index 5b5ae178068781ccd55c5a293c63daba5ba685af..1842f10150777fef91486eda6ea783e12fc3090b 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/generic.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/generic.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,7 +24,7 @@ #pragma once -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "depthwise.hpp" #include @@ -40,6 +40,7 @@ struct PackingArguments const size_t weight_element_size; const bool include_bias; const size_t bias_element_size; + const bool premultiply; arm_gemm::VLType vl_type; const size_t accumulator_element_size; const unsigned int accumulator_depth_vl; @@ -53,6 +54,7 @@ struct PackingArguments size_t weight_element_size, bool include_bias, size_t bias_element_size, + bool premultiply, arm_gemm::VLType vl_type, size_t accumulator_element_size, unsigned int accumulator_depth_vl, diff --git a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_s8q_3x3_dot.cpp b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_s8q_3x3_dot.cpp index 0cf8044733fe42362fc98235d73ddaf22166cc91..5d7b54f235a6ede951e9a454798fb88686c0126d 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_s8q_3x3_dot.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_s8q_3x3_dot.cpp @@ -25,8 +25,8 @@ #if defined(ARM_COMPUTE_ENABLE_SVE) #include "arm_gemm.hpp" -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" -#include "src/core/NEON/kernels/assembly/depthwise.hpp" +#include "utils.hpp" +#include "depthwise.hpp" #include namespace arm_conv { @@ -42,7 +42,7 @@ size_t interleave_sve_s8q_3x3_dot::get_packed_size(const DepthwiseArgs &args) { // We store 7 vectors for every of channels. const unsigned int n = arm_gemm::roundup( - arm_gemm::iceildiv((long unsigned int) args.input_channels, + arm_gemm::iceildiv((long unsigned int) args.input_channels * args.channel_multiplier, get_vector_length(arm_gemm::VLType::SVE)), 4lu ); return n * 7 * get_vector_length(arm_gemm::VLType::SVE); @@ -76,7 +76,6 @@ void interleave_sve_s8q_3x3_dot::pack_parameters(unsigned int n_channels, void * "cbz %x[bias], 1f\n" "ptrue p8.s\n" "1:" // No bias - "2:" // Loop "cntp x20, p2, p1.s\n" "whilelt p0.b, XZR, x20\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_u8q_3x3_dot.cpp b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_u8q_3x3_dot.cpp index e5bc8198f8cba7c97eaadf61c38d2509bfee377d..c3da81448bb051f447c5782c510308690983f0e8 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_u8q_3x3_dot.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_u8q_3x3_dot.cpp @@ -25,8 +25,8 @@ #if defined(ARM_COMPUTE_ENABLE_SVE) #include "arm_gemm.hpp" -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" -#include "src/core/NEON/kernels/assembly/depthwise.hpp" +#include "utils.hpp" +#include "depthwise.hpp" #include namespace arm_conv { @@ -42,7 +42,7 @@ size_t interleave_sve_u8q_3x3_dot::get_packed_size(const DepthwiseArgs &args) { // We store 7 vectors for every of channels. const unsigned int n = arm_gemm::roundup( - arm_gemm::iceildiv((long unsigned int) args.input_channels, + arm_gemm::iceildiv((long unsigned int) args.input_channels * args.channel_multiplier, get_vector_length(arm_gemm::VLType::SVE)), 4lu ); return n * 7 * get_vector_length(arm_gemm::VLType::SVE); @@ -76,7 +76,6 @@ void interleave_sve_u8q_3x3_dot::pack_parameters(unsigned int n_channels, void * "cbz %x[bias], 1f\n" "ptrue p8.s\n" "1:" // No bias - "2:" // Loop "cntp x20, p2, p1.s\n" "whilelt p0.b, XZR, x20\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp index d2db12535fe54f129a2a0b3684ab4e09466c71c2..6beaba841f88dfbe1066d9e3e99c49be42c1f3fb 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include @@ -33,8 +33,8 @@ namespace arm_conv { namespace depthwise { -void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); -void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); +void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const input_ptrs, __fp16 *const *const outptrs, const void *params, unsigned int n_channels, const __fp16 activation_min, const __fp16 activation_max); +void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int n_tile_rows, const unsigned int n_tile_cols, const __fp16 *inptr, int64_t ld_input_row, int64_t ld_input_col, __fp16 *outptr, int64_t ld_output_row, int64_t ld_output_col, const void *params, unsigned int n_channels, const __fp16 activation_min, const __fp16 activation_max); class a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirstStrategy<__fp16, __fp16, __fp16, __fp16> { @@ -57,7 +57,7 @@ class a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirst constexpr static unsigned int output_cols = 2; a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst(const CPUInfo *) - : DepthwiseDepthfirstStrategy<__fp16, __fp16, __fp16, __fp16>(2, 3, 1) {} + : Parent(output_rows, output_cols, kernel_rows, kernel_cols, stride_rows, stride_cols) {} arm_gemm::VLType get_vl_type(void) const override { return vl_type; } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp index f4027df375b26126f664faf88925b721e7c2a2f9..d8ca3d7437b63da1d60bd026649c707c513708dd 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -116,9 +116,9 @@ void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "add x28, x9, x25, LSL #1\n" "add x12, x12, x20, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16) "add x20, %x[params_struct], %[offsetof_args_min]\n" - "ld1r { v18.8h }, [x20]\n" + "ld1r { v27.8h }, [x20]\n" "add x20, %x[params_struct], %[offsetof_args_max]\n" - "ld1r { v17.8h }, [x20]\n" + "ld1r { v26.8h }, [x20]\n" "add x27, x28, x25, LSL #1\n" "add x26, x11, x15\n" "add x25, x12, x24, LSL #1\n" @@ -126,7 +126,7 @@ void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "mov x21, #0x0\n" "sub x20, XZR, x23\n" "cbz x22, 4f\n" - "ldr q16, [x10, #0x0]\n" + "ldr q25, [x10, #0x0]\n" "ldr q0, [x10, #0x10]\n" "cmp x23, x22, LSL #4\n" "ldr q1, [x10, #0x20]\n" @@ -145,162 +145,162 @@ void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "ldr q13, [x28, x15]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "mov v28.16b, v16.16b\n fmla v28.8h, v4.8h, v9.8h\n" - "mov v29.16b, v16.16b\n fmla v29.8h, v3.8h, v9.8h\n" + "mov v24.16b, v25.16b\n fmla v24.8h, v4.8h, v9.8h\n" + "mov v23.16b, v25.16b\n fmla v23.8h, v3.8h, v9.8h\n" "add x23, x23, #0x10\n" "cmp x23, x22, LSL #4\n" - "mov v30.16b, v16.16b\n fmla v30.8h, v1.8h, v9.8h\n" - "mov v31.16b, v16.16b\n fmla v31.8h, v0.8h, v9.8h\n" - "ld1 { v9.8h }, [x27]\n" - "ldr q16, [x10, #0x0]\n" - "fmla v28.8h, v0.8h, v10.8h\n" - "ldr q10, [x28, x11]\n" - "fmla v29.8h, v2.8h, v11.8h\n" - "ldr q11, [x27, x26]\n" - "fmla v30.8h, v2.8h, v12.8h\n" - "fmla v31.8h, v1.8h, v12.8h\n" + "mov v22.16b, v25.16b\n fmla v22.8h, v1.8h, v9.8h\n" + "mov v21.16b, v25.16b\n fmla v21.8h, v0.8h, v9.8h\n" + "ld1 { v18.8h }, [x27]\n" + "ldr q25, [x10, #0x0]\n" + "fmla v24.8h, v0.8h, v10.8h\n" + "ldr q20, [x28, x11]\n" + "fmla v23.8h, v2.8h, v11.8h\n" + "ldr q17, [x27, x26]\n" + "fmla v22.8h, v2.8h, v12.8h\n" + "fmla v21.8h, v1.8h, v12.8h\n" "add x20, x20, #0x10\n" "add x21, x21, #0x10\n" - "fmla v28.8h, v5.8h, v12.8h\n" - "fmla v29.8h, v4.8h, v12.8h\n" - "ldr q12, [x13, x15]\n" - "fmla v30.8h, v6.8h, v9.8h\n" - "ldr q9, [x13, x11]\n" - "fmla v31.8h, v3.8h, v13.8h\n" + "fmla v24.8h, v5.8h, v12.8h\n" + "fmla v23.8h, v4.8h, v12.8h\n" + "ldr q16, [x13, x15]\n" + "fmla v22.8h, v6.8h, v18.8h\n" + "ldr q18, [x13, x11]\n" + "fmla v21.8h, v3.8h, v13.8h\n" "add x13, x13, #0x10\n" - "fmla v28.8h, v7.8h, v13.8h\n" - "fmla v29.8h, v6.8h, v13.8h\n" - "fmla v30.8h, v4.8h, v13.8h\n" - "fmla v31.8h, v8.8h, v11.8h\n" - "ld1 { v11.8h }, [x9]\n" - "fmla v28.8h, v1.8h, v12.8h\n" - "fmla v29.8h, v0.8h, v12.8h\n" - "ldr q12, [x9, x26]\n" + "fmla v24.8h, v7.8h, v13.8h\n" + "fmla v23.8h, v6.8h, v13.8h\n" + "fmla v22.8h, v4.8h, v13.8h\n" + "fmla v21.8h, v8.8h, v17.8h\n" + "ld1 { v17.8h }, [x9]\n" + "fmla v24.8h, v1.8h, v16.8h\n" + "fmla v23.8h, v0.8h, v16.8h\n" + "ldr q16, [x9, x26]\n" "add x9, x9, #0x10\n" - "fmla v30.8h, v5.8h, v10.8h\n" - "fmla v31.8h, v4.8h, v10.8h\n" + "fmla v22.8h, v5.8h, v20.8h\n" + "fmla v21.8h, v4.8h, v20.8h\n" "ldr q4, [x10, #0x50]\n" - "fmla v28.8h, v2.8h, v9.8h\n" - "fmla v29.8h, v1.8h, v9.8h\n" - "ld1 { v9.8h }, [x28]\n" + "fmla v24.8h, v2.8h, v18.8h\n" + "fmla v23.8h, v1.8h, v18.8h\n" + "ld1 { v19.8h }, [x28]\n" "ldr q1, [x10, #0x20]\n" - "fmla v30.8h, v0.8h, v11.8h\n" + "fmla v22.8h, v0.8h, v17.8h\n" "ldr q0, [x10, #0x10]\n" - "fmla v31.8h, v2.8h, v12.8h\n" + "fmla v21.8h, v2.8h, v16.8h\n" "ldr q2, [x10, #0x30]\n" - "fmla v28.8h, v8.8h, v10.8h\n" - "fmla v29.8h, v7.8h, v10.8h\n" - "ldr q10, [x28, x26]\n" + "fmla v24.8h, v8.8h, v20.8h\n" + "fmla v23.8h, v7.8h, v20.8h\n" + "ldr q18, [x28, x26]\n" "add x28, x28, #0x10\n" "ldr q13, [x28, x15]\n" - "fmla v30.8h, v3.8h, v9.8h\n" - "fmla v31.8h, v5.8h, v10.8h\n" - "fmla v28.8h, v3.8h, v11.8h\n" - "ldr q11, [x27, x15]\n" + "fmla v22.8h, v3.8h, v19.8h\n" + "fmla v21.8h, v5.8h, v18.8h\n" + "fmla v24.8h, v3.8h, v17.8h\n" + "ldr q17, [x27, x15]\n" "ldr q3, [x10, #0x40]\n" - "fmla v29.8h, v5.8h, v12.8h\n" - "ldr q12, [x27, x11]\n" + "fmla v23.8h, v5.8h, v16.8h\n" + "ldr q16, [x27, x11]\n" "ldr q5, [x10, #0x60]\n" - "fmla v30.8h, v7.8h, v11.8h\n" - "fmla v31.8h, v6.8h, v11.8h\n" + "fmla v22.8h, v7.8h, v17.8h\n" + "fmla v21.8h, v6.8h, v17.8h\n" "ldr q11, [x13, x26]\n" - "fmla v28.8h, v6.8h, v9.8h\n" + "fmla v24.8h, v6.8h, v19.8h\n" "ldr q9, [x9, x15]\n" - "fmla v29.8h, v8.8h, v10.8h\n" + "fmla v23.8h, v8.8h, v18.8h\n" "ld1 { v10.8h }, [x13]\n" "ldr q6, [x10, #0x70]\n" - "fmla v30.8h, v8.8h, v12.8h\n" - "fmla v31.8h, v7.8h, v12.8h\n" + "fmla v22.8h, v8.8h, v16.8h\n" + "fmla v21.8h, v7.8h, v16.8h\n" "ldr q12, [x9, x11]\n" "ldr q7, [x10, #0x80]\n" - "fmax v28.8h, v28.8h, v18.8h\n" - "fmax v29.8h, v29.8h, v18.8h\n" + "fmax v24.8h, v24.8h, v27.8h\n" + "fmax v23.8h, v23.8h, v27.8h\n" "ldr q8, [x10, #0x90]\n" - "fmax v30.8h, v30.8h, v18.8h\n" - "fmax v31.8h, v31.8h, v18.8h\n" + "fmax v22.8h, v22.8h, v27.8h\n" + "fmax v21.8h, v21.8h, v27.8h\n" "add x27, x27, #0x10\n" - "fmin v28.8h, v28.8h, v17.8h\n" - "fmin v29.8h, v29.8h, v17.8h\n" - "st1 { v28.8h }, [x12]\n" + "fmin v24.8h, v24.8h, v26.8h\n" + "fmin v23.8h, v23.8h, v26.8h\n" + "st1 { v24.8h }, [x12]\n" "add x10, x10, #0xa0\n" - "fmin v30.8h, v30.8h, v17.8h\n" - "fmin v31.8h, v31.8h, v17.8h\n" - "str q29, [x12, x14]\n" + "fmin v22.8h, v22.8h, v26.8h\n" + "fmin v21.8h, v21.8h, v26.8h\n" + "str q23, [x12, x14]\n" "add x12, x12, #0x10\n" - "st1 { v30.8h }, [x25]\n" - "str q31, [x25, x14]\n" + "st1 { v22.8h }, [x25]\n" + "str q21, [x25, x14]\n" "add x25, x25, #0x10\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "mov v28.16b, v16.16b\n fmla v28.8h, v4.8h, v9.8h\n" - "mov v29.16b, v16.16b\n fmla v29.8h, v3.8h, v9.8h\n" - "mov v30.16b, v16.16b\n fmla v30.8h, v1.8h, v9.8h\n" - "mov v31.16b, v16.16b\n fmla v31.8h, v0.8h, v9.8h\n" - "ld1 { v9.8h }, [x27]\n" - "fmla v28.8h, v0.8h, v10.8h\n" - "ldr q10, [x28, x11]\n" - "fmla v29.8h, v2.8h, v11.8h\n" - "ldr q11, [x27, x26]\n" - "fmla v30.8h, v2.8h, v12.8h\n" - "fmla v31.8h, v1.8h, v12.8h\n" - "fmla v28.8h, v5.8h, v12.8h\n" - "fmla v29.8h, v4.8h, v12.8h\n" - "ldr q12, [x13, x15]\n" - "fmla v30.8h, v6.8h, v9.8h\n" - "ldr q9, [x13, x11]\n" - "fmla v31.8h, v3.8h, v13.8h\n" + "mov v24.16b, v25.16b\n fmla v24.8h, v4.8h, v9.8h\n" + "mov v23.16b, v25.16b\n fmla v23.8h, v3.8h, v9.8h\n" + "mov v22.16b, v25.16b\n fmla v22.8h, v1.8h, v9.8h\n" + "mov v21.16b, v25.16b\n fmla v21.8h, v0.8h, v9.8h\n" + "ld1 { v18.8h }, [x27]\n" + "fmla v24.8h, v0.8h, v10.8h\n" + "ldr q20, [x28, x11]\n" + "fmla v23.8h, v2.8h, v11.8h\n" + "ldr q17, [x27, x26]\n" + "fmla v22.8h, v2.8h, v12.8h\n" + "fmla v21.8h, v1.8h, v12.8h\n" + "fmla v24.8h, v5.8h, v12.8h\n" + "fmla v23.8h, v4.8h, v12.8h\n" + "ldr q16, [x13, x15]\n" + "fmla v22.8h, v6.8h, v18.8h\n" + "ldr q18, [x13, x11]\n" + "fmla v21.8h, v3.8h, v13.8h\n" "add x13, x13, #0x10\n" - "fmla v28.8h, v7.8h, v13.8h\n" - "fmla v29.8h, v6.8h, v13.8h\n" - "fmla v30.8h, v4.8h, v13.8h\n" - "fmla v31.8h, v8.8h, v11.8h\n" - "ld1 { v11.8h }, [x9]\n" - "fmla v28.8h, v1.8h, v12.8h\n" - "fmla v29.8h, v0.8h, v12.8h\n" - "ldr q12, [x9, x26]\n" + "fmla v24.8h, v7.8h, v13.8h\n" + "fmla v23.8h, v6.8h, v13.8h\n" + "fmla v22.8h, v4.8h, v13.8h\n" + "fmla v21.8h, v8.8h, v17.8h\n" + "ld1 { v17.8h }, [x9]\n" + "fmla v24.8h, v1.8h, v16.8h\n" + "fmla v23.8h, v0.8h, v16.8h\n" + "ldr q16, [x9, x26]\n" "add x9, x9, #0x10\n" - "fmla v30.8h, v5.8h, v10.8h\n" - "fmla v31.8h, v4.8h, v10.8h\n" - "fmla v28.8h, v2.8h, v9.8h\n" - "fmla v29.8h, v1.8h, v9.8h\n" - "ld1 { v9.8h }, [x28]\n" - "fmla v30.8h, v0.8h, v11.8h\n" - "fmla v31.8h, v2.8h, v12.8h\n" - "fmla v28.8h, v8.8h, v10.8h\n" - "fmla v29.8h, v7.8h, v10.8h\n" - "ldr q10, [x28, x26]\n" + "fmla v22.8h, v5.8h, v20.8h\n" + "fmla v21.8h, v4.8h, v20.8h\n" + "fmla v24.8h, v2.8h, v18.8h\n" + "fmla v23.8h, v1.8h, v18.8h\n" + "ld1 { v19.8h }, [x28]\n" + "fmla v22.8h, v0.8h, v17.8h\n" + "fmla v21.8h, v2.8h, v16.8h\n" + "fmla v24.8h, v8.8h, v20.8h\n" + "fmla v23.8h, v7.8h, v20.8h\n" + "ldr q18, [x28, x26]\n" "add x28, x28, #0x10\n" - "fmla v30.8h, v3.8h, v9.8h\n" - "fmla v31.8h, v5.8h, v10.8h\n" - "fmla v28.8h, v3.8h, v11.8h\n" - "ldr q11, [x27, x15]\n" - "fmla v29.8h, v5.8h, v12.8h\n" - "ldr q12, [x27, x11]\n" - "fmla v30.8h, v7.8h, v11.8h\n" - "fmla v31.8h, v6.8h, v11.8h\n" + "fmla v22.8h, v3.8h, v19.8h\n" + "fmla v21.8h, v5.8h, v18.8h\n" + "fmla v24.8h, v3.8h, v17.8h\n" + "ldr q17, [x27, x15]\n" + "fmla v23.8h, v5.8h, v16.8h\n" + "ldr q16, [x27, x11]\n" + "fmla v22.8h, v7.8h, v17.8h\n" + "fmla v21.8h, v6.8h, v17.8h\n" "add x27, x27, #0x10\n" - "fmla v28.8h, v6.8h, v9.8h\n" - "fmla v29.8h, v8.8h, v10.8h\n" - "fmax v28.8h, v28.8h, v18.8h\n" - "fmla v30.8h, v8.8h, v12.8h\n" - "fmla v31.8h, v7.8h, v12.8h\n" - "fmax v29.8h, v29.8h, v18.8h\n" - "fmax v30.8h, v30.8h, v18.8h\n" - "fmax v31.8h, v31.8h, v18.8h\n" - "fmin v28.8h, v28.8h, v17.8h\n" - "fmin v29.8h, v29.8h, v17.8h\n" - "st1 { v28.8h }, [x12]\n" - "fmin v30.8h, v30.8h, v17.8h\n" - "fmin v31.8h, v31.8h, v17.8h\n" - "str q29, [x12, x14]\n" + "fmla v24.8h, v6.8h, v19.8h\n" + "fmla v23.8h, v8.8h, v18.8h\n" + "fmax v24.8h, v24.8h, v27.8h\n" + "fmla v22.8h, v8.8h, v16.8h\n" + "fmla v21.8h, v7.8h, v16.8h\n" + "fmax v23.8h, v23.8h, v27.8h\n" + "fmax v22.8h, v22.8h, v27.8h\n" + "fmax v21.8h, v21.8h, v27.8h\n" + "fmin v24.8h, v24.8h, v26.8h\n" + "fmin v23.8h, v23.8h, v26.8h\n" + "st1 { v24.8h }, [x12]\n" + "fmin v22.8h, v22.8h, v26.8h\n" + "fmin v21.8h, v21.8h, v26.8h\n" + "str q23, [x12, x14]\n" "add x12, x12, #0x10\n" - "st1 { v30.8h }, [x25]\n" - "str q31, [x25, x14]\n" + "st1 { v22.8h }, [x25]\n" + "str q21, [x25, x14]\n" "add x25, x25, #0x10\n" "4:" // Tile loop: Oddments "tst %x[n_channels], #0x7\n" "beq 57f\n" - "ldr q16, [x10, #0x0]\n" + "ldr q25, [x10, #0x0]\n" "ldr q0, [x10, #0x10]\n" "add x24, x9, x15\n" "add x23, x13, XZR\n" @@ -363,11 +363,11 @@ void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "ldr h12, [x21, #0x0]\n" "ldr h13, [x20, #0x0]\n" "8:" // Tile loop: Oddments: Load inputs: (1, 1), (0, 0), (0, 3), (1, 2), (2, 1): Bit 2: End - "mov v28.16b, v16.16b\n fmla v28.8h, v4.8h, v9.8h\n" - "mov v29.16b, v16.16b\n fmla v29.8h, v3.8h, v9.8h\n" + "mov v28.16b, v25.16b\n fmla v28.8h, v4.8h, v9.8h\n" + "mov v29.16b, v25.16b\n fmla v29.8h, v3.8h, v9.8h\n" "add x20, x27, XZR\n" - "mov v30.16b, v16.16b\n fmla v30.8h, v1.8h, v9.8h\n" - "mov v31.16b, v16.16b\n fmla v31.8h, v0.8h, v9.8h\n" + "mov v30.16b, v25.16b\n fmla v30.8h, v1.8h, v9.8h\n" + "mov v31.16b, v25.16b\n fmla v31.8h, v0.8h, v9.8h\n" "fmla v28.8h, v0.8h, v10.8h\n" "fmla v29.8h, v2.8h, v11.8h\n" "fmla v28.8h, v5.8h, v12.8h\n" @@ -630,14 +630,14 @@ void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "52:" // Tile loop: Oddments: Load inputs: (3, 2): Bit 2: End "fmla v30.8h, v8.8h, v12.8h\n" "fmla v31.8h, v7.8h, v12.8h\n" - "fmax v28.8h, v28.8h, v18.8h\n" - "fmax v29.8h, v29.8h, v18.8h\n" - "fmax v30.8h, v30.8h, v18.8h\n" - "fmax v31.8h, v31.8h, v18.8h\n" - "fmin v28.8h, v28.8h, v17.8h\n" - "fmin v29.8h, v29.8h, v17.8h\n" - "fmin v30.8h, v30.8h, v17.8h\n" - "fmin v31.8h, v31.8h, v17.8h\n" + "fmax v28.8h, v28.8h, v27.8h\n" + "fmax v29.8h, v29.8h, v27.8h\n" + "fmax v30.8h, v30.8h, v27.8h\n" + "fmax v31.8h, v31.8h, v27.8h\n" + "fmin v28.8h, v28.8h, v26.8h\n" + "fmin v29.8h, v29.8h, v26.8h\n" + "fmin v30.8h, v30.8h, v26.8h\n" + "fmin v31.8h, v31.8h, v26.8h\n" "tbz %x[n_channels], #2, 54f\n" "mov x21, x12\n" "mov x20, x25\n" @@ -699,7 +699,6 @@ void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "st1 { v29.h }[0], [x21]\n" "st1 { v31.h }[0], [x20]\n" "56:" // Tile loop: Oddments: Store: Bit 2: End - "57:" // Tile loop: End "ldr x22, [%x[params_struct], %[offsetof_args_tile_j]]\n" "ldr x23, [%x[params_struct], %[offsetof_args_tile_i]]\n" @@ -714,7 +713,7 @@ void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v16", "v17", "v18", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp index bea471531391d5694f57f545b22f4dccdcd26e91..c9a554e9ad6fdb2ea50302b36c2f8d83c5da7988 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -83,16 +83,16 @@ void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( "lsr x15, %x[n_channels], #0x3\n" "ldr x14, [%x[params_struct], %[offsetof_args_params]]\n" "add x20, %x[params_struct], %[offsetof_args_min]\n" - "ld1r { v18.8h }, [x20]\n" + "ld1r { v27.8h }, [x20]\n" "add x20, %x[params_struct], %[offsetof_args_max]\n" - "ld1r { v17.8h }, [x20]\n" + "ld1r { v26.8h }, [x20]\n" "add x13, %x[params_struct], %[offsetof_Args_inptrs]\n" "ldp x12, x11, [x21, #0x0]\n" "ldp x10, x9, [x21, #0x10]\n" "mov x28, #0x0\n" "sub x27, XZR, x16\n" "cbz x15, 3f\n" - "ldr q16, [x14, #0x0]\n" + "ldr q25, [x14, #0x0]\n" "ldr q0, [x14, #0x10]\n" "cmp x16, x15, LSL #4\n" "ldr q1, [x14, #0x20]\n" @@ -104,197 +104,197 @@ void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( "ldr q7, [x14, #0x80]\n" "ldr q8, [x14, #0x90]\n" "add x14, x14, #0xa0\n" - "ldp x26, x22, [x13, #0x0]\n" - "ldr q9, [x26, x28]\n" - "ldr q10, [x22, x28]\n" - "ldp x25, x24, [x13, #0x10]\n" - "ldr q11, [x25, x28]\n" - "ldr q12, [x24, x28]\n" - "ldr x23, [x13, #0x20]\n" - "ldr q13, [x23, x28]\n" - "bge 2f\n" - "1:" // Channel loop - "mov v28.16b, v16.16b\n fmla v28.8h, v4.8h, v9.8h\n" - "mov v29.16b, v16.16b\n fmla v29.8h, v3.8h, v9.8h\n" - "ldr x22, [x13, #0x28]\n" - "ldr x21, [x13, #0x30]\n" - "mov v30.16b, v16.16b\n fmla v30.8h, v1.8h, v9.8h\n" - "mov v31.16b, v16.16b\n fmla v31.8h, v0.8h, v9.8h\n" - "ldr q9, [x22, x28]\n" - "ldr q16, [x14, #0x0]\n" - "fmla v28.8h, v0.8h, v10.8h\n" - "fmla v29.8h, v2.8h, v11.8h\n" + "ldp x21, x20, [x13, #0x0]\n" + "ldr q9, [x21, x28]\n" + "ldr q10, [x20, x28]\n" + "ldp x21, x20, [x13, #0x10]\n" "ldr q11, [x21, x28]\n" - "ldr x20, [x13, #0x38]\n" - "fmla v30.8h, v2.8h, v12.8h\n" - "fmla v31.8h, v1.8h, v12.8h\n" - "ldr x22, [x13, #0x48]\n" - "ldr q10, [x22, x28]\n" - "fmla v28.8h, v5.8h, v12.8h\n" - "fmla v29.8h, v4.8h, v12.8h\n" "ldr q12, [x20, x28]\n" - "ldr x26, [x13, #0x40]\n" - "fmla v30.8h, v6.8h, v9.8h\n" - "ldr q9, [x26, x28]\n" - "fmla v31.8h, v3.8h, v13.8h\n" - "ldr x25, [x13, #0x50]\n" - "fmla v28.8h, v7.8h, v13.8h\n" - "fmla v29.8h, v6.8h, v13.8h\n" - "ldr x24, [x13, #0x58]\n" - "ldr x23, [x13, #0x60]\n" - "fmla v30.8h, v4.8h, v13.8h\n" - "fmla v31.8h, v8.8h, v11.8h\n" - "ldr q11, [x25, x28]\n" - "ldr x22, [x13, #0x68]\n" - "fmla v28.8h, v1.8h, v12.8h\n" - "fmla v29.8h, v0.8h, v12.8h\n" - "ldr q12, [x24, x28]\n" - "ldr x21, [x13, #0x70]\n" - "fmla v30.8h, v5.8h, v10.8h\n" - "fmla v31.8h, v4.8h, v10.8h\n" + "ldr x20, [x13, #0x20]\n" + "ldr q13, [x20, x28]\n" + "bge 2f\n" + "1:" // Channel loop + "mov v24.16b, v25.16b\n fmla v24.8h, v4.8h, v9.8h\n" + "mov v23.16b, v25.16b\n fmla v23.8h, v3.8h, v9.8h\n" + "ldr x21, [x13, #0x28]\n" + "ldr x20, [x13, #0x30]\n" + "mov v22.16b, v25.16b\n fmla v22.8h, v1.8h, v9.8h\n" + "mov v21.16b, v25.16b\n fmla v21.8h, v0.8h, v9.8h\n" + "ldr q18, [x21, x28]\n" + "ldr q25, [x14, #0x0]\n" + "fmla v24.8h, v0.8h, v10.8h\n" + "fmla v23.8h, v2.8h, v11.8h\n" + "ldr q17, [x20, x28]\n" + "ldr x21, [x13, #0x38]\n" + "fmla v22.8h, v2.8h, v12.8h\n" + "fmla v21.8h, v1.8h, v12.8h\n" + "ldr x20, [x13, #0x48]\n" + "ldr q20, [x20, x28]\n" + "fmla v24.8h, v5.8h, v12.8h\n" + "fmla v23.8h, v4.8h, v12.8h\n" + "ldr q16, [x21, x28]\n" + "ldr x20, [x13, #0x40]\n" + "fmla v22.8h, v6.8h, v18.8h\n" + "ldr q18, [x20, x28]\n" + "fmla v21.8h, v3.8h, v13.8h\n" + "ldr x20, [x13, #0x50]\n" + "fmla v24.8h, v7.8h, v13.8h\n" + "fmla v23.8h, v6.8h, v13.8h\n" + "ldr x22, [x13, #0x58]\n" + "ldr x21, [x13, #0x60]\n" + "fmla v22.8h, v4.8h, v13.8h\n" + "fmla v21.8h, v8.8h, v17.8h\n" + "ldr q17, [x20, x28]\n" + "ldr x20, [x13, #0x68]\n" + "fmla v24.8h, v1.8h, v16.8h\n" + "fmla v23.8h, v0.8h, v16.8h\n" + "ldr q16, [x22, x28]\n" + "ldr x26, [x13, #0x70]\n" + "fmla v22.8h, v5.8h, v20.8h\n" + "fmla v21.8h, v4.8h, v20.8h\n" "ldr q4, [x14, #0x50]\n" - "ldr x20, [x13, #0x78]\n" - "fmla v28.8h, v2.8h, v9.8h\n" - "fmla v29.8h, v1.8h, v9.8h\n" - "ldr q9, [x23, x28]\n" + "ldr x25, [x13, #0x78]\n" + "fmla v24.8h, v2.8h, v18.8h\n" + "fmla v23.8h, v1.8h, v18.8h\n" + "ldr q19, [x21, x28]\n" "ldr q1, [x14, #0x20]\n" - "fmla v30.8h, v0.8h, v11.8h\n" + "fmla v22.8h, v0.8h, v17.8h\n" "ldr q0, [x14, #0x10]\n" - "fmla v31.8h, v2.8h, v12.8h\n" + "fmla v21.8h, v2.8h, v16.8h\n" "ldr q2, [x14, #0x30]\n" - "fmla v28.8h, v8.8h, v10.8h\n" - "fmla v29.8h, v7.8h, v10.8h\n" - "ldr q10, [x22, x28]\n" - "ldp x26, x22, [x13, #0x0]\n" - "fmla v30.8h, v3.8h, v9.8h\n" - "fmla v31.8h, v5.8h, v10.8h\n" - "ldp x25, x24, [x13, #0x10]\n" - "ldr x23, [x13, #0x20]\n" - "ldr q13, [x23, x16]\n" - "fmla v28.8h, v3.8h, v11.8h\n" - "ldr q11, [x21, x28]\n" - "fmla v29.8h, v5.8h, v12.8h\n" - "ldr q12, [x20, x28]\n" + "fmla v24.8h, v8.8h, v20.8h\n" + "fmla v23.8h, v7.8h, v20.8h\n" + "ldr q18, [x20, x28]\n" + "ldp x24, x23, [x13, #0x0]\n" + "fmla v22.8h, v3.8h, v19.8h\n" + "fmla v21.8h, v5.8h, v18.8h\n" + "ldp x22, x21, [x13, #0x10]\n" + "ldr x20, [x13, #0x20]\n" + "ldr q13, [x20, x16]\n" + "fmla v24.8h, v3.8h, v17.8h\n" + "ldr q17, [x26, x28]\n" + "fmla v23.8h, v5.8h, v16.8h\n" + "ldr q16, [x25, x28]\n" "ldr q3, [x14, #0x40]\n" - "fmla v30.8h, v7.8h, v11.8h\n" - "fmla v31.8h, v6.8h, v11.8h\n" - "ldr q11, [x25, x16]\n" + "fmla v22.8h, v7.8h, v17.8h\n" + "fmla v21.8h, v6.8h, v17.8h\n" + "ldr q11, [x22, x16]\n" "ldr q5, [x14, #0x60]\n" - "fmla v28.8h, v6.8h, v9.8h\n" - "fmla v29.8h, v8.8h, v10.8h\n" - "ldr q9, [x26, x16]\n" - "ldr q10, [x22, x16]\n" - "fmla v30.8h, v8.8h, v12.8h\n" - "fmla v31.8h, v7.8h, v12.8h\n" - "ldr q12, [x24, x16]\n" + "fmla v24.8h, v6.8h, v19.8h\n" + "fmla v23.8h, v8.8h, v18.8h\n" + "ldr q9, [x24, x16]\n" + "ldr q10, [x23, x16]\n" + "fmla v22.8h, v8.8h, v16.8h\n" + "fmla v21.8h, v7.8h, v16.8h\n" + "ldr q12, [x21, x16]\n" "ldr q6, [x14, #0x70]\n" - "fmax v28.8h, v28.8h, v18.8h\n" - "fmax v29.8h, v29.8h, v18.8h\n" + "fmax v24.8h, v24.8h, v27.8h\n" + "fmax v23.8h, v23.8h, v27.8h\n" "ldr q7, [x14, #0x80]\n" "ldr q8, [x14, #0x90]\n" - "fmax v30.8h, v30.8h, v18.8h\n" - "fmax v31.8h, v31.8h, v18.8h\n" + "fmax v22.8h, v22.8h, v27.8h\n" + "fmax v21.8h, v21.8h, v27.8h\n" "add x16, x16, #0x10\n" "add x27, x27, #0x10\n" - "fmin v28.8h, v28.8h, v17.8h\n" - "fmin v29.8h, v29.8h, v17.8h\n" + "fmin v24.8h, v24.8h, v26.8h\n" + "fmin v23.8h, v23.8h, v26.8h\n" "cmp x16, x15, LSL #4\n" - "fmin v30.8h, v30.8h, v17.8h\n" - "fmin v31.8h, v31.8h, v17.8h\n" + "fmin v22.8h, v22.8h, v26.8h\n" + "fmin v21.8h, v21.8h, v26.8h\n" "add x28, x28, #0x10\n" - "str q28, [x12, x27]\n" + "str q24, [x12, x27]\n" "add x14, x14, #0xa0\n" - "str q29, [x11, x27]\n" - "str q30, [x10, x27]\n" - "str q31, [x9, x27]\n" + "str q23, [x11, x27]\n" + "str q22, [x10, x27]\n" + "str q21, [x9, x27]\n" "blt 1b\n" "2:" // Channel tail - "mov v28.16b, v16.16b\n fmla v28.8h, v4.8h, v9.8h\n" - "mov v29.16b, v16.16b\n fmla v29.8h, v3.8h, v9.8h\n" - "ldr x22, [x13, #0x28]\n" - "ldr x21, [x13, #0x30]\n" - "mov v30.16b, v16.16b\n fmla v30.8h, v1.8h, v9.8h\n" - "mov v31.16b, v16.16b\n fmla v31.8h, v0.8h, v9.8h\n" - "ldr q9, [x22, x28]\n" - "ldr x20, [x13, #0x38]\n" - "fmla v28.8h, v0.8h, v10.8h\n" - "fmla v29.8h, v2.8h, v11.8h\n" - "ldr q11, [x21, x28]\n" - "ldr x22, [x13, #0x48]\n" - "ldr q10, [x22, x28]\n" - "fmla v30.8h, v2.8h, v12.8h\n" - "fmla v31.8h, v1.8h, v12.8h\n" - "ldr x26, [x13, #0x40]\n" - "fmla v28.8h, v5.8h, v12.8h\n" - "fmla v29.8h, v4.8h, v12.8h\n" - "ldr q12, [x20, x28]\n" - "ldr x25, [x13, #0x50]\n" - "fmla v30.8h, v6.8h, v9.8h\n" - "ldr q9, [x26, x28]\n" - "fmla v31.8h, v3.8h, v13.8h\n" - "ldr x24, [x13, #0x58]\n" - "fmla v28.8h, v7.8h, v13.8h\n" - "fmla v29.8h, v6.8h, v13.8h\n" + "mov v24.16b, v25.16b\n fmla v24.8h, v4.8h, v9.8h\n" + "mov v23.16b, v25.16b\n fmla v23.8h, v3.8h, v9.8h\n" + "ldr x21, [x13, #0x28]\n" + "ldr x20, [x13, #0x30]\n" + "mov v22.16b, v25.16b\n fmla v22.8h, v1.8h, v9.8h\n" + "mov v21.16b, v25.16b\n fmla v21.8h, v0.8h, v9.8h\n" + "ldr q18, [x21, x28]\n" + "ldr x21, [x13, #0x38]\n" + "fmla v24.8h, v0.8h, v10.8h\n" + "fmla v23.8h, v2.8h, v11.8h\n" + "ldr q17, [x20, x28]\n" + "ldr x20, [x13, #0x48]\n" + "ldr q20, [x20, x28]\n" + "fmla v22.8h, v2.8h, v12.8h\n" + "fmla v21.8h, v1.8h, v12.8h\n" + "ldr x20, [x13, #0x40]\n" + "fmla v24.8h, v5.8h, v12.8h\n" + "fmla v23.8h, v4.8h, v12.8h\n" + "ldr q16, [x21, x28]\n" + "ldr x21, [x13, #0x50]\n" + "fmla v22.8h, v6.8h, v18.8h\n" + "ldr q18, [x20, x28]\n" + "fmla v21.8h, v3.8h, v13.8h\n" + "ldr x20, [x13, #0x58]\n" + "fmla v24.8h, v7.8h, v13.8h\n" + "fmla v23.8h, v6.8h, v13.8h\n" "ldr x23, [x13, #0x60]\n" "ldr x22, [x13, #0x68]\n" - "fmla v30.8h, v4.8h, v13.8h\n" - "fmla v31.8h, v8.8h, v11.8h\n" - "ldr q11, [x25, x28]\n" + "fmla v22.8h, v4.8h, v13.8h\n" + "fmla v21.8h, v8.8h, v17.8h\n" + "ldr q17, [x21, x28]\n" "ldr x21, [x13, #0x70]\n" - "fmla v28.8h, v1.8h, v12.8h\n" - "fmla v29.8h, v0.8h, v12.8h\n" - "ldr q12, [x24, x28]\n" + "fmla v24.8h, v1.8h, v16.8h\n" + "fmla v23.8h, v0.8h, v16.8h\n" + "ldr q16, [x20, x28]\n" "ldr x20, [x13, #0x78]\n" - "fmla v30.8h, v5.8h, v10.8h\n" - "fmla v31.8h, v4.8h, v10.8h\n" + "fmla v22.8h, v5.8h, v20.8h\n" + "fmla v21.8h, v4.8h, v20.8h\n" "add x27, x27, #0x10\n" - "fmla v28.8h, v2.8h, v9.8h\n" - "fmla v29.8h, v1.8h, v9.8h\n" - "ldr q9, [x23, x28]\n" - "fmla v30.8h, v0.8h, v11.8h\n" - "fmla v31.8h, v2.8h, v12.8h\n" - "fmla v28.8h, v8.8h, v10.8h\n" - "fmla v29.8h, v7.8h, v10.8h\n" - "ldr q10, [x22, x28]\n" - "fmla v30.8h, v3.8h, v9.8h\n" - "fmla v31.8h, v5.8h, v10.8h\n" - "fmla v28.8h, v3.8h, v11.8h\n" - "ldr q11, [x21, x28]\n" - "fmla v29.8h, v5.8h, v12.8h\n" - "ldr q12, [x20, x28]\n" - "fmla v30.8h, v7.8h, v11.8h\n" - "fmla v31.8h, v6.8h, v11.8h\n" + "fmla v24.8h, v2.8h, v18.8h\n" + "fmla v23.8h, v1.8h, v18.8h\n" + "ldr q19, [x23, x28]\n" + "fmla v22.8h, v0.8h, v17.8h\n" + "fmla v21.8h, v2.8h, v16.8h\n" + "fmla v24.8h, v8.8h, v20.8h\n" + "fmla v23.8h, v7.8h, v20.8h\n" + "ldr q18, [x22, x28]\n" + "fmla v22.8h, v3.8h, v19.8h\n" + "fmla v21.8h, v5.8h, v18.8h\n" + "fmla v24.8h, v3.8h, v17.8h\n" + "ldr q17, [x21, x28]\n" + "fmla v23.8h, v5.8h, v16.8h\n" + "ldr q16, [x20, x28]\n" + "fmla v22.8h, v7.8h, v17.8h\n" + "fmla v21.8h, v6.8h, v17.8h\n" "add x28, x28, #0x10\n" - "fmla v28.8h, v6.8h, v9.8h\n" - "fmla v29.8h, v8.8h, v10.8h\n" - "fmax v28.8h, v28.8h, v18.8h\n" - "fmla v30.8h, v8.8h, v12.8h\n" - "fmla v31.8h, v7.8h, v12.8h\n" - "fmax v29.8h, v29.8h, v18.8h\n" - "fmax v30.8h, v30.8h, v18.8h\n" - "fmax v31.8h, v31.8h, v18.8h\n" - "fmin v28.8h, v28.8h, v17.8h\n" - "fmin v29.8h, v29.8h, v17.8h\n" - "str q28, [x12, x27]\n" - "fmin v30.8h, v30.8h, v17.8h\n" - "fmin v31.8h, v31.8h, v17.8h\n" - "str q29, [x11, x27]\n" - "str q30, [x10, x27]\n" - "str q31, [x9, x27]\n" + "fmla v24.8h, v6.8h, v19.8h\n" + "fmla v23.8h, v8.8h, v18.8h\n" + "fmax v24.8h, v24.8h, v27.8h\n" + "fmla v22.8h, v8.8h, v16.8h\n" + "fmla v21.8h, v7.8h, v16.8h\n" + "fmax v23.8h, v23.8h, v27.8h\n" + "fmax v22.8h, v22.8h, v27.8h\n" + "fmax v21.8h, v21.8h, v27.8h\n" + "fmin v24.8h, v24.8h, v26.8h\n" + "fmin v23.8h, v23.8h, v26.8h\n" + "str q24, [x12, x27]\n" + "fmin v22.8h, v22.8h, v26.8h\n" + "fmin v21.8h, v21.8h, v26.8h\n" + "str q23, [x11, x27]\n" + "str q22, [x10, x27]\n" + "str q21, [x9, x27]\n" "3:" // Oddments "tst %x[n_channels], #0x7\n" "beq 56f\n" - "ldr q16, [x14, #0x0]\n" + "ldr q25, [x14, #0x0]\n" "ldr q0, [x14, #0x10]\n" - "mov x27, x28\n" - "add x12, x12, x27\n" + "mov x20, x28\n" + "add x12, x12, x20\n" "ldr q1, [x14, #0x20]\n" "ldr q2, [x14, #0x30]\n" - "add x11, x11, x27\n" - "add x10, x10, x27\n" + "add x11, x11, x20\n" + "add x10, x10, x20\n" "ldr q3, [x14, #0x40]\n" "ldr q4, [x14, #0x50]\n" - "add x9, x9, x27\n" + "add x9, x9, x20\n" "ldr q5, [x14, #0x60]\n" "ldr q6, [x14, #0x70]\n" "ldr q7, [x14, #0x80]\n" @@ -357,12 +357,12 @@ void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( "ld1 { v12.h }[0], [x21], #0x2\n" "ld1 { v13.h }[0], [x20], #0x2\n" "7:" // Oddments: Load inputs (1, 1), (0, 0), (0, 3), (1, 2), (2, 1): Bit 2: End - "mov v28.16b, v16.16b\n fmla v28.8h, v4.8h, v9.8h\n" - "mov v29.16b, v16.16b\n fmla v29.8h, v3.8h, v9.8h\n" + "mov v28.16b, v25.16b\n fmla v28.8h, v4.8h, v9.8h\n" + "mov v29.16b, v25.16b\n fmla v29.8h, v3.8h, v9.8h\n" "ldr x20, [x13, #0x28]\n" "add x20, x20, x28\n" - "mov v30.16b, v16.16b\n fmla v30.8h, v1.8h, v9.8h\n" - "mov v31.16b, v16.16b\n fmla v31.8h, v0.8h, v9.8h\n" + "mov v30.16b, v25.16b\n fmla v30.8h, v1.8h, v9.8h\n" + "mov v31.16b, v25.16b\n fmla v31.8h, v0.8h, v9.8h\n" "fmla v28.8h, v0.8h, v10.8h\n" "fmla v29.8h, v2.8h, v11.8h\n" "fmla v28.8h, v5.8h, v12.8h\n" @@ -635,14 +635,14 @@ void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( "51:" // Oddments: Load input (3, 2): Bit 2: End "fmla v30.8h, v8.8h, v12.8h\n" "fmla v31.8h, v7.8h, v12.8h\n" - "fmax v28.8h, v28.8h, v18.8h\n" - "fmax v29.8h, v29.8h, v18.8h\n" - "fmax v30.8h, v30.8h, v18.8h\n" - "fmax v31.8h, v31.8h, v18.8h\n" - "fmin v28.8h, v28.8h, v17.8h\n" - "fmin v29.8h, v29.8h, v17.8h\n" - "fmin v30.8h, v30.8h, v17.8h\n" - "fmin v31.8h, v31.8h, v17.8h\n" + "fmax v28.8h, v28.8h, v27.8h\n" + "fmax v29.8h, v29.8h, v27.8h\n" + "fmax v30.8h, v30.8h, v27.8h\n" + "fmax v31.8h, v31.8h, v27.8h\n" + "fmin v28.8h, v28.8h, v26.8h\n" + "fmin v29.8h, v29.8h, v26.8h\n" + "fmin v30.8h, v30.8h, v26.8h\n" + "fmin v31.8h, v31.8h, v26.8h\n" "tbz %x[n_channels], #2, 53f\n" "st1 { v28.d }[0], [x12], #0x8\n" "st1 { v29.d }[0], [x11], #0x8\n" @@ -687,7 +687,7 @@ void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( "56:" // End : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v16", "v17", "v18", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp index 75368dfcf9e8e6fb2cae36dbf4dc7e903a03b63f..6bbd3508cb295a96caee7369f4c58eff9f214c1e 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include @@ -33,8 +33,8 @@ namespace arm_conv { namespace depthwise { -void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); -void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); +void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(const __fp16 *const *const input_ptrs, __fp16 *const *const outptrs, const void *params, unsigned int n_channels, const __fp16 activation_min, const __fp16 activation_max); +void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(const unsigned int n_tile_rows, const unsigned int n_tile_cols, const __fp16 *inptr, int64_t ld_input_row, int64_t ld_input_col, __fp16 *outptr, int64_t ld_output_row, int64_t ld_output_col, const void *params, unsigned int n_channels, const __fp16 activation_min, const __fp16 activation_max); class a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst : public DepthwiseDepthfirstStrategy<__fp16, __fp16, __fp16, __fp16> { @@ -57,7 +57,7 @@ class a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst : public DepthwiseDepthfirst constexpr static unsigned int output_cols = 3; a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst(const CPUInfo *) - : DepthwiseDepthfirstStrategy<__fp16, __fp16, __fp16, __fp16>(3, 3, 1) {} + : Parent(output_rows, output_cols, kernel_rows, kernel_cols, stride_rows, stride_cols) {} arm_gemm::VLType get_vl_type(void) const override { return vl_type; } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp index 2b1dc3646d7af030e346a27943a7a999d5046241..4e64a2bf2b67b4e08b38300c7ef83c9e79a7b3c6 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp @@ -120,9 +120,9 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "add x9, x11, x8\n" "add x28, x15, x22, LSL #1\n" "add x20, %x[params_struct], %[offsetof_args_min]\n" - "ld1r { v18.8h }, [x20]\n" + "ld1r { v15.8h }, [x20]\n" "add x20, %x[params_struct], %[offsetof_args_max]\n" - "ld1r { v17.8h }, [x20]\n" + "ld1r { v14.8h }, [x20]\n" "add x27, x10, x25, LSL #1\n" "add x26, x9, x8\n" "add x25, x28, x22, LSL #1\n" @@ -130,7 +130,7 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "mov x21, #0x0\n" "sub x20, XZR, x24\n" "cbz x23, 4f\n" - "ldr q16, [x14, #0x0]\n" + "ldr q31, [x14, #0x0]\n" "ldr q0, [x14, #0x10]\n" "cmp x24, x23, LSL #4\n" "ldr q1, [x14, #0x20]\n" @@ -149,304 +149,304 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "ldr q13, [x13, x11]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "mov v24.16b, v16.16b\n fmla v24.8h, v7.8h, v9.8h\n" - "mov v23.16b, v16.16b\n fmla v23.8h, v8.8h, v9.8h\n" + "mov v29.16b, v31.16b\n fmla v29.8h, v7.8h, v9.8h\n" + "mov v28.16b, v31.16b\n fmla v28.8h, v8.8h, v9.8h\n" "add x24, x24, #0x10\n" "cmp x24, x23, LSL #4\n" - "mov v25.16b, v16.16b\n fmla v25.8h, v6.8h, v9.8h\n" - "fmla v24.8h, v4.8h, v13.8h\n" + "mov v27.16b, v31.16b\n fmla v27.8h, v6.8h, v9.8h\n" + "fmla v29.8h, v4.8h, v13.8h\n" "add x20, x20, #0x10\n" "add x21, x21, #0x10\n" - "mov v26.16b, v16.16b\n fmla v26.8h, v5.8h, v9.8h\n" - "mov v27.16b, v16.16b\n fmla v27.8h, v4.8h, v9.8h\n" - "mov v28.16b, v16.16b\n fmla v28.8h, v3.8h, v9.8h\n" - "fmla v23.8h, v0.8h, v10.8h\n" - "ldr q10, [x12, x9]\n" - "fmla v25.8h, v2.8h, v11.8h\n" - "ldr q11, [x12, x8]\n" - "mov v29.16b, v16.16b\n fmla v29.8h, v2.8h, v9.8h\n" - "fmla v24.8h, v6.8h, v11.8h\n" - "mov v31.16b, v16.16b\n fmla v31.8h, v0.8h, v9.8h\n" - "fmla v23.8h, v5.8h, v13.8h\n" - "fmla v25.8h, v3.8h, v13.8h\n" + "mov v26.16b, v31.16b\n fmla v26.8h, v5.8h, v9.8h\n" + "mov v25.16b, v31.16b\n fmla v25.8h, v4.8h, v9.8h\n" + "mov v24.16b, v31.16b\n fmla v24.8h, v3.8h, v9.8h\n" + "fmla v28.8h, v0.8h, v10.8h\n" + "ldr q23, [x12, x9]\n" + "fmla v27.8h, v2.8h, v11.8h\n" + "ldr q18, [x12, x8]\n" + "mov v22.16b, v31.16b\n fmla v22.8h, v2.8h, v9.8h\n" + "fmla v29.8h, v6.8h, v18.8h\n" + "mov v21.16b, v31.16b\n fmla v21.8h, v0.8h, v9.8h\n" + "fmla v28.8h, v5.8h, v13.8h\n" + "fmla v27.8h, v3.8h, v13.8h\n" "fmla v26.8h, v2.8h, v13.8h\n" - "fmla v27.8h, v1.8h, v13.8h\n" - "fmla v28.8h, v0.8h, v13.8h\n" - "ldr q13, [x16, x8]\n" - "fmla v29.8h, v6.8h, v12.8h\n" - "ldr q12, [x27, x26]\n" - "mov v30.16b, v16.16b\n fmla v30.8h, v1.8h, v9.8h\n" - "ldr q16, [x14, #0x0]\n" + "fmla v25.8h, v1.8h, v13.8h\n" "fmla v24.8h, v0.8h, v13.8h\n" - "fmla v31.8h, v8.8h, v12.8h\n" - "ldr q12, [x16, x9]\n" - "fmla v23.8h, v7.8h, v11.8h\n" - "fmla v30.8h, v0.8h, v11.8h\n" - "fmla v26.8h, v4.8h, v11.8h\n" - "fmla v27.8h, v3.8h, v11.8h\n" - "fmla v29.8h, v1.8h, v11.8h\n" - "ld1 { v11.8h }, [x13]\n" - "fmla v24.8h, v2.8h, v12.8h\n" - "fmla v25.8h, v1.8h, v12.8h\n" - "ld1 { v12.8h }, [x10]\n" - "fmla v28.8h, v4.8h, v10.8h\n" - "fmla v23.8h, v1.8h, v13.8h\n" - "ldr q13, [x13, x26]\n" - "fmla v30.8h, v2.8h, v10.8h\n" - "fmla v31.8h, v1.8h, v10.8h\n" - "fmla v24.8h, v8.8h, v10.8h\n" - "fmla v25.8h, v7.8h, v10.8h\n" - "fmla v27.8h, v5.8h, v10.8h\n" - "ldr q10, [x10, x11]\n" - "fmla v26.8h, v0.8h, v11.8h\n" - "fmla v29.8h, v3.8h, v12.8h\n" - "fmla v28.8h, v2.8h, v13.8h\n" - "fmla v30.8h, v4.8h, v10.8h\n" - "fmla v31.8h, v3.8h, v10.8h\n" - "fmla v23.8h, v3.8h, v11.8h\n" - "ldr q11, [x10, x26]\n" - "fmla v25.8h, v5.8h, v13.8h\n" - "ldr q13, [x27, x8]\n" - "fmla v26.8h, v6.8h, v12.8h\n" - "ldr q12, [x13, x8]\n" - "fmla v27.8h, v7.8h, v10.8h\n" - "fmla v29.8h, v5.8h, v10.8h\n" - "fmla v28.8h, v6.8h, v10.8h\n" - "fmla v31.8h, v5.8h, v11.8h\n" - "fmla v30.8h, v6.8h, v13.8h\n" - "fmla v26.8h, v8.8h, v10.8h\n" - "fmla v29.8h, v7.8h, v13.8h\n" - "ldr q13, [x27, x9]\n" - "fmla v24.8h, v3.8h, v12.8h\n" - "fmla v27.8h, v0.8h, v12.8h\n" - "fmla v28.8h, v8.8h, v11.8h\n" - "ldr q11, [x13, x9]\n" - "fmla v30.8h, v8.8h, v13.8h\n" + "ldr q17, [x16, x8]\n" + "fmla v22.8h, v6.8h, v12.8h\n" + "ldr q16, [x27, x26]\n" + "mov v20.16b, v31.16b\n fmla v20.8h, v1.8h, v9.8h\n" + "ldr q31, [x14, #0x0]\n" + "fmla v29.8h, v0.8h, v17.8h\n" + "fmla v21.8h, v8.8h, v16.8h\n" + "ldr q16, [x16, x9]\n" + "fmla v28.8h, v7.8h, v18.8h\n" + "fmla v20.8h, v0.8h, v18.8h\n" + "fmla v26.8h, v4.8h, v18.8h\n" + "fmla v25.8h, v3.8h, v18.8h\n" + "fmla v22.8h, v1.8h, v18.8h\n" + "ld1 { v19.8h }, [x13]\n" + "fmla v29.8h, v2.8h, v16.8h\n" + "fmla v27.8h, v1.8h, v16.8h\n" + "ld1 { v18.8h }, [x10]\n" + "fmla v24.8h, v4.8h, v23.8h\n" + "fmla v28.8h, v1.8h, v17.8h\n" + "ldr q16, [x13, x26]\n" + "fmla v20.8h, v2.8h, v23.8h\n" + "fmla v21.8h, v1.8h, v23.8h\n" + "fmla v29.8h, v8.8h, v23.8h\n" + "fmla v27.8h, v7.8h, v23.8h\n" + "fmla v25.8h, v5.8h, v23.8h\n" + "ldr q17, [x10, x11]\n" + "fmla v26.8h, v0.8h, v19.8h\n" + "fmla v22.8h, v3.8h, v18.8h\n" + "fmla v24.8h, v2.8h, v16.8h\n" + "fmla v20.8h, v4.8h, v17.8h\n" + "fmla v21.8h, v3.8h, v17.8h\n" + "fmla v28.8h, v3.8h, v19.8h\n" + "ldr q19, [x10, x26]\n" + "fmla v27.8h, v5.8h, v16.8h\n" + "ldr q16, [x27, x8]\n" + "fmla v26.8h, v6.8h, v18.8h\n" + "ldr q18, [x13, x8]\n" + "fmla v25.8h, v7.8h, v17.8h\n" + "fmla v22.8h, v5.8h, v17.8h\n" + "fmla v24.8h, v6.8h, v17.8h\n" + "fmla v21.8h, v5.8h, v19.8h\n" + "fmla v20.8h, v6.8h, v16.8h\n" + "fmla v26.8h, v8.8h, v17.8h\n" + "fmla v22.8h, v7.8h, v16.8h\n" + "ldr q17, [x27, x9]\n" + "fmla v29.8h, v3.8h, v18.8h\n" + "fmla v25.8h, v0.8h, v18.8h\n" + "fmla v24.8h, v8.8h, v19.8h\n" + "ldr q16, [x13, x9]\n" + "fmla v20.8h, v8.8h, v17.8h\n" "add x13, x13, #0x10\n" - "fmla v31.8h, v7.8h, v13.8h\n" - "ldr q13, [x10, x9]\n" - "fmla v23.8h, v4.8h, v12.8h\n" - "fmla v26.8h, v1.8h, v12.8h\n" - "ldr q12, [x10, x8]\n" - "fmla v24.8h, v5.8h, v11.8h\n" + "fmla v21.8h, v7.8h, v17.8h\n" + "ldr q19, [x10, x9]\n" + "fmla v28.8h, v4.8h, v18.8h\n" + "fmla v26.8h, v1.8h, v18.8h\n" + "ldr q17, [x10, x8]\n" + "fmla v29.8h, v5.8h, v16.8h\n" "add x10, x10, #0x10\n" - "fmla v25.8h, v4.8h, v11.8h\n" - "fmla v27.8h, v2.8h, v11.8h\n" - "fmla v28.8h, v1.8h, v11.8h\n" - "ldr q11, [x16, x11]\n" - "fmla v29.8h, v4.8h, v12.8h\n" + "fmla v27.8h, v4.8h, v16.8h\n" + "fmla v25.8h, v2.8h, v16.8h\n" + "fmla v24.8h, v1.8h, v16.8h\n" + "ldr q16, [x16, x11]\n" + "fmla v22.8h, v4.8h, v17.8h\n" "add x16, x16, #0x10\n" "ld1 { v10.8h }, [x16]\n" - "fmla v30.8h, v3.8h, v12.8h\n" - "fmla v31.8h, v4.8h, v13.8h\n" + "fmla v20.8h, v3.8h, v17.8h\n" + "fmla v21.8h, v4.8h, v19.8h\n" "ldr q4, [x14, #0x50]\n" - "fmla v26.8h, v7.8h, v12.8h\n" - "fmla v27.8h, v6.8h, v12.8h\n" - "ld1 { v12.8h }, [x12]\n" - "fmla v23.8h, v2.8h, v11.8h\n" - "fmla v24.8h, v1.8h, v11.8h\n" + "fmla v26.8h, v7.8h, v17.8h\n" + "fmla v25.8h, v6.8h, v17.8h\n" + "ld1 { v18.8h }, [x12]\n" + "fmla v28.8h, v2.8h, v16.8h\n" + "fmla v29.8h, v1.8h, v16.8h\n" "ldr q1, [x14, #0x20]\n" - "fmax v24.8h, v24.8h, v18.8h\n" - "fmla v25.8h, v0.8h, v11.8h\n" - "ldr q11, [x12, x26]\n" - "fmla v28.8h, v7.8h, v13.8h\n" + "fmax v29.8h, v29.8h, v15.8h\n" + "fmla v27.8h, v0.8h, v16.8h\n" + "ldr q17, [x12, x26]\n" + "fmla v24.8h, v7.8h, v19.8h\n" "add x12, x12, #0x10\n" "ldr q9, [x12, x11]\n" - "fmla v30.8h, v5.8h, v13.8h\n" - "fmla v29.8h, v0.8h, v12.8h\n" + "fmla v20.8h, v5.8h, v19.8h\n" + "fmla v22.8h, v0.8h, v18.8h\n" "ldr q0, [x14, #0x10]\n" - "fmla v31.8h, v2.8h, v11.8h\n" + "fmla v21.8h, v2.8h, v17.8h\n" "ldr q2, [x14, #0x30]\n" - "fmla v27.8h, v8.8h, v13.8h\n" - "ldr q13, [x27, x11]\n" - "fmla v23.8h, v6.8h, v12.8h\n" - "fmla v26.8h, v3.8h, v12.8h\n" + "fmla v25.8h, v8.8h, v19.8h\n" + "ldr q16, [x27, x11]\n" + "fmla v28.8h, v6.8h, v18.8h\n" + "fmla v26.8h, v3.8h, v18.8h\n" "ldr q3, [x14, #0x40]\n" - "fmax v23.8h, v23.8h, v18.8h\n" - "fmla v25.8h, v8.8h, v11.8h\n" - "fmla v28.8h, v5.8h, v11.8h\n" + "fmax v28.8h, v28.8h, v15.8h\n" + "fmla v27.8h, v8.8h, v17.8h\n" + "fmla v24.8h, v5.8h, v17.8h\n" "ldr q11, [x16, x26]\n" "ldr q5, [x14, #0x60]\n" - "fmla v29.8h, v8.8h, v13.8h\n" + "fmla v22.8h, v8.8h, v16.8h\n" "ldr q8, [x14, #0x90]\n" - "fmla v30.8h, v7.8h, v13.8h\n" + "fmla v20.8h, v7.8h, v16.8h\n" "ldr q7, [x14, #0x80]\n" - "fmla v31.8h, v6.8h, v13.8h\n" + "fmla v21.8h, v6.8h, v16.8h\n" "ldr q13, [x13, x11]\n" "ldr q6, [x14, #0x70]\n" - "fmax v25.8h, v25.8h, v18.8h\n" - "fmax v26.8h, v26.8h, v18.8h\n" - "fmax v27.8h, v27.8h, v18.8h\n" + "fmax v27.8h, v27.8h, v15.8h\n" + "fmax v26.8h, v26.8h, v15.8h\n" + "fmax v25.8h, v25.8h, v15.8h\n" "add x27, x27, #0x10\n" "ld1 { v12.8h }, [x27]\n" - "fmax v28.8h, v28.8h, v18.8h\n" - "fmax v29.8h, v29.8h, v18.8h\n" + "fmax v24.8h, v24.8h, v15.8h\n" + "fmax v22.8h, v22.8h, v15.8h\n" "add x14, x14, #0xa0\n" - "fmax v30.8h, v30.8h, v18.8h\n" - "fmax v31.8h, v31.8h, v18.8h\n" - "fmin v23.8h, v23.8h, v17.8h\n" - "fmin v24.8h, v24.8h, v17.8h\n" - "st1 { v23.8h }, [x15]\n" - "fmin v25.8h, v25.8h, v17.8h\n" - "fmin v26.8h, v26.8h, v17.8h\n" - "str q24, [x15, x17]\n" - "fmin v27.8h, v27.8h, v17.8h\n" - "fmin v28.8h, v28.8h, v17.8h\n" - "str q25, [x15, x22]\n" + "fmax v20.8h, v20.8h, v15.8h\n" + "fmax v21.8h, v21.8h, v15.8h\n" + "fmin v28.8h, v28.8h, v14.8h\n" + "fmin v29.8h, v29.8h, v14.8h\n" + "st1 { v28.8h }, [x15]\n" + "fmin v27.8h, v27.8h, v14.8h\n" + "fmin v26.8h, v26.8h, v14.8h\n" + "str q29, [x15, x17]\n" + "fmin v25.8h, v25.8h, v14.8h\n" + "fmin v24.8h, v24.8h, v14.8h\n" + "str q27, [x15, x22]\n" "add x15, x15, #0x10\n" - "fmin v29.8h, v29.8h, v17.8h\n" - "fmin v30.8h, v30.8h, v17.8h\n" + "fmin v22.8h, v22.8h, v14.8h\n" + "fmin v20.8h, v20.8h, v14.8h\n" "st1 { v26.8h }, [x28]\n" - "fmin v31.8h, v31.8h, v17.8h\n" - "str q27, [x28, x17]\n" - "str q28, [x28, x22]\n" + "fmin v21.8h, v21.8h, v14.8h\n" + "str q25, [x28, x17]\n" + "str q24, [x28, x22]\n" "add x28, x28, #0x10\n" - "st1 { v29.8h }, [x25]\n" - "str q30, [x25, x17]\n" - "str q31, [x25, x22]\n" + "st1 { v22.8h }, [x25]\n" + "str q20, [x25, x17]\n" + "str q21, [x25, x22]\n" "add x25, x25, #0x10\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "mov v24.16b, v16.16b\n fmla v24.8h, v7.8h, v9.8h\n" - "mov v23.16b, v16.16b\n fmla v23.8h, v8.8h, v9.8h\n" - "mov v25.16b, v16.16b\n fmla v25.8h, v6.8h, v9.8h\n" - "fmla v24.8h, v4.8h, v13.8h\n" - "mov v26.16b, v16.16b\n fmla v26.8h, v5.8h, v9.8h\n" - "mov v27.16b, v16.16b\n fmla v27.8h, v4.8h, v9.8h\n" - "mov v28.16b, v16.16b\n fmla v28.8h, v3.8h, v9.8h\n" - "fmla v23.8h, v0.8h, v10.8h\n" - "ldr q10, [x12, x9]\n" - "fmla v25.8h, v2.8h, v11.8h\n" - "ldr q11, [x12, x8]\n" - "mov v29.16b, v16.16b\n fmla v29.8h, v2.8h, v9.8h\n" - "fmla v24.8h, v6.8h, v11.8h\n" - "mov v31.16b, v16.16b\n fmla v31.8h, v0.8h, v9.8h\n" - "fmla v23.8h, v5.8h, v13.8h\n" - "fmla v25.8h, v3.8h, v13.8h\n" + "mov v29.16b, v31.16b\n fmla v29.8h, v7.8h, v9.8h\n" + "mov v28.16b, v31.16b\n fmla v28.8h, v8.8h, v9.8h\n" + "mov v27.16b, v31.16b\n fmla v27.8h, v6.8h, v9.8h\n" + "fmla v29.8h, v4.8h, v13.8h\n" + "mov v26.16b, v31.16b\n fmla v26.8h, v5.8h, v9.8h\n" + "mov v25.16b, v31.16b\n fmla v25.8h, v4.8h, v9.8h\n" + "mov v24.16b, v31.16b\n fmla v24.8h, v3.8h, v9.8h\n" + "fmla v28.8h, v0.8h, v10.8h\n" + "ldr q23, [x12, x9]\n" + "fmla v27.8h, v2.8h, v11.8h\n" + "ldr q18, [x12, x8]\n" + "mov v22.16b, v31.16b\n fmla v22.8h, v2.8h, v9.8h\n" + "fmla v29.8h, v6.8h, v18.8h\n" + "mov v21.16b, v31.16b\n fmla v21.8h, v0.8h, v9.8h\n" + "fmla v28.8h, v5.8h, v13.8h\n" + "fmla v27.8h, v3.8h, v13.8h\n" "fmla v26.8h, v2.8h, v13.8h\n" - "fmla v27.8h, v1.8h, v13.8h\n" - "fmla v28.8h, v0.8h, v13.8h\n" - "ldr q13, [x16, x8]\n" - "fmla v29.8h, v6.8h, v12.8h\n" - "ldr q12, [x27, x26]\n" - "mov v30.16b, v16.16b\n fmla v30.8h, v1.8h, v9.8h\n" + "fmla v25.8h, v1.8h, v13.8h\n" "fmla v24.8h, v0.8h, v13.8h\n" - "fmla v31.8h, v8.8h, v12.8h\n" - "ldr q12, [x16, x9]\n" - "fmla v23.8h, v7.8h, v11.8h\n" - "fmla v30.8h, v0.8h, v11.8h\n" - "fmla v26.8h, v4.8h, v11.8h\n" - "fmla v27.8h, v3.8h, v11.8h\n" - "fmla v29.8h, v1.8h, v11.8h\n" - "ld1 { v11.8h }, [x13]\n" - "fmla v24.8h, v2.8h, v12.8h\n" - "fmla v25.8h, v1.8h, v12.8h\n" - "ld1 { v12.8h }, [x10]\n" - "fmla v28.8h, v4.8h, v10.8h\n" - "fmla v23.8h, v1.8h, v13.8h\n" - "ldr q13, [x13, x26]\n" - "fmla v30.8h, v2.8h, v10.8h\n" - "fmla v31.8h, v1.8h, v10.8h\n" - "fmla v24.8h, v8.8h, v10.8h\n" - "fmla v25.8h, v7.8h, v10.8h\n" - "fmla v27.8h, v5.8h, v10.8h\n" - "ldr q10, [x10, x11]\n" - "fmla v26.8h, v0.8h, v11.8h\n" - "fmla v29.8h, v3.8h, v12.8h\n" - "fmla v28.8h, v2.8h, v13.8h\n" - "fmla v30.8h, v4.8h, v10.8h\n" - "fmla v31.8h, v3.8h, v10.8h\n" - "fmla v23.8h, v3.8h, v11.8h\n" - "ldr q11, [x10, x26]\n" - "fmla v25.8h, v5.8h, v13.8h\n" - "ldr q13, [x27, x8]\n" - "fmla v26.8h, v6.8h, v12.8h\n" - "ldr q12, [x13, x8]\n" - "fmla v27.8h, v7.8h, v10.8h\n" - "fmla v29.8h, v5.8h, v10.8h\n" - "fmla v28.8h, v6.8h, v10.8h\n" - "fmla v31.8h, v5.8h, v11.8h\n" - "fmla v30.8h, v6.8h, v13.8h\n" - "fmla v26.8h, v8.8h, v10.8h\n" - "fmla v29.8h, v7.8h, v13.8h\n" - "ldr q13, [x27, x9]\n" - "fmla v24.8h, v3.8h, v12.8h\n" - "fmla v27.8h, v0.8h, v12.8h\n" - "fmla v28.8h, v8.8h, v11.8h\n" - "ldr q11, [x13, x9]\n" - "fmla v30.8h, v8.8h, v13.8h\n" + "ldr q17, [x16, x8]\n" + "fmla v22.8h, v6.8h, v12.8h\n" + "ldr q16, [x27, x26]\n" + "mov v20.16b, v31.16b\n fmla v20.8h, v1.8h, v9.8h\n" + "fmla v29.8h, v0.8h, v17.8h\n" + "fmla v21.8h, v8.8h, v16.8h\n" + "ldr q16, [x16, x9]\n" + "fmla v28.8h, v7.8h, v18.8h\n" + "fmla v20.8h, v0.8h, v18.8h\n" + "fmla v26.8h, v4.8h, v18.8h\n" + "fmla v25.8h, v3.8h, v18.8h\n" + "fmla v22.8h, v1.8h, v18.8h\n" + "ld1 { v19.8h }, [x13]\n" + "fmla v29.8h, v2.8h, v16.8h\n" + "fmla v27.8h, v1.8h, v16.8h\n" + "ld1 { v18.8h }, [x10]\n" + "fmla v24.8h, v4.8h, v23.8h\n" + "fmla v28.8h, v1.8h, v17.8h\n" + "ldr q16, [x13, x26]\n" + "fmla v20.8h, v2.8h, v23.8h\n" + "fmla v21.8h, v1.8h, v23.8h\n" + "fmla v29.8h, v8.8h, v23.8h\n" + "fmla v27.8h, v7.8h, v23.8h\n" + "fmla v25.8h, v5.8h, v23.8h\n" + "ldr q17, [x10, x11]\n" + "fmla v26.8h, v0.8h, v19.8h\n" + "fmla v22.8h, v3.8h, v18.8h\n" + "fmla v24.8h, v2.8h, v16.8h\n" + "fmla v20.8h, v4.8h, v17.8h\n" + "fmla v21.8h, v3.8h, v17.8h\n" + "fmla v28.8h, v3.8h, v19.8h\n" + "ldr q19, [x10, x26]\n" + "fmla v27.8h, v5.8h, v16.8h\n" + "ldr q16, [x27, x8]\n" + "fmla v26.8h, v6.8h, v18.8h\n" + "ldr q18, [x13, x8]\n" + "fmla v25.8h, v7.8h, v17.8h\n" + "fmla v22.8h, v5.8h, v17.8h\n" + "fmla v24.8h, v6.8h, v17.8h\n" + "fmla v21.8h, v5.8h, v19.8h\n" + "fmla v20.8h, v6.8h, v16.8h\n" + "fmla v26.8h, v8.8h, v17.8h\n" + "fmla v22.8h, v7.8h, v16.8h\n" + "ldr q17, [x27, x9]\n" + "fmla v29.8h, v3.8h, v18.8h\n" + "fmla v25.8h, v0.8h, v18.8h\n" + "fmla v24.8h, v8.8h, v19.8h\n" + "ldr q16, [x13, x9]\n" + "fmla v20.8h, v8.8h, v17.8h\n" "add x13, x13, #0x10\n" - "fmla v31.8h, v7.8h, v13.8h\n" - "ldr q13, [x10, x9]\n" - "fmla v23.8h, v4.8h, v12.8h\n" - "fmla v26.8h, v1.8h, v12.8h\n" - "ldr q12, [x10, x8]\n" - "fmla v24.8h, v5.8h, v11.8h\n" + "fmla v21.8h, v7.8h, v17.8h\n" + "ldr q19, [x10, x9]\n" + "fmla v28.8h, v4.8h, v18.8h\n" + "fmla v26.8h, v1.8h, v18.8h\n" + "ldr q17, [x10, x8]\n" + "fmla v29.8h, v5.8h, v16.8h\n" "add x10, x10, #0x10\n" - "fmla v25.8h, v4.8h, v11.8h\n" - "fmla v27.8h, v2.8h, v11.8h\n" - "fmla v28.8h, v1.8h, v11.8h\n" - "ldr q11, [x16, x11]\n" - "fmla v29.8h, v4.8h, v12.8h\n" + "fmla v27.8h, v4.8h, v16.8h\n" + "fmla v25.8h, v2.8h, v16.8h\n" + "fmla v24.8h, v1.8h, v16.8h\n" + "ldr q16, [x16, x11]\n" + "fmla v22.8h, v4.8h, v17.8h\n" "add x16, x16, #0x10\n" - "fmla v30.8h, v3.8h, v12.8h\n" - "fmla v31.8h, v4.8h, v13.8h\n" - "fmla v26.8h, v7.8h, v12.8h\n" - "fmla v27.8h, v6.8h, v12.8h\n" - "ld1 { v12.8h }, [x12]\n" - "fmla v23.8h, v2.8h, v11.8h\n" - "fmla v24.8h, v1.8h, v11.8h\n" - "fmax v24.8h, v24.8h, v18.8h\n" - "fmla v25.8h, v0.8h, v11.8h\n" - "ldr q11, [x12, x26]\n" - "fmla v28.8h, v7.8h, v13.8h\n" - "fmin v24.8h, v24.8h, v17.8h\n" - "fmla v30.8h, v5.8h, v13.8h\n" - "fmla v29.8h, v0.8h, v12.8h\n" + "fmla v20.8h, v3.8h, v17.8h\n" + "fmla v21.8h, v4.8h, v19.8h\n" + "fmla v26.8h, v7.8h, v17.8h\n" + "fmla v25.8h, v6.8h, v17.8h\n" + "ld1 { v18.8h }, [x12]\n" + "fmla v28.8h, v2.8h, v16.8h\n" + "fmla v29.8h, v1.8h, v16.8h\n" + "fmax v29.8h, v29.8h, v15.8h\n" + "fmla v27.8h, v0.8h, v16.8h\n" + "ldr q17, [x12, x26]\n" + "fmla v24.8h, v7.8h, v19.8h\n" + "fmin v29.8h, v29.8h, v14.8h\n" + "fmla v20.8h, v5.8h, v19.8h\n" + "fmla v22.8h, v0.8h, v18.8h\n" "add x12, x12, #0x10\n" - "fmla v31.8h, v2.8h, v11.8h\n" - "fmla v27.8h, v8.8h, v13.8h\n" - "ldr q13, [x27, x11]\n" - "fmax v27.8h, v27.8h, v18.8h\n" - "fmla v23.8h, v6.8h, v12.8h\n" - "fmla v26.8h, v3.8h, v12.8h\n" - "fmax v23.8h, v23.8h, v18.8h\n" + "fmla v21.8h, v2.8h, v17.8h\n" + "fmla v25.8h, v8.8h, v19.8h\n" + "ldr q16, [x27, x11]\n" + "fmax v25.8h, v25.8h, v15.8h\n" + "fmla v28.8h, v6.8h, v18.8h\n" + "fmla v26.8h, v3.8h, v18.8h\n" + "fmax v28.8h, v28.8h, v15.8h\n" "add x27, x27, #0x10\n" - "fmla v25.8h, v8.8h, v11.8h\n" - "fmla v28.8h, v5.8h, v11.8h\n" - "fmax v25.8h, v25.8h, v18.8h\n" - "fmla v29.8h, v8.8h, v13.8h\n" - "fmla v30.8h, v7.8h, v13.8h\n" - "fmax v26.8h, v26.8h, v18.8h\n" - "fmla v31.8h, v6.8h, v13.8h\n" - "fmax v28.8h, v28.8h, v18.8h\n" - "fmax v29.8h, v29.8h, v18.8h\n" - "fmax v30.8h, v30.8h, v18.8h\n" - "fmax v31.8h, v31.8h, v18.8h\n" - "fmin v23.8h, v23.8h, v17.8h\n" - "st1 { v23.8h }, [x15]\n" - "fmin v25.8h, v25.8h, v17.8h\n" - "fmin v26.8h, v26.8h, v17.8h\n" - "str q24, [x15, x17]\n" - "fmin v27.8h, v27.8h, v17.8h\n" - "fmin v28.8h, v28.8h, v17.8h\n" - "str q25, [x15, x22]\n" + "fmla v27.8h, v8.8h, v17.8h\n" + "fmla v24.8h, v5.8h, v17.8h\n" + "fmax v27.8h, v27.8h, v15.8h\n" + "fmla v22.8h, v8.8h, v16.8h\n" + "fmla v20.8h, v7.8h, v16.8h\n" + "fmax v26.8h, v26.8h, v15.8h\n" + "fmla v21.8h, v6.8h, v16.8h\n" + "fmax v24.8h, v24.8h, v15.8h\n" + "fmax v22.8h, v22.8h, v15.8h\n" + "fmax v20.8h, v20.8h, v15.8h\n" + "fmax v21.8h, v21.8h, v15.8h\n" + "fmin v28.8h, v28.8h, v14.8h\n" + "st1 { v28.8h }, [x15]\n" + "fmin v27.8h, v27.8h, v14.8h\n" + "fmin v26.8h, v26.8h, v14.8h\n" + "str q29, [x15, x17]\n" + "fmin v25.8h, v25.8h, v14.8h\n" + "fmin v24.8h, v24.8h, v14.8h\n" + "str q27, [x15, x22]\n" "add x15, x15, #0x10\n" - "fmin v29.8h, v29.8h, v17.8h\n" - "fmin v30.8h, v30.8h, v17.8h\n" + "fmin v22.8h, v22.8h, v14.8h\n" + "fmin v20.8h, v20.8h, v14.8h\n" "st1 { v26.8h }, [x28]\n" - "fmin v31.8h, v31.8h, v17.8h\n" - "str q27, [x28, x17]\n" - "str q28, [x28, x22]\n" + "fmin v21.8h, v21.8h, v14.8h\n" + "str q25, [x28, x17]\n" + "str q24, [x28, x22]\n" "add x28, x28, #0x10\n" - "st1 { v29.8h }, [x25]\n" - "str q30, [x25, x17]\n" - "str q31, [x25, x22]\n" + "st1 { v22.8h }, [x25]\n" + "str q20, [x25, x17]\n" + "str q21, [x25, x22]\n" "add x25, x25, #0x10\n" "4:" // Tile loop: Oddments "tst %x[n_channels], #0x7\n" "beq 93f\n" - "ldr q16, [x14, #0x0]\n" + "ldr q31, [x14, #0x0]\n" "ldr q0, [x14, #0x10]\n" "add x24, x12, x11\n" "add x23, x16, XZR\n" @@ -509,18 +509,18 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "ldr h12, [x21, #0x0]\n" "ldr h13, [x20, #0x0]\n" "8:" // Tile loop: Oddments: Load inputs: (2, 2), (0, 0), (0, 4), (4, 0), (1, 2): Bit 2: End - "mov v23.16b, v16.16b\n fmla v23.8h, v8.8h, v9.8h\n" - "mov v25.16b, v16.16b\n fmla v25.8h, v6.8h, v9.8h\n" + "mov v23.16b, v31.16b\n fmla v23.8h, v8.8h, v9.8h\n" + "mov v25.16b, v31.16b\n fmla v25.8h, v6.8h, v9.8h\n" "add x20, x27, x26\n" - "mov v24.16b, v16.16b\n fmla v24.8h, v7.8h, v9.8h\n" - "mov v26.16b, v16.16b\n fmla v26.8h, v5.8h, v9.8h\n" - "mov v27.16b, v16.16b\n fmla v27.8h, v4.8h, v9.8h\n" - "mov v28.16b, v16.16b\n fmla v28.8h, v3.8h, v9.8h\n" - "mov v29.16b, v16.16b\n fmla v29.8h, v2.8h, v9.8h\n" + "mov v24.16b, v31.16b\n fmla v24.8h, v7.8h, v9.8h\n" + "mov v26.16b, v31.16b\n fmla v26.8h, v5.8h, v9.8h\n" + "mov v27.16b, v31.16b\n fmla v27.8h, v4.8h, v9.8h\n" + "mov v28.16b, v31.16b\n fmla v28.8h, v3.8h, v9.8h\n" + "mov v29.16b, v31.16b\n fmla v29.8h, v2.8h, v9.8h\n" "fmla v23.8h, v0.8h, v10.8h\n" "fmla v25.8h, v2.8h, v11.8h\n" - "mov v30.16b, v16.16b\n fmla v30.8h, v1.8h, v9.8h\n" - "mov v31.16b, v16.16b\n fmla v31.8h, v0.8h, v9.8h\n" + "mov v30.16b, v31.16b\n fmla v30.8h, v1.8h, v9.8h\n" + "fmla v31.8h, v0.8h, v9.8h\n" "fmla v29.8h, v6.8h, v12.8h\n" "fmla v23.8h, v5.8h, v13.8h\n" "fmla v24.8h, v4.8h, v13.8h\n" @@ -1009,25 +1009,25 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "88:" // Tile loop: Oddments: Load inputs: (4, 2): Bit 2: End "fmla v29.8h, v8.8h, v13.8h\n" "fmla v30.8h, v7.8h, v13.8h\n" - "fmax v23.8h, v23.8h, v18.8h\n" + "fmax v23.8h, v23.8h, v15.8h\n" "fmla v31.8h, v6.8h, v13.8h\n" - "fmax v24.8h, v24.8h, v18.8h\n" - "fmax v25.8h, v25.8h, v18.8h\n" - "fmax v26.8h, v26.8h, v18.8h\n" - "fmax v27.8h, v27.8h, v18.8h\n" - "fmax v28.8h, v28.8h, v18.8h\n" - "fmax v29.8h, v29.8h, v18.8h\n" - "fmax v30.8h, v30.8h, v18.8h\n" - "fmax v31.8h, v31.8h, v18.8h\n" - "fmin v23.8h, v23.8h, v17.8h\n" - "fmin v24.8h, v24.8h, v17.8h\n" - "fmin v25.8h, v25.8h, v17.8h\n" - "fmin v26.8h, v26.8h, v17.8h\n" - "fmin v27.8h, v27.8h, v17.8h\n" - "fmin v28.8h, v28.8h, v17.8h\n" - "fmin v29.8h, v29.8h, v17.8h\n" - "fmin v30.8h, v30.8h, v17.8h\n" - "fmin v31.8h, v31.8h, v17.8h\n" + "fmax v24.8h, v24.8h, v15.8h\n" + "fmax v25.8h, v25.8h, v15.8h\n" + "fmax v26.8h, v26.8h, v15.8h\n" + "fmax v27.8h, v27.8h, v15.8h\n" + "fmax v28.8h, v28.8h, v15.8h\n" + "fmax v29.8h, v29.8h, v15.8h\n" + "fmax v30.8h, v30.8h, v15.8h\n" + "fmax v31.8h, v31.8h, v15.8h\n" + "fmin v23.8h, v23.8h, v14.8h\n" + "fmin v24.8h, v24.8h, v14.8h\n" + "fmin v25.8h, v25.8h, v14.8h\n" + "fmin v26.8h, v26.8h, v14.8h\n" + "fmin v27.8h, v27.8h, v14.8h\n" + "fmin v28.8h, v28.8h, v14.8h\n" + "fmin v29.8h, v29.8h, v14.8h\n" + "fmin v30.8h, v30.8h, v14.8h\n" + "fmin v31.8h, v31.8h, v14.8h\n" "tbz %x[n_channels], #2, 90f\n" "mov x22, x15\n" "mov x21, x28\n" @@ -1134,7 +1134,6 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "st1 { v28.h }[0], [x21]\n" "st1 { v31.h }[0], [x20]\n" "92:" // Tile loop: Oddments: Store: Bit 2: End - "93:" // Tile loop: End "ldr x23, [%x[params_struct], %[offsetof_args_tile_j]]\n" "ldr x24, [%x[params_struct], %[offsetof_args_tile_i]]\n" @@ -1149,7 +1148,7 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v16", "v17", "v18", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp index 878aa29bcf7320db1fd34ee42f2e01140be07f18..72e68482c663cbeb858bd62e1c9010d83e17368b 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp @@ -87,405 +87,405 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( activation_min, activation_max); __asm__ __volatile__( - "mov x8, #0x10\n" // cntb _, ALL, #1 - "lsr x17, %x[n_channels], #0x3\n" - "ldr x16, [%x[params_struct], %[offsetof_args_outptrs]]\n" - "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" + "mov x7, #0x10\n" // cntb _, ALL, #1 + "lsr x8, %x[n_channels], #0x3\n" + "ldr x17, [%x[params_struct], %[offsetof_args_outptrs]]\n" + "ldr x16, [%x[params_struct], %[offsetof_args_params]]\n" "add x20, %x[params_struct], %[offsetof_args_min]\n" - "ld1r { v18.8h }, [x20]\n" + "ld1r { v15.8h }, [x20]\n" "add x20, %x[params_struct], %[offsetof_args_max]\n" - "ld1r { v17.8h }, [x20]\n" - "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n" - "mov x13, #0x0\n" - "sub x12, XZR, x8\n" - "cbz x17, 3f\n" - "ldr q16, [x15, #0x0]\n" - "ldr q0, [x15, #0x10]\n" - "cmp x8, x17, LSL #4\n" - "ldr q1, [x15, #0x20]\n" - "ldr q2, [x15, #0x30]\n" - "ldr q3, [x15, #0x40]\n" - "ldr q4, [x15, #0x50]\n" - "ldr q5, [x15, #0x60]\n" - "ldr q6, [x15, #0x70]\n" - "ldr q7, [x15, #0x80]\n" - "ldr q8, [x15, #0x90]\n" - "add x15, x15, #0xa0\n" - "ldp x11, x10, [x14, #0x0]\n" - "ldr q9, [x11, x13]\n" - "ldr q10, [x10, x13]\n" - "ldp x9, x28, [x14, #0x10]\n" - "ldr q11, [x9, x13]\n" - "ldr q12, [x28, x13]\n" - "ldr x27, [x14, #0x20]\n" - "ldr q13, [x27, x13]\n" + "ld1r { v14.8h }, [x20]\n" + "add x15, %x[params_struct], %[offsetof_Args_inptrs]\n" + "mov x14, #0x0\n" + "sub x13, XZR, x7\n" + "cbz x8, 3f\n" + "ldr q31, [x16, #0x0]\n" + "ldr q0, [x16, #0x10]\n" + "cmp x7, x8, LSL #4\n" + "ldr q1, [x16, #0x20]\n" + "ldr q2, [x16, #0x30]\n" + "ldr q3, [x16, #0x40]\n" + "ldr q4, [x16, #0x50]\n" + "ldr q5, [x16, #0x60]\n" + "ldr q6, [x16, #0x70]\n" + "ldr q7, [x16, #0x80]\n" + "ldr q8, [x16, #0x90]\n" + "add x16, x16, #0xa0\n" + "ldp x21, x20, [x15, #0x0]\n" + "ldr q9, [x21, x14]\n" + "ldr q10, [x20, x14]\n" + "ldp x21, x20, [x15, #0x10]\n" + "ldr q11, [x21, x14]\n" + "ldr q12, [x20, x14]\n" + "ldr x20, [x15, #0x20]\n" + "ldr q13, [x20, x14]\n" "bge 2f\n" "1:" // Channel loop - "mov v23.16b, v16.16b\n fmla v23.8h, v8.8h, v9.8h\n" - "mov v24.16b, v16.16b\n fmla v24.8h, v7.8h, v9.8h\n" - "ldr x26, [x14, #0x30]\n" - "ldr x25, [x14, #0x38]\n" - "mov v25.16b, v16.16b\n fmla v25.8h, v6.8h, v9.8h\n" - "fmla v23.8h, v0.8h, v10.8h\n" - "ldr x24, [x14, #0x28]\n" - "ldr x10, [x14, #0x48]\n" - "ldr q10, [x10, x13]\n" - "fmla v24.8h, v4.8h, v13.8h\n" - "mov v26.16b, v16.16b\n fmla v26.8h, v5.8h, v9.8h\n" - "ldr x11, [x14, #0x40]\n" - "mov v27.16b, v16.16b\n fmla v27.8h, v4.8h, v9.8h\n" - "mov v28.16b, v16.16b\n fmla v28.8h, v3.8h, v9.8h\n" - "ldr x9, [x14, #0x50]\n" - "ldr x28, [x14, #0x58]\n" - "fmla v25.8h, v2.8h, v11.8h\n" - "ldr q11, [x26, x13]\n" - "mov v29.16b, v16.16b\n fmla v29.8h, v2.8h, v9.8h\n" - "ldr x27, [x14, #0x60]\n" - "fmla v23.8h, v5.8h, v13.8h\n" - "fmla v24.8h, v6.8h, v11.8h\n" - "ldr x26, [x14, #0x70]\n" - "ldr x10, [x14, #0x88]\n" - "mov v31.16b, v16.16b\n fmla v31.8h, v0.8h, v9.8h\n" - "fmla v25.8h, v3.8h, v13.8h\n" - "ldr x23, [x16, #0x0]\n" - "add x12, x12, #0x10\n" + "mov v29.16b, v31.16b\n fmla v29.8h, v8.8h, v9.8h\n" + "mov v28.16b, v31.16b\n fmla v28.8h, v7.8h, v9.8h\n" + "ldr x26, [x15, #0x30]\n" + "ldr x23, [x15, #0x38]\n" + "mov v27.16b, v31.16b\n fmla v27.8h, v6.8h, v9.8h\n" + "fmla v29.8h, v0.8h, v10.8h\n" + "ldr x22, [x15, #0x28]\n" + "ldr x20, [x15, #0x48]\n" + "ldr q19, [x20, x14]\n" + "fmla v28.8h, v4.8h, v13.8h\n" + "mov v26.16b, v31.16b\n fmla v26.8h, v5.8h, v9.8h\n" + "ldr x21, [x15, #0x40]\n" + "mov v25.16b, v31.16b\n fmla v25.8h, v4.8h, v9.8h\n" + "mov v24.16b, v31.16b\n fmla v24.8h, v3.8h, v9.8h\n" + "ldr x25, [x15, #0x50]\n" + "ldr x24, [x15, #0x58]\n" + "fmla v27.8h, v2.8h, v11.8h\n" + "ldr q17, [x26, x14]\n" + "mov v23.16b, v31.16b\n fmla v23.8h, v2.8h, v9.8h\n" + "ldr x20, [x15, #0x60]\n" + "fmla v29.8h, v5.8h, v13.8h\n" + "fmla v28.8h, v6.8h, v17.8h\n" + "ldr x12, [x15, #0x70]\n" + "ldr x11, [x15, #0x88]\n" + "mov v22.16b, v31.16b\n fmla v22.8h, v0.8h, v9.8h\n" + "fmla v27.8h, v3.8h, v13.8h\n" + "ldr x10, [x17, #0x0]\n" + "add x13, x13, #0x10\n" "fmla v26.8h, v2.8h, v13.8h\n" - "fmla v27.8h, v1.8h, v13.8h\n" - "ldr x22, [x16, #0x8]\n" - "ldr x21, [x16, #0x10]\n" - "fmla v28.8h, v0.8h, v13.8h\n" - "ldr q13, [x25, x13]\n" - "fmla v29.8h, v6.8h, v12.8h\n" - "ldr q12, [x24, x13]\n" - "mov v30.16b, v16.16b\n fmla v30.8h, v1.8h, v9.8h\n" - "ldr q16, [x15, #0x0]\n" - "fmla v23.8h, v7.8h, v11.8h\n" - "ldr x24, [x14, #0x68]\n" + "fmla v25.8h, v1.8h, v13.8h\n" + "ldr x9, [x17, #0x8]\n" + "ldr x28, [x17, #0x10]\n" "fmla v24.8h, v0.8h, v13.8h\n" - "fmla v31.8h, v8.8h, v12.8h\n" - "ldr q12, [x11, x13]\n" - "ldr x25, [x14, #0x78]\n" - "fmla v26.8h, v4.8h, v11.8h\n" - "fmla v27.8h, v3.8h, v11.8h\n" - "ldr x11, [x14, #0x80]\n" - "ldr x20, [x16, #0x18]\n" - "fmla v30.8h, v0.8h, v11.8h\n" - "fmla v28.8h, v4.8h, v10.8h\n" - "fmla v29.8h, v1.8h, v11.8h\n" - "ldr q11, [x9, x13]\n" - "fmla v23.8h, v1.8h, v13.8h\n" - "ldr q13, [x28, x13]\n" - "fmla v24.8h, v2.8h, v12.8h\n" - "fmla v25.8h, v1.8h, v12.8h\n" - "ldr q12, [x27, x13]\n" - "ldr x9, [x14, #0x90]\n" - "fmla v27.8h, v5.8h, v10.8h\n" - "fmla v30.8h, v2.8h, v10.8h\n" - "ldr x27, [x14, #0xa0]\n" - "ldr x28, [x14, #0x98]\n" - "fmla v26.8h, v0.8h, v11.8h\n" - "fmla v28.8h, v2.8h, v13.8h\n" - "fmla v24.8h, v8.8h, v10.8h\n" - "fmla v25.8h, v7.8h, v10.8h\n" - "fmla v31.8h, v1.8h, v10.8h\n" - "ldr q10, [x24, x13]\n" - "fmla v29.8h, v3.8h, v12.8h\n" - "ldr x24, [x14, #0xa8]\n" - "fmla v26.8h, v6.8h, v12.8h\n" - "ldr q12, [x11, x13]\n" - "fmla v27.8h, v7.8h, v10.8h\n" - "ldr x11, [x14, #0xc0]\n" - "fmla v28.8h, v6.8h, v10.8h\n" - "fmla v30.8h, v4.8h, v10.8h\n" - "fmla v23.8h, v3.8h, v11.8h\n" - "ldr q11, [x26, x13]\n" - "fmla v25.8h, v5.8h, v13.8h\n" - "ldr q13, [x25, x13]\n" - "fmla v29.8h, v5.8h, v10.8h\n" - "fmla v31.8h, v3.8h, v10.8h\n" - "ldr x26, [x14, #0xb0]\n" - "ldr x25, [x14, #0xb8]\n" - "fmla v26.8h, v8.8h, v10.8h\n" - "fmla v28.8h, v8.8h, v11.8h\n" - "fmla v30.8h, v6.8h, v13.8h\n" - "fmla v24.8h, v3.8h, v12.8h\n" - "fmla v27.8h, v0.8h, v12.8h\n" - "fmla v31.8h, v5.8h, v11.8h\n" - "ldr q11, [x10, x13]\n" - "fmla v29.8h, v7.8h, v13.8h\n" - "ldr q13, [x9, x13]\n" - "fmla v23.8h, v4.8h, v12.8h\n" - "fmla v26.8h, v1.8h, v12.8h\n" - "ldr q12, [x28, x13]\n" - "fmla v24.8h, v5.8h, v11.8h\n" - "fmla v25.8h, v4.8h, v11.8h\n" - "fmla v27.8h, v2.8h, v11.8h\n" - "fmla v28.8h, v1.8h, v11.8h\n" - "ldr q11, [x27, x13]\n" - "fmla v30.8h, v8.8h, v13.8h\n" - "ldr x27, [x14, #0x20]\n" - "fmla v31.8h, v7.8h, v13.8h\n" - "ldr q13, [x24, x13]\n" - "fmla v23.8h, v2.8h, v11.8h\n" - "fmla v26.8h, v7.8h, v12.8h\n" - "fmla v27.8h, v6.8h, v12.8h\n" - "fmla v29.8h, v4.8h, v12.8h\n" - "fmla v30.8h, v3.8h, v12.8h\n" - "ldr q12, [x26, x13]\n" - "fmla v31.8h, v4.8h, v13.8h\n" - "ldr q4, [x15, #0x50]\n" - "fmla v24.8h, v1.8h, v11.8h\n" - "ldr q1, [x15, #0x20]\n" - "fmla v25.8h, v0.8h, v11.8h\n" - "ldr q11, [x25, x13]\n" + "ldr q18, [x23, x14]\n" "fmla v23.8h, v6.8h, v12.8h\n" - "fmax v23.8h, v23.8h, v18.8h\n" - "fmla v28.8h, v7.8h, v13.8h\n" - "fmla v30.8h, v5.8h, v13.8h\n" - "fmin v23.8h, v23.8h, v17.8h\n" - "str q23, [x23, x12]\n" - "fmla v29.8h, v0.8h, v12.8h\n" - "ldr q0, [x15, #0x10]\n" - "fmla v31.8h, v2.8h, v11.8h\n" - "ldr q2, [x15, #0x30]\n" - "fmla v27.8h, v8.8h, v13.8h\n" - "ldr q13, [x11, x13]\n" - "fmla v26.8h, v3.8h, v12.8h\n" - "ldr q3, [x15, #0x40]\n" - "fmla v25.8h, v8.8h, v11.8h\n" - "fmla v28.8h, v5.8h, v11.8h\n" - "ldr q5, [x15, #0x60]\n" - "fmax v24.8h, v24.8h, v18.8h\n" - "fmla v29.8h, v8.8h, v13.8h\n" - "ldr q8, [x15, #0x90]\n" - "fmla v30.8h, v7.8h, v13.8h\n" - "ldr q7, [x15, #0x80]\n" - "fmla v31.8h, v6.8h, v13.8h\n" - "ldr q13, [x27, x8]\n" - "ldr q6, [x15, #0x70]\n" - "fmax v25.8h, v25.8h, v18.8h\n" - "fmax v26.8h, v26.8h, v18.8h\n" - "fmax v27.8h, v27.8h, v18.8h\n" - "ldr x23, [x16, #0x20]\n" - "ldp x11, x10, [x14, #0x0]\n" - "ldr q9, [x11, x8]\n" - "ldr q10, [x10, x8]\n" - "fmin v24.8h, v24.8h, v17.8h\n" - "fmin v25.8h, v25.8h, v17.8h\n" - "ldp x9, x28, [x14, #0x10]\n" - "ldr q11, [x9, x8]\n" - "fmin v26.8h, v26.8h, v17.8h\n" - "fmin v27.8h, v27.8h, v17.8h\n" - "ldr q12, [x28, x8]\n" - "fmax v28.8h, v28.8h, v18.8h\n" - "fmax v29.8h, v29.8h, v18.8h\n" - "str q24, [x22, x12]\n" - "fmax v30.8h, v30.8h, v18.8h\n" - "fmax v31.8h, v31.8h, v18.8h\n" - "str q25, [x21, x12]\n" - "ldr x22, [x16, #0x28]\n" - "str q26, [x20, x12]\n" - "ldr x21, [x16, #0x30]\n" - "ldr x20, [x16, #0x38]\n" - "add x8, x8, #0x10\n" - "str q27, [x23, x12]\n" - "ldr x23, [x16, #0x40]\n" - "cmp x8, x17, LSL #4\n" - "fmin v28.8h, v28.8h, v17.8h\n" - "fmin v29.8h, v29.8h, v17.8h\n" - "fmin v30.8h, v30.8h, v17.8h\n" - "add x13, x13, #0x10\n" - "str q28, [x22, x12]\n" - "fmin v31.8h, v31.8h, v17.8h\n" - "str q29, [x21, x12]\n" - "add x15, x15, #0xa0\n" - "str q30, [x20, x12]\n" - "str q31, [x23, x12]\n" + "ldr q16, [x22, x14]\n" + "mov v21.16b, v31.16b\n fmla v21.8h, v1.8h, v9.8h\n" + "ldr q31, [x16, #0x0]\n" + "fmla v29.8h, v7.8h, v17.8h\n" + "ldr x23, [x15, #0x68]\n" + "fmla v28.8h, v0.8h, v18.8h\n" + "fmla v22.8h, v8.8h, v16.8h\n" + "ldr q16, [x21, x14]\n" + "ldr x22, [x15, #0x78]\n" + "fmla v26.8h, v4.8h, v17.8h\n" + "fmla v25.8h, v3.8h, v17.8h\n" + "ldr x21, [x15, #0x80]\n" + "ldr x27, [x17, #0x18]\n" + "fmla v21.8h, v0.8h, v17.8h\n" + "fmla v24.8h, v4.8h, v19.8h\n" + "fmla v23.8h, v1.8h, v17.8h\n" + "ldr q17, [x25, x14]\n" + "fmla v29.8h, v1.8h, v18.8h\n" + "ldr q20, [x24, x14]\n" + "fmla v28.8h, v2.8h, v16.8h\n" + "fmla v27.8h, v1.8h, v16.8h\n" + "ldr q16, [x20, x14]\n" + "ldr x26, [x15, #0x90]\n" + "fmla v25.8h, v5.8h, v19.8h\n" + "fmla v21.8h, v2.8h, v19.8h\n" + "ldr x25, [x15, #0xa0]\n" + "ldr x20, [x15, #0x98]\n" + "fmla v26.8h, v0.8h, v17.8h\n" + "fmla v24.8h, v2.8h, v20.8h\n" + "fmla v28.8h, v8.8h, v19.8h\n" + "fmla v27.8h, v7.8h, v19.8h\n" + "fmla v22.8h, v1.8h, v19.8h\n" + "ldr q19, [x23, x14]\n" + "fmla v23.8h, v3.8h, v16.8h\n" + "ldr x24, [x15, #0xa8]\n" + "fmla v26.8h, v6.8h, v16.8h\n" + "ldr q18, [x21, x14]\n" + "fmla v25.8h, v7.8h, v19.8h\n" + "ldr x23, [x15, #0xc0]\n" + "fmla v24.8h, v6.8h, v19.8h\n" + "fmla v21.8h, v4.8h, v19.8h\n" + "fmla v29.8h, v3.8h, v17.8h\n" + "ldr q17, [x12, x14]\n" + "fmla v27.8h, v5.8h, v20.8h\n" + "ldr q16, [x22, x14]\n" + "fmla v23.8h, v5.8h, v19.8h\n" + "fmla v22.8h, v3.8h, v19.8h\n" + "ldr x22, [x15, #0xb0]\n" + "ldr x21, [x15, #0xb8]\n" + "fmla v26.8h, v8.8h, v19.8h\n" + "fmla v24.8h, v8.8h, v17.8h\n" + "fmla v21.8h, v6.8h, v16.8h\n" + "fmla v28.8h, v3.8h, v18.8h\n" + "fmla v25.8h, v0.8h, v18.8h\n" + "fmla v22.8h, v5.8h, v17.8h\n" + "ldr q17, [x11, x14]\n" + "fmla v23.8h, v7.8h, v16.8h\n" + "ldr q16, [x26, x14]\n" + "fmla v29.8h, v4.8h, v18.8h\n" + "fmla v26.8h, v1.8h, v18.8h\n" + "ldr q18, [x20, x14]\n" + "fmla v28.8h, v5.8h, v17.8h\n" + "fmla v27.8h, v4.8h, v17.8h\n" + "fmla v25.8h, v2.8h, v17.8h\n" + "fmla v24.8h, v1.8h, v17.8h\n" + "ldr q17, [x25, x14]\n" + "fmla v21.8h, v8.8h, v16.8h\n" + "ldr x20, [x15, #0x20]\n" + "fmla v22.8h, v7.8h, v16.8h\n" + "ldr q16, [x24, x14]\n" + "fmla v29.8h, v2.8h, v17.8h\n" + "fmla v26.8h, v7.8h, v18.8h\n" + "fmla v25.8h, v6.8h, v18.8h\n" + "fmla v23.8h, v4.8h, v18.8h\n" + "fmla v21.8h, v3.8h, v18.8h\n" + "ldr q18, [x22, x14]\n" + "fmla v22.8h, v4.8h, v16.8h\n" + "ldr q4, [x16, #0x50]\n" + "fmla v28.8h, v1.8h, v17.8h\n" + "ldr q1, [x16, #0x20]\n" + "fmla v27.8h, v0.8h, v17.8h\n" + "ldr q17, [x21, x14]\n" + "fmla v29.8h, v6.8h, v18.8h\n" + "fmax v29.8h, v29.8h, v15.8h\n" + "fmla v24.8h, v7.8h, v16.8h\n" + "fmla v21.8h, v5.8h, v16.8h\n" + "fmin v29.8h, v29.8h, v14.8h\n" + "str q29, [x10, x13]\n" + "fmla v23.8h, v0.8h, v18.8h\n" + "ldr q0, [x16, #0x10]\n" + "fmla v22.8h, v2.8h, v17.8h\n" + "ldr q2, [x16, #0x30]\n" + "fmla v25.8h, v8.8h, v16.8h\n" + "ldr q16, [x23, x14]\n" + "fmla v26.8h, v3.8h, v18.8h\n" + "ldr q3, [x16, #0x40]\n" + "fmla v27.8h, v8.8h, v17.8h\n" + "fmla v24.8h, v5.8h, v17.8h\n" + "ldr q5, [x16, #0x60]\n" + "fmax v28.8h, v28.8h, v15.8h\n" + "fmla v23.8h, v8.8h, v16.8h\n" + "ldr q8, [x16, #0x90]\n" + "fmla v21.8h, v7.8h, v16.8h\n" + "ldr q7, [x16, #0x80]\n" + "fmla v22.8h, v6.8h, v16.8h\n" + "ldr q13, [x20, x7]\n" + "ldr q6, [x16, #0x70]\n" + "fmax v27.8h, v27.8h, v15.8h\n" + "fmax v26.8h, v26.8h, v15.8h\n" + "fmax v25.8h, v25.8h, v15.8h\n" + "ldr x24, [x17, #0x20]\n" + "ldp x21, x20, [x15, #0x0]\n" + "ldr q9, [x21, x7]\n" + "ldr q10, [x20, x7]\n" + "fmin v28.8h, v28.8h, v14.8h\n" + "fmin v27.8h, v27.8h, v14.8h\n" + "ldp x21, x20, [x15, #0x10]\n" + "ldr q11, [x21, x7]\n" + "fmin v26.8h, v26.8h, v14.8h\n" + "fmin v25.8h, v25.8h, v14.8h\n" + "ldr q12, [x20, x7]\n" + "fmax v24.8h, v24.8h, v15.8h\n" + "fmax v23.8h, v23.8h, v15.8h\n" + "str q28, [x9, x13]\n" + "fmax v21.8h, v21.8h, v15.8h\n" + "fmax v22.8h, v22.8h, v15.8h\n" + "str q27, [x28, x13]\n" + "ldr x23, [x17, #0x28]\n" + "str q26, [x27, x13]\n" + "ldr x22, [x17, #0x30]\n" + "ldr x21, [x17, #0x38]\n" + "add x7, x7, #0x10\n" + "str q25, [x24, x13]\n" + "ldr x20, [x17, #0x40]\n" + "cmp x7, x8, LSL #4\n" + "fmin v24.8h, v24.8h, v14.8h\n" + "fmin v23.8h, v23.8h, v14.8h\n" + "fmin v21.8h, v21.8h, v14.8h\n" + "add x14, x14, #0x10\n" + "str q24, [x23, x13]\n" + "fmin v22.8h, v22.8h, v14.8h\n" + "str q23, [x22, x13]\n" + "add x16, x16, #0xa0\n" + "str q21, [x21, x13]\n" + "str q22, [x20, x13]\n" "blt 1b\n" "2:" // Channel tail - "mov v23.16b, v16.16b\n fmla v23.8h, v8.8h, v9.8h\n" - "mov v24.16b, v16.16b\n fmla v24.8h, v7.8h, v9.8h\n" - "ldr x26, [x14, #0x30]\n" - "ldr x25, [x14, #0x38]\n" - "mov v25.16b, v16.16b\n fmla v25.8h, v6.8h, v9.8h\n" - "fmla v23.8h, v0.8h, v10.8h\n" - "ldr x24, [x14, #0x28]\n" - "ldr x10, [x14, #0x48]\n" - "ldr q10, [x10, x13]\n" - "fmla v24.8h, v4.8h, v13.8h\n" - "mov v26.16b, v16.16b\n fmla v26.8h, v5.8h, v9.8h\n" - "ldr x11, [x14, #0x40]\n" - "mov v27.16b, v16.16b\n fmla v27.8h, v4.8h, v9.8h\n" - "mov v28.16b, v16.16b\n fmla v28.8h, v3.8h, v9.8h\n" - "ldr x9, [x14, #0x50]\n" - "ldr x28, [x14, #0x58]\n" - "fmla v25.8h, v2.8h, v11.8h\n" - "ldr q11, [x26, x13]\n" - "mov v29.16b, v16.16b\n fmla v29.8h, v2.8h, v9.8h\n" - "ldr x27, [x14, #0x60]\n" - "fmla v23.8h, v5.8h, v13.8h\n" - "fmla v24.8h, v6.8h, v11.8h\n" - "ldr x26, [x14, #0x70]\n" - "ldr x10, [x14, #0x88]\n" - "mov v31.16b, v16.16b\n fmla v31.8h, v0.8h, v9.8h\n" - "fmla v25.8h, v3.8h, v13.8h\n" - "ldr x23, [x16, #0x0]\n" - "add x12, x12, #0x10\n" + "mov v29.16b, v31.16b\n fmla v29.8h, v8.8h, v9.8h\n" + "mov v28.16b, v31.16b\n fmla v28.8h, v7.8h, v9.8h\n" + "ldr x23, [x15, #0x30]\n" + "ldr x22, [x15, #0x38]\n" + "mov v27.16b, v31.16b\n fmla v27.8h, v6.8h, v9.8h\n" + "fmla v29.8h, v0.8h, v10.8h\n" + "ldr x21, [x15, #0x28]\n" + "ldr x20, [x15, #0x48]\n" + "ldr q19, [x20, x14]\n" + "fmla v28.8h, v4.8h, v13.8h\n" + "mov v26.16b, v31.16b\n fmla v26.8h, v5.8h, v9.8h\n" + "ldr x20, [x15, #0x40]\n" + "mov v25.16b, v31.16b\n fmla v25.8h, v4.8h, v9.8h\n" + "mov v24.16b, v31.16b\n fmla v24.8h, v3.8h, v9.8h\n" + "ldr x25, [x15, #0x50]\n" + "ldr x24, [x15, #0x58]\n" + "fmla v27.8h, v2.8h, v11.8h\n" + "ldr q17, [x23, x14]\n" + "mov v23.16b, v31.16b\n fmla v23.8h, v2.8h, v9.8h\n" + "ldr x23, [x15, #0x60]\n" + "fmla v29.8h, v5.8h, v13.8h\n" + "fmla v28.8h, v6.8h, v17.8h\n" + "ldr x12, [x15, #0x70]\n" + "ldr x11, [x15, #0x88]\n" + "mov v22.16b, v31.16b\n fmla v22.8h, v0.8h, v9.8h\n" + "fmla v27.8h, v3.8h, v13.8h\n" + "ldr x10, [x17, #0x0]\n" + "add x13, x13, #0x10\n" "fmla v26.8h, v2.8h, v13.8h\n" - "fmla v27.8h, v1.8h, v13.8h\n" - "ldr x22, [x16, #0x8]\n" - "ldr x21, [x16, #0x10]\n" - "fmla v28.8h, v0.8h, v13.8h\n" - "ldr q13, [x25, x13]\n" - "fmla v29.8h, v6.8h, v12.8h\n" - "ldr q12, [x24, x13]\n" - "mov v30.16b, v16.16b\n fmla v30.8h, v1.8h, v9.8h\n" - "fmla v23.8h, v7.8h, v11.8h\n" - "ldr x24, [x14, #0x68]\n" - "ldr x25, [x14, #0x78]\n" + "fmla v25.8h, v1.8h, v13.8h\n" + "ldr x9, [x17, #0x8]\n" + "ldr x28, [x17, #0x10]\n" "fmla v24.8h, v0.8h, v13.8h\n" - "fmla v31.8h, v8.8h, v12.8h\n" - "ldr q12, [x11, x13]\n" - "ldr x11, [x14, #0x80]\n" - "fmla v26.8h, v4.8h, v11.8h\n" - "fmla v27.8h, v3.8h, v11.8h\n" - "ldr x20, [x16, #0x18]\n" - "fmla v30.8h, v0.8h, v11.8h\n" - "fmla v28.8h, v4.8h, v10.8h\n" - "fmla v29.8h, v1.8h, v11.8h\n" - "ldr q11, [x9, x13]\n" - "fmla v23.8h, v1.8h, v13.8h\n" - "ldr q13, [x28, x13]\n" - "fmla v24.8h, v2.8h, v12.8h\n" - "fmla v25.8h, v1.8h, v12.8h\n" - "ldr q12, [x27, x13]\n" - "ldr x9, [x14, #0x90]\n" - "fmla v27.8h, v5.8h, v10.8h\n" - "fmla v30.8h, v2.8h, v10.8h\n" - "ldr x27, [x14, #0xa0]\n" - "ldr x28, [x14, #0x98]\n" - "fmla v26.8h, v0.8h, v11.8h\n" - "fmla v28.8h, v2.8h, v13.8h\n" - "fmla v24.8h, v8.8h, v10.8h\n" - "fmla v25.8h, v7.8h, v10.8h\n" - "fmla v31.8h, v1.8h, v10.8h\n" - "ldr q10, [x24, x13]\n" - "fmla v29.8h, v3.8h, v12.8h\n" - "ldr x24, [x14, #0xa8]\n" - "fmla v26.8h, v6.8h, v12.8h\n" - "ldr q12, [x11, x13]\n" - "fmla v27.8h, v7.8h, v10.8h\n" - "ldr x11, [x14, #0xc0]\n" - "fmla v28.8h, v6.8h, v10.8h\n" - "fmla v30.8h, v4.8h, v10.8h\n" - "fmla v23.8h, v3.8h, v11.8h\n" - "ldr q11, [x26, x13]\n" - "fmla v25.8h, v5.8h, v13.8h\n" - "ldr q13, [x25, x13]\n" - "fmla v29.8h, v5.8h, v10.8h\n" - "fmla v31.8h, v3.8h, v10.8h\n" - "ldr x26, [x14, #0xb0]\n" - "ldr x25, [x14, #0xb8]\n" - "fmla v26.8h, v8.8h, v10.8h\n" - "fmla v28.8h, v8.8h, v11.8h\n" - "fmla v30.8h, v6.8h, v13.8h\n" - "fmla v24.8h, v3.8h, v12.8h\n" - "fmla v27.8h, v0.8h, v12.8h\n" - "fmla v31.8h, v5.8h, v11.8h\n" - "ldr q11, [x10, x13]\n" - "fmla v29.8h, v7.8h, v13.8h\n" - "ldr q13, [x9, x13]\n" - "fmla v23.8h, v4.8h, v12.8h\n" - "fmla v26.8h, v1.8h, v12.8h\n" - "ldr q12, [x28, x13]\n" - "fmla v24.8h, v5.8h, v11.8h\n" - "fmla v25.8h, v4.8h, v11.8h\n" - "fmla v27.8h, v2.8h, v11.8h\n" - "fmla v28.8h, v1.8h, v11.8h\n" - "ldr q11, [x27, x13]\n" - "fmla v30.8h, v8.8h, v13.8h\n" - "fmla v31.8h, v7.8h, v13.8h\n" - "ldr q13, [x24, x13]\n" - "fmla v23.8h, v2.8h, v11.8h\n" - "fmla v26.8h, v7.8h, v12.8h\n" - "fmla v27.8h, v6.8h, v12.8h\n" - "fmla v29.8h, v4.8h, v12.8h\n" - "fmla v30.8h, v3.8h, v12.8h\n" - "ldr q12, [x26, x13]\n" - "fmla v31.8h, v4.8h, v13.8h\n" - "fmla v24.8h, v1.8h, v11.8h\n" - "fmax v24.8h, v24.8h, v18.8h\n" - "fmla v25.8h, v0.8h, v11.8h\n" - "ldr q11, [x25, x13]\n" + "ldr q18, [x22, x14]\n" "fmla v23.8h, v6.8h, v12.8h\n" - "fmax v23.8h, v23.8h, v18.8h\n" - "fmla v28.8h, v7.8h, v13.8h\n" - "fmla v30.8h, v5.8h, v13.8h\n" - "fmin v23.8h, v23.8h, v17.8h\n" - "str q23, [x23, x12]\n" - "fmla v29.8h, v0.8h, v12.8h\n" - "fmla v31.8h, v2.8h, v11.8h\n" - "ldr x23, [x16, #0x20]\n" - "fmin v24.8h, v24.8h, v17.8h\n" - "fmla v27.8h, v8.8h, v13.8h\n" - "ldr q13, [x11, x13]\n" - "fmla v26.8h, v3.8h, v12.8h\n" - "fmax v26.8h, v26.8h, v18.8h\n" - "fmla v25.8h, v8.8h, v11.8h\n" - "fmla v28.8h, v5.8h, v11.8h\n" - "fmax v25.8h, v25.8h, v18.8h\n" - "str q24, [x22, x12]\n" - "fmla v29.8h, v8.8h, v13.8h\n" - "fmla v30.8h, v7.8h, v13.8h\n" - "fmax v27.8h, v27.8h, v18.8h\n" - "ldr x22, [x16, #0x28]\n" - "fmla v31.8h, v6.8h, v13.8h\n" - "fmin v25.8h, v25.8h, v17.8h\n" - "str q25, [x21, x12]\n" - "ldr x21, [x16, #0x30]\n" - "fmin v26.8h, v26.8h, v17.8h\n" - "fmin v27.8h, v27.8h, v17.8h\n" - "str q26, [x20, x12]\n" - "ldr x20, [x16, #0x38]\n" - "fmax v28.8h, v28.8h, v18.8h\n" - "fmax v29.8h, v29.8h, v18.8h\n" - "str q27, [x23, x12]\n" - "ldr x23, [x16, #0x40]\n" - "fmax v30.8h, v30.8h, v18.8h\n" - "fmax v31.8h, v31.8h, v18.8h\n" - "add x13, x13, #0x10\n" - "fmin v28.8h, v28.8h, v17.8h\n" - "fmin v29.8h, v29.8h, v17.8h\n" - "str q28, [x22, x12]\n" - "fmin v30.8h, v30.8h, v17.8h\n" - "fmin v31.8h, v31.8h, v17.8h\n" - "str q29, [x21, x12]\n" - "str q30, [x20, x12]\n" - "str q31, [x23, x12]\n" + "ldr q16, [x21, x14]\n" + "mov v21.16b, v31.16b\n fmla v21.8h, v1.8h, v9.8h\n" + "fmla v29.8h, v7.8h, v17.8h\n" + "ldr x22, [x15, #0x68]\n" + "ldr x21, [x15, #0x78]\n" + "fmla v28.8h, v0.8h, v18.8h\n" + "fmla v22.8h, v8.8h, v16.8h\n" + "ldr q16, [x20, x14]\n" + "ldr x20, [x15, #0x80]\n" + "fmla v26.8h, v4.8h, v17.8h\n" + "fmla v25.8h, v3.8h, v17.8h\n" + "ldr x27, [x17, #0x18]\n" + "fmla v21.8h, v0.8h, v17.8h\n" + "fmla v24.8h, v4.8h, v19.8h\n" + "fmla v23.8h, v1.8h, v17.8h\n" + "ldr q17, [x25, x14]\n" + "fmla v29.8h, v1.8h, v18.8h\n" + "ldr q20, [x24, x14]\n" + "fmla v28.8h, v2.8h, v16.8h\n" + "fmla v27.8h, v1.8h, v16.8h\n" + "ldr q16, [x23, x14]\n" + "ldr x26, [x15, #0x90]\n" + "fmla v25.8h, v5.8h, v19.8h\n" + "fmla v21.8h, v2.8h, v19.8h\n" + "ldr x25, [x15, #0xa0]\n" + "ldr x24, [x15, #0x98]\n" + "fmla v26.8h, v0.8h, v17.8h\n" + "fmla v24.8h, v2.8h, v20.8h\n" + "fmla v28.8h, v8.8h, v19.8h\n" + "fmla v27.8h, v7.8h, v19.8h\n" + "fmla v22.8h, v1.8h, v19.8h\n" + "ldr q19, [x22, x14]\n" + "fmla v23.8h, v3.8h, v16.8h\n" + "ldr x23, [x15, #0xa8]\n" + "fmla v26.8h, v6.8h, v16.8h\n" + "ldr q18, [x20, x14]\n" + "fmla v25.8h, v7.8h, v19.8h\n" + "ldr x22, [x15, #0xc0]\n" + "fmla v24.8h, v6.8h, v19.8h\n" + "fmla v21.8h, v4.8h, v19.8h\n" + "fmla v29.8h, v3.8h, v17.8h\n" + "ldr q17, [x12, x14]\n" + "fmla v27.8h, v5.8h, v20.8h\n" + "ldr q16, [x21, x14]\n" + "fmla v23.8h, v5.8h, v19.8h\n" + "fmla v22.8h, v3.8h, v19.8h\n" + "ldr x21, [x15, #0xb0]\n" + "ldr x20, [x15, #0xb8]\n" + "fmla v26.8h, v8.8h, v19.8h\n" + "fmla v24.8h, v8.8h, v17.8h\n" + "fmla v21.8h, v6.8h, v16.8h\n" + "fmla v28.8h, v3.8h, v18.8h\n" + "fmla v25.8h, v0.8h, v18.8h\n" + "fmla v22.8h, v5.8h, v17.8h\n" + "ldr q17, [x11, x14]\n" + "fmla v23.8h, v7.8h, v16.8h\n" + "ldr q16, [x26, x14]\n" + "fmla v29.8h, v4.8h, v18.8h\n" + "fmla v26.8h, v1.8h, v18.8h\n" + "ldr q18, [x24, x14]\n" + "fmla v28.8h, v5.8h, v17.8h\n" + "fmla v27.8h, v4.8h, v17.8h\n" + "fmla v25.8h, v2.8h, v17.8h\n" + "fmla v24.8h, v1.8h, v17.8h\n" + "ldr q17, [x25, x14]\n" + "fmla v21.8h, v8.8h, v16.8h\n" + "fmla v22.8h, v7.8h, v16.8h\n" + "ldr q16, [x23, x14]\n" + "fmla v29.8h, v2.8h, v17.8h\n" + "fmla v26.8h, v7.8h, v18.8h\n" + "fmla v25.8h, v6.8h, v18.8h\n" + "fmla v23.8h, v4.8h, v18.8h\n" + "fmla v21.8h, v3.8h, v18.8h\n" + "ldr q18, [x21, x14]\n" + "fmla v22.8h, v4.8h, v16.8h\n" + "fmla v28.8h, v1.8h, v17.8h\n" + "fmax v28.8h, v28.8h, v15.8h\n" + "fmla v27.8h, v0.8h, v17.8h\n" + "ldr q17, [x20, x14]\n" + "fmla v29.8h, v6.8h, v18.8h\n" + "fmax v29.8h, v29.8h, v15.8h\n" + "fmla v24.8h, v7.8h, v16.8h\n" + "fmla v21.8h, v5.8h, v16.8h\n" + "fmin v29.8h, v29.8h, v14.8h\n" + "str q29, [x10, x13]\n" + "fmla v23.8h, v0.8h, v18.8h\n" + "fmla v22.8h, v2.8h, v17.8h\n" + "ldr x20, [x17, #0x20]\n" + "fmin v28.8h, v28.8h, v14.8h\n" + "fmla v25.8h, v8.8h, v16.8h\n" + "ldr q16, [x22, x14]\n" + "fmla v26.8h, v3.8h, v18.8h\n" + "fmax v26.8h, v26.8h, v15.8h\n" + "fmla v27.8h, v8.8h, v17.8h\n" + "fmla v24.8h, v5.8h, v17.8h\n" + "fmax v27.8h, v27.8h, v15.8h\n" + "str q28, [x9, x13]\n" + "fmla v23.8h, v8.8h, v16.8h\n" + "fmla v21.8h, v7.8h, v16.8h\n" + "fmax v25.8h, v25.8h, v15.8h\n" + "ldr x23, [x17, #0x28]\n" + "fmla v22.8h, v6.8h, v16.8h\n" + "fmin v27.8h, v27.8h, v14.8h\n" + "str q27, [x28, x13]\n" + "ldr x22, [x17, #0x30]\n" + "fmin v26.8h, v26.8h, v14.8h\n" + "fmin v25.8h, v25.8h, v14.8h\n" + "str q26, [x27, x13]\n" + "ldr x21, [x17, #0x38]\n" + "fmax v24.8h, v24.8h, v15.8h\n" + "fmax v23.8h, v23.8h, v15.8h\n" + "str q25, [x20, x13]\n" + "ldr x20, [x17, #0x40]\n" + "fmax v21.8h, v21.8h, v15.8h\n" + "fmax v22.8h, v22.8h, v15.8h\n" + "add x14, x14, #0x10\n" + "fmin v24.8h, v24.8h, v14.8h\n" + "fmin v23.8h, v23.8h, v14.8h\n" + "str q24, [x23, x13]\n" + "fmin v21.8h, v21.8h, v14.8h\n" + "fmin v22.8h, v22.8h, v14.8h\n" + "str q23, [x22, x13]\n" + "str q21, [x21, x13]\n" + "str q22, [x20, x13]\n" "3:" // Oddments "tst %x[n_channels], #0x7\n" "beq 92f\n" - "ldr q16, [x15, #0x0]\n" - "ldr q0, [x15, #0x10]\n" - "mov x12, x13\n" - "ldr q1, [x15, #0x20]\n" - "ldr q2, [x15, #0x30]\n" - "ldr q3, [x15, #0x40]\n" - "ldr q4, [x15, #0x50]\n" - "ldr q5, [x15, #0x60]\n" - "ldr q6, [x15, #0x70]\n" - "ldr q7, [x15, #0x80]\n" - "ldr q8, [x15, #0x90]\n" - "ldr x24, [x14, #0x0]\n" - "ldr x23, [x14, #0x8]\n" - "add x24, x24, x13\n" - "add x23, x23, x13\n" - "ldr x22, [x14, #0x10]\n" - "ldr x21, [x14, #0x18]\n" - "add x22, x22, x13\n" - "add x21, x21, x13\n" - "ldr x20, [x14, #0x20]\n" - "add x20, x20, x13\n" + "ldr q31, [x16, #0x0]\n" + "ldr q0, [x16, #0x10]\n" + "mov x13, x14\n" + "ldr q1, [x16, #0x20]\n" + "ldr q2, [x16, #0x30]\n" + "ldr q3, [x16, #0x40]\n" + "ldr q4, [x16, #0x50]\n" + "ldr q5, [x16, #0x60]\n" + "ldr q6, [x16, #0x70]\n" + "ldr q7, [x16, #0x80]\n" + "ldr q8, [x16, #0x90]\n" + "ldr x24, [x15, #0x0]\n" + "ldr x23, [x15, #0x8]\n" + "add x24, x24, x14\n" + "add x23, x23, x14\n" + "ldr x22, [x15, #0x10]\n" + "ldr x21, [x15, #0x18]\n" + "add x22, x22, x14\n" + "add x21, x21, x14\n" + "ldr x20, [x15, #0x20]\n" + "add x20, x20, x14\n" "tbz %x[n_channels], #2, 5f\n" "ld1 { v9.d }[0], [x24], #0x8\n" "ld1 { v10.d }[0], [x23], #0x8\n" @@ -534,19 +534,19 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "ld1 { v12.h }[0], [x21], #0x2\n" "ld1 { v13.h }[0], [x20], #0x2\n" "7:" // Oddments: Load inputs (2, 2), (0, 0), (0, 4), (4, 0), (1, 2): Bit 2: End - "mov v23.16b, v16.16b\n fmla v23.8h, v8.8h, v9.8h\n" - "mov v25.16b, v16.16b\n fmla v25.8h, v6.8h, v9.8h\n" - "ldr x20, [x14, #0x28]\n" - "add x20, x20, x13\n" - "mov v24.16b, v16.16b\n fmla v24.8h, v7.8h, v9.8h\n" - "mov v26.16b, v16.16b\n fmla v26.8h, v5.8h, v9.8h\n" - "mov v27.16b, v16.16b\n fmla v27.8h, v4.8h, v9.8h\n" - "mov v28.16b, v16.16b\n fmla v28.8h, v3.8h, v9.8h\n" - "mov v29.16b, v16.16b\n fmla v29.8h, v2.8h, v9.8h\n" + "mov v23.16b, v31.16b\n fmla v23.8h, v8.8h, v9.8h\n" + "mov v25.16b, v31.16b\n fmla v25.8h, v6.8h, v9.8h\n" + "ldr x20, [x15, #0x28]\n" + "add x20, x20, x14\n" + "mov v24.16b, v31.16b\n fmla v24.8h, v7.8h, v9.8h\n" + "mov v26.16b, v31.16b\n fmla v26.8h, v5.8h, v9.8h\n" + "mov v27.16b, v31.16b\n fmla v27.8h, v4.8h, v9.8h\n" + "mov v28.16b, v31.16b\n fmla v28.8h, v3.8h, v9.8h\n" + "mov v29.16b, v31.16b\n fmla v29.8h, v2.8h, v9.8h\n" "fmla v23.8h, v0.8h, v10.8h\n" "fmla v25.8h, v2.8h, v11.8h\n" - "mov v30.16b, v16.16b\n fmla v30.8h, v1.8h, v9.8h\n" - "mov v31.16b, v16.16b\n fmla v31.8h, v0.8h, v9.8h\n" + "mov v30.16b, v31.16b\n fmla v30.8h, v1.8h, v9.8h\n" + "fmla v31.8h, v0.8h, v9.8h\n" "fmla v29.8h, v6.8h, v12.8h\n" "fmla v23.8h, v5.8h, v13.8h\n" "fmla v24.8h, v4.8h, v13.8h\n" @@ -574,9 +574,9 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "10:" // Oddments: Load input (4, 4): Bit 2: Unset: Bit 1: Unset "ld1 { v12.h }[0], [x20], #0x2\n" "11:" // Oddments: Load input (4, 4): Bit 2: End - "ldr x20, [x14, #0x30]\n" + "ldr x20, [x15, #0x30]\n" "fmla v31.8h, v8.8h, v12.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "tbz %x[n_channels], #2, 13f\n" "ld1 { v11.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #1, 12f\n" @@ -597,10 +597,10 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "14:" // Oddments: Load input (2, 1): Bit 2: Unset: Bit 1: Unset "ld1 { v11.h }[0], [x20], #0x2\n" "15:" // Oddments: Load input (2, 1): Bit 2: End - "ldr x20, [x14, #0x38]\n" + "ldr x20, [x15, #0x38]\n" "fmla v23.8h, v7.8h, v11.8h\n" "fmla v24.8h, v6.8h, v11.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "fmla v26.8h, v4.8h, v11.8h\n" "fmla v27.8h, v3.8h, v11.8h\n" "fmla v29.8h, v1.8h, v11.8h\n" @@ -625,10 +625,10 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "18:" // Oddments: Load input (0, 1): Bit 2: Unset: Bit 1: Unset "ld1 { v13.h }[0], [x20], #0x2\n" "19:" // Oddments: Load input (0, 1): Bit 2: End - "ldr x20, [x14, #0x40]\n" + "ldr x20, [x15, #0x40]\n" "fmla v23.8h, v1.8h, v13.8h\n" "fmla v24.8h, v0.8h, v13.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "tbz %x[n_channels], #2, 21f\n" "ld1 { v12.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #1, 20f\n" @@ -649,10 +649,10 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "22:" // Oddments: Load input (0, 3): Bit 2: Unset: Bit 1: Unset "ld1 { v12.h }[0], [x20], #0x2\n" "23:" // Oddments: Load input (0, 3): Bit 2: End - "ldr x20, [x14, #0x48]\n" + "ldr x20, [x15, #0x48]\n" "fmla v24.8h, v2.8h, v12.8h\n" "fmla v25.8h, v1.8h, v12.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "tbz %x[n_channels], #2, 25f\n" "ld1 { v10.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #1, 24f\n" @@ -673,10 +673,10 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "26:" // Oddments: Load input (2, 3): Bit 2: Unset: Bit 1: Unset "ld1 { v10.h }[0], [x20], #0x2\n" "27:" // Oddments: Load input (2, 3): Bit 2: End - "ldr x20, [x14, #0x50]\n" + "ldr x20, [x15, #0x50]\n" "fmla v24.8h, v8.8h, v10.8h\n" "fmla v25.8h, v7.8h, v10.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "fmla v27.8h, v5.8h, v10.8h\n" "fmla v28.8h, v4.8h, v10.8h\n" "fmla v30.8h, v2.8h, v10.8h\n" @@ -701,10 +701,10 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "30:" // Oddments: Load input (1, 0): Bit 2: Unset: Bit 1: Unset "ld1 { v11.h }[0], [x20], #0x2\n" "31:" // Oddments: Load input (1, 0): Bit 2: End - "ldr x20, [x14, #0x58]\n" + "ldr x20, [x15, #0x58]\n" "fmla v23.8h, v3.8h, v11.8h\n" "fmla v26.8h, v0.8h, v11.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "tbz %x[n_channels], #2, 33f\n" "ld1 { v13.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #1, 32f\n" @@ -725,10 +725,10 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "34:" // Oddments: Load input (1, 4): Bit 2: Unset: Bit 1: Unset "ld1 { v13.h }[0], [x20], #0x2\n" "35:" // Oddments: Load input (1, 4): Bit 2: End - "ldr x20, [x14, #0x60]\n" + "ldr x20, [x15, #0x60]\n" "fmla v25.8h, v5.8h, v13.8h\n" "fmla v28.8h, v2.8h, v13.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "tbz %x[n_channels], #2, 37f\n" "ld1 { v12.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #1, 36f\n" @@ -749,10 +749,10 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "38:" // Oddments: Load input (3, 0): Bit 2: Unset: Bit 1: Unset "ld1 { v12.h }[0], [x20], #0x2\n" "39:" // Oddments: Load input (3, 0): Bit 2: End - "ldr x20, [x14, #0x68]\n" + "ldr x20, [x15, #0x68]\n" "fmla v26.8h, v6.8h, v12.8h\n" "fmla v29.8h, v3.8h, v12.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "tbz %x[n_channels], #2, 41f\n" "ld1 { v10.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #1, 40f\n" @@ -773,10 +773,10 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "42:" // Oddments: Load input (3, 2): Bit 2: Unset: Bit 1: Unset "ld1 { v10.h }[0], [x20], #0x2\n" "43:" // Oddments: Load input (3, 2): Bit 2: End - "ldr x20, [x14, #0x70]\n" + "ldr x20, [x15, #0x70]\n" "fmla v26.8h, v8.8h, v10.8h\n" "fmla v27.8h, v7.8h, v10.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "fmla v28.8h, v6.8h, v10.8h\n" "fmla v29.8h, v5.8h, v10.8h\n" "fmla v30.8h, v4.8h, v10.8h\n" @@ -801,10 +801,10 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "46:" // Oddments: Load input (3, 4): Bit 2: Unset: Bit 1: Unset "ld1 { v11.h }[0], [x20], #0x2\n" "47:" // Oddments: Load input (3, 4): Bit 2: End - "ldr x20, [x14, #0x78]\n" + "ldr x20, [x15, #0x78]\n" "fmla v28.8h, v8.8h, v11.8h\n" "fmla v31.8h, v5.8h, v11.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "tbz %x[n_channels], #2, 49f\n" "ld1 { v13.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #1, 48f\n" @@ -825,10 +825,10 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "50:" // Oddments: Load input (4, 1): Bit 2: Unset: Bit 1: Unset "ld1 { v13.h }[0], [x20], #0x2\n" "51:" // Oddments: Load input (4, 1): Bit 2: End - "ldr x20, [x14, #0x80]\n" + "ldr x20, [x15, #0x80]\n" "fmla v29.8h, v7.8h, v13.8h\n" "fmla v30.8h, v6.8h, v13.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "tbz %x[n_channels], #2, 53f\n" "ld1 { v12.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #1, 52f\n" @@ -849,10 +849,10 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "54:" // Oddments: Load input (1, 1): Bit 2: Unset: Bit 1: Unset "ld1 { v12.h }[0], [x20], #0x2\n" "55:" // Oddments: Load input (1, 1): Bit 2: End - "ldr x20, [x14, #0x88]\n" + "ldr x20, [x15, #0x88]\n" "fmla v23.8h, v4.8h, v12.8h\n" "fmla v24.8h, v3.8h, v12.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "fmla v26.8h, v1.8h, v12.8h\n" "fmla v27.8h, v0.8h, v12.8h\n" "tbz %x[n_channels], #2, 57f\n" @@ -875,10 +875,10 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "58:" // Oddments: Load input (1, 3): Bit 2: Unset: Bit 1: Unset "ld1 { v11.h }[0], [x20], #0x2\n" "59:" // Oddments: Load input (1, 3): Bit 2: End - "ldr x20, [x14, #0x90]\n" + "ldr x20, [x15, #0x90]\n" "fmla v24.8h, v5.8h, v11.8h\n" "fmla v25.8h, v4.8h, v11.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "fmla v27.8h, v2.8h, v11.8h\n" "fmla v28.8h, v1.8h, v11.8h\n" "tbz %x[n_channels], #2, 61f\n" @@ -901,10 +901,10 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "62:" // Oddments: Load input (4, 3): Bit 2: Unset: Bit 1: Unset "ld1 { v13.h }[0], [x20], #0x2\n" "63:" // Oddments: Load input (4, 3): Bit 2: End - "ldr x20, [x14, #0x98]\n" + "ldr x20, [x15, #0x98]\n" "fmla v30.8h, v8.8h, v13.8h\n" "fmla v31.8h, v7.8h, v13.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "tbz %x[n_channels], #2, 65f\n" "ld1 { v12.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #1, 64f\n" @@ -925,10 +925,10 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "66:" // Oddments: Load input (3, 1): Bit 2: Unset: Bit 1: Unset "ld1 { v12.h }[0], [x20], #0x2\n" "67:" // Oddments: Load input (3, 1): Bit 2: End - "ldr x20, [x14, #0xa0]\n" + "ldr x20, [x15, #0xa0]\n" "fmla v26.8h, v7.8h, v12.8h\n" "fmla v27.8h, v6.8h, v12.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "fmla v29.8h, v4.8h, v12.8h\n" "fmla v30.8h, v3.8h, v12.8h\n" "tbz %x[n_channels], #2, 69f\n" @@ -951,10 +951,10 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "70:" // Oddments: Load input (0, 2): Bit 2: Unset: Bit 1: Unset "ld1 { v11.h }[0], [x20], #0x2\n" "71:" // Oddments: Load input (0, 2): Bit 2: End - "ldr x20, [x14, #0xa8]\n" + "ldr x20, [x15, #0xa8]\n" "fmla v23.8h, v2.8h, v11.8h\n" "fmla v24.8h, v1.8h, v11.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "fmla v25.8h, v0.8h, v11.8h\n" "tbz %x[n_channels], #2, 73f\n" "ld1 { v13.d }[0], [x20], #0x8\n" @@ -976,10 +976,10 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "74:" // Oddments: Load input (3, 3): Bit 2: Unset: Bit 1: Unset "ld1 { v13.h }[0], [x20], #0x2\n" "75:" // Oddments: Load input (3, 3): Bit 2: End - "ldr x20, [x14, #0xb0]\n" + "ldr x20, [x15, #0xb0]\n" "fmla v27.8h, v8.8h, v13.8h\n" "fmla v28.8h, v7.8h, v13.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "fmla v30.8h, v5.8h, v13.8h\n" "fmla v31.8h, v4.8h, v13.8h\n" "tbz %x[n_channels], #2, 77f\n" @@ -1002,10 +1002,10 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "78:" // Oddments: Load input (2, 0): Bit 2: Unset: Bit 1: Unset "ld1 { v12.h }[0], [x20], #0x2\n" "79:" // Oddments: Load input (2, 0): Bit 2: End - "ldr x20, [x14, #0xb8]\n" + "ldr x20, [x15, #0xb8]\n" "fmla v23.8h, v6.8h, v12.8h\n" "fmla v26.8h, v3.8h, v12.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "fmla v29.8h, v0.8h, v12.8h\n" "tbz %x[n_channels], #2, 81f\n" "ld1 { v11.d }[0], [x20], #0x8\n" @@ -1027,10 +1027,10 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "82:" // Oddments: Load input (2, 4): Bit 2: Unset: Bit 1: Unset "ld1 { v11.h }[0], [x20], #0x2\n" "83:" // Oddments: Load input (2, 4): Bit 2: End - "ldr x20, [x14, #0xc0]\n" + "ldr x20, [x15, #0xc0]\n" "fmla v25.8h, v8.8h, v11.8h\n" "fmla v28.8h, v5.8h, v11.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "fmla v31.8h, v2.8h, v11.8h\n" "tbz %x[n_channels], #2, 85f\n" "ld1 { v13.d }[0], [x20], #0x8\n" @@ -1054,236 +1054,234 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "87:" // Oddments: Load input (4, 2): Bit 2: End "fmla v29.8h, v8.8h, v13.8h\n" "fmla v30.8h, v7.8h, v13.8h\n" - "fmax v23.8h, v23.8h, v18.8h\n" + "fmax v23.8h, v23.8h, v15.8h\n" "fmla v31.8h, v6.8h, v13.8h\n" - "fmax v24.8h, v24.8h, v18.8h\n" - "fmax v25.8h, v25.8h, v18.8h\n" - "fmax v26.8h, v26.8h, v18.8h\n" - "fmax v27.8h, v27.8h, v18.8h\n" - "fmax v28.8h, v28.8h, v18.8h\n" - "fmax v29.8h, v29.8h, v18.8h\n" - "fmax v30.8h, v30.8h, v18.8h\n" - "fmax v31.8h, v31.8h, v18.8h\n" - "fmin v23.8h, v23.8h, v17.8h\n" - "fmin v24.8h, v24.8h, v17.8h\n" - "fmin v25.8h, v25.8h, v17.8h\n" - "fmin v26.8h, v26.8h, v17.8h\n" - "fmin v27.8h, v27.8h, v17.8h\n" - "fmin v28.8h, v28.8h, v17.8h\n" - "fmin v29.8h, v29.8h, v17.8h\n" - "fmin v30.8h, v30.8h, v17.8h\n" - "fmin v31.8h, v31.8h, v17.8h\n" + "fmax v24.8h, v24.8h, v15.8h\n" + "fmax v25.8h, v25.8h, v15.8h\n" + "fmax v26.8h, v26.8h, v15.8h\n" + "fmax v27.8h, v27.8h, v15.8h\n" + "fmax v28.8h, v28.8h, v15.8h\n" + "fmax v29.8h, v29.8h, v15.8h\n" + "fmax v30.8h, v30.8h, v15.8h\n" + "fmax v31.8h, v31.8h, v15.8h\n" + "fmin v23.8h, v23.8h, v14.8h\n" + "fmin v24.8h, v24.8h, v14.8h\n" + "fmin v25.8h, v25.8h, v14.8h\n" + "fmin v26.8h, v26.8h, v14.8h\n" + "fmin v27.8h, v27.8h, v14.8h\n" + "fmin v28.8h, v28.8h, v14.8h\n" + "fmin v29.8h, v29.8h, v14.8h\n" + "fmin v30.8h, v30.8h, v14.8h\n" + "fmin v31.8h, v31.8h, v14.8h\n" "tbz %x[n_channels], #2, 89f\n" - "ldr x23, [x16, #0x0]\n" - "add x23, x23, x12\n" - "st1 { v23.d }[0], [x23]\n" - "ldr x22, [x16, #0x8]\n" - "ldr x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x18]\n" - "add x22, x22, x12\n" - "add x21, x21, x12\n" - "ldr x23, [x16, #0x20]\n" - "add x20, x20, x12\n" - "add x23, x23, x12\n" - "st1 { v24.d }[0], [x22]\n" - "st1 { v25.d }[0], [x21]\n" - "ldr x22, [x16, #0x28]\n" - "ldr x21, [x16, #0x30]\n" - "add x22, x22, x12\n" - "st1 { v26.d }[0], [x20]\n" - "ldr x20, [x16, #0x38]\n" - "add x21, x21, x12\n" - "add x20, x20, x12\n" - "st1 { v27.d }[0], [x23]\n" - "ldr x23, [x16, #0x40]\n" - "add x23, x23, x12\n" - "add x12, x12, #0x8\n" - "st1 { v28.d }[0], [x22]\n" - "st1 { v29.d }[0], [x21]\n" - "st1 { v30.d }[0], [x20]\n" - "st1 { v31.d }[0], [x23]\n" + "ldr x20, [x17, #0x0]\n" + "add x20, x20, x13\n" + "st1 { v23.d }[0], [x20]\n" + "ldr x23, [x17, #0x8]\n" + "ldr x22, [x17, #0x10]\n" + "ldr x21, [x17, #0x18]\n" + "add x23, x23, x13\n" + "add x22, x22, x13\n" + "ldr x20, [x17, #0x20]\n" + "add x21, x21, x13\n" + "add x20, x20, x13\n" + "st1 { v24.d }[0], [x23]\n" + "st1 { v25.d }[0], [x22]\n" + "ldr x23, [x17, #0x28]\n" + "ldr x22, [x17, #0x30]\n" + "add x23, x23, x13\n" + "st1 { v26.d }[0], [x21]\n" + "ldr x21, [x17, #0x38]\n" + "add x22, x22, x13\n" + "add x21, x21, x13\n" + "st1 { v27.d }[0], [x20]\n" + "ldr x20, [x17, #0x40]\n" + "add x20, x20, x13\n" + "add x13, x13, #0x8\n" + "st1 { v28.d }[0], [x23]\n" + "st1 { v29.d }[0], [x22]\n" + "st1 { v30.d }[0], [x21]\n" + "st1 { v31.d }[0], [x20]\n" "tbz %x[n_channels], #1, 88f\n" - "ldr x23, [x16, #0x0]\n" - "add x23, x23, x12\n" - "st1 { v23.s }[2], [x23]\n" - "ldr x22, [x16, #0x8]\n" - "ldr x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x18]\n" - "add x22, x22, x12\n" - "add x21, x21, x12\n" - "ldr x23, [x16, #0x20]\n" - "add x20, x20, x12\n" - "add x23, x23, x12\n" - "st1 { v24.s }[2], [x22]\n" - "st1 { v25.s }[2], [x21]\n" - "ldr x22, [x16, #0x28]\n" - "ldr x21, [x16, #0x30]\n" - "add x22, x22, x12\n" - "st1 { v26.s }[2], [x20]\n" - "ldr x20, [x16, #0x38]\n" - "add x21, x21, x12\n" - "add x20, x20, x12\n" - "st1 { v27.s }[2], [x23]\n" - "ldr x23, [x16, #0x40]\n" - "add x23, x23, x12\n" - "add x12, x12, #0x4\n" - "st1 { v28.s }[2], [x22]\n" - "st1 { v29.s }[2], [x21]\n" - "st1 { v30.s }[2], [x20]\n" - "st1 { v31.s }[2], [x23]\n" + "ldr x20, [x17, #0x0]\n" + "add x20, x20, x13\n" + "st1 { v23.s }[2], [x20]\n" + "ldr x23, [x17, #0x8]\n" + "ldr x22, [x17, #0x10]\n" + "ldr x21, [x17, #0x18]\n" + "add x23, x23, x13\n" + "add x22, x22, x13\n" + "ldr x20, [x17, #0x20]\n" + "add x21, x21, x13\n" + "add x20, x20, x13\n" + "st1 { v24.s }[2], [x23]\n" + "st1 { v25.s }[2], [x22]\n" + "ldr x23, [x17, #0x28]\n" + "ldr x22, [x17, #0x30]\n" + "add x23, x23, x13\n" + "st1 { v26.s }[2], [x21]\n" + "ldr x21, [x17, #0x38]\n" + "add x22, x22, x13\n" + "add x21, x21, x13\n" + "st1 { v27.s }[2], [x20]\n" + "ldr x20, [x17, #0x40]\n" + "add x20, x20, x13\n" + "add x13, x13, #0x4\n" + "st1 { v28.s }[2], [x23]\n" + "st1 { v29.s }[2], [x22]\n" + "st1 { v30.s }[2], [x21]\n" + "st1 { v31.s }[2], [x20]\n" "tbz %x[n_channels], #0, 91f\n" - "ldr x23, [x16, #0x0]\n" - "add x23, x23, x12\n" - "st1 { v23.h }[6], [x23]\n" - "ldr x22, [x16, #0x8]\n" - "ldr x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x18]\n" - "add x22, x22, x12\n" - "add x21, x21, x12\n" - "ldr x23, [x16, #0x20]\n" - "add x20, x20, x12\n" - "add x23, x23, x12\n" - "st1 { v24.h }[6], [x22]\n" - "st1 { v25.h }[6], [x21]\n" - "ldr x22, [x16, #0x28]\n" - "ldr x21, [x16, #0x30]\n" - "add x22, x22, x12\n" - "st1 { v26.h }[6], [x20]\n" - "ldr x20, [x16, #0x38]\n" - "add x21, x21, x12\n" - "add x20, x20, x12\n" - "st1 { v27.h }[6], [x23]\n" - "ldr x23, [x16, #0x40]\n" - "add x23, x23, x12\n" - "st1 { v28.h }[6], [x22]\n" - "st1 { v29.h }[6], [x21]\n" - "st1 { v30.h }[6], [x20]\n" - "st1 { v31.h }[6], [x23]\n" + "ldr x20, [x17, #0x0]\n" + "add x20, x20, x13\n" + "st1 { v23.h }[6], [x20]\n" + "ldr x23, [x17, #0x8]\n" + "ldr x22, [x17, #0x10]\n" + "ldr x21, [x17, #0x18]\n" + "add x23, x23, x13\n" + "add x22, x22, x13\n" + "ldr x20, [x17, #0x20]\n" + "add x21, x21, x13\n" + "add x20, x20, x13\n" + "st1 { v24.h }[6], [x23]\n" + "st1 { v25.h }[6], [x22]\n" + "ldr x23, [x17, #0x28]\n" + "ldr x22, [x17, #0x30]\n" + "add x23, x23, x13\n" + "st1 { v26.h }[6], [x21]\n" + "ldr x21, [x17, #0x38]\n" + "add x22, x22, x13\n" + "add x21, x21, x13\n" + "st1 { v27.h }[6], [x20]\n" + "ldr x20, [x17, #0x40]\n" + "add x20, x20, x13\n" + "st1 { v28.h }[6], [x23]\n" + "st1 { v29.h }[6], [x22]\n" + "st1 { v30.h }[6], [x21]\n" + "st1 { v31.h }[6], [x20]\n" "b 91f\n" "88:" // Oddments: Store: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 91f\n" - "ldr x23, [x16, #0x0]\n" - "add x23, x23, x12\n" - "st1 { v23.h }[4], [x23]\n" - "ldr x22, [x16, #0x8]\n" - "ldr x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x18]\n" - "add x22, x22, x12\n" - "add x21, x21, x12\n" - "ldr x23, [x16, #0x20]\n" - "add x20, x20, x12\n" - "add x23, x23, x12\n" - "st1 { v24.h }[4], [x22]\n" - "st1 { v25.h }[4], [x21]\n" - "ldr x22, [x16, #0x28]\n" - "ldr x21, [x16, #0x30]\n" - "add x22, x22, x12\n" - "st1 { v26.h }[4], [x20]\n" - "ldr x20, [x16, #0x38]\n" - "add x21, x21, x12\n" - "add x20, x20, x12\n" - "st1 { v27.h }[4], [x23]\n" - "ldr x23, [x16, #0x40]\n" - "add x23, x23, x12\n" - "st1 { v28.h }[4], [x22]\n" - "st1 { v29.h }[4], [x21]\n" - "st1 { v30.h }[4], [x20]\n" - "st1 { v31.h }[4], [x23]\n" + "ldr x20, [x17, #0x0]\n" + "add x20, x20, x13\n" + "st1 { v23.h }[4], [x20]\n" + "ldr x23, [x17, #0x8]\n" + "ldr x22, [x17, #0x10]\n" + "ldr x21, [x17, #0x18]\n" + "add x23, x23, x13\n" + "add x22, x22, x13\n" + "ldr x20, [x17, #0x20]\n" + "add x21, x21, x13\n" + "add x20, x20, x13\n" + "st1 { v24.h }[4], [x23]\n" + "st1 { v25.h }[4], [x22]\n" + "ldr x23, [x17, #0x28]\n" + "ldr x22, [x17, #0x30]\n" + "add x23, x23, x13\n" + "st1 { v26.h }[4], [x21]\n" + "ldr x21, [x17, #0x38]\n" + "add x22, x22, x13\n" + "add x21, x21, x13\n" + "st1 { v27.h }[4], [x20]\n" + "ldr x20, [x17, #0x40]\n" + "add x20, x20, x13\n" + "st1 { v28.h }[4], [x23]\n" + "st1 { v29.h }[4], [x22]\n" + "st1 { v30.h }[4], [x21]\n" + "st1 { v31.h }[4], [x20]\n" "b 91f\n" "89:" // Oddments: Store: Bit 2: Unset "tbz %x[n_channels], #1, 90f\n" - "ldr x23, [x16, #0x0]\n" - "add x23, x23, x12\n" - "st1 { v23.s }[0], [x23]\n" - "ldr x22, [x16, #0x8]\n" - "ldr x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x18]\n" - "add x22, x22, x12\n" - "add x21, x21, x12\n" - "ldr x23, [x16, #0x20]\n" - "add x20, x20, x12\n" - "add x23, x23, x12\n" - "st1 { v24.s }[0], [x22]\n" - "st1 { v25.s }[0], [x21]\n" - "ldr x22, [x16, #0x28]\n" - "ldr x21, [x16, #0x30]\n" - "add x22, x22, x12\n" - "st1 { v26.s }[0], [x20]\n" - "ldr x20, [x16, #0x38]\n" - "add x21, x21, x12\n" - "add x20, x20, x12\n" - "st1 { v27.s }[0], [x23]\n" - "ldr x23, [x16, #0x40]\n" - "add x23, x23, x12\n" - "add x12, x12, #0x4\n" - "st1 { v28.s }[0], [x22]\n" - "st1 { v29.s }[0], [x21]\n" - "st1 { v30.s }[0], [x20]\n" - "st1 { v31.s }[0], [x23]\n" + "ldr x20, [x17, #0x0]\n" + "add x20, x20, x13\n" + "st1 { v23.s }[0], [x20]\n" + "ldr x23, [x17, #0x8]\n" + "ldr x22, [x17, #0x10]\n" + "ldr x21, [x17, #0x18]\n" + "add x23, x23, x13\n" + "add x22, x22, x13\n" + "ldr x20, [x17, #0x20]\n" + "add x21, x21, x13\n" + "add x20, x20, x13\n" + "st1 { v24.s }[0], [x23]\n" + "st1 { v25.s }[0], [x22]\n" + "ldr x23, [x17, #0x28]\n" + "ldr x22, [x17, #0x30]\n" + "add x23, x23, x13\n" + "st1 { v26.s }[0], [x21]\n" + "ldr x21, [x17, #0x38]\n" + "add x22, x22, x13\n" + "add x21, x21, x13\n" + "st1 { v27.s }[0], [x20]\n" + "ldr x20, [x17, #0x40]\n" + "add x20, x20, x13\n" + "add x13, x13, #0x4\n" + "st1 { v28.s }[0], [x23]\n" + "st1 { v29.s }[0], [x22]\n" + "st1 { v30.s }[0], [x21]\n" + "st1 { v31.s }[0], [x20]\n" "tbz %x[n_channels], #0, 91f\n" - "ldr x23, [x16, #0x0]\n" - "add x23, x23, x12\n" - "st1 { v23.h }[2], [x23]\n" - "ldr x22, [x16, #0x8]\n" - "ldr x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x18]\n" - "add x22, x22, x12\n" - "add x21, x21, x12\n" - "ldr x23, [x16, #0x20]\n" - "add x20, x20, x12\n" - "add x23, x23, x12\n" - "st1 { v24.h }[2], [x22]\n" - "st1 { v25.h }[2], [x21]\n" - "ldr x22, [x16, #0x28]\n" - "ldr x21, [x16, #0x30]\n" - "add x22, x22, x12\n" - "st1 { v26.h }[2], [x20]\n" - "ldr x20, [x16, #0x38]\n" - "add x21, x21, x12\n" - "add x20, x20, x12\n" - "st1 { v27.h }[2], [x23]\n" - "ldr x23, [x16, #0x40]\n" - "add x23, x23, x12\n" - "st1 { v28.h }[2], [x22]\n" - "st1 { v29.h }[2], [x21]\n" - "st1 { v30.h }[2], [x20]\n" - "st1 { v31.h }[2], [x23]\n" + "ldr x20, [x17, #0x0]\n" + "add x20, x20, x13\n" + "st1 { v23.h }[2], [x20]\n" + "ldr x23, [x17, #0x8]\n" + "ldr x22, [x17, #0x10]\n" + "ldr x21, [x17, #0x18]\n" + "add x23, x23, x13\n" + "add x22, x22, x13\n" + "ldr x20, [x17, #0x20]\n" + "add x21, x21, x13\n" + "add x20, x20, x13\n" + "st1 { v24.h }[2], [x23]\n" + "st1 { v25.h }[2], [x22]\n" + "ldr x23, [x17, #0x28]\n" + "ldr x22, [x17, #0x30]\n" + "add x23, x23, x13\n" + "st1 { v26.h }[2], [x21]\n" + "ldr x21, [x17, #0x38]\n" + "add x22, x22, x13\n" + "add x21, x21, x13\n" + "st1 { v27.h }[2], [x20]\n" + "ldr x20, [x17, #0x40]\n" + "add x20, x20, x13\n" + "st1 { v28.h }[2], [x23]\n" + "st1 { v29.h }[2], [x22]\n" + "st1 { v30.h }[2], [x21]\n" + "st1 { v31.h }[2], [x20]\n" "b 91f\n" "90:" // Oddments: Store: Bit 2: Unset: Bit 1: Unset - "ldr x23, [x16, #0x0]\n" - "add x23, x23, x12\n" - "st1 { v23.h }[0], [x23]\n" - "ldr x22, [x16, #0x8]\n" - "ldr x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x18]\n" - "add x22, x22, x12\n" - "add x21, x21, x12\n" - "ldr x23, [x16, #0x20]\n" - "add x20, x20, x12\n" - "add x23, x23, x12\n" - "st1 { v24.h }[0], [x22]\n" - "st1 { v25.h }[0], [x21]\n" - "ldr x22, [x16, #0x28]\n" - "ldr x21, [x16, #0x30]\n" - "add x22, x22, x12\n" - "st1 { v26.h }[0], [x20]\n" - "ldr x20, [x16, #0x38]\n" - "add x21, x21, x12\n" - "add x20, x20, x12\n" - "st1 { v27.h }[0], [x23]\n" - "ldr x23, [x16, #0x40]\n" - "add x23, x23, x12\n" - "st1 { v28.h }[0], [x22]\n" - "st1 { v29.h }[0], [x21]\n" - "st1 { v30.h }[0], [x20]\n" - "st1 { v31.h }[0], [x23]\n" + "ldr x20, [x17, #0x0]\n" + "add x20, x20, x13\n" + "st1 { v23.h }[0], [x20]\n" + "ldr x23, [x17, #0x8]\n" + "ldr x22, [x17, #0x10]\n" + "ldr x21, [x17, #0x18]\n" + "add x23, x23, x13\n" + "add x22, x22, x13\n" + "ldr x20, [x17, #0x20]\n" + "add x21, x21, x13\n" + "add x20, x20, x13\n" + "st1 { v24.h }[0], [x23]\n" + "st1 { v25.h }[0], [x22]\n" + "ldr x23, [x17, #0x28]\n" + "ldr x22, [x17, #0x30]\n" + "add x23, x23, x13\n" + "st1 { v26.h }[0], [x21]\n" + "ldr x21, [x17, #0x38]\n" + "add x22, x22, x13\n" + "add x21, x21, x13\n" + "st1 { v27.h }[0], [x20]\n" + "ldr x20, [x17, #0x40]\n" + "add x20, x20, x13\n" + "st1 { v28.h }[0], [x23]\n" + "st1 { v29.h }[0], [x22]\n" + "st1 { v30.h }[0], [x21]\n" + "st1 { v31.h }[0], [x20]\n" "91:" // Oddments: Store: Bit 2: End - "92:" // End - : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v16", "v17", "v18", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp index 4f0de6b61c6172a3f6242cb036df3b8bd3f2d5ec..04fb532937ef13616f116fc223c7e54b4ef79352 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include @@ -33,8 +33,8 @@ namespace arm_conv { namespace depthwise { -void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); -void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); +void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(const __fp16 *const *const input_ptrs, __fp16 *const *const outptrs, const void *params, unsigned int n_channels, const __fp16 activation_min, const __fp16 activation_max); +void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(const unsigned int n_tile_rows, const unsigned int n_tile_cols, const __fp16 *inptr, int64_t ld_input_row, int64_t ld_input_col, __fp16 *outptr, int64_t ld_output_row, int64_t ld_output_col, const void *params, unsigned int n_channels, const __fp16 activation_min, const __fp16 activation_max); class a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst : public DepthwiseDepthfirstStrategy<__fp16, __fp16, __fp16, __fp16> { @@ -57,7 +57,7 @@ class a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst : public DepthwiseDepthfirst constexpr static unsigned int output_cols = 4; a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst(const CPUInfo *) - : DepthwiseDepthfirstStrategy<__fp16, __fp16, __fp16, __fp16>(4, 3, 1) {} + : Parent(output_rows, output_cols, kernel_rows, kernel_cols, stride_rows, stride_cols) {} arm_gemm::VLType get_vl_type(void) const override { return vl_type; } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp index a3a372be05039961448f98c0fd7394bd1a9f9411..a1e1dd0e995a9374a2c7528e5d990a742ae0600e 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp @@ -124,9 +124,9 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "add x27, x10, x22, LSL #1\n" "add x23, x5, x5\n" "add x20, %x[params_struct], %[offsetof_args_min]\n" - "ld1r { v15.8h }, [x20]\n" + "ld1r { v13.8h }, [x20]\n" "add x20, %x[params_struct], %[offsetof_args_max]\n" - "ld1r { v14.8h }, [x20]\n" + "ld1r { v15.8h }, [x20]\n" "add x26, x9, x24, LSL #1\n" "add x25, x28, x4\n" "add x24, x27, x22, LSL #1\n" @@ -134,7 +134,7 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "mov x21, #0x0\n" "sub x20, XZR, x6\n" "cbz x13, 4f\n" - "ldr q13, [x16, #0x0]\n" + "ldr q14, [x16, #0x0]\n" "ldr q0, [x16, #0x10]\n" "cmp x6, x13, LSL #4\n" "ldr q1, [x16, #0x20]\n" @@ -152,499 +152,499 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "ldr q12, [x14, x11]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "mov v21.16b, v13.16b\n fmla v21.8h, v4.8h, v9.8h\n" - "mov v16.16b, v13.16b\n fmla v16.8h, v8.8h, v9.8h\n" + "mov v26.16b, v14.16b\n fmla v26.8h, v4.8h, v9.8h\n" + "mov v28.16b, v14.16b\n fmla v28.8h, v8.8h, v9.8h\n" "add x6, x6, #0x10\n" "cmp x6, x13, LSL #4\n" - "mov v22.16b, v13.16b\n fmla v22.8h, v3.8h, v9.8h\n" - "mov v25.16b, v13.16b\n fmla v25.8h, v1.8h, v9.8h\n" + "mov v16.16b, v14.16b\n fmla v16.8h, v3.8h, v9.8h\n" + "mov v22.16b, v14.16b\n fmla v22.8h, v1.8h, v9.8h\n" "add x20, x20, #0x10\n" "add x21, x21, #0x10\n" - "mov v26.16b, v13.16b\n fmla v26.8h, v0.8h, v9.8h\n" - "fmla v21.8h, v5.8h, v12.8h\n" - "mov v17.16b, v13.16b\n fmla v17.8h, v7.8h, v9.8h\n" - "mov v18.16b, v13.16b\n fmla v18.8h, v6.8h, v9.8h\n" - "mov v20.16b, v13.16b\n fmla v20.8h, v5.8h, v9.8h\n" - "mov v24.16b, v13.16b\n fmla v24.8h, v2.8h, v9.8h\n" + "mov v23.16b, v14.16b\n fmla v23.8h, v0.8h, v9.8h\n" + "fmla v26.8h, v5.8h, v12.8h\n" + "mov v25.16b, v14.16b\n fmla v25.8h, v7.8h, v9.8h\n" + "mov v17.16b, v14.16b\n fmla v17.8h, v6.8h, v9.8h\n" + "mov v31.16b, v14.16b\n fmla v31.8h, v5.8h, v9.8h\n" + "mov v20.16b, v14.16b\n fmla v20.8h, v2.8h, v9.8h\n" "ldr q9, [x12, x17]\n" - "fmla v16.8h, v0.8h, v10.8h\n" - "ld1 { v10.8h }, [x26]\n" - "mov v19.16b, v13.16b\n fmla v19.8h, v2.8h, v11.8h\n" - "ldr q11, [x26, x25]\n" - "fmla v22.8h, v4.8h, v12.8h\n" - "fmla v25.8h, v2.8h, v12.8h\n" - "fmla v26.8h, v1.8h, v12.8h\n" - "mov v28.16b, v13.16b\n fmla v28.8h, v6.8h, v10.8h\n" + "fmla v28.8h, v0.8h, v10.8h\n" + "ld1 { v30.8h }, [x26]\n" + "mov v29.16b, v14.16b\n fmla v29.8h, v2.8h, v11.8h\n" + "ldr q27, [x26, x25]\n" + "fmla v16.8h, v4.8h, v12.8h\n" + "fmla v22.8h, v2.8h, v12.8h\n" + "fmla v23.8h, v1.8h, v12.8h\n" + "mov v21.16b, v14.16b\n fmla v21.8h, v6.8h, v30.8h\n" "ldr q10, [x12, x11]\n" - "fmla v21.8h, v7.8h, v9.8h\n" - "fmla v17.8h, v8.8h, v12.8h\n" - "fmla v18.8h, v7.8h, v12.8h\n" - "fmla v19.8h, v6.8h, v12.8h\n" - "mov v23.16b, v13.16b\n fmla v23.8h, v3.8h, v12.8h\n" - "mov v27.16b, v13.16b\n fmla v27.8h, v0.8h, v12.8h\n" - "ldr q12, [x7, x4]\n" - "mov v31.16b, v13.16b\n fmla v31.8h, v8.8h, v11.8h\n" - "ldr q11, [x7, x28]\n" - "fmla v22.8h, v6.8h, v9.8h\n" - "fmla v25.8h, v4.8h, v9.8h\n" - "fmla v26.8h, v3.8h, v9.8h\n" - "fmla v20.8h, v8.8h, v9.8h\n" - "fmla v24.8h, v5.8h, v9.8h\n" - "fmla v28.8h, v2.8h, v9.8h\n" - "fmla v21.8h, v8.8h, v10.8h\n" - "fmla v16.8h, v1.8h, v12.8h\n" - "fmla v17.8h, v0.8h, v12.8h\n" - "ldr q12, [x15, x25]\n" - "fmla v18.8h, v2.8h, v11.8h\n" - "fmla v19.8h, v1.8h, v11.8h\n" - "ld1 { v11.8h }, [x9]\n" - "fmla v22.8h, v7.8h, v10.8h\n" - "fmla v23.8h, v6.8h, v10.8h\n" - "fmla v25.8h, v5.8h, v10.8h\n" - "fmla v26.8h, v4.8h, v10.8h\n" - "fmla v27.8h, v3.8h, v10.8h\n" - "fmla v31.8h, v0.8h, v10.8h\n" - "fmla v24.8h, v6.8h, v11.8h\n" - "fmla v28.8h, v3.8h, v11.8h\n" - "ldr q11, [x9, x25]\n" - "fmla v19.8h, v5.8h, v12.8h\n" - "fmla v23.8h, v2.8h, v12.8h\n" - "ldr q12, [x15, x11]\n" - "fmla v27.8h, v8.8h, v11.8h\n" - "fmla v31.8h, v5.8h, v11.8h\n" - "mov v29.16b, v13.16b\n fmla v29.8h, v1.8h, v9.8h\n" - "mov v30.16b, v13.16b\n fmla v30.8h, v0.8h, v9.8h\n" + "fmla v26.8h, v7.8h, v9.8h\n" + "fmla v25.8h, v8.8h, v12.8h\n" + "fmla v17.8h, v7.8h, v12.8h\n" + "fmla v29.8h, v6.8h, v12.8h\n" + "mov v24.16b, v14.16b\n fmla v24.8h, v3.8h, v12.8h\n" + "mov v19.16b, v14.16b\n fmla v19.8h, v0.8h, v12.8h\n" + "ldr q11, [x7, x4]\n" + "mov v30.16b, v14.16b\n fmla v30.8h, v8.8h, v27.8h\n" + "ldr q12, [x7, x28]\n" + "fmla v16.8h, v6.8h, v9.8h\n" + "fmla v22.8h, v4.8h, v9.8h\n" + "fmla v23.8h, v3.8h, v9.8h\n" + "mov v27.16b, v14.16b\n fmla v27.8h, v1.8h, v9.8h\n" + "mov v18.16b, v14.16b\n fmla v18.8h, v0.8h, v9.8h\n" + "ldr q14, [x16, #0x0]\n" + "fmla v31.8h, v8.8h, v9.8h\n" + "fmla v20.8h, v5.8h, v9.8h\n" + "fmla v21.8h, v2.8h, v9.8h\n" "ld1 { v9.8h }, [x15]\n" - "fmla v29.8h, v2.8h, v10.8h\n" - "fmla v30.8h, v1.8h, v10.8h\n" + "fmla v26.8h, v8.8h, v10.8h\n" + "fmla v28.8h, v1.8h, v11.8h\n" + "fmla v25.8h, v0.8h, v11.8h\n" + "ldr q11, [x15, x25]\n" + "fmla v17.8h, v2.8h, v12.8h\n" + "fmla v29.8h, v1.8h, v12.8h\n" + "ld1 { v12.8h }, [x9]\n" + "fmla v16.8h, v7.8h, v10.8h\n" + "fmla v24.8h, v6.8h, v10.8h\n" + "fmla v22.8h, v5.8h, v10.8h\n" + "fmla v23.8h, v4.8h, v10.8h\n" + "fmla v19.8h, v3.8h, v10.8h\n" + "fmla v27.8h, v2.8h, v10.8h\n" + "fmla v18.8h, v1.8h, v10.8h\n" + "fmla v30.8h, v0.8h, v10.8h\n" "ldr q10, [x15, x17]\n" - "fmla v20.8h, v0.8h, v9.8h\n" - "fmla v21.8h, v1.8h, v10.8h\n" - "fmla v16.8h, v3.8h, v9.8h\n" - "ldr q11, [x26, x4]\n" - "fmla v17.8h, v4.8h, v10.8h\n" - "fmla v18.8h, v3.8h, v10.8h\n" - "fmla v22.8h, v0.8h, v10.8h\n" - "fmla v20.8h, v2.8h, v10.8h\n" - "fmla v21.8h, v2.8h, v12.8h\n" - "fmla v16.8h, v5.8h, v10.8h\n" + "fmla v31.8h, v0.8h, v9.8h\n" + "fmla v20.8h, v6.8h, v12.8h\n" + "fmla v21.8h, v3.8h, v12.8h\n" + "ldr q12, [x9, x25]\n" + "fmla v26.8h, v1.8h, v10.8h\n" + "fmla v28.8h, v3.8h, v9.8h\n" + "fmla v29.8h, v5.8h, v11.8h\n" + "fmla v24.8h, v2.8h, v11.8h\n" + "ldr q11, [x15, x11]\n" + "fmla v25.8h, v4.8h, v10.8h\n" + "fmla v17.8h, v3.8h, v10.8h\n" + "fmla v16.8h, v0.8h, v10.8h\n" + "fmla v19.8h, v8.8h, v12.8h\n" + "fmla v30.8h, v5.8h, v12.8h\n" + "ldr q9, [x26, x4]\n" + "fmla v31.8h, v2.8h, v10.8h\n" + "fmla v26.8h, v2.8h, v11.8h\n" + "fmla v28.8h, v5.8h, v10.8h\n" "ldr q10, [x14, x4]\n" - "fmla v17.8h, v5.8h, v12.8h\n" - "fmla v18.8h, v4.8h, v12.8h\n" - "fmla v19.8h, v3.8h, v12.8h\n" - "fmla v22.8h, v1.8h, v12.8h\n" - "fmla v23.8h, v0.8h, v12.8h\n" - "ldr q12, [x14, x28]\n" - "fmla v28.8h, v7.8h, v11.8h\n" - "fmla v29.8h, v6.8h, v11.8h\n" - "ldr q11, [x26, x28]\n" - "fmla v20.8h, v4.8h, v10.8h\n" - "fmla v21.8h, v3.8h, v10.8h\n" - "fmla v24.8h, v1.8h, v10.8h\n" - "fmla v25.8h, v0.8h, v10.8h\n" - "fmla v16.8h, v7.8h, v10.8h\n" - "fmla v17.8h, v6.8h, v10.8h\n" + "fmla v25.8h, v5.8h, v11.8h\n" + "fmla v17.8h, v4.8h, v11.8h\n" + "fmla v29.8h, v3.8h, v11.8h\n" + "fmla v16.8h, v1.8h, v11.8h\n" + "fmla v24.8h, v0.8h, v11.8h\n" + "ldr q11, [x14, x28]\n" + "fmla v21.8h, v7.8h, v9.8h\n" + "fmla v27.8h, v6.8h, v9.8h\n" + "ldr q12, [x26, x28]\n" + "fmla v31.8h, v4.8h, v10.8h\n" + "fmla v26.8h, v3.8h, v10.8h\n" + "fmla v20.8h, v1.8h, v10.8h\n" + "fmla v22.8h, v0.8h, v10.8h\n" + "fmla v28.8h, v7.8h, v10.8h\n" + "fmla v25.8h, v6.8h, v10.8h\n" "ldr q10, [x7, x17]\n" - "fmla v30.8h, v8.8h, v11.8h\n" - "fmla v31.8h, v7.8h, v11.8h\n" - "ldr q11, [x12, x4]\n" "fmla v18.8h, v8.8h, v12.8h\n" - "fmla v19.8h, v7.8h, v12.8h\n" - "fmla v22.8h, v5.8h, v12.8h\n" - "fmla v23.8h, v4.8h, v12.8h\n" - "fmla v26.8h, v2.8h, v12.8h\n" - "fmla v27.8h, v1.8h, v12.8h\n" + "fmla v30.8h, v7.8h, v12.8h\n" + "ldr q9, [x12, x4]\n" + "fmla v17.8h, v8.8h, v11.8h\n" + "fmla v29.8h, v7.8h, v11.8h\n" + "fmla v16.8h, v5.8h, v11.8h\n" + "fmla v24.8h, v4.8h, v11.8h\n" + "fmla v23.8h, v2.8h, v11.8h\n" + "fmla v19.8h, v1.8h, v11.8h\n" "ldr q12, [x7, x11]\n" "add x7, x7, #0x10\n" - "fmla v20.8h, v7.8h, v11.8h\n" - "fmla v21.8h, v6.8h, v11.8h\n" - "fmla v24.8h, v4.8h, v11.8h\n" - "fmla v25.8h, v3.8h, v11.8h\n" - "fmla v28.8h, v1.8h, v11.8h\n" - "fmla v29.8h, v0.8h, v11.8h\n" - "ldr q11, [x12, x28]\n" - "fmla v16.8h, v2.8h, v10.8h\n" - "fmla v17.8h, v1.8h, v10.8h\n" - "fmla v18.8h, v0.8h, v10.8h\n" + "fmla v31.8h, v7.8h, v9.8h\n" + "fmla v26.8h, v6.8h, v9.8h\n" + "fmla v20.8h, v4.8h, v9.8h\n" + "fmla v22.8h, v3.8h, v9.8h\n" + "fmla v21.8h, v1.8h, v9.8h\n" + "fmla v27.8h, v0.8h, v9.8h\n" + "ldr q9, [x12, x28]\n" + "fmla v28.8h, v2.8h, v10.8h\n" + "fmla v25.8h, v1.8h, v10.8h\n" + "fmla v17.8h, v0.8h, v10.8h\n" "ld1 { v10.8h }, [x14]\n" - "fmla v30.8h, v2.8h, v11.8h\n" - "fmla v19.8h, v0.8h, v12.8h\n" - "fmla v20.8h, v3.8h, v10.8h\n" - "fmla v24.8h, v0.8h, v10.8h\n" - "fmla v22.8h, v8.8h, v11.8h\n" - "fmla v23.8h, v7.8h, v11.8h\n" - "fmla v26.8h, v5.8h, v11.8h\n" - "fmla v27.8h, v4.8h, v11.8h\n" - "fmla v31.8h, v1.8h, v11.8h\n" + "fmla v18.8h, v2.8h, v9.8h\n" + "fmla v29.8h, v0.8h, v12.8h\n" + "fmla v31.8h, v3.8h, v10.8h\n" + "fmla v20.8h, v0.8h, v10.8h\n" + "fmla v16.8h, v8.8h, v9.8h\n" + "fmla v24.8h, v7.8h, v9.8h\n" + "fmla v23.8h, v5.8h, v9.8h\n" + "fmla v19.8h, v4.8h, v9.8h\n" + "fmla v30.8h, v1.8h, v9.8h\n" "ldr q11, [x9, x17]\n" - "fmla v17.8h, v2.8h, v12.8h\n" - "fmla v18.8h, v1.8h, v12.8h\n" + "fmla v25.8h, v2.8h, v12.8h\n" + "fmla v17.8h, v1.8h, v12.8h\n" "ldr q12, [x14, x25]\n" "add x14, x14, #0x10\n" - "fmla v16.8h, v6.8h, v10.8h\n" + "ldr q9, [x14, x17]\n" + "fmla v28.8h, v6.8h, v10.8h\n" "ld1 { v10.8h }, [x12]\n" - "fmla v29.8h, v4.8h, v11.8h\n" - "fmla v30.8h, v3.8h, v11.8h\n" - "fmla v19.8h, v8.8h, v12.8h\n" - "fmla v23.8h, v5.8h, v12.8h\n" - "fmla v27.8h, v2.8h, v12.8h\n" + "fmla v27.8h, v4.8h, v11.8h\n" + "fmla v18.8h, v3.8h, v11.8h\n" + "fmla v29.8h, v8.8h, v12.8h\n" + "fmla v24.8h, v5.8h, v12.8h\n" + "fmla v19.8h, v2.8h, v12.8h\n" "ldr q12, [x12, x25]\n" "add x12, x12, #0x10\n" - "fmla v20.8h, v6.8h, v10.8h\n" - "fmla v24.8h, v3.8h, v10.8h\n" - "fmla v28.8h, v0.8h, v10.8h\n" + "fmla v31.8h, v6.8h, v10.8h\n" + "fmla v20.8h, v3.8h, v10.8h\n" + "fmla v21.8h, v0.8h, v10.8h\n" "ldr q10, [x26, x17]\n" - "fmla v31.8h, v2.8h, v12.8h\n" - "fmla v29.8h, v7.8h, v10.8h\n" - "fmla v30.8h, v6.8h, v10.8h\n" - "fmla v24.8h, v8.8h, v11.8h\n" - "fmla v25.8h, v7.8h, v11.8h\n" - "fmla v26.8h, v6.8h, v11.8h\n" - "fmla v28.8h, v5.8h, v11.8h\n" + "fmla v30.8h, v2.8h, v12.8h\n" + "fmla v27.8h, v7.8h, v10.8h\n" + "fmla v18.8h, v6.8h, v10.8h\n" + "fmla v20.8h, v8.8h, v11.8h\n" + "fmla v22.8h, v7.8h, v11.8h\n" + "fmla v23.8h, v6.8h, v11.8h\n" + "fmla v21.8h, v5.8h, v11.8h\n" "ldr q11, [x9, x11]\n" - "fmla v27.8h, v5.8h, v12.8h\n" - "fmla v29.8h, v5.8h, v11.8h\n" - "fmla v30.8h, v4.8h, v11.8h\n" - "fmla v31.8h, v3.8h, v11.8h\n" - "fmla v23.8h, v8.8h, v12.8h\n" + "fmla v19.8h, v5.8h, v12.8h\n" + "fmla v27.8h, v5.8h, v11.8h\n" + "fmla v18.8h, v4.8h, v11.8h\n" + "fmla v30.8h, v3.8h, v11.8h\n" + "fmla v24.8h, v8.8h, v12.8h\n" "ldr q12, [x26, x11]\n" - "fmla v28.8h, v8.8h, v10.8h\n" + "fmla v21.8h, v8.8h, v10.8h\n" "ldr q10, [x15, x4]\n" - "fmla v25.8h, v8.8h, v11.8h\n" - "fmla v26.8h, v7.8h, v11.8h\n" + "fmla v22.8h, v8.8h, v11.8h\n" + "fmla v23.8h, v7.8h, v11.8h\n" "add x26, x26, #0x10\n" - "fmla v27.8h, v6.8h, v11.8h\n" + "fmla v19.8h, v6.8h, v11.8h\n" "ldr q11, [x15, x28]\n" - "fmla v29.8h, v8.8h, v12.8h\n" + "fmla v27.8h, v8.8h, v12.8h\n" "add x15, x15, #0x10\n" - "fmla v30.8h, v7.8h, v12.8h\n" - "fmla v31.8h, v6.8h, v12.8h\n" + "fmla v18.8h, v7.8h, v12.8h\n" + "fmla v30.8h, v6.8h, v12.8h\n" "ldr q12, [x9, x4]\n" - "fmla v16.8h, v4.8h, v10.8h\n" - "fmla v17.8h, v3.8h, v10.8h\n" - "fmax v16.8h, v16.8h, v15.8h\n" - "fmla v20.8h, v1.8h, v10.8h\n" - "fmla v21.8h, v0.8h, v10.8h\n" + "fmla v28.8h, v4.8h, v10.8h\n" + "fmla v25.8h, v3.8h, v10.8h\n" + "fmax v28.8h, v28.8h, v13.8h\n" + "fmla v31.8h, v1.8h, v10.8h\n" + "fmla v26.8h, v0.8h, v10.8h\n" "ldr q10, [x9, x28]\n" - "ldr q9, [x14, x17]\n" - "fmla v18.8h, v5.8h, v11.8h\n" - "fmla v19.8h, v4.8h, v11.8h\n" - "fmax v17.8h, v17.8h, v15.8h\n" + "ldr q0, [x16, #0x10]\n" + "fmla v17.8h, v5.8h, v11.8h\n" + "fmla v29.8h, v4.8h, v11.8h\n" + "fmax v25.8h, v25.8h, v13.8h\n" "add x9, x9, #0x10\n" - "fmla v22.8h, v2.8h, v11.8h\n" - "ldr q13, [x16, #0x0]\n" - "fmla v23.8h, v1.8h, v11.8h\n" + "fmla v16.8h, v2.8h, v11.8h\n" + "ldr q2, [x16, #0x30]\n" + "fmla v24.8h, v1.8h, v11.8h\n" "ldr q11, [x7, x25]\n" - "ldr q0, [x16, #0x10]\n" - "fmla v24.8h, v7.8h, v12.8h\n" - "fmla v25.8h, v6.8h, v12.8h\n" "ldr q1, [x16, #0x20]\n" - "fmla v28.8h, v4.8h, v12.8h\n" - "fmla v29.8h, v3.8h, v12.8h\n" + "fmla v20.8h, v7.8h, v12.8h\n" + "fmla v22.8h, v6.8h, v12.8h\n" + "ldr q6, [x16, #0x70]\n" + "fmla v21.8h, v4.8h, v12.8h\n" + "fmla v27.8h, v3.8h, v12.8h\n" "ldr q12, [x14, x11]\n" - "ldr q2, [x16, #0x30]\n" - "fmla v26.8h, v8.8h, v10.8h\n" "ldr q3, [x16, #0x40]\n" - "fmla v27.8h, v7.8h, v10.8h\n" - "ldr q6, [x16, #0x70]\n" - "fmla v30.8h, v5.8h, v10.8h\n" + "fmla v23.8h, v8.8h, v10.8h\n" + "ldr q8, [x16, #0x90]\n" + "fmla v19.8h, v7.8h, v10.8h\n" + "ldr q7, [x16, #0x80]\n" + "fmla v18.8h, v5.8h, v10.8h\n" "ldr q5, [x16, #0x60]\n" - "fmla v31.8h, v4.8h, v10.8h\n" + "fmla v30.8h, v4.8h, v10.8h\n" "ld1 { v10.8h }, [x7]\n" "ldr q4, [x16, #0x50]\n" - "fmax v18.8h, v18.8h, v15.8h\n" - "fmax v19.8h, v19.8h, v15.8h\n" - "fmax v20.8h, v20.8h, v15.8h\n" - "fmax v21.8h, v21.8h, v15.8h\n" - "fmax v22.8h, v22.8h, v15.8h\n" - "fmax v23.8h, v23.8h, v15.8h\n" - "fmax v24.8h, v24.8h, v15.8h\n" - "fmax v25.8h, v25.8h, v15.8h\n" - "fmax v26.8h, v26.8h, v15.8h\n" - "fmax v27.8h, v27.8h, v15.8h\n" - "fmax v28.8h, v28.8h, v15.8h\n" - "fmax v29.8h, v29.8h, v15.8h\n" - "fmax v30.8h, v30.8h, v15.8h\n" - "fmax v31.8h, v31.8h, v15.8h\n" - "fmin v16.8h, v16.8h, v14.8h\n" - "fmin v17.8h, v17.8h, v14.8h\n" - "st1 { v16.8h }, [x8]\n" - "ldr q7, [x16, #0x80]\n" - "fmin v18.8h, v18.8h, v14.8h\n" - "fmin v19.8h, v19.8h, v14.8h\n" - "str q17, [x8, x5]\n" - "ldr q8, [x16, #0x90]\n" - "fmin v20.8h, v20.8h, v14.8h\n" - "fmin v21.8h, v21.8h, v14.8h\n" - "str q18, [x8, x23]\n" + "fmax v17.8h, v17.8h, v13.8h\n" + "fmax v29.8h, v29.8h, v13.8h\n" "add x16, x16, #0xa0\n" - "fmin v22.8h, v22.8h, v14.8h\n" - "fmin v23.8h, v23.8h, v14.8h\n" - "str q19, [x8, x22]\n" + "fmax v31.8h, v31.8h, v13.8h\n" + "fmax v26.8h, v26.8h, v13.8h\n" + "fmax v16.8h, v16.8h, v13.8h\n" + "fmax v24.8h, v24.8h, v13.8h\n" + "fmax v20.8h, v20.8h, v13.8h\n" + "fmax v22.8h, v22.8h, v13.8h\n" + "fmax v23.8h, v23.8h, v13.8h\n" + "fmax v19.8h, v19.8h, v13.8h\n" + "fmax v21.8h, v21.8h, v13.8h\n" + "fmax v27.8h, v27.8h, v13.8h\n" + "fmax v18.8h, v18.8h, v13.8h\n" + "fmax v30.8h, v30.8h, v13.8h\n" + "fmin v28.8h, v28.8h, v15.8h\n" + "fmin v25.8h, v25.8h, v15.8h\n" + "st1 { v28.8h }, [x8]\n" + "fmin v17.8h, v17.8h, v15.8h\n" + "fmin v29.8h, v29.8h, v15.8h\n" + "str q25, [x8, x5]\n" + "fmin v31.8h, v31.8h, v15.8h\n" + "fmin v26.8h, v26.8h, v15.8h\n" + "str q17, [x8, x23]\n" + "fmin v16.8h, v16.8h, v15.8h\n" + "fmin v24.8h, v24.8h, v15.8h\n" + "str q29, [x8, x22]\n" "add x8, x8, #0x10\n" - "fmin v24.8h, v24.8h, v14.8h\n" - "fmin v25.8h, v25.8h, v14.8h\n" - "st1 { v20.8h }, [x10]\n" - "fmin v26.8h, v26.8h, v14.8h\n" - "fmin v27.8h, v27.8h, v14.8h\n" - "str q21, [x10, x5]\n" - "fmin v28.8h, v28.8h, v14.8h\n" - "fmin v29.8h, v29.8h, v14.8h\n" - "str q22, [x10, x23]\n" - "fmin v30.8h, v30.8h, v14.8h\n" - "fmin v31.8h, v31.8h, v14.8h\n" - "str q23, [x10, x22]\n" + "fmin v20.8h, v20.8h, v15.8h\n" + "fmin v22.8h, v22.8h, v15.8h\n" + "st1 { v31.8h }, [x10]\n" + "fmin v23.8h, v23.8h, v15.8h\n" + "fmin v19.8h, v19.8h, v15.8h\n" + "str q26, [x10, x5]\n" + "fmin v21.8h, v21.8h, v15.8h\n" + "fmin v27.8h, v27.8h, v15.8h\n" + "str q16, [x10, x23]\n" + "fmin v18.8h, v18.8h, v15.8h\n" + "fmin v30.8h, v30.8h, v15.8h\n" + "str q24, [x10, x22]\n" "add x10, x10, #0x10\n" - "st1 { v24.8h }, [x27]\n" - "str q25, [x27, x5]\n" - "str q26, [x27, x23]\n" - "str q27, [x27, x22]\n" + "st1 { v20.8h }, [x27]\n" + "str q22, [x27, x5]\n" + "str q23, [x27, x23]\n" + "str q19, [x27, x22]\n" "add x27, x27, #0x10\n" - "st1 { v28.8h }, [x24]\n" - "str q29, [x24, x5]\n" - "str q30, [x24, x23]\n" - "str q31, [x24, x22]\n" + "st1 { v21.8h }, [x24]\n" + "str q27, [x24, x5]\n" + "str q18, [x24, x23]\n" + "str q30, [x24, x22]\n" "add x24, x24, #0x10\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "mov v21.16b, v13.16b\n fmla v21.8h, v4.8h, v9.8h\n" - "mov v16.16b, v13.16b\n fmla v16.8h, v8.8h, v9.8h\n" - "mov v22.16b, v13.16b\n fmla v22.8h, v3.8h, v9.8h\n" - "mov v25.16b, v13.16b\n fmla v25.8h, v1.8h, v9.8h\n" - "mov v26.16b, v13.16b\n fmla v26.8h, v0.8h, v9.8h\n" - "fmla v21.8h, v5.8h, v12.8h\n" - "mov v17.16b, v13.16b\n fmla v17.8h, v7.8h, v9.8h\n" - "mov v18.16b, v13.16b\n fmla v18.8h, v6.8h, v9.8h\n" - "mov v20.16b, v13.16b\n fmla v20.8h, v5.8h, v9.8h\n" - "mov v24.16b, v13.16b\n fmla v24.8h, v2.8h, v9.8h\n" - "ldr q9, [x12, x17]\n" - "fmla v16.8h, v0.8h, v10.8h\n" - "ld1 { v10.8h }, [x26]\n" - "mov v19.16b, v13.16b\n fmla v19.8h, v2.8h, v11.8h\n" - "ldr q11, [x26, x25]\n" - "fmla v22.8h, v4.8h, v12.8h\n" - "fmla v25.8h, v2.8h, v12.8h\n" - "fmla v26.8h, v1.8h, v12.8h\n" - "mov v28.16b, v13.16b\n fmla v28.8h, v6.8h, v10.8h\n" - "ldr q10, [x12, x11]\n" - "fmla v21.8h, v7.8h, v9.8h\n" + "mov v16.16b, v14.16b\n fmla v16.8h, v4.8h, v9.8h\n" + "mov v23.16b, v14.16b\n fmla v23.8h, v8.8h, v9.8h\n" + "mov v31.16b, v14.16b\n fmla v31.8h, v3.8h, v9.8h\n" + "mov v30.16b, v14.16b\n fmla v30.8h, v1.8h, v9.8h\n" + "mov v18.16b, v14.16b\n fmla v18.8h, v0.8h, v9.8h\n" + "fmla v16.8h, v5.8h, v12.8h\n" + "mov v17.16b, v14.16b\n fmla v17.8h, v7.8h, v9.8h\n" + "mov v19.16b, v14.16b\n fmla v19.8h, v6.8h, v9.8h\n" + "mov v28.16b, v14.16b\n fmla v28.8h, v5.8h, v9.8h\n" + "mov v27.16b, v14.16b\n fmla v27.8h, v2.8h, v9.8h\n" + "ldr q24, [x12, x17]\n" + "fmla v23.8h, v0.8h, v10.8h\n" + "ld1 { v21.8h }, [x26]\n" + "mov v29.16b, v14.16b\n fmla v29.8h, v2.8h, v11.8h\n" + "ldr q20, [x26, x25]\n" + "fmla v31.8h, v4.8h, v12.8h\n" + "fmla v30.8h, v2.8h, v12.8h\n" + "fmla v18.8h, v1.8h, v12.8h\n" + "mov v26.16b, v14.16b\n fmla v26.8h, v6.8h, v21.8h\n" + "ldr q9, [x12, x11]\n" + "fmla v16.8h, v7.8h, v24.8h\n" "fmla v17.8h, v8.8h, v12.8h\n" - "fmla v18.8h, v7.8h, v12.8h\n" - "fmla v19.8h, v6.8h, v12.8h\n" - "mov v23.16b, v13.16b\n fmla v23.8h, v3.8h, v12.8h\n" - "mov v27.16b, v13.16b\n fmla v27.8h, v0.8h, v12.8h\n" - "ldr q12, [x7, x4]\n" - "mov v31.16b, v13.16b\n fmla v31.8h, v8.8h, v11.8h\n" - "ldr q11, [x7, x28]\n" - "fmla v22.8h, v6.8h, v9.8h\n" - "fmla v25.8h, v4.8h, v9.8h\n" - "fmla v26.8h, v3.8h, v9.8h\n" - "fmla v20.8h, v8.8h, v9.8h\n" - "fmla v24.8h, v5.8h, v9.8h\n" - "fmla v28.8h, v2.8h, v9.8h\n" - "fmla v21.8h, v8.8h, v10.8h\n" - "fmla v16.8h, v1.8h, v12.8h\n" - "fmla v17.8h, v0.8h, v12.8h\n" - "ldr q12, [x15, x25]\n" - "fmla v18.8h, v2.8h, v11.8h\n" - "fmla v19.8h, v1.8h, v11.8h\n" - "ld1 { v11.8h }, [x9]\n" - "fmla v22.8h, v7.8h, v10.8h\n" - "fmla v23.8h, v6.8h, v10.8h\n" - "fmla v25.8h, v5.8h, v10.8h\n" - "fmla v26.8h, v4.8h, v10.8h\n" - "fmla v27.8h, v3.8h, v10.8h\n" - "fmla v31.8h, v0.8h, v10.8h\n" - "fmla v24.8h, v6.8h, v11.8h\n" - "fmla v28.8h, v3.8h, v11.8h\n" - "ldr q11, [x9, x25]\n" - "fmla v19.8h, v5.8h, v12.8h\n" - "fmla v23.8h, v2.8h, v12.8h\n" - "ldr q12, [x15, x11]\n" - "fmla v27.8h, v8.8h, v11.8h\n" - "fmla v31.8h, v5.8h, v11.8h\n" - "mov v29.16b, v13.16b\n fmla v29.8h, v1.8h, v9.8h\n" - "mov v30.16b, v13.16b\n fmla v30.8h, v0.8h, v9.8h\n" - "ld1 { v9.8h }, [x15]\n" - "fmla v29.8h, v2.8h, v10.8h\n" - "fmla v30.8h, v1.8h, v10.8h\n" - "ldr q10, [x15, x17]\n" - "fmla v20.8h, v0.8h, v9.8h\n" - "fmla v21.8h, v1.8h, v10.8h\n" - "fmla v16.8h, v3.8h, v9.8h\n" - "ldr q11, [x26, x4]\n" - "fmla v17.8h, v4.8h, v10.8h\n" - "fmla v18.8h, v3.8h, v10.8h\n" - "fmla v22.8h, v0.8h, v10.8h\n" - "fmla v20.8h, v2.8h, v10.8h\n" - "fmla v21.8h, v2.8h, v12.8h\n" - "fmla v16.8h, v5.8h, v10.8h\n" - "ldr q10, [x14, x4]\n" - "fmla v17.8h, v5.8h, v12.8h\n" - "fmla v18.8h, v4.8h, v12.8h\n" - "fmla v19.8h, v3.8h, v12.8h\n" - "fmla v22.8h, v1.8h, v12.8h\n" - "fmla v23.8h, v0.8h, v12.8h\n" - "ldr q12, [x14, x28]\n" - "fmla v28.8h, v7.8h, v11.8h\n" - "fmla v29.8h, v6.8h, v11.8h\n" - "ldr q11, [x26, x28]\n" - "fmla v20.8h, v4.8h, v10.8h\n" - "fmla v21.8h, v3.8h, v10.8h\n" - "fmla v24.8h, v1.8h, v10.8h\n" - "fmla v25.8h, v0.8h, v10.8h\n" - "fmla v16.8h, v7.8h, v10.8h\n" - "fmla v17.8h, v6.8h, v10.8h\n" - "ldr q10, [x7, x17]\n" - "fmla v30.8h, v8.8h, v11.8h\n" - "fmla v31.8h, v7.8h, v11.8h\n" - "ldr q11, [x12, x4]\n" - "fmla v18.8h, v8.8h, v12.8h\n" "fmla v19.8h, v7.8h, v12.8h\n" - "fmla v22.8h, v5.8h, v12.8h\n" - "fmla v23.8h, v4.8h, v12.8h\n" - "fmla v26.8h, v2.8h, v12.8h\n" - "fmla v27.8h, v1.8h, v12.8h\n" - "ldr q12, [x7, x11]\n" + "fmla v29.8h, v6.8h, v12.8h\n" + "mov v11.16b, v14.16b\n fmla v11.8h, v3.8h, v12.8h\n" + "mov v10.16b, v14.16b\n fmla v10.8h, v0.8h, v12.8h\n" + "ldr q22, [x7, x4]\n" + "mov v25.16b, v14.16b\n fmla v25.8h, v8.8h, v20.8h\n" + "ldr q21, [x7, x28]\n" + "fmla v31.8h, v6.8h, v24.8h\n" + "fmla v30.8h, v4.8h, v24.8h\n" + "fmla v18.8h, v3.8h, v24.8h\n" + "mov v12.16b, v14.16b\n fmla v12.8h, v1.8h, v24.8h\n" + "fmla v14.8h, v0.8h, v24.8h\n" + "fmla v28.8h, v8.8h, v24.8h\n" + "fmla v27.8h, v5.8h, v24.8h\n" + "fmla v26.8h, v2.8h, v24.8h\n" + "ld1 { v24.8h }, [x15]\n" + "fmla v16.8h, v8.8h, v9.8h\n" + "fmla v23.8h, v1.8h, v22.8h\n" + "fmla v17.8h, v0.8h, v22.8h\n" + "ldr q22, [x15, x25]\n" + "fmla v19.8h, v2.8h, v21.8h\n" + "fmla v29.8h, v1.8h, v21.8h\n" + "ld1 { v20.8h }, [x9]\n" + "fmla v31.8h, v7.8h, v9.8h\n" + "fmla v11.8h, v6.8h, v9.8h\n" + "fmla v30.8h, v5.8h, v9.8h\n" + "fmla v18.8h, v4.8h, v9.8h\n" + "fmla v10.8h, v3.8h, v9.8h\n" + "fmla v12.8h, v2.8h, v9.8h\n" + "fmla v14.8h, v1.8h, v9.8h\n" + "fmla v25.8h, v0.8h, v9.8h\n" + "ldr q21, [x15, x17]\n" + "fmla v28.8h, v0.8h, v24.8h\n" + "fmla v27.8h, v6.8h, v20.8h\n" + "fmla v26.8h, v3.8h, v20.8h\n" + "ldr q20, [x9, x25]\n" + "fmla v16.8h, v1.8h, v21.8h\n" + "fmla v23.8h, v3.8h, v24.8h\n" + "fmla v29.8h, v5.8h, v22.8h\n" + "fmla v11.8h, v2.8h, v22.8h\n" + "ldr q22, [x15, x11]\n" + "fmla v17.8h, v4.8h, v21.8h\n" + "fmla v19.8h, v3.8h, v21.8h\n" + "fmla v31.8h, v0.8h, v21.8h\n" + "fmla v10.8h, v8.8h, v20.8h\n" + "fmla v25.8h, v5.8h, v20.8h\n" + "ldr q20, [x26, x4]\n" + "fmla v28.8h, v2.8h, v21.8h\n" + "fmla v16.8h, v2.8h, v22.8h\n" + "fmla v23.8h, v5.8h, v21.8h\n" + "ldr q21, [x14, x4]\n" + "fmla v17.8h, v5.8h, v22.8h\n" + "fmla v19.8h, v4.8h, v22.8h\n" + "fmla v29.8h, v3.8h, v22.8h\n" + "fmla v31.8h, v1.8h, v22.8h\n" + "fmla v11.8h, v0.8h, v22.8h\n" + "ldr q22, [x14, x28]\n" + "fmla v26.8h, v7.8h, v20.8h\n" + "fmla v12.8h, v6.8h, v20.8h\n" + "ldr q20, [x26, x28]\n" + "fmla v28.8h, v4.8h, v21.8h\n" + "fmla v16.8h, v3.8h, v21.8h\n" + "fmla v27.8h, v1.8h, v21.8h\n" + "fmla v30.8h, v0.8h, v21.8h\n" + "fmla v23.8h, v7.8h, v21.8h\n" + "fmla v17.8h, v6.8h, v21.8h\n" + "ldr q21, [x7, x17]\n" + "fmla v14.8h, v8.8h, v20.8h\n" + "fmla v25.8h, v7.8h, v20.8h\n" + "ldr q20, [x12, x4]\n" + "fmla v19.8h, v8.8h, v22.8h\n" + "fmla v29.8h, v7.8h, v22.8h\n" + "fmla v31.8h, v5.8h, v22.8h\n" + "fmla v11.8h, v4.8h, v22.8h\n" + "fmla v18.8h, v2.8h, v22.8h\n" + "fmla v10.8h, v1.8h, v22.8h\n" + "ldr q22, [x7, x11]\n" "add x7, x7, #0x10\n" - "fmla v20.8h, v7.8h, v11.8h\n" - "fmla v21.8h, v6.8h, v11.8h\n" - "fmla v24.8h, v4.8h, v11.8h\n" - "fmla v25.8h, v3.8h, v11.8h\n" - "fmla v28.8h, v1.8h, v11.8h\n" - "fmla v29.8h, v0.8h, v11.8h\n" - "ldr q11, [x12, x28]\n" - "fmla v16.8h, v2.8h, v10.8h\n" - "fmla v17.8h, v1.8h, v10.8h\n" - "fmla v18.8h, v0.8h, v10.8h\n" - "ld1 { v10.8h }, [x14]\n" - "fmla v30.8h, v2.8h, v11.8h\n" - "fmla v19.8h, v0.8h, v12.8h\n" - "fmla v20.8h, v3.8h, v10.8h\n" - "fmla v24.8h, v0.8h, v10.8h\n" - "fmla v22.8h, v8.8h, v11.8h\n" - "fmla v23.8h, v7.8h, v11.8h\n" - "fmla v26.8h, v5.8h, v11.8h\n" - "fmla v27.8h, v4.8h, v11.8h\n" - "fmla v31.8h, v1.8h, v11.8h\n" - "ldr q11, [x9, x17]\n" - "fmla v17.8h, v2.8h, v12.8h\n" - "fmla v18.8h, v1.8h, v12.8h\n" - "ldr q12, [x14, x25]\n" + "fmla v28.8h, v7.8h, v20.8h\n" + "fmla v16.8h, v6.8h, v20.8h\n" + "fmla v27.8h, v4.8h, v20.8h\n" + "fmla v30.8h, v3.8h, v20.8h\n" + "fmla v26.8h, v1.8h, v20.8h\n" + "fmla v12.8h, v0.8h, v20.8h\n" + "ldr q20, [x12, x28]\n" + "fmla v23.8h, v2.8h, v21.8h\n" + "fmla v17.8h, v1.8h, v21.8h\n" + "fmla v19.8h, v0.8h, v21.8h\n" + "ld1 { v21.8h }, [x14]\n" + "fmla v14.8h, v2.8h, v20.8h\n" + "fmla v29.8h, v0.8h, v22.8h\n" + "fmla v28.8h, v3.8h, v21.8h\n" + "fmla v27.8h, v0.8h, v21.8h\n" + "fmla v31.8h, v8.8h, v20.8h\n" + "fmla v11.8h, v7.8h, v20.8h\n" + "fmla v18.8h, v5.8h, v20.8h\n" + "fmla v10.8h, v4.8h, v20.8h\n" + "fmla v25.8h, v1.8h, v20.8h\n" + "ldr q24, [x9, x17]\n" + "fmla v17.8h, v2.8h, v22.8h\n" + "fmla v19.8h, v1.8h, v22.8h\n" + "ldr q20, [x14, x25]\n" "add x14, x14, #0x10\n" - "fmla v16.8h, v6.8h, v10.8h\n" - "ld1 { v10.8h }, [x12]\n" - "fmla v29.8h, v4.8h, v11.8h\n" - "fmla v30.8h, v3.8h, v11.8h\n" - "fmla v19.8h, v8.8h, v12.8h\n" - "fmla v23.8h, v5.8h, v12.8h\n" - "fmla v27.8h, v2.8h, v12.8h\n" - "ldr q12, [x12, x25]\n" + "fmla v23.8h, v6.8h, v21.8h\n" + "ld1 { v21.8h }, [x12]\n" + "fmla v12.8h, v4.8h, v24.8h\n" + "fmla v14.8h, v3.8h, v24.8h\n" + "fmla v29.8h, v8.8h, v20.8h\n" + "fmla v11.8h, v5.8h, v20.8h\n" + "fmla v10.8h, v2.8h, v20.8h\n" + "ldr q20, [x12, x25]\n" "add x12, x12, #0x10\n" - "fmla v20.8h, v6.8h, v10.8h\n" - "fmla v24.8h, v3.8h, v10.8h\n" - "fmla v28.8h, v0.8h, v10.8h\n" - "ldr q10, [x26, x17]\n" - "fmla v31.8h, v2.8h, v12.8h\n" - "fmla v29.8h, v7.8h, v10.8h\n" - "fmla v30.8h, v6.8h, v10.8h\n" - "fmla v24.8h, v8.8h, v11.8h\n" - "fmla v25.8h, v7.8h, v11.8h\n" - "fmla v26.8h, v6.8h, v11.8h\n" - "fmla v28.8h, v5.8h, v11.8h\n" - "ldr q11, [x9, x11]\n" - "fmla v27.8h, v5.8h, v12.8h\n" - "fmla v29.8h, v5.8h, v11.8h\n" - "fmla v30.8h, v4.8h, v11.8h\n" - "fmla v31.8h, v3.8h, v11.8h\n" - "fmla v23.8h, v8.8h, v12.8h\n" - "ldr q12, [x26, x11]\n" - "fmla v28.8h, v8.8h, v10.8h\n" - "ldr q10, [x15, x4]\n" - "fmla v25.8h, v8.8h, v11.8h\n" - "fmla v26.8h, v7.8h, v11.8h\n" + "fmla v28.8h, v6.8h, v21.8h\n" + "fmla v27.8h, v3.8h, v21.8h\n" + "fmla v26.8h, v0.8h, v21.8h\n" + "ldr q22, [x26, x17]\n" + "fmla v25.8h, v2.8h, v20.8h\n" + "fmla v12.8h, v7.8h, v22.8h\n" + "fmla v14.8h, v6.8h, v22.8h\n" + "fmla v27.8h, v8.8h, v24.8h\n" + "fmla v30.8h, v7.8h, v24.8h\n" + "fmla v18.8h, v6.8h, v24.8h\n" + "fmla v26.8h, v5.8h, v24.8h\n" + "ldr q21, [x9, x11]\n" + "fmla v10.8h, v5.8h, v20.8h\n" + "fmla v12.8h, v5.8h, v21.8h\n" + "fmla v14.8h, v4.8h, v21.8h\n" + "fmla v25.8h, v3.8h, v21.8h\n" + "fmla v11.8h, v8.8h, v20.8h\n" + "ldr q20, [x26, x11]\n" + "fmla v26.8h, v8.8h, v22.8h\n" + "ldr q9, [x15, x4]\n" + "fmla v30.8h, v8.8h, v21.8h\n" + "fmla v18.8h, v7.8h, v21.8h\n" "add x26, x26, #0x10\n" - "fmla v27.8h, v6.8h, v11.8h\n" - "ldr q11, [x15, x28]\n" - "fmla v29.8h, v8.8h, v12.8h\n" + "fmla v10.8h, v6.8h, v21.8h\n" + "ldr q21, [x15, x28]\n" + "fmla v12.8h, v8.8h, v20.8h\n" "add x15, x15, #0x10\n" - "fmla v30.8h, v7.8h, v12.8h\n" - "fmla v31.8h, v6.8h, v12.8h\n" - "ldr q12, [x9, x4]\n" - "fmla v16.8h, v4.8h, v10.8h\n" - "fmla v17.8h, v3.8h, v10.8h\n" - "fmax v16.8h, v16.8h, v15.8h\n" - "fmla v20.8h, v1.8h, v10.8h\n" - "fmla v21.8h, v0.8h, v10.8h\n" - "ldr q10, [x9, x28]\n" - "fmax v17.8h, v17.8h, v15.8h\n" - "fmla v18.8h, v5.8h, v11.8h\n" - "fmla v19.8h, v4.8h, v11.8h\n" - "fmax v18.8h, v18.8h, v15.8h\n" + "fmla v14.8h, v7.8h, v20.8h\n" + "fmla v25.8h, v6.8h, v20.8h\n" + "ldr q24, [x9, x4]\n" + "fmla v23.8h, v4.8h, v9.8h\n" + "fmla v17.8h, v3.8h, v9.8h\n" + "fmax v23.8h, v23.8h, v13.8h\n" + "fmla v28.8h, v1.8h, v9.8h\n" + "fmla v16.8h, v0.8h, v9.8h\n" + "ldr q0, [x9, x28]\n" + "fmax v17.8h, v17.8h, v13.8h\n" + "fmla v19.8h, v5.8h, v21.8h\n" + "fmla v29.8h, v4.8h, v21.8h\n" + "fmax v19.8h, v19.8h, v13.8h\n" "add x9, x9, #0x10\n" - "fmla v22.8h, v2.8h, v11.8h\n" - "fmla v23.8h, v1.8h, v11.8h\n" - "fmax v19.8h, v19.8h, v15.8h\n" - "fmla v24.8h, v7.8h, v12.8h\n" - "fmla v25.8h, v6.8h, v12.8h\n" - "fmax v20.8h, v20.8h, v15.8h\n" - "fmla v28.8h, v4.8h, v12.8h\n" - "fmla v29.8h, v3.8h, v12.8h\n" - "fmax v21.8h, v21.8h, v15.8h\n" - "fmla v26.8h, v8.8h, v10.8h\n" - "fmla v27.8h, v7.8h, v10.8h\n" - "fmax v22.8h, v22.8h, v15.8h\n" - "fmla v30.8h, v5.8h, v10.8h\n" - "fmla v31.8h, v4.8h, v10.8h\n" - "fmax v23.8h, v23.8h, v15.8h\n" - "fmax v24.8h, v24.8h, v15.8h\n" - "fmax v25.8h, v25.8h, v15.8h\n" - "fmax v26.8h, v26.8h, v15.8h\n" - "fmax v27.8h, v27.8h, v15.8h\n" - "fmax v28.8h, v28.8h, v15.8h\n" - "fmax v29.8h, v29.8h, v15.8h\n" - "fmax v30.8h, v30.8h, v15.8h\n" - "fmax v31.8h, v31.8h, v15.8h\n" - "fmin v16.8h, v16.8h, v14.8h\n" - "fmin v17.8h, v17.8h, v14.8h\n" - "st1 { v16.8h }, [x8]\n" - "fmin v18.8h, v18.8h, v14.8h\n" - "fmin v19.8h, v19.8h, v14.8h\n" + "fmla v31.8h, v2.8h, v21.8h\n" + "fmla v11.8h, v1.8h, v21.8h\n" + "fmax v29.8h, v29.8h, v13.8h\n" + "fmla v27.8h, v7.8h, v24.8h\n" + "fmla v30.8h, v6.8h, v24.8h\n" + "fmax v28.8h, v28.8h, v13.8h\n" + "fmla v26.8h, v4.8h, v24.8h\n" + "fmla v12.8h, v3.8h, v24.8h\n" + "fmax v16.8h, v16.8h, v13.8h\n" + "fmla v18.8h, v8.8h, v0.8h\n" + "fmla v10.8h, v7.8h, v0.8h\n" + "fmax v31.8h, v31.8h, v13.8h\n" + "fmla v14.8h, v5.8h, v0.8h\n" + "fmla v25.8h, v4.8h, v0.8h\n" + "fmax v11.8h, v11.8h, v13.8h\n" + "fmax v27.8h, v27.8h, v13.8h\n" + "fmax v30.8h, v30.8h, v13.8h\n" + "fmax v18.8h, v18.8h, v13.8h\n" + "fmax v10.8h, v10.8h, v13.8h\n" + "fmax v26.8h, v26.8h, v13.8h\n" + "fmax v12.8h, v12.8h, v13.8h\n" + "fmax v14.8h, v14.8h, v13.8h\n" + "fmax v25.8h, v25.8h, v13.8h\n" + "fmin v23.8h, v23.8h, v15.8h\n" + "fmin v17.8h, v17.8h, v15.8h\n" + "st1 { v23.8h }, [x8]\n" + "fmin v19.8h, v19.8h, v15.8h\n" + "fmin v29.8h, v29.8h, v15.8h\n" "str q17, [x8, x5]\n" - "fmin v20.8h, v20.8h, v14.8h\n" - "fmin v21.8h, v21.8h, v14.8h\n" - "str q18, [x8, x23]\n" - "fmin v22.8h, v22.8h, v14.8h\n" - "fmin v23.8h, v23.8h, v14.8h\n" - "str q19, [x8, x22]\n" + "fmin v28.8h, v28.8h, v15.8h\n" + "fmin v16.8h, v16.8h, v15.8h\n" + "str q19, [x8, x23]\n" + "fmin v31.8h, v31.8h, v15.8h\n" + "fmin v11.8h, v11.8h, v15.8h\n" + "str q29, [x8, x22]\n" "add x8, x8, #0x10\n" - "fmin v24.8h, v24.8h, v14.8h\n" - "fmin v25.8h, v25.8h, v14.8h\n" - "st1 { v20.8h }, [x10]\n" - "fmin v26.8h, v26.8h, v14.8h\n" - "fmin v27.8h, v27.8h, v14.8h\n" - "str q21, [x10, x5]\n" - "fmin v28.8h, v28.8h, v14.8h\n" - "fmin v29.8h, v29.8h, v14.8h\n" - "str q22, [x10, x23]\n" - "fmin v30.8h, v30.8h, v14.8h\n" - "fmin v31.8h, v31.8h, v14.8h\n" - "str q23, [x10, x22]\n" + "fmin v27.8h, v27.8h, v15.8h\n" + "fmin v30.8h, v30.8h, v15.8h\n" + "st1 { v28.8h }, [x10]\n" + "fmin v18.8h, v18.8h, v15.8h\n" + "fmin v10.8h, v10.8h, v15.8h\n" + "str q16, [x10, x5]\n" + "fmin v26.8h, v26.8h, v15.8h\n" + "fmin v12.8h, v12.8h, v15.8h\n" + "str q31, [x10, x23]\n" + "fmin v14.8h, v14.8h, v15.8h\n" + "fmin v25.8h, v25.8h, v15.8h\n" + "str q11, [x10, x22]\n" "add x10, x10, #0x10\n" - "st1 { v24.8h }, [x27]\n" - "str q25, [x27, x5]\n" - "str q26, [x27, x23]\n" - "str q27, [x27, x22]\n" + "st1 { v27.8h }, [x27]\n" + "str q30, [x27, x5]\n" + "str q18, [x27, x23]\n" + "str q10, [x27, x22]\n" "add x27, x27, #0x10\n" - "st1 { v28.8h }, [x24]\n" - "str q29, [x24, x5]\n" - "str q30, [x24, x23]\n" - "str q31, [x24, x22]\n" + "st1 { v26.8h }, [x24]\n" + "str q12, [x24, x5]\n" + "str q14, [x24, x23]\n" + "str q25, [x24, x22]\n" "add x24, x24, #0x10\n" "4:" // Tile loop: Oddments "tst %x[n_channels], #0x7\n" "beq 141f\n" - "ldr q13, [x16, #0x0]\n" + "ldr q14, [x16, #0x0]\n" "ldr q0, [x16, #0x10]\n" "add x23, x14, x17\n" "add x22, x7, XZR\n" @@ -699,27 +699,27 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "ldr h11, [x21, #0x0]\n" "ldr h12, [x20, #0x0]\n" "8:" // Tile loop: Oddments: Load inputs: (2, 2), (0, 0), (0, 5), (2, 3): Bit 2: End - "mov v16.16b, v13.16b\n fmla v16.8h, v8.8h, v9.8h\n" - "mov v17.16b, v13.16b\n fmla v17.8h, v7.8h, v9.8h\n" + "mov v16.16b, v14.16b\n fmla v16.8h, v8.8h, v9.8h\n" + "mov v17.16b, v14.16b\n fmla v17.8h, v7.8h, v9.8h\n" "add x20, x26, XZR\n" - "mov v18.16b, v13.16b\n fmla v18.8h, v6.8h, v9.8h\n" - "mov v21.16b, v13.16b\n fmla v21.8h, v4.8h, v9.8h\n" - "mov v22.16b, v13.16b\n fmla v22.8h, v3.8h, v9.8h\n" - "mov v25.16b, v13.16b\n fmla v25.8h, v1.8h, v9.8h\n" - "mov v26.16b, v13.16b\n fmla v26.8h, v0.8h, v9.8h\n" - "mov v19.16b, v13.16b\n fmla v19.8h, v2.8h, v11.8h\n" - "mov v20.16b, v13.16b\n fmla v20.8h, v5.8h, v9.8h\n" - "mov v24.16b, v13.16b\n fmla v24.8h, v2.8h, v9.8h\n" + "mov v18.16b, v14.16b\n fmla v18.8h, v6.8h, v9.8h\n" + "mov v21.16b, v14.16b\n fmla v21.8h, v4.8h, v9.8h\n" + "mov v22.16b, v14.16b\n fmla v22.8h, v3.8h, v9.8h\n" + "mov v25.16b, v14.16b\n fmla v25.8h, v1.8h, v9.8h\n" + "mov v26.16b, v14.16b\n fmla v26.8h, v0.8h, v9.8h\n" + "mov v19.16b, v14.16b\n fmla v19.8h, v2.8h, v11.8h\n" + "mov v20.16b, v14.16b\n fmla v20.8h, v5.8h, v9.8h\n" + "mov v24.16b, v14.16b\n fmla v24.8h, v2.8h, v9.8h\n" "fmla v16.8h, v0.8h, v10.8h\n" "fmla v17.8h, v8.8h, v12.8h\n" "fmla v18.8h, v7.8h, v12.8h\n" "fmla v19.8h, v6.8h, v12.8h\n" "fmla v21.8h, v5.8h, v12.8h\n" "fmla v22.8h, v4.8h, v12.8h\n" - "mov v23.16b, v13.16b\n fmla v23.8h, v3.8h, v12.8h\n" + "mov v23.16b, v14.16b\n fmla v23.8h, v3.8h, v12.8h\n" "fmla v25.8h, v2.8h, v12.8h\n" "fmla v26.8h, v1.8h, v12.8h\n" - "mov v27.16b, v13.16b\n fmla v27.8h, v0.8h, v12.8h\n" + "mov v27.16b, v14.16b\n fmla v27.8h, v0.8h, v12.8h\n" "tbz %x[n_channels], #2, 10f\n" "ldr d10, [x20], #0x8\n" "tbz %x[n_channels], #1, 9f\n" @@ -740,7 +740,7 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "11:" // Tile loop: Oddments: Load inputs: (5, 0): Bit 2: Unset: Bit 1: Unset "ldr h10, [x20, #0x0]\n" "12:" // Tile loop: Oddments: Load inputs: (5, 0): Bit 2: End - "mov v28.16b, v13.16b\n fmla v28.8h, v6.8h, v10.8h\n" + "mov v28.16b, v14.16b\n fmla v28.8h, v6.8h, v10.8h\n" "add x20, x26, x25\n" "tbz %x[n_channels], #2, 14f\n" "ldr d11, [x20], #0x8\n" @@ -762,7 +762,7 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "15:" // Tile loop: Oddments: Load inputs: (5, 5): Bit 2: Unset: Bit 1: Unset "ldr h11, [x20, #0x0]\n" "16:" // Tile loop: Oddments: Load inputs: (5, 5): Bit 2: End - "mov v31.16b, v13.16b\n fmla v31.8h, v8.8h, v11.8h\n" + "mov v31.16b, v14.16b\n fmla v31.8h, v8.8h, v11.8h\n" "add x20, x12, x17\n" "tbz %x[n_channels], #2, 18f\n" "ldr d9, [x20], #0x8\n" @@ -792,8 +792,8 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "fmla v25.8h, v4.8h, v9.8h\n" "fmla v26.8h, v3.8h, v9.8h\n" "fmla v28.8h, v2.8h, v9.8h\n" - "mov v29.16b, v13.16b\n fmla v29.8h, v1.8h, v9.8h\n" - "mov v30.16b, v13.16b\n fmla v30.8h, v0.8h, v9.8h\n" + "mov v29.16b, v14.16b\n fmla v29.8h, v1.8h, v9.8h\n" + "mov v30.16b, v14.16b\n fmla v30.8h, v0.8h, v9.8h\n" "tbz %x[n_channels], #2, 22f\n" "ldr d12, [x20], #0x8\n" "tbz %x[n_channels], #1, 21f\n" @@ -1513,40 +1513,40 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "136:" // Tile loop: Oddments: Load inputs: (4, 4): Bit 2: End "fmla v26.8h, v8.8h, v10.8h\n" "fmla v27.8h, v7.8h, v10.8h\n" - "fmax v16.8h, v16.8h, v15.8h\n" + "fmax v16.8h, v16.8h, v13.8h\n" "fmla v30.8h, v5.8h, v10.8h\n" "fmla v31.8h, v4.8h, v10.8h\n" - "fmax v17.8h, v17.8h, v15.8h\n" - "fmax v18.8h, v18.8h, v15.8h\n" - "fmax v19.8h, v19.8h, v15.8h\n" - "fmax v20.8h, v20.8h, v15.8h\n" - "fmax v21.8h, v21.8h, v15.8h\n" - "fmax v22.8h, v22.8h, v15.8h\n" - "fmax v23.8h, v23.8h, v15.8h\n" - "fmax v24.8h, v24.8h, v15.8h\n" - "fmax v25.8h, v25.8h, v15.8h\n" - "fmax v26.8h, v26.8h, v15.8h\n" - "fmax v27.8h, v27.8h, v15.8h\n" - "fmax v28.8h, v28.8h, v15.8h\n" - "fmax v29.8h, v29.8h, v15.8h\n" - "fmax v30.8h, v30.8h, v15.8h\n" - "fmax v31.8h, v31.8h, v15.8h\n" - "fmin v16.8h, v16.8h, v14.8h\n" - "fmin v17.8h, v17.8h, v14.8h\n" - "fmin v18.8h, v18.8h, v14.8h\n" - "fmin v19.8h, v19.8h, v14.8h\n" - "fmin v20.8h, v20.8h, v14.8h\n" - "fmin v21.8h, v21.8h, v14.8h\n" - "fmin v22.8h, v22.8h, v14.8h\n" - "fmin v23.8h, v23.8h, v14.8h\n" - "fmin v24.8h, v24.8h, v14.8h\n" - "fmin v25.8h, v25.8h, v14.8h\n" - "fmin v26.8h, v26.8h, v14.8h\n" - "fmin v27.8h, v27.8h, v14.8h\n" - "fmin v28.8h, v28.8h, v14.8h\n" - "fmin v29.8h, v29.8h, v14.8h\n" - "fmin v30.8h, v30.8h, v14.8h\n" - "fmin v31.8h, v31.8h, v14.8h\n" + "fmax v17.8h, v17.8h, v13.8h\n" + "fmax v18.8h, v18.8h, v13.8h\n" + "fmax v19.8h, v19.8h, v13.8h\n" + "fmax v20.8h, v20.8h, v13.8h\n" + "fmax v21.8h, v21.8h, v13.8h\n" + "fmax v22.8h, v22.8h, v13.8h\n" + "fmax v23.8h, v23.8h, v13.8h\n" + "fmax v24.8h, v24.8h, v13.8h\n" + "fmax v25.8h, v25.8h, v13.8h\n" + "fmax v26.8h, v26.8h, v13.8h\n" + "fmax v27.8h, v27.8h, v13.8h\n" + "fmax v28.8h, v28.8h, v13.8h\n" + "fmax v29.8h, v29.8h, v13.8h\n" + "fmax v30.8h, v30.8h, v13.8h\n" + "fmax v31.8h, v31.8h, v13.8h\n" + "fmin v16.8h, v16.8h, v15.8h\n" + "fmin v17.8h, v17.8h, v15.8h\n" + "fmin v18.8h, v18.8h, v15.8h\n" + "fmin v19.8h, v19.8h, v15.8h\n" + "fmin v20.8h, v20.8h, v15.8h\n" + "fmin v21.8h, v21.8h, v15.8h\n" + "fmin v22.8h, v22.8h, v15.8h\n" + "fmin v23.8h, v23.8h, v15.8h\n" + "fmin v24.8h, v24.8h, v15.8h\n" + "fmin v25.8h, v25.8h, v15.8h\n" + "fmin v26.8h, v26.8h, v15.8h\n" + "fmin v27.8h, v27.8h, v15.8h\n" + "fmin v28.8h, v28.8h, v15.8h\n" + "fmin v29.8h, v29.8h, v15.8h\n" + "fmin v30.8h, v30.8h, v15.8h\n" + "fmin v31.8h, v31.8h, v15.8h\n" "tbz %x[n_channels], #2, 138f\n" "mov x23, x8\n" "mov x22, x10\n" @@ -1712,7 +1712,6 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "st1 { v27.h }[0], [x21]\n" "st1 { v31.h }[0], [x20]\n" "140:" // Tile loop: Oddments: Store: Bit 2: End - "141:" // Tile loop: End "ldr x26, [%x[params_struct], %[offsetof_args_tile_j]]\n" "ldr x27, [%x[params_struct], %[offsetof_args_tile_i]]\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp index 16326150fd3e0453cdd4911157c5ac9cefc3d0b3..96feeeeece7465be847a85e95c7851b9b751c446 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp @@ -98,629 +98,629 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( activation_min, activation_max); __asm__ __volatile__( - "mov x8, #0x10\n" // cntb _, ALL, #1 - "lsr x17, %x[n_channels], #0x3\n" - "ldr x16, [%x[params_struct], %[offsetof_args_outptrs]]\n" - "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" + "mov x6, #0x10\n" // cntb _, ALL, #1 + "lsr x7, %x[n_channels], #0x3\n" + "ldr x8, [%x[params_struct], %[offsetof_args_outptrs]]\n" + "ldr x17, [%x[params_struct], %[offsetof_args_params]]\n" "add x20, %x[params_struct], %[offsetof_args_min]\n" - "ld1r { v15.8h }, [x20]\n" + "ld1r { v13.8h }, [x20]\n" "add x20, %x[params_struct], %[offsetof_args_max]\n" "ld1r { v14.8h }, [x20]\n" - "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n" - "mov x13, #0x0\n" - "sub x12, XZR, x8\n" - "cbz x17, 3f\n" - "ldr q13, [x15, #0x0]\n" - "ldr q0, [x15, #0x10]\n" - "cmp x8, x17, LSL #4\n" - "ldr q1, [x15, #0x20]\n" - "ldr q2, [x15, #0x30]\n" - "ldr q3, [x15, #0x40]\n" - "ldr q4, [x15, #0x50]\n" - "ldr q5, [x15, #0x60]\n" - "ldr q6, [x15, #0x70]\n" - "ldr q7, [x15, #0x80]\n" - "ldr q8, [x15, #0x90]\n" - "add x15, x15, #0xa0\n" - "ldp x11, x10, [x14, #0x0]\n" - "ldr q9, [x11, x13]\n" - "ldr q10, [x10, x13]\n" - "ldp x9, x28, [x14, #0x10]\n" - "ldr q11, [x9, x13]\n" - "ldr q12, [x28, x13]\n" + "add x16, %x[params_struct], %[offsetof_Args_inptrs]\n" + "mov x15, #0x0\n" + "sub x14, XZR, x6\n" + "cbz x7, 3f\n" + "ldr q30, [x17, #0x0]\n" + "ldr q0, [x17, #0x10]\n" + "cmp x6, x7, LSL #4\n" + "ldr q1, [x17, #0x20]\n" + "ldr q2, [x17, #0x30]\n" + "ldr q3, [x17, #0x40]\n" + "ldr q4, [x17, #0x50]\n" + "ldr q5, [x17, #0x60]\n" + "ldr q6, [x17, #0x70]\n" + "ldr q7, [x17, #0x80]\n" + "ldr q8, [x17, #0x90]\n" + "add x17, x17, #0xa0\n" + "ldp x21, x20, [x16, #0x0]\n" + "ldr q9, [x21, x15]\n" + "ldr q10, [x20, x15]\n" + "ldp x21, x20, [x16, #0x10]\n" + "ldr q11, [x21, x15]\n" + "ldr q12, [x20, x15]\n" "bge 2f\n" "1:" // Channel loop - "mov v21.16b, v13.16b\n fmla v21.8h, v4.8h, v9.8h\n" - "mov v16.16b, v13.16b\n fmla v16.8h, v8.8h, v9.8h\n" - "ldr x27, [x14, #0x20]\n" - "ldr x26, [x14, #0x30]\n" - "mov v22.16b, v13.16b\n fmla v22.8h, v3.8h, v9.8h\n" - "mov v25.16b, v13.16b\n fmla v25.8h, v1.8h, v9.8h\n" - "ldr x25, [x14, #0x28]\n" - "ldr x24, [x14, #0x38]\n" - "mov v26.16b, v13.16b\n fmla v26.8h, v0.8h, v9.8h\n" - "mov v17.16b, v13.16b\n fmla v17.8h, v7.8h, v9.8h\n" - "ldr x11, [x14, #0x40]\n" - "ldr x10, [x14, #0x48]\n" - "mov v18.16b, v13.16b\n fmla v18.8h, v6.8h, v9.8h\n" - "fmla v21.8h, v5.8h, v12.8h\n" - "ldr x9, [x14, #0x50]\n" - "ldr x28, [x14, #0x58]\n" - "mov v20.16b, v13.16b\n fmla v20.8h, v5.8h, v9.8h\n" - "mov v24.16b, v13.16b\n fmla v24.8h, v2.8h, v9.8h\n" - "ldr q9, [x26, x13]\n" - "ldr x26, [x14, #0x70]\n" - "fmla v16.8h, v0.8h, v10.8h\n" - "ldr q10, [x27, x13]\n" - "mov v19.16b, v13.16b\n fmla v19.8h, v2.8h, v11.8h\n" - "ldr q11, [x25, x13]\n" - "fmla v22.8h, v4.8h, v12.8h\n" - "fmla v25.8h, v2.8h, v12.8h\n" - "ldr x27, [x14, #0x60]\n" - "ldr x25, [x14, #0x68]\n" - "fmla v26.8h, v1.8h, v12.8h\n" - "fmla v17.8h, v8.8h, v12.8h\n" - "ldr x23, [x16, #0x0]\n" - "ldr x22, [x16, #0x8]\n" - "fmla v18.8h, v7.8h, v12.8h\n" - "mov v28.16b, v13.16b\n fmla v28.8h, v6.8h, v10.8h\n" - "ldr q10, [x10, x13]\n" - "ldr x10, [x14, #0x88]\n" - "fmla v21.8h, v7.8h, v9.8h\n" - "fmla v19.8h, v6.8h, v12.8h\n" - "ldr x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x18]\n" - "mov v23.16b, v13.16b\n fmla v23.8h, v3.8h, v12.8h\n" - "mov v27.16b, v13.16b\n fmla v27.8h, v0.8h, v12.8h\n" - "ldr q12, [x24, x13]\n" - "ldr x24, [x14, #0x78]\n" - "mov v31.16b, v13.16b\n fmla v31.8h, v8.8h, v11.8h\n" - "ldr q11, [x11, x13]\n" - "fmla v22.8h, v6.8h, v9.8h\n" - "ldr x11, [x14, #0x80]\n" - "fmla v25.8h, v4.8h, v9.8h\n" - "fmla v26.8h, v3.8h, v9.8h\n" - "add x12, x12, #0x10\n" - "fmla v20.8h, v8.8h, v9.8h\n" + "mov v23.16b, v30.16b\n fmla v23.8h, v4.8h, v9.8h\n" + "mov v17.16b, v30.16b\n fmla v17.8h, v8.8h, v9.8h\n" + "ldr x27, [x16, #0x20]\n" + "ldr x24, [x16, #0x30]\n" + "mov v25.16b, v30.16b\n fmla v25.8h, v3.8h, v9.8h\n" + "mov v28.16b, v30.16b\n fmla v28.8h, v1.8h, v9.8h\n" + "ldr x23, [x16, #0x28]\n" + "ldr x22, [x16, #0x38]\n" + "mov v20.16b, v30.16b\n fmla v20.8h, v0.8h, v9.8h\n" + "mov v16.16b, v30.16b\n fmla v16.8h, v7.8h, v9.8h\n" + "ldr x26, [x16, #0x40]\n" + "ldr x20, [x16, #0x48]\n" + "mov v15.16b, v30.16b\n fmla v15.8h, v6.8h, v9.8h\n" + "fmla v23.8h, v5.8h, v12.8h\n" + "ldr x25, [x16, #0x50]\n" + "ldr x21, [x16, #0x58]\n" + "mov v27.16b, v30.16b\n fmla v27.8h, v5.8h, v9.8h\n" + "mov v31.16b, v30.16b\n fmla v31.8h, v2.8h, v9.8h\n" + "ldr q9, [x24, x15]\n" + "ldr x13, [x16, #0x70]\n" + "fmla v17.8h, v0.8h, v10.8h\n" + "ldr q22, [x27, x15]\n" + "mov v10.16b, v30.16b\n fmla v10.8h, v2.8h, v11.8h\n" + "ldr q18, [x23, x15]\n" + "fmla v25.8h, v4.8h, v12.8h\n" + "fmla v28.8h, v2.8h, v12.8h\n" + "ldr x24, [x16, #0x60]\n" + "ldr x23, [x16, #0x68]\n" + "fmla v20.8h, v1.8h, v12.8h\n" + "fmla v16.8h, v8.8h, v12.8h\n" + "ldr x12, [x8, #0x0]\n" + "ldr x11, [x8, #0x8]\n" + "fmla v15.8h, v7.8h, v12.8h\n" + "mov v29.16b, v30.16b\n fmla v29.8h, v6.8h, v22.8h\n" + "ldr q22, [x20, x15]\n" + "ldr x28, [x16, #0x88]\n" + "fmla v23.8h, v7.8h, v9.8h\n" + "fmla v10.8h, v6.8h, v12.8h\n" + "ldr x10, [x8, #0x10]\n" + "ldr x9, [x8, #0x18]\n" + "mov v21.16b, v30.16b\n fmla v21.8h, v3.8h, v12.8h\n" + "mov v19.16b, v30.16b\n fmla v19.8h, v0.8h, v12.8h\n" + "ldr q11, [x22, x15]\n" + "ldr x22, [x16, #0x78]\n" + "mov v24.16b, v30.16b\n fmla v24.8h, v8.8h, v18.8h\n" + "ldr q12, [x26, x15]\n" + "fmla v25.8h, v6.8h, v9.8h\n" + "ldr x20, [x16, #0x80]\n" + "fmla v28.8h, v4.8h, v9.8h\n" + "fmla v20.8h, v3.8h, v9.8h\n" + "add x14, x14, #0x10\n" + "mov v26.16b, v30.16b\n fmla v26.8h, v1.8h, v9.8h\n" + "mov v18.16b, v30.16b\n fmla v18.8h, v0.8h, v9.8h\n" + "ldr q30, [x17, #0x0]\n" + "fmla v27.8h, v8.8h, v9.8h\n" + "fmla v31.8h, v5.8h, v9.8h\n" + "fmla v29.8h, v2.8h, v9.8h\n" + "ldr q9, [x25, x15]\n" + "fmla v17.8h, v1.8h, v11.8h\n" + "ldr x27, [x16, #0x90]\n" + "fmla v16.8h, v0.8h, v11.8h\n" + "ldr q11, [x21, x15]\n" + "fmla v15.8h, v2.8h, v12.8h\n" + "ldr x21, [x16, #0x98]\n" + "fmla v23.8h, v8.8h, v22.8h\n" + "fmla v10.8h, v1.8h, v12.8h\n" + "ldr q12, [x24, x15]\n" + "ldr x26, [x16, #0xa0]\n" + "fmla v25.8h, v7.8h, v22.8h\n" + "fmla v21.8h, v6.8h, v22.8h\n" + "fmla v28.8h, v5.8h, v22.8h\n" + "fmla v20.8h, v4.8h, v22.8h\n" + "fmla v19.8h, v3.8h, v22.8h\n" + "fmla v26.8h, v2.8h, v22.8h\n" + "fmla v18.8h, v1.8h, v22.8h\n" + "fmla v24.8h, v0.8h, v22.8h\n" + "ldr q22, [x23, x15]\n" + "ldr x25, [x16, #0xa8]\n" + "fmla v17.8h, v3.8h, v9.8h\n" + "fmla v27.8h, v0.8h, v9.8h\n" + "fmla v31.8h, v6.8h, v12.8h\n" + "fmla v29.8h, v3.8h, v12.8h\n" + "ldr q9, [x13, x15]\n" + "ldr x24, [x16, #0xb0]\n" + "fmla v16.8h, v4.8h, v22.8h\n" + "fmla v15.8h, v3.8h, v22.8h\n" + "fmla v23.8h, v1.8h, v22.8h\n" + "fmla v10.8h, v5.8h, v11.8h\n" + "fmla v21.8h, v2.8h, v11.8h\n" + "ldr q12, [x22, x15]\n" + "fmla v25.8h, v0.8h, v22.8h\n" + "ldr x23, [x16, #0xb8]\n" + "fmla v19.8h, v8.8h, v9.8h\n" "fmla v24.8h, v5.8h, v9.8h\n" - "fmla v28.8h, v2.8h, v9.8h\n" - "fmla v16.8h, v1.8h, v12.8h\n" - "fmla v17.8h, v0.8h, v12.8h\n" - "ldr q12, [x28, x13]\n" - "fmla v18.8h, v2.8h, v11.8h\n" - "ldr x28, [x14, #0x98]\n" - "fmla v21.8h, v8.8h, v10.8h\n" - "fmla v19.8h, v1.8h, v11.8h\n" - "ldr q11, [x27, x13]\n" - "ldr x27, [x14, #0xa0]\n" - "fmla v22.8h, v7.8h, v10.8h\n" - "fmla v23.8h, v6.8h, v10.8h\n" - "fmla v25.8h, v5.8h, v10.8h\n" - "fmla v26.8h, v4.8h, v10.8h\n" - "fmla v27.8h, v3.8h, v10.8h\n" - "fmla v31.8h, v0.8h, v10.8h\n" - "fmla v24.8h, v6.8h, v11.8h\n" - "fmla v28.8h, v3.8h, v11.8h\n" - "ldr q11, [x26, x13]\n" - "ldr x26, [x14, #0xb0]\n" - "fmla v19.8h, v5.8h, v12.8h\n" + "ldr q11, [x20, x15]\n" + "ldr x22, [x16, #0xc0]\n" + "fmla v17.8h, v5.8h, v22.8h\n" + "fmla v27.8h, v2.8h, v22.8h\n" + "ldr q22, [x28, x15]\n" + "ldr x20, [x16, #0xc8]\n" + "fmla v16.8h, v5.8h, v12.8h\n" + "fmla v15.8h, v4.8h, v12.8h\n" "fmla v23.8h, v2.8h, v12.8h\n" - "ldr q12, [x24, x13]\n" - "ldr x24, [x14, #0xb8]\n" - "fmla v27.8h, v8.8h, v11.8h\n" - "fmla v31.8h, v5.8h, v11.8h\n" - "mov v29.16b, v13.16b\n fmla v29.8h, v1.8h, v9.8h\n" - "mov v30.16b, v13.16b\n fmla v30.8h, v0.8h, v9.8h\n" - "ldr q9, [x9, x13]\n" - "ldr x9, [x14, #0x90]\n" - "fmla v29.8h, v2.8h, v10.8h\n" - "fmla v30.8h, v1.8h, v10.8h\n" - "ldr q10, [x25, x13]\n" - "ldr x25, [x14, #0xa8]\n" - "fmla v16.8h, v3.8h, v9.8h\n" - "fmla v20.8h, v0.8h, v9.8h\n" - "ldr q11, [x11, x13]\n" - "ldr x11, [x14, #0xc0]\n" - "fmla v17.8h, v4.8h, v10.8h\n" - "fmla v18.8h, v3.8h, v10.8h\n" - "fmla v21.8h, v1.8h, v10.8h\n" - "fmla v22.8h, v0.8h, v10.8h\n" - "fmla v16.8h, v5.8h, v10.8h\n" - "fmla v20.8h, v2.8h, v10.8h\n" - "ldr q10, [x10, x13]\n" - "ldr x10, [x14, #0xc8]\n" - "fmla v17.8h, v5.8h, v12.8h\n" - "fmla v18.8h, v4.8h, v12.8h\n" - "fmla v21.8h, v2.8h, v12.8h\n" - "fmla v19.8h, v3.8h, v12.8h\n" - "fmla v22.8h, v1.8h, v12.8h\n" - "fmla v23.8h, v0.8h, v12.8h\n" - "ldr q12, [x28, x13]\n" - "ldr x28, [x14, #0xd8]\n" - "fmla v28.8h, v7.8h, v11.8h\n" - "fmla v29.8h, v6.8h, v11.8h\n" - "ldr q11, [x9, x13]\n" - "ldr x9, [x14, #0xd0]\n" - "fmla v16.8h, v7.8h, v10.8h\n" - "fmla v17.8h, v6.8h, v10.8h\n" - "fmla v20.8h, v4.8h, v10.8h\n" - "fmla v21.8h, v3.8h, v10.8h\n" - "fmla v24.8h, v1.8h, v10.8h\n" - "fmla v25.8h, v0.8h, v10.8h\n" - "ldr q10, [x27, x13]\n" - "ldr x27, [x14, #0xe0]\n" - "fmla v18.8h, v8.8h, v12.8h\n" - "fmla v30.8h, v8.8h, v11.8h\n" - "fmla v31.8h, v7.8h, v11.8h\n" - "ldr q11, [x25, x13]\n" - "fmla v27.8h, v1.8h, v12.8h\n" - "ldr x25, [x14, #0xe8]\n" - "fmla v19.8h, v7.8h, v12.8h\n" - "fmla v22.8h, v5.8h, v12.8h\n" - "fmla v23.8h, v4.8h, v12.8h\n" - "fmla v26.8h, v2.8h, v12.8h\n" - "ldr q12, [x26, x13]\n" - "ldr x26, [x14, #0xf0]\n" - "fmla v16.8h, v2.8h, v10.8h\n" - "fmla v17.8h, v1.8h, v10.8h\n" - "fmla v18.8h, v0.8h, v10.8h\n" - "ldr q10, [x24, x13]\n" - "fmla v20.8h, v7.8h, v11.8h\n" - "ldr x24, [x14, #0xf8]\n" - "fmla v21.8h, v6.8h, v11.8h\n" - "fmla v24.8h, v4.8h, v11.8h\n" - "fmla v25.8h, v3.8h, v11.8h\n" - "fmla v28.8h, v1.8h, v11.8h\n" - "fmla v29.8h, v0.8h, v11.8h\n" - "ldr q11, [x11, x13]\n" - "fmla v27.8h, v4.8h, v11.8h\n" - "ldr x11, [x14, #0x100]\n" - "fmla v30.8h, v2.8h, v11.8h\n" - "fmla v17.8h, v2.8h, v12.8h\n" - "fmla v18.8h, v1.8h, v12.8h\n" - "fmla v19.8h, v0.8h, v12.8h\n" - "ldr q12, [x10, x13]\n" - "ldr x10, [x14, #0x108]\n" - "fmla v16.8h, v6.8h, v10.8h\n" - "fmla v20.8h, v3.8h, v10.8h\n" - "fmla v24.8h, v0.8h, v10.8h\n" - "ldr q10, [x9, x13]\n" - "fmla v22.8h, v8.8h, v11.8h\n" - "ldr x9, [x14, #0x110]\n" - "fmla v23.8h, v7.8h, v11.8h\n" - "fmla v26.8h, v5.8h, v11.8h\n" - "fmla v31.8h, v1.8h, v11.8h\n" - "ldr q11, [x28, x13]\n" - "fmla v27.8h, v2.8h, v12.8h\n" - "ldr x28, [x14, #0x118]\n" - "fmla v28.8h, v0.8h, v10.8h\n" - "fmla v29.8h, v4.8h, v11.8h\n" - "fmla v30.8h, v3.8h, v11.8h\n" - "fmla v19.8h, v8.8h, v12.8h\n" - "fmla v23.8h, v5.8h, v12.8h\n" - "ldr q12, [x27, x13]\n" - "fmla v20.8h, v6.8h, v10.8h\n" - "fmla v24.8h, v3.8h, v10.8h\n" - "ldr q10, [x25, x13]\n" - "fmla v25.8h, v7.8h, v11.8h\n" + "fmla v10.8h, v3.8h, v12.8h\n" + "fmla v25.8h, v1.8h, v12.8h\n" + "fmla v21.8h, v0.8h, v12.8h\n" + "ldr q9, [x21, x15]\n" + "ldr x28, [x16, #0xd8]\n" + "fmla v29.8h, v7.8h, v11.8h\n" "fmla v26.8h, v6.8h, v11.8h\n" - "fmla v28.8h, v5.8h, v11.8h\n" - "fmla v27.8h, v5.8h, v12.8h\n" - "fmla v31.8h, v2.8h, v12.8h\n" - "fmla v29.8h, v7.8h, v10.8h\n" - "fmla v30.8h, v6.8h, v10.8h\n" - "fmla v24.8h, v8.8h, v11.8h\n" - "ldr q11, [x26, x13]\n" - "fmla v28.8h, v8.8h, v10.8h\n" - "ldr q10, [x11, x13]\n" - "fmla v25.8h, v8.8h, v11.8h\n" - "fmla v26.8h, v7.8h, v11.8h\n" - "fmla v27.8h, v6.8h, v11.8h\n" - "fmla v29.8h, v5.8h, v11.8h\n" - "fmla v30.8h, v4.8h, v11.8h\n" - "fmla v31.8h, v3.8h, v11.8h\n" - "ldr q11, [x10, x13]\n" - "ldp x11, x10, [x14, #0x0]\n" - "fmla v23.8h, v8.8h, v12.8h\n" - "ldr q12, [x24, x13]\n" - "fmla v16.8h, v4.8h, v10.8h\n" - "fmax v16.8h, v16.8h, v15.8h\n" - "fmla v17.8h, v3.8h, v10.8h\n" - "fmla v18.8h, v5.8h, v11.8h\n" - "fmax v17.8h, v17.8h, v15.8h\n" - "fmla v19.8h, v4.8h, v11.8h\n" - "fmla v29.8h, v8.8h, v12.8h\n" - "fmax v18.8h, v18.8h, v15.8h\n" - "fmla v30.8h, v7.8h, v12.8h\n" - "fmla v31.8h, v6.8h, v12.8h\n" - "ldr q12, [x9, x13]\n" - "fmax v19.8h, v19.8h, v15.8h\n" - "fmla v20.8h, v1.8h, v10.8h\n" - "fmla v21.8h, v0.8h, v10.8h\n" - "ldr q10, [x28, x13]\n" - "ldr q9, [x11, x8]\n" - "fmla v22.8h, v2.8h, v11.8h\n" - "ldr q13, [x15, #0x0]\n" - "fmla v23.8h, v1.8h, v11.8h\n" - "ldr q0, [x15, #0x10]\n" - "ldr q1, [x15, #0x20]\n" + "ldr q12, [x27, x15]\n" + "ldr x21, [x16, #0xd0]\n" + "fmla v17.8h, v7.8h, v22.8h\n" + "fmla v16.8h, v6.8h, v22.8h\n" + "fmla v27.8h, v4.8h, v22.8h\n" + "fmla v23.8h, v3.8h, v22.8h\n" + "fmla v31.8h, v1.8h, v22.8h\n" + "fmla v28.8h, v0.8h, v22.8h\n" + "ldr q11, [x26, x15]\n" + "ldr x27, [x16, #0xe0]\n" + "fmla v15.8h, v8.8h, v9.8h\n" + "fmla v18.8h, v8.8h, v12.8h\n" "fmla v24.8h, v7.8h, v12.8h\n" - "fmla v25.8h, v6.8h, v12.8h\n" - "ldr q2, [x15, #0x30]\n" - "fmla v26.8h, v8.8h, v10.8h\n" - "ldr q6, [x15, #0x70]\n" - "fmla v27.8h, v7.8h, v10.8h\n" - "ldr q7, [x15, #0x80]\n" - "fmin v16.8h, v16.8h, v14.8h\n" + "ldr q12, [x25, x15]\n" + "fmla v19.8h, v1.8h, v9.8h\n" + "ldr x26, [x16, #0xe8]\n" + "fmla v10.8h, v7.8h, v9.8h\n" + "fmla v25.8h, v5.8h, v9.8h\n" + "fmla v21.8h, v4.8h, v9.8h\n" + "fmla v20.8h, v2.8h, v9.8h\n" + "ldr q9, [x24, x15]\n" + "ldr x24, [x16, #0xf0]\n" + "fmla v17.8h, v2.8h, v11.8h\n" + "fmla v16.8h, v1.8h, v11.8h\n" + "fmla v15.8h, v0.8h, v11.8h\n" + "ldr q22, [x23, x15]\n" + "fmla v27.8h, v7.8h, v12.8h\n" + "ldr x25, [x16, #0xf8]\n" + "fmla v23.8h, v6.8h, v12.8h\n" + "fmla v31.8h, v4.8h, v12.8h\n" + "fmla v28.8h, v3.8h, v12.8h\n" + "fmla v29.8h, v1.8h, v12.8h\n" + "fmla v26.8h, v0.8h, v12.8h\n" + "ldr q11, [x22, x15]\n" + "fmla v19.8h, v4.8h, v11.8h\n" + "ldr x23, [x16, #0x100]\n" + "fmla v18.8h, v2.8h, v11.8h\n" + "fmla v16.8h, v2.8h, v9.8h\n" + "fmla v15.8h, v1.8h, v9.8h\n" + "fmla v10.8h, v0.8h, v9.8h\n" + "ldr q9, [x20, x15]\n" + "ldr x20, [x16, #0x108]\n" + "fmla v17.8h, v6.8h, v22.8h\n" + "fmla v27.8h, v3.8h, v22.8h\n" + "fmla v31.8h, v0.8h, v22.8h\n" + "ldr q22, [x21, x15]\n" + "fmla v25.8h, v8.8h, v11.8h\n" + "ldr x22, [x16, #0x110]\n" + "fmla v21.8h, v7.8h, v11.8h\n" + "fmla v20.8h, v5.8h, v11.8h\n" + "fmla v24.8h, v1.8h, v11.8h\n" + "ldr q12, [x28, x15]\n" + "fmla v19.8h, v2.8h, v9.8h\n" + "ldr x21, [x16, #0x118]\n" + "fmla v29.8h, v0.8h, v22.8h\n" + "fmla v26.8h, v4.8h, v12.8h\n" + "fmla v18.8h, v3.8h, v12.8h\n" + "fmla v10.8h, v8.8h, v9.8h\n" + "fmla v21.8h, v5.8h, v9.8h\n" + "ldr q11, [x27, x15]\n" + "fmla v27.8h, v6.8h, v22.8h\n" + "fmla v31.8h, v3.8h, v22.8h\n" + "ldr q22, [x26, x15]\n" + "fmla v28.8h, v7.8h, v12.8h\n" + "fmla v20.8h, v6.8h, v12.8h\n" + "fmla v29.8h, v5.8h, v12.8h\n" + "fmla v19.8h, v5.8h, v11.8h\n" + "fmla v24.8h, v2.8h, v11.8h\n" + "fmla v26.8h, v7.8h, v22.8h\n" + "fmla v18.8h, v6.8h, v22.8h\n" + "fmla v31.8h, v8.8h, v12.8h\n" + "ldr q12, [x24, x15]\n" + "fmla v29.8h, v8.8h, v22.8h\n" + "ldr q22, [x23, x15]\n" + "fmla v28.8h, v8.8h, v12.8h\n" + "fmla v20.8h, v7.8h, v12.8h\n" + "fmla v19.8h, v6.8h, v12.8h\n" + "fmla v26.8h, v5.8h, v12.8h\n" + "fmla v18.8h, v4.8h, v12.8h\n" + "fmla v24.8h, v3.8h, v12.8h\n" + "ldr q12, [x20, x15]\n" + "ldp x20, x24, [x16, #0x0]\n" + "ldr q9, [x20, x6]\n" + "fmla v21.8h, v8.8h, v11.8h\n" + "ldr q11, [x25, x15]\n" + "fmla v17.8h, v4.8h, v22.8h\n" + "fmla v16.8h, v3.8h, v22.8h\n" + "fmla v15.8h, v5.8h, v12.8h\n" + "fmax v17.8h, v17.8h, v13.8h\n" + "fmla v10.8h, v4.8h, v12.8h\n" + "fmla v26.8h, v8.8h, v11.8h\n" + "fmax v16.8h, v16.8h, v13.8h\n" + "fmla v18.8h, v7.8h, v11.8h\n" + "fmla v24.8h, v6.8h, v11.8h\n" + "ldr q11, [x22, x15]\n" + "fmax v15.8h, v15.8h, v13.8h\n" + "fmla v27.8h, v1.8h, v22.8h\n" + "fmla v23.8h, v0.8h, v22.8h\n" + "ldr q22, [x21, x15]\n" + "ldr q0, [x17, #0x10]\n" + "fmla v25.8h, v2.8h, v12.8h\n" + "ldr q2, [x17, #0x30]\n" + "fmla v21.8h, v1.8h, v12.8h\n" + "ldr q1, [x17, #0x20]\n" + "fmax v10.8h, v10.8h, v13.8h\n" + "fmla v31.8h, v7.8h, v11.8h\n" + "fmla v28.8h, v6.8h, v11.8h\n" + "ldr q6, [x17, #0x70]\n" + "fmla v20.8h, v8.8h, v22.8h\n" + "ldr q8, [x17, #0x90]\n" + "fmla v19.8h, v7.8h, v22.8h\n" + "ldr q7, [x17, #0x80]\n" "fmin v17.8h, v17.8h, v14.8h\n" - "str q16, [x23, x12]\n" - "ldr q8, [x15, #0x90]\n" - "fmin v18.8h, v18.8h, v14.8h\n" - "fmin v19.8h, v19.8h, v14.8h\n" - "str q17, [x22, x12]\n" - "ldr x23, [x16, #0x20]\n" - "fmax v20.8h, v20.8h, v15.8h\n" - "fmax v21.8h, v21.8h, v15.8h\n" - "str q18, [x21, x12]\n" - "ldr x22, [x16, #0x28]\n" - "fmax v22.8h, v22.8h, v15.8h\n" - "fmax v23.8h, v23.8h, v15.8h\n" - "str q19, [x20, x12]\n" - "ldr x21, [x16, #0x30]\n" - "ldr x20, [x16, #0x38]\n" - "fmla v28.8h, v4.8h, v12.8h\n" - "fmla v29.8h, v3.8h, v12.8h\n" - "ldr q3, [x15, #0x40]\n" - "fmla v30.8h, v5.8h, v10.8h\n" - "ldr q5, [x15, #0x60]\n" - "fmla v31.8h, v4.8h, v10.8h\n" - "ldr q10, [x10, x8]\n" - "ldr q4, [x15, #0x50]\n" - "fmin v20.8h, v20.8h, v14.8h\n" - "fmin v21.8h, v21.8h, v14.8h\n" - "str q20, [x23, x12]\n" - "fmin v22.8h, v22.8h, v14.8h\n" + "fmin v16.8h, v16.8h, v14.8h\n" + "str q17, [x12, x14]\n" + "ldr x23, [x8, #0x20]\n" + "fmin v15.8h, v15.8h, v14.8h\n" + "fmin v10.8h, v10.8h, v14.8h\n" + "str q16, [x11, x14]\n" + "ldr x22, [x8, #0x28]\n" + "fmax v27.8h, v27.8h, v13.8h\n" + "fmax v23.8h, v23.8h, v13.8h\n" + "str q15, [x10, x14]\n" + "ldr x21, [x8, #0x30]\n" + "fmax v25.8h, v25.8h, v13.8h\n" + "fmax v21.8h, v21.8h, v13.8h\n" + "str q10, [x9, x14]\n" + "ldr x20, [x8, #0x38]\n" + "fmla v29.8h, v4.8h, v11.8h\n" + "fmla v26.8h, v3.8h, v11.8h\n" + "ldr q3, [x17, #0x40]\n" + "fmin v27.8h, v27.8h, v14.8h\n" + "fmla v18.8h, v5.8h, v22.8h\n" + "ldr q5, [x17, #0x60]\n" + "fmla v24.8h, v4.8h, v22.8h\n" + "ldr q10, [x24, x6]\n" + "ldr q4, [x17, #0x50]\n" "fmin v23.8h, v23.8h, v14.8h\n" - "str q21, [x22, x12]\n" - "ldr x23, [x16, #0x40]\n" - "fmax v24.8h, v24.8h, v15.8h\n" - "fmax v25.8h, v25.8h, v15.8h\n" - "str q22, [x21, x12]\n" - "ldr x22, [x16, #0x48]\n" - "fmax v26.8h, v26.8h, v15.8h\n" - "fmax v27.8h, v27.8h, v15.8h\n" - "str q23, [x20, x12]\n" - "ldr x21, [x16, #0x50]\n" - "ldr x20, [x16, #0x58]\n" - "ldp x9, x28, [x14, #0x10]\n" - "fmin v24.8h, v24.8h, v14.8h\n" "fmin v25.8h, v25.8h, v14.8h\n" - "ldr q11, [x9, x8]\n" - "ldr q12, [x28, x8]\n" - "fmin v26.8h, v26.8h, v14.8h\n" - "fmin v27.8h, v27.8h, v14.8h\n" - "fmax v28.8h, v28.8h, v15.8h\n" - "fmax v29.8h, v29.8h, v15.8h\n" - "str q24, [x23, x12]\n" - "ldr x23, [x16, #0x60]\n" - "fmax v30.8h, v30.8h, v15.8h\n" - "fmax v31.8h, v31.8h, v15.8h\n" - "str q25, [x22, x12]\n" - "ldr x22, [x16, #0x68]\n" - "str q26, [x21, x12]\n" - "ldr x21, [x16, #0x70]\n" - "add x8, x8, #0x10\n" - "cmp x8, x17, LSL #4\n" - "str q27, [x20, x12]\n" - "ldr x20, [x16, #0x78]\n" + "str q27, [x23, x14]\n" + "fmin v21.8h, v21.8h, v14.8h\n" + "fmax v31.8h, v31.8h, v13.8h\n" + "str q23, [x22, x14]\n" + "ldr x25, [x8, #0x40]\n" + "fmax v28.8h, v28.8h, v13.8h\n" + "fmax v20.8h, v20.8h, v13.8h\n" + "str q25, [x21, x14]\n" + "ldr x23, [x8, #0x48]\n" + "fmax v19.8h, v19.8h, v13.8h\n" + "str q21, [x20, x14]\n" + "ldr x22, [x8, #0x50]\n" + "ldr x24, [x8, #0x58]\n" + "ldp x21, x20, [x16, #0x10]\n" + "ldr q11, [x21, x6]\n" + "fmin v31.8h, v31.8h, v14.8h\n" "fmin v28.8h, v28.8h, v14.8h\n" + "ldr q12, [x20, x6]\n" + "fmin v20.8h, v20.8h, v14.8h\n" + "fmin v19.8h, v19.8h, v14.8h\n" + "str q31, [x25, x14]\n" + "fmax v29.8h, v29.8h, v13.8h\n" + "fmax v26.8h, v26.8h, v13.8h\n" + "str q28, [x23, x14]\n" + "ldr x23, [x8, #0x60]\n" + "fmax v18.8h, v18.8h, v13.8h\n" + "fmax v24.8h, v24.8h, v13.8h\n" + "str q20, [x22, x14]\n" + "ldr x22, [x8, #0x68]\n" + "str q19, [x24, x14]\n" + "ldr x21, [x8, #0x70]\n" + "ldr x20, [x8, #0x78]\n" + "add x6, x6, #0x10\n" + "cmp x6, x7, LSL #4\n" "fmin v29.8h, v29.8h, v14.8h\n" - "fmin v30.8h, v30.8h, v14.8h\n" - "fmin v31.8h, v31.8h, v14.8h\n" - "add x13, x13, #0x10\n" - "str q28, [x23, x12]\n" - "str q29, [x22, x12]\n" - "add x15, x15, #0xa0\n" - "str q30, [x21, x12]\n" - "str q31, [x20, x12]\n" + "fmin v26.8h, v26.8h, v14.8h\n" + "add x15, x15, #0x10\n" + "fmin v18.8h, v18.8h, v14.8h\n" + "fmin v24.8h, v24.8h, v14.8h\n" + "str q29, [x23, x14]\n" + "add x17, x17, #0xa0\n" + "str q26, [x22, x14]\n" + "str q18, [x21, x14]\n" + "str q24, [x20, x14]\n" "blt 1b\n" "2:" // Channel tail - "mov v21.16b, v13.16b\n fmla v21.8h, v4.8h, v9.8h\n" - "mov v16.16b, v13.16b\n fmla v16.8h, v8.8h, v9.8h\n" - "ldr x27, [x14, #0x20]\n" - "ldr x26, [x14, #0x30]\n" - "mov v22.16b, v13.16b\n fmla v22.8h, v3.8h, v9.8h\n" - "mov v25.16b, v13.16b\n fmla v25.8h, v1.8h, v9.8h\n" - "ldr x25, [x14, #0x28]\n" - "ldr x24, [x14, #0x38]\n" - "mov v26.16b, v13.16b\n fmla v26.8h, v0.8h, v9.8h\n" - "mov v17.16b, v13.16b\n fmla v17.8h, v7.8h, v9.8h\n" - "ldr x11, [x14, #0x40]\n" - "ldr x10, [x14, #0x48]\n" - "mov v18.16b, v13.16b\n fmla v18.8h, v6.8h, v9.8h\n" - "fmla v21.8h, v5.8h, v12.8h\n" - "ldr x9, [x14, #0x50]\n" - "ldr x28, [x14, #0x58]\n" - "mov v20.16b, v13.16b\n fmla v20.8h, v5.8h, v9.8h\n" - "mov v24.16b, v13.16b\n fmla v24.8h, v2.8h, v9.8h\n" - "ldr q9, [x26, x13]\n" - "ldr x26, [x14, #0x70]\n" - "fmla v16.8h, v0.8h, v10.8h\n" - "ldr q10, [x27, x13]\n" - "mov v19.16b, v13.16b\n fmla v19.8h, v2.8h, v11.8h\n" - "ldr q11, [x25, x13]\n" - "fmla v22.8h, v4.8h, v12.8h\n" - "fmla v25.8h, v2.8h, v12.8h\n" - "ldr x27, [x14, #0x60]\n" - "ldr x25, [x14, #0x68]\n" - "fmla v26.8h, v1.8h, v12.8h\n" - "fmla v17.8h, v8.8h, v12.8h\n" - "ldr x23, [x16, #0x0]\n" - "ldr x22, [x16, #0x8]\n" - "fmla v18.8h, v7.8h, v12.8h\n" - "mov v28.16b, v13.16b\n fmla v28.8h, v6.8h, v10.8h\n" - "ldr q10, [x10, x13]\n" - "ldr x10, [x14, #0x88]\n" - "fmla v21.8h, v7.8h, v9.8h\n" - "fmla v19.8h, v6.8h, v12.8h\n" - "ldr x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x18]\n" - "mov v23.16b, v13.16b\n fmla v23.8h, v3.8h, v12.8h\n" - "mov v27.16b, v13.16b\n fmla v27.8h, v0.8h, v12.8h\n" - "ldr q12, [x24, x13]\n" - "ldr x24, [x14, #0x78]\n" - "mov v31.16b, v13.16b\n fmla v31.8h, v8.8h, v11.8h\n" - "ldr q11, [x11, x13]\n" - "fmla v22.8h, v6.8h, v9.8h\n" - "ldr x11, [x14, #0x80]\n" - "fmla v25.8h, v4.8h, v9.8h\n" - "fmla v26.8h, v3.8h, v9.8h\n" - "add x12, x12, #0x10\n" - "fmla v20.8h, v8.8h, v9.8h\n" - "fmla v24.8h, v5.8h, v9.8h\n" - "fmla v28.8h, v2.8h, v9.8h\n" - "fmla v16.8h, v1.8h, v12.8h\n" - "fmla v17.8h, v0.8h, v12.8h\n" - "ldr q12, [x28, x13]\n" - "fmla v18.8h, v2.8h, v11.8h\n" - "ldr x28, [x14, #0x98]\n" - "fmla v21.8h, v8.8h, v10.8h\n" - "fmla v19.8h, v1.8h, v11.8h\n" - "ldr q11, [x27, x13]\n" - "ldr x27, [x14, #0xa0]\n" - "fmla v22.8h, v7.8h, v10.8h\n" - "fmla v23.8h, v6.8h, v10.8h\n" - "fmla v25.8h, v5.8h, v10.8h\n" - "fmla v26.8h, v4.8h, v10.8h\n" - "fmla v27.8h, v3.8h, v10.8h\n" - "fmla v31.8h, v0.8h, v10.8h\n" - "fmla v24.8h, v6.8h, v11.8h\n" - "fmla v28.8h, v3.8h, v11.8h\n" - "ldr q11, [x26, x13]\n" - "ldr x26, [x14, #0xb0]\n" - "fmla v19.8h, v5.8h, v12.8h\n" - "fmla v23.8h, v2.8h, v12.8h\n" - "ldr q12, [x24, x13]\n" - "ldr x24, [x14, #0xb8]\n" - "fmla v27.8h, v8.8h, v11.8h\n" - "fmla v31.8h, v5.8h, v11.8h\n" - "mov v29.16b, v13.16b\n fmla v29.8h, v1.8h, v9.8h\n" - "mov v30.16b, v13.16b\n fmla v30.8h, v0.8h, v9.8h\n" - "ldr q9, [x9, x13]\n" - "ldr x9, [x14, #0x90]\n" - "fmla v29.8h, v2.8h, v10.8h\n" - "fmla v30.8h, v1.8h, v10.8h\n" - "ldr q10, [x25, x13]\n" - "ldr x25, [x14, #0xa8]\n" - "fmla v16.8h, v3.8h, v9.8h\n" - "fmla v20.8h, v0.8h, v9.8h\n" - "ldr q11, [x11, x13]\n" - "ldr x11, [x14, #0xc0]\n" - "fmla v17.8h, v4.8h, v10.8h\n" - "fmla v18.8h, v3.8h, v10.8h\n" - "fmla v21.8h, v1.8h, v10.8h\n" - "fmla v22.8h, v0.8h, v10.8h\n" - "fmla v16.8h, v5.8h, v10.8h\n" - "fmla v20.8h, v2.8h, v10.8h\n" - "ldr q10, [x10, x13]\n" - "ldr x10, [x14, #0xc8]\n" - "fmla v17.8h, v5.8h, v12.8h\n" - "fmla v18.8h, v4.8h, v12.8h\n" - "fmla v21.8h, v2.8h, v12.8h\n" - "fmla v19.8h, v3.8h, v12.8h\n" - "fmla v22.8h, v1.8h, v12.8h\n" - "fmla v23.8h, v0.8h, v12.8h\n" - "ldr q12, [x28, x13]\n" - "ldr x28, [x14, #0xd8]\n" - "fmla v28.8h, v7.8h, v11.8h\n" - "fmla v29.8h, v6.8h, v11.8h\n" - "ldr q11, [x9, x13]\n" - "ldr x9, [x14, #0xd0]\n" - "fmla v16.8h, v7.8h, v10.8h\n" - "fmla v17.8h, v6.8h, v10.8h\n" - "fmla v20.8h, v4.8h, v10.8h\n" - "fmla v21.8h, v3.8h, v10.8h\n" - "fmla v24.8h, v1.8h, v10.8h\n" - "fmla v25.8h, v0.8h, v10.8h\n" - "ldr q10, [x27, x13]\n" - "ldr x27, [x14, #0xe0]\n" - "fmla v18.8h, v8.8h, v12.8h\n" - "fmla v30.8h, v8.8h, v11.8h\n" - "fmla v31.8h, v7.8h, v11.8h\n" - "ldr q11, [x25, x13]\n" - "fmla v27.8h, v1.8h, v12.8h\n" - "ldr x25, [x14, #0xe8]\n" - "fmla v19.8h, v7.8h, v12.8h\n" - "fmla v22.8h, v5.8h, v12.8h\n" - "fmla v23.8h, v4.8h, v12.8h\n" - "fmla v26.8h, v2.8h, v12.8h\n" - "ldr q12, [x26, x13]\n" - "ldr x26, [x14, #0xf0]\n" - "fmla v16.8h, v2.8h, v10.8h\n" - "fmla v17.8h, v1.8h, v10.8h\n" - "fmla v18.8h, v0.8h, v10.8h\n" - "ldr q10, [x24, x13]\n" - "fmla v20.8h, v7.8h, v11.8h\n" - "ldr x24, [x14, #0xf8]\n" - "fmla v21.8h, v6.8h, v11.8h\n" - "fmla v24.8h, v4.8h, v11.8h\n" - "fmla v25.8h, v3.8h, v11.8h\n" - "fmla v28.8h, v1.8h, v11.8h\n" - "fmla v29.8h, v0.8h, v11.8h\n" - "ldr q11, [x11, x13]\n" - "fmla v27.8h, v4.8h, v11.8h\n" - "ldr x11, [x14, #0x100]\n" - "fmla v30.8h, v2.8h, v11.8h\n" - "fmla v17.8h, v2.8h, v12.8h\n" - "fmla v18.8h, v1.8h, v12.8h\n" - "fmla v19.8h, v0.8h, v12.8h\n" - "ldr q12, [x10, x13]\n" - "ldr x10, [x14, #0x108]\n" - "fmla v16.8h, v6.8h, v10.8h\n" - "fmla v20.8h, v3.8h, v10.8h\n" - "fmla v24.8h, v0.8h, v10.8h\n" - "ldr q10, [x9, x13]\n" - "fmla v22.8h, v8.8h, v11.8h\n" - "ldr x9, [x14, #0x110]\n" - "fmla v23.8h, v7.8h, v11.8h\n" - "fmla v26.8h, v5.8h, v11.8h\n" - "fmla v31.8h, v1.8h, v11.8h\n" - "ldr q11, [x28, x13]\n" - "fmla v27.8h, v2.8h, v12.8h\n" - "ldr x28, [x14, #0x118]\n" - "fmla v28.8h, v0.8h, v10.8h\n" - "fmla v29.8h, v4.8h, v11.8h\n" - "fmla v30.8h, v3.8h, v11.8h\n" - "fmla v19.8h, v8.8h, v12.8h\n" - "fmla v23.8h, v5.8h, v12.8h\n" - "ldr q12, [x27, x13]\n" - "fmla v20.8h, v6.8h, v10.8h\n" - "fmla v24.8h, v3.8h, v10.8h\n" - "ldr q10, [x25, x13]\n" - "fmla v25.8h, v7.8h, v11.8h\n" - "fmla v26.8h, v6.8h, v11.8h\n" - "fmla v28.8h, v5.8h, v11.8h\n" - "fmla v27.8h, v5.8h, v12.8h\n" - "fmla v31.8h, v2.8h, v12.8h\n" - "fmla v29.8h, v7.8h, v10.8h\n" - "fmla v30.8h, v6.8h, v10.8h\n" - "fmla v24.8h, v8.8h, v11.8h\n" - "ldr q11, [x26, x13]\n" - "fmla v28.8h, v8.8h, v10.8h\n" - "ldr q10, [x11, x13]\n" - "fmla v25.8h, v8.8h, v11.8h\n" - "fmla v26.8h, v7.8h, v11.8h\n" - "fmla v27.8h, v6.8h, v11.8h\n" - "fmla v29.8h, v5.8h, v11.8h\n" - "fmla v30.8h, v4.8h, v11.8h\n" - "fmla v31.8h, v3.8h, v11.8h\n" - "ldr q11, [x10, x13]\n" - "fmla v23.8h, v8.8h, v12.8h\n" - "ldr q12, [x24, x13]\n" - "fmla v16.8h, v4.8h, v10.8h\n" - "fmax v16.8h, v16.8h, v15.8h\n" - "fmla v17.8h, v3.8h, v10.8h\n" - "fmla v18.8h, v5.8h, v11.8h\n" - "fmax v17.8h, v17.8h, v15.8h\n" - "fmla v19.8h, v4.8h, v11.8h\n" - "fmla v29.8h, v8.8h, v12.8h\n" - "fmax v18.8h, v18.8h, v15.8h\n" - "fmla v30.8h, v7.8h, v12.8h\n" - "fmla v31.8h, v6.8h, v12.8h\n" - "ldr q12, [x9, x13]\n" - "fmax v19.8h, v19.8h, v15.8h\n" - "fmla v20.8h, v1.8h, v10.8h\n" - "fmla v21.8h, v0.8h, v10.8h\n" - "ldr q10, [x28, x13]\n" - "fmin v16.8h, v16.8h, v14.8h\n" - "fmla v22.8h, v2.8h, v11.8h\n" - "fmla v23.8h, v1.8h, v11.8h\n" + "mov v31.16b, v30.16b\n fmla v31.8h, v4.8h, v9.8h\n" + "mov v17.16b, v30.16b\n fmla v17.8h, v8.8h, v9.8h\n" + "ldr x27, [x16, #0x20]\n" + "ldr x24, [x16, #0x30]\n" + "mov v15.16b, v30.16b\n fmla v15.8h, v3.8h, v9.8h\n" + "mov v29.16b, v30.16b\n fmla v29.8h, v1.8h, v9.8h\n" + "ldr x23, [x16, #0x28]\n" + "ldr x22, [x16, #0x38]\n" + "mov v19.16b, v30.16b\n fmla v19.8h, v0.8h, v9.8h\n" + "mov v20.16b, v30.16b\n fmla v20.8h, v7.8h, v9.8h\n" + "ldr x26, [x16, #0x40]\n" + "ldr x21, [x16, #0x48]\n" + "mov v21.16b, v30.16b\n fmla v21.8h, v6.8h, v9.8h\n" + "fmla v31.8h, v5.8h, v12.8h\n" + "ldr x25, [x16, #0x50]\n" + "ldr x20, [x16, #0x58]\n" + "mov v18.16b, v30.16b\n fmla v18.8h, v5.8h, v9.8h\n" + "mov v27.16b, v30.16b\n fmla v27.8h, v2.8h, v9.8h\n" + "ldr q24, [x24, x15]\n" + "ldr x13, [x16, #0x70]\n" + "fmla v17.8h, v0.8h, v10.8h\n" + "ldr q22, [x27, x15]\n" + "mov v28.16b, v30.16b\n fmla v28.8h, v2.8h, v11.8h\n" + "ldr q16, [x23, x15]\n" + "fmla v15.8h, v4.8h, v12.8h\n" + "fmla v29.8h, v2.8h, v12.8h\n" + "ldr x24, [x16, #0x60]\n" + "ldr x23, [x16, #0x68]\n" + "fmla v19.8h, v1.8h, v12.8h\n" + "fmla v20.8h, v8.8h, v12.8h\n" + "ldr x12, [x8, #0x0]\n" + "ldr x11, [x8, #0x8]\n" + "fmla v21.8h, v7.8h, v12.8h\n" + "mov v10.16b, v30.16b\n fmla v10.8h, v6.8h, v22.8h\n" + "ldr q22, [x21, x15]\n" + "ldr x28, [x16, #0x88]\n" + "fmla v31.8h, v7.8h, v24.8h\n" + "fmla v28.8h, v6.8h, v12.8h\n" + "ldr x10, [x8, #0x10]\n" + "ldr x9, [x8, #0x18]\n" + "mov v9.16b, v30.16b\n fmla v9.8h, v3.8h, v12.8h\n" + "mov v11.16b, v30.16b\n fmla v11.8h, v0.8h, v12.8h\n" + "ldr q23, [x22, x15]\n" + "ldr x22, [x16, #0x78]\n" + "mov v12.16b, v30.16b\n fmla v12.8h, v8.8h, v16.8h\n" + "ldr q16, [x26, x15]\n" + "fmla v15.8h, v6.8h, v24.8h\n" + "ldr x21, [x16, #0x80]\n" + "fmla v29.8h, v4.8h, v24.8h\n" + "fmla v19.8h, v3.8h, v24.8h\n" + "add x14, x14, #0x10\n" + "mov v26.16b, v30.16b\n fmla v26.8h, v1.8h, v24.8h\n" + "mov v25.16b, v30.16b\n fmla v25.8h, v0.8h, v24.8h\n" + "fmla v18.8h, v8.8h, v24.8h\n" + "fmla v27.8h, v5.8h, v24.8h\n" + "fmla v10.8h, v2.8h, v24.8h\n" + "ldr q24, [x25, x15]\n" + "fmla v17.8h, v1.8h, v23.8h\n" + "ldr x27, [x16, #0x90]\n" + "fmla v20.8h, v0.8h, v23.8h\n" + "ldr q23, [x20, x15]\n" + "fmla v21.8h, v2.8h, v16.8h\n" + "ldr x20, [x16, #0x98]\n" + "fmla v31.8h, v8.8h, v22.8h\n" + "fmla v28.8h, v1.8h, v16.8h\n" + "ldr q16, [x24, x15]\n" + "ldr x26, [x16, #0xa0]\n" + "fmla v15.8h, v7.8h, v22.8h\n" + "fmla v9.8h, v6.8h, v22.8h\n" + "fmla v29.8h, v5.8h, v22.8h\n" + "fmla v19.8h, v4.8h, v22.8h\n" + "fmla v11.8h, v3.8h, v22.8h\n" + "fmla v26.8h, v2.8h, v22.8h\n" + "fmla v25.8h, v1.8h, v22.8h\n" + "fmla v12.8h, v0.8h, v22.8h\n" + "ldr q22, [x23, x15]\n" + "ldr x25, [x16, #0xa8]\n" + "fmla v17.8h, v3.8h, v24.8h\n" + "fmla v18.8h, v0.8h, v24.8h\n" + "fmla v27.8h, v6.8h, v16.8h\n" + "fmla v10.8h, v3.8h, v16.8h\n" + "ldr q16, [x13, x15]\n" + "ldr x24, [x16, #0xb0]\n" + "fmla v20.8h, v4.8h, v22.8h\n" + "fmla v21.8h, v3.8h, v22.8h\n" + "fmla v31.8h, v1.8h, v22.8h\n" + "fmla v28.8h, v5.8h, v23.8h\n" + "fmla v9.8h, v2.8h, v23.8h\n" + "ldr q23, [x22, x15]\n" + "fmla v15.8h, v0.8h, v22.8h\n" + "ldr x23, [x16, #0xb8]\n" + "fmla v11.8h, v8.8h, v16.8h\n" + "fmla v12.8h, v5.8h, v16.8h\n" + "ldr q16, [x21, x15]\n" + "ldr x22, [x16, #0xc0]\n" + "fmla v17.8h, v5.8h, v22.8h\n" + "fmla v18.8h, v2.8h, v22.8h\n" + "ldr q22, [x28, x15]\n" + "ldr x21, [x16, #0xc8]\n" + "fmla v20.8h, v5.8h, v23.8h\n" + "fmla v21.8h, v4.8h, v23.8h\n" + "fmla v31.8h, v2.8h, v23.8h\n" + "fmla v28.8h, v3.8h, v23.8h\n" + "fmla v15.8h, v1.8h, v23.8h\n" + "fmla v9.8h, v0.8h, v23.8h\n" + "ldr q23, [x20, x15]\n" + "ldr x28, [x16, #0xd8]\n" + "fmla v10.8h, v7.8h, v16.8h\n" + "fmla v26.8h, v6.8h, v16.8h\n" + "ldr q16, [x27, x15]\n" + "ldr x20, [x16, #0xd0]\n" + "fmla v17.8h, v7.8h, v22.8h\n" + "fmla v20.8h, v6.8h, v22.8h\n" + "fmla v18.8h, v4.8h, v22.8h\n" + "fmla v31.8h, v3.8h, v22.8h\n" + "fmla v27.8h, v1.8h, v22.8h\n" + "fmla v29.8h, v0.8h, v22.8h\n" + "ldr q22, [x26, x15]\n" + "ldr x27, [x16, #0xe0]\n" + "fmla v21.8h, v8.8h, v23.8h\n" + "fmla v25.8h, v8.8h, v16.8h\n" + "fmla v12.8h, v7.8h, v16.8h\n" + "ldr q16, [x25, x15]\n" + "fmla v11.8h, v1.8h, v23.8h\n" + "ldr x26, [x16, #0xe8]\n" + "fmla v28.8h, v7.8h, v23.8h\n" + "fmla v15.8h, v5.8h, v23.8h\n" + "fmla v9.8h, v4.8h, v23.8h\n" + "fmla v19.8h, v2.8h, v23.8h\n" + "ldr q23, [x24, x15]\n" + "ldr x25, [x16, #0xf0]\n" + "fmla v17.8h, v2.8h, v22.8h\n" + "fmla v20.8h, v1.8h, v22.8h\n" + "fmla v21.8h, v0.8h, v22.8h\n" + "ldr q22, [x23, x15]\n" + "fmla v18.8h, v7.8h, v16.8h\n" + "ldr x24, [x16, #0xf8]\n" + "fmla v31.8h, v6.8h, v16.8h\n" + "fmla v27.8h, v4.8h, v16.8h\n" + "fmla v29.8h, v3.8h, v16.8h\n" + "fmla v10.8h, v1.8h, v16.8h\n" + "fmla v26.8h, v0.8h, v16.8h\n" + "ldr q16, [x22, x15]\n" + "fmla v11.8h, v4.8h, v16.8h\n" + "ldr x23, [x16, #0x100]\n" + "fmla v25.8h, v2.8h, v16.8h\n" + "fmla v20.8h, v2.8h, v23.8h\n" + "fmla v21.8h, v1.8h, v23.8h\n" + "fmla v28.8h, v0.8h, v23.8h\n" + "ldr q23, [x21, x15]\n" + "ldr x22, [x16, #0x108]\n" + "fmla v17.8h, v6.8h, v22.8h\n" + "fmla v18.8h, v3.8h, v22.8h\n" + "fmla v27.8h, v0.8h, v22.8h\n" + "ldr q22, [x20, x15]\n" + "fmla v15.8h, v8.8h, v16.8h\n" + "ldr x21, [x16, #0x110]\n" + "fmla v9.8h, v7.8h, v16.8h\n" + "fmla v19.8h, v5.8h, v16.8h\n" + "fmla v12.8h, v1.8h, v16.8h\n" + "ldr q16, [x28, x15]\n" + "fmla v11.8h, v2.8h, v23.8h\n" + "ldr x20, [x16, #0x118]\n" + "fmla v10.8h, v0.8h, v22.8h\n" + "fmla v26.8h, v4.8h, v16.8h\n" + "fmla v25.8h, v3.8h, v16.8h\n" + "fmla v28.8h, v8.8h, v23.8h\n" + "fmla v9.8h, v5.8h, v23.8h\n" + "ldr q23, [x27, x15]\n" + "fmla v18.8h, v6.8h, v22.8h\n" + "fmla v27.8h, v3.8h, v22.8h\n" + "ldr q22, [x26, x15]\n" + "fmla v29.8h, v7.8h, v16.8h\n" + "fmla v19.8h, v6.8h, v16.8h\n" + "fmla v10.8h, v5.8h, v16.8h\n" + "fmla v11.8h, v5.8h, v23.8h\n" + "fmla v12.8h, v2.8h, v23.8h\n" + "fmla v26.8h, v7.8h, v22.8h\n" + "fmla v25.8h, v6.8h, v22.8h\n" + "fmla v27.8h, v8.8h, v16.8h\n" + "ldr q16, [x25, x15]\n" + "fmla v10.8h, v8.8h, v22.8h\n" + "ldr q30, [x23, x15]\n" + "fmla v29.8h, v8.8h, v16.8h\n" + "fmla v19.8h, v7.8h, v16.8h\n" + "fmla v11.8h, v6.8h, v16.8h\n" + "fmla v26.8h, v5.8h, v16.8h\n" + "fmla v25.8h, v4.8h, v16.8h\n" + "fmla v12.8h, v3.8h, v16.8h\n" + "ldr q24, [x22, x15]\n" + "fmla v9.8h, v8.8h, v23.8h\n" + "ldr q16, [x24, x15]\n" + "fmla v17.8h, v4.8h, v30.8h\n" + "fmax v17.8h, v17.8h, v13.8h\n" + "fmla v20.8h, v3.8h, v30.8h\n" + "fmla v21.8h, v5.8h, v24.8h\n" + "fmax v20.8h, v20.8h, v13.8h\n" + "fmla v28.8h, v4.8h, v24.8h\n" + "fmla v26.8h, v8.8h, v16.8h\n" + "fmax v21.8h, v21.8h, v13.8h\n" + "fmla v25.8h, v7.8h, v16.8h\n" + "fmla v12.8h, v6.8h, v16.8h\n" + "ldr q23, [x21, x15]\n" + "fmax v28.8h, v28.8h, v13.8h\n" + "fmla v18.8h, v1.8h, v30.8h\n" + "fmla v31.8h, v0.8h, v30.8h\n" + "ldr q16, [x20, x15]\n" "fmin v17.8h, v17.8h, v14.8h\n" - "str q16, [x23, x12]\n" - "fmla v24.8h, v7.8h, v12.8h\n" - "fmla v25.8h, v6.8h, v12.8h\n" - "fmin v18.8h, v18.8h, v14.8h\n" - "str q17, [x22, x12]\n" - "fmla v26.8h, v8.8h, v10.8h\n" - "fmla v27.8h, v7.8h, v10.8h\n" - "fmin v19.8h, v19.8h, v14.8h\n" - "str q18, [x21, x12]\n" - "fmax v20.8h, v20.8h, v15.8h\n" - "fmax v21.8h, v21.8h, v15.8h\n" - "str q19, [x20, x12]\n" - "ldr x23, [x16, #0x20]\n" - "fmax v22.8h, v22.8h, v15.8h\n" - "fmax v23.8h, v23.8h, v15.8h\n" - "ldr x22, [x16, #0x28]\n" - "ldr x21, [x16, #0x30]\n" - "ldr x20, [x16, #0x38]\n" - "fmla v28.8h, v4.8h, v12.8h\n" - "fmla v29.8h, v3.8h, v12.8h\n" + "fmla v15.8h, v2.8h, v24.8h\n" + "fmla v9.8h, v1.8h, v24.8h\n" "fmin v20.8h, v20.8h, v14.8h\n" - "fmla v30.8h, v5.8h, v10.8h\n" - "fmla v31.8h, v4.8h, v10.8h\n" + "str q17, [x12, x14]\n" + "fmla v27.8h, v7.8h, v23.8h\n" + "fmla v29.8h, v6.8h, v23.8h\n" "fmin v21.8h, v21.8h, v14.8h\n" - "str q20, [x23, x12]\n" - "fmin v22.8h, v22.8h, v14.8h\n" - "fmin v23.8h, v23.8h, v14.8h\n" - "str q21, [x22, x12]\n" - "ldr x23, [x16, #0x40]\n" - "fmax v24.8h, v24.8h, v15.8h\n" - "fmax v25.8h, v25.8h, v15.8h\n" - "str q22, [x21, x12]\n" - "ldr x22, [x16, #0x48]\n" - "fmax v26.8h, v26.8h, v15.8h\n" - "fmax v27.8h, v27.8h, v15.8h\n" - "str q23, [x20, x12]\n" - "ldr x21, [x16, #0x50]\n" - "ldr x20, [x16, #0x58]\n" - "fmin v24.8h, v24.8h, v14.8h\n" - "fmin v25.8h, v25.8h, v14.8h\n" - "str q24, [x23, x12]\n" - "fmin v26.8h, v26.8h, v14.8h\n" - "fmin v27.8h, v27.8h, v14.8h\n" - "str q25, [x22, x12]\n" - "ldr x23, [x16, #0x60]\n" - "fmax v28.8h, v28.8h, v15.8h\n" - "fmax v29.8h, v29.8h, v15.8h\n" - "str q26, [x21, x12]\n" - "ldr x22, [x16, #0x68]\n" - "fmax v30.8h, v30.8h, v15.8h\n" - "fmax v31.8h, v31.8h, v15.8h\n" - "str q27, [x20, x12]\n" - "ldr x21, [x16, #0x70]\n" - "ldr x20, [x16, #0x78]\n" + "str q20, [x11, x14]\n" + "fmla v19.8h, v8.8h, v16.8h\n" + "fmla v11.8h, v7.8h, v16.8h\n" "fmin v28.8h, v28.8h, v14.8h\n" - "fmin v29.8h, v29.8h, v14.8h\n" - "str q28, [x23, x12]\n" - "fmin v30.8h, v30.8h, v14.8h\n" + "str q21, [x10, x14]\n" + "fmax v18.8h, v18.8h, v13.8h\n" + "fmax v31.8h, v31.8h, v13.8h\n" + "str q28, [x9, x14]\n" + "ldr x23, [x8, #0x20]\n" + "fmax v15.8h, v15.8h, v13.8h\n" + "fmax v9.8h, v9.8h, v13.8h\n" + "ldr x22, [x8, #0x28]\n" + "ldr x21, [x8, #0x30]\n" + "ldr x20, [x8, #0x38]\n" + "fmla v10.8h, v4.8h, v23.8h\n" + "fmla v26.8h, v3.8h, v23.8h\n" + "fmin v18.8h, v18.8h, v14.8h\n" + "fmla v25.8h, v5.8h, v16.8h\n" + "fmla v12.8h, v4.8h, v16.8h\n" "fmin v31.8h, v31.8h, v14.8h\n" - "str q29, [x22, x12]\n" - "add x13, x13, #0x10\n" - "str q30, [x21, x12]\n" - "str q31, [x20, x12]\n" + "str q18, [x23, x14]\n" + "fmin v15.8h, v15.8h, v14.8h\n" + "fmin v9.8h, v9.8h, v14.8h\n" + "str q31, [x22, x14]\n" + "ldr x23, [x8, #0x40]\n" + "fmax v27.8h, v27.8h, v13.8h\n" + "fmax v29.8h, v29.8h, v13.8h\n" + "str q15, [x21, x14]\n" + "ldr x22, [x8, #0x48]\n" + "fmax v19.8h, v19.8h, v13.8h\n" + "fmax v11.8h, v11.8h, v13.8h\n" + "str q9, [x20, x14]\n" + "ldr x21, [x8, #0x50]\n" + "ldr x20, [x8, #0x58]\n" + "fmin v27.8h, v27.8h, v14.8h\n" + "fmin v29.8h, v29.8h, v14.8h\n" + "str q27, [x23, x14]\n" + "fmin v19.8h, v19.8h, v14.8h\n" + "fmin v11.8h, v11.8h, v14.8h\n" + "str q29, [x22, x14]\n" + "ldr x23, [x8, #0x60]\n" + "fmax v10.8h, v10.8h, v13.8h\n" + "fmax v26.8h, v26.8h, v13.8h\n" + "str q19, [x21, x14]\n" + "ldr x22, [x8, #0x68]\n" + "fmax v25.8h, v25.8h, v13.8h\n" + "fmax v12.8h, v12.8h, v13.8h\n" + "str q11, [x20, x14]\n" + "ldr x21, [x8, #0x70]\n" + "ldr x20, [x8, #0x78]\n" + "fmin v10.8h, v10.8h, v14.8h\n" + "fmin v26.8h, v26.8h, v14.8h\n" + "str q10, [x23, x14]\n" + "fmin v25.8h, v25.8h, v14.8h\n" + "fmin v12.8h, v12.8h, v14.8h\n" + "str q26, [x22, x14]\n" + "add x15, x15, #0x10\n" + "str q25, [x21, x14]\n" + "str q12, [x20, x14]\n" "3:" // Oddments "tst %x[n_channels], #0x7\n" "beq 140f\n" - "ldr q13, [x15, #0x0]\n" - "ldr q0, [x15, #0x10]\n" - "mov x12, x13\n" - "ldr q1, [x15, #0x20]\n" - "ldr q2, [x15, #0x30]\n" - "ldr q3, [x15, #0x40]\n" - "ldr q4, [x15, #0x50]\n" - "ldr q5, [x15, #0x60]\n" - "ldr q6, [x15, #0x70]\n" - "ldr q7, [x15, #0x80]\n" - "ldr q8, [x15, #0x90]\n" - "ldr x23, [x14, #0x0]\n" - "ldr x22, [x14, #0x8]\n" - "add x23, x23, x13\n" - "add x22, x22, x13\n" - "ldr x21, [x14, #0x10]\n" - "ldr x20, [x14, #0x18]\n" - "add x21, x21, x13\n" - "add x20, x20, x13\n" + "ldr q30, [x17, #0x0]\n" + "ldr q0, [x17, #0x10]\n" + "mov x14, x15\n" + "ldr q1, [x17, #0x20]\n" + "ldr q2, [x17, #0x30]\n" + "ldr q3, [x17, #0x40]\n" + "ldr q4, [x17, #0x50]\n" + "ldr q5, [x17, #0x60]\n" + "ldr q6, [x17, #0x70]\n" + "ldr q7, [x17, #0x80]\n" + "ldr q8, [x17, #0x90]\n" + "ldr x23, [x16, #0x0]\n" + "ldr x22, [x16, #0x8]\n" + "add x23, x23, x15\n" + "add x22, x22, x15\n" + "ldr x21, [x16, #0x10]\n" + "ldr x20, [x16, #0x18]\n" + "add x21, x21, x15\n" + "add x20, x20, x15\n" "tbz %x[n_channels], #2, 5f\n" "ld1 { v9.d }[0], [x23], #0x8\n" "ld1 { v10.d }[0], [x22], #0x8\n" @@ -762,28 +762,28 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "ld1 { v11.h }[0], [x21], #0x2\n" "ld1 { v12.h }[0], [x20], #0x2\n" "7:" // Oddments: Load inputs (2, 2), (0, 0), (0, 5), (2, 3): Bit 2: End - "mov v16.16b, v13.16b\n fmla v16.8h, v8.8h, v9.8h\n" - "mov v17.16b, v13.16b\n fmla v17.8h, v7.8h, v9.8h\n" - "ldr x20, [x14, #0x20]\n" - "add x20, x20, x13\n" - "mov v18.16b, v13.16b\n fmla v18.8h, v6.8h, v9.8h\n" - "mov v21.16b, v13.16b\n fmla v21.8h, v4.8h, v9.8h\n" - "mov v22.16b, v13.16b\n fmla v22.8h, v3.8h, v9.8h\n" - "mov v25.16b, v13.16b\n fmla v25.8h, v1.8h, v9.8h\n" - "mov v26.16b, v13.16b\n fmla v26.8h, v0.8h, v9.8h\n" - "mov v19.16b, v13.16b\n fmla v19.8h, v2.8h, v11.8h\n" - "mov v20.16b, v13.16b\n fmla v20.8h, v5.8h, v9.8h\n" - "mov v24.16b, v13.16b\n fmla v24.8h, v2.8h, v9.8h\n" + "mov v16.16b, v30.16b\n fmla v16.8h, v8.8h, v9.8h\n" + "mov v17.16b, v30.16b\n fmla v17.8h, v7.8h, v9.8h\n" + "ldr x20, [x16, #0x20]\n" + "add x20, x20, x15\n" + "mov v18.16b, v30.16b\n fmla v18.8h, v6.8h, v9.8h\n" + "mov v21.16b, v30.16b\n fmla v21.8h, v4.8h, v9.8h\n" + "mov v22.16b, v30.16b\n fmla v22.8h, v3.8h, v9.8h\n" + "mov v25.16b, v30.16b\n fmla v25.8h, v1.8h, v9.8h\n" + "mov v26.16b, v30.16b\n fmla v26.8h, v0.8h, v9.8h\n" + "mov v19.16b, v30.16b\n fmla v19.8h, v2.8h, v11.8h\n" + "mov v20.16b, v30.16b\n fmla v20.8h, v5.8h, v9.8h\n" + "mov v24.16b, v30.16b\n fmla v24.8h, v2.8h, v9.8h\n" "fmla v16.8h, v0.8h, v10.8h\n" "fmla v17.8h, v8.8h, v12.8h\n" "fmla v18.8h, v7.8h, v12.8h\n" "fmla v19.8h, v6.8h, v12.8h\n" "fmla v21.8h, v5.8h, v12.8h\n" "fmla v22.8h, v4.8h, v12.8h\n" - "mov v23.16b, v13.16b\n fmla v23.8h, v3.8h, v12.8h\n" + "mov v23.16b, v30.16b\n fmla v23.8h, v3.8h, v12.8h\n" "fmla v25.8h, v2.8h, v12.8h\n" "fmla v26.8h, v1.8h, v12.8h\n" - "mov v27.16b, v13.16b\n fmla v27.8h, v0.8h, v12.8h\n" + "mov v27.16b, v30.16b\n fmla v27.8h, v0.8h, v12.8h\n" "tbz %x[n_channels], #2, 9f\n" "ld1 { v10.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #1, 8f\n" @@ -804,9 +804,9 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "10:" // Oddments: Load input (5, 0): Bit 2: Unset: Bit 1: Unset "ld1 { v10.h }[0], [x20], #0x2\n" "11:" // Oddments: Load input (5, 0): Bit 2: End - "ldr x20, [x14, #0x28]\n" - "mov v28.16b, v13.16b\n fmla v28.8h, v6.8h, v10.8h\n" - "add x20, x20, x13\n" + "ldr x20, [x16, #0x28]\n" + "mov v28.16b, v30.16b\n fmla v28.8h, v6.8h, v10.8h\n" + "add x20, x20, x15\n" "tbz %x[n_channels], #2, 13f\n" "ld1 { v11.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #1, 12f\n" @@ -827,9 +827,9 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "14:" // Oddments: Load input (5, 5): Bit 2: Unset: Bit 1: Unset "ld1 { v11.h }[0], [x20], #0x2\n" "15:" // Oddments: Load input (5, 5): Bit 2: End - "ldr x20, [x14, #0x30]\n" - "mov v31.16b, v13.16b\n fmla v31.8h, v8.8h, v11.8h\n" - "add x20, x20, x13\n" + "ldr x20, [x16, #0x30]\n" + "mov v31.16b, v30.16b\n fmla v31.8h, v8.8h, v11.8h\n" + "add x20, x20, x15\n" "tbz %x[n_channels], #2, 17f\n" "ld1 { v9.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #1, 16f\n" @@ -850,17 +850,17 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "18:" // Oddments: Load input (3, 2): Bit 2: Unset: Bit 1: Unset "ld1 { v9.h }[0], [x20], #0x2\n" "19:" // Oddments: Load input (3, 2): Bit 2: End - "ldr x20, [x14, #0x38]\n" + "ldr x20, [x16, #0x38]\n" "fmla v20.8h, v8.8h, v9.8h\n" "fmla v21.8h, v7.8h, v9.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v22.8h, v6.8h, v9.8h\n" "fmla v24.8h, v5.8h, v9.8h\n" "fmla v25.8h, v4.8h, v9.8h\n" "fmla v26.8h, v3.8h, v9.8h\n" "fmla v28.8h, v2.8h, v9.8h\n" - "mov v29.16b, v13.16b\n fmla v29.8h, v1.8h, v9.8h\n" - "mov v30.16b, v13.16b\n fmla v30.8h, v0.8h, v9.8h\n" + "mov v29.16b, v30.16b\n fmla v29.8h, v1.8h, v9.8h\n" + "fmla v30.8h, v0.8h, v9.8h\n" "tbz %x[n_channels], #2, 21f\n" "ld1 { v12.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #1, 20f\n" @@ -881,10 +881,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "22:" // Oddments: Load input (0, 1): Bit 2: Unset: Bit 1: Unset "ld1 { v12.h }[0], [x20], #0x2\n" "23:" // Oddments: Load input (0, 1): Bit 2: End - "ldr x20, [x14, #0x40]\n" + "ldr x20, [x16, #0x40]\n" "fmla v16.8h, v1.8h, v12.8h\n" "fmla v17.8h, v0.8h, v12.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "tbz %x[n_channels], #2, 25f\n" "ld1 { v11.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #1, 24f\n" @@ -905,10 +905,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "26:" // Oddments: Load input (0, 4): Bit 2: Unset: Bit 1: Unset "ld1 { v11.h }[0], [x20], #0x2\n" "27:" // Oddments: Load input (0, 4): Bit 2: End - "ldr x20, [x14, #0x48]\n" + "ldr x20, [x16, #0x48]\n" "fmla v18.8h, v2.8h, v11.8h\n" "fmla v19.8h, v1.8h, v11.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "tbz %x[n_channels], #2, 29f\n" "ld1 { v10.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #1, 28f\n" @@ -929,10 +929,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "30:" // Oddments: Load input (3, 3): Bit 2: Unset: Bit 1: Unset "ld1 { v10.h }[0], [x20], #0x2\n" "31:" // Oddments: Load input (3, 3): Bit 2: End - "ldr x20, [x14, #0x50]\n" + "ldr x20, [x16, #0x50]\n" "fmla v21.8h, v8.8h, v10.8h\n" "fmla v22.8h, v7.8h, v10.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v23.8h, v6.8h, v10.8h\n" "fmla v25.8h, v5.8h, v10.8h\n" "fmla v26.8h, v4.8h, v10.8h\n" @@ -960,10 +960,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "34:" // Oddments: Load input (1, 0): Bit 2: Unset: Bit 1: Unset "ld1 { v9.h }[0], [x20], #0x2\n" "35:" // Oddments: Load input (1, 0): Bit 2: End - "ldr x20, [x14, #0x58]\n" + "ldr x20, [x16, #0x58]\n" "fmla v16.8h, v3.8h, v9.8h\n" "fmla v20.8h, v0.8h, v9.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "tbz %x[n_channels], #2, 37f\n" "ld1 { v12.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #1, 36f\n" @@ -984,10 +984,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "38:" // Oddments: Load input (1, 5): Bit 2: Unset: Bit 1: Unset "ld1 { v12.h }[0], [x20], #0x2\n" "39:" // Oddments: Load input (1, 5): Bit 2: End - "ldr x20, [x14, #0x60]\n" + "ldr x20, [x16, #0x60]\n" "fmla v19.8h, v5.8h, v12.8h\n" "fmla v23.8h, v2.8h, v12.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "tbz %x[n_channels], #2, 41f\n" "ld1 { v11.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #1, 40f\n" @@ -1008,10 +1008,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "42:" // Oddments: Load input (4, 0): Bit 2: Unset: Bit 1: Unset "ld1 { v11.h }[0], [x20], #0x2\n" "43:" // Oddments: Load input (4, 0): Bit 2: End - "ldr x20, [x14, #0x68]\n" + "ldr x20, [x16, #0x68]\n" "fmla v24.8h, v6.8h, v11.8h\n" "fmla v28.8h, v3.8h, v11.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "tbz %x[n_channels], #2, 45f\n" "ld1 { v10.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #1, 44f\n" @@ -1032,10 +1032,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "46:" // Oddments: Load input (1, 2): Bit 2: Unset: Bit 1: Unset "ld1 { v10.h }[0], [x20], #0x2\n" "47:" // Oddments: Load input (1, 2): Bit 2: End - "ldr x20, [x14, #0x70]\n" + "ldr x20, [x16, #0x70]\n" "fmla v16.8h, v5.8h, v10.8h\n" "fmla v17.8h, v4.8h, v10.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v18.8h, v3.8h, v10.8h\n" "fmla v20.8h, v2.8h, v10.8h\n" "fmla v21.8h, v1.8h, v10.8h\n" @@ -1060,10 +1060,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "50:" // Oddments: Load input (4, 5): Bit 2: Unset: Bit 1: Unset "ld1 { v11.h }[0], [x20], #0x2\n" "51:" // Oddments: Load input (4, 5): Bit 2: End - "ldr x20, [x14, #0x78]\n" + "ldr x20, [x16, #0x78]\n" "fmla v27.8h, v8.8h, v11.8h\n" "fmla v31.8h, v5.8h, v11.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "tbz %x[n_channels], #2, 53f\n" "ld1 { v12.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #1, 52f\n" @@ -1084,10 +1084,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "54:" // Oddments: Load input (1, 3): Bit 2: Unset: Bit 1: Unset "ld1 { v12.h }[0], [x20], #0x2\n" "55:" // Oddments: Load input (1, 3): Bit 2: End - "ldr x20, [x14, #0x80]\n" + "ldr x20, [x16, #0x80]\n" "fmla v17.8h, v5.8h, v12.8h\n" "fmla v18.8h, v4.8h, v12.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v19.8h, v3.8h, v12.8h\n" "fmla v21.8h, v2.8h, v12.8h\n" "fmla v22.8h, v1.8h, v12.8h\n" @@ -1112,10 +1112,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "58:" // Oddments: Load input (5, 1): Bit 2: Unset: Bit 1: Unset "ld1 { v11.h }[0], [x20], #0x2\n" "59:" // Oddments: Load input (5, 1): Bit 2: End - "ldr x20, [x14, #0x88]\n" + "ldr x20, [x16, #0x88]\n" "fmla v28.8h, v7.8h, v11.8h\n" "fmla v29.8h, v6.8h, v11.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "tbz %x[n_channels], #2, 61f\n" "ld1 { v10.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #1, 60f\n" @@ -1136,10 +1136,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "62:" // Oddments: Load input (2, 1): Bit 2: Unset: Bit 1: Unset "ld1 { v10.h }[0], [x20], #0x2\n" "63:" // Oddments: Load input (2, 1): Bit 2: End - "ldr x20, [x14, #0x90]\n" + "ldr x20, [x16, #0x90]\n" "fmla v16.8h, v7.8h, v10.8h\n" "fmla v17.8h, v6.8h, v10.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v20.8h, v4.8h, v10.8h\n" "fmla v21.8h, v3.8h, v10.8h\n" "fmla v24.8h, v1.8h, v10.8h\n" @@ -1164,10 +1164,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "66:" // Oddments: Load input (5, 4): Bit 2: Unset: Bit 1: Unset "ld1 { v11.h }[0], [x20], #0x2\n" "67:" // Oddments: Load input (5, 4): Bit 2: End - "ldr x20, [x14, #0x98]\n" + "ldr x20, [x16, #0x98]\n" "fmla v30.8h, v8.8h, v11.8h\n" "fmla v31.8h, v7.8h, v11.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "tbz %x[n_channels], #2, 69f\n" "ld1 { v12.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #1, 68f\n" @@ -1188,10 +1188,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "70:" // Oddments: Load input (2, 4): Bit 2: Unset: Bit 1: Unset "ld1 { v12.h }[0], [x20], #0x2\n" "71:" // Oddments: Load input (2, 4): Bit 2: End - "ldr x20, [x14, #0xa0]\n" + "ldr x20, [x16, #0xa0]\n" "fmla v18.8h, v8.8h, v12.8h\n" "fmla v19.8h, v7.8h, v12.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v22.8h, v5.8h, v12.8h\n" "fmla v23.8h, v4.8h, v12.8h\n" "fmla v26.8h, v2.8h, v12.8h\n" @@ -1216,10 +1216,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "74:" // Oddments: Load input (0, 2): Bit 2: Unset: Bit 1: Unset "ld1 { v10.h }[0], [x20], #0x2\n" "75:" // Oddments: Load input (0, 2): Bit 2: End - "ldr x20, [x14, #0xa8]\n" + "ldr x20, [x16, #0xa8]\n" "fmla v16.8h, v2.8h, v10.8h\n" "fmla v17.8h, v1.8h, v10.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v18.8h, v0.8h, v10.8h\n" "tbz %x[n_channels], #2, 77f\n" "ld1 { v11.d }[0], [x20], #0x8\n" @@ -1241,10 +1241,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "78:" // Oddments: Load input (3, 1): Bit 2: Unset: Bit 1: Unset "ld1 { v11.h }[0], [x20], #0x2\n" "79:" // Oddments: Load input (3, 1): Bit 2: End - "ldr x20, [x14, #0xb0]\n" + "ldr x20, [x16, #0xb0]\n" "fmla v20.8h, v7.8h, v11.8h\n" "fmla v21.8h, v6.8h, v11.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v24.8h, v4.8h, v11.8h\n" "fmla v25.8h, v3.8h, v11.8h\n" "fmla v28.8h, v1.8h, v11.8h\n" @@ -1269,10 +1269,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "82:" // Oddments: Load input (0, 3): Bit 2: Unset: Bit 1: Unset "ld1 { v12.h }[0], [x20], #0x2\n" "83:" // Oddments: Load input (0, 3): Bit 2: End - "ldr x20, [x14, #0xb8]\n" + "ldr x20, [x16, #0xb8]\n" "fmla v17.8h, v2.8h, v12.8h\n" "fmla v18.8h, v1.8h, v12.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v19.8h, v0.8h, v12.8h\n" "tbz %x[n_channels], #2, 85f\n" "ld1 { v10.d }[0], [x20], #0x8\n" @@ -1294,10 +1294,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "86:" // Oddments: Load input (2, 0): Bit 2: Unset: Bit 1: Unset "ld1 { v10.h }[0], [x20], #0x2\n" "87:" // Oddments: Load input (2, 0): Bit 2: End - "ldr x20, [x14, #0xc0]\n" + "ldr x20, [x16, #0xc0]\n" "fmla v16.8h, v6.8h, v10.8h\n" "fmla v20.8h, v3.8h, v10.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v24.8h, v0.8h, v10.8h\n" "tbz %x[n_channels], #2, 89f\n" "ld1 { v11.d }[0], [x20], #0x8\n" @@ -1319,10 +1319,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "90:" // Oddments: Load input (3, 4): Bit 2: Unset: Bit 1: Unset "ld1 { v11.h }[0], [x20], #0x2\n" "91:" // Oddments: Load input (3, 4): Bit 2: End - "ldr x20, [x14, #0xc8]\n" + "ldr x20, [x16, #0xc8]\n" "fmla v22.8h, v8.8h, v11.8h\n" "fmla v23.8h, v7.8h, v11.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v26.8h, v5.8h, v11.8h\n" "fmla v27.8h, v4.8h, v11.8h\n" "fmla v30.8h, v2.8h, v11.8h\n" @@ -1347,10 +1347,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "94:" // Oddments: Load input (2, 5): Bit 2: Unset: Bit 1: Unset "ld1 { v12.h }[0], [x20], #0x2\n" "95:" // Oddments: Load input (2, 5): Bit 2: End - "ldr x20, [x14, #0xd0]\n" + "ldr x20, [x16, #0xd0]\n" "fmla v19.8h, v8.8h, v12.8h\n" "fmla v23.8h, v5.8h, v12.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v27.8h, v2.8h, v12.8h\n" "tbz %x[n_channels], #2, 97f\n" "ld1 { v10.d }[0], [x20], #0x8\n" @@ -1372,10 +1372,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "98:" // Oddments: Load input (3, 0): Bit 2: Unset: Bit 1: Unset "ld1 { v10.h }[0], [x20], #0x2\n" "99:" // Oddments: Load input (3, 0): Bit 2: End - "ldr x20, [x14, #0xd8]\n" + "ldr x20, [x16, #0xd8]\n" "fmla v20.8h, v6.8h, v10.8h\n" "fmla v24.8h, v3.8h, v10.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v28.8h, v0.8h, v10.8h\n" "tbz %x[n_channels], #2, 101f\n" "ld1 { v11.d }[0], [x20], #0x8\n" @@ -1397,10 +1397,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "102:" // Oddments: Load input (4, 2): Bit 2: Unset: Bit 1: Unset "ld1 { v11.h }[0], [x20], #0x2\n" "103:" // Oddments: Load input (4, 2): Bit 2: End - "ldr x20, [x14, #0xe0]\n" + "ldr x20, [x16, #0xe0]\n" "fmla v24.8h, v8.8h, v11.8h\n" "fmla v25.8h, v7.8h, v11.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v26.8h, v6.8h, v11.8h\n" "fmla v28.8h, v5.8h, v11.8h\n" "fmla v29.8h, v4.8h, v11.8h\n" @@ -1425,10 +1425,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "106:" // Oddments: Load input (3, 5): Bit 2: Unset: Bit 1: Unset "ld1 { v12.h }[0], [x20], #0x2\n" "107:" // Oddments: Load input (3, 5): Bit 2: End - "ldr x20, [x14, #0xe8]\n" + "ldr x20, [x16, #0xe8]\n" "fmla v23.8h, v8.8h, v12.8h\n" "fmla v27.8h, v5.8h, v12.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v31.8h, v2.8h, v12.8h\n" "tbz %x[n_channels], #2, 109f\n" "ld1 { v10.d }[0], [x20], #0x8\n" @@ -1450,10 +1450,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "110:" // Oddments: Load input (5, 2): Bit 2: Unset: Bit 1: Unset "ld1 { v10.h }[0], [x20], #0x2\n" "111:" // Oddments: Load input (5, 2): Bit 2: End - "ldr x20, [x14, #0xf0]\n" + "ldr x20, [x16, #0xf0]\n" "fmla v28.8h, v8.8h, v10.8h\n" "fmla v29.8h, v7.8h, v10.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v30.8h, v6.8h, v10.8h\n" "tbz %x[n_channels], #2, 113f\n" "ld1 { v11.d }[0], [x20], #0x8\n" @@ -1475,10 +1475,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "114:" // Oddments: Load input (4, 3): Bit 2: Unset: Bit 1: Unset "ld1 { v11.h }[0], [x20], #0x2\n" "115:" // Oddments: Load input (4, 3): Bit 2: End - "ldr x20, [x14, #0xf8]\n" + "ldr x20, [x16, #0xf8]\n" "fmla v25.8h, v8.8h, v11.8h\n" "fmla v26.8h, v7.8h, v11.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v27.8h, v6.8h, v11.8h\n" "fmla v29.8h, v5.8h, v11.8h\n" "fmla v30.8h, v4.8h, v11.8h\n" @@ -1503,10 +1503,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "118:" // Oddments: Load input (5, 3): Bit 2: Unset: Bit 1: Unset "ld1 { v12.h }[0], [x20], #0x2\n" "119:" // Oddments: Load input (5, 3): Bit 2: End - "ldr x20, [x14, #0x100]\n" + "ldr x20, [x16, #0x100]\n" "fmla v29.8h, v8.8h, v12.8h\n" "fmla v30.8h, v7.8h, v12.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v31.8h, v6.8h, v12.8h\n" "tbz %x[n_channels], #2, 121f\n" "ld1 { v10.d }[0], [x20], #0x8\n" @@ -1528,10 +1528,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "122:" // Oddments: Load input (1, 1): Bit 2: Unset: Bit 1: Unset "ld1 { v10.h }[0], [x20], #0x2\n" "123:" // Oddments: Load input (1, 1): Bit 2: End - "ldr x20, [x14, #0x108]\n" + "ldr x20, [x16, #0x108]\n" "fmla v16.8h, v4.8h, v10.8h\n" "fmla v17.8h, v3.8h, v10.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v20.8h, v1.8h, v10.8h\n" "fmla v21.8h, v0.8h, v10.8h\n" "tbz %x[n_channels], #2, 125f\n" @@ -1554,10 +1554,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "126:" // Oddments: Load input (1, 4): Bit 2: Unset: Bit 1: Unset "ld1 { v11.h }[0], [x20], #0x2\n" "127:" // Oddments: Load input (1, 4): Bit 2: End - "ldr x20, [x14, #0x110]\n" + "ldr x20, [x16, #0x110]\n" "fmla v18.8h, v5.8h, v11.8h\n" "fmla v19.8h, v4.8h, v11.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v22.8h, v2.8h, v11.8h\n" "fmla v23.8h, v1.8h, v11.8h\n" "tbz %x[n_channels], #2, 129f\n" @@ -1580,10 +1580,10 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "130:" // Oddments: Load input (4, 1): Bit 2: Unset: Bit 1: Unset "ld1 { v12.h }[0], [x20], #0x2\n" "131:" // Oddments: Load input (4, 1): Bit 2: End - "ldr x20, [x14, #0x118]\n" + "ldr x20, [x16, #0x118]\n" "fmla v24.8h, v7.8h, v12.8h\n" "fmla v25.8h, v6.8h, v12.8h\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v28.8h, v4.8h, v12.8h\n" "fmla v29.8h, v3.8h, v12.8h\n" "tbz %x[n_channels], #2, 133f\n" @@ -1608,24 +1608,24 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "135:" // Oddments: Load input (4, 4): Bit 2: End "fmla v26.8h, v8.8h, v10.8h\n" "fmla v27.8h, v7.8h, v10.8h\n" - "fmax v16.8h, v16.8h, v15.8h\n" + "fmax v16.8h, v16.8h, v13.8h\n" "fmla v30.8h, v5.8h, v10.8h\n" "fmla v31.8h, v4.8h, v10.8h\n" - "fmax v17.8h, v17.8h, v15.8h\n" - "fmax v18.8h, v18.8h, v15.8h\n" - "fmax v19.8h, v19.8h, v15.8h\n" - "fmax v20.8h, v20.8h, v15.8h\n" - "fmax v21.8h, v21.8h, v15.8h\n" - "fmax v22.8h, v22.8h, v15.8h\n" - "fmax v23.8h, v23.8h, v15.8h\n" - "fmax v24.8h, v24.8h, v15.8h\n" - "fmax v25.8h, v25.8h, v15.8h\n" - "fmax v26.8h, v26.8h, v15.8h\n" - "fmax v27.8h, v27.8h, v15.8h\n" - "fmax v28.8h, v28.8h, v15.8h\n" - "fmax v29.8h, v29.8h, v15.8h\n" - "fmax v30.8h, v30.8h, v15.8h\n" - "fmax v31.8h, v31.8h, v15.8h\n" + "fmax v17.8h, v17.8h, v13.8h\n" + "fmax v18.8h, v18.8h, v13.8h\n" + "fmax v19.8h, v19.8h, v13.8h\n" + "fmax v20.8h, v20.8h, v13.8h\n" + "fmax v21.8h, v21.8h, v13.8h\n" + "fmax v22.8h, v22.8h, v13.8h\n" + "fmax v23.8h, v23.8h, v13.8h\n" + "fmax v24.8h, v24.8h, v13.8h\n" + "fmax v25.8h, v25.8h, v13.8h\n" + "fmax v26.8h, v26.8h, v13.8h\n" + "fmax v27.8h, v27.8h, v13.8h\n" + "fmax v28.8h, v28.8h, v13.8h\n" + "fmax v29.8h, v29.8h, v13.8h\n" + "fmax v30.8h, v30.8h, v13.8h\n" + "fmax v31.8h, v31.8h, v13.8h\n" "fmin v16.8h, v16.8h, v14.8h\n" "fmin v17.8h, v17.8h, v14.8h\n" "fmin v18.8h, v18.8h, v14.8h\n" @@ -1643,150 +1643,150 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "fmin v30.8h, v30.8h, v14.8h\n" "fmin v31.8h, v31.8h, v14.8h\n" "tbz %x[n_channels], #2, 137f\n" - "ldr x23, [x16, #0x0]\n" - "ldr x22, [x16, #0x8]\n" - "add x23, x23, x12\n" - "add x22, x22, x12\n" - "ldr x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x18]\n" - "add x21, x21, x12\n" - "add x20, x20, x12\n" + "ldr x23, [x8, #0x0]\n" + "ldr x22, [x8, #0x8]\n" + "add x23, x23, x14\n" + "add x22, x22, x14\n" + "ldr x21, [x8, #0x10]\n" + "ldr x20, [x8, #0x18]\n" + "add x21, x21, x14\n" + "add x20, x20, x14\n" "st1 { v16.d }[0], [x23]\n" - "ldr x23, [x16, #0x20]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x20]\n" + "add x23, x23, x14\n" "st1 { v17.d }[0], [x22]\n" - "ldr x22, [x16, #0x28]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x28]\n" + "add x22, x22, x14\n" "st1 { v18.d }[0], [x21]\n" - "ldr x21, [x16, #0x30]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x30]\n" + "add x21, x21, x14\n" "st1 { v19.d }[0], [x20]\n" - "ldr x20, [x16, #0x38]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x38]\n" + "add x20, x20, x14\n" "st1 { v20.d }[0], [x23]\n" - "ldr x23, [x16, #0x40]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x40]\n" + "add x23, x23, x14\n" "st1 { v21.d }[0], [x22]\n" - "ldr x22, [x16, #0x48]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x48]\n" + "add x22, x22, x14\n" "st1 { v22.d }[0], [x21]\n" - "ldr x21, [x16, #0x50]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x50]\n" + "add x21, x21, x14\n" "st1 { v23.d }[0], [x20]\n" - "ldr x20, [x16, #0x58]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x58]\n" + "add x20, x20, x14\n" "st1 { v24.d }[0], [x23]\n" - "ldr x23, [x16, #0x60]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x60]\n" + "add x23, x23, x14\n" "st1 { v25.d }[0], [x22]\n" - "ldr x22, [x16, #0x68]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x68]\n" + "add x22, x22, x14\n" "st1 { v26.d }[0], [x21]\n" - "ldr x21, [x16, #0x70]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x70]\n" + "add x21, x21, x14\n" "st1 { v27.d }[0], [x20]\n" - "ldr x20, [x16, #0x78]\n" - "add x20, x20, x12\n" - "add x12, x12, #0x8\n" + "ldr x20, [x8, #0x78]\n" + "add x20, x20, x14\n" + "add x14, x14, #0x8\n" "st1 { v28.d }[0], [x23]\n" "st1 { v29.d }[0], [x22]\n" "st1 { v30.d }[0], [x21]\n" "st1 { v31.d }[0], [x20]\n" "tbz %x[n_channels], #1, 136f\n" - "ldr x23, [x16, #0x0]\n" - "ldr x22, [x16, #0x8]\n" - "add x23, x23, x12\n" - "add x22, x22, x12\n" - "ldr x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x18]\n" - "add x21, x21, x12\n" - "add x20, x20, x12\n" + "ldr x23, [x8, #0x0]\n" + "ldr x22, [x8, #0x8]\n" + "add x23, x23, x14\n" + "add x22, x22, x14\n" + "ldr x21, [x8, #0x10]\n" + "ldr x20, [x8, #0x18]\n" + "add x21, x21, x14\n" + "add x20, x20, x14\n" "st1 { v16.s }[2], [x23]\n" - "ldr x23, [x16, #0x20]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x20]\n" + "add x23, x23, x14\n" "st1 { v17.s }[2], [x22]\n" - "ldr x22, [x16, #0x28]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x28]\n" + "add x22, x22, x14\n" "st1 { v18.s }[2], [x21]\n" - "ldr x21, [x16, #0x30]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x30]\n" + "add x21, x21, x14\n" "st1 { v19.s }[2], [x20]\n" - "ldr x20, [x16, #0x38]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x38]\n" + "add x20, x20, x14\n" "st1 { v20.s }[2], [x23]\n" - "ldr x23, [x16, #0x40]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x40]\n" + "add x23, x23, x14\n" "st1 { v21.s }[2], [x22]\n" - "ldr x22, [x16, #0x48]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x48]\n" + "add x22, x22, x14\n" "st1 { v22.s }[2], [x21]\n" - "ldr x21, [x16, #0x50]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x50]\n" + "add x21, x21, x14\n" "st1 { v23.s }[2], [x20]\n" - "ldr x20, [x16, #0x58]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x58]\n" + "add x20, x20, x14\n" "st1 { v24.s }[2], [x23]\n" - "ldr x23, [x16, #0x60]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x60]\n" + "add x23, x23, x14\n" "st1 { v25.s }[2], [x22]\n" - "ldr x22, [x16, #0x68]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x68]\n" + "add x22, x22, x14\n" "st1 { v26.s }[2], [x21]\n" - "ldr x21, [x16, #0x70]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x70]\n" + "add x21, x21, x14\n" "st1 { v27.s }[2], [x20]\n" - "ldr x20, [x16, #0x78]\n" - "add x20, x20, x12\n" - "add x12, x12, #0x4\n" + "ldr x20, [x8, #0x78]\n" + "add x20, x20, x14\n" + "add x14, x14, #0x4\n" "st1 { v28.s }[2], [x23]\n" "st1 { v29.s }[2], [x22]\n" "st1 { v30.s }[2], [x21]\n" "st1 { v31.s }[2], [x20]\n" "tbz %x[n_channels], #0, 139f\n" - "ldr x23, [x16, #0x0]\n" - "ldr x22, [x16, #0x8]\n" - "add x23, x23, x12\n" - "add x22, x22, x12\n" - "ldr x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x18]\n" - "add x21, x21, x12\n" - "add x20, x20, x12\n" + "ldr x23, [x8, #0x0]\n" + "ldr x22, [x8, #0x8]\n" + "add x23, x23, x14\n" + "add x22, x22, x14\n" + "ldr x21, [x8, #0x10]\n" + "ldr x20, [x8, #0x18]\n" + "add x21, x21, x14\n" + "add x20, x20, x14\n" "st1 { v16.h }[6], [x23]\n" - "ldr x23, [x16, #0x20]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x20]\n" + "add x23, x23, x14\n" "st1 { v17.h }[6], [x22]\n" - "ldr x22, [x16, #0x28]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x28]\n" + "add x22, x22, x14\n" "st1 { v18.h }[6], [x21]\n" - "ldr x21, [x16, #0x30]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x30]\n" + "add x21, x21, x14\n" "st1 { v19.h }[6], [x20]\n" - "ldr x20, [x16, #0x38]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x38]\n" + "add x20, x20, x14\n" "st1 { v20.h }[6], [x23]\n" - "ldr x23, [x16, #0x40]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x40]\n" + "add x23, x23, x14\n" "st1 { v21.h }[6], [x22]\n" - "ldr x22, [x16, #0x48]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x48]\n" + "add x22, x22, x14\n" "st1 { v22.h }[6], [x21]\n" - "ldr x21, [x16, #0x50]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x50]\n" + "add x21, x21, x14\n" "st1 { v23.h }[6], [x20]\n" - "ldr x20, [x16, #0x58]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x58]\n" + "add x20, x20, x14\n" "st1 { v24.h }[6], [x23]\n" - "ldr x23, [x16, #0x60]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x60]\n" + "add x23, x23, x14\n" "st1 { v25.h }[6], [x22]\n" - "ldr x22, [x16, #0x68]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x68]\n" + "add x22, x22, x14\n" "st1 { v26.h }[6], [x21]\n" - "ldr x21, [x16, #0x70]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x70]\n" + "add x21, x21, x14\n" "st1 { v27.h }[6], [x20]\n" - "ldr x20, [x16, #0x78]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x78]\n" + "add x20, x20, x14\n" "st1 { v28.h }[6], [x23]\n" "st1 { v29.h }[6], [x22]\n" "st1 { v30.h }[6], [x21]\n" @@ -1794,50 +1794,50 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "b 139f\n" "136:" // Oddments: Store: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 139f\n" - "ldr x23, [x16, #0x0]\n" - "ldr x22, [x16, #0x8]\n" - "add x23, x23, x12\n" - "add x22, x22, x12\n" - "ldr x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x18]\n" - "add x21, x21, x12\n" - "add x20, x20, x12\n" + "ldr x23, [x8, #0x0]\n" + "ldr x22, [x8, #0x8]\n" + "add x23, x23, x14\n" + "add x22, x22, x14\n" + "ldr x21, [x8, #0x10]\n" + "ldr x20, [x8, #0x18]\n" + "add x21, x21, x14\n" + "add x20, x20, x14\n" "st1 { v16.h }[4], [x23]\n" - "ldr x23, [x16, #0x20]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x20]\n" + "add x23, x23, x14\n" "st1 { v17.h }[4], [x22]\n" - "ldr x22, [x16, #0x28]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x28]\n" + "add x22, x22, x14\n" "st1 { v18.h }[4], [x21]\n" - "ldr x21, [x16, #0x30]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x30]\n" + "add x21, x21, x14\n" "st1 { v19.h }[4], [x20]\n" - "ldr x20, [x16, #0x38]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x38]\n" + "add x20, x20, x14\n" "st1 { v20.h }[4], [x23]\n" - "ldr x23, [x16, #0x40]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x40]\n" + "add x23, x23, x14\n" "st1 { v21.h }[4], [x22]\n" - "ldr x22, [x16, #0x48]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x48]\n" + "add x22, x22, x14\n" "st1 { v22.h }[4], [x21]\n" - "ldr x21, [x16, #0x50]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x50]\n" + "add x21, x21, x14\n" "st1 { v23.h }[4], [x20]\n" - "ldr x20, [x16, #0x58]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x58]\n" + "add x20, x20, x14\n" "st1 { v24.h }[4], [x23]\n" - "ldr x23, [x16, #0x60]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x60]\n" + "add x23, x23, x14\n" "st1 { v25.h }[4], [x22]\n" - "ldr x22, [x16, #0x68]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x68]\n" + "add x22, x22, x14\n" "st1 { v26.h }[4], [x21]\n" - "ldr x21, [x16, #0x70]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x70]\n" + "add x21, x21, x14\n" "st1 { v27.h }[4], [x20]\n" - "ldr x20, [x16, #0x78]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x78]\n" + "add x20, x20, x14\n" "st1 { v28.h }[4], [x23]\n" "st1 { v29.h }[4], [x22]\n" "st1 { v30.h }[4], [x21]\n" @@ -1845,161 +1845,159 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "b 139f\n" "137:" // Oddments: Store: Bit 2: Unset "tbz %x[n_channels], #1, 138f\n" - "ldr x23, [x16, #0x0]\n" - "ldr x22, [x16, #0x8]\n" - "add x23, x23, x12\n" - "add x22, x22, x12\n" - "ldr x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x18]\n" - "add x21, x21, x12\n" - "add x20, x20, x12\n" + "ldr x23, [x8, #0x0]\n" + "ldr x22, [x8, #0x8]\n" + "add x23, x23, x14\n" + "add x22, x22, x14\n" + "ldr x21, [x8, #0x10]\n" + "ldr x20, [x8, #0x18]\n" + "add x21, x21, x14\n" + "add x20, x20, x14\n" "st1 { v16.s }[0], [x23]\n" - "ldr x23, [x16, #0x20]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x20]\n" + "add x23, x23, x14\n" "st1 { v17.s }[0], [x22]\n" - "ldr x22, [x16, #0x28]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x28]\n" + "add x22, x22, x14\n" "st1 { v18.s }[0], [x21]\n" - "ldr x21, [x16, #0x30]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x30]\n" + "add x21, x21, x14\n" "st1 { v19.s }[0], [x20]\n" - "ldr x20, [x16, #0x38]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x38]\n" + "add x20, x20, x14\n" "st1 { v20.s }[0], [x23]\n" - "ldr x23, [x16, #0x40]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x40]\n" + "add x23, x23, x14\n" "st1 { v21.s }[0], [x22]\n" - "ldr x22, [x16, #0x48]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x48]\n" + "add x22, x22, x14\n" "st1 { v22.s }[0], [x21]\n" - "ldr x21, [x16, #0x50]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x50]\n" + "add x21, x21, x14\n" "st1 { v23.s }[0], [x20]\n" - "ldr x20, [x16, #0x58]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x58]\n" + "add x20, x20, x14\n" "st1 { v24.s }[0], [x23]\n" - "ldr x23, [x16, #0x60]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x60]\n" + "add x23, x23, x14\n" "st1 { v25.s }[0], [x22]\n" - "ldr x22, [x16, #0x68]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x68]\n" + "add x22, x22, x14\n" "st1 { v26.s }[0], [x21]\n" - "ldr x21, [x16, #0x70]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x70]\n" + "add x21, x21, x14\n" "st1 { v27.s }[0], [x20]\n" - "ldr x20, [x16, #0x78]\n" - "add x20, x20, x12\n" - "add x12, x12, #0x4\n" + "ldr x20, [x8, #0x78]\n" + "add x20, x20, x14\n" + "add x14, x14, #0x4\n" "st1 { v28.s }[0], [x23]\n" "st1 { v29.s }[0], [x22]\n" "st1 { v30.s }[0], [x21]\n" "st1 { v31.s }[0], [x20]\n" "tbz %x[n_channels], #0, 139f\n" - "ldr x23, [x16, #0x0]\n" - "ldr x22, [x16, #0x8]\n" - "add x23, x23, x12\n" - "add x22, x22, x12\n" - "ldr x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x18]\n" - "add x21, x21, x12\n" - "add x20, x20, x12\n" + "ldr x23, [x8, #0x0]\n" + "ldr x22, [x8, #0x8]\n" + "add x23, x23, x14\n" + "add x22, x22, x14\n" + "ldr x21, [x8, #0x10]\n" + "ldr x20, [x8, #0x18]\n" + "add x21, x21, x14\n" + "add x20, x20, x14\n" "st1 { v16.h }[2], [x23]\n" - "ldr x23, [x16, #0x20]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x20]\n" + "add x23, x23, x14\n" "st1 { v17.h }[2], [x22]\n" - "ldr x22, [x16, #0x28]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x28]\n" + "add x22, x22, x14\n" "st1 { v18.h }[2], [x21]\n" - "ldr x21, [x16, #0x30]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x30]\n" + "add x21, x21, x14\n" "st1 { v19.h }[2], [x20]\n" - "ldr x20, [x16, #0x38]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x38]\n" + "add x20, x20, x14\n" "st1 { v20.h }[2], [x23]\n" - "ldr x23, [x16, #0x40]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x40]\n" + "add x23, x23, x14\n" "st1 { v21.h }[2], [x22]\n" - "ldr x22, [x16, #0x48]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x48]\n" + "add x22, x22, x14\n" "st1 { v22.h }[2], [x21]\n" - "ldr x21, [x16, #0x50]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x50]\n" + "add x21, x21, x14\n" "st1 { v23.h }[2], [x20]\n" - "ldr x20, [x16, #0x58]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x58]\n" + "add x20, x20, x14\n" "st1 { v24.h }[2], [x23]\n" - "ldr x23, [x16, #0x60]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x60]\n" + "add x23, x23, x14\n" "st1 { v25.h }[2], [x22]\n" - "ldr x22, [x16, #0x68]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x68]\n" + "add x22, x22, x14\n" "st1 { v26.h }[2], [x21]\n" - "ldr x21, [x16, #0x70]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x70]\n" + "add x21, x21, x14\n" "st1 { v27.h }[2], [x20]\n" - "ldr x20, [x16, #0x78]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x78]\n" + "add x20, x20, x14\n" "st1 { v28.h }[2], [x23]\n" "st1 { v29.h }[2], [x22]\n" "st1 { v30.h }[2], [x21]\n" "st1 { v31.h }[2], [x20]\n" "b 139f\n" "138:" // Oddments: Store: Bit 2: Unset: Bit 1: Unset - "ldr x23, [x16, #0x0]\n" - "ldr x22, [x16, #0x8]\n" - "add x23, x23, x12\n" - "add x22, x22, x12\n" - "ldr x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x18]\n" - "add x21, x21, x12\n" - "add x20, x20, x12\n" + "ldr x23, [x8, #0x0]\n" + "ldr x22, [x8, #0x8]\n" + "add x23, x23, x14\n" + "add x22, x22, x14\n" + "ldr x21, [x8, #0x10]\n" + "ldr x20, [x8, #0x18]\n" + "add x21, x21, x14\n" + "add x20, x20, x14\n" "st1 { v16.h }[0], [x23]\n" - "ldr x23, [x16, #0x20]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x20]\n" + "add x23, x23, x14\n" "st1 { v17.h }[0], [x22]\n" - "ldr x22, [x16, #0x28]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x28]\n" + "add x22, x22, x14\n" "st1 { v18.h }[0], [x21]\n" - "ldr x21, [x16, #0x30]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x30]\n" + "add x21, x21, x14\n" "st1 { v19.h }[0], [x20]\n" - "ldr x20, [x16, #0x38]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x38]\n" + "add x20, x20, x14\n" "st1 { v20.h }[0], [x23]\n" - "ldr x23, [x16, #0x40]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x40]\n" + "add x23, x23, x14\n" "st1 { v21.h }[0], [x22]\n" - "ldr x22, [x16, #0x48]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x48]\n" + "add x22, x22, x14\n" "st1 { v22.h }[0], [x21]\n" - "ldr x21, [x16, #0x50]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x50]\n" + "add x21, x21, x14\n" "st1 { v23.h }[0], [x20]\n" - "ldr x20, [x16, #0x58]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x58]\n" + "add x20, x20, x14\n" "st1 { v24.h }[0], [x23]\n" - "ldr x23, [x16, #0x60]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x60]\n" + "add x23, x23, x14\n" "st1 { v25.h }[0], [x22]\n" - "ldr x22, [x16, #0x68]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x68]\n" + "add x22, x22, x14\n" "st1 { v26.h }[0], [x21]\n" - "ldr x21, [x16, #0x70]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x70]\n" + "add x21, x21, x14\n" "st1 { v27.h }[0], [x20]\n" - "ldr x20, [x16, #0x78]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x78]\n" + "add x20, x20, x14\n" "st1 { v28.h }[0], [x23]\n" "st1 { v29.h }[0], [x22]\n" "st1 { v30.h }[0], [x21]\n" "st1 { v31.h }[0], [x20]\n" "139:" // Oddments: Store: Bit 2: End - "140:" // End - : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp index d52f48064f24e67b26d52de910797bd0b0b26b76..8ad6a37fea8f21767394e0eba64c8dd654cc4b10 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include @@ -33,8 +33,8 @@ namespace arm_conv { namespace depthwise { -void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); -void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); +void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const input_ptrs, __fp16 *const *const outptrs, const void *params, unsigned int n_channels, const __fp16 activation_min, const __fp16 activation_max); +void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(const unsigned int n_tile_rows, const unsigned int n_tile_cols, const __fp16 *inptr, int64_t ld_input_row, int64_t ld_input_col, __fp16 *outptr, int64_t ld_output_row, int64_t ld_output_col, const void *params, unsigned int n_channels, const __fp16 activation_min, const __fp16 activation_max); class a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst : public DepthwiseDepthfirstStrategy<__fp16, __fp16, __fp16, __fp16> { @@ -57,7 +57,7 @@ class a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst : public DepthwiseDepthfirst constexpr static unsigned int output_cols = 2; a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst(const CPUInfo *) - : DepthwiseDepthfirstStrategy<__fp16, __fp16, __fp16, __fp16>(2, 3, 2) {} + : Parent(output_rows, output_cols, kernel_rows, kernel_cols, stride_rows, stride_cols) {} arm_gemm::VLType get_vl_type(void) const override { return vl_type; } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp index 268dda531da79652355cfe77bf88624984006ce6..8954999990793b5e15f5ffffb27a951b33317052 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp @@ -118,9 +118,9 @@ void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "add x11, x13, x6\n" "add x17, x17, x20, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16) "add x20, %x[params_struct], %[offsetof_args_min]\n" - "ld1r { v19.8h }, [x20]\n" + "ld1r { v26.8h }, [x20]\n" "add x20, %x[params_struct], %[offsetof_args_max]\n" - "ld1r { v18.8h }, [x20]\n" + "ld1r { v27.8h }, [x20]\n" "add x10, x12, x24, LSL #1\n" "add x9, x11, x6\n" "add x28, x17, x21, LSL #1\n" @@ -128,7 +128,7 @@ void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "mov x21, #0x0\n" "sub x20, XZR, x23\n" "cbz x22, 4f\n" - "ldr q17, [x15, #0x0]\n" + "ldr q31, [x15, #0x0]\n" "ldr q0, [x15, #0x10]\n" "cmp x23, x22, LSL #4\n" "ldr q1, [x15, #0x20]\n" @@ -150,179 +150,179 @@ void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "ldr q16, [x8, x13]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "mov v28.16b, v17.16b\n fmla v28.8h, v8.8h, v9.8h\n" - "mov v29.16b, v17.16b\n fmla v29.8h, v6.8h, v9.8h\n" + "mov v29.16b, v31.16b\n fmla v29.8h, v8.8h, v9.8h\n" + "mov v28.16b, v31.16b\n fmla v28.8h, v6.8h, v9.8h\n" "add x23, x23, #0x10\n" "add x8, x8, #0x10\n" - "fmla v28.8h, v0.8h, v10.8h\n" + "fmla v29.8h, v0.8h, v10.8h\n" "ld1 { v10.8h }, [x8]\n" - "fmla v29.8h, v1.8h, v12.8h\n" - "ldr q12, [x16, x9]\n" - "fmla v28.8h, v1.8h, v11.8h\n" - "ldr q11, [x16, x11]\n" - "fmla v29.8h, v2.8h, v13.8h\n" - "ldr q13, [x16, x13]\n" - "fmla v28.8h, v3.8h, v14.8h\n" - "ld1 { v14.8h }, [x12]\n" - "fmla v29.8h, v0.8h, v16.8h\n" + "fmla v28.8h, v1.8h, v12.8h\n" + "ldr q21, [x16, x9]\n" + "fmla v29.8h, v1.8h, v11.8h\n" + "ldr q18, [x16, x11]\n" + "fmla v28.8h, v2.8h, v13.8h\n" + "ldr q17, [x16, x13]\n" + "fmla v29.8h, v3.8h, v14.8h\n" + "ld1 { v20.8h }, [x12]\n" + "fmla v28.8h, v0.8h, v16.8h\n" "add x16, x16, #0x10\n" - "fmla v28.8h, v4.8h, v15.8h\n" - "ld1 { v15.8h }, [x14]\n" - "fmla v29.8h, v4.8h, v11.8h\n" - "ldr q11, [x12, x6]\n" - "fmla v28.8h, v2.8h, v16.8h\n" - "ldr q16, [x14, x6]\n" - "fmla v29.8h, v5.8h, v12.8h\n" - "ldr q12, [x14, x11]\n" - "mov v30.16b, v17.16b\n fmla v30.8h, v2.8h, v9.8h\n" - "mov v31.16b, v17.16b\n fmla v31.8h, v0.8h, v9.8h\n" - "ldr q17, [x15, #0x0]\n" + "fmla v29.8h, v4.8h, v15.8h\n" + "ld1 { v25.8h }, [x14]\n" + "fmla v28.8h, v4.8h, v18.8h\n" + "ldr q19, [x12, x6]\n" + "fmla v29.8h, v2.8h, v16.8h\n" + "ldr q18, [x14, x6]\n" + "fmla v28.8h, v5.8h, v21.8h\n" + "ldr q24, [x14, x11]\n" + "mov v23.16b, v31.16b\n fmla v23.8h, v2.8h, v9.8h\n" + "mov v22.16b, v31.16b\n fmla v22.8h, v0.8h, v9.8h\n" + "ldr q31, [x15, #0x0]\n" "cmp x23, x22, LSL #4\n" - "fmla v28.8h, v5.8h, v13.8h\n" - "fmla v29.8h, v3.8h, v13.8h\n" - "ldr q13, [x12, x11]\n" + "fmla v29.8h, v5.8h, v17.8h\n" + "fmla v28.8h, v3.8h, v17.8h\n" + "ldr q17, [x12, x11]\n" "add x20, x20, #0x10\n" - "fmla v30.8h, v3.8h, v14.8h\n" - "ldr q14, [x12, x9]\n" - "fmla v31.8h, v4.8h, v13.8h\n" - "ldr q13, [x10, x6]\n" - "fmla v30.8h, v0.8h, v15.8h\n" + "fmla v23.8h, v3.8h, v20.8h\n" + "ldr q16, [x12, x9]\n" + "fmla v22.8h, v4.8h, v17.8h\n" + "ldr q21, [x10, x6]\n" + "fmla v23.8h, v0.8h, v25.8h\n" "ldr q0, [x15, #0x10]\n" - "fmla v31.8h, v1.8h, v12.8h\n" + "fmla v22.8h, v1.8h, v24.8h\n" "add x21, x21, #0x10\n" - "fmla v30.8h, v4.8h, v11.8h\n" - "ldr q11, [x14, x9]\n" + "fmla v23.8h, v4.8h, v19.8h\n" + "ldr q20, [x14, x9]\n" "ldr q4, [x15, #0x50]\n" - "fmla v31.8h, v5.8h, v14.8h\n" - "ldr q14, [x10, x11]\n" - "fmla v28.8h, v6.8h, v15.8h\n" - "ld1 { v15.8h }, [x10]\n" - "fmla v30.8h, v1.8h, v16.8h\n" + "fmla v22.8h, v5.8h, v16.8h\n" + "ldr q19, [x10, x11]\n" + "fmla v29.8h, v6.8h, v25.8h\n" + "ld1 { v17.8h }, [x10]\n" + "fmla v23.8h, v1.8h, v18.8h\n" "ldr q1, [x15, #0x20]\n" - "fmla v31.8h, v2.8h, v11.8h\n" + "fmla v22.8h, v2.8h, v20.8h\n" "ldr q2, [x15, #0x30]\n" - "fmla v28.8h, v7.8h, v16.8h\n" + "fmla v29.8h, v7.8h, v18.8h\n" "ldr q16, [x12, x13]\n" - "fmla v30.8h, v6.8h, v15.8h\n" - "ldr q15, [x10, x13]\n" - "fmla v31.8h, v3.8h, v16.8h\n" + "fmla v23.8h, v6.8h, v17.8h\n" + "ldr q18, [x10, x13]\n" + "fmla v22.8h, v3.8h, v16.8h\n" "ldr q3, [x15, #0x40]\n" - "fmla v30.8h, v7.8h, v13.8h\n" + "fmla v23.8h, v7.8h, v21.8h\n" "ldr q13, [x8, x9]\n" - "fmla v31.8h, v7.8h, v14.8h\n" + "fmla v22.8h, v7.8h, v19.8h\n" "ld1 { v14.8h }, [x16]\n" - "fmla v29.8h, v7.8h, v12.8h\n" + "fmla v28.8h, v7.8h, v24.8h\n" "ldr q12, [x8, x11]\n" - "fmla v30.8h, v5.8h, v16.8h\n" + "fmla v23.8h, v5.8h, v16.8h\n" "ldr q16, [x8, x13]\n" "ldr q5, [x15, #0x60]\n" - "fmla v31.8h, v6.8h, v15.8h\n" - "fmla v29.8h, v8.8h, v11.8h\n" - "ldr q11, [x10, x9]\n" + "fmla v22.8h, v6.8h, v18.8h\n" + "fmla v28.8h, v8.8h, v20.8h\n" + "ldr q17, [x10, x9]\n" "ldr q6, [x15, #0x70]\n" - "fmla v30.8h, v8.8h, v15.8h\n" - "fmla v31.8h, v8.8h, v11.8h\n" + "fmla v23.8h, v8.8h, v18.8h\n" + "fmla v22.8h, v8.8h, v17.8h\n" "ldr q11, [x8, x6]\n" "ldr q15, [x16, x6]\n" - "fmax v28.8h, v28.8h, v19.8h\n" - "fmax v29.8h, v29.8h, v19.8h\n" + "fmax v29.8h, v29.8h, v26.8h\n" + "fmax v28.8h, v28.8h, v26.8h\n" "ldr q7, [x15, #0x80]\n" "ldr q8, [x15, #0x90]\n" - "fmax v30.8h, v30.8h, v19.8h\n" - "fmax v31.8h, v31.8h, v19.8h\n" + "fmax v23.8h, v23.8h, v26.8h\n" + "fmax v22.8h, v22.8h, v26.8h\n" "add x14, x14, #0x10\n" "ldr q9, [x14, x13]\n" - "fmin v28.8h, v28.8h, v18.8h\n" - "fmin v29.8h, v29.8h, v18.8h\n" - "fmin v30.8h, v30.8h, v18.8h\n" - "fmin v31.8h, v31.8h, v18.8h\n" + "fmin v29.8h, v29.8h, v27.8h\n" + "fmin v28.8h, v28.8h, v27.8h\n" + "fmin v23.8h, v23.8h, v27.8h\n" + "fmin v22.8h, v22.8h, v27.8h\n" "add x12, x12, #0x10\n" "add x10, x10, #0x10\n" - "st1 { v28.8h }, [x17]\n" + "st1 { v29.8h }, [x17]\n" "add x15, x15, #0xa0\n" - "str q29, [x17, x7]\n" + "str q28, [x17, x7]\n" "add x17, x17, #0x10\n" - "st1 { v30.8h }, [x28]\n" - "str q31, [x28, x7]\n" + "st1 { v23.8h }, [x28]\n" + "str q22, [x28, x7]\n" "add x28, x28, #0x10\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "mov v28.16b, v17.16b\n fmla v28.8h, v8.8h, v9.8h\n" - "mov v29.16b, v17.16b\n fmla v29.8h, v6.8h, v9.8h\n" + "mov v29.16b, v31.16b\n fmla v29.8h, v8.8h, v9.8h\n" + "mov v28.16b, v31.16b\n fmla v28.8h, v6.8h, v9.8h\n" "add x8, x8, #0x10\n" - "fmla v28.8h, v0.8h, v10.8h\n" - "fmla v29.8h, v1.8h, v12.8h\n" - "ldr q12, [x16, x9]\n" - "fmla v28.8h, v1.8h, v11.8h\n" - "ldr q11, [x16, x11]\n" - "fmla v29.8h, v2.8h, v13.8h\n" - "ldr q13, [x16, x13]\n" - "fmla v28.8h, v3.8h, v14.8h\n" - "ld1 { v14.8h }, [x12]\n" - "fmla v29.8h, v0.8h, v16.8h\n" + "fmla v29.8h, v0.8h, v10.8h\n" + "fmla v28.8h, v1.8h, v12.8h\n" + "ldr q20, [x16, x9]\n" + "fmla v29.8h, v1.8h, v11.8h\n" + "ldr q18, [x16, x11]\n" + "fmla v28.8h, v2.8h, v13.8h\n" + "ldr q17, [x16, x13]\n" + "fmla v29.8h, v3.8h, v14.8h\n" + "ld1 { v19.8h }, [x12]\n" + "fmla v28.8h, v0.8h, v16.8h\n" "add x16, x16, #0x10\n" - "fmla v28.8h, v4.8h, v15.8h\n" - "ld1 { v15.8h }, [x14]\n" - "fmla v29.8h, v4.8h, v11.8h\n" - "ldr q11, [x12, x6]\n" - "fmla v28.8h, v2.8h, v16.8h\n" - "ldr q16, [x14, x6]\n" - "fmla v29.8h, v5.8h, v12.8h\n" - "ldr q12, [x14, x11]\n" - "mov v30.16b, v17.16b\n fmla v30.8h, v2.8h, v9.8h\n" - "mov v31.16b, v17.16b\n fmla v31.8h, v0.8h, v9.8h\n" - "fmla v28.8h, v5.8h, v13.8h\n" - "fmla v29.8h, v3.8h, v13.8h\n" - "ldr q13, [x12, x11]\n" - "fmla v30.8h, v3.8h, v14.8h\n" - "ldr q14, [x12, x9]\n" - "fmla v31.8h, v4.8h, v13.8h\n" - "ldr q13, [x10, x6]\n" - "fmla v30.8h, v0.8h, v15.8h\n" - "fmla v31.8h, v1.8h, v12.8h\n" - "fmla v30.8h, v4.8h, v11.8h\n" - "ldr q11, [x14, x9]\n" - "fmla v31.8h, v5.8h, v14.8h\n" - "ldr q14, [x10, x11]\n" - "fmla v28.8h, v6.8h, v15.8h\n" - "ld1 { v15.8h }, [x10]\n" - "fmla v30.8h, v1.8h, v16.8h\n" + "fmla v29.8h, v4.8h, v15.8h\n" + "ld1 { v25.8h }, [x14]\n" + "fmla v28.8h, v4.8h, v18.8h\n" + "ldr q18, [x12, x6]\n" + "fmla v29.8h, v2.8h, v16.8h\n" + "ldr q24, [x14, x6]\n" + "fmla v28.8h, v5.8h, v20.8h\n" + "ldr q23, [x14, x11]\n" + "mov v22.16b, v31.16b\n fmla v22.8h, v2.8h, v9.8h\n" + "mov v21.16b, v31.16b\n fmla v21.8h, v0.8h, v9.8h\n" + "fmla v29.8h, v5.8h, v17.8h\n" + "fmla v28.8h, v3.8h, v17.8h\n" + "ldr q17, [x12, x11]\n" + "fmla v22.8h, v3.8h, v19.8h\n" + "ldr q16, [x12, x9]\n" + "fmla v21.8h, v4.8h, v17.8h\n" + "ldr q20, [x10, x6]\n" + "fmla v22.8h, v0.8h, v25.8h\n" + "fmla v21.8h, v1.8h, v23.8h\n" + "fmla v22.8h, v4.8h, v18.8h\n" + "ldr q19, [x14, x9]\n" + "fmla v21.8h, v5.8h, v16.8h\n" + "ldr q18, [x10, x11]\n" + "fmla v29.8h, v6.8h, v25.8h\n" + "ld1 { v17.8h }, [x10]\n" + "fmla v22.8h, v1.8h, v24.8h\n" "add x14, x14, #0x10\n" - "fmla v31.8h, v2.8h, v11.8h\n" - "fmla v28.8h, v7.8h, v16.8h\n" + "fmla v21.8h, v2.8h, v19.8h\n" + "fmla v29.8h, v7.8h, v24.8h\n" "ldr q16, [x12, x13]\n" - "fmax v28.8h, v28.8h, v19.8h\n" - "fmla v30.8h, v6.8h, v15.8h\n" - "ldr q15, [x10, x13]\n" - "fmla v31.8h, v3.8h, v16.8h\n" - "fmin v28.8h, v28.8h, v18.8h\n" - "fmla v30.8h, v7.8h, v13.8h\n" - "fmla v31.8h, v7.8h, v14.8h\n" - "st1 { v28.8h }, [x17]\n" + "fmax v29.8h, v29.8h, v26.8h\n" + "fmla v22.8h, v6.8h, v17.8h\n" + "ldr q17, [x10, x13]\n" + "fmla v21.8h, v3.8h, v16.8h\n" + "fmin v29.8h, v29.8h, v27.8h\n" + "fmla v22.8h, v7.8h, v20.8h\n" + "fmla v21.8h, v7.8h, v18.8h\n" + "st1 { v29.8h }, [x17]\n" "add x12, x12, #0x10\n" - "fmla v29.8h, v7.8h, v12.8h\n" - "fmla v30.8h, v5.8h, v16.8h\n" - "fmla v31.8h, v6.8h, v15.8h\n" - "fmla v29.8h, v8.8h, v11.8h\n" - "ldr q11, [x10, x9]\n" - "fmax v29.8h, v29.8h, v19.8h\n" - "fmla v30.8h, v8.8h, v15.8h\n" - "fmla v31.8h, v8.8h, v11.8h\n" - "fmax v30.8h, v30.8h, v19.8h\n" + "fmla v28.8h, v7.8h, v23.8h\n" + "fmla v22.8h, v5.8h, v16.8h\n" + "fmla v21.8h, v6.8h, v17.8h\n" + "fmla v28.8h, v8.8h, v19.8h\n" + "ldr q16, [x10, x9]\n" + "fmax v28.8h, v28.8h, v26.8h\n" + "fmla v22.8h, v8.8h, v17.8h\n" + "fmla v21.8h, v8.8h, v16.8h\n" + "fmax v22.8h, v22.8h, v26.8h\n" "add x10, x10, #0x10\n" - "fmax v31.8h, v31.8h, v19.8h\n" - "fmin v29.8h, v29.8h, v18.8h\n" - "str q29, [x17, x7]\n" + "fmax v21.8h, v21.8h, v26.8h\n" + "fmin v28.8h, v28.8h, v27.8h\n" + "str q28, [x17, x7]\n" "add x17, x17, #0x10\n" - "fmin v30.8h, v30.8h, v18.8h\n" - "fmin v31.8h, v31.8h, v18.8h\n" - "st1 { v30.8h }, [x28]\n" - "str q31, [x28, x7]\n" + "fmin v22.8h, v22.8h, v27.8h\n" + "fmin v21.8h, v21.8h, v27.8h\n" + "st1 { v22.8h }, [x28]\n" + "str q21, [x28, x7]\n" "add x28, x28, #0x10\n" "4:" // Tile loop: Oddments "tst %x[n_channels], #0x7\n" "beq 81f\n" - "ldr q17, [x15, #0x0]\n" + "ldr q31, [x15, #0x0]\n" "ldr q0, [x15, #0x10]\n" "add x27, x14, x13\n" "add x26, x8, XZR\n" @@ -409,17 +409,17 @@ void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "ldr h15, [x21, #0x0]\n" "ldr h16, [x20, #0x0]\n" "8:" // Tile loop: Oddments: Load inputs: (2, 2), (0, 0), (0, 1), (0, 3), (0, 4), (1, 0), (1, 1), (0, 2): Bit 2: End - "mov v28.16b, v17.16b\n fmla v28.8h, v8.8h, v9.8h\n" + "mov v28.16b, v31.16b\n fmla v28.8h, v8.8h, v9.8h\n" "fmla v28.8h, v0.8h, v10.8h\n" "add x20, x16, x11\n" - "mov v29.16b, v17.16b\n fmla v29.8h, v6.8h, v9.8h\n" + "mov v29.16b, v31.16b\n fmla v29.8h, v6.8h, v9.8h\n" "fmla v28.8h, v1.8h, v11.8h\n" "fmla v29.8h, v1.8h, v12.8h\n" "fmla v28.8h, v3.8h, v14.8h\n" "fmla v29.8h, v2.8h, v13.8h\n" "fmla v28.8h, v4.8h, v15.8h\n" - "mov v30.16b, v17.16b\n fmla v30.8h, v2.8h, v9.8h\n" - "mov v31.16b, v17.16b\n fmla v31.8h, v0.8h, v9.8h\n" + "mov v30.16b, v31.16b\n fmla v30.8h, v2.8h, v9.8h\n" + "fmla v31.8h, v0.8h, v9.8h\n" "fmla v28.8h, v2.8h, v16.8h\n" "fmla v29.8h, v0.8h, v16.8h\n" "tbz %x[n_channels], #2, 10f\n" @@ -802,14 +802,14 @@ void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "ldr h11, [x20, #0x0]\n" "76:" // Tile loop: Oddments: Load inputs: (4, 4): Bit 2: End "fmla v31.8h, v8.8h, v11.8h\n" - "fmax v28.8h, v28.8h, v19.8h\n" - "fmax v29.8h, v29.8h, v19.8h\n" - "fmax v30.8h, v30.8h, v19.8h\n" - "fmax v31.8h, v31.8h, v19.8h\n" - "fmin v28.8h, v28.8h, v18.8h\n" - "fmin v29.8h, v29.8h, v18.8h\n" - "fmin v30.8h, v30.8h, v18.8h\n" - "fmin v31.8h, v31.8h, v18.8h\n" + "fmax v28.8h, v28.8h, v26.8h\n" + "fmax v29.8h, v29.8h, v26.8h\n" + "fmax v30.8h, v30.8h, v26.8h\n" + "fmax v31.8h, v31.8h, v26.8h\n" + "fmin v28.8h, v28.8h, v27.8h\n" + "fmin v29.8h, v29.8h, v27.8h\n" + "fmin v30.8h, v30.8h, v27.8h\n" + "fmin v31.8h, v31.8h, v27.8h\n" "tbz %x[n_channels], #2, 78f\n" "mov x21, x17\n" "mov x20, x28\n" @@ -871,7 +871,6 @@ void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "st1 { v29.h }[0], [x21]\n" "st1 { v31.h }[0], [x20]\n" "80:" // Tile loop: Oddments: Store: Bit 2: End - "81:" // Tile loop: End "ldr x27, [%x[params_struct], %[offsetof_args_tile_j]]\n" "ldr x23, [%x[params_struct], %[offsetof_args_tile_i]]\n" @@ -886,7 +885,7 @@ void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v28", "v29", "v30", "v31", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp index 144d11fb39185778ccbfd87fc832b8f841989144..6ae0b30afd581be54e4e60c4ccbb483d38c5f2aa 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp @@ -88,258 +88,258 @@ void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( __asm__ __volatile__( "ldr x21, [%x[params_struct], %[offsetof_args_outptrs]]\n" - "mov x26, #0x10\n" // cntb _, ALL, #1 - "lsr x25, %x[n_channels], #0x3\n" - "ldr x24, [%x[params_struct], %[offsetof_args_params]]\n" + "mov x25, #0x10\n" // cntb _, ALL, #1 + "lsr x24, %x[n_channels], #0x3\n" + "ldr x23, [%x[params_struct], %[offsetof_args_params]]\n" "add x20, %x[params_struct], %[offsetof_args_min]\n" - "ld1r { v19.8h }, [x20]\n" + "ld1r { v26.8h }, [x20]\n" "add x20, %x[params_struct], %[offsetof_args_max]\n" - "ld1r { v18.8h }, [x20]\n" + "ld1r { v27.8h }, [x20]\n" "add x13, %x[params_struct], %[offsetof_Args_inptrs]\n" "ldp x12, x11, [x21, #0x0]\n" "ldp x10, x9, [x21, #0x10]\n" "mov x28, #0x0\n" - "sub x23, XZR, x26\n" - "cbz x25, 3f\n" - "ldr q17, [x24, #0x0]\n" - "ldr q0, [x24, #0x10]\n" - "cmp x26, x25, LSL #4\n" - "ldr q1, [x24, #0x20]\n" - "ldr q2, [x24, #0x30]\n" - "ldr q3, [x24, #0x40]\n" - "ldr q4, [x24, #0x50]\n" - "ldr q5, [x24, #0x60]\n" - "ldr q6, [x24, #0x70]\n" - "ldr q7, [x24, #0x80]\n" - "ldr q8, [x24, #0x90]\n" - "add x24, x24, #0xa0\n" - "ldp x22, x20, [x13, #0x0]\n" - "ldr q9, [x22, x28]\n" + "sub x22, XZR, x25\n" + "cbz x24, 3f\n" + "ldr q31, [x23, #0x0]\n" + "ldr q0, [x23, #0x10]\n" + "cmp x25, x24, LSL #4\n" + "ldr q1, [x23, #0x20]\n" + "ldr q2, [x23, #0x30]\n" + "ldr q3, [x23, #0x40]\n" + "ldr q4, [x23, #0x50]\n" + "ldr q5, [x23, #0x60]\n" + "ldr q6, [x23, #0x70]\n" + "ldr q7, [x23, #0x80]\n" + "ldr q8, [x23, #0x90]\n" + "add x23, x23, #0xa0\n" + "ldp x21, x20, [x13, #0x0]\n" + "ldr q9, [x21, x28]\n" "ldr q10, [x20, x28]\n" "ldp x21, x20, [x13, #0x10]\n" "ldr q11, [x21, x28]\n" "ldr q12, [x20, x28]\n" - "ldp x22, x21, [x13, #0x20]\n" - "ldr q13, [x22, x28]\n" - "ldr q14, [x21, x28]\n" + "ldp x21, x20, [x13, #0x20]\n" + "ldr q13, [x21, x28]\n" + "ldr q14, [x20, x28]\n" "ldp x21, x20, [x13, #0x30]\n" "ldr q15, [x21, x28]\n" "ldr q16, [x20, x28]\n" "bge 2f\n" "1:" // Channel loop - "mov v28.16b, v17.16b\n fmla v28.8h, v8.8h, v9.8h\n" - "mov v29.16b, v17.16b\n fmla v29.8h, v6.8h, v9.8h\n" - "ldr x22, [x13, #0x40]\n" + "mov v24.16b, v31.16b\n fmla v24.8h, v8.8h, v9.8h\n" + "mov v23.16b, v31.16b\n fmla v23.8h, v6.8h, v9.8h\n" + "ldr x21, [x13, #0x40]\n" "ldr x20, [x13, #0x48]\n" - "fmla v28.8h, v0.8h, v10.8h\n" - "fmla v29.8h, v1.8h, v12.8h\n" - "ldr q12, [x20, x28]\n" - "ldr x21, [x13, #0x50]\n" - "fmla v28.8h, v1.8h, v11.8h\n" - "ldr q11, [x22, x28]\n" - "fmla v29.8h, v2.8h, v13.8h\n" - "ldr q13, [x21, x28]\n" - "fmla v28.8h, v3.8h, v14.8h\n" - "fmla v29.8h, v0.8h, v16.8h\n" + "fmla v24.8h, v0.8h, v10.8h\n" + "fmla v23.8h, v1.8h, v12.8h\n" + "ldr q20, [x20, x28]\n" + "ldr x20, [x13, #0x50]\n" + "fmla v24.8h, v1.8h, v11.8h\n" + "ldr q19, [x21, x28]\n" + "fmla v23.8h, v2.8h, v13.8h\n" + "ldr q18, [x20, x28]\n" + "fmla v24.8h, v3.8h, v14.8h\n" + "fmla v23.8h, v0.8h, v16.8h\n" "ldr x20, [x13, #0x58]\n" - "ldr q14, [x20, x28]\n" - "fmla v28.8h, v4.8h, v15.8h\n" - "fmla v29.8h, v4.8h, v11.8h\n" - "ldr x20, [x13, #0x78]\n" - "ldr x22, [x13, #0x60]\n" - "ldr q15, [x22, x28]\n" - "fmla v28.8h, v2.8h, v16.8h\n" - "fmla v29.8h, v5.8h, v12.8h\n" - "ldr x22, [x13, #0x80]\n" - "ldr q12, [x22, x28]\n" - "mov v30.16b, v17.16b\n fmla v30.8h, v2.8h, v9.8h\n" - "mov v31.16b, v17.16b\n fmla v31.8h, v0.8h, v9.8h\n" - "ldr q17, [x24, #0x0]\n" - "fmla v28.8h, v5.8h, v13.8h\n" - "fmla v29.8h, v3.8h, v13.8h\n" - "ldr q13, [x20, x28]\n" - "ldr x21, [x13, #0x68]\n" - "ldr q11, [x21, x28]\n" - "fmla v30.8h, v3.8h, v14.8h\n" - "fmla v31.8h, v4.8h, v13.8h\n" - "ldr x20, [x13, #0x88]\n" - "ldr q14, [x20, x28]\n" - "fmla v30.8h, v0.8h, v15.8h\n" - "ldr q0, [x24, #0x10]\n" - "fmla v31.8h, v1.8h, v12.8h\n" - "ldr x21, [x13, #0x70]\n" + "ldr q17, [x20, x28]\n" + "fmla v24.8h, v4.8h, v15.8h\n" + "fmla v23.8h, v4.8h, v19.8h\n" + "ldr x21, [x13, #0x78]\n" + "ldr x20, [x13, #0x60]\n" + "ldr q22, [x20, x28]\n" + "fmla v24.8h, v2.8h, v16.8h\n" + "fmla v23.8h, v5.8h, v20.8h\n" + "ldr x20, [x13, #0x80]\n" + "ldr q21, [x20, x28]\n" + "mov v20.16b, v31.16b\n fmla v20.8h, v2.8h, v9.8h\n" + "mov v19.16b, v31.16b\n fmla v19.8h, v0.8h, v9.8h\n" + "ldr q31, [x23, #0x0]\n" + "fmla v24.8h, v5.8h, v18.8h\n" + "fmla v23.8h, v3.8h, v18.8h\n" "ldr q16, [x21, x28]\n" - "fmla v30.8h, v4.8h, v11.8h\n" - "fmla v31.8h, v5.8h, v14.8h\n" - "ldr q4, [x24, #0x50]\n" + "ldr x20, [x13, #0x68]\n" + "ldr q18, [x20, x28]\n" + "fmla v20.8h, v3.8h, v17.8h\n" + "fmla v19.8h, v4.8h, v16.8h\n" + "ldr x20, [x13, #0x88]\n" + "ldr q16, [x20, x28]\n" + "fmla v20.8h, v0.8h, v22.8h\n" + "ldr q0, [x23, #0x10]\n" + "fmla v19.8h, v1.8h, v21.8h\n" + "ldr x20, [x13, #0x70]\n" + "ldr q17, [x20, x28]\n" + "fmla v20.8h, v4.8h, v18.8h\n" + "fmla v19.8h, v5.8h, v16.8h\n" + "ldr q4, [x23, #0x50]\n" "ldr x20, [x13, #0x98]\n" - "fmla v28.8h, v6.8h, v15.8h\n" - "fmla v30.8h, v1.8h, v16.8h\n" - "ldr q11, [x20, x28]\n" - "ldr q1, [x24, #0x20]\n" - "fmla v31.8h, v2.8h, v11.8h\n" - "fmla v28.8h, v7.8h, v16.8h\n" - "ldr q2, [x24, #0x30]\n" - "ldr x21, [x13, #0x90]\n" - "fmla v29.8h, v7.8h, v12.8h\n" - "fmla v29.8h, v8.8h, v11.8h\n" - "ldr q15, [x21, x28]\n" - "ldr x21, [x13, #0xa8]\n" - "fmla v30.8h, v6.8h, v15.8h\n" - "fmax v28.8h, v28.8h, v19.8h\n" - "ldr q16, [x21, x28]\n" - "ldr x22, [x13, #0xa0]\n" - "fmla v31.8h, v3.8h, v16.8h\n" - "fmax v29.8h, v29.8h, v19.8h\n" - "ldr q13, [x22, x28]\n" - "ldr q3, [x24, #0x40]\n" - "fmla v30.8h, v7.8h, v13.8h\n" - "fmla v30.8h, v5.8h, v16.8h\n" - "ldr q5, [x24, #0x60]\n" - "ldr x21, [x13, #0xb0]\n" - "add x23, x23, #0x10\n" - "fmin v28.8h, v28.8h, v18.8h\n" - "ldr q14, [x21, x28]\n" + "fmla v24.8h, v6.8h, v22.8h\n" + "fmla v20.8h, v1.8h, v17.8h\n" + "ldr q16, [x20, x28]\n" + "ldr q1, [x23, #0x20]\n" + "fmla v19.8h, v2.8h, v16.8h\n" + "fmla v24.8h, v7.8h, v17.8h\n" + "ldr q2, [x23, #0x30]\n" + "ldr x20, [x13, #0x90]\n" + "fmla v23.8h, v7.8h, v21.8h\n" + "fmla v23.8h, v8.8h, v16.8h\n" + "ldr q16, [x20, x28]\n" + "ldr x20, [x13, #0xa8]\n" + "fmla v20.8h, v6.8h, v16.8h\n" + "fmax v24.8h, v24.8h, v26.8h\n" + "ldr q17, [x20, x28]\n" + "ldr x20, [x13, #0xa0]\n" + "fmla v19.8h, v3.8h, v17.8h\n" + "fmax v23.8h, v23.8h, v26.8h\n" + "ldr q16, [x20, x28]\n" + "ldr q3, [x23, #0x40]\n" + "fmla v20.8h, v7.8h, v16.8h\n" + "fmla v20.8h, v5.8h, v17.8h\n" + "ldr q5, [x23, #0x60]\n" + "ldr x20, [x13, #0xb0]\n" + "add x22, x22, #0x10\n" + "fmin v24.8h, v24.8h, v27.8h\n" + "ldr q16, [x20, x28]\n" "ldr x20, [x13, #0xb8]\n" - "fmla v31.8h, v7.8h, v14.8h\n" - "fmin v29.8h, v29.8h, v18.8h\n" - "ldr q15, [x20, x28]\n" - "ldr q7, [x24, #0x80]\n" - "fmla v31.8h, v6.8h, v15.8h\n" - "fmla v30.8h, v8.8h, v15.8h\n" - "ldr q6, [x24, #0x70]\n" - "ldr x22, [x13, #0xc0]\n" - "fmax v30.8h, v30.8h, v19.8h\n" - "fmin v30.8h, v30.8h, v18.8h\n" - "ldr q11, [x22, x28]\n" - "fmla v31.8h, v8.8h, v11.8h\n" - "ldr q8, [x24, #0x90]\n" - "fmax v31.8h, v31.8h, v19.8h\n" - "ldp x22, x20, [x13, #0x0]\n" - "ldr q9, [x22, x26]\n" - "fmin v31.8h, v31.8h, v18.8h\n" + "fmla v19.8h, v7.8h, v16.8h\n" + "fmin v23.8h, v23.8h, v27.8h\n" + "ldr q16, [x20, x28]\n" + "ldr q7, [x23, #0x80]\n" + "fmla v19.8h, v6.8h, v16.8h\n" + "fmla v20.8h, v8.8h, v16.8h\n" + "ldr q6, [x23, #0x70]\n" + "ldr x20, [x13, #0xc0]\n" + "fmax v20.8h, v20.8h, v26.8h\n" + "fmin v20.8h, v20.8h, v27.8h\n" + "ldr q16, [x20, x28]\n" + "fmla v19.8h, v8.8h, v16.8h\n" + "ldr q8, [x23, #0x90]\n" + "fmax v19.8h, v19.8h, v26.8h\n" + "ldp x21, x20, [x13, #0x0]\n" + "ldr q9, [x21, x25]\n" + "fmin v19.8h, v19.8h, v27.8h\n" "add x28, x28, #0x10\n" - "ldr q10, [x20, x26]\n" + "ldr q10, [x20, x25]\n" "ldp x21, x20, [x13, #0x10]\n" - "str q28, [x12, x23]\n" - "add x24, x24, #0xa0\n" - "ldr q11, [x21, x26]\n" - "ldr q12, [x20, x26]\n" - "str q29, [x11, x23]\n" - "ldp x22, x21, [x13, #0x20]\n" - "ldr q13, [x22, x26]\n" - "str q30, [x10, x23]\n" - "ldr q14, [x21, x26]\n" + "str q24, [x12, x22]\n" + "add x23, x23, #0xa0\n" + "ldr q11, [x21, x25]\n" + "ldr q12, [x20, x25]\n" + "str q23, [x11, x22]\n" + "ldp x21, x20, [x13, #0x20]\n" + "ldr q13, [x21, x25]\n" + "str q20, [x10, x22]\n" + "ldr q14, [x20, x25]\n" "ldp x21, x20, [x13, #0x30]\n" - "str q31, [x9, x23]\n" - "ldr q15, [x21, x26]\n" - "ldr q16, [x20, x26]\n" - "add x26, x26, #0x10\n" - "cmp x26, x25, LSL #4\n" + "str q19, [x9, x22]\n" + "ldr q15, [x21, x25]\n" + "ldr q16, [x20, x25]\n" + "add x25, x25, #0x10\n" + "cmp x25, x24, LSL #4\n" "blt 1b\n" "2:" // Channel tail - "mov v28.16b, v17.16b\n fmla v28.8h, v8.8h, v9.8h\n" - "mov v29.16b, v17.16b\n fmla v29.8h, v6.8h, v9.8h\n" - "ldr x22, [x13, #0x40]\n" + "mov v25.16b, v31.16b\n fmla v25.8h, v8.8h, v9.8h\n" + "mov v24.16b, v31.16b\n fmla v24.8h, v6.8h, v9.8h\n" + "ldr x21, [x13, #0x40]\n" "ldr x20, [x13, #0x48]\n" - "fmla v28.8h, v0.8h, v10.8h\n" - "fmla v29.8h, v1.8h, v12.8h\n" - "ldr q12, [x20, x28]\n" - "ldr x21, [x13, #0x50]\n" - "fmla v28.8h, v1.8h, v11.8h\n" - "ldr q11, [x22, x28]\n" - "fmla v29.8h, v2.8h, v13.8h\n" - "ldr q13, [x21, x28]\n" - "fmla v28.8h, v3.8h, v14.8h\n" - "fmla v29.8h, v0.8h, v16.8h\n" + "fmla v25.8h, v0.8h, v10.8h\n" + "fmla v24.8h, v1.8h, v12.8h\n" + "ldr q20, [x20, x28]\n" + "ldr x20, [x13, #0x50]\n" + "fmla v25.8h, v1.8h, v11.8h\n" + "ldr q18, [x21, x28]\n" + "fmla v24.8h, v2.8h, v13.8h\n" + "ldr q19, [x20, x28]\n" + "fmla v25.8h, v3.8h, v14.8h\n" + "fmla v24.8h, v0.8h, v16.8h\n" "ldr x20, [x13, #0x58]\n" - "ldr q14, [x20, x28]\n" - "fmla v28.8h, v4.8h, v15.8h\n" - "fmla v29.8h, v4.8h, v11.8h\n" - "ldr x20, [x13, #0x78]\n" - "ldr x22, [x13, #0x60]\n" - "ldr q15, [x22, x28]\n" - "fmla v28.8h, v2.8h, v16.8h\n" - "fmla v29.8h, v5.8h, v12.8h\n" - "ldr x22, [x13, #0x80]\n" - "ldr q12, [x22, x28]\n" - "mov v30.16b, v17.16b\n fmla v30.8h, v2.8h, v9.8h\n" - "mov v31.16b, v17.16b\n fmla v31.8h, v0.8h, v9.8h\n" - "ldr x21, [x13, #0x68]\n" - "ldr q11, [x21, x28]\n" - "fmla v28.8h, v5.8h, v13.8h\n" - "fmla v29.8h, v3.8h, v13.8h\n" - "ldr q13, [x20, x28]\n" - "fmla v30.8h, v3.8h, v14.8h\n" - "fmla v31.8h, v4.8h, v13.8h\n" - "ldr x20, [x13, #0x88]\n" - "ldr q14, [x20, x28]\n" - "fmla v30.8h, v0.8h, v15.8h\n" - "fmla v31.8h, v1.8h, v12.8h\n" - "ldr x21, [x13, #0x70]\n" + "ldr q17, [x20, x28]\n" + "fmla v25.8h, v4.8h, v15.8h\n" + "fmla v24.8h, v4.8h, v18.8h\n" + "ldr x21, [x13, #0x78]\n" + "ldr x20, [x13, #0x60]\n" + "ldr q23, [x20, x28]\n" + "fmla v25.8h, v2.8h, v16.8h\n" + "fmla v24.8h, v5.8h, v20.8h\n" + "ldr x20, [x13, #0x80]\n" + "ldr q22, [x20, x28]\n" + "mov v21.16b, v31.16b\n fmla v21.8h, v2.8h, v9.8h\n" + "mov v20.16b, v31.16b\n fmla v20.8h, v0.8h, v9.8h\n" + "ldr x20, [x13, #0x68]\n" + "ldr q18, [x20, x28]\n" + "fmla v25.8h, v5.8h, v19.8h\n" + "fmla v24.8h, v3.8h, v19.8h\n" "ldr q16, [x21, x28]\n" + "fmla v21.8h, v3.8h, v17.8h\n" + "fmla v20.8h, v4.8h, v16.8h\n" + "ldr x20, [x13, #0x88]\n" + "ldr q16, [x20, x28]\n" + "fmla v21.8h, v0.8h, v23.8h\n" + "fmla v20.8h, v1.8h, v22.8h\n" + "ldr x20, [x13, #0x70]\n" + "ldr q17, [x20, x28]\n" "ldr x20, [x13, #0x98]\n" - "fmla v30.8h, v4.8h, v11.8h\n" - "ldr q11, [x20, x28]\n" - "fmla v31.8h, v5.8h, v14.8h\n" - "fmla v28.8h, v6.8h, v15.8h\n" - "ldr x21, [x13, #0x90]\n" - "ldr q15, [x21, x28]\n" - "fmla v30.8h, v1.8h, v16.8h\n" - "ldr x21, [x13, #0xa8]\n" - "fmla v31.8h, v2.8h, v11.8h\n" - "fmla v28.8h, v7.8h, v16.8h\n" - "ldr q16, [x21, x28]\n" - "ldr x22, [x13, #0xa0]\n" - "ldr q13, [x22, x28]\n" - "fmla v30.8h, v6.8h, v15.8h\n" - "fmla v31.8h, v3.8h, v16.8h\n" - "ldr x21, [x13, #0xb0]\n" - "ldr q14, [x21, x28]\n" - "fmla v30.8h, v7.8h, v13.8h\n" - "fmla v31.8h, v7.8h, v14.8h\n" + "fmla v21.8h, v4.8h, v18.8h\n" + "ldr q19, [x20, x28]\n" + "fmla v20.8h, v5.8h, v16.8h\n" + "fmla v25.8h, v6.8h, v23.8h\n" + "ldr x20, [x13, #0x90]\n" + "ldr q16, [x20, x28]\n" + "fmla v21.8h, v1.8h, v17.8h\n" + "ldr x20, [x13, #0xa8]\n" + "fmla v20.8h, v2.8h, v19.8h\n" + "fmla v25.8h, v7.8h, v17.8h\n" + "ldr q18, [x20, x28]\n" + "ldr x20, [x13, #0xa0]\n" + "ldr q17, [x20, x28]\n" + "fmla v21.8h, v6.8h, v16.8h\n" + "fmla v20.8h, v3.8h, v18.8h\n" + "ldr x20, [x13, #0xb0]\n" + "ldr q16, [x20, x28]\n" + "fmla v21.8h, v7.8h, v17.8h\n" + "fmla v20.8h, v7.8h, v16.8h\n" "ldr x20, [x13, #0xb8]\n" - "ldr q15, [x20, x28]\n" - "fmla v29.8h, v7.8h, v12.8h\n" - "fmla v30.8h, v5.8h, v16.8h\n" - "ldr x22, [x13, #0xc0]\n" - "fmla v31.8h, v6.8h, v15.8h\n" - "fmla v29.8h, v8.8h, v11.8h\n" - "ldr q11, [x22, x28]\n" - "fmla v30.8h, v8.8h, v15.8h\n" - "fmla v31.8h, v8.8h, v11.8h\n" - "fmax v28.8h, v28.8h, v19.8h\n" - "add x23, x23, #0x10\n" - "fmax v29.8h, v29.8h, v19.8h\n" - "fmax v30.8h, v30.8h, v19.8h\n" + "ldr q17, [x20, x28]\n" + "fmla v24.8h, v7.8h, v22.8h\n" + "fmla v21.8h, v5.8h, v18.8h\n" + "ldr x20, [x13, #0xc0]\n" + "fmla v20.8h, v6.8h, v17.8h\n" + "fmla v24.8h, v8.8h, v19.8h\n" + "ldr q16, [x20, x28]\n" + "fmla v21.8h, v8.8h, v17.8h\n" + "fmla v20.8h, v8.8h, v16.8h\n" + "fmax v25.8h, v25.8h, v26.8h\n" + "add x22, x22, #0x10\n" + "fmax v24.8h, v24.8h, v26.8h\n" + "fmax v21.8h, v21.8h, v26.8h\n" "add x28, x28, #0x10\n" - "fmax v31.8h, v31.8h, v19.8h\n" - "fmin v28.8h, v28.8h, v18.8h\n" - "str q28, [x12, x23]\n" - "fmin v29.8h, v29.8h, v18.8h\n" - "fmin v30.8h, v30.8h, v18.8h\n" - "str q29, [x11, x23]\n" - "fmin v31.8h, v31.8h, v18.8h\n" - "str q30, [x10, x23]\n" - "str q31, [x9, x23]\n" + "fmax v20.8h, v20.8h, v26.8h\n" + "fmin v25.8h, v25.8h, v27.8h\n" + "str q25, [x12, x22]\n" + "fmin v24.8h, v24.8h, v27.8h\n" + "fmin v21.8h, v21.8h, v27.8h\n" + "str q24, [x11, x22]\n" + "fmin v20.8h, v20.8h, v27.8h\n" + "str q21, [x10, x22]\n" + "str q20, [x9, x22]\n" "3:" // Oddments "tst %x[n_channels], #0x7\n" "beq 80f\n" - "ldr q17, [x24, #0x0]\n" - "ldr q0, [x24, #0x10]\n" - "mov x23, x28\n" - "add x12, x12, x23\n" - "ldr q1, [x24, #0x20]\n" - "ldr q2, [x24, #0x30]\n" - "add x11, x11, x23\n" - "add x10, x10, x23\n" - "ldr q3, [x24, #0x40]\n" - "ldr q4, [x24, #0x50]\n" - "add x9, x9, x23\n" - "ldr q5, [x24, #0x60]\n" - "ldr q6, [x24, #0x70]\n" - "ldr q7, [x24, #0x80]\n" - "ldr q8, [x24, #0x90]\n" + "ldr q31, [x23, #0x0]\n" + "ldr q0, [x23, #0x10]\n" + "mov x20, x28\n" + "add x12, x12, x20\n" + "ldr q1, [x23, #0x20]\n" + "ldr q2, [x23, #0x30]\n" + "add x11, x11, x20\n" + "add x10, x10, x20\n" + "ldr q3, [x23, #0x40]\n" + "ldr q4, [x23, #0x50]\n" + "add x9, x9, x20\n" + "ldr q5, [x23, #0x60]\n" + "ldr q6, [x23, #0x70]\n" + "ldr q7, [x23, #0x80]\n" + "ldr q8, [x23, #0x90]\n" "ldr x27, [x13, #0x0]\n" "ldr x26, [x13, #0x8]\n" "add x27, x27, x28\n" @@ -425,18 +425,18 @@ void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( "ld1 { v15.h }[0], [x21], #0x2\n" "ld1 { v16.h }[0], [x20], #0x2\n" "7:" // Oddments: Load inputs (2, 2), (0, 0), (0, 1), (0, 3), (0, 4), (1, 0), (1, 1), (0, 2): Bit 2: End - "mov v28.16b, v17.16b\n fmla v28.8h, v8.8h, v9.8h\n" + "mov v28.16b, v31.16b\n fmla v28.8h, v8.8h, v9.8h\n" "fmla v28.8h, v0.8h, v10.8h\n" "ldr x20, [x13, #0x40]\n" "add x20, x20, x28\n" - "mov v29.16b, v17.16b\n fmla v29.8h, v6.8h, v9.8h\n" + "mov v29.16b, v31.16b\n fmla v29.8h, v6.8h, v9.8h\n" "fmla v28.8h, v1.8h, v11.8h\n" "fmla v29.8h, v1.8h, v12.8h\n" "fmla v28.8h, v3.8h, v14.8h\n" "fmla v29.8h, v2.8h, v13.8h\n" "fmla v28.8h, v4.8h, v15.8h\n" - "mov v30.16b, v17.16b\n fmla v30.8h, v2.8h, v9.8h\n" - "mov v31.16b, v17.16b\n fmla v31.8h, v0.8h, v9.8h\n" + "mov v30.16b, v31.16b\n fmla v30.8h, v2.8h, v9.8h\n" + "fmla v31.8h, v0.8h, v9.8h\n" "fmla v28.8h, v2.8h, v16.8h\n" "fmla v29.8h, v0.8h, v16.8h\n" "tbz %x[n_channels], #2, 9f\n" @@ -835,14 +835,14 @@ void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( "ld1 { v11.h }[0], [x20], #0x2\n" "75:" // Oddments: Load input (4, 4): Bit 2: End "fmla v31.8h, v8.8h, v11.8h\n" - "fmax v28.8h, v28.8h, v19.8h\n" - "fmax v29.8h, v29.8h, v19.8h\n" - "fmax v30.8h, v30.8h, v19.8h\n" - "fmax v31.8h, v31.8h, v19.8h\n" - "fmin v28.8h, v28.8h, v18.8h\n" - "fmin v29.8h, v29.8h, v18.8h\n" - "fmin v30.8h, v30.8h, v18.8h\n" - "fmin v31.8h, v31.8h, v18.8h\n" + "fmax v28.8h, v28.8h, v26.8h\n" + "fmax v29.8h, v29.8h, v26.8h\n" + "fmax v30.8h, v30.8h, v26.8h\n" + "fmax v31.8h, v31.8h, v26.8h\n" + "fmin v28.8h, v28.8h, v27.8h\n" + "fmin v29.8h, v29.8h, v27.8h\n" + "fmin v30.8h, v30.8h, v27.8h\n" + "fmin v31.8h, v31.8h, v27.8h\n" "tbz %x[n_channels], #2, 77f\n" "st1 { v28.d }[0], [x12], #0x8\n" "st1 { v29.d }[0], [x11], #0x8\n" @@ -887,7 +887,7 @@ void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( "80:" // End : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp index 81a608e349d1b85ed3bbc6146e9574284f991a32..1d1d491c280d99ab8ddf64719bc18a1fae943ca0 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include @@ -33,8 +33,8 @@ namespace arm_conv { namespace depthwise { -void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); -void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); +void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const input_ptrs, __fp16 *const *const outptrs, const void *params, unsigned int n_channels, const __fp16 activation_min, const __fp16 activation_max); +void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int n_tile_rows, const unsigned int n_tile_cols, const __fp16 *inptr, int64_t ld_input_row, int64_t ld_input_col, __fp16 *outptr, int64_t ld_output_row, int64_t ld_output_col, const void *params, unsigned int n_channels, const __fp16 activation_min, const __fp16 activation_max); class a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirstStrategy<__fp16, __fp16, __fp16, __fp16> { @@ -57,7 +57,7 @@ class a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirst constexpr static unsigned int output_cols = 2; a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst(const CPUInfo *) - : DepthwiseDepthfirstStrategy<__fp16, __fp16, __fp16, __fp16>(2, 5, 1) {} + : Parent(output_rows, output_cols, kernel_rows, kernel_cols, stride_rows, stride_cols) {} arm_gemm::VLType get_vl_type(void) const override { return vl_type; } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp index 8807f5d3061fc7ef77aa9f35df2b3d584e53003c..cecaf797042391f41898beb501db40554396ff56 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -120,9 +120,9 @@ void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "add x13, x15, x2\n" "add x5, x5, x20, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16) "add x20, %x[params_struct], %[offsetof_args_min]\n" - "ld1r { v18.8h }, [x20]\n" + "ld1r { v27.8h }, [x20]\n" "add x20, %x[params_struct], %[offsetof_args_max]\n" - "ld1r { v17.8h }, [x20]\n" + "ld1r { v15.8h }, [x20]\n" "add x12, x14, x24, LSL #1\n" "add x11, x13, x2\n" "add x10, x5, x21, LSL #1\n" @@ -130,7 +130,7 @@ void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "mov x21, #0x0\n" "sub x20, XZR, x23\n" "cbz x22, 4f\n" - "ldr q16, [x8, #0x0]\n" + "ldr q25, [x8, #0x0]\n" "ldr q0, [x8, #0x10]\n" "cmp x23, x22, LSL #4\n" "ldr q1, [x8, #0x20]\n" @@ -150,366 +150,366 @@ void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "ld1 { v14.8h }, [x17]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "mov v28.16b, v16.16b\n fmla v28.8h, v0.8h, v5.8h\n" - "ldr q5, [x7, x15]\n" - "mov v29.16b, v16.16b\n fmla v29.8h, v0.8h, v6.8h\n" + "mov v30.16b, v25.16b\n fmla v30.8h, v0.8h, v5.8h\n" + "ldr q23, [x7, x15]\n" + "mov v31.16b, v25.16b\n fmla v31.8h, v0.8h, v6.8h\n" "add x23, x23, #0x10\n" - "mov v30.16b, v16.16b\n fmla v30.8h, v0.8h, v7.8h\n" - "mov v31.16b, v16.16b\n fmla v31.8h, v0.8h, v8.8h\n" - "ldr q0, [x8, #0x0]\n" - "ldr q16, [x8, #0x140]\n" - "fmla v28.8h, v1.8h, v6.8h\n" - "ldr q6, [x7, x13]\n" - "fmla v29.8h, v1.8h, v9.8h\n" + "mov v29.16b, v25.16b\n fmla v29.8h, v0.8h, v7.8h\n" + "mov v28.16b, v25.16b\n fmla v28.8h, v0.8h, v8.8h\n" + "ldr q19, [x8, #0x0]\n" + "ldr q25, [x8, #0x140]\n" + "fmla v30.8h, v1.8h, v6.8h\n" + "ldr q21, [x7, x13]\n" + "fmla v31.8h, v1.8h, v9.8h\n" "add x7, x7, #0x10\n" - "fmla v30.8h, v1.8h, v8.8h\n" - "fmla v31.8h, v1.8h, v13.8h\n" + "fmla v29.8h, v1.8h, v8.8h\n" + "fmla v28.8h, v1.8h, v13.8h\n" "ldr q1, [x8, #0x10]\n" "cmp x23, x22, LSL #4\n" - "fmla v28.8h, v2.8h, v9.8h\n" - "ldr q9, [x4, x11]\n" - "fmla v29.8h, v2.8h, v11.8h\n" + "fmla v30.8h, v2.8h, v9.8h\n" + "ldr q18, [x4, x11]\n" + "fmla v31.8h, v2.8h, v11.8h\n" "add x4, x4, #0x10\n" - "fmla v30.8h, v2.8h, v13.8h\n" - "fmla v31.8h, v2.8h, v5.8h\n" - "ldr q2, [x8, #0x20]\n" + "fmla v29.8h, v2.8h, v13.8h\n" + "fmla v28.8h, v2.8h, v23.8h\n" + "ldr q17, [x8, #0x20]\n" "add x20, x20, #0x10\n" - "fmla v28.8h, v3.8h, v11.8h\n" - "ldr q11, [x17, x2]\n" - "fmla v29.8h, v3.8h, v12.8h\n" + "fmla v30.8h, v3.8h, v11.8h\n" + "ldr q6, [x17, x2]\n" + "fmla v31.8h, v3.8h, v12.8h\n" "add x21, x21, #0x10\n" - "fmla v30.8h, v3.8h, v5.8h\n" - "fmla v31.8h, v3.8h, v6.8h\n" - "ldr q3, [x8, #0x30]\n" - "fmla v28.8h, v4.8h, v12.8h\n" - "ldr q12, [x17, x6]\n" - "fmla v29.8h, v4.8h, v9.8h\n" - "ldr q9, [x17, x15]\n" - "fmla v30.8h, v4.8h, v6.8h\n" - "fmla v31.8h, v4.8h, v10.8h\n" - "ldr q4, [x8, #0x40]\n" - "fmla v28.8h, v0.8h, v7.8h\n" + "fmla v29.8h, v3.8h, v23.8h\n" + "fmla v28.8h, v3.8h, v21.8h\n" + "ldr q16, [x8, #0x30]\n" + "fmla v30.8h, v4.8h, v12.8h\n" + "ldr q2, [x17, x6]\n" + "fmla v31.8h, v4.8h, v18.8h\n" + "ldr q0, [x17, x15]\n" + "fmla v29.8h, v4.8h, v21.8h\n" + "fmla v28.8h, v4.8h, v10.8h\n" + "ldr q20, [x8, #0x40]\n" + "fmla v30.8h, v19.8h, v7.8h\n" "ld1 { v7.8h }, [x7]\n" - "fmla v29.8h, v0.8h, v8.8h\n" - "fmla v30.8h, v0.8h, v14.8h\n" - "fmla v31.8h, v0.8h, v11.8h\n" - "ldr q0, [x8, #0x50]\n" - "fmla v28.8h, v1.8h, v8.8h\n" - "ldr q8, [x17, x11]\n" - "fmla v29.8h, v1.8h, v13.8h\n" - "fmla v30.8h, v1.8h, v11.8h\n" - "fmla v31.8h, v1.8h, v12.8h\n" - "ldr q1, [x8, #0x60]\n" - "fmla v28.8h, v2.8h, v13.8h\n" - "ldr q13, [x17, x13]\n" - "fmla v29.8h, v2.8h, v5.8h\n" + "fmla v31.8h, v19.8h, v8.8h\n" + "fmla v29.8h, v19.8h, v14.8h\n" + "fmla v28.8h, v19.8h, v6.8h\n" + "ldr q19, [x8, #0x50]\n" + "fmla v30.8h, v1.8h, v8.8h\n" + "ldr q26, [x17, x11]\n" + "fmla v31.8h, v1.8h, v13.8h\n" + "fmla v29.8h, v1.8h, v6.8h\n" + "fmla v28.8h, v1.8h, v2.8h\n" + "ldr q18, [x8, #0x60]\n" + "fmla v30.8h, v17.8h, v13.8h\n" + "ldr q1, [x17, x13]\n" + "fmla v31.8h, v17.8h, v23.8h\n" "add x17, x17, #0x10\n" - "fmla v30.8h, v2.8h, v12.8h\n" - "fmla v31.8h, v2.8h, v9.8h\n" - "ldr q2, [x8, #0x70]\n" - "fmla v28.8h, v3.8h, v5.8h\n" - "ld1 { v5.8h }, [x16]\n" - "fmla v29.8h, v3.8h, v6.8h\n" - "fmla v30.8h, v3.8h, v9.8h\n" - "fmla v31.8h, v3.8h, v13.8h\n" - "ldr q3, [x8, #0x80]\n" - "fmla v28.8h, v4.8h, v6.8h\n" - "ldr q6, [x16, x2]\n" - "fmla v29.8h, v4.8h, v10.8h\n" - "ldr q10, [x16, x6]\n" - "fmla v30.8h, v4.8h, v13.8h\n" - "fmla v31.8h, v4.8h, v8.8h\n" - "ldr q4, [x8, #0x90]\n" - "fmla v28.8h, v0.8h, v14.8h\n" - "ldr q14, [x16, x11]\n" - "fmla v29.8h, v0.8h, v11.8h\n" - "fmla v30.8h, v0.8h, v5.8h\n" - "fmla v31.8h, v0.8h, v6.8h\n" - "ldr q0, [x8, #0xa0]\n" - "fmla v28.8h, v1.8h, v11.8h\n" - "ldr q11, [x16, x15]\n" - "fmla v29.8h, v1.8h, v12.8h\n" - "fmla v30.8h, v1.8h, v6.8h\n" - "fmla v31.8h, v1.8h, v10.8h\n" - "ldr q1, [x8, #0xb0]\n" - "fmla v28.8h, v2.8h, v12.8h\n" - "ldr q12, [x16, x13]\n" - "fmla v29.8h, v2.8h, v9.8h\n" + "fmla v29.8h, v17.8h, v2.8h\n" + "fmla v28.8h, v17.8h, v0.8h\n" + "ldr q17, [x8, #0x70]\n" + "fmla v30.8h, v16.8h, v23.8h\n" + "ld1 { v24.8h }, [x16]\n" + "fmla v31.8h, v16.8h, v21.8h\n" + "fmla v29.8h, v16.8h, v0.8h\n" + "fmla v28.8h, v16.8h, v1.8h\n" + "ldr q16, [x8, #0x80]\n" + "fmla v30.8h, v20.8h, v21.8h\n" + "ldr q23, [x16, x2]\n" + "fmla v31.8h, v20.8h, v10.8h\n" + "ldr q22, [x16, x6]\n" + "fmla v29.8h, v20.8h, v1.8h\n" + "fmla v28.8h, v20.8h, v26.8h\n" + "ldr q21, [x8, #0x90]\n" + "fmla v30.8h, v19.8h, v14.8h\n" + "ldr q5, [x16, x11]\n" + "fmla v31.8h, v19.8h, v6.8h\n" + "fmla v29.8h, v19.8h, v24.8h\n" + "fmla v28.8h, v19.8h, v23.8h\n" + "ldr q11, [x8, #0xa0]\n" + "fmla v30.8h, v18.8h, v6.8h\n" + "ldr q20, [x16, x15]\n" + "fmla v31.8h, v18.8h, v2.8h\n" + "fmla v29.8h, v18.8h, v23.8h\n" + "fmla v28.8h, v18.8h, v22.8h\n" + "ldr q18, [x8, #0xb0]\n" + "fmla v30.8h, v17.8h, v2.8h\n" + "ldr q19, [x16, x13]\n" + "fmla v31.8h, v17.8h, v0.8h\n" "add x16, x16, #0x10\n" - "fmla v30.8h, v2.8h, v10.8h\n" - "fmla v31.8h, v2.8h, v11.8h\n" - "ldr q2, [x8, #0xc0]\n" - "fmla v28.8h, v3.8h, v9.8h\n" - "ld1 { v9.8h }, [x14]\n" - "fmla v29.8h, v3.8h, v13.8h\n" - "fmla v30.8h, v3.8h, v11.8h\n" - "fmla v31.8h, v3.8h, v12.8h\n" - "ldr q3, [x8, #0xd0]\n" - "fmla v28.8h, v4.8h, v13.8h\n" - "ldr q13, [x14, x2]\n" - "fmla v29.8h, v4.8h, v8.8h\n" - "ldr q8, [x14, x13]\n" - "fmla v30.8h, v4.8h, v12.8h\n" - "fmla v31.8h, v4.8h, v14.8h\n" - "ldr q4, [x8, #0xe0]\n" - "fmla v28.8h, v0.8h, v5.8h\n" - "ldr q5, [x14, x6]\n" - "fmla v29.8h, v0.8h, v6.8h\n" - "fmla v30.8h, v0.8h, v9.8h\n" - "fmla v31.8h, v0.8h, v13.8h\n" - "ldr q0, [x8, #0xf0]\n" - "fmla v28.8h, v1.8h, v6.8h\n" - "ldr q6, [x14, x15]\n" - "fmla v29.8h, v1.8h, v10.8h\n" - "fmla v30.8h, v1.8h, v13.8h\n" - "fmla v31.8h, v1.8h, v5.8h\n" - "ldr q1, [x8, #0x100]\n" - "fmla v28.8h, v2.8h, v10.8h\n" - "ldr q10, [x14, x11]\n" - "fmla v29.8h, v2.8h, v11.8h\n" + "fmla v29.8h, v17.8h, v22.8h\n" + "fmla v28.8h, v17.8h, v20.8h\n" + "ldr q17, [x8, #0xc0]\n" + "fmla v30.8h, v16.8h, v0.8h\n" + "ld1 { v0.8h }, [x14]\n" + "fmla v31.8h, v16.8h, v1.8h\n" + "fmla v29.8h, v16.8h, v20.8h\n" + "fmla v28.8h, v16.8h, v19.8h\n" + "ldr q16, [x8, #0xd0]\n" + "fmla v30.8h, v21.8h, v1.8h\n" + "ldr q4, [x14, x2]\n" + "fmla v31.8h, v21.8h, v26.8h\n" + "ldr q12, [x14, x13]\n" + "fmla v29.8h, v21.8h, v19.8h\n" + "fmla v28.8h, v21.8h, v5.8h\n" + "ldr q13, [x8, #0xe0]\n" + "fmla v30.8h, v11.8h, v24.8h\n" + "ldr q6, [x14, x6]\n" + "fmla v31.8h, v11.8h, v23.8h\n" + "fmla v29.8h, v11.8h, v0.8h\n" + "fmla v28.8h, v11.8h, v4.8h\n" + "ldr q24, [x8, #0xf0]\n" + "fmla v30.8h, v18.8h, v23.8h\n" + "ldr q26, [x14, x15]\n" + "fmla v31.8h, v18.8h, v22.8h\n" + "fmla v29.8h, v18.8h, v4.8h\n" + "fmla v28.8h, v18.8h, v6.8h\n" + "ldr q23, [x8, #0x100]\n" + "fmla v30.8h, v17.8h, v22.8h\n" + "ldr q22, [x14, x11]\n" + "fmla v31.8h, v17.8h, v20.8h\n" "add x14, x14, #0x10\n" - "fmla v30.8h, v2.8h, v5.8h\n" - "fmla v31.8h, v2.8h, v6.8h\n" - "ldr q2, [x8, #0x110]\n" - "fmla v28.8h, v3.8h, v11.8h\n" - "ld1 { v11.8h }, [x12]\n" - "fmla v29.8h, v3.8h, v12.8h\n" - "fmla v30.8h, v3.8h, v6.8h\n" - "fmla v31.8h, v3.8h, v8.8h\n" - "ldr q3, [x8, #0x120]\n" - "fmla v28.8h, v4.8h, v12.8h\n" - "ldr q12, [x12, x2]\n" - "fmla v29.8h, v4.8h, v14.8h\n" + "fmla v29.8h, v17.8h, v6.8h\n" + "fmla v28.8h, v17.8h, v26.8h\n" + "ldr q21, [x8, #0x110]\n" + "fmla v30.8h, v16.8h, v20.8h\n" + "ld1 { v18.8h }, [x12]\n" + "fmla v31.8h, v16.8h, v19.8h\n" + "fmla v29.8h, v16.8h, v26.8h\n" + "fmla v28.8h, v16.8h, v12.8h\n" + "ldr q20, [x8, #0x120]\n" + "fmla v30.8h, v13.8h, v19.8h\n" + "ldr q17, [x12, x2]\n" + "fmla v31.8h, v13.8h, v5.8h\n" "ld1 { v14.8h }, [x17]\n" - "fmla v30.8h, v4.8h, v8.8h\n" - "fmla v31.8h, v4.8h, v10.8h\n" - "ldr q4, [x8, #0x130]\n" - "fmla v28.8h, v0.8h, v9.8h\n" - "ldr q9, [x12, x6]\n" - "fmla v29.8h, v0.8h, v13.8h\n" - "fmla v30.8h, v0.8h, v11.8h\n" - "ldr q11, [x12, x15]\n" - "fmla v31.8h, v0.8h, v12.8h\n" + "fmla v29.8h, v13.8h, v12.8h\n" + "fmla v28.8h, v13.8h, v22.8h\n" + "ldr q19, [x8, #0x130]\n" + "fmla v30.8h, v24.8h, v0.8h\n" + "ldr q16, [x12, x6]\n" + "fmla v31.8h, v24.8h, v4.8h\n" + "fmla v29.8h, v24.8h, v18.8h\n" + "ldr q18, [x12, x15]\n" + "fmla v28.8h, v24.8h, v17.8h\n" "ldr q0, [x8, #0x150]\n" - "fmla v28.8h, v1.8h, v13.8h\n" + "fmla v30.8h, v23.8h, v4.8h\n" "ldr q13, [x7, x6]\n" - "fmla v29.8h, v1.8h, v5.8h\n" - "fmla v30.8h, v1.8h, v12.8h\n" - "ldr q12, [x12, x13]\n" - "fmla v31.8h, v1.8h, v9.8h\n" + "fmla v31.8h, v23.8h, v6.8h\n" + "fmla v29.8h, v23.8h, v17.8h\n" + "ldr q17, [x12, x13]\n" + "fmla v28.8h, v23.8h, v16.8h\n" "ldr q1, [x8, #0x160]\n" - "fmla v28.8h, v2.8h, v5.8h\n" + "fmla v30.8h, v21.8h, v6.8h\n" "ld1 { v5.8h }, [x4]\n" - "fmla v29.8h, v2.8h, v6.8h\n" - "fmla v30.8h, v2.8h, v9.8h\n" - "ldr q9, [x12, x11]\n" - "fmla v31.8h, v2.8h, v11.8h\n" + "fmla v31.8h, v21.8h, v26.8h\n" + "fmla v29.8h, v21.8h, v16.8h\n" + "ldr q16, [x12, x11]\n" + "fmla v28.8h, v21.8h, v18.8h\n" "ldr q2, [x8, #0x170]\n" - "fmla v28.8h, v3.8h, v6.8h\n" + "fmla v30.8h, v20.8h, v26.8h\n" "ldr q6, [x4, x2]\n" - "fmla v29.8h, v3.8h, v8.8h\n" + "fmla v31.8h, v20.8h, v12.8h\n" "add x12, x12, #0x10\n" - "fmla v30.8h, v3.8h, v11.8h\n" + "fmla v29.8h, v20.8h, v18.8h\n" "ldr q11, [x4, x15]\n" - "fmla v31.8h, v3.8h, v12.8h\n" + "fmla v28.8h, v20.8h, v17.8h\n" "ldr q3, [x8, #0x180]\n" - "fmla v28.8h, v4.8h, v8.8h\n" + "fmla v30.8h, v19.8h, v12.8h\n" "ldr q8, [x7, x2]\n" - "fmla v29.8h, v4.8h, v10.8h\n" + "fmla v31.8h, v19.8h, v22.8h\n" "ldr q10, [x7, x11]\n" - "fmla v30.8h, v4.8h, v12.8h\n" + "fmla v29.8h, v19.8h, v17.8h\n" "ldr q12, [x4, x13]\n" - "fmla v31.8h, v4.8h, v9.8h\n" + "fmla v28.8h, v19.8h, v16.8h\n" "ldr q9, [x4, x6]\n" "ldr q4, [x8, #0x190]\n" - "fmax v28.8h, v28.8h, v18.8h\n" - "fmax v29.8h, v29.8h, v18.8h\n" + "fmax v30.8h, v30.8h, v27.8h\n" + "fmax v31.8h, v31.8h, v27.8h\n" "add x8, x8, #0x1a0\n" - "fmax v30.8h, v30.8h, v18.8h\n" - "fmax v31.8h, v31.8h, v18.8h\n" - "fmin v28.8h, v28.8h, v17.8h\n" - "fmin v29.8h, v29.8h, v17.8h\n" - "st1 { v28.8h }, [x5]\n" - "fmin v30.8h, v30.8h, v17.8h\n" - "fmin v31.8h, v31.8h, v17.8h\n" - "str q29, [x5, x3]\n" + "fmax v29.8h, v29.8h, v27.8h\n" + "fmax v28.8h, v28.8h, v27.8h\n" + "fmin v30.8h, v30.8h, v15.8h\n" + "fmin v31.8h, v31.8h, v15.8h\n" + "st1 { v30.8h }, [x5]\n" + "fmin v29.8h, v29.8h, v15.8h\n" + "fmin v28.8h, v28.8h, v15.8h\n" + "str q31, [x5, x3]\n" "add x5, x5, #0x10\n" - "st1 { v30.8h }, [x10]\n" - "str q31, [x10, x3]\n" + "st1 { v29.8h }, [x10]\n" + "str q28, [x10, x3]\n" "add x10, x10, #0x10\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "mov v28.16b, v16.16b\n fmla v28.8h, v0.8h, v5.8h\n" - "ldr q5, [x7, x15]\n" - "mov v29.16b, v16.16b\n fmla v29.8h, v0.8h, v6.8h\n" - "mov v30.16b, v16.16b\n fmla v30.8h, v0.8h, v7.8h\n" - "mov v31.16b, v16.16b\n fmla v31.8h, v0.8h, v8.8h\n" - "ldr q0, [x8, #0x0]\n" - "fmla v28.8h, v1.8h, v6.8h\n" - "ldr q6, [x7, x13]\n" - "fmla v29.8h, v1.8h, v9.8h\n" + "mov v31.16b, v25.16b\n fmla v31.8h, v0.8h, v5.8h\n" + "ldr q22, [x7, x15]\n" + "mov v5.16b, v25.16b\n fmla v5.8h, v0.8h, v6.8h\n" + "mov v30.16b, v25.16b\n fmla v30.8h, v0.8h, v7.8h\n" + "mov v29.16b, v25.16b\n fmla v29.8h, v0.8h, v8.8h\n" + "ldr q19, [x8, #0x0]\n" + "fmla v31.8h, v1.8h, v6.8h\n" + "ldr q21, [x7, x13]\n" + "fmla v5.8h, v1.8h, v9.8h\n" "add x7, x7, #0x10\n" "fmla v30.8h, v1.8h, v8.8h\n" - "fmla v31.8h, v1.8h, v13.8h\n" - "ldr q1, [x8, #0x10]\n" - "fmla v28.8h, v2.8h, v9.8h\n" - "ldr q9, [x4, x11]\n" - "fmla v29.8h, v2.8h, v11.8h\n" - "add x4, x4, #0x10\n" - "fmla v30.8h, v2.8h, v13.8h\n" - "fmla v31.8h, v2.8h, v5.8h\n" - "ldr q2, [x8, #0x20]\n" - "fmla v28.8h, v3.8h, v11.8h\n" - "ldr q11, [x17, x2]\n" - "fmla v29.8h, v3.8h, v12.8h\n" - "fmla v30.8h, v3.8h, v5.8h\n" - "fmla v31.8h, v3.8h, v6.8h\n" - "ldr q3, [x8, #0x30]\n" - "fmla v28.8h, v4.8h, v12.8h\n" - "ldr q12, [x17, x6]\n" - "fmla v29.8h, v4.8h, v9.8h\n" - "ldr q9, [x17, x15]\n" - "fmla v30.8h, v4.8h, v6.8h\n" - "fmla v31.8h, v4.8h, v10.8h\n" - "ldr q4, [x8, #0x40]\n" - "fmla v28.8h, v0.8h, v7.8h\n" - "fmla v29.8h, v0.8h, v8.8h\n" - "fmla v30.8h, v0.8h, v14.8h\n" - "fmla v31.8h, v0.8h, v11.8h\n" - "ldr q0, [x8, #0x50]\n" - "fmla v28.8h, v1.8h, v8.8h\n" - "ldr q8, [x17, x11]\n" "fmla v29.8h, v1.8h, v13.8h\n" - "fmla v30.8h, v1.8h, v11.8h\n" - "fmla v31.8h, v1.8h, v12.8h\n" - "ldr q1, [x8, #0x60]\n" - "fmla v28.8h, v2.8h, v13.8h\n" - "ldr q13, [x17, x13]\n" - "fmla v29.8h, v2.8h, v5.8h\n" - "add x17, x17, #0x10\n" - "fmla v30.8h, v2.8h, v12.8h\n" + "ldr q18, [x8, #0x10]\n" "fmla v31.8h, v2.8h, v9.8h\n" - "ldr q2, [x8, #0x70]\n" - "fmla v28.8h, v3.8h, v5.8h\n" - "ld1 { v5.8h }, [x16]\n" - "fmla v29.8h, v3.8h, v6.8h\n" - "fmla v30.8h, v3.8h, v9.8h\n" - "fmla v31.8h, v3.8h, v13.8h\n" - "ldr q3, [x8, #0x80]\n" - "fmla v28.8h, v4.8h, v6.8h\n" - "ldr q6, [x16, x2]\n" + "ldr q16, [x4, x11]\n" + "fmla v5.8h, v2.8h, v11.8h\n" + "add x4, x4, #0x10\n" + "fmla v30.8h, v2.8h, v13.8h\n" + "fmla v29.8h, v2.8h, v22.8h\n" + "ldr q17, [x8, #0x20]\n" + "fmla v31.8h, v3.8h, v11.8h\n" + "ldr q6, [x17, x2]\n" + "fmla v5.8h, v3.8h, v12.8h\n" + "fmla v30.8h, v3.8h, v22.8h\n" + "fmla v29.8h, v3.8h, v21.8h\n" + "ldr q20, [x8, #0x30]\n" + "fmla v31.8h, v4.8h, v12.8h\n" + "ldr q2, [x17, x6]\n" + "fmla v5.8h, v4.8h, v16.8h\n" + "ldr q28, [x17, x15]\n" + "fmla v30.8h, v4.8h, v21.8h\n" "fmla v29.8h, v4.8h, v10.8h\n" - "ldr q10, [x16, x6]\n" - "fmla v30.8h, v4.8h, v13.8h\n" - "fmla v31.8h, v4.8h, v8.8h\n" - "ldr q4, [x8, #0x90]\n" - "fmla v28.8h, v0.8h, v14.8h\n" - "ldr q14, [x16, x11]\n" - "fmla v29.8h, v0.8h, v11.8h\n" - "fmla v30.8h, v0.8h, v5.8h\n" - "fmla v31.8h, v0.8h, v6.8h\n" - "ldr q0, [x8, #0xa0]\n" - "fmla v28.8h, v1.8h, v11.8h\n" - "ldr q11, [x16, x15]\n" - "fmla v29.8h, v1.8h, v12.8h\n" - "fmla v30.8h, v1.8h, v6.8h\n" - "fmla v31.8h, v1.8h, v10.8h\n" - "ldr q1, [x8, #0xb0]\n" - "fmla v28.8h, v2.8h, v12.8h\n" - "ldr q12, [x16, x13]\n" - "fmla v29.8h, v2.8h, v9.8h\n" + "ldr q16, [x8, #0x40]\n" + "fmla v31.8h, v19.8h, v7.8h\n" + "fmla v5.8h, v19.8h, v8.8h\n" + "fmla v30.8h, v19.8h, v14.8h\n" + "fmla v29.8h, v19.8h, v6.8h\n" + "ldr q19, [x8, #0x50]\n" + "fmla v31.8h, v18.8h, v8.8h\n" + "ldr q1, [x17, x11]\n" + "fmla v5.8h, v18.8h, v13.8h\n" + "fmla v30.8h, v18.8h, v6.8h\n" + "fmla v29.8h, v18.8h, v2.8h\n" + "ldr q18, [x8, #0x60]\n" + "fmla v31.8h, v17.8h, v13.8h\n" + "ldr q26, [x17, x13]\n" + "fmla v5.8h, v17.8h, v22.8h\n" + "add x17, x17, #0x10\n" + "fmla v30.8h, v17.8h, v2.8h\n" + "fmla v29.8h, v17.8h, v28.8h\n" + "ldr q17, [x8, #0x70]\n" + "fmla v31.8h, v20.8h, v22.8h\n" + "ld1 { v25.8h }, [x16]\n" + "fmla v5.8h, v20.8h, v21.8h\n" + "fmla v30.8h, v20.8h, v28.8h\n" + "fmla v29.8h, v20.8h, v26.8h\n" + "ldr q24, [x8, #0x80]\n" + "fmla v31.8h, v16.8h, v21.8h\n" + "ldr q23, [x16, x2]\n" + "fmla v5.8h, v16.8h, v10.8h\n" + "ldr q0, [x16, x6]\n" + "fmla v30.8h, v16.8h, v26.8h\n" + "fmla v29.8h, v16.8h, v1.8h\n" + "ldr q22, [x8, #0x90]\n" + "fmla v31.8h, v19.8h, v14.8h\n" + "ldr q16, [x16, x11]\n" + "fmla v5.8h, v19.8h, v6.8h\n" + "fmla v30.8h, v19.8h, v25.8h\n" + "fmla v29.8h, v19.8h, v23.8h\n" + "ldr q21, [x8, #0xa0]\n" + "fmla v31.8h, v18.8h, v6.8h\n" + "ldr q20, [x16, x15]\n" + "fmla v5.8h, v18.8h, v2.8h\n" + "fmla v30.8h, v18.8h, v23.8h\n" + "fmla v29.8h, v18.8h, v0.8h\n" + "ldr q18, [x8, #0xb0]\n" + "fmla v31.8h, v17.8h, v2.8h\n" + "ldr q19, [x16, x13]\n" + "fmla v5.8h, v17.8h, v28.8h\n" "add x16, x16, #0x10\n" - "fmla v30.8h, v2.8h, v10.8h\n" - "fmla v31.8h, v2.8h, v11.8h\n" - "ldr q2, [x8, #0xc0]\n" - "fmla v28.8h, v3.8h, v9.8h\n" - "ld1 { v9.8h }, [x14]\n" - "fmla v29.8h, v3.8h, v13.8h\n" - "fmla v30.8h, v3.8h, v11.8h\n" - "fmla v31.8h, v3.8h, v12.8h\n" - "ldr q3, [x8, #0xd0]\n" - "fmla v28.8h, v4.8h, v13.8h\n" - "ldr q13, [x14, x2]\n" - "fmla v29.8h, v4.8h, v8.8h\n" - "ldr q8, [x14, x13]\n" - "fmla v30.8h, v4.8h, v12.8h\n" - "fmla v31.8h, v4.8h, v14.8h\n" - "ldr q4, [x8, #0xe0]\n" - "fmla v28.8h, v0.8h, v5.8h\n" - "ldr q5, [x14, x6]\n" - "fmla v29.8h, v0.8h, v6.8h\n" - "fmla v30.8h, v0.8h, v9.8h\n" - "fmla v31.8h, v0.8h, v13.8h\n" - "ldr q0, [x8, #0xf0]\n" - "fmla v28.8h, v1.8h, v6.8h\n" - "ldr q6, [x14, x15]\n" - "fmla v29.8h, v1.8h, v10.8h\n" - "fmla v30.8h, v1.8h, v13.8h\n" - "fmla v31.8h, v1.8h, v5.8h\n" - "ldr q1, [x8, #0x100]\n" - "fmla v28.8h, v2.8h, v10.8h\n" - "ldr q10, [x14, x11]\n" - "fmla v29.8h, v2.8h, v11.8h\n" + "fmla v30.8h, v17.8h, v0.8h\n" + "fmla v29.8h, v17.8h, v20.8h\n" + "ldr q17, [x8, #0xc0]\n" + "fmla v31.8h, v24.8h, v28.8h\n" + "ld1 { v7.8h }, [x14]\n" + "fmla v5.8h, v24.8h, v26.8h\n" + "fmla v30.8h, v24.8h, v20.8h\n" + "fmla v29.8h, v24.8h, v19.8h\n" + "ldr q2, [x8, #0xd0]\n" + "fmla v31.8h, v22.8h, v26.8h\n" + "ldr q28, [x14, x2]\n" + "fmla v5.8h, v22.8h, v1.8h\n" + "ldr q13, [x14, x13]\n" + "fmla v30.8h, v22.8h, v19.8h\n" + "fmla v29.8h, v22.8h, v16.8h\n" + "ldr q14, [x8, #0xe0]\n" + "fmla v31.8h, v21.8h, v25.8h\n" + "ldr q26, [x14, x6]\n" + "fmla v5.8h, v21.8h, v23.8h\n" + "fmla v30.8h, v21.8h, v7.8h\n" + "fmla v29.8h, v21.8h, v28.8h\n" + "ldr q25, [x8, #0xf0]\n" + "fmla v31.8h, v18.8h, v23.8h\n" + "ldr q24, [x14, x15]\n" + "fmla v5.8h, v18.8h, v0.8h\n" + "fmla v30.8h, v18.8h, v28.8h\n" + "fmla v29.8h, v18.8h, v26.8h\n" + "ldr q23, [x8, #0x100]\n" + "fmla v31.8h, v17.8h, v0.8h\n" + "ldr q22, [x14, x11]\n" + "fmla v5.8h, v17.8h, v20.8h\n" "add x14, x14, #0x10\n" - "fmla v30.8h, v2.8h, v5.8h\n" - "fmla v31.8h, v2.8h, v6.8h\n" - "ldr q2, [x8, #0x110]\n" - "fmla v28.8h, v3.8h, v11.8h\n" - "ld1 { v11.8h }, [x12]\n" - "fmla v29.8h, v3.8h, v12.8h\n" - "fmla v30.8h, v3.8h, v6.8h\n" - "fmla v31.8h, v3.8h, v8.8h\n" - "ldr q3, [x8, #0x120]\n" - "fmla v28.8h, v4.8h, v12.8h\n" - "ldr q12, [x12, x2]\n" - "fmla v29.8h, v4.8h, v14.8h\n" - "fmla v30.8h, v4.8h, v8.8h\n" - "fmla v31.8h, v4.8h, v10.8h\n" - "ldr q4, [x8, #0x130]\n" + "fmla v30.8h, v17.8h, v26.8h\n" + "fmla v29.8h, v17.8h, v24.8h\n" + "ldr q21, [x8, #0x110]\n" + "fmla v31.8h, v2.8h, v20.8h\n" + "ld1 { v18.8h }, [x12]\n" + "fmla v5.8h, v2.8h, v19.8h\n" + "fmla v30.8h, v2.8h, v24.8h\n" + "fmla v29.8h, v2.8h, v13.8h\n" + "ldr q20, [x8, #0x120]\n" + "fmla v31.8h, v14.8h, v19.8h\n" + "ldr q17, [x12, x2]\n" + "fmla v5.8h, v14.8h, v16.8h\n" + "fmla v30.8h, v14.8h, v13.8h\n" + "fmla v29.8h, v14.8h, v22.8h\n" + "ldr q19, [x8, #0x130]\n" "add x8, x8, #0x140\n" - "fmla v28.8h, v0.8h, v9.8h\n" - "ldr q9, [x12, x6]\n" - "fmla v29.8h, v0.8h, v13.8h\n" - "fmla v30.8h, v0.8h, v11.8h\n" - "ldr q11, [x12, x15]\n" - "fmla v31.8h, v0.8h, v12.8h\n" - "fmla v28.8h, v1.8h, v13.8h\n" - "fmla v29.8h, v1.8h, v5.8h\n" - "fmla v30.8h, v1.8h, v12.8h\n" - "ldr q12, [x12, x13]\n" - "fmla v31.8h, v1.8h, v9.8h\n" - "fmla v28.8h, v2.8h, v5.8h\n" - "fmla v29.8h, v2.8h, v6.8h\n" - "fmla v30.8h, v2.8h, v9.8h\n" - "ldr q9, [x12, x11]\n" - "fmla v31.8h, v2.8h, v11.8h\n" + "fmla v31.8h, v25.8h, v7.8h\n" + "ldr q16, [x12, x6]\n" + "fmla v5.8h, v25.8h, v28.8h\n" + "fmla v30.8h, v25.8h, v18.8h\n" + "ldr q18, [x12, x15]\n" + "fmla v29.8h, v25.8h, v17.8h\n" + "fmla v31.8h, v23.8h, v28.8h\n" + "fmla v5.8h, v23.8h, v26.8h\n" + "fmla v30.8h, v23.8h, v17.8h\n" + "ldr q17, [x12, x13]\n" + "fmla v29.8h, v23.8h, v16.8h\n" + "fmla v31.8h, v21.8h, v26.8h\n" + "fmla v5.8h, v21.8h, v24.8h\n" + "fmla v30.8h, v21.8h, v16.8h\n" + "ldr q16, [x12, x11]\n" + "fmla v29.8h, v21.8h, v18.8h\n" "add x12, x12, #0x10\n" - "fmla v28.8h, v3.8h, v6.8h\n" - "fmla v29.8h, v3.8h, v8.8h\n" - "fmla v30.8h, v3.8h, v11.8h\n" - "fmla v31.8h, v3.8h, v12.8h\n" - "fmla v28.8h, v4.8h, v8.8h\n" - "fmla v29.8h, v4.8h, v10.8h\n" - "fmax v28.8h, v28.8h, v18.8h\n" - "fmla v30.8h, v4.8h, v12.8h\n" - "fmla v31.8h, v4.8h, v9.8h\n" - "fmax v29.8h, v29.8h, v18.8h\n" - "fmax v30.8h, v30.8h, v18.8h\n" - "fmax v31.8h, v31.8h, v18.8h\n" - "fmin v28.8h, v28.8h, v17.8h\n" - "fmin v29.8h, v29.8h, v17.8h\n" - "st1 { v28.8h }, [x5]\n" - "fmin v30.8h, v30.8h, v17.8h\n" - "fmin v31.8h, v31.8h, v17.8h\n" - "str q29, [x5, x3]\n" + "fmla v31.8h, v20.8h, v24.8h\n" + "fmla v5.8h, v20.8h, v13.8h\n" + "fmla v30.8h, v20.8h, v18.8h\n" + "fmla v29.8h, v20.8h, v17.8h\n" + "fmla v31.8h, v19.8h, v13.8h\n" + "fmla v5.8h, v19.8h, v22.8h\n" + "fmax v31.8h, v31.8h, v27.8h\n" + "fmla v30.8h, v19.8h, v17.8h\n" + "fmla v29.8h, v19.8h, v16.8h\n" + "fmax v5.8h, v5.8h, v27.8h\n" + "fmax v30.8h, v30.8h, v27.8h\n" + "fmax v29.8h, v29.8h, v27.8h\n" + "fmin v31.8h, v31.8h, v15.8h\n" + "fmin v5.8h, v5.8h, v15.8h\n" + "st1 { v31.8h }, [x5]\n" + "fmin v30.8h, v30.8h, v15.8h\n" + "fmin v29.8h, v29.8h, v15.8h\n" + "str q5, [x5, x3]\n" "add x5, x5, #0x10\n" "st1 { v30.8h }, [x10]\n" - "str q31, [x10, x3]\n" + "str q29, [x10, x3]\n" "add x10, x10, #0x10\n" "4:" // Tile loop: Oddments "tst %x[n_channels], #0x7\n" "beq 117f\n" - "ldr q16, [x8, #0x0]\n" + "ldr q25, [x8, #0x0]\n" "ldr q0, [x8, #0x10]\n" "add x9, x4, XZR\n" "add x28, x4, x2\n" @@ -609,11 +609,11 @@ void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "ldr h10, [x21, #0x0]\n" "ldr h14, [x20, #0x0]\n" "8:" // Tile loop: Oddments: Load inputs: (0, 0), (0, 1), (1, 0), (1, 1), (0, 2), (1, 2), (0, 3), (0, 4), (1, 5), (2, 0): Bit 2: End - "mov v28.16b, v16.16b\n fmla v28.8h, v0.8h, v5.8h\n" - "mov v29.16b, v16.16b\n fmla v29.8h, v0.8h, v6.8h\n" + "mov v28.16b, v25.16b\n fmla v28.8h, v0.8h, v5.8h\n" + "mov v29.16b, v25.16b\n fmla v29.8h, v0.8h, v6.8h\n" "add x20, x7, x15\n" - "mov v30.16b, v16.16b\n fmla v30.8h, v0.8h, v7.8h\n" - "mov v31.16b, v16.16b\n fmla v31.8h, v0.8h, v8.8h\n" + "mov v30.16b, v25.16b\n fmla v30.8h, v0.8h, v7.8h\n" + "mov v31.16b, v25.16b\n fmla v31.8h, v0.8h, v8.8h\n" "fmla v28.8h, v1.8h, v6.8h\n" "fmla v29.8h, v1.8h, v9.8h\n" "fmla v30.8h, v1.8h, v8.8h\n" @@ -1294,14 +1294,14 @@ void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "ldr h9, [x20, #0x0]\n" "112:" // Tile loop: Oddments: Load inputs: (5, 5): Bit 2: End "fmla v31.8h, v4.8h, v9.8h\n" - "fmax v28.8h, v28.8h, v18.8h\n" - "fmax v29.8h, v29.8h, v18.8h\n" - "fmax v30.8h, v30.8h, v18.8h\n" - "fmax v31.8h, v31.8h, v18.8h\n" - "fmin v28.8h, v28.8h, v17.8h\n" - "fmin v29.8h, v29.8h, v17.8h\n" - "fmin v30.8h, v30.8h, v17.8h\n" - "fmin v31.8h, v31.8h, v17.8h\n" + "fmax v28.8h, v28.8h, v27.8h\n" + "fmax v29.8h, v29.8h, v27.8h\n" + "fmax v30.8h, v30.8h, v27.8h\n" + "fmax v31.8h, v31.8h, v27.8h\n" + "fmin v28.8h, v28.8h, v15.8h\n" + "fmin v29.8h, v29.8h, v15.8h\n" + "fmin v30.8h, v30.8h, v15.8h\n" + "fmin v31.8h, v31.8h, v15.8h\n" "tbz %x[n_channels], #2, 114f\n" "mov x21, x5\n" "mov x20, x10\n" @@ -1363,7 +1363,6 @@ void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "st1 { v29.h }[0], [x21]\n" "st1 { v31.h }[0], [x20]\n" "116:" // Tile loop: Oddments: Store: Bit 2: End - "117:" // Tile loop: End "ldr x26, [%x[params_struct], %[offsetof_args_tile_j]]\n" "ldr x27, [%x[params_struct], %[offsetof_args_tile_i]]\n" @@ -1378,7 +1377,7 @@ void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v16", "v17", "v18", "v28", "v29", "v30", "v31", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp index a2791d277e24452c90ceea9c9664164c15ad5792..4913340c4cc6169c87fd663daad91c809c1c4f65 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -103,16 +103,16 @@ void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl( "lsr x9, %x[n_channels], #0x3\n" "ldr x16, [%x[params_struct], %[offsetof_args_params]]\n" "add x20, %x[params_struct], %[offsetof_args_min]\n" - "ld1r { v18.8h }, [x20]\n" + "ld1r { v27.8h }, [x20]\n" "add x20, %x[params_struct], %[offsetof_args_max]\n" - "ld1r { v17.8h }, [x20]\n" + "ld1r { v15.8h }, [x20]\n" "add x15, %x[params_struct], %[offsetof_Args_inptrs]\n" "ldp x14, x13, [x21, #0x0]\n" "ldp x12, x11, [x21, #0x10]\n" "mov x10, #0x0\n" "sub x28, XZR, x17\n" "cbz x9, 3f\n" - "ldr q16, [x16, #0x0]\n" + "ldr q26, [x16, #0x0]\n" "ldr q0, [x16, #0x10]\n" "cmp x17, x9, LSL #4\n" "ldr q1, [x16, #0x20]\n" @@ -120,436 +120,436 @@ void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl( "ldr q3, [x16, #0x40]\n" "ldr q4, [x16, #0x50]\n" "add x16, x16, #0x60\n" - "ldp x27, x26, [x15, #0x0]\n" - "ldr q5, [x27, x10]\n" - "ldr q6, [x26, x10]\n" - "ldp x25, x24, [x15, #0x10]\n" - "ldr q7, [x25, x10]\n" - "ldr q8, [x24, x10]\n" - "ldp x23, x22, [x15, #0x20]\n" - "ldr q9, [x23, x10]\n" - "ldr q13, [x22, x10]\n" + "ldp x21, x20, [x15, #0x0]\n" + "ldr q5, [x21, x10]\n" + "ldr q6, [x20, x10]\n" + "ldp x21, x20, [x15, #0x10]\n" + "ldr q7, [x21, x10]\n" + "ldr q8, [x20, x10]\n" + "ldp x21, x20, [x15, #0x20]\n" + "ldr q9, [x21, x10]\n" + "ldr q13, [x20, x10]\n" "ldp x21, x20, [x15, #0x30]\n" "ldr q11, [x21, x10]\n" "ldr q12, [x20, x10]\n" - "ldp x27, x26, [x15, #0x40]\n" - "ldr q10, [x27, x10]\n" - "ldr q14, [x26, x10]\n" + "ldp x21, x20, [x15, #0x40]\n" + "ldr q10, [x21, x10]\n" + "ldr q14, [x20, x10]\n" "bge 2f\n" "1:" // Channel loop - "mov v28.16b, v16.16b\n fmla v28.8h, v0.8h, v5.8h\n" - "mov v29.16b, v16.16b\n fmla v29.8h, v0.8h, v6.8h\n" - "ldr x25, [x15, #0x50]\n" - "ldr q5, [x25, x10]\n" - "mov v30.16b, v16.16b\n fmla v30.8h, v0.8h, v7.8h\n" - "mov v31.16b, v16.16b\n fmla v31.8h, v0.8h, v8.8h\n" - "ldr q0, [x16, #0x0]\n" - "ldr q16, [x16, #0x140]\n" - "fmla v28.8h, v1.8h, v6.8h\n" - "fmla v29.8h, v1.8h, v9.8h\n" - "ldr x24, [x15, #0x58]\n" - "ldr q6, [x24, x10]\n" - "fmla v30.8h, v1.8h, v8.8h\n" - "fmla v31.8h, v1.8h, v13.8h\n" - "ldr q1, [x16, #0x10]\n" - "ldr x23, [x15, #0x60]\n" - "fmla v28.8h, v2.8h, v9.8h\n" - "ldr q9, [x23, x10]\n" - "fmla v29.8h, v2.8h, v11.8h\n" - "ldr x22, [x15, #0x68]\n" - "fmla v30.8h, v2.8h, v13.8h\n" - "fmla v31.8h, v2.8h, v5.8h\n" - "ldr q2, [x16, #0x20]\n" - "ldr x21, [x15, #0x70]\n" - "fmla v28.8h, v3.8h, v11.8h\n" - "ldr q11, [x22, x10]\n" - "fmla v29.8h, v3.8h, v12.8h\n" - "ldr x20, [x15, #0x78]\n" - "fmla v30.8h, v3.8h, v5.8h\n" - "fmla v31.8h, v3.8h, v6.8h\n" - "ldr q3, [x16, #0x30]\n" - "ldr x27, [x15, #0x80]\n" - "fmla v28.8h, v4.8h, v12.8h\n" - "ldr q12, [x21, x10]\n" - "fmla v29.8h, v4.8h, v9.8h\n" - "ldr q9, [x20, x10]\n" - "fmla v30.8h, v4.8h, v6.8h\n" - "fmla v31.8h, v4.8h, v10.8h\n" - "ldr q4, [x16, #0x40]\n" - "ldr x26, [x15, #0x88]\n" - "fmla v28.8h, v0.8h, v7.8h\n" - "fmla v29.8h, v0.8h, v8.8h\n" - "ldr x25, [x15, #0x90]\n" - "ldr x24, [x15, #0x98]\n" - "fmla v30.8h, v0.8h, v14.8h\n" - "fmla v31.8h, v0.8h, v11.8h\n" - "ldr q0, [x16, #0x50]\n" - "ldr x23, [x15, #0xa0]\n" + "mov v30.16b, v26.16b\n fmla v30.8h, v0.8h, v5.8h\n" + "mov v31.16b, v26.16b\n fmla v31.8h, v0.8h, v6.8h\n" + "ldr x20, [x15, #0x50]\n" + "ldr q24, [x20, x10]\n" + "mov v28.16b, v26.16b\n fmla v28.8h, v0.8h, v7.8h\n" + "mov v29.16b, v26.16b\n fmla v29.8h, v0.8h, v8.8h\n" + "ldr q23, [x16, #0x0]\n" + "ldr q26, [x16, #0x140]\n" + "fmla v30.8h, v1.8h, v6.8h\n" + "fmla v31.8h, v1.8h, v9.8h\n" + "ldr x20, [x15, #0x58]\n" + "ldr q22, [x20, x10]\n" "fmla v28.8h, v1.8h, v8.8h\n" - "ldr q8, [x26, x10]\n" "fmla v29.8h, v1.8h, v13.8h\n" - "ldr x22, [x15, #0xa8]\n" - "fmla v30.8h, v1.8h, v11.8h\n" - "fmla v31.8h, v1.8h, v12.8h\n" - "ldr q1, [x16, #0x60]\n" - "ldr x21, [x15, #0xb0]\n" - "fmla v28.8h, v2.8h, v13.8h\n" - "ldr q13, [x27, x10]\n" - "fmla v29.8h, v2.8h, v5.8h\n" - "ldr x20, [x15, #0xb8]\n" - "fmla v30.8h, v2.8h, v12.8h\n" - "fmla v31.8h, v2.8h, v9.8h\n" - "ldr q2, [x16, #0x70]\n" - "ldr x27, [x15, #0xc0]\n" - "fmla v28.8h, v3.8h, v5.8h\n" - "ldr q5, [x25, x10]\n" - "fmla v29.8h, v3.8h, v6.8h\n" - "ldr x26, [x15, #0xc8]\n" - "fmla v30.8h, v3.8h, v9.8h\n" - "fmla v31.8h, v3.8h, v13.8h\n" - "ldr q3, [x16, #0x80]\n" - "ldr x25, [x15, #0xd0]\n" - "fmla v28.8h, v4.8h, v6.8h\n" - "ldr q6, [x24, x10]\n" - "fmla v29.8h, v4.8h, v10.8h\n" - "ldr q10, [x23, x10]\n" - "fmla v30.8h, v4.8h, v13.8h\n" - "fmla v31.8h, v4.8h, v8.8h\n" - "ldr q4, [x16, #0x90]\n" - "ldr x24, [x15, #0xd8]\n" - "fmla v28.8h, v0.8h, v14.8h\n" - "ldr q14, [x20, x10]\n" - "fmla v29.8h, v0.8h, v11.8h\n" - "ldr x23, [x15, #0xe0]\n" - "fmla v30.8h, v0.8h, v5.8h\n" - "fmla v31.8h, v0.8h, v6.8h\n" - "ldr q0, [x16, #0xa0]\n" - "ldr x20, [x15, #0xf8]\n" - "fmla v28.8h, v1.8h, v11.8h\n" - "ldr q11, [x22, x10]\n" - "fmla v29.8h, v1.8h, v12.8h\n" - "ldr x22, [x15, #0xe8]\n" - "fmla v30.8h, v1.8h, v6.8h\n" - "fmla v31.8h, v1.8h, v10.8h\n" - "ldr q1, [x16, #0xb0]\n" - "add x28, x28, #0x10\n" - "fmla v28.8h, v2.8h, v12.8h\n" - "ldr q12, [x21, x10]\n" - "fmla v29.8h, v2.8h, v9.8h\n" - "ldr x21, [x15, #0xf0]\n" - "fmla v30.8h, v2.8h, v10.8h\n" + "ldr q21, [x16, #0x10]\n" + "ldr x20, [x15, #0x60]\n" + "fmla v30.8h, v2.8h, v9.8h\n" + "ldr q17, [x20, x10]\n" "fmla v31.8h, v2.8h, v11.8h\n" - "ldr q2, [x16, #0xc0]\n" - "fmla v28.8h, v3.8h, v9.8h\n" - "ldr q9, [x27, x10]\n" - "fmla v29.8h, v3.8h, v13.8h\n" - "ldr x27, [x15, #0x100]\n" + "ldr x20, [x15, #0x68]\n" + "fmla v28.8h, v2.8h, v13.8h\n" + "fmla v29.8h, v2.8h, v24.8h\n" + "ldr q16, [x16, #0x20]\n" + "ldr x22, [x15, #0x70]\n" "fmla v30.8h, v3.8h, v11.8h\n" + "ldr q5, [x20, x10]\n" "fmla v31.8h, v3.8h, v12.8h\n" - "ldr q3, [x16, #0xd0]\n" - "fmla v28.8h, v4.8h, v13.8h\n" - "ldr q13, [x26, x10]\n" - "fmla v29.8h, v4.8h, v8.8h\n" - "ldr q8, [x23, x10]\n" + "ldr x20, [x15, #0x78]\n" + "fmla v28.8h, v3.8h, v24.8h\n" + "fmla v29.8h, v3.8h, v22.8h\n" + "ldr q20, [x16, #0x30]\n" + "ldr x21, [x15, #0x80]\n" "fmla v30.8h, v4.8h, v12.8h\n" - "fmla v31.8h, v4.8h, v14.8h\n" - "ldr q4, [x16, #0xe0]\n" - "ldr x26, [x15, #0x108]\n" - "fmla v28.8h, v0.8h, v5.8h\n" - "ldr q5, [x25, x10]\n" - "fmla v29.8h, v0.8h, v6.8h\n" - "ldr x25, [x15, #0x110]\n" - "fmla v30.8h, v0.8h, v9.8h\n" - "fmla v31.8h, v0.8h, v13.8h\n" - "ldr q0, [x16, #0xf0]\n" - "fmla v28.8h, v1.8h, v6.8h\n" - "ldr q6, [x24, x10]\n" - "fmla v29.8h, v1.8h, v10.8h\n" - "ldr x24, [x15, #0x118]\n" - "fmla v30.8h, v1.8h, v13.8h\n" + "ldr q19, [x22, x10]\n" + "fmla v31.8h, v4.8h, v17.8h\n" + "ldr q2, [x20, x10]\n" + "fmla v28.8h, v4.8h, v22.8h\n" + "fmla v29.8h, v4.8h, v10.8h\n" + "ldr q18, [x16, #0x40]\n" + "ldr x20, [x15, #0x88]\n" + "fmla v30.8h, v23.8h, v7.8h\n" + "fmla v31.8h, v23.8h, v8.8h\n" + "ldr x23, [x15, #0x90]\n" + "ldr x26, [x15, #0x98]\n" + "fmla v28.8h, v23.8h, v14.8h\n" + "fmla v29.8h, v23.8h, v5.8h\n" + "ldr q1, [x16, #0x50]\n" + "ldr x22, [x15, #0xa0]\n" + "fmla v30.8h, v21.8h, v8.8h\n" + "ldr q25, [x20, x10]\n" + "fmla v31.8h, v21.8h, v13.8h\n" + "ldr x25, [x15, #0xa8]\n" + "fmla v28.8h, v21.8h, v5.8h\n" + "fmla v29.8h, v21.8h, v19.8h\n" + "ldr q17, [x16, #0x60]\n" + "ldr x24, [x15, #0xb0]\n" + "fmla v30.8h, v16.8h, v13.8h\n" + "ldr q8, [x21, x10]\n" + "fmla v31.8h, v16.8h, v24.8h\n" + "ldr x20, [x15, #0xb8]\n" + "fmla v28.8h, v16.8h, v19.8h\n" + "fmla v29.8h, v16.8h, v2.8h\n" + "ldr q16, [x16, #0x70]\n" + "ldr x21, [x15, #0xc0]\n" + "fmla v30.8h, v20.8h, v24.8h\n" + "ldr q24, [x23, x10]\n" + "fmla v31.8h, v20.8h, v22.8h\n" + "ldr x27, [x15, #0xc8]\n" + "fmla v28.8h, v20.8h, v2.8h\n" + "fmla v29.8h, v20.8h, v8.8h\n" + "ldr q23, [x16, #0x80]\n" + "ldr x23, [x15, #0xd0]\n" + "fmla v30.8h, v18.8h, v22.8h\n" + "ldr q22, [x26, x10]\n" + "fmla v31.8h, v18.8h, v10.8h\n" + "ldr q21, [x22, x10]\n" + "fmla v28.8h, v18.8h, v8.8h\n" + "fmla v29.8h, v18.8h, v25.8h\n" + "ldr q20, [x16, #0x90]\n" + "ldr x22, [x15, #0xd8]\n" + "fmla v30.8h, v1.8h, v14.8h\n" + "ldr q0, [x20, x10]\n" "fmla v31.8h, v1.8h, v5.8h\n" - "ldr q1, [x16, #0x100]\n" - "fmla v28.8h, v2.8h, v10.8h\n" - "ldr q10, [x22, x10]\n" - "fmla v29.8h, v2.8h, v11.8h\n" - "fmla v30.8h, v2.8h, v5.8h\n" - "fmla v31.8h, v2.8h, v6.8h\n" - "ldr q2, [x16, #0x110]\n" - "fmla v28.8h, v3.8h, v11.8h\n" - "ldr q11, [x21, x10]\n" - "fmla v29.8h, v3.8h, v12.8h\n" - "fmla v30.8h, v3.8h, v6.8h\n" - "fmla v31.8h, v3.8h, v8.8h\n" - "ldr q3, [x16, #0x120]\n" - "fmla v28.8h, v4.8h, v12.8h\n" - "ldr q12, [x20, x10]\n" - "fmla v29.8h, v4.8h, v14.8h\n" - "fmla v30.8h, v4.8h, v8.8h\n" - "fmla v31.8h, v4.8h, v10.8h\n" - "ldr q4, [x16, #0x130]\n" - "fmla v28.8h, v0.8h, v9.8h\n" - "ldr q9, [x27, x10]\n" - "fmla v29.8h, v0.8h, v13.8h\n" - "fmla v30.8h, v0.8h, v11.8h\n" - "ldr q11, [x26, x10]\n" - "fmla v31.8h, v0.8h, v12.8h\n" + "ldr x20, [x15, #0xe0]\n" + "fmla v28.8h, v1.8h, v24.8h\n" + "fmla v29.8h, v1.8h, v22.8h\n" + "ldr q6, [x16, #0xa0]\n" + "ldr x26, [x15, #0xf8]\n" + "fmla v30.8h, v17.8h, v5.8h\n" + "ldr q1, [x25, x10]\n" + "fmla v31.8h, v17.8h, v19.8h\n" + "ldr x25, [x15, #0xe8]\n" + "fmla v28.8h, v17.8h, v22.8h\n" + "fmla v29.8h, v17.8h, v21.8h\n" + "ldr q18, [x16, #0xb0]\n" + "add x28, x28, #0x10\n" + "fmla v30.8h, v16.8h, v19.8h\n" + "ldr q19, [x24, x10]\n" + "fmla v31.8h, v16.8h, v2.8h\n" + "ldr x24, [x15, #0xf0]\n" + "fmla v28.8h, v16.8h, v21.8h\n" + "fmla v29.8h, v16.8h, v1.8h\n" + "ldr q17, [x16, #0xc0]\n" + "fmla v30.8h, v23.8h, v2.8h\n" + "ldr q16, [x21, x10]\n" + "fmla v31.8h, v23.8h, v8.8h\n" + "ldr x21, [x15, #0x100]\n" + "fmla v28.8h, v23.8h, v1.8h\n" + "fmla v29.8h, v23.8h, v19.8h\n" + "ldr q13, [x16, #0xd0]\n" + "fmla v30.8h, v20.8h, v8.8h\n" + "ldr q2, [x27, x10]\n" + "fmla v31.8h, v20.8h, v25.8h\n" + "ldr q10, [x20, x10]\n" + "fmla v28.8h, v20.8h, v19.8h\n" + "fmla v29.8h, v20.8h, v0.8h\n" + "ldr q9, [x16, #0xe0]\n" + "ldr x20, [x15, #0x108]\n" + "fmla v30.8h, v6.8h, v24.8h\n" + "ldr q5, [x23, x10]\n" + "fmla v31.8h, v6.8h, v22.8h\n" + "ldr x23, [x15, #0x110]\n" + "fmla v28.8h, v6.8h, v16.8h\n" + "fmla v29.8h, v6.8h, v2.8h\n" + "ldr q24, [x16, #0xf0]\n" + "fmla v30.8h, v18.8h, v22.8h\n" + "ldr q25, [x22, x10]\n" + "fmla v31.8h, v18.8h, v21.8h\n" + "ldr x22, [x15, #0x118]\n" + "fmla v28.8h, v18.8h, v2.8h\n" + "fmla v29.8h, v18.8h, v5.8h\n" + "ldr q23, [x16, #0x100]\n" + "fmla v30.8h, v17.8h, v21.8h\n" + "ldr q22, [x25, x10]\n" + "fmla v31.8h, v17.8h, v1.8h\n" + "fmla v28.8h, v17.8h, v5.8h\n" + "fmla v29.8h, v17.8h, v25.8h\n" + "ldr q21, [x16, #0x110]\n" + "fmla v30.8h, v13.8h, v1.8h\n" + "ldr q18, [x24, x10]\n" + "fmla v31.8h, v13.8h, v19.8h\n" + "fmla v28.8h, v13.8h, v25.8h\n" + "fmla v29.8h, v13.8h, v10.8h\n" + "ldr q20, [x16, #0x120]\n" + "fmla v30.8h, v9.8h, v19.8h\n" + "ldr q17, [x26, x10]\n" + "fmla v31.8h, v9.8h, v0.8h\n" + "fmla v28.8h, v9.8h, v10.8h\n" + "fmla v29.8h, v9.8h, v22.8h\n" + "ldr q19, [x16, #0x130]\n" + "fmla v30.8h, v24.8h, v16.8h\n" + "ldr q16, [x21, x10]\n" + "fmla v31.8h, v24.8h, v2.8h\n" + "fmla v28.8h, v24.8h, v18.8h\n" + "ldr q18, [x20, x10]\n" + "fmla v29.8h, v24.8h, v17.8h\n" "ldr q0, [x16, #0x150]\n" - "fmla v28.8h, v1.8h, v13.8h\n" - "fmla v29.8h, v1.8h, v5.8h\n" - "ldp x27, x26, [x15, #0x0]\n" - "fmla v30.8h, v1.8h, v12.8h\n" - "ldr q12, [x25, x10]\n" - "fmla v31.8h, v1.8h, v9.8h\n" + "fmla v30.8h, v23.8h, v2.8h\n" + "fmla v31.8h, v23.8h, v5.8h\n" + "ldp x21, x20, [x15, #0x0]\n" + "fmla v28.8h, v23.8h, v17.8h\n" + "ldr q17, [x23, x10]\n" + "fmla v29.8h, v23.8h, v16.8h\n" "ldr q1, [x16, #0x160]\n" - "fmla v28.8h, v2.8h, v5.8h\n" - "ldr q5, [x27, x17]\n" - "fmla v29.8h, v2.8h, v6.8h\n" - "fmla v30.8h, v2.8h, v9.8h\n" - "ldr q9, [x24, x10]\n" - "fmla v31.8h, v2.8h, v11.8h\n" + "fmla v30.8h, v21.8h, v5.8h\n" + "ldr q5, [x21, x17]\n" + "fmla v31.8h, v21.8h, v25.8h\n" + "fmla v28.8h, v21.8h, v16.8h\n" + "ldr q16, [x22, x10]\n" + "fmla v29.8h, v21.8h, v18.8h\n" "ldr q2, [x16, #0x170]\n" - "fmla v28.8h, v3.8h, v6.8h\n" - "ldr q6, [x26, x17]\n" - "fmla v29.8h, v3.8h, v8.8h\n" - "ldp x25, x24, [x15, #0x10]\n" - "ldr q7, [x25, x17]\n" - "fmla v30.8h, v3.8h, v11.8h\n" - "fmla v31.8h, v3.8h, v12.8h\n" + "fmla v30.8h, v20.8h, v25.8h\n" + "ldr q6, [x20, x17]\n" + "fmla v31.8h, v20.8h, v10.8h\n" + "ldp x21, x20, [x15, #0x10]\n" + "ldr q7, [x21, x17]\n" + "fmla v28.8h, v20.8h, v18.8h\n" + "fmla v29.8h, v20.8h, v17.8h\n" "ldr q3, [x16, #0x180]\n" - "fmla v28.8h, v4.8h, v8.8h\n" - "ldr q8, [x24, x17]\n" - "fmla v29.8h, v4.8h, v10.8h\n" - "ldp x23, x22, [x15, #0x20]\n" - "ldr q13, [x22, x17]\n" - "fmla v30.8h, v4.8h, v12.8h\n" - "fmla v31.8h, v4.8h, v9.8h\n" - "ldr q9, [x23, x17]\n" + "fmla v30.8h, v19.8h, v10.8h\n" + "ldr q8, [x20, x17]\n" + "fmla v31.8h, v19.8h, v22.8h\n" + "ldp x21, x20, [x15, #0x20]\n" + "ldr q13, [x20, x17]\n" + "fmla v28.8h, v19.8h, v17.8h\n" + "fmla v29.8h, v19.8h, v16.8h\n" + "ldr q9, [x21, x17]\n" "ldr q4, [x16, #0x190]\n" "ldp x21, x20, [x15, #0x30]\n" - "fmax v28.8h, v28.8h, v18.8h\n" - "fmax v29.8h, v29.8h, v18.8h\n" + "fmax v30.8h, v30.8h, v27.8h\n" + "fmax v31.8h, v31.8h, v27.8h\n" "ldr q11, [x21, x17]\n" "ldr q12, [x20, x17]\n" - "fmax v30.8h, v30.8h, v18.8h\n" - "fmax v31.8h, v31.8h, v18.8h\n" - "ldp x27, x26, [x15, #0x40]\n" - "ldr q10, [x27, x17]\n" - "fmin v28.8h, v28.8h, v17.8h\n" - "fmin v29.8h, v29.8h, v17.8h\n" - "ldr q14, [x26, x17]\n" + "fmax v28.8h, v28.8h, v27.8h\n" + "fmax v29.8h, v29.8h, v27.8h\n" + "ldp x21, x20, [x15, #0x40]\n" + "ldr q10, [x21, x17]\n" + "fmin v30.8h, v30.8h, v15.8h\n" + "fmin v31.8h, v31.8h, v15.8h\n" + "ldr q14, [x20, x17]\n" "add x17, x17, #0x10\n" "cmp x17, x9, LSL #4\n" - "fmin v30.8h, v30.8h, v17.8h\n" - "fmin v31.8h, v31.8h, v17.8h\n" + "fmin v28.8h, v28.8h, v15.8h\n" + "fmin v29.8h, v29.8h, v15.8h\n" "add x10, x10, #0x10\n" - "str q28, [x14, x28]\n" + "str q30, [x14, x28]\n" "add x16, x16, #0x1a0\n" - "str q29, [x13, x28]\n" - "str q30, [x12, x28]\n" - "str q31, [x11, x28]\n" + "str q31, [x13, x28]\n" + "str q28, [x12, x28]\n" + "str q29, [x11, x28]\n" "blt 1b\n" "2:" // Channel tail - "mov v28.16b, v16.16b\n fmla v28.8h, v0.8h, v5.8h\n" - "mov v29.16b, v16.16b\n fmla v29.8h, v0.8h, v6.8h\n" - "ldr x25, [x15, #0x50]\n" - "ldr q5, [x25, x10]\n" - "mov v30.16b, v16.16b\n fmla v30.8h, v0.8h, v7.8h\n" - "mov v31.16b, v16.16b\n fmla v31.8h, v0.8h, v8.8h\n" - "ldr q0, [x16, #0x0]\n" - "ldr x24, [x15, #0x58]\n" - "fmla v28.8h, v1.8h, v6.8h\n" - "ldr q6, [x24, x10]\n" - "fmla v29.8h, v1.8h, v9.8h\n" - "ldr x23, [x15, #0x60]\n" + "mov v31.16b, v26.16b\n fmla v31.8h, v0.8h, v5.8h\n" + "mov v5.16b, v26.16b\n fmla v5.8h, v0.8h, v6.8h\n" + "ldr x20, [x15, #0x50]\n" + "ldr q22, [x20, x10]\n" + "mov v30.16b, v26.16b\n fmla v30.8h, v0.8h, v7.8h\n" + "mov v29.16b, v26.16b\n fmla v29.8h, v0.8h, v8.8h\n" + "ldr q19, [x16, #0x0]\n" + "ldr x20, [x15, #0x58]\n" + "fmla v31.8h, v1.8h, v6.8h\n" + "ldr q21, [x20, x10]\n" + "fmla v5.8h, v1.8h, v9.8h\n" + "ldr x21, [x15, #0x60]\n" "fmla v30.8h, v1.8h, v8.8h\n" - "fmla v31.8h, v1.8h, v13.8h\n" - "ldr q1, [x16, #0x10]\n" - "ldr x22, [x15, #0x68]\n" - "fmla v28.8h, v2.8h, v9.8h\n" - "ldr q9, [x23, x10]\n" - "fmla v29.8h, v2.8h, v11.8h\n" - "ldr x21, [x15, #0x70]\n" - "fmla v30.8h, v2.8h, v13.8h\n" - "fmla v31.8h, v2.8h, v5.8h\n" - "ldr q2, [x16, #0x20]\n" - "ldr x20, [x15, #0x78]\n" - "fmla v28.8h, v3.8h, v11.8h\n" - "ldr q11, [x22, x10]\n" - "fmla v29.8h, v3.8h, v12.8h\n" - "ldr x27, [x15, #0x80]\n" - "fmla v30.8h, v3.8h, v5.8h\n" - "fmla v31.8h, v3.8h, v6.8h\n" - "ldr q3, [x16, #0x30]\n" - "ldr x26, [x15, #0x88]\n" - "fmla v28.8h, v4.8h, v12.8h\n" - "ldr q12, [x21, x10]\n" - "fmla v29.8h, v4.8h, v9.8h\n" - "ldr q9, [x20, x10]\n" - "fmla v30.8h, v4.8h, v6.8h\n" - "fmla v31.8h, v4.8h, v10.8h\n" - "ldr q4, [x16, #0x40]\n" - "ldr x25, [x15, #0x90]\n" - "fmla v28.8h, v0.8h, v7.8h\n" - "fmla v29.8h, v0.8h, v8.8h\n" - "ldr x24, [x15, #0x98]\n" - "ldr x23, [x15, #0xa0]\n" - "fmla v30.8h, v0.8h, v14.8h\n" - "fmla v31.8h, v0.8h, v11.8h\n" - "ldr q0, [x16, #0x50]\n" - "ldr x22, [x15, #0xa8]\n" - "fmla v28.8h, v1.8h, v8.8h\n" - "ldr q8, [x26, x10]\n" "fmla v29.8h, v1.8h, v13.8h\n" - "ldr x21, [x15, #0xb0]\n" - "fmla v30.8h, v1.8h, v11.8h\n" - "fmla v31.8h, v1.8h, v12.8h\n" - "ldr q1, [x16, #0x60]\n" - "ldr x20, [x15, #0xb8]\n" - "fmla v28.8h, v2.8h, v13.8h\n" - "ldr q13, [x27, x10]\n" - "fmla v29.8h, v2.8h, v5.8h\n" - "ldr x27, [x15, #0xc0]\n" - "fmla v30.8h, v2.8h, v12.8h\n" + "ldr q18, [x16, #0x10]\n" + "ldr x20, [x15, #0x68]\n" "fmla v31.8h, v2.8h, v9.8h\n" - "ldr q2, [x16, #0x70]\n" - "ldr x26, [x15, #0xc8]\n" - "fmla v28.8h, v3.8h, v5.8h\n" - "ldr q5, [x25, x10]\n" - "fmla v29.8h, v3.8h, v6.8h\n" - "ldr x25, [x15, #0xd0]\n" - "fmla v30.8h, v3.8h, v9.8h\n" - "fmla v31.8h, v3.8h, v13.8h\n" - "ldr q3, [x16, #0x80]\n" - "add x28, x28, #0x10\n" - "fmla v28.8h, v4.8h, v6.8h\n" - "ldr q6, [x24, x10]\n" + "ldr q16, [x21, x10]\n" + "fmla v5.8h, v2.8h, v11.8h\n" + "ldr x23, [x15, #0x70]\n" + "fmla v30.8h, v2.8h, v13.8h\n" + "fmla v29.8h, v2.8h, v22.8h\n" + "ldr q17, [x16, #0x20]\n" + "ldr x21, [x15, #0x78]\n" + "fmla v31.8h, v3.8h, v11.8h\n" + "ldr q6, [x20, x10]\n" + "fmla v5.8h, v3.8h, v12.8h\n" + "ldr x22, [x15, #0x80]\n" + "fmla v30.8h, v3.8h, v22.8h\n" + "fmla v29.8h, v3.8h, v21.8h\n" + "ldr q20, [x16, #0x30]\n" + "ldr x20, [x15, #0x88]\n" + "fmla v31.8h, v4.8h, v12.8h\n" + "ldr q2, [x23, x10]\n" + "fmla v5.8h, v4.8h, v16.8h\n" + "ldr q28, [x21, x10]\n" + "fmla v30.8h, v4.8h, v21.8h\n" "fmla v29.8h, v4.8h, v10.8h\n" - "ldr q10, [x23, x10]\n" - "fmla v30.8h, v4.8h, v13.8h\n" - "fmla v31.8h, v4.8h, v8.8h\n" - "ldr q4, [x16, #0x90]\n" - "ldr x24, [x15, #0xd8]\n" - "fmla v28.8h, v0.8h, v14.8h\n" - "ldr q14, [x20, x10]\n" - "fmla v29.8h, v0.8h, v11.8h\n" - "ldr x23, [x15, #0xe0]\n" - "fmla v30.8h, v0.8h, v5.8h\n" - "fmla v31.8h, v0.8h, v6.8h\n" - "ldr q0, [x16, #0xa0]\n" - "ldr x20, [x15, #0xf8]\n" - "fmla v28.8h, v1.8h, v11.8h\n" - "ldr q11, [x22, x10]\n" - "fmla v29.8h, v1.8h, v12.8h\n" - "ldr x22, [x15, #0xe8]\n" - "fmla v30.8h, v1.8h, v6.8h\n" - "fmla v31.8h, v1.8h, v10.8h\n" - "ldr q1, [x16, #0xb0]\n" - "fmla v28.8h, v2.8h, v12.8h\n" - "ldr q12, [x21, x10]\n" - "fmla v29.8h, v2.8h, v9.8h\n" - "ldr x21, [x15, #0xf0]\n" - "fmla v30.8h, v2.8h, v10.8h\n" - "fmla v31.8h, v2.8h, v11.8h\n" - "ldr q2, [x16, #0xc0]\n" - "fmla v28.8h, v3.8h, v9.8h\n" - "ldr q9, [x27, x10]\n" - "fmla v29.8h, v3.8h, v13.8h\n" - "ldr x27, [x15, #0x100]\n" - "fmla v30.8h, v3.8h, v11.8h\n" - "fmla v31.8h, v3.8h, v12.8h\n" + "ldr q16, [x16, #0x40]\n" + "ldr x21, [x15, #0x90]\n" + "fmla v31.8h, v19.8h, v7.8h\n" + "fmla v5.8h, v19.8h, v8.8h\n" + "ldr x27, [x15, #0x98]\n" + "ldr x26, [x15, #0xa0]\n" + "fmla v30.8h, v19.8h, v14.8h\n" + "fmla v29.8h, v19.8h, v6.8h\n" + "ldr q19, [x16, #0x50]\n" + "ldr x25, [x15, #0xa8]\n" + "fmla v31.8h, v18.8h, v8.8h\n" + "ldr q1, [x20, x10]\n" + "fmla v5.8h, v18.8h, v13.8h\n" + "ldr x24, [x15, #0xb0]\n" + "fmla v30.8h, v18.8h, v6.8h\n" + "fmla v29.8h, v18.8h, v2.8h\n" + "ldr q18, [x16, #0x60]\n" + "ldr x20, [x15, #0xb8]\n" + "fmla v31.8h, v17.8h, v13.8h\n" + "ldr q26, [x22, x10]\n" + "fmla v5.8h, v17.8h, v22.8h\n" + "ldr x23, [x15, #0xc0]\n" + "fmla v30.8h, v17.8h, v2.8h\n" + "fmla v29.8h, v17.8h, v28.8h\n" + "ldr q17, [x16, #0x70]\n" + "ldr x22, [x15, #0xc8]\n" + "fmla v31.8h, v20.8h, v22.8h\n" + "ldr q25, [x21, x10]\n" + "fmla v5.8h, v20.8h, v21.8h\n" + "ldr x21, [x15, #0xd0]\n" + "fmla v30.8h, v20.8h, v28.8h\n" + "fmla v29.8h, v20.8h, v26.8h\n" + "ldr q24, [x16, #0x80]\n" + "add x28, x28, #0x10\n" + "fmla v31.8h, v16.8h, v21.8h\n" + "ldr q23, [x27, x10]\n" + "fmla v5.8h, v16.8h, v10.8h\n" + "ldr q0, [x26, x10]\n" + "fmla v30.8h, v16.8h, v26.8h\n" + "fmla v29.8h, v16.8h, v1.8h\n" + "ldr q22, [x16, #0x90]\n" + "ldr x27, [x15, #0xd8]\n" + "fmla v31.8h, v19.8h, v14.8h\n" + "ldr q16, [x20, x10]\n" + "fmla v5.8h, v19.8h, v6.8h\n" + "ldr x20, [x15, #0xe0]\n" + "fmla v30.8h, v19.8h, v25.8h\n" + "fmla v29.8h, v19.8h, v23.8h\n" + "ldr q21, [x16, #0xa0]\n" + "ldr x26, [x15, #0xf8]\n" + "fmla v31.8h, v18.8h, v6.8h\n" + "ldr q20, [x25, x10]\n" + "fmla v5.8h, v18.8h, v2.8h\n" + "ldr x25, [x15, #0xe8]\n" + "fmla v30.8h, v18.8h, v23.8h\n" + "fmla v29.8h, v18.8h, v0.8h\n" + "ldr q18, [x16, #0xb0]\n" + "fmla v31.8h, v17.8h, v2.8h\n" + "ldr q19, [x24, x10]\n" + "fmla v5.8h, v17.8h, v28.8h\n" + "ldr x24, [x15, #0xf0]\n" + "fmla v30.8h, v17.8h, v0.8h\n" + "fmla v29.8h, v17.8h, v20.8h\n" + "ldr q17, [x16, #0xc0]\n" + "fmla v31.8h, v24.8h, v28.8h\n" + "ldr q7, [x23, x10]\n" + "fmla v5.8h, v24.8h, v26.8h\n" + "ldr x23, [x15, #0x100]\n" + "fmla v30.8h, v24.8h, v20.8h\n" + "fmla v29.8h, v24.8h, v19.8h\n" "ldr q3, [x16, #0xd0]\n" - "fmla v28.8h, v4.8h, v13.8h\n" - "ldr q13, [x26, x10]\n" - "fmla v29.8h, v4.8h, v8.8h\n" - "ldr q8, [x23, x10]\n" - "fmla v30.8h, v4.8h, v12.8h\n" - "fmla v31.8h, v4.8h, v14.8h\n" - "ldr q4, [x16, #0xe0]\n" - "ldr x26, [x15, #0x108]\n" - "fmla v28.8h, v0.8h, v5.8h\n" - "ldr q5, [x25, x10]\n" - "fmla v29.8h, v0.8h, v6.8h\n" - "ldr x25, [x15, #0x110]\n" - "fmla v30.8h, v0.8h, v9.8h\n" - "fmla v31.8h, v0.8h, v13.8h\n" - "ldr q0, [x16, #0xf0]\n" - "fmla v28.8h, v1.8h, v6.8h\n" - "ldr q6, [x24, x10]\n" - "fmla v29.8h, v1.8h, v10.8h\n" - "ldr x24, [x15, #0x118]\n" - "fmla v30.8h, v1.8h, v13.8h\n" - "fmla v31.8h, v1.8h, v5.8h\n" - "ldr q1, [x16, #0x100]\n" - "fmla v28.8h, v2.8h, v10.8h\n" - "ldr q10, [x22, x10]\n" - "fmla v29.8h, v2.8h, v11.8h\n" - "fmla v30.8h, v2.8h, v5.8h\n" - "fmla v31.8h, v2.8h, v6.8h\n" - "ldr q2, [x16, #0x110]\n" - "fmla v28.8h, v3.8h, v11.8h\n" - "ldr q11, [x21, x10]\n" - "fmla v29.8h, v3.8h, v12.8h\n" - "fmla v30.8h, v3.8h, v6.8h\n" - "fmla v31.8h, v3.8h, v8.8h\n" - "ldr q3, [x16, #0x120]\n" - "fmla v28.8h, v4.8h, v12.8h\n" - "ldr q12, [x20, x10]\n" - "fmla v29.8h, v4.8h, v14.8h\n" - "fmla v30.8h, v4.8h, v8.8h\n" - "fmla v31.8h, v4.8h, v10.8h\n" - "ldr q4, [x16, #0x130]\n" + "fmla v31.8h, v22.8h, v26.8h\n" + "ldr q28, [x22, x10]\n" + "fmla v5.8h, v22.8h, v1.8h\n" + "ldr q13, [x20, x10]\n" + "fmla v30.8h, v22.8h, v19.8h\n" + "fmla v29.8h, v22.8h, v16.8h\n" + "ldr q11, [x16, #0xe0]\n" + "ldr x22, [x15, #0x108]\n" + "fmla v31.8h, v21.8h, v25.8h\n" + "ldr q26, [x21, x10]\n" + "fmla v5.8h, v21.8h, v23.8h\n" + "ldr x21, [x15, #0x110]\n" + "fmla v30.8h, v21.8h, v7.8h\n" + "fmla v29.8h, v21.8h, v28.8h\n" + "ldr q25, [x16, #0xf0]\n" + "fmla v31.8h, v18.8h, v23.8h\n" + "ldr q24, [x27, x10]\n" + "fmla v5.8h, v18.8h, v0.8h\n" + "ldr x20, [x15, #0x118]\n" + "fmla v30.8h, v18.8h, v28.8h\n" + "fmla v29.8h, v18.8h, v26.8h\n" + "ldr q23, [x16, #0x100]\n" + "fmla v31.8h, v17.8h, v0.8h\n" + "ldr q22, [x25, x10]\n" + "fmla v5.8h, v17.8h, v20.8h\n" + "fmla v30.8h, v17.8h, v26.8h\n" + "fmla v29.8h, v17.8h, v24.8h\n" + "ldr q21, [x16, #0x110]\n" + "fmla v31.8h, v3.8h, v20.8h\n" + "ldr q18, [x24, x10]\n" + "fmla v5.8h, v3.8h, v19.8h\n" + "fmla v30.8h, v3.8h, v24.8h\n" + "fmla v29.8h, v3.8h, v13.8h\n" + "ldr q20, [x16, #0x120]\n" + "fmla v31.8h, v11.8h, v19.8h\n" + "ldr q17, [x26, x10]\n" + "fmla v5.8h, v11.8h, v16.8h\n" + "fmla v30.8h, v11.8h, v13.8h\n" + "fmla v29.8h, v11.8h, v22.8h\n" + "ldr q19, [x16, #0x130]\n" "add x16, x16, #0x140\n" - "fmla v28.8h, v0.8h, v9.8h\n" - "ldr q9, [x27, x10]\n" - "fmla v29.8h, v0.8h, v13.8h\n" - "fmla v30.8h, v0.8h, v11.8h\n" - "ldr q11, [x26, x10]\n" - "fmla v31.8h, v0.8h, v12.8h\n" - "fmla v28.8h, v1.8h, v13.8h\n" - "fmla v29.8h, v1.8h, v5.8h\n" - "fmla v30.8h, v1.8h, v12.8h\n" - "ldr q12, [x25, x10]\n" - "fmla v31.8h, v1.8h, v9.8h\n" - "fmla v28.8h, v2.8h, v5.8h\n" - "fmla v29.8h, v2.8h, v6.8h\n" - "fmla v30.8h, v2.8h, v9.8h\n" - "ldr q9, [x24, x10]\n" - "fmla v31.8h, v2.8h, v11.8h\n" + "fmla v31.8h, v25.8h, v7.8h\n" + "ldr q16, [x23, x10]\n" + "fmla v5.8h, v25.8h, v28.8h\n" + "fmla v30.8h, v25.8h, v18.8h\n" + "ldr q18, [x22, x10]\n" + "fmla v29.8h, v25.8h, v17.8h\n" + "fmla v31.8h, v23.8h, v28.8h\n" + "fmla v5.8h, v23.8h, v26.8h\n" + "fmla v30.8h, v23.8h, v17.8h\n" + "ldr q17, [x21, x10]\n" + "fmla v29.8h, v23.8h, v16.8h\n" + "fmla v31.8h, v21.8h, v26.8h\n" + "fmla v5.8h, v21.8h, v24.8h\n" + "fmla v30.8h, v21.8h, v16.8h\n" + "ldr q16, [x20, x10]\n" + "fmla v29.8h, v21.8h, v18.8h\n" "add x10, x10, #0x10\n" - "fmla v28.8h, v3.8h, v6.8h\n" - "fmla v29.8h, v3.8h, v8.8h\n" - "fmla v30.8h, v3.8h, v11.8h\n" - "fmla v31.8h, v3.8h, v12.8h\n" - "fmla v28.8h, v4.8h, v8.8h\n" - "fmla v29.8h, v4.8h, v10.8h\n" - "fmax v28.8h, v28.8h, v18.8h\n" - "fmla v30.8h, v4.8h, v12.8h\n" - "fmla v31.8h, v4.8h, v9.8h\n" - "fmax v29.8h, v29.8h, v18.8h\n" - "fmax v30.8h, v30.8h, v18.8h\n" - "fmax v31.8h, v31.8h, v18.8h\n" - "fmin v28.8h, v28.8h, v17.8h\n" - "fmin v29.8h, v29.8h, v17.8h\n" - "str q28, [x14, x28]\n" - "fmin v30.8h, v30.8h, v17.8h\n" - "fmin v31.8h, v31.8h, v17.8h\n" - "str q29, [x13, x28]\n" + "fmla v31.8h, v20.8h, v24.8h\n" + "fmla v5.8h, v20.8h, v13.8h\n" + "fmla v30.8h, v20.8h, v18.8h\n" + "fmla v29.8h, v20.8h, v17.8h\n" + "fmla v31.8h, v19.8h, v13.8h\n" + "fmla v5.8h, v19.8h, v22.8h\n" + "fmax v31.8h, v31.8h, v27.8h\n" + "fmla v30.8h, v19.8h, v17.8h\n" + "fmla v29.8h, v19.8h, v16.8h\n" + "fmax v5.8h, v5.8h, v27.8h\n" + "fmax v30.8h, v30.8h, v27.8h\n" + "fmax v29.8h, v29.8h, v27.8h\n" + "fmin v31.8h, v31.8h, v15.8h\n" + "fmin v5.8h, v5.8h, v15.8h\n" + "str q31, [x14, x28]\n" + "fmin v30.8h, v30.8h, v15.8h\n" + "fmin v29.8h, v29.8h, v15.8h\n" + "str q5, [x13, x28]\n" "str q30, [x12, x28]\n" - "str q31, [x11, x28]\n" + "str q29, [x11, x28]\n" "3:" // Oddments "tst %x[n_channels], #0x7\n" "beq 116f\n" - "ldr q16, [x16, #0x0]\n" + "ldr q26, [x16, #0x0]\n" "ldr q0, [x16, #0x10]\n" - "mov x28, x10\n" - "add x14, x14, x28\n" + "mov x20, x10\n" + "add x14, x14, x20\n" "ldr q1, [x16, #0x20]\n" "ldr q2, [x16, #0x30]\n" - "add x13, x13, x28\n" - "add x12, x12, x28\n" + "add x13, x13, x20\n" + "add x12, x12, x20\n" "ldr q3, [x16, #0x40]\n" "ldr q4, [x16, #0x50]\n" - "add x11, x11, x28\n" + "add x11, x11, x20\n" "ldr x9, [x15, #0x0]\n" "ldr x28, [x15, #0x8]\n" "add x9, x9, x10\n" @@ -654,12 +654,12 @@ void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl( "ld1 { v10.h }[0], [x21], #0x2\n" "ld1 { v14.h }[0], [x20], #0x2\n" "7:" // Oddments: Load inputs (0, 0), (0, 1), (1, 0), (1, 1), (0, 2), (1, 2), (0, 3), (0, 4), (1, 5), (2, 0): Bit 2: End - "mov v28.16b, v16.16b\n fmla v28.8h, v0.8h, v5.8h\n" - "mov v29.16b, v16.16b\n fmla v29.8h, v0.8h, v6.8h\n" + "mov v28.16b, v26.16b\n fmla v28.8h, v0.8h, v5.8h\n" + "mov v29.16b, v26.16b\n fmla v29.8h, v0.8h, v6.8h\n" "ldr x20, [x15, #0x50]\n" "add x20, x20, x10\n" - "mov v30.16b, v16.16b\n fmla v30.8h, v0.8h, v7.8h\n" - "mov v31.16b, v16.16b\n fmla v31.8h, v0.8h, v8.8h\n" + "mov v30.16b, v26.16b\n fmla v30.8h, v0.8h, v7.8h\n" + "mov v31.16b, v26.16b\n fmla v31.8h, v0.8h, v8.8h\n" "fmla v28.8h, v1.8h, v6.8h\n" "fmla v29.8h, v1.8h, v9.8h\n" "fmla v30.8h, v1.8h, v8.8h\n" @@ -1365,14 +1365,14 @@ void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl( "ld1 { v9.h }[0], [x20], #0x2\n" "111:" // Oddments: Load input (5, 5): Bit 2: End "fmla v31.8h, v4.8h, v9.8h\n" - "fmax v28.8h, v28.8h, v18.8h\n" - "fmax v29.8h, v29.8h, v18.8h\n" - "fmax v30.8h, v30.8h, v18.8h\n" - "fmax v31.8h, v31.8h, v18.8h\n" - "fmin v28.8h, v28.8h, v17.8h\n" - "fmin v29.8h, v29.8h, v17.8h\n" - "fmin v30.8h, v30.8h, v17.8h\n" - "fmin v31.8h, v31.8h, v17.8h\n" + "fmax v28.8h, v28.8h, v27.8h\n" + "fmax v29.8h, v29.8h, v27.8h\n" + "fmax v30.8h, v30.8h, v27.8h\n" + "fmax v31.8h, v31.8h, v27.8h\n" + "fmin v28.8h, v28.8h, v15.8h\n" + "fmin v29.8h, v29.8h, v15.8h\n" + "fmin v30.8h, v30.8h, v15.8h\n" + "fmin v31.8h, v31.8h, v15.8h\n" "tbz %x[n_channels], #2, 113f\n" "st1 { v28.d }[0], [x14], #0x8\n" "st1 { v29.d }[0], [x13], #0x8\n" @@ -1417,7 +1417,7 @@ void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl( "116:" // End : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v16", "v17", "v18", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_generic_output9_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_generic_output9_mla_depthfirst.hpp index 1ccd3408e269fc8ce0795f14e2c67e073e9bb703..b7608af7219a62ac064a0cd291bea1f2cb16d705 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_generic_output9_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_generic_output9_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include @@ -42,7 +42,7 @@ class a64_fp16_nhwc_generic_output9_mla_depthfirst : public GenericDepthfirstKer public: a64_fp16_nhwc_generic_output9_mla_depthfirst(const CPUInfo *) : GenericDepthfirstKernelStrategy<__fp16, __fp16, __fp16, __fp16>(9, arm_gemm::VLType::None) {} - virtual KernelType get_kernel() const override { return kernel; } + KernelType get_kernel() const override { return kernel; } }; } // namespace depthwise diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_generic_output9_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_generic_output9_mla_depthfirst/generic.cpp index 418530fdc4fbe14030acdf107d08f6d389363068..08f40b785fe3e1cae4d298692623e8703367bd3b 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_generic_output9_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_generic_output9_mla_depthfirst/generic.cpp @@ -45,70 +45,70 @@ void a64_fp16_nhwc_generic_output9_mla_depthfirst_impl( __asm__ __volatile__( "ld1r { v2.8h }, [%x[minmax_vals]]\n" - "lsr x12, %x[n_channels], #0x3\n" + "lsr x9, %x[n_channels], #0x3\n" "add x20, %x[minmax_vals], #0x2\n" "ld1r { v1.8h }, [x20]\n" "mov x11, #0x0\n" - "cbz x12, 5f\n" + "cbz x9, 5f\n" "1:" // Channel loop "movi v23.16b, #0x0\n" "cbz %x[bias], 2f\n" "ldr q23, [%x[bias], x11]\n" "2:" // Channel loop: Load bias: Done "ldr q0, [%x[params], #0x0]\n" - "mov x21, %x[inptrs]\n" - "ldp x10, x9, [x21], #0x10\n" - "subs x20, %x[n_points], #0x1\n" - "ldr q14, [x10, x11]\n" - "ldr q15, [x9, x11]\n" + "mov x26, %x[inptrs]\n" + "ldp x21, x20, [x26], #0x10\n" + "subs x25, %x[n_points], #0x1\n" + "ldr q14, [x21, x11]\n" + "ldr q15, [x20, x11]\n" "mov v24.16b, v23.16b\n" "mov v25.16b, v23.16b\n" - "ldp x28, x27, [x21], #0x10\n" - "ldr q16, [x28, x11]\n" + "ldp x21, x20, [x26], #0x10\n" + "ldr q16, [x21, x11]\n" "mov v26.16b, v23.16b\n" "mov v27.16b, v23.16b\n" - "ldr q17, [x27, x11]\n" - "ldp x26, x25, [x21], #0x10\n" + "ldr q17, [x20, x11]\n" + "ldp x21, x20, [x26], #0x10\n" "mov v28.16b, v23.16b\n" "mov v29.16b, v23.16b\n" - "ldr q18, [x26, x11]\n" - "ldr q19, [x25, x11]\n" + "ldr q18, [x21, x11]\n" + "ldr q19, [x20, x11]\n" "mov v30.16b, v23.16b\n" "mov v31.16b, v23.16b\n" - "ldp x24, x23, [x21], #0x10\n" - "ldr q20, [x24, x11]\n" + "ldp x21, x20, [x26], #0x10\n" + "ldr q20, [x21, x11]\n" "add %x[params], %x[params], #0x10\n" - "ldr q21, [x23, x11]\n" - "ldr x22, [x21], #0x8\n" - "ldr q22, [x22, x11]\n" + "ldr q21, [x20, x11]\n" + "ldr x20, [x26], #0x8\n" + "ldr q22, [x20, x11]\n" "ble 4f\n" "3:" // Channel loop: Planar loop - "ldp x10, x9, [x21], #0x10\n" - "ldp x28, x27, [x21], #0x10\n" - "subs x20, x20, #0x1\n" + "ldp x20, x24, [x26], #0x10\n" + "ldp x23, x22, [x26], #0x10\n" + "subs x25, x25, #0x1\n" "fmla v23.8h, v14.8h, v0.8h\n" - "ldr q14, [x10, x11]\n" - "ldp x26, x25, [x21], #0x10\n" + "ldr q14, [x20, x11]\n" + "ldp x21, x20, [x26], #0x10\n" "fmla v24.8h, v15.8h, v0.8h\n" "fmla v25.8h, v16.8h, v0.8h\n" - "ldr q15, [x9, x11]\n" - "ldr q16, [x28, x11]\n" + "ldr q15, [x24, x11]\n" + "ldr q16, [x23, x11]\n" "fmla v26.8h, v17.8h, v0.8h\n" "fmla v27.8h, v18.8h, v0.8h\n" - "ldr q17, [x27, x11]\n" - "ldr q18, [x26, x11]\n" + "ldr q17, [x22, x11]\n" + "ldr q18, [x21, x11]\n" "fmla v28.8h, v19.8h, v0.8h\n" "fmla v29.8h, v20.8h, v0.8h\n" - "ldr q19, [x25, x11]\n" - "ldp x24, x23, [x21], #0x10\n" + "ldr q19, [x20, x11]\n" + "ldp x21, x20, [x26], #0x10\n" "fmla v30.8h, v21.8h, v0.8h\n" "fmla v31.8h, v22.8h, v0.8h\n" "ldr q0, [%x[params], #0x0]\n" - "ldr q20, [x24, x11]\n" + "ldr q20, [x21, x11]\n" "add %x[params], %x[params], #0x10\n" - "ldr q21, [x23, x11]\n" - "ldr x22, [x21], #0x8\n" - "ldr q22, [x22, x11]\n" + "ldr q21, [x20, x11]\n" + "ldr x20, [x26], #0x8\n" + "ldr q22, [x20, x11]\n" "bgt 3b\n" "4:" // Channel loop: Planar tail "fmla v23.8h, v14.8h, v0.8h\n" @@ -153,7 +153,7 @@ void a64_fp16_nhwc_generic_output9_mla_depthfirst_impl( "str q30, [x21, x11]\n" "str q31, [x20, x11]\n" "add x11, x11, #0x10\n" - "cmp x11, x12, LSL #4\n" + "cmp x11, x9, LSL #4\n" "blt 1b\n" "5:" // Oddments "tst %x[n_channels], #0x7\n" @@ -183,209 +183,209 @@ void a64_fp16_nhwc_generic_output9_mla_depthfirst_impl( "9:" // Oddments: Load bias: Bit 2: End "10:" // Oddments: Load bias: Done "ldr q0, [%x[params], #0x0]\n" - "mov x21, %x[inptrs]\n" - "ldp x10, x9, [x21], #0x10\n" + "mov x10, %x[inptrs]\n" + "ldp x9, x28, [x10], #0x10\n" "mov v24.16b, v23.16b\n" - "ldp x28, x27, [x21], #0x10\n" - "ldp x26, x25, [x21], #0x10\n" + "ldp x27, x26, [x10], #0x10\n" + "ldp x25, x24, [x10], #0x10\n" "mov v25.16b, v23.16b\n" "mov v26.16b, v23.16b\n" - "ldp x24, x23, [x21], #0x10\n" - "ldr x22, [x21], #0x8\n" + "ldp x23, x22, [x10], #0x10\n" + "ldr x21, [x10], #0x8\n" "mov v27.16b, v23.16b\n" "mov v28.16b, v23.16b\n" "mov v29.16b, v23.16b\n" "mov v30.16b, v23.16b\n" - "add x10, x10, x11\n" "add x9, x9, x11\n" - "mov v31.16b, v23.16b\n" "add x28, x28, x11\n" + "mov v31.16b, v23.16b\n" "add x27, x27, x11\n" "add x26, x26, x11\n" "add x25, x25, x11\n" "add x24, x24, x11\n" "add x23, x23, x11\n" "add x22, x22, x11\n" + "add x21, x21, x11\n" "add %x[params], %x[params], #0x10\n" "tbz %x[n_channels], #2, 12f\n" - "ldr d14, [x10], #0x8\n" - "ldr d15, [x9], #0x8\n" - "ldr d16, [x28], #0x8\n" - "ldr d17, [x27], #0x8\n" - "ldr d18, [x26], #0x8\n" - "ldr d19, [x25], #0x8\n" - "ldr d20, [x24], #0x8\n" - "ldr d21, [x23], #0x8\n" - "ldr d22, [x22], #0x8\n" + "ldr d14, [x9], #0x8\n" + "ldr d15, [x28], #0x8\n" + "ldr d16, [x27], #0x8\n" + "ldr d17, [x26], #0x8\n" + "ldr d18, [x25], #0x8\n" + "ldr d19, [x24], #0x8\n" + "ldr d20, [x23], #0x8\n" + "ldr d21, [x22], #0x8\n" + "ldr d22, [x21], #0x8\n" "tbz %x[n_channels], #1, 11f\n" - "ld1 { v14.s }[2], [x10], #0x4\n" - "ld1 { v15.s }[2], [x9], #0x4\n" - "ld1 { v16.s }[2], [x28], #0x4\n" - "ld1 { v17.s }[2], [x27], #0x4\n" - "ld1 { v18.s }[2], [x26], #0x4\n" - "ld1 { v19.s }[2], [x25], #0x4\n" - "ld1 { v20.s }[2], [x24], #0x4\n" - "ld1 { v21.s }[2], [x23], #0x4\n" - "ld1 { v22.s }[2], [x22], #0x4\n" + "ld1 { v14.s }[2], [x9], #0x4\n" + "ld1 { v15.s }[2], [x28], #0x4\n" + "ld1 { v16.s }[2], [x27], #0x4\n" + "ld1 { v17.s }[2], [x26], #0x4\n" + "ld1 { v18.s }[2], [x25], #0x4\n" + "ld1 { v19.s }[2], [x24], #0x4\n" + "ld1 { v20.s }[2], [x23], #0x4\n" + "ld1 { v21.s }[2], [x22], #0x4\n" + "ld1 { v22.s }[2], [x21], #0x4\n" "tbz %x[n_channels], #0, 14f\n" - "ld1 { v14.h }[6], [x10], #0x2\n" - "ld1 { v15.h }[6], [x9], #0x2\n" - "ld1 { v16.h }[6], [x28], #0x2\n" - "ld1 { v17.h }[6], [x27], #0x2\n" - "ld1 { v18.h }[6], [x26], #0x2\n" - "ld1 { v19.h }[6], [x25], #0x2\n" - "ld1 { v20.h }[6], [x24], #0x2\n" - "ld1 { v21.h }[6], [x23], #0x2\n" - "ld1 { v22.h }[6], [x22], #0x2\n" + "ld1 { v14.h }[6], [x9], #0x2\n" + "ld1 { v15.h }[6], [x28], #0x2\n" + "ld1 { v16.h }[6], [x27], #0x2\n" + "ld1 { v17.h }[6], [x26], #0x2\n" + "ld1 { v18.h }[6], [x25], #0x2\n" + "ld1 { v19.h }[6], [x24], #0x2\n" + "ld1 { v20.h }[6], [x23], #0x2\n" + "ld1 { v21.h }[6], [x22], #0x2\n" + "ld1 { v22.h }[6], [x21], #0x2\n" "b 14f\n" "11:" // Oddments: Load: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 14f\n" - "ld1 { v14.h }[4], [x10], #0x2\n" - "ld1 { v15.h }[4], [x9], #0x2\n" - "ld1 { v16.h }[4], [x28], #0x2\n" - "ld1 { v17.h }[4], [x27], #0x2\n" - "ld1 { v18.h }[4], [x26], #0x2\n" - "ld1 { v19.h }[4], [x25], #0x2\n" - "ld1 { v20.h }[4], [x24], #0x2\n" - "ld1 { v21.h }[4], [x23], #0x2\n" - "ld1 { v22.h }[4], [x22], #0x2\n" + "ld1 { v14.h }[4], [x9], #0x2\n" + "ld1 { v15.h }[4], [x28], #0x2\n" + "ld1 { v16.h }[4], [x27], #0x2\n" + "ld1 { v17.h }[4], [x26], #0x2\n" + "ld1 { v18.h }[4], [x25], #0x2\n" + "ld1 { v19.h }[4], [x24], #0x2\n" + "ld1 { v20.h }[4], [x23], #0x2\n" + "ld1 { v21.h }[4], [x22], #0x2\n" + "ld1 { v22.h }[4], [x21], #0x2\n" "b 14f\n" "12:" // Oddments: Load: Bit 2: Unset "tbz %x[n_channels], #1, 13f\n" - "ldr s14, [x10], #0x4\n" - "ldr s15, [x9], #0x4\n" - "ldr s16, [x28], #0x4\n" - "ldr s17, [x27], #0x4\n" - "ldr s18, [x26], #0x4\n" - "ldr s19, [x25], #0x4\n" - "ldr s20, [x24], #0x4\n" - "ldr s21, [x23], #0x4\n" - "ldr s22, [x22], #0x4\n" + "ldr s14, [x9], #0x4\n" + "ldr s15, [x28], #0x4\n" + "ldr s16, [x27], #0x4\n" + "ldr s17, [x26], #0x4\n" + "ldr s18, [x25], #0x4\n" + "ldr s19, [x24], #0x4\n" + "ldr s20, [x23], #0x4\n" + "ldr s21, [x22], #0x4\n" + "ldr s22, [x21], #0x4\n" "tbz %x[n_channels], #0, 14f\n" - "ld1 { v14.h }[2], [x10], #0x2\n" - "ld1 { v15.h }[2], [x9], #0x2\n" - "ld1 { v16.h }[2], [x28], #0x2\n" - "ld1 { v17.h }[2], [x27], #0x2\n" - "ld1 { v18.h }[2], [x26], #0x2\n" - "ld1 { v19.h }[2], [x25], #0x2\n" - "ld1 { v20.h }[2], [x24], #0x2\n" - "ld1 { v21.h }[2], [x23], #0x2\n" - "ld1 { v22.h }[2], [x22], #0x2\n" + "ld1 { v14.h }[2], [x9], #0x2\n" + "ld1 { v15.h }[2], [x28], #0x2\n" + "ld1 { v16.h }[2], [x27], #0x2\n" + "ld1 { v17.h }[2], [x26], #0x2\n" + "ld1 { v18.h }[2], [x25], #0x2\n" + "ld1 { v19.h }[2], [x24], #0x2\n" + "ld1 { v20.h }[2], [x23], #0x2\n" + "ld1 { v21.h }[2], [x22], #0x2\n" + "ld1 { v22.h }[2], [x21], #0x2\n" "b 14f\n" "13:" // Oddments: Load: Bit 2: Unset: Bit 1: Unset - "ldr h14, [x10], #0x2\n" - "ldr h15, [x9], #0x2\n" - "ldr h16, [x28], #0x2\n" - "ldr h17, [x27], #0x2\n" - "ldr h18, [x26], #0x2\n" - "ldr h19, [x25], #0x2\n" - "ldr h20, [x24], #0x2\n" - "ldr h21, [x23], #0x2\n" - "ldr h22, [x22], #0x2\n" + "ldr h14, [x9], #0x2\n" + "ldr h15, [x28], #0x2\n" + "ldr h16, [x27], #0x2\n" + "ldr h17, [x26], #0x2\n" + "ldr h18, [x25], #0x2\n" + "ldr h19, [x24], #0x2\n" + "ldr h20, [x23], #0x2\n" + "ldr h21, [x22], #0x2\n" + "ldr h22, [x21], #0x2\n" "14:" // Oddments: Load: Bit 2: End "subs x20, %x[n_points], #0x1\n" "ble 20f\n" "15:" // Oddments: Planar loop - "ldp x10, x9, [x21], #0x10\n" - "ldp x28, x27, [x21], #0x10\n" + "ldp x9, x28, [x10], #0x10\n" + "ldp x27, x26, [x10], #0x10\n" "fmla v23.8h, v14.8h, v0.8h\n" "fmla v24.8h, v15.8h, v0.8h\n" - "ldp x26, x25, [x21], #0x10\n" - "ldp x24, x23, [x21], #0x10\n" + "ldp x25, x24, [x10], #0x10\n" + "ldp x23, x22, [x10], #0x10\n" "fmla v25.8h, v16.8h, v0.8h\n" "fmla v26.8h, v17.8h, v0.8h\n" - "ldr x22, [x21], #0x8\n" + "ldr x21, [x10], #0x8\n" "fmla v27.8h, v18.8h, v0.8h\n" "fmla v28.8h, v19.8h, v0.8h\n" - "add x10, x10, x11\n" + "add x9, x9, x11\n" "fmla v29.8h, v20.8h, v0.8h\n" "fmla v30.8h, v21.8h, v0.8h\n" - "add x9, x9, x11\n" "add x28, x28, x11\n" + "add x27, x27, x11\n" "fmla v31.8h, v22.8h, v0.8h\n" "ldr q0, [%x[params], #0x0]\n" - "add x27, x27, x11\n" "add x26, x26, x11\n" "add x25, x25, x11\n" "add x24, x24, x11\n" "add x23, x23, x11\n" "add x22, x22, x11\n" + "add x21, x21, x11\n" "add %x[params], %x[params], #0x10\n" "tbz %x[n_channels], #2, 17f\n" - "ldr d14, [x10], #0x8\n" - "ldr d15, [x9], #0x8\n" - "ldr d16, [x28], #0x8\n" - "ldr d17, [x27], #0x8\n" - "ldr d18, [x26], #0x8\n" - "ldr d19, [x25], #0x8\n" - "ldr d20, [x24], #0x8\n" - "ldr d21, [x23], #0x8\n" - "ldr d22, [x22], #0x8\n" + "ldr d14, [x9], #0x8\n" + "ldr d15, [x28], #0x8\n" + "ldr d16, [x27], #0x8\n" + "ldr d17, [x26], #0x8\n" + "ldr d18, [x25], #0x8\n" + "ldr d19, [x24], #0x8\n" + "ldr d20, [x23], #0x8\n" + "ldr d21, [x22], #0x8\n" + "ldr d22, [x21], #0x8\n" "tbz %x[n_channels], #1, 16f\n" - "ld1 { v14.s }[2], [x10], #0x4\n" - "ld1 { v15.s }[2], [x9], #0x4\n" - "ld1 { v16.s }[2], [x28], #0x4\n" - "ld1 { v17.s }[2], [x27], #0x4\n" - "ld1 { v18.s }[2], [x26], #0x4\n" - "ld1 { v19.s }[2], [x25], #0x4\n" - "ld1 { v20.s }[2], [x24], #0x4\n" - "ld1 { v21.s }[2], [x23], #0x4\n" - "ld1 { v22.s }[2], [x22], #0x4\n" + "ld1 { v14.s }[2], [x9], #0x4\n" + "ld1 { v15.s }[2], [x28], #0x4\n" + "ld1 { v16.s }[2], [x27], #0x4\n" + "ld1 { v17.s }[2], [x26], #0x4\n" + "ld1 { v18.s }[2], [x25], #0x4\n" + "ld1 { v19.s }[2], [x24], #0x4\n" + "ld1 { v20.s }[2], [x23], #0x4\n" + "ld1 { v21.s }[2], [x22], #0x4\n" + "ld1 { v22.s }[2], [x21], #0x4\n" "tbz %x[n_channels], #0, 19f\n" - "ld1 { v14.h }[6], [x10], #0x2\n" - "ld1 { v15.h }[6], [x9], #0x2\n" - "ld1 { v16.h }[6], [x28], #0x2\n" - "ld1 { v17.h }[6], [x27], #0x2\n" - "ld1 { v18.h }[6], [x26], #0x2\n" - "ld1 { v19.h }[6], [x25], #0x2\n" - "ld1 { v20.h }[6], [x24], #0x2\n" - "ld1 { v21.h }[6], [x23], #0x2\n" - "ld1 { v22.h }[6], [x22], #0x2\n" + "ld1 { v14.h }[6], [x9], #0x2\n" + "ld1 { v15.h }[6], [x28], #0x2\n" + "ld1 { v16.h }[6], [x27], #0x2\n" + "ld1 { v17.h }[6], [x26], #0x2\n" + "ld1 { v18.h }[6], [x25], #0x2\n" + "ld1 { v19.h }[6], [x24], #0x2\n" + "ld1 { v20.h }[6], [x23], #0x2\n" + "ld1 { v21.h }[6], [x22], #0x2\n" + "ld1 { v22.h }[6], [x21], #0x2\n" "b 19f\n" "16:" // Oddments: Planar loop: Load: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 19f\n" - "ld1 { v14.h }[4], [x10], #0x2\n" - "ld1 { v15.h }[4], [x9], #0x2\n" - "ld1 { v16.h }[4], [x28], #0x2\n" - "ld1 { v17.h }[4], [x27], #0x2\n" - "ld1 { v18.h }[4], [x26], #0x2\n" - "ld1 { v19.h }[4], [x25], #0x2\n" - "ld1 { v20.h }[4], [x24], #0x2\n" - "ld1 { v21.h }[4], [x23], #0x2\n" - "ld1 { v22.h }[4], [x22], #0x2\n" + "ld1 { v14.h }[4], [x9], #0x2\n" + "ld1 { v15.h }[4], [x28], #0x2\n" + "ld1 { v16.h }[4], [x27], #0x2\n" + "ld1 { v17.h }[4], [x26], #0x2\n" + "ld1 { v18.h }[4], [x25], #0x2\n" + "ld1 { v19.h }[4], [x24], #0x2\n" + "ld1 { v20.h }[4], [x23], #0x2\n" + "ld1 { v21.h }[4], [x22], #0x2\n" + "ld1 { v22.h }[4], [x21], #0x2\n" "b 19f\n" "17:" // Oddments: Planar loop: Load: Bit 2: Unset "tbz %x[n_channels], #1, 18f\n" - "ldr s14, [x10], #0x4\n" - "ldr s15, [x9], #0x4\n" - "ldr s16, [x28], #0x4\n" - "ldr s17, [x27], #0x4\n" - "ldr s18, [x26], #0x4\n" - "ldr s19, [x25], #0x4\n" - "ldr s20, [x24], #0x4\n" - "ldr s21, [x23], #0x4\n" - "ldr s22, [x22], #0x4\n" + "ldr s14, [x9], #0x4\n" + "ldr s15, [x28], #0x4\n" + "ldr s16, [x27], #0x4\n" + "ldr s17, [x26], #0x4\n" + "ldr s18, [x25], #0x4\n" + "ldr s19, [x24], #0x4\n" + "ldr s20, [x23], #0x4\n" + "ldr s21, [x22], #0x4\n" + "ldr s22, [x21], #0x4\n" "tbz %x[n_channels], #0, 19f\n" - "ld1 { v14.h }[2], [x10], #0x2\n" - "ld1 { v15.h }[2], [x9], #0x2\n" - "ld1 { v16.h }[2], [x28], #0x2\n" - "ld1 { v17.h }[2], [x27], #0x2\n" - "ld1 { v18.h }[2], [x26], #0x2\n" - "ld1 { v19.h }[2], [x25], #0x2\n" - "ld1 { v20.h }[2], [x24], #0x2\n" - "ld1 { v21.h }[2], [x23], #0x2\n" - "ld1 { v22.h }[2], [x22], #0x2\n" + "ld1 { v14.h }[2], [x9], #0x2\n" + "ld1 { v15.h }[2], [x28], #0x2\n" + "ld1 { v16.h }[2], [x27], #0x2\n" + "ld1 { v17.h }[2], [x26], #0x2\n" + "ld1 { v18.h }[2], [x25], #0x2\n" + "ld1 { v19.h }[2], [x24], #0x2\n" + "ld1 { v20.h }[2], [x23], #0x2\n" + "ld1 { v21.h }[2], [x22], #0x2\n" + "ld1 { v22.h }[2], [x21], #0x2\n" "b 19f\n" "18:" // Oddments: Planar loop: Load: Bit 2: Unset: Bit 1: Unset - "ldr h14, [x10], #0x2\n" - "ldr h15, [x9], #0x2\n" - "ldr h16, [x28], #0x2\n" - "ldr h17, [x27], #0x2\n" - "ldr h18, [x26], #0x2\n" - "ldr h19, [x25], #0x2\n" - "ldr h20, [x24], #0x2\n" - "ldr h21, [x23], #0x2\n" - "ldr h22, [x22], #0x2\n" + "ldr h14, [x9], #0x2\n" + "ldr h15, [x28], #0x2\n" + "ldr h16, [x27], #0x2\n" + "ldr h17, [x26], #0x2\n" + "ldr h18, [x25], #0x2\n" + "ldr h19, [x24], #0x2\n" + "ldr h20, [x23], #0x2\n" + "ldr h21, [x22], #0x2\n" + "ldr h22, [x21], #0x2\n" "19:" // Oddments: Planar loop: Load: Bit 2: End "subs x20, x20, #0x1\n" "bgt 15b\n" @@ -507,12 +507,10 @@ void a64_fp16_nhwc_generic_output9_mla_depthfirst_impl( "st1 { v30.h }[0], [x21], #0x2\n" "st1 { v31.h }[0], [x20], #0x2\n" "24:" // Oddments: Store: Bit 2: End - "25:" // End - : [params] "+&r" (params) : [bias] "r" (bias), [inptrs] "r" (inptrs), [minmax_vals] "r" (minmax_vals), [n_channels] "r" ((uint64_t) n_channels), [n_points] "r" ((uint64_t) n_points), [outptrs] "r" (outptrs) - : "cc", "memory", "v0", "v1", "v2", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp index 8fcbce2cfe8b141893fa6266a5f327db92ca88b8..3646c18b0471d377067e2c34bfe20d5a267cec62 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp index f246cec87e862169e253878fc393672aae224062..cee3fb59c5d98757e81c377b835c20dbc3aaaa7f 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp @@ -44,10 +44,10 @@ void a64_fp16_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_im const __fp16 minmax_vals[2] = { activation_min, activation_max }; __asm__ __volatile__( - "ld1r { v7.8h }, [%x[minmax_vals]]\n" + "ld1r { v8.8h }, [%x[minmax_vals]]\n" "lsr x11, %x[n_output_channels], #0x3\n" "add x20, %x[minmax_vals], #0x2\n" - "ld1r { v6.8h }, [x20]\n" + "ld1r { v7.8h }, [x20]\n" "mov x10, #0x0\n" "cbz x11, 8f\n" "1:" // Output channel loop @@ -56,12 +56,12 @@ void a64_fp16_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_im "lsl x20, x10, #0x1\n" "ldr q31, [%x[bias], x20]\n" "2:" // Output channel loop: Load bias: Done - "ldr q5, [%x[weights], #0x0]\n" - "mov x20, %x[inptrs]\n" - "ldp x24, x9, [x20], #0x10\n" - "lsr x21, %x[kernel_points], #0x1\n" - "ldr q4, [x24, #0x0]\n" - "ldr q3, [x9, #0x0]\n" + "ldr q6, [%x[weights], #0x0]\n" + "mov x22, %x[inptrs]\n" + "ldp x21, x20, [x22], #0x10\n" + "lsr x23, %x[kernel_points], #0x1\n" + "ldr q1, [x21, #0x0]\n" + "ldr q0, [x20, #0x0]\n" "mov v16.16b, v31.16b\n" "mov v17.16b, v31.16b\n" "mov v18.16b, v31.16b\n" @@ -79,26 +79,46 @@ void a64_fp16_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_im "mov v29.16b, v31.16b\n" "mov v30.16b, v31.16b\n" "mov v31.16b, v31.16b\n" - "cbz x21, 6f\n" - "ldr q2, [%x[weights], #0x0]\n" - "ldp x24, x9, [x20], #0x10\n" - "subs x21, x21, #0x1\n" + "cbz x23, 6f\n" + "ldr q5, [%x[weights], #0x0]\n" + "ldp x21, x20, [x22], #0x10\n" + "subs x23, x23, #0x1\n" "add %x[weights], %x[weights], #0x10\n" - "ldr q1, [x24, #0x0]\n" - "ldr q0, [x9, #0x0]\n" + "ldr q4, [x21, #0x0]\n" + "ldr q3, [x20, #0x0]\n" "beq 4f\n" "3:" // Output channel loop: Kernel loop - "ldp x24, x9, [x20], #0x10\n" + "ldp x21, x20, [x22], #0x10\n" + "fmla v16.8h, v6.8h, v1.h[0]\n" + "fmla v17.8h, v6.8h, v1.h[1]\n" + "subs x23, x23, #0x1\n" + "fmla v18.8h, v6.8h, v1.h[2]\n" + "fmla v19.8h, v6.8h, v1.h[3]\n" + "fmla v20.8h, v6.8h, v1.h[4]\n" + "fmla v21.8h, v6.8h, v1.h[5]\n" + "fmla v22.8h, v6.8h, v1.h[6]\n" + "fmla v23.8h, v6.8h, v1.h[7]\n" + "ldr q1, [x21, #0x0]\n" + "fmla v24.8h, v6.8h, v0.h[0]\n" + "fmla v25.8h, v6.8h, v0.h[1]\n" + "fmla v26.8h, v6.8h, v0.h[2]\n" + "fmla v27.8h, v6.8h, v0.h[3]\n" + "fmla v28.8h, v6.8h, v0.h[4]\n" + "fmla v29.8h, v6.8h, v0.h[5]\n" + "fmla v30.8h, v6.8h, v0.h[6]\n" + "fmla v31.8h, v6.8h, v0.h[7]\n" + "ldr q0, [x20, #0x0]\n" + "ldr q6, [%x[weights], #0x0]\n" + "ldp x21, x20, [x22], #0x10\n" "fmla v16.8h, v5.8h, v4.h[0]\n" "fmla v17.8h, v5.8h, v4.h[1]\n" - "subs x21, x21, #0x1\n" "fmla v18.8h, v5.8h, v4.h[2]\n" "fmla v19.8h, v5.8h, v4.h[3]\n" "fmla v20.8h, v5.8h, v4.h[4]\n" "fmla v21.8h, v5.8h, v4.h[5]\n" "fmla v22.8h, v5.8h, v4.h[6]\n" "fmla v23.8h, v5.8h, v4.h[7]\n" - "ldr q4, [x24, #0x0]\n" + "ldr q4, [x21, #0x0]\n" "fmla v24.8h, v5.8h, v3.h[0]\n" "fmla v25.8h, v5.8h, v3.h[1]\n" "fmla v26.8h, v5.8h, v3.h[2]\n" @@ -107,332 +127,312 @@ void a64_fp16_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_im "fmla v29.8h, v5.8h, v3.h[5]\n" "fmla v30.8h, v5.8h, v3.h[6]\n" "fmla v31.8h, v5.8h, v3.h[7]\n" - "ldr q3, [x9, #0x0]\n" - "ldr q5, [%x[weights], #0x0]\n" - "ldp x24, x9, [x20], #0x10\n" - "fmla v16.8h, v2.8h, v1.h[0]\n" - "fmla v17.8h, v2.8h, v1.h[1]\n" - "fmla v18.8h, v2.8h, v1.h[2]\n" - "fmla v19.8h, v2.8h, v1.h[3]\n" - "fmla v20.8h, v2.8h, v1.h[4]\n" - "fmla v21.8h, v2.8h, v1.h[5]\n" - "fmla v22.8h, v2.8h, v1.h[6]\n" - "fmla v23.8h, v2.8h, v1.h[7]\n" - "ldr q1, [x24, #0x0]\n" - "fmla v24.8h, v2.8h, v0.h[0]\n" - "fmla v25.8h, v2.8h, v0.h[1]\n" - "fmla v26.8h, v2.8h, v0.h[2]\n" - "fmla v27.8h, v2.8h, v0.h[3]\n" - "fmla v28.8h, v2.8h, v0.h[4]\n" - "fmla v29.8h, v2.8h, v0.h[5]\n" - "fmla v30.8h, v2.8h, v0.h[6]\n" - "fmla v31.8h, v2.8h, v0.h[7]\n" - "ldr q0, [x9, #0x0]\n" - "ldr q2, [%x[weights], #0x10]\n" + "ldr q3, [x20, #0x0]\n" + "ldr q5, [%x[weights], #0x10]\n" "add %x[weights], %x[weights], #0x20\n" "bgt 3b\n" "4:" // Output channel loop: Kernel loop tail "tbnz %x[kernel_points], #0, 5f\n" + "fmla v16.8h, v6.8h, v1.h[0]\n" + "fmla v17.8h, v6.8h, v1.h[1]\n" + "lsl x28, x10, #0x1\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "fmla v18.8h, v6.8h, v1.h[2]\n" + "fmla v19.8h, v6.8h, v1.h[3]\n" + "ldr x26, [%x[outptrs], #0x8]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "fmla v20.8h, v6.8h, v1.h[4]\n" + "fmla v21.8h, v6.8h, v1.h[5]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "fmla v22.8h, v6.8h, v1.h[6]\n" + "fmla v23.8h, v6.8h, v1.h[7]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "fmla v24.8h, v6.8h, v0.h[0]\n" + "fmla v25.8h, v6.8h, v0.h[1]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "fmla v26.8h, v6.8h, v0.h[2]\n" + "fmla v27.8h, v6.8h, v0.h[3]\n" + "fmla v28.8h, v6.8h, v0.h[4]\n" + "fmla v29.8h, v6.8h, v0.h[5]\n" + "fmla v30.8h, v6.8h, v0.h[6]\n" + "fmla v31.8h, v6.8h, v0.h[7]\n" "fmla v16.8h, v5.8h, v4.h[0]\n" "fmla v17.8h, v5.8h, v4.h[1]\n" - "lsl x28, x10, #0x1\n" - "ldr x20, [%x[outptrs], #0x0]\n" + "fmin v16.8h, v16.8h, v7.8h\n" "fmla v18.8h, v5.8h, v4.h[2]\n" "fmla v19.8h, v5.8h, v4.h[3]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "ldr x22, [%x[outptrs], #0x10]\n" + "fmin v17.8h, v17.8h, v7.8h\n" "fmla v20.8h, v5.8h, v4.h[4]\n" "fmla v21.8h, v5.8h, v4.h[5]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "ldr x24, [%x[outptrs], #0x20]\n" + "fmin v18.8h, v18.8h, v7.8h\n" "fmla v22.8h, v5.8h, v4.h[6]\n" "fmla v23.8h, v5.8h, v4.h[7]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "ldr x26, [%x[outptrs], #0x30]\n" + "fmin v19.8h, v19.8h, v7.8h\n" "fmla v24.8h, v5.8h, v3.h[0]\n" "fmla v25.8h, v5.8h, v3.h[1]\n" - "ldr x27, [%x[outptrs], #0x38]\n" + "fmin v20.8h, v20.8h, v7.8h\n" "fmla v26.8h, v5.8h, v3.h[2]\n" "fmla v27.8h, v5.8h, v3.h[3]\n" + "fmin v21.8h, v21.8h, v7.8h\n" "fmla v28.8h, v5.8h, v3.h[4]\n" "fmla v29.8h, v5.8h, v3.h[5]\n" + "fmin v22.8h, v22.8h, v7.8h\n" "fmla v30.8h, v5.8h, v3.h[6]\n" "fmla v31.8h, v5.8h, v3.h[7]\n" - "fmla v16.8h, v2.8h, v1.h[0]\n" - "fmla v17.8h, v2.8h, v1.h[1]\n" - "fmin v16.8h, v16.8h, v6.8h\n" - "fmla v18.8h, v2.8h, v1.h[2]\n" - "fmla v19.8h, v2.8h, v1.h[3]\n" - "fmin v17.8h, v17.8h, v6.8h\n" - "fmla v20.8h, v2.8h, v1.h[4]\n" - "fmla v21.8h, v2.8h, v1.h[5]\n" - "fmin v18.8h, v18.8h, v6.8h\n" - "fmla v22.8h, v2.8h, v1.h[6]\n" - "fmla v23.8h, v2.8h, v1.h[7]\n" - "fmin v19.8h, v19.8h, v6.8h\n" - "fmla v24.8h, v2.8h, v0.h[0]\n" - "fmla v25.8h, v2.8h, v0.h[1]\n" - "fmin v20.8h, v20.8h, v6.8h\n" - "fmla v26.8h, v2.8h, v0.h[2]\n" - "fmla v27.8h, v2.8h, v0.h[3]\n" - "fmin v21.8h, v21.8h, v6.8h\n" - "fmla v28.8h, v2.8h, v0.h[4]\n" - "fmla v29.8h, v2.8h, v0.h[5]\n" - "fmin v22.8h, v22.8h, v6.8h\n" - "fmla v30.8h, v2.8h, v0.h[6]\n" - "fmla v31.8h, v2.8h, v0.h[7]\n" - "fmin v23.8h, v23.8h, v6.8h\n" - "fmax v16.8h, v16.8h, v7.8h\n" - "fmax v17.8h, v17.8h, v7.8h\n" - "str q16, [x20, x28]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "fmax v18.8h, v18.8h, v7.8h\n" - "fmax v19.8h, v19.8h, v7.8h\n" - "str q17, [x21, x28]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "fmax v20.8h, v20.8h, v7.8h\n" - "fmax v21.8h, v21.8h, v7.8h\n" - "str q18, [x22, x28]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "fmax v22.8h, v22.8h, v7.8h\n" - "fmax v23.8h, v23.8h, v7.8h\n" - "str q19, [x23, x28]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "fmin v24.8h, v24.8h, v6.8h\n" - "fmin v25.8h, v25.8h, v6.8h\n" - "str q20, [x24, x28]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "fmin v26.8h, v26.8h, v6.8h\n" - "fmin v27.8h, v27.8h, v6.8h\n" - "str q21, [x25, x28]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "fmin v28.8h, v28.8h, v6.8h\n" - "fmin v29.8h, v29.8h, v6.8h\n" - "str q22, [x26, x28]\n" - "ldr x26, [%x[outptrs], #0x70]\n" - "fmin v30.8h, v30.8h, v6.8h\n" - "fmin v31.8h, v31.8h, v6.8h\n" - "str q23, [x27, x28]\n" - "ldr x27, [%x[outptrs], #0x78]\n" - "fmax v24.8h, v24.8h, v7.8h\n" - "fmax v25.8h, v25.8h, v7.8h\n" - "str q24, [x20, x28]\n" - "fmax v26.8h, v26.8h, v7.8h\n" - "fmax v27.8h, v27.8h, v7.8h\n" - "str q25, [x21, x28]\n" - "fmax v28.8h, v28.8h, v7.8h\n" - "fmax v29.8h, v29.8h, v7.8h\n" - "str q26, [x22, x28]\n" - "fmax v30.8h, v30.8h, v7.8h\n" - "fmax v31.8h, v31.8h, v7.8h\n" - "str q27, [x23, x28]\n" - "str q28, [x24, x28]\n" - "str q29, [x25, x28]\n" - "str q30, [x26, x28]\n" - "str q31, [x27, x28]\n" + "fmin v23.8h, v23.8h, v7.8h\n" + "fmax v16.8h, v16.8h, v8.8h\n" + "fmax v17.8h, v17.8h, v8.8h\n" + "str q16, [x27, x28]\n" + "ldr x27, [%x[outptrs], #0x40]\n" + "fmax v18.8h, v18.8h, v8.8h\n" + "fmax v19.8h, v19.8h, v8.8h\n" + "str q17, [x26, x28]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "fmax v20.8h, v20.8h, v8.8h\n" + "fmax v21.8h, v21.8h, v8.8h\n" + "str q18, [x25, x28]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "fmax v22.8h, v22.8h, v8.8h\n" + "fmax v23.8h, v23.8h, v8.8h\n" + "str q19, [x24, x28]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "fmin v24.8h, v24.8h, v7.8h\n" + "fmin v25.8h, v25.8h, v7.8h\n" + "str q20, [x23, x28]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "fmin v26.8h, v26.8h, v7.8h\n" + "fmin v27.8h, v27.8h, v7.8h\n" + "str q21, [x22, x28]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "fmin v28.8h, v28.8h, v7.8h\n" + "fmin v29.8h, v29.8h, v7.8h\n" + "str q22, [x21, x28]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "fmin v30.8h, v30.8h, v7.8h\n" + "fmin v31.8h, v31.8h, v7.8h\n" + "str q23, [x20, x28]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "fmax v24.8h, v24.8h, v8.8h\n" + "fmax v25.8h, v25.8h, v8.8h\n" + "str q24, [x27, x28]\n" + "fmax v26.8h, v26.8h, v8.8h\n" + "fmax v27.8h, v27.8h, v8.8h\n" + "str q25, [x26, x28]\n" + "fmax v28.8h, v28.8h, v8.8h\n" + "fmax v29.8h, v29.8h, v8.8h\n" + "str q26, [x25, x28]\n" + "fmax v30.8h, v30.8h, v8.8h\n" + "fmax v31.8h, v31.8h, v8.8h\n" + "str q27, [x24, x28]\n" + "str q28, [x23, x28]\n" + "str q29, [x22, x28]\n" + "str q30, [x21, x28]\n" + "str q31, [x20, x28]\n" "b 7f\n" "5:" // Output channel loop: Odd tail - "fmla v16.8h, v5.8h, v4.h[0]\n" - "fmla v17.8h, v5.8h, v4.h[1]\n" - "ldp x24, x9, [x20], #0x10\n" + "fmla v16.8h, v6.8h, v1.h[0]\n" + "fmla v17.8h, v6.8h, v1.h[1]\n" + "ldp x20, x9, [x22], #0x10\n" "lsl x28, x10, #0x1\n" - "fmla v18.8h, v5.8h, v4.h[2]\n" - "fmla v19.8h, v5.8h, v4.h[3]\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "fmla v20.8h, v5.8h, v4.h[4]\n" - "fmla v21.8h, v5.8h, v4.h[5]\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "fmla v22.8h, v5.8h, v4.h[6]\n" - "fmla v23.8h, v5.8h, v4.h[7]\n" - "ldr q4, [x24, #0x0]\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "fmla v24.8h, v5.8h, v3.h[0]\n" - "fmla v25.8h, v5.8h, v3.h[1]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "fmla v26.8h, v5.8h, v3.h[2]\n" - "fmla v27.8h, v5.8h, v3.h[3]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "fmla v28.8h, v5.8h, v3.h[4]\n" - "fmla v29.8h, v5.8h, v3.h[5]\n" - "fmla v30.8h, v5.8h, v3.h[6]\n" - "fmla v31.8h, v5.8h, v3.h[7]\n" - "ldr q5, [%x[weights], #0x0]\n" - "ldr q3, [x9, #0x0]\n" - "fmla v16.8h, v2.8h, v1.h[0]\n" - "fmla v17.8h, v2.8h, v1.h[1]\n" - "add %x[weights], %x[weights], #0x10\n" - "fmla v18.8h, v2.8h, v1.h[2]\n" - "fmla v19.8h, v2.8h, v1.h[3]\n" - "fmla v20.8h, v2.8h, v1.h[4]\n" - "fmla v21.8h, v2.8h, v1.h[5]\n" - "fmla v22.8h, v2.8h, v1.h[6]\n" - "fmla v23.8h, v2.8h, v1.h[7]\n" - "fmla v24.8h, v2.8h, v0.h[0]\n" - "fmla v25.8h, v2.8h, v0.h[1]\n" - "fmla v26.8h, v2.8h, v0.h[2]\n" - "fmla v27.8h, v2.8h, v0.h[3]\n" - "fmla v28.8h, v2.8h, v0.h[4]\n" - "fmla v29.8h, v2.8h, v0.h[5]\n" - "fmla v30.8h, v2.8h, v0.h[6]\n" - "fmla v31.8h, v2.8h, v0.h[7]\n" + "fmla v18.8h, v6.8h, v1.h[2]\n" + "fmla v19.8h, v6.8h, v1.h[3]\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" + "fmla v20.8h, v6.8h, v1.h[4]\n" + "fmla v21.8h, v6.8h, v1.h[5]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "fmla v22.8h, v6.8h, v1.h[6]\n" + "fmla v23.8h, v6.8h, v1.h[7]\n" + "ldr q2, [x20, #0x0]\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "fmla v24.8h, v6.8h, v0.h[0]\n" + "fmla v25.8h, v6.8h, v0.h[1]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "fmla v26.8h, v6.8h, v0.h[2]\n" + "fmla v27.8h, v6.8h, v0.h[3]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "fmla v28.8h, v6.8h, v0.h[4]\n" + "fmla v29.8h, v6.8h, v0.h[5]\n" + "fmla v30.8h, v6.8h, v0.h[6]\n" + "fmla v31.8h, v6.8h, v0.h[7]\n" + "ldr q1, [%x[weights], #0x0]\n" + "ldr q0, [x9, #0x0]\n" "fmla v16.8h, v5.8h, v4.h[0]\n" "fmla v17.8h, v5.8h, v4.h[1]\n" - "fmin v16.8h, v16.8h, v6.8h\n" + "add %x[weights], %x[weights], #0x10\n" "fmla v18.8h, v5.8h, v4.h[2]\n" "fmla v19.8h, v5.8h, v4.h[3]\n" - "fmin v17.8h, v17.8h, v6.8h\n" "fmla v20.8h, v5.8h, v4.h[4]\n" "fmla v21.8h, v5.8h, v4.h[5]\n" - "fmin v18.8h, v18.8h, v6.8h\n" "fmla v22.8h, v5.8h, v4.h[6]\n" "fmla v23.8h, v5.8h, v4.h[7]\n" - "fmin v19.8h, v19.8h, v6.8h\n" "fmla v24.8h, v5.8h, v3.h[0]\n" "fmla v25.8h, v5.8h, v3.h[1]\n" - "fmin v20.8h, v20.8h, v6.8h\n" "fmla v26.8h, v5.8h, v3.h[2]\n" "fmla v27.8h, v5.8h, v3.h[3]\n" - "fmin v21.8h, v21.8h, v6.8h\n" "fmla v28.8h, v5.8h, v3.h[4]\n" "fmla v29.8h, v5.8h, v3.h[5]\n" - "fmin v22.8h, v22.8h, v6.8h\n" "fmla v30.8h, v5.8h, v3.h[6]\n" "fmla v31.8h, v5.8h, v3.h[7]\n" - "fmin v23.8h, v23.8h, v6.8h\n" - "fmax v16.8h, v16.8h, v7.8h\n" - "fmax v17.8h, v17.8h, v7.8h\n" - "str q16, [x20, x28]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "fmax v18.8h, v18.8h, v7.8h\n" - "fmax v19.8h, v19.8h, v7.8h\n" - "str q17, [x21, x28]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "fmax v20.8h, v20.8h, v7.8h\n" - "fmax v21.8h, v21.8h, v7.8h\n" - "str q18, [x22, x28]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "fmax v22.8h, v22.8h, v7.8h\n" - "fmax v23.8h, v23.8h, v7.8h\n" - "str q19, [x23, x28]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "fmin v24.8h, v24.8h, v6.8h\n" - "fmin v25.8h, v25.8h, v6.8h\n" - "str q20, [x24, x28]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "fmin v26.8h, v26.8h, v6.8h\n" - "fmin v27.8h, v27.8h, v6.8h\n" - "str q21, [x25, x28]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "fmin v28.8h, v28.8h, v6.8h\n" - "fmin v29.8h, v29.8h, v6.8h\n" - "str q22, [x26, x28]\n" - "ldr x26, [%x[outptrs], #0x70]\n" - "fmin v30.8h, v30.8h, v6.8h\n" - "fmin v31.8h, v31.8h, v6.8h\n" - "str q23, [x27, x28]\n" - "ldr x27, [%x[outptrs], #0x78]\n" - "fmax v24.8h, v24.8h, v7.8h\n" - "fmax v25.8h, v25.8h, v7.8h\n" - "str q24, [x20, x28]\n" - "fmax v26.8h, v26.8h, v7.8h\n" - "fmax v27.8h, v27.8h, v7.8h\n" - "str q25, [x21, x28]\n" - "fmax v28.8h, v28.8h, v7.8h\n" - "fmax v29.8h, v29.8h, v7.8h\n" - "str q26, [x22, x28]\n" - "fmax v30.8h, v30.8h, v7.8h\n" - "fmax v31.8h, v31.8h, v7.8h\n" - "str q27, [x23, x28]\n" - "str q28, [x24, x28]\n" - "str q29, [x25, x28]\n" - "str q30, [x26, x28]\n" - "str q31, [x27, x28]\n" + "fmla v16.8h, v1.8h, v2.h[0]\n" + "fmla v17.8h, v1.8h, v2.h[1]\n" + "fmin v16.8h, v16.8h, v7.8h\n" + "fmla v18.8h, v1.8h, v2.h[2]\n" + "fmla v19.8h, v1.8h, v2.h[3]\n" + "fmin v17.8h, v17.8h, v7.8h\n" + "fmla v20.8h, v1.8h, v2.h[4]\n" + "fmla v21.8h, v1.8h, v2.h[5]\n" + "fmin v18.8h, v18.8h, v7.8h\n" + "fmla v22.8h, v1.8h, v2.h[6]\n" + "fmla v23.8h, v1.8h, v2.h[7]\n" + "fmin v19.8h, v19.8h, v7.8h\n" + "fmla v24.8h, v1.8h, v0.h[0]\n" + "fmla v25.8h, v1.8h, v0.h[1]\n" + "fmin v20.8h, v20.8h, v7.8h\n" + "fmla v26.8h, v1.8h, v0.h[2]\n" + "fmla v27.8h, v1.8h, v0.h[3]\n" + "fmin v21.8h, v21.8h, v7.8h\n" + "fmla v28.8h, v1.8h, v0.h[4]\n" + "fmla v29.8h, v1.8h, v0.h[5]\n" + "fmin v22.8h, v22.8h, v7.8h\n" + "fmla v30.8h, v1.8h, v0.h[6]\n" + "fmla v31.8h, v1.8h, v0.h[7]\n" + "fmin v23.8h, v23.8h, v7.8h\n" + "fmax v16.8h, v16.8h, v8.8h\n" + "fmax v17.8h, v17.8h, v8.8h\n" + "str q16, [x27, x28]\n" + "ldr x27, [%x[outptrs], #0x40]\n" + "fmax v18.8h, v18.8h, v8.8h\n" + "fmax v19.8h, v19.8h, v8.8h\n" + "str q17, [x26, x28]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "fmax v20.8h, v20.8h, v8.8h\n" + "fmax v21.8h, v21.8h, v8.8h\n" + "str q18, [x25, x28]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "fmax v22.8h, v22.8h, v8.8h\n" + "fmax v23.8h, v23.8h, v8.8h\n" + "str q19, [x24, x28]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "fmin v24.8h, v24.8h, v7.8h\n" + "fmin v25.8h, v25.8h, v7.8h\n" + "str q20, [x23, x28]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "fmin v26.8h, v26.8h, v7.8h\n" + "fmin v27.8h, v27.8h, v7.8h\n" + "str q21, [x22, x28]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "fmin v28.8h, v28.8h, v7.8h\n" + "fmin v29.8h, v29.8h, v7.8h\n" + "str q22, [x21, x28]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "fmin v30.8h, v30.8h, v7.8h\n" + "fmin v31.8h, v31.8h, v7.8h\n" + "str q23, [x20, x28]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "fmax v24.8h, v24.8h, v8.8h\n" + "fmax v25.8h, v25.8h, v8.8h\n" + "str q24, [x27, x28]\n" + "fmax v26.8h, v26.8h, v8.8h\n" + "fmax v27.8h, v27.8h, v8.8h\n" + "str q25, [x26, x28]\n" + "fmax v28.8h, v28.8h, v8.8h\n" + "fmax v29.8h, v29.8h, v8.8h\n" + "str q26, [x25, x28]\n" + "fmax v30.8h, v30.8h, v8.8h\n" + "fmax v31.8h, v31.8h, v8.8h\n" + "str q27, [x24, x28]\n" + "str q28, [x23, x28]\n" + "str q29, [x22, x28]\n" + "str q30, [x21, x28]\n" + "str q31, [x20, x28]\n" "b 7f\n" "6:" // Output channel loop: Single kernel point - "fmla v16.8h, v5.8h, v4.h[0]\n" - "fmla v17.8h, v5.8h, v4.h[1]\n" - "fmin v16.8h, v16.8h, v6.8h\n" + "fmla v16.8h, v6.8h, v1.h[0]\n" + "fmla v17.8h, v6.8h, v1.h[1]\n" + "fmin v16.8h, v16.8h, v7.8h\n" "lsl x28, x10, #0x1\n" - "fmla v18.8h, v5.8h, v4.h[2]\n" - "fmla v19.8h, v5.8h, v4.h[3]\n" - "fmin v17.8h, v17.8h, v6.8h\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "fmla v20.8h, v5.8h, v4.h[4]\n" - "fmla v21.8h, v5.8h, v4.h[5]\n" - "fmin v18.8h, v18.8h, v6.8h\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "fmla v22.8h, v5.8h, v4.h[6]\n" - "fmla v23.8h, v5.8h, v4.h[7]\n" - "fmin v19.8h, v19.8h, v6.8h\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "fmla v24.8h, v5.8h, v3.h[0]\n" - "fmla v25.8h, v5.8h, v3.h[1]\n" - "fmin v20.8h, v20.8h, v6.8h\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "fmla v26.8h, v5.8h, v3.h[2]\n" - "fmla v27.8h, v5.8h, v3.h[3]\n" - "fmin v21.8h, v21.8h, v6.8h\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "fmla v28.8h, v5.8h, v3.h[4]\n" - "fmla v29.8h, v5.8h, v3.h[5]\n" - "fmin v22.8h, v22.8h, v6.8h\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "fmla v30.8h, v5.8h, v3.h[6]\n" - "fmla v31.8h, v5.8h, v3.h[7]\n" - "fmin v23.8h, v23.8h, v6.8h\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "fmax v16.8h, v16.8h, v7.8h\n" - "fmax v17.8h, v17.8h, v7.8h\n" - "str q16, [x20, x28]\n" - "fmax v18.8h, v18.8h, v7.8h\n" - "fmax v19.8h, v19.8h, v7.8h\n" - "str q17, [x21, x28]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "fmax v20.8h, v20.8h, v7.8h\n" - "fmax v21.8h, v21.8h, v7.8h\n" - "str q18, [x22, x28]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "fmax v22.8h, v22.8h, v7.8h\n" - "fmax v23.8h, v23.8h, v7.8h\n" - "str q19, [x23, x28]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "fmin v24.8h, v24.8h, v6.8h\n" - "fmin v25.8h, v25.8h, v6.8h\n" - "str q20, [x24, x28]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "fmin v26.8h, v26.8h, v6.8h\n" - "fmin v27.8h, v27.8h, v6.8h\n" - "str q21, [x25, x28]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "fmin v28.8h, v28.8h, v6.8h\n" - "fmin v29.8h, v29.8h, v6.8h\n" - "str q22, [x26, x28]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "fmin v30.8h, v30.8h, v6.8h\n" - "fmin v31.8h, v31.8h, v6.8h\n" - "str q23, [x27, x28]\n" - "ldr x26, [%x[outptrs], #0x70]\n" - "ldr x27, [%x[outptrs], #0x78]\n" - "fmax v24.8h, v24.8h, v7.8h\n" - "fmax v25.8h, v25.8h, v7.8h\n" - "str q24, [x20, x28]\n" - "fmax v26.8h, v26.8h, v7.8h\n" - "fmax v27.8h, v27.8h, v7.8h\n" - "str q25, [x21, x28]\n" - "fmax v28.8h, v28.8h, v7.8h\n" - "fmax v29.8h, v29.8h, v7.8h\n" - "str q26, [x22, x28]\n" - "fmax v30.8h, v30.8h, v7.8h\n" - "fmax v31.8h, v31.8h, v7.8h\n" - "str q27, [x23, x28]\n" - "str q28, [x24, x28]\n" - "str q29, [x25, x28]\n" - "str q30, [x26, x28]\n" - "str q31, [x27, x28]\n" + "fmla v18.8h, v6.8h, v1.h[2]\n" + "fmla v19.8h, v6.8h, v1.h[3]\n" + "fmin v17.8h, v17.8h, v7.8h\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "fmla v20.8h, v6.8h, v1.h[4]\n" + "fmla v21.8h, v6.8h, v1.h[5]\n" + "fmin v18.8h, v18.8h, v7.8h\n" + "ldr x26, [%x[outptrs], #0x8]\n" + "fmla v22.8h, v6.8h, v1.h[6]\n" + "fmla v23.8h, v6.8h, v1.h[7]\n" + "fmin v19.8h, v19.8h, v7.8h\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "fmla v24.8h, v6.8h, v0.h[0]\n" + "fmla v25.8h, v6.8h, v0.h[1]\n" + "fmin v20.8h, v20.8h, v7.8h\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "fmla v26.8h, v6.8h, v0.h[2]\n" + "fmla v27.8h, v6.8h, v0.h[3]\n" + "fmin v21.8h, v21.8h, v7.8h\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "fmla v28.8h, v6.8h, v0.h[4]\n" + "fmla v29.8h, v6.8h, v0.h[5]\n" + "fmin v22.8h, v22.8h, v7.8h\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "fmla v30.8h, v6.8h, v0.h[6]\n" + "fmla v31.8h, v6.8h, v0.h[7]\n" + "fmin v23.8h, v23.8h, v7.8h\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "fmax v16.8h, v16.8h, v8.8h\n" + "fmax v17.8h, v17.8h, v8.8h\n" + "str q16, [x27, x28]\n" + "fmax v18.8h, v18.8h, v8.8h\n" + "fmax v19.8h, v19.8h, v8.8h\n" + "str q17, [x26, x28]\n" + "ldr x27, [%x[outptrs], #0x40]\n" + "fmax v20.8h, v20.8h, v8.8h\n" + "fmax v21.8h, v21.8h, v8.8h\n" + "str q18, [x25, x28]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "fmax v22.8h, v22.8h, v8.8h\n" + "fmax v23.8h, v23.8h, v8.8h\n" + "str q19, [x24, x28]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "fmin v24.8h, v24.8h, v7.8h\n" + "fmin v25.8h, v25.8h, v7.8h\n" + "str q20, [x23, x28]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "fmin v26.8h, v26.8h, v7.8h\n" + "fmin v27.8h, v27.8h, v7.8h\n" + "str q21, [x22, x28]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "fmin v28.8h, v28.8h, v7.8h\n" + "fmin v29.8h, v29.8h, v7.8h\n" + "str q22, [x21, x28]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "fmin v30.8h, v30.8h, v7.8h\n" + "fmin v31.8h, v31.8h, v7.8h\n" + "str q23, [x20, x28]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "fmax v24.8h, v24.8h, v8.8h\n" + "fmax v25.8h, v25.8h, v8.8h\n" + "str q24, [x27, x28]\n" + "fmax v26.8h, v26.8h, v8.8h\n" + "fmax v27.8h, v27.8h, v8.8h\n" + "str q25, [x26, x28]\n" + "fmax v28.8h, v28.8h, v8.8h\n" + "fmax v29.8h, v29.8h, v8.8h\n" + "str q26, [x25, x28]\n" + "fmax v30.8h, v30.8h, v8.8h\n" + "fmax v31.8h, v31.8h, v8.8h\n" + "str q27, [x24, x28]\n" + "str q28, [x23, x28]\n" + "str q29, [x22, x28]\n" + "str q30, [x21, x28]\n" + "str q31, [x20, x28]\n" "7:" // Output channel loop: Done "add x10, x10, #0x8\n" "cmp x10, x11, LSL #3\n" @@ -464,12 +464,12 @@ void a64_fp16_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_im "ld1 { v31.h }[0], [x20]\n" "12:" // Output channel oddments: Load bias: Bit 2: End "13:" // Output channel oddments: Load bias: Done - "ldr q5, [%x[weights], #0x0]\n" - "mov x20, %x[inptrs]\n" - "ldp x24, x9, [x20], #0x10\n" - "lsr x21, %x[kernel_points], #0x1\n" - "ldr q4, [x24, #0x0]\n" - "ldr q3, [x9, #0x0]\n" + "ldr q6, [%x[weights], #0x0]\n" + "mov x22, %x[inptrs]\n" + "ldp x21, x20, [x22], #0x10\n" + "lsr x23, %x[kernel_points], #0x1\n" + "ldr q1, [x21, #0x0]\n" + "ldr q0, [x20, #0x0]\n" "mov v16.16b, v31.16b\n" "mov v17.16b, v31.16b\n" "mov v18.16b, v31.16b\n" @@ -487,26 +487,46 @@ void a64_fp16_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_im "mov v29.16b, v31.16b\n" "mov v30.16b, v31.16b\n" "mov v31.16b, v31.16b\n" - "cbz x21, 17f\n" - "ldr q2, [%x[weights], #0x0]\n" - "ldp x24, x9, [x20], #0x10\n" - "subs x21, x21, #0x1\n" + "cbz x23, 17f\n" + "ldr q5, [%x[weights], #0x0]\n" + "ldp x21, x20, [x22], #0x10\n" + "subs x23, x23, #0x1\n" "add %x[weights], %x[weights], #0x10\n" - "ldr q1, [x24, #0x0]\n" - "ldr q0, [x9, #0x0]\n" + "ldr q4, [x21, #0x0]\n" + "ldr q3, [x20, #0x0]\n" "beq 15f\n" "14:" // Output channel oddments: Kernel loop - "ldp x24, x9, [x20], #0x10\n" + "ldp x21, x20, [x22], #0x10\n" + "fmla v16.8h, v6.8h, v1.h[0]\n" + "fmla v17.8h, v6.8h, v1.h[1]\n" + "subs x23, x23, #0x1\n" + "fmla v18.8h, v6.8h, v1.h[2]\n" + "fmla v19.8h, v6.8h, v1.h[3]\n" + "fmla v20.8h, v6.8h, v1.h[4]\n" + "fmla v21.8h, v6.8h, v1.h[5]\n" + "fmla v22.8h, v6.8h, v1.h[6]\n" + "fmla v23.8h, v6.8h, v1.h[7]\n" + "ldr q1, [x21, #0x0]\n" + "fmla v24.8h, v6.8h, v0.h[0]\n" + "fmla v25.8h, v6.8h, v0.h[1]\n" + "fmla v26.8h, v6.8h, v0.h[2]\n" + "fmla v27.8h, v6.8h, v0.h[3]\n" + "fmla v28.8h, v6.8h, v0.h[4]\n" + "fmla v29.8h, v6.8h, v0.h[5]\n" + "fmla v30.8h, v6.8h, v0.h[6]\n" + "fmla v31.8h, v6.8h, v0.h[7]\n" + "ldr q0, [x20, #0x0]\n" + "ldr q6, [%x[weights], #0x0]\n" + "ldp x21, x20, [x22], #0x10\n" "fmla v16.8h, v5.8h, v4.h[0]\n" "fmla v17.8h, v5.8h, v4.h[1]\n" - "subs x21, x21, #0x1\n" "fmla v18.8h, v5.8h, v4.h[2]\n" "fmla v19.8h, v5.8h, v4.h[3]\n" "fmla v20.8h, v5.8h, v4.h[4]\n" "fmla v21.8h, v5.8h, v4.h[5]\n" "fmla v22.8h, v5.8h, v4.h[6]\n" "fmla v23.8h, v5.8h, v4.h[7]\n" - "ldr q4, [x24, #0x0]\n" + "ldr q4, [x21, #0x0]\n" "fmla v24.8h, v5.8h, v3.h[0]\n" "fmla v25.8h, v5.8h, v3.h[1]\n" "fmla v26.8h, v5.8h, v3.h[2]\n" @@ -515,32 +535,28 @@ void a64_fp16_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_im "fmla v29.8h, v5.8h, v3.h[5]\n" "fmla v30.8h, v5.8h, v3.h[6]\n" "fmla v31.8h, v5.8h, v3.h[7]\n" - "ldr q3, [x9, #0x0]\n" - "ldr q5, [%x[weights], #0x0]\n" - "ldp x24, x9, [x20], #0x10\n" - "fmla v16.8h, v2.8h, v1.h[0]\n" - "fmla v17.8h, v2.8h, v1.h[1]\n" - "fmla v18.8h, v2.8h, v1.h[2]\n" - "fmla v19.8h, v2.8h, v1.h[3]\n" - "fmla v20.8h, v2.8h, v1.h[4]\n" - "fmla v21.8h, v2.8h, v1.h[5]\n" - "fmla v22.8h, v2.8h, v1.h[6]\n" - "fmla v23.8h, v2.8h, v1.h[7]\n" - "ldr q1, [x24, #0x0]\n" - "fmla v24.8h, v2.8h, v0.h[0]\n" - "fmla v25.8h, v2.8h, v0.h[1]\n" - "fmla v26.8h, v2.8h, v0.h[2]\n" - "fmla v27.8h, v2.8h, v0.h[3]\n" - "fmla v28.8h, v2.8h, v0.h[4]\n" - "fmla v29.8h, v2.8h, v0.h[5]\n" - "fmla v30.8h, v2.8h, v0.h[6]\n" - "fmla v31.8h, v2.8h, v0.h[7]\n" - "ldr q0, [x9, #0x0]\n" - "ldr q2, [%x[weights], #0x10]\n" + "ldr q3, [x20, #0x0]\n" + "ldr q5, [%x[weights], #0x10]\n" "add %x[weights], %x[weights], #0x20\n" "bgt 14b\n" "15:" // Output channel oddments: Kernel loop tail "tbnz %x[kernel_points], #0, 16f\n" + "fmla v16.8h, v6.8h, v1.h[0]\n" + "fmla v17.8h, v6.8h, v1.h[1]\n" + "fmla v18.8h, v6.8h, v1.h[2]\n" + "fmla v19.8h, v6.8h, v1.h[3]\n" + "fmla v20.8h, v6.8h, v1.h[4]\n" + "fmla v21.8h, v6.8h, v1.h[5]\n" + "fmla v22.8h, v6.8h, v1.h[6]\n" + "fmla v23.8h, v6.8h, v1.h[7]\n" + "fmla v24.8h, v6.8h, v0.h[0]\n" + "fmla v25.8h, v6.8h, v0.h[1]\n" + "fmla v26.8h, v6.8h, v0.h[2]\n" + "fmla v27.8h, v6.8h, v0.h[3]\n" + "fmla v28.8h, v6.8h, v0.h[4]\n" + "fmla v29.8h, v6.8h, v0.h[5]\n" + "fmla v30.8h, v6.8h, v0.h[6]\n" + "fmla v31.8h, v6.8h, v0.h[7]\n" "fmla v16.8h, v5.8h, v4.h[0]\n" "fmla v17.8h, v5.8h, v4.h[1]\n" "fmla v18.8h, v5.8h, v4.h[2]\n" @@ -557,63 +573,31 @@ void a64_fp16_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_im "fmla v29.8h, v5.8h, v3.h[5]\n" "fmla v30.8h, v5.8h, v3.h[6]\n" "fmla v31.8h, v5.8h, v3.h[7]\n" - "fmla v16.8h, v2.8h, v1.h[0]\n" - "fmla v17.8h, v2.8h, v1.h[1]\n" - "fmla v18.8h, v2.8h, v1.h[2]\n" - "fmla v19.8h, v2.8h, v1.h[3]\n" - "fmla v20.8h, v2.8h, v1.h[4]\n" - "fmla v21.8h, v2.8h, v1.h[5]\n" - "fmla v22.8h, v2.8h, v1.h[6]\n" - "fmla v23.8h, v2.8h, v1.h[7]\n" - "fmla v24.8h, v2.8h, v0.h[0]\n" - "fmla v25.8h, v2.8h, v0.h[1]\n" - "fmla v26.8h, v2.8h, v0.h[2]\n" - "fmla v27.8h, v2.8h, v0.h[3]\n" - "fmla v28.8h, v2.8h, v0.h[4]\n" - "fmla v29.8h, v2.8h, v0.h[5]\n" - "fmla v30.8h, v2.8h, v0.h[6]\n" - "fmla v31.8h, v2.8h, v0.h[7]\n" "b 18f\n" "16:" // Output channel oddments: Odd tail + "fmla v16.8h, v6.8h, v1.h[0]\n" + "fmla v17.8h, v6.8h, v1.h[1]\n" + "ldp x21, x20, [x22], #0x10\n" + "fmla v18.8h, v6.8h, v1.h[2]\n" + "fmla v19.8h, v6.8h, v1.h[3]\n" + "fmla v20.8h, v6.8h, v1.h[4]\n" + "fmla v21.8h, v6.8h, v1.h[5]\n" + "fmla v22.8h, v6.8h, v1.h[6]\n" + "fmla v23.8h, v6.8h, v1.h[7]\n" + "ldr q2, [x21, #0x0]\n" + "fmla v24.8h, v6.8h, v0.h[0]\n" + "fmla v25.8h, v6.8h, v0.h[1]\n" + "fmla v26.8h, v6.8h, v0.h[2]\n" + "fmla v27.8h, v6.8h, v0.h[3]\n" + "fmla v28.8h, v6.8h, v0.h[4]\n" + "fmla v29.8h, v6.8h, v0.h[5]\n" + "fmla v30.8h, v6.8h, v0.h[6]\n" + "fmla v31.8h, v6.8h, v0.h[7]\n" + "ldr q1, [x20, #0x0]\n" + "ldr q0, [%x[weights], #0x0]\n" "fmla v16.8h, v5.8h, v4.h[0]\n" "fmla v17.8h, v5.8h, v4.h[1]\n" - "ldp x24, x9, [x20], #0x10\n" - "fmla v18.8h, v5.8h, v4.h[2]\n" - "fmla v19.8h, v5.8h, v4.h[3]\n" - "fmla v20.8h, v5.8h, v4.h[4]\n" - "fmla v21.8h, v5.8h, v4.h[5]\n" - "fmla v22.8h, v5.8h, v4.h[6]\n" - "fmla v23.8h, v5.8h, v4.h[7]\n" - "ldr q4, [x24, #0x0]\n" - "fmla v24.8h, v5.8h, v3.h[0]\n" - "fmla v25.8h, v5.8h, v3.h[1]\n" - "fmla v26.8h, v5.8h, v3.h[2]\n" - "fmla v27.8h, v5.8h, v3.h[3]\n" - "fmla v28.8h, v5.8h, v3.h[4]\n" - "fmla v29.8h, v5.8h, v3.h[5]\n" - "fmla v30.8h, v5.8h, v3.h[6]\n" - "fmla v31.8h, v5.8h, v3.h[7]\n" - "ldr q3, [x9, #0x0]\n" - "ldr q5, [%x[weights], #0x0]\n" - "fmla v16.8h, v2.8h, v1.h[0]\n" - "fmla v17.8h, v2.8h, v1.h[1]\n" "add %x[weights], %x[weights], #0x10\n" - "fmla v18.8h, v2.8h, v1.h[2]\n" - "fmla v19.8h, v2.8h, v1.h[3]\n" - "fmla v20.8h, v2.8h, v1.h[4]\n" - "fmla v21.8h, v2.8h, v1.h[5]\n" - "fmla v22.8h, v2.8h, v1.h[6]\n" - "fmla v23.8h, v2.8h, v1.h[7]\n" - "fmla v24.8h, v2.8h, v0.h[0]\n" - "fmla v25.8h, v2.8h, v0.h[1]\n" - "fmla v26.8h, v2.8h, v0.h[2]\n" - "fmla v27.8h, v2.8h, v0.h[3]\n" - "fmla v28.8h, v2.8h, v0.h[4]\n" - "fmla v29.8h, v2.8h, v0.h[5]\n" - "fmla v30.8h, v2.8h, v0.h[6]\n" - "fmla v31.8h, v2.8h, v0.h[7]\n" - "fmla v16.8h, v5.8h, v4.h[0]\n" - "fmla v17.8h, v5.8h, v4.h[1]\n" "fmla v18.8h, v5.8h, v4.h[2]\n" "fmla v19.8h, v5.8h, v4.h[3]\n" "fmla v20.8h, v5.8h, v4.h[4]\n" @@ -628,415 +612,429 @@ void a64_fp16_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_im "fmla v29.8h, v5.8h, v3.h[5]\n" "fmla v30.8h, v5.8h, v3.h[6]\n" "fmla v31.8h, v5.8h, v3.h[7]\n" + "fmla v16.8h, v0.8h, v2.h[0]\n" + "fmla v17.8h, v0.8h, v2.h[1]\n" + "fmla v18.8h, v0.8h, v2.h[2]\n" + "fmla v19.8h, v0.8h, v2.h[3]\n" + "fmla v20.8h, v0.8h, v2.h[4]\n" + "fmla v21.8h, v0.8h, v2.h[5]\n" + "fmla v22.8h, v0.8h, v2.h[6]\n" + "fmla v23.8h, v0.8h, v2.h[7]\n" + "fmla v24.8h, v0.8h, v1.h[0]\n" + "fmla v25.8h, v0.8h, v1.h[1]\n" + "fmla v26.8h, v0.8h, v1.h[2]\n" + "fmla v27.8h, v0.8h, v1.h[3]\n" + "fmla v28.8h, v0.8h, v1.h[4]\n" + "fmla v29.8h, v0.8h, v1.h[5]\n" + "fmla v30.8h, v0.8h, v1.h[6]\n" + "fmla v31.8h, v0.8h, v1.h[7]\n" "b 18f\n" "17:" // Output channel oddments: Single kernel point - "fmla v16.8h, v5.8h, v4.h[0]\n" - "fmla v17.8h, v5.8h, v4.h[1]\n" - "fmla v18.8h, v5.8h, v4.h[2]\n" - "fmla v19.8h, v5.8h, v4.h[3]\n" - "fmla v20.8h, v5.8h, v4.h[4]\n" - "fmla v21.8h, v5.8h, v4.h[5]\n" - "fmla v22.8h, v5.8h, v4.h[6]\n" - "fmla v23.8h, v5.8h, v4.h[7]\n" - "fmla v24.8h, v5.8h, v3.h[0]\n" - "fmla v25.8h, v5.8h, v3.h[1]\n" - "fmla v26.8h, v5.8h, v3.h[2]\n" - "fmla v27.8h, v5.8h, v3.h[3]\n" - "fmla v28.8h, v5.8h, v3.h[4]\n" - "fmla v29.8h, v5.8h, v3.h[5]\n" - "fmla v30.8h, v5.8h, v3.h[6]\n" - "fmla v31.8h, v5.8h, v3.h[7]\n" + "fmla v16.8h, v6.8h, v1.h[0]\n" + "fmla v17.8h, v6.8h, v1.h[1]\n" + "fmla v18.8h, v6.8h, v1.h[2]\n" + "fmla v19.8h, v6.8h, v1.h[3]\n" + "fmla v20.8h, v6.8h, v1.h[4]\n" + "fmla v21.8h, v6.8h, v1.h[5]\n" + "fmla v22.8h, v6.8h, v1.h[6]\n" + "fmla v23.8h, v6.8h, v1.h[7]\n" + "fmla v24.8h, v6.8h, v0.h[0]\n" + "fmla v25.8h, v6.8h, v0.h[1]\n" + "fmla v26.8h, v6.8h, v0.h[2]\n" + "fmla v27.8h, v6.8h, v0.h[3]\n" + "fmla v28.8h, v6.8h, v0.h[4]\n" + "fmla v29.8h, v6.8h, v0.h[5]\n" + "fmla v30.8h, v6.8h, v0.h[6]\n" + "fmla v31.8h, v6.8h, v0.h[7]\n" "18:" // Output channel oddments: Done - "fmin v16.8h, v16.8h, v6.8h\n" - "fmin v17.8h, v17.8h, v6.8h\n" - "fmin v18.8h, v18.8h, v6.8h\n" - "fmin v19.8h, v19.8h, v6.8h\n" - "fmin v20.8h, v20.8h, v6.8h\n" - "fmin v21.8h, v21.8h, v6.8h\n" - "fmin v22.8h, v22.8h, v6.8h\n" - "fmin v23.8h, v23.8h, v6.8h\n" - "fmin v24.8h, v24.8h, v6.8h\n" - "fmin v25.8h, v25.8h, v6.8h\n" - "fmin v26.8h, v26.8h, v6.8h\n" - "fmin v27.8h, v27.8h, v6.8h\n" - "fmin v28.8h, v28.8h, v6.8h\n" - "fmin v29.8h, v29.8h, v6.8h\n" - "fmin v30.8h, v30.8h, v6.8h\n" - "fmin v31.8h, v31.8h, v6.8h\n" - "fmax v16.8h, v16.8h, v7.8h\n" - "fmax v17.8h, v17.8h, v7.8h\n" - "fmax v18.8h, v18.8h, v7.8h\n" - "fmax v19.8h, v19.8h, v7.8h\n" - "fmax v20.8h, v20.8h, v7.8h\n" - "fmax v21.8h, v21.8h, v7.8h\n" - "fmax v22.8h, v22.8h, v7.8h\n" - "fmax v23.8h, v23.8h, v7.8h\n" - "fmax v24.8h, v24.8h, v7.8h\n" - "fmax v25.8h, v25.8h, v7.8h\n" - "fmax v26.8h, v26.8h, v7.8h\n" - "fmax v27.8h, v27.8h, v7.8h\n" - "fmax v28.8h, v28.8h, v7.8h\n" - "fmax v29.8h, v29.8h, v7.8h\n" - "fmax v30.8h, v30.8h, v7.8h\n" - "fmax v31.8h, v31.8h, v7.8h\n" + "fmin v16.8h, v16.8h, v7.8h\n" + "fmin v17.8h, v17.8h, v7.8h\n" + "fmin v18.8h, v18.8h, v7.8h\n" + "fmin v19.8h, v19.8h, v7.8h\n" + "fmin v20.8h, v20.8h, v7.8h\n" + "fmin v21.8h, v21.8h, v7.8h\n" + "fmin v22.8h, v22.8h, v7.8h\n" + "fmin v23.8h, v23.8h, v7.8h\n" + "fmin v24.8h, v24.8h, v7.8h\n" + "fmin v25.8h, v25.8h, v7.8h\n" + "fmin v26.8h, v26.8h, v7.8h\n" + "fmin v27.8h, v27.8h, v7.8h\n" + "fmin v28.8h, v28.8h, v7.8h\n" + "fmin v29.8h, v29.8h, v7.8h\n" + "fmin v30.8h, v30.8h, v7.8h\n" + "fmin v31.8h, v31.8h, v7.8h\n" + "fmax v16.8h, v16.8h, v8.8h\n" + "fmax v17.8h, v17.8h, v8.8h\n" + "fmax v18.8h, v18.8h, v8.8h\n" + "fmax v19.8h, v19.8h, v8.8h\n" + "fmax v20.8h, v20.8h, v8.8h\n" + "fmax v21.8h, v21.8h, v8.8h\n" + "fmax v22.8h, v22.8h, v8.8h\n" + "fmax v23.8h, v23.8h, v8.8h\n" + "fmax v24.8h, v24.8h, v8.8h\n" + "fmax v25.8h, v25.8h, v8.8h\n" + "fmax v26.8h, v26.8h, v8.8h\n" + "fmax v27.8h, v27.8h, v8.8h\n" + "fmax v28.8h, v28.8h, v8.8h\n" + "fmax v29.8h, v29.8h, v8.8h\n" + "fmax v30.8h, v30.8h, v8.8h\n" + "fmax v31.8h, v31.8h, v8.8h\n" "tbz %x[n_output_channels], #2, 20f\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "add x20, x20, x10, LSL #1\n" - "add x21, x21, x10, LSL #1\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "add x22, x22, x10, LSL #1\n" - "add x23, x23, x10, LSL #1\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "add x24, x24, x10, LSL #1\n" - "add x25, x25, x10, LSL #1\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "add x26, x26, x10, LSL #1\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" "add x27, x27, x10, LSL #1\n" - "st1 { v16.d }[0], [x20]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "add x20, x20, x10, LSL #1\n" - "st1 { v17.d }[0], [x21]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "add x21, x21, x10, LSL #1\n" - "st1 { v18.d }[0], [x22]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "add x22, x22, x10, LSL #1\n" - "st1 { v19.d }[0], [x23]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "add x23, x23, x10, LSL #1\n" - "st1 { v20.d }[0], [x24]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "add x24, x24, x10, LSL #1\n" - "st1 { v21.d }[0], [x25]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "add x25, x25, x10, LSL #1\n" - "st1 { v22.d }[0], [x26]\n" - "ldr x26, [%x[outptrs], #0x70]\n" "add x26, x26, x10, LSL #1\n" - "st1 { v23.d }[0], [x27]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "add x25, x25, x10, LSL #1\n" + "add x24, x24, x10, LSL #1\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "add x23, x23, x10, LSL #1\n" + "add x22, x22, x10, LSL #1\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "add x21, x21, x10, LSL #1\n" + "add x20, x20, x10, LSL #1\n" + "st1 { v16.d }[0], [x27]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "add x27, x27, x10, LSL #1\n" + "st1 { v17.d }[0], [x26]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "add x26, x26, x10, LSL #1\n" + "st1 { v18.d }[0], [x25]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "add x25, x25, x10, LSL #1\n" + "st1 { v19.d }[0], [x24]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "add x24, x24, x10, LSL #1\n" + "st1 { v20.d }[0], [x23]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "add x23, x23, x10, LSL #1\n" + "st1 { v21.d }[0], [x22]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "add x22, x22, x10, LSL #1\n" + "st1 { v22.d }[0], [x21]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "add x21, x21, x10, LSL #1\n" + "st1 { v23.d }[0], [x20]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "add x20, x20, x10, LSL #1\n" "add x10, x10, #0x4\n" - "st1 { v24.d }[0], [x20]\n" - "st1 { v25.d }[0], [x21]\n" - "st1 { v26.d }[0], [x22]\n" - "st1 { v27.d }[0], [x23]\n" - "st1 { v28.d }[0], [x24]\n" - "st1 { v29.d }[0], [x25]\n" - "st1 { v30.d }[0], [x26]\n" - "st1 { v31.d }[0], [x27]\n" + "st1 { v24.d }[0], [x27]\n" + "st1 { v25.d }[0], [x26]\n" + "st1 { v26.d }[0], [x25]\n" + "st1 { v27.d }[0], [x24]\n" + "st1 { v28.d }[0], [x23]\n" + "st1 { v29.d }[0], [x22]\n" + "st1 { v30.d }[0], [x21]\n" + "st1 { v31.d }[0], [x20]\n" "tbz %x[n_output_channels], #1, 19f\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "add x20, x20, x10, LSL #1\n" - "add x21, x21, x10, LSL #1\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "add x22, x22, x10, LSL #1\n" - "add x23, x23, x10, LSL #1\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "add x24, x24, x10, LSL #1\n" - "add x25, x25, x10, LSL #1\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "add x26, x26, x10, LSL #1\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" "add x27, x27, x10, LSL #1\n" - "st1 { v16.s }[2], [x20]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "add x20, x20, x10, LSL #1\n" - "st1 { v17.s }[2], [x21]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "add x21, x21, x10, LSL #1\n" - "st1 { v18.s }[2], [x22]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "add x22, x22, x10, LSL #1\n" - "st1 { v19.s }[2], [x23]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "add x23, x23, x10, LSL #1\n" - "st1 { v20.s }[2], [x24]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "add x24, x24, x10, LSL #1\n" - "st1 { v21.s }[2], [x25]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "add x25, x25, x10, LSL #1\n" - "st1 { v22.s }[2], [x26]\n" - "ldr x26, [%x[outptrs], #0x70]\n" "add x26, x26, x10, LSL #1\n" - "st1 { v23.s }[2], [x27]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "add x25, x25, x10, LSL #1\n" + "add x24, x24, x10, LSL #1\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "add x23, x23, x10, LSL #1\n" + "add x22, x22, x10, LSL #1\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "add x21, x21, x10, LSL #1\n" + "add x20, x20, x10, LSL #1\n" + "st1 { v16.s }[2], [x27]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "add x27, x27, x10, LSL #1\n" + "st1 { v17.s }[2], [x26]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "add x26, x26, x10, LSL #1\n" + "st1 { v18.s }[2], [x25]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "add x25, x25, x10, LSL #1\n" + "st1 { v19.s }[2], [x24]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "add x24, x24, x10, LSL #1\n" + "st1 { v20.s }[2], [x23]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "add x23, x23, x10, LSL #1\n" + "st1 { v21.s }[2], [x22]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "add x22, x22, x10, LSL #1\n" + "st1 { v22.s }[2], [x21]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "add x21, x21, x10, LSL #1\n" + "st1 { v23.s }[2], [x20]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "add x20, x20, x10, LSL #1\n" "add x10, x10, #0x2\n" - "st1 { v24.s }[2], [x20]\n" - "st1 { v25.s }[2], [x21]\n" - "st1 { v26.s }[2], [x22]\n" - "st1 { v27.s }[2], [x23]\n" - "st1 { v28.s }[2], [x24]\n" - "st1 { v29.s }[2], [x25]\n" - "st1 { v30.s }[2], [x26]\n" - "st1 { v31.s }[2], [x27]\n" + "st1 { v24.s }[2], [x27]\n" + "st1 { v25.s }[2], [x26]\n" + "st1 { v26.s }[2], [x25]\n" + "st1 { v27.s }[2], [x24]\n" + "st1 { v28.s }[2], [x23]\n" + "st1 { v29.s }[2], [x22]\n" + "st1 { v30.s }[2], [x21]\n" + "st1 { v31.s }[2], [x20]\n" "tbz %x[n_output_channels], #0, 22f\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "add x20, x20, x10, LSL #1\n" - "add x21, x21, x10, LSL #1\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "add x22, x22, x10, LSL #1\n" - "add x23, x23, x10, LSL #1\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "add x24, x24, x10, LSL #1\n" - "add x25, x25, x10, LSL #1\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "add x26, x26, x10, LSL #1\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" "add x27, x27, x10, LSL #1\n" - "st1 { v16.h }[6], [x20]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "add x20, x20, x10, LSL #1\n" - "st1 { v17.h }[6], [x21]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "add x21, x21, x10, LSL #1\n" - "st1 { v18.h }[6], [x22]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "add x22, x22, x10, LSL #1\n" - "st1 { v19.h }[6], [x23]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "add x23, x23, x10, LSL #1\n" - "st1 { v20.h }[6], [x24]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "add x24, x24, x10, LSL #1\n" - "st1 { v21.h }[6], [x25]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "add x25, x25, x10, LSL #1\n" - "st1 { v22.h }[6], [x26]\n" - "ldr x26, [%x[outptrs], #0x70]\n" "add x26, x26, x10, LSL #1\n" - "st1 { v23.h }[6], [x27]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "add x25, x25, x10, LSL #1\n" + "add x24, x24, x10, LSL #1\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "add x23, x23, x10, LSL #1\n" + "add x22, x22, x10, LSL #1\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "add x21, x21, x10, LSL #1\n" + "add x20, x20, x10, LSL #1\n" + "st1 { v16.h }[6], [x27]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "add x27, x27, x10, LSL #1\n" - "st1 { v24.h }[6], [x20]\n" - "st1 { v25.h }[6], [x21]\n" - "st1 { v26.h }[6], [x22]\n" - "st1 { v27.h }[6], [x23]\n" - "st1 { v28.h }[6], [x24]\n" - "st1 { v29.h }[6], [x25]\n" - "st1 { v30.h }[6], [x26]\n" - "st1 { v31.h }[6], [x27]\n" + "st1 { v17.h }[6], [x26]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "add x26, x26, x10, LSL #1\n" + "st1 { v18.h }[6], [x25]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "add x25, x25, x10, LSL #1\n" + "st1 { v19.h }[6], [x24]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "add x24, x24, x10, LSL #1\n" + "st1 { v20.h }[6], [x23]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "add x23, x23, x10, LSL #1\n" + "st1 { v21.h }[6], [x22]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "add x22, x22, x10, LSL #1\n" + "st1 { v22.h }[6], [x21]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "add x21, x21, x10, LSL #1\n" + "st1 { v23.h }[6], [x20]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "add x20, x20, x10, LSL #1\n" + "st1 { v24.h }[6], [x27]\n" + "st1 { v25.h }[6], [x26]\n" + "st1 { v26.h }[6], [x25]\n" + "st1 { v27.h }[6], [x24]\n" + "st1 { v28.h }[6], [x23]\n" + "st1 { v29.h }[6], [x22]\n" + "st1 { v30.h }[6], [x21]\n" + "st1 { v31.h }[6], [x20]\n" "b 22f\n" "19:" // Output channel oddments: Done: Store: Bit 2: Bit 1: Unset "tbz %x[n_output_channels], #0, 22f\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "add x20, x20, x10, LSL #1\n" - "add x21, x21, x10, LSL #1\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "add x22, x22, x10, LSL #1\n" - "add x23, x23, x10, LSL #1\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "add x24, x24, x10, LSL #1\n" - "add x25, x25, x10, LSL #1\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "add x26, x26, x10, LSL #1\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" "add x27, x27, x10, LSL #1\n" - "st1 { v16.h }[4], [x20]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "add x20, x20, x10, LSL #1\n" - "st1 { v17.h }[4], [x21]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "add x21, x21, x10, LSL #1\n" - "st1 { v18.h }[4], [x22]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "add x22, x22, x10, LSL #1\n" - "st1 { v19.h }[4], [x23]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "add x23, x23, x10, LSL #1\n" - "st1 { v20.h }[4], [x24]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "add x24, x24, x10, LSL #1\n" - "st1 { v21.h }[4], [x25]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "add x25, x25, x10, LSL #1\n" - "st1 { v22.h }[4], [x26]\n" - "ldr x26, [%x[outptrs], #0x70]\n" "add x26, x26, x10, LSL #1\n" - "st1 { v23.h }[4], [x27]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "add x25, x25, x10, LSL #1\n" + "add x24, x24, x10, LSL #1\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "add x23, x23, x10, LSL #1\n" + "add x22, x22, x10, LSL #1\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "add x21, x21, x10, LSL #1\n" + "add x20, x20, x10, LSL #1\n" + "st1 { v16.h }[4], [x27]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "add x27, x27, x10, LSL #1\n" - "st1 { v24.h }[4], [x20]\n" - "st1 { v25.h }[4], [x21]\n" - "st1 { v26.h }[4], [x22]\n" - "st1 { v27.h }[4], [x23]\n" - "st1 { v28.h }[4], [x24]\n" - "st1 { v29.h }[4], [x25]\n" - "st1 { v30.h }[4], [x26]\n" - "st1 { v31.h }[4], [x27]\n" + "st1 { v17.h }[4], [x26]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "add x26, x26, x10, LSL #1\n" + "st1 { v18.h }[4], [x25]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "add x25, x25, x10, LSL #1\n" + "st1 { v19.h }[4], [x24]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "add x24, x24, x10, LSL #1\n" + "st1 { v20.h }[4], [x23]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "add x23, x23, x10, LSL #1\n" + "st1 { v21.h }[4], [x22]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "add x22, x22, x10, LSL #1\n" + "st1 { v22.h }[4], [x21]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "add x21, x21, x10, LSL #1\n" + "st1 { v23.h }[4], [x20]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "add x20, x20, x10, LSL #1\n" + "st1 { v24.h }[4], [x27]\n" + "st1 { v25.h }[4], [x26]\n" + "st1 { v26.h }[4], [x25]\n" + "st1 { v27.h }[4], [x24]\n" + "st1 { v28.h }[4], [x23]\n" + "st1 { v29.h }[4], [x22]\n" + "st1 { v30.h }[4], [x21]\n" + "st1 { v31.h }[4], [x20]\n" "b 22f\n" "20:" // Output channel oddments: Done: Store: Bit 2: Unset "tbz %x[n_output_channels], #1, 21f\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "add x20, x20, x10, LSL #1\n" - "add x21, x21, x10, LSL #1\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "add x22, x22, x10, LSL #1\n" - "add x23, x23, x10, LSL #1\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "add x24, x24, x10, LSL #1\n" - "add x25, x25, x10, LSL #1\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "add x26, x26, x10, LSL #1\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" "add x27, x27, x10, LSL #1\n" - "st1 { v16.s }[0], [x20]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "add x20, x20, x10, LSL #1\n" - "st1 { v17.s }[0], [x21]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "add x21, x21, x10, LSL #1\n" - "st1 { v18.s }[0], [x22]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "add x22, x22, x10, LSL #1\n" - "st1 { v19.s }[0], [x23]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "add x23, x23, x10, LSL #1\n" - "st1 { v20.s }[0], [x24]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "add x24, x24, x10, LSL #1\n" - "st1 { v21.s }[0], [x25]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "add x25, x25, x10, LSL #1\n" - "st1 { v22.s }[0], [x26]\n" - "ldr x26, [%x[outptrs], #0x70]\n" "add x26, x26, x10, LSL #1\n" - "st1 { v23.s }[0], [x27]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "add x25, x25, x10, LSL #1\n" + "add x24, x24, x10, LSL #1\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "add x23, x23, x10, LSL #1\n" + "add x22, x22, x10, LSL #1\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "add x21, x21, x10, LSL #1\n" + "add x20, x20, x10, LSL #1\n" + "st1 { v16.s }[0], [x27]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "add x27, x27, x10, LSL #1\n" + "st1 { v17.s }[0], [x26]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "add x26, x26, x10, LSL #1\n" + "st1 { v18.s }[0], [x25]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "add x25, x25, x10, LSL #1\n" + "st1 { v19.s }[0], [x24]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "add x24, x24, x10, LSL #1\n" + "st1 { v20.s }[0], [x23]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "add x23, x23, x10, LSL #1\n" + "st1 { v21.s }[0], [x22]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "add x22, x22, x10, LSL #1\n" + "st1 { v22.s }[0], [x21]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "add x21, x21, x10, LSL #1\n" + "st1 { v23.s }[0], [x20]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "add x20, x20, x10, LSL #1\n" "add x10, x10, #0x2\n" - "st1 { v24.s }[0], [x20]\n" - "st1 { v25.s }[0], [x21]\n" - "st1 { v26.s }[0], [x22]\n" - "st1 { v27.s }[0], [x23]\n" - "st1 { v28.s }[0], [x24]\n" - "st1 { v29.s }[0], [x25]\n" - "st1 { v30.s }[0], [x26]\n" - "st1 { v31.s }[0], [x27]\n" + "st1 { v24.s }[0], [x27]\n" + "st1 { v25.s }[0], [x26]\n" + "st1 { v26.s }[0], [x25]\n" + "st1 { v27.s }[0], [x24]\n" + "st1 { v28.s }[0], [x23]\n" + "st1 { v29.s }[0], [x22]\n" + "st1 { v30.s }[0], [x21]\n" + "st1 { v31.s }[0], [x20]\n" "tbz %x[n_output_channels], #0, 22f\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "add x20, x20, x10, LSL #1\n" - "add x21, x21, x10, LSL #1\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "add x22, x22, x10, LSL #1\n" - "add x23, x23, x10, LSL #1\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "add x24, x24, x10, LSL #1\n" - "add x25, x25, x10, LSL #1\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "add x26, x26, x10, LSL #1\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" "add x27, x27, x10, LSL #1\n" - "st1 { v16.h }[2], [x20]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "add x20, x20, x10, LSL #1\n" - "st1 { v17.h }[2], [x21]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "add x21, x21, x10, LSL #1\n" - "st1 { v18.h }[2], [x22]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "add x22, x22, x10, LSL #1\n" - "st1 { v19.h }[2], [x23]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "add x23, x23, x10, LSL #1\n" - "st1 { v20.h }[2], [x24]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "add x24, x24, x10, LSL #1\n" - "st1 { v21.h }[2], [x25]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "add x25, x25, x10, LSL #1\n" - "st1 { v22.h }[2], [x26]\n" - "ldr x26, [%x[outptrs], #0x70]\n" "add x26, x26, x10, LSL #1\n" - "st1 { v23.h }[2], [x27]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "add x25, x25, x10, LSL #1\n" + "add x24, x24, x10, LSL #1\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "add x23, x23, x10, LSL #1\n" + "add x22, x22, x10, LSL #1\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "add x21, x21, x10, LSL #1\n" + "add x20, x20, x10, LSL #1\n" + "st1 { v16.h }[2], [x27]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "add x27, x27, x10, LSL #1\n" - "st1 { v24.h }[2], [x20]\n" - "st1 { v25.h }[2], [x21]\n" - "st1 { v26.h }[2], [x22]\n" - "st1 { v27.h }[2], [x23]\n" - "st1 { v28.h }[2], [x24]\n" - "st1 { v29.h }[2], [x25]\n" - "st1 { v30.h }[2], [x26]\n" - "st1 { v31.h }[2], [x27]\n" + "st1 { v17.h }[2], [x26]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "add x26, x26, x10, LSL #1\n" + "st1 { v18.h }[2], [x25]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "add x25, x25, x10, LSL #1\n" + "st1 { v19.h }[2], [x24]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "add x24, x24, x10, LSL #1\n" + "st1 { v20.h }[2], [x23]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "add x23, x23, x10, LSL #1\n" + "st1 { v21.h }[2], [x22]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "add x22, x22, x10, LSL #1\n" + "st1 { v22.h }[2], [x21]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "add x21, x21, x10, LSL #1\n" + "st1 { v23.h }[2], [x20]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "add x20, x20, x10, LSL #1\n" + "st1 { v24.h }[2], [x27]\n" + "st1 { v25.h }[2], [x26]\n" + "st1 { v26.h }[2], [x25]\n" + "st1 { v27.h }[2], [x24]\n" + "st1 { v28.h }[2], [x23]\n" + "st1 { v29.h }[2], [x22]\n" + "st1 { v30.h }[2], [x21]\n" + "st1 { v31.h }[2], [x20]\n" "b 22f\n" "21:" // Output channel oddments: Done: Store: Bit 2: Unset: Bit 1: Unset - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "add x20, x20, x10, LSL #1\n" - "add x21, x21, x10, LSL #1\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "add x22, x22, x10, LSL #1\n" - "add x23, x23, x10, LSL #1\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "add x24, x24, x10, LSL #1\n" - "add x25, x25, x10, LSL #1\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "add x26, x26, x10, LSL #1\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" "add x27, x27, x10, LSL #1\n" - "st1 { v16.h }[0], [x20]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "add x20, x20, x10, LSL #1\n" - "st1 { v17.h }[0], [x21]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "add x21, x21, x10, LSL #1\n" - "st1 { v18.h }[0], [x22]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "add x22, x22, x10, LSL #1\n" - "st1 { v19.h }[0], [x23]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "add x23, x23, x10, LSL #1\n" - "st1 { v20.h }[0], [x24]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "add x24, x24, x10, LSL #1\n" - "st1 { v21.h }[0], [x25]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "add x25, x25, x10, LSL #1\n" - "st1 { v22.h }[0], [x26]\n" - "ldr x26, [%x[outptrs], #0x70]\n" "add x26, x26, x10, LSL #1\n" - "st1 { v23.h }[0], [x27]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "add x25, x25, x10, LSL #1\n" + "add x24, x24, x10, LSL #1\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "add x23, x23, x10, LSL #1\n" + "add x22, x22, x10, LSL #1\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "add x21, x21, x10, LSL #1\n" + "add x20, x20, x10, LSL #1\n" + "st1 { v16.h }[0], [x27]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "add x27, x27, x10, LSL #1\n" - "st1 { v24.h }[0], [x20]\n" - "st1 { v25.h }[0], [x21]\n" - "st1 { v26.h }[0], [x22]\n" - "st1 { v27.h }[0], [x23]\n" - "st1 { v28.h }[0], [x24]\n" - "st1 { v29.h }[0], [x25]\n" - "st1 { v30.h }[0], [x26]\n" - "st1 { v31.h }[0], [x27]\n" + "st1 { v17.h }[0], [x26]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "add x26, x26, x10, LSL #1\n" + "st1 { v18.h }[0], [x25]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "add x25, x25, x10, LSL #1\n" + "st1 { v19.h }[0], [x24]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "add x24, x24, x10, LSL #1\n" + "st1 { v20.h }[0], [x23]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "add x23, x23, x10, LSL #1\n" + "st1 { v21.h }[0], [x22]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "add x22, x22, x10, LSL #1\n" + "st1 { v22.h }[0], [x21]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "add x21, x21, x10, LSL #1\n" + "st1 { v23.h }[0], [x20]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "add x20, x20, x10, LSL #1\n" + "st1 { v24.h }[0], [x27]\n" + "st1 { v25.h }[0], [x26]\n" + "st1 { v26.h }[0], [x25]\n" + "st1 { v27.h }[0], [x24]\n" + "st1 { v28.h }[0], [x23]\n" + "st1 { v29.h }[0], [x22]\n" + "st1 { v30.h }[0], [x21]\n" + "st1 { v31.h }[0], [x20]\n" "22:" // Output channel oddments: Done: Store: Bit 2: End - "23:" // Done - : [weights] "+&r" (weights) : [bias] "r" (bias), [inptrs] "r" (inptrs), [kernel_points] "r" ((uint64_t) kernel_points), [minmax_vals] "r" (minmax_vals), [n_output_channels] "r" ((uint64_t) n_output_channels), [outptrs] "r" (outptrs) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp index 420e95384d03b41fb6ce75a1b4586df6382becb9..5d3db974f0522eac3c8af0185f4c9dd04ccaac65 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include @@ -33,8 +33,8 @@ namespace arm_conv { namespace depthwise { -void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); -void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); +void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(const float *const *const input_ptrs, float *const *const outptrs, const void *params, unsigned int n_channels, const float activation_min, const float activation_max); +void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int n_tile_rows, const unsigned int n_tile_cols, const float *inptr, int64_t ld_input_row, int64_t ld_input_col, float *outptr, int64_t ld_output_row, int64_t ld_output_col, const void *params, unsigned int n_channels, const float activation_min, const float activation_max); class a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirstStrategy { @@ -57,7 +57,7 @@ class a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirst constexpr static unsigned int output_cols = 2; a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst(const CPUInfo *) - : DepthwiseDepthfirstStrategy(2, 3, 1) {} + : Parent(output_rows, output_cols, kernel_rows, kernel_cols, stride_rows, stride_cols) {} arm_gemm::VLType get_vl_type(void) const override { return vl_type; } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp index 2ff03aa15ae9ff67e6c8596936d8267e0277b8b0..fd8686c15e06ec52acecd828a40d6ea187e67d44 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ +#if defined(__aarch64__) namespace arm_conv { namespace depthwise { @@ -110,15 +110,15 @@ void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "add x11, x15, x15\n" "ldr x10, [%x[params_struct], %[offsetof_args_params]]\n" "mul x21, x21, x27\n" // offset *= kernel_stride * output_size - "add x13, x13, x21, LSL #2\n" // inptr[0] += offset * sizeof(float) + "add x13, x13, x21, LSL #2\n" // inptr[0] += offset * sizeof(float) "add x9, x13, x25, LSL #2\n" "mul x20, x20, x26\n" // offset *= output_tile_size "add x28, x9, x25, LSL #2\n" "add x12, x12, x20, LSL #2\n" // outptrs[0] += offset * sizeof(float) "add x20, %x[params_struct], %[offsetof_args_min]\n" - "ld1r { v18.4s }, [x20]\n" + "ld1r { v27.4s }, [x20]\n" "add x20, %x[params_struct], %[offsetof_args_max]\n" - "ld1r { v17.4s }, [x20]\n" + "ld1r { v26.4s }, [x20]\n" "add x27, x28, x25, LSL #2\n" "add x26, x11, x15\n" "add x25, x12, x24, LSL #2\n" @@ -126,7 +126,7 @@ void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "mov x21, #0x0\n" "sub x20, XZR, x23\n" "cbz x22, 4f\n" - "ldr q16, [x10, #0x0]\n" + "ldr q25, [x10, #0x0]\n" "ldr q0, [x10, #0x10]\n" "cmp x23, x22, LSL #4\n" "ldr q1, [x10, #0x20]\n" @@ -145,162 +145,162 @@ void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "ldr q13, [x28, x15]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "mov v28.16b, v16.16b\n fmla v28.4s, v4.4s, v9.4s\n" - "mov v29.16b, v16.16b\n fmla v29.4s, v3.4s, v9.4s\n" + "mov v24.16b, v25.16b\n fmla v24.4s, v4.4s, v9.4s\n" + "mov v23.16b, v25.16b\n fmla v23.4s, v3.4s, v9.4s\n" "add x23, x23, #0x10\n" "cmp x23, x22, LSL #4\n" - "mov v30.16b, v16.16b\n fmla v30.4s, v1.4s, v9.4s\n" - "mov v31.16b, v16.16b\n fmla v31.4s, v0.4s, v9.4s\n" - "ld1 { v9.4s }, [x27]\n" - "ldr q16, [x10, #0x0]\n" - "fmla v28.4s, v0.4s, v10.4s\n" - "ldr q10, [x28, x11]\n" - "fmla v29.4s, v2.4s, v11.4s\n" - "ldr q11, [x27, x26]\n" - "fmla v30.4s, v2.4s, v12.4s\n" - "fmla v31.4s, v1.4s, v12.4s\n" + "mov v22.16b, v25.16b\n fmla v22.4s, v1.4s, v9.4s\n" + "mov v21.16b, v25.16b\n fmla v21.4s, v0.4s, v9.4s\n" + "ld1 { v18.4s }, [x27]\n" + "ldr q25, [x10, #0x0]\n" + "fmla v24.4s, v0.4s, v10.4s\n" + "ldr q20, [x28, x11]\n" + "fmla v23.4s, v2.4s, v11.4s\n" + "ldr q17, [x27, x26]\n" + "fmla v22.4s, v2.4s, v12.4s\n" + "fmla v21.4s, v1.4s, v12.4s\n" "add x20, x20, #0x10\n" "add x21, x21, #0x10\n" - "fmla v28.4s, v5.4s, v12.4s\n" - "fmla v29.4s, v4.4s, v12.4s\n" - "ldr q12, [x13, x15]\n" - "fmla v30.4s, v6.4s, v9.4s\n" - "ldr q9, [x13, x11]\n" - "fmla v31.4s, v3.4s, v13.4s\n" + "fmla v24.4s, v5.4s, v12.4s\n" + "fmla v23.4s, v4.4s, v12.4s\n" + "ldr q16, [x13, x15]\n" + "fmla v22.4s, v6.4s, v18.4s\n" + "ldr q18, [x13, x11]\n" + "fmla v21.4s, v3.4s, v13.4s\n" "add x13, x13, #0x10\n" - "fmla v28.4s, v7.4s, v13.4s\n" - "fmla v29.4s, v6.4s, v13.4s\n" - "fmla v30.4s, v4.4s, v13.4s\n" - "fmla v31.4s, v8.4s, v11.4s\n" - "ld1 { v11.4s }, [x9]\n" - "fmla v28.4s, v1.4s, v12.4s\n" - "fmla v29.4s, v0.4s, v12.4s\n" - "ldr q12, [x9, x26]\n" + "fmla v24.4s, v7.4s, v13.4s\n" + "fmla v23.4s, v6.4s, v13.4s\n" + "fmla v22.4s, v4.4s, v13.4s\n" + "fmla v21.4s, v8.4s, v17.4s\n" + "ld1 { v17.4s }, [x9]\n" + "fmla v24.4s, v1.4s, v16.4s\n" + "fmla v23.4s, v0.4s, v16.4s\n" + "ldr q16, [x9, x26]\n" "add x9, x9, #0x10\n" - "fmla v30.4s, v5.4s, v10.4s\n" - "fmla v31.4s, v4.4s, v10.4s\n" + "fmla v22.4s, v5.4s, v20.4s\n" + "fmla v21.4s, v4.4s, v20.4s\n" "ldr q4, [x10, #0x50]\n" - "fmla v28.4s, v2.4s, v9.4s\n" - "fmla v29.4s, v1.4s, v9.4s\n" - "ld1 { v9.4s }, [x28]\n" + "fmla v24.4s, v2.4s, v18.4s\n" + "fmla v23.4s, v1.4s, v18.4s\n" + "ld1 { v19.4s }, [x28]\n" "ldr q1, [x10, #0x20]\n" - "fmla v30.4s, v0.4s, v11.4s\n" + "fmla v22.4s, v0.4s, v17.4s\n" "ldr q0, [x10, #0x10]\n" - "fmla v31.4s, v2.4s, v12.4s\n" + "fmla v21.4s, v2.4s, v16.4s\n" "ldr q2, [x10, #0x30]\n" - "fmla v28.4s, v8.4s, v10.4s\n" - "fmla v29.4s, v7.4s, v10.4s\n" - "ldr q10, [x28, x26]\n" + "fmla v24.4s, v8.4s, v20.4s\n" + "fmla v23.4s, v7.4s, v20.4s\n" + "ldr q18, [x28, x26]\n" "add x28, x28, #0x10\n" "ldr q13, [x28, x15]\n" - "fmla v30.4s, v3.4s, v9.4s\n" - "fmla v31.4s, v5.4s, v10.4s\n" - "fmla v28.4s, v3.4s, v11.4s\n" - "ldr q11, [x27, x15]\n" + "fmla v22.4s, v3.4s, v19.4s\n" + "fmla v21.4s, v5.4s, v18.4s\n" + "fmla v24.4s, v3.4s, v17.4s\n" + "ldr q17, [x27, x15]\n" "ldr q3, [x10, #0x40]\n" - "fmla v29.4s, v5.4s, v12.4s\n" - "ldr q12, [x27, x11]\n" + "fmla v23.4s, v5.4s, v16.4s\n" + "ldr q16, [x27, x11]\n" "ldr q5, [x10, #0x60]\n" - "fmla v30.4s, v7.4s, v11.4s\n" - "fmla v31.4s, v6.4s, v11.4s\n" + "fmla v22.4s, v7.4s, v17.4s\n" + "fmla v21.4s, v6.4s, v17.4s\n" "ldr q11, [x13, x26]\n" - "fmla v28.4s, v6.4s, v9.4s\n" + "fmla v24.4s, v6.4s, v19.4s\n" "ldr q9, [x9, x15]\n" - "fmla v29.4s, v8.4s, v10.4s\n" + "fmla v23.4s, v8.4s, v18.4s\n" "ld1 { v10.4s }, [x13]\n" "ldr q6, [x10, #0x70]\n" - "fmla v30.4s, v8.4s, v12.4s\n" - "fmla v31.4s, v7.4s, v12.4s\n" + "fmla v22.4s, v8.4s, v16.4s\n" + "fmla v21.4s, v7.4s, v16.4s\n" "ldr q12, [x9, x11]\n" "ldr q7, [x10, #0x80]\n" - "fmax v28.4s, v28.4s, v18.4s\n" - "fmax v29.4s, v29.4s, v18.4s\n" + "fmax v24.4s, v24.4s, v27.4s\n" + "fmax v23.4s, v23.4s, v27.4s\n" "ldr q8, [x10, #0x90]\n" - "fmax v30.4s, v30.4s, v18.4s\n" - "fmax v31.4s, v31.4s, v18.4s\n" + "fmax v22.4s, v22.4s, v27.4s\n" + "fmax v21.4s, v21.4s, v27.4s\n" "add x27, x27, #0x10\n" - "fmin v28.4s, v28.4s, v17.4s\n" - "fmin v29.4s, v29.4s, v17.4s\n" - "st1 { v28.4s }, [x12]\n" + "fmin v24.4s, v24.4s, v26.4s\n" + "fmin v23.4s, v23.4s, v26.4s\n" + "st1 { v24.4s }, [x12]\n" "add x10, x10, #0xa0\n" - "fmin v30.4s, v30.4s, v17.4s\n" - "fmin v31.4s, v31.4s, v17.4s\n" - "str q29, [x12, x14]\n" + "fmin v22.4s, v22.4s, v26.4s\n" + "fmin v21.4s, v21.4s, v26.4s\n" + "str q23, [x12, x14]\n" "add x12, x12, #0x10\n" - "st1 { v30.4s }, [x25]\n" - "str q31, [x25, x14]\n" + "st1 { v22.4s }, [x25]\n" + "str q21, [x25, x14]\n" "add x25, x25, #0x10\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "mov v28.16b, v16.16b\n fmla v28.4s, v4.4s, v9.4s\n" - "mov v29.16b, v16.16b\n fmla v29.4s, v3.4s, v9.4s\n" - "mov v30.16b, v16.16b\n fmla v30.4s, v1.4s, v9.4s\n" - "mov v31.16b, v16.16b\n fmla v31.4s, v0.4s, v9.4s\n" - "ld1 { v9.4s }, [x27]\n" - "fmla v28.4s, v0.4s, v10.4s\n" - "ldr q10, [x28, x11]\n" - "fmla v29.4s, v2.4s, v11.4s\n" - "ldr q11, [x27, x26]\n" - "fmla v30.4s, v2.4s, v12.4s\n" - "fmla v31.4s, v1.4s, v12.4s\n" - "fmla v28.4s, v5.4s, v12.4s\n" - "fmla v29.4s, v4.4s, v12.4s\n" - "ldr q12, [x13, x15]\n" - "fmla v30.4s, v6.4s, v9.4s\n" - "ldr q9, [x13, x11]\n" - "fmla v31.4s, v3.4s, v13.4s\n" + "mov v24.16b, v25.16b\n fmla v24.4s, v4.4s, v9.4s\n" + "mov v23.16b, v25.16b\n fmla v23.4s, v3.4s, v9.4s\n" + "mov v22.16b, v25.16b\n fmla v22.4s, v1.4s, v9.4s\n" + "mov v21.16b, v25.16b\n fmla v21.4s, v0.4s, v9.4s\n" + "ld1 { v18.4s }, [x27]\n" + "fmla v24.4s, v0.4s, v10.4s\n" + "ldr q20, [x28, x11]\n" + "fmla v23.4s, v2.4s, v11.4s\n" + "ldr q17, [x27, x26]\n" + "fmla v22.4s, v2.4s, v12.4s\n" + "fmla v21.4s, v1.4s, v12.4s\n" + "fmla v24.4s, v5.4s, v12.4s\n" + "fmla v23.4s, v4.4s, v12.4s\n" + "ldr q16, [x13, x15]\n" + "fmla v22.4s, v6.4s, v18.4s\n" + "ldr q18, [x13, x11]\n" + "fmla v21.4s, v3.4s, v13.4s\n" "add x13, x13, #0x10\n" - "fmla v28.4s, v7.4s, v13.4s\n" - "fmla v29.4s, v6.4s, v13.4s\n" - "fmla v30.4s, v4.4s, v13.4s\n" - "fmla v31.4s, v8.4s, v11.4s\n" - "ld1 { v11.4s }, [x9]\n" - "fmla v28.4s, v1.4s, v12.4s\n" - "fmla v29.4s, v0.4s, v12.4s\n" - "ldr q12, [x9, x26]\n" + "fmla v24.4s, v7.4s, v13.4s\n" + "fmla v23.4s, v6.4s, v13.4s\n" + "fmla v22.4s, v4.4s, v13.4s\n" + "fmla v21.4s, v8.4s, v17.4s\n" + "ld1 { v17.4s }, [x9]\n" + "fmla v24.4s, v1.4s, v16.4s\n" + "fmla v23.4s, v0.4s, v16.4s\n" + "ldr q16, [x9, x26]\n" "add x9, x9, #0x10\n" - "fmla v30.4s, v5.4s, v10.4s\n" - "fmla v31.4s, v4.4s, v10.4s\n" - "fmla v28.4s, v2.4s, v9.4s\n" - "fmla v29.4s, v1.4s, v9.4s\n" - "ld1 { v9.4s }, [x28]\n" - "fmla v30.4s, v0.4s, v11.4s\n" - "fmla v31.4s, v2.4s, v12.4s\n" - "fmla v28.4s, v8.4s, v10.4s\n" - "fmla v29.4s, v7.4s, v10.4s\n" - "ldr q10, [x28, x26]\n" + "fmla v22.4s, v5.4s, v20.4s\n" + "fmla v21.4s, v4.4s, v20.4s\n" + "fmla v24.4s, v2.4s, v18.4s\n" + "fmla v23.4s, v1.4s, v18.4s\n" + "ld1 { v19.4s }, [x28]\n" + "fmla v22.4s, v0.4s, v17.4s\n" + "fmla v21.4s, v2.4s, v16.4s\n" + "fmla v24.4s, v8.4s, v20.4s\n" + "fmla v23.4s, v7.4s, v20.4s\n" + "ldr q18, [x28, x26]\n" "add x28, x28, #0x10\n" - "fmla v30.4s, v3.4s, v9.4s\n" - "fmla v31.4s, v5.4s, v10.4s\n" - "fmla v28.4s, v3.4s, v11.4s\n" - "ldr q11, [x27, x15]\n" - "fmla v29.4s, v5.4s, v12.4s\n" - "ldr q12, [x27, x11]\n" - "fmla v30.4s, v7.4s, v11.4s\n" - "fmla v31.4s, v6.4s, v11.4s\n" + "fmla v22.4s, v3.4s, v19.4s\n" + "fmla v21.4s, v5.4s, v18.4s\n" + "fmla v24.4s, v3.4s, v17.4s\n" + "ldr q17, [x27, x15]\n" + "fmla v23.4s, v5.4s, v16.4s\n" + "ldr q16, [x27, x11]\n" + "fmla v22.4s, v7.4s, v17.4s\n" + "fmla v21.4s, v6.4s, v17.4s\n" "add x27, x27, #0x10\n" - "fmla v28.4s, v6.4s, v9.4s\n" - "fmla v29.4s, v8.4s, v10.4s\n" - "fmax v28.4s, v28.4s, v18.4s\n" - "fmla v30.4s, v8.4s, v12.4s\n" - "fmla v31.4s, v7.4s, v12.4s\n" - "fmax v29.4s, v29.4s, v18.4s\n" - "fmax v30.4s, v30.4s, v18.4s\n" - "fmax v31.4s, v31.4s, v18.4s\n" - "fmin v28.4s, v28.4s, v17.4s\n" - "fmin v29.4s, v29.4s, v17.4s\n" - "st1 { v28.4s }, [x12]\n" - "fmin v30.4s, v30.4s, v17.4s\n" - "fmin v31.4s, v31.4s, v17.4s\n" - "str q29, [x12, x14]\n" + "fmla v24.4s, v6.4s, v19.4s\n" + "fmla v23.4s, v8.4s, v18.4s\n" + "fmax v24.4s, v24.4s, v27.4s\n" + "fmla v22.4s, v8.4s, v16.4s\n" + "fmla v21.4s, v7.4s, v16.4s\n" + "fmax v23.4s, v23.4s, v27.4s\n" + "fmax v22.4s, v22.4s, v27.4s\n" + "fmax v21.4s, v21.4s, v27.4s\n" + "fmin v24.4s, v24.4s, v26.4s\n" + "fmin v23.4s, v23.4s, v26.4s\n" + "st1 { v24.4s }, [x12]\n" + "fmin v22.4s, v22.4s, v26.4s\n" + "fmin v21.4s, v21.4s, v26.4s\n" + "str q23, [x12, x14]\n" "add x12, x12, #0x10\n" - "st1 { v30.4s }, [x25]\n" - "str q31, [x25, x14]\n" + "st1 { v22.4s }, [x25]\n" + "str q21, [x25, x14]\n" "add x25, x25, #0x10\n" "4:" // Tile loop: Oddments "tst %x[n_channels], #0x3\n" "beq 31f\n" - "ldr q16, [x10, #0x0]\n" + "ldr q25, [x10, #0x0]\n" "ldr q0, [x10, #0x10]\n" "add x24, x9, x15\n" "add x23, x13, XZR\n" @@ -335,11 +335,11 @@ void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "ldr s12, [x21, #0x0]\n" "ldr s13, [x20, #0x0]\n" "6:" // Tile loop: Oddments: Load inputs: (1, 1), (0, 0), (0, 3), (1, 2), (2, 1): Bit 1: End - "mov v28.16b, v16.16b\n fmla v28.4s, v4.4s, v9.4s\n" - "mov v29.16b, v16.16b\n fmla v29.4s, v3.4s, v9.4s\n" + "mov v28.16b, v25.16b\n fmla v28.4s, v4.4s, v9.4s\n" + "mov v29.16b, v25.16b\n fmla v29.4s, v3.4s, v9.4s\n" "add x20, x27, XZR\n" - "mov v30.16b, v16.16b\n fmla v30.4s, v1.4s, v9.4s\n" - "mov v31.16b, v16.16b\n fmla v31.4s, v0.4s, v9.4s\n" + "mov v30.16b, v25.16b\n fmla v30.4s, v1.4s, v9.4s\n" + "mov v31.16b, v25.16b\n fmla v31.4s, v0.4s, v9.4s\n" "fmla v28.4s, v0.4s, v10.4s\n" "fmla v29.4s, v2.4s, v11.4s\n" "fmla v28.4s, v5.4s, v12.4s\n" @@ -470,14 +470,14 @@ void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "28:" // Tile loop: Oddments: Load inputs: (3, 2): Bit 1: End "fmla v30.4s, v8.4s, v12.4s\n" "fmla v31.4s, v7.4s, v12.4s\n" - "fmax v28.4s, v28.4s, v18.4s\n" - "fmax v29.4s, v29.4s, v18.4s\n" - "fmax v30.4s, v30.4s, v18.4s\n" - "fmax v31.4s, v31.4s, v18.4s\n" - "fmin v28.4s, v28.4s, v17.4s\n" - "fmin v29.4s, v29.4s, v17.4s\n" - "fmin v30.4s, v30.4s, v17.4s\n" - "fmin v31.4s, v31.4s, v17.4s\n" + "fmax v28.4s, v28.4s, v27.4s\n" + "fmax v29.4s, v29.4s, v27.4s\n" + "fmax v30.4s, v30.4s, v27.4s\n" + "fmax v31.4s, v31.4s, v27.4s\n" + "fmin v28.4s, v28.4s, v26.4s\n" + "fmin v29.4s, v29.4s, v26.4s\n" + "fmin v30.4s, v30.4s, v26.4s\n" + "fmin v31.4s, v31.4s, v26.4s\n" "tbz %x[n_channels], #1, 29f\n" "mov x21, x12\n" "mov x20, x25\n" @@ -503,7 +503,6 @@ void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "st1 { v29.s }[0], [x21]\n" "st1 { v31.s }[0], [x20]\n" "30:" // Tile loop: Oddments: Store: Bit 1: End - "31:" // Tile loop: End "ldr x22, [%x[params_struct], %[offsetof_args_tile_j]]\n" "ldr x23, [%x[params_struct], %[offsetof_args_tile_i]]\n" @@ -518,11 +517,11 @@ void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v16", "v17", "v18", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp index 56e9ed2e1ba347957096b8592902d3c6af7f1313..7dedfd972aa37b8a410dc6fb1cdabbf2fb8e374e 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ +#if defined(__aarch64__) namespace arm_conv { namespace depthwise { @@ -83,16 +83,16 @@ void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( "lsr x15, %x[n_channels], #0x2\n" "ldr x14, [%x[params_struct], %[offsetof_args_params]]\n" "add x20, %x[params_struct], %[offsetof_args_min]\n" - "ld1r { v18.4s }, [x20]\n" + "ld1r { v27.4s }, [x20]\n" "add x20, %x[params_struct], %[offsetof_args_max]\n" - "ld1r { v17.4s }, [x20]\n" + "ld1r { v26.4s }, [x20]\n" "add x13, %x[params_struct], %[offsetof_Args_inptrs]\n" "ldp x12, x11, [x21, #0x0]\n" "ldp x10, x9, [x21, #0x10]\n" "mov x28, #0x0\n" "sub x27, XZR, x16\n" "cbz x15, 3f\n" - "ldr q16, [x14, #0x0]\n" + "ldr q25, [x14, #0x0]\n" "ldr q0, [x14, #0x10]\n" "cmp x16, x15, LSL #4\n" "ldr q1, [x14, #0x20]\n" @@ -104,197 +104,197 @@ void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( "ldr q7, [x14, #0x80]\n" "ldr q8, [x14, #0x90]\n" "add x14, x14, #0xa0\n" - "ldp x26, x22, [x13, #0x0]\n" - "ldr q9, [x26, x28]\n" - "ldr q10, [x22, x28]\n" - "ldp x25, x24, [x13, #0x10]\n" - "ldr q11, [x25, x28]\n" - "ldr q12, [x24, x28]\n" - "ldr x23, [x13, #0x20]\n" - "ldr q13, [x23, x28]\n" - "bge 2f\n" - "1:" // Channel loop - "mov v28.16b, v16.16b\n fmla v28.4s, v4.4s, v9.4s\n" - "mov v29.16b, v16.16b\n fmla v29.4s, v3.4s, v9.4s\n" - "ldr x22, [x13, #0x28]\n" - "ldr x21, [x13, #0x30]\n" - "mov v30.16b, v16.16b\n fmla v30.4s, v1.4s, v9.4s\n" - "mov v31.16b, v16.16b\n fmla v31.4s, v0.4s, v9.4s\n" - "ldr q9, [x22, x28]\n" - "ldr q16, [x14, #0x0]\n" - "fmla v28.4s, v0.4s, v10.4s\n" - "fmla v29.4s, v2.4s, v11.4s\n" + "ldp x21, x20, [x13, #0x0]\n" + "ldr q9, [x21, x28]\n" + "ldr q10, [x20, x28]\n" + "ldp x21, x20, [x13, #0x10]\n" "ldr q11, [x21, x28]\n" - "ldr x20, [x13, #0x38]\n" - "fmla v30.4s, v2.4s, v12.4s\n" - "fmla v31.4s, v1.4s, v12.4s\n" - "ldr x22, [x13, #0x48]\n" - "ldr q10, [x22, x28]\n" - "fmla v28.4s, v5.4s, v12.4s\n" - "fmla v29.4s, v4.4s, v12.4s\n" "ldr q12, [x20, x28]\n" - "ldr x26, [x13, #0x40]\n" - "fmla v30.4s, v6.4s, v9.4s\n" - "ldr q9, [x26, x28]\n" - "fmla v31.4s, v3.4s, v13.4s\n" - "ldr x25, [x13, #0x50]\n" - "fmla v28.4s, v7.4s, v13.4s\n" - "fmla v29.4s, v6.4s, v13.4s\n" - "ldr x24, [x13, #0x58]\n" - "ldr x23, [x13, #0x60]\n" - "fmla v30.4s, v4.4s, v13.4s\n" - "fmla v31.4s, v8.4s, v11.4s\n" - "ldr q11, [x25, x28]\n" - "ldr x22, [x13, #0x68]\n" - "fmla v28.4s, v1.4s, v12.4s\n" - "fmla v29.4s, v0.4s, v12.4s\n" - "ldr q12, [x24, x28]\n" - "ldr x21, [x13, #0x70]\n" - "fmla v30.4s, v5.4s, v10.4s\n" - "fmla v31.4s, v4.4s, v10.4s\n" + "ldr x20, [x13, #0x20]\n" + "ldr q13, [x20, x28]\n" + "bge 2f\n" + "1:" // Channel loop + "mov v24.16b, v25.16b\n fmla v24.4s, v4.4s, v9.4s\n" + "mov v23.16b, v25.16b\n fmla v23.4s, v3.4s, v9.4s\n" + "ldr x21, [x13, #0x28]\n" + "ldr x20, [x13, #0x30]\n" + "mov v22.16b, v25.16b\n fmla v22.4s, v1.4s, v9.4s\n" + "mov v21.16b, v25.16b\n fmla v21.4s, v0.4s, v9.4s\n" + "ldr q18, [x21, x28]\n" + "ldr q25, [x14, #0x0]\n" + "fmla v24.4s, v0.4s, v10.4s\n" + "fmla v23.4s, v2.4s, v11.4s\n" + "ldr q17, [x20, x28]\n" + "ldr x21, [x13, #0x38]\n" + "fmla v22.4s, v2.4s, v12.4s\n" + "fmla v21.4s, v1.4s, v12.4s\n" + "ldr x20, [x13, #0x48]\n" + "ldr q20, [x20, x28]\n" + "fmla v24.4s, v5.4s, v12.4s\n" + "fmla v23.4s, v4.4s, v12.4s\n" + "ldr q16, [x21, x28]\n" + "ldr x20, [x13, #0x40]\n" + "fmla v22.4s, v6.4s, v18.4s\n" + "ldr q18, [x20, x28]\n" + "fmla v21.4s, v3.4s, v13.4s\n" + "ldr x20, [x13, #0x50]\n" + "fmla v24.4s, v7.4s, v13.4s\n" + "fmla v23.4s, v6.4s, v13.4s\n" + "ldr x22, [x13, #0x58]\n" + "ldr x21, [x13, #0x60]\n" + "fmla v22.4s, v4.4s, v13.4s\n" + "fmla v21.4s, v8.4s, v17.4s\n" + "ldr q17, [x20, x28]\n" + "ldr x20, [x13, #0x68]\n" + "fmla v24.4s, v1.4s, v16.4s\n" + "fmla v23.4s, v0.4s, v16.4s\n" + "ldr q16, [x22, x28]\n" + "ldr x26, [x13, #0x70]\n" + "fmla v22.4s, v5.4s, v20.4s\n" + "fmla v21.4s, v4.4s, v20.4s\n" "ldr q4, [x14, #0x50]\n" - "ldr x20, [x13, #0x78]\n" - "fmla v28.4s, v2.4s, v9.4s\n" - "fmla v29.4s, v1.4s, v9.4s\n" - "ldr q9, [x23, x28]\n" + "ldr x25, [x13, #0x78]\n" + "fmla v24.4s, v2.4s, v18.4s\n" + "fmla v23.4s, v1.4s, v18.4s\n" + "ldr q19, [x21, x28]\n" "ldr q1, [x14, #0x20]\n" - "fmla v30.4s, v0.4s, v11.4s\n" + "fmla v22.4s, v0.4s, v17.4s\n" "ldr q0, [x14, #0x10]\n" - "fmla v31.4s, v2.4s, v12.4s\n" + "fmla v21.4s, v2.4s, v16.4s\n" "ldr q2, [x14, #0x30]\n" - "fmla v28.4s, v8.4s, v10.4s\n" - "fmla v29.4s, v7.4s, v10.4s\n" - "ldr q10, [x22, x28]\n" - "ldp x26, x22, [x13, #0x0]\n" - "fmla v30.4s, v3.4s, v9.4s\n" - "fmla v31.4s, v5.4s, v10.4s\n" - "ldp x25, x24, [x13, #0x10]\n" - "ldr x23, [x13, #0x20]\n" - "ldr q13, [x23, x16]\n" - "fmla v28.4s, v3.4s, v11.4s\n" - "ldr q11, [x21, x28]\n" - "fmla v29.4s, v5.4s, v12.4s\n" - "ldr q12, [x20, x28]\n" + "fmla v24.4s, v8.4s, v20.4s\n" + "fmla v23.4s, v7.4s, v20.4s\n" + "ldr q18, [x20, x28]\n" + "ldp x24, x23, [x13, #0x0]\n" + "fmla v22.4s, v3.4s, v19.4s\n" + "fmla v21.4s, v5.4s, v18.4s\n" + "ldp x22, x21, [x13, #0x10]\n" + "ldr x20, [x13, #0x20]\n" + "ldr q13, [x20, x16]\n" + "fmla v24.4s, v3.4s, v17.4s\n" + "ldr q17, [x26, x28]\n" + "fmla v23.4s, v5.4s, v16.4s\n" + "ldr q16, [x25, x28]\n" "ldr q3, [x14, #0x40]\n" - "fmla v30.4s, v7.4s, v11.4s\n" - "fmla v31.4s, v6.4s, v11.4s\n" - "ldr q11, [x25, x16]\n" + "fmla v22.4s, v7.4s, v17.4s\n" + "fmla v21.4s, v6.4s, v17.4s\n" + "ldr q11, [x22, x16]\n" "ldr q5, [x14, #0x60]\n" - "fmla v28.4s, v6.4s, v9.4s\n" - "fmla v29.4s, v8.4s, v10.4s\n" - "ldr q9, [x26, x16]\n" - "ldr q10, [x22, x16]\n" - "fmla v30.4s, v8.4s, v12.4s\n" - "fmla v31.4s, v7.4s, v12.4s\n" - "ldr q12, [x24, x16]\n" + "fmla v24.4s, v6.4s, v19.4s\n" + "fmla v23.4s, v8.4s, v18.4s\n" + "ldr q9, [x24, x16]\n" + "ldr q10, [x23, x16]\n" + "fmla v22.4s, v8.4s, v16.4s\n" + "fmla v21.4s, v7.4s, v16.4s\n" + "ldr q12, [x21, x16]\n" "ldr q6, [x14, #0x70]\n" - "fmax v28.4s, v28.4s, v18.4s\n" - "fmax v29.4s, v29.4s, v18.4s\n" + "fmax v24.4s, v24.4s, v27.4s\n" + "fmax v23.4s, v23.4s, v27.4s\n" "ldr q7, [x14, #0x80]\n" "ldr q8, [x14, #0x90]\n" - "fmax v30.4s, v30.4s, v18.4s\n" - "fmax v31.4s, v31.4s, v18.4s\n" + "fmax v22.4s, v22.4s, v27.4s\n" + "fmax v21.4s, v21.4s, v27.4s\n" "add x16, x16, #0x10\n" "add x27, x27, #0x10\n" - "fmin v28.4s, v28.4s, v17.4s\n" - "fmin v29.4s, v29.4s, v17.4s\n" + "fmin v24.4s, v24.4s, v26.4s\n" + "fmin v23.4s, v23.4s, v26.4s\n" "cmp x16, x15, LSL #4\n" - "fmin v30.4s, v30.4s, v17.4s\n" - "fmin v31.4s, v31.4s, v17.4s\n" + "fmin v22.4s, v22.4s, v26.4s\n" + "fmin v21.4s, v21.4s, v26.4s\n" "add x28, x28, #0x10\n" - "str q28, [x12, x27]\n" + "str q24, [x12, x27]\n" "add x14, x14, #0xa0\n" - "str q29, [x11, x27]\n" - "str q30, [x10, x27]\n" - "str q31, [x9, x27]\n" + "str q23, [x11, x27]\n" + "str q22, [x10, x27]\n" + "str q21, [x9, x27]\n" "blt 1b\n" "2:" // Channel tail - "mov v28.16b, v16.16b\n fmla v28.4s, v4.4s, v9.4s\n" - "mov v29.16b, v16.16b\n fmla v29.4s, v3.4s, v9.4s\n" - "ldr x22, [x13, #0x28]\n" - "ldr x21, [x13, #0x30]\n" - "mov v30.16b, v16.16b\n fmla v30.4s, v1.4s, v9.4s\n" - "mov v31.16b, v16.16b\n fmla v31.4s, v0.4s, v9.4s\n" - "ldr q9, [x22, x28]\n" - "ldr x20, [x13, #0x38]\n" - "fmla v28.4s, v0.4s, v10.4s\n" - "fmla v29.4s, v2.4s, v11.4s\n" - "ldr q11, [x21, x28]\n" - "ldr x22, [x13, #0x48]\n" - "ldr q10, [x22, x28]\n" - "fmla v30.4s, v2.4s, v12.4s\n" - "fmla v31.4s, v1.4s, v12.4s\n" - "ldr x26, [x13, #0x40]\n" - "fmla v28.4s, v5.4s, v12.4s\n" - "fmla v29.4s, v4.4s, v12.4s\n" - "ldr q12, [x20, x28]\n" - "ldr x25, [x13, #0x50]\n" - "fmla v30.4s, v6.4s, v9.4s\n" - "ldr q9, [x26, x28]\n" - "fmla v31.4s, v3.4s, v13.4s\n" - "ldr x24, [x13, #0x58]\n" - "fmla v28.4s, v7.4s, v13.4s\n" - "fmla v29.4s, v6.4s, v13.4s\n" + "mov v24.16b, v25.16b\n fmla v24.4s, v4.4s, v9.4s\n" + "mov v23.16b, v25.16b\n fmla v23.4s, v3.4s, v9.4s\n" + "ldr x21, [x13, #0x28]\n" + "ldr x20, [x13, #0x30]\n" + "mov v22.16b, v25.16b\n fmla v22.4s, v1.4s, v9.4s\n" + "mov v21.16b, v25.16b\n fmla v21.4s, v0.4s, v9.4s\n" + "ldr q18, [x21, x28]\n" + "ldr x21, [x13, #0x38]\n" + "fmla v24.4s, v0.4s, v10.4s\n" + "fmla v23.4s, v2.4s, v11.4s\n" + "ldr q17, [x20, x28]\n" + "ldr x20, [x13, #0x48]\n" + "ldr q20, [x20, x28]\n" + "fmla v22.4s, v2.4s, v12.4s\n" + "fmla v21.4s, v1.4s, v12.4s\n" + "ldr x20, [x13, #0x40]\n" + "fmla v24.4s, v5.4s, v12.4s\n" + "fmla v23.4s, v4.4s, v12.4s\n" + "ldr q16, [x21, x28]\n" + "ldr x21, [x13, #0x50]\n" + "fmla v22.4s, v6.4s, v18.4s\n" + "ldr q18, [x20, x28]\n" + "fmla v21.4s, v3.4s, v13.4s\n" + "ldr x20, [x13, #0x58]\n" + "fmla v24.4s, v7.4s, v13.4s\n" + "fmla v23.4s, v6.4s, v13.4s\n" "ldr x23, [x13, #0x60]\n" "ldr x22, [x13, #0x68]\n" - "fmla v30.4s, v4.4s, v13.4s\n" - "fmla v31.4s, v8.4s, v11.4s\n" - "ldr q11, [x25, x28]\n" + "fmla v22.4s, v4.4s, v13.4s\n" + "fmla v21.4s, v8.4s, v17.4s\n" + "ldr q17, [x21, x28]\n" "ldr x21, [x13, #0x70]\n" - "fmla v28.4s, v1.4s, v12.4s\n" - "fmla v29.4s, v0.4s, v12.4s\n" - "ldr q12, [x24, x28]\n" + "fmla v24.4s, v1.4s, v16.4s\n" + "fmla v23.4s, v0.4s, v16.4s\n" + "ldr q16, [x20, x28]\n" "ldr x20, [x13, #0x78]\n" - "fmla v30.4s, v5.4s, v10.4s\n" - "fmla v31.4s, v4.4s, v10.4s\n" + "fmla v22.4s, v5.4s, v20.4s\n" + "fmla v21.4s, v4.4s, v20.4s\n" "add x27, x27, #0x10\n" - "fmla v28.4s, v2.4s, v9.4s\n" - "fmla v29.4s, v1.4s, v9.4s\n" - "ldr q9, [x23, x28]\n" - "fmla v30.4s, v0.4s, v11.4s\n" - "fmla v31.4s, v2.4s, v12.4s\n" - "fmla v28.4s, v8.4s, v10.4s\n" - "fmla v29.4s, v7.4s, v10.4s\n" - "ldr q10, [x22, x28]\n" - "fmla v30.4s, v3.4s, v9.4s\n" - "fmla v31.4s, v5.4s, v10.4s\n" - "fmla v28.4s, v3.4s, v11.4s\n" - "ldr q11, [x21, x28]\n" - "fmla v29.4s, v5.4s, v12.4s\n" - "ldr q12, [x20, x28]\n" - "fmla v30.4s, v7.4s, v11.4s\n" - "fmla v31.4s, v6.4s, v11.4s\n" + "fmla v24.4s, v2.4s, v18.4s\n" + "fmla v23.4s, v1.4s, v18.4s\n" + "ldr q19, [x23, x28]\n" + "fmla v22.4s, v0.4s, v17.4s\n" + "fmla v21.4s, v2.4s, v16.4s\n" + "fmla v24.4s, v8.4s, v20.4s\n" + "fmla v23.4s, v7.4s, v20.4s\n" + "ldr q18, [x22, x28]\n" + "fmla v22.4s, v3.4s, v19.4s\n" + "fmla v21.4s, v5.4s, v18.4s\n" + "fmla v24.4s, v3.4s, v17.4s\n" + "ldr q17, [x21, x28]\n" + "fmla v23.4s, v5.4s, v16.4s\n" + "ldr q16, [x20, x28]\n" + "fmla v22.4s, v7.4s, v17.4s\n" + "fmla v21.4s, v6.4s, v17.4s\n" "add x28, x28, #0x10\n" - "fmla v28.4s, v6.4s, v9.4s\n" - "fmla v29.4s, v8.4s, v10.4s\n" - "fmax v28.4s, v28.4s, v18.4s\n" - "fmla v30.4s, v8.4s, v12.4s\n" - "fmla v31.4s, v7.4s, v12.4s\n" - "fmax v29.4s, v29.4s, v18.4s\n" - "fmax v30.4s, v30.4s, v18.4s\n" - "fmax v31.4s, v31.4s, v18.4s\n" - "fmin v28.4s, v28.4s, v17.4s\n" - "fmin v29.4s, v29.4s, v17.4s\n" - "str q28, [x12, x27]\n" - "fmin v30.4s, v30.4s, v17.4s\n" - "fmin v31.4s, v31.4s, v17.4s\n" - "str q29, [x11, x27]\n" - "str q30, [x10, x27]\n" - "str q31, [x9, x27]\n" + "fmla v24.4s, v6.4s, v19.4s\n" + "fmla v23.4s, v8.4s, v18.4s\n" + "fmax v24.4s, v24.4s, v27.4s\n" + "fmla v22.4s, v8.4s, v16.4s\n" + "fmla v21.4s, v7.4s, v16.4s\n" + "fmax v23.4s, v23.4s, v27.4s\n" + "fmax v22.4s, v22.4s, v27.4s\n" + "fmax v21.4s, v21.4s, v27.4s\n" + "fmin v24.4s, v24.4s, v26.4s\n" + "fmin v23.4s, v23.4s, v26.4s\n" + "str q24, [x12, x27]\n" + "fmin v22.4s, v22.4s, v26.4s\n" + "fmin v21.4s, v21.4s, v26.4s\n" + "str q23, [x11, x27]\n" + "str q22, [x10, x27]\n" + "str q21, [x9, x27]\n" "3:" // Oddments "tst %x[n_channels], #0x3\n" "beq 30f\n" - "ldr q16, [x14, #0x0]\n" + "ldr q25, [x14, #0x0]\n" "ldr q0, [x14, #0x10]\n" - "mov x27, x28\n" - "add x12, x12, x27\n" + "mov x20, x28\n" + "add x12, x12, x20\n" "ldr q1, [x14, #0x20]\n" "ldr q2, [x14, #0x30]\n" - "add x11, x11, x27\n" - "add x10, x10, x27\n" + "add x11, x11, x20\n" + "add x10, x10, x20\n" "ldr q3, [x14, #0x40]\n" "ldr q4, [x14, #0x50]\n" - "add x9, x9, x27\n" + "add x9, x9, x20\n" "ldr q5, [x14, #0x60]\n" "ldr q6, [x14, #0x70]\n" "ldr q7, [x14, #0x80]\n" @@ -329,12 +329,12 @@ void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( "ld1 { v12.s }[0], [x21], #0x4\n" "ld1 { v13.s }[0], [x20], #0x4\n" "5:" // Oddments: Load inputs (1, 1), (0, 0), (0, 3), (1, 2), (2, 1): Bit 1: End - "mov v28.16b, v16.16b\n fmla v28.4s, v4.4s, v9.4s\n" - "mov v29.16b, v16.16b\n fmla v29.4s, v3.4s, v9.4s\n" + "mov v28.16b, v25.16b\n fmla v28.4s, v4.4s, v9.4s\n" + "mov v29.16b, v25.16b\n fmla v29.4s, v3.4s, v9.4s\n" "ldr x20, [x13, #0x28]\n" "add x20, x20, x28\n" - "mov v30.16b, v16.16b\n fmla v30.4s, v1.4s, v9.4s\n" - "mov v31.16b, v16.16b\n fmla v31.4s, v0.4s, v9.4s\n" + "mov v30.16b, v25.16b\n fmla v30.4s, v1.4s, v9.4s\n" + "mov v31.16b, v25.16b\n fmla v31.4s, v0.4s, v9.4s\n" "fmla v28.4s, v0.4s, v10.4s\n" "fmla v29.4s, v2.4s, v11.4s\n" "fmla v28.4s, v5.4s, v12.4s\n" @@ -475,14 +475,14 @@ void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( "27:" // Oddments: Load input (3, 2): Bit 1: End "fmla v30.4s, v8.4s, v12.4s\n" "fmla v31.4s, v7.4s, v12.4s\n" - "fmax v28.4s, v28.4s, v18.4s\n" - "fmax v29.4s, v29.4s, v18.4s\n" - "fmax v30.4s, v30.4s, v18.4s\n" - "fmax v31.4s, v31.4s, v18.4s\n" - "fmin v28.4s, v28.4s, v17.4s\n" - "fmin v29.4s, v29.4s, v17.4s\n" - "fmin v30.4s, v30.4s, v17.4s\n" - "fmin v31.4s, v31.4s, v17.4s\n" + "fmax v28.4s, v28.4s, v27.4s\n" + "fmax v29.4s, v29.4s, v27.4s\n" + "fmax v30.4s, v30.4s, v27.4s\n" + "fmax v31.4s, v31.4s, v27.4s\n" + "fmin v28.4s, v28.4s, v26.4s\n" + "fmin v29.4s, v29.4s, v26.4s\n" + "fmin v30.4s, v30.4s, v26.4s\n" + "fmin v31.4s, v31.4s, v26.4s\n" "tbz %x[n_channels], #1, 28f\n" "st1 { v28.d }[0], [x12], #0x8\n" "st1 { v29.d }[0], [x11], #0x8\n" @@ -503,11 +503,11 @@ void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( "30:" // End : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v16", "v17", "v18", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp index 0e9a3ba3fced003129a7fc3ca701339302bb7a12..c2d86615e3d8712c231c5dfa336b39fb21927ebe 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include @@ -33,8 +33,8 @@ namespace arm_conv { namespace depthwise { -void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); -void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); +void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(const float *const *const input_ptrs, float *const *const outptrs, const void *params, unsigned int n_channels, const float activation_min, const float activation_max); +void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(const unsigned int n_tile_rows, const unsigned int n_tile_cols, const float *inptr, int64_t ld_input_row, int64_t ld_input_col, float *outptr, int64_t ld_output_row, int64_t ld_output_col, const void *params, unsigned int n_channels, const float activation_min, const float activation_max); class a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst : public DepthwiseDepthfirstStrategy { @@ -57,7 +57,7 @@ class a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst : public DepthwiseDepthfirst constexpr static unsigned int output_cols = 3; a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst(const CPUInfo *) - : DepthwiseDepthfirstStrategy(3, 3, 1) {} + : Parent(output_rows, output_cols, kernel_rows, kernel_cols, stride_rows, stride_cols) {} arm_gemm::VLType get_vl_type(void) const override { return vl_type; } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp index 620319bc7c1ebc5f783e443365029e94dee34aa7..9bfcd9cd3c0b3f3e9865d173dad6690eb32b8ad1 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ +#if defined(__aarch64__) namespace arm_conv { namespace depthwise { @@ -110,7 +110,7 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "lsr x23, %x[n_channels], #0x2\n" "ldr x14, [%x[params_struct], %[offsetof_args_params]]\n" "mul x21, x21, x27\n" // offset *= kernel_stride * output_size - "add x16, x16, x21, LSL #2\n" // inptr[0] += offset * sizeof(float) + "add x16, x16, x21, LSL #2\n" // inptr[0] += offset * sizeof(float) "add x13, x16, x25, LSL #2\n" "mul x20, x20, x26\n" // offset *= output_tile_size "add x12, x13, x25, LSL #2\n" @@ -120,9 +120,9 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "add x9, x11, x8\n" "add x28, x15, x22, LSL #2\n" "add x20, %x[params_struct], %[offsetof_args_min]\n" - "ld1r { v18.4s }, [x20]\n" + "ld1r { v15.4s }, [x20]\n" "add x20, %x[params_struct], %[offsetof_args_max]\n" - "ld1r { v17.4s }, [x20]\n" + "ld1r { v14.4s }, [x20]\n" "add x27, x10, x25, LSL #2\n" "add x26, x9, x8\n" "add x25, x28, x22, LSL #2\n" @@ -130,7 +130,7 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "mov x21, #0x0\n" "sub x20, XZR, x24\n" "cbz x23, 4f\n" - "ldr q16, [x14, #0x0]\n" + "ldr q31, [x14, #0x0]\n" "ldr q0, [x14, #0x10]\n" "cmp x24, x23, LSL #4\n" "ldr q1, [x14, #0x20]\n" @@ -149,304 +149,304 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "ldr q13, [x13, x11]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "mov v24.16b, v16.16b\n fmla v24.4s, v7.4s, v9.4s\n" - "mov v23.16b, v16.16b\n fmla v23.4s, v8.4s, v9.4s\n" + "mov v29.16b, v31.16b\n fmla v29.4s, v7.4s, v9.4s\n" + "mov v28.16b, v31.16b\n fmla v28.4s, v8.4s, v9.4s\n" "add x24, x24, #0x10\n" "cmp x24, x23, LSL #4\n" - "mov v25.16b, v16.16b\n fmla v25.4s, v6.4s, v9.4s\n" - "fmla v24.4s, v4.4s, v13.4s\n" + "mov v27.16b, v31.16b\n fmla v27.4s, v6.4s, v9.4s\n" + "fmla v29.4s, v4.4s, v13.4s\n" "add x20, x20, #0x10\n" "add x21, x21, #0x10\n" - "mov v26.16b, v16.16b\n fmla v26.4s, v5.4s, v9.4s\n" - "mov v27.16b, v16.16b\n fmla v27.4s, v4.4s, v9.4s\n" - "mov v28.16b, v16.16b\n fmla v28.4s, v3.4s, v9.4s\n" - "fmla v23.4s, v0.4s, v10.4s\n" - "ldr q10, [x12, x9]\n" - "fmla v25.4s, v2.4s, v11.4s\n" - "ldr q11, [x12, x8]\n" - "mov v29.16b, v16.16b\n fmla v29.4s, v2.4s, v9.4s\n" - "fmla v24.4s, v6.4s, v11.4s\n" - "mov v31.16b, v16.16b\n fmla v31.4s, v0.4s, v9.4s\n" - "fmla v23.4s, v5.4s, v13.4s\n" - "fmla v25.4s, v3.4s, v13.4s\n" + "mov v26.16b, v31.16b\n fmla v26.4s, v5.4s, v9.4s\n" + "mov v25.16b, v31.16b\n fmla v25.4s, v4.4s, v9.4s\n" + "mov v24.16b, v31.16b\n fmla v24.4s, v3.4s, v9.4s\n" + "fmla v28.4s, v0.4s, v10.4s\n" + "ldr q23, [x12, x9]\n" + "fmla v27.4s, v2.4s, v11.4s\n" + "ldr q18, [x12, x8]\n" + "mov v22.16b, v31.16b\n fmla v22.4s, v2.4s, v9.4s\n" + "fmla v29.4s, v6.4s, v18.4s\n" + "mov v21.16b, v31.16b\n fmla v21.4s, v0.4s, v9.4s\n" + "fmla v28.4s, v5.4s, v13.4s\n" + "fmla v27.4s, v3.4s, v13.4s\n" "fmla v26.4s, v2.4s, v13.4s\n" - "fmla v27.4s, v1.4s, v13.4s\n" - "fmla v28.4s, v0.4s, v13.4s\n" - "ldr q13, [x16, x8]\n" - "fmla v29.4s, v6.4s, v12.4s\n" - "ldr q12, [x27, x26]\n" - "mov v30.16b, v16.16b\n fmla v30.4s, v1.4s, v9.4s\n" - "ldr q16, [x14, #0x0]\n" + "fmla v25.4s, v1.4s, v13.4s\n" "fmla v24.4s, v0.4s, v13.4s\n" - "fmla v31.4s, v8.4s, v12.4s\n" - "ldr q12, [x16, x9]\n" - "fmla v23.4s, v7.4s, v11.4s\n" - "fmla v30.4s, v0.4s, v11.4s\n" - "fmla v26.4s, v4.4s, v11.4s\n" - "fmla v27.4s, v3.4s, v11.4s\n" - "fmla v29.4s, v1.4s, v11.4s\n" - "ld1 { v11.4s }, [x13]\n" - "fmla v24.4s, v2.4s, v12.4s\n" - "fmla v25.4s, v1.4s, v12.4s\n" - "ld1 { v12.4s }, [x10]\n" - "fmla v28.4s, v4.4s, v10.4s\n" - "fmla v23.4s, v1.4s, v13.4s\n" - "ldr q13, [x13, x26]\n" - "fmla v30.4s, v2.4s, v10.4s\n" - "fmla v31.4s, v1.4s, v10.4s\n" - "fmla v24.4s, v8.4s, v10.4s\n" - "fmla v25.4s, v7.4s, v10.4s\n" - "fmla v27.4s, v5.4s, v10.4s\n" - "ldr q10, [x10, x11]\n" - "fmla v26.4s, v0.4s, v11.4s\n" - "fmla v29.4s, v3.4s, v12.4s\n" - "fmla v28.4s, v2.4s, v13.4s\n" - "fmla v30.4s, v4.4s, v10.4s\n" - "fmla v31.4s, v3.4s, v10.4s\n" - "fmla v23.4s, v3.4s, v11.4s\n" - "ldr q11, [x10, x26]\n" - "fmla v25.4s, v5.4s, v13.4s\n" - "ldr q13, [x27, x8]\n" - "fmla v26.4s, v6.4s, v12.4s\n" - "ldr q12, [x13, x8]\n" - "fmla v27.4s, v7.4s, v10.4s\n" - "fmla v29.4s, v5.4s, v10.4s\n" - "fmla v28.4s, v6.4s, v10.4s\n" - "fmla v31.4s, v5.4s, v11.4s\n" - "fmla v30.4s, v6.4s, v13.4s\n" - "fmla v26.4s, v8.4s, v10.4s\n" - "fmla v29.4s, v7.4s, v13.4s\n" - "ldr q13, [x27, x9]\n" - "fmla v24.4s, v3.4s, v12.4s\n" - "fmla v27.4s, v0.4s, v12.4s\n" - "fmla v28.4s, v8.4s, v11.4s\n" - "ldr q11, [x13, x9]\n" - "fmla v30.4s, v8.4s, v13.4s\n" + "ldr q17, [x16, x8]\n" + "fmla v22.4s, v6.4s, v12.4s\n" + "ldr q16, [x27, x26]\n" + "mov v20.16b, v31.16b\n fmla v20.4s, v1.4s, v9.4s\n" + "ldr q31, [x14, #0x0]\n" + "fmla v29.4s, v0.4s, v17.4s\n" + "fmla v21.4s, v8.4s, v16.4s\n" + "ldr q16, [x16, x9]\n" + "fmla v28.4s, v7.4s, v18.4s\n" + "fmla v20.4s, v0.4s, v18.4s\n" + "fmla v26.4s, v4.4s, v18.4s\n" + "fmla v25.4s, v3.4s, v18.4s\n" + "fmla v22.4s, v1.4s, v18.4s\n" + "ld1 { v19.4s }, [x13]\n" + "fmla v29.4s, v2.4s, v16.4s\n" + "fmla v27.4s, v1.4s, v16.4s\n" + "ld1 { v18.4s }, [x10]\n" + "fmla v24.4s, v4.4s, v23.4s\n" + "fmla v28.4s, v1.4s, v17.4s\n" + "ldr q16, [x13, x26]\n" + "fmla v20.4s, v2.4s, v23.4s\n" + "fmla v21.4s, v1.4s, v23.4s\n" + "fmla v29.4s, v8.4s, v23.4s\n" + "fmla v27.4s, v7.4s, v23.4s\n" + "fmla v25.4s, v5.4s, v23.4s\n" + "ldr q17, [x10, x11]\n" + "fmla v26.4s, v0.4s, v19.4s\n" + "fmla v22.4s, v3.4s, v18.4s\n" + "fmla v24.4s, v2.4s, v16.4s\n" + "fmla v20.4s, v4.4s, v17.4s\n" + "fmla v21.4s, v3.4s, v17.4s\n" + "fmla v28.4s, v3.4s, v19.4s\n" + "ldr q19, [x10, x26]\n" + "fmla v27.4s, v5.4s, v16.4s\n" + "ldr q16, [x27, x8]\n" + "fmla v26.4s, v6.4s, v18.4s\n" + "ldr q18, [x13, x8]\n" + "fmla v25.4s, v7.4s, v17.4s\n" + "fmla v22.4s, v5.4s, v17.4s\n" + "fmla v24.4s, v6.4s, v17.4s\n" + "fmla v21.4s, v5.4s, v19.4s\n" + "fmla v20.4s, v6.4s, v16.4s\n" + "fmla v26.4s, v8.4s, v17.4s\n" + "fmla v22.4s, v7.4s, v16.4s\n" + "ldr q17, [x27, x9]\n" + "fmla v29.4s, v3.4s, v18.4s\n" + "fmla v25.4s, v0.4s, v18.4s\n" + "fmla v24.4s, v8.4s, v19.4s\n" + "ldr q16, [x13, x9]\n" + "fmla v20.4s, v8.4s, v17.4s\n" "add x13, x13, #0x10\n" - "fmla v31.4s, v7.4s, v13.4s\n" - "ldr q13, [x10, x9]\n" - "fmla v23.4s, v4.4s, v12.4s\n" - "fmla v26.4s, v1.4s, v12.4s\n" - "ldr q12, [x10, x8]\n" - "fmla v24.4s, v5.4s, v11.4s\n" + "fmla v21.4s, v7.4s, v17.4s\n" + "ldr q19, [x10, x9]\n" + "fmla v28.4s, v4.4s, v18.4s\n" + "fmla v26.4s, v1.4s, v18.4s\n" + "ldr q17, [x10, x8]\n" + "fmla v29.4s, v5.4s, v16.4s\n" "add x10, x10, #0x10\n" - "fmla v25.4s, v4.4s, v11.4s\n" - "fmla v27.4s, v2.4s, v11.4s\n" - "fmla v28.4s, v1.4s, v11.4s\n" - "ldr q11, [x16, x11]\n" - "fmla v29.4s, v4.4s, v12.4s\n" + "fmla v27.4s, v4.4s, v16.4s\n" + "fmla v25.4s, v2.4s, v16.4s\n" + "fmla v24.4s, v1.4s, v16.4s\n" + "ldr q16, [x16, x11]\n" + "fmla v22.4s, v4.4s, v17.4s\n" "add x16, x16, #0x10\n" "ld1 { v10.4s }, [x16]\n" - "fmla v30.4s, v3.4s, v12.4s\n" - "fmla v31.4s, v4.4s, v13.4s\n" + "fmla v20.4s, v3.4s, v17.4s\n" + "fmla v21.4s, v4.4s, v19.4s\n" "ldr q4, [x14, #0x50]\n" - "fmla v26.4s, v7.4s, v12.4s\n" - "fmla v27.4s, v6.4s, v12.4s\n" - "ld1 { v12.4s }, [x12]\n" - "fmla v23.4s, v2.4s, v11.4s\n" - "fmla v24.4s, v1.4s, v11.4s\n" + "fmla v26.4s, v7.4s, v17.4s\n" + "fmla v25.4s, v6.4s, v17.4s\n" + "ld1 { v18.4s }, [x12]\n" + "fmla v28.4s, v2.4s, v16.4s\n" + "fmla v29.4s, v1.4s, v16.4s\n" "ldr q1, [x14, #0x20]\n" - "fmax v24.4s, v24.4s, v18.4s\n" - "fmla v25.4s, v0.4s, v11.4s\n" - "ldr q11, [x12, x26]\n" - "fmla v28.4s, v7.4s, v13.4s\n" + "fmax v29.4s, v29.4s, v15.4s\n" + "fmla v27.4s, v0.4s, v16.4s\n" + "ldr q17, [x12, x26]\n" + "fmla v24.4s, v7.4s, v19.4s\n" "add x12, x12, #0x10\n" "ldr q9, [x12, x11]\n" - "fmla v30.4s, v5.4s, v13.4s\n" - "fmla v29.4s, v0.4s, v12.4s\n" + "fmla v20.4s, v5.4s, v19.4s\n" + "fmla v22.4s, v0.4s, v18.4s\n" "ldr q0, [x14, #0x10]\n" - "fmla v31.4s, v2.4s, v11.4s\n" + "fmla v21.4s, v2.4s, v17.4s\n" "ldr q2, [x14, #0x30]\n" - "fmla v27.4s, v8.4s, v13.4s\n" - "ldr q13, [x27, x11]\n" - "fmla v23.4s, v6.4s, v12.4s\n" - "fmla v26.4s, v3.4s, v12.4s\n" + "fmla v25.4s, v8.4s, v19.4s\n" + "ldr q16, [x27, x11]\n" + "fmla v28.4s, v6.4s, v18.4s\n" + "fmla v26.4s, v3.4s, v18.4s\n" "ldr q3, [x14, #0x40]\n" - "fmax v23.4s, v23.4s, v18.4s\n" - "fmla v25.4s, v8.4s, v11.4s\n" - "fmla v28.4s, v5.4s, v11.4s\n" + "fmax v28.4s, v28.4s, v15.4s\n" + "fmla v27.4s, v8.4s, v17.4s\n" + "fmla v24.4s, v5.4s, v17.4s\n" "ldr q11, [x16, x26]\n" "ldr q5, [x14, #0x60]\n" - "fmla v29.4s, v8.4s, v13.4s\n" + "fmla v22.4s, v8.4s, v16.4s\n" "ldr q8, [x14, #0x90]\n" - "fmla v30.4s, v7.4s, v13.4s\n" + "fmla v20.4s, v7.4s, v16.4s\n" "ldr q7, [x14, #0x80]\n" - "fmla v31.4s, v6.4s, v13.4s\n" + "fmla v21.4s, v6.4s, v16.4s\n" "ldr q13, [x13, x11]\n" "ldr q6, [x14, #0x70]\n" - "fmax v25.4s, v25.4s, v18.4s\n" - "fmax v26.4s, v26.4s, v18.4s\n" - "fmax v27.4s, v27.4s, v18.4s\n" + "fmax v27.4s, v27.4s, v15.4s\n" + "fmax v26.4s, v26.4s, v15.4s\n" + "fmax v25.4s, v25.4s, v15.4s\n" "add x27, x27, #0x10\n" "ld1 { v12.4s }, [x27]\n" - "fmax v28.4s, v28.4s, v18.4s\n" - "fmax v29.4s, v29.4s, v18.4s\n" + "fmax v24.4s, v24.4s, v15.4s\n" + "fmax v22.4s, v22.4s, v15.4s\n" "add x14, x14, #0xa0\n" - "fmax v30.4s, v30.4s, v18.4s\n" - "fmax v31.4s, v31.4s, v18.4s\n" - "fmin v23.4s, v23.4s, v17.4s\n" - "fmin v24.4s, v24.4s, v17.4s\n" - "st1 { v23.4s }, [x15]\n" - "fmin v25.4s, v25.4s, v17.4s\n" - "fmin v26.4s, v26.4s, v17.4s\n" - "str q24, [x15, x17]\n" - "fmin v27.4s, v27.4s, v17.4s\n" - "fmin v28.4s, v28.4s, v17.4s\n" - "str q25, [x15, x22]\n" + "fmax v20.4s, v20.4s, v15.4s\n" + "fmax v21.4s, v21.4s, v15.4s\n" + "fmin v28.4s, v28.4s, v14.4s\n" + "fmin v29.4s, v29.4s, v14.4s\n" + "st1 { v28.4s }, [x15]\n" + "fmin v27.4s, v27.4s, v14.4s\n" + "fmin v26.4s, v26.4s, v14.4s\n" + "str q29, [x15, x17]\n" + "fmin v25.4s, v25.4s, v14.4s\n" + "fmin v24.4s, v24.4s, v14.4s\n" + "str q27, [x15, x22]\n" "add x15, x15, #0x10\n" - "fmin v29.4s, v29.4s, v17.4s\n" - "fmin v30.4s, v30.4s, v17.4s\n" + "fmin v22.4s, v22.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v14.4s\n" "st1 { v26.4s }, [x28]\n" - "fmin v31.4s, v31.4s, v17.4s\n" - "str q27, [x28, x17]\n" - "str q28, [x28, x22]\n" + "fmin v21.4s, v21.4s, v14.4s\n" + "str q25, [x28, x17]\n" + "str q24, [x28, x22]\n" "add x28, x28, #0x10\n" - "st1 { v29.4s }, [x25]\n" - "str q30, [x25, x17]\n" - "str q31, [x25, x22]\n" + "st1 { v22.4s }, [x25]\n" + "str q20, [x25, x17]\n" + "str q21, [x25, x22]\n" "add x25, x25, #0x10\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "mov v24.16b, v16.16b\n fmla v24.4s, v7.4s, v9.4s\n" - "mov v23.16b, v16.16b\n fmla v23.4s, v8.4s, v9.4s\n" - "mov v25.16b, v16.16b\n fmla v25.4s, v6.4s, v9.4s\n" - "fmla v24.4s, v4.4s, v13.4s\n" - "mov v26.16b, v16.16b\n fmla v26.4s, v5.4s, v9.4s\n" - "mov v27.16b, v16.16b\n fmla v27.4s, v4.4s, v9.4s\n" - "mov v28.16b, v16.16b\n fmla v28.4s, v3.4s, v9.4s\n" - "fmla v23.4s, v0.4s, v10.4s\n" - "ldr q10, [x12, x9]\n" - "fmla v25.4s, v2.4s, v11.4s\n" - "ldr q11, [x12, x8]\n" - "mov v29.16b, v16.16b\n fmla v29.4s, v2.4s, v9.4s\n" - "fmla v24.4s, v6.4s, v11.4s\n" - "mov v31.16b, v16.16b\n fmla v31.4s, v0.4s, v9.4s\n" - "fmla v23.4s, v5.4s, v13.4s\n" - "fmla v25.4s, v3.4s, v13.4s\n" + "mov v29.16b, v31.16b\n fmla v29.4s, v7.4s, v9.4s\n" + "mov v28.16b, v31.16b\n fmla v28.4s, v8.4s, v9.4s\n" + "mov v27.16b, v31.16b\n fmla v27.4s, v6.4s, v9.4s\n" + "fmla v29.4s, v4.4s, v13.4s\n" + "mov v26.16b, v31.16b\n fmla v26.4s, v5.4s, v9.4s\n" + "mov v25.16b, v31.16b\n fmla v25.4s, v4.4s, v9.4s\n" + "mov v24.16b, v31.16b\n fmla v24.4s, v3.4s, v9.4s\n" + "fmla v28.4s, v0.4s, v10.4s\n" + "ldr q23, [x12, x9]\n" + "fmla v27.4s, v2.4s, v11.4s\n" + "ldr q18, [x12, x8]\n" + "mov v22.16b, v31.16b\n fmla v22.4s, v2.4s, v9.4s\n" + "fmla v29.4s, v6.4s, v18.4s\n" + "mov v21.16b, v31.16b\n fmla v21.4s, v0.4s, v9.4s\n" + "fmla v28.4s, v5.4s, v13.4s\n" + "fmla v27.4s, v3.4s, v13.4s\n" "fmla v26.4s, v2.4s, v13.4s\n" - "fmla v27.4s, v1.4s, v13.4s\n" - "fmla v28.4s, v0.4s, v13.4s\n" - "ldr q13, [x16, x8]\n" - "fmla v29.4s, v6.4s, v12.4s\n" - "ldr q12, [x27, x26]\n" - "mov v30.16b, v16.16b\n fmla v30.4s, v1.4s, v9.4s\n" + "fmla v25.4s, v1.4s, v13.4s\n" "fmla v24.4s, v0.4s, v13.4s\n" - "fmla v31.4s, v8.4s, v12.4s\n" - "ldr q12, [x16, x9]\n" - "fmla v23.4s, v7.4s, v11.4s\n" - "fmla v30.4s, v0.4s, v11.4s\n" - "fmla v26.4s, v4.4s, v11.4s\n" - "fmla v27.4s, v3.4s, v11.4s\n" - "fmla v29.4s, v1.4s, v11.4s\n" - "ld1 { v11.4s }, [x13]\n" - "fmla v24.4s, v2.4s, v12.4s\n" - "fmla v25.4s, v1.4s, v12.4s\n" - "ld1 { v12.4s }, [x10]\n" - "fmla v28.4s, v4.4s, v10.4s\n" - "fmla v23.4s, v1.4s, v13.4s\n" - "ldr q13, [x13, x26]\n" - "fmla v30.4s, v2.4s, v10.4s\n" - "fmla v31.4s, v1.4s, v10.4s\n" - "fmla v24.4s, v8.4s, v10.4s\n" - "fmla v25.4s, v7.4s, v10.4s\n" - "fmla v27.4s, v5.4s, v10.4s\n" - "ldr q10, [x10, x11]\n" - "fmla v26.4s, v0.4s, v11.4s\n" - "fmla v29.4s, v3.4s, v12.4s\n" - "fmla v28.4s, v2.4s, v13.4s\n" - "fmla v30.4s, v4.4s, v10.4s\n" - "fmla v31.4s, v3.4s, v10.4s\n" - "fmla v23.4s, v3.4s, v11.4s\n" - "ldr q11, [x10, x26]\n" - "fmla v25.4s, v5.4s, v13.4s\n" - "ldr q13, [x27, x8]\n" - "fmla v26.4s, v6.4s, v12.4s\n" - "ldr q12, [x13, x8]\n" - "fmla v27.4s, v7.4s, v10.4s\n" - "fmla v29.4s, v5.4s, v10.4s\n" - "fmla v28.4s, v6.4s, v10.4s\n" - "fmla v31.4s, v5.4s, v11.4s\n" - "fmla v30.4s, v6.4s, v13.4s\n" - "fmla v26.4s, v8.4s, v10.4s\n" - "fmla v29.4s, v7.4s, v13.4s\n" - "ldr q13, [x27, x9]\n" - "fmla v24.4s, v3.4s, v12.4s\n" - "fmla v27.4s, v0.4s, v12.4s\n" - "fmla v28.4s, v8.4s, v11.4s\n" - "ldr q11, [x13, x9]\n" - "fmla v30.4s, v8.4s, v13.4s\n" + "ldr q17, [x16, x8]\n" + "fmla v22.4s, v6.4s, v12.4s\n" + "ldr q16, [x27, x26]\n" + "mov v20.16b, v31.16b\n fmla v20.4s, v1.4s, v9.4s\n" + "fmla v29.4s, v0.4s, v17.4s\n" + "fmla v21.4s, v8.4s, v16.4s\n" + "ldr q16, [x16, x9]\n" + "fmla v28.4s, v7.4s, v18.4s\n" + "fmla v20.4s, v0.4s, v18.4s\n" + "fmla v26.4s, v4.4s, v18.4s\n" + "fmla v25.4s, v3.4s, v18.4s\n" + "fmla v22.4s, v1.4s, v18.4s\n" + "ld1 { v19.4s }, [x13]\n" + "fmla v29.4s, v2.4s, v16.4s\n" + "fmla v27.4s, v1.4s, v16.4s\n" + "ld1 { v18.4s }, [x10]\n" + "fmla v24.4s, v4.4s, v23.4s\n" + "fmla v28.4s, v1.4s, v17.4s\n" + "ldr q16, [x13, x26]\n" + "fmla v20.4s, v2.4s, v23.4s\n" + "fmla v21.4s, v1.4s, v23.4s\n" + "fmla v29.4s, v8.4s, v23.4s\n" + "fmla v27.4s, v7.4s, v23.4s\n" + "fmla v25.4s, v5.4s, v23.4s\n" + "ldr q17, [x10, x11]\n" + "fmla v26.4s, v0.4s, v19.4s\n" + "fmla v22.4s, v3.4s, v18.4s\n" + "fmla v24.4s, v2.4s, v16.4s\n" + "fmla v20.4s, v4.4s, v17.4s\n" + "fmla v21.4s, v3.4s, v17.4s\n" + "fmla v28.4s, v3.4s, v19.4s\n" + "ldr q19, [x10, x26]\n" + "fmla v27.4s, v5.4s, v16.4s\n" + "ldr q16, [x27, x8]\n" + "fmla v26.4s, v6.4s, v18.4s\n" + "ldr q18, [x13, x8]\n" + "fmla v25.4s, v7.4s, v17.4s\n" + "fmla v22.4s, v5.4s, v17.4s\n" + "fmla v24.4s, v6.4s, v17.4s\n" + "fmla v21.4s, v5.4s, v19.4s\n" + "fmla v20.4s, v6.4s, v16.4s\n" + "fmla v26.4s, v8.4s, v17.4s\n" + "fmla v22.4s, v7.4s, v16.4s\n" + "ldr q17, [x27, x9]\n" + "fmla v29.4s, v3.4s, v18.4s\n" + "fmla v25.4s, v0.4s, v18.4s\n" + "fmla v24.4s, v8.4s, v19.4s\n" + "ldr q16, [x13, x9]\n" + "fmla v20.4s, v8.4s, v17.4s\n" "add x13, x13, #0x10\n" - "fmla v31.4s, v7.4s, v13.4s\n" - "ldr q13, [x10, x9]\n" - "fmla v23.4s, v4.4s, v12.4s\n" - "fmla v26.4s, v1.4s, v12.4s\n" - "ldr q12, [x10, x8]\n" - "fmla v24.4s, v5.4s, v11.4s\n" + "fmla v21.4s, v7.4s, v17.4s\n" + "ldr q19, [x10, x9]\n" + "fmla v28.4s, v4.4s, v18.4s\n" + "fmla v26.4s, v1.4s, v18.4s\n" + "ldr q17, [x10, x8]\n" + "fmla v29.4s, v5.4s, v16.4s\n" "add x10, x10, #0x10\n" - "fmla v25.4s, v4.4s, v11.4s\n" - "fmla v27.4s, v2.4s, v11.4s\n" - "fmla v28.4s, v1.4s, v11.4s\n" - "ldr q11, [x16, x11]\n" - "fmla v29.4s, v4.4s, v12.4s\n" + "fmla v27.4s, v4.4s, v16.4s\n" + "fmla v25.4s, v2.4s, v16.4s\n" + "fmla v24.4s, v1.4s, v16.4s\n" + "ldr q16, [x16, x11]\n" + "fmla v22.4s, v4.4s, v17.4s\n" "add x16, x16, #0x10\n" - "fmla v30.4s, v3.4s, v12.4s\n" - "fmla v31.4s, v4.4s, v13.4s\n" - "fmla v26.4s, v7.4s, v12.4s\n" - "fmla v27.4s, v6.4s, v12.4s\n" - "ld1 { v12.4s }, [x12]\n" - "fmla v23.4s, v2.4s, v11.4s\n" - "fmla v24.4s, v1.4s, v11.4s\n" - "fmax v24.4s, v24.4s, v18.4s\n" - "fmla v25.4s, v0.4s, v11.4s\n" - "ldr q11, [x12, x26]\n" - "fmla v28.4s, v7.4s, v13.4s\n" - "fmin v24.4s, v24.4s, v17.4s\n" - "fmla v30.4s, v5.4s, v13.4s\n" - "fmla v29.4s, v0.4s, v12.4s\n" + "fmla v20.4s, v3.4s, v17.4s\n" + "fmla v21.4s, v4.4s, v19.4s\n" + "fmla v26.4s, v7.4s, v17.4s\n" + "fmla v25.4s, v6.4s, v17.4s\n" + "ld1 { v18.4s }, [x12]\n" + "fmla v28.4s, v2.4s, v16.4s\n" + "fmla v29.4s, v1.4s, v16.4s\n" + "fmax v29.4s, v29.4s, v15.4s\n" + "fmla v27.4s, v0.4s, v16.4s\n" + "ldr q17, [x12, x26]\n" + "fmla v24.4s, v7.4s, v19.4s\n" + "fmin v29.4s, v29.4s, v14.4s\n" + "fmla v20.4s, v5.4s, v19.4s\n" + "fmla v22.4s, v0.4s, v18.4s\n" "add x12, x12, #0x10\n" - "fmla v31.4s, v2.4s, v11.4s\n" - "fmla v27.4s, v8.4s, v13.4s\n" - "ldr q13, [x27, x11]\n" - "fmax v27.4s, v27.4s, v18.4s\n" - "fmla v23.4s, v6.4s, v12.4s\n" - "fmla v26.4s, v3.4s, v12.4s\n" - "fmax v23.4s, v23.4s, v18.4s\n" + "fmla v21.4s, v2.4s, v17.4s\n" + "fmla v25.4s, v8.4s, v19.4s\n" + "ldr q16, [x27, x11]\n" + "fmax v25.4s, v25.4s, v15.4s\n" + "fmla v28.4s, v6.4s, v18.4s\n" + "fmla v26.4s, v3.4s, v18.4s\n" + "fmax v28.4s, v28.4s, v15.4s\n" "add x27, x27, #0x10\n" - "fmla v25.4s, v8.4s, v11.4s\n" - "fmla v28.4s, v5.4s, v11.4s\n" - "fmax v25.4s, v25.4s, v18.4s\n" - "fmla v29.4s, v8.4s, v13.4s\n" - "fmla v30.4s, v7.4s, v13.4s\n" - "fmax v26.4s, v26.4s, v18.4s\n" - "fmla v31.4s, v6.4s, v13.4s\n" - "fmax v28.4s, v28.4s, v18.4s\n" - "fmax v29.4s, v29.4s, v18.4s\n" - "fmax v30.4s, v30.4s, v18.4s\n" - "fmax v31.4s, v31.4s, v18.4s\n" - "fmin v23.4s, v23.4s, v17.4s\n" - "st1 { v23.4s }, [x15]\n" - "fmin v25.4s, v25.4s, v17.4s\n" - "fmin v26.4s, v26.4s, v17.4s\n" - "str q24, [x15, x17]\n" - "fmin v27.4s, v27.4s, v17.4s\n" - "fmin v28.4s, v28.4s, v17.4s\n" - "str q25, [x15, x22]\n" + "fmla v27.4s, v8.4s, v17.4s\n" + "fmla v24.4s, v5.4s, v17.4s\n" + "fmax v27.4s, v27.4s, v15.4s\n" + "fmla v22.4s, v8.4s, v16.4s\n" + "fmla v20.4s, v7.4s, v16.4s\n" + "fmax v26.4s, v26.4s, v15.4s\n" + "fmla v21.4s, v6.4s, v16.4s\n" + "fmax v24.4s, v24.4s, v15.4s\n" + "fmax v22.4s, v22.4s, v15.4s\n" + "fmax v20.4s, v20.4s, v15.4s\n" + "fmax v21.4s, v21.4s, v15.4s\n" + "fmin v28.4s, v28.4s, v14.4s\n" + "st1 { v28.4s }, [x15]\n" + "fmin v27.4s, v27.4s, v14.4s\n" + "fmin v26.4s, v26.4s, v14.4s\n" + "str q29, [x15, x17]\n" + "fmin v25.4s, v25.4s, v14.4s\n" + "fmin v24.4s, v24.4s, v14.4s\n" + "str q27, [x15, x22]\n" "add x15, x15, #0x10\n" - "fmin v29.4s, v29.4s, v17.4s\n" - "fmin v30.4s, v30.4s, v17.4s\n" + "fmin v22.4s, v22.4s, v14.4s\n" + "fmin v20.4s, v20.4s, v14.4s\n" "st1 { v26.4s }, [x28]\n" - "fmin v31.4s, v31.4s, v17.4s\n" - "str q27, [x28, x17]\n" - "str q28, [x28, x22]\n" + "fmin v21.4s, v21.4s, v14.4s\n" + "str q25, [x28, x17]\n" + "str q24, [x28, x22]\n" "add x28, x28, #0x10\n" - "st1 { v29.4s }, [x25]\n" - "str q30, [x25, x17]\n" - "str q31, [x25, x22]\n" + "st1 { v22.4s }, [x25]\n" + "str q20, [x25, x17]\n" + "str q21, [x25, x22]\n" "add x25, x25, #0x10\n" "4:" // Tile loop: Oddments "tst %x[n_channels], #0x3\n" "beq 49f\n" - "ldr q16, [x14, #0x0]\n" + "ldr q31, [x14, #0x0]\n" "ldr q0, [x14, #0x10]\n" "add x24, x12, x11\n" "add x23, x16, XZR\n" @@ -481,18 +481,18 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "ldr s12, [x21, #0x0]\n" "ldr s13, [x20, #0x0]\n" "6:" // Tile loop: Oddments: Load inputs: (2, 2), (0, 0), (0, 4), (4, 0), (1, 2): Bit 1: End - "mov v23.16b, v16.16b\n fmla v23.4s, v8.4s, v9.4s\n" - "mov v25.16b, v16.16b\n fmla v25.4s, v6.4s, v9.4s\n" + "mov v23.16b, v31.16b\n fmla v23.4s, v8.4s, v9.4s\n" + "mov v25.16b, v31.16b\n fmla v25.4s, v6.4s, v9.4s\n" "add x20, x27, x26\n" - "mov v24.16b, v16.16b\n fmla v24.4s, v7.4s, v9.4s\n" - "mov v26.16b, v16.16b\n fmla v26.4s, v5.4s, v9.4s\n" - "mov v27.16b, v16.16b\n fmla v27.4s, v4.4s, v9.4s\n" - "mov v28.16b, v16.16b\n fmla v28.4s, v3.4s, v9.4s\n" - "mov v29.16b, v16.16b\n fmla v29.4s, v2.4s, v9.4s\n" + "mov v24.16b, v31.16b\n fmla v24.4s, v7.4s, v9.4s\n" + "mov v26.16b, v31.16b\n fmla v26.4s, v5.4s, v9.4s\n" + "mov v27.16b, v31.16b\n fmla v27.4s, v4.4s, v9.4s\n" + "mov v28.16b, v31.16b\n fmla v28.4s, v3.4s, v9.4s\n" + "mov v29.16b, v31.16b\n fmla v29.4s, v2.4s, v9.4s\n" "fmla v23.4s, v0.4s, v10.4s\n" "fmla v25.4s, v2.4s, v11.4s\n" - "mov v30.16b, v16.16b\n fmla v30.4s, v1.4s, v9.4s\n" - "mov v31.16b, v16.16b\n fmla v31.4s, v0.4s, v9.4s\n" + "mov v30.16b, v31.16b\n fmla v30.4s, v1.4s, v9.4s\n" + "fmla v31.4s, v0.4s, v9.4s\n" "fmla v29.4s, v6.4s, v12.4s\n" "fmla v23.4s, v5.4s, v13.4s\n" "fmla v24.4s, v4.4s, v13.4s\n" @@ -741,25 +741,25 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "46:" // Tile loop: Oddments: Load inputs: (4, 2): Bit 1: End "fmla v29.4s, v8.4s, v13.4s\n" "fmla v30.4s, v7.4s, v13.4s\n" - "fmax v23.4s, v23.4s, v18.4s\n" + "fmax v23.4s, v23.4s, v15.4s\n" "fmla v31.4s, v6.4s, v13.4s\n" - "fmax v24.4s, v24.4s, v18.4s\n" - "fmax v25.4s, v25.4s, v18.4s\n" - "fmax v26.4s, v26.4s, v18.4s\n" - "fmax v27.4s, v27.4s, v18.4s\n" - "fmax v28.4s, v28.4s, v18.4s\n" - "fmax v29.4s, v29.4s, v18.4s\n" - "fmax v30.4s, v30.4s, v18.4s\n" - "fmax v31.4s, v31.4s, v18.4s\n" - "fmin v23.4s, v23.4s, v17.4s\n" - "fmin v24.4s, v24.4s, v17.4s\n" - "fmin v25.4s, v25.4s, v17.4s\n" - "fmin v26.4s, v26.4s, v17.4s\n" - "fmin v27.4s, v27.4s, v17.4s\n" - "fmin v28.4s, v28.4s, v17.4s\n" - "fmin v29.4s, v29.4s, v17.4s\n" - "fmin v30.4s, v30.4s, v17.4s\n" - "fmin v31.4s, v31.4s, v17.4s\n" + "fmax v24.4s, v24.4s, v15.4s\n" + "fmax v25.4s, v25.4s, v15.4s\n" + "fmax v26.4s, v26.4s, v15.4s\n" + "fmax v27.4s, v27.4s, v15.4s\n" + "fmax v28.4s, v28.4s, v15.4s\n" + "fmax v29.4s, v29.4s, v15.4s\n" + "fmax v30.4s, v30.4s, v15.4s\n" + "fmax v31.4s, v31.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v14.4s\n" + "fmin v24.4s, v24.4s, v14.4s\n" + "fmin v25.4s, v25.4s, v14.4s\n" + "fmin v26.4s, v26.4s, v14.4s\n" + "fmin v27.4s, v27.4s, v14.4s\n" + "fmin v28.4s, v28.4s, v14.4s\n" + "fmin v29.4s, v29.4s, v14.4s\n" + "fmin v30.4s, v30.4s, v14.4s\n" + "fmin v31.4s, v31.4s, v14.4s\n" "tbz %x[n_channels], #1, 47f\n" "mov x22, x15\n" "mov x21, x28\n" @@ -804,7 +804,6 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "st1 { v28.s }[0], [x21]\n" "st1 { v31.s }[0], [x20]\n" "48:" // Tile loop: Oddments: Store: Bit 1: End - "49:" // Tile loop: End "ldr x23, [%x[params_struct], %[offsetof_args_tile_j]]\n" "ldr x24, [%x[params_struct], %[offsetof_args_tile_i]]\n" @@ -819,11 +818,11 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v16", "v17", "v18", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp index 15053a337a34edc43f389f241f63e5b374e3f922..972f7eb5353f21f252af19b9729cff15f3981ad0 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ +#if defined(__aarch64__) namespace arm_conv { namespace depthwise { @@ -87,405 +87,405 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( activation_min, activation_max); __asm__ __volatile__( - "mov x8, #0x10\n" // cntb _, ALL, #1 - "lsr x17, %x[n_channels], #0x2\n" - "ldr x16, [%x[params_struct], %[offsetof_args_outptrs]]\n" - "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" + "mov x7, #0x10\n" // cntb _, ALL, #1 + "lsr x8, %x[n_channels], #0x2\n" + "ldr x17, [%x[params_struct], %[offsetof_args_outptrs]]\n" + "ldr x16, [%x[params_struct], %[offsetof_args_params]]\n" "add x20, %x[params_struct], %[offsetof_args_min]\n" - "ld1r { v18.4s }, [x20]\n" + "ld1r { v15.4s }, [x20]\n" "add x20, %x[params_struct], %[offsetof_args_max]\n" - "ld1r { v17.4s }, [x20]\n" - "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n" - "mov x13, #0x0\n" - "sub x12, XZR, x8\n" - "cbz x17, 3f\n" - "ldr q16, [x15, #0x0]\n" - "ldr q0, [x15, #0x10]\n" - "cmp x8, x17, LSL #4\n" - "ldr q1, [x15, #0x20]\n" - "ldr q2, [x15, #0x30]\n" - "ldr q3, [x15, #0x40]\n" - "ldr q4, [x15, #0x50]\n" - "ldr q5, [x15, #0x60]\n" - "ldr q6, [x15, #0x70]\n" - "ldr q7, [x15, #0x80]\n" - "ldr q8, [x15, #0x90]\n" - "add x15, x15, #0xa0\n" - "ldp x11, x10, [x14, #0x0]\n" - "ldr q9, [x11, x13]\n" - "ldr q10, [x10, x13]\n" - "ldp x9, x28, [x14, #0x10]\n" - "ldr q11, [x9, x13]\n" - "ldr q12, [x28, x13]\n" - "ldr x27, [x14, #0x20]\n" - "ldr q13, [x27, x13]\n" + "ld1r { v14.4s }, [x20]\n" + "add x15, %x[params_struct], %[offsetof_Args_inptrs]\n" + "mov x14, #0x0\n" + "sub x13, XZR, x7\n" + "cbz x8, 3f\n" + "ldr q31, [x16, #0x0]\n" + "ldr q0, [x16, #0x10]\n" + "cmp x7, x8, LSL #4\n" + "ldr q1, [x16, #0x20]\n" + "ldr q2, [x16, #0x30]\n" + "ldr q3, [x16, #0x40]\n" + "ldr q4, [x16, #0x50]\n" + "ldr q5, [x16, #0x60]\n" + "ldr q6, [x16, #0x70]\n" + "ldr q7, [x16, #0x80]\n" + "ldr q8, [x16, #0x90]\n" + "add x16, x16, #0xa0\n" + "ldp x21, x20, [x15, #0x0]\n" + "ldr q9, [x21, x14]\n" + "ldr q10, [x20, x14]\n" + "ldp x21, x20, [x15, #0x10]\n" + "ldr q11, [x21, x14]\n" + "ldr q12, [x20, x14]\n" + "ldr x20, [x15, #0x20]\n" + "ldr q13, [x20, x14]\n" "bge 2f\n" "1:" // Channel loop - "mov v23.16b, v16.16b\n fmla v23.4s, v8.4s, v9.4s\n" - "mov v24.16b, v16.16b\n fmla v24.4s, v7.4s, v9.4s\n" - "ldr x26, [x14, #0x30]\n" - "ldr x25, [x14, #0x38]\n" - "mov v25.16b, v16.16b\n fmla v25.4s, v6.4s, v9.4s\n" - "fmla v23.4s, v0.4s, v10.4s\n" - "ldr x24, [x14, #0x28]\n" - "ldr x10, [x14, #0x48]\n" - "ldr q10, [x10, x13]\n" - "fmla v24.4s, v4.4s, v13.4s\n" - "mov v26.16b, v16.16b\n fmla v26.4s, v5.4s, v9.4s\n" - "ldr x11, [x14, #0x40]\n" - "mov v27.16b, v16.16b\n fmla v27.4s, v4.4s, v9.4s\n" - "mov v28.16b, v16.16b\n fmla v28.4s, v3.4s, v9.4s\n" - "ldr x9, [x14, #0x50]\n" - "ldr x28, [x14, #0x58]\n" - "fmla v25.4s, v2.4s, v11.4s\n" - "ldr q11, [x26, x13]\n" - "mov v29.16b, v16.16b\n fmla v29.4s, v2.4s, v9.4s\n" - "ldr x27, [x14, #0x60]\n" - "fmla v23.4s, v5.4s, v13.4s\n" - "fmla v24.4s, v6.4s, v11.4s\n" - "ldr x26, [x14, #0x70]\n" - "ldr x10, [x14, #0x88]\n" - "mov v31.16b, v16.16b\n fmla v31.4s, v0.4s, v9.4s\n" - "fmla v25.4s, v3.4s, v13.4s\n" - "ldr x23, [x16, #0x0]\n" - "add x12, x12, #0x10\n" + "mov v29.16b, v31.16b\n fmla v29.4s, v8.4s, v9.4s\n" + "mov v28.16b, v31.16b\n fmla v28.4s, v7.4s, v9.4s\n" + "ldr x26, [x15, #0x30]\n" + "ldr x23, [x15, #0x38]\n" + "mov v27.16b, v31.16b\n fmla v27.4s, v6.4s, v9.4s\n" + "fmla v29.4s, v0.4s, v10.4s\n" + "ldr x22, [x15, #0x28]\n" + "ldr x20, [x15, #0x48]\n" + "ldr q19, [x20, x14]\n" + "fmla v28.4s, v4.4s, v13.4s\n" + "mov v26.16b, v31.16b\n fmla v26.4s, v5.4s, v9.4s\n" + "ldr x21, [x15, #0x40]\n" + "mov v25.16b, v31.16b\n fmla v25.4s, v4.4s, v9.4s\n" + "mov v24.16b, v31.16b\n fmla v24.4s, v3.4s, v9.4s\n" + "ldr x25, [x15, #0x50]\n" + "ldr x24, [x15, #0x58]\n" + "fmla v27.4s, v2.4s, v11.4s\n" + "ldr q17, [x26, x14]\n" + "mov v23.16b, v31.16b\n fmla v23.4s, v2.4s, v9.4s\n" + "ldr x20, [x15, #0x60]\n" + "fmla v29.4s, v5.4s, v13.4s\n" + "fmla v28.4s, v6.4s, v17.4s\n" + "ldr x12, [x15, #0x70]\n" + "ldr x11, [x15, #0x88]\n" + "mov v22.16b, v31.16b\n fmla v22.4s, v0.4s, v9.4s\n" + "fmla v27.4s, v3.4s, v13.4s\n" + "ldr x10, [x17, #0x0]\n" + "add x13, x13, #0x10\n" "fmla v26.4s, v2.4s, v13.4s\n" - "fmla v27.4s, v1.4s, v13.4s\n" - "ldr x22, [x16, #0x8]\n" - "ldr x21, [x16, #0x10]\n" - "fmla v28.4s, v0.4s, v13.4s\n" - "ldr q13, [x25, x13]\n" - "fmla v29.4s, v6.4s, v12.4s\n" - "ldr q12, [x24, x13]\n" - "mov v30.16b, v16.16b\n fmla v30.4s, v1.4s, v9.4s\n" - "ldr q16, [x15, #0x0]\n" - "fmla v23.4s, v7.4s, v11.4s\n" - "ldr x24, [x14, #0x68]\n" + "fmla v25.4s, v1.4s, v13.4s\n" + "ldr x9, [x17, #0x8]\n" + "ldr x28, [x17, #0x10]\n" "fmla v24.4s, v0.4s, v13.4s\n" - "fmla v31.4s, v8.4s, v12.4s\n" - "ldr q12, [x11, x13]\n" - "ldr x25, [x14, #0x78]\n" - "fmla v26.4s, v4.4s, v11.4s\n" - "fmla v27.4s, v3.4s, v11.4s\n" - "ldr x11, [x14, #0x80]\n" - "ldr x20, [x16, #0x18]\n" - "fmla v30.4s, v0.4s, v11.4s\n" - "fmla v28.4s, v4.4s, v10.4s\n" - "fmla v29.4s, v1.4s, v11.4s\n" - "ldr q11, [x9, x13]\n" - "fmla v23.4s, v1.4s, v13.4s\n" - "ldr q13, [x28, x13]\n" - "fmla v24.4s, v2.4s, v12.4s\n" - "fmla v25.4s, v1.4s, v12.4s\n" - "ldr q12, [x27, x13]\n" - "ldr x9, [x14, #0x90]\n" - "fmla v27.4s, v5.4s, v10.4s\n" - "fmla v30.4s, v2.4s, v10.4s\n" - "ldr x27, [x14, #0xa0]\n" - "ldr x28, [x14, #0x98]\n" - "fmla v26.4s, v0.4s, v11.4s\n" - "fmla v28.4s, v2.4s, v13.4s\n" - "fmla v24.4s, v8.4s, v10.4s\n" - "fmla v25.4s, v7.4s, v10.4s\n" - "fmla v31.4s, v1.4s, v10.4s\n" - "ldr q10, [x24, x13]\n" - "fmla v29.4s, v3.4s, v12.4s\n" - "ldr x24, [x14, #0xa8]\n" - "fmla v26.4s, v6.4s, v12.4s\n" - "ldr q12, [x11, x13]\n" - "fmla v27.4s, v7.4s, v10.4s\n" - "ldr x11, [x14, #0xc0]\n" - "fmla v28.4s, v6.4s, v10.4s\n" - "fmla v30.4s, v4.4s, v10.4s\n" - "fmla v23.4s, v3.4s, v11.4s\n" - "ldr q11, [x26, x13]\n" - "fmla v25.4s, v5.4s, v13.4s\n" - "ldr q13, [x25, x13]\n" - "fmla v29.4s, v5.4s, v10.4s\n" - "fmla v31.4s, v3.4s, v10.4s\n" - "ldr x26, [x14, #0xb0]\n" - "ldr x25, [x14, #0xb8]\n" - "fmla v26.4s, v8.4s, v10.4s\n" - "fmla v28.4s, v8.4s, v11.4s\n" - "fmla v30.4s, v6.4s, v13.4s\n" - "fmla v24.4s, v3.4s, v12.4s\n" - "fmla v27.4s, v0.4s, v12.4s\n" - "fmla v31.4s, v5.4s, v11.4s\n" - "ldr q11, [x10, x13]\n" - "fmla v29.4s, v7.4s, v13.4s\n" - "ldr q13, [x9, x13]\n" - "fmla v23.4s, v4.4s, v12.4s\n" - "fmla v26.4s, v1.4s, v12.4s\n" - "ldr q12, [x28, x13]\n" - "fmla v24.4s, v5.4s, v11.4s\n" - "fmla v25.4s, v4.4s, v11.4s\n" - "fmla v27.4s, v2.4s, v11.4s\n" - "fmla v28.4s, v1.4s, v11.4s\n" - "ldr q11, [x27, x13]\n" - "fmla v30.4s, v8.4s, v13.4s\n" - "ldr x27, [x14, #0x20]\n" - "fmla v31.4s, v7.4s, v13.4s\n" - "ldr q13, [x24, x13]\n" - "fmla v23.4s, v2.4s, v11.4s\n" - "fmla v26.4s, v7.4s, v12.4s\n" - "fmla v27.4s, v6.4s, v12.4s\n" - "fmla v29.4s, v4.4s, v12.4s\n" - "fmla v30.4s, v3.4s, v12.4s\n" - "ldr q12, [x26, x13]\n" - "fmla v31.4s, v4.4s, v13.4s\n" - "ldr q4, [x15, #0x50]\n" - "fmla v24.4s, v1.4s, v11.4s\n" - "ldr q1, [x15, #0x20]\n" - "fmla v25.4s, v0.4s, v11.4s\n" - "ldr q11, [x25, x13]\n" + "ldr q18, [x23, x14]\n" "fmla v23.4s, v6.4s, v12.4s\n" - "fmax v23.4s, v23.4s, v18.4s\n" - "fmla v28.4s, v7.4s, v13.4s\n" - "fmla v30.4s, v5.4s, v13.4s\n" - "fmin v23.4s, v23.4s, v17.4s\n" - "str q23, [x23, x12]\n" - "fmla v29.4s, v0.4s, v12.4s\n" - "ldr q0, [x15, #0x10]\n" - "fmla v31.4s, v2.4s, v11.4s\n" - "ldr q2, [x15, #0x30]\n" - "fmla v27.4s, v8.4s, v13.4s\n" - "ldr q13, [x11, x13]\n" - "fmla v26.4s, v3.4s, v12.4s\n" - "ldr q3, [x15, #0x40]\n" - "fmla v25.4s, v8.4s, v11.4s\n" - "fmla v28.4s, v5.4s, v11.4s\n" - "ldr q5, [x15, #0x60]\n" - "fmax v24.4s, v24.4s, v18.4s\n" - "fmla v29.4s, v8.4s, v13.4s\n" - "ldr q8, [x15, #0x90]\n" - "fmla v30.4s, v7.4s, v13.4s\n" - "ldr q7, [x15, #0x80]\n" - "fmla v31.4s, v6.4s, v13.4s\n" - "ldr q13, [x27, x8]\n" - "ldr q6, [x15, #0x70]\n" - "fmax v25.4s, v25.4s, v18.4s\n" - "fmax v26.4s, v26.4s, v18.4s\n" - "fmax v27.4s, v27.4s, v18.4s\n" - "ldr x23, [x16, #0x20]\n" - "ldp x11, x10, [x14, #0x0]\n" - "ldr q9, [x11, x8]\n" - "ldr q10, [x10, x8]\n" - "fmin v24.4s, v24.4s, v17.4s\n" - "fmin v25.4s, v25.4s, v17.4s\n" - "ldp x9, x28, [x14, #0x10]\n" - "ldr q11, [x9, x8]\n" - "fmin v26.4s, v26.4s, v17.4s\n" - "fmin v27.4s, v27.4s, v17.4s\n" - "ldr q12, [x28, x8]\n" - "fmax v28.4s, v28.4s, v18.4s\n" - "fmax v29.4s, v29.4s, v18.4s\n" - "str q24, [x22, x12]\n" - "fmax v30.4s, v30.4s, v18.4s\n" - "fmax v31.4s, v31.4s, v18.4s\n" - "str q25, [x21, x12]\n" - "ldr x22, [x16, #0x28]\n" - "str q26, [x20, x12]\n" - "ldr x21, [x16, #0x30]\n" - "ldr x20, [x16, #0x38]\n" - "add x8, x8, #0x10\n" - "str q27, [x23, x12]\n" - "ldr x23, [x16, #0x40]\n" - "cmp x8, x17, LSL #4\n" - "fmin v28.4s, v28.4s, v17.4s\n" - "fmin v29.4s, v29.4s, v17.4s\n" - "fmin v30.4s, v30.4s, v17.4s\n" - "add x13, x13, #0x10\n" - "str q28, [x22, x12]\n" - "fmin v31.4s, v31.4s, v17.4s\n" - "str q29, [x21, x12]\n" - "add x15, x15, #0xa0\n" - "str q30, [x20, x12]\n" - "str q31, [x23, x12]\n" + "ldr q16, [x22, x14]\n" + "mov v21.16b, v31.16b\n fmla v21.4s, v1.4s, v9.4s\n" + "ldr q31, [x16, #0x0]\n" + "fmla v29.4s, v7.4s, v17.4s\n" + "ldr x23, [x15, #0x68]\n" + "fmla v28.4s, v0.4s, v18.4s\n" + "fmla v22.4s, v8.4s, v16.4s\n" + "ldr q16, [x21, x14]\n" + "ldr x22, [x15, #0x78]\n" + "fmla v26.4s, v4.4s, v17.4s\n" + "fmla v25.4s, v3.4s, v17.4s\n" + "ldr x21, [x15, #0x80]\n" + "ldr x27, [x17, #0x18]\n" + "fmla v21.4s, v0.4s, v17.4s\n" + "fmla v24.4s, v4.4s, v19.4s\n" + "fmla v23.4s, v1.4s, v17.4s\n" + "ldr q17, [x25, x14]\n" + "fmla v29.4s, v1.4s, v18.4s\n" + "ldr q20, [x24, x14]\n" + "fmla v28.4s, v2.4s, v16.4s\n" + "fmla v27.4s, v1.4s, v16.4s\n" + "ldr q16, [x20, x14]\n" + "ldr x26, [x15, #0x90]\n" + "fmla v25.4s, v5.4s, v19.4s\n" + "fmla v21.4s, v2.4s, v19.4s\n" + "ldr x25, [x15, #0xa0]\n" + "ldr x20, [x15, #0x98]\n" + "fmla v26.4s, v0.4s, v17.4s\n" + "fmla v24.4s, v2.4s, v20.4s\n" + "fmla v28.4s, v8.4s, v19.4s\n" + "fmla v27.4s, v7.4s, v19.4s\n" + "fmla v22.4s, v1.4s, v19.4s\n" + "ldr q19, [x23, x14]\n" + "fmla v23.4s, v3.4s, v16.4s\n" + "ldr x24, [x15, #0xa8]\n" + "fmla v26.4s, v6.4s, v16.4s\n" + "ldr q18, [x21, x14]\n" + "fmla v25.4s, v7.4s, v19.4s\n" + "ldr x23, [x15, #0xc0]\n" + "fmla v24.4s, v6.4s, v19.4s\n" + "fmla v21.4s, v4.4s, v19.4s\n" + "fmla v29.4s, v3.4s, v17.4s\n" + "ldr q17, [x12, x14]\n" + "fmla v27.4s, v5.4s, v20.4s\n" + "ldr q16, [x22, x14]\n" + "fmla v23.4s, v5.4s, v19.4s\n" + "fmla v22.4s, v3.4s, v19.4s\n" + "ldr x22, [x15, #0xb0]\n" + "ldr x21, [x15, #0xb8]\n" + "fmla v26.4s, v8.4s, v19.4s\n" + "fmla v24.4s, v8.4s, v17.4s\n" + "fmla v21.4s, v6.4s, v16.4s\n" + "fmla v28.4s, v3.4s, v18.4s\n" + "fmla v25.4s, v0.4s, v18.4s\n" + "fmla v22.4s, v5.4s, v17.4s\n" + "ldr q17, [x11, x14]\n" + "fmla v23.4s, v7.4s, v16.4s\n" + "ldr q16, [x26, x14]\n" + "fmla v29.4s, v4.4s, v18.4s\n" + "fmla v26.4s, v1.4s, v18.4s\n" + "ldr q18, [x20, x14]\n" + "fmla v28.4s, v5.4s, v17.4s\n" + "fmla v27.4s, v4.4s, v17.4s\n" + "fmla v25.4s, v2.4s, v17.4s\n" + "fmla v24.4s, v1.4s, v17.4s\n" + "ldr q17, [x25, x14]\n" + "fmla v21.4s, v8.4s, v16.4s\n" + "ldr x20, [x15, #0x20]\n" + "fmla v22.4s, v7.4s, v16.4s\n" + "ldr q16, [x24, x14]\n" + "fmla v29.4s, v2.4s, v17.4s\n" + "fmla v26.4s, v7.4s, v18.4s\n" + "fmla v25.4s, v6.4s, v18.4s\n" + "fmla v23.4s, v4.4s, v18.4s\n" + "fmla v21.4s, v3.4s, v18.4s\n" + "ldr q18, [x22, x14]\n" + "fmla v22.4s, v4.4s, v16.4s\n" + "ldr q4, [x16, #0x50]\n" + "fmla v28.4s, v1.4s, v17.4s\n" + "ldr q1, [x16, #0x20]\n" + "fmla v27.4s, v0.4s, v17.4s\n" + "ldr q17, [x21, x14]\n" + "fmla v29.4s, v6.4s, v18.4s\n" + "fmax v29.4s, v29.4s, v15.4s\n" + "fmla v24.4s, v7.4s, v16.4s\n" + "fmla v21.4s, v5.4s, v16.4s\n" + "fmin v29.4s, v29.4s, v14.4s\n" + "str q29, [x10, x13]\n" + "fmla v23.4s, v0.4s, v18.4s\n" + "ldr q0, [x16, #0x10]\n" + "fmla v22.4s, v2.4s, v17.4s\n" + "ldr q2, [x16, #0x30]\n" + "fmla v25.4s, v8.4s, v16.4s\n" + "ldr q16, [x23, x14]\n" + "fmla v26.4s, v3.4s, v18.4s\n" + "ldr q3, [x16, #0x40]\n" + "fmla v27.4s, v8.4s, v17.4s\n" + "fmla v24.4s, v5.4s, v17.4s\n" + "ldr q5, [x16, #0x60]\n" + "fmax v28.4s, v28.4s, v15.4s\n" + "fmla v23.4s, v8.4s, v16.4s\n" + "ldr q8, [x16, #0x90]\n" + "fmla v21.4s, v7.4s, v16.4s\n" + "ldr q7, [x16, #0x80]\n" + "fmla v22.4s, v6.4s, v16.4s\n" + "ldr q13, [x20, x7]\n" + "ldr q6, [x16, #0x70]\n" + "fmax v27.4s, v27.4s, v15.4s\n" + "fmax v26.4s, v26.4s, v15.4s\n" + "fmax v25.4s, v25.4s, v15.4s\n" + "ldr x24, [x17, #0x20]\n" + "ldp x21, x20, [x15, #0x0]\n" + "ldr q9, [x21, x7]\n" + "ldr q10, [x20, x7]\n" + "fmin v28.4s, v28.4s, v14.4s\n" + "fmin v27.4s, v27.4s, v14.4s\n" + "ldp x21, x20, [x15, #0x10]\n" + "ldr q11, [x21, x7]\n" + "fmin v26.4s, v26.4s, v14.4s\n" + "fmin v25.4s, v25.4s, v14.4s\n" + "ldr q12, [x20, x7]\n" + "fmax v24.4s, v24.4s, v15.4s\n" + "fmax v23.4s, v23.4s, v15.4s\n" + "str q28, [x9, x13]\n" + "fmax v21.4s, v21.4s, v15.4s\n" + "fmax v22.4s, v22.4s, v15.4s\n" + "str q27, [x28, x13]\n" + "ldr x23, [x17, #0x28]\n" + "str q26, [x27, x13]\n" + "ldr x22, [x17, #0x30]\n" + "ldr x21, [x17, #0x38]\n" + "add x7, x7, #0x10\n" + "str q25, [x24, x13]\n" + "ldr x20, [x17, #0x40]\n" + "cmp x7, x8, LSL #4\n" + "fmin v24.4s, v24.4s, v14.4s\n" + "fmin v23.4s, v23.4s, v14.4s\n" + "fmin v21.4s, v21.4s, v14.4s\n" + "add x14, x14, #0x10\n" + "str q24, [x23, x13]\n" + "fmin v22.4s, v22.4s, v14.4s\n" + "str q23, [x22, x13]\n" + "add x16, x16, #0xa0\n" + "str q21, [x21, x13]\n" + "str q22, [x20, x13]\n" "blt 1b\n" "2:" // Channel tail - "mov v23.16b, v16.16b\n fmla v23.4s, v8.4s, v9.4s\n" - "mov v24.16b, v16.16b\n fmla v24.4s, v7.4s, v9.4s\n" - "ldr x26, [x14, #0x30]\n" - "ldr x25, [x14, #0x38]\n" - "mov v25.16b, v16.16b\n fmla v25.4s, v6.4s, v9.4s\n" - "fmla v23.4s, v0.4s, v10.4s\n" - "ldr x24, [x14, #0x28]\n" - "ldr x10, [x14, #0x48]\n" - "ldr q10, [x10, x13]\n" - "fmla v24.4s, v4.4s, v13.4s\n" - "mov v26.16b, v16.16b\n fmla v26.4s, v5.4s, v9.4s\n" - "ldr x11, [x14, #0x40]\n" - "mov v27.16b, v16.16b\n fmla v27.4s, v4.4s, v9.4s\n" - "mov v28.16b, v16.16b\n fmla v28.4s, v3.4s, v9.4s\n" - "ldr x9, [x14, #0x50]\n" - "ldr x28, [x14, #0x58]\n" - "fmla v25.4s, v2.4s, v11.4s\n" - "ldr q11, [x26, x13]\n" - "mov v29.16b, v16.16b\n fmla v29.4s, v2.4s, v9.4s\n" - "ldr x27, [x14, #0x60]\n" - "fmla v23.4s, v5.4s, v13.4s\n" - "fmla v24.4s, v6.4s, v11.4s\n" - "ldr x26, [x14, #0x70]\n" - "ldr x10, [x14, #0x88]\n" - "mov v31.16b, v16.16b\n fmla v31.4s, v0.4s, v9.4s\n" - "fmla v25.4s, v3.4s, v13.4s\n" - "ldr x23, [x16, #0x0]\n" - "add x12, x12, #0x10\n" + "mov v29.16b, v31.16b\n fmla v29.4s, v8.4s, v9.4s\n" + "mov v28.16b, v31.16b\n fmla v28.4s, v7.4s, v9.4s\n" + "ldr x23, [x15, #0x30]\n" + "ldr x22, [x15, #0x38]\n" + "mov v27.16b, v31.16b\n fmla v27.4s, v6.4s, v9.4s\n" + "fmla v29.4s, v0.4s, v10.4s\n" + "ldr x21, [x15, #0x28]\n" + "ldr x20, [x15, #0x48]\n" + "ldr q19, [x20, x14]\n" + "fmla v28.4s, v4.4s, v13.4s\n" + "mov v26.16b, v31.16b\n fmla v26.4s, v5.4s, v9.4s\n" + "ldr x20, [x15, #0x40]\n" + "mov v25.16b, v31.16b\n fmla v25.4s, v4.4s, v9.4s\n" + "mov v24.16b, v31.16b\n fmla v24.4s, v3.4s, v9.4s\n" + "ldr x25, [x15, #0x50]\n" + "ldr x24, [x15, #0x58]\n" + "fmla v27.4s, v2.4s, v11.4s\n" + "ldr q17, [x23, x14]\n" + "mov v23.16b, v31.16b\n fmla v23.4s, v2.4s, v9.4s\n" + "ldr x23, [x15, #0x60]\n" + "fmla v29.4s, v5.4s, v13.4s\n" + "fmla v28.4s, v6.4s, v17.4s\n" + "ldr x12, [x15, #0x70]\n" + "ldr x11, [x15, #0x88]\n" + "mov v22.16b, v31.16b\n fmla v22.4s, v0.4s, v9.4s\n" + "fmla v27.4s, v3.4s, v13.4s\n" + "ldr x10, [x17, #0x0]\n" + "add x13, x13, #0x10\n" "fmla v26.4s, v2.4s, v13.4s\n" - "fmla v27.4s, v1.4s, v13.4s\n" - "ldr x22, [x16, #0x8]\n" - "ldr x21, [x16, #0x10]\n" - "fmla v28.4s, v0.4s, v13.4s\n" - "ldr q13, [x25, x13]\n" - "fmla v29.4s, v6.4s, v12.4s\n" - "ldr q12, [x24, x13]\n" - "mov v30.16b, v16.16b\n fmla v30.4s, v1.4s, v9.4s\n" - "fmla v23.4s, v7.4s, v11.4s\n" - "ldr x24, [x14, #0x68]\n" - "ldr x25, [x14, #0x78]\n" + "fmla v25.4s, v1.4s, v13.4s\n" + "ldr x9, [x17, #0x8]\n" + "ldr x28, [x17, #0x10]\n" "fmla v24.4s, v0.4s, v13.4s\n" - "fmla v31.4s, v8.4s, v12.4s\n" - "ldr q12, [x11, x13]\n" - "ldr x11, [x14, #0x80]\n" - "fmla v26.4s, v4.4s, v11.4s\n" - "fmla v27.4s, v3.4s, v11.4s\n" - "ldr x20, [x16, #0x18]\n" - "fmla v30.4s, v0.4s, v11.4s\n" - "fmla v28.4s, v4.4s, v10.4s\n" - "fmla v29.4s, v1.4s, v11.4s\n" - "ldr q11, [x9, x13]\n" - "fmla v23.4s, v1.4s, v13.4s\n" - "ldr q13, [x28, x13]\n" - "fmla v24.4s, v2.4s, v12.4s\n" - "fmla v25.4s, v1.4s, v12.4s\n" - "ldr q12, [x27, x13]\n" - "ldr x9, [x14, #0x90]\n" - "fmla v27.4s, v5.4s, v10.4s\n" - "fmla v30.4s, v2.4s, v10.4s\n" - "ldr x27, [x14, #0xa0]\n" - "ldr x28, [x14, #0x98]\n" - "fmla v26.4s, v0.4s, v11.4s\n" - "fmla v28.4s, v2.4s, v13.4s\n" - "fmla v24.4s, v8.4s, v10.4s\n" - "fmla v25.4s, v7.4s, v10.4s\n" - "fmla v31.4s, v1.4s, v10.4s\n" - "ldr q10, [x24, x13]\n" - "fmla v29.4s, v3.4s, v12.4s\n" - "ldr x24, [x14, #0xa8]\n" - "fmla v26.4s, v6.4s, v12.4s\n" - "ldr q12, [x11, x13]\n" - "fmla v27.4s, v7.4s, v10.4s\n" - "ldr x11, [x14, #0xc0]\n" - "fmla v28.4s, v6.4s, v10.4s\n" - "fmla v30.4s, v4.4s, v10.4s\n" - "fmla v23.4s, v3.4s, v11.4s\n" - "ldr q11, [x26, x13]\n" - "fmla v25.4s, v5.4s, v13.4s\n" - "ldr q13, [x25, x13]\n" - "fmla v29.4s, v5.4s, v10.4s\n" - "fmla v31.4s, v3.4s, v10.4s\n" - "ldr x26, [x14, #0xb0]\n" - "ldr x25, [x14, #0xb8]\n" - "fmla v26.4s, v8.4s, v10.4s\n" - "fmla v28.4s, v8.4s, v11.4s\n" - "fmla v30.4s, v6.4s, v13.4s\n" - "fmla v24.4s, v3.4s, v12.4s\n" - "fmla v27.4s, v0.4s, v12.4s\n" - "fmla v31.4s, v5.4s, v11.4s\n" - "ldr q11, [x10, x13]\n" - "fmla v29.4s, v7.4s, v13.4s\n" - "ldr q13, [x9, x13]\n" - "fmla v23.4s, v4.4s, v12.4s\n" - "fmla v26.4s, v1.4s, v12.4s\n" - "ldr q12, [x28, x13]\n" - "fmla v24.4s, v5.4s, v11.4s\n" - "fmla v25.4s, v4.4s, v11.4s\n" - "fmla v27.4s, v2.4s, v11.4s\n" - "fmla v28.4s, v1.4s, v11.4s\n" - "ldr q11, [x27, x13]\n" - "fmla v30.4s, v8.4s, v13.4s\n" - "fmla v31.4s, v7.4s, v13.4s\n" - "ldr q13, [x24, x13]\n" - "fmla v23.4s, v2.4s, v11.4s\n" - "fmla v26.4s, v7.4s, v12.4s\n" - "fmla v27.4s, v6.4s, v12.4s\n" - "fmla v29.4s, v4.4s, v12.4s\n" - "fmla v30.4s, v3.4s, v12.4s\n" - "ldr q12, [x26, x13]\n" - "fmla v31.4s, v4.4s, v13.4s\n" - "fmla v24.4s, v1.4s, v11.4s\n" - "fmax v24.4s, v24.4s, v18.4s\n" - "fmla v25.4s, v0.4s, v11.4s\n" - "ldr q11, [x25, x13]\n" + "ldr q18, [x22, x14]\n" "fmla v23.4s, v6.4s, v12.4s\n" - "fmax v23.4s, v23.4s, v18.4s\n" - "fmla v28.4s, v7.4s, v13.4s\n" - "fmla v30.4s, v5.4s, v13.4s\n" - "fmin v23.4s, v23.4s, v17.4s\n" - "str q23, [x23, x12]\n" - "fmla v29.4s, v0.4s, v12.4s\n" - "fmla v31.4s, v2.4s, v11.4s\n" - "ldr x23, [x16, #0x20]\n" - "fmin v24.4s, v24.4s, v17.4s\n" - "fmla v27.4s, v8.4s, v13.4s\n" - "ldr q13, [x11, x13]\n" - "fmla v26.4s, v3.4s, v12.4s\n" - "fmax v26.4s, v26.4s, v18.4s\n" - "fmla v25.4s, v8.4s, v11.4s\n" - "fmla v28.4s, v5.4s, v11.4s\n" - "fmax v25.4s, v25.4s, v18.4s\n" - "str q24, [x22, x12]\n" - "fmla v29.4s, v8.4s, v13.4s\n" - "fmla v30.4s, v7.4s, v13.4s\n" - "fmax v27.4s, v27.4s, v18.4s\n" - "ldr x22, [x16, #0x28]\n" - "fmla v31.4s, v6.4s, v13.4s\n" - "fmin v25.4s, v25.4s, v17.4s\n" - "str q25, [x21, x12]\n" - "ldr x21, [x16, #0x30]\n" - "fmin v26.4s, v26.4s, v17.4s\n" - "fmin v27.4s, v27.4s, v17.4s\n" - "str q26, [x20, x12]\n" - "ldr x20, [x16, #0x38]\n" - "fmax v28.4s, v28.4s, v18.4s\n" - "fmax v29.4s, v29.4s, v18.4s\n" - "str q27, [x23, x12]\n" - "ldr x23, [x16, #0x40]\n" - "fmax v30.4s, v30.4s, v18.4s\n" - "fmax v31.4s, v31.4s, v18.4s\n" - "add x13, x13, #0x10\n" - "fmin v28.4s, v28.4s, v17.4s\n" - "fmin v29.4s, v29.4s, v17.4s\n" - "str q28, [x22, x12]\n" - "fmin v30.4s, v30.4s, v17.4s\n" - "fmin v31.4s, v31.4s, v17.4s\n" - "str q29, [x21, x12]\n" - "str q30, [x20, x12]\n" - "str q31, [x23, x12]\n" + "ldr q16, [x21, x14]\n" + "mov v21.16b, v31.16b\n fmla v21.4s, v1.4s, v9.4s\n" + "fmla v29.4s, v7.4s, v17.4s\n" + "ldr x22, [x15, #0x68]\n" + "ldr x21, [x15, #0x78]\n" + "fmla v28.4s, v0.4s, v18.4s\n" + "fmla v22.4s, v8.4s, v16.4s\n" + "ldr q16, [x20, x14]\n" + "ldr x20, [x15, #0x80]\n" + "fmla v26.4s, v4.4s, v17.4s\n" + "fmla v25.4s, v3.4s, v17.4s\n" + "ldr x27, [x17, #0x18]\n" + "fmla v21.4s, v0.4s, v17.4s\n" + "fmla v24.4s, v4.4s, v19.4s\n" + "fmla v23.4s, v1.4s, v17.4s\n" + "ldr q17, [x25, x14]\n" + "fmla v29.4s, v1.4s, v18.4s\n" + "ldr q20, [x24, x14]\n" + "fmla v28.4s, v2.4s, v16.4s\n" + "fmla v27.4s, v1.4s, v16.4s\n" + "ldr q16, [x23, x14]\n" + "ldr x26, [x15, #0x90]\n" + "fmla v25.4s, v5.4s, v19.4s\n" + "fmla v21.4s, v2.4s, v19.4s\n" + "ldr x25, [x15, #0xa0]\n" + "ldr x24, [x15, #0x98]\n" + "fmla v26.4s, v0.4s, v17.4s\n" + "fmla v24.4s, v2.4s, v20.4s\n" + "fmla v28.4s, v8.4s, v19.4s\n" + "fmla v27.4s, v7.4s, v19.4s\n" + "fmla v22.4s, v1.4s, v19.4s\n" + "ldr q19, [x22, x14]\n" + "fmla v23.4s, v3.4s, v16.4s\n" + "ldr x23, [x15, #0xa8]\n" + "fmla v26.4s, v6.4s, v16.4s\n" + "ldr q18, [x20, x14]\n" + "fmla v25.4s, v7.4s, v19.4s\n" + "ldr x22, [x15, #0xc0]\n" + "fmla v24.4s, v6.4s, v19.4s\n" + "fmla v21.4s, v4.4s, v19.4s\n" + "fmla v29.4s, v3.4s, v17.4s\n" + "ldr q17, [x12, x14]\n" + "fmla v27.4s, v5.4s, v20.4s\n" + "ldr q16, [x21, x14]\n" + "fmla v23.4s, v5.4s, v19.4s\n" + "fmla v22.4s, v3.4s, v19.4s\n" + "ldr x21, [x15, #0xb0]\n" + "ldr x20, [x15, #0xb8]\n" + "fmla v26.4s, v8.4s, v19.4s\n" + "fmla v24.4s, v8.4s, v17.4s\n" + "fmla v21.4s, v6.4s, v16.4s\n" + "fmla v28.4s, v3.4s, v18.4s\n" + "fmla v25.4s, v0.4s, v18.4s\n" + "fmla v22.4s, v5.4s, v17.4s\n" + "ldr q17, [x11, x14]\n" + "fmla v23.4s, v7.4s, v16.4s\n" + "ldr q16, [x26, x14]\n" + "fmla v29.4s, v4.4s, v18.4s\n" + "fmla v26.4s, v1.4s, v18.4s\n" + "ldr q18, [x24, x14]\n" + "fmla v28.4s, v5.4s, v17.4s\n" + "fmla v27.4s, v4.4s, v17.4s\n" + "fmla v25.4s, v2.4s, v17.4s\n" + "fmla v24.4s, v1.4s, v17.4s\n" + "ldr q17, [x25, x14]\n" + "fmla v21.4s, v8.4s, v16.4s\n" + "fmla v22.4s, v7.4s, v16.4s\n" + "ldr q16, [x23, x14]\n" + "fmla v29.4s, v2.4s, v17.4s\n" + "fmla v26.4s, v7.4s, v18.4s\n" + "fmla v25.4s, v6.4s, v18.4s\n" + "fmla v23.4s, v4.4s, v18.4s\n" + "fmla v21.4s, v3.4s, v18.4s\n" + "ldr q18, [x21, x14]\n" + "fmla v22.4s, v4.4s, v16.4s\n" + "fmla v28.4s, v1.4s, v17.4s\n" + "fmax v28.4s, v28.4s, v15.4s\n" + "fmla v27.4s, v0.4s, v17.4s\n" + "ldr q17, [x20, x14]\n" + "fmla v29.4s, v6.4s, v18.4s\n" + "fmax v29.4s, v29.4s, v15.4s\n" + "fmla v24.4s, v7.4s, v16.4s\n" + "fmla v21.4s, v5.4s, v16.4s\n" + "fmin v29.4s, v29.4s, v14.4s\n" + "str q29, [x10, x13]\n" + "fmla v23.4s, v0.4s, v18.4s\n" + "fmla v22.4s, v2.4s, v17.4s\n" + "ldr x20, [x17, #0x20]\n" + "fmin v28.4s, v28.4s, v14.4s\n" + "fmla v25.4s, v8.4s, v16.4s\n" + "ldr q16, [x22, x14]\n" + "fmla v26.4s, v3.4s, v18.4s\n" + "fmax v26.4s, v26.4s, v15.4s\n" + "fmla v27.4s, v8.4s, v17.4s\n" + "fmla v24.4s, v5.4s, v17.4s\n" + "fmax v27.4s, v27.4s, v15.4s\n" + "str q28, [x9, x13]\n" + "fmla v23.4s, v8.4s, v16.4s\n" + "fmla v21.4s, v7.4s, v16.4s\n" + "fmax v25.4s, v25.4s, v15.4s\n" + "ldr x23, [x17, #0x28]\n" + "fmla v22.4s, v6.4s, v16.4s\n" + "fmin v27.4s, v27.4s, v14.4s\n" + "str q27, [x28, x13]\n" + "ldr x22, [x17, #0x30]\n" + "fmin v26.4s, v26.4s, v14.4s\n" + "fmin v25.4s, v25.4s, v14.4s\n" + "str q26, [x27, x13]\n" + "ldr x21, [x17, #0x38]\n" + "fmax v24.4s, v24.4s, v15.4s\n" + "fmax v23.4s, v23.4s, v15.4s\n" + "str q25, [x20, x13]\n" + "ldr x20, [x17, #0x40]\n" + "fmax v21.4s, v21.4s, v15.4s\n" + "fmax v22.4s, v22.4s, v15.4s\n" + "add x14, x14, #0x10\n" + "fmin v24.4s, v24.4s, v14.4s\n" + "fmin v23.4s, v23.4s, v14.4s\n" + "str q24, [x23, x13]\n" + "fmin v21.4s, v21.4s, v14.4s\n" + "fmin v22.4s, v22.4s, v14.4s\n" + "str q23, [x22, x13]\n" + "str q21, [x21, x13]\n" + "str q22, [x20, x13]\n" "3:" // Oddments "tst %x[n_channels], #0x3\n" "beq 48f\n" - "ldr q16, [x15, #0x0]\n" - "ldr q0, [x15, #0x10]\n" - "mov x12, x13\n" - "ldr q1, [x15, #0x20]\n" - "ldr q2, [x15, #0x30]\n" - "ldr q3, [x15, #0x40]\n" - "ldr q4, [x15, #0x50]\n" - "ldr q5, [x15, #0x60]\n" - "ldr q6, [x15, #0x70]\n" - "ldr q7, [x15, #0x80]\n" - "ldr q8, [x15, #0x90]\n" - "ldr x24, [x14, #0x0]\n" - "ldr x23, [x14, #0x8]\n" - "add x24, x24, x13\n" - "add x23, x23, x13\n" - "ldr x22, [x14, #0x10]\n" - "ldr x21, [x14, #0x18]\n" - "add x22, x22, x13\n" - "add x21, x21, x13\n" - "ldr x20, [x14, #0x20]\n" - "add x20, x20, x13\n" + "ldr q31, [x16, #0x0]\n" + "ldr q0, [x16, #0x10]\n" + "mov x13, x14\n" + "ldr q1, [x16, #0x20]\n" + "ldr q2, [x16, #0x30]\n" + "ldr q3, [x16, #0x40]\n" + "ldr q4, [x16, #0x50]\n" + "ldr q5, [x16, #0x60]\n" + "ldr q6, [x16, #0x70]\n" + "ldr q7, [x16, #0x80]\n" + "ldr q8, [x16, #0x90]\n" + "ldr x24, [x15, #0x0]\n" + "ldr x23, [x15, #0x8]\n" + "add x24, x24, x14\n" + "add x23, x23, x14\n" + "ldr x22, [x15, #0x10]\n" + "ldr x21, [x15, #0x18]\n" + "add x22, x22, x14\n" + "add x21, x21, x14\n" + "ldr x20, [x15, #0x20]\n" + "add x20, x20, x14\n" "tbz %x[n_channels], #1, 4f\n" "ld1 { v9.d }[0], [x24], #0x8\n" "ld1 { v10.d }[0], [x23], #0x8\n" @@ -506,19 +506,19 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "ld1 { v12.s }[0], [x21], #0x4\n" "ld1 { v13.s }[0], [x20], #0x4\n" "5:" // Oddments: Load inputs (2, 2), (0, 0), (0, 4), (4, 0), (1, 2): Bit 1: End - "mov v23.16b, v16.16b\n fmla v23.4s, v8.4s, v9.4s\n" - "mov v25.16b, v16.16b\n fmla v25.4s, v6.4s, v9.4s\n" - "ldr x20, [x14, #0x28]\n" - "add x20, x20, x13\n" - "mov v24.16b, v16.16b\n fmla v24.4s, v7.4s, v9.4s\n" - "mov v26.16b, v16.16b\n fmla v26.4s, v5.4s, v9.4s\n" - "mov v27.16b, v16.16b\n fmla v27.4s, v4.4s, v9.4s\n" - "mov v28.16b, v16.16b\n fmla v28.4s, v3.4s, v9.4s\n" - "mov v29.16b, v16.16b\n fmla v29.4s, v2.4s, v9.4s\n" + "mov v23.16b, v31.16b\n fmla v23.4s, v8.4s, v9.4s\n" + "mov v25.16b, v31.16b\n fmla v25.4s, v6.4s, v9.4s\n" + "ldr x20, [x15, #0x28]\n" + "add x20, x20, x14\n" + "mov v24.16b, v31.16b\n fmla v24.4s, v7.4s, v9.4s\n" + "mov v26.16b, v31.16b\n fmla v26.4s, v5.4s, v9.4s\n" + "mov v27.16b, v31.16b\n fmla v27.4s, v4.4s, v9.4s\n" + "mov v28.16b, v31.16b\n fmla v28.4s, v3.4s, v9.4s\n" + "mov v29.16b, v31.16b\n fmla v29.4s, v2.4s, v9.4s\n" "fmla v23.4s, v0.4s, v10.4s\n" "fmla v25.4s, v2.4s, v11.4s\n" - "mov v30.16b, v16.16b\n fmla v30.4s, v1.4s, v9.4s\n" - "mov v31.16b, v16.16b\n fmla v31.4s, v0.4s, v9.4s\n" + "mov v30.16b, v31.16b\n fmla v30.4s, v1.4s, v9.4s\n" + "fmla v31.4s, v0.4s, v9.4s\n" "fmla v29.4s, v6.4s, v12.4s\n" "fmla v23.4s, v5.4s, v13.4s\n" "fmla v24.4s, v4.4s, v13.4s\n" @@ -534,9 +534,9 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "6:" // Oddments: Load input (4, 4): Bit 1: Unset "ld1 { v12.s }[0], [x20], #0x4\n" "7:" // Oddments: Load input (4, 4): Bit 1: End - "ldr x20, [x14, #0x30]\n" + "ldr x20, [x15, #0x30]\n" "fmla v31.4s, v8.4s, v12.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "tbz %x[n_channels], #1, 8f\n" "ld1 { v11.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #0, 9f\n" @@ -545,10 +545,10 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "8:" // Oddments: Load input (2, 1): Bit 1: Unset "ld1 { v11.s }[0], [x20], #0x4\n" "9:" // Oddments: Load input (2, 1): Bit 1: End - "ldr x20, [x14, #0x38]\n" + "ldr x20, [x15, #0x38]\n" "fmla v23.4s, v7.4s, v11.4s\n" "fmla v24.4s, v6.4s, v11.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "fmla v26.4s, v4.4s, v11.4s\n" "fmla v27.4s, v3.4s, v11.4s\n" "fmla v29.4s, v1.4s, v11.4s\n" @@ -561,10 +561,10 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "10:" // Oddments: Load input (0, 1): Bit 1: Unset "ld1 { v13.s }[0], [x20], #0x4\n" "11:" // Oddments: Load input (0, 1): Bit 1: End - "ldr x20, [x14, #0x40]\n" + "ldr x20, [x15, #0x40]\n" "fmla v23.4s, v1.4s, v13.4s\n" "fmla v24.4s, v0.4s, v13.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "tbz %x[n_channels], #1, 12f\n" "ld1 { v12.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #0, 13f\n" @@ -573,10 +573,10 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "12:" // Oddments: Load input (0, 3): Bit 1: Unset "ld1 { v12.s }[0], [x20], #0x4\n" "13:" // Oddments: Load input (0, 3): Bit 1: End - "ldr x20, [x14, #0x48]\n" + "ldr x20, [x15, #0x48]\n" "fmla v24.4s, v2.4s, v12.4s\n" "fmla v25.4s, v1.4s, v12.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "tbz %x[n_channels], #1, 14f\n" "ld1 { v10.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #0, 15f\n" @@ -585,10 +585,10 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "14:" // Oddments: Load input (2, 3): Bit 1: Unset "ld1 { v10.s }[0], [x20], #0x4\n" "15:" // Oddments: Load input (2, 3): Bit 1: End - "ldr x20, [x14, #0x50]\n" + "ldr x20, [x15, #0x50]\n" "fmla v24.4s, v8.4s, v10.4s\n" "fmla v25.4s, v7.4s, v10.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "fmla v27.4s, v5.4s, v10.4s\n" "fmla v28.4s, v4.4s, v10.4s\n" "fmla v30.4s, v2.4s, v10.4s\n" @@ -601,10 +601,10 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "16:" // Oddments: Load input (1, 0): Bit 1: Unset "ld1 { v11.s }[0], [x20], #0x4\n" "17:" // Oddments: Load input (1, 0): Bit 1: End - "ldr x20, [x14, #0x58]\n" + "ldr x20, [x15, #0x58]\n" "fmla v23.4s, v3.4s, v11.4s\n" "fmla v26.4s, v0.4s, v11.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "tbz %x[n_channels], #1, 18f\n" "ld1 { v13.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #0, 19f\n" @@ -613,10 +613,10 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "18:" // Oddments: Load input (1, 4): Bit 1: Unset "ld1 { v13.s }[0], [x20], #0x4\n" "19:" // Oddments: Load input (1, 4): Bit 1: End - "ldr x20, [x14, #0x60]\n" + "ldr x20, [x15, #0x60]\n" "fmla v25.4s, v5.4s, v13.4s\n" "fmla v28.4s, v2.4s, v13.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "tbz %x[n_channels], #1, 20f\n" "ld1 { v12.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #0, 21f\n" @@ -625,10 +625,10 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "20:" // Oddments: Load input (3, 0): Bit 1: Unset "ld1 { v12.s }[0], [x20], #0x4\n" "21:" // Oddments: Load input (3, 0): Bit 1: End - "ldr x20, [x14, #0x68]\n" + "ldr x20, [x15, #0x68]\n" "fmla v26.4s, v6.4s, v12.4s\n" "fmla v29.4s, v3.4s, v12.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "tbz %x[n_channels], #1, 22f\n" "ld1 { v10.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #0, 23f\n" @@ -637,10 +637,10 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "22:" // Oddments: Load input (3, 2): Bit 1: Unset "ld1 { v10.s }[0], [x20], #0x4\n" "23:" // Oddments: Load input (3, 2): Bit 1: End - "ldr x20, [x14, #0x70]\n" + "ldr x20, [x15, #0x70]\n" "fmla v26.4s, v8.4s, v10.4s\n" "fmla v27.4s, v7.4s, v10.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "fmla v28.4s, v6.4s, v10.4s\n" "fmla v29.4s, v5.4s, v10.4s\n" "fmla v30.4s, v4.4s, v10.4s\n" @@ -653,10 +653,10 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "24:" // Oddments: Load input (3, 4): Bit 1: Unset "ld1 { v11.s }[0], [x20], #0x4\n" "25:" // Oddments: Load input (3, 4): Bit 1: End - "ldr x20, [x14, #0x78]\n" + "ldr x20, [x15, #0x78]\n" "fmla v28.4s, v8.4s, v11.4s\n" "fmla v31.4s, v5.4s, v11.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "tbz %x[n_channels], #1, 26f\n" "ld1 { v13.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #0, 27f\n" @@ -665,10 +665,10 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "26:" // Oddments: Load input (4, 1): Bit 1: Unset "ld1 { v13.s }[0], [x20], #0x4\n" "27:" // Oddments: Load input (4, 1): Bit 1: End - "ldr x20, [x14, #0x80]\n" + "ldr x20, [x15, #0x80]\n" "fmla v29.4s, v7.4s, v13.4s\n" "fmla v30.4s, v6.4s, v13.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "tbz %x[n_channels], #1, 28f\n" "ld1 { v12.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #0, 29f\n" @@ -677,10 +677,10 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "28:" // Oddments: Load input (1, 1): Bit 1: Unset "ld1 { v12.s }[0], [x20], #0x4\n" "29:" // Oddments: Load input (1, 1): Bit 1: End - "ldr x20, [x14, #0x88]\n" + "ldr x20, [x15, #0x88]\n" "fmla v23.4s, v4.4s, v12.4s\n" "fmla v24.4s, v3.4s, v12.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "fmla v26.4s, v1.4s, v12.4s\n" "fmla v27.4s, v0.4s, v12.4s\n" "tbz %x[n_channels], #1, 30f\n" @@ -691,10 +691,10 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "30:" // Oddments: Load input (1, 3): Bit 1: Unset "ld1 { v11.s }[0], [x20], #0x4\n" "31:" // Oddments: Load input (1, 3): Bit 1: End - "ldr x20, [x14, #0x90]\n" + "ldr x20, [x15, #0x90]\n" "fmla v24.4s, v5.4s, v11.4s\n" "fmla v25.4s, v4.4s, v11.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "fmla v27.4s, v2.4s, v11.4s\n" "fmla v28.4s, v1.4s, v11.4s\n" "tbz %x[n_channels], #1, 32f\n" @@ -705,10 +705,10 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "32:" // Oddments: Load input (4, 3): Bit 1: Unset "ld1 { v13.s }[0], [x20], #0x4\n" "33:" // Oddments: Load input (4, 3): Bit 1: End - "ldr x20, [x14, #0x98]\n" + "ldr x20, [x15, #0x98]\n" "fmla v30.4s, v8.4s, v13.4s\n" "fmla v31.4s, v7.4s, v13.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "tbz %x[n_channels], #1, 34f\n" "ld1 { v12.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #0, 35f\n" @@ -717,10 +717,10 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "34:" // Oddments: Load input (3, 1): Bit 1: Unset "ld1 { v12.s }[0], [x20], #0x4\n" "35:" // Oddments: Load input (3, 1): Bit 1: End - "ldr x20, [x14, #0xa0]\n" + "ldr x20, [x15, #0xa0]\n" "fmla v26.4s, v7.4s, v12.4s\n" "fmla v27.4s, v6.4s, v12.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "fmla v29.4s, v4.4s, v12.4s\n" "fmla v30.4s, v3.4s, v12.4s\n" "tbz %x[n_channels], #1, 36f\n" @@ -731,10 +731,10 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "36:" // Oddments: Load input (0, 2): Bit 1: Unset "ld1 { v11.s }[0], [x20], #0x4\n" "37:" // Oddments: Load input (0, 2): Bit 1: End - "ldr x20, [x14, #0xa8]\n" + "ldr x20, [x15, #0xa8]\n" "fmla v23.4s, v2.4s, v11.4s\n" "fmla v24.4s, v1.4s, v11.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "fmla v25.4s, v0.4s, v11.4s\n" "tbz %x[n_channels], #1, 38f\n" "ld1 { v13.d }[0], [x20], #0x8\n" @@ -744,10 +744,10 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "38:" // Oddments: Load input (3, 3): Bit 1: Unset "ld1 { v13.s }[0], [x20], #0x4\n" "39:" // Oddments: Load input (3, 3): Bit 1: End - "ldr x20, [x14, #0xb0]\n" + "ldr x20, [x15, #0xb0]\n" "fmla v27.4s, v8.4s, v13.4s\n" "fmla v28.4s, v7.4s, v13.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "fmla v30.4s, v5.4s, v13.4s\n" "fmla v31.4s, v4.4s, v13.4s\n" "tbz %x[n_channels], #1, 40f\n" @@ -758,10 +758,10 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "40:" // Oddments: Load input (2, 0): Bit 1: Unset "ld1 { v12.s }[0], [x20], #0x4\n" "41:" // Oddments: Load input (2, 0): Bit 1: End - "ldr x20, [x14, #0xb8]\n" + "ldr x20, [x15, #0xb8]\n" "fmla v23.4s, v6.4s, v12.4s\n" "fmla v26.4s, v3.4s, v12.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "fmla v29.4s, v0.4s, v12.4s\n" "tbz %x[n_channels], #1, 42f\n" "ld1 { v11.d }[0], [x20], #0x8\n" @@ -771,10 +771,10 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "42:" // Oddments: Load input (2, 4): Bit 1: Unset "ld1 { v11.s }[0], [x20], #0x4\n" "43:" // Oddments: Load input (2, 4): Bit 1: End - "ldr x20, [x14, #0xc0]\n" + "ldr x20, [x15, #0xc0]\n" "fmla v25.4s, v8.4s, v11.4s\n" "fmla v28.4s, v5.4s, v11.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x14\n" "fmla v31.4s, v2.4s, v11.4s\n" "tbz %x[n_channels], #1, 44f\n" "ld1 { v13.d }[0], [x20], #0x8\n" @@ -786,120 +786,120 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "45:" // Oddments: Load input (4, 2): Bit 1: End "fmla v29.4s, v8.4s, v13.4s\n" "fmla v30.4s, v7.4s, v13.4s\n" - "fmax v23.4s, v23.4s, v18.4s\n" + "fmax v23.4s, v23.4s, v15.4s\n" "fmla v31.4s, v6.4s, v13.4s\n" - "fmax v24.4s, v24.4s, v18.4s\n" - "fmax v25.4s, v25.4s, v18.4s\n" - "fmax v26.4s, v26.4s, v18.4s\n" - "fmax v27.4s, v27.4s, v18.4s\n" - "fmax v28.4s, v28.4s, v18.4s\n" - "fmax v29.4s, v29.4s, v18.4s\n" - "fmax v30.4s, v30.4s, v18.4s\n" - "fmax v31.4s, v31.4s, v18.4s\n" - "fmin v23.4s, v23.4s, v17.4s\n" - "fmin v24.4s, v24.4s, v17.4s\n" - "fmin v25.4s, v25.4s, v17.4s\n" - "fmin v26.4s, v26.4s, v17.4s\n" - "fmin v27.4s, v27.4s, v17.4s\n" - "fmin v28.4s, v28.4s, v17.4s\n" - "fmin v29.4s, v29.4s, v17.4s\n" - "fmin v30.4s, v30.4s, v17.4s\n" - "fmin v31.4s, v31.4s, v17.4s\n" + "fmax v24.4s, v24.4s, v15.4s\n" + "fmax v25.4s, v25.4s, v15.4s\n" + "fmax v26.4s, v26.4s, v15.4s\n" + "fmax v27.4s, v27.4s, v15.4s\n" + "fmax v28.4s, v28.4s, v15.4s\n" + "fmax v29.4s, v29.4s, v15.4s\n" + "fmax v30.4s, v30.4s, v15.4s\n" + "fmax v31.4s, v31.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v14.4s\n" + "fmin v24.4s, v24.4s, v14.4s\n" + "fmin v25.4s, v25.4s, v14.4s\n" + "fmin v26.4s, v26.4s, v14.4s\n" + "fmin v27.4s, v27.4s, v14.4s\n" + "fmin v28.4s, v28.4s, v14.4s\n" + "fmin v29.4s, v29.4s, v14.4s\n" + "fmin v30.4s, v30.4s, v14.4s\n" + "fmin v31.4s, v31.4s, v14.4s\n" "tbz %x[n_channels], #1, 46f\n" - "ldr x23, [x16, #0x0]\n" - "add x23, x23, x12\n" - "st1 { v23.d }[0], [x23]\n" - "ldr x22, [x16, #0x8]\n" - "ldr x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x18]\n" - "add x22, x22, x12\n" - "add x21, x21, x12\n" - "ldr x23, [x16, #0x20]\n" - "add x20, x20, x12\n" - "add x23, x23, x12\n" - "st1 { v24.d }[0], [x22]\n" - "st1 { v25.d }[0], [x21]\n" - "ldr x22, [x16, #0x28]\n" - "ldr x21, [x16, #0x30]\n" - "add x22, x22, x12\n" - "st1 { v26.d }[0], [x20]\n" - "ldr x20, [x16, #0x38]\n" - "add x21, x21, x12\n" - "add x20, x20, x12\n" - "st1 { v27.d }[0], [x23]\n" - "ldr x23, [x16, #0x40]\n" - "add x23, x23, x12\n" - "add x12, x12, #0x8\n" - "st1 { v28.d }[0], [x22]\n" - "st1 { v29.d }[0], [x21]\n" - "st1 { v30.d }[0], [x20]\n" - "st1 { v31.d }[0], [x23]\n" + "ldr x20, [x17, #0x0]\n" + "add x20, x20, x13\n" + "st1 { v23.d }[0], [x20]\n" + "ldr x23, [x17, #0x8]\n" + "ldr x22, [x17, #0x10]\n" + "ldr x21, [x17, #0x18]\n" + "add x23, x23, x13\n" + "add x22, x22, x13\n" + "ldr x20, [x17, #0x20]\n" + "add x21, x21, x13\n" + "add x20, x20, x13\n" + "st1 { v24.d }[0], [x23]\n" + "st1 { v25.d }[0], [x22]\n" + "ldr x23, [x17, #0x28]\n" + "ldr x22, [x17, #0x30]\n" + "add x23, x23, x13\n" + "st1 { v26.d }[0], [x21]\n" + "ldr x21, [x17, #0x38]\n" + "add x22, x22, x13\n" + "add x21, x21, x13\n" + "st1 { v27.d }[0], [x20]\n" + "ldr x20, [x17, #0x40]\n" + "add x20, x20, x13\n" + "add x13, x13, #0x8\n" + "st1 { v28.d }[0], [x23]\n" + "st1 { v29.d }[0], [x22]\n" + "st1 { v30.d }[0], [x21]\n" + "st1 { v31.d }[0], [x20]\n" "tbz %x[n_channels], #0, 47f\n" - "ldr x23, [x16, #0x0]\n" - "add x23, x23, x12\n" - "st1 { v23.s }[2], [x23]\n" - "ldr x22, [x16, #0x8]\n" - "ldr x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x18]\n" - "add x22, x22, x12\n" - "add x21, x21, x12\n" - "ldr x23, [x16, #0x20]\n" - "add x20, x20, x12\n" - "add x23, x23, x12\n" - "st1 { v24.s }[2], [x22]\n" - "st1 { v25.s }[2], [x21]\n" - "ldr x22, [x16, #0x28]\n" - "ldr x21, [x16, #0x30]\n" - "add x22, x22, x12\n" - "st1 { v26.s }[2], [x20]\n" - "ldr x20, [x16, #0x38]\n" - "add x21, x21, x12\n" - "add x20, x20, x12\n" - "st1 { v27.s }[2], [x23]\n" - "ldr x23, [x16, #0x40]\n" - "add x23, x23, x12\n" - "st1 { v28.s }[2], [x22]\n" - "st1 { v29.s }[2], [x21]\n" - "st1 { v30.s }[2], [x20]\n" - "st1 { v31.s }[2], [x23]\n" + "ldr x20, [x17, #0x0]\n" + "add x20, x20, x13\n" + "st1 { v23.s }[2], [x20]\n" + "ldr x23, [x17, #0x8]\n" + "ldr x22, [x17, #0x10]\n" + "ldr x21, [x17, #0x18]\n" + "add x23, x23, x13\n" + "add x22, x22, x13\n" + "ldr x20, [x17, #0x20]\n" + "add x21, x21, x13\n" + "add x20, x20, x13\n" + "st1 { v24.s }[2], [x23]\n" + "st1 { v25.s }[2], [x22]\n" + "ldr x23, [x17, #0x28]\n" + "ldr x22, [x17, #0x30]\n" + "add x23, x23, x13\n" + "st1 { v26.s }[2], [x21]\n" + "ldr x21, [x17, #0x38]\n" + "add x22, x22, x13\n" + "add x21, x21, x13\n" + "st1 { v27.s }[2], [x20]\n" + "ldr x20, [x17, #0x40]\n" + "add x20, x20, x13\n" + "st1 { v28.s }[2], [x23]\n" + "st1 { v29.s }[2], [x22]\n" + "st1 { v30.s }[2], [x21]\n" + "st1 { v31.s }[2], [x20]\n" "b 47f\n" "46:" // Oddments: Store: Bit 1: Unset - "ldr x23, [x16, #0x0]\n" - "add x23, x23, x12\n" - "st1 { v23.s }[0], [x23]\n" - "ldr x22, [x16, #0x8]\n" - "ldr x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x18]\n" - "add x22, x22, x12\n" - "add x21, x21, x12\n" - "ldr x23, [x16, #0x20]\n" - "add x20, x20, x12\n" - "add x23, x23, x12\n" - "st1 { v24.s }[0], [x22]\n" - "st1 { v25.s }[0], [x21]\n" - "ldr x22, [x16, #0x28]\n" - "ldr x21, [x16, #0x30]\n" - "add x22, x22, x12\n" - "st1 { v26.s }[0], [x20]\n" - "ldr x20, [x16, #0x38]\n" - "add x21, x21, x12\n" - "add x20, x20, x12\n" - "st1 { v27.s }[0], [x23]\n" - "ldr x23, [x16, #0x40]\n" - "add x23, x23, x12\n" - "st1 { v28.s }[0], [x22]\n" - "st1 { v29.s }[0], [x21]\n" - "st1 { v30.s }[0], [x20]\n" - "st1 { v31.s }[0], [x23]\n" + "ldr x20, [x17, #0x0]\n" + "add x20, x20, x13\n" + "st1 { v23.s }[0], [x20]\n" + "ldr x23, [x17, #0x8]\n" + "ldr x22, [x17, #0x10]\n" + "ldr x21, [x17, #0x18]\n" + "add x23, x23, x13\n" + "add x22, x22, x13\n" + "ldr x20, [x17, #0x20]\n" + "add x21, x21, x13\n" + "add x20, x20, x13\n" + "st1 { v24.s }[0], [x23]\n" + "st1 { v25.s }[0], [x22]\n" + "ldr x23, [x17, #0x28]\n" + "ldr x22, [x17, #0x30]\n" + "add x23, x23, x13\n" + "st1 { v26.s }[0], [x21]\n" + "ldr x21, [x17, #0x38]\n" + "add x22, x22, x13\n" + "add x21, x21, x13\n" + "st1 { v27.s }[0], [x20]\n" + "ldr x20, [x17, #0x40]\n" + "add x20, x20, x13\n" + "st1 { v28.s }[0], [x23]\n" + "st1 { v29.s }[0], [x22]\n" + "st1 { v30.s }[0], [x21]\n" + "st1 { v31.s }[0], [x20]\n" "47:" // Oddments: Store: Bit 1: End "48:" // End : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v16", "v17", "v18", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp index 6c897d6eaa9f4b1136354792788f02be247a93f6..8a198c181822ff7537a6d23e599104bdea28e5d7 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include @@ -33,8 +33,8 @@ namespace arm_conv { namespace depthwise { -void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); -void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); +void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(const float *const *const input_ptrs, float *const *const outptrs, const void *params, unsigned int n_channels, const float activation_min, const float activation_max); +void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(const unsigned int n_tile_rows, const unsigned int n_tile_cols, const float *inptr, int64_t ld_input_row, int64_t ld_input_col, float *outptr, int64_t ld_output_row, int64_t ld_output_col, const void *params, unsigned int n_channels, const float activation_min, const float activation_max); class a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst : public DepthwiseDepthfirstStrategy { @@ -57,7 +57,7 @@ class a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst : public DepthwiseDepthfirst constexpr static unsigned int output_cols = 4; a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst(const CPUInfo *) - : DepthwiseDepthfirstStrategy(4, 3, 1) {} + : Parent(output_rows, output_cols, kernel_rows, kernel_cols, stride_rows, stride_cols) {} arm_gemm::VLType get_vl_type(void) const override { return vl_type; } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp index 6d2b6ee9980b461d15f7e324f87b3e5681081091..3adf8b0d9ff3738a1517640e09e8058bb1dc75eb 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ +#if defined(__aarch64__) namespace arm_conv { namespace depthwise { @@ -124,9 +124,9 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "add x27, x10, x22, LSL #2\n" "add x23, x5, x5\n" "add x20, %x[params_struct], %[offsetof_args_min]\n" - "ld1r { v15.4s }, [x20]\n" + "ld1r { v13.4s }, [x20]\n" "add x20, %x[params_struct], %[offsetof_args_max]\n" - "ld1r { v14.4s }, [x20]\n" + "ld1r { v15.4s }, [x20]\n" "add x26, x9, x24, LSL #2\n" "add x25, x28, x4\n" "add x24, x27, x22, LSL #2\n" @@ -134,7 +134,7 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "mov x21, #0x0\n" "sub x20, XZR, x6\n" "cbz x13, 4f\n" - "ldr q13, [x16, #0x0]\n" + "ldr q14, [x16, #0x0]\n" "ldr q0, [x16, #0x10]\n" "cmp x6, x13, LSL #4\n" "ldr q1, [x16, #0x20]\n" @@ -152,499 +152,499 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "ldr q12, [x14, x11]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "mov v21.16b, v13.16b\n fmla v21.4s, v4.4s, v9.4s\n" - "mov v16.16b, v13.16b\n fmla v16.4s, v8.4s, v9.4s\n" + "mov v26.16b, v14.16b\n fmla v26.4s, v4.4s, v9.4s\n" + "mov v28.16b, v14.16b\n fmla v28.4s, v8.4s, v9.4s\n" "add x6, x6, #0x10\n" "cmp x6, x13, LSL #4\n" - "mov v22.16b, v13.16b\n fmla v22.4s, v3.4s, v9.4s\n" - "mov v25.16b, v13.16b\n fmla v25.4s, v1.4s, v9.4s\n" + "mov v16.16b, v14.16b\n fmla v16.4s, v3.4s, v9.4s\n" + "mov v22.16b, v14.16b\n fmla v22.4s, v1.4s, v9.4s\n" "add x20, x20, #0x10\n" "add x21, x21, #0x10\n" - "mov v26.16b, v13.16b\n fmla v26.4s, v0.4s, v9.4s\n" - "fmla v21.4s, v5.4s, v12.4s\n" - "mov v17.16b, v13.16b\n fmla v17.4s, v7.4s, v9.4s\n" - "mov v18.16b, v13.16b\n fmla v18.4s, v6.4s, v9.4s\n" - "mov v20.16b, v13.16b\n fmla v20.4s, v5.4s, v9.4s\n" - "mov v24.16b, v13.16b\n fmla v24.4s, v2.4s, v9.4s\n" + "mov v23.16b, v14.16b\n fmla v23.4s, v0.4s, v9.4s\n" + "fmla v26.4s, v5.4s, v12.4s\n" + "mov v25.16b, v14.16b\n fmla v25.4s, v7.4s, v9.4s\n" + "mov v17.16b, v14.16b\n fmla v17.4s, v6.4s, v9.4s\n" + "mov v31.16b, v14.16b\n fmla v31.4s, v5.4s, v9.4s\n" + "mov v20.16b, v14.16b\n fmla v20.4s, v2.4s, v9.4s\n" "ldr q9, [x12, x17]\n" - "fmla v16.4s, v0.4s, v10.4s\n" - "ld1 { v10.4s }, [x26]\n" - "mov v19.16b, v13.16b\n fmla v19.4s, v2.4s, v11.4s\n" - "ldr q11, [x26, x25]\n" - "fmla v22.4s, v4.4s, v12.4s\n" - "fmla v25.4s, v2.4s, v12.4s\n" - "fmla v26.4s, v1.4s, v12.4s\n" - "mov v28.16b, v13.16b\n fmla v28.4s, v6.4s, v10.4s\n" + "fmla v28.4s, v0.4s, v10.4s\n" + "ld1 { v30.4s }, [x26]\n" + "mov v29.16b, v14.16b\n fmla v29.4s, v2.4s, v11.4s\n" + "ldr q27, [x26, x25]\n" + "fmla v16.4s, v4.4s, v12.4s\n" + "fmla v22.4s, v2.4s, v12.4s\n" + "fmla v23.4s, v1.4s, v12.4s\n" + "mov v21.16b, v14.16b\n fmla v21.4s, v6.4s, v30.4s\n" "ldr q10, [x12, x11]\n" - "fmla v21.4s, v7.4s, v9.4s\n" - "fmla v17.4s, v8.4s, v12.4s\n" - "fmla v18.4s, v7.4s, v12.4s\n" - "fmla v19.4s, v6.4s, v12.4s\n" - "mov v23.16b, v13.16b\n fmla v23.4s, v3.4s, v12.4s\n" - "mov v27.16b, v13.16b\n fmla v27.4s, v0.4s, v12.4s\n" - "ldr q12, [x7, x4]\n" - "mov v31.16b, v13.16b\n fmla v31.4s, v8.4s, v11.4s\n" - "ldr q11, [x7, x28]\n" - "fmla v22.4s, v6.4s, v9.4s\n" - "fmla v25.4s, v4.4s, v9.4s\n" - "fmla v26.4s, v3.4s, v9.4s\n" - "fmla v20.4s, v8.4s, v9.4s\n" - "fmla v24.4s, v5.4s, v9.4s\n" - "fmla v28.4s, v2.4s, v9.4s\n" - "fmla v21.4s, v8.4s, v10.4s\n" - "fmla v16.4s, v1.4s, v12.4s\n" - "fmla v17.4s, v0.4s, v12.4s\n" - "ldr q12, [x15, x25]\n" - "fmla v18.4s, v2.4s, v11.4s\n" - "fmla v19.4s, v1.4s, v11.4s\n" - "ld1 { v11.4s }, [x9]\n" - "fmla v22.4s, v7.4s, v10.4s\n" - "fmla v23.4s, v6.4s, v10.4s\n" - "fmla v25.4s, v5.4s, v10.4s\n" - "fmla v26.4s, v4.4s, v10.4s\n" - "fmla v27.4s, v3.4s, v10.4s\n" - "fmla v31.4s, v0.4s, v10.4s\n" - "fmla v24.4s, v6.4s, v11.4s\n" - "fmla v28.4s, v3.4s, v11.4s\n" - "ldr q11, [x9, x25]\n" - "fmla v19.4s, v5.4s, v12.4s\n" - "fmla v23.4s, v2.4s, v12.4s\n" - "ldr q12, [x15, x11]\n" - "fmla v27.4s, v8.4s, v11.4s\n" - "fmla v31.4s, v5.4s, v11.4s\n" - "mov v29.16b, v13.16b\n fmla v29.4s, v1.4s, v9.4s\n" - "mov v30.16b, v13.16b\n fmla v30.4s, v0.4s, v9.4s\n" + "fmla v26.4s, v7.4s, v9.4s\n" + "fmla v25.4s, v8.4s, v12.4s\n" + "fmla v17.4s, v7.4s, v12.4s\n" + "fmla v29.4s, v6.4s, v12.4s\n" + "mov v24.16b, v14.16b\n fmla v24.4s, v3.4s, v12.4s\n" + "mov v19.16b, v14.16b\n fmla v19.4s, v0.4s, v12.4s\n" + "ldr q11, [x7, x4]\n" + "mov v30.16b, v14.16b\n fmla v30.4s, v8.4s, v27.4s\n" + "ldr q12, [x7, x28]\n" + "fmla v16.4s, v6.4s, v9.4s\n" + "fmla v22.4s, v4.4s, v9.4s\n" + "fmla v23.4s, v3.4s, v9.4s\n" + "mov v27.16b, v14.16b\n fmla v27.4s, v1.4s, v9.4s\n" + "mov v18.16b, v14.16b\n fmla v18.4s, v0.4s, v9.4s\n" + "ldr q14, [x16, #0x0]\n" + "fmla v31.4s, v8.4s, v9.4s\n" + "fmla v20.4s, v5.4s, v9.4s\n" + "fmla v21.4s, v2.4s, v9.4s\n" "ld1 { v9.4s }, [x15]\n" - "fmla v29.4s, v2.4s, v10.4s\n" - "fmla v30.4s, v1.4s, v10.4s\n" + "fmla v26.4s, v8.4s, v10.4s\n" + "fmla v28.4s, v1.4s, v11.4s\n" + "fmla v25.4s, v0.4s, v11.4s\n" + "ldr q11, [x15, x25]\n" + "fmla v17.4s, v2.4s, v12.4s\n" + "fmla v29.4s, v1.4s, v12.4s\n" + "ld1 { v12.4s }, [x9]\n" + "fmla v16.4s, v7.4s, v10.4s\n" + "fmla v24.4s, v6.4s, v10.4s\n" + "fmla v22.4s, v5.4s, v10.4s\n" + "fmla v23.4s, v4.4s, v10.4s\n" + "fmla v19.4s, v3.4s, v10.4s\n" + "fmla v27.4s, v2.4s, v10.4s\n" + "fmla v18.4s, v1.4s, v10.4s\n" + "fmla v30.4s, v0.4s, v10.4s\n" "ldr q10, [x15, x17]\n" - "fmla v20.4s, v0.4s, v9.4s\n" - "fmla v21.4s, v1.4s, v10.4s\n" - "fmla v16.4s, v3.4s, v9.4s\n" - "ldr q11, [x26, x4]\n" - "fmla v17.4s, v4.4s, v10.4s\n" - "fmla v18.4s, v3.4s, v10.4s\n" - "fmla v22.4s, v0.4s, v10.4s\n" - "fmla v20.4s, v2.4s, v10.4s\n" - "fmla v21.4s, v2.4s, v12.4s\n" - "fmla v16.4s, v5.4s, v10.4s\n" + "fmla v31.4s, v0.4s, v9.4s\n" + "fmla v20.4s, v6.4s, v12.4s\n" + "fmla v21.4s, v3.4s, v12.4s\n" + "ldr q12, [x9, x25]\n" + "fmla v26.4s, v1.4s, v10.4s\n" + "fmla v28.4s, v3.4s, v9.4s\n" + "fmla v29.4s, v5.4s, v11.4s\n" + "fmla v24.4s, v2.4s, v11.4s\n" + "ldr q11, [x15, x11]\n" + "fmla v25.4s, v4.4s, v10.4s\n" + "fmla v17.4s, v3.4s, v10.4s\n" + "fmla v16.4s, v0.4s, v10.4s\n" + "fmla v19.4s, v8.4s, v12.4s\n" + "fmla v30.4s, v5.4s, v12.4s\n" + "ldr q9, [x26, x4]\n" + "fmla v31.4s, v2.4s, v10.4s\n" + "fmla v26.4s, v2.4s, v11.4s\n" + "fmla v28.4s, v5.4s, v10.4s\n" "ldr q10, [x14, x4]\n" - "fmla v17.4s, v5.4s, v12.4s\n" - "fmla v18.4s, v4.4s, v12.4s\n" - "fmla v19.4s, v3.4s, v12.4s\n" - "fmla v22.4s, v1.4s, v12.4s\n" - "fmla v23.4s, v0.4s, v12.4s\n" - "ldr q12, [x14, x28]\n" - "fmla v28.4s, v7.4s, v11.4s\n" - "fmla v29.4s, v6.4s, v11.4s\n" - "ldr q11, [x26, x28]\n" - "fmla v20.4s, v4.4s, v10.4s\n" - "fmla v21.4s, v3.4s, v10.4s\n" - "fmla v24.4s, v1.4s, v10.4s\n" - "fmla v25.4s, v0.4s, v10.4s\n" - "fmla v16.4s, v7.4s, v10.4s\n" - "fmla v17.4s, v6.4s, v10.4s\n" + "fmla v25.4s, v5.4s, v11.4s\n" + "fmla v17.4s, v4.4s, v11.4s\n" + "fmla v29.4s, v3.4s, v11.4s\n" + "fmla v16.4s, v1.4s, v11.4s\n" + "fmla v24.4s, v0.4s, v11.4s\n" + "ldr q11, [x14, x28]\n" + "fmla v21.4s, v7.4s, v9.4s\n" + "fmla v27.4s, v6.4s, v9.4s\n" + "ldr q12, [x26, x28]\n" + "fmla v31.4s, v4.4s, v10.4s\n" + "fmla v26.4s, v3.4s, v10.4s\n" + "fmla v20.4s, v1.4s, v10.4s\n" + "fmla v22.4s, v0.4s, v10.4s\n" + "fmla v28.4s, v7.4s, v10.4s\n" + "fmla v25.4s, v6.4s, v10.4s\n" "ldr q10, [x7, x17]\n" - "fmla v30.4s, v8.4s, v11.4s\n" - "fmla v31.4s, v7.4s, v11.4s\n" - "ldr q11, [x12, x4]\n" "fmla v18.4s, v8.4s, v12.4s\n" - "fmla v19.4s, v7.4s, v12.4s\n" - "fmla v22.4s, v5.4s, v12.4s\n" - "fmla v23.4s, v4.4s, v12.4s\n" - "fmla v26.4s, v2.4s, v12.4s\n" - "fmla v27.4s, v1.4s, v12.4s\n" + "fmla v30.4s, v7.4s, v12.4s\n" + "ldr q9, [x12, x4]\n" + "fmla v17.4s, v8.4s, v11.4s\n" + "fmla v29.4s, v7.4s, v11.4s\n" + "fmla v16.4s, v5.4s, v11.4s\n" + "fmla v24.4s, v4.4s, v11.4s\n" + "fmla v23.4s, v2.4s, v11.4s\n" + "fmla v19.4s, v1.4s, v11.4s\n" "ldr q12, [x7, x11]\n" "add x7, x7, #0x10\n" - "fmla v20.4s, v7.4s, v11.4s\n" - "fmla v21.4s, v6.4s, v11.4s\n" - "fmla v24.4s, v4.4s, v11.4s\n" - "fmla v25.4s, v3.4s, v11.4s\n" - "fmla v28.4s, v1.4s, v11.4s\n" - "fmla v29.4s, v0.4s, v11.4s\n" - "ldr q11, [x12, x28]\n" - "fmla v16.4s, v2.4s, v10.4s\n" - "fmla v17.4s, v1.4s, v10.4s\n" - "fmla v18.4s, v0.4s, v10.4s\n" + "fmla v31.4s, v7.4s, v9.4s\n" + "fmla v26.4s, v6.4s, v9.4s\n" + "fmla v20.4s, v4.4s, v9.4s\n" + "fmla v22.4s, v3.4s, v9.4s\n" + "fmla v21.4s, v1.4s, v9.4s\n" + "fmla v27.4s, v0.4s, v9.4s\n" + "ldr q9, [x12, x28]\n" + "fmla v28.4s, v2.4s, v10.4s\n" + "fmla v25.4s, v1.4s, v10.4s\n" + "fmla v17.4s, v0.4s, v10.4s\n" "ld1 { v10.4s }, [x14]\n" - "fmla v30.4s, v2.4s, v11.4s\n" - "fmla v19.4s, v0.4s, v12.4s\n" - "fmla v20.4s, v3.4s, v10.4s\n" - "fmla v24.4s, v0.4s, v10.4s\n" - "fmla v22.4s, v8.4s, v11.4s\n" - "fmla v23.4s, v7.4s, v11.4s\n" - "fmla v26.4s, v5.4s, v11.4s\n" - "fmla v27.4s, v4.4s, v11.4s\n" - "fmla v31.4s, v1.4s, v11.4s\n" + "fmla v18.4s, v2.4s, v9.4s\n" + "fmla v29.4s, v0.4s, v12.4s\n" + "fmla v31.4s, v3.4s, v10.4s\n" + "fmla v20.4s, v0.4s, v10.4s\n" + "fmla v16.4s, v8.4s, v9.4s\n" + "fmla v24.4s, v7.4s, v9.4s\n" + "fmla v23.4s, v5.4s, v9.4s\n" + "fmla v19.4s, v4.4s, v9.4s\n" + "fmla v30.4s, v1.4s, v9.4s\n" "ldr q11, [x9, x17]\n" - "fmla v17.4s, v2.4s, v12.4s\n" - "fmla v18.4s, v1.4s, v12.4s\n" + "fmla v25.4s, v2.4s, v12.4s\n" + "fmla v17.4s, v1.4s, v12.4s\n" "ldr q12, [x14, x25]\n" "add x14, x14, #0x10\n" - "fmla v16.4s, v6.4s, v10.4s\n" + "ldr q9, [x14, x17]\n" + "fmla v28.4s, v6.4s, v10.4s\n" "ld1 { v10.4s }, [x12]\n" - "fmla v29.4s, v4.4s, v11.4s\n" - "fmla v30.4s, v3.4s, v11.4s\n" - "fmla v19.4s, v8.4s, v12.4s\n" - "fmla v23.4s, v5.4s, v12.4s\n" - "fmla v27.4s, v2.4s, v12.4s\n" + "fmla v27.4s, v4.4s, v11.4s\n" + "fmla v18.4s, v3.4s, v11.4s\n" + "fmla v29.4s, v8.4s, v12.4s\n" + "fmla v24.4s, v5.4s, v12.4s\n" + "fmla v19.4s, v2.4s, v12.4s\n" "ldr q12, [x12, x25]\n" "add x12, x12, #0x10\n" - "fmla v20.4s, v6.4s, v10.4s\n" - "fmla v24.4s, v3.4s, v10.4s\n" - "fmla v28.4s, v0.4s, v10.4s\n" + "fmla v31.4s, v6.4s, v10.4s\n" + "fmla v20.4s, v3.4s, v10.4s\n" + "fmla v21.4s, v0.4s, v10.4s\n" "ldr q10, [x26, x17]\n" - "fmla v31.4s, v2.4s, v12.4s\n" - "fmla v29.4s, v7.4s, v10.4s\n" - "fmla v30.4s, v6.4s, v10.4s\n" - "fmla v24.4s, v8.4s, v11.4s\n" - "fmla v25.4s, v7.4s, v11.4s\n" - "fmla v26.4s, v6.4s, v11.4s\n" - "fmla v28.4s, v5.4s, v11.4s\n" + "fmla v30.4s, v2.4s, v12.4s\n" + "fmla v27.4s, v7.4s, v10.4s\n" + "fmla v18.4s, v6.4s, v10.4s\n" + "fmla v20.4s, v8.4s, v11.4s\n" + "fmla v22.4s, v7.4s, v11.4s\n" + "fmla v23.4s, v6.4s, v11.4s\n" + "fmla v21.4s, v5.4s, v11.4s\n" "ldr q11, [x9, x11]\n" - "fmla v27.4s, v5.4s, v12.4s\n" - "fmla v29.4s, v5.4s, v11.4s\n" - "fmla v30.4s, v4.4s, v11.4s\n" - "fmla v31.4s, v3.4s, v11.4s\n" - "fmla v23.4s, v8.4s, v12.4s\n" + "fmla v19.4s, v5.4s, v12.4s\n" + "fmla v27.4s, v5.4s, v11.4s\n" + "fmla v18.4s, v4.4s, v11.4s\n" + "fmla v30.4s, v3.4s, v11.4s\n" + "fmla v24.4s, v8.4s, v12.4s\n" "ldr q12, [x26, x11]\n" - "fmla v28.4s, v8.4s, v10.4s\n" + "fmla v21.4s, v8.4s, v10.4s\n" "ldr q10, [x15, x4]\n" - "fmla v25.4s, v8.4s, v11.4s\n" - "fmla v26.4s, v7.4s, v11.4s\n" + "fmla v22.4s, v8.4s, v11.4s\n" + "fmla v23.4s, v7.4s, v11.4s\n" "add x26, x26, #0x10\n" - "fmla v27.4s, v6.4s, v11.4s\n" + "fmla v19.4s, v6.4s, v11.4s\n" "ldr q11, [x15, x28]\n" - "fmla v29.4s, v8.4s, v12.4s\n" + "fmla v27.4s, v8.4s, v12.4s\n" "add x15, x15, #0x10\n" - "fmla v30.4s, v7.4s, v12.4s\n" - "fmla v31.4s, v6.4s, v12.4s\n" + "fmla v18.4s, v7.4s, v12.4s\n" + "fmla v30.4s, v6.4s, v12.4s\n" "ldr q12, [x9, x4]\n" - "fmla v16.4s, v4.4s, v10.4s\n" - "fmla v17.4s, v3.4s, v10.4s\n" - "fmax v16.4s, v16.4s, v15.4s\n" - "fmla v20.4s, v1.4s, v10.4s\n" - "fmla v21.4s, v0.4s, v10.4s\n" + "fmla v28.4s, v4.4s, v10.4s\n" + "fmla v25.4s, v3.4s, v10.4s\n" + "fmax v28.4s, v28.4s, v13.4s\n" + "fmla v31.4s, v1.4s, v10.4s\n" + "fmla v26.4s, v0.4s, v10.4s\n" "ldr q10, [x9, x28]\n" - "ldr q9, [x14, x17]\n" - "fmla v18.4s, v5.4s, v11.4s\n" - "fmla v19.4s, v4.4s, v11.4s\n" - "fmax v17.4s, v17.4s, v15.4s\n" + "ldr q0, [x16, #0x10]\n" + "fmla v17.4s, v5.4s, v11.4s\n" + "fmla v29.4s, v4.4s, v11.4s\n" + "fmax v25.4s, v25.4s, v13.4s\n" "add x9, x9, #0x10\n" - "fmla v22.4s, v2.4s, v11.4s\n" - "ldr q13, [x16, #0x0]\n" - "fmla v23.4s, v1.4s, v11.4s\n" + "fmla v16.4s, v2.4s, v11.4s\n" + "ldr q2, [x16, #0x30]\n" + "fmla v24.4s, v1.4s, v11.4s\n" "ldr q11, [x7, x25]\n" - "ldr q0, [x16, #0x10]\n" - "fmla v24.4s, v7.4s, v12.4s\n" - "fmla v25.4s, v6.4s, v12.4s\n" "ldr q1, [x16, #0x20]\n" - "fmla v28.4s, v4.4s, v12.4s\n" - "fmla v29.4s, v3.4s, v12.4s\n" + "fmla v20.4s, v7.4s, v12.4s\n" + "fmla v22.4s, v6.4s, v12.4s\n" + "ldr q6, [x16, #0x70]\n" + "fmla v21.4s, v4.4s, v12.4s\n" + "fmla v27.4s, v3.4s, v12.4s\n" "ldr q12, [x14, x11]\n" - "ldr q2, [x16, #0x30]\n" - "fmla v26.4s, v8.4s, v10.4s\n" "ldr q3, [x16, #0x40]\n" - "fmla v27.4s, v7.4s, v10.4s\n" - "ldr q6, [x16, #0x70]\n" - "fmla v30.4s, v5.4s, v10.4s\n" + "fmla v23.4s, v8.4s, v10.4s\n" + "ldr q8, [x16, #0x90]\n" + "fmla v19.4s, v7.4s, v10.4s\n" + "ldr q7, [x16, #0x80]\n" + "fmla v18.4s, v5.4s, v10.4s\n" "ldr q5, [x16, #0x60]\n" - "fmla v31.4s, v4.4s, v10.4s\n" + "fmla v30.4s, v4.4s, v10.4s\n" "ld1 { v10.4s }, [x7]\n" "ldr q4, [x16, #0x50]\n" - "fmax v18.4s, v18.4s, v15.4s\n" - "fmax v19.4s, v19.4s, v15.4s\n" - "fmax v20.4s, v20.4s, v15.4s\n" - "fmax v21.4s, v21.4s, v15.4s\n" - "fmax v22.4s, v22.4s, v15.4s\n" - "fmax v23.4s, v23.4s, v15.4s\n" - "fmax v24.4s, v24.4s, v15.4s\n" - "fmax v25.4s, v25.4s, v15.4s\n" - "fmax v26.4s, v26.4s, v15.4s\n" - "fmax v27.4s, v27.4s, v15.4s\n" - "fmax v28.4s, v28.4s, v15.4s\n" - "fmax v29.4s, v29.4s, v15.4s\n" - "fmax v30.4s, v30.4s, v15.4s\n" - "fmax v31.4s, v31.4s, v15.4s\n" - "fmin v16.4s, v16.4s, v14.4s\n" - "fmin v17.4s, v17.4s, v14.4s\n" - "st1 { v16.4s }, [x8]\n" - "ldr q7, [x16, #0x80]\n" - "fmin v18.4s, v18.4s, v14.4s\n" - "fmin v19.4s, v19.4s, v14.4s\n" - "str q17, [x8, x5]\n" - "ldr q8, [x16, #0x90]\n" - "fmin v20.4s, v20.4s, v14.4s\n" - "fmin v21.4s, v21.4s, v14.4s\n" - "str q18, [x8, x23]\n" + "fmax v17.4s, v17.4s, v13.4s\n" + "fmax v29.4s, v29.4s, v13.4s\n" "add x16, x16, #0xa0\n" - "fmin v22.4s, v22.4s, v14.4s\n" - "fmin v23.4s, v23.4s, v14.4s\n" - "str q19, [x8, x22]\n" + "fmax v31.4s, v31.4s, v13.4s\n" + "fmax v26.4s, v26.4s, v13.4s\n" + "fmax v16.4s, v16.4s, v13.4s\n" + "fmax v24.4s, v24.4s, v13.4s\n" + "fmax v20.4s, v20.4s, v13.4s\n" + "fmax v22.4s, v22.4s, v13.4s\n" + "fmax v23.4s, v23.4s, v13.4s\n" + "fmax v19.4s, v19.4s, v13.4s\n" + "fmax v21.4s, v21.4s, v13.4s\n" + "fmax v27.4s, v27.4s, v13.4s\n" + "fmax v18.4s, v18.4s, v13.4s\n" + "fmax v30.4s, v30.4s, v13.4s\n" + "fmin v28.4s, v28.4s, v15.4s\n" + "fmin v25.4s, v25.4s, v15.4s\n" + "st1 { v28.4s }, [x8]\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v29.4s, v29.4s, v15.4s\n" + "str q25, [x8, x5]\n" + "fmin v31.4s, v31.4s, v15.4s\n" + "fmin v26.4s, v26.4s, v15.4s\n" + "str q17, [x8, x23]\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v24.4s, v24.4s, v15.4s\n" + "str q29, [x8, x22]\n" "add x8, x8, #0x10\n" - "fmin v24.4s, v24.4s, v14.4s\n" - "fmin v25.4s, v25.4s, v14.4s\n" - "st1 { v20.4s }, [x10]\n" - "fmin v26.4s, v26.4s, v14.4s\n" - "fmin v27.4s, v27.4s, v14.4s\n" - "str q21, [x10, x5]\n" - "fmin v28.4s, v28.4s, v14.4s\n" - "fmin v29.4s, v29.4s, v14.4s\n" - "str q22, [x10, x23]\n" - "fmin v30.4s, v30.4s, v14.4s\n" - "fmin v31.4s, v31.4s, v14.4s\n" - "str q23, [x10, x22]\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "st1 { v31.4s }, [x10]\n" + "fmin v23.4s, v23.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" + "str q26, [x10, x5]\n" + "fmin v21.4s, v21.4s, v15.4s\n" + "fmin v27.4s, v27.4s, v15.4s\n" + "str q16, [x10, x23]\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v30.4s, v30.4s, v15.4s\n" + "str q24, [x10, x22]\n" "add x10, x10, #0x10\n" - "st1 { v24.4s }, [x27]\n" - "str q25, [x27, x5]\n" - "str q26, [x27, x23]\n" - "str q27, [x27, x22]\n" + "st1 { v20.4s }, [x27]\n" + "str q22, [x27, x5]\n" + "str q23, [x27, x23]\n" + "str q19, [x27, x22]\n" "add x27, x27, #0x10\n" - "st1 { v28.4s }, [x24]\n" - "str q29, [x24, x5]\n" - "str q30, [x24, x23]\n" - "str q31, [x24, x22]\n" + "st1 { v21.4s }, [x24]\n" + "str q27, [x24, x5]\n" + "str q18, [x24, x23]\n" + "str q30, [x24, x22]\n" "add x24, x24, #0x10\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "mov v21.16b, v13.16b\n fmla v21.4s, v4.4s, v9.4s\n" - "mov v16.16b, v13.16b\n fmla v16.4s, v8.4s, v9.4s\n" - "mov v22.16b, v13.16b\n fmla v22.4s, v3.4s, v9.4s\n" - "mov v25.16b, v13.16b\n fmla v25.4s, v1.4s, v9.4s\n" - "mov v26.16b, v13.16b\n fmla v26.4s, v0.4s, v9.4s\n" - "fmla v21.4s, v5.4s, v12.4s\n" - "mov v17.16b, v13.16b\n fmla v17.4s, v7.4s, v9.4s\n" - "mov v18.16b, v13.16b\n fmla v18.4s, v6.4s, v9.4s\n" - "mov v20.16b, v13.16b\n fmla v20.4s, v5.4s, v9.4s\n" - "mov v24.16b, v13.16b\n fmla v24.4s, v2.4s, v9.4s\n" - "ldr q9, [x12, x17]\n" - "fmla v16.4s, v0.4s, v10.4s\n" - "ld1 { v10.4s }, [x26]\n" - "mov v19.16b, v13.16b\n fmla v19.4s, v2.4s, v11.4s\n" - "ldr q11, [x26, x25]\n" - "fmla v22.4s, v4.4s, v12.4s\n" - "fmla v25.4s, v2.4s, v12.4s\n" - "fmla v26.4s, v1.4s, v12.4s\n" - "mov v28.16b, v13.16b\n fmla v28.4s, v6.4s, v10.4s\n" - "ldr q10, [x12, x11]\n" - "fmla v21.4s, v7.4s, v9.4s\n" + "mov v16.16b, v14.16b\n fmla v16.4s, v4.4s, v9.4s\n" + "mov v23.16b, v14.16b\n fmla v23.4s, v8.4s, v9.4s\n" + "mov v31.16b, v14.16b\n fmla v31.4s, v3.4s, v9.4s\n" + "mov v30.16b, v14.16b\n fmla v30.4s, v1.4s, v9.4s\n" + "mov v18.16b, v14.16b\n fmla v18.4s, v0.4s, v9.4s\n" + "fmla v16.4s, v5.4s, v12.4s\n" + "mov v17.16b, v14.16b\n fmla v17.4s, v7.4s, v9.4s\n" + "mov v19.16b, v14.16b\n fmla v19.4s, v6.4s, v9.4s\n" + "mov v28.16b, v14.16b\n fmla v28.4s, v5.4s, v9.4s\n" + "mov v27.16b, v14.16b\n fmla v27.4s, v2.4s, v9.4s\n" + "ldr q24, [x12, x17]\n" + "fmla v23.4s, v0.4s, v10.4s\n" + "ld1 { v21.4s }, [x26]\n" + "mov v29.16b, v14.16b\n fmla v29.4s, v2.4s, v11.4s\n" + "ldr q20, [x26, x25]\n" + "fmla v31.4s, v4.4s, v12.4s\n" + "fmla v30.4s, v2.4s, v12.4s\n" + "fmla v18.4s, v1.4s, v12.4s\n" + "mov v26.16b, v14.16b\n fmla v26.4s, v6.4s, v21.4s\n" + "ldr q9, [x12, x11]\n" + "fmla v16.4s, v7.4s, v24.4s\n" "fmla v17.4s, v8.4s, v12.4s\n" - "fmla v18.4s, v7.4s, v12.4s\n" - "fmla v19.4s, v6.4s, v12.4s\n" - "mov v23.16b, v13.16b\n fmla v23.4s, v3.4s, v12.4s\n" - "mov v27.16b, v13.16b\n fmla v27.4s, v0.4s, v12.4s\n" - "ldr q12, [x7, x4]\n" - "mov v31.16b, v13.16b\n fmla v31.4s, v8.4s, v11.4s\n" - "ldr q11, [x7, x28]\n" - "fmla v22.4s, v6.4s, v9.4s\n" - "fmla v25.4s, v4.4s, v9.4s\n" - "fmla v26.4s, v3.4s, v9.4s\n" - "fmla v20.4s, v8.4s, v9.4s\n" - "fmla v24.4s, v5.4s, v9.4s\n" - "fmla v28.4s, v2.4s, v9.4s\n" - "fmla v21.4s, v8.4s, v10.4s\n" - "fmla v16.4s, v1.4s, v12.4s\n" - "fmla v17.4s, v0.4s, v12.4s\n" - "ldr q12, [x15, x25]\n" - "fmla v18.4s, v2.4s, v11.4s\n" - "fmla v19.4s, v1.4s, v11.4s\n" - "ld1 { v11.4s }, [x9]\n" - "fmla v22.4s, v7.4s, v10.4s\n" - "fmla v23.4s, v6.4s, v10.4s\n" - "fmla v25.4s, v5.4s, v10.4s\n" - "fmla v26.4s, v4.4s, v10.4s\n" - "fmla v27.4s, v3.4s, v10.4s\n" - "fmla v31.4s, v0.4s, v10.4s\n" - "fmla v24.4s, v6.4s, v11.4s\n" - "fmla v28.4s, v3.4s, v11.4s\n" - "ldr q11, [x9, x25]\n" - "fmla v19.4s, v5.4s, v12.4s\n" - "fmla v23.4s, v2.4s, v12.4s\n" - "ldr q12, [x15, x11]\n" - "fmla v27.4s, v8.4s, v11.4s\n" - "fmla v31.4s, v5.4s, v11.4s\n" - "mov v29.16b, v13.16b\n fmla v29.4s, v1.4s, v9.4s\n" - "mov v30.16b, v13.16b\n fmla v30.4s, v0.4s, v9.4s\n" - "ld1 { v9.4s }, [x15]\n" - "fmla v29.4s, v2.4s, v10.4s\n" - "fmla v30.4s, v1.4s, v10.4s\n" - "ldr q10, [x15, x17]\n" - "fmla v20.4s, v0.4s, v9.4s\n" - "fmla v21.4s, v1.4s, v10.4s\n" - "fmla v16.4s, v3.4s, v9.4s\n" - "ldr q11, [x26, x4]\n" - "fmla v17.4s, v4.4s, v10.4s\n" - "fmla v18.4s, v3.4s, v10.4s\n" - "fmla v22.4s, v0.4s, v10.4s\n" - "fmla v20.4s, v2.4s, v10.4s\n" - "fmla v21.4s, v2.4s, v12.4s\n" - "fmla v16.4s, v5.4s, v10.4s\n" - "ldr q10, [x14, x4]\n" - "fmla v17.4s, v5.4s, v12.4s\n" - "fmla v18.4s, v4.4s, v12.4s\n" - "fmla v19.4s, v3.4s, v12.4s\n" - "fmla v22.4s, v1.4s, v12.4s\n" - "fmla v23.4s, v0.4s, v12.4s\n" - "ldr q12, [x14, x28]\n" - "fmla v28.4s, v7.4s, v11.4s\n" - "fmla v29.4s, v6.4s, v11.4s\n" - "ldr q11, [x26, x28]\n" - "fmla v20.4s, v4.4s, v10.4s\n" - "fmla v21.4s, v3.4s, v10.4s\n" - "fmla v24.4s, v1.4s, v10.4s\n" - "fmla v25.4s, v0.4s, v10.4s\n" - "fmla v16.4s, v7.4s, v10.4s\n" - "fmla v17.4s, v6.4s, v10.4s\n" - "ldr q10, [x7, x17]\n" - "fmla v30.4s, v8.4s, v11.4s\n" - "fmla v31.4s, v7.4s, v11.4s\n" - "ldr q11, [x12, x4]\n" - "fmla v18.4s, v8.4s, v12.4s\n" "fmla v19.4s, v7.4s, v12.4s\n" - "fmla v22.4s, v5.4s, v12.4s\n" - "fmla v23.4s, v4.4s, v12.4s\n" - "fmla v26.4s, v2.4s, v12.4s\n" - "fmla v27.4s, v1.4s, v12.4s\n" - "ldr q12, [x7, x11]\n" + "fmla v29.4s, v6.4s, v12.4s\n" + "mov v11.16b, v14.16b\n fmla v11.4s, v3.4s, v12.4s\n" + "mov v10.16b, v14.16b\n fmla v10.4s, v0.4s, v12.4s\n" + "ldr q22, [x7, x4]\n" + "mov v25.16b, v14.16b\n fmla v25.4s, v8.4s, v20.4s\n" + "ldr q21, [x7, x28]\n" + "fmla v31.4s, v6.4s, v24.4s\n" + "fmla v30.4s, v4.4s, v24.4s\n" + "fmla v18.4s, v3.4s, v24.4s\n" + "mov v12.16b, v14.16b\n fmla v12.4s, v1.4s, v24.4s\n" + "fmla v14.4s, v0.4s, v24.4s\n" + "fmla v28.4s, v8.4s, v24.4s\n" + "fmla v27.4s, v5.4s, v24.4s\n" + "fmla v26.4s, v2.4s, v24.4s\n" + "ld1 { v24.4s }, [x15]\n" + "fmla v16.4s, v8.4s, v9.4s\n" + "fmla v23.4s, v1.4s, v22.4s\n" + "fmla v17.4s, v0.4s, v22.4s\n" + "ldr q22, [x15, x25]\n" + "fmla v19.4s, v2.4s, v21.4s\n" + "fmla v29.4s, v1.4s, v21.4s\n" + "ld1 { v20.4s }, [x9]\n" + "fmla v31.4s, v7.4s, v9.4s\n" + "fmla v11.4s, v6.4s, v9.4s\n" + "fmla v30.4s, v5.4s, v9.4s\n" + "fmla v18.4s, v4.4s, v9.4s\n" + "fmla v10.4s, v3.4s, v9.4s\n" + "fmla v12.4s, v2.4s, v9.4s\n" + "fmla v14.4s, v1.4s, v9.4s\n" + "fmla v25.4s, v0.4s, v9.4s\n" + "ldr q21, [x15, x17]\n" + "fmla v28.4s, v0.4s, v24.4s\n" + "fmla v27.4s, v6.4s, v20.4s\n" + "fmla v26.4s, v3.4s, v20.4s\n" + "ldr q20, [x9, x25]\n" + "fmla v16.4s, v1.4s, v21.4s\n" + "fmla v23.4s, v3.4s, v24.4s\n" + "fmla v29.4s, v5.4s, v22.4s\n" + "fmla v11.4s, v2.4s, v22.4s\n" + "ldr q22, [x15, x11]\n" + "fmla v17.4s, v4.4s, v21.4s\n" + "fmla v19.4s, v3.4s, v21.4s\n" + "fmla v31.4s, v0.4s, v21.4s\n" + "fmla v10.4s, v8.4s, v20.4s\n" + "fmla v25.4s, v5.4s, v20.4s\n" + "ldr q20, [x26, x4]\n" + "fmla v28.4s, v2.4s, v21.4s\n" + "fmla v16.4s, v2.4s, v22.4s\n" + "fmla v23.4s, v5.4s, v21.4s\n" + "ldr q21, [x14, x4]\n" + "fmla v17.4s, v5.4s, v22.4s\n" + "fmla v19.4s, v4.4s, v22.4s\n" + "fmla v29.4s, v3.4s, v22.4s\n" + "fmla v31.4s, v1.4s, v22.4s\n" + "fmla v11.4s, v0.4s, v22.4s\n" + "ldr q22, [x14, x28]\n" + "fmla v26.4s, v7.4s, v20.4s\n" + "fmla v12.4s, v6.4s, v20.4s\n" + "ldr q20, [x26, x28]\n" + "fmla v28.4s, v4.4s, v21.4s\n" + "fmla v16.4s, v3.4s, v21.4s\n" + "fmla v27.4s, v1.4s, v21.4s\n" + "fmla v30.4s, v0.4s, v21.4s\n" + "fmla v23.4s, v7.4s, v21.4s\n" + "fmla v17.4s, v6.4s, v21.4s\n" + "ldr q21, [x7, x17]\n" + "fmla v14.4s, v8.4s, v20.4s\n" + "fmla v25.4s, v7.4s, v20.4s\n" + "ldr q20, [x12, x4]\n" + "fmla v19.4s, v8.4s, v22.4s\n" + "fmla v29.4s, v7.4s, v22.4s\n" + "fmla v31.4s, v5.4s, v22.4s\n" + "fmla v11.4s, v4.4s, v22.4s\n" + "fmla v18.4s, v2.4s, v22.4s\n" + "fmla v10.4s, v1.4s, v22.4s\n" + "ldr q22, [x7, x11]\n" "add x7, x7, #0x10\n" - "fmla v20.4s, v7.4s, v11.4s\n" - "fmla v21.4s, v6.4s, v11.4s\n" - "fmla v24.4s, v4.4s, v11.4s\n" - "fmla v25.4s, v3.4s, v11.4s\n" - "fmla v28.4s, v1.4s, v11.4s\n" - "fmla v29.4s, v0.4s, v11.4s\n" - "ldr q11, [x12, x28]\n" - "fmla v16.4s, v2.4s, v10.4s\n" - "fmla v17.4s, v1.4s, v10.4s\n" - "fmla v18.4s, v0.4s, v10.4s\n" - "ld1 { v10.4s }, [x14]\n" - "fmla v30.4s, v2.4s, v11.4s\n" - "fmla v19.4s, v0.4s, v12.4s\n" - "fmla v20.4s, v3.4s, v10.4s\n" - "fmla v24.4s, v0.4s, v10.4s\n" - "fmla v22.4s, v8.4s, v11.4s\n" - "fmla v23.4s, v7.4s, v11.4s\n" - "fmla v26.4s, v5.4s, v11.4s\n" - "fmla v27.4s, v4.4s, v11.4s\n" - "fmla v31.4s, v1.4s, v11.4s\n" - "ldr q11, [x9, x17]\n" - "fmla v17.4s, v2.4s, v12.4s\n" - "fmla v18.4s, v1.4s, v12.4s\n" - "ldr q12, [x14, x25]\n" + "fmla v28.4s, v7.4s, v20.4s\n" + "fmla v16.4s, v6.4s, v20.4s\n" + "fmla v27.4s, v4.4s, v20.4s\n" + "fmla v30.4s, v3.4s, v20.4s\n" + "fmla v26.4s, v1.4s, v20.4s\n" + "fmla v12.4s, v0.4s, v20.4s\n" + "ldr q20, [x12, x28]\n" + "fmla v23.4s, v2.4s, v21.4s\n" + "fmla v17.4s, v1.4s, v21.4s\n" + "fmla v19.4s, v0.4s, v21.4s\n" + "ld1 { v21.4s }, [x14]\n" + "fmla v14.4s, v2.4s, v20.4s\n" + "fmla v29.4s, v0.4s, v22.4s\n" + "fmla v28.4s, v3.4s, v21.4s\n" + "fmla v27.4s, v0.4s, v21.4s\n" + "fmla v31.4s, v8.4s, v20.4s\n" + "fmla v11.4s, v7.4s, v20.4s\n" + "fmla v18.4s, v5.4s, v20.4s\n" + "fmla v10.4s, v4.4s, v20.4s\n" + "fmla v25.4s, v1.4s, v20.4s\n" + "ldr q24, [x9, x17]\n" + "fmla v17.4s, v2.4s, v22.4s\n" + "fmla v19.4s, v1.4s, v22.4s\n" + "ldr q20, [x14, x25]\n" "add x14, x14, #0x10\n" - "fmla v16.4s, v6.4s, v10.4s\n" - "ld1 { v10.4s }, [x12]\n" - "fmla v29.4s, v4.4s, v11.4s\n" - "fmla v30.4s, v3.4s, v11.4s\n" - "fmla v19.4s, v8.4s, v12.4s\n" - "fmla v23.4s, v5.4s, v12.4s\n" - "fmla v27.4s, v2.4s, v12.4s\n" - "ldr q12, [x12, x25]\n" + "fmla v23.4s, v6.4s, v21.4s\n" + "ld1 { v21.4s }, [x12]\n" + "fmla v12.4s, v4.4s, v24.4s\n" + "fmla v14.4s, v3.4s, v24.4s\n" + "fmla v29.4s, v8.4s, v20.4s\n" + "fmla v11.4s, v5.4s, v20.4s\n" + "fmla v10.4s, v2.4s, v20.4s\n" + "ldr q20, [x12, x25]\n" "add x12, x12, #0x10\n" - "fmla v20.4s, v6.4s, v10.4s\n" - "fmla v24.4s, v3.4s, v10.4s\n" - "fmla v28.4s, v0.4s, v10.4s\n" - "ldr q10, [x26, x17]\n" - "fmla v31.4s, v2.4s, v12.4s\n" - "fmla v29.4s, v7.4s, v10.4s\n" - "fmla v30.4s, v6.4s, v10.4s\n" - "fmla v24.4s, v8.4s, v11.4s\n" - "fmla v25.4s, v7.4s, v11.4s\n" - "fmla v26.4s, v6.4s, v11.4s\n" - "fmla v28.4s, v5.4s, v11.4s\n" - "ldr q11, [x9, x11]\n" - "fmla v27.4s, v5.4s, v12.4s\n" - "fmla v29.4s, v5.4s, v11.4s\n" - "fmla v30.4s, v4.4s, v11.4s\n" - "fmla v31.4s, v3.4s, v11.4s\n" - "fmla v23.4s, v8.4s, v12.4s\n" - "ldr q12, [x26, x11]\n" - "fmla v28.4s, v8.4s, v10.4s\n" - "ldr q10, [x15, x4]\n" - "fmla v25.4s, v8.4s, v11.4s\n" - "fmla v26.4s, v7.4s, v11.4s\n" + "fmla v28.4s, v6.4s, v21.4s\n" + "fmla v27.4s, v3.4s, v21.4s\n" + "fmla v26.4s, v0.4s, v21.4s\n" + "ldr q22, [x26, x17]\n" + "fmla v25.4s, v2.4s, v20.4s\n" + "fmla v12.4s, v7.4s, v22.4s\n" + "fmla v14.4s, v6.4s, v22.4s\n" + "fmla v27.4s, v8.4s, v24.4s\n" + "fmla v30.4s, v7.4s, v24.4s\n" + "fmla v18.4s, v6.4s, v24.4s\n" + "fmla v26.4s, v5.4s, v24.4s\n" + "ldr q21, [x9, x11]\n" + "fmla v10.4s, v5.4s, v20.4s\n" + "fmla v12.4s, v5.4s, v21.4s\n" + "fmla v14.4s, v4.4s, v21.4s\n" + "fmla v25.4s, v3.4s, v21.4s\n" + "fmla v11.4s, v8.4s, v20.4s\n" + "ldr q20, [x26, x11]\n" + "fmla v26.4s, v8.4s, v22.4s\n" + "ldr q9, [x15, x4]\n" + "fmla v30.4s, v8.4s, v21.4s\n" + "fmla v18.4s, v7.4s, v21.4s\n" "add x26, x26, #0x10\n" - "fmla v27.4s, v6.4s, v11.4s\n" - "ldr q11, [x15, x28]\n" - "fmla v29.4s, v8.4s, v12.4s\n" + "fmla v10.4s, v6.4s, v21.4s\n" + "ldr q21, [x15, x28]\n" + "fmla v12.4s, v8.4s, v20.4s\n" "add x15, x15, #0x10\n" - "fmla v30.4s, v7.4s, v12.4s\n" - "fmla v31.4s, v6.4s, v12.4s\n" - "ldr q12, [x9, x4]\n" - "fmla v16.4s, v4.4s, v10.4s\n" - "fmla v17.4s, v3.4s, v10.4s\n" - "fmax v16.4s, v16.4s, v15.4s\n" - "fmla v20.4s, v1.4s, v10.4s\n" - "fmla v21.4s, v0.4s, v10.4s\n" - "ldr q10, [x9, x28]\n" - "fmax v17.4s, v17.4s, v15.4s\n" - "fmla v18.4s, v5.4s, v11.4s\n" - "fmla v19.4s, v4.4s, v11.4s\n" - "fmax v18.4s, v18.4s, v15.4s\n" + "fmla v14.4s, v7.4s, v20.4s\n" + "fmla v25.4s, v6.4s, v20.4s\n" + "ldr q24, [x9, x4]\n" + "fmla v23.4s, v4.4s, v9.4s\n" + "fmla v17.4s, v3.4s, v9.4s\n" + "fmax v23.4s, v23.4s, v13.4s\n" + "fmla v28.4s, v1.4s, v9.4s\n" + "fmla v16.4s, v0.4s, v9.4s\n" + "ldr q0, [x9, x28]\n" + "fmax v17.4s, v17.4s, v13.4s\n" + "fmla v19.4s, v5.4s, v21.4s\n" + "fmla v29.4s, v4.4s, v21.4s\n" + "fmax v19.4s, v19.4s, v13.4s\n" "add x9, x9, #0x10\n" - "fmla v22.4s, v2.4s, v11.4s\n" - "fmla v23.4s, v1.4s, v11.4s\n" - "fmax v19.4s, v19.4s, v15.4s\n" - "fmla v24.4s, v7.4s, v12.4s\n" - "fmla v25.4s, v6.4s, v12.4s\n" - "fmax v20.4s, v20.4s, v15.4s\n" - "fmla v28.4s, v4.4s, v12.4s\n" - "fmla v29.4s, v3.4s, v12.4s\n" - "fmax v21.4s, v21.4s, v15.4s\n" - "fmla v26.4s, v8.4s, v10.4s\n" - "fmla v27.4s, v7.4s, v10.4s\n" - "fmax v22.4s, v22.4s, v15.4s\n" - "fmla v30.4s, v5.4s, v10.4s\n" - "fmla v31.4s, v4.4s, v10.4s\n" - "fmax v23.4s, v23.4s, v15.4s\n" - "fmax v24.4s, v24.4s, v15.4s\n" - "fmax v25.4s, v25.4s, v15.4s\n" - "fmax v26.4s, v26.4s, v15.4s\n" - "fmax v27.4s, v27.4s, v15.4s\n" - "fmax v28.4s, v28.4s, v15.4s\n" - "fmax v29.4s, v29.4s, v15.4s\n" - "fmax v30.4s, v30.4s, v15.4s\n" - "fmax v31.4s, v31.4s, v15.4s\n" - "fmin v16.4s, v16.4s, v14.4s\n" - "fmin v17.4s, v17.4s, v14.4s\n" - "st1 { v16.4s }, [x8]\n" - "fmin v18.4s, v18.4s, v14.4s\n" - "fmin v19.4s, v19.4s, v14.4s\n" + "fmla v31.4s, v2.4s, v21.4s\n" + "fmla v11.4s, v1.4s, v21.4s\n" + "fmax v29.4s, v29.4s, v13.4s\n" + "fmla v27.4s, v7.4s, v24.4s\n" + "fmla v30.4s, v6.4s, v24.4s\n" + "fmax v28.4s, v28.4s, v13.4s\n" + "fmla v26.4s, v4.4s, v24.4s\n" + "fmla v12.4s, v3.4s, v24.4s\n" + "fmax v16.4s, v16.4s, v13.4s\n" + "fmla v18.4s, v8.4s, v0.4s\n" + "fmla v10.4s, v7.4s, v0.4s\n" + "fmax v31.4s, v31.4s, v13.4s\n" + "fmla v14.4s, v5.4s, v0.4s\n" + "fmla v25.4s, v4.4s, v0.4s\n" + "fmax v11.4s, v11.4s, v13.4s\n" + "fmax v27.4s, v27.4s, v13.4s\n" + "fmax v30.4s, v30.4s, v13.4s\n" + "fmax v18.4s, v18.4s, v13.4s\n" + "fmax v10.4s, v10.4s, v13.4s\n" + "fmax v26.4s, v26.4s, v13.4s\n" + "fmax v12.4s, v12.4s, v13.4s\n" + "fmax v14.4s, v14.4s, v13.4s\n" + "fmax v25.4s, v25.4s, v13.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "st1 { v23.4s }, [x8]\n" + "fmin v19.4s, v19.4s, v15.4s\n" + "fmin v29.4s, v29.4s, v15.4s\n" "str q17, [x8, x5]\n" - "fmin v20.4s, v20.4s, v14.4s\n" - "fmin v21.4s, v21.4s, v14.4s\n" - "str q18, [x8, x23]\n" - "fmin v22.4s, v22.4s, v14.4s\n" - "fmin v23.4s, v23.4s, v14.4s\n" - "str q19, [x8, x22]\n" + "fmin v28.4s, v28.4s, v15.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "str q19, [x8, x23]\n" + "fmin v31.4s, v31.4s, v15.4s\n" + "fmin v11.4s, v11.4s, v15.4s\n" + "str q29, [x8, x22]\n" "add x8, x8, #0x10\n" - "fmin v24.4s, v24.4s, v14.4s\n" - "fmin v25.4s, v25.4s, v14.4s\n" - "st1 { v20.4s }, [x10]\n" - "fmin v26.4s, v26.4s, v14.4s\n" - "fmin v27.4s, v27.4s, v14.4s\n" - "str q21, [x10, x5]\n" - "fmin v28.4s, v28.4s, v14.4s\n" - "fmin v29.4s, v29.4s, v14.4s\n" - "str q22, [x10, x23]\n" - "fmin v30.4s, v30.4s, v14.4s\n" - "fmin v31.4s, v31.4s, v14.4s\n" - "str q23, [x10, x22]\n" + "fmin v27.4s, v27.4s, v15.4s\n" + "fmin v30.4s, v30.4s, v15.4s\n" + "st1 { v28.4s }, [x10]\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v10.4s, v10.4s, v15.4s\n" + "str q16, [x10, x5]\n" + "fmin v26.4s, v26.4s, v15.4s\n" + "fmin v12.4s, v12.4s, v15.4s\n" + "str q31, [x10, x23]\n" + "fmin v14.4s, v14.4s, v15.4s\n" + "fmin v25.4s, v25.4s, v15.4s\n" + "str q11, [x10, x22]\n" "add x10, x10, #0x10\n" - "st1 { v24.4s }, [x27]\n" - "str q25, [x27, x5]\n" - "str q26, [x27, x23]\n" - "str q27, [x27, x22]\n" + "st1 { v27.4s }, [x27]\n" + "str q30, [x27, x5]\n" + "str q18, [x27, x23]\n" + "str q10, [x27, x22]\n" "add x27, x27, #0x10\n" - "st1 { v28.4s }, [x24]\n" - "str q29, [x24, x5]\n" - "str q30, [x24, x23]\n" - "str q31, [x24, x22]\n" + "st1 { v26.4s }, [x24]\n" + "str q12, [x24, x5]\n" + "str q14, [x24, x23]\n" + "str q25, [x24, x22]\n" "add x24, x24, #0x10\n" "4:" // Tile loop: Oddments "tst %x[n_channels], #0x3\n" "beq 73f\n" - "ldr q13, [x16, #0x0]\n" + "ldr q14, [x16, #0x0]\n" "ldr q0, [x16, #0x10]\n" "add x23, x14, x17\n" "add x22, x7, XZR\n" @@ -675,27 +675,27 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "ldr s11, [x21, #0x0]\n" "ldr s12, [x20, #0x0]\n" "6:" // Tile loop: Oddments: Load inputs: (2, 2), (0, 0), (0, 5), (2, 3): Bit 1: End - "mov v16.16b, v13.16b\n fmla v16.4s, v8.4s, v9.4s\n" - "mov v17.16b, v13.16b\n fmla v17.4s, v7.4s, v9.4s\n" + "mov v16.16b, v14.16b\n fmla v16.4s, v8.4s, v9.4s\n" + "mov v17.16b, v14.16b\n fmla v17.4s, v7.4s, v9.4s\n" "add x20, x26, XZR\n" - "mov v18.16b, v13.16b\n fmla v18.4s, v6.4s, v9.4s\n" - "mov v21.16b, v13.16b\n fmla v21.4s, v4.4s, v9.4s\n" - "mov v22.16b, v13.16b\n fmla v22.4s, v3.4s, v9.4s\n" - "mov v25.16b, v13.16b\n fmla v25.4s, v1.4s, v9.4s\n" - "mov v26.16b, v13.16b\n fmla v26.4s, v0.4s, v9.4s\n" - "mov v19.16b, v13.16b\n fmla v19.4s, v2.4s, v11.4s\n" - "mov v20.16b, v13.16b\n fmla v20.4s, v5.4s, v9.4s\n" - "mov v24.16b, v13.16b\n fmla v24.4s, v2.4s, v9.4s\n" + "mov v18.16b, v14.16b\n fmla v18.4s, v6.4s, v9.4s\n" + "mov v21.16b, v14.16b\n fmla v21.4s, v4.4s, v9.4s\n" + "mov v22.16b, v14.16b\n fmla v22.4s, v3.4s, v9.4s\n" + "mov v25.16b, v14.16b\n fmla v25.4s, v1.4s, v9.4s\n" + "mov v26.16b, v14.16b\n fmla v26.4s, v0.4s, v9.4s\n" + "mov v19.16b, v14.16b\n fmla v19.4s, v2.4s, v11.4s\n" + "mov v20.16b, v14.16b\n fmla v20.4s, v5.4s, v9.4s\n" + "mov v24.16b, v14.16b\n fmla v24.4s, v2.4s, v9.4s\n" "fmla v16.4s, v0.4s, v10.4s\n" "fmla v17.4s, v8.4s, v12.4s\n" "fmla v18.4s, v7.4s, v12.4s\n" "fmla v19.4s, v6.4s, v12.4s\n" "fmla v21.4s, v5.4s, v12.4s\n" "fmla v22.4s, v4.4s, v12.4s\n" - "mov v23.16b, v13.16b\n fmla v23.4s, v3.4s, v12.4s\n" + "mov v23.16b, v14.16b\n fmla v23.4s, v3.4s, v12.4s\n" "fmla v25.4s, v2.4s, v12.4s\n" "fmla v26.4s, v1.4s, v12.4s\n" - "mov v27.16b, v13.16b\n fmla v27.4s, v0.4s, v12.4s\n" + "mov v27.16b, v14.16b\n fmla v27.4s, v0.4s, v12.4s\n" "tbz %x[n_channels], #1, 7f\n" "ldr d10, [x20], #0x8\n" "tbz %x[n_channels], #0, 8f\n" @@ -704,7 +704,7 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "7:" // Tile loop: Oddments: Load inputs: (5, 0): Bit 1: Unset "ldr s10, [x20, #0x0]\n" "8:" // Tile loop: Oddments: Load inputs: (5, 0): Bit 1: End - "mov v28.16b, v13.16b\n fmla v28.4s, v6.4s, v10.4s\n" + "mov v28.16b, v14.16b\n fmla v28.4s, v6.4s, v10.4s\n" "add x20, x26, x25\n" "tbz %x[n_channels], #1, 9f\n" "ldr d11, [x20], #0x8\n" @@ -714,7 +714,7 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "9:" // Tile loop: Oddments: Load inputs: (5, 5): Bit 1: Unset "ldr s11, [x20, #0x0]\n" "10:" // Tile loop: Oddments: Load inputs: (5, 5): Bit 1: End - "mov v31.16b, v13.16b\n fmla v31.4s, v8.4s, v11.4s\n" + "mov v31.16b, v14.16b\n fmla v31.4s, v8.4s, v11.4s\n" "add x20, x12, x17\n" "tbz %x[n_channels], #1, 11f\n" "ldr d9, [x20], #0x8\n" @@ -732,8 +732,8 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "fmla v25.4s, v4.4s, v9.4s\n" "fmla v26.4s, v3.4s, v9.4s\n" "fmla v28.4s, v2.4s, v9.4s\n" - "mov v29.16b, v13.16b\n fmla v29.4s, v1.4s, v9.4s\n" - "mov v30.16b, v13.16b\n fmla v30.4s, v0.4s, v9.4s\n" + "mov v29.16b, v14.16b\n fmla v29.4s, v1.4s, v9.4s\n" + "mov v30.16b, v14.16b\n fmla v30.4s, v0.4s, v9.4s\n" "tbz %x[n_channels], #1, 13f\n" "ldr d12, [x20], #0x8\n" "tbz %x[n_channels], #0, 14f\n" @@ -1105,40 +1105,40 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "70:" // Tile loop: Oddments: Load inputs: (4, 4): Bit 1: End "fmla v26.4s, v8.4s, v10.4s\n" "fmla v27.4s, v7.4s, v10.4s\n" - "fmax v16.4s, v16.4s, v15.4s\n" + "fmax v16.4s, v16.4s, v13.4s\n" "fmla v30.4s, v5.4s, v10.4s\n" "fmla v31.4s, v4.4s, v10.4s\n" - "fmax v17.4s, v17.4s, v15.4s\n" - "fmax v18.4s, v18.4s, v15.4s\n" - "fmax v19.4s, v19.4s, v15.4s\n" - "fmax v20.4s, v20.4s, v15.4s\n" - "fmax v21.4s, v21.4s, v15.4s\n" - "fmax v22.4s, v22.4s, v15.4s\n" - "fmax v23.4s, v23.4s, v15.4s\n" - "fmax v24.4s, v24.4s, v15.4s\n" - "fmax v25.4s, v25.4s, v15.4s\n" - "fmax v26.4s, v26.4s, v15.4s\n" - "fmax v27.4s, v27.4s, v15.4s\n" - "fmax v28.4s, v28.4s, v15.4s\n" - "fmax v29.4s, v29.4s, v15.4s\n" - "fmax v30.4s, v30.4s, v15.4s\n" - "fmax v31.4s, v31.4s, v15.4s\n" - "fmin v16.4s, v16.4s, v14.4s\n" - "fmin v17.4s, v17.4s, v14.4s\n" - "fmin v18.4s, v18.4s, v14.4s\n" - "fmin v19.4s, v19.4s, v14.4s\n" - "fmin v20.4s, v20.4s, v14.4s\n" - "fmin v21.4s, v21.4s, v14.4s\n" - "fmin v22.4s, v22.4s, v14.4s\n" - "fmin v23.4s, v23.4s, v14.4s\n" - "fmin v24.4s, v24.4s, v14.4s\n" - "fmin v25.4s, v25.4s, v14.4s\n" - "fmin v26.4s, v26.4s, v14.4s\n" - "fmin v27.4s, v27.4s, v14.4s\n" - "fmin v28.4s, v28.4s, v14.4s\n" - "fmin v29.4s, v29.4s, v14.4s\n" - "fmin v30.4s, v30.4s, v14.4s\n" - "fmin v31.4s, v31.4s, v14.4s\n" + "fmax v17.4s, v17.4s, v13.4s\n" + "fmax v18.4s, v18.4s, v13.4s\n" + "fmax v19.4s, v19.4s, v13.4s\n" + "fmax v20.4s, v20.4s, v13.4s\n" + "fmax v21.4s, v21.4s, v13.4s\n" + "fmax v22.4s, v22.4s, v13.4s\n" + "fmax v23.4s, v23.4s, v13.4s\n" + "fmax v24.4s, v24.4s, v13.4s\n" + "fmax v25.4s, v25.4s, v13.4s\n" + "fmax v26.4s, v26.4s, v13.4s\n" + "fmax v27.4s, v27.4s, v13.4s\n" + "fmax v28.4s, v28.4s, v13.4s\n" + "fmax v29.4s, v29.4s, v13.4s\n" + "fmax v30.4s, v30.4s, v13.4s\n" + "fmax v31.4s, v31.4s, v13.4s\n" + "fmin v16.4s, v16.4s, v15.4s\n" + "fmin v17.4s, v17.4s, v15.4s\n" + "fmin v18.4s, v18.4s, v15.4s\n" + "fmin v19.4s, v19.4s, v15.4s\n" + "fmin v20.4s, v20.4s, v15.4s\n" + "fmin v21.4s, v21.4s, v15.4s\n" + "fmin v22.4s, v22.4s, v15.4s\n" + "fmin v23.4s, v23.4s, v15.4s\n" + "fmin v24.4s, v24.4s, v15.4s\n" + "fmin v25.4s, v25.4s, v15.4s\n" + "fmin v26.4s, v26.4s, v15.4s\n" + "fmin v27.4s, v27.4s, v15.4s\n" + "fmin v28.4s, v28.4s, v15.4s\n" + "fmin v29.4s, v29.4s, v15.4s\n" + "fmin v30.4s, v30.4s, v15.4s\n" + "fmin v31.4s, v31.4s, v15.4s\n" "tbz %x[n_channels], #1, 71f\n" "mov x23, x8\n" "mov x22, x10\n" @@ -1229,4 +1229,4 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp index 23530450218dd34594c1be279e42aa6ea21a9b65..76045f30d6e1139b9211a70af44ca237124f2837 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ +#if defined(__aarch64__) namespace arm_conv { namespace depthwise { @@ -98,629 +98,629 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( activation_min, activation_max); __asm__ __volatile__( - "mov x8, #0x10\n" // cntb _, ALL, #1 - "lsr x17, %x[n_channels], #0x2\n" - "ldr x16, [%x[params_struct], %[offsetof_args_outptrs]]\n" - "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" + "mov x6, #0x10\n" // cntb _, ALL, #1 + "lsr x7, %x[n_channels], #0x2\n" + "ldr x8, [%x[params_struct], %[offsetof_args_outptrs]]\n" + "ldr x17, [%x[params_struct], %[offsetof_args_params]]\n" "add x20, %x[params_struct], %[offsetof_args_min]\n" - "ld1r { v15.4s }, [x20]\n" + "ld1r { v13.4s }, [x20]\n" "add x20, %x[params_struct], %[offsetof_args_max]\n" "ld1r { v14.4s }, [x20]\n" - "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n" - "mov x13, #0x0\n" - "sub x12, XZR, x8\n" - "cbz x17, 3f\n" - "ldr q13, [x15, #0x0]\n" - "ldr q0, [x15, #0x10]\n" - "cmp x8, x17, LSL #4\n" - "ldr q1, [x15, #0x20]\n" - "ldr q2, [x15, #0x30]\n" - "ldr q3, [x15, #0x40]\n" - "ldr q4, [x15, #0x50]\n" - "ldr q5, [x15, #0x60]\n" - "ldr q6, [x15, #0x70]\n" - "ldr q7, [x15, #0x80]\n" - "ldr q8, [x15, #0x90]\n" - "add x15, x15, #0xa0\n" - "ldp x11, x10, [x14, #0x0]\n" - "ldr q9, [x11, x13]\n" - "ldr q10, [x10, x13]\n" - "ldp x9, x28, [x14, #0x10]\n" - "ldr q11, [x9, x13]\n" - "ldr q12, [x28, x13]\n" + "add x16, %x[params_struct], %[offsetof_Args_inptrs]\n" + "mov x15, #0x0\n" + "sub x14, XZR, x6\n" + "cbz x7, 3f\n" + "ldr q30, [x17, #0x0]\n" + "ldr q0, [x17, #0x10]\n" + "cmp x6, x7, LSL #4\n" + "ldr q1, [x17, #0x20]\n" + "ldr q2, [x17, #0x30]\n" + "ldr q3, [x17, #0x40]\n" + "ldr q4, [x17, #0x50]\n" + "ldr q5, [x17, #0x60]\n" + "ldr q6, [x17, #0x70]\n" + "ldr q7, [x17, #0x80]\n" + "ldr q8, [x17, #0x90]\n" + "add x17, x17, #0xa0\n" + "ldp x21, x20, [x16, #0x0]\n" + "ldr q9, [x21, x15]\n" + "ldr q10, [x20, x15]\n" + "ldp x21, x20, [x16, #0x10]\n" + "ldr q11, [x21, x15]\n" + "ldr q12, [x20, x15]\n" "bge 2f\n" "1:" // Channel loop - "mov v21.16b, v13.16b\n fmla v21.4s, v4.4s, v9.4s\n" - "mov v16.16b, v13.16b\n fmla v16.4s, v8.4s, v9.4s\n" - "ldr x27, [x14, #0x20]\n" - "ldr x26, [x14, #0x30]\n" - "mov v22.16b, v13.16b\n fmla v22.4s, v3.4s, v9.4s\n" - "mov v25.16b, v13.16b\n fmla v25.4s, v1.4s, v9.4s\n" - "ldr x25, [x14, #0x28]\n" - "ldr x24, [x14, #0x38]\n" - "mov v26.16b, v13.16b\n fmla v26.4s, v0.4s, v9.4s\n" - "mov v17.16b, v13.16b\n fmla v17.4s, v7.4s, v9.4s\n" - "ldr x11, [x14, #0x40]\n" - "ldr x10, [x14, #0x48]\n" - "mov v18.16b, v13.16b\n fmla v18.4s, v6.4s, v9.4s\n" - "fmla v21.4s, v5.4s, v12.4s\n" - "ldr x9, [x14, #0x50]\n" - "ldr x28, [x14, #0x58]\n" - "mov v20.16b, v13.16b\n fmla v20.4s, v5.4s, v9.4s\n" - "mov v24.16b, v13.16b\n fmla v24.4s, v2.4s, v9.4s\n" - "ldr q9, [x26, x13]\n" - "ldr x26, [x14, #0x70]\n" - "fmla v16.4s, v0.4s, v10.4s\n" - "ldr q10, [x27, x13]\n" - "mov v19.16b, v13.16b\n fmla v19.4s, v2.4s, v11.4s\n" - "ldr q11, [x25, x13]\n" - "fmla v22.4s, v4.4s, v12.4s\n" - "fmla v25.4s, v2.4s, v12.4s\n" - "ldr x27, [x14, #0x60]\n" - "ldr x25, [x14, #0x68]\n" - "fmla v26.4s, v1.4s, v12.4s\n" - "fmla v17.4s, v8.4s, v12.4s\n" - "ldr x23, [x16, #0x0]\n" - "ldr x22, [x16, #0x8]\n" - "fmla v18.4s, v7.4s, v12.4s\n" - "mov v28.16b, v13.16b\n fmla v28.4s, v6.4s, v10.4s\n" - "ldr q10, [x10, x13]\n" - "ldr x10, [x14, #0x88]\n" - "fmla v21.4s, v7.4s, v9.4s\n" - "fmla v19.4s, v6.4s, v12.4s\n" - "ldr x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x18]\n" - "mov v23.16b, v13.16b\n fmla v23.4s, v3.4s, v12.4s\n" - "mov v27.16b, v13.16b\n fmla v27.4s, v0.4s, v12.4s\n" - "ldr q12, [x24, x13]\n" - "ldr x24, [x14, #0x78]\n" - "mov v31.16b, v13.16b\n fmla v31.4s, v8.4s, v11.4s\n" - "ldr q11, [x11, x13]\n" - "fmla v22.4s, v6.4s, v9.4s\n" - "ldr x11, [x14, #0x80]\n" - "fmla v25.4s, v4.4s, v9.4s\n" - "fmla v26.4s, v3.4s, v9.4s\n" - "add x12, x12, #0x10\n" - "fmla v20.4s, v8.4s, v9.4s\n" + "mov v23.16b, v30.16b\n fmla v23.4s, v4.4s, v9.4s\n" + "mov v17.16b, v30.16b\n fmla v17.4s, v8.4s, v9.4s\n" + "ldr x27, [x16, #0x20]\n" + "ldr x24, [x16, #0x30]\n" + "mov v25.16b, v30.16b\n fmla v25.4s, v3.4s, v9.4s\n" + "mov v28.16b, v30.16b\n fmla v28.4s, v1.4s, v9.4s\n" + "ldr x23, [x16, #0x28]\n" + "ldr x22, [x16, #0x38]\n" + "mov v20.16b, v30.16b\n fmla v20.4s, v0.4s, v9.4s\n" + "mov v16.16b, v30.16b\n fmla v16.4s, v7.4s, v9.4s\n" + "ldr x26, [x16, #0x40]\n" + "ldr x20, [x16, #0x48]\n" + "mov v15.16b, v30.16b\n fmla v15.4s, v6.4s, v9.4s\n" + "fmla v23.4s, v5.4s, v12.4s\n" + "ldr x25, [x16, #0x50]\n" + "ldr x21, [x16, #0x58]\n" + "mov v27.16b, v30.16b\n fmla v27.4s, v5.4s, v9.4s\n" + "mov v31.16b, v30.16b\n fmla v31.4s, v2.4s, v9.4s\n" + "ldr q9, [x24, x15]\n" + "ldr x13, [x16, #0x70]\n" + "fmla v17.4s, v0.4s, v10.4s\n" + "ldr q22, [x27, x15]\n" + "mov v10.16b, v30.16b\n fmla v10.4s, v2.4s, v11.4s\n" + "ldr q18, [x23, x15]\n" + "fmla v25.4s, v4.4s, v12.4s\n" + "fmla v28.4s, v2.4s, v12.4s\n" + "ldr x24, [x16, #0x60]\n" + "ldr x23, [x16, #0x68]\n" + "fmla v20.4s, v1.4s, v12.4s\n" + "fmla v16.4s, v8.4s, v12.4s\n" + "ldr x12, [x8, #0x0]\n" + "ldr x11, [x8, #0x8]\n" + "fmla v15.4s, v7.4s, v12.4s\n" + "mov v29.16b, v30.16b\n fmla v29.4s, v6.4s, v22.4s\n" + "ldr q22, [x20, x15]\n" + "ldr x28, [x16, #0x88]\n" + "fmla v23.4s, v7.4s, v9.4s\n" + "fmla v10.4s, v6.4s, v12.4s\n" + "ldr x10, [x8, #0x10]\n" + "ldr x9, [x8, #0x18]\n" + "mov v21.16b, v30.16b\n fmla v21.4s, v3.4s, v12.4s\n" + "mov v19.16b, v30.16b\n fmla v19.4s, v0.4s, v12.4s\n" + "ldr q11, [x22, x15]\n" + "ldr x22, [x16, #0x78]\n" + "mov v24.16b, v30.16b\n fmla v24.4s, v8.4s, v18.4s\n" + "ldr q12, [x26, x15]\n" + "fmla v25.4s, v6.4s, v9.4s\n" + "ldr x20, [x16, #0x80]\n" + "fmla v28.4s, v4.4s, v9.4s\n" + "fmla v20.4s, v3.4s, v9.4s\n" + "add x14, x14, #0x10\n" + "mov v26.16b, v30.16b\n fmla v26.4s, v1.4s, v9.4s\n" + "mov v18.16b, v30.16b\n fmla v18.4s, v0.4s, v9.4s\n" + "ldr q30, [x17, #0x0]\n" + "fmla v27.4s, v8.4s, v9.4s\n" + "fmla v31.4s, v5.4s, v9.4s\n" + "fmla v29.4s, v2.4s, v9.4s\n" + "ldr q9, [x25, x15]\n" + "fmla v17.4s, v1.4s, v11.4s\n" + "ldr x27, [x16, #0x90]\n" + "fmla v16.4s, v0.4s, v11.4s\n" + "ldr q11, [x21, x15]\n" + "fmla v15.4s, v2.4s, v12.4s\n" + "ldr x21, [x16, #0x98]\n" + "fmla v23.4s, v8.4s, v22.4s\n" + "fmla v10.4s, v1.4s, v12.4s\n" + "ldr q12, [x24, x15]\n" + "ldr x26, [x16, #0xa0]\n" + "fmla v25.4s, v7.4s, v22.4s\n" + "fmla v21.4s, v6.4s, v22.4s\n" + "fmla v28.4s, v5.4s, v22.4s\n" + "fmla v20.4s, v4.4s, v22.4s\n" + "fmla v19.4s, v3.4s, v22.4s\n" + "fmla v26.4s, v2.4s, v22.4s\n" + "fmla v18.4s, v1.4s, v22.4s\n" + "fmla v24.4s, v0.4s, v22.4s\n" + "ldr q22, [x23, x15]\n" + "ldr x25, [x16, #0xa8]\n" + "fmla v17.4s, v3.4s, v9.4s\n" + "fmla v27.4s, v0.4s, v9.4s\n" + "fmla v31.4s, v6.4s, v12.4s\n" + "fmla v29.4s, v3.4s, v12.4s\n" + "ldr q9, [x13, x15]\n" + "ldr x24, [x16, #0xb0]\n" + "fmla v16.4s, v4.4s, v22.4s\n" + "fmla v15.4s, v3.4s, v22.4s\n" + "fmla v23.4s, v1.4s, v22.4s\n" + "fmla v10.4s, v5.4s, v11.4s\n" + "fmla v21.4s, v2.4s, v11.4s\n" + "ldr q12, [x22, x15]\n" + "fmla v25.4s, v0.4s, v22.4s\n" + "ldr x23, [x16, #0xb8]\n" + "fmla v19.4s, v8.4s, v9.4s\n" "fmla v24.4s, v5.4s, v9.4s\n" - "fmla v28.4s, v2.4s, v9.4s\n" - "fmla v16.4s, v1.4s, v12.4s\n" - "fmla v17.4s, v0.4s, v12.4s\n" - "ldr q12, [x28, x13]\n" - "fmla v18.4s, v2.4s, v11.4s\n" - "ldr x28, [x14, #0x98]\n" - "fmla v21.4s, v8.4s, v10.4s\n" - "fmla v19.4s, v1.4s, v11.4s\n" - "ldr q11, [x27, x13]\n" - "ldr x27, [x14, #0xa0]\n" - "fmla v22.4s, v7.4s, v10.4s\n" - "fmla v23.4s, v6.4s, v10.4s\n" - "fmla v25.4s, v5.4s, v10.4s\n" - "fmla v26.4s, v4.4s, v10.4s\n" - "fmla v27.4s, v3.4s, v10.4s\n" - "fmla v31.4s, v0.4s, v10.4s\n" - "fmla v24.4s, v6.4s, v11.4s\n" - "fmla v28.4s, v3.4s, v11.4s\n" - "ldr q11, [x26, x13]\n" - "ldr x26, [x14, #0xb0]\n" - "fmla v19.4s, v5.4s, v12.4s\n" + "ldr q11, [x20, x15]\n" + "ldr x22, [x16, #0xc0]\n" + "fmla v17.4s, v5.4s, v22.4s\n" + "fmla v27.4s, v2.4s, v22.4s\n" + "ldr q22, [x28, x15]\n" + "ldr x20, [x16, #0xc8]\n" + "fmla v16.4s, v5.4s, v12.4s\n" + "fmla v15.4s, v4.4s, v12.4s\n" "fmla v23.4s, v2.4s, v12.4s\n" - "ldr q12, [x24, x13]\n" - "ldr x24, [x14, #0xb8]\n" - "fmla v27.4s, v8.4s, v11.4s\n" - "fmla v31.4s, v5.4s, v11.4s\n" - "mov v29.16b, v13.16b\n fmla v29.4s, v1.4s, v9.4s\n" - "mov v30.16b, v13.16b\n fmla v30.4s, v0.4s, v9.4s\n" - "ldr q9, [x9, x13]\n" - "ldr x9, [x14, #0x90]\n" - "fmla v29.4s, v2.4s, v10.4s\n" - "fmla v30.4s, v1.4s, v10.4s\n" - "ldr q10, [x25, x13]\n" - "ldr x25, [x14, #0xa8]\n" - "fmla v16.4s, v3.4s, v9.4s\n" - "fmla v20.4s, v0.4s, v9.4s\n" - "ldr q11, [x11, x13]\n" - "ldr x11, [x14, #0xc0]\n" - "fmla v17.4s, v4.4s, v10.4s\n" - "fmla v18.4s, v3.4s, v10.4s\n" - "fmla v21.4s, v1.4s, v10.4s\n" - "fmla v22.4s, v0.4s, v10.4s\n" - "fmla v16.4s, v5.4s, v10.4s\n" - "fmla v20.4s, v2.4s, v10.4s\n" - "ldr q10, [x10, x13]\n" - "ldr x10, [x14, #0xc8]\n" - "fmla v17.4s, v5.4s, v12.4s\n" - "fmla v18.4s, v4.4s, v12.4s\n" - "fmla v21.4s, v2.4s, v12.4s\n" - "fmla v19.4s, v3.4s, v12.4s\n" - "fmla v22.4s, v1.4s, v12.4s\n" - "fmla v23.4s, v0.4s, v12.4s\n" - "ldr q12, [x28, x13]\n" - "ldr x28, [x14, #0xd8]\n" - "fmla v28.4s, v7.4s, v11.4s\n" - "fmla v29.4s, v6.4s, v11.4s\n" - "ldr q11, [x9, x13]\n" - "ldr x9, [x14, #0xd0]\n" - "fmla v16.4s, v7.4s, v10.4s\n" - "fmla v17.4s, v6.4s, v10.4s\n" - "fmla v20.4s, v4.4s, v10.4s\n" - "fmla v21.4s, v3.4s, v10.4s\n" - "fmla v24.4s, v1.4s, v10.4s\n" - "fmla v25.4s, v0.4s, v10.4s\n" - "ldr q10, [x27, x13]\n" - "ldr x27, [x14, #0xe0]\n" - "fmla v18.4s, v8.4s, v12.4s\n" - "fmla v30.4s, v8.4s, v11.4s\n" - "fmla v31.4s, v7.4s, v11.4s\n" - "ldr q11, [x25, x13]\n" - "fmla v27.4s, v1.4s, v12.4s\n" - "ldr x25, [x14, #0xe8]\n" - "fmla v19.4s, v7.4s, v12.4s\n" - "fmla v22.4s, v5.4s, v12.4s\n" - "fmla v23.4s, v4.4s, v12.4s\n" - "fmla v26.4s, v2.4s, v12.4s\n" - "ldr q12, [x26, x13]\n" - "ldr x26, [x14, #0xf0]\n" - "fmla v16.4s, v2.4s, v10.4s\n" - "fmla v17.4s, v1.4s, v10.4s\n" - "fmla v18.4s, v0.4s, v10.4s\n" - "ldr q10, [x24, x13]\n" - "fmla v20.4s, v7.4s, v11.4s\n" - "ldr x24, [x14, #0xf8]\n" - "fmla v21.4s, v6.4s, v11.4s\n" - "fmla v24.4s, v4.4s, v11.4s\n" - "fmla v25.4s, v3.4s, v11.4s\n" - "fmla v28.4s, v1.4s, v11.4s\n" - "fmla v29.4s, v0.4s, v11.4s\n" - "ldr q11, [x11, x13]\n" - "fmla v27.4s, v4.4s, v11.4s\n" - "ldr x11, [x14, #0x100]\n" - "fmla v30.4s, v2.4s, v11.4s\n" - "fmla v17.4s, v2.4s, v12.4s\n" - "fmla v18.4s, v1.4s, v12.4s\n" - "fmla v19.4s, v0.4s, v12.4s\n" - "ldr q12, [x10, x13]\n" - "ldr x10, [x14, #0x108]\n" - "fmla v16.4s, v6.4s, v10.4s\n" - "fmla v20.4s, v3.4s, v10.4s\n" - "fmla v24.4s, v0.4s, v10.4s\n" - "ldr q10, [x9, x13]\n" - "fmla v22.4s, v8.4s, v11.4s\n" - "ldr x9, [x14, #0x110]\n" - "fmla v23.4s, v7.4s, v11.4s\n" - "fmla v26.4s, v5.4s, v11.4s\n" - "fmla v31.4s, v1.4s, v11.4s\n" - "ldr q11, [x28, x13]\n" - "fmla v27.4s, v2.4s, v12.4s\n" - "ldr x28, [x14, #0x118]\n" - "fmla v28.4s, v0.4s, v10.4s\n" - "fmla v29.4s, v4.4s, v11.4s\n" - "fmla v30.4s, v3.4s, v11.4s\n" - "fmla v19.4s, v8.4s, v12.4s\n" - "fmla v23.4s, v5.4s, v12.4s\n" - "ldr q12, [x27, x13]\n" - "fmla v20.4s, v6.4s, v10.4s\n" - "fmla v24.4s, v3.4s, v10.4s\n" - "ldr q10, [x25, x13]\n" - "fmla v25.4s, v7.4s, v11.4s\n" + "fmla v10.4s, v3.4s, v12.4s\n" + "fmla v25.4s, v1.4s, v12.4s\n" + "fmla v21.4s, v0.4s, v12.4s\n" + "ldr q9, [x21, x15]\n" + "ldr x28, [x16, #0xd8]\n" + "fmla v29.4s, v7.4s, v11.4s\n" "fmla v26.4s, v6.4s, v11.4s\n" - "fmla v28.4s, v5.4s, v11.4s\n" - "fmla v27.4s, v5.4s, v12.4s\n" - "fmla v31.4s, v2.4s, v12.4s\n" - "fmla v29.4s, v7.4s, v10.4s\n" - "fmla v30.4s, v6.4s, v10.4s\n" - "fmla v24.4s, v8.4s, v11.4s\n" - "ldr q11, [x26, x13]\n" - "fmla v28.4s, v8.4s, v10.4s\n" - "ldr q10, [x11, x13]\n" - "fmla v25.4s, v8.4s, v11.4s\n" - "fmla v26.4s, v7.4s, v11.4s\n" - "fmla v27.4s, v6.4s, v11.4s\n" - "fmla v29.4s, v5.4s, v11.4s\n" - "fmla v30.4s, v4.4s, v11.4s\n" - "fmla v31.4s, v3.4s, v11.4s\n" - "ldr q11, [x10, x13]\n" - "ldp x11, x10, [x14, #0x0]\n" - "fmla v23.4s, v8.4s, v12.4s\n" - "ldr q12, [x24, x13]\n" - "fmla v16.4s, v4.4s, v10.4s\n" - "fmax v16.4s, v16.4s, v15.4s\n" - "fmla v17.4s, v3.4s, v10.4s\n" - "fmla v18.4s, v5.4s, v11.4s\n" - "fmax v17.4s, v17.4s, v15.4s\n" - "fmla v19.4s, v4.4s, v11.4s\n" - "fmla v29.4s, v8.4s, v12.4s\n" - "fmax v18.4s, v18.4s, v15.4s\n" - "fmla v30.4s, v7.4s, v12.4s\n" - "fmla v31.4s, v6.4s, v12.4s\n" - "ldr q12, [x9, x13]\n" - "fmax v19.4s, v19.4s, v15.4s\n" - "fmla v20.4s, v1.4s, v10.4s\n" - "fmla v21.4s, v0.4s, v10.4s\n" - "ldr q10, [x28, x13]\n" - "ldr q9, [x11, x8]\n" - "fmla v22.4s, v2.4s, v11.4s\n" - "ldr q13, [x15, #0x0]\n" - "fmla v23.4s, v1.4s, v11.4s\n" - "ldr q0, [x15, #0x10]\n" - "ldr q1, [x15, #0x20]\n" + "ldr q12, [x27, x15]\n" + "ldr x21, [x16, #0xd0]\n" + "fmla v17.4s, v7.4s, v22.4s\n" + "fmla v16.4s, v6.4s, v22.4s\n" + "fmla v27.4s, v4.4s, v22.4s\n" + "fmla v23.4s, v3.4s, v22.4s\n" + "fmla v31.4s, v1.4s, v22.4s\n" + "fmla v28.4s, v0.4s, v22.4s\n" + "ldr q11, [x26, x15]\n" + "ldr x27, [x16, #0xe0]\n" + "fmla v15.4s, v8.4s, v9.4s\n" + "fmla v18.4s, v8.4s, v12.4s\n" "fmla v24.4s, v7.4s, v12.4s\n" - "fmla v25.4s, v6.4s, v12.4s\n" - "ldr q2, [x15, #0x30]\n" - "fmla v26.4s, v8.4s, v10.4s\n" - "ldr q6, [x15, #0x70]\n" - "fmla v27.4s, v7.4s, v10.4s\n" - "ldr q7, [x15, #0x80]\n" - "fmin v16.4s, v16.4s, v14.4s\n" + "ldr q12, [x25, x15]\n" + "fmla v19.4s, v1.4s, v9.4s\n" + "ldr x26, [x16, #0xe8]\n" + "fmla v10.4s, v7.4s, v9.4s\n" + "fmla v25.4s, v5.4s, v9.4s\n" + "fmla v21.4s, v4.4s, v9.4s\n" + "fmla v20.4s, v2.4s, v9.4s\n" + "ldr q9, [x24, x15]\n" + "ldr x24, [x16, #0xf0]\n" + "fmla v17.4s, v2.4s, v11.4s\n" + "fmla v16.4s, v1.4s, v11.4s\n" + "fmla v15.4s, v0.4s, v11.4s\n" + "ldr q22, [x23, x15]\n" + "fmla v27.4s, v7.4s, v12.4s\n" + "ldr x25, [x16, #0xf8]\n" + "fmla v23.4s, v6.4s, v12.4s\n" + "fmla v31.4s, v4.4s, v12.4s\n" + "fmla v28.4s, v3.4s, v12.4s\n" + "fmla v29.4s, v1.4s, v12.4s\n" + "fmla v26.4s, v0.4s, v12.4s\n" + "ldr q11, [x22, x15]\n" + "fmla v19.4s, v4.4s, v11.4s\n" + "ldr x23, [x16, #0x100]\n" + "fmla v18.4s, v2.4s, v11.4s\n" + "fmla v16.4s, v2.4s, v9.4s\n" + "fmla v15.4s, v1.4s, v9.4s\n" + "fmla v10.4s, v0.4s, v9.4s\n" + "ldr q9, [x20, x15]\n" + "ldr x20, [x16, #0x108]\n" + "fmla v17.4s, v6.4s, v22.4s\n" + "fmla v27.4s, v3.4s, v22.4s\n" + "fmla v31.4s, v0.4s, v22.4s\n" + "ldr q22, [x21, x15]\n" + "fmla v25.4s, v8.4s, v11.4s\n" + "ldr x22, [x16, #0x110]\n" + "fmla v21.4s, v7.4s, v11.4s\n" + "fmla v20.4s, v5.4s, v11.4s\n" + "fmla v24.4s, v1.4s, v11.4s\n" + "ldr q12, [x28, x15]\n" + "fmla v19.4s, v2.4s, v9.4s\n" + "ldr x21, [x16, #0x118]\n" + "fmla v29.4s, v0.4s, v22.4s\n" + "fmla v26.4s, v4.4s, v12.4s\n" + "fmla v18.4s, v3.4s, v12.4s\n" + "fmla v10.4s, v8.4s, v9.4s\n" + "fmla v21.4s, v5.4s, v9.4s\n" + "ldr q11, [x27, x15]\n" + "fmla v27.4s, v6.4s, v22.4s\n" + "fmla v31.4s, v3.4s, v22.4s\n" + "ldr q22, [x26, x15]\n" + "fmla v28.4s, v7.4s, v12.4s\n" + "fmla v20.4s, v6.4s, v12.4s\n" + "fmla v29.4s, v5.4s, v12.4s\n" + "fmla v19.4s, v5.4s, v11.4s\n" + "fmla v24.4s, v2.4s, v11.4s\n" + "fmla v26.4s, v7.4s, v22.4s\n" + "fmla v18.4s, v6.4s, v22.4s\n" + "fmla v31.4s, v8.4s, v12.4s\n" + "ldr q12, [x24, x15]\n" + "fmla v29.4s, v8.4s, v22.4s\n" + "ldr q22, [x23, x15]\n" + "fmla v28.4s, v8.4s, v12.4s\n" + "fmla v20.4s, v7.4s, v12.4s\n" + "fmla v19.4s, v6.4s, v12.4s\n" + "fmla v26.4s, v5.4s, v12.4s\n" + "fmla v18.4s, v4.4s, v12.4s\n" + "fmla v24.4s, v3.4s, v12.4s\n" + "ldr q12, [x20, x15]\n" + "ldp x20, x24, [x16, #0x0]\n" + "ldr q9, [x20, x6]\n" + "fmla v21.4s, v8.4s, v11.4s\n" + "ldr q11, [x25, x15]\n" + "fmla v17.4s, v4.4s, v22.4s\n" + "fmla v16.4s, v3.4s, v22.4s\n" + "fmla v15.4s, v5.4s, v12.4s\n" + "fmax v17.4s, v17.4s, v13.4s\n" + "fmla v10.4s, v4.4s, v12.4s\n" + "fmla v26.4s, v8.4s, v11.4s\n" + "fmax v16.4s, v16.4s, v13.4s\n" + "fmla v18.4s, v7.4s, v11.4s\n" + "fmla v24.4s, v6.4s, v11.4s\n" + "ldr q11, [x22, x15]\n" + "fmax v15.4s, v15.4s, v13.4s\n" + "fmla v27.4s, v1.4s, v22.4s\n" + "fmla v23.4s, v0.4s, v22.4s\n" + "ldr q22, [x21, x15]\n" + "ldr q0, [x17, #0x10]\n" + "fmla v25.4s, v2.4s, v12.4s\n" + "ldr q2, [x17, #0x30]\n" + "fmla v21.4s, v1.4s, v12.4s\n" + "ldr q1, [x17, #0x20]\n" + "fmax v10.4s, v10.4s, v13.4s\n" + "fmla v31.4s, v7.4s, v11.4s\n" + "fmla v28.4s, v6.4s, v11.4s\n" + "ldr q6, [x17, #0x70]\n" + "fmla v20.4s, v8.4s, v22.4s\n" + "ldr q8, [x17, #0x90]\n" + "fmla v19.4s, v7.4s, v22.4s\n" + "ldr q7, [x17, #0x80]\n" "fmin v17.4s, v17.4s, v14.4s\n" - "str q16, [x23, x12]\n" - "ldr q8, [x15, #0x90]\n" - "fmin v18.4s, v18.4s, v14.4s\n" - "fmin v19.4s, v19.4s, v14.4s\n" - "str q17, [x22, x12]\n" - "ldr x23, [x16, #0x20]\n" - "fmax v20.4s, v20.4s, v15.4s\n" - "fmax v21.4s, v21.4s, v15.4s\n" - "str q18, [x21, x12]\n" - "ldr x22, [x16, #0x28]\n" - "fmax v22.4s, v22.4s, v15.4s\n" - "fmax v23.4s, v23.4s, v15.4s\n" - "str q19, [x20, x12]\n" - "ldr x21, [x16, #0x30]\n" - "ldr x20, [x16, #0x38]\n" - "fmla v28.4s, v4.4s, v12.4s\n" - "fmla v29.4s, v3.4s, v12.4s\n" - "ldr q3, [x15, #0x40]\n" - "fmla v30.4s, v5.4s, v10.4s\n" - "ldr q5, [x15, #0x60]\n" - "fmla v31.4s, v4.4s, v10.4s\n" - "ldr q10, [x10, x8]\n" - "ldr q4, [x15, #0x50]\n" - "fmin v20.4s, v20.4s, v14.4s\n" - "fmin v21.4s, v21.4s, v14.4s\n" - "str q20, [x23, x12]\n" - "fmin v22.4s, v22.4s, v14.4s\n" + "fmin v16.4s, v16.4s, v14.4s\n" + "str q17, [x12, x14]\n" + "ldr x23, [x8, #0x20]\n" + "fmin v15.4s, v15.4s, v14.4s\n" + "fmin v10.4s, v10.4s, v14.4s\n" + "str q16, [x11, x14]\n" + "ldr x22, [x8, #0x28]\n" + "fmax v27.4s, v27.4s, v13.4s\n" + "fmax v23.4s, v23.4s, v13.4s\n" + "str q15, [x10, x14]\n" + "ldr x21, [x8, #0x30]\n" + "fmax v25.4s, v25.4s, v13.4s\n" + "fmax v21.4s, v21.4s, v13.4s\n" + "str q10, [x9, x14]\n" + "ldr x20, [x8, #0x38]\n" + "fmla v29.4s, v4.4s, v11.4s\n" + "fmla v26.4s, v3.4s, v11.4s\n" + "ldr q3, [x17, #0x40]\n" + "fmin v27.4s, v27.4s, v14.4s\n" + "fmla v18.4s, v5.4s, v22.4s\n" + "ldr q5, [x17, #0x60]\n" + "fmla v24.4s, v4.4s, v22.4s\n" + "ldr q10, [x24, x6]\n" + "ldr q4, [x17, #0x50]\n" "fmin v23.4s, v23.4s, v14.4s\n" - "str q21, [x22, x12]\n" - "ldr x23, [x16, #0x40]\n" - "fmax v24.4s, v24.4s, v15.4s\n" - "fmax v25.4s, v25.4s, v15.4s\n" - "str q22, [x21, x12]\n" - "ldr x22, [x16, #0x48]\n" - "fmax v26.4s, v26.4s, v15.4s\n" - "fmax v27.4s, v27.4s, v15.4s\n" - "str q23, [x20, x12]\n" - "ldr x21, [x16, #0x50]\n" - "ldr x20, [x16, #0x58]\n" - "ldp x9, x28, [x14, #0x10]\n" - "fmin v24.4s, v24.4s, v14.4s\n" "fmin v25.4s, v25.4s, v14.4s\n" - "ldr q11, [x9, x8]\n" - "ldr q12, [x28, x8]\n" - "fmin v26.4s, v26.4s, v14.4s\n" - "fmin v27.4s, v27.4s, v14.4s\n" - "fmax v28.4s, v28.4s, v15.4s\n" - "fmax v29.4s, v29.4s, v15.4s\n" - "str q24, [x23, x12]\n" - "ldr x23, [x16, #0x60]\n" - "fmax v30.4s, v30.4s, v15.4s\n" - "fmax v31.4s, v31.4s, v15.4s\n" - "str q25, [x22, x12]\n" - "ldr x22, [x16, #0x68]\n" - "str q26, [x21, x12]\n" - "ldr x21, [x16, #0x70]\n" - "add x8, x8, #0x10\n" - "cmp x8, x17, LSL #4\n" - "str q27, [x20, x12]\n" - "ldr x20, [x16, #0x78]\n" + "str q27, [x23, x14]\n" + "fmin v21.4s, v21.4s, v14.4s\n" + "fmax v31.4s, v31.4s, v13.4s\n" + "str q23, [x22, x14]\n" + "ldr x25, [x8, #0x40]\n" + "fmax v28.4s, v28.4s, v13.4s\n" + "fmax v20.4s, v20.4s, v13.4s\n" + "str q25, [x21, x14]\n" + "ldr x23, [x8, #0x48]\n" + "fmax v19.4s, v19.4s, v13.4s\n" + "str q21, [x20, x14]\n" + "ldr x22, [x8, #0x50]\n" + "ldr x24, [x8, #0x58]\n" + "ldp x21, x20, [x16, #0x10]\n" + "ldr q11, [x21, x6]\n" + "fmin v31.4s, v31.4s, v14.4s\n" "fmin v28.4s, v28.4s, v14.4s\n" + "ldr q12, [x20, x6]\n" + "fmin v20.4s, v20.4s, v14.4s\n" + "fmin v19.4s, v19.4s, v14.4s\n" + "str q31, [x25, x14]\n" + "fmax v29.4s, v29.4s, v13.4s\n" + "fmax v26.4s, v26.4s, v13.4s\n" + "str q28, [x23, x14]\n" + "ldr x23, [x8, #0x60]\n" + "fmax v18.4s, v18.4s, v13.4s\n" + "fmax v24.4s, v24.4s, v13.4s\n" + "str q20, [x22, x14]\n" + "ldr x22, [x8, #0x68]\n" + "str q19, [x24, x14]\n" + "ldr x21, [x8, #0x70]\n" + "ldr x20, [x8, #0x78]\n" + "add x6, x6, #0x10\n" + "cmp x6, x7, LSL #4\n" "fmin v29.4s, v29.4s, v14.4s\n" - "fmin v30.4s, v30.4s, v14.4s\n" - "fmin v31.4s, v31.4s, v14.4s\n" - "add x13, x13, #0x10\n" - "str q28, [x23, x12]\n" - "str q29, [x22, x12]\n" - "add x15, x15, #0xa0\n" - "str q30, [x21, x12]\n" - "str q31, [x20, x12]\n" + "fmin v26.4s, v26.4s, v14.4s\n" + "add x15, x15, #0x10\n" + "fmin v18.4s, v18.4s, v14.4s\n" + "fmin v24.4s, v24.4s, v14.4s\n" + "str q29, [x23, x14]\n" + "add x17, x17, #0xa0\n" + "str q26, [x22, x14]\n" + "str q18, [x21, x14]\n" + "str q24, [x20, x14]\n" "blt 1b\n" "2:" // Channel tail - "mov v21.16b, v13.16b\n fmla v21.4s, v4.4s, v9.4s\n" - "mov v16.16b, v13.16b\n fmla v16.4s, v8.4s, v9.4s\n" - "ldr x27, [x14, #0x20]\n" - "ldr x26, [x14, #0x30]\n" - "mov v22.16b, v13.16b\n fmla v22.4s, v3.4s, v9.4s\n" - "mov v25.16b, v13.16b\n fmla v25.4s, v1.4s, v9.4s\n" - "ldr x25, [x14, #0x28]\n" - "ldr x24, [x14, #0x38]\n" - "mov v26.16b, v13.16b\n fmla v26.4s, v0.4s, v9.4s\n" - "mov v17.16b, v13.16b\n fmla v17.4s, v7.4s, v9.4s\n" - "ldr x11, [x14, #0x40]\n" - "ldr x10, [x14, #0x48]\n" - "mov v18.16b, v13.16b\n fmla v18.4s, v6.4s, v9.4s\n" - "fmla v21.4s, v5.4s, v12.4s\n" - "ldr x9, [x14, #0x50]\n" - "ldr x28, [x14, #0x58]\n" - "mov v20.16b, v13.16b\n fmla v20.4s, v5.4s, v9.4s\n" - "mov v24.16b, v13.16b\n fmla v24.4s, v2.4s, v9.4s\n" - "ldr q9, [x26, x13]\n" - "ldr x26, [x14, #0x70]\n" - "fmla v16.4s, v0.4s, v10.4s\n" - "ldr q10, [x27, x13]\n" - "mov v19.16b, v13.16b\n fmla v19.4s, v2.4s, v11.4s\n" - "ldr q11, [x25, x13]\n" - "fmla v22.4s, v4.4s, v12.4s\n" - "fmla v25.4s, v2.4s, v12.4s\n" - "ldr x27, [x14, #0x60]\n" - "ldr x25, [x14, #0x68]\n" - "fmla v26.4s, v1.4s, v12.4s\n" - "fmla v17.4s, v8.4s, v12.4s\n" - "ldr x23, [x16, #0x0]\n" - "ldr x22, [x16, #0x8]\n" - "fmla v18.4s, v7.4s, v12.4s\n" - "mov v28.16b, v13.16b\n fmla v28.4s, v6.4s, v10.4s\n" - "ldr q10, [x10, x13]\n" - "ldr x10, [x14, #0x88]\n" - "fmla v21.4s, v7.4s, v9.4s\n" - "fmla v19.4s, v6.4s, v12.4s\n" - "ldr x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x18]\n" - "mov v23.16b, v13.16b\n fmla v23.4s, v3.4s, v12.4s\n" - "mov v27.16b, v13.16b\n fmla v27.4s, v0.4s, v12.4s\n" - "ldr q12, [x24, x13]\n" - "ldr x24, [x14, #0x78]\n" - "mov v31.16b, v13.16b\n fmla v31.4s, v8.4s, v11.4s\n" - "ldr q11, [x11, x13]\n" - "fmla v22.4s, v6.4s, v9.4s\n" - "ldr x11, [x14, #0x80]\n" - "fmla v25.4s, v4.4s, v9.4s\n" - "fmla v26.4s, v3.4s, v9.4s\n" - "add x12, x12, #0x10\n" - "fmla v20.4s, v8.4s, v9.4s\n" - "fmla v24.4s, v5.4s, v9.4s\n" - "fmla v28.4s, v2.4s, v9.4s\n" - "fmla v16.4s, v1.4s, v12.4s\n" - "fmla v17.4s, v0.4s, v12.4s\n" - "ldr q12, [x28, x13]\n" - "fmla v18.4s, v2.4s, v11.4s\n" - "ldr x28, [x14, #0x98]\n" - "fmla v21.4s, v8.4s, v10.4s\n" - "fmla v19.4s, v1.4s, v11.4s\n" - "ldr q11, [x27, x13]\n" - "ldr x27, [x14, #0xa0]\n" - "fmla v22.4s, v7.4s, v10.4s\n" - "fmla v23.4s, v6.4s, v10.4s\n" - "fmla v25.4s, v5.4s, v10.4s\n" - "fmla v26.4s, v4.4s, v10.4s\n" - "fmla v27.4s, v3.4s, v10.4s\n" - "fmla v31.4s, v0.4s, v10.4s\n" - "fmla v24.4s, v6.4s, v11.4s\n" - "fmla v28.4s, v3.4s, v11.4s\n" - "ldr q11, [x26, x13]\n" - "ldr x26, [x14, #0xb0]\n" - "fmla v19.4s, v5.4s, v12.4s\n" - "fmla v23.4s, v2.4s, v12.4s\n" - "ldr q12, [x24, x13]\n" - "ldr x24, [x14, #0xb8]\n" - "fmla v27.4s, v8.4s, v11.4s\n" - "fmla v31.4s, v5.4s, v11.4s\n" - "mov v29.16b, v13.16b\n fmla v29.4s, v1.4s, v9.4s\n" - "mov v30.16b, v13.16b\n fmla v30.4s, v0.4s, v9.4s\n" - "ldr q9, [x9, x13]\n" - "ldr x9, [x14, #0x90]\n" - "fmla v29.4s, v2.4s, v10.4s\n" - "fmla v30.4s, v1.4s, v10.4s\n" - "ldr q10, [x25, x13]\n" - "ldr x25, [x14, #0xa8]\n" - "fmla v16.4s, v3.4s, v9.4s\n" - "fmla v20.4s, v0.4s, v9.4s\n" - "ldr q11, [x11, x13]\n" - "ldr x11, [x14, #0xc0]\n" - "fmla v17.4s, v4.4s, v10.4s\n" - "fmla v18.4s, v3.4s, v10.4s\n" - "fmla v21.4s, v1.4s, v10.4s\n" - "fmla v22.4s, v0.4s, v10.4s\n" - "fmla v16.4s, v5.4s, v10.4s\n" - "fmla v20.4s, v2.4s, v10.4s\n" - "ldr q10, [x10, x13]\n" - "ldr x10, [x14, #0xc8]\n" - "fmla v17.4s, v5.4s, v12.4s\n" - "fmla v18.4s, v4.4s, v12.4s\n" - "fmla v21.4s, v2.4s, v12.4s\n" - "fmla v19.4s, v3.4s, v12.4s\n" - "fmla v22.4s, v1.4s, v12.4s\n" - "fmla v23.4s, v0.4s, v12.4s\n" - "ldr q12, [x28, x13]\n" - "ldr x28, [x14, #0xd8]\n" - "fmla v28.4s, v7.4s, v11.4s\n" - "fmla v29.4s, v6.4s, v11.4s\n" - "ldr q11, [x9, x13]\n" - "ldr x9, [x14, #0xd0]\n" - "fmla v16.4s, v7.4s, v10.4s\n" - "fmla v17.4s, v6.4s, v10.4s\n" - "fmla v20.4s, v4.4s, v10.4s\n" - "fmla v21.4s, v3.4s, v10.4s\n" - "fmla v24.4s, v1.4s, v10.4s\n" - "fmla v25.4s, v0.4s, v10.4s\n" - "ldr q10, [x27, x13]\n" - "ldr x27, [x14, #0xe0]\n" - "fmla v18.4s, v8.4s, v12.4s\n" - "fmla v30.4s, v8.4s, v11.4s\n" - "fmla v31.4s, v7.4s, v11.4s\n" - "ldr q11, [x25, x13]\n" - "fmla v27.4s, v1.4s, v12.4s\n" - "ldr x25, [x14, #0xe8]\n" - "fmla v19.4s, v7.4s, v12.4s\n" - "fmla v22.4s, v5.4s, v12.4s\n" - "fmla v23.4s, v4.4s, v12.4s\n" - "fmla v26.4s, v2.4s, v12.4s\n" - "ldr q12, [x26, x13]\n" - "ldr x26, [x14, #0xf0]\n" - "fmla v16.4s, v2.4s, v10.4s\n" - "fmla v17.4s, v1.4s, v10.4s\n" - "fmla v18.4s, v0.4s, v10.4s\n" - "ldr q10, [x24, x13]\n" - "fmla v20.4s, v7.4s, v11.4s\n" - "ldr x24, [x14, #0xf8]\n" - "fmla v21.4s, v6.4s, v11.4s\n" - "fmla v24.4s, v4.4s, v11.4s\n" - "fmla v25.4s, v3.4s, v11.4s\n" - "fmla v28.4s, v1.4s, v11.4s\n" - "fmla v29.4s, v0.4s, v11.4s\n" - "ldr q11, [x11, x13]\n" - "fmla v27.4s, v4.4s, v11.4s\n" - "ldr x11, [x14, #0x100]\n" - "fmla v30.4s, v2.4s, v11.4s\n" - "fmla v17.4s, v2.4s, v12.4s\n" - "fmla v18.4s, v1.4s, v12.4s\n" - "fmla v19.4s, v0.4s, v12.4s\n" - "ldr q12, [x10, x13]\n" - "ldr x10, [x14, #0x108]\n" - "fmla v16.4s, v6.4s, v10.4s\n" - "fmla v20.4s, v3.4s, v10.4s\n" - "fmla v24.4s, v0.4s, v10.4s\n" - "ldr q10, [x9, x13]\n" - "fmla v22.4s, v8.4s, v11.4s\n" - "ldr x9, [x14, #0x110]\n" - "fmla v23.4s, v7.4s, v11.4s\n" - "fmla v26.4s, v5.4s, v11.4s\n" - "fmla v31.4s, v1.4s, v11.4s\n" - "ldr q11, [x28, x13]\n" - "fmla v27.4s, v2.4s, v12.4s\n" - "ldr x28, [x14, #0x118]\n" - "fmla v28.4s, v0.4s, v10.4s\n" - "fmla v29.4s, v4.4s, v11.4s\n" - "fmla v30.4s, v3.4s, v11.4s\n" - "fmla v19.4s, v8.4s, v12.4s\n" - "fmla v23.4s, v5.4s, v12.4s\n" - "ldr q12, [x27, x13]\n" - "fmla v20.4s, v6.4s, v10.4s\n" - "fmla v24.4s, v3.4s, v10.4s\n" - "ldr q10, [x25, x13]\n" - "fmla v25.4s, v7.4s, v11.4s\n" - "fmla v26.4s, v6.4s, v11.4s\n" - "fmla v28.4s, v5.4s, v11.4s\n" - "fmla v27.4s, v5.4s, v12.4s\n" - "fmla v31.4s, v2.4s, v12.4s\n" - "fmla v29.4s, v7.4s, v10.4s\n" - "fmla v30.4s, v6.4s, v10.4s\n" - "fmla v24.4s, v8.4s, v11.4s\n" - "ldr q11, [x26, x13]\n" - "fmla v28.4s, v8.4s, v10.4s\n" - "ldr q10, [x11, x13]\n" - "fmla v25.4s, v8.4s, v11.4s\n" - "fmla v26.4s, v7.4s, v11.4s\n" - "fmla v27.4s, v6.4s, v11.4s\n" - "fmla v29.4s, v5.4s, v11.4s\n" - "fmla v30.4s, v4.4s, v11.4s\n" - "fmla v31.4s, v3.4s, v11.4s\n" - "ldr q11, [x10, x13]\n" - "fmla v23.4s, v8.4s, v12.4s\n" - "ldr q12, [x24, x13]\n" - "fmla v16.4s, v4.4s, v10.4s\n" - "fmax v16.4s, v16.4s, v15.4s\n" - "fmla v17.4s, v3.4s, v10.4s\n" - "fmla v18.4s, v5.4s, v11.4s\n" - "fmax v17.4s, v17.4s, v15.4s\n" - "fmla v19.4s, v4.4s, v11.4s\n" - "fmla v29.4s, v8.4s, v12.4s\n" - "fmax v18.4s, v18.4s, v15.4s\n" - "fmla v30.4s, v7.4s, v12.4s\n" - "fmla v31.4s, v6.4s, v12.4s\n" - "ldr q12, [x9, x13]\n" - "fmax v19.4s, v19.4s, v15.4s\n" - "fmla v20.4s, v1.4s, v10.4s\n" - "fmla v21.4s, v0.4s, v10.4s\n" - "ldr q10, [x28, x13]\n" - "fmin v16.4s, v16.4s, v14.4s\n" - "fmla v22.4s, v2.4s, v11.4s\n" - "fmla v23.4s, v1.4s, v11.4s\n" + "mov v31.16b, v30.16b\n fmla v31.4s, v4.4s, v9.4s\n" + "mov v17.16b, v30.16b\n fmla v17.4s, v8.4s, v9.4s\n" + "ldr x27, [x16, #0x20]\n" + "ldr x24, [x16, #0x30]\n" + "mov v15.16b, v30.16b\n fmla v15.4s, v3.4s, v9.4s\n" + "mov v29.16b, v30.16b\n fmla v29.4s, v1.4s, v9.4s\n" + "ldr x23, [x16, #0x28]\n" + "ldr x22, [x16, #0x38]\n" + "mov v19.16b, v30.16b\n fmla v19.4s, v0.4s, v9.4s\n" + "mov v20.16b, v30.16b\n fmla v20.4s, v7.4s, v9.4s\n" + "ldr x26, [x16, #0x40]\n" + "ldr x21, [x16, #0x48]\n" + "mov v21.16b, v30.16b\n fmla v21.4s, v6.4s, v9.4s\n" + "fmla v31.4s, v5.4s, v12.4s\n" + "ldr x25, [x16, #0x50]\n" + "ldr x20, [x16, #0x58]\n" + "mov v18.16b, v30.16b\n fmla v18.4s, v5.4s, v9.4s\n" + "mov v27.16b, v30.16b\n fmla v27.4s, v2.4s, v9.4s\n" + "ldr q24, [x24, x15]\n" + "ldr x13, [x16, #0x70]\n" + "fmla v17.4s, v0.4s, v10.4s\n" + "ldr q22, [x27, x15]\n" + "mov v28.16b, v30.16b\n fmla v28.4s, v2.4s, v11.4s\n" + "ldr q16, [x23, x15]\n" + "fmla v15.4s, v4.4s, v12.4s\n" + "fmla v29.4s, v2.4s, v12.4s\n" + "ldr x24, [x16, #0x60]\n" + "ldr x23, [x16, #0x68]\n" + "fmla v19.4s, v1.4s, v12.4s\n" + "fmla v20.4s, v8.4s, v12.4s\n" + "ldr x12, [x8, #0x0]\n" + "ldr x11, [x8, #0x8]\n" + "fmla v21.4s, v7.4s, v12.4s\n" + "mov v10.16b, v30.16b\n fmla v10.4s, v6.4s, v22.4s\n" + "ldr q22, [x21, x15]\n" + "ldr x28, [x16, #0x88]\n" + "fmla v31.4s, v7.4s, v24.4s\n" + "fmla v28.4s, v6.4s, v12.4s\n" + "ldr x10, [x8, #0x10]\n" + "ldr x9, [x8, #0x18]\n" + "mov v9.16b, v30.16b\n fmla v9.4s, v3.4s, v12.4s\n" + "mov v11.16b, v30.16b\n fmla v11.4s, v0.4s, v12.4s\n" + "ldr q23, [x22, x15]\n" + "ldr x22, [x16, #0x78]\n" + "mov v12.16b, v30.16b\n fmla v12.4s, v8.4s, v16.4s\n" + "ldr q16, [x26, x15]\n" + "fmla v15.4s, v6.4s, v24.4s\n" + "ldr x21, [x16, #0x80]\n" + "fmla v29.4s, v4.4s, v24.4s\n" + "fmla v19.4s, v3.4s, v24.4s\n" + "add x14, x14, #0x10\n" + "mov v26.16b, v30.16b\n fmla v26.4s, v1.4s, v24.4s\n" + "mov v25.16b, v30.16b\n fmla v25.4s, v0.4s, v24.4s\n" + "fmla v18.4s, v8.4s, v24.4s\n" + "fmla v27.4s, v5.4s, v24.4s\n" + "fmla v10.4s, v2.4s, v24.4s\n" + "ldr q24, [x25, x15]\n" + "fmla v17.4s, v1.4s, v23.4s\n" + "ldr x27, [x16, #0x90]\n" + "fmla v20.4s, v0.4s, v23.4s\n" + "ldr q23, [x20, x15]\n" + "fmla v21.4s, v2.4s, v16.4s\n" + "ldr x20, [x16, #0x98]\n" + "fmla v31.4s, v8.4s, v22.4s\n" + "fmla v28.4s, v1.4s, v16.4s\n" + "ldr q16, [x24, x15]\n" + "ldr x26, [x16, #0xa0]\n" + "fmla v15.4s, v7.4s, v22.4s\n" + "fmla v9.4s, v6.4s, v22.4s\n" + "fmla v29.4s, v5.4s, v22.4s\n" + "fmla v19.4s, v4.4s, v22.4s\n" + "fmla v11.4s, v3.4s, v22.4s\n" + "fmla v26.4s, v2.4s, v22.4s\n" + "fmla v25.4s, v1.4s, v22.4s\n" + "fmla v12.4s, v0.4s, v22.4s\n" + "ldr q22, [x23, x15]\n" + "ldr x25, [x16, #0xa8]\n" + "fmla v17.4s, v3.4s, v24.4s\n" + "fmla v18.4s, v0.4s, v24.4s\n" + "fmla v27.4s, v6.4s, v16.4s\n" + "fmla v10.4s, v3.4s, v16.4s\n" + "ldr q16, [x13, x15]\n" + "ldr x24, [x16, #0xb0]\n" + "fmla v20.4s, v4.4s, v22.4s\n" + "fmla v21.4s, v3.4s, v22.4s\n" + "fmla v31.4s, v1.4s, v22.4s\n" + "fmla v28.4s, v5.4s, v23.4s\n" + "fmla v9.4s, v2.4s, v23.4s\n" + "ldr q23, [x22, x15]\n" + "fmla v15.4s, v0.4s, v22.4s\n" + "ldr x23, [x16, #0xb8]\n" + "fmla v11.4s, v8.4s, v16.4s\n" + "fmla v12.4s, v5.4s, v16.4s\n" + "ldr q16, [x21, x15]\n" + "ldr x22, [x16, #0xc0]\n" + "fmla v17.4s, v5.4s, v22.4s\n" + "fmla v18.4s, v2.4s, v22.4s\n" + "ldr q22, [x28, x15]\n" + "ldr x21, [x16, #0xc8]\n" + "fmla v20.4s, v5.4s, v23.4s\n" + "fmla v21.4s, v4.4s, v23.4s\n" + "fmla v31.4s, v2.4s, v23.4s\n" + "fmla v28.4s, v3.4s, v23.4s\n" + "fmla v15.4s, v1.4s, v23.4s\n" + "fmla v9.4s, v0.4s, v23.4s\n" + "ldr q23, [x20, x15]\n" + "ldr x28, [x16, #0xd8]\n" + "fmla v10.4s, v7.4s, v16.4s\n" + "fmla v26.4s, v6.4s, v16.4s\n" + "ldr q16, [x27, x15]\n" + "ldr x20, [x16, #0xd0]\n" + "fmla v17.4s, v7.4s, v22.4s\n" + "fmla v20.4s, v6.4s, v22.4s\n" + "fmla v18.4s, v4.4s, v22.4s\n" + "fmla v31.4s, v3.4s, v22.4s\n" + "fmla v27.4s, v1.4s, v22.4s\n" + "fmla v29.4s, v0.4s, v22.4s\n" + "ldr q22, [x26, x15]\n" + "ldr x27, [x16, #0xe0]\n" + "fmla v21.4s, v8.4s, v23.4s\n" + "fmla v25.4s, v8.4s, v16.4s\n" + "fmla v12.4s, v7.4s, v16.4s\n" + "ldr q16, [x25, x15]\n" + "fmla v11.4s, v1.4s, v23.4s\n" + "ldr x26, [x16, #0xe8]\n" + "fmla v28.4s, v7.4s, v23.4s\n" + "fmla v15.4s, v5.4s, v23.4s\n" + "fmla v9.4s, v4.4s, v23.4s\n" + "fmla v19.4s, v2.4s, v23.4s\n" + "ldr q23, [x24, x15]\n" + "ldr x25, [x16, #0xf0]\n" + "fmla v17.4s, v2.4s, v22.4s\n" + "fmla v20.4s, v1.4s, v22.4s\n" + "fmla v21.4s, v0.4s, v22.4s\n" + "ldr q22, [x23, x15]\n" + "fmla v18.4s, v7.4s, v16.4s\n" + "ldr x24, [x16, #0xf8]\n" + "fmla v31.4s, v6.4s, v16.4s\n" + "fmla v27.4s, v4.4s, v16.4s\n" + "fmla v29.4s, v3.4s, v16.4s\n" + "fmla v10.4s, v1.4s, v16.4s\n" + "fmla v26.4s, v0.4s, v16.4s\n" + "ldr q16, [x22, x15]\n" + "fmla v11.4s, v4.4s, v16.4s\n" + "ldr x23, [x16, #0x100]\n" + "fmla v25.4s, v2.4s, v16.4s\n" + "fmla v20.4s, v2.4s, v23.4s\n" + "fmla v21.4s, v1.4s, v23.4s\n" + "fmla v28.4s, v0.4s, v23.4s\n" + "ldr q23, [x21, x15]\n" + "ldr x22, [x16, #0x108]\n" + "fmla v17.4s, v6.4s, v22.4s\n" + "fmla v18.4s, v3.4s, v22.4s\n" + "fmla v27.4s, v0.4s, v22.4s\n" + "ldr q22, [x20, x15]\n" + "fmla v15.4s, v8.4s, v16.4s\n" + "ldr x21, [x16, #0x110]\n" + "fmla v9.4s, v7.4s, v16.4s\n" + "fmla v19.4s, v5.4s, v16.4s\n" + "fmla v12.4s, v1.4s, v16.4s\n" + "ldr q16, [x28, x15]\n" + "fmla v11.4s, v2.4s, v23.4s\n" + "ldr x20, [x16, #0x118]\n" + "fmla v10.4s, v0.4s, v22.4s\n" + "fmla v26.4s, v4.4s, v16.4s\n" + "fmla v25.4s, v3.4s, v16.4s\n" + "fmla v28.4s, v8.4s, v23.4s\n" + "fmla v9.4s, v5.4s, v23.4s\n" + "ldr q23, [x27, x15]\n" + "fmla v18.4s, v6.4s, v22.4s\n" + "fmla v27.4s, v3.4s, v22.4s\n" + "ldr q22, [x26, x15]\n" + "fmla v29.4s, v7.4s, v16.4s\n" + "fmla v19.4s, v6.4s, v16.4s\n" + "fmla v10.4s, v5.4s, v16.4s\n" + "fmla v11.4s, v5.4s, v23.4s\n" + "fmla v12.4s, v2.4s, v23.4s\n" + "fmla v26.4s, v7.4s, v22.4s\n" + "fmla v25.4s, v6.4s, v22.4s\n" + "fmla v27.4s, v8.4s, v16.4s\n" + "ldr q16, [x25, x15]\n" + "fmla v10.4s, v8.4s, v22.4s\n" + "ldr q30, [x23, x15]\n" + "fmla v29.4s, v8.4s, v16.4s\n" + "fmla v19.4s, v7.4s, v16.4s\n" + "fmla v11.4s, v6.4s, v16.4s\n" + "fmla v26.4s, v5.4s, v16.4s\n" + "fmla v25.4s, v4.4s, v16.4s\n" + "fmla v12.4s, v3.4s, v16.4s\n" + "ldr q24, [x22, x15]\n" + "fmla v9.4s, v8.4s, v23.4s\n" + "ldr q16, [x24, x15]\n" + "fmla v17.4s, v4.4s, v30.4s\n" + "fmax v17.4s, v17.4s, v13.4s\n" + "fmla v20.4s, v3.4s, v30.4s\n" + "fmla v21.4s, v5.4s, v24.4s\n" + "fmax v20.4s, v20.4s, v13.4s\n" + "fmla v28.4s, v4.4s, v24.4s\n" + "fmla v26.4s, v8.4s, v16.4s\n" + "fmax v21.4s, v21.4s, v13.4s\n" + "fmla v25.4s, v7.4s, v16.4s\n" + "fmla v12.4s, v6.4s, v16.4s\n" + "ldr q23, [x21, x15]\n" + "fmax v28.4s, v28.4s, v13.4s\n" + "fmla v18.4s, v1.4s, v30.4s\n" + "fmla v31.4s, v0.4s, v30.4s\n" + "ldr q16, [x20, x15]\n" "fmin v17.4s, v17.4s, v14.4s\n" - "str q16, [x23, x12]\n" - "fmla v24.4s, v7.4s, v12.4s\n" - "fmla v25.4s, v6.4s, v12.4s\n" - "fmin v18.4s, v18.4s, v14.4s\n" - "str q17, [x22, x12]\n" - "fmla v26.4s, v8.4s, v10.4s\n" - "fmla v27.4s, v7.4s, v10.4s\n" - "fmin v19.4s, v19.4s, v14.4s\n" - "str q18, [x21, x12]\n" - "fmax v20.4s, v20.4s, v15.4s\n" - "fmax v21.4s, v21.4s, v15.4s\n" - "str q19, [x20, x12]\n" - "ldr x23, [x16, #0x20]\n" - "fmax v22.4s, v22.4s, v15.4s\n" - "fmax v23.4s, v23.4s, v15.4s\n" - "ldr x22, [x16, #0x28]\n" - "ldr x21, [x16, #0x30]\n" - "ldr x20, [x16, #0x38]\n" - "fmla v28.4s, v4.4s, v12.4s\n" - "fmla v29.4s, v3.4s, v12.4s\n" + "fmla v15.4s, v2.4s, v24.4s\n" + "fmla v9.4s, v1.4s, v24.4s\n" "fmin v20.4s, v20.4s, v14.4s\n" - "fmla v30.4s, v5.4s, v10.4s\n" - "fmla v31.4s, v4.4s, v10.4s\n" + "str q17, [x12, x14]\n" + "fmla v27.4s, v7.4s, v23.4s\n" + "fmla v29.4s, v6.4s, v23.4s\n" "fmin v21.4s, v21.4s, v14.4s\n" - "str q20, [x23, x12]\n" - "fmin v22.4s, v22.4s, v14.4s\n" - "fmin v23.4s, v23.4s, v14.4s\n" - "str q21, [x22, x12]\n" - "ldr x23, [x16, #0x40]\n" - "fmax v24.4s, v24.4s, v15.4s\n" - "fmax v25.4s, v25.4s, v15.4s\n" - "str q22, [x21, x12]\n" - "ldr x22, [x16, #0x48]\n" - "fmax v26.4s, v26.4s, v15.4s\n" - "fmax v27.4s, v27.4s, v15.4s\n" - "str q23, [x20, x12]\n" - "ldr x21, [x16, #0x50]\n" - "ldr x20, [x16, #0x58]\n" - "fmin v24.4s, v24.4s, v14.4s\n" - "fmin v25.4s, v25.4s, v14.4s\n" - "str q24, [x23, x12]\n" - "fmin v26.4s, v26.4s, v14.4s\n" - "fmin v27.4s, v27.4s, v14.4s\n" - "str q25, [x22, x12]\n" - "ldr x23, [x16, #0x60]\n" - "fmax v28.4s, v28.4s, v15.4s\n" - "fmax v29.4s, v29.4s, v15.4s\n" - "str q26, [x21, x12]\n" - "ldr x22, [x16, #0x68]\n" - "fmax v30.4s, v30.4s, v15.4s\n" - "fmax v31.4s, v31.4s, v15.4s\n" - "str q27, [x20, x12]\n" - "ldr x21, [x16, #0x70]\n" - "ldr x20, [x16, #0x78]\n" + "str q20, [x11, x14]\n" + "fmla v19.4s, v8.4s, v16.4s\n" + "fmla v11.4s, v7.4s, v16.4s\n" "fmin v28.4s, v28.4s, v14.4s\n" - "fmin v29.4s, v29.4s, v14.4s\n" - "str q28, [x23, x12]\n" - "fmin v30.4s, v30.4s, v14.4s\n" + "str q21, [x10, x14]\n" + "fmax v18.4s, v18.4s, v13.4s\n" + "fmax v31.4s, v31.4s, v13.4s\n" + "str q28, [x9, x14]\n" + "ldr x23, [x8, #0x20]\n" + "fmax v15.4s, v15.4s, v13.4s\n" + "fmax v9.4s, v9.4s, v13.4s\n" + "ldr x22, [x8, #0x28]\n" + "ldr x21, [x8, #0x30]\n" + "ldr x20, [x8, #0x38]\n" + "fmla v10.4s, v4.4s, v23.4s\n" + "fmla v26.4s, v3.4s, v23.4s\n" + "fmin v18.4s, v18.4s, v14.4s\n" + "fmla v25.4s, v5.4s, v16.4s\n" + "fmla v12.4s, v4.4s, v16.4s\n" "fmin v31.4s, v31.4s, v14.4s\n" - "str q29, [x22, x12]\n" - "add x13, x13, #0x10\n" - "str q30, [x21, x12]\n" - "str q31, [x20, x12]\n" + "str q18, [x23, x14]\n" + "fmin v15.4s, v15.4s, v14.4s\n" + "fmin v9.4s, v9.4s, v14.4s\n" + "str q31, [x22, x14]\n" + "ldr x23, [x8, #0x40]\n" + "fmax v27.4s, v27.4s, v13.4s\n" + "fmax v29.4s, v29.4s, v13.4s\n" + "str q15, [x21, x14]\n" + "ldr x22, [x8, #0x48]\n" + "fmax v19.4s, v19.4s, v13.4s\n" + "fmax v11.4s, v11.4s, v13.4s\n" + "str q9, [x20, x14]\n" + "ldr x21, [x8, #0x50]\n" + "ldr x20, [x8, #0x58]\n" + "fmin v27.4s, v27.4s, v14.4s\n" + "fmin v29.4s, v29.4s, v14.4s\n" + "str q27, [x23, x14]\n" + "fmin v19.4s, v19.4s, v14.4s\n" + "fmin v11.4s, v11.4s, v14.4s\n" + "str q29, [x22, x14]\n" + "ldr x23, [x8, #0x60]\n" + "fmax v10.4s, v10.4s, v13.4s\n" + "fmax v26.4s, v26.4s, v13.4s\n" + "str q19, [x21, x14]\n" + "ldr x22, [x8, #0x68]\n" + "fmax v25.4s, v25.4s, v13.4s\n" + "fmax v12.4s, v12.4s, v13.4s\n" + "str q11, [x20, x14]\n" + "ldr x21, [x8, #0x70]\n" + "ldr x20, [x8, #0x78]\n" + "fmin v10.4s, v10.4s, v14.4s\n" + "fmin v26.4s, v26.4s, v14.4s\n" + "str q10, [x23, x14]\n" + "fmin v25.4s, v25.4s, v14.4s\n" + "fmin v12.4s, v12.4s, v14.4s\n" + "str q26, [x22, x14]\n" + "add x15, x15, #0x10\n" + "str q25, [x21, x14]\n" + "str q12, [x20, x14]\n" "3:" // Oddments "tst %x[n_channels], #0x3\n" "beq 72f\n" - "ldr q13, [x15, #0x0]\n" - "ldr q0, [x15, #0x10]\n" - "mov x12, x13\n" - "ldr q1, [x15, #0x20]\n" - "ldr q2, [x15, #0x30]\n" - "ldr q3, [x15, #0x40]\n" - "ldr q4, [x15, #0x50]\n" - "ldr q5, [x15, #0x60]\n" - "ldr q6, [x15, #0x70]\n" - "ldr q7, [x15, #0x80]\n" - "ldr q8, [x15, #0x90]\n" - "ldr x23, [x14, #0x0]\n" - "ldr x22, [x14, #0x8]\n" - "add x23, x23, x13\n" - "add x22, x22, x13\n" - "ldr x21, [x14, #0x10]\n" - "ldr x20, [x14, #0x18]\n" - "add x21, x21, x13\n" - "add x20, x20, x13\n" + "ldr q30, [x17, #0x0]\n" + "ldr q0, [x17, #0x10]\n" + "mov x14, x15\n" + "ldr q1, [x17, #0x20]\n" + "ldr q2, [x17, #0x30]\n" + "ldr q3, [x17, #0x40]\n" + "ldr q4, [x17, #0x50]\n" + "ldr q5, [x17, #0x60]\n" + "ldr q6, [x17, #0x70]\n" + "ldr q7, [x17, #0x80]\n" + "ldr q8, [x17, #0x90]\n" + "ldr x23, [x16, #0x0]\n" + "ldr x22, [x16, #0x8]\n" + "add x23, x23, x15\n" + "add x22, x22, x15\n" + "ldr x21, [x16, #0x10]\n" + "ldr x20, [x16, #0x18]\n" + "add x21, x21, x15\n" + "add x20, x20, x15\n" "tbz %x[n_channels], #1, 4f\n" "ld1 { v9.d }[0], [x23], #0x8\n" "ld1 { v10.d }[0], [x22], #0x8\n" @@ -738,28 +738,28 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "ld1 { v11.s }[0], [x21], #0x4\n" "ld1 { v12.s }[0], [x20], #0x4\n" "5:" // Oddments: Load inputs (2, 2), (0, 0), (0, 5), (2, 3): Bit 1: End - "mov v16.16b, v13.16b\n fmla v16.4s, v8.4s, v9.4s\n" - "mov v17.16b, v13.16b\n fmla v17.4s, v7.4s, v9.4s\n" - "ldr x20, [x14, #0x20]\n" - "add x20, x20, x13\n" - "mov v18.16b, v13.16b\n fmla v18.4s, v6.4s, v9.4s\n" - "mov v21.16b, v13.16b\n fmla v21.4s, v4.4s, v9.4s\n" - "mov v22.16b, v13.16b\n fmla v22.4s, v3.4s, v9.4s\n" - "mov v25.16b, v13.16b\n fmla v25.4s, v1.4s, v9.4s\n" - "mov v26.16b, v13.16b\n fmla v26.4s, v0.4s, v9.4s\n" - "mov v19.16b, v13.16b\n fmla v19.4s, v2.4s, v11.4s\n" - "mov v20.16b, v13.16b\n fmla v20.4s, v5.4s, v9.4s\n" - "mov v24.16b, v13.16b\n fmla v24.4s, v2.4s, v9.4s\n" + "mov v16.16b, v30.16b\n fmla v16.4s, v8.4s, v9.4s\n" + "mov v17.16b, v30.16b\n fmla v17.4s, v7.4s, v9.4s\n" + "ldr x20, [x16, #0x20]\n" + "add x20, x20, x15\n" + "mov v18.16b, v30.16b\n fmla v18.4s, v6.4s, v9.4s\n" + "mov v21.16b, v30.16b\n fmla v21.4s, v4.4s, v9.4s\n" + "mov v22.16b, v30.16b\n fmla v22.4s, v3.4s, v9.4s\n" + "mov v25.16b, v30.16b\n fmla v25.4s, v1.4s, v9.4s\n" + "mov v26.16b, v30.16b\n fmla v26.4s, v0.4s, v9.4s\n" + "mov v19.16b, v30.16b\n fmla v19.4s, v2.4s, v11.4s\n" + "mov v20.16b, v30.16b\n fmla v20.4s, v5.4s, v9.4s\n" + "mov v24.16b, v30.16b\n fmla v24.4s, v2.4s, v9.4s\n" "fmla v16.4s, v0.4s, v10.4s\n" "fmla v17.4s, v8.4s, v12.4s\n" "fmla v18.4s, v7.4s, v12.4s\n" "fmla v19.4s, v6.4s, v12.4s\n" "fmla v21.4s, v5.4s, v12.4s\n" "fmla v22.4s, v4.4s, v12.4s\n" - "mov v23.16b, v13.16b\n fmla v23.4s, v3.4s, v12.4s\n" + "mov v23.16b, v30.16b\n fmla v23.4s, v3.4s, v12.4s\n" "fmla v25.4s, v2.4s, v12.4s\n" "fmla v26.4s, v1.4s, v12.4s\n" - "mov v27.16b, v13.16b\n fmla v27.4s, v0.4s, v12.4s\n" + "mov v27.16b, v30.16b\n fmla v27.4s, v0.4s, v12.4s\n" "tbz %x[n_channels], #1, 6f\n" "ld1 { v10.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #0, 7f\n" @@ -768,9 +768,9 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "6:" // Oddments: Load input (5, 0): Bit 1: Unset "ld1 { v10.s }[0], [x20], #0x4\n" "7:" // Oddments: Load input (5, 0): Bit 1: End - "ldr x20, [x14, #0x28]\n" - "mov v28.16b, v13.16b\n fmla v28.4s, v6.4s, v10.4s\n" - "add x20, x20, x13\n" + "ldr x20, [x16, #0x28]\n" + "mov v28.16b, v30.16b\n fmla v28.4s, v6.4s, v10.4s\n" + "add x20, x20, x15\n" "tbz %x[n_channels], #1, 8f\n" "ld1 { v11.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #0, 9f\n" @@ -779,9 +779,9 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "8:" // Oddments: Load input (5, 5): Bit 1: Unset "ld1 { v11.s }[0], [x20], #0x4\n" "9:" // Oddments: Load input (5, 5): Bit 1: End - "ldr x20, [x14, #0x30]\n" - "mov v31.16b, v13.16b\n fmla v31.4s, v8.4s, v11.4s\n" - "add x20, x20, x13\n" + "ldr x20, [x16, #0x30]\n" + "mov v31.16b, v30.16b\n fmla v31.4s, v8.4s, v11.4s\n" + "add x20, x20, x15\n" "tbz %x[n_channels], #1, 10f\n" "ld1 { v9.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #0, 11f\n" @@ -790,17 +790,17 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "10:" // Oddments: Load input (3, 2): Bit 1: Unset "ld1 { v9.s }[0], [x20], #0x4\n" "11:" // Oddments: Load input (3, 2): Bit 1: End - "ldr x20, [x14, #0x38]\n" + "ldr x20, [x16, #0x38]\n" "fmla v20.4s, v8.4s, v9.4s\n" "fmla v21.4s, v7.4s, v9.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v22.4s, v6.4s, v9.4s\n" "fmla v24.4s, v5.4s, v9.4s\n" "fmla v25.4s, v4.4s, v9.4s\n" "fmla v26.4s, v3.4s, v9.4s\n" "fmla v28.4s, v2.4s, v9.4s\n" - "mov v29.16b, v13.16b\n fmla v29.4s, v1.4s, v9.4s\n" - "mov v30.16b, v13.16b\n fmla v30.4s, v0.4s, v9.4s\n" + "mov v29.16b, v30.16b\n fmla v29.4s, v1.4s, v9.4s\n" + "fmla v30.4s, v0.4s, v9.4s\n" "tbz %x[n_channels], #1, 12f\n" "ld1 { v12.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #0, 13f\n" @@ -809,10 +809,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "12:" // Oddments: Load input (0, 1): Bit 1: Unset "ld1 { v12.s }[0], [x20], #0x4\n" "13:" // Oddments: Load input (0, 1): Bit 1: End - "ldr x20, [x14, #0x40]\n" + "ldr x20, [x16, #0x40]\n" "fmla v16.4s, v1.4s, v12.4s\n" "fmla v17.4s, v0.4s, v12.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "tbz %x[n_channels], #1, 14f\n" "ld1 { v11.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #0, 15f\n" @@ -821,10 +821,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "14:" // Oddments: Load input (0, 4): Bit 1: Unset "ld1 { v11.s }[0], [x20], #0x4\n" "15:" // Oddments: Load input (0, 4): Bit 1: End - "ldr x20, [x14, #0x48]\n" + "ldr x20, [x16, #0x48]\n" "fmla v18.4s, v2.4s, v11.4s\n" "fmla v19.4s, v1.4s, v11.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "tbz %x[n_channels], #1, 16f\n" "ld1 { v10.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #0, 17f\n" @@ -833,10 +833,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "16:" // Oddments: Load input (3, 3): Bit 1: Unset "ld1 { v10.s }[0], [x20], #0x4\n" "17:" // Oddments: Load input (3, 3): Bit 1: End - "ldr x20, [x14, #0x50]\n" + "ldr x20, [x16, #0x50]\n" "fmla v21.4s, v8.4s, v10.4s\n" "fmla v22.4s, v7.4s, v10.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v23.4s, v6.4s, v10.4s\n" "fmla v25.4s, v5.4s, v10.4s\n" "fmla v26.4s, v4.4s, v10.4s\n" @@ -852,10 +852,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "18:" // Oddments: Load input (1, 0): Bit 1: Unset "ld1 { v9.s }[0], [x20], #0x4\n" "19:" // Oddments: Load input (1, 0): Bit 1: End - "ldr x20, [x14, #0x58]\n" + "ldr x20, [x16, #0x58]\n" "fmla v16.4s, v3.4s, v9.4s\n" "fmla v20.4s, v0.4s, v9.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "tbz %x[n_channels], #1, 20f\n" "ld1 { v12.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #0, 21f\n" @@ -864,10 +864,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "20:" // Oddments: Load input (1, 5): Bit 1: Unset "ld1 { v12.s }[0], [x20], #0x4\n" "21:" // Oddments: Load input (1, 5): Bit 1: End - "ldr x20, [x14, #0x60]\n" + "ldr x20, [x16, #0x60]\n" "fmla v19.4s, v5.4s, v12.4s\n" "fmla v23.4s, v2.4s, v12.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "tbz %x[n_channels], #1, 22f\n" "ld1 { v11.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #0, 23f\n" @@ -876,10 +876,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "22:" // Oddments: Load input (4, 0): Bit 1: Unset "ld1 { v11.s }[0], [x20], #0x4\n" "23:" // Oddments: Load input (4, 0): Bit 1: End - "ldr x20, [x14, #0x68]\n" + "ldr x20, [x16, #0x68]\n" "fmla v24.4s, v6.4s, v11.4s\n" "fmla v28.4s, v3.4s, v11.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "tbz %x[n_channels], #1, 24f\n" "ld1 { v10.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #0, 25f\n" @@ -888,10 +888,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "24:" // Oddments: Load input (1, 2): Bit 1: Unset "ld1 { v10.s }[0], [x20], #0x4\n" "25:" // Oddments: Load input (1, 2): Bit 1: End - "ldr x20, [x14, #0x70]\n" + "ldr x20, [x16, #0x70]\n" "fmla v16.4s, v5.4s, v10.4s\n" "fmla v17.4s, v4.4s, v10.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v18.4s, v3.4s, v10.4s\n" "fmla v20.4s, v2.4s, v10.4s\n" "fmla v21.4s, v1.4s, v10.4s\n" @@ -904,10 +904,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "26:" // Oddments: Load input (4, 5): Bit 1: Unset "ld1 { v11.s }[0], [x20], #0x4\n" "27:" // Oddments: Load input (4, 5): Bit 1: End - "ldr x20, [x14, #0x78]\n" + "ldr x20, [x16, #0x78]\n" "fmla v27.4s, v8.4s, v11.4s\n" "fmla v31.4s, v5.4s, v11.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "tbz %x[n_channels], #1, 28f\n" "ld1 { v12.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #0, 29f\n" @@ -916,10 +916,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "28:" // Oddments: Load input (1, 3): Bit 1: Unset "ld1 { v12.s }[0], [x20], #0x4\n" "29:" // Oddments: Load input (1, 3): Bit 1: End - "ldr x20, [x14, #0x80]\n" + "ldr x20, [x16, #0x80]\n" "fmla v17.4s, v5.4s, v12.4s\n" "fmla v18.4s, v4.4s, v12.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v19.4s, v3.4s, v12.4s\n" "fmla v21.4s, v2.4s, v12.4s\n" "fmla v22.4s, v1.4s, v12.4s\n" @@ -932,10 +932,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "30:" // Oddments: Load input (5, 1): Bit 1: Unset "ld1 { v11.s }[0], [x20], #0x4\n" "31:" // Oddments: Load input (5, 1): Bit 1: End - "ldr x20, [x14, #0x88]\n" + "ldr x20, [x16, #0x88]\n" "fmla v28.4s, v7.4s, v11.4s\n" "fmla v29.4s, v6.4s, v11.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "tbz %x[n_channels], #1, 32f\n" "ld1 { v10.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #0, 33f\n" @@ -944,10 +944,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "32:" // Oddments: Load input (2, 1): Bit 1: Unset "ld1 { v10.s }[0], [x20], #0x4\n" "33:" // Oddments: Load input (2, 1): Bit 1: End - "ldr x20, [x14, #0x90]\n" + "ldr x20, [x16, #0x90]\n" "fmla v16.4s, v7.4s, v10.4s\n" "fmla v17.4s, v6.4s, v10.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v20.4s, v4.4s, v10.4s\n" "fmla v21.4s, v3.4s, v10.4s\n" "fmla v24.4s, v1.4s, v10.4s\n" @@ -960,10 +960,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "34:" // Oddments: Load input (5, 4): Bit 1: Unset "ld1 { v11.s }[0], [x20], #0x4\n" "35:" // Oddments: Load input (5, 4): Bit 1: End - "ldr x20, [x14, #0x98]\n" + "ldr x20, [x16, #0x98]\n" "fmla v30.4s, v8.4s, v11.4s\n" "fmla v31.4s, v7.4s, v11.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "tbz %x[n_channels], #1, 36f\n" "ld1 { v12.d }[0], [x20], #0x8\n" "tbz %x[n_channels], #0, 37f\n" @@ -972,10 +972,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "36:" // Oddments: Load input (2, 4): Bit 1: Unset "ld1 { v12.s }[0], [x20], #0x4\n" "37:" // Oddments: Load input (2, 4): Bit 1: End - "ldr x20, [x14, #0xa0]\n" + "ldr x20, [x16, #0xa0]\n" "fmla v18.4s, v8.4s, v12.4s\n" "fmla v19.4s, v7.4s, v12.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v22.4s, v5.4s, v12.4s\n" "fmla v23.4s, v4.4s, v12.4s\n" "fmla v26.4s, v2.4s, v12.4s\n" @@ -988,10 +988,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "38:" // Oddments: Load input (0, 2): Bit 1: Unset "ld1 { v10.s }[0], [x20], #0x4\n" "39:" // Oddments: Load input (0, 2): Bit 1: End - "ldr x20, [x14, #0xa8]\n" + "ldr x20, [x16, #0xa8]\n" "fmla v16.4s, v2.4s, v10.4s\n" "fmla v17.4s, v1.4s, v10.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v18.4s, v0.4s, v10.4s\n" "tbz %x[n_channels], #1, 40f\n" "ld1 { v11.d }[0], [x20], #0x8\n" @@ -1001,10 +1001,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "40:" // Oddments: Load input (3, 1): Bit 1: Unset "ld1 { v11.s }[0], [x20], #0x4\n" "41:" // Oddments: Load input (3, 1): Bit 1: End - "ldr x20, [x14, #0xb0]\n" + "ldr x20, [x16, #0xb0]\n" "fmla v20.4s, v7.4s, v11.4s\n" "fmla v21.4s, v6.4s, v11.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v24.4s, v4.4s, v11.4s\n" "fmla v25.4s, v3.4s, v11.4s\n" "fmla v28.4s, v1.4s, v11.4s\n" @@ -1017,10 +1017,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "42:" // Oddments: Load input (0, 3): Bit 1: Unset "ld1 { v12.s }[0], [x20], #0x4\n" "43:" // Oddments: Load input (0, 3): Bit 1: End - "ldr x20, [x14, #0xb8]\n" + "ldr x20, [x16, #0xb8]\n" "fmla v17.4s, v2.4s, v12.4s\n" "fmla v18.4s, v1.4s, v12.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v19.4s, v0.4s, v12.4s\n" "tbz %x[n_channels], #1, 44f\n" "ld1 { v10.d }[0], [x20], #0x8\n" @@ -1030,10 +1030,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "44:" // Oddments: Load input (2, 0): Bit 1: Unset "ld1 { v10.s }[0], [x20], #0x4\n" "45:" // Oddments: Load input (2, 0): Bit 1: End - "ldr x20, [x14, #0xc0]\n" + "ldr x20, [x16, #0xc0]\n" "fmla v16.4s, v6.4s, v10.4s\n" "fmla v20.4s, v3.4s, v10.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v24.4s, v0.4s, v10.4s\n" "tbz %x[n_channels], #1, 46f\n" "ld1 { v11.d }[0], [x20], #0x8\n" @@ -1043,10 +1043,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "46:" // Oddments: Load input (3, 4): Bit 1: Unset "ld1 { v11.s }[0], [x20], #0x4\n" "47:" // Oddments: Load input (3, 4): Bit 1: End - "ldr x20, [x14, #0xc8]\n" + "ldr x20, [x16, #0xc8]\n" "fmla v22.4s, v8.4s, v11.4s\n" "fmla v23.4s, v7.4s, v11.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v26.4s, v5.4s, v11.4s\n" "fmla v27.4s, v4.4s, v11.4s\n" "fmla v30.4s, v2.4s, v11.4s\n" @@ -1059,10 +1059,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "48:" // Oddments: Load input (2, 5): Bit 1: Unset "ld1 { v12.s }[0], [x20], #0x4\n" "49:" // Oddments: Load input (2, 5): Bit 1: End - "ldr x20, [x14, #0xd0]\n" + "ldr x20, [x16, #0xd0]\n" "fmla v19.4s, v8.4s, v12.4s\n" "fmla v23.4s, v5.4s, v12.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v27.4s, v2.4s, v12.4s\n" "tbz %x[n_channels], #1, 50f\n" "ld1 { v10.d }[0], [x20], #0x8\n" @@ -1072,10 +1072,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "50:" // Oddments: Load input (3, 0): Bit 1: Unset "ld1 { v10.s }[0], [x20], #0x4\n" "51:" // Oddments: Load input (3, 0): Bit 1: End - "ldr x20, [x14, #0xd8]\n" + "ldr x20, [x16, #0xd8]\n" "fmla v20.4s, v6.4s, v10.4s\n" "fmla v24.4s, v3.4s, v10.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v28.4s, v0.4s, v10.4s\n" "tbz %x[n_channels], #1, 52f\n" "ld1 { v11.d }[0], [x20], #0x8\n" @@ -1085,10 +1085,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "52:" // Oddments: Load input (4, 2): Bit 1: Unset "ld1 { v11.s }[0], [x20], #0x4\n" "53:" // Oddments: Load input (4, 2): Bit 1: End - "ldr x20, [x14, #0xe0]\n" + "ldr x20, [x16, #0xe0]\n" "fmla v24.4s, v8.4s, v11.4s\n" "fmla v25.4s, v7.4s, v11.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v26.4s, v6.4s, v11.4s\n" "fmla v28.4s, v5.4s, v11.4s\n" "fmla v29.4s, v4.4s, v11.4s\n" @@ -1101,10 +1101,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "54:" // Oddments: Load input (3, 5): Bit 1: Unset "ld1 { v12.s }[0], [x20], #0x4\n" "55:" // Oddments: Load input (3, 5): Bit 1: End - "ldr x20, [x14, #0xe8]\n" + "ldr x20, [x16, #0xe8]\n" "fmla v23.4s, v8.4s, v12.4s\n" "fmla v27.4s, v5.4s, v12.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v31.4s, v2.4s, v12.4s\n" "tbz %x[n_channels], #1, 56f\n" "ld1 { v10.d }[0], [x20], #0x8\n" @@ -1114,10 +1114,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "56:" // Oddments: Load input (5, 2): Bit 1: Unset "ld1 { v10.s }[0], [x20], #0x4\n" "57:" // Oddments: Load input (5, 2): Bit 1: End - "ldr x20, [x14, #0xf0]\n" + "ldr x20, [x16, #0xf0]\n" "fmla v28.4s, v8.4s, v10.4s\n" "fmla v29.4s, v7.4s, v10.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v30.4s, v6.4s, v10.4s\n" "tbz %x[n_channels], #1, 58f\n" "ld1 { v11.d }[0], [x20], #0x8\n" @@ -1127,10 +1127,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "58:" // Oddments: Load input (4, 3): Bit 1: Unset "ld1 { v11.s }[0], [x20], #0x4\n" "59:" // Oddments: Load input (4, 3): Bit 1: End - "ldr x20, [x14, #0xf8]\n" + "ldr x20, [x16, #0xf8]\n" "fmla v25.4s, v8.4s, v11.4s\n" "fmla v26.4s, v7.4s, v11.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v27.4s, v6.4s, v11.4s\n" "fmla v29.4s, v5.4s, v11.4s\n" "fmla v30.4s, v4.4s, v11.4s\n" @@ -1143,10 +1143,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "60:" // Oddments: Load input (5, 3): Bit 1: Unset "ld1 { v12.s }[0], [x20], #0x4\n" "61:" // Oddments: Load input (5, 3): Bit 1: End - "ldr x20, [x14, #0x100]\n" + "ldr x20, [x16, #0x100]\n" "fmla v29.4s, v8.4s, v12.4s\n" "fmla v30.4s, v7.4s, v12.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v31.4s, v6.4s, v12.4s\n" "tbz %x[n_channels], #1, 62f\n" "ld1 { v10.d }[0], [x20], #0x8\n" @@ -1156,10 +1156,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "62:" // Oddments: Load input (1, 1): Bit 1: Unset "ld1 { v10.s }[0], [x20], #0x4\n" "63:" // Oddments: Load input (1, 1): Bit 1: End - "ldr x20, [x14, #0x108]\n" + "ldr x20, [x16, #0x108]\n" "fmla v16.4s, v4.4s, v10.4s\n" "fmla v17.4s, v3.4s, v10.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v20.4s, v1.4s, v10.4s\n" "fmla v21.4s, v0.4s, v10.4s\n" "tbz %x[n_channels], #1, 64f\n" @@ -1170,10 +1170,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "64:" // Oddments: Load input (1, 4): Bit 1: Unset "ld1 { v11.s }[0], [x20], #0x4\n" "65:" // Oddments: Load input (1, 4): Bit 1: End - "ldr x20, [x14, #0x110]\n" + "ldr x20, [x16, #0x110]\n" "fmla v18.4s, v5.4s, v11.4s\n" "fmla v19.4s, v4.4s, v11.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v22.4s, v2.4s, v11.4s\n" "fmla v23.4s, v1.4s, v11.4s\n" "tbz %x[n_channels], #1, 66f\n" @@ -1184,10 +1184,10 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "66:" // Oddments: Load input (4, 1): Bit 1: Unset "ld1 { v12.s }[0], [x20], #0x4\n" "67:" // Oddments: Load input (4, 1): Bit 1: End - "ldr x20, [x14, #0x118]\n" + "ldr x20, [x16, #0x118]\n" "fmla v24.4s, v7.4s, v12.4s\n" "fmla v25.4s, v6.4s, v12.4s\n" - "add x20, x20, x13\n" + "add x20, x20, x15\n" "fmla v28.4s, v4.4s, v12.4s\n" "fmla v29.4s, v3.4s, v12.4s\n" "tbz %x[n_channels], #1, 68f\n" @@ -1200,24 +1200,24 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "69:" // Oddments: Load input (4, 4): Bit 1: End "fmla v26.4s, v8.4s, v10.4s\n" "fmla v27.4s, v7.4s, v10.4s\n" - "fmax v16.4s, v16.4s, v15.4s\n" + "fmax v16.4s, v16.4s, v13.4s\n" "fmla v30.4s, v5.4s, v10.4s\n" "fmla v31.4s, v4.4s, v10.4s\n" - "fmax v17.4s, v17.4s, v15.4s\n" - "fmax v18.4s, v18.4s, v15.4s\n" - "fmax v19.4s, v19.4s, v15.4s\n" - "fmax v20.4s, v20.4s, v15.4s\n" - "fmax v21.4s, v21.4s, v15.4s\n" - "fmax v22.4s, v22.4s, v15.4s\n" - "fmax v23.4s, v23.4s, v15.4s\n" - "fmax v24.4s, v24.4s, v15.4s\n" - "fmax v25.4s, v25.4s, v15.4s\n" - "fmax v26.4s, v26.4s, v15.4s\n" - "fmax v27.4s, v27.4s, v15.4s\n" - "fmax v28.4s, v28.4s, v15.4s\n" - "fmax v29.4s, v29.4s, v15.4s\n" - "fmax v30.4s, v30.4s, v15.4s\n" - "fmax v31.4s, v31.4s, v15.4s\n" + "fmax v17.4s, v17.4s, v13.4s\n" + "fmax v18.4s, v18.4s, v13.4s\n" + "fmax v19.4s, v19.4s, v13.4s\n" + "fmax v20.4s, v20.4s, v13.4s\n" + "fmax v21.4s, v21.4s, v13.4s\n" + "fmax v22.4s, v22.4s, v13.4s\n" + "fmax v23.4s, v23.4s, v13.4s\n" + "fmax v24.4s, v24.4s, v13.4s\n" + "fmax v25.4s, v25.4s, v13.4s\n" + "fmax v26.4s, v26.4s, v13.4s\n" + "fmax v27.4s, v27.4s, v13.4s\n" + "fmax v28.4s, v28.4s, v13.4s\n" + "fmax v29.4s, v29.4s, v13.4s\n" + "fmax v30.4s, v30.4s, v13.4s\n" + "fmax v31.4s, v31.4s, v13.4s\n" "fmin v16.4s, v16.4s, v14.4s\n" "fmin v17.4s, v17.4s, v14.4s\n" "fmin v18.4s, v18.4s, v14.4s\n" @@ -1235,150 +1235,150 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "fmin v30.4s, v30.4s, v14.4s\n" "fmin v31.4s, v31.4s, v14.4s\n" "tbz %x[n_channels], #1, 70f\n" - "ldr x23, [x16, #0x0]\n" - "ldr x22, [x16, #0x8]\n" - "add x23, x23, x12\n" - "add x22, x22, x12\n" - "ldr x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x18]\n" - "add x21, x21, x12\n" - "add x20, x20, x12\n" + "ldr x23, [x8, #0x0]\n" + "ldr x22, [x8, #0x8]\n" + "add x23, x23, x14\n" + "add x22, x22, x14\n" + "ldr x21, [x8, #0x10]\n" + "ldr x20, [x8, #0x18]\n" + "add x21, x21, x14\n" + "add x20, x20, x14\n" "st1 { v16.d }[0], [x23]\n" - "ldr x23, [x16, #0x20]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x20]\n" + "add x23, x23, x14\n" "st1 { v17.d }[0], [x22]\n" - "ldr x22, [x16, #0x28]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x28]\n" + "add x22, x22, x14\n" "st1 { v18.d }[0], [x21]\n" - "ldr x21, [x16, #0x30]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x30]\n" + "add x21, x21, x14\n" "st1 { v19.d }[0], [x20]\n" - "ldr x20, [x16, #0x38]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x38]\n" + "add x20, x20, x14\n" "st1 { v20.d }[0], [x23]\n" - "ldr x23, [x16, #0x40]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x40]\n" + "add x23, x23, x14\n" "st1 { v21.d }[0], [x22]\n" - "ldr x22, [x16, #0x48]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x48]\n" + "add x22, x22, x14\n" "st1 { v22.d }[0], [x21]\n" - "ldr x21, [x16, #0x50]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x50]\n" + "add x21, x21, x14\n" "st1 { v23.d }[0], [x20]\n" - "ldr x20, [x16, #0x58]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x58]\n" + "add x20, x20, x14\n" "st1 { v24.d }[0], [x23]\n" - "ldr x23, [x16, #0x60]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x60]\n" + "add x23, x23, x14\n" "st1 { v25.d }[0], [x22]\n" - "ldr x22, [x16, #0x68]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x68]\n" + "add x22, x22, x14\n" "st1 { v26.d }[0], [x21]\n" - "ldr x21, [x16, #0x70]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x70]\n" + "add x21, x21, x14\n" "st1 { v27.d }[0], [x20]\n" - "ldr x20, [x16, #0x78]\n" - "add x20, x20, x12\n" - "add x12, x12, #0x8\n" + "ldr x20, [x8, #0x78]\n" + "add x20, x20, x14\n" + "add x14, x14, #0x8\n" "st1 { v28.d }[0], [x23]\n" "st1 { v29.d }[0], [x22]\n" "st1 { v30.d }[0], [x21]\n" "st1 { v31.d }[0], [x20]\n" "tbz %x[n_channels], #0, 71f\n" - "ldr x23, [x16, #0x0]\n" - "ldr x22, [x16, #0x8]\n" - "add x23, x23, x12\n" - "add x22, x22, x12\n" - "ldr x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x18]\n" - "add x21, x21, x12\n" - "add x20, x20, x12\n" + "ldr x23, [x8, #0x0]\n" + "ldr x22, [x8, #0x8]\n" + "add x23, x23, x14\n" + "add x22, x22, x14\n" + "ldr x21, [x8, #0x10]\n" + "ldr x20, [x8, #0x18]\n" + "add x21, x21, x14\n" + "add x20, x20, x14\n" "st1 { v16.s }[2], [x23]\n" - "ldr x23, [x16, #0x20]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x20]\n" + "add x23, x23, x14\n" "st1 { v17.s }[2], [x22]\n" - "ldr x22, [x16, #0x28]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x28]\n" + "add x22, x22, x14\n" "st1 { v18.s }[2], [x21]\n" - "ldr x21, [x16, #0x30]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x30]\n" + "add x21, x21, x14\n" "st1 { v19.s }[2], [x20]\n" - "ldr x20, [x16, #0x38]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x38]\n" + "add x20, x20, x14\n" "st1 { v20.s }[2], [x23]\n" - "ldr x23, [x16, #0x40]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x40]\n" + "add x23, x23, x14\n" "st1 { v21.s }[2], [x22]\n" - "ldr x22, [x16, #0x48]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x48]\n" + "add x22, x22, x14\n" "st1 { v22.s }[2], [x21]\n" - "ldr x21, [x16, #0x50]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x50]\n" + "add x21, x21, x14\n" "st1 { v23.s }[2], [x20]\n" - "ldr x20, [x16, #0x58]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x58]\n" + "add x20, x20, x14\n" "st1 { v24.s }[2], [x23]\n" - "ldr x23, [x16, #0x60]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x60]\n" + "add x23, x23, x14\n" "st1 { v25.s }[2], [x22]\n" - "ldr x22, [x16, #0x68]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x68]\n" + "add x22, x22, x14\n" "st1 { v26.s }[2], [x21]\n" - "ldr x21, [x16, #0x70]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x70]\n" + "add x21, x21, x14\n" "st1 { v27.s }[2], [x20]\n" - "ldr x20, [x16, #0x78]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x78]\n" + "add x20, x20, x14\n" "st1 { v28.s }[2], [x23]\n" "st1 { v29.s }[2], [x22]\n" "st1 { v30.s }[2], [x21]\n" "st1 { v31.s }[2], [x20]\n" "b 71f\n" "70:" // Oddments: Store: Bit 1: Unset - "ldr x23, [x16, #0x0]\n" - "ldr x22, [x16, #0x8]\n" - "add x23, x23, x12\n" - "add x22, x22, x12\n" - "ldr x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x18]\n" - "add x21, x21, x12\n" - "add x20, x20, x12\n" + "ldr x23, [x8, #0x0]\n" + "ldr x22, [x8, #0x8]\n" + "add x23, x23, x14\n" + "add x22, x22, x14\n" + "ldr x21, [x8, #0x10]\n" + "ldr x20, [x8, #0x18]\n" + "add x21, x21, x14\n" + "add x20, x20, x14\n" "st1 { v16.s }[0], [x23]\n" - "ldr x23, [x16, #0x20]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x20]\n" + "add x23, x23, x14\n" "st1 { v17.s }[0], [x22]\n" - "ldr x22, [x16, #0x28]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x28]\n" + "add x22, x22, x14\n" "st1 { v18.s }[0], [x21]\n" - "ldr x21, [x16, #0x30]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x30]\n" + "add x21, x21, x14\n" "st1 { v19.s }[0], [x20]\n" - "ldr x20, [x16, #0x38]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x38]\n" + "add x20, x20, x14\n" "st1 { v20.s }[0], [x23]\n" - "ldr x23, [x16, #0x40]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x40]\n" + "add x23, x23, x14\n" "st1 { v21.s }[0], [x22]\n" - "ldr x22, [x16, #0x48]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x48]\n" + "add x22, x22, x14\n" "st1 { v22.s }[0], [x21]\n" - "ldr x21, [x16, #0x50]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x50]\n" + "add x21, x21, x14\n" "st1 { v23.s }[0], [x20]\n" - "ldr x20, [x16, #0x58]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x58]\n" + "add x20, x20, x14\n" "st1 { v24.s }[0], [x23]\n" - "ldr x23, [x16, #0x60]\n" - "add x23, x23, x12\n" + "ldr x23, [x8, #0x60]\n" + "add x23, x23, x14\n" "st1 { v25.s }[0], [x22]\n" - "ldr x22, [x16, #0x68]\n" - "add x22, x22, x12\n" + "ldr x22, [x8, #0x68]\n" + "add x22, x22, x14\n" "st1 { v26.s }[0], [x21]\n" - "ldr x21, [x16, #0x70]\n" - "add x21, x21, x12\n" + "ldr x21, [x8, #0x70]\n" + "add x21, x21, x14\n" "st1 { v27.s }[0], [x20]\n" - "ldr x20, [x16, #0x78]\n" - "add x20, x20, x12\n" + "ldr x20, [x8, #0x78]\n" + "add x20, x20, x14\n" "st1 { v28.s }[0], [x23]\n" "st1 { v29.s }[0], [x22]\n" "st1 { v30.s }[0], [x21]\n" @@ -1387,11 +1387,11 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "72:" // End : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp index ff521fb2caa64e43524f24507befe191189d4c09..f727efea806ee16b7411b2fa522192e723927b05 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include @@ -33,8 +33,8 @@ namespace arm_conv { namespace depthwise { -void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); -void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); +void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(const float *const *const input_ptrs, float *const *const outptrs, const void *params, unsigned int n_channels, const float activation_min, const float activation_max); +void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(const unsigned int n_tile_rows, const unsigned int n_tile_cols, const float *inptr, int64_t ld_input_row, int64_t ld_input_col, float *outptr, int64_t ld_output_row, int64_t ld_output_col, const void *params, unsigned int n_channels, const float activation_min, const float activation_max); class a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst : public DepthwiseDepthfirstStrategy { @@ -57,7 +57,7 @@ class a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst : public DepthwiseDepthfirst constexpr static unsigned int output_cols = 2; a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst(const CPUInfo *) - : DepthwiseDepthfirstStrategy(2, 3, 2) {} + : Parent(output_rows, output_cols, kernel_rows, kernel_cols, stride_rows, stride_cols) {} arm_gemm::VLType get_vl_type(void) const override { return vl_type; } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp index e42ceffb50aa3c5aa7df3e5f3c877fa4dd9d0dc7..5ab61fad4c6089fe840208c1f3aff70bc40d0313 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ +#if defined(__aarch64__) namespace arm_conv { namespace depthwise { @@ -106,7 +106,7 @@ void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "ldr x17, [%x[params_struct], %[offsetof_args_outptr]]\n" "mov x23, #0x10\n" // cntb _, ALL, #1 "mul x22, x22, x26\n" // offset *= kernel_stride * output_size - "add x8, x8, x22, LSL #2\n" // inptr[0] += offset * sizeof(float) + "add x8, x8, x22, LSL #2\n" // inptr[0] += offset * sizeof(float) "add x16, x8, x24, LSL #2\n" "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" "madd x20, x27, x7, x20\n" // offset += tile_j * ld_output_col @@ -118,9 +118,9 @@ void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "add x11, x13, x6\n" "add x17, x17, x20, LSL #2\n" // outptrs[0] += offset * sizeof(float) "add x20, %x[params_struct], %[offsetof_args_min]\n" - "ld1r { v19.4s }, [x20]\n" + "ld1r { v26.4s }, [x20]\n" "add x20, %x[params_struct], %[offsetof_args_max]\n" - "ld1r { v18.4s }, [x20]\n" + "ld1r { v27.4s }, [x20]\n" "add x10, x12, x24, LSL #2\n" "add x9, x11, x6\n" "add x28, x17, x21, LSL #2\n" @@ -128,7 +128,7 @@ void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "mov x21, #0x0\n" "sub x20, XZR, x23\n" "cbz x22, 4f\n" - "ldr q17, [x15, #0x0]\n" + "ldr q31, [x15, #0x0]\n" "ldr q0, [x15, #0x10]\n" "cmp x23, x22, LSL #4\n" "ldr q1, [x15, #0x20]\n" @@ -150,179 +150,179 @@ void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "ldr q16, [x8, x13]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "mov v28.16b, v17.16b\n fmla v28.4s, v8.4s, v9.4s\n" - "mov v29.16b, v17.16b\n fmla v29.4s, v6.4s, v9.4s\n" + "mov v29.16b, v31.16b\n fmla v29.4s, v8.4s, v9.4s\n" + "mov v28.16b, v31.16b\n fmla v28.4s, v6.4s, v9.4s\n" "add x23, x23, #0x10\n" "add x8, x8, #0x10\n" - "fmla v28.4s, v0.4s, v10.4s\n" + "fmla v29.4s, v0.4s, v10.4s\n" "ld1 { v10.4s }, [x8]\n" - "fmla v29.4s, v1.4s, v12.4s\n" - "ldr q12, [x16, x9]\n" - "fmla v28.4s, v1.4s, v11.4s\n" - "ldr q11, [x16, x11]\n" - "fmla v29.4s, v2.4s, v13.4s\n" - "ldr q13, [x16, x13]\n" - "fmla v28.4s, v3.4s, v14.4s\n" - "ld1 { v14.4s }, [x12]\n" - "fmla v29.4s, v0.4s, v16.4s\n" + "fmla v28.4s, v1.4s, v12.4s\n" + "ldr q21, [x16, x9]\n" + "fmla v29.4s, v1.4s, v11.4s\n" + "ldr q18, [x16, x11]\n" + "fmla v28.4s, v2.4s, v13.4s\n" + "ldr q17, [x16, x13]\n" + "fmla v29.4s, v3.4s, v14.4s\n" + "ld1 { v20.4s }, [x12]\n" + "fmla v28.4s, v0.4s, v16.4s\n" "add x16, x16, #0x10\n" - "fmla v28.4s, v4.4s, v15.4s\n" - "ld1 { v15.4s }, [x14]\n" - "fmla v29.4s, v4.4s, v11.4s\n" - "ldr q11, [x12, x6]\n" - "fmla v28.4s, v2.4s, v16.4s\n" - "ldr q16, [x14, x6]\n" - "fmla v29.4s, v5.4s, v12.4s\n" - "ldr q12, [x14, x11]\n" - "mov v30.16b, v17.16b\n fmla v30.4s, v2.4s, v9.4s\n" - "mov v31.16b, v17.16b\n fmla v31.4s, v0.4s, v9.4s\n" - "ldr q17, [x15, #0x0]\n" + "fmla v29.4s, v4.4s, v15.4s\n" + "ld1 { v25.4s }, [x14]\n" + "fmla v28.4s, v4.4s, v18.4s\n" + "ldr q19, [x12, x6]\n" + "fmla v29.4s, v2.4s, v16.4s\n" + "ldr q18, [x14, x6]\n" + "fmla v28.4s, v5.4s, v21.4s\n" + "ldr q24, [x14, x11]\n" + "mov v23.16b, v31.16b\n fmla v23.4s, v2.4s, v9.4s\n" + "mov v22.16b, v31.16b\n fmla v22.4s, v0.4s, v9.4s\n" + "ldr q31, [x15, #0x0]\n" "cmp x23, x22, LSL #4\n" - "fmla v28.4s, v5.4s, v13.4s\n" - "fmla v29.4s, v3.4s, v13.4s\n" - "ldr q13, [x12, x11]\n" + "fmla v29.4s, v5.4s, v17.4s\n" + "fmla v28.4s, v3.4s, v17.4s\n" + "ldr q17, [x12, x11]\n" "add x20, x20, #0x10\n" - "fmla v30.4s, v3.4s, v14.4s\n" - "ldr q14, [x12, x9]\n" - "fmla v31.4s, v4.4s, v13.4s\n" - "ldr q13, [x10, x6]\n" - "fmla v30.4s, v0.4s, v15.4s\n" + "fmla v23.4s, v3.4s, v20.4s\n" + "ldr q16, [x12, x9]\n" + "fmla v22.4s, v4.4s, v17.4s\n" + "ldr q21, [x10, x6]\n" + "fmla v23.4s, v0.4s, v25.4s\n" "ldr q0, [x15, #0x10]\n" - "fmla v31.4s, v1.4s, v12.4s\n" + "fmla v22.4s, v1.4s, v24.4s\n" "add x21, x21, #0x10\n" - "fmla v30.4s, v4.4s, v11.4s\n" - "ldr q11, [x14, x9]\n" + "fmla v23.4s, v4.4s, v19.4s\n" + "ldr q20, [x14, x9]\n" "ldr q4, [x15, #0x50]\n" - "fmla v31.4s, v5.4s, v14.4s\n" - "ldr q14, [x10, x11]\n" - "fmla v28.4s, v6.4s, v15.4s\n" - "ld1 { v15.4s }, [x10]\n" - "fmla v30.4s, v1.4s, v16.4s\n" + "fmla v22.4s, v5.4s, v16.4s\n" + "ldr q19, [x10, x11]\n" + "fmla v29.4s, v6.4s, v25.4s\n" + "ld1 { v17.4s }, [x10]\n" + "fmla v23.4s, v1.4s, v18.4s\n" "ldr q1, [x15, #0x20]\n" - "fmla v31.4s, v2.4s, v11.4s\n" + "fmla v22.4s, v2.4s, v20.4s\n" "ldr q2, [x15, #0x30]\n" - "fmla v28.4s, v7.4s, v16.4s\n" + "fmla v29.4s, v7.4s, v18.4s\n" "ldr q16, [x12, x13]\n" - "fmla v30.4s, v6.4s, v15.4s\n" - "ldr q15, [x10, x13]\n" - "fmla v31.4s, v3.4s, v16.4s\n" + "fmla v23.4s, v6.4s, v17.4s\n" + "ldr q18, [x10, x13]\n" + "fmla v22.4s, v3.4s, v16.4s\n" "ldr q3, [x15, #0x40]\n" - "fmla v30.4s, v7.4s, v13.4s\n" + "fmla v23.4s, v7.4s, v21.4s\n" "ldr q13, [x8, x9]\n" - "fmla v31.4s, v7.4s, v14.4s\n" + "fmla v22.4s, v7.4s, v19.4s\n" "ld1 { v14.4s }, [x16]\n" - "fmla v29.4s, v7.4s, v12.4s\n" + "fmla v28.4s, v7.4s, v24.4s\n" "ldr q12, [x8, x11]\n" - "fmla v30.4s, v5.4s, v16.4s\n" + "fmla v23.4s, v5.4s, v16.4s\n" "ldr q16, [x8, x13]\n" "ldr q5, [x15, #0x60]\n" - "fmla v31.4s, v6.4s, v15.4s\n" - "fmla v29.4s, v8.4s, v11.4s\n" - "ldr q11, [x10, x9]\n" + "fmla v22.4s, v6.4s, v18.4s\n" + "fmla v28.4s, v8.4s, v20.4s\n" + "ldr q17, [x10, x9]\n" "ldr q6, [x15, #0x70]\n" - "fmla v30.4s, v8.4s, v15.4s\n" - "fmla v31.4s, v8.4s, v11.4s\n" + "fmla v23.4s, v8.4s, v18.4s\n" + "fmla v22.4s, v8.4s, v17.4s\n" "ldr q11, [x8, x6]\n" "ldr q15, [x16, x6]\n" - "fmax v28.4s, v28.4s, v19.4s\n" - "fmax v29.4s, v29.4s, v19.4s\n" + "fmax v29.4s, v29.4s, v26.4s\n" + "fmax v28.4s, v28.4s, v26.4s\n" "ldr q7, [x15, #0x80]\n" "ldr q8, [x15, #0x90]\n" - "fmax v30.4s, v30.4s, v19.4s\n" - "fmax v31.4s, v31.4s, v19.4s\n" + "fmax v23.4s, v23.4s, v26.4s\n" + "fmax v22.4s, v22.4s, v26.4s\n" "add x14, x14, #0x10\n" "ldr q9, [x14, x13]\n" - "fmin v28.4s, v28.4s, v18.4s\n" - "fmin v29.4s, v29.4s, v18.4s\n" - "fmin v30.4s, v30.4s, v18.4s\n" - "fmin v31.4s, v31.4s, v18.4s\n" + "fmin v29.4s, v29.4s, v27.4s\n" + "fmin v28.4s, v28.4s, v27.4s\n" + "fmin v23.4s, v23.4s, v27.4s\n" + "fmin v22.4s, v22.4s, v27.4s\n" "add x12, x12, #0x10\n" "add x10, x10, #0x10\n" - "st1 { v28.4s }, [x17]\n" + "st1 { v29.4s }, [x17]\n" "add x15, x15, #0xa0\n" - "str q29, [x17, x7]\n" + "str q28, [x17, x7]\n" "add x17, x17, #0x10\n" - "st1 { v30.4s }, [x28]\n" - "str q31, [x28, x7]\n" + "st1 { v23.4s }, [x28]\n" + "str q22, [x28, x7]\n" "add x28, x28, #0x10\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "mov v28.16b, v17.16b\n fmla v28.4s, v8.4s, v9.4s\n" - "mov v29.16b, v17.16b\n fmla v29.4s, v6.4s, v9.4s\n" + "mov v29.16b, v31.16b\n fmla v29.4s, v8.4s, v9.4s\n" + "mov v28.16b, v31.16b\n fmla v28.4s, v6.4s, v9.4s\n" "add x8, x8, #0x10\n" - "fmla v28.4s, v0.4s, v10.4s\n" - "fmla v29.4s, v1.4s, v12.4s\n" - "ldr q12, [x16, x9]\n" - "fmla v28.4s, v1.4s, v11.4s\n" - "ldr q11, [x16, x11]\n" - "fmla v29.4s, v2.4s, v13.4s\n" - "ldr q13, [x16, x13]\n" - "fmla v28.4s, v3.4s, v14.4s\n" - "ld1 { v14.4s }, [x12]\n" - "fmla v29.4s, v0.4s, v16.4s\n" + "fmla v29.4s, v0.4s, v10.4s\n" + "fmla v28.4s, v1.4s, v12.4s\n" + "ldr q20, [x16, x9]\n" + "fmla v29.4s, v1.4s, v11.4s\n" + "ldr q18, [x16, x11]\n" + "fmla v28.4s, v2.4s, v13.4s\n" + "ldr q17, [x16, x13]\n" + "fmla v29.4s, v3.4s, v14.4s\n" + "ld1 { v19.4s }, [x12]\n" + "fmla v28.4s, v0.4s, v16.4s\n" "add x16, x16, #0x10\n" - "fmla v28.4s, v4.4s, v15.4s\n" - "ld1 { v15.4s }, [x14]\n" - "fmla v29.4s, v4.4s, v11.4s\n" - "ldr q11, [x12, x6]\n" - "fmla v28.4s, v2.4s, v16.4s\n" - "ldr q16, [x14, x6]\n" - "fmla v29.4s, v5.4s, v12.4s\n" - "ldr q12, [x14, x11]\n" - "mov v30.16b, v17.16b\n fmla v30.4s, v2.4s, v9.4s\n" - "mov v31.16b, v17.16b\n fmla v31.4s, v0.4s, v9.4s\n" - "fmla v28.4s, v5.4s, v13.4s\n" - "fmla v29.4s, v3.4s, v13.4s\n" - "ldr q13, [x12, x11]\n" - "fmla v30.4s, v3.4s, v14.4s\n" - "ldr q14, [x12, x9]\n" - "fmla v31.4s, v4.4s, v13.4s\n" - "ldr q13, [x10, x6]\n" - "fmla v30.4s, v0.4s, v15.4s\n" - "fmla v31.4s, v1.4s, v12.4s\n" - "fmla v30.4s, v4.4s, v11.4s\n" - "ldr q11, [x14, x9]\n" - "fmla v31.4s, v5.4s, v14.4s\n" - "ldr q14, [x10, x11]\n" - "fmla v28.4s, v6.4s, v15.4s\n" - "ld1 { v15.4s }, [x10]\n" - "fmla v30.4s, v1.4s, v16.4s\n" + "fmla v29.4s, v4.4s, v15.4s\n" + "ld1 { v25.4s }, [x14]\n" + "fmla v28.4s, v4.4s, v18.4s\n" + "ldr q18, [x12, x6]\n" + "fmla v29.4s, v2.4s, v16.4s\n" + "ldr q24, [x14, x6]\n" + "fmla v28.4s, v5.4s, v20.4s\n" + "ldr q23, [x14, x11]\n" + "mov v22.16b, v31.16b\n fmla v22.4s, v2.4s, v9.4s\n" + "mov v21.16b, v31.16b\n fmla v21.4s, v0.4s, v9.4s\n" + "fmla v29.4s, v5.4s, v17.4s\n" + "fmla v28.4s, v3.4s, v17.4s\n" + "ldr q17, [x12, x11]\n" + "fmla v22.4s, v3.4s, v19.4s\n" + "ldr q16, [x12, x9]\n" + "fmla v21.4s, v4.4s, v17.4s\n" + "ldr q20, [x10, x6]\n" + "fmla v22.4s, v0.4s, v25.4s\n" + "fmla v21.4s, v1.4s, v23.4s\n" + "fmla v22.4s, v4.4s, v18.4s\n" + "ldr q19, [x14, x9]\n" + "fmla v21.4s, v5.4s, v16.4s\n" + "ldr q18, [x10, x11]\n" + "fmla v29.4s, v6.4s, v25.4s\n" + "ld1 { v17.4s }, [x10]\n" + "fmla v22.4s, v1.4s, v24.4s\n" "add x14, x14, #0x10\n" - "fmla v31.4s, v2.4s, v11.4s\n" - "fmla v28.4s, v7.4s, v16.4s\n" + "fmla v21.4s, v2.4s, v19.4s\n" + "fmla v29.4s, v7.4s, v24.4s\n" "ldr q16, [x12, x13]\n" - "fmax v28.4s, v28.4s, v19.4s\n" - "fmla v30.4s, v6.4s, v15.4s\n" - "ldr q15, [x10, x13]\n" - "fmla v31.4s, v3.4s, v16.4s\n" - "fmin v28.4s, v28.4s, v18.4s\n" - "fmla v30.4s, v7.4s, v13.4s\n" - "fmla v31.4s, v7.4s, v14.4s\n" - "st1 { v28.4s }, [x17]\n" + "fmax v29.4s, v29.4s, v26.4s\n" + "fmla v22.4s, v6.4s, v17.4s\n" + "ldr q17, [x10, x13]\n" + "fmla v21.4s, v3.4s, v16.4s\n" + "fmin v29.4s, v29.4s, v27.4s\n" + "fmla v22.4s, v7.4s, v20.4s\n" + "fmla v21.4s, v7.4s, v18.4s\n" + "st1 { v29.4s }, [x17]\n" "add x12, x12, #0x10\n" - "fmla v29.4s, v7.4s, v12.4s\n" - "fmla v30.4s, v5.4s, v16.4s\n" - "fmla v31.4s, v6.4s, v15.4s\n" - "fmla v29.4s, v8.4s, v11.4s\n" - "ldr q11, [x10, x9]\n" - "fmax v29.4s, v29.4s, v19.4s\n" - "fmla v30.4s, v8.4s, v15.4s\n" - "fmla v31.4s, v8.4s, v11.4s\n" - "fmax v30.4s, v30.4s, v19.4s\n" + "fmla v28.4s, v7.4s, v23.4s\n" + "fmla v22.4s, v5.4s, v16.4s\n" + "fmla v21.4s, v6.4s, v17.4s\n" + "fmla v28.4s, v8.4s, v19.4s\n" + "ldr q16, [x10, x9]\n" + "fmax v28.4s, v28.4s, v26.4s\n" + "fmla v22.4s, v8.4s, v17.4s\n" + "fmla v21.4s, v8.4s, v16.4s\n" + "fmax v22.4s, v22.4s, v26.4s\n" "add x10, x10, #0x10\n" - "fmax v31.4s, v31.4s, v19.4s\n" - "fmin v29.4s, v29.4s, v18.4s\n" - "str q29, [x17, x7]\n" + "fmax v21.4s, v21.4s, v26.4s\n" + "fmin v28.4s, v28.4s, v27.4s\n" + "str q28, [x17, x7]\n" "add x17, x17, #0x10\n" - "fmin v30.4s, v30.4s, v18.4s\n" - "fmin v31.4s, v31.4s, v18.4s\n" - "st1 { v30.4s }, [x28]\n" - "str q31, [x28, x7]\n" + "fmin v22.4s, v22.4s, v27.4s\n" + "fmin v21.4s, v21.4s, v27.4s\n" + "st1 { v22.4s }, [x28]\n" + "str q21, [x28, x7]\n" "add x28, x28, #0x10\n" "4:" // Tile loop: Oddments "tst %x[n_channels], #0x3\n" "beq 43f\n" - "ldr q17, [x15, #0x0]\n" + "ldr q31, [x15, #0x0]\n" "ldr q0, [x15, #0x10]\n" "add x27, x14, x13\n" "add x26, x8, XZR\n" @@ -369,17 +369,17 @@ void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "ldr s15, [x21, #0x0]\n" "ldr s16, [x20, #0x0]\n" "6:" // Tile loop: Oddments: Load inputs: (2, 2), (0, 0), (0, 1), (0, 3), (0, 4), (1, 0), (1, 1), (0, 2): Bit 1: End - "mov v28.16b, v17.16b\n fmla v28.4s, v8.4s, v9.4s\n" + "mov v28.16b, v31.16b\n fmla v28.4s, v8.4s, v9.4s\n" "fmla v28.4s, v0.4s, v10.4s\n" "add x20, x16, x11\n" - "mov v29.16b, v17.16b\n fmla v29.4s, v6.4s, v9.4s\n" + "mov v29.16b, v31.16b\n fmla v29.4s, v6.4s, v9.4s\n" "fmla v28.4s, v1.4s, v11.4s\n" "fmla v29.4s, v1.4s, v12.4s\n" "fmla v28.4s, v3.4s, v14.4s\n" "fmla v29.4s, v2.4s, v13.4s\n" "fmla v28.4s, v4.4s, v15.4s\n" - "mov v30.16b, v17.16b\n fmla v30.4s, v2.4s, v9.4s\n" - "mov v31.16b, v17.16b\n fmla v31.4s, v0.4s, v9.4s\n" + "mov v30.16b, v31.16b\n fmla v30.4s, v2.4s, v9.4s\n" + "fmla v31.4s, v0.4s, v9.4s\n" "fmla v28.4s, v2.4s, v16.4s\n" "fmla v29.4s, v0.4s, v16.4s\n" "tbz %x[n_channels], #1, 7f\n" @@ -558,14 +558,14 @@ void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "ldr s11, [x20, #0x0]\n" "40:" // Tile loop: Oddments: Load inputs: (4, 4): Bit 1: End "fmla v31.4s, v8.4s, v11.4s\n" - "fmax v28.4s, v28.4s, v19.4s\n" - "fmax v29.4s, v29.4s, v19.4s\n" - "fmax v30.4s, v30.4s, v19.4s\n" - "fmax v31.4s, v31.4s, v19.4s\n" - "fmin v28.4s, v28.4s, v18.4s\n" - "fmin v29.4s, v29.4s, v18.4s\n" - "fmin v30.4s, v30.4s, v18.4s\n" - "fmin v31.4s, v31.4s, v18.4s\n" + "fmax v28.4s, v28.4s, v26.4s\n" + "fmax v29.4s, v29.4s, v26.4s\n" + "fmax v30.4s, v30.4s, v26.4s\n" + "fmax v31.4s, v31.4s, v26.4s\n" + "fmin v28.4s, v28.4s, v27.4s\n" + "fmin v29.4s, v29.4s, v27.4s\n" + "fmin v30.4s, v30.4s, v27.4s\n" + "fmin v31.4s, v31.4s, v27.4s\n" "tbz %x[n_channels], #1, 41f\n" "mov x21, x17\n" "mov x20, x28\n" @@ -591,7 +591,6 @@ void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "st1 { v29.s }[0], [x21]\n" "st1 { v31.s }[0], [x20]\n" "42:" // Tile loop: Oddments: Store: Bit 1: End - "43:" // Tile loop: End "ldr x27, [%x[params_struct], %[offsetof_args_tile_j]]\n" "ldr x23, [%x[params_struct], %[offsetof_args_tile_i]]\n" @@ -606,11 +605,11 @@ void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v28", "v29", "v30", "v31", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp index f65633002e8ea89413691f688e8d78fbb5c3702c..24fe255dfb3e3c5fdf7be1f8fbad25cf6b4b969a 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ +#if defined(__aarch64__) namespace arm_conv { namespace depthwise { @@ -88,258 +88,258 @@ void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( __asm__ __volatile__( "ldr x21, [%x[params_struct], %[offsetof_args_outptrs]]\n" - "mov x26, #0x10\n" // cntb _, ALL, #1 - "lsr x25, %x[n_channels], #0x2\n" - "ldr x24, [%x[params_struct], %[offsetof_args_params]]\n" + "mov x25, #0x10\n" // cntb _, ALL, #1 + "lsr x24, %x[n_channels], #0x2\n" + "ldr x23, [%x[params_struct], %[offsetof_args_params]]\n" "add x20, %x[params_struct], %[offsetof_args_min]\n" - "ld1r { v19.4s }, [x20]\n" + "ld1r { v26.4s }, [x20]\n" "add x20, %x[params_struct], %[offsetof_args_max]\n" - "ld1r { v18.4s }, [x20]\n" + "ld1r { v27.4s }, [x20]\n" "add x13, %x[params_struct], %[offsetof_Args_inptrs]\n" "ldp x12, x11, [x21, #0x0]\n" "ldp x10, x9, [x21, #0x10]\n" "mov x28, #0x0\n" - "sub x23, XZR, x26\n" - "cbz x25, 3f\n" - "ldr q17, [x24, #0x0]\n" - "ldr q0, [x24, #0x10]\n" - "cmp x26, x25, LSL #4\n" - "ldr q1, [x24, #0x20]\n" - "ldr q2, [x24, #0x30]\n" - "ldr q3, [x24, #0x40]\n" - "ldr q4, [x24, #0x50]\n" - "ldr q5, [x24, #0x60]\n" - "ldr q6, [x24, #0x70]\n" - "ldr q7, [x24, #0x80]\n" - "ldr q8, [x24, #0x90]\n" - "add x24, x24, #0xa0\n" - "ldp x22, x20, [x13, #0x0]\n" - "ldr q9, [x22, x28]\n" + "sub x22, XZR, x25\n" + "cbz x24, 3f\n" + "ldr q31, [x23, #0x0]\n" + "ldr q0, [x23, #0x10]\n" + "cmp x25, x24, LSL #4\n" + "ldr q1, [x23, #0x20]\n" + "ldr q2, [x23, #0x30]\n" + "ldr q3, [x23, #0x40]\n" + "ldr q4, [x23, #0x50]\n" + "ldr q5, [x23, #0x60]\n" + "ldr q6, [x23, #0x70]\n" + "ldr q7, [x23, #0x80]\n" + "ldr q8, [x23, #0x90]\n" + "add x23, x23, #0xa0\n" + "ldp x21, x20, [x13, #0x0]\n" + "ldr q9, [x21, x28]\n" "ldr q10, [x20, x28]\n" "ldp x21, x20, [x13, #0x10]\n" "ldr q11, [x21, x28]\n" "ldr q12, [x20, x28]\n" - "ldp x22, x21, [x13, #0x20]\n" - "ldr q13, [x22, x28]\n" - "ldr q14, [x21, x28]\n" + "ldp x21, x20, [x13, #0x20]\n" + "ldr q13, [x21, x28]\n" + "ldr q14, [x20, x28]\n" "ldp x21, x20, [x13, #0x30]\n" "ldr q15, [x21, x28]\n" "ldr q16, [x20, x28]\n" "bge 2f\n" "1:" // Channel loop - "mov v28.16b, v17.16b\n fmla v28.4s, v8.4s, v9.4s\n" - "mov v29.16b, v17.16b\n fmla v29.4s, v6.4s, v9.4s\n" - "ldr x22, [x13, #0x40]\n" + "mov v24.16b, v31.16b\n fmla v24.4s, v8.4s, v9.4s\n" + "mov v23.16b, v31.16b\n fmla v23.4s, v6.4s, v9.4s\n" + "ldr x21, [x13, #0x40]\n" "ldr x20, [x13, #0x48]\n" - "fmla v28.4s, v0.4s, v10.4s\n" - "fmla v29.4s, v1.4s, v12.4s\n" - "ldr q12, [x20, x28]\n" - "ldr x21, [x13, #0x50]\n" - "fmla v28.4s, v1.4s, v11.4s\n" - "ldr q11, [x22, x28]\n" - "fmla v29.4s, v2.4s, v13.4s\n" - "ldr q13, [x21, x28]\n" - "fmla v28.4s, v3.4s, v14.4s\n" - "fmla v29.4s, v0.4s, v16.4s\n" + "fmla v24.4s, v0.4s, v10.4s\n" + "fmla v23.4s, v1.4s, v12.4s\n" + "ldr q20, [x20, x28]\n" + "ldr x20, [x13, #0x50]\n" + "fmla v24.4s, v1.4s, v11.4s\n" + "ldr q19, [x21, x28]\n" + "fmla v23.4s, v2.4s, v13.4s\n" + "ldr q18, [x20, x28]\n" + "fmla v24.4s, v3.4s, v14.4s\n" + "fmla v23.4s, v0.4s, v16.4s\n" "ldr x20, [x13, #0x58]\n" - "ldr q14, [x20, x28]\n" - "fmla v28.4s, v4.4s, v15.4s\n" - "fmla v29.4s, v4.4s, v11.4s\n" - "ldr x20, [x13, #0x78]\n" - "ldr x22, [x13, #0x60]\n" - "ldr q15, [x22, x28]\n" - "fmla v28.4s, v2.4s, v16.4s\n" - "fmla v29.4s, v5.4s, v12.4s\n" - "ldr x22, [x13, #0x80]\n" - "ldr q12, [x22, x28]\n" - "mov v30.16b, v17.16b\n fmla v30.4s, v2.4s, v9.4s\n" - "mov v31.16b, v17.16b\n fmla v31.4s, v0.4s, v9.4s\n" - "ldr q17, [x24, #0x0]\n" - "fmla v28.4s, v5.4s, v13.4s\n" - "fmla v29.4s, v3.4s, v13.4s\n" - "ldr q13, [x20, x28]\n" - "ldr x21, [x13, #0x68]\n" - "ldr q11, [x21, x28]\n" - "fmla v30.4s, v3.4s, v14.4s\n" - "fmla v31.4s, v4.4s, v13.4s\n" - "ldr x20, [x13, #0x88]\n" - "ldr q14, [x20, x28]\n" - "fmla v30.4s, v0.4s, v15.4s\n" - "ldr q0, [x24, #0x10]\n" - "fmla v31.4s, v1.4s, v12.4s\n" - "ldr x21, [x13, #0x70]\n" + "ldr q17, [x20, x28]\n" + "fmla v24.4s, v4.4s, v15.4s\n" + "fmla v23.4s, v4.4s, v19.4s\n" + "ldr x21, [x13, #0x78]\n" + "ldr x20, [x13, #0x60]\n" + "ldr q22, [x20, x28]\n" + "fmla v24.4s, v2.4s, v16.4s\n" + "fmla v23.4s, v5.4s, v20.4s\n" + "ldr x20, [x13, #0x80]\n" + "ldr q21, [x20, x28]\n" + "mov v20.16b, v31.16b\n fmla v20.4s, v2.4s, v9.4s\n" + "mov v19.16b, v31.16b\n fmla v19.4s, v0.4s, v9.4s\n" + "ldr q31, [x23, #0x0]\n" + "fmla v24.4s, v5.4s, v18.4s\n" + "fmla v23.4s, v3.4s, v18.4s\n" "ldr q16, [x21, x28]\n" - "fmla v30.4s, v4.4s, v11.4s\n" - "fmla v31.4s, v5.4s, v14.4s\n" - "ldr q4, [x24, #0x50]\n" + "ldr x20, [x13, #0x68]\n" + "ldr q18, [x20, x28]\n" + "fmla v20.4s, v3.4s, v17.4s\n" + "fmla v19.4s, v4.4s, v16.4s\n" + "ldr x20, [x13, #0x88]\n" + "ldr q16, [x20, x28]\n" + "fmla v20.4s, v0.4s, v22.4s\n" + "ldr q0, [x23, #0x10]\n" + "fmla v19.4s, v1.4s, v21.4s\n" + "ldr x20, [x13, #0x70]\n" + "ldr q17, [x20, x28]\n" + "fmla v20.4s, v4.4s, v18.4s\n" + "fmla v19.4s, v5.4s, v16.4s\n" + "ldr q4, [x23, #0x50]\n" "ldr x20, [x13, #0x98]\n" - "fmla v28.4s, v6.4s, v15.4s\n" - "fmla v30.4s, v1.4s, v16.4s\n" - "ldr q11, [x20, x28]\n" - "ldr q1, [x24, #0x20]\n" - "fmla v31.4s, v2.4s, v11.4s\n" - "fmla v28.4s, v7.4s, v16.4s\n" - "ldr q2, [x24, #0x30]\n" - "ldr x21, [x13, #0x90]\n" - "fmla v29.4s, v7.4s, v12.4s\n" - "fmla v29.4s, v8.4s, v11.4s\n" - "ldr q15, [x21, x28]\n" - "ldr x21, [x13, #0xa8]\n" - "fmla v30.4s, v6.4s, v15.4s\n" - "fmax v28.4s, v28.4s, v19.4s\n" - "ldr q16, [x21, x28]\n" - "ldr x22, [x13, #0xa0]\n" - "fmla v31.4s, v3.4s, v16.4s\n" - "fmax v29.4s, v29.4s, v19.4s\n" - "ldr q13, [x22, x28]\n" - "ldr q3, [x24, #0x40]\n" - "fmla v30.4s, v7.4s, v13.4s\n" - "fmla v30.4s, v5.4s, v16.4s\n" - "ldr q5, [x24, #0x60]\n" - "ldr x21, [x13, #0xb0]\n" - "add x23, x23, #0x10\n" - "fmin v28.4s, v28.4s, v18.4s\n" - "ldr q14, [x21, x28]\n" + "fmla v24.4s, v6.4s, v22.4s\n" + "fmla v20.4s, v1.4s, v17.4s\n" + "ldr q16, [x20, x28]\n" + "ldr q1, [x23, #0x20]\n" + "fmla v19.4s, v2.4s, v16.4s\n" + "fmla v24.4s, v7.4s, v17.4s\n" + "ldr q2, [x23, #0x30]\n" + "ldr x20, [x13, #0x90]\n" + "fmla v23.4s, v7.4s, v21.4s\n" + "fmla v23.4s, v8.4s, v16.4s\n" + "ldr q16, [x20, x28]\n" + "ldr x20, [x13, #0xa8]\n" + "fmla v20.4s, v6.4s, v16.4s\n" + "fmax v24.4s, v24.4s, v26.4s\n" + "ldr q17, [x20, x28]\n" + "ldr x20, [x13, #0xa0]\n" + "fmla v19.4s, v3.4s, v17.4s\n" + "fmax v23.4s, v23.4s, v26.4s\n" + "ldr q16, [x20, x28]\n" + "ldr q3, [x23, #0x40]\n" + "fmla v20.4s, v7.4s, v16.4s\n" + "fmla v20.4s, v5.4s, v17.4s\n" + "ldr q5, [x23, #0x60]\n" + "ldr x20, [x13, #0xb0]\n" + "add x22, x22, #0x10\n" + "fmin v24.4s, v24.4s, v27.4s\n" + "ldr q16, [x20, x28]\n" "ldr x20, [x13, #0xb8]\n" - "fmla v31.4s, v7.4s, v14.4s\n" - "fmin v29.4s, v29.4s, v18.4s\n" - "ldr q15, [x20, x28]\n" - "ldr q7, [x24, #0x80]\n" - "fmla v31.4s, v6.4s, v15.4s\n" - "fmla v30.4s, v8.4s, v15.4s\n" - "ldr q6, [x24, #0x70]\n" - "ldr x22, [x13, #0xc0]\n" - "fmax v30.4s, v30.4s, v19.4s\n" - "fmin v30.4s, v30.4s, v18.4s\n" - "ldr q11, [x22, x28]\n" - "fmla v31.4s, v8.4s, v11.4s\n" - "ldr q8, [x24, #0x90]\n" - "fmax v31.4s, v31.4s, v19.4s\n" - "ldp x22, x20, [x13, #0x0]\n" - "ldr q9, [x22, x26]\n" - "fmin v31.4s, v31.4s, v18.4s\n" + "fmla v19.4s, v7.4s, v16.4s\n" + "fmin v23.4s, v23.4s, v27.4s\n" + "ldr q16, [x20, x28]\n" + "ldr q7, [x23, #0x80]\n" + "fmla v19.4s, v6.4s, v16.4s\n" + "fmla v20.4s, v8.4s, v16.4s\n" + "ldr q6, [x23, #0x70]\n" + "ldr x20, [x13, #0xc0]\n" + "fmax v20.4s, v20.4s, v26.4s\n" + "fmin v20.4s, v20.4s, v27.4s\n" + "ldr q16, [x20, x28]\n" + "fmla v19.4s, v8.4s, v16.4s\n" + "ldr q8, [x23, #0x90]\n" + "fmax v19.4s, v19.4s, v26.4s\n" + "ldp x21, x20, [x13, #0x0]\n" + "ldr q9, [x21, x25]\n" + "fmin v19.4s, v19.4s, v27.4s\n" "add x28, x28, #0x10\n" - "ldr q10, [x20, x26]\n" + "ldr q10, [x20, x25]\n" "ldp x21, x20, [x13, #0x10]\n" - "str q28, [x12, x23]\n" - "add x24, x24, #0xa0\n" - "ldr q11, [x21, x26]\n" - "ldr q12, [x20, x26]\n" - "str q29, [x11, x23]\n" - "ldp x22, x21, [x13, #0x20]\n" - "ldr q13, [x22, x26]\n" - "str q30, [x10, x23]\n" - "ldr q14, [x21, x26]\n" + "str q24, [x12, x22]\n" + "add x23, x23, #0xa0\n" + "ldr q11, [x21, x25]\n" + "ldr q12, [x20, x25]\n" + "str q23, [x11, x22]\n" + "ldp x21, x20, [x13, #0x20]\n" + "ldr q13, [x21, x25]\n" + "str q20, [x10, x22]\n" + "ldr q14, [x20, x25]\n" "ldp x21, x20, [x13, #0x30]\n" - "str q31, [x9, x23]\n" - "ldr q15, [x21, x26]\n" - "ldr q16, [x20, x26]\n" - "add x26, x26, #0x10\n" - "cmp x26, x25, LSL #4\n" + "str q19, [x9, x22]\n" + "ldr q15, [x21, x25]\n" + "ldr q16, [x20, x25]\n" + "add x25, x25, #0x10\n" + "cmp x25, x24, LSL #4\n" "blt 1b\n" "2:" // Channel tail - "mov v28.16b, v17.16b\n fmla v28.4s, v8.4s, v9.4s\n" - "mov v29.16b, v17.16b\n fmla v29.4s, v6.4s, v9.4s\n" - "ldr x22, [x13, #0x40]\n" + "mov v25.16b, v31.16b\n fmla v25.4s, v8.4s, v9.4s\n" + "mov v24.16b, v31.16b\n fmla v24.4s, v6.4s, v9.4s\n" + "ldr x21, [x13, #0x40]\n" "ldr x20, [x13, #0x48]\n" - "fmla v28.4s, v0.4s, v10.4s\n" - "fmla v29.4s, v1.4s, v12.4s\n" - "ldr q12, [x20, x28]\n" - "ldr x21, [x13, #0x50]\n" - "fmla v28.4s, v1.4s, v11.4s\n" - "ldr q11, [x22, x28]\n" - "fmla v29.4s, v2.4s, v13.4s\n" - "ldr q13, [x21, x28]\n" - "fmla v28.4s, v3.4s, v14.4s\n" - "fmla v29.4s, v0.4s, v16.4s\n" + "fmla v25.4s, v0.4s, v10.4s\n" + "fmla v24.4s, v1.4s, v12.4s\n" + "ldr q20, [x20, x28]\n" + "ldr x20, [x13, #0x50]\n" + "fmla v25.4s, v1.4s, v11.4s\n" + "ldr q18, [x21, x28]\n" + "fmla v24.4s, v2.4s, v13.4s\n" + "ldr q19, [x20, x28]\n" + "fmla v25.4s, v3.4s, v14.4s\n" + "fmla v24.4s, v0.4s, v16.4s\n" "ldr x20, [x13, #0x58]\n" - "ldr q14, [x20, x28]\n" - "fmla v28.4s, v4.4s, v15.4s\n" - "fmla v29.4s, v4.4s, v11.4s\n" - "ldr x20, [x13, #0x78]\n" - "ldr x22, [x13, #0x60]\n" - "ldr q15, [x22, x28]\n" - "fmla v28.4s, v2.4s, v16.4s\n" - "fmla v29.4s, v5.4s, v12.4s\n" - "ldr x22, [x13, #0x80]\n" - "ldr q12, [x22, x28]\n" - "mov v30.16b, v17.16b\n fmla v30.4s, v2.4s, v9.4s\n" - "mov v31.16b, v17.16b\n fmla v31.4s, v0.4s, v9.4s\n" - "ldr x21, [x13, #0x68]\n" - "ldr q11, [x21, x28]\n" - "fmla v28.4s, v5.4s, v13.4s\n" - "fmla v29.4s, v3.4s, v13.4s\n" - "ldr q13, [x20, x28]\n" - "fmla v30.4s, v3.4s, v14.4s\n" - "fmla v31.4s, v4.4s, v13.4s\n" - "ldr x20, [x13, #0x88]\n" - "ldr q14, [x20, x28]\n" - "fmla v30.4s, v0.4s, v15.4s\n" - "fmla v31.4s, v1.4s, v12.4s\n" - "ldr x21, [x13, #0x70]\n" + "ldr q17, [x20, x28]\n" + "fmla v25.4s, v4.4s, v15.4s\n" + "fmla v24.4s, v4.4s, v18.4s\n" + "ldr x21, [x13, #0x78]\n" + "ldr x20, [x13, #0x60]\n" + "ldr q23, [x20, x28]\n" + "fmla v25.4s, v2.4s, v16.4s\n" + "fmla v24.4s, v5.4s, v20.4s\n" + "ldr x20, [x13, #0x80]\n" + "ldr q22, [x20, x28]\n" + "mov v21.16b, v31.16b\n fmla v21.4s, v2.4s, v9.4s\n" + "mov v20.16b, v31.16b\n fmla v20.4s, v0.4s, v9.4s\n" + "ldr x20, [x13, #0x68]\n" + "ldr q18, [x20, x28]\n" + "fmla v25.4s, v5.4s, v19.4s\n" + "fmla v24.4s, v3.4s, v19.4s\n" "ldr q16, [x21, x28]\n" + "fmla v21.4s, v3.4s, v17.4s\n" + "fmla v20.4s, v4.4s, v16.4s\n" + "ldr x20, [x13, #0x88]\n" + "ldr q16, [x20, x28]\n" + "fmla v21.4s, v0.4s, v23.4s\n" + "fmla v20.4s, v1.4s, v22.4s\n" + "ldr x20, [x13, #0x70]\n" + "ldr q17, [x20, x28]\n" "ldr x20, [x13, #0x98]\n" - "fmla v30.4s, v4.4s, v11.4s\n" - "ldr q11, [x20, x28]\n" - "fmla v31.4s, v5.4s, v14.4s\n" - "fmla v28.4s, v6.4s, v15.4s\n" - "ldr x21, [x13, #0x90]\n" - "ldr q15, [x21, x28]\n" - "fmla v30.4s, v1.4s, v16.4s\n" - "ldr x21, [x13, #0xa8]\n" - "fmla v31.4s, v2.4s, v11.4s\n" - "fmla v28.4s, v7.4s, v16.4s\n" - "ldr q16, [x21, x28]\n" - "ldr x22, [x13, #0xa0]\n" - "ldr q13, [x22, x28]\n" - "fmla v30.4s, v6.4s, v15.4s\n" - "fmla v31.4s, v3.4s, v16.4s\n" - "ldr x21, [x13, #0xb0]\n" - "ldr q14, [x21, x28]\n" - "fmla v30.4s, v7.4s, v13.4s\n" - "fmla v31.4s, v7.4s, v14.4s\n" + "fmla v21.4s, v4.4s, v18.4s\n" + "ldr q19, [x20, x28]\n" + "fmla v20.4s, v5.4s, v16.4s\n" + "fmla v25.4s, v6.4s, v23.4s\n" + "ldr x20, [x13, #0x90]\n" + "ldr q16, [x20, x28]\n" + "fmla v21.4s, v1.4s, v17.4s\n" + "ldr x20, [x13, #0xa8]\n" + "fmla v20.4s, v2.4s, v19.4s\n" + "fmla v25.4s, v7.4s, v17.4s\n" + "ldr q18, [x20, x28]\n" + "ldr x20, [x13, #0xa0]\n" + "ldr q17, [x20, x28]\n" + "fmla v21.4s, v6.4s, v16.4s\n" + "fmla v20.4s, v3.4s, v18.4s\n" + "ldr x20, [x13, #0xb0]\n" + "ldr q16, [x20, x28]\n" + "fmla v21.4s, v7.4s, v17.4s\n" + "fmla v20.4s, v7.4s, v16.4s\n" "ldr x20, [x13, #0xb8]\n" - "ldr q15, [x20, x28]\n" - "fmla v29.4s, v7.4s, v12.4s\n" - "fmla v30.4s, v5.4s, v16.4s\n" - "ldr x22, [x13, #0xc0]\n" - "fmla v31.4s, v6.4s, v15.4s\n" - "fmla v29.4s, v8.4s, v11.4s\n" - "ldr q11, [x22, x28]\n" - "fmla v30.4s, v8.4s, v15.4s\n" - "fmla v31.4s, v8.4s, v11.4s\n" - "fmax v28.4s, v28.4s, v19.4s\n" - "add x23, x23, #0x10\n" - "fmax v29.4s, v29.4s, v19.4s\n" - "fmax v30.4s, v30.4s, v19.4s\n" + "ldr q17, [x20, x28]\n" + "fmla v24.4s, v7.4s, v22.4s\n" + "fmla v21.4s, v5.4s, v18.4s\n" + "ldr x20, [x13, #0xc0]\n" + "fmla v20.4s, v6.4s, v17.4s\n" + "fmla v24.4s, v8.4s, v19.4s\n" + "ldr q16, [x20, x28]\n" + "fmla v21.4s, v8.4s, v17.4s\n" + "fmla v20.4s, v8.4s, v16.4s\n" + "fmax v25.4s, v25.4s, v26.4s\n" + "add x22, x22, #0x10\n" + "fmax v24.4s, v24.4s, v26.4s\n" + "fmax v21.4s, v21.4s, v26.4s\n" "add x28, x28, #0x10\n" - "fmax v31.4s, v31.4s, v19.4s\n" - "fmin v28.4s, v28.4s, v18.4s\n" - "str q28, [x12, x23]\n" - "fmin v29.4s, v29.4s, v18.4s\n" - "fmin v30.4s, v30.4s, v18.4s\n" - "str q29, [x11, x23]\n" - "fmin v31.4s, v31.4s, v18.4s\n" - "str q30, [x10, x23]\n" - "str q31, [x9, x23]\n" + "fmax v20.4s, v20.4s, v26.4s\n" + "fmin v25.4s, v25.4s, v27.4s\n" + "str q25, [x12, x22]\n" + "fmin v24.4s, v24.4s, v27.4s\n" + "fmin v21.4s, v21.4s, v27.4s\n" + "str q24, [x11, x22]\n" + "fmin v20.4s, v20.4s, v27.4s\n" + "str q21, [x10, x22]\n" + "str q20, [x9, x22]\n" "3:" // Oddments "tst %x[n_channels], #0x3\n" "beq 42f\n" - "ldr q17, [x24, #0x0]\n" - "ldr q0, [x24, #0x10]\n" - "mov x23, x28\n" - "add x12, x12, x23\n" - "ldr q1, [x24, #0x20]\n" - "ldr q2, [x24, #0x30]\n" - "add x11, x11, x23\n" - "add x10, x10, x23\n" - "ldr q3, [x24, #0x40]\n" - "ldr q4, [x24, #0x50]\n" - "add x9, x9, x23\n" - "ldr q5, [x24, #0x60]\n" - "ldr q6, [x24, #0x70]\n" - "ldr q7, [x24, #0x80]\n" - "ldr q8, [x24, #0x90]\n" + "ldr q31, [x23, #0x0]\n" + "ldr q0, [x23, #0x10]\n" + "mov x20, x28\n" + "add x12, x12, x20\n" + "ldr q1, [x23, #0x20]\n" + "ldr q2, [x23, #0x30]\n" + "add x11, x11, x20\n" + "add x10, x10, x20\n" + "ldr q3, [x23, #0x40]\n" + "ldr q4, [x23, #0x50]\n" + "add x9, x9, x20\n" + "ldr q5, [x23, #0x60]\n" + "ldr q6, [x23, #0x70]\n" + "ldr q7, [x23, #0x80]\n" + "ldr q8, [x23, #0x90]\n" "ldr x27, [x13, #0x0]\n" "ldr x26, [x13, #0x8]\n" "add x27, x27, x28\n" @@ -385,18 +385,18 @@ void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( "ld1 { v15.s }[0], [x21], #0x4\n" "ld1 { v16.s }[0], [x20], #0x4\n" "5:" // Oddments: Load inputs (2, 2), (0, 0), (0, 1), (0, 3), (0, 4), (1, 0), (1, 1), (0, 2): Bit 1: End - "mov v28.16b, v17.16b\n fmla v28.4s, v8.4s, v9.4s\n" + "mov v28.16b, v31.16b\n fmla v28.4s, v8.4s, v9.4s\n" "fmla v28.4s, v0.4s, v10.4s\n" "ldr x20, [x13, #0x40]\n" "add x20, x20, x28\n" - "mov v29.16b, v17.16b\n fmla v29.4s, v6.4s, v9.4s\n" + "mov v29.16b, v31.16b\n fmla v29.4s, v6.4s, v9.4s\n" "fmla v28.4s, v1.4s, v11.4s\n" "fmla v29.4s, v1.4s, v12.4s\n" "fmla v28.4s, v3.4s, v14.4s\n" "fmla v29.4s, v2.4s, v13.4s\n" "fmla v28.4s, v4.4s, v15.4s\n" - "mov v30.16b, v17.16b\n fmla v30.4s, v2.4s, v9.4s\n" - "mov v31.16b, v17.16b\n fmla v31.4s, v0.4s, v9.4s\n" + "mov v30.16b, v31.16b\n fmla v30.4s, v2.4s, v9.4s\n" + "fmla v31.4s, v0.4s, v9.4s\n" "fmla v28.4s, v2.4s, v16.4s\n" "fmla v29.4s, v0.4s, v16.4s\n" "tbz %x[n_channels], #1, 6f\n" @@ -591,14 +591,14 @@ void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( "ld1 { v11.s }[0], [x20], #0x4\n" "39:" // Oddments: Load input (4, 4): Bit 1: End "fmla v31.4s, v8.4s, v11.4s\n" - "fmax v28.4s, v28.4s, v19.4s\n" - "fmax v29.4s, v29.4s, v19.4s\n" - "fmax v30.4s, v30.4s, v19.4s\n" - "fmax v31.4s, v31.4s, v19.4s\n" - "fmin v28.4s, v28.4s, v18.4s\n" - "fmin v29.4s, v29.4s, v18.4s\n" - "fmin v30.4s, v30.4s, v18.4s\n" - "fmin v31.4s, v31.4s, v18.4s\n" + "fmax v28.4s, v28.4s, v26.4s\n" + "fmax v29.4s, v29.4s, v26.4s\n" + "fmax v30.4s, v30.4s, v26.4s\n" + "fmax v31.4s, v31.4s, v26.4s\n" + "fmin v28.4s, v28.4s, v27.4s\n" + "fmin v29.4s, v29.4s, v27.4s\n" + "fmin v30.4s, v30.4s, v27.4s\n" + "fmin v31.4s, v31.4s, v27.4s\n" "tbz %x[n_channels], #1, 40f\n" "st1 { v28.d }[0], [x12], #0x8\n" "st1 { v29.d }[0], [x11], #0x8\n" @@ -619,11 +619,11 @@ void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( "42:" // End : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp index c88a7d57ced92544e29b1489969cc70b966d90c6..de8a1e45142279bdcb079de4993feb2bb3494701 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include @@ -33,8 +33,8 @@ namespace arm_conv { namespace depthwise { -void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); -void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); +void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(const float *const *const input_ptrs, float *const *const outptrs, const void *params, unsigned int n_channels, const float activation_min, const float activation_max); +void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int n_tile_rows, const unsigned int n_tile_cols, const float *inptr, int64_t ld_input_row, int64_t ld_input_col, float *outptr, int64_t ld_output_row, int64_t ld_output_col, const void *params, unsigned int n_channels, const float activation_min, const float activation_max); class a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirstStrategy { @@ -57,7 +57,7 @@ class a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirst constexpr static unsigned int output_cols = 2; a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst(const CPUInfo *) - : DepthwiseDepthfirstStrategy(2, 5, 1) {} + : Parent(output_rows, output_cols, kernel_rows, kernel_cols, stride_rows, stride_cols) {} arm_gemm::VLType get_vl_type(void) const override { return vl_type; } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp index 6ca3976f0245a696309de752383938eb47a08eb8..3426fbc3f952f9b0f1de8282de1472faab32dcef 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -120,9 +120,9 @@ void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "add x13, x15, x2\n" "add x5, x5, x20, LSL #2\n" // outptrs[0] += offset * sizeof(float) "add x20, %x[params_struct], %[offsetof_args_min]\n" - "ld1r { v18.4s }, [x20]\n" + "ld1r { v27.4s }, [x20]\n" "add x20, %x[params_struct], %[offsetof_args_max]\n" - "ld1r { v17.4s }, [x20]\n" + "ld1r { v15.4s }, [x20]\n" "add x12, x14, x24, LSL #2\n" "add x11, x13, x2\n" "add x10, x5, x21, LSL #2\n" @@ -130,7 +130,7 @@ void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "mov x21, #0x0\n" "sub x20, XZR, x23\n" "cbz x22, 4f\n" - "ldr q16, [x8, #0x0]\n" + "ldr q25, [x8, #0x0]\n" "ldr q0, [x8, #0x10]\n" "cmp x23, x22, LSL #4\n" "ldr q1, [x8, #0x20]\n" @@ -150,366 +150,366 @@ void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "ld1 { v14.4s }, [x17]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "mov v28.16b, v16.16b\n fmla v28.4s, v0.4s, v5.4s\n" - "ldr q5, [x7, x15]\n" - "mov v29.16b, v16.16b\n fmla v29.4s, v0.4s, v6.4s\n" + "mov v30.16b, v25.16b\n fmla v30.4s, v0.4s, v5.4s\n" + "ldr q23, [x7, x15]\n" + "mov v31.16b, v25.16b\n fmla v31.4s, v0.4s, v6.4s\n" "add x23, x23, #0x10\n" - "mov v30.16b, v16.16b\n fmla v30.4s, v0.4s, v7.4s\n" - "mov v31.16b, v16.16b\n fmla v31.4s, v0.4s, v8.4s\n" - "ldr q0, [x8, #0x0]\n" - "ldr q16, [x8, #0x140]\n" - "fmla v28.4s, v1.4s, v6.4s\n" - "ldr q6, [x7, x13]\n" - "fmla v29.4s, v1.4s, v9.4s\n" + "mov v29.16b, v25.16b\n fmla v29.4s, v0.4s, v7.4s\n" + "mov v28.16b, v25.16b\n fmla v28.4s, v0.4s, v8.4s\n" + "ldr q19, [x8, #0x0]\n" + "ldr q25, [x8, #0x140]\n" + "fmla v30.4s, v1.4s, v6.4s\n" + "ldr q21, [x7, x13]\n" + "fmla v31.4s, v1.4s, v9.4s\n" "add x7, x7, #0x10\n" - "fmla v30.4s, v1.4s, v8.4s\n" - "fmla v31.4s, v1.4s, v13.4s\n" + "fmla v29.4s, v1.4s, v8.4s\n" + "fmla v28.4s, v1.4s, v13.4s\n" "ldr q1, [x8, #0x10]\n" "cmp x23, x22, LSL #4\n" - "fmla v28.4s, v2.4s, v9.4s\n" - "ldr q9, [x4, x11]\n" - "fmla v29.4s, v2.4s, v11.4s\n" + "fmla v30.4s, v2.4s, v9.4s\n" + "ldr q18, [x4, x11]\n" + "fmla v31.4s, v2.4s, v11.4s\n" "add x4, x4, #0x10\n" - "fmla v30.4s, v2.4s, v13.4s\n" - "fmla v31.4s, v2.4s, v5.4s\n" - "ldr q2, [x8, #0x20]\n" + "fmla v29.4s, v2.4s, v13.4s\n" + "fmla v28.4s, v2.4s, v23.4s\n" + "ldr q17, [x8, #0x20]\n" "add x20, x20, #0x10\n" - "fmla v28.4s, v3.4s, v11.4s\n" - "ldr q11, [x17, x2]\n" - "fmla v29.4s, v3.4s, v12.4s\n" + "fmla v30.4s, v3.4s, v11.4s\n" + "ldr q6, [x17, x2]\n" + "fmla v31.4s, v3.4s, v12.4s\n" "add x21, x21, #0x10\n" - "fmla v30.4s, v3.4s, v5.4s\n" - "fmla v31.4s, v3.4s, v6.4s\n" - "ldr q3, [x8, #0x30]\n" - "fmla v28.4s, v4.4s, v12.4s\n" - "ldr q12, [x17, x6]\n" - "fmla v29.4s, v4.4s, v9.4s\n" - "ldr q9, [x17, x15]\n" - "fmla v30.4s, v4.4s, v6.4s\n" - "fmla v31.4s, v4.4s, v10.4s\n" - "ldr q4, [x8, #0x40]\n" - "fmla v28.4s, v0.4s, v7.4s\n" + "fmla v29.4s, v3.4s, v23.4s\n" + "fmla v28.4s, v3.4s, v21.4s\n" + "ldr q16, [x8, #0x30]\n" + "fmla v30.4s, v4.4s, v12.4s\n" + "ldr q2, [x17, x6]\n" + "fmla v31.4s, v4.4s, v18.4s\n" + "ldr q0, [x17, x15]\n" + "fmla v29.4s, v4.4s, v21.4s\n" + "fmla v28.4s, v4.4s, v10.4s\n" + "ldr q20, [x8, #0x40]\n" + "fmla v30.4s, v19.4s, v7.4s\n" "ld1 { v7.4s }, [x7]\n" - "fmla v29.4s, v0.4s, v8.4s\n" - "fmla v30.4s, v0.4s, v14.4s\n" - "fmla v31.4s, v0.4s, v11.4s\n" - "ldr q0, [x8, #0x50]\n" - "fmla v28.4s, v1.4s, v8.4s\n" - "ldr q8, [x17, x11]\n" - "fmla v29.4s, v1.4s, v13.4s\n" - "fmla v30.4s, v1.4s, v11.4s\n" - "fmla v31.4s, v1.4s, v12.4s\n" - "ldr q1, [x8, #0x60]\n" - "fmla v28.4s, v2.4s, v13.4s\n" - "ldr q13, [x17, x13]\n" - "fmla v29.4s, v2.4s, v5.4s\n" + "fmla v31.4s, v19.4s, v8.4s\n" + "fmla v29.4s, v19.4s, v14.4s\n" + "fmla v28.4s, v19.4s, v6.4s\n" + "ldr q19, [x8, #0x50]\n" + "fmla v30.4s, v1.4s, v8.4s\n" + "ldr q26, [x17, x11]\n" + "fmla v31.4s, v1.4s, v13.4s\n" + "fmla v29.4s, v1.4s, v6.4s\n" + "fmla v28.4s, v1.4s, v2.4s\n" + "ldr q18, [x8, #0x60]\n" + "fmla v30.4s, v17.4s, v13.4s\n" + "ldr q1, [x17, x13]\n" + "fmla v31.4s, v17.4s, v23.4s\n" "add x17, x17, #0x10\n" - "fmla v30.4s, v2.4s, v12.4s\n" - "fmla v31.4s, v2.4s, v9.4s\n" - "ldr q2, [x8, #0x70]\n" - "fmla v28.4s, v3.4s, v5.4s\n" - "ld1 { v5.4s }, [x16]\n" - "fmla v29.4s, v3.4s, v6.4s\n" - "fmla v30.4s, v3.4s, v9.4s\n" - "fmla v31.4s, v3.4s, v13.4s\n" - "ldr q3, [x8, #0x80]\n" - "fmla v28.4s, v4.4s, v6.4s\n" - "ldr q6, [x16, x2]\n" - "fmla v29.4s, v4.4s, v10.4s\n" - "ldr q10, [x16, x6]\n" - "fmla v30.4s, v4.4s, v13.4s\n" - "fmla v31.4s, v4.4s, v8.4s\n" - "ldr q4, [x8, #0x90]\n" - "fmla v28.4s, v0.4s, v14.4s\n" - "ldr q14, [x16, x11]\n" - "fmla v29.4s, v0.4s, v11.4s\n" - "fmla v30.4s, v0.4s, v5.4s\n" - "fmla v31.4s, v0.4s, v6.4s\n" - "ldr q0, [x8, #0xa0]\n" - "fmla v28.4s, v1.4s, v11.4s\n" - "ldr q11, [x16, x15]\n" - "fmla v29.4s, v1.4s, v12.4s\n" - "fmla v30.4s, v1.4s, v6.4s\n" - "fmla v31.4s, v1.4s, v10.4s\n" - "ldr q1, [x8, #0xb0]\n" - "fmla v28.4s, v2.4s, v12.4s\n" - "ldr q12, [x16, x13]\n" - "fmla v29.4s, v2.4s, v9.4s\n" + "fmla v29.4s, v17.4s, v2.4s\n" + "fmla v28.4s, v17.4s, v0.4s\n" + "ldr q17, [x8, #0x70]\n" + "fmla v30.4s, v16.4s, v23.4s\n" + "ld1 { v24.4s }, [x16]\n" + "fmla v31.4s, v16.4s, v21.4s\n" + "fmla v29.4s, v16.4s, v0.4s\n" + "fmla v28.4s, v16.4s, v1.4s\n" + "ldr q16, [x8, #0x80]\n" + "fmla v30.4s, v20.4s, v21.4s\n" + "ldr q23, [x16, x2]\n" + "fmla v31.4s, v20.4s, v10.4s\n" + "ldr q22, [x16, x6]\n" + "fmla v29.4s, v20.4s, v1.4s\n" + "fmla v28.4s, v20.4s, v26.4s\n" + "ldr q21, [x8, #0x90]\n" + "fmla v30.4s, v19.4s, v14.4s\n" + "ldr q5, [x16, x11]\n" + "fmla v31.4s, v19.4s, v6.4s\n" + "fmla v29.4s, v19.4s, v24.4s\n" + "fmla v28.4s, v19.4s, v23.4s\n" + "ldr q11, [x8, #0xa0]\n" + "fmla v30.4s, v18.4s, v6.4s\n" + "ldr q20, [x16, x15]\n" + "fmla v31.4s, v18.4s, v2.4s\n" + "fmla v29.4s, v18.4s, v23.4s\n" + "fmla v28.4s, v18.4s, v22.4s\n" + "ldr q18, [x8, #0xb0]\n" + "fmla v30.4s, v17.4s, v2.4s\n" + "ldr q19, [x16, x13]\n" + "fmla v31.4s, v17.4s, v0.4s\n" "add x16, x16, #0x10\n" - "fmla v30.4s, v2.4s, v10.4s\n" - "fmla v31.4s, v2.4s, v11.4s\n" - "ldr q2, [x8, #0xc0]\n" - "fmla v28.4s, v3.4s, v9.4s\n" - "ld1 { v9.4s }, [x14]\n" - "fmla v29.4s, v3.4s, v13.4s\n" - "fmla v30.4s, v3.4s, v11.4s\n" - "fmla v31.4s, v3.4s, v12.4s\n" - "ldr q3, [x8, #0xd0]\n" - "fmla v28.4s, v4.4s, v13.4s\n" - "ldr q13, [x14, x2]\n" - "fmla v29.4s, v4.4s, v8.4s\n" - "ldr q8, [x14, x13]\n" - "fmla v30.4s, v4.4s, v12.4s\n" - "fmla v31.4s, v4.4s, v14.4s\n" - "ldr q4, [x8, #0xe0]\n" - "fmla v28.4s, v0.4s, v5.4s\n" - "ldr q5, [x14, x6]\n" - "fmla v29.4s, v0.4s, v6.4s\n" - "fmla v30.4s, v0.4s, v9.4s\n" - "fmla v31.4s, v0.4s, v13.4s\n" - "ldr q0, [x8, #0xf0]\n" - "fmla v28.4s, v1.4s, v6.4s\n" - "ldr q6, [x14, x15]\n" - "fmla v29.4s, v1.4s, v10.4s\n" - "fmla v30.4s, v1.4s, v13.4s\n" - "fmla v31.4s, v1.4s, v5.4s\n" - "ldr q1, [x8, #0x100]\n" - "fmla v28.4s, v2.4s, v10.4s\n" - "ldr q10, [x14, x11]\n" - "fmla v29.4s, v2.4s, v11.4s\n" + "fmla v29.4s, v17.4s, v22.4s\n" + "fmla v28.4s, v17.4s, v20.4s\n" + "ldr q17, [x8, #0xc0]\n" + "fmla v30.4s, v16.4s, v0.4s\n" + "ld1 { v0.4s }, [x14]\n" + "fmla v31.4s, v16.4s, v1.4s\n" + "fmla v29.4s, v16.4s, v20.4s\n" + "fmla v28.4s, v16.4s, v19.4s\n" + "ldr q16, [x8, #0xd0]\n" + "fmla v30.4s, v21.4s, v1.4s\n" + "ldr q4, [x14, x2]\n" + "fmla v31.4s, v21.4s, v26.4s\n" + "ldr q12, [x14, x13]\n" + "fmla v29.4s, v21.4s, v19.4s\n" + "fmla v28.4s, v21.4s, v5.4s\n" + "ldr q13, [x8, #0xe0]\n" + "fmla v30.4s, v11.4s, v24.4s\n" + "ldr q6, [x14, x6]\n" + "fmla v31.4s, v11.4s, v23.4s\n" + "fmla v29.4s, v11.4s, v0.4s\n" + "fmla v28.4s, v11.4s, v4.4s\n" + "ldr q24, [x8, #0xf0]\n" + "fmla v30.4s, v18.4s, v23.4s\n" + "ldr q26, [x14, x15]\n" + "fmla v31.4s, v18.4s, v22.4s\n" + "fmla v29.4s, v18.4s, v4.4s\n" + "fmla v28.4s, v18.4s, v6.4s\n" + "ldr q23, [x8, #0x100]\n" + "fmla v30.4s, v17.4s, v22.4s\n" + "ldr q22, [x14, x11]\n" + "fmla v31.4s, v17.4s, v20.4s\n" "add x14, x14, #0x10\n" - "fmla v30.4s, v2.4s, v5.4s\n" - "fmla v31.4s, v2.4s, v6.4s\n" - "ldr q2, [x8, #0x110]\n" - "fmla v28.4s, v3.4s, v11.4s\n" - "ld1 { v11.4s }, [x12]\n" - "fmla v29.4s, v3.4s, v12.4s\n" - "fmla v30.4s, v3.4s, v6.4s\n" - "fmla v31.4s, v3.4s, v8.4s\n" - "ldr q3, [x8, #0x120]\n" - "fmla v28.4s, v4.4s, v12.4s\n" - "ldr q12, [x12, x2]\n" - "fmla v29.4s, v4.4s, v14.4s\n" + "fmla v29.4s, v17.4s, v6.4s\n" + "fmla v28.4s, v17.4s, v26.4s\n" + "ldr q21, [x8, #0x110]\n" + "fmla v30.4s, v16.4s, v20.4s\n" + "ld1 { v18.4s }, [x12]\n" + "fmla v31.4s, v16.4s, v19.4s\n" + "fmla v29.4s, v16.4s, v26.4s\n" + "fmla v28.4s, v16.4s, v12.4s\n" + "ldr q20, [x8, #0x120]\n" + "fmla v30.4s, v13.4s, v19.4s\n" + "ldr q17, [x12, x2]\n" + "fmla v31.4s, v13.4s, v5.4s\n" "ld1 { v14.4s }, [x17]\n" - "fmla v30.4s, v4.4s, v8.4s\n" - "fmla v31.4s, v4.4s, v10.4s\n" - "ldr q4, [x8, #0x130]\n" - "fmla v28.4s, v0.4s, v9.4s\n" - "ldr q9, [x12, x6]\n" - "fmla v29.4s, v0.4s, v13.4s\n" - "fmla v30.4s, v0.4s, v11.4s\n" - "ldr q11, [x12, x15]\n" - "fmla v31.4s, v0.4s, v12.4s\n" + "fmla v29.4s, v13.4s, v12.4s\n" + "fmla v28.4s, v13.4s, v22.4s\n" + "ldr q19, [x8, #0x130]\n" + "fmla v30.4s, v24.4s, v0.4s\n" + "ldr q16, [x12, x6]\n" + "fmla v31.4s, v24.4s, v4.4s\n" + "fmla v29.4s, v24.4s, v18.4s\n" + "ldr q18, [x12, x15]\n" + "fmla v28.4s, v24.4s, v17.4s\n" "ldr q0, [x8, #0x150]\n" - "fmla v28.4s, v1.4s, v13.4s\n" + "fmla v30.4s, v23.4s, v4.4s\n" "ldr q13, [x7, x6]\n" - "fmla v29.4s, v1.4s, v5.4s\n" - "fmla v30.4s, v1.4s, v12.4s\n" - "ldr q12, [x12, x13]\n" - "fmla v31.4s, v1.4s, v9.4s\n" + "fmla v31.4s, v23.4s, v6.4s\n" + "fmla v29.4s, v23.4s, v17.4s\n" + "ldr q17, [x12, x13]\n" + "fmla v28.4s, v23.4s, v16.4s\n" "ldr q1, [x8, #0x160]\n" - "fmla v28.4s, v2.4s, v5.4s\n" + "fmla v30.4s, v21.4s, v6.4s\n" "ld1 { v5.4s }, [x4]\n" - "fmla v29.4s, v2.4s, v6.4s\n" - "fmla v30.4s, v2.4s, v9.4s\n" - "ldr q9, [x12, x11]\n" - "fmla v31.4s, v2.4s, v11.4s\n" + "fmla v31.4s, v21.4s, v26.4s\n" + "fmla v29.4s, v21.4s, v16.4s\n" + "ldr q16, [x12, x11]\n" + "fmla v28.4s, v21.4s, v18.4s\n" "ldr q2, [x8, #0x170]\n" - "fmla v28.4s, v3.4s, v6.4s\n" + "fmla v30.4s, v20.4s, v26.4s\n" "ldr q6, [x4, x2]\n" - "fmla v29.4s, v3.4s, v8.4s\n" + "fmla v31.4s, v20.4s, v12.4s\n" "add x12, x12, #0x10\n" - "fmla v30.4s, v3.4s, v11.4s\n" + "fmla v29.4s, v20.4s, v18.4s\n" "ldr q11, [x4, x15]\n" - "fmla v31.4s, v3.4s, v12.4s\n" + "fmla v28.4s, v20.4s, v17.4s\n" "ldr q3, [x8, #0x180]\n" - "fmla v28.4s, v4.4s, v8.4s\n" + "fmla v30.4s, v19.4s, v12.4s\n" "ldr q8, [x7, x2]\n" - "fmla v29.4s, v4.4s, v10.4s\n" + "fmla v31.4s, v19.4s, v22.4s\n" "ldr q10, [x7, x11]\n" - "fmla v30.4s, v4.4s, v12.4s\n" + "fmla v29.4s, v19.4s, v17.4s\n" "ldr q12, [x4, x13]\n" - "fmla v31.4s, v4.4s, v9.4s\n" + "fmla v28.4s, v19.4s, v16.4s\n" "ldr q9, [x4, x6]\n" "ldr q4, [x8, #0x190]\n" - "fmax v28.4s, v28.4s, v18.4s\n" - "fmax v29.4s, v29.4s, v18.4s\n" + "fmax v30.4s, v30.4s, v27.4s\n" + "fmax v31.4s, v31.4s, v27.4s\n" "add x8, x8, #0x1a0\n" - "fmax v30.4s, v30.4s, v18.4s\n" - "fmax v31.4s, v31.4s, v18.4s\n" - "fmin v28.4s, v28.4s, v17.4s\n" - "fmin v29.4s, v29.4s, v17.4s\n" - "st1 { v28.4s }, [x5]\n" - "fmin v30.4s, v30.4s, v17.4s\n" - "fmin v31.4s, v31.4s, v17.4s\n" - "str q29, [x5, x3]\n" + "fmax v29.4s, v29.4s, v27.4s\n" + "fmax v28.4s, v28.4s, v27.4s\n" + "fmin v30.4s, v30.4s, v15.4s\n" + "fmin v31.4s, v31.4s, v15.4s\n" + "st1 { v30.4s }, [x5]\n" + "fmin v29.4s, v29.4s, v15.4s\n" + "fmin v28.4s, v28.4s, v15.4s\n" + "str q31, [x5, x3]\n" "add x5, x5, #0x10\n" - "st1 { v30.4s }, [x10]\n" - "str q31, [x10, x3]\n" + "st1 { v29.4s }, [x10]\n" + "str q28, [x10, x3]\n" "add x10, x10, #0x10\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "mov v28.16b, v16.16b\n fmla v28.4s, v0.4s, v5.4s\n" - "ldr q5, [x7, x15]\n" - "mov v29.16b, v16.16b\n fmla v29.4s, v0.4s, v6.4s\n" - "mov v30.16b, v16.16b\n fmla v30.4s, v0.4s, v7.4s\n" - "mov v31.16b, v16.16b\n fmla v31.4s, v0.4s, v8.4s\n" - "ldr q0, [x8, #0x0]\n" - "fmla v28.4s, v1.4s, v6.4s\n" - "ldr q6, [x7, x13]\n" - "fmla v29.4s, v1.4s, v9.4s\n" + "mov v31.16b, v25.16b\n fmla v31.4s, v0.4s, v5.4s\n" + "ldr q22, [x7, x15]\n" + "mov v5.16b, v25.16b\n fmla v5.4s, v0.4s, v6.4s\n" + "mov v30.16b, v25.16b\n fmla v30.4s, v0.4s, v7.4s\n" + "mov v29.16b, v25.16b\n fmla v29.4s, v0.4s, v8.4s\n" + "ldr q19, [x8, #0x0]\n" + "fmla v31.4s, v1.4s, v6.4s\n" + "ldr q21, [x7, x13]\n" + "fmla v5.4s, v1.4s, v9.4s\n" "add x7, x7, #0x10\n" "fmla v30.4s, v1.4s, v8.4s\n" - "fmla v31.4s, v1.4s, v13.4s\n" - "ldr q1, [x8, #0x10]\n" - "fmla v28.4s, v2.4s, v9.4s\n" - "ldr q9, [x4, x11]\n" - "fmla v29.4s, v2.4s, v11.4s\n" - "add x4, x4, #0x10\n" - "fmla v30.4s, v2.4s, v13.4s\n" - "fmla v31.4s, v2.4s, v5.4s\n" - "ldr q2, [x8, #0x20]\n" - "fmla v28.4s, v3.4s, v11.4s\n" - "ldr q11, [x17, x2]\n" - "fmla v29.4s, v3.4s, v12.4s\n" - "fmla v30.4s, v3.4s, v5.4s\n" - "fmla v31.4s, v3.4s, v6.4s\n" - "ldr q3, [x8, #0x30]\n" - "fmla v28.4s, v4.4s, v12.4s\n" - "ldr q12, [x17, x6]\n" - "fmla v29.4s, v4.4s, v9.4s\n" - "ldr q9, [x17, x15]\n" - "fmla v30.4s, v4.4s, v6.4s\n" - "fmla v31.4s, v4.4s, v10.4s\n" - "ldr q4, [x8, #0x40]\n" - "fmla v28.4s, v0.4s, v7.4s\n" - "fmla v29.4s, v0.4s, v8.4s\n" - "fmla v30.4s, v0.4s, v14.4s\n" - "fmla v31.4s, v0.4s, v11.4s\n" - "ldr q0, [x8, #0x50]\n" - "fmla v28.4s, v1.4s, v8.4s\n" - "ldr q8, [x17, x11]\n" "fmla v29.4s, v1.4s, v13.4s\n" - "fmla v30.4s, v1.4s, v11.4s\n" - "fmla v31.4s, v1.4s, v12.4s\n" - "ldr q1, [x8, #0x60]\n" - "fmla v28.4s, v2.4s, v13.4s\n" - "ldr q13, [x17, x13]\n" - "fmla v29.4s, v2.4s, v5.4s\n" - "add x17, x17, #0x10\n" - "fmla v30.4s, v2.4s, v12.4s\n" + "ldr q18, [x8, #0x10]\n" "fmla v31.4s, v2.4s, v9.4s\n" - "ldr q2, [x8, #0x70]\n" - "fmla v28.4s, v3.4s, v5.4s\n" - "ld1 { v5.4s }, [x16]\n" - "fmla v29.4s, v3.4s, v6.4s\n" - "fmla v30.4s, v3.4s, v9.4s\n" - "fmla v31.4s, v3.4s, v13.4s\n" - "ldr q3, [x8, #0x80]\n" - "fmla v28.4s, v4.4s, v6.4s\n" - "ldr q6, [x16, x2]\n" + "ldr q16, [x4, x11]\n" + "fmla v5.4s, v2.4s, v11.4s\n" + "add x4, x4, #0x10\n" + "fmla v30.4s, v2.4s, v13.4s\n" + "fmla v29.4s, v2.4s, v22.4s\n" + "ldr q17, [x8, #0x20]\n" + "fmla v31.4s, v3.4s, v11.4s\n" + "ldr q6, [x17, x2]\n" + "fmla v5.4s, v3.4s, v12.4s\n" + "fmla v30.4s, v3.4s, v22.4s\n" + "fmla v29.4s, v3.4s, v21.4s\n" + "ldr q20, [x8, #0x30]\n" + "fmla v31.4s, v4.4s, v12.4s\n" + "ldr q2, [x17, x6]\n" + "fmla v5.4s, v4.4s, v16.4s\n" + "ldr q28, [x17, x15]\n" + "fmla v30.4s, v4.4s, v21.4s\n" "fmla v29.4s, v4.4s, v10.4s\n" - "ldr q10, [x16, x6]\n" - "fmla v30.4s, v4.4s, v13.4s\n" - "fmla v31.4s, v4.4s, v8.4s\n" - "ldr q4, [x8, #0x90]\n" - "fmla v28.4s, v0.4s, v14.4s\n" - "ldr q14, [x16, x11]\n" - "fmla v29.4s, v0.4s, v11.4s\n" - "fmla v30.4s, v0.4s, v5.4s\n" - "fmla v31.4s, v0.4s, v6.4s\n" - "ldr q0, [x8, #0xa0]\n" - "fmla v28.4s, v1.4s, v11.4s\n" - "ldr q11, [x16, x15]\n" - "fmla v29.4s, v1.4s, v12.4s\n" - "fmla v30.4s, v1.4s, v6.4s\n" - "fmla v31.4s, v1.4s, v10.4s\n" - "ldr q1, [x8, #0xb0]\n" - "fmla v28.4s, v2.4s, v12.4s\n" - "ldr q12, [x16, x13]\n" - "fmla v29.4s, v2.4s, v9.4s\n" + "ldr q16, [x8, #0x40]\n" + "fmla v31.4s, v19.4s, v7.4s\n" + "fmla v5.4s, v19.4s, v8.4s\n" + "fmla v30.4s, v19.4s, v14.4s\n" + "fmla v29.4s, v19.4s, v6.4s\n" + "ldr q19, [x8, #0x50]\n" + "fmla v31.4s, v18.4s, v8.4s\n" + "ldr q1, [x17, x11]\n" + "fmla v5.4s, v18.4s, v13.4s\n" + "fmla v30.4s, v18.4s, v6.4s\n" + "fmla v29.4s, v18.4s, v2.4s\n" + "ldr q18, [x8, #0x60]\n" + "fmla v31.4s, v17.4s, v13.4s\n" + "ldr q26, [x17, x13]\n" + "fmla v5.4s, v17.4s, v22.4s\n" + "add x17, x17, #0x10\n" + "fmla v30.4s, v17.4s, v2.4s\n" + "fmla v29.4s, v17.4s, v28.4s\n" + "ldr q17, [x8, #0x70]\n" + "fmla v31.4s, v20.4s, v22.4s\n" + "ld1 { v25.4s }, [x16]\n" + "fmla v5.4s, v20.4s, v21.4s\n" + "fmla v30.4s, v20.4s, v28.4s\n" + "fmla v29.4s, v20.4s, v26.4s\n" + "ldr q24, [x8, #0x80]\n" + "fmla v31.4s, v16.4s, v21.4s\n" + "ldr q23, [x16, x2]\n" + "fmla v5.4s, v16.4s, v10.4s\n" + "ldr q0, [x16, x6]\n" + "fmla v30.4s, v16.4s, v26.4s\n" + "fmla v29.4s, v16.4s, v1.4s\n" + "ldr q22, [x8, #0x90]\n" + "fmla v31.4s, v19.4s, v14.4s\n" + "ldr q16, [x16, x11]\n" + "fmla v5.4s, v19.4s, v6.4s\n" + "fmla v30.4s, v19.4s, v25.4s\n" + "fmla v29.4s, v19.4s, v23.4s\n" + "ldr q21, [x8, #0xa0]\n" + "fmla v31.4s, v18.4s, v6.4s\n" + "ldr q20, [x16, x15]\n" + "fmla v5.4s, v18.4s, v2.4s\n" + "fmla v30.4s, v18.4s, v23.4s\n" + "fmla v29.4s, v18.4s, v0.4s\n" + "ldr q18, [x8, #0xb0]\n" + "fmla v31.4s, v17.4s, v2.4s\n" + "ldr q19, [x16, x13]\n" + "fmla v5.4s, v17.4s, v28.4s\n" "add x16, x16, #0x10\n" - "fmla v30.4s, v2.4s, v10.4s\n" - "fmla v31.4s, v2.4s, v11.4s\n" - "ldr q2, [x8, #0xc0]\n" - "fmla v28.4s, v3.4s, v9.4s\n" - "ld1 { v9.4s }, [x14]\n" - "fmla v29.4s, v3.4s, v13.4s\n" - "fmla v30.4s, v3.4s, v11.4s\n" - "fmla v31.4s, v3.4s, v12.4s\n" - "ldr q3, [x8, #0xd0]\n" - "fmla v28.4s, v4.4s, v13.4s\n" - "ldr q13, [x14, x2]\n" - "fmla v29.4s, v4.4s, v8.4s\n" - "ldr q8, [x14, x13]\n" - "fmla v30.4s, v4.4s, v12.4s\n" - "fmla v31.4s, v4.4s, v14.4s\n" - "ldr q4, [x8, #0xe0]\n" - "fmla v28.4s, v0.4s, v5.4s\n" - "ldr q5, [x14, x6]\n" - "fmla v29.4s, v0.4s, v6.4s\n" - "fmla v30.4s, v0.4s, v9.4s\n" - "fmla v31.4s, v0.4s, v13.4s\n" - "ldr q0, [x8, #0xf0]\n" - "fmla v28.4s, v1.4s, v6.4s\n" - "ldr q6, [x14, x15]\n" - "fmla v29.4s, v1.4s, v10.4s\n" - "fmla v30.4s, v1.4s, v13.4s\n" - "fmla v31.4s, v1.4s, v5.4s\n" - "ldr q1, [x8, #0x100]\n" - "fmla v28.4s, v2.4s, v10.4s\n" - "ldr q10, [x14, x11]\n" - "fmla v29.4s, v2.4s, v11.4s\n" + "fmla v30.4s, v17.4s, v0.4s\n" + "fmla v29.4s, v17.4s, v20.4s\n" + "ldr q17, [x8, #0xc0]\n" + "fmla v31.4s, v24.4s, v28.4s\n" + "ld1 { v7.4s }, [x14]\n" + "fmla v5.4s, v24.4s, v26.4s\n" + "fmla v30.4s, v24.4s, v20.4s\n" + "fmla v29.4s, v24.4s, v19.4s\n" + "ldr q2, [x8, #0xd0]\n" + "fmla v31.4s, v22.4s, v26.4s\n" + "ldr q28, [x14, x2]\n" + "fmla v5.4s, v22.4s, v1.4s\n" + "ldr q13, [x14, x13]\n" + "fmla v30.4s, v22.4s, v19.4s\n" + "fmla v29.4s, v22.4s, v16.4s\n" + "ldr q14, [x8, #0xe0]\n" + "fmla v31.4s, v21.4s, v25.4s\n" + "ldr q26, [x14, x6]\n" + "fmla v5.4s, v21.4s, v23.4s\n" + "fmla v30.4s, v21.4s, v7.4s\n" + "fmla v29.4s, v21.4s, v28.4s\n" + "ldr q25, [x8, #0xf0]\n" + "fmla v31.4s, v18.4s, v23.4s\n" + "ldr q24, [x14, x15]\n" + "fmla v5.4s, v18.4s, v0.4s\n" + "fmla v30.4s, v18.4s, v28.4s\n" + "fmla v29.4s, v18.4s, v26.4s\n" + "ldr q23, [x8, #0x100]\n" + "fmla v31.4s, v17.4s, v0.4s\n" + "ldr q22, [x14, x11]\n" + "fmla v5.4s, v17.4s, v20.4s\n" "add x14, x14, #0x10\n" - "fmla v30.4s, v2.4s, v5.4s\n" - "fmla v31.4s, v2.4s, v6.4s\n" - "ldr q2, [x8, #0x110]\n" - "fmla v28.4s, v3.4s, v11.4s\n" - "ld1 { v11.4s }, [x12]\n" - "fmla v29.4s, v3.4s, v12.4s\n" - "fmla v30.4s, v3.4s, v6.4s\n" - "fmla v31.4s, v3.4s, v8.4s\n" - "ldr q3, [x8, #0x120]\n" - "fmla v28.4s, v4.4s, v12.4s\n" - "ldr q12, [x12, x2]\n" - "fmla v29.4s, v4.4s, v14.4s\n" - "fmla v30.4s, v4.4s, v8.4s\n" - "fmla v31.4s, v4.4s, v10.4s\n" - "ldr q4, [x8, #0x130]\n" + "fmla v30.4s, v17.4s, v26.4s\n" + "fmla v29.4s, v17.4s, v24.4s\n" + "ldr q21, [x8, #0x110]\n" + "fmla v31.4s, v2.4s, v20.4s\n" + "ld1 { v18.4s }, [x12]\n" + "fmla v5.4s, v2.4s, v19.4s\n" + "fmla v30.4s, v2.4s, v24.4s\n" + "fmla v29.4s, v2.4s, v13.4s\n" + "ldr q20, [x8, #0x120]\n" + "fmla v31.4s, v14.4s, v19.4s\n" + "ldr q17, [x12, x2]\n" + "fmla v5.4s, v14.4s, v16.4s\n" + "fmla v30.4s, v14.4s, v13.4s\n" + "fmla v29.4s, v14.4s, v22.4s\n" + "ldr q19, [x8, #0x130]\n" "add x8, x8, #0x140\n" - "fmla v28.4s, v0.4s, v9.4s\n" - "ldr q9, [x12, x6]\n" - "fmla v29.4s, v0.4s, v13.4s\n" - "fmla v30.4s, v0.4s, v11.4s\n" - "ldr q11, [x12, x15]\n" - "fmla v31.4s, v0.4s, v12.4s\n" - "fmla v28.4s, v1.4s, v13.4s\n" - "fmla v29.4s, v1.4s, v5.4s\n" - "fmla v30.4s, v1.4s, v12.4s\n" - "ldr q12, [x12, x13]\n" - "fmla v31.4s, v1.4s, v9.4s\n" - "fmla v28.4s, v2.4s, v5.4s\n" - "fmla v29.4s, v2.4s, v6.4s\n" - "fmla v30.4s, v2.4s, v9.4s\n" - "ldr q9, [x12, x11]\n" - "fmla v31.4s, v2.4s, v11.4s\n" + "fmla v31.4s, v25.4s, v7.4s\n" + "ldr q16, [x12, x6]\n" + "fmla v5.4s, v25.4s, v28.4s\n" + "fmla v30.4s, v25.4s, v18.4s\n" + "ldr q18, [x12, x15]\n" + "fmla v29.4s, v25.4s, v17.4s\n" + "fmla v31.4s, v23.4s, v28.4s\n" + "fmla v5.4s, v23.4s, v26.4s\n" + "fmla v30.4s, v23.4s, v17.4s\n" + "ldr q17, [x12, x13]\n" + "fmla v29.4s, v23.4s, v16.4s\n" + "fmla v31.4s, v21.4s, v26.4s\n" + "fmla v5.4s, v21.4s, v24.4s\n" + "fmla v30.4s, v21.4s, v16.4s\n" + "ldr q16, [x12, x11]\n" + "fmla v29.4s, v21.4s, v18.4s\n" "add x12, x12, #0x10\n" - "fmla v28.4s, v3.4s, v6.4s\n" - "fmla v29.4s, v3.4s, v8.4s\n" - "fmla v30.4s, v3.4s, v11.4s\n" - "fmla v31.4s, v3.4s, v12.4s\n" - "fmla v28.4s, v4.4s, v8.4s\n" - "fmla v29.4s, v4.4s, v10.4s\n" - "fmax v28.4s, v28.4s, v18.4s\n" - "fmla v30.4s, v4.4s, v12.4s\n" - "fmla v31.4s, v4.4s, v9.4s\n" - "fmax v29.4s, v29.4s, v18.4s\n" - "fmax v30.4s, v30.4s, v18.4s\n" - "fmax v31.4s, v31.4s, v18.4s\n" - "fmin v28.4s, v28.4s, v17.4s\n" - "fmin v29.4s, v29.4s, v17.4s\n" - "st1 { v28.4s }, [x5]\n" - "fmin v30.4s, v30.4s, v17.4s\n" - "fmin v31.4s, v31.4s, v17.4s\n" - "str q29, [x5, x3]\n" + "fmla v31.4s, v20.4s, v24.4s\n" + "fmla v5.4s, v20.4s, v13.4s\n" + "fmla v30.4s, v20.4s, v18.4s\n" + "fmla v29.4s, v20.4s, v17.4s\n" + "fmla v31.4s, v19.4s, v13.4s\n" + "fmla v5.4s, v19.4s, v22.4s\n" + "fmax v31.4s, v31.4s, v27.4s\n" + "fmla v30.4s, v19.4s, v17.4s\n" + "fmla v29.4s, v19.4s, v16.4s\n" + "fmax v5.4s, v5.4s, v27.4s\n" + "fmax v30.4s, v30.4s, v27.4s\n" + "fmax v29.4s, v29.4s, v27.4s\n" + "fmin v31.4s, v31.4s, v15.4s\n" + "fmin v5.4s, v5.4s, v15.4s\n" + "st1 { v31.4s }, [x5]\n" + "fmin v30.4s, v30.4s, v15.4s\n" + "fmin v29.4s, v29.4s, v15.4s\n" + "str q5, [x5, x3]\n" "add x5, x5, #0x10\n" "st1 { v30.4s }, [x10]\n" - "str q31, [x10, x3]\n" + "str q29, [x10, x3]\n" "add x10, x10, #0x10\n" "4:" // Tile loop: Oddments "tst %x[n_channels], #0x3\n" "beq 61f\n" - "ldr q16, [x8, #0x0]\n" + "ldr q25, [x8, #0x0]\n" "ldr q0, [x8, #0x10]\n" "add x9, x4, XZR\n" "add x28, x4, x2\n" @@ -561,11 +561,11 @@ void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "ldr s10, [x21, #0x0]\n" "ldr s14, [x20, #0x0]\n" "6:" // Tile loop: Oddments: Load inputs: (0, 0), (0, 1), (1, 0), (1, 1), (0, 2), (1, 2), (0, 3), (0, 4), (1, 5), (2, 0): Bit 1: End - "mov v28.16b, v16.16b\n fmla v28.4s, v0.4s, v5.4s\n" - "mov v29.16b, v16.16b\n fmla v29.4s, v0.4s, v6.4s\n" + "mov v28.16b, v25.16b\n fmla v28.4s, v0.4s, v5.4s\n" + "mov v29.16b, v25.16b\n fmla v29.4s, v0.4s, v6.4s\n" "add x20, x7, x15\n" - "mov v30.16b, v16.16b\n fmla v30.4s, v0.4s, v7.4s\n" - "mov v31.16b, v16.16b\n fmla v31.4s, v0.4s, v8.4s\n" + "mov v30.16b, v25.16b\n fmla v30.4s, v0.4s, v7.4s\n" + "mov v31.16b, v25.16b\n fmla v31.4s, v0.4s, v8.4s\n" "fmla v28.4s, v1.4s, v6.4s\n" "fmla v29.4s, v1.4s, v9.4s\n" "fmla v30.4s, v1.4s, v8.4s\n" @@ -934,14 +934,14 @@ void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "ldr s9, [x20, #0x0]\n" "58:" // Tile loop: Oddments: Load inputs: (5, 5): Bit 1: End "fmla v31.4s, v4.4s, v9.4s\n" - "fmax v28.4s, v28.4s, v18.4s\n" - "fmax v29.4s, v29.4s, v18.4s\n" - "fmax v30.4s, v30.4s, v18.4s\n" - "fmax v31.4s, v31.4s, v18.4s\n" - "fmin v28.4s, v28.4s, v17.4s\n" - "fmin v29.4s, v29.4s, v17.4s\n" - "fmin v30.4s, v30.4s, v17.4s\n" - "fmin v31.4s, v31.4s, v17.4s\n" + "fmax v28.4s, v28.4s, v27.4s\n" + "fmax v29.4s, v29.4s, v27.4s\n" + "fmax v30.4s, v30.4s, v27.4s\n" + "fmax v31.4s, v31.4s, v27.4s\n" + "fmin v28.4s, v28.4s, v15.4s\n" + "fmin v29.4s, v29.4s, v15.4s\n" + "fmin v30.4s, v30.4s, v15.4s\n" + "fmin v31.4s, v31.4s, v15.4s\n" "tbz %x[n_channels], #1, 59f\n" "mov x21, x5\n" "mov x20, x10\n" @@ -967,7 +967,6 @@ void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "st1 { v29.s }[0], [x21]\n" "st1 { v31.s }[0], [x20]\n" "60:" // Tile loop: Oddments: Store: Bit 1: End - "61:" // Tile loop: End "ldr x26, [%x[params_struct], %[offsetof_args_tile_j]]\n" "ldr x27, [%x[params_struct], %[offsetof_args_tile_i]]\n" @@ -982,7 +981,7 @@ void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v16", "v17", "v18", "v28", "v29", "v30", "v31", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp index 860adac3268936f6ebf15a8b26258d8053f560f2..32939eb6dc870c4f974ebafd905bc912b332ef8f 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -103,16 +103,16 @@ void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl( "lsr x9, %x[n_channels], #0x2\n" "ldr x16, [%x[params_struct], %[offsetof_args_params]]\n" "add x20, %x[params_struct], %[offsetof_args_min]\n" - "ld1r { v18.4s }, [x20]\n" + "ld1r { v27.4s }, [x20]\n" "add x20, %x[params_struct], %[offsetof_args_max]\n" - "ld1r { v17.4s }, [x20]\n" + "ld1r { v15.4s }, [x20]\n" "add x15, %x[params_struct], %[offsetof_Args_inptrs]\n" "ldp x14, x13, [x21, #0x0]\n" "ldp x12, x11, [x21, #0x10]\n" "mov x10, #0x0\n" "sub x28, XZR, x17\n" "cbz x9, 3f\n" - "ldr q16, [x16, #0x0]\n" + "ldr q26, [x16, #0x0]\n" "ldr q0, [x16, #0x10]\n" "cmp x17, x9, LSL #4\n" "ldr q1, [x16, #0x20]\n" @@ -120,436 +120,436 @@ void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl( "ldr q3, [x16, #0x40]\n" "ldr q4, [x16, #0x50]\n" "add x16, x16, #0x60\n" - "ldp x27, x26, [x15, #0x0]\n" - "ldr q5, [x27, x10]\n" - "ldr q6, [x26, x10]\n" - "ldp x25, x24, [x15, #0x10]\n" - "ldr q7, [x25, x10]\n" - "ldr q8, [x24, x10]\n" - "ldp x23, x22, [x15, #0x20]\n" - "ldr q9, [x23, x10]\n" - "ldr q13, [x22, x10]\n" + "ldp x21, x20, [x15, #0x0]\n" + "ldr q5, [x21, x10]\n" + "ldr q6, [x20, x10]\n" + "ldp x21, x20, [x15, #0x10]\n" + "ldr q7, [x21, x10]\n" + "ldr q8, [x20, x10]\n" + "ldp x21, x20, [x15, #0x20]\n" + "ldr q9, [x21, x10]\n" + "ldr q13, [x20, x10]\n" "ldp x21, x20, [x15, #0x30]\n" "ldr q11, [x21, x10]\n" "ldr q12, [x20, x10]\n" - "ldp x27, x26, [x15, #0x40]\n" - "ldr q10, [x27, x10]\n" - "ldr q14, [x26, x10]\n" + "ldp x21, x20, [x15, #0x40]\n" + "ldr q10, [x21, x10]\n" + "ldr q14, [x20, x10]\n" "bge 2f\n" "1:" // Channel loop - "mov v28.16b, v16.16b\n fmla v28.4s, v0.4s, v5.4s\n" - "mov v29.16b, v16.16b\n fmla v29.4s, v0.4s, v6.4s\n" - "ldr x25, [x15, #0x50]\n" - "ldr q5, [x25, x10]\n" - "mov v30.16b, v16.16b\n fmla v30.4s, v0.4s, v7.4s\n" - "mov v31.16b, v16.16b\n fmla v31.4s, v0.4s, v8.4s\n" - "ldr q0, [x16, #0x0]\n" - "ldr q16, [x16, #0x140]\n" - "fmla v28.4s, v1.4s, v6.4s\n" - "fmla v29.4s, v1.4s, v9.4s\n" - "ldr x24, [x15, #0x58]\n" - "ldr q6, [x24, x10]\n" - "fmla v30.4s, v1.4s, v8.4s\n" - "fmla v31.4s, v1.4s, v13.4s\n" - "ldr q1, [x16, #0x10]\n" - "ldr x23, [x15, #0x60]\n" - "fmla v28.4s, v2.4s, v9.4s\n" - "ldr q9, [x23, x10]\n" - "fmla v29.4s, v2.4s, v11.4s\n" - "ldr x22, [x15, #0x68]\n" - "fmla v30.4s, v2.4s, v13.4s\n" - "fmla v31.4s, v2.4s, v5.4s\n" - "ldr q2, [x16, #0x20]\n" - "ldr x21, [x15, #0x70]\n" - "fmla v28.4s, v3.4s, v11.4s\n" - "ldr q11, [x22, x10]\n" - "fmla v29.4s, v3.4s, v12.4s\n" - "ldr x20, [x15, #0x78]\n" - "fmla v30.4s, v3.4s, v5.4s\n" - "fmla v31.4s, v3.4s, v6.4s\n" - "ldr q3, [x16, #0x30]\n" - "ldr x27, [x15, #0x80]\n" - "fmla v28.4s, v4.4s, v12.4s\n" - "ldr q12, [x21, x10]\n" - "fmla v29.4s, v4.4s, v9.4s\n" - "ldr q9, [x20, x10]\n" - "fmla v30.4s, v4.4s, v6.4s\n" - "fmla v31.4s, v4.4s, v10.4s\n" - "ldr q4, [x16, #0x40]\n" - "ldr x26, [x15, #0x88]\n" - "fmla v28.4s, v0.4s, v7.4s\n" - "fmla v29.4s, v0.4s, v8.4s\n" - "ldr x25, [x15, #0x90]\n" - "ldr x24, [x15, #0x98]\n" - "fmla v30.4s, v0.4s, v14.4s\n" - "fmla v31.4s, v0.4s, v11.4s\n" - "ldr q0, [x16, #0x50]\n" - "ldr x23, [x15, #0xa0]\n" + "mov v30.16b, v26.16b\n fmla v30.4s, v0.4s, v5.4s\n" + "mov v31.16b, v26.16b\n fmla v31.4s, v0.4s, v6.4s\n" + "ldr x20, [x15, #0x50]\n" + "ldr q24, [x20, x10]\n" + "mov v28.16b, v26.16b\n fmla v28.4s, v0.4s, v7.4s\n" + "mov v29.16b, v26.16b\n fmla v29.4s, v0.4s, v8.4s\n" + "ldr q23, [x16, #0x0]\n" + "ldr q26, [x16, #0x140]\n" + "fmla v30.4s, v1.4s, v6.4s\n" + "fmla v31.4s, v1.4s, v9.4s\n" + "ldr x20, [x15, #0x58]\n" + "ldr q22, [x20, x10]\n" "fmla v28.4s, v1.4s, v8.4s\n" - "ldr q8, [x26, x10]\n" "fmla v29.4s, v1.4s, v13.4s\n" - "ldr x22, [x15, #0xa8]\n" - "fmla v30.4s, v1.4s, v11.4s\n" - "fmla v31.4s, v1.4s, v12.4s\n" - "ldr q1, [x16, #0x60]\n" - "ldr x21, [x15, #0xb0]\n" - "fmla v28.4s, v2.4s, v13.4s\n" - "ldr q13, [x27, x10]\n" - "fmla v29.4s, v2.4s, v5.4s\n" - "ldr x20, [x15, #0xb8]\n" - "fmla v30.4s, v2.4s, v12.4s\n" - "fmla v31.4s, v2.4s, v9.4s\n" - "ldr q2, [x16, #0x70]\n" - "ldr x27, [x15, #0xc0]\n" - "fmla v28.4s, v3.4s, v5.4s\n" - "ldr q5, [x25, x10]\n" - "fmla v29.4s, v3.4s, v6.4s\n" - "ldr x26, [x15, #0xc8]\n" - "fmla v30.4s, v3.4s, v9.4s\n" - "fmla v31.4s, v3.4s, v13.4s\n" - "ldr q3, [x16, #0x80]\n" - "ldr x25, [x15, #0xd0]\n" - "fmla v28.4s, v4.4s, v6.4s\n" - "ldr q6, [x24, x10]\n" - "fmla v29.4s, v4.4s, v10.4s\n" - "ldr q10, [x23, x10]\n" - "fmla v30.4s, v4.4s, v13.4s\n" - "fmla v31.4s, v4.4s, v8.4s\n" - "ldr q4, [x16, #0x90]\n" - "ldr x24, [x15, #0xd8]\n" - "fmla v28.4s, v0.4s, v14.4s\n" - "ldr q14, [x20, x10]\n" - "fmla v29.4s, v0.4s, v11.4s\n" - "ldr x23, [x15, #0xe0]\n" - "fmla v30.4s, v0.4s, v5.4s\n" - "fmla v31.4s, v0.4s, v6.4s\n" - "ldr q0, [x16, #0xa0]\n" - "ldr x20, [x15, #0xf8]\n" - "fmla v28.4s, v1.4s, v11.4s\n" - "ldr q11, [x22, x10]\n" - "fmla v29.4s, v1.4s, v12.4s\n" - "ldr x22, [x15, #0xe8]\n" - "fmla v30.4s, v1.4s, v6.4s\n" - "fmla v31.4s, v1.4s, v10.4s\n" - "ldr q1, [x16, #0xb0]\n" - "add x28, x28, #0x10\n" - "fmla v28.4s, v2.4s, v12.4s\n" - "ldr q12, [x21, x10]\n" - "fmla v29.4s, v2.4s, v9.4s\n" - "ldr x21, [x15, #0xf0]\n" - "fmla v30.4s, v2.4s, v10.4s\n" + "ldr q21, [x16, #0x10]\n" + "ldr x20, [x15, #0x60]\n" + "fmla v30.4s, v2.4s, v9.4s\n" + "ldr q17, [x20, x10]\n" "fmla v31.4s, v2.4s, v11.4s\n" - "ldr q2, [x16, #0xc0]\n" - "fmla v28.4s, v3.4s, v9.4s\n" - "ldr q9, [x27, x10]\n" - "fmla v29.4s, v3.4s, v13.4s\n" - "ldr x27, [x15, #0x100]\n" + "ldr x20, [x15, #0x68]\n" + "fmla v28.4s, v2.4s, v13.4s\n" + "fmla v29.4s, v2.4s, v24.4s\n" + "ldr q16, [x16, #0x20]\n" + "ldr x22, [x15, #0x70]\n" "fmla v30.4s, v3.4s, v11.4s\n" + "ldr q5, [x20, x10]\n" "fmla v31.4s, v3.4s, v12.4s\n" - "ldr q3, [x16, #0xd0]\n" - "fmla v28.4s, v4.4s, v13.4s\n" - "ldr q13, [x26, x10]\n" - "fmla v29.4s, v4.4s, v8.4s\n" - "ldr q8, [x23, x10]\n" + "ldr x20, [x15, #0x78]\n" + "fmla v28.4s, v3.4s, v24.4s\n" + "fmla v29.4s, v3.4s, v22.4s\n" + "ldr q20, [x16, #0x30]\n" + "ldr x21, [x15, #0x80]\n" "fmla v30.4s, v4.4s, v12.4s\n" - "fmla v31.4s, v4.4s, v14.4s\n" - "ldr q4, [x16, #0xe0]\n" - "ldr x26, [x15, #0x108]\n" - "fmla v28.4s, v0.4s, v5.4s\n" - "ldr q5, [x25, x10]\n" - "fmla v29.4s, v0.4s, v6.4s\n" - "ldr x25, [x15, #0x110]\n" - "fmla v30.4s, v0.4s, v9.4s\n" - "fmla v31.4s, v0.4s, v13.4s\n" - "ldr q0, [x16, #0xf0]\n" - "fmla v28.4s, v1.4s, v6.4s\n" - "ldr q6, [x24, x10]\n" - "fmla v29.4s, v1.4s, v10.4s\n" - "ldr x24, [x15, #0x118]\n" - "fmla v30.4s, v1.4s, v13.4s\n" + "ldr q19, [x22, x10]\n" + "fmla v31.4s, v4.4s, v17.4s\n" + "ldr q2, [x20, x10]\n" + "fmla v28.4s, v4.4s, v22.4s\n" + "fmla v29.4s, v4.4s, v10.4s\n" + "ldr q18, [x16, #0x40]\n" + "ldr x20, [x15, #0x88]\n" + "fmla v30.4s, v23.4s, v7.4s\n" + "fmla v31.4s, v23.4s, v8.4s\n" + "ldr x23, [x15, #0x90]\n" + "ldr x26, [x15, #0x98]\n" + "fmla v28.4s, v23.4s, v14.4s\n" + "fmla v29.4s, v23.4s, v5.4s\n" + "ldr q1, [x16, #0x50]\n" + "ldr x22, [x15, #0xa0]\n" + "fmla v30.4s, v21.4s, v8.4s\n" + "ldr q25, [x20, x10]\n" + "fmla v31.4s, v21.4s, v13.4s\n" + "ldr x25, [x15, #0xa8]\n" + "fmla v28.4s, v21.4s, v5.4s\n" + "fmla v29.4s, v21.4s, v19.4s\n" + "ldr q17, [x16, #0x60]\n" + "ldr x24, [x15, #0xb0]\n" + "fmla v30.4s, v16.4s, v13.4s\n" + "ldr q8, [x21, x10]\n" + "fmla v31.4s, v16.4s, v24.4s\n" + "ldr x20, [x15, #0xb8]\n" + "fmla v28.4s, v16.4s, v19.4s\n" + "fmla v29.4s, v16.4s, v2.4s\n" + "ldr q16, [x16, #0x70]\n" + "ldr x21, [x15, #0xc0]\n" + "fmla v30.4s, v20.4s, v24.4s\n" + "ldr q24, [x23, x10]\n" + "fmla v31.4s, v20.4s, v22.4s\n" + "ldr x27, [x15, #0xc8]\n" + "fmla v28.4s, v20.4s, v2.4s\n" + "fmla v29.4s, v20.4s, v8.4s\n" + "ldr q23, [x16, #0x80]\n" + "ldr x23, [x15, #0xd0]\n" + "fmla v30.4s, v18.4s, v22.4s\n" + "ldr q22, [x26, x10]\n" + "fmla v31.4s, v18.4s, v10.4s\n" + "ldr q21, [x22, x10]\n" + "fmla v28.4s, v18.4s, v8.4s\n" + "fmla v29.4s, v18.4s, v25.4s\n" + "ldr q20, [x16, #0x90]\n" + "ldr x22, [x15, #0xd8]\n" + "fmla v30.4s, v1.4s, v14.4s\n" + "ldr q0, [x20, x10]\n" "fmla v31.4s, v1.4s, v5.4s\n" - "ldr q1, [x16, #0x100]\n" - "fmla v28.4s, v2.4s, v10.4s\n" - "ldr q10, [x22, x10]\n" - "fmla v29.4s, v2.4s, v11.4s\n" - "fmla v30.4s, v2.4s, v5.4s\n" - "fmla v31.4s, v2.4s, v6.4s\n" - "ldr q2, [x16, #0x110]\n" - "fmla v28.4s, v3.4s, v11.4s\n" - "ldr q11, [x21, x10]\n" - "fmla v29.4s, v3.4s, v12.4s\n" - "fmla v30.4s, v3.4s, v6.4s\n" - "fmla v31.4s, v3.4s, v8.4s\n" - "ldr q3, [x16, #0x120]\n" - "fmla v28.4s, v4.4s, v12.4s\n" - "ldr q12, [x20, x10]\n" - "fmla v29.4s, v4.4s, v14.4s\n" - "fmla v30.4s, v4.4s, v8.4s\n" - "fmla v31.4s, v4.4s, v10.4s\n" - "ldr q4, [x16, #0x130]\n" - "fmla v28.4s, v0.4s, v9.4s\n" - "ldr q9, [x27, x10]\n" - "fmla v29.4s, v0.4s, v13.4s\n" - "fmla v30.4s, v0.4s, v11.4s\n" - "ldr q11, [x26, x10]\n" - "fmla v31.4s, v0.4s, v12.4s\n" + "ldr x20, [x15, #0xe0]\n" + "fmla v28.4s, v1.4s, v24.4s\n" + "fmla v29.4s, v1.4s, v22.4s\n" + "ldr q6, [x16, #0xa0]\n" + "ldr x26, [x15, #0xf8]\n" + "fmla v30.4s, v17.4s, v5.4s\n" + "ldr q1, [x25, x10]\n" + "fmla v31.4s, v17.4s, v19.4s\n" + "ldr x25, [x15, #0xe8]\n" + "fmla v28.4s, v17.4s, v22.4s\n" + "fmla v29.4s, v17.4s, v21.4s\n" + "ldr q18, [x16, #0xb0]\n" + "add x28, x28, #0x10\n" + "fmla v30.4s, v16.4s, v19.4s\n" + "ldr q19, [x24, x10]\n" + "fmla v31.4s, v16.4s, v2.4s\n" + "ldr x24, [x15, #0xf0]\n" + "fmla v28.4s, v16.4s, v21.4s\n" + "fmla v29.4s, v16.4s, v1.4s\n" + "ldr q17, [x16, #0xc0]\n" + "fmla v30.4s, v23.4s, v2.4s\n" + "ldr q16, [x21, x10]\n" + "fmla v31.4s, v23.4s, v8.4s\n" + "ldr x21, [x15, #0x100]\n" + "fmla v28.4s, v23.4s, v1.4s\n" + "fmla v29.4s, v23.4s, v19.4s\n" + "ldr q13, [x16, #0xd0]\n" + "fmla v30.4s, v20.4s, v8.4s\n" + "ldr q2, [x27, x10]\n" + "fmla v31.4s, v20.4s, v25.4s\n" + "ldr q10, [x20, x10]\n" + "fmla v28.4s, v20.4s, v19.4s\n" + "fmla v29.4s, v20.4s, v0.4s\n" + "ldr q9, [x16, #0xe0]\n" + "ldr x20, [x15, #0x108]\n" + "fmla v30.4s, v6.4s, v24.4s\n" + "ldr q5, [x23, x10]\n" + "fmla v31.4s, v6.4s, v22.4s\n" + "ldr x23, [x15, #0x110]\n" + "fmla v28.4s, v6.4s, v16.4s\n" + "fmla v29.4s, v6.4s, v2.4s\n" + "ldr q24, [x16, #0xf0]\n" + "fmla v30.4s, v18.4s, v22.4s\n" + "ldr q25, [x22, x10]\n" + "fmla v31.4s, v18.4s, v21.4s\n" + "ldr x22, [x15, #0x118]\n" + "fmla v28.4s, v18.4s, v2.4s\n" + "fmla v29.4s, v18.4s, v5.4s\n" + "ldr q23, [x16, #0x100]\n" + "fmla v30.4s, v17.4s, v21.4s\n" + "ldr q22, [x25, x10]\n" + "fmla v31.4s, v17.4s, v1.4s\n" + "fmla v28.4s, v17.4s, v5.4s\n" + "fmla v29.4s, v17.4s, v25.4s\n" + "ldr q21, [x16, #0x110]\n" + "fmla v30.4s, v13.4s, v1.4s\n" + "ldr q18, [x24, x10]\n" + "fmla v31.4s, v13.4s, v19.4s\n" + "fmla v28.4s, v13.4s, v25.4s\n" + "fmla v29.4s, v13.4s, v10.4s\n" + "ldr q20, [x16, #0x120]\n" + "fmla v30.4s, v9.4s, v19.4s\n" + "ldr q17, [x26, x10]\n" + "fmla v31.4s, v9.4s, v0.4s\n" + "fmla v28.4s, v9.4s, v10.4s\n" + "fmla v29.4s, v9.4s, v22.4s\n" + "ldr q19, [x16, #0x130]\n" + "fmla v30.4s, v24.4s, v16.4s\n" + "ldr q16, [x21, x10]\n" + "fmla v31.4s, v24.4s, v2.4s\n" + "fmla v28.4s, v24.4s, v18.4s\n" + "ldr q18, [x20, x10]\n" + "fmla v29.4s, v24.4s, v17.4s\n" "ldr q0, [x16, #0x150]\n" - "fmla v28.4s, v1.4s, v13.4s\n" - "fmla v29.4s, v1.4s, v5.4s\n" - "ldp x27, x26, [x15, #0x0]\n" - "fmla v30.4s, v1.4s, v12.4s\n" - "ldr q12, [x25, x10]\n" - "fmla v31.4s, v1.4s, v9.4s\n" + "fmla v30.4s, v23.4s, v2.4s\n" + "fmla v31.4s, v23.4s, v5.4s\n" + "ldp x21, x20, [x15, #0x0]\n" + "fmla v28.4s, v23.4s, v17.4s\n" + "ldr q17, [x23, x10]\n" + "fmla v29.4s, v23.4s, v16.4s\n" "ldr q1, [x16, #0x160]\n" - "fmla v28.4s, v2.4s, v5.4s\n" - "ldr q5, [x27, x17]\n" - "fmla v29.4s, v2.4s, v6.4s\n" - "fmla v30.4s, v2.4s, v9.4s\n" - "ldr q9, [x24, x10]\n" - "fmla v31.4s, v2.4s, v11.4s\n" + "fmla v30.4s, v21.4s, v5.4s\n" + "ldr q5, [x21, x17]\n" + "fmla v31.4s, v21.4s, v25.4s\n" + "fmla v28.4s, v21.4s, v16.4s\n" + "ldr q16, [x22, x10]\n" + "fmla v29.4s, v21.4s, v18.4s\n" "ldr q2, [x16, #0x170]\n" - "fmla v28.4s, v3.4s, v6.4s\n" - "ldr q6, [x26, x17]\n" - "fmla v29.4s, v3.4s, v8.4s\n" - "ldp x25, x24, [x15, #0x10]\n" - "ldr q7, [x25, x17]\n" - "fmla v30.4s, v3.4s, v11.4s\n" - "fmla v31.4s, v3.4s, v12.4s\n" + "fmla v30.4s, v20.4s, v25.4s\n" + "ldr q6, [x20, x17]\n" + "fmla v31.4s, v20.4s, v10.4s\n" + "ldp x21, x20, [x15, #0x10]\n" + "ldr q7, [x21, x17]\n" + "fmla v28.4s, v20.4s, v18.4s\n" + "fmla v29.4s, v20.4s, v17.4s\n" "ldr q3, [x16, #0x180]\n" - "fmla v28.4s, v4.4s, v8.4s\n" - "ldr q8, [x24, x17]\n" - "fmla v29.4s, v4.4s, v10.4s\n" - "ldp x23, x22, [x15, #0x20]\n" - "ldr q13, [x22, x17]\n" - "fmla v30.4s, v4.4s, v12.4s\n" - "fmla v31.4s, v4.4s, v9.4s\n" - "ldr q9, [x23, x17]\n" + "fmla v30.4s, v19.4s, v10.4s\n" + "ldr q8, [x20, x17]\n" + "fmla v31.4s, v19.4s, v22.4s\n" + "ldp x21, x20, [x15, #0x20]\n" + "ldr q13, [x20, x17]\n" + "fmla v28.4s, v19.4s, v17.4s\n" + "fmla v29.4s, v19.4s, v16.4s\n" + "ldr q9, [x21, x17]\n" "ldr q4, [x16, #0x190]\n" "ldp x21, x20, [x15, #0x30]\n" - "fmax v28.4s, v28.4s, v18.4s\n" - "fmax v29.4s, v29.4s, v18.4s\n" + "fmax v30.4s, v30.4s, v27.4s\n" + "fmax v31.4s, v31.4s, v27.4s\n" "ldr q11, [x21, x17]\n" "ldr q12, [x20, x17]\n" - "fmax v30.4s, v30.4s, v18.4s\n" - "fmax v31.4s, v31.4s, v18.4s\n" - "ldp x27, x26, [x15, #0x40]\n" - "ldr q10, [x27, x17]\n" - "fmin v28.4s, v28.4s, v17.4s\n" - "fmin v29.4s, v29.4s, v17.4s\n" - "ldr q14, [x26, x17]\n" + "fmax v28.4s, v28.4s, v27.4s\n" + "fmax v29.4s, v29.4s, v27.4s\n" + "ldp x21, x20, [x15, #0x40]\n" + "ldr q10, [x21, x17]\n" + "fmin v30.4s, v30.4s, v15.4s\n" + "fmin v31.4s, v31.4s, v15.4s\n" + "ldr q14, [x20, x17]\n" "add x17, x17, #0x10\n" "cmp x17, x9, LSL #4\n" - "fmin v30.4s, v30.4s, v17.4s\n" - "fmin v31.4s, v31.4s, v17.4s\n" + "fmin v28.4s, v28.4s, v15.4s\n" + "fmin v29.4s, v29.4s, v15.4s\n" "add x10, x10, #0x10\n" - "str q28, [x14, x28]\n" + "str q30, [x14, x28]\n" "add x16, x16, #0x1a0\n" - "str q29, [x13, x28]\n" - "str q30, [x12, x28]\n" - "str q31, [x11, x28]\n" + "str q31, [x13, x28]\n" + "str q28, [x12, x28]\n" + "str q29, [x11, x28]\n" "blt 1b\n" "2:" // Channel tail - "mov v28.16b, v16.16b\n fmla v28.4s, v0.4s, v5.4s\n" - "mov v29.16b, v16.16b\n fmla v29.4s, v0.4s, v6.4s\n" - "ldr x25, [x15, #0x50]\n" - "ldr q5, [x25, x10]\n" - "mov v30.16b, v16.16b\n fmla v30.4s, v0.4s, v7.4s\n" - "mov v31.16b, v16.16b\n fmla v31.4s, v0.4s, v8.4s\n" - "ldr q0, [x16, #0x0]\n" - "ldr x24, [x15, #0x58]\n" - "fmla v28.4s, v1.4s, v6.4s\n" - "ldr q6, [x24, x10]\n" - "fmla v29.4s, v1.4s, v9.4s\n" - "ldr x23, [x15, #0x60]\n" + "mov v31.16b, v26.16b\n fmla v31.4s, v0.4s, v5.4s\n" + "mov v5.16b, v26.16b\n fmla v5.4s, v0.4s, v6.4s\n" + "ldr x20, [x15, #0x50]\n" + "ldr q22, [x20, x10]\n" + "mov v30.16b, v26.16b\n fmla v30.4s, v0.4s, v7.4s\n" + "mov v29.16b, v26.16b\n fmla v29.4s, v0.4s, v8.4s\n" + "ldr q19, [x16, #0x0]\n" + "ldr x20, [x15, #0x58]\n" + "fmla v31.4s, v1.4s, v6.4s\n" + "ldr q21, [x20, x10]\n" + "fmla v5.4s, v1.4s, v9.4s\n" + "ldr x21, [x15, #0x60]\n" "fmla v30.4s, v1.4s, v8.4s\n" - "fmla v31.4s, v1.4s, v13.4s\n" - "ldr q1, [x16, #0x10]\n" - "ldr x22, [x15, #0x68]\n" - "fmla v28.4s, v2.4s, v9.4s\n" - "ldr q9, [x23, x10]\n" - "fmla v29.4s, v2.4s, v11.4s\n" - "ldr x21, [x15, #0x70]\n" - "fmla v30.4s, v2.4s, v13.4s\n" - "fmla v31.4s, v2.4s, v5.4s\n" - "ldr q2, [x16, #0x20]\n" - "ldr x20, [x15, #0x78]\n" - "fmla v28.4s, v3.4s, v11.4s\n" - "ldr q11, [x22, x10]\n" - "fmla v29.4s, v3.4s, v12.4s\n" - "ldr x27, [x15, #0x80]\n" - "fmla v30.4s, v3.4s, v5.4s\n" - "fmla v31.4s, v3.4s, v6.4s\n" - "ldr q3, [x16, #0x30]\n" - "ldr x26, [x15, #0x88]\n" - "fmla v28.4s, v4.4s, v12.4s\n" - "ldr q12, [x21, x10]\n" - "fmla v29.4s, v4.4s, v9.4s\n" - "ldr q9, [x20, x10]\n" - "fmla v30.4s, v4.4s, v6.4s\n" - "fmla v31.4s, v4.4s, v10.4s\n" - "ldr q4, [x16, #0x40]\n" - "ldr x25, [x15, #0x90]\n" - "fmla v28.4s, v0.4s, v7.4s\n" - "fmla v29.4s, v0.4s, v8.4s\n" - "ldr x24, [x15, #0x98]\n" - "ldr x23, [x15, #0xa0]\n" - "fmla v30.4s, v0.4s, v14.4s\n" - "fmla v31.4s, v0.4s, v11.4s\n" - "ldr q0, [x16, #0x50]\n" - "ldr x22, [x15, #0xa8]\n" - "fmla v28.4s, v1.4s, v8.4s\n" - "ldr q8, [x26, x10]\n" "fmla v29.4s, v1.4s, v13.4s\n" - "ldr x21, [x15, #0xb0]\n" - "fmla v30.4s, v1.4s, v11.4s\n" - "fmla v31.4s, v1.4s, v12.4s\n" - "ldr q1, [x16, #0x60]\n" - "ldr x20, [x15, #0xb8]\n" - "fmla v28.4s, v2.4s, v13.4s\n" - "ldr q13, [x27, x10]\n" - "fmla v29.4s, v2.4s, v5.4s\n" - "ldr x27, [x15, #0xc0]\n" - "fmla v30.4s, v2.4s, v12.4s\n" + "ldr q18, [x16, #0x10]\n" + "ldr x20, [x15, #0x68]\n" "fmla v31.4s, v2.4s, v9.4s\n" - "ldr q2, [x16, #0x70]\n" - "ldr x26, [x15, #0xc8]\n" - "fmla v28.4s, v3.4s, v5.4s\n" - "ldr q5, [x25, x10]\n" - "fmla v29.4s, v3.4s, v6.4s\n" - "ldr x25, [x15, #0xd0]\n" - "fmla v30.4s, v3.4s, v9.4s\n" - "fmla v31.4s, v3.4s, v13.4s\n" - "ldr q3, [x16, #0x80]\n" - "add x28, x28, #0x10\n" - "fmla v28.4s, v4.4s, v6.4s\n" - "ldr q6, [x24, x10]\n" + "ldr q16, [x21, x10]\n" + "fmla v5.4s, v2.4s, v11.4s\n" + "ldr x23, [x15, #0x70]\n" + "fmla v30.4s, v2.4s, v13.4s\n" + "fmla v29.4s, v2.4s, v22.4s\n" + "ldr q17, [x16, #0x20]\n" + "ldr x21, [x15, #0x78]\n" + "fmla v31.4s, v3.4s, v11.4s\n" + "ldr q6, [x20, x10]\n" + "fmla v5.4s, v3.4s, v12.4s\n" + "ldr x22, [x15, #0x80]\n" + "fmla v30.4s, v3.4s, v22.4s\n" + "fmla v29.4s, v3.4s, v21.4s\n" + "ldr q20, [x16, #0x30]\n" + "ldr x20, [x15, #0x88]\n" + "fmla v31.4s, v4.4s, v12.4s\n" + "ldr q2, [x23, x10]\n" + "fmla v5.4s, v4.4s, v16.4s\n" + "ldr q28, [x21, x10]\n" + "fmla v30.4s, v4.4s, v21.4s\n" "fmla v29.4s, v4.4s, v10.4s\n" - "ldr q10, [x23, x10]\n" - "fmla v30.4s, v4.4s, v13.4s\n" - "fmla v31.4s, v4.4s, v8.4s\n" - "ldr q4, [x16, #0x90]\n" - "ldr x24, [x15, #0xd8]\n" - "fmla v28.4s, v0.4s, v14.4s\n" - "ldr q14, [x20, x10]\n" - "fmla v29.4s, v0.4s, v11.4s\n" - "ldr x23, [x15, #0xe0]\n" - "fmla v30.4s, v0.4s, v5.4s\n" - "fmla v31.4s, v0.4s, v6.4s\n" - "ldr q0, [x16, #0xa0]\n" - "ldr x20, [x15, #0xf8]\n" - "fmla v28.4s, v1.4s, v11.4s\n" - "ldr q11, [x22, x10]\n" - "fmla v29.4s, v1.4s, v12.4s\n" - "ldr x22, [x15, #0xe8]\n" - "fmla v30.4s, v1.4s, v6.4s\n" - "fmla v31.4s, v1.4s, v10.4s\n" - "ldr q1, [x16, #0xb0]\n" - "fmla v28.4s, v2.4s, v12.4s\n" - "ldr q12, [x21, x10]\n" - "fmla v29.4s, v2.4s, v9.4s\n" - "ldr x21, [x15, #0xf0]\n" - "fmla v30.4s, v2.4s, v10.4s\n" - "fmla v31.4s, v2.4s, v11.4s\n" - "ldr q2, [x16, #0xc0]\n" - "fmla v28.4s, v3.4s, v9.4s\n" - "ldr q9, [x27, x10]\n" - "fmla v29.4s, v3.4s, v13.4s\n" - "ldr x27, [x15, #0x100]\n" - "fmla v30.4s, v3.4s, v11.4s\n" - "fmla v31.4s, v3.4s, v12.4s\n" + "ldr q16, [x16, #0x40]\n" + "ldr x21, [x15, #0x90]\n" + "fmla v31.4s, v19.4s, v7.4s\n" + "fmla v5.4s, v19.4s, v8.4s\n" + "ldr x27, [x15, #0x98]\n" + "ldr x26, [x15, #0xa0]\n" + "fmla v30.4s, v19.4s, v14.4s\n" + "fmla v29.4s, v19.4s, v6.4s\n" + "ldr q19, [x16, #0x50]\n" + "ldr x25, [x15, #0xa8]\n" + "fmla v31.4s, v18.4s, v8.4s\n" + "ldr q1, [x20, x10]\n" + "fmla v5.4s, v18.4s, v13.4s\n" + "ldr x24, [x15, #0xb0]\n" + "fmla v30.4s, v18.4s, v6.4s\n" + "fmla v29.4s, v18.4s, v2.4s\n" + "ldr q18, [x16, #0x60]\n" + "ldr x20, [x15, #0xb8]\n" + "fmla v31.4s, v17.4s, v13.4s\n" + "ldr q26, [x22, x10]\n" + "fmla v5.4s, v17.4s, v22.4s\n" + "ldr x23, [x15, #0xc0]\n" + "fmla v30.4s, v17.4s, v2.4s\n" + "fmla v29.4s, v17.4s, v28.4s\n" + "ldr q17, [x16, #0x70]\n" + "ldr x22, [x15, #0xc8]\n" + "fmla v31.4s, v20.4s, v22.4s\n" + "ldr q25, [x21, x10]\n" + "fmla v5.4s, v20.4s, v21.4s\n" + "ldr x21, [x15, #0xd0]\n" + "fmla v30.4s, v20.4s, v28.4s\n" + "fmla v29.4s, v20.4s, v26.4s\n" + "ldr q24, [x16, #0x80]\n" + "add x28, x28, #0x10\n" + "fmla v31.4s, v16.4s, v21.4s\n" + "ldr q23, [x27, x10]\n" + "fmla v5.4s, v16.4s, v10.4s\n" + "ldr q0, [x26, x10]\n" + "fmla v30.4s, v16.4s, v26.4s\n" + "fmla v29.4s, v16.4s, v1.4s\n" + "ldr q22, [x16, #0x90]\n" + "ldr x27, [x15, #0xd8]\n" + "fmla v31.4s, v19.4s, v14.4s\n" + "ldr q16, [x20, x10]\n" + "fmla v5.4s, v19.4s, v6.4s\n" + "ldr x20, [x15, #0xe0]\n" + "fmla v30.4s, v19.4s, v25.4s\n" + "fmla v29.4s, v19.4s, v23.4s\n" + "ldr q21, [x16, #0xa0]\n" + "ldr x26, [x15, #0xf8]\n" + "fmla v31.4s, v18.4s, v6.4s\n" + "ldr q20, [x25, x10]\n" + "fmla v5.4s, v18.4s, v2.4s\n" + "ldr x25, [x15, #0xe8]\n" + "fmla v30.4s, v18.4s, v23.4s\n" + "fmla v29.4s, v18.4s, v0.4s\n" + "ldr q18, [x16, #0xb0]\n" + "fmla v31.4s, v17.4s, v2.4s\n" + "ldr q19, [x24, x10]\n" + "fmla v5.4s, v17.4s, v28.4s\n" + "ldr x24, [x15, #0xf0]\n" + "fmla v30.4s, v17.4s, v0.4s\n" + "fmla v29.4s, v17.4s, v20.4s\n" + "ldr q17, [x16, #0xc0]\n" + "fmla v31.4s, v24.4s, v28.4s\n" + "ldr q7, [x23, x10]\n" + "fmla v5.4s, v24.4s, v26.4s\n" + "ldr x23, [x15, #0x100]\n" + "fmla v30.4s, v24.4s, v20.4s\n" + "fmla v29.4s, v24.4s, v19.4s\n" "ldr q3, [x16, #0xd0]\n" - "fmla v28.4s, v4.4s, v13.4s\n" - "ldr q13, [x26, x10]\n" - "fmla v29.4s, v4.4s, v8.4s\n" - "ldr q8, [x23, x10]\n" - "fmla v30.4s, v4.4s, v12.4s\n" - "fmla v31.4s, v4.4s, v14.4s\n" - "ldr q4, [x16, #0xe0]\n" - "ldr x26, [x15, #0x108]\n" - "fmla v28.4s, v0.4s, v5.4s\n" - "ldr q5, [x25, x10]\n" - "fmla v29.4s, v0.4s, v6.4s\n" - "ldr x25, [x15, #0x110]\n" - "fmla v30.4s, v0.4s, v9.4s\n" - "fmla v31.4s, v0.4s, v13.4s\n" - "ldr q0, [x16, #0xf0]\n" - "fmla v28.4s, v1.4s, v6.4s\n" - "ldr q6, [x24, x10]\n" - "fmla v29.4s, v1.4s, v10.4s\n" - "ldr x24, [x15, #0x118]\n" - "fmla v30.4s, v1.4s, v13.4s\n" - "fmla v31.4s, v1.4s, v5.4s\n" - "ldr q1, [x16, #0x100]\n" - "fmla v28.4s, v2.4s, v10.4s\n" - "ldr q10, [x22, x10]\n" - "fmla v29.4s, v2.4s, v11.4s\n" - "fmla v30.4s, v2.4s, v5.4s\n" - "fmla v31.4s, v2.4s, v6.4s\n" - "ldr q2, [x16, #0x110]\n" - "fmla v28.4s, v3.4s, v11.4s\n" - "ldr q11, [x21, x10]\n" - "fmla v29.4s, v3.4s, v12.4s\n" - "fmla v30.4s, v3.4s, v6.4s\n" - "fmla v31.4s, v3.4s, v8.4s\n" - "ldr q3, [x16, #0x120]\n" - "fmla v28.4s, v4.4s, v12.4s\n" - "ldr q12, [x20, x10]\n" - "fmla v29.4s, v4.4s, v14.4s\n" - "fmla v30.4s, v4.4s, v8.4s\n" - "fmla v31.4s, v4.4s, v10.4s\n" - "ldr q4, [x16, #0x130]\n" + "fmla v31.4s, v22.4s, v26.4s\n" + "ldr q28, [x22, x10]\n" + "fmla v5.4s, v22.4s, v1.4s\n" + "ldr q13, [x20, x10]\n" + "fmla v30.4s, v22.4s, v19.4s\n" + "fmla v29.4s, v22.4s, v16.4s\n" + "ldr q11, [x16, #0xe0]\n" + "ldr x22, [x15, #0x108]\n" + "fmla v31.4s, v21.4s, v25.4s\n" + "ldr q26, [x21, x10]\n" + "fmla v5.4s, v21.4s, v23.4s\n" + "ldr x21, [x15, #0x110]\n" + "fmla v30.4s, v21.4s, v7.4s\n" + "fmla v29.4s, v21.4s, v28.4s\n" + "ldr q25, [x16, #0xf0]\n" + "fmla v31.4s, v18.4s, v23.4s\n" + "ldr q24, [x27, x10]\n" + "fmla v5.4s, v18.4s, v0.4s\n" + "ldr x20, [x15, #0x118]\n" + "fmla v30.4s, v18.4s, v28.4s\n" + "fmla v29.4s, v18.4s, v26.4s\n" + "ldr q23, [x16, #0x100]\n" + "fmla v31.4s, v17.4s, v0.4s\n" + "ldr q22, [x25, x10]\n" + "fmla v5.4s, v17.4s, v20.4s\n" + "fmla v30.4s, v17.4s, v26.4s\n" + "fmla v29.4s, v17.4s, v24.4s\n" + "ldr q21, [x16, #0x110]\n" + "fmla v31.4s, v3.4s, v20.4s\n" + "ldr q18, [x24, x10]\n" + "fmla v5.4s, v3.4s, v19.4s\n" + "fmla v30.4s, v3.4s, v24.4s\n" + "fmla v29.4s, v3.4s, v13.4s\n" + "ldr q20, [x16, #0x120]\n" + "fmla v31.4s, v11.4s, v19.4s\n" + "ldr q17, [x26, x10]\n" + "fmla v5.4s, v11.4s, v16.4s\n" + "fmla v30.4s, v11.4s, v13.4s\n" + "fmla v29.4s, v11.4s, v22.4s\n" + "ldr q19, [x16, #0x130]\n" "add x16, x16, #0x140\n" - "fmla v28.4s, v0.4s, v9.4s\n" - "ldr q9, [x27, x10]\n" - "fmla v29.4s, v0.4s, v13.4s\n" - "fmla v30.4s, v0.4s, v11.4s\n" - "ldr q11, [x26, x10]\n" - "fmla v31.4s, v0.4s, v12.4s\n" - "fmla v28.4s, v1.4s, v13.4s\n" - "fmla v29.4s, v1.4s, v5.4s\n" - "fmla v30.4s, v1.4s, v12.4s\n" - "ldr q12, [x25, x10]\n" - "fmla v31.4s, v1.4s, v9.4s\n" - "fmla v28.4s, v2.4s, v5.4s\n" - "fmla v29.4s, v2.4s, v6.4s\n" - "fmla v30.4s, v2.4s, v9.4s\n" - "ldr q9, [x24, x10]\n" - "fmla v31.4s, v2.4s, v11.4s\n" + "fmla v31.4s, v25.4s, v7.4s\n" + "ldr q16, [x23, x10]\n" + "fmla v5.4s, v25.4s, v28.4s\n" + "fmla v30.4s, v25.4s, v18.4s\n" + "ldr q18, [x22, x10]\n" + "fmla v29.4s, v25.4s, v17.4s\n" + "fmla v31.4s, v23.4s, v28.4s\n" + "fmla v5.4s, v23.4s, v26.4s\n" + "fmla v30.4s, v23.4s, v17.4s\n" + "ldr q17, [x21, x10]\n" + "fmla v29.4s, v23.4s, v16.4s\n" + "fmla v31.4s, v21.4s, v26.4s\n" + "fmla v5.4s, v21.4s, v24.4s\n" + "fmla v30.4s, v21.4s, v16.4s\n" + "ldr q16, [x20, x10]\n" + "fmla v29.4s, v21.4s, v18.4s\n" "add x10, x10, #0x10\n" - "fmla v28.4s, v3.4s, v6.4s\n" - "fmla v29.4s, v3.4s, v8.4s\n" - "fmla v30.4s, v3.4s, v11.4s\n" - "fmla v31.4s, v3.4s, v12.4s\n" - "fmla v28.4s, v4.4s, v8.4s\n" - "fmla v29.4s, v4.4s, v10.4s\n" - "fmax v28.4s, v28.4s, v18.4s\n" - "fmla v30.4s, v4.4s, v12.4s\n" - "fmla v31.4s, v4.4s, v9.4s\n" - "fmax v29.4s, v29.4s, v18.4s\n" - "fmax v30.4s, v30.4s, v18.4s\n" - "fmax v31.4s, v31.4s, v18.4s\n" - "fmin v28.4s, v28.4s, v17.4s\n" - "fmin v29.4s, v29.4s, v17.4s\n" - "str q28, [x14, x28]\n" - "fmin v30.4s, v30.4s, v17.4s\n" - "fmin v31.4s, v31.4s, v17.4s\n" - "str q29, [x13, x28]\n" + "fmla v31.4s, v20.4s, v24.4s\n" + "fmla v5.4s, v20.4s, v13.4s\n" + "fmla v30.4s, v20.4s, v18.4s\n" + "fmla v29.4s, v20.4s, v17.4s\n" + "fmla v31.4s, v19.4s, v13.4s\n" + "fmla v5.4s, v19.4s, v22.4s\n" + "fmax v31.4s, v31.4s, v27.4s\n" + "fmla v30.4s, v19.4s, v17.4s\n" + "fmla v29.4s, v19.4s, v16.4s\n" + "fmax v5.4s, v5.4s, v27.4s\n" + "fmax v30.4s, v30.4s, v27.4s\n" + "fmax v29.4s, v29.4s, v27.4s\n" + "fmin v31.4s, v31.4s, v15.4s\n" + "fmin v5.4s, v5.4s, v15.4s\n" + "str q31, [x14, x28]\n" + "fmin v30.4s, v30.4s, v15.4s\n" + "fmin v29.4s, v29.4s, v15.4s\n" + "str q5, [x13, x28]\n" "str q30, [x12, x28]\n" - "str q31, [x11, x28]\n" + "str q29, [x11, x28]\n" "3:" // Oddments "tst %x[n_channels], #0x3\n" "beq 60f\n" - "ldr q16, [x16, #0x0]\n" + "ldr q26, [x16, #0x0]\n" "ldr q0, [x16, #0x10]\n" - "mov x28, x10\n" - "add x14, x14, x28\n" + "mov x20, x10\n" + "add x14, x14, x20\n" "ldr q1, [x16, #0x20]\n" "ldr q2, [x16, #0x30]\n" - "add x13, x13, x28\n" - "add x12, x12, x28\n" + "add x13, x13, x20\n" + "add x12, x12, x20\n" "ldr q3, [x16, #0x40]\n" "ldr q4, [x16, #0x50]\n" - "add x11, x11, x28\n" + "add x11, x11, x20\n" "ldr x9, [x15, #0x0]\n" "ldr x28, [x15, #0x8]\n" "add x9, x9, x10\n" @@ -606,12 +606,12 @@ void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl( "ld1 { v10.s }[0], [x21], #0x4\n" "ld1 { v14.s }[0], [x20], #0x4\n" "5:" // Oddments: Load inputs (0, 0), (0, 1), (1, 0), (1, 1), (0, 2), (1, 2), (0, 3), (0, 4), (1, 5), (2, 0): Bit 1: End - "mov v28.16b, v16.16b\n fmla v28.4s, v0.4s, v5.4s\n" - "mov v29.16b, v16.16b\n fmla v29.4s, v0.4s, v6.4s\n" + "mov v28.16b, v26.16b\n fmla v28.4s, v0.4s, v5.4s\n" + "mov v29.16b, v26.16b\n fmla v29.4s, v0.4s, v6.4s\n" "ldr x20, [x15, #0x50]\n" "add x20, x20, x10\n" - "mov v30.16b, v16.16b\n fmla v30.4s, v0.4s, v7.4s\n" - "mov v31.16b, v16.16b\n fmla v31.4s, v0.4s, v8.4s\n" + "mov v30.16b, v26.16b\n fmla v30.4s, v0.4s, v7.4s\n" + "mov v31.16b, v26.16b\n fmla v31.4s, v0.4s, v8.4s\n" "fmla v28.4s, v1.4s, v6.4s\n" "fmla v29.4s, v1.4s, v9.4s\n" "fmla v30.4s, v1.4s, v8.4s\n" @@ -1005,14 +1005,14 @@ void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl( "ld1 { v9.s }[0], [x20], #0x4\n" "57:" // Oddments: Load input (5, 5): Bit 1: End "fmla v31.4s, v4.4s, v9.4s\n" - "fmax v28.4s, v28.4s, v18.4s\n" - "fmax v29.4s, v29.4s, v18.4s\n" - "fmax v30.4s, v30.4s, v18.4s\n" - "fmax v31.4s, v31.4s, v18.4s\n" - "fmin v28.4s, v28.4s, v17.4s\n" - "fmin v29.4s, v29.4s, v17.4s\n" - "fmin v30.4s, v30.4s, v17.4s\n" - "fmin v31.4s, v31.4s, v17.4s\n" + "fmax v28.4s, v28.4s, v27.4s\n" + "fmax v29.4s, v29.4s, v27.4s\n" + "fmax v30.4s, v30.4s, v27.4s\n" + "fmax v31.4s, v31.4s, v27.4s\n" + "fmin v28.4s, v28.4s, v15.4s\n" + "fmin v29.4s, v29.4s, v15.4s\n" + "fmin v30.4s, v30.4s, v15.4s\n" + "fmin v31.4s, v31.4s, v15.4s\n" "tbz %x[n_channels], #1, 58f\n" "st1 { v28.d }[0], [x14], #0x8\n" "st1 { v29.d }[0], [x13], #0x8\n" @@ -1030,12 +1030,10 @@ void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl( "st1 { v30.s }[0], [x12], #0x4\n" "st1 { v31.s }[0], [x11], #0x4\n" "59:" // Oddments: Store: Bit 1: End - "60:" // End - : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v16", "v17", "v18", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_generic_output9_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_generic_output9_mla_depthfirst.hpp index 6fa02b781ecca7fee37b168176bfc0c122ffacb2..8a8060770cd40a046edb12ab1cea9d2ed975d89d 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_generic_output9_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_generic_output9_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include @@ -48,4 +48,4 @@ class a64_fp32_nhwc_generic_output9_mla_depthfirst : public GenericDepthfirstKer } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_generic_output9_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_generic_output9_mla_depthfirst/generic.cpp index 0ea3a8fbeda92d7e9da14f17cb0bf375e6a7ef68..a2f577784ffd6f3a3526e86d6b5f902eb976b2fc 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_generic_output9_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_generic_output9_mla_depthfirst/generic.cpp @@ -22,10 +22,11 @@ * SOFTWARE. */ -#if defined(__aarch64__) #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace depthwise { @@ -44,70 +45,70 @@ void a64_fp32_nhwc_generic_output9_mla_depthfirst_impl( __asm__ __volatile__( "ld1r { v2.4s }, [%x[minmax_vals]]\n" - "lsr x12, %x[n_channels], #0x2\n" + "lsr x9, %x[n_channels], #0x2\n" "add x20, %x[minmax_vals], #0x4\n" "ld1r { v1.4s }, [x20]\n" "mov x11, #0x0\n" - "cbz x12, 5f\n" + "cbz x9, 5f\n" "1:" // Channel loop "movi v23.16b, #0x0\n" "cbz %x[bias], 2f\n" "ldr q23, [%x[bias], x11]\n" "2:" // Channel loop: Load bias: Done "ldr q0, [%x[params], #0x0]\n" - "mov x21, %x[inptrs]\n" - "ldp x10, x9, [x21], #0x10\n" - "subs x20, %x[n_points], #0x1\n" - "ldr q14, [x10, x11]\n" - "ldr q15, [x9, x11]\n" + "mov x26, %x[inptrs]\n" + "ldp x21, x20, [x26], #0x10\n" + "subs x25, %x[n_points], #0x1\n" + "ldr q14, [x21, x11]\n" + "ldr q15, [x20, x11]\n" "mov v24.16b, v23.16b\n" "mov v25.16b, v23.16b\n" - "ldp x28, x27, [x21], #0x10\n" - "ldr q16, [x28, x11]\n" + "ldp x21, x20, [x26], #0x10\n" + "ldr q16, [x21, x11]\n" "mov v26.16b, v23.16b\n" "mov v27.16b, v23.16b\n" - "ldr q17, [x27, x11]\n" - "ldp x26, x25, [x21], #0x10\n" + "ldr q17, [x20, x11]\n" + "ldp x21, x20, [x26], #0x10\n" "mov v28.16b, v23.16b\n" "mov v29.16b, v23.16b\n" - "ldr q18, [x26, x11]\n" - "ldr q19, [x25, x11]\n" + "ldr q18, [x21, x11]\n" + "ldr q19, [x20, x11]\n" "mov v30.16b, v23.16b\n" "mov v31.16b, v23.16b\n" - "ldp x24, x23, [x21], #0x10\n" - "ldr q20, [x24, x11]\n" + "ldp x21, x20, [x26], #0x10\n" + "ldr q20, [x21, x11]\n" "add %x[params], %x[params], #0x10\n" - "ldr q21, [x23, x11]\n" - "ldr x22, [x21], #0x8\n" - "ldr q22, [x22, x11]\n" + "ldr q21, [x20, x11]\n" + "ldr x20, [x26], #0x8\n" + "ldr q22, [x20, x11]\n" "ble 4f\n" "3:" // Channel loop: Planar loop - "ldp x10, x9, [x21], #0x10\n" - "ldp x28, x27, [x21], #0x10\n" - "subs x20, x20, #0x1\n" + "ldp x20, x24, [x26], #0x10\n" + "ldp x23, x22, [x26], #0x10\n" + "subs x25, x25, #0x1\n" "fmla v23.4s, v14.4s, v0.4s\n" - "ldr q14, [x10, x11]\n" - "ldp x26, x25, [x21], #0x10\n" + "ldr q14, [x20, x11]\n" + "ldp x21, x20, [x26], #0x10\n" "fmla v24.4s, v15.4s, v0.4s\n" "fmla v25.4s, v16.4s, v0.4s\n" - "ldr q15, [x9, x11]\n" - "ldr q16, [x28, x11]\n" + "ldr q15, [x24, x11]\n" + "ldr q16, [x23, x11]\n" "fmla v26.4s, v17.4s, v0.4s\n" "fmla v27.4s, v18.4s, v0.4s\n" - "ldr q17, [x27, x11]\n" - "ldr q18, [x26, x11]\n" + "ldr q17, [x22, x11]\n" + "ldr q18, [x21, x11]\n" "fmla v28.4s, v19.4s, v0.4s\n" "fmla v29.4s, v20.4s, v0.4s\n" - "ldr q19, [x25, x11]\n" - "ldp x24, x23, [x21], #0x10\n" + "ldr q19, [x20, x11]\n" + "ldp x21, x20, [x26], #0x10\n" "fmla v30.4s, v21.4s, v0.4s\n" "fmla v31.4s, v22.4s, v0.4s\n" "ldr q0, [%x[params], #0x0]\n" - "ldr q20, [x24, x11]\n" + "ldr q20, [x21, x11]\n" "add %x[params], %x[params], #0x10\n" - "ldr q21, [x23, x11]\n" - "ldr x22, [x21], #0x8\n" - "ldr q22, [x22, x11]\n" + "ldr q21, [x20, x11]\n" + "ldr x20, [x26], #0x8\n" + "ldr q22, [x20, x11]\n" "bgt 3b\n" "4:" // Channel loop: Planar tail "fmla v23.4s, v14.4s, v0.4s\n" @@ -152,7 +153,7 @@ void a64_fp32_nhwc_generic_output9_mla_depthfirst_impl( "str q30, [x21, x11]\n" "str q31, [x20, x11]\n" "add x11, x11, #0x10\n" - "cmp x11, x12, LSL #4\n" + "cmp x11, x9, LSL #4\n" "blt 1b\n" "5:" // Oddments "tst %x[n_channels], #0x3\n" @@ -170,121 +171,121 @@ void a64_fp32_nhwc_generic_output9_mla_depthfirst_impl( "7:" // Oddments: Load bias: Bit 1: End "8:" // Oddments: Load bias: Done "ldr q0, [%x[params], #0x0]\n" - "mov x21, %x[inptrs]\n" - "ldp x10, x9, [x21], #0x10\n" + "mov x10, %x[inptrs]\n" + "ldp x9, x28, [x10], #0x10\n" "mov v24.16b, v23.16b\n" - "ldp x28, x27, [x21], #0x10\n" - "ldp x26, x25, [x21], #0x10\n" + "ldp x27, x26, [x10], #0x10\n" + "ldp x25, x24, [x10], #0x10\n" "mov v25.16b, v23.16b\n" "mov v26.16b, v23.16b\n" - "ldp x24, x23, [x21], #0x10\n" - "ldr x22, [x21], #0x8\n" + "ldp x23, x22, [x10], #0x10\n" + "ldr x21, [x10], #0x8\n" "mov v27.16b, v23.16b\n" "mov v28.16b, v23.16b\n" "mov v29.16b, v23.16b\n" "mov v30.16b, v23.16b\n" - "add x10, x10, x11\n" "add x9, x9, x11\n" - "mov v31.16b, v23.16b\n" "add x28, x28, x11\n" + "mov v31.16b, v23.16b\n" "add x27, x27, x11\n" "add x26, x26, x11\n" "add x25, x25, x11\n" "add x24, x24, x11\n" "add x23, x23, x11\n" "add x22, x22, x11\n" + "add x21, x21, x11\n" "add %x[params], %x[params], #0x10\n" "tbz %x[n_channels], #1, 9f\n" - "ldr d14, [x10], #0x8\n" - "ldr d15, [x9], #0x8\n" - "ldr d16, [x28], #0x8\n" - "ldr d17, [x27], #0x8\n" - "ldr d18, [x26], #0x8\n" - "ldr d19, [x25], #0x8\n" - "ldr d20, [x24], #0x8\n" - "ldr d21, [x23], #0x8\n" - "ldr d22, [x22], #0x8\n" + "ldr d14, [x9], #0x8\n" + "ldr d15, [x28], #0x8\n" + "ldr d16, [x27], #0x8\n" + "ldr d17, [x26], #0x8\n" + "ldr d18, [x25], #0x8\n" + "ldr d19, [x24], #0x8\n" + "ldr d20, [x23], #0x8\n" + "ldr d21, [x22], #0x8\n" + "ldr d22, [x21], #0x8\n" "tbz %x[n_channels], #0, 10f\n" - "ld1 { v14.s }[2], [x10], #0x4\n" - "ld1 { v15.s }[2], [x9], #0x4\n" - "ld1 { v16.s }[2], [x28], #0x4\n" - "ld1 { v17.s }[2], [x27], #0x4\n" - "ld1 { v18.s }[2], [x26], #0x4\n" - "ld1 { v19.s }[2], [x25], #0x4\n" - "ld1 { v20.s }[2], [x24], #0x4\n" - "ld1 { v21.s }[2], [x23], #0x4\n" - "ld1 { v22.s }[2], [x22], #0x4\n" + "ld1 { v14.s }[2], [x9], #0x4\n" + "ld1 { v15.s }[2], [x28], #0x4\n" + "ld1 { v16.s }[2], [x27], #0x4\n" + "ld1 { v17.s }[2], [x26], #0x4\n" + "ld1 { v18.s }[2], [x25], #0x4\n" + "ld1 { v19.s }[2], [x24], #0x4\n" + "ld1 { v20.s }[2], [x23], #0x4\n" + "ld1 { v21.s }[2], [x22], #0x4\n" + "ld1 { v22.s }[2], [x21], #0x4\n" "b 10f\n" "9:" // Oddments: Load: Bit 1: Unset - "ldr s14, [x10], #0x4\n" - "ldr s15, [x9], #0x4\n" - "ldr s16, [x28], #0x4\n" - "ldr s17, [x27], #0x4\n" - "ldr s18, [x26], #0x4\n" - "ldr s19, [x25], #0x4\n" - "ldr s20, [x24], #0x4\n" - "ldr s21, [x23], #0x4\n" - "ldr s22, [x22], #0x4\n" + "ldr s14, [x9], #0x4\n" + "ldr s15, [x28], #0x4\n" + "ldr s16, [x27], #0x4\n" + "ldr s17, [x26], #0x4\n" + "ldr s18, [x25], #0x4\n" + "ldr s19, [x24], #0x4\n" + "ldr s20, [x23], #0x4\n" + "ldr s21, [x22], #0x4\n" + "ldr s22, [x21], #0x4\n" "10:" // Oddments: Load: Bit 1: End "subs x20, %x[n_points], #0x1\n" "ble 14f\n" "11:" // Oddments: Planar loop - "ldp x10, x9, [x21], #0x10\n" - "ldp x28, x27, [x21], #0x10\n" + "ldp x9, x28, [x10], #0x10\n" + "ldp x27, x26, [x10], #0x10\n" "fmla v23.4s, v14.4s, v0.4s\n" "fmla v24.4s, v15.4s, v0.4s\n" - "ldp x26, x25, [x21], #0x10\n" - "ldp x24, x23, [x21], #0x10\n" + "ldp x25, x24, [x10], #0x10\n" + "ldp x23, x22, [x10], #0x10\n" "fmla v25.4s, v16.4s, v0.4s\n" "fmla v26.4s, v17.4s, v0.4s\n" - "ldr x22, [x21], #0x8\n" + "ldr x21, [x10], #0x8\n" "fmla v27.4s, v18.4s, v0.4s\n" "fmla v28.4s, v19.4s, v0.4s\n" - "add x10, x10, x11\n" + "add x9, x9, x11\n" "fmla v29.4s, v20.4s, v0.4s\n" "fmla v30.4s, v21.4s, v0.4s\n" - "add x9, x9, x11\n" "add x28, x28, x11\n" + "add x27, x27, x11\n" "fmla v31.4s, v22.4s, v0.4s\n" "ldr q0, [%x[params], #0x0]\n" - "add x27, x27, x11\n" "add x26, x26, x11\n" "add x25, x25, x11\n" "add x24, x24, x11\n" "add x23, x23, x11\n" "add x22, x22, x11\n" + "add x21, x21, x11\n" "add %x[params], %x[params], #0x10\n" "tbz %x[n_channels], #1, 12f\n" - "ldr d14, [x10], #0x8\n" - "ldr d15, [x9], #0x8\n" - "ldr d16, [x28], #0x8\n" - "ldr d17, [x27], #0x8\n" - "ldr d18, [x26], #0x8\n" - "ldr d19, [x25], #0x8\n" - "ldr d20, [x24], #0x8\n" - "ldr d21, [x23], #0x8\n" - "ldr d22, [x22], #0x8\n" + "ldr d14, [x9], #0x8\n" + "ldr d15, [x28], #0x8\n" + "ldr d16, [x27], #0x8\n" + "ldr d17, [x26], #0x8\n" + "ldr d18, [x25], #0x8\n" + "ldr d19, [x24], #0x8\n" + "ldr d20, [x23], #0x8\n" + "ldr d21, [x22], #0x8\n" + "ldr d22, [x21], #0x8\n" "tbz %x[n_channels], #0, 13f\n" - "ld1 { v14.s }[2], [x10], #0x4\n" - "ld1 { v15.s }[2], [x9], #0x4\n" - "ld1 { v16.s }[2], [x28], #0x4\n" - "ld1 { v17.s }[2], [x27], #0x4\n" - "ld1 { v18.s }[2], [x26], #0x4\n" - "ld1 { v19.s }[2], [x25], #0x4\n" - "ld1 { v20.s }[2], [x24], #0x4\n" - "ld1 { v21.s }[2], [x23], #0x4\n" - "ld1 { v22.s }[2], [x22], #0x4\n" + "ld1 { v14.s }[2], [x9], #0x4\n" + "ld1 { v15.s }[2], [x28], #0x4\n" + "ld1 { v16.s }[2], [x27], #0x4\n" + "ld1 { v17.s }[2], [x26], #0x4\n" + "ld1 { v18.s }[2], [x25], #0x4\n" + "ld1 { v19.s }[2], [x24], #0x4\n" + "ld1 { v20.s }[2], [x23], #0x4\n" + "ld1 { v21.s }[2], [x22], #0x4\n" + "ld1 { v22.s }[2], [x21], #0x4\n" "b 13f\n" "12:" // Oddments: Planar loop: Load: Bit 1: Unset - "ldr s14, [x10], #0x4\n" - "ldr s15, [x9], #0x4\n" - "ldr s16, [x28], #0x4\n" - "ldr s17, [x27], #0x4\n" - "ldr s18, [x26], #0x4\n" - "ldr s19, [x25], #0x4\n" - "ldr s20, [x24], #0x4\n" - "ldr s21, [x23], #0x4\n" - "ldr s22, [x22], #0x4\n" + "ldr s14, [x9], #0x4\n" + "ldr s15, [x28], #0x4\n" + "ldr s16, [x27], #0x4\n" + "ldr s17, [x26], #0x4\n" + "ldr s18, [x25], #0x4\n" + "ldr s19, [x24], #0x4\n" + "ldr s20, [x23], #0x4\n" + "ldr s21, [x22], #0x4\n" + "ldr s22, [x21], #0x4\n" "13:" // Oddments: Planar loop: Load: Bit 1: End "subs x20, x20, #0x1\n" "bgt 11b\n" @@ -365,10 +366,11 @@ void a64_fp32_nhwc_generic_output9_mla_depthfirst_impl( "17:" // End : [params] "+&r" (params) : [bias] "r" (bias), [inptrs] "r" (inptrs), [minmax_vals] "r" (minmax_vals), [n_channels] "r" ((uint64_t) n_channels), [n_points] "r" ((uint64_t) n_points), [outptrs] "r" (outptrs) - : "cc", "memory", "v0", "v1", "v2", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst.hpp index 2ec0525226101d2e35c640c64efcb7e8257cb56a..6c07fa645c783d7cf4105c8ce4e0360c99fd3e87 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include @@ -58,4 +58,4 @@ struct a64_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst : } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) \ No newline at end of file +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst/generic.cpp index 69b3865a65230069cf9dd55537380fb1ec38d424..9cafd23fb83737f8b0bf9e92de4d3f13715bee56 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst/generic.cpp @@ -22,10 +22,11 @@ * SOFTWARE. */ -#if defined(__aarch64__) #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace depthwise { @@ -41,7 +42,7 @@ void a64_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst_imp const float minmax_vals[2] = { activation_min, activation_max }; __asm__ __volatile__( - "ld1r { v24.4s }, [%x[clamps]]\n" + "ld1r { v27.4s }, [%x[clamps]]\n" "ldr x21, [%x[inptrs], #0x0]\n" "lsr x22, %x[channel_multiplier], #0x2\n" "add x20, %x[clamps], #0x4\n" @@ -49,7 +50,7 @@ void a64_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst_imp "ldr q1, [x21, #0x10]\n" "mov x21, #0x0\n" "mov x14, #0x0\n" - "ld1r { v23.4s }, [x20]\n" + "ld1r { v26.4s }, [x20]\n" "ldr x20, [%x[inptrs], #0x8]\n" "ldr q2, [x20, #0x0]\n" "ldr q3, [x20, #0x10]\n" @@ -101,7 +102,7 @@ void a64_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst_imp "fmla v20.4s, v31.4s, v8.s[0]\n" "fmla v21.4s, v31.4s, v8.s[2]\n" "fmla v22.4s, v31.4s, v9.s[0]\n" - "ldr q31, [%x[params], #0x0]\n" + "ldr q25, [%x[params], #0x0]\n" "fmla v14.4s, v30.4s, v0.s[1]\n" "fmla v15.4s, v30.4s, v0.s[3]\n" "fmla v16.4s, v30.4s, v1.s[1]\n" @@ -111,7 +112,7 @@ void a64_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst_imp "fmla v20.4s, v30.4s, v8.s[1]\n" "fmla v21.4s, v30.4s, v8.s[3]\n" "fmla v22.4s, v30.4s, v9.s[1]\n" - "ldr q30, [%x[params], #0x10]\n" + "ldr q24, [%x[params], #0x10]\n" "fmla v14.4s, v29.4s, v0.s[2]\n" "fmla v15.4s, v29.4s, v1.s[0]\n" "fmla v16.4s, v29.4s, v1.s[2]\n" @@ -121,92 +122,92 @@ void a64_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst_imp "fmla v20.4s, v29.4s, v8.s[2]\n" "fmla v21.4s, v29.4s, v9.s[0]\n" "fmla v22.4s, v29.4s, v9.s[2]\n" - "ldr q29, [%x[params], #0x20]\n" - "fmla v14.4s, v31.4s, v2.s[0]\n" - "fmla v15.4s, v31.4s, v2.s[2]\n" - "fmla v16.4s, v31.4s, v3.s[0]\n" - "fmla v17.4s, v31.4s, v6.s[0]\n" - "fmla v18.4s, v31.4s, v6.s[2]\n" - "fmla v19.4s, v31.4s, v7.s[0]\n" - "fmla v20.4s, v31.4s, v10.s[0]\n" - "fmla v21.4s, v31.4s, v10.s[2]\n" - "fmla v22.4s, v31.4s, v11.s[0]\n" - "ldr q31, [%x[params], #0x30]\n" - "fmla v14.4s, v30.4s, v2.s[1]\n" - "fmla v15.4s, v30.4s, v2.s[3]\n" - "fmla v16.4s, v30.4s, v3.s[1]\n" - "fmla v17.4s, v30.4s, v6.s[1]\n" - "fmla v18.4s, v30.4s, v6.s[3]\n" - "fmla v19.4s, v30.4s, v7.s[1]\n" - "fmla v20.4s, v30.4s, v10.s[1]\n" - "fmla v21.4s, v30.4s, v10.s[3]\n" - "fmla v22.4s, v30.4s, v11.s[1]\n" - "ldr q30, [%x[params], #0x40]\n" - "fmla v14.4s, v29.4s, v2.s[2]\n" - "fmla v15.4s, v29.4s, v3.s[0]\n" - "fmla v16.4s, v29.4s, v3.s[2]\n" - "fmla v17.4s, v29.4s, v6.s[2]\n" - "fmla v18.4s, v29.4s, v7.s[0]\n" - "fmla v19.4s, v29.4s, v7.s[2]\n" - "fmla v20.4s, v29.4s, v10.s[2]\n" - "fmla v21.4s, v29.4s, v11.s[0]\n" - "fmla v22.4s, v29.4s, v11.s[2]\n" - "ldr q29, [%x[params], #0x50]\n" - "fmla v14.4s, v31.4s, v4.s[0]\n" - "fmla v15.4s, v31.4s, v4.s[2]\n" - "fmla v16.4s, v31.4s, v5.s[0]\n" - "fmla v17.4s, v31.4s, v8.s[0]\n" - "fmla v18.4s, v31.4s, v8.s[2]\n" - "fmla v19.4s, v31.4s, v9.s[0]\n" - "fmla v20.4s, v31.4s, v12.s[0]\n" - "fmla v21.4s, v31.4s, v12.s[2]\n" - "fmla v22.4s, v31.4s, v13.s[0]\n" + "ldr q23, [%x[params], #0x20]\n" + "fmla v14.4s, v25.4s, v2.s[0]\n" + "fmla v15.4s, v25.4s, v2.s[2]\n" + "fmla v16.4s, v25.4s, v3.s[0]\n" + "fmla v17.4s, v25.4s, v6.s[0]\n" + "fmla v18.4s, v25.4s, v6.s[2]\n" + "fmla v19.4s, v25.4s, v7.s[0]\n" + "fmla v20.4s, v25.4s, v10.s[0]\n" + "fmla v21.4s, v25.4s, v10.s[2]\n" + "fmla v22.4s, v25.4s, v11.s[0]\n" + "ldr q25, [%x[params], #0x30]\n" + "fmla v14.4s, v24.4s, v2.s[1]\n" + "fmla v15.4s, v24.4s, v2.s[3]\n" + "fmla v16.4s, v24.4s, v3.s[1]\n" + "fmla v17.4s, v24.4s, v6.s[1]\n" + "fmla v18.4s, v24.4s, v6.s[3]\n" + "fmla v19.4s, v24.4s, v7.s[1]\n" + "fmla v20.4s, v24.4s, v10.s[1]\n" + "fmla v21.4s, v24.4s, v10.s[3]\n" + "fmla v22.4s, v24.4s, v11.s[1]\n" + "ldr q24, [%x[params], #0x40]\n" + "fmla v14.4s, v23.4s, v2.s[2]\n" + "fmla v15.4s, v23.4s, v3.s[0]\n" + "fmla v16.4s, v23.4s, v3.s[2]\n" + "fmla v17.4s, v23.4s, v6.s[2]\n" + "fmla v18.4s, v23.4s, v7.s[0]\n" + "fmla v19.4s, v23.4s, v7.s[2]\n" + "fmla v20.4s, v23.4s, v10.s[2]\n" + "fmla v21.4s, v23.4s, v11.s[0]\n" + "fmla v22.4s, v23.4s, v11.s[2]\n" + "ldr q23, [%x[params], #0x50]\n" + "fmla v14.4s, v25.4s, v4.s[0]\n" + "fmla v15.4s, v25.4s, v4.s[2]\n" + "fmla v16.4s, v25.4s, v5.s[0]\n" + "fmla v17.4s, v25.4s, v8.s[0]\n" + "fmla v18.4s, v25.4s, v8.s[2]\n" + "fmla v19.4s, v25.4s, v9.s[0]\n" + "fmla v20.4s, v25.4s, v12.s[0]\n" + "fmla v21.4s, v25.4s, v12.s[2]\n" + "fmla v22.4s, v25.4s, v13.s[0]\n" "ldr q31, [%x[params], #0x70]\n" - "fmla v14.4s, v30.4s, v4.s[1]\n" - "fmla v15.4s, v30.4s, v4.s[3]\n" - "fmla v16.4s, v30.4s, v5.s[1]\n" - "fmla v17.4s, v30.4s, v8.s[1]\n" - "fmla v18.4s, v30.4s, v8.s[3]\n" - "fmla v19.4s, v30.4s, v9.s[1]\n" - "fmla v20.4s, v30.4s, v12.s[1]\n" - "fmla v21.4s, v30.4s, v12.s[3]\n" - "fmla v22.4s, v30.4s, v13.s[1]\n" + "fmla v14.4s, v24.4s, v4.s[1]\n" + "fmla v15.4s, v24.4s, v4.s[3]\n" + "fmla v16.4s, v24.4s, v5.s[1]\n" + "fmla v17.4s, v24.4s, v8.s[1]\n" + "fmla v18.4s, v24.4s, v8.s[3]\n" + "fmla v19.4s, v24.4s, v9.s[1]\n" + "fmla v20.4s, v24.4s, v12.s[1]\n" + "fmla v21.4s, v24.4s, v12.s[3]\n" + "fmla v22.4s, v24.4s, v13.s[1]\n" "ldr q30, [%x[params], #0x80]\n" - "fmla v14.4s, v29.4s, v4.s[2]\n" - "fmla v15.4s, v29.4s, v5.s[0]\n" - "fmin v14.4s, v14.4s, v23.4s\n" - "fmla v16.4s, v29.4s, v5.s[2]\n" - "fmla v17.4s, v29.4s, v8.s[2]\n" - "fmax v14.4s, v14.4s, v24.4s\n" + "fmla v14.4s, v23.4s, v4.s[2]\n" + "fmla v15.4s, v23.4s, v5.s[0]\n" + "fmin v14.4s, v14.4s, v26.4s\n" + "fmla v16.4s, v23.4s, v5.s[2]\n" + "fmla v17.4s, v23.4s, v8.s[2]\n" + "fmax v14.4s, v14.4s, v27.4s\n" "str q14, [x13, x14]\n" "ldr q14, [%x[params], #0x60]\n" - "fmla v18.4s, v29.4s, v9.s[0]\n" - "fmla v19.4s, v29.4s, v9.s[2]\n" - "fmin v15.4s, v15.4s, v23.4s\n" - "fmla v20.4s, v29.4s, v12.s[2]\n" - "fmla v21.4s, v29.4s, v13.s[0]\n" - "fmin v16.4s, v16.4s, v23.4s\n" - "fmla v22.4s, v29.4s, v13.s[2]\n" + "fmla v18.4s, v23.4s, v9.s[0]\n" + "fmla v19.4s, v23.4s, v9.s[2]\n" + "fmin v15.4s, v15.4s, v26.4s\n" + "fmla v20.4s, v23.4s, v12.s[2]\n" + "fmla v21.4s, v23.4s, v13.s[0]\n" + "fmin v16.4s, v16.4s, v26.4s\n" + "fmla v22.4s, v23.4s, v13.s[2]\n" "ldr q29, [%x[params], #0x90]\n" - "fmin v17.4s, v17.4s, v23.4s\n" + "fmin v17.4s, v17.4s, v26.4s\n" "add %x[params], %x[params], #0xa0\n" - "fmin v18.4s, v18.4s, v23.4s\n" - "fmin v19.4s, v19.4s, v23.4s\n" - "fmin v20.4s, v20.4s, v23.4s\n" - "fmin v21.4s, v21.4s, v23.4s\n" - "fmin v22.4s, v22.4s, v23.4s\n" - "fmax v15.4s, v15.4s, v24.4s\n" + "fmin v18.4s, v18.4s, v26.4s\n" + "fmin v19.4s, v19.4s, v26.4s\n" + "fmin v20.4s, v20.4s, v26.4s\n" + "fmin v21.4s, v21.4s, v26.4s\n" + "fmin v22.4s, v22.4s, v26.4s\n" + "fmax v15.4s, v15.4s, v27.4s\n" "str q15, [x12, x14]\n" - "fmax v16.4s, v16.4s, v24.4s\n" - "fmax v17.4s, v17.4s, v24.4s\n" + "fmax v16.4s, v16.4s, v27.4s\n" + "fmax v17.4s, v17.4s, v27.4s\n" "str q16, [x11, x14]\n" - "fmax v18.4s, v18.4s, v24.4s\n" - "fmax v19.4s, v19.4s, v24.4s\n" + "fmax v18.4s, v18.4s, v27.4s\n" + "fmax v19.4s, v19.4s, v27.4s\n" "str q17, [x10, x14]\n" - "fmax v20.4s, v20.4s, v24.4s\n" - "fmax v21.4s, v21.4s, v24.4s\n" + "fmax v20.4s, v20.4s, v27.4s\n" + "fmax v21.4s, v21.4s, v27.4s\n" "str q18, [x9, x14]\n" - "fmax v22.4s, v22.4s, v24.4s\n" + "fmax v22.4s, v22.4s, v27.4s\n" "str q19, [x28, x14]\n" "mov v15.16b, v14.16b\n" "str q20, [x27, x14]\n" @@ -231,7 +232,7 @@ void a64_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst_imp "fmla v20.4s, v31.4s, v8.s[0]\n" "fmla v21.4s, v31.4s, v8.s[2]\n" "fmla v22.4s, v31.4s, v9.s[0]\n" - "ldr q31, [%x[params], #0x0]\n" + "ldr q25, [%x[params], #0x0]\n" "fmla v14.4s, v30.4s, v0.s[1]\n" "fmla v15.4s, v30.4s, v0.s[3]\n" "fmla v16.4s, v30.4s, v1.s[1]\n" @@ -241,7 +242,7 @@ void a64_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst_imp "fmla v20.4s, v30.4s, v8.s[1]\n" "fmla v21.4s, v30.4s, v8.s[3]\n" "fmla v22.4s, v30.4s, v9.s[1]\n" - "ldr q30, [%x[params], #0x10]\n" + "ldr q24, [%x[params], #0x10]\n" "fmla v14.4s, v29.4s, v0.s[2]\n" "fmla v15.4s, v29.4s, v1.s[0]\n" "fmla v16.4s, v29.4s, v1.s[2]\n" @@ -251,87 +252,87 @@ void a64_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst_imp "fmla v20.4s, v29.4s, v8.s[2]\n" "fmla v21.4s, v29.4s, v9.s[0]\n" "fmla v22.4s, v29.4s, v9.s[2]\n" - "ldr q29, [%x[params], #0x20]\n" - "fmla v14.4s, v31.4s, v2.s[0]\n" - "fmla v15.4s, v31.4s, v2.s[2]\n" - "fmla v16.4s, v31.4s, v3.s[0]\n" - "fmla v17.4s, v31.4s, v6.s[0]\n" - "fmla v18.4s, v31.4s, v6.s[2]\n" - "fmla v19.4s, v31.4s, v7.s[0]\n" - "fmla v20.4s, v31.4s, v10.s[0]\n" - "fmla v21.4s, v31.4s, v10.s[2]\n" - "fmla v22.4s, v31.4s, v11.s[0]\n" - "ldr q31, [%x[params], #0x30]\n" - "fmla v14.4s, v30.4s, v2.s[1]\n" - "fmla v15.4s, v30.4s, v2.s[3]\n" - "fmla v16.4s, v30.4s, v3.s[1]\n" - "fmla v17.4s, v30.4s, v6.s[1]\n" - "fmla v18.4s, v30.4s, v6.s[3]\n" - "fmla v19.4s, v30.4s, v7.s[1]\n" - "fmla v20.4s, v30.4s, v10.s[1]\n" - "fmla v21.4s, v30.4s, v10.s[3]\n" - "fmla v22.4s, v30.4s, v11.s[1]\n" - "ldr q30, [%x[params], #0x40]\n" - "fmla v14.4s, v29.4s, v2.s[2]\n" - "fmla v15.4s, v29.4s, v3.s[0]\n" - "fmla v16.4s, v29.4s, v3.s[2]\n" - "fmla v17.4s, v29.4s, v6.s[2]\n" - "fmla v18.4s, v29.4s, v7.s[0]\n" - "fmla v19.4s, v29.4s, v7.s[2]\n" - "fmla v20.4s, v29.4s, v10.s[2]\n" - "fmla v21.4s, v29.4s, v11.s[0]\n" - "fmla v22.4s, v29.4s, v11.s[2]\n" - "ldr q29, [%x[params], #0x50]\n" + "ldr q23, [%x[params], #0x20]\n" + "fmla v14.4s, v25.4s, v2.s[0]\n" + "fmla v15.4s, v25.4s, v2.s[2]\n" + "fmla v16.4s, v25.4s, v3.s[0]\n" + "fmla v17.4s, v25.4s, v6.s[0]\n" + "fmla v18.4s, v25.4s, v6.s[2]\n" + "fmla v19.4s, v25.4s, v7.s[0]\n" + "fmla v20.4s, v25.4s, v10.s[0]\n" + "fmla v21.4s, v25.4s, v10.s[2]\n" + "fmla v22.4s, v25.4s, v11.s[0]\n" + "ldr q25, [%x[params], #0x30]\n" + "fmla v14.4s, v24.4s, v2.s[1]\n" + "fmla v15.4s, v24.4s, v2.s[3]\n" + "fmla v16.4s, v24.4s, v3.s[1]\n" + "fmla v17.4s, v24.4s, v6.s[1]\n" + "fmla v18.4s, v24.4s, v6.s[3]\n" + "fmla v19.4s, v24.4s, v7.s[1]\n" + "fmla v20.4s, v24.4s, v10.s[1]\n" + "fmla v21.4s, v24.4s, v10.s[3]\n" + "fmla v22.4s, v24.4s, v11.s[1]\n" + "ldr q24, [%x[params], #0x40]\n" + "fmla v14.4s, v23.4s, v2.s[2]\n" + "fmla v15.4s, v23.4s, v3.s[0]\n" + "fmla v16.4s, v23.4s, v3.s[2]\n" + "fmla v17.4s, v23.4s, v6.s[2]\n" + "fmla v18.4s, v23.4s, v7.s[0]\n" + "fmla v19.4s, v23.4s, v7.s[2]\n" + "fmla v20.4s, v23.4s, v10.s[2]\n" + "fmla v21.4s, v23.4s, v11.s[0]\n" + "fmla v22.4s, v23.4s, v11.s[2]\n" + "ldr q23, [%x[params], #0x50]\n" "add %x[params], %x[params], #0x60\n" - "fmla v14.4s, v31.4s, v4.s[0]\n" - "fmla v15.4s, v31.4s, v4.s[2]\n" - "fmla v16.4s, v31.4s, v5.s[0]\n" - "fmla v17.4s, v31.4s, v8.s[0]\n" - "fmla v18.4s, v31.4s, v8.s[2]\n" - "fmla v19.4s, v31.4s, v9.s[0]\n" - "fmla v20.4s, v31.4s, v12.s[0]\n" - "fmla v21.4s, v31.4s, v12.s[2]\n" - "fmla v22.4s, v31.4s, v13.s[0]\n" - "fmla v14.4s, v30.4s, v4.s[1]\n" - "fmla v15.4s, v30.4s, v4.s[3]\n" - "fmla v16.4s, v30.4s, v5.s[1]\n" - "fmla v17.4s, v30.4s, v8.s[1]\n" - "fmla v18.4s, v30.4s, v8.s[3]\n" - "fmla v19.4s, v30.4s, v9.s[1]\n" - "fmla v20.4s, v30.4s, v12.s[1]\n" - "fmla v21.4s, v30.4s, v12.s[3]\n" - "fmla v22.4s, v30.4s, v13.s[1]\n" - "fmla v14.4s, v29.4s, v4.s[2]\n" - "fmla v15.4s, v29.4s, v5.s[0]\n" - "fmin v14.4s, v14.4s, v23.4s\n" - "fmla v16.4s, v29.4s, v5.s[2]\n" - "fmla v17.4s, v29.4s, v8.s[2]\n" - "fmin v15.4s, v15.4s, v23.4s\n" - "fmla v18.4s, v29.4s, v9.s[0]\n" - "fmla v19.4s, v29.4s, v9.s[2]\n" - "fmin v16.4s, v16.4s, v23.4s\n" - "fmla v20.4s, v29.4s, v12.s[2]\n" - "fmla v21.4s, v29.4s, v13.s[0]\n" - "fmin v17.4s, v17.4s, v23.4s\n" - "fmla v22.4s, v29.4s, v13.s[2]\n" - "fmin v18.4s, v18.4s, v23.4s\n" - "fmin v19.4s, v19.4s, v23.4s\n" - "fmin v20.4s, v20.4s, v23.4s\n" - "fmin v21.4s, v21.4s, v23.4s\n" - "fmin v22.4s, v22.4s, v23.4s\n" - "fmax v14.4s, v14.4s, v24.4s\n" - "fmax v15.4s, v15.4s, v24.4s\n" + "fmla v14.4s, v25.4s, v4.s[0]\n" + "fmla v15.4s, v25.4s, v4.s[2]\n" + "fmla v16.4s, v25.4s, v5.s[0]\n" + "fmla v17.4s, v25.4s, v8.s[0]\n" + "fmla v18.4s, v25.4s, v8.s[2]\n" + "fmla v19.4s, v25.4s, v9.s[0]\n" + "fmla v20.4s, v25.4s, v12.s[0]\n" + "fmla v21.4s, v25.4s, v12.s[2]\n" + "fmla v22.4s, v25.4s, v13.s[0]\n" + "fmla v14.4s, v24.4s, v4.s[1]\n" + "fmla v15.4s, v24.4s, v4.s[3]\n" + "fmla v16.4s, v24.4s, v5.s[1]\n" + "fmla v17.4s, v24.4s, v8.s[1]\n" + "fmla v18.4s, v24.4s, v8.s[3]\n" + "fmla v19.4s, v24.4s, v9.s[1]\n" + "fmla v20.4s, v24.4s, v12.s[1]\n" + "fmla v21.4s, v24.4s, v12.s[3]\n" + "fmla v22.4s, v24.4s, v13.s[1]\n" + "fmla v14.4s, v23.4s, v4.s[2]\n" + "fmla v15.4s, v23.4s, v5.s[0]\n" + "fmin v14.4s, v14.4s, v26.4s\n" + "fmla v16.4s, v23.4s, v5.s[2]\n" + "fmla v17.4s, v23.4s, v8.s[2]\n" + "fmin v15.4s, v15.4s, v26.4s\n" + "fmla v18.4s, v23.4s, v9.s[0]\n" + "fmla v19.4s, v23.4s, v9.s[2]\n" + "fmin v16.4s, v16.4s, v26.4s\n" + "fmla v20.4s, v23.4s, v12.s[2]\n" + "fmla v21.4s, v23.4s, v13.s[0]\n" + "fmin v17.4s, v17.4s, v26.4s\n" + "fmla v22.4s, v23.4s, v13.s[2]\n" + "fmin v18.4s, v18.4s, v26.4s\n" + "fmin v19.4s, v19.4s, v26.4s\n" + "fmin v20.4s, v20.4s, v26.4s\n" + "fmin v21.4s, v21.4s, v26.4s\n" + "fmin v22.4s, v22.4s, v26.4s\n" + "fmax v14.4s, v14.4s, v27.4s\n" + "fmax v15.4s, v15.4s, v27.4s\n" "str q14, [x13, x14]\n" - "fmax v16.4s, v16.4s, v24.4s\n" - "fmax v17.4s, v17.4s, v24.4s\n" + "fmax v16.4s, v16.4s, v27.4s\n" + "fmax v17.4s, v17.4s, v27.4s\n" "str q15, [x12, x14]\n" - "fmax v18.4s, v18.4s, v24.4s\n" - "fmax v19.4s, v19.4s, v24.4s\n" + "fmax v18.4s, v18.4s, v27.4s\n" + "fmax v19.4s, v19.4s, v27.4s\n" "str q16, [x11, x14]\n" - "fmax v20.4s, v20.4s, v24.4s\n" - "fmax v21.4s, v21.4s, v24.4s\n" + "fmax v20.4s, v20.4s, v27.4s\n" + "fmax v21.4s, v21.4s, v27.4s\n" "str q17, [x10, x14]\n" - "fmax v22.4s, v22.4s, v24.4s\n" + "fmax v22.4s, v22.4s, v27.4s\n" "str q18, [x9, x14]\n" "str q19, [x28, x14]\n" "str q20, [x27, x14]\n" @@ -342,123 +343,123 @@ void a64_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst_imp "tst %x[channel_multiplier], #0x3\n" "beq 6f\n" "ldr q14, [%x[params], #0x0]\n" - "ldr q31, [%x[params], #0x10]\n" + "ldr q25, [%x[params], #0x10]\n" "mov v15.16b, v14.16b\n" "mov v16.16b, v14.16b\n" - "ldr q30, [%x[params], #0x20]\n" - "ldr q29, [%x[params], #0x30]\n" + "ldr q24, [%x[params], #0x20]\n" + "ldr q23, [%x[params], #0x30]\n" "mov v17.16b, v14.16b\n" "mov v18.16b, v14.16b\n" "mov v19.16b, v14.16b\n" "mov v20.16b, v14.16b\n" - "fmla v15.4s, v31.4s, v0.s[2]\n" + "fmla v15.4s, v25.4s, v0.s[2]\n" "mov v21.16b, v14.16b\n" "mov v22.16b, v14.16b\n" - "fmla v14.4s, v31.4s, v0.s[0]\n" - "fmla v16.4s, v31.4s, v1.s[0]\n" - "fmla v17.4s, v31.4s, v4.s[0]\n" - "fmla v18.4s, v31.4s, v4.s[2]\n" - "fmla v19.4s, v31.4s, v5.s[0]\n" - "fmla v20.4s, v31.4s, v8.s[0]\n" - "fmla v21.4s, v31.4s, v8.s[2]\n" - "fmla v22.4s, v31.4s, v9.s[0]\n" - "ldr q31, [%x[params], #0x40]\n" - "fmla v14.4s, v30.4s, v0.s[1]\n" - "fmla v15.4s, v30.4s, v0.s[3]\n" - "fmla v16.4s, v30.4s, v1.s[1]\n" - "fmla v17.4s, v30.4s, v4.s[1]\n" - "fmla v18.4s, v30.4s, v4.s[3]\n" - "fmla v19.4s, v30.4s, v5.s[1]\n" - "fmla v20.4s, v30.4s, v8.s[1]\n" - "fmla v21.4s, v30.4s, v8.s[3]\n" - "fmla v22.4s, v30.4s, v9.s[1]\n" - "ldr q30, [%x[params], #0x50]\n" - "fmla v14.4s, v29.4s, v0.s[2]\n" - "fmla v15.4s, v29.4s, v1.s[0]\n" - "fmla v16.4s, v29.4s, v1.s[2]\n" - "fmla v17.4s, v29.4s, v4.s[2]\n" - "fmla v18.4s, v29.4s, v5.s[0]\n" - "fmla v19.4s, v29.4s, v5.s[2]\n" - "fmla v20.4s, v29.4s, v8.s[2]\n" - "fmla v21.4s, v29.4s, v9.s[0]\n" - "fmla v22.4s, v29.4s, v9.s[2]\n" - "ldr q29, [%x[params], #0x60]\n" - "fmla v14.4s, v31.4s, v2.s[0]\n" - "fmla v15.4s, v31.4s, v2.s[2]\n" - "fmla v16.4s, v31.4s, v3.s[0]\n" - "fmla v17.4s, v31.4s, v6.s[0]\n" - "fmla v18.4s, v31.4s, v6.s[2]\n" - "fmla v19.4s, v31.4s, v7.s[0]\n" - "fmla v20.4s, v31.4s, v10.s[0]\n" - "fmla v21.4s, v31.4s, v10.s[2]\n" - "fmla v22.4s, v31.4s, v11.s[0]\n" - "ldr q31, [%x[params], #0x70]\n" - "fmla v14.4s, v30.4s, v2.s[1]\n" - "fmla v15.4s, v30.4s, v2.s[3]\n" - "fmla v16.4s, v30.4s, v3.s[1]\n" - "fmla v17.4s, v30.4s, v6.s[1]\n" - "fmla v18.4s, v30.4s, v6.s[3]\n" - "fmla v19.4s, v30.4s, v7.s[1]\n" - "fmla v20.4s, v30.4s, v10.s[1]\n" - "fmla v21.4s, v30.4s, v10.s[3]\n" - "fmla v22.4s, v30.4s, v11.s[1]\n" - "ldr q30, [%x[params], #0x80]\n" - "fmla v14.4s, v29.4s, v2.s[2]\n" - "fmla v15.4s, v29.4s, v3.s[0]\n" - "fmla v16.4s, v29.4s, v3.s[2]\n" - "fmla v17.4s, v29.4s, v6.s[2]\n" - "fmla v18.4s, v29.4s, v7.s[0]\n" - "fmla v19.4s, v29.4s, v7.s[2]\n" - "fmla v20.4s, v29.4s, v10.s[2]\n" - "fmla v21.4s, v29.4s, v11.s[0]\n" - "fmla v22.4s, v29.4s, v11.s[2]\n" - "ldr q29, [%x[params], #0x90]\n" + "fmla v14.4s, v25.4s, v0.s[0]\n" + "fmla v16.4s, v25.4s, v1.s[0]\n" + "fmla v17.4s, v25.4s, v4.s[0]\n" + "fmla v18.4s, v25.4s, v4.s[2]\n" + "fmla v19.4s, v25.4s, v5.s[0]\n" + "fmla v20.4s, v25.4s, v8.s[0]\n" + "fmla v21.4s, v25.4s, v8.s[2]\n" + "fmla v22.4s, v25.4s, v9.s[0]\n" + "ldr q25, [%x[params], #0x40]\n" + "fmla v14.4s, v24.4s, v0.s[1]\n" + "fmla v15.4s, v24.4s, v0.s[3]\n" + "fmla v16.4s, v24.4s, v1.s[1]\n" + "fmla v17.4s, v24.4s, v4.s[1]\n" + "fmla v18.4s, v24.4s, v4.s[3]\n" + "fmla v19.4s, v24.4s, v5.s[1]\n" + "fmla v20.4s, v24.4s, v8.s[1]\n" + "fmla v21.4s, v24.4s, v8.s[3]\n" + "fmla v22.4s, v24.4s, v9.s[1]\n" + "ldr q24, [%x[params], #0x50]\n" + "fmla v14.4s, v23.4s, v0.s[2]\n" + "fmla v15.4s, v23.4s, v1.s[0]\n" + "fmla v16.4s, v23.4s, v1.s[2]\n" + "fmla v17.4s, v23.4s, v4.s[2]\n" + "fmla v18.4s, v23.4s, v5.s[0]\n" + "fmla v19.4s, v23.4s, v5.s[2]\n" + "fmla v20.4s, v23.4s, v8.s[2]\n" + "fmla v21.4s, v23.4s, v9.s[0]\n" + "fmla v22.4s, v23.4s, v9.s[2]\n" + "ldr q23, [%x[params], #0x60]\n" + "fmla v14.4s, v25.4s, v2.s[0]\n" + "fmla v15.4s, v25.4s, v2.s[2]\n" + "fmla v16.4s, v25.4s, v3.s[0]\n" + "fmla v17.4s, v25.4s, v6.s[0]\n" + "fmla v18.4s, v25.4s, v6.s[2]\n" + "fmla v19.4s, v25.4s, v7.s[0]\n" + "fmla v20.4s, v25.4s, v10.s[0]\n" + "fmla v21.4s, v25.4s, v10.s[2]\n" + "fmla v22.4s, v25.4s, v11.s[0]\n" + "ldr q25, [%x[params], #0x70]\n" + "fmla v14.4s, v24.4s, v2.s[1]\n" + "fmla v15.4s, v24.4s, v2.s[3]\n" + "fmla v16.4s, v24.4s, v3.s[1]\n" + "fmla v17.4s, v24.4s, v6.s[1]\n" + "fmla v18.4s, v24.4s, v6.s[3]\n" + "fmla v19.4s, v24.4s, v7.s[1]\n" + "fmla v20.4s, v24.4s, v10.s[1]\n" + "fmla v21.4s, v24.4s, v10.s[3]\n" + "fmla v22.4s, v24.4s, v11.s[1]\n" + "ldr q24, [%x[params], #0x80]\n" + "fmla v14.4s, v23.4s, v2.s[2]\n" + "fmla v15.4s, v23.4s, v3.s[0]\n" + "fmla v16.4s, v23.4s, v3.s[2]\n" + "fmla v17.4s, v23.4s, v6.s[2]\n" + "fmla v18.4s, v23.4s, v7.s[0]\n" + "fmla v19.4s, v23.4s, v7.s[2]\n" + "fmla v20.4s, v23.4s, v10.s[2]\n" + "fmla v21.4s, v23.4s, v11.s[0]\n" + "fmla v22.4s, v23.4s, v11.s[2]\n" + "ldr q23, [%x[params], #0x90]\n" "add %x[params], %x[params], #0xa0\n" - "fmla v14.4s, v31.4s, v4.s[0]\n" - "fmla v15.4s, v31.4s, v4.s[2]\n" - "fmla v16.4s, v31.4s, v5.s[0]\n" - "fmla v17.4s, v31.4s, v8.s[0]\n" - "fmla v18.4s, v31.4s, v8.s[2]\n" - "fmla v19.4s, v31.4s, v9.s[0]\n" - "fmla v20.4s, v31.4s, v12.s[0]\n" - "fmla v21.4s, v31.4s, v12.s[2]\n" - "fmla v22.4s, v31.4s, v13.s[0]\n" - "fmla v14.4s, v30.4s, v4.s[1]\n" - "fmla v15.4s, v30.4s, v4.s[3]\n" - "fmla v16.4s, v30.4s, v5.s[1]\n" - "fmla v17.4s, v30.4s, v8.s[1]\n" - "fmla v18.4s, v30.4s, v8.s[3]\n" - "fmla v19.4s, v30.4s, v9.s[1]\n" - "fmla v20.4s, v30.4s, v12.s[1]\n" - "fmla v21.4s, v30.4s, v12.s[3]\n" - "fmla v22.4s, v30.4s, v13.s[1]\n" - "fmla v14.4s, v29.4s, v4.s[2]\n" - "fmla v15.4s, v29.4s, v5.s[0]\n" - "fmin v14.4s, v14.4s, v23.4s\n" - "fmla v16.4s, v29.4s, v5.s[2]\n" - "fmla v17.4s, v29.4s, v8.s[2]\n" - "fmin v15.4s, v15.4s, v23.4s\n" - "fmla v18.4s, v29.4s, v9.s[0]\n" - "fmla v19.4s, v29.4s, v9.s[2]\n" - "fmin v16.4s, v16.4s, v23.4s\n" - "fmla v20.4s, v29.4s, v12.s[2]\n" - "fmla v21.4s, v29.4s, v13.s[0]\n" - "fmin v17.4s, v17.4s, v23.4s\n" - "fmla v22.4s, v29.4s, v13.s[2]\n" - "fmin v18.4s, v18.4s, v23.4s\n" - "fmin v19.4s, v19.4s, v23.4s\n" - "fmin v20.4s, v20.4s, v23.4s\n" - "fmin v21.4s, v21.4s, v23.4s\n" - "fmin v22.4s, v22.4s, v23.4s\n" - "fmax v14.4s, v14.4s, v24.4s\n" - "fmax v15.4s, v15.4s, v24.4s\n" - "fmax v16.4s, v16.4s, v24.4s\n" - "fmax v17.4s, v17.4s, v24.4s\n" - "fmax v18.4s, v18.4s, v24.4s\n" - "fmax v19.4s, v19.4s, v24.4s\n" - "fmax v20.4s, v20.4s, v24.4s\n" - "fmax v21.4s, v21.4s, v24.4s\n" - "fmax v22.4s, v22.4s, v24.4s\n" + "fmla v14.4s, v25.4s, v4.s[0]\n" + "fmla v15.4s, v25.4s, v4.s[2]\n" + "fmla v16.4s, v25.4s, v5.s[0]\n" + "fmla v17.4s, v25.4s, v8.s[0]\n" + "fmla v18.4s, v25.4s, v8.s[2]\n" + "fmla v19.4s, v25.4s, v9.s[0]\n" + "fmla v20.4s, v25.4s, v12.s[0]\n" + "fmla v21.4s, v25.4s, v12.s[2]\n" + "fmla v22.4s, v25.4s, v13.s[0]\n" + "fmla v14.4s, v24.4s, v4.s[1]\n" + "fmla v15.4s, v24.4s, v4.s[3]\n" + "fmla v16.4s, v24.4s, v5.s[1]\n" + "fmla v17.4s, v24.4s, v8.s[1]\n" + "fmla v18.4s, v24.4s, v8.s[3]\n" + "fmla v19.4s, v24.4s, v9.s[1]\n" + "fmla v20.4s, v24.4s, v12.s[1]\n" + "fmla v21.4s, v24.4s, v12.s[3]\n" + "fmla v22.4s, v24.4s, v13.s[1]\n" + "fmla v14.4s, v23.4s, v4.s[2]\n" + "fmla v15.4s, v23.4s, v5.s[0]\n" + "fmin v14.4s, v14.4s, v26.4s\n" + "fmla v16.4s, v23.4s, v5.s[2]\n" + "fmla v17.4s, v23.4s, v8.s[2]\n" + "fmin v15.4s, v15.4s, v26.4s\n" + "fmla v18.4s, v23.4s, v9.s[0]\n" + "fmla v19.4s, v23.4s, v9.s[2]\n" + "fmin v16.4s, v16.4s, v26.4s\n" + "fmla v20.4s, v23.4s, v12.s[2]\n" + "fmla v21.4s, v23.4s, v13.s[0]\n" + "fmin v17.4s, v17.4s, v26.4s\n" + "fmla v22.4s, v23.4s, v13.s[2]\n" + "fmin v18.4s, v18.4s, v26.4s\n" + "fmin v19.4s, v19.4s, v26.4s\n" + "fmin v20.4s, v20.4s, v26.4s\n" + "fmin v21.4s, v21.4s, v26.4s\n" + "fmin v22.4s, v22.4s, v26.4s\n" + "fmax v14.4s, v14.4s, v27.4s\n" + "fmax v15.4s, v15.4s, v27.4s\n" + "fmax v16.4s, v16.4s, v27.4s\n" + "fmax v17.4s, v17.4s, v27.4s\n" + "fmax v18.4s, v18.4s, v27.4s\n" + "fmax v19.4s, v19.4s, v27.4s\n" + "fmax v20.4s, v20.4s, v27.4s\n" + "fmax v21.4s, v21.4s, v27.4s\n" + "fmax v22.4s, v22.4s, v27.4s\n" "tbz %x[channel_multiplier], #1, 4f\n" "add x20, x13, x14\n" "add x22, x12, x14\n" @@ -519,15 +520,14 @@ void a64_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst_imp "st1 { v21.s }[0], [x21]\n" "st1 { v22.s }[0], [x20]\n" "5:" // Output channel oddments: Store: Bit 1: End - "6:" // End - : [params] "+&r" (params) : [channel_multiplier] "r" (n_output_channels), [clamps] "r" (minmax_vals), [inptrs] "r" (inptrs), [outptrs] "r" (outptrs) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst.hpp index 5ae8dd36530bfa44e97a5977cdc09b4c974f7cf4..9f514c78e75eba9741c2e2979cbc36a3cbcef3aa 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include @@ -58,4 +58,4 @@ struct a64_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst : } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst/generic.cpp index 50848cc2e8d2ce89fb14b01d8b65efd560590c8d..c9bb1f41da985f4bee94d202e4988678a95ceb94 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst/generic.cpp @@ -22,11 +22,11 @@ * SOFTWARE. */ -#if defined(__aarch64__) - #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace depthwise { @@ -42,7 +42,7 @@ void a64_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst_imp const float minmax_vals[2] = { activation_min, activation_max }; __asm__ __volatile__( - "ld1r { v21.4s }, [%x[clamps]]\n" + "ld1r { v26.4s }, [%x[clamps]]\n" "ldr x21, [%x[inptrs], #0x0]\n" "lsr x22, %x[channel_multiplier], #0x2\n" "add x20, %x[clamps], #0x4\n" @@ -50,7 +50,7 @@ void a64_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst_imp "ldr q1, [x21, #0x10]\n" "mov x21, #0x0\n" "mov x13, #0x0\n" - "ld1r { v20.4s }, [x20]\n" + "ld1r { v25.4s }, [x20]\n" "ldr x20, [%x[inptrs], #0x8]\n" "ldr q2, [x20, #0x0]\n" "ldr q3, [x20, #0x10]\n" @@ -98,7 +98,7 @@ void a64_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst_imp "fmla v17.4s, v31.4s, v2.s[1]\n" "fmla v18.4s, v31.4s, v2.s[2]\n" "fmla v19.4s, v31.4s, v2.s[3]\n" - "ldr q31, [%x[params], #0x0]\n" + "ldr q24, [%x[params], #0x0]\n" "fmla v12.4s, v30.4s, v0.s[1]\n" "fmla v13.4s, v30.4s, v0.s[2]\n" "fmla v14.4s, v30.4s, v0.s[3]\n" @@ -107,7 +107,7 @@ void a64_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst_imp "fmla v17.4s, v30.4s, v2.s[2]\n" "fmla v18.4s, v30.4s, v2.s[3]\n" "fmla v19.4s, v30.4s, v3.s[0]\n" - "ldr q30, [%x[params], #0x10]\n" + "ldr q23, [%x[params], #0x10]\n" "fmla v12.4s, v29.4s, v0.s[2]\n" "fmla v13.4s, v29.4s, v0.s[3]\n" "fmla v14.4s, v29.4s, v1.s[0]\n" @@ -116,7 +116,7 @@ void a64_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst_imp "fmla v17.4s, v29.4s, v2.s[3]\n" "fmla v18.4s, v29.4s, v3.s[0]\n" "fmla v19.4s, v29.4s, v3.s[1]\n" - "ldr q29, [%x[params], #0x20]\n" + "ldr q22, [%x[params], #0x20]\n" "fmla v12.4s, v28.4s, v0.s[3]\n" "fmla v13.4s, v28.4s, v1.s[0]\n" "fmla v14.4s, v28.4s, v1.s[1]\n" @@ -125,7 +125,7 @@ void a64_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst_imp "fmla v17.4s, v28.4s, v3.s[0]\n" "fmla v18.4s, v28.4s, v3.s[1]\n" "fmla v19.4s, v28.4s, v3.s[2]\n" - "ldr q28, [%x[params], #0x30]\n" + "ldr q21, [%x[params], #0x30]\n" "fmla v12.4s, v27.4s, v1.s[0]\n" "fmla v13.4s, v27.4s, v1.s[1]\n" "fmla v14.4s, v27.4s, v1.s[2]\n" @@ -134,209 +134,209 @@ void a64_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst_imp "fmla v17.4s, v27.4s, v3.s[1]\n" "fmla v18.4s, v27.4s, v3.s[2]\n" "fmla v19.4s, v27.4s, v3.s[3]\n" - "ldr q27, [%x[params], #0x40]\n" - "fmla v12.4s, v31.4s, v2.s[0]\n" - "fmla v13.4s, v31.4s, v2.s[1]\n" - "fmla v14.4s, v31.4s, v2.s[2]\n" - "fmla v15.4s, v31.4s, v2.s[3]\n" - "fmla v16.4s, v31.4s, v4.s[0]\n" - "fmla v17.4s, v31.4s, v4.s[1]\n" - "fmla v18.4s, v31.4s, v4.s[2]\n" - "fmla v19.4s, v31.4s, v4.s[3]\n" - "ldr q31, [%x[params], #0x50]\n" - "fmla v12.4s, v30.4s, v2.s[1]\n" - "fmla v13.4s, v30.4s, v2.s[2]\n" - "fmla v14.4s, v30.4s, v2.s[3]\n" - "fmla v15.4s, v30.4s, v3.s[0]\n" - "fmla v16.4s, v30.4s, v4.s[1]\n" - "fmla v17.4s, v30.4s, v4.s[2]\n" - "fmla v18.4s, v30.4s, v4.s[3]\n" - "fmla v19.4s, v30.4s, v5.s[0]\n" - "ldr q30, [%x[params], #0x60]\n" - "fmla v12.4s, v29.4s, v2.s[2]\n" - "fmla v13.4s, v29.4s, v2.s[3]\n" - "fmla v14.4s, v29.4s, v3.s[0]\n" - "fmla v15.4s, v29.4s, v3.s[1]\n" - "fmla v16.4s, v29.4s, v4.s[2]\n" - "fmla v17.4s, v29.4s, v4.s[3]\n" - "fmla v18.4s, v29.4s, v5.s[0]\n" - "fmla v19.4s, v29.4s, v5.s[1]\n" - "ldr q29, [%x[params], #0x70]\n" - "fmla v12.4s, v28.4s, v2.s[3]\n" - "fmla v13.4s, v28.4s, v3.s[0]\n" - "fmla v14.4s, v28.4s, v3.s[1]\n" - "fmla v15.4s, v28.4s, v3.s[2]\n" - "fmla v16.4s, v28.4s, v4.s[3]\n" - "fmla v17.4s, v28.4s, v5.s[0]\n" - "fmla v18.4s, v28.4s, v5.s[1]\n" - "fmla v19.4s, v28.4s, v5.s[2]\n" - "ldr q28, [%x[params], #0x80]\n" - "fmla v12.4s, v27.4s, v3.s[0]\n" - "fmla v13.4s, v27.4s, v3.s[1]\n" - "fmla v14.4s, v27.4s, v3.s[2]\n" - "fmla v15.4s, v27.4s, v3.s[3]\n" - "fmla v16.4s, v27.4s, v5.s[0]\n" - "fmla v17.4s, v27.4s, v5.s[1]\n" - "fmla v18.4s, v27.4s, v5.s[2]\n" - "fmla v19.4s, v27.4s, v5.s[3]\n" - "ldr q27, [%x[params], #0x90]\n" - "fmla v12.4s, v31.4s, v4.s[0]\n" - "fmla v13.4s, v31.4s, v4.s[1]\n" - "fmla v14.4s, v31.4s, v4.s[2]\n" - "fmla v15.4s, v31.4s, v4.s[3]\n" - "fmla v16.4s, v31.4s, v6.s[0]\n" - "fmla v17.4s, v31.4s, v6.s[1]\n" - "fmla v18.4s, v31.4s, v6.s[2]\n" - "fmla v19.4s, v31.4s, v6.s[3]\n" - "ldr q31, [%x[params], #0xa0]\n" - "fmla v12.4s, v30.4s, v4.s[1]\n" - "fmla v13.4s, v30.4s, v4.s[2]\n" - "fmla v14.4s, v30.4s, v4.s[3]\n" - "fmla v15.4s, v30.4s, v5.s[0]\n" - "fmla v16.4s, v30.4s, v6.s[1]\n" - "fmla v17.4s, v30.4s, v6.s[2]\n" - "fmla v18.4s, v30.4s, v6.s[3]\n" - "fmla v19.4s, v30.4s, v7.s[0]\n" - "ldr q30, [%x[params], #0xb0]\n" - "fmla v12.4s, v29.4s, v4.s[2]\n" - "fmla v13.4s, v29.4s, v4.s[3]\n" - "fmla v14.4s, v29.4s, v5.s[0]\n" - "fmla v15.4s, v29.4s, v5.s[1]\n" - "fmla v16.4s, v29.4s, v6.s[2]\n" - "fmla v17.4s, v29.4s, v6.s[3]\n" - "fmla v18.4s, v29.4s, v7.s[0]\n" - "fmla v19.4s, v29.4s, v7.s[1]\n" - "ldr q29, [%x[params], #0xc0]\n" - "fmla v12.4s, v28.4s, v4.s[3]\n" - "fmla v13.4s, v28.4s, v5.s[0]\n" - "fmla v14.4s, v28.4s, v5.s[1]\n" - "fmla v15.4s, v28.4s, v5.s[2]\n" - "fmla v16.4s, v28.4s, v6.s[3]\n" - "fmla v17.4s, v28.4s, v7.s[0]\n" - "fmla v18.4s, v28.4s, v7.s[1]\n" - "fmla v19.4s, v28.4s, v7.s[2]\n" - "ldr q28, [%x[params], #0xd0]\n" - "fmla v12.4s, v27.4s, v5.s[0]\n" - "fmla v13.4s, v27.4s, v5.s[1]\n" - "fmla v14.4s, v27.4s, v5.s[2]\n" - "fmla v15.4s, v27.4s, v5.s[3]\n" - "fmla v16.4s, v27.4s, v7.s[0]\n" - "fmla v17.4s, v27.4s, v7.s[1]\n" - "fmla v18.4s, v27.4s, v7.s[2]\n" - "fmla v19.4s, v27.4s, v7.s[3]\n" - "ldr q27, [%x[params], #0xe0]\n" - "fmla v12.4s, v31.4s, v6.s[0]\n" - "fmla v13.4s, v31.4s, v6.s[1]\n" - "fmla v14.4s, v31.4s, v6.s[2]\n" - "fmla v15.4s, v31.4s, v6.s[3]\n" - "fmla v16.4s, v31.4s, v8.s[0]\n" - "fmla v17.4s, v31.4s, v8.s[1]\n" - "fmla v18.4s, v31.4s, v8.s[2]\n" - "fmla v19.4s, v31.4s, v8.s[3]\n" - "ldr q31, [%x[params], #0xf0]\n" - "fmla v12.4s, v30.4s, v6.s[1]\n" - "fmla v13.4s, v30.4s, v6.s[2]\n" - "fmla v14.4s, v30.4s, v6.s[3]\n" - "fmla v15.4s, v30.4s, v7.s[0]\n" - "fmla v16.4s, v30.4s, v8.s[1]\n" - "fmla v17.4s, v30.4s, v8.s[2]\n" - "fmla v18.4s, v30.4s, v8.s[3]\n" - "fmla v19.4s, v30.4s, v9.s[0]\n" - "ldr q30, [%x[params], #0x100]\n" - "fmla v12.4s, v29.4s, v6.s[2]\n" - "fmla v13.4s, v29.4s, v6.s[3]\n" - "fmla v14.4s, v29.4s, v7.s[0]\n" - "fmla v15.4s, v29.4s, v7.s[1]\n" - "fmla v16.4s, v29.4s, v8.s[2]\n" - "fmla v17.4s, v29.4s, v8.s[3]\n" - "fmla v18.4s, v29.4s, v9.s[0]\n" - "fmla v19.4s, v29.4s, v9.s[1]\n" - "ldr q29, [%x[params], #0x110]\n" - "fmla v12.4s, v28.4s, v6.s[3]\n" - "fmla v13.4s, v28.4s, v7.s[0]\n" - "fmla v14.4s, v28.4s, v7.s[1]\n" - "fmla v15.4s, v28.4s, v7.s[2]\n" - "fmla v16.4s, v28.4s, v8.s[3]\n" - "fmla v17.4s, v28.4s, v9.s[0]\n" - "fmla v18.4s, v28.4s, v9.s[1]\n" - "fmla v19.4s, v28.4s, v9.s[2]\n" - "ldr q28, [%x[params], #0x120]\n" - "fmla v12.4s, v27.4s, v7.s[0]\n" - "fmla v13.4s, v27.4s, v7.s[1]\n" - "fmla v14.4s, v27.4s, v7.s[2]\n" - "fmla v15.4s, v27.4s, v7.s[3]\n" - "fmla v16.4s, v27.4s, v9.s[0]\n" - "fmla v17.4s, v27.4s, v9.s[1]\n" - "fmla v18.4s, v27.4s, v9.s[2]\n" - "fmla v19.4s, v27.4s, v9.s[3]\n" - "ldr q27, [%x[params], #0x130]\n" - "fmla v12.4s, v31.4s, v8.s[0]\n" - "fmla v13.4s, v31.4s, v8.s[1]\n" - "fmla v14.4s, v31.4s, v8.s[2]\n" - "fmla v15.4s, v31.4s, v8.s[3]\n" - "fmla v16.4s, v31.4s, v10.s[0]\n" - "fmla v17.4s, v31.4s, v10.s[1]\n" - "fmla v18.4s, v31.4s, v10.s[2]\n" - "fmla v19.4s, v31.4s, v10.s[3]\n" + "ldr q20, [%x[params], #0x40]\n" + "fmla v12.4s, v24.4s, v2.s[0]\n" + "fmla v13.4s, v24.4s, v2.s[1]\n" + "fmla v14.4s, v24.4s, v2.s[2]\n" + "fmla v15.4s, v24.4s, v2.s[3]\n" + "fmla v16.4s, v24.4s, v4.s[0]\n" + "fmla v17.4s, v24.4s, v4.s[1]\n" + "fmla v18.4s, v24.4s, v4.s[2]\n" + "fmla v19.4s, v24.4s, v4.s[3]\n" + "ldr q24, [%x[params], #0x50]\n" + "fmla v12.4s, v23.4s, v2.s[1]\n" + "fmla v13.4s, v23.4s, v2.s[2]\n" + "fmla v14.4s, v23.4s, v2.s[3]\n" + "fmla v15.4s, v23.4s, v3.s[0]\n" + "fmla v16.4s, v23.4s, v4.s[1]\n" + "fmla v17.4s, v23.4s, v4.s[2]\n" + "fmla v18.4s, v23.4s, v4.s[3]\n" + "fmla v19.4s, v23.4s, v5.s[0]\n" + "ldr q23, [%x[params], #0x60]\n" + "fmla v12.4s, v22.4s, v2.s[2]\n" + "fmla v13.4s, v22.4s, v2.s[3]\n" + "fmla v14.4s, v22.4s, v3.s[0]\n" + "fmla v15.4s, v22.4s, v3.s[1]\n" + "fmla v16.4s, v22.4s, v4.s[2]\n" + "fmla v17.4s, v22.4s, v4.s[3]\n" + "fmla v18.4s, v22.4s, v5.s[0]\n" + "fmla v19.4s, v22.4s, v5.s[1]\n" + "ldr q22, [%x[params], #0x70]\n" + "fmla v12.4s, v21.4s, v2.s[3]\n" + "fmla v13.4s, v21.4s, v3.s[0]\n" + "fmla v14.4s, v21.4s, v3.s[1]\n" + "fmla v15.4s, v21.4s, v3.s[2]\n" + "fmla v16.4s, v21.4s, v4.s[3]\n" + "fmla v17.4s, v21.4s, v5.s[0]\n" + "fmla v18.4s, v21.4s, v5.s[1]\n" + "fmla v19.4s, v21.4s, v5.s[2]\n" + "ldr q21, [%x[params], #0x80]\n" + "fmla v12.4s, v20.4s, v3.s[0]\n" + "fmla v13.4s, v20.4s, v3.s[1]\n" + "fmla v14.4s, v20.4s, v3.s[2]\n" + "fmla v15.4s, v20.4s, v3.s[3]\n" + "fmla v16.4s, v20.4s, v5.s[0]\n" + "fmla v17.4s, v20.4s, v5.s[1]\n" + "fmla v18.4s, v20.4s, v5.s[2]\n" + "fmla v19.4s, v20.4s, v5.s[3]\n" + "ldr q20, [%x[params], #0x90]\n" + "fmla v12.4s, v24.4s, v4.s[0]\n" + "fmla v13.4s, v24.4s, v4.s[1]\n" + "fmla v14.4s, v24.4s, v4.s[2]\n" + "fmla v15.4s, v24.4s, v4.s[3]\n" + "fmla v16.4s, v24.4s, v6.s[0]\n" + "fmla v17.4s, v24.4s, v6.s[1]\n" + "fmla v18.4s, v24.4s, v6.s[2]\n" + "fmla v19.4s, v24.4s, v6.s[3]\n" + "ldr q24, [%x[params], #0xa0]\n" + "fmla v12.4s, v23.4s, v4.s[1]\n" + "fmla v13.4s, v23.4s, v4.s[2]\n" + "fmla v14.4s, v23.4s, v4.s[3]\n" + "fmla v15.4s, v23.4s, v5.s[0]\n" + "fmla v16.4s, v23.4s, v6.s[1]\n" + "fmla v17.4s, v23.4s, v6.s[2]\n" + "fmla v18.4s, v23.4s, v6.s[3]\n" + "fmla v19.4s, v23.4s, v7.s[0]\n" + "ldr q23, [%x[params], #0xb0]\n" + "fmla v12.4s, v22.4s, v4.s[2]\n" + "fmla v13.4s, v22.4s, v4.s[3]\n" + "fmla v14.4s, v22.4s, v5.s[0]\n" + "fmla v15.4s, v22.4s, v5.s[1]\n" + "fmla v16.4s, v22.4s, v6.s[2]\n" + "fmla v17.4s, v22.4s, v6.s[3]\n" + "fmla v18.4s, v22.4s, v7.s[0]\n" + "fmla v19.4s, v22.4s, v7.s[1]\n" + "ldr q22, [%x[params], #0xc0]\n" + "fmla v12.4s, v21.4s, v4.s[3]\n" + "fmla v13.4s, v21.4s, v5.s[0]\n" + "fmla v14.4s, v21.4s, v5.s[1]\n" + "fmla v15.4s, v21.4s, v5.s[2]\n" + "fmla v16.4s, v21.4s, v6.s[3]\n" + "fmla v17.4s, v21.4s, v7.s[0]\n" + "fmla v18.4s, v21.4s, v7.s[1]\n" + "fmla v19.4s, v21.4s, v7.s[2]\n" + "ldr q21, [%x[params], #0xd0]\n" + "fmla v12.4s, v20.4s, v5.s[0]\n" + "fmla v13.4s, v20.4s, v5.s[1]\n" + "fmla v14.4s, v20.4s, v5.s[2]\n" + "fmla v15.4s, v20.4s, v5.s[3]\n" + "fmla v16.4s, v20.4s, v7.s[0]\n" + "fmla v17.4s, v20.4s, v7.s[1]\n" + "fmla v18.4s, v20.4s, v7.s[2]\n" + "fmla v19.4s, v20.4s, v7.s[3]\n" + "ldr q20, [%x[params], #0xe0]\n" + "fmla v12.4s, v24.4s, v6.s[0]\n" + "fmla v13.4s, v24.4s, v6.s[1]\n" + "fmla v14.4s, v24.4s, v6.s[2]\n" + "fmla v15.4s, v24.4s, v6.s[3]\n" + "fmla v16.4s, v24.4s, v8.s[0]\n" + "fmla v17.4s, v24.4s, v8.s[1]\n" + "fmla v18.4s, v24.4s, v8.s[2]\n" + "fmla v19.4s, v24.4s, v8.s[3]\n" + "ldr q24, [%x[params], #0xf0]\n" + "fmla v12.4s, v23.4s, v6.s[1]\n" + "fmla v13.4s, v23.4s, v6.s[2]\n" + "fmla v14.4s, v23.4s, v6.s[3]\n" + "fmla v15.4s, v23.4s, v7.s[0]\n" + "fmla v16.4s, v23.4s, v8.s[1]\n" + "fmla v17.4s, v23.4s, v8.s[2]\n" + "fmla v18.4s, v23.4s, v8.s[3]\n" + "fmla v19.4s, v23.4s, v9.s[0]\n" + "ldr q23, [%x[params], #0x100]\n" + "fmla v12.4s, v22.4s, v6.s[2]\n" + "fmla v13.4s, v22.4s, v6.s[3]\n" + "fmla v14.4s, v22.4s, v7.s[0]\n" + "fmla v15.4s, v22.4s, v7.s[1]\n" + "fmla v16.4s, v22.4s, v8.s[2]\n" + "fmla v17.4s, v22.4s, v8.s[3]\n" + "fmla v18.4s, v22.4s, v9.s[0]\n" + "fmla v19.4s, v22.4s, v9.s[1]\n" + "ldr q22, [%x[params], #0x110]\n" + "fmla v12.4s, v21.4s, v6.s[3]\n" + "fmla v13.4s, v21.4s, v7.s[0]\n" + "fmla v14.4s, v21.4s, v7.s[1]\n" + "fmla v15.4s, v21.4s, v7.s[2]\n" + "fmla v16.4s, v21.4s, v8.s[3]\n" + "fmla v17.4s, v21.4s, v9.s[0]\n" + "fmla v18.4s, v21.4s, v9.s[1]\n" + "fmla v19.4s, v21.4s, v9.s[2]\n" + "ldr q21, [%x[params], #0x120]\n" + "fmla v12.4s, v20.4s, v7.s[0]\n" + "fmla v13.4s, v20.4s, v7.s[1]\n" + "fmla v14.4s, v20.4s, v7.s[2]\n" + "fmla v15.4s, v20.4s, v7.s[3]\n" + "fmla v16.4s, v20.4s, v9.s[0]\n" + "fmla v17.4s, v20.4s, v9.s[1]\n" + "fmla v18.4s, v20.4s, v9.s[2]\n" + "fmla v19.4s, v20.4s, v9.s[3]\n" + "ldr q20, [%x[params], #0x130]\n" + "fmla v12.4s, v24.4s, v8.s[0]\n" + "fmla v13.4s, v24.4s, v8.s[1]\n" + "fmla v14.4s, v24.4s, v8.s[2]\n" + "fmla v15.4s, v24.4s, v8.s[3]\n" + "fmla v16.4s, v24.4s, v10.s[0]\n" + "fmla v17.4s, v24.4s, v10.s[1]\n" + "fmla v18.4s, v24.4s, v10.s[2]\n" + "fmla v19.4s, v24.4s, v10.s[3]\n" "ldr q31, [%x[params], #0x150]\n" - "fmla v12.4s, v30.4s, v8.s[1]\n" - "fmla v13.4s, v30.4s, v8.s[2]\n" - "fmla v14.4s, v30.4s, v8.s[3]\n" - "fmla v15.4s, v30.4s, v9.s[0]\n" - "fmla v16.4s, v30.4s, v10.s[1]\n" - "fmla v17.4s, v30.4s, v10.s[2]\n" - "fmla v18.4s, v30.4s, v10.s[3]\n" - "fmla v19.4s, v30.4s, v11.s[0]\n" + "fmla v12.4s, v23.4s, v8.s[1]\n" + "fmla v13.4s, v23.4s, v8.s[2]\n" + "fmla v14.4s, v23.4s, v8.s[3]\n" + "fmla v15.4s, v23.4s, v9.s[0]\n" + "fmla v16.4s, v23.4s, v10.s[1]\n" + "fmla v17.4s, v23.4s, v10.s[2]\n" + "fmla v18.4s, v23.4s, v10.s[3]\n" + "fmla v19.4s, v23.4s, v11.s[0]\n" "ldr q30, [%x[params], #0x160]\n" - "fmla v12.4s, v29.4s, v8.s[2]\n" - "fmla v13.4s, v29.4s, v8.s[3]\n" - "fmla v14.4s, v29.4s, v9.s[0]\n" - "fmla v15.4s, v29.4s, v9.s[1]\n" - "fmla v16.4s, v29.4s, v10.s[2]\n" - "fmla v17.4s, v29.4s, v10.s[3]\n" - "fmla v18.4s, v29.4s, v11.s[0]\n" - "fmla v19.4s, v29.4s, v11.s[1]\n" + "fmla v12.4s, v22.4s, v8.s[2]\n" + "fmla v13.4s, v22.4s, v8.s[3]\n" + "fmla v14.4s, v22.4s, v9.s[0]\n" + "fmla v15.4s, v22.4s, v9.s[1]\n" + "fmla v16.4s, v22.4s, v10.s[2]\n" + "fmla v17.4s, v22.4s, v10.s[3]\n" + "fmla v18.4s, v22.4s, v11.s[0]\n" + "fmla v19.4s, v22.4s, v11.s[1]\n" "ldr q29, [%x[params], #0x170]\n" - "fmla v12.4s, v28.4s, v8.s[3]\n" - "fmla v13.4s, v28.4s, v9.s[0]\n" - "fmla v14.4s, v28.4s, v9.s[1]\n" - "fmla v15.4s, v28.4s, v9.s[2]\n" - "fmla v16.4s, v28.4s, v10.s[3]\n" - "fmla v17.4s, v28.4s, v11.s[0]\n" - "fmla v18.4s, v28.4s, v11.s[1]\n" - "fmla v19.4s, v28.4s, v11.s[2]\n" + "fmla v12.4s, v21.4s, v8.s[3]\n" + "fmla v13.4s, v21.4s, v9.s[0]\n" + "fmla v14.4s, v21.4s, v9.s[1]\n" + "fmla v15.4s, v21.4s, v9.s[2]\n" + "fmla v16.4s, v21.4s, v10.s[3]\n" + "fmla v17.4s, v21.4s, v11.s[0]\n" + "fmla v18.4s, v21.4s, v11.s[1]\n" + "fmla v19.4s, v21.4s, v11.s[2]\n" "ldr q28, [%x[params], #0x180]\n" - "fmla v12.4s, v27.4s, v9.s[0]\n" - "fmla v13.4s, v27.4s, v9.s[1]\n" - "fmin v12.4s, v12.4s, v20.4s\n" - "fmla v14.4s, v27.4s, v9.s[2]\n" - "fmla v15.4s, v27.4s, v9.s[3]\n" - "fmax v12.4s, v12.4s, v21.4s\n" + "fmla v12.4s, v20.4s, v9.s[0]\n" + "fmla v13.4s, v20.4s, v9.s[1]\n" + "fmin v12.4s, v12.4s, v25.4s\n" + "fmla v14.4s, v20.4s, v9.s[2]\n" + "fmla v15.4s, v20.4s, v9.s[3]\n" + "fmax v12.4s, v12.4s, v26.4s\n" "str q12, [x12, x13]\n" "ldr q12, [%x[params], #0x140]\n" - "fmla v16.4s, v27.4s, v11.s[0]\n" - "fmla v17.4s, v27.4s, v11.s[1]\n" - "fmin v13.4s, v13.4s, v20.4s\n" - "fmla v18.4s, v27.4s, v11.s[2]\n" - "fmla v19.4s, v27.4s, v11.s[3]\n" + "fmla v16.4s, v20.4s, v11.s[0]\n" + "fmla v17.4s, v20.4s, v11.s[1]\n" + "fmin v13.4s, v13.4s, v25.4s\n" + "fmla v18.4s, v20.4s, v11.s[2]\n" + "fmla v19.4s, v20.4s, v11.s[3]\n" "ldr q27, [%x[params], #0x190]\n" - "fmin v14.4s, v14.4s, v20.4s\n" - "fmin v15.4s, v15.4s, v20.4s\n" - "fmin v16.4s, v16.4s, v20.4s\n" + "fmin v14.4s, v14.4s, v25.4s\n" + "fmin v15.4s, v15.4s, v25.4s\n" + "fmin v16.4s, v16.4s, v25.4s\n" "add %x[params], %x[params], #0x1a0\n" - "fmin v17.4s, v17.4s, v20.4s\n" - "fmin v18.4s, v18.4s, v20.4s\n" - "fmin v19.4s, v19.4s, v20.4s\n" - "fmax v13.4s, v13.4s, v21.4s\n" + "fmin v17.4s, v17.4s, v25.4s\n" + "fmin v18.4s, v18.4s, v25.4s\n" + "fmin v19.4s, v19.4s, v25.4s\n" + "fmax v13.4s, v13.4s, v26.4s\n" "str q13, [x11, x13]\n" - "fmax v14.4s, v14.4s, v21.4s\n" - "fmax v15.4s, v15.4s, v21.4s\n" + "fmax v14.4s, v14.4s, v26.4s\n" + "fmax v15.4s, v15.4s, v26.4s\n" "str q14, [x10, x13]\n" - "fmax v16.4s, v16.4s, v21.4s\n" - "fmax v17.4s, v17.4s, v21.4s\n" + "fmax v16.4s, v16.4s, v26.4s\n" + "fmax v17.4s, v17.4s, v26.4s\n" "str q15, [x9, x13]\n" - "fmax v18.4s, v18.4s, v21.4s\n" - "fmax v19.4s, v19.4s, v21.4s\n" + "fmax v18.4s, v18.4s, v26.4s\n" + "fmax v19.4s, v19.4s, v26.4s\n" "str q16, [x28, x13]\n" "str q17, [x27, x13]\n" "mov v13.16b, v12.16b\n" @@ -359,7 +359,7 @@ void a64_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst_imp "fmla v17.4s, v31.4s, v2.s[1]\n" "fmla v18.4s, v31.4s, v2.s[2]\n" "fmla v19.4s, v31.4s, v2.s[3]\n" - "ldr q31, [%x[params], #0x0]\n" + "ldr q24, [%x[params], #0x0]\n" "fmla v12.4s, v30.4s, v0.s[1]\n" "fmla v13.4s, v30.4s, v0.s[2]\n" "fmla v14.4s, v30.4s, v0.s[3]\n" @@ -368,7 +368,7 @@ void a64_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst_imp "fmla v17.4s, v30.4s, v2.s[2]\n" "fmla v18.4s, v30.4s, v2.s[3]\n" "fmla v19.4s, v30.4s, v3.s[0]\n" - "ldr q30, [%x[params], #0x10]\n" + "ldr q23, [%x[params], #0x10]\n" "fmla v12.4s, v29.4s, v0.s[2]\n" "fmla v13.4s, v29.4s, v0.s[3]\n" "fmla v14.4s, v29.4s, v1.s[0]\n" @@ -377,7 +377,7 @@ void a64_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst_imp "fmla v17.4s, v29.4s, v2.s[3]\n" "fmla v18.4s, v29.4s, v3.s[0]\n" "fmla v19.4s, v29.4s, v3.s[1]\n" - "ldr q29, [%x[params], #0x20]\n" + "ldr q22, [%x[params], #0x20]\n" "fmla v12.4s, v28.4s, v0.s[3]\n" "fmla v13.4s, v28.4s, v1.s[0]\n" "fmla v14.4s, v28.4s, v1.s[1]\n" @@ -386,7 +386,7 @@ void a64_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst_imp "fmla v17.4s, v28.4s, v3.s[0]\n" "fmla v18.4s, v28.4s, v3.s[1]\n" "fmla v19.4s, v28.4s, v3.s[2]\n" - "ldr q28, [%x[params], #0x30]\n" + "ldr q21, [%x[params], #0x30]\n" "fmla v12.4s, v27.4s, v1.s[0]\n" "fmla v13.4s, v27.4s, v1.s[1]\n" "fmla v14.4s, v27.4s, v1.s[2]\n" @@ -395,202 +395,202 @@ void a64_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst_imp "fmla v17.4s, v27.4s, v3.s[1]\n" "fmla v18.4s, v27.4s, v3.s[2]\n" "fmla v19.4s, v27.4s, v3.s[3]\n" - "ldr q27, [%x[params], #0x40]\n" - "fmla v12.4s, v31.4s, v2.s[0]\n" - "fmla v13.4s, v31.4s, v2.s[1]\n" - "fmla v14.4s, v31.4s, v2.s[2]\n" - "fmla v15.4s, v31.4s, v2.s[3]\n" - "fmla v16.4s, v31.4s, v4.s[0]\n" - "fmla v17.4s, v31.4s, v4.s[1]\n" - "fmla v18.4s, v31.4s, v4.s[2]\n" - "fmla v19.4s, v31.4s, v4.s[3]\n" - "ldr q31, [%x[params], #0x50]\n" - "fmla v12.4s, v30.4s, v2.s[1]\n" - "fmla v13.4s, v30.4s, v2.s[2]\n" - "fmla v14.4s, v30.4s, v2.s[3]\n" - "fmla v15.4s, v30.4s, v3.s[0]\n" - "fmla v16.4s, v30.4s, v4.s[1]\n" - "fmla v17.4s, v30.4s, v4.s[2]\n" - "fmla v18.4s, v30.4s, v4.s[3]\n" - "fmla v19.4s, v30.4s, v5.s[0]\n" - "ldr q30, [%x[params], #0x60]\n" - "fmla v12.4s, v29.4s, v2.s[2]\n" - "fmla v13.4s, v29.4s, v2.s[3]\n" - "fmla v14.4s, v29.4s, v3.s[0]\n" - "fmla v15.4s, v29.4s, v3.s[1]\n" - "fmla v16.4s, v29.4s, v4.s[2]\n" - "fmla v17.4s, v29.4s, v4.s[3]\n" - "fmla v18.4s, v29.4s, v5.s[0]\n" - "fmla v19.4s, v29.4s, v5.s[1]\n" - "ldr q29, [%x[params], #0x70]\n" - "fmla v12.4s, v28.4s, v2.s[3]\n" - "fmla v13.4s, v28.4s, v3.s[0]\n" - "fmla v14.4s, v28.4s, v3.s[1]\n" - "fmla v15.4s, v28.4s, v3.s[2]\n" - "fmla v16.4s, v28.4s, v4.s[3]\n" - "fmla v17.4s, v28.4s, v5.s[0]\n" - "fmla v18.4s, v28.4s, v5.s[1]\n" - "fmla v19.4s, v28.4s, v5.s[2]\n" - "ldr q28, [%x[params], #0x80]\n" - "fmla v12.4s, v27.4s, v3.s[0]\n" - "fmla v13.4s, v27.4s, v3.s[1]\n" - "fmla v14.4s, v27.4s, v3.s[2]\n" - "fmla v15.4s, v27.4s, v3.s[3]\n" - "fmla v16.4s, v27.4s, v5.s[0]\n" - "fmla v17.4s, v27.4s, v5.s[1]\n" - "fmla v18.4s, v27.4s, v5.s[2]\n" - "fmla v19.4s, v27.4s, v5.s[3]\n" - "ldr q27, [%x[params], #0x90]\n" - "fmla v12.4s, v31.4s, v4.s[0]\n" - "fmla v13.4s, v31.4s, v4.s[1]\n" - "fmla v14.4s, v31.4s, v4.s[2]\n" - "fmla v15.4s, v31.4s, v4.s[3]\n" - "fmla v16.4s, v31.4s, v6.s[0]\n" - "fmla v17.4s, v31.4s, v6.s[1]\n" - "fmla v18.4s, v31.4s, v6.s[2]\n" - "fmla v19.4s, v31.4s, v6.s[3]\n" - "ldr q31, [%x[params], #0xa0]\n" - "fmla v12.4s, v30.4s, v4.s[1]\n" - "fmla v13.4s, v30.4s, v4.s[2]\n" - "fmla v14.4s, v30.4s, v4.s[3]\n" - "fmla v15.4s, v30.4s, v5.s[0]\n" - "fmla v16.4s, v30.4s, v6.s[1]\n" - "fmla v17.4s, v30.4s, v6.s[2]\n" - "fmla v18.4s, v30.4s, v6.s[3]\n" - "fmla v19.4s, v30.4s, v7.s[0]\n" - "ldr q30, [%x[params], #0xb0]\n" - "fmla v12.4s, v29.4s, v4.s[2]\n" - "fmla v13.4s, v29.4s, v4.s[3]\n" - "fmla v14.4s, v29.4s, v5.s[0]\n" - "fmla v15.4s, v29.4s, v5.s[1]\n" - "fmla v16.4s, v29.4s, v6.s[2]\n" - "fmla v17.4s, v29.4s, v6.s[3]\n" - "fmla v18.4s, v29.4s, v7.s[0]\n" - "fmla v19.4s, v29.4s, v7.s[1]\n" - "ldr q29, [%x[params], #0xc0]\n" - "fmla v12.4s, v28.4s, v4.s[3]\n" - "fmla v13.4s, v28.4s, v5.s[0]\n" - "fmla v14.4s, v28.4s, v5.s[1]\n" - "fmla v15.4s, v28.4s, v5.s[2]\n" - "fmla v16.4s, v28.4s, v6.s[3]\n" - "fmla v17.4s, v28.4s, v7.s[0]\n" - "fmla v18.4s, v28.4s, v7.s[1]\n" - "fmla v19.4s, v28.4s, v7.s[2]\n" - "ldr q28, [%x[params], #0xd0]\n" - "fmla v12.4s, v27.4s, v5.s[0]\n" - "fmla v13.4s, v27.4s, v5.s[1]\n" - "fmla v14.4s, v27.4s, v5.s[2]\n" - "fmla v15.4s, v27.4s, v5.s[3]\n" - "fmla v16.4s, v27.4s, v7.s[0]\n" - "fmla v17.4s, v27.4s, v7.s[1]\n" - "fmla v18.4s, v27.4s, v7.s[2]\n" - "fmla v19.4s, v27.4s, v7.s[3]\n" - "ldr q27, [%x[params], #0xe0]\n" - "fmla v12.4s, v31.4s, v6.s[0]\n" - "fmla v13.4s, v31.4s, v6.s[1]\n" - "fmla v14.4s, v31.4s, v6.s[2]\n" - "fmla v15.4s, v31.4s, v6.s[3]\n" - "fmla v16.4s, v31.4s, v8.s[0]\n" - "fmla v17.4s, v31.4s, v8.s[1]\n" - "fmla v18.4s, v31.4s, v8.s[2]\n" - "fmla v19.4s, v31.4s, v8.s[3]\n" - "ldr q31, [%x[params], #0xf0]\n" - "fmla v12.4s, v30.4s, v6.s[1]\n" - "fmla v13.4s, v30.4s, v6.s[2]\n" - "fmla v14.4s, v30.4s, v6.s[3]\n" - "fmla v15.4s, v30.4s, v7.s[0]\n" - "fmla v16.4s, v30.4s, v8.s[1]\n" - "fmla v17.4s, v30.4s, v8.s[2]\n" - "fmla v18.4s, v30.4s, v8.s[3]\n" - "fmla v19.4s, v30.4s, v9.s[0]\n" - "ldr q30, [%x[params], #0x100]\n" - "fmla v12.4s, v29.4s, v6.s[2]\n" - "fmla v13.4s, v29.4s, v6.s[3]\n" - "fmla v14.4s, v29.4s, v7.s[0]\n" - "fmla v15.4s, v29.4s, v7.s[1]\n" - "fmla v16.4s, v29.4s, v8.s[2]\n" - "fmla v17.4s, v29.4s, v8.s[3]\n" - "fmla v18.4s, v29.4s, v9.s[0]\n" - "fmla v19.4s, v29.4s, v9.s[1]\n" - "ldr q29, [%x[params], #0x110]\n" - "fmla v12.4s, v28.4s, v6.s[3]\n" - "fmla v13.4s, v28.4s, v7.s[0]\n" - "fmla v14.4s, v28.4s, v7.s[1]\n" - "fmla v15.4s, v28.4s, v7.s[2]\n" - "fmla v16.4s, v28.4s, v8.s[3]\n" - "fmla v17.4s, v28.4s, v9.s[0]\n" - "fmla v18.4s, v28.4s, v9.s[1]\n" - "fmla v19.4s, v28.4s, v9.s[2]\n" - "ldr q28, [%x[params], #0x120]\n" - "fmla v12.4s, v27.4s, v7.s[0]\n" - "fmla v13.4s, v27.4s, v7.s[1]\n" - "fmla v14.4s, v27.4s, v7.s[2]\n" - "fmla v15.4s, v27.4s, v7.s[3]\n" - "fmla v16.4s, v27.4s, v9.s[0]\n" - "fmla v17.4s, v27.4s, v9.s[1]\n" - "fmla v18.4s, v27.4s, v9.s[2]\n" - "fmla v19.4s, v27.4s, v9.s[3]\n" - "ldr q27, [%x[params], #0x130]\n" + "ldr q20, [%x[params], #0x40]\n" + "fmla v12.4s, v24.4s, v2.s[0]\n" + "fmla v13.4s, v24.4s, v2.s[1]\n" + "fmla v14.4s, v24.4s, v2.s[2]\n" + "fmla v15.4s, v24.4s, v2.s[3]\n" + "fmla v16.4s, v24.4s, v4.s[0]\n" + "fmla v17.4s, v24.4s, v4.s[1]\n" + "fmla v18.4s, v24.4s, v4.s[2]\n" + "fmla v19.4s, v24.4s, v4.s[3]\n" + "ldr q24, [%x[params], #0x50]\n" + "fmla v12.4s, v23.4s, v2.s[1]\n" + "fmla v13.4s, v23.4s, v2.s[2]\n" + "fmla v14.4s, v23.4s, v2.s[3]\n" + "fmla v15.4s, v23.4s, v3.s[0]\n" + "fmla v16.4s, v23.4s, v4.s[1]\n" + "fmla v17.4s, v23.4s, v4.s[2]\n" + "fmla v18.4s, v23.4s, v4.s[3]\n" + "fmla v19.4s, v23.4s, v5.s[0]\n" + "ldr q23, [%x[params], #0x60]\n" + "fmla v12.4s, v22.4s, v2.s[2]\n" + "fmla v13.4s, v22.4s, v2.s[3]\n" + "fmla v14.4s, v22.4s, v3.s[0]\n" + "fmla v15.4s, v22.4s, v3.s[1]\n" + "fmla v16.4s, v22.4s, v4.s[2]\n" + "fmla v17.4s, v22.4s, v4.s[3]\n" + "fmla v18.4s, v22.4s, v5.s[0]\n" + "fmla v19.4s, v22.4s, v5.s[1]\n" + "ldr q22, [%x[params], #0x70]\n" + "fmla v12.4s, v21.4s, v2.s[3]\n" + "fmla v13.4s, v21.4s, v3.s[0]\n" + "fmla v14.4s, v21.4s, v3.s[1]\n" + "fmla v15.4s, v21.4s, v3.s[2]\n" + "fmla v16.4s, v21.4s, v4.s[3]\n" + "fmla v17.4s, v21.4s, v5.s[0]\n" + "fmla v18.4s, v21.4s, v5.s[1]\n" + "fmla v19.4s, v21.4s, v5.s[2]\n" + "ldr q21, [%x[params], #0x80]\n" + "fmla v12.4s, v20.4s, v3.s[0]\n" + "fmla v13.4s, v20.4s, v3.s[1]\n" + "fmla v14.4s, v20.4s, v3.s[2]\n" + "fmla v15.4s, v20.4s, v3.s[3]\n" + "fmla v16.4s, v20.4s, v5.s[0]\n" + "fmla v17.4s, v20.4s, v5.s[1]\n" + "fmla v18.4s, v20.4s, v5.s[2]\n" + "fmla v19.4s, v20.4s, v5.s[3]\n" + "ldr q20, [%x[params], #0x90]\n" + "fmla v12.4s, v24.4s, v4.s[0]\n" + "fmla v13.4s, v24.4s, v4.s[1]\n" + "fmla v14.4s, v24.4s, v4.s[2]\n" + "fmla v15.4s, v24.4s, v4.s[3]\n" + "fmla v16.4s, v24.4s, v6.s[0]\n" + "fmla v17.4s, v24.4s, v6.s[1]\n" + "fmla v18.4s, v24.4s, v6.s[2]\n" + "fmla v19.4s, v24.4s, v6.s[3]\n" + "ldr q24, [%x[params], #0xa0]\n" + "fmla v12.4s, v23.4s, v4.s[1]\n" + "fmla v13.4s, v23.4s, v4.s[2]\n" + "fmla v14.4s, v23.4s, v4.s[3]\n" + "fmla v15.4s, v23.4s, v5.s[0]\n" + "fmla v16.4s, v23.4s, v6.s[1]\n" + "fmla v17.4s, v23.4s, v6.s[2]\n" + "fmla v18.4s, v23.4s, v6.s[3]\n" + "fmla v19.4s, v23.4s, v7.s[0]\n" + "ldr q23, [%x[params], #0xb0]\n" + "fmla v12.4s, v22.4s, v4.s[2]\n" + "fmla v13.4s, v22.4s, v4.s[3]\n" + "fmla v14.4s, v22.4s, v5.s[0]\n" + "fmla v15.4s, v22.4s, v5.s[1]\n" + "fmla v16.4s, v22.4s, v6.s[2]\n" + "fmla v17.4s, v22.4s, v6.s[3]\n" + "fmla v18.4s, v22.4s, v7.s[0]\n" + "fmla v19.4s, v22.4s, v7.s[1]\n" + "ldr q22, [%x[params], #0xc0]\n" + "fmla v12.4s, v21.4s, v4.s[3]\n" + "fmla v13.4s, v21.4s, v5.s[0]\n" + "fmla v14.4s, v21.4s, v5.s[1]\n" + "fmla v15.4s, v21.4s, v5.s[2]\n" + "fmla v16.4s, v21.4s, v6.s[3]\n" + "fmla v17.4s, v21.4s, v7.s[0]\n" + "fmla v18.4s, v21.4s, v7.s[1]\n" + "fmla v19.4s, v21.4s, v7.s[2]\n" + "ldr q21, [%x[params], #0xd0]\n" + "fmla v12.4s, v20.4s, v5.s[0]\n" + "fmla v13.4s, v20.4s, v5.s[1]\n" + "fmla v14.4s, v20.4s, v5.s[2]\n" + "fmla v15.4s, v20.4s, v5.s[3]\n" + "fmla v16.4s, v20.4s, v7.s[0]\n" + "fmla v17.4s, v20.4s, v7.s[1]\n" + "fmla v18.4s, v20.4s, v7.s[2]\n" + "fmla v19.4s, v20.4s, v7.s[3]\n" + "ldr q20, [%x[params], #0xe0]\n" + "fmla v12.4s, v24.4s, v6.s[0]\n" + "fmla v13.4s, v24.4s, v6.s[1]\n" + "fmla v14.4s, v24.4s, v6.s[2]\n" + "fmla v15.4s, v24.4s, v6.s[3]\n" + "fmla v16.4s, v24.4s, v8.s[0]\n" + "fmla v17.4s, v24.4s, v8.s[1]\n" + "fmla v18.4s, v24.4s, v8.s[2]\n" + "fmla v19.4s, v24.4s, v8.s[3]\n" + "ldr q24, [%x[params], #0xf0]\n" + "fmla v12.4s, v23.4s, v6.s[1]\n" + "fmla v13.4s, v23.4s, v6.s[2]\n" + "fmla v14.4s, v23.4s, v6.s[3]\n" + "fmla v15.4s, v23.4s, v7.s[0]\n" + "fmla v16.4s, v23.4s, v8.s[1]\n" + "fmla v17.4s, v23.4s, v8.s[2]\n" + "fmla v18.4s, v23.4s, v8.s[3]\n" + "fmla v19.4s, v23.4s, v9.s[0]\n" + "ldr q23, [%x[params], #0x100]\n" + "fmla v12.4s, v22.4s, v6.s[2]\n" + "fmla v13.4s, v22.4s, v6.s[3]\n" + "fmla v14.4s, v22.4s, v7.s[0]\n" + "fmla v15.4s, v22.4s, v7.s[1]\n" + "fmla v16.4s, v22.4s, v8.s[2]\n" + "fmla v17.4s, v22.4s, v8.s[3]\n" + "fmla v18.4s, v22.4s, v9.s[0]\n" + "fmla v19.4s, v22.4s, v9.s[1]\n" + "ldr q22, [%x[params], #0x110]\n" + "fmla v12.4s, v21.4s, v6.s[3]\n" + "fmla v13.4s, v21.4s, v7.s[0]\n" + "fmla v14.4s, v21.4s, v7.s[1]\n" + "fmla v15.4s, v21.4s, v7.s[2]\n" + "fmla v16.4s, v21.4s, v8.s[3]\n" + "fmla v17.4s, v21.4s, v9.s[0]\n" + "fmla v18.4s, v21.4s, v9.s[1]\n" + "fmla v19.4s, v21.4s, v9.s[2]\n" + "ldr q21, [%x[params], #0x120]\n" + "fmla v12.4s, v20.4s, v7.s[0]\n" + "fmla v13.4s, v20.4s, v7.s[1]\n" + "fmla v14.4s, v20.4s, v7.s[2]\n" + "fmla v15.4s, v20.4s, v7.s[3]\n" + "fmla v16.4s, v20.4s, v9.s[0]\n" + "fmla v17.4s, v20.4s, v9.s[1]\n" + "fmla v18.4s, v20.4s, v9.s[2]\n" + "fmla v19.4s, v20.4s, v9.s[3]\n" + "ldr q20, [%x[params], #0x130]\n" "add %x[params], %x[params], #0x140\n" - "fmla v12.4s, v31.4s, v8.s[0]\n" - "fmla v13.4s, v31.4s, v8.s[1]\n" - "fmla v14.4s, v31.4s, v8.s[2]\n" - "fmla v15.4s, v31.4s, v8.s[3]\n" - "fmla v16.4s, v31.4s, v10.s[0]\n" - "fmla v17.4s, v31.4s, v10.s[1]\n" - "fmla v18.4s, v31.4s, v10.s[2]\n" - "fmla v19.4s, v31.4s, v10.s[3]\n" - "fmla v12.4s, v30.4s, v8.s[1]\n" - "fmla v13.4s, v30.4s, v8.s[2]\n" - "fmla v14.4s, v30.4s, v8.s[3]\n" - "fmla v15.4s, v30.4s, v9.s[0]\n" - "fmla v16.4s, v30.4s, v10.s[1]\n" - "fmla v17.4s, v30.4s, v10.s[2]\n" - "fmla v18.4s, v30.4s, v10.s[3]\n" - "fmla v19.4s, v30.4s, v11.s[0]\n" - "fmla v12.4s, v29.4s, v8.s[2]\n" - "fmla v13.4s, v29.4s, v8.s[3]\n" - "fmla v14.4s, v29.4s, v9.s[0]\n" - "fmla v15.4s, v29.4s, v9.s[1]\n" - "fmla v16.4s, v29.4s, v10.s[2]\n" - "fmla v17.4s, v29.4s, v10.s[3]\n" - "fmla v18.4s, v29.4s, v11.s[0]\n" - "fmla v19.4s, v29.4s, v11.s[1]\n" - "fmla v12.4s, v28.4s, v8.s[3]\n" - "fmla v13.4s, v28.4s, v9.s[0]\n" - "fmla v14.4s, v28.4s, v9.s[1]\n" - "fmla v15.4s, v28.4s, v9.s[2]\n" - "fmla v16.4s, v28.4s, v10.s[3]\n" - "fmla v17.4s, v28.4s, v11.s[0]\n" - "fmla v18.4s, v28.4s, v11.s[1]\n" - "fmla v19.4s, v28.4s, v11.s[2]\n" - "fmla v12.4s, v27.4s, v9.s[0]\n" - "fmla v13.4s, v27.4s, v9.s[1]\n" - "fmin v12.4s, v12.4s, v20.4s\n" - "fmla v14.4s, v27.4s, v9.s[2]\n" - "fmla v15.4s, v27.4s, v9.s[3]\n" - "fmin v13.4s, v13.4s, v20.4s\n" - "fmla v16.4s, v27.4s, v11.s[0]\n" - "fmla v17.4s, v27.4s, v11.s[1]\n" - "fmin v14.4s, v14.4s, v20.4s\n" - "fmla v18.4s, v27.4s, v11.s[2]\n" - "fmla v19.4s, v27.4s, v11.s[3]\n" - "fmin v15.4s, v15.4s, v20.4s\n" - "fmin v16.4s, v16.4s, v20.4s\n" - "fmin v17.4s, v17.4s, v20.4s\n" - "fmin v18.4s, v18.4s, v20.4s\n" - "fmin v19.4s, v19.4s, v20.4s\n" - "fmax v12.4s, v12.4s, v21.4s\n" - "fmax v13.4s, v13.4s, v21.4s\n" + "fmla v12.4s, v24.4s, v8.s[0]\n" + "fmla v13.4s, v24.4s, v8.s[1]\n" + "fmla v14.4s, v24.4s, v8.s[2]\n" + "fmla v15.4s, v24.4s, v8.s[3]\n" + "fmla v16.4s, v24.4s, v10.s[0]\n" + "fmla v17.4s, v24.4s, v10.s[1]\n" + "fmla v18.4s, v24.4s, v10.s[2]\n" + "fmla v19.4s, v24.4s, v10.s[3]\n" + "fmla v12.4s, v23.4s, v8.s[1]\n" + "fmla v13.4s, v23.4s, v8.s[2]\n" + "fmla v14.4s, v23.4s, v8.s[3]\n" + "fmla v15.4s, v23.4s, v9.s[0]\n" + "fmla v16.4s, v23.4s, v10.s[1]\n" + "fmla v17.4s, v23.4s, v10.s[2]\n" + "fmla v18.4s, v23.4s, v10.s[3]\n" + "fmla v19.4s, v23.4s, v11.s[0]\n" + "fmla v12.4s, v22.4s, v8.s[2]\n" + "fmla v13.4s, v22.4s, v8.s[3]\n" + "fmla v14.4s, v22.4s, v9.s[0]\n" + "fmla v15.4s, v22.4s, v9.s[1]\n" + "fmla v16.4s, v22.4s, v10.s[2]\n" + "fmla v17.4s, v22.4s, v10.s[3]\n" + "fmla v18.4s, v22.4s, v11.s[0]\n" + "fmla v19.4s, v22.4s, v11.s[1]\n" + "fmla v12.4s, v21.4s, v8.s[3]\n" + "fmla v13.4s, v21.4s, v9.s[0]\n" + "fmla v14.4s, v21.4s, v9.s[1]\n" + "fmla v15.4s, v21.4s, v9.s[2]\n" + "fmla v16.4s, v21.4s, v10.s[3]\n" + "fmla v17.4s, v21.4s, v11.s[0]\n" + "fmla v18.4s, v21.4s, v11.s[1]\n" + "fmla v19.4s, v21.4s, v11.s[2]\n" + "fmla v12.4s, v20.4s, v9.s[0]\n" + "fmla v13.4s, v20.4s, v9.s[1]\n" + "fmin v12.4s, v12.4s, v25.4s\n" + "fmla v14.4s, v20.4s, v9.s[2]\n" + "fmla v15.4s, v20.4s, v9.s[3]\n" + "fmin v13.4s, v13.4s, v25.4s\n" + "fmla v16.4s, v20.4s, v11.s[0]\n" + "fmla v17.4s, v20.4s, v11.s[1]\n" + "fmin v14.4s, v14.4s, v25.4s\n" + "fmla v18.4s, v20.4s, v11.s[2]\n" + "fmla v19.4s, v20.4s, v11.s[3]\n" + "fmin v15.4s, v15.4s, v25.4s\n" + "fmin v16.4s, v16.4s, v25.4s\n" + "fmin v17.4s, v17.4s, v25.4s\n" + "fmin v18.4s, v18.4s, v25.4s\n" + "fmin v19.4s, v19.4s, v25.4s\n" + "fmax v12.4s, v12.4s, v26.4s\n" + "fmax v13.4s, v13.4s, v26.4s\n" "str q12, [x12, x13]\n" - "fmax v14.4s, v14.4s, v21.4s\n" - "fmax v15.4s, v15.4s, v21.4s\n" + "fmax v14.4s, v14.4s, v26.4s\n" + "fmax v15.4s, v15.4s, v26.4s\n" "str q13, [x11, x13]\n" - "fmax v16.4s, v16.4s, v21.4s\n" - "fmax v17.4s, v17.4s, v21.4s\n" + "fmax v16.4s, v16.4s, v26.4s\n" + "fmax v17.4s, v17.4s, v26.4s\n" "str q14, [x10, x13]\n" - "fmax v18.4s, v18.4s, v21.4s\n" - "fmax v19.4s, v19.4s, v21.4s\n" + "fmax v18.4s, v18.4s, v26.4s\n" + "fmax v19.4s, v19.4s, v26.4s\n" "str q15, [x9, x13]\n" "str q16, [x28, x13]\n" "str q17, [x27, x13]\n" @@ -601,255 +601,255 @@ void a64_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst_imp "tst %x[channel_multiplier], #0x3\n" "beq 6f\n" "ldr q12, [%x[params], #0x0]\n" - "ldr q31, [%x[params], #0x10]\n" + "ldr q24, [%x[params], #0x10]\n" "mov v13.16b, v12.16b\n" "mov v14.16b, v12.16b\n" - "ldr q30, [%x[params], #0x20]\n" - "ldr q29, [%x[params], #0x30]\n" + "ldr q23, [%x[params], #0x20]\n" + "ldr q22, [%x[params], #0x30]\n" "mov v15.16b, v12.16b\n" "mov v16.16b, v12.16b\n" - "ldr q28, [%x[params], #0x40]\n" - "ldr q27, [%x[params], #0x50]\n" + "ldr q21, [%x[params], #0x40]\n" + "ldr q20, [%x[params], #0x50]\n" "mov v17.16b, v12.16b\n" "mov v18.16b, v12.16b\n" "mov v19.16b, v12.16b\n" - "fmla v12.4s, v31.4s, v0.s[0]\n" - "fmla v13.4s, v31.4s, v0.s[1]\n" - "fmla v14.4s, v31.4s, v0.s[2]\n" - "fmla v15.4s, v31.4s, v0.s[3]\n" - "fmla v16.4s, v31.4s, v2.s[0]\n" - "fmla v17.4s, v31.4s, v2.s[1]\n" - "fmla v18.4s, v31.4s, v2.s[2]\n" - "fmla v19.4s, v31.4s, v2.s[3]\n" - "ldr q31, [%x[params], #0x60]\n" - "fmla v12.4s, v30.4s, v0.s[1]\n" - "fmla v13.4s, v30.4s, v0.s[2]\n" - "fmla v14.4s, v30.4s, v0.s[3]\n" - "fmla v15.4s, v30.4s, v1.s[0]\n" - "fmla v16.4s, v30.4s, v2.s[1]\n" - "fmla v17.4s, v30.4s, v2.s[2]\n" - "fmla v18.4s, v30.4s, v2.s[3]\n" - "fmla v19.4s, v30.4s, v3.s[0]\n" - "ldr q30, [%x[params], #0x70]\n" - "fmla v12.4s, v29.4s, v0.s[2]\n" - "fmla v13.4s, v29.4s, v0.s[3]\n" - "fmla v14.4s, v29.4s, v1.s[0]\n" - "fmla v15.4s, v29.4s, v1.s[1]\n" - "fmla v16.4s, v29.4s, v2.s[2]\n" - "fmla v17.4s, v29.4s, v2.s[3]\n" - "fmla v18.4s, v29.4s, v3.s[0]\n" - "fmla v19.4s, v29.4s, v3.s[1]\n" - "ldr q29, [%x[params], #0x80]\n" - "fmla v12.4s, v28.4s, v0.s[3]\n" - "fmla v13.4s, v28.4s, v1.s[0]\n" - "fmla v14.4s, v28.4s, v1.s[1]\n" - "fmla v15.4s, v28.4s, v1.s[2]\n" - "fmla v16.4s, v28.4s, v2.s[3]\n" - "fmla v17.4s, v28.4s, v3.s[0]\n" - "fmla v18.4s, v28.4s, v3.s[1]\n" - "fmla v19.4s, v28.4s, v3.s[2]\n" - "ldr q28, [%x[params], #0x90]\n" - "fmla v12.4s, v27.4s, v1.s[0]\n" - "fmla v13.4s, v27.4s, v1.s[1]\n" - "fmla v14.4s, v27.4s, v1.s[2]\n" - "fmla v15.4s, v27.4s, v1.s[3]\n" - "fmla v16.4s, v27.4s, v3.s[0]\n" - "fmla v17.4s, v27.4s, v3.s[1]\n" - "fmla v18.4s, v27.4s, v3.s[2]\n" - "fmla v19.4s, v27.4s, v3.s[3]\n" - "ldr q27, [%x[params], #0xa0]\n" - "fmla v12.4s, v31.4s, v2.s[0]\n" - "fmla v13.4s, v31.4s, v2.s[1]\n" - "fmla v14.4s, v31.4s, v2.s[2]\n" - "fmla v15.4s, v31.4s, v2.s[3]\n" - "fmla v16.4s, v31.4s, v4.s[0]\n" - "fmla v17.4s, v31.4s, v4.s[1]\n" - "fmla v18.4s, v31.4s, v4.s[2]\n" - "fmla v19.4s, v31.4s, v4.s[3]\n" - "ldr q31, [%x[params], #0xb0]\n" - "fmla v12.4s, v30.4s, v2.s[1]\n" - "fmla v13.4s, v30.4s, v2.s[2]\n" - "fmla v14.4s, v30.4s, v2.s[3]\n" - "fmla v15.4s, v30.4s, v3.s[0]\n" - "fmla v16.4s, v30.4s, v4.s[1]\n" - "fmla v17.4s, v30.4s, v4.s[2]\n" - "fmla v18.4s, v30.4s, v4.s[3]\n" - "fmla v19.4s, v30.4s, v5.s[0]\n" - "ldr q30, [%x[params], #0xc0]\n" - "fmla v12.4s, v29.4s, v2.s[2]\n" - "fmla v13.4s, v29.4s, v2.s[3]\n" - "fmla v14.4s, v29.4s, v3.s[0]\n" - "fmla v15.4s, v29.4s, v3.s[1]\n" - "fmla v16.4s, v29.4s, v4.s[2]\n" - "fmla v17.4s, v29.4s, v4.s[3]\n" - "fmla v18.4s, v29.4s, v5.s[0]\n" - "fmla v19.4s, v29.4s, v5.s[1]\n" - "ldr q29, [%x[params], #0xd0]\n" - "fmla v12.4s, v28.4s, v2.s[3]\n" - "fmla v13.4s, v28.4s, v3.s[0]\n" - "fmla v14.4s, v28.4s, v3.s[1]\n" - "fmla v15.4s, v28.4s, v3.s[2]\n" - "fmla v16.4s, v28.4s, v4.s[3]\n" - "fmla v17.4s, v28.4s, v5.s[0]\n" - "fmla v18.4s, v28.4s, v5.s[1]\n" - "fmla v19.4s, v28.4s, v5.s[2]\n" - "ldr q28, [%x[params], #0xe0]\n" - "fmla v12.4s, v27.4s, v3.s[0]\n" - "fmla v13.4s, v27.4s, v3.s[1]\n" - "fmla v14.4s, v27.4s, v3.s[2]\n" - "fmla v15.4s, v27.4s, v3.s[3]\n" - "fmla v16.4s, v27.4s, v5.s[0]\n" - "fmla v17.4s, v27.4s, v5.s[1]\n" - "fmla v18.4s, v27.4s, v5.s[2]\n" - "fmla v19.4s, v27.4s, v5.s[3]\n" - "ldr q27, [%x[params], #0xf0]\n" - "fmla v12.4s, v31.4s, v4.s[0]\n" - "fmla v13.4s, v31.4s, v4.s[1]\n" - "fmla v14.4s, v31.4s, v4.s[2]\n" - "fmla v15.4s, v31.4s, v4.s[3]\n" - "fmla v16.4s, v31.4s, v6.s[0]\n" - "fmla v17.4s, v31.4s, v6.s[1]\n" - "fmla v18.4s, v31.4s, v6.s[2]\n" - "fmla v19.4s, v31.4s, v6.s[3]\n" - "ldr q31, [%x[params], #0x100]\n" - "fmla v12.4s, v30.4s, v4.s[1]\n" - "fmla v13.4s, v30.4s, v4.s[2]\n" - "fmla v14.4s, v30.4s, v4.s[3]\n" - "fmla v15.4s, v30.4s, v5.s[0]\n" - "fmla v16.4s, v30.4s, v6.s[1]\n" - "fmla v17.4s, v30.4s, v6.s[2]\n" - "fmla v18.4s, v30.4s, v6.s[3]\n" - "fmla v19.4s, v30.4s, v7.s[0]\n" - "ldr q30, [%x[params], #0x110]\n" - "fmla v12.4s, v29.4s, v4.s[2]\n" - "fmla v13.4s, v29.4s, v4.s[3]\n" - "fmla v14.4s, v29.4s, v5.s[0]\n" - "fmla v15.4s, v29.4s, v5.s[1]\n" - "fmla v16.4s, v29.4s, v6.s[2]\n" - "fmla v17.4s, v29.4s, v6.s[3]\n" - "fmla v18.4s, v29.4s, v7.s[0]\n" - "fmla v19.4s, v29.4s, v7.s[1]\n" - "ldr q29, [%x[params], #0x120]\n" - "fmla v12.4s, v28.4s, v4.s[3]\n" - "fmla v13.4s, v28.4s, v5.s[0]\n" - "fmla v14.4s, v28.4s, v5.s[1]\n" - "fmla v15.4s, v28.4s, v5.s[2]\n" - "fmla v16.4s, v28.4s, v6.s[3]\n" - "fmla v17.4s, v28.4s, v7.s[0]\n" - "fmla v18.4s, v28.4s, v7.s[1]\n" - "fmla v19.4s, v28.4s, v7.s[2]\n" - "ldr q28, [%x[params], #0x130]\n" - "fmla v12.4s, v27.4s, v5.s[0]\n" - "fmla v13.4s, v27.4s, v5.s[1]\n" - "fmla v14.4s, v27.4s, v5.s[2]\n" - "fmla v15.4s, v27.4s, v5.s[3]\n" - "fmla v16.4s, v27.4s, v7.s[0]\n" - "fmla v17.4s, v27.4s, v7.s[1]\n" - "fmla v18.4s, v27.4s, v7.s[2]\n" - "fmla v19.4s, v27.4s, v7.s[3]\n" - "ldr q27, [%x[params], #0x140]\n" - "fmla v12.4s, v31.4s, v6.s[0]\n" - "fmla v13.4s, v31.4s, v6.s[1]\n" - "fmla v14.4s, v31.4s, v6.s[2]\n" - "fmla v15.4s, v31.4s, v6.s[3]\n" - "fmla v16.4s, v31.4s, v8.s[0]\n" - "fmla v17.4s, v31.4s, v8.s[1]\n" - "fmla v18.4s, v31.4s, v8.s[2]\n" - "fmla v19.4s, v31.4s, v8.s[3]\n" - "ldr q31, [%x[params], #0x150]\n" - "fmla v12.4s, v30.4s, v6.s[1]\n" - "fmla v13.4s, v30.4s, v6.s[2]\n" - "fmla v14.4s, v30.4s, v6.s[3]\n" - "fmla v15.4s, v30.4s, v7.s[0]\n" - "fmla v16.4s, v30.4s, v8.s[1]\n" - "fmla v17.4s, v30.4s, v8.s[2]\n" - "fmla v18.4s, v30.4s, v8.s[3]\n" - "fmla v19.4s, v30.4s, v9.s[0]\n" - "ldr q30, [%x[params], #0x160]\n" - "fmla v12.4s, v29.4s, v6.s[2]\n" - "fmla v13.4s, v29.4s, v6.s[3]\n" - "fmla v14.4s, v29.4s, v7.s[0]\n" - "fmla v15.4s, v29.4s, v7.s[1]\n" - "fmla v16.4s, v29.4s, v8.s[2]\n" - "fmla v17.4s, v29.4s, v8.s[3]\n" - "fmla v18.4s, v29.4s, v9.s[0]\n" - "fmla v19.4s, v29.4s, v9.s[1]\n" - "ldr q29, [%x[params], #0x170]\n" - "fmla v12.4s, v28.4s, v6.s[3]\n" - "fmla v13.4s, v28.4s, v7.s[0]\n" - "fmla v14.4s, v28.4s, v7.s[1]\n" - "fmla v15.4s, v28.4s, v7.s[2]\n" - "fmla v16.4s, v28.4s, v8.s[3]\n" - "fmla v17.4s, v28.4s, v9.s[0]\n" - "fmla v18.4s, v28.4s, v9.s[1]\n" - "fmla v19.4s, v28.4s, v9.s[2]\n" - "ldr q28, [%x[params], #0x180]\n" - "fmla v12.4s, v27.4s, v7.s[0]\n" - "fmla v13.4s, v27.4s, v7.s[1]\n" - "fmla v14.4s, v27.4s, v7.s[2]\n" - "fmla v15.4s, v27.4s, v7.s[3]\n" - "fmla v16.4s, v27.4s, v9.s[0]\n" - "fmla v17.4s, v27.4s, v9.s[1]\n" - "fmla v18.4s, v27.4s, v9.s[2]\n" - "fmla v19.4s, v27.4s, v9.s[3]\n" - "ldr q27, [%x[params], #0x190]\n" + "fmla v12.4s, v24.4s, v0.s[0]\n" + "fmla v13.4s, v24.4s, v0.s[1]\n" + "fmla v14.4s, v24.4s, v0.s[2]\n" + "fmla v15.4s, v24.4s, v0.s[3]\n" + "fmla v16.4s, v24.4s, v2.s[0]\n" + "fmla v17.4s, v24.4s, v2.s[1]\n" + "fmla v18.4s, v24.4s, v2.s[2]\n" + "fmla v19.4s, v24.4s, v2.s[3]\n" + "ldr q24, [%x[params], #0x60]\n" + "fmla v12.4s, v23.4s, v0.s[1]\n" + "fmla v13.4s, v23.4s, v0.s[2]\n" + "fmla v14.4s, v23.4s, v0.s[3]\n" + "fmla v15.4s, v23.4s, v1.s[0]\n" + "fmla v16.4s, v23.4s, v2.s[1]\n" + "fmla v17.4s, v23.4s, v2.s[2]\n" + "fmla v18.4s, v23.4s, v2.s[3]\n" + "fmla v19.4s, v23.4s, v3.s[0]\n" + "ldr q23, [%x[params], #0x70]\n" + "fmla v12.4s, v22.4s, v0.s[2]\n" + "fmla v13.4s, v22.4s, v0.s[3]\n" + "fmla v14.4s, v22.4s, v1.s[0]\n" + "fmla v15.4s, v22.4s, v1.s[1]\n" + "fmla v16.4s, v22.4s, v2.s[2]\n" + "fmla v17.4s, v22.4s, v2.s[3]\n" + "fmla v18.4s, v22.4s, v3.s[0]\n" + "fmla v19.4s, v22.4s, v3.s[1]\n" + "ldr q22, [%x[params], #0x80]\n" + "fmla v12.4s, v21.4s, v0.s[3]\n" + "fmla v13.4s, v21.4s, v1.s[0]\n" + "fmla v14.4s, v21.4s, v1.s[1]\n" + "fmla v15.4s, v21.4s, v1.s[2]\n" + "fmla v16.4s, v21.4s, v2.s[3]\n" + "fmla v17.4s, v21.4s, v3.s[0]\n" + "fmla v18.4s, v21.4s, v3.s[1]\n" + "fmla v19.4s, v21.4s, v3.s[2]\n" + "ldr q21, [%x[params], #0x90]\n" + "fmla v12.4s, v20.4s, v1.s[0]\n" + "fmla v13.4s, v20.4s, v1.s[1]\n" + "fmla v14.4s, v20.4s, v1.s[2]\n" + "fmla v15.4s, v20.4s, v1.s[3]\n" + "fmla v16.4s, v20.4s, v3.s[0]\n" + "fmla v17.4s, v20.4s, v3.s[1]\n" + "fmla v18.4s, v20.4s, v3.s[2]\n" + "fmla v19.4s, v20.4s, v3.s[3]\n" + "ldr q20, [%x[params], #0xa0]\n" + "fmla v12.4s, v24.4s, v2.s[0]\n" + "fmla v13.4s, v24.4s, v2.s[1]\n" + "fmla v14.4s, v24.4s, v2.s[2]\n" + "fmla v15.4s, v24.4s, v2.s[3]\n" + "fmla v16.4s, v24.4s, v4.s[0]\n" + "fmla v17.4s, v24.4s, v4.s[1]\n" + "fmla v18.4s, v24.4s, v4.s[2]\n" + "fmla v19.4s, v24.4s, v4.s[3]\n" + "ldr q24, [%x[params], #0xb0]\n" + "fmla v12.4s, v23.4s, v2.s[1]\n" + "fmla v13.4s, v23.4s, v2.s[2]\n" + "fmla v14.4s, v23.4s, v2.s[3]\n" + "fmla v15.4s, v23.4s, v3.s[0]\n" + "fmla v16.4s, v23.4s, v4.s[1]\n" + "fmla v17.4s, v23.4s, v4.s[2]\n" + "fmla v18.4s, v23.4s, v4.s[3]\n" + "fmla v19.4s, v23.4s, v5.s[0]\n" + "ldr q23, [%x[params], #0xc0]\n" + "fmla v12.4s, v22.4s, v2.s[2]\n" + "fmla v13.4s, v22.4s, v2.s[3]\n" + "fmla v14.4s, v22.4s, v3.s[0]\n" + "fmla v15.4s, v22.4s, v3.s[1]\n" + "fmla v16.4s, v22.4s, v4.s[2]\n" + "fmla v17.4s, v22.4s, v4.s[3]\n" + "fmla v18.4s, v22.4s, v5.s[0]\n" + "fmla v19.4s, v22.4s, v5.s[1]\n" + "ldr q22, [%x[params], #0xd0]\n" + "fmla v12.4s, v21.4s, v2.s[3]\n" + "fmla v13.4s, v21.4s, v3.s[0]\n" + "fmla v14.4s, v21.4s, v3.s[1]\n" + "fmla v15.4s, v21.4s, v3.s[2]\n" + "fmla v16.4s, v21.4s, v4.s[3]\n" + "fmla v17.4s, v21.4s, v5.s[0]\n" + "fmla v18.4s, v21.4s, v5.s[1]\n" + "fmla v19.4s, v21.4s, v5.s[2]\n" + "ldr q21, [%x[params], #0xe0]\n" + "fmla v12.4s, v20.4s, v3.s[0]\n" + "fmla v13.4s, v20.4s, v3.s[1]\n" + "fmla v14.4s, v20.4s, v3.s[2]\n" + "fmla v15.4s, v20.4s, v3.s[3]\n" + "fmla v16.4s, v20.4s, v5.s[0]\n" + "fmla v17.4s, v20.4s, v5.s[1]\n" + "fmla v18.4s, v20.4s, v5.s[2]\n" + "fmla v19.4s, v20.4s, v5.s[3]\n" + "ldr q20, [%x[params], #0xf0]\n" + "fmla v12.4s, v24.4s, v4.s[0]\n" + "fmla v13.4s, v24.4s, v4.s[1]\n" + "fmla v14.4s, v24.4s, v4.s[2]\n" + "fmla v15.4s, v24.4s, v4.s[3]\n" + "fmla v16.4s, v24.4s, v6.s[0]\n" + "fmla v17.4s, v24.4s, v6.s[1]\n" + "fmla v18.4s, v24.4s, v6.s[2]\n" + "fmla v19.4s, v24.4s, v6.s[3]\n" + "ldr q24, [%x[params], #0x100]\n" + "fmla v12.4s, v23.4s, v4.s[1]\n" + "fmla v13.4s, v23.4s, v4.s[2]\n" + "fmla v14.4s, v23.4s, v4.s[3]\n" + "fmla v15.4s, v23.4s, v5.s[0]\n" + "fmla v16.4s, v23.4s, v6.s[1]\n" + "fmla v17.4s, v23.4s, v6.s[2]\n" + "fmla v18.4s, v23.4s, v6.s[3]\n" + "fmla v19.4s, v23.4s, v7.s[0]\n" + "ldr q23, [%x[params], #0x110]\n" + "fmla v12.4s, v22.4s, v4.s[2]\n" + "fmla v13.4s, v22.4s, v4.s[3]\n" + "fmla v14.4s, v22.4s, v5.s[0]\n" + "fmla v15.4s, v22.4s, v5.s[1]\n" + "fmla v16.4s, v22.4s, v6.s[2]\n" + "fmla v17.4s, v22.4s, v6.s[3]\n" + "fmla v18.4s, v22.4s, v7.s[0]\n" + "fmla v19.4s, v22.4s, v7.s[1]\n" + "ldr q22, [%x[params], #0x120]\n" + "fmla v12.4s, v21.4s, v4.s[3]\n" + "fmla v13.4s, v21.4s, v5.s[0]\n" + "fmla v14.4s, v21.4s, v5.s[1]\n" + "fmla v15.4s, v21.4s, v5.s[2]\n" + "fmla v16.4s, v21.4s, v6.s[3]\n" + "fmla v17.4s, v21.4s, v7.s[0]\n" + "fmla v18.4s, v21.4s, v7.s[1]\n" + "fmla v19.4s, v21.4s, v7.s[2]\n" + "ldr q21, [%x[params], #0x130]\n" + "fmla v12.4s, v20.4s, v5.s[0]\n" + "fmla v13.4s, v20.4s, v5.s[1]\n" + "fmla v14.4s, v20.4s, v5.s[2]\n" + "fmla v15.4s, v20.4s, v5.s[3]\n" + "fmla v16.4s, v20.4s, v7.s[0]\n" + "fmla v17.4s, v20.4s, v7.s[1]\n" + "fmla v18.4s, v20.4s, v7.s[2]\n" + "fmla v19.4s, v20.4s, v7.s[3]\n" + "ldr q20, [%x[params], #0x140]\n" + "fmla v12.4s, v24.4s, v6.s[0]\n" + "fmla v13.4s, v24.4s, v6.s[1]\n" + "fmla v14.4s, v24.4s, v6.s[2]\n" + "fmla v15.4s, v24.4s, v6.s[3]\n" + "fmla v16.4s, v24.4s, v8.s[0]\n" + "fmla v17.4s, v24.4s, v8.s[1]\n" + "fmla v18.4s, v24.4s, v8.s[2]\n" + "fmla v19.4s, v24.4s, v8.s[3]\n" + "ldr q24, [%x[params], #0x150]\n" + "fmla v12.4s, v23.4s, v6.s[1]\n" + "fmla v13.4s, v23.4s, v6.s[2]\n" + "fmla v14.4s, v23.4s, v6.s[3]\n" + "fmla v15.4s, v23.4s, v7.s[0]\n" + "fmla v16.4s, v23.4s, v8.s[1]\n" + "fmla v17.4s, v23.4s, v8.s[2]\n" + "fmla v18.4s, v23.4s, v8.s[3]\n" + "fmla v19.4s, v23.4s, v9.s[0]\n" + "ldr q23, [%x[params], #0x160]\n" + "fmla v12.4s, v22.4s, v6.s[2]\n" + "fmla v13.4s, v22.4s, v6.s[3]\n" + "fmla v14.4s, v22.4s, v7.s[0]\n" + "fmla v15.4s, v22.4s, v7.s[1]\n" + "fmla v16.4s, v22.4s, v8.s[2]\n" + "fmla v17.4s, v22.4s, v8.s[3]\n" + "fmla v18.4s, v22.4s, v9.s[0]\n" + "fmla v19.4s, v22.4s, v9.s[1]\n" + "ldr q22, [%x[params], #0x170]\n" + "fmla v12.4s, v21.4s, v6.s[3]\n" + "fmla v13.4s, v21.4s, v7.s[0]\n" + "fmla v14.4s, v21.4s, v7.s[1]\n" + "fmla v15.4s, v21.4s, v7.s[2]\n" + "fmla v16.4s, v21.4s, v8.s[3]\n" + "fmla v17.4s, v21.4s, v9.s[0]\n" + "fmla v18.4s, v21.4s, v9.s[1]\n" + "fmla v19.4s, v21.4s, v9.s[2]\n" + "ldr q21, [%x[params], #0x180]\n" + "fmla v12.4s, v20.4s, v7.s[0]\n" + "fmla v13.4s, v20.4s, v7.s[1]\n" + "fmla v14.4s, v20.4s, v7.s[2]\n" + "fmla v15.4s, v20.4s, v7.s[3]\n" + "fmla v16.4s, v20.4s, v9.s[0]\n" + "fmla v17.4s, v20.4s, v9.s[1]\n" + "fmla v18.4s, v20.4s, v9.s[2]\n" + "fmla v19.4s, v20.4s, v9.s[3]\n" + "ldr q20, [%x[params], #0x190]\n" "add %x[params], %x[params], #0x1a0\n" - "fmla v12.4s, v31.4s, v8.s[0]\n" - "fmla v13.4s, v31.4s, v8.s[1]\n" - "fmla v14.4s, v31.4s, v8.s[2]\n" - "fmla v15.4s, v31.4s, v8.s[3]\n" - "fmla v16.4s, v31.4s, v10.s[0]\n" - "fmla v17.4s, v31.4s, v10.s[1]\n" - "fmla v18.4s, v31.4s, v10.s[2]\n" - "fmla v19.4s, v31.4s, v10.s[3]\n" - "fmla v12.4s, v30.4s, v8.s[1]\n" - "fmla v13.4s, v30.4s, v8.s[2]\n" - "fmla v14.4s, v30.4s, v8.s[3]\n" - "fmla v15.4s, v30.4s, v9.s[0]\n" - "fmla v16.4s, v30.4s, v10.s[1]\n" - "fmla v17.4s, v30.4s, v10.s[2]\n" - "fmla v18.4s, v30.4s, v10.s[3]\n" - "fmla v19.4s, v30.4s, v11.s[0]\n" - "fmla v12.4s, v29.4s, v8.s[2]\n" - "fmla v13.4s, v29.4s, v8.s[3]\n" - "fmla v14.4s, v29.4s, v9.s[0]\n" - "fmla v15.4s, v29.4s, v9.s[1]\n" - "fmla v16.4s, v29.4s, v10.s[2]\n" - "fmla v17.4s, v29.4s, v10.s[3]\n" - "fmla v18.4s, v29.4s, v11.s[0]\n" - "fmla v19.4s, v29.4s, v11.s[1]\n" - "fmla v12.4s, v28.4s, v8.s[3]\n" - "fmla v13.4s, v28.4s, v9.s[0]\n" - "fmla v14.4s, v28.4s, v9.s[1]\n" - "fmla v15.4s, v28.4s, v9.s[2]\n" - "fmla v16.4s, v28.4s, v10.s[3]\n" - "fmla v17.4s, v28.4s, v11.s[0]\n" - "fmla v18.4s, v28.4s, v11.s[1]\n" - "fmla v19.4s, v28.4s, v11.s[2]\n" - "fmla v12.4s, v27.4s, v9.s[0]\n" - "fmla v13.4s, v27.4s, v9.s[1]\n" - "fmin v12.4s, v12.4s, v20.4s\n" - "fmla v14.4s, v27.4s, v9.s[2]\n" - "fmla v15.4s, v27.4s, v9.s[3]\n" - "fmin v13.4s, v13.4s, v20.4s\n" - "fmla v16.4s, v27.4s, v11.s[0]\n" - "fmla v17.4s, v27.4s, v11.s[1]\n" - "fmin v14.4s, v14.4s, v20.4s\n" - "fmla v18.4s, v27.4s, v11.s[2]\n" - "fmla v19.4s, v27.4s, v11.s[3]\n" - "fmin v15.4s, v15.4s, v20.4s\n" - "fmin v16.4s, v16.4s, v20.4s\n" - "fmin v17.4s, v17.4s, v20.4s\n" - "fmin v18.4s, v18.4s, v20.4s\n" - "fmin v19.4s, v19.4s, v20.4s\n" - "fmax v12.4s, v12.4s, v21.4s\n" - "fmax v13.4s, v13.4s, v21.4s\n" - "fmax v14.4s, v14.4s, v21.4s\n" - "fmax v15.4s, v15.4s, v21.4s\n" - "fmax v16.4s, v16.4s, v21.4s\n" - "fmax v17.4s, v17.4s, v21.4s\n" - "fmax v18.4s, v18.4s, v21.4s\n" - "fmax v19.4s, v19.4s, v21.4s\n" + "fmla v12.4s, v24.4s, v8.s[0]\n" + "fmla v13.4s, v24.4s, v8.s[1]\n" + "fmla v14.4s, v24.4s, v8.s[2]\n" + "fmla v15.4s, v24.4s, v8.s[3]\n" + "fmla v16.4s, v24.4s, v10.s[0]\n" + "fmla v17.4s, v24.4s, v10.s[1]\n" + "fmla v18.4s, v24.4s, v10.s[2]\n" + "fmla v19.4s, v24.4s, v10.s[3]\n" + "fmla v12.4s, v23.4s, v8.s[1]\n" + "fmla v13.4s, v23.4s, v8.s[2]\n" + "fmla v14.4s, v23.4s, v8.s[3]\n" + "fmla v15.4s, v23.4s, v9.s[0]\n" + "fmla v16.4s, v23.4s, v10.s[1]\n" + "fmla v17.4s, v23.4s, v10.s[2]\n" + "fmla v18.4s, v23.4s, v10.s[3]\n" + "fmla v19.4s, v23.4s, v11.s[0]\n" + "fmla v12.4s, v22.4s, v8.s[2]\n" + "fmla v13.4s, v22.4s, v8.s[3]\n" + "fmla v14.4s, v22.4s, v9.s[0]\n" + "fmla v15.4s, v22.4s, v9.s[1]\n" + "fmla v16.4s, v22.4s, v10.s[2]\n" + "fmla v17.4s, v22.4s, v10.s[3]\n" + "fmla v18.4s, v22.4s, v11.s[0]\n" + "fmla v19.4s, v22.4s, v11.s[1]\n" + "fmla v12.4s, v21.4s, v8.s[3]\n" + "fmla v13.4s, v21.4s, v9.s[0]\n" + "fmla v14.4s, v21.4s, v9.s[1]\n" + "fmla v15.4s, v21.4s, v9.s[2]\n" + "fmla v16.4s, v21.4s, v10.s[3]\n" + "fmla v17.4s, v21.4s, v11.s[0]\n" + "fmla v18.4s, v21.4s, v11.s[1]\n" + "fmla v19.4s, v21.4s, v11.s[2]\n" + "fmla v12.4s, v20.4s, v9.s[0]\n" + "fmla v13.4s, v20.4s, v9.s[1]\n" + "fmin v12.4s, v12.4s, v25.4s\n" + "fmla v14.4s, v20.4s, v9.s[2]\n" + "fmla v15.4s, v20.4s, v9.s[3]\n" + "fmin v13.4s, v13.4s, v25.4s\n" + "fmla v16.4s, v20.4s, v11.s[0]\n" + "fmla v17.4s, v20.4s, v11.s[1]\n" + "fmin v14.4s, v14.4s, v25.4s\n" + "fmla v18.4s, v20.4s, v11.s[2]\n" + "fmla v19.4s, v20.4s, v11.s[3]\n" + "fmin v15.4s, v15.4s, v25.4s\n" + "fmin v16.4s, v16.4s, v25.4s\n" + "fmin v17.4s, v17.4s, v25.4s\n" + "fmin v18.4s, v18.4s, v25.4s\n" + "fmin v19.4s, v19.4s, v25.4s\n" + "fmax v12.4s, v12.4s, v26.4s\n" + "fmax v13.4s, v13.4s, v26.4s\n" + "fmax v14.4s, v14.4s, v26.4s\n" + "fmax v15.4s, v15.4s, v26.4s\n" + "fmax v16.4s, v16.4s, v26.4s\n" + "fmax v17.4s, v17.4s, v26.4s\n" + "fmax v18.4s, v18.4s, v26.4s\n" + "fmax v19.4s, v19.4s, v26.4s\n" "tbz %x[channel_multiplier], #1, 4f\n" "add x20, x12, x13\n" "add x21, x11, x13\n" @@ -904,15 +904,14 @@ void a64_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst_imp "st1 { v18.s }[0], [x21]\n" "st1 { v19.s }[0], [x20]\n" "5:" // Output channel oddments: Store: Bit 1: End - "6:" // End - : [params] "+&r" (params) : [channel_multiplier] "r" (n_output_channels), [clamps] "r" (minmax_vals), [inptrs] "r" (inptrs), [outptrs] "r" (outptrs) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp index d60e15ec847503609d0934e36a761446e3697f78..3bece73973d3dec15d0b3f81862b60db11c9fed8 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp index c28f29c4f9b5b05e428906bafeaf7c4f64052a0d..cc18dd4bb440529158febd3151970ccd4d2274c9 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp @@ -22,10 +22,11 @@ * SOFTWARE. */ -#if defined(__aarch64__) #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace depthwise { @@ -43,10 +44,10 @@ void a64_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_im const float minmax_vals[2] = { activation_min, activation_max }; __asm__ __volatile__( - "ld1r { v11.4s }, [%x[minmax_vals]]\n" + "ld1r { v12.4s }, [%x[minmax_vals]]\n" "lsr x11, %x[n_output_channels], #0x2\n" "add x20, %x[minmax_vals], #0x4\n" - "ld1r { v10.4s }, [x20]\n" + "ld1r { v11.4s }, [x20]\n" "mov x10, #0x0\n" "cbz x11, 8f\n" "1:" // Output channel loop @@ -55,16 +56,16 @@ void a64_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_im "lsl x20, x10, #0x2\n" "ldr q31, [%x[bias], x20]\n" "2:" // Output channel loop: Load bias: Done - "ldr q9, [%x[weights], #0x0]\n" - "mov x20, %x[inptrs]\n" - "ldp x23, x9, [x20], #0x10\n" - "lsr x21, %x[kernel_points], #0x1\n" - "ldr q8, [x23, #0x0]\n" - "ldr q7, [x23, #0x10]\n" + "ldr q10, [%x[weights], #0x0]\n" + "mov x22, %x[inptrs]\n" + "ldp x21, x20, [x22], #0x10\n" + "lsr x23, %x[kernel_points], #0x1\n" + "ldr q3, [x21, #0x0]\n" + "ldr q2, [x21, #0x10]\n" "mov v16.16b, v31.16b\n" "mov v17.16b, v31.16b\n" - "ldr q6, [x9, #0x0]\n" - "ldr q5, [x9, #0x10]\n" + "ldr q1, [x20, #0x0]\n" + "ldr q0, [x20, #0x10]\n" "mov v18.16b, v31.16b\n" "mov v19.16b, v31.16b\n" "mov v20.16b, v31.16b\n" @@ -80,368 +81,368 @@ void a64_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_im "mov v29.16b, v31.16b\n" "mov v30.16b, v31.16b\n" "mov v31.16b, v31.16b\n" - "cbz x21, 6f\n" - "ldr q4, [%x[weights], #0x0]\n" - "ldp x23, x9, [x20], #0x10\n" - "subs x21, x21, #0x1\n" + "cbz x23, 6f\n" + "ldr q9, [%x[weights], #0x0]\n" + "ldp x21, x20, [x22], #0x10\n" + "subs x23, x23, #0x1\n" "add %x[weights], %x[weights], #0x10\n" - "ldr q3, [x23, #0x0]\n" - "ldr q2, [x23, #0x10]\n" - "ldr q1, [x9, #0x0]\n" - "ldr q0, [x9, #0x10]\n" + "ldr q8, [x21, #0x0]\n" + "ldr q7, [x21, #0x10]\n" + "ldr q6, [x20, #0x0]\n" + "ldr q5, [x20, #0x10]\n" "beq 4f\n" "3:" // Output channel loop: Kernel loop - "ldp x23, x9, [x20], #0x10\n" + "ldp x21, x20, [x22], #0x10\n" + "fmla v16.4s, v10.4s, v3.s[0]\n" + "fmla v17.4s, v10.4s, v3.s[1]\n" + "subs x23, x23, #0x1\n" + "fmla v18.4s, v10.4s, v3.s[2]\n" + "fmla v19.4s, v10.4s, v3.s[3]\n" + "ldr q3, [x21, #0x0]\n" + "fmla v20.4s, v10.4s, v2.s[0]\n" + "fmla v21.4s, v10.4s, v2.s[1]\n" + "fmla v22.4s, v10.4s, v2.s[2]\n" + "fmla v23.4s, v10.4s, v2.s[3]\n" + "ldr q2, [x21, #0x10]\n" + "fmla v24.4s, v10.4s, v1.s[0]\n" + "fmla v25.4s, v10.4s, v1.s[1]\n" + "fmla v26.4s, v10.4s, v1.s[2]\n" + "fmla v27.4s, v10.4s, v1.s[3]\n" + "ldr q1, [x20, #0x0]\n" + "fmla v28.4s, v10.4s, v0.s[0]\n" + "fmla v29.4s, v10.4s, v0.s[1]\n" + "fmla v30.4s, v10.4s, v0.s[2]\n" + "fmla v31.4s, v10.4s, v0.s[3]\n" + "ldr q0, [x20, #0x10]\n" + "ldr q10, [%x[weights], #0x0]\n" + "ldp x21, x20, [x22], #0x10\n" "fmla v16.4s, v9.4s, v8.s[0]\n" "fmla v17.4s, v9.4s, v8.s[1]\n" - "subs x21, x21, #0x1\n" "fmla v18.4s, v9.4s, v8.s[2]\n" "fmla v19.4s, v9.4s, v8.s[3]\n" - "ldr q8, [x23, #0x0]\n" + "ldr q8, [x21, #0x0]\n" "fmla v20.4s, v9.4s, v7.s[0]\n" "fmla v21.4s, v9.4s, v7.s[1]\n" "fmla v22.4s, v9.4s, v7.s[2]\n" "fmla v23.4s, v9.4s, v7.s[3]\n" - "ldr q7, [x23, #0x10]\n" + "ldr q7, [x21, #0x10]\n" "fmla v24.4s, v9.4s, v6.s[0]\n" "fmla v25.4s, v9.4s, v6.s[1]\n" "fmla v26.4s, v9.4s, v6.s[2]\n" "fmla v27.4s, v9.4s, v6.s[3]\n" - "ldr q6, [x9, #0x0]\n" + "ldr q6, [x20, #0x0]\n" "fmla v28.4s, v9.4s, v5.s[0]\n" "fmla v29.4s, v9.4s, v5.s[1]\n" "fmla v30.4s, v9.4s, v5.s[2]\n" "fmla v31.4s, v9.4s, v5.s[3]\n" - "ldr q5, [x9, #0x10]\n" - "ldr q9, [%x[weights], #0x0]\n" - "ldp x23, x9, [x20], #0x10\n" - "fmla v16.4s, v4.4s, v3.s[0]\n" - "fmla v17.4s, v4.4s, v3.s[1]\n" - "fmla v18.4s, v4.4s, v3.s[2]\n" - "fmla v19.4s, v4.4s, v3.s[3]\n" - "ldr q3, [x23, #0x0]\n" - "fmla v20.4s, v4.4s, v2.s[0]\n" - "fmla v21.4s, v4.4s, v2.s[1]\n" - "fmla v22.4s, v4.4s, v2.s[2]\n" - "fmla v23.4s, v4.4s, v2.s[3]\n" - "ldr q2, [x23, #0x10]\n" - "fmla v24.4s, v4.4s, v1.s[0]\n" - "fmla v25.4s, v4.4s, v1.s[1]\n" - "fmla v26.4s, v4.4s, v1.s[2]\n" - "fmla v27.4s, v4.4s, v1.s[3]\n" - "ldr q1, [x9, #0x0]\n" - "fmla v28.4s, v4.4s, v0.s[0]\n" - "fmla v29.4s, v4.4s, v0.s[1]\n" - "fmla v30.4s, v4.4s, v0.s[2]\n" - "fmla v31.4s, v4.4s, v0.s[3]\n" - "ldr q0, [x9, #0x10]\n" - "ldr q4, [%x[weights], #0x10]\n" + "ldr q5, [x20, #0x10]\n" + "ldr q9, [%x[weights], #0x10]\n" "add %x[weights], %x[weights], #0x20\n" "bgt 3b\n" "4:" // Output channel loop: Kernel loop tail "tbnz %x[kernel_points], #0, 5f\n" + "fmla v16.4s, v10.4s, v3.s[0]\n" + "fmla v17.4s, v10.4s, v3.s[1]\n" + "lsl x28, x10, #0x2\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "fmla v18.4s, v10.4s, v3.s[2]\n" + "fmla v19.4s, v10.4s, v3.s[3]\n" + "ldr x26, [%x[outptrs], #0x8]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "fmla v20.4s, v10.4s, v2.s[0]\n" + "fmla v21.4s, v10.4s, v2.s[1]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "fmla v22.4s, v10.4s, v2.s[2]\n" + "fmla v23.4s, v10.4s, v2.s[3]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "fmla v24.4s, v10.4s, v1.s[0]\n" + "fmla v25.4s, v10.4s, v1.s[1]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "fmla v26.4s, v10.4s, v1.s[2]\n" + "fmla v27.4s, v10.4s, v1.s[3]\n" + "fmla v28.4s, v10.4s, v0.s[0]\n" + "fmla v29.4s, v10.4s, v0.s[1]\n" + "fmla v30.4s, v10.4s, v0.s[2]\n" + "fmla v31.4s, v10.4s, v0.s[3]\n" "fmla v16.4s, v9.4s, v8.s[0]\n" "fmla v17.4s, v9.4s, v8.s[1]\n" - "lsl x28, x10, #0x2\n" - "ldr x20, [%x[outptrs], #0x0]\n" + "fmin v16.4s, v16.4s, v11.4s\n" "fmla v18.4s, v9.4s, v8.s[2]\n" "fmla v19.4s, v9.4s, v8.s[3]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "ldr x22, [%x[outptrs], #0x10]\n" + "fmin v17.4s, v17.4s, v11.4s\n" "fmla v20.4s, v9.4s, v7.s[0]\n" "fmla v21.4s, v9.4s, v7.s[1]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "ldr x24, [%x[outptrs], #0x20]\n" + "fmin v18.4s, v18.4s, v11.4s\n" "fmla v22.4s, v9.4s, v7.s[2]\n" "fmla v23.4s, v9.4s, v7.s[3]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "ldr x26, [%x[outptrs], #0x30]\n" + "fmin v19.4s, v19.4s, v11.4s\n" "fmla v24.4s, v9.4s, v6.s[0]\n" "fmla v25.4s, v9.4s, v6.s[1]\n" - "ldr x27, [%x[outptrs], #0x38]\n" + "fmin v20.4s, v20.4s, v11.4s\n" "fmla v26.4s, v9.4s, v6.s[2]\n" "fmla v27.4s, v9.4s, v6.s[3]\n" + "fmin v21.4s, v21.4s, v11.4s\n" "fmla v28.4s, v9.4s, v5.s[0]\n" "fmla v29.4s, v9.4s, v5.s[1]\n" + "fmin v22.4s, v22.4s, v11.4s\n" "fmla v30.4s, v9.4s, v5.s[2]\n" "fmla v31.4s, v9.4s, v5.s[3]\n" - "fmla v16.4s, v4.4s, v3.s[0]\n" - "fmla v17.4s, v4.4s, v3.s[1]\n" - "fmin v16.4s, v16.4s, v10.4s\n" - "fmla v18.4s, v4.4s, v3.s[2]\n" - "fmla v19.4s, v4.4s, v3.s[3]\n" - "fmin v17.4s, v17.4s, v10.4s\n" - "fmla v20.4s, v4.4s, v2.s[0]\n" - "fmla v21.4s, v4.4s, v2.s[1]\n" - "fmin v18.4s, v18.4s, v10.4s\n" - "fmla v22.4s, v4.4s, v2.s[2]\n" - "fmla v23.4s, v4.4s, v2.s[3]\n" - "fmin v19.4s, v19.4s, v10.4s\n" - "fmla v24.4s, v4.4s, v1.s[0]\n" - "fmla v25.4s, v4.4s, v1.s[1]\n" - "fmin v20.4s, v20.4s, v10.4s\n" - "fmla v26.4s, v4.4s, v1.s[2]\n" - "fmla v27.4s, v4.4s, v1.s[3]\n" - "fmin v21.4s, v21.4s, v10.4s\n" - "fmla v28.4s, v4.4s, v0.s[0]\n" - "fmla v29.4s, v4.4s, v0.s[1]\n" - "fmin v22.4s, v22.4s, v10.4s\n" - "fmla v30.4s, v4.4s, v0.s[2]\n" - "fmla v31.4s, v4.4s, v0.s[3]\n" - "fmin v23.4s, v23.4s, v10.4s\n" - "fmax v16.4s, v16.4s, v11.4s\n" - "fmax v17.4s, v17.4s, v11.4s\n" - "str q16, [x20, x28]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "fmax v18.4s, v18.4s, v11.4s\n" - "fmax v19.4s, v19.4s, v11.4s\n" - "str q17, [x21, x28]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "fmax v20.4s, v20.4s, v11.4s\n" - "fmax v21.4s, v21.4s, v11.4s\n" - "str q18, [x22, x28]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "fmax v22.4s, v22.4s, v11.4s\n" - "fmax v23.4s, v23.4s, v11.4s\n" - "str q19, [x23, x28]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "fmin v24.4s, v24.4s, v10.4s\n" - "fmin v25.4s, v25.4s, v10.4s\n" - "str q20, [x24, x28]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "fmin v26.4s, v26.4s, v10.4s\n" - "fmin v27.4s, v27.4s, v10.4s\n" - "str q21, [x25, x28]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "fmin v28.4s, v28.4s, v10.4s\n" - "fmin v29.4s, v29.4s, v10.4s\n" - "str q22, [x26, x28]\n" - "ldr x26, [%x[outptrs], #0x70]\n" - "fmin v30.4s, v30.4s, v10.4s\n" - "fmin v31.4s, v31.4s, v10.4s\n" - "str q23, [x27, x28]\n" - "ldr x27, [%x[outptrs], #0x78]\n" - "fmax v24.4s, v24.4s, v11.4s\n" - "fmax v25.4s, v25.4s, v11.4s\n" - "str q24, [x20, x28]\n" - "fmax v26.4s, v26.4s, v11.4s\n" - "fmax v27.4s, v27.4s, v11.4s\n" - "str q25, [x21, x28]\n" - "fmax v28.4s, v28.4s, v11.4s\n" - "fmax v29.4s, v29.4s, v11.4s\n" - "str q26, [x22, x28]\n" - "fmax v30.4s, v30.4s, v11.4s\n" - "fmax v31.4s, v31.4s, v11.4s\n" - "str q27, [x23, x28]\n" - "str q28, [x24, x28]\n" - "str q29, [x25, x28]\n" - "str q30, [x26, x28]\n" - "str q31, [x27, x28]\n" + "fmin v23.4s, v23.4s, v11.4s\n" + "fmax v16.4s, v16.4s, v12.4s\n" + "fmax v17.4s, v17.4s, v12.4s\n" + "str q16, [x27, x28]\n" + "ldr x27, [%x[outptrs], #0x40]\n" + "fmax v18.4s, v18.4s, v12.4s\n" + "fmax v19.4s, v19.4s, v12.4s\n" + "str q17, [x26, x28]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "fmax v20.4s, v20.4s, v12.4s\n" + "fmax v21.4s, v21.4s, v12.4s\n" + "str q18, [x25, x28]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "fmax v22.4s, v22.4s, v12.4s\n" + "fmax v23.4s, v23.4s, v12.4s\n" + "str q19, [x24, x28]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "fmin v24.4s, v24.4s, v11.4s\n" + "fmin v25.4s, v25.4s, v11.4s\n" + "str q20, [x23, x28]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "fmin v26.4s, v26.4s, v11.4s\n" + "fmin v27.4s, v27.4s, v11.4s\n" + "str q21, [x22, x28]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "fmin v28.4s, v28.4s, v11.4s\n" + "fmin v29.4s, v29.4s, v11.4s\n" + "str q22, [x21, x28]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "fmin v30.4s, v30.4s, v11.4s\n" + "fmin v31.4s, v31.4s, v11.4s\n" + "str q23, [x20, x28]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "fmax v24.4s, v24.4s, v12.4s\n" + "fmax v25.4s, v25.4s, v12.4s\n" + "str q24, [x27, x28]\n" + "fmax v26.4s, v26.4s, v12.4s\n" + "fmax v27.4s, v27.4s, v12.4s\n" + "str q25, [x26, x28]\n" + "fmax v28.4s, v28.4s, v12.4s\n" + "fmax v29.4s, v29.4s, v12.4s\n" + "str q26, [x25, x28]\n" + "fmax v30.4s, v30.4s, v12.4s\n" + "fmax v31.4s, v31.4s, v12.4s\n" + "str q27, [x24, x28]\n" + "str q28, [x23, x28]\n" + "str q29, [x22, x28]\n" + "str q30, [x21, x28]\n" + "str q31, [x20, x28]\n" "b 7f\n" "5:" // Output channel loop: Odd tail - "fmla v16.4s, v9.4s, v8.s[0]\n" - "fmla v17.4s, v9.4s, v8.s[1]\n" - "ldp x23, x9, [x20], #0x10\n" + "fmla v16.4s, v10.4s, v3.s[0]\n" + "fmla v17.4s, v10.4s, v3.s[1]\n" + "ldp x20, x9, [x22], #0x10\n" "lsl x28, x10, #0x2\n" - "fmla v18.4s, v9.4s, v8.s[2]\n" - "fmla v19.4s, v9.4s, v8.s[3]\n" - "ldr q8, [x23, #0x0]\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "fmla v20.4s, v9.4s, v7.s[0]\n" - "fmla v21.4s, v9.4s, v7.s[1]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "fmla v22.4s, v9.4s, v7.s[2]\n" - "fmla v23.4s, v9.4s, v7.s[3]\n" - "ldr q7, [x23, #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "fmla v24.4s, v9.4s, v6.s[0]\n" - "fmla v25.4s, v9.4s, v6.s[1]\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "fmla v26.4s, v9.4s, v6.s[2]\n" - "fmla v27.4s, v9.4s, v6.s[3]\n" - "ldr q6, [x9, #0x0]\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "fmla v28.4s, v9.4s, v5.s[0]\n" - "fmla v29.4s, v9.4s, v5.s[1]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "fmla v30.4s, v9.4s, v5.s[2]\n" - "fmla v31.4s, v9.4s, v5.s[3]\n" - "ldr q9, [%x[weights], #0x0]\n" - "ldr q5, [x9, #0x10]\n" - "fmla v16.4s, v4.4s, v3.s[0]\n" - "fmla v17.4s, v4.4s, v3.s[1]\n" - "add %x[weights], %x[weights], #0x10\n" - "fmla v18.4s, v4.4s, v3.s[2]\n" - "fmla v19.4s, v4.4s, v3.s[3]\n" - "fmla v20.4s, v4.4s, v2.s[0]\n" - "fmla v21.4s, v4.4s, v2.s[1]\n" - "fmla v22.4s, v4.4s, v2.s[2]\n" - "fmla v23.4s, v4.4s, v2.s[3]\n" - "fmla v24.4s, v4.4s, v1.s[0]\n" - "fmla v25.4s, v4.4s, v1.s[1]\n" - "fmla v26.4s, v4.4s, v1.s[2]\n" - "fmla v27.4s, v4.4s, v1.s[3]\n" - "fmla v28.4s, v4.4s, v0.s[0]\n" - "fmla v29.4s, v4.4s, v0.s[1]\n" - "fmla v30.4s, v4.4s, v0.s[2]\n" - "fmla v31.4s, v4.4s, v0.s[3]\n" + "fmla v18.4s, v10.4s, v3.s[2]\n" + "fmla v19.4s, v10.4s, v3.s[3]\n" + "ldr q4, [x20, #0x0]\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "fmla v20.4s, v10.4s, v2.s[0]\n" + "fmla v21.4s, v10.4s, v2.s[1]\n" + "ldr x26, [%x[outptrs], #0x8]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "fmla v22.4s, v10.4s, v2.s[2]\n" + "fmla v23.4s, v10.4s, v2.s[3]\n" + "ldr q3, [x20, #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "fmla v24.4s, v10.4s, v1.s[0]\n" + "fmla v25.4s, v10.4s, v1.s[1]\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "fmla v26.4s, v10.4s, v1.s[2]\n" + "fmla v27.4s, v10.4s, v1.s[3]\n" + "ldr q2, [x9, #0x0]\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "fmla v28.4s, v10.4s, v0.s[0]\n" + "fmla v29.4s, v10.4s, v0.s[1]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "fmla v30.4s, v10.4s, v0.s[2]\n" + "fmla v31.4s, v10.4s, v0.s[3]\n" + "ldr q1, [%x[weights], #0x0]\n" + "ldr q0, [x9, #0x10]\n" "fmla v16.4s, v9.4s, v8.s[0]\n" "fmla v17.4s, v9.4s, v8.s[1]\n" - "fmin v16.4s, v16.4s, v10.4s\n" + "add %x[weights], %x[weights], #0x10\n" "fmla v18.4s, v9.4s, v8.s[2]\n" "fmla v19.4s, v9.4s, v8.s[3]\n" - "fmin v17.4s, v17.4s, v10.4s\n" "fmla v20.4s, v9.4s, v7.s[0]\n" "fmla v21.4s, v9.4s, v7.s[1]\n" - "fmin v18.4s, v18.4s, v10.4s\n" "fmla v22.4s, v9.4s, v7.s[2]\n" "fmla v23.4s, v9.4s, v7.s[3]\n" - "fmin v19.4s, v19.4s, v10.4s\n" "fmla v24.4s, v9.4s, v6.s[0]\n" "fmla v25.4s, v9.4s, v6.s[1]\n" - "fmin v20.4s, v20.4s, v10.4s\n" "fmla v26.4s, v9.4s, v6.s[2]\n" "fmla v27.4s, v9.4s, v6.s[3]\n" - "fmin v21.4s, v21.4s, v10.4s\n" "fmla v28.4s, v9.4s, v5.s[0]\n" "fmla v29.4s, v9.4s, v5.s[1]\n" - "fmin v22.4s, v22.4s, v10.4s\n" "fmla v30.4s, v9.4s, v5.s[2]\n" "fmla v31.4s, v9.4s, v5.s[3]\n" - "fmin v23.4s, v23.4s, v10.4s\n" - "fmax v16.4s, v16.4s, v11.4s\n" - "fmax v17.4s, v17.4s, v11.4s\n" - "str q16, [x20, x28]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "fmax v18.4s, v18.4s, v11.4s\n" - "fmax v19.4s, v19.4s, v11.4s\n" - "str q17, [x21, x28]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "fmax v20.4s, v20.4s, v11.4s\n" - "fmax v21.4s, v21.4s, v11.4s\n" - "str q18, [x22, x28]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "fmax v22.4s, v22.4s, v11.4s\n" - "fmax v23.4s, v23.4s, v11.4s\n" - "str q19, [x23, x28]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "fmin v24.4s, v24.4s, v10.4s\n" - "fmin v25.4s, v25.4s, v10.4s\n" - "str q20, [x24, x28]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "fmin v26.4s, v26.4s, v10.4s\n" - "fmin v27.4s, v27.4s, v10.4s\n" - "str q21, [x25, x28]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "fmin v28.4s, v28.4s, v10.4s\n" - "fmin v29.4s, v29.4s, v10.4s\n" - "str q22, [x26, x28]\n" - "ldr x26, [%x[outptrs], #0x70]\n" - "fmin v30.4s, v30.4s, v10.4s\n" - "fmin v31.4s, v31.4s, v10.4s\n" - "str q23, [x27, x28]\n" - "ldr x27, [%x[outptrs], #0x78]\n" - "fmax v24.4s, v24.4s, v11.4s\n" - "fmax v25.4s, v25.4s, v11.4s\n" - "str q24, [x20, x28]\n" - "fmax v26.4s, v26.4s, v11.4s\n" - "fmax v27.4s, v27.4s, v11.4s\n" - "str q25, [x21, x28]\n" - "fmax v28.4s, v28.4s, v11.4s\n" - "fmax v29.4s, v29.4s, v11.4s\n" - "str q26, [x22, x28]\n" - "fmax v30.4s, v30.4s, v11.4s\n" - "fmax v31.4s, v31.4s, v11.4s\n" - "str q27, [x23, x28]\n" - "str q28, [x24, x28]\n" - "str q29, [x25, x28]\n" - "str q30, [x26, x28]\n" - "str q31, [x27, x28]\n" + "fmla v16.4s, v1.4s, v4.s[0]\n" + "fmla v17.4s, v1.4s, v4.s[1]\n" + "fmin v16.4s, v16.4s, v11.4s\n" + "fmla v18.4s, v1.4s, v4.s[2]\n" + "fmla v19.4s, v1.4s, v4.s[3]\n" + "fmin v17.4s, v17.4s, v11.4s\n" + "fmla v20.4s, v1.4s, v3.s[0]\n" + "fmla v21.4s, v1.4s, v3.s[1]\n" + "fmin v18.4s, v18.4s, v11.4s\n" + "fmla v22.4s, v1.4s, v3.s[2]\n" + "fmla v23.4s, v1.4s, v3.s[3]\n" + "fmin v19.4s, v19.4s, v11.4s\n" + "fmla v24.4s, v1.4s, v2.s[0]\n" + "fmla v25.4s, v1.4s, v2.s[1]\n" + "fmin v20.4s, v20.4s, v11.4s\n" + "fmla v26.4s, v1.4s, v2.s[2]\n" + "fmla v27.4s, v1.4s, v2.s[3]\n" + "fmin v21.4s, v21.4s, v11.4s\n" + "fmla v28.4s, v1.4s, v0.s[0]\n" + "fmla v29.4s, v1.4s, v0.s[1]\n" + "fmin v22.4s, v22.4s, v11.4s\n" + "fmla v30.4s, v1.4s, v0.s[2]\n" + "fmla v31.4s, v1.4s, v0.s[3]\n" + "fmin v23.4s, v23.4s, v11.4s\n" + "fmax v16.4s, v16.4s, v12.4s\n" + "fmax v17.4s, v17.4s, v12.4s\n" + "str q16, [x27, x28]\n" + "ldr x27, [%x[outptrs], #0x40]\n" + "fmax v18.4s, v18.4s, v12.4s\n" + "fmax v19.4s, v19.4s, v12.4s\n" + "str q17, [x26, x28]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "fmax v20.4s, v20.4s, v12.4s\n" + "fmax v21.4s, v21.4s, v12.4s\n" + "str q18, [x25, x28]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "fmax v22.4s, v22.4s, v12.4s\n" + "fmax v23.4s, v23.4s, v12.4s\n" + "str q19, [x24, x28]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "fmin v24.4s, v24.4s, v11.4s\n" + "fmin v25.4s, v25.4s, v11.4s\n" + "str q20, [x23, x28]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "fmin v26.4s, v26.4s, v11.4s\n" + "fmin v27.4s, v27.4s, v11.4s\n" + "str q21, [x22, x28]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "fmin v28.4s, v28.4s, v11.4s\n" + "fmin v29.4s, v29.4s, v11.4s\n" + "str q22, [x21, x28]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "fmin v30.4s, v30.4s, v11.4s\n" + "fmin v31.4s, v31.4s, v11.4s\n" + "str q23, [x20, x28]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "fmax v24.4s, v24.4s, v12.4s\n" + "fmax v25.4s, v25.4s, v12.4s\n" + "str q24, [x27, x28]\n" + "fmax v26.4s, v26.4s, v12.4s\n" + "fmax v27.4s, v27.4s, v12.4s\n" + "str q25, [x26, x28]\n" + "fmax v28.4s, v28.4s, v12.4s\n" + "fmax v29.4s, v29.4s, v12.4s\n" + "str q26, [x25, x28]\n" + "fmax v30.4s, v30.4s, v12.4s\n" + "fmax v31.4s, v31.4s, v12.4s\n" + "str q27, [x24, x28]\n" + "str q28, [x23, x28]\n" + "str q29, [x22, x28]\n" + "str q30, [x21, x28]\n" + "str q31, [x20, x28]\n" "b 7f\n" "6:" // Output channel loop: Single kernel point - "fmla v16.4s, v9.4s, v8.s[0]\n" - "fmla v17.4s, v9.4s, v8.s[1]\n" - "fmin v16.4s, v16.4s, v10.4s\n" + "fmla v16.4s, v10.4s, v3.s[0]\n" + "fmla v17.4s, v10.4s, v3.s[1]\n" + "fmin v16.4s, v16.4s, v11.4s\n" "lsl x28, x10, #0x2\n" - "fmla v18.4s, v9.4s, v8.s[2]\n" - "fmla v19.4s, v9.4s, v8.s[3]\n" - "fmin v17.4s, v17.4s, v10.4s\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "fmla v20.4s, v9.4s, v7.s[0]\n" - "fmla v21.4s, v9.4s, v7.s[1]\n" - "fmin v18.4s, v18.4s, v10.4s\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "fmla v22.4s, v9.4s, v7.s[2]\n" - "fmla v23.4s, v9.4s, v7.s[3]\n" - "fmin v19.4s, v19.4s, v10.4s\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "fmla v24.4s, v9.4s, v6.s[0]\n" - "fmla v25.4s, v9.4s, v6.s[1]\n" - "fmin v20.4s, v20.4s, v10.4s\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "fmla v26.4s, v9.4s, v6.s[2]\n" - "fmla v27.4s, v9.4s, v6.s[3]\n" - "fmin v21.4s, v21.4s, v10.4s\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "fmla v28.4s, v9.4s, v5.s[0]\n" - "fmla v29.4s, v9.4s, v5.s[1]\n" - "fmin v22.4s, v22.4s, v10.4s\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "fmla v30.4s, v9.4s, v5.s[2]\n" - "fmla v31.4s, v9.4s, v5.s[3]\n" - "fmin v23.4s, v23.4s, v10.4s\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "fmax v16.4s, v16.4s, v11.4s\n" - "fmax v17.4s, v17.4s, v11.4s\n" - "str q16, [x20, x28]\n" - "fmax v18.4s, v18.4s, v11.4s\n" - "fmax v19.4s, v19.4s, v11.4s\n" - "str q17, [x21, x28]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "fmax v20.4s, v20.4s, v11.4s\n" - "fmax v21.4s, v21.4s, v11.4s\n" - "str q18, [x22, x28]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "fmax v22.4s, v22.4s, v11.4s\n" - "fmax v23.4s, v23.4s, v11.4s\n" - "str q19, [x23, x28]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "fmin v24.4s, v24.4s, v10.4s\n" - "fmin v25.4s, v25.4s, v10.4s\n" - "str q20, [x24, x28]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "fmin v26.4s, v26.4s, v10.4s\n" - "fmin v27.4s, v27.4s, v10.4s\n" - "str q21, [x25, x28]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "fmin v28.4s, v28.4s, v10.4s\n" - "fmin v29.4s, v29.4s, v10.4s\n" - "str q22, [x26, x28]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "fmin v30.4s, v30.4s, v10.4s\n" - "fmin v31.4s, v31.4s, v10.4s\n" - "str q23, [x27, x28]\n" - "ldr x26, [%x[outptrs], #0x70]\n" - "ldr x27, [%x[outptrs], #0x78]\n" - "fmax v24.4s, v24.4s, v11.4s\n" - "fmax v25.4s, v25.4s, v11.4s\n" - "str q24, [x20, x28]\n" - "fmax v26.4s, v26.4s, v11.4s\n" - "fmax v27.4s, v27.4s, v11.4s\n" - "str q25, [x21, x28]\n" - "fmax v28.4s, v28.4s, v11.4s\n" - "fmax v29.4s, v29.4s, v11.4s\n" - "str q26, [x22, x28]\n" - "fmax v30.4s, v30.4s, v11.4s\n" - "fmax v31.4s, v31.4s, v11.4s\n" - "str q27, [x23, x28]\n" - "str q28, [x24, x28]\n" - "str q29, [x25, x28]\n" - "str q30, [x26, x28]\n" - "str q31, [x27, x28]\n" + "fmla v18.4s, v10.4s, v3.s[2]\n" + "fmla v19.4s, v10.4s, v3.s[3]\n" + "fmin v17.4s, v17.4s, v11.4s\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "fmla v20.4s, v10.4s, v2.s[0]\n" + "fmla v21.4s, v10.4s, v2.s[1]\n" + "fmin v18.4s, v18.4s, v11.4s\n" + "ldr x26, [%x[outptrs], #0x8]\n" + "fmla v22.4s, v10.4s, v2.s[2]\n" + "fmla v23.4s, v10.4s, v2.s[3]\n" + "fmin v19.4s, v19.4s, v11.4s\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "fmla v24.4s, v10.4s, v1.s[0]\n" + "fmla v25.4s, v10.4s, v1.s[1]\n" + "fmin v20.4s, v20.4s, v11.4s\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "fmla v26.4s, v10.4s, v1.s[2]\n" + "fmla v27.4s, v10.4s, v1.s[3]\n" + "fmin v21.4s, v21.4s, v11.4s\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "fmla v28.4s, v10.4s, v0.s[0]\n" + "fmla v29.4s, v10.4s, v0.s[1]\n" + "fmin v22.4s, v22.4s, v11.4s\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "fmla v30.4s, v10.4s, v0.s[2]\n" + "fmla v31.4s, v10.4s, v0.s[3]\n" + "fmin v23.4s, v23.4s, v11.4s\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "fmax v16.4s, v16.4s, v12.4s\n" + "fmax v17.4s, v17.4s, v12.4s\n" + "str q16, [x27, x28]\n" + "fmax v18.4s, v18.4s, v12.4s\n" + "fmax v19.4s, v19.4s, v12.4s\n" + "str q17, [x26, x28]\n" + "ldr x27, [%x[outptrs], #0x40]\n" + "fmax v20.4s, v20.4s, v12.4s\n" + "fmax v21.4s, v21.4s, v12.4s\n" + "str q18, [x25, x28]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "fmax v22.4s, v22.4s, v12.4s\n" + "fmax v23.4s, v23.4s, v12.4s\n" + "str q19, [x24, x28]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "fmin v24.4s, v24.4s, v11.4s\n" + "fmin v25.4s, v25.4s, v11.4s\n" + "str q20, [x23, x28]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "fmin v26.4s, v26.4s, v11.4s\n" + "fmin v27.4s, v27.4s, v11.4s\n" + "str q21, [x22, x28]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "fmin v28.4s, v28.4s, v11.4s\n" + "fmin v29.4s, v29.4s, v11.4s\n" + "str q22, [x21, x28]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "fmin v30.4s, v30.4s, v11.4s\n" + "fmin v31.4s, v31.4s, v11.4s\n" + "str q23, [x20, x28]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "fmax v24.4s, v24.4s, v12.4s\n" + "fmax v25.4s, v25.4s, v12.4s\n" + "str q24, [x27, x28]\n" + "fmax v26.4s, v26.4s, v12.4s\n" + "fmax v27.4s, v27.4s, v12.4s\n" + "str q25, [x26, x28]\n" + "fmax v28.4s, v28.4s, v12.4s\n" + "fmax v29.4s, v29.4s, v12.4s\n" + "str q26, [x25, x28]\n" + "fmax v30.4s, v30.4s, v12.4s\n" + "fmax v31.4s, v31.4s, v12.4s\n" + "str q27, [x24, x28]\n" + "str q28, [x23, x28]\n" + "str q29, [x22, x28]\n" + "str q30, [x21, x28]\n" + "str q31, [x20, x28]\n" "7:" // Output channel loop: Done "add x10, x10, #0x4\n" "cmp x10, x11, LSL #2\n" @@ -461,16 +462,16 @@ void a64_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_im "ld1 { v31.s }[0], [x20]\n" "10:" // Output channel oddments: Load bias: Bit 1: End "11:" // Output channel oddments: Load bias: Done - "ldr q9, [%x[weights], #0x0]\n" - "mov x20, %x[inptrs]\n" - "ldp x23, x9, [x20], #0x10\n" - "lsr x21, %x[kernel_points], #0x1\n" - "ldr q8, [x23, #0x0]\n" - "ldr q7, [x23, #0x10]\n" + "ldr q10, [%x[weights], #0x0]\n" + "mov x22, %x[inptrs]\n" + "ldp x21, x20, [x22], #0x10\n" + "lsr x23, %x[kernel_points], #0x1\n" + "ldr q3, [x21, #0x0]\n" + "ldr q2, [x21, #0x10]\n" "mov v16.16b, v31.16b\n" "mov v17.16b, v31.16b\n" - "ldr q6, [x9, #0x0]\n" - "ldr q5, [x9, #0x10]\n" + "ldr q1, [x20, #0x0]\n" + "ldr q0, [x20, #0x10]\n" "mov v18.16b, v31.16b\n" "mov v19.16b, v31.16b\n" "mov v20.16b, v31.16b\n" @@ -486,66 +487,82 @@ void a64_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_im "mov v29.16b, v31.16b\n" "mov v30.16b, v31.16b\n" "mov v31.16b, v31.16b\n" - "cbz x21, 15f\n" - "ldr q4, [%x[weights], #0x0]\n" - "ldp x23, x9, [x20], #0x10\n" - "subs x21, x21, #0x1\n" + "cbz x23, 15f\n" + "ldr q9, [%x[weights], #0x0]\n" + "ldp x21, x20, [x22], #0x10\n" + "subs x23, x23, #0x1\n" "add %x[weights], %x[weights], #0x10\n" - "ldr q3, [x23, #0x0]\n" - "ldr q2, [x23, #0x10]\n" - "ldr q1, [x9, #0x0]\n" - "ldr q0, [x9, #0x10]\n" + "ldr q8, [x21, #0x0]\n" + "ldr q7, [x21, #0x10]\n" + "ldr q6, [x20, #0x0]\n" + "ldr q5, [x20, #0x10]\n" "beq 13f\n" "12:" // Output channel oddments: Kernel loop - "ldp x23, x9, [x20], #0x10\n" + "ldp x21, x20, [x22], #0x10\n" + "fmla v16.4s, v10.4s, v3.s[0]\n" + "fmla v17.4s, v10.4s, v3.s[1]\n" + "subs x23, x23, #0x1\n" + "fmla v18.4s, v10.4s, v3.s[2]\n" + "fmla v19.4s, v10.4s, v3.s[3]\n" + "ldr q3, [x21, #0x0]\n" + "fmla v20.4s, v10.4s, v2.s[0]\n" + "fmla v21.4s, v10.4s, v2.s[1]\n" + "fmla v22.4s, v10.4s, v2.s[2]\n" + "fmla v23.4s, v10.4s, v2.s[3]\n" + "ldr q2, [x21, #0x10]\n" + "fmla v24.4s, v10.4s, v1.s[0]\n" + "fmla v25.4s, v10.4s, v1.s[1]\n" + "fmla v26.4s, v10.4s, v1.s[2]\n" + "fmla v27.4s, v10.4s, v1.s[3]\n" + "ldr q1, [x20, #0x0]\n" + "fmla v28.4s, v10.4s, v0.s[0]\n" + "fmla v29.4s, v10.4s, v0.s[1]\n" + "fmla v30.4s, v10.4s, v0.s[2]\n" + "fmla v31.4s, v10.4s, v0.s[3]\n" + "ldr q0, [x20, #0x10]\n" + "ldr q10, [%x[weights], #0x0]\n" + "ldp x21, x20, [x22], #0x10\n" "fmla v16.4s, v9.4s, v8.s[0]\n" "fmla v17.4s, v9.4s, v8.s[1]\n" - "subs x21, x21, #0x1\n" "fmla v18.4s, v9.4s, v8.s[2]\n" "fmla v19.4s, v9.4s, v8.s[3]\n" - "ldr q8, [x23, #0x0]\n" + "ldr q8, [x21, #0x0]\n" "fmla v20.4s, v9.4s, v7.s[0]\n" "fmla v21.4s, v9.4s, v7.s[1]\n" "fmla v22.4s, v9.4s, v7.s[2]\n" "fmla v23.4s, v9.4s, v7.s[3]\n" - "ldr q7, [x23, #0x10]\n" + "ldr q7, [x21, #0x10]\n" "fmla v24.4s, v9.4s, v6.s[0]\n" "fmla v25.4s, v9.4s, v6.s[1]\n" "fmla v26.4s, v9.4s, v6.s[2]\n" "fmla v27.4s, v9.4s, v6.s[3]\n" - "ldr q6, [x9, #0x0]\n" + "ldr q6, [x20, #0x0]\n" "fmla v28.4s, v9.4s, v5.s[0]\n" "fmla v29.4s, v9.4s, v5.s[1]\n" "fmla v30.4s, v9.4s, v5.s[2]\n" "fmla v31.4s, v9.4s, v5.s[3]\n" - "ldr q5, [x9, #0x10]\n" - "ldr q9, [%x[weights], #0x0]\n" - "ldp x23, x9, [x20], #0x10\n" - "fmla v16.4s, v4.4s, v3.s[0]\n" - "fmla v17.4s, v4.4s, v3.s[1]\n" - "fmla v18.4s, v4.4s, v3.s[2]\n" - "fmla v19.4s, v4.4s, v3.s[3]\n" - "ldr q3, [x23, #0x0]\n" - "fmla v20.4s, v4.4s, v2.s[0]\n" - "fmla v21.4s, v4.4s, v2.s[1]\n" - "fmla v22.4s, v4.4s, v2.s[2]\n" - "fmla v23.4s, v4.4s, v2.s[3]\n" - "ldr q2, [x23, #0x10]\n" - "fmla v24.4s, v4.4s, v1.s[0]\n" - "fmla v25.4s, v4.4s, v1.s[1]\n" - "fmla v26.4s, v4.4s, v1.s[2]\n" - "fmla v27.4s, v4.4s, v1.s[3]\n" - "ldr q1, [x9, #0x0]\n" - "fmla v28.4s, v4.4s, v0.s[0]\n" - "fmla v29.4s, v4.4s, v0.s[1]\n" - "fmla v30.4s, v4.4s, v0.s[2]\n" - "fmla v31.4s, v4.4s, v0.s[3]\n" - "ldr q0, [x9, #0x10]\n" - "ldr q4, [%x[weights], #0x10]\n" + "ldr q5, [x20, #0x10]\n" + "ldr q9, [%x[weights], #0x10]\n" "add %x[weights], %x[weights], #0x20\n" "bgt 12b\n" "13:" // Output channel oddments: Kernel loop tail "tbnz %x[kernel_points], #0, 14f\n" + "fmla v16.4s, v10.4s, v3.s[0]\n" + "fmla v17.4s, v10.4s, v3.s[1]\n" + "fmla v18.4s, v10.4s, v3.s[2]\n" + "fmla v19.4s, v10.4s, v3.s[3]\n" + "fmla v20.4s, v10.4s, v2.s[0]\n" + "fmla v21.4s, v10.4s, v2.s[1]\n" + "fmla v22.4s, v10.4s, v2.s[2]\n" + "fmla v23.4s, v10.4s, v2.s[3]\n" + "fmla v24.4s, v10.4s, v1.s[0]\n" + "fmla v25.4s, v10.4s, v1.s[1]\n" + "fmla v26.4s, v10.4s, v1.s[2]\n" + "fmla v27.4s, v10.4s, v1.s[3]\n" + "fmla v28.4s, v10.4s, v0.s[0]\n" + "fmla v29.4s, v10.4s, v0.s[1]\n" + "fmla v30.4s, v10.4s, v0.s[2]\n" + "fmla v31.4s, v10.4s, v0.s[3]\n" "fmla v16.4s, v9.4s, v8.s[0]\n" "fmla v17.4s, v9.4s, v8.s[1]\n" "fmla v18.4s, v9.4s, v8.s[2]\n" @@ -562,65 +579,33 @@ void a64_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_im "fmla v29.4s, v9.4s, v5.s[1]\n" "fmla v30.4s, v9.4s, v5.s[2]\n" "fmla v31.4s, v9.4s, v5.s[3]\n" - "fmla v16.4s, v4.4s, v3.s[0]\n" - "fmla v17.4s, v4.4s, v3.s[1]\n" - "fmla v18.4s, v4.4s, v3.s[2]\n" - "fmla v19.4s, v4.4s, v3.s[3]\n" - "fmla v20.4s, v4.4s, v2.s[0]\n" - "fmla v21.4s, v4.4s, v2.s[1]\n" - "fmla v22.4s, v4.4s, v2.s[2]\n" - "fmla v23.4s, v4.4s, v2.s[3]\n" - "fmla v24.4s, v4.4s, v1.s[0]\n" - "fmla v25.4s, v4.4s, v1.s[1]\n" - "fmla v26.4s, v4.4s, v1.s[2]\n" - "fmla v27.4s, v4.4s, v1.s[3]\n" - "fmla v28.4s, v4.4s, v0.s[0]\n" - "fmla v29.4s, v4.4s, v0.s[1]\n" - "fmla v30.4s, v4.4s, v0.s[2]\n" - "fmla v31.4s, v4.4s, v0.s[3]\n" "b 16f\n" "14:" // Output channel oddments: Odd tail + "fmla v16.4s, v10.4s, v3.s[0]\n" + "fmla v17.4s, v10.4s, v3.s[1]\n" + "ldp x21, x20, [x22], #0x10\n" + "fmla v18.4s, v10.4s, v3.s[2]\n" + "fmla v19.4s, v10.4s, v3.s[3]\n" + "ldr q4, [x21, #0x0]\n" + "fmla v20.4s, v10.4s, v2.s[0]\n" + "fmla v21.4s, v10.4s, v2.s[1]\n" + "fmla v22.4s, v10.4s, v2.s[2]\n" + "fmla v23.4s, v10.4s, v2.s[3]\n" + "ldr q3, [x21, #0x10]\n" + "fmla v24.4s, v10.4s, v1.s[0]\n" + "fmla v25.4s, v10.4s, v1.s[1]\n" + "fmla v26.4s, v10.4s, v1.s[2]\n" + "fmla v27.4s, v10.4s, v1.s[3]\n" + "ldr q2, [x20, #0x0]\n" + "fmla v28.4s, v10.4s, v0.s[0]\n" + "fmla v29.4s, v10.4s, v0.s[1]\n" + "fmla v30.4s, v10.4s, v0.s[2]\n" + "fmla v31.4s, v10.4s, v0.s[3]\n" + "ldr q1, [x20, #0x10]\n" + "ldr q0, [%x[weights], #0x0]\n" "fmla v16.4s, v9.4s, v8.s[0]\n" "fmla v17.4s, v9.4s, v8.s[1]\n" - "ldp x23, x9, [x20], #0x10\n" - "fmla v18.4s, v9.4s, v8.s[2]\n" - "fmla v19.4s, v9.4s, v8.s[3]\n" - "ldr q8, [x23, #0x0]\n" - "fmla v20.4s, v9.4s, v7.s[0]\n" - "fmla v21.4s, v9.4s, v7.s[1]\n" - "fmla v22.4s, v9.4s, v7.s[2]\n" - "fmla v23.4s, v9.4s, v7.s[3]\n" - "ldr q7, [x23, #0x10]\n" - "fmla v24.4s, v9.4s, v6.s[0]\n" - "fmla v25.4s, v9.4s, v6.s[1]\n" - "fmla v26.4s, v9.4s, v6.s[2]\n" - "fmla v27.4s, v9.4s, v6.s[3]\n" - "ldr q6, [x9, #0x0]\n" - "fmla v28.4s, v9.4s, v5.s[0]\n" - "fmla v29.4s, v9.4s, v5.s[1]\n" - "fmla v30.4s, v9.4s, v5.s[2]\n" - "fmla v31.4s, v9.4s, v5.s[3]\n" - "ldr q5, [x9, #0x10]\n" - "ldr q9, [%x[weights], #0x0]\n" - "fmla v16.4s, v4.4s, v3.s[0]\n" - "fmla v17.4s, v4.4s, v3.s[1]\n" "add %x[weights], %x[weights], #0x10\n" - "fmla v18.4s, v4.4s, v3.s[2]\n" - "fmla v19.4s, v4.4s, v3.s[3]\n" - "fmla v20.4s, v4.4s, v2.s[0]\n" - "fmla v21.4s, v4.4s, v2.s[1]\n" - "fmla v22.4s, v4.4s, v2.s[2]\n" - "fmla v23.4s, v4.4s, v2.s[3]\n" - "fmla v24.4s, v4.4s, v1.s[0]\n" - "fmla v25.4s, v4.4s, v1.s[1]\n" - "fmla v26.4s, v4.4s, v1.s[2]\n" - "fmla v27.4s, v4.4s, v1.s[3]\n" - "fmla v28.4s, v4.4s, v0.s[0]\n" - "fmla v29.4s, v4.4s, v0.s[1]\n" - "fmla v30.4s, v4.4s, v0.s[2]\n" - "fmla v31.4s, v4.4s, v0.s[3]\n" - "fmla v16.4s, v9.4s, v8.s[0]\n" - "fmla v17.4s, v9.4s, v8.s[1]\n" "fmla v18.4s, v9.4s, v8.s[2]\n" "fmla v19.4s, v9.4s, v8.s[3]\n" "fmla v20.4s, v9.4s, v7.s[0]\n" @@ -635,216 +620,231 @@ void a64_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_im "fmla v29.4s, v9.4s, v5.s[1]\n" "fmla v30.4s, v9.4s, v5.s[2]\n" "fmla v31.4s, v9.4s, v5.s[3]\n" + "fmla v16.4s, v0.4s, v4.s[0]\n" + "fmla v17.4s, v0.4s, v4.s[1]\n" + "fmla v18.4s, v0.4s, v4.s[2]\n" + "fmla v19.4s, v0.4s, v4.s[3]\n" + "fmla v20.4s, v0.4s, v3.s[0]\n" + "fmla v21.4s, v0.4s, v3.s[1]\n" + "fmla v22.4s, v0.4s, v3.s[2]\n" + "fmla v23.4s, v0.4s, v3.s[3]\n" + "fmla v24.4s, v0.4s, v2.s[0]\n" + "fmla v25.4s, v0.4s, v2.s[1]\n" + "fmla v26.4s, v0.4s, v2.s[2]\n" + "fmla v27.4s, v0.4s, v2.s[3]\n" + "fmla v28.4s, v0.4s, v1.s[0]\n" + "fmla v29.4s, v0.4s, v1.s[1]\n" + "fmla v30.4s, v0.4s, v1.s[2]\n" + "fmla v31.4s, v0.4s, v1.s[3]\n" "b 16f\n" "15:" // Output channel oddments: Single kernel point - "fmla v16.4s, v9.4s, v8.s[0]\n" - "fmla v17.4s, v9.4s, v8.s[1]\n" - "fmla v18.4s, v9.4s, v8.s[2]\n" - "fmla v19.4s, v9.4s, v8.s[3]\n" - "fmla v20.4s, v9.4s, v7.s[0]\n" - "fmla v21.4s, v9.4s, v7.s[1]\n" - "fmla v22.4s, v9.4s, v7.s[2]\n" - "fmla v23.4s, v9.4s, v7.s[3]\n" - "fmla v24.4s, v9.4s, v6.s[0]\n" - "fmla v25.4s, v9.4s, v6.s[1]\n" - "fmla v26.4s, v9.4s, v6.s[2]\n" - "fmla v27.4s, v9.4s, v6.s[3]\n" - "fmla v28.4s, v9.4s, v5.s[0]\n" - "fmla v29.4s, v9.4s, v5.s[1]\n" - "fmla v30.4s, v9.4s, v5.s[2]\n" - "fmla v31.4s, v9.4s, v5.s[3]\n" + "fmla v16.4s, v10.4s, v3.s[0]\n" + "fmla v17.4s, v10.4s, v3.s[1]\n" + "fmla v18.4s, v10.4s, v3.s[2]\n" + "fmla v19.4s, v10.4s, v3.s[3]\n" + "fmla v20.4s, v10.4s, v2.s[0]\n" + "fmla v21.4s, v10.4s, v2.s[1]\n" + "fmla v22.4s, v10.4s, v2.s[2]\n" + "fmla v23.4s, v10.4s, v2.s[3]\n" + "fmla v24.4s, v10.4s, v1.s[0]\n" + "fmla v25.4s, v10.4s, v1.s[1]\n" + "fmla v26.4s, v10.4s, v1.s[2]\n" + "fmla v27.4s, v10.4s, v1.s[3]\n" + "fmla v28.4s, v10.4s, v0.s[0]\n" + "fmla v29.4s, v10.4s, v0.s[1]\n" + "fmla v30.4s, v10.4s, v0.s[2]\n" + "fmla v31.4s, v10.4s, v0.s[3]\n" "16:" // Output channel oddments: Done - "fmin v16.4s, v16.4s, v10.4s\n" - "fmin v17.4s, v17.4s, v10.4s\n" - "fmin v18.4s, v18.4s, v10.4s\n" - "fmin v19.4s, v19.4s, v10.4s\n" - "fmin v20.4s, v20.4s, v10.4s\n" - "fmin v21.4s, v21.4s, v10.4s\n" - "fmin v22.4s, v22.4s, v10.4s\n" - "fmin v23.4s, v23.4s, v10.4s\n" - "fmin v24.4s, v24.4s, v10.4s\n" - "fmin v25.4s, v25.4s, v10.4s\n" - "fmin v26.4s, v26.4s, v10.4s\n" - "fmin v27.4s, v27.4s, v10.4s\n" - "fmin v28.4s, v28.4s, v10.4s\n" - "fmin v29.4s, v29.4s, v10.4s\n" - "fmin v30.4s, v30.4s, v10.4s\n" - "fmin v31.4s, v31.4s, v10.4s\n" - "fmax v16.4s, v16.4s, v11.4s\n" - "fmax v17.4s, v17.4s, v11.4s\n" - "fmax v18.4s, v18.4s, v11.4s\n" - "fmax v19.4s, v19.4s, v11.4s\n" - "fmax v20.4s, v20.4s, v11.4s\n" - "fmax v21.4s, v21.4s, v11.4s\n" - "fmax v22.4s, v22.4s, v11.4s\n" - "fmax v23.4s, v23.4s, v11.4s\n" - "fmax v24.4s, v24.4s, v11.4s\n" - "fmax v25.4s, v25.4s, v11.4s\n" - "fmax v26.4s, v26.4s, v11.4s\n" - "fmax v27.4s, v27.4s, v11.4s\n" - "fmax v28.4s, v28.4s, v11.4s\n" - "fmax v29.4s, v29.4s, v11.4s\n" - "fmax v30.4s, v30.4s, v11.4s\n" - "fmax v31.4s, v31.4s, v11.4s\n" + "fmin v16.4s, v16.4s, v11.4s\n" + "fmin v17.4s, v17.4s, v11.4s\n" + "fmin v18.4s, v18.4s, v11.4s\n" + "fmin v19.4s, v19.4s, v11.4s\n" + "fmin v20.4s, v20.4s, v11.4s\n" + "fmin v21.4s, v21.4s, v11.4s\n" + "fmin v22.4s, v22.4s, v11.4s\n" + "fmin v23.4s, v23.4s, v11.4s\n" + "fmin v24.4s, v24.4s, v11.4s\n" + "fmin v25.4s, v25.4s, v11.4s\n" + "fmin v26.4s, v26.4s, v11.4s\n" + "fmin v27.4s, v27.4s, v11.4s\n" + "fmin v28.4s, v28.4s, v11.4s\n" + "fmin v29.4s, v29.4s, v11.4s\n" + "fmin v30.4s, v30.4s, v11.4s\n" + "fmin v31.4s, v31.4s, v11.4s\n" + "fmax v16.4s, v16.4s, v12.4s\n" + "fmax v17.4s, v17.4s, v12.4s\n" + "fmax v18.4s, v18.4s, v12.4s\n" + "fmax v19.4s, v19.4s, v12.4s\n" + "fmax v20.4s, v20.4s, v12.4s\n" + "fmax v21.4s, v21.4s, v12.4s\n" + "fmax v22.4s, v22.4s, v12.4s\n" + "fmax v23.4s, v23.4s, v12.4s\n" + "fmax v24.4s, v24.4s, v12.4s\n" + "fmax v25.4s, v25.4s, v12.4s\n" + "fmax v26.4s, v26.4s, v12.4s\n" + "fmax v27.4s, v27.4s, v12.4s\n" + "fmax v28.4s, v28.4s, v12.4s\n" + "fmax v29.4s, v29.4s, v12.4s\n" + "fmax v30.4s, v30.4s, v12.4s\n" + "fmax v31.4s, v31.4s, v12.4s\n" "tbz %x[n_output_channels], #1, 17f\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "add x20, x20, x10, LSL #2\n" - "add x21, x21, x10, LSL #2\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "add x22, x22, x10, LSL #2\n" - "add x23, x23, x10, LSL #2\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "add x24, x24, x10, LSL #2\n" - "add x25, x25, x10, LSL #2\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "add x26, x26, x10, LSL #2\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" "add x27, x27, x10, LSL #2\n" - "st1 { v16.d }[0], [x20]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "add x20, x20, x10, LSL #2\n" - "st1 { v17.d }[0], [x21]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "add x21, x21, x10, LSL #2\n" - "st1 { v18.d }[0], [x22]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "add x22, x22, x10, LSL #2\n" - "st1 { v19.d }[0], [x23]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "add x23, x23, x10, LSL #2\n" - "st1 { v20.d }[0], [x24]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "add x24, x24, x10, LSL #2\n" - "st1 { v21.d }[0], [x25]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "add x25, x25, x10, LSL #2\n" - "st1 { v22.d }[0], [x26]\n" - "ldr x26, [%x[outptrs], #0x70]\n" "add x26, x26, x10, LSL #2\n" - "st1 { v23.d }[0], [x27]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "add x25, x25, x10, LSL #2\n" + "add x24, x24, x10, LSL #2\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "add x23, x23, x10, LSL #2\n" + "add x22, x22, x10, LSL #2\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "add x21, x21, x10, LSL #2\n" + "add x20, x20, x10, LSL #2\n" + "st1 { v16.d }[0], [x27]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "add x27, x27, x10, LSL #2\n" + "st1 { v17.d }[0], [x26]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "add x26, x26, x10, LSL #2\n" + "st1 { v18.d }[0], [x25]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "add x25, x25, x10, LSL #2\n" + "st1 { v19.d }[0], [x24]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "add x24, x24, x10, LSL #2\n" + "st1 { v20.d }[0], [x23]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "add x23, x23, x10, LSL #2\n" + "st1 { v21.d }[0], [x22]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "add x22, x22, x10, LSL #2\n" + "st1 { v22.d }[0], [x21]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "add x21, x21, x10, LSL #2\n" + "st1 { v23.d }[0], [x20]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "add x20, x20, x10, LSL #2\n" "add x10, x10, #0x2\n" - "st1 { v24.d }[0], [x20]\n" - "st1 { v25.d }[0], [x21]\n" - "st1 { v26.d }[0], [x22]\n" - "st1 { v27.d }[0], [x23]\n" - "st1 { v28.d }[0], [x24]\n" - "st1 { v29.d }[0], [x25]\n" - "st1 { v30.d }[0], [x26]\n" - "st1 { v31.d }[0], [x27]\n" + "st1 { v24.d }[0], [x27]\n" + "st1 { v25.d }[0], [x26]\n" + "st1 { v26.d }[0], [x25]\n" + "st1 { v27.d }[0], [x24]\n" + "st1 { v28.d }[0], [x23]\n" + "st1 { v29.d }[0], [x22]\n" + "st1 { v30.d }[0], [x21]\n" + "st1 { v31.d }[0], [x20]\n" "tbz %x[n_output_channels], #0, 18f\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "add x20, x20, x10, LSL #2\n" - "add x21, x21, x10, LSL #2\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "add x22, x22, x10, LSL #2\n" - "add x23, x23, x10, LSL #2\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "add x24, x24, x10, LSL #2\n" - "add x25, x25, x10, LSL #2\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "add x26, x26, x10, LSL #2\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" "add x27, x27, x10, LSL #2\n" - "st1 { v16.s }[2], [x20]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "add x20, x20, x10, LSL #2\n" - "st1 { v17.s }[2], [x21]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "add x21, x21, x10, LSL #2\n" - "st1 { v18.s }[2], [x22]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "add x22, x22, x10, LSL #2\n" - "st1 { v19.s }[2], [x23]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "add x23, x23, x10, LSL #2\n" - "st1 { v20.s }[2], [x24]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "add x24, x24, x10, LSL #2\n" - "st1 { v21.s }[2], [x25]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "add x25, x25, x10, LSL #2\n" - "st1 { v22.s }[2], [x26]\n" - "ldr x26, [%x[outptrs], #0x70]\n" "add x26, x26, x10, LSL #2\n" - "st1 { v23.s }[2], [x27]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "add x25, x25, x10, LSL #2\n" + "add x24, x24, x10, LSL #2\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "add x23, x23, x10, LSL #2\n" + "add x22, x22, x10, LSL #2\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "add x21, x21, x10, LSL #2\n" + "add x20, x20, x10, LSL #2\n" + "st1 { v16.s }[2], [x27]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "add x27, x27, x10, LSL #2\n" - "st1 { v24.s }[2], [x20]\n" - "st1 { v25.s }[2], [x21]\n" - "st1 { v26.s }[2], [x22]\n" - "st1 { v27.s }[2], [x23]\n" - "st1 { v28.s }[2], [x24]\n" - "st1 { v29.s }[2], [x25]\n" - "st1 { v30.s }[2], [x26]\n" - "st1 { v31.s }[2], [x27]\n" + "st1 { v17.s }[2], [x26]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "add x26, x26, x10, LSL #2\n" + "st1 { v18.s }[2], [x25]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "add x25, x25, x10, LSL #2\n" + "st1 { v19.s }[2], [x24]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "add x24, x24, x10, LSL #2\n" + "st1 { v20.s }[2], [x23]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "add x23, x23, x10, LSL #2\n" + "st1 { v21.s }[2], [x22]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "add x22, x22, x10, LSL #2\n" + "st1 { v22.s }[2], [x21]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "add x21, x21, x10, LSL #2\n" + "st1 { v23.s }[2], [x20]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "add x20, x20, x10, LSL #2\n" + "st1 { v24.s }[2], [x27]\n" + "st1 { v25.s }[2], [x26]\n" + "st1 { v26.s }[2], [x25]\n" + "st1 { v27.s }[2], [x24]\n" + "st1 { v28.s }[2], [x23]\n" + "st1 { v29.s }[2], [x22]\n" + "st1 { v30.s }[2], [x21]\n" + "st1 { v31.s }[2], [x20]\n" "b 18f\n" "17:" // Output channel oddments: Done: Store: Bit 1: Unset - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "add x20, x20, x10, LSL #2\n" - "add x21, x21, x10, LSL #2\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "add x22, x22, x10, LSL #2\n" - "add x23, x23, x10, LSL #2\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "add x24, x24, x10, LSL #2\n" - "add x25, x25, x10, LSL #2\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "add x26, x26, x10, LSL #2\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" "add x27, x27, x10, LSL #2\n" - "st1 { v16.s }[0], [x20]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "add x20, x20, x10, LSL #2\n" - "st1 { v17.s }[0], [x21]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "add x21, x21, x10, LSL #2\n" - "st1 { v18.s }[0], [x22]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "add x22, x22, x10, LSL #2\n" - "st1 { v19.s }[0], [x23]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "add x23, x23, x10, LSL #2\n" - "st1 { v20.s }[0], [x24]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "add x24, x24, x10, LSL #2\n" - "st1 { v21.s }[0], [x25]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "add x25, x25, x10, LSL #2\n" - "st1 { v22.s }[0], [x26]\n" - "ldr x26, [%x[outptrs], #0x70]\n" "add x26, x26, x10, LSL #2\n" - "st1 { v23.s }[0], [x27]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "add x25, x25, x10, LSL #2\n" + "add x24, x24, x10, LSL #2\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "add x23, x23, x10, LSL #2\n" + "add x22, x22, x10, LSL #2\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "add x21, x21, x10, LSL #2\n" + "add x20, x20, x10, LSL #2\n" + "st1 { v16.s }[0], [x27]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "add x27, x27, x10, LSL #2\n" - "st1 { v24.s }[0], [x20]\n" - "st1 { v25.s }[0], [x21]\n" - "st1 { v26.s }[0], [x22]\n" - "st1 { v27.s }[0], [x23]\n" - "st1 { v28.s }[0], [x24]\n" - "st1 { v29.s }[0], [x25]\n" - "st1 { v30.s }[0], [x26]\n" - "st1 { v31.s }[0], [x27]\n" + "st1 { v17.s }[0], [x26]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "add x26, x26, x10, LSL #2\n" + "st1 { v18.s }[0], [x25]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "add x25, x25, x10, LSL #2\n" + "st1 { v19.s }[0], [x24]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "add x24, x24, x10, LSL #2\n" + "st1 { v20.s }[0], [x23]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "add x23, x23, x10, LSL #2\n" + "st1 { v21.s }[0], [x22]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "add x22, x22, x10, LSL #2\n" + "st1 { v22.s }[0], [x21]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "add x21, x21, x10, LSL #2\n" + "st1 { v23.s }[0], [x20]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "add x20, x20, x10, LSL #2\n" + "st1 { v24.s }[0], [x27]\n" + "st1 { v25.s }[0], [x26]\n" + "st1 { v26.s }[0], [x25]\n" + "st1 { v27.s }[0], [x24]\n" + "st1 { v28.s }[0], [x23]\n" + "st1 { v29.s }[0], [x22]\n" + "st1 { v30.s }[0], [x21]\n" + "st1 { v31.s }[0], [x20]\n" "18:" // Output channel oddments: Done: Store: Bit 1: End - "19:" // Done - : [weights] "+&r" (weights) : [bias] "r" (bias), [inptrs] "r" (inptrs), [kernel_points] "r" ((uint64_t) kernel_points), [minmax_vals] "r" (minmax_vals), [n_output_channels] "r" ((uint64_t) n_output_channels), [outptrs] "r" (outptrs) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp index 79bba40ca3b67baa6b25532cf7dbbcb9dce0daa0..2b97ad816a6f1c644eea3ab729746ceb7d18b5d9 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include @@ -34,15 +34,7 @@ namespace arm_conv { namespace depthwise { -void a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst_impl( - const unsigned int, - const int8_t *const *const, - const int8_t *, - const int32_t *, - const arm_gemm::Requantize32&, - const int32_t *, const int32_t *, - int8_t *const *const -); +void a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst_impl(unsigned int, const int8_t *const *, const int8_t *, const int32_t *, const arm_gemm::Requantize32&, const int32_t *, const int32_t *, int8_t *const *); class a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst : public DepthwiseDepthfirstStrategy { @@ -72,7 +64,7 @@ class a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst : public DepthwiseDepthfirstS ) const override { interleave_a64_s8q_3x3_dot::pack_parameters( - args.input_channels, buffer, reinterpret_cast(biases), + args.input_channels * args.channel_multiplier, buffer, reinterpret_cast(biases), reinterpret_cast(weights), qp, ld_weight_col, ld_weight_row ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp index fda88f94bb215cb549b464b19fbe760132ded6d0..916c8a4afe6391984be313f0ced7080b2180c63a 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp @@ -30,15 +30,7 @@ namespace arm_conv { namespace depthwise { -void a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst_impl( - const unsigned int n_channels, - const int8_t *const *const inptrs, - const int8_t *params, - const int32_t *, // Bias, should be wrapped into the parameters - const arm_gemm::Requantize32& qp, - const int32_t *, const int32_t *, // Requant parameters, also wrapped - int8_t *const *const outptrs -) +void a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst_impl(const unsigned int n_channels, const int8_t *const *const inptrs, const int8_t *params, const int32_t *, const arm_gemm::Requantize32& qp, const int32_t *, const int32_t *, int8_t *const *const outptrs) { __asm__ __volatile__( "mov x20, #0x1\n" @@ -47,817 +39,817 @@ void a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst_impl( "ldp x13, x12, [%x[inptrs], #0x10]\n" "orr x20, x20, #0x10000\n" "lsr x11, %x[n_channels], #0x4\n" - "dup v14.4s, w20\n" + "dup v12.4s, w20\n" "ldp x10, x9, [%x[inptrs], #0x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_minval]\n" "ld1r { v13.4s }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_maxval]\n" - "ld1r { v12.4s }, [x20]\n" - "add x20, %x[qp], %[offsetof_Requantize32_b_offset]\n" "ld1r { v11.4s }, [x20]\n" + "add x20, %x[qp], %[offsetof_Requantize32_b_offset]\n" + "ld1r { v16.4s }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_c_offset]\n" - "ld1r { v10.4s }, [x20]\n" + "ld1r { v14.4s }, [x20]\n" "mov x28, #0x0\n" "mov x27, #0x0\n" - "ldp x26, x25, [%x[inptrs], #0x30]\n" - "ldp x24, x23, [%x[outptrs], #0x0]\n" - "ldp x22, x21, [%x[outptrs], #0x10]\n" + "ldp x26, x21, [%x[inptrs], #0x30]\n" + "ldp x25, x24, [%x[outptrs], #0x0]\n" + "ldp x23, x22, [%x[outptrs], #0x10]\n" "cbz x11, 3f\n" - "ldr q9, [x15, x28]\n" - "ldr q8, [x14, x28]\n" - "subs x11, x11, #0x1\n" - "ldr q7, [x13, x28]\n" - "ldr q6, [x12, x28]\n" - "zip2 v5.16b, v9.16b, v7.16b\n" - "zip1 v9.16b, v9.16b, v7.16b\n" - "ldr q4, [x10, x28]\n" - "ldr q3, [x9, x28]\n" - "zip1 v7.16b, v8.16b, v6.16b\n" - "zip2 v6.16b, v8.16b, v6.16b\n" - "ldr q2, [x26, x28]\n" - "ldr q1, [x25, x28]\n" - "zip2 v8.16b, v9.16b, v7.16b\n" - "zip1 v9.16b, v9.16b, v7.16b\n" - "ldr q0, [%x[params], #0x10]\n" - "ldr q16, [%x[params], #0x20]\n" - "zip1 v7.16b, v5.16b, v6.16b\n" - "zip2 v6.16b, v5.16b, v6.16b\n" - "ldr q5, [%x[params], #0x0]\n" - "ldr q31, [%x[params], #0x30]\n" - "zip2 v30.16b, v4.16b, v2.16b\n" - "zip1 v4.16b, v4.16b, v2.16b\n" - "ldp x15, x14, [%x[inptrs], #0x40]\n" - "ldr q29, [x15, x28]\n" - "zip1 v2.16b, v3.16b, v1.16b\n" - "zip2 v1.16b, v3.16b, v1.16b\n" + "ldr q15, [x15, x28]\n" "ldr q28, [x14, x28]\n" - "ldp x13, x12, [%x[inptrs], #0x50]\n" - "zip2 v3.16b, v4.16b, v2.16b\n" - "zip1 v4.16b, v4.16b, v2.16b\n" - "ldr q27, [x13, x28]\n" - "ldr q26, [x12, x28]\n" - "zip2 v25.16b, v29.16b, v27.16b\n" - "zip1 v29.16b, v29.16b, v27.16b\n" - "ldp x10, x9, [%x[inptrs], #0x60]\n" - "ldr q24, [x10, x28]\n" - "zip1 v27.16b, v28.16b, v26.16b\n" - "zip2 v26.16b, v28.16b, v26.16b\n" - "ldr q23, [x9, x28]\n" - "ldp x26, x25, [%x[inptrs], #0x70]\n" - "zip1 v2.16b, v30.16b, v1.16b\n" - "zip2 v1.16b, v30.16b, v1.16b\n" - "ldr q22, [x26, x28]\n" - "ldr q21, [x25, x28]\n" - "zip2 v20.16b, v24.16b, v22.16b\n" - "zip1 v24.16b, v24.16b, v22.16b\n" - "zip1 v22.16b, v23.16b, v21.16b\n" - "zip2 v21.16b, v23.16b, v21.16b\n" + "subs x11, x11, #0x1\n" + "ldr q30, [x13, x28]\n" + "ldr q8, [x12, x28]\n" + "zip2 v19.16b, v15.16b, v30.16b\n" + "zip1 v15.16b, v15.16b, v30.16b\n" + "ldr q26, [x10, x28]\n" + "ldr q0, [x9, x28]\n" + "zip1 v7.16b, v28.16b, v8.16b\n" + "zip2 v8.16b, v28.16b, v8.16b\n" + "ldr q29, [x26, x28]\n" + "ldr q10, [x21, x28]\n" + "zip2 v25.16b, v15.16b, v7.16b\n" + "zip1 v15.16b, v15.16b, v7.16b\n" + "ldr q1, [%x[params], #0x10]\n" + "ldr q6, [%x[params], #0x20]\n" + "zip1 v7.16b, v19.16b, v8.16b\n" + "zip2 v8.16b, v19.16b, v8.16b\n" + "ldr q31, [%x[params], #0x0]\n" + "ldr q20, [%x[params], #0x30]\n" + "zip2 v21.16b, v26.16b, v29.16b\n" + "zip1 v26.16b, v26.16b, v29.16b\n" + "ldp x21, x20, [%x[inptrs], #0x40]\n" + "ldr q22, [x21, x28]\n" + "zip1 v27.16b, v0.16b, v10.16b\n" + "zip2 v10.16b, v0.16b, v10.16b\n" + "ldr q17, [x20, x28]\n" + "ldp x21, x20, [%x[inptrs], #0x50]\n" + "zip2 v23.16b, v26.16b, v27.16b\n" + "zip1 v26.16b, v26.16b, v27.16b\n" + "ldr q9, [x21, x28]\n" + "ldr q5, [x20, x28]\n" + "zip2 v28.16b, v22.16b, v9.16b\n" + "zip1 v22.16b, v22.16b, v9.16b\n" + "ldp x21, x20, [%x[inptrs], #0x60]\n" + "ldr q27, [x21, x28]\n" + "zip1 v24.16b, v17.16b, v5.16b\n" + "zip2 v5.16b, v17.16b, v5.16b\n" + "ldr q18, [x20, x28]\n" + "ldp x21, x20, [%x[inptrs], #0x70]\n" + "zip1 v3.16b, v21.16b, v10.16b\n" + "zip2 v10.16b, v21.16b, v10.16b\n" + "ldr q4, [x21, x28]\n" + "ldr q9, [x20, x28]\n" + "zip2 v17.16b, v27.16b, v4.16b\n" + "zip1 v27.16b, v27.16b, v4.16b\n" + "zip1 v4.16b, v18.16b, v9.16b\n" + "zip2 v9.16b, v18.16b, v9.16b\n" "ldp x15, x14, [%x[inptrs], #0x0]\n" "ldp x13, x12, [%x[inptrs], #0x10]\n" "ldp x10, x9, [%x[inptrs], #0x20]\n" - "ldp x26, x25, [%x[inptrs], #0x30]\n" - "zip2 v28.16b, v29.16b, v27.16b\n" - "zip1 v29.16b, v29.16b, v27.16b\n" - "zip1 v27.16b, v25.16b, v26.16b\n" - "zip2 v26.16b, v25.16b, v26.16b\n" + "ldp x26, x21, [%x[inptrs], #0x30]\n" + "zip2 v19.16b, v22.16b, v24.16b\n" + "zip1 v22.16b, v22.16b, v24.16b\n" + "zip1 v0.16b, v28.16b, v5.16b\n" + "zip2 v5.16b, v28.16b, v5.16b\n" "add %x[params], %x[params], #0x40\n" - "zip2 v23.16b, v24.16b, v22.16b\n" - "zip1 v24.16b, v24.16b, v22.16b\n" - "zip1 v22.16b, v20.16b, v21.16b\n" - "zip2 v21.16b, v20.16b, v21.16b\n" - "mov v30.16b, v5.16b\n" - "mov v25.16b, v5.16b\n" - "mov v20.16b, v5.16b\n" + "zip2 v24.16b, v27.16b, v4.16b\n" + "zip1 v27.16b, v27.16b, v4.16b\n" + "zip1 v2.16b, v17.16b, v9.16b\n" + "zip2 v9.16b, v17.16b, v9.16b\n" + "mov v30.16b, v31.16b\n" + "mov v29.16b, v31.16b\n" + "mov v28.16b, v31.16b\n" "beq 2f\n" "1:" // Loop - "movi v19.4s, #0x0\n" - ".inst 0x4e8495d3 // sdot v19.4s, v14.16b, v4.16b\n" - ".inst 0x4e899405 // sdot v5.4s, v0.16b, v9.16b\n" + "movi v21.4s, #0x0\n" + ".inst 0x4e9a9595 // sdot v21.4s, v12.16b, v26.16b\n" + ".inst 0x4e8f943f // sdot v31.4s, v1.16b, v15.16b\n" "add x28, x28, #0x10\n" - ".inst 0x4e9d95d3 // sdot v19.4s, v14.16b, v29.16b\n" - ".inst 0x4e849419 // sdot v25.4s, v0.16b, v4.16b\n" + ".inst 0x4e969595 // sdot v21.4s, v12.16b, v22.16b\n" + ".inst 0x4e9a943d // sdot v29.4s, v1.16b, v26.16b\n" + "movi v18.4s, #0x0\n" "subs x11, x11, #0x1\n" - ".inst 0x4e849605 // sdot v5.4s, v16.16b, v4.16b\n" - "ext v4.16b, v4.16b, v4.16b, #0x1\n" - "mov v18.16b, v19.16b\n .inst 0x4e9895d2 // sdot v18.4s, v14.16b, v24.16b\n" - ".inst 0x4e8995d3 // sdot v19.4s, v14.16b, v9.16b\n" - "ext v9.16b, v9.16b, v9.16b, #0x1\n" - ".inst 0x4e9d9619 // sdot v25.4s, v16.16b, v29.16b\n" - ".inst 0x4e9d97e5 // sdot v5.4s, v31.16b, v29.16b\n" - "ext v29.16b, v29.16b, v29.16b, #0x1\n" - ".inst 0x4e89941e // sdot v30.4s, v0.16b, v9.16b\n" - ".inst 0x4e849414 // sdot v20.4s, v0.16b, v4.16b\n" - "movi v17.4s, #0x0\n" - ".inst 0x4e8495d1 // sdot v17.4s, v14.16b, v4.16b\n" - ".inst 0x4e9d95d1 // sdot v17.4s, v14.16b, v29.16b\n" - ".inst 0x4e9897f9 // sdot v25.4s, v31.16b, v24.16b\n" - "ext v24.16b, v24.16b, v24.16b, #0x1\n" - ".inst 0x4e84961e // sdot v30.4s, v16.16b, v4.16b\n" - "ldr q4, [%x[params], #0x10]\n" - ".inst 0x4e9d9614 // sdot v20.4s, v16.16b, v29.16b\n" - "mls v5.4s, v19.4s, v11.4s\n" - "mov v16.16b, v17.16b\n .inst 0x4e9895d0 // sdot v16.4s, v14.16b, v24.16b\n" - ".inst 0x4e8995d1 // sdot v17.4s, v14.16b, v9.16b\n" - "ldr q9, [%x[params], #0x0]\n" - "sqrdmulh v5.4s, v5.4s, v9.4s\n" - ".inst 0x4e9d97fe // sdot v30.4s, v31.16b, v29.16b\n" - ".inst 0x4e9897f4 // sdot v20.4s, v31.16b, v24.16b\n" - "mls v30.4s, v17.4s, v11.4s\n" - "mls v25.4s, v18.4s, v11.4s\n" - "mls v20.4s, v16.4s, v11.4s\n" - "and v0.16b, v5.16b, v4.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqrdmulh v30.4s, v30.4s, v9.4s\n" - "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqrdmulh v20.4s, v20.4s, v9.4s\n" - "ldr q9, [%x[params], #0x60]\n" - "sqadd v5.4s, v5.4s, v0.4s\n" - "and v16.16b, v30.16b, v4.16b\n" - "and v31.16b, v25.16b, v4.16b\n" - "and v0.16b, v20.16b, v4.16b\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "srshl v5.4s, v5.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v16.4s\n" - "ldr q16, [%x[params], #0x40]\n" - "sqadd v25.4s, v25.4s, v31.4s\n" - "ldr q31, [%x[params], #0x50]\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "ldr q0, [%x[params], #0x30]\n" - "add v5.4s, v5.4s, v10.4s\n" - "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" - "srshl v20.4s, v20.4s, v4.4s\n" - "ldr q4, [%x[params], #0x70]\n" - "smax v5.4s, v5.4s, v13.4s\n" - "add v30.4s, v30.4s, v10.4s\n" - "add v25.4s, v25.4s, v10.4s\n" - "add v20.4s, v20.4s, v10.4s\n" - "smin v5.4s, v5.4s, v12.4s\n" + ".inst 0x4e9a94df // sdot v31.4s, v6.16b, v26.16b\n" + "ext v26.16b, v26.16b, v26.16b, #0x1\n" + "mov v17.16b, v21.16b\n .inst 0x4e9b9591 // sdot v17.4s, v12.16b, v27.16b\n" + ".inst 0x4e8f9595 // sdot v21.4s, v12.16b, v15.16b\n" + "ext v15.16b, v15.16b, v15.16b, #0x1\n" + ".inst 0x4e9a9592 // sdot v18.4s, v12.16b, v26.16b\n" + ".inst 0x4e9694dd // sdot v29.4s, v6.16b, v22.16b\n" + ".inst 0x4e96969f // sdot v31.4s, v20.16b, v22.16b\n" + "ext v22.16b, v22.16b, v22.16b, #0x1\n" + ".inst 0x4e8f943e // sdot v30.4s, v1.16b, v15.16b\n" + ".inst 0x4e9a943c // sdot v28.4s, v1.16b, v26.16b\n" + "mls v31.4s, v21.4s, v16.4s\n" + ".inst 0x4e969592 // sdot v18.4s, v12.16b, v22.16b\n" + ".inst 0x4e9b969d // sdot v29.4s, v20.16b, v27.16b\n" + "ext v27.16b, v27.16b, v27.16b, #0x1\n" + ".inst 0x4e9a94de // sdot v30.4s, v6.16b, v26.16b\n" + "ldr q26, [%x[params], #0x10]\n" + ".inst 0x4e9694dc // sdot v28.4s, v6.16b, v22.16b\n" + "mls v29.4s, v17.4s, v16.4s\n" + "mov v21.16b, v18.16b\n .inst 0x4e9b9595 // sdot v21.4s, v12.16b, v27.16b\n" + ".inst 0x4e8f9592 // sdot v18.4s, v12.16b, v15.16b\n" + "ldr q17, [%x[params], #0x0]\n" + "sqrdmulh v31.4s, v31.4s, v17.4s\n" + ".inst 0x4e96969e // sdot v30.4s, v20.16b, v22.16b\n" + ".inst 0x4e9b969c // sdot v28.4s, v20.16b, v27.16b\n" + "mls v30.4s, v18.4s, v16.4s\n" + "mls v28.4s, v21.4s, v16.4s\n" + "and v15.16b, v31.16b, v26.16b\n" + "sshr v15.4s, v15.4s, #0x1f\n" + "sqrdmulh v30.4s, v30.4s, v17.4s\n" + "sqrdmulh v29.4s, v29.4s, v17.4s\n" + "sqrdmulh v28.4s, v28.4s, v17.4s\n" + "ldr q1, [%x[params], #0x60]\n" + "sqadd v31.4s, v31.4s, v15.4s\n" + "and v18.16b, v30.16b, v26.16b\n" + "and v21.16b, v29.16b, v26.16b\n" + "and v17.16b, v28.16b, v26.16b\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v21.4s, v21.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "srshl v31.4s, v31.4s, v26.4s\n" + "sqadd v30.4s, v30.4s, v18.4s\n" + "ldr q18, [%x[params], #0x40]\n" + "sqadd v29.4s, v29.4s, v21.4s\n" + "ldr q27, [%x[params], #0x50]\n" + "sqadd v28.4s, v28.4s, v17.4s\n" + "ldr q15, [%x[params], #0x30]\n" + "add v31.4s, v31.4s, v14.4s\n" + "srshl v30.4s, v30.4s, v26.4s\n" + "srshl v29.4s, v29.4s, v26.4s\n" + "srshl v28.4s, v28.4s, v26.4s\n" + "ldr q20, [%x[params], #0x70]\n" + "smax v31.4s, v31.4s, v13.4s\n" + "add v30.4s, v30.4s, v14.4s\n" + "add v29.4s, v29.4s, v14.4s\n" + "add v28.4s, v28.4s, v14.4s\n" + "smin v31.4s, v31.4s, v11.4s\n" "smax v30.4s, v30.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smin v30.4s, v30.4s, v12.4s\n" - "smin v25.4s, v25.4s, v12.4s\n" - "smin v20.4s, v20.4s, v12.4s\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "movi v19.4s, #0x0\n" - ".inst 0x4e8395d3 // sdot v19.4s, v14.16b, v3.16b\n" - ".inst 0x4e9c95d3 // sdot v19.4s, v14.16b, v28.16b\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" + "smax v29.4s, v29.4s, v13.4s\n" + "smax v28.4s, v28.4s, v13.4s\n" + "smin v30.4s, v30.4s, v11.4s\n" + "smin v29.4s, v29.4s, v11.4s\n" + "smin v28.4s, v28.4s, v11.4s\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" + "movi v22.4s, #0x0\n" + ".inst 0x4e979596 // sdot v22.4s, v12.16b, v23.16b\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s5, [x24, x27]\n" - "ldr q5, [%x[params], #0x20]\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "mov v18.16b, v19.16b\n .inst 0x4e9795d2 // sdot v18.4s, v14.16b, v23.16b\n" + "str s31, [x25, x27]\n" + "ldr q26, [%x[params], #0x20]\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + ".inst 0x4e939596 // sdot v22.4s, v12.16b, v19.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "str s30, [x23, x27]\n" - ".inst 0x4e8895d3 // sdot v19.4s, v14.16b, v8.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s25, [x22, x27]\n" - "mov v30.16b, v5.16b\n" - "str s20, [x21, x27]\n" - "mov v25.16b, v5.16b\n" - "mov v20.16b, v5.16b\n" - ".inst 0x4e889405 // sdot v5.4s, v0.16b, v8.16b\n" - ".inst 0x4e839419 // sdot v25.4s, v0.16b, v3.16b\n" - ".inst 0x4e839605 // sdot v5.4s, v16.16b, v3.16b\n" - "ext v8.16b, v8.16b, v8.16b, #0x1\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "str s30, [x24, x27]\n" + "mov v6.16b, v22.16b\n .inst 0x4e989586 // sdot v6.4s, v12.16b, v24.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + "str s29, [x23, x27]\n" + "mov v30.16b, v26.16b\n" + ".inst 0x4e999596 // sdot v22.4s, v12.16b, v25.16b\n" + "str s28, [x22, x27]\n" + "mov v29.16b, v26.16b\n" + "mov v21.16b, v26.16b\n" + ".inst 0x4e9995fa // sdot v26.4s, v15.16b, v25.16b\n" + ".inst 0x4e9795fd // sdot v29.4s, v15.16b, v23.16b\n" + ".inst 0x4e97965a // sdot v26.4s, v18.16b, v23.16b\n" + "ext v25.16b, v25.16b, v25.16b, #0x1\n" "add x27, x27, #0x4\n" - "ext v3.16b, v3.16b, v3.16b, #0x1\n" - "movi v17.4s, #0x0\n" - ".inst 0x4e88941e // sdot v30.4s, v0.16b, v8.16b\n" - ".inst 0x4e839414 // sdot v20.4s, v0.16b, v3.16b\n" - ".inst 0x4e8395d1 // sdot v17.4s, v14.16b, v3.16b\n" - ".inst 0x4e9c9619 // sdot v25.4s, v16.16b, v28.16b\n" - ".inst 0x4e9c97e5 // sdot v5.4s, v31.16b, v28.16b\n" - "ext v28.16b, v28.16b, v28.16b, #0x1\n" - ".inst 0x4e83961e // sdot v30.4s, v16.16b, v3.16b\n" - "ldr q3, [x9, x28]\n" - ".inst 0x4e9c9614 // sdot v20.4s, v16.16b, v28.16b\n" - "mls v5.4s, v19.4s, v11.4s\n" - ".inst 0x4e9c95d1 // sdot v17.4s, v14.16b, v28.16b\n" - ".inst 0x4e9797f9 // sdot v25.4s, v31.16b, v23.16b\n" "ext v23.16b, v23.16b, v23.16b, #0x1\n" - ".inst 0x4e9c97fe // sdot v30.4s, v31.16b, v28.16b\n" - ".inst 0x4e9797f4 // sdot v20.4s, v31.16b, v23.16b\n" - "sqrdmulh v5.4s, v5.4s, v9.4s\n" - "mov v16.16b, v17.16b\n .inst 0x4e9795d0 // sdot v16.4s, v14.16b, v23.16b\n" - ".inst 0x4e8895d1 // sdot v17.4s, v14.16b, v8.16b\n" - "ldr q8, [x14, x28]\n" - "mls v30.4s, v17.4s, v11.4s\n" - "mls v25.4s, v18.4s, v11.4s\n" - "mls v20.4s, v16.4s, v11.4s\n" - "and v0.16b, v5.16b, v4.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqrdmulh v30.4s, v30.4s, v9.4s\n" - "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqrdmulh v20.4s, v20.4s, v9.4s\n" - "ldr q9, [%x[params], #0xc0]\n" - "sqadd v5.4s, v5.4s, v0.4s\n" - "and v16.16b, v30.16b, v4.16b\n" - "and v31.16b, v25.16b, v4.16b\n" - "and v0.16b, v20.16b, v4.16b\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "srshl v5.4s, v5.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v16.4s\n" - "ldr q16, [%x[params], #0xa0]\n" - "sqadd v25.4s, v25.4s, v31.4s\n" - "ldr q31, [%x[params], #0xb0]\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "ldr q0, [%x[params], #0x90]\n" - "add v5.4s, v5.4s, v10.4s\n" - "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" - "srshl v20.4s, v20.4s, v4.4s\n" - "ldr q4, [%x[params], #0xd0]\n" - "smax v5.4s, v5.4s, v13.4s\n" - "add v30.4s, v30.4s, v10.4s\n" - "add v25.4s, v25.4s, v10.4s\n" - "add v20.4s, v20.4s, v10.4s\n" - "smin v5.4s, v5.4s, v12.4s\n" + "movi v28.4s, #0x0\n" + ".inst 0x4e9995fe // sdot v30.4s, v15.16b, v25.16b\n" + ".inst 0x4e9795f5 // sdot v21.4s, v15.16b, v23.16b\n" + ".inst 0x4e97959c // sdot v28.4s, v12.16b, v23.16b\n" + ".inst 0x4e93965d // sdot v29.4s, v18.16b, v19.16b\n" + ".inst 0x4e93977a // sdot v26.4s, v27.16b, v19.16b\n" + "ext v19.16b, v19.16b, v19.16b, #0x1\n" + ".inst 0x4e97965e // sdot v30.4s, v18.16b, v23.16b\n" + "ldr q4, [x9, x28]\n" + ".inst 0x4e939655 // sdot v21.4s, v18.16b, v19.16b\n" + "mls v26.4s, v22.4s, v16.4s\n" + ".inst 0x4e93959c // sdot v28.4s, v12.16b, v19.16b\n" + ".inst 0x4e98977d // sdot v29.4s, v27.16b, v24.16b\n" + "ext v24.16b, v24.16b, v24.16b, #0x1\n" + ".inst 0x4e93977e // sdot v30.4s, v27.16b, v19.16b\n" + ".inst 0x4e989775 // sdot v21.4s, v27.16b, v24.16b\n" + "sqrdmulh v26.4s, v26.4s, v1.4s\n" + "mov v17.16b, v28.16b\n .inst 0x4e989591 // sdot v17.4s, v12.16b, v24.16b\n" + ".inst 0x4e99959c // sdot v28.4s, v12.16b, v25.16b\n" + "ldr q31, [x14, x28]\n" + "mls v30.4s, v28.4s, v16.4s\n" + "mls v29.4s, v6.4s, v16.4s\n" + "mls v21.4s, v17.4s, v16.4s\n" + "and v17.16b, v26.16b, v20.16b\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqrdmulh v30.4s, v30.4s, v1.4s\n" + "sqrdmulh v29.4s, v29.4s, v1.4s\n" + "sqrdmulh v21.4s, v21.4s, v1.4s\n" + "ldr q27, [%x[params], #0xc0]\n" + "sqadd v26.4s, v26.4s, v17.4s\n" + "and v18.16b, v30.16b, v20.16b\n" + "and v6.16b, v29.16b, v20.16b\n" + "and v17.16b, v21.16b, v20.16b\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v6.4s, v6.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "srshl v26.4s, v26.4s, v20.4s\n" + "sqadd v30.4s, v30.4s, v18.4s\n" + "ldr q28, [%x[params], #0xa0]\n" + "sqadd v29.4s, v29.4s, v6.4s\n" + "ldr q24, [%x[params], #0xb0]\n" + "sqadd v21.4s, v21.4s, v17.4s\n" + "ldr q15, [%x[params], #0x90]\n" + "add v26.4s, v26.4s, v14.4s\n" + "srshl v30.4s, v30.4s, v20.4s\n" + "srshl v29.4s, v29.4s, v20.4s\n" + "srshl v21.4s, v21.4s, v20.4s\n" + "ldr q1, [%x[params], #0xd0]\n" + "smax v26.4s, v26.4s, v13.4s\n" + "add v30.4s, v30.4s, v14.4s\n" + "add v29.4s, v29.4s, v14.4s\n" + "add v21.4s, v21.4s, v14.4s\n" + "smin v26.4s, v26.4s, v11.4s\n" "smax v30.4s, v30.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smin v30.4s, v30.4s, v12.4s\n" - "smin v25.4s, v25.4s, v12.4s\n" - "smin v20.4s, v20.4s, v12.4s\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "movi v19.4s, #0x0\n" - ".inst 0x4e8295d3 // sdot v19.4s, v14.16b, v2.16b\n" - ".inst 0x4e9b95d3 // sdot v19.4s, v14.16b, v27.16b\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" + "smax v29.4s, v29.4s, v13.4s\n" + "smax v21.4s, v21.4s, v13.4s\n" + "smin v30.4s, v30.4s, v11.4s\n" + "smin v29.4s, v29.4s, v11.4s\n" + "smin v21.4s, v21.4s, v11.4s\n" + "uzp1 v26.16b, v26.16b, v26.16b\n" + "movi v22.4s, #0x0\n" + ".inst 0x4e839596 // sdot v22.4s, v12.16b, v3.16b\n" + ".inst 0x4e809596 // sdot v22.4s, v12.16b, v0.16b\n" + "uzp1 v26.16b, v26.16b, v26.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s5, [x24, x27]\n" - "ldr q5, [%x[params], #0x80]\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "mov v18.16b, v19.16b\n .inst 0x4e9695d2 // sdot v18.4s, v14.16b, v22.16b\n" + "str s26, [x25, x27]\n" + "ldr q26, [%x[params], #0x80]\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "uzp1 v21.16b, v21.16b, v21.16b\n" + "mov v18.16b, v22.16b\n .inst 0x4e829592 // sdot v18.4s, v12.16b, v2.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "str s30, [x23, x27]\n" - ".inst 0x4e8795d3 // sdot v19.4s, v14.16b, v7.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s25, [x22, x27]\n" - "mov v30.16b, v5.16b\n" - "str s20, [x21, x27]\n" - "mov v25.16b, v5.16b\n" - "mov v20.16b, v5.16b\n" - ".inst 0x4e879405 // sdot v5.4s, v0.16b, v7.16b\n" - ".inst 0x4e829419 // sdot v25.4s, v0.16b, v2.16b\n" - ".inst 0x4e829605 // sdot v5.4s, v16.16b, v2.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "str s30, [x24, x27]\n" + ".inst 0x4e879596 // sdot v22.4s, v12.16b, v7.16b\n" + "uzp1 v21.16b, v21.16b, v21.16b\n" + "str s29, [x23, x27]\n" + "mov v6.16b, v26.16b\n" + "str s21, [x22, x27]\n" + "mov v25.16b, v26.16b\n" + "mov v20.16b, v26.16b\n" + ".inst 0x4e8795fa // sdot v26.4s, v15.16b, v7.16b\n" + ".inst 0x4e8395f9 // sdot v25.4s, v15.16b, v3.16b\n" + ".inst 0x4e83979a // sdot v26.4s, v28.16b, v3.16b\n" "ext v7.16b, v7.16b, v7.16b, #0x1\n" "add x27, x27, #0x4\n" + "ext v3.16b, v3.16b, v3.16b, #0x1\n" + "movi v23.4s, #0x0\n" + ".inst 0x4e8795e6 // sdot v6.4s, v15.16b, v7.16b\n" + ".inst 0x4e8395f4 // sdot v20.4s, v15.16b, v3.16b\n" + ".inst 0x4e839597 // sdot v23.4s, v12.16b, v3.16b\n" + ".inst 0x4e809799 // sdot v25.4s, v28.16b, v0.16b\n" + ".inst 0x4e80971a // sdot v26.4s, v24.16b, v0.16b\n" + "ext v0.16b, v0.16b, v0.16b, #0x1\n" + ".inst 0x4e839786 // sdot v6.4s, v28.16b, v3.16b\n" + "ldr q19, [x26, x28]\n" + ".inst 0x4e809794 // sdot v20.4s, v28.16b, v0.16b\n" + "mls v26.4s, v22.4s, v16.4s\n" + ".inst 0x4e809597 // sdot v23.4s, v12.16b, v0.16b\n" + ".inst 0x4e829719 // sdot v25.4s, v24.16b, v2.16b\n" "ext v2.16b, v2.16b, v2.16b, #0x1\n" - "movi v17.4s, #0x0\n" - ".inst 0x4e87941e // sdot v30.4s, v0.16b, v7.16b\n" - ".inst 0x4e829414 // sdot v20.4s, v0.16b, v2.16b\n" - ".inst 0x4e8295d1 // sdot v17.4s, v14.16b, v2.16b\n" - ".inst 0x4e9b9619 // sdot v25.4s, v16.16b, v27.16b\n" - ".inst 0x4e9b97e5 // sdot v5.4s, v31.16b, v27.16b\n" - "ext v27.16b, v27.16b, v27.16b, #0x1\n" - ".inst 0x4e82961e // sdot v30.4s, v16.16b, v2.16b\n" - "ldr q2, [x26, x28]\n" - ".inst 0x4e9b9614 // sdot v20.4s, v16.16b, v27.16b\n" - "mls v5.4s, v19.4s, v11.4s\n" - ".inst 0x4e9b95d1 // sdot v17.4s, v14.16b, v27.16b\n" - ".inst 0x4e9697f9 // sdot v25.4s, v31.16b, v22.16b\n" - "ext v22.16b, v22.16b, v22.16b, #0x1\n" - ".inst 0x4e9b97fe // sdot v30.4s, v31.16b, v27.16b\n" - ".inst 0x4e9697f4 // sdot v20.4s, v31.16b, v22.16b\n" - "sqrdmulh v5.4s, v5.4s, v9.4s\n" - "mov v16.16b, v17.16b\n .inst 0x4e9695d0 // sdot v16.4s, v14.16b, v22.16b\n" - ".inst 0x4e8795d1 // sdot v17.4s, v14.16b, v7.16b\n" - "ldr q7, [x13, x28]\n" - "mls v30.4s, v17.4s, v11.4s\n" - "mls v25.4s, v18.4s, v11.4s\n" - "mls v20.4s, v16.4s, v11.4s\n" - "and v0.16b, v5.16b, v4.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqrdmulh v30.4s, v30.4s, v9.4s\n" - "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqrdmulh v20.4s, v20.4s, v9.4s\n" - "ldr q9, [%x[params], #0x120]\n" - "sqadd v5.4s, v5.4s, v0.4s\n" - "and v16.16b, v30.16b, v4.16b\n" - "and v31.16b, v25.16b, v4.16b\n" - "and v0.16b, v20.16b, v4.16b\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "srshl v5.4s, v5.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v16.4s\n" - "ldr q16, [%x[params], #0x100]\n" - "sqadd v25.4s, v25.4s, v31.4s\n" - "ldr q31, [%x[params], #0x110]\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "ldr q0, [%x[params], #0xf0]\n" - "add v5.4s, v5.4s, v10.4s\n" - "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" - "srshl v20.4s, v20.4s, v4.4s\n" - "ldr q4, [%x[params], #0x130]\n" - "smax v5.4s, v5.4s, v13.4s\n" - "add v30.4s, v30.4s, v10.4s\n" - "add v25.4s, v25.4s, v10.4s\n" - "add v20.4s, v20.4s, v10.4s\n" - "smin v5.4s, v5.4s, v12.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" + ".inst 0x4e809706 // sdot v6.4s, v24.16b, v0.16b\n" + ".inst 0x4e829714 // sdot v20.4s, v24.16b, v2.16b\n" + "sqrdmulh v26.4s, v26.4s, v27.4s\n" + "mov v17.16b, v23.16b\n .inst 0x4e829591 // sdot v17.4s, v12.16b, v2.16b\n" + ".inst 0x4e879597 // sdot v23.4s, v12.16b, v7.16b\n" + "ldr q21, [x13, x28]\n" + "mls v6.4s, v23.4s, v16.4s\n" + "mls v25.4s, v18.4s, v16.4s\n" + "mls v20.4s, v17.4s, v16.4s\n" + "and v17.16b, v26.16b, v1.16b\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqrdmulh v6.4s, v6.4s, v27.4s\n" + "sqrdmulh v25.4s, v25.4s, v27.4s\n" + "sqrdmulh v20.4s, v20.4s, v27.4s\n" + "ldr q15, [%x[params], #0x120]\n" + "sqadd v26.4s, v26.4s, v17.4s\n" + "and v18.16b, v6.16b, v1.16b\n" + "and v22.16b, v25.16b, v1.16b\n" + "and v17.16b, v20.16b, v1.16b\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "srshl v26.4s, v26.4s, v1.4s\n" + "sqadd v6.4s, v6.4s, v18.4s\n" + "ldr q30, [%x[params], #0x100]\n" + "sqadd v25.4s, v25.4s, v22.4s\n" + "ldr q27, [%x[params], #0x110]\n" + "sqadd v20.4s, v20.4s, v17.4s\n" + "ldr q24, [%x[params], #0xf0]\n" + "add v26.4s, v26.4s, v14.4s\n" + "srshl v6.4s, v6.4s, v1.4s\n" + "srshl v25.4s, v25.4s, v1.4s\n" + "srshl v20.4s, v20.4s, v1.4s\n" + "ldr q23, [%x[params], #0x130]\n" + "smax v26.4s, v26.4s, v13.4s\n" + "add v6.4s, v6.4s, v14.4s\n" + "add v25.4s, v25.4s, v14.4s\n" + "add v20.4s, v20.4s, v14.4s\n" + "smin v26.4s, v26.4s, v11.4s\n" + "smax v6.4s, v6.4s, v13.4s\n" "smax v25.4s, v25.4s, v13.4s\n" "smax v20.4s, v20.4s, v13.4s\n" - "smin v30.4s, v30.4s, v12.4s\n" - "smin v25.4s, v25.4s, v12.4s\n" - "smin v20.4s, v20.4s, v12.4s\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "movi v19.4s, #0x0\n" - ".inst 0x4e8195d3 // sdot v19.4s, v14.16b, v1.16b\n" - ".inst 0x4e9a95d3 // sdot v19.4s, v14.16b, v26.16b\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s5, [x24, x27]\n" - "ldr q5, [%x[params], #0xe0]\n" + "smin v6.4s, v6.4s, v11.4s\n" + "smin v25.4s, v25.4s, v11.4s\n" + "smin v20.4s, v20.4s, v11.4s\n" + "uzp1 v26.16b, v26.16b, v26.16b\n" + "movi v0.4s, #0x0\n" + ".inst 0x4e8a9580 // sdot v0.4s, v12.16b, v10.16b\n" + ".inst 0x4e859580 // sdot v0.4s, v12.16b, v5.16b\n" + "uzp1 v26.16b, v26.16b, v26.16b\n" + "uzp1 v6.16b, v6.16b, v6.16b\n" + "str s26, [x25, x27]\n" + "ldr q28, [%x[params], #0xe0]\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "mov v18.16b, v19.16b\n .inst 0x4e9595d2 // sdot v18.4s, v14.16b, v21.16b\n" - "uzp1 v30.16b, v30.16b, v30.16b\n" + "mov v22.16b, v0.16b\n .inst 0x4e899596 // sdot v22.4s, v12.16b, v9.16b\n" + "uzp1 v6.16b, v6.16b, v6.16b\n" "uzp1 v25.16b, v25.16b, v25.16b\n" - "str s30, [x23, x27]\n" - ".inst 0x4e8695d3 // sdot v19.4s, v14.16b, v6.16b\n" + "str s6, [x24, x27]\n" + ".inst 0x4e889580 // sdot v0.4s, v12.16b, v8.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s25, [x22, x27]\n" - "mov v30.16b, v5.16b\n" - "str s20, [x21, x27]\n" - "mov v25.16b, v5.16b\n" - "mov v20.16b, v5.16b\n" - ".inst 0x4e869405 // sdot v5.4s, v0.16b, v6.16b\n" - ".inst 0x4e819419 // sdot v25.4s, v0.16b, v1.16b\n" - ".inst 0x4e819605 // sdot v5.4s, v16.16b, v1.16b\n" - "ext v6.16b, v6.16b, v6.16b, #0x1\n" + "str s25, [x23, x27]\n" + "mov v29.16b, v28.16b\n" + "str s20, [x22, x27]\n" + "mov v25.16b, v28.16b\n" + "mov v7.16b, v28.16b\n" + ".inst 0x4e88971c // sdot v28.4s, v24.16b, v8.16b\n" + ".inst 0x4e8a9719 // sdot v25.4s, v24.16b, v10.16b\n" + ".inst 0x4e8a97dc // sdot v28.4s, v30.16b, v10.16b\n" + "ext v8.16b, v8.16b, v8.16b, #0x1\n" "add x27, x27, #0x4\n" - "ext v1.16b, v1.16b, v1.16b, #0x1\n" + "ext v10.16b, v10.16b, v10.16b, #0x1\n" "movi v17.4s, #0x0\n" - ".inst 0x4e86941e // sdot v30.4s, v0.16b, v6.16b\n" - ".inst 0x4e819414 // sdot v20.4s, v0.16b, v1.16b\n" - ".inst 0x4e8195d1 // sdot v17.4s, v14.16b, v1.16b\n" - ".inst 0x4e9a9619 // sdot v25.4s, v16.16b, v26.16b\n" - ".inst 0x4e9a97e5 // sdot v5.4s, v31.16b, v26.16b\n" - "ext v26.16b, v26.16b, v26.16b, #0x1\n" - ".inst 0x4e81961e // sdot v30.4s, v16.16b, v1.16b\n" - "ldr q1, [x25, x28]\n" - ".inst 0x4e9a9614 // sdot v20.4s, v16.16b, v26.16b\n" - "mls v5.4s, v19.4s, v11.4s\n" - ".inst 0x4e9a95d1 // sdot v17.4s, v14.16b, v26.16b\n" - ".inst 0x4e9597f9 // sdot v25.4s, v31.16b, v21.16b\n" - "ext v21.16b, v21.16b, v21.16b, #0x1\n" - ".inst 0x4e9a97fe // sdot v30.4s, v31.16b, v26.16b\n" - ".inst 0x4e9597f4 // sdot v20.4s, v31.16b, v21.16b\n" - "sqrdmulh v5.4s, v5.4s, v9.4s\n" - "mov v16.16b, v17.16b\n .inst 0x4e9595d0 // sdot v16.4s, v14.16b, v21.16b\n" - ".inst 0x4e8695d1 // sdot v17.4s, v14.16b, v6.16b\n" - "ldr q6, [x12, x28]\n" - "mls v30.4s, v17.4s, v11.4s\n" - "mls v25.4s, v18.4s, v11.4s\n" - "mls v20.4s, v16.4s, v11.4s\n" - "and v0.16b, v5.16b, v4.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqrdmulh v30.4s, v30.4s, v9.4s\n" - "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqrdmulh v20.4s, v20.4s, v9.4s\n" - "ldr q9, [x15, x28]\n" - "sqadd v5.4s, v5.4s, v0.4s\n" - "ldp x15, x14, [%x[inptrs], #0x40]\n" - "ldr q29, [x15, x28]\n" - "ldr q28, [x14, x28]\n" - "and v16.16b, v30.16b, v4.16b\n" - "and v31.16b, v25.16b, v4.16b\n" - "and v0.16b, v20.16b, v4.16b\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "ldp x13, x12, [%x[inptrs], #0x50]\n" - "ldr q27, [x13, x28]\n" - "ldr q26, [x12, x28]\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "srshl v5.4s, v5.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v16.4s\n" - "ldr q16, [%x[params], #0x160]\n" - "sqadd v25.4s, v25.4s, v31.4s\n" - "ldr q31, [%x[params], #0x170]\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "ldr q0, [%x[params], #0x150]\n" - "add v5.4s, v5.4s, v10.4s\n" - "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" - "srshl v20.4s, v20.4s, v4.4s\n" - "ldr q4, [x10, x28]\n" - "ldp x10, x9, [%x[inptrs], #0x60]\n" - "ldr q24, [x10, x28]\n" - "ldr q23, [x9, x28]\n" - "smax v5.4s, v5.4s, v13.4s\n" - "add v30.4s, v30.4s, v10.4s\n" - "add v25.4s, v25.4s, v10.4s\n" - "add v20.4s, v20.4s, v10.4s\n" - "ldp x26, x25, [%x[inptrs], #0x70]\n" - "ldr q22, [x26, x28]\n" - "ldr q21, [x25, x28]\n" - "smin v5.4s, v5.4s, v12.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" + ".inst 0x4e88971d // sdot v29.4s, v24.16b, v8.16b\n" + ".inst 0x4e8a9707 // sdot v7.4s, v24.16b, v10.16b\n" + ".inst 0x4e8a9591 // sdot v17.4s, v12.16b, v10.16b\n" + ".inst 0x4e8597d9 // sdot v25.4s, v30.16b, v5.16b\n" + ".inst 0x4e85977c // sdot v28.4s, v27.16b, v5.16b\n" + "ext v5.16b, v5.16b, v5.16b, #0x1\n" + ".inst 0x4e8a97dd // sdot v29.4s, v30.16b, v10.16b\n" + "ldr q10, [x21, x28]\n" + ".inst 0x4e8597c7 // sdot v7.4s, v30.16b, v5.16b\n" + "mls v28.4s, v0.4s, v16.4s\n" + ".inst 0x4e859591 // sdot v17.4s, v12.16b, v5.16b\n" + ".inst 0x4e899779 // sdot v25.4s, v27.16b, v9.16b\n" + "ext v9.16b, v9.16b, v9.16b, #0x1\n" + ".inst 0x4e85977d // sdot v29.4s, v27.16b, v5.16b\n" + ".inst 0x4e899767 // sdot v7.4s, v27.16b, v9.16b\n" + "sqrdmulh v28.4s, v28.4s, v15.4s\n" + "mov v18.16b, v17.16b\n .inst 0x4e899592 // sdot v18.4s, v12.16b, v9.16b\n" + ".inst 0x4e889591 // sdot v17.4s, v12.16b, v8.16b\n" + "ldr q8, [x12, x28]\n" + "mls v29.4s, v17.4s, v16.4s\n" + "mls v25.4s, v22.4s, v16.4s\n" + "mls v7.4s, v18.4s, v16.4s\n" + "and v17.16b, v28.16b, v23.16b\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqrdmulh v29.4s, v29.4s, v15.4s\n" + "sqrdmulh v25.4s, v25.4s, v15.4s\n" + "sqrdmulh v7.4s, v7.4s, v15.4s\n" + "ldr q15, [x15, x28]\n" + "sqadd v28.4s, v28.4s, v17.4s\n" + "ldp x21, x20, [%x[inptrs], #0x40]\n" + "ldr q22, [x21, x28]\n" + "ldr q3, [x20, x28]\n" + "and v24.16b, v29.16b, v23.16b\n" + "and v20.16b, v25.16b, v23.16b\n" + "and v17.16b, v7.16b, v23.16b\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "ldp x21, x20, [%x[inptrs], #0x50]\n" + "ldr q2, [x21, x28]\n" + "ldr q5, [x20, x28]\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "srshl v28.4s, v28.4s, v23.4s\n" + "sqadd v29.4s, v29.4s, v24.4s\n" + "ldr q6, [%x[params], #0x160]\n" + "sqadd v25.4s, v25.4s, v20.4s\n" + "ldr q20, [%x[params], #0x170]\n" + "sqadd v7.4s, v7.4s, v17.4s\n" + "ldr q1, [%x[params], #0x150]\n" + "add v28.4s, v28.4s, v14.4s\n" + "srshl v29.4s, v29.4s, v23.4s\n" + "srshl v25.4s, v25.4s, v23.4s\n" + "srshl v7.4s, v7.4s, v23.4s\n" + "ldr q26, [x10, x28]\n" + "ldp x21, x20, [%x[inptrs], #0x60]\n" + "ldr q27, [x21, x28]\n" + "ldr q30, [x20, x28]\n" + "smax v28.4s, v28.4s, v13.4s\n" + "add v29.4s, v29.4s, v14.4s\n" + "add v25.4s, v25.4s, v14.4s\n" + "add v7.4s, v7.4s, v14.4s\n" + "ldp x21, x20, [%x[inptrs], #0x70]\n" + "ldr q23, [x21, x28]\n" + "ldr q9, [x20, x28]\n" + "smin v28.4s, v28.4s, v11.4s\n" + "smax v29.4s, v29.4s, v13.4s\n" "ldp x15, x14, [%x[inptrs], #0x0]\n" "smax v25.4s, v25.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" + "smax v7.4s, v7.4s, v13.4s\n" "ldp x13, x12, [%x[inptrs], #0x10]\n" "ldp x10, x9, [%x[inptrs], #0x20]\n" - "smin v30.4s, v30.4s, v12.4s\n" - "smin v25.4s, v25.4s, v12.4s\n" - "ldp x26, x25, [%x[inptrs], #0x30]\n" - "smin v20.4s, v20.4s, v12.4s\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s5, [x24, x27]\n" + "smin v29.4s, v29.4s, v11.4s\n" + "smin v25.4s, v25.4s, v11.4s\n" + "ldp x26, x21, [%x[inptrs], #0x30]\n" + "smin v7.4s, v7.4s, v11.4s\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "str s28, [x25, x27]\n" "uzp1 v25.16b, v25.16b, v25.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "zip2 v5.16b, v9.16b, v7.16b\n" - "zip1 v9.16b, v9.16b, v7.16b\n" - "zip1 v7.16b, v8.16b, v6.16b\n" - "zip2 v6.16b, v8.16b, v6.16b\n" - "uzp1 v30.16b, v30.16b, v30.16b\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" + "zip2 v17.16b, v15.16b, v21.16b\n" + "zip1 v15.16b, v15.16b, v21.16b\n" + "zip1 v18.16b, v31.16b, v8.16b\n" + "zip2 v8.16b, v31.16b, v8.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" "uzp1 v25.16b, v25.16b, v25.16b\n" - "str s30, [x23, x27]\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s25, [x22, x27]\n" - "zip2 v8.16b, v9.16b, v7.16b\n" - "str s20, [x21, x27]\n" - "zip1 v9.16b, v9.16b, v7.16b\n" - "zip1 v7.16b, v5.16b, v6.16b\n" + "str s29, [x24, x27]\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" + "str s25, [x23, x27]\n" + "zip2 v25.16b, v15.16b, v18.16b\n" + "str s7, [x22, x27]\n" + "zip1 v15.16b, v15.16b, v18.16b\n" + "zip1 v7.16b, v17.16b, v8.16b\n" "add x27, x27, #0x4\n" - "zip2 v6.16b, v5.16b, v6.16b\n" - "ldr q5, [%x[params], #0x140]\n" - "zip2 v30.16b, v4.16b, v2.16b\n" + "zip2 v8.16b, v17.16b, v8.16b\n" + "ldr q31, [%x[params], #0x140]\n" + "zip2 v29.16b, v26.16b, v19.16b\n" "add %x[params], %x[params], #0x180\n" - "zip1 v4.16b, v4.16b, v2.16b\n" - "zip1 v2.16b, v3.16b, v1.16b\n" - "zip2 v1.16b, v3.16b, v1.16b\n" - "zip2 v25.16b, v29.16b, v27.16b\n" - "zip1 v29.16b, v29.16b, v27.16b\n" - "zip1 v27.16b, v28.16b, v26.16b\n" - "zip2 v26.16b, v28.16b, v26.16b\n" - "zip2 v20.16b, v24.16b, v22.16b\n" - "zip1 v24.16b, v24.16b, v22.16b\n" - "zip1 v22.16b, v23.16b, v21.16b\n" - "zip2 v21.16b, v23.16b, v21.16b\n" - "zip2 v3.16b, v4.16b, v2.16b\n" - "zip1 v4.16b, v4.16b, v2.16b\n" - "zip1 v2.16b, v30.16b, v1.16b\n" - "zip2 v1.16b, v30.16b, v1.16b\n" - "zip2 v28.16b, v29.16b, v27.16b\n" - "zip1 v29.16b, v29.16b, v27.16b\n" - "zip1 v27.16b, v25.16b, v26.16b\n" - "zip2 v26.16b, v25.16b, v26.16b\n" - "zip2 v23.16b, v24.16b, v22.16b\n" - "zip1 v24.16b, v24.16b, v22.16b\n" - "zip1 v22.16b, v20.16b, v21.16b\n" - "zip2 v21.16b, v20.16b, v21.16b\n" - "mov v30.16b, v5.16b\n" - "mov v25.16b, v5.16b\n" - "mov v20.16b, v5.16b\n" + "zip1 v26.16b, v26.16b, v19.16b\n" + "zip1 v28.16b, v4.16b, v10.16b\n" + "zip2 v10.16b, v4.16b, v10.16b\n" + "zip2 v24.16b, v22.16b, v2.16b\n" + "zip1 v22.16b, v22.16b, v2.16b\n" + "zip1 v21.16b, v3.16b, v5.16b\n" + "zip2 v5.16b, v3.16b, v5.16b\n" + "zip2 v18.16b, v27.16b, v23.16b\n" + "zip1 v27.16b, v27.16b, v23.16b\n" + "zip1 v17.16b, v30.16b, v9.16b\n" + "zip2 v9.16b, v30.16b, v9.16b\n" + "zip2 v23.16b, v26.16b, v28.16b\n" + "zip1 v26.16b, v26.16b, v28.16b\n" + "zip1 v3.16b, v29.16b, v10.16b\n" + "zip2 v10.16b, v29.16b, v10.16b\n" + "zip2 v19.16b, v22.16b, v21.16b\n" + "zip1 v22.16b, v22.16b, v21.16b\n" + "zip1 v0.16b, v24.16b, v5.16b\n" + "zip2 v5.16b, v24.16b, v5.16b\n" + "zip2 v24.16b, v27.16b, v17.16b\n" + "zip1 v27.16b, v27.16b, v17.16b\n" + "zip1 v2.16b, v18.16b, v9.16b\n" + "zip2 v9.16b, v18.16b, v9.16b\n" + "mov v30.16b, v31.16b\n" + "mov v29.16b, v31.16b\n" + "mov v28.16b, v31.16b\n" "bgt 1b\n" "2:" // Detached iteration - "movi v19.4s, #0x0\n" - ".inst 0x4e8495d3 // sdot v19.4s, v14.16b, v4.16b\n" - ".inst 0x4e899405 // sdot v5.4s, v0.16b, v9.16b\n" + "movi v21.4s, #0x0\n" + ".inst 0x4e9a9595 // sdot v21.4s, v12.16b, v26.16b\n" + ".inst 0x4e8f943f // sdot v31.4s, v1.16b, v15.16b\n" "tst %x[n_channels], #0xf\n" - ".inst 0x4e9d95d3 // sdot v19.4s, v14.16b, v29.16b\n" - ".inst 0x4e849419 // sdot v25.4s, v0.16b, v4.16b\n" + ".inst 0x4e969595 // sdot v21.4s, v12.16b, v22.16b\n" + ".inst 0x4e9a943d // sdot v29.4s, v1.16b, v26.16b\n" + "movi v18.4s, #0x0\n" "add x28, x28, #0x10\n" - ".inst 0x4e849605 // sdot v5.4s, v16.16b, v4.16b\n" - "ext v4.16b, v4.16b, v4.16b, #0x1\n" - "mov v18.16b, v19.16b\n .inst 0x4e9895d2 // sdot v18.4s, v14.16b, v24.16b\n" - ".inst 0x4e8995d3 // sdot v19.4s, v14.16b, v9.16b\n" - "ext v9.16b, v9.16b, v9.16b, #0x1\n" - ".inst 0x4e9d9619 // sdot v25.4s, v16.16b, v29.16b\n" - ".inst 0x4e9d97e5 // sdot v5.4s, v31.16b, v29.16b\n" - "ext v29.16b, v29.16b, v29.16b, #0x1\n" - ".inst 0x4e89941e // sdot v30.4s, v0.16b, v9.16b\n" - ".inst 0x4e849414 // sdot v20.4s, v0.16b, v4.16b\n" - "movi v17.4s, #0x0\n" - ".inst 0x4e8495d1 // sdot v17.4s, v14.16b, v4.16b\n" - ".inst 0x4e9d95d1 // sdot v17.4s, v14.16b, v29.16b\n" - ".inst 0x4e9897f9 // sdot v25.4s, v31.16b, v24.16b\n" - "ext v24.16b, v24.16b, v24.16b, #0x1\n" - ".inst 0x4e84961e // sdot v30.4s, v16.16b, v4.16b\n" + ".inst 0x4e9a94df // sdot v31.4s, v6.16b, v26.16b\n" + "ext v26.16b, v26.16b, v26.16b, #0x1\n" + "mov v17.16b, v21.16b\n .inst 0x4e9b9591 // sdot v17.4s, v12.16b, v27.16b\n" + ".inst 0x4e8f9595 // sdot v21.4s, v12.16b, v15.16b\n" + "ext v15.16b, v15.16b, v15.16b, #0x1\n" + ".inst 0x4e9a9592 // sdot v18.4s, v12.16b, v26.16b\n" + ".inst 0x4e9694dd // sdot v29.4s, v6.16b, v22.16b\n" + ".inst 0x4e96969f // sdot v31.4s, v20.16b, v22.16b\n" + "ext v22.16b, v22.16b, v22.16b, #0x1\n" + ".inst 0x4e8f943e // sdot v30.4s, v1.16b, v15.16b\n" + ".inst 0x4e9a943c // sdot v28.4s, v1.16b, v26.16b\n" + "mls v31.4s, v21.4s, v16.4s\n" + ".inst 0x4e969592 // sdot v18.4s, v12.16b, v22.16b\n" + ".inst 0x4e9b969d // sdot v29.4s, v20.16b, v27.16b\n" + "ext v27.16b, v27.16b, v27.16b, #0x1\n" + ".inst 0x4e9a94de // sdot v30.4s, v6.16b, v26.16b\n" "ldr q4, [%x[params], #0x10]\n" - ".inst 0x4e9d9614 // sdot v20.4s, v16.16b, v29.16b\n" - "mls v5.4s, v19.4s, v11.4s\n" - "mov v16.16b, v17.16b\n .inst 0x4e9895d0 // sdot v16.4s, v14.16b, v24.16b\n" - ".inst 0x4e8995d1 // sdot v17.4s, v14.16b, v9.16b\n" - "ldr q9, [%x[params], #0x0]\n" - "sqrdmulh v5.4s, v5.4s, v9.4s\n" - ".inst 0x4e9d97fe // sdot v30.4s, v31.16b, v29.16b\n" - ".inst 0x4e9897f4 // sdot v20.4s, v31.16b, v24.16b\n" - "mls v30.4s, v17.4s, v11.4s\n" - "mls v25.4s, v18.4s, v11.4s\n" - "mls v20.4s, v16.4s, v11.4s\n" - "and v0.16b, v5.16b, v4.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqrdmulh v30.4s, v30.4s, v9.4s\n" - "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqrdmulh v20.4s, v20.4s, v9.4s\n" - "ldr q9, [%x[params], #0x60]\n" - "sqadd v5.4s, v5.4s, v0.4s\n" - "and v16.16b, v30.16b, v4.16b\n" - "and v31.16b, v25.16b, v4.16b\n" - "and v0.16b, v20.16b, v4.16b\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "srshl v5.4s, v5.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v16.4s\n" - "ldr q16, [%x[params], #0x40]\n" - "sqadd v25.4s, v25.4s, v31.4s\n" - "ldr q31, [%x[params], #0x50]\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "ldr q0, [%x[params], #0x30]\n" - "add v5.4s, v5.4s, v10.4s\n" + ".inst 0x4e9694dc // sdot v28.4s, v6.16b, v22.16b\n" + "mls v29.4s, v17.4s, v16.4s\n" + "mov v21.16b, v18.16b\n .inst 0x4e9b9595 // sdot v21.4s, v12.16b, v27.16b\n" + ".inst 0x4e8f9592 // sdot v18.4s, v12.16b, v15.16b\n" + "ldr q17, [%x[params], #0x0]\n" + "sqrdmulh v31.4s, v31.4s, v17.4s\n" + ".inst 0x4e96969e // sdot v30.4s, v20.16b, v22.16b\n" + ".inst 0x4e9b969c // sdot v28.4s, v20.16b, v27.16b\n" + "mls v30.4s, v18.4s, v16.4s\n" + "mls v28.4s, v21.4s, v16.4s\n" + "and v27.16b, v31.16b, v4.16b\n" + "sshr v27.4s, v27.4s, #0x1f\n" + "sqrdmulh v30.4s, v30.4s, v17.4s\n" + "sqrdmulh v29.4s, v29.4s, v17.4s\n" + "sqrdmulh v28.4s, v28.4s, v17.4s\n" + "ldr q15, [%x[params], #0x60]\n" + "sqadd v31.4s, v31.4s, v27.4s\n" + "and v20.16b, v30.16b, v4.16b\n" + "and v18.16b, v29.16b, v4.16b\n" + "and v17.16b, v28.16b, v4.16b\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "srshl v31.4s, v31.4s, v4.4s\n" + "sqadd v30.4s, v30.4s, v20.4s\n" + "ldr q27, [%x[params], #0x40]\n" + "sqadd v29.4s, v29.4s, v18.4s\n" + "ldr q26, [%x[params], #0x50]\n" + "sqadd v28.4s, v28.4s, v17.4s\n" + "ldr q6, [%x[params], #0x30]\n" + "add v31.4s, v31.4s, v14.4s\n" "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" - "srshl v20.4s, v20.4s, v4.4s\n" + "srshl v29.4s, v29.4s, v4.4s\n" + "srshl v28.4s, v28.4s, v4.4s\n" "ldr q4, [%x[params], #0x70]\n" - "smax v5.4s, v5.4s, v13.4s\n" - "add v30.4s, v30.4s, v10.4s\n" - "add v25.4s, v25.4s, v10.4s\n" - "add v20.4s, v20.4s, v10.4s\n" - "smin v5.4s, v5.4s, v12.4s\n" + "smax v31.4s, v31.4s, v13.4s\n" + "add v30.4s, v30.4s, v14.4s\n" + "add v29.4s, v29.4s, v14.4s\n" + "add v28.4s, v28.4s, v14.4s\n" + "smin v31.4s, v31.4s, v11.4s\n" "smax v30.4s, v30.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smin v30.4s, v30.4s, v12.4s\n" - "smin v25.4s, v25.4s, v12.4s\n" - "smin v20.4s, v20.4s, v12.4s\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "movi v19.4s, #0x0\n" - ".inst 0x4e8395d3 // sdot v19.4s, v14.16b, v3.16b\n" - ".inst 0x4e9c95d3 // sdot v19.4s, v14.16b, v28.16b\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" + "smax v29.4s, v29.4s, v13.4s\n" + "smax v28.4s, v28.4s, v13.4s\n" + "smin v30.4s, v30.4s, v11.4s\n" + "smin v29.4s, v29.4s, v11.4s\n" + "smin v28.4s, v28.4s, v11.4s\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" + "movi v1.4s, #0x0\n" + ".inst 0x4e979581 // sdot v1.4s, v12.16b, v23.16b\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s5, [x24, x27]\n" - "ldr q5, [%x[params], #0x20]\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "mov v18.16b, v19.16b\n .inst 0x4e9795d2 // sdot v18.4s, v14.16b, v23.16b\n" + "str s31, [x25, x27]\n" + "ldr q31, [%x[params], #0x20]\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + ".inst 0x4e939581 // sdot v1.4s, v12.16b, v19.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "str s30, [x23, x27]\n" - ".inst 0x4e8895d3 // sdot v19.4s, v14.16b, v8.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s25, [x22, x27]\n" - "mov v30.16b, v5.16b\n" - "str s20, [x21, x27]\n" - "mov v25.16b, v5.16b\n" - "mov v20.16b, v5.16b\n" - ".inst 0x4e889405 // sdot v5.4s, v0.16b, v8.16b\n" - ".inst 0x4e839419 // sdot v25.4s, v0.16b, v3.16b\n" - ".inst 0x4e839605 // sdot v5.4s, v16.16b, v3.16b\n" - "ext v8.16b, v8.16b, v8.16b, #0x1\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "str s30, [x24, x27]\n" + "mov v22.16b, v1.16b\n .inst 0x4e989596 // sdot v22.4s, v12.16b, v24.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + "str s29, [x23, x27]\n" + "mov v29.16b, v31.16b\n" + ".inst 0x4e999581 // sdot v1.4s, v12.16b, v25.16b\n" + "str s28, [x22, x27]\n" + "mov v21.16b, v31.16b\n" + "mov v20.16b, v31.16b\n" + ".inst 0x4e9994df // sdot v31.4s, v6.16b, v25.16b\n" + ".inst 0x4e9794d5 // sdot v21.4s, v6.16b, v23.16b\n" + ".inst 0x4e97977f // sdot v31.4s, v27.16b, v23.16b\n" + "ext v25.16b, v25.16b, v25.16b, #0x1\n" "add x27, x27, #0x4\n" - "ext v3.16b, v3.16b, v3.16b, #0x1\n" - "movi v17.4s, #0x0\n" - ".inst 0x4e88941e // sdot v30.4s, v0.16b, v8.16b\n" - ".inst 0x4e839414 // sdot v20.4s, v0.16b, v3.16b\n" - ".inst 0x4e8395d1 // sdot v17.4s, v14.16b, v3.16b\n" - ".inst 0x4e9c9619 // sdot v25.4s, v16.16b, v28.16b\n" - ".inst 0x4e9c97e5 // sdot v5.4s, v31.16b, v28.16b\n" - "ext v28.16b, v28.16b, v28.16b, #0x1\n" - ".inst 0x4e83961e // sdot v30.4s, v16.16b, v3.16b\n" - ".inst 0x4e9c9614 // sdot v20.4s, v16.16b, v28.16b\n" - "mls v5.4s, v19.4s, v11.4s\n" - ".inst 0x4e9c95d1 // sdot v17.4s, v14.16b, v28.16b\n" - ".inst 0x4e9797f9 // sdot v25.4s, v31.16b, v23.16b\n" "ext v23.16b, v23.16b, v23.16b, #0x1\n" - ".inst 0x4e9c97fe // sdot v30.4s, v31.16b, v28.16b\n" - ".inst 0x4e9797f4 // sdot v20.4s, v31.16b, v23.16b\n" - "sqrdmulh v5.4s, v5.4s, v9.4s\n" - "mov v16.16b, v17.16b\n .inst 0x4e9795d0 // sdot v16.4s, v14.16b, v23.16b\n" - ".inst 0x4e8895d1 // sdot v17.4s, v14.16b, v8.16b\n" - "mls v30.4s, v17.4s, v11.4s\n" - "mls v25.4s, v18.4s, v11.4s\n" - "mls v20.4s, v16.4s, v11.4s\n" - "and v0.16b, v5.16b, v4.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqrdmulh v30.4s, v30.4s, v9.4s\n" - "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqrdmulh v20.4s, v20.4s, v9.4s\n" - "ldr q9, [%x[params], #0xc0]\n" - "sqadd v5.4s, v5.4s, v0.4s\n" - "and v16.16b, v30.16b, v4.16b\n" - "and v31.16b, v25.16b, v4.16b\n" - "and v0.16b, v20.16b, v4.16b\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "srshl v5.4s, v5.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v16.4s\n" - "ldr q16, [%x[params], #0xa0]\n" - "sqadd v25.4s, v25.4s, v31.4s\n" - "ldr q31, [%x[params], #0xb0]\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "ldr q0, [%x[params], #0x90]\n" - "add v5.4s, v5.4s, v10.4s\n" - "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" + "movi v18.4s, #0x0\n" + ".inst 0x4e9994dd // sdot v29.4s, v6.16b, v25.16b\n" + ".inst 0x4e9794d4 // sdot v20.4s, v6.16b, v23.16b\n" + ".inst 0x4e979592 // sdot v18.4s, v12.16b, v23.16b\n" + ".inst 0x4e939775 // sdot v21.4s, v27.16b, v19.16b\n" + ".inst 0x4e93975f // sdot v31.4s, v26.16b, v19.16b\n" + "ext v19.16b, v19.16b, v19.16b, #0x1\n" + ".inst 0x4e97977d // sdot v29.4s, v27.16b, v23.16b\n" + ".inst 0x4e939774 // sdot v20.4s, v27.16b, v19.16b\n" + "mls v31.4s, v1.4s, v16.4s\n" + ".inst 0x4e939592 // sdot v18.4s, v12.16b, v19.16b\n" + ".inst 0x4e989755 // sdot v21.4s, v26.16b, v24.16b\n" + "ext v24.16b, v24.16b, v24.16b, #0x1\n" + ".inst 0x4e93975d // sdot v29.4s, v26.16b, v19.16b\n" + ".inst 0x4e989754 // sdot v20.4s, v26.16b, v24.16b\n" + "sqrdmulh v31.4s, v31.4s, v15.4s\n" + "mov v17.16b, v18.16b\n .inst 0x4e989591 // sdot v17.4s, v12.16b, v24.16b\n" + ".inst 0x4e999592 // sdot v18.4s, v12.16b, v25.16b\n" + "mls v29.4s, v18.4s, v16.4s\n" + "mls v21.4s, v22.4s, v16.4s\n" + "mls v20.4s, v17.4s, v16.4s\n" + "and v17.16b, v31.16b, v4.16b\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqrdmulh v29.4s, v29.4s, v15.4s\n" + "sqrdmulh v21.4s, v21.4s, v15.4s\n" + "sqrdmulh v20.4s, v20.4s, v15.4s\n" + "ldr q27, [%x[params], #0xc0]\n" + "sqadd v31.4s, v31.4s, v17.4s\n" + "and v19.16b, v29.16b, v4.16b\n" + "and v18.16b, v21.16b, v4.16b\n" + "and v17.16b, v20.16b, v4.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "srshl v31.4s, v31.4s, v4.4s\n" + "sqadd v29.4s, v29.4s, v19.4s\n" + "ldr q26, [%x[params], #0xa0]\n" + "sqadd v21.4s, v21.4s, v18.4s\n" + "ldr q25, [%x[params], #0xb0]\n" + "sqadd v20.4s, v20.4s, v17.4s\n" + "ldr q24, [%x[params], #0x90]\n" + "add v31.4s, v31.4s, v14.4s\n" + "srshl v29.4s, v29.4s, v4.4s\n" + "srshl v21.4s, v21.4s, v4.4s\n" "srshl v20.4s, v20.4s, v4.4s\n" - "ldr q4, [%x[params], #0xd0]\n" - "smax v5.4s, v5.4s, v13.4s\n" - "add v30.4s, v30.4s, v10.4s\n" - "add v25.4s, v25.4s, v10.4s\n" - "add v20.4s, v20.4s, v10.4s\n" - "smin v5.4s, v5.4s, v12.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" + "ldr q1, [%x[params], #0xd0]\n" + "smax v31.4s, v31.4s, v13.4s\n" + "add v29.4s, v29.4s, v14.4s\n" + "add v21.4s, v21.4s, v14.4s\n" + "add v20.4s, v20.4s, v14.4s\n" + "smin v31.4s, v31.4s, v11.4s\n" + "smax v29.4s, v29.4s, v13.4s\n" + "smax v21.4s, v21.4s, v13.4s\n" "smax v20.4s, v20.4s, v13.4s\n" - "smin v30.4s, v30.4s, v12.4s\n" - "smin v25.4s, v25.4s, v12.4s\n" - "smin v20.4s, v20.4s, v12.4s\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "movi v19.4s, #0x0\n" - ".inst 0x4e8295d3 // sdot v19.4s, v14.16b, v2.16b\n" - ".inst 0x4e9b95d3 // sdot v19.4s, v14.16b, v27.16b\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s5, [x24, x27]\n" - "ldr q5, [%x[params], #0x80]\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" + "smin v29.4s, v29.4s, v11.4s\n" + "smin v21.4s, v21.4s, v11.4s\n" + "smin v20.4s, v20.4s, v11.4s\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" + "movi v23.4s, #0x0\n" + ".inst 0x4e839597 // sdot v23.4s, v12.16b, v3.16b\n" + ".inst 0x4e809597 // sdot v23.4s, v12.16b, v0.16b\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "str s31, [x25, x27]\n" + "ldr q31, [%x[params], #0x80]\n" + "uzp1 v21.16b, v21.16b, v21.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "mov v18.16b, v19.16b\n .inst 0x4e9695d2 // sdot v18.4s, v14.16b, v22.16b\n" - "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "str s30, [x23, x27]\n" - ".inst 0x4e8795d3 // sdot v19.4s, v14.16b, v7.16b\n" + "mov v22.16b, v23.16b\n .inst 0x4e829596 // sdot v22.4s, v12.16b, v2.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "uzp1 v21.16b, v21.16b, v21.16b\n" + "str s29, [x24, x27]\n" + ".inst 0x4e879597 // sdot v23.4s, v12.16b, v7.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s25, [x22, x27]\n" - "mov v30.16b, v5.16b\n" - "str s20, [x21, x27]\n" - "mov v25.16b, v5.16b\n" - "mov v20.16b, v5.16b\n" - ".inst 0x4e879405 // sdot v5.4s, v0.16b, v7.16b\n" - ".inst 0x4e829419 // sdot v25.4s, v0.16b, v2.16b\n" - ".inst 0x4e829605 // sdot v5.4s, v16.16b, v2.16b\n" + "str s21, [x23, x27]\n" + "mov v21.16b, v31.16b\n" + "str s20, [x22, x27]\n" + "mov v4.16b, v31.16b\n" + "mov v20.16b, v31.16b\n" + ".inst 0x4e87971f // sdot v31.4s, v24.16b, v7.16b\n" + ".inst 0x4e839704 // sdot v4.4s, v24.16b, v3.16b\n" + ".inst 0x4e83975f // sdot v31.4s, v26.16b, v3.16b\n" "ext v7.16b, v7.16b, v7.16b, #0x1\n" "add x27, x27, #0x4\n" + "ext v3.16b, v3.16b, v3.16b, #0x1\n" + "movi v18.4s, #0x0\n" + ".inst 0x4e879715 // sdot v21.4s, v24.16b, v7.16b\n" + ".inst 0x4e839714 // sdot v20.4s, v24.16b, v3.16b\n" + ".inst 0x4e839592 // sdot v18.4s, v12.16b, v3.16b\n" + ".inst 0x4e809744 // sdot v4.4s, v26.16b, v0.16b\n" + ".inst 0x4e80973f // sdot v31.4s, v25.16b, v0.16b\n" + "ext v0.16b, v0.16b, v0.16b, #0x1\n" + ".inst 0x4e839755 // sdot v21.4s, v26.16b, v3.16b\n" + ".inst 0x4e809754 // sdot v20.4s, v26.16b, v0.16b\n" + "mls v31.4s, v23.4s, v16.4s\n" + ".inst 0x4e809592 // sdot v18.4s, v12.16b, v0.16b\n" + ".inst 0x4e829724 // sdot v4.4s, v25.16b, v2.16b\n" "ext v2.16b, v2.16b, v2.16b, #0x1\n" - "movi v17.4s, #0x0\n" - ".inst 0x4e87941e // sdot v30.4s, v0.16b, v7.16b\n" - ".inst 0x4e829414 // sdot v20.4s, v0.16b, v2.16b\n" - ".inst 0x4e8295d1 // sdot v17.4s, v14.16b, v2.16b\n" - ".inst 0x4e9b9619 // sdot v25.4s, v16.16b, v27.16b\n" - ".inst 0x4e9b97e5 // sdot v5.4s, v31.16b, v27.16b\n" - "ext v27.16b, v27.16b, v27.16b, #0x1\n" - ".inst 0x4e82961e // sdot v30.4s, v16.16b, v2.16b\n" - ".inst 0x4e9b9614 // sdot v20.4s, v16.16b, v27.16b\n" - "mls v5.4s, v19.4s, v11.4s\n" - ".inst 0x4e9b95d1 // sdot v17.4s, v14.16b, v27.16b\n" - ".inst 0x4e9697f9 // sdot v25.4s, v31.16b, v22.16b\n" - "ext v22.16b, v22.16b, v22.16b, #0x1\n" - ".inst 0x4e9b97fe // sdot v30.4s, v31.16b, v27.16b\n" - ".inst 0x4e9697f4 // sdot v20.4s, v31.16b, v22.16b\n" - "sqrdmulh v5.4s, v5.4s, v9.4s\n" - "mov v16.16b, v17.16b\n .inst 0x4e9695d0 // sdot v16.4s, v14.16b, v22.16b\n" - ".inst 0x4e8795d1 // sdot v17.4s, v14.16b, v7.16b\n" - "mls v30.4s, v17.4s, v11.4s\n" - "mls v25.4s, v18.4s, v11.4s\n" - "mls v20.4s, v16.4s, v11.4s\n" - "and v0.16b, v5.16b, v4.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqrdmulh v30.4s, v30.4s, v9.4s\n" - "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqrdmulh v20.4s, v20.4s, v9.4s\n" - "ldr q9, [%x[params], #0x120]\n" - "sqadd v5.4s, v5.4s, v0.4s\n" - "and v16.16b, v30.16b, v4.16b\n" - "and v31.16b, v25.16b, v4.16b\n" - "and v0.16b, v20.16b, v4.16b\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "srshl v5.4s, v5.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v16.4s\n" - "ldr q16, [%x[params], #0x100]\n" - "sqadd v25.4s, v25.4s, v31.4s\n" - "ldr q31, [%x[params], #0x110]\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "ldr q0, [%x[params], #0xf0]\n" - "add v5.4s, v5.4s, v10.4s\n" - "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" - "srshl v20.4s, v20.4s, v4.4s\n" - "ldr q4, [%x[params], #0x130]\n" - "smax v5.4s, v5.4s, v13.4s\n" - "add v30.4s, v30.4s, v10.4s\n" - "add v25.4s, v25.4s, v10.4s\n" - "add v20.4s, v20.4s, v10.4s\n" - "smin v5.4s, v5.4s, v12.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" + ".inst 0x4e809735 // sdot v21.4s, v25.16b, v0.16b\n" + ".inst 0x4e829734 // sdot v20.4s, v25.16b, v2.16b\n" + "sqrdmulh v31.4s, v31.4s, v27.4s\n" + "mov v17.16b, v18.16b\n .inst 0x4e829591 // sdot v17.4s, v12.16b, v2.16b\n" + ".inst 0x4e879592 // sdot v18.4s, v12.16b, v7.16b\n" + "mls v21.4s, v18.4s, v16.4s\n" + "mls v4.4s, v22.4s, v16.4s\n" + "mls v20.4s, v17.4s, v16.4s\n" + "and v17.16b, v31.16b, v1.16b\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqrdmulh v21.4s, v21.4s, v27.4s\n" + "sqrdmulh v4.4s, v4.4s, v27.4s\n" + "sqrdmulh v20.4s, v20.4s, v27.4s\n" + "ldr q30, [%x[params], #0x120]\n" + "sqadd v31.4s, v31.4s, v17.4s\n" + "and v19.16b, v21.16b, v1.16b\n" + "and v18.16b, v4.16b, v1.16b\n" + "and v17.16b, v20.16b, v1.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "srshl v31.4s, v31.4s, v1.4s\n" + "sqadd v21.4s, v21.4s, v19.4s\n" + "ldr q29, [%x[params], #0x100]\n" + "sqadd v4.4s, v4.4s, v18.4s\n" + "ldr q28, [%x[params], #0x110]\n" + "sqadd v20.4s, v20.4s, v17.4s\n" + "ldr q27, [%x[params], #0xf0]\n" + "add v31.4s, v31.4s, v14.4s\n" + "srshl v21.4s, v21.4s, v1.4s\n" + "srshl v4.4s, v4.4s, v1.4s\n" + "srshl v20.4s, v20.4s, v1.4s\n" + "ldr q26, [%x[params], #0x130]\n" + "smax v31.4s, v31.4s, v13.4s\n" + "add v21.4s, v21.4s, v14.4s\n" + "add v4.4s, v4.4s, v14.4s\n" + "add v20.4s, v20.4s, v14.4s\n" + "smin v31.4s, v31.4s, v11.4s\n" + "smax v21.4s, v21.4s, v13.4s\n" + "smax v4.4s, v4.4s, v13.4s\n" "smax v20.4s, v20.4s, v13.4s\n" - "smin v30.4s, v30.4s, v12.4s\n" - "smin v25.4s, v25.4s, v12.4s\n" - "smin v20.4s, v20.4s, v12.4s\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "movi v19.4s, #0x0\n" - ".inst 0x4e8195d3 // sdot v19.4s, v14.16b, v1.16b\n" - ".inst 0x4e9a95d3 // sdot v19.4s, v14.16b, v26.16b\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s5, [x24, x27]\n" - "ldr q5, [%x[params], #0xe0]\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" + "smin v21.4s, v21.4s, v11.4s\n" + "smin v4.4s, v4.4s, v11.4s\n" + "smin v20.4s, v20.4s, v11.4s\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" + "movi v25.4s, #0x0\n" + ".inst 0x4e8a9599 // sdot v25.4s, v12.16b, v10.16b\n" + ".inst 0x4e859599 // sdot v25.4s, v12.16b, v5.16b\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" + "uzp1 v21.16b, v21.16b, v21.16b\n" + "str s31, [x25, x27]\n" + "ldr q24, [%x[params], #0xe0]\n" + "uzp1 v4.16b, v4.16b, v4.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "mov v18.16b, v19.16b\n .inst 0x4e9595d2 // sdot v18.4s, v14.16b, v21.16b\n" + "mov v23.16b, v25.16b\n .inst 0x4e899597 // sdot v23.4s, v12.16b, v9.16b\n" "add %x[params], %x[params], #0x140\n" - "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "str s30, [x23, x27]\n" - ".inst 0x4e8695d3 // sdot v19.4s, v14.16b, v6.16b\n" + "uzp1 v21.16b, v21.16b, v21.16b\n" + "uzp1 v4.16b, v4.16b, v4.16b\n" + "str s21, [x24, x27]\n" + ".inst 0x4e889599 // sdot v25.4s, v12.16b, v8.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s25, [x22, x27]\n" - "mov v30.16b, v5.16b\n" - "str s20, [x21, x27]\n" - "mov v25.16b, v5.16b\n" - "mov v20.16b, v5.16b\n" - ".inst 0x4e869405 // sdot v5.4s, v0.16b, v6.16b\n" - ".inst 0x4e819419 // sdot v25.4s, v0.16b, v1.16b\n" - ".inst 0x4e819605 // sdot v5.4s, v16.16b, v1.16b\n" - "ext v6.16b, v6.16b, v6.16b, #0x1\n" + "str s4, [x23, x27]\n" + "mov v22.16b, v24.16b\n" + "str s20, [x22, x27]\n" + "mov v21.16b, v24.16b\n" + "mov v20.16b, v24.16b\n" + ".inst 0x4e889778 // sdot v24.4s, v27.16b, v8.16b\n" + ".inst 0x4e8a9775 // sdot v21.4s, v27.16b, v10.16b\n" + ".inst 0x4e8a97b8 // sdot v24.4s, v29.16b, v10.16b\n" + "ext v8.16b, v8.16b, v8.16b, #0x1\n" "add x27, x27, #0x4\n" - "ext v1.16b, v1.16b, v1.16b, #0x1\n" - "movi v17.4s, #0x0\n" - ".inst 0x4e86941e // sdot v30.4s, v0.16b, v6.16b\n" - ".inst 0x4e819414 // sdot v20.4s, v0.16b, v1.16b\n" - ".inst 0x4e8195d1 // sdot v17.4s, v14.16b, v1.16b\n" - ".inst 0x4e9a9619 // sdot v25.4s, v16.16b, v26.16b\n" - ".inst 0x4e9a97e5 // sdot v5.4s, v31.16b, v26.16b\n" - "ext v26.16b, v26.16b, v26.16b, #0x1\n" - ".inst 0x4e81961e // sdot v30.4s, v16.16b, v1.16b\n" - ".inst 0x4e9a9614 // sdot v20.4s, v16.16b, v26.16b\n" - "mls v5.4s, v19.4s, v11.4s\n" - ".inst 0x4e9a95d1 // sdot v17.4s, v14.16b, v26.16b\n" - ".inst 0x4e9597f9 // sdot v25.4s, v31.16b, v21.16b\n" - "ext v21.16b, v21.16b, v21.16b, #0x1\n" - ".inst 0x4e9a97fe // sdot v30.4s, v31.16b, v26.16b\n" - ".inst 0x4e9597f4 // sdot v20.4s, v31.16b, v21.16b\n" - "sqrdmulh v5.4s, v5.4s, v9.4s\n" - "mov v16.16b, v17.16b\n .inst 0x4e9595d0 // sdot v16.4s, v14.16b, v21.16b\n" - ".inst 0x4e8695d1 // sdot v17.4s, v14.16b, v6.16b\n" - "mls v30.4s, v17.4s, v11.4s\n" - "mls v25.4s, v18.4s, v11.4s\n" - "mls v20.4s, v16.4s, v11.4s\n" - "and v0.16b, v5.16b, v4.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqrdmulh v30.4s, v30.4s, v9.4s\n" - "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqrdmulh v20.4s, v20.4s, v9.4s\n" - "sqadd v5.4s, v5.4s, v0.4s\n" - "and v16.16b, v30.16b, v4.16b\n" - "and v31.16b, v25.16b, v4.16b\n" - "and v0.16b, v20.16b, v4.16b\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqadd v30.4s, v30.4s, v16.4s\n" - "sqadd v25.4s, v25.4s, v31.4s\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "srshl v5.4s, v5.4s, v4.4s\n" - "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" - "srshl v20.4s, v20.4s, v4.4s\n" - "add v5.4s, v5.4s, v10.4s\n" - "add v30.4s, v30.4s, v10.4s\n" - "add v25.4s, v25.4s, v10.4s\n" - "add v20.4s, v20.4s, v10.4s\n" - "smax v5.4s, v5.4s, v13.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" + "ext v10.16b, v10.16b, v10.16b, #0x1\n" + "movi v18.4s, #0x0\n" + ".inst 0x4e889776 // sdot v22.4s, v27.16b, v8.16b\n" + ".inst 0x4e8a9774 // sdot v20.4s, v27.16b, v10.16b\n" + ".inst 0x4e8a9592 // sdot v18.4s, v12.16b, v10.16b\n" + ".inst 0x4e8597b5 // sdot v21.4s, v29.16b, v5.16b\n" + ".inst 0x4e859798 // sdot v24.4s, v28.16b, v5.16b\n" + "ext v5.16b, v5.16b, v5.16b, #0x1\n" + ".inst 0x4e8a97b6 // sdot v22.4s, v29.16b, v10.16b\n" + ".inst 0x4e8597b4 // sdot v20.4s, v29.16b, v5.16b\n" + "mls v24.4s, v25.4s, v16.4s\n" + ".inst 0x4e859592 // sdot v18.4s, v12.16b, v5.16b\n" + ".inst 0x4e899795 // sdot v21.4s, v28.16b, v9.16b\n" + "ext v9.16b, v9.16b, v9.16b, #0x1\n" + ".inst 0x4e859796 // sdot v22.4s, v28.16b, v5.16b\n" + ".inst 0x4e899794 // sdot v20.4s, v28.16b, v9.16b\n" + "sqrdmulh v24.4s, v24.4s, v30.4s\n" + "mov v17.16b, v18.16b\n .inst 0x4e899591 // sdot v17.4s, v12.16b, v9.16b\n" + ".inst 0x4e889592 // sdot v18.4s, v12.16b, v8.16b\n" + "mls v22.4s, v18.4s, v16.4s\n" + "mls v21.4s, v23.4s, v16.4s\n" + "mls v20.4s, v17.4s, v16.4s\n" + "and v17.16b, v24.16b, v26.16b\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqrdmulh v22.4s, v22.4s, v30.4s\n" + "sqrdmulh v21.4s, v21.4s, v30.4s\n" + "sqrdmulh v20.4s, v20.4s, v30.4s\n" + "sqadd v24.4s, v24.4s, v17.4s\n" + "and v19.16b, v22.16b, v26.16b\n" + "and v18.16b, v21.16b, v26.16b\n" + "and v17.16b, v20.16b, v26.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqadd v22.4s, v22.4s, v19.4s\n" + "sqadd v21.4s, v21.4s, v18.4s\n" + "sqadd v20.4s, v20.4s, v17.4s\n" + "srshl v24.4s, v24.4s, v26.4s\n" + "srshl v22.4s, v22.4s, v26.4s\n" + "srshl v21.4s, v21.4s, v26.4s\n" + "srshl v20.4s, v20.4s, v26.4s\n" + "add v24.4s, v24.4s, v14.4s\n" + "add v22.4s, v22.4s, v14.4s\n" + "add v21.4s, v21.4s, v14.4s\n" + "add v20.4s, v20.4s, v14.4s\n" + "smax v24.4s, v24.4s, v13.4s\n" + "smax v22.4s, v22.4s, v13.4s\n" + "smax v21.4s, v21.4s, v13.4s\n" "smax v20.4s, v20.4s, v13.4s\n" - "smin v5.4s, v5.4s, v12.4s\n" - "smin v30.4s, v30.4s, v12.4s\n" - "smin v25.4s, v25.4s, v12.4s\n" - "smin v20.4s, v20.4s, v12.4s\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" + "smin v24.4s, v24.4s, v11.4s\n" + "smin v22.4s, v22.4s, v11.4s\n" + "smin v21.4s, v21.4s, v11.4s\n" + "smin v20.4s, v20.4s, v11.4s\n" + "uzp1 v24.16b, v24.16b, v24.16b\n" + "uzp1 v22.16b, v22.16b, v22.16b\n" + "uzp1 v21.16b, v21.16b, v21.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "str s5, [x24, x27]\n" - "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "str s30, [x23, x27]\n" + "uzp1 v24.16b, v24.16b, v24.16b\n" + "str s24, [x25, x27]\n" + "uzp1 v22.16b, v22.16b, v22.16b\n" + "uzp1 v21.16b, v21.16b, v21.16b\n" + "str s22, [x24, x27]\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s25, [x22, x27]\n" - "str s20, [x21, x27]\n" + "str s21, [x23, x27]\n" + "str s20, [x22, x27]\n" "add x27, x27, #0x4\n" "beq 35f\n" "3:" // Oddments @@ -869,794 +861,794 @@ void a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst_impl( "add x10, x10, x28\n" "add x9, x9, x28\n" "add x26, x26, x28\n" - "add x25, x25, x28\n" + "add x21, x21, x28\n" "tbz %x[n_channels], #3, 7f\n" - "ldr d9, [x15], #0x8\n" - "ldr d8, [x14], #0x8\n" + "ldr d15, [x15], #0x8\n" + "ldr d25, [x14], #0x8\n" "ldr d7, [x13], #0x8\n" - "ldr d6, [x12], #0x8\n" - "ldr d4, [x10], #0x8\n" - "ldr d3, [x9], #0x8\n" - "ldr d2, [x26], #0x8\n" - "ldr d1, [x25], #0x8\n" + "ldr d8, [x12], #0x8\n" + "ldr d26, [x10], #0x8\n" + "ldr d23, [x9], #0x8\n" + "ldr d3, [x26], #0x8\n" + "ldr d10, [x21], #0x8\n" "tbz %x[n_channels], #2, 5f\n" - "ld1 { v9.s }[2], [x15], #0x4\n" - "ld1 { v8.s }[2], [x14], #0x4\n" + "ld1 { v15.s }[2], [x15], #0x4\n" + "ld1 { v25.s }[2], [x14], #0x4\n" "ld1 { v7.s }[2], [x13], #0x4\n" - "ld1 { v6.s }[2], [x12], #0x4\n" - "ld1 { v4.s }[2], [x10], #0x4\n" - "ld1 { v3.s }[2], [x9], #0x4\n" - "ld1 { v2.s }[2], [x26], #0x4\n" - "ld1 { v1.s }[2], [x25], #0x4\n" + "ld1 { v8.s }[2], [x12], #0x4\n" + "ld1 { v26.s }[2], [x10], #0x4\n" + "ld1 { v23.s }[2], [x9], #0x4\n" + "ld1 { v3.s }[2], [x26], #0x4\n" + "ld1 { v10.s }[2], [x21], #0x4\n" "tbz %x[n_channels], #1, 4f\n" - "ld1 { v9.h }[6], [x15], #0x2\n" - "ld1 { v8.h }[6], [x14], #0x2\n" + "ld1 { v15.h }[6], [x15], #0x2\n" + "ld1 { v25.h }[6], [x14], #0x2\n" "ld1 { v7.h }[6], [x13], #0x2\n" - "ld1 { v6.h }[6], [x12], #0x2\n" - "ld1 { v4.h }[6], [x10], #0x2\n" - "ld1 { v3.h }[6], [x9], #0x2\n" - "ld1 { v2.h }[6], [x26], #0x2\n" - "ld1 { v1.h }[6], [x25], #0x2\n" + "ld1 { v8.h }[6], [x12], #0x2\n" + "ld1 { v26.h }[6], [x10], #0x2\n" + "ld1 { v23.h }[6], [x9], #0x2\n" + "ld1 { v3.h }[6], [x26], #0x2\n" + "ld1 { v10.h }[6], [x21], #0x2\n" "tbz %x[n_channels], #0, 11f\n" - "ld1 { v9.b }[14], [x15], #0x1\n" - "ld1 { v8.b }[14], [x14], #0x1\n" + "ld1 { v15.b }[14], [x15], #0x1\n" + "ld1 { v25.b }[14], [x14], #0x1\n" "ld1 { v7.b }[14], [x13], #0x1\n" - "ld1 { v6.b }[14], [x12], #0x1\n" - "ld1 { v4.b }[14], [x10], #0x1\n" - "ld1 { v3.b }[14], [x9], #0x1\n" - "ld1 { v2.b }[14], [x26], #0x1\n" - "ld1 { v1.b }[14], [x25], #0x1\n" + "ld1 { v8.b }[14], [x12], #0x1\n" + "ld1 { v26.b }[14], [x10], #0x1\n" + "ld1 { v23.b }[14], [x9], #0x1\n" + "ld1 { v3.b }[14], [x26], #0x1\n" + "ld1 { v10.b }[14], [x21], #0x1\n" "b 11f\n" "4:" // Oddments: Load (A): Bit 3: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 11f\n" - "ld1 { v9.b }[12], [x15], #0x1\n" - "ld1 { v8.b }[12], [x14], #0x1\n" + "ld1 { v15.b }[12], [x15], #0x1\n" + "ld1 { v25.b }[12], [x14], #0x1\n" "ld1 { v7.b }[12], [x13], #0x1\n" - "ld1 { v6.b }[12], [x12], #0x1\n" - "ld1 { v4.b }[12], [x10], #0x1\n" - "ld1 { v3.b }[12], [x9], #0x1\n" - "ld1 { v2.b }[12], [x26], #0x1\n" - "ld1 { v1.b }[12], [x25], #0x1\n" + "ld1 { v8.b }[12], [x12], #0x1\n" + "ld1 { v26.b }[12], [x10], #0x1\n" + "ld1 { v23.b }[12], [x9], #0x1\n" + "ld1 { v3.b }[12], [x26], #0x1\n" + "ld1 { v10.b }[12], [x21], #0x1\n" "b 11f\n" "5:" // Oddments: Load (A): Bit 3: Bit 2: Unset "tbz %x[n_channels], #1, 6f\n" - "ld1 { v9.h }[4], [x15], #0x2\n" - "ld1 { v8.h }[4], [x14], #0x2\n" + "ld1 { v15.h }[4], [x15], #0x2\n" + "ld1 { v25.h }[4], [x14], #0x2\n" "ld1 { v7.h }[4], [x13], #0x2\n" - "ld1 { v6.h }[4], [x12], #0x2\n" - "ld1 { v4.h }[4], [x10], #0x2\n" - "ld1 { v3.h }[4], [x9], #0x2\n" - "ld1 { v2.h }[4], [x26], #0x2\n" - "ld1 { v1.h }[4], [x25], #0x2\n" + "ld1 { v8.h }[4], [x12], #0x2\n" + "ld1 { v26.h }[4], [x10], #0x2\n" + "ld1 { v23.h }[4], [x9], #0x2\n" + "ld1 { v3.h }[4], [x26], #0x2\n" + "ld1 { v10.h }[4], [x21], #0x2\n" "tbz %x[n_channels], #0, 11f\n" - "ld1 { v9.b }[10], [x15], #0x1\n" - "ld1 { v8.b }[10], [x14], #0x1\n" + "ld1 { v15.b }[10], [x15], #0x1\n" + "ld1 { v25.b }[10], [x14], #0x1\n" "ld1 { v7.b }[10], [x13], #0x1\n" - "ld1 { v6.b }[10], [x12], #0x1\n" - "ld1 { v4.b }[10], [x10], #0x1\n" - "ld1 { v3.b }[10], [x9], #0x1\n" - "ld1 { v2.b }[10], [x26], #0x1\n" - "ld1 { v1.b }[10], [x25], #0x1\n" + "ld1 { v8.b }[10], [x12], #0x1\n" + "ld1 { v26.b }[10], [x10], #0x1\n" + "ld1 { v23.b }[10], [x9], #0x1\n" + "ld1 { v3.b }[10], [x26], #0x1\n" + "ld1 { v10.b }[10], [x21], #0x1\n" "b 11f\n" "6:" // Oddments: Load (A): Bit 3: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 11f\n" - "ld1 { v9.b }[8], [x15], #0x1\n" - "ld1 { v8.b }[8], [x14], #0x1\n" + "ld1 { v15.b }[8], [x15], #0x1\n" + "ld1 { v25.b }[8], [x14], #0x1\n" "ld1 { v7.b }[8], [x13], #0x1\n" - "ld1 { v6.b }[8], [x12], #0x1\n" - "ld1 { v4.b }[8], [x10], #0x1\n" - "ld1 { v3.b }[8], [x9], #0x1\n" - "ld1 { v2.b }[8], [x26], #0x1\n" - "ld1 { v1.b }[8], [x25], #0x1\n" + "ld1 { v8.b }[8], [x12], #0x1\n" + "ld1 { v26.b }[8], [x10], #0x1\n" + "ld1 { v23.b }[8], [x9], #0x1\n" + "ld1 { v3.b }[8], [x26], #0x1\n" + "ld1 { v10.b }[8], [x21], #0x1\n" "b 11f\n" "7:" // Oddments: Load (A): Bit 3: Unset "tbz %x[n_channels], #2, 9f\n" - "ldr s9, [x15], #0x4\n" - "ldr s8, [x14], #0x4\n" + "ldr s15, [x15], #0x4\n" + "ldr s25, [x14], #0x4\n" "ldr s7, [x13], #0x4\n" - "ldr s6, [x12], #0x4\n" - "ldr s4, [x10], #0x4\n" - "ldr s3, [x9], #0x4\n" - "ldr s2, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s8, [x12], #0x4\n" + "ldr s26, [x10], #0x4\n" + "ldr s23, [x9], #0x4\n" + "ldr s3, [x26], #0x4\n" + "ldr s10, [x21], #0x4\n" "tbz %x[n_channels], #1, 8f\n" - "ld1 { v9.h }[2], [x15], #0x2\n" - "ld1 { v8.h }[2], [x14], #0x2\n" + "ld1 { v15.h }[2], [x15], #0x2\n" + "ld1 { v25.h }[2], [x14], #0x2\n" "ld1 { v7.h }[2], [x13], #0x2\n" - "ld1 { v6.h }[2], [x12], #0x2\n" - "ld1 { v4.h }[2], [x10], #0x2\n" - "ld1 { v3.h }[2], [x9], #0x2\n" - "ld1 { v2.h }[2], [x26], #0x2\n" - "ld1 { v1.h }[2], [x25], #0x2\n" + "ld1 { v8.h }[2], [x12], #0x2\n" + "ld1 { v26.h }[2], [x10], #0x2\n" + "ld1 { v23.h }[2], [x9], #0x2\n" + "ld1 { v3.h }[2], [x26], #0x2\n" + "ld1 { v10.h }[2], [x21], #0x2\n" "tbz %x[n_channels], #0, 11f\n" - "ld1 { v9.b }[6], [x15], #0x1\n" - "ld1 { v8.b }[6], [x14], #0x1\n" + "ld1 { v15.b }[6], [x15], #0x1\n" + "ld1 { v25.b }[6], [x14], #0x1\n" "ld1 { v7.b }[6], [x13], #0x1\n" - "ld1 { v6.b }[6], [x12], #0x1\n" - "ld1 { v4.b }[6], [x10], #0x1\n" - "ld1 { v3.b }[6], [x9], #0x1\n" - "ld1 { v2.b }[6], [x26], #0x1\n" - "ld1 { v1.b }[6], [x25], #0x1\n" + "ld1 { v8.b }[6], [x12], #0x1\n" + "ld1 { v26.b }[6], [x10], #0x1\n" + "ld1 { v23.b }[6], [x9], #0x1\n" + "ld1 { v3.b }[6], [x26], #0x1\n" + "ld1 { v10.b }[6], [x21], #0x1\n" "b 11f\n" "8:" // Oddments: Load (A): Bit 3: Unset: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 11f\n" - "ld1 { v9.b }[4], [x15], #0x1\n" - "ld1 { v8.b }[4], [x14], #0x1\n" + "ld1 { v15.b }[4], [x15], #0x1\n" + "ld1 { v25.b }[4], [x14], #0x1\n" "ld1 { v7.b }[4], [x13], #0x1\n" - "ld1 { v6.b }[4], [x12], #0x1\n" - "ld1 { v4.b }[4], [x10], #0x1\n" - "ld1 { v3.b }[4], [x9], #0x1\n" - "ld1 { v2.b }[4], [x26], #0x1\n" - "ld1 { v1.b }[4], [x25], #0x1\n" + "ld1 { v8.b }[4], [x12], #0x1\n" + "ld1 { v26.b }[4], [x10], #0x1\n" + "ld1 { v23.b }[4], [x9], #0x1\n" + "ld1 { v3.b }[4], [x26], #0x1\n" + "ld1 { v10.b }[4], [x21], #0x1\n" "b 11f\n" "9:" // Oddments: Load (A): Bit 3: Unset: Bit 2: Unset "tbz %x[n_channels], #1, 10f\n" - "ldr h9, [x15], #0x2\n" - "ldr h8, [x14], #0x2\n" + "ldr h15, [x15], #0x2\n" + "ldr h25, [x14], #0x2\n" "ldr h7, [x13], #0x2\n" - "ldr h6, [x12], #0x2\n" - "ldr h4, [x10], #0x2\n" - "ldr h3, [x9], #0x2\n" - "ldr h2, [x26], #0x2\n" - "ldr h1, [x25], #0x2\n" + "ldr h8, [x12], #0x2\n" + "ldr h26, [x10], #0x2\n" + "ldr h23, [x9], #0x2\n" + "ldr h3, [x26], #0x2\n" + "ldr h10, [x21], #0x2\n" "tbz %x[n_channels], #0, 11f\n" - "ld1 { v9.b }[2], [x15], #0x1\n" - "ld1 { v8.b }[2], [x14], #0x1\n" + "ld1 { v15.b }[2], [x15], #0x1\n" + "ld1 { v25.b }[2], [x14], #0x1\n" "ld1 { v7.b }[2], [x13], #0x1\n" - "ld1 { v6.b }[2], [x12], #0x1\n" - "ld1 { v4.b }[2], [x10], #0x1\n" - "ld1 { v3.b }[2], [x9], #0x1\n" - "ld1 { v2.b }[2], [x26], #0x1\n" - "ld1 { v1.b }[2], [x25], #0x1\n" + "ld1 { v8.b }[2], [x12], #0x1\n" + "ld1 { v26.b }[2], [x10], #0x1\n" + "ld1 { v23.b }[2], [x9], #0x1\n" + "ld1 { v3.b }[2], [x26], #0x1\n" + "ld1 { v10.b }[2], [x21], #0x1\n" "b 11f\n" "10:" // Oddments: Load (A): Bit 3: Unset: Bit 2: Unset: Bit 1: Unset - "ldr b9, [x15], #0x1\n" - "ldr b8, [x14], #0x1\n" + "ldr b15, [x15], #0x1\n" + "ldr b25, [x14], #0x1\n" "ldr b7, [x13], #0x1\n" - "ldr b6, [x12], #0x1\n" - "ldr b4, [x10], #0x1\n" - "ldr b3, [x9], #0x1\n" - "ldr b2, [x26], #0x1\n" - "ldr b1, [x25], #0x1\n" + "ldr b8, [x12], #0x1\n" + "ldr b26, [x10], #0x1\n" + "ldr b23, [x9], #0x1\n" + "ldr b3, [x26], #0x1\n" + "ldr b10, [x21], #0x1\n" "11:" // Oddments: Load (A): Bit 3: End "ldp x15, x14, [%x[inptrs], #0x40]\n" "ldp x13, x12, [%x[inptrs], #0x50]\n" "add x15, x15, x28\n" "add x14, x14, x28\n" "ldp x10, x9, [%x[inptrs], #0x60]\n" - "ldp x26, x25, [%x[inptrs], #0x70]\n" + "ldp x26, x21, [%x[inptrs], #0x70]\n" "add x13, x13, x28\n" "add x12, x12, x28\n" "add x10, x10, x28\n" "add x9, x9, x28\n" "add x26, x26, x28\n" - "add x25, x25, x28\n" + "add x21, x21, x28\n" "tbz %x[n_channels], #3, 15f\n" - "ldr d29, [x15], #0x8\n" - "ldr d28, [x14], #0x8\n" - "ldr d27, [x13], #0x8\n" - "ldr d26, [x12], #0x8\n" - "ldr d24, [x10], #0x8\n" - "ldr d23, [x9], #0x8\n" - "ldr d22, [x26], #0x8\n" - "ldr d21, [x25], #0x8\n" + "ldr d22, [x15], #0x8\n" + "ldr d19, [x14], #0x8\n" + "ldr d0, [x13], #0x8\n" + "ldr d5, [x12], #0x8\n" + "ldr d27, [x10], #0x8\n" + "ldr d24, [x9], #0x8\n" + "ldr d2, [x26], #0x8\n" + "ldr d9, [x21], #0x8\n" "tbz %x[n_channels], #2, 13f\n" - "ld1 { v29.s }[2], [x15], #0x4\n" - "ld1 { v28.s }[2], [x14], #0x4\n" - "ld1 { v27.s }[2], [x13], #0x4\n" - "ld1 { v26.s }[2], [x12], #0x4\n" - "ld1 { v24.s }[2], [x10], #0x4\n" - "ld1 { v23.s }[2], [x9], #0x4\n" - "ld1 { v22.s }[2], [x26], #0x4\n" - "ld1 { v21.s }[2], [x25], #0x4\n" + "ld1 { v22.s }[2], [x15], #0x4\n" + "ld1 { v19.s }[2], [x14], #0x4\n" + "ld1 { v0.s }[2], [x13], #0x4\n" + "ld1 { v5.s }[2], [x12], #0x4\n" + "ld1 { v27.s }[2], [x10], #0x4\n" + "ld1 { v24.s }[2], [x9], #0x4\n" + "ld1 { v2.s }[2], [x26], #0x4\n" + "ld1 { v9.s }[2], [x21], #0x4\n" "tbz %x[n_channels], #1, 12f\n" - "ld1 { v29.h }[6], [x15], #0x2\n" - "ld1 { v28.h }[6], [x14], #0x2\n" - "ld1 { v27.h }[6], [x13], #0x2\n" - "ld1 { v26.h }[6], [x12], #0x2\n" - "ld1 { v24.h }[6], [x10], #0x2\n" - "ld1 { v23.h }[6], [x9], #0x2\n" - "ld1 { v22.h }[6], [x26], #0x2\n" - "ld1 { v21.h }[6], [x25], #0x2\n" + "ld1 { v22.h }[6], [x15], #0x2\n" + "ld1 { v19.h }[6], [x14], #0x2\n" + "ld1 { v0.h }[6], [x13], #0x2\n" + "ld1 { v5.h }[6], [x12], #0x2\n" + "ld1 { v27.h }[6], [x10], #0x2\n" + "ld1 { v24.h }[6], [x9], #0x2\n" + "ld1 { v2.h }[6], [x26], #0x2\n" + "ld1 { v9.h }[6], [x21], #0x2\n" "tbz %x[n_channels], #0, 19f\n" - "ld1 { v29.b }[14], [x15], #0x1\n" - "ld1 { v28.b }[14], [x14], #0x1\n" - "ld1 { v27.b }[14], [x13], #0x1\n" - "ld1 { v26.b }[14], [x12], #0x1\n" - "ld1 { v24.b }[14], [x10], #0x1\n" - "ld1 { v23.b }[14], [x9], #0x1\n" - "ld1 { v22.b }[14], [x26], #0x1\n" - "ld1 { v21.b }[14], [x25], #0x1\n" + "ld1 { v22.b }[14], [x15], #0x1\n" + "ld1 { v19.b }[14], [x14], #0x1\n" + "ld1 { v0.b }[14], [x13], #0x1\n" + "ld1 { v5.b }[14], [x12], #0x1\n" + "ld1 { v27.b }[14], [x10], #0x1\n" + "ld1 { v24.b }[14], [x9], #0x1\n" + "ld1 { v2.b }[14], [x26], #0x1\n" + "ld1 { v9.b }[14], [x21], #0x1\n" "b 19f\n" "12:" // Oddments: Load (B): Bit 3: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 19f\n" - "ld1 { v29.b }[12], [x15], #0x1\n" - "ld1 { v28.b }[12], [x14], #0x1\n" - "ld1 { v27.b }[12], [x13], #0x1\n" - "ld1 { v26.b }[12], [x12], #0x1\n" - "ld1 { v24.b }[12], [x10], #0x1\n" - "ld1 { v23.b }[12], [x9], #0x1\n" - "ld1 { v22.b }[12], [x26], #0x1\n" - "ld1 { v21.b }[12], [x25], #0x1\n" + "ld1 { v22.b }[12], [x15], #0x1\n" + "ld1 { v19.b }[12], [x14], #0x1\n" + "ld1 { v0.b }[12], [x13], #0x1\n" + "ld1 { v5.b }[12], [x12], #0x1\n" + "ld1 { v27.b }[12], [x10], #0x1\n" + "ld1 { v24.b }[12], [x9], #0x1\n" + "ld1 { v2.b }[12], [x26], #0x1\n" + "ld1 { v9.b }[12], [x21], #0x1\n" "b 19f\n" "13:" // Oddments: Load (B): Bit 3: Bit 2: Unset "tbz %x[n_channels], #1, 14f\n" - "ld1 { v29.h }[4], [x15], #0x2\n" - "ld1 { v28.h }[4], [x14], #0x2\n" - "ld1 { v27.h }[4], [x13], #0x2\n" - "ld1 { v26.h }[4], [x12], #0x2\n" - "ld1 { v24.h }[4], [x10], #0x2\n" - "ld1 { v23.h }[4], [x9], #0x2\n" - "ld1 { v22.h }[4], [x26], #0x2\n" - "ld1 { v21.h }[4], [x25], #0x2\n" + "ld1 { v22.h }[4], [x15], #0x2\n" + "ld1 { v19.h }[4], [x14], #0x2\n" + "ld1 { v0.h }[4], [x13], #0x2\n" + "ld1 { v5.h }[4], [x12], #0x2\n" + "ld1 { v27.h }[4], [x10], #0x2\n" + "ld1 { v24.h }[4], [x9], #0x2\n" + "ld1 { v2.h }[4], [x26], #0x2\n" + "ld1 { v9.h }[4], [x21], #0x2\n" "tbz %x[n_channels], #0, 19f\n" - "ld1 { v29.b }[10], [x15], #0x1\n" - "ld1 { v28.b }[10], [x14], #0x1\n" - "ld1 { v27.b }[10], [x13], #0x1\n" - "ld1 { v26.b }[10], [x12], #0x1\n" - "ld1 { v24.b }[10], [x10], #0x1\n" - "ld1 { v23.b }[10], [x9], #0x1\n" - "ld1 { v22.b }[10], [x26], #0x1\n" - "ld1 { v21.b }[10], [x25], #0x1\n" + "ld1 { v22.b }[10], [x15], #0x1\n" + "ld1 { v19.b }[10], [x14], #0x1\n" + "ld1 { v0.b }[10], [x13], #0x1\n" + "ld1 { v5.b }[10], [x12], #0x1\n" + "ld1 { v27.b }[10], [x10], #0x1\n" + "ld1 { v24.b }[10], [x9], #0x1\n" + "ld1 { v2.b }[10], [x26], #0x1\n" + "ld1 { v9.b }[10], [x21], #0x1\n" "b 19f\n" "14:" // Oddments: Load (B): Bit 3: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 19f\n" - "ld1 { v29.b }[8], [x15], #0x1\n" - "ld1 { v28.b }[8], [x14], #0x1\n" - "ld1 { v27.b }[8], [x13], #0x1\n" - "ld1 { v26.b }[8], [x12], #0x1\n" - "ld1 { v24.b }[8], [x10], #0x1\n" - "ld1 { v23.b }[8], [x9], #0x1\n" - "ld1 { v22.b }[8], [x26], #0x1\n" - "ld1 { v21.b }[8], [x25], #0x1\n" + "ld1 { v22.b }[8], [x15], #0x1\n" + "ld1 { v19.b }[8], [x14], #0x1\n" + "ld1 { v0.b }[8], [x13], #0x1\n" + "ld1 { v5.b }[8], [x12], #0x1\n" + "ld1 { v27.b }[8], [x10], #0x1\n" + "ld1 { v24.b }[8], [x9], #0x1\n" + "ld1 { v2.b }[8], [x26], #0x1\n" + "ld1 { v9.b }[8], [x21], #0x1\n" "b 19f\n" "15:" // Oddments: Load (B): Bit 3: Unset "tbz %x[n_channels], #2, 17f\n" - "ldr s29, [x15], #0x4\n" - "ldr s28, [x14], #0x4\n" - "ldr s27, [x13], #0x4\n" - "ldr s26, [x12], #0x4\n" - "ldr s24, [x10], #0x4\n" - "ldr s23, [x9], #0x4\n" - "ldr s22, [x26], #0x4\n" - "ldr s21, [x25], #0x4\n" + "ldr s22, [x15], #0x4\n" + "ldr s19, [x14], #0x4\n" + "ldr s0, [x13], #0x4\n" + "ldr s5, [x12], #0x4\n" + "ldr s27, [x10], #0x4\n" + "ldr s24, [x9], #0x4\n" + "ldr s2, [x26], #0x4\n" + "ldr s9, [x21], #0x4\n" "tbz %x[n_channels], #1, 16f\n" - "ld1 { v29.h }[2], [x15], #0x2\n" - "ld1 { v28.h }[2], [x14], #0x2\n" - "ld1 { v27.h }[2], [x13], #0x2\n" - "ld1 { v26.h }[2], [x12], #0x2\n" - "ld1 { v24.h }[2], [x10], #0x2\n" - "ld1 { v23.h }[2], [x9], #0x2\n" - "ld1 { v22.h }[2], [x26], #0x2\n" - "ld1 { v21.h }[2], [x25], #0x2\n" + "ld1 { v22.h }[2], [x15], #0x2\n" + "ld1 { v19.h }[2], [x14], #0x2\n" + "ld1 { v0.h }[2], [x13], #0x2\n" + "ld1 { v5.h }[2], [x12], #0x2\n" + "ld1 { v27.h }[2], [x10], #0x2\n" + "ld1 { v24.h }[2], [x9], #0x2\n" + "ld1 { v2.h }[2], [x26], #0x2\n" + "ld1 { v9.h }[2], [x21], #0x2\n" "tbz %x[n_channels], #0, 19f\n" - "ld1 { v29.b }[6], [x15], #0x1\n" - "ld1 { v28.b }[6], [x14], #0x1\n" - "ld1 { v27.b }[6], [x13], #0x1\n" - "ld1 { v26.b }[6], [x12], #0x1\n" - "ld1 { v24.b }[6], [x10], #0x1\n" - "ld1 { v23.b }[6], [x9], #0x1\n" - "ld1 { v22.b }[6], [x26], #0x1\n" - "ld1 { v21.b }[6], [x25], #0x1\n" + "ld1 { v22.b }[6], [x15], #0x1\n" + "ld1 { v19.b }[6], [x14], #0x1\n" + "ld1 { v0.b }[6], [x13], #0x1\n" + "ld1 { v5.b }[6], [x12], #0x1\n" + "ld1 { v27.b }[6], [x10], #0x1\n" + "ld1 { v24.b }[6], [x9], #0x1\n" + "ld1 { v2.b }[6], [x26], #0x1\n" + "ld1 { v9.b }[6], [x21], #0x1\n" "b 19f\n" "16:" // Oddments: Load (B): Bit 3: Unset: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 19f\n" - "ld1 { v29.b }[4], [x15], #0x1\n" - "ld1 { v28.b }[4], [x14], #0x1\n" - "ld1 { v27.b }[4], [x13], #0x1\n" - "ld1 { v26.b }[4], [x12], #0x1\n" - "ld1 { v24.b }[4], [x10], #0x1\n" - "ld1 { v23.b }[4], [x9], #0x1\n" - "ld1 { v22.b }[4], [x26], #0x1\n" - "ld1 { v21.b }[4], [x25], #0x1\n" + "ld1 { v22.b }[4], [x15], #0x1\n" + "ld1 { v19.b }[4], [x14], #0x1\n" + "ld1 { v0.b }[4], [x13], #0x1\n" + "ld1 { v5.b }[4], [x12], #0x1\n" + "ld1 { v27.b }[4], [x10], #0x1\n" + "ld1 { v24.b }[4], [x9], #0x1\n" + "ld1 { v2.b }[4], [x26], #0x1\n" + "ld1 { v9.b }[4], [x21], #0x1\n" "b 19f\n" "17:" // Oddments: Load (B): Bit 3: Unset: Bit 2: Unset "tbz %x[n_channels], #1, 18f\n" - "ldr h29, [x15], #0x2\n" - "ldr h28, [x14], #0x2\n" - "ldr h27, [x13], #0x2\n" - "ldr h26, [x12], #0x2\n" - "ldr h24, [x10], #0x2\n" - "ldr h23, [x9], #0x2\n" - "ldr h22, [x26], #0x2\n" - "ldr h21, [x25], #0x2\n" + "ldr h22, [x15], #0x2\n" + "ldr h19, [x14], #0x2\n" + "ldr h0, [x13], #0x2\n" + "ldr h5, [x12], #0x2\n" + "ldr h27, [x10], #0x2\n" + "ldr h24, [x9], #0x2\n" + "ldr h2, [x26], #0x2\n" + "ldr h9, [x21], #0x2\n" "tbz %x[n_channels], #0, 19f\n" - "ld1 { v29.b }[2], [x15], #0x1\n" - "ld1 { v28.b }[2], [x14], #0x1\n" - "ld1 { v27.b }[2], [x13], #0x1\n" - "ld1 { v26.b }[2], [x12], #0x1\n" - "ld1 { v24.b }[2], [x10], #0x1\n" - "ld1 { v23.b }[2], [x9], #0x1\n" - "ld1 { v22.b }[2], [x26], #0x1\n" - "ld1 { v21.b }[2], [x25], #0x1\n" + "ld1 { v22.b }[2], [x15], #0x1\n" + "ld1 { v19.b }[2], [x14], #0x1\n" + "ld1 { v0.b }[2], [x13], #0x1\n" + "ld1 { v5.b }[2], [x12], #0x1\n" + "ld1 { v27.b }[2], [x10], #0x1\n" + "ld1 { v24.b }[2], [x9], #0x1\n" + "ld1 { v2.b }[2], [x26], #0x1\n" + "ld1 { v9.b }[2], [x21], #0x1\n" "b 19f\n" "18:" // Oddments: Load (B): Bit 3: Unset: Bit 2: Unset: Bit 1: Unset - "ldr b29, [x15], #0x1\n" - "ldr b28, [x14], #0x1\n" - "ldr b27, [x13], #0x1\n" - "ldr b26, [x12], #0x1\n" - "ldr b24, [x10], #0x1\n" - "ldr b23, [x9], #0x1\n" - "ldr b22, [x26], #0x1\n" - "ldr b21, [x25], #0x1\n" + "ldr b22, [x15], #0x1\n" + "ldr b19, [x14], #0x1\n" + "ldr b0, [x13], #0x1\n" + "ldr b5, [x12], #0x1\n" + "ldr b27, [x10], #0x1\n" + "ldr b24, [x9], #0x1\n" + "ldr b2, [x26], #0x1\n" + "ldr b9, [x21], #0x1\n" "19:" // Oddments: Load (B): Bit 3: End - "ldr q0, [%x[params], #0x10]\n" - "ldr q16, [%x[params], #0x20]\n" - "zip2 v30.16b, v4.16b, v2.16b\n" - "zip1 v4.16b, v4.16b, v2.16b\n" - "ldr q31, [%x[params], #0x30]\n" - "zip1 v2.16b, v3.16b, v1.16b\n" - "zip2 v5.16b, v9.16b, v7.16b\n" + "ldr q20, [%x[params], #0x10]\n" + "ldr q6, [%x[params], #0x20]\n" + "zip2 v1.16b, v26.16b, v3.16b\n" + "zip1 v26.16b, v26.16b, v3.16b\n" + "ldr q4, [%x[params], #0x30]\n" + "zip1 v18.16b, v23.16b, v10.16b\n" + "zip2 v30.16b, v15.16b, v7.16b\n" "cmp x20, #0x4\n" - "zip1 v9.16b, v9.16b, v7.16b\n" - "zip1 v7.16b, v8.16b, v6.16b\n" - "zip2 v6.16b, v8.16b, v6.16b\n" - "zip2 v1.16b, v3.16b, v1.16b\n" - "zip2 v3.16b, v4.16b, v2.16b\n" - "zip1 v4.16b, v4.16b, v2.16b\n" - "zip2 v25.16b, v29.16b, v27.16b\n" - "zip1 v29.16b, v29.16b, v27.16b\n" - "zip1 v27.16b, v28.16b, v26.16b\n" - "movi v19.4s, #0x0\n" - ".inst 0x4e8495d3 // sdot v19.4s, v14.16b, v4.16b\n" - "zip2 v8.16b, v9.16b, v7.16b\n" - "zip1 v9.16b, v9.16b, v7.16b\n" - "zip1 v7.16b, v5.16b, v6.16b\n" - "zip2 v6.16b, v5.16b, v6.16b\n" - "ldr q5, [%x[params], #0x0]\n" - "zip2 v26.16b, v28.16b, v26.16b\n" - "zip2 v20.16b, v24.16b, v22.16b\n" - "zip1 v24.16b, v24.16b, v22.16b\n" - "zip1 v22.16b, v23.16b, v21.16b\n" - "zip2 v21.16b, v23.16b, v21.16b\n" - "zip2 v28.16b, v29.16b, v27.16b\n" - "zip1 v29.16b, v29.16b, v27.16b\n" - "zip1 v2.16b, v30.16b, v1.16b\n" - ".inst 0x4e9d95d3 // sdot v19.4s, v14.16b, v29.16b\n" - "zip2 v1.16b, v30.16b, v1.16b\n" - "zip1 v27.16b, v25.16b, v26.16b\n" - "zip2 v26.16b, v25.16b, v26.16b\n" - "zip2 v23.16b, v24.16b, v22.16b\n" - "zip1 v24.16b, v24.16b, v22.16b\n" - "zip1 v22.16b, v20.16b, v21.16b\n" - "mov v18.16b, v19.16b\n .inst 0x4e9895d2 // sdot v18.4s, v14.16b, v24.16b\n" - "zip2 v21.16b, v20.16b, v21.16b\n" - "mov v30.16b, v5.16b\n" - ".inst 0x4e8995d3 // sdot v19.4s, v14.16b, v9.16b\n" - "mov v25.16b, v5.16b\n" - "mov v20.16b, v5.16b\n" - ".inst 0x4e899405 // sdot v5.4s, v0.16b, v9.16b\n" - ".inst 0x4e849419 // sdot v25.4s, v0.16b, v4.16b\n" - ".inst 0x4e849605 // sdot v5.4s, v16.16b, v4.16b\n" - "ext v4.16b, v4.16b, v4.16b, #0x1\n" - "ext v9.16b, v9.16b, v9.16b, #0x1\n" - ".inst 0x4e9d9619 // sdot v25.4s, v16.16b, v29.16b\n" - ".inst 0x4e9d97e5 // sdot v5.4s, v31.16b, v29.16b\n" - "ext v29.16b, v29.16b, v29.16b, #0x1\n" - ".inst 0x4e89941e // sdot v30.4s, v0.16b, v9.16b\n" - ".inst 0x4e849414 // sdot v20.4s, v0.16b, v4.16b\n" + "zip1 v15.16b, v15.16b, v7.16b\n" + "zip1 v29.16b, v25.16b, v8.16b\n" + "zip2 v8.16b, v25.16b, v8.16b\n" + "zip2 v10.16b, v23.16b, v10.16b\n" + "zip2 v23.16b, v26.16b, v18.16b\n" + "zip1 v26.16b, v26.16b, v18.16b\n" + "zip2 v28.16b, v22.16b, v0.16b\n" + "zip1 v22.16b, v22.16b, v0.16b\n" + "zip1 v21.16b, v19.16b, v5.16b\n" "movi v17.4s, #0x0\n" - ".inst 0x4e8495d1 // sdot v17.4s, v14.16b, v4.16b\n" - ".inst 0x4e9d95d1 // sdot v17.4s, v14.16b, v29.16b\n" - ".inst 0x4e9897f9 // sdot v25.4s, v31.16b, v24.16b\n" - "ext v24.16b, v24.16b, v24.16b, #0x1\n" - ".inst 0x4e84961e // sdot v30.4s, v16.16b, v4.16b\n" - "ldr q4, [%x[params], #0x50]\n" - ".inst 0x4e9d9614 // sdot v20.4s, v16.16b, v29.16b\n" - "mov v16.16b, v17.16b\n .inst 0x4e9895d0 // sdot v16.4s, v14.16b, v24.16b\n" - "mls v5.4s, v19.4s, v11.4s\n" - ".inst 0x4e8995d1 // sdot v17.4s, v14.16b, v9.16b\n" - "ldr q9, [%x[params], #0x40]\n" - ".inst 0x4e9d97fe // sdot v30.4s, v31.16b, v29.16b\n" - "sqrdmulh v5.4s, v5.4s, v9.4s\n" - ".inst 0x4e9897f4 // sdot v20.4s, v31.16b, v24.16b\n" - "mls v30.4s, v17.4s, v11.4s\n" + ".inst 0x4e9a9591 // sdot v17.4s, v12.16b, v26.16b\n" + "zip2 v25.16b, v15.16b, v29.16b\n" + "zip1 v15.16b, v15.16b, v29.16b\n" + "zip1 v7.16b, v30.16b, v8.16b\n" + "zip2 v8.16b, v30.16b, v8.16b\n" + "ldr q31, [%x[params], #0x0]\n" + "zip2 v5.16b, v19.16b, v5.16b\n" + "zip2 v30.16b, v27.16b, v2.16b\n" + "zip1 v27.16b, v27.16b, v2.16b\n" + "zip1 v18.16b, v24.16b, v9.16b\n" + "zip2 v9.16b, v24.16b, v9.16b\n" + "zip2 v19.16b, v22.16b, v21.16b\n" + "zip1 v22.16b, v22.16b, v21.16b\n" + "zip1 v3.16b, v1.16b, v10.16b\n" + ".inst 0x4e969591 // sdot v17.4s, v12.16b, v22.16b\n" + "zip2 v10.16b, v1.16b, v10.16b\n" + "zip1 v0.16b, v28.16b, v5.16b\n" + "zip2 v5.16b, v28.16b, v5.16b\n" + "zip2 v24.16b, v27.16b, v18.16b\n" + "zip1 v27.16b, v27.16b, v18.16b\n" + "zip1 v2.16b, v30.16b, v9.16b\n" + "mov v18.16b, v17.16b\n .inst 0x4e9b9592 // sdot v18.4s, v12.16b, v27.16b\n" + "zip2 v9.16b, v30.16b, v9.16b\n" + "mov v30.16b, v31.16b\n" + ".inst 0x4e8f9591 // sdot v17.4s, v12.16b, v15.16b\n" + "mov v29.16b, v31.16b\n" + "mov v28.16b, v31.16b\n" + ".inst 0x4e8f969f // sdot v31.4s, v20.16b, v15.16b\n" + ".inst 0x4e9a969d // sdot v29.4s, v20.16b, v26.16b\n" + ".inst 0x4e9a94df // sdot v31.4s, v6.16b, v26.16b\n" + "ext v26.16b, v26.16b, v26.16b, #0x1\n" + "movi v1.4s, #0x0\n" + "ext v15.16b, v15.16b, v15.16b, #0x1\n" + ".inst 0x4e9a9581 // sdot v1.4s, v12.16b, v26.16b\n" + ".inst 0x4e9694dd // sdot v29.4s, v6.16b, v22.16b\n" + ".inst 0x4e96949f // sdot v31.4s, v4.16b, v22.16b\n" + "ext v22.16b, v22.16b, v22.16b, #0x1\n" + ".inst 0x4e8f969e // sdot v30.4s, v20.16b, v15.16b\n" + ".inst 0x4e9a969c // sdot v28.4s, v20.16b, v26.16b\n" + "mls v31.4s, v17.4s, v16.4s\n" + ".inst 0x4e969581 // sdot v1.4s, v12.16b, v22.16b\n" + ".inst 0x4e9b949d // sdot v29.4s, v4.16b, v27.16b\n" + "ext v27.16b, v27.16b, v27.16b, #0x1\n" + ".inst 0x4e9a94de // sdot v30.4s, v6.16b, v26.16b\n" + "ldr q21, [%x[params], #0x50]\n" + ".inst 0x4e9694dc // sdot v28.4s, v6.16b, v22.16b\n" + "mls v29.4s, v18.4s, v16.4s\n" + "mov v20.16b, v1.16b\n .inst 0x4e9b9594 // sdot v20.4s, v12.16b, v27.16b\n" + ".inst 0x4e8f9581 // sdot v1.4s, v12.16b, v15.16b\n" + "ldr q18, [%x[params], #0x40]\n" + "sqrdmulh v31.4s, v31.4s, v18.4s\n" + ".inst 0x4e96949e // sdot v30.4s, v4.16b, v22.16b\n" + ".inst 0x4e9b949c // sdot v28.4s, v4.16b, v27.16b\n" + "mls v30.4s, v1.4s, v16.4s\n" "add %x[params], %x[params], #0x60\n" - "mls v25.4s, v18.4s, v11.4s\n" - "mls v20.4s, v16.4s, v11.4s\n" - "and v0.16b, v5.16b, v4.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqrdmulh v30.4s, v30.4s, v9.4s\n" - "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqrdmulh v20.4s, v20.4s, v9.4s\n" - "sqadd v5.4s, v5.4s, v0.4s\n" - "and v16.16b, v30.16b, v4.16b\n" - "and v31.16b, v25.16b, v4.16b\n" - "and v0.16b, v20.16b, v4.16b\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqadd v30.4s, v30.4s, v16.4s\n" - "sqadd v25.4s, v25.4s, v31.4s\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "srshl v5.4s, v5.4s, v4.4s\n" - "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" - "srshl v20.4s, v20.4s, v4.4s\n" - "add v5.4s, v5.4s, v10.4s\n" - "add v30.4s, v30.4s, v10.4s\n" - "add v25.4s, v25.4s, v10.4s\n" - "add v20.4s, v20.4s, v10.4s\n" - "smax v5.4s, v5.4s, v13.4s\n" + "mls v28.4s, v20.4s, v16.4s\n" + "and v17.16b, v31.16b, v21.16b\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqrdmulh v30.4s, v30.4s, v18.4s\n" + "sqrdmulh v29.4s, v29.4s, v18.4s\n" + "sqrdmulh v28.4s, v28.4s, v18.4s\n" + "sqadd v31.4s, v31.4s, v17.4s\n" + "and v17.16b, v30.16b, v21.16b\n" + "and v18.16b, v29.16b, v21.16b\n" + "and v26.16b, v28.16b, v21.16b\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v26.4s, v26.4s, #0x1f\n" + "sqadd v30.4s, v30.4s, v17.4s\n" + "sqadd v29.4s, v29.4s, v18.4s\n" + "sqadd v28.4s, v28.4s, v26.4s\n" + "srshl v31.4s, v31.4s, v21.4s\n" + "srshl v30.4s, v30.4s, v21.4s\n" + "srshl v29.4s, v29.4s, v21.4s\n" + "srshl v28.4s, v28.4s, v21.4s\n" + "add v31.4s, v31.4s, v14.4s\n" + "add v30.4s, v30.4s, v14.4s\n" + "add v29.4s, v29.4s, v14.4s\n" + "add v28.4s, v28.4s, v14.4s\n" + "smax v31.4s, v31.4s, v13.4s\n" "smax v30.4s, v30.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smin v5.4s, v5.4s, v12.4s\n" - "smin v30.4s, v30.4s, v12.4s\n" - "smin v25.4s, v25.4s, v12.4s\n" - "smin v20.4s, v20.4s, v12.4s\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" + "smax v29.4s, v29.4s, v13.4s\n" + "smax v28.4s, v28.4s, v13.4s\n" + "smin v31.4s, v31.4s, v11.4s\n" + "smin v30.4s, v30.4s, v11.4s\n" + "smin v29.4s, v29.4s, v11.4s\n" + "smin v28.4s, v28.4s, v11.4s\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" "blt 20f\n" - "str s5, [x24, x27]\n" - "str s30, [x23, x27]\n" - "str s25, [x22, x27]\n" - "str s20, [x21, x27]\n" + "str s31, [x25, x27]\n" + "str s30, [x24, x27]\n" + "str s29, [x23, x27]\n" + "str s28, [x22, x27]\n" "b 23f\n" "20:" // Oddments: Unroll 0: Oddment store + "add x25, x25, x27\n" "add x24, x24, x27\n" "add x23, x23, x27\n" "add x22, x22, x27\n" - "add x21, x21, x27\n" "tbz x20, #1, 21f\n" - "st1 { v5.h }[0], [x24], #0x2\n" - "st1 { v30.h }[0], [x23], #0x2\n" - "st1 { v25.h }[0], [x22], #0x2\n" - "st1 { v20.h }[0], [x21], #0x2\n" + "st1 { v31.h }[0], [x25], #0x2\n" + "st1 { v30.h }[0], [x24], #0x2\n" + "st1 { v29.h }[0], [x23], #0x2\n" + "st1 { v28.h }[0], [x22], #0x2\n" "tbz x20, #0, 22f\n" - "st1 { v5.b }[2], [x24], #0x1\n" - "st1 { v30.b }[2], [x23], #0x1\n" - "st1 { v25.b }[2], [x22], #0x1\n" - "st1 { v20.b }[2], [x21], #0x1\n" + "st1 { v31.b }[2], [x25], #0x1\n" + "st1 { v30.b }[2], [x24], #0x1\n" + "st1 { v29.b }[2], [x23], #0x1\n" + "st1 { v28.b }[2], [x22], #0x1\n" "b 22f\n" "21:" // Oddments: Unroll 0: Oddment store: Bit 1: Unset - "st1 { v5.b }[0], [x24], #0x1\n" - "st1 { v30.b }[0], [x23], #0x1\n" - "st1 { v25.b }[0], [x22], #0x1\n" - "st1 { v20.b }[0], [x21], #0x1\n" + "st1 { v31.b }[0], [x25], #0x1\n" + "st1 { v30.b }[0], [x24], #0x1\n" + "st1 { v29.b }[0], [x23], #0x1\n" + "st1 { v28.b }[0], [x22], #0x1\n" "22:" // Oddments: Unroll 0: Oddment store: Bit 1: End "23:" // Oddments: Unroll 0: After oddment store "subs x20, x20, #0x4\n" "add x27, x27, #0x4\n" "ble 35f\n" - "ldr q5, [%x[params], #0x0]\n" - "ldr q0, [%x[params], #0x10]\n" - "movi v19.4s, #0x0\n" - ".inst 0x4e8395d3 // sdot v19.4s, v14.16b, v3.16b\n" - "ldr q16, [%x[params], #0x20]\n" - "ldr q31, [%x[params], #0x30]\n" - "mov v30.16b, v5.16b\n" - "mov v25.16b, v5.16b\n" - "ldr q9, [%x[params], #0x40]\n" - "ldr q4, [%x[params], #0x50]\n" - "mov v20.16b, v5.16b\n" - ".inst 0x4e889405 // sdot v5.4s, v0.16b, v8.16b\n" - ".inst 0x4e9c95d3 // sdot v19.4s, v14.16b, v28.16b\n" - ".inst 0x4e839419 // sdot v25.4s, v0.16b, v3.16b\n" - "movi v17.4s, #0x0\n" + "ldr q31, [%x[params], #0x0]\n" + "ldr q27, [%x[params], #0x10]\n" + "movi v1.4s, #0x0\n" + ".inst 0x4e979581 // sdot v1.4s, v12.16b, v23.16b\n" + "ldr q26, [%x[params], #0x20]\n" + "ldr q22, [%x[params], #0x30]\n" + "mov v30.16b, v31.16b\n" + "mov v29.16b, v31.16b\n" + "ldr q4, [%x[params], #0x40]\n" + "ldr q21, [%x[params], #0x50]\n" + "mov v28.16b, v31.16b\n" + ".inst 0x4e99977f // sdot v31.4s, v27.16b, v25.16b\n" + ".inst 0x4e939581 // sdot v1.4s, v12.16b, v19.16b\n" + ".inst 0x4e97977d // sdot v29.4s, v27.16b, v23.16b\n" + "movi v20.4s, #0x0\n" "cmp x20, #0x4\n" - ".inst 0x4e839605 // sdot v5.4s, v16.16b, v3.16b\n" - "mov v18.16b, v19.16b\n .inst 0x4e9795d2 // sdot v18.4s, v14.16b, v23.16b\n" - "ext v3.16b, v3.16b, v3.16b, #0x1\n" - "add %x[params], %x[params], #0x60\n" - ".inst 0x4e8895d3 // sdot v19.4s, v14.16b, v8.16b\n" - "ext v8.16b, v8.16b, v8.16b, #0x1\n" - ".inst 0x4e88941e // sdot v30.4s, v0.16b, v8.16b\n" - ".inst 0x4e839414 // sdot v20.4s, v0.16b, v3.16b\n" - ".inst 0x4e8395d1 // sdot v17.4s, v14.16b, v3.16b\n" - ".inst 0x4e9c9619 // sdot v25.4s, v16.16b, v28.16b\n" - ".inst 0x4e9c97e5 // sdot v5.4s, v31.16b, v28.16b\n" - "ext v28.16b, v28.16b, v28.16b, #0x1\n" - ".inst 0x4e83961e // sdot v30.4s, v16.16b, v3.16b\n" - ".inst 0x4e9c9614 // sdot v20.4s, v16.16b, v28.16b\n" - "mls v5.4s, v19.4s, v11.4s\n" - ".inst 0x4e9c95d1 // sdot v17.4s, v14.16b, v28.16b\n" - ".inst 0x4e9797f9 // sdot v25.4s, v31.16b, v23.16b\n" + ".inst 0x4e97975f // sdot v31.4s, v26.16b, v23.16b\n" + "mov v18.16b, v1.16b\n .inst 0x4e989592 // sdot v18.4s, v12.16b, v24.16b\n" "ext v23.16b, v23.16b, v23.16b, #0x1\n" - ".inst 0x4e9c97fe // sdot v30.4s, v31.16b, v28.16b\n" - ".inst 0x4e9797f4 // sdot v20.4s, v31.16b, v23.16b\n" - "sqrdmulh v5.4s, v5.4s, v9.4s\n" - "mov v16.16b, v17.16b\n .inst 0x4e9795d0 // sdot v16.4s, v14.16b, v23.16b\n" - ".inst 0x4e8895d1 // sdot v17.4s, v14.16b, v8.16b\n" - "mls v30.4s, v17.4s, v11.4s\n" - "mls v25.4s, v18.4s, v11.4s\n" - "mls v20.4s, v16.4s, v11.4s\n" - "and v0.16b, v5.16b, v4.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqrdmulh v30.4s, v30.4s, v9.4s\n" - "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqrdmulh v20.4s, v20.4s, v9.4s\n" - "sqadd v5.4s, v5.4s, v0.4s\n" - "and v16.16b, v30.16b, v4.16b\n" - "and v31.16b, v25.16b, v4.16b\n" - "and v0.16b, v20.16b, v4.16b\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqadd v30.4s, v30.4s, v16.4s\n" - "sqadd v25.4s, v25.4s, v31.4s\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "srshl v5.4s, v5.4s, v4.4s\n" - "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" - "srshl v20.4s, v20.4s, v4.4s\n" - "add v5.4s, v5.4s, v10.4s\n" - "add v30.4s, v30.4s, v10.4s\n" - "add v25.4s, v25.4s, v10.4s\n" - "add v20.4s, v20.4s, v10.4s\n" - "smax v5.4s, v5.4s, v13.4s\n" + "add %x[params], %x[params], #0x60\n" + ".inst 0x4e999581 // sdot v1.4s, v12.16b, v25.16b\n" + "ext v25.16b, v25.16b, v25.16b, #0x1\n" + ".inst 0x4e99977e // sdot v30.4s, v27.16b, v25.16b\n" + ".inst 0x4e97977c // sdot v28.4s, v27.16b, v23.16b\n" + ".inst 0x4e979594 // sdot v20.4s, v12.16b, v23.16b\n" + ".inst 0x4e93975d // sdot v29.4s, v26.16b, v19.16b\n" + ".inst 0x4e9396df // sdot v31.4s, v22.16b, v19.16b\n" + "ext v19.16b, v19.16b, v19.16b, #0x1\n" + ".inst 0x4e97975e // sdot v30.4s, v26.16b, v23.16b\n" + ".inst 0x4e93975c // sdot v28.4s, v26.16b, v19.16b\n" + "mls v31.4s, v1.4s, v16.4s\n" + ".inst 0x4e939594 // sdot v20.4s, v12.16b, v19.16b\n" + ".inst 0x4e9896dd // sdot v29.4s, v22.16b, v24.16b\n" + "ext v24.16b, v24.16b, v24.16b, #0x1\n" + ".inst 0x4e9396de // sdot v30.4s, v22.16b, v19.16b\n" + ".inst 0x4e9896dc // sdot v28.4s, v22.16b, v24.16b\n" + "sqrdmulh v31.4s, v31.4s, v4.4s\n" + "mov v17.16b, v20.16b\n .inst 0x4e989591 // sdot v17.4s, v12.16b, v24.16b\n" + ".inst 0x4e999594 // sdot v20.4s, v12.16b, v25.16b\n" + "mls v30.4s, v20.4s, v16.4s\n" + "mls v29.4s, v18.4s, v16.4s\n" + "mls v28.4s, v17.4s, v16.4s\n" + "and v17.16b, v31.16b, v21.16b\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqrdmulh v30.4s, v30.4s, v4.4s\n" + "sqrdmulh v29.4s, v29.4s, v4.4s\n" + "sqrdmulh v28.4s, v28.4s, v4.4s\n" + "sqadd v31.4s, v31.4s, v17.4s\n" + "and v19.16b, v30.16b, v21.16b\n" + "and v18.16b, v29.16b, v21.16b\n" + "and v17.16b, v28.16b, v21.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqadd v30.4s, v30.4s, v19.4s\n" + "sqadd v29.4s, v29.4s, v18.4s\n" + "sqadd v28.4s, v28.4s, v17.4s\n" + "srshl v31.4s, v31.4s, v21.4s\n" + "srshl v30.4s, v30.4s, v21.4s\n" + "srshl v29.4s, v29.4s, v21.4s\n" + "srshl v28.4s, v28.4s, v21.4s\n" + "add v31.4s, v31.4s, v14.4s\n" + "add v30.4s, v30.4s, v14.4s\n" + "add v29.4s, v29.4s, v14.4s\n" + "add v28.4s, v28.4s, v14.4s\n" + "smax v31.4s, v31.4s, v13.4s\n" "smax v30.4s, v30.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smin v5.4s, v5.4s, v12.4s\n" - "smin v30.4s, v30.4s, v12.4s\n" - "smin v25.4s, v25.4s, v12.4s\n" - "smin v20.4s, v20.4s, v12.4s\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" + "smax v29.4s, v29.4s, v13.4s\n" + "smax v28.4s, v28.4s, v13.4s\n" + "smin v31.4s, v31.4s, v11.4s\n" + "smin v30.4s, v30.4s, v11.4s\n" + "smin v29.4s, v29.4s, v11.4s\n" + "smin v28.4s, v28.4s, v11.4s\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" "blt 24f\n" - "str s5, [x24, x27]\n" - "str s30, [x23, x27]\n" - "str s25, [x22, x27]\n" - "str s20, [x21, x27]\n" + "str s31, [x25, x27]\n" + "str s30, [x24, x27]\n" + "str s29, [x23, x27]\n" + "str s28, [x22, x27]\n" "b 27f\n" "24:" // Oddments: Unroll 1: Oddment store + "add x25, x25, x27\n" "add x24, x24, x27\n" "add x23, x23, x27\n" "add x22, x22, x27\n" - "add x21, x21, x27\n" "tbz x20, #1, 25f\n" - "st1 { v5.h }[0], [x24], #0x2\n" - "st1 { v30.h }[0], [x23], #0x2\n" - "st1 { v25.h }[0], [x22], #0x2\n" - "st1 { v20.h }[0], [x21], #0x2\n" + "st1 { v31.h }[0], [x25], #0x2\n" + "st1 { v30.h }[0], [x24], #0x2\n" + "st1 { v29.h }[0], [x23], #0x2\n" + "st1 { v28.h }[0], [x22], #0x2\n" "tbz x20, #0, 26f\n" - "st1 { v5.b }[2], [x24], #0x1\n" - "st1 { v30.b }[2], [x23], #0x1\n" - "st1 { v25.b }[2], [x22], #0x1\n" - "st1 { v20.b }[2], [x21], #0x1\n" + "st1 { v31.b }[2], [x25], #0x1\n" + "st1 { v30.b }[2], [x24], #0x1\n" + "st1 { v29.b }[2], [x23], #0x1\n" + "st1 { v28.b }[2], [x22], #0x1\n" "b 26f\n" "25:" // Oddments: Unroll 1: Oddment store: Bit 1: Unset - "st1 { v5.b }[0], [x24], #0x1\n" - "st1 { v30.b }[0], [x23], #0x1\n" - "st1 { v25.b }[0], [x22], #0x1\n" - "st1 { v20.b }[0], [x21], #0x1\n" + "st1 { v31.b }[0], [x25], #0x1\n" + "st1 { v30.b }[0], [x24], #0x1\n" + "st1 { v29.b }[0], [x23], #0x1\n" + "st1 { v28.b }[0], [x22], #0x1\n" "26:" // Oddments: Unroll 1: Oddment store: Bit 1: End "27:" // Oddments: Unroll 1: After oddment store "subs x20, x20, #0x4\n" "add x27, x27, #0x4\n" "ble 35f\n" - "ldr q5, [%x[params], #0x0]\n" - "ldr q0, [%x[params], #0x10]\n" + "ldr q31, [%x[params], #0x0]\n" + "ldr q25, [%x[params], #0x10]\n" + "movi v24.4s, #0x0\n" + ".inst 0x4e839598 // sdot v24.4s, v12.16b, v3.16b\n" + "ldr q23, [%x[params], #0x20]\n" + "ldr q22, [%x[params], #0x30]\n" + "mov v30.16b, v31.16b\n" + "mov v29.16b, v31.16b\n" + "ldr q21, [%x[params], #0x40]\n" + "ldr q20, [%x[params], #0x50]\n" + "mov v28.16b, v31.16b\n" + ".inst 0x4e87973f // sdot v31.4s, v25.16b, v7.16b\n" + ".inst 0x4e809598 // sdot v24.4s, v12.16b, v0.16b\n" + ".inst 0x4e83973d // sdot v29.4s, v25.16b, v3.16b\n" "movi v19.4s, #0x0\n" - ".inst 0x4e8295d3 // sdot v19.4s, v14.16b, v2.16b\n" - "ldr q16, [%x[params], #0x20]\n" - "ldr q31, [%x[params], #0x30]\n" - "mov v30.16b, v5.16b\n" - "mov v25.16b, v5.16b\n" - "ldr q9, [%x[params], #0x40]\n" - "ldr q4, [%x[params], #0x50]\n" - "mov v20.16b, v5.16b\n" - ".inst 0x4e879405 // sdot v5.4s, v0.16b, v7.16b\n" - ".inst 0x4e9b95d3 // sdot v19.4s, v14.16b, v27.16b\n" - ".inst 0x4e829419 // sdot v25.4s, v0.16b, v2.16b\n" - "movi v17.4s, #0x0\n" "cmp x20, #0x4\n" - ".inst 0x4e829605 // sdot v5.4s, v16.16b, v2.16b\n" - "mov v18.16b, v19.16b\n .inst 0x4e9695d2 // sdot v18.4s, v14.16b, v22.16b\n" - "ext v2.16b, v2.16b, v2.16b, #0x1\n" + ".inst 0x4e8396ff // sdot v31.4s, v23.16b, v3.16b\n" + "mov v18.16b, v24.16b\n .inst 0x4e829592 // sdot v18.4s, v12.16b, v2.16b\n" + "ext v3.16b, v3.16b, v3.16b, #0x1\n" "add %x[params], %x[params], #0x60\n" - ".inst 0x4e8795d3 // sdot v19.4s, v14.16b, v7.16b\n" + ".inst 0x4e879598 // sdot v24.4s, v12.16b, v7.16b\n" "ext v7.16b, v7.16b, v7.16b, #0x1\n" - ".inst 0x4e87941e // sdot v30.4s, v0.16b, v7.16b\n" - ".inst 0x4e829414 // sdot v20.4s, v0.16b, v2.16b\n" - ".inst 0x4e8295d1 // sdot v17.4s, v14.16b, v2.16b\n" - ".inst 0x4e9b9619 // sdot v25.4s, v16.16b, v27.16b\n" - ".inst 0x4e9b97e5 // sdot v5.4s, v31.16b, v27.16b\n" - "ext v27.16b, v27.16b, v27.16b, #0x1\n" - ".inst 0x4e82961e // sdot v30.4s, v16.16b, v2.16b\n" - ".inst 0x4e9b9614 // sdot v20.4s, v16.16b, v27.16b\n" - "mls v5.4s, v19.4s, v11.4s\n" - ".inst 0x4e9b95d1 // sdot v17.4s, v14.16b, v27.16b\n" - ".inst 0x4e9697f9 // sdot v25.4s, v31.16b, v22.16b\n" - "ext v22.16b, v22.16b, v22.16b, #0x1\n" - ".inst 0x4e9b97fe // sdot v30.4s, v31.16b, v27.16b\n" - ".inst 0x4e9697f4 // sdot v20.4s, v31.16b, v22.16b\n" - "sqrdmulh v5.4s, v5.4s, v9.4s\n" - "mov v16.16b, v17.16b\n .inst 0x4e9695d0 // sdot v16.4s, v14.16b, v22.16b\n" - ".inst 0x4e8795d1 // sdot v17.4s, v14.16b, v7.16b\n" - "mls v30.4s, v17.4s, v11.4s\n" - "mls v25.4s, v18.4s, v11.4s\n" - "mls v20.4s, v16.4s, v11.4s\n" - "and v0.16b, v5.16b, v4.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqrdmulh v30.4s, v30.4s, v9.4s\n" - "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqrdmulh v20.4s, v20.4s, v9.4s\n" - "sqadd v5.4s, v5.4s, v0.4s\n" - "and v16.16b, v30.16b, v4.16b\n" - "and v31.16b, v25.16b, v4.16b\n" - "and v0.16b, v20.16b, v4.16b\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqadd v30.4s, v30.4s, v16.4s\n" - "sqadd v25.4s, v25.4s, v31.4s\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "srshl v5.4s, v5.4s, v4.4s\n" - "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" - "srshl v20.4s, v20.4s, v4.4s\n" - "add v5.4s, v5.4s, v10.4s\n" - "add v30.4s, v30.4s, v10.4s\n" - "add v25.4s, v25.4s, v10.4s\n" - "add v20.4s, v20.4s, v10.4s\n" - "smax v5.4s, v5.4s, v13.4s\n" + ".inst 0x4e87973e // sdot v30.4s, v25.16b, v7.16b\n" + ".inst 0x4e83973c // sdot v28.4s, v25.16b, v3.16b\n" + ".inst 0x4e839593 // sdot v19.4s, v12.16b, v3.16b\n" + ".inst 0x4e8096fd // sdot v29.4s, v23.16b, v0.16b\n" + ".inst 0x4e8096df // sdot v31.4s, v22.16b, v0.16b\n" + "ext v0.16b, v0.16b, v0.16b, #0x1\n" + ".inst 0x4e8396fe // sdot v30.4s, v23.16b, v3.16b\n" + ".inst 0x4e8096fc // sdot v28.4s, v23.16b, v0.16b\n" + "mls v31.4s, v24.4s, v16.4s\n" + ".inst 0x4e809593 // sdot v19.4s, v12.16b, v0.16b\n" + ".inst 0x4e8296dd // sdot v29.4s, v22.16b, v2.16b\n" + "ext v2.16b, v2.16b, v2.16b, #0x1\n" + ".inst 0x4e8096de // sdot v30.4s, v22.16b, v0.16b\n" + ".inst 0x4e8296dc // sdot v28.4s, v22.16b, v2.16b\n" + "sqrdmulh v31.4s, v31.4s, v21.4s\n" + "mov v17.16b, v19.16b\n .inst 0x4e829591 // sdot v17.4s, v12.16b, v2.16b\n" + ".inst 0x4e879593 // sdot v19.4s, v12.16b, v7.16b\n" + "mls v30.4s, v19.4s, v16.4s\n" + "mls v29.4s, v18.4s, v16.4s\n" + "mls v28.4s, v17.4s, v16.4s\n" + "and v17.16b, v31.16b, v20.16b\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqrdmulh v30.4s, v30.4s, v21.4s\n" + "sqrdmulh v29.4s, v29.4s, v21.4s\n" + "sqrdmulh v28.4s, v28.4s, v21.4s\n" + "sqadd v31.4s, v31.4s, v17.4s\n" + "and v19.16b, v30.16b, v20.16b\n" + "and v18.16b, v29.16b, v20.16b\n" + "and v17.16b, v28.16b, v20.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqadd v30.4s, v30.4s, v19.4s\n" + "sqadd v29.4s, v29.4s, v18.4s\n" + "sqadd v28.4s, v28.4s, v17.4s\n" + "srshl v31.4s, v31.4s, v20.4s\n" + "srshl v30.4s, v30.4s, v20.4s\n" + "srshl v29.4s, v29.4s, v20.4s\n" + "srshl v28.4s, v28.4s, v20.4s\n" + "add v31.4s, v31.4s, v14.4s\n" + "add v30.4s, v30.4s, v14.4s\n" + "add v29.4s, v29.4s, v14.4s\n" + "add v28.4s, v28.4s, v14.4s\n" + "smax v31.4s, v31.4s, v13.4s\n" "smax v30.4s, v30.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smin v5.4s, v5.4s, v12.4s\n" - "smin v30.4s, v30.4s, v12.4s\n" - "smin v25.4s, v25.4s, v12.4s\n" - "smin v20.4s, v20.4s, v12.4s\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" + "smax v29.4s, v29.4s, v13.4s\n" + "smax v28.4s, v28.4s, v13.4s\n" + "smin v31.4s, v31.4s, v11.4s\n" + "smin v30.4s, v30.4s, v11.4s\n" + "smin v29.4s, v29.4s, v11.4s\n" + "smin v28.4s, v28.4s, v11.4s\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" "blt 28f\n" - "str s5, [x24, x27]\n" - "str s30, [x23, x27]\n" - "str s25, [x22, x27]\n" - "str s20, [x21, x27]\n" + "str s31, [x25, x27]\n" + "str s30, [x24, x27]\n" + "str s29, [x23, x27]\n" + "str s28, [x22, x27]\n" "b 31f\n" "28:" // Oddments: Unroll 2: Oddment store + "add x25, x25, x27\n" "add x24, x24, x27\n" "add x23, x23, x27\n" "add x22, x22, x27\n" - "add x21, x21, x27\n" "tbz x20, #1, 29f\n" - "st1 { v5.h }[0], [x24], #0x2\n" - "st1 { v30.h }[0], [x23], #0x2\n" - "st1 { v25.h }[0], [x22], #0x2\n" - "st1 { v20.h }[0], [x21], #0x2\n" + "st1 { v31.h }[0], [x25], #0x2\n" + "st1 { v30.h }[0], [x24], #0x2\n" + "st1 { v29.h }[0], [x23], #0x2\n" + "st1 { v28.h }[0], [x22], #0x2\n" "tbz x20, #0, 30f\n" - "st1 { v5.b }[2], [x24], #0x1\n" - "st1 { v30.b }[2], [x23], #0x1\n" - "st1 { v25.b }[2], [x22], #0x1\n" - "st1 { v20.b }[2], [x21], #0x1\n" + "st1 { v31.b }[2], [x25], #0x1\n" + "st1 { v30.b }[2], [x24], #0x1\n" + "st1 { v29.b }[2], [x23], #0x1\n" + "st1 { v28.b }[2], [x22], #0x1\n" "b 30f\n" "29:" // Oddments: Unroll 2: Oddment store: Bit 1: Unset - "st1 { v5.b }[0], [x24], #0x1\n" - "st1 { v30.b }[0], [x23], #0x1\n" - "st1 { v25.b }[0], [x22], #0x1\n" - "st1 { v20.b }[0], [x21], #0x1\n" + "st1 { v31.b }[0], [x25], #0x1\n" + "st1 { v30.b }[0], [x24], #0x1\n" + "st1 { v29.b }[0], [x23], #0x1\n" + "st1 { v28.b }[0], [x22], #0x1\n" "30:" // Oddments: Unroll 2: Oddment store: Bit 1: End "31:" // Oddments: Unroll 2: After oddment store "subs x20, x20, #0x4\n" "add x27, x27, #0x4\n" "ble 35f\n" - "ldr q5, [%x[params], #0x0]\n" - "ldr q0, [%x[params], #0x10]\n" - "movi v19.4s, #0x0\n" - ".inst 0x4e8195d3 // sdot v19.4s, v14.16b, v1.16b\n" - "ldr q16, [%x[params], #0x20]\n" - "ldr q31, [%x[params], #0x30]\n" - "mov v30.16b, v5.16b\n" - "mov v25.16b, v5.16b\n" - "ldr q9, [%x[params], #0x40]\n" - "ldr q4, [%x[params], #0x50]\n" - "mov v20.16b, v5.16b\n" - ".inst 0x4e869405 // sdot v5.4s, v0.16b, v6.16b\n" - ".inst 0x4e9a95d3 // sdot v19.4s, v14.16b, v26.16b\n" - ".inst 0x4e819419 // sdot v25.4s, v0.16b, v1.16b\n" - "movi v17.4s, #0x0\n" + "ldr q31, [%x[params], #0x0]\n" + "ldr q23, [%x[params], #0x10]\n" + "movi v22.4s, #0x0\n" + ".inst 0x4e8a9596 // sdot v22.4s, v12.16b, v10.16b\n" + "ldr q21, [%x[params], #0x20]\n" + "ldr q19, [%x[params], #0x30]\n" + "mov v30.16b, v31.16b\n" + "mov v29.16b, v31.16b\n" + "ldr q20, [%x[params], #0x40]\n" + "ldr q26, [%x[params], #0x50]\n" + "mov v28.16b, v31.16b\n" + ".inst 0x4e8896ff // sdot v31.4s, v23.16b, v8.16b\n" + ".inst 0x4e859596 // sdot v22.4s, v12.16b, v5.16b\n" + ".inst 0x4e8a96fd // sdot v29.4s, v23.16b, v10.16b\n" + "movi v18.4s, #0x0\n" "add %x[params], %x[params], #0x60\n" - ".inst 0x4e819605 // sdot v5.4s, v16.16b, v1.16b\n" - "mov v18.16b, v19.16b\n .inst 0x4e9595d2 // sdot v18.4s, v14.16b, v21.16b\n" - "ext v1.16b, v1.16b, v1.16b, #0x1\n" - ".inst 0x4e8695d3 // sdot v19.4s, v14.16b, v6.16b\n" - "ext v6.16b, v6.16b, v6.16b, #0x1\n" - ".inst 0x4e86941e // sdot v30.4s, v0.16b, v6.16b\n" - ".inst 0x4e819414 // sdot v20.4s, v0.16b, v1.16b\n" - ".inst 0x4e8195d1 // sdot v17.4s, v14.16b, v1.16b\n" - ".inst 0x4e9a9619 // sdot v25.4s, v16.16b, v26.16b\n" - ".inst 0x4e9a97e5 // sdot v5.4s, v31.16b, v26.16b\n" - "ext v26.16b, v26.16b, v26.16b, #0x1\n" - ".inst 0x4e81961e // sdot v30.4s, v16.16b, v1.16b\n" - ".inst 0x4e9a9614 // sdot v20.4s, v16.16b, v26.16b\n" - "mls v5.4s, v19.4s, v11.4s\n" - ".inst 0x4e9a95d1 // sdot v17.4s, v14.16b, v26.16b\n" - ".inst 0x4e9597f9 // sdot v25.4s, v31.16b, v21.16b\n" - "ext v21.16b, v21.16b, v21.16b, #0x1\n" - ".inst 0x4e9a97fe // sdot v30.4s, v31.16b, v26.16b\n" - ".inst 0x4e9597f4 // sdot v20.4s, v31.16b, v21.16b\n" - "sqrdmulh v5.4s, v5.4s, v9.4s\n" - "mov v16.16b, v17.16b\n .inst 0x4e9595d0 // sdot v16.4s, v14.16b, v21.16b\n" - ".inst 0x4e8695d1 // sdot v17.4s, v14.16b, v6.16b\n" - "mls v30.4s, v17.4s, v11.4s\n" - "mls v25.4s, v18.4s, v11.4s\n" - "mls v20.4s, v16.4s, v11.4s\n" - "and v0.16b, v5.16b, v4.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqrdmulh v30.4s, v30.4s, v9.4s\n" - "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqrdmulh v20.4s, v20.4s, v9.4s\n" - "sqadd v5.4s, v5.4s, v0.4s\n" - "and v16.16b, v30.16b, v4.16b\n" - "and v31.16b, v25.16b, v4.16b\n" - "and v0.16b, v20.16b, v4.16b\n" + ".inst 0x4e8a96bf // sdot v31.4s, v21.16b, v10.16b\n" + "mov v17.16b, v22.16b\n .inst 0x4e899591 // sdot v17.4s, v12.16b, v9.16b\n" + "ext v10.16b, v10.16b, v10.16b, #0x1\n" + ".inst 0x4e889596 // sdot v22.4s, v12.16b, v8.16b\n" + "ext v8.16b, v8.16b, v8.16b, #0x1\n" + ".inst 0x4e8896fe // sdot v30.4s, v23.16b, v8.16b\n" + ".inst 0x4e8a96fc // sdot v28.4s, v23.16b, v10.16b\n" + ".inst 0x4e8a9592 // sdot v18.4s, v12.16b, v10.16b\n" + ".inst 0x4e8596bd // sdot v29.4s, v21.16b, v5.16b\n" + ".inst 0x4e85967f // sdot v31.4s, v19.16b, v5.16b\n" + "ext v5.16b, v5.16b, v5.16b, #0x1\n" + ".inst 0x4e8a96be // sdot v30.4s, v21.16b, v10.16b\n" + ".inst 0x4e8596bc // sdot v28.4s, v21.16b, v5.16b\n" + "mls v31.4s, v22.4s, v16.4s\n" + ".inst 0x4e859592 // sdot v18.4s, v12.16b, v5.16b\n" + ".inst 0x4e89967d // sdot v29.4s, v19.16b, v9.16b\n" + "ext v9.16b, v9.16b, v9.16b, #0x1\n" + ".inst 0x4e85967e // sdot v30.4s, v19.16b, v5.16b\n" + ".inst 0x4e89967c // sdot v28.4s, v19.16b, v9.16b\n" + "sqrdmulh v31.4s, v31.4s, v20.4s\n" + "mov v7.16b, v18.16b\n .inst 0x4e899587 // sdot v7.4s, v12.16b, v9.16b\n" + ".inst 0x4e889592 // sdot v18.4s, v12.16b, v8.16b\n" + "mls v30.4s, v18.4s, v16.4s\n" + "mls v29.4s, v17.4s, v16.4s\n" + "mls v28.4s, v7.4s, v16.4s\n" + "and v16.16b, v31.16b, v26.16b\n" "sshr v16.4s, v16.4s, #0x1f\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqadd v30.4s, v30.4s, v16.4s\n" - "sqadd v25.4s, v25.4s, v31.4s\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "srshl v5.4s, v5.4s, v4.4s\n" - "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" - "srshl v20.4s, v20.4s, v4.4s\n" - "add v5.4s, v5.4s, v10.4s\n" - "add v30.4s, v30.4s, v10.4s\n" - "add v25.4s, v25.4s, v10.4s\n" - "add v20.4s, v20.4s, v10.4s\n" - "smax v5.4s, v5.4s, v13.4s\n" + "sqrdmulh v30.4s, v30.4s, v20.4s\n" + "sqrdmulh v29.4s, v29.4s, v20.4s\n" + "sqrdmulh v28.4s, v28.4s, v20.4s\n" + "sqadd v31.4s, v31.4s, v16.4s\n" + "and v18.16b, v30.16b, v26.16b\n" + "and v17.16b, v29.16b, v26.16b\n" + "and v16.16b, v28.16b, v26.16b\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v30.4s, v30.4s, v18.4s\n" + "sqadd v29.4s, v29.4s, v17.4s\n" + "sqadd v28.4s, v28.4s, v16.4s\n" + "srshl v31.4s, v31.4s, v26.4s\n" + "srshl v30.4s, v30.4s, v26.4s\n" + "srshl v29.4s, v29.4s, v26.4s\n" + "srshl v28.4s, v28.4s, v26.4s\n" + "add v31.4s, v31.4s, v14.4s\n" + "add v30.4s, v30.4s, v14.4s\n" + "add v29.4s, v29.4s, v14.4s\n" + "add v28.4s, v28.4s, v14.4s\n" + "smax v31.4s, v31.4s, v13.4s\n" "smax v30.4s, v30.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smin v5.4s, v5.4s, v12.4s\n" - "smin v30.4s, v30.4s, v12.4s\n" - "smin v25.4s, v25.4s, v12.4s\n" - "smin v20.4s, v20.4s, v12.4s\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" + "smax v29.4s, v29.4s, v13.4s\n" + "smax v28.4s, v28.4s, v13.4s\n" + "smin v31.4s, v31.4s, v11.4s\n" + "smin v30.4s, v30.4s, v11.4s\n" + "smin v29.4s, v29.4s, v11.4s\n" + "smin v28.4s, v28.4s, v11.4s\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" "32:" // Oddments: Unroll 3: Oddment store + "add x25, x25, x27\n" "add x24, x24, x27\n" "add x23, x23, x27\n" "add x22, x22, x27\n" - "add x21, x21, x27\n" "tbz x20, #1, 33f\n" - "st1 { v5.h }[0], [x24], #0x2\n" - "st1 { v30.h }[0], [x23], #0x2\n" - "st1 { v25.h }[0], [x22], #0x2\n" - "st1 { v20.h }[0], [x21], #0x2\n" + "st1 { v31.h }[0], [x25], #0x2\n" + "st1 { v30.h }[0], [x24], #0x2\n" + "st1 { v29.h }[0], [x23], #0x2\n" + "st1 { v28.h }[0], [x22], #0x2\n" "tbz x20, #0, 34f\n" - "st1 { v5.b }[2], [x24], #0x1\n" - "st1 { v30.b }[2], [x23], #0x1\n" - "st1 { v25.b }[2], [x22], #0x1\n" - "st1 { v20.b }[2], [x21], #0x1\n" + "st1 { v31.b }[2], [x25], #0x1\n" + "st1 { v30.b }[2], [x24], #0x1\n" + "st1 { v29.b }[2], [x23], #0x1\n" + "st1 { v28.b }[2], [x22], #0x1\n" "b 34f\n" "33:" // Oddments: Unroll 3: Oddment store: Bit 1: Unset - "st1 { v5.b }[0], [x24], #0x1\n" - "st1 { v30.b }[0], [x23], #0x1\n" - "st1 { v25.b }[0], [x22], #0x1\n" - "st1 { v20.b }[0], [x21], #0x1\n" + "st1 { v31.b }[0], [x25], #0x1\n" + "st1 { v30.b }[0], [x24], #0x1\n" + "st1 { v29.b }[0], [x23], #0x1\n" + "st1 { v28.b }[0], [x22], #0x1\n" "34:" // Oddments: Unroll 3: Oddment store: Bit 1: End "35:" // End : [params] "+&r" (params) : [inptrs] "r" (inptrs), [n_channels] "r" (n_channels), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [outptrs] "r" (outptrs), [qp] "r" (&qp) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp index 411b4788d813f339931be6fac8b0601d9c064f65..a679b02f7c8933cb248379d9647e5f66824c9ce0 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include @@ -34,15 +34,7 @@ namespace arm_conv { namespace depthwise { -void a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( - const unsigned int, - const int8_t *const *const, - const int8_t *const, - const int32_t *const, - const arm_gemm::Requantize32 &, - const int32_t *const, - const int32_t *const, - int8_t *const *const); +void a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl(unsigned int, const int8_t *const *, const int8_t *, const int32_t *, const arm_gemm::Requantize32 &, const int32_t *, const int32_t *, int8_t *const *); class a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirstStrategy { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp index d69d0e1ef2d2972ca3bd6a9abd95373ee094728a..a181603f1edf1ce83121f9fcd688cb3fdfbd6cde 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp @@ -91,1072 +91,1072 @@ void a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( requant_muls, requant_shifts, outptrs); __asm__ __volatile__( - "ldr x6, [%x[params], %[offsetof_Params_n_channels]]\n" + "ldr x7, [%x[params], %[offsetof_Params_n_channels]]\n" "ldr x23, [%x[params], %[offsetof_Params_requant]]\n" - "lsr x7, x6, #0x3\n" + "lsr x8, x7, #0x3\n" "add x20, x23, %[offsetof_Requantize32_a_offset]\n" - "ld1r { v24.16b }, [x20]\n" + "ld1r { v14.16b }, [x20]\n" "ldr x22, [%x[params], %[offsetof_Params_outptrs]]\n" "add x21, x23, %[offsetof_Requantize32_b_offset]\n" "add x20, x23, %[offsetof_Requantize32_c_offset]\n" - "ld1r { v15.16b }, [x21]\n" - "ld1r { v14.8h }, [x20]\n" + "ld1r { v19.16b }, [x21]\n" + "ld1r { v13.8h }, [x20]\n" "add x21, x23, %[offsetof_Requantize32_minval]\n" "add x20, x23, %[offsetof_Requantize32_maxval]\n" - "ld1r { v12.8h }, [x21]\n" - "ld1r { v11.8h }, [x20]\n" - "mov x8, #0x0\n" + "ld1r { v29.8h }, [x21]\n" + "ld1r { v12.8h }, [x20]\n" "mov x17, #0x0\n" - "add x16, %x[params], %[offsetof_Params_inptrs]\n" - "ldr x15, [%x[params], %[offsetof_Params_weights]]\n" - "ldr x14, [%x[params], %[offsetof_Params_requant_muls]]\n" - "ldr x13, [%x[params], %[offsetof_Params_requant_shifts]]\n" - "ldp x12, x11, [x22, #0x0]\n" - "ldp x10, x9, [x22, #0x10]\n" - "cbz x7, 3f\n" - "ldr d0, [x15, #0x0]\n" - "ldr d1, [x15, #0x8]\n" - "subs x7, x7, #0x1\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "ldr d2, [x15, #0x10]\n" - "ldr d3, [x15, #0x18]\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "ldr d4, [x15, #0x20]\n" - "ldr d5, [x15, #0x28]\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "ldr d6, [x15, #0x30]\n" - "ldr d7, [x15, #0x38]\n" - "ssubl v5.8h, v5.8b, v15.8b\n" - "ssubl v6.8h, v6.8b, v15.8b\n" - "ldr d8, [x15, #0x40]\n" - "ldr x28, [%x[params], %[offsetof_Params_bias]]\n" - "ssubl v7.8h, v7.8b, v15.8b\n" - "ssubl v8.8h, v8.8b, v15.8b\n" - "ldr q13, [x28, #0x0]\n" - "ldr q20, [x28, #0x10]\n" - "add x28, x28, #0x20\n" - "str x28, [%x[params], %[offsetof_Params_bias]]\n" - "ldp x24, x23, [x16, #0x0]\n" - "ldp x22, x21, [x16, #0x10]\n" - "mov v9.16b, v13.16b\n" - "mov v18.16b, v20.16b\n" - "ldr d31, [x24, x8]\n" - "ldr d30, [x23, x8]\n" - "mov v16.16b, v13.16b\n" - "mov v26.16b, v20.16b\n" - "ldr d29, [x22, x8]\n" - "ldr d28, [x21, x8]\n" - "mov v25.16b, v13.16b\n" - "mov v10.16b, v20.16b\n" - "ldr x20, [x16, #0x20]\n" - "ldr d27, [x20, x8]\n" - "ssubl v31.8h, v31.8b, v24.8b\n" - "ssubl v30.8h, v30.8b, v24.8b\n" - "ssubl v29.8h, v29.8b, v24.8b\n" - "ssubl v28.8h, v28.8b, v24.8b\n" - "ssubl v27.8h, v27.8b, v24.8b\n" + "mov x16, #0x0\n" + "add x15, %x[params], %[offsetof_Params_inptrs]\n" + "ldr x14, [%x[params], %[offsetof_Params_weights]]\n" + "ldr x13, [%x[params], %[offsetof_Params_requant_muls]]\n" + "ldr x12, [%x[params], %[offsetof_Params_requant_shifts]]\n" + "ldp x11, x10, [x22, #0x0]\n" + "ldp x9, x28, [x22, #0x10]\n" + "cbz x8, 3f\n" + "ldr d23, [x14, #0x0]\n" + "ldr d16, [x14, #0x8]\n" + "subs x8, x8, #0x1\n" + "ssubl v23.8h, v23.8b, v19.8b\n" + "ldr d1, [x14, #0x10]\n" + "ldr d5, [x14, #0x18]\n" + "ssubl v16.8h, v16.8b, v19.8b\n" + "ssubl v1.8h, v1.8b, v19.8b\n" + "ldr d26, [x14, #0x20]\n" + "ldr d18, [x14, #0x28]\n" + "ssubl v5.8h, v5.8b, v19.8b\n" + "ssubl v26.8h, v26.8b, v19.8b\n" + "ldr d31, [x14, #0x30]\n" + "ldr d25, [x14, #0x38]\n" + "ssubl v18.8h, v18.8b, v19.8b\n" + "ssubl v31.8h, v31.8b, v19.8b\n" + "ldr d20, [x14, #0x40]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "ssubl v25.8h, v25.8b, v19.8b\n" + "ssubl v20.8h, v20.8b, v19.8b\n" + "ldr q9, [x20, #0x0]\n" + "ldr q24, [x20, #0x10]\n" + "add x20, x20, #0x20\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" + "ldp x23, x22, [x15, #0x0]\n" + "ldp x21, x20, [x15, #0x10]\n" + "mov v7.16b, v9.16b\n" + "mov v0.16b, v24.16b\n" + "ldr d22, [x23, x17]\n" + "ldr d4, [x22, x17]\n" + "mov v2.16b, v9.16b\n" + "mov v30.16b, v24.16b\n" + "ldr d8, [x21, x17]\n" + "ldr d27, [x20, x17]\n" + "mov v10.16b, v9.16b\n" + "mov v6.16b, v24.16b\n" + "ldr x20, [x15, #0x20]\n" + "ldr d15, [x20, x17]\n" + "ssubl v22.8h, v22.8b, v14.8b\n" + "ssubl v4.8h, v4.8b, v14.8b\n" + "ssubl v8.8h, v8.8b, v14.8b\n" + "ssubl v27.8h, v27.8b, v14.8b\n" + "ssubl v15.8h, v15.8b, v14.8b\n" "beq 2f\n" "1:" // Loop - "ldr q17, [x14, #0x0]\n" - "ldr q22, [x13, #0x0]\n" - "smlal v13.4s, v31.4h, v4.4h\n" - "smlal2 v20.4s, v31.8h, v4.8h\n" - "ldr q23, [x14, #0x10]\n" - "smlal v9.4s, v31.4h, v3.4h\n" - "smlal2 v18.4s, v31.8h, v3.8h\n" - "ldr x21, [x16, #0x28]\n" - "smlal v13.4s, v30.4h, v0.4h\n" - "smlal2 v20.4s, v30.8h, v0.8h\n" - "ldr q19, [x13, #0x10]\n" - "ldr x28, [x16, #0x38]\n" - "smlal v9.4s, v29.4h, v2.4h\n" - "smlal2 v18.4s, v29.8h, v2.8h\n" - "ldr x20, [x16, #0x30]\n" - "ldr d29, [x20, x8]\n" - "smlal v16.4s, v31.4h, v1.4h\n" - "smlal2 v26.4s, v31.8h, v1.8h\n" - "ldr x27, [x16, #0x40]\n" - "ldr x26, [x16, #0x48]\n" - "smlal v25.4s, v31.4h, v0.4h\n" - "smlal2 v10.4s, v31.8h, v0.8h\n" - "ldr d31, [x21, x8]\n" - "ssubl v31.8h, v31.8b, v24.8b\n" - "smlal v13.4s, v28.4h, v5.4h\n" - "smlal2 v20.4s, v28.8h, v5.8h\n" - "ssubl v29.8h, v29.8b, v24.8b\n" - "ldr x25, [x16, #0x50]\n" - "smlal v9.4s, v28.4h, v4.4h\n" - "smlal2 v18.4s, v28.8h, v4.8h\n" - "ldr x24, [x16, #0x58]\n" - "ldr x23, [x16, #0x60]\n" - "smlal v16.4s, v28.4h, v2.4h\n" - "smlal2 v26.4s, v28.8h, v2.8h\n" - "ldr x22, [x16, #0x68]\n" - "ldr x21, [x16, #0x70]\n" - "smlal v25.4s, v28.4h, v1.4h\n" - "smlal2 v10.4s, v28.8h, v1.8h\n" - "ldr d28, [x28, x8]\n" - "ssubl v28.8h, v28.8b, v24.8b\n" - "smlal v13.4s, v27.4h, v7.4h\n" - "smlal2 v20.4s, v27.8h, v7.8h\n" - "ldr x20, [x16, #0x78]\n" - "ldr x28, [%x[params], %[offsetof_Params_bias]]\n" - "smlal v9.4s, v27.4h, v6.4h\n" - "smlal2 v18.4s, v27.8h, v6.8h\n" - "add x15, x15, #0x48\n" - "subs x7, x7, #0x1\n" - "smlal v16.4s, v31.4h, v6.4h\n" - "smlal2 v26.4s, v31.8h, v6.8h\n" - "ldr d31, [x27, x8]\n" - "ssubl v31.8h, v31.8b, v24.8b\n" - "smlal v25.4s, v27.4h, v3.4h\n" - "smlal2 v10.4s, v27.8h, v3.8h\n" - "add x14, x14, #0x20\n" + "ldr q3, [x13, #0x0]\n" + "ldr q17, [x12, #0x0]\n" + "smlal v9.4s, v22.4h, v26.4h\n" + "smlal2 v24.4s, v22.8h, v26.8h\n" + "ldr q21, [x13, #0x10]\n" + "ldr q28, [x12, #0x10]\n" + "smlal v9.4s, v4.4h, v23.4h\n" + "smlal v7.4s, v22.4h, v5.4h\n" + "ldr x20, [x15, #0x28]\n" + "ldr d11, [x20, x17]\n" + "smlal v2.4s, v22.4h, v16.4h\n" + "smlal v10.4s, v22.4h, v23.4h\n" + "smlal2 v24.4s, v4.8h, v23.8h\n" + "ldr x20, [x15, #0x38]\n" + "ldr d4, [x20, x17]\n" + "smlal v9.4s, v27.4h, v18.4h\n" + "smlal2 v0.4s, v22.8h, v5.8h\n" + "smlal2 v30.4s, v22.8h, v16.8h\n" + "ldr x20, [x15, #0x30]\n" + "ssubl v11.8h, v11.8b, v14.8b\n" + "smlal2 v6.4s, v22.8h, v23.8h\n" + "ldr d22, [x20, x17]\n" + "smlal v7.4s, v8.4h, v1.4h\n" + "ldr x20, [x15, #0x40]\n" + "smlal v2.4s, v27.4h, v1.4h\n" + "smlal v10.4s, v27.4h, v16.4h\n" + "ssubl v4.8h, v4.8b, v14.8b\n" + "ldr x27, [x15, #0x48]\n" + "smlal2 v24.4s, v27.8h, v18.8h\n" + "smlal v9.4s, v15.4h, v25.4h\n" + "ssubl v22.8h, v22.8b, v14.8b\n" + "ldr x26, [x15, #0x50]\n" + "smlal2 v0.4s, v8.8h, v1.8h\n" + "ldr d8, [x20, x17]\n" + "smlal2 v30.4s, v27.8h, v1.8h\n" + "ssubl v8.8h, v8.8b, v14.8b\n" + "smlal2 v6.4s, v27.8h, v16.8h\n" + "smlal v7.4s, v27.4h, v26.4h\n" + "ldr x25, [x15, #0x58]\n" + "ldr x24, [x15, #0x60]\n" + "smlal v2.4s, v11.4h, v31.4h\n" + "smlal v10.4s, v15.4h, v5.4h\n" + "ldr x23, [x15, #0x68]\n" + "ldr x22, [x15, #0x70]\n" + "smlal2 v24.4s, v15.8h, v25.8h\n" + "smlal v9.4s, v4.4h, v16.4h\n" + "ldr x21, [x15, #0x78]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "smlal2 v0.4s, v27.8h, v26.8h\n" + "ldr d27, [x27, x17]\n" + "smlal2 v30.4s, v11.8h, v31.8h\n" + "ldr d11, [x26, x17]\n" + "smlal2 v6.4s, v15.8h, v5.8h\n" + "smlal v7.4s, v15.4h, v31.4h\n" + "ssubl v27.8h, v27.8b, v14.8b\n" + "add x14, x14, #0x48\n" + "smlal v2.4s, v15.4h, v26.4h\n" + "smlal v10.4s, v22.4h, v20.4h\n" + "ssubl v11.8h, v11.8b, v14.8b\n" + "subs x8, x8, #0x1\n" + "smlal2 v24.4s, v4.8h, v16.8h\n" + "smlal v9.4s, v8.4h, v1.4h\n" "add x13, x13, #0x20\n" - "smlal v13.4s, v28.4h, v1.4h\n" - "smlal2 v20.4s, v28.8h, v1.8h\n" - "smlal v9.4s, v28.4h, v0.4h\n" - "smlal2 v18.4s, v28.8h, v0.8h\n" - "ldr d30, [x26, x8]\n" - "ssubl v30.8h, v30.8b, v24.8b\n" - "smlal v16.4s, v27.4h, v4.4h\n" - "smlal v25.4s, v29.4h, v8.4h\n" - "smlal2 v26.4s, v27.8h, v4.8h\n" - "ldr d28, [x24, x8]\n" - "smlal2 v10.4s, v29.8h, v8.8h\n" - "ldr d29, [x25, x8]\n" - "smlal v13.4s, v31.4h, v2.4h\n" - "smlal2 v20.4s, v31.8h, v2.8h\n" - "ssubl v29.8h, v29.8b, v24.8b\n" - "smlal v9.4s, v31.4h, v1.4h\n" - "smlal2 v18.4s, v31.8h, v1.8h\n" - "ldr d31, [x23, x8]\n" - "ssubl v28.8h, v28.8b, v24.8b\n" - "smlal v16.4s, v30.4h, v5.4h\n" - "smlal v25.4s, v30.4h, v4.4h\n" - "ssubl v31.8h, v31.8b, v24.8b\n" - "smlal v13.4s, v30.4h, v8.4h\n" - "smlal2 v20.4s, v30.8h, v8.8h\n" - "smlal v9.4s, v30.4h, v7.4h\n" - "smlal2 v18.4s, v30.8h, v7.8h\n" - "smlal2 v26.4s, v30.8h, v5.8h\n" - "smlal2 v10.4s, v30.8h, v4.8h\n" - "ldr d30, [x22, x8]\n" - "ssubl v30.8h, v30.8b, v24.8b\n" - "smlal v16.4s, v29.4h, v0.4h\n" - "smlal v25.4s, v28.4h, v2.4h\n" - "smlal v13.4s, v29.4h, v3.4h\n" - "smlal2 v20.4s, v29.8h, v3.8h\n" - "smlal2 v26.4s, v29.8h, v0.8h\n" - "ldr d29, [x21, x8]\n" - "smlal2 v10.4s, v28.8h, v2.8h\n" - "ssubl v29.8h, v29.8b, v24.8b\n" - "smlal v16.4s, v31.4h, v3.4h\n" - "smlal v25.4s, v30.4h, v5.4h\n" - "smlal v9.4s, v28.4h, v5.4h\n" - "smlal2 v18.4s, v28.8h, v5.8h\n" - "ldr d28, [x20, x8]\n" - "ssubl v28.8h, v28.8b, v24.8b\n" - "smlal v13.4s, v31.4h, v6.4h\n" - "smlal2 v26.4s, v31.8h, v3.8h\n" - "sqrdmulh v13.4s, v13.4s, v17.4s\n" - "add x8, x8, #0x8\n" - "smlal2 v10.4s, v30.8h, v5.8h\n" - "smlal v16.4s, v29.4h, v7.4h\n" - "and v21.16b, v13.16b, v22.16b\n" - "smlal v25.4s, v29.4h, v6.4h\n" - "smlal2 v20.4s, v31.8h, v6.8h\n" - "sqrdmulh v20.4s, v20.4s, v23.4s\n" - "smlal2 v26.4s, v29.8h, v7.8h\n" - "smlal2 v10.4s, v29.8h, v6.8h\n" - "sshr v21.4s, v21.4s, #0x1f\n" - "smlal v9.4s, v30.4h, v8.4h\n" - "smlal v16.4s, v28.4h, v8.4h\n" - "and v29.16b, v20.16b, v19.16b\n" - "smlal v25.4s, v28.4h, v7.4h\n" - "smlal2 v18.4s, v30.8h, v8.8h\n" - "sqrdmulh v9.4s, v9.4s, v17.4s\n" - "smlal2 v26.4s, v28.8h, v8.8h\n" - "smlal2 v10.4s, v28.8h, v7.8h\n" - "sqrdmulh v16.4s, v16.4s, v17.4s\n" - "sqrdmulh v25.4s, v25.4s, v17.4s\n" - "sqadd v13.4s, v13.4s, v21.4s\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "and v0.16b, v9.16b, v22.16b\n" - "sqrdmulh v18.4s, v18.4s, v23.4s\n" - "and v27.16b, v16.16b, v22.16b\n" - "sqrdmulh v26.4s, v26.4s, v23.4s\n" - "and v21.16b, v25.16b, v22.16b\n" - "sqrdmulh v10.4s, v10.4s, v23.4s\n" - "sqadd v20.4s, v20.4s, v29.4s\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v17.16b, v18.16b, v19.16b\n" + "add x12, x12, #0x20\n" + "smlal2 v0.4s, v15.8h, v31.8h\n" + "smlal2 v30.4s, v15.8h, v26.8h\n" + "ldr d15, [x25, x17]\n" + "ssubl v15.8h, v15.8b, v14.8b\n" + "smlal2 v6.4s, v22.8h, v20.8h\n" + "ldr d22, [x24, x17]\n" + "smlal v7.4s, v4.4h, v23.4h\n" + "ssubl v22.8h, v22.8b, v14.8b\n" + "smlal v2.4s, v27.4h, v18.4h\n" + "smlal v10.4s, v27.4h, v26.4h\n" + "smlal2 v24.4s, v8.8h, v1.8h\n" + "smlal v9.4s, v27.4h, v20.4h\n" + "smlal2 v0.4s, v4.8h, v23.8h\n" + "ldr d4, [x23, x17]\n" + "smlal2 v30.4s, v27.8h, v18.8h\n" + "ssubl v4.8h, v4.8b, v14.8b\n" + "smlal2 v6.4s, v27.8h, v26.8h\n" + "ldr d26, [x22, x17]\n" + "smlal v7.4s, v8.4h, v16.4h\n" + "ssubl v26.8h, v26.8b, v14.8b\n" + "smlal v2.4s, v11.4h, v23.4h\n" + "smlal v10.4s, v15.4h, v1.4h\n" + "smlal2 v24.4s, v27.8h, v20.8h\n" + "smlal v9.4s, v11.4h, v5.4h\n" + "smlal2 v0.4s, v8.8h, v16.8h\n" + "ldr d8, [x21, x17]\n" + "smlal2 v30.4s, v11.8h, v23.8h\n" + "ssubl v8.8h, v8.8b, v14.8b\n" + "smlal2 v6.4s, v15.8h, v1.8h\n" + "smlal v7.4s, v27.4h, v25.4h\n" + "add x17, x17, #0x8\n" + "smlal v2.4s, v22.4h, v5.4h\n" + "smlal v10.4s, v4.4h, v18.4h\n" + "smlal2 v24.4s, v11.8h, v5.8h\n" + "smlal v9.4s, v22.4h, v31.4h\n" + "sqrdmulh v9.4s, v9.4s, v3.4s\n" + "smlal2 v0.4s, v27.8h, v25.8h\n" + "smlal2 v30.4s, v22.8h, v5.8h\n" + "and v27.16b, v9.16b, v17.16b\n" + "smlal2 v6.4s, v4.8h, v18.8h\n" + "smlal v7.4s, v15.4h, v18.4h\n" "sshr v27.4s, v27.4s, #0x1f\n" - "and v7.16b, v26.16b, v19.16b\n" - "sshr v21.4s, v21.4s, #0x1f\n" - "and v29.16b, v10.16b, v19.16b\n" - "sqadd v9.4s, v9.4s, v0.4s\n" - "sshr v17.4s, v17.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v27.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v25.4s, v25.4s, v21.4s\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "srshl v13.4s, v13.4s, v22.4s\n" - "srshl v9.4s, v9.4s, v22.4s\n" - "sqadd v18.4s, v18.4s, v17.4s\n" - "srshl v16.4s, v16.4s, v22.4s\n" - "sqadd v26.4s, v26.4s, v7.4s\n" - "srshl v25.4s, v25.4s, v22.4s\n" - "sqadd v10.4s, v10.4s, v29.4s\n" - "srshl v20.4s, v20.4s, v19.4s\n" - "sqxtn v13.4h, v13.4s\n" - "srshl v18.4s, v18.4s, v19.4s\n" + "smlal v2.4s, v26.4h, v25.4h\n" + "smlal v10.4s, v26.4h, v31.4h\n" + "sqadd v9.4s, v9.4s, v27.4s\n" + "smlal2 v24.4s, v22.8h, v31.8h\n" + "smlal2 v0.4s, v15.8h, v18.8h\n" + "sqrdmulh v24.4s, v24.4s, v21.4s\n" + "smlal2 v30.4s, v26.8h, v25.8h\n" + "smlal2 v6.4s, v26.8h, v31.8h\n" + "and v31.16b, v24.16b, v28.16b\n" + "smlal v7.4s, v4.4h, v20.4h\n" + "smlal v2.4s, v8.4h, v20.4h\n" + "sqrdmulh v7.4s, v7.4s, v3.4s\n" + "smlal v10.4s, v8.4h, v25.4h\n" + "smlal2 v0.4s, v4.8h, v20.8h\n" + "sqrdmulh v2.4s, v2.4s, v3.4s\n" + "smlal2 v30.4s, v8.8h, v20.8h\n" + "smlal2 v6.4s, v8.8h, v25.8h\n" + "sqrdmulh v10.4s, v10.4s, v3.4s\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "and v22.16b, v7.16b, v17.16b\n" + "sqrdmulh v0.4s, v0.4s, v21.4s\n" + "and v3.16b, v2.16b, v17.16b\n" + "sqrdmulh v30.4s, v30.4s, v21.4s\n" + "and v11.16b, v10.16b, v17.16b\n" + "sqrdmulh v6.4s, v6.4s, v21.4s\n" + "sqadd v24.4s, v24.4s, v31.4s\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "and v20.16b, v0.16b, v28.16b\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "and v31.16b, v30.16b, v28.16b\n" + "sshr v11.4s, v11.4s, #0x1f\n" + "and v18.16b, v6.16b, v28.16b\n" + "sqadd v7.4s, v7.4s, v22.4s\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "sqadd v2.4s, v2.4s, v3.4s\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "sqadd v10.4s, v10.4s, v11.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "srshl v9.4s, v9.4s, v17.4s\n" + "srshl v7.4s, v7.4s, v17.4s\n" + "sqadd v0.4s, v0.4s, v20.4s\n" + "srshl v2.4s, v2.4s, v17.4s\n" + "sqadd v30.4s, v30.4s, v31.4s\n" + "srshl v10.4s, v10.4s, v17.4s\n" + "sqadd v6.4s, v6.4s, v18.4s\n" + "srshl v24.4s, v24.4s, v28.4s\n" "sqxtn v9.4h, v9.4s\n" - "srshl v26.4s, v26.4s, v19.4s\n" - "sqxtn v16.4h, v16.4s\n" - "srshl v10.4s, v10.4s, v19.4s\n" - "sqxtn v25.4h, v25.4s\n" - "sqxtn2 v13.8h, v20.4s\n" - "sqxtn2 v9.8h, v18.4s\n" - "sqxtn2 v16.8h, v26.4s\n" - "sqxtn2 v25.8h, v10.4s\n" - "sqadd v13.8h, v13.8h, v14.8h\n" - "sqadd v9.8h, v9.8h, v14.8h\n" - "sqadd v16.8h, v16.8h, v14.8h\n" - "sqadd v25.8h, v25.8h, v14.8h\n" - "smax v13.8h, v13.8h, v12.8h\n" - "smax v9.8h, v9.8h, v12.8h\n" - "smax v16.8h, v16.8h, v12.8h\n" - "smax v25.8h, v25.8h, v12.8h\n" - "smin v13.8h, v13.8h, v11.8h\n" - "smin v9.8h, v9.8h, v11.8h\n" - "smin v16.8h, v16.8h, v11.8h\n" - "smin v25.8h, v25.8h, v11.8h\n" - "uzp1 v13.16b, v13.16b, v13.16b\n" - "str d13, [x12, x17]\n" + "srshl v0.4s, v0.4s, v28.4s\n" + "sqxtn v7.4h, v7.4s\n" + "srshl v30.4s, v30.4s, v28.4s\n" + "sqxtn v2.4h, v2.4s\n" + "srshl v6.4s, v6.4s, v28.4s\n" + "sqxtn v10.4h, v10.4s\n" + "sqxtn2 v9.8h, v24.4s\n" + "sqxtn2 v7.8h, v0.4s\n" + "sqxtn2 v2.8h, v30.4s\n" + "sqxtn2 v10.8h, v6.4s\n" + "sqadd v9.8h, v9.8h, v13.8h\n" + "sqadd v7.8h, v7.8h, v13.8h\n" + "sqadd v2.8h, v2.8h, v13.8h\n" + "sqadd v10.8h, v10.8h, v13.8h\n" + "smax v9.8h, v9.8h, v29.8h\n" + "smax v7.8h, v7.8h, v29.8h\n" + "smax v2.8h, v2.8h, v29.8h\n" + "smax v10.8h, v10.8h, v29.8h\n" + "smin v9.8h, v9.8h, v12.8h\n" + "smin v7.8h, v7.8h, v12.8h\n" + "smin v2.8h, v2.8h, v12.8h\n" + "smin v10.8h, v10.8h, v12.8h\n" "uzp1 v9.16b, v9.16b, v9.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "str d9, [x11, x17]\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "str d16, [x10, x17]\n" - "str d25, [x9, x17]\n" - "ldr q13, [x28, #0x0]\n" - "ldr q20, [x28, #0x10]\n" - "add x28, x28, #0x20\n" - "ldr d0, [x15, #0x0]\n" - "ldr d1, [x15, #0x8]\n" - "add x17, x17, #0x8\n" - "str x28, [%x[params], %[offsetof_Params_bias]]\n" - "ldr d2, [x15, #0x10]\n" - "ldr d3, [x15, #0x18]\n" - "mov v9.16b, v13.16b\n" - "mov v18.16b, v20.16b\n" - "ldr d4, [x15, #0x20]\n" - "ldr d5, [x15, #0x28]\n" - "mov v16.16b, v13.16b\n" - "mov v26.16b, v20.16b\n" - "ldr d6, [x15, #0x30]\n" - "ldr d7, [x15, #0x38]\n" - "mov v25.16b, v13.16b\n" - "mov v10.16b, v20.16b\n" - "ldr d8, [x15, #0x40]\n" - "ldp x24, x23, [x16, #0x0]\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "ldp x22, x21, [x16, #0x10]\n" - "ldr d31, [x24, x8]\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "ldr d30, [x23, x8]\n" - "ldr d29, [x22, x8]\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "ssubl v5.8h, v5.8b, v15.8b\n" - "ldr d28, [x21, x8]\n" - "ldr x20, [x16, #0x20]\n" - "ssubl v6.8h, v6.8b, v15.8b\n" - "ssubl v7.8h, v7.8b, v15.8b\n" - "ldr d27, [x20, x8]\n" - "ssubl v8.8h, v8.8b, v15.8b\n" - "ssubl v31.8h, v31.8b, v24.8b\n" - "ssubl v30.8h, v30.8b, v24.8b\n" - "ssubl v29.8h, v29.8b, v24.8b\n" - "ssubl v28.8h, v28.8b, v24.8b\n" - "ssubl v27.8h, v27.8b, v24.8b\n" + "str d9, [x11, x16]\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" + "uzp1 v2.16b, v2.16b, v2.16b\n" + "str d7, [x10, x16]\n" + "uzp1 v10.16b, v10.16b, v10.16b\n" + "str d2, [x9, x16]\n" + "str d10, [x28, x16]\n" + "ldr q9, [x20, #0x0]\n" + "ldr q24, [x20, #0x10]\n" + "add x20, x20, #0x20\n" + "ldr d23, [x14, #0x0]\n" + "ldr d16, [x14, #0x8]\n" + "add x16, x16, #0x8\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" + "ldr d1, [x14, #0x10]\n" + "ldr d5, [x14, #0x18]\n" + "mov v7.16b, v9.16b\n" + "mov v0.16b, v24.16b\n" + "ldr d26, [x14, #0x20]\n" + "ldr d18, [x14, #0x28]\n" + "mov v2.16b, v9.16b\n" + "mov v30.16b, v24.16b\n" + "ldr d31, [x14, #0x30]\n" + "ldr d25, [x14, #0x38]\n" + "mov v10.16b, v9.16b\n" + "mov v6.16b, v24.16b\n" + "ldr d20, [x14, #0x40]\n" + "ldp x23, x22, [x15, #0x0]\n" + "ssubl v23.8h, v23.8b, v19.8b\n" + "ssubl v16.8h, v16.8b, v19.8b\n" + "ldp x21, x20, [x15, #0x10]\n" + "ldr d22, [x23, x17]\n" + "ssubl v1.8h, v1.8b, v19.8b\n" + "ssubl v5.8h, v5.8b, v19.8b\n" + "ldr d4, [x22, x17]\n" + "ldr d8, [x21, x17]\n" + "ssubl v26.8h, v26.8b, v19.8b\n" + "ssubl v18.8h, v18.8b, v19.8b\n" + "ldr d27, [x20, x17]\n" + "ldr x20, [x15, #0x20]\n" + "ssubl v31.8h, v31.8b, v19.8b\n" + "ssubl v25.8h, v25.8b, v19.8b\n" + "ldr d15, [x20, x17]\n" + "ssubl v20.8h, v20.8b, v19.8b\n" + "ssubl v22.8h, v22.8b, v14.8b\n" + "ssubl v4.8h, v4.8b, v14.8b\n" + "ssubl v8.8h, v8.8b, v14.8b\n" + "ssubl v27.8h, v27.8b, v14.8b\n" + "ssubl v15.8h, v15.8b, v14.8b\n" "bgt 1b\n" "2:" // Tail - "ldr q17, [x14, #0x0]\n" - "ldr q22, [x13, #0x0]\n" - "smlal v13.4s, v31.4h, v4.4h\n" - "smlal2 v20.4s, v31.8h, v4.8h\n" - "ldr q23, [x14, #0x10]\n" - "smlal v9.4s, v31.4h, v3.4h\n" - "smlal2 v18.4s, v31.8h, v3.8h\n" - "ldr x21, [x16, #0x28]\n" - "smlal v13.4s, v30.4h, v0.4h\n" - "smlal2 v20.4s, v30.8h, v0.8h\n" - "ldr q19, [x13, #0x10]\n" - "ldr x28, [x16, #0x38]\n" - "smlal v9.4s, v29.4h, v2.4h\n" - "smlal2 v18.4s, v29.8h, v2.8h\n" - "ldr x20, [x16, #0x30]\n" - "ldr d29, [x20, x8]\n" - "smlal v16.4s, v31.4h, v1.4h\n" - "smlal2 v26.4s, v31.8h, v1.8h\n" - "ldr x27, [x16, #0x40]\n" - "ldr x26, [x16, #0x48]\n" - "smlal v25.4s, v31.4h, v0.4h\n" - "smlal2 v10.4s, v31.8h, v0.8h\n" - "ldr d31, [x21, x8]\n" - "ssubl v31.8h, v31.8b, v24.8b\n" - "smlal v13.4s, v28.4h, v5.4h\n" - "smlal2 v20.4s, v28.8h, v5.8h\n" - "ssubl v29.8h, v29.8b, v24.8b\n" - "ldr x25, [x16, #0x50]\n" - "smlal v9.4s, v28.4h, v4.4h\n" - "smlal2 v18.4s, v28.8h, v4.8h\n" - "ldr x24, [x16, #0x58]\n" - "ldr x23, [x16, #0x60]\n" - "smlal v16.4s, v28.4h, v2.4h\n" - "smlal2 v26.4s, v28.8h, v2.8h\n" - "ldr x22, [x16, #0x68]\n" - "ldr x21, [x16, #0x70]\n" - "smlal v25.4s, v28.4h, v1.4h\n" - "smlal2 v10.4s, v28.8h, v1.8h\n" - "ldr d28, [x28, x8]\n" - "ssubl v28.8h, v28.8b, v24.8b\n" - "smlal v13.4s, v27.4h, v7.4h\n" - "smlal2 v20.4s, v27.8h, v7.8h\n" - "ldr x20, [x16, #0x78]\n" - "tst x6, #0x7\n" - "smlal v9.4s, v27.4h, v6.4h\n" - "smlal2 v18.4s, v27.8h, v6.8h\n" - "add x14, x14, #0x20\n" + "ldr q28, [x13, #0x0]\n" + "ldr q17, [x12, #0x0]\n" + "smlal v9.4s, v22.4h, v26.4h\n" + "smlal2 v24.4s, v22.8h, v26.8h\n" + "ldr q21, [x13, #0x10]\n" + "ldr q3, [x12, #0x10]\n" + "smlal v9.4s, v4.4h, v23.4h\n" + "smlal v7.4s, v22.4h, v5.4h\n" + "ldr x20, [x15, #0x28]\n" + "ldr d11, [x20, x17]\n" + "smlal v2.4s, v22.4h, v16.4h\n" + "smlal v10.4s, v22.4h, v23.4h\n" + "smlal2 v24.4s, v4.8h, v23.8h\n" + "ldr x20, [x15, #0x38]\n" + "ldr d4, [x20, x17]\n" + "smlal v9.4s, v27.4h, v18.4h\n" + "smlal2 v0.4s, v22.8h, v5.8h\n" + "smlal2 v30.4s, v22.8h, v16.8h\n" + "ldr x20, [x15, #0x30]\n" + "ssubl v11.8h, v11.8b, v14.8b\n" + "smlal2 v6.4s, v22.8h, v23.8h\n" + "ldr d22, [x20, x17]\n" + "smlal v7.4s, v8.4h, v1.4h\n" + "ldr x20, [x15, #0x40]\n" + "smlal v2.4s, v27.4h, v1.4h\n" + "smlal v10.4s, v27.4h, v16.4h\n" + "ssubl v4.8h, v4.8b, v14.8b\n" + "ldr x26, [x15, #0x48]\n" + "smlal2 v24.4s, v27.8h, v18.8h\n" + "smlal v9.4s, v15.4h, v25.4h\n" + "ssubl v22.8h, v22.8b, v14.8b\n" + "ldr x25, [x15, #0x50]\n" + "smlal2 v0.4s, v8.8h, v1.8h\n" + "ldr d8, [x20, x17]\n" + "smlal2 v30.4s, v27.8h, v1.8h\n" + "ssubl v8.8h, v8.8b, v14.8b\n" + "smlal2 v6.4s, v27.8h, v16.8h\n" + "smlal v7.4s, v27.4h, v26.4h\n" + "ldr x24, [x15, #0x58]\n" + "ldr x23, [x15, #0x60]\n" + "smlal v2.4s, v11.4h, v31.4h\n" + "smlal v10.4s, v15.4h, v5.4h\n" + "ldr x22, [x15, #0x68]\n" + "ldr x21, [x15, #0x70]\n" + "smlal2 v24.4s, v15.8h, v25.8h\n" + "smlal v9.4s, v4.4h, v16.4h\n" + "ldr x20, [x15, #0x78]\n" + "tst x7, #0x7\n" + "smlal2 v0.4s, v27.8h, v26.8h\n" + "ldr d27, [x26, x17]\n" + "smlal2 v30.4s, v11.8h, v31.8h\n" + "ldr d11, [x25, x17]\n" + "smlal2 v6.4s, v15.8h, v5.8h\n" + "smlal v7.4s, v15.4h, v31.4h\n" + "ssubl v27.8h, v27.8b, v14.8b\n" "add x13, x13, #0x20\n" - "smlal v16.4s, v31.4h, v6.4h\n" - "smlal2 v26.4s, v31.8h, v6.8h\n" - "ldr d31, [x27, x8]\n" - "ssubl v31.8h, v31.8b, v24.8b\n" - "smlal v25.4s, v27.4h, v3.4h\n" - "smlal2 v10.4s, v27.8h, v3.8h\n" - "smlal v13.4s, v28.4h, v1.4h\n" - "smlal2 v20.4s, v28.8h, v1.8h\n" - "smlal v9.4s, v28.4h, v0.4h\n" - "smlal2 v18.4s, v28.8h, v0.8h\n" - "ldr d30, [x26, x8]\n" - "ssubl v30.8h, v30.8b, v24.8b\n" - "smlal v16.4s, v27.4h, v4.4h\n" - "smlal v25.4s, v29.4h, v8.4h\n" - "smlal2 v26.4s, v27.8h, v4.8h\n" - "ldr d28, [x24, x8]\n" - "smlal2 v10.4s, v29.8h, v8.8h\n" - "ldr d29, [x25, x8]\n" - "smlal v13.4s, v31.4h, v2.4h\n" - "smlal2 v20.4s, v31.8h, v2.8h\n" - "ssubl v29.8h, v29.8b, v24.8b\n" - "smlal v9.4s, v31.4h, v1.4h\n" - "smlal2 v18.4s, v31.8h, v1.8h\n" - "ldr d31, [x23, x8]\n" - "ssubl v28.8h, v28.8b, v24.8b\n" - "smlal v16.4s, v30.4h, v5.4h\n" - "smlal v25.4s, v30.4h, v4.4h\n" - "ssubl v31.8h, v31.8b, v24.8b\n" - "smlal v13.4s, v30.4h, v8.4h\n" - "smlal2 v20.4s, v30.8h, v8.8h\n" - "smlal v9.4s, v30.4h, v7.4h\n" - "smlal2 v18.4s, v30.8h, v7.8h\n" - "smlal2 v26.4s, v30.8h, v5.8h\n" - "smlal2 v10.4s, v30.8h, v4.8h\n" - "ldr d30, [x22, x8]\n" - "ssubl v30.8h, v30.8b, v24.8b\n" - "smlal v16.4s, v29.4h, v0.4h\n" - "smlal v25.4s, v28.4h, v2.4h\n" - "smlal v13.4s, v29.4h, v3.4h\n" - "smlal2 v20.4s, v29.8h, v3.8h\n" - "smlal2 v26.4s, v29.8h, v0.8h\n" - "ldr d29, [x21, x8]\n" - "smlal2 v10.4s, v28.8h, v2.8h\n" - "ssubl v29.8h, v29.8b, v24.8b\n" - "smlal v16.4s, v31.4h, v3.4h\n" - "smlal v25.4s, v30.4h, v5.4h\n" - "smlal v9.4s, v28.4h, v5.4h\n" - "smlal2 v18.4s, v28.8h, v5.8h\n" - "ldr d28, [x20, x8]\n" - "ssubl v28.8h, v28.8b, v24.8b\n" - "smlal v13.4s, v31.4h, v6.4h\n" - "smlal2 v26.4s, v31.8h, v3.8h\n" - "sqrdmulh v13.4s, v13.4s, v17.4s\n" - "add x8, x8, #0x8\n" - "smlal2 v10.4s, v30.8h, v5.8h\n" - "smlal v16.4s, v29.4h, v7.4h\n" - "and v21.16b, v13.16b, v22.16b\n" - "smlal v25.4s, v29.4h, v6.4h\n" - "smlal2 v20.4s, v31.8h, v6.8h\n" - "sqrdmulh v20.4s, v20.4s, v23.4s\n" - "smlal2 v26.4s, v29.8h, v7.8h\n" - "smlal2 v10.4s, v29.8h, v6.8h\n" - "sshr v21.4s, v21.4s, #0x1f\n" - "smlal v9.4s, v30.4h, v8.4h\n" - "smlal v16.4s, v28.4h, v8.4h\n" - "and v29.16b, v20.16b, v19.16b\n" - "smlal v25.4s, v28.4h, v7.4h\n" - "smlal2 v18.4s, v30.8h, v8.8h\n" - "sqrdmulh v9.4s, v9.4s, v17.4s\n" - "smlal2 v26.4s, v28.8h, v8.8h\n" - "smlal2 v10.4s, v28.8h, v7.8h\n" - "sqrdmulh v16.4s, v16.4s, v17.4s\n" - "sqrdmulh v25.4s, v25.4s, v17.4s\n" - "sqadd v13.4s, v13.4s, v21.4s\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "and v0.16b, v9.16b, v22.16b\n" - "sqrdmulh v18.4s, v18.4s, v23.4s\n" - "and v27.16b, v16.16b, v22.16b\n" - "sqrdmulh v26.4s, v26.4s, v23.4s\n" - "and v21.16b, v25.16b, v22.16b\n" - "sqrdmulh v10.4s, v10.4s, v23.4s\n" - "sqadd v20.4s, v20.4s, v29.4s\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v17.16b, v18.16b, v19.16b\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "and v7.16b, v26.16b, v19.16b\n" + "smlal v2.4s, v15.4h, v26.4h\n" + "smlal v10.4s, v22.4h, v20.4h\n" + "ssubl v11.8h, v11.8b, v14.8b\n" + "add x12, x12, #0x20\n" + "smlal2 v24.4s, v4.8h, v16.8h\n" + "smlal v9.4s, v8.4h, v1.4h\n" + "smlal2 v0.4s, v15.8h, v31.8h\n" + "smlal2 v30.4s, v15.8h, v26.8h\n" + "ldr d15, [x24, x17]\n" + "ssubl v15.8h, v15.8b, v14.8b\n" + "smlal2 v6.4s, v22.8h, v20.8h\n" + "ldr d22, [x23, x17]\n" + "smlal v7.4s, v4.4h, v23.4h\n" + "ssubl v22.8h, v22.8b, v14.8b\n" + "smlal v2.4s, v27.4h, v18.4h\n" + "smlal v10.4s, v27.4h, v26.4h\n" + "smlal2 v24.4s, v8.8h, v1.8h\n" + "smlal v9.4s, v27.4h, v20.4h\n" + "smlal2 v0.4s, v4.8h, v23.8h\n" + "ldr d4, [x22, x17]\n" + "smlal2 v30.4s, v27.8h, v18.8h\n" + "ssubl v4.8h, v4.8b, v14.8b\n" + "smlal2 v6.4s, v27.8h, v26.8h\n" + "ldr d26, [x21, x17]\n" + "smlal v7.4s, v8.4h, v16.4h\n" + "ssubl v26.8h, v26.8b, v14.8b\n" + "smlal v2.4s, v11.4h, v23.4h\n" + "smlal v10.4s, v15.4h, v1.4h\n" + "smlal2 v24.4s, v27.8h, v20.8h\n" + "smlal v9.4s, v11.4h, v5.4h\n" + "smlal2 v0.4s, v8.8h, v16.8h\n" + "ldr d16, [x20, x17]\n" + "smlal2 v30.4s, v11.8h, v23.8h\n" + "ssubl v16.8h, v16.8b, v14.8b\n" + "smlal2 v6.4s, v15.8h, v1.8h\n" + "smlal v7.4s, v27.4h, v25.4h\n" + "add x17, x17, #0x8\n" + "smlal v2.4s, v22.4h, v5.4h\n" + "smlal v10.4s, v4.4h, v18.4h\n" + "smlal2 v24.4s, v11.8h, v5.8h\n" + "smlal v9.4s, v22.4h, v31.4h\n" + "sqrdmulh v9.4s, v9.4s, v28.4s\n" + "smlal2 v0.4s, v27.8h, v25.8h\n" + "smlal2 v30.4s, v22.8h, v5.8h\n" + "and v1.16b, v9.16b, v17.16b\n" + "smlal2 v6.4s, v4.8h, v18.8h\n" + "smlal v7.4s, v15.4h, v18.4h\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "smlal v2.4s, v26.4h, v25.4h\n" + "smlal v10.4s, v26.4h, v31.4h\n" + "sqadd v9.4s, v9.4s, v1.4s\n" + "smlal2 v24.4s, v22.8h, v31.8h\n" + "smlal2 v0.4s, v15.8h, v18.8h\n" + "sqrdmulh v24.4s, v24.4s, v21.4s\n" + "smlal2 v30.4s, v26.8h, v25.8h\n" + "smlal2 v6.4s, v26.8h, v31.8h\n" + "and v31.16b, v24.16b, v3.16b\n" + "smlal v7.4s, v4.4h, v20.4h\n" + "smlal v2.4s, v16.4h, v20.4h\n" + "sqrdmulh v7.4s, v7.4s, v28.4s\n" + "smlal v10.4s, v16.4h, v25.4h\n" + "smlal2 v0.4s, v4.8h, v20.8h\n" + "sqrdmulh v2.4s, v2.4s, v28.4s\n" + "smlal2 v30.4s, v16.8h, v20.8h\n" + "smlal2 v6.4s, v16.8h, v25.8h\n" + "sqrdmulh v10.4s, v10.4s, v28.4s\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "and v22.16b, v7.16b, v17.16b\n" + "sqrdmulh v0.4s, v0.4s, v21.4s\n" + "and v15.16b, v2.16b, v17.16b\n" + "sqrdmulh v30.4s, v30.4s, v21.4s\n" + "and v11.16b, v10.16b, v17.16b\n" + "sqrdmulh v6.4s, v6.4s, v21.4s\n" + "sqadd v24.4s, v24.4s, v31.4s\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "and v18.16b, v0.16b, v3.16b\n" + "sshr v15.4s, v15.4s, #0x1f\n" + "and v23.16b, v30.16b, v3.16b\n" + "sshr v11.4s, v11.4s, #0x1f\n" + "and v21.16b, v6.16b, v3.16b\n" + "sqadd v7.4s, v7.4s, v22.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sqadd v2.4s, v2.4s, v15.4s\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "sqadd v10.4s, v10.4s, v11.4s\n" "sshr v21.4s, v21.4s, #0x1f\n" - "and v29.16b, v10.16b, v19.16b\n" - "sqadd v9.4s, v9.4s, v0.4s\n" - "sshr v17.4s, v17.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v27.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v25.4s, v25.4s, v21.4s\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "srshl v13.4s, v13.4s, v22.4s\n" - "srshl v9.4s, v9.4s, v22.4s\n" - "sqadd v18.4s, v18.4s, v17.4s\n" - "srshl v16.4s, v16.4s, v22.4s\n" - "sqadd v26.4s, v26.4s, v7.4s\n" - "srshl v25.4s, v25.4s, v22.4s\n" - "sqadd v10.4s, v10.4s, v29.4s\n" - "srshl v20.4s, v20.4s, v19.4s\n" - "sqxtn v13.4h, v13.4s\n" - "srshl v18.4s, v18.4s, v19.4s\n" + "srshl v9.4s, v9.4s, v17.4s\n" + "srshl v7.4s, v7.4s, v17.4s\n" + "sqadd v0.4s, v0.4s, v18.4s\n" + "srshl v2.4s, v2.4s, v17.4s\n" + "sqadd v30.4s, v30.4s, v23.4s\n" + "srshl v10.4s, v10.4s, v17.4s\n" + "sqadd v6.4s, v6.4s, v21.4s\n" + "srshl v24.4s, v24.4s, v3.4s\n" "sqxtn v9.4h, v9.4s\n" - "srshl v26.4s, v26.4s, v19.4s\n" - "sqxtn v16.4h, v16.4s\n" - "srshl v10.4s, v10.4s, v19.4s\n" - "sqxtn v25.4h, v25.4s\n" - "sqxtn2 v13.8h, v20.4s\n" - "sqxtn2 v9.8h, v18.4s\n" - "sqxtn2 v16.8h, v26.4s\n" - "sqxtn2 v25.8h, v10.4s\n" - "sqadd v13.8h, v13.8h, v14.8h\n" - "sqadd v9.8h, v9.8h, v14.8h\n" - "sqadd v16.8h, v16.8h, v14.8h\n" - "sqadd v25.8h, v25.8h, v14.8h\n" - "smax v13.8h, v13.8h, v12.8h\n" - "smax v9.8h, v9.8h, v12.8h\n" - "smax v16.8h, v16.8h, v12.8h\n" - "smax v25.8h, v25.8h, v12.8h\n" - "smin v13.8h, v13.8h, v11.8h\n" - "smin v9.8h, v9.8h, v11.8h\n" - "smin v16.8h, v16.8h, v11.8h\n" - "smin v25.8h, v25.8h, v11.8h\n" - "uzp1 v13.16b, v13.16b, v13.16b\n" - "str d13, [x12, x17]\n" + "srshl v0.4s, v0.4s, v3.4s\n" + "sqxtn v7.4h, v7.4s\n" + "srshl v30.4s, v30.4s, v3.4s\n" + "sqxtn v2.4h, v2.4s\n" + "srshl v6.4s, v6.4s, v3.4s\n" + "sqxtn v10.4h, v10.4s\n" + "sqxtn2 v9.8h, v24.4s\n" + "sqxtn2 v7.8h, v0.4s\n" + "sqxtn2 v2.8h, v30.4s\n" + "sqxtn2 v10.8h, v6.4s\n" + "sqadd v9.8h, v9.8h, v13.8h\n" + "sqadd v7.8h, v7.8h, v13.8h\n" + "sqadd v2.8h, v2.8h, v13.8h\n" + "sqadd v10.8h, v10.8h, v13.8h\n" + "smax v9.8h, v9.8h, v29.8h\n" + "smax v7.8h, v7.8h, v29.8h\n" + "smax v2.8h, v2.8h, v29.8h\n" + "smax v10.8h, v10.8h, v29.8h\n" + "smin v9.8h, v9.8h, v12.8h\n" + "smin v7.8h, v7.8h, v12.8h\n" + "smin v2.8h, v2.8h, v12.8h\n" + "smin v10.8h, v10.8h, v12.8h\n" "uzp1 v9.16b, v9.16b, v9.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "str d9, [x11, x17]\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "str d16, [x10, x17]\n" - "str d25, [x9, x17]\n" - "add x17, x17, #0x8\n" + "str d9, [x11, x16]\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" + "uzp1 v2.16b, v2.16b, v2.16b\n" + "str d7, [x10, x16]\n" + "uzp1 v10.16b, v10.16b, v10.16b\n" + "str d2, [x9, x16]\n" + "str d10, [x28, x16]\n" + "add x16, x16, #0x8\n" "beq 64f\n" - "add x15, x15, #0x48\n" + "add x14, x14, #0x48\n" "3:" // Oddments - "ldr x28, [%x[params], %[offsetof_Params_bias]]\n" - "tbz x6, #2, 5f\n" - "ld1 { v13.4s }, [x28], #0x10\n" - "tbz x6, #1, 4f\n" - "ld1 { v20.d }[0], [x28], #0x8\n" - "tbz x6, #0, 7f\n" - "ld1 { v20.s }[2], [x28]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "tbz x7, #2, 5f\n" + "ld1 { v9.4s }, [x20], #0x10\n" + "tbz x7, #1, 4f\n" + "ld1 { v24.d }[0], [x20], #0x8\n" + "tbz x7, #0, 7f\n" + "ld1 { v24.s }[2], [x20]\n" "b 7f\n" "4:" // Oddments: Load bias: Bit 2: Bit 1: Unset - "tbz x6, #0, 7f\n" - "ld1 { v20.s }[0], [x28]\n" + "tbz x7, #0, 7f\n" + "ld1 { v24.s }[0], [x20]\n" "b 7f\n" "5:" // Oddments: Load bias: Bit 2: Unset - "tbz x6, #1, 6f\n" - "ld1 { v13.d }[0], [x28], #0x8\n" - "tbz x6, #0, 7f\n" - "ld1 { v13.s }[2], [x28]\n" + "tbz x7, #1, 6f\n" + "ld1 { v9.d }[0], [x20], #0x8\n" + "tbz x7, #0, 7f\n" + "ld1 { v9.s }[2], [x20]\n" "b 7f\n" "6:" // Oddments: Load bias: Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 7f\n" - "ld1 { v13.s }[0], [x28]\n" + "tbz x7, #0, 7f\n" + "ld1 { v9.s }[0], [x20]\n" "7:" // Oddments: Load bias: Bit 2: End - "ldr d0, [x15, #0x0]\n" - "ldr d1, [x15, #0x8]\n" - "mov v9.16b, v13.16b\n" - "mov v18.16b, v20.16b\n" - "ldr d2, [x15, #0x10]\n" - "ldr d3, [x15, #0x18]\n" - "mov v16.16b, v13.16b\n" - "mov v26.16b, v20.16b\n" - "ldr d4, [x15, #0x20]\n" - "ldr d5, [x15, #0x28]\n" - "mov v25.16b, v13.16b\n" - "mov v10.16b, v20.16b\n" - "ldr d6, [x15, #0x30]\n" - "ldr d7, [x15, #0x38]\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "ldr d8, [x15, #0x40]\n" - "ldp x24, x23, [x16, #0x0]\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "ldp x22, x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x20]\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "ssubl v5.8h, v5.8b, v15.8b\n" - "ssubl v6.8h, v6.8b, v15.8b\n" - "ssubl v7.8h, v7.8b, v15.8b\n" - "ssubl v8.8h, v8.8b, v15.8b\n" - "add x24, x24, x8\n" - "add x23, x23, x8\n" - "add x22, x22, x8\n" - "add x21, x21, x8\n" - "add x20, x20, x8\n" - "tbz x6, #2, 9f\n" - "ld1 { v31.s }[0], [x24], #0x4\n" - "ld1 { v30.s }[0], [x23], #0x4\n" - "ld1 { v29.s }[0], [x22], #0x4\n" - "ld1 { v28.s }[0], [x21], #0x4\n" - "ld1 { v27.s }[0], [x20], #0x4\n" - "tbz x6, #1, 8f\n" - "ld1 { v31.h }[2], [x24], #0x2\n" - "ld1 { v30.h }[2], [x23], #0x2\n" - "ld1 { v29.h }[2], [x22], #0x2\n" - "ld1 { v28.h }[2], [x21], #0x2\n" - "ld1 { v27.h }[2], [x20], #0x2\n" - "tbz x6, #0, 11f\n" - "ld1 { v31.b }[6], [x24]\n" - "ld1 { v30.b }[6], [x23]\n" - "ld1 { v29.b }[6], [x22]\n" - "ld1 { v28.b }[6], [x21]\n" - "ld1 { v27.b }[6], [x20]\n" + "ldr d23, [x14, #0x0]\n" + "ldr d16, [x14, #0x8]\n" + "mov v7.16b, v9.16b\n" + "mov v0.16b, v24.16b\n" + "ldr d1, [x14, #0x10]\n" + "ldr d5, [x14, #0x18]\n" + "mov v2.16b, v9.16b\n" + "mov v30.16b, v24.16b\n" + "ldr d26, [x14, #0x20]\n" + "ldr d18, [x14, #0x28]\n" + "mov v10.16b, v9.16b\n" + "mov v6.16b, v24.16b\n" + "ldr d31, [x14, #0x30]\n" + "ldr d25, [x14, #0x38]\n" + "ssubl v23.8h, v23.8b, v19.8b\n" + "ssubl v16.8h, v16.8b, v19.8b\n" + "ldr d20, [x14, #0x40]\n" + "ldp x24, x23, [x15, #0x0]\n" + "ssubl v1.8h, v1.8b, v19.8b\n" + "ssubl v5.8h, v5.8b, v19.8b\n" + "ldp x22, x21, [x15, #0x10]\n" + "ldr x20, [x15, #0x20]\n" + "ssubl v26.8h, v26.8b, v19.8b\n" + "ssubl v18.8h, v18.8b, v19.8b\n" + "ssubl v31.8h, v31.8b, v19.8b\n" + "ssubl v25.8h, v25.8b, v19.8b\n" + "ssubl v20.8h, v20.8b, v19.8b\n" + "add x24, x24, x17\n" + "add x23, x23, x17\n" + "add x22, x22, x17\n" + "add x21, x21, x17\n" + "add x20, x20, x17\n" + "tbz x7, #2, 9f\n" + "ld1 { v22.s }[0], [x24], #0x4\n" + "ld1 { v4.s }[0], [x23], #0x4\n" + "ld1 { v8.s }[0], [x22], #0x4\n" + "ld1 { v27.s }[0], [x21], #0x4\n" + "ld1 { v15.s }[0], [x20], #0x4\n" + "tbz x7, #1, 8f\n" + "ld1 { v22.h }[2], [x24], #0x2\n" + "ld1 { v4.h }[2], [x23], #0x2\n" + "ld1 { v8.h }[2], [x22], #0x2\n" + "ld1 { v27.h }[2], [x21], #0x2\n" + "ld1 { v15.h }[2], [x20], #0x2\n" + "tbz x7, #0, 11f\n" + "ld1 { v22.b }[6], [x24]\n" + "ld1 { v4.b }[6], [x23]\n" + "ld1 { v8.b }[6], [x22]\n" + "ld1 { v27.b }[6], [x21]\n" + "ld1 { v15.b }[6], [x20]\n" "b 11f\n" "8:" // Oddments: Initial loads: Bit 2: Bit 1: Unset - "tbz x6, #0, 11f\n" - "ld1 { v31.b }[4], [x24]\n" - "ld1 { v30.b }[4], [x23]\n" - "ld1 { v29.b }[4], [x22]\n" - "ld1 { v28.b }[4], [x21]\n" - "ld1 { v27.b }[4], [x20]\n" + "tbz x7, #0, 11f\n" + "ld1 { v22.b }[4], [x24]\n" + "ld1 { v4.b }[4], [x23]\n" + "ld1 { v8.b }[4], [x22]\n" + "ld1 { v27.b }[4], [x21]\n" + "ld1 { v15.b }[4], [x20]\n" "b 11f\n" "9:" // Oddments: Initial loads: Bit 2: Unset - "tbz x6, #1, 10f\n" - "ld1 { v31.h }[0], [x24], #0x2\n" - "ld1 { v30.h }[0], [x23], #0x2\n" - "ld1 { v29.h }[0], [x22], #0x2\n" - "ld1 { v28.h }[0], [x21], #0x2\n" - "ld1 { v27.h }[0], [x20], #0x2\n" - "tbz x6, #0, 11f\n" - "ld1 { v31.b }[2], [x24]\n" - "ld1 { v30.b }[2], [x23]\n" - "ld1 { v29.b }[2], [x22]\n" - "ld1 { v28.b }[2], [x21]\n" - "ld1 { v27.b }[2], [x20]\n" + "tbz x7, #1, 10f\n" + "ld1 { v22.h }[0], [x24], #0x2\n" + "ld1 { v4.h }[0], [x23], #0x2\n" + "ld1 { v8.h }[0], [x22], #0x2\n" + "ld1 { v27.h }[0], [x21], #0x2\n" + "ld1 { v15.h }[0], [x20], #0x2\n" + "tbz x7, #0, 11f\n" + "ld1 { v22.b }[2], [x24]\n" + "ld1 { v4.b }[2], [x23]\n" + "ld1 { v8.b }[2], [x22]\n" + "ld1 { v27.b }[2], [x21]\n" + "ld1 { v15.b }[2], [x20]\n" "b 11f\n" "10:" // Oddments: Initial loads: Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 11f\n" - "ld1 { v31.b }[0], [x24]\n" - "ld1 { v30.b }[0], [x23]\n" - "ld1 { v29.b }[0], [x22]\n" - "ld1 { v28.b }[0], [x21]\n" - "ld1 { v27.b }[0], [x20]\n" + "tbz x7, #0, 11f\n" + "ld1 { v22.b }[0], [x24]\n" + "ld1 { v4.b }[0], [x23]\n" + "ld1 { v8.b }[0], [x22]\n" + "ld1 { v27.b }[0], [x21]\n" + "ld1 { v15.b }[0], [x20]\n" "11:" // Oddments: Initial loads: Bit 2: End - "ssubl v31.8h, v31.8b, v24.8b\n" - "smlal v13.4s, v31.4h, v4.4h\n" - "smlal2 v20.4s, v31.8h, v4.8h\n" - "ldr x21, [x16, #0x28]\n" - "smlal v9.4s, v31.4h, v3.4h\n" - "smlal2 v18.4s, v31.8h, v3.8h\n" - "ssubl v30.8h, v30.8b, v24.8b\n" - "add x21, x21, x8\n" - "ssubl v29.8h, v29.8b, v24.8b\n" - "smlal v16.4s, v31.4h, v1.4h\n" - "smlal2 v26.4s, v31.8h, v1.8h\n" - "smlal v25.4s, v31.4h, v0.4h\n" - "smlal2 v10.4s, v31.8h, v0.8h\n" - "ssubl v28.8h, v28.8b, v24.8b\n" - "smlal v13.4s, v30.4h, v0.4h\n" - "smlal2 v20.4s, v30.8h, v0.8h\n" - "ssubl v27.8h, v27.8b, v24.8b\n" - "smlal v9.4s, v29.4h, v2.4h\n" - "smlal2 v18.4s, v29.8h, v2.8h\n" - "smlal v13.4s, v28.4h, v5.4h\n" - "smlal2 v20.4s, v28.8h, v5.8h\n" - "smlal v9.4s, v28.4h, v4.4h\n" - "smlal2 v18.4s, v28.8h, v4.8h\n" - "smlal v16.4s, v28.4h, v2.4h\n" - "smlal2 v26.4s, v28.8h, v2.8h\n" - "smlal v25.4s, v28.4h, v1.4h\n" - "smlal2 v10.4s, v28.8h, v1.8h\n" - "tbz x6, #2, 13f\n" - "ld1 { v31.s }[0], [x21], #0x4\n" - "tbz x6, #1, 12f\n" - "ld1 { v31.h }[2], [x21], #0x2\n" - "tbz x6, #0, 15f\n" - "ld1 { v31.b }[6], [x21]\n" + "ssubl v22.8h, v22.8b, v14.8b\n" + "smlal v9.4s, v22.4h, v26.4h\n" + "smlal2 v24.4s, v22.8h, v26.8h\n" + "ldr x20, [x15, #0x28]\n" + "smlal v7.4s, v22.4h, v5.4h\n" + "smlal2 v0.4s, v22.8h, v5.8h\n" + "ssubl v4.8h, v4.8b, v14.8b\n" + "ssubl v8.8h, v8.8b, v14.8b\n" + "smlal v2.4s, v22.4h, v16.4h\n" + "smlal2 v30.4s, v22.8h, v16.8h\n" + "add x20, x20, x17\n" + "smlal v10.4s, v22.4h, v23.4h\n" + "smlal2 v6.4s, v22.8h, v23.8h\n" + "ssubl v27.8h, v27.8b, v14.8b\n" + "smlal v9.4s, v4.4h, v23.4h\n" + "smlal2 v24.4s, v4.8h, v23.8h\n" + "ssubl v15.8h, v15.8b, v14.8b\n" + "smlal v7.4s, v8.4h, v1.4h\n" + "smlal2 v0.4s, v8.8h, v1.8h\n" + "smlal v9.4s, v27.4h, v18.4h\n" + "smlal2 v24.4s, v27.8h, v18.8h\n" + "smlal v7.4s, v27.4h, v26.4h\n" + "smlal2 v0.4s, v27.8h, v26.8h\n" + "smlal v2.4s, v27.4h, v1.4h\n" + "smlal2 v30.4s, v27.8h, v1.8h\n" + "smlal v10.4s, v27.4h, v16.4h\n" + "smlal2 v6.4s, v27.8h, v16.8h\n" + "tbz x7, #2, 13f\n" + "ld1 { v21.s }[0], [x20], #0x4\n" + "tbz x7, #1, 12f\n" + "ld1 { v21.h }[2], [x20], #0x2\n" + "tbz x7, #0, 15f\n" + "ld1 { v21.b }[6], [x20]\n" "b 15f\n" "12:" // Oddments: Load (3, 0): Bit 2: Bit 1: Unset - "tbz x6, #0, 15f\n" - "ld1 { v31.b }[4], [x21]\n" + "tbz x7, #0, 15f\n" + "ld1 { v21.b }[4], [x20]\n" "b 15f\n" "13:" // Oddments: Load (3, 0): Bit 2: Unset - "tbz x6, #1, 14f\n" - "ld1 { v31.h }[0], [x21], #0x2\n" - "tbz x6, #0, 15f\n" - "ld1 { v31.b }[2], [x21]\n" + "tbz x7, #1, 14f\n" + "ld1 { v21.h }[0], [x20], #0x2\n" + "tbz x7, #0, 15f\n" + "ld1 { v21.b }[2], [x20]\n" "b 15f\n" "14:" // Oddments: Load (3, 0): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 15f\n" - "ld1 { v31.b }[0], [x21]\n" + "tbz x7, #0, 15f\n" + "ld1 { v21.b }[0], [x20]\n" "15:" // Oddments: Load (3, 0): Bit 2: End - "ssubl v31.8h, v31.8b, v24.8b\n" - "smlal v16.4s, v31.4h, v6.4h\n" - "smlal2 v26.4s, v31.8h, v6.8h\n" - "ldr x20, [x16, #0x30]\n" - "smlal v13.4s, v27.4h, v7.4h\n" - "smlal2 v20.4s, v27.8h, v7.8h\n" - "add x20, x20, x8\n" - "smlal v9.4s, v27.4h, v6.4h\n" - "smlal2 v18.4s, v27.8h, v6.8h\n" - "smlal v16.4s, v27.4h, v4.4h\n" - "smlal2 v26.4s, v27.8h, v4.8h\n" - "smlal v25.4s, v27.4h, v3.4h\n" - "smlal2 v10.4s, v27.8h, v3.8h\n" - "tbz x6, #2, 17f\n" - "ld1 { v29.s }[0], [x20], #0x4\n" - "tbz x6, #1, 16f\n" - "ld1 { v29.h }[2], [x20], #0x2\n" - "tbz x6, #0, 19f\n" - "ld1 { v29.b }[6], [x20]\n" + "ssubl v21.8h, v21.8b, v14.8b\n" + "smlal v2.4s, v21.4h, v31.4h\n" + "smlal2 v30.4s, v21.8h, v31.8h\n" + "ldr x20, [x15, #0x30]\n" + "smlal v9.4s, v15.4h, v25.4h\n" + "smlal2 v24.4s, v15.8h, v25.8h\n" + "add x20, x20, x17\n" + "smlal v7.4s, v15.4h, v31.4h\n" + "smlal2 v0.4s, v15.8h, v31.8h\n" + "smlal v2.4s, v15.4h, v26.4h\n" + "smlal2 v30.4s, v15.8h, v26.8h\n" + "smlal v10.4s, v15.4h, v5.4h\n" + "smlal2 v6.4s, v15.8h, v5.8h\n" + "tbz x7, #2, 17f\n" + "ld1 { v28.s }[0], [x20], #0x4\n" + "tbz x7, #1, 16f\n" + "ld1 { v28.h }[2], [x20], #0x2\n" + "tbz x7, #0, 19f\n" + "ld1 { v28.b }[6], [x20]\n" "b 19f\n" "16:" // Oddments: Load (3, 3): Bit 2: Bit 1: Unset - "tbz x6, #0, 19f\n" - "ld1 { v29.b }[4], [x20]\n" + "tbz x7, #0, 19f\n" + "ld1 { v28.b }[4], [x20]\n" "b 19f\n" "17:" // Oddments: Load (3, 3): Bit 2: Unset - "tbz x6, #1, 18f\n" - "ld1 { v29.h }[0], [x20], #0x2\n" - "tbz x6, #0, 19f\n" - "ld1 { v29.b }[2], [x20]\n" + "tbz x7, #1, 18f\n" + "ld1 { v28.h }[0], [x20], #0x2\n" + "tbz x7, #0, 19f\n" + "ld1 { v28.b }[2], [x20]\n" "b 19f\n" "18:" // Oddments: Load (3, 3): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 19f\n" - "ld1 { v29.b }[0], [x20]\n" + "tbz x7, #0, 19f\n" + "ld1 { v28.b }[0], [x20]\n" "19:" // Oddments: Load (3, 3): Bit 2: End - "ssubl v29.8h, v29.8b, v24.8b\n" - "ldr x28, [x16, #0x38]\n" - "smlal v25.4s, v29.4h, v8.4h\n" - "smlal2 v10.4s, v29.8h, v8.8h\n" - "add x28, x28, x8\n" - "tbz x6, #2, 21f\n" - "ld1 { v28.s }[0], [x28], #0x4\n" - "tbz x6, #1, 20f\n" - "ld1 { v28.h }[2], [x28], #0x2\n" - "tbz x6, #0, 23f\n" - "ld1 { v28.b }[6], [x28]\n" + "ssubl v28.8h, v28.8b, v14.8b\n" + "ldr x20, [x15, #0x38]\n" + "smlal v10.4s, v28.4h, v20.4h\n" + "smlal2 v6.4s, v28.8h, v20.8h\n" + "add x20, x20, x17\n" + "tbz x7, #2, 21f\n" + "ld1 { v22.s }[0], [x20], #0x4\n" + "tbz x7, #1, 20f\n" + "ld1 { v22.h }[2], [x20], #0x2\n" + "tbz x7, #0, 23f\n" + "ld1 { v22.b }[6], [x20]\n" "b 23f\n" "20:" // Oddments: Load (0, 1): Bit 2: Bit 1: Unset - "tbz x6, #0, 23f\n" - "ld1 { v28.b }[4], [x28]\n" + "tbz x7, #0, 23f\n" + "ld1 { v22.b }[4], [x20]\n" "b 23f\n" "21:" // Oddments: Load (0, 1): Bit 2: Unset - "tbz x6, #1, 22f\n" - "ld1 { v28.h }[0], [x28], #0x2\n" - "tbz x6, #0, 23f\n" - "ld1 { v28.b }[2], [x28]\n" + "tbz x7, #1, 22f\n" + "ld1 { v22.h }[0], [x20], #0x2\n" + "tbz x7, #0, 23f\n" + "ld1 { v22.b }[2], [x20]\n" "b 23f\n" "22:" // Oddments: Load (0, 1): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 23f\n" - "ld1 { v28.b }[0], [x28]\n" + "tbz x7, #0, 23f\n" + "ld1 { v22.b }[0], [x20]\n" "23:" // Oddments: Load (0, 1): Bit 2: End - "ssubl v28.8h, v28.8b, v24.8b\n" - "ldr x27, [x16, #0x40]\n" - "smlal v13.4s, v28.4h, v1.4h\n" - "smlal2 v20.4s, v28.8h, v1.8h\n" - "smlal v9.4s, v28.4h, v0.4h\n" - "smlal2 v18.4s, v28.8h, v0.8h\n" - "add x27, x27, x8\n" - "tbz x6, #2, 25f\n" - "ld1 { v31.s }[0], [x27], #0x4\n" - "tbz x6, #1, 24f\n" - "ld1 { v31.h }[2], [x27], #0x2\n" - "tbz x6, #0, 27f\n" - "ld1 { v31.b }[6], [x27]\n" + "ssubl v22.8h, v22.8b, v14.8b\n" + "ldr x20, [x15, #0x40]\n" + "smlal v9.4s, v22.4h, v16.4h\n" + "smlal2 v24.4s, v22.8h, v16.8h\n" + "smlal v7.4s, v22.4h, v23.4h\n" + "smlal2 v0.4s, v22.8h, v23.8h\n" + "add x20, x20, x17\n" + "tbz x7, #2, 25f\n" + "ld1 { v21.s }[0], [x20], #0x4\n" + "tbz x7, #1, 24f\n" + "ld1 { v21.h }[2], [x20], #0x2\n" + "tbz x7, #0, 27f\n" + "ld1 { v21.b }[6], [x20]\n" "b 27f\n" "24:" // Oddments: Load (0, 2): Bit 2: Bit 1: Unset - "tbz x6, #0, 27f\n" - "ld1 { v31.b }[4], [x27]\n" + "tbz x7, #0, 27f\n" + "ld1 { v21.b }[4], [x20]\n" "b 27f\n" "25:" // Oddments: Load (0, 2): Bit 2: Unset - "tbz x6, #1, 26f\n" - "ld1 { v31.h }[0], [x27], #0x2\n" - "tbz x6, #0, 27f\n" - "ld1 { v31.b }[2], [x27]\n" + "tbz x7, #1, 26f\n" + "ld1 { v21.h }[0], [x20], #0x2\n" + "tbz x7, #0, 27f\n" + "ld1 { v21.b }[2], [x20]\n" "b 27f\n" "26:" // Oddments: Load (0, 2): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 27f\n" - "ld1 { v31.b }[0], [x27]\n" + "tbz x7, #0, 27f\n" + "ld1 { v21.b }[0], [x20]\n" "27:" // Oddments: Load (0, 2): Bit 2: End - "ssubl v31.8h, v31.8b, v24.8b\n" - "ldr x26, [x16, #0x48]\n" - "smlal v13.4s, v31.4h, v2.4h\n" - "smlal2 v20.4s, v31.8h, v2.8h\n" - "smlal v9.4s, v31.4h, v1.4h\n" - "smlal2 v18.4s, v31.8h, v1.8h\n" - "add x26, x26, x8\n" - "tbz x6, #2, 29f\n" - "ld1 { v30.s }[0], [x26], #0x4\n" - "tbz x6, #1, 28f\n" - "ld1 { v30.h }[2], [x26], #0x2\n" - "tbz x6, #0, 31f\n" - "ld1 { v30.b }[6], [x26]\n" + "ssubl v21.8h, v21.8b, v14.8b\n" + "ldr x20, [x15, #0x48]\n" + "smlal v9.4s, v21.4h, v1.4h\n" + "smlal2 v24.4s, v21.8h, v1.8h\n" + "smlal v7.4s, v21.4h, v16.4h\n" + "smlal2 v0.4s, v21.8h, v16.8h\n" + "add x20, x20, x17\n" + "tbz x7, #2, 29f\n" + "ld1 { v28.s }[0], [x20], #0x4\n" + "tbz x7, #1, 28f\n" + "ld1 { v28.h }[2], [x20], #0x2\n" + "tbz x7, #0, 31f\n" + "ld1 { v28.b }[6], [x20]\n" "b 31f\n" "28:" // Oddments: Load (2, 2): Bit 2: Bit 1: Unset - "tbz x6, #0, 31f\n" - "ld1 { v30.b }[4], [x26]\n" + "tbz x7, #0, 31f\n" + "ld1 { v28.b }[4], [x20]\n" "b 31f\n" "29:" // Oddments: Load (2, 2): Bit 2: Unset - "tbz x6, #1, 30f\n" - "ld1 { v30.h }[0], [x26], #0x2\n" - "tbz x6, #0, 31f\n" - "ld1 { v30.b }[2], [x26]\n" + "tbz x7, #1, 30f\n" + "ld1 { v28.h }[0], [x20], #0x2\n" + "tbz x7, #0, 31f\n" + "ld1 { v28.b }[2], [x20]\n" "b 31f\n" "30:" // Oddments: Load (2, 2): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 31f\n" - "ld1 { v30.b }[0], [x26]\n" + "tbz x7, #0, 31f\n" + "ld1 { v28.b }[0], [x20]\n" "31:" // Oddments: Load (2, 2): Bit 2: End - "ssubl v30.8h, v30.8b, v24.8b\n" - "ldr x25, [x16, #0x50]\n" - "smlal v13.4s, v30.4h, v8.4h\n" - "smlal2 v20.4s, v30.8h, v8.8h\n" - "smlal v9.4s, v30.4h, v7.4h\n" - "smlal2 v18.4s, v30.8h, v7.8h\n" - "add x25, x25, x8\n" - "smlal v16.4s, v30.4h, v5.4h\n" - "smlal2 v26.4s, v30.8h, v5.8h\n" - "smlal v25.4s, v30.4h, v4.4h\n" - "smlal2 v10.4s, v30.8h, v4.8h\n" - "tbz x6, #2, 33f\n" - "ld1 { v29.s }[0], [x25], #0x4\n" - "tbz x6, #1, 32f\n" - "ld1 { v29.h }[2], [x25], #0x2\n" - "tbz x6, #0, 35f\n" - "ld1 { v29.b }[6], [x25]\n" + "ssubl v28.8h, v28.8b, v14.8b\n" + "ldr x20, [x15, #0x50]\n" + "smlal v9.4s, v28.4h, v20.4h\n" + "smlal2 v24.4s, v28.8h, v20.8h\n" + "smlal v7.4s, v28.4h, v25.4h\n" + "smlal2 v0.4s, v28.8h, v25.8h\n" + "add x20, x20, x17\n" + "smlal v2.4s, v28.4h, v18.4h\n" + "smlal2 v30.4s, v28.8h, v18.8h\n" + "smlal v10.4s, v28.4h, v26.4h\n" + "smlal2 v6.4s, v28.8h, v26.8h\n" + "tbz x7, #2, 33f\n" + "ld1 { v8.s }[0], [x20], #0x4\n" + "tbz x7, #1, 32f\n" + "ld1 { v8.h }[2], [x20], #0x2\n" + "tbz x7, #0, 35f\n" + "ld1 { v8.b }[6], [x20]\n" "b 35f\n" "32:" // Oddments: Load (1, 0): Bit 2: Bit 1: Unset - "tbz x6, #0, 35f\n" - "ld1 { v29.b }[4], [x25]\n" + "tbz x7, #0, 35f\n" + "ld1 { v8.b }[4], [x20]\n" "b 35f\n" "33:" // Oddments: Load (1, 0): Bit 2: Unset - "tbz x6, #1, 34f\n" - "ld1 { v29.h }[0], [x25], #0x2\n" - "tbz x6, #0, 35f\n" - "ld1 { v29.b }[2], [x25]\n" + "tbz x7, #1, 34f\n" + "ld1 { v8.h }[0], [x20], #0x2\n" + "tbz x7, #0, 35f\n" + "ld1 { v8.b }[2], [x20]\n" "b 35f\n" "34:" // Oddments: Load (1, 0): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 35f\n" - "ld1 { v29.b }[0], [x25]\n" + "tbz x7, #0, 35f\n" + "ld1 { v8.b }[0], [x20]\n" "35:" // Oddments: Load (1, 0): Bit 2: End - "ssubl v29.8h, v29.8b, v24.8b\n" - "ldr x24, [x16, #0x58]\n" - "smlal v13.4s, v29.4h, v3.4h\n" - "smlal2 v20.4s, v29.8h, v3.8h\n" - "smlal v16.4s, v29.4h, v0.4h\n" - "smlal2 v26.4s, v29.8h, v0.8h\n" - "add x24, x24, x8\n" - "tbz x6, #2, 37f\n" - "ld1 { v28.s }[0], [x24], #0x4\n" - "tbz x6, #1, 36f\n" - "ld1 { v28.h }[2], [x24], #0x2\n" - "tbz x6, #0, 39f\n" - "ld1 { v28.b }[6], [x24]\n" + "ssubl v8.8h, v8.8b, v14.8b\n" + "ldr x20, [x15, #0x58]\n" + "smlal v9.4s, v8.4h, v5.4h\n" + "smlal2 v24.4s, v8.8h, v5.8h\n" + "smlal v2.4s, v8.4h, v23.4h\n" + "smlal2 v30.4s, v8.8h, v23.8h\n" + "add x20, x20, x17\n" + "tbz x7, #2, 37f\n" + "ld1 { v8.s }[0], [x20], #0x4\n" + "tbz x7, #1, 36f\n" + "ld1 { v8.h }[2], [x20], #0x2\n" + "tbz x7, #0, 39f\n" + "ld1 { v8.b }[6], [x20]\n" "b 39f\n" "36:" // Oddments: Load (1, 3): Bit 2: Bit 1: Unset - "tbz x6, #0, 39f\n" - "ld1 { v28.b }[4], [x24]\n" + "tbz x7, #0, 39f\n" + "ld1 { v8.b }[4], [x20]\n" "b 39f\n" "37:" // Oddments: Load (1, 3): Bit 2: Unset - "tbz x6, #1, 38f\n" - "ld1 { v28.h }[0], [x24], #0x2\n" - "tbz x6, #0, 39f\n" - "ld1 { v28.b }[2], [x24]\n" + "tbz x7, #1, 38f\n" + "ld1 { v8.h }[0], [x20], #0x2\n" + "tbz x7, #0, 39f\n" + "ld1 { v8.b }[2], [x20]\n" "b 39f\n" "38:" // Oddments: Load (1, 3): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 39f\n" - "ld1 { v28.b }[0], [x24]\n" + "tbz x7, #0, 39f\n" + "ld1 { v8.b }[0], [x20]\n" "39:" // Oddments: Load (1, 3): Bit 2: End - "ssubl v28.8h, v28.8b, v24.8b\n" - "ldr x23, [x16, #0x60]\n" - "smlal v9.4s, v28.4h, v5.4h\n" - "smlal2 v18.4s, v28.8h, v5.8h\n" - "smlal v25.4s, v28.4h, v2.4h\n" - "smlal2 v10.4s, v28.8h, v2.8h\n" - "add x23, x23, x8\n" - "tbz x6, #2, 41f\n" - "ld1 { v31.s }[0], [x23], #0x4\n" - "tbz x6, #1, 40f\n" - "ld1 { v31.h }[2], [x23], #0x2\n" - "tbz x6, #0, 43f\n" - "ld1 { v31.b }[6], [x23]\n" + "ssubl v8.8h, v8.8b, v14.8b\n" + "ldr x20, [x15, #0x60]\n" + "smlal v7.4s, v8.4h, v18.4h\n" + "smlal2 v0.4s, v8.8h, v18.8h\n" + "smlal v10.4s, v8.4h, v1.4h\n" + "smlal2 v6.4s, v8.8h, v1.8h\n" + "add x20, x20, x17\n" + "tbz x7, #2, 41f\n" + "ld1 { v17.s }[0], [x20], #0x4\n" + "tbz x7, #1, 40f\n" + "ld1 { v17.h }[2], [x20], #0x2\n" + "tbz x7, #0, 43f\n" + "ld1 { v17.b }[6], [x20]\n" "b 43f\n" "40:" // Oddments: Load (2, 0): Bit 2: Bit 1: Unset - "tbz x6, #0, 43f\n" - "ld1 { v31.b }[4], [x23]\n" + "tbz x7, #0, 43f\n" + "ld1 { v17.b }[4], [x20]\n" "b 43f\n" "41:" // Oddments: Load (2, 0): Bit 2: Unset - "tbz x6, #1, 42f\n" - "ld1 { v31.h }[0], [x23], #0x2\n" - "tbz x6, #0, 43f\n" - "ld1 { v31.b }[2], [x23]\n" + "tbz x7, #1, 42f\n" + "ld1 { v17.h }[0], [x20], #0x2\n" + "tbz x7, #0, 43f\n" + "ld1 { v17.b }[2], [x20]\n" "b 43f\n" "42:" // Oddments: Load (2, 0): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 43f\n" - "ld1 { v31.b }[0], [x23]\n" + "tbz x7, #0, 43f\n" + "ld1 { v17.b }[0], [x20]\n" "43:" // Oddments: Load (2, 0): Bit 2: End - "ssubl v31.8h, v31.8b, v24.8b\n" - "ldr x22, [x16, #0x68]\n" - "smlal v13.4s, v31.4h, v6.4h\n" - "smlal2 v20.4s, v31.8h, v6.8h\n" - "smlal v16.4s, v31.4h, v3.4h\n" - "smlal2 v26.4s, v31.8h, v3.8h\n" - "add x22, x22, x8\n" - "tbz x6, #2, 45f\n" - "ld1 { v30.s }[0], [x22], #0x4\n" - "tbz x6, #1, 44f\n" - "ld1 { v30.h }[2], [x22], #0x2\n" - "tbz x6, #0, 47f\n" - "ld1 { v30.b }[6], [x22]\n" + "ssubl v17.8h, v17.8b, v14.8b\n" + "ldr x20, [x15, #0x68]\n" + "smlal v9.4s, v17.4h, v31.4h\n" + "smlal2 v24.4s, v17.8h, v31.8h\n" + "smlal v2.4s, v17.4h, v5.4h\n" + "smlal2 v30.4s, v17.8h, v5.8h\n" + "add x20, x20, x17\n" + "tbz x7, #2, 45f\n" + "ld1 { v23.s }[0], [x20], #0x4\n" + "tbz x7, #1, 44f\n" + "ld1 { v23.h }[2], [x20], #0x2\n" + "tbz x7, #0, 47f\n" + "ld1 { v23.b }[6], [x20]\n" "b 47f\n" "44:" // Oddments: Load (2, 3): Bit 2: Bit 1: Unset - "tbz x6, #0, 47f\n" - "ld1 { v30.b }[4], [x22]\n" + "tbz x7, #0, 47f\n" + "ld1 { v23.b }[4], [x20]\n" "b 47f\n" "45:" // Oddments: Load (2, 3): Bit 2: Unset - "tbz x6, #1, 46f\n" - "ld1 { v30.h }[0], [x22], #0x2\n" - "tbz x6, #0, 47f\n" - "ld1 { v30.b }[2], [x22]\n" + "tbz x7, #1, 46f\n" + "ld1 { v23.h }[0], [x20], #0x2\n" + "tbz x7, #0, 47f\n" + "ld1 { v23.b }[2], [x20]\n" "b 47f\n" "46:" // Oddments: Load (2, 3): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 47f\n" - "ld1 { v30.b }[0], [x22]\n" + "tbz x7, #0, 47f\n" + "ld1 { v23.b }[0], [x20]\n" "47:" // Oddments: Load (2, 3): Bit 2: End - "ssubl v30.8h, v30.8b, v24.8b\n" - "ldr x21, [x16, #0x70]\n" - "smlal v9.4s, v30.4h, v8.4h\n" - "smlal2 v18.4s, v30.8h, v8.8h\n" - "smlal v25.4s, v30.4h, v5.4h\n" - "smlal2 v10.4s, v30.8h, v5.8h\n" - "add x21, x21, x8\n" - "tbz x6, #2, 49f\n" - "ld1 { v29.s }[0], [x21], #0x4\n" - "tbz x6, #1, 48f\n" - "ld1 { v29.h }[2], [x21], #0x2\n" - "tbz x6, #0, 51f\n" - "ld1 { v29.b }[6], [x21]\n" + "ssubl v23.8h, v23.8b, v14.8b\n" + "ldr x20, [x15, #0x70]\n" + "smlal v7.4s, v23.4h, v20.4h\n" + "smlal2 v0.4s, v23.8h, v20.8h\n" + "smlal v10.4s, v23.4h, v18.4h\n" + "smlal2 v6.4s, v23.8h, v18.8h\n" + "add x20, x20, x17\n" + "tbz x7, #2, 49f\n" + "ld1 { v5.s }[0], [x20], #0x4\n" + "tbz x7, #1, 48f\n" + "ld1 { v5.h }[2], [x20], #0x2\n" + "tbz x7, #0, 51f\n" + "ld1 { v5.b }[6], [x20]\n" "b 51f\n" "48:" // Oddments: Load (3, 1): Bit 2: Bit 1: Unset - "tbz x6, #0, 51f\n" - "ld1 { v29.b }[4], [x21]\n" + "tbz x7, #0, 51f\n" + "ld1 { v5.b }[4], [x20]\n" "b 51f\n" "49:" // Oddments: Load (3, 1): Bit 2: Unset - "tbz x6, #1, 50f\n" - "ld1 { v29.h }[0], [x21], #0x2\n" - "tbz x6, #0, 51f\n" - "ld1 { v29.b }[2], [x21]\n" + "tbz x7, #1, 50f\n" + "ld1 { v5.h }[0], [x20], #0x2\n" + "tbz x7, #0, 51f\n" + "ld1 { v5.b }[2], [x20]\n" "b 51f\n" "50:" // Oddments: Load (3, 1): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 51f\n" - "ld1 { v29.b }[0], [x21]\n" + "tbz x7, #0, 51f\n" + "ld1 { v5.b }[0], [x20]\n" "51:" // Oddments: Load (3, 1): Bit 2: End - "ssubl v29.8h, v29.8b, v24.8b\n" - "ldr x20, [x16, #0x78]\n" - "smlal v16.4s, v29.4h, v7.4h\n" - "smlal2 v26.4s, v29.8h, v7.8h\n" - "smlal v25.4s, v29.4h, v6.4h\n" - "smlal2 v10.4s, v29.8h, v6.8h\n" - "add x20, x20, x8\n" - "tbz x6, #2, 53f\n" - "ld1 { v28.s }[0], [x20], #0x4\n" - "tbz x6, #1, 52f\n" - "ld1 { v28.h }[2], [x20], #0x2\n" - "tbz x6, #0, 55f\n" - "ld1 { v28.b }[6], [x20]\n" + "ssubl v5.8h, v5.8b, v14.8b\n" + "ldr x20, [x15, #0x78]\n" + "smlal v2.4s, v5.4h, v25.4h\n" + "smlal2 v30.4s, v5.8h, v25.8h\n" + "smlal v10.4s, v5.4h, v31.4h\n" + "smlal2 v6.4s, v5.8h, v31.8h\n" + "add x20, x20, x17\n" + "tbz x7, #2, 53f\n" + "ld1 { v23.s }[0], [x20], #0x4\n" + "tbz x7, #1, 52f\n" + "ld1 { v23.h }[2], [x20], #0x2\n" + "tbz x7, #0, 55f\n" + "ld1 { v23.b }[6], [x20]\n" "b 55f\n" "52:" // Oddments: Load (3, 2): Bit 2: Bit 1: Unset - "tbz x6, #0, 55f\n" - "ld1 { v28.b }[4], [x20]\n" + "tbz x7, #0, 55f\n" + "ld1 { v23.b }[4], [x20]\n" "b 55f\n" "53:" // Oddments: Load (3, 2): Bit 2: Unset - "tbz x6, #1, 54f\n" - "ld1 { v28.h }[0], [x20], #0x2\n" - "tbz x6, #0, 55f\n" - "ld1 { v28.b }[2], [x20]\n" + "tbz x7, #1, 54f\n" + "ld1 { v23.h }[0], [x20], #0x2\n" + "tbz x7, #0, 55f\n" + "ld1 { v23.b }[2], [x20]\n" "b 55f\n" "54:" // Oddments: Load (3, 2): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 55f\n" - "ld1 { v28.b }[0], [x20]\n" + "tbz x7, #0, 55f\n" + "ld1 { v23.b }[0], [x20]\n" "55:" // Oddments: Load (3, 2): Bit 2: End - "ssubl v28.8h, v28.8b, v24.8b\n" - "smlal v16.4s, v28.4h, v8.4h\n" - "smlal2 v26.4s, v28.8h, v8.8h\n" - "smlal v25.4s, v28.4h, v7.4h\n" - "smlal2 v10.4s, v28.8h, v7.8h\n" - "tbz x6, #2, 57f\n" - "ld1 { v17.4s }, [x14], #0x10\n" - "ld1 { v22.4s }, [x13], #0x10\n" - "tbz x6, #1, 56f\n" - "ld1 { v23.d }[0], [x14], #0x8\n" - "ld1 { v19.d }[0], [x13], #0x8\n" - "tbz x6, #0, 59f\n" - "ld1 { v23.s }[2], [x14]\n" - "ld1 { v19.s }[2], [x13]\n" + "ssubl v23.8h, v23.8b, v14.8b\n" + "smlal v2.4s, v23.4h, v20.4h\n" + "smlal2 v30.4s, v23.8h, v20.8h\n" + "smlal v10.4s, v23.4h, v25.4h\n" + "smlal2 v6.4s, v23.8h, v25.8h\n" + "tbz x7, #2, 57f\n" + "ld1 { v15.4s }, [x13], #0x10\n" + "ld1 { v19.4s }, [x12], #0x10\n" + "tbz x7, #1, 56f\n" + "ld1 { v18.d }[0], [x13], #0x8\n" + "ld1 { v22.d }[0], [x12], #0x8\n" + "tbz x7, #0, 59f\n" + "ld1 { v18.s }[2], [x13]\n" + "ld1 { v22.s }[2], [x12]\n" "b 59f\n" "56:" // Oddments: Load requant params: Bit 2: Bit 1: Unset - "tbz x6, #0, 59f\n" - "ld1 { v23.s }[0], [x14]\n" - "ld1 { v19.s }[0], [x13]\n" + "tbz x7, #0, 59f\n" + "ld1 { v18.s }[0], [x13]\n" + "ld1 { v22.s }[0], [x12]\n" "b 59f\n" "57:" // Oddments: Load requant params: Bit 2: Unset - "tbz x6, #1, 58f\n" - "ld1 { v17.d }[0], [x14], #0x8\n" - "ld1 { v22.d }[0], [x13], #0x8\n" - "tbz x6, #0, 59f\n" - "ld1 { v17.s }[2], [x14]\n" - "ld1 { v22.s }[2], [x13]\n" + "tbz x7, #1, 58f\n" + "ld1 { v15.d }[0], [x13], #0x8\n" + "ld1 { v19.d }[0], [x12], #0x8\n" + "tbz x7, #0, 59f\n" + "ld1 { v15.s }[2], [x13]\n" + "ld1 { v19.s }[2], [x12]\n" "b 59f\n" "58:" // Oddments: Load requant params: Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 59f\n" - "ld1 { v17.s }[0], [x14]\n" - "ld1 { v22.s }[0], [x13]\n" + "tbz x7, #0, 59f\n" + "ld1 { v15.s }[0], [x13]\n" + "ld1 { v19.s }[0], [x12]\n" "59:" // Oddments: Load requant params: Bit 2: End - "sqrdmulh v13.4s, v13.4s, v17.4s\n" - "and v21.16b, v13.16b, v22.16b\n" - "add x12, x12, x17\n" - "add x11, x11, x17\n" - "sqrdmulh v20.4s, v20.4s, v23.4s\n" - "sshr v21.4s, v21.4s, #0x1f\n" - "add x10, x10, x17\n" - "add x9, x9, x17\n" - "and v29.16b, v20.16b, v19.16b\n" - "sqrdmulh v9.4s, v9.4s, v17.4s\n" - "sqrdmulh v16.4s, v16.4s, v17.4s\n" - "sqrdmulh v25.4s, v25.4s, v17.4s\n" - "sqadd v13.4s, v13.4s, v21.4s\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "and v0.16b, v9.16b, v22.16b\n" - "sqrdmulh v18.4s, v18.4s, v23.4s\n" - "and v27.16b, v16.16b, v22.16b\n" - "sqrdmulh v26.4s, v26.4s, v23.4s\n" - "and v21.16b, v25.16b, v22.16b\n" - "sqrdmulh v10.4s, v10.4s, v23.4s\n" - "sqadd v20.4s, v20.4s, v29.4s\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v17.16b, v18.16b, v19.16b\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "and v7.16b, v26.16b, v19.16b\n" + "sqrdmulh v9.4s, v9.4s, v15.4s\n" + "and v17.16b, v9.16b, v19.16b\n" + "add x11, x11, x16\n" + "add x10, x10, x16\n" + "sqrdmulh v24.4s, v24.4s, v18.4s\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "add x9, x9, x16\n" + "add x28, x28, x16\n" + "and v20.16b, v24.16b, v22.16b\n" + "sqrdmulh v7.4s, v7.4s, v15.4s\n" + "sqrdmulh v2.4s, v2.4s, v15.4s\n" + "sqrdmulh v10.4s, v10.4s, v15.4s\n" + "sqadd v9.4s, v9.4s, v17.4s\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "and v21.16b, v7.16b, v19.16b\n" + "sqrdmulh v0.4s, v0.4s, v18.4s\n" + "and v15.16b, v2.16b, v19.16b\n" + "sqrdmulh v30.4s, v30.4s, v18.4s\n" + "and v23.16b, v10.16b, v19.16b\n" + "sqrdmulh v6.4s, v6.4s, v18.4s\n" + "sqadd v24.4s, v24.4s, v20.4s\n" "sshr v21.4s, v21.4s, #0x1f\n" - "and v29.16b, v10.16b, v19.16b\n" - "sqadd v9.4s, v9.4s, v0.4s\n" + "and v18.16b, v0.16b, v22.16b\n" + "sshr v15.4s, v15.4s, #0x1f\n" + "and v17.16b, v30.16b, v22.16b\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "and v28.16b, v6.16b, v22.16b\n" + "sqadd v7.4s, v7.4s, v21.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sqadd v2.4s, v2.4s, v15.4s\n" "sshr v17.4s, v17.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v27.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v25.4s, v25.4s, v21.4s\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "srshl v13.4s, v13.4s, v22.4s\n" - "srshl v9.4s, v9.4s, v22.4s\n" - "sqadd v18.4s, v18.4s, v17.4s\n" - "srshl v16.4s, v16.4s, v22.4s\n" - "sqadd v26.4s, v26.4s, v7.4s\n" - "srshl v25.4s, v25.4s, v22.4s\n" - "sqadd v10.4s, v10.4s, v29.4s\n" - "srshl v20.4s, v20.4s, v19.4s\n" - "sqxtn v13.4h, v13.4s\n" - "srshl v18.4s, v18.4s, v19.4s\n" - "sqxtn v9.4h, v9.4s\n" - "srshl v26.4s, v26.4s, v19.4s\n" - "sqxtn v16.4h, v16.4s\n" + "sqadd v10.4s, v10.4s, v23.4s\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "srshl v9.4s, v9.4s, v19.4s\n" + "srshl v7.4s, v7.4s, v19.4s\n" + "sqadd v0.4s, v0.4s, v18.4s\n" + "srshl v2.4s, v2.4s, v19.4s\n" + "sqadd v30.4s, v30.4s, v17.4s\n" "srshl v10.4s, v10.4s, v19.4s\n" - "sqxtn v25.4h, v25.4s\n" - "sqxtn2 v13.8h, v20.4s\n" - "sqxtn2 v9.8h, v18.4s\n" - "sqxtn2 v16.8h, v26.4s\n" - "sqxtn2 v25.8h, v10.4s\n" - "sqadd v13.8h, v13.8h, v14.8h\n" - "sqadd v9.8h, v9.8h, v14.8h\n" - "sqadd v16.8h, v16.8h, v14.8h\n" - "sqadd v25.8h, v25.8h, v14.8h\n" - "smax v13.8h, v13.8h, v12.8h\n" - "smax v9.8h, v9.8h, v12.8h\n" - "smax v16.8h, v16.8h, v12.8h\n" - "smax v25.8h, v25.8h, v12.8h\n" - "smin v13.8h, v13.8h, v11.8h\n" - "smin v9.8h, v9.8h, v11.8h\n" - "smin v16.8h, v16.8h, v11.8h\n" - "smin v25.8h, v25.8h, v11.8h\n" - "uzp1 v13.16b, v13.16b, v13.16b\n" + "sqadd v6.4s, v6.4s, v28.4s\n" + "srshl v24.4s, v24.4s, v22.4s\n" + "sqxtn v9.4h, v9.4s\n" + "srshl v0.4s, v0.4s, v22.4s\n" + "sqxtn v7.4h, v7.4s\n" + "srshl v30.4s, v30.4s, v22.4s\n" + "sqxtn v2.4h, v2.4s\n" + "srshl v6.4s, v6.4s, v22.4s\n" + "sqxtn v10.4h, v10.4s\n" + "sqxtn2 v9.8h, v24.4s\n" + "sqxtn2 v7.8h, v0.4s\n" + "sqxtn2 v2.8h, v30.4s\n" + "sqxtn2 v10.8h, v6.4s\n" + "sqadd v9.8h, v9.8h, v13.8h\n" + "sqadd v7.8h, v7.8h, v13.8h\n" + "sqadd v2.8h, v2.8h, v13.8h\n" + "sqadd v10.8h, v10.8h, v13.8h\n" + "smax v9.8h, v9.8h, v29.8h\n" + "smax v7.8h, v7.8h, v29.8h\n" + "smax v2.8h, v2.8h, v29.8h\n" + "smax v10.8h, v10.8h, v29.8h\n" + "smin v9.8h, v9.8h, v12.8h\n" + "smin v7.8h, v7.8h, v12.8h\n" + "smin v2.8h, v2.8h, v12.8h\n" + "smin v10.8h, v10.8h, v12.8h\n" "uzp1 v9.16b, v9.16b, v9.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "tbz x6, #2, 61f\n" - "st1 { v13.s }[0], [x12], #0x4\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" + "uzp1 v2.16b, v2.16b, v2.16b\n" + "uzp1 v10.16b, v10.16b, v10.16b\n" + "tbz x7, #2, 61f\n" "st1 { v9.s }[0], [x11], #0x4\n" - "st1 { v16.s }[0], [x10], #0x4\n" - "st1 { v25.s }[0], [x9], #0x4\n" - "tbz x6, #1, 60f\n" - "st1 { v13.h }[2], [x12], #0x2\n" + "st1 { v7.s }[0], [x10], #0x4\n" + "st1 { v2.s }[0], [x9], #0x4\n" + "st1 { v10.s }[0], [x28], #0x4\n" + "tbz x7, #1, 60f\n" "st1 { v9.h }[2], [x11], #0x2\n" - "st1 { v16.h }[2], [x10], #0x2\n" - "st1 { v25.h }[2], [x9], #0x2\n" - "tbz x6, #0, 63f\n" - "st1 { v13.b }[6], [x12], #0x1\n" + "st1 { v7.h }[2], [x10], #0x2\n" + "st1 { v2.h }[2], [x9], #0x2\n" + "st1 { v10.h }[2], [x28], #0x2\n" + "tbz x7, #0, 63f\n" "st1 { v9.b }[6], [x11], #0x1\n" - "st1 { v16.b }[6], [x10], #0x1\n" - "st1 { v25.b }[6], [x9], #0x1\n" + "st1 { v7.b }[6], [x10], #0x1\n" + "st1 { v2.b }[6], [x9], #0x1\n" + "st1 { v10.b }[6], [x28], #0x1\n" "b 63f\n" "60:" // Oddments: Bit 2: Bit 1: Unset - "tbz x6, #0, 63f\n" - "st1 { v13.b }[4], [x12], #0x1\n" + "tbz x7, #0, 63f\n" "st1 { v9.b }[4], [x11], #0x1\n" - "st1 { v16.b }[4], [x10], #0x1\n" - "st1 { v25.b }[4], [x9], #0x1\n" + "st1 { v7.b }[4], [x10], #0x1\n" + "st1 { v2.b }[4], [x9], #0x1\n" + "st1 { v10.b }[4], [x28], #0x1\n" "b 63f\n" "61:" // Oddments: Bit 2: Unset - "tbz x6, #1, 62f\n" - "st1 { v13.h }[0], [x12], #0x2\n" + "tbz x7, #1, 62f\n" "st1 { v9.h }[0], [x11], #0x2\n" - "st1 { v16.h }[0], [x10], #0x2\n" - "st1 { v25.h }[0], [x9], #0x2\n" - "tbz x6, #0, 63f\n" - "st1 { v13.b }[2], [x12], #0x1\n" + "st1 { v7.h }[0], [x10], #0x2\n" + "st1 { v2.h }[0], [x9], #0x2\n" + "st1 { v10.h }[0], [x28], #0x2\n" + "tbz x7, #0, 63f\n" "st1 { v9.b }[2], [x11], #0x1\n" - "st1 { v16.b }[2], [x10], #0x1\n" - "st1 { v25.b }[2], [x9], #0x1\n" + "st1 { v7.b }[2], [x10], #0x1\n" + "st1 { v2.b }[2], [x9], #0x1\n" + "st1 { v10.b }[2], [x28], #0x1\n" "b 63f\n" "62:" // Oddments: Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 63f\n" - "st1 { v13.b }[0], [x12], #0x1\n" + "tbz x7, #0, 63f\n" "st1 { v9.b }[0], [x11], #0x1\n" - "st1 { v16.b }[0], [x10], #0x1\n" - "st1 { v25.b }[0], [x9], #0x1\n" + "st1 { v7.b }[0], [x10], #0x1\n" + "st1 { v2.b }[0], [x9], #0x1\n" + "st1 { v10.b }[0], [x28], #0x1\n" "63:" // Oddments: Bit 2: End "64:" // End : : [offsetof_Params_bias] "I" (offsetof(Params, bias)), [offsetof_Params_inptrs] "I" (offsetof(Params, inptrs)), [offsetof_Params_n_channels] "I" (offsetof(Params, n_channels)), [offsetof_Params_outptrs] "I" (offsetof(Params, outptrs)), [offsetof_Params_requant] "I" (offsetof(Params, requant)), [offsetof_Params_requant_muls] "I" (offsetof(Params, requant_muls)), [offsetof_Params_requant_shifts] "I" (offsetof(Params, requant_shifts)), [offsetof_Params_weights] "I" (offsetof(Params, weights)), [offsetof_Requantize32_a_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [params] "r" (¶ms) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp index 852466c48debd2e7d7a25f5f1f63e8c28603f155..7370f896993d2f3736ccceb0c784500e273114e1 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include @@ -34,15 +34,7 @@ namespace arm_conv { namespace depthwise { -void a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( - const unsigned int, - const int8_t *const *const, - const int8_t *const, - const int32_t *const, - const arm_gemm::Requantize32 &, - const int32_t *const, - const int32_t *const, - int8_t *const *const); +void a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl(unsigned int, const int8_t *const *, const int8_t *, const int32_t *, const arm_gemm::Requantize32 &, const int32_t *, const int32_t *, int8_t *const *); class a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst : public DepthwiseDepthfirstStrategy { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp index fa9ae97dee3815c9fd021cc95b7eb86e1d45cfb0..6432417c35950200d6e9dbdbe8e9b0fc7827348c 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp @@ -104,16 +104,16 @@ void a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ldr x23, [%x[params], %[offsetof_Params_requant]]\n" "lsr x8, x7, #0x3\n" "add x20, x23, %[offsetof_Requantize32_a_offset]\n" - "ld1r { v12.16b }, [x20]\n" + "ld1r { v6.16b }, [x20]\n" "ldr x22, [%x[params], %[offsetof_Params_outptrs]]\n" "add x21, x23, %[offsetof_Requantize32_b_offset]\n" "add x20, x23, %[offsetof_Requantize32_c_offset]\n" - "ld1r { v13.16b }, [x21]\n" - "ld1r { v11.8h }, [x20]\n" + "ld1r { v15.16b }, [x21]\n" + "ld1r { v13.8h }, [x20]\n" "add x21, x23, %[offsetof_Requantize32_minval]\n" "add x20, x23, %[offsetof_Requantize32_maxval]\n" - "ld1r { v16.8h }, [x21]\n" - "ld1r { v14.8h }, [x20]\n" + "ld1r { v17.8h }, [x21]\n" + "ld1r { v24.8h }, [x20]\n" "mov x17, #0x0\n" "mov x16, #0x0\n" "add x15, %x[params], %[offsetof_Params_inptrs]\n" @@ -123,563 +123,563 @@ void a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ldp x11, x10, [x22, #0x0]\n" "ldp x9, x28, [x22, #0x10]\n" "cbz x8, 3f\n" - "ldr d0, [x14, #0x0]\n" - "ldr d1, [x14, #0x8]\n" + "ldr d11, [x14, #0x0]\n" + "ldr d22, [x14, #0x8]\n" "subs x8, x8, #0x1\n" - "ssubl v0.8h, v0.8b, v13.8b\n" - "ldr d2, [x14, #0x10]\n" - "ldr d3, [x14, #0x18]\n" - "ssubl v1.8h, v1.8b, v13.8b\n" - "ssubl v2.8h, v2.8b, v13.8b\n" - "ldr d4, [x14, #0x20]\n" - "ldr d5, [x14, #0x28]\n" - "ssubl v3.8h, v3.8b, v13.8b\n" - "ssubl v4.8h, v4.8b, v13.8b\n" - "ldr d6, [x14, #0x30]\n" + "ssubl v11.8h, v11.8b, v15.8b\n" + "ldr d14, [x14, #0x10]\n" + "ldr d28, [x14, #0x18]\n" + "ssubl v22.8h, v22.8b, v15.8b\n" + "ssubl v14.8h, v14.8b, v15.8b\n" + "ldr d18, [x14, #0x20]\n" + "ldr d9, [x14, #0x28]\n" + "ssubl v28.8h, v28.8b, v15.8b\n" + "ssubl v18.8h, v18.8b, v15.8b\n" + "ldr d26, [x14, #0x30]\n" "ldr d7, [x14, #0x38]\n" - "ssubl v5.8h, v5.8b, v13.8b\n" - "ssubl v6.8h, v6.8b, v13.8b\n" - "ldr d8, [x14, #0x40]\n" - "ldr x24, [%x[params], %[offsetof_Params_bias]]\n" - "ssubl v7.8h, v7.8b, v13.8b\n" - "ssubl v8.8h, v8.8b, v13.8b\n" - "ldr q15, [x24, #0x0]\n" - "ldr q17, [x24, #0x10]\n" - "add x24, x24, #0x20\n" - "str x24, [%x[params], %[offsetof_Params_bias]]\n" + "ssubl v9.8h, v9.8b, v15.8b\n" + "ssubl v26.8h, v26.8b, v15.8b\n" + "ldr d4, [x14, #0x40]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "ssubl v7.8h, v7.8b, v15.8b\n" + "ssubl v4.8h, v4.8b, v15.8b\n" + "ldr q5, [x20, #0x0]\n" + "ldr q3, [x20, #0x10]\n" + "add x20, x20, #0x20\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" "ldp x27, x26, [x15, #0x0]\n" "ldp x25, x24, [x15, #0x10]\n" - "mov v10.16b, v15.16b\n" - "mov v20.16b, v17.16b\n" + "mov v21.16b, v5.16b\n" + "mov v8.16b, v3.16b\n" "ldp x23, x22, [x15, #0x20]\n" "ldp x21, x20, [x15, #0x30]\n" - "mov v9.16b, v15.16b\n" - "mov v23.16b, v17.16b\n" - "ldr d31, [x27, x17]\n" - "ldr d30, [x26, x17]\n" - "mov v21.16b, v15.16b\n" - "mov v22.16b, v17.16b\n" - "ldr d29, [x25, x17]\n" - "ldr d28, [x24, x17]\n" - "ssubl v31.8h, v31.8b, v12.8b\n" - "ssubl v30.8h, v30.8b, v12.8b\n" - "ldr d27, [x23, x17]\n" - "ldr d26, [x22, x17]\n" - "ssubl v29.8h, v29.8b, v12.8b\n" - "ssubl v28.8h, v28.8b, v12.8b\n" - "ldr d25, [x21, x17]\n" - "ldr d24, [x20, x17]\n" - "ssubl v27.8h, v27.8b, v12.8b\n" - "ssubl v26.8h, v26.8b, v12.8b\n" - "ssubl v25.8h, v25.8b, v12.8b\n" - "ssubl v24.8h, v24.8b, v12.8b\n" + "mov v20.16b, v5.16b\n" + "mov v0.16b, v3.16b\n" + "ldr d25, [x27, x17]\n" + "ldr d27, [x26, x17]\n" + "mov v19.16b, v5.16b\n" + "mov v31.16b, v3.16b\n" + "ldr d1, [x25, x17]\n" + "ldr d2, [x24, x17]\n" + "ssubl v25.8h, v25.8b, v6.8b\n" + "ssubl v27.8h, v27.8b, v6.8b\n" + "ldr d12, [x23, x17]\n" + "ldr d16, [x22, x17]\n" + "ssubl v1.8h, v1.8b, v6.8b\n" + "ssubl v2.8h, v2.8b, v6.8b\n" + "ldr d23, [x21, x17]\n" + "ldr d10, [x20, x17]\n" + "ssubl v12.8h, v12.8b, v6.8b\n" + "ssubl v16.8h, v16.8b, v6.8b\n" + "ssubl v23.8h, v23.8b, v6.8b\n" + "ssubl v10.8h, v10.8b, v6.8b\n" "beq 2f\n" "1:" // Loop - "smlal v15.4s, v31.4h, v8.4h\n" - "smlal2 v17.4s, v31.8h, v8.8h\n" - "ldr x24, [x15, #0x40]\n" - "ldr x22, [x15, #0x48]\n" - "smlal v10.4s, v31.4h, v6.4h\n" - "smlal2 v20.4s, v31.8h, v6.8h\n" - "ldr x21, [x15, #0x50]\n" - "ldr x20, [x15, #0x58]\n" - "smlal v15.4s, v30.4h, v0.4h\n" - "smlal2 v17.4s, v30.8h, v0.8h\n" - "ldr q19, [x13, #0x0]\n" - "ldr x23, [x15, #0x78]\n" - "smlal v10.4s, v28.4h, v1.4h\n" - "smlal2 v20.4s, v28.8h, v1.8h\n" - "ldr d28, [x22, x17]\n" - "ssubl v28.8h, v28.8b, v12.8b\n" - "smlal v15.4s, v29.4h, v1.4h\n" - "smlal2 v17.4s, v29.8h, v1.8h\n" - "ldr d29, [x24, x17]\n" - "ssubl v29.8h, v29.8b, v12.8b\n" - "smlal v10.4s, v27.4h, v2.4h\n" - "smlal2 v20.4s, v27.8h, v2.8h\n" + "ldr q30, [x13, #0x0]\n" + "ldr q29, [x12, #0x0]\n" + "smlal v5.4s, v25.4h, v4.4h\n" + "smlal2 v3.4s, v25.8h, v4.8h\n" + "ldr x21, [x15, #0x58]\n" + "ldr x20, [x15, #0x78]\n" + "smlal v5.4s, v27.4h, v11.4h\n" + "smlal v21.4s, v25.4h, v26.4h\n" + "ldr x25, [x15, #0x60]\n" + "ldr x24, [x15, #0x80]\n" + "smlal v20.4s, v25.4h, v14.4h\n" + "smlal v19.4s, v25.4h, v11.4h\n" + "smlal2 v3.4s, v27.8h, v11.8h\n" "ldr d27, [x21, x17]\n" - "ssubl v27.8h, v27.8b, v12.8b\n" - "smlal v15.4s, v26.4h, v3.4h\n" - "smlal2 v17.4s, v26.8h, v3.8h\n" - "ldr d26, [x20, x17]\n" - "ldr x20, [x15, #0x60]\n" - "smlal v10.4s, v24.4h, v0.4h\n" - "smlal2 v20.4s, v24.8h, v0.8h\n" - "ssubl v26.8h, v26.8b, v12.8b\n" - "ldr x21, [x15, #0x80]\n" - "smlal v15.4s, v25.4h, v4.4h\n" - "smlal2 v17.4s, v25.8h, v4.8h\n" - "ldr d25, [x20, x17]\n" - "ldr x20, [x15, #0x68]\n" - "smlal v10.4s, v29.4h, v4.4h\n" - "smlal2 v20.4s, v29.8h, v4.8h\n" - "ldr d29, [x20, x17]\n" - "ssubl v25.8h, v25.8b, v12.8b\n" - "smlal v15.4s, v24.4h, v2.4h\n" - "smlal2 v17.4s, v24.8h, v2.8h\n" - "ldr q18, [x12, #0x0]\n" + "ssubl v27.8h, v27.8b, v6.8b\n" + "smlal v5.4s, v1.4h, v22.4h\n" + "smlal2 v8.4s, v25.8h, v26.8h\n" + "smlal2 v0.4s, v25.8h, v14.8h\n" + "ldr x23, [x15, #0x68]\n" "ldr x22, [x15, #0x88]\n" - "smlal v10.4s, v28.4h, v5.4h\n" - "smlal2 v20.4s, v28.8h, v5.8h\n" - "ldr d28, [x21, x17]\n" - "ldr x21, [x15, #0x70]\n" - "smlal v9.4s, v31.4h, v2.4h\n" - "smlal2 v23.4s, v31.8h, v2.8h\n" - "ssubl v28.8h, v28.8b, v12.8b\n" + "smlal2 v31.4s, v25.8h, v11.8h\n" + "ldr d25, [x20, x17]\n" + "ssubl v25.8h, v25.8b, v6.8b\n" + "smlal v21.4s, v2.4h, v22.4h\n" + "smlal v20.4s, v27.4h, v28.4h\n" + "smlal v19.4s, v25.4h, v18.4h\n" + "ldr x21, [x15, #0x40]\n" + "ldr x20, [x15, #0x70]\n" + "smlal2 v3.4s, v1.8h, v22.8h\n" + "ldr d1, [x25, x17]\n" + "ssubl v1.8h, v1.8b, v6.8b\n" + "smlal v5.4s, v16.4h, v28.4h\n" + "smlal2 v8.4s, v2.8h, v22.8h\n" + "ldr d2, [x24, x17]\n" + "ssubl v2.8h, v2.8b, v6.8b\n" + "smlal2 v0.4s, v27.8h, v28.8h\n" + "ldr d27, [x23, x17]\n" + "smlal2 v31.4s, v25.8h, v18.8h\n" + "ldr d25, [x22, x17]\n" + "smlal v21.4s, v12.4h, v14.4h\n" "ldr x25, [x15, #0x98]\n" - "smlal v15.4s, v27.4h, v5.4h\n" - "smlal2 v17.4s, v27.8h, v5.8h\n" - "ssubl v29.8h, v29.8b, v12.8b\n" - "ldr x24, [x15, #0x90]\n" - "smlal v10.4s, v27.4h, v3.4h\n" - "smlal2 v20.4s, v27.8h, v3.8h\n" + "smlal v20.4s, v1.4h, v11.4h\n" + "smlal v19.4s, v2.4h, v22.4h\n" + "ldr x24, [x15, #0x50]\n" + "smlal2 v3.4s, v16.8h, v28.8h\n" + "ldr d16, [x21, x17]\n" + "ssubl v27.8h, v27.8b, v6.8b\n" + "smlal v5.4s, v23.4h, v18.4h\n" + "ssubl v25.8h, v25.8b, v6.8b\n" + "smlal2 v8.4s, v12.8h, v14.8h\n" + "ldr d12, [x20, x17]\n" + "ldr x23, [x15, #0x48]\n" + "smlal2 v0.4s, v1.8h, v11.8h\n" + "smlal2 v31.4s, v2.8h, v22.8h\n" + "ldr x21, [x15, #0x90]\n" + "ldr x20, [x15, #0xa8]\n" + "smlal v21.4s, v10.4h, v11.4h\n" + "smlal v20.4s, v27.4h, v18.4h\n" + "ssubl v16.8h, v16.8b, v6.8b\n" + "ldr x22, [x15, #0xa0]\n" + "smlal v19.4s, v25.4h, v9.4h\n" + "smlal2 v3.4s, v23.8h, v18.8h\n" + "ldr d23, [x25, x17]\n" + "ssubl v12.8h, v12.8b, v6.8b\n" + "ssubl v23.8h, v23.8b, v6.8b\n" + "smlal v5.4s, v10.4h, v14.4h\n" + "smlal2 v8.4s, v10.8h, v11.8h\n" + "ldr d11, [x24, x17]\n" + "ssubl v11.8h, v11.8b, v6.8b\n" + "smlal2 v0.4s, v27.8h, v18.8h\n" "ldr d27, [x23, x17]\n" - "ssubl v27.8h, v27.8b, v12.8b\n" - "smlal v21.4s, v31.4h, v0.4h\n" - "smlal v9.4s, v26.4h, v3.4h\n" - "ldr x23, [x15, #0xa8]\n" - "ldr x20, [x15, #0xa0]\n" - "smlal2 v23.4s, v26.8h, v3.8h\n" - "ldr d26, [x22, x17]\n" - "smlal2 v22.4s, v31.8h, v0.8h\n" - "ldr d24, [x21, x17]\n" - "smlal v21.4s, v27.4h, v4.4h\n" - "smlal v9.4s, v25.4h, v0.4h\n" - "ssubl v26.8h, v26.8b, v12.8b\n" - "ldr x22, [x15, #0xb0]\n" - "smlal2 v23.4s, v25.8h, v0.8h\n" - "ldr q30, [x13, #0x10]\n" - "smlal2 v22.4s, v27.8h, v4.8h\n" - "ldr d27, [x20, x17]\n" - "smlal v21.4s, v28.4h, v1.4h\n" - "smlal v15.4s, v25.4h, v6.4h\n" - "ssubl v24.8h, v24.8b, v12.8b\n" - "ldr x21, [x15, #0xb8]\n" - "smlal2 v17.4s, v25.8h, v6.8h\n" - "ldr d25, [x24, x17]\n" - "smlal v9.4s, v29.4h, v4.4h\n" - "ssubl v25.8h, v25.8b, v12.8b\n" - "smlal2 v23.4s, v29.8h, v4.8h\n" - "ldr d29, [x25, x17]\n" - "ldr q31, [x12, #0x10]\n" - "smlal2 v22.4s, v28.8h, v1.8h\n" - "smlal v21.4s, v26.4h, v5.4h\n" - "ssubl v29.8h, v29.8b, v12.8b\n" - "smlal v15.4s, v24.4h, v7.4h\n" - "ldr x20, [x15, #0xc0]\n" - "smlal2 v17.4s, v24.8h, v7.8h\n" - "smlal v9.4s, v24.4h, v1.4h\n" - "ssubl v27.8h, v27.8b, v12.8b\n" - "ldr x24, [%x[params], %[offsetof_Params_bias]]\n" - "smlal2 v23.4s, v24.8h, v1.8h\n" - "ldr d24, [x23, x17]\n" - "smlal2 v22.4s, v26.8h, v5.8h\n" - "ldr d26, [x22, x17]\n" - "smlal v21.4s, v29.4h, v2.4h\n" - "ssubl v24.8h, v24.8b, v12.8b\n" - "smlal2 v22.4s, v29.8h, v2.8h\n" + "smlal2 v31.4s, v25.8h, v9.8h\n" + "ldr d25, [x21, x17]\n" + "ldr x21, [x15, #0xb0]\n" + "smlal v21.4s, v16.4h, v18.4h\n" + "smlal v20.4s, v12.4h, v22.4h\n" + "smlal v19.4s, v23.4h, v14.4h\n" + "smlal2 v3.4s, v10.8h, v14.8h\n" + "ldr d10, [x20, x17]\n" + "ssubl v27.8h, v27.8b, v6.8b\n" + "ssubl v25.8h, v25.8b, v6.8b\n" + "ssubl v10.8h, v10.8b, v6.8b\n" + "smlal v5.4s, v11.4h, v9.4h\n" + "ldr x20, [x15, #0xb8]\n" + "smlal2 v8.4s, v16.8h, v18.8h\n" + "ldr d18, [x22, x17]\n" + "ldr d16, [x21, x17]\n" + "smlal2 v0.4s, v12.8h, v22.8h\n" + "ldr d22, [x20, x17]\n" + "smlal2 v31.4s, v23.8h, v14.8h\n" + "ldr q14, [x13, #0x10]\n" + "smlal v21.4s, v27.4h, v9.4h\n" + "smlal v20.4s, v25.4h, v26.4h\n" + "smlal v19.4s, v10.4h, v28.4h\n" + "ssubl v18.8h, v18.8b, v6.8b\n" + "ldr x21, [x15, #0xc0]\n" + "smlal2 v3.4s, v11.8h, v9.8h\n" + "ssubl v16.8h, v16.8b, v6.8b\n" + "smlal v5.4s, v1.4h, v26.4h\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "smlal2 v8.4s, v27.8h, v9.8h\n" + "ldr d27, [x21, x17]\n" + "smlal2 v0.4s, v25.8h, v26.8h\n" + "ldr q25, [x12, #0x10]\n" + "smlal2 v31.4s, v10.8h, v28.8h\n" + "smlal v21.4s, v11.4h, v28.4h\n" + "ssubl v22.8h, v22.8b, v6.8b\n" "add x14, x14, #0x48\n" - "smlal v9.4s, v25.4h, v6.4h\n" - "smlal v21.4s, v24.4h, v3.4h\n" - "ssubl v26.8h, v26.8b, v12.8b\n" + "smlal v20.4s, v18.4h, v7.4h\n" + "smlal v19.4s, v16.4h, v7.4h\n" + "ssubl v27.8h, v27.8b, v6.8b\n" + "add x17, x17, #0x8\n" + "smlal2 v3.4s, v1.8h, v26.8h\n" + "smlal v5.4s, v12.4h, v7.4h\n" + "sqrdmulh v5.4s, v5.4s, v30.4s\n" "subs x8, x8, #0x1\n" - "smlal v10.4s, v28.4h, v7.4h\n" - "smlal2 v20.4s, v28.8h, v7.8h\n" - "sqrdmulh v15.4s, v15.4s, v19.4s\n" + "smlal2 v8.4s, v11.8h, v28.8h\n" + "smlal2 v0.4s, v18.8h, v7.8h\n" + "and v28.16b, v5.16b, v29.16b\n" "add x13, x13, #0x20\n" - "smlal2 v23.4s, v25.8h, v6.8h\n" - "ldr d25, [x21, x17]\n" - "smlal2 v22.4s, v24.8h, v3.8h\n" - "ssubl v25.8h, v25.8b, v12.8b\n" - "smlal v9.4s, v27.4h, v7.4h\n" - "smlal v21.4s, v26.4h, v7.4h\n" - "and v0.16b, v15.16b, v18.16b\n" + "smlal2 v31.4s, v16.8h, v7.8h\n" + "smlal v21.4s, v2.4h, v7.4h\n" + "sshr v28.4s, v28.4s, #0x1f\n" "add x12, x12, #0x20\n" - "smlal v10.4s, v29.4h, v8.4h\n" - "smlal2 v20.4s, v29.8h, v8.8h\n" - "ldr d29, [x20, x17]\n" - "ssubl v29.8h, v29.8b, v12.8b\n" - "smlal2 v23.4s, v27.8h, v7.8h\n" - "smlal2 v22.4s, v26.8h, v7.8h\n" - "sqrdmulh v17.4s, v17.4s, v30.4s\n" - "add x17, x17, #0x8\n" - "smlal v9.4s, v24.4h, v5.4h\n" - "smlal v21.4s, v25.4h, v6.4h\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "smlal2 v23.4s, v24.8h, v5.8h\n" - "smlal2 v22.4s, v25.8h, v6.8h\n" - "and v7.16b, v17.16b, v31.16b\n" - "smlal v9.4s, v25.4h, v8.4h\n" - "smlal v21.4s, v29.4h, v8.4h\n" - "sqrdmulh v10.4s, v10.4s, v19.4s\n" - "smlal2 v23.4s, v25.8h, v8.8h\n" - "smlal2 v22.4s, v29.8h, v8.8h\n" - "sqrdmulh v9.4s, v9.4s, v19.4s\n" - "sqrdmulh v21.4s, v21.4s, v19.4s\n" - "sqadd v15.4s, v15.4s, v0.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "and v19.16b, v10.16b, v18.16b\n" + "smlal v20.4s, v10.4h, v9.4h\n" + "smlal v19.4s, v22.4h, v26.4h\n" + "sqadd v5.4s, v5.4s, v28.4s\n" + "smlal2 v3.4s, v12.8h, v7.8h\n" + "smlal2 v8.4s, v2.8h, v7.8h\n" + "sqrdmulh v3.4s, v3.4s, v14.4s\n" + "smlal2 v0.4s, v10.8h, v9.8h\n" + "smlal2 v31.4s, v22.8h, v26.8h\n" + "and v16.16b, v3.16b, v25.16b\n" + "smlal v21.4s, v23.4h, v4.4h\n" + "smlal v20.4s, v22.4h, v4.4h\n" + "sqrdmulh v21.4s, v21.4s, v30.4s\n" + "smlal v19.4s, v27.4h, v4.4h\n" + "smlal2 v8.4s, v23.8h, v4.8h\n" "sqrdmulh v20.4s, v20.4s, v30.4s\n" - "and v27.16b, v9.16b, v18.16b\n" - "sqrdmulh v23.4s, v23.4s, v30.4s\n" - "and v0.16b, v21.16b, v18.16b\n" - "sqrdmulh v22.4s, v22.4s, v30.4s\n" - "sqadd v17.4s, v17.4s, v7.4s\n" - "sshr v19.4s, v19.4s, #0x1f\n" - "and v5.16b, v20.16b, v31.16b\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "and v4.16b, v23.16b, v31.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v7.16b, v22.16b, v31.16b\n" - "sqadd v10.4s, v10.4s, v19.4s\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v9.4s, v9.4s, v27.4s\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v21.4s, v21.4s, v0.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "srshl v15.4s, v15.4s, v18.4s\n" - "srshl v10.4s, v10.4s, v18.4s\n" - "sqadd v20.4s, v20.4s, v5.4s\n" - "srshl v9.4s, v9.4s, v18.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "srshl v21.4s, v21.4s, v18.4s\n" - "sqadd v22.4s, v22.4s, v7.4s\n" - "srshl v17.4s, v17.4s, v31.4s\n" - "sqxtn v15.4h, v15.4s\n" - "srshl v20.4s, v20.4s, v31.4s\n" - "sqxtn v10.4h, v10.4s\n" - "srshl v23.4s, v23.4s, v31.4s\n" - "sqxtn v9.4h, v9.4s\n" - "srshl v22.4s, v22.4s, v31.4s\n" + "smlal2 v0.4s, v22.8h, v4.8h\n" + "smlal2 v31.4s, v27.8h, v4.8h\n" + "sqrdmulh v19.4s, v19.4s, v30.4s\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "and v12.16b, v21.16b, v29.16b\n" + "sqrdmulh v8.4s, v8.4s, v14.4s\n" + "and v23.16b, v20.16b, v29.16b\n" + "sqrdmulh v0.4s, v0.4s, v14.4s\n" + "and v9.16b, v19.16b, v29.16b\n" + "sqrdmulh v31.4s, v31.4s, v14.4s\n" + "sqadd v3.4s, v3.4s, v16.4s\n" + "sshr v12.4s, v12.4s, #0x1f\n" + "and v18.16b, v8.16b, v25.16b\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "and v22.16b, v0.16b, v25.16b\n" + "sshr v9.4s, v9.4s, #0x1f\n" + "and v16.16b, v31.16b, v25.16b\n" + "sqadd v21.4s, v21.4s, v12.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v23.4s\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "sqadd v19.4s, v19.4s, v9.4s\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "srshl v5.4s, v5.4s, v29.4s\n" + "srshl v21.4s, v21.4s, v29.4s\n" + "sqadd v8.4s, v8.4s, v18.4s\n" + "srshl v20.4s, v20.4s, v29.4s\n" + "sqadd v0.4s, v0.4s, v22.4s\n" + "srshl v19.4s, v19.4s, v29.4s\n" + "sqadd v31.4s, v31.4s, v16.4s\n" + "srshl v3.4s, v3.4s, v25.4s\n" + "sqxtn v5.4h, v5.4s\n" + "srshl v8.4s, v8.4s, v25.4s\n" "sqxtn v21.4h, v21.4s\n" - "sqxtn2 v15.8h, v17.4s\n" - "sqxtn2 v10.8h, v20.4s\n" - "sqxtn2 v9.8h, v23.4s\n" - "sqxtn2 v21.8h, v22.4s\n" - "sqadd v15.8h, v15.8h, v11.8h\n" - "sqadd v10.8h, v10.8h, v11.8h\n" - "sqadd v9.8h, v9.8h, v11.8h\n" - "sqadd v21.8h, v21.8h, v11.8h\n" - "smax v15.8h, v15.8h, v16.8h\n" - "smax v10.8h, v10.8h, v16.8h\n" - "smax v9.8h, v9.8h, v16.8h\n" - "smax v21.8h, v21.8h, v16.8h\n" - "smin v15.8h, v15.8h, v14.8h\n" - "smin v10.8h, v10.8h, v14.8h\n" - "smin v9.8h, v9.8h, v14.8h\n" - "smin v21.8h, v21.8h, v14.8h\n" - "uzp1 v15.16b, v15.16b, v15.16b\n" - "str d15, [x11, x16]\n" - "uzp1 v10.16b, v10.16b, v10.16b\n" - "uzp1 v9.16b, v9.16b, v9.16b\n" - "str d10, [x10, x16]\n" + "srshl v0.4s, v0.4s, v25.4s\n" + "sqxtn v20.4h, v20.4s\n" + "srshl v31.4s, v31.4s, v25.4s\n" + "sqxtn v19.4h, v19.4s\n" + "sqxtn2 v5.8h, v3.4s\n" + "sqxtn2 v21.8h, v8.4s\n" + "sqxtn2 v20.8h, v0.4s\n" + "sqxtn2 v19.8h, v31.4s\n" + "sqadd v5.8h, v5.8h, v13.8h\n" + "sqadd v21.8h, v21.8h, v13.8h\n" + "sqadd v20.8h, v20.8h, v13.8h\n" + "sqadd v19.8h, v19.8h, v13.8h\n" + "smax v5.8h, v5.8h, v17.8h\n" + "smax v21.8h, v21.8h, v17.8h\n" + "smax v20.8h, v20.8h, v17.8h\n" + "smax v19.8h, v19.8h, v17.8h\n" + "smin v5.8h, v5.8h, v24.8h\n" + "smin v21.8h, v21.8h, v24.8h\n" + "smin v20.8h, v20.8h, v24.8h\n" + "smin v19.8h, v19.8h, v24.8h\n" + "uzp1 v5.16b, v5.16b, v5.16b\n" + "str d5, [x11, x16]\n" "uzp1 v21.16b, v21.16b, v21.16b\n" - "str d9, [x9, x16]\n" - "str d21, [x28, x16]\n" - "ldr q15, [x24, #0x0]\n" - "ldr q17, [x24, #0x10]\n" - "add x24, x24, #0x20\n" - "ldr d0, [x14, #0x0]\n" - "ldr d1, [x14, #0x8]\n" + "uzp1 v20.16b, v20.16b, v20.16b\n" + "str d21, [x10, x16]\n" + "uzp1 v19.16b, v19.16b, v19.16b\n" + "str d20, [x9, x16]\n" + "str d19, [x28, x16]\n" + "ldr q5, [x20, #0x0]\n" + "ldr q3, [x20, #0x10]\n" + "add x20, x20, #0x20\n" + "ldr d11, [x14, #0x0]\n" + "ldr d22, [x14, #0x8]\n" "add x16, x16, #0x8\n" - "str x24, [%x[params], %[offsetof_Params_bias]]\n" - "ldr d2, [x14, #0x10]\n" - "ldr d3, [x14, #0x18]\n" - "mov v10.16b, v15.16b\n" - "mov v20.16b, v17.16b\n" - "ldr d4, [x14, #0x20]\n" - "ldr d5, [x14, #0x28]\n" - "mov v9.16b, v15.16b\n" - "mov v23.16b, v17.16b\n" - "ldr d6, [x14, #0x30]\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" + "ldr d14, [x14, #0x10]\n" + "ldr d28, [x14, #0x18]\n" + "mov v21.16b, v5.16b\n" + "mov v8.16b, v3.16b\n" + "ldr d18, [x14, #0x20]\n" + "ldr d9, [x14, #0x28]\n" + "mov v20.16b, v5.16b\n" + "mov v0.16b, v3.16b\n" + "ldr d26, [x14, #0x30]\n" "ldr d7, [x14, #0x38]\n" - "mov v21.16b, v15.16b\n" - "mov v22.16b, v17.16b\n" - "ldr d8, [x14, #0x40]\n" + "mov v19.16b, v5.16b\n" + "mov v31.16b, v3.16b\n" + "ldr d4, [x14, #0x40]\n" "ldp x27, x26, [x15, #0x0]\n" - "ssubl v0.8h, v0.8b, v13.8b\n" - "ssubl v1.8h, v1.8b, v13.8b\n" + "ssubl v11.8h, v11.8b, v15.8b\n" + "ssubl v22.8h, v22.8b, v15.8b\n" "ldp x25, x24, [x15, #0x10]\n" "ldp x23, x22, [x15, #0x20]\n" - "ssubl v2.8h, v2.8b, v13.8b\n" - "ssubl v3.8h, v3.8b, v13.8b\n" + "ssubl v14.8h, v14.8b, v15.8b\n" + "ssubl v28.8h, v28.8b, v15.8b\n" "ldp x21, x20, [x15, #0x30]\n" - "ldr d31, [x27, x17]\n" - "ssubl v4.8h, v4.8b, v13.8b\n" - "ssubl v5.8h, v5.8b, v13.8b\n" - "ldr d30, [x26, x17]\n" - "ldr d29, [x25, x17]\n" - "ssubl v6.8h, v6.8b, v13.8b\n" - "ssubl v7.8h, v7.8b, v13.8b\n" - "ldr d28, [x24, x17]\n" - "ldr d27, [x23, x17]\n" - "ssubl v8.8h, v8.8b, v13.8b\n" - "ssubl v31.8h, v31.8b, v12.8b\n" - "ldr d26, [x22, x17]\n" - "ldr d25, [x21, x17]\n" - "ssubl v30.8h, v30.8b, v12.8b\n" - "ssubl v29.8h, v29.8b, v12.8b\n" - "ldr d24, [x20, x17]\n" - "ssubl v28.8h, v28.8b, v12.8b\n" - "ssubl v27.8h, v27.8b, v12.8b\n" - "ssubl v26.8h, v26.8b, v12.8b\n" - "ssubl v25.8h, v25.8b, v12.8b\n" - "ssubl v24.8h, v24.8b, v12.8b\n" + "ldr d25, [x27, x17]\n" + "ssubl v18.8h, v18.8b, v15.8b\n" + "ssubl v9.8h, v9.8b, v15.8b\n" + "ldr d27, [x26, x17]\n" + "ldr d1, [x25, x17]\n" + "ssubl v26.8h, v26.8b, v15.8b\n" + "ssubl v7.8h, v7.8b, v15.8b\n" + "ldr d2, [x24, x17]\n" + "ldr d12, [x23, x17]\n" + "ssubl v4.8h, v4.8b, v15.8b\n" + "ssubl v25.8h, v25.8b, v6.8b\n" + "ldr d16, [x22, x17]\n" + "ldr d23, [x21, x17]\n" + "ssubl v27.8h, v27.8b, v6.8b\n" + "ssubl v1.8h, v1.8b, v6.8b\n" + "ldr d10, [x20, x17]\n" + "ssubl v2.8h, v2.8b, v6.8b\n" + "ssubl v12.8h, v12.8b, v6.8b\n" + "ssubl v16.8h, v16.8b, v6.8b\n" + "ssubl v23.8h, v23.8b, v6.8b\n" + "ssubl v10.8h, v10.8b, v6.8b\n" "bgt 1b\n" "2:" // Tail - "smlal v15.4s, v31.4h, v8.4h\n" - "smlal2 v17.4s, v31.8h, v8.8h\n" - "ldr x24, [x15, #0x40]\n" - "ldr x22, [x15, #0x48]\n" - "smlal v10.4s, v31.4h, v6.4h\n" - "smlal2 v20.4s, v31.8h, v6.8h\n" - "ldr x21, [x15, #0x50]\n" - "ldr x20, [x15, #0x58]\n" - "smlal v15.4s, v30.4h, v0.4h\n" - "smlal2 v17.4s, v30.8h, v0.8h\n" - "ldr q19, [x13, #0x0]\n" - "ldr x23, [x15, #0x78]\n" - "smlal v10.4s, v28.4h, v1.4h\n" - "smlal2 v20.4s, v28.8h, v1.8h\n" - "ldr d28, [x22, x17]\n" - "ssubl v28.8h, v28.8b, v12.8b\n" - "smlal v15.4s, v29.4h, v1.4h\n" - "smlal2 v17.4s, v29.8h, v1.8h\n" - "ldr d29, [x24, x17]\n" - "ssubl v29.8h, v29.8b, v12.8b\n" - "smlal v10.4s, v27.4h, v2.4h\n" - "smlal2 v20.4s, v27.8h, v2.8h\n" + "ldr q29, [x13, #0x0]\n" + "ldr q30, [x12, #0x0]\n" + "smlal v5.4s, v25.4h, v4.4h\n" + "smlal2 v3.4s, v25.8h, v4.8h\n" + "ldr x21, [x15, #0x58]\n" + "ldr x20, [x15, #0x78]\n" + "smlal v5.4s, v27.4h, v11.4h\n" + "smlal v21.4s, v25.4h, v26.4h\n" + "ldr x25, [x15, #0x60]\n" + "ldr x24, [x15, #0x80]\n" + "smlal v20.4s, v25.4h, v14.4h\n" + "smlal v19.4s, v25.4h, v11.4h\n" + "smlal2 v3.4s, v27.8h, v11.8h\n" "ldr d27, [x21, x17]\n" - "ssubl v27.8h, v27.8b, v12.8b\n" - "smlal v15.4s, v26.4h, v3.4h\n" - "smlal2 v17.4s, v26.8h, v3.8h\n" - "ldr d26, [x20, x17]\n" - "ldr x20, [x15, #0x60]\n" - "smlal v10.4s, v24.4h, v0.4h\n" - "smlal2 v20.4s, v24.8h, v0.8h\n" - "ssubl v26.8h, v26.8b, v12.8b\n" - "ldr x21, [x15, #0x80]\n" - "smlal v15.4s, v25.4h, v4.4h\n" - "smlal2 v17.4s, v25.8h, v4.8h\n" - "ldr d25, [x20, x17]\n" - "ldr x20, [x15, #0x68]\n" - "smlal v10.4s, v29.4h, v4.4h\n" - "smlal2 v20.4s, v29.8h, v4.8h\n" - "ldr d29, [x20, x17]\n" - "ssubl v25.8h, v25.8b, v12.8b\n" - "smlal v15.4s, v24.4h, v2.4h\n" - "smlal2 v17.4s, v24.8h, v2.8h\n" - "ldr q18, [x12, #0x0]\n" + "ssubl v27.8h, v27.8b, v6.8b\n" + "smlal v5.4s, v1.4h, v22.4h\n" + "smlal2 v8.4s, v25.8h, v26.8h\n" + "smlal2 v0.4s, v25.8h, v14.8h\n" + "ldr x23, [x15, #0x68]\n" "ldr x22, [x15, #0x88]\n" - "smlal v10.4s, v28.4h, v5.4h\n" - "smlal2 v20.4s, v28.8h, v5.8h\n" - "ldr d28, [x21, x17]\n" - "ldr x21, [x15, #0x70]\n" - "smlal v9.4s, v31.4h, v2.4h\n" - "smlal2 v23.4s, v31.8h, v2.8h\n" - "ssubl v28.8h, v28.8b, v12.8b\n" + "smlal2 v31.4s, v25.8h, v11.8h\n" + "ldr d25, [x20, x17]\n" + "ssubl v25.8h, v25.8b, v6.8b\n" + "smlal v21.4s, v2.4h, v22.4h\n" + "smlal v20.4s, v27.4h, v28.4h\n" + "smlal v19.4s, v25.4h, v18.4h\n" + "ldr x21, [x15, #0x40]\n" + "ldr x20, [x15, #0x70]\n" + "smlal2 v3.4s, v1.8h, v22.8h\n" + "ldr d1, [x25, x17]\n" + "ssubl v1.8h, v1.8b, v6.8b\n" + "smlal v5.4s, v16.4h, v28.4h\n" + "smlal2 v8.4s, v2.8h, v22.8h\n" + "ldr d2, [x24, x17]\n" + "ssubl v2.8h, v2.8b, v6.8b\n" + "smlal2 v0.4s, v27.8h, v28.8h\n" + "ldr d27, [x23, x17]\n" + "smlal2 v31.4s, v25.8h, v18.8h\n" + "ldr d25, [x22, x17]\n" + "smlal v21.4s, v12.4h, v14.4h\n" "ldr x25, [x15, #0x98]\n" - "smlal v15.4s, v27.4h, v5.4h\n" - "smlal2 v17.4s, v27.8h, v5.8h\n" - "ssubl v29.8h, v29.8b, v12.8b\n" - "ldr x24, [x15, #0x90]\n" - "smlal v10.4s, v27.4h, v3.4h\n" - "smlal2 v20.4s, v27.8h, v3.8h\n" + "smlal v20.4s, v1.4h, v11.4h\n" + "smlal v19.4s, v2.4h, v22.4h\n" + "ldr x24, [x15, #0x50]\n" + "smlal2 v3.4s, v16.8h, v28.8h\n" + "ldr d16, [x21, x17]\n" + "ssubl v27.8h, v27.8b, v6.8b\n" + "smlal v5.4s, v23.4h, v18.4h\n" + "ssubl v25.8h, v25.8b, v6.8b\n" + "smlal2 v8.4s, v12.8h, v14.8h\n" + "ldr d12, [x20, x17]\n" + "ldr x23, [x15, #0x48]\n" + "smlal2 v0.4s, v1.8h, v11.8h\n" + "smlal2 v31.4s, v2.8h, v22.8h\n" + "ldr x21, [x15, #0x90]\n" + "ldr x20, [x15, #0xa8]\n" + "smlal v21.4s, v10.4h, v11.4h\n" + "smlal v20.4s, v27.4h, v18.4h\n" + "ssubl v16.8h, v16.8b, v6.8b\n" + "ldr x22, [x15, #0xa0]\n" + "smlal v19.4s, v25.4h, v9.4h\n" + "smlal2 v3.4s, v23.8h, v18.8h\n" + "ldr d23, [x25, x17]\n" + "ssubl v12.8h, v12.8b, v6.8b\n" + "ssubl v23.8h, v23.8b, v6.8b\n" + "smlal v5.4s, v10.4h, v14.4h\n" + "smlal2 v8.4s, v10.8h, v11.8h\n" + "ldr d11, [x24, x17]\n" + "ssubl v11.8h, v11.8b, v6.8b\n" + "smlal2 v0.4s, v27.8h, v18.8h\n" "ldr d27, [x23, x17]\n" - "ssubl v27.8h, v27.8b, v12.8b\n" - "smlal v21.4s, v31.4h, v0.4h\n" - "smlal v9.4s, v26.4h, v3.4h\n" - "ldr x23, [x15, #0xa8]\n" - "ldr x20, [x15, #0xa0]\n" - "smlal2 v23.4s, v26.8h, v3.8h\n" - "ldr d26, [x22, x17]\n" - "smlal2 v22.4s, v31.8h, v0.8h\n" - "ldr d24, [x21, x17]\n" - "smlal v21.4s, v27.4h, v4.4h\n" - "smlal v9.4s, v25.4h, v0.4h\n" - "ssubl v26.8h, v26.8b, v12.8b\n" - "ldr x22, [x15, #0xb0]\n" - "smlal2 v23.4s, v25.8h, v0.8h\n" - "ldr q30, [x13, #0x10]\n" - "smlal2 v22.4s, v27.8h, v4.8h\n" - "ldr d27, [x20, x17]\n" - "smlal v21.4s, v28.4h, v1.4h\n" - "smlal v15.4s, v25.4h, v6.4h\n" - "ssubl v24.8h, v24.8b, v12.8b\n" - "ldr x21, [x15, #0xb8]\n" - "smlal2 v17.4s, v25.8h, v6.8h\n" - "ldr d25, [x24, x17]\n" - "smlal v9.4s, v29.4h, v4.4h\n" - "ssubl v25.8h, v25.8b, v12.8b\n" - "smlal2 v23.4s, v29.8h, v4.8h\n" - "ldr d29, [x25, x17]\n" - "ldr q31, [x12, #0x10]\n" - "smlal2 v22.4s, v28.8h, v1.8h\n" - "smlal v21.4s, v26.4h, v5.4h\n" - "ssubl v29.8h, v29.8b, v12.8b\n" - "smlal v15.4s, v24.4h, v7.4h\n" + "smlal2 v31.4s, v25.8h, v9.8h\n" + "ldr d25, [x21, x17]\n" + "ldr x21, [x15, #0xb0]\n" + "smlal v21.4s, v16.4h, v18.4h\n" + "smlal v20.4s, v12.4h, v22.4h\n" + "smlal v19.4s, v23.4h, v14.4h\n" + "smlal2 v3.4s, v10.8h, v14.8h\n" + "ldr d10, [x20, x17]\n" + "ssubl v27.8h, v27.8b, v6.8b\n" + "ssubl v25.8h, v25.8b, v6.8b\n" + "ssubl v10.8h, v10.8b, v6.8b\n" + "smlal v5.4s, v11.4h, v9.4h\n" + "ldr x20, [x15, #0xb8]\n" + "smlal2 v8.4s, v16.8h, v18.8h\n" + "ldr d16, [x22, x17]\n" + "ldr d18, [x21, x17]\n" + "smlal2 v0.4s, v12.8h, v22.8h\n" + "ldr d22, [x20, x17]\n" + "smlal2 v31.4s, v23.8h, v14.8h\n" + "ldr q14, [x13, #0x10]\n" + "smlal v21.4s, v27.4h, v9.4h\n" + "smlal v20.4s, v25.4h, v26.4h\n" + "smlal v19.4s, v10.4h, v28.4h\n" + "ssubl v16.8h, v16.8b, v6.8b\n" "ldr x20, [x15, #0xc0]\n" - "smlal2 v17.4s, v24.8h, v7.8h\n" - "smlal v9.4s, v24.4h, v1.4h\n" - "ssubl v27.8h, v27.8b, v12.8b\n" + "smlal2 v3.4s, v11.8h, v9.8h\n" + "ssubl v18.8h, v18.8b, v6.8b\n" + "smlal v5.4s, v1.4h, v26.4h\n" "tst x7, #0x7\n" - "smlal2 v23.4s, v24.8h, v1.8h\n" - "ldr d24, [x23, x17]\n" - "smlal2 v22.4s, v26.8h, v5.8h\n" - "ldr d26, [x22, x17]\n" - "smlal v21.4s, v29.4h, v2.4h\n" - "ssubl v24.8h, v24.8b, v12.8b\n" - "smlal2 v22.4s, v29.8h, v2.8h\n" + "smlal2 v8.4s, v27.8h, v9.8h\n" + "ldr d27, [x20, x17]\n" + "smlal2 v0.4s, v25.8h, v26.8h\n" + "ldr q25, [x12, #0x10]\n" + "smlal2 v31.4s, v10.8h, v28.8h\n" + "smlal v21.4s, v11.4h, v28.4h\n" + "ssubl v22.8h, v22.8b, v6.8b\n" + "add x17, x17, #0x8\n" + "smlal v20.4s, v16.4h, v7.4h\n" + "smlal v19.4s, v18.4h, v7.4h\n" + "ssubl v27.8h, v27.8b, v6.8b\n" "add x13, x13, #0x20\n" - "smlal v9.4s, v25.4h, v6.4h\n" - "smlal v21.4s, v24.4h, v3.4h\n" - "ssubl v26.8h, v26.8b, v12.8b\n" + "smlal2 v3.4s, v1.8h, v26.8h\n" + "smlal v5.4s, v12.4h, v7.4h\n" + "sqrdmulh v5.4s, v5.4s, v29.4s\n" "add x12, x12, #0x20\n" - "smlal v10.4s, v28.4h, v7.4h\n" - "smlal2 v20.4s, v28.8h, v7.8h\n" - "sqrdmulh v15.4s, v15.4s, v19.4s\n" - "smlal2 v23.4s, v25.8h, v6.8h\n" - "ldr d25, [x21, x17]\n" - "smlal2 v22.4s, v24.8h, v3.8h\n" - "ssubl v25.8h, v25.8b, v12.8b\n" - "smlal v9.4s, v27.4h, v7.4h\n" - "smlal v21.4s, v26.4h, v7.4h\n" - "and v0.16b, v15.16b, v18.16b\n" - "smlal v10.4s, v29.4h, v8.4h\n" - "smlal2 v20.4s, v29.8h, v8.8h\n" - "ldr d29, [x20, x17]\n" - "ssubl v29.8h, v29.8b, v12.8b\n" - "smlal2 v23.4s, v27.8h, v7.8h\n" - "smlal2 v22.4s, v26.8h, v7.8h\n" - "sqrdmulh v17.4s, v17.4s, v30.4s\n" - "add x17, x17, #0x8\n" - "smlal v9.4s, v24.4h, v5.4h\n" - "smlal v21.4s, v25.4h, v6.4h\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "smlal2 v23.4s, v24.8h, v5.8h\n" - "smlal2 v22.4s, v25.8h, v6.8h\n" - "and v7.16b, v17.16b, v31.16b\n" - "smlal v9.4s, v25.4h, v8.4h\n" - "smlal v21.4s, v29.4h, v8.4h\n" - "sqrdmulh v10.4s, v10.4s, v19.4s\n" - "smlal2 v23.4s, v25.8h, v8.8h\n" - "smlal2 v22.4s, v29.8h, v8.8h\n" - "sqrdmulh v9.4s, v9.4s, v19.4s\n" - "sqrdmulh v21.4s, v21.4s, v19.4s\n" - "sqadd v15.4s, v15.4s, v0.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "and v19.16b, v10.16b, v18.16b\n" - "sqrdmulh v20.4s, v20.4s, v30.4s\n" - "and v27.16b, v9.16b, v18.16b\n" - "sqrdmulh v23.4s, v23.4s, v30.4s\n" - "and v0.16b, v21.16b, v18.16b\n" - "sqrdmulh v22.4s, v22.4s, v30.4s\n" - "sqadd v17.4s, v17.4s, v7.4s\n" - "sshr v19.4s, v19.4s, #0x1f\n" - "and v5.16b, v20.16b, v31.16b\n" + "smlal2 v8.4s, v11.8h, v28.8h\n" + "smlal2 v0.4s, v16.8h, v7.8h\n" + "and v16.16b, v5.16b, v30.16b\n" + "smlal2 v31.4s, v18.8h, v7.8h\n" + "smlal v21.4s, v2.4h, v7.4h\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "smlal v20.4s, v10.4h, v9.4h\n" + "smlal v19.4s, v22.4h, v26.4h\n" + "sqadd v5.4s, v5.4s, v16.4s\n" + "smlal2 v3.4s, v12.8h, v7.8h\n" + "smlal2 v8.4s, v2.8h, v7.8h\n" + "sqrdmulh v3.4s, v3.4s, v14.4s\n" + "smlal2 v0.4s, v10.8h, v9.8h\n" + "smlal2 v31.4s, v22.8h, v26.8h\n" + "and v16.16b, v3.16b, v25.16b\n" + "smlal v21.4s, v23.4h, v4.4h\n" + "smlal v20.4s, v22.4h, v4.4h\n" + "sqrdmulh v21.4s, v21.4s, v29.4s\n" + "smlal v19.4s, v27.4h, v4.4h\n" + "smlal2 v8.4s, v23.8h, v4.8h\n" + "sqrdmulh v20.4s, v20.4s, v29.4s\n" + "smlal2 v0.4s, v22.8h, v4.8h\n" + "smlal2 v31.4s, v27.8h, v4.8h\n" + "sqrdmulh v19.4s, v19.4s, v29.4s\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "and v23.16b, v21.16b, v30.16b\n" + "sqrdmulh v8.4s, v8.4s, v14.4s\n" + "and v27.16b, v20.16b, v30.16b\n" + "sqrdmulh v0.4s, v0.4s, v14.4s\n" + "and v22.16b, v19.16b, v30.16b\n" + "sqrdmulh v31.4s, v31.4s, v14.4s\n" + "sqadd v3.4s, v3.4s, v16.4s\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "and v14.16b, v8.16b, v25.16b\n" "sshr v27.4s, v27.4s, #0x1f\n" - "and v4.16b, v23.16b, v31.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v7.16b, v22.16b, v31.16b\n" - "sqadd v10.4s, v10.4s, v19.4s\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v9.4s, v9.4s, v27.4s\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v21.4s, v21.4s, v0.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "srshl v15.4s, v15.4s, v18.4s\n" - "srshl v10.4s, v10.4s, v18.4s\n" - "sqadd v20.4s, v20.4s, v5.4s\n" - "srshl v9.4s, v9.4s, v18.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "srshl v21.4s, v21.4s, v18.4s\n" - "sqadd v22.4s, v22.4s, v7.4s\n" - "srshl v17.4s, v17.4s, v31.4s\n" - "sqxtn v15.4h, v15.4s\n" - "srshl v20.4s, v20.4s, v31.4s\n" - "sqxtn v10.4h, v10.4s\n" - "srshl v23.4s, v23.4s, v31.4s\n" - "sqxtn v9.4h, v9.4s\n" - "srshl v22.4s, v22.4s, v31.4s\n" + "and v18.16b, v0.16b, v25.16b\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "and v16.16b, v31.16b, v25.16b\n" + "sqadd v21.4s, v21.4s, v23.4s\n" + "sshr v14.4s, v14.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v27.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sqadd v19.4s, v19.4s, v22.4s\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "srshl v5.4s, v5.4s, v30.4s\n" + "srshl v21.4s, v21.4s, v30.4s\n" + "sqadd v8.4s, v8.4s, v14.4s\n" + "srshl v20.4s, v20.4s, v30.4s\n" + "sqadd v0.4s, v0.4s, v18.4s\n" + "srshl v19.4s, v19.4s, v30.4s\n" + "sqadd v31.4s, v31.4s, v16.4s\n" + "srshl v3.4s, v3.4s, v25.4s\n" + "sqxtn v5.4h, v5.4s\n" + "srshl v8.4s, v8.4s, v25.4s\n" "sqxtn v21.4h, v21.4s\n" - "sqxtn2 v15.8h, v17.4s\n" - "sqxtn2 v10.8h, v20.4s\n" - "sqxtn2 v9.8h, v23.4s\n" - "sqxtn2 v21.8h, v22.4s\n" - "sqadd v15.8h, v15.8h, v11.8h\n" - "sqadd v10.8h, v10.8h, v11.8h\n" - "sqadd v9.8h, v9.8h, v11.8h\n" - "sqadd v21.8h, v21.8h, v11.8h\n" - "smax v15.8h, v15.8h, v16.8h\n" - "smax v10.8h, v10.8h, v16.8h\n" - "smax v9.8h, v9.8h, v16.8h\n" - "smax v21.8h, v21.8h, v16.8h\n" - "smin v15.8h, v15.8h, v14.8h\n" - "smin v10.8h, v10.8h, v14.8h\n" - "smin v9.8h, v9.8h, v14.8h\n" - "smin v21.8h, v21.8h, v14.8h\n" - "uzp1 v15.16b, v15.16b, v15.16b\n" - "str d15, [x11, x16]\n" - "uzp1 v10.16b, v10.16b, v10.16b\n" - "uzp1 v9.16b, v9.16b, v9.16b\n" - "str d10, [x10, x16]\n" + "srshl v0.4s, v0.4s, v25.4s\n" + "sqxtn v20.4h, v20.4s\n" + "srshl v31.4s, v31.4s, v25.4s\n" + "sqxtn v19.4h, v19.4s\n" + "sqxtn2 v5.8h, v3.4s\n" + "sqxtn2 v21.8h, v8.4s\n" + "sqxtn2 v20.8h, v0.4s\n" + "sqxtn2 v19.8h, v31.4s\n" + "sqadd v5.8h, v5.8h, v13.8h\n" + "sqadd v21.8h, v21.8h, v13.8h\n" + "sqadd v20.8h, v20.8h, v13.8h\n" + "sqadd v19.8h, v19.8h, v13.8h\n" + "smax v5.8h, v5.8h, v17.8h\n" + "smax v21.8h, v21.8h, v17.8h\n" + "smax v20.8h, v20.8h, v17.8h\n" + "smax v19.8h, v19.8h, v17.8h\n" + "smin v5.8h, v5.8h, v24.8h\n" + "smin v21.8h, v21.8h, v24.8h\n" + "smin v20.8h, v20.8h, v24.8h\n" + "smin v19.8h, v19.8h, v24.8h\n" + "uzp1 v5.16b, v5.16b, v5.16b\n" + "str d5, [x11, x16]\n" "uzp1 v21.16b, v21.16b, v21.16b\n" - "str d9, [x9, x16]\n" - "str d21, [x28, x16]\n" + "uzp1 v20.16b, v20.16b, v20.16b\n" + "str d21, [x10, x16]\n" + "uzp1 v19.16b, v19.16b, v19.16b\n" + "str d20, [x9, x16]\n" + "str d19, [x28, x16]\n" "add x16, x16, #0x8\n" "beq 88f\n" "add x14, x14, #0x48\n" "3:" // Oddments - "ldr x24, [%x[params], %[offsetof_Params_bias]]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" "tbz x7, #2, 5f\n" - "ld1 { v15.4s }, [x24], #0x10\n" + "ld1 { v5.4s }, [x20], #0x10\n" "tbz x7, #1, 4f\n" - "ld1 { v17.d }[0], [x24], #0x8\n" + "ld1 { v3.d }[0], [x20], #0x8\n" "tbz x7, #0, 7f\n" - "ld1 { v17.s }[2], [x24]\n" + "ld1 { v3.s }[2], [x20]\n" "b 7f\n" "4:" // Oddments: Load bias: Bit 2: Bit 1: Unset "tbz x7, #0, 7f\n" - "ld1 { v17.s }[0], [x24]\n" + "ld1 { v3.s }[0], [x20]\n" "b 7f\n" "5:" // Oddments: Load bias: Bit 2: Unset "tbz x7, #1, 6f\n" - "ld1 { v15.d }[0], [x24], #0x8\n" + "ld1 { v5.d }[0], [x20], #0x8\n" "tbz x7, #0, 7f\n" - "ld1 { v15.s }[2], [x24]\n" + "ld1 { v5.s }[2], [x20]\n" "b 7f\n" "6:" // Oddments: Load bias: Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 7f\n" - "ld1 { v15.s }[0], [x24]\n" + "ld1 { v5.s }[0], [x20]\n" "7:" // Oddments: Load bias: Bit 2: End - "ldr d0, [x14, #0x0]\n" - "ldr d1, [x14, #0x8]\n" - "mov v10.16b, v15.16b\n" - "mov v20.16b, v17.16b\n" - "ldr d2, [x14, #0x10]\n" - "ldr d3, [x14, #0x18]\n" - "mov v9.16b, v15.16b\n" - "mov v23.16b, v17.16b\n" - "ldr d4, [x14, #0x20]\n" - "ldr d5, [x14, #0x28]\n" - "mov v21.16b, v15.16b\n" - "mov v22.16b, v17.16b\n" - "ldr d6, [x14, #0x30]\n" + "ldr d11, [x14, #0x0]\n" + "ldr d22, [x14, #0x8]\n" + "mov v21.16b, v5.16b\n" + "mov v8.16b, v3.16b\n" + "ldr d14, [x14, #0x10]\n" + "ldr d28, [x14, #0x18]\n" + "mov v20.16b, v5.16b\n" + "mov v0.16b, v3.16b\n" + "ldr d18, [x14, #0x20]\n" + "ldr d9, [x14, #0x28]\n" + "mov v19.16b, v5.16b\n" + "mov v31.16b, v3.16b\n" + "ldr d26, [x14, #0x30]\n" "ldr d7, [x14, #0x38]\n" - "ssubl v0.8h, v0.8b, v13.8b\n" - "ssubl v1.8h, v1.8b, v13.8b\n" - "ldr d8, [x14, #0x40]\n" + "ssubl v11.8h, v11.8b, v15.8b\n" + "ssubl v22.8h, v22.8b, v15.8b\n" + "ldr d4, [x14, #0x40]\n" "ldp x27, x26, [x15, #0x0]\n" - "ssubl v2.8h, v2.8b, v13.8b\n" - "ssubl v3.8h, v3.8b, v13.8b\n" + "ssubl v14.8h, v14.8b, v15.8b\n" + "ssubl v28.8h, v28.8b, v15.8b\n" "ldp x25, x24, [x15, #0x10]\n" "ldp x23, x22, [x15, #0x20]\n" - "ssubl v4.8h, v4.8b, v13.8b\n" - "ssubl v5.8h, v5.8b, v13.8b\n" + "ssubl v18.8h, v18.8b, v15.8b\n" + "ssubl v9.8h, v9.8b, v15.8b\n" "ldp x21, x20, [x15, #0x30]\n" - "ssubl v6.8h, v6.8b, v13.8b\n" - "ssubl v7.8h, v7.8b, v13.8b\n" - "ssubl v8.8h, v8.8b, v13.8b\n" + "ssubl v26.8h, v26.8b, v15.8b\n" + "ssubl v7.8h, v7.8b, v15.8b\n" + "ssubl v4.8h, v4.8b, v15.8b\n" "add x27, x27, x17\n" "add x26, x26, x17\n" "add x25, x25, x17\n" @@ -689,700 +689,700 @@ void a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "add x21, x21, x17\n" "add x20, x20, x17\n" "tbz x7, #2, 9f\n" - "ld1 { v31.s }[0], [x27], #0x4\n" - "ld1 { v30.s }[0], [x26], #0x4\n" - "ld1 { v29.s }[0], [x25], #0x4\n" - "ld1 { v28.s }[0], [x24], #0x4\n" - "ld1 { v27.s }[0], [x23], #0x4\n" - "ld1 { v26.s }[0], [x22], #0x4\n" - "ld1 { v25.s }[0], [x21], #0x4\n" - "ld1 { v24.s }[0], [x20], #0x4\n" + "ld1 { v25.s }[0], [x27], #0x4\n" + "ld1 { v27.s }[0], [x26], #0x4\n" + "ld1 { v1.s }[0], [x25], #0x4\n" + "ld1 { v2.s }[0], [x24], #0x4\n" + "ld1 { v12.s }[0], [x23], #0x4\n" + "ld1 { v16.s }[0], [x22], #0x4\n" + "ld1 { v23.s }[0], [x21], #0x4\n" + "ld1 { v10.s }[0], [x20], #0x4\n" "tbz x7, #1, 8f\n" - "ld1 { v31.h }[2], [x27], #0x2\n" - "ld1 { v30.h }[2], [x26], #0x2\n" - "ld1 { v29.h }[2], [x25], #0x2\n" - "ld1 { v28.h }[2], [x24], #0x2\n" - "ld1 { v27.h }[2], [x23], #0x2\n" - "ld1 { v26.h }[2], [x22], #0x2\n" - "ld1 { v25.h }[2], [x21], #0x2\n" - "ld1 { v24.h }[2], [x20], #0x2\n" + "ld1 { v25.h }[2], [x27], #0x2\n" + "ld1 { v27.h }[2], [x26], #0x2\n" + "ld1 { v1.h }[2], [x25], #0x2\n" + "ld1 { v2.h }[2], [x24], #0x2\n" + "ld1 { v12.h }[2], [x23], #0x2\n" + "ld1 { v16.h }[2], [x22], #0x2\n" + "ld1 { v23.h }[2], [x21], #0x2\n" + "ld1 { v10.h }[2], [x20], #0x2\n" "tbz x7, #0, 11f\n" - "ld1 { v31.b }[6], [x27]\n" - "ld1 { v30.b }[6], [x26]\n" - "ld1 { v29.b }[6], [x25]\n" - "ld1 { v28.b }[6], [x24]\n" - "ld1 { v27.b }[6], [x23]\n" - "ld1 { v26.b }[6], [x22]\n" - "ld1 { v25.b }[6], [x21]\n" - "ld1 { v24.b }[6], [x20]\n" + "ld1 { v25.b }[6], [x27]\n" + "ld1 { v27.b }[6], [x26]\n" + "ld1 { v1.b }[6], [x25]\n" + "ld1 { v2.b }[6], [x24]\n" + "ld1 { v12.b }[6], [x23]\n" + "ld1 { v16.b }[6], [x22]\n" + "ld1 { v23.b }[6], [x21]\n" + "ld1 { v10.b }[6], [x20]\n" "b 11f\n" "8:" // Oddments: Initial loads: Bit 2: Bit 1: Unset "tbz x7, #0, 11f\n" - "ld1 { v31.b }[4], [x27]\n" - "ld1 { v30.b }[4], [x26]\n" - "ld1 { v29.b }[4], [x25]\n" - "ld1 { v28.b }[4], [x24]\n" - "ld1 { v27.b }[4], [x23]\n" - "ld1 { v26.b }[4], [x22]\n" - "ld1 { v25.b }[4], [x21]\n" - "ld1 { v24.b }[4], [x20]\n" + "ld1 { v25.b }[4], [x27]\n" + "ld1 { v27.b }[4], [x26]\n" + "ld1 { v1.b }[4], [x25]\n" + "ld1 { v2.b }[4], [x24]\n" + "ld1 { v12.b }[4], [x23]\n" + "ld1 { v16.b }[4], [x22]\n" + "ld1 { v23.b }[4], [x21]\n" + "ld1 { v10.b }[4], [x20]\n" "b 11f\n" "9:" // Oddments: Initial loads: Bit 2: Unset "tbz x7, #1, 10f\n" - "ld1 { v31.h }[0], [x27], #0x2\n" - "ld1 { v30.h }[0], [x26], #0x2\n" - "ld1 { v29.h }[0], [x25], #0x2\n" - "ld1 { v28.h }[0], [x24], #0x2\n" - "ld1 { v27.h }[0], [x23], #0x2\n" - "ld1 { v26.h }[0], [x22], #0x2\n" - "ld1 { v25.h }[0], [x21], #0x2\n" - "ld1 { v24.h }[0], [x20], #0x2\n" + "ld1 { v25.h }[0], [x27], #0x2\n" + "ld1 { v27.h }[0], [x26], #0x2\n" + "ld1 { v1.h }[0], [x25], #0x2\n" + "ld1 { v2.h }[0], [x24], #0x2\n" + "ld1 { v12.h }[0], [x23], #0x2\n" + "ld1 { v16.h }[0], [x22], #0x2\n" + "ld1 { v23.h }[0], [x21], #0x2\n" + "ld1 { v10.h }[0], [x20], #0x2\n" "tbz x7, #0, 11f\n" - "ld1 { v31.b }[2], [x27]\n" - "ld1 { v30.b }[2], [x26]\n" - "ld1 { v29.b }[2], [x25]\n" - "ld1 { v28.b }[2], [x24]\n" - "ld1 { v27.b }[2], [x23]\n" - "ld1 { v26.b }[2], [x22]\n" - "ld1 { v25.b }[2], [x21]\n" - "ld1 { v24.b }[2], [x20]\n" + "ld1 { v25.b }[2], [x27]\n" + "ld1 { v27.b }[2], [x26]\n" + "ld1 { v1.b }[2], [x25]\n" + "ld1 { v2.b }[2], [x24]\n" + "ld1 { v12.b }[2], [x23]\n" + "ld1 { v16.b }[2], [x22]\n" + "ld1 { v23.b }[2], [x21]\n" + "ld1 { v10.b }[2], [x20]\n" "b 11f\n" "10:" // Oddments: Initial loads: Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 11f\n" - "ld1 { v31.b }[0], [x27]\n" - "ld1 { v30.b }[0], [x26]\n" - "ld1 { v29.b }[0], [x25]\n" - "ld1 { v28.b }[0], [x24]\n" - "ld1 { v27.b }[0], [x23]\n" - "ld1 { v26.b }[0], [x22]\n" - "ld1 { v25.b }[0], [x21]\n" - "ld1 { v24.b }[0], [x20]\n" + "ld1 { v25.b }[0], [x27]\n" + "ld1 { v27.b }[0], [x26]\n" + "ld1 { v1.b }[0], [x25]\n" + "ld1 { v2.b }[0], [x24]\n" + "ld1 { v12.b }[0], [x23]\n" + "ld1 { v16.b }[0], [x22]\n" + "ld1 { v23.b }[0], [x21]\n" + "ld1 { v10.b }[0], [x20]\n" "11:" // Oddments: Initial loads: Bit 2: End - "ssubl v31.8h, v31.8b, v12.8b\n" - "smlal v15.4s, v31.4h, v8.4h\n" - "smlal2 v17.4s, v31.8h, v8.8h\n" - "ldr x24, [x15, #0x40]\n" - "ssubl v30.8h, v30.8b, v12.8b\n" - "smlal v15.4s, v30.4h, v0.4h\n" - "smlal2 v17.4s, v30.8h, v0.8h\n" - "add x24, x24, x17\n" - "ssubl v29.8h, v29.8b, v12.8b\n" - "smlal v10.4s, v31.4h, v6.4h\n" - "smlal2 v20.4s, v31.8h, v6.8h\n" - "smlal v15.4s, v29.4h, v1.4h\n" - "smlal2 v17.4s, v29.8h, v1.8h\n" - "ssubl v28.8h, v28.8b, v12.8b\n" - "ssubl v26.8h, v26.8b, v12.8b\n" - "smlal v10.4s, v28.4h, v1.4h\n" - "smlal2 v20.4s, v28.8h, v1.8h\n" - "smlal v15.4s, v26.4h, v3.4h\n" - "smlal2 v17.4s, v26.8h, v3.8h\n" - "ssubl v27.8h, v27.8b, v12.8b\n" - "ssubl v25.8h, v25.8b, v12.8b\n" - "smlal v10.4s, v27.4h, v2.4h\n" - "smlal2 v20.4s, v27.8h, v2.8h\n" - "smlal v15.4s, v25.4h, v4.4h\n" - "smlal2 v17.4s, v25.8h, v4.8h\n" - "ssubl v24.8h, v24.8b, v12.8b\n" - "smlal v9.4s, v31.4h, v2.4h\n" - "smlal2 v23.4s, v31.8h, v2.8h\n" - "smlal v21.4s, v31.4h, v0.4h\n" - "smlal2 v22.4s, v31.8h, v0.8h\n" - "smlal v15.4s, v24.4h, v2.4h\n" - "smlal2 v17.4s, v24.8h, v2.8h\n" - "smlal v10.4s, v24.4h, v0.4h\n" - "smlal2 v20.4s, v24.8h, v0.8h\n" + "ssubl v25.8h, v25.8b, v6.8b\n" + "smlal v5.4s, v25.4h, v4.4h\n" + "smlal2 v3.4s, v25.8h, v4.8h\n" + "ldr x20, [x15, #0x40]\n" + "ssubl v27.8h, v27.8b, v6.8b\n" + "smlal v5.4s, v27.4h, v11.4h\n" + "smlal2 v3.4s, v27.8h, v11.8h\n" + "ssubl v1.8h, v1.8b, v6.8b\n" + "smlal v21.4s, v25.4h, v26.4h\n" + "smlal2 v8.4s, v25.8h, v26.8h\n" + "add x20, x20, x17\n" + "smlal v5.4s, v1.4h, v22.4h\n" + "smlal2 v3.4s, v1.8h, v22.8h\n" + "ssubl v2.8h, v2.8b, v6.8b\n" + "ssubl v16.8h, v16.8b, v6.8b\n" + "smlal v21.4s, v2.4h, v22.4h\n" + "smlal2 v8.4s, v2.8h, v22.8h\n" + "smlal v5.4s, v16.4h, v28.4h\n" + "smlal2 v3.4s, v16.8h, v28.8h\n" + "ssubl v12.8h, v12.8b, v6.8b\n" + "ssubl v23.8h, v23.8b, v6.8b\n" + "smlal v21.4s, v12.4h, v14.4h\n" + "smlal2 v8.4s, v12.8h, v14.8h\n" + "smlal v5.4s, v23.4h, v18.4h\n" + "smlal2 v3.4s, v23.8h, v18.8h\n" + "ssubl v10.8h, v10.8b, v6.8b\n" + "smlal v20.4s, v25.4h, v14.4h\n" + "smlal2 v0.4s, v25.8h, v14.8h\n" + "smlal v19.4s, v25.4h, v11.4h\n" + "smlal2 v31.4s, v25.8h, v11.8h\n" + "smlal v5.4s, v10.4h, v14.4h\n" + "smlal2 v3.4s, v10.8h, v14.8h\n" + "smlal v21.4s, v10.4h, v11.4h\n" + "smlal2 v8.4s, v10.8h, v11.8h\n" "tbz x7, #2, 13f\n" - "ld1 { v29.s }[0], [x24], #0x4\n" + "ld1 { v15.s }[0], [x20], #0x4\n" "tbz x7, #1, 12f\n" - "ld1 { v29.h }[2], [x24], #0x2\n" + "ld1 { v15.h }[2], [x20], #0x2\n" "tbz x7, #0, 15f\n" - "ld1 { v29.b }[6], [x24]\n" + "ld1 { v15.b }[6], [x20]\n" "b 15f\n" "12:" // Oddments: Load (1, 3): Bit 2: Bit 1: Unset "tbz x7, #0, 15f\n" - "ld1 { v29.b }[4], [x24]\n" + "ld1 { v15.b }[4], [x20]\n" "b 15f\n" "13:" // Oddments: Load (1, 3): Bit 2: Unset "tbz x7, #1, 14f\n" - "ld1 { v29.h }[0], [x24], #0x2\n" + "ld1 { v15.h }[0], [x20], #0x2\n" "tbz x7, #0, 15f\n" - "ld1 { v29.b }[2], [x24]\n" + "ld1 { v15.b }[2], [x20]\n" "b 15f\n" "14:" // Oddments: Load (1, 3): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 15f\n" - "ld1 { v29.b }[0], [x24]\n" + "ld1 { v15.b }[0], [x20]\n" "15:" // Oddments: Load (1, 3): Bit 2: End - "ssubl v29.8h, v29.8b, v12.8b\n" - "ldr x22, [x15, #0x48]\n" - "smlal v10.4s, v29.4h, v4.4h\n" - "smlal2 v20.4s, v29.8h, v4.8h\n" - "add x22, x22, x17\n" + "ssubl v15.8h, v15.8b, v6.8b\n" + "ldr x20, [x15, #0x48]\n" + "smlal v21.4s, v15.4h, v18.4h\n" + "smlal2 v8.4s, v15.8h, v18.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 17f\n" - "ld1 { v28.s }[0], [x22], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 16f\n" - "ld1 { v28.h }[2], [x22], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 19f\n" - "ld1 { v28.b }[6], [x22]\n" + "ld1 { v16.b }[6], [x20]\n" "b 19f\n" "16:" // Oddments: Load (1, 4): Bit 2: Bit 1: Unset "tbz x7, #0, 19f\n" - "ld1 { v28.b }[4], [x22]\n" + "ld1 { v16.b }[4], [x20]\n" "b 19f\n" "17:" // Oddments: Load (1, 4): Bit 2: Unset "tbz x7, #1, 18f\n" - "ld1 { v28.h }[0], [x22], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 19f\n" - "ld1 { v28.b }[2], [x22]\n" + "ld1 { v16.b }[2], [x20]\n" "b 19f\n" "18:" // Oddments: Load (1, 4): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 19f\n" - "ld1 { v28.b }[0], [x22]\n" + "ld1 { v16.b }[0], [x20]\n" "19:" // Oddments: Load (1, 4): Bit 2: End - "ssubl v28.8h, v28.8b, v12.8b\n" - "ldr x21, [x15, #0x50]\n" - "smlal v10.4s, v28.4h, v5.4h\n" - "smlal2 v20.4s, v28.8h, v5.8h\n" - "add x21, x21, x17\n" + "ssubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0x50]\n" + "smlal v21.4s, v16.4h, v9.4h\n" + "smlal2 v8.4s, v16.8h, v9.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 21f\n" - "ld1 { v27.s }[0], [x21], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 20f\n" - "ld1 { v27.h }[2], [x21], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 23f\n" - "ld1 { v27.b }[6], [x21]\n" + "ld1 { v16.b }[6], [x20]\n" "b 23f\n" "20:" // Oddments: Load (1, 2): Bit 2: Bit 1: Unset "tbz x7, #0, 23f\n" - "ld1 { v27.b }[4], [x21]\n" + "ld1 { v16.b }[4], [x20]\n" "b 23f\n" "21:" // Oddments: Load (1, 2): Bit 2: Unset "tbz x7, #1, 22f\n" - "ld1 { v27.h }[0], [x21], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 23f\n" - "ld1 { v27.b }[2], [x21]\n" + "ld1 { v16.b }[2], [x20]\n" "b 23f\n" "22:" // Oddments: Load (1, 2): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 23f\n" - "ld1 { v27.b }[0], [x21]\n" + "ld1 { v16.b }[0], [x20]\n" "23:" // Oddments: Load (1, 2): Bit 2: End - "ssubl v27.8h, v27.8b, v12.8b\n" + "ssubl v16.8h, v16.8b, v6.8b\n" "ldr x20, [x15, #0x58]\n" - "smlal v15.4s, v27.4h, v5.4h\n" - "smlal2 v17.4s, v27.8h, v5.8h\n" - "smlal v10.4s, v27.4h, v3.4h\n" - "smlal2 v20.4s, v27.8h, v3.8h\n" + "smlal v5.4s, v16.4h, v9.4h\n" + "smlal2 v3.4s, v16.8h, v9.8h\n" + "smlal v21.4s, v16.4h, v28.4h\n" + "smlal2 v8.4s, v16.8h, v28.8h\n" "add x20, x20, x17\n" "tbz x7, #2, 25f\n" - "ld1 { v26.s }[0], [x20], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 24f\n" - "ld1 { v26.h }[2], [x20], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 27f\n" - "ld1 { v26.b }[6], [x20]\n" + "ld1 { v16.b }[6], [x20]\n" "b 27f\n" "24:" // Oddments: Load (3, 0): Bit 2: Bit 1: Unset "tbz x7, #0, 27f\n" - "ld1 { v26.b }[4], [x20]\n" + "ld1 { v16.b }[4], [x20]\n" "b 27f\n" "25:" // Oddments: Load (3, 0): Bit 2: Unset "tbz x7, #1, 26f\n" - "ld1 { v26.h }[0], [x20], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 27f\n" - "ld1 { v26.b }[2], [x20]\n" + "ld1 { v16.b }[2], [x20]\n" "b 27f\n" "26:" // Oddments: Load (3, 0): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 27f\n" - "ld1 { v26.b }[0], [x20]\n" + "ld1 { v16.b }[0], [x20]\n" "27:" // Oddments: Load (3, 0): Bit 2: End - "ssubl v26.8h, v26.8b, v12.8b\n" + "ssubl v16.8h, v16.8b, v6.8b\n" "ldr x20, [x15, #0x60]\n" - "smlal v9.4s, v26.4h, v3.4h\n" - "smlal2 v23.4s, v26.8h, v3.8h\n" + "smlal v20.4s, v16.4h, v28.4h\n" + "smlal2 v0.4s, v16.8h, v28.8h\n" "add x20, x20, x17\n" "tbz x7, #2, 29f\n" - "ld1 { v25.s }[0], [x20], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 28f\n" - "ld1 { v25.h }[2], [x20], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 31f\n" - "ld1 { v25.b }[6], [x20]\n" + "ld1 { v16.b }[6], [x20]\n" "b 31f\n" "28:" // Oddments: Load (2, 0): Bit 2: Bit 1: Unset "tbz x7, #0, 31f\n" - "ld1 { v25.b }[4], [x20]\n" + "ld1 { v16.b }[4], [x20]\n" "b 31f\n" "29:" // Oddments: Load (2, 0): Bit 2: Unset "tbz x7, #1, 30f\n" - "ld1 { v25.h }[0], [x20], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 31f\n" - "ld1 { v25.b }[2], [x20]\n" + "ld1 { v16.b }[2], [x20]\n" "b 31f\n" "30:" // Oddments: Load (2, 0): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 31f\n" - "ld1 { v25.b }[0], [x20]\n" + "ld1 { v16.b }[0], [x20]\n" "31:" // Oddments: Load (2, 0): Bit 2: End - "ssubl v25.8h, v25.8b, v12.8b\n" + "ssubl v16.8h, v16.8b, v6.8b\n" "ldr x20, [x15, #0x68]\n" - "smlal v15.4s, v25.4h, v6.4h\n" - "smlal2 v17.4s, v25.8h, v6.8h\n" - "smlal v9.4s, v25.4h, v0.4h\n" - "smlal2 v23.4s, v25.8h, v0.8h\n" + "smlal v5.4s, v16.4h, v26.4h\n" + "smlal2 v3.4s, v16.8h, v26.8h\n" + "smlal v20.4s, v16.4h, v11.4h\n" + "smlal2 v0.4s, v16.8h, v11.8h\n" "add x20, x20, x17\n" "tbz x7, #2, 33f\n" - "ld1 { v29.s }[0], [x20], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 32f\n" - "ld1 { v29.h }[2], [x20], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 35f\n" - "ld1 { v29.b }[6], [x20]\n" + "ld1 { v16.b }[6], [x20]\n" "b 35f\n" "32:" // Oddments: Load (3, 1): Bit 2: Bit 1: Unset "tbz x7, #0, 35f\n" - "ld1 { v29.b }[4], [x20]\n" + "ld1 { v16.b }[4], [x20]\n" "b 35f\n" "33:" // Oddments: Load (3, 1): Bit 2: Unset "tbz x7, #1, 34f\n" - "ld1 { v29.h }[0], [x20], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 35f\n" - "ld1 { v29.b }[2], [x20]\n" + "ld1 { v16.b }[2], [x20]\n" "b 35f\n" "34:" // Oddments: Load (3, 1): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 35f\n" - "ld1 { v29.b }[0], [x20]\n" + "ld1 { v16.b }[0], [x20]\n" "35:" // Oddments: Load (3, 1): Bit 2: End - "ssubl v29.8h, v29.8b, v12.8b\n" - "ldr x21, [x15, #0x70]\n" - "smlal v9.4s, v29.4h, v4.4h\n" - "smlal2 v23.4s, v29.8h, v4.8h\n" - "add x21, x21, x17\n" + "ssubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0x70]\n" + "smlal v20.4s, v16.4h, v18.4h\n" + "smlal2 v0.4s, v16.8h, v18.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 37f\n" - "ld1 { v24.s }[0], [x21], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 36f\n" - "ld1 { v24.h }[2], [x21], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 39f\n" - "ld1 { v24.b }[6], [x21]\n" + "ld1 { v16.b }[6], [x20]\n" "b 39f\n" "36:" // Oddments: Load (2, 1): Bit 2: Bit 1: Unset "tbz x7, #0, 39f\n" - "ld1 { v24.b }[4], [x21]\n" + "ld1 { v16.b }[4], [x20]\n" "b 39f\n" "37:" // Oddments: Load (2, 1): Bit 2: Unset "tbz x7, #1, 38f\n" - "ld1 { v24.h }[0], [x21], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 39f\n" - "ld1 { v24.b }[2], [x21]\n" + "ld1 { v16.b }[2], [x20]\n" "b 39f\n" "38:" // Oddments: Load (2, 1): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 39f\n" - "ld1 { v24.b }[0], [x21]\n" + "ld1 { v16.b }[0], [x20]\n" "39:" // Oddments: Load (2, 1): Bit 2: End - "ssubl v24.8h, v24.8b, v12.8b\n" - "ldr x23, [x15, #0x78]\n" - "smlal v15.4s, v24.4h, v7.4h\n" - "smlal2 v17.4s, v24.8h, v7.8h\n" - "smlal v9.4s, v24.4h, v1.4h\n" - "smlal2 v23.4s, v24.8h, v1.8h\n" - "add x23, x23, x17\n" + "ssubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0x78]\n" + "smlal v5.4s, v16.4h, v7.4h\n" + "smlal2 v3.4s, v16.8h, v7.8h\n" + "smlal v20.4s, v16.4h, v22.4h\n" + "smlal2 v0.4s, v16.8h, v22.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 41f\n" - "ld1 { v27.s }[0], [x23], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 40f\n" - "ld1 { v27.h }[2], [x23], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 43f\n" - "ld1 { v27.b }[6], [x23]\n" + "ld1 { v16.b }[6], [x20]\n" "b 43f\n" "40:" // Oddments: Load (3, 3): Bit 2: Bit 1: Unset "tbz x7, #0, 43f\n" - "ld1 { v27.b }[4], [x23]\n" + "ld1 { v16.b }[4], [x20]\n" "b 43f\n" "41:" // Oddments: Load (3, 3): Bit 2: Unset "tbz x7, #1, 42f\n" - "ld1 { v27.h }[0], [x23], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 43f\n" - "ld1 { v27.b }[2], [x23]\n" + "ld1 { v16.b }[2], [x20]\n" "b 43f\n" "42:" // Oddments: Load (3, 3): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 43f\n" - "ld1 { v27.b }[0], [x23]\n" + "ld1 { v16.b }[0], [x20]\n" "43:" // Oddments: Load (3, 3): Bit 2: End - "ssubl v27.8h, v27.8b, v12.8b\n" - "ldr x21, [x15, #0x80]\n" - "smlal v21.4s, v27.4h, v4.4h\n" - "smlal2 v22.4s, v27.8h, v4.8h\n" - "add x21, x21, x17\n" + "ssubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0x80]\n" + "smlal v19.4s, v16.4h, v18.4h\n" + "smlal2 v31.4s, v16.8h, v18.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 45f\n" - "ld1 { v28.s }[0], [x21], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 44f\n" - "ld1 { v28.h }[2], [x21], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 47f\n" - "ld1 { v28.b }[6], [x21]\n" + "ld1 { v16.b }[6], [x20]\n" "b 47f\n" "44:" // Oddments: Load (2, 3): Bit 2: Bit 1: Unset "tbz x7, #0, 47f\n" - "ld1 { v28.b }[4], [x21]\n" + "ld1 { v16.b }[4], [x20]\n" "b 47f\n" "45:" // Oddments: Load (2, 3): Bit 2: Unset "tbz x7, #1, 46f\n" - "ld1 { v28.h }[0], [x21], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 47f\n" - "ld1 { v28.b }[2], [x21]\n" + "ld1 { v16.b }[2], [x20]\n" "b 47f\n" "46:" // Oddments: Load (2, 3): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 47f\n" - "ld1 { v28.b }[0], [x21]\n" + "ld1 { v16.b }[0], [x20]\n" "47:" // Oddments: Load (2, 3): Bit 2: End - "ssubl v28.8h, v28.8b, v12.8b\n" - "ldr x22, [x15, #0x88]\n" - "smlal v10.4s, v28.4h, v7.4h\n" - "smlal2 v20.4s, v28.8h, v7.8h\n" - "smlal v21.4s, v28.4h, v1.4h\n" - "smlal2 v22.4s, v28.8h, v1.8h\n" - "add x22, x22, x17\n" + "ssubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0x88]\n" + "smlal v21.4s, v16.4h, v7.4h\n" + "smlal2 v8.4s, v16.8h, v7.8h\n" + "smlal v19.4s, v16.4h, v22.4h\n" + "smlal2 v31.4s, v16.8h, v22.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 49f\n" - "ld1 { v26.s }[0], [x22], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 48f\n" - "ld1 { v26.h }[2], [x22], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 51f\n" - "ld1 { v26.b }[6], [x22]\n" + "ld1 { v16.b }[6], [x20]\n" "b 51f\n" "48:" // Oddments: Load (3, 4): Bit 2: Bit 1: Unset "tbz x7, #0, 51f\n" - "ld1 { v26.b }[4], [x22]\n" + "ld1 { v16.b }[4], [x20]\n" "b 51f\n" "49:" // Oddments: Load (3, 4): Bit 2: Unset "tbz x7, #1, 50f\n" - "ld1 { v26.h }[0], [x22], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 51f\n" - "ld1 { v26.b }[2], [x22]\n" + "ld1 { v16.b }[2], [x20]\n" "b 51f\n" "50:" // Oddments: Load (3, 4): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 51f\n" - "ld1 { v26.b }[0], [x22]\n" + "ld1 { v16.b }[0], [x20]\n" "51:" // Oddments: Load (3, 4): Bit 2: End - "ssubl v26.8h, v26.8b, v12.8b\n" - "ldr x24, [x15, #0x90]\n" - "smlal v21.4s, v26.4h, v5.4h\n" - "smlal2 v22.4s, v26.8h, v5.8h\n" - "add x24, x24, x17\n" + "ssubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0x90]\n" + "smlal v19.4s, v16.4h, v9.4h\n" + "smlal2 v31.4s, v16.8h, v9.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 53f\n" - "ld1 { v25.s }[0], [x24], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 52f\n" - "ld1 { v25.h }[2], [x24], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 55f\n" - "ld1 { v25.b }[6], [x24]\n" + "ld1 { v16.b }[6], [x20]\n" "b 55f\n" "52:" // Oddments: Load (4, 0): Bit 2: Bit 1: Unset "tbz x7, #0, 55f\n" - "ld1 { v25.b }[4], [x24]\n" + "ld1 { v16.b }[4], [x20]\n" "b 55f\n" "53:" // Oddments: Load (4, 0): Bit 2: Unset "tbz x7, #1, 54f\n" - "ld1 { v25.h }[0], [x24], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 55f\n" - "ld1 { v25.b }[2], [x24]\n" + "ld1 { v16.b }[2], [x20]\n" "b 55f\n" "54:" // Oddments: Load (4, 0): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 55f\n" - "ld1 { v25.b }[0], [x24]\n" + "ld1 { v16.b }[0], [x20]\n" "55:" // Oddments: Load (4, 0): Bit 2: End - "ssubl v25.8h, v25.8b, v12.8b\n" - "ldr x25, [x15, #0x98]\n" - "smlal v9.4s, v25.4h, v6.4h\n" - "smlal2 v23.4s, v25.8h, v6.8h\n" - "add x25, x25, x17\n" + "ssubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0x98]\n" + "smlal v20.4s, v16.4h, v26.4h\n" + "smlal2 v0.4s, v16.8h, v26.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 57f\n" - "ld1 { v29.s }[0], [x25], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 56f\n" - "ld1 { v29.h }[2], [x25], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 59f\n" - "ld1 { v29.b }[6], [x25]\n" + "ld1 { v16.b }[6], [x20]\n" "b 59f\n" "56:" // Oddments: Load (2, 4): Bit 2: Bit 1: Unset "tbz x7, #0, 59f\n" - "ld1 { v29.b }[4], [x25]\n" + "ld1 { v16.b }[4], [x20]\n" "b 59f\n" "57:" // Oddments: Load (2, 4): Bit 2: Unset "tbz x7, #1, 58f\n" - "ld1 { v29.h }[0], [x25], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 59f\n" - "ld1 { v29.b }[2], [x25]\n" + "ld1 { v16.b }[2], [x20]\n" "b 59f\n" "58:" // Oddments: Load (2, 4): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 59f\n" - "ld1 { v29.b }[0], [x25]\n" + "ld1 { v16.b }[0], [x20]\n" "59:" // Oddments: Load (2, 4): Bit 2: End - "ssubl v29.8h, v29.8b, v12.8b\n" + "ssubl v16.8h, v16.8b, v6.8b\n" "ldr x20, [x15, #0xa0]\n" - "smlal v10.4s, v29.4h, v8.4h\n" - "smlal2 v20.4s, v29.8h, v8.8h\n" - "smlal v21.4s, v29.4h, v2.4h\n" - "smlal2 v22.4s, v29.8h, v2.8h\n" + "smlal v21.4s, v16.4h, v4.4h\n" + "smlal2 v8.4s, v16.8h, v4.8h\n" + "smlal v19.4s, v16.4h, v14.4h\n" + "smlal2 v31.4s, v16.8h, v14.8h\n" "add x20, x20, x17\n" "tbz x7, #2, 61f\n" - "ld1 { v27.s }[0], [x20], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 60f\n" - "ld1 { v27.h }[2], [x20], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 63f\n" - "ld1 { v27.b }[6], [x20]\n" + "ld1 { v16.b }[6], [x20]\n" "b 63f\n" "60:" // Oddments: Load (4, 1): Bit 2: Bit 1: Unset "tbz x7, #0, 63f\n" - "ld1 { v27.b }[4], [x20]\n" + "ld1 { v16.b }[4], [x20]\n" "b 63f\n" "61:" // Oddments: Load (4, 1): Bit 2: Unset "tbz x7, #1, 62f\n" - "ld1 { v27.h }[0], [x20], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 63f\n" - "ld1 { v27.b }[2], [x20]\n" + "ld1 { v16.b }[2], [x20]\n" "b 63f\n" "62:" // Oddments: Load (4, 1): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 63f\n" - "ld1 { v27.b }[0], [x20]\n" + "ld1 { v16.b }[0], [x20]\n" "63:" // Oddments: Load (4, 1): Bit 2: End - "ssubl v27.8h, v27.8b, v12.8b\n" - "ldr x23, [x15, #0xa8]\n" - "smlal v9.4s, v27.4h, v7.4h\n" - "smlal2 v23.4s, v27.8h, v7.8h\n" - "add x23, x23, x17\n" + "ssubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0xa8]\n" + "smlal v20.4s, v16.4h, v7.4h\n" + "smlal2 v0.4s, v16.8h, v7.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 65f\n" - "ld1 { v24.s }[0], [x23], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 64f\n" - "ld1 { v24.h }[2], [x23], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 67f\n" - "ld1 { v24.b }[6], [x23]\n" + "ld1 { v16.b }[6], [x20]\n" "b 67f\n" "64:" // Oddments: Load (3, 2): Bit 2: Bit 1: Unset "tbz x7, #0, 67f\n" - "ld1 { v24.b }[4], [x23]\n" + "ld1 { v16.b }[4], [x20]\n" "b 67f\n" "65:" // Oddments: Load (3, 2): Bit 2: Unset "tbz x7, #1, 66f\n" - "ld1 { v24.h }[0], [x23], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 67f\n" - "ld1 { v24.b }[2], [x23]\n" + "ld1 { v16.b }[2], [x20]\n" "b 67f\n" "66:" // Oddments: Load (3, 2): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 67f\n" - "ld1 { v24.b }[0], [x23]\n" + "ld1 { v16.b }[0], [x20]\n" "67:" // Oddments: Load (3, 2): Bit 2: End - "ssubl v24.8h, v24.8b, v12.8b\n" - "ldr x22, [x15, #0xb0]\n" - "smlal v9.4s, v24.4h, v5.4h\n" - "smlal2 v23.4s, v24.8h, v5.8h\n" - "smlal v21.4s, v24.4h, v3.4h\n" - "smlal2 v22.4s, v24.8h, v3.8h\n" - "add x22, x22, x17\n" + "ssubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0xb0]\n" + "smlal v20.4s, v16.4h, v9.4h\n" + "smlal2 v0.4s, v16.8h, v9.8h\n" + "smlal v19.4s, v16.4h, v28.4h\n" + "smlal2 v31.4s, v16.8h, v28.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 69f\n" - "ld1 { v26.s }[0], [x22], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 68f\n" - "ld1 { v26.h }[2], [x22], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 71f\n" - "ld1 { v26.b }[6], [x22]\n" + "ld1 { v16.b }[6], [x20]\n" "b 71f\n" "68:" // Oddments: Load (4, 3): Bit 2: Bit 1: Unset "tbz x7, #0, 71f\n" - "ld1 { v26.b }[4], [x22]\n" + "ld1 { v16.b }[4], [x20]\n" "b 71f\n" "69:" // Oddments: Load (4, 3): Bit 2: Unset "tbz x7, #1, 70f\n" - "ld1 { v26.h }[0], [x22], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 71f\n" - "ld1 { v26.b }[2], [x22]\n" + "ld1 { v16.b }[2], [x20]\n" "b 71f\n" "70:" // Oddments: Load (4, 3): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 71f\n" - "ld1 { v26.b }[0], [x22]\n" + "ld1 { v16.b }[0], [x20]\n" "71:" // Oddments: Load (4, 3): Bit 2: End - "ssubl v26.8h, v26.8b, v12.8b\n" - "ldr x21, [x15, #0xb8]\n" - "smlal v21.4s, v26.4h, v7.4h\n" - "smlal2 v22.4s, v26.8h, v7.8h\n" - "add x21, x21, x17\n" + "ssubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0xb8]\n" + "smlal v19.4s, v16.4h, v7.4h\n" + "smlal2 v31.4s, v16.8h, v7.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 73f\n" - "ld1 { v25.s }[0], [x21], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 72f\n" - "ld1 { v25.h }[2], [x21], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 75f\n" - "ld1 { v25.b }[6], [x21]\n" + "ld1 { v16.b }[6], [x20]\n" "b 75f\n" "72:" // Oddments: Load (4, 2): Bit 2: Bit 1: Unset "tbz x7, #0, 75f\n" - "ld1 { v25.b }[4], [x21]\n" + "ld1 { v16.b }[4], [x20]\n" "b 75f\n" "73:" // Oddments: Load (4, 2): Bit 2: Unset "tbz x7, #1, 74f\n" - "ld1 { v25.h }[0], [x21], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 75f\n" - "ld1 { v25.b }[2], [x21]\n" + "ld1 { v16.b }[2], [x20]\n" "b 75f\n" "74:" // Oddments: Load (4, 2): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 75f\n" - "ld1 { v25.b }[0], [x21]\n" + "ld1 { v16.b }[0], [x20]\n" "75:" // Oddments: Load (4, 2): Bit 2: End - "ssubl v25.8h, v25.8b, v12.8b\n" + "ssubl v16.8h, v16.8b, v6.8b\n" "ldr x20, [x15, #0xc0]\n" - "smlal v9.4s, v25.4h, v8.4h\n" - "smlal2 v23.4s, v25.8h, v8.8h\n" - "smlal v21.4s, v25.4h, v6.4h\n" - "smlal2 v22.4s, v25.8h, v6.8h\n" + "smlal v20.4s, v16.4h, v4.4h\n" + "smlal2 v0.4s, v16.8h, v4.8h\n" + "smlal v19.4s, v16.4h, v26.4h\n" + "smlal2 v31.4s, v16.8h, v26.8h\n" "add x20, x20, x17\n" "tbz x7, #2, 77f\n" - "ld1 { v29.s }[0], [x20], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 76f\n" - "ld1 { v29.h }[2], [x20], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 79f\n" - "ld1 { v29.b }[6], [x20]\n" + "ld1 { v16.b }[6], [x20]\n" "b 79f\n" "76:" // Oddments: Load (4, 4): Bit 2: Bit 1: Unset "tbz x7, #0, 79f\n" - "ld1 { v29.b }[4], [x20]\n" + "ld1 { v16.b }[4], [x20]\n" "b 79f\n" "77:" // Oddments: Load (4, 4): Bit 2: Unset "tbz x7, #1, 78f\n" - "ld1 { v29.h }[0], [x20], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 79f\n" - "ld1 { v29.b }[2], [x20]\n" + "ld1 { v16.b }[2], [x20]\n" "b 79f\n" "78:" // Oddments: Load (4, 4): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 79f\n" - "ld1 { v29.b }[0], [x20]\n" + "ld1 { v16.b }[0], [x20]\n" "79:" // Oddments: Load (4, 4): Bit 2: End - "ssubl v29.8h, v29.8b, v12.8b\n" - "smlal v21.4s, v29.4h, v8.4h\n" - "smlal2 v22.4s, v29.8h, v8.8h\n" + "ssubl v16.8h, v16.8b, v6.8b\n" + "smlal v19.4s, v16.4h, v4.4h\n" + "smlal2 v31.4s, v16.8h, v4.8h\n" "tbz x7, #2, 81f\n" - "ld1 { v19.4s }, [x13], #0x10\n" - "ld1 { v18.4s }, [x12], #0x10\n" + "ld1 { v14.4s }, [x13], #0x10\n" + "ld1 { v25.4s }, [x12], #0x10\n" "tbz x7, #1, 80f\n" - "ld1 { v30.d }[0], [x13], #0x8\n" - "ld1 { v31.d }[0], [x12], #0x8\n" + "ld1 { v18.d }[0], [x13], #0x8\n" + "ld1 { v12.d }[0], [x12], #0x8\n" "tbz x7, #0, 83f\n" - "ld1 { v30.s }[2], [x13]\n" - "ld1 { v31.s }[2], [x12]\n" + "ld1 { v18.s }[2], [x13]\n" + "ld1 { v12.s }[2], [x12]\n" "b 83f\n" "80:" // Oddments: Load requant params: Bit 2: Bit 1: Unset "tbz x7, #0, 83f\n" - "ld1 { v30.s }[0], [x13]\n" - "ld1 { v31.s }[0], [x12]\n" + "ld1 { v18.s }[0], [x13]\n" + "ld1 { v12.s }[0], [x12]\n" "b 83f\n" "81:" // Oddments: Load requant params: Bit 2: Unset "tbz x7, #1, 82f\n" - "ld1 { v19.d }[0], [x13], #0x8\n" - "ld1 { v18.d }[0], [x12], #0x8\n" + "ld1 { v14.d }[0], [x13], #0x8\n" + "ld1 { v25.d }[0], [x12], #0x8\n" "tbz x7, #0, 83f\n" - "ld1 { v19.s }[2], [x13]\n" - "ld1 { v18.s }[2], [x12]\n" + "ld1 { v14.s }[2], [x13]\n" + "ld1 { v25.s }[2], [x12]\n" "b 83f\n" "82:" // Oddments: Load requant params: Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 83f\n" - "ld1 { v19.s }[0], [x13]\n" - "ld1 { v18.s }[0], [x12]\n" + "ld1 { v14.s }[0], [x13]\n" + "ld1 { v25.s }[0], [x12]\n" "83:" // Oddments: Load requant params: Bit 2: End - "sqrdmulh v15.4s, v15.4s, v19.4s\n" - "and v0.16b, v15.16b, v18.16b\n" + "sqrdmulh v5.4s, v5.4s, v14.4s\n" + "and v28.16b, v5.16b, v25.16b\n" "add x11, x11, x16\n" "add x10, x10, x16\n" - "sqrdmulh v17.4s, v17.4s, v30.4s\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqrdmulh v3.4s, v3.4s, v18.4s\n" + "sshr v28.4s, v28.4s, #0x1f\n" "add x9, x9, x16\n" "add x28, x28, x16\n" - "and v7.16b, v17.16b, v31.16b\n" - "sqrdmulh v10.4s, v10.4s, v19.4s\n" - "sqrdmulh v9.4s, v9.4s, v19.4s\n" - "sqrdmulh v21.4s, v21.4s, v19.4s\n" - "sqadd v15.4s, v15.4s, v0.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "and v19.16b, v10.16b, v18.16b\n" - "sqrdmulh v20.4s, v20.4s, v30.4s\n" - "and v27.16b, v9.16b, v18.16b\n" - "sqrdmulh v23.4s, v23.4s, v30.4s\n" - "and v0.16b, v21.16b, v18.16b\n" - "sqrdmulh v22.4s, v22.4s, v30.4s\n" - "sqadd v17.4s, v17.4s, v7.4s\n" - "sshr v19.4s, v19.4s, #0x1f\n" - "and v5.16b, v20.16b, v31.16b\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "and v4.16b, v23.16b, v31.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v7.16b, v22.16b, v31.16b\n" - "sqadd v10.4s, v10.4s, v19.4s\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v9.4s, v9.4s, v27.4s\n" + "and v16.16b, v3.16b, v12.16b\n" + "sqrdmulh v21.4s, v21.4s, v14.4s\n" + "sqrdmulh v20.4s, v20.4s, v14.4s\n" + "sqrdmulh v19.4s, v19.4s, v14.4s\n" + "sqadd v5.4s, v5.4s, v28.4s\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "and v14.16b, v21.16b, v25.16b\n" + "sqrdmulh v8.4s, v8.4s, v18.4s\n" + "and v6.16b, v20.16b, v25.16b\n" + "sqrdmulh v0.4s, v0.4s, v18.4s\n" + "and v4.16b, v19.16b, v25.16b\n" + "sqrdmulh v31.4s, v31.4s, v18.4s\n" + "sqadd v3.4s, v3.4s, v16.4s\n" + "sshr v14.4s, v14.4s, #0x1f\n" + "and v18.16b, v8.16b, v12.16b\n" + "sshr v6.4s, v6.4s, #0x1f\n" + "and v7.16b, v0.16b, v12.16b\n" "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v21.4s, v21.4s, v0.4s\n" + "and v16.16b, v31.16b, v12.16b\n" + "sqadd v21.4s, v21.4s, v14.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v6.4s\n" "sshr v7.4s, v7.4s, #0x1f\n" - "srshl v15.4s, v15.4s, v18.4s\n" - "srshl v10.4s, v10.4s, v18.4s\n" - "sqadd v20.4s, v20.4s, v5.4s\n" - "srshl v9.4s, v9.4s, v18.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "srshl v21.4s, v21.4s, v18.4s\n" - "sqadd v22.4s, v22.4s, v7.4s\n" - "srshl v17.4s, v17.4s, v31.4s\n" - "sqxtn v15.4h, v15.4s\n" - "srshl v20.4s, v20.4s, v31.4s\n" - "sqxtn v10.4h, v10.4s\n" - "srshl v23.4s, v23.4s, v31.4s\n" - "sqxtn v9.4h, v9.4s\n" - "srshl v22.4s, v22.4s, v31.4s\n" + "sqadd v19.4s, v19.4s, v4.4s\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "srshl v5.4s, v5.4s, v25.4s\n" + "srshl v21.4s, v21.4s, v25.4s\n" + "sqadd v8.4s, v8.4s, v18.4s\n" + "srshl v20.4s, v20.4s, v25.4s\n" + "sqadd v0.4s, v0.4s, v7.4s\n" + "srshl v19.4s, v19.4s, v25.4s\n" + "sqadd v31.4s, v31.4s, v16.4s\n" + "srshl v3.4s, v3.4s, v12.4s\n" + "sqxtn v5.4h, v5.4s\n" + "srshl v8.4s, v8.4s, v12.4s\n" "sqxtn v21.4h, v21.4s\n" - "sqxtn2 v15.8h, v17.4s\n" - "sqxtn2 v10.8h, v20.4s\n" - "sqxtn2 v9.8h, v23.4s\n" - "sqxtn2 v21.8h, v22.4s\n" - "sqadd v15.8h, v15.8h, v11.8h\n" - "sqadd v10.8h, v10.8h, v11.8h\n" - "sqadd v9.8h, v9.8h, v11.8h\n" - "sqadd v21.8h, v21.8h, v11.8h\n" - "smax v15.8h, v15.8h, v16.8h\n" - "smax v10.8h, v10.8h, v16.8h\n" - "smax v9.8h, v9.8h, v16.8h\n" - "smax v21.8h, v21.8h, v16.8h\n" - "smin v15.8h, v15.8h, v14.8h\n" - "smin v10.8h, v10.8h, v14.8h\n" - "smin v9.8h, v9.8h, v14.8h\n" - "smin v21.8h, v21.8h, v14.8h\n" - "uzp1 v15.16b, v15.16b, v15.16b\n" - "uzp1 v10.16b, v10.16b, v10.16b\n" - "uzp1 v9.16b, v9.16b, v9.16b\n" + "srshl v0.4s, v0.4s, v12.4s\n" + "sqxtn v20.4h, v20.4s\n" + "srshl v31.4s, v31.4s, v12.4s\n" + "sqxtn v19.4h, v19.4s\n" + "sqxtn2 v5.8h, v3.4s\n" + "sqxtn2 v21.8h, v8.4s\n" + "sqxtn2 v20.8h, v0.4s\n" + "sqxtn2 v19.8h, v31.4s\n" + "sqadd v5.8h, v5.8h, v13.8h\n" + "sqadd v21.8h, v21.8h, v13.8h\n" + "sqadd v20.8h, v20.8h, v13.8h\n" + "sqadd v19.8h, v19.8h, v13.8h\n" + "smax v5.8h, v5.8h, v17.8h\n" + "smax v21.8h, v21.8h, v17.8h\n" + "smax v20.8h, v20.8h, v17.8h\n" + "smax v19.8h, v19.8h, v17.8h\n" + "smin v5.8h, v5.8h, v24.8h\n" + "smin v21.8h, v21.8h, v24.8h\n" + "smin v20.8h, v20.8h, v24.8h\n" + "smin v19.8h, v19.8h, v24.8h\n" + "uzp1 v5.16b, v5.16b, v5.16b\n" "uzp1 v21.16b, v21.16b, v21.16b\n" + "uzp1 v20.16b, v20.16b, v20.16b\n" + "uzp1 v19.16b, v19.16b, v19.16b\n" "tbz x7, #2, 85f\n" - "st1 { v15.s }[0], [x11], #0x4\n" - "st1 { v10.s }[0], [x10], #0x4\n" - "st1 { v9.s }[0], [x9], #0x4\n" - "st1 { v21.s }[0], [x28], #0x4\n" + "st1 { v5.s }[0], [x11], #0x4\n" + "st1 { v21.s }[0], [x10], #0x4\n" + "st1 { v20.s }[0], [x9], #0x4\n" + "st1 { v19.s }[0], [x28], #0x4\n" "tbz x7, #1, 84f\n" - "st1 { v15.h }[2], [x11], #0x2\n" - "st1 { v10.h }[2], [x10], #0x2\n" - "st1 { v9.h }[2], [x9], #0x2\n" - "st1 { v21.h }[2], [x28], #0x2\n" + "st1 { v5.h }[2], [x11], #0x2\n" + "st1 { v21.h }[2], [x10], #0x2\n" + "st1 { v20.h }[2], [x9], #0x2\n" + "st1 { v19.h }[2], [x28], #0x2\n" "tbz x7, #0, 87f\n" - "st1 { v15.b }[6], [x11], #0x1\n" - "st1 { v10.b }[6], [x10], #0x1\n" - "st1 { v9.b }[6], [x9], #0x1\n" - "st1 { v21.b }[6], [x28], #0x1\n" + "st1 { v5.b }[6], [x11], #0x1\n" + "st1 { v21.b }[6], [x10], #0x1\n" + "st1 { v20.b }[6], [x9], #0x1\n" + "st1 { v19.b }[6], [x28], #0x1\n" "b 87f\n" "84:" // Oddments: Bit 2: Bit 1: Unset "tbz x7, #0, 87f\n" - "st1 { v15.b }[4], [x11], #0x1\n" - "st1 { v10.b }[4], [x10], #0x1\n" - "st1 { v9.b }[4], [x9], #0x1\n" - "st1 { v21.b }[4], [x28], #0x1\n" + "st1 { v5.b }[4], [x11], #0x1\n" + "st1 { v21.b }[4], [x10], #0x1\n" + "st1 { v20.b }[4], [x9], #0x1\n" + "st1 { v19.b }[4], [x28], #0x1\n" "b 87f\n" "85:" // Oddments: Bit 2: Unset "tbz x7, #1, 86f\n" - "st1 { v15.h }[0], [x11], #0x2\n" - "st1 { v10.h }[0], [x10], #0x2\n" - "st1 { v9.h }[0], [x9], #0x2\n" - "st1 { v21.h }[0], [x28], #0x2\n" + "st1 { v5.h }[0], [x11], #0x2\n" + "st1 { v21.h }[0], [x10], #0x2\n" + "st1 { v20.h }[0], [x9], #0x2\n" + "st1 { v19.h }[0], [x28], #0x2\n" "tbz x7, #0, 87f\n" - "st1 { v15.b }[2], [x11], #0x1\n" - "st1 { v10.b }[2], [x10], #0x1\n" - "st1 { v9.b }[2], [x9], #0x1\n" - "st1 { v21.b }[2], [x28], #0x1\n" + "st1 { v5.b }[2], [x11], #0x1\n" + "st1 { v21.b }[2], [x10], #0x1\n" + "st1 { v20.b }[2], [x9], #0x1\n" + "st1 { v19.b }[2], [x28], #0x1\n" "b 87f\n" "86:" // Oddments: Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 87f\n" - "st1 { v15.b }[0], [x11], #0x1\n" - "st1 { v10.b }[0], [x10], #0x1\n" - "st1 { v9.b }[0], [x9], #0x1\n" - "st1 { v21.b }[0], [x28], #0x1\n" + "st1 { v5.b }[0], [x11], #0x1\n" + "st1 { v21.b }[0], [x10], #0x1\n" + "st1 { v20.b }[0], [x9], #0x1\n" + "st1 { v19.b }[0], [x28], #0x1\n" "87:" // Oddments: Bit 2: End "88:" // End : diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp index e60597d390f94d513d57a0983b581bcbfbd7e0e9..65ebe627efc26f087c3e8b1dd77a22b723258663 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include @@ -34,15 +34,7 @@ namespace arm_conv { namespace depthwise { -void a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( - const unsigned int, - const int8_t *const *const, - const int8_t *const, - const int32_t *const, - const arm_gemm::Requantize32 &, - const int32_t *const, - const int32_t *const, - int8_t *const *const); +void a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl(unsigned int, const int8_t *const *, const int8_t *, const int32_t *, const arm_gemm::Requantize32 &, const int32_t *, const int32_t *, int8_t *const *); class a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirstStrategy { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp index 4b0ad00187f5ce449e0bc6e4ffffb35fbf362838..1dc0f33186ef9325fe78ba581920fa5899413c08 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp @@ -112,1188 +112,1188 @@ void a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( __asm__ __volatile__( "ldr x1, [%x[params], %[offsetof_Params_n_channels]]\n" - "ldr x13, [%x[params], %[offsetof_Params_requant]]\n" + "ldr x23, [%x[params], %[offsetof_Params_requant]]\n" "lsr x2, x1, #0x3\n" - "add x3, x13, %[offsetof_Requantize32_a_offset]\n" - "ld1r { v9.16b }, [x3]\n" - "ldr x24, [%x[params], %[offsetof_Params_outptrs]]\n" - "add x11, x13, %[offsetof_Requantize32_b_offset]\n" - "add x5, x13, %[offsetof_Requantize32_c_offset]\n" - "ld1r { v15.16b }, [x11]\n" - "ld1r { v14.8h }, [x5]\n" - "add x3, x13, %[offsetof_Requantize32_minval]\n" - "add x15, x13, %[offsetof_Requantize32_maxval]\n" - "ld1r { v12.8h }, [x3]\n" - "ld1r { v11.8h }, [x15]\n" - "mov x0, #0x0\n" - "mov x10, #0x0\n" - "add x4, %x[params], %[offsetof_Params_inptrs]\n" - "ldr x3, [%x[params], %[offsetof_Params_weights]]\n" - "ldr x5, [%x[params], %[offsetof_Params_requant_muls]]\n" + "add x20, x23, %[offsetof_Requantize32_a_offset]\n" + "ld1r { v18.16b }, [x20]\n" + "ldr x22, [%x[params], %[offsetof_Params_outptrs]]\n" + "add x21, x23, %[offsetof_Requantize32_b_offset]\n" + "add x20, x23, %[offsetof_Requantize32_c_offset]\n" + "ld1r { v13.16b }, [x21]\n" + "ld1r { v26.8h }, [x20]\n" + "add x21, x23, %[offsetof_Requantize32_minval]\n" + "add x20, x23, %[offsetof_Requantize32_maxval]\n" + "ld1r { v11.8h }, [x21]\n" + "ld1r { v0.8h }, [x20]\n" + "mov x3, #0x0\n" + "mov x4, #0x0\n" + "add x5, %x[params], %[offsetof_Params_inptrs]\n" + "ldr x6, [%x[params], %[offsetof_Params_weights]]\n" + "ldr x7, [%x[params], %[offsetof_Params_requant_muls]]\n" "ldr x8, [%x[params], %[offsetof_Params_requant_shifts]]\n" - "ldp x17, x6, [x24, #0x0]\n" - "ldp x7, x16, [x24, #0x10]\n" + "ldp x17, x16, [x22, #0x0]\n" + "ldp x15, x14, [x22, #0x10]\n" "cbz x2, 3f\n" - "ldr d0, [x3, #0x0]\n" - "ldr d1, [x3, #0x8]\n" + "ldr d6, [x6, #0x0]\n" + "ldr d14, [x6, #0x8]\n" "subs x2, x2, #0x1\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "ldr d2, [x3, #0x10]\n" - "ldr d3, [x3, #0x18]\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "ldr d4, [x3, #0x20]\n" - "ldr x13, [%x[params], %[offsetof_Params_bias]]\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "ldr q13, [x13, #0x0]\n" - "ldr q19, [x13, #0x10]\n" - "add x13, x13, #0x20\n" - "str x13, [%x[params], %[offsetof_Params_bias]]\n" - "ldp x9, x28, [x4, #0x0]\n" - "ldp x27, x26, [x4, #0x10]\n" - "mov v20.16b, v13.16b\n" - "mov v10.16b, v19.16b\n" - "ldp x25, x24, [x4, #0x20]\n" - "ldp x23, x22, [x4, #0x30]\n" - "mov v8.16b, v13.16b\n" - "mov v7.16b, v19.16b\n" - "ldp x21, x20, [x4, #0x40]\n" - "ldr d31, [x9, x0]\n" - "mov v17.16b, v13.16b\n" - "mov v21.16b, v19.16b\n" - "ldr d30, [x28, x0]\n" - "ldr d29, [x27, x0]\n" - "ssubl v31.8h, v31.8b, v9.8b\n" - "ssubl v30.8h, v30.8b, v9.8b\n" - "ldr d28, [x26, x0]\n" - "ldr d27, [x25, x0]\n" - "ssubl v29.8h, v29.8b, v9.8b\n" - "ssubl v28.8h, v28.8b, v9.8b\n" - "ldr d23, [x24, x0]\n" - "ldr d25, [x23, x0]\n" - "ssubl v27.8h, v27.8b, v9.8b\n" - "ssubl v23.8h, v23.8b, v9.8b\n" - "ldr d24, [x22, x0]\n" - "ldr d26, [x21, x0]\n" - "ssubl v25.8h, v25.8b, v9.8b\n" - "ssubl v24.8h, v24.8b, v9.8b\n" - "ldr d22, [x20, x0]\n" - "ssubl v26.8h, v26.8b, v9.8b\n" - "ssubl v22.8h, v22.8b, v9.8b\n" + "ssubl v6.8h, v6.8b, v13.8b\n" + "ldr d10, [x6, #0x10]\n" + "ldr d21, [x6, #0x18]\n" + "ssubl v14.8h, v14.8b, v13.8b\n" + "ssubl v10.8h, v10.8b, v13.8b\n" + "ldr d12, [x6, #0x20]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "ssubl v21.8h, v21.8b, v13.8b\n" + "ssubl v12.8h, v12.8b, v13.8b\n" + "ldr q7, [x20, #0x0]\n" + "ldr q15, [x20, #0x10]\n" + "add x20, x20, #0x20\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" + "ldp x9, x28, [x5, #0x0]\n" + "ldp x27, x26, [x5, #0x10]\n" + "mov v20.16b, v7.16b\n" + "mov v5.16b, v15.16b\n" + "ldp x25, x24, [x5, #0x20]\n" + "ldp x23, x22, [x5, #0x30]\n" + "mov v24.16b, v7.16b\n" + "mov v22.16b, v15.16b\n" + "ldp x21, x20, [x5, #0x40]\n" + "ldr d31, [x9, x3]\n" + "mov v23.16b, v7.16b\n" + "mov v19.16b, v15.16b\n" + "ldr d17, [x28, x3]\n" + "ldr d30, [x27, x3]\n" + "ssubl v31.8h, v31.8b, v18.8b\n" + "ssubl v17.8h, v17.8b, v18.8b\n" + "ldr d16, [x26, x3]\n" + "ldr d3, [x25, x3]\n" + "ssubl v30.8h, v30.8b, v18.8b\n" + "ssubl v16.8h, v16.8b, v18.8b\n" + "ldr d4, [x24, x3]\n" + "ldr d25, [x23, x3]\n" + "ssubl v3.8h, v3.8b, v18.8b\n" + "ssubl v4.8h, v4.8b, v18.8b\n" + "ldr d9, [x22, x3]\n" + "ldr d29, [x21, x3]\n" + "ssubl v25.8h, v25.8b, v18.8b\n" + "ssubl v9.8h, v9.8b, v18.8b\n" + "ldr d28, [x20, x3]\n" + "ssubl v29.8h, v29.8b, v18.8b\n" + "ssubl v28.8h, v28.8b, v18.8b\n" "beq 2f\n" "1:" // Loop - "ldr q18, [x5, #0x0]\n" - "ldr q6, [x8, #0x0]\n" - "smlal v13.4s, v31.4h, v0.4h\n" - "smlal2 v19.4s, v31.8h, v0.8h\n" - "ldr q5, [x5, #0x10]\n" - "smlal v13.4s, v30.4h, v1.4h\n" - "ldr x20, [x4, #0x50]\n" - "smlal v20.4s, v30.4h, v0.4h\n" - "smlal v8.4s, v29.4h, v0.4h\n" - "smlal v17.4s, v28.4h, v0.4h\n" - "ldr x22, [x4, #0x58]\n" - "ldr x21, [x4, #0x60]\n" - "smlal2 v19.4s, v30.8h, v1.8h\n" - "smlal2 v10.4s, v30.8h, v0.8h\n" - "ldr d31, [x20, x0]\n" - "ssubl v31.8h, v31.8b, v9.8b\n" - "smlal2 v7.4s, v29.8h, v0.8h\n" - "smlal v13.4s, v27.4h, v2.4h\n" - "ldr x20, [x4, #0x68]\n" - "ldr x26, [x4, #0x70]\n" - "smlal2 v21.4s, v28.8h, v0.8h\n" - "ldr d30, [x22, x0]\n" - "smlal v20.4s, v27.4h, v1.4h\n" - "ssubl v30.8h, v30.8b, v9.8b\n" - "smlal v8.4s, v28.4h, v1.4h\n" - "smlal v17.4s, v23.4h, v1.4h\n" - "ldr x25, [x4, #0x78]\n" - "ldr x23, [x4, #0x80]\n" - "smlal2 v19.4s, v27.8h, v2.8h\n" - "smlal2 v10.4s, v27.8h, v1.8h\n" - "ldr d0, [x3, #0x28]\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "smlal2 v7.4s, v28.8h, v1.8h\n" - "smlal v13.4s, v25.4h, v3.4h\n" - "ldr x24, [x4, #0x88]\n" - "ldr x15, [x4, #0x90]\n" - "smlal2 v21.4s, v23.8h, v1.8h\n" - "ldr d27, [x21, x0]\n" - "smlal v20.4s, v25.4h, v2.4h\n" - "ssubl v27.8h, v27.8b, v9.8b\n" - "smlal v8.4s, v23.4h, v2.4h\n" - "smlal v17.4s, v31.4h, v2.4h\n" - "ldr x21, [x4, #0x98]\n" - "ldr x14, [x4, #0xa0]\n" - "smlal2 v19.4s, v25.8h, v3.8h\n" - "smlal2 v10.4s, v25.8h, v2.8h\n" - "ldr d1, [x3, #0x30]\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "smlal2 v7.4s, v23.8h, v2.8h\n" - "smlal v13.4s, v24.4h, v4.4h\n" - "ldr x13, [x4, #0xa8]\n" - "ldr x12, [x4, #0xb0]\n" - "smlal2 v21.4s, v31.8h, v2.8h\n" - "ldr d25, [x20, x0]\n" - "smlal v20.4s, v24.4h, v3.4h\n" - "ssubl v25.8h, v25.8b, v9.8b\n" - "smlal v8.4s, v31.4h, v3.4h\n" - "smlal v17.4s, v30.4h, v3.4h\n" - "ldr x20, [x4, #0xb8]\n" - "ldr x11, [x4, #0xc0]\n" - "smlal2 v19.4s, v24.8h, v4.8h\n" - "smlal2 v10.4s, v24.8h, v3.8h\n" - "ldr d2, [x3, #0x38]\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "smlal2 v7.4s, v31.8h, v3.8h\n" - "smlal v13.4s, v29.4h, v0.4h\n" - "ldr x22, [x4, #0xc8]\n" - "ldr x9, [x4, #0xd0]\n" - "smlal2 v21.4s, v30.8h, v3.8h\n" - "ldr d24, [x26, x0]\n" - "smlal v20.4s, v27.4h, v4.4h\n" - "ssubl v24.8h, v24.8b, v9.8b\n" - "smlal v8.4s, v30.4h, v4.4h\n" - "smlal v17.4s, v26.4h, v4.4h\n" - "ldr x28, [x4, #0xd8]\n" - "ldr x27, [x4, #0xe0]\n" - "smlal2 v19.4s, v29.8h, v0.8h\n" - "ldr d3, [x3, #0x40]\n" - "smlal2 v10.4s, v27.8h, v4.8h\n" - "ldr d27, [x25, x0]\n" - "smlal2 v7.4s, v30.8h, v4.8h\n" - "smlal v13.4s, v28.4h, v1.4h\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "ldr x26, [x4, #0xe8]\n" - "smlal2 v21.4s, v26.8h, v4.8h\n" - "ldr d4, [x3, #0x48]\n" - "smlal v20.4s, v28.4h, v0.4h\n" - "ssubl v27.8h, v27.8b, v9.8b\n" - "smlal v8.4s, v22.4h, v0.4h\n" - "smlal v17.4s, v25.4h, v0.4h\n" - "ssubl v4.8h, v4.8b, v15.8b\n" + "ldr d2, [x6, #0x28]\n" + "ldr d27, [x6, #0x30]\n" + "smlal v7.4s, v31.4h, v6.4h\n" + "smlal2 v15.4s, v31.8h, v6.8h\n" + "ldr d1, [x6, #0x38]\n" + "ldr d31, [x6, #0x40]\n" + "smlal v7.4s, v17.4h, v14.4h\n" + "smlal v20.4s, v17.4h, v6.4h\n" + "ldr d8, [x6, #0x48]\n" + "ldr x22, [x5, #0x50]\n" + "smlal v24.4s, v30.4h, v6.4h\n" + "smlal v23.4s, v16.4h, v6.4h\n" + "smlal2 v15.4s, v17.8h, v14.8h\n" + "smlal v7.4s, v3.4h, v10.4h\n" + "ldr x20, [x5, #0x58]\n" + "ldr x21, [x5, #0x60]\n" + "smlal2 v5.4s, v17.8h, v6.8h\n" + "ldr d17, [x22, x3]\n" + "smlal2 v22.4s, v30.8h, v6.8h\n" + "ssubl v17.8h, v17.8b, v18.8b\n" + "smlal2 v19.4s, v16.8h, v6.8h\n" + "ldr d6, [x20, x3]\n" + "smlal v20.4s, v3.4h, v14.4h\n" + "ssubl v6.8h, v6.8b, v18.8b\n" + "smlal v24.4s, v16.4h, v14.4h\n" + "smlal v23.4s, v4.4h, v14.4h\n" + "ssubl v2.8h, v2.8b, v13.8b\n" + "ldr x20, [x5, #0x68]\n" + "smlal2 v15.4s, v3.8h, v10.8h\n" + "smlal v7.4s, v25.4h, v21.4h\n" + "ssubl v27.8h, v27.8b, v13.8b\n" + "ldr x22, [x5, #0x70]\n" + "smlal2 v5.4s, v3.8h, v14.8h\n" + "ldr d3, [x21, x3]\n" + "smlal2 v22.4s, v16.8h, v14.8h\n" + "ssubl v3.8h, v3.8b, v18.8b\n" + "smlal2 v19.4s, v4.8h, v14.8h\n" + "ldr d14, [x20, x3]\n" + "smlal v20.4s, v25.4h, v10.4h\n" + "ssubl v14.8h, v14.8b, v18.8b\n" + "smlal v24.4s, v4.4h, v10.4h\n" + "smlal v23.4s, v17.4h, v10.4h\n" + "ssubl v1.8h, v1.8b, v13.8b\n" + "ldr x20, [x5, #0x78]\n" + "smlal2 v15.4s, v25.8h, v21.8h\n" + "smlal v7.4s, v9.4h, v12.4h\n" + "ssubl v31.8h, v31.8b, v13.8b\n" + "ldr x21, [x5, #0x80]\n" + "smlal2 v5.4s, v25.8h, v10.8h\n" + "ldr d25, [x22, x3]\n" + "smlal2 v22.4s, v4.8h, v10.8h\n" + "ssubl v25.8h, v25.8b, v18.8b\n" + "smlal2 v19.4s, v17.8h, v10.8h\n" + "ldr d10, [x20, x3]\n" + "smlal v20.4s, v9.4h, v21.4h\n" + "ssubl v10.8h, v10.8b, v18.8b\n" + "smlal v24.4s, v17.4h, v21.4h\n" + "smlal v23.4s, v6.4h, v21.4h\n" + "ssubl v8.8h, v8.8b, v13.8b\n" + "ldr x24, [x5, #0x88]\n" + "smlal2 v15.4s, v9.8h, v12.8h\n" + "smlal v7.4s, v30.4h, v2.4h\n" + "ldr x20, [x5, #0x90]\n" + "ldr x23, [x5, #0x98]\n" + "smlal2 v5.4s, v9.8h, v21.8h\n" + "ldr d9, [x21, x3]\n" + "smlal2 v22.4s, v17.8h, v21.8h\n" + "ssubl v9.8h, v9.8b, v18.8b\n" + "smlal2 v19.4s, v6.8h, v21.8h\n" + "ldr d21, [x6, #0x50]\n" + "smlal v20.4s, v3.4h, v12.4h\n" + "ssubl v21.8h, v21.8b, v13.8b\n" + "smlal v24.4s, v6.4h, v12.4h\n" + "smlal v23.4s, v29.4h, v12.4h\n" + "ldr x22, [x5, #0xa0]\n" + "ldr x21, [x5, #0xa8]\n" + "smlal2 v15.4s, v30.8h, v2.8h\n" + "ldr d30, [x24, x3]\n" + "smlal v7.4s, v16.4h, v27.4h\n" + "ssubl v30.8h, v30.8b, v18.8b\n" + "smlal2 v5.4s, v3.8h, v12.8h\n" + "ldr d3, [x6, #0x58]\n" + "smlal2 v22.4s, v6.8h, v12.8h\n" + "ssubl v3.8h, v3.8b, v13.8b\n" + "smlal2 v19.4s, v29.8h, v12.8h\n" + "ldr d12, [x20, x3]\n" + "smlal v20.4s, v16.4h, v2.4h\n" + "ssubl v12.8h, v12.8b, v18.8b\n" + "smlal v24.4s, v28.4h, v2.4h\n" + "smlal v23.4s, v14.4h, v2.4h\n" + "ldr x20, [x5, #0xb0]\n" + "ldr x13, [x5, #0xb8]\n" + "smlal2 v15.4s, v16.8h, v27.8h\n" + "smlal v7.4s, v4.4h, v1.4h\n" + "ldr x12, [x5, #0xc0]\n" + "ldr x11, [x5, #0xc8]\n" + "smlal2 v5.4s, v16.8h, v2.8h\n" + "ldr d16, [x23, x3]\n" + "smlal2 v22.4s, v28.8h, v2.8h\n" + "ssubl v16.8h, v16.8b, v18.8b\n" + "smlal2 v19.4s, v14.8h, v2.8h\n" + "ldr d2, [x6, #0x60]\n" + "smlal v20.4s, v4.4h, v27.4h\n" + "ssubl v2.8h, v2.8b, v13.8b\n" + "smlal v24.4s, v14.4h, v27.4h\n" + "smlal v23.4s, v25.4h, v27.4h\n" + "ldr x10, [x5, #0xd0]\n" + "ldr x9, [x5, #0xd8]\n" + "smlal2 v15.4s, v4.8h, v1.8h\n" + "smlal v7.4s, v17.4h, v31.4h\n" + "ldr x28, [x5, #0xe0]\n" + "ldr x27, [x5, #0xe8]\n" + "smlal2 v5.4s, v4.8h, v27.8h\n" + "ldr d4, [x22, x3]\n" + "smlal2 v22.4s, v14.8h, v27.8h\n" + "ssubl v4.8h, v4.8b, v18.8b\n" + "smlal2 v19.4s, v25.8h, v27.8h\n" + "ldr d27, [x6, #0x68]\n" + "smlal v20.4s, v17.4h, v1.4h\n" + "ssubl v27.8h, v27.8b, v13.8b\n" + "smlal v24.4s, v25.4h, v1.4h\n" + "smlal v23.4s, v10.4h, v1.4h\n" + "ldr x26, [x5, #0xf0]\n" + "ldr x25, [x5, #0xf8]\n" + "smlal2 v15.4s, v17.8h, v31.8h\n" + "smlal v7.4s, v6.4h, v8.4h\n" + "ldr x24, [x5, #0x100]\n" + "ldr x23, [x5, #0x108]\n" + "smlal2 v5.4s, v17.8h, v1.8h\n" + "ldr d17, [x21, x3]\n" + "smlal2 v22.4s, v25.8h, v1.8h\n" + "ssubl v17.8h, v17.8b, v18.8b\n" + "smlal2 v19.4s, v10.8h, v1.8h\n" + "ldr d1, [x6, #0x70]\n" + "smlal v20.4s, v6.4h, v31.4h\n" + "ssubl v1.8h, v1.8b, v13.8b\n" + "smlal v24.4s, v10.4h, v31.4h\n" + "smlal v23.4s, v9.4h, v31.4h\n" + "ldr x22, [x5, #0x110]\n" + "ldr x21, [x5, #0x118]\n" + "smlal2 v15.4s, v6.8h, v8.8h\n" + "smlal v7.4s, v28.4h, v21.4h\n" "subs x2, x2, #0x1\n" + "smlal2 v5.4s, v6.8h, v31.8h\n" + "ldr d6, [x20, x3]\n" + "smlal2 v22.4s, v10.8h, v31.8h\n" + "ssubl v6.8h, v6.8b, v18.8b\n" + "smlal2 v19.4s, v9.8h, v31.8h\n" + "ldr d31, [x6, #0x78]\n" + "smlal v20.4s, v29.4h, v8.4h\n" + "ssubl v31.8h, v31.8b, v13.8b\n" + "smlal v24.4s, v9.4h, v8.4h\n" + "smlal v23.4s, v30.4h, v8.4h\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "smlal2 v15.4s, v28.8h, v21.8h\n" + "ldr d28, [x13, x3]\n" + "smlal v7.4s, v14.4h, v3.4h\n" + "ssubl v28.8h, v28.8b, v18.8b\n" + "smlal2 v5.4s, v29.8h, v8.8h\n" + "ldr d29, [x6, #0x80]\n" + "smlal2 v22.4s, v9.8h, v8.8h\n" + "ssubl v29.8h, v29.8b, v13.8b\n" + "smlal2 v19.4s, v30.8h, v8.8h\n" + "ldr d8, [x12, x3]\n" + "smlal v20.4s, v14.4h, v21.4h\n" + "ssubl v8.8h, v8.8b, v18.8b\n" + "smlal v24.4s, v12.4h, v21.4h\n" + "smlal v23.4s, v16.4h, v21.4h\n" + "smlal2 v15.4s, v14.8h, v3.8h\n" + "smlal v7.4s, v25.4h, v2.4h\n" + "smlal2 v5.4s, v14.8h, v21.8h\n" + "ldr d14, [x11, x3]\n" + "smlal2 v22.4s, v12.8h, v21.8h\n" + "ssubl v14.8h, v14.8b, v18.8b\n" + "smlal2 v19.4s, v16.8h, v21.8h\n" + "ldr d21, [x6, #0x88]\n" + "smlal v20.4s, v25.4h, v3.4h\n" + "ssubl v21.8h, v21.8b, v13.8b\n" + "smlal v24.4s, v16.4h, v3.4h\n" + "smlal v23.4s, v4.4h, v3.4h\n" + "smlal2 v15.4s, v25.8h, v2.8h\n" + "smlal v7.4s, v10.4h, v27.4h\n" + "smlal2 v5.4s, v25.8h, v3.8h\n" + "ldr d25, [x10, x3]\n" + "smlal2 v22.4s, v16.8h, v3.8h\n" + "ssubl v25.8h, v25.8b, v18.8b\n" + "smlal2 v19.4s, v4.8h, v3.8h\n" + "ldr d3, [x6, #0x90]\n" + "smlal v20.4s, v10.4h, v2.4h\n" + "ssubl v3.8h, v3.8b, v13.8b\n" + "smlal v24.4s, v4.4h, v2.4h\n" + "smlal v23.4s, v17.4h, v2.4h\n" + "smlal2 v15.4s, v10.8h, v27.8h\n" + "smlal v7.4s, v9.4h, v1.4h\n" + "smlal2 v5.4s, v10.8h, v2.8h\n" + "ldr d10, [x9, x3]\n" + "smlal2 v22.4s, v4.8h, v2.8h\n" + "ssubl v10.8h, v10.8b, v18.8b\n" + "smlal2 v19.4s, v17.8h, v2.8h\n" + "ldr d2, [x6, #0x98]\n" + "smlal v20.4s, v9.4h, v27.4h\n" + "ssubl v2.8h, v2.8b, v13.8b\n" + "smlal v24.4s, v17.4h, v27.4h\n" + "smlal v23.4s, v6.4h, v27.4h\n" + "smlal2 v15.4s, v9.8h, v1.8h\n" + "smlal v7.4s, v12.4h, v31.4h\n" + "smlal2 v5.4s, v9.8h, v27.8h\n" + "ldr d9, [x28, x3]\n" + "smlal2 v22.4s, v17.8h, v27.8h\n" + "ssubl v9.8h, v9.8b, v18.8b\n" + "smlal2 v19.4s, v6.8h, v27.8h\n" + "ldr d27, [x6, #0xa0]\n" + "smlal v20.4s, v30.4h, v1.4h\n" + "ssubl v27.8h, v27.8b, v13.8b\n" + "smlal v24.4s, v6.4h, v1.4h\n" + "smlal v23.4s, v28.4h, v1.4h\n" + "smlal2 v15.4s, v12.8h, v31.8h\n" + "ldr d12, [x27, x3]\n" + "smlal v7.4s, v16.4h, v29.4h\n" + "ssubl v12.8h, v12.8b, v18.8b\n" + "smlal2 v5.4s, v30.8h, v1.8h\n" + "ldr d30, [x6, #0xa8]\n" + "smlal2 v22.4s, v6.8h, v1.8h\n" + "ssubl v30.8h, v30.8b, v13.8b\n" "smlal2 v19.4s, v28.8h, v1.8h\n" - "smlal2 v10.4s, v28.8h, v0.8h\n" - "ldr d28, [x24, x0]\n" - "ssubl v28.8h, v28.8b, v9.8b\n" - "smlal2 v7.4s, v22.8h, v0.8h\n" - "smlal v13.4s, v23.4h, v2.4h\n" - "ldr x25, [x4, #0xf0]\n" - "add x5, x5, #0x20\n" - "smlal2 v21.4s, v25.8h, v0.8h\n" - "ldr d0, [x3, #0x50]\n" - "smlal v20.4s, v23.4h, v1.4h\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "smlal v8.4s, v25.4h, v1.4h\n" - "smlal v17.4s, v24.4h, v1.4h\n" - "smlal2 v19.4s, v23.8h, v2.8h\n" - "smlal2 v10.4s, v23.8h, v1.8h\n" - "ldr d23, [x23, x0]\n" - "ssubl v23.8h, v23.8b, v9.8b\n" - "smlal2 v7.4s, v25.8h, v1.8h\n" - "smlal v13.4s, v31.4h, v3.4h\n" - "ldr x24, [x4, #0xf8]\n" - "smlal2 v21.4s, v24.8h, v1.8h\n" - "ldr d1, [x3, #0x58]\n" - "smlal v20.4s, v31.4h, v2.4h\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "smlal v8.4s, v24.4h, v2.4h\n" - "smlal v17.4s, v27.4h, v2.4h\n" - "smlal2 v19.4s, v31.8h, v3.8h\n" - "smlal2 v10.4s, v31.8h, v2.8h\n" - "ldr d31, [x15, x0]\n" - "ssubl v31.8h, v31.8b, v9.8b\n" - "smlal2 v7.4s, v24.8h, v2.8h\n" - "smlal v13.4s, v30.4h, v4.4h\n" - "ldr x23, [x4, #0x100]\n" - "smlal2 v21.4s, v27.8h, v2.8h\n" - "ldr d2, [x3, #0x60]\n" - "smlal v20.4s, v30.4h, v3.4h\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "smlal v8.4s, v27.4h, v3.4h\n" - "smlal v17.4s, v23.4h, v3.4h\n" - "smlal2 v19.4s, v30.8h, v4.8h\n" - "smlal2 v10.4s, v30.8h, v3.8h\n" - "ldr d30, [x21, x0]\n" - "ssubl v30.8h, v30.8b, v9.8b\n" - "smlal2 v7.4s, v27.8h, v3.8h\n" - "smlal v13.4s, v22.4h, v0.4h\n" - "ldr x15, [x4, #0x108]\n" - "smlal2 v21.4s, v23.8h, v3.8h\n" - "ldr d3, [x3, #0x68]\n" - "smlal v20.4s, v26.4h, v4.4h\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "smlal v8.4s, v23.4h, v4.4h\n" - "smlal v17.4s, v28.4h, v4.4h\n" - "smlal2 v19.4s, v22.8h, v0.8h\n" - "ldr d22, [x20, x0]\n" - "smlal2 v10.4s, v26.8h, v4.8h\n" - "ldr d26, [x14, x0]\n" - "smlal2 v7.4s, v23.8h, v4.8h\n" - "smlal v13.4s, v25.4h, v1.4h\n" - "ssubl v26.8h, v26.8b, v9.8b\n" - "ldr x21, [x4, #0x110]\n" - "smlal2 v21.4s, v28.8h, v4.8h\n" - "ldr d4, [x3, #0x70]\n" - "smlal v20.4s, v25.4h, v0.4h\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "smlal v8.4s, v31.4h, v0.4h\n" - "smlal v17.4s, v30.4h, v0.4h\n" - "ssubl v22.8h, v22.8b, v9.8b\n" - "ldr x20, [x4, #0x118]\n" - "smlal2 v19.4s, v25.8h, v1.8h\n" - "smlal2 v10.4s, v25.8h, v0.8h\n" - "ldr d25, [x13, x0]\n" - "ssubl v25.8h, v25.8b, v9.8b\n" - "smlal2 v7.4s, v31.8h, v0.8h\n" - "smlal v13.4s, v24.4h, v2.4h\n" - "ldr x13, [%x[params], %[offsetof_Params_bias]]\n" - "smlal2 v21.4s, v30.8h, v0.8h\n" - "ldr d0, [x3, #0x78]\n" - "smlal v20.4s, v24.4h, v1.4h\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "smlal v8.4s, v30.4h, v1.4h\n" - "smlal v17.4s, v26.4h, v1.4h\n" - "smlal2 v19.4s, v24.8h, v2.8h\n" - "smlal2 v10.4s, v24.8h, v1.8h\n" - "ldr d24, [x12, x0]\n" - "ssubl v24.8h, v24.8b, v9.8b\n" - "smlal2 v7.4s, v30.8h, v1.8h\n" - "smlal v13.4s, v27.4h, v3.4h\n" - "smlal2 v21.4s, v26.8h, v1.8h\n" - "ldr d1, [x3, #0x80]\n" - "smlal v20.4s, v27.4h, v2.4h\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "smlal v8.4s, v26.4h, v2.4h\n" - "smlal v17.4s, v25.4h, v2.4h\n" - "smlal2 v19.4s, v27.8h, v3.8h\n" - "smlal2 v10.4s, v27.8h, v2.8h\n" - "ldr d27, [x11, x0]\n" - "ssubl v27.8h, v27.8b, v9.8b\n" - "smlal2 v7.4s, v26.8h, v2.8h\n" - "smlal v13.4s, v23.4h, v4.4h\n" - "smlal2 v21.4s, v25.8h, v2.8h\n" - "ldr d2, [x3, #0x88]\n" - "smlal v20.4s, v23.4h, v3.4h\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "smlal v8.4s, v25.4h, v3.4h\n" - "smlal v17.4s, v24.4h, v3.4h\n" - "smlal2 v19.4s, v23.8h, v4.8h\n" - "smlal2 v10.4s, v23.8h, v3.8h\n" - "ldr d23, [x22, x0]\n" - "ssubl v23.8h, v23.8b, v9.8b\n" - "smlal2 v7.4s, v25.8h, v3.8h\n" - "smlal v13.4s, v31.4h, v0.4h\n" - "smlal2 v21.4s, v24.8h, v3.8h\n" - "ldr d3, [x3, #0x90]\n" - "smlal v20.4s, v28.4h, v4.4h\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "smlal v8.4s, v24.4h, v4.4h\n" - "smlal v17.4s, v22.4h, v4.4h\n" - "smlal2 v19.4s, v31.8h, v0.8h\n" - "ldr d31, [x9, x0]\n" - "smlal2 v10.4s, v28.8h, v4.8h\n" - "ldr d28, [x27, x0]\n" - "smlal2 v7.4s, v24.8h, v4.8h\n" - "smlal v13.4s, v30.4h, v1.4h\n" - "ssubl v31.8h, v31.8b, v9.8b\n" - "smlal2 v21.4s, v22.8h, v4.8h\n" - "ldr d4, [x3, #0x98]\n" - "smlal v20.4s, v30.4h, v0.4h\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "smlal v8.4s, v27.4h, v0.4h\n" - "smlal v17.4s, v23.4h, v0.4h\n" - "ssubl v28.8h, v28.8b, v9.8b\n" - "smlal2 v19.4s, v30.8h, v1.8h\n" - "smlal2 v10.4s, v30.8h, v0.8h\n" - "ldr d30, [x28, x0]\n" - "ssubl v30.8h, v30.8b, v9.8b\n" - "smlal2 v7.4s, v27.8h, v0.8h\n" - "smlal v13.4s, v26.4h, v2.4h\n" - "smlal2 v21.4s, v23.8h, v0.8h\n" - "ldr d0, [x3, #0xa0]\n" - "smlal v20.4s, v26.4h, v1.4h\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "smlal v8.4s, v23.4h, v1.4h\n" - "smlal v17.4s, v31.4h, v1.4h\n" - "smlal2 v19.4s, v26.8h, v2.8h\n" - "smlal2 v10.4s, v26.8h, v1.8h\n" - "ldr d26, [x26, x0]\n" - "ssubl v26.8h, v26.8b, v9.8b\n" - "smlal2 v7.4s, v23.8h, v1.8h\n" - "smlal v13.4s, v25.4h, v3.4h\n" - "smlal2 v21.4s, v31.8h, v1.8h\n" - "ldr d1, [x3, #0xa8]\n" - "smlal v20.4s, v25.4h, v2.4h\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "smlal v8.4s, v31.4h, v2.4h\n" - "smlal v17.4s, v30.4h, v2.4h\n" - "smlal2 v19.4s, v25.8h, v3.8h\n" - "smlal2 v10.4s, v25.8h, v2.8h\n" - "ldr d25, [x25, x0]\n" - "ssubl v25.8h, v25.8b, v9.8b\n" - "smlal2 v7.4s, v31.8h, v2.8h\n" - "smlal v13.4s, v24.4h, v4.4h\n" - "smlal2 v21.4s, v30.8h, v2.8h\n" - "ldr d2, [x3, #0xb0]\n" - "smlal v20.4s, v24.4h, v3.4h\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "smlal v8.4s, v30.4h, v3.4h\n" - "smlal v17.4s, v28.4h, v3.4h\n" - "smlal2 v19.4s, v24.8h, v4.8h\n" - "smlal2 v10.4s, v24.8h, v3.8h\n" - "ldr d24, [x24, x0]\n" - "ssubl v24.8h, v24.8b, v9.8b\n" - "smlal2 v7.4s, v30.8h, v3.8h\n" - "smlal v13.4s, v27.4h, v0.4h\n" - "smlal2 v21.4s, v28.8h, v3.8h\n" - "ldr d3, [x3, #0xb8]\n" - "smlal v20.4s, v22.4h, v4.4h\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "smlal v8.4s, v28.4h, v4.4h\n" - "smlal v17.4s, v26.4h, v4.4h\n" - "smlal2 v19.4s, v27.8h, v0.8h\n" - "ldr d27, [x23, x0]\n" - "smlal2 v7.4s, v28.8h, v4.8h\n" - "ssubl v27.8h, v27.8b, v9.8b\n" - "smlal v13.4s, v23.4h, v1.4h\n" - "smlal2 v10.4s, v22.8h, v4.8h\n" - "ldr q22, [x8, #0x10]\n" + "ldr d1, [x26, x3]\n" + "smlal v20.4s, v16.4h, v31.4h\n" + "ssubl v1.8h, v1.8b, v18.8b\n" + "smlal v24.4s, v8.4h, v31.4h\n" + "smlal v23.4s, v14.4h, v31.4h\n" + "smlal2 v15.4s, v16.8h, v29.8h\n" + "smlal v7.4s, v4.4h, v21.4h\n" + "smlal2 v5.4s, v16.8h, v31.8h\n" + "ldr d16, [x25, x3]\n" + "smlal2 v22.4s, v8.8h, v31.8h\n" + "ssubl v16.8h, v16.8b, v18.8b\n" + "smlal2 v19.4s, v14.8h, v31.8h\n" + "ldr d31, [x6, #0xb0]\n" + "smlal v20.4s, v4.4h, v29.4h\n" + "ssubl v31.8h, v31.8b, v13.8b\n" + "smlal v24.4s, v14.4h, v29.4h\n" + "smlal v23.4s, v25.4h, v29.4h\n" + "smlal2 v15.4s, v4.8h, v21.8h\n" + "smlal v7.4s, v17.4h, v3.4h\n" + "smlal2 v5.4s, v4.8h, v29.8h\n" + "ldr d4, [x24, x3]\n" + "smlal2 v22.4s, v14.8h, v29.8h\n" + "ssubl v4.8h, v4.8b, v18.8b\n" + "smlal2 v19.4s, v25.8h, v29.8h\n" + "ldr d29, [x6, #0xb8]\n" + "smlal v20.4s, v17.4h, v21.4h\n" + "ssubl v29.8h, v29.8b, v13.8b\n" + "smlal v24.4s, v25.4h, v21.4h\n" + "smlal v23.4s, v10.4h, v21.4h\n" + "smlal2 v15.4s, v17.8h, v3.8h\n" + "smlal v7.4s, v6.4h, v2.4h\n" + "smlal2 v5.4s, v17.8h, v21.8h\n" + "ldr d17, [x23, x3]\n" + "smlal2 v22.4s, v25.8h, v21.8h\n" + "ssubl v17.8h, v17.8b, v18.8b\n" + "smlal2 v19.4s, v10.8h, v21.8h\n" + "ldr d21, [x6, #0xc0]\n" + "smlal v20.4s, v6.4h, v3.4h\n" + "ssubl v21.8h, v21.8b, v13.8b\n" + "smlal v24.4s, v10.4h, v3.4h\n" + "smlal v23.4s, v9.4h, v3.4h\n" + "add x6, x6, #0xc8\n" + "smlal2 v15.4s, v6.8h, v2.8h\n" + "smlal v7.4s, v8.4h, v27.4h\n" + "smlal2 v5.4s, v6.8h, v3.8h\n" + "ldr d6, [x22, x3]\n" + "smlal2 v22.4s, v10.8h, v3.8h\n" + "ssubl v6.8h, v6.8b, v18.8b\n" + "smlal2 v19.4s, v9.8h, v3.8h\n" + "ldr d3, [x21, x3]\n" + "smlal v20.4s, v28.4h, v2.4h\n" + "ssubl v3.8h, v3.8b, v18.8b\n" + "smlal v24.4s, v9.4h, v2.4h\n" + "smlal v23.4s, v12.4h, v2.4h\n" + "add x3, x3, #0x8\n" + "smlal2 v15.4s, v8.8h, v27.8h\n" + "ldr q8, [x7, #0x0]\n" + "smlal v7.4s, v14.4h, v30.4h\n" + "smlal2 v5.4s, v28.8h, v2.8h\n" + "ldr q28, [x8, #0x0]\n" + "smlal2 v22.4s, v9.8h, v2.8h\n" + "smlal2 v19.4s, v12.8h, v2.8h\n" + "ldr q2, [x7, #0x10]\n" + "smlal v20.4s, v14.4h, v27.4h\n" + "add x7, x7, #0x20\n" + "smlal v24.4s, v1.4h, v27.4h\n" + "smlal v23.4s, v16.4h, v27.4h\n" + "smlal2 v15.4s, v14.8h, v30.8h\n" + "smlal v7.4s, v25.4h, v31.4h\n" + "smlal2 v5.4s, v14.8h, v27.8h\n" + "ldr q14, [x8, #0x10]\n" + "smlal2 v22.4s, v1.8h, v27.8h\n" "add x8, x8, #0x20\n" - "smlal2 v21.4s, v26.8h, v4.8h\n" - "ldr d4, [x3, #0xc0]\n" - "smlal v20.4s, v23.4h, v0.4h\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "smlal v8.4s, v25.4h, v0.4h\n" - "smlal v17.4s, v24.4h, v0.4h\n" - "add x3, x3, #0xc8\n" - "smlal2 v19.4s, v23.8h, v1.8h\n" - "smlal2 v7.4s, v25.8h, v0.8h\n" - "ldr d25, [x15, x0]\n" - "ssubl v25.8h, v25.8b, v9.8b\n" - "smlal v13.4s, v31.4h, v2.4h\n" - "smlal2 v10.4s, v23.8h, v0.8h\n" - "smlal2 v21.4s, v24.8h, v0.8h\n" - "smlal v20.4s, v31.4h, v1.4h\n" - "smlal v8.4s, v24.4h, v1.4h\n" - "smlal v17.4s, v27.4h, v1.4h\n" - "smlal2 v19.4s, v31.8h, v2.8h\n" - "smlal2 v7.4s, v24.8h, v1.8h\n" - "ldr d24, [x21, x0]\n" - "ssubl v24.8h, v24.8b, v9.8b\n" - "smlal v13.4s, v30.4h, v3.4h\n" - "smlal2 v10.4s, v31.8h, v1.8h\n" - "smlal2 v21.4s, v27.8h, v1.8h\n" - "smlal v20.4s, v30.4h, v2.4h\n" - "smlal v8.4s, v27.4h, v2.4h\n" - "smlal v17.4s, v25.4h, v2.4h\n" - "smlal2 v19.4s, v30.8h, v3.8h\n" - "smlal2 v7.4s, v27.8h, v2.8h\n" - "ldr d27, [x20, x0]\n" - "ssubl v27.8h, v27.8b, v9.8b\n" - "smlal v13.4s, v28.4h, v4.4h\n" - "smlal2 v10.4s, v30.8h, v2.8h\n" - "sqrdmulh v13.4s, v13.4s, v18.4s\n" - "add x0, x0, #0x8\n" - "smlal2 v21.4s, v25.8h, v2.8h\n" - "smlal v20.4s, v28.4h, v3.4h\n" - "and v30.16b, v13.16b, v6.16b\n" - "smlal v8.4s, v25.4h, v3.4h\n" - "smlal v17.4s, v24.4h, v3.4h\n" - "sshr v30.4s, v30.4s, #0x1f\n" - "smlal2 v19.4s, v28.8h, v4.8h\n" - "smlal2 v10.4s, v28.8h, v3.8h\n" - "sqrdmulh v19.4s, v19.4s, v5.4s\n" - "smlal2 v7.4s, v25.8h, v3.8h\n" - "smlal2 v21.4s, v24.8h, v3.8h\n" - "and v16.16b, v19.16b, v22.16b\n" - "smlal v20.4s, v26.4h, v4.4h\n" - "smlal v8.4s, v24.4h, v4.4h\n" - "sqrdmulh v20.4s, v20.4s, v18.4s\n" - "smlal v17.4s, v27.4h, v4.4h\n" - "smlal2 v10.4s, v26.8h, v4.8h\n" - "sqrdmulh v8.4s, v8.4s, v18.4s\n" - "smlal2 v7.4s, v24.8h, v4.8h\n" - "smlal2 v21.4s, v27.8h, v4.8h\n" - "sqrdmulh v17.4s, v17.4s, v18.4s\n" - "sqadd v13.4s, v13.4s, v30.4s\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "and v0.16b, v20.16b, v6.16b\n" - "sqrdmulh v10.4s, v10.4s, v5.4s\n" - "and v18.16b, v8.16b, v6.16b\n" - "sqrdmulh v7.4s, v7.4s, v5.4s\n" - "and v30.16b, v17.16b, v6.16b\n" - "sqrdmulh v21.4s, v21.4s, v5.4s\n" - "sqadd v19.4s, v19.4s, v16.4s\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v26.16b, v10.16b, v22.16b\n" - "sshr v18.4s, v18.4s, #0x1f\n" - "and v23.16b, v7.16b, v22.16b\n" - "sshr v30.4s, v30.4s, #0x1f\n" - "and v16.16b, v21.16b, v22.16b\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "sshr v26.4s, v26.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v18.4s\n" - "sshr v23.4s, v23.4s, #0x1f\n" - "sqadd v17.4s, v17.4s, v30.4s\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "srshl v13.4s, v13.4s, v6.4s\n" - "srshl v20.4s, v20.4s, v6.4s\n" - "sqadd v10.4s, v10.4s, v26.4s\n" - "srshl v8.4s, v8.4s, v6.4s\n" - "sqadd v7.4s, v7.4s, v23.4s\n" - "srshl v17.4s, v17.4s, v6.4s\n" - "sqadd v21.4s, v21.4s, v16.4s\n" - "srshl v19.4s, v19.4s, v22.4s\n" - "sqxtn v13.4h, v13.4s\n" - "srshl v10.4s, v10.4s, v22.4s\n" + "smlal2 v19.4s, v16.8h, v27.8h\n" + "smlal v20.4s, v25.4h, v30.4h\n" + "smlal v24.4s, v16.4h, v30.4h\n" + "smlal v23.4s, v4.4h, v30.4h\n" + "smlal2 v15.4s, v25.8h, v31.8h\n" + "smlal v7.4s, v10.4h, v29.4h\n" + "smlal2 v5.4s, v25.8h, v30.8h\n" + "smlal2 v22.4s, v16.8h, v30.8h\n" + "smlal2 v19.4s, v4.8h, v30.8h\n" + "smlal v20.4s, v10.4h, v31.4h\n" + "smlal v24.4s, v4.4h, v31.4h\n" + "smlal v23.4s, v17.4h, v31.4h\n" + "smlal2 v15.4s, v10.8h, v29.8h\n" + "smlal v7.4s, v9.4h, v21.4h\n" + "sqrdmulh v7.4s, v7.4s, v8.4s\n" + "smlal2 v5.4s, v10.8h, v31.8h\n" + "smlal2 v22.4s, v4.8h, v31.8h\n" + "and v27.16b, v7.16b, v28.16b\n" + "smlal2 v19.4s, v17.8h, v31.8h\n" + "smlal v20.4s, v9.4h, v29.4h\n" + "sshr v27.4s, v27.4s, #0x1f\n" + "smlal v24.4s, v17.4h, v29.4h\n" + "smlal v23.4s, v6.4h, v29.4h\n" + "sqadd v7.4s, v7.4s, v27.4s\n" + "smlal2 v15.4s, v9.8h, v21.8h\n" + "smlal2 v5.4s, v9.8h, v29.8h\n" + "sqrdmulh v15.4s, v15.4s, v2.4s\n" + "smlal2 v22.4s, v17.8h, v29.8h\n" + "smlal2 v19.4s, v6.8h, v29.8h\n" + "and v9.16b, v15.16b, v14.16b\n" + "smlal v20.4s, v12.4h, v21.4h\n" + "smlal v24.4s, v6.4h, v21.4h\n" + "sqrdmulh v20.4s, v20.4s, v8.4s\n" + "smlal v23.4s, v3.4h, v21.4h\n" + "smlal2 v5.4s, v12.8h, v21.8h\n" + "sqrdmulh v24.4s, v24.4s, v8.4s\n" + "smlal2 v22.4s, v6.8h, v21.8h\n" + "smlal2 v19.4s, v3.8h, v21.8h\n" + "sqrdmulh v23.4s, v23.4s, v8.4s\n" + "sshr v9.4s, v9.4s, #0x1f\n" + "and v25.16b, v20.16b, v28.16b\n" + "sqrdmulh v5.4s, v5.4s, v2.4s\n" + "and v10.16b, v24.16b, v28.16b\n" + "sqrdmulh v22.4s, v22.4s, v2.4s\n" + "and v21.16b, v23.16b, v28.16b\n" + "sqrdmulh v19.4s, v19.4s, v2.4s\n" + "sqadd v15.4s, v15.4s, v9.4s\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "and v9.16b, v5.16b, v14.16b\n" + "sshr v10.4s, v10.4s, #0x1f\n" + "and v12.16b, v22.16b, v14.16b\n" + "sshr v21.4s, v21.4s, #0x1f\n" + "and v17.16b, v19.16b, v14.16b\n" + "sqadd v20.4s, v20.4s, v25.4s\n" + "sshr v9.4s, v9.4s, #0x1f\n" + "sqadd v24.4s, v24.4s, v10.4s\n" + "sshr v12.4s, v12.4s, #0x1f\n" + "sqadd v23.4s, v23.4s, v21.4s\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "srshl v7.4s, v7.4s, v28.4s\n" + "srshl v20.4s, v20.4s, v28.4s\n" + "sqadd v5.4s, v5.4s, v9.4s\n" + "srshl v24.4s, v24.4s, v28.4s\n" + "sqadd v22.4s, v22.4s, v12.4s\n" + "srshl v23.4s, v23.4s, v28.4s\n" + "sqadd v19.4s, v19.4s, v17.4s\n" + "srshl v15.4s, v15.4s, v14.4s\n" + "sqxtn v7.4h, v7.4s\n" + "srshl v5.4s, v5.4s, v14.4s\n" "sqxtn v20.4h, v20.4s\n" - "srshl v7.4s, v7.4s, v22.4s\n" - "sqxtn v8.4h, v8.4s\n" - "srshl v21.4s, v21.4s, v22.4s\n" - "sqxtn v17.4h, v17.4s\n" - "sqxtn2 v13.8h, v19.4s\n" - "sqxtn2 v20.8h, v10.4s\n" - "sqxtn2 v8.8h, v7.4s\n" - "sqxtn2 v17.8h, v21.4s\n" - "sqadd v13.8h, v13.8h, v14.8h\n" - "sqadd v20.8h, v20.8h, v14.8h\n" - "sqadd v8.8h, v8.8h, v14.8h\n" - "sqadd v17.8h, v17.8h, v14.8h\n" - "smax v13.8h, v13.8h, v12.8h\n" - "smax v20.8h, v20.8h, v12.8h\n" - "smax v8.8h, v8.8h, v12.8h\n" - "smax v17.8h, v17.8h, v12.8h\n" - "smin v13.8h, v13.8h, v11.8h\n" - "smin v20.8h, v20.8h, v11.8h\n" - "smin v8.8h, v8.8h, v11.8h\n" - "smin v17.8h, v17.8h, v11.8h\n" - "uzp1 v13.16b, v13.16b, v13.16b\n" + "srshl v22.4s, v22.4s, v14.4s\n" + "sqxtn v24.4h, v24.4s\n" + "srshl v19.4s, v19.4s, v14.4s\n" + "sqxtn v23.4h, v23.4s\n" + "sqxtn2 v7.8h, v15.4s\n" + "sqxtn2 v20.8h, v5.4s\n" + "sqxtn2 v24.8h, v22.4s\n" + "sqxtn2 v23.8h, v19.4s\n" + "sqadd v7.8h, v7.8h, v26.8h\n" + "sqadd v20.8h, v20.8h, v26.8h\n" + "sqadd v24.8h, v24.8h, v26.8h\n" + "sqadd v23.8h, v23.8h, v26.8h\n" + "smax v7.8h, v7.8h, v11.8h\n" + "smax v20.8h, v20.8h, v11.8h\n" + "smax v24.8h, v24.8h, v11.8h\n" + "smax v23.8h, v23.8h, v11.8h\n" + "smin v7.8h, v7.8h, v0.8h\n" + "smin v20.8h, v20.8h, v0.8h\n" + "smin v24.8h, v24.8h, v0.8h\n" + "smin v23.8h, v23.8h, v0.8h\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" + "str d7, [x17, x4]\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "str d13, [x17, x10]\n" - "uzp1 v8.16b, v8.16b, v8.16b\n" - "uzp1 v17.16b, v17.16b, v17.16b\n" - "str d20, [x6, x10]\n" - "str d8, [x7, x10]\n" - "str d17, [x16, x10]\n" - "ldr q13, [x13, #0x0]\n" - "ldr q19, [x13, #0x10]\n" - "add x13, x13, #0x20\n" - "ldr d0, [x3, #0x0]\n" - "ldr d1, [x3, #0x8]\n" - "add x10, x10, #0x8\n" - "str x13, [%x[params], %[offsetof_Params_bias]]\n" - "ldr d2, [x3, #0x10]\n" - "ldr d3, [x3, #0x18]\n" - "mov v20.16b, v13.16b\n" - "mov v10.16b, v19.16b\n" - "ldr d4, [x3, #0x20]\n" - "ldp x9, x28, [x4, #0x0]\n" - "mov v8.16b, v13.16b\n" - "mov v7.16b, v19.16b\n" - "ldp x27, x26, [x4, #0x10]\n" - "ldp x25, x24, [x4, #0x20]\n" - "mov v17.16b, v13.16b\n" - "mov v21.16b, v19.16b\n" - "ldp x23, x22, [x4, #0x30]\n" - "ldp x21, x20, [x4, #0x40]\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "ldr d31, [x9, x0]\n" - "ldr d30, [x28, x0]\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "ldr d29, [x27, x0]\n" - "ldr d28, [x26, x0]\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "ssubl v31.8h, v31.8b, v9.8b\n" - "ldr d27, [x25, x0]\n" - "ldr d23, [x24, x0]\n" - "ssubl v30.8h, v30.8b, v9.8b\n" - "ssubl v29.8h, v29.8b, v9.8b\n" - "ldr d25, [x23, x0]\n" - "ldr d24, [x22, x0]\n" - "ssubl v28.8h, v28.8b, v9.8b\n" - "ssubl v27.8h, v27.8b, v9.8b\n" - "ldr d26, [x21, x0]\n" - "ldr d22, [x20, x0]\n" - "ssubl v23.8h, v23.8b, v9.8b\n" - "ssubl v25.8h, v25.8b, v9.8b\n" - "ssubl v24.8h, v24.8b, v9.8b\n" - "ssubl v26.8h, v26.8b, v9.8b\n" - "ssubl v22.8h, v22.8b, v9.8b\n" + "uzp1 v24.16b, v24.16b, v24.16b\n" + "str d20, [x16, x4]\n" + "uzp1 v23.16b, v23.16b, v23.16b\n" + "str d24, [x15, x4]\n" + "str d23, [x14, x4]\n" + "ldr q7, [x20, #0x0]\n" + "ldr q15, [x20, #0x10]\n" + "add x20, x20, #0x20\n" + "ldr d6, [x6, #0x0]\n" + "ldr d14, [x6, #0x8]\n" + "add x4, x4, #0x8\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" + "ldr d10, [x6, #0x10]\n" + "ldr d21, [x6, #0x18]\n" + "mov v20.16b, v7.16b\n" + "mov v5.16b, v15.16b\n" + "ldr d12, [x6, #0x20]\n" + "ldp x9, x28, [x5, #0x0]\n" + "mov v24.16b, v7.16b\n" + "mov v22.16b, v15.16b\n" + "ldp x27, x26, [x5, #0x10]\n" + "ldp x25, x24, [x5, #0x20]\n" + "mov v23.16b, v7.16b\n" + "mov v19.16b, v15.16b\n" + "ldp x23, x22, [x5, #0x30]\n" + "ldp x21, x20, [x5, #0x40]\n" + "ssubl v6.8h, v6.8b, v13.8b\n" + "ssubl v14.8h, v14.8b, v13.8b\n" + "ldr d31, [x9, x3]\n" + "ldr d17, [x28, x3]\n" + "ssubl v10.8h, v10.8b, v13.8b\n" + "ssubl v21.8h, v21.8b, v13.8b\n" + "ldr d30, [x27, x3]\n" + "ldr d16, [x26, x3]\n" + "ssubl v12.8h, v12.8b, v13.8b\n" + "ssubl v31.8h, v31.8b, v18.8b\n" + "ldr d3, [x25, x3]\n" + "ldr d4, [x24, x3]\n" + "ssubl v17.8h, v17.8b, v18.8b\n" + "ssubl v30.8h, v30.8b, v18.8b\n" + "ldr d25, [x23, x3]\n" + "ldr d9, [x22, x3]\n" + "ssubl v16.8h, v16.8b, v18.8b\n" + "ssubl v3.8h, v3.8b, v18.8b\n" + "ldr d29, [x21, x3]\n" + "ldr d28, [x20, x3]\n" + "ssubl v4.8h, v4.8b, v18.8b\n" + "ssubl v25.8h, v25.8b, v18.8b\n" + "ssubl v9.8h, v9.8b, v18.8b\n" + "ssubl v29.8h, v29.8b, v18.8b\n" + "ssubl v28.8h, v28.8b, v18.8b\n" "bgt 1b\n" "2:" // Tail - "ldr q18, [x5, #0x0]\n" - "ldr q6, [x8, #0x0]\n" - "smlal v13.4s, v31.4h, v0.4h\n" - "smlal2 v19.4s, v31.8h, v0.8h\n" - "ldr q5, [x5, #0x10]\n" - "smlal v13.4s, v30.4h, v1.4h\n" - "ldr x20, [x4, #0x50]\n" - "smlal v20.4s, v30.4h, v0.4h\n" - "smlal v8.4s, v29.4h, v0.4h\n" - "smlal v17.4s, v28.4h, v0.4h\n" - "ldr x22, [x4, #0x58]\n" - "ldr x21, [x4, #0x60]\n" - "smlal2 v19.4s, v30.8h, v1.8h\n" - "smlal2 v10.4s, v30.8h, v0.8h\n" - "ldr d31, [x20, x0]\n" - "ssubl v31.8h, v31.8b, v9.8b\n" - "smlal2 v7.4s, v29.8h, v0.8h\n" - "smlal v13.4s, v27.4h, v2.4h\n" - "ldr x20, [x4, #0x68]\n" - "ldr x26, [x4, #0x70]\n" - "smlal2 v21.4s, v28.8h, v0.8h\n" - "ldr d30, [x22, x0]\n" - "smlal v20.4s, v27.4h, v1.4h\n" - "ssubl v30.8h, v30.8b, v9.8b\n" - "smlal v8.4s, v28.4h, v1.4h\n" - "smlal v17.4s, v23.4h, v1.4h\n" - "ldr x25, [x4, #0x78]\n" - "ldr x23, [x4, #0x80]\n" - "smlal2 v19.4s, v27.8h, v2.8h\n" - "smlal2 v10.4s, v27.8h, v1.8h\n" - "ldr d0, [x3, #0x28]\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "smlal2 v7.4s, v28.8h, v1.8h\n" - "smlal v13.4s, v25.4h, v3.4h\n" - "ldr x24, [x4, #0x88]\n" - "ldr x15, [x4, #0x90]\n" - "smlal2 v21.4s, v23.8h, v1.8h\n" - "ldr d27, [x21, x0]\n" - "smlal v20.4s, v25.4h, v2.4h\n" - "ssubl v27.8h, v27.8b, v9.8b\n" - "smlal v8.4s, v23.4h, v2.4h\n" - "smlal v17.4s, v31.4h, v2.4h\n" - "ldr x21, [x4, #0x98]\n" - "ldr x14, [x4, #0xa0]\n" - "smlal2 v19.4s, v25.8h, v3.8h\n" - "smlal2 v10.4s, v25.8h, v2.8h\n" - "ldr d1, [x3, #0x30]\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "smlal2 v7.4s, v23.8h, v2.8h\n" - "smlal v13.4s, v24.4h, v4.4h\n" - "ldr x13, [x4, #0xa8]\n" - "ldr x12, [x4, #0xb0]\n" - "smlal2 v21.4s, v31.8h, v2.8h\n" - "ldr d25, [x20, x0]\n" - "smlal v20.4s, v24.4h, v3.4h\n" - "ssubl v25.8h, v25.8b, v9.8b\n" - "smlal v8.4s, v31.4h, v3.4h\n" - "smlal v17.4s, v30.4h, v3.4h\n" - "ldr x20, [x4, #0xb8]\n" - "ldr x11, [x4, #0xc0]\n" - "smlal2 v19.4s, v24.8h, v4.8h\n" - "smlal2 v10.4s, v24.8h, v3.8h\n" - "ldr d2, [x3, #0x38]\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "smlal2 v7.4s, v31.8h, v3.8h\n" - "smlal v13.4s, v29.4h, v0.4h\n" - "ldr x22, [x4, #0xc8]\n" - "ldr x9, [x4, #0xd0]\n" - "smlal2 v21.4s, v30.8h, v3.8h\n" - "ldr d24, [x26, x0]\n" - "smlal v20.4s, v27.4h, v4.4h\n" - "ssubl v24.8h, v24.8b, v9.8b\n" - "smlal v8.4s, v30.4h, v4.4h\n" - "smlal v17.4s, v26.4h, v4.4h\n" - "ldr x28, [x4, #0xd8]\n" - "ldr x27, [x4, #0xe0]\n" - "smlal2 v19.4s, v29.8h, v0.8h\n" - "ldr d3, [x3, #0x40]\n" - "smlal2 v10.4s, v27.8h, v4.8h\n" - "ldr d27, [x25, x0]\n" - "smlal2 v7.4s, v30.8h, v4.8h\n" - "smlal v13.4s, v28.4h, v1.4h\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "ldr x26, [x4, #0xe8]\n" - "smlal2 v21.4s, v26.8h, v4.8h\n" - "ldr d4, [x3, #0x48]\n" - "smlal v20.4s, v28.4h, v0.4h\n" - "ssubl v27.8h, v27.8b, v9.8b\n" - "smlal v8.4s, v22.4h, v0.4h\n" - "smlal v17.4s, v25.4h, v0.4h\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "ldr x25, [x4, #0xf0]\n" - "smlal2 v19.4s, v28.8h, v1.8h\n" - "smlal2 v10.4s, v28.8h, v0.8h\n" - "ldr d28, [x24, x0]\n" - "ssubl v28.8h, v28.8b, v9.8b\n" - "smlal2 v7.4s, v22.8h, v0.8h\n" - "smlal v13.4s, v23.4h, v2.4h\n" - "ldr x24, [x4, #0xf8]\n" - "tst x1, #0x7\n" - "smlal2 v21.4s, v25.8h, v0.8h\n" - "ldr d0, [x3, #0x50]\n" - "smlal v20.4s, v23.4h, v1.4h\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "smlal v8.4s, v25.4h, v1.4h\n" - "smlal v17.4s, v24.4h, v1.4h\n" - "add x5, x5, #0x20\n" - "smlal2 v19.4s, v23.8h, v2.8h\n" - "smlal2 v10.4s, v23.8h, v1.8h\n" - "ldr d23, [x23, x0]\n" - "ssubl v23.8h, v23.8b, v9.8b\n" - "smlal2 v7.4s, v25.8h, v1.8h\n" - "smlal v13.4s, v31.4h, v3.4h\n" - "ldr x23, [x4, #0x100]\n" - "smlal2 v21.4s, v24.8h, v1.8h\n" - "ldr d1, [x3, #0x58]\n" - "smlal v20.4s, v31.4h, v2.4h\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "smlal v8.4s, v24.4h, v2.4h\n" - "smlal v17.4s, v27.4h, v2.4h\n" - "smlal2 v19.4s, v31.8h, v3.8h\n" - "smlal2 v10.4s, v31.8h, v2.8h\n" - "ldr d31, [x15, x0]\n" - "ssubl v31.8h, v31.8b, v9.8b\n" - "smlal2 v7.4s, v24.8h, v2.8h\n" - "smlal v13.4s, v30.4h, v4.4h\n" - "ldr x15, [x4, #0x108]\n" - "smlal2 v21.4s, v27.8h, v2.8h\n" - "ldr d2, [x3, #0x60]\n" - "smlal v20.4s, v30.4h, v3.4h\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "smlal v8.4s, v27.4h, v3.4h\n" - "smlal v17.4s, v23.4h, v3.4h\n" - "smlal2 v19.4s, v30.8h, v4.8h\n" - "smlal2 v10.4s, v30.8h, v3.8h\n" - "ldr d30, [x21, x0]\n" - "ssubl v30.8h, v30.8b, v9.8b\n" - "smlal2 v7.4s, v27.8h, v3.8h\n" - "smlal v13.4s, v22.4h, v0.4h\n" - "ldr x21, [x4, #0x110]\n" - "smlal2 v21.4s, v23.8h, v3.8h\n" - "ldr d3, [x3, #0x68]\n" - "smlal v20.4s, v26.4h, v4.4h\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "smlal v8.4s, v23.4h, v4.4h\n" - "smlal v17.4s, v28.4h, v4.4h\n" - "smlal2 v19.4s, v22.8h, v0.8h\n" - "ldr d22, [x20, x0]\n" - "smlal2 v10.4s, v26.8h, v4.8h\n" - "ldr d26, [x14, x0]\n" - "smlal2 v7.4s, v23.8h, v4.8h\n" - "smlal v13.4s, v25.4h, v1.4h\n" - "ssubl v26.8h, v26.8b, v9.8b\n" - "ldr x20, [x4, #0x118]\n" - "smlal2 v21.4s, v28.8h, v4.8h\n" - "ldr d4, [x3, #0x70]\n" - "smlal v20.4s, v25.4h, v0.4h\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "smlal v8.4s, v31.4h, v0.4h\n" - "smlal v17.4s, v30.4h, v0.4h\n" - "ssubl v22.8h, v22.8b, v9.8b\n" + "ldr d27, [x6, #0x28]\n" + "ldr d1, [x6, #0x30]\n" + "smlal v7.4s, v31.4h, v6.4h\n" + "smlal2 v15.4s, v31.8h, v6.8h\n" + "ldr d2, [x6, #0x38]\n" + "ldr d31, [x6, #0x40]\n" + "smlal v7.4s, v17.4h, v14.4h\n" + "smlal v20.4s, v17.4h, v6.4h\n" + "ldr d8, [x6, #0x48]\n" + "ldr x22, [x5, #0x50]\n" + "smlal v24.4s, v30.4h, v6.4h\n" + "smlal v23.4s, v16.4h, v6.4h\n" + "smlal2 v15.4s, v17.8h, v14.8h\n" + "smlal v7.4s, v3.4h, v10.4h\n" + "ldr x20, [x5, #0x58]\n" + "ldr x21, [x5, #0x60]\n" + "smlal2 v5.4s, v17.8h, v6.8h\n" + "ldr d17, [x22, x3]\n" + "smlal2 v22.4s, v30.8h, v6.8h\n" + "ssubl v17.8h, v17.8b, v18.8b\n" + "smlal2 v19.4s, v16.8h, v6.8h\n" + "ldr d6, [x20, x3]\n" + "smlal v20.4s, v3.4h, v14.4h\n" + "ssubl v6.8h, v6.8b, v18.8b\n" + "smlal v24.4s, v16.4h, v14.4h\n" + "smlal v23.4s, v4.4h, v14.4h\n" + "ssubl v27.8h, v27.8b, v13.8b\n" + "ldr x20, [x5, #0x68]\n" + "smlal2 v15.4s, v3.8h, v10.8h\n" + "smlal v7.4s, v25.4h, v21.4h\n" + "ssubl v1.8h, v1.8b, v13.8b\n" + "ldr x22, [x5, #0x70]\n" + "smlal2 v5.4s, v3.8h, v14.8h\n" + "ldr d3, [x21, x3]\n" + "smlal2 v22.4s, v16.8h, v14.8h\n" + "ssubl v3.8h, v3.8b, v18.8b\n" + "smlal2 v19.4s, v4.8h, v14.8h\n" + "ldr d14, [x20, x3]\n" + "smlal v20.4s, v25.4h, v10.4h\n" + "ssubl v14.8h, v14.8b, v18.8b\n" + "smlal v24.4s, v4.4h, v10.4h\n" + "smlal v23.4s, v17.4h, v10.4h\n" + "ssubl v2.8h, v2.8b, v13.8b\n" + "ldr x21, [x5, #0x78]\n" + "smlal2 v15.4s, v25.8h, v21.8h\n" + "smlal v7.4s, v9.4h, v12.4h\n" + "ssubl v31.8h, v31.8b, v13.8b\n" + "ldr x20, [x5, #0x80]\n" + "smlal2 v5.4s, v25.8h, v10.8h\n" + "ldr d25, [x22, x3]\n" + "smlal2 v22.4s, v4.8h, v10.8h\n" + "ssubl v25.8h, v25.8b, v18.8b\n" + "smlal2 v19.4s, v17.8h, v10.8h\n" + "ldr d10, [x21, x3]\n" + "smlal v20.4s, v9.4h, v21.4h\n" + "ssubl v10.8h, v10.8b, v18.8b\n" + "smlal v24.4s, v17.4h, v21.4h\n" + "smlal v23.4s, v6.4h, v21.4h\n" + "ssubl v8.8h, v8.8b, v13.8b\n" + "ldr x24, [x5, #0x88]\n" + "smlal2 v15.4s, v9.8h, v12.8h\n" + "smlal v7.4s, v30.4h, v27.4h\n" + "ldr x23, [x5, #0x90]\n" + "ldr x22, [x5, #0x98]\n" + "smlal2 v5.4s, v9.8h, v21.8h\n" + "ldr d9, [x20, x3]\n" + "smlal2 v22.4s, v17.8h, v21.8h\n" + "ssubl v9.8h, v9.8b, v18.8b\n" + "smlal2 v19.4s, v6.8h, v21.8h\n" + "ldr d21, [x6, #0x50]\n" + "smlal v20.4s, v3.4h, v12.4h\n" + "ssubl v21.8h, v21.8b, v13.8b\n" + "smlal v24.4s, v6.4h, v12.4h\n" + "smlal v23.4s, v29.4h, v12.4h\n" + "ldr x21, [x5, #0xa0]\n" + "ldr x20, [x5, #0xa8]\n" + "smlal2 v15.4s, v30.8h, v27.8h\n" + "ldr d30, [x24, x3]\n" + "smlal v7.4s, v16.4h, v1.4h\n" + "ssubl v30.8h, v30.8b, v18.8b\n" + "smlal2 v5.4s, v3.8h, v12.8h\n" + "ldr d3, [x6, #0x58]\n" + "smlal2 v22.4s, v6.8h, v12.8h\n" + "ssubl v3.8h, v3.8b, v13.8b\n" + "smlal2 v19.4s, v29.8h, v12.8h\n" + "ldr d12, [x23, x3]\n" + "smlal v20.4s, v16.4h, v27.4h\n" + "ssubl v12.8h, v12.8b, v18.8b\n" + "smlal v24.4s, v28.4h, v27.4h\n" + "smlal v23.4s, v14.4h, v27.4h\n" + "ldr x13, [x5, #0xb0]\n" + "ldr x12, [x5, #0xb8]\n" + "smlal2 v15.4s, v16.8h, v1.8h\n" + "smlal v7.4s, v4.4h, v2.4h\n" + "ldr x11, [x5, #0xc0]\n" + "ldr x10, [x5, #0xc8]\n" + "smlal2 v5.4s, v16.8h, v27.8h\n" + "ldr d16, [x22, x3]\n" + "smlal2 v22.4s, v28.8h, v27.8h\n" + "ssubl v16.8h, v16.8b, v18.8b\n" + "smlal2 v19.4s, v14.8h, v27.8h\n" + "ldr d27, [x6, #0x60]\n" + "smlal v20.4s, v4.4h, v1.4h\n" + "ssubl v27.8h, v27.8b, v13.8b\n" + "smlal v24.4s, v14.4h, v1.4h\n" + "smlal v23.4s, v25.4h, v1.4h\n" + "ldr x9, [x5, #0xd0]\n" + "ldr x28, [x5, #0xd8]\n" + "smlal2 v15.4s, v4.8h, v2.8h\n" + "smlal v7.4s, v17.4h, v31.4h\n" + "ldr x27, [x5, #0xe0]\n" + "ldr x26, [x5, #0xe8]\n" + "smlal2 v5.4s, v4.8h, v1.8h\n" + "ldr d4, [x21, x3]\n" + "smlal2 v22.4s, v14.8h, v1.8h\n" + "ssubl v4.8h, v4.8b, v18.8b\n" "smlal2 v19.4s, v25.8h, v1.8h\n" - "smlal2 v10.4s, v25.8h, v0.8h\n" - "ldr d25, [x13, x0]\n" - "ssubl v25.8h, v25.8b, v9.8b\n" - "smlal2 v7.4s, v31.8h, v0.8h\n" - "smlal v13.4s, v24.4h, v2.4h\n" - "smlal2 v21.4s, v30.8h, v0.8h\n" - "ldr d0, [x3, #0x78]\n" - "smlal v20.4s, v24.4h, v1.4h\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "smlal v8.4s, v30.4h, v1.4h\n" - "smlal v17.4s, v26.4h, v1.4h\n" - "smlal2 v19.4s, v24.8h, v2.8h\n" - "smlal2 v10.4s, v24.8h, v1.8h\n" - "ldr d24, [x12, x0]\n" - "ssubl v24.8h, v24.8b, v9.8b\n" - "smlal2 v7.4s, v30.8h, v1.8h\n" - "smlal v13.4s, v27.4h, v3.4h\n" - "smlal2 v21.4s, v26.8h, v1.8h\n" - "ldr d1, [x3, #0x80]\n" - "smlal v20.4s, v27.4h, v2.4h\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "smlal v8.4s, v26.4h, v2.4h\n" - "smlal v17.4s, v25.4h, v2.4h\n" - "smlal2 v19.4s, v27.8h, v3.8h\n" - "smlal2 v10.4s, v27.8h, v2.8h\n" - "ldr d27, [x11, x0]\n" - "ssubl v27.8h, v27.8b, v9.8b\n" - "smlal2 v7.4s, v26.8h, v2.8h\n" - "smlal v13.4s, v23.4h, v4.4h\n" - "smlal2 v21.4s, v25.8h, v2.8h\n" - "ldr d2, [x3, #0x88]\n" - "smlal v20.4s, v23.4h, v3.4h\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "smlal v8.4s, v25.4h, v3.4h\n" - "smlal v17.4s, v24.4h, v3.4h\n" - "smlal2 v19.4s, v23.8h, v4.8h\n" - "smlal2 v10.4s, v23.8h, v3.8h\n" - "ldr d23, [x22, x0]\n" - "ssubl v23.8h, v23.8b, v9.8b\n" - "smlal2 v7.4s, v25.8h, v3.8h\n" - "smlal v13.4s, v31.4h, v0.4h\n" - "smlal2 v21.4s, v24.8h, v3.8h\n" - "ldr d3, [x3, #0x90]\n" - "smlal v20.4s, v28.4h, v4.4h\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "smlal v8.4s, v24.4h, v4.4h\n" - "smlal v17.4s, v22.4h, v4.4h\n" - "smlal2 v19.4s, v31.8h, v0.8h\n" - "ldr d31, [x9, x0]\n" - "smlal2 v10.4s, v28.8h, v4.8h\n" - "ldr d28, [x27, x0]\n" - "smlal2 v7.4s, v24.8h, v4.8h\n" - "smlal v13.4s, v30.4h, v1.4h\n" - "ssubl v31.8h, v31.8b, v9.8b\n" - "smlal2 v21.4s, v22.8h, v4.8h\n" - "ldr d4, [x3, #0x98]\n" - "smlal v20.4s, v30.4h, v0.4h\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "smlal v8.4s, v27.4h, v0.4h\n" - "smlal v17.4s, v23.4h, v0.4h\n" - "ssubl v28.8h, v28.8b, v9.8b\n" - "smlal2 v19.4s, v30.8h, v1.8h\n" - "smlal2 v10.4s, v30.8h, v0.8h\n" - "ldr d30, [x28, x0]\n" - "ssubl v30.8h, v30.8b, v9.8b\n" - "smlal2 v7.4s, v27.8h, v0.8h\n" - "smlal v13.4s, v26.4h, v2.4h\n" - "smlal2 v21.4s, v23.8h, v0.8h\n" - "ldr d0, [x3, #0xa0]\n" - "smlal v20.4s, v26.4h, v1.4h\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "smlal v8.4s, v23.4h, v1.4h\n" - "smlal v17.4s, v31.4h, v1.4h\n" - "smlal2 v19.4s, v26.8h, v2.8h\n" - "smlal2 v10.4s, v26.8h, v1.8h\n" - "ldr d26, [x26, x0]\n" - "ssubl v26.8h, v26.8b, v9.8b\n" - "smlal2 v7.4s, v23.8h, v1.8h\n" - "smlal v13.4s, v25.4h, v3.4h\n" - "smlal2 v21.4s, v31.8h, v1.8h\n" - "ldr d1, [x3, #0xa8]\n" - "smlal v20.4s, v25.4h, v2.4h\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "smlal v8.4s, v31.4h, v2.4h\n" - "smlal v17.4s, v30.4h, v2.4h\n" - "smlal2 v19.4s, v25.8h, v3.8h\n" - "smlal2 v10.4s, v25.8h, v2.8h\n" - "ldr d25, [x25, x0]\n" - "ssubl v25.8h, v25.8b, v9.8b\n" - "smlal2 v7.4s, v31.8h, v2.8h\n" - "smlal v13.4s, v24.4h, v4.4h\n" - "smlal2 v21.4s, v30.8h, v2.8h\n" - "ldr d2, [x3, #0xb0]\n" - "smlal v20.4s, v24.4h, v3.4h\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "smlal v8.4s, v30.4h, v3.4h\n" - "smlal v17.4s, v28.4h, v3.4h\n" - "smlal2 v19.4s, v24.8h, v4.8h\n" - "smlal2 v10.4s, v24.8h, v3.8h\n" - "ldr d24, [x24, x0]\n" - "ssubl v24.8h, v24.8b, v9.8b\n" - "smlal2 v7.4s, v30.8h, v3.8h\n" - "smlal v13.4s, v27.4h, v0.4h\n" - "smlal2 v21.4s, v28.8h, v3.8h\n" - "ldr d3, [x3, #0xb8]\n" - "smlal v20.4s, v22.4h, v4.4h\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "smlal v8.4s, v28.4h, v4.4h\n" - "smlal v17.4s, v26.4h, v4.4h\n" - "smlal2 v19.4s, v27.8h, v0.8h\n" - "ldr d27, [x23, x0]\n" - "smlal2 v7.4s, v28.8h, v4.8h\n" - "ssubl v27.8h, v27.8b, v9.8b\n" - "smlal v13.4s, v23.4h, v1.4h\n" - "smlal2 v10.4s, v22.8h, v4.8h\n" - "ldr q22, [x8, #0x10]\n" - "add x8, x8, #0x20\n" - "smlal2 v21.4s, v26.8h, v4.8h\n" - "ldr d4, [x3, #0xc0]\n" - "smlal v20.4s, v23.4h, v0.4h\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "smlal v8.4s, v25.4h, v0.4h\n" - "smlal v17.4s, v24.4h, v0.4h\n" - "smlal2 v19.4s, v23.8h, v1.8h\n" - "smlal2 v7.4s, v25.8h, v0.8h\n" - "ldr d25, [x15, x0]\n" - "ssubl v25.8h, v25.8b, v9.8b\n" - "smlal v13.4s, v31.4h, v2.4h\n" - "smlal2 v10.4s, v23.8h, v0.8h\n" - "smlal2 v21.4s, v24.8h, v0.8h\n" - "smlal v20.4s, v31.4h, v1.4h\n" - "smlal v8.4s, v24.4h, v1.4h\n" - "smlal v17.4s, v27.4h, v1.4h\n" - "smlal2 v19.4s, v31.8h, v2.8h\n" - "smlal2 v7.4s, v24.8h, v1.8h\n" - "ldr d24, [x21, x0]\n" - "ssubl v24.8h, v24.8b, v9.8b\n" - "smlal v13.4s, v30.4h, v3.4h\n" - "smlal2 v10.4s, v31.8h, v1.8h\n" - "smlal2 v21.4s, v27.8h, v1.8h\n" + "ldr d1, [x6, #0x68]\n" + "smlal v20.4s, v17.4h, v2.4h\n" + "ssubl v1.8h, v1.8b, v13.8b\n" + "smlal v24.4s, v25.4h, v2.4h\n" + "smlal v23.4s, v10.4h, v2.4h\n" + "ldr x25, [x5, #0xf0]\n" + "ldr x24, [x5, #0xf8]\n" + "smlal2 v15.4s, v17.8h, v31.8h\n" + "smlal v7.4s, v6.4h, v8.4h\n" + "ldr x23, [x5, #0x100]\n" + "ldr x22, [x5, #0x108]\n" + "smlal2 v5.4s, v17.8h, v2.8h\n" + "ldr d17, [x20, x3]\n" + "smlal2 v22.4s, v25.8h, v2.8h\n" + "ssubl v17.8h, v17.8b, v18.8b\n" + "smlal2 v19.4s, v10.8h, v2.8h\n" + "ldr d2, [x6, #0x70]\n" + "smlal v20.4s, v6.4h, v31.4h\n" + "ssubl v2.8h, v2.8b, v13.8b\n" + "smlal v24.4s, v10.4h, v31.4h\n" + "smlal v23.4s, v9.4h, v31.4h\n" + "ldr x21, [x5, #0x110]\n" + "ldr x20, [x5, #0x118]\n" + "smlal2 v15.4s, v6.8h, v8.8h\n" + "smlal v7.4s, v28.4h, v21.4h\n" + "tst x1, #0x7\n" + "smlal2 v5.4s, v6.8h, v31.8h\n" + "ldr d6, [x13, x3]\n" + "smlal2 v22.4s, v10.8h, v31.8h\n" + "ssubl v6.8h, v6.8b, v18.8b\n" + "smlal2 v19.4s, v9.8h, v31.8h\n" + "ldr d31, [x6, #0x78]\n" + "smlal v20.4s, v29.4h, v8.4h\n" + "ssubl v31.8h, v31.8b, v13.8b\n" + "smlal v24.4s, v9.4h, v8.4h\n" + "smlal v23.4s, v30.4h, v8.4h\n" + "smlal2 v15.4s, v28.8h, v21.8h\n" + "ldr d28, [x12, x3]\n" + "smlal v7.4s, v14.4h, v3.4h\n" + "ssubl v28.8h, v28.8b, v18.8b\n" + "smlal2 v5.4s, v29.8h, v8.8h\n" + "ldr d29, [x6, #0x80]\n" + "smlal2 v22.4s, v9.8h, v8.8h\n" + "ssubl v29.8h, v29.8b, v13.8b\n" + "smlal2 v19.4s, v30.8h, v8.8h\n" + "ldr d8, [x11, x3]\n" + "smlal v20.4s, v14.4h, v21.4h\n" + "ssubl v8.8h, v8.8b, v18.8b\n" + "smlal v24.4s, v12.4h, v21.4h\n" + "smlal v23.4s, v16.4h, v21.4h\n" + "smlal2 v15.4s, v14.8h, v3.8h\n" + "smlal v7.4s, v25.4h, v27.4h\n" + "smlal2 v5.4s, v14.8h, v21.8h\n" + "ldr d14, [x10, x3]\n" + "smlal2 v22.4s, v12.8h, v21.8h\n" + "ssubl v14.8h, v14.8b, v18.8b\n" + "smlal2 v19.4s, v16.8h, v21.8h\n" + "ldr d21, [x6, #0x88]\n" + "smlal v20.4s, v25.4h, v3.4h\n" + "ssubl v21.8h, v21.8b, v13.8b\n" + "smlal v24.4s, v16.4h, v3.4h\n" + "smlal v23.4s, v4.4h, v3.4h\n" + "smlal2 v15.4s, v25.8h, v27.8h\n" + "smlal v7.4s, v10.4h, v1.4h\n" + "smlal2 v5.4s, v25.8h, v3.8h\n" + "ldr d25, [x9, x3]\n" + "smlal2 v22.4s, v16.8h, v3.8h\n" + "ssubl v25.8h, v25.8b, v18.8b\n" + "smlal2 v19.4s, v4.8h, v3.8h\n" + "ldr d3, [x6, #0x90]\n" + "smlal v20.4s, v10.4h, v27.4h\n" + "ssubl v3.8h, v3.8b, v13.8b\n" + "smlal v24.4s, v4.4h, v27.4h\n" + "smlal v23.4s, v17.4h, v27.4h\n" + "smlal2 v15.4s, v10.8h, v1.8h\n" + "smlal v7.4s, v9.4h, v2.4h\n" + "smlal2 v5.4s, v10.8h, v27.8h\n" + "ldr d10, [x28, x3]\n" + "smlal2 v22.4s, v4.8h, v27.8h\n" + "ssubl v10.8h, v10.8b, v18.8b\n" + "smlal2 v19.4s, v17.8h, v27.8h\n" + "ldr d27, [x6, #0x98]\n" + "smlal v20.4s, v9.4h, v1.4h\n" + "ssubl v27.8h, v27.8b, v13.8b\n" + "smlal v24.4s, v17.4h, v1.4h\n" + "smlal v23.4s, v6.4h, v1.4h\n" + "smlal2 v15.4s, v9.8h, v2.8h\n" + "smlal v7.4s, v12.4h, v31.4h\n" + "smlal2 v5.4s, v9.8h, v1.8h\n" + "ldr d9, [x27, x3]\n" + "smlal2 v22.4s, v17.8h, v1.8h\n" + "ssubl v9.8h, v9.8b, v18.8b\n" + "smlal2 v19.4s, v6.8h, v1.8h\n" + "ldr d1, [x6, #0xa0]\n" "smlal v20.4s, v30.4h, v2.4h\n" - "smlal v8.4s, v27.4h, v2.4h\n" - "smlal v17.4s, v25.4h, v2.4h\n" - "smlal2 v19.4s, v30.8h, v3.8h\n" - "smlal2 v7.4s, v27.8h, v2.8h\n" - "ldr d27, [x20, x0]\n" - "ssubl v27.8h, v27.8b, v9.8b\n" - "smlal v13.4s, v28.4h, v4.4h\n" - "smlal2 v10.4s, v30.8h, v2.8h\n" - "sqrdmulh v13.4s, v13.4s, v18.4s\n" - "add x0, x0, #0x8\n" - "smlal2 v21.4s, v25.8h, v2.8h\n" - "smlal v20.4s, v28.4h, v3.4h\n" - "and v30.16b, v13.16b, v6.16b\n" - "smlal v8.4s, v25.4h, v3.4h\n" - "smlal v17.4s, v24.4h, v3.4h\n" - "sshr v30.4s, v30.4s, #0x1f\n" - "smlal2 v19.4s, v28.8h, v4.8h\n" - "smlal2 v10.4s, v28.8h, v3.8h\n" - "sqrdmulh v19.4s, v19.4s, v5.4s\n" - "smlal2 v7.4s, v25.8h, v3.8h\n" - "smlal2 v21.4s, v24.8h, v3.8h\n" - "and v16.16b, v19.16b, v22.16b\n" - "smlal v20.4s, v26.4h, v4.4h\n" - "smlal v8.4s, v24.4h, v4.4h\n" - "sqrdmulh v20.4s, v20.4s, v18.4s\n" - "smlal v17.4s, v27.4h, v4.4h\n" - "smlal2 v10.4s, v26.8h, v4.8h\n" - "sqrdmulh v8.4s, v8.4s, v18.4s\n" - "smlal2 v7.4s, v24.8h, v4.8h\n" - "smlal2 v21.4s, v27.8h, v4.8h\n" - "sqrdmulh v17.4s, v17.4s, v18.4s\n" - "sqadd v13.4s, v13.4s, v30.4s\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "and v0.16b, v20.16b, v6.16b\n" - "sqrdmulh v10.4s, v10.4s, v5.4s\n" - "and v18.16b, v8.16b, v6.16b\n" - "sqrdmulh v7.4s, v7.4s, v5.4s\n" - "and v30.16b, v17.16b, v6.16b\n" - "sqrdmulh v21.4s, v21.4s, v5.4s\n" - "sqadd v19.4s, v19.4s, v16.4s\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v26.16b, v10.16b, v22.16b\n" - "sshr v18.4s, v18.4s, #0x1f\n" - "and v23.16b, v7.16b, v22.16b\n" + "ssubl v1.8h, v1.8b, v13.8b\n" + "smlal v24.4s, v6.4h, v2.4h\n" + "smlal v23.4s, v28.4h, v2.4h\n" + "smlal2 v15.4s, v12.8h, v31.8h\n" + "ldr d12, [x26, x3]\n" + "smlal v7.4s, v16.4h, v29.4h\n" + "ssubl v12.8h, v12.8b, v18.8b\n" + "smlal2 v5.4s, v30.8h, v2.8h\n" + "ldr d30, [x6, #0xa8]\n" + "smlal2 v22.4s, v6.8h, v2.8h\n" + "ssubl v30.8h, v30.8b, v13.8b\n" + "smlal2 v19.4s, v28.8h, v2.8h\n" + "ldr d2, [x25, x3]\n" + "smlal v20.4s, v16.4h, v31.4h\n" + "ssubl v2.8h, v2.8b, v18.8b\n" + "smlal v24.4s, v8.4h, v31.4h\n" + "smlal v23.4s, v14.4h, v31.4h\n" + "smlal2 v15.4s, v16.8h, v29.8h\n" + "smlal v7.4s, v4.4h, v21.4h\n" + "smlal2 v5.4s, v16.8h, v31.8h\n" + "ldr d16, [x24, x3]\n" + "smlal2 v22.4s, v8.8h, v31.8h\n" + "ssubl v16.8h, v16.8b, v18.8b\n" + "smlal2 v19.4s, v14.8h, v31.8h\n" + "ldr d31, [x6, #0xb0]\n" + "smlal v20.4s, v4.4h, v29.4h\n" + "ssubl v31.8h, v31.8b, v13.8b\n" + "smlal v24.4s, v14.4h, v29.4h\n" + "smlal v23.4s, v25.4h, v29.4h\n" + "smlal2 v15.4s, v4.8h, v21.8h\n" + "smlal v7.4s, v17.4h, v3.4h\n" + "smlal2 v5.4s, v4.8h, v29.8h\n" + "ldr d4, [x23, x3]\n" + "smlal2 v22.4s, v14.8h, v29.8h\n" + "ssubl v4.8h, v4.8b, v18.8b\n" + "smlal2 v19.4s, v25.8h, v29.8h\n" + "ldr d29, [x6, #0xb8]\n" + "smlal v20.4s, v17.4h, v21.4h\n" + "ssubl v29.8h, v29.8b, v13.8b\n" + "smlal v24.4s, v25.4h, v21.4h\n" + "smlal v23.4s, v10.4h, v21.4h\n" + "smlal2 v15.4s, v17.8h, v3.8h\n" + "smlal v7.4s, v6.4h, v27.4h\n" + "smlal2 v5.4s, v17.8h, v21.8h\n" + "ldr d17, [x22, x3]\n" + "smlal2 v22.4s, v25.8h, v21.8h\n" + "ssubl v17.8h, v17.8b, v18.8b\n" + "smlal2 v19.4s, v10.8h, v21.8h\n" + "ldr d21, [x6, #0xc0]\n" + "smlal v20.4s, v6.4h, v3.4h\n" + "ssubl v21.8h, v21.8b, v13.8b\n" + "smlal v24.4s, v10.4h, v3.4h\n" + "smlal v23.4s, v9.4h, v3.4h\n" + "smlal2 v15.4s, v6.8h, v27.8h\n" + "smlal v7.4s, v8.4h, v1.4h\n" + "smlal2 v5.4s, v6.8h, v3.8h\n" + "ldr d6, [x21, x3]\n" + "smlal2 v22.4s, v10.8h, v3.8h\n" + "ssubl v6.8h, v6.8b, v18.8b\n" + "smlal2 v19.4s, v9.8h, v3.8h\n" + "ldr d3, [x20, x3]\n" + "smlal v20.4s, v28.4h, v27.4h\n" + "ssubl v3.8h, v3.8b, v18.8b\n" + "smlal v24.4s, v9.4h, v27.4h\n" + "smlal v23.4s, v12.4h, v27.4h\n" + "add x3, x3, #0x8\n" + "smlal2 v15.4s, v8.8h, v1.8h\n" + "ldr q8, [x7, #0x0]\n" + "smlal v7.4s, v14.4h, v30.4h\n" + "smlal2 v5.4s, v28.8h, v27.8h\n" + "ldr q28, [x8, #0x0]\n" + "smlal2 v22.4s, v9.8h, v27.8h\n" + "smlal2 v19.4s, v12.8h, v27.8h\n" + "ldr q27, [x7, #0x10]\n" + "smlal v20.4s, v14.4h, v1.4h\n" + "add x7, x7, #0x20\n" + "smlal v24.4s, v2.4h, v1.4h\n" + "smlal v23.4s, v16.4h, v1.4h\n" + "smlal2 v15.4s, v14.8h, v30.8h\n" + "smlal v7.4s, v25.4h, v31.4h\n" + "smlal2 v5.4s, v14.8h, v1.8h\n" + "ldr q14, [x8, #0x10]\n" + "smlal2 v22.4s, v2.8h, v1.8h\n" + "add x8, x8, #0x20\n" + "smlal2 v19.4s, v16.8h, v1.8h\n" + "smlal v20.4s, v25.4h, v30.4h\n" + "smlal v24.4s, v16.4h, v30.4h\n" + "smlal v23.4s, v4.4h, v30.4h\n" + "smlal2 v15.4s, v25.8h, v31.8h\n" + "smlal v7.4s, v10.4h, v29.4h\n" + "smlal2 v5.4s, v25.8h, v30.8h\n" + "smlal2 v22.4s, v16.8h, v30.8h\n" + "smlal2 v19.4s, v4.8h, v30.8h\n" + "smlal v20.4s, v10.4h, v31.4h\n" + "smlal v24.4s, v4.4h, v31.4h\n" + "smlal v23.4s, v17.4h, v31.4h\n" + "smlal2 v15.4s, v10.8h, v29.8h\n" + "smlal v7.4s, v9.4h, v21.4h\n" + "sqrdmulh v7.4s, v7.4s, v8.4s\n" + "smlal2 v5.4s, v10.8h, v31.8h\n" + "smlal2 v22.4s, v4.8h, v31.8h\n" + "and v4.16b, v7.16b, v28.16b\n" + "smlal2 v19.4s, v17.8h, v31.8h\n" + "smlal v20.4s, v9.4h, v29.4h\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "smlal v24.4s, v17.4h, v29.4h\n" + "smlal v23.4s, v6.4h, v29.4h\n" + "sqadd v7.4s, v7.4s, v4.4s\n" + "smlal2 v15.4s, v9.8h, v21.8h\n" + "smlal2 v5.4s, v9.8h, v29.8h\n" + "sqrdmulh v15.4s, v15.4s, v27.4s\n" + "smlal2 v22.4s, v17.8h, v29.8h\n" + "smlal2 v19.4s, v6.8h, v29.8h\n" + "and v30.16b, v15.16b, v14.16b\n" + "smlal v20.4s, v12.4h, v21.4h\n" + "smlal v24.4s, v6.4h, v21.4h\n" + "sqrdmulh v20.4s, v20.4s, v8.4s\n" + "smlal v23.4s, v3.4h, v21.4h\n" + "smlal2 v5.4s, v12.8h, v21.8h\n" + "sqrdmulh v24.4s, v24.4s, v8.4s\n" + "smlal2 v22.4s, v6.8h, v21.8h\n" + "smlal2 v19.4s, v3.8h, v21.8h\n" + "sqrdmulh v23.4s, v23.4s, v8.4s\n" "sshr v30.4s, v30.4s, #0x1f\n" - "and v16.16b, v21.16b, v22.16b\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "sshr v26.4s, v26.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v18.4s\n" - "sshr v23.4s, v23.4s, #0x1f\n" - "sqadd v17.4s, v17.4s, v30.4s\n" + "and v3.16b, v20.16b, v28.16b\n" + "sqrdmulh v5.4s, v5.4s, v27.4s\n" + "and v25.16b, v24.16b, v28.16b\n" + "sqrdmulh v22.4s, v22.4s, v27.4s\n" + "and v16.16b, v23.16b, v28.16b\n" + "sqrdmulh v19.4s, v19.4s, v27.4s\n" + "sqadd v15.4s, v15.4s, v30.4s\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "and v4.16b, v5.16b, v14.16b\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "and v10.16b, v22.16b, v14.16b\n" "sshr v16.4s, v16.4s, #0x1f\n" - "srshl v13.4s, v13.4s, v6.4s\n" - "srshl v20.4s, v20.4s, v6.4s\n" - "sqadd v10.4s, v10.4s, v26.4s\n" - "srshl v8.4s, v8.4s, v6.4s\n" - "sqadd v7.4s, v7.4s, v23.4s\n" - "srshl v17.4s, v17.4s, v6.4s\n" - "sqadd v21.4s, v21.4s, v16.4s\n" - "srshl v19.4s, v19.4s, v22.4s\n" - "sqxtn v13.4h, v13.4s\n" - "srshl v10.4s, v10.4s, v22.4s\n" + "and v12.16b, v19.16b, v14.16b\n" + "sqadd v20.4s, v20.4s, v3.4s\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sqadd v24.4s, v24.4s, v25.4s\n" + "sshr v10.4s, v10.4s, #0x1f\n" + "sqadd v23.4s, v23.4s, v16.4s\n" + "sshr v12.4s, v12.4s, #0x1f\n" + "srshl v7.4s, v7.4s, v28.4s\n" + "srshl v20.4s, v20.4s, v28.4s\n" + "sqadd v5.4s, v5.4s, v4.4s\n" + "srshl v24.4s, v24.4s, v28.4s\n" + "sqadd v22.4s, v22.4s, v10.4s\n" + "srshl v23.4s, v23.4s, v28.4s\n" + "sqadd v19.4s, v19.4s, v12.4s\n" + "srshl v15.4s, v15.4s, v14.4s\n" + "sqxtn v7.4h, v7.4s\n" + "srshl v5.4s, v5.4s, v14.4s\n" "sqxtn v20.4h, v20.4s\n" - "srshl v7.4s, v7.4s, v22.4s\n" - "sqxtn v8.4h, v8.4s\n" - "srshl v21.4s, v21.4s, v22.4s\n" - "sqxtn v17.4h, v17.4s\n" - "sqxtn2 v13.8h, v19.4s\n" - "sqxtn2 v20.8h, v10.4s\n" - "sqxtn2 v8.8h, v7.4s\n" - "sqxtn2 v17.8h, v21.4s\n" - "sqadd v13.8h, v13.8h, v14.8h\n" - "sqadd v20.8h, v20.8h, v14.8h\n" - "sqadd v8.8h, v8.8h, v14.8h\n" - "sqadd v17.8h, v17.8h, v14.8h\n" - "smax v13.8h, v13.8h, v12.8h\n" - "smax v20.8h, v20.8h, v12.8h\n" - "smax v8.8h, v8.8h, v12.8h\n" - "smax v17.8h, v17.8h, v12.8h\n" - "smin v13.8h, v13.8h, v11.8h\n" - "smin v20.8h, v20.8h, v11.8h\n" - "smin v8.8h, v8.8h, v11.8h\n" - "smin v17.8h, v17.8h, v11.8h\n" - "uzp1 v13.16b, v13.16b, v13.16b\n" + "srshl v22.4s, v22.4s, v14.4s\n" + "sqxtn v24.4h, v24.4s\n" + "srshl v19.4s, v19.4s, v14.4s\n" + "sqxtn v23.4h, v23.4s\n" + "sqxtn2 v7.8h, v15.4s\n" + "sqxtn2 v20.8h, v5.4s\n" + "sqxtn2 v24.8h, v22.4s\n" + "sqxtn2 v23.8h, v19.4s\n" + "sqadd v7.8h, v7.8h, v26.8h\n" + "sqadd v20.8h, v20.8h, v26.8h\n" + "sqadd v24.8h, v24.8h, v26.8h\n" + "sqadd v23.8h, v23.8h, v26.8h\n" + "smax v7.8h, v7.8h, v11.8h\n" + "smax v20.8h, v20.8h, v11.8h\n" + "smax v24.8h, v24.8h, v11.8h\n" + "smax v23.8h, v23.8h, v11.8h\n" + "smin v7.8h, v7.8h, v0.8h\n" + "smin v20.8h, v20.8h, v0.8h\n" + "smin v24.8h, v24.8h, v0.8h\n" + "smin v23.8h, v23.8h, v0.8h\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" + "str d7, [x17, x4]\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "str d13, [x17, x10]\n" - "uzp1 v8.16b, v8.16b, v8.16b\n" - "uzp1 v17.16b, v17.16b, v17.16b\n" - "str d20, [x6, x10]\n" - "str d8, [x7, x10]\n" - "str d17, [x16, x10]\n" - "add x10, x10, #0x8\n" + "uzp1 v24.16b, v24.16b, v24.16b\n" + "str d20, [x16, x4]\n" + "uzp1 v23.16b, v23.16b, v23.16b\n" + "str d24, [x15, x4]\n" + "str d23, [x14, x4]\n" + "add x4, x4, #0x8\n" "beq 124f\n" - "add x3, x3, #0xc8\n" + "add x6, x6, #0xc8\n" "3:" // Oddments - "ldr x13, [%x[params], %[offsetof_Params_bias]]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" "tbz x1, #2, 5f\n" - "ld1 { v13.4s }, [x13], #0x10\n" + "ld1 { v7.4s }, [x20], #0x10\n" "tbz x1, #1, 4f\n" - "ld1 { v19.d }[0], [x13], #0x8\n" + "ld1 { v15.d }[0], [x20], #0x8\n" "tbz x1, #0, 7f\n" - "ld1 { v19.s }[2], [x13]\n" + "ld1 { v15.s }[2], [x20]\n" "b 7f\n" "4:" // Oddments: Load bias: Bit 2: Bit 1: Unset "tbz x1, #0, 7f\n" - "ld1 { v19.s }[0], [x13]\n" + "ld1 { v15.s }[0], [x20]\n" "b 7f\n" "5:" // Oddments: Load bias: Bit 2: Unset "tbz x1, #1, 6f\n" - "ld1 { v13.d }[0], [x13], #0x8\n" + "ld1 { v7.d }[0], [x20], #0x8\n" "tbz x1, #0, 7f\n" - "ld1 { v13.s }[2], [x13]\n" + "ld1 { v7.s }[2], [x20]\n" "b 7f\n" "6:" // Oddments: Load bias: Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 7f\n" - "ld1 { v13.s }[0], [x13]\n" + "ld1 { v7.s }[0], [x20]\n" "7:" // Oddments: Load bias: Bit 2: End - "ldr d0, [x3, #0x0]\n" - "ldr d1, [x3, #0x8]\n" - "mov v20.16b, v13.16b\n" - "mov v10.16b, v19.16b\n" - "ldr d2, [x3, #0x10]\n" - "ldr d3, [x3, #0x18]\n" - "mov v8.16b, v13.16b\n" - "mov v7.16b, v19.16b\n" - "ldr d4, [x3, #0x20]\n" - "ldp x9, x28, [x4, #0x0]\n" - "mov v17.16b, v13.16b\n" - "mov v21.16b, v19.16b\n" - "ldp x27, x26, [x4, #0x10]\n" - "ldp x25, x24, [x4, #0x20]\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "ldp x23, x22, [x4, #0x30]\n" - "ldp x21, x20, [x4, #0x40]\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "add x9, x9, x0\n" - "add x28, x28, x0\n" - "add x27, x27, x0\n" - "add x26, x26, x0\n" - "add x25, x25, x0\n" - "add x24, x24, x0\n" - "add x23, x23, x0\n" - "add x22, x22, x0\n" - "add x21, x21, x0\n" - "add x20, x20, x0\n" + "ldr d6, [x6, #0x0]\n" + "ldr d14, [x6, #0x8]\n" + "mov v20.16b, v7.16b\n" + "mov v5.16b, v15.16b\n" + "ldr d10, [x6, #0x10]\n" + "ldr d21, [x6, #0x18]\n" + "mov v24.16b, v7.16b\n" + "mov v22.16b, v15.16b\n" + "ldr d12, [x6, #0x20]\n" + "ldp x9, x28, [x5, #0x0]\n" + "mov v23.16b, v7.16b\n" + "mov v19.16b, v15.16b\n" + "ldp x27, x26, [x5, #0x10]\n" + "ldp x25, x24, [x5, #0x20]\n" + "ssubl v6.8h, v6.8b, v13.8b\n" + "ssubl v14.8h, v14.8b, v13.8b\n" + "ldp x23, x22, [x5, #0x30]\n" + "ldp x21, x20, [x5, #0x40]\n" + "ssubl v10.8h, v10.8b, v13.8b\n" + "ssubl v21.8h, v21.8b, v13.8b\n" + "ssubl v12.8h, v12.8b, v13.8b\n" + "add x9, x9, x3\n" + "add x28, x28, x3\n" + "add x27, x27, x3\n" + "add x26, x26, x3\n" + "add x25, x25, x3\n" + "add x24, x24, x3\n" + "add x23, x23, x3\n" + "add x22, x22, x3\n" + "add x21, x21, x3\n" + "add x20, x20, x3\n" "tbz x1, #2, 9f\n" "ld1 { v31.s }[0], [x9], #0x4\n" - "ld1 { v30.s }[0], [x28], #0x4\n" - "ld1 { v29.s }[0], [x27], #0x4\n" - "ld1 { v28.s }[0], [x26], #0x4\n" - "ld1 { v27.s }[0], [x25], #0x4\n" - "ld1 { v23.s }[0], [x24], #0x4\n" + "ld1 { v17.s }[0], [x28], #0x4\n" + "ld1 { v30.s }[0], [x27], #0x4\n" + "ld1 { v16.s }[0], [x26], #0x4\n" + "ld1 { v3.s }[0], [x25], #0x4\n" + "ld1 { v4.s }[0], [x24], #0x4\n" "ld1 { v25.s }[0], [x23], #0x4\n" - "ld1 { v24.s }[0], [x22], #0x4\n" - "ld1 { v26.s }[0], [x21], #0x4\n" - "ld1 { v22.s }[0], [x20], #0x4\n" + "ld1 { v9.s }[0], [x22], #0x4\n" + "ld1 { v29.s }[0], [x21], #0x4\n" + "ld1 { v28.s }[0], [x20], #0x4\n" "tbz x1, #1, 8f\n" "ld1 { v31.h }[2], [x9], #0x2\n" - "ld1 { v30.h }[2], [x28], #0x2\n" - "ld1 { v29.h }[2], [x27], #0x2\n" - "ld1 { v28.h }[2], [x26], #0x2\n" - "ld1 { v27.h }[2], [x25], #0x2\n" - "ld1 { v23.h }[2], [x24], #0x2\n" + "ld1 { v17.h }[2], [x28], #0x2\n" + "ld1 { v30.h }[2], [x27], #0x2\n" + "ld1 { v16.h }[2], [x26], #0x2\n" + "ld1 { v3.h }[2], [x25], #0x2\n" + "ld1 { v4.h }[2], [x24], #0x2\n" "ld1 { v25.h }[2], [x23], #0x2\n" - "ld1 { v24.h }[2], [x22], #0x2\n" - "ld1 { v26.h }[2], [x21], #0x2\n" - "ld1 { v22.h }[2], [x20], #0x2\n" + "ld1 { v9.h }[2], [x22], #0x2\n" + "ld1 { v29.h }[2], [x21], #0x2\n" + "ld1 { v28.h }[2], [x20], #0x2\n" "tbz x1, #0, 11f\n" "ld1 { v31.b }[6], [x9]\n" - "ld1 { v30.b }[6], [x28]\n" - "ld1 { v29.b }[6], [x27]\n" - "ld1 { v28.b }[6], [x26]\n" - "ld1 { v27.b }[6], [x25]\n" - "ld1 { v23.b }[6], [x24]\n" + "ld1 { v17.b }[6], [x28]\n" + "ld1 { v30.b }[6], [x27]\n" + "ld1 { v16.b }[6], [x26]\n" + "ld1 { v3.b }[6], [x25]\n" + "ld1 { v4.b }[6], [x24]\n" "ld1 { v25.b }[6], [x23]\n" - "ld1 { v24.b }[6], [x22]\n" - "ld1 { v26.b }[6], [x21]\n" - "ld1 { v22.b }[6], [x20]\n" + "ld1 { v9.b }[6], [x22]\n" + "ld1 { v29.b }[6], [x21]\n" + "ld1 { v28.b }[6], [x20]\n" "b 11f\n" "8:" // Oddments: Initial loads: Bit 2: Bit 1: Unset "tbz x1, #0, 11f\n" "ld1 { v31.b }[4], [x9]\n" - "ld1 { v30.b }[4], [x28]\n" - "ld1 { v29.b }[4], [x27]\n" - "ld1 { v28.b }[4], [x26]\n" - "ld1 { v27.b }[4], [x25]\n" - "ld1 { v23.b }[4], [x24]\n" + "ld1 { v17.b }[4], [x28]\n" + "ld1 { v30.b }[4], [x27]\n" + "ld1 { v16.b }[4], [x26]\n" + "ld1 { v3.b }[4], [x25]\n" + "ld1 { v4.b }[4], [x24]\n" "ld1 { v25.b }[4], [x23]\n" - "ld1 { v24.b }[4], [x22]\n" - "ld1 { v26.b }[4], [x21]\n" - "ld1 { v22.b }[4], [x20]\n" + "ld1 { v9.b }[4], [x22]\n" + "ld1 { v29.b }[4], [x21]\n" + "ld1 { v28.b }[4], [x20]\n" "b 11f\n" "9:" // Oddments: Initial loads: Bit 2: Unset "tbz x1, #1, 10f\n" "ld1 { v31.h }[0], [x9], #0x2\n" - "ld1 { v30.h }[0], [x28], #0x2\n" - "ld1 { v29.h }[0], [x27], #0x2\n" - "ld1 { v28.h }[0], [x26], #0x2\n" - "ld1 { v27.h }[0], [x25], #0x2\n" - "ld1 { v23.h }[0], [x24], #0x2\n" + "ld1 { v17.h }[0], [x28], #0x2\n" + "ld1 { v30.h }[0], [x27], #0x2\n" + "ld1 { v16.h }[0], [x26], #0x2\n" + "ld1 { v3.h }[0], [x25], #0x2\n" + "ld1 { v4.h }[0], [x24], #0x2\n" "ld1 { v25.h }[0], [x23], #0x2\n" - "ld1 { v24.h }[0], [x22], #0x2\n" - "ld1 { v26.h }[0], [x21], #0x2\n" - "ld1 { v22.h }[0], [x20], #0x2\n" + "ld1 { v9.h }[0], [x22], #0x2\n" + "ld1 { v29.h }[0], [x21], #0x2\n" + "ld1 { v28.h }[0], [x20], #0x2\n" "tbz x1, #0, 11f\n" "ld1 { v31.b }[2], [x9]\n" - "ld1 { v30.b }[2], [x28]\n" - "ld1 { v29.b }[2], [x27]\n" - "ld1 { v28.b }[2], [x26]\n" - "ld1 { v27.b }[2], [x25]\n" - "ld1 { v23.b }[2], [x24]\n" + "ld1 { v17.b }[2], [x28]\n" + "ld1 { v30.b }[2], [x27]\n" + "ld1 { v16.b }[2], [x26]\n" + "ld1 { v3.b }[2], [x25]\n" + "ld1 { v4.b }[2], [x24]\n" "ld1 { v25.b }[2], [x23]\n" - "ld1 { v24.b }[2], [x22]\n" - "ld1 { v26.b }[2], [x21]\n" - "ld1 { v22.b }[2], [x20]\n" + "ld1 { v9.b }[2], [x22]\n" + "ld1 { v29.b }[2], [x21]\n" + "ld1 { v28.b }[2], [x20]\n" "b 11f\n" "10:" // Oddments: Initial loads: Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 11f\n" "ld1 { v31.b }[0], [x9]\n" - "ld1 { v30.b }[0], [x28]\n" - "ld1 { v29.b }[0], [x27]\n" - "ld1 { v28.b }[0], [x26]\n" - "ld1 { v27.b }[0], [x25]\n" - "ld1 { v23.b }[0], [x24]\n" + "ld1 { v17.b }[0], [x28]\n" + "ld1 { v30.b }[0], [x27]\n" + "ld1 { v16.b }[0], [x26]\n" + "ld1 { v3.b }[0], [x25]\n" + "ld1 { v4.b }[0], [x24]\n" "ld1 { v25.b }[0], [x23]\n" - "ld1 { v24.b }[0], [x22]\n" - "ld1 { v26.b }[0], [x21]\n" - "ld1 { v22.b }[0], [x20]\n" + "ld1 { v9.b }[0], [x22]\n" + "ld1 { v29.b }[0], [x21]\n" + "ld1 { v28.b }[0], [x20]\n" "11:" // Oddments: Initial loads: Bit 2: End - "ssubl v31.8h, v31.8b, v9.8b\n" - "ssubl v30.8h, v30.8b, v9.8b\n" - "smlal v13.4s, v31.4h, v0.4h\n" - "ldr x20, [x4, #0x50]\n" - "ssubl v29.8h, v29.8b, v9.8b\n" - "smlal2 v19.4s, v31.8h, v0.8h\n" - "smlal v20.4s, v30.4h, v0.4h\n" - "smlal2 v10.4s, v30.8h, v0.8h\n" - "smlal v8.4s, v29.4h, v0.4h\n" - "ssubl v28.8h, v28.8b, v9.8b\n" - "add x20, x20, x0\n" - "smlal2 v7.4s, v29.8h, v0.8h\n" - "ssubl v27.8h, v27.8b, v9.8b\n" - "smlal v17.4s, v28.4h, v0.4h\n" - "smlal2 v21.4s, v28.8h, v0.8h\n" - "smlal v13.4s, v30.4h, v1.4h\n" - "ssubl v23.8h, v23.8b, v9.8b\n" - "smlal2 v19.4s, v30.8h, v1.8h\n" - "smlal v20.4s, v27.4h, v1.4h\n" - "ssubl v25.8h, v25.8b, v9.8b\n" - "smlal2 v10.4s, v27.8h, v1.8h\n" - "smlal v8.4s, v28.4h, v1.4h\n" - "ssubl v24.8h, v24.8b, v9.8b\n" - "smlal2 v7.4s, v28.8h, v1.8h\n" - "ssubl v26.8h, v26.8b, v9.8b\n" - "smlal v17.4s, v23.4h, v1.4h\n" - "ssubl v22.8h, v22.8b, v9.8b\n" - "smlal2 v21.4s, v23.8h, v1.8h\n" - "smlal v13.4s, v27.4h, v2.4h\n" - "smlal2 v19.4s, v27.8h, v2.8h\n" - "smlal v20.4s, v25.4h, v2.4h\n" - "smlal2 v10.4s, v25.8h, v2.8h\n" - "smlal v8.4s, v23.4h, v2.4h\n" - "smlal2 v7.4s, v23.8h, v2.8h\n" + "ssubl v31.8h, v31.8b, v18.8b\n" + "ssubl v17.8h, v17.8b, v18.8b\n" + "smlal v7.4s, v31.4h, v6.4h\n" + "ldr x20, [x5, #0x50]\n" + "ssubl v30.8h, v30.8b, v18.8b\n" + "smlal2 v15.4s, v31.8h, v6.8h\n" + "smlal v20.4s, v17.4h, v6.4h\n" + "smlal2 v5.4s, v17.8h, v6.8h\n" + "smlal v24.4s, v30.4h, v6.4h\n" + "ssubl v16.8h, v16.8b, v18.8b\n" + "add x20, x20, x3\n" + "smlal2 v22.4s, v30.8h, v6.8h\n" + "ssubl v3.8h, v3.8b, v18.8b\n" + "smlal v23.4s, v16.4h, v6.4h\n" + "smlal2 v19.4s, v16.8h, v6.8h\n" + "smlal v7.4s, v17.4h, v14.4h\n" + "ssubl v4.8h, v4.8b, v18.8b\n" + "smlal2 v15.4s, v17.8h, v14.8h\n" + "smlal v20.4s, v3.4h, v14.4h\n" + "ssubl v25.8h, v25.8b, v18.8b\n" + "smlal2 v5.4s, v3.8h, v14.8h\n" + "smlal v24.4s, v16.4h, v14.4h\n" + "ssubl v9.8h, v9.8b, v18.8b\n" + "smlal2 v22.4s, v16.8h, v14.8h\n" + "ssubl v29.8h, v29.8b, v18.8b\n" + "smlal v23.4s, v4.4h, v14.4h\n" + "ssubl v28.8h, v28.8b, v18.8b\n" + "smlal2 v19.4s, v4.8h, v14.8h\n" + "smlal v7.4s, v3.4h, v10.4h\n" + "smlal2 v15.4s, v3.8h, v10.8h\n" + "smlal v20.4s, v25.4h, v10.4h\n" + "smlal2 v5.4s, v25.8h, v10.8h\n" + "smlal v24.4s, v4.4h, v10.4h\n" + "smlal2 v22.4s, v4.8h, v10.8h\n" "tbz x1, #2, 13f\n" - "ld1 { v31.s }[0], [x20], #0x4\n" + "ld1 { v27.s }[0], [x20], #0x4\n" "tbz x1, #1, 12f\n" - "ld1 { v31.h }[2], [x20], #0x2\n" + "ld1 { v27.h }[2], [x20], #0x2\n" "tbz x1, #0, 15f\n" - "ld1 { v31.b }[6], [x20]\n" + "ld1 { v27.b }[6], [x20]\n" "b 15f\n" "12:" // Oddments: Load (1, 3): Bit 2: Bit 1: Unset "tbz x1, #0, 15f\n" - "ld1 { v31.b }[4], [x20]\n" + "ld1 { v27.b }[4], [x20]\n" "b 15f\n" "13:" // Oddments: Load (1, 3): Bit 2: Unset "tbz x1, #1, 14f\n" - "ld1 { v31.h }[0], [x20], #0x2\n" + "ld1 { v27.h }[0], [x20], #0x2\n" "tbz x1, #0, 15f\n" - "ld1 { v31.b }[2], [x20]\n" + "ld1 { v27.b }[2], [x20]\n" "b 15f\n" "14:" // Oddments: Load (1, 3): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 15f\n" - "ld1 { v31.b }[0], [x20]\n" + "ld1 { v27.b }[0], [x20]\n" "15:" // Oddments: Load (1, 3): Bit 2: End - "ssubl v31.8h, v31.8b, v9.8b\n" - "ldr x22, [x4, #0x58]\n" - "smlal v17.4s, v31.4h, v2.4h\n" - "smlal2 v21.4s, v31.8h, v2.8h\n" - "smlal v13.4s, v25.4h, v3.4h\n" - "smlal2 v19.4s, v25.8h, v3.8h\n" - "add x22, x22, x0\n" - "smlal v20.4s, v24.4h, v3.4h\n" - "smlal2 v10.4s, v24.8h, v3.8h\n" - "smlal v8.4s, v31.4h, v3.4h\n" - "smlal2 v7.4s, v31.8h, v3.8h\n" + "ssubl v27.8h, v27.8b, v18.8b\n" + "ldr x20, [x5, #0x58]\n" + "smlal v23.4s, v27.4h, v10.4h\n" + "smlal2 v19.4s, v27.8h, v10.8h\n" + "smlal v7.4s, v25.4h, v21.4h\n" + "smlal2 v15.4s, v25.8h, v21.8h\n" + "add x20, x20, x3\n" + "smlal v20.4s, v9.4h, v21.4h\n" + "smlal2 v5.4s, v9.8h, v21.8h\n" + "smlal v24.4s, v27.4h, v21.4h\n" + "smlal2 v22.4s, v27.8h, v21.8h\n" "tbz x1, #2, 17f\n" - "ld1 { v30.s }[0], [x22], #0x4\n" + "ld1 { v6.s }[0], [x20], #0x4\n" "tbz x1, #1, 16f\n" - "ld1 { v30.h }[2], [x22], #0x2\n" + "ld1 { v6.h }[2], [x20], #0x2\n" "tbz x1, #0, 19f\n" - "ld1 { v30.b }[6], [x22]\n" + "ld1 { v6.b }[6], [x20]\n" "b 19f\n" "16:" // Oddments: Load (1, 4): Bit 2: Bit 1: Unset "tbz x1, #0, 19f\n" - "ld1 { v30.b }[4], [x22]\n" + "ld1 { v6.b }[4], [x20]\n" "b 19f\n" "17:" // Oddments: Load (1, 4): Bit 2: Unset "tbz x1, #1, 18f\n" - "ld1 { v30.h }[0], [x22], #0x2\n" + "ld1 { v6.h }[0], [x20], #0x2\n" "tbz x1, #0, 19f\n" - "ld1 { v30.b }[2], [x22]\n" + "ld1 { v6.b }[2], [x20]\n" "b 19f\n" "18:" // Oddments: Load (1, 4): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 19f\n" - "ld1 { v30.b }[0], [x22]\n" + "ld1 { v6.b }[0], [x20]\n" "19:" // Oddments: Load (1, 4): Bit 2: End - "ssubl v30.8h, v30.8b, v9.8b\n" - "ldr x21, [x4, #0x60]\n" - "smlal v17.4s, v30.4h, v3.4h\n" - "smlal2 v21.4s, v30.8h, v3.8h\n" - "smlal v13.4s, v24.4h, v4.4h\n" - "smlal2 v19.4s, v24.8h, v4.8h\n" - "add x21, x21, x0\n" + "ssubl v6.8h, v6.8b, v18.8b\n" + "ldr x20, [x5, #0x60]\n" + "smlal v23.4s, v6.4h, v21.4h\n" + "smlal2 v19.4s, v6.8h, v21.8h\n" + "smlal v7.4s, v9.4h, v12.4h\n" + "smlal2 v15.4s, v9.8h, v12.8h\n" + "add x20, x20, x3\n" "tbz x1, #2, 21f\n" - "ld1 { v27.s }[0], [x21], #0x4\n" + "ld1 { v9.s }[0], [x20], #0x4\n" "tbz x1, #1, 20f\n" - "ld1 { v27.h }[2], [x21], #0x2\n" + "ld1 { v9.h }[2], [x20], #0x2\n" "tbz x1, #0, 23f\n" - "ld1 { v27.b }[6], [x21]\n" + "ld1 { v9.b }[6], [x20]\n" "b 23f\n" "20:" // Oddments: Load (0, 5): Bit 2: Bit 1: Unset "tbz x1, #0, 23f\n" - "ld1 { v27.b }[4], [x21]\n" + "ld1 { v9.b }[4], [x20]\n" "b 23f\n" "21:" // Oddments: Load (0, 5): Bit 2: Unset "tbz x1, #1, 22f\n" - "ld1 { v27.h }[0], [x21], #0x2\n" + "ld1 { v9.h }[0], [x20], #0x2\n" "tbz x1, #0, 23f\n" - "ld1 { v27.b }[2], [x21]\n" + "ld1 { v9.b }[2], [x20]\n" "b 23f\n" "22:" // Oddments: Load (0, 5): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 23f\n" - "ld1 { v27.b }[0], [x21]\n" + "ld1 { v9.b }[0], [x20]\n" "23:" // Oddments: Load (0, 5): Bit 2: End - "ldr d0, [x3, #0x28]\n" - "ssubl v27.8h, v27.8b, v9.8b\n" - "smlal v20.4s, v27.4h, v4.4h\n" - "smlal2 v10.4s, v27.8h, v4.8h\n" - "smlal v8.4s, v30.4h, v4.4h\n" - "smlal2 v7.4s, v30.8h, v4.8h\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "ldr x20, [x4, #0x68]\n" - "smlal v17.4s, v26.4h, v4.4h\n" - "smlal2 v21.4s, v26.8h, v4.8h\n" - "add x20, x20, x0\n" - "smlal v13.4s, v29.4h, v0.4h\n" - "smlal2 v19.4s, v29.8h, v0.8h\n" - "smlal v20.4s, v28.4h, v0.4h\n" - "smlal2 v10.4s, v28.8h, v0.8h\n" - "smlal v8.4s, v22.4h, v0.4h\n" - "smlal2 v7.4s, v22.8h, v0.8h\n" + "ldr d14, [x6, #0x28]\n" + "ssubl v9.8h, v9.8b, v18.8b\n" + "smlal v20.4s, v9.4h, v12.4h\n" + "smlal2 v5.4s, v9.8h, v12.8h\n" + "smlal v24.4s, v6.4h, v12.4h\n" + "smlal2 v22.4s, v6.8h, v12.8h\n" + "ssubl v14.8h, v14.8b, v13.8b\n" + "ldr x20, [x5, #0x68]\n" + "smlal v23.4s, v29.4h, v12.4h\n" + "smlal2 v19.4s, v29.8h, v12.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v30.4h, v14.4h\n" + "smlal2 v15.4s, v30.8h, v14.8h\n" + "smlal v20.4s, v16.4h, v14.4h\n" + "smlal2 v5.4s, v16.8h, v14.8h\n" + "smlal v24.4s, v28.4h, v14.4h\n" + "smlal2 v22.4s, v28.8h, v14.8h\n" "tbz x1, #2, 25f\n" "ld1 { v25.s }[0], [x20], #0x4\n" "tbz x1, #1, 24f\n" @@ -1315,869 +1315,869 @@ void a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( "tbz x1, #0, 27f\n" "ld1 { v25.b }[0], [x20]\n" "27:" // Oddments: Load (2, 1): Bit 2: End - "ldr d1, [x3, #0x30]\n" - "ssubl v25.8h, v25.8b, v9.8b\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "ldr x26, [x4, #0x70]\n" - "smlal v17.4s, v25.4h, v0.4h\n" - "smlal2 v21.4s, v25.8h, v0.8h\n" - "add x26, x26, x0\n" - "smlal v13.4s, v28.4h, v1.4h\n" - "smlal2 v19.4s, v28.8h, v1.8h\n" - "smlal v20.4s, v23.4h, v1.4h\n" - "smlal2 v10.4s, v23.8h, v1.8h\n" - "smlal v8.4s, v25.4h, v1.4h\n" - "smlal2 v7.4s, v25.8h, v1.8h\n" + "ldr d21, [x6, #0x30]\n" + "ssubl v25.8h, v25.8b, v18.8b\n" + "ssubl v21.8h, v21.8b, v13.8b\n" + "ldr x20, [x5, #0x70]\n" + "smlal v23.4s, v25.4h, v14.4h\n" + "smlal2 v19.4s, v25.8h, v14.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v16.4h, v21.4h\n" + "smlal2 v15.4s, v16.8h, v21.8h\n" + "smlal v20.4s, v4.4h, v21.4h\n" + "smlal2 v5.4s, v4.8h, v21.8h\n" + "smlal v24.4s, v25.4h, v21.4h\n" + "smlal2 v22.4s, v25.8h, v21.8h\n" "tbz x1, #2, 29f\n" - "ld1 { v24.s }[0], [x26], #0x4\n" + "ld1 { v10.s }[0], [x20], #0x4\n" "tbz x1, #1, 28f\n" - "ld1 { v24.h }[2], [x26], #0x2\n" + "ld1 { v10.h }[2], [x20], #0x2\n" "tbz x1, #0, 31f\n" - "ld1 { v24.b }[6], [x26]\n" + "ld1 { v10.b }[6], [x20]\n" "b 31f\n" "28:" // Oddments: Load (2, 2): Bit 2: Bit 1: Unset "tbz x1, #0, 31f\n" - "ld1 { v24.b }[4], [x26]\n" + "ld1 { v10.b }[4], [x20]\n" "b 31f\n" "29:" // Oddments: Load (2, 2): Bit 2: Unset "tbz x1, #1, 30f\n" - "ld1 { v24.h }[0], [x26], #0x2\n" + "ld1 { v10.h }[0], [x20], #0x2\n" "tbz x1, #0, 31f\n" - "ld1 { v24.b }[2], [x26]\n" + "ld1 { v10.b }[2], [x20]\n" "b 31f\n" "30:" // Oddments: Load (2, 2): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 31f\n" - "ld1 { v24.b }[0], [x26]\n" + "ld1 { v10.b }[0], [x20]\n" "31:" // Oddments: Load (2, 2): Bit 2: End - "ldr d2, [x3, #0x38]\n" - "ssubl v24.8h, v24.8b, v9.8b\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "ldr x25, [x4, #0x78]\n" - "smlal v17.4s, v24.4h, v1.4h\n" - "smlal2 v21.4s, v24.8h, v1.8h\n" - "add x25, x25, x0\n" - "smlal v13.4s, v23.4h, v2.4h\n" - "smlal2 v19.4s, v23.8h, v2.8h\n" - "smlal v20.4s, v31.4h, v2.4h\n" - "smlal2 v10.4s, v31.8h, v2.8h\n" - "smlal v8.4s, v24.4h, v2.4h\n" - "smlal2 v7.4s, v24.8h, v2.8h\n" + "ldr d9, [x6, #0x38]\n" + "ssubl v10.8h, v10.8b, v18.8b\n" + "ssubl v9.8h, v9.8b, v13.8b\n" + "ldr x20, [x5, #0x78]\n" + "smlal v23.4s, v10.4h, v21.4h\n" + "smlal2 v19.4s, v10.8h, v21.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v4.4h, v9.4h\n" + "smlal2 v15.4s, v4.8h, v9.8h\n" + "smlal v20.4s, v27.4h, v9.4h\n" + "smlal2 v5.4s, v27.8h, v9.8h\n" + "smlal v24.4s, v10.4h, v9.4h\n" + "smlal2 v22.4s, v10.8h, v9.8h\n" "tbz x1, #2, 33f\n" - "ld1 { v27.s }[0], [x25], #0x4\n" + "ld1 { v12.s }[0], [x20], #0x4\n" "tbz x1, #1, 32f\n" - "ld1 { v27.h }[2], [x25], #0x2\n" + "ld1 { v12.h }[2], [x20], #0x2\n" "tbz x1, #0, 35f\n" - "ld1 { v27.b }[6], [x25]\n" + "ld1 { v12.b }[6], [x20]\n" "b 35f\n" "32:" // Oddments: Load (2, 3): Bit 2: Bit 1: Unset "tbz x1, #0, 35f\n" - "ld1 { v27.b }[4], [x25]\n" + "ld1 { v12.b }[4], [x20]\n" "b 35f\n" "33:" // Oddments: Load (2, 3): Bit 2: Unset "tbz x1, #1, 34f\n" - "ld1 { v27.h }[0], [x25], #0x2\n" + "ld1 { v12.h }[0], [x20], #0x2\n" "tbz x1, #0, 35f\n" - "ld1 { v27.b }[2], [x25]\n" + "ld1 { v12.b }[2], [x20]\n" "b 35f\n" "34:" // Oddments: Load (2, 3): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 35f\n" - "ld1 { v27.b }[0], [x25]\n" + "ld1 { v12.b }[0], [x20]\n" "35:" // Oddments: Load (2, 3): Bit 2: End - "ldr d3, [x3, #0x40]\n" - "ssubl v27.8h, v27.8b, v9.8b\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "ldr x23, [x4, #0x80]\n" - "smlal v17.4s, v27.4h, v2.4h\n" - "smlal2 v21.4s, v27.8h, v2.8h\n" - "add x23, x23, x0\n" - "smlal v13.4s, v31.4h, v3.4h\n" - "smlal2 v19.4s, v31.8h, v3.8h\n" - "smlal v20.4s, v30.4h, v3.4h\n" - "smlal2 v10.4s, v30.8h, v3.8h\n" - "smlal v8.4s, v27.4h, v3.4h\n" - "smlal2 v7.4s, v27.8h, v3.8h\n" + "ldr d31, [x6, #0x40]\n" + "ssubl v12.8h, v12.8b, v18.8b\n" + "ssubl v31.8h, v31.8b, v13.8b\n" + "ldr x20, [x5, #0x80]\n" + "smlal v23.4s, v12.4h, v9.4h\n" + "smlal2 v19.4s, v12.8h, v9.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v27.4h, v31.4h\n" + "smlal2 v15.4s, v27.8h, v31.8h\n" + "smlal v20.4s, v6.4h, v31.4h\n" + "smlal2 v5.4s, v6.8h, v31.8h\n" + "smlal v24.4s, v12.4h, v31.4h\n" + "smlal2 v22.4s, v12.8h, v31.8h\n" "tbz x1, #2, 37f\n" - "ld1 { v23.s }[0], [x23], #0x4\n" + "ld1 { v8.s }[0], [x20], #0x4\n" "tbz x1, #1, 36f\n" - "ld1 { v23.h }[2], [x23], #0x2\n" + "ld1 { v8.h }[2], [x20], #0x2\n" "tbz x1, #0, 39f\n" - "ld1 { v23.b }[6], [x23]\n" + "ld1 { v8.b }[6], [x20]\n" "b 39f\n" "36:" // Oddments: Load (2, 4): Bit 2: Bit 1: Unset "tbz x1, #0, 39f\n" - "ld1 { v23.b }[4], [x23]\n" + "ld1 { v8.b }[4], [x20]\n" "b 39f\n" "37:" // Oddments: Load (2, 4): Bit 2: Unset "tbz x1, #1, 38f\n" - "ld1 { v23.h }[0], [x23], #0x2\n" + "ld1 { v8.h }[0], [x20], #0x2\n" "tbz x1, #0, 39f\n" - "ld1 { v23.b }[2], [x23]\n" + "ld1 { v8.b }[2], [x20]\n" "b 39f\n" "38:" // Oddments: Load (2, 4): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 39f\n" - "ld1 { v23.b }[0], [x23]\n" + "ld1 { v8.b }[0], [x20]\n" "39:" // Oddments: Load (2, 4): Bit 2: End - "ldr d4, [x3, #0x48]\n" - "ssubl v23.8h, v23.8b, v9.8b\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "ldr x24, [x4, #0x88]\n" - "smlal v17.4s, v23.4h, v3.4h\n" - "smlal2 v21.4s, v23.8h, v3.8h\n" - "add x24, x24, x0\n" - "smlal v13.4s, v30.4h, v4.4h\n" - "smlal2 v19.4s, v30.8h, v4.8h\n" - "smlal v20.4s, v26.4h, v4.4h\n" - "smlal2 v10.4s, v26.8h, v4.8h\n" - "smlal v8.4s, v23.4h, v4.4h\n" - "smlal2 v7.4s, v23.8h, v4.8h\n" + "ldr d16, [x6, #0x48]\n" + "ssubl v8.8h, v8.8b, v18.8b\n" + "ssubl v16.8h, v16.8b, v13.8b\n" + "ldr x20, [x5, #0x88]\n" + "smlal v23.4s, v8.4h, v31.4h\n" + "smlal2 v19.4s, v8.8h, v31.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v6.4h, v16.4h\n" + "smlal2 v15.4s, v6.8h, v16.8h\n" + "smlal v20.4s, v29.4h, v16.4h\n" + "smlal2 v5.4s, v29.8h, v16.8h\n" + "smlal v24.4s, v8.4h, v16.4h\n" + "smlal2 v22.4s, v8.8h, v16.8h\n" "tbz x1, #2, 41f\n" - "ld1 { v28.s }[0], [x24], #0x4\n" + "ld1 { v27.s }[0], [x20], #0x4\n" "tbz x1, #1, 40f\n" - "ld1 { v28.h }[2], [x24], #0x2\n" + "ld1 { v27.h }[2], [x20], #0x2\n" "tbz x1, #0, 43f\n" - "ld1 { v28.b }[6], [x24]\n" + "ld1 { v27.b }[6], [x20]\n" "b 43f\n" "40:" // Oddments: Load (2, 5): Bit 2: Bit 1: Unset "tbz x1, #0, 43f\n" - "ld1 { v28.b }[4], [x24]\n" + "ld1 { v27.b }[4], [x20]\n" "b 43f\n" "41:" // Oddments: Load (2, 5): Bit 2: Unset "tbz x1, #1, 42f\n" - "ld1 { v28.h }[0], [x24], #0x2\n" + "ld1 { v27.h }[0], [x20], #0x2\n" "tbz x1, #0, 43f\n" - "ld1 { v28.b }[2], [x24]\n" + "ld1 { v27.b }[2], [x20]\n" "b 43f\n" "42:" // Oddments: Load (2, 5): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 43f\n" - "ld1 { v28.b }[0], [x24]\n" + "ld1 { v27.b }[0], [x20]\n" "43:" // Oddments: Load (2, 5): Bit 2: End - "ldr d0, [x3, #0x50]\n" - "ssubl v28.8h, v28.8b, v9.8b\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "ldr x15, [x4, #0x90]\n" - "smlal v17.4s, v28.4h, v4.4h\n" - "smlal2 v21.4s, v28.8h, v4.8h\n" - "add x15, x15, x0\n" - "smlal v13.4s, v22.4h, v0.4h\n" - "smlal2 v19.4s, v22.8h, v0.8h\n" - "smlal v20.4s, v25.4h, v0.4h\n" - "smlal2 v10.4s, v25.8h, v0.8h\n" + "ldr d21, [x6, #0x50]\n" + "ssubl v27.8h, v27.8b, v18.8b\n" + "ssubl v21.8h, v21.8b, v13.8b\n" + "ldr x20, [x5, #0x90]\n" + "smlal v23.4s, v27.4h, v16.4h\n" + "smlal2 v19.4s, v27.8h, v16.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v28.4h, v21.4h\n" + "smlal2 v15.4s, v28.8h, v21.8h\n" + "smlal v20.4s, v25.4h, v21.4h\n" + "smlal2 v5.4s, v25.8h, v21.8h\n" "tbz x1, #2, 45f\n" - "ld1 { v31.s }[0], [x15], #0x4\n" + "ld1 { v31.s }[0], [x20], #0x4\n" "tbz x1, #1, 44f\n" - "ld1 { v31.h }[2], [x15], #0x2\n" + "ld1 { v31.h }[2], [x20], #0x2\n" "tbz x1, #0, 47f\n" - "ld1 { v31.b }[6], [x15]\n" + "ld1 { v31.b }[6], [x20]\n" "b 47f\n" "44:" // Oddments: Load (3, 0): Bit 2: Bit 1: Unset "tbz x1, #0, 47f\n" - "ld1 { v31.b }[4], [x15]\n" + "ld1 { v31.b }[4], [x20]\n" "b 47f\n" "45:" // Oddments: Load (3, 0): Bit 2: Unset "tbz x1, #1, 46f\n" - "ld1 { v31.h }[0], [x15], #0x2\n" + "ld1 { v31.h }[0], [x20], #0x2\n" "tbz x1, #0, 47f\n" - "ld1 { v31.b }[2], [x15]\n" + "ld1 { v31.b }[2], [x20]\n" "b 47f\n" "46:" // Oddments: Load (3, 0): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 47f\n" - "ld1 { v31.b }[0], [x15]\n" + "ld1 { v31.b }[0], [x20]\n" "47:" // Oddments: Load (3, 0): Bit 2: End - "ssubl v31.8h, v31.8b, v9.8b\n" - "ldr x21, [x4, #0x98]\n" - "smlal v8.4s, v31.4h, v0.4h\n" - "smlal2 v7.4s, v31.8h, v0.8h\n" - "add x21, x21, x0\n" + "ssubl v31.8h, v31.8b, v18.8b\n" + "ldr x20, [x5, #0x98]\n" + "smlal v24.4s, v31.4h, v21.4h\n" + "smlal2 v22.4s, v31.8h, v21.8h\n" + "add x20, x20, x3\n" "tbz x1, #2, 49f\n" - "ld1 { v30.s }[0], [x21], #0x4\n" + "ld1 { v28.s }[0], [x20], #0x4\n" "tbz x1, #1, 48f\n" - "ld1 { v30.h }[2], [x21], #0x2\n" + "ld1 { v28.h }[2], [x20], #0x2\n" "tbz x1, #0, 51f\n" - "ld1 { v30.b }[6], [x21]\n" + "ld1 { v28.b }[6], [x20]\n" "b 51f\n" "48:" // Oddments: Load (3, 1): Bit 2: Bit 1: Unset "tbz x1, #0, 51f\n" - "ld1 { v30.b }[4], [x21]\n" + "ld1 { v28.b }[4], [x20]\n" "b 51f\n" "49:" // Oddments: Load (3, 1): Bit 2: Unset "tbz x1, #1, 50f\n" - "ld1 { v30.h }[0], [x21], #0x2\n" + "ld1 { v28.h }[0], [x20], #0x2\n" "tbz x1, #0, 51f\n" - "ld1 { v30.b }[2], [x21]\n" + "ld1 { v28.b }[2], [x20]\n" "b 51f\n" "50:" // Oddments: Load (3, 1): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 51f\n" - "ld1 { v30.b }[0], [x21]\n" + "ld1 { v28.b }[0], [x20]\n" "51:" // Oddments: Load (3, 1): Bit 2: End - "ldr d1, [x3, #0x58]\n" - "ssubl v30.8h, v30.8b, v9.8b\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "ldr x14, [x4, #0xa0]\n" - "smlal v17.4s, v30.4h, v0.4h\n" - "smlal2 v21.4s, v30.8h, v0.8h\n" - "add x14, x14, x0\n" - "smlal v13.4s, v25.4h, v1.4h\n" - "smlal2 v19.4s, v25.8h, v1.8h\n" - "smlal v20.4s, v24.4h, v1.4h\n" - "smlal2 v10.4s, v24.8h, v1.8h\n" - "smlal v8.4s, v30.4h, v1.4h\n" - "smlal2 v7.4s, v30.8h, v1.8h\n" + "ldr d2, [x6, #0x58]\n" + "ssubl v28.8h, v28.8b, v18.8b\n" + "ssubl v2.8h, v2.8b, v13.8b\n" + "ldr x20, [x5, #0xa0]\n" + "smlal v23.4s, v28.4h, v21.4h\n" + "smlal2 v19.4s, v28.8h, v21.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v25.4h, v2.4h\n" + "smlal2 v15.4s, v25.8h, v2.8h\n" + "smlal v20.4s, v10.4h, v2.4h\n" + "smlal2 v5.4s, v10.8h, v2.8h\n" + "smlal v24.4s, v28.4h, v2.4h\n" + "smlal2 v22.4s, v28.8h, v2.8h\n" "tbz x1, #2, 53f\n" - "ld1 { v26.s }[0], [x14], #0x4\n" + "ld1 { v21.s }[0], [x20], #0x4\n" "tbz x1, #1, 52f\n" - "ld1 { v26.h }[2], [x14], #0x2\n" + "ld1 { v21.h }[2], [x20], #0x2\n" "tbz x1, #0, 55f\n" - "ld1 { v26.b }[6], [x14]\n" + "ld1 { v21.b }[6], [x20]\n" "b 55f\n" "52:" // Oddments: Load (3, 2): Bit 2: Bit 1: Unset "tbz x1, #0, 55f\n" - "ld1 { v26.b }[4], [x14]\n" + "ld1 { v21.b }[4], [x20]\n" "b 55f\n" "53:" // Oddments: Load (3, 2): Bit 2: Unset "tbz x1, #1, 54f\n" - "ld1 { v26.h }[0], [x14], #0x2\n" + "ld1 { v21.h }[0], [x20], #0x2\n" "tbz x1, #0, 55f\n" - "ld1 { v26.b }[2], [x14]\n" + "ld1 { v21.b }[2], [x20]\n" "b 55f\n" "54:" // Oddments: Load (3, 2): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 55f\n" - "ld1 { v26.b }[0], [x14]\n" + "ld1 { v21.b }[0], [x20]\n" "55:" // Oddments: Load (3, 2): Bit 2: End - "ldr d2, [x3, #0x60]\n" - "ssubl v26.8h, v26.8b, v9.8b\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "ldr x13, [x4, #0xa8]\n" - "smlal v17.4s, v26.4h, v1.4h\n" - "smlal2 v21.4s, v26.8h, v1.8h\n" - "add x13, x13, x0\n" - "smlal v13.4s, v24.4h, v2.4h\n" - "smlal2 v19.4s, v24.8h, v2.8h\n" - "smlal v20.4s, v27.4h, v2.4h\n" - "smlal2 v10.4s, v27.8h, v2.8h\n" - "smlal v8.4s, v26.4h, v2.4h\n" - "smlal2 v7.4s, v26.8h, v2.8h\n" + "ldr d25, [x6, #0x60]\n" + "ssubl v21.8h, v21.8b, v18.8b\n" + "ssubl v25.8h, v25.8b, v13.8b\n" + "ldr x20, [x5, #0xa8]\n" + "smlal v23.4s, v21.4h, v2.4h\n" + "smlal2 v19.4s, v21.8h, v2.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v10.4h, v25.4h\n" + "smlal2 v15.4s, v10.8h, v25.8h\n" + "smlal v20.4s, v12.4h, v25.4h\n" + "smlal2 v5.4s, v12.8h, v25.8h\n" + "smlal v24.4s, v21.4h, v25.4h\n" + "smlal2 v22.4s, v21.8h, v25.8h\n" "tbz x1, #2, 57f\n" - "ld1 { v25.s }[0], [x13], #0x4\n" + "ld1 { v9.s }[0], [x20], #0x4\n" "tbz x1, #1, 56f\n" - "ld1 { v25.h }[2], [x13], #0x2\n" + "ld1 { v9.h }[2], [x20], #0x2\n" "tbz x1, #0, 59f\n" - "ld1 { v25.b }[6], [x13]\n" + "ld1 { v9.b }[6], [x20]\n" "b 59f\n" "56:" // Oddments: Load (3, 3): Bit 2: Bit 1: Unset "tbz x1, #0, 59f\n" - "ld1 { v25.b }[4], [x13]\n" + "ld1 { v9.b }[4], [x20]\n" "b 59f\n" "57:" // Oddments: Load (3, 3): Bit 2: Unset "tbz x1, #1, 58f\n" - "ld1 { v25.h }[0], [x13], #0x2\n" + "ld1 { v9.h }[0], [x20], #0x2\n" "tbz x1, #0, 59f\n" - "ld1 { v25.b }[2], [x13]\n" + "ld1 { v9.b }[2], [x20]\n" "b 59f\n" "58:" // Oddments: Load (3, 3): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 59f\n" - "ld1 { v25.b }[0], [x13]\n" + "ld1 { v9.b }[0], [x20]\n" "59:" // Oddments: Load (3, 3): Bit 2: End - "ldr d3, [x3, #0x68]\n" - "ssubl v25.8h, v25.8b, v9.8b\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "ldr x12, [x4, #0xb0]\n" - "smlal v17.4s, v25.4h, v2.4h\n" - "smlal2 v21.4s, v25.8h, v2.8h\n" - "add x12, x12, x0\n" - "smlal v13.4s, v27.4h, v3.4h\n" - "smlal2 v19.4s, v27.8h, v3.8h\n" - "smlal v20.4s, v23.4h, v3.4h\n" - "smlal2 v10.4s, v23.8h, v3.8h\n" - "smlal v8.4s, v25.4h, v3.4h\n" - "smlal2 v7.4s, v25.8h, v3.8h\n" + "ldr d1, [x6, #0x68]\n" + "ssubl v9.8h, v9.8b, v18.8b\n" + "ssubl v1.8h, v1.8b, v13.8b\n" + "ldr x20, [x5, #0xb0]\n" + "smlal v23.4s, v9.4h, v25.4h\n" + "smlal2 v19.4s, v9.8h, v25.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v12.4h, v1.4h\n" + "smlal2 v15.4s, v12.8h, v1.8h\n" + "smlal v20.4s, v8.4h, v1.4h\n" + "smlal2 v5.4s, v8.8h, v1.8h\n" + "smlal v24.4s, v9.4h, v1.4h\n" + "smlal2 v22.4s, v9.8h, v1.8h\n" "tbz x1, #2, 61f\n" - "ld1 { v24.s }[0], [x12], #0x4\n" + "ld1 { v3.s }[0], [x20], #0x4\n" "tbz x1, #1, 60f\n" - "ld1 { v24.h }[2], [x12], #0x2\n" + "ld1 { v3.h }[2], [x20], #0x2\n" "tbz x1, #0, 63f\n" - "ld1 { v24.b }[6], [x12]\n" + "ld1 { v3.b }[6], [x20]\n" "b 63f\n" "60:" // Oddments: Load (3, 4): Bit 2: Bit 1: Unset "tbz x1, #0, 63f\n" - "ld1 { v24.b }[4], [x12]\n" + "ld1 { v3.b }[4], [x20]\n" "b 63f\n" "61:" // Oddments: Load (3, 4): Bit 2: Unset "tbz x1, #1, 62f\n" - "ld1 { v24.h }[0], [x12], #0x2\n" + "ld1 { v3.h }[0], [x20], #0x2\n" "tbz x1, #0, 63f\n" - "ld1 { v24.b }[2], [x12]\n" + "ld1 { v3.b }[2], [x20]\n" "b 63f\n" "62:" // Oddments: Load (3, 4): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 63f\n" - "ld1 { v24.b }[0], [x12]\n" + "ld1 { v3.b }[0], [x20]\n" "63:" // Oddments: Load (3, 4): Bit 2: End - "ldr d4, [x3, #0x70]\n" - "ssubl v24.8h, v24.8b, v9.8b\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "ldr x20, [x4, #0xb8]\n" - "smlal v17.4s, v24.4h, v3.4h\n" - "smlal2 v21.4s, v24.8h, v3.8h\n" - "add x20, x20, x0\n" - "smlal v13.4s, v23.4h, v4.4h\n" - "smlal2 v19.4s, v23.8h, v4.8h\n" - "smlal v20.4s, v28.4h, v4.4h\n" - "smlal2 v10.4s, v28.8h, v4.8h\n" - "smlal v8.4s, v24.4h, v4.4h\n" - "smlal2 v7.4s, v24.8h, v4.8h\n" + "ldr d16, [x6, #0x70]\n" + "ssubl v3.8h, v3.8b, v18.8b\n" + "ssubl v16.8h, v16.8b, v13.8b\n" + "ldr x20, [x5, #0xb8]\n" + "smlal v23.4s, v3.4h, v1.4h\n" + "smlal2 v19.4s, v3.8h, v1.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v8.4h, v16.4h\n" + "smlal2 v15.4s, v8.8h, v16.8h\n" + "smlal v20.4s, v27.4h, v16.4h\n" + "smlal2 v5.4s, v27.8h, v16.8h\n" + "smlal v24.4s, v3.4h, v16.4h\n" + "smlal2 v22.4s, v3.8h, v16.8h\n" "tbz x1, #2, 65f\n" - "ld1 { v22.s }[0], [x20], #0x4\n" + "ld1 { v14.s }[0], [x20], #0x4\n" "tbz x1, #1, 64f\n" - "ld1 { v22.h }[2], [x20], #0x2\n" + "ld1 { v14.h }[2], [x20], #0x2\n" "tbz x1, #0, 67f\n" - "ld1 { v22.b }[6], [x20]\n" + "ld1 { v14.b }[6], [x20]\n" "b 67f\n" "64:" // Oddments: Load (3, 5): Bit 2: Bit 1: Unset "tbz x1, #0, 67f\n" - "ld1 { v22.b }[4], [x20]\n" + "ld1 { v14.b }[4], [x20]\n" "b 67f\n" "65:" // Oddments: Load (3, 5): Bit 2: Unset "tbz x1, #1, 66f\n" - "ld1 { v22.h }[0], [x20], #0x2\n" + "ld1 { v14.h }[0], [x20], #0x2\n" "tbz x1, #0, 67f\n" - "ld1 { v22.b }[2], [x20]\n" + "ld1 { v14.b }[2], [x20]\n" "b 67f\n" "66:" // Oddments: Load (3, 5): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 67f\n" - "ld1 { v22.b }[0], [x20]\n" + "ld1 { v14.b }[0], [x20]\n" "67:" // Oddments: Load (3, 5): Bit 2: End - "ldr d0, [x3, #0x78]\n" - "ssubl v22.8h, v22.8b, v9.8b\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "ldr x11, [x4, #0xc0]\n" - "smlal v17.4s, v22.4h, v4.4h\n" - "smlal2 v21.4s, v22.8h, v4.8h\n" - "add x11, x11, x0\n" - "smlal v13.4s, v31.4h, v0.4h\n" - "smlal2 v19.4s, v31.8h, v0.8h\n" - "smlal v20.4s, v30.4h, v0.4h\n" - "smlal2 v10.4s, v30.8h, v0.8h\n" + "ldr d17, [x6, #0x78]\n" + "ssubl v14.8h, v14.8b, v18.8b\n" + "ssubl v17.8h, v17.8b, v13.8b\n" + "ldr x20, [x5, #0xc0]\n" + "smlal v23.4s, v14.4h, v16.4h\n" + "smlal2 v19.4s, v14.8h, v16.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v31.4h, v17.4h\n" + "smlal2 v15.4s, v31.8h, v17.8h\n" + "smlal v20.4s, v28.4h, v17.4h\n" + "smlal2 v5.4s, v28.8h, v17.8h\n" "tbz x1, #2, 69f\n" - "ld1 { v27.s }[0], [x11], #0x4\n" + "ld1 { v1.s }[0], [x20], #0x4\n" "tbz x1, #1, 68f\n" - "ld1 { v27.h }[2], [x11], #0x2\n" + "ld1 { v1.h }[2], [x20], #0x2\n" "tbz x1, #0, 71f\n" - "ld1 { v27.b }[6], [x11]\n" + "ld1 { v1.b }[6], [x20]\n" "b 71f\n" "68:" // Oddments: Load (4, 0): Bit 2: Bit 1: Unset "tbz x1, #0, 71f\n" - "ld1 { v27.b }[4], [x11]\n" + "ld1 { v1.b }[4], [x20]\n" "b 71f\n" "69:" // Oddments: Load (4, 0): Bit 2: Unset "tbz x1, #1, 70f\n" - "ld1 { v27.h }[0], [x11], #0x2\n" + "ld1 { v1.h }[0], [x20], #0x2\n" "tbz x1, #0, 71f\n" - "ld1 { v27.b }[2], [x11]\n" + "ld1 { v1.b }[2], [x20]\n" "b 71f\n" "70:" // Oddments: Load (4, 0): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 71f\n" - "ld1 { v27.b }[0], [x11]\n" + "ld1 { v1.b }[0], [x20]\n" "71:" // Oddments: Load (4, 0): Bit 2: End - "ssubl v27.8h, v27.8b, v9.8b\n" - "ldr x22, [x4, #0xc8]\n" - "smlal v8.4s, v27.4h, v0.4h\n" - "smlal2 v7.4s, v27.8h, v0.8h\n" - "add x22, x22, x0\n" + "ssubl v1.8h, v1.8b, v18.8b\n" + "ldr x20, [x5, #0xc8]\n" + "smlal v24.4s, v1.4h, v17.4h\n" + "smlal2 v22.4s, v1.8h, v17.8h\n" + "add x20, x20, x3\n" "tbz x1, #2, 73f\n" - "ld1 { v23.s }[0], [x22], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x1, #1, 72f\n" - "ld1 { v23.h }[2], [x22], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x1, #0, 75f\n" - "ld1 { v23.b }[6], [x22]\n" + "ld1 { v16.b }[6], [x20]\n" "b 75f\n" "72:" // Oddments: Load (4, 1): Bit 2: Bit 1: Unset "tbz x1, #0, 75f\n" - "ld1 { v23.b }[4], [x22]\n" + "ld1 { v16.b }[4], [x20]\n" "b 75f\n" "73:" // Oddments: Load (4, 1): Bit 2: Unset "tbz x1, #1, 74f\n" - "ld1 { v23.h }[0], [x22], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x1, #0, 75f\n" - "ld1 { v23.b }[2], [x22]\n" + "ld1 { v16.b }[2], [x20]\n" "b 75f\n" "74:" // Oddments: Load (4, 1): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 75f\n" - "ld1 { v23.b }[0], [x22]\n" + "ld1 { v16.b }[0], [x20]\n" "75:" // Oddments: Load (4, 1): Bit 2: End - "ldr d1, [x3, #0x80]\n" - "ssubl v23.8h, v23.8b, v9.8b\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "ldr x9, [x4, #0xd0]\n" - "smlal v17.4s, v23.4h, v0.4h\n" - "smlal2 v21.4s, v23.8h, v0.8h\n" - "add x9, x9, x0\n" - "smlal v13.4s, v30.4h, v1.4h\n" - "smlal2 v19.4s, v30.8h, v1.8h\n" - "smlal v20.4s, v26.4h, v1.4h\n" - "smlal2 v10.4s, v26.8h, v1.8h\n" - "smlal v8.4s, v23.4h, v1.4h\n" - "smlal2 v7.4s, v23.8h, v1.8h\n" + "ldr d29, [x6, #0x80]\n" + "ssubl v16.8h, v16.8b, v18.8b\n" + "ssubl v29.8h, v29.8b, v13.8b\n" + "ldr x20, [x5, #0xd0]\n" + "smlal v23.4s, v16.4h, v17.4h\n" + "smlal2 v19.4s, v16.8h, v17.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v28.4h, v29.4h\n" + "smlal2 v15.4s, v28.8h, v29.8h\n" + "smlal v20.4s, v21.4h, v29.4h\n" + "smlal2 v5.4s, v21.8h, v29.8h\n" + "smlal v24.4s, v16.4h, v29.4h\n" + "smlal2 v22.4s, v16.8h, v29.8h\n" "tbz x1, #2, 77f\n" - "ld1 { v31.s }[0], [x9], #0x4\n" + "ld1 { v30.s }[0], [x20], #0x4\n" "tbz x1, #1, 76f\n" - "ld1 { v31.h }[2], [x9], #0x2\n" + "ld1 { v30.h }[2], [x20], #0x2\n" "tbz x1, #0, 79f\n" - "ld1 { v31.b }[6], [x9]\n" + "ld1 { v30.b }[6], [x20]\n" "b 79f\n" "76:" // Oddments: Load (4, 2): Bit 2: Bit 1: Unset "tbz x1, #0, 79f\n" - "ld1 { v31.b }[4], [x9]\n" + "ld1 { v30.b }[4], [x20]\n" "b 79f\n" "77:" // Oddments: Load (4, 2): Bit 2: Unset "tbz x1, #1, 78f\n" - "ld1 { v31.h }[0], [x9], #0x2\n" + "ld1 { v30.h }[0], [x20], #0x2\n" "tbz x1, #0, 79f\n" - "ld1 { v31.b }[2], [x9]\n" + "ld1 { v30.b }[2], [x20]\n" "b 79f\n" "78:" // Oddments: Load (4, 2): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 79f\n" - "ld1 { v31.b }[0], [x9]\n" + "ld1 { v30.b }[0], [x20]\n" "79:" // Oddments: Load (4, 2): Bit 2: End - "ldr d2, [x3, #0x88]\n" - "ssubl v31.8h, v31.8b, v9.8b\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "ldr x28, [x4, #0xd8]\n" - "smlal v17.4s, v31.4h, v1.4h\n" - "smlal2 v21.4s, v31.8h, v1.8h\n" - "add x28, x28, x0\n" - "smlal v13.4s, v26.4h, v2.4h\n" - "smlal2 v19.4s, v26.8h, v2.8h\n" - "smlal v20.4s, v25.4h, v2.4h\n" - "smlal2 v10.4s, v25.8h, v2.8h\n" - "smlal v8.4s, v31.4h, v2.4h\n" - "smlal2 v7.4s, v31.8h, v2.8h\n" + "ldr d12, [x6, #0x88]\n" + "ssubl v30.8h, v30.8b, v18.8b\n" + "ssubl v12.8h, v12.8b, v13.8b\n" + "ldr x20, [x5, #0xd8]\n" + "smlal v23.4s, v30.4h, v29.4h\n" + "smlal2 v19.4s, v30.8h, v29.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v21.4h, v12.4h\n" + "smlal2 v15.4s, v21.8h, v12.8h\n" + "smlal v20.4s, v9.4h, v12.4h\n" + "smlal2 v5.4s, v9.8h, v12.8h\n" + "smlal v24.4s, v30.4h, v12.4h\n" + "smlal2 v22.4s, v30.8h, v12.8h\n" "tbz x1, #2, 81f\n" - "ld1 { v30.s }[0], [x28], #0x4\n" + "ld1 { v29.s }[0], [x20], #0x4\n" "tbz x1, #1, 80f\n" - "ld1 { v30.h }[2], [x28], #0x2\n" + "ld1 { v29.h }[2], [x20], #0x2\n" "tbz x1, #0, 83f\n" - "ld1 { v30.b }[6], [x28]\n" + "ld1 { v29.b }[6], [x20]\n" "b 83f\n" "80:" // Oddments: Load (4, 3): Bit 2: Bit 1: Unset "tbz x1, #0, 83f\n" - "ld1 { v30.b }[4], [x28]\n" + "ld1 { v29.b }[4], [x20]\n" "b 83f\n" "81:" // Oddments: Load (4, 3): Bit 2: Unset "tbz x1, #1, 82f\n" - "ld1 { v30.h }[0], [x28], #0x2\n" + "ld1 { v29.h }[0], [x20], #0x2\n" "tbz x1, #0, 83f\n" - "ld1 { v30.b }[2], [x28]\n" + "ld1 { v29.b }[2], [x20]\n" "b 83f\n" "82:" // Oddments: Load (4, 3): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 83f\n" - "ld1 { v30.b }[0], [x28]\n" + "ld1 { v29.b }[0], [x20]\n" "83:" // Oddments: Load (4, 3): Bit 2: End - "ldr d3, [x3, #0x90]\n" - "ssubl v30.8h, v30.8b, v9.8b\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "ldr x27, [x4, #0xe0]\n" - "smlal v17.4s, v30.4h, v2.4h\n" - "smlal2 v21.4s, v30.8h, v2.8h\n" - "add x27, x27, x0\n" - "smlal v13.4s, v25.4h, v3.4h\n" - "smlal2 v19.4s, v25.8h, v3.8h\n" - "smlal v20.4s, v24.4h, v3.4h\n" - "smlal2 v10.4s, v24.8h, v3.8h\n" - "smlal v8.4s, v30.4h, v3.4h\n" - "smlal2 v7.4s, v30.8h, v3.8h\n" + "ldr d21, [x6, #0x90]\n" + "ssubl v29.8h, v29.8b, v18.8b\n" + "ssubl v21.8h, v21.8b, v13.8b\n" + "ldr x20, [x5, #0xe0]\n" + "smlal v23.4s, v29.4h, v12.4h\n" + "smlal2 v19.4s, v29.8h, v12.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v9.4h, v21.4h\n" + "smlal2 v15.4s, v9.8h, v21.8h\n" + "smlal v20.4s, v3.4h, v21.4h\n" + "smlal2 v5.4s, v3.8h, v21.8h\n" + "smlal v24.4s, v29.4h, v21.4h\n" + "smlal2 v22.4s, v29.8h, v21.8h\n" "tbz x1, #2, 85f\n" - "ld1 { v28.s }[0], [x27], #0x4\n" + "ld1 { v25.s }[0], [x20], #0x4\n" "tbz x1, #1, 84f\n" - "ld1 { v28.h }[2], [x27], #0x2\n" + "ld1 { v25.h }[2], [x20], #0x2\n" "tbz x1, #0, 87f\n" - "ld1 { v28.b }[6], [x27]\n" + "ld1 { v25.b }[6], [x20]\n" "b 87f\n" "84:" // Oddments: Load (4, 4): Bit 2: Bit 1: Unset "tbz x1, #0, 87f\n" - "ld1 { v28.b }[4], [x27]\n" + "ld1 { v25.b }[4], [x20]\n" "b 87f\n" "85:" // Oddments: Load (4, 4): Bit 2: Unset "tbz x1, #1, 86f\n" - "ld1 { v28.h }[0], [x27], #0x2\n" + "ld1 { v25.h }[0], [x20], #0x2\n" "tbz x1, #0, 87f\n" - "ld1 { v28.b }[2], [x27]\n" + "ld1 { v25.b }[2], [x20]\n" "b 87f\n" "86:" // Oddments: Load (4, 4): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 87f\n" - "ld1 { v28.b }[0], [x27]\n" + "ld1 { v25.b }[0], [x20]\n" "87:" // Oddments: Load (4, 4): Bit 2: End - "ldr d4, [x3, #0x98]\n" - "ssubl v28.8h, v28.8b, v9.8b\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "ldr x26, [x4, #0xe8]\n" - "smlal v17.4s, v28.4h, v3.4h\n" - "smlal2 v21.4s, v28.8h, v3.8h\n" - "add x26, x26, x0\n" - "smlal v13.4s, v24.4h, v4.4h\n" - "smlal2 v19.4s, v24.8h, v4.8h\n" - "smlal v20.4s, v22.4h, v4.4h\n" - "smlal2 v10.4s, v22.8h, v4.8h\n" - "smlal v8.4s, v28.4h, v4.4h\n" - "smlal2 v7.4s, v28.8h, v4.8h\n" + "ldr d8, [x6, #0x98]\n" + "ssubl v25.8h, v25.8b, v18.8b\n" + "ssubl v8.8h, v8.8b, v13.8b\n" + "ldr x20, [x5, #0xe8]\n" + "smlal v23.4s, v25.4h, v21.4h\n" + "smlal2 v19.4s, v25.8h, v21.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v3.4h, v8.4h\n" + "smlal2 v15.4s, v3.8h, v8.8h\n" + "smlal v20.4s, v14.4h, v8.4h\n" + "smlal2 v5.4s, v14.8h, v8.8h\n" + "smlal v24.4s, v25.4h, v8.4h\n" + "smlal2 v22.4s, v25.8h, v8.8h\n" "tbz x1, #2, 89f\n" - "ld1 { v26.s }[0], [x26], #0x4\n" + "ld1 { v21.s }[0], [x20], #0x4\n" "tbz x1, #1, 88f\n" - "ld1 { v26.h }[2], [x26], #0x2\n" + "ld1 { v21.h }[2], [x20], #0x2\n" "tbz x1, #0, 91f\n" - "ld1 { v26.b }[6], [x26]\n" + "ld1 { v21.b }[6], [x20]\n" "b 91f\n" "88:" // Oddments: Load (4, 5): Bit 2: Bit 1: Unset "tbz x1, #0, 91f\n" - "ld1 { v26.b }[4], [x26]\n" + "ld1 { v21.b }[4], [x20]\n" "b 91f\n" "89:" // Oddments: Load (4, 5): Bit 2: Unset "tbz x1, #1, 90f\n" - "ld1 { v26.h }[0], [x26], #0x2\n" + "ld1 { v21.h }[0], [x20], #0x2\n" "tbz x1, #0, 91f\n" - "ld1 { v26.b }[2], [x26]\n" + "ld1 { v21.b }[2], [x20]\n" "b 91f\n" "90:" // Oddments: Load (4, 5): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 91f\n" - "ld1 { v26.b }[0], [x26]\n" + "ld1 { v21.b }[0], [x20]\n" "91:" // Oddments: Load (4, 5): Bit 2: End - "ldr d0, [x3, #0xa0]\n" - "ssubl v26.8h, v26.8b, v9.8b\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "ldr x25, [x4, #0xf0]\n" - "smlal v17.4s, v26.4h, v4.4h\n" - "smlal2 v21.4s, v26.8h, v4.8h\n" - "add x25, x25, x0\n" - "smlal v13.4s, v27.4h, v0.4h\n" - "smlal2 v19.4s, v27.8h, v0.8h\n" - "smlal v20.4s, v23.4h, v0.4h\n" - "smlal2 v10.4s, v23.8h, v0.8h\n" + "ldr d9, [x6, #0xa0]\n" + "ssubl v21.8h, v21.8b, v18.8b\n" + "ssubl v9.8h, v9.8b, v13.8b\n" + "ldr x20, [x5, #0xf0]\n" + "smlal v23.4s, v21.4h, v8.4h\n" + "smlal2 v19.4s, v21.8h, v8.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v1.4h, v9.4h\n" + "smlal2 v15.4s, v1.8h, v9.8h\n" + "smlal v20.4s, v16.4h, v9.4h\n" + "smlal2 v5.4s, v16.8h, v9.8h\n" "tbz x1, #2, 93f\n" - "ld1 { v25.s }[0], [x25], #0x4\n" + "ld1 { v12.s }[0], [x20], #0x4\n" "tbz x1, #1, 92f\n" - "ld1 { v25.h }[2], [x25], #0x2\n" + "ld1 { v12.h }[2], [x20], #0x2\n" "tbz x1, #0, 95f\n" - "ld1 { v25.b }[6], [x25]\n" + "ld1 { v12.b }[6], [x20]\n" "b 95f\n" "92:" // Oddments: Load (5, 0): Bit 2: Bit 1: Unset "tbz x1, #0, 95f\n" - "ld1 { v25.b }[4], [x25]\n" + "ld1 { v12.b }[4], [x20]\n" "b 95f\n" "93:" // Oddments: Load (5, 0): Bit 2: Unset "tbz x1, #1, 94f\n" - "ld1 { v25.h }[0], [x25], #0x2\n" + "ld1 { v12.h }[0], [x20], #0x2\n" "tbz x1, #0, 95f\n" - "ld1 { v25.b }[2], [x25]\n" + "ld1 { v12.b }[2], [x20]\n" "b 95f\n" "94:" // Oddments: Load (5, 0): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 95f\n" - "ld1 { v25.b }[0], [x25]\n" + "ld1 { v12.b }[0], [x20]\n" "95:" // Oddments: Load (5, 0): Bit 2: End - "ssubl v25.8h, v25.8b, v9.8b\n" - "ldr x24, [x4, #0xf8]\n" - "smlal v8.4s, v25.4h, v0.4h\n" - "smlal2 v7.4s, v25.8h, v0.8h\n" - "add x24, x24, x0\n" + "ssubl v12.8h, v12.8b, v18.8b\n" + "ldr x20, [x5, #0xf8]\n" + "smlal v24.4s, v12.4h, v9.4h\n" + "smlal2 v22.4s, v12.8h, v9.8h\n" + "add x20, x20, x3\n" "tbz x1, #2, 97f\n" - "ld1 { v24.s }[0], [x24], #0x4\n" + "ld1 { v10.s }[0], [x20], #0x4\n" "tbz x1, #1, 96f\n" - "ld1 { v24.h }[2], [x24], #0x2\n" + "ld1 { v10.h }[2], [x20], #0x2\n" "tbz x1, #0, 99f\n" - "ld1 { v24.b }[6], [x24]\n" + "ld1 { v10.b }[6], [x20]\n" "b 99f\n" "96:" // Oddments: Load (5, 1): Bit 2: Bit 1: Unset "tbz x1, #0, 99f\n" - "ld1 { v24.b }[4], [x24]\n" + "ld1 { v10.b }[4], [x20]\n" "b 99f\n" "97:" // Oddments: Load (5, 1): Bit 2: Unset "tbz x1, #1, 98f\n" - "ld1 { v24.h }[0], [x24], #0x2\n" + "ld1 { v10.h }[0], [x20], #0x2\n" "tbz x1, #0, 99f\n" - "ld1 { v24.b }[2], [x24]\n" + "ld1 { v10.b }[2], [x20]\n" "b 99f\n" "98:" // Oddments: Load (5, 1): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 99f\n" - "ld1 { v24.b }[0], [x24]\n" + "ld1 { v10.b }[0], [x20]\n" "99:" // Oddments: Load (5, 1): Bit 2: End - "ldr d1, [x3, #0xa8]\n" - "ssubl v24.8h, v24.8b, v9.8b\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "ldr x23, [x4, #0x100]\n" - "smlal v17.4s, v24.4h, v0.4h\n" - "smlal2 v21.4s, v24.8h, v0.8h\n" - "add x23, x23, x0\n" - "smlal v13.4s, v23.4h, v1.4h\n" - "smlal2 v19.4s, v23.8h, v1.8h\n" - "smlal v20.4s, v31.4h, v1.4h\n" - "smlal2 v10.4s, v31.8h, v1.8h\n" - "smlal v8.4s, v24.4h, v1.4h\n" - "smlal2 v7.4s, v24.8h, v1.8h\n" + "ldr d12, [x6, #0xa8]\n" + "ssubl v10.8h, v10.8b, v18.8b\n" + "ssubl v12.8h, v12.8b, v13.8b\n" + "ldr x20, [x5, #0x100]\n" + "smlal v23.4s, v10.4h, v9.4h\n" + "smlal2 v19.4s, v10.8h, v9.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v16.4h, v12.4h\n" + "smlal2 v15.4s, v16.8h, v12.8h\n" + "smlal v20.4s, v30.4h, v12.4h\n" + "smlal2 v5.4s, v30.8h, v12.8h\n" + "smlal v24.4s, v10.4h, v12.4h\n" + "smlal2 v22.4s, v10.8h, v12.8h\n" "tbz x1, #2, 101f\n" - "ld1 { v27.s }[0], [x23], #0x4\n" + "ld1 { v9.s }[0], [x20], #0x4\n" "tbz x1, #1, 100f\n" - "ld1 { v27.h }[2], [x23], #0x2\n" + "ld1 { v9.h }[2], [x20], #0x2\n" "tbz x1, #0, 103f\n" - "ld1 { v27.b }[6], [x23]\n" + "ld1 { v9.b }[6], [x20]\n" "b 103f\n" "100:" // Oddments: Load (5, 2): Bit 2: Bit 1: Unset "tbz x1, #0, 103f\n" - "ld1 { v27.b }[4], [x23]\n" + "ld1 { v9.b }[4], [x20]\n" "b 103f\n" "101:" // Oddments: Load (5, 2): Bit 2: Unset "tbz x1, #1, 102f\n" - "ld1 { v27.h }[0], [x23], #0x2\n" + "ld1 { v9.h }[0], [x20], #0x2\n" "tbz x1, #0, 103f\n" - "ld1 { v27.b }[2], [x23]\n" + "ld1 { v9.b }[2], [x20]\n" "b 103f\n" "102:" // Oddments: Load (5, 2): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 103f\n" - "ld1 { v27.b }[0], [x23]\n" + "ld1 { v9.b }[0], [x20]\n" "103:" // Oddments: Load (5, 2): Bit 2: End - "ldr d2, [x3, #0xb0]\n" - "ssubl v27.8h, v27.8b, v9.8b\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "ldr x15, [x4, #0x108]\n" - "smlal v17.4s, v27.4h, v1.4h\n" - "smlal2 v21.4s, v27.8h, v1.8h\n" - "add x15, x15, x0\n" - "smlal v13.4s, v31.4h, v2.4h\n" - "smlal2 v19.4s, v31.8h, v2.8h\n" - "smlal v20.4s, v30.4h, v2.4h\n" - "smlal2 v10.4s, v30.8h, v2.8h\n" - "smlal v8.4s, v27.4h, v2.4h\n" - "smlal2 v7.4s, v27.8h, v2.8h\n" + "ldr d28, [x6, #0xb0]\n" + "ssubl v9.8h, v9.8b, v18.8b\n" + "ssubl v28.8h, v28.8b, v13.8b\n" + "ldr x20, [x5, #0x108]\n" + "smlal v23.4s, v9.4h, v12.4h\n" + "smlal2 v19.4s, v9.8h, v12.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v30.4h, v28.4h\n" + "smlal2 v15.4s, v30.8h, v28.8h\n" + "smlal v20.4s, v29.4h, v28.4h\n" + "smlal2 v5.4s, v29.8h, v28.8h\n" + "smlal v24.4s, v9.4h, v28.4h\n" + "smlal2 v22.4s, v9.8h, v28.8h\n" "tbz x1, #2, 105f\n" - "ld1 { v25.s }[0], [x15], #0x4\n" + "ld1 { v2.s }[0], [x20], #0x4\n" "tbz x1, #1, 104f\n" - "ld1 { v25.h }[2], [x15], #0x2\n" + "ld1 { v2.h }[2], [x20], #0x2\n" "tbz x1, #0, 107f\n" - "ld1 { v25.b }[6], [x15]\n" + "ld1 { v2.b }[6], [x20]\n" "b 107f\n" "104:" // Oddments: Load (5, 3): Bit 2: Bit 1: Unset "tbz x1, #0, 107f\n" - "ld1 { v25.b }[4], [x15]\n" + "ld1 { v2.b }[4], [x20]\n" "b 107f\n" "105:" // Oddments: Load (5, 3): Bit 2: Unset "tbz x1, #1, 106f\n" - "ld1 { v25.h }[0], [x15], #0x2\n" + "ld1 { v2.h }[0], [x20], #0x2\n" "tbz x1, #0, 107f\n" - "ld1 { v25.b }[2], [x15]\n" + "ld1 { v2.b }[2], [x20]\n" "b 107f\n" "106:" // Oddments: Load (5, 3): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 107f\n" - "ld1 { v25.b }[0], [x15]\n" + "ld1 { v2.b }[0], [x20]\n" "107:" // Oddments: Load (5, 3): Bit 2: End - "ldr d3, [x3, #0xb8]\n" - "ssubl v25.8h, v25.8b, v9.8b\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "ldr x21, [x4, #0x110]\n" - "smlal v17.4s, v25.4h, v2.4h\n" - "smlal2 v21.4s, v25.8h, v2.8h\n" - "add x21, x21, x0\n" - "smlal v13.4s, v30.4h, v3.4h\n" - "smlal2 v19.4s, v30.8h, v3.8h\n" - "smlal v20.4s, v28.4h, v3.4h\n" - "smlal2 v10.4s, v28.8h, v3.8h\n" - "smlal v8.4s, v25.4h, v3.4h\n" - "smlal2 v7.4s, v25.8h, v3.8h\n" + "ldr d30, [x6, #0xb8]\n" + "ssubl v2.8h, v2.8b, v18.8b\n" + "ssubl v30.8h, v30.8b, v13.8b\n" + "ldr x20, [x5, #0x110]\n" + "smlal v23.4s, v2.4h, v28.4h\n" + "smlal2 v19.4s, v2.8h, v28.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v29.4h, v30.4h\n" + "smlal2 v15.4s, v29.8h, v30.8h\n" + "smlal v20.4s, v25.4h, v30.4h\n" + "smlal2 v5.4s, v25.8h, v30.8h\n" + "smlal v24.4s, v2.4h, v30.4h\n" + "smlal2 v22.4s, v2.8h, v30.8h\n" "tbz x1, #2, 109f\n" - "ld1 { v24.s }[0], [x21], #0x4\n" + "ld1 { v27.s }[0], [x20], #0x4\n" "tbz x1, #1, 108f\n" - "ld1 { v24.h }[2], [x21], #0x2\n" + "ld1 { v27.h }[2], [x20], #0x2\n" "tbz x1, #0, 111f\n" - "ld1 { v24.b }[6], [x21]\n" + "ld1 { v27.b }[6], [x20]\n" "b 111f\n" "108:" // Oddments: Load (5, 4): Bit 2: Bit 1: Unset "tbz x1, #0, 111f\n" - "ld1 { v24.b }[4], [x21]\n" + "ld1 { v27.b }[4], [x20]\n" "b 111f\n" "109:" // Oddments: Load (5, 4): Bit 2: Unset "tbz x1, #1, 110f\n" - "ld1 { v24.h }[0], [x21], #0x2\n" + "ld1 { v27.h }[0], [x20], #0x2\n" "tbz x1, #0, 111f\n" - "ld1 { v24.b }[2], [x21]\n" + "ld1 { v27.b }[2], [x20]\n" "b 111f\n" "110:" // Oddments: Load (5, 4): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 111f\n" - "ld1 { v24.b }[0], [x21]\n" + "ld1 { v27.b }[0], [x20]\n" "111:" // Oddments: Load (5, 4): Bit 2: End - "ldr d4, [x3, #0xc0]\n" - "ssubl v24.8h, v24.8b, v9.8b\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "ldr x20, [x4, #0x118]\n" - "smlal v17.4s, v24.4h, v3.4h\n" - "smlal2 v21.4s, v24.8h, v3.8h\n" - "add x20, x20, x0\n" - "smlal v13.4s, v28.4h, v4.4h\n" - "smlal2 v19.4s, v28.8h, v4.8h\n" - "smlal v20.4s, v26.4h, v4.4h\n" - "smlal2 v10.4s, v26.8h, v4.8h\n" - "smlal v8.4s, v24.4h, v4.4h\n" - "smlal2 v7.4s, v24.8h, v4.8h\n" + "ldr d8, [x6, #0xc0]\n" + "ssubl v27.8h, v27.8b, v18.8b\n" + "ssubl v8.8h, v8.8b, v13.8b\n" + "ldr x20, [x5, #0x118]\n" + "smlal v23.4s, v27.4h, v30.4h\n" + "smlal2 v19.4s, v27.8h, v30.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v25.4h, v8.4h\n" + "smlal2 v15.4s, v25.8h, v8.8h\n" + "smlal v20.4s, v21.4h, v8.4h\n" + "smlal2 v5.4s, v21.8h, v8.8h\n" + "smlal v24.4s, v27.4h, v8.4h\n" + "smlal2 v22.4s, v27.8h, v8.8h\n" "tbz x1, #2, 113f\n" - "ld1 { v27.s }[0], [x20], #0x4\n" + "ld1 { v9.s }[0], [x20], #0x4\n" "tbz x1, #1, 112f\n" - "ld1 { v27.h }[2], [x20], #0x2\n" + "ld1 { v9.h }[2], [x20], #0x2\n" "tbz x1, #0, 115f\n" - "ld1 { v27.b }[6], [x20]\n" + "ld1 { v9.b }[6], [x20]\n" "b 115f\n" "112:" // Oddments: Load (5, 5): Bit 2: Bit 1: Unset "tbz x1, #0, 115f\n" - "ld1 { v27.b }[4], [x20]\n" + "ld1 { v9.b }[4], [x20]\n" "b 115f\n" "113:" // Oddments: Load (5, 5): Bit 2: Unset "tbz x1, #1, 114f\n" - "ld1 { v27.h }[0], [x20], #0x2\n" + "ld1 { v9.h }[0], [x20], #0x2\n" "tbz x1, #0, 115f\n" - "ld1 { v27.b }[2], [x20]\n" + "ld1 { v9.b }[2], [x20]\n" "b 115f\n" "114:" // Oddments: Load (5, 5): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 115f\n" - "ld1 { v27.b }[0], [x20]\n" + "ld1 { v9.b }[0], [x20]\n" "115:" // Oddments: Load (5, 5): Bit 2: End - "ssubl v27.8h, v27.8b, v9.8b\n" - "smlal v17.4s, v27.4h, v4.4h\n" - "smlal2 v21.4s, v27.8h, v4.8h\n" + "ssubl v9.8h, v9.8b, v18.8b\n" + "smlal v23.4s, v9.4h, v8.4h\n" + "smlal2 v19.4s, v9.8h, v8.8h\n" "tbz x1, #2, 117f\n" - "ld1 { v18.4s }, [x5], #0x10\n" - "ld1 { v6.4s }, [x8], #0x10\n" + "ld1 { v30.4s }, [x7], #0x10\n" + "ld1 { v12.4s }, [x8], #0x10\n" "tbz x1, #1, 116f\n" - "ld1 { v5.d }[0], [x5], #0x8\n" - "ld1 { v22.d }[0], [x8], #0x8\n" + "ld1 { v14.d }[0], [x7], #0x8\n" + "ld1 { v27.d }[0], [x8], #0x8\n" "tbz x1, #0, 119f\n" - "ld1 { v5.s }[2], [x5]\n" - "ld1 { v22.s }[2], [x8]\n" + "ld1 { v14.s }[2], [x7]\n" + "ld1 { v27.s }[2], [x8]\n" "b 119f\n" "116:" // Oddments: Load requant params: Bit 2: Bit 1: Unset "tbz x1, #0, 119f\n" - "ld1 { v5.s }[0], [x5]\n" - "ld1 { v22.s }[0], [x8]\n" + "ld1 { v14.s }[0], [x7]\n" + "ld1 { v27.s }[0], [x8]\n" "b 119f\n" "117:" // Oddments: Load requant params: Bit 2: Unset "tbz x1, #1, 118f\n" - "ld1 { v18.d }[0], [x5], #0x8\n" - "ld1 { v6.d }[0], [x8], #0x8\n" + "ld1 { v30.d }[0], [x7], #0x8\n" + "ld1 { v12.d }[0], [x8], #0x8\n" "tbz x1, #0, 119f\n" - "ld1 { v18.s }[2], [x5]\n" - "ld1 { v6.s }[2], [x8]\n" + "ld1 { v30.s }[2], [x7]\n" + "ld1 { v12.s }[2], [x8]\n" "b 119f\n" "118:" // Oddments: Load requant params: Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 119f\n" - "ld1 { v18.s }[0], [x5]\n" - "ld1 { v6.s }[0], [x8]\n" + "ld1 { v30.s }[0], [x7]\n" + "ld1 { v12.s }[0], [x8]\n" "119:" // Oddments: Load requant params: Bit 2: End - "sqrdmulh v13.4s, v13.4s, v18.4s\n" - "and v30.16b, v13.16b, v6.16b\n" - "add x17, x17, x10\n" - "add x6, x6, x10\n" - "sqrdmulh v19.4s, v19.4s, v5.4s\n" - "sshr v30.4s, v30.4s, #0x1f\n" - "add x7, x7, x10\n" - "add x16, x16, x10\n" - "and v16.16b, v19.16b, v22.16b\n" - "sqrdmulh v20.4s, v20.4s, v18.4s\n" - "sqrdmulh v8.4s, v8.4s, v18.4s\n" - "sqrdmulh v17.4s, v17.4s, v18.4s\n" - "sqadd v13.4s, v13.4s, v30.4s\n" + "sqrdmulh v7.4s, v7.4s, v30.4s\n" + "and v16.16b, v7.16b, v12.16b\n" + "add x17, x17, x4\n" + "add x16, x16, x4\n" + "sqrdmulh v15.4s, v15.4s, v14.4s\n" "sshr v16.4s, v16.4s, #0x1f\n" - "and v0.16b, v20.16b, v6.16b\n" - "sqrdmulh v10.4s, v10.4s, v5.4s\n" - "and v18.16b, v8.16b, v6.16b\n" - "sqrdmulh v7.4s, v7.4s, v5.4s\n" - "and v30.16b, v17.16b, v6.16b\n" - "sqrdmulh v21.4s, v21.4s, v5.4s\n" - "sqadd v19.4s, v19.4s, v16.4s\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v26.16b, v10.16b, v22.16b\n" + "add x15, x15, x4\n" + "add x14, x14, x4\n" + "and v2.16b, v15.16b, v27.16b\n" + "sqrdmulh v20.4s, v20.4s, v30.4s\n" + "sqrdmulh v24.4s, v24.4s, v30.4s\n" + "sqrdmulh v23.4s, v23.4s, v30.4s\n" + "sqadd v7.4s, v7.4s, v16.4s\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "and v21.16b, v20.16b, v12.16b\n" + "sqrdmulh v5.4s, v5.4s, v14.4s\n" + "and v18.16b, v24.16b, v12.16b\n" + "sqrdmulh v22.4s, v22.4s, v14.4s\n" + "and v31.16b, v23.16b, v12.16b\n" + "sqrdmulh v19.4s, v19.4s, v14.4s\n" + "sqadd v15.4s, v15.4s, v2.4s\n" + "sshr v21.4s, v21.4s, #0x1f\n" + "and v9.16b, v5.16b, v27.16b\n" "sshr v18.4s, v18.4s, #0x1f\n" - "and v23.16b, v7.16b, v22.16b\n" - "sshr v30.4s, v30.4s, #0x1f\n" - "and v16.16b, v21.16b, v22.16b\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "sshr v26.4s, v26.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v18.4s\n" - "sshr v23.4s, v23.4s, #0x1f\n" - "sqadd v17.4s, v17.4s, v30.4s\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "srshl v13.4s, v13.4s, v6.4s\n" - "srshl v20.4s, v20.4s, v6.4s\n" - "sqadd v10.4s, v10.4s, v26.4s\n" - "srshl v8.4s, v8.4s, v6.4s\n" - "sqadd v7.4s, v7.4s, v23.4s\n" - "srshl v17.4s, v17.4s, v6.4s\n" - "sqadd v21.4s, v21.4s, v16.4s\n" - "srshl v19.4s, v19.4s, v22.4s\n" - "sqxtn v13.4h, v13.4s\n" - "srshl v10.4s, v10.4s, v22.4s\n" + "and v4.16b, v22.16b, v27.16b\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "and v28.16b, v19.16b, v27.16b\n" + "sqadd v20.4s, v20.4s, v21.4s\n" + "sshr v9.4s, v9.4s, #0x1f\n" + "sqadd v24.4s, v24.4s, v18.4s\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sqadd v23.4s, v23.4s, v31.4s\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "srshl v7.4s, v7.4s, v12.4s\n" + "srshl v20.4s, v20.4s, v12.4s\n" + "sqadd v5.4s, v5.4s, v9.4s\n" + "srshl v24.4s, v24.4s, v12.4s\n" + "sqadd v22.4s, v22.4s, v4.4s\n" + "srshl v23.4s, v23.4s, v12.4s\n" + "sqadd v19.4s, v19.4s, v28.4s\n" + "srshl v15.4s, v15.4s, v27.4s\n" + "sqxtn v7.4h, v7.4s\n" + "srshl v5.4s, v5.4s, v27.4s\n" "sqxtn v20.4h, v20.4s\n" - "srshl v7.4s, v7.4s, v22.4s\n" - "sqxtn v8.4h, v8.4s\n" - "srshl v21.4s, v21.4s, v22.4s\n" - "sqxtn v17.4h, v17.4s\n" - "sqxtn2 v13.8h, v19.4s\n" - "sqxtn2 v20.8h, v10.4s\n" - "sqxtn2 v8.8h, v7.4s\n" - "sqxtn2 v17.8h, v21.4s\n" - "sqadd v13.8h, v13.8h, v14.8h\n" - "sqadd v20.8h, v20.8h, v14.8h\n" - "sqadd v8.8h, v8.8h, v14.8h\n" - "sqadd v17.8h, v17.8h, v14.8h\n" - "smax v13.8h, v13.8h, v12.8h\n" - "smax v20.8h, v20.8h, v12.8h\n" - "smax v8.8h, v8.8h, v12.8h\n" - "smax v17.8h, v17.8h, v12.8h\n" - "smin v13.8h, v13.8h, v11.8h\n" - "smin v20.8h, v20.8h, v11.8h\n" - "smin v8.8h, v8.8h, v11.8h\n" - "smin v17.8h, v17.8h, v11.8h\n" - "uzp1 v13.16b, v13.16b, v13.16b\n" + "srshl v22.4s, v22.4s, v27.4s\n" + "sqxtn v24.4h, v24.4s\n" + "srshl v19.4s, v19.4s, v27.4s\n" + "sqxtn v23.4h, v23.4s\n" + "sqxtn2 v7.8h, v15.4s\n" + "sqxtn2 v20.8h, v5.4s\n" + "sqxtn2 v24.8h, v22.4s\n" + "sqxtn2 v23.8h, v19.4s\n" + "sqadd v7.8h, v7.8h, v26.8h\n" + "sqadd v20.8h, v20.8h, v26.8h\n" + "sqadd v24.8h, v24.8h, v26.8h\n" + "sqadd v23.8h, v23.8h, v26.8h\n" + "smax v7.8h, v7.8h, v11.8h\n" + "smax v20.8h, v20.8h, v11.8h\n" + "smax v24.8h, v24.8h, v11.8h\n" + "smax v23.8h, v23.8h, v11.8h\n" + "smin v7.8h, v7.8h, v0.8h\n" + "smin v20.8h, v20.8h, v0.8h\n" + "smin v24.8h, v24.8h, v0.8h\n" + "smin v23.8h, v23.8h, v0.8h\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "uzp1 v8.16b, v8.16b, v8.16b\n" - "uzp1 v17.16b, v17.16b, v17.16b\n" + "uzp1 v24.16b, v24.16b, v24.16b\n" + "uzp1 v23.16b, v23.16b, v23.16b\n" "tbz x1, #2, 121f\n" - "st1 { v13.s }[0], [x17], #0x4\n" - "st1 { v20.s }[0], [x6], #0x4\n" - "st1 { v8.s }[0], [x7], #0x4\n" - "st1 { v17.s }[0], [x16], #0x4\n" + "st1 { v7.s }[0], [x17], #0x4\n" + "st1 { v20.s }[0], [x16], #0x4\n" + "st1 { v24.s }[0], [x15], #0x4\n" + "st1 { v23.s }[0], [x14], #0x4\n" "tbz x1, #1, 120f\n" - "st1 { v13.h }[2], [x17], #0x2\n" - "st1 { v20.h }[2], [x6], #0x2\n" - "st1 { v8.h }[2], [x7], #0x2\n" - "st1 { v17.h }[2], [x16], #0x2\n" + "st1 { v7.h }[2], [x17], #0x2\n" + "st1 { v20.h }[2], [x16], #0x2\n" + "st1 { v24.h }[2], [x15], #0x2\n" + "st1 { v23.h }[2], [x14], #0x2\n" "tbz x1, #0, 123f\n" - "st1 { v13.b }[6], [x17], #0x1\n" - "st1 { v20.b }[6], [x6], #0x1\n" - "st1 { v8.b }[6], [x7], #0x1\n" - "st1 { v17.b }[6], [x16], #0x1\n" + "st1 { v7.b }[6], [x17], #0x1\n" + "st1 { v20.b }[6], [x16], #0x1\n" + "st1 { v24.b }[6], [x15], #0x1\n" + "st1 { v23.b }[6], [x14], #0x1\n" "b 123f\n" "120:" // Oddments: Bit 2: Bit 1: Unset "tbz x1, #0, 123f\n" - "st1 { v13.b }[4], [x17], #0x1\n" - "st1 { v20.b }[4], [x6], #0x1\n" - "st1 { v8.b }[4], [x7], #0x1\n" - "st1 { v17.b }[4], [x16], #0x1\n" + "st1 { v7.b }[4], [x17], #0x1\n" + "st1 { v20.b }[4], [x16], #0x1\n" + "st1 { v24.b }[4], [x15], #0x1\n" + "st1 { v23.b }[4], [x14], #0x1\n" "b 123f\n" "121:" // Oddments: Bit 2: Unset "tbz x1, #1, 122f\n" - "st1 { v13.h }[0], [x17], #0x2\n" - "st1 { v20.h }[0], [x6], #0x2\n" - "st1 { v8.h }[0], [x7], #0x2\n" - "st1 { v17.h }[0], [x16], #0x2\n" + "st1 { v7.h }[0], [x17], #0x2\n" + "st1 { v20.h }[0], [x16], #0x2\n" + "st1 { v24.h }[0], [x15], #0x2\n" + "st1 { v23.h }[0], [x14], #0x2\n" "tbz x1, #0, 123f\n" - "st1 { v13.b }[2], [x17], #0x1\n" - "st1 { v20.b }[2], [x6], #0x1\n" - "st1 { v8.b }[2], [x7], #0x1\n" - "st1 { v17.b }[2], [x16], #0x1\n" + "st1 { v7.b }[2], [x17], #0x1\n" + "st1 { v20.b }[2], [x16], #0x1\n" + "st1 { v24.b }[2], [x15], #0x1\n" + "st1 { v23.b }[2], [x14], #0x1\n" "b 123f\n" "122:" // Oddments: Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 123f\n" - "st1 { v13.b }[0], [x17], #0x1\n" - "st1 { v20.b }[0], [x6], #0x1\n" - "st1 { v8.b }[0], [x7], #0x1\n" - "st1 { v17.b }[0], [x16], #0x1\n" + "st1 { v7.b }[0], [x17], #0x1\n" + "st1 { v20.b }[0], [x16], #0x1\n" + "st1 { v24.b }[0], [x15], #0x1\n" + "st1 { v23.b }[0], [x14], #0x1\n" "123:" // Oddments: Bit 2: End "124:" // End : : [offsetof_Params_bias] "I" (offsetof(Params, bias)), [offsetof_Params_inptrs] "I" (offsetof(Params, inptrs)), [offsetof_Params_n_channels] "I" (offsetof(Params, n_channels)), [offsetof_Params_outptrs] "I" (offsetof(Params, outptrs)), [offsetof_Params_requant] "I" (offsetof(Params, requant)), [offsetof_Params_requant_muls] "I" (offsetof(Params, requant_muls)), [offsetof_Params_requant_shifts] "I" (offsetof(Params, requant_shifts)), [offsetof_Params_weights] "I" (offsetof(Params, weights)), [offsetof_Requantize32_a_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [params] "r" (¶ms) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_generic_output9_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_generic_output9_mla_depthfirst.hpp index 9b1f7c239fccee906576b2e1f92860de39e8fd91..9c92a9dd46e8b78c9fa13ed79ce082c5b59c24a9 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_generic_output9_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_generic_output9_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include @@ -47,4 +47,5 @@ class a64_s8q_nhwc_generic_output9_mla_depthfirst : public GenericDepthfirstKern } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_generic_output9_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_generic_output9_mla_depthfirst/generic.cpp index 3f345cf95a13288cdc580d4c9c581c8922c9ff92..77b7d231e05955f93d055a494659ee8bbfad5886 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_generic_output9_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_generic_output9_mla_depthfirst/generic.cpp @@ -22,12 +22,13 @@ * SOFTWARE. */ -#if defined(__aarch64__) #include "arm_gemm.hpp" #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace depthwise { @@ -41,7 +42,7 @@ void a64_s8q_nhwc_generic_output9_mla_depthfirst_impl( ) { __asm__ __volatile__( - "lsr x12, %x[n_channels], #0x2\n" + "lsr x9, %x[n_channels], #0x2\n" "add x20, %x[qp], %[offsetof_Requantize32_minval]\n" "ld1r { v8.4s }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_maxval]\n" @@ -59,7 +60,7 @@ void a64_s8q_nhwc_generic_output9_mla_depthfirst_impl( "add x20, %x[qp], %[offsetof_Requantize32_per_layer_right_shift]\n" "ld1r { v1.4s }, [x20]\n" "mov x11, #0x0\n" - "cbz x12, 6f\n" + "cbz x9, 6f\n" "1:" // Channel loop "movi v23.4s, #0x0\n" "cbz %x[bias], 2f\n" @@ -67,34 +68,34 @@ void a64_s8q_nhwc_generic_output9_mla_depthfirst_impl( "ldr q23, [%x[bias], x20]\n" "2:" // Channel loop: Load bias: Done "ldr s0, [%x[params]], #0x4\n" - "mov x21, %x[inptrs]\n" - "ldp x10, x9, [x21], #0x10\n" - "subs x20, %x[n_points], #0x1\n" - "ldr s14, [x10, x11]\n" - "ldr s15, [x9, x11]\n" + "mov x25, %x[inptrs]\n" + "ldp x21, x20, [x25], #0x10\n" + "subs x24, %x[n_points], #0x1\n" + "ldr s14, [x21, x11]\n" + "ldr s15, [x20, x11]\n" "mov v24.16b, v23.16b\n" "mov v25.16b, v23.16b\n" - "ldp x28, x27, [x21], #0x10\n" - "ldr s16, [x28, x11]\n" + "ldp x21, x20, [x25], #0x10\n" + "ldr s16, [x21, x11]\n" "mov v26.16b, v23.16b\n" "mov v27.16b, v23.16b\n" - "ldr s17, [x27, x11]\n" - "ldp x26, x25, [x21], #0x10\n" + "ldr s17, [x20, x11]\n" + "ldp x21, x20, [x25], #0x10\n" "mov v28.16b, v23.16b\n" "mov v29.16b, v23.16b\n" - "ldr s18, [x26, x11]\n" - "ldr s19, [x25, x11]\n" + "ldr s18, [x21, x11]\n" + "ldr s19, [x20, x11]\n" "mov v30.16b, v23.16b\n" "mov v31.16b, v23.16b\n" - "ldp x24, x23, [x21], #0x10\n" - "ldr s20, [x24, x11]\n" + "ldp x21, x20, [x25], #0x10\n" + "ldr s20, [x21, x11]\n" "ssubl v0.8h, v0.8b, v5.8b\n" "ssubl v14.8h, v14.8b, v6.8b\n" - "ldr s21, [x23, x11]\n" - "ldr x22, [x21], #0x8\n" + "ldr s21, [x20, x11]\n" + "ldr x20, [x25], #0x8\n" "ssubl v15.8h, v15.8b, v6.8b\n" "ssubl v16.8h, v16.8b, v6.8b\n" - "ldr s22, [x22, x11]\n" + "ldr s22, [x20, x11]\n" "ssubl v17.8h, v17.8b, v6.8b\n" "ssubl v18.8h, v18.8b, v6.8b\n" "ssubl v19.8h, v19.8b, v6.8b\n" @@ -103,35 +104,35 @@ void a64_s8q_nhwc_generic_output9_mla_depthfirst_impl( "ssubl v22.8h, v22.8b, v6.8b\n" "ble 4f\n" "3:" // Channel loop: Planar loop - "ldp x10, x9, [x21], #0x10\n" - "ldp x28, x27, [x21], #0x10\n" + "ldp x23, x22, [x25], #0x10\n" + "ldp x21, x20, [x25], #0x10\n" "smlal v23.4s, v14.4h, v0.4h\n" "smlal v24.4s, v15.4h, v0.4h\n" - "ldr s14, [x10, x11]\n" - "ldr s15, [x9, x11]\n" + "ldr s14, [x23, x11]\n" + "ldr s15, [x22, x11]\n" "smlal v25.4s, v16.4h, v0.4h\n" "smlal v26.4s, v17.4h, v0.4h\n" - "ldr s16, [x28, x11]\n" - "ldr s17, [x27, x11]\n" + "ldr s16, [x21, x11]\n" + "ldr s17, [x20, x11]\n" "smlal v27.4s, v18.4h, v0.4h\n" "smlal v28.4s, v19.4h, v0.4h\n" - "ldp x26, x25, [x21], #0x10\n" - "ldr s18, [x26, x11]\n" + "ldp x21, x20, [x25], #0x10\n" + "ldr s18, [x21, x11]\n" "smlal v29.4s, v20.4h, v0.4h\n" "smlal v30.4s, v21.4h, v0.4h\n" - "ldr s19, [x25, x11]\n" - "ldp x24, x23, [x21], #0x10\n" + "ldr s19, [x20, x11]\n" + "ldp x21, x20, [x25], #0x10\n" "smlal v31.4s, v22.4h, v0.4h\n" - "subs x20, x20, #0x1\n" + "subs x24, x24, #0x1\n" "ldr s0, [%x[params]], #0x4\n" - "ldr s20, [x24, x11]\n" + "ldr s20, [x21, x11]\n" "ssubl v0.8h, v0.8b, v5.8b\n" "ssubl v14.8h, v14.8b, v6.8b\n" - "ldr s21, [x23, x11]\n" - "ldr x22, [x21], #0x8\n" + "ldr s21, [x20, x11]\n" + "ldr x20, [x25], #0x8\n" "ssubl v15.8h, v15.8b, v6.8b\n" "ssubl v16.8h, v16.8b, v6.8b\n" - "ldr s22, [x22, x11]\n" + "ldr s22, [x20, x11]\n" "ssubl v17.8h, v17.8b, v6.8b\n" "ssubl v18.8h, v18.8b, v6.8b\n" "ssubl v19.8h, v19.8b, v6.8b\n" @@ -167,45 +168,45 @@ void a64_s8q_nhwc_generic_output9_mla_depthfirst_impl( "sqrdmulh v24.4s, v24.4s, v2.4s\n" "sqrdmulh v25.4s, v25.4s, v2.4s\n" "ldr x20, [%x[outptrs], #0x40]\n" - "and v21.16b, v23.16b, v1.16b\n" - "and v20.16b, v24.16b, v1.16b\n" - "and v19.16b, v25.16b, v1.16b\n" + "and v18.16b, v23.16b, v1.16b\n" + "and v17.16b, v24.16b, v1.16b\n" + "and v16.16b, v25.16b, v1.16b\n" "sshl v26.4s, v26.4s, v3.4s\n" "sshl v27.4s, v27.4s, v3.4s\n" "sshl v28.4s, v28.4s, v3.4s\n" "sshl v29.4s, v29.4s, v3.4s\n" "sshl v30.4s, v30.4s, v3.4s\n" "sshl v31.4s, v31.4s, v3.4s\n" - "sshr v21.4s, v21.4s, #0x1f\n" - "sshr v20.4s, v20.4s, #0x1f\n" - "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" "sqrdmulh v26.4s, v26.4s, v2.4s\n" "sqrdmulh v27.4s, v27.4s, v2.4s\n" "sqrdmulh v28.4s, v28.4s, v2.4s\n" "sqrdmulh v29.4s, v29.4s, v2.4s\n" "sqrdmulh v30.4s, v30.4s, v2.4s\n" "sqrdmulh v31.4s, v31.4s, v2.4s\n" - "sqadd v23.4s, v23.4s, v21.4s\n" - "sqadd v24.4s, v24.4s, v20.4s\n" - "sqadd v25.4s, v25.4s, v19.4s\n" - "and v18.16b, v26.16b, v1.16b\n" - "and v17.16b, v27.16b, v1.16b\n" - "and v16.16b, v28.16b, v1.16b\n" - "and v21.16b, v29.16b, v1.16b\n" - "and v20.16b, v30.16b, v1.16b\n" - "and v19.16b, v31.16b, v1.16b\n" - "sshr v18.4s, v18.4s, #0x1f\n" - "sshr v17.4s, v17.4s, #0x1f\n" - "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v23.4s, v23.4s, v18.4s\n" + "sqadd v24.4s, v24.4s, v17.4s\n" + "sqadd v25.4s, v25.4s, v16.4s\n" + "and v21.16b, v26.16b, v1.16b\n" + "and v20.16b, v27.16b, v1.16b\n" + "and v19.16b, v28.16b, v1.16b\n" + "and v18.16b, v29.16b, v1.16b\n" + "and v17.16b, v30.16b, v1.16b\n" + "and v16.16b, v31.16b, v1.16b\n" "sshr v21.4s, v21.4s, #0x1f\n" "sshr v20.4s, v20.4s, #0x1f\n" "sshr v19.4s, v19.4s, #0x1f\n" - "sqadd v26.4s, v26.4s, v18.4s\n" - "sqadd v27.4s, v27.4s, v17.4s\n" - "sqadd v28.4s, v28.4s, v16.4s\n" - "sqadd v29.4s, v29.4s, v21.4s\n" - "sqadd v30.4s, v30.4s, v20.4s\n" - "sqadd v31.4s, v31.4s, v19.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v26.4s, v26.4s, v21.4s\n" + "sqadd v27.4s, v27.4s, v20.4s\n" + "sqadd v28.4s, v28.4s, v19.4s\n" + "sqadd v29.4s, v29.4s, v18.4s\n" + "sqadd v30.4s, v30.4s, v17.4s\n" + "sqadd v31.4s, v31.4s, v16.4s\n" "srshl v23.4s, v23.4s, v1.4s\n" "srshl v24.4s, v24.4s, v1.4s\n" "srshl v25.4s, v25.4s, v1.4s\n" @@ -270,7 +271,7 @@ void a64_s8q_nhwc_generic_output9_mla_depthfirst_impl( "str s30, [x21, x11]\n" "str s31, [x20, x11]\n" "add x11, x11, #0x4\n" - "cmp x11, x12, LSL #2\n" + "cmp x11, x9, LSL #2\n" "blt 1b\n" "6:" // Oddments "tst %x[n_channels], #0x3\n" @@ -288,61 +289,61 @@ void a64_s8q_nhwc_generic_output9_mla_depthfirst_impl( "8:" // Oddments: Load bias: Bit 1: End "9:" // Oddments: Load bias: Done "ldr s0, [%x[params]], #0x4\n" - "mov x21, %x[inptrs]\n" - "ldp x10, x9, [x21], #0x10\n" + "mov x10, %x[inptrs]\n" + "ldp x9, x28, [x10], #0x10\n" "mov v24.16b, v23.16b\n" - "ldp x28, x27, [x21], #0x10\n" - "ldp x26, x25, [x21], #0x10\n" + "ldp x27, x26, [x10], #0x10\n" + "ldp x25, x24, [x10], #0x10\n" "mov v25.16b, v23.16b\n" "mov v26.16b, v23.16b\n" - "ldp x24, x23, [x21], #0x10\n" - "ldr x22, [x21], #0x8\n" + "ldp x23, x22, [x10], #0x10\n" + "ldr x21, [x10], #0x8\n" "mov v27.16b, v23.16b\n" "mov v28.16b, v23.16b\n" "mov v29.16b, v23.16b\n" "mov v30.16b, v23.16b\n" - "add x10, x10, x11\n" "add x9, x9, x11\n" + "add x28, x28, x11\n" "mov v31.16b, v23.16b\n" "ssubl v0.8h, v0.8b, v5.8b\n" - "add x28, x28, x11\n" "add x27, x27, x11\n" "add x26, x26, x11\n" "add x25, x25, x11\n" "add x24, x24, x11\n" "add x23, x23, x11\n" "add x22, x22, x11\n" + "add x21, x21, x11\n" "tbz %x[n_channels], #1, 10f\n" - "ldr h14, [x10], #0x2\n" - "ldr h15, [x9], #0x2\n" - "ldr h16, [x28], #0x2\n" - "ldr h17, [x27], #0x2\n" - "ldr h18, [x26], #0x2\n" - "ldr h19, [x25], #0x2\n" - "ldr h20, [x24], #0x2\n" - "ldr h21, [x23], #0x2\n" - "ldr h22, [x22], #0x2\n" + "ldr h14, [x9], #0x2\n" + "ldr h15, [x28], #0x2\n" + "ldr h16, [x27], #0x2\n" + "ldr h17, [x26], #0x2\n" + "ldr h18, [x25], #0x2\n" + "ldr h19, [x24], #0x2\n" + "ldr h20, [x23], #0x2\n" + "ldr h21, [x22], #0x2\n" + "ldr h22, [x21], #0x2\n" "tbz %x[n_channels], #0, 11f\n" - "ld1 { v14.b }[2], [x10], #0x1\n" - "ld1 { v15.b }[2], [x9], #0x1\n" - "ld1 { v16.b }[2], [x28], #0x1\n" - "ld1 { v17.b }[2], [x27], #0x1\n" - "ld1 { v18.b }[2], [x26], #0x1\n" - "ld1 { v19.b }[2], [x25], #0x1\n" - "ld1 { v20.b }[2], [x24], #0x1\n" - "ld1 { v21.b }[2], [x23], #0x1\n" - "ld1 { v22.b }[2], [x22], #0x1\n" + "ld1 { v14.b }[2], [x9], #0x1\n" + "ld1 { v15.b }[2], [x28], #0x1\n" + "ld1 { v16.b }[2], [x27], #0x1\n" + "ld1 { v17.b }[2], [x26], #0x1\n" + "ld1 { v18.b }[2], [x25], #0x1\n" + "ld1 { v19.b }[2], [x24], #0x1\n" + "ld1 { v20.b }[2], [x23], #0x1\n" + "ld1 { v21.b }[2], [x22], #0x1\n" + "ld1 { v22.b }[2], [x21], #0x1\n" "b 11f\n" "10:" // Oddments: Load: Bit 1: Unset - "ldr b14, [x10], #0x1\n" - "ldr b15, [x9], #0x1\n" - "ldr b16, [x28], #0x1\n" - "ldr b17, [x27], #0x1\n" - "ldr b18, [x26], #0x1\n" - "ldr b19, [x25], #0x1\n" - "ldr b20, [x24], #0x1\n" - "ldr b21, [x23], #0x1\n" - "ldr b22, [x22], #0x1\n" + "ldr b14, [x9], #0x1\n" + "ldr b15, [x28], #0x1\n" + "ldr b16, [x27], #0x1\n" + "ldr b17, [x26], #0x1\n" + "ldr b18, [x25], #0x1\n" + "ldr b19, [x24], #0x1\n" + "ldr b20, [x23], #0x1\n" + "ldr b21, [x22], #0x1\n" + "ldr b22, [x21], #0x1\n" "11:" // Oddments: Load: Bit 1: End "subs x20, %x[n_points], #0x1\n" "ssubl v14.8h, v14.8b, v6.8b\n" @@ -356,62 +357,62 @@ void a64_s8q_nhwc_generic_output9_mla_depthfirst_impl( "ssubl v22.8h, v22.8b, v6.8b\n" "ble 15f\n" "12:" // Oddments: Planar loop - "ldp x10, x9, [x21], #0x10\n" - "ldp x28, x27, [x21], #0x10\n" + "ldp x9, x28, [x10], #0x10\n" + "ldp x27, x26, [x10], #0x10\n" "smlal v23.4s, v14.4h, v0.4h\n" "smlal v24.4s, v15.4h, v0.4h\n" - "ldp x26, x25, [x21], #0x10\n" - "ldp x24, x23, [x21], #0x10\n" + "ldp x25, x24, [x10], #0x10\n" + "ldp x23, x22, [x10], #0x10\n" "smlal v25.4s, v16.4h, v0.4h\n" "smlal v26.4s, v17.4h, v0.4h\n" "smlal v27.4s, v18.4h, v0.4h\n" "smlal v28.4s, v19.4h, v0.4h\n" - "ldr x22, [x21], #0x8\n" - "add x10, x10, x11\n" + "ldr x21, [x10], #0x8\n" + "add x9, x9, x11\n" "smlal v29.4s, v20.4h, v0.4h\n" "smlal v30.4s, v21.4h, v0.4h\n" - "add x9, x9, x11\n" "add x28, x28, x11\n" + "add x27, x27, x11\n" "smlal v31.4s, v22.4h, v0.4h\n" "ldr s0, [%x[params]], #0x4\n" "ssubl v0.8h, v0.8b, v5.8b\n" - "add x27, x27, x11\n" "add x26, x26, x11\n" "add x25, x25, x11\n" "add x24, x24, x11\n" "add x23, x23, x11\n" "add x22, x22, x11\n" + "add x21, x21, x11\n" "tbz %x[n_channels], #1, 13f\n" - "ldr h14, [x10], #0x2\n" - "ldr h15, [x9], #0x2\n" - "ldr h16, [x28], #0x2\n" - "ldr h17, [x27], #0x2\n" - "ldr h18, [x26], #0x2\n" - "ldr h19, [x25], #0x2\n" - "ldr h20, [x24], #0x2\n" - "ldr h21, [x23], #0x2\n" - "ldr h22, [x22], #0x2\n" + "ldr h14, [x9], #0x2\n" + "ldr h15, [x28], #0x2\n" + "ldr h16, [x27], #0x2\n" + "ldr h17, [x26], #0x2\n" + "ldr h18, [x25], #0x2\n" + "ldr h19, [x24], #0x2\n" + "ldr h20, [x23], #0x2\n" + "ldr h21, [x22], #0x2\n" + "ldr h22, [x21], #0x2\n" "tbz %x[n_channels], #0, 14f\n" - "ld1 { v14.b }[2], [x10], #0x1\n" - "ld1 { v15.b }[2], [x9], #0x1\n" - "ld1 { v16.b }[2], [x28], #0x1\n" - "ld1 { v17.b }[2], [x27], #0x1\n" - "ld1 { v18.b }[2], [x26], #0x1\n" - "ld1 { v19.b }[2], [x25], #0x1\n" - "ld1 { v20.b }[2], [x24], #0x1\n" - "ld1 { v21.b }[2], [x23], #0x1\n" - "ld1 { v22.b }[2], [x22], #0x1\n" + "ld1 { v14.b }[2], [x9], #0x1\n" + "ld1 { v15.b }[2], [x28], #0x1\n" + "ld1 { v16.b }[2], [x27], #0x1\n" + "ld1 { v17.b }[2], [x26], #0x1\n" + "ld1 { v18.b }[2], [x25], #0x1\n" + "ld1 { v19.b }[2], [x24], #0x1\n" + "ld1 { v20.b }[2], [x23], #0x1\n" + "ld1 { v21.b }[2], [x22], #0x1\n" + "ld1 { v22.b }[2], [x21], #0x1\n" "b 14f\n" "13:" // Oddments: Planar loop: Load: Bit 1: Unset - "ldr b14, [x10], #0x1\n" - "ldr b15, [x9], #0x1\n" - "ldr b16, [x28], #0x1\n" - "ldr b17, [x27], #0x1\n" - "ldr b18, [x26], #0x1\n" - "ldr b19, [x25], #0x1\n" - "ldr b20, [x24], #0x1\n" - "ldr b21, [x23], #0x1\n" - "ldr b22, [x22], #0x1\n" + "ldr b14, [x9], #0x1\n" + "ldr b15, [x28], #0x1\n" + "ldr b16, [x27], #0x1\n" + "ldr b17, [x26], #0x1\n" + "ldr b18, [x25], #0x1\n" + "ldr b19, [x24], #0x1\n" + "ldr b20, [x23], #0x1\n" + "ldr b21, [x22], #0x1\n" + "ldr b22, [x21], #0x1\n" "14:" // Oddments: Planar loop: Load: Bit 1: End "subs x20, x20, #0x1\n" "ssubl v14.8h, v14.8b, v6.8b\n" @@ -457,9 +458,7 @@ void a64_s8q_nhwc_generic_output9_mla_depthfirst_impl( "cbz %x[rq_left_shift_ptr], 19f\n" "ld1 { v3.s }[0], [x20], #0x4\n" "19:" // Oddments: Load quantisation parameters: Bit 1: Unset: Bit 0: Load left shift: Done - "20:" // Oddments: Load quantisation parameters: Bit 1: End - "21:" // Oddments: Load quantisation parameters: Done "sshl v23.4s, v23.4s, v3.4s\n" "sshl v24.4s, v24.4s, v3.4s\n" @@ -473,11 +472,11 @@ void a64_s8q_nhwc_generic_output9_mla_depthfirst_impl( "sqrdmulh v25.4s, v25.4s, v2.4s\n" "ldr x20, [%x[outptrs], #0x40]\n" "add x28, x28, x11\n" - "and v21.16b, v23.16b, v1.16b\n" - "and v20.16b, v24.16b, v1.16b\n" + "and v18.16b, v23.16b, v1.16b\n" + "and v17.16b, v24.16b, v1.16b\n" "add x27, x27, x11\n" "add x26, x26, x11\n" - "and v19.16b, v25.16b, v1.16b\n" + "and v16.16b, v25.16b, v1.16b\n" "sshl v26.4s, v26.4s, v3.4s\n" "add x25, x25, x11\n" "add x24, x24, x11\n" @@ -490,36 +489,36 @@ void a64_s8q_nhwc_generic_output9_mla_depthfirst_impl( "add x21, x21, x11\n" "add x20, x20, x11\n" "sshl v31.4s, v31.4s, v3.4s\n" - "sshr v21.4s, v21.4s, #0x1f\n" - "sshr v20.4s, v20.4s, #0x1f\n" - "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" "sqrdmulh v26.4s, v26.4s, v2.4s\n" "sqrdmulh v27.4s, v27.4s, v2.4s\n" "sqrdmulh v28.4s, v28.4s, v2.4s\n" "sqrdmulh v29.4s, v29.4s, v2.4s\n" "sqrdmulh v30.4s, v30.4s, v2.4s\n" "sqrdmulh v31.4s, v31.4s, v2.4s\n" - "sqadd v23.4s, v23.4s, v21.4s\n" - "sqadd v24.4s, v24.4s, v20.4s\n" - "sqadd v25.4s, v25.4s, v19.4s\n" - "and v18.16b, v26.16b, v1.16b\n" - "and v17.16b, v27.16b, v1.16b\n" - "and v16.16b, v28.16b, v1.16b\n" - "and v21.16b, v29.16b, v1.16b\n" - "and v20.16b, v30.16b, v1.16b\n" - "and v19.16b, v31.16b, v1.16b\n" - "sshr v18.4s, v18.4s, #0x1f\n" - "sshr v17.4s, v17.4s, #0x1f\n" - "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v23.4s, v23.4s, v18.4s\n" + "sqadd v24.4s, v24.4s, v17.4s\n" + "sqadd v25.4s, v25.4s, v16.4s\n" + "and v21.16b, v26.16b, v1.16b\n" + "and v20.16b, v27.16b, v1.16b\n" + "and v19.16b, v28.16b, v1.16b\n" + "and v18.16b, v29.16b, v1.16b\n" + "and v17.16b, v30.16b, v1.16b\n" + "and v16.16b, v31.16b, v1.16b\n" "sshr v21.4s, v21.4s, #0x1f\n" "sshr v20.4s, v20.4s, #0x1f\n" "sshr v19.4s, v19.4s, #0x1f\n" - "sqadd v26.4s, v26.4s, v18.4s\n" - "sqadd v27.4s, v27.4s, v17.4s\n" - "sqadd v28.4s, v28.4s, v16.4s\n" - "sqadd v29.4s, v29.4s, v21.4s\n" - "sqadd v30.4s, v30.4s, v20.4s\n" - "sqadd v31.4s, v31.4s, v19.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v26.4s, v26.4s, v21.4s\n" + "sqadd v27.4s, v27.4s, v20.4s\n" + "sqadd v28.4s, v28.4s, v19.4s\n" + "sqadd v29.4s, v29.4s, v18.4s\n" + "sqadd v30.4s, v30.4s, v17.4s\n" + "sqadd v31.4s, v31.4s, v16.4s\n" "srshl v23.4s, v23.4s, v1.4s\n" "srshl v24.4s, v24.4s, v1.4s\n" "srshl v25.4s, v25.4s, v1.4s\n" @@ -606,15 +605,14 @@ void a64_s8q_nhwc_generic_output9_mla_depthfirst_impl( "st1 { v30.b }[0], [x21], #0x1\n" "st1 { v31.b }[0], [x20], #0x1\n" "23:" // Oddments: Store: Bit 1: End - "24:" // End - : [params] "+&r" (params) : [bias] "r" (qp.bias), [inptrs] "r" (inptrs), [n_channels] "r" ((uint64_t) n_channels), [n_points] "r" ((uint64_t) n_points), [offsetof_Requantize32_a_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [offsetof_Requantize32_per_layer_left_shift] "I" (offsetof(arm_gemm::Requantize32, per_layer_left_shift)), [offsetof_Requantize32_per_layer_mul] "I" (offsetof(arm_gemm::Requantize32, per_layer_mul)), [offsetof_Requantize32_per_layer_right_shift] "I" (offsetof(arm_gemm::Requantize32, per_layer_right_shift)), [outptrs] "r" (outptrs), [qp] "r" (&qp), [rq_left_shift_ptr] "r" (qp.per_channel_left_shifts), [rq_mul_ptr] "r" (qp.per_channel_muls), [rq_right_shift_ptr] "r" (qp.per_channel_right_shifts) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst.hpp index 5ca3ccd4bfeb6d64b7558513555e5429b5a4a22c..14adf8880fe6a87de79dc3acc9445550cb97d42e 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include @@ -58,4 +58,4 @@ struct a64_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst : } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst/generic.cpp index 342a297dd461aec73f3ad1a438fa9aef0d91af4c..be8fbfa0e253478ca264c784c6edbaaa6d0bc00f 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst/generic.cpp @@ -22,12 +22,13 @@ * SOFTWARE. */ -#if defined(__aarch64__) #include "arm_gemm.hpp" #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace depthwise { @@ -40,169 +41,169 @@ void a64_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst_impl ) { __asm__ __volatile__( - "ldr q14, [%x[params], #0x0]\n" + "ldr q11, [%x[params], #0x0]\n" "ldr q5, [%x[params], #0x10]\n" - "movi v15.16b, #0x1\n" - "ushr v15.4s, v15.4s, #0x8\n" + "movi v8.16b, #0x1\n" + "ushr v8.4s, v8.4s, #0x8\n" "ldr q6, [%x[params], #0x20]\n" "ldr q7, [%x[params], #0x30]\n" - "movi v26.4s, #0x0\n" - "movi v27.4s, #0x0\n" + "movi v24.4s, #0x0\n" + "movi v25.4s, #0x0\n" "ldr x20, [%x[inptrs], #0x8]\n" "ld1 { v1.16b }, [x20]\n" - "mov v29.16b, v1.16b\n" - "mov v16.16b, v1.16b\n" + "mov v28.16b, v1.16b\n" + "mov v23.16b, v1.16b\n" "ldr x20, [%x[inptrs], #0x10]\n" "ld1 { v2.16b }, [x20]\n" - "mov v28.16b, v1.16b\n" - "mov v22.16b, v2.16b\n" + "mov v30.16b, v1.16b\n" + "mov v21.16b, v2.16b\n" "ldr x20, [%x[inptrs], #0x20]\n" "ld1 { v4.16b }, [x20]\n" - "mov v31.16b, v2.16b\n" - "mov v30.16b, v2.16b\n" + "mov v20.16b, v2.16b\n" + "mov v29.16b, v2.16b\n" "ldr x20, [%x[inptrs], #0x0]\n" "ld1 { v0.16b }, [x20]\n" - "mov v23.16b, v4.16b\n" - "mov v21.16b, v4.16b\n" + "mov v9.16b, v4.16b\n" + "mov v22.16b, v4.16b\n" "ldr x20, [%x[inptrs], #0x18]\n" "ld1 { v3.16b }, [x20]\n" - "mov v20.16b, v4.16b\n" - "ext v29.16b, v29.16b, v29.16b, #0x2\n" - "ext v16.16b, v16.16b, v16.16b, #0x4\n" - "ext v28.16b, v28.16b, v28.16b, #0x6\n" + "mov v31.16b, v4.16b\n" + "ext v28.16b, v28.16b, v28.16b, #0x2\n" + "ext v23.16b, v23.16b, v23.16b, #0x4\n" + "ext v30.16b, v30.16b, v30.16b, #0x6\n" "add x20, %x[qp], %[offsetof_Requantize32_b_offset]\n" - "ld1r { v13.4s }, [x20]\n" - "ext v22.16b, v22.16b, v22.16b, #0x2\n" - "ext v31.16b, v31.16b, v31.16b, #0x4\n" - "add x20, %x[qp], %[offsetof_Requantize32_c_offset]\n" "ld1r { v12.4s }, [x20]\n" - "ext v30.16b, v30.16b, v30.16b, #0x6\n" - "ext v23.16b, v23.16b, v23.16b, #0x2\n" + "ext v21.16b, v21.16b, v21.16b, #0x2\n" + "ext v20.16b, v20.16b, v20.16b, #0x4\n" + "add x20, %x[qp], %[offsetof_Requantize32_c_offset]\n" + "ld1r { v14.4s }, [x20]\n" + "ext v29.16b, v29.16b, v29.16b, #0x6\n" + "ext v9.16b, v9.16b, v9.16b, #0x2\n" "add x20, %x[qp], %[offsetof_Requantize32_minval]\n" - "ld1r { v11.4s }, [x20]\n" - "ext v21.16b, v21.16b, v21.16b, #0x4\n" - "ext v20.16b, v20.16b, v20.16b, #0x6\n" + "ld1r { v13.4s }, [x20]\n" + "ext v22.16b, v22.16b, v22.16b, #0x4\n" + "ext v31.16b, v31.16b, v31.16b, #0x6\n" "add x20, %x[qp], %[offsetof_Requantize32_maxval]\n" - "ld1r { v10.4s }, [x20]\n" - "mov v25.16b, v0.16b\n" + "ld1r { v15.4s }, [x20]\n" + "mov v27.16b, v0.16b\n" "mov v19.16b, v0.16b\n" "cmp %x[n_channels], #0x4\n" "mov x9, #0x0\n" "mov v18.16b, v0.16b\n" - "mov v24.16b, v3.16b\n" + "mov v26.16b, v3.16b\n" "mov x28, #0x0\n" "ldp x27, x26, [%x[outptrs], #0x0]\n" "mov v17.16b, v3.16b\n" - "ext v25.16b, v25.16b, v25.16b, #0x2\n" + "mov v16.16b, v3.16b\n" "ldp x25, x24, [%x[outptrs], #0x10]\n" "ldp x23, x22, [%x[outptrs], #0x20]\n" + "ext v27.16b, v27.16b, v27.16b, #0x2\n" "ext v19.16b, v19.16b, v19.16b, #0x4\n" - "ext v18.16b, v18.16b, v18.16b, #0x6\n" "ldp x21, x20, [%x[outptrs], #0x30]\n" "add %x[params], %x[params], #0x40\n" - "zip1 v1.4s, v1.4s, v16.4s\n" - "mov v16.16b, v3.16b\n" - "zip1 v29.4s, v29.4s, v28.4s\n" - "zip1 v2.4s, v2.4s, v31.4s\n" - "zip1 v22.4s, v22.4s, v30.4s\n" - "ext v24.16b, v24.16b, v24.16b, #0x2\n" + "ext v18.16b, v18.16b, v18.16b, #0x6\n" + "zip1 v1.4s, v1.4s, v23.4s\n" + "zip1 v28.4s, v28.4s, v30.4s\n" + "zip1 v2.4s, v2.4s, v20.4s\n" + "zip1 v21.4s, v21.4s, v29.4s\n" + "ext v26.16b, v26.16b, v26.16b, #0x2\n" "ext v17.16b, v17.16b, v17.16b, #0x4\n" "ext v16.16b, v16.16b, v16.16b, #0x6\n" - "zip1 v4.4s, v4.4s, v21.4s\n" - "zip1 v23.4s, v23.4s, v20.4s\n" + "zip1 v4.4s, v4.4s, v22.4s\n" + "zip1 v9.4s, v9.4s, v31.4s\n" "zip1 v0.4s, v0.4s, v19.4s\n" - "zip1 v25.4s, v25.4s, v18.4s\n" - "zip1 v1.4s, v1.4s, v29.4s\n" - "zip1 v2.4s, v2.4s, v22.4s\n" - ".inst 0x4f81e1fa // sdot v26.4s, v15.16b, v1.4b[0]\n" + "zip1 v27.4s, v27.4s, v18.4s\n" + "zip1 v1.4s, v1.4s, v28.4s\n" + "zip1 v2.4s, v2.4s, v21.4s\n" + ".inst 0x4f81e118 // sdot v24.4s, v8.16b, v1.4b[0]\n" "zip1 v3.4s, v3.4s, v17.4s\n" - "zip1 v24.4s, v24.4s, v16.4s\n" - ".inst 0x4fa1e1fb // sdot v27.4s, v15.16b, v1.4b[1]\n" - "zip1 v4.4s, v4.4s, v23.4s\n" + "zip1 v26.4s, v26.4s, v16.4s\n" + ".inst 0x4fa1e119 // sdot v25.4s, v8.16b, v1.4b[1]\n" + "zip1 v4.4s, v4.4s, v9.4s\n" "movi v23.4s, #0x0\n" - ".inst 0x4f81e9f7 // sdot v23.4s, v15.16b, v1.4b[2]\n" + ".inst 0x4f81e917 // sdot v23.4s, v8.16b, v1.4b[2]\n" "movi v22.4s, #0x0\n" "movi v21.4s, #0x0\n" - ".inst 0x4fa1e9f6 // sdot v22.4s, v15.16b, v1.4b[3]\n" - "movi v20.4s, #0x0\n" - "movi v9.4s, #0x0\n" - ".inst 0x4f82e1f5 // sdot v21.4s, v15.16b, v2.4b[0]\n" - "movi v8.4s, #0x0\n" + ".inst 0x4fa1e916 // sdot v22.4s, v8.16b, v1.4b[3]\n" "movi v19.4s, #0x0\n" - ".inst 0x4fa2e1f4 // sdot v20.4s, v15.16b, v2.4b[1]\n" + "movi v9.4s, #0x0\n" + ".inst 0x4f82e115 // sdot v21.4s, v8.16b, v2.4b[0]\n" + "movi v10.4s, #0x0\n" + "movi v20.4s, #0x0\n" + ".inst 0x4fa2e113 // sdot v19.4s, v8.16b, v2.4b[1]\n" "movi v18.4s, #0x0\n" "movi v17.4s, #0x0\n" - ".inst 0x4f82e9e9 // sdot v9.4s, v15.16b, v2.4b[2]\n" + ".inst 0x4f82e909 // sdot v9.4s, v8.16b, v2.4b[2]\n" "movi v16.4s, #0x0\n" - "zip1 v0.4s, v0.4s, v25.4s\n" - ".inst 0x4fa2e9e8 // sdot v8.4s, v15.16b, v2.4b[3]\n" - "zip1 v3.4s, v3.4s, v24.4s\n" - ".inst 0x4f84e1f3 // sdot v19.4s, v15.16b, v4.4b[0]\n" - ".inst 0x4fa4e1f2 // sdot v18.4s, v15.16b, v4.4b[1]\n" - ".inst 0x4f84e9f1 // sdot v17.4s, v15.16b, v4.4b[2]\n" - ".inst 0x4fa4e9f0 // sdot v16.4s, v15.16b, v4.4b[3]\n" + "zip1 v0.4s, v0.4s, v27.4s\n" + ".inst 0x4fa2e90a // sdot v10.4s, v8.16b, v2.4b[3]\n" + "zip1 v3.4s, v3.4s, v26.4s\n" + ".inst 0x4f84e114 // sdot v20.4s, v8.16b, v4.4b[0]\n" + ".inst 0x4fa4e112 // sdot v18.4s, v8.16b, v4.4b[1]\n" + ".inst 0x4f84e911 // sdot v17.4s, v8.16b, v4.4b[2]\n" + ".inst 0x4fa4e910 // sdot v16.4s, v8.16b, v4.4b[3]\n" "movi v31.4s, #0x0\n" "movi v30.4s, #0x0\n" - "movi v29.4s, #0x0\n" - ".inst 0x4f80e1ff // sdot v31.4s, v15.16b, v0.4b[0]\n" + "movi v26.4s, #0x0\n" + ".inst 0x4f80e11f // sdot v31.4s, v8.16b, v0.4b[0]\n" + "movi v27.4s, #0x0\n" "movi v28.4s, #0x0\n" - ".inst 0x4fa0e1fe // sdot v30.4s, v15.16b, v0.4b[1]\n" - ".inst 0x4f80e9fd // sdot v29.4s, v15.16b, v0.4b[2]\n" - ".inst 0x4fa0e9fc // sdot v28.4s, v15.16b, v0.4b[3]\n" - "add v24.4s, v26.4s, v21.4s\n" - "add v25.4s, v27.4s, v20.4s\n" - "add v26.4s, v23.4s, v9.4s\n" - "add v27.4s, v22.4s, v8.4s\n" - "add v23.4s, v19.4s, v21.4s\n" - "movi v22.4s, #0x0\n" - ".inst 0x4f83e1f6 // sdot v22.4s, v15.16b, v3.4b[0]\n" - "add v21.4s, v18.4s, v20.4s\n" + ".inst 0x4fa0e11e // sdot v30.4s, v8.16b, v0.4b[1]\n" + "movi v29.4s, #0x0\n" + ".inst 0x4f80e91a // sdot v26.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4fa0e91b // sdot v27.4s, v8.16b, v0.4b[3]\n" + ".inst 0x4f83e11c // sdot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x4fa3e11d // sdot v29.4s, v8.16b, v3.4b[1]\n" + "add v24.4s, v24.4s, v21.4s\n" + "add v25.4s, v25.4s, v19.4s\n" + "add v23.4s, v23.4s, v9.4s\n" + "add v22.4s, v22.4s, v10.4s\n" + "add v21.4s, v20.4s, v21.4s\n" "movi v20.4s, #0x0\n" - ".inst 0x4fa3e1f4 // sdot v20.4s, v15.16b, v3.4b[1]\n" - "add v19.4s, v17.4s, v9.4s\n" + ".inst 0x4f83e914 // sdot v20.4s, v8.16b, v3.4b[2]\n" + "add v19.4s, v18.4s, v19.4s\n" "movi v18.4s, #0x0\n" - ".inst 0x4f83e9f2 // sdot v18.4s, v15.16b, v3.4b[2]\n" - "add v17.4s, v16.4s, v8.4s\n" - "movi v16.4s, #0x0\n" - ".inst 0x4fa3e9f0 // sdot v16.4s, v15.16b, v3.4b[3]\n" + ".inst 0x4fa3e912 // sdot v18.4s, v8.16b, v3.4b[3]\n" + "add v17.4s, v17.4s, v9.4s\n" + "add v16.4s, v16.4s, v10.4s\n" "add v24.4s, v24.4s, v31.4s\n" "add v25.4s, v25.4s, v30.4s\n" - "add v26.4s, v26.4s, v29.4s\n" - "add v27.4s, v27.4s, v28.4s\n" - "add v28.4s, v23.4s, v22.4s\n" - "add v29.4s, v21.4s, v20.4s\n" - "add v30.4s, v19.4s, v18.4s\n" - "add v31.4s, v17.4s, v16.4s\n" - "neg v13.4s, v13.4s\n" - "mul v24.4s, v24.4s, v13.4s\n" - "mul v25.4s, v25.4s, v13.4s\n" - "mul v26.4s, v26.4s, v13.4s\n" - "mul v27.4s, v27.4s, v13.4s\n" - "mul v28.4s, v28.4s, v13.4s\n" - "mul v29.4s, v29.4s, v13.4s\n" - "mul v30.4s, v30.4s, v13.4s\n" - "mul v31.4s, v31.4s, v13.4s\n" + "add v26.4s, v23.4s, v26.4s\n" + "add v27.4s, v22.4s, v27.4s\n" + "add v28.4s, v21.4s, v28.4s\n" + "add v29.4s, v19.4s, v29.4s\n" + "add v30.4s, v17.4s, v20.4s\n" + "add v31.4s, v16.4s, v18.4s\n" + "neg v12.4s, v12.4s\n" + "mul v24.4s, v24.4s, v12.4s\n" + "mul v25.4s, v25.4s, v12.4s\n" + "mul v26.4s, v26.4s, v12.4s\n" + "mul v27.4s, v27.4s, v12.4s\n" + "mul v28.4s, v28.4s, v12.4s\n" + "mul v29.4s, v29.4s, v12.4s\n" + "mul v30.4s, v30.4s, v12.4s\n" + "mul v31.4s, v31.4s, v12.4s\n" "zip1 v19.4s, v24.4s, v26.4s\n" "zip1 v18.4s, v25.4s, v27.4s\n" "zip1 v17.4s, v28.4s, v30.4s\n" "zip1 v16.4s, v29.4s, v31.4s\n" "zip1 v22.4s, v19.4s, v18.4s\n" "zip1 v23.4s, v17.4s, v16.4s\n" - "add v24.4s, v24.4s, v14.4s\n" - "add v25.4s, v25.4s, v14.4s\n" - "add v26.4s, v26.4s, v14.4s\n" - "add v27.4s, v27.4s, v14.4s\n" - "add v28.4s, v28.4s, v14.4s\n" - "add v29.4s, v29.4s, v14.4s\n" - "add v30.4s, v30.4s, v14.4s\n" - "add v31.4s, v31.4s, v14.4s\n" + "add v24.4s, v24.4s, v11.4s\n" + "add v25.4s, v25.4s, v11.4s\n" + "add v26.4s, v26.4s, v11.4s\n" + "add v27.4s, v27.4s, v11.4s\n" + "add v28.4s, v28.4s, v11.4s\n" + "add v29.4s, v29.4s, v11.4s\n" + "add v30.4s, v30.4s, v11.4s\n" + "add v31.4s, v31.4s, v11.4s\n" "ble 2f\n" "1:" // Loop - "ldr q21, [%x[params], #0x0]\n" - "ldr q20, [%x[params], #0x10]\n" + "ldr q8, [%x[params], #0x0]\n" + "ldr q21, [%x[params], #0x10]\n" ".inst 0x4f80e0b8 // sdot v24.4s, v5.16b, v0.4b[0]\n" ".inst 0x4fa0e0b9 // sdot v25.4s, v5.16b, v0.4b[1]\n" - "ldr q14, [%x[params], #0x20]\n" + "ldr q20, [%x[params], #0x20]\n" ".inst 0x4f80e8ba // sdot v26.4s, v5.16b, v0.4b[2]\n" ".inst 0x4fa0e8bb // sdot v27.4s, v5.16b, v0.4b[3]\n" "sub %x[n_channels], %x[n_channels], #0x4\n" @@ -219,43 +220,43 @@ void a64_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst_impl "ldr q5, [%x[params], #0x30]\n" ".inst 0x4f82e0f8 // sdot v24.4s, v7.16b, v2.4b[0]\n" ".inst 0x4fa2e0f9 // sdot v25.4s, v7.16b, v2.4b[1]\n" - "sqrdmulh v24.4s, v24.4s, v21.4s\n" + "sqrdmulh v24.4s, v24.4s, v8.4s\n" ".inst 0x4f82e8fa // sdot v26.4s, v7.16b, v2.4b[2]\n" ".inst 0x4fa2e8fb // sdot v27.4s, v7.16b, v2.4b[3]\n" - "sqrdmulh v25.4s, v25.4s, v21.4s\n" + "sqrdmulh v25.4s, v25.4s, v8.4s\n" ".inst 0x4f83e0dc // sdot v28.4s, v6.16b, v3.4b[0]\n" ".inst 0x4fa3e0dd // sdot v29.4s, v6.16b, v3.4b[1]\n" - "sqrdmulh v26.4s, v26.4s, v21.4s\n" + "sqrdmulh v26.4s, v26.4s, v8.4s\n" ".inst 0x4f83e8de // sdot v30.4s, v6.16b, v3.4b[2]\n" ".inst 0x4fa3e8df // sdot v31.4s, v6.16b, v3.4b[3]\n" "ldr q6, [%x[params], #0x40]\n" - "sqrdmulh v27.4s, v27.4s, v21.4s\n" + "sqrdmulh v27.4s, v27.4s, v8.4s\n" ".inst 0x4f84e0fc // sdot v28.4s, v7.16b, v4.4b[0]\n" ".inst 0x4fa4e0fd // sdot v29.4s, v7.16b, v4.4b[1]\n" - "and v19.16b, v24.16b, v20.16b\n" + "and v19.16b, v24.16b, v21.16b\n" ".inst 0x4f84e8fe // sdot v30.4s, v7.16b, v4.4b[2]\n" ".inst 0x4fa4e8ff // sdot v31.4s, v7.16b, v4.4b[3]\n" "ldr q7, [%x[params], #0x50]\n" - "and v18.16b, v25.16b, v20.16b\n" - "and v17.16b, v26.16b, v20.16b\n" - "and v16.16b, v27.16b, v20.16b\n" + "and v18.16b, v25.16b, v21.16b\n" + "and v17.16b, v26.16b, v21.16b\n" + "and v16.16b, v27.16b, v21.16b\n" "add %x[params], %x[params], #0x60\n" "sshr v19.4s, v19.4s, #0x1f\n" "sshr v18.4s, v18.4s, #0x1f\n" "sshr v17.4s, v17.4s, #0x1f\n" "sshr v16.4s, v16.4s, #0x1f\n" - "sqrdmulh v28.4s, v28.4s, v21.4s\n" - "sqrdmulh v29.4s, v29.4s, v21.4s\n" - "sqrdmulh v30.4s, v30.4s, v21.4s\n" - "sqrdmulh v31.4s, v31.4s, v21.4s\n" + "sqrdmulh v28.4s, v28.4s, v8.4s\n" + "sqrdmulh v29.4s, v29.4s, v8.4s\n" + "sqrdmulh v30.4s, v30.4s, v8.4s\n" + "sqrdmulh v31.4s, v31.4s, v8.4s\n" "sqadd v24.4s, v24.4s, v19.4s\n" "sqadd v25.4s, v25.4s, v18.4s\n" "sqadd v26.4s, v26.4s, v17.4s\n" "sqadd v27.4s, v27.4s, v16.4s\n" - "and v19.16b, v28.16b, v20.16b\n" - "and v18.16b, v29.16b, v20.16b\n" - "and v17.16b, v30.16b, v20.16b\n" - "and v16.16b, v31.16b, v20.16b\n" + "and v19.16b, v28.16b, v21.16b\n" + "and v18.16b, v29.16b, v21.16b\n" + "and v17.16b, v30.16b, v21.16b\n" + "and v16.16b, v31.16b, v21.16b\n" "sshr v19.4s, v19.4s, #0x1f\n" "sshr v18.4s, v18.4s, #0x1f\n" "sshr v17.4s, v17.4s, #0x1f\n" @@ -264,38 +265,38 @@ void a64_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst_impl "sqadd v29.4s, v29.4s, v18.4s\n" "sqadd v30.4s, v30.4s, v17.4s\n" "sqadd v31.4s, v31.4s, v16.4s\n" - "srshl v24.4s, v24.4s, v20.4s\n" - "srshl v25.4s, v25.4s, v20.4s\n" - "srshl v26.4s, v26.4s, v20.4s\n" - "srshl v27.4s, v27.4s, v20.4s\n" - "srshl v28.4s, v28.4s, v20.4s\n" - "srshl v29.4s, v29.4s, v20.4s\n" - "srshl v30.4s, v30.4s, v20.4s\n" - "srshl v31.4s, v31.4s, v20.4s\n" - "add v24.4s, v24.4s, v12.4s\n" - "add v25.4s, v25.4s, v12.4s\n" - "add v26.4s, v26.4s, v12.4s\n" - "add v27.4s, v27.4s, v12.4s\n" - "add v28.4s, v28.4s, v12.4s\n" - "add v29.4s, v29.4s, v12.4s\n" - "add v30.4s, v30.4s, v12.4s\n" - "add v31.4s, v31.4s, v12.4s\n" - "smin v24.4s, v24.4s, v10.4s\n" - "smin v25.4s, v25.4s, v10.4s\n" - "smin v26.4s, v26.4s, v10.4s\n" - "smin v27.4s, v27.4s, v10.4s\n" - "smin v28.4s, v28.4s, v10.4s\n" - "smin v29.4s, v29.4s, v10.4s\n" - "smin v30.4s, v30.4s, v10.4s\n" - "smin v31.4s, v31.4s, v10.4s\n" - "smax v24.4s, v24.4s, v11.4s\n" - "smax v25.4s, v25.4s, v11.4s\n" - "smax v26.4s, v26.4s, v11.4s\n" - "smax v27.4s, v27.4s, v11.4s\n" - "smax v28.4s, v28.4s, v11.4s\n" - "smax v29.4s, v29.4s, v11.4s\n" - "smax v30.4s, v30.4s, v11.4s\n" - "smax v31.4s, v31.4s, v11.4s\n" + "srshl v24.4s, v24.4s, v21.4s\n" + "srshl v25.4s, v25.4s, v21.4s\n" + "srshl v26.4s, v26.4s, v21.4s\n" + "srshl v27.4s, v27.4s, v21.4s\n" + "srshl v28.4s, v28.4s, v21.4s\n" + "srshl v29.4s, v29.4s, v21.4s\n" + "srshl v30.4s, v30.4s, v21.4s\n" + "srshl v31.4s, v31.4s, v21.4s\n" + "add v24.4s, v24.4s, v14.4s\n" + "add v25.4s, v25.4s, v14.4s\n" + "add v26.4s, v26.4s, v14.4s\n" + "add v27.4s, v27.4s, v14.4s\n" + "add v28.4s, v28.4s, v14.4s\n" + "add v29.4s, v29.4s, v14.4s\n" + "add v30.4s, v30.4s, v14.4s\n" + "add v31.4s, v31.4s, v14.4s\n" + "smin v24.4s, v24.4s, v15.4s\n" + "smin v25.4s, v25.4s, v15.4s\n" + "smin v26.4s, v26.4s, v15.4s\n" + "smin v27.4s, v27.4s, v15.4s\n" + "smin v28.4s, v28.4s, v15.4s\n" + "smin v29.4s, v29.4s, v15.4s\n" + "smin v30.4s, v30.4s, v15.4s\n" + "smin v31.4s, v31.4s, v15.4s\n" + "smax v24.4s, v24.4s, v13.4s\n" + "smax v25.4s, v25.4s, v13.4s\n" + "smax v26.4s, v26.4s, v13.4s\n" + "smax v27.4s, v27.4s, v13.4s\n" + "smax v28.4s, v28.4s, v13.4s\n" + "smax v29.4s, v29.4s, v13.4s\n" + "smax v30.4s, v30.4s, v13.4s\n" + "smax v31.4s, v31.4s, v13.4s\n" "uzp1 v24.16b, v24.16b, v24.16b\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" @@ -329,14 +330,14 @@ void a64_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst_impl "dup v30.4s, v23.s[2]\n" "dup v31.4s, v23.s[3]\n" "add x28, x28, #0x4\n" - "add v24.4s, v24.4s, v14.4s\n" - "add v25.4s, v25.4s, v14.4s\n" - "add v26.4s, v26.4s, v14.4s\n" - "add v27.4s, v27.4s, v14.4s\n" - "add v28.4s, v28.4s, v14.4s\n" - "add v29.4s, v29.4s, v14.4s\n" - "add v30.4s, v30.4s, v14.4s\n" - "add v31.4s, v31.4s, v14.4s\n" + "add v24.4s, v24.4s, v20.4s\n" + "add v25.4s, v25.4s, v20.4s\n" + "add v26.4s, v26.4s, v20.4s\n" + "add v27.4s, v27.4s, v20.4s\n" + "add v28.4s, v28.4s, v20.4s\n" + "add v29.4s, v29.4s, v20.4s\n" + "add v30.4s, v30.4s, v20.4s\n" + "add v31.4s, v31.4s, v20.4s\n" "bgt 1b\n" "2:" // Tail "ldr q21, [%x[params], #0x0]\n" @@ -415,30 +416,30 @@ void a64_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst_impl "srshl v29.4s, v29.4s, v20.4s\n" "srshl v30.4s, v30.4s, v20.4s\n" "srshl v31.4s, v31.4s, v20.4s\n" - "add v24.4s, v24.4s, v12.4s\n" - "add v25.4s, v25.4s, v12.4s\n" - "add v26.4s, v26.4s, v12.4s\n" - "add v27.4s, v27.4s, v12.4s\n" - "add v28.4s, v28.4s, v12.4s\n" - "add v29.4s, v29.4s, v12.4s\n" - "add v30.4s, v30.4s, v12.4s\n" - "add v31.4s, v31.4s, v12.4s\n" - "smin v24.4s, v24.4s, v10.4s\n" - "smin v25.4s, v25.4s, v10.4s\n" - "smin v26.4s, v26.4s, v10.4s\n" - "smin v27.4s, v27.4s, v10.4s\n" - "smin v28.4s, v28.4s, v10.4s\n" - "smin v29.4s, v29.4s, v10.4s\n" - "smin v30.4s, v30.4s, v10.4s\n" - "smin v31.4s, v31.4s, v10.4s\n" - "smax v24.4s, v24.4s, v11.4s\n" - "smax v25.4s, v25.4s, v11.4s\n" - "smax v26.4s, v26.4s, v11.4s\n" - "smax v27.4s, v27.4s, v11.4s\n" - "smax v28.4s, v28.4s, v11.4s\n" - "smax v29.4s, v29.4s, v11.4s\n" - "smax v30.4s, v30.4s, v11.4s\n" - "smax v31.4s, v31.4s, v11.4s\n" + "add v24.4s, v24.4s, v14.4s\n" + "add v25.4s, v25.4s, v14.4s\n" + "add v26.4s, v26.4s, v14.4s\n" + "add v27.4s, v27.4s, v14.4s\n" + "add v28.4s, v28.4s, v14.4s\n" + "add v29.4s, v29.4s, v14.4s\n" + "add v30.4s, v30.4s, v14.4s\n" + "add v31.4s, v31.4s, v14.4s\n" + "smin v24.4s, v24.4s, v15.4s\n" + "smin v25.4s, v25.4s, v15.4s\n" + "smin v26.4s, v26.4s, v15.4s\n" + "smin v27.4s, v27.4s, v15.4s\n" + "smin v28.4s, v28.4s, v15.4s\n" + "smin v29.4s, v29.4s, v15.4s\n" + "smin v30.4s, v30.4s, v15.4s\n" + "smin v31.4s, v31.4s, v15.4s\n" + "smax v24.4s, v24.4s, v13.4s\n" + "smax v25.4s, v25.4s, v13.4s\n" + "smax v26.4s, v26.4s, v13.4s\n" + "smax v27.4s, v27.4s, v13.4s\n" + "smax v28.4s, v28.4s, v13.4s\n" + "smax v29.4s, v29.4s, v13.4s\n" + "smax v30.4s, v30.4s, v13.4s\n" + "smax v31.4s, v31.4s, v13.4s\n" "uzp1 v24.16b, v24.16b, v24.16b\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" @@ -514,4 +515,5 @@ void a64_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst_impl } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst.hpp index 0641229aa7139e8172ca5646e465d12824c535e6..62b033f48d1e245d66e2914f852a2e493b69bd36 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,14 +22,14 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include -#if defined(__aarch64__) - #pragma once +#if defined(__aarch64__) + namespace arm_conv { namespace depthwise { @@ -57,4 +57,5 @@ struct a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst : } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst/generic.cpp index 9fa38c6efec187c81988ba97ab1073f969e64ed6..17afc92e3034192f74fa8527991f6a2cd526216a 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst/generic.cpp @@ -22,12 +22,13 @@ * SOFTWARE. */ -#if defined(__aarch64__) #include "arm_gemm.hpp" #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace depthwise { @@ -42,133 +43,133 @@ void a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl __asm__ __volatile__( "ldr q12, [%x[params], #0x0]\n" "ldr q8, [%x[params], #0x10]\n" - "movi v28.16b, #0x1\n" - "movi v18.4s, #0x0\n" + "movi v30.16b, #0x1\n" + "movi v17.4s, #0x0\n" "ldr q9, [%x[params], #0x20]\n" "ldr q10, [%x[params], #0x30]\n" - "movi v31.4s, #0x0\n" - "movi v24.4s, #0x0\n" + "movi v16.4s, #0x0\n" + "movi v25.4s, #0x0\n" "ldr q11, [%x[params], #0x40]\n" "ldr x20, [%x[inptrs], #0x18]\n" - "movi v30.4s, #0x0\n" - "movi v21.4s, #0x0\n" + "movi v24.4s, #0x0\n" + "movi v31.4s, #0x0\n" "ld1 { v3.16b }, [x20]\n" "ldr x20, [%x[inptrs], #0x20]\n" - "mov v16.16b, v3.16b\n" - "ext v16.16b, v16.16b, v16.16b, #0x1\n" + "mov v26.16b, v3.16b\n" + "ext v26.16b, v26.16b, v26.16b, #0x1\n" "ld1 { v4.16b }, [x20]\n" "ldr x20, [%x[inptrs], #0x10]\n" - "mov v15.16b, v4.16b\n" - "ext v15.16b, v15.16b, v15.16b, #0x1\n" + "mov v21.16b, v4.16b\n" + "ext v21.16b, v21.16b, v21.16b, #0x1\n" "ld1 { v2.16b }, [x20]\n" "ldr x20, [%x[inptrs], #0x8]\n" - "mov v20.16b, v2.16b\n" - "ext v20.16b, v20.16b, v20.16b, #0x1\n" + "mov v27.16b, v2.16b\n" + "ext v27.16b, v27.16b, v27.16b, #0x1\n" "ld1 { v1.16b }, [x20]\n" "ldr x20, [%x[inptrs], #0x28]\n" - "zip1 v3.2d, v3.2d, v16.2d\n" - "zip1 v4.2d, v4.2d, v15.2d\n" + "zip1 v3.2d, v3.2d, v26.2d\n" + "zip1 v4.2d, v4.2d, v21.2d\n" "ld1 { v5.16b }, [x20]\n" "ldr x20, [%x[inptrs], #0x30]\n" "mov v26.16b, v1.16b\n" - "mov v13.16b, v5.16b\n" + "mov v22.16b, v5.16b\n" "ld1 { v6.16b }, [x20]\n" "ldr x20, [%x[inptrs], #0x38]\n" "mov v19.16b, v6.16b\n" "ext v26.16b, v26.16b, v26.16b, #0x1\n" "ld1 { v7.16b }, [x20]\n" "ldr x20, [%x[inptrs], #0x0]\n" - "mov v17.16b, v7.16b\n" - "zip1 v2.2d, v2.2d, v20.2d\n" + "mov v21.16b, v7.16b\n" + "zip1 v2.2d, v2.2d, v27.2d\n" "ld1 { v0.16b }, [x20]\n" - "ext v13.16b, v13.16b, v13.16b, #0x1\n" + "ext v22.16b, v22.16b, v22.16b, #0x1\n" "ext v19.16b, v19.16b, v19.16b, #0x1\n" - ".inst 0x4f83e392 // sdot v18.4s, v28.16b, v3.4b[0]\n" - "ext v17.16b, v17.16b, v17.16b, #0x1\n" - ".inst 0x4f83eb9f // sdot v31.4s, v28.16b, v3.4b[2]\n" - ".inst 0x4f84e398 // sdot v24.4s, v28.16b, v4.4b[0]\n" + ".inst 0x4f83e3d1 // sdot v17.4s, v30.16b, v3.4b[0]\n" + "ext v21.16b, v21.16b, v21.16b, #0x1\n" + ".inst 0x4f83ebd0 // sdot v16.4s, v30.16b, v3.4b[2]\n" + ".inst 0x4f84e3d9 // sdot v25.4s, v30.16b, v4.4b[0]\n" "add x20, %x[qp], %[offsetof_Requantize32_b_offset]\n" "ld1r { v23.4s }, [x20]\n" - ".inst 0x4f84eb9e // sdot v30.4s, v28.16b, v4.4b[2]\n" - "mov v16.16b, v0.16b\n" - ".inst 0x4f82e395 // sdot v21.4s, v28.16b, v2.4b[0]\n" - "movi v20.4s, #0x0\n" - "movi v29.4s, #0x1\n" - ".inst 0x4f82eb94 // sdot v20.4s, v28.16b, v2.4b[2]\n" + ".inst 0x4f84ebd8 // sdot v24.4s, v30.16b, v4.4b[2]\n" + "mov v18.16b, v0.16b\n" + ".inst 0x4f82e3df // sdot v31.4s, v30.16b, v2.4b[0]\n" + "movi v29.4s, #0x0\n" + "movi v28.4s, #0x1\n" + ".inst 0x4f82ebdd // sdot v29.4s, v30.16b, v2.4b[2]\n" "add x20, %x[qp], %[offsetof_Requantize32_c_offset]\n" - "ld1r { v14.4s }, [x20]\n" - "ext v16.16b, v16.16b, v16.16b, #0x1\n" + "ld1r { v13.4s }, [x20]\n" + "ext v18.16b, v18.16b, v18.16b, #0x1\n" "zip1 v1.2d, v1.2d, v26.2d\n" - ".inst 0x4fa3e3b2 // sdot v18.4s, v29.16b, v3.4b[1]\n" - "zip1 v5.2d, v5.2d, v13.2d\n" + ".inst 0x4fa3e391 // sdot v17.4s, v28.16b, v3.4b[1]\n" + "zip1 v5.2d, v5.2d, v22.2d\n" "zip1 v6.2d, v6.2d, v19.2d\n" - ".inst 0x4fa3ebbf // sdot v31.4s, v29.16b, v3.4b[3]\n" + ".inst 0x4fa3eb90 // sdot v16.4s, v28.16b, v3.4b[3]\n" "add x20, %x[qp], %[offsetof_Requantize32_minval]\n" - "ld1r { v13.4s }, [x20]\n" - "zip1 v7.2d, v7.2d, v17.2d\n" + "ld1r { v14.4s }, [x20]\n" + "zip1 v7.2d, v7.2d, v21.2d\n" "movi v22.4s, #0x0\n" - ".inst 0x4fa4e3b8 // sdot v24.4s, v29.16b, v4.4b[1]\n" - "movi v26.4s, #0x0\n" - ".inst 0x4fa4ebbe // sdot v30.4s, v29.16b, v4.4b[3]\n" - ".inst 0x4f81e396 // sdot v22.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4fa4e399 // sdot v25.4s, v28.16b, v4.4b[1]\n" + "movi v21.4s, #0x0\n" + ".inst 0x4fa4eb98 // sdot v24.4s, v28.16b, v4.4b[3]\n" + ".inst 0x4f81e3d6 // sdot v22.4s, v30.16b, v1.4b[0]\n" "add x20, %x[qp], %[offsetof_Requantize32_maxval]\n" "ld1r { v15.4s }, [x20]\n" - "movi v25.4s, #0x0\n" + "movi v26.4s, #0x0\n" "movi v27.4s, #0x0\n" - ".inst 0x4f81eb9a // sdot v26.4s, v28.16b, v1.4b[2]\n" - "zip1 v0.2d, v0.2d, v16.2d\n" + ".inst 0x4f81ebd5 // sdot v21.4s, v30.16b, v1.4b[2]\n" + "movi v20.4s, #0x0\n" "movi v19.4s, #0x0\n" - ".inst 0x4f85e399 // sdot v25.4s, v28.16b, v5.4b[0]\n" + ".inst 0x4f85e3da // sdot v26.4s, v30.16b, v5.4b[0]\n" "cmp %x[n_channels], #0x4\n" - ".inst 0x4f85eb9b // sdot v27.4s, v28.16b, v5.4b[2]\n" - ".inst 0x4f86e393 // sdot v19.4s, v28.16b, v6.4b[0]\n" - "add v24.4s, v18.4s, v24.4s\n" - "mov x9, #0x0\n" + "zip1 v0.2d, v0.2d, v18.2d\n" "movi v18.4s, #0x0\n" - ".inst 0x4f86eb92 // sdot v18.4s, v28.16b, v6.4b[2]\n" - ".inst 0x4fa2e3b5 // sdot v21.4s, v29.16b, v2.4b[1]\n" + ".inst 0x4f85ebdb // sdot v27.4s, v30.16b, v5.4b[2]\n" + "mov x9, #0x0\n" + ".inst 0x4f86e3d4 // sdot v20.4s, v30.16b, v6.4b[0]\n" + ".inst 0x4f86ebd3 // sdot v19.4s, v30.16b, v6.4b[2]\n" + "add v17.4s, v17.4s, v25.4s\n" "mov x28, #0x0\n" - ".inst 0x4fa2ebb4 // sdot v20.4s, v29.16b, v2.4b[3]\n" - "add v17.4s, v31.4s, v30.4s\n" - ".inst 0x4fa1e3b6 // sdot v22.4s, v29.16b, v1.4b[1]\n" + "movi v25.4s, #0x0\n" + ".inst 0x4f87e3d2 // sdot v18.4s, v30.16b, v7.4b[0]\n" + ".inst 0x4f87ebd9 // sdot v25.4s, v30.16b, v7.4b[2]\n" "ldp x27, x26, [%x[outptrs], #0x0]\n" - "movi v16.4s, #0x0\n" - ".inst 0x4f87e390 // sdot v16.4s, v28.16b, v7.4b[0]\n" - ".inst 0x4fa1ebba // sdot v26.4s, v29.16b, v1.4b[3]\n" + ".inst 0x4fa2e39f // sdot v31.4s, v28.16b, v2.4b[1]\n" + ".inst 0x4fa2eb9d // sdot v29.4s, v28.16b, v2.4b[3]\n" + "add v16.4s, v16.4s, v24.4s\n" "ldp x25, x24, [%x[outptrs], #0x10]\n" - ".inst 0x4fa5e3b9 // sdot v25.4s, v29.16b, v5.4b[1]\n" - ".inst 0x4fa5ebbb // sdot v27.4s, v29.16b, v5.4b[3]\n" - "add v30.4s, v21.4s, v24.4s\n" + "movi v24.4s, #0x0\n" + ".inst 0x4f80e3d8 // sdot v24.4s, v30.16b, v0.4b[0]\n" + ".inst 0x4fa1e396 // sdot v22.4s, v28.16b, v1.4b[1]\n" "ldp x23, x22, [%x[outptrs], #0x20]\n" - ".inst 0x4fa6e3b3 // sdot v19.4s, v29.16b, v6.4b[1]\n" - ".inst 0x4fa6ebb2 // sdot v18.4s, v29.16b, v6.4b[3]\n" - "add v31.4s, v20.4s, v17.4s\n" + ".inst 0x4fa1eb95 // sdot v21.4s, v28.16b, v1.4b[3]\n" + ".inst 0x4fa5e39a // sdot v26.4s, v28.16b, v5.4b[1]\n" + "add v31.4s, v31.4s, v17.4s\n" "ldp x21, x20, [%x[outptrs], #0x30]\n" - ".inst 0x4fa7e3b0 // sdot v16.4s, v29.16b, v7.4b[1]\n" - "add v22.4s, v22.4s, v30.4s\n" + ".inst 0x4fa5eb9b // sdot v27.4s, v28.16b, v5.4b[3]\n" + ".inst 0x4fa6e394 // sdot v20.4s, v28.16b, v6.4b[1]\n" + "add v29.4s, v29.4s, v16.4s\n" "add %x[params], %x[params], #0x50\n" - "add v21.4s, v26.4s, v31.4s\n" - "add v20.4s, v25.4s, v19.4s\n" - "add v19.4s, v27.4s, v18.4s\n" - "add v18.4s, v16.4s, v24.4s\n" - "movi v16.4s, #0x0\n" - ".inst 0x4f87eb90 // sdot v16.4s, v28.16b, v7.4b[2]\n" - ".inst 0x4fa7ebb0 // sdot v16.4s, v29.16b, v7.4b[3]\n" - "add v17.4s, v16.4s, v17.4s\n" - "movi v16.4s, #0x0\n" - ".inst 0x4f80e390 // sdot v16.4s, v28.16b, v0.4b[0]\n" - ".inst 0x4fa0e3b0 // sdot v16.4s, v29.16b, v0.4b[1]\n" - "add v24.4s, v22.4s, v16.4s\n" - "add v26.4s, v22.4s, v25.4s\n" - "movi v16.4s, #0x0\n" - ".inst 0x4f80eb90 // sdot v16.4s, v28.16b, v0.4b[2]\n" - ".inst 0x4fa0ebb0 // sdot v16.4s, v29.16b, v0.4b[3]\n" - "add v25.4s, v21.4s, v16.4s\n" - "add v27.4s, v21.4s, v27.4s\n" - "add v28.4s, v20.4s, v30.4s\n" - "add v29.4s, v19.4s, v31.4s\n" - "add v30.4s, v18.4s, v20.4s\n" - "add v31.4s, v17.4s, v19.4s\n" + ".inst 0x4fa6eb93 // sdot v19.4s, v28.16b, v6.4b[3]\n" + ".inst 0x4fa7e392 // sdot v18.4s, v28.16b, v7.4b[1]\n" + "add v22.4s, v22.4s, v31.4s\n" + ".inst 0x4fa7eb99 // sdot v25.4s, v28.16b, v7.4b[3]\n" + ".inst 0x4fa0e398 // sdot v24.4s, v28.16b, v0.4b[1]\n" + "add v21.4s, v21.4s, v29.4s\n" + "add v20.4s, v26.4s, v20.4s\n" + "add v19.4s, v27.4s, v19.4s\n" + "add v18.4s, v18.4s, v17.4s\n" + "movi v17.4s, #0x0\n" + ".inst 0x4f80ebd1 // sdot v17.4s, v30.16b, v0.4b[2]\n" + ".inst 0x4fa0eb91 // sdot v17.4s, v28.16b, v0.4b[3]\n" + "add v16.4s, v25.4s, v16.4s\n" + "add v24.4s, v22.4s, v24.4s\n" + "add v25.4s, v21.4s, v17.4s\n" + "add v26.4s, v26.4s, v22.4s\n" + "add v27.4s, v27.4s, v21.4s\n" + "add v28.4s, v20.4s, v31.4s\n" + "add v29.4s, v19.4s, v29.4s\n" + "add v30.4s, v20.4s, v18.4s\n" + "add v31.4s, v19.4s, v16.4s\n" "neg v23.4s, v23.4s\n" "mul v24.4s, v24.4s, v23.4s\n" "mul v25.4s, v25.4s, v23.4s\n" @@ -194,11 +195,11 @@ void a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl "add v31.4s, v31.4s, v12.4s\n" "ble 2f\n" "1:" // Loop - "ldr q21, [%x[params], #0x60]\n" - "ldr q20, [%x[params], #0x70]\n" + "ldr q12, [%x[params], #0x60]\n" + "ldr q21, [%x[params], #0x70]\n" ".inst 0x4f80e118 // sdot v24.4s, v8.16b, v0.4b[0]\n" ".inst 0x4f80e919 // sdot v25.4s, v8.16b, v0.4b[2]\n" - "ldr q12, [%x[params], #0x80]\n" + "ldr q20, [%x[params], #0x80]\n" ".inst 0x4f81e11a // sdot v26.4s, v8.16b, v1.4b[0]\n" ".inst 0x4f81e91b // sdot v27.4s, v8.16b, v1.4b[2]\n" "sub %x[n_channels], %x[n_channels], #0x4\n" @@ -212,7 +213,7 @@ void a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl ".inst 0x4f82e91d // sdot v29.4s, v8.16b, v2.4b[2]\n" ".inst 0x4f83e11e // sdot v30.4s, v8.16b, v3.4b[0]\n" ".inst 0x4f83e91f // sdot v31.4s, v8.16b, v3.4b[2]\n" - "ldr q8, [%x[params], #0x0]\n" + "ldr q17, [%x[params], #0x0]\n" ".inst 0x4f81e158 // sdot v24.4s, v10.16b, v1.4b[0]\n" ".inst 0x4f81e959 // sdot v25.4s, v10.16b, v1.4b[2]\n" ".inst 0x4f82e15a // sdot v26.4s, v10.16b, v2.4b[0]\n" @@ -221,7 +222,7 @@ void a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl ".inst 0x4fa2e93d // sdot v29.4s, v9.16b, v2.4b[3]\n" ".inst 0x4fa3e13e // sdot v30.4s, v9.16b, v3.4b[1]\n" ".inst 0x4fa3e93f // sdot v31.4s, v9.16b, v3.4b[3]\n" - "ldr q9, [%x[params], #0x10]\n" + "ldr q16, [%x[params], #0x10]\n" ".inst 0x4fa1e178 // sdot v24.4s, v11.16b, v1.4b[1]\n" ".inst 0x4fa1e979 // sdot v25.4s, v11.16b, v1.4b[3]\n" ".inst 0x4fa2e17a // sdot v26.4s, v11.16b, v2.4b[1]\n" @@ -230,115 +231,115 @@ void a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl ".inst 0x4f83e95d // sdot v29.4s, v10.16b, v3.4b[2]\n" ".inst 0x4f84e15e // sdot v30.4s, v10.16b, v4.4b[0]\n" ".inst 0x4f84e95f // sdot v31.4s, v10.16b, v4.4b[2]\n" - "ldr q10, [%x[params], #0x20]\n" - ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" - ".inst 0x4f82e919 // sdot v25.4s, v8.16b, v2.4b[2]\n" - ".inst 0x4f83e11a // sdot v26.4s, v8.16b, v3.4b[0]\n" - ".inst 0x4f83e91b // sdot v27.4s, v8.16b, v3.4b[2]\n" + "ldr q19, [%x[params], #0x20]\n" + ".inst 0x4f82e238 // sdot v24.4s, v17.16b, v2.4b[0]\n" + ".inst 0x4f82ea39 // sdot v25.4s, v17.16b, v2.4b[2]\n" + ".inst 0x4f83e23a // sdot v26.4s, v17.16b, v3.4b[0]\n" + ".inst 0x4f83ea3b // sdot v27.4s, v17.16b, v3.4b[2]\n" ".inst 0x4fa3e17c // sdot v28.4s, v11.16b, v3.4b[1]\n" ".inst 0x4fa3e97d // sdot v29.4s, v11.16b, v3.4b[3]\n" ".inst 0x4fa4e17e // sdot v30.4s, v11.16b, v4.4b[1]\n" ".inst 0x4fa4e97f // sdot v31.4s, v11.16b, v4.4b[3]\n" - "ldr q11, [%x[params], #0x30]\n" - ".inst 0x4fa2e138 // sdot v24.4s, v9.16b, v2.4b[1]\n" - ".inst 0x4fa2e939 // sdot v25.4s, v9.16b, v2.4b[3]\n" - ".inst 0x4fa3e13a // sdot v26.4s, v9.16b, v3.4b[1]\n" - ".inst 0x4fa3e93b // sdot v27.4s, v9.16b, v3.4b[3]\n" - ".inst 0x4f84e11c // sdot v28.4s, v8.16b, v4.4b[0]\n" - ".inst 0x4f84e91d // sdot v29.4s, v8.16b, v4.4b[2]\n" - ".inst 0x4f85e11e // sdot v30.4s, v8.16b, v5.4b[0]\n" - ".inst 0x4f85e91f // sdot v31.4s, v8.16b, v5.4b[2]\n" - "ldr q8, [%x[params], #0x40]\n" - ".inst 0x4f83e158 // sdot v24.4s, v10.16b, v3.4b[0]\n" - ".inst 0x4f83e959 // sdot v25.4s, v10.16b, v3.4b[2]\n" - ".inst 0x4f84e15a // sdot v26.4s, v10.16b, v4.4b[0]\n" - ".inst 0x4f84e95b // sdot v27.4s, v10.16b, v4.4b[2]\n" - ".inst 0x4fa4e13c // sdot v28.4s, v9.16b, v4.4b[1]\n" - ".inst 0x4fa4e93d // sdot v29.4s, v9.16b, v4.4b[3]\n" - ".inst 0x4fa5e13e // sdot v30.4s, v9.16b, v5.4b[1]\n" - ".inst 0x4fa5e93f // sdot v31.4s, v9.16b, v5.4b[3]\n" - "ldr q9, [%x[params], #0x50]\n" - ".inst 0x4fa3e178 // sdot v24.4s, v11.16b, v3.4b[1]\n" - ".inst 0x4fa3e979 // sdot v25.4s, v11.16b, v3.4b[3]\n" - ".inst 0x4fa4e17a // sdot v26.4s, v11.16b, v4.4b[1]\n" - ".inst 0x4fa4e97b // sdot v27.4s, v11.16b, v4.4b[3]\n" - ".inst 0x4f85e15c // sdot v28.4s, v10.16b, v5.4b[0]\n" - ".inst 0x4f85e95d // sdot v29.4s, v10.16b, v5.4b[2]\n" - ".inst 0x4f86e15e // sdot v30.4s, v10.16b, v6.4b[0]\n" - ".inst 0x4f86e95f // sdot v31.4s, v10.16b, v6.4b[2]\n" + "ldr q18, [%x[params], #0x30]\n" + ".inst 0x4fa2e218 // sdot v24.4s, v16.16b, v2.4b[1]\n" + ".inst 0x4fa2ea19 // sdot v25.4s, v16.16b, v2.4b[3]\n" + ".inst 0x4fa3e21a // sdot v26.4s, v16.16b, v3.4b[1]\n" + ".inst 0x4fa3ea1b // sdot v27.4s, v16.16b, v3.4b[3]\n" + ".inst 0x4f84e23c // sdot v28.4s, v17.16b, v4.4b[0]\n" + ".inst 0x4f84ea3d // sdot v29.4s, v17.16b, v4.4b[2]\n" + ".inst 0x4f85e23e // sdot v30.4s, v17.16b, v5.4b[0]\n" + ".inst 0x4f85ea3f // sdot v31.4s, v17.16b, v5.4b[2]\n" + "ldr q17, [%x[params], #0x40]\n" + ".inst 0x4f83e278 // sdot v24.4s, v19.16b, v3.4b[0]\n" + ".inst 0x4f83ea79 // sdot v25.4s, v19.16b, v3.4b[2]\n" + ".inst 0x4f84e27a // sdot v26.4s, v19.16b, v4.4b[0]\n" + ".inst 0x4f84ea7b // sdot v27.4s, v19.16b, v4.4b[2]\n" + ".inst 0x4fa4e21c // sdot v28.4s, v16.16b, v4.4b[1]\n" + ".inst 0x4fa4ea1d // sdot v29.4s, v16.16b, v4.4b[3]\n" + ".inst 0x4fa5e21e // sdot v30.4s, v16.16b, v5.4b[1]\n" + ".inst 0x4fa5ea1f // sdot v31.4s, v16.16b, v5.4b[3]\n" + "ldr q16, [%x[params], #0x50]\n" + ".inst 0x4fa3e258 // sdot v24.4s, v18.16b, v3.4b[1]\n" + ".inst 0x4fa3ea59 // sdot v25.4s, v18.16b, v3.4b[3]\n" + ".inst 0x4fa4e25a // sdot v26.4s, v18.16b, v4.4b[1]\n" + ".inst 0x4fa4ea5b // sdot v27.4s, v18.16b, v4.4b[3]\n" + ".inst 0x4f85e27c // sdot v28.4s, v19.16b, v5.4b[0]\n" + ".inst 0x4f85ea7d // sdot v29.4s, v19.16b, v5.4b[2]\n" + ".inst 0x4f86e27e // sdot v30.4s, v19.16b, v6.4b[0]\n" + ".inst 0x4f86ea7f // sdot v31.4s, v19.16b, v6.4b[2]\n" "ldr q10, [%x[params], #0xb0]\n" - ".inst 0x4f84e118 // sdot v24.4s, v8.16b, v4.4b[0]\n" - ".inst 0x4f84e919 // sdot v25.4s, v8.16b, v4.4b[2]\n" - ".inst 0x4f85e11a // sdot v26.4s, v8.16b, v5.4b[0]\n" - ".inst 0x4f85e91b // sdot v27.4s, v8.16b, v5.4b[2]\n" - ".inst 0x4fa5e17c // sdot v28.4s, v11.16b, v5.4b[1]\n" - ".inst 0x4fa5e97d // sdot v29.4s, v11.16b, v5.4b[3]\n" - ".inst 0x4fa6e17e // sdot v30.4s, v11.16b, v6.4b[1]\n" - ".inst 0x4fa6e97f // sdot v31.4s, v11.16b, v6.4b[3]\n" + ".inst 0x4f84e238 // sdot v24.4s, v17.16b, v4.4b[0]\n" + ".inst 0x4f84ea39 // sdot v25.4s, v17.16b, v4.4b[2]\n" + ".inst 0x4f85e23a // sdot v26.4s, v17.16b, v5.4b[0]\n" + ".inst 0x4f85ea3b // sdot v27.4s, v17.16b, v5.4b[2]\n" + ".inst 0x4fa5e25c // sdot v28.4s, v18.16b, v5.4b[1]\n" + ".inst 0x4fa5ea5d // sdot v29.4s, v18.16b, v5.4b[3]\n" + ".inst 0x4fa6e25e // sdot v30.4s, v18.16b, v6.4b[1]\n" + ".inst 0x4fa6ea5f // sdot v31.4s, v18.16b, v6.4b[3]\n" "ldr q11, [%x[params], #0xc0]\n" - ".inst 0x4fa4e138 // sdot v24.4s, v9.16b, v4.4b[1]\n" - ".inst 0x4fa4e939 // sdot v25.4s, v9.16b, v4.4b[3]\n" - "sqrdmulh v24.4s, v24.4s, v21.4s\n" - ".inst 0x4fa5e13a // sdot v26.4s, v9.16b, v5.4b[1]\n" - ".inst 0x4fa5e93b // sdot v27.4s, v9.16b, v5.4b[3]\n" - "sqrdmulh v25.4s, v25.4s, v21.4s\n" - ".inst 0x4f86e11c // sdot v28.4s, v8.16b, v6.4b[0]\n" - ".inst 0x4f86e91d // sdot v29.4s, v8.16b, v6.4b[2]\n" - "sqrdmulh v26.4s, v26.4s, v21.4s\n" - ".inst 0x4f87e11e // sdot v30.4s, v8.16b, v7.4b[0]\n" - ".inst 0x4f87e91f // sdot v31.4s, v8.16b, v7.4b[2]\n" + ".inst 0x4fa4e218 // sdot v24.4s, v16.16b, v4.4b[1]\n" + ".inst 0x4fa4ea19 // sdot v25.4s, v16.16b, v4.4b[3]\n" + "sqrdmulh v24.4s, v24.4s, v12.4s\n" + ".inst 0x4fa5e21a // sdot v26.4s, v16.16b, v5.4b[1]\n" + ".inst 0x4fa5ea1b // sdot v27.4s, v16.16b, v5.4b[3]\n" + "sqrdmulh v25.4s, v25.4s, v12.4s\n" + ".inst 0x4f86e23c // sdot v28.4s, v17.16b, v6.4b[0]\n" + ".inst 0x4f86ea3d // sdot v29.4s, v17.16b, v6.4b[2]\n" + "sqrdmulh v26.4s, v26.4s, v12.4s\n" + ".inst 0x4f87e23e // sdot v30.4s, v17.16b, v7.4b[0]\n" + ".inst 0x4f87ea3f // sdot v31.4s, v17.16b, v7.4b[2]\n" "ldr q8, [%x[params], #0x90]\n" - "sqrdmulh v27.4s, v27.4s, v21.4s\n" - ".inst 0x4fa6e13c // sdot v28.4s, v9.16b, v6.4b[1]\n" - ".inst 0x4fa6e93d // sdot v29.4s, v9.16b, v6.4b[3]\n" - "and v19.16b, v24.16b, v20.16b\n" - ".inst 0x4fa7e13e // sdot v30.4s, v9.16b, v7.4b[1]\n" - ".inst 0x4fa7e93f // sdot v31.4s, v9.16b, v7.4b[3]\n" + "sqrdmulh v27.4s, v27.4s, v12.4s\n" + ".inst 0x4fa6e21c // sdot v28.4s, v16.16b, v6.4b[1]\n" + ".inst 0x4fa6ea1d // sdot v29.4s, v16.16b, v6.4b[3]\n" + "and v19.16b, v24.16b, v21.16b\n" + ".inst 0x4fa7e21e // sdot v30.4s, v16.16b, v7.4b[1]\n" + ".inst 0x4fa7ea1f // sdot v31.4s, v16.16b, v7.4b[3]\n" "ldr q9, [%x[params], #0xa0]\n" - "and v18.16b, v25.16b, v20.16b\n" + "and v18.16b, v25.16b, v21.16b\n" + "and v17.16b, v26.16b, v21.16b\n" + "and v16.16b, v27.16b, v21.16b\n" + "add %x[params], %x[params], #0xd0\n" "sshr v19.4s, v19.4s, #0x1f\n" "sshr v18.4s, v18.4s, #0x1f\n" - "add %x[params], %x[params], #0xd0\n" - "sqrdmulh v28.4s, v28.4s, v21.4s\n" - "sqrdmulh v29.4s, v29.4s, v21.4s\n" - "sqrdmulh v30.4s, v30.4s, v21.4s\n" - "sqrdmulh v31.4s, v31.4s, v21.4s\n" - "and v17.16b, v26.16b, v20.16b\n" "sshr v17.4s, v17.4s, #0x1f\n" - "sqadd v24.4s, v24.4s, v19.4s\n" - "and v16.16b, v27.16b, v20.16b\n" "sshr v16.4s, v16.4s, #0x1f\n" + "sqrdmulh v28.4s, v28.4s, v12.4s\n" + "sqrdmulh v29.4s, v29.4s, v12.4s\n" + "sqrdmulh v30.4s, v30.4s, v12.4s\n" + "sqrdmulh v31.4s, v31.4s, v12.4s\n" + "sqadd v24.4s, v24.4s, v19.4s\n" "sqadd v25.4s, v25.4s, v18.4s\n" "sqadd v26.4s, v26.4s, v17.4s\n" "sqadd v27.4s, v27.4s, v16.4s\n" - "and v19.16b, v28.16b, v20.16b\n" - "and v18.16b, v29.16b, v20.16b\n" - "and v17.16b, v30.16b, v20.16b\n" + "and v19.16b, v28.16b, v21.16b\n" + "and v18.16b, v29.16b, v21.16b\n" + "and v17.16b, v30.16b, v21.16b\n" + "and v16.16b, v31.16b, v21.16b\n" "sshr v19.4s, v19.4s, #0x1f\n" "sshr v18.4s, v18.4s, #0x1f\n" "sshr v17.4s, v17.4s, #0x1f\n" - "sqadd v28.4s, v28.4s, v19.4s\n" - "and v16.16b, v31.16b, v20.16b\n" "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v28.4s, v28.4s, v19.4s\n" "sqadd v29.4s, v29.4s, v18.4s\n" "sqadd v30.4s, v30.4s, v17.4s\n" "sqadd v31.4s, v31.4s, v16.4s\n" - "srshl v24.4s, v24.4s, v20.4s\n" - "srshl v25.4s, v25.4s, v20.4s\n" - "srshl v26.4s, v26.4s, v20.4s\n" - "srshl v27.4s, v27.4s, v20.4s\n" - "srshl v28.4s, v28.4s, v20.4s\n" - "srshl v29.4s, v29.4s, v20.4s\n" - "srshl v30.4s, v30.4s, v20.4s\n" - "srshl v31.4s, v31.4s, v20.4s\n" - "add v24.4s, v24.4s, v14.4s\n" - "add v25.4s, v25.4s, v14.4s\n" - "add v26.4s, v26.4s, v14.4s\n" - "add v27.4s, v27.4s, v14.4s\n" - "add v28.4s, v28.4s, v14.4s\n" - "add v29.4s, v29.4s, v14.4s\n" - "add v30.4s, v30.4s, v14.4s\n" - "add v31.4s, v31.4s, v14.4s\n" + "srshl v24.4s, v24.4s, v21.4s\n" + "srshl v25.4s, v25.4s, v21.4s\n" + "srshl v26.4s, v26.4s, v21.4s\n" + "srshl v27.4s, v27.4s, v21.4s\n" + "srshl v28.4s, v28.4s, v21.4s\n" + "srshl v29.4s, v29.4s, v21.4s\n" + "srshl v30.4s, v30.4s, v21.4s\n" + "srshl v31.4s, v31.4s, v21.4s\n" + "add v24.4s, v24.4s, v13.4s\n" + "add v25.4s, v25.4s, v13.4s\n" + "add v26.4s, v26.4s, v13.4s\n" + "add v27.4s, v27.4s, v13.4s\n" + "add v28.4s, v28.4s, v13.4s\n" + "add v29.4s, v29.4s, v13.4s\n" + "add v30.4s, v30.4s, v13.4s\n" + "add v31.4s, v31.4s, v13.4s\n" "smin v24.4s, v24.4s, v15.4s\n" "smin v25.4s, v25.4s, v15.4s\n" "smin v26.4s, v26.4s, v15.4s\n" @@ -347,14 +348,14 @@ void a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl "smin v29.4s, v29.4s, v15.4s\n" "smin v30.4s, v30.4s, v15.4s\n" "smin v31.4s, v31.4s, v15.4s\n" - "smax v24.4s, v24.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v26.4s, v26.4s, v13.4s\n" - "smax v27.4s, v27.4s, v13.4s\n" - "smax v28.4s, v28.4s, v13.4s\n" - "smax v29.4s, v29.4s, v13.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" - "smax v31.4s, v31.4s, v13.4s\n" + "smax v24.4s, v24.4s, v14.4s\n" + "smax v25.4s, v25.4s, v14.4s\n" + "smax v26.4s, v26.4s, v14.4s\n" + "smax v27.4s, v27.4s, v14.4s\n" + "smax v28.4s, v28.4s, v14.4s\n" + "smax v29.4s, v29.4s, v14.4s\n" + "smax v30.4s, v30.4s, v14.4s\n" + "smax v31.4s, v31.4s, v14.4s\n" "uzp1 v24.16b, v24.16b, v24.16b\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" @@ -388,14 +389,14 @@ void a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl "dup v30.4s, v23.s[2]\n" "dup v31.4s, v23.s[3]\n" "add x28, x28, #0x4\n" - "add v24.4s, v24.4s, v12.4s\n" - "add v25.4s, v25.4s, v12.4s\n" - "add v26.4s, v26.4s, v12.4s\n" - "add v27.4s, v27.4s, v12.4s\n" - "add v28.4s, v28.4s, v12.4s\n" - "add v29.4s, v29.4s, v12.4s\n" - "add v30.4s, v30.4s, v12.4s\n" - "add v31.4s, v31.4s, v12.4s\n" + "add v24.4s, v24.4s, v20.4s\n" + "add v25.4s, v25.4s, v20.4s\n" + "add v26.4s, v26.4s, v20.4s\n" + "add v27.4s, v27.4s, v20.4s\n" + "add v28.4s, v28.4s, v20.4s\n" + "add v29.4s, v29.4s, v20.4s\n" + "add v30.4s, v30.4s, v20.4s\n" + "add v31.4s, v31.4s, v20.4s\n" "bgt 1b\n" "2:" // Tail "ldr q21, [%x[params], #0x60]\n" @@ -420,7 +421,7 @@ void a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl "add x21, x21, x28\n" ".inst 0x4f83e11e // sdot v30.4s, v8.16b, v3.4b[0]\n" ".inst 0x4f83e91f // sdot v31.4s, v8.16b, v3.4b[2]\n" - "ldr q8, [%x[params], #0x0]\n" + "ldr q17, [%x[params], #0x0]\n" "add x20, x20, x28\n" ".inst 0x4f81e158 // sdot v24.4s, v10.16b, v1.4b[0]\n" ".inst 0x4f81e959 // sdot v25.4s, v10.16b, v1.4b[2]\n" @@ -430,7 +431,7 @@ void a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl ".inst 0x4fa2e93d // sdot v29.4s, v9.16b, v2.4b[3]\n" ".inst 0x4fa3e13e // sdot v30.4s, v9.16b, v3.4b[1]\n" ".inst 0x4fa3e93f // sdot v31.4s, v9.16b, v3.4b[3]\n" - "ldr q9, [%x[params], #0x10]\n" + "ldr q16, [%x[params], #0x10]\n" ".inst 0x4fa1e178 // sdot v24.4s, v11.16b, v1.4b[1]\n" ".inst 0x4fa1e979 // sdot v25.4s, v11.16b, v1.4b[3]\n" ".inst 0x4fa2e17a // sdot v26.4s, v11.16b, v2.4b[1]\n" @@ -439,68 +440,68 @@ void a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl ".inst 0x4f83e95d // sdot v29.4s, v10.16b, v3.4b[2]\n" ".inst 0x4f84e15e // sdot v30.4s, v10.16b, v4.4b[0]\n" ".inst 0x4f84e95f // sdot v31.4s, v10.16b, v4.4b[2]\n" - "ldr q10, [%x[params], #0x20]\n" - ".inst 0x4f82e118 // sdot v24.4s, v8.16b, v2.4b[0]\n" - ".inst 0x4f82e919 // sdot v25.4s, v8.16b, v2.4b[2]\n" - ".inst 0x4f83e11a // sdot v26.4s, v8.16b, v3.4b[0]\n" - ".inst 0x4f83e91b // sdot v27.4s, v8.16b, v3.4b[2]\n" + "ldr q19, [%x[params], #0x20]\n" + ".inst 0x4f82e238 // sdot v24.4s, v17.16b, v2.4b[0]\n" + ".inst 0x4f82ea39 // sdot v25.4s, v17.16b, v2.4b[2]\n" + ".inst 0x4f83e23a // sdot v26.4s, v17.16b, v3.4b[0]\n" + ".inst 0x4f83ea3b // sdot v27.4s, v17.16b, v3.4b[2]\n" ".inst 0x4fa3e17c // sdot v28.4s, v11.16b, v3.4b[1]\n" ".inst 0x4fa3e97d // sdot v29.4s, v11.16b, v3.4b[3]\n" ".inst 0x4fa4e17e // sdot v30.4s, v11.16b, v4.4b[1]\n" ".inst 0x4fa4e97f // sdot v31.4s, v11.16b, v4.4b[3]\n" - "ldr q11, [%x[params], #0x30]\n" - ".inst 0x4fa2e138 // sdot v24.4s, v9.16b, v2.4b[1]\n" - ".inst 0x4fa2e939 // sdot v25.4s, v9.16b, v2.4b[3]\n" - ".inst 0x4fa3e13a // sdot v26.4s, v9.16b, v3.4b[1]\n" - ".inst 0x4fa3e93b // sdot v27.4s, v9.16b, v3.4b[3]\n" - ".inst 0x4f84e11c // sdot v28.4s, v8.16b, v4.4b[0]\n" - ".inst 0x4f84e91d // sdot v29.4s, v8.16b, v4.4b[2]\n" - ".inst 0x4f85e11e // sdot v30.4s, v8.16b, v5.4b[0]\n" - ".inst 0x4f85e91f // sdot v31.4s, v8.16b, v5.4b[2]\n" - "ldr q8, [%x[params], #0x40]\n" - ".inst 0x4f83e158 // sdot v24.4s, v10.16b, v3.4b[0]\n" - ".inst 0x4f83e959 // sdot v25.4s, v10.16b, v3.4b[2]\n" - ".inst 0x4f84e15a // sdot v26.4s, v10.16b, v4.4b[0]\n" - ".inst 0x4f84e95b // sdot v27.4s, v10.16b, v4.4b[2]\n" - ".inst 0x4fa4e13c // sdot v28.4s, v9.16b, v4.4b[1]\n" - ".inst 0x4fa4e93d // sdot v29.4s, v9.16b, v4.4b[3]\n" - ".inst 0x4fa5e13e // sdot v30.4s, v9.16b, v5.4b[1]\n" - ".inst 0x4fa5e93f // sdot v31.4s, v9.16b, v5.4b[3]\n" - "ldr q9, [%x[params], #0x50]\n" + "ldr q18, [%x[params], #0x30]\n" + ".inst 0x4fa2e218 // sdot v24.4s, v16.16b, v2.4b[1]\n" + ".inst 0x4fa2ea19 // sdot v25.4s, v16.16b, v2.4b[3]\n" + ".inst 0x4fa3e21a // sdot v26.4s, v16.16b, v3.4b[1]\n" + ".inst 0x4fa3ea1b // sdot v27.4s, v16.16b, v3.4b[3]\n" + ".inst 0x4f84e23c // sdot v28.4s, v17.16b, v4.4b[0]\n" + ".inst 0x4f84ea3d // sdot v29.4s, v17.16b, v4.4b[2]\n" + ".inst 0x4f85e23e // sdot v30.4s, v17.16b, v5.4b[0]\n" + ".inst 0x4f85ea3f // sdot v31.4s, v17.16b, v5.4b[2]\n" + "ldr q17, [%x[params], #0x40]\n" + ".inst 0x4f83e278 // sdot v24.4s, v19.16b, v3.4b[0]\n" + ".inst 0x4f83ea79 // sdot v25.4s, v19.16b, v3.4b[2]\n" + ".inst 0x4f84e27a // sdot v26.4s, v19.16b, v4.4b[0]\n" + ".inst 0x4f84ea7b // sdot v27.4s, v19.16b, v4.4b[2]\n" + ".inst 0x4fa4e21c // sdot v28.4s, v16.16b, v4.4b[1]\n" + ".inst 0x4fa4ea1d // sdot v29.4s, v16.16b, v4.4b[3]\n" + ".inst 0x4fa5e21e // sdot v30.4s, v16.16b, v5.4b[1]\n" + ".inst 0x4fa5ea1f // sdot v31.4s, v16.16b, v5.4b[3]\n" + "ldr q16, [%x[params], #0x50]\n" "add %x[params], %x[params], #0x80\n" - ".inst 0x4fa3e178 // sdot v24.4s, v11.16b, v3.4b[1]\n" - ".inst 0x4fa3e979 // sdot v25.4s, v11.16b, v3.4b[3]\n" - ".inst 0x4fa4e17a // sdot v26.4s, v11.16b, v4.4b[1]\n" - ".inst 0x4fa4e97b // sdot v27.4s, v11.16b, v4.4b[3]\n" - ".inst 0x4f85e15c // sdot v28.4s, v10.16b, v5.4b[0]\n" - ".inst 0x4f85e95d // sdot v29.4s, v10.16b, v5.4b[2]\n" - ".inst 0x4f86e15e // sdot v30.4s, v10.16b, v6.4b[0]\n" - ".inst 0x4f86e95f // sdot v31.4s, v10.16b, v6.4b[2]\n" - ".inst 0x4f84e118 // sdot v24.4s, v8.16b, v4.4b[0]\n" - ".inst 0x4f84e919 // sdot v25.4s, v8.16b, v4.4b[2]\n" - ".inst 0x4f85e11a // sdot v26.4s, v8.16b, v5.4b[0]\n" - ".inst 0x4f85e91b // sdot v27.4s, v8.16b, v5.4b[2]\n" - ".inst 0x4fa5e17c // sdot v28.4s, v11.16b, v5.4b[1]\n" - ".inst 0x4fa5e97d // sdot v29.4s, v11.16b, v5.4b[3]\n" - ".inst 0x4fa6e17e // sdot v30.4s, v11.16b, v6.4b[1]\n" - ".inst 0x4fa6e97f // sdot v31.4s, v11.16b, v6.4b[3]\n" - ".inst 0x4fa4e138 // sdot v24.4s, v9.16b, v4.4b[1]\n" - ".inst 0x4fa4e939 // sdot v25.4s, v9.16b, v4.4b[3]\n" + ".inst 0x4fa3e258 // sdot v24.4s, v18.16b, v3.4b[1]\n" + ".inst 0x4fa3ea59 // sdot v25.4s, v18.16b, v3.4b[3]\n" + ".inst 0x4fa4e25a // sdot v26.4s, v18.16b, v4.4b[1]\n" + ".inst 0x4fa4ea5b // sdot v27.4s, v18.16b, v4.4b[3]\n" + ".inst 0x4f85e27c // sdot v28.4s, v19.16b, v5.4b[0]\n" + ".inst 0x4f85ea7d // sdot v29.4s, v19.16b, v5.4b[2]\n" + ".inst 0x4f86e27e // sdot v30.4s, v19.16b, v6.4b[0]\n" + ".inst 0x4f86ea7f // sdot v31.4s, v19.16b, v6.4b[2]\n" + ".inst 0x4f84e238 // sdot v24.4s, v17.16b, v4.4b[0]\n" + ".inst 0x4f84ea39 // sdot v25.4s, v17.16b, v4.4b[2]\n" + ".inst 0x4f85e23a // sdot v26.4s, v17.16b, v5.4b[0]\n" + ".inst 0x4f85ea3b // sdot v27.4s, v17.16b, v5.4b[2]\n" + ".inst 0x4fa5e25c // sdot v28.4s, v18.16b, v5.4b[1]\n" + ".inst 0x4fa5ea5d // sdot v29.4s, v18.16b, v5.4b[3]\n" + ".inst 0x4fa6e25e // sdot v30.4s, v18.16b, v6.4b[1]\n" + ".inst 0x4fa6ea5f // sdot v31.4s, v18.16b, v6.4b[3]\n" + ".inst 0x4fa4e218 // sdot v24.4s, v16.16b, v4.4b[1]\n" + ".inst 0x4fa4ea19 // sdot v25.4s, v16.16b, v4.4b[3]\n" "sqrdmulh v24.4s, v24.4s, v21.4s\n" - ".inst 0x4fa5e13a // sdot v26.4s, v9.16b, v5.4b[1]\n" - ".inst 0x4fa5e93b // sdot v27.4s, v9.16b, v5.4b[3]\n" + ".inst 0x4fa5e21a // sdot v26.4s, v16.16b, v5.4b[1]\n" + ".inst 0x4fa5ea1b // sdot v27.4s, v16.16b, v5.4b[3]\n" "sqrdmulh v25.4s, v25.4s, v21.4s\n" - ".inst 0x4f86e11c // sdot v28.4s, v8.16b, v6.4b[0]\n" - ".inst 0x4f86e91d // sdot v29.4s, v8.16b, v6.4b[2]\n" + ".inst 0x4f86e23c // sdot v28.4s, v17.16b, v6.4b[0]\n" + ".inst 0x4f86ea3d // sdot v29.4s, v17.16b, v6.4b[2]\n" "sqrdmulh v26.4s, v26.4s, v21.4s\n" - ".inst 0x4f87e11e // sdot v30.4s, v8.16b, v7.4b[0]\n" - ".inst 0x4f87e91f // sdot v31.4s, v8.16b, v7.4b[2]\n" + ".inst 0x4f87e23e // sdot v30.4s, v17.16b, v7.4b[0]\n" + ".inst 0x4f87ea3f // sdot v31.4s, v17.16b, v7.4b[2]\n" "sqrdmulh v27.4s, v27.4s, v21.4s\n" - ".inst 0x4fa6e13c // sdot v28.4s, v9.16b, v6.4b[1]\n" - ".inst 0x4fa6e93d // sdot v29.4s, v9.16b, v6.4b[3]\n" + ".inst 0x4fa6e21c // sdot v28.4s, v16.16b, v6.4b[1]\n" + ".inst 0x4fa6ea1d // sdot v29.4s, v16.16b, v6.4b[3]\n" "and v19.16b, v24.16b, v20.16b\n" - ".inst 0x4fa7e13e // sdot v30.4s, v9.16b, v7.4b[1]\n" - ".inst 0x4fa7e93f // sdot v31.4s, v9.16b, v7.4b[3]\n" + ".inst 0x4fa7e21e // sdot v30.4s, v16.16b, v7.4b[1]\n" + ".inst 0x4fa7ea1f // sdot v31.4s, v16.16b, v7.4b[3]\n" "and v18.16b, v25.16b, v20.16b\n" "and v17.16b, v26.16b, v20.16b\n" "and v16.16b, v27.16b, v20.16b\n" @@ -536,14 +537,14 @@ void a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl "srshl v29.4s, v29.4s, v20.4s\n" "srshl v30.4s, v30.4s, v20.4s\n" "srshl v31.4s, v31.4s, v20.4s\n" - "add v24.4s, v24.4s, v14.4s\n" - "add v25.4s, v25.4s, v14.4s\n" - "add v26.4s, v26.4s, v14.4s\n" - "add v27.4s, v27.4s, v14.4s\n" - "add v28.4s, v28.4s, v14.4s\n" - "add v29.4s, v29.4s, v14.4s\n" - "add v30.4s, v30.4s, v14.4s\n" - "add v31.4s, v31.4s, v14.4s\n" + "add v24.4s, v24.4s, v13.4s\n" + "add v25.4s, v25.4s, v13.4s\n" + "add v26.4s, v26.4s, v13.4s\n" + "add v27.4s, v27.4s, v13.4s\n" + "add v28.4s, v28.4s, v13.4s\n" + "add v29.4s, v29.4s, v13.4s\n" + "add v30.4s, v30.4s, v13.4s\n" + "add v31.4s, v31.4s, v13.4s\n" "smin v24.4s, v24.4s, v15.4s\n" "smin v25.4s, v25.4s, v15.4s\n" "smin v26.4s, v26.4s, v15.4s\n" @@ -552,14 +553,14 @@ void a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl "smin v29.4s, v29.4s, v15.4s\n" "smin v30.4s, v30.4s, v15.4s\n" "smin v31.4s, v31.4s, v15.4s\n" - "smax v24.4s, v24.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v26.4s, v26.4s, v13.4s\n" - "smax v27.4s, v27.4s, v13.4s\n" - "smax v28.4s, v28.4s, v13.4s\n" - "smax v29.4s, v29.4s, v13.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" - "smax v31.4s, v31.4s, v13.4s\n" + "smax v24.4s, v24.4s, v14.4s\n" + "smax v25.4s, v25.4s, v14.4s\n" + "smax v26.4s, v26.4s, v14.4s\n" + "smax v27.4s, v27.4s, v14.4s\n" + "smax v28.4s, v28.4s, v14.4s\n" + "smax v29.4s, v29.4s, v14.4s\n" + "smax v30.4s, v30.4s, v14.4s\n" + "smax v31.4s, v31.4s, v14.4s\n" "uzp1 v24.16b, v24.16b, v24.16b\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" @@ -635,4 +636,5 @@ void a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp index 3dad8d560436af45167776e13e6cb6a80d0b0c62..3f71c5fb641a8f516ac364a2dbdcb846fdb13825 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp index 3a544e069784948689ed03609f24a63420665412..b21ad484e55cc045ca913565bffc6b17ceb02a3b 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp @@ -22,12 +22,13 @@ * SOFTWARE. */ -#if defined(__aarch64__) #include "arm_gemm.hpp" #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace depthwise { @@ -47,21 +48,21 @@ void a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_imp __asm__ __volatile__( "lsr x10, %x[n_output_channels], #0x2\n" "add x20, %x[qp], %[offsetof_Requantize32_minval]\n" - "ld1r { v13.4s }, [x20]\n" + "ld1r { v15.4s }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_maxval]\n" - "ld1r { v11.4s }, [x20]\n" + "ld1r { v14.4s }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_a_offset]\n" - "ld1r { v3.16b }, [x20]\n" + "ld1r { v13.16b }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_b_offset]\n" "ld1r { v12.16b }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_c_offset]\n" - "ld1r { v14.4s }, [x20]\n" + "ld1r { v11.4s }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_per_layer_left_shift]\n" - "ld1r { v15.4s }, [x20]\n" + "ld1r { v10.4s }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_per_layer_mul]\n" "ld1r { v9.4s }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_per_layer_right_shift]\n" - "ld1r { v10.4s }, [x20]\n" + "ld1r { v8.4s }, [x20]\n" "mov x9, #0x0\n" "cbz x10, 9f\n" "1:" // Output channel loop @@ -89,256 +90,256 @@ void a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_imp "cbz %x[rq_mul_ptr], 3f\n" "lsl x20, x9, #0x2\n" "ldr q9, [%x[rq_mul_ptr], x20]\n" - "ldr q10, [%x[rq_right_shift_ptr], x20]\n" + "ldr q8, [%x[rq_right_shift_ptr], x20]\n" "cbz %x[rq_left_shift_ptr], 3f\n" - "ldr q15, [%x[rq_left_shift_ptr], x20]\n" + "ldr q10, [%x[rq_left_shift_ptr], x20]\n" "3:" // Output channel loop: Load quantization parameters: Done - "ldr s8, [%x[weights]], #0x4\n" - "mov x20, %x[inptrs]\n" - "ldp x25, x28, [x20], #0x10\n" - "lsr x21, %x[kernel_points], #0x1\n" - "ldr d2, [x25, #0x0]\n" - "ldr d7, [x28, #0x0]\n" - "ssubl v2.8h, v2.8b, v3.8b\n" - "ssubl v7.8h, v7.8b, v3.8b\n" - "ssubl v8.8h, v8.8b, v12.8b\n" - "cbz x21, 7f\n" - "ldr s6, [%x[weights]], #0x4\n" - "ldp x25, x28, [x20], #0x10\n" - "subs x21, x21, #0x1\n" - "ssubl v6.8h, v6.8b, v12.8b\n" - "ldr d1, [x25, #0x0]\n" - "ldr d0, [x28, #0x0]\n" - "ssubl v1.8h, v1.8b, v3.8b\n" - "ssubl v0.8h, v0.8b, v3.8b\n" + "ldr s5, [%x[weights]], #0x4\n" + "mov x22, %x[inptrs]\n" + "ldp x21, x20, [x22], #0x10\n" + "lsr x23, %x[kernel_points], #0x1\n" + "ldr d0, [x21, #0x0]\n" + "ldr d4, [x20, #0x0]\n" + "ssubl v0.8h, v0.8b, v13.8b\n" + "ssubl v4.8h, v4.8b, v13.8b\n" + "ssubl v5.8h, v5.8b, v12.8b\n" + "cbz x23, 7f\n" + "ldr s7, [%x[weights]], #0x4\n" + "ldp x21, x20, [x22], #0x10\n" + "subs x23, x23, #0x1\n" + "ssubl v7.8h, v7.8b, v12.8b\n" + "ldr d3, [x21, #0x0]\n" + "ldr d6, [x20, #0x0]\n" + "ssubl v3.8h, v3.8b, v13.8b\n" + "ssubl v6.8h, v6.8b, v13.8b\n" "beq 5f\n" "4:" // Output channel loop: Kernel loop - "ldp x25, x28, [x20], #0x10\n" - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "subs x21, x21, #0x1\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "ldr d2, [x25, #0x0]\n" - "ssubl v2.8h, v2.8b, v3.8b\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" - "ldr d7, [x28, #0x0]\n" - "ldr s8, [%x[weights]], #0x4\n" - "ldp x25, x28, [x20], #0x10\n" - "smlal v16.4s, v6.4h, v1.h[0]\n" - "smlal v17.4s, v6.4h, v1.h[1]\n" - "ssubl v7.8h, v7.8b, v3.8b\n" - "smlal v18.4s, v6.4h, v1.h[2]\n" - "smlal v19.4s, v6.4h, v1.h[3]\n" - "ssubl v8.8h, v8.8b, v12.8b\n" - "smlal v20.4s, v6.4h, v1.h[4]\n" - "smlal v21.4s, v6.4h, v1.h[5]\n" - "smlal v22.4s, v6.4h, v1.h[6]\n" - "smlal v23.4s, v6.4h, v1.h[7]\n" - "ldr d1, [x25, #0x0]\n" - "ssubl v1.8h, v1.8b, v3.8b\n" - "smlal v24.4s, v6.4h, v0.h[0]\n" - "smlal v25.4s, v6.4h, v0.h[1]\n" - "smlal v26.4s, v6.4h, v0.h[2]\n" - "smlal v27.4s, v6.4h, v0.h[3]\n" - "smlal v28.4s, v6.4h, v0.h[4]\n" - "smlal v29.4s, v6.4h, v0.h[5]\n" - "smlal v30.4s, v6.4h, v0.h[6]\n" - "smlal v31.4s, v6.4h, v0.h[7]\n" - "ldr d0, [x28, #0x0]\n" - "ldr s6, [%x[weights]], #0x4\n" - "ssubl v0.8h, v0.8b, v3.8b\n" - "ssubl v6.8h, v6.8b, v12.8b\n" + "ldp x21, x20, [x22], #0x10\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "subs x23, x23, #0x1\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" + "ldr d0, [x21, #0x0]\n" + "ssubl v0.8h, v0.8b, v13.8b\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" + "ldr d4, [x20, #0x0]\n" + "ldr s5, [%x[weights]], #0x4\n" + "ldp x21, x20, [x22], #0x10\n" + "smlal v16.4s, v7.4h, v3.h[0]\n" + "smlal v17.4s, v7.4h, v3.h[1]\n" + "ssubl v4.8h, v4.8b, v13.8b\n" + "smlal v18.4s, v7.4h, v3.h[2]\n" + "smlal v19.4s, v7.4h, v3.h[3]\n" + "ssubl v5.8h, v5.8b, v12.8b\n" + "smlal v20.4s, v7.4h, v3.h[4]\n" + "smlal v21.4s, v7.4h, v3.h[5]\n" + "smlal v22.4s, v7.4h, v3.h[6]\n" + "smlal v23.4s, v7.4h, v3.h[7]\n" + "ldr d3, [x21, #0x0]\n" + "ssubl v3.8h, v3.8b, v13.8b\n" + "smlal v24.4s, v7.4h, v6.h[0]\n" + "smlal v25.4s, v7.4h, v6.h[1]\n" + "smlal v26.4s, v7.4h, v6.h[2]\n" + "smlal v27.4s, v7.4h, v6.h[3]\n" + "smlal v28.4s, v7.4h, v6.h[4]\n" + "smlal v29.4s, v7.4h, v6.h[5]\n" + "smlal v30.4s, v7.4h, v6.h[6]\n" + "smlal v31.4s, v7.4h, v6.h[7]\n" + "ldr d6, [x20, #0x0]\n" + "ldr s7, [%x[weights]], #0x4\n" + "ssubl v6.8h, v6.8b, v13.8b\n" + "ssubl v7.8h, v7.8b, v12.8b\n" "bgt 4b\n" "5:" // Output channel loop: Kernel loop tail "tbnz %x[kernel_points], #0, 6f\n" - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "smlal v16.4s, v6.4h, v1.h[0]\n" - "smlal v17.4s, v6.4h, v1.h[1]\n" - "sshl v16.4s, v16.4s, v15.4s\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "smlal v18.4s, v6.4h, v1.h[2]\n" - "smlal v19.4s, v6.4h, v1.h[3]\n" - "sshl v17.4s, v17.4s, v15.4s\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "sshl v18.4s, v18.4s, v15.4s\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "sshl v19.4s, v19.4s, v15.4s\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "smlal v16.4s, v7.4h, v3.h[0]\n" + "smlal v17.4s, v7.4h, v3.h[1]\n" + "sshl v16.4s, v16.4s, v10.4s\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "smlal v18.4s, v7.4h, v3.h[2]\n" + "smlal v19.4s, v7.4h, v3.h[3]\n" + "sshl v17.4s, v17.4s, v10.4s\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" + "sshl v18.4s, v18.4s, v10.4s\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" + "sshl v19.4s, v19.4s, v10.4s\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" "sqrdmulh v16.4s, v16.4s, v9.4s\n" - "smlal v20.4s, v6.4h, v1.h[4]\n" - "smlal v21.4s, v6.4h, v1.h[5]\n" + "smlal v20.4s, v7.4h, v3.h[4]\n" + "smlal v21.4s, v7.4h, v3.h[5]\n" "sqrdmulh v17.4s, v17.4s, v9.4s\n" - "smlal v22.4s, v6.4h, v1.h[6]\n" - "smlal v23.4s, v6.4h, v1.h[7]\n" + "smlal v22.4s, v7.4h, v3.h[6]\n" + "smlal v23.4s, v7.4h, v3.h[7]\n" "sqrdmulh v18.4s, v18.4s, v9.4s\n" - "smlal v24.4s, v6.4h, v0.h[0]\n" - "smlal v25.4s, v6.4h, v0.h[1]\n" + "smlal v24.4s, v7.4h, v6.h[0]\n" + "smlal v25.4s, v7.4h, v6.h[1]\n" "sqrdmulh v19.4s, v19.4s, v9.4s\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "and v5.16b, v16.16b, v10.16b\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "and v4.16b, v17.16b, v10.16b\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" - "and v2.16b, v18.16b, v10.16b\n" - "and v1.16b, v19.16b, v10.16b\n" - "sshl v20.4s, v20.4s, v15.4s\n" - "smlal v26.4s, v6.4h, v0.h[2]\n" - "sshl v21.4s, v21.4s, v15.4s\n" - "sshl v22.4s, v22.4s, v15.4s\n" - "smlal v27.4s, v6.4h, v0.h[3]\n" - "sshl v23.4s, v23.4s, v15.4s\n" - "sshl v24.4s, v24.4s, v15.4s\n" - "smlal v28.4s, v6.4h, v0.h[4]\n" - "sshl v25.4s, v25.4s, v15.4s\n" - "smlal v29.4s, v6.4h, v0.h[5]\n" - "smlal v30.4s, v6.4h, v0.h[6]\n" - "smlal v31.4s, v6.4h, v0.h[7]\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "and v3.16b, v16.16b, v8.16b\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "and v2.16b, v17.16b, v8.16b\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" + "and v1.16b, v18.16b, v8.16b\n" + "and v0.16b, v19.16b, v8.16b\n" + "sshl v20.4s, v20.4s, v10.4s\n" + "smlal v26.4s, v7.4h, v6.h[2]\n" + "sshl v21.4s, v21.4s, v10.4s\n" + "sshl v22.4s, v22.4s, v10.4s\n" + "smlal v27.4s, v7.4h, v6.h[3]\n" + "sshl v23.4s, v23.4s, v10.4s\n" + "sshl v24.4s, v24.4s, v10.4s\n" + "smlal v28.4s, v7.4h, v6.h[4]\n" + "sshl v25.4s, v25.4s, v10.4s\n" + "smlal v29.4s, v7.4h, v6.h[5]\n" + "smlal v30.4s, v7.4h, v6.h[6]\n" + "smlal v31.4s, v7.4h, v6.h[7]\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v0.4s, v0.4s, #0x1f\n" "sqrdmulh v20.4s, v20.4s, v9.4s\n" "sqrdmulh v21.4s, v21.4s, v9.4s\n" "sqrdmulh v22.4s, v22.4s, v9.4s\n" "sqrdmulh v23.4s, v23.4s, v9.4s\n" "sqrdmulh v24.4s, v24.4s, v9.4s\n" "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqadd v16.4s, v16.4s, v5.4s\n" - "sqadd v17.4s, v17.4s, v4.4s\n" - "sqadd v18.4s, v18.4s, v2.4s\n" - "sqadd v19.4s, v19.4s, v1.4s\n" - "and v8.16b, v20.16b, v10.16b\n" - "and v0.16b, v21.16b, v10.16b\n" - "and v5.16b, v22.16b, v10.16b\n" - "and v4.16b, v23.16b, v10.16b\n" - "and v2.16b, v24.16b, v10.16b\n" - "and v1.16b, v25.16b, v10.16b\n" - "sshl v26.4s, v26.4s, v15.4s\n" - "sshl v27.4s, v27.4s, v15.4s\n" - "sshl v28.4s, v28.4s, v15.4s\n" - "sshl v29.4s, v29.4s, v15.4s\n" - "sshl v30.4s, v30.4s, v15.4s\n" - "sshl v31.4s, v31.4s, v15.4s\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v3.4s\n" + "sqadd v17.4s, v17.4s, v2.4s\n" + "sqadd v18.4s, v18.4s, v1.4s\n" + "sqadd v19.4s, v19.4s, v0.4s\n" + "and v5.16b, v20.16b, v8.16b\n" + "and v4.16b, v21.16b, v8.16b\n" + "and v3.16b, v22.16b, v8.16b\n" + "and v2.16b, v23.16b, v8.16b\n" + "and v1.16b, v24.16b, v8.16b\n" + "and v0.16b, v25.16b, v8.16b\n" + "sshl v26.4s, v26.4s, v10.4s\n" + "sshl v27.4s, v27.4s, v10.4s\n" + "sshl v28.4s, v28.4s, v10.4s\n" + "sshl v29.4s, v29.4s, v10.4s\n" + "sshl v30.4s, v30.4s, v10.4s\n" + "sshl v31.4s, v31.4s, v10.4s\n" "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v0.4s, v0.4s, #0x1f\n" "sqrdmulh v26.4s, v26.4s, v9.4s\n" "sqrdmulh v27.4s, v27.4s, v9.4s\n" "sqrdmulh v28.4s, v28.4s, v9.4s\n" "sqrdmulh v29.4s, v29.4s, v9.4s\n" "sqrdmulh v30.4s, v30.4s, v9.4s\n" "sqrdmulh v31.4s, v31.4s, v9.4s\n" - "sqadd v20.4s, v20.4s, v8.4s\n" - "sqadd v21.4s, v21.4s, v0.4s\n" - "sqadd v22.4s, v22.4s, v5.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v24.4s, v24.4s, v2.4s\n" - "sqadd v25.4s, v25.4s, v1.4s\n" - "and v8.16b, v26.16b, v10.16b\n" - "and v0.16b, v27.16b, v10.16b\n" - "and v5.16b, v28.16b, v10.16b\n" - "and v4.16b, v29.16b, v10.16b\n" - "and v2.16b, v30.16b, v10.16b\n" - "and v1.16b, v31.16b, v10.16b\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v5.4s\n" + "sqadd v21.4s, v21.4s, v4.4s\n" + "sqadd v22.4s, v22.4s, v3.4s\n" + "sqadd v23.4s, v23.4s, v2.4s\n" + "sqadd v24.4s, v24.4s, v1.4s\n" + "sqadd v25.4s, v25.4s, v0.4s\n" + "and v5.16b, v26.16b, v8.16b\n" + "and v4.16b, v27.16b, v8.16b\n" + "and v3.16b, v28.16b, v8.16b\n" + "and v2.16b, v29.16b, v8.16b\n" + "and v1.16b, v30.16b, v8.16b\n" + "and v0.16b, v31.16b, v8.16b\n" "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" - "srshl v16.4s, v16.4s, v10.4s\n" - "srshl v17.4s, v17.4s, v10.4s\n" - "srshl v18.4s, v18.4s, v10.4s\n" - "srshl v19.4s, v19.4s, v10.4s\n" - "srshl v20.4s, v20.4s, v10.4s\n" - "srshl v21.4s, v21.4s, v10.4s\n" - "srshl v22.4s, v22.4s, v10.4s\n" - "srshl v23.4s, v23.4s, v10.4s\n" - "sqadd v26.4s, v26.4s, v8.4s\n" - "sqadd v27.4s, v27.4s, v0.4s\n" - "sqadd v28.4s, v28.4s, v5.4s\n" - "sqadd v29.4s, v29.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v2.4s\n" - "sqadd v31.4s, v31.4s, v1.4s\n" - "add v16.4s, v16.4s, v14.4s\n" - "add v17.4s, v17.4s, v14.4s\n" - "add v18.4s, v18.4s, v14.4s\n" - "add v19.4s, v19.4s, v14.4s\n" - "add v20.4s, v20.4s, v14.4s\n" - "add v21.4s, v21.4s, v14.4s\n" - "add v22.4s, v22.4s, v14.4s\n" - "add v23.4s, v23.4s, v14.4s\n" - "srshl v24.4s, v24.4s, v10.4s\n" - "srshl v25.4s, v25.4s, v10.4s\n" - "srshl v26.4s, v26.4s, v10.4s\n" - "srshl v27.4s, v27.4s, v10.4s\n" - "srshl v28.4s, v28.4s, v10.4s\n" - "srshl v29.4s, v29.4s, v10.4s\n" - "srshl v30.4s, v30.4s, v10.4s\n" - "srshl v31.4s, v31.4s, v10.4s\n" - "smin v16.4s, v16.4s, v11.4s\n" - "smin v17.4s, v17.4s, v11.4s\n" - "smin v18.4s, v18.4s, v11.4s\n" - "smin v19.4s, v19.4s, v11.4s\n" - "smin v20.4s, v20.4s, v11.4s\n" - "smin v21.4s, v21.4s, v11.4s\n" - "smin v22.4s, v22.4s, v11.4s\n" - "smin v23.4s, v23.4s, v11.4s\n" - "add v24.4s, v24.4s, v14.4s\n" - "add v25.4s, v25.4s, v14.4s\n" - "add v26.4s, v26.4s, v14.4s\n" - "add v27.4s, v27.4s, v14.4s\n" - "add v28.4s, v28.4s, v14.4s\n" - "add v29.4s, v29.4s, v14.4s\n" - "add v30.4s, v30.4s, v14.4s\n" - "add v31.4s, v31.4s, v14.4s\n" - "smax v16.4s, v16.4s, v13.4s\n" - "smax v17.4s, v17.4s, v13.4s\n" - "smax v18.4s, v18.4s, v13.4s\n" - "smax v19.4s, v19.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smax v21.4s, v21.4s, v13.4s\n" - "smax v22.4s, v22.4s, v13.4s\n" - "smax v23.4s, v23.4s, v13.4s\n" - "smin v24.4s, v24.4s, v11.4s\n" - "smin v25.4s, v25.4s, v11.4s\n" - "smin v26.4s, v26.4s, v11.4s\n" - "smin v27.4s, v27.4s, v11.4s\n" - "smin v28.4s, v28.4s, v11.4s\n" - "smin v29.4s, v29.4s, v11.4s\n" - "smin v30.4s, v30.4s, v11.4s\n" - "smin v31.4s, v31.4s, v11.4s\n" + "sshr v0.4s, v0.4s, #0x1f\n" + "srshl v16.4s, v16.4s, v8.4s\n" + "srshl v17.4s, v17.4s, v8.4s\n" + "srshl v18.4s, v18.4s, v8.4s\n" + "srshl v19.4s, v19.4s, v8.4s\n" + "srshl v20.4s, v20.4s, v8.4s\n" + "srshl v21.4s, v21.4s, v8.4s\n" + "srshl v22.4s, v22.4s, v8.4s\n" + "srshl v23.4s, v23.4s, v8.4s\n" + "sqadd v26.4s, v26.4s, v5.4s\n" + "sqadd v27.4s, v27.4s, v4.4s\n" + "sqadd v28.4s, v28.4s, v3.4s\n" + "sqadd v29.4s, v29.4s, v2.4s\n" + "sqadd v30.4s, v30.4s, v1.4s\n" + "sqadd v31.4s, v31.4s, v0.4s\n" + "add v16.4s, v16.4s, v11.4s\n" + "add v17.4s, v17.4s, v11.4s\n" + "add v18.4s, v18.4s, v11.4s\n" + "add v19.4s, v19.4s, v11.4s\n" + "add v20.4s, v20.4s, v11.4s\n" + "add v21.4s, v21.4s, v11.4s\n" + "add v22.4s, v22.4s, v11.4s\n" + "add v23.4s, v23.4s, v11.4s\n" + "srshl v24.4s, v24.4s, v8.4s\n" + "srshl v25.4s, v25.4s, v8.4s\n" + "srshl v26.4s, v26.4s, v8.4s\n" + "srshl v27.4s, v27.4s, v8.4s\n" + "srshl v28.4s, v28.4s, v8.4s\n" + "srshl v29.4s, v29.4s, v8.4s\n" + "srshl v30.4s, v30.4s, v8.4s\n" + "srshl v31.4s, v31.4s, v8.4s\n" + "smin v16.4s, v16.4s, v14.4s\n" + "smin v17.4s, v17.4s, v14.4s\n" + "smin v18.4s, v18.4s, v14.4s\n" + "smin v19.4s, v19.4s, v14.4s\n" + "smin v20.4s, v20.4s, v14.4s\n" + "smin v21.4s, v21.4s, v14.4s\n" + "smin v22.4s, v22.4s, v14.4s\n" + "smin v23.4s, v23.4s, v14.4s\n" + "add v24.4s, v24.4s, v11.4s\n" + "add v25.4s, v25.4s, v11.4s\n" + "add v26.4s, v26.4s, v11.4s\n" + "add v27.4s, v27.4s, v11.4s\n" + "add v28.4s, v28.4s, v11.4s\n" + "add v29.4s, v29.4s, v11.4s\n" + "add v30.4s, v30.4s, v11.4s\n" + "add v31.4s, v31.4s, v11.4s\n" + "smax v16.4s, v16.4s, v15.4s\n" + "smax v17.4s, v17.4s, v15.4s\n" + "smax v18.4s, v18.4s, v15.4s\n" + "smax v19.4s, v19.4s, v15.4s\n" + "smax v20.4s, v20.4s, v15.4s\n" + "smax v21.4s, v21.4s, v15.4s\n" + "smax v22.4s, v22.4s, v15.4s\n" + "smax v23.4s, v23.4s, v15.4s\n" + "smin v24.4s, v24.4s, v14.4s\n" + "smin v25.4s, v25.4s, v14.4s\n" + "smin v26.4s, v26.4s, v14.4s\n" + "smin v27.4s, v27.4s, v14.4s\n" + "smin v28.4s, v28.4s, v14.4s\n" + "smin v29.4s, v29.4s, v14.4s\n" + "smin v30.4s, v30.4s, v14.4s\n" + "smin v31.4s, v31.4s, v14.4s\n" "uzp1 v16.16b, v16.16b, v16.16b\n" "uzp1 v17.16b, v17.16b, v17.16b\n" "uzp1 v18.16b, v18.16b, v18.16b\n" @@ -347,263 +348,263 @@ void a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_imp "uzp1 v21.16b, v21.16b, v21.16b\n" "uzp1 v22.16b, v22.16b, v22.16b\n" "uzp1 v23.16b, v23.16b, v23.16b\n" - "smax v24.4s, v24.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v26.4s, v26.4s, v13.4s\n" - "smax v27.4s, v27.4s, v13.4s\n" - "smax v28.4s, v28.4s, v13.4s\n" - "smax v29.4s, v29.4s, v13.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" - "smax v31.4s, v31.4s, v13.4s\n" + "smax v24.4s, v24.4s, v15.4s\n" + "smax v25.4s, v25.4s, v15.4s\n" + "smax v26.4s, v26.4s, v15.4s\n" + "smax v27.4s, v27.4s, v15.4s\n" + "smax v28.4s, v28.4s, v15.4s\n" + "smax v29.4s, v29.4s, v15.4s\n" + "smax v30.4s, v30.4s, v15.4s\n" + "smax v31.4s, v31.4s, v15.4s\n" "uzp1 v16.16b, v16.16b, v16.16b\n" - "str s16, [x20, x9]\n" - "ldr x20, [%x[outptrs], #0x40]\n" + "str s16, [x27, x9]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "uzp1 v17.16b, v17.16b, v17.16b\n" "uzp1 v18.16b, v18.16b, v18.16b\n" - "str s17, [x21, x9]\n" - "ldr x21, [%x[outptrs], #0x48]\n" + "str s17, [x26, x9]\n" + "ldr x26, [%x[outptrs], #0x48]\n" "uzp1 v19.16b, v19.16b, v19.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s18, [x22, x9]\n" - "ldr x22, [%x[outptrs], #0x50]\n" + "str s18, [x25, x9]\n" + "ldr x25, [%x[outptrs], #0x50]\n" "uzp1 v21.16b, v21.16b, v21.16b\n" "uzp1 v22.16b, v22.16b, v22.16b\n" - "str s19, [x23, x9]\n" - "ldr x23, [%x[outptrs], #0x58]\n" + "str s19, [x24, x9]\n" + "ldr x24, [%x[outptrs], #0x58]\n" "uzp1 v23.16b, v23.16b, v23.16b\n" "uzp1 v24.16b, v24.16b, v24.16b\n" - "str s20, [x24, x9]\n" - "ldr x24, [%x[outptrs], #0x60]\n" + "str s20, [x23, x9]\n" + "ldr x23, [%x[outptrs], #0x60]\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s21, [x25, x9]\n" - "ldr x25, [%x[outptrs], #0x68]\n" + "str s21, [x22, x9]\n" + "ldr x22, [%x[outptrs], #0x68]\n" "uzp1 v27.16b, v27.16b, v27.16b\n" "uzp1 v28.16b, v28.16b, v28.16b\n" - "str s22, [x26, x9]\n" - "ldr x26, [%x[outptrs], #0x70]\n" + "str s22, [x21, x9]\n" + "ldr x21, [%x[outptrs], #0x70]\n" "uzp1 v29.16b, v29.16b, v29.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s23, [x27, x9]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "str s23, [x20, x9]\n" + "ldr x20, [%x[outptrs], #0x78]\n" "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v24.16b, v24.16b, v24.16b\n" - "str s24, [x20, x9]\n" + "str s24, [x27, x9]\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s25, [x21, x9]\n" + "str s25, [x26, x9]\n" "uzp1 v27.16b, v27.16b, v27.16b\n" "uzp1 v28.16b, v28.16b, v28.16b\n" - "str s26, [x22, x9]\n" + "str s26, [x25, x9]\n" "uzp1 v29.16b, v29.16b, v29.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s27, [x23, x9]\n" + "str s27, [x24, x9]\n" "uzp1 v31.16b, v31.16b, v31.16b\n" - "str s28, [x24, x9]\n" - "str s29, [x25, x9]\n" - "str s30, [x26, x9]\n" - "str s31, [x27, x9]\n" + "str s28, [x23, x9]\n" + "str s29, [x22, x9]\n" + "str s30, [x21, x9]\n" + "str s31, [x20, x9]\n" "b 8f\n" "6:" // Output channel loop: Odd tail - "ldp x25, x28, [x20], #0x10\n" - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "ldr d2, [x25, #0x0]\n" - "ssubl v2.8h, v2.8b, v3.8b\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" - "ldr s8, [%x[weights]], #0x4\n" - "ldr d7, [x28, #0x0]\n" - "smlal v16.4s, v6.4h, v1.h[0]\n" - "smlal v17.4s, v6.4h, v1.h[1]\n" - "ssubl v8.8h, v8.8b, v12.8b\n" - "smlal v18.4s, v6.4h, v1.h[2]\n" - "smlal v19.4s, v6.4h, v1.h[3]\n" - "ssubl v7.8h, v7.8b, v3.8b\n" - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "sshl v16.4s, v16.4s, v15.4s\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "sshl v17.4s, v17.4s, v15.4s\n" - "smlal v20.4s, v6.4h, v1.h[4]\n" - "smlal v21.4s, v6.4h, v1.h[5]\n" - "sshl v18.4s, v18.4s, v15.4s\n" - "smlal v22.4s, v6.4h, v1.h[6]\n" - "smlal v23.4s, v6.4h, v1.h[7]\n" - "sshl v19.4s, v19.4s, v15.4s\n" - "smlal v24.4s, v6.4h, v0.h[0]\n" - "smlal v25.4s, v6.4h, v0.h[1]\n" + "ldp x20, x28, [x22], #0x10\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "ldr x26, [%x[outptrs], #0x8]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" + "ldr d0, [x20, #0x0]\n" + "ssubl v0.8h, v0.8b, v13.8b\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" + "ldr s5, [%x[weights]], #0x4\n" + "ldr d4, [x28, #0x0]\n" + "smlal v16.4s, v7.4h, v3.h[0]\n" + "smlal v17.4s, v7.4h, v3.h[1]\n" + "ssubl v5.8h, v5.8b, v12.8b\n" + "smlal v18.4s, v7.4h, v3.h[2]\n" + "smlal v19.4s, v7.4h, v3.h[3]\n" + "ssubl v4.8h, v4.8b, v13.8b\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "sshl v16.4s, v16.4s, v10.4s\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "sshl v17.4s, v17.4s, v10.4s\n" + "smlal v20.4s, v7.4h, v3.h[4]\n" + "smlal v21.4s, v7.4h, v3.h[5]\n" + "sshl v18.4s, v18.4s, v10.4s\n" + "smlal v22.4s, v7.4h, v3.h[6]\n" + "smlal v23.4s, v7.4h, v3.h[7]\n" + "sshl v19.4s, v19.4s, v10.4s\n" + "smlal v24.4s, v7.4h, v6.h[0]\n" + "smlal v25.4s, v7.4h, v6.h[1]\n" "sqrdmulh v16.4s, v16.4s, v9.4s\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" "sqrdmulh v17.4s, v17.4s, v9.4s\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" "sqrdmulh v18.4s, v18.4s, v9.4s\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" "sqrdmulh v19.4s, v19.4s, v9.4s\n" - "smlal v26.4s, v6.4h, v0.h[2]\n" - "smlal v27.4s, v6.4h, v0.h[3]\n" - "and v5.16b, v16.16b, v10.16b\n" - "smlal v28.4s, v6.4h, v0.h[4]\n" - "smlal v29.4s, v6.4h, v0.h[5]\n" - "and v4.16b, v17.16b, v10.16b\n" - "smlal v30.4s, v6.4h, v0.h[6]\n" - "smlal v31.4s, v6.4h, v0.h[7]\n" - "and v2.16b, v18.16b, v10.16b\n" - "and v1.16b, v19.16b, v10.16b\n" - "sshl v20.4s, v20.4s, v15.4s\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "sshl v21.4s, v21.4s, v15.4s\n" - "sshl v22.4s, v22.4s, v15.4s\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "sshl v23.4s, v23.4s, v15.4s\n" - "sshl v24.4s, v24.4s, v15.4s\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "sshl v25.4s, v25.4s, v15.4s\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" + "smlal v26.4s, v7.4h, v6.h[2]\n" + "smlal v27.4s, v7.4h, v6.h[3]\n" + "and v3.16b, v16.16b, v8.16b\n" + "smlal v28.4s, v7.4h, v6.h[4]\n" + "smlal v29.4s, v7.4h, v6.h[5]\n" + "and v2.16b, v17.16b, v8.16b\n" + "smlal v30.4s, v7.4h, v6.h[6]\n" + "smlal v31.4s, v7.4h, v6.h[7]\n" + "and v1.16b, v18.16b, v8.16b\n" + "and v0.16b, v19.16b, v8.16b\n" + "sshl v20.4s, v20.4s, v10.4s\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "sshl v21.4s, v21.4s, v10.4s\n" + "sshl v22.4s, v22.4s, v10.4s\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "sshl v23.4s, v23.4s, v10.4s\n" + "sshl v24.4s, v24.4s, v10.4s\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "sshl v25.4s, v25.4s, v10.4s\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v0.4s, v0.4s, #0x1f\n" "sqrdmulh v20.4s, v20.4s, v9.4s\n" "sqrdmulh v21.4s, v21.4s, v9.4s\n" "sqrdmulh v22.4s, v22.4s, v9.4s\n" "sqrdmulh v23.4s, v23.4s, v9.4s\n" "sqrdmulh v24.4s, v24.4s, v9.4s\n" "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqadd v16.4s, v16.4s, v5.4s\n" - "sqadd v17.4s, v17.4s, v4.4s\n" - "sqadd v18.4s, v18.4s, v2.4s\n" - "sqadd v19.4s, v19.4s, v1.4s\n" - "and v8.16b, v20.16b, v10.16b\n" - "and v0.16b, v21.16b, v10.16b\n" - "and v5.16b, v22.16b, v10.16b\n" - "and v4.16b, v23.16b, v10.16b\n" - "and v2.16b, v24.16b, v10.16b\n" - "and v1.16b, v25.16b, v10.16b\n" - "sshl v26.4s, v26.4s, v15.4s\n" - "sshl v27.4s, v27.4s, v15.4s\n" - "sshl v28.4s, v28.4s, v15.4s\n" - "sshl v29.4s, v29.4s, v15.4s\n" - "sshl v30.4s, v30.4s, v15.4s\n" - "sshl v31.4s, v31.4s, v15.4s\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v3.4s\n" + "sqadd v17.4s, v17.4s, v2.4s\n" + "sqadd v18.4s, v18.4s, v1.4s\n" + "sqadd v19.4s, v19.4s, v0.4s\n" + "and v5.16b, v20.16b, v8.16b\n" + "and v4.16b, v21.16b, v8.16b\n" + "and v3.16b, v22.16b, v8.16b\n" + "and v2.16b, v23.16b, v8.16b\n" + "and v1.16b, v24.16b, v8.16b\n" + "and v0.16b, v25.16b, v8.16b\n" + "sshl v26.4s, v26.4s, v10.4s\n" + "sshl v27.4s, v27.4s, v10.4s\n" + "sshl v28.4s, v28.4s, v10.4s\n" + "sshl v29.4s, v29.4s, v10.4s\n" + "sshl v30.4s, v30.4s, v10.4s\n" + "sshl v31.4s, v31.4s, v10.4s\n" "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v0.4s, v0.4s, #0x1f\n" "sqrdmulh v26.4s, v26.4s, v9.4s\n" "sqrdmulh v27.4s, v27.4s, v9.4s\n" "sqrdmulh v28.4s, v28.4s, v9.4s\n" "sqrdmulh v29.4s, v29.4s, v9.4s\n" "sqrdmulh v30.4s, v30.4s, v9.4s\n" "sqrdmulh v31.4s, v31.4s, v9.4s\n" - "sqadd v20.4s, v20.4s, v8.4s\n" - "sqadd v21.4s, v21.4s, v0.4s\n" - "sqadd v22.4s, v22.4s, v5.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v24.4s, v24.4s, v2.4s\n" - "sqadd v25.4s, v25.4s, v1.4s\n" - "and v8.16b, v26.16b, v10.16b\n" - "and v0.16b, v27.16b, v10.16b\n" - "and v5.16b, v28.16b, v10.16b\n" - "and v4.16b, v29.16b, v10.16b\n" - "and v2.16b, v30.16b, v10.16b\n" - "and v1.16b, v31.16b, v10.16b\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v5.4s\n" + "sqadd v21.4s, v21.4s, v4.4s\n" + "sqadd v22.4s, v22.4s, v3.4s\n" + "sqadd v23.4s, v23.4s, v2.4s\n" + "sqadd v24.4s, v24.4s, v1.4s\n" + "sqadd v25.4s, v25.4s, v0.4s\n" + "and v5.16b, v26.16b, v8.16b\n" + "and v4.16b, v27.16b, v8.16b\n" + "and v3.16b, v28.16b, v8.16b\n" + "and v2.16b, v29.16b, v8.16b\n" + "and v1.16b, v30.16b, v8.16b\n" + "and v0.16b, v31.16b, v8.16b\n" "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" - "srshl v16.4s, v16.4s, v10.4s\n" - "srshl v17.4s, v17.4s, v10.4s\n" - "srshl v18.4s, v18.4s, v10.4s\n" - "srshl v19.4s, v19.4s, v10.4s\n" - "srshl v20.4s, v20.4s, v10.4s\n" - "srshl v21.4s, v21.4s, v10.4s\n" - "srshl v22.4s, v22.4s, v10.4s\n" - "srshl v23.4s, v23.4s, v10.4s\n" - "sqadd v26.4s, v26.4s, v8.4s\n" - "sqadd v27.4s, v27.4s, v0.4s\n" - "sqadd v28.4s, v28.4s, v5.4s\n" - "sqadd v29.4s, v29.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v2.4s\n" - "sqadd v31.4s, v31.4s, v1.4s\n" - "add v16.4s, v16.4s, v14.4s\n" - "add v17.4s, v17.4s, v14.4s\n" - "add v18.4s, v18.4s, v14.4s\n" - "add v19.4s, v19.4s, v14.4s\n" - "add v20.4s, v20.4s, v14.4s\n" - "add v21.4s, v21.4s, v14.4s\n" - "add v22.4s, v22.4s, v14.4s\n" - "add v23.4s, v23.4s, v14.4s\n" - "srshl v24.4s, v24.4s, v10.4s\n" - "srshl v25.4s, v25.4s, v10.4s\n" - "srshl v26.4s, v26.4s, v10.4s\n" - "srshl v27.4s, v27.4s, v10.4s\n" - "srshl v28.4s, v28.4s, v10.4s\n" - "srshl v29.4s, v29.4s, v10.4s\n" - "srshl v30.4s, v30.4s, v10.4s\n" - "srshl v31.4s, v31.4s, v10.4s\n" - "smin v16.4s, v16.4s, v11.4s\n" - "smin v17.4s, v17.4s, v11.4s\n" - "smin v18.4s, v18.4s, v11.4s\n" - "smin v19.4s, v19.4s, v11.4s\n" - "smin v20.4s, v20.4s, v11.4s\n" - "smin v21.4s, v21.4s, v11.4s\n" - "smin v22.4s, v22.4s, v11.4s\n" - "smin v23.4s, v23.4s, v11.4s\n" - "add v24.4s, v24.4s, v14.4s\n" - "add v25.4s, v25.4s, v14.4s\n" - "add v26.4s, v26.4s, v14.4s\n" - "add v27.4s, v27.4s, v14.4s\n" - "add v28.4s, v28.4s, v14.4s\n" - "add v29.4s, v29.4s, v14.4s\n" - "add v30.4s, v30.4s, v14.4s\n" - "add v31.4s, v31.4s, v14.4s\n" - "smax v16.4s, v16.4s, v13.4s\n" - "smax v17.4s, v17.4s, v13.4s\n" - "smax v18.4s, v18.4s, v13.4s\n" - "smax v19.4s, v19.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smax v21.4s, v21.4s, v13.4s\n" - "smax v22.4s, v22.4s, v13.4s\n" - "smax v23.4s, v23.4s, v13.4s\n" - "smin v24.4s, v24.4s, v11.4s\n" - "smin v25.4s, v25.4s, v11.4s\n" - "smin v26.4s, v26.4s, v11.4s\n" - "smin v27.4s, v27.4s, v11.4s\n" - "smin v28.4s, v28.4s, v11.4s\n" - "smin v29.4s, v29.4s, v11.4s\n" - "smin v30.4s, v30.4s, v11.4s\n" - "smin v31.4s, v31.4s, v11.4s\n" + "sshr v0.4s, v0.4s, #0x1f\n" + "srshl v16.4s, v16.4s, v8.4s\n" + "srshl v17.4s, v17.4s, v8.4s\n" + "srshl v18.4s, v18.4s, v8.4s\n" + "srshl v19.4s, v19.4s, v8.4s\n" + "srshl v20.4s, v20.4s, v8.4s\n" + "srshl v21.4s, v21.4s, v8.4s\n" + "srshl v22.4s, v22.4s, v8.4s\n" + "srshl v23.4s, v23.4s, v8.4s\n" + "sqadd v26.4s, v26.4s, v5.4s\n" + "sqadd v27.4s, v27.4s, v4.4s\n" + "sqadd v28.4s, v28.4s, v3.4s\n" + "sqadd v29.4s, v29.4s, v2.4s\n" + "sqadd v30.4s, v30.4s, v1.4s\n" + "sqadd v31.4s, v31.4s, v0.4s\n" + "add v16.4s, v16.4s, v11.4s\n" + "add v17.4s, v17.4s, v11.4s\n" + "add v18.4s, v18.4s, v11.4s\n" + "add v19.4s, v19.4s, v11.4s\n" + "add v20.4s, v20.4s, v11.4s\n" + "add v21.4s, v21.4s, v11.4s\n" + "add v22.4s, v22.4s, v11.4s\n" + "add v23.4s, v23.4s, v11.4s\n" + "srshl v24.4s, v24.4s, v8.4s\n" + "srshl v25.4s, v25.4s, v8.4s\n" + "srshl v26.4s, v26.4s, v8.4s\n" + "srshl v27.4s, v27.4s, v8.4s\n" + "srshl v28.4s, v28.4s, v8.4s\n" + "srshl v29.4s, v29.4s, v8.4s\n" + "srshl v30.4s, v30.4s, v8.4s\n" + "srshl v31.4s, v31.4s, v8.4s\n" + "smin v16.4s, v16.4s, v14.4s\n" + "smin v17.4s, v17.4s, v14.4s\n" + "smin v18.4s, v18.4s, v14.4s\n" + "smin v19.4s, v19.4s, v14.4s\n" + "smin v20.4s, v20.4s, v14.4s\n" + "smin v21.4s, v21.4s, v14.4s\n" + "smin v22.4s, v22.4s, v14.4s\n" + "smin v23.4s, v23.4s, v14.4s\n" + "add v24.4s, v24.4s, v11.4s\n" + "add v25.4s, v25.4s, v11.4s\n" + "add v26.4s, v26.4s, v11.4s\n" + "add v27.4s, v27.4s, v11.4s\n" + "add v28.4s, v28.4s, v11.4s\n" + "add v29.4s, v29.4s, v11.4s\n" + "add v30.4s, v30.4s, v11.4s\n" + "add v31.4s, v31.4s, v11.4s\n" + "smax v16.4s, v16.4s, v15.4s\n" + "smax v17.4s, v17.4s, v15.4s\n" + "smax v18.4s, v18.4s, v15.4s\n" + "smax v19.4s, v19.4s, v15.4s\n" + "smax v20.4s, v20.4s, v15.4s\n" + "smax v21.4s, v21.4s, v15.4s\n" + "smax v22.4s, v22.4s, v15.4s\n" + "smax v23.4s, v23.4s, v15.4s\n" + "smin v24.4s, v24.4s, v14.4s\n" + "smin v25.4s, v25.4s, v14.4s\n" + "smin v26.4s, v26.4s, v14.4s\n" + "smin v27.4s, v27.4s, v14.4s\n" + "smin v28.4s, v28.4s, v14.4s\n" + "smin v29.4s, v29.4s, v14.4s\n" + "smin v30.4s, v30.4s, v14.4s\n" + "smin v31.4s, v31.4s, v14.4s\n" "uzp1 v16.16b, v16.16b, v16.16b\n" "uzp1 v17.16b, v17.16b, v17.16b\n" "uzp1 v18.16b, v18.16b, v18.16b\n" @@ -612,224 +613,224 @@ void a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_imp "uzp1 v21.16b, v21.16b, v21.16b\n" "uzp1 v22.16b, v22.16b, v22.16b\n" "uzp1 v23.16b, v23.16b, v23.16b\n" - "smax v24.4s, v24.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v26.4s, v26.4s, v13.4s\n" - "smax v27.4s, v27.4s, v13.4s\n" - "smax v28.4s, v28.4s, v13.4s\n" - "smax v29.4s, v29.4s, v13.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" - "smax v31.4s, v31.4s, v13.4s\n" + "smax v24.4s, v24.4s, v15.4s\n" + "smax v25.4s, v25.4s, v15.4s\n" + "smax v26.4s, v26.4s, v15.4s\n" + "smax v27.4s, v27.4s, v15.4s\n" + "smax v28.4s, v28.4s, v15.4s\n" + "smax v29.4s, v29.4s, v15.4s\n" + "smax v30.4s, v30.4s, v15.4s\n" + "smax v31.4s, v31.4s, v15.4s\n" "uzp1 v16.16b, v16.16b, v16.16b\n" - "str s16, [x20, x9]\n" - "ldr x20, [%x[outptrs], #0x40]\n" + "str s16, [x27, x9]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "uzp1 v17.16b, v17.16b, v17.16b\n" "uzp1 v18.16b, v18.16b, v18.16b\n" - "str s17, [x21, x9]\n" - "ldr x21, [%x[outptrs], #0x48]\n" + "str s17, [x26, x9]\n" + "ldr x26, [%x[outptrs], #0x48]\n" "uzp1 v19.16b, v19.16b, v19.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s18, [x22, x9]\n" - "ldr x22, [%x[outptrs], #0x50]\n" + "str s18, [x25, x9]\n" + "ldr x25, [%x[outptrs], #0x50]\n" "uzp1 v21.16b, v21.16b, v21.16b\n" "uzp1 v22.16b, v22.16b, v22.16b\n" - "str s19, [x23, x9]\n" - "ldr x23, [%x[outptrs], #0x58]\n" + "str s19, [x24, x9]\n" + "ldr x24, [%x[outptrs], #0x58]\n" "uzp1 v23.16b, v23.16b, v23.16b\n" "uzp1 v24.16b, v24.16b, v24.16b\n" - "str s20, [x24, x9]\n" - "ldr x24, [%x[outptrs], #0x60]\n" + "str s20, [x23, x9]\n" + "ldr x23, [%x[outptrs], #0x60]\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s21, [x25, x9]\n" - "ldr x25, [%x[outptrs], #0x68]\n" + "str s21, [x22, x9]\n" + "ldr x22, [%x[outptrs], #0x68]\n" "uzp1 v27.16b, v27.16b, v27.16b\n" "uzp1 v28.16b, v28.16b, v28.16b\n" - "str s22, [x26, x9]\n" - "ldr x26, [%x[outptrs], #0x70]\n" + "str s22, [x21, x9]\n" + "ldr x21, [%x[outptrs], #0x70]\n" "uzp1 v29.16b, v29.16b, v29.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s23, [x27, x9]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "str s23, [x20, x9]\n" + "ldr x20, [%x[outptrs], #0x78]\n" "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v24.16b, v24.16b, v24.16b\n" - "str s24, [x20, x9]\n" + "str s24, [x27, x9]\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s25, [x21, x9]\n" + "str s25, [x26, x9]\n" "uzp1 v27.16b, v27.16b, v27.16b\n" "uzp1 v28.16b, v28.16b, v28.16b\n" - "str s26, [x22, x9]\n" + "str s26, [x25, x9]\n" "uzp1 v29.16b, v29.16b, v29.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s27, [x23, x9]\n" + "str s27, [x24, x9]\n" "uzp1 v31.16b, v31.16b, v31.16b\n" - "str s28, [x24, x9]\n" - "str s29, [x25, x9]\n" - "str s30, [x26, x9]\n" - "str s31, [x27, x9]\n" + "str s28, [x23, x9]\n" + "str s29, [x22, x9]\n" + "str s30, [x21, x9]\n" + "str s31, [x20, x9]\n" "b 8f\n" "7:" // Output channel loop: Single kernel point - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "sshl v16.4s, v16.4s, v15.4s\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "sshl v17.4s, v17.4s, v15.4s\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "sshl v18.4s, v18.4s, v15.4s\n" - "sshl v19.4s, v19.4s, v15.4s\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "sshl v16.4s, v16.4s, v10.4s\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "sshl v17.4s, v17.4s, v10.4s\n" + "ldr x26, [%x[outptrs], #0x8]\n" + "sshl v18.4s, v18.4s, v10.4s\n" + "sshl v19.4s, v19.4s, v10.4s\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" "sqrdmulh v16.4s, v16.4s, v9.4s\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" "sqrdmulh v17.4s, v17.4s, v9.4s\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" "sqrdmulh v18.4s, v18.4s, v9.4s\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "ldr x25, [%x[outptrs], #0x28]\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "ldr x22, [%x[outptrs], #0x28]\n" "sqrdmulh v19.4s, v19.4s, v9.4s\n" - "and v5.16b, v16.16b, v10.16b\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "and v4.16b, v17.16b, v10.16b\n" - "and v2.16b, v18.16b, v10.16b\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "and v1.16b, v19.16b, v10.16b\n" - "sshl v20.4s, v20.4s, v15.4s\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "sshl v21.4s, v21.4s, v15.4s\n" - "sshl v22.4s, v22.4s, v15.4s\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "sshl v23.4s, v23.4s, v15.4s\n" - "sshl v24.4s, v24.4s, v15.4s\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" - "sshl v25.4s, v25.4s, v15.4s\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" + "and v3.16b, v16.16b, v8.16b\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "and v2.16b, v17.16b, v8.16b\n" + "and v1.16b, v18.16b, v8.16b\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "and v0.16b, v19.16b, v8.16b\n" + "sshl v20.4s, v20.4s, v10.4s\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "sshl v21.4s, v21.4s, v10.4s\n" + "sshl v22.4s, v22.4s, v10.4s\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "sshl v23.4s, v23.4s, v10.4s\n" + "sshl v24.4s, v24.4s, v10.4s\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" + "sshl v25.4s, v25.4s, v10.4s\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v0.4s, v0.4s, #0x1f\n" "sqrdmulh v20.4s, v20.4s, v9.4s\n" "sqrdmulh v21.4s, v21.4s, v9.4s\n" "sqrdmulh v22.4s, v22.4s, v9.4s\n" "sqrdmulh v23.4s, v23.4s, v9.4s\n" "sqrdmulh v24.4s, v24.4s, v9.4s\n" "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqadd v16.4s, v16.4s, v5.4s\n" - "sqadd v17.4s, v17.4s, v4.4s\n" - "sqadd v18.4s, v18.4s, v2.4s\n" - "sqadd v19.4s, v19.4s, v1.4s\n" - "and v8.16b, v20.16b, v10.16b\n" - "and v0.16b, v21.16b, v10.16b\n" - "and v5.16b, v22.16b, v10.16b\n" - "and v4.16b, v23.16b, v10.16b\n" - "and v2.16b, v24.16b, v10.16b\n" - "and v1.16b, v25.16b, v10.16b\n" - "sshl v26.4s, v26.4s, v15.4s\n" - "sshl v27.4s, v27.4s, v15.4s\n" - "sshl v28.4s, v28.4s, v15.4s\n" - "sshl v29.4s, v29.4s, v15.4s\n" - "sshl v30.4s, v30.4s, v15.4s\n" - "sshl v31.4s, v31.4s, v15.4s\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v3.4s\n" + "sqadd v17.4s, v17.4s, v2.4s\n" + "sqadd v18.4s, v18.4s, v1.4s\n" + "sqadd v19.4s, v19.4s, v0.4s\n" + "and v5.16b, v20.16b, v8.16b\n" + "and v4.16b, v21.16b, v8.16b\n" + "and v3.16b, v22.16b, v8.16b\n" + "and v2.16b, v23.16b, v8.16b\n" + "and v1.16b, v24.16b, v8.16b\n" + "and v0.16b, v25.16b, v8.16b\n" + "sshl v26.4s, v26.4s, v10.4s\n" + "sshl v27.4s, v27.4s, v10.4s\n" + "sshl v28.4s, v28.4s, v10.4s\n" + "sshl v29.4s, v29.4s, v10.4s\n" + "sshl v30.4s, v30.4s, v10.4s\n" + "sshl v31.4s, v31.4s, v10.4s\n" "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v0.4s, v0.4s, #0x1f\n" "sqrdmulh v26.4s, v26.4s, v9.4s\n" "sqrdmulh v27.4s, v27.4s, v9.4s\n" "sqrdmulh v28.4s, v28.4s, v9.4s\n" "sqrdmulh v29.4s, v29.4s, v9.4s\n" "sqrdmulh v30.4s, v30.4s, v9.4s\n" "sqrdmulh v31.4s, v31.4s, v9.4s\n" - "sqadd v20.4s, v20.4s, v8.4s\n" - "sqadd v21.4s, v21.4s, v0.4s\n" - "sqadd v22.4s, v22.4s, v5.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v24.4s, v24.4s, v2.4s\n" - "sqadd v25.4s, v25.4s, v1.4s\n" - "and v8.16b, v26.16b, v10.16b\n" - "and v0.16b, v27.16b, v10.16b\n" - "and v5.16b, v28.16b, v10.16b\n" - "and v4.16b, v29.16b, v10.16b\n" - "and v2.16b, v30.16b, v10.16b\n" - "and v1.16b, v31.16b, v10.16b\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v5.4s\n" + "sqadd v21.4s, v21.4s, v4.4s\n" + "sqadd v22.4s, v22.4s, v3.4s\n" + "sqadd v23.4s, v23.4s, v2.4s\n" + "sqadd v24.4s, v24.4s, v1.4s\n" + "sqadd v25.4s, v25.4s, v0.4s\n" + "and v5.16b, v26.16b, v8.16b\n" + "and v4.16b, v27.16b, v8.16b\n" + "and v3.16b, v28.16b, v8.16b\n" + "and v2.16b, v29.16b, v8.16b\n" + "and v1.16b, v30.16b, v8.16b\n" + "and v0.16b, v31.16b, v8.16b\n" "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" - "srshl v16.4s, v16.4s, v10.4s\n" - "srshl v17.4s, v17.4s, v10.4s\n" - "srshl v18.4s, v18.4s, v10.4s\n" - "srshl v19.4s, v19.4s, v10.4s\n" - "srshl v20.4s, v20.4s, v10.4s\n" - "srshl v21.4s, v21.4s, v10.4s\n" - "srshl v22.4s, v22.4s, v10.4s\n" - "srshl v23.4s, v23.4s, v10.4s\n" - "sqadd v26.4s, v26.4s, v8.4s\n" - "sqadd v27.4s, v27.4s, v0.4s\n" - "sqadd v28.4s, v28.4s, v5.4s\n" - "sqadd v29.4s, v29.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v2.4s\n" - "sqadd v31.4s, v31.4s, v1.4s\n" - "add v16.4s, v16.4s, v14.4s\n" - "add v17.4s, v17.4s, v14.4s\n" - "add v18.4s, v18.4s, v14.4s\n" - "add v19.4s, v19.4s, v14.4s\n" - "add v20.4s, v20.4s, v14.4s\n" - "add v21.4s, v21.4s, v14.4s\n" - "add v22.4s, v22.4s, v14.4s\n" - "add v23.4s, v23.4s, v14.4s\n" - "srshl v24.4s, v24.4s, v10.4s\n" - "srshl v25.4s, v25.4s, v10.4s\n" - "srshl v26.4s, v26.4s, v10.4s\n" - "srshl v27.4s, v27.4s, v10.4s\n" - "srshl v28.4s, v28.4s, v10.4s\n" - "srshl v29.4s, v29.4s, v10.4s\n" - "srshl v30.4s, v30.4s, v10.4s\n" - "srshl v31.4s, v31.4s, v10.4s\n" - "smin v16.4s, v16.4s, v11.4s\n" - "smin v17.4s, v17.4s, v11.4s\n" - "smin v18.4s, v18.4s, v11.4s\n" - "smin v19.4s, v19.4s, v11.4s\n" - "smin v20.4s, v20.4s, v11.4s\n" - "smin v21.4s, v21.4s, v11.4s\n" - "smin v22.4s, v22.4s, v11.4s\n" - "smin v23.4s, v23.4s, v11.4s\n" - "add v24.4s, v24.4s, v14.4s\n" - "add v25.4s, v25.4s, v14.4s\n" - "add v26.4s, v26.4s, v14.4s\n" - "add v27.4s, v27.4s, v14.4s\n" - "add v28.4s, v28.4s, v14.4s\n" - "add v29.4s, v29.4s, v14.4s\n" - "add v30.4s, v30.4s, v14.4s\n" - "add v31.4s, v31.4s, v14.4s\n" - "smax v16.4s, v16.4s, v13.4s\n" - "smax v17.4s, v17.4s, v13.4s\n" - "smax v18.4s, v18.4s, v13.4s\n" - "smax v19.4s, v19.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smax v21.4s, v21.4s, v13.4s\n" - "smax v22.4s, v22.4s, v13.4s\n" - "smax v23.4s, v23.4s, v13.4s\n" - "smin v24.4s, v24.4s, v11.4s\n" - "smin v25.4s, v25.4s, v11.4s\n" - "smin v26.4s, v26.4s, v11.4s\n" - "smin v27.4s, v27.4s, v11.4s\n" - "smin v28.4s, v28.4s, v11.4s\n" - "smin v29.4s, v29.4s, v11.4s\n" - "smin v30.4s, v30.4s, v11.4s\n" - "smin v31.4s, v31.4s, v11.4s\n" + "sshr v0.4s, v0.4s, #0x1f\n" + "srshl v16.4s, v16.4s, v8.4s\n" + "srshl v17.4s, v17.4s, v8.4s\n" + "srshl v18.4s, v18.4s, v8.4s\n" + "srshl v19.4s, v19.4s, v8.4s\n" + "srshl v20.4s, v20.4s, v8.4s\n" + "srshl v21.4s, v21.4s, v8.4s\n" + "srshl v22.4s, v22.4s, v8.4s\n" + "srshl v23.4s, v23.4s, v8.4s\n" + "sqadd v26.4s, v26.4s, v5.4s\n" + "sqadd v27.4s, v27.4s, v4.4s\n" + "sqadd v28.4s, v28.4s, v3.4s\n" + "sqadd v29.4s, v29.4s, v2.4s\n" + "sqadd v30.4s, v30.4s, v1.4s\n" + "sqadd v31.4s, v31.4s, v0.4s\n" + "add v16.4s, v16.4s, v11.4s\n" + "add v17.4s, v17.4s, v11.4s\n" + "add v18.4s, v18.4s, v11.4s\n" + "add v19.4s, v19.4s, v11.4s\n" + "add v20.4s, v20.4s, v11.4s\n" + "add v21.4s, v21.4s, v11.4s\n" + "add v22.4s, v22.4s, v11.4s\n" + "add v23.4s, v23.4s, v11.4s\n" + "srshl v24.4s, v24.4s, v8.4s\n" + "srshl v25.4s, v25.4s, v8.4s\n" + "srshl v26.4s, v26.4s, v8.4s\n" + "srshl v27.4s, v27.4s, v8.4s\n" + "srshl v28.4s, v28.4s, v8.4s\n" + "srshl v29.4s, v29.4s, v8.4s\n" + "srshl v30.4s, v30.4s, v8.4s\n" + "srshl v31.4s, v31.4s, v8.4s\n" + "smin v16.4s, v16.4s, v14.4s\n" + "smin v17.4s, v17.4s, v14.4s\n" + "smin v18.4s, v18.4s, v14.4s\n" + "smin v19.4s, v19.4s, v14.4s\n" + "smin v20.4s, v20.4s, v14.4s\n" + "smin v21.4s, v21.4s, v14.4s\n" + "smin v22.4s, v22.4s, v14.4s\n" + "smin v23.4s, v23.4s, v14.4s\n" + "add v24.4s, v24.4s, v11.4s\n" + "add v25.4s, v25.4s, v11.4s\n" + "add v26.4s, v26.4s, v11.4s\n" + "add v27.4s, v27.4s, v11.4s\n" + "add v28.4s, v28.4s, v11.4s\n" + "add v29.4s, v29.4s, v11.4s\n" + "add v30.4s, v30.4s, v11.4s\n" + "add v31.4s, v31.4s, v11.4s\n" + "smax v16.4s, v16.4s, v15.4s\n" + "smax v17.4s, v17.4s, v15.4s\n" + "smax v18.4s, v18.4s, v15.4s\n" + "smax v19.4s, v19.4s, v15.4s\n" + "smax v20.4s, v20.4s, v15.4s\n" + "smax v21.4s, v21.4s, v15.4s\n" + "smax v22.4s, v22.4s, v15.4s\n" + "smax v23.4s, v23.4s, v15.4s\n" + "smin v24.4s, v24.4s, v14.4s\n" + "smin v25.4s, v25.4s, v14.4s\n" + "smin v26.4s, v26.4s, v14.4s\n" + "smin v27.4s, v27.4s, v14.4s\n" + "smin v28.4s, v28.4s, v14.4s\n" + "smin v29.4s, v29.4s, v14.4s\n" + "smin v30.4s, v30.4s, v14.4s\n" + "smin v31.4s, v31.4s, v14.4s\n" "uzp1 v16.16b, v16.16b, v16.16b\n" "uzp1 v17.16b, v17.16b, v17.16b\n" "uzp1 v18.16b, v18.16b, v18.16b\n" @@ -838,62 +839,62 @@ void a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_imp "uzp1 v21.16b, v21.16b, v21.16b\n" "uzp1 v22.16b, v22.16b, v22.16b\n" "uzp1 v23.16b, v23.16b, v23.16b\n" - "smax v24.4s, v24.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v26.4s, v26.4s, v13.4s\n" - "smax v27.4s, v27.4s, v13.4s\n" - "smax v28.4s, v28.4s, v13.4s\n" - "smax v29.4s, v29.4s, v13.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" - "smax v31.4s, v31.4s, v13.4s\n" + "smax v24.4s, v24.4s, v15.4s\n" + "smax v25.4s, v25.4s, v15.4s\n" + "smax v26.4s, v26.4s, v15.4s\n" + "smax v27.4s, v27.4s, v15.4s\n" + "smax v28.4s, v28.4s, v15.4s\n" + "smax v29.4s, v29.4s, v15.4s\n" + "smax v30.4s, v30.4s, v15.4s\n" + "smax v31.4s, v31.4s, v15.4s\n" "uzp1 v16.16b, v16.16b, v16.16b\n" - "str s16, [x20, x9]\n" - "ldr x20, [%x[outptrs], #0x40]\n" + "str s16, [x27, x9]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "uzp1 v17.16b, v17.16b, v17.16b\n" "uzp1 v18.16b, v18.16b, v18.16b\n" - "str s17, [x21, x9]\n" - "ldr x21, [%x[outptrs], #0x48]\n" + "str s17, [x26, x9]\n" + "ldr x26, [%x[outptrs], #0x48]\n" "uzp1 v19.16b, v19.16b, v19.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s18, [x22, x9]\n" - "ldr x22, [%x[outptrs], #0x50]\n" + "str s18, [x25, x9]\n" + "ldr x25, [%x[outptrs], #0x50]\n" "uzp1 v21.16b, v21.16b, v21.16b\n" "uzp1 v22.16b, v22.16b, v22.16b\n" - "str s19, [x23, x9]\n" - "ldr x23, [%x[outptrs], #0x58]\n" + "str s19, [x24, x9]\n" + "ldr x24, [%x[outptrs], #0x58]\n" "uzp1 v23.16b, v23.16b, v23.16b\n" "uzp1 v24.16b, v24.16b, v24.16b\n" - "str s20, [x24, x9]\n" - "ldr x24, [%x[outptrs], #0x60]\n" + "str s20, [x23, x9]\n" + "ldr x23, [%x[outptrs], #0x60]\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s21, [x25, x9]\n" - "ldr x25, [%x[outptrs], #0x68]\n" + "str s21, [x22, x9]\n" + "ldr x22, [%x[outptrs], #0x68]\n" "uzp1 v27.16b, v27.16b, v27.16b\n" "uzp1 v28.16b, v28.16b, v28.16b\n" - "str s22, [x26, x9]\n" - "ldr x26, [%x[outptrs], #0x70]\n" + "str s22, [x21, x9]\n" + "ldr x21, [%x[outptrs], #0x70]\n" "uzp1 v29.16b, v29.16b, v29.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s23, [x27, x9]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "str s23, [x20, x9]\n" + "ldr x20, [%x[outptrs], #0x78]\n" "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v24.16b, v24.16b, v24.16b\n" - "str s24, [x20, x9]\n" + "str s24, [x27, x9]\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s25, [x21, x9]\n" + "str s25, [x26, x9]\n" "uzp1 v27.16b, v27.16b, v27.16b\n" "uzp1 v28.16b, v28.16b, v28.16b\n" - "str s26, [x22, x9]\n" + "str s26, [x25, x9]\n" "uzp1 v29.16b, v29.16b, v29.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s27, [x23, x9]\n" + "str s27, [x24, x9]\n" "uzp1 v31.16b, v31.16b, v31.16b\n" - "str s28, [x24, x9]\n" - "str s29, [x25, x9]\n" - "str s30, [x26, x9]\n" - "str s31, [x27, x9]\n" + "str s28, [x23, x9]\n" + "str s29, [x22, x9]\n" + "str s30, [x21, x9]\n" + "str s31, [x20, x9]\n" "8:" // Output channel loop: Done "add x9, x9, #0x4\n" "cmp x9, x10, LSL #2\n" @@ -936,354 +937,354 @@ void a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_imp "cbz %x[rq_left_shift_ptr], 15f\n" "tbz %x[n_output_channels], #1, 13f\n" "ld1 { v9.d }[0], [x22], #0x8\n" - "ld1 { v10.d }[0], [x21], #0x8\n" - "ld1 { v15.d }[0], [x20], #0x8\n" + "ld1 { v8.d }[0], [x21], #0x8\n" + "ld1 { v10.d }[0], [x20], #0x8\n" "tbz %x[n_output_channels], #0, 14f\n" "ld1 { v9.s }[2], [x22], #0x4\n" - "ld1 { v10.s }[2], [x21], #0x4\n" - "ld1 { v15.s }[2], [x20], #0x4\n" + "ld1 { v8.s }[2], [x21], #0x4\n" + "ld1 { v10.s }[2], [x20], #0x4\n" "b 14f\n" "13:" // Output channel oddments: Load quantization parameters: With left shift: Bit 1: Unset "ld1 { v9.s }[0], [x22], #0x4\n" - "ld1 { v10.s }[0], [x21], #0x4\n" - "ld1 { v15.s }[0], [x20], #0x4\n" + "ld1 { v8.s }[0], [x21], #0x4\n" + "ld1 { v10.s }[0], [x20], #0x4\n" "14:" // Output channel oddments: Load quantization parameters: With left shift: Bit 1: End "b 18f\n" "15:" // Output channel oddments: Load quantization parameters: No left shift "tbz %x[n_output_channels], #1, 16f\n" "ld1 { v9.d }[0], [x22], #0x8\n" - "ld1 { v10.d }[0], [x21], #0x8\n" + "ld1 { v8.d }[0], [x21], #0x8\n" "tbz %x[n_output_channels], #0, 17f\n" "ld1 { v9.s }[2], [x22], #0x4\n" - "ld1 { v10.s }[2], [x21], #0x4\n" + "ld1 { v8.s }[2], [x21], #0x4\n" "b 17f\n" "16:" // Output channel oddments: Load quantization parameters: No left shift: Bit 1: Unset "ld1 { v9.s }[0], [x22], #0x4\n" - "ld1 { v10.s }[0], [x21], #0x4\n" + "ld1 { v8.s }[0], [x21], #0x4\n" "17:" // Output channel oddments: Load quantization parameters: No left shift: Bit 1: End "18:" // Output channel oddments: Load quantization parameters: Done - "ldr s8, [%x[weights]], #0x4\n" - "mov x20, %x[inptrs]\n" - "ldp x25, x28, [x20], #0x10\n" - "lsr x21, %x[kernel_points], #0x1\n" - "ldr d2, [x25, #0x0]\n" - "ldr d7, [x28, #0x0]\n" - "ssubl v2.8h, v2.8b, v3.8b\n" - "ssubl v7.8h, v7.8b, v3.8b\n" - "ssubl v8.8h, v8.8b, v12.8b\n" - "cbz x21, 22f\n" - "ldr s6, [%x[weights]], #0x4\n" - "ldp x25, x28, [x20], #0x10\n" - "subs x21, x21, #0x1\n" - "ssubl v6.8h, v6.8b, v12.8b\n" - "ldr d1, [x25, #0x0]\n" - "ldr d0, [x28, #0x0]\n" - "ssubl v1.8h, v1.8b, v3.8b\n" - "ssubl v0.8h, v0.8b, v3.8b\n" + "ldr s5, [%x[weights]], #0x4\n" + "mov x22, %x[inptrs]\n" + "ldp x21, x20, [x22], #0x10\n" + "lsr x23, %x[kernel_points], #0x1\n" + "ldr d0, [x21, #0x0]\n" + "ldr d4, [x20, #0x0]\n" + "ssubl v0.8h, v0.8b, v13.8b\n" + "ssubl v4.8h, v4.8b, v13.8b\n" + "ssubl v5.8h, v5.8b, v12.8b\n" + "cbz x23, 22f\n" + "ldr s7, [%x[weights]], #0x4\n" + "ldp x21, x20, [x22], #0x10\n" + "subs x23, x23, #0x1\n" + "ssubl v7.8h, v7.8b, v12.8b\n" + "ldr d3, [x21, #0x0]\n" + "ldr d6, [x20, #0x0]\n" + "ssubl v3.8h, v3.8b, v13.8b\n" + "ssubl v6.8h, v6.8b, v13.8b\n" "beq 20f\n" "19:" // Output channel oddments: Kernel loop - "ldp x25, x28, [x20], #0x10\n" - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "subs x21, x21, #0x1\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "ldr d2, [x25, #0x0]\n" - "ssubl v2.8h, v2.8b, v3.8b\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" - "ldr d7, [x28, #0x0]\n" - "ldr s8, [%x[weights]], #0x4\n" - "ldp x25, x28, [x20], #0x10\n" - "smlal v16.4s, v6.4h, v1.h[0]\n" - "smlal v17.4s, v6.4h, v1.h[1]\n" - "ssubl v7.8h, v7.8b, v3.8b\n" - "smlal v18.4s, v6.4h, v1.h[2]\n" - "smlal v19.4s, v6.4h, v1.h[3]\n" - "ssubl v8.8h, v8.8b, v12.8b\n" - "smlal v20.4s, v6.4h, v1.h[4]\n" - "smlal v21.4s, v6.4h, v1.h[5]\n" - "smlal v22.4s, v6.4h, v1.h[6]\n" - "smlal v23.4s, v6.4h, v1.h[7]\n" - "ldr d1, [x25, #0x0]\n" - "ssubl v1.8h, v1.8b, v3.8b\n" - "smlal v24.4s, v6.4h, v0.h[0]\n" - "smlal v25.4s, v6.4h, v0.h[1]\n" - "smlal v26.4s, v6.4h, v0.h[2]\n" - "smlal v27.4s, v6.4h, v0.h[3]\n" - "smlal v28.4s, v6.4h, v0.h[4]\n" - "smlal v29.4s, v6.4h, v0.h[5]\n" - "smlal v30.4s, v6.4h, v0.h[6]\n" - "smlal v31.4s, v6.4h, v0.h[7]\n" - "ldr d0, [x28, #0x0]\n" - "ldr s6, [%x[weights]], #0x4\n" - "ssubl v0.8h, v0.8b, v3.8b\n" - "ssubl v6.8h, v6.8b, v12.8b\n" + "ldp x21, x20, [x22], #0x10\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "subs x23, x23, #0x1\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" + "ldr d0, [x21, #0x0]\n" + "ssubl v0.8h, v0.8b, v13.8b\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" + "ldr d4, [x20, #0x0]\n" + "ldr s5, [%x[weights]], #0x4\n" + "ldp x21, x20, [x22], #0x10\n" + "smlal v16.4s, v7.4h, v3.h[0]\n" + "smlal v17.4s, v7.4h, v3.h[1]\n" + "ssubl v4.8h, v4.8b, v13.8b\n" + "smlal v18.4s, v7.4h, v3.h[2]\n" + "smlal v19.4s, v7.4h, v3.h[3]\n" + "ssubl v5.8h, v5.8b, v12.8b\n" + "smlal v20.4s, v7.4h, v3.h[4]\n" + "smlal v21.4s, v7.4h, v3.h[5]\n" + "smlal v22.4s, v7.4h, v3.h[6]\n" + "smlal v23.4s, v7.4h, v3.h[7]\n" + "ldr d3, [x21, #0x0]\n" + "ssubl v3.8h, v3.8b, v13.8b\n" + "smlal v24.4s, v7.4h, v6.h[0]\n" + "smlal v25.4s, v7.4h, v6.h[1]\n" + "smlal v26.4s, v7.4h, v6.h[2]\n" + "smlal v27.4s, v7.4h, v6.h[3]\n" + "smlal v28.4s, v7.4h, v6.h[4]\n" + "smlal v29.4s, v7.4h, v6.h[5]\n" + "smlal v30.4s, v7.4h, v6.h[6]\n" + "smlal v31.4s, v7.4h, v6.h[7]\n" + "ldr d6, [x20, #0x0]\n" + "ldr s7, [%x[weights]], #0x4\n" + "ssubl v6.8h, v6.8b, v13.8b\n" + "ssubl v7.8h, v7.8b, v12.8b\n" "bgt 19b\n" "20:" // Output channel oddments: Kernel loop tail "tbnz %x[kernel_points], #0, 21f\n" - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" - "smlal v16.4s, v6.4h, v1.h[0]\n" - "smlal v17.4s, v6.4h, v1.h[1]\n" - "smlal v18.4s, v6.4h, v1.h[2]\n" - "smlal v19.4s, v6.4h, v1.h[3]\n" - "smlal v20.4s, v6.4h, v1.h[4]\n" - "smlal v21.4s, v6.4h, v1.h[5]\n" - "smlal v22.4s, v6.4h, v1.h[6]\n" - "smlal v23.4s, v6.4h, v1.h[7]\n" - "smlal v24.4s, v6.4h, v0.h[0]\n" - "smlal v25.4s, v6.4h, v0.h[1]\n" - "smlal v26.4s, v6.4h, v0.h[2]\n" - "smlal v27.4s, v6.4h, v0.h[3]\n" - "smlal v28.4s, v6.4h, v0.h[4]\n" - "smlal v29.4s, v6.4h, v0.h[5]\n" - "smlal v30.4s, v6.4h, v0.h[6]\n" - "smlal v31.4s, v6.4h, v0.h[7]\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" + "smlal v16.4s, v7.4h, v3.h[0]\n" + "smlal v17.4s, v7.4h, v3.h[1]\n" + "smlal v18.4s, v7.4h, v3.h[2]\n" + "smlal v19.4s, v7.4h, v3.h[3]\n" + "smlal v20.4s, v7.4h, v3.h[4]\n" + "smlal v21.4s, v7.4h, v3.h[5]\n" + "smlal v22.4s, v7.4h, v3.h[6]\n" + "smlal v23.4s, v7.4h, v3.h[7]\n" + "smlal v24.4s, v7.4h, v6.h[0]\n" + "smlal v25.4s, v7.4h, v6.h[1]\n" + "smlal v26.4s, v7.4h, v6.h[2]\n" + "smlal v27.4s, v7.4h, v6.h[3]\n" + "smlal v28.4s, v7.4h, v6.h[4]\n" + "smlal v29.4s, v7.4h, v6.h[5]\n" + "smlal v30.4s, v7.4h, v6.h[6]\n" + "smlal v31.4s, v7.4h, v6.h[7]\n" "b 23f\n" "21:" // Output channel oddments: Odd tail - "ldp x25, x28, [x20], #0x10\n" - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "ldr d2, [x25, #0x0]\n" - "ssubl v2.8h, v2.8b, v3.8b\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" - "ldr d7, [x28, #0x0]\n" - "ldr s8, [%x[weights]], #0x4\n" - "smlal v16.4s, v6.4h, v1.h[0]\n" - "smlal v17.4s, v6.4h, v1.h[1]\n" - "ssubl v7.8h, v7.8b, v3.8b\n" - "smlal v18.4s, v6.4h, v1.h[2]\n" - "smlal v19.4s, v6.4h, v1.h[3]\n" - "ssubl v8.8h, v8.8b, v12.8b\n" - "smlal v20.4s, v6.4h, v1.h[4]\n" - "smlal v21.4s, v6.4h, v1.h[5]\n" - "smlal v22.4s, v6.4h, v1.h[6]\n" - "smlal v23.4s, v6.4h, v1.h[7]\n" - "smlal v24.4s, v6.4h, v0.h[0]\n" - "smlal v25.4s, v6.4h, v0.h[1]\n" - "smlal v26.4s, v6.4h, v0.h[2]\n" - "smlal v27.4s, v6.4h, v0.h[3]\n" - "smlal v28.4s, v6.4h, v0.h[4]\n" - "smlal v29.4s, v6.4h, v0.h[5]\n" - "smlal v30.4s, v6.4h, v0.h[6]\n" - "smlal v31.4s, v6.4h, v0.h[7]\n" - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" + "ldp x21, x20, [x22], #0x10\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" + "ldr d2, [x21, #0x0]\n" + "ssubl v2.8h, v2.8b, v13.8b\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" + "ldr d1, [x20, #0x0]\n" + "ldr s0, [%x[weights]], #0x4\n" + "smlal v16.4s, v7.4h, v3.h[0]\n" + "smlal v17.4s, v7.4h, v3.h[1]\n" + "ssubl v1.8h, v1.8b, v13.8b\n" + "smlal v18.4s, v7.4h, v3.h[2]\n" + "smlal v19.4s, v7.4h, v3.h[3]\n" + "ssubl v0.8h, v0.8b, v12.8b\n" + "smlal v20.4s, v7.4h, v3.h[4]\n" + "smlal v21.4s, v7.4h, v3.h[5]\n" + "smlal v22.4s, v7.4h, v3.h[6]\n" + "smlal v23.4s, v7.4h, v3.h[7]\n" + "smlal v24.4s, v7.4h, v6.h[0]\n" + "smlal v25.4s, v7.4h, v6.h[1]\n" + "smlal v26.4s, v7.4h, v6.h[2]\n" + "smlal v27.4s, v7.4h, v6.h[3]\n" + "smlal v28.4s, v7.4h, v6.h[4]\n" + "smlal v29.4s, v7.4h, v6.h[5]\n" + "smlal v30.4s, v7.4h, v6.h[6]\n" + "smlal v31.4s, v7.4h, v6.h[7]\n" + "smlal v16.4s, v0.4h, v2.h[0]\n" + "smlal v17.4s, v0.4h, v2.h[1]\n" + "smlal v18.4s, v0.4h, v2.h[2]\n" + "smlal v19.4s, v0.4h, v2.h[3]\n" + "smlal v20.4s, v0.4h, v2.h[4]\n" + "smlal v21.4s, v0.4h, v2.h[5]\n" + "smlal v22.4s, v0.4h, v2.h[6]\n" + "smlal v23.4s, v0.4h, v2.h[7]\n" + "smlal v24.4s, v0.4h, v1.h[0]\n" + "smlal v25.4s, v0.4h, v1.h[1]\n" + "smlal v26.4s, v0.4h, v1.h[2]\n" + "smlal v27.4s, v0.4h, v1.h[3]\n" + "smlal v28.4s, v0.4h, v1.h[4]\n" + "smlal v29.4s, v0.4h, v1.h[5]\n" + "smlal v30.4s, v0.4h, v1.h[6]\n" + "smlal v31.4s, v0.4h, v1.h[7]\n" "b 23f\n" "22:" // Output channel oddments: Single kernel point - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" "23:" // Output channel oddments: Done - "sshl v16.4s, v16.4s, v15.4s\n" - "sshl v17.4s, v17.4s, v15.4s\n" - "sshl v18.4s, v18.4s, v15.4s\n" - "sshl v19.4s, v19.4s, v15.4s\n" + "sshl v16.4s, v16.4s, v10.4s\n" + "sshl v17.4s, v17.4s, v10.4s\n" + "sshl v18.4s, v18.4s, v10.4s\n" + "sshl v19.4s, v19.4s, v10.4s\n" "sqrdmulh v16.4s, v16.4s, v9.4s\n" "sqrdmulh v17.4s, v17.4s, v9.4s\n" "sqrdmulh v18.4s, v18.4s, v9.4s\n" "sqrdmulh v19.4s, v19.4s, v9.4s\n" - "and v5.16b, v16.16b, v10.16b\n" - "and v4.16b, v17.16b, v10.16b\n" - "and v2.16b, v18.16b, v10.16b\n" - "and v1.16b, v19.16b, v10.16b\n" - "sshl v20.4s, v20.4s, v15.4s\n" - "sshl v21.4s, v21.4s, v15.4s\n" - "sshl v22.4s, v22.4s, v15.4s\n" - "sshl v23.4s, v23.4s, v15.4s\n" - "sshl v24.4s, v24.4s, v15.4s\n" - "sshl v25.4s, v25.4s, v15.4s\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" + "and v3.16b, v16.16b, v8.16b\n" + "and v2.16b, v17.16b, v8.16b\n" + "and v1.16b, v18.16b, v8.16b\n" + "and v0.16b, v19.16b, v8.16b\n" + "sshl v20.4s, v20.4s, v10.4s\n" + "sshl v21.4s, v21.4s, v10.4s\n" + "sshl v22.4s, v22.4s, v10.4s\n" + "sshl v23.4s, v23.4s, v10.4s\n" + "sshl v24.4s, v24.4s, v10.4s\n" + "sshl v25.4s, v25.4s, v10.4s\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v0.4s, v0.4s, #0x1f\n" "sqrdmulh v20.4s, v20.4s, v9.4s\n" "sqrdmulh v21.4s, v21.4s, v9.4s\n" "sqrdmulh v22.4s, v22.4s, v9.4s\n" "sqrdmulh v23.4s, v23.4s, v9.4s\n" "sqrdmulh v24.4s, v24.4s, v9.4s\n" "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqadd v16.4s, v16.4s, v5.4s\n" - "sqadd v17.4s, v17.4s, v4.4s\n" - "sqadd v18.4s, v18.4s, v2.4s\n" - "sqadd v19.4s, v19.4s, v1.4s\n" - "and v8.16b, v20.16b, v10.16b\n" - "and v0.16b, v21.16b, v10.16b\n" - "and v5.16b, v22.16b, v10.16b\n" - "and v4.16b, v23.16b, v10.16b\n" - "and v2.16b, v24.16b, v10.16b\n" - "and v1.16b, v25.16b, v10.16b\n" - "sshl v26.4s, v26.4s, v15.4s\n" - "sshl v27.4s, v27.4s, v15.4s\n" - "sshl v28.4s, v28.4s, v15.4s\n" - "sshl v29.4s, v29.4s, v15.4s\n" - "sshl v30.4s, v30.4s, v15.4s\n" - "sshl v31.4s, v31.4s, v15.4s\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v3.4s\n" + "sqadd v17.4s, v17.4s, v2.4s\n" + "sqadd v18.4s, v18.4s, v1.4s\n" + "sqadd v19.4s, v19.4s, v0.4s\n" + "and v5.16b, v20.16b, v8.16b\n" + "and v4.16b, v21.16b, v8.16b\n" + "and v3.16b, v22.16b, v8.16b\n" + "and v2.16b, v23.16b, v8.16b\n" + "and v1.16b, v24.16b, v8.16b\n" + "and v0.16b, v25.16b, v8.16b\n" + "sshl v26.4s, v26.4s, v10.4s\n" + "sshl v27.4s, v27.4s, v10.4s\n" + "sshl v28.4s, v28.4s, v10.4s\n" + "sshl v29.4s, v29.4s, v10.4s\n" + "sshl v30.4s, v30.4s, v10.4s\n" + "sshl v31.4s, v31.4s, v10.4s\n" "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v0.4s, v0.4s, #0x1f\n" "sqrdmulh v26.4s, v26.4s, v9.4s\n" "sqrdmulh v27.4s, v27.4s, v9.4s\n" "sqrdmulh v28.4s, v28.4s, v9.4s\n" "sqrdmulh v29.4s, v29.4s, v9.4s\n" "sqrdmulh v30.4s, v30.4s, v9.4s\n" "sqrdmulh v31.4s, v31.4s, v9.4s\n" - "sqadd v20.4s, v20.4s, v8.4s\n" - "sqadd v21.4s, v21.4s, v0.4s\n" - "sqadd v22.4s, v22.4s, v5.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v24.4s, v24.4s, v2.4s\n" - "sqadd v25.4s, v25.4s, v1.4s\n" - "and v8.16b, v26.16b, v10.16b\n" - "and v0.16b, v27.16b, v10.16b\n" - "and v5.16b, v28.16b, v10.16b\n" - "and v4.16b, v29.16b, v10.16b\n" - "and v2.16b, v30.16b, v10.16b\n" - "and v1.16b, v31.16b, v10.16b\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v5.4s\n" + "sqadd v21.4s, v21.4s, v4.4s\n" + "sqadd v22.4s, v22.4s, v3.4s\n" + "sqadd v23.4s, v23.4s, v2.4s\n" + "sqadd v24.4s, v24.4s, v1.4s\n" + "sqadd v25.4s, v25.4s, v0.4s\n" + "and v5.16b, v26.16b, v8.16b\n" + "and v4.16b, v27.16b, v8.16b\n" + "and v3.16b, v28.16b, v8.16b\n" + "and v2.16b, v29.16b, v8.16b\n" + "and v1.16b, v30.16b, v8.16b\n" + "and v0.16b, v31.16b, v8.16b\n" "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" - "sqadd v26.4s, v26.4s, v8.4s\n" - "sqadd v27.4s, v27.4s, v0.4s\n" - "sqadd v28.4s, v28.4s, v5.4s\n" - "sqadd v29.4s, v29.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v2.4s\n" - "sqadd v31.4s, v31.4s, v1.4s\n" - "srshl v16.4s, v16.4s, v10.4s\n" - "srshl v17.4s, v17.4s, v10.4s\n" - "srshl v18.4s, v18.4s, v10.4s\n" - "srshl v19.4s, v19.4s, v10.4s\n" - "srshl v20.4s, v20.4s, v10.4s\n" - "srshl v21.4s, v21.4s, v10.4s\n" - "srshl v22.4s, v22.4s, v10.4s\n" - "srshl v23.4s, v23.4s, v10.4s\n" - "srshl v24.4s, v24.4s, v10.4s\n" - "srshl v25.4s, v25.4s, v10.4s\n" - "srshl v26.4s, v26.4s, v10.4s\n" - "srshl v27.4s, v27.4s, v10.4s\n" - "srshl v28.4s, v28.4s, v10.4s\n" - "srshl v29.4s, v29.4s, v10.4s\n" - "srshl v30.4s, v30.4s, v10.4s\n" - "srshl v31.4s, v31.4s, v10.4s\n" - "add v16.4s, v16.4s, v14.4s\n" - "add v17.4s, v17.4s, v14.4s\n" - "add v18.4s, v18.4s, v14.4s\n" - "add v19.4s, v19.4s, v14.4s\n" - "add v20.4s, v20.4s, v14.4s\n" - "add v21.4s, v21.4s, v14.4s\n" - "add v22.4s, v22.4s, v14.4s\n" - "add v23.4s, v23.4s, v14.4s\n" - "add v24.4s, v24.4s, v14.4s\n" - "add v25.4s, v25.4s, v14.4s\n" - "add v26.4s, v26.4s, v14.4s\n" - "add v27.4s, v27.4s, v14.4s\n" - "add v28.4s, v28.4s, v14.4s\n" - "add v29.4s, v29.4s, v14.4s\n" - "add v30.4s, v30.4s, v14.4s\n" - "add v31.4s, v31.4s, v14.4s\n" - "smin v16.4s, v16.4s, v11.4s\n" - "smin v17.4s, v17.4s, v11.4s\n" - "smin v18.4s, v18.4s, v11.4s\n" - "smin v19.4s, v19.4s, v11.4s\n" - "smin v20.4s, v20.4s, v11.4s\n" - "smin v21.4s, v21.4s, v11.4s\n" - "smin v22.4s, v22.4s, v11.4s\n" - "smin v23.4s, v23.4s, v11.4s\n" - "smin v24.4s, v24.4s, v11.4s\n" - "smin v25.4s, v25.4s, v11.4s\n" - "smin v26.4s, v26.4s, v11.4s\n" - "smin v27.4s, v27.4s, v11.4s\n" - "smin v28.4s, v28.4s, v11.4s\n" - "smin v29.4s, v29.4s, v11.4s\n" - "smin v30.4s, v30.4s, v11.4s\n" - "smin v31.4s, v31.4s, v11.4s\n" - "smax v16.4s, v16.4s, v13.4s\n" - "smax v17.4s, v17.4s, v13.4s\n" - "smax v18.4s, v18.4s, v13.4s\n" - "smax v19.4s, v19.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smax v21.4s, v21.4s, v13.4s\n" - "smax v22.4s, v22.4s, v13.4s\n" - "smax v23.4s, v23.4s, v13.4s\n" - "smax v24.4s, v24.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v26.4s, v26.4s, v13.4s\n" - "smax v27.4s, v27.4s, v13.4s\n" - "smax v28.4s, v28.4s, v13.4s\n" - "smax v29.4s, v29.4s, v13.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" - "smax v31.4s, v31.4s, v13.4s\n" + "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v26.4s, v26.4s, v5.4s\n" + "sqadd v27.4s, v27.4s, v4.4s\n" + "sqadd v28.4s, v28.4s, v3.4s\n" + "sqadd v29.4s, v29.4s, v2.4s\n" + "sqadd v30.4s, v30.4s, v1.4s\n" + "sqadd v31.4s, v31.4s, v0.4s\n" + "srshl v16.4s, v16.4s, v8.4s\n" + "srshl v17.4s, v17.4s, v8.4s\n" + "srshl v18.4s, v18.4s, v8.4s\n" + "srshl v19.4s, v19.4s, v8.4s\n" + "srshl v20.4s, v20.4s, v8.4s\n" + "srshl v21.4s, v21.4s, v8.4s\n" + "srshl v22.4s, v22.4s, v8.4s\n" + "srshl v23.4s, v23.4s, v8.4s\n" + "srshl v24.4s, v24.4s, v8.4s\n" + "srshl v25.4s, v25.4s, v8.4s\n" + "srshl v26.4s, v26.4s, v8.4s\n" + "srshl v27.4s, v27.4s, v8.4s\n" + "srshl v28.4s, v28.4s, v8.4s\n" + "srshl v29.4s, v29.4s, v8.4s\n" + "srshl v30.4s, v30.4s, v8.4s\n" + "srshl v31.4s, v31.4s, v8.4s\n" + "add v16.4s, v16.4s, v11.4s\n" + "add v17.4s, v17.4s, v11.4s\n" + "add v18.4s, v18.4s, v11.4s\n" + "add v19.4s, v19.4s, v11.4s\n" + "add v20.4s, v20.4s, v11.4s\n" + "add v21.4s, v21.4s, v11.4s\n" + "add v22.4s, v22.4s, v11.4s\n" + "add v23.4s, v23.4s, v11.4s\n" + "add v24.4s, v24.4s, v11.4s\n" + "add v25.4s, v25.4s, v11.4s\n" + "add v26.4s, v26.4s, v11.4s\n" + "add v27.4s, v27.4s, v11.4s\n" + "add v28.4s, v28.4s, v11.4s\n" + "add v29.4s, v29.4s, v11.4s\n" + "add v30.4s, v30.4s, v11.4s\n" + "add v31.4s, v31.4s, v11.4s\n" + "smin v16.4s, v16.4s, v14.4s\n" + "smin v17.4s, v17.4s, v14.4s\n" + "smin v18.4s, v18.4s, v14.4s\n" + "smin v19.4s, v19.4s, v14.4s\n" + "smin v20.4s, v20.4s, v14.4s\n" + "smin v21.4s, v21.4s, v14.4s\n" + "smin v22.4s, v22.4s, v14.4s\n" + "smin v23.4s, v23.4s, v14.4s\n" + "smin v24.4s, v24.4s, v14.4s\n" + "smin v25.4s, v25.4s, v14.4s\n" + "smin v26.4s, v26.4s, v14.4s\n" + "smin v27.4s, v27.4s, v14.4s\n" + "smin v28.4s, v28.4s, v14.4s\n" + "smin v29.4s, v29.4s, v14.4s\n" + "smin v30.4s, v30.4s, v14.4s\n" + "smin v31.4s, v31.4s, v14.4s\n" + "smax v16.4s, v16.4s, v15.4s\n" + "smax v17.4s, v17.4s, v15.4s\n" + "smax v18.4s, v18.4s, v15.4s\n" + "smax v19.4s, v19.4s, v15.4s\n" + "smax v20.4s, v20.4s, v15.4s\n" + "smax v21.4s, v21.4s, v15.4s\n" + "smax v22.4s, v22.4s, v15.4s\n" + "smax v23.4s, v23.4s, v15.4s\n" + "smax v24.4s, v24.4s, v15.4s\n" + "smax v25.4s, v25.4s, v15.4s\n" + "smax v26.4s, v26.4s, v15.4s\n" + "smax v27.4s, v27.4s, v15.4s\n" + "smax v28.4s, v28.4s, v15.4s\n" + "smax v29.4s, v29.4s, v15.4s\n" + "smax v30.4s, v30.4s, v15.4s\n" + "smax v31.4s, v31.4s, v15.4s\n" "uzp1 v16.16b, v16.16b, v16.16b\n" "uzp1 v17.16b, v17.16b, v17.16b\n" "uzp1 v18.16b, v18.16b, v18.16b\n" @@ -1317,158 +1318,156 @@ void a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_imp "uzp1 v30.16b, v30.16b, v30.16b\n" "uzp1 v31.16b, v31.16b, v31.16b\n" "tbz %x[n_output_channels], #1, 24f\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "add x20, x20, x9\n" - "add x21, x21, x9\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "add x22, x22, x9\n" - "add x23, x23, x9\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "add x24, x24, x9\n" - "add x25, x25, x9\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "add x26, x26, x9\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" "add x27, x27, x9\n" - "st1 { v16.h }[0], [x20]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "add x20, x20, x9\n" - "st1 { v17.h }[0], [x21]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "add x21, x21, x9\n" - "st1 { v18.h }[0], [x22]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "add x22, x22, x9\n" - "st1 { v19.h }[0], [x23]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "add x23, x23, x9\n" - "st1 { v20.h }[0], [x24]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "add x24, x24, x9\n" - "st1 { v21.h }[0], [x25]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "add x25, x25, x9\n" - "st1 { v22.h }[0], [x26]\n" - "ldr x26, [%x[outptrs], #0x70]\n" "add x26, x26, x9\n" - "st1 { v23.h }[0], [x27]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "add x25, x25, x9\n" + "add x24, x24, x9\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "add x23, x23, x9\n" + "add x22, x22, x9\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "add x21, x21, x9\n" + "add x20, x20, x9\n" + "st1 { v16.h }[0], [x27]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "add x27, x27, x9\n" + "st1 { v17.h }[0], [x26]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "add x26, x26, x9\n" + "st1 { v18.h }[0], [x25]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "add x25, x25, x9\n" + "st1 { v19.h }[0], [x24]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "add x24, x24, x9\n" + "st1 { v20.h }[0], [x23]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "add x23, x23, x9\n" + "st1 { v21.h }[0], [x22]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "add x22, x22, x9\n" + "st1 { v22.h }[0], [x21]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "add x21, x21, x9\n" + "st1 { v23.h }[0], [x20]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "add x20, x20, x9\n" "add x9, x9, #0x2\n" - "st1 { v24.h }[0], [x20]\n" - "st1 { v25.h }[0], [x21]\n" - "st1 { v26.h }[0], [x22]\n" - "st1 { v27.h }[0], [x23]\n" - "st1 { v28.h }[0], [x24]\n" - "st1 { v29.h }[0], [x25]\n" - "st1 { v30.h }[0], [x26]\n" - "st1 { v31.h }[0], [x27]\n" + "st1 { v24.h }[0], [x27]\n" + "st1 { v25.h }[0], [x26]\n" + "st1 { v26.h }[0], [x25]\n" + "st1 { v27.h }[0], [x24]\n" + "st1 { v28.h }[0], [x23]\n" + "st1 { v29.h }[0], [x22]\n" + "st1 { v30.h }[0], [x21]\n" + "st1 { v31.h }[0], [x20]\n" "tbz %x[n_output_channels], #0, 25f\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "add x20, x20, x9\n" - "add x21, x21, x9\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "add x22, x22, x9\n" - "add x23, x23, x9\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "add x24, x24, x9\n" - "add x25, x25, x9\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "add x26, x26, x9\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" "add x27, x27, x9\n" - "st1 { v16.b }[2], [x20]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "add x20, x20, x9\n" - "st1 { v17.b }[2], [x21]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "add x21, x21, x9\n" - "st1 { v18.b }[2], [x22]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "add x22, x22, x9\n" - "st1 { v19.b }[2], [x23]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "add x23, x23, x9\n" - "st1 { v20.b }[2], [x24]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "add x24, x24, x9\n" - "st1 { v21.b }[2], [x25]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "add x25, x25, x9\n" - "st1 { v22.b }[2], [x26]\n" - "ldr x26, [%x[outptrs], #0x70]\n" "add x26, x26, x9\n" - "st1 { v23.b }[2], [x27]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "add x25, x25, x9\n" + "add x24, x24, x9\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "add x23, x23, x9\n" + "add x22, x22, x9\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "add x21, x21, x9\n" + "add x20, x20, x9\n" + "st1 { v16.b }[2], [x27]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "add x27, x27, x9\n" - "st1 { v24.b }[2], [x20]\n" - "st1 { v25.b }[2], [x21]\n" - "st1 { v26.b }[2], [x22]\n" - "st1 { v27.b }[2], [x23]\n" - "st1 { v28.b }[2], [x24]\n" - "st1 { v29.b }[2], [x25]\n" - "st1 { v30.b }[2], [x26]\n" - "st1 { v31.b }[2], [x27]\n" + "st1 { v17.b }[2], [x26]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "add x26, x26, x9\n" + "st1 { v18.b }[2], [x25]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "add x25, x25, x9\n" + "st1 { v19.b }[2], [x24]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "add x24, x24, x9\n" + "st1 { v20.b }[2], [x23]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "add x23, x23, x9\n" + "st1 { v21.b }[2], [x22]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "add x22, x22, x9\n" + "st1 { v22.b }[2], [x21]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "add x21, x21, x9\n" + "st1 { v23.b }[2], [x20]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "add x20, x20, x9\n" + "st1 { v24.b }[2], [x27]\n" + "st1 { v25.b }[2], [x26]\n" + "st1 { v26.b }[2], [x25]\n" + "st1 { v27.b }[2], [x24]\n" + "st1 { v28.b }[2], [x23]\n" + "st1 { v29.b }[2], [x22]\n" + "st1 { v30.b }[2], [x21]\n" + "st1 { v31.b }[2], [x20]\n" "b 25f\n" "24:" // Output channel oddments: Done: Store: Bit 1: Unset - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "add x20, x20, x9\n" - "add x21, x21, x9\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "add x22, x22, x9\n" - "add x23, x23, x9\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "add x24, x24, x9\n" - "add x25, x25, x9\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "add x26, x26, x9\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" "add x27, x27, x9\n" - "st1 { v16.b }[0], [x20]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "add x20, x20, x9\n" - "st1 { v17.b }[0], [x21]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "add x21, x21, x9\n" - "st1 { v18.b }[0], [x22]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "add x22, x22, x9\n" - "st1 { v19.b }[0], [x23]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "add x23, x23, x9\n" - "st1 { v20.b }[0], [x24]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "add x24, x24, x9\n" - "st1 { v21.b }[0], [x25]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "add x25, x25, x9\n" - "st1 { v22.b }[0], [x26]\n" - "ldr x26, [%x[outptrs], #0x70]\n" "add x26, x26, x9\n" - "st1 { v23.b }[0], [x27]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "add x25, x25, x9\n" + "add x24, x24, x9\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "add x23, x23, x9\n" + "add x22, x22, x9\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "add x21, x21, x9\n" + "add x20, x20, x9\n" + "st1 { v16.b }[0], [x27]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "add x27, x27, x9\n" - "st1 { v24.b }[0], [x20]\n" - "st1 { v25.b }[0], [x21]\n" - "st1 { v26.b }[0], [x22]\n" - "st1 { v27.b }[0], [x23]\n" - "st1 { v28.b }[0], [x24]\n" - "st1 { v29.b }[0], [x25]\n" - "st1 { v30.b }[0], [x26]\n" - "st1 { v31.b }[0], [x27]\n" + "st1 { v17.b }[0], [x26]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "add x26, x26, x9\n" + "st1 { v18.b }[0], [x25]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "add x25, x25, x9\n" + "st1 { v19.b }[0], [x24]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "add x24, x24, x9\n" + "st1 { v20.b }[0], [x23]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "add x23, x23, x9\n" + "st1 { v21.b }[0], [x22]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "add x22, x22, x9\n" + "st1 { v22.b }[0], [x21]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "add x21, x21, x9\n" + "st1 { v23.b }[0], [x20]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "add x20, x20, x9\n" + "st1 { v24.b }[0], [x27]\n" + "st1 { v25.b }[0], [x26]\n" + "st1 { v26.b }[0], [x25]\n" + "st1 { v27.b }[0], [x24]\n" + "st1 { v28.b }[0], [x23]\n" + "st1 { v29.b }[0], [x22]\n" + "st1 { v30.b }[0], [x21]\n" + "st1 { v31.b }[0], [x20]\n" "25:" // Output channel oddments: Done: Store: Bit 1: End - "26:" // Done - : [weights] "+&r" (weights) : [bias] "r" (bias), [inptrs] "r" (inptrs), [kernel_points] "r" ((uint64_t) kernel_points), [n_output_channels] "r" ((uint64_t) n_output_channels), [offsetof_Requantize32_a_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [offsetof_Requantize32_per_layer_left_shift] "I" (offsetof(arm_gemm::Requantize32, per_layer_left_shift)), [offsetof_Requantize32_per_layer_mul] "I" (offsetof(arm_gemm::Requantize32, per_layer_mul)), [offsetof_Requantize32_per_layer_right_shift] "I" (offsetof(arm_gemm::Requantize32, per_layer_right_shift)), [outptrs] "r" (outptrs), [qp] "r" (&qp), [rq_left_shift_ptr] "r" (per_channel_left_shifts), [rq_mul_ptr] "r" (per_channel_muls), [rq_right_shift_ptr] "r" (per_channel_right_shifts) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" @@ -1477,4 +1476,5 @@ void a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_imp } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp index 1d4580471417d0d8c7523aece1eb6aaab67b499e..bd2941b3d65380588ae310fc4d6ccb5c202ade04 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include @@ -34,14 +34,7 @@ namespace arm_conv { namespace depthwise { -void a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst_impl( - const unsigned int, - const int8_t *const *const, - const int8_t *, - const int32_t *, - const arm_gemm::Requantize32&, - const int32_t *, const int32_t *, - int8_t *const *const); +void a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst_impl(unsigned int, const int8_t *const *, const int8_t *, const int32_t *, const arm_gemm::Requantize32&, const int32_t *, const int32_t *, int8_t *const *); class a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst : public DepthwiseDepthfirstStrategy { @@ -71,7 +64,7 @@ class a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst : public DepthwiseDepthfirst ) const override { interleave_a64_s8q_3x3_dot::pack_parameters( - args.input_channels, buffer, reinterpret_cast(biases), + args.input_channels * args.channel_multiplier, buffer, reinterpret_cast(biases), reinterpret_cast(weights), qp, ld_weight_col, ld_weight_row ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp index 3fc1b13d9c10b8207c25e8a367b0754973a8b141..aad34c4c257f000f0c9e8706ca92785d35da2bbf 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp @@ -30,716 +30,708 @@ namespace arm_conv { namespace depthwise { -void a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst_impl( - const unsigned int n_channels, - const int8_t *const *const inptrs, - const int8_t *params, - const int32_t *, // Bias, should be wrapped into the parameters - const arm_gemm::Requantize32& qp, - const int32_t *, const int32_t *, // Requant parameters, also wrapped - int8_t *const *const outptrs -) +void a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst_impl(const unsigned int n_channels, const int8_t *const *const inptrs, const int8_t *params, const int32_t *, const arm_gemm::Requantize32& qp, const int32_t *, const int32_t *, int8_t *const *const outptrs) { __asm__ __volatile__( "lsr x15, %x[n_channels], #0x4\n" "add x20, %x[qp], %[offsetof_Requantize32_minval]\n" - "ld1r { v9.4s }, [x20]\n" + "ld1r { v8.4s }, [x20]\n" "ldp x14, x13, [%x[inptrs], #0x0]\n" "add x20, %x[qp], %[offsetof_Requantize32_maxval]\n" - "ld1r { v8.4s }, [x20]\n" + "ld1r { v12.4s }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_c_offset]\n" - "ld1r { v7.4s }, [x20]\n" + "ld1r { v15.4s }, [x20]\n" "mov x12, #0x0\n" "mov x11, #0x0\n" "ldp x10, x9, [%x[inptrs], #0x10]\n" "ldp x28, x27, [%x[inptrs], #0x20]\n" - "ldp x26, x25, [%x[inptrs], #0x30]\n" - "ldp x24, x23, [%x[outptrs], #0x0]\n" - "ldp x22, x21, [%x[outptrs], #0x10]\n" + "ldp x26, x21, [%x[inptrs], #0x30]\n" + "ldp x25, x24, [%x[outptrs], #0x0]\n" + "ldp x23, x22, [%x[outptrs], #0x10]\n" "cbz x15, 3f\n" - "ldr q6, [x14, x12]\n" - "ldr q5, [x13, x12]\n" + "ldr q11, [x14, x12]\n" + "ldr q20, [x13, x12]\n" "subs x15, x15, #0x1\n" - "ldr q4, [x10, x12]\n" - "ldr q3, [x9, x12]\n" - "zip2 v2.16b, v6.16b, v4.16b\n" - "zip1 v6.16b, v6.16b, v4.16b\n" - "ldr q1, [x28, x12]\n" - "ldr q0, [x27, x12]\n" - "zip1 v4.16b, v5.16b, v3.16b\n" - "zip2 v3.16b, v5.16b, v3.16b\n" - "ldr q31, [x26, x12]\n" - "ldr q30, [x25, x12]\n" - "zip2 v5.16b, v6.16b, v4.16b\n" - "zip1 v6.16b, v6.16b, v4.16b\n" - "ldr q29, [%x[params], #0x10]\n" - "ldr q28, [%x[params], #0x20]\n" - "zip1 v4.16b, v2.16b, v3.16b\n" - "zip2 v3.16b, v2.16b, v3.16b\n" - "ldr q2, [%x[params], #0x0]\n" - "ldr q27, [%x[params], #0x30]\n" - "zip2 v26.16b, v1.16b, v31.16b\n" - "zip1 v1.16b, v1.16b, v31.16b\n" - "ldp x14, x13, [%x[inptrs], #0x40]\n" - "ldr q25, [x14, x12]\n" - "zip1 v31.16b, v0.16b, v30.16b\n" - "zip2 v30.16b, v0.16b, v30.16b\n" - "ldr q24, [x13, x12]\n" - "ldp x10, x9, [%x[inptrs], #0x50]\n" - "zip2 v0.16b, v1.16b, v31.16b\n" - "zip1 v1.16b, v1.16b, v31.16b\n" - "ldr q23, [x10, x12]\n" - "ldr q22, [x9, x12]\n" - "zip2 v21.16b, v25.16b, v23.16b\n" - "zip1 v25.16b, v25.16b, v23.16b\n" - "ldp x28, x27, [%x[inptrs], #0x60]\n" - "ldr q20, [x28, x12]\n" - "zip1 v23.16b, v24.16b, v22.16b\n" - "zip2 v22.16b, v24.16b, v22.16b\n" - "ldr q19, [x27, x12]\n" - "ldp x26, x25, [%x[inptrs], #0x70]\n" - "zip1 v31.16b, v26.16b, v30.16b\n" - "zip2 v30.16b, v26.16b, v30.16b\n" - "ldr q18, [x26, x12]\n" - "ldr q17, [x25, x12]\n" - "zip2 v16.16b, v20.16b, v18.16b\n" - "zip1 v20.16b, v20.16b, v18.16b\n" - "zip1 v18.16b, v19.16b, v17.16b\n" - "zip2 v17.16b, v19.16b, v17.16b\n" + "ldr q16, [x10, x12]\n" + "ldr q14, [x9, x12]\n" + "zip2 v19.16b, v11.16b, v16.16b\n" + "zip1 v11.16b, v11.16b, v16.16b\n" + "ldr q13, [x28, x12]\n" + "ldr q18, [x27, x12]\n" + "zip1 v17.16b, v20.16b, v14.16b\n" + "zip2 v14.16b, v20.16b, v14.16b\n" + "ldr q16, [x26, x12]\n" + "ldr q27, [x21, x12]\n" + "zip2 v10.16b, v11.16b, v17.16b\n" + "zip1 v11.16b, v11.16b, v17.16b\n" + "ldr q24, [%x[params], #0x10]\n" + "ldr q9, [%x[params], #0x20]\n" + "zip1 v3.16b, v19.16b, v14.16b\n" + "zip2 v14.16b, v19.16b, v14.16b\n" + "ldr q31, [%x[params], #0x0]\n" + "ldr q6, [%x[params], #0x30]\n" + "zip2 v30.16b, v13.16b, v16.16b\n" + "zip1 v13.16b, v13.16b, v16.16b\n" + "ldp x21, x20, [%x[inptrs], #0x40]\n" + "ldr q5, [x21, x12]\n" + "zip1 v16.16b, v18.16b, v27.16b\n" + "zip2 v27.16b, v18.16b, v27.16b\n" + "ldr q17, [x20, x12]\n" + "ldp x21, x20, [%x[inptrs], #0x50]\n" + "zip2 v28.16b, v13.16b, v16.16b\n" + "zip1 v13.16b, v13.16b, v16.16b\n" + "ldr q16, [x21, x12]\n" + "ldr q7, [x20, x12]\n" + "zip2 v20.16b, v5.16b, v16.16b\n" + "zip1 v5.16b, v5.16b, v16.16b\n" + "ldp x21, x20, [%x[inptrs], #0x60]\n" + "ldr q16, [x21, x12]\n" + "zip1 v22.16b, v17.16b, v7.16b\n" + "zip2 v7.16b, v17.16b, v7.16b\n" + "ldr q19, [x20, x12]\n" + "ldp x21, x20, [%x[inptrs], #0x70]\n" + "zip1 v21.16b, v30.16b, v27.16b\n" + "zip2 v27.16b, v30.16b, v27.16b\n" + "ldr q30, [x21, x12]\n" + "ldr q1, [x20, x12]\n" + "zip2 v17.16b, v16.16b, v30.16b\n" + "zip1 v16.16b, v16.16b, v30.16b\n" + "zip1 v18.16b, v19.16b, v1.16b\n" + "zip2 v1.16b, v19.16b, v1.16b\n" "ldp x14, x13, [%x[inptrs], #0x0]\n" "ldp x10, x9, [%x[inptrs], #0x10]\n" "ldp x28, x27, [%x[inptrs], #0x20]\n" - "ldp x26, x25, [%x[inptrs], #0x30]\n" - "zip2 v24.16b, v25.16b, v23.16b\n" - "zip1 v25.16b, v25.16b, v23.16b\n" - "zip1 v23.16b, v21.16b, v22.16b\n" - "zip2 v22.16b, v21.16b, v22.16b\n" + "ldp x26, x21, [%x[inptrs], #0x30]\n" + "zip2 v29.16b, v5.16b, v22.16b\n" + "zip1 v5.16b, v5.16b, v22.16b\n" + "zip1 v0.16b, v20.16b, v7.16b\n" + "zip2 v7.16b, v20.16b, v7.16b\n" "add %x[params], %x[params], #0x40\n" - "zip2 v19.16b, v20.16b, v18.16b\n" - "zip1 v20.16b, v20.16b, v18.16b\n" - "zip1 v18.16b, v16.16b, v17.16b\n" - "zip2 v17.16b, v16.16b, v17.16b\n" - "mov v26.16b, v2.16b\n" - "mov v21.16b, v2.16b\n" - "mov v16.16b, v2.16b\n" + "zip2 v30.16b, v16.16b, v18.16b\n" + "zip1 v16.16b, v16.16b, v18.16b\n" + "zip1 v2.16b, v17.16b, v1.16b\n" + "zip2 v1.16b, v17.16b, v1.16b\n" + "mov v26.16b, v31.16b\n" + "mov v18.16b, v31.16b\n" + "mov v4.16b, v31.16b\n" "beq 2f\n" "1:" // Loop - ".inst 0x4e8697a2 // sdot v2.4s, v29.16b, v6.16b\n" - ".inst 0x4e8197b5 // sdot v21.4s, v29.16b, v1.16b\n" - "ext v6.16b, v6.16b, v6.16b, #0x1\n" + ".inst 0x4e8b971f // sdot v31.4s, v24.16b, v11.16b\n" + ".inst 0x4e8d9712 // sdot v18.4s, v24.16b, v13.16b\n" + "ext v11.16b, v11.16b, v11.16b, #0x1\n" "add x12, x12, #0x10\n" - ".inst 0x4e819782 // sdot v2.4s, v28.16b, v1.16b\n" - "ext v1.16b, v1.16b, v1.16b, #0x1\n" - ".inst 0x4e8697ba // sdot v26.4s, v29.16b, v6.16b\n" - "ldr q6, [%x[params], #0x0]\n" - ".inst 0x4e8197b0 // sdot v16.4s, v29.16b, v1.16b\n" - ".inst 0x4e999795 // sdot v21.4s, v28.16b, v25.16b\n" + ".inst 0x4e8d953f // sdot v31.4s, v9.16b, v13.16b\n" + "ext v13.16b, v13.16b, v13.16b, #0x1\n" + ".inst 0x4e8b971a // sdot v26.4s, v24.16b, v11.16b\n" + "ldr q17, [%x[params], #0x0]\n" + ".inst 0x4e8d9704 // sdot v4.4s, v24.16b, v13.16b\n" + ".inst 0x4e859532 // sdot v18.4s, v9.16b, v5.16b\n" "subs x15, x15, #0x1\n" - ".inst 0x4e999762 // sdot v2.4s, v27.16b, v25.16b\n" - "ext v25.16b, v25.16b, v25.16b, #0x1\n" - ".inst 0x4e81979a // sdot v26.4s, v28.16b, v1.16b\n" - "ldr q1, [%x[params], #0x10]\n" - ".inst 0x4e999790 // sdot v16.4s, v28.16b, v25.16b\n" - ".inst 0x4e949775 // sdot v21.4s, v27.16b, v20.16b\n" - "ext v20.16b, v20.16b, v20.16b, #0x1\n" - "sqrdmulh v2.4s, v2.4s, v6.4s\n" - ".inst 0x4e99977a // sdot v26.4s, v27.16b, v25.16b\n" - ".inst 0x4e949770 // sdot v16.4s, v27.16b, v20.16b\n" - "and v29.16b, v2.16b, v1.16b\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "sqrdmulh v26.4s, v26.4s, v6.4s\n" - "sqrdmulh v21.4s, v21.4s, v6.4s\n" - "sqrdmulh v16.4s, v16.4s, v6.4s\n" - "ldr q6, [%x[params], #0x60]\n" - "sqadd v2.4s, v2.4s, v29.4s\n" - "and v28.16b, v26.16b, v1.16b\n" - "and v27.16b, v21.16b, v1.16b\n" - "and v29.16b, v16.16b, v1.16b\n" - "sshr v28.4s, v28.4s, #0x1f\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "srshl v2.4s, v2.4s, v1.4s\n" - "sqadd v26.4s, v26.4s, v28.4s\n" - "ldr q28, [%x[params], #0x40]\n" - "sqadd v21.4s, v21.4s, v27.4s\n" - "ldr q27, [%x[params], #0x50]\n" - "sqadd v16.4s, v16.4s, v29.4s\n" - "ldr q29, [%x[params], #0x30]\n" - "add v2.4s, v2.4s, v7.4s\n" - "srshl v26.4s, v26.4s, v1.4s\n" - "srshl v21.4s, v21.4s, v1.4s\n" - "srshl v16.4s, v16.4s, v1.4s\n" - "ldr q1, [%x[params], #0x70]\n" - "smax v2.4s, v2.4s, v9.4s\n" - "add v26.4s, v26.4s, v7.4s\n" - "add v21.4s, v21.4s, v7.4s\n" - "add v16.4s, v16.4s, v7.4s\n" - "smin v2.4s, v2.4s, v8.4s\n" - "smax v26.4s, v26.4s, v9.4s\n" - "smax v21.4s, v21.4s, v9.4s\n" - "smax v16.4s, v16.4s, v9.4s\n" - "smin v26.4s, v26.4s, v8.4s\n" - "smin v21.4s, v21.4s, v8.4s\n" - "smin v16.4s, v16.4s, v8.4s\n" - "uzp1 v2.16b, v2.16b, v2.16b\n" - "uzp1 v2.16b, v2.16b, v2.16b\n" + ".inst 0x4e8594df // sdot v31.4s, v6.16b, v5.16b\n" + "ext v5.16b, v5.16b, v5.16b, #0x1\n" + ".inst 0x4e8d953a // sdot v26.4s, v9.16b, v13.16b\n" + "ldr q20, [%x[params], #0x10]\n" + ".inst 0x4e859524 // sdot v4.4s, v9.16b, v5.16b\n" + ".inst 0x4e9094d2 // sdot v18.4s, v6.16b, v16.16b\n" + "ext v16.16b, v16.16b, v16.16b, #0x1\n" + "sqrdmulh v31.4s, v31.4s, v17.4s\n" + ".inst 0x4e8594da // sdot v26.4s, v6.16b, v5.16b\n" + ".inst 0x4e9094c4 // sdot v4.4s, v6.16b, v16.16b\n" + "and v16.16b, v31.16b, v20.16b\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqrdmulh v26.4s, v26.4s, v17.4s\n" + "sqrdmulh v18.4s, v18.4s, v17.4s\n" + "sqrdmulh v4.4s, v4.4s, v17.4s\n" + "ldr q5, [%x[params], #0x60]\n" + "sqadd v31.4s, v31.4s, v16.4s\n" + "and v19.16b, v26.16b, v20.16b\n" + "and v17.16b, v18.16b, v20.16b\n" + "and v16.16b, v4.16b, v20.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "srshl v31.4s, v31.4s, v20.4s\n" + "sqadd v26.4s, v26.4s, v19.4s\n" + "ldr q13, [%x[params], #0x40]\n" + "sqadd v18.4s, v18.4s, v17.4s\n" + "ldr q17, [%x[params], #0x50]\n" + "sqadd v4.4s, v4.4s, v16.4s\n" + "ldr q16, [%x[params], #0x30]\n" + "add v31.4s, v31.4s, v15.4s\n" + "srshl v26.4s, v26.4s, v20.4s\n" + "srshl v18.4s, v18.4s, v20.4s\n" + "srshl v4.4s, v4.4s, v20.4s\n" + "ldr q22, [%x[params], #0x70]\n" + "smax v31.4s, v31.4s, v8.4s\n" + "add v26.4s, v26.4s, v15.4s\n" + "add v18.4s, v18.4s, v15.4s\n" + "add v4.4s, v4.4s, v15.4s\n" + "smin v31.4s, v31.4s, v12.4s\n" + "smax v26.4s, v26.4s, v8.4s\n" + "smax v18.4s, v18.4s, v8.4s\n" + "smax v4.4s, v4.4s, v8.4s\n" + "smin v26.4s, v26.4s, v12.4s\n" + "smin v18.4s, v18.4s, v12.4s\n" + "smin v4.4s, v4.4s, v12.4s\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s2, [x24, x11]\n" - "ldr q2, [%x[params], #0x20]\n" - "uzp1 v21.16b, v21.16b, v21.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" + "str s31, [x25, x11]\n" + "ldr q24, [%x[params], #0x20]\n" + "uzp1 v18.16b, v18.16b, v18.16b\n" + "uzp1 v4.16b, v4.16b, v4.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "uzp1 v21.16b, v21.16b, v21.16b\n" - "str s26, [x23, x11]\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "str s21, [x22, x11]\n" - "mov v26.16b, v2.16b\n" - "str s16, [x21, x11]\n" - "mov v21.16b, v2.16b\n" - "mov v16.16b, v2.16b\n" - ".inst 0x4e8597a2 // sdot v2.4s, v29.16b, v5.16b\n" - ".inst 0x4e8097b5 // sdot v21.4s, v29.16b, v0.16b\n" - ".inst 0x4e809782 // sdot v2.4s, v28.16b, v0.16b\n" - "ext v5.16b, v5.16b, v5.16b, #0x1\n" + "uzp1 v18.16b, v18.16b, v18.16b\n" + "str s26, [x24, x11]\n" + "uzp1 v4.16b, v4.16b, v4.16b\n" + "str s18, [x23, x11]\n" + "mov v26.16b, v24.16b\n" + "str s4, [x22, x11]\n" + "mov v25.16b, v24.16b\n" + "mov v23.16b, v24.16b\n" + ".inst 0x4e8a9618 // sdot v24.4s, v16.16b, v10.16b\n" + ".inst 0x4e9c9619 // sdot v25.4s, v16.16b, v28.16b\n" + ".inst 0x4e9c95b8 // sdot v24.4s, v13.16b, v28.16b\n" + "ext v10.16b, v10.16b, v10.16b, #0x1\n" "add x11, x11, #0x4\n" - "ext v0.16b, v0.16b, v0.16b, #0x1\n" - ".inst 0x4e8597ba // sdot v26.4s, v29.16b, v5.16b\n" - "ldr q5, [x13, x12]\n" - ".inst 0x4e8097b0 // sdot v16.4s, v29.16b, v0.16b\n" - ".inst 0x4e989795 // sdot v21.4s, v28.16b, v24.16b\n" - ".inst 0x4e989762 // sdot v2.4s, v27.16b, v24.16b\n" - "ext v24.16b, v24.16b, v24.16b, #0x1\n" - ".inst 0x4e80979a // sdot v26.4s, v28.16b, v0.16b\n" - "ldr q0, [x27, x12]\n" - ".inst 0x4e989790 // sdot v16.4s, v28.16b, v24.16b\n" - "sqrdmulh v2.4s, v2.4s, v6.4s\n" - ".inst 0x4e939775 // sdot v21.4s, v27.16b, v19.16b\n" - "ext v19.16b, v19.16b, v19.16b, #0x1\n" - ".inst 0x4e98977a // sdot v26.4s, v27.16b, v24.16b\n" - ".inst 0x4e939770 // sdot v16.4s, v27.16b, v19.16b\n" - "and v29.16b, v2.16b, v1.16b\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "sqrdmulh v26.4s, v26.4s, v6.4s\n" - "sqrdmulh v21.4s, v21.4s, v6.4s\n" - "sqrdmulh v16.4s, v16.4s, v6.4s\n" - "ldr q6, [%x[params], #0xc0]\n" - "sqadd v2.4s, v2.4s, v29.4s\n" - "and v28.16b, v26.16b, v1.16b\n" - "and v27.16b, v21.16b, v1.16b\n" - "and v29.16b, v16.16b, v1.16b\n" - "sshr v28.4s, v28.4s, #0x1f\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "srshl v2.4s, v2.4s, v1.4s\n" - "sqadd v26.4s, v26.4s, v28.4s\n" - "ldr q28, [%x[params], #0xa0]\n" - "sqadd v21.4s, v21.4s, v27.4s\n" - "ldr q27, [%x[params], #0xb0]\n" - "sqadd v16.4s, v16.4s, v29.4s\n" - "ldr q29, [%x[params], #0x90]\n" - "add v2.4s, v2.4s, v7.4s\n" - "srshl v26.4s, v26.4s, v1.4s\n" - "srshl v21.4s, v21.4s, v1.4s\n" - "srshl v16.4s, v16.4s, v1.4s\n" - "ldr q1, [%x[params], #0xd0]\n" - "smax v2.4s, v2.4s, v9.4s\n" - "add v26.4s, v26.4s, v7.4s\n" - "add v21.4s, v21.4s, v7.4s\n" - "add v16.4s, v16.4s, v7.4s\n" - "smin v2.4s, v2.4s, v8.4s\n" - "smax v26.4s, v26.4s, v9.4s\n" - "smax v21.4s, v21.4s, v9.4s\n" - "smax v16.4s, v16.4s, v9.4s\n" - "smin v26.4s, v26.4s, v8.4s\n" - "smin v21.4s, v21.4s, v8.4s\n" - "smin v16.4s, v16.4s, v8.4s\n" - "uzp1 v2.16b, v2.16b, v2.16b\n" - "uzp1 v2.16b, v2.16b, v2.16b\n" - "str s2, [x24, x11]\n" - "ldr q2, [%x[params], #0x80]\n" + "ext v28.16b, v28.16b, v28.16b, #0x1\n" + ".inst 0x4e8a961a // sdot v26.4s, v16.16b, v10.16b\n" + "ldr q10, [x13, x12]\n" + ".inst 0x4e9c9617 // sdot v23.4s, v16.16b, v28.16b\n" + ".inst 0x4e9d95b9 // sdot v25.4s, v13.16b, v29.16b\n" + ".inst 0x4e9d9638 // sdot v24.4s, v17.16b, v29.16b\n" + "ext v29.16b, v29.16b, v29.16b, #0x1\n" + ".inst 0x4e9c95ba // sdot v26.4s, v13.16b, v28.16b\n" + "ldr q20, [x27, x12]\n" + ".inst 0x4e9d95b7 // sdot v23.4s, v13.16b, v29.16b\n" + "sqrdmulh v24.4s, v24.4s, v5.4s\n" + ".inst 0x4e9e9639 // sdot v25.4s, v17.16b, v30.16b\n" + "ext v30.16b, v30.16b, v30.16b, #0x1\n" + ".inst 0x4e9d963a // sdot v26.4s, v17.16b, v29.16b\n" + ".inst 0x4e9e9637 // sdot v23.4s, v17.16b, v30.16b\n" + "and v16.16b, v24.16b, v22.16b\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqrdmulh v26.4s, v26.4s, v5.4s\n" + "sqrdmulh v25.4s, v25.4s, v5.4s\n" + "sqrdmulh v23.4s, v23.4s, v5.4s\n" + "ldr q19, [%x[params], #0xc0]\n" + "sqadd v24.4s, v24.4s, v16.4s\n" + "and v18.16b, v26.16b, v22.16b\n" + "and v17.16b, v25.16b, v22.16b\n" + "and v16.16b, v23.16b, v22.16b\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "srshl v24.4s, v24.4s, v22.4s\n" + "sqadd v26.4s, v26.4s, v18.4s\n" + "ldr q18, [%x[params], #0xa0]\n" + "sqadd v25.4s, v25.4s, v17.4s\n" + "ldr q17, [%x[params], #0xb0]\n" + "sqadd v23.4s, v23.4s, v16.4s\n" + "ldr q16, [%x[params], #0x90]\n" + "add v24.4s, v24.4s, v15.4s\n" + "srshl v26.4s, v26.4s, v22.4s\n" + "srshl v25.4s, v25.4s, v22.4s\n" + "srshl v23.4s, v23.4s, v22.4s\n" + "ldr q22, [%x[params], #0xd0]\n" + "smax v24.4s, v24.4s, v8.4s\n" + "add v26.4s, v26.4s, v15.4s\n" + "add v25.4s, v25.4s, v15.4s\n" + "add v23.4s, v23.4s, v15.4s\n" + "smin v24.4s, v24.4s, v12.4s\n" + "smax v26.4s, v26.4s, v8.4s\n" + "smax v25.4s, v25.4s, v8.4s\n" + "smax v23.4s, v23.4s, v8.4s\n" + "smin v26.4s, v26.4s, v12.4s\n" + "smin v25.4s, v25.4s, v12.4s\n" + "smin v23.4s, v23.4s, v12.4s\n" + "uzp1 v24.16b, v24.16b, v24.16b\n" + "uzp1 v24.16b, v24.16b, v24.16b\n" + "str s24, [x25, x11]\n" + "ldr q24, [%x[params], #0x80]\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "uzp1 v21.16b, v21.16b, v21.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" + "uzp1 v25.16b, v25.16b, v25.16b\n" + "uzp1 v23.16b, v23.16b, v23.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s26, [x23, x11]\n" - "uzp1 v21.16b, v21.16b, v21.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "str s21, [x22, x11]\n" - "str s16, [x21, x11]\n" - "mov v26.16b, v2.16b\n" - "mov v21.16b, v2.16b\n" - ".inst 0x4e9f97b5 // sdot v21.4s, v29.16b, v31.16b\n" - "mov v16.16b, v2.16b\n" - ".inst 0x4e8497a2 // sdot v2.4s, v29.16b, v4.16b\n" - ".inst 0x4e9f9782 // sdot v2.4s, v28.16b, v31.16b\n" + "str s26, [x24, x11]\n" + "uzp1 v25.16b, v25.16b, v25.16b\n" + "uzp1 v23.16b, v23.16b, v23.16b\n" + "str s25, [x23, x11]\n" + "str s23, [x22, x11]\n" + "mov v23.16b, v24.16b\n" + "mov v31.16b, v24.16b\n" + ".inst 0x4e95961f // sdot v31.4s, v16.16b, v21.16b\n" + "mov v13.16b, v24.16b\n" + ".inst 0x4e839618 // sdot v24.4s, v16.16b, v3.16b\n" + ".inst 0x4e959658 // sdot v24.4s, v18.16b, v21.16b\n" "add x11, x11, #0x4\n" - "ext v4.16b, v4.16b, v4.16b, #0x1\n" - "ext v31.16b, v31.16b, v31.16b, #0x1\n" - ".inst 0x4e8497ba // sdot v26.4s, v29.16b, v4.16b\n" - "ldr q4, [x10, x12]\n" - ".inst 0x4e9f97b0 // sdot v16.4s, v29.16b, v31.16b\n" - ".inst 0x4e979795 // sdot v21.4s, v28.16b, v23.16b\n" - ".inst 0x4e979762 // sdot v2.4s, v27.16b, v23.16b\n" - "ext v23.16b, v23.16b, v23.16b, #0x1\n" - ".inst 0x4e9f979a // sdot v26.4s, v28.16b, v31.16b\n" - "ldr q31, [x26, x12]\n" - ".inst 0x4e979790 // sdot v16.4s, v28.16b, v23.16b\n" - ".inst 0x4e929775 // sdot v21.4s, v27.16b, v18.16b\n" - "ext v18.16b, v18.16b, v18.16b, #0x1\n" - "sqrdmulh v2.4s, v2.4s, v6.4s\n" - ".inst 0x4e97977a // sdot v26.4s, v27.16b, v23.16b\n" - ".inst 0x4e929770 // sdot v16.4s, v27.16b, v18.16b\n" - "and v29.16b, v2.16b, v1.16b\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "sqrdmulh v26.4s, v26.4s, v6.4s\n" - "sqrdmulh v21.4s, v21.4s, v6.4s\n" - "sqrdmulh v16.4s, v16.4s, v6.4s\n" - "ldr q6, [%x[params], #0x120]\n" - "sqadd v2.4s, v2.4s, v29.4s\n" - "and v28.16b, v26.16b, v1.16b\n" - "and v27.16b, v21.16b, v1.16b\n" - "and v29.16b, v16.16b, v1.16b\n" - "sshr v28.4s, v28.4s, #0x1f\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "srshl v2.4s, v2.4s, v1.4s\n" - "sqadd v26.4s, v26.4s, v28.4s\n" - "ldr q28, [%x[params], #0x100]\n" - "sqadd v21.4s, v21.4s, v27.4s\n" - "ldr q27, [%x[params], #0x110]\n" - "sqadd v16.4s, v16.4s, v29.4s\n" - "ldr q29, [%x[params], #0xf0]\n" - "add v2.4s, v2.4s, v7.4s\n" - "srshl v26.4s, v26.4s, v1.4s\n" - "srshl v21.4s, v21.4s, v1.4s\n" - "srshl v16.4s, v16.4s, v1.4s\n" - "ldr q1, [%x[params], #0x130]\n" - "smax v2.4s, v2.4s, v9.4s\n" - "add v26.4s, v26.4s, v7.4s\n" - "add v21.4s, v21.4s, v7.4s\n" - "add v16.4s, v16.4s, v7.4s\n" - "smin v2.4s, v2.4s, v8.4s\n" - "smax v26.4s, v26.4s, v9.4s\n" - "smax v21.4s, v21.4s, v9.4s\n" - "smax v16.4s, v16.4s, v9.4s\n" - "smin v26.4s, v26.4s, v8.4s\n" - "smin v21.4s, v21.4s, v8.4s\n" - "smin v16.4s, v16.4s, v8.4s\n" - "uzp1 v2.16b, v2.16b, v2.16b\n" - "uzp1 v2.16b, v2.16b, v2.16b\n" - "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s2, [x24, x11]\n" + "ext v3.16b, v3.16b, v3.16b, #0x1\n" + "ext v21.16b, v21.16b, v21.16b, #0x1\n" + ".inst 0x4e839617 // sdot v23.4s, v16.16b, v3.16b\n" + "ldr q3, [x10, x12]\n" + ".inst 0x4e95960d // sdot v13.4s, v16.16b, v21.16b\n" + ".inst 0x4e80965f // sdot v31.4s, v18.16b, v0.16b\n" + ".inst 0x4e809638 // sdot v24.4s, v17.16b, v0.16b\n" + "ext v0.16b, v0.16b, v0.16b, #0x1\n" + ".inst 0x4e959657 // sdot v23.4s, v18.16b, v21.16b\n" + "ldr q4, [x26, x12]\n" + ".inst 0x4e80964d // sdot v13.4s, v18.16b, v0.16b\n" + ".inst 0x4e82963f // sdot v31.4s, v17.16b, v2.16b\n" + "ext v2.16b, v2.16b, v2.16b, #0x1\n" + "sqrdmulh v24.4s, v24.4s, v19.4s\n" + ".inst 0x4e809637 // sdot v23.4s, v17.16b, v0.16b\n" + ".inst 0x4e82962d // sdot v13.4s, v17.16b, v2.16b\n" + "and v16.16b, v24.16b, v22.16b\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqrdmulh v23.4s, v23.4s, v19.4s\n" + "sqrdmulh v31.4s, v31.4s, v19.4s\n" + "sqrdmulh v13.4s, v13.4s, v19.4s\n" + "ldr q19, [%x[params], #0x120]\n" + "sqadd v24.4s, v24.4s, v16.4s\n" + "and v18.16b, v23.16b, v22.16b\n" + "and v17.16b, v31.16b, v22.16b\n" + "and v16.16b, v13.16b, v22.16b\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "srshl v24.4s, v24.4s, v22.4s\n" + "sqadd v23.4s, v23.4s, v18.4s\n" + "ldr q18, [%x[params], #0x100]\n" + "sqadd v31.4s, v31.4s, v17.4s\n" + "ldr q17, [%x[params], #0x110]\n" + "sqadd v13.4s, v13.4s, v16.4s\n" + "ldr q16, [%x[params], #0xf0]\n" + "add v24.4s, v24.4s, v15.4s\n" + "srshl v23.4s, v23.4s, v22.4s\n" + "srshl v31.4s, v31.4s, v22.4s\n" + "srshl v13.4s, v13.4s, v22.4s\n" + "ldr q22, [%x[params], #0x130]\n" + "smax v24.4s, v24.4s, v8.4s\n" + "add v23.4s, v23.4s, v15.4s\n" + "add v31.4s, v31.4s, v15.4s\n" + "add v13.4s, v13.4s, v15.4s\n" + "smin v24.4s, v24.4s, v12.4s\n" + "smax v23.4s, v23.4s, v8.4s\n" + "smax v31.4s, v31.4s, v8.4s\n" + "smax v13.4s, v13.4s, v8.4s\n" + "smin v23.4s, v23.4s, v12.4s\n" + "smin v31.4s, v31.4s, v12.4s\n" + "smin v13.4s, v13.4s, v12.4s\n" + "uzp1 v24.16b, v24.16b, v24.16b\n" + "uzp1 v24.16b, v24.16b, v24.16b\n" + "uzp1 v23.16b, v23.16b, v23.16b\n" + "str s24, [x25, x11]\n" "ldr q2, [%x[params], #0xe0]\n" - "uzp1 v21.16b, v21.16b, v21.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "uzp1 v26.16b, v26.16b, v26.16b\n" - "uzp1 v21.16b, v21.16b, v21.16b\n" - "str s26, [x23, x11]\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "str s21, [x22, x11]\n" - "mov v26.16b, v2.16b\n" - "str s16, [x21, x11]\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" + "uzp1 v13.16b, v13.16b, v13.16b\n" + "uzp1 v23.16b, v23.16b, v23.16b\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" + "str s23, [x24, x11]\n" + "uzp1 v13.16b, v13.16b, v13.16b\n" + "str s31, [x23, x11]\n" + "mov v25.16b, v2.16b\n" + "str s13, [x22, x11]\n" "mov v21.16b, v2.16b\n" - "mov v16.16b, v2.16b\n" - ".inst 0x4e8397a2 // sdot v2.4s, v29.16b, v3.16b\n" - ".inst 0x4e9e97b5 // sdot v21.4s, v29.16b, v30.16b\n" - ".inst 0x4e9e9782 // sdot v2.4s, v28.16b, v30.16b\n" - "ext v3.16b, v3.16b, v3.16b, #0x1\n" + "mov v30.16b, v2.16b\n" + ".inst 0x4e8e9602 // sdot v2.4s, v16.16b, v14.16b\n" + ".inst 0x4e9b9615 // sdot v21.4s, v16.16b, v27.16b\n" + ".inst 0x4e9b9642 // sdot v2.4s, v18.16b, v27.16b\n" + "ext v14.16b, v14.16b, v14.16b, #0x1\n" "add x11, x11, #0x4\n" - "ext v30.16b, v30.16b, v30.16b, #0x1\n" - ".inst 0x4e8397ba // sdot v26.4s, v29.16b, v3.16b\n" - "ldr q3, [x9, x12]\n" - ".inst 0x4e9e97b0 // sdot v16.4s, v29.16b, v30.16b\n" - ".inst 0x4e969795 // sdot v21.4s, v28.16b, v22.16b\n" - ".inst 0x4e969762 // sdot v2.4s, v27.16b, v22.16b\n" - "ext v22.16b, v22.16b, v22.16b, #0x1\n" - ".inst 0x4e9e979a // sdot v26.4s, v28.16b, v30.16b\n" - "ldr q30, [x25, x12]\n" - ".inst 0x4e969790 // sdot v16.4s, v28.16b, v22.16b\n" - "sqrdmulh v2.4s, v2.4s, v6.4s\n" - ".inst 0x4e919775 // sdot v21.4s, v27.16b, v17.16b\n" - "ext v17.16b, v17.16b, v17.16b, #0x1\n" - ".inst 0x4e96977a // sdot v26.4s, v27.16b, v22.16b\n" - ".inst 0x4e919770 // sdot v16.4s, v27.16b, v17.16b\n" - "and v29.16b, v2.16b, v1.16b\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "sqrdmulh v26.4s, v26.4s, v6.4s\n" - "sqrdmulh v21.4s, v21.4s, v6.4s\n" - "sqrdmulh v16.4s, v16.4s, v6.4s\n" - "ldr q6, [x14, x12]\n" - "ldp x14, x13, [%x[inptrs], #0x40]\n" - "ldr q25, [x14, x12]\n" - "ldr q24, [x13, x12]\n" - "sqadd v2.4s, v2.4s, v29.4s\n" - "and v28.16b, v26.16b, v1.16b\n" - "and v27.16b, v21.16b, v1.16b\n" - "and v29.16b, v16.16b, v1.16b\n" - "ldp x10, x9, [%x[inptrs], #0x50]\n" - "ldr q23, [x10, x12]\n" - "ldr q22, [x9, x12]\n" - "sshr v28.4s, v28.4s, #0x1f\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "srshl v2.4s, v2.4s, v1.4s\n" - "sqadd v26.4s, v26.4s, v28.4s\n" - "ldr q28, [%x[params], #0x160]\n" - "sqadd v21.4s, v21.4s, v27.4s\n" - "ldr q27, [%x[params], #0x170]\n" - "sqadd v16.4s, v16.4s, v29.4s\n" - "ldr q29, [%x[params], #0x150]\n" - "add v2.4s, v2.4s, v7.4s\n" - "srshl v26.4s, v26.4s, v1.4s\n" - "srshl v21.4s, v21.4s, v1.4s\n" - "srshl v16.4s, v16.4s, v1.4s\n" - "ldr q1, [x28, x12]\n" - "smax v2.4s, v2.4s, v9.4s\n" - "ldp x28, x27, [%x[inptrs], #0x60]\n" - "ldr q20, [x28, x12]\n" - "ldr q19, [x27, x12]\n" - "add v26.4s, v26.4s, v7.4s\n" - "add v21.4s, v21.4s, v7.4s\n" - "add v16.4s, v16.4s, v7.4s\n" - "smin v2.4s, v2.4s, v8.4s\n" - "ldp x26, x25, [%x[inptrs], #0x70]\n" - "ldr q18, [x26, x12]\n" - "ldr q17, [x25, x12]\n" - "smax v26.4s, v26.4s, v9.4s\n" - "smax v21.4s, v21.4s, v9.4s\n" + "ext v27.16b, v27.16b, v27.16b, #0x1\n" + ".inst 0x4e8e9619 // sdot v25.4s, v16.16b, v14.16b\n" + "ldr q14, [x9, x12]\n" + ".inst 0x4e9b961e // sdot v30.4s, v16.16b, v27.16b\n" + ".inst 0x4e879655 // sdot v21.4s, v18.16b, v7.16b\n" + ".inst 0x4e879622 // sdot v2.4s, v17.16b, v7.16b\n" + "ext v7.16b, v7.16b, v7.16b, #0x1\n" + ".inst 0x4e9b9659 // sdot v25.4s, v18.16b, v27.16b\n" + "ldr q27, [x21, x12]\n" + ".inst 0x4e87965e // sdot v30.4s, v18.16b, v7.16b\n" + "sqrdmulh v2.4s, v2.4s, v19.4s\n" + ".inst 0x4e819635 // sdot v21.4s, v17.16b, v1.16b\n" + "ext v1.16b, v1.16b, v1.16b, #0x1\n" + ".inst 0x4e879639 // sdot v25.4s, v17.16b, v7.16b\n" + ".inst 0x4e81963e // sdot v30.4s, v17.16b, v1.16b\n" + "and v16.16b, v2.16b, v22.16b\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqrdmulh v25.4s, v25.4s, v19.4s\n" + "sqrdmulh v21.4s, v21.4s, v19.4s\n" + "sqrdmulh v30.4s, v30.4s, v19.4s\n" + "ldr q11, [x14, x12]\n" + "ldp x21, x20, [%x[inptrs], #0x40]\n" + "ldr q5, [x21, x12]\n" + "ldr q29, [x20, x12]\n" + "sqadd v2.4s, v2.4s, v16.4s\n" + "and v19.16b, v25.16b, v22.16b\n" + "and v17.16b, v21.16b, v22.16b\n" + "and v16.16b, v30.16b, v22.16b\n" + "ldp x21, x20, [%x[inptrs], #0x50]\n" + "ldr q26, [x21, x12]\n" + "ldr q7, [x20, x12]\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "srshl v2.4s, v2.4s, v22.4s\n" + "sqadd v25.4s, v25.4s, v19.4s\n" + "ldr q9, [%x[params], #0x160]\n" + "sqadd v21.4s, v21.4s, v17.4s\n" + "ldr q6, [%x[params], #0x170]\n" + "sqadd v30.4s, v30.4s, v16.4s\n" + "ldr q24, [%x[params], #0x150]\n" + "add v2.4s, v2.4s, v15.4s\n" + "srshl v25.4s, v25.4s, v22.4s\n" + "srshl v21.4s, v21.4s, v22.4s\n" + "srshl v30.4s, v30.4s, v22.4s\n" + "ldr q13, [x28, x12]\n" + "smax v2.4s, v2.4s, v8.4s\n" + "ldp x21, x20, [%x[inptrs], #0x60]\n" + "ldr q16, [x21, x12]\n" + "ldr q28, [x20, x12]\n" + "add v25.4s, v25.4s, v15.4s\n" + "add v21.4s, v21.4s, v15.4s\n" + "add v30.4s, v30.4s, v15.4s\n" + "smin v2.4s, v2.4s, v12.4s\n" + "ldp x21, x20, [%x[inptrs], #0x70]\n" + "ldr q23, [x21, x12]\n" + "ldr q1, [x20, x12]\n" + "smax v25.4s, v25.4s, v8.4s\n" + "smax v21.4s, v21.4s, v8.4s\n" "ldp x14, x13, [%x[inptrs], #0x0]\n" - "smax v16.4s, v16.4s, v9.4s\n" - "smin v26.4s, v26.4s, v8.4s\n" + "smax v30.4s, v30.4s, v8.4s\n" + "smin v25.4s, v25.4s, v12.4s\n" "ldp x10, x9, [%x[inptrs], #0x10]\n" "ldp x28, x27, [%x[inptrs], #0x20]\n" - "smin v21.4s, v21.4s, v8.4s\n" - "smin v16.4s, v16.4s, v8.4s\n" - "ldp x26, x25, [%x[inptrs], #0x30]\n" + "smin v21.4s, v21.4s, v12.4s\n" + "smin v30.4s, v30.4s, v12.4s\n" + "ldp x26, x21, [%x[inptrs], #0x30]\n" "uzp1 v2.16b, v2.16b, v2.16b\n" "uzp1 v2.16b, v2.16b, v2.16b\n" - "str s2, [x24, x11]\n" - "uzp1 v26.16b, v26.16b, v26.16b\n" + "str s2, [x25, x11]\n" + "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v21.16b, v21.16b, v21.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "zip2 v2.16b, v6.16b, v4.16b\n" - "zip1 v6.16b, v6.16b, v4.16b\n" - "zip1 v4.16b, v5.16b, v3.16b\n" - "zip2 v3.16b, v5.16b, v3.16b\n" - "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s26, [x23, x11]\n" + "uzp1 v30.16b, v30.16b, v30.16b\n" + "zip2 v18.16b, v11.16b, v3.16b\n" + "zip1 v11.16b, v11.16b, v3.16b\n" + "zip1 v17.16b, v10.16b, v14.16b\n" + "zip2 v14.16b, v10.16b, v14.16b\n" + "uzp1 v25.16b, v25.16b, v25.16b\n" + "str s25, [x24, x11]\n" "uzp1 v21.16b, v21.16b, v21.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "str s21, [x22, x11]\n" - "str s16, [x21, x11]\n" - "zip2 v5.16b, v6.16b, v4.16b\n" - "zip1 v6.16b, v6.16b, v4.16b\n" + "uzp1 v30.16b, v30.16b, v30.16b\n" + "str s21, [x23, x11]\n" + "str s30, [x22, x11]\n" + "zip2 v10.16b, v11.16b, v17.16b\n" + "zip1 v11.16b, v11.16b, v17.16b\n" "add x11, x11, #0x4\n" - "zip1 v4.16b, v2.16b, v3.16b\n" - "zip2 v3.16b, v2.16b, v3.16b\n" - "ldr q2, [%x[params], #0x140]\n" + "zip1 v3.16b, v18.16b, v14.16b\n" + "zip2 v14.16b, v18.16b, v14.16b\n" + "ldr q31, [%x[params], #0x140]\n" "add %x[params], %x[params], #0x180\n" - "zip2 v26.16b, v1.16b, v31.16b\n" - "zip1 v1.16b, v1.16b, v31.16b\n" - "zip1 v31.16b, v0.16b, v30.16b\n" - "zip2 v30.16b, v0.16b, v30.16b\n" - "zip2 v21.16b, v25.16b, v23.16b\n" - "zip1 v25.16b, v25.16b, v23.16b\n" - "zip1 v23.16b, v24.16b, v22.16b\n" - "zip2 v22.16b, v24.16b, v22.16b\n" - "zip2 v16.16b, v20.16b, v18.16b\n" - "zip1 v20.16b, v20.16b, v18.16b\n" - "zip1 v18.16b, v19.16b, v17.16b\n" - "zip2 v17.16b, v19.16b, v17.16b\n" - "zip2 v0.16b, v1.16b, v31.16b\n" - "zip1 v1.16b, v1.16b, v31.16b\n" - "zip1 v31.16b, v26.16b, v30.16b\n" - "zip2 v30.16b, v26.16b, v30.16b\n" - "zip2 v24.16b, v25.16b, v23.16b\n" - "zip1 v25.16b, v25.16b, v23.16b\n" - "zip1 v23.16b, v21.16b, v22.16b\n" - "zip2 v22.16b, v21.16b, v22.16b\n" - "zip2 v19.16b, v20.16b, v18.16b\n" - "zip1 v20.16b, v20.16b, v18.16b\n" - "zip1 v18.16b, v16.16b, v17.16b\n" - "zip2 v17.16b, v16.16b, v17.16b\n" - "mov v26.16b, v2.16b\n" - "mov v21.16b, v2.16b\n" - "mov v16.16b, v2.16b\n" + "zip2 v22.16b, v13.16b, v4.16b\n" + "zip1 v13.16b, v13.16b, v4.16b\n" + "zip1 v2.16b, v20.16b, v27.16b\n" + "zip2 v27.16b, v20.16b, v27.16b\n" + "zip2 v19.16b, v5.16b, v26.16b\n" + "zip1 v5.16b, v5.16b, v26.16b\n" + "zip1 v18.16b, v29.16b, v7.16b\n" + "zip2 v7.16b, v29.16b, v7.16b\n" + "zip2 v4.16b, v16.16b, v23.16b\n" + "zip1 v16.16b, v16.16b, v23.16b\n" + "zip1 v17.16b, v28.16b, v1.16b\n" + "zip2 v1.16b, v28.16b, v1.16b\n" + "zip2 v28.16b, v13.16b, v2.16b\n" + "zip1 v13.16b, v13.16b, v2.16b\n" + "zip1 v21.16b, v22.16b, v27.16b\n" + "zip2 v27.16b, v22.16b, v27.16b\n" + "zip2 v29.16b, v5.16b, v18.16b\n" + "zip1 v5.16b, v5.16b, v18.16b\n" + "zip1 v0.16b, v19.16b, v7.16b\n" + "zip2 v7.16b, v19.16b, v7.16b\n" + "zip2 v30.16b, v16.16b, v17.16b\n" + "zip1 v16.16b, v16.16b, v17.16b\n" + "zip1 v2.16b, v4.16b, v1.16b\n" + "zip2 v1.16b, v4.16b, v1.16b\n" + "mov v26.16b, v31.16b\n" + "mov v18.16b, v31.16b\n" + "mov v4.16b, v31.16b\n" "bgt 1b\n" "2:" // Detached iteration - ".inst 0x4e8697a2 // sdot v2.4s, v29.16b, v6.16b\n" - ".inst 0x4e8197b5 // sdot v21.4s, v29.16b, v1.16b\n" - "ext v6.16b, v6.16b, v6.16b, #0x1\n" + ".inst 0x4e8b971f // sdot v31.4s, v24.16b, v11.16b\n" + ".inst 0x4e8d9712 // sdot v18.4s, v24.16b, v13.16b\n" + "ext v11.16b, v11.16b, v11.16b, #0x1\n" "tst %x[n_channels], #0xf\n" - ".inst 0x4e819782 // sdot v2.4s, v28.16b, v1.16b\n" - "ext v1.16b, v1.16b, v1.16b, #0x1\n" - ".inst 0x4e8697ba // sdot v26.4s, v29.16b, v6.16b\n" - "ldr q6, [%x[params], #0x0]\n" - ".inst 0x4e8197b0 // sdot v16.4s, v29.16b, v1.16b\n" - ".inst 0x4e999795 // sdot v21.4s, v28.16b, v25.16b\n" + ".inst 0x4e8d953f // sdot v31.4s, v9.16b, v13.16b\n" + "ext v13.16b, v13.16b, v13.16b, #0x1\n" + ".inst 0x4e8b971a // sdot v26.4s, v24.16b, v11.16b\n" + "ldr q17, [%x[params], #0x0]\n" + ".inst 0x4e8d9704 // sdot v4.4s, v24.16b, v13.16b\n" + ".inst 0x4e859532 // sdot v18.4s, v9.16b, v5.16b\n" "add x12, x12, #0x10\n" - ".inst 0x4e999762 // sdot v2.4s, v27.16b, v25.16b\n" - "ext v25.16b, v25.16b, v25.16b, #0x1\n" - ".inst 0x4e81979a // sdot v26.4s, v28.16b, v1.16b\n" - "ldr q1, [%x[params], #0x10]\n" - ".inst 0x4e999790 // sdot v16.4s, v28.16b, v25.16b\n" - ".inst 0x4e949775 // sdot v21.4s, v27.16b, v20.16b\n" - "ext v20.16b, v20.16b, v20.16b, #0x1\n" - "sqrdmulh v2.4s, v2.4s, v6.4s\n" - ".inst 0x4e99977a // sdot v26.4s, v27.16b, v25.16b\n" - ".inst 0x4e949770 // sdot v16.4s, v27.16b, v20.16b\n" - "and v29.16b, v2.16b, v1.16b\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "sqrdmulh v26.4s, v26.4s, v6.4s\n" - "sqrdmulh v21.4s, v21.4s, v6.4s\n" - "sqrdmulh v16.4s, v16.4s, v6.4s\n" - "ldr q6, [%x[params], #0x60]\n" - "sqadd v2.4s, v2.4s, v29.4s\n" - "and v28.16b, v26.16b, v1.16b\n" - "and v27.16b, v21.16b, v1.16b\n" - "and v29.16b, v16.16b, v1.16b\n" - "sshr v28.4s, v28.4s, #0x1f\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "srshl v2.4s, v2.4s, v1.4s\n" - "sqadd v26.4s, v26.4s, v28.4s\n" - "ldr q28, [%x[params], #0x40]\n" - "sqadd v21.4s, v21.4s, v27.4s\n" - "ldr q27, [%x[params], #0x50]\n" - "sqadd v16.4s, v16.4s, v29.4s\n" - "ldr q29, [%x[params], #0x30]\n" - "add v2.4s, v2.4s, v7.4s\n" - "srshl v26.4s, v26.4s, v1.4s\n" - "srshl v21.4s, v21.4s, v1.4s\n" - "srshl v16.4s, v16.4s, v1.4s\n" - "ldr q1, [%x[params], #0x70]\n" - "smax v2.4s, v2.4s, v9.4s\n" - "add v26.4s, v26.4s, v7.4s\n" - "add v21.4s, v21.4s, v7.4s\n" - "add v16.4s, v16.4s, v7.4s\n" - "smin v2.4s, v2.4s, v8.4s\n" - "smax v26.4s, v26.4s, v9.4s\n" - "smax v21.4s, v21.4s, v9.4s\n" - "smax v16.4s, v16.4s, v9.4s\n" - "smin v26.4s, v26.4s, v8.4s\n" - "smin v21.4s, v21.4s, v8.4s\n" - "smin v16.4s, v16.4s, v8.4s\n" - "uzp1 v2.16b, v2.16b, v2.16b\n" - "uzp1 v2.16b, v2.16b, v2.16b\n" - "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s2, [x24, x11]\n" - "ldr q2, [%x[params], #0x20]\n" - "uzp1 v21.16b, v21.16b, v21.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "uzp1 v26.16b, v26.16b, v26.16b\n" - "uzp1 v21.16b, v21.16b, v21.16b\n" - "str s26, [x23, x11]\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "str s21, [x22, x11]\n" - "mov v26.16b, v2.16b\n" - "str s16, [x21, x11]\n" - "mov v21.16b, v2.16b\n" - "mov v16.16b, v2.16b\n" - ".inst 0x4e8597a2 // sdot v2.4s, v29.16b, v5.16b\n" - ".inst 0x4e8097b5 // sdot v21.4s, v29.16b, v0.16b\n" - ".inst 0x4e809782 // sdot v2.4s, v28.16b, v0.16b\n" + ".inst 0x4e8594df // sdot v31.4s, v6.16b, v5.16b\n" "ext v5.16b, v5.16b, v5.16b, #0x1\n" - "add x11, x11, #0x4\n" - "ext v0.16b, v0.16b, v0.16b, #0x1\n" - ".inst 0x4e8597ba // sdot v26.4s, v29.16b, v5.16b\n" - ".inst 0x4e8097b0 // sdot v16.4s, v29.16b, v0.16b\n" - ".inst 0x4e989795 // sdot v21.4s, v28.16b, v24.16b\n" - ".inst 0x4e989762 // sdot v2.4s, v27.16b, v24.16b\n" - "ext v24.16b, v24.16b, v24.16b, #0x1\n" - ".inst 0x4e80979a // sdot v26.4s, v28.16b, v0.16b\n" - ".inst 0x4e989790 // sdot v16.4s, v28.16b, v24.16b\n" - "sqrdmulh v2.4s, v2.4s, v6.4s\n" - ".inst 0x4e939775 // sdot v21.4s, v27.16b, v19.16b\n" - "ext v19.16b, v19.16b, v19.16b, #0x1\n" - ".inst 0x4e98977a // sdot v26.4s, v27.16b, v24.16b\n" - ".inst 0x4e939770 // sdot v16.4s, v27.16b, v19.16b\n" - "and v29.16b, v2.16b, v1.16b\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "sqrdmulh v26.4s, v26.4s, v6.4s\n" - "sqrdmulh v21.4s, v21.4s, v6.4s\n" - "sqrdmulh v16.4s, v16.4s, v6.4s\n" - "ldr q6, [%x[params], #0xc0]\n" - "sqadd v2.4s, v2.4s, v29.4s\n" - "and v28.16b, v26.16b, v1.16b\n" - "and v27.16b, v21.16b, v1.16b\n" - "and v29.16b, v16.16b, v1.16b\n" - "sshr v28.4s, v28.4s, #0x1f\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "srshl v2.4s, v2.4s, v1.4s\n" - "sqadd v26.4s, v26.4s, v28.4s\n" - "ldr q28, [%x[params], #0xa0]\n" - "sqadd v21.4s, v21.4s, v27.4s\n" - "ldr q27, [%x[params], #0xb0]\n" - "sqadd v16.4s, v16.4s, v29.4s\n" - "ldr q29, [%x[params], #0x90]\n" - "add v2.4s, v2.4s, v7.4s\n" - "srshl v26.4s, v26.4s, v1.4s\n" - "srshl v21.4s, v21.4s, v1.4s\n" - "srshl v16.4s, v16.4s, v1.4s\n" - "ldr q1, [%x[params], #0xd0]\n" - "smax v2.4s, v2.4s, v9.4s\n" - "add v26.4s, v26.4s, v7.4s\n" - "add v21.4s, v21.4s, v7.4s\n" - "add v16.4s, v16.4s, v7.4s\n" - "smin v2.4s, v2.4s, v8.4s\n" - "smax v26.4s, v26.4s, v9.4s\n" - "smax v21.4s, v21.4s, v9.4s\n" - "smax v16.4s, v16.4s, v9.4s\n" - "smin v26.4s, v26.4s, v8.4s\n" - "smin v21.4s, v21.4s, v8.4s\n" - "smin v16.4s, v16.4s, v8.4s\n" - "uzp1 v2.16b, v2.16b, v2.16b\n" - "uzp1 v2.16b, v2.16b, v2.16b\n" - "str s2, [x24, x11]\n" - "ldr q2, [%x[params], #0x80]\n" + ".inst 0x4e8d953a // sdot v26.4s, v9.16b, v13.16b\n" + "ldr q19, [%x[params], #0x10]\n" + ".inst 0x4e859524 // sdot v4.4s, v9.16b, v5.16b\n" + ".inst 0x4e9094d2 // sdot v18.4s, v6.16b, v16.16b\n" + "ext v16.16b, v16.16b, v16.16b, #0x1\n" + "sqrdmulh v31.4s, v31.4s, v17.4s\n" + ".inst 0x4e8594da // sdot v26.4s, v6.16b, v5.16b\n" + ".inst 0x4e9094c4 // sdot v4.4s, v6.16b, v16.16b\n" + "and v16.16b, v31.16b, v19.16b\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqrdmulh v26.4s, v26.4s, v17.4s\n" + "sqrdmulh v18.4s, v18.4s, v17.4s\n" + "sqrdmulh v4.4s, v4.4s, v17.4s\n" + "ldr q24, [%x[params], #0x60]\n" + "sqadd v31.4s, v31.4s, v16.4s\n" + "and v20.16b, v26.16b, v19.16b\n" + "and v17.16b, v18.16b, v19.16b\n" + "and v16.16b, v4.16b, v19.16b\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "srshl v31.4s, v31.4s, v19.4s\n" + "sqadd v26.4s, v26.4s, v20.4s\n" + "ldr q5, [%x[params], #0x40]\n" + "sqadd v18.4s, v18.4s, v17.4s\n" + "ldr q17, [%x[params], #0x50]\n" + "sqadd v4.4s, v4.4s, v16.4s\n" + "ldr q16, [%x[params], #0x30]\n" + "add v31.4s, v31.4s, v15.4s\n" + "srshl v26.4s, v26.4s, v19.4s\n" + "srshl v18.4s, v18.4s, v19.4s\n" + "srshl v4.4s, v4.4s, v19.4s\n" + "ldr q23, [%x[params], #0x70]\n" + "smax v31.4s, v31.4s, v8.4s\n" + "add v26.4s, v26.4s, v15.4s\n" + "add v18.4s, v18.4s, v15.4s\n" + "add v4.4s, v4.4s, v15.4s\n" + "smin v31.4s, v31.4s, v12.4s\n" + "smax v26.4s, v26.4s, v8.4s\n" + "smax v18.4s, v18.4s, v8.4s\n" + "smax v4.4s, v4.4s, v8.4s\n" + "smin v26.4s, v26.4s, v12.4s\n" + "smin v18.4s, v18.4s, v12.4s\n" + "smin v4.4s, v4.4s, v12.4s\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "uzp1 v21.16b, v21.16b, v21.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" + "str s31, [x25, x11]\n" + "ldr q25, [%x[params], #0x20]\n" + "uzp1 v18.16b, v18.16b, v18.16b\n" + "uzp1 v4.16b, v4.16b, v4.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s26, [x23, x11]\n" - "uzp1 v21.16b, v21.16b, v21.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "str s21, [x22, x11]\n" - "str s16, [x21, x11]\n" - "mov v26.16b, v2.16b\n" - "mov v21.16b, v2.16b\n" - ".inst 0x4e9f97b5 // sdot v21.4s, v29.16b, v31.16b\n" - "mov v16.16b, v2.16b\n" - ".inst 0x4e8497a2 // sdot v2.4s, v29.16b, v4.16b\n" - ".inst 0x4e9f9782 // sdot v2.4s, v28.16b, v31.16b\n" + "uzp1 v18.16b, v18.16b, v18.16b\n" + "str s26, [x24, x11]\n" + "uzp1 v4.16b, v4.16b, v4.16b\n" + "str s18, [x23, x11]\n" + "mov v22.16b, v25.16b\n" + "str s4, [x22, x11]\n" + "mov v20.16b, v25.16b\n" + "mov v19.16b, v25.16b\n" + ".inst 0x4e8a9619 // sdot v25.4s, v16.16b, v10.16b\n" + ".inst 0x4e9c9614 // sdot v20.4s, v16.16b, v28.16b\n" + ".inst 0x4e9c94b9 // sdot v25.4s, v5.16b, v28.16b\n" + "ext v10.16b, v10.16b, v10.16b, #0x1\n" + "add x11, x11, #0x4\n" + "ext v28.16b, v28.16b, v28.16b, #0x1\n" + ".inst 0x4e8a9616 // sdot v22.4s, v16.16b, v10.16b\n" + ".inst 0x4e9c9613 // sdot v19.4s, v16.16b, v28.16b\n" + ".inst 0x4e9d94b4 // sdot v20.4s, v5.16b, v29.16b\n" + ".inst 0x4e9d9639 // sdot v25.4s, v17.16b, v29.16b\n" + "ext v29.16b, v29.16b, v29.16b, #0x1\n" + ".inst 0x4e9c94b6 // sdot v22.4s, v5.16b, v28.16b\n" + ".inst 0x4e9d94b3 // sdot v19.4s, v5.16b, v29.16b\n" + "sqrdmulh v25.4s, v25.4s, v24.4s\n" + ".inst 0x4e9e9634 // sdot v20.4s, v17.16b, v30.16b\n" + "ext v30.16b, v30.16b, v30.16b, #0x1\n" + ".inst 0x4e9d9636 // sdot v22.4s, v17.16b, v29.16b\n" + ".inst 0x4e9e9633 // sdot v19.4s, v17.16b, v30.16b\n" + "and v16.16b, v25.16b, v23.16b\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqrdmulh v22.4s, v22.4s, v24.4s\n" + "sqrdmulh v20.4s, v20.4s, v24.4s\n" + "sqrdmulh v19.4s, v19.4s, v24.4s\n" + "ldr q24, [%x[params], #0xc0]\n" + "sqadd v25.4s, v25.4s, v16.4s\n" + "and v18.16b, v22.16b, v23.16b\n" + "and v17.16b, v20.16b, v23.16b\n" + "and v16.16b, v19.16b, v23.16b\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "srshl v25.4s, v25.4s, v23.4s\n" + "sqadd v22.4s, v22.4s, v18.4s\n" + "ldr q18, [%x[params], #0xa0]\n" + "sqadd v20.4s, v20.4s, v17.4s\n" + "ldr q17, [%x[params], #0xb0]\n" + "sqadd v19.4s, v19.4s, v16.4s\n" + "ldr q16, [%x[params], #0x90]\n" + "add v25.4s, v25.4s, v15.4s\n" + "srshl v22.4s, v22.4s, v23.4s\n" + "srshl v20.4s, v20.4s, v23.4s\n" + "srshl v19.4s, v19.4s, v23.4s\n" + "ldr q23, [%x[params], #0xd0]\n" + "smax v25.4s, v25.4s, v8.4s\n" + "add v22.4s, v22.4s, v15.4s\n" + "add v20.4s, v20.4s, v15.4s\n" + "add v19.4s, v19.4s, v15.4s\n" + "smin v25.4s, v25.4s, v12.4s\n" + "smax v22.4s, v22.4s, v8.4s\n" + "smax v20.4s, v20.4s, v8.4s\n" + "smax v19.4s, v19.4s, v8.4s\n" + "smin v22.4s, v22.4s, v12.4s\n" + "smin v20.4s, v20.4s, v12.4s\n" + "smin v19.4s, v19.4s, v12.4s\n" + "uzp1 v25.16b, v25.16b, v25.16b\n" + "uzp1 v25.16b, v25.16b, v25.16b\n" + "str s25, [x25, x11]\n" + "ldr q10, [%x[params], #0x80]\n" + "uzp1 v22.16b, v22.16b, v22.16b\n" + "uzp1 v20.16b, v20.16b, v20.16b\n" + "uzp1 v19.16b, v19.16b, v19.16b\n" + "uzp1 v22.16b, v22.16b, v22.16b\n" + "str s22, [x24, x11]\n" + "uzp1 v20.16b, v20.16b, v20.16b\n" + "uzp1 v19.16b, v19.16b, v19.16b\n" + "str s20, [x23, x11]\n" + "str s19, [x22, x11]\n" + "mov v28.16b, v10.16b\n" + "mov v20.16b, v10.16b\n" + ".inst 0x4e959614 // sdot v20.4s, v16.16b, v21.16b\n" + "mov v19.16b, v10.16b\n" + ".inst 0x4e83960a // sdot v10.4s, v16.16b, v3.16b\n" + ".inst 0x4e95964a // sdot v10.4s, v18.16b, v21.16b\n" "add x11, x11, #0x4\n" - "ext v4.16b, v4.16b, v4.16b, #0x1\n" - "ext v31.16b, v31.16b, v31.16b, #0x1\n" - ".inst 0x4e8497ba // sdot v26.4s, v29.16b, v4.16b\n" - ".inst 0x4e9f97b0 // sdot v16.4s, v29.16b, v31.16b\n" - ".inst 0x4e979795 // sdot v21.4s, v28.16b, v23.16b\n" - ".inst 0x4e979762 // sdot v2.4s, v27.16b, v23.16b\n" - "ext v23.16b, v23.16b, v23.16b, #0x1\n" - ".inst 0x4e9f979a // sdot v26.4s, v28.16b, v31.16b\n" - ".inst 0x4e979790 // sdot v16.4s, v28.16b, v23.16b\n" - ".inst 0x4e929775 // sdot v21.4s, v27.16b, v18.16b\n" - "ext v18.16b, v18.16b, v18.16b, #0x1\n" - "sqrdmulh v2.4s, v2.4s, v6.4s\n" - ".inst 0x4e97977a // sdot v26.4s, v27.16b, v23.16b\n" - ".inst 0x4e929770 // sdot v16.4s, v27.16b, v18.16b\n" - "and v29.16b, v2.16b, v1.16b\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "sqrdmulh v26.4s, v26.4s, v6.4s\n" - "sqrdmulh v21.4s, v21.4s, v6.4s\n" - "sqrdmulh v16.4s, v16.4s, v6.4s\n" - "ldr q6, [%x[params], #0x120]\n" - "sqadd v2.4s, v2.4s, v29.4s\n" - "and v28.16b, v26.16b, v1.16b\n" - "and v27.16b, v21.16b, v1.16b\n" - "and v29.16b, v16.16b, v1.16b\n" - "sshr v28.4s, v28.4s, #0x1f\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "srshl v2.4s, v2.4s, v1.4s\n" - "sqadd v26.4s, v26.4s, v28.4s\n" - "ldr q28, [%x[params], #0x100]\n" - "sqadd v21.4s, v21.4s, v27.4s\n" - "ldr q27, [%x[params], #0x110]\n" - "sqadd v16.4s, v16.4s, v29.4s\n" - "ldr q29, [%x[params], #0xf0]\n" - "add v2.4s, v2.4s, v7.4s\n" - "srshl v26.4s, v26.4s, v1.4s\n" - "srshl v21.4s, v21.4s, v1.4s\n" - "srshl v16.4s, v16.4s, v1.4s\n" - "ldr q1, [%x[params], #0x130]\n" - "smax v2.4s, v2.4s, v9.4s\n" - "add v26.4s, v26.4s, v7.4s\n" - "add v21.4s, v21.4s, v7.4s\n" - "add v16.4s, v16.4s, v7.4s\n" - "smin v2.4s, v2.4s, v8.4s\n" - "smax v26.4s, v26.4s, v9.4s\n" - "smax v21.4s, v21.4s, v9.4s\n" - "smax v16.4s, v16.4s, v9.4s\n" - "smin v26.4s, v26.4s, v8.4s\n" - "smin v21.4s, v21.4s, v8.4s\n" - "smin v16.4s, v16.4s, v8.4s\n" - "uzp1 v2.16b, v2.16b, v2.16b\n" - "uzp1 v2.16b, v2.16b, v2.16b\n" - "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s2, [x24, x11]\n" - "ldr q2, [%x[params], #0xe0]\n" - "uzp1 v21.16b, v21.16b, v21.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "add %x[params], %x[params], #0x140\n" - "uzp1 v26.16b, v26.16b, v26.16b\n" - "uzp1 v21.16b, v21.16b, v21.16b\n" - "str s26, [x23, x11]\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "str s21, [x22, x11]\n" - "mov v26.16b, v2.16b\n" - "str s16, [x21, x11]\n" - "mov v21.16b, v2.16b\n" - "mov v16.16b, v2.16b\n" - ".inst 0x4e8397a2 // sdot v2.4s, v29.16b, v3.16b\n" - ".inst 0x4e9e97b5 // sdot v21.4s, v29.16b, v30.16b\n" - ".inst 0x4e9e9782 // sdot v2.4s, v28.16b, v30.16b\n" "ext v3.16b, v3.16b, v3.16b, #0x1\n" + "ext v21.16b, v21.16b, v21.16b, #0x1\n" + ".inst 0x4e83961c // sdot v28.4s, v16.16b, v3.16b\n" + ".inst 0x4e959613 // sdot v19.4s, v16.16b, v21.16b\n" + ".inst 0x4e809654 // sdot v20.4s, v18.16b, v0.16b\n" + ".inst 0x4e80962a // sdot v10.4s, v17.16b, v0.16b\n" + "ext v0.16b, v0.16b, v0.16b, #0x1\n" + ".inst 0x4e95965c // sdot v28.4s, v18.16b, v21.16b\n" + ".inst 0x4e809653 // sdot v19.4s, v18.16b, v0.16b\n" + ".inst 0x4e829634 // sdot v20.4s, v17.16b, v2.16b\n" + "ext v2.16b, v2.16b, v2.16b, #0x1\n" + "sqrdmulh v10.4s, v10.4s, v24.4s\n" + ".inst 0x4e80963c // sdot v28.4s, v17.16b, v0.16b\n" + ".inst 0x4e829633 // sdot v19.4s, v17.16b, v2.16b\n" + "and v16.16b, v10.16b, v23.16b\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqrdmulh v28.4s, v28.4s, v24.4s\n" + "sqrdmulh v20.4s, v20.4s, v24.4s\n" + "sqrdmulh v19.4s, v19.4s, v24.4s\n" + "ldr q24, [%x[params], #0x120]\n" + "sqadd v10.4s, v10.4s, v16.4s\n" + "and v18.16b, v28.16b, v23.16b\n" + "and v17.16b, v20.16b, v23.16b\n" + "and v16.16b, v19.16b, v23.16b\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "srshl v10.4s, v10.4s, v23.4s\n" + "sqadd v28.4s, v28.4s, v18.4s\n" + "ldr q18, [%x[params], #0x100]\n" + "sqadd v20.4s, v20.4s, v17.4s\n" + "ldr q17, [%x[params], #0x110]\n" + "sqadd v19.4s, v19.4s, v16.4s\n" + "ldr q16, [%x[params], #0xf0]\n" + "add v10.4s, v10.4s, v15.4s\n" + "srshl v28.4s, v28.4s, v23.4s\n" + "srshl v20.4s, v20.4s, v23.4s\n" + "srshl v19.4s, v19.4s, v23.4s\n" + "ldr q23, [%x[params], #0x130]\n" + "smax v10.4s, v10.4s, v8.4s\n" + "add v28.4s, v28.4s, v15.4s\n" + "add v20.4s, v20.4s, v15.4s\n" + "add v19.4s, v19.4s, v15.4s\n" + "smin v10.4s, v10.4s, v12.4s\n" + "smax v28.4s, v28.4s, v8.4s\n" + "smax v20.4s, v20.4s, v8.4s\n" + "smax v19.4s, v19.4s, v8.4s\n" + "smin v28.4s, v28.4s, v12.4s\n" + "smin v20.4s, v20.4s, v12.4s\n" + "smin v19.4s, v19.4s, v12.4s\n" + "uzp1 v10.16b, v10.16b, v10.16b\n" + "uzp1 v10.16b, v10.16b, v10.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + "str s10, [x25, x11]\n" + "ldr q22, [%x[params], #0xe0]\n" + "uzp1 v20.16b, v20.16b, v20.16b\n" + "uzp1 v19.16b, v19.16b, v19.16b\n" + "add %x[params], %x[params], #0x140\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + "uzp1 v20.16b, v20.16b, v20.16b\n" + "str s28, [x24, x11]\n" + "uzp1 v19.16b, v19.16b, v19.16b\n" + "str s20, [x23, x11]\n" + "mov v21.16b, v22.16b\n" + "str s19, [x22, x11]\n" + "mov v20.16b, v22.16b\n" + "mov v19.16b, v22.16b\n" + ".inst 0x4e8e9616 // sdot v22.4s, v16.16b, v14.16b\n" + ".inst 0x4e9b9614 // sdot v20.4s, v16.16b, v27.16b\n" + ".inst 0x4e9b9656 // sdot v22.4s, v18.16b, v27.16b\n" + "ext v14.16b, v14.16b, v14.16b, #0x1\n" "add x11, x11, #0x4\n" - "ext v30.16b, v30.16b, v30.16b, #0x1\n" - ".inst 0x4e8397ba // sdot v26.4s, v29.16b, v3.16b\n" - ".inst 0x4e9e97b0 // sdot v16.4s, v29.16b, v30.16b\n" - ".inst 0x4e969795 // sdot v21.4s, v28.16b, v22.16b\n" - ".inst 0x4e969762 // sdot v2.4s, v27.16b, v22.16b\n" - "ext v22.16b, v22.16b, v22.16b, #0x1\n" - ".inst 0x4e9e979a // sdot v26.4s, v28.16b, v30.16b\n" - ".inst 0x4e969790 // sdot v16.4s, v28.16b, v22.16b\n" - "sqrdmulh v2.4s, v2.4s, v6.4s\n" - ".inst 0x4e919775 // sdot v21.4s, v27.16b, v17.16b\n" - "ext v17.16b, v17.16b, v17.16b, #0x1\n" - ".inst 0x4e96977a // sdot v26.4s, v27.16b, v22.16b\n" - ".inst 0x4e919770 // sdot v16.4s, v27.16b, v17.16b\n" - "and v29.16b, v2.16b, v1.16b\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "sqrdmulh v26.4s, v26.4s, v6.4s\n" - "sqrdmulh v21.4s, v21.4s, v6.4s\n" - "sqrdmulh v16.4s, v16.4s, v6.4s\n" - "sqadd v2.4s, v2.4s, v29.4s\n" - "and v28.16b, v26.16b, v1.16b\n" - "and v27.16b, v21.16b, v1.16b\n" - "and v29.16b, v16.16b, v1.16b\n" - "sshr v28.4s, v28.4s, #0x1f\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "sqadd v26.4s, v26.4s, v28.4s\n" - "sqadd v21.4s, v21.4s, v27.4s\n" - "sqadd v16.4s, v16.4s, v29.4s\n" - "srshl v2.4s, v2.4s, v1.4s\n" - "srshl v26.4s, v26.4s, v1.4s\n" - "srshl v21.4s, v21.4s, v1.4s\n" - "srshl v16.4s, v16.4s, v1.4s\n" - "add v2.4s, v2.4s, v7.4s\n" - "add v26.4s, v26.4s, v7.4s\n" - "add v21.4s, v21.4s, v7.4s\n" - "add v16.4s, v16.4s, v7.4s\n" - "smax v2.4s, v2.4s, v9.4s\n" - "smax v26.4s, v26.4s, v9.4s\n" - "smax v21.4s, v21.4s, v9.4s\n" - "smax v16.4s, v16.4s, v9.4s\n" - "smin v2.4s, v2.4s, v8.4s\n" - "smin v26.4s, v26.4s, v8.4s\n" - "smin v21.4s, v21.4s, v8.4s\n" - "smin v16.4s, v16.4s, v8.4s\n" - "uzp1 v2.16b, v2.16b, v2.16b\n" - "uzp1 v26.16b, v26.16b, v26.16b\n" + "ext v27.16b, v27.16b, v27.16b, #0x1\n" + ".inst 0x4e8e9615 // sdot v21.4s, v16.16b, v14.16b\n" + ".inst 0x4e9b9613 // sdot v19.4s, v16.16b, v27.16b\n" + ".inst 0x4e879654 // sdot v20.4s, v18.16b, v7.16b\n" + ".inst 0x4e879636 // sdot v22.4s, v17.16b, v7.16b\n" + "ext v7.16b, v7.16b, v7.16b, #0x1\n" + ".inst 0x4e9b9655 // sdot v21.4s, v18.16b, v27.16b\n" + ".inst 0x4e879653 // sdot v19.4s, v18.16b, v7.16b\n" + "sqrdmulh v22.4s, v22.4s, v24.4s\n" + ".inst 0x4e819634 // sdot v20.4s, v17.16b, v1.16b\n" + "ext v1.16b, v1.16b, v1.16b, #0x1\n" + ".inst 0x4e879635 // sdot v21.4s, v17.16b, v7.16b\n" + ".inst 0x4e819633 // sdot v19.4s, v17.16b, v1.16b\n" + "and v16.16b, v22.16b, v23.16b\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqrdmulh v21.4s, v21.4s, v24.4s\n" + "sqrdmulh v20.4s, v20.4s, v24.4s\n" + "sqrdmulh v19.4s, v19.4s, v24.4s\n" + "sqadd v22.4s, v22.4s, v16.4s\n" + "and v18.16b, v21.16b, v23.16b\n" + "and v17.16b, v20.16b, v23.16b\n" + "and v16.16b, v19.16b, v23.16b\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v21.4s, v21.4s, v18.4s\n" + "sqadd v20.4s, v20.4s, v17.4s\n" + "sqadd v19.4s, v19.4s, v16.4s\n" + "srshl v22.4s, v22.4s, v23.4s\n" + "srshl v21.4s, v21.4s, v23.4s\n" + "srshl v20.4s, v20.4s, v23.4s\n" + "srshl v19.4s, v19.4s, v23.4s\n" + "add v22.4s, v22.4s, v15.4s\n" + "add v21.4s, v21.4s, v15.4s\n" + "add v20.4s, v20.4s, v15.4s\n" + "add v19.4s, v19.4s, v15.4s\n" + "smax v22.4s, v22.4s, v8.4s\n" + "smax v21.4s, v21.4s, v8.4s\n" + "smax v20.4s, v20.4s, v8.4s\n" + "smax v19.4s, v19.4s, v8.4s\n" + "smin v22.4s, v22.4s, v12.4s\n" + "smin v21.4s, v21.4s, v12.4s\n" + "smin v20.4s, v20.4s, v12.4s\n" + "smin v19.4s, v19.4s, v12.4s\n" + "uzp1 v22.16b, v22.16b, v22.16b\n" "uzp1 v21.16b, v21.16b, v21.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "uzp1 v2.16b, v2.16b, v2.16b\n" - "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s2, [x24, x11]\n" + "uzp1 v20.16b, v20.16b, v20.16b\n" + "uzp1 v19.16b, v19.16b, v19.16b\n" + "uzp1 v22.16b, v22.16b, v22.16b\n" "uzp1 v21.16b, v21.16b, v21.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "str s26, [x23, x11]\n" - "str s21, [x22, x11]\n" - "str s16, [x21, x11]\n" + "str s22, [x25, x11]\n" + "uzp1 v20.16b, v20.16b, v20.16b\n" + "uzp1 v19.16b, v19.16b, v19.16b\n" + "str s21, [x24, x11]\n" + "str s20, [x23, x11]\n" + "str s19, [x22, x11]\n" "add x11, x11, #0x4\n" "beq 35f\n" "3:" // Oddments @@ -751,740 +743,738 @@ void a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst_impl( "add x28, x28, x12\n" "add x27, x27, x12\n" "add x26, x26, x12\n" - "add x25, x25, x12\n" + "add x21, x21, x12\n" "tbz %x[n_channels], #3, 7f\n" - "ldr d6, [x14], #0x8\n" - "ldr d5, [x13], #0x8\n" - "ldr d4, [x10], #0x8\n" - "ldr d3, [x9], #0x8\n" - "ldr d1, [x28], #0x8\n" - "ldr d0, [x27], #0x8\n" - "ldr d31, [x26], #0x8\n" - "ldr d30, [x25], #0x8\n" + "ldr d11, [x14], #0x8\n" + "ldr d10, [x13], #0x8\n" + "ldr d3, [x10], #0x8\n" + "ldr d14, [x9], #0x8\n" + "ldr d13, [x28], #0x8\n" + "ldr d28, [x27], #0x8\n" + "ldr d21, [x26], #0x8\n" + "ldr d27, [x21], #0x8\n" "tbz %x[n_channels], #2, 5f\n" - "ld1 { v6.s }[2], [x14], #0x4\n" - "ld1 { v5.s }[2], [x13], #0x4\n" - "ld1 { v4.s }[2], [x10], #0x4\n" - "ld1 { v3.s }[2], [x9], #0x4\n" - "ld1 { v1.s }[2], [x28], #0x4\n" - "ld1 { v0.s }[2], [x27], #0x4\n" - "ld1 { v31.s }[2], [x26], #0x4\n" - "ld1 { v30.s }[2], [x25], #0x4\n" + "ld1 { v11.s }[2], [x14], #0x4\n" + "ld1 { v10.s }[2], [x13], #0x4\n" + "ld1 { v3.s }[2], [x10], #0x4\n" + "ld1 { v14.s }[2], [x9], #0x4\n" + "ld1 { v13.s }[2], [x28], #0x4\n" + "ld1 { v28.s }[2], [x27], #0x4\n" + "ld1 { v21.s }[2], [x26], #0x4\n" + "ld1 { v27.s }[2], [x21], #0x4\n" "tbz %x[n_channels], #1, 4f\n" - "ld1 { v6.h }[6], [x14], #0x2\n" - "ld1 { v5.h }[6], [x13], #0x2\n" - "ld1 { v4.h }[6], [x10], #0x2\n" - "ld1 { v3.h }[6], [x9], #0x2\n" - "ld1 { v1.h }[6], [x28], #0x2\n" - "ld1 { v0.h }[6], [x27], #0x2\n" - "ld1 { v31.h }[6], [x26], #0x2\n" - "ld1 { v30.h }[6], [x25], #0x2\n" + "ld1 { v11.h }[6], [x14], #0x2\n" + "ld1 { v10.h }[6], [x13], #0x2\n" + "ld1 { v3.h }[6], [x10], #0x2\n" + "ld1 { v14.h }[6], [x9], #0x2\n" + "ld1 { v13.h }[6], [x28], #0x2\n" + "ld1 { v28.h }[6], [x27], #0x2\n" + "ld1 { v21.h }[6], [x26], #0x2\n" + "ld1 { v27.h }[6], [x21], #0x2\n" "tbz %x[n_channels], #0, 11f\n" - "ld1 { v6.b }[14], [x14], #0x1\n" - "ld1 { v5.b }[14], [x13], #0x1\n" - "ld1 { v4.b }[14], [x10], #0x1\n" - "ld1 { v3.b }[14], [x9], #0x1\n" - "ld1 { v1.b }[14], [x28], #0x1\n" - "ld1 { v0.b }[14], [x27], #0x1\n" - "ld1 { v31.b }[14], [x26], #0x1\n" - "ld1 { v30.b }[14], [x25], #0x1\n" + "ld1 { v11.b }[14], [x14], #0x1\n" + "ld1 { v10.b }[14], [x13], #0x1\n" + "ld1 { v3.b }[14], [x10], #0x1\n" + "ld1 { v14.b }[14], [x9], #0x1\n" + "ld1 { v13.b }[14], [x28], #0x1\n" + "ld1 { v28.b }[14], [x27], #0x1\n" + "ld1 { v21.b }[14], [x26], #0x1\n" + "ld1 { v27.b }[14], [x21], #0x1\n" "b 11f\n" "4:" // Oddments: Load (A): Bit 3: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 11f\n" - "ld1 { v6.b }[12], [x14], #0x1\n" - "ld1 { v5.b }[12], [x13], #0x1\n" - "ld1 { v4.b }[12], [x10], #0x1\n" - "ld1 { v3.b }[12], [x9], #0x1\n" - "ld1 { v1.b }[12], [x28], #0x1\n" - "ld1 { v0.b }[12], [x27], #0x1\n" - "ld1 { v31.b }[12], [x26], #0x1\n" - "ld1 { v30.b }[12], [x25], #0x1\n" + "ld1 { v11.b }[12], [x14], #0x1\n" + "ld1 { v10.b }[12], [x13], #0x1\n" + "ld1 { v3.b }[12], [x10], #0x1\n" + "ld1 { v14.b }[12], [x9], #0x1\n" + "ld1 { v13.b }[12], [x28], #0x1\n" + "ld1 { v28.b }[12], [x27], #0x1\n" + "ld1 { v21.b }[12], [x26], #0x1\n" + "ld1 { v27.b }[12], [x21], #0x1\n" "b 11f\n" "5:" // Oddments: Load (A): Bit 3: Bit 2: Unset "tbz %x[n_channels], #1, 6f\n" - "ld1 { v6.h }[4], [x14], #0x2\n" - "ld1 { v5.h }[4], [x13], #0x2\n" - "ld1 { v4.h }[4], [x10], #0x2\n" - "ld1 { v3.h }[4], [x9], #0x2\n" - "ld1 { v1.h }[4], [x28], #0x2\n" - "ld1 { v0.h }[4], [x27], #0x2\n" - "ld1 { v31.h }[4], [x26], #0x2\n" - "ld1 { v30.h }[4], [x25], #0x2\n" + "ld1 { v11.h }[4], [x14], #0x2\n" + "ld1 { v10.h }[4], [x13], #0x2\n" + "ld1 { v3.h }[4], [x10], #0x2\n" + "ld1 { v14.h }[4], [x9], #0x2\n" + "ld1 { v13.h }[4], [x28], #0x2\n" + "ld1 { v28.h }[4], [x27], #0x2\n" + "ld1 { v21.h }[4], [x26], #0x2\n" + "ld1 { v27.h }[4], [x21], #0x2\n" "tbz %x[n_channels], #0, 11f\n" - "ld1 { v6.b }[10], [x14], #0x1\n" - "ld1 { v5.b }[10], [x13], #0x1\n" - "ld1 { v4.b }[10], [x10], #0x1\n" - "ld1 { v3.b }[10], [x9], #0x1\n" - "ld1 { v1.b }[10], [x28], #0x1\n" - "ld1 { v0.b }[10], [x27], #0x1\n" - "ld1 { v31.b }[10], [x26], #0x1\n" - "ld1 { v30.b }[10], [x25], #0x1\n" + "ld1 { v11.b }[10], [x14], #0x1\n" + "ld1 { v10.b }[10], [x13], #0x1\n" + "ld1 { v3.b }[10], [x10], #0x1\n" + "ld1 { v14.b }[10], [x9], #0x1\n" + "ld1 { v13.b }[10], [x28], #0x1\n" + "ld1 { v28.b }[10], [x27], #0x1\n" + "ld1 { v21.b }[10], [x26], #0x1\n" + "ld1 { v27.b }[10], [x21], #0x1\n" "b 11f\n" "6:" // Oddments: Load (A): Bit 3: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 11f\n" - "ld1 { v6.b }[8], [x14], #0x1\n" - "ld1 { v5.b }[8], [x13], #0x1\n" - "ld1 { v4.b }[8], [x10], #0x1\n" - "ld1 { v3.b }[8], [x9], #0x1\n" - "ld1 { v1.b }[8], [x28], #0x1\n" - "ld1 { v0.b }[8], [x27], #0x1\n" - "ld1 { v31.b }[8], [x26], #0x1\n" - "ld1 { v30.b }[8], [x25], #0x1\n" + "ld1 { v11.b }[8], [x14], #0x1\n" + "ld1 { v10.b }[8], [x13], #0x1\n" + "ld1 { v3.b }[8], [x10], #0x1\n" + "ld1 { v14.b }[8], [x9], #0x1\n" + "ld1 { v13.b }[8], [x28], #0x1\n" + "ld1 { v28.b }[8], [x27], #0x1\n" + "ld1 { v21.b }[8], [x26], #0x1\n" + "ld1 { v27.b }[8], [x21], #0x1\n" "b 11f\n" "7:" // Oddments: Load (A): Bit 3: Unset "tbz %x[n_channels], #2, 9f\n" - "ldr s6, [x14], #0x4\n" - "ldr s5, [x13], #0x4\n" - "ldr s4, [x10], #0x4\n" - "ldr s3, [x9], #0x4\n" - "ldr s1, [x28], #0x4\n" - "ldr s0, [x27], #0x4\n" - "ldr s31, [x26], #0x4\n" - "ldr s30, [x25], #0x4\n" + "ldr s11, [x14], #0x4\n" + "ldr s10, [x13], #0x4\n" + "ldr s3, [x10], #0x4\n" + "ldr s14, [x9], #0x4\n" + "ldr s13, [x28], #0x4\n" + "ldr s28, [x27], #0x4\n" + "ldr s21, [x26], #0x4\n" + "ldr s27, [x21], #0x4\n" "tbz %x[n_channels], #1, 8f\n" - "ld1 { v6.h }[2], [x14], #0x2\n" - "ld1 { v5.h }[2], [x13], #0x2\n" - "ld1 { v4.h }[2], [x10], #0x2\n" - "ld1 { v3.h }[2], [x9], #0x2\n" - "ld1 { v1.h }[2], [x28], #0x2\n" - "ld1 { v0.h }[2], [x27], #0x2\n" - "ld1 { v31.h }[2], [x26], #0x2\n" - "ld1 { v30.h }[2], [x25], #0x2\n" + "ld1 { v11.h }[2], [x14], #0x2\n" + "ld1 { v10.h }[2], [x13], #0x2\n" + "ld1 { v3.h }[2], [x10], #0x2\n" + "ld1 { v14.h }[2], [x9], #0x2\n" + "ld1 { v13.h }[2], [x28], #0x2\n" + "ld1 { v28.h }[2], [x27], #0x2\n" + "ld1 { v21.h }[2], [x26], #0x2\n" + "ld1 { v27.h }[2], [x21], #0x2\n" "tbz %x[n_channels], #0, 11f\n" - "ld1 { v6.b }[6], [x14], #0x1\n" - "ld1 { v5.b }[6], [x13], #0x1\n" - "ld1 { v4.b }[6], [x10], #0x1\n" - "ld1 { v3.b }[6], [x9], #0x1\n" - "ld1 { v1.b }[6], [x28], #0x1\n" - "ld1 { v0.b }[6], [x27], #0x1\n" - "ld1 { v31.b }[6], [x26], #0x1\n" - "ld1 { v30.b }[6], [x25], #0x1\n" + "ld1 { v11.b }[6], [x14], #0x1\n" + "ld1 { v10.b }[6], [x13], #0x1\n" + "ld1 { v3.b }[6], [x10], #0x1\n" + "ld1 { v14.b }[6], [x9], #0x1\n" + "ld1 { v13.b }[6], [x28], #0x1\n" + "ld1 { v28.b }[6], [x27], #0x1\n" + "ld1 { v21.b }[6], [x26], #0x1\n" + "ld1 { v27.b }[6], [x21], #0x1\n" "b 11f\n" "8:" // Oddments: Load (A): Bit 3: Unset: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 11f\n" - "ld1 { v6.b }[4], [x14], #0x1\n" - "ld1 { v5.b }[4], [x13], #0x1\n" - "ld1 { v4.b }[4], [x10], #0x1\n" - "ld1 { v3.b }[4], [x9], #0x1\n" - "ld1 { v1.b }[4], [x28], #0x1\n" - "ld1 { v0.b }[4], [x27], #0x1\n" - "ld1 { v31.b }[4], [x26], #0x1\n" - "ld1 { v30.b }[4], [x25], #0x1\n" + "ld1 { v11.b }[4], [x14], #0x1\n" + "ld1 { v10.b }[4], [x13], #0x1\n" + "ld1 { v3.b }[4], [x10], #0x1\n" + "ld1 { v14.b }[4], [x9], #0x1\n" + "ld1 { v13.b }[4], [x28], #0x1\n" + "ld1 { v28.b }[4], [x27], #0x1\n" + "ld1 { v21.b }[4], [x26], #0x1\n" + "ld1 { v27.b }[4], [x21], #0x1\n" "b 11f\n" "9:" // Oddments: Load (A): Bit 3: Unset: Bit 2: Unset "tbz %x[n_channels], #1, 10f\n" - "ldr h6, [x14], #0x2\n" - "ldr h5, [x13], #0x2\n" - "ldr h4, [x10], #0x2\n" - "ldr h3, [x9], #0x2\n" - "ldr h1, [x28], #0x2\n" - "ldr h0, [x27], #0x2\n" - "ldr h31, [x26], #0x2\n" - "ldr h30, [x25], #0x2\n" + "ldr h11, [x14], #0x2\n" + "ldr h10, [x13], #0x2\n" + "ldr h3, [x10], #0x2\n" + "ldr h14, [x9], #0x2\n" + "ldr h13, [x28], #0x2\n" + "ldr h28, [x27], #0x2\n" + "ldr h21, [x26], #0x2\n" + "ldr h27, [x21], #0x2\n" "tbz %x[n_channels], #0, 11f\n" - "ld1 { v6.b }[2], [x14], #0x1\n" - "ld1 { v5.b }[2], [x13], #0x1\n" - "ld1 { v4.b }[2], [x10], #0x1\n" - "ld1 { v3.b }[2], [x9], #0x1\n" - "ld1 { v1.b }[2], [x28], #0x1\n" - "ld1 { v0.b }[2], [x27], #0x1\n" - "ld1 { v31.b }[2], [x26], #0x1\n" - "ld1 { v30.b }[2], [x25], #0x1\n" + "ld1 { v11.b }[2], [x14], #0x1\n" + "ld1 { v10.b }[2], [x13], #0x1\n" + "ld1 { v3.b }[2], [x10], #0x1\n" + "ld1 { v14.b }[2], [x9], #0x1\n" + "ld1 { v13.b }[2], [x28], #0x1\n" + "ld1 { v28.b }[2], [x27], #0x1\n" + "ld1 { v21.b }[2], [x26], #0x1\n" + "ld1 { v27.b }[2], [x21], #0x1\n" "b 11f\n" "10:" // Oddments: Load (A): Bit 3: Unset: Bit 2: Unset: Bit 1: Unset - "ldr b6, [x14], #0x1\n" - "ldr b5, [x13], #0x1\n" - "ldr b4, [x10], #0x1\n" - "ldr b3, [x9], #0x1\n" - "ldr b1, [x28], #0x1\n" - "ldr b0, [x27], #0x1\n" - "ldr b31, [x26], #0x1\n" - "ldr b30, [x25], #0x1\n" + "ldr b11, [x14], #0x1\n" + "ldr b10, [x13], #0x1\n" + "ldr b3, [x10], #0x1\n" + "ldr b14, [x9], #0x1\n" + "ldr b13, [x28], #0x1\n" + "ldr b28, [x27], #0x1\n" + "ldr b21, [x26], #0x1\n" + "ldr b27, [x21], #0x1\n" "11:" // Oddments: Load (A): Bit 3: End "ldp x14, x13, [%x[inptrs], #0x40]\n" "ldp x10, x9, [%x[inptrs], #0x50]\n" "add x14, x14, x12\n" "add x13, x13, x12\n" "ldp x28, x27, [%x[inptrs], #0x60]\n" - "ldp x26, x25, [%x[inptrs], #0x70]\n" + "ldp x26, x21, [%x[inptrs], #0x70]\n" "add x10, x10, x12\n" "add x9, x9, x12\n" "add x28, x28, x12\n" "add x27, x27, x12\n" "add x26, x26, x12\n" - "add x25, x25, x12\n" + "add x21, x21, x12\n" "tbz %x[n_channels], #3, 15f\n" - "ldr d25, [x14], #0x8\n" - "ldr d24, [x13], #0x8\n" - "ldr d23, [x10], #0x8\n" - "ldr d22, [x9], #0x8\n" - "ldr d20, [x28], #0x8\n" - "ldr d19, [x27], #0x8\n" - "ldr d18, [x26], #0x8\n" - "ldr d17, [x25], #0x8\n" + "ldr d5, [x14], #0x8\n" + "ldr d29, [x13], #0x8\n" + "ldr d0, [x10], #0x8\n" + "ldr d7, [x9], #0x8\n" + "ldr d16, [x28], #0x8\n" + "ldr d30, [x27], #0x8\n" + "ldr d2, [x26], #0x8\n" + "ldr d1, [x21], #0x8\n" "tbz %x[n_channels], #2, 13f\n" - "ld1 { v25.s }[2], [x14], #0x4\n" - "ld1 { v24.s }[2], [x13], #0x4\n" - "ld1 { v23.s }[2], [x10], #0x4\n" - "ld1 { v22.s }[2], [x9], #0x4\n" - "ld1 { v20.s }[2], [x28], #0x4\n" - "ld1 { v19.s }[2], [x27], #0x4\n" - "ld1 { v18.s }[2], [x26], #0x4\n" - "ld1 { v17.s }[2], [x25], #0x4\n" + "ld1 { v5.s }[2], [x14], #0x4\n" + "ld1 { v29.s }[2], [x13], #0x4\n" + "ld1 { v0.s }[2], [x10], #0x4\n" + "ld1 { v7.s }[2], [x9], #0x4\n" + "ld1 { v16.s }[2], [x28], #0x4\n" + "ld1 { v30.s }[2], [x27], #0x4\n" + "ld1 { v2.s }[2], [x26], #0x4\n" + "ld1 { v1.s }[2], [x21], #0x4\n" "tbz %x[n_channels], #1, 12f\n" - "ld1 { v25.h }[6], [x14], #0x2\n" - "ld1 { v24.h }[6], [x13], #0x2\n" - "ld1 { v23.h }[6], [x10], #0x2\n" - "ld1 { v22.h }[6], [x9], #0x2\n" - "ld1 { v20.h }[6], [x28], #0x2\n" - "ld1 { v19.h }[6], [x27], #0x2\n" - "ld1 { v18.h }[6], [x26], #0x2\n" - "ld1 { v17.h }[6], [x25], #0x2\n" + "ld1 { v5.h }[6], [x14], #0x2\n" + "ld1 { v29.h }[6], [x13], #0x2\n" + "ld1 { v0.h }[6], [x10], #0x2\n" + "ld1 { v7.h }[6], [x9], #0x2\n" + "ld1 { v16.h }[6], [x28], #0x2\n" + "ld1 { v30.h }[6], [x27], #0x2\n" + "ld1 { v2.h }[6], [x26], #0x2\n" + "ld1 { v1.h }[6], [x21], #0x2\n" "tbz %x[n_channels], #0, 19f\n" - "ld1 { v25.b }[14], [x14], #0x1\n" - "ld1 { v24.b }[14], [x13], #0x1\n" - "ld1 { v23.b }[14], [x10], #0x1\n" - "ld1 { v22.b }[14], [x9], #0x1\n" - "ld1 { v20.b }[14], [x28], #0x1\n" - "ld1 { v19.b }[14], [x27], #0x1\n" - "ld1 { v18.b }[14], [x26], #0x1\n" - "ld1 { v17.b }[14], [x25], #0x1\n" + "ld1 { v5.b }[14], [x14], #0x1\n" + "ld1 { v29.b }[14], [x13], #0x1\n" + "ld1 { v0.b }[14], [x10], #0x1\n" + "ld1 { v7.b }[14], [x9], #0x1\n" + "ld1 { v16.b }[14], [x28], #0x1\n" + "ld1 { v30.b }[14], [x27], #0x1\n" + "ld1 { v2.b }[14], [x26], #0x1\n" + "ld1 { v1.b }[14], [x21], #0x1\n" "b 19f\n" "12:" // Oddments: Load (B): Bit 3: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 19f\n" - "ld1 { v25.b }[12], [x14], #0x1\n" - "ld1 { v24.b }[12], [x13], #0x1\n" - "ld1 { v23.b }[12], [x10], #0x1\n" - "ld1 { v22.b }[12], [x9], #0x1\n" - "ld1 { v20.b }[12], [x28], #0x1\n" - "ld1 { v19.b }[12], [x27], #0x1\n" - "ld1 { v18.b }[12], [x26], #0x1\n" - "ld1 { v17.b }[12], [x25], #0x1\n" + "ld1 { v5.b }[12], [x14], #0x1\n" + "ld1 { v29.b }[12], [x13], #0x1\n" + "ld1 { v0.b }[12], [x10], #0x1\n" + "ld1 { v7.b }[12], [x9], #0x1\n" + "ld1 { v16.b }[12], [x28], #0x1\n" + "ld1 { v30.b }[12], [x27], #0x1\n" + "ld1 { v2.b }[12], [x26], #0x1\n" + "ld1 { v1.b }[12], [x21], #0x1\n" "b 19f\n" "13:" // Oddments: Load (B): Bit 3: Bit 2: Unset "tbz %x[n_channels], #1, 14f\n" - "ld1 { v25.h }[4], [x14], #0x2\n" - "ld1 { v24.h }[4], [x13], #0x2\n" - "ld1 { v23.h }[4], [x10], #0x2\n" - "ld1 { v22.h }[4], [x9], #0x2\n" - "ld1 { v20.h }[4], [x28], #0x2\n" - "ld1 { v19.h }[4], [x27], #0x2\n" - "ld1 { v18.h }[4], [x26], #0x2\n" - "ld1 { v17.h }[4], [x25], #0x2\n" + "ld1 { v5.h }[4], [x14], #0x2\n" + "ld1 { v29.h }[4], [x13], #0x2\n" + "ld1 { v0.h }[4], [x10], #0x2\n" + "ld1 { v7.h }[4], [x9], #0x2\n" + "ld1 { v16.h }[4], [x28], #0x2\n" + "ld1 { v30.h }[4], [x27], #0x2\n" + "ld1 { v2.h }[4], [x26], #0x2\n" + "ld1 { v1.h }[4], [x21], #0x2\n" "tbz %x[n_channels], #0, 19f\n" - "ld1 { v25.b }[10], [x14], #0x1\n" - "ld1 { v24.b }[10], [x13], #0x1\n" - "ld1 { v23.b }[10], [x10], #0x1\n" - "ld1 { v22.b }[10], [x9], #0x1\n" - "ld1 { v20.b }[10], [x28], #0x1\n" - "ld1 { v19.b }[10], [x27], #0x1\n" - "ld1 { v18.b }[10], [x26], #0x1\n" - "ld1 { v17.b }[10], [x25], #0x1\n" + "ld1 { v5.b }[10], [x14], #0x1\n" + "ld1 { v29.b }[10], [x13], #0x1\n" + "ld1 { v0.b }[10], [x10], #0x1\n" + "ld1 { v7.b }[10], [x9], #0x1\n" + "ld1 { v16.b }[10], [x28], #0x1\n" + "ld1 { v30.b }[10], [x27], #0x1\n" + "ld1 { v2.b }[10], [x26], #0x1\n" + "ld1 { v1.b }[10], [x21], #0x1\n" "b 19f\n" "14:" // Oddments: Load (B): Bit 3: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 19f\n" - "ld1 { v25.b }[8], [x14], #0x1\n" - "ld1 { v24.b }[8], [x13], #0x1\n" - "ld1 { v23.b }[8], [x10], #0x1\n" - "ld1 { v22.b }[8], [x9], #0x1\n" - "ld1 { v20.b }[8], [x28], #0x1\n" - "ld1 { v19.b }[8], [x27], #0x1\n" - "ld1 { v18.b }[8], [x26], #0x1\n" - "ld1 { v17.b }[8], [x25], #0x1\n" + "ld1 { v5.b }[8], [x14], #0x1\n" + "ld1 { v29.b }[8], [x13], #0x1\n" + "ld1 { v0.b }[8], [x10], #0x1\n" + "ld1 { v7.b }[8], [x9], #0x1\n" + "ld1 { v16.b }[8], [x28], #0x1\n" + "ld1 { v30.b }[8], [x27], #0x1\n" + "ld1 { v2.b }[8], [x26], #0x1\n" + "ld1 { v1.b }[8], [x21], #0x1\n" "b 19f\n" "15:" // Oddments: Load (B): Bit 3: Unset "tbz %x[n_channels], #2, 17f\n" - "ldr s25, [x14], #0x4\n" - "ldr s24, [x13], #0x4\n" - "ldr s23, [x10], #0x4\n" - "ldr s22, [x9], #0x4\n" - "ldr s20, [x28], #0x4\n" - "ldr s19, [x27], #0x4\n" - "ldr s18, [x26], #0x4\n" - "ldr s17, [x25], #0x4\n" + "ldr s5, [x14], #0x4\n" + "ldr s29, [x13], #0x4\n" + "ldr s0, [x10], #0x4\n" + "ldr s7, [x9], #0x4\n" + "ldr s16, [x28], #0x4\n" + "ldr s30, [x27], #0x4\n" + "ldr s2, [x26], #0x4\n" + "ldr s1, [x21], #0x4\n" "tbz %x[n_channels], #1, 16f\n" - "ld1 { v25.h }[2], [x14], #0x2\n" - "ld1 { v24.h }[2], [x13], #0x2\n" - "ld1 { v23.h }[2], [x10], #0x2\n" - "ld1 { v22.h }[2], [x9], #0x2\n" - "ld1 { v20.h }[2], [x28], #0x2\n" - "ld1 { v19.h }[2], [x27], #0x2\n" - "ld1 { v18.h }[2], [x26], #0x2\n" - "ld1 { v17.h }[2], [x25], #0x2\n" + "ld1 { v5.h }[2], [x14], #0x2\n" + "ld1 { v29.h }[2], [x13], #0x2\n" + "ld1 { v0.h }[2], [x10], #0x2\n" + "ld1 { v7.h }[2], [x9], #0x2\n" + "ld1 { v16.h }[2], [x28], #0x2\n" + "ld1 { v30.h }[2], [x27], #0x2\n" + "ld1 { v2.h }[2], [x26], #0x2\n" + "ld1 { v1.h }[2], [x21], #0x2\n" "tbz %x[n_channels], #0, 19f\n" - "ld1 { v25.b }[6], [x14], #0x1\n" - "ld1 { v24.b }[6], [x13], #0x1\n" - "ld1 { v23.b }[6], [x10], #0x1\n" - "ld1 { v22.b }[6], [x9], #0x1\n" - "ld1 { v20.b }[6], [x28], #0x1\n" - "ld1 { v19.b }[6], [x27], #0x1\n" - "ld1 { v18.b }[6], [x26], #0x1\n" - "ld1 { v17.b }[6], [x25], #0x1\n" + "ld1 { v5.b }[6], [x14], #0x1\n" + "ld1 { v29.b }[6], [x13], #0x1\n" + "ld1 { v0.b }[6], [x10], #0x1\n" + "ld1 { v7.b }[6], [x9], #0x1\n" + "ld1 { v16.b }[6], [x28], #0x1\n" + "ld1 { v30.b }[6], [x27], #0x1\n" + "ld1 { v2.b }[6], [x26], #0x1\n" + "ld1 { v1.b }[6], [x21], #0x1\n" "b 19f\n" "16:" // Oddments: Load (B): Bit 3: Unset: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 19f\n" - "ld1 { v25.b }[4], [x14], #0x1\n" - "ld1 { v24.b }[4], [x13], #0x1\n" - "ld1 { v23.b }[4], [x10], #0x1\n" - "ld1 { v22.b }[4], [x9], #0x1\n" - "ld1 { v20.b }[4], [x28], #0x1\n" - "ld1 { v19.b }[4], [x27], #0x1\n" - "ld1 { v18.b }[4], [x26], #0x1\n" - "ld1 { v17.b }[4], [x25], #0x1\n" + "ld1 { v5.b }[4], [x14], #0x1\n" + "ld1 { v29.b }[4], [x13], #0x1\n" + "ld1 { v0.b }[4], [x10], #0x1\n" + "ld1 { v7.b }[4], [x9], #0x1\n" + "ld1 { v16.b }[4], [x28], #0x1\n" + "ld1 { v30.b }[4], [x27], #0x1\n" + "ld1 { v2.b }[4], [x26], #0x1\n" + "ld1 { v1.b }[4], [x21], #0x1\n" "b 19f\n" "17:" // Oddments: Load (B): Bit 3: Unset: Bit 2: Unset "tbz %x[n_channels], #1, 18f\n" - "ldr h25, [x14], #0x2\n" - "ldr h24, [x13], #0x2\n" - "ldr h23, [x10], #0x2\n" - "ldr h22, [x9], #0x2\n" - "ldr h20, [x28], #0x2\n" - "ldr h19, [x27], #0x2\n" - "ldr h18, [x26], #0x2\n" - "ldr h17, [x25], #0x2\n" + "ldr h5, [x14], #0x2\n" + "ldr h29, [x13], #0x2\n" + "ldr h0, [x10], #0x2\n" + "ldr h7, [x9], #0x2\n" + "ldr h16, [x28], #0x2\n" + "ldr h30, [x27], #0x2\n" + "ldr h2, [x26], #0x2\n" + "ldr h1, [x21], #0x2\n" "tbz %x[n_channels], #0, 19f\n" - "ld1 { v25.b }[2], [x14], #0x1\n" - "ld1 { v24.b }[2], [x13], #0x1\n" - "ld1 { v23.b }[2], [x10], #0x1\n" - "ld1 { v22.b }[2], [x9], #0x1\n" - "ld1 { v20.b }[2], [x28], #0x1\n" - "ld1 { v19.b }[2], [x27], #0x1\n" - "ld1 { v18.b }[2], [x26], #0x1\n" - "ld1 { v17.b }[2], [x25], #0x1\n" + "ld1 { v5.b }[2], [x14], #0x1\n" + "ld1 { v29.b }[2], [x13], #0x1\n" + "ld1 { v0.b }[2], [x10], #0x1\n" + "ld1 { v7.b }[2], [x9], #0x1\n" + "ld1 { v16.b }[2], [x28], #0x1\n" + "ld1 { v30.b }[2], [x27], #0x1\n" + "ld1 { v2.b }[2], [x26], #0x1\n" + "ld1 { v1.b }[2], [x21], #0x1\n" "b 19f\n" "18:" // Oddments: Load (B): Bit 3: Unset: Bit 2: Unset: Bit 1: Unset - "ldr b25, [x14], #0x1\n" - "ldr b24, [x13], #0x1\n" - "ldr b23, [x10], #0x1\n" - "ldr b22, [x9], #0x1\n" - "ldr b20, [x28], #0x1\n" - "ldr b19, [x27], #0x1\n" - "ldr b18, [x26], #0x1\n" - "ldr b17, [x25], #0x1\n" + "ldr b5, [x14], #0x1\n" + "ldr b29, [x13], #0x1\n" + "ldr b0, [x10], #0x1\n" + "ldr b7, [x9], #0x1\n" + "ldr b16, [x28], #0x1\n" + "ldr b30, [x27], #0x1\n" + "ldr b2, [x26], #0x1\n" + "ldr b1, [x21], #0x1\n" "19:" // Oddments: Load (B): Bit 3: End - "ldr q29, [%x[params], #0x10]\n" - "ldr q28, [%x[params], #0x20]\n" - "zip2 v2.16b, v6.16b, v4.16b\n" - "zip1 v6.16b, v6.16b, v4.16b\n" - "ldr q27, [%x[params], #0x30]\n" - "zip1 v4.16b, v5.16b, v3.16b\n" - "zip2 v3.16b, v5.16b, v3.16b\n" + "ldr q25, [%x[params], #0x10]\n" + "ldr q24, [%x[params], #0x20]\n" + "zip2 v18.16b, v11.16b, v3.16b\n" + "zip1 v11.16b, v11.16b, v3.16b\n" + "ldr q23, [%x[params], #0x30]\n" + "zip1 v17.16b, v10.16b, v14.16b\n" + "zip2 v14.16b, v10.16b, v14.16b\n" "cmp x20, #0x4\n" - "zip2 v5.16b, v6.16b, v4.16b\n" - "zip1 v6.16b, v6.16b, v4.16b\n" - "zip1 v4.16b, v2.16b, v3.16b\n" - "zip2 v3.16b, v2.16b, v3.16b\n" - "ldr q2, [%x[params], #0x0]\n" - "zip2 v26.16b, v1.16b, v31.16b\n" - "zip1 v1.16b, v1.16b, v31.16b\n" - "zip1 v31.16b, v0.16b, v30.16b\n" - "zip2 v30.16b, v0.16b, v30.16b\n" - "zip2 v21.16b, v25.16b, v23.16b\n" - "zip1 v25.16b, v25.16b, v23.16b\n" - "zip1 v23.16b, v24.16b, v22.16b\n" - "zip2 v22.16b, v24.16b, v22.16b\n" - "zip2 v16.16b, v20.16b, v18.16b\n" - "zip1 v20.16b, v20.16b, v18.16b\n" - "zip1 v18.16b, v19.16b, v17.16b\n" - "zip2 v17.16b, v19.16b, v17.16b\n" - "zip2 v0.16b, v1.16b, v31.16b\n" - "zip1 v1.16b, v1.16b, v31.16b\n" - "zip1 v31.16b, v26.16b, v30.16b\n" - "zip2 v30.16b, v26.16b, v30.16b\n" - "zip2 v24.16b, v25.16b, v23.16b\n" - "zip1 v25.16b, v25.16b, v23.16b\n" - "zip1 v23.16b, v21.16b, v22.16b\n" - "zip2 v22.16b, v21.16b, v22.16b\n" - "zip2 v19.16b, v20.16b, v18.16b\n" - "zip1 v20.16b, v20.16b, v18.16b\n" - "zip1 v18.16b, v16.16b, v17.16b\n" - "zip2 v17.16b, v16.16b, v17.16b\n" - "mov v26.16b, v2.16b\n" - "mov v21.16b, v2.16b\n" - ".inst 0x4e8197b5 // sdot v21.4s, v29.16b, v1.16b\n" - "mov v16.16b, v2.16b\n" - ".inst 0x4e8697a2 // sdot v2.4s, v29.16b, v6.16b\n" - ".inst 0x4e819782 // sdot v2.4s, v28.16b, v1.16b\n" - "ext v6.16b, v6.16b, v6.16b, #0x1\n" - "ext v1.16b, v1.16b, v1.16b, #0x1\n" - ".inst 0x4e8697ba // sdot v26.4s, v29.16b, v6.16b\n" - "ldr q6, [%x[params], #0x40]\n" - ".inst 0x4e8197b0 // sdot v16.4s, v29.16b, v1.16b\n" - ".inst 0x4e999795 // sdot v21.4s, v28.16b, v25.16b\n" - ".inst 0x4e999762 // sdot v2.4s, v27.16b, v25.16b\n" - "ext v25.16b, v25.16b, v25.16b, #0x1\n" - ".inst 0x4e81979a // sdot v26.4s, v28.16b, v1.16b\n" - "ldr q1, [%x[params], #0x50]\n" - ".inst 0x4e999790 // sdot v16.4s, v28.16b, v25.16b\n" - ".inst 0x4e949775 // sdot v21.4s, v27.16b, v20.16b\n" - "ext v20.16b, v20.16b, v20.16b, #0x1\n" + "zip2 v10.16b, v11.16b, v17.16b\n" + "zip1 v11.16b, v11.16b, v17.16b\n" + "zip1 v3.16b, v18.16b, v14.16b\n" + "zip2 v14.16b, v18.16b, v14.16b\n" + "ldr q31, [%x[params], #0x0]\n" + "zip2 v22.16b, v13.16b, v21.16b\n" + "zip1 v13.16b, v13.16b, v21.16b\n" + "zip1 v21.16b, v28.16b, v27.16b\n" + "zip2 v27.16b, v28.16b, v27.16b\n" + "zip2 v20.16b, v5.16b, v0.16b\n" + "zip1 v5.16b, v5.16b, v0.16b\n" + "zip1 v19.16b, v29.16b, v7.16b\n" + "zip2 v7.16b, v29.16b, v7.16b\n" + "zip2 v18.16b, v16.16b, v2.16b\n" + "zip1 v16.16b, v16.16b, v2.16b\n" + "zip1 v17.16b, v30.16b, v1.16b\n" + "zip2 v1.16b, v30.16b, v1.16b\n" + "zip2 v28.16b, v13.16b, v21.16b\n" + "zip1 v13.16b, v13.16b, v21.16b\n" + "zip1 v21.16b, v22.16b, v27.16b\n" + "zip2 v27.16b, v22.16b, v27.16b\n" + "zip2 v29.16b, v5.16b, v19.16b\n" + "zip1 v5.16b, v5.16b, v19.16b\n" + "zip1 v0.16b, v20.16b, v7.16b\n" + "zip2 v7.16b, v20.16b, v7.16b\n" + "zip2 v30.16b, v16.16b, v17.16b\n" + "zip1 v16.16b, v16.16b, v17.16b\n" + "zip1 v2.16b, v18.16b, v1.16b\n" + "zip2 v1.16b, v18.16b, v1.16b\n" + "mov v26.16b, v31.16b\n" + "mov v18.16b, v31.16b\n" + ".inst 0x4e8d9732 // sdot v18.4s, v25.16b, v13.16b\n" + "mov v4.16b, v31.16b\n" + ".inst 0x4e8b973f // sdot v31.4s, v25.16b, v11.16b\n" + ".inst 0x4e8d971f // sdot v31.4s, v24.16b, v13.16b\n" + "ext v11.16b, v11.16b, v11.16b, #0x1\n" + "ext v13.16b, v13.16b, v13.16b, #0x1\n" + ".inst 0x4e8b973a // sdot v26.4s, v25.16b, v11.16b\n" + "ldr q17, [%x[params], #0x40]\n" + ".inst 0x4e8d9724 // sdot v4.4s, v25.16b, v13.16b\n" + ".inst 0x4e859712 // sdot v18.4s, v24.16b, v5.16b\n" + ".inst 0x4e8596ff // sdot v31.4s, v23.16b, v5.16b\n" + "ext v5.16b, v5.16b, v5.16b, #0x1\n" + ".inst 0x4e8d971a // sdot v26.4s, v24.16b, v13.16b\n" + "ldr q20, [%x[params], #0x50]\n" + ".inst 0x4e859704 // sdot v4.4s, v24.16b, v5.16b\n" + ".inst 0x4e9096f2 // sdot v18.4s, v23.16b, v16.16b\n" + "ext v16.16b, v16.16b, v16.16b, #0x1\n" "add %x[params], %x[params], #0x60\n" - "sqrdmulh v2.4s, v2.4s, v6.4s\n" - ".inst 0x4e99977a // sdot v26.4s, v27.16b, v25.16b\n" - ".inst 0x4e949770 // sdot v16.4s, v27.16b, v20.16b\n" - "and v29.16b, v2.16b, v1.16b\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "sqrdmulh v26.4s, v26.4s, v6.4s\n" - "sqrdmulh v21.4s, v21.4s, v6.4s\n" - "sqrdmulh v16.4s, v16.4s, v6.4s\n" - "sqadd v2.4s, v2.4s, v29.4s\n" - "and v28.16b, v26.16b, v1.16b\n" - "and v27.16b, v21.16b, v1.16b\n" - "and v29.16b, v16.16b, v1.16b\n" - "sshr v28.4s, v28.4s, #0x1f\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "sqadd v26.4s, v26.4s, v28.4s\n" - "sqadd v21.4s, v21.4s, v27.4s\n" - "sqadd v16.4s, v16.4s, v29.4s\n" - "srshl v2.4s, v2.4s, v1.4s\n" - "srshl v26.4s, v26.4s, v1.4s\n" - "srshl v21.4s, v21.4s, v1.4s\n" - "srshl v16.4s, v16.4s, v1.4s\n" - "add v2.4s, v2.4s, v7.4s\n" - "add v26.4s, v26.4s, v7.4s\n" - "add v21.4s, v21.4s, v7.4s\n" - "add v16.4s, v16.4s, v7.4s\n" - "smax v2.4s, v2.4s, v9.4s\n" - "smax v26.4s, v26.4s, v9.4s\n" - "smax v21.4s, v21.4s, v9.4s\n" - "smax v16.4s, v16.4s, v9.4s\n" - "smin v2.4s, v2.4s, v8.4s\n" - "smin v26.4s, v26.4s, v8.4s\n" - "smin v21.4s, v21.4s, v8.4s\n" - "smin v16.4s, v16.4s, v8.4s\n" - "uzp1 v2.16b, v2.16b, v2.16b\n" + "sqrdmulh v31.4s, v31.4s, v17.4s\n" + ".inst 0x4e8596fa // sdot v26.4s, v23.16b, v5.16b\n" + ".inst 0x4e9096e4 // sdot v4.4s, v23.16b, v16.16b\n" + "and v16.16b, v31.16b, v20.16b\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqrdmulh v26.4s, v26.4s, v17.4s\n" + "sqrdmulh v18.4s, v18.4s, v17.4s\n" + "sqrdmulh v4.4s, v4.4s, v17.4s\n" + "sqadd v31.4s, v31.4s, v16.4s\n" + "and v19.16b, v26.16b, v20.16b\n" + "and v17.16b, v18.16b, v20.16b\n" + "and v16.16b, v4.16b, v20.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v26.4s, v26.4s, v19.4s\n" + "sqadd v18.4s, v18.4s, v17.4s\n" + "sqadd v4.4s, v4.4s, v16.4s\n" + "srshl v31.4s, v31.4s, v20.4s\n" + "srshl v26.4s, v26.4s, v20.4s\n" + "srshl v18.4s, v18.4s, v20.4s\n" + "srshl v4.4s, v4.4s, v20.4s\n" + "add v31.4s, v31.4s, v15.4s\n" + "add v26.4s, v26.4s, v15.4s\n" + "add v18.4s, v18.4s, v15.4s\n" + "add v4.4s, v4.4s, v15.4s\n" + "smax v31.4s, v31.4s, v8.4s\n" + "smax v26.4s, v26.4s, v8.4s\n" + "smax v18.4s, v18.4s, v8.4s\n" + "smax v4.4s, v4.4s, v8.4s\n" + "smin v31.4s, v31.4s, v12.4s\n" + "smin v26.4s, v26.4s, v12.4s\n" + "smin v18.4s, v18.4s, v12.4s\n" + "smin v4.4s, v4.4s, v12.4s\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "uzp1 v21.16b, v21.16b, v21.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "uzp1 v2.16b, v2.16b, v2.16b\n" + "uzp1 v18.16b, v18.16b, v18.16b\n" + "uzp1 v4.16b, v4.16b, v4.16b\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "uzp1 v21.16b, v21.16b, v21.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" + "uzp1 v18.16b, v18.16b, v18.16b\n" + "uzp1 v4.16b, v4.16b, v4.16b\n" "blt 20f\n" - "str s2, [x24, x11]\n" - "str s26, [x23, x11]\n" - "str s21, [x22, x11]\n" - "str s16, [x21, x11]\n" + "str s31, [x25, x11]\n" + "str s26, [x24, x11]\n" + "str s18, [x23, x11]\n" + "str s4, [x22, x11]\n" "b 23f\n" "20:" // Oddments: Unroll 0: Oddment store + "add x25, x25, x11\n" "add x24, x24, x11\n" "add x23, x23, x11\n" "add x22, x22, x11\n" - "add x21, x21, x11\n" "tbz x20, #1, 21f\n" - "st1 { v2.h }[0], [x24], #0x2\n" - "st1 { v26.h }[0], [x23], #0x2\n" - "st1 { v21.h }[0], [x22], #0x2\n" - "st1 { v16.h }[0], [x21], #0x2\n" + "st1 { v31.h }[0], [x25], #0x2\n" + "st1 { v26.h }[0], [x24], #0x2\n" + "st1 { v18.h }[0], [x23], #0x2\n" + "st1 { v4.h }[0], [x22], #0x2\n" "tbz x20, #0, 22f\n" - "st1 { v2.b }[2], [x24], #0x1\n" - "st1 { v26.b }[2], [x23], #0x1\n" - "st1 { v21.b }[2], [x22], #0x1\n" - "st1 { v16.b }[2], [x21], #0x1\n" + "st1 { v31.b }[2], [x25], #0x1\n" + "st1 { v26.b }[2], [x24], #0x1\n" + "st1 { v18.b }[2], [x23], #0x1\n" + "st1 { v4.b }[2], [x22], #0x1\n" "b 22f\n" "21:" // Oddments: Unroll 0: Oddment store: Bit 1: Unset - "st1 { v2.b }[0], [x24], #0x1\n" - "st1 { v26.b }[0], [x23], #0x1\n" - "st1 { v21.b }[0], [x22], #0x1\n" - "st1 { v16.b }[0], [x21], #0x1\n" + "st1 { v31.b }[0], [x25], #0x1\n" + "st1 { v26.b }[0], [x24], #0x1\n" + "st1 { v18.b }[0], [x23], #0x1\n" + "st1 { v4.b }[0], [x22], #0x1\n" "22:" // Oddments: Unroll 0: Oddment store: Bit 1: End "23:" // Oddments: Unroll 0: After oddment store "subs x20, x20, #0x4\n" "add x11, x11, #0x4\n" "ble 35f\n" - "ldr q2, [%x[params], #0x0]\n" - "ldr q29, [%x[params], #0x10]\n" - "mov v26.16b, v2.16b\n" - "mov v21.16b, v2.16b\n" - "ldr q28, [%x[params], #0x20]\n" - "ldr q27, [%x[params], #0x30]\n" - "mov v16.16b, v2.16b\n" - ".inst 0x4e8597a2 // sdot v2.4s, v29.16b, v5.16b\n" - "ldr q6, [%x[params], #0x40]\n" - "ldr q1, [%x[params], #0x50]\n" - ".inst 0x4e8097b5 // sdot v21.4s, v29.16b, v0.16b\n" - ".inst 0x4e809782 // sdot v2.4s, v28.16b, v0.16b\n" - "ext v5.16b, v5.16b, v5.16b, #0x1\n" - "ext v0.16b, v0.16b, v0.16b, #0x1\n" - ".inst 0x4e8597ba // sdot v26.4s, v29.16b, v5.16b\n" + "ldr q31, [%x[params], #0x0]\n" + "ldr q23, [%x[params], #0x10]\n" + "mov v26.16b, v31.16b\n" + "mov v18.16b, v31.16b\n" + "ldr q22, [%x[params], #0x20]\n" + "ldr q16, [%x[params], #0x30]\n" + "mov v4.16b, v31.16b\n" + ".inst 0x4e8a96ff // sdot v31.4s, v23.16b, v10.16b\n" + "ldr q17, [%x[params], #0x40]\n" + "ldr q20, [%x[params], #0x50]\n" + ".inst 0x4e9c96f2 // sdot v18.4s, v23.16b, v28.16b\n" + ".inst 0x4e9c96df // sdot v31.4s, v22.16b, v28.16b\n" + "ext v10.16b, v10.16b, v10.16b, #0x1\n" + "ext v28.16b, v28.16b, v28.16b, #0x1\n" + ".inst 0x4e8a96fa // sdot v26.4s, v23.16b, v10.16b\n" "cmp x20, #0x4\n" - ".inst 0x4e8097b0 // sdot v16.4s, v29.16b, v0.16b\n" - ".inst 0x4e989795 // sdot v21.4s, v28.16b, v24.16b\n" + ".inst 0x4e9c96e4 // sdot v4.4s, v23.16b, v28.16b\n" + ".inst 0x4e9d96d2 // sdot v18.4s, v22.16b, v29.16b\n" "add %x[params], %x[params], #0x60\n" - ".inst 0x4e989762 // sdot v2.4s, v27.16b, v24.16b\n" - "ext v24.16b, v24.16b, v24.16b, #0x1\n" - ".inst 0x4e80979a // sdot v26.4s, v28.16b, v0.16b\n" - ".inst 0x4e989790 // sdot v16.4s, v28.16b, v24.16b\n" - ".inst 0x4e939775 // sdot v21.4s, v27.16b, v19.16b\n" - "ext v19.16b, v19.16b, v19.16b, #0x1\n" - "sqrdmulh v2.4s, v2.4s, v6.4s\n" - ".inst 0x4e98977a // sdot v26.4s, v27.16b, v24.16b\n" - ".inst 0x4e939770 // sdot v16.4s, v27.16b, v19.16b\n" - "and v29.16b, v2.16b, v1.16b\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "sqrdmulh v26.4s, v26.4s, v6.4s\n" - "sqrdmulh v21.4s, v21.4s, v6.4s\n" - "sqrdmulh v16.4s, v16.4s, v6.4s\n" - "sqadd v2.4s, v2.4s, v29.4s\n" - "and v28.16b, v26.16b, v1.16b\n" - "and v27.16b, v21.16b, v1.16b\n" - "and v29.16b, v16.16b, v1.16b\n" - "sshr v28.4s, v28.4s, #0x1f\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "sqadd v26.4s, v26.4s, v28.4s\n" - "sqadd v21.4s, v21.4s, v27.4s\n" - "sqadd v16.4s, v16.4s, v29.4s\n" - "srshl v2.4s, v2.4s, v1.4s\n" - "srshl v26.4s, v26.4s, v1.4s\n" - "srshl v21.4s, v21.4s, v1.4s\n" - "srshl v16.4s, v16.4s, v1.4s\n" - "add v2.4s, v2.4s, v7.4s\n" - "add v26.4s, v26.4s, v7.4s\n" - "add v21.4s, v21.4s, v7.4s\n" - "add v16.4s, v16.4s, v7.4s\n" - "smax v2.4s, v2.4s, v9.4s\n" - "smax v26.4s, v26.4s, v9.4s\n" - "smax v21.4s, v21.4s, v9.4s\n" - "smax v16.4s, v16.4s, v9.4s\n" - "smin v2.4s, v2.4s, v8.4s\n" - "smin v26.4s, v26.4s, v8.4s\n" - "smin v21.4s, v21.4s, v8.4s\n" - "smin v16.4s, v16.4s, v8.4s\n" - "uzp1 v2.16b, v2.16b, v2.16b\n" + ".inst 0x4e9d961f // sdot v31.4s, v16.16b, v29.16b\n" + "ext v29.16b, v29.16b, v29.16b, #0x1\n" + ".inst 0x4e9c96da // sdot v26.4s, v22.16b, v28.16b\n" + ".inst 0x4e9d96c4 // sdot v4.4s, v22.16b, v29.16b\n" + ".inst 0x4e9e9612 // sdot v18.4s, v16.16b, v30.16b\n" + "ext v30.16b, v30.16b, v30.16b, #0x1\n" + "sqrdmulh v31.4s, v31.4s, v17.4s\n" + ".inst 0x4e9d961a // sdot v26.4s, v16.16b, v29.16b\n" + ".inst 0x4e9e9604 // sdot v4.4s, v16.16b, v30.16b\n" + "and v16.16b, v31.16b, v20.16b\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqrdmulh v26.4s, v26.4s, v17.4s\n" + "sqrdmulh v18.4s, v18.4s, v17.4s\n" + "sqrdmulh v4.4s, v4.4s, v17.4s\n" + "sqadd v31.4s, v31.4s, v16.4s\n" + "and v19.16b, v26.16b, v20.16b\n" + "and v17.16b, v18.16b, v20.16b\n" + "and v16.16b, v4.16b, v20.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v26.4s, v26.4s, v19.4s\n" + "sqadd v18.4s, v18.4s, v17.4s\n" + "sqadd v4.4s, v4.4s, v16.4s\n" + "srshl v31.4s, v31.4s, v20.4s\n" + "srshl v26.4s, v26.4s, v20.4s\n" + "srshl v18.4s, v18.4s, v20.4s\n" + "srshl v4.4s, v4.4s, v20.4s\n" + "add v31.4s, v31.4s, v15.4s\n" + "add v26.4s, v26.4s, v15.4s\n" + "add v18.4s, v18.4s, v15.4s\n" + "add v4.4s, v4.4s, v15.4s\n" + "smax v31.4s, v31.4s, v8.4s\n" + "smax v26.4s, v26.4s, v8.4s\n" + "smax v18.4s, v18.4s, v8.4s\n" + "smax v4.4s, v4.4s, v8.4s\n" + "smin v31.4s, v31.4s, v12.4s\n" + "smin v26.4s, v26.4s, v12.4s\n" + "smin v18.4s, v18.4s, v12.4s\n" + "smin v4.4s, v4.4s, v12.4s\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "uzp1 v21.16b, v21.16b, v21.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "uzp1 v2.16b, v2.16b, v2.16b\n" + "uzp1 v18.16b, v18.16b, v18.16b\n" + "uzp1 v4.16b, v4.16b, v4.16b\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "uzp1 v21.16b, v21.16b, v21.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" + "uzp1 v18.16b, v18.16b, v18.16b\n" + "uzp1 v4.16b, v4.16b, v4.16b\n" "blt 24f\n" - "str s2, [x24, x11]\n" - "str s26, [x23, x11]\n" - "str s21, [x22, x11]\n" - "str s16, [x21, x11]\n" + "str s31, [x25, x11]\n" + "str s26, [x24, x11]\n" + "str s18, [x23, x11]\n" + "str s4, [x22, x11]\n" "b 27f\n" "24:" // Oddments: Unroll 1: Oddment store + "add x25, x25, x11\n" "add x24, x24, x11\n" "add x23, x23, x11\n" "add x22, x22, x11\n" - "add x21, x21, x11\n" "tbz x20, #1, 25f\n" - "st1 { v2.h }[0], [x24], #0x2\n" - "st1 { v26.h }[0], [x23], #0x2\n" - "st1 { v21.h }[0], [x22], #0x2\n" - "st1 { v16.h }[0], [x21], #0x2\n" + "st1 { v31.h }[0], [x25], #0x2\n" + "st1 { v26.h }[0], [x24], #0x2\n" + "st1 { v18.h }[0], [x23], #0x2\n" + "st1 { v4.h }[0], [x22], #0x2\n" "tbz x20, #0, 26f\n" - "st1 { v2.b }[2], [x24], #0x1\n" - "st1 { v26.b }[2], [x23], #0x1\n" - "st1 { v21.b }[2], [x22], #0x1\n" - "st1 { v16.b }[2], [x21], #0x1\n" + "st1 { v31.b }[2], [x25], #0x1\n" + "st1 { v26.b }[2], [x24], #0x1\n" + "st1 { v18.b }[2], [x23], #0x1\n" + "st1 { v4.b }[2], [x22], #0x1\n" "b 26f\n" "25:" // Oddments: Unroll 1: Oddment store: Bit 1: Unset - "st1 { v2.b }[0], [x24], #0x1\n" - "st1 { v26.b }[0], [x23], #0x1\n" - "st1 { v21.b }[0], [x22], #0x1\n" - "st1 { v16.b }[0], [x21], #0x1\n" + "st1 { v31.b }[0], [x25], #0x1\n" + "st1 { v26.b }[0], [x24], #0x1\n" + "st1 { v18.b }[0], [x23], #0x1\n" + "st1 { v4.b }[0], [x22], #0x1\n" "26:" // Oddments: Unroll 1: Oddment store: Bit 1: End "27:" // Oddments: Unroll 1: After oddment store "subs x20, x20, #0x4\n" "add x11, x11, #0x4\n" "ble 35f\n" - "ldr q2, [%x[params], #0x0]\n" - "ldr q29, [%x[params], #0x10]\n" - "mov v26.16b, v2.16b\n" - "mov v21.16b, v2.16b\n" - "ldr q28, [%x[params], #0x20]\n" - "ldr q27, [%x[params], #0x30]\n" - "mov v16.16b, v2.16b\n" - ".inst 0x4e8497a2 // sdot v2.4s, v29.16b, v4.16b\n" - "ldr q6, [%x[params], #0x40]\n" - "ldr q1, [%x[params], #0x50]\n" - ".inst 0x4e9f97b5 // sdot v21.4s, v29.16b, v31.16b\n" - ".inst 0x4e9f9782 // sdot v2.4s, v28.16b, v31.16b\n" - "ext v4.16b, v4.16b, v4.16b, #0x1\n" - "ext v31.16b, v31.16b, v31.16b, #0x1\n" - ".inst 0x4e8497ba // sdot v26.4s, v29.16b, v4.16b\n" + "ldr q31, [%x[params], #0x0]\n" + "ldr q23, [%x[params], #0x10]\n" + "mov v26.16b, v31.16b\n" + "mov v18.16b, v31.16b\n" + "ldr q22, [%x[params], #0x20]\n" + "ldr q16, [%x[params], #0x30]\n" + "mov v4.16b, v31.16b\n" + ".inst 0x4e8396ff // sdot v31.4s, v23.16b, v3.16b\n" + "ldr q17, [%x[params], #0x40]\n" + "ldr q20, [%x[params], #0x50]\n" + ".inst 0x4e9596f2 // sdot v18.4s, v23.16b, v21.16b\n" + ".inst 0x4e9596df // sdot v31.4s, v22.16b, v21.16b\n" + "ext v3.16b, v3.16b, v3.16b, #0x1\n" + "ext v21.16b, v21.16b, v21.16b, #0x1\n" + ".inst 0x4e8396fa // sdot v26.4s, v23.16b, v3.16b\n" "cmp x20, #0x4\n" - ".inst 0x4e9f97b0 // sdot v16.4s, v29.16b, v31.16b\n" - ".inst 0x4e979795 // sdot v21.4s, v28.16b, v23.16b\n" + ".inst 0x4e9596e4 // sdot v4.4s, v23.16b, v21.16b\n" + ".inst 0x4e8096d2 // sdot v18.4s, v22.16b, v0.16b\n" "add %x[params], %x[params], #0x60\n" - ".inst 0x4e979762 // sdot v2.4s, v27.16b, v23.16b\n" - "ext v23.16b, v23.16b, v23.16b, #0x1\n" - ".inst 0x4e9f979a // sdot v26.4s, v28.16b, v31.16b\n" - ".inst 0x4e979790 // sdot v16.4s, v28.16b, v23.16b\n" - ".inst 0x4e929775 // sdot v21.4s, v27.16b, v18.16b\n" - "ext v18.16b, v18.16b, v18.16b, #0x1\n" - "sqrdmulh v2.4s, v2.4s, v6.4s\n" - ".inst 0x4e97977a // sdot v26.4s, v27.16b, v23.16b\n" - ".inst 0x4e929770 // sdot v16.4s, v27.16b, v18.16b\n" - "and v29.16b, v2.16b, v1.16b\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "sqrdmulh v26.4s, v26.4s, v6.4s\n" - "sqrdmulh v21.4s, v21.4s, v6.4s\n" - "sqrdmulh v16.4s, v16.4s, v6.4s\n" - "sqadd v2.4s, v2.4s, v29.4s\n" - "and v28.16b, v26.16b, v1.16b\n" - "and v27.16b, v21.16b, v1.16b\n" - "and v29.16b, v16.16b, v1.16b\n" - "sshr v28.4s, v28.4s, #0x1f\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "sqadd v26.4s, v26.4s, v28.4s\n" - "sqadd v21.4s, v21.4s, v27.4s\n" - "sqadd v16.4s, v16.4s, v29.4s\n" - "srshl v2.4s, v2.4s, v1.4s\n" - "srshl v26.4s, v26.4s, v1.4s\n" - "srshl v21.4s, v21.4s, v1.4s\n" - "srshl v16.4s, v16.4s, v1.4s\n" - "add v2.4s, v2.4s, v7.4s\n" - "add v26.4s, v26.4s, v7.4s\n" - "add v21.4s, v21.4s, v7.4s\n" - "add v16.4s, v16.4s, v7.4s\n" - "smax v2.4s, v2.4s, v9.4s\n" - "smax v26.4s, v26.4s, v9.4s\n" - "smax v21.4s, v21.4s, v9.4s\n" - "smax v16.4s, v16.4s, v9.4s\n" - "smin v2.4s, v2.4s, v8.4s\n" - "smin v26.4s, v26.4s, v8.4s\n" - "smin v21.4s, v21.4s, v8.4s\n" - "smin v16.4s, v16.4s, v8.4s\n" - "uzp1 v2.16b, v2.16b, v2.16b\n" + ".inst 0x4e80961f // sdot v31.4s, v16.16b, v0.16b\n" + "ext v0.16b, v0.16b, v0.16b, #0x1\n" + ".inst 0x4e9596da // sdot v26.4s, v22.16b, v21.16b\n" + ".inst 0x4e8096c4 // sdot v4.4s, v22.16b, v0.16b\n" + ".inst 0x4e829612 // sdot v18.4s, v16.16b, v2.16b\n" + "ext v2.16b, v2.16b, v2.16b, #0x1\n" + "sqrdmulh v31.4s, v31.4s, v17.4s\n" + ".inst 0x4e80961a // sdot v26.4s, v16.16b, v0.16b\n" + ".inst 0x4e829604 // sdot v4.4s, v16.16b, v2.16b\n" + "and v16.16b, v31.16b, v20.16b\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqrdmulh v26.4s, v26.4s, v17.4s\n" + "sqrdmulh v18.4s, v18.4s, v17.4s\n" + "sqrdmulh v4.4s, v4.4s, v17.4s\n" + "sqadd v31.4s, v31.4s, v16.4s\n" + "and v19.16b, v26.16b, v20.16b\n" + "and v17.16b, v18.16b, v20.16b\n" + "and v16.16b, v4.16b, v20.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v26.4s, v26.4s, v19.4s\n" + "sqadd v18.4s, v18.4s, v17.4s\n" + "sqadd v4.4s, v4.4s, v16.4s\n" + "srshl v31.4s, v31.4s, v20.4s\n" + "srshl v26.4s, v26.4s, v20.4s\n" + "srshl v18.4s, v18.4s, v20.4s\n" + "srshl v4.4s, v4.4s, v20.4s\n" + "add v31.4s, v31.4s, v15.4s\n" + "add v26.4s, v26.4s, v15.4s\n" + "add v18.4s, v18.4s, v15.4s\n" + "add v4.4s, v4.4s, v15.4s\n" + "smax v31.4s, v31.4s, v8.4s\n" + "smax v26.4s, v26.4s, v8.4s\n" + "smax v18.4s, v18.4s, v8.4s\n" + "smax v4.4s, v4.4s, v8.4s\n" + "smin v31.4s, v31.4s, v12.4s\n" + "smin v26.4s, v26.4s, v12.4s\n" + "smin v18.4s, v18.4s, v12.4s\n" + "smin v4.4s, v4.4s, v12.4s\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "uzp1 v21.16b, v21.16b, v21.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "uzp1 v2.16b, v2.16b, v2.16b\n" + "uzp1 v18.16b, v18.16b, v18.16b\n" + "uzp1 v4.16b, v4.16b, v4.16b\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "uzp1 v21.16b, v21.16b, v21.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" + "uzp1 v18.16b, v18.16b, v18.16b\n" + "uzp1 v4.16b, v4.16b, v4.16b\n" "blt 28f\n" - "str s2, [x24, x11]\n" - "str s26, [x23, x11]\n" - "str s21, [x22, x11]\n" - "str s16, [x21, x11]\n" + "str s31, [x25, x11]\n" + "str s26, [x24, x11]\n" + "str s18, [x23, x11]\n" + "str s4, [x22, x11]\n" "b 31f\n" "28:" // Oddments: Unroll 2: Oddment store + "add x25, x25, x11\n" "add x24, x24, x11\n" "add x23, x23, x11\n" "add x22, x22, x11\n" - "add x21, x21, x11\n" "tbz x20, #1, 29f\n" - "st1 { v2.h }[0], [x24], #0x2\n" - "st1 { v26.h }[0], [x23], #0x2\n" - "st1 { v21.h }[0], [x22], #0x2\n" - "st1 { v16.h }[0], [x21], #0x2\n" + "st1 { v31.h }[0], [x25], #0x2\n" + "st1 { v26.h }[0], [x24], #0x2\n" + "st1 { v18.h }[0], [x23], #0x2\n" + "st1 { v4.h }[0], [x22], #0x2\n" "tbz x20, #0, 30f\n" - "st1 { v2.b }[2], [x24], #0x1\n" - "st1 { v26.b }[2], [x23], #0x1\n" - "st1 { v21.b }[2], [x22], #0x1\n" - "st1 { v16.b }[2], [x21], #0x1\n" + "st1 { v31.b }[2], [x25], #0x1\n" + "st1 { v26.b }[2], [x24], #0x1\n" + "st1 { v18.b }[2], [x23], #0x1\n" + "st1 { v4.b }[2], [x22], #0x1\n" "b 30f\n" "29:" // Oddments: Unroll 2: Oddment store: Bit 1: Unset - "st1 { v2.b }[0], [x24], #0x1\n" - "st1 { v26.b }[0], [x23], #0x1\n" - "st1 { v21.b }[0], [x22], #0x1\n" - "st1 { v16.b }[0], [x21], #0x1\n" + "st1 { v31.b }[0], [x25], #0x1\n" + "st1 { v26.b }[0], [x24], #0x1\n" + "st1 { v18.b }[0], [x23], #0x1\n" + "st1 { v4.b }[0], [x22], #0x1\n" "30:" // Oddments: Unroll 2: Oddment store: Bit 1: End - "31:" // Oddments: Unroll 2: After oddment store "subs x20, x20, #0x4\n" "add x11, x11, #0x4\n" "ble 35f\n" - "ldr q2, [%x[params], #0x0]\n" - "ldr q29, [%x[params], #0x10]\n" - "mov v26.16b, v2.16b\n" - "mov v21.16b, v2.16b\n" - "ldr q28, [%x[params], #0x20]\n" - "ldr q27, [%x[params], #0x30]\n" - "mov v16.16b, v2.16b\n" - ".inst 0x4e8397a2 // sdot v2.4s, v29.16b, v3.16b\n" - "ldr q6, [%x[params], #0x40]\n" - "ldr q1, [%x[params], #0x50]\n" - ".inst 0x4e9e97b5 // sdot v21.4s, v29.16b, v30.16b\n" - ".inst 0x4e9e9782 // sdot v2.4s, v28.16b, v30.16b\n" - "ext v3.16b, v3.16b, v3.16b, #0x1\n" - "ext v30.16b, v30.16b, v30.16b, #0x1\n" - ".inst 0x4e8397ba // sdot v26.4s, v29.16b, v3.16b\n" + "ldr q31, [%x[params], #0x0]\n" + "ldr q20, [%x[params], #0x10]\n" + "mov v26.16b, v31.16b\n" + "mov v18.16b, v31.16b\n" + "ldr q19, [%x[params], #0x20]\n" + "ldr q16, [%x[params], #0x30]\n" + "mov v4.16b, v31.16b\n" + ".inst 0x4e8e969f // sdot v31.4s, v20.16b, v14.16b\n" + "ldr q17, [%x[params], #0x40]\n" + "ldr q22, [%x[params], #0x50]\n" + ".inst 0x4e9b9692 // sdot v18.4s, v20.16b, v27.16b\n" + ".inst 0x4e9b967f // sdot v31.4s, v19.16b, v27.16b\n" + "ext v14.16b, v14.16b, v14.16b, #0x1\n" + "ext v27.16b, v27.16b, v27.16b, #0x1\n" + ".inst 0x4e8e969a // sdot v26.4s, v20.16b, v14.16b\n" "add %x[params], %x[params], #0x60\n" - ".inst 0x4e9e97b0 // sdot v16.4s, v29.16b, v30.16b\n" - ".inst 0x4e969795 // sdot v21.4s, v28.16b, v22.16b\n" - ".inst 0x4e969762 // sdot v2.4s, v27.16b, v22.16b\n" - "ext v22.16b, v22.16b, v22.16b, #0x1\n" - ".inst 0x4e9e979a // sdot v26.4s, v28.16b, v30.16b\n" - ".inst 0x4e969790 // sdot v16.4s, v28.16b, v22.16b\n" - ".inst 0x4e919775 // sdot v21.4s, v27.16b, v17.16b\n" - "ext v17.16b, v17.16b, v17.16b, #0x1\n" - "sqrdmulh v2.4s, v2.4s, v6.4s\n" - ".inst 0x4e96977a // sdot v26.4s, v27.16b, v22.16b\n" - ".inst 0x4e919770 // sdot v16.4s, v27.16b, v17.16b\n" - "and v29.16b, v2.16b, v1.16b\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "sqrdmulh v26.4s, v26.4s, v6.4s\n" - "sqrdmulh v21.4s, v21.4s, v6.4s\n" - "sqrdmulh v16.4s, v16.4s, v6.4s\n" - "sqadd v2.4s, v2.4s, v29.4s\n" - "and v28.16b, v26.16b, v1.16b\n" - "and v27.16b, v21.16b, v1.16b\n" - "and v29.16b, v16.16b, v1.16b\n" - "sshr v28.4s, v28.4s, #0x1f\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "sqadd v26.4s, v26.4s, v28.4s\n" - "sqadd v21.4s, v21.4s, v27.4s\n" - "sqadd v16.4s, v16.4s, v29.4s\n" - "srshl v2.4s, v2.4s, v1.4s\n" - "srshl v26.4s, v26.4s, v1.4s\n" - "srshl v21.4s, v21.4s, v1.4s\n" - "srshl v16.4s, v16.4s, v1.4s\n" - "add v2.4s, v2.4s, v7.4s\n" - "add v26.4s, v26.4s, v7.4s\n" - "add v21.4s, v21.4s, v7.4s\n" - "add v16.4s, v16.4s, v7.4s\n" - "smax v2.4s, v2.4s, v9.4s\n" - "smax v26.4s, v26.4s, v9.4s\n" - "smax v21.4s, v21.4s, v9.4s\n" - "smax v16.4s, v16.4s, v9.4s\n" - "smin v2.4s, v2.4s, v8.4s\n" - "smin v26.4s, v26.4s, v8.4s\n" - "smin v21.4s, v21.4s, v8.4s\n" - "smin v16.4s, v16.4s, v8.4s\n" - "uzp1 v2.16b, v2.16b, v2.16b\n" + ".inst 0x4e9b9684 // sdot v4.4s, v20.16b, v27.16b\n" + ".inst 0x4e879672 // sdot v18.4s, v19.16b, v7.16b\n" + ".inst 0x4e87961f // sdot v31.4s, v16.16b, v7.16b\n" + "ext v7.16b, v7.16b, v7.16b, #0x1\n" + ".inst 0x4e9b967a // sdot v26.4s, v19.16b, v27.16b\n" + ".inst 0x4e879664 // sdot v4.4s, v19.16b, v7.16b\n" + ".inst 0x4e819612 // sdot v18.4s, v16.16b, v1.16b\n" + "ext v1.16b, v1.16b, v1.16b, #0x1\n" + "sqrdmulh v31.4s, v31.4s, v17.4s\n" + ".inst 0x4e87961a // sdot v26.4s, v16.16b, v7.16b\n" + ".inst 0x4e819604 // sdot v4.4s, v16.16b, v1.16b\n" + "and v16.16b, v31.16b, v22.16b\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqrdmulh v26.4s, v26.4s, v17.4s\n" + "sqrdmulh v18.4s, v18.4s, v17.4s\n" + "sqrdmulh v4.4s, v4.4s, v17.4s\n" + "sqadd v31.4s, v31.4s, v16.4s\n" + "and v23.16b, v26.16b, v22.16b\n" + "and v17.16b, v18.16b, v22.16b\n" + "and v16.16b, v4.16b, v22.16b\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v26.4s, v26.4s, v23.4s\n" + "sqadd v18.4s, v18.4s, v17.4s\n" + "sqadd v4.4s, v4.4s, v16.4s\n" + "srshl v31.4s, v31.4s, v22.4s\n" + "srshl v26.4s, v26.4s, v22.4s\n" + "srshl v18.4s, v18.4s, v22.4s\n" + "srshl v4.4s, v4.4s, v22.4s\n" + "add v31.4s, v31.4s, v15.4s\n" + "add v26.4s, v26.4s, v15.4s\n" + "add v18.4s, v18.4s, v15.4s\n" + "add v4.4s, v4.4s, v15.4s\n" + "smax v31.4s, v31.4s, v8.4s\n" + "smax v26.4s, v26.4s, v8.4s\n" + "smax v18.4s, v18.4s, v8.4s\n" + "smax v4.4s, v4.4s, v8.4s\n" + "smin v31.4s, v31.4s, v12.4s\n" + "smin v26.4s, v26.4s, v12.4s\n" + "smin v18.4s, v18.4s, v12.4s\n" + "smin v4.4s, v4.4s, v12.4s\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "uzp1 v21.16b, v21.16b, v21.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "uzp1 v2.16b, v2.16b, v2.16b\n" + "uzp1 v18.16b, v18.16b, v18.16b\n" + "uzp1 v4.16b, v4.16b, v4.16b\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "uzp1 v21.16b, v21.16b, v21.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" + "uzp1 v18.16b, v18.16b, v18.16b\n" + "uzp1 v4.16b, v4.16b, v4.16b\n" "32:" // Oddments: Unroll 3: Oddment store + "add x25, x25, x11\n" "add x24, x24, x11\n" "add x23, x23, x11\n" "add x22, x22, x11\n" - "add x21, x21, x11\n" "tbz x20, #1, 33f\n" - "st1 { v2.h }[0], [x24], #0x2\n" - "st1 { v26.h }[0], [x23], #0x2\n" - "st1 { v21.h }[0], [x22], #0x2\n" - "st1 { v16.h }[0], [x21], #0x2\n" + "st1 { v31.h }[0], [x25], #0x2\n" + "st1 { v26.h }[0], [x24], #0x2\n" + "st1 { v18.h }[0], [x23], #0x2\n" + "st1 { v4.h }[0], [x22], #0x2\n" "tbz x20, #0, 34f\n" - "st1 { v2.b }[2], [x24], #0x1\n" - "st1 { v26.b }[2], [x23], #0x1\n" - "st1 { v21.b }[2], [x22], #0x1\n" - "st1 { v16.b }[2], [x21], #0x1\n" + "st1 { v31.b }[2], [x25], #0x1\n" + "st1 { v26.b }[2], [x24], #0x1\n" + "st1 { v18.b }[2], [x23], #0x1\n" + "st1 { v4.b }[2], [x22], #0x1\n" "b 34f\n" "33:" // Oddments: Unroll 3: Oddment store: Bit 1: Unset - "st1 { v2.b }[0], [x24], #0x1\n" - "st1 { v26.b }[0], [x23], #0x1\n" - "st1 { v21.b }[0], [x22], #0x1\n" - "st1 { v16.b }[0], [x21], #0x1\n" + "st1 { v31.b }[0], [x25], #0x1\n" + "st1 { v26.b }[0], [x24], #0x1\n" + "st1 { v18.b }[0], [x23], #0x1\n" + "st1 { v4.b }[0], [x22], #0x1\n" "34:" // Oddments: Unroll 3: Oddment store: Bit 1: End - "35:" // End : [params] "+&r" (params) : [inptrs] "r" (inptrs), [n_channels] "r" (n_channels), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [outptrs] "r" (outptrs), [qp] "r" (&qp) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp index 8366b0a270452ecfeb2a1556ce643d57cf53d70c..ed24f8fa3c9589bf310aa457236f843ffef4aed9 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include @@ -34,7 +34,7 @@ namespace arm_conv { namespace depthwise { -void a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst_impl(const unsigned int, const uint8_t *const * const, const uint8_t *, const int32_t *, const arm_gemm::Requantize32&, const int32_t *, const int32_t *, uint8_t *const *const); +void a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst_impl(unsigned int, const uint8_t *const *, const uint8_t *, const int32_t *, const arm_gemm::Requantize32&, const int32_t *, const int32_t *, uint8_t *const *); class a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst : public DepthwiseDepthfirstStrategy { @@ -64,7 +64,7 @@ class a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst : public DepthwiseDepthfirstS ) const override { interleave_a64_u8q_3x3_dot::pack_parameters( - args.input_channels, buffer, reinterpret_cast(biases), + args.input_channels * args.channel_multiplier, buffer, reinterpret_cast(biases), reinterpret_cast(weights), qp, ld_weight_col, ld_weight_row ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp index 986937f3b47f8f2b9f0ed49ea1223e1d1f4586ca..5a28daffbf5890794f610a80b3c80e77b1bd07dc 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp @@ -30,15 +30,7 @@ namespace arm_conv { namespace depthwise { -void a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst_impl( - const unsigned int n_channels, - const uint8_t *const *const inptrs, - const uint8_t *params, - const int32_t *, // Bias, should be wrapped into the parameters - const arm_gemm::Requantize32& qp, - const int32_t *, const int32_t *, // Requant parameters, also wrapped - uint8_t *const *const outptrs -) +void a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst_impl(const unsigned int n_channels, const uint8_t *const *const inptrs, const uint8_t *params, const int32_t *, const arm_gemm::Requantize32& qp, const int32_t *, const int32_t *, uint8_t *const *const outptrs) { __asm__ __volatile__( "mov x20, #0x1\n" @@ -47,817 +39,817 @@ void a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst_impl( "ldp x13, x12, [%x[inptrs], #0x10]\n" "orr x20, x20, #0x10000\n" "lsr x11, %x[n_channels], #0x4\n" - "dup v14.4s, w20\n" + "dup v12.4s, w20\n" "ldp x10, x9, [%x[inptrs], #0x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_minval]\n" "ld1r { v13.4s }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_maxval]\n" - "ld1r { v12.4s }, [x20]\n" - "add x20, %x[qp], %[offsetof_Requantize32_b_offset]\n" "ld1r { v11.4s }, [x20]\n" + "add x20, %x[qp], %[offsetof_Requantize32_b_offset]\n" + "ld1r { v16.4s }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_c_offset]\n" - "ld1r { v10.4s }, [x20]\n" + "ld1r { v14.4s }, [x20]\n" "mov x28, #0x0\n" "mov x27, #0x0\n" - "ldp x26, x25, [%x[inptrs], #0x30]\n" - "ldp x24, x23, [%x[outptrs], #0x0]\n" - "ldp x22, x21, [%x[outptrs], #0x10]\n" + "ldp x26, x21, [%x[inptrs], #0x30]\n" + "ldp x25, x24, [%x[outptrs], #0x0]\n" + "ldp x23, x22, [%x[outptrs], #0x10]\n" "cbz x11, 3f\n" - "ldr q9, [x15, x28]\n" - "ldr q8, [x14, x28]\n" - "subs x11, x11, #0x1\n" - "ldr q7, [x13, x28]\n" - "ldr q6, [x12, x28]\n" - "zip2 v5.16b, v9.16b, v7.16b\n" - "zip1 v9.16b, v9.16b, v7.16b\n" - "ldr q4, [x10, x28]\n" - "ldr q3, [x9, x28]\n" - "zip1 v7.16b, v8.16b, v6.16b\n" - "zip2 v6.16b, v8.16b, v6.16b\n" - "ldr q2, [x26, x28]\n" - "ldr q1, [x25, x28]\n" - "zip2 v8.16b, v9.16b, v7.16b\n" - "zip1 v9.16b, v9.16b, v7.16b\n" - "ldr q0, [%x[params], #0x10]\n" - "ldr q16, [%x[params], #0x20]\n" - "zip1 v7.16b, v5.16b, v6.16b\n" - "zip2 v6.16b, v5.16b, v6.16b\n" - "ldr q5, [%x[params], #0x0]\n" - "ldr q31, [%x[params], #0x30]\n" - "zip2 v30.16b, v4.16b, v2.16b\n" - "zip1 v4.16b, v4.16b, v2.16b\n" - "ldp x15, x14, [%x[inptrs], #0x40]\n" - "ldr q29, [x15, x28]\n" - "zip1 v2.16b, v3.16b, v1.16b\n" - "zip2 v1.16b, v3.16b, v1.16b\n" + "ldr q15, [x15, x28]\n" "ldr q28, [x14, x28]\n" - "ldp x13, x12, [%x[inptrs], #0x50]\n" - "zip2 v3.16b, v4.16b, v2.16b\n" - "zip1 v4.16b, v4.16b, v2.16b\n" - "ldr q27, [x13, x28]\n" - "ldr q26, [x12, x28]\n" - "zip2 v25.16b, v29.16b, v27.16b\n" - "zip1 v29.16b, v29.16b, v27.16b\n" - "ldp x10, x9, [%x[inptrs], #0x60]\n" - "ldr q24, [x10, x28]\n" - "zip1 v27.16b, v28.16b, v26.16b\n" - "zip2 v26.16b, v28.16b, v26.16b\n" - "ldr q23, [x9, x28]\n" - "ldp x26, x25, [%x[inptrs], #0x70]\n" - "zip1 v2.16b, v30.16b, v1.16b\n" - "zip2 v1.16b, v30.16b, v1.16b\n" - "ldr q22, [x26, x28]\n" - "ldr q21, [x25, x28]\n" - "zip2 v20.16b, v24.16b, v22.16b\n" - "zip1 v24.16b, v24.16b, v22.16b\n" - "zip1 v22.16b, v23.16b, v21.16b\n" - "zip2 v21.16b, v23.16b, v21.16b\n" + "subs x11, x11, #0x1\n" + "ldr q30, [x13, x28]\n" + "ldr q8, [x12, x28]\n" + "zip2 v19.16b, v15.16b, v30.16b\n" + "zip1 v15.16b, v15.16b, v30.16b\n" + "ldr q26, [x10, x28]\n" + "ldr q0, [x9, x28]\n" + "zip1 v7.16b, v28.16b, v8.16b\n" + "zip2 v8.16b, v28.16b, v8.16b\n" + "ldr q29, [x26, x28]\n" + "ldr q10, [x21, x28]\n" + "zip2 v25.16b, v15.16b, v7.16b\n" + "zip1 v15.16b, v15.16b, v7.16b\n" + "ldr q1, [%x[params], #0x10]\n" + "ldr q6, [%x[params], #0x20]\n" + "zip1 v7.16b, v19.16b, v8.16b\n" + "zip2 v8.16b, v19.16b, v8.16b\n" + "ldr q31, [%x[params], #0x0]\n" + "ldr q20, [%x[params], #0x30]\n" + "zip2 v21.16b, v26.16b, v29.16b\n" + "zip1 v26.16b, v26.16b, v29.16b\n" + "ldp x21, x20, [%x[inptrs], #0x40]\n" + "ldr q22, [x21, x28]\n" + "zip1 v27.16b, v0.16b, v10.16b\n" + "zip2 v10.16b, v0.16b, v10.16b\n" + "ldr q17, [x20, x28]\n" + "ldp x21, x20, [%x[inptrs], #0x50]\n" + "zip2 v23.16b, v26.16b, v27.16b\n" + "zip1 v26.16b, v26.16b, v27.16b\n" + "ldr q9, [x21, x28]\n" + "ldr q5, [x20, x28]\n" + "zip2 v28.16b, v22.16b, v9.16b\n" + "zip1 v22.16b, v22.16b, v9.16b\n" + "ldp x21, x20, [%x[inptrs], #0x60]\n" + "ldr q27, [x21, x28]\n" + "zip1 v24.16b, v17.16b, v5.16b\n" + "zip2 v5.16b, v17.16b, v5.16b\n" + "ldr q18, [x20, x28]\n" + "ldp x21, x20, [%x[inptrs], #0x70]\n" + "zip1 v3.16b, v21.16b, v10.16b\n" + "zip2 v10.16b, v21.16b, v10.16b\n" + "ldr q4, [x21, x28]\n" + "ldr q9, [x20, x28]\n" + "zip2 v17.16b, v27.16b, v4.16b\n" + "zip1 v27.16b, v27.16b, v4.16b\n" + "zip1 v4.16b, v18.16b, v9.16b\n" + "zip2 v9.16b, v18.16b, v9.16b\n" "ldp x15, x14, [%x[inptrs], #0x0]\n" "ldp x13, x12, [%x[inptrs], #0x10]\n" "ldp x10, x9, [%x[inptrs], #0x20]\n" - "ldp x26, x25, [%x[inptrs], #0x30]\n" - "zip2 v28.16b, v29.16b, v27.16b\n" - "zip1 v29.16b, v29.16b, v27.16b\n" - "zip1 v27.16b, v25.16b, v26.16b\n" - "zip2 v26.16b, v25.16b, v26.16b\n" + "ldp x26, x21, [%x[inptrs], #0x30]\n" + "zip2 v19.16b, v22.16b, v24.16b\n" + "zip1 v22.16b, v22.16b, v24.16b\n" + "zip1 v0.16b, v28.16b, v5.16b\n" + "zip2 v5.16b, v28.16b, v5.16b\n" "add %x[params], %x[params], #0x40\n" - "zip2 v23.16b, v24.16b, v22.16b\n" - "zip1 v24.16b, v24.16b, v22.16b\n" - "zip1 v22.16b, v20.16b, v21.16b\n" - "zip2 v21.16b, v20.16b, v21.16b\n" - "mov v30.16b, v5.16b\n" - "mov v25.16b, v5.16b\n" - "mov v20.16b, v5.16b\n" + "zip2 v24.16b, v27.16b, v4.16b\n" + "zip1 v27.16b, v27.16b, v4.16b\n" + "zip1 v2.16b, v17.16b, v9.16b\n" + "zip2 v9.16b, v17.16b, v9.16b\n" + "mov v30.16b, v31.16b\n" + "mov v29.16b, v31.16b\n" + "mov v28.16b, v31.16b\n" "beq 2f\n" "1:" // Loop - "movi v19.4s, #0x0\n" - ".inst 0x6e8495d3 // udot v19.4s, v14.16b, v4.16b\n" - ".inst 0x6e899405 // udot v5.4s, v0.16b, v9.16b\n" + "movi v21.4s, #0x0\n" + ".inst 0x6e9a9595 // udot v21.4s, v12.16b, v26.16b\n" + ".inst 0x6e8f943f // udot v31.4s, v1.16b, v15.16b\n" "add x28, x28, #0x10\n" - ".inst 0x6e9d95d3 // udot v19.4s, v14.16b, v29.16b\n" - ".inst 0x6e849419 // udot v25.4s, v0.16b, v4.16b\n" + ".inst 0x6e969595 // udot v21.4s, v12.16b, v22.16b\n" + ".inst 0x6e9a943d // udot v29.4s, v1.16b, v26.16b\n" + "movi v18.4s, #0x0\n" "subs x11, x11, #0x1\n" - ".inst 0x6e849605 // udot v5.4s, v16.16b, v4.16b\n" - "ext v4.16b, v4.16b, v4.16b, #0x1\n" - "mov v18.16b, v19.16b\n .inst 0x6e9895d2 // udot v18.4s, v14.16b, v24.16b\n" - ".inst 0x6e8995d3 // udot v19.4s, v14.16b, v9.16b\n" - "ext v9.16b, v9.16b, v9.16b, #0x1\n" - ".inst 0x6e9d9619 // udot v25.4s, v16.16b, v29.16b\n" - ".inst 0x6e9d97e5 // udot v5.4s, v31.16b, v29.16b\n" - "ext v29.16b, v29.16b, v29.16b, #0x1\n" - ".inst 0x6e89941e // udot v30.4s, v0.16b, v9.16b\n" - ".inst 0x6e849414 // udot v20.4s, v0.16b, v4.16b\n" - "movi v17.4s, #0x0\n" - ".inst 0x6e8495d1 // udot v17.4s, v14.16b, v4.16b\n" - ".inst 0x6e9d95d1 // udot v17.4s, v14.16b, v29.16b\n" - ".inst 0x6e9897f9 // udot v25.4s, v31.16b, v24.16b\n" - "ext v24.16b, v24.16b, v24.16b, #0x1\n" - ".inst 0x6e84961e // udot v30.4s, v16.16b, v4.16b\n" - "ldr q4, [%x[params], #0x10]\n" - ".inst 0x6e9d9614 // udot v20.4s, v16.16b, v29.16b\n" - "mls v5.4s, v19.4s, v11.4s\n" - "mov v16.16b, v17.16b\n .inst 0x6e9895d0 // udot v16.4s, v14.16b, v24.16b\n" - ".inst 0x6e8995d1 // udot v17.4s, v14.16b, v9.16b\n" - "ldr q9, [%x[params], #0x0]\n" - "sqrdmulh v5.4s, v5.4s, v9.4s\n" - ".inst 0x6e9d97fe // udot v30.4s, v31.16b, v29.16b\n" - ".inst 0x6e9897f4 // udot v20.4s, v31.16b, v24.16b\n" - "mls v30.4s, v17.4s, v11.4s\n" - "mls v25.4s, v18.4s, v11.4s\n" - "mls v20.4s, v16.4s, v11.4s\n" - "and v0.16b, v5.16b, v4.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqrdmulh v30.4s, v30.4s, v9.4s\n" - "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqrdmulh v20.4s, v20.4s, v9.4s\n" - "ldr q9, [%x[params], #0x60]\n" - "sqadd v5.4s, v5.4s, v0.4s\n" - "and v16.16b, v30.16b, v4.16b\n" - "and v31.16b, v25.16b, v4.16b\n" - "and v0.16b, v20.16b, v4.16b\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "srshl v5.4s, v5.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v16.4s\n" - "ldr q16, [%x[params], #0x40]\n" - "sqadd v25.4s, v25.4s, v31.4s\n" - "ldr q31, [%x[params], #0x50]\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "ldr q0, [%x[params], #0x30]\n" - "add v5.4s, v5.4s, v10.4s\n" - "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" - "srshl v20.4s, v20.4s, v4.4s\n" - "ldr q4, [%x[params], #0x70]\n" - "smax v5.4s, v5.4s, v13.4s\n" - "add v30.4s, v30.4s, v10.4s\n" - "add v25.4s, v25.4s, v10.4s\n" - "add v20.4s, v20.4s, v10.4s\n" - "smin v5.4s, v5.4s, v12.4s\n" + ".inst 0x6e9a94df // udot v31.4s, v6.16b, v26.16b\n" + "ext v26.16b, v26.16b, v26.16b, #0x1\n" + "mov v17.16b, v21.16b\n .inst 0x6e9b9591 // udot v17.4s, v12.16b, v27.16b\n" + ".inst 0x6e8f9595 // udot v21.4s, v12.16b, v15.16b\n" + "ext v15.16b, v15.16b, v15.16b, #0x1\n" + ".inst 0x6e9a9592 // udot v18.4s, v12.16b, v26.16b\n" + ".inst 0x6e9694dd // udot v29.4s, v6.16b, v22.16b\n" + ".inst 0x6e96969f // udot v31.4s, v20.16b, v22.16b\n" + "ext v22.16b, v22.16b, v22.16b, #0x1\n" + ".inst 0x6e8f943e // udot v30.4s, v1.16b, v15.16b\n" + ".inst 0x6e9a943c // udot v28.4s, v1.16b, v26.16b\n" + "mls v31.4s, v21.4s, v16.4s\n" + ".inst 0x6e969592 // udot v18.4s, v12.16b, v22.16b\n" + ".inst 0x6e9b969d // udot v29.4s, v20.16b, v27.16b\n" + "ext v27.16b, v27.16b, v27.16b, #0x1\n" + ".inst 0x6e9a94de // udot v30.4s, v6.16b, v26.16b\n" + "ldr q26, [%x[params], #0x10]\n" + ".inst 0x6e9694dc // udot v28.4s, v6.16b, v22.16b\n" + "mls v29.4s, v17.4s, v16.4s\n" + "mov v21.16b, v18.16b\n .inst 0x6e9b9595 // udot v21.4s, v12.16b, v27.16b\n" + ".inst 0x6e8f9592 // udot v18.4s, v12.16b, v15.16b\n" + "ldr q17, [%x[params], #0x0]\n" + "sqrdmulh v31.4s, v31.4s, v17.4s\n" + ".inst 0x6e96969e // udot v30.4s, v20.16b, v22.16b\n" + ".inst 0x6e9b969c // udot v28.4s, v20.16b, v27.16b\n" + "mls v30.4s, v18.4s, v16.4s\n" + "mls v28.4s, v21.4s, v16.4s\n" + "and v15.16b, v31.16b, v26.16b\n" + "sshr v15.4s, v15.4s, #0x1f\n" + "sqrdmulh v30.4s, v30.4s, v17.4s\n" + "sqrdmulh v29.4s, v29.4s, v17.4s\n" + "sqrdmulh v28.4s, v28.4s, v17.4s\n" + "ldr q1, [%x[params], #0x60]\n" + "sqadd v31.4s, v31.4s, v15.4s\n" + "and v18.16b, v30.16b, v26.16b\n" + "and v21.16b, v29.16b, v26.16b\n" + "and v17.16b, v28.16b, v26.16b\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v21.4s, v21.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "srshl v31.4s, v31.4s, v26.4s\n" + "sqadd v30.4s, v30.4s, v18.4s\n" + "ldr q18, [%x[params], #0x40]\n" + "sqadd v29.4s, v29.4s, v21.4s\n" + "ldr q27, [%x[params], #0x50]\n" + "sqadd v28.4s, v28.4s, v17.4s\n" + "ldr q15, [%x[params], #0x30]\n" + "add v31.4s, v31.4s, v14.4s\n" + "srshl v30.4s, v30.4s, v26.4s\n" + "srshl v29.4s, v29.4s, v26.4s\n" + "srshl v28.4s, v28.4s, v26.4s\n" + "ldr q20, [%x[params], #0x70]\n" + "smax v31.4s, v31.4s, v13.4s\n" + "add v30.4s, v30.4s, v14.4s\n" + "add v29.4s, v29.4s, v14.4s\n" + "add v28.4s, v28.4s, v14.4s\n" + "smin v31.4s, v31.4s, v11.4s\n" "smax v30.4s, v30.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smin v30.4s, v30.4s, v12.4s\n" - "smin v25.4s, v25.4s, v12.4s\n" - "smin v20.4s, v20.4s, v12.4s\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "movi v19.4s, #0x0\n" - ".inst 0x6e8395d3 // udot v19.4s, v14.16b, v3.16b\n" - ".inst 0x6e9c95d3 // udot v19.4s, v14.16b, v28.16b\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" + "smax v29.4s, v29.4s, v13.4s\n" + "smax v28.4s, v28.4s, v13.4s\n" + "smin v30.4s, v30.4s, v11.4s\n" + "smin v29.4s, v29.4s, v11.4s\n" + "smin v28.4s, v28.4s, v11.4s\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" + "movi v22.4s, #0x0\n" + ".inst 0x6e979596 // udot v22.4s, v12.16b, v23.16b\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s5, [x24, x27]\n" - "ldr q5, [%x[params], #0x20]\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "mov v18.16b, v19.16b\n .inst 0x6e9795d2 // udot v18.4s, v14.16b, v23.16b\n" + "str s31, [x25, x27]\n" + "ldr q26, [%x[params], #0x20]\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + ".inst 0x6e939596 // udot v22.4s, v12.16b, v19.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "str s30, [x23, x27]\n" - ".inst 0x6e8895d3 // udot v19.4s, v14.16b, v8.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s25, [x22, x27]\n" - "mov v30.16b, v5.16b\n" - "str s20, [x21, x27]\n" - "mov v25.16b, v5.16b\n" - "mov v20.16b, v5.16b\n" - ".inst 0x6e889405 // udot v5.4s, v0.16b, v8.16b\n" - ".inst 0x6e839419 // udot v25.4s, v0.16b, v3.16b\n" - ".inst 0x6e839605 // udot v5.4s, v16.16b, v3.16b\n" - "ext v8.16b, v8.16b, v8.16b, #0x1\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "str s30, [x24, x27]\n" + "mov v6.16b, v22.16b\n .inst 0x6e989586 // udot v6.4s, v12.16b, v24.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + "str s29, [x23, x27]\n" + "mov v30.16b, v26.16b\n" + ".inst 0x6e999596 // udot v22.4s, v12.16b, v25.16b\n" + "str s28, [x22, x27]\n" + "mov v29.16b, v26.16b\n" + "mov v21.16b, v26.16b\n" + ".inst 0x6e9995fa // udot v26.4s, v15.16b, v25.16b\n" + ".inst 0x6e9795fd // udot v29.4s, v15.16b, v23.16b\n" + ".inst 0x6e97965a // udot v26.4s, v18.16b, v23.16b\n" + "ext v25.16b, v25.16b, v25.16b, #0x1\n" "add x27, x27, #0x4\n" - "ext v3.16b, v3.16b, v3.16b, #0x1\n" - "movi v17.4s, #0x0\n" - ".inst 0x6e88941e // udot v30.4s, v0.16b, v8.16b\n" - ".inst 0x6e839414 // udot v20.4s, v0.16b, v3.16b\n" - ".inst 0x6e8395d1 // udot v17.4s, v14.16b, v3.16b\n" - ".inst 0x6e9c9619 // udot v25.4s, v16.16b, v28.16b\n" - ".inst 0x6e9c97e5 // udot v5.4s, v31.16b, v28.16b\n" - "ext v28.16b, v28.16b, v28.16b, #0x1\n" - ".inst 0x6e83961e // udot v30.4s, v16.16b, v3.16b\n" - "ldr q3, [x9, x28]\n" - ".inst 0x6e9c9614 // udot v20.4s, v16.16b, v28.16b\n" - "mls v5.4s, v19.4s, v11.4s\n" - ".inst 0x6e9c95d1 // udot v17.4s, v14.16b, v28.16b\n" - ".inst 0x6e9797f9 // udot v25.4s, v31.16b, v23.16b\n" "ext v23.16b, v23.16b, v23.16b, #0x1\n" - ".inst 0x6e9c97fe // udot v30.4s, v31.16b, v28.16b\n" - ".inst 0x6e9797f4 // udot v20.4s, v31.16b, v23.16b\n" - "sqrdmulh v5.4s, v5.4s, v9.4s\n" - "mov v16.16b, v17.16b\n .inst 0x6e9795d0 // udot v16.4s, v14.16b, v23.16b\n" - ".inst 0x6e8895d1 // udot v17.4s, v14.16b, v8.16b\n" - "ldr q8, [x14, x28]\n" - "mls v30.4s, v17.4s, v11.4s\n" - "mls v25.4s, v18.4s, v11.4s\n" - "mls v20.4s, v16.4s, v11.4s\n" - "and v0.16b, v5.16b, v4.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqrdmulh v30.4s, v30.4s, v9.4s\n" - "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqrdmulh v20.4s, v20.4s, v9.4s\n" - "ldr q9, [%x[params], #0xc0]\n" - "sqadd v5.4s, v5.4s, v0.4s\n" - "and v16.16b, v30.16b, v4.16b\n" - "and v31.16b, v25.16b, v4.16b\n" - "and v0.16b, v20.16b, v4.16b\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "srshl v5.4s, v5.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v16.4s\n" - "ldr q16, [%x[params], #0xa0]\n" - "sqadd v25.4s, v25.4s, v31.4s\n" - "ldr q31, [%x[params], #0xb0]\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "ldr q0, [%x[params], #0x90]\n" - "add v5.4s, v5.4s, v10.4s\n" - "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" - "srshl v20.4s, v20.4s, v4.4s\n" - "ldr q4, [%x[params], #0xd0]\n" - "smax v5.4s, v5.4s, v13.4s\n" - "add v30.4s, v30.4s, v10.4s\n" - "add v25.4s, v25.4s, v10.4s\n" - "add v20.4s, v20.4s, v10.4s\n" - "smin v5.4s, v5.4s, v12.4s\n" + "movi v28.4s, #0x0\n" + ".inst 0x6e9995fe // udot v30.4s, v15.16b, v25.16b\n" + ".inst 0x6e9795f5 // udot v21.4s, v15.16b, v23.16b\n" + ".inst 0x6e97959c // udot v28.4s, v12.16b, v23.16b\n" + ".inst 0x6e93965d // udot v29.4s, v18.16b, v19.16b\n" + ".inst 0x6e93977a // udot v26.4s, v27.16b, v19.16b\n" + "ext v19.16b, v19.16b, v19.16b, #0x1\n" + ".inst 0x6e97965e // udot v30.4s, v18.16b, v23.16b\n" + "ldr q4, [x9, x28]\n" + ".inst 0x6e939655 // udot v21.4s, v18.16b, v19.16b\n" + "mls v26.4s, v22.4s, v16.4s\n" + ".inst 0x6e93959c // udot v28.4s, v12.16b, v19.16b\n" + ".inst 0x6e98977d // udot v29.4s, v27.16b, v24.16b\n" + "ext v24.16b, v24.16b, v24.16b, #0x1\n" + ".inst 0x6e93977e // udot v30.4s, v27.16b, v19.16b\n" + ".inst 0x6e989775 // udot v21.4s, v27.16b, v24.16b\n" + "sqrdmulh v26.4s, v26.4s, v1.4s\n" + "mov v17.16b, v28.16b\n .inst 0x6e989591 // udot v17.4s, v12.16b, v24.16b\n" + ".inst 0x6e99959c // udot v28.4s, v12.16b, v25.16b\n" + "ldr q31, [x14, x28]\n" + "mls v30.4s, v28.4s, v16.4s\n" + "mls v29.4s, v6.4s, v16.4s\n" + "mls v21.4s, v17.4s, v16.4s\n" + "and v17.16b, v26.16b, v20.16b\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqrdmulh v30.4s, v30.4s, v1.4s\n" + "sqrdmulh v29.4s, v29.4s, v1.4s\n" + "sqrdmulh v21.4s, v21.4s, v1.4s\n" + "ldr q27, [%x[params], #0xc0]\n" + "sqadd v26.4s, v26.4s, v17.4s\n" + "and v18.16b, v30.16b, v20.16b\n" + "and v6.16b, v29.16b, v20.16b\n" + "and v17.16b, v21.16b, v20.16b\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v6.4s, v6.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "srshl v26.4s, v26.4s, v20.4s\n" + "sqadd v30.4s, v30.4s, v18.4s\n" + "ldr q28, [%x[params], #0xa0]\n" + "sqadd v29.4s, v29.4s, v6.4s\n" + "ldr q24, [%x[params], #0xb0]\n" + "sqadd v21.4s, v21.4s, v17.4s\n" + "ldr q15, [%x[params], #0x90]\n" + "add v26.4s, v26.4s, v14.4s\n" + "srshl v30.4s, v30.4s, v20.4s\n" + "srshl v29.4s, v29.4s, v20.4s\n" + "srshl v21.4s, v21.4s, v20.4s\n" + "ldr q1, [%x[params], #0xd0]\n" + "smax v26.4s, v26.4s, v13.4s\n" + "add v30.4s, v30.4s, v14.4s\n" + "add v29.4s, v29.4s, v14.4s\n" + "add v21.4s, v21.4s, v14.4s\n" + "smin v26.4s, v26.4s, v11.4s\n" "smax v30.4s, v30.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smin v30.4s, v30.4s, v12.4s\n" - "smin v25.4s, v25.4s, v12.4s\n" - "smin v20.4s, v20.4s, v12.4s\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "movi v19.4s, #0x0\n" - ".inst 0x6e8295d3 // udot v19.4s, v14.16b, v2.16b\n" - ".inst 0x6e9b95d3 // udot v19.4s, v14.16b, v27.16b\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" + "smax v29.4s, v29.4s, v13.4s\n" + "smax v21.4s, v21.4s, v13.4s\n" + "smin v30.4s, v30.4s, v11.4s\n" + "smin v29.4s, v29.4s, v11.4s\n" + "smin v21.4s, v21.4s, v11.4s\n" + "uzp1 v26.16b, v26.16b, v26.16b\n" + "movi v22.4s, #0x0\n" + ".inst 0x6e839596 // udot v22.4s, v12.16b, v3.16b\n" + ".inst 0x6e809596 // udot v22.4s, v12.16b, v0.16b\n" + "uzp1 v26.16b, v26.16b, v26.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s5, [x24, x27]\n" - "ldr q5, [%x[params], #0x80]\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "mov v18.16b, v19.16b\n .inst 0x6e9695d2 // udot v18.4s, v14.16b, v22.16b\n" + "str s26, [x25, x27]\n" + "ldr q26, [%x[params], #0x80]\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "uzp1 v21.16b, v21.16b, v21.16b\n" + "mov v18.16b, v22.16b\n .inst 0x6e829592 // udot v18.4s, v12.16b, v2.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "str s30, [x23, x27]\n" - ".inst 0x6e8795d3 // udot v19.4s, v14.16b, v7.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s25, [x22, x27]\n" - "mov v30.16b, v5.16b\n" - "str s20, [x21, x27]\n" - "mov v25.16b, v5.16b\n" - "mov v20.16b, v5.16b\n" - ".inst 0x6e879405 // udot v5.4s, v0.16b, v7.16b\n" - ".inst 0x6e829419 // udot v25.4s, v0.16b, v2.16b\n" - ".inst 0x6e829605 // udot v5.4s, v16.16b, v2.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "str s30, [x24, x27]\n" + ".inst 0x6e879596 // udot v22.4s, v12.16b, v7.16b\n" + "uzp1 v21.16b, v21.16b, v21.16b\n" + "str s29, [x23, x27]\n" + "mov v6.16b, v26.16b\n" + "str s21, [x22, x27]\n" + "mov v25.16b, v26.16b\n" + "mov v20.16b, v26.16b\n" + ".inst 0x6e8795fa // udot v26.4s, v15.16b, v7.16b\n" + ".inst 0x6e8395f9 // udot v25.4s, v15.16b, v3.16b\n" + ".inst 0x6e83979a // udot v26.4s, v28.16b, v3.16b\n" "ext v7.16b, v7.16b, v7.16b, #0x1\n" "add x27, x27, #0x4\n" + "ext v3.16b, v3.16b, v3.16b, #0x1\n" + "movi v23.4s, #0x0\n" + ".inst 0x6e8795e6 // udot v6.4s, v15.16b, v7.16b\n" + ".inst 0x6e8395f4 // udot v20.4s, v15.16b, v3.16b\n" + ".inst 0x6e839597 // udot v23.4s, v12.16b, v3.16b\n" + ".inst 0x6e809799 // udot v25.4s, v28.16b, v0.16b\n" + ".inst 0x6e80971a // udot v26.4s, v24.16b, v0.16b\n" + "ext v0.16b, v0.16b, v0.16b, #0x1\n" + ".inst 0x6e839786 // udot v6.4s, v28.16b, v3.16b\n" + "ldr q19, [x26, x28]\n" + ".inst 0x6e809794 // udot v20.4s, v28.16b, v0.16b\n" + "mls v26.4s, v22.4s, v16.4s\n" + ".inst 0x6e809597 // udot v23.4s, v12.16b, v0.16b\n" + ".inst 0x6e829719 // udot v25.4s, v24.16b, v2.16b\n" "ext v2.16b, v2.16b, v2.16b, #0x1\n" - "movi v17.4s, #0x0\n" - ".inst 0x6e87941e // udot v30.4s, v0.16b, v7.16b\n" - ".inst 0x6e829414 // udot v20.4s, v0.16b, v2.16b\n" - ".inst 0x6e8295d1 // udot v17.4s, v14.16b, v2.16b\n" - ".inst 0x6e9b9619 // udot v25.4s, v16.16b, v27.16b\n" - ".inst 0x6e9b97e5 // udot v5.4s, v31.16b, v27.16b\n" - "ext v27.16b, v27.16b, v27.16b, #0x1\n" - ".inst 0x6e82961e // udot v30.4s, v16.16b, v2.16b\n" - "ldr q2, [x26, x28]\n" - ".inst 0x6e9b9614 // udot v20.4s, v16.16b, v27.16b\n" - "mls v5.4s, v19.4s, v11.4s\n" - ".inst 0x6e9b95d1 // udot v17.4s, v14.16b, v27.16b\n" - ".inst 0x6e9697f9 // udot v25.4s, v31.16b, v22.16b\n" - "ext v22.16b, v22.16b, v22.16b, #0x1\n" - ".inst 0x6e9b97fe // udot v30.4s, v31.16b, v27.16b\n" - ".inst 0x6e9697f4 // udot v20.4s, v31.16b, v22.16b\n" - "sqrdmulh v5.4s, v5.4s, v9.4s\n" - "mov v16.16b, v17.16b\n .inst 0x6e9695d0 // udot v16.4s, v14.16b, v22.16b\n" - ".inst 0x6e8795d1 // udot v17.4s, v14.16b, v7.16b\n" - "ldr q7, [x13, x28]\n" - "mls v30.4s, v17.4s, v11.4s\n" - "mls v25.4s, v18.4s, v11.4s\n" - "mls v20.4s, v16.4s, v11.4s\n" - "and v0.16b, v5.16b, v4.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqrdmulh v30.4s, v30.4s, v9.4s\n" - "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqrdmulh v20.4s, v20.4s, v9.4s\n" - "ldr q9, [%x[params], #0x120]\n" - "sqadd v5.4s, v5.4s, v0.4s\n" - "and v16.16b, v30.16b, v4.16b\n" - "and v31.16b, v25.16b, v4.16b\n" - "and v0.16b, v20.16b, v4.16b\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "srshl v5.4s, v5.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v16.4s\n" - "ldr q16, [%x[params], #0x100]\n" - "sqadd v25.4s, v25.4s, v31.4s\n" - "ldr q31, [%x[params], #0x110]\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "ldr q0, [%x[params], #0xf0]\n" - "add v5.4s, v5.4s, v10.4s\n" - "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" - "srshl v20.4s, v20.4s, v4.4s\n" - "ldr q4, [%x[params], #0x130]\n" - "smax v5.4s, v5.4s, v13.4s\n" - "add v30.4s, v30.4s, v10.4s\n" - "add v25.4s, v25.4s, v10.4s\n" - "add v20.4s, v20.4s, v10.4s\n" - "smin v5.4s, v5.4s, v12.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" + ".inst 0x6e809706 // udot v6.4s, v24.16b, v0.16b\n" + ".inst 0x6e829714 // udot v20.4s, v24.16b, v2.16b\n" + "sqrdmulh v26.4s, v26.4s, v27.4s\n" + "mov v17.16b, v23.16b\n .inst 0x6e829591 // udot v17.4s, v12.16b, v2.16b\n" + ".inst 0x6e879597 // udot v23.4s, v12.16b, v7.16b\n" + "ldr q21, [x13, x28]\n" + "mls v6.4s, v23.4s, v16.4s\n" + "mls v25.4s, v18.4s, v16.4s\n" + "mls v20.4s, v17.4s, v16.4s\n" + "and v17.16b, v26.16b, v1.16b\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqrdmulh v6.4s, v6.4s, v27.4s\n" + "sqrdmulh v25.4s, v25.4s, v27.4s\n" + "sqrdmulh v20.4s, v20.4s, v27.4s\n" + "ldr q15, [%x[params], #0x120]\n" + "sqadd v26.4s, v26.4s, v17.4s\n" + "and v18.16b, v6.16b, v1.16b\n" + "and v22.16b, v25.16b, v1.16b\n" + "and v17.16b, v20.16b, v1.16b\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "srshl v26.4s, v26.4s, v1.4s\n" + "sqadd v6.4s, v6.4s, v18.4s\n" + "ldr q30, [%x[params], #0x100]\n" + "sqadd v25.4s, v25.4s, v22.4s\n" + "ldr q27, [%x[params], #0x110]\n" + "sqadd v20.4s, v20.4s, v17.4s\n" + "ldr q24, [%x[params], #0xf0]\n" + "add v26.4s, v26.4s, v14.4s\n" + "srshl v6.4s, v6.4s, v1.4s\n" + "srshl v25.4s, v25.4s, v1.4s\n" + "srshl v20.4s, v20.4s, v1.4s\n" + "ldr q23, [%x[params], #0x130]\n" + "smax v26.4s, v26.4s, v13.4s\n" + "add v6.4s, v6.4s, v14.4s\n" + "add v25.4s, v25.4s, v14.4s\n" + "add v20.4s, v20.4s, v14.4s\n" + "smin v26.4s, v26.4s, v11.4s\n" + "smax v6.4s, v6.4s, v13.4s\n" "smax v25.4s, v25.4s, v13.4s\n" "smax v20.4s, v20.4s, v13.4s\n" - "smin v30.4s, v30.4s, v12.4s\n" - "smin v25.4s, v25.4s, v12.4s\n" - "smin v20.4s, v20.4s, v12.4s\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "movi v19.4s, #0x0\n" - ".inst 0x6e8195d3 // udot v19.4s, v14.16b, v1.16b\n" - ".inst 0x6e9a95d3 // udot v19.4s, v14.16b, v26.16b\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s5, [x24, x27]\n" - "ldr q5, [%x[params], #0xe0]\n" + "smin v6.4s, v6.4s, v11.4s\n" + "smin v25.4s, v25.4s, v11.4s\n" + "smin v20.4s, v20.4s, v11.4s\n" + "uzp1 v26.16b, v26.16b, v26.16b\n" + "movi v0.4s, #0x0\n" + ".inst 0x6e8a9580 // udot v0.4s, v12.16b, v10.16b\n" + ".inst 0x6e859580 // udot v0.4s, v12.16b, v5.16b\n" + "uzp1 v26.16b, v26.16b, v26.16b\n" + "uzp1 v6.16b, v6.16b, v6.16b\n" + "str s26, [x25, x27]\n" + "ldr q28, [%x[params], #0xe0]\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "mov v18.16b, v19.16b\n .inst 0x6e9595d2 // udot v18.4s, v14.16b, v21.16b\n" - "uzp1 v30.16b, v30.16b, v30.16b\n" + "mov v22.16b, v0.16b\n .inst 0x6e899596 // udot v22.4s, v12.16b, v9.16b\n" + "uzp1 v6.16b, v6.16b, v6.16b\n" "uzp1 v25.16b, v25.16b, v25.16b\n" - "str s30, [x23, x27]\n" - ".inst 0x6e8695d3 // udot v19.4s, v14.16b, v6.16b\n" + "str s6, [x24, x27]\n" + ".inst 0x6e889580 // udot v0.4s, v12.16b, v8.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s25, [x22, x27]\n" - "mov v30.16b, v5.16b\n" - "str s20, [x21, x27]\n" - "mov v25.16b, v5.16b\n" - "mov v20.16b, v5.16b\n" - ".inst 0x6e869405 // udot v5.4s, v0.16b, v6.16b\n" - ".inst 0x6e819419 // udot v25.4s, v0.16b, v1.16b\n" - ".inst 0x6e819605 // udot v5.4s, v16.16b, v1.16b\n" - "ext v6.16b, v6.16b, v6.16b, #0x1\n" + "str s25, [x23, x27]\n" + "mov v29.16b, v28.16b\n" + "str s20, [x22, x27]\n" + "mov v25.16b, v28.16b\n" + "mov v7.16b, v28.16b\n" + ".inst 0x6e88971c // udot v28.4s, v24.16b, v8.16b\n" + ".inst 0x6e8a9719 // udot v25.4s, v24.16b, v10.16b\n" + ".inst 0x6e8a97dc // udot v28.4s, v30.16b, v10.16b\n" + "ext v8.16b, v8.16b, v8.16b, #0x1\n" "add x27, x27, #0x4\n" - "ext v1.16b, v1.16b, v1.16b, #0x1\n" + "ext v10.16b, v10.16b, v10.16b, #0x1\n" "movi v17.4s, #0x0\n" - ".inst 0x6e86941e // udot v30.4s, v0.16b, v6.16b\n" - ".inst 0x6e819414 // udot v20.4s, v0.16b, v1.16b\n" - ".inst 0x6e8195d1 // udot v17.4s, v14.16b, v1.16b\n" - ".inst 0x6e9a9619 // udot v25.4s, v16.16b, v26.16b\n" - ".inst 0x6e9a97e5 // udot v5.4s, v31.16b, v26.16b\n" - "ext v26.16b, v26.16b, v26.16b, #0x1\n" - ".inst 0x6e81961e // udot v30.4s, v16.16b, v1.16b\n" - "ldr q1, [x25, x28]\n" - ".inst 0x6e9a9614 // udot v20.4s, v16.16b, v26.16b\n" - "mls v5.4s, v19.4s, v11.4s\n" - ".inst 0x6e9a95d1 // udot v17.4s, v14.16b, v26.16b\n" - ".inst 0x6e9597f9 // udot v25.4s, v31.16b, v21.16b\n" - "ext v21.16b, v21.16b, v21.16b, #0x1\n" - ".inst 0x6e9a97fe // udot v30.4s, v31.16b, v26.16b\n" - ".inst 0x6e9597f4 // udot v20.4s, v31.16b, v21.16b\n" - "sqrdmulh v5.4s, v5.4s, v9.4s\n" - "mov v16.16b, v17.16b\n .inst 0x6e9595d0 // udot v16.4s, v14.16b, v21.16b\n" - ".inst 0x6e8695d1 // udot v17.4s, v14.16b, v6.16b\n" - "ldr q6, [x12, x28]\n" - "mls v30.4s, v17.4s, v11.4s\n" - "mls v25.4s, v18.4s, v11.4s\n" - "mls v20.4s, v16.4s, v11.4s\n" - "and v0.16b, v5.16b, v4.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqrdmulh v30.4s, v30.4s, v9.4s\n" - "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqrdmulh v20.4s, v20.4s, v9.4s\n" - "ldr q9, [x15, x28]\n" - "sqadd v5.4s, v5.4s, v0.4s\n" - "ldp x15, x14, [%x[inptrs], #0x40]\n" - "ldr q29, [x15, x28]\n" - "ldr q28, [x14, x28]\n" - "and v16.16b, v30.16b, v4.16b\n" - "and v31.16b, v25.16b, v4.16b\n" - "and v0.16b, v20.16b, v4.16b\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "ldp x13, x12, [%x[inptrs], #0x50]\n" - "ldr q27, [x13, x28]\n" - "ldr q26, [x12, x28]\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "srshl v5.4s, v5.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v16.4s\n" - "ldr q16, [%x[params], #0x160]\n" - "sqadd v25.4s, v25.4s, v31.4s\n" - "ldr q31, [%x[params], #0x170]\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "ldr q0, [%x[params], #0x150]\n" - "add v5.4s, v5.4s, v10.4s\n" - "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" - "srshl v20.4s, v20.4s, v4.4s\n" - "ldr q4, [x10, x28]\n" - "ldp x10, x9, [%x[inptrs], #0x60]\n" - "ldr q24, [x10, x28]\n" - "ldr q23, [x9, x28]\n" - "smax v5.4s, v5.4s, v13.4s\n" - "add v30.4s, v30.4s, v10.4s\n" - "add v25.4s, v25.4s, v10.4s\n" - "add v20.4s, v20.4s, v10.4s\n" - "ldp x26, x25, [%x[inptrs], #0x70]\n" - "ldr q22, [x26, x28]\n" - "ldr q21, [x25, x28]\n" - "smin v5.4s, v5.4s, v12.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" + ".inst 0x6e88971d // udot v29.4s, v24.16b, v8.16b\n" + ".inst 0x6e8a9707 // udot v7.4s, v24.16b, v10.16b\n" + ".inst 0x6e8a9591 // udot v17.4s, v12.16b, v10.16b\n" + ".inst 0x6e8597d9 // udot v25.4s, v30.16b, v5.16b\n" + ".inst 0x6e85977c // udot v28.4s, v27.16b, v5.16b\n" + "ext v5.16b, v5.16b, v5.16b, #0x1\n" + ".inst 0x6e8a97dd // udot v29.4s, v30.16b, v10.16b\n" + "ldr q10, [x21, x28]\n" + ".inst 0x6e8597c7 // udot v7.4s, v30.16b, v5.16b\n" + "mls v28.4s, v0.4s, v16.4s\n" + ".inst 0x6e859591 // udot v17.4s, v12.16b, v5.16b\n" + ".inst 0x6e899779 // udot v25.4s, v27.16b, v9.16b\n" + "ext v9.16b, v9.16b, v9.16b, #0x1\n" + ".inst 0x6e85977d // udot v29.4s, v27.16b, v5.16b\n" + ".inst 0x6e899767 // udot v7.4s, v27.16b, v9.16b\n" + "sqrdmulh v28.4s, v28.4s, v15.4s\n" + "mov v18.16b, v17.16b\n .inst 0x6e899592 // udot v18.4s, v12.16b, v9.16b\n" + ".inst 0x6e889591 // udot v17.4s, v12.16b, v8.16b\n" + "ldr q8, [x12, x28]\n" + "mls v29.4s, v17.4s, v16.4s\n" + "mls v25.4s, v22.4s, v16.4s\n" + "mls v7.4s, v18.4s, v16.4s\n" + "and v17.16b, v28.16b, v23.16b\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqrdmulh v29.4s, v29.4s, v15.4s\n" + "sqrdmulh v25.4s, v25.4s, v15.4s\n" + "sqrdmulh v7.4s, v7.4s, v15.4s\n" + "ldr q15, [x15, x28]\n" + "sqadd v28.4s, v28.4s, v17.4s\n" + "ldp x21, x20, [%x[inptrs], #0x40]\n" + "ldr q22, [x21, x28]\n" + "ldr q3, [x20, x28]\n" + "and v24.16b, v29.16b, v23.16b\n" + "and v20.16b, v25.16b, v23.16b\n" + "and v17.16b, v7.16b, v23.16b\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "ldp x21, x20, [%x[inptrs], #0x50]\n" + "ldr q2, [x21, x28]\n" + "ldr q5, [x20, x28]\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "srshl v28.4s, v28.4s, v23.4s\n" + "sqadd v29.4s, v29.4s, v24.4s\n" + "ldr q6, [%x[params], #0x160]\n" + "sqadd v25.4s, v25.4s, v20.4s\n" + "ldr q20, [%x[params], #0x170]\n" + "sqadd v7.4s, v7.4s, v17.4s\n" + "ldr q1, [%x[params], #0x150]\n" + "add v28.4s, v28.4s, v14.4s\n" + "srshl v29.4s, v29.4s, v23.4s\n" + "srshl v25.4s, v25.4s, v23.4s\n" + "srshl v7.4s, v7.4s, v23.4s\n" + "ldr q26, [x10, x28]\n" + "ldp x21, x20, [%x[inptrs], #0x60]\n" + "ldr q27, [x21, x28]\n" + "ldr q30, [x20, x28]\n" + "smax v28.4s, v28.4s, v13.4s\n" + "add v29.4s, v29.4s, v14.4s\n" + "add v25.4s, v25.4s, v14.4s\n" + "add v7.4s, v7.4s, v14.4s\n" + "ldp x21, x20, [%x[inptrs], #0x70]\n" + "ldr q23, [x21, x28]\n" + "ldr q9, [x20, x28]\n" + "smin v28.4s, v28.4s, v11.4s\n" + "smax v29.4s, v29.4s, v13.4s\n" "ldp x15, x14, [%x[inptrs], #0x0]\n" "smax v25.4s, v25.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" + "smax v7.4s, v7.4s, v13.4s\n" "ldp x13, x12, [%x[inptrs], #0x10]\n" "ldp x10, x9, [%x[inptrs], #0x20]\n" - "smin v30.4s, v30.4s, v12.4s\n" - "smin v25.4s, v25.4s, v12.4s\n" - "ldp x26, x25, [%x[inptrs], #0x30]\n" - "smin v20.4s, v20.4s, v12.4s\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s5, [x24, x27]\n" + "smin v29.4s, v29.4s, v11.4s\n" + "smin v25.4s, v25.4s, v11.4s\n" + "ldp x26, x21, [%x[inptrs], #0x30]\n" + "smin v7.4s, v7.4s, v11.4s\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "str s28, [x25, x27]\n" "uzp1 v25.16b, v25.16b, v25.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "zip2 v5.16b, v9.16b, v7.16b\n" - "zip1 v9.16b, v9.16b, v7.16b\n" - "zip1 v7.16b, v8.16b, v6.16b\n" - "zip2 v6.16b, v8.16b, v6.16b\n" - "uzp1 v30.16b, v30.16b, v30.16b\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" + "zip2 v17.16b, v15.16b, v21.16b\n" + "zip1 v15.16b, v15.16b, v21.16b\n" + "zip1 v18.16b, v31.16b, v8.16b\n" + "zip2 v8.16b, v31.16b, v8.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" "uzp1 v25.16b, v25.16b, v25.16b\n" - "str s30, [x23, x27]\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s25, [x22, x27]\n" - "zip2 v8.16b, v9.16b, v7.16b\n" - "str s20, [x21, x27]\n" - "zip1 v9.16b, v9.16b, v7.16b\n" - "zip1 v7.16b, v5.16b, v6.16b\n" + "str s29, [x24, x27]\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" + "str s25, [x23, x27]\n" + "zip2 v25.16b, v15.16b, v18.16b\n" + "str s7, [x22, x27]\n" + "zip1 v15.16b, v15.16b, v18.16b\n" + "zip1 v7.16b, v17.16b, v8.16b\n" "add x27, x27, #0x4\n" - "zip2 v6.16b, v5.16b, v6.16b\n" - "ldr q5, [%x[params], #0x140]\n" - "zip2 v30.16b, v4.16b, v2.16b\n" + "zip2 v8.16b, v17.16b, v8.16b\n" + "ldr q31, [%x[params], #0x140]\n" + "zip2 v29.16b, v26.16b, v19.16b\n" "add %x[params], %x[params], #0x180\n" - "zip1 v4.16b, v4.16b, v2.16b\n" - "zip1 v2.16b, v3.16b, v1.16b\n" - "zip2 v1.16b, v3.16b, v1.16b\n" - "zip2 v25.16b, v29.16b, v27.16b\n" - "zip1 v29.16b, v29.16b, v27.16b\n" - "zip1 v27.16b, v28.16b, v26.16b\n" - "zip2 v26.16b, v28.16b, v26.16b\n" - "zip2 v20.16b, v24.16b, v22.16b\n" - "zip1 v24.16b, v24.16b, v22.16b\n" - "zip1 v22.16b, v23.16b, v21.16b\n" - "zip2 v21.16b, v23.16b, v21.16b\n" - "zip2 v3.16b, v4.16b, v2.16b\n" - "zip1 v4.16b, v4.16b, v2.16b\n" - "zip1 v2.16b, v30.16b, v1.16b\n" - "zip2 v1.16b, v30.16b, v1.16b\n" - "zip2 v28.16b, v29.16b, v27.16b\n" - "zip1 v29.16b, v29.16b, v27.16b\n" - "zip1 v27.16b, v25.16b, v26.16b\n" - "zip2 v26.16b, v25.16b, v26.16b\n" - "zip2 v23.16b, v24.16b, v22.16b\n" - "zip1 v24.16b, v24.16b, v22.16b\n" - "zip1 v22.16b, v20.16b, v21.16b\n" - "zip2 v21.16b, v20.16b, v21.16b\n" - "mov v30.16b, v5.16b\n" - "mov v25.16b, v5.16b\n" - "mov v20.16b, v5.16b\n" + "zip1 v26.16b, v26.16b, v19.16b\n" + "zip1 v28.16b, v4.16b, v10.16b\n" + "zip2 v10.16b, v4.16b, v10.16b\n" + "zip2 v24.16b, v22.16b, v2.16b\n" + "zip1 v22.16b, v22.16b, v2.16b\n" + "zip1 v21.16b, v3.16b, v5.16b\n" + "zip2 v5.16b, v3.16b, v5.16b\n" + "zip2 v18.16b, v27.16b, v23.16b\n" + "zip1 v27.16b, v27.16b, v23.16b\n" + "zip1 v17.16b, v30.16b, v9.16b\n" + "zip2 v9.16b, v30.16b, v9.16b\n" + "zip2 v23.16b, v26.16b, v28.16b\n" + "zip1 v26.16b, v26.16b, v28.16b\n" + "zip1 v3.16b, v29.16b, v10.16b\n" + "zip2 v10.16b, v29.16b, v10.16b\n" + "zip2 v19.16b, v22.16b, v21.16b\n" + "zip1 v22.16b, v22.16b, v21.16b\n" + "zip1 v0.16b, v24.16b, v5.16b\n" + "zip2 v5.16b, v24.16b, v5.16b\n" + "zip2 v24.16b, v27.16b, v17.16b\n" + "zip1 v27.16b, v27.16b, v17.16b\n" + "zip1 v2.16b, v18.16b, v9.16b\n" + "zip2 v9.16b, v18.16b, v9.16b\n" + "mov v30.16b, v31.16b\n" + "mov v29.16b, v31.16b\n" + "mov v28.16b, v31.16b\n" "bgt 1b\n" "2:" // Detached iteration - "movi v19.4s, #0x0\n" - ".inst 0x6e8495d3 // udot v19.4s, v14.16b, v4.16b\n" - ".inst 0x6e899405 // udot v5.4s, v0.16b, v9.16b\n" + "movi v21.4s, #0x0\n" + ".inst 0x6e9a9595 // udot v21.4s, v12.16b, v26.16b\n" + ".inst 0x6e8f943f // udot v31.4s, v1.16b, v15.16b\n" "tst %x[n_channels], #0xf\n" - ".inst 0x6e9d95d3 // udot v19.4s, v14.16b, v29.16b\n" - ".inst 0x6e849419 // udot v25.4s, v0.16b, v4.16b\n" + ".inst 0x6e969595 // udot v21.4s, v12.16b, v22.16b\n" + ".inst 0x6e9a943d // udot v29.4s, v1.16b, v26.16b\n" + "movi v18.4s, #0x0\n" "add x28, x28, #0x10\n" - ".inst 0x6e849605 // udot v5.4s, v16.16b, v4.16b\n" - "ext v4.16b, v4.16b, v4.16b, #0x1\n" - "mov v18.16b, v19.16b\n .inst 0x6e9895d2 // udot v18.4s, v14.16b, v24.16b\n" - ".inst 0x6e8995d3 // udot v19.4s, v14.16b, v9.16b\n" - "ext v9.16b, v9.16b, v9.16b, #0x1\n" - ".inst 0x6e9d9619 // udot v25.4s, v16.16b, v29.16b\n" - ".inst 0x6e9d97e5 // udot v5.4s, v31.16b, v29.16b\n" - "ext v29.16b, v29.16b, v29.16b, #0x1\n" - ".inst 0x6e89941e // udot v30.4s, v0.16b, v9.16b\n" - ".inst 0x6e849414 // udot v20.4s, v0.16b, v4.16b\n" - "movi v17.4s, #0x0\n" - ".inst 0x6e8495d1 // udot v17.4s, v14.16b, v4.16b\n" - ".inst 0x6e9d95d1 // udot v17.4s, v14.16b, v29.16b\n" - ".inst 0x6e9897f9 // udot v25.4s, v31.16b, v24.16b\n" - "ext v24.16b, v24.16b, v24.16b, #0x1\n" - ".inst 0x6e84961e // udot v30.4s, v16.16b, v4.16b\n" + ".inst 0x6e9a94df // udot v31.4s, v6.16b, v26.16b\n" + "ext v26.16b, v26.16b, v26.16b, #0x1\n" + "mov v17.16b, v21.16b\n .inst 0x6e9b9591 // udot v17.4s, v12.16b, v27.16b\n" + ".inst 0x6e8f9595 // udot v21.4s, v12.16b, v15.16b\n" + "ext v15.16b, v15.16b, v15.16b, #0x1\n" + ".inst 0x6e9a9592 // udot v18.4s, v12.16b, v26.16b\n" + ".inst 0x6e9694dd // udot v29.4s, v6.16b, v22.16b\n" + ".inst 0x6e96969f // udot v31.4s, v20.16b, v22.16b\n" + "ext v22.16b, v22.16b, v22.16b, #0x1\n" + ".inst 0x6e8f943e // udot v30.4s, v1.16b, v15.16b\n" + ".inst 0x6e9a943c // udot v28.4s, v1.16b, v26.16b\n" + "mls v31.4s, v21.4s, v16.4s\n" + ".inst 0x6e969592 // udot v18.4s, v12.16b, v22.16b\n" + ".inst 0x6e9b969d // udot v29.4s, v20.16b, v27.16b\n" + "ext v27.16b, v27.16b, v27.16b, #0x1\n" + ".inst 0x6e9a94de // udot v30.4s, v6.16b, v26.16b\n" "ldr q4, [%x[params], #0x10]\n" - ".inst 0x6e9d9614 // udot v20.4s, v16.16b, v29.16b\n" - "mls v5.4s, v19.4s, v11.4s\n" - "mov v16.16b, v17.16b\n .inst 0x6e9895d0 // udot v16.4s, v14.16b, v24.16b\n" - ".inst 0x6e8995d1 // udot v17.4s, v14.16b, v9.16b\n" - "ldr q9, [%x[params], #0x0]\n" - "sqrdmulh v5.4s, v5.4s, v9.4s\n" - ".inst 0x6e9d97fe // udot v30.4s, v31.16b, v29.16b\n" - ".inst 0x6e9897f4 // udot v20.4s, v31.16b, v24.16b\n" - "mls v30.4s, v17.4s, v11.4s\n" - "mls v25.4s, v18.4s, v11.4s\n" - "mls v20.4s, v16.4s, v11.4s\n" - "and v0.16b, v5.16b, v4.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqrdmulh v30.4s, v30.4s, v9.4s\n" - "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqrdmulh v20.4s, v20.4s, v9.4s\n" - "ldr q9, [%x[params], #0x60]\n" - "sqadd v5.4s, v5.4s, v0.4s\n" - "and v16.16b, v30.16b, v4.16b\n" - "and v31.16b, v25.16b, v4.16b\n" - "and v0.16b, v20.16b, v4.16b\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "srshl v5.4s, v5.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v16.4s\n" - "ldr q16, [%x[params], #0x40]\n" - "sqadd v25.4s, v25.4s, v31.4s\n" - "ldr q31, [%x[params], #0x50]\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "ldr q0, [%x[params], #0x30]\n" - "add v5.4s, v5.4s, v10.4s\n" + ".inst 0x6e9694dc // udot v28.4s, v6.16b, v22.16b\n" + "mls v29.4s, v17.4s, v16.4s\n" + "mov v21.16b, v18.16b\n .inst 0x6e9b9595 // udot v21.4s, v12.16b, v27.16b\n" + ".inst 0x6e8f9592 // udot v18.4s, v12.16b, v15.16b\n" + "ldr q17, [%x[params], #0x0]\n" + "sqrdmulh v31.4s, v31.4s, v17.4s\n" + ".inst 0x6e96969e // udot v30.4s, v20.16b, v22.16b\n" + ".inst 0x6e9b969c // udot v28.4s, v20.16b, v27.16b\n" + "mls v30.4s, v18.4s, v16.4s\n" + "mls v28.4s, v21.4s, v16.4s\n" + "and v27.16b, v31.16b, v4.16b\n" + "sshr v27.4s, v27.4s, #0x1f\n" + "sqrdmulh v30.4s, v30.4s, v17.4s\n" + "sqrdmulh v29.4s, v29.4s, v17.4s\n" + "sqrdmulh v28.4s, v28.4s, v17.4s\n" + "ldr q15, [%x[params], #0x60]\n" + "sqadd v31.4s, v31.4s, v27.4s\n" + "and v20.16b, v30.16b, v4.16b\n" + "and v18.16b, v29.16b, v4.16b\n" + "and v17.16b, v28.16b, v4.16b\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "srshl v31.4s, v31.4s, v4.4s\n" + "sqadd v30.4s, v30.4s, v20.4s\n" + "ldr q27, [%x[params], #0x40]\n" + "sqadd v29.4s, v29.4s, v18.4s\n" + "ldr q26, [%x[params], #0x50]\n" + "sqadd v28.4s, v28.4s, v17.4s\n" + "ldr q6, [%x[params], #0x30]\n" + "add v31.4s, v31.4s, v14.4s\n" "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" - "srshl v20.4s, v20.4s, v4.4s\n" + "srshl v29.4s, v29.4s, v4.4s\n" + "srshl v28.4s, v28.4s, v4.4s\n" "ldr q4, [%x[params], #0x70]\n" - "smax v5.4s, v5.4s, v13.4s\n" - "add v30.4s, v30.4s, v10.4s\n" - "add v25.4s, v25.4s, v10.4s\n" - "add v20.4s, v20.4s, v10.4s\n" - "smin v5.4s, v5.4s, v12.4s\n" + "smax v31.4s, v31.4s, v13.4s\n" + "add v30.4s, v30.4s, v14.4s\n" + "add v29.4s, v29.4s, v14.4s\n" + "add v28.4s, v28.4s, v14.4s\n" + "smin v31.4s, v31.4s, v11.4s\n" "smax v30.4s, v30.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smin v30.4s, v30.4s, v12.4s\n" - "smin v25.4s, v25.4s, v12.4s\n" - "smin v20.4s, v20.4s, v12.4s\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "movi v19.4s, #0x0\n" - ".inst 0x6e8395d3 // udot v19.4s, v14.16b, v3.16b\n" - ".inst 0x6e9c95d3 // udot v19.4s, v14.16b, v28.16b\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" + "smax v29.4s, v29.4s, v13.4s\n" + "smax v28.4s, v28.4s, v13.4s\n" + "smin v30.4s, v30.4s, v11.4s\n" + "smin v29.4s, v29.4s, v11.4s\n" + "smin v28.4s, v28.4s, v11.4s\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" + "movi v1.4s, #0x0\n" + ".inst 0x6e979581 // udot v1.4s, v12.16b, v23.16b\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s5, [x24, x27]\n" - "ldr q5, [%x[params], #0x20]\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "mov v18.16b, v19.16b\n .inst 0x6e9795d2 // udot v18.4s, v14.16b, v23.16b\n" + "str s31, [x25, x27]\n" + "ldr q31, [%x[params], #0x20]\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + ".inst 0x6e939581 // udot v1.4s, v12.16b, v19.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "str s30, [x23, x27]\n" - ".inst 0x6e8895d3 // udot v19.4s, v14.16b, v8.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s25, [x22, x27]\n" - "mov v30.16b, v5.16b\n" - "str s20, [x21, x27]\n" - "mov v25.16b, v5.16b\n" - "mov v20.16b, v5.16b\n" - ".inst 0x6e889405 // udot v5.4s, v0.16b, v8.16b\n" - ".inst 0x6e839419 // udot v25.4s, v0.16b, v3.16b\n" - ".inst 0x6e839605 // udot v5.4s, v16.16b, v3.16b\n" - "ext v8.16b, v8.16b, v8.16b, #0x1\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "str s30, [x24, x27]\n" + "mov v22.16b, v1.16b\n .inst 0x6e989596 // udot v22.4s, v12.16b, v24.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + "str s29, [x23, x27]\n" + "mov v29.16b, v31.16b\n" + ".inst 0x6e999581 // udot v1.4s, v12.16b, v25.16b\n" + "str s28, [x22, x27]\n" + "mov v21.16b, v31.16b\n" + "mov v20.16b, v31.16b\n" + ".inst 0x6e9994df // udot v31.4s, v6.16b, v25.16b\n" + ".inst 0x6e9794d5 // udot v21.4s, v6.16b, v23.16b\n" + ".inst 0x6e97977f // udot v31.4s, v27.16b, v23.16b\n" + "ext v25.16b, v25.16b, v25.16b, #0x1\n" "add x27, x27, #0x4\n" - "ext v3.16b, v3.16b, v3.16b, #0x1\n" - "movi v17.4s, #0x0\n" - ".inst 0x6e88941e // udot v30.4s, v0.16b, v8.16b\n" - ".inst 0x6e839414 // udot v20.4s, v0.16b, v3.16b\n" - ".inst 0x6e8395d1 // udot v17.4s, v14.16b, v3.16b\n" - ".inst 0x6e9c9619 // udot v25.4s, v16.16b, v28.16b\n" - ".inst 0x6e9c97e5 // udot v5.4s, v31.16b, v28.16b\n" - "ext v28.16b, v28.16b, v28.16b, #0x1\n" - ".inst 0x6e83961e // udot v30.4s, v16.16b, v3.16b\n" - ".inst 0x6e9c9614 // udot v20.4s, v16.16b, v28.16b\n" - "mls v5.4s, v19.4s, v11.4s\n" - ".inst 0x6e9c95d1 // udot v17.4s, v14.16b, v28.16b\n" - ".inst 0x6e9797f9 // udot v25.4s, v31.16b, v23.16b\n" "ext v23.16b, v23.16b, v23.16b, #0x1\n" - ".inst 0x6e9c97fe // udot v30.4s, v31.16b, v28.16b\n" - ".inst 0x6e9797f4 // udot v20.4s, v31.16b, v23.16b\n" - "sqrdmulh v5.4s, v5.4s, v9.4s\n" - "mov v16.16b, v17.16b\n .inst 0x6e9795d0 // udot v16.4s, v14.16b, v23.16b\n" - ".inst 0x6e8895d1 // udot v17.4s, v14.16b, v8.16b\n" - "mls v30.4s, v17.4s, v11.4s\n" - "mls v25.4s, v18.4s, v11.4s\n" - "mls v20.4s, v16.4s, v11.4s\n" - "and v0.16b, v5.16b, v4.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqrdmulh v30.4s, v30.4s, v9.4s\n" - "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqrdmulh v20.4s, v20.4s, v9.4s\n" - "ldr q9, [%x[params], #0xc0]\n" - "sqadd v5.4s, v5.4s, v0.4s\n" - "and v16.16b, v30.16b, v4.16b\n" - "and v31.16b, v25.16b, v4.16b\n" - "and v0.16b, v20.16b, v4.16b\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "srshl v5.4s, v5.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v16.4s\n" - "ldr q16, [%x[params], #0xa0]\n" - "sqadd v25.4s, v25.4s, v31.4s\n" - "ldr q31, [%x[params], #0xb0]\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "ldr q0, [%x[params], #0x90]\n" - "add v5.4s, v5.4s, v10.4s\n" - "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" + "movi v18.4s, #0x0\n" + ".inst 0x6e9994dd // udot v29.4s, v6.16b, v25.16b\n" + ".inst 0x6e9794d4 // udot v20.4s, v6.16b, v23.16b\n" + ".inst 0x6e979592 // udot v18.4s, v12.16b, v23.16b\n" + ".inst 0x6e939775 // udot v21.4s, v27.16b, v19.16b\n" + ".inst 0x6e93975f // udot v31.4s, v26.16b, v19.16b\n" + "ext v19.16b, v19.16b, v19.16b, #0x1\n" + ".inst 0x6e97977d // udot v29.4s, v27.16b, v23.16b\n" + ".inst 0x6e939774 // udot v20.4s, v27.16b, v19.16b\n" + "mls v31.4s, v1.4s, v16.4s\n" + ".inst 0x6e939592 // udot v18.4s, v12.16b, v19.16b\n" + ".inst 0x6e989755 // udot v21.4s, v26.16b, v24.16b\n" + "ext v24.16b, v24.16b, v24.16b, #0x1\n" + ".inst 0x6e93975d // udot v29.4s, v26.16b, v19.16b\n" + ".inst 0x6e989754 // udot v20.4s, v26.16b, v24.16b\n" + "sqrdmulh v31.4s, v31.4s, v15.4s\n" + "mov v17.16b, v18.16b\n .inst 0x6e989591 // udot v17.4s, v12.16b, v24.16b\n" + ".inst 0x6e999592 // udot v18.4s, v12.16b, v25.16b\n" + "mls v29.4s, v18.4s, v16.4s\n" + "mls v21.4s, v22.4s, v16.4s\n" + "mls v20.4s, v17.4s, v16.4s\n" + "and v17.16b, v31.16b, v4.16b\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqrdmulh v29.4s, v29.4s, v15.4s\n" + "sqrdmulh v21.4s, v21.4s, v15.4s\n" + "sqrdmulh v20.4s, v20.4s, v15.4s\n" + "ldr q27, [%x[params], #0xc0]\n" + "sqadd v31.4s, v31.4s, v17.4s\n" + "and v19.16b, v29.16b, v4.16b\n" + "and v18.16b, v21.16b, v4.16b\n" + "and v17.16b, v20.16b, v4.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "srshl v31.4s, v31.4s, v4.4s\n" + "sqadd v29.4s, v29.4s, v19.4s\n" + "ldr q26, [%x[params], #0xa0]\n" + "sqadd v21.4s, v21.4s, v18.4s\n" + "ldr q25, [%x[params], #0xb0]\n" + "sqadd v20.4s, v20.4s, v17.4s\n" + "ldr q24, [%x[params], #0x90]\n" + "add v31.4s, v31.4s, v14.4s\n" + "srshl v29.4s, v29.4s, v4.4s\n" + "srshl v21.4s, v21.4s, v4.4s\n" "srshl v20.4s, v20.4s, v4.4s\n" - "ldr q4, [%x[params], #0xd0]\n" - "smax v5.4s, v5.4s, v13.4s\n" - "add v30.4s, v30.4s, v10.4s\n" - "add v25.4s, v25.4s, v10.4s\n" - "add v20.4s, v20.4s, v10.4s\n" - "smin v5.4s, v5.4s, v12.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" + "ldr q1, [%x[params], #0xd0]\n" + "smax v31.4s, v31.4s, v13.4s\n" + "add v29.4s, v29.4s, v14.4s\n" + "add v21.4s, v21.4s, v14.4s\n" + "add v20.4s, v20.4s, v14.4s\n" + "smin v31.4s, v31.4s, v11.4s\n" + "smax v29.4s, v29.4s, v13.4s\n" + "smax v21.4s, v21.4s, v13.4s\n" "smax v20.4s, v20.4s, v13.4s\n" - "smin v30.4s, v30.4s, v12.4s\n" - "smin v25.4s, v25.4s, v12.4s\n" - "smin v20.4s, v20.4s, v12.4s\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "movi v19.4s, #0x0\n" - ".inst 0x6e8295d3 // udot v19.4s, v14.16b, v2.16b\n" - ".inst 0x6e9b95d3 // udot v19.4s, v14.16b, v27.16b\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s5, [x24, x27]\n" - "ldr q5, [%x[params], #0x80]\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" + "smin v29.4s, v29.4s, v11.4s\n" + "smin v21.4s, v21.4s, v11.4s\n" + "smin v20.4s, v20.4s, v11.4s\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" + "movi v23.4s, #0x0\n" + ".inst 0x6e839597 // udot v23.4s, v12.16b, v3.16b\n" + ".inst 0x6e809597 // udot v23.4s, v12.16b, v0.16b\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "str s31, [x25, x27]\n" + "ldr q31, [%x[params], #0x80]\n" + "uzp1 v21.16b, v21.16b, v21.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "mov v18.16b, v19.16b\n .inst 0x6e9695d2 // udot v18.4s, v14.16b, v22.16b\n" - "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "str s30, [x23, x27]\n" - ".inst 0x6e8795d3 // udot v19.4s, v14.16b, v7.16b\n" + "mov v22.16b, v23.16b\n .inst 0x6e829596 // udot v22.4s, v12.16b, v2.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "uzp1 v21.16b, v21.16b, v21.16b\n" + "str s29, [x24, x27]\n" + ".inst 0x6e879597 // udot v23.4s, v12.16b, v7.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s25, [x22, x27]\n" - "mov v30.16b, v5.16b\n" - "str s20, [x21, x27]\n" - "mov v25.16b, v5.16b\n" - "mov v20.16b, v5.16b\n" - ".inst 0x6e879405 // udot v5.4s, v0.16b, v7.16b\n" - ".inst 0x6e829419 // udot v25.4s, v0.16b, v2.16b\n" - ".inst 0x6e829605 // udot v5.4s, v16.16b, v2.16b\n" + "str s21, [x23, x27]\n" + "mov v21.16b, v31.16b\n" + "str s20, [x22, x27]\n" + "mov v4.16b, v31.16b\n" + "mov v20.16b, v31.16b\n" + ".inst 0x6e87971f // udot v31.4s, v24.16b, v7.16b\n" + ".inst 0x6e839704 // udot v4.4s, v24.16b, v3.16b\n" + ".inst 0x6e83975f // udot v31.4s, v26.16b, v3.16b\n" "ext v7.16b, v7.16b, v7.16b, #0x1\n" "add x27, x27, #0x4\n" + "ext v3.16b, v3.16b, v3.16b, #0x1\n" + "movi v18.4s, #0x0\n" + ".inst 0x6e879715 // udot v21.4s, v24.16b, v7.16b\n" + ".inst 0x6e839714 // udot v20.4s, v24.16b, v3.16b\n" + ".inst 0x6e839592 // udot v18.4s, v12.16b, v3.16b\n" + ".inst 0x6e809744 // udot v4.4s, v26.16b, v0.16b\n" + ".inst 0x6e80973f // udot v31.4s, v25.16b, v0.16b\n" + "ext v0.16b, v0.16b, v0.16b, #0x1\n" + ".inst 0x6e839755 // udot v21.4s, v26.16b, v3.16b\n" + ".inst 0x6e809754 // udot v20.4s, v26.16b, v0.16b\n" + "mls v31.4s, v23.4s, v16.4s\n" + ".inst 0x6e809592 // udot v18.4s, v12.16b, v0.16b\n" + ".inst 0x6e829724 // udot v4.4s, v25.16b, v2.16b\n" "ext v2.16b, v2.16b, v2.16b, #0x1\n" - "movi v17.4s, #0x0\n" - ".inst 0x6e87941e // udot v30.4s, v0.16b, v7.16b\n" - ".inst 0x6e829414 // udot v20.4s, v0.16b, v2.16b\n" - ".inst 0x6e8295d1 // udot v17.4s, v14.16b, v2.16b\n" - ".inst 0x6e9b9619 // udot v25.4s, v16.16b, v27.16b\n" - ".inst 0x6e9b97e5 // udot v5.4s, v31.16b, v27.16b\n" - "ext v27.16b, v27.16b, v27.16b, #0x1\n" - ".inst 0x6e82961e // udot v30.4s, v16.16b, v2.16b\n" - ".inst 0x6e9b9614 // udot v20.4s, v16.16b, v27.16b\n" - "mls v5.4s, v19.4s, v11.4s\n" - ".inst 0x6e9b95d1 // udot v17.4s, v14.16b, v27.16b\n" - ".inst 0x6e9697f9 // udot v25.4s, v31.16b, v22.16b\n" - "ext v22.16b, v22.16b, v22.16b, #0x1\n" - ".inst 0x6e9b97fe // udot v30.4s, v31.16b, v27.16b\n" - ".inst 0x6e9697f4 // udot v20.4s, v31.16b, v22.16b\n" - "sqrdmulh v5.4s, v5.4s, v9.4s\n" - "mov v16.16b, v17.16b\n .inst 0x6e9695d0 // udot v16.4s, v14.16b, v22.16b\n" - ".inst 0x6e8795d1 // udot v17.4s, v14.16b, v7.16b\n" - "mls v30.4s, v17.4s, v11.4s\n" - "mls v25.4s, v18.4s, v11.4s\n" - "mls v20.4s, v16.4s, v11.4s\n" - "and v0.16b, v5.16b, v4.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqrdmulh v30.4s, v30.4s, v9.4s\n" - "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqrdmulh v20.4s, v20.4s, v9.4s\n" - "ldr q9, [%x[params], #0x120]\n" - "sqadd v5.4s, v5.4s, v0.4s\n" - "and v16.16b, v30.16b, v4.16b\n" - "and v31.16b, v25.16b, v4.16b\n" - "and v0.16b, v20.16b, v4.16b\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "srshl v5.4s, v5.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v16.4s\n" - "ldr q16, [%x[params], #0x100]\n" - "sqadd v25.4s, v25.4s, v31.4s\n" - "ldr q31, [%x[params], #0x110]\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "ldr q0, [%x[params], #0xf0]\n" - "add v5.4s, v5.4s, v10.4s\n" - "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" - "srshl v20.4s, v20.4s, v4.4s\n" - "ldr q4, [%x[params], #0x130]\n" - "smax v5.4s, v5.4s, v13.4s\n" - "add v30.4s, v30.4s, v10.4s\n" - "add v25.4s, v25.4s, v10.4s\n" - "add v20.4s, v20.4s, v10.4s\n" - "smin v5.4s, v5.4s, v12.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" + ".inst 0x6e809735 // udot v21.4s, v25.16b, v0.16b\n" + ".inst 0x6e829734 // udot v20.4s, v25.16b, v2.16b\n" + "sqrdmulh v31.4s, v31.4s, v27.4s\n" + "mov v17.16b, v18.16b\n .inst 0x6e829591 // udot v17.4s, v12.16b, v2.16b\n" + ".inst 0x6e879592 // udot v18.4s, v12.16b, v7.16b\n" + "mls v21.4s, v18.4s, v16.4s\n" + "mls v4.4s, v22.4s, v16.4s\n" + "mls v20.4s, v17.4s, v16.4s\n" + "and v17.16b, v31.16b, v1.16b\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqrdmulh v21.4s, v21.4s, v27.4s\n" + "sqrdmulh v4.4s, v4.4s, v27.4s\n" + "sqrdmulh v20.4s, v20.4s, v27.4s\n" + "ldr q30, [%x[params], #0x120]\n" + "sqadd v31.4s, v31.4s, v17.4s\n" + "and v19.16b, v21.16b, v1.16b\n" + "and v18.16b, v4.16b, v1.16b\n" + "and v17.16b, v20.16b, v1.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "srshl v31.4s, v31.4s, v1.4s\n" + "sqadd v21.4s, v21.4s, v19.4s\n" + "ldr q29, [%x[params], #0x100]\n" + "sqadd v4.4s, v4.4s, v18.4s\n" + "ldr q28, [%x[params], #0x110]\n" + "sqadd v20.4s, v20.4s, v17.4s\n" + "ldr q27, [%x[params], #0xf0]\n" + "add v31.4s, v31.4s, v14.4s\n" + "srshl v21.4s, v21.4s, v1.4s\n" + "srshl v4.4s, v4.4s, v1.4s\n" + "srshl v20.4s, v20.4s, v1.4s\n" + "ldr q26, [%x[params], #0x130]\n" + "smax v31.4s, v31.4s, v13.4s\n" + "add v21.4s, v21.4s, v14.4s\n" + "add v4.4s, v4.4s, v14.4s\n" + "add v20.4s, v20.4s, v14.4s\n" + "smin v31.4s, v31.4s, v11.4s\n" + "smax v21.4s, v21.4s, v13.4s\n" + "smax v4.4s, v4.4s, v13.4s\n" "smax v20.4s, v20.4s, v13.4s\n" - "smin v30.4s, v30.4s, v12.4s\n" - "smin v25.4s, v25.4s, v12.4s\n" - "smin v20.4s, v20.4s, v12.4s\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "movi v19.4s, #0x0\n" - ".inst 0x6e8195d3 // udot v19.4s, v14.16b, v1.16b\n" - ".inst 0x6e9a95d3 // udot v19.4s, v14.16b, v26.16b\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s5, [x24, x27]\n" - "ldr q5, [%x[params], #0xe0]\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" + "smin v21.4s, v21.4s, v11.4s\n" + "smin v4.4s, v4.4s, v11.4s\n" + "smin v20.4s, v20.4s, v11.4s\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" + "movi v25.4s, #0x0\n" + ".inst 0x6e8a9599 // udot v25.4s, v12.16b, v10.16b\n" + ".inst 0x6e859599 // udot v25.4s, v12.16b, v5.16b\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" + "uzp1 v21.16b, v21.16b, v21.16b\n" + "str s31, [x25, x27]\n" + "ldr q24, [%x[params], #0xe0]\n" + "uzp1 v4.16b, v4.16b, v4.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "mov v18.16b, v19.16b\n .inst 0x6e9595d2 // udot v18.4s, v14.16b, v21.16b\n" + "mov v23.16b, v25.16b\n .inst 0x6e899597 // udot v23.4s, v12.16b, v9.16b\n" "add %x[params], %x[params], #0x140\n" - "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "str s30, [x23, x27]\n" - ".inst 0x6e8695d3 // udot v19.4s, v14.16b, v6.16b\n" + "uzp1 v21.16b, v21.16b, v21.16b\n" + "uzp1 v4.16b, v4.16b, v4.16b\n" + "str s21, [x24, x27]\n" + ".inst 0x6e889599 // udot v25.4s, v12.16b, v8.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s25, [x22, x27]\n" - "mov v30.16b, v5.16b\n" - "str s20, [x21, x27]\n" - "mov v25.16b, v5.16b\n" - "mov v20.16b, v5.16b\n" - ".inst 0x6e869405 // udot v5.4s, v0.16b, v6.16b\n" - ".inst 0x6e819419 // udot v25.4s, v0.16b, v1.16b\n" - ".inst 0x6e819605 // udot v5.4s, v16.16b, v1.16b\n" - "ext v6.16b, v6.16b, v6.16b, #0x1\n" + "str s4, [x23, x27]\n" + "mov v22.16b, v24.16b\n" + "str s20, [x22, x27]\n" + "mov v21.16b, v24.16b\n" + "mov v20.16b, v24.16b\n" + ".inst 0x6e889778 // udot v24.4s, v27.16b, v8.16b\n" + ".inst 0x6e8a9775 // udot v21.4s, v27.16b, v10.16b\n" + ".inst 0x6e8a97b8 // udot v24.4s, v29.16b, v10.16b\n" + "ext v8.16b, v8.16b, v8.16b, #0x1\n" "add x27, x27, #0x4\n" - "ext v1.16b, v1.16b, v1.16b, #0x1\n" - "movi v17.4s, #0x0\n" - ".inst 0x6e86941e // udot v30.4s, v0.16b, v6.16b\n" - ".inst 0x6e819414 // udot v20.4s, v0.16b, v1.16b\n" - ".inst 0x6e8195d1 // udot v17.4s, v14.16b, v1.16b\n" - ".inst 0x6e9a9619 // udot v25.4s, v16.16b, v26.16b\n" - ".inst 0x6e9a97e5 // udot v5.4s, v31.16b, v26.16b\n" - "ext v26.16b, v26.16b, v26.16b, #0x1\n" - ".inst 0x6e81961e // udot v30.4s, v16.16b, v1.16b\n" - ".inst 0x6e9a9614 // udot v20.4s, v16.16b, v26.16b\n" - "mls v5.4s, v19.4s, v11.4s\n" - ".inst 0x6e9a95d1 // udot v17.4s, v14.16b, v26.16b\n" - ".inst 0x6e9597f9 // udot v25.4s, v31.16b, v21.16b\n" - "ext v21.16b, v21.16b, v21.16b, #0x1\n" - ".inst 0x6e9a97fe // udot v30.4s, v31.16b, v26.16b\n" - ".inst 0x6e9597f4 // udot v20.4s, v31.16b, v21.16b\n" - "sqrdmulh v5.4s, v5.4s, v9.4s\n" - "mov v16.16b, v17.16b\n .inst 0x6e9595d0 // udot v16.4s, v14.16b, v21.16b\n" - ".inst 0x6e8695d1 // udot v17.4s, v14.16b, v6.16b\n" - "mls v30.4s, v17.4s, v11.4s\n" - "mls v25.4s, v18.4s, v11.4s\n" - "mls v20.4s, v16.4s, v11.4s\n" - "and v0.16b, v5.16b, v4.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqrdmulh v30.4s, v30.4s, v9.4s\n" - "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqrdmulh v20.4s, v20.4s, v9.4s\n" - "sqadd v5.4s, v5.4s, v0.4s\n" - "and v16.16b, v30.16b, v4.16b\n" - "and v31.16b, v25.16b, v4.16b\n" - "and v0.16b, v20.16b, v4.16b\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqadd v30.4s, v30.4s, v16.4s\n" - "sqadd v25.4s, v25.4s, v31.4s\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "srshl v5.4s, v5.4s, v4.4s\n" - "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" - "srshl v20.4s, v20.4s, v4.4s\n" - "add v5.4s, v5.4s, v10.4s\n" - "add v30.4s, v30.4s, v10.4s\n" - "add v25.4s, v25.4s, v10.4s\n" - "add v20.4s, v20.4s, v10.4s\n" - "smax v5.4s, v5.4s, v13.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" + "ext v10.16b, v10.16b, v10.16b, #0x1\n" + "movi v18.4s, #0x0\n" + ".inst 0x6e889776 // udot v22.4s, v27.16b, v8.16b\n" + ".inst 0x6e8a9774 // udot v20.4s, v27.16b, v10.16b\n" + ".inst 0x6e8a9592 // udot v18.4s, v12.16b, v10.16b\n" + ".inst 0x6e8597b5 // udot v21.4s, v29.16b, v5.16b\n" + ".inst 0x6e859798 // udot v24.4s, v28.16b, v5.16b\n" + "ext v5.16b, v5.16b, v5.16b, #0x1\n" + ".inst 0x6e8a97b6 // udot v22.4s, v29.16b, v10.16b\n" + ".inst 0x6e8597b4 // udot v20.4s, v29.16b, v5.16b\n" + "mls v24.4s, v25.4s, v16.4s\n" + ".inst 0x6e859592 // udot v18.4s, v12.16b, v5.16b\n" + ".inst 0x6e899795 // udot v21.4s, v28.16b, v9.16b\n" + "ext v9.16b, v9.16b, v9.16b, #0x1\n" + ".inst 0x6e859796 // udot v22.4s, v28.16b, v5.16b\n" + ".inst 0x6e899794 // udot v20.4s, v28.16b, v9.16b\n" + "sqrdmulh v24.4s, v24.4s, v30.4s\n" + "mov v17.16b, v18.16b\n .inst 0x6e899591 // udot v17.4s, v12.16b, v9.16b\n" + ".inst 0x6e889592 // udot v18.4s, v12.16b, v8.16b\n" + "mls v22.4s, v18.4s, v16.4s\n" + "mls v21.4s, v23.4s, v16.4s\n" + "mls v20.4s, v17.4s, v16.4s\n" + "and v17.16b, v24.16b, v26.16b\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqrdmulh v22.4s, v22.4s, v30.4s\n" + "sqrdmulh v21.4s, v21.4s, v30.4s\n" + "sqrdmulh v20.4s, v20.4s, v30.4s\n" + "sqadd v24.4s, v24.4s, v17.4s\n" + "and v19.16b, v22.16b, v26.16b\n" + "and v18.16b, v21.16b, v26.16b\n" + "and v17.16b, v20.16b, v26.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqadd v22.4s, v22.4s, v19.4s\n" + "sqadd v21.4s, v21.4s, v18.4s\n" + "sqadd v20.4s, v20.4s, v17.4s\n" + "srshl v24.4s, v24.4s, v26.4s\n" + "srshl v22.4s, v22.4s, v26.4s\n" + "srshl v21.4s, v21.4s, v26.4s\n" + "srshl v20.4s, v20.4s, v26.4s\n" + "add v24.4s, v24.4s, v14.4s\n" + "add v22.4s, v22.4s, v14.4s\n" + "add v21.4s, v21.4s, v14.4s\n" + "add v20.4s, v20.4s, v14.4s\n" + "smax v24.4s, v24.4s, v13.4s\n" + "smax v22.4s, v22.4s, v13.4s\n" + "smax v21.4s, v21.4s, v13.4s\n" "smax v20.4s, v20.4s, v13.4s\n" - "smin v5.4s, v5.4s, v12.4s\n" - "smin v30.4s, v30.4s, v12.4s\n" - "smin v25.4s, v25.4s, v12.4s\n" - "smin v20.4s, v20.4s, v12.4s\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" + "smin v24.4s, v24.4s, v11.4s\n" + "smin v22.4s, v22.4s, v11.4s\n" + "smin v21.4s, v21.4s, v11.4s\n" + "smin v20.4s, v20.4s, v11.4s\n" + "uzp1 v24.16b, v24.16b, v24.16b\n" + "uzp1 v22.16b, v22.16b, v22.16b\n" + "uzp1 v21.16b, v21.16b, v21.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" - "str s5, [x24, x27]\n" - "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "str s30, [x23, x27]\n" + "uzp1 v24.16b, v24.16b, v24.16b\n" + "str s24, [x25, x27]\n" + "uzp1 v22.16b, v22.16b, v22.16b\n" + "uzp1 v21.16b, v21.16b, v21.16b\n" + "str s22, [x24, x27]\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s25, [x22, x27]\n" - "str s20, [x21, x27]\n" + "str s21, [x23, x27]\n" + "str s20, [x22, x27]\n" "add x27, x27, #0x4\n" "beq 35f\n" "3:" // Oddments @@ -869,794 +861,794 @@ void a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst_impl( "add x10, x10, x28\n" "add x9, x9, x28\n" "add x26, x26, x28\n" - "add x25, x25, x28\n" + "add x21, x21, x28\n" "tbz %x[n_channels], #3, 7f\n" - "ldr d9, [x15], #0x8\n" - "ldr d8, [x14], #0x8\n" + "ldr d15, [x15], #0x8\n" + "ldr d25, [x14], #0x8\n" "ldr d7, [x13], #0x8\n" - "ldr d6, [x12], #0x8\n" - "ldr d4, [x10], #0x8\n" - "ldr d3, [x9], #0x8\n" - "ldr d2, [x26], #0x8\n" - "ldr d1, [x25], #0x8\n" + "ldr d8, [x12], #0x8\n" + "ldr d26, [x10], #0x8\n" + "ldr d23, [x9], #0x8\n" + "ldr d3, [x26], #0x8\n" + "ldr d10, [x21], #0x8\n" "tbz %x[n_channels], #2, 5f\n" - "ld1 { v9.s }[2], [x15], #0x4\n" - "ld1 { v8.s }[2], [x14], #0x4\n" + "ld1 { v15.s }[2], [x15], #0x4\n" + "ld1 { v25.s }[2], [x14], #0x4\n" "ld1 { v7.s }[2], [x13], #0x4\n" - "ld1 { v6.s }[2], [x12], #0x4\n" - "ld1 { v4.s }[2], [x10], #0x4\n" - "ld1 { v3.s }[2], [x9], #0x4\n" - "ld1 { v2.s }[2], [x26], #0x4\n" - "ld1 { v1.s }[2], [x25], #0x4\n" + "ld1 { v8.s }[2], [x12], #0x4\n" + "ld1 { v26.s }[2], [x10], #0x4\n" + "ld1 { v23.s }[2], [x9], #0x4\n" + "ld1 { v3.s }[2], [x26], #0x4\n" + "ld1 { v10.s }[2], [x21], #0x4\n" "tbz %x[n_channels], #1, 4f\n" - "ld1 { v9.h }[6], [x15], #0x2\n" - "ld1 { v8.h }[6], [x14], #0x2\n" + "ld1 { v15.h }[6], [x15], #0x2\n" + "ld1 { v25.h }[6], [x14], #0x2\n" "ld1 { v7.h }[6], [x13], #0x2\n" - "ld1 { v6.h }[6], [x12], #0x2\n" - "ld1 { v4.h }[6], [x10], #0x2\n" - "ld1 { v3.h }[6], [x9], #0x2\n" - "ld1 { v2.h }[6], [x26], #0x2\n" - "ld1 { v1.h }[6], [x25], #0x2\n" + "ld1 { v8.h }[6], [x12], #0x2\n" + "ld1 { v26.h }[6], [x10], #0x2\n" + "ld1 { v23.h }[6], [x9], #0x2\n" + "ld1 { v3.h }[6], [x26], #0x2\n" + "ld1 { v10.h }[6], [x21], #0x2\n" "tbz %x[n_channels], #0, 11f\n" - "ld1 { v9.b }[14], [x15], #0x1\n" - "ld1 { v8.b }[14], [x14], #0x1\n" + "ld1 { v15.b }[14], [x15], #0x1\n" + "ld1 { v25.b }[14], [x14], #0x1\n" "ld1 { v7.b }[14], [x13], #0x1\n" - "ld1 { v6.b }[14], [x12], #0x1\n" - "ld1 { v4.b }[14], [x10], #0x1\n" - "ld1 { v3.b }[14], [x9], #0x1\n" - "ld1 { v2.b }[14], [x26], #0x1\n" - "ld1 { v1.b }[14], [x25], #0x1\n" + "ld1 { v8.b }[14], [x12], #0x1\n" + "ld1 { v26.b }[14], [x10], #0x1\n" + "ld1 { v23.b }[14], [x9], #0x1\n" + "ld1 { v3.b }[14], [x26], #0x1\n" + "ld1 { v10.b }[14], [x21], #0x1\n" "b 11f\n" "4:" // Oddments: Load (A): Bit 3: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 11f\n" - "ld1 { v9.b }[12], [x15], #0x1\n" - "ld1 { v8.b }[12], [x14], #0x1\n" + "ld1 { v15.b }[12], [x15], #0x1\n" + "ld1 { v25.b }[12], [x14], #0x1\n" "ld1 { v7.b }[12], [x13], #0x1\n" - "ld1 { v6.b }[12], [x12], #0x1\n" - "ld1 { v4.b }[12], [x10], #0x1\n" - "ld1 { v3.b }[12], [x9], #0x1\n" - "ld1 { v2.b }[12], [x26], #0x1\n" - "ld1 { v1.b }[12], [x25], #0x1\n" + "ld1 { v8.b }[12], [x12], #0x1\n" + "ld1 { v26.b }[12], [x10], #0x1\n" + "ld1 { v23.b }[12], [x9], #0x1\n" + "ld1 { v3.b }[12], [x26], #0x1\n" + "ld1 { v10.b }[12], [x21], #0x1\n" "b 11f\n" "5:" // Oddments: Load (A): Bit 3: Bit 2: Unset "tbz %x[n_channels], #1, 6f\n" - "ld1 { v9.h }[4], [x15], #0x2\n" - "ld1 { v8.h }[4], [x14], #0x2\n" + "ld1 { v15.h }[4], [x15], #0x2\n" + "ld1 { v25.h }[4], [x14], #0x2\n" "ld1 { v7.h }[4], [x13], #0x2\n" - "ld1 { v6.h }[4], [x12], #0x2\n" - "ld1 { v4.h }[4], [x10], #0x2\n" - "ld1 { v3.h }[4], [x9], #0x2\n" - "ld1 { v2.h }[4], [x26], #0x2\n" - "ld1 { v1.h }[4], [x25], #0x2\n" + "ld1 { v8.h }[4], [x12], #0x2\n" + "ld1 { v26.h }[4], [x10], #0x2\n" + "ld1 { v23.h }[4], [x9], #0x2\n" + "ld1 { v3.h }[4], [x26], #0x2\n" + "ld1 { v10.h }[4], [x21], #0x2\n" "tbz %x[n_channels], #0, 11f\n" - "ld1 { v9.b }[10], [x15], #0x1\n" - "ld1 { v8.b }[10], [x14], #0x1\n" + "ld1 { v15.b }[10], [x15], #0x1\n" + "ld1 { v25.b }[10], [x14], #0x1\n" "ld1 { v7.b }[10], [x13], #0x1\n" - "ld1 { v6.b }[10], [x12], #0x1\n" - "ld1 { v4.b }[10], [x10], #0x1\n" - "ld1 { v3.b }[10], [x9], #0x1\n" - "ld1 { v2.b }[10], [x26], #0x1\n" - "ld1 { v1.b }[10], [x25], #0x1\n" + "ld1 { v8.b }[10], [x12], #0x1\n" + "ld1 { v26.b }[10], [x10], #0x1\n" + "ld1 { v23.b }[10], [x9], #0x1\n" + "ld1 { v3.b }[10], [x26], #0x1\n" + "ld1 { v10.b }[10], [x21], #0x1\n" "b 11f\n" "6:" // Oddments: Load (A): Bit 3: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 11f\n" - "ld1 { v9.b }[8], [x15], #0x1\n" - "ld1 { v8.b }[8], [x14], #0x1\n" + "ld1 { v15.b }[8], [x15], #0x1\n" + "ld1 { v25.b }[8], [x14], #0x1\n" "ld1 { v7.b }[8], [x13], #0x1\n" - "ld1 { v6.b }[8], [x12], #0x1\n" - "ld1 { v4.b }[8], [x10], #0x1\n" - "ld1 { v3.b }[8], [x9], #0x1\n" - "ld1 { v2.b }[8], [x26], #0x1\n" - "ld1 { v1.b }[8], [x25], #0x1\n" + "ld1 { v8.b }[8], [x12], #0x1\n" + "ld1 { v26.b }[8], [x10], #0x1\n" + "ld1 { v23.b }[8], [x9], #0x1\n" + "ld1 { v3.b }[8], [x26], #0x1\n" + "ld1 { v10.b }[8], [x21], #0x1\n" "b 11f\n" "7:" // Oddments: Load (A): Bit 3: Unset "tbz %x[n_channels], #2, 9f\n" - "ldr s9, [x15], #0x4\n" - "ldr s8, [x14], #0x4\n" + "ldr s15, [x15], #0x4\n" + "ldr s25, [x14], #0x4\n" "ldr s7, [x13], #0x4\n" - "ldr s6, [x12], #0x4\n" - "ldr s4, [x10], #0x4\n" - "ldr s3, [x9], #0x4\n" - "ldr s2, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s8, [x12], #0x4\n" + "ldr s26, [x10], #0x4\n" + "ldr s23, [x9], #0x4\n" + "ldr s3, [x26], #0x4\n" + "ldr s10, [x21], #0x4\n" "tbz %x[n_channels], #1, 8f\n" - "ld1 { v9.h }[2], [x15], #0x2\n" - "ld1 { v8.h }[2], [x14], #0x2\n" + "ld1 { v15.h }[2], [x15], #0x2\n" + "ld1 { v25.h }[2], [x14], #0x2\n" "ld1 { v7.h }[2], [x13], #0x2\n" - "ld1 { v6.h }[2], [x12], #0x2\n" - "ld1 { v4.h }[2], [x10], #0x2\n" - "ld1 { v3.h }[2], [x9], #0x2\n" - "ld1 { v2.h }[2], [x26], #0x2\n" - "ld1 { v1.h }[2], [x25], #0x2\n" + "ld1 { v8.h }[2], [x12], #0x2\n" + "ld1 { v26.h }[2], [x10], #0x2\n" + "ld1 { v23.h }[2], [x9], #0x2\n" + "ld1 { v3.h }[2], [x26], #0x2\n" + "ld1 { v10.h }[2], [x21], #0x2\n" "tbz %x[n_channels], #0, 11f\n" - "ld1 { v9.b }[6], [x15], #0x1\n" - "ld1 { v8.b }[6], [x14], #0x1\n" + "ld1 { v15.b }[6], [x15], #0x1\n" + "ld1 { v25.b }[6], [x14], #0x1\n" "ld1 { v7.b }[6], [x13], #0x1\n" - "ld1 { v6.b }[6], [x12], #0x1\n" - "ld1 { v4.b }[6], [x10], #0x1\n" - "ld1 { v3.b }[6], [x9], #0x1\n" - "ld1 { v2.b }[6], [x26], #0x1\n" - "ld1 { v1.b }[6], [x25], #0x1\n" + "ld1 { v8.b }[6], [x12], #0x1\n" + "ld1 { v26.b }[6], [x10], #0x1\n" + "ld1 { v23.b }[6], [x9], #0x1\n" + "ld1 { v3.b }[6], [x26], #0x1\n" + "ld1 { v10.b }[6], [x21], #0x1\n" "b 11f\n" "8:" // Oddments: Load (A): Bit 3: Unset: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 11f\n" - "ld1 { v9.b }[4], [x15], #0x1\n" - "ld1 { v8.b }[4], [x14], #0x1\n" + "ld1 { v15.b }[4], [x15], #0x1\n" + "ld1 { v25.b }[4], [x14], #0x1\n" "ld1 { v7.b }[4], [x13], #0x1\n" - "ld1 { v6.b }[4], [x12], #0x1\n" - "ld1 { v4.b }[4], [x10], #0x1\n" - "ld1 { v3.b }[4], [x9], #0x1\n" - "ld1 { v2.b }[4], [x26], #0x1\n" - "ld1 { v1.b }[4], [x25], #0x1\n" + "ld1 { v8.b }[4], [x12], #0x1\n" + "ld1 { v26.b }[4], [x10], #0x1\n" + "ld1 { v23.b }[4], [x9], #0x1\n" + "ld1 { v3.b }[4], [x26], #0x1\n" + "ld1 { v10.b }[4], [x21], #0x1\n" "b 11f\n" "9:" // Oddments: Load (A): Bit 3: Unset: Bit 2: Unset "tbz %x[n_channels], #1, 10f\n" - "ldr h9, [x15], #0x2\n" - "ldr h8, [x14], #0x2\n" + "ldr h15, [x15], #0x2\n" + "ldr h25, [x14], #0x2\n" "ldr h7, [x13], #0x2\n" - "ldr h6, [x12], #0x2\n" - "ldr h4, [x10], #0x2\n" - "ldr h3, [x9], #0x2\n" - "ldr h2, [x26], #0x2\n" - "ldr h1, [x25], #0x2\n" + "ldr h8, [x12], #0x2\n" + "ldr h26, [x10], #0x2\n" + "ldr h23, [x9], #0x2\n" + "ldr h3, [x26], #0x2\n" + "ldr h10, [x21], #0x2\n" "tbz %x[n_channels], #0, 11f\n" - "ld1 { v9.b }[2], [x15], #0x1\n" - "ld1 { v8.b }[2], [x14], #0x1\n" + "ld1 { v15.b }[2], [x15], #0x1\n" + "ld1 { v25.b }[2], [x14], #0x1\n" "ld1 { v7.b }[2], [x13], #0x1\n" - "ld1 { v6.b }[2], [x12], #0x1\n" - "ld1 { v4.b }[2], [x10], #0x1\n" - "ld1 { v3.b }[2], [x9], #0x1\n" - "ld1 { v2.b }[2], [x26], #0x1\n" - "ld1 { v1.b }[2], [x25], #0x1\n" + "ld1 { v8.b }[2], [x12], #0x1\n" + "ld1 { v26.b }[2], [x10], #0x1\n" + "ld1 { v23.b }[2], [x9], #0x1\n" + "ld1 { v3.b }[2], [x26], #0x1\n" + "ld1 { v10.b }[2], [x21], #0x1\n" "b 11f\n" "10:" // Oddments: Load (A): Bit 3: Unset: Bit 2: Unset: Bit 1: Unset - "ldr b9, [x15], #0x1\n" - "ldr b8, [x14], #0x1\n" + "ldr b15, [x15], #0x1\n" + "ldr b25, [x14], #0x1\n" "ldr b7, [x13], #0x1\n" - "ldr b6, [x12], #0x1\n" - "ldr b4, [x10], #0x1\n" - "ldr b3, [x9], #0x1\n" - "ldr b2, [x26], #0x1\n" - "ldr b1, [x25], #0x1\n" + "ldr b8, [x12], #0x1\n" + "ldr b26, [x10], #0x1\n" + "ldr b23, [x9], #0x1\n" + "ldr b3, [x26], #0x1\n" + "ldr b10, [x21], #0x1\n" "11:" // Oddments: Load (A): Bit 3: End "ldp x15, x14, [%x[inptrs], #0x40]\n" "ldp x13, x12, [%x[inptrs], #0x50]\n" "add x15, x15, x28\n" "add x14, x14, x28\n" "ldp x10, x9, [%x[inptrs], #0x60]\n" - "ldp x26, x25, [%x[inptrs], #0x70]\n" + "ldp x26, x21, [%x[inptrs], #0x70]\n" "add x13, x13, x28\n" "add x12, x12, x28\n" "add x10, x10, x28\n" "add x9, x9, x28\n" "add x26, x26, x28\n" - "add x25, x25, x28\n" + "add x21, x21, x28\n" "tbz %x[n_channels], #3, 15f\n" - "ldr d29, [x15], #0x8\n" - "ldr d28, [x14], #0x8\n" - "ldr d27, [x13], #0x8\n" - "ldr d26, [x12], #0x8\n" - "ldr d24, [x10], #0x8\n" - "ldr d23, [x9], #0x8\n" - "ldr d22, [x26], #0x8\n" - "ldr d21, [x25], #0x8\n" + "ldr d22, [x15], #0x8\n" + "ldr d19, [x14], #0x8\n" + "ldr d0, [x13], #0x8\n" + "ldr d5, [x12], #0x8\n" + "ldr d27, [x10], #0x8\n" + "ldr d24, [x9], #0x8\n" + "ldr d2, [x26], #0x8\n" + "ldr d9, [x21], #0x8\n" "tbz %x[n_channels], #2, 13f\n" - "ld1 { v29.s }[2], [x15], #0x4\n" - "ld1 { v28.s }[2], [x14], #0x4\n" - "ld1 { v27.s }[2], [x13], #0x4\n" - "ld1 { v26.s }[2], [x12], #0x4\n" - "ld1 { v24.s }[2], [x10], #0x4\n" - "ld1 { v23.s }[2], [x9], #0x4\n" - "ld1 { v22.s }[2], [x26], #0x4\n" - "ld1 { v21.s }[2], [x25], #0x4\n" + "ld1 { v22.s }[2], [x15], #0x4\n" + "ld1 { v19.s }[2], [x14], #0x4\n" + "ld1 { v0.s }[2], [x13], #0x4\n" + "ld1 { v5.s }[2], [x12], #0x4\n" + "ld1 { v27.s }[2], [x10], #0x4\n" + "ld1 { v24.s }[2], [x9], #0x4\n" + "ld1 { v2.s }[2], [x26], #0x4\n" + "ld1 { v9.s }[2], [x21], #0x4\n" "tbz %x[n_channels], #1, 12f\n" - "ld1 { v29.h }[6], [x15], #0x2\n" - "ld1 { v28.h }[6], [x14], #0x2\n" - "ld1 { v27.h }[6], [x13], #0x2\n" - "ld1 { v26.h }[6], [x12], #0x2\n" - "ld1 { v24.h }[6], [x10], #0x2\n" - "ld1 { v23.h }[6], [x9], #0x2\n" - "ld1 { v22.h }[6], [x26], #0x2\n" - "ld1 { v21.h }[6], [x25], #0x2\n" + "ld1 { v22.h }[6], [x15], #0x2\n" + "ld1 { v19.h }[6], [x14], #0x2\n" + "ld1 { v0.h }[6], [x13], #0x2\n" + "ld1 { v5.h }[6], [x12], #0x2\n" + "ld1 { v27.h }[6], [x10], #0x2\n" + "ld1 { v24.h }[6], [x9], #0x2\n" + "ld1 { v2.h }[6], [x26], #0x2\n" + "ld1 { v9.h }[6], [x21], #0x2\n" "tbz %x[n_channels], #0, 19f\n" - "ld1 { v29.b }[14], [x15], #0x1\n" - "ld1 { v28.b }[14], [x14], #0x1\n" - "ld1 { v27.b }[14], [x13], #0x1\n" - "ld1 { v26.b }[14], [x12], #0x1\n" - "ld1 { v24.b }[14], [x10], #0x1\n" - "ld1 { v23.b }[14], [x9], #0x1\n" - "ld1 { v22.b }[14], [x26], #0x1\n" - "ld1 { v21.b }[14], [x25], #0x1\n" + "ld1 { v22.b }[14], [x15], #0x1\n" + "ld1 { v19.b }[14], [x14], #0x1\n" + "ld1 { v0.b }[14], [x13], #0x1\n" + "ld1 { v5.b }[14], [x12], #0x1\n" + "ld1 { v27.b }[14], [x10], #0x1\n" + "ld1 { v24.b }[14], [x9], #0x1\n" + "ld1 { v2.b }[14], [x26], #0x1\n" + "ld1 { v9.b }[14], [x21], #0x1\n" "b 19f\n" "12:" // Oddments: Load (B): Bit 3: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 19f\n" - "ld1 { v29.b }[12], [x15], #0x1\n" - "ld1 { v28.b }[12], [x14], #0x1\n" - "ld1 { v27.b }[12], [x13], #0x1\n" - "ld1 { v26.b }[12], [x12], #0x1\n" - "ld1 { v24.b }[12], [x10], #0x1\n" - "ld1 { v23.b }[12], [x9], #0x1\n" - "ld1 { v22.b }[12], [x26], #0x1\n" - "ld1 { v21.b }[12], [x25], #0x1\n" + "ld1 { v22.b }[12], [x15], #0x1\n" + "ld1 { v19.b }[12], [x14], #0x1\n" + "ld1 { v0.b }[12], [x13], #0x1\n" + "ld1 { v5.b }[12], [x12], #0x1\n" + "ld1 { v27.b }[12], [x10], #0x1\n" + "ld1 { v24.b }[12], [x9], #0x1\n" + "ld1 { v2.b }[12], [x26], #0x1\n" + "ld1 { v9.b }[12], [x21], #0x1\n" "b 19f\n" "13:" // Oddments: Load (B): Bit 3: Bit 2: Unset "tbz %x[n_channels], #1, 14f\n" - "ld1 { v29.h }[4], [x15], #0x2\n" - "ld1 { v28.h }[4], [x14], #0x2\n" - "ld1 { v27.h }[4], [x13], #0x2\n" - "ld1 { v26.h }[4], [x12], #0x2\n" - "ld1 { v24.h }[4], [x10], #0x2\n" - "ld1 { v23.h }[4], [x9], #0x2\n" - "ld1 { v22.h }[4], [x26], #0x2\n" - "ld1 { v21.h }[4], [x25], #0x2\n" + "ld1 { v22.h }[4], [x15], #0x2\n" + "ld1 { v19.h }[4], [x14], #0x2\n" + "ld1 { v0.h }[4], [x13], #0x2\n" + "ld1 { v5.h }[4], [x12], #0x2\n" + "ld1 { v27.h }[4], [x10], #0x2\n" + "ld1 { v24.h }[4], [x9], #0x2\n" + "ld1 { v2.h }[4], [x26], #0x2\n" + "ld1 { v9.h }[4], [x21], #0x2\n" "tbz %x[n_channels], #0, 19f\n" - "ld1 { v29.b }[10], [x15], #0x1\n" - "ld1 { v28.b }[10], [x14], #0x1\n" - "ld1 { v27.b }[10], [x13], #0x1\n" - "ld1 { v26.b }[10], [x12], #0x1\n" - "ld1 { v24.b }[10], [x10], #0x1\n" - "ld1 { v23.b }[10], [x9], #0x1\n" - "ld1 { v22.b }[10], [x26], #0x1\n" - "ld1 { v21.b }[10], [x25], #0x1\n" + "ld1 { v22.b }[10], [x15], #0x1\n" + "ld1 { v19.b }[10], [x14], #0x1\n" + "ld1 { v0.b }[10], [x13], #0x1\n" + "ld1 { v5.b }[10], [x12], #0x1\n" + "ld1 { v27.b }[10], [x10], #0x1\n" + "ld1 { v24.b }[10], [x9], #0x1\n" + "ld1 { v2.b }[10], [x26], #0x1\n" + "ld1 { v9.b }[10], [x21], #0x1\n" "b 19f\n" "14:" // Oddments: Load (B): Bit 3: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 19f\n" - "ld1 { v29.b }[8], [x15], #0x1\n" - "ld1 { v28.b }[8], [x14], #0x1\n" - "ld1 { v27.b }[8], [x13], #0x1\n" - "ld1 { v26.b }[8], [x12], #0x1\n" - "ld1 { v24.b }[8], [x10], #0x1\n" - "ld1 { v23.b }[8], [x9], #0x1\n" - "ld1 { v22.b }[8], [x26], #0x1\n" - "ld1 { v21.b }[8], [x25], #0x1\n" + "ld1 { v22.b }[8], [x15], #0x1\n" + "ld1 { v19.b }[8], [x14], #0x1\n" + "ld1 { v0.b }[8], [x13], #0x1\n" + "ld1 { v5.b }[8], [x12], #0x1\n" + "ld1 { v27.b }[8], [x10], #0x1\n" + "ld1 { v24.b }[8], [x9], #0x1\n" + "ld1 { v2.b }[8], [x26], #0x1\n" + "ld1 { v9.b }[8], [x21], #0x1\n" "b 19f\n" "15:" // Oddments: Load (B): Bit 3: Unset "tbz %x[n_channels], #2, 17f\n" - "ldr s29, [x15], #0x4\n" - "ldr s28, [x14], #0x4\n" - "ldr s27, [x13], #0x4\n" - "ldr s26, [x12], #0x4\n" - "ldr s24, [x10], #0x4\n" - "ldr s23, [x9], #0x4\n" - "ldr s22, [x26], #0x4\n" - "ldr s21, [x25], #0x4\n" + "ldr s22, [x15], #0x4\n" + "ldr s19, [x14], #0x4\n" + "ldr s0, [x13], #0x4\n" + "ldr s5, [x12], #0x4\n" + "ldr s27, [x10], #0x4\n" + "ldr s24, [x9], #0x4\n" + "ldr s2, [x26], #0x4\n" + "ldr s9, [x21], #0x4\n" "tbz %x[n_channels], #1, 16f\n" - "ld1 { v29.h }[2], [x15], #0x2\n" - "ld1 { v28.h }[2], [x14], #0x2\n" - "ld1 { v27.h }[2], [x13], #0x2\n" - "ld1 { v26.h }[2], [x12], #0x2\n" - "ld1 { v24.h }[2], [x10], #0x2\n" - "ld1 { v23.h }[2], [x9], #0x2\n" - "ld1 { v22.h }[2], [x26], #0x2\n" - "ld1 { v21.h }[2], [x25], #0x2\n" + "ld1 { v22.h }[2], [x15], #0x2\n" + "ld1 { v19.h }[2], [x14], #0x2\n" + "ld1 { v0.h }[2], [x13], #0x2\n" + "ld1 { v5.h }[2], [x12], #0x2\n" + "ld1 { v27.h }[2], [x10], #0x2\n" + "ld1 { v24.h }[2], [x9], #0x2\n" + "ld1 { v2.h }[2], [x26], #0x2\n" + "ld1 { v9.h }[2], [x21], #0x2\n" "tbz %x[n_channels], #0, 19f\n" - "ld1 { v29.b }[6], [x15], #0x1\n" - "ld1 { v28.b }[6], [x14], #0x1\n" - "ld1 { v27.b }[6], [x13], #0x1\n" - "ld1 { v26.b }[6], [x12], #0x1\n" - "ld1 { v24.b }[6], [x10], #0x1\n" - "ld1 { v23.b }[6], [x9], #0x1\n" - "ld1 { v22.b }[6], [x26], #0x1\n" - "ld1 { v21.b }[6], [x25], #0x1\n" + "ld1 { v22.b }[6], [x15], #0x1\n" + "ld1 { v19.b }[6], [x14], #0x1\n" + "ld1 { v0.b }[6], [x13], #0x1\n" + "ld1 { v5.b }[6], [x12], #0x1\n" + "ld1 { v27.b }[6], [x10], #0x1\n" + "ld1 { v24.b }[6], [x9], #0x1\n" + "ld1 { v2.b }[6], [x26], #0x1\n" + "ld1 { v9.b }[6], [x21], #0x1\n" "b 19f\n" "16:" // Oddments: Load (B): Bit 3: Unset: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 19f\n" - "ld1 { v29.b }[4], [x15], #0x1\n" - "ld1 { v28.b }[4], [x14], #0x1\n" - "ld1 { v27.b }[4], [x13], #0x1\n" - "ld1 { v26.b }[4], [x12], #0x1\n" - "ld1 { v24.b }[4], [x10], #0x1\n" - "ld1 { v23.b }[4], [x9], #0x1\n" - "ld1 { v22.b }[4], [x26], #0x1\n" - "ld1 { v21.b }[4], [x25], #0x1\n" + "ld1 { v22.b }[4], [x15], #0x1\n" + "ld1 { v19.b }[4], [x14], #0x1\n" + "ld1 { v0.b }[4], [x13], #0x1\n" + "ld1 { v5.b }[4], [x12], #0x1\n" + "ld1 { v27.b }[4], [x10], #0x1\n" + "ld1 { v24.b }[4], [x9], #0x1\n" + "ld1 { v2.b }[4], [x26], #0x1\n" + "ld1 { v9.b }[4], [x21], #0x1\n" "b 19f\n" "17:" // Oddments: Load (B): Bit 3: Unset: Bit 2: Unset "tbz %x[n_channels], #1, 18f\n" - "ldr h29, [x15], #0x2\n" - "ldr h28, [x14], #0x2\n" - "ldr h27, [x13], #0x2\n" - "ldr h26, [x12], #0x2\n" - "ldr h24, [x10], #0x2\n" - "ldr h23, [x9], #0x2\n" - "ldr h22, [x26], #0x2\n" - "ldr h21, [x25], #0x2\n" + "ldr h22, [x15], #0x2\n" + "ldr h19, [x14], #0x2\n" + "ldr h0, [x13], #0x2\n" + "ldr h5, [x12], #0x2\n" + "ldr h27, [x10], #0x2\n" + "ldr h24, [x9], #0x2\n" + "ldr h2, [x26], #0x2\n" + "ldr h9, [x21], #0x2\n" "tbz %x[n_channels], #0, 19f\n" - "ld1 { v29.b }[2], [x15], #0x1\n" - "ld1 { v28.b }[2], [x14], #0x1\n" - "ld1 { v27.b }[2], [x13], #0x1\n" - "ld1 { v26.b }[2], [x12], #0x1\n" - "ld1 { v24.b }[2], [x10], #0x1\n" - "ld1 { v23.b }[2], [x9], #0x1\n" - "ld1 { v22.b }[2], [x26], #0x1\n" - "ld1 { v21.b }[2], [x25], #0x1\n" + "ld1 { v22.b }[2], [x15], #0x1\n" + "ld1 { v19.b }[2], [x14], #0x1\n" + "ld1 { v0.b }[2], [x13], #0x1\n" + "ld1 { v5.b }[2], [x12], #0x1\n" + "ld1 { v27.b }[2], [x10], #0x1\n" + "ld1 { v24.b }[2], [x9], #0x1\n" + "ld1 { v2.b }[2], [x26], #0x1\n" + "ld1 { v9.b }[2], [x21], #0x1\n" "b 19f\n" "18:" // Oddments: Load (B): Bit 3: Unset: Bit 2: Unset: Bit 1: Unset - "ldr b29, [x15], #0x1\n" - "ldr b28, [x14], #0x1\n" - "ldr b27, [x13], #0x1\n" - "ldr b26, [x12], #0x1\n" - "ldr b24, [x10], #0x1\n" - "ldr b23, [x9], #0x1\n" - "ldr b22, [x26], #0x1\n" - "ldr b21, [x25], #0x1\n" + "ldr b22, [x15], #0x1\n" + "ldr b19, [x14], #0x1\n" + "ldr b0, [x13], #0x1\n" + "ldr b5, [x12], #0x1\n" + "ldr b27, [x10], #0x1\n" + "ldr b24, [x9], #0x1\n" + "ldr b2, [x26], #0x1\n" + "ldr b9, [x21], #0x1\n" "19:" // Oddments: Load (B): Bit 3: End - "ldr q0, [%x[params], #0x10]\n" - "ldr q16, [%x[params], #0x20]\n" - "zip2 v30.16b, v4.16b, v2.16b\n" - "zip1 v4.16b, v4.16b, v2.16b\n" - "ldr q31, [%x[params], #0x30]\n" - "zip1 v2.16b, v3.16b, v1.16b\n" - "zip2 v5.16b, v9.16b, v7.16b\n" + "ldr q20, [%x[params], #0x10]\n" + "ldr q6, [%x[params], #0x20]\n" + "zip2 v1.16b, v26.16b, v3.16b\n" + "zip1 v26.16b, v26.16b, v3.16b\n" + "ldr q4, [%x[params], #0x30]\n" + "zip1 v18.16b, v23.16b, v10.16b\n" + "zip2 v30.16b, v15.16b, v7.16b\n" "cmp x20, #0x4\n" - "zip1 v9.16b, v9.16b, v7.16b\n" - "zip1 v7.16b, v8.16b, v6.16b\n" - "zip2 v6.16b, v8.16b, v6.16b\n" - "zip2 v1.16b, v3.16b, v1.16b\n" - "zip2 v3.16b, v4.16b, v2.16b\n" - "zip1 v4.16b, v4.16b, v2.16b\n" - "zip2 v25.16b, v29.16b, v27.16b\n" - "zip1 v29.16b, v29.16b, v27.16b\n" - "zip1 v27.16b, v28.16b, v26.16b\n" - "movi v19.4s, #0x0\n" - ".inst 0x6e8495d3 // udot v19.4s, v14.16b, v4.16b\n" - "zip2 v8.16b, v9.16b, v7.16b\n" - "zip1 v9.16b, v9.16b, v7.16b\n" - "zip1 v7.16b, v5.16b, v6.16b\n" - "zip2 v6.16b, v5.16b, v6.16b\n" - "ldr q5, [%x[params], #0x0]\n" - "zip2 v26.16b, v28.16b, v26.16b\n" - "zip2 v20.16b, v24.16b, v22.16b\n" - "zip1 v24.16b, v24.16b, v22.16b\n" - "zip1 v22.16b, v23.16b, v21.16b\n" - "zip2 v21.16b, v23.16b, v21.16b\n" - "zip2 v28.16b, v29.16b, v27.16b\n" - "zip1 v29.16b, v29.16b, v27.16b\n" - "zip1 v2.16b, v30.16b, v1.16b\n" - ".inst 0x6e9d95d3 // udot v19.4s, v14.16b, v29.16b\n" - "zip2 v1.16b, v30.16b, v1.16b\n" - "zip1 v27.16b, v25.16b, v26.16b\n" - "zip2 v26.16b, v25.16b, v26.16b\n" - "zip2 v23.16b, v24.16b, v22.16b\n" - "zip1 v24.16b, v24.16b, v22.16b\n" - "zip1 v22.16b, v20.16b, v21.16b\n" - "mov v18.16b, v19.16b\n .inst 0x6e9895d2 // udot v18.4s, v14.16b, v24.16b\n" - "zip2 v21.16b, v20.16b, v21.16b\n" - "mov v30.16b, v5.16b\n" - ".inst 0x6e8995d3 // udot v19.4s, v14.16b, v9.16b\n" - "mov v25.16b, v5.16b\n" - "mov v20.16b, v5.16b\n" - ".inst 0x6e899405 // udot v5.4s, v0.16b, v9.16b\n" - ".inst 0x6e849419 // udot v25.4s, v0.16b, v4.16b\n" - ".inst 0x6e849605 // udot v5.4s, v16.16b, v4.16b\n" - "ext v4.16b, v4.16b, v4.16b, #0x1\n" - "ext v9.16b, v9.16b, v9.16b, #0x1\n" - ".inst 0x6e9d9619 // udot v25.4s, v16.16b, v29.16b\n" - ".inst 0x6e9d97e5 // udot v5.4s, v31.16b, v29.16b\n" - "ext v29.16b, v29.16b, v29.16b, #0x1\n" - ".inst 0x6e89941e // udot v30.4s, v0.16b, v9.16b\n" - ".inst 0x6e849414 // udot v20.4s, v0.16b, v4.16b\n" + "zip1 v15.16b, v15.16b, v7.16b\n" + "zip1 v29.16b, v25.16b, v8.16b\n" + "zip2 v8.16b, v25.16b, v8.16b\n" + "zip2 v10.16b, v23.16b, v10.16b\n" + "zip2 v23.16b, v26.16b, v18.16b\n" + "zip1 v26.16b, v26.16b, v18.16b\n" + "zip2 v28.16b, v22.16b, v0.16b\n" + "zip1 v22.16b, v22.16b, v0.16b\n" + "zip1 v21.16b, v19.16b, v5.16b\n" "movi v17.4s, #0x0\n" - ".inst 0x6e8495d1 // udot v17.4s, v14.16b, v4.16b\n" - ".inst 0x6e9d95d1 // udot v17.4s, v14.16b, v29.16b\n" - ".inst 0x6e9897f9 // udot v25.4s, v31.16b, v24.16b\n" - "ext v24.16b, v24.16b, v24.16b, #0x1\n" - ".inst 0x6e84961e // udot v30.4s, v16.16b, v4.16b\n" - "ldr q4, [%x[params], #0x50]\n" - ".inst 0x6e9d9614 // udot v20.4s, v16.16b, v29.16b\n" - "mov v16.16b, v17.16b\n .inst 0x6e9895d0 // udot v16.4s, v14.16b, v24.16b\n" - "mls v5.4s, v19.4s, v11.4s\n" - ".inst 0x6e8995d1 // udot v17.4s, v14.16b, v9.16b\n" - "ldr q9, [%x[params], #0x40]\n" - ".inst 0x6e9d97fe // udot v30.4s, v31.16b, v29.16b\n" - "sqrdmulh v5.4s, v5.4s, v9.4s\n" - ".inst 0x6e9897f4 // udot v20.4s, v31.16b, v24.16b\n" - "mls v30.4s, v17.4s, v11.4s\n" + ".inst 0x6e9a9591 // udot v17.4s, v12.16b, v26.16b\n" + "zip2 v25.16b, v15.16b, v29.16b\n" + "zip1 v15.16b, v15.16b, v29.16b\n" + "zip1 v7.16b, v30.16b, v8.16b\n" + "zip2 v8.16b, v30.16b, v8.16b\n" + "ldr q31, [%x[params], #0x0]\n" + "zip2 v5.16b, v19.16b, v5.16b\n" + "zip2 v30.16b, v27.16b, v2.16b\n" + "zip1 v27.16b, v27.16b, v2.16b\n" + "zip1 v18.16b, v24.16b, v9.16b\n" + "zip2 v9.16b, v24.16b, v9.16b\n" + "zip2 v19.16b, v22.16b, v21.16b\n" + "zip1 v22.16b, v22.16b, v21.16b\n" + "zip1 v3.16b, v1.16b, v10.16b\n" + ".inst 0x6e969591 // udot v17.4s, v12.16b, v22.16b\n" + "zip2 v10.16b, v1.16b, v10.16b\n" + "zip1 v0.16b, v28.16b, v5.16b\n" + "zip2 v5.16b, v28.16b, v5.16b\n" + "zip2 v24.16b, v27.16b, v18.16b\n" + "zip1 v27.16b, v27.16b, v18.16b\n" + "zip1 v2.16b, v30.16b, v9.16b\n" + "mov v18.16b, v17.16b\n .inst 0x6e9b9592 // udot v18.4s, v12.16b, v27.16b\n" + "zip2 v9.16b, v30.16b, v9.16b\n" + "mov v30.16b, v31.16b\n" + ".inst 0x6e8f9591 // udot v17.4s, v12.16b, v15.16b\n" + "mov v29.16b, v31.16b\n" + "mov v28.16b, v31.16b\n" + ".inst 0x6e8f969f // udot v31.4s, v20.16b, v15.16b\n" + ".inst 0x6e9a969d // udot v29.4s, v20.16b, v26.16b\n" + ".inst 0x6e9a94df // udot v31.4s, v6.16b, v26.16b\n" + "ext v26.16b, v26.16b, v26.16b, #0x1\n" + "movi v1.4s, #0x0\n" + "ext v15.16b, v15.16b, v15.16b, #0x1\n" + ".inst 0x6e9a9581 // udot v1.4s, v12.16b, v26.16b\n" + ".inst 0x6e9694dd // udot v29.4s, v6.16b, v22.16b\n" + ".inst 0x6e96949f // udot v31.4s, v4.16b, v22.16b\n" + "ext v22.16b, v22.16b, v22.16b, #0x1\n" + ".inst 0x6e8f969e // udot v30.4s, v20.16b, v15.16b\n" + ".inst 0x6e9a969c // udot v28.4s, v20.16b, v26.16b\n" + "mls v31.4s, v17.4s, v16.4s\n" + ".inst 0x6e969581 // udot v1.4s, v12.16b, v22.16b\n" + ".inst 0x6e9b949d // udot v29.4s, v4.16b, v27.16b\n" + "ext v27.16b, v27.16b, v27.16b, #0x1\n" + ".inst 0x6e9a94de // udot v30.4s, v6.16b, v26.16b\n" + "ldr q21, [%x[params], #0x50]\n" + ".inst 0x6e9694dc // udot v28.4s, v6.16b, v22.16b\n" + "mls v29.4s, v18.4s, v16.4s\n" + "mov v20.16b, v1.16b\n .inst 0x6e9b9594 // udot v20.4s, v12.16b, v27.16b\n" + ".inst 0x6e8f9581 // udot v1.4s, v12.16b, v15.16b\n" + "ldr q18, [%x[params], #0x40]\n" + "sqrdmulh v31.4s, v31.4s, v18.4s\n" + ".inst 0x6e96949e // udot v30.4s, v4.16b, v22.16b\n" + ".inst 0x6e9b949c // udot v28.4s, v4.16b, v27.16b\n" + "mls v30.4s, v1.4s, v16.4s\n" "add %x[params], %x[params], #0x60\n" - "mls v25.4s, v18.4s, v11.4s\n" - "mls v20.4s, v16.4s, v11.4s\n" - "and v0.16b, v5.16b, v4.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqrdmulh v30.4s, v30.4s, v9.4s\n" - "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqrdmulh v20.4s, v20.4s, v9.4s\n" - "sqadd v5.4s, v5.4s, v0.4s\n" - "and v16.16b, v30.16b, v4.16b\n" - "and v31.16b, v25.16b, v4.16b\n" - "and v0.16b, v20.16b, v4.16b\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqadd v30.4s, v30.4s, v16.4s\n" - "sqadd v25.4s, v25.4s, v31.4s\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "srshl v5.4s, v5.4s, v4.4s\n" - "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" - "srshl v20.4s, v20.4s, v4.4s\n" - "add v5.4s, v5.4s, v10.4s\n" - "add v30.4s, v30.4s, v10.4s\n" - "add v25.4s, v25.4s, v10.4s\n" - "add v20.4s, v20.4s, v10.4s\n" - "smax v5.4s, v5.4s, v13.4s\n" + "mls v28.4s, v20.4s, v16.4s\n" + "and v17.16b, v31.16b, v21.16b\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqrdmulh v30.4s, v30.4s, v18.4s\n" + "sqrdmulh v29.4s, v29.4s, v18.4s\n" + "sqrdmulh v28.4s, v28.4s, v18.4s\n" + "sqadd v31.4s, v31.4s, v17.4s\n" + "and v17.16b, v30.16b, v21.16b\n" + "and v18.16b, v29.16b, v21.16b\n" + "and v26.16b, v28.16b, v21.16b\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v26.4s, v26.4s, #0x1f\n" + "sqadd v30.4s, v30.4s, v17.4s\n" + "sqadd v29.4s, v29.4s, v18.4s\n" + "sqadd v28.4s, v28.4s, v26.4s\n" + "srshl v31.4s, v31.4s, v21.4s\n" + "srshl v30.4s, v30.4s, v21.4s\n" + "srshl v29.4s, v29.4s, v21.4s\n" + "srshl v28.4s, v28.4s, v21.4s\n" + "add v31.4s, v31.4s, v14.4s\n" + "add v30.4s, v30.4s, v14.4s\n" + "add v29.4s, v29.4s, v14.4s\n" + "add v28.4s, v28.4s, v14.4s\n" + "smax v31.4s, v31.4s, v13.4s\n" "smax v30.4s, v30.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smin v5.4s, v5.4s, v12.4s\n" - "smin v30.4s, v30.4s, v12.4s\n" - "smin v25.4s, v25.4s, v12.4s\n" - "smin v20.4s, v20.4s, v12.4s\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" + "smax v29.4s, v29.4s, v13.4s\n" + "smax v28.4s, v28.4s, v13.4s\n" + "smin v31.4s, v31.4s, v11.4s\n" + "smin v30.4s, v30.4s, v11.4s\n" + "smin v29.4s, v29.4s, v11.4s\n" + "smin v28.4s, v28.4s, v11.4s\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" "blt 20f\n" - "str s5, [x24, x27]\n" - "str s30, [x23, x27]\n" - "str s25, [x22, x27]\n" - "str s20, [x21, x27]\n" + "str s31, [x25, x27]\n" + "str s30, [x24, x27]\n" + "str s29, [x23, x27]\n" + "str s28, [x22, x27]\n" "b 23f\n" "20:" // Oddments: Unroll 0: Oddment store + "add x25, x25, x27\n" "add x24, x24, x27\n" "add x23, x23, x27\n" "add x22, x22, x27\n" - "add x21, x21, x27\n" "tbz x20, #1, 21f\n" - "st1 { v5.h }[0], [x24], #0x2\n" - "st1 { v30.h }[0], [x23], #0x2\n" - "st1 { v25.h }[0], [x22], #0x2\n" - "st1 { v20.h }[0], [x21], #0x2\n" + "st1 { v31.h }[0], [x25], #0x2\n" + "st1 { v30.h }[0], [x24], #0x2\n" + "st1 { v29.h }[0], [x23], #0x2\n" + "st1 { v28.h }[0], [x22], #0x2\n" "tbz x20, #0, 22f\n" - "st1 { v5.b }[2], [x24], #0x1\n" - "st1 { v30.b }[2], [x23], #0x1\n" - "st1 { v25.b }[2], [x22], #0x1\n" - "st1 { v20.b }[2], [x21], #0x1\n" + "st1 { v31.b }[2], [x25], #0x1\n" + "st1 { v30.b }[2], [x24], #0x1\n" + "st1 { v29.b }[2], [x23], #0x1\n" + "st1 { v28.b }[2], [x22], #0x1\n" "b 22f\n" "21:" // Oddments: Unroll 0: Oddment store: Bit 1: Unset - "st1 { v5.b }[0], [x24], #0x1\n" - "st1 { v30.b }[0], [x23], #0x1\n" - "st1 { v25.b }[0], [x22], #0x1\n" - "st1 { v20.b }[0], [x21], #0x1\n" + "st1 { v31.b }[0], [x25], #0x1\n" + "st1 { v30.b }[0], [x24], #0x1\n" + "st1 { v29.b }[0], [x23], #0x1\n" + "st1 { v28.b }[0], [x22], #0x1\n" "22:" // Oddments: Unroll 0: Oddment store: Bit 1: End "23:" // Oddments: Unroll 0: After oddment store "subs x20, x20, #0x4\n" "add x27, x27, #0x4\n" "ble 35f\n" - "ldr q5, [%x[params], #0x0]\n" - "ldr q0, [%x[params], #0x10]\n" - "movi v19.4s, #0x0\n" - ".inst 0x6e8395d3 // udot v19.4s, v14.16b, v3.16b\n" - "ldr q16, [%x[params], #0x20]\n" - "ldr q31, [%x[params], #0x30]\n" - "mov v30.16b, v5.16b\n" - "mov v25.16b, v5.16b\n" - "ldr q9, [%x[params], #0x40]\n" - "ldr q4, [%x[params], #0x50]\n" - "mov v20.16b, v5.16b\n" - ".inst 0x6e889405 // udot v5.4s, v0.16b, v8.16b\n" - ".inst 0x6e9c95d3 // udot v19.4s, v14.16b, v28.16b\n" - ".inst 0x6e839419 // udot v25.4s, v0.16b, v3.16b\n" - "movi v17.4s, #0x0\n" + "ldr q31, [%x[params], #0x0]\n" + "ldr q27, [%x[params], #0x10]\n" + "movi v1.4s, #0x0\n" + ".inst 0x6e979581 // udot v1.4s, v12.16b, v23.16b\n" + "ldr q26, [%x[params], #0x20]\n" + "ldr q22, [%x[params], #0x30]\n" + "mov v30.16b, v31.16b\n" + "mov v29.16b, v31.16b\n" + "ldr q4, [%x[params], #0x40]\n" + "ldr q21, [%x[params], #0x50]\n" + "mov v28.16b, v31.16b\n" + ".inst 0x6e99977f // udot v31.4s, v27.16b, v25.16b\n" + ".inst 0x6e939581 // udot v1.4s, v12.16b, v19.16b\n" + ".inst 0x6e97977d // udot v29.4s, v27.16b, v23.16b\n" + "movi v20.4s, #0x0\n" "cmp x20, #0x4\n" - ".inst 0x6e839605 // udot v5.4s, v16.16b, v3.16b\n" - "mov v18.16b, v19.16b\n .inst 0x6e9795d2 // udot v18.4s, v14.16b, v23.16b\n" - "ext v3.16b, v3.16b, v3.16b, #0x1\n" - "add %x[params], %x[params], #0x60\n" - ".inst 0x6e8895d3 // udot v19.4s, v14.16b, v8.16b\n" - "ext v8.16b, v8.16b, v8.16b, #0x1\n" - ".inst 0x6e88941e // udot v30.4s, v0.16b, v8.16b\n" - ".inst 0x6e839414 // udot v20.4s, v0.16b, v3.16b\n" - ".inst 0x6e8395d1 // udot v17.4s, v14.16b, v3.16b\n" - ".inst 0x6e9c9619 // udot v25.4s, v16.16b, v28.16b\n" - ".inst 0x6e9c97e5 // udot v5.4s, v31.16b, v28.16b\n" - "ext v28.16b, v28.16b, v28.16b, #0x1\n" - ".inst 0x6e83961e // udot v30.4s, v16.16b, v3.16b\n" - ".inst 0x6e9c9614 // udot v20.4s, v16.16b, v28.16b\n" - "mls v5.4s, v19.4s, v11.4s\n" - ".inst 0x6e9c95d1 // udot v17.4s, v14.16b, v28.16b\n" - ".inst 0x6e9797f9 // udot v25.4s, v31.16b, v23.16b\n" + ".inst 0x6e97975f // udot v31.4s, v26.16b, v23.16b\n" + "mov v18.16b, v1.16b\n .inst 0x6e989592 // udot v18.4s, v12.16b, v24.16b\n" "ext v23.16b, v23.16b, v23.16b, #0x1\n" - ".inst 0x6e9c97fe // udot v30.4s, v31.16b, v28.16b\n" - ".inst 0x6e9797f4 // udot v20.4s, v31.16b, v23.16b\n" - "sqrdmulh v5.4s, v5.4s, v9.4s\n" - "mov v16.16b, v17.16b\n .inst 0x6e9795d0 // udot v16.4s, v14.16b, v23.16b\n" - ".inst 0x6e8895d1 // udot v17.4s, v14.16b, v8.16b\n" - "mls v30.4s, v17.4s, v11.4s\n" - "mls v25.4s, v18.4s, v11.4s\n" - "mls v20.4s, v16.4s, v11.4s\n" - "and v0.16b, v5.16b, v4.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqrdmulh v30.4s, v30.4s, v9.4s\n" - "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqrdmulh v20.4s, v20.4s, v9.4s\n" - "sqadd v5.4s, v5.4s, v0.4s\n" - "and v16.16b, v30.16b, v4.16b\n" - "and v31.16b, v25.16b, v4.16b\n" - "and v0.16b, v20.16b, v4.16b\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqadd v30.4s, v30.4s, v16.4s\n" - "sqadd v25.4s, v25.4s, v31.4s\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "srshl v5.4s, v5.4s, v4.4s\n" - "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" - "srshl v20.4s, v20.4s, v4.4s\n" - "add v5.4s, v5.4s, v10.4s\n" - "add v30.4s, v30.4s, v10.4s\n" - "add v25.4s, v25.4s, v10.4s\n" - "add v20.4s, v20.4s, v10.4s\n" - "smax v5.4s, v5.4s, v13.4s\n" + "add %x[params], %x[params], #0x60\n" + ".inst 0x6e999581 // udot v1.4s, v12.16b, v25.16b\n" + "ext v25.16b, v25.16b, v25.16b, #0x1\n" + ".inst 0x6e99977e // udot v30.4s, v27.16b, v25.16b\n" + ".inst 0x6e97977c // udot v28.4s, v27.16b, v23.16b\n" + ".inst 0x6e979594 // udot v20.4s, v12.16b, v23.16b\n" + ".inst 0x6e93975d // udot v29.4s, v26.16b, v19.16b\n" + ".inst 0x6e9396df // udot v31.4s, v22.16b, v19.16b\n" + "ext v19.16b, v19.16b, v19.16b, #0x1\n" + ".inst 0x6e97975e // udot v30.4s, v26.16b, v23.16b\n" + ".inst 0x6e93975c // udot v28.4s, v26.16b, v19.16b\n" + "mls v31.4s, v1.4s, v16.4s\n" + ".inst 0x6e939594 // udot v20.4s, v12.16b, v19.16b\n" + ".inst 0x6e9896dd // udot v29.4s, v22.16b, v24.16b\n" + "ext v24.16b, v24.16b, v24.16b, #0x1\n" + ".inst 0x6e9396de // udot v30.4s, v22.16b, v19.16b\n" + ".inst 0x6e9896dc // udot v28.4s, v22.16b, v24.16b\n" + "sqrdmulh v31.4s, v31.4s, v4.4s\n" + "mov v17.16b, v20.16b\n .inst 0x6e989591 // udot v17.4s, v12.16b, v24.16b\n" + ".inst 0x6e999594 // udot v20.4s, v12.16b, v25.16b\n" + "mls v30.4s, v20.4s, v16.4s\n" + "mls v29.4s, v18.4s, v16.4s\n" + "mls v28.4s, v17.4s, v16.4s\n" + "and v17.16b, v31.16b, v21.16b\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqrdmulh v30.4s, v30.4s, v4.4s\n" + "sqrdmulh v29.4s, v29.4s, v4.4s\n" + "sqrdmulh v28.4s, v28.4s, v4.4s\n" + "sqadd v31.4s, v31.4s, v17.4s\n" + "and v19.16b, v30.16b, v21.16b\n" + "and v18.16b, v29.16b, v21.16b\n" + "and v17.16b, v28.16b, v21.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqadd v30.4s, v30.4s, v19.4s\n" + "sqadd v29.4s, v29.4s, v18.4s\n" + "sqadd v28.4s, v28.4s, v17.4s\n" + "srshl v31.4s, v31.4s, v21.4s\n" + "srshl v30.4s, v30.4s, v21.4s\n" + "srshl v29.4s, v29.4s, v21.4s\n" + "srshl v28.4s, v28.4s, v21.4s\n" + "add v31.4s, v31.4s, v14.4s\n" + "add v30.4s, v30.4s, v14.4s\n" + "add v29.4s, v29.4s, v14.4s\n" + "add v28.4s, v28.4s, v14.4s\n" + "smax v31.4s, v31.4s, v13.4s\n" "smax v30.4s, v30.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smin v5.4s, v5.4s, v12.4s\n" - "smin v30.4s, v30.4s, v12.4s\n" - "smin v25.4s, v25.4s, v12.4s\n" - "smin v20.4s, v20.4s, v12.4s\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" + "smax v29.4s, v29.4s, v13.4s\n" + "smax v28.4s, v28.4s, v13.4s\n" + "smin v31.4s, v31.4s, v11.4s\n" + "smin v30.4s, v30.4s, v11.4s\n" + "smin v29.4s, v29.4s, v11.4s\n" + "smin v28.4s, v28.4s, v11.4s\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" "blt 24f\n" - "str s5, [x24, x27]\n" - "str s30, [x23, x27]\n" - "str s25, [x22, x27]\n" - "str s20, [x21, x27]\n" + "str s31, [x25, x27]\n" + "str s30, [x24, x27]\n" + "str s29, [x23, x27]\n" + "str s28, [x22, x27]\n" "b 27f\n" "24:" // Oddments: Unroll 1: Oddment store + "add x25, x25, x27\n" "add x24, x24, x27\n" "add x23, x23, x27\n" "add x22, x22, x27\n" - "add x21, x21, x27\n" "tbz x20, #1, 25f\n" - "st1 { v5.h }[0], [x24], #0x2\n" - "st1 { v30.h }[0], [x23], #0x2\n" - "st1 { v25.h }[0], [x22], #0x2\n" - "st1 { v20.h }[0], [x21], #0x2\n" + "st1 { v31.h }[0], [x25], #0x2\n" + "st1 { v30.h }[0], [x24], #0x2\n" + "st1 { v29.h }[0], [x23], #0x2\n" + "st1 { v28.h }[0], [x22], #0x2\n" "tbz x20, #0, 26f\n" - "st1 { v5.b }[2], [x24], #0x1\n" - "st1 { v30.b }[2], [x23], #0x1\n" - "st1 { v25.b }[2], [x22], #0x1\n" - "st1 { v20.b }[2], [x21], #0x1\n" + "st1 { v31.b }[2], [x25], #0x1\n" + "st1 { v30.b }[2], [x24], #0x1\n" + "st1 { v29.b }[2], [x23], #0x1\n" + "st1 { v28.b }[2], [x22], #0x1\n" "b 26f\n" "25:" // Oddments: Unroll 1: Oddment store: Bit 1: Unset - "st1 { v5.b }[0], [x24], #0x1\n" - "st1 { v30.b }[0], [x23], #0x1\n" - "st1 { v25.b }[0], [x22], #0x1\n" - "st1 { v20.b }[0], [x21], #0x1\n" + "st1 { v31.b }[0], [x25], #0x1\n" + "st1 { v30.b }[0], [x24], #0x1\n" + "st1 { v29.b }[0], [x23], #0x1\n" + "st1 { v28.b }[0], [x22], #0x1\n" "26:" // Oddments: Unroll 1: Oddment store: Bit 1: End "27:" // Oddments: Unroll 1: After oddment store "subs x20, x20, #0x4\n" "add x27, x27, #0x4\n" "ble 35f\n" - "ldr q5, [%x[params], #0x0]\n" - "ldr q0, [%x[params], #0x10]\n" + "ldr q31, [%x[params], #0x0]\n" + "ldr q25, [%x[params], #0x10]\n" + "movi v24.4s, #0x0\n" + ".inst 0x6e839598 // udot v24.4s, v12.16b, v3.16b\n" + "ldr q23, [%x[params], #0x20]\n" + "ldr q22, [%x[params], #0x30]\n" + "mov v30.16b, v31.16b\n" + "mov v29.16b, v31.16b\n" + "ldr q21, [%x[params], #0x40]\n" + "ldr q20, [%x[params], #0x50]\n" + "mov v28.16b, v31.16b\n" + ".inst 0x6e87973f // udot v31.4s, v25.16b, v7.16b\n" + ".inst 0x6e809598 // udot v24.4s, v12.16b, v0.16b\n" + ".inst 0x6e83973d // udot v29.4s, v25.16b, v3.16b\n" "movi v19.4s, #0x0\n" - ".inst 0x6e8295d3 // udot v19.4s, v14.16b, v2.16b\n" - "ldr q16, [%x[params], #0x20]\n" - "ldr q31, [%x[params], #0x30]\n" - "mov v30.16b, v5.16b\n" - "mov v25.16b, v5.16b\n" - "ldr q9, [%x[params], #0x40]\n" - "ldr q4, [%x[params], #0x50]\n" - "mov v20.16b, v5.16b\n" - ".inst 0x6e879405 // udot v5.4s, v0.16b, v7.16b\n" - ".inst 0x6e9b95d3 // udot v19.4s, v14.16b, v27.16b\n" - ".inst 0x6e829419 // udot v25.4s, v0.16b, v2.16b\n" - "movi v17.4s, #0x0\n" "cmp x20, #0x4\n" - ".inst 0x6e829605 // udot v5.4s, v16.16b, v2.16b\n" - "mov v18.16b, v19.16b\n .inst 0x6e9695d2 // udot v18.4s, v14.16b, v22.16b\n" - "ext v2.16b, v2.16b, v2.16b, #0x1\n" + ".inst 0x6e8396ff // udot v31.4s, v23.16b, v3.16b\n" + "mov v18.16b, v24.16b\n .inst 0x6e829592 // udot v18.4s, v12.16b, v2.16b\n" + "ext v3.16b, v3.16b, v3.16b, #0x1\n" "add %x[params], %x[params], #0x60\n" - ".inst 0x6e8795d3 // udot v19.4s, v14.16b, v7.16b\n" + ".inst 0x6e879598 // udot v24.4s, v12.16b, v7.16b\n" "ext v7.16b, v7.16b, v7.16b, #0x1\n" - ".inst 0x6e87941e // udot v30.4s, v0.16b, v7.16b\n" - ".inst 0x6e829414 // udot v20.4s, v0.16b, v2.16b\n" - ".inst 0x6e8295d1 // udot v17.4s, v14.16b, v2.16b\n" - ".inst 0x6e9b9619 // udot v25.4s, v16.16b, v27.16b\n" - ".inst 0x6e9b97e5 // udot v5.4s, v31.16b, v27.16b\n" - "ext v27.16b, v27.16b, v27.16b, #0x1\n" - ".inst 0x6e82961e // udot v30.4s, v16.16b, v2.16b\n" - ".inst 0x6e9b9614 // udot v20.4s, v16.16b, v27.16b\n" - "mls v5.4s, v19.4s, v11.4s\n" - ".inst 0x6e9b95d1 // udot v17.4s, v14.16b, v27.16b\n" - ".inst 0x6e9697f9 // udot v25.4s, v31.16b, v22.16b\n" - "ext v22.16b, v22.16b, v22.16b, #0x1\n" - ".inst 0x6e9b97fe // udot v30.4s, v31.16b, v27.16b\n" - ".inst 0x6e9697f4 // udot v20.4s, v31.16b, v22.16b\n" - "sqrdmulh v5.4s, v5.4s, v9.4s\n" - "mov v16.16b, v17.16b\n .inst 0x6e9695d0 // udot v16.4s, v14.16b, v22.16b\n" - ".inst 0x6e8795d1 // udot v17.4s, v14.16b, v7.16b\n" - "mls v30.4s, v17.4s, v11.4s\n" - "mls v25.4s, v18.4s, v11.4s\n" - "mls v20.4s, v16.4s, v11.4s\n" - "and v0.16b, v5.16b, v4.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqrdmulh v30.4s, v30.4s, v9.4s\n" - "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqrdmulh v20.4s, v20.4s, v9.4s\n" - "sqadd v5.4s, v5.4s, v0.4s\n" - "and v16.16b, v30.16b, v4.16b\n" - "and v31.16b, v25.16b, v4.16b\n" - "and v0.16b, v20.16b, v4.16b\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqadd v30.4s, v30.4s, v16.4s\n" - "sqadd v25.4s, v25.4s, v31.4s\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "srshl v5.4s, v5.4s, v4.4s\n" - "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" - "srshl v20.4s, v20.4s, v4.4s\n" - "add v5.4s, v5.4s, v10.4s\n" - "add v30.4s, v30.4s, v10.4s\n" - "add v25.4s, v25.4s, v10.4s\n" - "add v20.4s, v20.4s, v10.4s\n" - "smax v5.4s, v5.4s, v13.4s\n" + ".inst 0x6e87973e // udot v30.4s, v25.16b, v7.16b\n" + ".inst 0x6e83973c // udot v28.4s, v25.16b, v3.16b\n" + ".inst 0x6e839593 // udot v19.4s, v12.16b, v3.16b\n" + ".inst 0x6e8096fd // udot v29.4s, v23.16b, v0.16b\n" + ".inst 0x6e8096df // udot v31.4s, v22.16b, v0.16b\n" + "ext v0.16b, v0.16b, v0.16b, #0x1\n" + ".inst 0x6e8396fe // udot v30.4s, v23.16b, v3.16b\n" + ".inst 0x6e8096fc // udot v28.4s, v23.16b, v0.16b\n" + "mls v31.4s, v24.4s, v16.4s\n" + ".inst 0x6e809593 // udot v19.4s, v12.16b, v0.16b\n" + ".inst 0x6e8296dd // udot v29.4s, v22.16b, v2.16b\n" + "ext v2.16b, v2.16b, v2.16b, #0x1\n" + ".inst 0x6e8096de // udot v30.4s, v22.16b, v0.16b\n" + ".inst 0x6e8296dc // udot v28.4s, v22.16b, v2.16b\n" + "sqrdmulh v31.4s, v31.4s, v21.4s\n" + "mov v17.16b, v19.16b\n .inst 0x6e829591 // udot v17.4s, v12.16b, v2.16b\n" + ".inst 0x6e879593 // udot v19.4s, v12.16b, v7.16b\n" + "mls v30.4s, v19.4s, v16.4s\n" + "mls v29.4s, v18.4s, v16.4s\n" + "mls v28.4s, v17.4s, v16.4s\n" + "and v17.16b, v31.16b, v20.16b\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqrdmulh v30.4s, v30.4s, v21.4s\n" + "sqrdmulh v29.4s, v29.4s, v21.4s\n" + "sqrdmulh v28.4s, v28.4s, v21.4s\n" + "sqadd v31.4s, v31.4s, v17.4s\n" + "and v19.16b, v30.16b, v20.16b\n" + "and v18.16b, v29.16b, v20.16b\n" + "and v17.16b, v28.16b, v20.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqadd v30.4s, v30.4s, v19.4s\n" + "sqadd v29.4s, v29.4s, v18.4s\n" + "sqadd v28.4s, v28.4s, v17.4s\n" + "srshl v31.4s, v31.4s, v20.4s\n" + "srshl v30.4s, v30.4s, v20.4s\n" + "srshl v29.4s, v29.4s, v20.4s\n" + "srshl v28.4s, v28.4s, v20.4s\n" + "add v31.4s, v31.4s, v14.4s\n" + "add v30.4s, v30.4s, v14.4s\n" + "add v29.4s, v29.4s, v14.4s\n" + "add v28.4s, v28.4s, v14.4s\n" + "smax v31.4s, v31.4s, v13.4s\n" "smax v30.4s, v30.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smin v5.4s, v5.4s, v12.4s\n" - "smin v30.4s, v30.4s, v12.4s\n" - "smin v25.4s, v25.4s, v12.4s\n" - "smin v20.4s, v20.4s, v12.4s\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" + "smax v29.4s, v29.4s, v13.4s\n" + "smax v28.4s, v28.4s, v13.4s\n" + "smin v31.4s, v31.4s, v11.4s\n" + "smin v30.4s, v30.4s, v11.4s\n" + "smin v29.4s, v29.4s, v11.4s\n" + "smin v28.4s, v28.4s, v11.4s\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" "blt 28f\n" - "str s5, [x24, x27]\n" - "str s30, [x23, x27]\n" - "str s25, [x22, x27]\n" - "str s20, [x21, x27]\n" + "str s31, [x25, x27]\n" + "str s30, [x24, x27]\n" + "str s29, [x23, x27]\n" + "str s28, [x22, x27]\n" "b 31f\n" "28:" // Oddments: Unroll 2: Oddment store + "add x25, x25, x27\n" "add x24, x24, x27\n" "add x23, x23, x27\n" "add x22, x22, x27\n" - "add x21, x21, x27\n" "tbz x20, #1, 29f\n" - "st1 { v5.h }[0], [x24], #0x2\n" - "st1 { v30.h }[0], [x23], #0x2\n" - "st1 { v25.h }[0], [x22], #0x2\n" - "st1 { v20.h }[0], [x21], #0x2\n" + "st1 { v31.h }[0], [x25], #0x2\n" + "st1 { v30.h }[0], [x24], #0x2\n" + "st1 { v29.h }[0], [x23], #0x2\n" + "st1 { v28.h }[0], [x22], #0x2\n" "tbz x20, #0, 30f\n" - "st1 { v5.b }[2], [x24], #0x1\n" - "st1 { v30.b }[2], [x23], #0x1\n" - "st1 { v25.b }[2], [x22], #0x1\n" - "st1 { v20.b }[2], [x21], #0x1\n" + "st1 { v31.b }[2], [x25], #0x1\n" + "st1 { v30.b }[2], [x24], #0x1\n" + "st1 { v29.b }[2], [x23], #0x1\n" + "st1 { v28.b }[2], [x22], #0x1\n" "b 30f\n" "29:" // Oddments: Unroll 2: Oddment store: Bit 1: Unset - "st1 { v5.b }[0], [x24], #0x1\n" - "st1 { v30.b }[0], [x23], #0x1\n" - "st1 { v25.b }[0], [x22], #0x1\n" - "st1 { v20.b }[0], [x21], #0x1\n" + "st1 { v31.b }[0], [x25], #0x1\n" + "st1 { v30.b }[0], [x24], #0x1\n" + "st1 { v29.b }[0], [x23], #0x1\n" + "st1 { v28.b }[0], [x22], #0x1\n" "30:" // Oddments: Unroll 2: Oddment store: Bit 1: End "31:" // Oddments: Unroll 2: After oddment store "subs x20, x20, #0x4\n" "add x27, x27, #0x4\n" "ble 35f\n" - "ldr q5, [%x[params], #0x0]\n" - "ldr q0, [%x[params], #0x10]\n" - "movi v19.4s, #0x0\n" - ".inst 0x6e8195d3 // udot v19.4s, v14.16b, v1.16b\n" - "ldr q16, [%x[params], #0x20]\n" - "ldr q31, [%x[params], #0x30]\n" - "mov v30.16b, v5.16b\n" - "mov v25.16b, v5.16b\n" - "ldr q9, [%x[params], #0x40]\n" - "ldr q4, [%x[params], #0x50]\n" - "mov v20.16b, v5.16b\n" - ".inst 0x6e869405 // udot v5.4s, v0.16b, v6.16b\n" - ".inst 0x6e9a95d3 // udot v19.4s, v14.16b, v26.16b\n" - ".inst 0x6e819419 // udot v25.4s, v0.16b, v1.16b\n" - "movi v17.4s, #0x0\n" + "ldr q31, [%x[params], #0x0]\n" + "ldr q23, [%x[params], #0x10]\n" + "movi v22.4s, #0x0\n" + ".inst 0x6e8a9596 // udot v22.4s, v12.16b, v10.16b\n" + "ldr q21, [%x[params], #0x20]\n" + "ldr q19, [%x[params], #0x30]\n" + "mov v30.16b, v31.16b\n" + "mov v29.16b, v31.16b\n" + "ldr q20, [%x[params], #0x40]\n" + "ldr q26, [%x[params], #0x50]\n" + "mov v28.16b, v31.16b\n" + ".inst 0x6e8896ff // udot v31.4s, v23.16b, v8.16b\n" + ".inst 0x6e859596 // udot v22.4s, v12.16b, v5.16b\n" + ".inst 0x6e8a96fd // udot v29.4s, v23.16b, v10.16b\n" + "movi v18.4s, #0x0\n" "add %x[params], %x[params], #0x60\n" - ".inst 0x6e819605 // udot v5.4s, v16.16b, v1.16b\n" - "mov v18.16b, v19.16b\n .inst 0x6e9595d2 // udot v18.4s, v14.16b, v21.16b\n" - "ext v1.16b, v1.16b, v1.16b, #0x1\n" - ".inst 0x6e8695d3 // udot v19.4s, v14.16b, v6.16b\n" - "ext v6.16b, v6.16b, v6.16b, #0x1\n" - ".inst 0x6e86941e // udot v30.4s, v0.16b, v6.16b\n" - ".inst 0x6e819414 // udot v20.4s, v0.16b, v1.16b\n" - ".inst 0x6e8195d1 // udot v17.4s, v14.16b, v1.16b\n" - ".inst 0x6e9a9619 // udot v25.4s, v16.16b, v26.16b\n" - ".inst 0x6e9a97e5 // udot v5.4s, v31.16b, v26.16b\n" - "ext v26.16b, v26.16b, v26.16b, #0x1\n" - ".inst 0x6e81961e // udot v30.4s, v16.16b, v1.16b\n" - ".inst 0x6e9a9614 // udot v20.4s, v16.16b, v26.16b\n" - "mls v5.4s, v19.4s, v11.4s\n" - ".inst 0x6e9a95d1 // udot v17.4s, v14.16b, v26.16b\n" - ".inst 0x6e9597f9 // udot v25.4s, v31.16b, v21.16b\n" - "ext v21.16b, v21.16b, v21.16b, #0x1\n" - ".inst 0x6e9a97fe // udot v30.4s, v31.16b, v26.16b\n" - ".inst 0x6e9597f4 // udot v20.4s, v31.16b, v21.16b\n" - "sqrdmulh v5.4s, v5.4s, v9.4s\n" - "mov v16.16b, v17.16b\n .inst 0x6e9595d0 // udot v16.4s, v14.16b, v21.16b\n" - ".inst 0x6e8695d1 // udot v17.4s, v14.16b, v6.16b\n" - "mls v30.4s, v17.4s, v11.4s\n" - "mls v25.4s, v18.4s, v11.4s\n" - "mls v20.4s, v16.4s, v11.4s\n" - "and v0.16b, v5.16b, v4.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqrdmulh v30.4s, v30.4s, v9.4s\n" - "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqrdmulh v20.4s, v20.4s, v9.4s\n" - "sqadd v5.4s, v5.4s, v0.4s\n" - "and v16.16b, v30.16b, v4.16b\n" - "and v31.16b, v25.16b, v4.16b\n" - "and v0.16b, v20.16b, v4.16b\n" + ".inst 0x6e8a96bf // udot v31.4s, v21.16b, v10.16b\n" + "mov v17.16b, v22.16b\n .inst 0x6e899591 // udot v17.4s, v12.16b, v9.16b\n" + "ext v10.16b, v10.16b, v10.16b, #0x1\n" + ".inst 0x6e889596 // udot v22.4s, v12.16b, v8.16b\n" + "ext v8.16b, v8.16b, v8.16b, #0x1\n" + ".inst 0x6e8896fe // udot v30.4s, v23.16b, v8.16b\n" + ".inst 0x6e8a96fc // udot v28.4s, v23.16b, v10.16b\n" + ".inst 0x6e8a9592 // udot v18.4s, v12.16b, v10.16b\n" + ".inst 0x6e8596bd // udot v29.4s, v21.16b, v5.16b\n" + ".inst 0x6e85967f // udot v31.4s, v19.16b, v5.16b\n" + "ext v5.16b, v5.16b, v5.16b, #0x1\n" + ".inst 0x6e8a96be // udot v30.4s, v21.16b, v10.16b\n" + ".inst 0x6e8596bc // udot v28.4s, v21.16b, v5.16b\n" + "mls v31.4s, v22.4s, v16.4s\n" + ".inst 0x6e859592 // udot v18.4s, v12.16b, v5.16b\n" + ".inst 0x6e89967d // udot v29.4s, v19.16b, v9.16b\n" + "ext v9.16b, v9.16b, v9.16b, #0x1\n" + ".inst 0x6e85967e // udot v30.4s, v19.16b, v5.16b\n" + ".inst 0x6e89967c // udot v28.4s, v19.16b, v9.16b\n" + "sqrdmulh v31.4s, v31.4s, v20.4s\n" + "mov v7.16b, v18.16b\n .inst 0x6e899587 // udot v7.4s, v12.16b, v9.16b\n" + ".inst 0x6e889592 // udot v18.4s, v12.16b, v8.16b\n" + "mls v30.4s, v18.4s, v16.4s\n" + "mls v29.4s, v17.4s, v16.4s\n" + "mls v28.4s, v7.4s, v16.4s\n" + "and v16.16b, v31.16b, v26.16b\n" "sshr v16.4s, v16.4s, #0x1f\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "sqadd v30.4s, v30.4s, v16.4s\n" - "sqadd v25.4s, v25.4s, v31.4s\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "srshl v5.4s, v5.4s, v4.4s\n" - "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" - "srshl v20.4s, v20.4s, v4.4s\n" - "add v5.4s, v5.4s, v10.4s\n" - "add v30.4s, v30.4s, v10.4s\n" - "add v25.4s, v25.4s, v10.4s\n" - "add v20.4s, v20.4s, v10.4s\n" - "smax v5.4s, v5.4s, v13.4s\n" + "sqrdmulh v30.4s, v30.4s, v20.4s\n" + "sqrdmulh v29.4s, v29.4s, v20.4s\n" + "sqrdmulh v28.4s, v28.4s, v20.4s\n" + "sqadd v31.4s, v31.4s, v16.4s\n" + "and v18.16b, v30.16b, v26.16b\n" + "and v17.16b, v29.16b, v26.16b\n" + "and v16.16b, v28.16b, v26.16b\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v30.4s, v30.4s, v18.4s\n" + "sqadd v29.4s, v29.4s, v17.4s\n" + "sqadd v28.4s, v28.4s, v16.4s\n" + "srshl v31.4s, v31.4s, v26.4s\n" + "srshl v30.4s, v30.4s, v26.4s\n" + "srshl v29.4s, v29.4s, v26.4s\n" + "srshl v28.4s, v28.4s, v26.4s\n" + "add v31.4s, v31.4s, v14.4s\n" + "add v30.4s, v30.4s, v14.4s\n" + "add v29.4s, v29.4s, v14.4s\n" + "add v28.4s, v28.4s, v14.4s\n" + "smax v31.4s, v31.4s, v13.4s\n" "smax v30.4s, v30.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smin v5.4s, v5.4s, v12.4s\n" - "smin v30.4s, v30.4s, v12.4s\n" - "smin v25.4s, v25.4s, v12.4s\n" - "smin v20.4s, v20.4s, v12.4s\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" + "smax v29.4s, v29.4s, v13.4s\n" + "smax v28.4s, v28.4s, v13.4s\n" + "smin v31.4s, v31.4s, v11.4s\n" + "smin v30.4s, v30.4s, v11.4s\n" + "smin v29.4s, v29.4s, v11.4s\n" + "smin v28.4s, v28.4s, v11.4s\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "uzp1 v5.16b, v5.16b, v5.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" + "uzp1 v29.16b, v29.16b, v29.16b\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" "32:" // Oddments: Unroll 3: Oddment store + "add x25, x25, x27\n" "add x24, x24, x27\n" "add x23, x23, x27\n" "add x22, x22, x27\n" - "add x21, x21, x27\n" "tbz x20, #1, 33f\n" - "st1 { v5.h }[0], [x24], #0x2\n" - "st1 { v30.h }[0], [x23], #0x2\n" - "st1 { v25.h }[0], [x22], #0x2\n" - "st1 { v20.h }[0], [x21], #0x2\n" + "st1 { v31.h }[0], [x25], #0x2\n" + "st1 { v30.h }[0], [x24], #0x2\n" + "st1 { v29.h }[0], [x23], #0x2\n" + "st1 { v28.h }[0], [x22], #0x2\n" "tbz x20, #0, 34f\n" - "st1 { v5.b }[2], [x24], #0x1\n" - "st1 { v30.b }[2], [x23], #0x1\n" - "st1 { v25.b }[2], [x22], #0x1\n" - "st1 { v20.b }[2], [x21], #0x1\n" + "st1 { v31.b }[2], [x25], #0x1\n" + "st1 { v30.b }[2], [x24], #0x1\n" + "st1 { v29.b }[2], [x23], #0x1\n" + "st1 { v28.b }[2], [x22], #0x1\n" "b 34f\n" "33:" // Oddments: Unroll 3: Oddment store: Bit 1: Unset - "st1 { v5.b }[0], [x24], #0x1\n" - "st1 { v30.b }[0], [x23], #0x1\n" - "st1 { v25.b }[0], [x22], #0x1\n" - "st1 { v20.b }[0], [x21], #0x1\n" + "st1 { v31.b }[0], [x25], #0x1\n" + "st1 { v30.b }[0], [x24], #0x1\n" + "st1 { v29.b }[0], [x23], #0x1\n" + "st1 { v28.b }[0], [x22], #0x1\n" "34:" // Oddments: Unroll 3: Oddment store: Bit 1: End "35:" // End : [params] "+&r" (params) : [inptrs] "r" (inptrs), [n_channels] "r" (n_channels), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [outptrs] "r" (outptrs), [qp] "r" (&qp) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp index 49ef5dc0d99351a2a4c68ce5b63d27e3ab39b0cf..9fc6a5bc345d9dbb67772f6d3f90d85a13a87739 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include @@ -34,15 +34,7 @@ namespace arm_conv { namespace depthwise { -void a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( - const unsigned int, - const uint8_t *const *const, - const uint8_t *const, - const int32_t *const, - const arm_gemm::Requantize32 &, - const int32_t *const, - const int32_t *const, - uint8_t *const *const); +void a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl(unsigned int, const uint8_t *const *, const uint8_t *, const int32_t *, const arm_gemm::Requantize32 &, const int32_t *, const int32_t *, uint8_t *const *); class a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirstStrategy { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp index 15bbb314134de77a734d01012b8d2786f6c21adb..26fe4c8a101ced7904a9d75520a2f2434b3f6d40 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp @@ -91,1072 +91,1072 @@ void a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( requant_muls, requant_shifts, outptrs); __asm__ __volatile__( - "ldr x6, [%x[params], %[offsetof_Params_n_channels]]\n" + "ldr x7, [%x[params], %[offsetof_Params_n_channels]]\n" "ldr x23, [%x[params], %[offsetof_Params_requant]]\n" - "lsr x7, x6, #0x3\n" + "lsr x8, x7, #0x3\n" "add x20, x23, %[offsetof_Requantize32_a_offset]\n" - "ld1r { v24.16b }, [x20]\n" + "ld1r { v14.16b }, [x20]\n" "ldr x22, [%x[params], %[offsetof_Params_outptrs]]\n" "add x21, x23, %[offsetof_Requantize32_b_offset]\n" "add x20, x23, %[offsetof_Requantize32_c_offset]\n" - "ld1r { v15.16b }, [x21]\n" - "ld1r { v14.8h }, [x20]\n" + "ld1r { v19.16b }, [x21]\n" + "ld1r { v13.8h }, [x20]\n" "add x21, x23, %[offsetof_Requantize32_minval]\n" "add x20, x23, %[offsetof_Requantize32_maxval]\n" - "ld1r { v12.8h }, [x21]\n" - "ld1r { v11.8h }, [x20]\n" - "mov x8, #0x0\n" + "ld1r { v29.8h }, [x21]\n" + "ld1r { v12.8h }, [x20]\n" "mov x17, #0x0\n" - "add x16, %x[params], %[offsetof_Params_inptrs]\n" - "ldr x15, [%x[params], %[offsetof_Params_weights]]\n" - "ldr x14, [%x[params], %[offsetof_Params_requant_muls]]\n" - "ldr x13, [%x[params], %[offsetof_Params_requant_shifts]]\n" - "ldp x12, x11, [x22, #0x0]\n" - "ldp x10, x9, [x22, #0x10]\n" - "cbz x7, 3f\n" - "ldr d0, [x15, #0x0]\n" - "ldr d1, [x15, #0x8]\n" - "subs x7, x7, #0x1\n" - "usubl v0.8h, v0.8b, v15.8b\n" - "ldr d2, [x15, #0x10]\n" - "ldr d3, [x15, #0x18]\n" - "usubl v1.8h, v1.8b, v15.8b\n" - "usubl v2.8h, v2.8b, v15.8b\n" - "ldr d4, [x15, #0x20]\n" - "ldr d5, [x15, #0x28]\n" - "usubl v3.8h, v3.8b, v15.8b\n" - "usubl v4.8h, v4.8b, v15.8b\n" - "ldr d6, [x15, #0x30]\n" - "ldr d7, [x15, #0x38]\n" - "usubl v5.8h, v5.8b, v15.8b\n" - "usubl v6.8h, v6.8b, v15.8b\n" - "ldr d8, [x15, #0x40]\n" - "ldr x28, [%x[params], %[offsetof_Params_bias]]\n" - "usubl v7.8h, v7.8b, v15.8b\n" - "usubl v8.8h, v8.8b, v15.8b\n" - "ldr q13, [x28, #0x0]\n" - "ldr q20, [x28, #0x10]\n" - "add x28, x28, #0x20\n" - "str x28, [%x[params], %[offsetof_Params_bias]]\n" - "ldp x24, x23, [x16, #0x0]\n" - "ldp x22, x21, [x16, #0x10]\n" - "mov v9.16b, v13.16b\n" - "mov v18.16b, v20.16b\n" - "ldr d31, [x24, x8]\n" - "ldr d30, [x23, x8]\n" - "mov v16.16b, v13.16b\n" - "mov v26.16b, v20.16b\n" - "ldr d29, [x22, x8]\n" - "ldr d28, [x21, x8]\n" - "mov v25.16b, v13.16b\n" - "mov v10.16b, v20.16b\n" - "ldr x20, [x16, #0x20]\n" - "ldr d27, [x20, x8]\n" - "usubl v31.8h, v31.8b, v24.8b\n" - "usubl v30.8h, v30.8b, v24.8b\n" - "usubl v29.8h, v29.8b, v24.8b\n" - "usubl v28.8h, v28.8b, v24.8b\n" - "usubl v27.8h, v27.8b, v24.8b\n" + "mov x16, #0x0\n" + "add x15, %x[params], %[offsetof_Params_inptrs]\n" + "ldr x14, [%x[params], %[offsetof_Params_weights]]\n" + "ldr x13, [%x[params], %[offsetof_Params_requant_muls]]\n" + "ldr x12, [%x[params], %[offsetof_Params_requant_shifts]]\n" + "ldp x11, x10, [x22, #0x0]\n" + "ldp x9, x28, [x22, #0x10]\n" + "cbz x8, 3f\n" + "ldr d23, [x14, #0x0]\n" + "ldr d16, [x14, #0x8]\n" + "subs x8, x8, #0x1\n" + "usubl v23.8h, v23.8b, v19.8b\n" + "ldr d1, [x14, #0x10]\n" + "ldr d5, [x14, #0x18]\n" + "usubl v16.8h, v16.8b, v19.8b\n" + "usubl v1.8h, v1.8b, v19.8b\n" + "ldr d26, [x14, #0x20]\n" + "ldr d18, [x14, #0x28]\n" + "usubl v5.8h, v5.8b, v19.8b\n" + "usubl v26.8h, v26.8b, v19.8b\n" + "ldr d31, [x14, #0x30]\n" + "ldr d25, [x14, #0x38]\n" + "usubl v18.8h, v18.8b, v19.8b\n" + "usubl v31.8h, v31.8b, v19.8b\n" + "ldr d20, [x14, #0x40]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "usubl v25.8h, v25.8b, v19.8b\n" + "usubl v20.8h, v20.8b, v19.8b\n" + "ldr q9, [x20, #0x0]\n" + "ldr q24, [x20, #0x10]\n" + "add x20, x20, #0x20\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" + "ldp x23, x22, [x15, #0x0]\n" + "ldp x21, x20, [x15, #0x10]\n" + "mov v7.16b, v9.16b\n" + "mov v0.16b, v24.16b\n" + "ldr d22, [x23, x17]\n" + "ldr d4, [x22, x17]\n" + "mov v2.16b, v9.16b\n" + "mov v30.16b, v24.16b\n" + "ldr d8, [x21, x17]\n" + "ldr d27, [x20, x17]\n" + "mov v10.16b, v9.16b\n" + "mov v6.16b, v24.16b\n" + "ldr x20, [x15, #0x20]\n" + "ldr d15, [x20, x17]\n" + "usubl v22.8h, v22.8b, v14.8b\n" + "usubl v4.8h, v4.8b, v14.8b\n" + "usubl v8.8h, v8.8b, v14.8b\n" + "usubl v27.8h, v27.8b, v14.8b\n" + "usubl v15.8h, v15.8b, v14.8b\n" "beq 2f\n" "1:" // Loop - "ldr q17, [x14, #0x0]\n" - "ldr q22, [x13, #0x0]\n" - "smlal v13.4s, v31.4h, v4.4h\n" - "smlal2 v20.4s, v31.8h, v4.8h\n" - "ldr q23, [x14, #0x10]\n" - "smlal v9.4s, v31.4h, v3.4h\n" - "smlal2 v18.4s, v31.8h, v3.8h\n" - "ldr x21, [x16, #0x28]\n" - "smlal v13.4s, v30.4h, v0.4h\n" - "smlal2 v20.4s, v30.8h, v0.8h\n" - "ldr q19, [x13, #0x10]\n" - "ldr x28, [x16, #0x38]\n" - "smlal v9.4s, v29.4h, v2.4h\n" - "smlal2 v18.4s, v29.8h, v2.8h\n" - "ldr x20, [x16, #0x30]\n" - "ldr d29, [x20, x8]\n" - "smlal v16.4s, v31.4h, v1.4h\n" - "smlal2 v26.4s, v31.8h, v1.8h\n" - "ldr x27, [x16, #0x40]\n" - "ldr x26, [x16, #0x48]\n" - "smlal v25.4s, v31.4h, v0.4h\n" - "smlal2 v10.4s, v31.8h, v0.8h\n" - "ldr d31, [x21, x8]\n" - "usubl v31.8h, v31.8b, v24.8b\n" - "smlal v13.4s, v28.4h, v5.4h\n" - "smlal2 v20.4s, v28.8h, v5.8h\n" - "usubl v29.8h, v29.8b, v24.8b\n" - "ldr x25, [x16, #0x50]\n" - "smlal v9.4s, v28.4h, v4.4h\n" - "smlal2 v18.4s, v28.8h, v4.8h\n" - "ldr x24, [x16, #0x58]\n" - "ldr x23, [x16, #0x60]\n" - "smlal v16.4s, v28.4h, v2.4h\n" - "smlal2 v26.4s, v28.8h, v2.8h\n" - "ldr x22, [x16, #0x68]\n" - "ldr x21, [x16, #0x70]\n" - "smlal v25.4s, v28.4h, v1.4h\n" - "smlal2 v10.4s, v28.8h, v1.8h\n" - "ldr d28, [x28, x8]\n" - "usubl v28.8h, v28.8b, v24.8b\n" - "smlal v13.4s, v27.4h, v7.4h\n" - "smlal2 v20.4s, v27.8h, v7.8h\n" - "ldr x20, [x16, #0x78]\n" - "ldr x28, [%x[params], %[offsetof_Params_bias]]\n" - "smlal v9.4s, v27.4h, v6.4h\n" - "smlal2 v18.4s, v27.8h, v6.8h\n" - "add x15, x15, #0x48\n" - "subs x7, x7, #0x1\n" - "smlal v16.4s, v31.4h, v6.4h\n" - "smlal2 v26.4s, v31.8h, v6.8h\n" - "ldr d31, [x27, x8]\n" - "usubl v31.8h, v31.8b, v24.8b\n" - "smlal v25.4s, v27.4h, v3.4h\n" - "smlal2 v10.4s, v27.8h, v3.8h\n" - "add x14, x14, #0x20\n" + "ldr q3, [x13, #0x0]\n" + "ldr q17, [x12, #0x0]\n" + "smlal v9.4s, v22.4h, v26.4h\n" + "smlal2 v24.4s, v22.8h, v26.8h\n" + "ldr q21, [x13, #0x10]\n" + "ldr q28, [x12, #0x10]\n" + "smlal v9.4s, v4.4h, v23.4h\n" + "smlal v7.4s, v22.4h, v5.4h\n" + "ldr x20, [x15, #0x28]\n" + "ldr d11, [x20, x17]\n" + "smlal v2.4s, v22.4h, v16.4h\n" + "smlal v10.4s, v22.4h, v23.4h\n" + "smlal2 v24.4s, v4.8h, v23.8h\n" + "ldr x20, [x15, #0x38]\n" + "ldr d4, [x20, x17]\n" + "smlal v9.4s, v27.4h, v18.4h\n" + "smlal2 v0.4s, v22.8h, v5.8h\n" + "smlal2 v30.4s, v22.8h, v16.8h\n" + "ldr x20, [x15, #0x30]\n" + "usubl v11.8h, v11.8b, v14.8b\n" + "smlal2 v6.4s, v22.8h, v23.8h\n" + "ldr d22, [x20, x17]\n" + "smlal v7.4s, v8.4h, v1.4h\n" + "ldr x20, [x15, #0x40]\n" + "smlal v2.4s, v27.4h, v1.4h\n" + "smlal v10.4s, v27.4h, v16.4h\n" + "usubl v4.8h, v4.8b, v14.8b\n" + "ldr x27, [x15, #0x48]\n" + "smlal2 v24.4s, v27.8h, v18.8h\n" + "smlal v9.4s, v15.4h, v25.4h\n" + "usubl v22.8h, v22.8b, v14.8b\n" + "ldr x26, [x15, #0x50]\n" + "smlal2 v0.4s, v8.8h, v1.8h\n" + "ldr d8, [x20, x17]\n" + "smlal2 v30.4s, v27.8h, v1.8h\n" + "usubl v8.8h, v8.8b, v14.8b\n" + "smlal2 v6.4s, v27.8h, v16.8h\n" + "smlal v7.4s, v27.4h, v26.4h\n" + "ldr x25, [x15, #0x58]\n" + "ldr x24, [x15, #0x60]\n" + "smlal v2.4s, v11.4h, v31.4h\n" + "smlal v10.4s, v15.4h, v5.4h\n" + "ldr x23, [x15, #0x68]\n" + "ldr x22, [x15, #0x70]\n" + "smlal2 v24.4s, v15.8h, v25.8h\n" + "smlal v9.4s, v4.4h, v16.4h\n" + "ldr x21, [x15, #0x78]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "smlal2 v0.4s, v27.8h, v26.8h\n" + "ldr d27, [x27, x17]\n" + "smlal2 v30.4s, v11.8h, v31.8h\n" + "ldr d11, [x26, x17]\n" + "smlal2 v6.4s, v15.8h, v5.8h\n" + "smlal v7.4s, v15.4h, v31.4h\n" + "usubl v27.8h, v27.8b, v14.8b\n" + "add x14, x14, #0x48\n" + "smlal v2.4s, v15.4h, v26.4h\n" + "smlal v10.4s, v22.4h, v20.4h\n" + "usubl v11.8h, v11.8b, v14.8b\n" + "subs x8, x8, #0x1\n" + "smlal2 v24.4s, v4.8h, v16.8h\n" + "smlal v9.4s, v8.4h, v1.4h\n" "add x13, x13, #0x20\n" - "smlal v13.4s, v28.4h, v1.4h\n" - "smlal2 v20.4s, v28.8h, v1.8h\n" - "smlal v9.4s, v28.4h, v0.4h\n" - "smlal2 v18.4s, v28.8h, v0.8h\n" - "ldr d30, [x26, x8]\n" - "usubl v30.8h, v30.8b, v24.8b\n" - "smlal v16.4s, v27.4h, v4.4h\n" - "smlal v25.4s, v29.4h, v8.4h\n" - "smlal2 v26.4s, v27.8h, v4.8h\n" - "ldr d28, [x24, x8]\n" - "smlal2 v10.4s, v29.8h, v8.8h\n" - "ldr d29, [x25, x8]\n" - "smlal v13.4s, v31.4h, v2.4h\n" - "smlal2 v20.4s, v31.8h, v2.8h\n" - "usubl v29.8h, v29.8b, v24.8b\n" - "smlal v9.4s, v31.4h, v1.4h\n" - "smlal2 v18.4s, v31.8h, v1.8h\n" - "ldr d31, [x23, x8]\n" - "usubl v28.8h, v28.8b, v24.8b\n" - "smlal v16.4s, v30.4h, v5.4h\n" - "smlal v25.4s, v30.4h, v4.4h\n" - "usubl v31.8h, v31.8b, v24.8b\n" - "smlal v13.4s, v30.4h, v8.4h\n" - "smlal2 v20.4s, v30.8h, v8.8h\n" - "smlal v9.4s, v30.4h, v7.4h\n" - "smlal2 v18.4s, v30.8h, v7.8h\n" - "smlal2 v26.4s, v30.8h, v5.8h\n" - "smlal2 v10.4s, v30.8h, v4.8h\n" - "ldr d30, [x22, x8]\n" - "usubl v30.8h, v30.8b, v24.8b\n" - "smlal v16.4s, v29.4h, v0.4h\n" - "smlal v25.4s, v28.4h, v2.4h\n" - "smlal v13.4s, v29.4h, v3.4h\n" - "smlal2 v20.4s, v29.8h, v3.8h\n" - "smlal2 v26.4s, v29.8h, v0.8h\n" - "ldr d29, [x21, x8]\n" - "smlal2 v10.4s, v28.8h, v2.8h\n" - "usubl v29.8h, v29.8b, v24.8b\n" - "smlal v16.4s, v31.4h, v3.4h\n" - "smlal v25.4s, v30.4h, v5.4h\n" - "smlal v9.4s, v28.4h, v5.4h\n" - "smlal2 v18.4s, v28.8h, v5.8h\n" - "ldr d28, [x20, x8]\n" - "usubl v28.8h, v28.8b, v24.8b\n" - "smlal v13.4s, v31.4h, v6.4h\n" - "smlal2 v26.4s, v31.8h, v3.8h\n" - "sqrdmulh v13.4s, v13.4s, v17.4s\n" - "add x8, x8, #0x8\n" - "smlal2 v10.4s, v30.8h, v5.8h\n" - "smlal v16.4s, v29.4h, v7.4h\n" - "and v21.16b, v13.16b, v22.16b\n" - "smlal v25.4s, v29.4h, v6.4h\n" - "smlal2 v20.4s, v31.8h, v6.8h\n" - "sqrdmulh v20.4s, v20.4s, v23.4s\n" - "smlal2 v26.4s, v29.8h, v7.8h\n" - "smlal2 v10.4s, v29.8h, v6.8h\n" - "sshr v21.4s, v21.4s, #0x1f\n" - "smlal v9.4s, v30.4h, v8.4h\n" - "smlal v16.4s, v28.4h, v8.4h\n" - "and v29.16b, v20.16b, v19.16b\n" - "smlal v25.4s, v28.4h, v7.4h\n" - "smlal2 v18.4s, v30.8h, v8.8h\n" - "sqrdmulh v9.4s, v9.4s, v17.4s\n" - "smlal2 v26.4s, v28.8h, v8.8h\n" - "smlal2 v10.4s, v28.8h, v7.8h\n" - "sqrdmulh v16.4s, v16.4s, v17.4s\n" - "sqrdmulh v25.4s, v25.4s, v17.4s\n" - "sqadd v13.4s, v13.4s, v21.4s\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "and v0.16b, v9.16b, v22.16b\n" - "sqrdmulh v18.4s, v18.4s, v23.4s\n" - "and v27.16b, v16.16b, v22.16b\n" - "sqrdmulh v26.4s, v26.4s, v23.4s\n" - "and v21.16b, v25.16b, v22.16b\n" - "sqrdmulh v10.4s, v10.4s, v23.4s\n" - "sqadd v20.4s, v20.4s, v29.4s\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v17.16b, v18.16b, v19.16b\n" + "add x12, x12, #0x20\n" + "smlal2 v0.4s, v15.8h, v31.8h\n" + "smlal2 v30.4s, v15.8h, v26.8h\n" + "ldr d15, [x25, x17]\n" + "usubl v15.8h, v15.8b, v14.8b\n" + "smlal2 v6.4s, v22.8h, v20.8h\n" + "ldr d22, [x24, x17]\n" + "smlal v7.4s, v4.4h, v23.4h\n" + "usubl v22.8h, v22.8b, v14.8b\n" + "smlal v2.4s, v27.4h, v18.4h\n" + "smlal v10.4s, v27.4h, v26.4h\n" + "smlal2 v24.4s, v8.8h, v1.8h\n" + "smlal v9.4s, v27.4h, v20.4h\n" + "smlal2 v0.4s, v4.8h, v23.8h\n" + "ldr d4, [x23, x17]\n" + "smlal2 v30.4s, v27.8h, v18.8h\n" + "usubl v4.8h, v4.8b, v14.8b\n" + "smlal2 v6.4s, v27.8h, v26.8h\n" + "ldr d26, [x22, x17]\n" + "smlal v7.4s, v8.4h, v16.4h\n" + "usubl v26.8h, v26.8b, v14.8b\n" + "smlal v2.4s, v11.4h, v23.4h\n" + "smlal v10.4s, v15.4h, v1.4h\n" + "smlal2 v24.4s, v27.8h, v20.8h\n" + "smlal v9.4s, v11.4h, v5.4h\n" + "smlal2 v0.4s, v8.8h, v16.8h\n" + "ldr d8, [x21, x17]\n" + "smlal2 v30.4s, v11.8h, v23.8h\n" + "usubl v8.8h, v8.8b, v14.8b\n" + "smlal2 v6.4s, v15.8h, v1.8h\n" + "smlal v7.4s, v27.4h, v25.4h\n" + "add x17, x17, #0x8\n" + "smlal v2.4s, v22.4h, v5.4h\n" + "smlal v10.4s, v4.4h, v18.4h\n" + "smlal2 v24.4s, v11.8h, v5.8h\n" + "smlal v9.4s, v22.4h, v31.4h\n" + "sqrdmulh v9.4s, v9.4s, v3.4s\n" + "smlal2 v0.4s, v27.8h, v25.8h\n" + "smlal2 v30.4s, v22.8h, v5.8h\n" + "and v27.16b, v9.16b, v17.16b\n" + "smlal2 v6.4s, v4.8h, v18.8h\n" + "smlal v7.4s, v15.4h, v18.4h\n" "sshr v27.4s, v27.4s, #0x1f\n" - "and v7.16b, v26.16b, v19.16b\n" - "sshr v21.4s, v21.4s, #0x1f\n" - "and v29.16b, v10.16b, v19.16b\n" - "sqadd v9.4s, v9.4s, v0.4s\n" - "sshr v17.4s, v17.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v27.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v25.4s, v25.4s, v21.4s\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "srshl v13.4s, v13.4s, v22.4s\n" - "srshl v9.4s, v9.4s, v22.4s\n" - "sqadd v18.4s, v18.4s, v17.4s\n" - "srshl v16.4s, v16.4s, v22.4s\n" - "sqadd v26.4s, v26.4s, v7.4s\n" - "srshl v25.4s, v25.4s, v22.4s\n" - "sqadd v10.4s, v10.4s, v29.4s\n" - "srshl v20.4s, v20.4s, v19.4s\n" - "sqxtn v13.4h, v13.4s\n" - "srshl v18.4s, v18.4s, v19.4s\n" + "smlal v2.4s, v26.4h, v25.4h\n" + "smlal v10.4s, v26.4h, v31.4h\n" + "sqadd v9.4s, v9.4s, v27.4s\n" + "smlal2 v24.4s, v22.8h, v31.8h\n" + "smlal2 v0.4s, v15.8h, v18.8h\n" + "sqrdmulh v24.4s, v24.4s, v21.4s\n" + "smlal2 v30.4s, v26.8h, v25.8h\n" + "smlal2 v6.4s, v26.8h, v31.8h\n" + "and v31.16b, v24.16b, v28.16b\n" + "smlal v7.4s, v4.4h, v20.4h\n" + "smlal v2.4s, v8.4h, v20.4h\n" + "sqrdmulh v7.4s, v7.4s, v3.4s\n" + "smlal v10.4s, v8.4h, v25.4h\n" + "smlal2 v0.4s, v4.8h, v20.8h\n" + "sqrdmulh v2.4s, v2.4s, v3.4s\n" + "smlal2 v30.4s, v8.8h, v20.8h\n" + "smlal2 v6.4s, v8.8h, v25.8h\n" + "sqrdmulh v10.4s, v10.4s, v3.4s\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "and v22.16b, v7.16b, v17.16b\n" + "sqrdmulh v0.4s, v0.4s, v21.4s\n" + "and v3.16b, v2.16b, v17.16b\n" + "sqrdmulh v30.4s, v30.4s, v21.4s\n" + "and v11.16b, v10.16b, v17.16b\n" + "sqrdmulh v6.4s, v6.4s, v21.4s\n" + "sqadd v24.4s, v24.4s, v31.4s\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "and v20.16b, v0.16b, v28.16b\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "and v31.16b, v30.16b, v28.16b\n" + "sshr v11.4s, v11.4s, #0x1f\n" + "and v18.16b, v6.16b, v28.16b\n" + "sqadd v7.4s, v7.4s, v22.4s\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "sqadd v2.4s, v2.4s, v3.4s\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "sqadd v10.4s, v10.4s, v11.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "srshl v9.4s, v9.4s, v17.4s\n" + "srshl v7.4s, v7.4s, v17.4s\n" + "sqadd v0.4s, v0.4s, v20.4s\n" + "srshl v2.4s, v2.4s, v17.4s\n" + "sqadd v30.4s, v30.4s, v31.4s\n" + "srshl v10.4s, v10.4s, v17.4s\n" + "sqadd v6.4s, v6.4s, v18.4s\n" + "srshl v24.4s, v24.4s, v28.4s\n" "sqxtn v9.4h, v9.4s\n" - "srshl v26.4s, v26.4s, v19.4s\n" - "sqxtn v16.4h, v16.4s\n" - "srshl v10.4s, v10.4s, v19.4s\n" - "sqxtn v25.4h, v25.4s\n" - "sqxtn2 v13.8h, v20.4s\n" - "sqxtn2 v9.8h, v18.4s\n" - "sqxtn2 v16.8h, v26.4s\n" - "sqxtn2 v25.8h, v10.4s\n" - "sqadd v13.8h, v13.8h, v14.8h\n" - "sqadd v9.8h, v9.8h, v14.8h\n" - "sqadd v16.8h, v16.8h, v14.8h\n" - "sqadd v25.8h, v25.8h, v14.8h\n" - "smax v13.8h, v13.8h, v12.8h\n" - "smax v9.8h, v9.8h, v12.8h\n" - "smax v16.8h, v16.8h, v12.8h\n" - "smax v25.8h, v25.8h, v12.8h\n" - "smin v13.8h, v13.8h, v11.8h\n" - "smin v9.8h, v9.8h, v11.8h\n" - "smin v16.8h, v16.8h, v11.8h\n" - "smin v25.8h, v25.8h, v11.8h\n" - "uzp1 v13.16b, v13.16b, v13.16b\n" - "str d13, [x12, x17]\n" + "srshl v0.4s, v0.4s, v28.4s\n" + "sqxtn v7.4h, v7.4s\n" + "srshl v30.4s, v30.4s, v28.4s\n" + "sqxtn v2.4h, v2.4s\n" + "srshl v6.4s, v6.4s, v28.4s\n" + "sqxtn v10.4h, v10.4s\n" + "sqxtn2 v9.8h, v24.4s\n" + "sqxtn2 v7.8h, v0.4s\n" + "sqxtn2 v2.8h, v30.4s\n" + "sqxtn2 v10.8h, v6.4s\n" + "sqadd v9.8h, v9.8h, v13.8h\n" + "sqadd v7.8h, v7.8h, v13.8h\n" + "sqadd v2.8h, v2.8h, v13.8h\n" + "sqadd v10.8h, v10.8h, v13.8h\n" + "smax v9.8h, v9.8h, v29.8h\n" + "smax v7.8h, v7.8h, v29.8h\n" + "smax v2.8h, v2.8h, v29.8h\n" + "smax v10.8h, v10.8h, v29.8h\n" + "smin v9.8h, v9.8h, v12.8h\n" + "smin v7.8h, v7.8h, v12.8h\n" + "smin v2.8h, v2.8h, v12.8h\n" + "smin v10.8h, v10.8h, v12.8h\n" "uzp1 v9.16b, v9.16b, v9.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "str d9, [x11, x17]\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "str d16, [x10, x17]\n" - "str d25, [x9, x17]\n" - "ldr q13, [x28, #0x0]\n" - "ldr q20, [x28, #0x10]\n" - "add x28, x28, #0x20\n" - "ldr d0, [x15, #0x0]\n" - "ldr d1, [x15, #0x8]\n" - "add x17, x17, #0x8\n" - "str x28, [%x[params], %[offsetof_Params_bias]]\n" - "ldr d2, [x15, #0x10]\n" - "ldr d3, [x15, #0x18]\n" - "mov v9.16b, v13.16b\n" - "mov v18.16b, v20.16b\n" - "ldr d4, [x15, #0x20]\n" - "ldr d5, [x15, #0x28]\n" - "mov v16.16b, v13.16b\n" - "mov v26.16b, v20.16b\n" - "ldr d6, [x15, #0x30]\n" - "ldr d7, [x15, #0x38]\n" - "mov v25.16b, v13.16b\n" - "mov v10.16b, v20.16b\n" - "ldr d8, [x15, #0x40]\n" - "ldp x24, x23, [x16, #0x0]\n" - "usubl v0.8h, v0.8b, v15.8b\n" - "usubl v1.8h, v1.8b, v15.8b\n" - "ldp x22, x21, [x16, #0x10]\n" - "ldr d31, [x24, x8]\n" - "usubl v2.8h, v2.8b, v15.8b\n" - "usubl v3.8h, v3.8b, v15.8b\n" - "ldr d30, [x23, x8]\n" - "ldr d29, [x22, x8]\n" - "usubl v4.8h, v4.8b, v15.8b\n" - "usubl v5.8h, v5.8b, v15.8b\n" - "ldr d28, [x21, x8]\n" - "ldr x20, [x16, #0x20]\n" - "usubl v6.8h, v6.8b, v15.8b\n" - "usubl v7.8h, v7.8b, v15.8b\n" - "ldr d27, [x20, x8]\n" - "usubl v8.8h, v8.8b, v15.8b\n" - "usubl v31.8h, v31.8b, v24.8b\n" - "usubl v30.8h, v30.8b, v24.8b\n" - "usubl v29.8h, v29.8b, v24.8b\n" - "usubl v28.8h, v28.8b, v24.8b\n" - "usubl v27.8h, v27.8b, v24.8b\n" + "str d9, [x11, x16]\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" + "uzp1 v2.16b, v2.16b, v2.16b\n" + "str d7, [x10, x16]\n" + "uzp1 v10.16b, v10.16b, v10.16b\n" + "str d2, [x9, x16]\n" + "str d10, [x28, x16]\n" + "ldr q9, [x20, #0x0]\n" + "ldr q24, [x20, #0x10]\n" + "add x20, x20, #0x20\n" + "ldr d23, [x14, #0x0]\n" + "ldr d16, [x14, #0x8]\n" + "add x16, x16, #0x8\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" + "ldr d1, [x14, #0x10]\n" + "ldr d5, [x14, #0x18]\n" + "mov v7.16b, v9.16b\n" + "mov v0.16b, v24.16b\n" + "ldr d26, [x14, #0x20]\n" + "ldr d18, [x14, #0x28]\n" + "mov v2.16b, v9.16b\n" + "mov v30.16b, v24.16b\n" + "ldr d31, [x14, #0x30]\n" + "ldr d25, [x14, #0x38]\n" + "mov v10.16b, v9.16b\n" + "mov v6.16b, v24.16b\n" + "ldr d20, [x14, #0x40]\n" + "ldp x23, x22, [x15, #0x0]\n" + "usubl v23.8h, v23.8b, v19.8b\n" + "usubl v16.8h, v16.8b, v19.8b\n" + "ldp x21, x20, [x15, #0x10]\n" + "ldr d22, [x23, x17]\n" + "usubl v1.8h, v1.8b, v19.8b\n" + "usubl v5.8h, v5.8b, v19.8b\n" + "ldr d4, [x22, x17]\n" + "ldr d8, [x21, x17]\n" + "usubl v26.8h, v26.8b, v19.8b\n" + "usubl v18.8h, v18.8b, v19.8b\n" + "ldr d27, [x20, x17]\n" + "ldr x20, [x15, #0x20]\n" + "usubl v31.8h, v31.8b, v19.8b\n" + "usubl v25.8h, v25.8b, v19.8b\n" + "ldr d15, [x20, x17]\n" + "usubl v20.8h, v20.8b, v19.8b\n" + "usubl v22.8h, v22.8b, v14.8b\n" + "usubl v4.8h, v4.8b, v14.8b\n" + "usubl v8.8h, v8.8b, v14.8b\n" + "usubl v27.8h, v27.8b, v14.8b\n" + "usubl v15.8h, v15.8b, v14.8b\n" "bgt 1b\n" "2:" // Tail - "ldr q17, [x14, #0x0]\n" - "ldr q22, [x13, #0x0]\n" - "smlal v13.4s, v31.4h, v4.4h\n" - "smlal2 v20.4s, v31.8h, v4.8h\n" - "ldr q23, [x14, #0x10]\n" - "smlal v9.4s, v31.4h, v3.4h\n" - "smlal2 v18.4s, v31.8h, v3.8h\n" - "ldr x21, [x16, #0x28]\n" - "smlal v13.4s, v30.4h, v0.4h\n" - "smlal2 v20.4s, v30.8h, v0.8h\n" - "ldr q19, [x13, #0x10]\n" - "ldr x28, [x16, #0x38]\n" - "smlal v9.4s, v29.4h, v2.4h\n" - "smlal2 v18.4s, v29.8h, v2.8h\n" - "ldr x20, [x16, #0x30]\n" - "ldr d29, [x20, x8]\n" - "smlal v16.4s, v31.4h, v1.4h\n" - "smlal2 v26.4s, v31.8h, v1.8h\n" - "ldr x27, [x16, #0x40]\n" - "ldr x26, [x16, #0x48]\n" - "smlal v25.4s, v31.4h, v0.4h\n" - "smlal2 v10.4s, v31.8h, v0.8h\n" - "ldr d31, [x21, x8]\n" - "usubl v31.8h, v31.8b, v24.8b\n" - "smlal v13.4s, v28.4h, v5.4h\n" - "smlal2 v20.4s, v28.8h, v5.8h\n" - "usubl v29.8h, v29.8b, v24.8b\n" - "ldr x25, [x16, #0x50]\n" - "smlal v9.4s, v28.4h, v4.4h\n" - "smlal2 v18.4s, v28.8h, v4.8h\n" - "ldr x24, [x16, #0x58]\n" - "ldr x23, [x16, #0x60]\n" - "smlal v16.4s, v28.4h, v2.4h\n" - "smlal2 v26.4s, v28.8h, v2.8h\n" - "ldr x22, [x16, #0x68]\n" - "ldr x21, [x16, #0x70]\n" - "smlal v25.4s, v28.4h, v1.4h\n" - "smlal2 v10.4s, v28.8h, v1.8h\n" - "ldr d28, [x28, x8]\n" - "usubl v28.8h, v28.8b, v24.8b\n" - "smlal v13.4s, v27.4h, v7.4h\n" - "smlal2 v20.4s, v27.8h, v7.8h\n" - "ldr x20, [x16, #0x78]\n" - "tst x6, #0x7\n" - "smlal v9.4s, v27.4h, v6.4h\n" - "smlal2 v18.4s, v27.8h, v6.8h\n" - "add x14, x14, #0x20\n" + "ldr q28, [x13, #0x0]\n" + "ldr q17, [x12, #0x0]\n" + "smlal v9.4s, v22.4h, v26.4h\n" + "smlal2 v24.4s, v22.8h, v26.8h\n" + "ldr q21, [x13, #0x10]\n" + "ldr q3, [x12, #0x10]\n" + "smlal v9.4s, v4.4h, v23.4h\n" + "smlal v7.4s, v22.4h, v5.4h\n" + "ldr x20, [x15, #0x28]\n" + "ldr d11, [x20, x17]\n" + "smlal v2.4s, v22.4h, v16.4h\n" + "smlal v10.4s, v22.4h, v23.4h\n" + "smlal2 v24.4s, v4.8h, v23.8h\n" + "ldr x20, [x15, #0x38]\n" + "ldr d4, [x20, x17]\n" + "smlal v9.4s, v27.4h, v18.4h\n" + "smlal2 v0.4s, v22.8h, v5.8h\n" + "smlal2 v30.4s, v22.8h, v16.8h\n" + "ldr x20, [x15, #0x30]\n" + "usubl v11.8h, v11.8b, v14.8b\n" + "smlal2 v6.4s, v22.8h, v23.8h\n" + "ldr d22, [x20, x17]\n" + "smlal v7.4s, v8.4h, v1.4h\n" + "ldr x20, [x15, #0x40]\n" + "smlal v2.4s, v27.4h, v1.4h\n" + "smlal v10.4s, v27.4h, v16.4h\n" + "usubl v4.8h, v4.8b, v14.8b\n" + "ldr x26, [x15, #0x48]\n" + "smlal2 v24.4s, v27.8h, v18.8h\n" + "smlal v9.4s, v15.4h, v25.4h\n" + "usubl v22.8h, v22.8b, v14.8b\n" + "ldr x25, [x15, #0x50]\n" + "smlal2 v0.4s, v8.8h, v1.8h\n" + "ldr d8, [x20, x17]\n" + "smlal2 v30.4s, v27.8h, v1.8h\n" + "usubl v8.8h, v8.8b, v14.8b\n" + "smlal2 v6.4s, v27.8h, v16.8h\n" + "smlal v7.4s, v27.4h, v26.4h\n" + "ldr x24, [x15, #0x58]\n" + "ldr x23, [x15, #0x60]\n" + "smlal v2.4s, v11.4h, v31.4h\n" + "smlal v10.4s, v15.4h, v5.4h\n" + "ldr x22, [x15, #0x68]\n" + "ldr x21, [x15, #0x70]\n" + "smlal2 v24.4s, v15.8h, v25.8h\n" + "smlal v9.4s, v4.4h, v16.4h\n" + "ldr x20, [x15, #0x78]\n" + "tst x7, #0x7\n" + "smlal2 v0.4s, v27.8h, v26.8h\n" + "ldr d27, [x26, x17]\n" + "smlal2 v30.4s, v11.8h, v31.8h\n" + "ldr d11, [x25, x17]\n" + "smlal2 v6.4s, v15.8h, v5.8h\n" + "smlal v7.4s, v15.4h, v31.4h\n" + "usubl v27.8h, v27.8b, v14.8b\n" "add x13, x13, #0x20\n" - "smlal v16.4s, v31.4h, v6.4h\n" - "smlal2 v26.4s, v31.8h, v6.8h\n" - "ldr d31, [x27, x8]\n" - "usubl v31.8h, v31.8b, v24.8b\n" - "smlal v25.4s, v27.4h, v3.4h\n" - "smlal2 v10.4s, v27.8h, v3.8h\n" - "smlal v13.4s, v28.4h, v1.4h\n" - "smlal2 v20.4s, v28.8h, v1.8h\n" - "smlal v9.4s, v28.4h, v0.4h\n" - "smlal2 v18.4s, v28.8h, v0.8h\n" - "ldr d30, [x26, x8]\n" - "usubl v30.8h, v30.8b, v24.8b\n" - "smlal v16.4s, v27.4h, v4.4h\n" - "smlal v25.4s, v29.4h, v8.4h\n" - "smlal2 v26.4s, v27.8h, v4.8h\n" - "ldr d28, [x24, x8]\n" - "smlal2 v10.4s, v29.8h, v8.8h\n" - "ldr d29, [x25, x8]\n" - "smlal v13.4s, v31.4h, v2.4h\n" - "smlal2 v20.4s, v31.8h, v2.8h\n" - "usubl v29.8h, v29.8b, v24.8b\n" - "smlal v9.4s, v31.4h, v1.4h\n" - "smlal2 v18.4s, v31.8h, v1.8h\n" - "ldr d31, [x23, x8]\n" - "usubl v28.8h, v28.8b, v24.8b\n" - "smlal v16.4s, v30.4h, v5.4h\n" - "smlal v25.4s, v30.4h, v4.4h\n" - "usubl v31.8h, v31.8b, v24.8b\n" - "smlal v13.4s, v30.4h, v8.4h\n" - "smlal2 v20.4s, v30.8h, v8.8h\n" - "smlal v9.4s, v30.4h, v7.4h\n" - "smlal2 v18.4s, v30.8h, v7.8h\n" - "smlal2 v26.4s, v30.8h, v5.8h\n" - "smlal2 v10.4s, v30.8h, v4.8h\n" - "ldr d30, [x22, x8]\n" - "usubl v30.8h, v30.8b, v24.8b\n" - "smlal v16.4s, v29.4h, v0.4h\n" - "smlal v25.4s, v28.4h, v2.4h\n" - "smlal v13.4s, v29.4h, v3.4h\n" - "smlal2 v20.4s, v29.8h, v3.8h\n" - "smlal2 v26.4s, v29.8h, v0.8h\n" - "ldr d29, [x21, x8]\n" - "smlal2 v10.4s, v28.8h, v2.8h\n" - "usubl v29.8h, v29.8b, v24.8b\n" - "smlal v16.4s, v31.4h, v3.4h\n" - "smlal v25.4s, v30.4h, v5.4h\n" - "smlal v9.4s, v28.4h, v5.4h\n" - "smlal2 v18.4s, v28.8h, v5.8h\n" - "ldr d28, [x20, x8]\n" - "usubl v28.8h, v28.8b, v24.8b\n" - "smlal v13.4s, v31.4h, v6.4h\n" - "smlal2 v26.4s, v31.8h, v3.8h\n" - "sqrdmulh v13.4s, v13.4s, v17.4s\n" - "add x8, x8, #0x8\n" - "smlal2 v10.4s, v30.8h, v5.8h\n" - "smlal v16.4s, v29.4h, v7.4h\n" - "and v21.16b, v13.16b, v22.16b\n" - "smlal v25.4s, v29.4h, v6.4h\n" - "smlal2 v20.4s, v31.8h, v6.8h\n" - "sqrdmulh v20.4s, v20.4s, v23.4s\n" - "smlal2 v26.4s, v29.8h, v7.8h\n" - "smlal2 v10.4s, v29.8h, v6.8h\n" - "sshr v21.4s, v21.4s, #0x1f\n" - "smlal v9.4s, v30.4h, v8.4h\n" - "smlal v16.4s, v28.4h, v8.4h\n" - "and v29.16b, v20.16b, v19.16b\n" - "smlal v25.4s, v28.4h, v7.4h\n" - "smlal2 v18.4s, v30.8h, v8.8h\n" - "sqrdmulh v9.4s, v9.4s, v17.4s\n" - "smlal2 v26.4s, v28.8h, v8.8h\n" - "smlal2 v10.4s, v28.8h, v7.8h\n" - "sqrdmulh v16.4s, v16.4s, v17.4s\n" - "sqrdmulh v25.4s, v25.4s, v17.4s\n" - "sqadd v13.4s, v13.4s, v21.4s\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "and v0.16b, v9.16b, v22.16b\n" - "sqrdmulh v18.4s, v18.4s, v23.4s\n" - "and v27.16b, v16.16b, v22.16b\n" - "sqrdmulh v26.4s, v26.4s, v23.4s\n" - "and v21.16b, v25.16b, v22.16b\n" - "sqrdmulh v10.4s, v10.4s, v23.4s\n" - "sqadd v20.4s, v20.4s, v29.4s\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v17.16b, v18.16b, v19.16b\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "and v7.16b, v26.16b, v19.16b\n" + "smlal v2.4s, v15.4h, v26.4h\n" + "smlal v10.4s, v22.4h, v20.4h\n" + "usubl v11.8h, v11.8b, v14.8b\n" + "add x12, x12, #0x20\n" + "smlal2 v24.4s, v4.8h, v16.8h\n" + "smlal v9.4s, v8.4h, v1.4h\n" + "smlal2 v0.4s, v15.8h, v31.8h\n" + "smlal2 v30.4s, v15.8h, v26.8h\n" + "ldr d15, [x24, x17]\n" + "usubl v15.8h, v15.8b, v14.8b\n" + "smlal2 v6.4s, v22.8h, v20.8h\n" + "ldr d22, [x23, x17]\n" + "smlal v7.4s, v4.4h, v23.4h\n" + "usubl v22.8h, v22.8b, v14.8b\n" + "smlal v2.4s, v27.4h, v18.4h\n" + "smlal v10.4s, v27.4h, v26.4h\n" + "smlal2 v24.4s, v8.8h, v1.8h\n" + "smlal v9.4s, v27.4h, v20.4h\n" + "smlal2 v0.4s, v4.8h, v23.8h\n" + "ldr d4, [x22, x17]\n" + "smlal2 v30.4s, v27.8h, v18.8h\n" + "usubl v4.8h, v4.8b, v14.8b\n" + "smlal2 v6.4s, v27.8h, v26.8h\n" + "ldr d26, [x21, x17]\n" + "smlal v7.4s, v8.4h, v16.4h\n" + "usubl v26.8h, v26.8b, v14.8b\n" + "smlal v2.4s, v11.4h, v23.4h\n" + "smlal v10.4s, v15.4h, v1.4h\n" + "smlal2 v24.4s, v27.8h, v20.8h\n" + "smlal v9.4s, v11.4h, v5.4h\n" + "smlal2 v0.4s, v8.8h, v16.8h\n" + "ldr d16, [x20, x17]\n" + "smlal2 v30.4s, v11.8h, v23.8h\n" + "usubl v16.8h, v16.8b, v14.8b\n" + "smlal2 v6.4s, v15.8h, v1.8h\n" + "smlal v7.4s, v27.4h, v25.4h\n" + "add x17, x17, #0x8\n" + "smlal v2.4s, v22.4h, v5.4h\n" + "smlal v10.4s, v4.4h, v18.4h\n" + "smlal2 v24.4s, v11.8h, v5.8h\n" + "smlal v9.4s, v22.4h, v31.4h\n" + "sqrdmulh v9.4s, v9.4s, v28.4s\n" + "smlal2 v0.4s, v27.8h, v25.8h\n" + "smlal2 v30.4s, v22.8h, v5.8h\n" + "and v1.16b, v9.16b, v17.16b\n" + "smlal2 v6.4s, v4.8h, v18.8h\n" + "smlal v7.4s, v15.4h, v18.4h\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "smlal v2.4s, v26.4h, v25.4h\n" + "smlal v10.4s, v26.4h, v31.4h\n" + "sqadd v9.4s, v9.4s, v1.4s\n" + "smlal2 v24.4s, v22.8h, v31.8h\n" + "smlal2 v0.4s, v15.8h, v18.8h\n" + "sqrdmulh v24.4s, v24.4s, v21.4s\n" + "smlal2 v30.4s, v26.8h, v25.8h\n" + "smlal2 v6.4s, v26.8h, v31.8h\n" + "and v31.16b, v24.16b, v3.16b\n" + "smlal v7.4s, v4.4h, v20.4h\n" + "smlal v2.4s, v16.4h, v20.4h\n" + "sqrdmulh v7.4s, v7.4s, v28.4s\n" + "smlal v10.4s, v16.4h, v25.4h\n" + "smlal2 v0.4s, v4.8h, v20.8h\n" + "sqrdmulh v2.4s, v2.4s, v28.4s\n" + "smlal2 v30.4s, v16.8h, v20.8h\n" + "smlal2 v6.4s, v16.8h, v25.8h\n" + "sqrdmulh v10.4s, v10.4s, v28.4s\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "and v22.16b, v7.16b, v17.16b\n" + "sqrdmulh v0.4s, v0.4s, v21.4s\n" + "and v15.16b, v2.16b, v17.16b\n" + "sqrdmulh v30.4s, v30.4s, v21.4s\n" + "and v11.16b, v10.16b, v17.16b\n" + "sqrdmulh v6.4s, v6.4s, v21.4s\n" + "sqadd v24.4s, v24.4s, v31.4s\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "and v18.16b, v0.16b, v3.16b\n" + "sshr v15.4s, v15.4s, #0x1f\n" + "and v23.16b, v30.16b, v3.16b\n" + "sshr v11.4s, v11.4s, #0x1f\n" + "and v21.16b, v6.16b, v3.16b\n" + "sqadd v7.4s, v7.4s, v22.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sqadd v2.4s, v2.4s, v15.4s\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "sqadd v10.4s, v10.4s, v11.4s\n" "sshr v21.4s, v21.4s, #0x1f\n" - "and v29.16b, v10.16b, v19.16b\n" - "sqadd v9.4s, v9.4s, v0.4s\n" - "sshr v17.4s, v17.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v27.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v25.4s, v25.4s, v21.4s\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "srshl v13.4s, v13.4s, v22.4s\n" - "srshl v9.4s, v9.4s, v22.4s\n" - "sqadd v18.4s, v18.4s, v17.4s\n" - "srshl v16.4s, v16.4s, v22.4s\n" - "sqadd v26.4s, v26.4s, v7.4s\n" - "srshl v25.4s, v25.4s, v22.4s\n" - "sqadd v10.4s, v10.4s, v29.4s\n" - "srshl v20.4s, v20.4s, v19.4s\n" - "sqxtn v13.4h, v13.4s\n" - "srshl v18.4s, v18.4s, v19.4s\n" + "srshl v9.4s, v9.4s, v17.4s\n" + "srshl v7.4s, v7.4s, v17.4s\n" + "sqadd v0.4s, v0.4s, v18.4s\n" + "srshl v2.4s, v2.4s, v17.4s\n" + "sqadd v30.4s, v30.4s, v23.4s\n" + "srshl v10.4s, v10.4s, v17.4s\n" + "sqadd v6.4s, v6.4s, v21.4s\n" + "srshl v24.4s, v24.4s, v3.4s\n" "sqxtn v9.4h, v9.4s\n" - "srshl v26.4s, v26.4s, v19.4s\n" - "sqxtn v16.4h, v16.4s\n" - "srshl v10.4s, v10.4s, v19.4s\n" - "sqxtn v25.4h, v25.4s\n" - "sqxtn2 v13.8h, v20.4s\n" - "sqxtn2 v9.8h, v18.4s\n" - "sqxtn2 v16.8h, v26.4s\n" - "sqxtn2 v25.8h, v10.4s\n" - "sqadd v13.8h, v13.8h, v14.8h\n" - "sqadd v9.8h, v9.8h, v14.8h\n" - "sqadd v16.8h, v16.8h, v14.8h\n" - "sqadd v25.8h, v25.8h, v14.8h\n" - "smax v13.8h, v13.8h, v12.8h\n" - "smax v9.8h, v9.8h, v12.8h\n" - "smax v16.8h, v16.8h, v12.8h\n" - "smax v25.8h, v25.8h, v12.8h\n" - "smin v13.8h, v13.8h, v11.8h\n" - "smin v9.8h, v9.8h, v11.8h\n" - "smin v16.8h, v16.8h, v11.8h\n" - "smin v25.8h, v25.8h, v11.8h\n" - "uzp1 v13.16b, v13.16b, v13.16b\n" - "str d13, [x12, x17]\n" + "srshl v0.4s, v0.4s, v3.4s\n" + "sqxtn v7.4h, v7.4s\n" + "srshl v30.4s, v30.4s, v3.4s\n" + "sqxtn v2.4h, v2.4s\n" + "srshl v6.4s, v6.4s, v3.4s\n" + "sqxtn v10.4h, v10.4s\n" + "sqxtn2 v9.8h, v24.4s\n" + "sqxtn2 v7.8h, v0.4s\n" + "sqxtn2 v2.8h, v30.4s\n" + "sqxtn2 v10.8h, v6.4s\n" + "sqadd v9.8h, v9.8h, v13.8h\n" + "sqadd v7.8h, v7.8h, v13.8h\n" + "sqadd v2.8h, v2.8h, v13.8h\n" + "sqadd v10.8h, v10.8h, v13.8h\n" + "smax v9.8h, v9.8h, v29.8h\n" + "smax v7.8h, v7.8h, v29.8h\n" + "smax v2.8h, v2.8h, v29.8h\n" + "smax v10.8h, v10.8h, v29.8h\n" + "smin v9.8h, v9.8h, v12.8h\n" + "smin v7.8h, v7.8h, v12.8h\n" + "smin v2.8h, v2.8h, v12.8h\n" + "smin v10.8h, v10.8h, v12.8h\n" "uzp1 v9.16b, v9.16b, v9.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "str d9, [x11, x17]\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "str d16, [x10, x17]\n" - "str d25, [x9, x17]\n" - "add x17, x17, #0x8\n" + "str d9, [x11, x16]\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" + "uzp1 v2.16b, v2.16b, v2.16b\n" + "str d7, [x10, x16]\n" + "uzp1 v10.16b, v10.16b, v10.16b\n" + "str d2, [x9, x16]\n" + "str d10, [x28, x16]\n" + "add x16, x16, #0x8\n" "beq 64f\n" - "add x15, x15, #0x48\n" + "add x14, x14, #0x48\n" "3:" // Oddments - "ldr x28, [%x[params], %[offsetof_Params_bias]]\n" - "tbz x6, #2, 5f\n" - "ld1 { v13.4s }, [x28], #0x10\n" - "tbz x6, #1, 4f\n" - "ld1 { v20.d }[0], [x28], #0x8\n" - "tbz x6, #0, 7f\n" - "ld1 { v20.s }[2], [x28]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "tbz x7, #2, 5f\n" + "ld1 { v9.4s }, [x20], #0x10\n" + "tbz x7, #1, 4f\n" + "ld1 { v24.d }[0], [x20], #0x8\n" + "tbz x7, #0, 7f\n" + "ld1 { v24.s }[2], [x20]\n" "b 7f\n" "4:" // Oddments: Load bias: Bit 2: Bit 1: Unset - "tbz x6, #0, 7f\n" - "ld1 { v20.s }[0], [x28]\n" + "tbz x7, #0, 7f\n" + "ld1 { v24.s }[0], [x20]\n" "b 7f\n" "5:" // Oddments: Load bias: Bit 2: Unset - "tbz x6, #1, 6f\n" - "ld1 { v13.d }[0], [x28], #0x8\n" - "tbz x6, #0, 7f\n" - "ld1 { v13.s }[2], [x28]\n" + "tbz x7, #1, 6f\n" + "ld1 { v9.d }[0], [x20], #0x8\n" + "tbz x7, #0, 7f\n" + "ld1 { v9.s }[2], [x20]\n" "b 7f\n" "6:" // Oddments: Load bias: Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 7f\n" - "ld1 { v13.s }[0], [x28]\n" + "tbz x7, #0, 7f\n" + "ld1 { v9.s }[0], [x20]\n" "7:" // Oddments: Load bias: Bit 2: End - "ldr d0, [x15, #0x0]\n" - "ldr d1, [x15, #0x8]\n" - "mov v9.16b, v13.16b\n" - "mov v18.16b, v20.16b\n" - "ldr d2, [x15, #0x10]\n" - "ldr d3, [x15, #0x18]\n" - "mov v16.16b, v13.16b\n" - "mov v26.16b, v20.16b\n" - "ldr d4, [x15, #0x20]\n" - "ldr d5, [x15, #0x28]\n" - "mov v25.16b, v13.16b\n" - "mov v10.16b, v20.16b\n" - "ldr d6, [x15, #0x30]\n" - "ldr d7, [x15, #0x38]\n" - "usubl v0.8h, v0.8b, v15.8b\n" - "usubl v1.8h, v1.8b, v15.8b\n" - "ldr d8, [x15, #0x40]\n" - "ldp x24, x23, [x16, #0x0]\n" - "usubl v2.8h, v2.8b, v15.8b\n" - "usubl v3.8h, v3.8b, v15.8b\n" - "ldp x22, x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x20]\n" - "usubl v4.8h, v4.8b, v15.8b\n" - "usubl v5.8h, v5.8b, v15.8b\n" - "usubl v6.8h, v6.8b, v15.8b\n" - "usubl v7.8h, v7.8b, v15.8b\n" - "usubl v8.8h, v8.8b, v15.8b\n" - "add x24, x24, x8\n" - "add x23, x23, x8\n" - "add x22, x22, x8\n" - "add x21, x21, x8\n" - "add x20, x20, x8\n" - "tbz x6, #2, 9f\n" - "ld1 { v31.s }[0], [x24], #0x4\n" - "ld1 { v30.s }[0], [x23], #0x4\n" - "ld1 { v29.s }[0], [x22], #0x4\n" - "ld1 { v28.s }[0], [x21], #0x4\n" - "ld1 { v27.s }[0], [x20], #0x4\n" - "tbz x6, #1, 8f\n" - "ld1 { v31.h }[2], [x24], #0x2\n" - "ld1 { v30.h }[2], [x23], #0x2\n" - "ld1 { v29.h }[2], [x22], #0x2\n" - "ld1 { v28.h }[2], [x21], #0x2\n" - "ld1 { v27.h }[2], [x20], #0x2\n" - "tbz x6, #0, 11f\n" - "ld1 { v31.b }[6], [x24]\n" - "ld1 { v30.b }[6], [x23]\n" - "ld1 { v29.b }[6], [x22]\n" - "ld1 { v28.b }[6], [x21]\n" - "ld1 { v27.b }[6], [x20]\n" + "ldr d23, [x14, #0x0]\n" + "ldr d16, [x14, #0x8]\n" + "mov v7.16b, v9.16b\n" + "mov v0.16b, v24.16b\n" + "ldr d1, [x14, #0x10]\n" + "ldr d5, [x14, #0x18]\n" + "mov v2.16b, v9.16b\n" + "mov v30.16b, v24.16b\n" + "ldr d26, [x14, #0x20]\n" + "ldr d18, [x14, #0x28]\n" + "mov v10.16b, v9.16b\n" + "mov v6.16b, v24.16b\n" + "ldr d31, [x14, #0x30]\n" + "ldr d25, [x14, #0x38]\n" + "usubl v23.8h, v23.8b, v19.8b\n" + "usubl v16.8h, v16.8b, v19.8b\n" + "ldr d20, [x14, #0x40]\n" + "ldp x24, x23, [x15, #0x0]\n" + "usubl v1.8h, v1.8b, v19.8b\n" + "usubl v5.8h, v5.8b, v19.8b\n" + "ldp x22, x21, [x15, #0x10]\n" + "ldr x20, [x15, #0x20]\n" + "usubl v26.8h, v26.8b, v19.8b\n" + "usubl v18.8h, v18.8b, v19.8b\n" + "usubl v31.8h, v31.8b, v19.8b\n" + "usubl v25.8h, v25.8b, v19.8b\n" + "usubl v20.8h, v20.8b, v19.8b\n" + "add x24, x24, x17\n" + "add x23, x23, x17\n" + "add x22, x22, x17\n" + "add x21, x21, x17\n" + "add x20, x20, x17\n" + "tbz x7, #2, 9f\n" + "ld1 { v22.s }[0], [x24], #0x4\n" + "ld1 { v4.s }[0], [x23], #0x4\n" + "ld1 { v8.s }[0], [x22], #0x4\n" + "ld1 { v27.s }[0], [x21], #0x4\n" + "ld1 { v15.s }[0], [x20], #0x4\n" + "tbz x7, #1, 8f\n" + "ld1 { v22.h }[2], [x24], #0x2\n" + "ld1 { v4.h }[2], [x23], #0x2\n" + "ld1 { v8.h }[2], [x22], #0x2\n" + "ld1 { v27.h }[2], [x21], #0x2\n" + "ld1 { v15.h }[2], [x20], #0x2\n" + "tbz x7, #0, 11f\n" + "ld1 { v22.b }[6], [x24]\n" + "ld1 { v4.b }[6], [x23]\n" + "ld1 { v8.b }[6], [x22]\n" + "ld1 { v27.b }[6], [x21]\n" + "ld1 { v15.b }[6], [x20]\n" "b 11f\n" "8:" // Oddments: Initial loads: Bit 2: Bit 1: Unset - "tbz x6, #0, 11f\n" - "ld1 { v31.b }[4], [x24]\n" - "ld1 { v30.b }[4], [x23]\n" - "ld1 { v29.b }[4], [x22]\n" - "ld1 { v28.b }[4], [x21]\n" - "ld1 { v27.b }[4], [x20]\n" + "tbz x7, #0, 11f\n" + "ld1 { v22.b }[4], [x24]\n" + "ld1 { v4.b }[4], [x23]\n" + "ld1 { v8.b }[4], [x22]\n" + "ld1 { v27.b }[4], [x21]\n" + "ld1 { v15.b }[4], [x20]\n" "b 11f\n" "9:" // Oddments: Initial loads: Bit 2: Unset - "tbz x6, #1, 10f\n" - "ld1 { v31.h }[0], [x24], #0x2\n" - "ld1 { v30.h }[0], [x23], #0x2\n" - "ld1 { v29.h }[0], [x22], #0x2\n" - "ld1 { v28.h }[0], [x21], #0x2\n" - "ld1 { v27.h }[0], [x20], #0x2\n" - "tbz x6, #0, 11f\n" - "ld1 { v31.b }[2], [x24]\n" - "ld1 { v30.b }[2], [x23]\n" - "ld1 { v29.b }[2], [x22]\n" - "ld1 { v28.b }[2], [x21]\n" - "ld1 { v27.b }[2], [x20]\n" + "tbz x7, #1, 10f\n" + "ld1 { v22.h }[0], [x24], #0x2\n" + "ld1 { v4.h }[0], [x23], #0x2\n" + "ld1 { v8.h }[0], [x22], #0x2\n" + "ld1 { v27.h }[0], [x21], #0x2\n" + "ld1 { v15.h }[0], [x20], #0x2\n" + "tbz x7, #0, 11f\n" + "ld1 { v22.b }[2], [x24]\n" + "ld1 { v4.b }[2], [x23]\n" + "ld1 { v8.b }[2], [x22]\n" + "ld1 { v27.b }[2], [x21]\n" + "ld1 { v15.b }[2], [x20]\n" "b 11f\n" "10:" // Oddments: Initial loads: Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 11f\n" - "ld1 { v31.b }[0], [x24]\n" - "ld1 { v30.b }[0], [x23]\n" - "ld1 { v29.b }[0], [x22]\n" - "ld1 { v28.b }[0], [x21]\n" - "ld1 { v27.b }[0], [x20]\n" + "tbz x7, #0, 11f\n" + "ld1 { v22.b }[0], [x24]\n" + "ld1 { v4.b }[0], [x23]\n" + "ld1 { v8.b }[0], [x22]\n" + "ld1 { v27.b }[0], [x21]\n" + "ld1 { v15.b }[0], [x20]\n" "11:" // Oddments: Initial loads: Bit 2: End - "usubl v31.8h, v31.8b, v24.8b\n" - "smlal v13.4s, v31.4h, v4.4h\n" - "smlal2 v20.4s, v31.8h, v4.8h\n" - "ldr x21, [x16, #0x28]\n" - "smlal v9.4s, v31.4h, v3.4h\n" - "smlal2 v18.4s, v31.8h, v3.8h\n" - "usubl v30.8h, v30.8b, v24.8b\n" - "add x21, x21, x8\n" - "usubl v29.8h, v29.8b, v24.8b\n" - "smlal v16.4s, v31.4h, v1.4h\n" - "smlal2 v26.4s, v31.8h, v1.8h\n" - "smlal v25.4s, v31.4h, v0.4h\n" - "smlal2 v10.4s, v31.8h, v0.8h\n" - "usubl v28.8h, v28.8b, v24.8b\n" - "smlal v13.4s, v30.4h, v0.4h\n" - "smlal2 v20.4s, v30.8h, v0.8h\n" - "usubl v27.8h, v27.8b, v24.8b\n" - "smlal v9.4s, v29.4h, v2.4h\n" - "smlal2 v18.4s, v29.8h, v2.8h\n" - "smlal v13.4s, v28.4h, v5.4h\n" - "smlal2 v20.4s, v28.8h, v5.8h\n" - "smlal v9.4s, v28.4h, v4.4h\n" - "smlal2 v18.4s, v28.8h, v4.8h\n" - "smlal v16.4s, v28.4h, v2.4h\n" - "smlal2 v26.4s, v28.8h, v2.8h\n" - "smlal v25.4s, v28.4h, v1.4h\n" - "smlal2 v10.4s, v28.8h, v1.8h\n" - "tbz x6, #2, 13f\n" - "ld1 { v31.s }[0], [x21], #0x4\n" - "tbz x6, #1, 12f\n" - "ld1 { v31.h }[2], [x21], #0x2\n" - "tbz x6, #0, 15f\n" - "ld1 { v31.b }[6], [x21]\n" + "usubl v22.8h, v22.8b, v14.8b\n" + "smlal v9.4s, v22.4h, v26.4h\n" + "smlal2 v24.4s, v22.8h, v26.8h\n" + "ldr x20, [x15, #0x28]\n" + "smlal v7.4s, v22.4h, v5.4h\n" + "smlal2 v0.4s, v22.8h, v5.8h\n" + "usubl v4.8h, v4.8b, v14.8b\n" + "usubl v8.8h, v8.8b, v14.8b\n" + "smlal v2.4s, v22.4h, v16.4h\n" + "smlal2 v30.4s, v22.8h, v16.8h\n" + "add x20, x20, x17\n" + "smlal v10.4s, v22.4h, v23.4h\n" + "smlal2 v6.4s, v22.8h, v23.8h\n" + "usubl v27.8h, v27.8b, v14.8b\n" + "smlal v9.4s, v4.4h, v23.4h\n" + "smlal2 v24.4s, v4.8h, v23.8h\n" + "usubl v15.8h, v15.8b, v14.8b\n" + "smlal v7.4s, v8.4h, v1.4h\n" + "smlal2 v0.4s, v8.8h, v1.8h\n" + "smlal v9.4s, v27.4h, v18.4h\n" + "smlal2 v24.4s, v27.8h, v18.8h\n" + "smlal v7.4s, v27.4h, v26.4h\n" + "smlal2 v0.4s, v27.8h, v26.8h\n" + "smlal v2.4s, v27.4h, v1.4h\n" + "smlal2 v30.4s, v27.8h, v1.8h\n" + "smlal v10.4s, v27.4h, v16.4h\n" + "smlal2 v6.4s, v27.8h, v16.8h\n" + "tbz x7, #2, 13f\n" + "ld1 { v21.s }[0], [x20], #0x4\n" + "tbz x7, #1, 12f\n" + "ld1 { v21.h }[2], [x20], #0x2\n" + "tbz x7, #0, 15f\n" + "ld1 { v21.b }[6], [x20]\n" "b 15f\n" "12:" // Oddments: Load (3, 0): Bit 2: Bit 1: Unset - "tbz x6, #0, 15f\n" - "ld1 { v31.b }[4], [x21]\n" + "tbz x7, #0, 15f\n" + "ld1 { v21.b }[4], [x20]\n" "b 15f\n" "13:" // Oddments: Load (3, 0): Bit 2: Unset - "tbz x6, #1, 14f\n" - "ld1 { v31.h }[0], [x21], #0x2\n" - "tbz x6, #0, 15f\n" - "ld1 { v31.b }[2], [x21]\n" + "tbz x7, #1, 14f\n" + "ld1 { v21.h }[0], [x20], #0x2\n" + "tbz x7, #0, 15f\n" + "ld1 { v21.b }[2], [x20]\n" "b 15f\n" "14:" // Oddments: Load (3, 0): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 15f\n" - "ld1 { v31.b }[0], [x21]\n" + "tbz x7, #0, 15f\n" + "ld1 { v21.b }[0], [x20]\n" "15:" // Oddments: Load (3, 0): Bit 2: End - "usubl v31.8h, v31.8b, v24.8b\n" - "smlal v16.4s, v31.4h, v6.4h\n" - "smlal2 v26.4s, v31.8h, v6.8h\n" - "ldr x20, [x16, #0x30]\n" - "smlal v13.4s, v27.4h, v7.4h\n" - "smlal2 v20.4s, v27.8h, v7.8h\n" - "add x20, x20, x8\n" - "smlal v9.4s, v27.4h, v6.4h\n" - "smlal2 v18.4s, v27.8h, v6.8h\n" - "smlal v16.4s, v27.4h, v4.4h\n" - "smlal2 v26.4s, v27.8h, v4.8h\n" - "smlal v25.4s, v27.4h, v3.4h\n" - "smlal2 v10.4s, v27.8h, v3.8h\n" - "tbz x6, #2, 17f\n" - "ld1 { v29.s }[0], [x20], #0x4\n" - "tbz x6, #1, 16f\n" - "ld1 { v29.h }[2], [x20], #0x2\n" - "tbz x6, #0, 19f\n" - "ld1 { v29.b }[6], [x20]\n" + "usubl v21.8h, v21.8b, v14.8b\n" + "smlal v2.4s, v21.4h, v31.4h\n" + "smlal2 v30.4s, v21.8h, v31.8h\n" + "ldr x20, [x15, #0x30]\n" + "smlal v9.4s, v15.4h, v25.4h\n" + "smlal2 v24.4s, v15.8h, v25.8h\n" + "add x20, x20, x17\n" + "smlal v7.4s, v15.4h, v31.4h\n" + "smlal2 v0.4s, v15.8h, v31.8h\n" + "smlal v2.4s, v15.4h, v26.4h\n" + "smlal2 v30.4s, v15.8h, v26.8h\n" + "smlal v10.4s, v15.4h, v5.4h\n" + "smlal2 v6.4s, v15.8h, v5.8h\n" + "tbz x7, #2, 17f\n" + "ld1 { v28.s }[0], [x20], #0x4\n" + "tbz x7, #1, 16f\n" + "ld1 { v28.h }[2], [x20], #0x2\n" + "tbz x7, #0, 19f\n" + "ld1 { v28.b }[6], [x20]\n" "b 19f\n" "16:" // Oddments: Load (3, 3): Bit 2: Bit 1: Unset - "tbz x6, #0, 19f\n" - "ld1 { v29.b }[4], [x20]\n" + "tbz x7, #0, 19f\n" + "ld1 { v28.b }[4], [x20]\n" "b 19f\n" "17:" // Oddments: Load (3, 3): Bit 2: Unset - "tbz x6, #1, 18f\n" - "ld1 { v29.h }[0], [x20], #0x2\n" - "tbz x6, #0, 19f\n" - "ld1 { v29.b }[2], [x20]\n" + "tbz x7, #1, 18f\n" + "ld1 { v28.h }[0], [x20], #0x2\n" + "tbz x7, #0, 19f\n" + "ld1 { v28.b }[2], [x20]\n" "b 19f\n" "18:" // Oddments: Load (3, 3): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 19f\n" - "ld1 { v29.b }[0], [x20]\n" + "tbz x7, #0, 19f\n" + "ld1 { v28.b }[0], [x20]\n" "19:" // Oddments: Load (3, 3): Bit 2: End - "usubl v29.8h, v29.8b, v24.8b\n" - "ldr x28, [x16, #0x38]\n" - "smlal v25.4s, v29.4h, v8.4h\n" - "smlal2 v10.4s, v29.8h, v8.8h\n" - "add x28, x28, x8\n" - "tbz x6, #2, 21f\n" - "ld1 { v28.s }[0], [x28], #0x4\n" - "tbz x6, #1, 20f\n" - "ld1 { v28.h }[2], [x28], #0x2\n" - "tbz x6, #0, 23f\n" - "ld1 { v28.b }[6], [x28]\n" + "usubl v28.8h, v28.8b, v14.8b\n" + "ldr x20, [x15, #0x38]\n" + "smlal v10.4s, v28.4h, v20.4h\n" + "smlal2 v6.4s, v28.8h, v20.8h\n" + "add x20, x20, x17\n" + "tbz x7, #2, 21f\n" + "ld1 { v22.s }[0], [x20], #0x4\n" + "tbz x7, #1, 20f\n" + "ld1 { v22.h }[2], [x20], #0x2\n" + "tbz x7, #0, 23f\n" + "ld1 { v22.b }[6], [x20]\n" "b 23f\n" "20:" // Oddments: Load (0, 1): Bit 2: Bit 1: Unset - "tbz x6, #0, 23f\n" - "ld1 { v28.b }[4], [x28]\n" + "tbz x7, #0, 23f\n" + "ld1 { v22.b }[4], [x20]\n" "b 23f\n" "21:" // Oddments: Load (0, 1): Bit 2: Unset - "tbz x6, #1, 22f\n" - "ld1 { v28.h }[0], [x28], #0x2\n" - "tbz x6, #0, 23f\n" - "ld1 { v28.b }[2], [x28]\n" + "tbz x7, #1, 22f\n" + "ld1 { v22.h }[0], [x20], #0x2\n" + "tbz x7, #0, 23f\n" + "ld1 { v22.b }[2], [x20]\n" "b 23f\n" "22:" // Oddments: Load (0, 1): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 23f\n" - "ld1 { v28.b }[0], [x28]\n" + "tbz x7, #0, 23f\n" + "ld1 { v22.b }[0], [x20]\n" "23:" // Oddments: Load (0, 1): Bit 2: End - "usubl v28.8h, v28.8b, v24.8b\n" - "ldr x27, [x16, #0x40]\n" - "smlal v13.4s, v28.4h, v1.4h\n" - "smlal2 v20.4s, v28.8h, v1.8h\n" - "smlal v9.4s, v28.4h, v0.4h\n" - "smlal2 v18.4s, v28.8h, v0.8h\n" - "add x27, x27, x8\n" - "tbz x6, #2, 25f\n" - "ld1 { v31.s }[0], [x27], #0x4\n" - "tbz x6, #1, 24f\n" - "ld1 { v31.h }[2], [x27], #0x2\n" - "tbz x6, #0, 27f\n" - "ld1 { v31.b }[6], [x27]\n" + "usubl v22.8h, v22.8b, v14.8b\n" + "ldr x20, [x15, #0x40]\n" + "smlal v9.4s, v22.4h, v16.4h\n" + "smlal2 v24.4s, v22.8h, v16.8h\n" + "smlal v7.4s, v22.4h, v23.4h\n" + "smlal2 v0.4s, v22.8h, v23.8h\n" + "add x20, x20, x17\n" + "tbz x7, #2, 25f\n" + "ld1 { v21.s }[0], [x20], #0x4\n" + "tbz x7, #1, 24f\n" + "ld1 { v21.h }[2], [x20], #0x2\n" + "tbz x7, #0, 27f\n" + "ld1 { v21.b }[6], [x20]\n" "b 27f\n" "24:" // Oddments: Load (0, 2): Bit 2: Bit 1: Unset - "tbz x6, #0, 27f\n" - "ld1 { v31.b }[4], [x27]\n" + "tbz x7, #0, 27f\n" + "ld1 { v21.b }[4], [x20]\n" "b 27f\n" "25:" // Oddments: Load (0, 2): Bit 2: Unset - "tbz x6, #1, 26f\n" - "ld1 { v31.h }[0], [x27], #0x2\n" - "tbz x6, #0, 27f\n" - "ld1 { v31.b }[2], [x27]\n" + "tbz x7, #1, 26f\n" + "ld1 { v21.h }[0], [x20], #0x2\n" + "tbz x7, #0, 27f\n" + "ld1 { v21.b }[2], [x20]\n" "b 27f\n" "26:" // Oddments: Load (0, 2): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 27f\n" - "ld1 { v31.b }[0], [x27]\n" + "tbz x7, #0, 27f\n" + "ld1 { v21.b }[0], [x20]\n" "27:" // Oddments: Load (0, 2): Bit 2: End - "usubl v31.8h, v31.8b, v24.8b\n" - "ldr x26, [x16, #0x48]\n" - "smlal v13.4s, v31.4h, v2.4h\n" - "smlal2 v20.4s, v31.8h, v2.8h\n" - "smlal v9.4s, v31.4h, v1.4h\n" - "smlal2 v18.4s, v31.8h, v1.8h\n" - "add x26, x26, x8\n" - "tbz x6, #2, 29f\n" - "ld1 { v30.s }[0], [x26], #0x4\n" - "tbz x6, #1, 28f\n" - "ld1 { v30.h }[2], [x26], #0x2\n" - "tbz x6, #0, 31f\n" - "ld1 { v30.b }[6], [x26]\n" + "usubl v21.8h, v21.8b, v14.8b\n" + "ldr x20, [x15, #0x48]\n" + "smlal v9.4s, v21.4h, v1.4h\n" + "smlal2 v24.4s, v21.8h, v1.8h\n" + "smlal v7.4s, v21.4h, v16.4h\n" + "smlal2 v0.4s, v21.8h, v16.8h\n" + "add x20, x20, x17\n" + "tbz x7, #2, 29f\n" + "ld1 { v28.s }[0], [x20], #0x4\n" + "tbz x7, #1, 28f\n" + "ld1 { v28.h }[2], [x20], #0x2\n" + "tbz x7, #0, 31f\n" + "ld1 { v28.b }[6], [x20]\n" "b 31f\n" "28:" // Oddments: Load (2, 2): Bit 2: Bit 1: Unset - "tbz x6, #0, 31f\n" - "ld1 { v30.b }[4], [x26]\n" + "tbz x7, #0, 31f\n" + "ld1 { v28.b }[4], [x20]\n" "b 31f\n" "29:" // Oddments: Load (2, 2): Bit 2: Unset - "tbz x6, #1, 30f\n" - "ld1 { v30.h }[0], [x26], #0x2\n" - "tbz x6, #0, 31f\n" - "ld1 { v30.b }[2], [x26]\n" + "tbz x7, #1, 30f\n" + "ld1 { v28.h }[0], [x20], #0x2\n" + "tbz x7, #0, 31f\n" + "ld1 { v28.b }[2], [x20]\n" "b 31f\n" "30:" // Oddments: Load (2, 2): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 31f\n" - "ld1 { v30.b }[0], [x26]\n" + "tbz x7, #0, 31f\n" + "ld1 { v28.b }[0], [x20]\n" "31:" // Oddments: Load (2, 2): Bit 2: End - "usubl v30.8h, v30.8b, v24.8b\n" - "ldr x25, [x16, #0x50]\n" - "smlal v13.4s, v30.4h, v8.4h\n" - "smlal2 v20.4s, v30.8h, v8.8h\n" - "smlal v9.4s, v30.4h, v7.4h\n" - "smlal2 v18.4s, v30.8h, v7.8h\n" - "add x25, x25, x8\n" - "smlal v16.4s, v30.4h, v5.4h\n" - "smlal2 v26.4s, v30.8h, v5.8h\n" - "smlal v25.4s, v30.4h, v4.4h\n" - "smlal2 v10.4s, v30.8h, v4.8h\n" - "tbz x6, #2, 33f\n" - "ld1 { v29.s }[0], [x25], #0x4\n" - "tbz x6, #1, 32f\n" - "ld1 { v29.h }[2], [x25], #0x2\n" - "tbz x6, #0, 35f\n" - "ld1 { v29.b }[6], [x25]\n" + "usubl v28.8h, v28.8b, v14.8b\n" + "ldr x20, [x15, #0x50]\n" + "smlal v9.4s, v28.4h, v20.4h\n" + "smlal2 v24.4s, v28.8h, v20.8h\n" + "smlal v7.4s, v28.4h, v25.4h\n" + "smlal2 v0.4s, v28.8h, v25.8h\n" + "add x20, x20, x17\n" + "smlal v2.4s, v28.4h, v18.4h\n" + "smlal2 v30.4s, v28.8h, v18.8h\n" + "smlal v10.4s, v28.4h, v26.4h\n" + "smlal2 v6.4s, v28.8h, v26.8h\n" + "tbz x7, #2, 33f\n" + "ld1 { v8.s }[0], [x20], #0x4\n" + "tbz x7, #1, 32f\n" + "ld1 { v8.h }[2], [x20], #0x2\n" + "tbz x7, #0, 35f\n" + "ld1 { v8.b }[6], [x20]\n" "b 35f\n" "32:" // Oddments: Load (1, 0): Bit 2: Bit 1: Unset - "tbz x6, #0, 35f\n" - "ld1 { v29.b }[4], [x25]\n" + "tbz x7, #0, 35f\n" + "ld1 { v8.b }[4], [x20]\n" "b 35f\n" "33:" // Oddments: Load (1, 0): Bit 2: Unset - "tbz x6, #1, 34f\n" - "ld1 { v29.h }[0], [x25], #0x2\n" - "tbz x6, #0, 35f\n" - "ld1 { v29.b }[2], [x25]\n" + "tbz x7, #1, 34f\n" + "ld1 { v8.h }[0], [x20], #0x2\n" + "tbz x7, #0, 35f\n" + "ld1 { v8.b }[2], [x20]\n" "b 35f\n" "34:" // Oddments: Load (1, 0): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 35f\n" - "ld1 { v29.b }[0], [x25]\n" + "tbz x7, #0, 35f\n" + "ld1 { v8.b }[0], [x20]\n" "35:" // Oddments: Load (1, 0): Bit 2: End - "usubl v29.8h, v29.8b, v24.8b\n" - "ldr x24, [x16, #0x58]\n" - "smlal v13.4s, v29.4h, v3.4h\n" - "smlal2 v20.4s, v29.8h, v3.8h\n" - "smlal v16.4s, v29.4h, v0.4h\n" - "smlal2 v26.4s, v29.8h, v0.8h\n" - "add x24, x24, x8\n" - "tbz x6, #2, 37f\n" - "ld1 { v28.s }[0], [x24], #0x4\n" - "tbz x6, #1, 36f\n" - "ld1 { v28.h }[2], [x24], #0x2\n" - "tbz x6, #0, 39f\n" - "ld1 { v28.b }[6], [x24]\n" + "usubl v8.8h, v8.8b, v14.8b\n" + "ldr x20, [x15, #0x58]\n" + "smlal v9.4s, v8.4h, v5.4h\n" + "smlal2 v24.4s, v8.8h, v5.8h\n" + "smlal v2.4s, v8.4h, v23.4h\n" + "smlal2 v30.4s, v8.8h, v23.8h\n" + "add x20, x20, x17\n" + "tbz x7, #2, 37f\n" + "ld1 { v8.s }[0], [x20], #0x4\n" + "tbz x7, #1, 36f\n" + "ld1 { v8.h }[2], [x20], #0x2\n" + "tbz x7, #0, 39f\n" + "ld1 { v8.b }[6], [x20]\n" "b 39f\n" "36:" // Oddments: Load (1, 3): Bit 2: Bit 1: Unset - "tbz x6, #0, 39f\n" - "ld1 { v28.b }[4], [x24]\n" + "tbz x7, #0, 39f\n" + "ld1 { v8.b }[4], [x20]\n" "b 39f\n" "37:" // Oddments: Load (1, 3): Bit 2: Unset - "tbz x6, #1, 38f\n" - "ld1 { v28.h }[0], [x24], #0x2\n" - "tbz x6, #0, 39f\n" - "ld1 { v28.b }[2], [x24]\n" + "tbz x7, #1, 38f\n" + "ld1 { v8.h }[0], [x20], #0x2\n" + "tbz x7, #0, 39f\n" + "ld1 { v8.b }[2], [x20]\n" "b 39f\n" "38:" // Oddments: Load (1, 3): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 39f\n" - "ld1 { v28.b }[0], [x24]\n" + "tbz x7, #0, 39f\n" + "ld1 { v8.b }[0], [x20]\n" "39:" // Oddments: Load (1, 3): Bit 2: End - "usubl v28.8h, v28.8b, v24.8b\n" - "ldr x23, [x16, #0x60]\n" - "smlal v9.4s, v28.4h, v5.4h\n" - "smlal2 v18.4s, v28.8h, v5.8h\n" - "smlal v25.4s, v28.4h, v2.4h\n" - "smlal2 v10.4s, v28.8h, v2.8h\n" - "add x23, x23, x8\n" - "tbz x6, #2, 41f\n" - "ld1 { v31.s }[0], [x23], #0x4\n" - "tbz x6, #1, 40f\n" - "ld1 { v31.h }[2], [x23], #0x2\n" - "tbz x6, #0, 43f\n" - "ld1 { v31.b }[6], [x23]\n" + "usubl v8.8h, v8.8b, v14.8b\n" + "ldr x20, [x15, #0x60]\n" + "smlal v7.4s, v8.4h, v18.4h\n" + "smlal2 v0.4s, v8.8h, v18.8h\n" + "smlal v10.4s, v8.4h, v1.4h\n" + "smlal2 v6.4s, v8.8h, v1.8h\n" + "add x20, x20, x17\n" + "tbz x7, #2, 41f\n" + "ld1 { v17.s }[0], [x20], #0x4\n" + "tbz x7, #1, 40f\n" + "ld1 { v17.h }[2], [x20], #0x2\n" + "tbz x7, #0, 43f\n" + "ld1 { v17.b }[6], [x20]\n" "b 43f\n" "40:" // Oddments: Load (2, 0): Bit 2: Bit 1: Unset - "tbz x6, #0, 43f\n" - "ld1 { v31.b }[4], [x23]\n" + "tbz x7, #0, 43f\n" + "ld1 { v17.b }[4], [x20]\n" "b 43f\n" "41:" // Oddments: Load (2, 0): Bit 2: Unset - "tbz x6, #1, 42f\n" - "ld1 { v31.h }[0], [x23], #0x2\n" - "tbz x6, #0, 43f\n" - "ld1 { v31.b }[2], [x23]\n" + "tbz x7, #1, 42f\n" + "ld1 { v17.h }[0], [x20], #0x2\n" + "tbz x7, #0, 43f\n" + "ld1 { v17.b }[2], [x20]\n" "b 43f\n" "42:" // Oddments: Load (2, 0): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 43f\n" - "ld1 { v31.b }[0], [x23]\n" + "tbz x7, #0, 43f\n" + "ld1 { v17.b }[0], [x20]\n" "43:" // Oddments: Load (2, 0): Bit 2: End - "usubl v31.8h, v31.8b, v24.8b\n" - "ldr x22, [x16, #0x68]\n" - "smlal v13.4s, v31.4h, v6.4h\n" - "smlal2 v20.4s, v31.8h, v6.8h\n" - "smlal v16.4s, v31.4h, v3.4h\n" - "smlal2 v26.4s, v31.8h, v3.8h\n" - "add x22, x22, x8\n" - "tbz x6, #2, 45f\n" - "ld1 { v30.s }[0], [x22], #0x4\n" - "tbz x6, #1, 44f\n" - "ld1 { v30.h }[2], [x22], #0x2\n" - "tbz x6, #0, 47f\n" - "ld1 { v30.b }[6], [x22]\n" + "usubl v17.8h, v17.8b, v14.8b\n" + "ldr x20, [x15, #0x68]\n" + "smlal v9.4s, v17.4h, v31.4h\n" + "smlal2 v24.4s, v17.8h, v31.8h\n" + "smlal v2.4s, v17.4h, v5.4h\n" + "smlal2 v30.4s, v17.8h, v5.8h\n" + "add x20, x20, x17\n" + "tbz x7, #2, 45f\n" + "ld1 { v23.s }[0], [x20], #0x4\n" + "tbz x7, #1, 44f\n" + "ld1 { v23.h }[2], [x20], #0x2\n" + "tbz x7, #0, 47f\n" + "ld1 { v23.b }[6], [x20]\n" "b 47f\n" "44:" // Oddments: Load (2, 3): Bit 2: Bit 1: Unset - "tbz x6, #0, 47f\n" - "ld1 { v30.b }[4], [x22]\n" + "tbz x7, #0, 47f\n" + "ld1 { v23.b }[4], [x20]\n" "b 47f\n" "45:" // Oddments: Load (2, 3): Bit 2: Unset - "tbz x6, #1, 46f\n" - "ld1 { v30.h }[0], [x22], #0x2\n" - "tbz x6, #0, 47f\n" - "ld1 { v30.b }[2], [x22]\n" + "tbz x7, #1, 46f\n" + "ld1 { v23.h }[0], [x20], #0x2\n" + "tbz x7, #0, 47f\n" + "ld1 { v23.b }[2], [x20]\n" "b 47f\n" "46:" // Oddments: Load (2, 3): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 47f\n" - "ld1 { v30.b }[0], [x22]\n" + "tbz x7, #0, 47f\n" + "ld1 { v23.b }[0], [x20]\n" "47:" // Oddments: Load (2, 3): Bit 2: End - "usubl v30.8h, v30.8b, v24.8b\n" - "ldr x21, [x16, #0x70]\n" - "smlal v9.4s, v30.4h, v8.4h\n" - "smlal2 v18.4s, v30.8h, v8.8h\n" - "smlal v25.4s, v30.4h, v5.4h\n" - "smlal2 v10.4s, v30.8h, v5.8h\n" - "add x21, x21, x8\n" - "tbz x6, #2, 49f\n" - "ld1 { v29.s }[0], [x21], #0x4\n" - "tbz x6, #1, 48f\n" - "ld1 { v29.h }[2], [x21], #0x2\n" - "tbz x6, #0, 51f\n" - "ld1 { v29.b }[6], [x21]\n" + "usubl v23.8h, v23.8b, v14.8b\n" + "ldr x20, [x15, #0x70]\n" + "smlal v7.4s, v23.4h, v20.4h\n" + "smlal2 v0.4s, v23.8h, v20.8h\n" + "smlal v10.4s, v23.4h, v18.4h\n" + "smlal2 v6.4s, v23.8h, v18.8h\n" + "add x20, x20, x17\n" + "tbz x7, #2, 49f\n" + "ld1 { v5.s }[0], [x20], #0x4\n" + "tbz x7, #1, 48f\n" + "ld1 { v5.h }[2], [x20], #0x2\n" + "tbz x7, #0, 51f\n" + "ld1 { v5.b }[6], [x20]\n" "b 51f\n" "48:" // Oddments: Load (3, 1): Bit 2: Bit 1: Unset - "tbz x6, #0, 51f\n" - "ld1 { v29.b }[4], [x21]\n" + "tbz x7, #0, 51f\n" + "ld1 { v5.b }[4], [x20]\n" "b 51f\n" "49:" // Oddments: Load (3, 1): Bit 2: Unset - "tbz x6, #1, 50f\n" - "ld1 { v29.h }[0], [x21], #0x2\n" - "tbz x6, #0, 51f\n" - "ld1 { v29.b }[2], [x21]\n" + "tbz x7, #1, 50f\n" + "ld1 { v5.h }[0], [x20], #0x2\n" + "tbz x7, #0, 51f\n" + "ld1 { v5.b }[2], [x20]\n" "b 51f\n" "50:" // Oddments: Load (3, 1): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 51f\n" - "ld1 { v29.b }[0], [x21]\n" + "tbz x7, #0, 51f\n" + "ld1 { v5.b }[0], [x20]\n" "51:" // Oddments: Load (3, 1): Bit 2: End - "usubl v29.8h, v29.8b, v24.8b\n" - "ldr x20, [x16, #0x78]\n" - "smlal v16.4s, v29.4h, v7.4h\n" - "smlal2 v26.4s, v29.8h, v7.8h\n" - "smlal v25.4s, v29.4h, v6.4h\n" - "smlal2 v10.4s, v29.8h, v6.8h\n" - "add x20, x20, x8\n" - "tbz x6, #2, 53f\n" - "ld1 { v28.s }[0], [x20], #0x4\n" - "tbz x6, #1, 52f\n" - "ld1 { v28.h }[2], [x20], #0x2\n" - "tbz x6, #0, 55f\n" - "ld1 { v28.b }[6], [x20]\n" + "usubl v5.8h, v5.8b, v14.8b\n" + "ldr x20, [x15, #0x78]\n" + "smlal v2.4s, v5.4h, v25.4h\n" + "smlal2 v30.4s, v5.8h, v25.8h\n" + "smlal v10.4s, v5.4h, v31.4h\n" + "smlal2 v6.4s, v5.8h, v31.8h\n" + "add x20, x20, x17\n" + "tbz x7, #2, 53f\n" + "ld1 { v23.s }[0], [x20], #0x4\n" + "tbz x7, #1, 52f\n" + "ld1 { v23.h }[2], [x20], #0x2\n" + "tbz x7, #0, 55f\n" + "ld1 { v23.b }[6], [x20]\n" "b 55f\n" "52:" // Oddments: Load (3, 2): Bit 2: Bit 1: Unset - "tbz x6, #0, 55f\n" - "ld1 { v28.b }[4], [x20]\n" + "tbz x7, #0, 55f\n" + "ld1 { v23.b }[4], [x20]\n" "b 55f\n" "53:" // Oddments: Load (3, 2): Bit 2: Unset - "tbz x6, #1, 54f\n" - "ld1 { v28.h }[0], [x20], #0x2\n" - "tbz x6, #0, 55f\n" - "ld1 { v28.b }[2], [x20]\n" + "tbz x7, #1, 54f\n" + "ld1 { v23.h }[0], [x20], #0x2\n" + "tbz x7, #0, 55f\n" + "ld1 { v23.b }[2], [x20]\n" "b 55f\n" "54:" // Oddments: Load (3, 2): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 55f\n" - "ld1 { v28.b }[0], [x20]\n" + "tbz x7, #0, 55f\n" + "ld1 { v23.b }[0], [x20]\n" "55:" // Oddments: Load (3, 2): Bit 2: End - "usubl v28.8h, v28.8b, v24.8b\n" - "smlal v16.4s, v28.4h, v8.4h\n" - "smlal2 v26.4s, v28.8h, v8.8h\n" - "smlal v25.4s, v28.4h, v7.4h\n" - "smlal2 v10.4s, v28.8h, v7.8h\n" - "tbz x6, #2, 57f\n" - "ld1 { v17.4s }, [x14], #0x10\n" - "ld1 { v22.4s }, [x13], #0x10\n" - "tbz x6, #1, 56f\n" - "ld1 { v23.d }[0], [x14], #0x8\n" - "ld1 { v19.d }[0], [x13], #0x8\n" - "tbz x6, #0, 59f\n" - "ld1 { v23.s }[2], [x14]\n" - "ld1 { v19.s }[2], [x13]\n" + "usubl v23.8h, v23.8b, v14.8b\n" + "smlal v2.4s, v23.4h, v20.4h\n" + "smlal2 v30.4s, v23.8h, v20.8h\n" + "smlal v10.4s, v23.4h, v25.4h\n" + "smlal2 v6.4s, v23.8h, v25.8h\n" + "tbz x7, #2, 57f\n" + "ld1 { v15.4s }, [x13], #0x10\n" + "ld1 { v19.4s }, [x12], #0x10\n" + "tbz x7, #1, 56f\n" + "ld1 { v18.d }[0], [x13], #0x8\n" + "ld1 { v22.d }[0], [x12], #0x8\n" + "tbz x7, #0, 59f\n" + "ld1 { v18.s }[2], [x13]\n" + "ld1 { v22.s }[2], [x12]\n" "b 59f\n" "56:" // Oddments: Load requant params: Bit 2: Bit 1: Unset - "tbz x6, #0, 59f\n" - "ld1 { v23.s }[0], [x14]\n" - "ld1 { v19.s }[0], [x13]\n" + "tbz x7, #0, 59f\n" + "ld1 { v18.s }[0], [x13]\n" + "ld1 { v22.s }[0], [x12]\n" "b 59f\n" "57:" // Oddments: Load requant params: Bit 2: Unset - "tbz x6, #1, 58f\n" - "ld1 { v17.d }[0], [x14], #0x8\n" - "ld1 { v22.d }[0], [x13], #0x8\n" - "tbz x6, #0, 59f\n" - "ld1 { v17.s }[2], [x14]\n" - "ld1 { v22.s }[2], [x13]\n" + "tbz x7, #1, 58f\n" + "ld1 { v15.d }[0], [x13], #0x8\n" + "ld1 { v19.d }[0], [x12], #0x8\n" + "tbz x7, #0, 59f\n" + "ld1 { v15.s }[2], [x13]\n" + "ld1 { v19.s }[2], [x12]\n" "b 59f\n" "58:" // Oddments: Load requant params: Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 59f\n" - "ld1 { v17.s }[0], [x14]\n" - "ld1 { v22.s }[0], [x13]\n" + "tbz x7, #0, 59f\n" + "ld1 { v15.s }[0], [x13]\n" + "ld1 { v19.s }[0], [x12]\n" "59:" // Oddments: Load requant params: Bit 2: End - "sqrdmulh v13.4s, v13.4s, v17.4s\n" - "and v21.16b, v13.16b, v22.16b\n" - "add x12, x12, x17\n" - "add x11, x11, x17\n" - "sqrdmulh v20.4s, v20.4s, v23.4s\n" - "sshr v21.4s, v21.4s, #0x1f\n" - "add x10, x10, x17\n" - "add x9, x9, x17\n" - "and v29.16b, v20.16b, v19.16b\n" - "sqrdmulh v9.4s, v9.4s, v17.4s\n" - "sqrdmulh v16.4s, v16.4s, v17.4s\n" - "sqrdmulh v25.4s, v25.4s, v17.4s\n" - "sqadd v13.4s, v13.4s, v21.4s\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "and v0.16b, v9.16b, v22.16b\n" - "sqrdmulh v18.4s, v18.4s, v23.4s\n" - "and v27.16b, v16.16b, v22.16b\n" - "sqrdmulh v26.4s, v26.4s, v23.4s\n" - "and v21.16b, v25.16b, v22.16b\n" - "sqrdmulh v10.4s, v10.4s, v23.4s\n" - "sqadd v20.4s, v20.4s, v29.4s\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v17.16b, v18.16b, v19.16b\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "and v7.16b, v26.16b, v19.16b\n" + "sqrdmulh v9.4s, v9.4s, v15.4s\n" + "and v17.16b, v9.16b, v19.16b\n" + "add x11, x11, x16\n" + "add x10, x10, x16\n" + "sqrdmulh v24.4s, v24.4s, v18.4s\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "add x9, x9, x16\n" + "add x28, x28, x16\n" + "and v20.16b, v24.16b, v22.16b\n" + "sqrdmulh v7.4s, v7.4s, v15.4s\n" + "sqrdmulh v2.4s, v2.4s, v15.4s\n" + "sqrdmulh v10.4s, v10.4s, v15.4s\n" + "sqadd v9.4s, v9.4s, v17.4s\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "and v21.16b, v7.16b, v19.16b\n" + "sqrdmulh v0.4s, v0.4s, v18.4s\n" + "and v15.16b, v2.16b, v19.16b\n" + "sqrdmulh v30.4s, v30.4s, v18.4s\n" + "and v23.16b, v10.16b, v19.16b\n" + "sqrdmulh v6.4s, v6.4s, v18.4s\n" + "sqadd v24.4s, v24.4s, v20.4s\n" "sshr v21.4s, v21.4s, #0x1f\n" - "and v29.16b, v10.16b, v19.16b\n" - "sqadd v9.4s, v9.4s, v0.4s\n" + "and v18.16b, v0.16b, v22.16b\n" + "sshr v15.4s, v15.4s, #0x1f\n" + "and v17.16b, v30.16b, v22.16b\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "and v28.16b, v6.16b, v22.16b\n" + "sqadd v7.4s, v7.4s, v21.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sqadd v2.4s, v2.4s, v15.4s\n" "sshr v17.4s, v17.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v27.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v25.4s, v25.4s, v21.4s\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "srshl v13.4s, v13.4s, v22.4s\n" - "srshl v9.4s, v9.4s, v22.4s\n" - "sqadd v18.4s, v18.4s, v17.4s\n" - "srshl v16.4s, v16.4s, v22.4s\n" - "sqadd v26.4s, v26.4s, v7.4s\n" - "srshl v25.4s, v25.4s, v22.4s\n" - "sqadd v10.4s, v10.4s, v29.4s\n" - "srshl v20.4s, v20.4s, v19.4s\n" - "sqxtn v13.4h, v13.4s\n" - "srshl v18.4s, v18.4s, v19.4s\n" - "sqxtn v9.4h, v9.4s\n" - "srshl v26.4s, v26.4s, v19.4s\n" - "sqxtn v16.4h, v16.4s\n" + "sqadd v10.4s, v10.4s, v23.4s\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "srshl v9.4s, v9.4s, v19.4s\n" + "srshl v7.4s, v7.4s, v19.4s\n" + "sqadd v0.4s, v0.4s, v18.4s\n" + "srshl v2.4s, v2.4s, v19.4s\n" + "sqadd v30.4s, v30.4s, v17.4s\n" "srshl v10.4s, v10.4s, v19.4s\n" - "sqxtn v25.4h, v25.4s\n" - "sqxtn2 v13.8h, v20.4s\n" - "sqxtn2 v9.8h, v18.4s\n" - "sqxtn2 v16.8h, v26.4s\n" - "sqxtn2 v25.8h, v10.4s\n" - "sqadd v13.8h, v13.8h, v14.8h\n" - "sqadd v9.8h, v9.8h, v14.8h\n" - "sqadd v16.8h, v16.8h, v14.8h\n" - "sqadd v25.8h, v25.8h, v14.8h\n" - "smax v13.8h, v13.8h, v12.8h\n" - "smax v9.8h, v9.8h, v12.8h\n" - "smax v16.8h, v16.8h, v12.8h\n" - "smax v25.8h, v25.8h, v12.8h\n" - "smin v13.8h, v13.8h, v11.8h\n" - "smin v9.8h, v9.8h, v11.8h\n" - "smin v16.8h, v16.8h, v11.8h\n" - "smin v25.8h, v25.8h, v11.8h\n" - "uzp1 v13.16b, v13.16b, v13.16b\n" + "sqadd v6.4s, v6.4s, v28.4s\n" + "srshl v24.4s, v24.4s, v22.4s\n" + "sqxtn v9.4h, v9.4s\n" + "srshl v0.4s, v0.4s, v22.4s\n" + "sqxtn v7.4h, v7.4s\n" + "srshl v30.4s, v30.4s, v22.4s\n" + "sqxtn v2.4h, v2.4s\n" + "srshl v6.4s, v6.4s, v22.4s\n" + "sqxtn v10.4h, v10.4s\n" + "sqxtn2 v9.8h, v24.4s\n" + "sqxtn2 v7.8h, v0.4s\n" + "sqxtn2 v2.8h, v30.4s\n" + "sqxtn2 v10.8h, v6.4s\n" + "sqadd v9.8h, v9.8h, v13.8h\n" + "sqadd v7.8h, v7.8h, v13.8h\n" + "sqadd v2.8h, v2.8h, v13.8h\n" + "sqadd v10.8h, v10.8h, v13.8h\n" + "smax v9.8h, v9.8h, v29.8h\n" + "smax v7.8h, v7.8h, v29.8h\n" + "smax v2.8h, v2.8h, v29.8h\n" + "smax v10.8h, v10.8h, v29.8h\n" + "smin v9.8h, v9.8h, v12.8h\n" + "smin v7.8h, v7.8h, v12.8h\n" + "smin v2.8h, v2.8h, v12.8h\n" + "smin v10.8h, v10.8h, v12.8h\n" "uzp1 v9.16b, v9.16b, v9.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "tbz x6, #2, 61f\n" - "st1 { v13.s }[0], [x12], #0x4\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" + "uzp1 v2.16b, v2.16b, v2.16b\n" + "uzp1 v10.16b, v10.16b, v10.16b\n" + "tbz x7, #2, 61f\n" "st1 { v9.s }[0], [x11], #0x4\n" - "st1 { v16.s }[0], [x10], #0x4\n" - "st1 { v25.s }[0], [x9], #0x4\n" - "tbz x6, #1, 60f\n" - "st1 { v13.h }[2], [x12], #0x2\n" + "st1 { v7.s }[0], [x10], #0x4\n" + "st1 { v2.s }[0], [x9], #0x4\n" + "st1 { v10.s }[0], [x28], #0x4\n" + "tbz x7, #1, 60f\n" "st1 { v9.h }[2], [x11], #0x2\n" - "st1 { v16.h }[2], [x10], #0x2\n" - "st1 { v25.h }[2], [x9], #0x2\n" - "tbz x6, #0, 63f\n" - "st1 { v13.b }[6], [x12], #0x1\n" + "st1 { v7.h }[2], [x10], #0x2\n" + "st1 { v2.h }[2], [x9], #0x2\n" + "st1 { v10.h }[2], [x28], #0x2\n" + "tbz x7, #0, 63f\n" "st1 { v9.b }[6], [x11], #0x1\n" - "st1 { v16.b }[6], [x10], #0x1\n" - "st1 { v25.b }[6], [x9], #0x1\n" + "st1 { v7.b }[6], [x10], #0x1\n" + "st1 { v2.b }[6], [x9], #0x1\n" + "st1 { v10.b }[6], [x28], #0x1\n" "b 63f\n" "60:" // Oddments: Bit 2: Bit 1: Unset - "tbz x6, #0, 63f\n" - "st1 { v13.b }[4], [x12], #0x1\n" + "tbz x7, #0, 63f\n" "st1 { v9.b }[4], [x11], #0x1\n" - "st1 { v16.b }[4], [x10], #0x1\n" - "st1 { v25.b }[4], [x9], #0x1\n" + "st1 { v7.b }[4], [x10], #0x1\n" + "st1 { v2.b }[4], [x9], #0x1\n" + "st1 { v10.b }[4], [x28], #0x1\n" "b 63f\n" "61:" // Oddments: Bit 2: Unset - "tbz x6, #1, 62f\n" - "st1 { v13.h }[0], [x12], #0x2\n" + "tbz x7, #1, 62f\n" "st1 { v9.h }[0], [x11], #0x2\n" - "st1 { v16.h }[0], [x10], #0x2\n" - "st1 { v25.h }[0], [x9], #0x2\n" - "tbz x6, #0, 63f\n" - "st1 { v13.b }[2], [x12], #0x1\n" + "st1 { v7.h }[0], [x10], #0x2\n" + "st1 { v2.h }[0], [x9], #0x2\n" + "st1 { v10.h }[0], [x28], #0x2\n" + "tbz x7, #0, 63f\n" "st1 { v9.b }[2], [x11], #0x1\n" - "st1 { v16.b }[2], [x10], #0x1\n" - "st1 { v25.b }[2], [x9], #0x1\n" + "st1 { v7.b }[2], [x10], #0x1\n" + "st1 { v2.b }[2], [x9], #0x1\n" + "st1 { v10.b }[2], [x28], #0x1\n" "b 63f\n" "62:" // Oddments: Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 63f\n" - "st1 { v13.b }[0], [x12], #0x1\n" + "tbz x7, #0, 63f\n" "st1 { v9.b }[0], [x11], #0x1\n" - "st1 { v16.b }[0], [x10], #0x1\n" - "st1 { v25.b }[0], [x9], #0x1\n" + "st1 { v7.b }[0], [x10], #0x1\n" + "st1 { v2.b }[0], [x9], #0x1\n" + "st1 { v10.b }[0], [x28], #0x1\n" "63:" // Oddments: Bit 2: End "64:" // End : : [offsetof_Params_bias] "I" (offsetof(Params, bias)), [offsetof_Params_inptrs] "I" (offsetof(Params, inptrs)), [offsetof_Params_n_channels] "I" (offsetof(Params, n_channels)), [offsetof_Params_outptrs] "I" (offsetof(Params, outptrs)), [offsetof_Params_requant] "I" (offsetof(Params, requant)), [offsetof_Params_requant_muls] "I" (offsetof(Params, requant_muls)), [offsetof_Params_requant_shifts] "I" (offsetof(Params, requant_shifts)), [offsetof_Params_weights] "I" (offsetof(Params, weights)), [offsetof_Requantize32_a_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [params] "r" (¶ms) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp index 0baebafa3f96ed321090c55d388888b124ab4763..f4f2bc82e15d031ecd9186e89e5f0f567b3333f6 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include @@ -34,15 +34,7 @@ namespace arm_conv { namespace depthwise { -void a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( - const unsigned int, - const uint8_t *const *const, - const uint8_t *const, - const int32_t *const, - const arm_gemm::Requantize32 &, - const int32_t *const, - const int32_t *const, - uint8_t *const *const); +void a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl(unsigned int, const uint8_t *const *, const uint8_t *, const int32_t *, const arm_gemm::Requantize32 &, const int32_t *, const int32_t *, uint8_t *const *); class a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst : public DepthwiseDepthfirstStrategy { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp index de072a7d55fd49f6291e36a9ea2cb313d001c07e..fb533893a60c27916f5dd71b443254e29af224af 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp @@ -104,16 +104,16 @@ void a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ldr x23, [%x[params], %[offsetof_Params_requant]]\n" "lsr x8, x7, #0x3\n" "add x20, x23, %[offsetof_Requantize32_a_offset]\n" - "ld1r { v12.16b }, [x20]\n" + "ld1r { v6.16b }, [x20]\n" "ldr x22, [%x[params], %[offsetof_Params_outptrs]]\n" "add x21, x23, %[offsetof_Requantize32_b_offset]\n" "add x20, x23, %[offsetof_Requantize32_c_offset]\n" - "ld1r { v13.16b }, [x21]\n" - "ld1r { v11.8h }, [x20]\n" + "ld1r { v15.16b }, [x21]\n" + "ld1r { v13.8h }, [x20]\n" "add x21, x23, %[offsetof_Requantize32_minval]\n" "add x20, x23, %[offsetof_Requantize32_maxval]\n" - "ld1r { v16.8h }, [x21]\n" - "ld1r { v14.8h }, [x20]\n" + "ld1r { v17.8h }, [x21]\n" + "ld1r { v24.8h }, [x20]\n" "mov x17, #0x0\n" "mov x16, #0x0\n" "add x15, %x[params], %[offsetof_Params_inptrs]\n" @@ -123,563 +123,563 @@ void a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ldp x11, x10, [x22, #0x0]\n" "ldp x9, x28, [x22, #0x10]\n" "cbz x8, 3f\n" - "ldr d0, [x14, #0x0]\n" - "ldr d1, [x14, #0x8]\n" + "ldr d11, [x14, #0x0]\n" + "ldr d22, [x14, #0x8]\n" "subs x8, x8, #0x1\n" - "usubl v0.8h, v0.8b, v13.8b\n" - "ldr d2, [x14, #0x10]\n" - "ldr d3, [x14, #0x18]\n" - "usubl v1.8h, v1.8b, v13.8b\n" - "usubl v2.8h, v2.8b, v13.8b\n" - "ldr d4, [x14, #0x20]\n" - "ldr d5, [x14, #0x28]\n" - "usubl v3.8h, v3.8b, v13.8b\n" - "usubl v4.8h, v4.8b, v13.8b\n" - "ldr d6, [x14, #0x30]\n" + "usubl v11.8h, v11.8b, v15.8b\n" + "ldr d14, [x14, #0x10]\n" + "ldr d28, [x14, #0x18]\n" + "usubl v22.8h, v22.8b, v15.8b\n" + "usubl v14.8h, v14.8b, v15.8b\n" + "ldr d18, [x14, #0x20]\n" + "ldr d9, [x14, #0x28]\n" + "usubl v28.8h, v28.8b, v15.8b\n" + "usubl v18.8h, v18.8b, v15.8b\n" + "ldr d26, [x14, #0x30]\n" "ldr d7, [x14, #0x38]\n" - "usubl v5.8h, v5.8b, v13.8b\n" - "usubl v6.8h, v6.8b, v13.8b\n" - "ldr d8, [x14, #0x40]\n" - "ldr x24, [%x[params], %[offsetof_Params_bias]]\n" - "usubl v7.8h, v7.8b, v13.8b\n" - "usubl v8.8h, v8.8b, v13.8b\n" - "ldr q15, [x24, #0x0]\n" - "ldr q17, [x24, #0x10]\n" - "add x24, x24, #0x20\n" - "str x24, [%x[params], %[offsetof_Params_bias]]\n" + "usubl v9.8h, v9.8b, v15.8b\n" + "usubl v26.8h, v26.8b, v15.8b\n" + "ldr d4, [x14, #0x40]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "usubl v7.8h, v7.8b, v15.8b\n" + "usubl v4.8h, v4.8b, v15.8b\n" + "ldr q5, [x20, #0x0]\n" + "ldr q3, [x20, #0x10]\n" + "add x20, x20, #0x20\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" "ldp x27, x26, [x15, #0x0]\n" "ldp x25, x24, [x15, #0x10]\n" - "mov v10.16b, v15.16b\n" - "mov v20.16b, v17.16b\n" + "mov v21.16b, v5.16b\n" + "mov v8.16b, v3.16b\n" "ldp x23, x22, [x15, #0x20]\n" "ldp x21, x20, [x15, #0x30]\n" - "mov v9.16b, v15.16b\n" - "mov v23.16b, v17.16b\n" - "ldr d31, [x27, x17]\n" - "ldr d30, [x26, x17]\n" - "mov v21.16b, v15.16b\n" - "mov v22.16b, v17.16b\n" - "ldr d29, [x25, x17]\n" - "ldr d28, [x24, x17]\n" - "usubl v31.8h, v31.8b, v12.8b\n" - "usubl v30.8h, v30.8b, v12.8b\n" - "ldr d27, [x23, x17]\n" - "ldr d26, [x22, x17]\n" - "usubl v29.8h, v29.8b, v12.8b\n" - "usubl v28.8h, v28.8b, v12.8b\n" - "ldr d25, [x21, x17]\n" - "ldr d24, [x20, x17]\n" - "usubl v27.8h, v27.8b, v12.8b\n" - "usubl v26.8h, v26.8b, v12.8b\n" - "usubl v25.8h, v25.8b, v12.8b\n" - "usubl v24.8h, v24.8b, v12.8b\n" + "mov v20.16b, v5.16b\n" + "mov v0.16b, v3.16b\n" + "ldr d25, [x27, x17]\n" + "ldr d27, [x26, x17]\n" + "mov v19.16b, v5.16b\n" + "mov v31.16b, v3.16b\n" + "ldr d1, [x25, x17]\n" + "ldr d2, [x24, x17]\n" + "usubl v25.8h, v25.8b, v6.8b\n" + "usubl v27.8h, v27.8b, v6.8b\n" + "ldr d12, [x23, x17]\n" + "ldr d16, [x22, x17]\n" + "usubl v1.8h, v1.8b, v6.8b\n" + "usubl v2.8h, v2.8b, v6.8b\n" + "ldr d23, [x21, x17]\n" + "ldr d10, [x20, x17]\n" + "usubl v12.8h, v12.8b, v6.8b\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "usubl v23.8h, v23.8b, v6.8b\n" + "usubl v10.8h, v10.8b, v6.8b\n" "beq 2f\n" "1:" // Loop - "smlal v15.4s, v31.4h, v8.4h\n" - "smlal2 v17.4s, v31.8h, v8.8h\n" - "ldr x24, [x15, #0x40]\n" - "ldr x22, [x15, #0x48]\n" - "smlal v10.4s, v31.4h, v6.4h\n" - "smlal2 v20.4s, v31.8h, v6.8h\n" - "ldr x21, [x15, #0x50]\n" - "ldr x20, [x15, #0x58]\n" - "smlal v15.4s, v30.4h, v0.4h\n" - "smlal2 v17.4s, v30.8h, v0.8h\n" - "ldr q19, [x13, #0x0]\n" - "ldr x23, [x15, #0x78]\n" - "smlal v10.4s, v28.4h, v1.4h\n" - "smlal2 v20.4s, v28.8h, v1.8h\n" - "ldr d28, [x22, x17]\n" - "usubl v28.8h, v28.8b, v12.8b\n" - "smlal v15.4s, v29.4h, v1.4h\n" - "smlal2 v17.4s, v29.8h, v1.8h\n" - "ldr d29, [x24, x17]\n" - "usubl v29.8h, v29.8b, v12.8b\n" - "smlal v10.4s, v27.4h, v2.4h\n" - "smlal2 v20.4s, v27.8h, v2.8h\n" + "ldr q30, [x13, #0x0]\n" + "ldr q29, [x12, #0x0]\n" + "smlal v5.4s, v25.4h, v4.4h\n" + "smlal2 v3.4s, v25.8h, v4.8h\n" + "ldr x21, [x15, #0x58]\n" + "ldr x20, [x15, #0x78]\n" + "smlal v5.4s, v27.4h, v11.4h\n" + "smlal v21.4s, v25.4h, v26.4h\n" + "ldr x25, [x15, #0x60]\n" + "ldr x24, [x15, #0x80]\n" + "smlal v20.4s, v25.4h, v14.4h\n" + "smlal v19.4s, v25.4h, v11.4h\n" + "smlal2 v3.4s, v27.8h, v11.8h\n" "ldr d27, [x21, x17]\n" - "usubl v27.8h, v27.8b, v12.8b\n" - "smlal v15.4s, v26.4h, v3.4h\n" - "smlal2 v17.4s, v26.8h, v3.8h\n" - "ldr d26, [x20, x17]\n" - "ldr x20, [x15, #0x60]\n" - "smlal v10.4s, v24.4h, v0.4h\n" - "smlal2 v20.4s, v24.8h, v0.8h\n" - "usubl v26.8h, v26.8b, v12.8b\n" - "ldr x21, [x15, #0x80]\n" - "smlal v15.4s, v25.4h, v4.4h\n" - "smlal2 v17.4s, v25.8h, v4.8h\n" - "ldr d25, [x20, x17]\n" - "ldr x20, [x15, #0x68]\n" - "smlal v10.4s, v29.4h, v4.4h\n" - "smlal2 v20.4s, v29.8h, v4.8h\n" - "ldr d29, [x20, x17]\n" - "usubl v25.8h, v25.8b, v12.8b\n" - "smlal v15.4s, v24.4h, v2.4h\n" - "smlal2 v17.4s, v24.8h, v2.8h\n" - "ldr q18, [x12, #0x0]\n" + "usubl v27.8h, v27.8b, v6.8b\n" + "smlal v5.4s, v1.4h, v22.4h\n" + "smlal2 v8.4s, v25.8h, v26.8h\n" + "smlal2 v0.4s, v25.8h, v14.8h\n" + "ldr x23, [x15, #0x68]\n" "ldr x22, [x15, #0x88]\n" - "smlal v10.4s, v28.4h, v5.4h\n" - "smlal2 v20.4s, v28.8h, v5.8h\n" - "ldr d28, [x21, x17]\n" - "ldr x21, [x15, #0x70]\n" - "smlal v9.4s, v31.4h, v2.4h\n" - "smlal2 v23.4s, v31.8h, v2.8h\n" - "usubl v28.8h, v28.8b, v12.8b\n" + "smlal2 v31.4s, v25.8h, v11.8h\n" + "ldr d25, [x20, x17]\n" + "usubl v25.8h, v25.8b, v6.8b\n" + "smlal v21.4s, v2.4h, v22.4h\n" + "smlal v20.4s, v27.4h, v28.4h\n" + "smlal v19.4s, v25.4h, v18.4h\n" + "ldr x21, [x15, #0x40]\n" + "ldr x20, [x15, #0x70]\n" + "smlal2 v3.4s, v1.8h, v22.8h\n" + "ldr d1, [x25, x17]\n" + "usubl v1.8h, v1.8b, v6.8b\n" + "smlal v5.4s, v16.4h, v28.4h\n" + "smlal2 v8.4s, v2.8h, v22.8h\n" + "ldr d2, [x24, x17]\n" + "usubl v2.8h, v2.8b, v6.8b\n" + "smlal2 v0.4s, v27.8h, v28.8h\n" + "ldr d27, [x23, x17]\n" + "smlal2 v31.4s, v25.8h, v18.8h\n" + "ldr d25, [x22, x17]\n" + "smlal v21.4s, v12.4h, v14.4h\n" "ldr x25, [x15, #0x98]\n" - "smlal v15.4s, v27.4h, v5.4h\n" - "smlal2 v17.4s, v27.8h, v5.8h\n" - "usubl v29.8h, v29.8b, v12.8b\n" - "ldr x24, [x15, #0x90]\n" - "smlal v10.4s, v27.4h, v3.4h\n" - "smlal2 v20.4s, v27.8h, v3.8h\n" + "smlal v20.4s, v1.4h, v11.4h\n" + "smlal v19.4s, v2.4h, v22.4h\n" + "ldr x24, [x15, #0x50]\n" + "smlal2 v3.4s, v16.8h, v28.8h\n" + "ldr d16, [x21, x17]\n" + "usubl v27.8h, v27.8b, v6.8b\n" + "smlal v5.4s, v23.4h, v18.4h\n" + "usubl v25.8h, v25.8b, v6.8b\n" + "smlal2 v8.4s, v12.8h, v14.8h\n" + "ldr d12, [x20, x17]\n" + "ldr x23, [x15, #0x48]\n" + "smlal2 v0.4s, v1.8h, v11.8h\n" + "smlal2 v31.4s, v2.8h, v22.8h\n" + "ldr x21, [x15, #0x90]\n" + "ldr x20, [x15, #0xa8]\n" + "smlal v21.4s, v10.4h, v11.4h\n" + "smlal v20.4s, v27.4h, v18.4h\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "ldr x22, [x15, #0xa0]\n" + "smlal v19.4s, v25.4h, v9.4h\n" + "smlal2 v3.4s, v23.8h, v18.8h\n" + "ldr d23, [x25, x17]\n" + "usubl v12.8h, v12.8b, v6.8b\n" + "usubl v23.8h, v23.8b, v6.8b\n" + "smlal v5.4s, v10.4h, v14.4h\n" + "smlal2 v8.4s, v10.8h, v11.8h\n" + "ldr d11, [x24, x17]\n" + "usubl v11.8h, v11.8b, v6.8b\n" + "smlal2 v0.4s, v27.8h, v18.8h\n" "ldr d27, [x23, x17]\n" - "usubl v27.8h, v27.8b, v12.8b\n" - "smlal v21.4s, v31.4h, v0.4h\n" - "smlal v9.4s, v26.4h, v3.4h\n" - "ldr x23, [x15, #0xa8]\n" - "ldr x20, [x15, #0xa0]\n" - "smlal2 v23.4s, v26.8h, v3.8h\n" - "ldr d26, [x22, x17]\n" - "smlal2 v22.4s, v31.8h, v0.8h\n" - "ldr d24, [x21, x17]\n" - "smlal v21.4s, v27.4h, v4.4h\n" - "smlal v9.4s, v25.4h, v0.4h\n" - "usubl v26.8h, v26.8b, v12.8b\n" - "ldr x22, [x15, #0xb0]\n" - "smlal2 v23.4s, v25.8h, v0.8h\n" - "ldr q30, [x13, #0x10]\n" - "smlal2 v22.4s, v27.8h, v4.8h\n" - "ldr d27, [x20, x17]\n" - "smlal v21.4s, v28.4h, v1.4h\n" - "smlal v15.4s, v25.4h, v6.4h\n" - "usubl v24.8h, v24.8b, v12.8b\n" - "ldr x21, [x15, #0xb8]\n" - "smlal2 v17.4s, v25.8h, v6.8h\n" - "ldr d25, [x24, x17]\n" - "smlal v9.4s, v29.4h, v4.4h\n" - "usubl v25.8h, v25.8b, v12.8b\n" - "smlal2 v23.4s, v29.8h, v4.8h\n" - "ldr d29, [x25, x17]\n" - "ldr q31, [x12, #0x10]\n" - "smlal2 v22.4s, v28.8h, v1.8h\n" - "smlal v21.4s, v26.4h, v5.4h\n" - "usubl v29.8h, v29.8b, v12.8b\n" - "smlal v15.4s, v24.4h, v7.4h\n" - "ldr x20, [x15, #0xc0]\n" - "smlal2 v17.4s, v24.8h, v7.8h\n" - "smlal v9.4s, v24.4h, v1.4h\n" - "usubl v27.8h, v27.8b, v12.8b\n" - "ldr x24, [%x[params], %[offsetof_Params_bias]]\n" - "smlal2 v23.4s, v24.8h, v1.8h\n" - "ldr d24, [x23, x17]\n" - "smlal2 v22.4s, v26.8h, v5.8h\n" - "ldr d26, [x22, x17]\n" - "smlal v21.4s, v29.4h, v2.4h\n" - "usubl v24.8h, v24.8b, v12.8b\n" - "smlal2 v22.4s, v29.8h, v2.8h\n" + "smlal2 v31.4s, v25.8h, v9.8h\n" + "ldr d25, [x21, x17]\n" + "ldr x21, [x15, #0xb0]\n" + "smlal v21.4s, v16.4h, v18.4h\n" + "smlal v20.4s, v12.4h, v22.4h\n" + "smlal v19.4s, v23.4h, v14.4h\n" + "smlal2 v3.4s, v10.8h, v14.8h\n" + "ldr d10, [x20, x17]\n" + "usubl v27.8h, v27.8b, v6.8b\n" + "usubl v25.8h, v25.8b, v6.8b\n" + "usubl v10.8h, v10.8b, v6.8b\n" + "smlal v5.4s, v11.4h, v9.4h\n" + "ldr x20, [x15, #0xb8]\n" + "smlal2 v8.4s, v16.8h, v18.8h\n" + "ldr d18, [x22, x17]\n" + "ldr d16, [x21, x17]\n" + "smlal2 v0.4s, v12.8h, v22.8h\n" + "ldr d22, [x20, x17]\n" + "smlal2 v31.4s, v23.8h, v14.8h\n" + "ldr q14, [x13, #0x10]\n" + "smlal v21.4s, v27.4h, v9.4h\n" + "smlal v20.4s, v25.4h, v26.4h\n" + "smlal v19.4s, v10.4h, v28.4h\n" + "usubl v18.8h, v18.8b, v6.8b\n" + "ldr x21, [x15, #0xc0]\n" + "smlal2 v3.4s, v11.8h, v9.8h\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "smlal v5.4s, v1.4h, v26.4h\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "smlal2 v8.4s, v27.8h, v9.8h\n" + "ldr d27, [x21, x17]\n" + "smlal2 v0.4s, v25.8h, v26.8h\n" + "ldr q25, [x12, #0x10]\n" + "smlal2 v31.4s, v10.8h, v28.8h\n" + "smlal v21.4s, v11.4h, v28.4h\n" + "usubl v22.8h, v22.8b, v6.8b\n" "add x14, x14, #0x48\n" - "smlal v9.4s, v25.4h, v6.4h\n" - "smlal v21.4s, v24.4h, v3.4h\n" - "usubl v26.8h, v26.8b, v12.8b\n" + "smlal v20.4s, v18.4h, v7.4h\n" + "smlal v19.4s, v16.4h, v7.4h\n" + "usubl v27.8h, v27.8b, v6.8b\n" + "add x17, x17, #0x8\n" + "smlal2 v3.4s, v1.8h, v26.8h\n" + "smlal v5.4s, v12.4h, v7.4h\n" + "sqrdmulh v5.4s, v5.4s, v30.4s\n" "subs x8, x8, #0x1\n" - "smlal v10.4s, v28.4h, v7.4h\n" - "smlal2 v20.4s, v28.8h, v7.8h\n" - "sqrdmulh v15.4s, v15.4s, v19.4s\n" + "smlal2 v8.4s, v11.8h, v28.8h\n" + "smlal2 v0.4s, v18.8h, v7.8h\n" + "and v28.16b, v5.16b, v29.16b\n" "add x13, x13, #0x20\n" - "smlal2 v23.4s, v25.8h, v6.8h\n" - "ldr d25, [x21, x17]\n" - "smlal2 v22.4s, v24.8h, v3.8h\n" - "usubl v25.8h, v25.8b, v12.8b\n" - "smlal v9.4s, v27.4h, v7.4h\n" - "smlal v21.4s, v26.4h, v7.4h\n" - "and v0.16b, v15.16b, v18.16b\n" + "smlal2 v31.4s, v16.8h, v7.8h\n" + "smlal v21.4s, v2.4h, v7.4h\n" + "sshr v28.4s, v28.4s, #0x1f\n" "add x12, x12, #0x20\n" - "smlal v10.4s, v29.4h, v8.4h\n" - "smlal2 v20.4s, v29.8h, v8.8h\n" - "ldr d29, [x20, x17]\n" - "usubl v29.8h, v29.8b, v12.8b\n" - "smlal2 v23.4s, v27.8h, v7.8h\n" - "smlal2 v22.4s, v26.8h, v7.8h\n" - "sqrdmulh v17.4s, v17.4s, v30.4s\n" - "add x17, x17, #0x8\n" - "smlal v9.4s, v24.4h, v5.4h\n" - "smlal v21.4s, v25.4h, v6.4h\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "smlal2 v23.4s, v24.8h, v5.8h\n" - "smlal2 v22.4s, v25.8h, v6.8h\n" - "and v7.16b, v17.16b, v31.16b\n" - "smlal v9.4s, v25.4h, v8.4h\n" - "smlal v21.4s, v29.4h, v8.4h\n" - "sqrdmulh v10.4s, v10.4s, v19.4s\n" - "smlal2 v23.4s, v25.8h, v8.8h\n" - "smlal2 v22.4s, v29.8h, v8.8h\n" - "sqrdmulh v9.4s, v9.4s, v19.4s\n" - "sqrdmulh v21.4s, v21.4s, v19.4s\n" - "sqadd v15.4s, v15.4s, v0.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "and v19.16b, v10.16b, v18.16b\n" + "smlal v20.4s, v10.4h, v9.4h\n" + "smlal v19.4s, v22.4h, v26.4h\n" + "sqadd v5.4s, v5.4s, v28.4s\n" + "smlal2 v3.4s, v12.8h, v7.8h\n" + "smlal2 v8.4s, v2.8h, v7.8h\n" + "sqrdmulh v3.4s, v3.4s, v14.4s\n" + "smlal2 v0.4s, v10.8h, v9.8h\n" + "smlal2 v31.4s, v22.8h, v26.8h\n" + "and v16.16b, v3.16b, v25.16b\n" + "smlal v21.4s, v23.4h, v4.4h\n" + "smlal v20.4s, v22.4h, v4.4h\n" + "sqrdmulh v21.4s, v21.4s, v30.4s\n" + "smlal v19.4s, v27.4h, v4.4h\n" + "smlal2 v8.4s, v23.8h, v4.8h\n" "sqrdmulh v20.4s, v20.4s, v30.4s\n" - "and v27.16b, v9.16b, v18.16b\n" - "sqrdmulh v23.4s, v23.4s, v30.4s\n" - "and v0.16b, v21.16b, v18.16b\n" - "sqrdmulh v22.4s, v22.4s, v30.4s\n" - "sqadd v17.4s, v17.4s, v7.4s\n" - "sshr v19.4s, v19.4s, #0x1f\n" - "and v5.16b, v20.16b, v31.16b\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "and v4.16b, v23.16b, v31.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v7.16b, v22.16b, v31.16b\n" - "sqadd v10.4s, v10.4s, v19.4s\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v9.4s, v9.4s, v27.4s\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v21.4s, v21.4s, v0.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "srshl v15.4s, v15.4s, v18.4s\n" - "srshl v10.4s, v10.4s, v18.4s\n" - "sqadd v20.4s, v20.4s, v5.4s\n" - "srshl v9.4s, v9.4s, v18.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "srshl v21.4s, v21.4s, v18.4s\n" - "sqadd v22.4s, v22.4s, v7.4s\n" - "srshl v17.4s, v17.4s, v31.4s\n" - "sqxtn v15.4h, v15.4s\n" - "srshl v20.4s, v20.4s, v31.4s\n" - "sqxtn v10.4h, v10.4s\n" - "srshl v23.4s, v23.4s, v31.4s\n" - "sqxtn v9.4h, v9.4s\n" - "srshl v22.4s, v22.4s, v31.4s\n" + "smlal2 v0.4s, v22.8h, v4.8h\n" + "smlal2 v31.4s, v27.8h, v4.8h\n" + "sqrdmulh v19.4s, v19.4s, v30.4s\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "and v12.16b, v21.16b, v29.16b\n" + "sqrdmulh v8.4s, v8.4s, v14.4s\n" + "and v23.16b, v20.16b, v29.16b\n" + "sqrdmulh v0.4s, v0.4s, v14.4s\n" + "and v9.16b, v19.16b, v29.16b\n" + "sqrdmulh v31.4s, v31.4s, v14.4s\n" + "sqadd v3.4s, v3.4s, v16.4s\n" + "sshr v12.4s, v12.4s, #0x1f\n" + "and v18.16b, v8.16b, v25.16b\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "and v22.16b, v0.16b, v25.16b\n" + "sshr v9.4s, v9.4s, #0x1f\n" + "and v16.16b, v31.16b, v25.16b\n" + "sqadd v21.4s, v21.4s, v12.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v23.4s\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "sqadd v19.4s, v19.4s, v9.4s\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "srshl v5.4s, v5.4s, v29.4s\n" + "srshl v21.4s, v21.4s, v29.4s\n" + "sqadd v8.4s, v8.4s, v18.4s\n" + "srshl v20.4s, v20.4s, v29.4s\n" + "sqadd v0.4s, v0.4s, v22.4s\n" + "srshl v19.4s, v19.4s, v29.4s\n" + "sqadd v31.4s, v31.4s, v16.4s\n" + "srshl v3.4s, v3.4s, v25.4s\n" + "sqxtn v5.4h, v5.4s\n" + "srshl v8.4s, v8.4s, v25.4s\n" "sqxtn v21.4h, v21.4s\n" - "sqxtn2 v15.8h, v17.4s\n" - "sqxtn2 v10.8h, v20.4s\n" - "sqxtn2 v9.8h, v23.4s\n" - "sqxtn2 v21.8h, v22.4s\n" - "sqadd v15.8h, v15.8h, v11.8h\n" - "sqadd v10.8h, v10.8h, v11.8h\n" - "sqadd v9.8h, v9.8h, v11.8h\n" - "sqadd v21.8h, v21.8h, v11.8h\n" - "smax v15.8h, v15.8h, v16.8h\n" - "smax v10.8h, v10.8h, v16.8h\n" - "smax v9.8h, v9.8h, v16.8h\n" - "smax v21.8h, v21.8h, v16.8h\n" - "smin v15.8h, v15.8h, v14.8h\n" - "smin v10.8h, v10.8h, v14.8h\n" - "smin v9.8h, v9.8h, v14.8h\n" - "smin v21.8h, v21.8h, v14.8h\n" - "uzp1 v15.16b, v15.16b, v15.16b\n" - "str d15, [x11, x16]\n" - "uzp1 v10.16b, v10.16b, v10.16b\n" - "uzp1 v9.16b, v9.16b, v9.16b\n" - "str d10, [x10, x16]\n" + "srshl v0.4s, v0.4s, v25.4s\n" + "sqxtn v20.4h, v20.4s\n" + "srshl v31.4s, v31.4s, v25.4s\n" + "sqxtn v19.4h, v19.4s\n" + "sqxtn2 v5.8h, v3.4s\n" + "sqxtn2 v21.8h, v8.4s\n" + "sqxtn2 v20.8h, v0.4s\n" + "sqxtn2 v19.8h, v31.4s\n" + "sqadd v5.8h, v5.8h, v13.8h\n" + "sqadd v21.8h, v21.8h, v13.8h\n" + "sqadd v20.8h, v20.8h, v13.8h\n" + "sqadd v19.8h, v19.8h, v13.8h\n" + "smax v5.8h, v5.8h, v17.8h\n" + "smax v21.8h, v21.8h, v17.8h\n" + "smax v20.8h, v20.8h, v17.8h\n" + "smax v19.8h, v19.8h, v17.8h\n" + "smin v5.8h, v5.8h, v24.8h\n" + "smin v21.8h, v21.8h, v24.8h\n" + "smin v20.8h, v20.8h, v24.8h\n" + "smin v19.8h, v19.8h, v24.8h\n" + "uzp1 v5.16b, v5.16b, v5.16b\n" + "str d5, [x11, x16]\n" "uzp1 v21.16b, v21.16b, v21.16b\n" - "str d9, [x9, x16]\n" - "str d21, [x28, x16]\n" - "ldr q15, [x24, #0x0]\n" - "ldr q17, [x24, #0x10]\n" - "add x24, x24, #0x20\n" - "ldr d0, [x14, #0x0]\n" - "ldr d1, [x14, #0x8]\n" + "uzp1 v20.16b, v20.16b, v20.16b\n" + "str d21, [x10, x16]\n" + "uzp1 v19.16b, v19.16b, v19.16b\n" + "str d20, [x9, x16]\n" + "str d19, [x28, x16]\n" + "ldr q5, [x20, #0x0]\n" + "ldr q3, [x20, #0x10]\n" + "add x20, x20, #0x20\n" + "ldr d11, [x14, #0x0]\n" + "ldr d22, [x14, #0x8]\n" "add x16, x16, #0x8\n" - "str x24, [%x[params], %[offsetof_Params_bias]]\n" - "ldr d2, [x14, #0x10]\n" - "ldr d3, [x14, #0x18]\n" - "mov v10.16b, v15.16b\n" - "mov v20.16b, v17.16b\n" - "ldr d4, [x14, #0x20]\n" - "ldr d5, [x14, #0x28]\n" - "mov v9.16b, v15.16b\n" - "mov v23.16b, v17.16b\n" - "ldr d6, [x14, #0x30]\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" + "ldr d14, [x14, #0x10]\n" + "ldr d28, [x14, #0x18]\n" + "mov v21.16b, v5.16b\n" + "mov v8.16b, v3.16b\n" + "ldr d18, [x14, #0x20]\n" + "ldr d9, [x14, #0x28]\n" + "mov v20.16b, v5.16b\n" + "mov v0.16b, v3.16b\n" + "ldr d26, [x14, #0x30]\n" "ldr d7, [x14, #0x38]\n" - "mov v21.16b, v15.16b\n" - "mov v22.16b, v17.16b\n" - "ldr d8, [x14, #0x40]\n" + "mov v19.16b, v5.16b\n" + "mov v31.16b, v3.16b\n" + "ldr d4, [x14, #0x40]\n" "ldp x27, x26, [x15, #0x0]\n" - "usubl v0.8h, v0.8b, v13.8b\n" - "usubl v1.8h, v1.8b, v13.8b\n" + "usubl v11.8h, v11.8b, v15.8b\n" + "usubl v22.8h, v22.8b, v15.8b\n" "ldp x25, x24, [x15, #0x10]\n" "ldp x23, x22, [x15, #0x20]\n" - "usubl v2.8h, v2.8b, v13.8b\n" - "usubl v3.8h, v3.8b, v13.8b\n" + "usubl v14.8h, v14.8b, v15.8b\n" + "usubl v28.8h, v28.8b, v15.8b\n" "ldp x21, x20, [x15, #0x30]\n" - "ldr d31, [x27, x17]\n" - "usubl v4.8h, v4.8b, v13.8b\n" - "usubl v5.8h, v5.8b, v13.8b\n" - "ldr d30, [x26, x17]\n" - "ldr d29, [x25, x17]\n" - "usubl v6.8h, v6.8b, v13.8b\n" - "usubl v7.8h, v7.8b, v13.8b\n" - "ldr d28, [x24, x17]\n" - "ldr d27, [x23, x17]\n" - "usubl v8.8h, v8.8b, v13.8b\n" - "usubl v31.8h, v31.8b, v12.8b\n" - "ldr d26, [x22, x17]\n" - "ldr d25, [x21, x17]\n" - "usubl v30.8h, v30.8b, v12.8b\n" - "usubl v29.8h, v29.8b, v12.8b\n" - "ldr d24, [x20, x17]\n" - "usubl v28.8h, v28.8b, v12.8b\n" - "usubl v27.8h, v27.8b, v12.8b\n" - "usubl v26.8h, v26.8b, v12.8b\n" - "usubl v25.8h, v25.8b, v12.8b\n" - "usubl v24.8h, v24.8b, v12.8b\n" + "ldr d25, [x27, x17]\n" + "usubl v18.8h, v18.8b, v15.8b\n" + "usubl v9.8h, v9.8b, v15.8b\n" + "ldr d27, [x26, x17]\n" + "ldr d1, [x25, x17]\n" + "usubl v26.8h, v26.8b, v15.8b\n" + "usubl v7.8h, v7.8b, v15.8b\n" + "ldr d2, [x24, x17]\n" + "ldr d12, [x23, x17]\n" + "usubl v4.8h, v4.8b, v15.8b\n" + "usubl v25.8h, v25.8b, v6.8b\n" + "ldr d16, [x22, x17]\n" + "ldr d23, [x21, x17]\n" + "usubl v27.8h, v27.8b, v6.8b\n" + "usubl v1.8h, v1.8b, v6.8b\n" + "ldr d10, [x20, x17]\n" + "usubl v2.8h, v2.8b, v6.8b\n" + "usubl v12.8h, v12.8b, v6.8b\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "usubl v23.8h, v23.8b, v6.8b\n" + "usubl v10.8h, v10.8b, v6.8b\n" "bgt 1b\n" "2:" // Tail - "smlal v15.4s, v31.4h, v8.4h\n" - "smlal2 v17.4s, v31.8h, v8.8h\n" - "ldr x24, [x15, #0x40]\n" - "ldr x22, [x15, #0x48]\n" - "smlal v10.4s, v31.4h, v6.4h\n" - "smlal2 v20.4s, v31.8h, v6.8h\n" - "ldr x21, [x15, #0x50]\n" - "ldr x20, [x15, #0x58]\n" - "smlal v15.4s, v30.4h, v0.4h\n" - "smlal2 v17.4s, v30.8h, v0.8h\n" - "ldr q19, [x13, #0x0]\n" - "ldr x23, [x15, #0x78]\n" - "smlal v10.4s, v28.4h, v1.4h\n" - "smlal2 v20.4s, v28.8h, v1.8h\n" - "ldr d28, [x22, x17]\n" - "usubl v28.8h, v28.8b, v12.8b\n" - "smlal v15.4s, v29.4h, v1.4h\n" - "smlal2 v17.4s, v29.8h, v1.8h\n" - "ldr d29, [x24, x17]\n" - "usubl v29.8h, v29.8b, v12.8b\n" - "smlal v10.4s, v27.4h, v2.4h\n" - "smlal2 v20.4s, v27.8h, v2.8h\n" + "ldr q29, [x13, #0x0]\n" + "ldr q30, [x12, #0x0]\n" + "smlal v5.4s, v25.4h, v4.4h\n" + "smlal2 v3.4s, v25.8h, v4.8h\n" + "ldr x21, [x15, #0x58]\n" + "ldr x20, [x15, #0x78]\n" + "smlal v5.4s, v27.4h, v11.4h\n" + "smlal v21.4s, v25.4h, v26.4h\n" + "ldr x25, [x15, #0x60]\n" + "ldr x24, [x15, #0x80]\n" + "smlal v20.4s, v25.4h, v14.4h\n" + "smlal v19.4s, v25.4h, v11.4h\n" + "smlal2 v3.4s, v27.8h, v11.8h\n" "ldr d27, [x21, x17]\n" - "usubl v27.8h, v27.8b, v12.8b\n" - "smlal v15.4s, v26.4h, v3.4h\n" - "smlal2 v17.4s, v26.8h, v3.8h\n" - "ldr d26, [x20, x17]\n" - "ldr x20, [x15, #0x60]\n" - "smlal v10.4s, v24.4h, v0.4h\n" - "smlal2 v20.4s, v24.8h, v0.8h\n" - "usubl v26.8h, v26.8b, v12.8b\n" - "ldr x21, [x15, #0x80]\n" - "smlal v15.4s, v25.4h, v4.4h\n" - "smlal2 v17.4s, v25.8h, v4.8h\n" - "ldr d25, [x20, x17]\n" - "ldr x20, [x15, #0x68]\n" - "smlal v10.4s, v29.4h, v4.4h\n" - "smlal2 v20.4s, v29.8h, v4.8h\n" - "ldr d29, [x20, x17]\n" - "usubl v25.8h, v25.8b, v12.8b\n" - "smlal v15.4s, v24.4h, v2.4h\n" - "smlal2 v17.4s, v24.8h, v2.8h\n" - "ldr q18, [x12, #0x0]\n" + "usubl v27.8h, v27.8b, v6.8b\n" + "smlal v5.4s, v1.4h, v22.4h\n" + "smlal2 v8.4s, v25.8h, v26.8h\n" + "smlal2 v0.4s, v25.8h, v14.8h\n" + "ldr x23, [x15, #0x68]\n" "ldr x22, [x15, #0x88]\n" - "smlal v10.4s, v28.4h, v5.4h\n" - "smlal2 v20.4s, v28.8h, v5.8h\n" - "ldr d28, [x21, x17]\n" - "ldr x21, [x15, #0x70]\n" - "smlal v9.4s, v31.4h, v2.4h\n" - "smlal2 v23.4s, v31.8h, v2.8h\n" - "usubl v28.8h, v28.8b, v12.8b\n" + "smlal2 v31.4s, v25.8h, v11.8h\n" + "ldr d25, [x20, x17]\n" + "usubl v25.8h, v25.8b, v6.8b\n" + "smlal v21.4s, v2.4h, v22.4h\n" + "smlal v20.4s, v27.4h, v28.4h\n" + "smlal v19.4s, v25.4h, v18.4h\n" + "ldr x21, [x15, #0x40]\n" + "ldr x20, [x15, #0x70]\n" + "smlal2 v3.4s, v1.8h, v22.8h\n" + "ldr d1, [x25, x17]\n" + "usubl v1.8h, v1.8b, v6.8b\n" + "smlal v5.4s, v16.4h, v28.4h\n" + "smlal2 v8.4s, v2.8h, v22.8h\n" + "ldr d2, [x24, x17]\n" + "usubl v2.8h, v2.8b, v6.8b\n" + "smlal2 v0.4s, v27.8h, v28.8h\n" + "ldr d27, [x23, x17]\n" + "smlal2 v31.4s, v25.8h, v18.8h\n" + "ldr d25, [x22, x17]\n" + "smlal v21.4s, v12.4h, v14.4h\n" "ldr x25, [x15, #0x98]\n" - "smlal v15.4s, v27.4h, v5.4h\n" - "smlal2 v17.4s, v27.8h, v5.8h\n" - "usubl v29.8h, v29.8b, v12.8b\n" - "ldr x24, [x15, #0x90]\n" - "smlal v10.4s, v27.4h, v3.4h\n" - "smlal2 v20.4s, v27.8h, v3.8h\n" + "smlal v20.4s, v1.4h, v11.4h\n" + "smlal v19.4s, v2.4h, v22.4h\n" + "ldr x24, [x15, #0x50]\n" + "smlal2 v3.4s, v16.8h, v28.8h\n" + "ldr d16, [x21, x17]\n" + "usubl v27.8h, v27.8b, v6.8b\n" + "smlal v5.4s, v23.4h, v18.4h\n" + "usubl v25.8h, v25.8b, v6.8b\n" + "smlal2 v8.4s, v12.8h, v14.8h\n" + "ldr d12, [x20, x17]\n" + "ldr x23, [x15, #0x48]\n" + "smlal2 v0.4s, v1.8h, v11.8h\n" + "smlal2 v31.4s, v2.8h, v22.8h\n" + "ldr x21, [x15, #0x90]\n" + "ldr x20, [x15, #0xa8]\n" + "smlal v21.4s, v10.4h, v11.4h\n" + "smlal v20.4s, v27.4h, v18.4h\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "ldr x22, [x15, #0xa0]\n" + "smlal v19.4s, v25.4h, v9.4h\n" + "smlal2 v3.4s, v23.8h, v18.8h\n" + "ldr d23, [x25, x17]\n" + "usubl v12.8h, v12.8b, v6.8b\n" + "usubl v23.8h, v23.8b, v6.8b\n" + "smlal v5.4s, v10.4h, v14.4h\n" + "smlal2 v8.4s, v10.8h, v11.8h\n" + "ldr d11, [x24, x17]\n" + "usubl v11.8h, v11.8b, v6.8b\n" + "smlal2 v0.4s, v27.8h, v18.8h\n" "ldr d27, [x23, x17]\n" - "usubl v27.8h, v27.8b, v12.8b\n" - "smlal v21.4s, v31.4h, v0.4h\n" - "smlal v9.4s, v26.4h, v3.4h\n" - "ldr x23, [x15, #0xa8]\n" - "ldr x20, [x15, #0xa0]\n" - "smlal2 v23.4s, v26.8h, v3.8h\n" - "ldr d26, [x22, x17]\n" - "smlal2 v22.4s, v31.8h, v0.8h\n" - "ldr d24, [x21, x17]\n" - "smlal v21.4s, v27.4h, v4.4h\n" - "smlal v9.4s, v25.4h, v0.4h\n" - "usubl v26.8h, v26.8b, v12.8b\n" - "ldr x22, [x15, #0xb0]\n" - "smlal2 v23.4s, v25.8h, v0.8h\n" - "ldr q30, [x13, #0x10]\n" - "smlal2 v22.4s, v27.8h, v4.8h\n" - "ldr d27, [x20, x17]\n" - "smlal v21.4s, v28.4h, v1.4h\n" - "smlal v15.4s, v25.4h, v6.4h\n" - "usubl v24.8h, v24.8b, v12.8b\n" - "ldr x21, [x15, #0xb8]\n" - "smlal2 v17.4s, v25.8h, v6.8h\n" - "ldr d25, [x24, x17]\n" - "smlal v9.4s, v29.4h, v4.4h\n" - "usubl v25.8h, v25.8b, v12.8b\n" - "smlal2 v23.4s, v29.8h, v4.8h\n" - "ldr d29, [x25, x17]\n" - "ldr q31, [x12, #0x10]\n" - "smlal2 v22.4s, v28.8h, v1.8h\n" - "smlal v21.4s, v26.4h, v5.4h\n" - "usubl v29.8h, v29.8b, v12.8b\n" - "smlal v15.4s, v24.4h, v7.4h\n" + "smlal2 v31.4s, v25.8h, v9.8h\n" + "ldr d25, [x21, x17]\n" + "ldr x21, [x15, #0xb0]\n" + "smlal v21.4s, v16.4h, v18.4h\n" + "smlal v20.4s, v12.4h, v22.4h\n" + "smlal v19.4s, v23.4h, v14.4h\n" + "smlal2 v3.4s, v10.8h, v14.8h\n" + "ldr d10, [x20, x17]\n" + "usubl v27.8h, v27.8b, v6.8b\n" + "usubl v25.8h, v25.8b, v6.8b\n" + "usubl v10.8h, v10.8b, v6.8b\n" + "smlal v5.4s, v11.4h, v9.4h\n" + "ldr x20, [x15, #0xb8]\n" + "smlal2 v8.4s, v16.8h, v18.8h\n" + "ldr d16, [x22, x17]\n" + "ldr d18, [x21, x17]\n" + "smlal2 v0.4s, v12.8h, v22.8h\n" + "ldr d22, [x20, x17]\n" + "smlal2 v31.4s, v23.8h, v14.8h\n" + "ldr q14, [x13, #0x10]\n" + "smlal v21.4s, v27.4h, v9.4h\n" + "smlal v20.4s, v25.4h, v26.4h\n" + "smlal v19.4s, v10.4h, v28.4h\n" + "usubl v16.8h, v16.8b, v6.8b\n" "ldr x20, [x15, #0xc0]\n" - "smlal2 v17.4s, v24.8h, v7.8h\n" - "smlal v9.4s, v24.4h, v1.4h\n" - "usubl v27.8h, v27.8b, v12.8b\n" + "smlal2 v3.4s, v11.8h, v9.8h\n" + "usubl v18.8h, v18.8b, v6.8b\n" + "smlal v5.4s, v1.4h, v26.4h\n" "tst x7, #0x7\n" - "smlal2 v23.4s, v24.8h, v1.8h\n" - "ldr d24, [x23, x17]\n" - "smlal2 v22.4s, v26.8h, v5.8h\n" - "ldr d26, [x22, x17]\n" - "smlal v21.4s, v29.4h, v2.4h\n" - "usubl v24.8h, v24.8b, v12.8b\n" - "smlal2 v22.4s, v29.8h, v2.8h\n" + "smlal2 v8.4s, v27.8h, v9.8h\n" + "ldr d27, [x20, x17]\n" + "smlal2 v0.4s, v25.8h, v26.8h\n" + "ldr q25, [x12, #0x10]\n" + "smlal2 v31.4s, v10.8h, v28.8h\n" + "smlal v21.4s, v11.4h, v28.4h\n" + "usubl v22.8h, v22.8b, v6.8b\n" + "add x17, x17, #0x8\n" + "smlal v20.4s, v16.4h, v7.4h\n" + "smlal v19.4s, v18.4h, v7.4h\n" + "usubl v27.8h, v27.8b, v6.8b\n" "add x13, x13, #0x20\n" - "smlal v9.4s, v25.4h, v6.4h\n" - "smlal v21.4s, v24.4h, v3.4h\n" - "usubl v26.8h, v26.8b, v12.8b\n" + "smlal2 v3.4s, v1.8h, v26.8h\n" + "smlal v5.4s, v12.4h, v7.4h\n" + "sqrdmulh v5.4s, v5.4s, v29.4s\n" "add x12, x12, #0x20\n" - "smlal v10.4s, v28.4h, v7.4h\n" - "smlal2 v20.4s, v28.8h, v7.8h\n" - "sqrdmulh v15.4s, v15.4s, v19.4s\n" - "smlal2 v23.4s, v25.8h, v6.8h\n" - "ldr d25, [x21, x17]\n" - "smlal2 v22.4s, v24.8h, v3.8h\n" - "usubl v25.8h, v25.8b, v12.8b\n" - "smlal v9.4s, v27.4h, v7.4h\n" - "smlal v21.4s, v26.4h, v7.4h\n" - "and v0.16b, v15.16b, v18.16b\n" - "smlal v10.4s, v29.4h, v8.4h\n" - "smlal2 v20.4s, v29.8h, v8.8h\n" - "ldr d29, [x20, x17]\n" - "usubl v29.8h, v29.8b, v12.8b\n" - "smlal2 v23.4s, v27.8h, v7.8h\n" - "smlal2 v22.4s, v26.8h, v7.8h\n" - "sqrdmulh v17.4s, v17.4s, v30.4s\n" - "add x17, x17, #0x8\n" - "smlal v9.4s, v24.4h, v5.4h\n" - "smlal v21.4s, v25.4h, v6.4h\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "smlal2 v23.4s, v24.8h, v5.8h\n" - "smlal2 v22.4s, v25.8h, v6.8h\n" - "and v7.16b, v17.16b, v31.16b\n" - "smlal v9.4s, v25.4h, v8.4h\n" - "smlal v21.4s, v29.4h, v8.4h\n" - "sqrdmulh v10.4s, v10.4s, v19.4s\n" - "smlal2 v23.4s, v25.8h, v8.8h\n" - "smlal2 v22.4s, v29.8h, v8.8h\n" - "sqrdmulh v9.4s, v9.4s, v19.4s\n" - "sqrdmulh v21.4s, v21.4s, v19.4s\n" - "sqadd v15.4s, v15.4s, v0.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "and v19.16b, v10.16b, v18.16b\n" - "sqrdmulh v20.4s, v20.4s, v30.4s\n" - "and v27.16b, v9.16b, v18.16b\n" - "sqrdmulh v23.4s, v23.4s, v30.4s\n" - "and v0.16b, v21.16b, v18.16b\n" - "sqrdmulh v22.4s, v22.4s, v30.4s\n" - "sqadd v17.4s, v17.4s, v7.4s\n" - "sshr v19.4s, v19.4s, #0x1f\n" - "and v5.16b, v20.16b, v31.16b\n" + "smlal2 v8.4s, v11.8h, v28.8h\n" + "smlal2 v0.4s, v16.8h, v7.8h\n" + "and v16.16b, v5.16b, v30.16b\n" + "smlal2 v31.4s, v18.8h, v7.8h\n" + "smlal v21.4s, v2.4h, v7.4h\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "smlal v20.4s, v10.4h, v9.4h\n" + "smlal v19.4s, v22.4h, v26.4h\n" + "sqadd v5.4s, v5.4s, v16.4s\n" + "smlal2 v3.4s, v12.8h, v7.8h\n" + "smlal2 v8.4s, v2.8h, v7.8h\n" + "sqrdmulh v3.4s, v3.4s, v14.4s\n" + "smlal2 v0.4s, v10.8h, v9.8h\n" + "smlal2 v31.4s, v22.8h, v26.8h\n" + "and v16.16b, v3.16b, v25.16b\n" + "smlal v21.4s, v23.4h, v4.4h\n" + "smlal v20.4s, v22.4h, v4.4h\n" + "sqrdmulh v21.4s, v21.4s, v29.4s\n" + "smlal v19.4s, v27.4h, v4.4h\n" + "smlal2 v8.4s, v23.8h, v4.8h\n" + "sqrdmulh v20.4s, v20.4s, v29.4s\n" + "smlal2 v0.4s, v22.8h, v4.8h\n" + "smlal2 v31.4s, v27.8h, v4.8h\n" + "sqrdmulh v19.4s, v19.4s, v29.4s\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "and v23.16b, v21.16b, v30.16b\n" + "sqrdmulh v8.4s, v8.4s, v14.4s\n" + "and v27.16b, v20.16b, v30.16b\n" + "sqrdmulh v0.4s, v0.4s, v14.4s\n" + "and v22.16b, v19.16b, v30.16b\n" + "sqrdmulh v31.4s, v31.4s, v14.4s\n" + "sqadd v3.4s, v3.4s, v16.4s\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "and v14.16b, v8.16b, v25.16b\n" "sshr v27.4s, v27.4s, #0x1f\n" - "and v4.16b, v23.16b, v31.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v7.16b, v22.16b, v31.16b\n" - "sqadd v10.4s, v10.4s, v19.4s\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v9.4s, v9.4s, v27.4s\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v21.4s, v21.4s, v0.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "srshl v15.4s, v15.4s, v18.4s\n" - "srshl v10.4s, v10.4s, v18.4s\n" - "sqadd v20.4s, v20.4s, v5.4s\n" - "srshl v9.4s, v9.4s, v18.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "srshl v21.4s, v21.4s, v18.4s\n" - "sqadd v22.4s, v22.4s, v7.4s\n" - "srshl v17.4s, v17.4s, v31.4s\n" - "sqxtn v15.4h, v15.4s\n" - "srshl v20.4s, v20.4s, v31.4s\n" - "sqxtn v10.4h, v10.4s\n" - "srshl v23.4s, v23.4s, v31.4s\n" - "sqxtn v9.4h, v9.4s\n" - "srshl v22.4s, v22.4s, v31.4s\n" + "and v18.16b, v0.16b, v25.16b\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "and v16.16b, v31.16b, v25.16b\n" + "sqadd v21.4s, v21.4s, v23.4s\n" + "sshr v14.4s, v14.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v27.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sqadd v19.4s, v19.4s, v22.4s\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "srshl v5.4s, v5.4s, v30.4s\n" + "srshl v21.4s, v21.4s, v30.4s\n" + "sqadd v8.4s, v8.4s, v14.4s\n" + "srshl v20.4s, v20.4s, v30.4s\n" + "sqadd v0.4s, v0.4s, v18.4s\n" + "srshl v19.4s, v19.4s, v30.4s\n" + "sqadd v31.4s, v31.4s, v16.4s\n" + "srshl v3.4s, v3.4s, v25.4s\n" + "sqxtn v5.4h, v5.4s\n" + "srshl v8.4s, v8.4s, v25.4s\n" "sqxtn v21.4h, v21.4s\n" - "sqxtn2 v15.8h, v17.4s\n" - "sqxtn2 v10.8h, v20.4s\n" - "sqxtn2 v9.8h, v23.4s\n" - "sqxtn2 v21.8h, v22.4s\n" - "sqadd v15.8h, v15.8h, v11.8h\n" - "sqadd v10.8h, v10.8h, v11.8h\n" - "sqadd v9.8h, v9.8h, v11.8h\n" - "sqadd v21.8h, v21.8h, v11.8h\n" - "smax v15.8h, v15.8h, v16.8h\n" - "smax v10.8h, v10.8h, v16.8h\n" - "smax v9.8h, v9.8h, v16.8h\n" - "smax v21.8h, v21.8h, v16.8h\n" - "smin v15.8h, v15.8h, v14.8h\n" - "smin v10.8h, v10.8h, v14.8h\n" - "smin v9.8h, v9.8h, v14.8h\n" - "smin v21.8h, v21.8h, v14.8h\n" - "uzp1 v15.16b, v15.16b, v15.16b\n" - "str d15, [x11, x16]\n" - "uzp1 v10.16b, v10.16b, v10.16b\n" - "uzp1 v9.16b, v9.16b, v9.16b\n" - "str d10, [x10, x16]\n" + "srshl v0.4s, v0.4s, v25.4s\n" + "sqxtn v20.4h, v20.4s\n" + "srshl v31.4s, v31.4s, v25.4s\n" + "sqxtn v19.4h, v19.4s\n" + "sqxtn2 v5.8h, v3.4s\n" + "sqxtn2 v21.8h, v8.4s\n" + "sqxtn2 v20.8h, v0.4s\n" + "sqxtn2 v19.8h, v31.4s\n" + "sqadd v5.8h, v5.8h, v13.8h\n" + "sqadd v21.8h, v21.8h, v13.8h\n" + "sqadd v20.8h, v20.8h, v13.8h\n" + "sqadd v19.8h, v19.8h, v13.8h\n" + "smax v5.8h, v5.8h, v17.8h\n" + "smax v21.8h, v21.8h, v17.8h\n" + "smax v20.8h, v20.8h, v17.8h\n" + "smax v19.8h, v19.8h, v17.8h\n" + "smin v5.8h, v5.8h, v24.8h\n" + "smin v21.8h, v21.8h, v24.8h\n" + "smin v20.8h, v20.8h, v24.8h\n" + "smin v19.8h, v19.8h, v24.8h\n" + "uzp1 v5.16b, v5.16b, v5.16b\n" + "str d5, [x11, x16]\n" "uzp1 v21.16b, v21.16b, v21.16b\n" - "str d9, [x9, x16]\n" - "str d21, [x28, x16]\n" + "uzp1 v20.16b, v20.16b, v20.16b\n" + "str d21, [x10, x16]\n" + "uzp1 v19.16b, v19.16b, v19.16b\n" + "str d20, [x9, x16]\n" + "str d19, [x28, x16]\n" "add x16, x16, #0x8\n" "beq 88f\n" "add x14, x14, #0x48\n" "3:" // Oddments - "ldr x24, [%x[params], %[offsetof_Params_bias]]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" "tbz x7, #2, 5f\n" - "ld1 { v15.4s }, [x24], #0x10\n" + "ld1 { v5.4s }, [x20], #0x10\n" "tbz x7, #1, 4f\n" - "ld1 { v17.d }[0], [x24], #0x8\n" + "ld1 { v3.d }[0], [x20], #0x8\n" "tbz x7, #0, 7f\n" - "ld1 { v17.s }[2], [x24]\n" + "ld1 { v3.s }[2], [x20]\n" "b 7f\n" "4:" // Oddments: Load bias: Bit 2: Bit 1: Unset "tbz x7, #0, 7f\n" - "ld1 { v17.s }[0], [x24]\n" + "ld1 { v3.s }[0], [x20]\n" "b 7f\n" "5:" // Oddments: Load bias: Bit 2: Unset "tbz x7, #1, 6f\n" - "ld1 { v15.d }[0], [x24], #0x8\n" + "ld1 { v5.d }[0], [x20], #0x8\n" "tbz x7, #0, 7f\n" - "ld1 { v15.s }[2], [x24]\n" + "ld1 { v5.s }[2], [x20]\n" "b 7f\n" "6:" // Oddments: Load bias: Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 7f\n" - "ld1 { v15.s }[0], [x24]\n" + "ld1 { v5.s }[0], [x20]\n" "7:" // Oddments: Load bias: Bit 2: End - "ldr d0, [x14, #0x0]\n" - "ldr d1, [x14, #0x8]\n" - "mov v10.16b, v15.16b\n" - "mov v20.16b, v17.16b\n" - "ldr d2, [x14, #0x10]\n" - "ldr d3, [x14, #0x18]\n" - "mov v9.16b, v15.16b\n" - "mov v23.16b, v17.16b\n" - "ldr d4, [x14, #0x20]\n" - "ldr d5, [x14, #0x28]\n" - "mov v21.16b, v15.16b\n" - "mov v22.16b, v17.16b\n" - "ldr d6, [x14, #0x30]\n" + "ldr d11, [x14, #0x0]\n" + "ldr d22, [x14, #0x8]\n" + "mov v21.16b, v5.16b\n" + "mov v8.16b, v3.16b\n" + "ldr d14, [x14, #0x10]\n" + "ldr d28, [x14, #0x18]\n" + "mov v20.16b, v5.16b\n" + "mov v0.16b, v3.16b\n" + "ldr d18, [x14, #0x20]\n" + "ldr d9, [x14, #0x28]\n" + "mov v19.16b, v5.16b\n" + "mov v31.16b, v3.16b\n" + "ldr d26, [x14, #0x30]\n" "ldr d7, [x14, #0x38]\n" - "usubl v0.8h, v0.8b, v13.8b\n" - "usubl v1.8h, v1.8b, v13.8b\n" - "ldr d8, [x14, #0x40]\n" + "usubl v11.8h, v11.8b, v15.8b\n" + "usubl v22.8h, v22.8b, v15.8b\n" + "ldr d4, [x14, #0x40]\n" "ldp x27, x26, [x15, #0x0]\n" - "usubl v2.8h, v2.8b, v13.8b\n" - "usubl v3.8h, v3.8b, v13.8b\n" + "usubl v14.8h, v14.8b, v15.8b\n" + "usubl v28.8h, v28.8b, v15.8b\n" "ldp x25, x24, [x15, #0x10]\n" "ldp x23, x22, [x15, #0x20]\n" - "usubl v4.8h, v4.8b, v13.8b\n" - "usubl v5.8h, v5.8b, v13.8b\n" + "usubl v18.8h, v18.8b, v15.8b\n" + "usubl v9.8h, v9.8b, v15.8b\n" "ldp x21, x20, [x15, #0x30]\n" - "usubl v6.8h, v6.8b, v13.8b\n" - "usubl v7.8h, v7.8b, v13.8b\n" - "usubl v8.8h, v8.8b, v13.8b\n" + "usubl v26.8h, v26.8b, v15.8b\n" + "usubl v7.8h, v7.8b, v15.8b\n" + "usubl v4.8h, v4.8b, v15.8b\n" "add x27, x27, x17\n" "add x26, x26, x17\n" "add x25, x25, x17\n" @@ -689,700 +689,700 @@ void a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "add x21, x21, x17\n" "add x20, x20, x17\n" "tbz x7, #2, 9f\n" - "ld1 { v31.s }[0], [x27], #0x4\n" - "ld1 { v30.s }[0], [x26], #0x4\n" - "ld1 { v29.s }[0], [x25], #0x4\n" - "ld1 { v28.s }[0], [x24], #0x4\n" - "ld1 { v27.s }[0], [x23], #0x4\n" - "ld1 { v26.s }[0], [x22], #0x4\n" - "ld1 { v25.s }[0], [x21], #0x4\n" - "ld1 { v24.s }[0], [x20], #0x4\n" + "ld1 { v25.s }[0], [x27], #0x4\n" + "ld1 { v27.s }[0], [x26], #0x4\n" + "ld1 { v1.s }[0], [x25], #0x4\n" + "ld1 { v2.s }[0], [x24], #0x4\n" + "ld1 { v12.s }[0], [x23], #0x4\n" + "ld1 { v16.s }[0], [x22], #0x4\n" + "ld1 { v23.s }[0], [x21], #0x4\n" + "ld1 { v10.s }[0], [x20], #0x4\n" "tbz x7, #1, 8f\n" - "ld1 { v31.h }[2], [x27], #0x2\n" - "ld1 { v30.h }[2], [x26], #0x2\n" - "ld1 { v29.h }[2], [x25], #0x2\n" - "ld1 { v28.h }[2], [x24], #0x2\n" - "ld1 { v27.h }[2], [x23], #0x2\n" - "ld1 { v26.h }[2], [x22], #0x2\n" - "ld1 { v25.h }[2], [x21], #0x2\n" - "ld1 { v24.h }[2], [x20], #0x2\n" + "ld1 { v25.h }[2], [x27], #0x2\n" + "ld1 { v27.h }[2], [x26], #0x2\n" + "ld1 { v1.h }[2], [x25], #0x2\n" + "ld1 { v2.h }[2], [x24], #0x2\n" + "ld1 { v12.h }[2], [x23], #0x2\n" + "ld1 { v16.h }[2], [x22], #0x2\n" + "ld1 { v23.h }[2], [x21], #0x2\n" + "ld1 { v10.h }[2], [x20], #0x2\n" "tbz x7, #0, 11f\n" - "ld1 { v31.b }[6], [x27]\n" - "ld1 { v30.b }[6], [x26]\n" - "ld1 { v29.b }[6], [x25]\n" - "ld1 { v28.b }[6], [x24]\n" - "ld1 { v27.b }[6], [x23]\n" - "ld1 { v26.b }[6], [x22]\n" - "ld1 { v25.b }[6], [x21]\n" - "ld1 { v24.b }[6], [x20]\n" + "ld1 { v25.b }[6], [x27]\n" + "ld1 { v27.b }[6], [x26]\n" + "ld1 { v1.b }[6], [x25]\n" + "ld1 { v2.b }[6], [x24]\n" + "ld1 { v12.b }[6], [x23]\n" + "ld1 { v16.b }[6], [x22]\n" + "ld1 { v23.b }[6], [x21]\n" + "ld1 { v10.b }[6], [x20]\n" "b 11f\n" "8:" // Oddments: Initial loads: Bit 2: Bit 1: Unset "tbz x7, #0, 11f\n" - "ld1 { v31.b }[4], [x27]\n" - "ld1 { v30.b }[4], [x26]\n" - "ld1 { v29.b }[4], [x25]\n" - "ld1 { v28.b }[4], [x24]\n" - "ld1 { v27.b }[4], [x23]\n" - "ld1 { v26.b }[4], [x22]\n" - "ld1 { v25.b }[4], [x21]\n" - "ld1 { v24.b }[4], [x20]\n" + "ld1 { v25.b }[4], [x27]\n" + "ld1 { v27.b }[4], [x26]\n" + "ld1 { v1.b }[4], [x25]\n" + "ld1 { v2.b }[4], [x24]\n" + "ld1 { v12.b }[4], [x23]\n" + "ld1 { v16.b }[4], [x22]\n" + "ld1 { v23.b }[4], [x21]\n" + "ld1 { v10.b }[4], [x20]\n" "b 11f\n" "9:" // Oddments: Initial loads: Bit 2: Unset "tbz x7, #1, 10f\n" - "ld1 { v31.h }[0], [x27], #0x2\n" - "ld1 { v30.h }[0], [x26], #0x2\n" - "ld1 { v29.h }[0], [x25], #0x2\n" - "ld1 { v28.h }[0], [x24], #0x2\n" - "ld1 { v27.h }[0], [x23], #0x2\n" - "ld1 { v26.h }[0], [x22], #0x2\n" - "ld1 { v25.h }[0], [x21], #0x2\n" - "ld1 { v24.h }[0], [x20], #0x2\n" + "ld1 { v25.h }[0], [x27], #0x2\n" + "ld1 { v27.h }[0], [x26], #0x2\n" + "ld1 { v1.h }[0], [x25], #0x2\n" + "ld1 { v2.h }[0], [x24], #0x2\n" + "ld1 { v12.h }[0], [x23], #0x2\n" + "ld1 { v16.h }[0], [x22], #0x2\n" + "ld1 { v23.h }[0], [x21], #0x2\n" + "ld1 { v10.h }[0], [x20], #0x2\n" "tbz x7, #0, 11f\n" - "ld1 { v31.b }[2], [x27]\n" - "ld1 { v30.b }[2], [x26]\n" - "ld1 { v29.b }[2], [x25]\n" - "ld1 { v28.b }[2], [x24]\n" - "ld1 { v27.b }[2], [x23]\n" - "ld1 { v26.b }[2], [x22]\n" - "ld1 { v25.b }[2], [x21]\n" - "ld1 { v24.b }[2], [x20]\n" + "ld1 { v25.b }[2], [x27]\n" + "ld1 { v27.b }[2], [x26]\n" + "ld1 { v1.b }[2], [x25]\n" + "ld1 { v2.b }[2], [x24]\n" + "ld1 { v12.b }[2], [x23]\n" + "ld1 { v16.b }[2], [x22]\n" + "ld1 { v23.b }[2], [x21]\n" + "ld1 { v10.b }[2], [x20]\n" "b 11f\n" "10:" // Oddments: Initial loads: Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 11f\n" - "ld1 { v31.b }[0], [x27]\n" - "ld1 { v30.b }[0], [x26]\n" - "ld1 { v29.b }[0], [x25]\n" - "ld1 { v28.b }[0], [x24]\n" - "ld1 { v27.b }[0], [x23]\n" - "ld1 { v26.b }[0], [x22]\n" - "ld1 { v25.b }[0], [x21]\n" - "ld1 { v24.b }[0], [x20]\n" + "ld1 { v25.b }[0], [x27]\n" + "ld1 { v27.b }[0], [x26]\n" + "ld1 { v1.b }[0], [x25]\n" + "ld1 { v2.b }[0], [x24]\n" + "ld1 { v12.b }[0], [x23]\n" + "ld1 { v16.b }[0], [x22]\n" + "ld1 { v23.b }[0], [x21]\n" + "ld1 { v10.b }[0], [x20]\n" "11:" // Oddments: Initial loads: Bit 2: End - "usubl v31.8h, v31.8b, v12.8b\n" - "smlal v15.4s, v31.4h, v8.4h\n" - "smlal2 v17.4s, v31.8h, v8.8h\n" - "ldr x24, [x15, #0x40]\n" - "usubl v30.8h, v30.8b, v12.8b\n" - "smlal v15.4s, v30.4h, v0.4h\n" - "smlal2 v17.4s, v30.8h, v0.8h\n" - "add x24, x24, x17\n" - "usubl v29.8h, v29.8b, v12.8b\n" - "smlal v10.4s, v31.4h, v6.4h\n" - "smlal2 v20.4s, v31.8h, v6.8h\n" - "smlal v15.4s, v29.4h, v1.4h\n" - "smlal2 v17.4s, v29.8h, v1.8h\n" - "usubl v28.8h, v28.8b, v12.8b\n" - "usubl v26.8h, v26.8b, v12.8b\n" - "smlal v10.4s, v28.4h, v1.4h\n" - "smlal2 v20.4s, v28.8h, v1.8h\n" - "smlal v15.4s, v26.4h, v3.4h\n" - "smlal2 v17.4s, v26.8h, v3.8h\n" - "usubl v27.8h, v27.8b, v12.8b\n" - "usubl v25.8h, v25.8b, v12.8b\n" - "smlal v10.4s, v27.4h, v2.4h\n" - "smlal2 v20.4s, v27.8h, v2.8h\n" - "smlal v15.4s, v25.4h, v4.4h\n" - "smlal2 v17.4s, v25.8h, v4.8h\n" - "usubl v24.8h, v24.8b, v12.8b\n" - "smlal v9.4s, v31.4h, v2.4h\n" - "smlal2 v23.4s, v31.8h, v2.8h\n" - "smlal v21.4s, v31.4h, v0.4h\n" - "smlal2 v22.4s, v31.8h, v0.8h\n" - "smlal v15.4s, v24.4h, v2.4h\n" - "smlal2 v17.4s, v24.8h, v2.8h\n" - "smlal v10.4s, v24.4h, v0.4h\n" - "smlal2 v20.4s, v24.8h, v0.8h\n" + "usubl v25.8h, v25.8b, v6.8b\n" + "smlal v5.4s, v25.4h, v4.4h\n" + "smlal2 v3.4s, v25.8h, v4.8h\n" + "ldr x20, [x15, #0x40]\n" + "usubl v27.8h, v27.8b, v6.8b\n" + "smlal v5.4s, v27.4h, v11.4h\n" + "smlal2 v3.4s, v27.8h, v11.8h\n" + "usubl v1.8h, v1.8b, v6.8b\n" + "smlal v21.4s, v25.4h, v26.4h\n" + "smlal2 v8.4s, v25.8h, v26.8h\n" + "add x20, x20, x17\n" + "smlal v5.4s, v1.4h, v22.4h\n" + "smlal2 v3.4s, v1.8h, v22.8h\n" + "usubl v2.8h, v2.8b, v6.8b\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "smlal v21.4s, v2.4h, v22.4h\n" + "smlal2 v8.4s, v2.8h, v22.8h\n" + "smlal v5.4s, v16.4h, v28.4h\n" + "smlal2 v3.4s, v16.8h, v28.8h\n" + "usubl v12.8h, v12.8b, v6.8b\n" + "usubl v23.8h, v23.8b, v6.8b\n" + "smlal v21.4s, v12.4h, v14.4h\n" + "smlal2 v8.4s, v12.8h, v14.8h\n" + "smlal v5.4s, v23.4h, v18.4h\n" + "smlal2 v3.4s, v23.8h, v18.8h\n" + "usubl v10.8h, v10.8b, v6.8b\n" + "smlal v20.4s, v25.4h, v14.4h\n" + "smlal2 v0.4s, v25.8h, v14.8h\n" + "smlal v19.4s, v25.4h, v11.4h\n" + "smlal2 v31.4s, v25.8h, v11.8h\n" + "smlal v5.4s, v10.4h, v14.4h\n" + "smlal2 v3.4s, v10.8h, v14.8h\n" + "smlal v21.4s, v10.4h, v11.4h\n" + "smlal2 v8.4s, v10.8h, v11.8h\n" "tbz x7, #2, 13f\n" - "ld1 { v29.s }[0], [x24], #0x4\n" + "ld1 { v15.s }[0], [x20], #0x4\n" "tbz x7, #1, 12f\n" - "ld1 { v29.h }[2], [x24], #0x2\n" + "ld1 { v15.h }[2], [x20], #0x2\n" "tbz x7, #0, 15f\n" - "ld1 { v29.b }[6], [x24]\n" + "ld1 { v15.b }[6], [x20]\n" "b 15f\n" "12:" // Oddments: Load (1, 3): Bit 2: Bit 1: Unset "tbz x7, #0, 15f\n" - "ld1 { v29.b }[4], [x24]\n" + "ld1 { v15.b }[4], [x20]\n" "b 15f\n" "13:" // Oddments: Load (1, 3): Bit 2: Unset "tbz x7, #1, 14f\n" - "ld1 { v29.h }[0], [x24], #0x2\n" + "ld1 { v15.h }[0], [x20], #0x2\n" "tbz x7, #0, 15f\n" - "ld1 { v29.b }[2], [x24]\n" + "ld1 { v15.b }[2], [x20]\n" "b 15f\n" "14:" // Oddments: Load (1, 3): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 15f\n" - "ld1 { v29.b }[0], [x24]\n" + "ld1 { v15.b }[0], [x20]\n" "15:" // Oddments: Load (1, 3): Bit 2: End - "usubl v29.8h, v29.8b, v12.8b\n" - "ldr x22, [x15, #0x48]\n" - "smlal v10.4s, v29.4h, v4.4h\n" - "smlal2 v20.4s, v29.8h, v4.8h\n" - "add x22, x22, x17\n" + "usubl v15.8h, v15.8b, v6.8b\n" + "ldr x20, [x15, #0x48]\n" + "smlal v21.4s, v15.4h, v18.4h\n" + "smlal2 v8.4s, v15.8h, v18.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 17f\n" - "ld1 { v28.s }[0], [x22], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 16f\n" - "ld1 { v28.h }[2], [x22], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 19f\n" - "ld1 { v28.b }[6], [x22]\n" + "ld1 { v16.b }[6], [x20]\n" "b 19f\n" "16:" // Oddments: Load (1, 4): Bit 2: Bit 1: Unset "tbz x7, #0, 19f\n" - "ld1 { v28.b }[4], [x22]\n" + "ld1 { v16.b }[4], [x20]\n" "b 19f\n" "17:" // Oddments: Load (1, 4): Bit 2: Unset "tbz x7, #1, 18f\n" - "ld1 { v28.h }[0], [x22], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 19f\n" - "ld1 { v28.b }[2], [x22]\n" + "ld1 { v16.b }[2], [x20]\n" "b 19f\n" "18:" // Oddments: Load (1, 4): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 19f\n" - "ld1 { v28.b }[0], [x22]\n" + "ld1 { v16.b }[0], [x20]\n" "19:" // Oddments: Load (1, 4): Bit 2: End - "usubl v28.8h, v28.8b, v12.8b\n" - "ldr x21, [x15, #0x50]\n" - "smlal v10.4s, v28.4h, v5.4h\n" - "smlal2 v20.4s, v28.8h, v5.8h\n" - "add x21, x21, x17\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0x50]\n" + "smlal v21.4s, v16.4h, v9.4h\n" + "smlal2 v8.4s, v16.8h, v9.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 21f\n" - "ld1 { v27.s }[0], [x21], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 20f\n" - "ld1 { v27.h }[2], [x21], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 23f\n" - "ld1 { v27.b }[6], [x21]\n" + "ld1 { v16.b }[6], [x20]\n" "b 23f\n" "20:" // Oddments: Load (1, 2): Bit 2: Bit 1: Unset "tbz x7, #0, 23f\n" - "ld1 { v27.b }[4], [x21]\n" + "ld1 { v16.b }[4], [x20]\n" "b 23f\n" "21:" // Oddments: Load (1, 2): Bit 2: Unset "tbz x7, #1, 22f\n" - "ld1 { v27.h }[0], [x21], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 23f\n" - "ld1 { v27.b }[2], [x21]\n" + "ld1 { v16.b }[2], [x20]\n" "b 23f\n" "22:" // Oddments: Load (1, 2): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 23f\n" - "ld1 { v27.b }[0], [x21]\n" + "ld1 { v16.b }[0], [x20]\n" "23:" // Oddments: Load (1, 2): Bit 2: End - "usubl v27.8h, v27.8b, v12.8b\n" + "usubl v16.8h, v16.8b, v6.8b\n" "ldr x20, [x15, #0x58]\n" - "smlal v15.4s, v27.4h, v5.4h\n" - "smlal2 v17.4s, v27.8h, v5.8h\n" - "smlal v10.4s, v27.4h, v3.4h\n" - "smlal2 v20.4s, v27.8h, v3.8h\n" + "smlal v5.4s, v16.4h, v9.4h\n" + "smlal2 v3.4s, v16.8h, v9.8h\n" + "smlal v21.4s, v16.4h, v28.4h\n" + "smlal2 v8.4s, v16.8h, v28.8h\n" "add x20, x20, x17\n" "tbz x7, #2, 25f\n" - "ld1 { v26.s }[0], [x20], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 24f\n" - "ld1 { v26.h }[2], [x20], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 27f\n" - "ld1 { v26.b }[6], [x20]\n" + "ld1 { v16.b }[6], [x20]\n" "b 27f\n" "24:" // Oddments: Load (3, 0): Bit 2: Bit 1: Unset "tbz x7, #0, 27f\n" - "ld1 { v26.b }[4], [x20]\n" + "ld1 { v16.b }[4], [x20]\n" "b 27f\n" "25:" // Oddments: Load (3, 0): Bit 2: Unset "tbz x7, #1, 26f\n" - "ld1 { v26.h }[0], [x20], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 27f\n" - "ld1 { v26.b }[2], [x20]\n" + "ld1 { v16.b }[2], [x20]\n" "b 27f\n" "26:" // Oddments: Load (3, 0): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 27f\n" - "ld1 { v26.b }[0], [x20]\n" + "ld1 { v16.b }[0], [x20]\n" "27:" // Oddments: Load (3, 0): Bit 2: End - "usubl v26.8h, v26.8b, v12.8b\n" + "usubl v16.8h, v16.8b, v6.8b\n" "ldr x20, [x15, #0x60]\n" - "smlal v9.4s, v26.4h, v3.4h\n" - "smlal2 v23.4s, v26.8h, v3.8h\n" + "smlal v20.4s, v16.4h, v28.4h\n" + "smlal2 v0.4s, v16.8h, v28.8h\n" "add x20, x20, x17\n" "tbz x7, #2, 29f\n" - "ld1 { v25.s }[0], [x20], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 28f\n" - "ld1 { v25.h }[2], [x20], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 31f\n" - "ld1 { v25.b }[6], [x20]\n" + "ld1 { v16.b }[6], [x20]\n" "b 31f\n" "28:" // Oddments: Load (2, 0): Bit 2: Bit 1: Unset "tbz x7, #0, 31f\n" - "ld1 { v25.b }[4], [x20]\n" + "ld1 { v16.b }[4], [x20]\n" "b 31f\n" "29:" // Oddments: Load (2, 0): Bit 2: Unset "tbz x7, #1, 30f\n" - "ld1 { v25.h }[0], [x20], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 31f\n" - "ld1 { v25.b }[2], [x20]\n" + "ld1 { v16.b }[2], [x20]\n" "b 31f\n" "30:" // Oddments: Load (2, 0): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 31f\n" - "ld1 { v25.b }[0], [x20]\n" + "ld1 { v16.b }[0], [x20]\n" "31:" // Oddments: Load (2, 0): Bit 2: End - "usubl v25.8h, v25.8b, v12.8b\n" + "usubl v16.8h, v16.8b, v6.8b\n" "ldr x20, [x15, #0x68]\n" - "smlal v15.4s, v25.4h, v6.4h\n" - "smlal2 v17.4s, v25.8h, v6.8h\n" - "smlal v9.4s, v25.4h, v0.4h\n" - "smlal2 v23.4s, v25.8h, v0.8h\n" + "smlal v5.4s, v16.4h, v26.4h\n" + "smlal2 v3.4s, v16.8h, v26.8h\n" + "smlal v20.4s, v16.4h, v11.4h\n" + "smlal2 v0.4s, v16.8h, v11.8h\n" "add x20, x20, x17\n" "tbz x7, #2, 33f\n" - "ld1 { v29.s }[0], [x20], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 32f\n" - "ld1 { v29.h }[2], [x20], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 35f\n" - "ld1 { v29.b }[6], [x20]\n" + "ld1 { v16.b }[6], [x20]\n" "b 35f\n" "32:" // Oddments: Load (3, 1): Bit 2: Bit 1: Unset "tbz x7, #0, 35f\n" - "ld1 { v29.b }[4], [x20]\n" + "ld1 { v16.b }[4], [x20]\n" "b 35f\n" "33:" // Oddments: Load (3, 1): Bit 2: Unset "tbz x7, #1, 34f\n" - "ld1 { v29.h }[0], [x20], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 35f\n" - "ld1 { v29.b }[2], [x20]\n" + "ld1 { v16.b }[2], [x20]\n" "b 35f\n" "34:" // Oddments: Load (3, 1): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 35f\n" - "ld1 { v29.b }[0], [x20]\n" + "ld1 { v16.b }[0], [x20]\n" "35:" // Oddments: Load (3, 1): Bit 2: End - "usubl v29.8h, v29.8b, v12.8b\n" - "ldr x21, [x15, #0x70]\n" - "smlal v9.4s, v29.4h, v4.4h\n" - "smlal2 v23.4s, v29.8h, v4.8h\n" - "add x21, x21, x17\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0x70]\n" + "smlal v20.4s, v16.4h, v18.4h\n" + "smlal2 v0.4s, v16.8h, v18.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 37f\n" - "ld1 { v24.s }[0], [x21], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 36f\n" - "ld1 { v24.h }[2], [x21], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 39f\n" - "ld1 { v24.b }[6], [x21]\n" + "ld1 { v16.b }[6], [x20]\n" "b 39f\n" "36:" // Oddments: Load (2, 1): Bit 2: Bit 1: Unset "tbz x7, #0, 39f\n" - "ld1 { v24.b }[4], [x21]\n" + "ld1 { v16.b }[4], [x20]\n" "b 39f\n" "37:" // Oddments: Load (2, 1): Bit 2: Unset "tbz x7, #1, 38f\n" - "ld1 { v24.h }[0], [x21], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 39f\n" - "ld1 { v24.b }[2], [x21]\n" + "ld1 { v16.b }[2], [x20]\n" "b 39f\n" "38:" // Oddments: Load (2, 1): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 39f\n" - "ld1 { v24.b }[0], [x21]\n" + "ld1 { v16.b }[0], [x20]\n" "39:" // Oddments: Load (2, 1): Bit 2: End - "usubl v24.8h, v24.8b, v12.8b\n" - "ldr x23, [x15, #0x78]\n" - "smlal v15.4s, v24.4h, v7.4h\n" - "smlal2 v17.4s, v24.8h, v7.8h\n" - "smlal v9.4s, v24.4h, v1.4h\n" - "smlal2 v23.4s, v24.8h, v1.8h\n" - "add x23, x23, x17\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0x78]\n" + "smlal v5.4s, v16.4h, v7.4h\n" + "smlal2 v3.4s, v16.8h, v7.8h\n" + "smlal v20.4s, v16.4h, v22.4h\n" + "smlal2 v0.4s, v16.8h, v22.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 41f\n" - "ld1 { v27.s }[0], [x23], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 40f\n" - "ld1 { v27.h }[2], [x23], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 43f\n" - "ld1 { v27.b }[6], [x23]\n" + "ld1 { v16.b }[6], [x20]\n" "b 43f\n" "40:" // Oddments: Load (3, 3): Bit 2: Bit 1: Unset "tbz x7, #0, 43f\n" - "ld1 { v27.b }[4], [x23]\n" + "ld1 { v16.b }[4], [x20]\n" "b 43f\n" "41:" // Oddments: Load (3, 3): Bit 2: Unset "tbz x7, #1, 42f\n" - "ld1 { v27.h }[0], [x23], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 43f\n" - "ld1 { v27.b }[2], [x23]\n" + "ld1 { v16.b }[2], [x20]\n" "b 43f\n" "42:" // Oddments: Load (3, 3): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 43f\n" - "ld1 { v27.b }[0], [x23]\n" + "ld1 { v16.b }[0], [x20]\n" "43:" // Oddments: Load (3, 3): Bit 2: End - "usubl v27.8h, v27.8b, v12.8b\n" - "ldr x21, [x15, #0x80]\n" - "smlal v21.4s, v27.4h, v4.4h\n" - "smlal2 v22.4s, v27.8h, v4.8h\n" - "add x21, x21, x17\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0x80]\n" + "smlal v19.4s, v16.4h, v18.4h\n" + "smlal2 v31.4s, v16.8h, v18.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 45f\n" - "ld1 { v28.s }[0], [x21], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 44f\n" - "ld1 { v28.h }[2], [x21], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 47f\n" - "ld1 { v28.b }[6], [x21]\n" + "ld1 { v16.b }[6], [x20]\n" "b 47f\n" "44:" // Oddments: Load (2, 3): Bit 2: Bit 1: Unset "tbz x7, #0, 47f\n" - "ld1 { v28.b }[4], [x21]\n" + "ld1 { v16.b }[4], [x20]\n" "b 47f\n" "45:" // Oddments: Load (2, 3): Bit 2: Unset "tbz x7, #1, 46f\n" - "ld1 { v28.h }[0], [x21], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 47f\n" - "ld1 { v28.b }[2], [x21]\n" + "ld1 { v16.b }[2], [x20]\n" "b 47f\n" "46:" // Oddments: Load (2, 3): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 47f\n" - "ld1 { v28.b }[0], [x21]\n" + "ld1 { v16.b }[0], [x20]\n" "47:" // Oddments: Load (2, 3): Bit 2: End - "usubl v28.8h, v28.8b, v12.8b\n" - "ldr x22, [x15, #0x88]\n" - "smlal v10.4s, v28.4h, v7.4h\n" - "smlal2 v20.4s, v28.8h, v7.8h\n" - "smlal v21.4s, v28.4h, v1.4h\n" - "smlal2 v22.4s, v28.8h, v1.8h\n" - "add x22, x22, x17\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0x88]\n" + "smlal v21.4s, v16.4h, v7.4h\n" + "smlal2 v8.4s, v16.8h, v7.8h\n" + "smlal v19.4s, v16.4h, v22.4h\n" + "smlal2 v31.4s, v16.8h, v22.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 49f\n" - "ld1 { v26.s }[0], [x22], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 48f\n" - "ld1 { v26.h }[2], [x22], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 51f\n" - "ld1 { v26.b }[6], [x22]\n" + "ld1 { v16.b }[6], [x20]\n" "b 51f\n" "48:" // Oddments: Load (3, 4): Bit 2: Bit 1: Unset "tbz x7, #0, 51f\n" - "ld1 { v26.b }[4], [x22]\n" + "ld1 { v16.b }[4], [x20]\n" "b 51f\n" "49:" // Oddments: Load (3, 4): Bit 2: Unset "tbz x7, #1, 50f\n" - "ld1 { v26.h }[0], [x22], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 51f\n" - "ld1 { v26.b }[2], [x22]\n" + "ld1 { v16.b }[2], [x20]\n" "b 51f\n" "50:" // Oddments: Load (3, 4): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 51f\n" - "ld1 { v26.b }[0], [x22]\n" + "ld1 { v16.b }[0], [x20]\n" "51:" // Oddments: Load (3, 4): Bit 2: End - "usubl v26.8h, v26.8b, v12.8b\n" - "ldr x24, [x15, #0x90]\n" - "smlal v21.4s, v26.4h, v5.4h\n" - "smlal2 v22.4s, v26.8h, v5.8h\n" - "add x24, x24, x17\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0x90]\n" + "smlal v19.4s, v16.4h, v9.4h\n" + "smlal2 v31.4s, v16.8h, v9.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 53f\n" - "ld1 { v25.s }[0], [x24], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 52f\n" - "ld1 { v25.h }[2], [x24], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 55f\n" - "ld1 { v25.b }[6], [x24]\n" + "ld1 { v16.b }[6], [x20]\n" "b 55f\n" "52:" // Oddments: Load (4, 0): Bit 2: Bit 1: Unset "tbz x7, #0, 55f\n" - "ld1 { v25.b }[4], [x24]\n" + "ld1 { v16.b }[4], [x20]\n" "b 55f\n" "53:" // Oddments: Load (4, 0): Bit 2: Unset "tbz x7, #1, 54f\n" - "ld1 { v25.h }[0], [x24], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 55f\n" - "ld1 { v25.b }[2], [x24]\n" + "ld1 { v16.b }[2], [x20]\n" "b 55f\n" "54:" // Oddments: Load (4, 0): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 55f\n" - "ld1 { v25.b }[0], [x24]\n" + "ld1 { v16.b }[0], [x20]\n" "55:" // Oddments: Load (4, 0): Bit 2: End - "usubl v25.8h, v25.8b, v12.8b\n" - "ldr x25, [x15, #0x98]\n" - "smlal v9.4s, v25.4h, v6.4h\n" - "smlal2 v23.4s, v25.8h, v6.8h\n" - "add x25, x25, x17\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0x98]\n" + "smlal v20.4s, v16.4h, v26.4h\n" + "smlal2 v0.4s, v16.8h, v26.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 57f\n" - "ld1 { v29.s }[0], [x25], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 56f\n" - "ld1 { v29.h }[2], [x25], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 59f\n" - "ld1 { v29.b }[6], [x25]\n" + "ld1 { v16.b }[6], [x20]\n" "b 59f\n" "56:" // Oddments: Load (2, 4): Bit 2: Bit 1: Unset "tbz x7, #0, 59f\n" - "ld1 { v29.b }[4], [x25]\n" + "ld1 { v16.b }[4], [x20]\n" "b 59f\n" "57:" // Oddments: Load (2, 4): Bit 2: Unset "tbz x7, #1, 58f\n" - "ld1 { v29.h }[0], [x25], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 59f\n" - "ld1 { v29.b }[2], [x25]\n" + "ld1 { v16.b }[2], [x20]\n" "b 59f\n" "58:" // Oddments: Load (2, 4): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 59f\n" - "ld1 { v29.b }[0], [x25]\n" + "ld1 { v16.b }[0], [x20]\n" "59:" // Oddments: Load (2, 4): Bit 2: End - "usubl v29.8h, v29.8b, v12.8b\n" + "usubl v16.8h, v16.8b, v6.8b\n" "ldr x20, [x15, #0xa0]\n" - "smlal v10.4s, v29.4h, v8.4h\n" - "smlal2 v20.4s, v29.8h, v8.8h\n" - "smlal v21.4s, v29.4h, v2.4h\n" - "smlal2 v22.4s, v29.8h, v2.8h\n" + "smlal v21.4s, v16.4h, v4.4h\n" + "smlal2 v8.4s, v16.8h, v4.8h\n" + "smlal v19.4s, v16.4h, v14.4h\n" + "smlal2 v31.4s, v16.8h, v14.8h\n" "add x20, x20, x17\n" "tbz x7, #2, 61f\n" - "ld1 { v27.s }[0], [x20], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 60f\n" - "ld1 { v27.h }[2], [x20], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 63f\n" - "ld1 { v27.b }[6], [x20]\n" + "ld1 { v16.b }[6], [x20]\n" "b 63f\n" "60:" // Oddments: Load (4, 1): Bit 2: Bit 1: Unset "tbz x7, #0, 63f\n" - "ld1 { v27.b }[4], [x20]\n" + "ld1 { v16.b }[4], [x20]\n" "b 63f\n" "61:" // Oddments: Load (4, 1): Bit 2: Unset "tbz x7, #1, 62f\n" - "ld1 { v27.h }[0], [x20], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 63f\n" - "ld1 { v27.b }[2], [x20]\n" + "ld1 { v16.b }[2], [x20]\n" "b 63f\n" "62:" // Oddments: Load (4, 1): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 63f\n" - "ld1 { v27.b }[0], [x20]\n" + "ld1 { v16.b }[0], [x20]\n" "63:" // Oddments: Load (4, 1): Bit 2: End - "usubl v27.8h, v27.8b, v12.8b\n" - "ldr x23, [x15, #0xa8]\n" - "smlal v9.4s, v27.4h, v7.4h\n" - "smlal2 v23.4s, v27.8h, v7.8h\n" - "add x23, x23, x17\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0xa8]\n" + "smlal v20.4s, v16.4h, v7.4h\n" + "smlal2 v0.4s, v16.8h, v7.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 65f\n" - "ld1 { v24.s }[0], [x23], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 64f\n" - "ld1 { v24.h }[2], [x23], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 67f\n" - "ld1 { v24.b }[6], [x23]\n" + "ld1 { v16.b }[6], [x20]\n" "b 67f\n" "64:" // Oddments: Load (3, 2): Bit 2: Bit 1: Unset "tbz x7, #0, 67f\n" - "ld1 { v24.b }[4], [x23]\n" + "ld1 { v16.b }[4], [x20]\n" "b 67f\n" "65:" // Oddments: Load (3, 2): Bit 2: Unset "tbz x7, #1, 66f\n" - "ld1 { v24.h }[0], [x23], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 67f\n" - "ld1 { v24.b }[2], [x23]\n" + "ld1 { v16.b }[2], [x20]\n" "b 67f\n" "66:" // Oddments: Load (3, 2): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 67f\n" - "ld1 { v24.b }[0], [x23]\n" + "ld1 { v16.b }[0], [x20]\n" "67:" // Oddments: Load (3, 2): Bit 2: End - "usubl v24.8h, v24.8b, v12.8b\n" - "ldr x22, [x15, #0xb0]\n" - "smlal v9.4s, v24.4h, v5.4h\n" - "smlal2 v23.4s, v24.8h, v5.8h\n" - "smlal v21.4s, v24.4h, v3.4h\n" - "smlal2 v22.4s, v24.8h, v3.8h\n" - "add x22, x22, x17\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0xb0]\n" + "smlal v20.4s, v16.4h, v9.4h\n" + "smlal2 v0.4s, v16.8h, v9.8h\n" + "smlal v19.4s, v16.4h, v28.4h\n" + "smlal2 v31.4s, v16.8h, v28.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 69f\n" - "ld1 { v26.s }[0], [x22], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 68f\n" - "ld1 { v26.h }[2], [x22], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 71f\n" - "ld1 { v26.b }[6], [x22]\n" + "ld1 { v16.b }[6], [x20]\n" "b 71f\n" "68:" // Oddments: Load (4, 3): Bit 2: Bit 1: Unset "tbz x7, #0, 71f\n" - "ld1 { v26.b }[4], [x22]\n" + "ld1 { v16.b }[4], [x20]\n" "b 71f\n" "69:" // Oddments: Load (4, 3): Bit 2: Unset "tbz x7, #1, 70f\n" - "ld1 { v26.h }[0], [x22], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 71f\n" - "ld1 { v26.b }[2], [x22]\n" + "ld1 { v16.b }[2], [x20]\n" "b 71f\n" "70:" // Oddments: Load (4, 3): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 71f\n" - "ld1 { v26.b }[0], [x22]\n" + "ld1 { v16.b }[0], [x20]\n" "71:" // Oddments: Load (4, 3): Bit 2: End - "usubl v26.8h, v26.8b, v12.8b\n" - "ldr x21, [x15, #0xb8]\n" - "smlal v21.4s, v26.4h, v7.4h\n" - "smlal2 v22.4s, v26.8h, v7.8h\n" - "add x21, x21, x17\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0xb8]\n" + "smlal v19.4s, v16.4h, v7.4h\n" + "smlal2 v31.4s, v16.8h, v7.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 73f\n" - "ld1 { v25.s }[0], [x21], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 72f\n" - "ld1 { v25.h }[2], [x21], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 75f\n" - "ld1 { v25.b }[6], [x21]\n" + "ld1 { v16.b }[6], [x20]\n" "b 75f\n" "72:" // Oddments: Load (4, 2): Bit 2: Bit 1: Unset "tbz x7, #0, 75f\n" - "ld1 { v25.b }[4], [x21]\n" + "ld1 { v16.b }[4], [x20]\n" "b 75f\n" "73:" // Oddments: Load (4, 2): Bit 2: Unset "tbz x7, #1, 74f\n" - "ld1 { v25.h }[0], [x21], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 75f\n" - "ld1 { v25.b }[2], [x21]\n" + "ld1 { v16.b }[2], [x20]\n" "b 75f\n" "74:" // Oddments: Load (4, 2): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 75f\n" - "ld1 { v25.b }[0], [x21]\n" + "ld1 { v16.b }[0], [x20]\n" "75:" // Oddments: Load (4, 2): Bit 2: End - "usubl v25.8h, v25.8b, v12.8b\n" + "usubl v16.8h, v16.8b, v6.8b\n" "ldr x20, [x15, #0xc0]\n" - "smlal v9.4s, v25.4h, v8.4h\n" - "smlal2 v23.4s, v25.8h, v8.8h\n" - "smlal v21.4s, v25.4h, v6.4h\n" - "smlal2 v22.4s, v25.8h, v6.8h\n" + "smlal v20.4s, v16.4h, v4.4h\n" + "smlal2 v0.4s, v16.8h, v4.8h\n" + "smlal v19.4s, v16.4h, v26.4h\n" + "smlal2 v31.4s, v16.8h, v26.8h\n" "add x20, x20, x17\n" "tbz x7, #2, 77f\n" - "ld1 { v29.s }[0], [x20], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 76f\n" - "ld1 { v29.h }[2], [x20], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 79f\n" - "ld1 { v29.b }[6], [x20]\n" + "ld1 { v16.b }[6], [x20]\n" "b 79f\n" "76:" // Oddments: Load (4, 4): Bit 2: Bit 1: Unset "tbz x7, #0, 79f\n" - "ld1 { v29.b }[4], [x20]\n" + "ld1 { v16.b }[4], [x20]\n" "b 79f\n" "77:" // Oddments: Load (4, 4): Bit 2: Unset "tbz x7, #1, 78f\n" - "ld1 { v29.h }[0], [x20], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 79f\n" - "ld1 { v29.b }[2], [x20]\n" + "ld1 { v16.b }[2], [x20]\n" "b 79f\n" "78:" // Oddments: Load (4, 4): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 79f\n" - "ld1 { v29.b }[0], [x20]\n" + "ld1 { v16.b }[0], [x20]\n" "79:" // Oddments: Load (4, 4): Bit 2: End - "usubl v29.8h, v29.8b, v12.8b\n" - "smlal v21.4s, v29.4h, v8.4h\n" - "smlal2 v22.4s, v29.8h, v8.8h\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "smlal v19.4s, v16.4h, v4.4h\n" + "smlal2 v31.4s, v16.8h, v4.8h\n" "tbz x7, #2, 81f\n" - "ld1 { v19.4s }, [x13], #0x10\n" - "ld1 { v18.4s }, [x12], #0x10\n" + "ld1 { v14.4s }, [x13], #0x10\n" + "ld1 { v25.4s }, [x12], #0x10\n" "tbz x7, #1, 80f\n" - "ld1 { v30.d }[0], [x13], #0x8\n" - "ld1 { v31.d }[0], [x12], #0x8\n" + "ld1 { v18.d }[0], [x13], #0x8\n" + "ld1 { v12.d }[0], [x12], #0x8\n" "tbz x7, #0, 83f\n" - "ld1 { v30.s }[2], [x13]\n" - "ld1 { v31.s }[2], [x12]\n" + "ld1 { v18.s }[2], [x13]\n" + "ld1 { v12.s }[2], [x12]\n" "b 83f\n" "80:" // Oddments: Load requant params: Bit 2: Bit 1: Unset "tbz x7, #0, 83f\n" - "ld1 { v30.s }[0], [x13]\n" - "ld1 { v31.s }[0], [x12]\n" + "ld1 { v18.s }[0], [x13]\n" + "ld1 { v12.s }[0], [x12]\n" "b 83f\n" "81:" // Oddments: Load requant params: Bit 2: Unset "tbz x7, #1, 82f\n" - "ld1 { v19.d }[0], [x13], #0x8\n" - "ld1 { v18.d }[0], [x12], #0x8\n" + "ld1 { v14.d }[0], [x13], #0x8\n" + "ld1 { v25.d }[0], [x12], #0x8\n" "tbz x7, #0, 83f\n" - "ld1 { v19.s }[2], [x13]\n" - "ld1 { v18.s }[2], [x12]\n" + "ld1 { v14.s }[2], [x13]\n" + "ld1 { v25.s }[2], [x12]\n" "b 83f\n" "82:" // Oddments: Load requant params: Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 83f\n" - "ld1 { v19.s }[0], [x13]\n" - "ld1 { v18.s }[0], [x12]\n" + "ld1 { v14.s }[0], [x13]\n" + "ld1 { v25.s }[0], [x12]\n" "83:" // Oddments: Load requant params: Bit 2: End - "sqrdmulh v15.4s, v15.4s, v19.4s\n" - "and v0.16b, v15.16b, v18.16b\n" + "sqrdmulh v5.4s, v5.4s, v14.4s\n" + "and v28.16b, v5.16b, v25.16b\n" "add x11, x11, x16\n" "add x10, x10, x16\n" - "sqrdmulh v17.4s, v17.4s, v30.4s\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqrdmulh v3.4s, v3.4s, v18.4s\n" + "sshr v28.4s, v28.4s, #0x1f\n" "add x9, x9, x16\n" "add x28, x28, x16\n" - "and v7.16b, v17.16b, v31.16b\n" - "sqrdmulh v10.4s, v10.4s, v19.4s\n" - "sqrdmulh v9.4s, v9.4s, v19.4s\n" - "sqrdmulh v21.4s, v21.4s, v19.4s\n" - "sqadd v15.4s, v15.4s, v0.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "and v19.16b, v10.16b, v18.16b\n" - "sqrdmulh v20.4s, v20.4s, v30.4s\n" - "and v27.16b, v9.16b, v18.16b\n" - "sqrdmulh v23.4s, v23.4s, v30.4s\n" - "and v0.16b, v21.16b, v18.16b\n" - "sqrdmulh v22.4s, v22.4s, v30.4s\n" - "sqadd v17.4s, v17.4s, v7.4s\n" - "sshr v19.4s, v19.4s, #0x1f\n" - "and v5.16b, v20.16b, v31.16b\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "and v4.16b, v23.16b, v31.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v7.16b, v22.16b, v31.16b\n" - "sqadd v10.4s, v10.4s, v19.4s\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v9.4s, v9.4s, v27.4s\n" + "and v16.16b, v3.16b, v12.16b\n" + "sqrdmulh v21.4s, v21.4s, v14.4s\n" + "sqrdmulh v20.4s, v20.4s, v14.4s\n" + "sqrdmulh v19.4s, v19.4s, v14.4s\n" + "sqadd v5.4s, v5.4s, v28.4s\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "and v14.16b, v21.16b, v25.16b\n" + "sqrdmulh v8.4s, v8.4s, v18.4s\n" + "and v6.16b, v20.16b, v25.16b\n" + "sqrdmulh v0.4s, v0.4s, v18.4s\n" + "and v4.16b, v19.16b, v25.16b\n" + "sqrdmulh v31.4s, v31.4s, v18.4s\n" + "sqadd v3.4s, v3.4s, v16.4s\n" + "sshr v14.4s, v14.4s, #0x1f\n" + "and v18.16b, v8.16b, v12.16b\n" + "sshr v6.4s, v6.4s, #0x1f\n" + "and v7.16b, v0.16b, v12.16b\n" "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v21.4s, v21.4s, v0.4s\n" + "and v16.16b, v31.16b, v12.16b\n" + "sqadd v21.4s, v21.4s, v14.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v6.4s\n" "sshr v7.4s, v7.4s, #0x1f\n" - "srshl v15.4s, v15.4s, v18.4s\n" - "srshl v10.4s, v10.4s, v18.4s\n" - "sqadd v20.4s, v20.4s, v5.4s\n" - "srshl v9.4s, v9.4s, v18.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "srshl v21.4s, v21.4s, v18.4s\n" - "sqadd v22.4s, v22.4s, v7.4s\n" - "srshl v17.4s, v17.4s, v31.4s\n" - "sqxtn v15.4h, v15.4s\n" - "srshl v20.4s, v20.4s, v31.4s\n" - "sqxtn v10.4h, v10.4s\n" - "srshl v23.4s, v23.4s, v31.4s\n" - "sqxtn v9.4h, v9.4s\n" - "srshl v22.4s, v22.4s, v31.4s\n" + "sqadd v19.4s, v19.4s, v4.4s\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "srshl v5.4s, v5.4s, v25.4s\n" + "srshl v21.4s, v21.4s, v25.4s\n" + "sqadd v8.4s, v8.4s, v18.4s\n" + "srshl v20.4s, v20.4s, v25.4s\n" + "sqadd v0.4s, v0.4s, v7.4s\n" + "srshl v19.4s, v19.4s, v25.4s\n" + "sqadd v31.4s, v31.4s, v16.4s\n" + "srshl v3.4s, v3.4s, v12.4s\n" + "sqxtn v5.4h, v5.4s\n" + "srshl v8.4s, v8.4s, v12.4s\n" "sqxtn v21.4h, v21.4s\n" - "sqxtn2 v15.8h, v17.4s\n" - "sqxtn2 v10.8h, v20.4s\n" - "sqxtn2 v9.8h, v23.4s\n" - "sqxtn2 v21.8h, v22.4s\n" - "sqadd v15.8h, v15.8h, v11.8h\n" - "sqadd v10.8h, v10.8h, v11.8h\n" - "sqadd v9.8h, v9.8h, v11.8h\n" - "sqadd v21.8h, v21.8h, v11.8h\n" - "smax v15.8h, v15.8h, v16.8h\n" - "smax v10.8h, v10.8h, v16.8h\n" - "smax v9.8h, v9.8h, v16.8h\n" - "smax v21.8h, v21.8h, v16.8h\n" - "smin v15.8h, v15.8h, v14.8h\n" - "smin v10.8h, v10.8h, v14.8h\n" - "smin v9.8h, v9.8h, v14.8h\n" - "smin v21.8h, v21.8h, v14.8h\n" - "uzp1 v15.16b, v15.16b, v15.16b\n" - "uzp1 v10.16b, v10.16b, v10.16b\n" - "uzp1 v9.16b, v9.16b, v9.16b\n" + "srshl v0.4s, v0.4s, v12.4s\n" + "sqxtn v20.4h, v20.4s\n" + "srshl v31.4s, v31.4s, v12.4s\n" + "sqxtn v19.4h, v19.4s\n" + "sqxtn2 v5.8h, v3.4s\n" + "sqxtn2 v21.8h, v8.4s\n" + "sqxtn2 v20.8h, v0.4s\n" + "sqxtn2 v19.8h, v31.4s\n" + "sqadd v5.8h, v5.8h, v13.8h\n" + "sqadd v21.8h, v21.8h, v13.8h\n" + "sqadd v20.8h, v20.8h, v13.8h\n" + "sqadd v19.8h, v19.8h, v13.8h\n" + "smax v5.8h, v5.8h, v17.8h\n" + "smax v21.8h, v21.8h, v17.8h\n" + "smax v20.8h, v20.8h, v17.8h\n" + "smax v19.8h, v19.8h, v17.8h\n" + "smin v5.8h, v5.8h, v24.8h\n" + "smin v21.8h, v21.8h, v24.8h\n" + "smin v20.8h, v20.8h, v24.8h\n" + "smin v19.8h, v19.8h, v24.8h\n" + "uzp1 v5.16b, v5.16b, v5.16b\n" "uzp1 v21.16b, v21.16b, v21.16b\n" + "uzp1 v20.16b, v20.16b, v20.16b\n" + "uzp1 v19.16b, v19.16b, v19.16b\n" "tbz x7, #2, 85f\n" - "st1 { v15.s }[0], [x11], #0x4\n" - "st1 { v10.s }[0], [x10], #0x4\n" - "st1 { v9.s }[0], [x9], #0x4\n" - "st1 { v21.s }[0], [x28], #0x4\n" + "st1 { v5.s }[0], [x11], #0x4\n" + "st1 { v21.s }[0], [x10], #0x4\n" + "st1 { v20.s }[0], [x9], #0x4\n" + "st1 { v19.s }[0], [x28], #0x4\n" "tbz x7, #1, 84f\n" - "st1 { v15.h }[2], [x11], #0x2\n" - "st1 { v10.h }[2], [x10], #0x2\n" - "st1 { v9.h }[2], [x9], #0x2\n" - "st1 { v21.h }[2], [x28], #0x2\n" + "st1 { v5.h }[2], [x11], #0x2\n" + "st1 { v21.h }[2], [x10], #0x2\n" + "st1 { v20.h }[2], [x9], #0x2\n" + "st1 { v19.h }[2], [x28], #0x2\n" "tbz x7, #0, 87f\n" - "st1 { v15.b }[6], [x11], #0x1\n" - "st1 { v10.b }[6], [x10], #0x1\n" - "st1 { v9.b }[6], [x9], #0x1\n" - "st1 { v21.b }[6], [x28], #0x1\n" + "st1 { v5.b }[6], [x11], #0x1\n" + "st1 { v21.b }[6], [x10], #0x1\n" + "st1 { v20.b }[6], [x9], #0x1\n" + "st1 { v19.b }[6], [x28], #0x1\n" "b 87f\n" "84:" // Oddments: Bit 2: Bit 1: Unset "tbz x7, #0, 87f\n" - "st1 { v15.b }[4], [x11], #0x1\n" - "st1 { v10.b }[4], [x10], #0x1\n" - "st1 { v9.b }[4], [x9], #0x1\n" - "st1 { v21.b }[4], [x28], #0x1\n" + "st1 { v5.b }[4], [x11], #0x1\n" + "st1 { v21.b }[4], [x10], #0x1\n" + "st1 { v20.b }[4], [x9], #0x1\n" + "st1 { v19.b }[4], [x28], #0x1\n" "b 87f\n" "85:" // Oddments: Bit 2: Unset "tbz x7, #1, 86f\n" - "st1 { v15.h }[0], [x11], #0x2\n" - "st1 { v10.h }[0], [x10], #0x2\n" - "st1 { v9.h }[0], [x9], #0x2\n" - "st1 { v21.h }[0], [x28], #0x2\n" + "st1 { v5.h }[0], [x11], #0x2\n" + "st1 { v21.h }[0], [x10], #0x2\n" + "st1 { v20.h }[0], [x9], #0x2\n" + "st1 { v19.h }[0], [x28], #0x2\n" "tbz x7, #0, 87f\n" - "st1 { v15.b }[2], [x11], #0x1\n" - "st1 { v10.b }[2], [x10], #0x1\n" - "st1 { v9.b }[2], [x9], #0x1\n" - "st1 { v21.b }[2], [x28], #0x1\n" + "st1 { v5.b }[2], [x11], #0x1\n" + "st1 { v21.b }[2], [x10], #0x1\n" + "st1 { v20.b }[2], [x9], #0x1\n" + "st1 { v19.b }[2], [x28], #0x1\n" "b 87f\n" "86:" // Oddments: Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 87f\n" - "st1 { v15.b }[0], [x11], #0x1\n" - "st1 { v10.b }[0], [x10], #0x1\n" - "st1 { v9.b }[0], [x9], #0x1\n" - "st1 { v21.b }[0], [x28], #0x1\n" + "st1 { v5.b }[0], [x11], #0x1\n" + "st1 { v21.b }[0], [x10], #0x1\n" + "st1 { v20.b }[0], [x9], #0x1\n" + "st1 { v19.b }[0], [x28], #0x1\n" "87:" // Oddments: Bit 2: End "88:" // End : diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp index 407807fcc1ff681febd6a4db1c8f850ee4aaed97..375e6f8f15d62b1630b3f1c4cd90b0e8603e3552 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include @@ -34,15 +34,7 @@ namespace arm_conv { namespace depthwise { -void a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( - const unsigned int, - const uint8_t *const *const, - const uint8_t *const, - const int32_t *const, - const arm_gemm::Requantize32 &, - const int32_t *const, - const int32_t *const, - uint8_t *const *const); +void a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl(unsigned int, const uint8_t *const *, const uint8_t *, const int32_t *, const arm_gemm::Requantize32 &, const int32_t *, const int32_t *, uint8_t *const *); class a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirstStrategy { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp index 2fe688a65e4fd59be5c912165b625bf5a2c2d1a7..ae663585a2b2f335ab0de7b943f3df49e461d40e 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp @@ -112,1188 +112,1188 @@ void a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( __asm__ __volatile__( "ldr x1, [%x[params], %[offsetof_Params_n_channels]]\n" - "ldr x13, [%x[params], %[offsetof_Params_requant]]\n" + "ldr x23, [%x[params], %[offsetof_Params_requant]]\n" "lsr x2, x1, #0x3\n" - "add x3, x13, %[offsetof_Requantize32_a_offset]\n" - "ld1r { v9.16b }, [x3]\n" - "ldr x24, [%x[params], %[offsetof_Params_outptrs]]\n" - "add x11, x13, %[offsetof_Requantize32_b_offset]\n" - "add x5, x13, %[offsetof_Requantize32_c_offset]\n" - "ld1r { v15.16b }, [x11]\n" - "ld1r { v14.8h }, [x5]\n" - "add x3, x13, %[offsetof_Requantize32_minval]\n" - "add x15, x13, %[offsetof_Requantize32_maxval]\n" - "ld1r { v12.8h }, [x3]\n" - "ld1r { v11.8h }, [x15]\n" - "mov x0, #0x0\n" - "mov x10, #0x0\n" - "add x4, %x[params], %[offsetof_Params_inptrs]\n" - "ldr x3, [%x[params], %[offsetof_Params_weights]]\n" - "ldr x5, [%x[params], %[offsetof_Params_requant_muls]]\n" + "add x20, x23, %[offsetof_Requantize32_a_offset]\n" + "ld1r { v18.16b }, [x20]\n" + "ldr x22, [%x[params], %[offsetof_Params_outptrs]]\n" + "add x21, x23, %[offsetof_Requantize32_b_offset]\n" + "add x20, x23, %[offsetof_Requantize32_c_offset]\n" + "ld1r { v13.16b }, [x21]\n" + "ld1r { v26.8h }, [x20]\n" + "add x21, x23, %[offsetof_Requantize32_minval]\n" + "add x20, x23, %[offsetof_Requantize32_maxval]\n" + "ld1r { v11.8h }, [x21]\n" + "ld1r { v0.8h }, [x20]\n" + "mov x3, #0x0\n" + "mov x4, #0x0\n" + "add x5, %x[params], %[offsetof_Params_inptrs]\n" + "ldr x6, [%x[params], %[offsetof_Params_weights]]\n" + "ldr x7, [%x[params], %[offsetof_Params_requant_muls]]\n" "ldr x8, [%x[params], %[offsetof_Params_requant_shifts]]\n" - "ldp x17, x6, [x24, #0x0]\n" - "ldp x7, x16, [x24, #0x10]\n" + "ldp x17, x16, [x22, #0x0]\n" + "ldp x15, x14, [x22, #0x10]\n" "cbz x2, 3f\n" - "ldr d0, [x3, #0x0]\n" - "ldr d1, [x3, #0x8]\n" + "ldr d6, [x6, #0x0]\n" + "ldr d14, [x6, #0x8]\n" "subs x2, x2, #0x1\n" - "usubl v0.8h, v0.8b, v15.8b\n" - "ldr d2, [x3, #0x10]\n" - "ldr d3, [x3, #0x18]\n" - "usubl v1.8h, v1.8b, v15.8b\n" - "usubl v2.8h, v2.8b, v15.8b\n" - "ldr d4, [x3, #0x20]\n" - "ldr x13, [%x[params], %[offsetof_Params_bias]]\n" - "usubl v3.8h, v3.8b, v15.8b\n" - "usubl v4.8h, v4.8b, v15.8b\n" - "ldr q13, [x13, #0x0]\n" - "ldr q19, [x13, #0x10]\n" - "add x13, x13, #0x20\n" - "str x13, [%x[params], %[offsetof_Params_bias]]\n" - "ldp x9, x28, [x4, #0x0]\n" - "ldp x27, x26, [x4, #0x10]\n" - "mov v20.16b, v13.16b\n" - "mov v10.16b, v19.16b\n" - "ldp x25, x24, [x4, #0x20]\n" - "ldp x23, x22, [x4, #0x30]\n" - "mov v8.16b, v13.16b\n" - "mov v7.16b, v19.16b\n" - "ldp x21, x20, [x4, #0x40]\n" - "ldr d31, [x9, x0]\n" - "mov v17.16b, v13.16b\n" - "mov v21.16b, v19.16b\n" - "ldr d30, [x28, x0]\n" - "ldr d29, [x27, x0]\n" - "usubl v31.8h, v31.8b, v9.8b\n" - "usubl v30.8h, v30.8b, v9.8b\n" - "ldr d28, [x26, x0]\n" - "ldr d27, [x25, x0]\n" - "usubl v29.8h, v29.8b, v9.8b\n" - "usubl v28.8h, v28.8b, v9.8b\n" - "ldr d23, [x24, x0]\n" - "ldr d25, [x23, x0]\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "usubl v23.8h, v23.8b, v9.8b\n" - "ldr d24, [x22, x0]\n" - "ldr d26, [x21, x0]\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "ldr d22, [x20, x0]\n" - "usubl v26.8h, v26.8b, v9.8b\n" - "usubl v22.8h, v22.8b, v9.8b\n" + "usubl v6.8h, v6.8b, v13.8b\n" + "ldr d10, [x6, #0x10]\n" + "ldr d21, [x6, #0x18]\n" + "usubl v14.8h, v14.8b, v13.8b\n" + "usubl v10.8h, v10.8b, v13.8b\n" + "ldr d12, [x6, #0x20]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "usubl v21.8h, v21.8b, v13.8b\n" + "usubl v12.8h, v12.8b, v13.8b\n" + "ldr q7, [x20, #0x0]\n" + "ldr q15, [x20, #0x10]\n" + "add x20, x20, #0x20\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" + "ldp x9, x28, [x5, #0x0]\n" + "ldp x27, x26, [x5, #0x10]\n" + "mov v20.16b, v7.16b\n" + "mov v5.16b, v15.16b\n" + "ldp x25, x24, [x5, #0x20]\n" + "ldp x23, x22, [x5, #0x30]\n" + "mov v24.16b, v7.16b\n" + "mov v22.16b, v15.16b\n" + "ldp x21, x20, [x5, #0x40]\n" + "ldr d31, [x9, x3]\n" + "mov v23.16b, v7.16b\n" + "mov v19.16b, v15.16b\n" + "ldr d17, [x28, x3]\n" + "ldr d30, [x27, x3]\n" + "usubl v31.8h, v31.8b, v18.8b\n" + "usubl v17.8h, v17.8b, v18.8b\n" + "ldr d16, [x26, x3]\n" + "ldr d3, [x25, x3]\n" + "usubl v30.8h, v30.8b, v18.8b\n" + "usubl v16.8h, v16.8b, v18.8b\n" + "ldr d4, [x24, x3]\n" + "ldr d25, [x23, x3]\n" + "usubl v3.8h, v3.8b, v18.8b\n" + "usubl v4.8h, v4.8b, v18.8b\n" + "ldr d9, [x22, x3]\n" + "ldr d29, [x21, x3]\n" + "usubl v25.8h, v25.8b, v18.8b\n" + "usubl v9.8h, v9.8b, v18.8b\n" + "ldr d28, [x20, x3]\n" + "usubl v29.8h, v29.8b, v18.8b\n" + "usubl v28.8h, v28.8b, v18.8b\n" "beq 2f\n" "1:" // Loop - "ldr q18, [x5, #0x0]\n" - "ldr q6, [x8, #0x0]\n" - "smlal v13.4s, v31.4h, v0.4h\n" - "smlal2 v19.4s, v31.8h, v0.8h\n" - "ldr q5, [x5, #0x10]\n" - "smlal v13.4s, v30.4h, v1.4h\n" - "ldr x20, [x4, #0x50]\n" - "smlal v20.4s, v30.4h, v0.4h\n" - "smlal v8.4s, v29.4h, v0.4h\n" - "smlal v17.4s, v28.4h, v0.4h\n" - "ldr x22, [x4, #0x58]\n" - "ldr x21, [x4, #0x60]\n" - "smlal2 v19.4s, v30.8h, v1.8h\n" - "smlal2 v10.4s, v30.8h, v0.8h\n" - "ldr d31, [x20, x0]\n" - "usubl v31.8h, v31.8b, v9.8b\n" - "smlal2 v7.4s, v29.8h, v0.8h\n" - "smlal v13.4s, v27.4h, v2.4h\n" - "ldr x20, [x4, #0x68]\n" - "ldr x26, [x4, #0x70]\n" - "smlal2 v21.4s, v28.8h, v0.8h\n" - "ldr d30, [x22, x0]\n" - "smlal v20.4s, v27.4h, v1.4h\n" - "usubl v30.8h, v30.8b, v9.8b\n" - "smlal v8.4s, v28.4h, v1.4h\n" - "smlal v17.4s, v23.4h, v1.4h\n" - "ldr x25, [x4, #0x78]\n" - "ldr x23, [x4, #0x80]\n" - "smlal2 v19.4s, v27.8h, v2.8h\n" - "smlal2 v10.4s, v27.8h, v1.8h\n" - "ldr d0, [x3, #0x28]\n" - "usubl v0.8h, v0.8b, v15.8b\n" - "smlal2 v7.4s, v28.8h, v1.8h\n" - "smlal v13.4s, v25.4h, v3.4h\n" - "ldr x24, [x4, #0x88]\n" - "ldr x15, [x4, #0x90]\n" - "smlal2 v21.4s, v23.8h, v1.8h\n" - "ldr d27, [x21, x0]\n" - "smlal v20.4s, v25.4h, v2.4h\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal v8.4s, v23.4h, v2.4h\n" - "smlal v17.4s, v31.4h, v2.4h\n" - "ldr x21, [x4, #0x98]\n" - "ldr x14, [x4, #0xa0]\n" - "smlal2 v19.4s, v25.8h, v3.8h\n" - "smlal2 v10.4s, v25.8h, v2.8h\n" - "ldr d1, [x3, #0x30]\n" - "usubl v1.8h, v1.8b, v15.8b\n" - "smlal2 v7.4s, v23.8h, v2.8h\n" - "smlal v13.4s, v24.4h, v4.4h\n" - "ldr x13, [x4, #0xa8]\n" - "ldr x12, [x4, #0xb0]\n" - "smlal2 v21.4s, v31.8h, v2.8h\n" - "ldr d25, [x20, x0]\n" - "smlal v20.4s, v24.4h, v3.4h\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "smlal v8.4s, v31.4h, v3.4h\n" - "smlal v17.4s, v30.4h, v3.4h\n" - "ldr x20, [x4, #0xb8]\n" - "ldr x11, [x4, #0xc0]\n" - "smlal2 v19.4s, v24.8h, v4.8h\n" - "smlal2 v10.4s, v24.8h, v3.8h\n" - "ldr d2, [x3, #0x38]\n" - "usubl v2.8h, v2.8b, v15.8b\n" - "smlal2 v7.4s, v31.8h, v3.8h\n" - "smlal v13.4s, v29.4h, v0.4h\n" - "ldr x22, [x4, #0xc8]\n" - "ldr x9, [x4, #0xd0]\n" - "smlal2 v21.4s, v30.8h, v3.8h\n" - "ldr d24, [x26, x0]\n" - "smlal v20.4s, v27.4h, v4.4h\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "smlal v8.4s, v30.4h, v4.4h\n" - "smlal v17.4s, v26.4h, v4.4h\n" - "ldr x28, [x4, #0xd8]\n" - "ldr x27, [x4, #0xe0]\n" - "smlal2 v19.4s, v29.8h, v0.8h\n" - "ldr d3, [x3, #0x40]\n" - "smlal2 v10.4s, v27.8h, v4.8h\n" - "ldr d27, [x25, x0]\n" - "smlal2 v7.4s, v30.8h, v4.8h\n" - "smlal v13.4s, v28.4h, v1.4h\n" - "usubl v3.8h, v3.8b, v15.8b\n" - "ldr x26, [x4, #0xe8]\n" - "smlal2 v21.4s, v26.8h, v4.8h\n" - "ldr d4, [x3, #0x48]\n" - "smlal v20.4s, v28.4h, v0.4h\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal v8.4s, v22.4h, v0.4h\n" - "smlal v17.4s, v25.4h, v0.4h\n" - "usubl v4.8h, v4.8b, v15.8b\n" + "ldr d2, [x6, #0x28]\n" + "ldr d27, [x6, #0x30]\n" + "smlal v7.4s, v31.4h, v6.4h\n" + "smlal2 v15.4s, v31.8h, v6.8h\n" + "ldr d1, [x6, #0x38]\n" + "ldr d31, [x6, #0x40]\n" + "smlal v7.4s, v17.4h, v14.4h\n" + "smlal v20.4s, v17.4h, v6.4h\n" + "ldr d8, [x6, #0x48]\n" + "ldr x22, [x5, #0x50]\n" + "smlal v24.4s, v30.4h, v6.4h\n" + "smlal v23.4s, v16.4h, v6.4h\n" + "smlal2 v15.4s, v17.8h, v14.8h\n" + "smlal v7.4s, v3.4h, v10.4h\n" + "ldr x20, [x5, #0x58]\n" + "ldr x21, [x5, #0x60]\n" + "smlal2 v5.4s, v17.8h, v6.8h\n" + "ldr d17, [x22, x3]\n" + "smlal2 v22.4s, v30.8h, v6.8h\n" + "usubl v17.8h, v17.8b, v18.8b\n" + "smlal2 v19.4s, v16.8h, v6.8h\n" + "ldr d6, [x20, x3]\n" + "smlal v20.4s, v3.4h, v14.4h\n" + "usubl v6.8h, v6.8b, v18.8b\n" + "smlal v24.4s, v16.4h, v14.4h\n" + "smlal v23.4s, v4.4h, v14.4h\n" + "usubl v2.8h, v2.8b, v13.8b\n" + "ldr x20, [x5, #0x68]\n" + "smlal2 v15.4s, v3.8h, v10.8h\n" + "smlal v7.4s, v25.4h, v21.4h\n" + "usubl v27.8h, v27.8b, v13.8b\n" + "ldr x22, [x5, #0x70]\n" + "smlal2 v5.4s, v3.8h, v14.8h\n" + "ldr d3, [x21, x3]\n" + "smlal2 v22.4s, v16.8h, v14.8h\n" + "usubl v3.8h, v3.8b, v18.8b\n" + "smlal2 v19.4s, v4.8h, v14.8h\n" + "ldr d14, [x20, x3]\n" + "smlal v20.4s, v25.4h, v10.4h\n" + "usubl v14.8h, v14.8b, v18.8b\n" + "smlal v24.4s, v4.4h, v10.4h\n" + "smlal v23.4s, v17.4h, v10.4h\n" + "usubl v1.8h, v1.8b, v13.8b\n" + "ldr x20, [x5, #0x78]\n" + "smlal2 v15.4s, v25.8h, v21.8h\n" + "smlal v7.4s, v9.4h, v12.4h\n" + "usubl v31.8h, v31.8b, v13.8b\n" + "ldr x21, [x5, #0x80]\n" + "smlal2 v5.4s, v25.8h, v10.8h\n" + "ldr d25, [x22, x3]\n" + "smlal2 v22.4s, v4.8h, v10.8h\n" + "usubl v25.8h, v25.8b, v18.8b\n" + "smlal2 v19.4s, v17.8h, v10.8h\n" + "ldr d10, [x20, x3]\n" + "smlal v20.4s, v9.4h, v21.4h\n" + "usubl v10.8h, v10.8b, v18.8b\n" + "smlal v24.4s, v17.4h, v21.4h\n" + "smlal v23.4s, v6.4h, v21.4h\n" + "usubl v8.8h, v8.8b, v13.8b\n" + "ldr x24, [x5, #0x88]\n" + "smlal2 v15.4s, v9.8h, v12.8h\n" + "smlal v7.4s, v30.4h, v2.4h\n" + "ldr x20, [x5, #0x90]\n" + "ldr x23, [x5, #0x98]\n" + "smlal2 v5.4s, v9.8h, v21.8h\n" + "ldr d9, [x21, x3]\n" + "smlal2 v22.4s, v17.8h, v21.8h\n" + "usubl v9.8h, v9.8b, v18.8b\n" + "smlal2 v19.4s, v6.8h, v21.8h\n" + "ldr d21, [x6, #0x50]\n" + "smlal v20.4s, v3.4h, v12.4h\n" + "usubl v21.8h, v21.8b, v13.8b\n" + "smlal v24.4s, v6.4h, v12.4h\n" + "smlal v23.4s, v29.4h, v12.4h\n" + "ldr x22, [x5, #0xa0]\n" + "ldr x21, [x5, #0xa8]\n" + "smlal2 v15.4s, v30.8h, v2.8h\n" + "ldr d30, [x24, x3]\n" + "smlal v7.4s, v16.4h, v27.4h\n" + "usubl v30.8h, v30.8b, v18.8b\n" + "smlal2 v5.4s, v3.8h, v12.8h\n" + "ldr d3, [x6, #0x58]\n" + "smlal2 v22.4s, v6.8h, v12.8h\n" + "usubl v3.8h, v3.8b, v13.8b\n" + "smlal2 v19.4s, v29.8h, v12.8h\n" + "ldr d12, [x20, x3]\n" + "smlal v20.4s, v16.4h, v2.4h\n" + "usubl v12.8h, v12.8b, v18.8b\n" + "smlal v24.4s, v28.4h, v2.4h\n" + "smlal v23.4s, v14.4h, v2.4h\n" + "ldr x20, [x5, #0xb0]\n" + "ldr x13, [x5, #0xb8]\n" + "smlal2 v15.4s, v16.8h, v27.8h\n" + "smlal v7.4s, v4.4h, v1.4h\n" + "ldr x12, [x5, #0xc0]\n" + "ldr x11, [x5, #0xc8]\n" + "smlal2 v5.4s, v16.8h, v2.8h\n" + "ldr d16, [x23, x3]\n" + "smlal2 v22.4s, v28.8h, v2.8h\n" + "usubl v16.8h, v16.8b, v18.8b\n" + "smlal2 v19.4s, v14.8h, v2.8h\n" + "ldr d2, [x6, #0x60]\n" + "smlal v20.4s, v4.4h, v27.4h\n" + "usubl v2.8h, v2.8b, v13.8b\n" + "smlal v24.4s, v14.4h, v27.4h\n" + "smlal v23.4s, v25.4h, v27.4h\n" + "ldr x10, [x5, #0xd0]\n" + "ldr x9, [x5, #0xd8]\n" + "smlal2 v15.4s, v4.8h, v1.8h\n" + "smlal v7.4s, v17.4h, v31.4h\n" + "ldr x28, [x5, #0xe0]\n" + "ldr x27, [x5, #0xe8]\n" + "smlal2 v5.4s, v4.8h, v27.8h\n" + "ldr d4, [x22, x3]\n" + "smlal2 v22.4s, v14.8h, v27.8h\n" + "usubl v4.8h, v4.8b, v18.8b\n" + "smlal2 v19.4s, v25.8h, v27.8h\n" + "ldr d27, [x6, #0x68]\n" + "smlal v20.4s, v17.4h, v1.4h\n" + "usubl v27.8h, v27.8b, v13.8b\n" + "smlal v24.4s, v25.4h, v1.4h\n" + "smlal v23.4s, v10.4h, v1.4h\n" + "ldr x26, [x5, #0xf0]\n" + "ldr x25, [x5, #0xf8]\n" + "smlal2 v15.4s, v17.8h, v31.8h\n" + "smlal v7.4s, v6.4h, v8.4h\n" + "ldr x24, [x5, #0x100]\n" + "ldr x23, [x5, #0x108]\n" + "smlal2 v5.4s, v17.8h, v1.8h\n" + "ldr d17, [x21, x3]\n" + "smlal2 v22.4s, v25.8h, v1.8h\n" + "usubl v17.8h, v17.8b, v18.8b\n" + "smlal2 v19.4s, v10.8h, v1.8h\n" + "ldr d1, [x6, #0x70]\n" + "smlal v20.4s, v6.4h, v31.4h\n" + "usubl v1.8h, v1.8b, v13.8b\n" + "smlal v24.4s, v10.4h, v31.4h\n" + "smlal v23.4s, v9.4h, v31.4h\n" + "ldr x22, [x5, #0x110]\n" + "ldr x21, [x5, #0x118]\n" + "smlal2 v15.4s, v6.8h, v8.8h\n" + "smlal v7.4s, v28.4h, v21.4h\n" "subs x2, x2, #0x1\n" + "smlal2 v5.4s, v6.8h, v31.8h\n" + "ldr d6, [x20, x3]\n" + "smlal2 v22.4s, v10.8h, v31.8h\n" + "usubl v6.8h, v6.8b, v18.8b\n" + "smlal2 v19.4s, v9.8h, v31.8h\n" + "ldr d31, [x6, #0x78]\n" + "smlal v20.4s, v29.4h, v8.4h\n" + "usubl v31.8h, v31.8b, v13.8b\n" + "smlal v24.4s, v9.4h, v8.4h\n" + "smlal v23.4s, v30.4h, v8.4h\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "smlal2 v15.4s, v28.8h, v21.8h\n" + "ldr d28, [x13, x3]\n" + "smlal v7.4s, v14.4h, v3.4h\n" + "usubl v28.8h, v28.8b, v18.8b\n" + "smlal2 v5.4s, v29.8h, v8.8h\n" + "ldr d29, [x6, #0x80]\n" + "smlal2 v22.4s, v9.8h, v8.8h\n" + "usubl v29.8h, v29.8b, v13.8b\n" + "smlal2 v19.4s, v30.8h, v8.8h\n" + "ldr d8, [x12, x3]\n" + "smlal v20.4s, v14.4h, v21.4h\n" + "usubl v8.8h, v8.8b, v18.8b\n" + "smlal v24.4s, v12.4h, v21.4h\n" + "smlal v23.4s, v16.4h, v21.4h\n" + "smlal2 v15.4s, v14.8h, v3.8h\n" + "smlal v7.4s, v25.4h, v2.4h\n" + "smlal2 v5.4s, v14.8h, v21.8h\n" + "ldr d14, [x11, x3]\n" + "smlal2 v22.4s, v12.8h, v21.8h\n" + "usubl v14.8h, v14.8b, v18.8b\n" + "smlal2 v19.4s, v16.8h, v21.8h\n" + "ldr d21, [x6, #0x88]\n" + "smlal v20.4s, v25.4h, v3.4h\n" + "usubl v21.8h, v21.8b, v13.8b\n" + "smlal v24.4s, v16.4h, v3.4h\n" + "smlal v23.4s, v4.4h, v3.4h\n" + "smlal2 v15.4s, v25.8h, v2.8h\n" + "smlal v7.4s, v10.4h, v27.4h\n" + "smlal2 v5.4s, v25.8h, v3.8h\n" + "ldr d25, [x10, x3]\n" + "smlal2 v22.4s, v16.8h, v3.8h\n" + "usubl v25.8h, v25.8b, v18.8b\n" + "smlal2 v19.4s, v4.8h, v3.8h\n" + "ldr d3, [x6, #0x90]\n" + "smlal v20.4s, v10.4h, v2.4h\n" + "usubl v3.8h, v3.8b, v13.8b\n" + "smlal v24.4s, v4.4h, v2.4h\n" + "smlal v23.4s, v17.4h, v2.4h\n" + "smlal2 v15.4s, v10.8h, v27.8h\n" + "smlal v7.4s, v9.4h, v1.4h\n" + "smlal2 v5.4s, v10.8h, v2.8h\n" + "ldr d10, [x9, x3]\n" + "smlal2 v22.4s, v4.8h, v2.8h\n" + "usubl v10.8h, v10.8b, v18.8b\n" + "smlal2 v19.4s, v17.8h, v2.8h\n" + "ldr d2, [x6, #0x98]\n" + "smlal v20.4s, v9.4h, v27.4h\n" + "usubl v2.8h, v2.8b, v13.8b\n" + "smlal v24.4s, v17.4h, v27.4h\n" + "smlal v23.4s, v6.4h, v27.4h\n" + "smlal2 v15.4s, v9.8h, v1.8h\n" + "smlal v7.4s, v12.4h, v31.4h\n" + "smlal2 v5.4s, v9.8h, v27.8h\n" + "ldr d9, [x28, x3]\n" + "smlal2 v22.4s, v17.8h, v27.8h\n" + "usubl v9.8h, v9.8b, v18.8b\n" + "smlal2 v19.4s, v6.8h, v27.8h\n" + "ldr d27, [x6, #0xa0]\n" + "smlal v20.4s, v30.4h, v1.4h\n" + "usubl v27.8h, v27.8b, v13.8b\n" + "smlal v24.4s, v6.4h, v1.4h\n" + "smlal v23.4s, v28.4h, v1.4h\n" + "smlal2 v15.4s, v12.8h, v31.8h\n" + "ldr d12, [x27, x3]\n" + "smlal v7.4s, v16.4h, v29.4h\n" + "usubl v12.8h, v12.8b, v18.8b\n" + "smlal2 v5.4s, v30.8h, v1.8h\n" + "ldr d30, [x6, #0xa8]\n" + "smlal2 v22.4s, v6.8h, v1.8h\n" + "usubl v30.8h, v30.8b, v13.8b\n" "smlal2 v19.4s, v28.8h, v1.8h\n" - "smlal2 v10.4s, v28.8h, v0.8h\n" - "ldr d28, [x24, x0]\n" - "usubl v28.8h, v28.8b, v9.8b\n" - "smlal2 v7.4s, v22.8h, v0.8h\n" - "smlal v13.4s, v23.4h, v2.4h\n" - "ldr x25, [x4, #0xf0]\n" - "add x5, x5, #0x20\n" - "smlal2 v21.4s, v25.8h, v0.8h\n" - "ldr d0, [x3, #0x50]\n" - "smlal v20.4s, v23.4h, v1.4h\n" - "usubl v0.8h, v0.8b, v15.8b\n" - "smlal v8.4s, v25.4h, v1.4h\n" - "smlal v17.4s, v24.4h, v1.4h\n" - "smlal2 v19.4s, v23.8h, v2.8h\n" - "smlal2 v10.4s, v23.8h, v1.8h\n" - "ldr d23, [x23, x0]\n" - "usubl v23.8h, v23.8b, v9.8b\n" - "smlal2 v7.4s, v25.8h, v1.8h\n" - "smlal v13.4s, v31.4h, v3.4h\n" - "ldr x24, [x4, #0xf8]\n" - "smlal2 v21.4s, v24.8h, v1.8h\n" - "ldr d1, [x3, #0x58]\n" - "smlal v20.4s, v31.4h, v2.4h\n" - "usubl v1.8h, v1.8b, v15.8b\n" - "smlal v8.4s, v24.4h, v2.4h\n" - "smlal v17.4s, v27.4h, v2.4h\n" - "smlal2 v19.4s, v31.8h, v3.8h\n" - "smlal2 v10.4s, v31.8h, v2.8h\n" - "ldr d31, [x15, x0]\n" - "usubl v31.8h, v31.8b, v9.8b\n" - "smlal2 v7.4s, v24.8h, v2.8h\n" - "smlal v13.4s, v30.4h, v4.4h\n" - "ldr x23, [x4, #0x100]\n" - "smlal2 v21.4s, v27.8h, v2.8h\n" - "ldr d2, [x3, #0x60]\n" - "smlal v20.4s, v30.4h, v3.4h\n" - "usubl v2.8h, v2.8b, v15.8b\n" - "smlal v8.4s, v27.4h, v3.4h\n" - "smlal v17.4s, v23.4h, v3.4h\n" - "smlal2 v19.4s, v30.8h, v4.8h\n" - "smlal2 v10.4s, v30.8h, v3.8h\n" - "ldr d30, [x21, x0]\n" - "usubl v30.8h, v30.8b, v9.8b\n" - "smlal2 v7.4s, v27.8h, v3.8h\n" - "smlal v13.4s, v22.4h, v0.4h\n" - "ldr x15, [x4, #0x108]\n" - "smlal2 v21.4s, v23.8h, v3.8h\n" - "ldr d3, [x3, #0x68]\n" - "smlal v20.4s, v26.4h, v4.4h\n" - "usubl v3.8h, v3.8b, v15.8b\n" - "smlal v8.4s, v23.4h, v4.4h\n" - "smlal v17.4s, v28.4h, v4.4h\n" - "smlal2 v19.4s, v22.8h, v0.8h\n" - "ldr d22, [x20, x0]\n" - "smlal2 v10.4s, v26.8h, v4.8h\n" - "ldr d26, [x14, x0]\n" - "smlal2 v7.4s, v23.8h, v4.8h\n" - "smlal v13.4s, v25.4h, v1.4h\n" - "usubl v26.8h, v26.8b, v9.8b\n" - "ldr x21, [x4, #0x110]\n" - "smlal2 v21.4s, v28.8h, v4.8h\n" - "ldr d4, [x3, #0x70]\n" - "smlal v20.4s, v25.4h, v0.4h\n" - "usubl v4.8h, v4.8b, v15.8b\n" - "smlal v8.4s, v31.4h, v0.4h\n" - "smlal v17.4s, v30.4h, v0.4h\n" - "usubl v22.8h, v22.8b, v9.8b\n" - "ldr x20, [x4, #0x118]\n" - "smlal2 v19.4s, v25.8h, v1.8h\n" - "smlal2 v10.4s, v25.8h, v0.8h\n" - "ldr d25, [x13, x0]\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "smlal2 v7.4s, v31.8h, v0.8h\n" - "smlal v13.4s, v24.4h, v2.4h\n" - "ldr x13, [%x[params], %[offsetof_Params_bias]]\n" - "smlal2 v21.4s, v30.8h, v0.8h\n" - "ldr d0, [x3, #0x78]\n" - "smlal v20.4s, v24.4h, v1.4h\n" - "usubl v0.8h, v0.8b, v15.8b\n" - "smlal v8.4s, v30.4h, v1.4h\n" - "smlal v17.4s, v26.4h, v1.4h\n" - "smlal2 v19.4s, v24.8h, v2.8h\n" - "smlal2 v10.4s, v24.8h, v1.8h\n" - "ldr d24, [x12, x0]\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "smlal2 v7.4s, v30.8h, v1.8h\n" - "smlal v13.4s, v27.4h, v3.4h\n" - "smlal2 v21.4s, v26.8h, v1.8h\n" - "ldr d1, [x3, #0x80]\n" - "smlal v20.4s, v27.4h, v2.4h\n" - "usubl v1.8h, v1.8b, v15.8b\n" - "smlal v8.4s, v26.4h, v2.4h\n" - "smlal v17.4s, v25.4h, v2.4h\n" - "smlal2 v19.4s, v27.8h, v3.8h\n" - "smlal2 v10.4s, v27.8h, v2.8h\n" - "ldr d27, [x11, x0]\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal2 v7.4s, v26.8h, v2.8h\n" - "smlal v13.4s, v23.4h, v4.4h\n" - "smlal2 v21.4s, v25.8h, v2.8h\n" - "ldr d2, [x3, #0x88]\n" - "smlal v20.4s, v23.4h, v3.4h\n" - "usubl v2.8h, v2.8b, v15.8b\n" - "smlal v8.4s, v25.4h, v3.4h\n" - "smlal v17.4s, v24.4h, v3.4h\n" - "smlal2 v19.4s, v23.8h, v4.8h\n" - "smlal2 v10.4s, v23.8h, v3.8h\n" - "ldr d23, [x22, x0]\n" - "usubl v23.8h, v23.8b, v9.8b\n" - "smlal2 v7.4s, v25.8h, v3.8h\n" - "smlal v13.4s, v31.4h, v0.4h\n" - "smlal2 v21.4s, v24.8h, v3.8h\n" - "ldr d3, [x3, #0x90]\n" - "smlal v20.4s, v28.4h, v4.4h\n" - "usubl v3.8h, v3.8b, v15.8b\n" - "smlal v8.4s, v24.4h, v4.4h\n" - "smlal v17.4s, v22.4h, v4.4h\n" - "smlal2 v19.4s, v31.8h, v0.8h\n" - "ldr d31, [x9, x0]\n" - "smlal2 v10.4s, v28.8h, v4.8h\n" - "ldr d28, [x27, x0]\n" - "smlal2 v7.4s, v24.8h, v4.8h\n" - "smlal v13.4s, v30.4h, v1.4h\n" - "usubl v31.8h, v31.8b, v9.8b\n" - "smlal2 v21.4s, v22.8h, v4.8h\n" - "ldr d4, [x3, #0x98]\n" - "smlal v20.4s, v30.4h, v0.4h\n" - "usubl v4.8h, v4.8b, v15.8b\n" - "smlal v8.4s, v27.4h, v0.4h\n" - "smlal v17.4s, v23.4h, v0.4h\n" - "usubl v28.8h, v28.8b, v9.8b\n" - "smlal2 v19.4s, v30.8h, v1.8h\n" - "smlal2 v10.4s, v30.8h, v0.8h\n" - "ldr d30, [x28, x0]\n" - "usubl v30.8h, v30.8b, v9.8b\n" - "smlal2 v7.4s, v27.8h, v0.8h\n" - "smlal v13.4s, v26.4h, v2.4h\n" - "smlal2 v21.4s, v23.8h, v0.8h\n" - "ldr d0, [x3, #0xa0]\n" - "smlal v20.4s, v26.4h, v1.4h\n" - "usubl v0.8h, v0.8b, v15.8b\n" - "smlal v8.4s, v23.4h, v1.4h\n" - "smlal v17.4s, v31.4h, v1.4h\n" - "smlal2 v19.4s, v26.8h, v2.8h\n" - "smlal2 v10.4s, v26.8h, v1.8h\n" - "ldr d26, [x26, x0]\n" - "usubl v26.8h, v26.8b, v9.8b\n" - "smlal2 v7.4s, v23.8h, v1.8h\n" - "smlal v13.4s, v25.4h, v3.4h\n" - "smlal2 v21.4s, v31.8h, v1.8h\n" - "ldr d1, [x3, #0xa8]\n" - "smlal v20.4s, v25.4h, v2.4h\n" - "usubl v1.8h, v1.8b, v15.8b\n" - "smlal v8.4s, v31.4h, v2.4h\n" - "smlal v17.4s, v30.4h, v2.4h\n" - "smlal2 v19.4s, v25.8h, v3.8h\n" - "smlal2 v10.4s, v25.8h, v2.8h\n" - "ldr d25, [x25, x0]\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "smlal2 v7.4s, v31.8h, v2.8h\n" - "smlal v13.4s, v24.4h, v4.4h\n" - "smlal2 v21.4s, v30.8h, v2.8h\n" - "ldr d2, [x3, #0xb0]\n" - "smlal v20.4s, v24.4h, v3.4h\n" - "usubl v2.8h, v2.8b, v15.8b\n" - "smlal v8.4s, v30.4h, v3.4h\n" - "smlal v17.4s, v28.4h, v3.4h\n" - "smlal2 v19.4s, v24.8h, v4.8h\n" - "smlal2 v10.4s, v24.8h, v3.8h\n" - "ldr d24, [x24, x0]\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "smlal2 v7.4s, v30.8h, v3.8h\n" - "smlal v13.4s, v27.4h, v0.4h\n" - "smlal2 v21.4s, v28.8h, v3.8h\n" - "ldr d3, [x3, #0xb8]\n" - "smlal v20.4s, v22.4h, v4.4h\n" - "usubl v3.8h, v3.8b, v15.8b\n" - "smlal v8.4s, v28.4h, v4.4h\n" - "smlal v17.4s, v26.4h, v4.4h\n" - "smlal2 v19.4s, v27.8h, v0.8h\n" - "ldr d27, [x23, x0]\n" - "smlal2 v7.4s, v28.8h, v4.8h\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal v13.4s, v23.4h, v1.4h\n" - "smlal2 v10.4s, v22.8h, v4.8h\n" - "ldr q22, [x8, #0x10]\n" + "ldr d1, [x26, x3]\n" + "smlal v20.4s, v16.4h, v31.4h\n" + "usubl v1.8h, v1.8b, v18.8b\n" + "smlal v24.4s, v8.4h, v31.4h\n" + "smlal v23.4s, v14.4h, v31.4h\n" + "smlal2 v15.4s, v16.8h, v29.8h\n" + "smlal v7.4s, v4.4h, v21.4h\n" + "smlal2 v5.4s, v16.8h, v31.8h\n" + "ldr d16, [x25, x3]\n" + "smlal2 v22.4s, v8.8h, v31.8h\n" + "usubl v16.8h, v16.8b, v18.8b\n" + "smlal2 v19.4s, v14.8h, v31.8h\n" + "ldr d31, [x6, #0xb0]\n" + "smlal v20.4s, v4.4h, v29.4h\n" + "usubl v31.8h, v31.8b, v13.8b\n" + "smlal v24.4s, v14.4h, v29.4h\n" + "smlal v23.4s, v25.4h, v29.4h\n" + "smlal2 v15.4s, v4.8h, v21.8h\n" + "smlal v7.4s, v17.4h, v3.4h\n" + "smlal2 v5.4s, v4.8h, v29.8h\n" + "ldr d4, [x24, x3]\n" + "smlal2 v22.4s, v14.8h, v29.8h\n" + "usubl v4.8h, v4.8b, v18.8b\n" + "smlal2 v19.4s, v25.8h, v29.8h\n" + "ldr d29, [x6, #0xb8]\n" + "smlal v20.4s, v17.4h, v21.4h\n" + "usubl v29.8h, v29.8b, v13.8b\n" + "smlal v24.4s, v25.4h, v21.4h\n" + "smlal v23.4s, v10.4h, v21.4h\n" + "smlal2 v15.4s, v17.8h, v3.8h\n" + "smlal v7.4s, v6.4h, v2.4h\n" + "smlal2 v5.4s, v17.8h, v21.8h\n" + "ldr d17, [x23, x3]\n" + "smlal2 v22.4s, v25.8h, v21.8h\n" + "usubl v17.8h, v17.8b, v18.8b\n" + "smlal2 v19.4s, v10.8h, v21.8h\n" + "ldr d21, [x6, #0xc0]\n" + "smlal v20.4s, v6.4h, v3.4h\n" + "usubl v21.8h, v21.8b, v13.8b\n" + "smlal v24.4s, v10.4h, v3.4h\n" + "smlal v23.4s, v9.4h, v3.4h\n" + "add x6, x6, #0xc8\n" + "smlal2 v15.4s, v6.8h, v2.8h\n" + "smlal v7.4s, v8.4h, v27.4h\n" + "smlal2 v5.4s, v6.8h, v3.8h\n" + "ldr d6, [x22, x3]\n" + "smlal2 v22.4s, v10.8h, v3.8h\n" + "usubl v6.8h, v6.8b, v18.8b\n" + "smlal2 v19.4s, v9.8h, v3.8h\n" + "ldr d3, [x21, x3]\n" + "smlal v20.4s, v28.4h, v2.4h\n" + "usubl v3.8h, v3.8b, v18.8b\n" + "smlal v24.4s, v9.4h, v2.4h\n" + "smlal v23.4s, v12.4h, v2.4h\n" + "add x3, x3, #0x8\n" + "smlal2 v15.4s, v8.8h, v27.8h\n" + "ldr q8, [x7, #0x0]\n" + "smlal v7.4s, v14.4h, v30.4h\n" + "smlal2 v5.4s, v28.8h, v2.8h\n" + "ldr q28, [x8, #0x0]\n" + "smlal2 v22.4s, v9.8h, v2.8h\n" + "smlal2 v19.4s, v12.8h, v2.8h\n" + "ldr q2, [x7, #0x10]\n" + "smlal v20.4s, v14.4h, v27.4h\n" + "add x7, x7, #0x20\n" + "smlal v24.4s, v1.4h, v27.4h\n" + "smlal v23.4s, v16.4h, v27.4h\n" + "smlal2 v15.4s, v14.8h, v30.8h\n" + "smlal v7.4s, v25.4h, v31.4h\n" + "smlal2 v5.4s, v14.8h, v27.8h\n" + "ldr q14, [x8, #0x10]\n" + "smlal2 v22.4s, v1.8h, v27.8h\n" "add x8, x8, #0x20\n" - "smlal2 v21.4s, v26.8h, v4.8h\n" - "ldr d4, [x3, #0xc0]\n" - "smlal v20.4s, v23.4h, v0.4h\n" - "usubl v4.8h, v4.8b, v15.8b\n" - "smlal v8.4s, v25.4h, v0.4h\n" - "smlal v17.4s, v24.4h, v0.4h\n" - "add x3, x3, #0xc8\n" - "smlal2 v19.4s, v23.8h, v1.8h\n" - "smlal2 v7.4s, v25.8h, v0.8h\n" - "ldr d25, [x15, x0]\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "smlal v13.4s, v31.4h, v2.4h\n" - "smlal2 v10.4s, v23.8h, v0.8h\n" - "smlal2 v21.4s, v24.8h, v0.8h\n" - "smlal v20.4s, v31.4h, v1.4h\n" - "smlal v8.4s, v24.4h, v1.4h\n" - "smlal v17.4s, v27.4h, v1.4h\n" - "smlal2 v19.4s, v31.8h, v2.8h\n" - "smlal2 v7.4s, v24.8h, v1.8h\n" - "ldr d24, [x21, x0]\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "smlal v13.4s, v30.4h, v3.4h\n" - "smlal2 v10.4s, v31.8h, v1.8h\n" - "smlal2 v21.4s, v27.8h, v1.8h\n" - "smlal v20.4s, v30.4h, v2.4h\n" - "smlal v8.4s, v27.4h, v2.4h\n" - "smlal v17.4s, v25.4h, v2.4h\n" - "smlal2 v19.4s, v30.8h, v3.8h\n" - "smlal2 v7.4s, v27.8h, v2.8h\n" - "ldr d27, [x20, x0]\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal v13.4s, v28.4h, v4.4h\n" - "smlal2 v10.4s, v30.8h, v2.8h\n" - "sqrdmulh v13.4s, v13.4s, v18.4s\n" - "add x0, x0, #0x8\n" - "smlal2 v21.4s, v25.8h, v2.8h\n" - "smlal v20.4s, v28.4h, v3.4h\n" - "and v30.16b, v13.16b, v6.16b\n" - "smlal v8.4s, v25.4h, v3.4h\n" - "smlal v17.4s, v24.4h, v3.4h\n" - "sshr v30.4s, v30.4s, #0x1f\n" - "smlal2 v19.4s, v28.8h, v4.8h\n" - "smlal2 v10.4s, v28.8h, v3.8h\n" - "sqrdmulh v19.4s, v19.4s, v5.4s\n" - "smlal2 v7.4s, v25.8h, v3.8h\n" - "smlal2 v21.4s, v24.8h, v3.8h\n" - "and v16.16b, v19.16b, v22.16b\n" - "smlal v20.4s, v26.4h, v4.4h\n" - "smlal v8.4s, v24.4h, v4.4h\n" - "sqrdmulh v20.4s, v20.4s, v18.4s\n" - "smlal v17.4s, v27.4h, v4.4h\n" - "smlal2 v10.4s, v26.8h, v4.8h\n" - "sqrdmulh v8.4s, v8.4s, v18.4s\n" - "smlal2 v7.4s, v24.8h, v4.8h\n" - "smlal2 v21.4s, v27.8h, v4.8h\n" - "sqrdmulh v17.4s, v17.4s, v18.4s\n" - "sqadd v13.4s, v13.4s, v30.4s\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "and v0.16b, v20.16b, v6.16b\n" - "sqrdmulh v10.4s, v10.4s, v5.4s\n" - "and v18.16b, v8.16b, v6.16b\n" - "sqrdmulh v7.4s, v7.4s, v5.4s\n" - "and v30.16b, v17.16b, v6.16b\n" - "sqrdmulh v21.4s, v21.4s, v5.4s\n" - "sqadd v19.4s, v19.4s, v16.4s\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v26.16b, v10.16b, v22.16b\n" - "sshr v18.4s, v18.4s, #0x1f\n" - "and v23.16b, v7.16b, v22.16b\n" - "sshr v30.4s, v30.4s, #0x1f\n" - "and v16.16b, v21.16b, v22.16b\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "sshr v26.4s, v26.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v18.4s\n" - "sshr v23.4s, v23.4s, #0x1f\n" - "sqadd v17.4s, v17.4s, v30.4s\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "srshl v13.4s, v13.4s, v6.4s\n" - "srshl v20.4s, v20.4s, v6.4s\n" - "sqadd v10.4s, v10.4s, v26.4s\n" - "srshl v8.4s, v8.4s, v6.4s\n" - "sqadd v7.4s, v7.4s, v23.4s\n" - "srshl v17.4s, v17.4s, v6.4s\n" - "sqadd v21.4s, v21.4s, v16.4s\n" - "srshl v19.4s, v19.4s, v22.4s\n" - "sqxtn v13.4h, v13.4s\n" - "srshl v10.4s, v10.4s, v22.4s\n" + "smlal2 v19.4s, v16.8h, v27.8h\n" + "smlal v20.4s, v25.4h, v30.4h\n" + "smlal v24.4s, v16.4h, v30.4h\n" + "smlal v23.4s, v4.4h, v30.4h\n" + "smlal2 v15.4s, v25.8h, v31.8h\n" + "smlal v7.4s, v10.4h, v29.4h\n" + "smlal2 v5.4s, v25.8h, v30.8h\n" + "smlal2 v22.4s, v16.8h, v30.8h\n" + "smlal2 v19.4s, v4.8h, v30.8h\n" + "smlal v20.4s, v10.4h, v31.4h\n" + "smlal v24.4s, v4.4h, v31.4h\n" + "smlal v23.4s, v17.4h, v31.4h\n" + "smlal2 v15.4s, v10.8h, v29.8h\n" + "smlal v7.4s, v9.4h, v21.4h\n" + "sqrdmulh v7.4s, v7.4s, v8.4s\n" + "smlal2 v5.4s, v10.8h, v31.8h\n" + "smlal2 v22.4s, v4.8h, v31.8h\n" + "and v27.16b, v7.16b, v28.16b\n" + "smlal2 v19.4s, v17.8h, v31.8h\n" + "smlal v20.4s, v9.4h, v29.4h\n" + "sshr v27.4s, v27.4s, #0x1f\n" + "smlal v24.4s, v17.4h, v29.4h\n" + "smlal v23.4s, v6.4h, v29.4h\n" + "sqadd v7.4s, v7.4s, v27.4s\n" + "smlal2 v15.4s, v9.8h, v21.8h\n" + "smlal2 v5.4s, v9.8h, v29.8h\n" + "sqrdmulh v15.4s, v15.4s, v2.4s\n" + "smlal2 v22.4s, v17.8h, v29.8h\n" + "smlal2 v19.4s, v6.8h, v29.8h\n" + "and v9.16b, v15.16b, v14.16b\n" + "smlal v20.4s, v12.4h, v21.4h\n" + "smlal v24.4s, v6.4h, v21.4h\n" + "sqrdmulh v20.4s, v20.4s, v8.4s\n" + "smlal v23.4s, v3.4h, v21.4h\n" + "smlal2 v5.4s, v12.8h, v21.8h\n" + "sqrdmulh v24.4s, v24.4s, v8.4s\n" + "smlal2 v22.4s, v6.8h, v21.8h\n" + "smlal2 v19.4s, v3.8h, v21.8h\n" + "sqrdmulh v23.4s, v23.4s, v8.4s\n" + "sshr v9.4s, v9.4s, #0x1f\n" + "and v25.16b, v20.16b, v28.16b\n" + "sqrdmulh v5.4s, v5.4s, v2.4s\n" + "and v10.16b, v24.16b, v28.16b\n" + "sqrdmulh v22.4s, v22.4s, v2.4s\n" + "and v21.16b, v23.16b, v28.16b\n" + "sqrdmulh v19.4s, v19.4s, v2.4s\n" + "sqadd v15.4s, v15.4s, v9.4s\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "and v9.16b, v5.16b, v14.16b\n" + "sshr v10.4s, v10.4s, #0x1f\n" + "and v12.16b, v22.16b, v14.16b\n" + "sshr v21.4s, v21.4s, #0x1f\n" + "and v17.16b, v19.16b, v14.16b\n" + "sqadd v20.4s, v20.4s, v25.4s\n" + "sshr v9.4s, v9.4s, #0x1f\n" + "sqadd v24.4s, v24.4s, v10.4s\n" + "sshr v12.4s, v12.4s, #0x1f\n" + "sqadd v23.4s, v23.4s, v21.4s\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "srshl v7.4s, v7.4s, v28.4s\n" + "srshl v20.4s, v20.4s, v28.4s\n" + "sqadd v5.4s, v5.4s, v9.4s\n" + "srshl v24.4s, v24.4s, v28.4s\n" + "sqadd v22.4s, v22.4s, v12.4s\n" + "srshl v23.4s, v23.4s, v28.4s\n" + "sqadd v19.4s, v19.4s, v17.4s\n" + "srshl v15.4s, v15.4s, v14.4s\n" + "sqxtn v7.4h, v7.4s\n" + "srshl v5.4s, v5.4s, v14.4s\n" "sqxtn v20.4h, v20.4s\n" - "srshl v7.4s, v7.4s, v22.4s\n" - "sqxtn v8.4h, v8.4s\n" - "srshl v21.4s, v21.4s, v22.4s\n" - "sqxtn v17.4h, v17.4s\n" - "sqxtn2 v13.8h, v19.4s\n" - "sqxtn2 v20.8h, v10.4s\n" - "sqxtn2 v8.8h, v7.4s\n" - "sqxtn2 v17.8h, v21.4s\n" - "sqadd v13.8h, v13.8h, v14.8h\n" - "sqadd v20.8h, v20.8h, v14.8h\n" - "sqadd v8.8h, v8.8h, v14.8h\n" - "sqadd v17.8h, v17.8h, v14.8h\n" - "smax v13.8h, v13.8h, v12.8h\n" - "smax v20.8h, v20.8h, v12.8h\n" - "smax v8.8h, v8.8h, v12.8h\n" - "smax v17.8h, v17.8h, v12.8h\n" - "smin v13.8h, v13.8h, v11.8h\n" - "smin v20.8h, v20.8h, v11.8h\n" - "smin v8.8h, v8.8h, v11.8h\n" - "smin v17.8h, v17.8h, v11.8h\n" - "uzp1 v13.16b, v13.16b, v13.16b\n" + "srshl v22.4s, v22.4s, v14.4s\n" + "sqxtn v24.4h, v24.4s\n" + "srshl v19.4s, v19.4s, v14.4s\n" + "sqxtn v23.4h, v23.4s\n" + "sqxtn2 v7.8h, v15.4s\n" + "sqxtn2 v20.8h, v5.4s\n" + "sqxtn2 v24.8h, v22.4s\n" + "sqxtn2 v23.8h, v19.4s\n" + "sqadd v7.8h, v7.8h, v26.8h\n" + "sqadd v20.8h, v20.8h, v26.8h\n" + "sqadd v24.8h, v24.8h, v26.8h\n" + "sqadd v23.8h, v23.8h, v26.8h\n" + "smax v7.8h, v7.8h, v11.8h\n" + "smax v20.8h, v20.8h, v11.8h\n" + "smax v24.8h, v24.8h, v11.8h\n" + "smax v23.8h, v23.8h, v11.8h\n" + "smin v7.8h, v7.8h, v0.8h\n" + "smin v20.8h, v20.8h, v0.8h\n" + "smin v24.8h, v24.8h, v0.8h\n" + "smin v23.8h, v23.8h, v0.8h\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" + "str d7, [x17, x4]\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "str d13, [x17, x10]\n" - "uzp1 v8.16b, v8.16b, v8.16b\n" - "uzp1 v17.16b, v17.16b, v17.16b\n" - "str d20, [x6, x10]\n" - "str d8, [x7, x10]\n" - "str d17, [x16, x10]\n" - "ldr q13, [x13, #0x0]\n" - "ldr q19, [x13, #0x10]\n" - "add x13, x13, #0x20\n" - "ldr d0, [x3, #0x0]\n" - "ldr d1, [x3, #0x8]\n" - "add x10, x10, #0x8\n" - "str x13, [%x[params], %[offsetof_Params_bias]]\n" - "ldr d2, [x3, #0x10]\n" - "ldr d3, [x3, #0x18]\n" - "mov v20.16b, v13.16b\n" - "mov v10.16b, v19.16b\n" - "ldr d4, [x3, #0x20]\n" - "ldp x9, x28, [x4, #0x0]\n" - "mov v8.16b, v13.16b\n" - "mov v7.16b, v19.16b\n" - "ldp x27, x26, [x4, #0x10]\n" - "ldp x25, x24, [x4, #0x20]\n" - "mov v17.16b, v13.16b\n" - "mov v21.16b, v19.16b\n" - "ldp x23, x22, [x4, #0x30]\n" - "ldp x21, x20, [x4, #0x40]\n" - "usubl v0.8h, v0.8b, v15.8b\n" - "usubl v1.8h, v1.8b, v15.8b\n" - "ldr d31, [x9, x0]\n" - "ldr d30, [x28, x0]\n" - "usubl v2.8h, v2.8b, v15.8b\n" - "usubl v3.8h, v3.8b, v15.8b\n" - "ldr d29, [x27, x0]\n" - "ldr d28, [x26, x0]\n" - "usubl v4.8h, v4.8b, v15.8b\n" - "usubl v31.8h, v31.8b, v9.8b\n" - "ldr d27, [x25, x0]\n" - "ldr d23, [x24, x0]\n" - "usubl v30.8h, v30.8b, v9.8b\n" - "usubl v29.8h, v29.8b, v9.8b\n" - "ldr d25, [x23, x0]\n" - "ldr d24, [x22, x0]\n" - "usubl v28.8h, v28.8b, v9.8b\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "ldr d26, [x21, x0]\n" - "ldr d22, [x20, x0]\n" - "usubl v23.8h, v23.8b, v9.8b\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "usubl v26.8h, v26.8b, v9.8b\n" - "usubl v22.8h, v22.8b, v9.8b\n" + "uzp1 v24.16b, v24.16b, v24.16b\n" + "str d20, [x16, x4]\n" + "uzp1 v23.16b, v23.16b, v23.16b\n" + "str d24, [x15, x4]\n" + "str d23, [x14, x4]\n" + "ldr q7, [x20, #0x0]\n" + "ldr q15, [x20, #0x10]\n" + "add x20, x20, #0x20\n" + "ldr d6, [x6, #0x0]\n" + "ldr d14, [x6, #0x8]\n" + "add x4, x4, #0x8\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" + "ldr d10, [x6, #0x10]\n" + "ldr d21, [x6, #0x18]\n" + "mov v20.16b, v7.16b\n" + "mov v5.16b, v15.16b\n" + "ldr d12, [x6, #0x20]\n" + "ldp x9, x28, [x5, #0x0]\n" + "mov v24.16b, v7.16b\n" + "mov v22.16b, v15.16b\n" + "ldp x27, x26, [x5, #0x10]\n" + "ldp x25, x24, [x5, #0x20]\n" + "mov v23.16b, v7.16b\n" + "mov v19.16b, v15.16b\n" + "ldp x23, x22, [x5, #0x30]\n" + "ldp x21, x20, [x5, #0x40]\n" + "usubl v6.8h, v6.8b, v13.8b\n" + "usubl v14.8h, v14.8b, v13.8b\n" + "ldr d31, [x9, x3]\n" + "ldr d17, [x28, x3]\n" + "usubl v10.8h, v10.8b, v13.8b\n" + "usubl v21.8h, v21.8b, v13.8b\n" + "ldr d30, [x27, x3]\n" + "ldr d16, [x26, x3]\n" + "usubl v12.8h, v12.8b, v13.8b\n" + "usubl v31.8h, v31.8b, v18.8b\n" + "ldr d3, [x25, x3]\n" + "ldr d4, [x24, x3]\n" + "usubl v17.8h, v17.8b, v18.8b\n" + "usubl v30.8h, v30.8b, v18.8b\n" + "ldr d25, [x23, x3]\n" + "ldr d9, [x22, x3]\n" + "usubl v16.8h, v16.8b, v18.8b\n" + "usubl v3.8h, v3.8b, v18.8b\n" + "ldr d29, [x21, x3]\n" + "ldr d28, [x20, x3]\n" + "usubl v4.8h, v4.8b, v18.8b\n" + "usubl v25.8h, v25.8b, v18.8b\n" + "usubl v9.8h, v9.8b, v18.8b\n" + "usubl v29.8h, v29.8b, v18.8b\n" + "usubl v28.8h, v28.8b, v18.8b\n" "bgt 1b\n" "2:" // Tail - "ldr q18, [x5, #0x0]\n" - "ldr q6, [x8, #0x0]\n" - "smlal v13.4s, v31.4h, v0.4h\n" - "smlal2 v19.4s, v31.8h, v0.8h\n" - "ldr q5, [x5, #0x10]\n" - "smlal v13.4s, v30.4h, v1.4h\n" - "ldr x20, [x4, #0x50]\n" - "smlal v20.4s, v30.4h, v0.4h\n" - "smlal v8.4s, v29.4h, v0.4h\n" - "smlal v17.4s, v28.4h, v0.4h\n" - "ldr x22, [x4, #0x58]\n" - "ldr x21, [x4, #0x60]\n" - "smlal2 v19.4s, v30.8h, v1.8h\n" - "smlal2 v10.4s, v30.8h, v0.8h\n" - "ldr d31, [x20, x0]\n" - "usubl v31.8h, v31.8b, v9.8b\n" - "smlal2 v7.4s, v29.8h, v0.8h\n" - "smlal v13.4s, v27.4h, v2.4h\n" - "ldr x20, [x4, #0x68]\n" - "ldr x26, [x4, #0x70]\n" - "smlal2 v21.4s, v28.8h, v0.8h\n" - "ldr d30, [x22, x0]\n" - "smlal v20.4s, v27.4h, v1.4h\n" - "usubl v30.8h, v30.8b, v9.8b\n" - "smlal v8.4s, v28.4h, v1.4h\n" - "smlal v17.4s, v23.4h, v1.4h\n" - "ldr x25, [x4, #0x78]\n" - "ldr x23, [x4, #0x80]\n" - "smlal2 v19.4s, v27.8h, v2.8h\n" - "smlal2 v10.4s, v27.8h, v1.8h\n" - "ldr d0, [x3, #0x28]\n" - "usubl v0.8h, v0.8b, v15.8b\n" - "smlal2 v7.4s, v28.8h, v1.8h\n" - "smlal v13.4s, v25.4h, v3.4h\n" - "ldr x24, [x4, #0x88]\n" - "ldr x15, [x4, #0x90]\n" - "smlal2 v21.4s, v23.8h, v1.8h\n" - "ldr d27, [x21, x0]\n" - "smlal v20.4s, v25.4h, v2.4h\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal v8.4s, v23.4h, v2.4h\n" - "smlal v17.4s, v31.4h, v2.4h\n" - "ldr x21, [x4, #0x98]\n" - "ldr x14, [x4, #0xa0]\n" - "smlal2 v19.4s, v25.8h, v3.8h\n" - "smlal2 v10.4s, v25.8h, v2.8h\n" - "ldr d1, [x3, #0x30]\n" - "usubl v1.8h, v1.8b, v15.8b\n" - "smlal2 v7.4s, v23.8h, v2.8h\n" - "smlal v13.4s, v24.4h, v4.4h\n" - "ldr x13, [x4, #0xa8]\n" - "ldr x12, [x4, #0xb0]\n" - "smlal2 v21.4s, v31.8h, v2.8h\n" - "ldr d25, [x20, x0]\n" - "smlal v20.4s, v24.4h, v3.4h\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "smlal v8.4s, v31.4h, v3.4h\n" - "smlal v17.4s, v30.4h, v3.4h\n" - "ldr x20, [x4, #0xb8]\n" - "ldr x11, [x4, #0xc0]\n" - "smlal2 v19.4s, v24.8h, v4.8h\n" - "smlal2 v10.4s, v24.8h, v3.8h\n" - "ldr d2, [x3, #0x38]\n" - "usubl v2.8h, v2.8b, v15.8b\n" - "smlal2 v7.4s, v31.8h, v3.8h\n" - "smlal v13.4s, v29.4h, v0.4h\n" - "ldr x22, [x4, #0xc8]\n" - "ldr x9, [x4, #0xd0]\n" - "smlal2 v21.4s, v30.8h, v3.8h\n" - "ldr d24, [x26, x0]\n" - "smlal v20.4s, v27.4h, v4.4h\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "smlal v8.4s, v30.4h, v4.4h\n" - "smlal v17.4s, v26.4h, v4.4h\n" - "ldr x28, [x4, #0xd8]\n" - "ldr x27, [x4, #0xe0]\n" - "smlal2 v19.4s, v29.8h, v0.8h\n" - "ldr d3, [x3, #0x40]\n" - "smlal2 v10.4s, v27.8h, v4.8h\n" - "ldr d27, [x25, x0]\n" - "smlal2 v7.4s, v30.8h, v4.8h\n" - "smlal v13.4s, v28.4h, v1.4h\n" - "usubl v3.8h, v3.8b, v15.8b\n" - "ldr x26, [x4, #0xe8]\n" - "smlal2 v21.4s, v26.8h, v4.8h\n" - "ldr d4, [x3, #0x48]\n" - "smlal v20.4s, v28.4h, v0.4h\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal v8.4s, v22.4h, v0.4h\n" - "smlal v17.4s, v25.4h, v0.4h\n" - "usubl v4.8h, v4.8b, v15.8b\n" - "ldr x25, [x4, #0xf0]\n" - "smlal2 v19.4s, v28.8h, v1.8h\n" - "smlal2 v10.4s, v28.8h, v0.8h\n" - "ldr d28, [x24, x0]\n" - "usubl v28.8h, v28.8b, v9.8b\n" - "smlal2 v7.4s, v22.8h, v0.8h\n" - "smlal v13.4s, v23.4h, v2.4h\n" - "ldr x24, [x4, #0xf8]\n" - "tst x1, #0x7\n" - "smlal2 v21.4s, v25.8h, v0.8h\n" - "ldr d0, [x3, #0x50]\n" - "smlal v20.4s, v23.4h, v1.4h\n" - "usubl v0.8h, v0.8b, v15.8b\n" - "smlal v8.4s, v25.4h, v1.4h\n" - "smlal v17.4s, v24.4h, v1.4h\n" - "add x5, x5, #0x20\n" - "smlal2 v19.4s, v23.8h, v2.8h\n" - "smlal2 v10.4s, v23.8h, v1.8h\n" - "ldr d23, [x23, x0]\n" - "usubl v23.8h, v23.8b, v9.8b\n" - "smlal2 v7.4s, v25.8h, v1.8h\n" - "smlal v13.4s, v31.4h, v3.4h\n" - "ldr x23, [x4, #0x100]\n" - "smlal2 v21.4s, v24.8h, v1.8h\n" - "ldr d1, [x3, #0x58]\n" - "smlal v20.4s, v31.4h, v2.4h\n" - "usubl v1.8h, v1.8b, v15.8b\n" - "smlal v8.4s, v24.4h, v2.4h\n" - "smlal v17.4s, v27.4h, v2.4h\n" - "smlal2 v19.4s, v31.8h, v3.8h\n" - "smlal2 v10.4s, v31.8h, v2.8h\n" - "ldr d31, [x15, x0]\n" - "usubl v31.8h, v31.8b, v9.8b\n" - "smlal2 v7.4s, v24.8h, v2.8h\n" - "smlal v13.4s, v30.4h, v4.4h\n" - "ldr x15, [x4, #0x108]\n" - "smlal2 v21.4s, v27.8h, v2.8h\n" - "ldr d2, [x3, #0x60]\n" - "smlal v20.4s, v30.4h, v3.4h\n" - "usubl v2.8h, v2.8b, v15.8b\n" - "smlal v8.4s, v27.4h, v3.4h\n" - "smlal v17.4s, v23.4h, v3.4h\n" - "smlal2 v19.4s, v30.8h, v4.8h\n" - "smlal2 v10.4s, v30.8h, v3.8h\n" - "ldr d30, [x21, x0]\n" - "usubl v30.8h, v30.8b, v9.8b\n" - "smlal2 v7.4s, v27.8h, v3.8h\n" - "smlal v13.4s, v22.4h, v0.4h\n" - "ldr x21, [x4, #0x110]\n" - "smlal2 v21.4s, v23.8h, v3.8h\n" - "ldr d3, [x3, #0x68]\n" - "smlal v20.4s, v26.4h, v4.4h\n" - "usubl v3.8h, v3.8b, v15.8b\n" - "smlal v8.4s, v23.4h, v4.4h\n" - "smlal v17.4s, v28.4h, v4.4h\n" - "smlal2 v19.4s, v22.8h, v0.8h\n" - "ldr d22, [x20, x0]\n" - "smlal2 v10.4s, v26.8h, v4.8h\n" - "ldr d26, [x14, x0]\n" - "smlal2 v7.4s, v23.8h, v4.8h\n" - "smlal v13.4s, v25.4h, v1.4h\n" - "usubl v26.8h, v26.8b, v9.8b\n" - "ldr x20, [x4, #0x118]\n" - "smlal2 v21.4s, v28.8h, v4.8h\n" - "ldr d4, [x3, #0x70]\n" - "smlal v20.4s, v25.4h, v0.4h\n" - "usubl v4.8h, v4.8b, v15.8b\n" - "smlal v8.4s, v31.4h, v0.4h\n" - "smlal v17.4s, v30.4h, v0.4h\n" - "usubl v22.8h, v22.8b, v9.8b\n" + "ldr d27, [x6, #0x28]\n" + "ldr d1, [x6, #0x30]\n" + "smlal v7.4s, v31.4h, v6.4h\n" + "smlal2 v15.4s, v31.8h, v6.8h\n" + "ldr d2, [x6, #0x38]\n" + "ldr d31, [x6, #0x40]\n" + "smlal v7.4s, v17.4h, v14.4h\n" + "smlal v20.4s, v17.4h, v6.4h\n" + "ldr d8, [x6, #0x48]\n" + "ldr x22, [x5, #0x50]\n" + "smlal v24.4s, v30.4h, v6.4h\n" + "smlal v23.4s, v16.4h, v6.4h\n" + "smlal2 v15.4s, v17.8h, v14.8h\n" + "smlal v7.4s, v3.4h, v10.4h\n" + "ldr x20, [x5, #0x58]\n" + "ldr x21, [x5, #0x60]\n" + "smlal2 v5.4s, v17.8h, v6.8h\n" + "ldr d17, [x22, x3]\n" + "smlal2 v22.4s, v30.8h, v6.8h\n" + "usubl v17.8h, v17.8b, v18.8b\n" + "smlal2 v19.4s, v16.8h, v6.8h\n" + "ldr d6, [x20, x3]\n" + "smlal v20.4s, v3.4h, v14.4h\n" + "usubl v6.8h, v6.8b, v18.8b\n" + "smlal v24.4s, v16.4h, v14.4h\n" + "smlal v23.4s, v4.4h, v14.4h\n" + "usubl v27.8h, v27.8b, v13.8b\n" + "ldr x20, [x5, #0x68]\n" + "smlal2 v15.4s, v3.8h, v10.8h\n" + "smlal v7.4s, v25.4h, v21.4h\n" + "usubl v1.8h, v1.8b, v13.8b\n" + "ldr x22, [x5, #0x70]\n" + "smlal2 v5.4s, v3.8h, v14.8h\n" + "ldr d3, [x21, x3]\n" + "smlal2 v22.4s, v16.8h, v14.8h\n" + "usubl v3.8h, v3.8b, v18.8b\n" + "smlal2 v19.4s, v4.8h, v14.8h\n" + "ldr d14, [x20, x3]\n" + "smlal v20.4s, v25.4h, v10.4h\n" + "usubl v14.8h, v14.8b, v18.8b\n" + "smlal v24.4s, v4.4h, v10.4h\n" + "smlal v23.4s, v17.4h, v10.4h\n" + "usubl v2.8h, v2.8b, v13.8b\n" + "ldr x21, [x5, #0x78]\n" + "smlal2 v15.4s, v25.8h, v21.8h\n" + "smlal v7.4s, v9.4h, v12.4h\n" + "usubl v31.8h, v31.8b, v13.8b\n" + "ldr x20, [x5, #0x80]\n" + "smlal2 v5.4s, v25.8h, v10.8h\n" + "ldr d25, [x22, x3]\n" + "smlal2 v22.4s, v4.8h, v10.8h\n" + "usubl v25.8h, v25.8b, v18.8b\n" + "smlal2 v19.4s, v17.8h, v10.8h\n" + "ldr d10, [x21, x3]\n" + "smlal v20.4s, v9.4h, v21.4h\n" + "usubl v10.8h, v10.8b, v18.8b\n" + "smlal v24.4s, v17.4h, v21.4h\n" + "smlal v23.4s, v6.4h, v21.4h\n" + "usubl v8.8h, v8.8b, v13.8b\n" + "ldr x24, [x5, #0x88]\n" + "smlal2 v15.4s, v9.8h, v12.8h\n" + "smlal v7.4s, v30.4h, v27.4h\n" + "ldr x23, [x5, #0x90]\n" + "ldr x22, [x5, #0x98]\n" + "smlal2 v5.4s, v9.8h, v21.8h\n" + "ldr d9, [x20, x3]\n" + "smlal2 v22.4s, v17.8h, v21.8h\n" + "usubl v9.8h, v9.8b, v18.8b\n" + "smlal2 v19.4s, v6.8h, v21.8h\n" + "ldr d21, [x6, #0x50]\n" + "smlal v20.4s, v3.4h, v12.4h\n" + "usubl v21.8h, v21.8b, v13.8b\n" + "smlal v24.4s, v6.4h, v12.4h\n" + "smlal v23.4s, v29.4h, v12.4h\n" + "ldr x21, [x5, #0xa0]\n" + "ldr x20, [x5, #0xa8]\n" + "smlal2 v15.4s, v30.8h, v27.8h\n" + "ldr d30, [x24, x3]\n" + "smlal v7.4s, v16.4h, v1.4h\n" + "usubl v30.8h, v30.8b, v18.8b\n" + "smlal2 v5.4s, v3.8h, v12.8h\n" + "ldr d3, [x6, #0x58]\n" + "smlal2 v22.4s, v6.8h, v12.8h\n" + "usubl v3.8h, v3.8b, v13.8b\n" + "smlal2 v19.4s, v29.8h, v12.8h\n" + "ldr d12, [x23, x3]\n" + "smlal v20.4s, v16.4h, v27.4h\n" + "usubl v12.8h, v12.8b, v18.8b\n" + "smlal v24.4s, v28.4h, v27.4h\n" + "smlal v23.4s, v14.4h, v27.4h\n" + "ldr x13, [x5, #0xb0]\n" + "ldr x12, [x5, #0xb8]\n" + "smlal2 v15.4s, v16.8h, v1.8h\n" + "smlal v7.4s, v4.4h, v2.4h\n" + "ldr x11, [x5, #0xc0]\n" + "ldr x10, [x5, #0xc8]\n" + "smlal2 v5.4s, v16.8h, v27.8h\n" + "ldr d16, [x22, x3]\n" + "smlal2 v22.4s, v28.8h, v27.8h\n" + "usubl v16.8h, v16.8b, v18.8b\n" + "smlal2 v19.4s, v14.8h, v27.8h\n" + "ldr d27, [x6, #0x60]\n" + "smlal v20.4s, v4.4h, v1.4h\n" + "usubl v27.8h, v27.8b, v13.8b\n" + "smlal v24.4s, v14.4h, v1.4h\n" + "smlal v23.4s, v25.4h, v1.4h\n" + "ldr x9, [x5, #0xd0]\n" + "ldr x28, [x5, #0xd8]\n" + "smlal2 v15.4s, v4.8h, v2.8h\n" + "smlal v7.4s, v17.4h, v31.4h\n" + "ldr x27, [x5, #0xe0]\n" + "ldr x26, [x5, #0xe8]\n" + "smlal2 v5.4s, v4.8h, v1.8h\n" + "ldr d4, [x21, x3]\n" + "smlal2 v22.4s, v14.8h, v1.8h\n" + "usubl v4.8h, v4.8b, v18.8b\n" "smlal2 v19.4s, v25.8h, v1.8h\n" - "smlal2 v10.4s, v25.8h, v0.8h\n" - "ldr d25, [x13, x0]\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "smlal2 v7.4s, v31.8h, v0.8h\n" - "smlal v13.4s, v24.4h, v2.4h\n" - "smlal2 v21.4s, v30.8h, v0.8h\n" - "ldr d0, [x3, #0x78]\n" - "smlal v20.4s, v24.4h, v1.4h\n" - "usubl v0.8h, v0.8b, v15.8b\n" - "smlal v8.4s, v30.4h, v1.4h\n" - "smlal v17.4s, v26.4h, v1.4h\n" - "smlal2 v19.4s, v24.8h, v2.8h\n" - "smlal2 v10.4s, v24.8h, v1.8h\n" - "ldr d24, [x12, x0]\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "smlal2 v7.4s, v30.8h, v1.8h\n" - "smlal v13.4s, v27.4h, v3.4h\n" - "smlal2 v21.4s, v26.8h, v1.8h\n" - "ldr d1, [x3, #0x80]\n" - "smlal v20.4s, v27.4h, v2.4h\n" - "usubl v1.8h, v1.8b, v15.8b\n" - "smlal v8.4s, v26.4h, v2.4h\n" - "smlal v17.4s, v25.4h, v2.4h\n" - "smlal2 v19.4s, v27.8h, v3.8h\n" - "smlal2 v10.4s, v27.8h, v2.8h\n" - "ldr d27, [x11, x0]\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal2 v7.4s, v26.8h, v2.8h\n" - "smlal v13.4s, v23.4h, v4.4h\n" - "smlal2 v21.4s, v25.8h, v2.8h\n" - "ldr d2, [x3, #0x88]\n" - "smlal v20.4s, v23.4h, v3.4h\n" - "usubl v2.8h, v2.8b, v15.8b\n" - "smlal v8.4s, v25.4h, v3.4h\n" - "smlal v17.4s, v24.4h, v3.4h\n" - "smlal2 v19.4s, v23.8h, v4.8h\n" - "smlal2 v10.4s, v23.8h, v3.8h\n" - "ldr d23, [x22, x0]\n" - "usubl v23.8h, v23.8b, v9.8b\n" - "smlal2 v7.4s, v25.8h, v3.8h\n" - "smlal v13.4s, v31.4h, v0.4h\n" - "smlal2 v21.4s, v24.8h, v3.8h\n" - "ldr d3, [x3, #0x90]\n" - "smlal v20.4s, v28.4h, v4.4h\n" - "usubl v3.8h, v3.8b, v15.8b\n" - "smlal v8.4s, v24.4h, v4.4h\n" - "smlal v17.4s, v22.4h, v4.4h\n" - "smlal2 v19.4s, v31.8h, v0.8h\n" - "ldr d31, [x9, x0]\n" - "smlal2 v10.4s, v28.8h, v4.8h\n" - "ldr d28, [x27, x0]\n" - "smlal2 v7.4s, v24.8h, v4.8h\n" - "smlal v13.4s, v30.4h, v1.4h\n" - "usubl v31.8h, v31.8b, v9.8b\n" - "smlal2 v21.4s, v22.8h, v4.8h\n" - "ldr d4, [x3, #0x98]\n" - "smlal v20.4s, v30.4h, v0.4h\n" - "usubl v4.8h, v4.8b, v15.8b\n" - "smlal v8.4s, v27.4h, v0.4h\n" - "smlal v17.4s, v23.4h, v0.4h\n" - "usubl v28.8h, v28.8b, v9.8b\n" - "smlal2 v19.4s, v30.8h, v1.8h\n" - "smlal2 v10.4s, v30.8h, v0.8h\n" - "ldr d30, [x28, x0]\n" - "usubl v30.8h, v30.8b, v9.8b\n" - "smlal2 v7.4s, v27.8h, v0.8h\n" - "smlal v13.4s, v26.4h, v2.4h\n" - "smlal2 v21.4s, v23.8h, v0.8h\n" - "ldr d0, [x3, #0xa0]\n" - "smlal v20.4s, v26.4h, v1.4h\n" - "usubl v0.8h, v0.8b, v15.8b\n" - "smlal v8.4s, v23.4h, v1.4h\n" - "smlal v17.4s, v31.4h, v1.4h\n" - "smlal2 v19.4s, v26.8h, v2.8h\n" - "smlal2 v10.4s, v26.8h, v1.8h\n" - "ldr d26, [x26, x0]\n" - "usubl v26.8h, v26.8b, v9.8b\n" - "smlal2 v7.4s, v23.8h, v1.8h\n" - "smlal v13.4s, v25.4h, v3.4h\n" - "smlal2 v21.4s, v31.8h, v1.8h\n" - "ldr d1, [x3, #0xa8]\n" - "smlal v20.4s, v25.4h, v2.4h\n" - "usubl v1.8h, v1.8b, v15.8b\n" - "smlal v8.4s, v31.4h, v2.4h\n" - "smlal v17.4s, v30.4h, v2.4h\n" - "smlal2 v19.4s, v25.8h, v3.8h\n" - "smlal2 v10.4s, v25.8h, v2.8h\n" - "ldr d25, [x25, x0]\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "smlal2 v7.4s, v31.8h, v2.8h\n" - "smlal v13.4s, v24.4h, v4.4h\n" - "smlal2 v21.4s, v30.8h, v2.8h\n" - "ldr d2, [x3, #0xb0]\n" - "smlal v20.4s, v24.4h, v3.4h\n" - "usubl v2.8h, v2.8b, v15.8b\n" - "smlal v8.4s, v30.4h, v3.4h\n" - "smlal v17.4s, v28.4h, v3.4h\n" - "smlal2 v19.4s, v24.8h, v4.8h\n" - "smlal2 v10.4s, v24.8h, v3.8h\n" - "ldr d24, [x24, x0]\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "smlal2 v7.4s, v30.8h, v3.8h\n" - "smlal v13.4s, v27.4h, v0.4h\n" - "smlal2 v21.4s, v28.8h, v3.8h\n" - "ldr d3, [x3, #0xb8]\n" - "smlal v20.4s, v22.4h, v4.4h\n" - "usubl v3.8h, v3.8b, v15.8b\n" - "smlal v8.4s, v28.4h, v4.4h\n" - "smlal v17.4s, v26.4h, v4.4h\n" - "smlal2 v19.4s, v27.8h, v0.8h\n" - "ldr d27, [x23, x0]\n" - "smlal2 v7.4s, v28.8h, v4.8h\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal v13.4s, v23.4h, v1.4h\n" - "smlal2 v10.4s, v22.8h, v4.8h\n" - "ldr q22, [x8, #0x10]\n" - "add x8, x8, #0x20\n" - "smlal2 v21.4s, v26.8h, v4.8h\n" - "ldr d4, [x3, #0xc0]\n" - "smlal v20.4s, v23.4h, v0.4h\n" - "usubl v4.8h, v4.8b, v15.8b\n" - "smlal v8.4s, v25.4h, v0.4h\n" - "smlal v17.4s, v24.4h, v0.4h\n" - "smlal2 v19.4s, v23.8h, v1.8h\n" - "smlal2 v7.4s, v25.8h, v0.8h\n" - "ldr d25, [x15, x0]\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "smlal v13.4s, v31.4h, v2.4h\n" - "smlal2 v10.4s, v23.8h, v0.8h\n" - "smlal2 v21.4s, v24.8h, v0.8h\n" - "smlal v20.4s, v31.4h, v1.4h\n" - "smlal v8.4s, v24.4h, v1.4h\n" - "smlal v17.4s, v27.4h, v1.4h\n" - "smlal2 v19.4s, v31.8h, v2.8h\n" - "smlal2 v7.4s, v24.8h, v1.8h\n" - "ldr d24, [x21, x0]\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "smlal v13.4s, v30.4h, v3.4h\n" - "smlal2 v10.4s, v31.8h, v1.8h\n" - "smlal2 v21.4s, v27.8h, v1.8h\n" + "ldr d1, [x6, #0x68]\n" + "smlal v20.4s, v17.4h, v2.4h\n" + "usubl v1.8h, v1.8b, v13.8b\n" + "smlal v24.4s, v25.4h, v2.4h\n" + "smlal v23.4s, v10.4h, v2.4h\n" + "ldr x25, [x5, #0xf0]\n" + "ldr x24, [x5, #0xf8]\n" + "smlal2 v15.4s, v17.8h, v31.8h\n" + "smlal v7.4s, v6.4h, v8.4h\n" + "ldr x23, [x5, #0x100]\n" + "ldr x22, [x5, #0x108]\n" + "smlal2 v5.4s, v17.8h, v2.8h\n" + "ldr d17, [x20, x3]\n" + "smlal2 v22.4s, v25.8h, v2.8h\n" + "usubl v17.8h, v17.8b, v18.8b\n" + "smlal2 v19.4s, v10.8h, v2.8h\n" + "ldr d2, [x6, #0x70]\n" + "smlal v20.4s, v6.4h, v31.4h\n" + "usubl v2.8h, v2.8b, v13.8b\n" + "smlal v24.4s, v10.4h, v31.4h\n" + "smlal v23.4s, v9.4h, v31.4h\n" + "ldr x21, [x5, #0x110]\n" + "ldr x20, [x5, #0x118]\n" + "smlal2 v15.4s, v6.8h, v8.8h\n" + "smlal v7.4s, v28.4h, v21.4h\n" + "tst x1, #0x7\n" + "smlal2 v5.4s, v6.8h, v31.8h\n" + "ldr d6, [x13, x3]\n" + "smlal2 v22.4s, v10.8h, v31.8h\n" + "usubl v6.8h, v6.8b, v18.8b\n" + "smlal2 v19.4s, v9.8h, v31.8h\n" + "ldr d31, [x6, #0x78]\n" + "smlal v20.4s, v29.4h, v8.4h\n" + "usubl v31.8h, v31.8b, v13.8b\n" + "smlal v24.4s, v9.4h, v8.4h\n" + "smlal v23.4s, v30.4h, v8.4h\n" + "smlal2 v15.4s, v28.8h, v21.8h\n" + "ldr d28, [x12, x3]\n" + "smlal v7.4s, v14.4h, v3.4h\n" + "usubl v28.8h, v28.8b, v18.8b\n" + "smlal2 v5.4s, v29.8h, v8.8h\n" + "ldr d29, [x6, #0x80]\n" + "smlal2 v22.4s, v9.8h, v8.8h\n" + "usubl v29.8h, v29.8b, v13.8b\n" + "smlal2 v19.4s, v30.8h, v8.8h\n" + "ldr d8, [x11, x3]\n" + "smlal v20.4s, v14.4h, v21.4h\n" + "usubl v8.8h, v8.8b, v18.8b\n" + "smlal v24.4s, v12.4h, v21.4h\n" + "smlal v23.4s, v16.4h, v21.4h\n" + "smlal2 v15.4s, v14.8h, v3.8h\n" + "smlal v7.4s, v25.4h, v27.4h\n" + "smlal2 v5.4s, v14.8h, v21.8h\n" + "ldr d14, [x10, x3]\n" + "smlal2 v22.4s, v12.8h, v21.8h\n" + "usubl v14.8h, v14.8b, v18.8b\n" + "smlal2 v19.4s, v16.8h, v21.8h\n" + "ldr d21, [x6, #0x88]\n" + "smlal v20.4s, v25.4h, v3.4h\n" + "usubl v21.8h, v21.8b, v13.8b\n" + "smlal v24.4s, v16.4h, v3.4h\n" + "smlal v23.4s, v4.4h, v3.4h\n" + "smlal2 v15.4s, v25.8h, v27.8h\n" + "smlal v7.4s, v10.4h, v1.4h\n" + "smlal2 v5.4s, v25.8h, v3.8h\n" + "ldr d25, [x9, x3]\n" + "smlal2 v22.4s, v16.8h, v3.8h\n" + "usubl v25.8h, v25.8b, v18.8b\n" + "smlal2 v19.4s, v4.8h, v3.8h\n" + "ldr d3, [x6, #0x90]\n" + "smlal v20.4s, v10.4h, v27.4h\n" + "usubl v3.8h, v3.8b, v13.8b\n" + "smlal v24.4s, v4.4h, v27.4h\n" + "smlal v23.4s, v17.4h, v27.4h\n" + "smlal2 v15.4s, v10.8h, v1.8h\n" + "smlal v7.4s, v9.4h, v2.4h\n" + "smlal2 v5.4s, v10.8h, v27.8h\n" + "ldr d10, [x28, x3]\n" + "smlal2 v22.4s, v4.8h, v27.8h\n" + "usubl v10.8h, v10.8b, v18.8b\n" + "smlal2 v19.4s, v17.8h, v27.8h\n" + "ldr d27, [x6, #0x98]\n" + "smlal v20.4s, v9.4h, v1.4h\n" + "usubl v27.8h, v27.8b, v13.8b\n" + "smlal v24.4s, v17.4h, v1.4h\n" + "smlal v23.4s, v6.4h, v1.4h\n" + "smlal2 v15.4s, v9.8h, v2.8h\n" + "smlal v7.4s, v12.4h, v31.4h\n" + "smlal2 v5.4s, v9.8h, v1.8h\n" + "ldr d9, [x27, x3]\n" + "smlal2 v22.4s, v17.8h, v1.8h\n" + "usubl v9.8h, v9.8b, v18.8b\n" + "smlal2 v19.4s, v6.8h, v1.8h\n" + "ldr d1, [x6, #0xa0]\n" "smlal v20.4s, v30.4h, v2.4h\n" - "smlal v8.4s, v27.4h, v2.4h\n" - "smlal v17.4s, v25.4h, v2.4h\n" - "smlal2 v19.4s, v30.8h, v3.8h\n" - "smlal2 v7.4s, v27.8h, v2.8h\n" - "ldr d27, [x20, x0]\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal v13.4s, v28.4h, v4.4h\n" - "smlal2 v10.4s, v30.8h, v2.8h\n" - "sqrdmulh v13.4s, v13.4s, v18.4s\n" - "add x0, x0, #0x8\n" - "smlal2 v21.4s, v25.8h, v2.8h\n" - "smlal v20.4s, v28.4h, v3.4h\n" - "and v30.16b, v13.16b, v6.16b\n" - "smlal v8.4s, v25.4h, v3.4h\n" - "smlal v17.4s, v24.4h, v3.4h\n" - "sshr v30.4s, v30.4s, #0x1f\n" - "smlal2 v19.4s, v28.8h, v4.8h\n" - "smlal2 v10.4s, v28.8h, v3.8h\n" - "sqrdmulh v19.4s, v19.4s, v5.4s\n" - "smlal2 v7.4s, v25.8h, v3.8h\n" - "smlal2 v21.4s, v24.8h, v3.8h\n" - "and v16.16b, v19.16b, v22.16b\n" - "smlal v20.4s, v26.4h, v4.4h\n" - "smlal v8.4s, v24.4h, v4.4h\n" - "sqrdmulh v20.4s, v20.4s, v18.4s\n" - "smlal v17.4s, v27.4h, v4.4h\n" - "smlal2 v10.4s, v26.8h, v4.8h\n" - "sqrdmulh v8.4s, v8.4s, v18.4s\n" - "smlal2 v7.4s, v24.8h, v4.8h\n" - "smlal2 v21.4s, v27.8h, v4.8h\n" - "sqrdmulh v17.4s, v17.4s, v18.4s\n" - "sqadd v13.4s, v13.4s, v30.4s\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "and v0.16b, v20.16b, v6.16b\n" - "sqrdmulh v10.4s, v10.4s, v5.4s\n" - "and v18.16b, v8.16b, v6.16b\n" - "sqrdmulh v7.4s, v7.4s, v5.4s\n" - "and v30.16b, v17.16b, v6.16b\n" - "sqrdmulh v21.4s, v21.4s, v5.4s\n" - "sqadd v19.4s, v19.4s, v16.4s\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v26.16b, v10.16b, v22.16b\n" - "sshr v18.4s, v18.4s, #0x1f\n" - "and v23.16b, v7.16b, v22.16b\n" + "usubl v1.8h, v1.8b, v13.8b\n" + "smlal v24.4s, v6.4h, v2.4h\n" + "smlal v23.4s, v28.4h, v2.4h\n" + "smlal2 v15.4s, v12.8h, v31.8h\n" + "ldr d12, [x26, x3]\n" + "smlal v7.4s, v16.4h, v29.4h\n" + "usubl v12.8h, v12.8b, v18.8b\n" + "smlal2 v5.4s, v30.8h, v2.8h\n" + "ldr d30, [x6, #0xa8]\n" + "smlal2 v22.4s, v6.8h, v2.8h\n" + "usubl v30.8h, v30.8b, v13.8b\n" + "smlal2 v19.4s, v28.8h, v2.8h\n" + "ldr d2, [x25, x3]\n" + "smlal v20.4s, v16.4h, v31.4h\n" + "usubl v2.8h, v2.8b, v18.8b\n" + "smlal v24.4s, v8.4h, v31.4h\n" + "smlal v23.4s, v14.4h, v31.4h\n" + "smlal2 v15.4s, v16.8h, v29.8h\n" + "smlal v7.4s, v4.4h, v21.4h\n" + "smlal2 v5.4s, v16.8h, v31.8h\n" + "ldr d16, [x24, x3]\n" + "smlal2 v22.4s, v8.8h, v31.8h\n" + "usubl v16.8h, v16.8b, v18.8b\n" + "smlal2 v19.4s, v14.8h, v31.8h\n" + "ldr d31, [x6, #0xb0]\n" + "smlal v20.4s, v4.4h, v29.4h\n" + "usubl v31.8h, v31.8b, v13.8b\n" + "smlal v24.4s, v14.4h, v29.4h\n" + "smlal v23.4s, v25.4h, v29.4h\n" + "smlal2 v15.4s, v4.8h, v21.8h\n" + "smlal v7.4s, v17.4h, v3.4h\n" + "smlal2 v5.4s, v4.8h, v29.8h\n" + "ldr d4, [x23, x3]\n" + "smlal2 v22.4s, v14.8h, v29.8h\n" + "usubl v4.8h, v4.8b, v18.8b\n" + "smlal2 v19.4s, v25.8h, v29.8h\n" + "ldr d29, [x6, #0xb8]\n" + "smlal v20.4s, v17.4h, v21.4h\n" + "usubl v29.8h, v29.8b, v13.8b\n" + "smlal v24.4s, v25.4h, v21.4h\n" + "smlal v23.4s, v10.4h, v21.4h\n" + "smlal2 v15.4s, v17.8h, v3.8h\n" + "smlal v7.4s, v6.4h, v27.4h\n" + "smlal2 v5.4s, v17.8h, v21.8h\n" + "ldr d17, [x22, x3]\n" + "smlal2 v22.4s, v25.8h, v21.8h\n" + "usubl v17.8h, v17.8b, v18.8b\n" + "smlal2 v19.4s, v10.8h, v21.8h\n" + "ldr d21, [x6, #0xc0]\n" + "smlal v20.4s, v6.4h, v3.4h\n" + "usubl v21.8h, v21.8b, v13.8b\n" + "smlal v24.4s, v10.4h, v3.4h\n" + "smlal v23.4s, v9.4h, v3.4h\n" + "smlal2 v15.4s, v6.8h, v27.8h\n" + "smlal v7.4s, v8.4h, v1.4h\n" + "smlal2 v5.4s, v6.8h, v3.8h\n" + "ldr d6, [x21, x3]\n" + "smlal2 v22.4s, v10.8h, v3.8h\n" + "usubl v6.8h, v6.8b, v18.8b\n" + "smlal2 v19.4s, v9.8h, v3.8h\n" + "ldr d3, [x20, x3]\n" + "smlal v20.4s, v28.4h, v27.4h\n" + "usubl v3.8h, v3.8b, v18.8b\n" + "smlal v24.4s, v9.4h, v27.4h\n" + "smlal v23.4s, v12.4h, v27.4h\n" + "add x3, x3, #0x8\n" + "smlal2 v15.4s, v8.8h, v1.8h\n" + "ldr q8, [x7, #0x0]\n" + "smlal v7.4s, v14.4h, v30.4h\n" + "smlal2 v5.4s, v28.8h, v27.8h\n" + "ldr q28, [x8, #0x0]\n" + "smlal2 v22.4s, v9.8h, v27.8h\n" + "smlal2 v19.4s, v12.8h, v27.8h\n" + "ldr q27, [x7, #0x10]\n" + "smlal v20.4s, v14.4h, v1.4h\n" + "add x7, x7, #0x20\n" + "smlal v24.4s, v2.4h, v1.4h\n" + "smlal v23.4s, v16.4h, v1.4h\n" + "smlal2 v15.4s, v14.8h, v30.8h\n" + "smlal v7.4s, v25.4h, v31.4h\n" + "smlal2 v5.4s, v14.8h, v1.8h\n" + "ldr q14, [x8, #0x10]\n" + "smlal2 v22.4s, v2.8h, v1.8h\n" + "add x8, x8, #0x20\n" + "smlal2 v19.4s, v16.8h, v1.8h\n" + "smlal v20.4s, v25.4h, v30.4h\n" + "smlal v24.4s, v16.4h, v30.4h\n" + "smlal v23.4s, v4.4h, v30.4h\n" + "smlal2 v15.4s, v25.8h, v31.8h\n" + "smlal v7.4s, v10.4h, v29.4h\n" + "smlal2 v5.4s, v25.8h, v30.8h\n" + "smlal2 v22.4s, v16.8h, v30.8h\n" + "smlal2 v19.4s, v4.8h, v30.8h\n" + "smlal v20.4s, v10.4h, v31.4h\n" + "smlal v24.4s, v4.4h, v31.4h\n" + "smlal v23.4s, v17.4h, v31.4h\n" + "smlal2 v15.4s, v10.8h, v29.8h\n" + "smlal v7.4s, v9.4h, v21.4h\n" + "sqrdmulh v7.4s, v7.4s, v8.4s\n" + "smlal2 v5.4s, v10.8h, v31.8h\n" + "smlal2 v22.4s, v4.8h, v31.8h\n" + "and v4.16b, v7.16b, v28.16b\n" + "smlal2 v19.4s, v17.8h, v31.8h\n" + "smlal v20.4s, v9.4h, v29.4h\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "smlal v24.4s, v17.4h, v29.4h\n" + "smlal v23.4s, v6.4h, v29.4h\n" + "sqadd v7.4s, v7.4s, v4.4s\n" + "smlal2 v15.4s, v9.8h, v21.8h\n" + "smlal2 v5.4s, v9.8h, v29.8h\n" + "sqrdmulh v15.4s, v15.4s, v27.4s\n" + "smlal2 v22.4s, v17.8h, v29.8h\n" + "smlal2 v19.4s, v6.8h, v29.8h\n" + "and v30.16b, v15.16b, v14.16b\n" + "smlal v20.4s, v12.4h, v21.4h\n" + "smlal v24.4s, v6.4h, v21.4h\n" + "sqrdmulh v20.4s, v20.4s, v8.4s\n" + "smlal v23.4s, v3.4h, v21.4h\n" + "smlal2 v5.4s, v12.8h, v21.8h\n" + "sqrdmulh v24.4s, v24.4s, v8.4s\n" + "smlal2 v22.4s, v6.8h, v21.8h\n" + "smlal2 v19.4s, v3.8h, v21.8h\n" + "sqrdmulh v23.4s, v23.4s, v8.4s\n" "sshr v30.4s, v30.4s, #0x1f\n" - "and v16.16b, v21.16b, v22.16b\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "sshr v26.4s, v26.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v18.4s\n" - "sshr v23.4s, v23.4s, #0x1f\n" - "sqadd v17.4s, v17.4s, v30.4s\n" + "and v3.16b, v20.16b, v28.16b\n" + "sqrdmulh v5.4s, v5.4s, v27.4s\n" + "and v25.16b, v24.16b, v28.16b\n" + "sqrdmulh v22.4s, v22.4s, v27.4s\n" + "and v16.16b, v23.16b, v28.16b\n" + "sqrdmulh v19.4s, v19.4s, v27.4s\n" + "sqadd v15.4s, v15.4s, v30.4s\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "and v4.16b, v5.16b, v14.16b\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "and v10.16b, v22.16b, v14.16b\n" "sshr v16.4s, v16.4s, #0x1f\n" - "srshl v13.4s, v13.4s, v6.4s\n" - "srshl v20.4s, v20.4s, v6.4s\n" - "sqadd v10.4s, v10.4s, v26.4s\n" - "srshl v8.4s, v8.4s, v6.4s\n" - "sqadd v7.4s, v7.4s, v23.4s\n" - "srshl v17.4s, v17.4s, v6.4s\n" - "sqadd v21.4s, v21.4s, v16.4s\n" - "srshl v19.4s, v19.4s, v22.4s\n" - "sqxtn v13.4h, v13.4s\n" - "srshl v10.4s, v10.4s, v22.4s\n" + "and v12.16b, v19.16b, v14.16b\n" + "sqadd v20.4s, v20.4s, v3.4s\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sqadd v24.4s, v24.4s, v25.4s\n" + "sshr v10.4s, v10.4s, #0x1f\n" + "sqadd v23.4s, v23.4s, v16.4s\n" + "sshr v12.4s, v12.4s, #0x1f\n" + "srshl v7.4s, v7.4s, v28.4s\n" + "srshl v20.4s, v20.4s, v28.4s\n" + "sqadd v5.4s, v5.4s, v4.4s\n" + "srshl v24.4s, v24.4s, v28.4s\n" + "sqadd v22.4s, v22.4s, v10.4s\n" + "srshl v23.4s, v23.4s, v28.4s\n" + "sqadd v19.4s, v19.4s, v12.4s\n" + "srshl v15.4s, v15.4s, v14.4s\n" + "sqxtn v7.4h, v7.4s\n" + "srshl v5.4s, v5.4s, v14.4s\n" "sqxtn v20.4h, v20.4s\n" - "srshl v7.4s, v7.4s, v22.4s\n" - "sqxtn v8.4h, v8.4s\n" - "srshl v21.4s, v21.4s, v22.4s\n" - "sqxtn v17.4h, v17.4s\n" - "sqxtn2 v13.8h, v19.4s\n" - "sqxtn2 v20.8h, v10.4s\n" - "sqxtn2 v8.8h, v7.4s\n" - "sqxtn2 v17.8h, v21.4s\n" - "sqadd v13.8h, v13.8h, v14.8h\n" - "sqadd v20.8h, v20.8h, v14.8h\n" - "sqadd v8.8h, v8.8h, v14.8h\n" - "sqadd v17.8h, v17.8h, v14.8h\n" - "smax v13.8h, v13.8h, v12.8h\n" - "smax v20.8h, v20.8h, v12.8h\n" - "smax v8.8h, v8.8h, v12.8h\n" - "smax v17.8h, v17.8h, v12.8h\n" - "smin v13.8h, v13.8h, v11.8h\n" - "smin v20.8h, v20.8h, v11.8h\n" - "smin v8.8h, v8.8h, v11.8h\n" - "smin v17.8h, v17.8h, v11.8h\n" - "uzp1 v13.16b, v13.16b, v13.16b\n" + "srshl v22.4s, v22.4s, v14.4s\n" + "sqxtn v24.4h, v24.4s\n" + "srshl v19.4s, v19.4s, v14.4s\n" + "sqxtn v23.4h, v23.4s\n" + "sqxtn2 v7.8h, v15.4s\n" + "sqxtn2 v20.8h, v5.4s\n" + "sqxtn2 v24.8h, v22.4s\n" + "sqxtn2 v23.8h, v19.4s\n" + "sqadd v7.8h, v7.8h, v26.8h\n" + "sqadd v20.8h, v20.8h, v26.8h\n" + "sqadd v24.8h, v24.8h, v26.8h\n" + "sqadd v23.8h, v23.8h, v26.8h\n" + "smax v7.8h, v7.8h, v11.8h\n" + "smax v20.8h, v20.8h, v11.8h\n" + "smax v24.8h, v24.8h, v11.8h\n" + "smax v23.8h, v23.8h, v11.8h\n" + "smin v7.8h, v7.8h, v0.8h\n" + "smin v20.8h, v20.8h, v0.8h\n" + "smin v24.8h, v24.8h, v0.8h\n" + "smin v23.8h, v23.8h, v0.8h\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" + "str d7, [x17, x4]\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "str d13, [x17, x10]\n" - "uzp1 v8.16b, v8.16b, v8.16b\n" - "uzp1 v17.16b, v17.16b, v17.16b\n" - "str d20, [x6, x10]\n" - "str d8, [x7, x10]\n" - "str d17, [x16, x10]\n" - "add x10, x10, #0x8\n" + "uzp1 v24.16b, v24.16b, v24.16b\n" + "str d20, [x16, x4]\n" + "uzp1 v23.16b, v23.16b, v23.16b\n" + "str d24, [x15, x4]\n" + "str d23, [x14, x4]\n" + "add x4, x4, #0x8\n" "beq 124f\n" - "add x3, x3, #0xc8\n" + "add x6, x6, #0xc8\n" "3:" // Oddments - "ldr x13, [%x[params], %[offsetof_Params_bias]]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" "tbz x1, #2, 5f\n" - "ld1 { v13.4s }, [x13], #0x10\n" + "ld1 { v7.4s }, [x20], #0x10\n" "tbz x1, #1, 4f\n" - "ld1 { v19.d }[0], [x13], #0x8\n" + "ld1 { v15.d }[0], [x20], #0x8\n" "tbz x1, #0, 7f\n" - "ld1 { v19.s }[2], [x13]\n" + "ld1 { v15.s }[2], [x20]\n" "b 7f\n" "4:" // Oddments: Load bias: Bit 2: Bit 1: Unset "tbz x1, #0, 7f\n" - "ld1 { v19.s }[0], [x13]\n" + "ld1 { v15.s }[0], [x20]\n" "b 7f\n" "5:" // Oddments: Load bias: Bit 2: Unset "tbz x1, #1, 6f\n" - "ld1 { v13.d }[0], [x13], #0x8\n" + "ld1 { v7.d }[0], [x20], #0x8\n" "tbz x1, #0, 7f\n" - "ld1 { v13.s }[2], [x13]\n" + "ld1 { v7.s }[2], [x20]\n" "b 7f\n" "6:" // Oddments: Load bias: Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 7f\n" - "ld1 { v13.s }[0], [x13]\n" + "ld1 { v7.s }[0], [x20]\n" "7:" // Oddments: Load bias: Bit 2: End - "ldr d0, [x3, #0x0]\n" - "ldr d1, [x3, #0x8]\n" - "mov v20.16b, v13.16b\n" - "mov v10.16b, v19.16b\n" - "ldr d2, [x3, #0x10]\n" - "ldr d3, [x3, #0x18]\n" - "mov v8.16b, v13.16b\n" - "mov v7.16b, v19.16b\n" - "ldr d4, [x3, #0x20]\n" - "ldp x9, x28, [x4, #0x0]\n" - "mov v17.16b, v13.16b\n" - "mov v21.16b, v19.16b\n" - "ldp x27, x26, [x4, #0x10]\n" - "ldp x25, x24, [x4, #0x20]\n" - "usubl v0.8h, v0.8b, v15.8b\n" - "usubl v1.8h, v1.8b, v15.8b\n" - "ldp x23, x22, [x4, #0x30]\n" - "ldp x21, x20, [x4, #0x40]\n" - "usubl v2.8h, v2.8b, v15.8b\n" - "usubl v3.8h, v3.8b, v15.8b\n" - "usubl v4.8h, v4.8b, v15.8b\n" - "add x9, x9, x0\n" - "add x28, x28, x0\n" - "add x27, x27, x0\n" - "add x26, x26, x0\n" - "add x25, x25, x0\n" - "add x24, x24, x0\n" - "add x23, x23, x0\n" - "add x22, x22, x0\n" - "add x21, x21, x0\n" - "add x20, x20, x0\n" + "ldr d6, [x6, #0x0]\n" + "ldr d14, [x6, #0x8]\n" + "mov v20.16b, v7.16b\n" + "mov v5.16b, v15.16b\n" + "ldr d10, [x6, #0x10]\n" + "ldr d21, [x6, #0x18]\n" + "mov v24.16b, v7.16b\n" + "mov v22.16b, v15.16b\n" + "ldr d12, [x6, #0x20]\n" + "ldp x9, x28, [x5, #0x0]\n" + "mov v23.16b, v7.16b\n" + "mov v19.16b, v15.16b\n" + "ldp x27, x26, [x5, #0x10]\n" + "ldp x25, x24, [x5, #0x20]\n" + "usubl v6.8h, v6.8b, v13.8b\n" + "usubl v14.8h, v14.8b, v13.8b\n" + "ldp x23, x22, [x5, #0x30]\n" + "ldp x21, x20, [x5, #0x40]\n" + "usubl v10.8h, v10.8b, v13.8b\n" + "usubl v21.8h, v21.8b, v13.8b\n" + "usubl v12.8h, v12.8b, v13.8b\n" + "add x9, x9, x3\n" + "add x28, x28, x3\n" + "add x27, x27, x3\n" + "add x26, x26, x3\n" + "add x25, x25, x3\n" + "add x24, x24, x3\n" + "add x23, x23, x3\n" + "add x22, x22, x3\n" + "add x21, x21, x3\n" + "add x20, x20, x3\n" "tbz x1, #2, 9f\n" "ld1 { v31.s }[0], [x9], #0x4\n" - "ld1 { v30.s }[0], [x28], #0x4\n" - "ld1 { v29.s }[0], [x27], #0x4\n" - "ld1 { v28.s }[0], [x26], #0x4\n" - "ld1 { v27.s }[0], [x25], #0x4\n" - "ld1 { v23.s }[0], [x24], #0x4\n" + "ld1 { v17.s }[0], [x28], #0x4\n" + "ld1 { v30.s }[0], [x27], #0x4\n" + "ld1 { v16.s }[0], [x26], #0x4\n" + "ld1 { v3.s }[0], [x25], #0x4\n" + "ld1 { v4.s }[0], [x24], #0x4\n" "ld1 { v25.s }[0], [x23], #0x4\n" - "ld1 { v24.s }[0], [x22], #0x4\n" - "ld1 { v26.s }[0], [x21], #0x4\n" - "ld1 { v22.s }[0], [x20], #0x4\n" + "ld1 { v9.s }[0], [x22], #0x4\n" + "ld1 { v29.s }[0], [x21], #0x4\n" + "ld1 { v28.s }[0], [x20], #0x4\n" "tbz x1, #1, 8f\n" "ld1 { v31.h }[2], [x9], #0x2\n" - "ld1 { v30.h }[2], [x28], #0x2\n" - "ld1 { v29.h }[2], [x27], #0x2\n" - "ld1 { v28.h }[2], [x26], #0x2\n" - "ld1 { v27.h }[2], [x25], #0x2\n" - "ld1 { v23.h }[2], [x24], #0x2\n" + "ld1 { v17.h }[2], [x28], #0x2\n" + "ld1 { v30.h }[2], [x27], #0x2\n" + "ld1 { v16.h }[2], [x26], #0x2\n" + "ld1 { v3.h }[2], [x25], #0x2\n" + "ld1 { v4.h }[2], [x24], #0x2\n" "ld1 { v25.h }[2], [x23], #0x2\n" - "ld1 { v24.h }[2], [x22], #0x2\n" - "ld1 { v26.h }[2], [x21], #0x2\n" - "ld1 { v22.h }[2], [x20], #0x2\n" + "ld1 { v9.h }[2], [x22], #0x2\n" + "ld1 { v29.h }[2], [x21], #0x2\n" + "ld1 { v28.h }[2], [x20], #0x2\n" "tbz x1, #0, 11f\n" "ld1 { v31.b }[6], [x9]\n" - "ld1 { v30.b }[6], [x28]\n" - "ld1 { v29.b }[6], [x27]\n" - "ld1 { v28.b }[6], [x26]\n" - "ld1 { v27.b }[6], [x25]\n" - "ld1 { v23.b }[6], [x24]\n" + "ld1 { v17.b }[6], [x28]\n" + "ld1 { v30.b }[6], [x27]\n" + "ld1 { v16.b }[6], [x26]\n" + "ld1 { v3.b }[6], [x25]\n" + "ld1 { v4.b }[6], [x24]\n" "ld1 { v25.b }[6], [x23]\n" - "ld1 { v24.b }[6], [x22]\n" - "ld1 { v26.b }[6], [x21]\n" - "ld1 { v22.b }[6], [x20]\n" + "ld1 { v9.b }[6], [x22]\n" + "ld1 { v29.b }[6], [x21]\n" + "ld1 { v28.b }[6], [x20]\n" "b 11f\n" "8:" // Oddments: Initial loads: Bit 2: Bit 1: Unset "tbz x1, #0, 11f\n" "ld1 { v31.b }[4], [x9]\n" - "ld1 { v30.b }[4], [x28]\n" - "ld1 { v29.b }[4], [x27]\n" - "ld1 { v28.b }[4], [x26]\n" - "ld1 { v27.b }[4], [x25]\n" - "ld1 { v23.b }[4], [x24]\n" + "ld1 { v17.b }[4], [x28]\n" + "ld1 { v30.b }[4], [x27]\n" + "ld1 { v16.b }[4], [x26]\n" + "ld1 { v3.b }[4], [x25]\n" + "ld1 { v4.b }[4], [x24]\n" "ld1 { v25.b }[4], [x23]\n" - "ld1 { v24.b }[4], [x22]\n" - "ld1 { v26.b }[4], [x21]\n" - "ld1 { v22.b }[4], [x20]\n" + "ld1 { v9.b }[4], [x22]\n" + "ld1 { v29.b }[4], [x21]\n" + "ld1 { v28.b }[4], [x20]\n" "b 11f\n" "9:" // Oddments: Initial loads: Bit 2: Unset "tbz x1, #1, 10f\n" "ld1 { v31.h }[0], [x9], #0x2\n" - "ld1 { v30.h }[0], [x28], #0x2\n" - "ld1 { v29.h }[0], [x27], #0x2\n" - "ld1 { v28.h }[0], [x26], #0x2\n" - "ld1 { v27.h }[0], [x25], #0x2\n" - "ld1 { v23.h }[0], [x24], #0x2\n" + "ld1 { v17.h }[0], [x28], #0x2\n" + "ld1 { v30.h }[0], [x27], #0x2\n" + "ld1 { v16.h }[0], [x26], #0x2\n" + "ld1 { v3.h }[0], [x25], #0x2\n" + "ld1 { v4.h }[0], [x24], #0x2\n" "ld1 { v25.h }[0], [x23], #0x2\n" - "ld1 { v24.h }[0], [x22], #0x2\n" - "ld1 { v26.h }[0], [x21], #0x2\n" - "ld1 { v22.h }[0], [x20], #0x2\n" + "ld1 { v9.h }[0], [x22], #0x2\n" + "ld1 { v29.h }[0], [x21], #0x2\n" + "ld1 { v28.h }[0], [x20], #0x2\n" "tbz x1, #0, 11f\n" "ld1 { v31.b }[2], [x9]\n" - "ld1 { v30.b }[2], [x28]\n" - "ld1 { v29.b }[2], [x27]\n" - "ld1 { v28.b }[2], [x26]\n" - "ld1 { v27.b }[2], [x25]\n" - "ld1 { v23.b }[2], [x24]\n" + "ld1 { v17.b }[2], [x28]\n" + "ld1 { v30.b }[2], [x27]\n" + "ld1 { v16.b }[2], [x26]\n" + "ld1 { v3.b }[2], [x25]\n" + "ld1 { v4.b }[2], [x24]\n" "ld1 { v25.b }[2], [x23]\n" - "ld1 { v24.b }[2], [x22]\n" - "ld1 { v26.b }[2], [x21]\n" - "ld1 { v22.b }[2], [x20]\n" + "ld1 { v9.b }[2], [x22]\n" + "ld1 { v29.b }[2], [x21]\n" + "ld1 { v28.b }[2], [x20]\n" "b 11f\n" "10:" // Oddments: Initial loads: Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 11f\n" "ld1 { v31.b }[0], [x9]\n" - "ld1 { v30.b }[0], [x28]\n" - "ld1 { v29.b }[0], [x27]\n" - "ld1 { v28.b }[0], [x26]\n" - "ld1 { v27.b }[0], [x25]\n" - "ld1 { v23.b }[0], [x24]\n" + "ld1 { v17.b }[0], [x28]\n" + "ld1 { v30.b }[0], [x27]\n" + "ld1 { v16.b }[0], [x26]\n" + "ld1 { v3.b }[0], [x25]\n" + "ld1 { v4.b }[0], [x24]\n" "ld1 { v25.b }[0], [x23]\n" - "ld1 { v24.b }[0], [x22]\n" - "ld1 { v26.b }[0], [x21]\n" - "ld1 { v22.b }[0], [x20]\n" + "ld1 { v9.b }[0], [x22]\n" + "ld1 { v29.b }[0], [x21]\n" + "ld1 { v28.b }[0], [x20]\n" "11:" // Oddments: Initial loads: Bit 2: End - "usubl v31.8h, v31.8b, v9.8b\n" - "usubl v30.8h, v30.8b, v9.8b\n" - "smlal v13.4s, v31.4h, v0.4h\n" - "ldr x20, [x4, #0x50]\n" - "usubl v29.8h, v29.8b, v9.8b\n" - "smlal2 v19.4s, v31.8h, v0.8h\n" - "smlal v20.4s, v30.4h, v0.4h\n" - "smlal2 v10.4s, v30.8h, v0.8h\n" - "smlal v8.4s, v29.4h, v0.4h\n" - "usubl v28.8h, v28.8b, v9.8b\n" - "add x20, x20, x0\n" - "smlal2 v7.4s, v29.8h, v0.8h\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal v17.4s, v28.4h, v0.4h\n" - "smlal2 v21.4s, v28.8h, v0.8h\n" - "smlal v13.4s, v30.4h, v1.4h\n" - "usubl v23.8h, v23.8b, v9.8b\n" - "smlal2 v19.4s, v30.8h, v1.8h\n" - "smlal v20.4s, v27.4h, v1.4h\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "smlal2 v10.4s, v27.8h, v1.8h\n" - "smlal v8.4s, v28.4h, v1.4h\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "smlal2 v7.4s, v28.8h, v1.8h\n" - "usubl v26.8h, v26.8b, v9.8b\n" - "smlal v17.4s, v23.4h, v1.4h\n" - "usubl v22.8h, v22.8b, v9.8b\n" - "smlal2 v21.4s, v23.8h, v1.8h\n" - "smlal v13.4s, v27.4h, v2.4h\n" - "smlal2 v19.4s, v27.8h, v2.8h\n" - "smlal v20.4s, v25.4h, v2.4h\n" - "smlal2 v10.4s, v25.8h, v2.8h\n" - "smlal v8.4s, v23.4h, v2.4h\n" - "smlal2 v7.4s, v23.8h, v2.8h\n" + "usubl v31.8h, v31.8b, v18.8b\n" + "usubl v17.8h, v17.8b, v18.8b\n" + "smlal v7.4s, v31.4h, v6.4h\n" + "ldr x20, [x5, #0x50]\n" + "usubl v30.8h, v30.8b, v18.8b\n" + "smlal2 v15.4s, v31.8h, v6.8h\n" + "smlal v20.4s, v17.4h, v6.4h\n" + "smlal2 v5.4s, v17.8h, v6.8h\n" + "smlal v24.4s, v30.4h, v6.4h\n" + "usubl v16.8h, v16.8b, v18.8b\n" + "add x20, x20, x3\n" + "smlal2 v22.4s, v30.8h, v6.8h\n" + "usubl v3.8h, v3.8b, v18.8b\n" + "smlal v23.4s, v16.4h, v6.4h\n" + "smlal2 v19.4s, v16.8h, v6.8h\n" + "smlal v7.4s, v17.4h, v14.4h\n" + "usubl v4.8h, v4.8b, v18.8b\n" + "smlal2 v15.4s, v17.8h, v14.8h\n" + "smlal v20.4s, v3.4h, v14.4h\n" + "usubl v25.8h, v25.8b, v18.8b\n" + "smlal2 v5.4s, v3.8h, v14.8h\n" + "smlal v24.4s, v16.4h, v14.4h\n" + "usubl v9.8h, v9.8b, v18.8b\n" + "smlal2 v22.4s, v16.8h, v14.8h\n" + "usubl v29.8h, v29.8b, v18.8b\n" + "smlal v23.4s, v4.4h, v14.4h\n" + "usubl v28.8h, v28.8b, v18.8b\n" + "smlal2 v19.4s, v4.8h, v14.8h\n" + "smlal v7.4s, v3.4h, v10.4h\n" + "smlal2 v15.4s, v3.8h, v10.8h\n" + "smlal v20.4s, v25.4h, v10.4h\n" + "smlal2 v5.4s, v25.8h, v10.8h\n" + "smlal v24.4s, v4.4h, v10.4h\n" + "smlal2 v22.4s, v4.8h, v10.8h\n" "tbz x1, #2, 13f\n" - "ld1 { v31.s }[0], [x20], #0x4\n" + "ld1 { v27.s }[0], [x20], #0x4\n" "tbz x1, #1, 12f\n" - "ld1 { v31.h }[2], [x20], #0x2\n" + "ld1 { v27.h }[2], [x20], #0x2\n" "tbz x1, #0, 15f\n" - "ld1 { v31.b }[6], [x20]\n" + "ld1 { v27.b }[6], [x20]\n" "b 15f\n" "12:" // Oddments: Load (1, 3): Bit 2: Bit 1: Unset "tbz x1, #0, 15f\n" - "ld1 { v31.b }[4], [x20]\n" + "ld1 { v27.b }[4], [x20]\n" "b 15f\n" "13:" // Oddments: Load (1, 3): Bit 2: Unset "tbz x1, #1, 14f\n" - "ld1 { v31.h }[0], [x20], #0x2\n" + "ld1 { v27.h }[0], [x20], #0x2\n" "tbz x1, #0, 15f\n" - "ld1 { v31.b }[2], [x20]\n" + "ld1 { v27.b }[2], [x20]\n" "b 15f\n" "14:" // Oddments: Load (1, 3): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 15f\n" - "ld1 { v31.b }[0], [x20]\n" + "ld1 { v27.b }[0], [x20]\n" "15:" // Oddments: Load (1, 3): Bit 2: End - "usubl v31.8h, v31.8b, v9.8b\n" - "ldr x22, [x4, #0x58]\n" - "smlal v17.4s, v31.4h, v2.4h\n" - "smlal2 v21.4s, v31.8h, v2.8h\n" - "smlal v13.4s, v25.4h, v3.4h\n" - "smlal2 v19.4s, v25.8h, v3.8h\n" - "add x22, x22, x0\n" - "smlal v20.4s, v24.4h, v3.4h\n" - "smlal2 v10.4s, v24.8h, v3.8h\n" - "smlal v8.4s, v31.4h, v3.4h\n" - "smlal2 v7.4s, v31.8h, v3.8h\n" + "usubl v27.8h, v27.8b, v18.8b\n" + "ldr x20, [x5, #0x58]\n" + "smlal v23.4s, v27.4h, v10.4h\n" + "smlal2 v19.4s, v27.8h, v10.8h\n" + "smlal v7.4s, v25.4h, v21.4h\n" + "smlal2 v15.4s, v25.8h, v21.8h\n" + "add x20, x20, x3\n" + "smlal v20.4s, v9.4h, v21.4h\n" + "smlal2 v5.4s, v9.8h, v21.8h\n" + "smlal v24.4s, v27.4h, v21.4h\n" + "smlal2 v22.4s, v27.8h, v21.8h\n" "tbz x1, #2, 17f\n" - "ld1 { v30.s }[0], [x22], #0x4\n" + "ld1 { v6.s }[0], [x20], #0x4\n" "tbz x1, #1, 16f\n" - "ld1 { v30.h }[2], [x22], #0x2\n" + "ld1 { v6.h }[2], [x20], #0x2\n" "tbz x1, #0, 19f\n" - "ld1 { v30.b }[6], [x22]\n" + "ld1 { v6.b }[6], [x20]\n" "b 19f\n" "16:" // Oddments: Load (1, 4): Bit 2: Bit 1: Unset "tbz x1, #0, 19f\n" - "ld1 { v30.b }[4], [x22]\n" + "ld1 { v6.b }[4], [x20]\n" "b 19f\n" "17:" // Oddments: Load (1, 4): Bit 2: Unset "tbz x1, #1, 18f\n" - "ld1 { v30.h }[0], [x22], #0x2\n" + "ld1 { v6.h }[0], [x20], #0x2\n" "tbz x1, #0, 19f\n" - "ld1 { v30.b }[2], [x22]\n" + "ld1 { v6.b }[2], [x20]\n" "b 19f\n" "18:" // Oddments: Load (1, 4): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 19f\n" - "ld1 { v30.b }[0], [x22]\n" + "ld1 { v6.b }[0], [x20]\n" "19:" // Oddments: Load (1, 4): Bit 2: End - "usubl v30.8h, v30.8b, v9.8b\n" - "ldr x21, [x4, #0x60]\n" - "smlal v17.4s, v30.4h, v3.4h\n" - "smlal2 v21.4s, v30.8h, v3.8h\n" - "smlal v13.4s, v24.4h, v4.4h\n" - "smlal2 v19.4s, v24.8h, v4.8h\n" - "add x21, x21, x0\n" + "usubl v6.8h, v6.8b, v18.8b\n" + "ldr x20, [x5, #0x60]\n" + "smlal v23.4s, v6.4h, v21.4h\n" + "smlal2 v19.4s, v6.8h, v21.8h\n" + "smlal v7.4s, v9.4h, v12.4h\n" + "smlal2 v15.4s, v9.8h, v12.8h\n" + "add x20, x20, x3\n" "tbz x1, #2, 21f\n" - "ld1 { v27.s }[0], [x21], #0x4\n" + "ld1 { v9.s }[0], [x20], #0x4\n" "tbz x1, #1, 20f\n" - "ld1 { v27.h }[2], [x21], #0x2\n" + "ld1 { v9.h }[2], [x20], #0x2\n" "tbz x1, #0, 23f\n" - "ld1 { v27.b }[6], [x21]\n" + "ld1 { v9.b }[6], [x20]\n" "b 23f\n" "20:" // Oddments: Load (0, 5): Bit 2: Bit 1: Unset "tbz x1, #0, 23f\n" - "ld1 { v27.b }[4], [x21]\n" + "ld1 { v9.b }[4], [x20]\n" "b 23f\n" "21:" // Oddments: Load (0, 5): Bit 2: Unset "tbz x1, #1, 22f\n" - "ld1 { v27.h }[0], [x21], #0x2\n" + "ld1 { v9.h }[0], [x20], #0x2\n" "tbz x1, #0, 23f\n" - "ld1 { v27.b }[2], [x21]\n" + "ld1 { v9.b }[2], [x20]\n" "b 23f\n" "22:" // Oddments: Load (0, 5): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 23f\n" - "ld1 { v27.b }[0], [x21]\n" + "ld1 { v9.b }[0], [x20]\n" "23:" // Oddments: Load (0, 5): Bit 2: End - "ldr d0, [x3, #0x28]\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal v20.4s, v27.4h, v4.4h\n" - "smlal2 v10.4s, v27.8h, v4.8h\n" - "smlal v8.4s, v30.4h, v4.4h\n" - "smlal2 v7.4s, v30.8h, v4.8h\n" - "usubl v0.8h, v0.8b, v15.8b\n" - "ldr x20, [x4, #0x68]\n" - "smlal v17.4s, v26.4h, v4.4h\n" - "smlal2 v21.4s, v26.8h, v4.8h\n" - "add x20, x20, x0\n" - "smlal v13.4s, v29.4h, v0.4h\n" - "smlal2 v19.4s, v29.8h, v0.8h\n" - "smlal v20.4s, v28.4h, v0.4h\n" - "smlal2 v10.4s, v28.8h, v0.8h\n" - "smlal v8.4s, v22.4h, v0.4h\n" - "smlal2 v7.4s, v22.8h, v0.8h\n" + "ldr d14, [x6, #0x28]\n" + "usubl v9.8h, v9.8b, v18.8b\n" + "smlal v20.4s, v9.4h, v12.4h\n" + "smlal2 v5.4s, v9.8h, v12.8h\n" + "smlal v24.4s, v6.4h, v12.4h\n" + "smlal2 v22.4s, v6.8h, v12.8h\n" + "usubl v14.8h, v14.8b, v13.8b\n" + "ldr x20, [x5, #0x68]\n" + "smlal v23.4s, v29.4h, v12.4h\n" + "smlal2 v19.4s, v29.8h, v12.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v30.4h, v14.4h\n" + "smlal2 v15.4s, v30.8h, v14.8h\n" + "smlal v20.4s, v16.4h, v14.4h\n" + "smlal2 v5.4s, v16.8h, v14.8h\n" + "smlal v24.4s, v28.4h, v14.4h\n" + "smlal2 v22.4s, v28.8h, v14.8h\n" "tbz x1, #2, 25f\n" "ld1 { v25.s }[0], [x20], #0x4\n" "tbz x1, #1, 24f\n" @@ -1315,869 +1315,869 @@ void a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( "tbz x1, #0, 27f\n" "ld1 { v25.b }[0], [x20]\n" "27:" // Oddments: Load (2, 1): Bit 2: End - "ldr d1, [x3, #0x30]\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "usubl v1.8h, v1.8b, v15.8b\n" - "ldr x26, [x4, #0x70]\n" - "smlal v17.4s, v25.4h, v0.4h\n" - "smlal2 v21.4s, v25.8h, v0.8h\n" - "add x26, x26, x0\n" - "smlal v13.4s, v28.4h, v1.4h\n" - "smlal2 v19.4s, v28.8h, v1.8h\n" - "smlal v20.4s, v23.4h, v1.4h\n" - "smlal2 v10.4s, v23.8h, v1.8h\n" - "smlal v8.4s, v25.4h, v1.4h\n" - "smlal2 v7.4s, v25.8h, v1.8h\n" + "ldr d21, [x6, #0x30]\n" + "usubl v25.8h, v25.8b, v18.8b\n" + "usubl v21.8h, v21.8b, v13.8b\n" + "ldr x20, [x5, #0x70]\n" + "smlal v23.4s, v25.4h, v14.4h\n" + "smlal2 v19.4s, v25.8h, v14.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v16.4h, v21.4h\n" + "smlal2 v15.4s, v16.8h, v21.8h\n" + "smlal v20.4s, v4.4h, v21.4h\n" + "smlal2 v5.4s, v4.8h, v21.8h\n" + "smlal v24.4s, v25.4h, v21.4h\n" + "smlal2 v22.4s, v25.8h, v21.8h\n" "tbz x1, #2, 29f\n" - "ld1 { v24.s }[0], [x26], #0x4\n" + "ld1 { v10.s }[0], [x20], #0x4\n" "tbz x1, #1, 28f\n" - "ld1 { v24.h }[2], [x26], #0x2\n" + "ld1 { v10.h }[2], [x20], #0x2\n" "tbz x1, #0, 31f\n" - "ld1 { v24.b }[6], [x26]\n" + "ld1 { v10.b }[6], [x20]\n" "b 31f\n" "28:" // Oddments: Load (2, 2): Bit 2: Bit 1: Unset "tbz x1, #0, 31f\n" - "ld1 { v24.b }[4], [x26]\n" + "ld1 { v10.b }[4], [x20]\n" "b 31f\n" "29:" // Oddments: Load (2, 2): Bit 2: Unset "tbz x1, #1, 30f\n" - "ld1 { v24.h }[0], [x26], #0x2\n" + "ld1 { v10.h }[0], [x20], #0x2\n" "tbz x1, #0, 31f\n" - "ld1 { v24.b }[2], [x26]\n" + "ld1 { v10.b }[2], [x20]\n" "b 31f\n" "30:" // Oddments: Load (2, 2): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 31f\n" - "ld1 { v24.b }[0], [x26]\n" + "ld1 { v10.b }[0], [x20]\n" "31:" // Oddments: Load (2, 2): Bit 2: End - "ldr d2, [x3, #0x38]\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "usubl v2.8h, v2.8b, v15.8b\n" - "ldr x25, [x4, #0x78]\n" - "smlal v17.4s, v24.4h, v1.4h\n" - "smlal2 v21.4s, v24.8h, v1.8h\n" - "add x25, x25, x0\n" - "smlal v13.4s, v23.4h, v2.4h\n" - "smlal2 v19.4s, v23.8h, v2.8h\n" - "smlal v20.4s, v31.4h, v2.4h\n" - "smlal2 v10.4s, v31.8h, v2.8h\n" - "smlal v8.4s, v24.4h, v2.4h\n" - "smlal2 v7.4s, v24.8h, v2.8h\n" + "ldr d9, [x6, #0x38]\n" + "usubl v10.8h, v10.8b, v18.8b\n" + "usubl v9.8h, v9.8b, v13.8b\n" + "ldr x20, [x5, #0x78]\n" + "smlal v23.4s, v10.4h, v21.4h\n" + "smlal2 v19.4s, v10.8h, v21.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v4.4h, v9.4h\n" + "smlal2 v15.4s, v4.8h, v9.8h\n" + "smlal v20.4s, v27.4h, v9.4h\n" + "smlal2 v5.4s, v27.8h, v9.8h\n" + "smlal v24.4s, v10.4h, v9.4h\n" + "smlal2 v22.4s, v10.8h, v9.8h\n" "tbz x1, #2, 33f\n" - "ld1 { v27.s }[0], [x25], #0x4\n" + "ld1 { v12.s }[0], [x20], #0x4\n" "tbz x1, #1, 32f\n" - "ld1 { v27.h }[2], [x25], #0x2\n" + "ld1 { v12.h }[2], [x20], #0x2\n" "tbz x1, #0, 35f\n" - "ld1 { v27.b }[6], [x25]\n" + "ld1 { v12.b }[6], [x20]\n" "b 35f\n" "32:" // Oddments: Load (2, 3): Bit 2: Bit 1: Unset "tbz x1, #0, 35f\n" - "ld1 { v27.b }[4], [x25]\n" + "ld1 { v12.b }[4], [x20]\n" "b 35f\n" "33:" // Oddments: Load (2, 3): Bit 2: Unset "tbz x1, #1, 34f\n" - "ld1 { v27.h }[0], [x25], #0x2\n" + "ld1 { v12.h }[0], [x20], #0x2\n" "tbz x1, #0, 35f\n" - "ld1 { v27.b }[2], [x25]\n" + "ld1 { v12.b }[2], [x20]\n" "b 35f\n" "34:" // Oddments: Load (2, 3): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 35f\n" - "ld1 { v27.b }[0], [x25]\n" + "ld1 { v12.b }[0], [x20]\n" "35:" // Oddments: Load (2, 3): Bit 2: End - "ldr d3, [x3, #0x40]\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "usubl v3.8h, v3.8b, v15.8b\n" - "ldr x23, [x4, #0x80]\n" - "smlal v17.4s, v27.4h, v2.4h\n" - "smlal2 v21.4s, v27.8h, v2.8h\n" - "add x23, x23, x0\n" - "smlal v13.4s, v31.4h, v3.4h\n" - "smlal2 v19.4s, v31.8h, v3.8h\n" - "smlal v20.4s, v30.4h, v3.4h\n" - "smlal2 v10.4s, v30.8h, v3.8h\n" - "smlal v8.4s, v27.4h, v3.4h\n" - "smlal2 v7.4s, v27.8h, v3.8h\n" + "ldr d31, [x6, #0x40]\n" + "usubl v12.8h, v12.8b, v18.8b\n" + "usubl v31.8h, v31.8b, v13.8b\n" + "ldr x20, [x5, #0x80]\n" + "smlal v23.4s, v12.4h, v9.4h\n" + "smlal2 v19.4s, v12.8h, v9.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v27.4h, v31.4h\n" + "smlal2 v15.4s, v27.8h, v31.8h\n" + "smlal v20.4s, v6.4h, v31.4h\n" + "smlal2 v5.4s, v6.8h, v31.8h\n" + "smlal v24.4s, v12.4h, v31.4h\n" + "smlal2 v22.4s, v12.8h, v31.8h\n" "tbz x1, #2, 37f\n" - "ld1 { v23.s }[0], [x23], #0x4\n" + "ld1 { v8.s }[0], [x20], #0x4\n" "tbz x1, #1, 36f\n" - "ld1 { v23.h }[2], [x23], #0x2\n" + "ld1 { v8.h }[2], [x20], #0x2\n" "tbz x1, #0, 39f\n" - "ld1 { v23.b }[6], [x23]\n" + "ld1 { v8.b }[6], [x20]\n" "b 39f\n" "36:" // Oddments: Load (2, 4): Bit 2: Bit 1: Unset "tbz x1, #0, 39f\n" - "ld1 { v23.b }[4], [x23]\n" + "ld1 { v8.b }[4], [x20]\n" "b 39f\n" "37:" // Oddments: Load (2, 4): Bit 2: Unset "tbz x1, #1, 38f\n" - "ld1 { v23.h }[0], [x23], #0x2\n" + "ld1 { v8.h }[0], [x20], #0x2\n" "tbz x1, #0, 39f\n" - "ld1 { v23.b }[2], [x23]\n" + "ld1 { v8.b }[2], [x20]\n" "b 39f\n" "38:" // Oddments: Load (2, 4): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 39f\n" - "ld1 { v23.b }[0], [x23]\n" + "ld1 { v8.b }[0], [x20]\n" "39:" // Oddments: Load (2, 4): Bit 2: End - "ldr d4, [x3, #0x48]\n" - "usubl v23.8h, v23.8b, v9.8b\n" - "usubl v4.8h, v4.8b, v15.8b\n" - "ldr x24, [x4, #0x88]\n" - "smlal v17.4s, v23.4h, v3.4h\n" - "smlal2 v21.4s, v23.8h, v3.8h\n" - "add x24, x24, x0\n" - "smlal v13.4s, v30.4h, v4.4h\n" - "smlal2 v19.4s, v30.8h, v4.8h\n" - "smlal v20.4s, v26.4h, v4.4h\n" - "smlal2 v10.4s, v26.8h, v4.8h\n" - "smlal v8.4s, v23.4h, v4.4h\n" - "smlal2 v7.4s, v23.8h, v4.8h\n" + "ldr d16, [x6, #0x48]\n" + "usubl v8.8h, v8.8b, v18.8b\n" + "usubl v16.8h, v16.8b, v13.8b\n" + "ldr x20, [x5, #0x88]\n" + "smlal v23.4s, v8.4h, v31.4h\n" + "smlal2 v19.4s, v8.8h, v31.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v6.4h, v16.4h\n" + "smlal2 v15.4s, v6.8h, v16.8h\n" + "smlal v20.4s, v29.4h, v16.4h\n" + "smlal2 v5.4s, v29.8h, v16.8h\n" + "smlal v24.4s, v8.4h, v16.4h\n" + "smlal2 v22.4s, v8.8h, v16.8h\n" "tbz x1, #2, 41f\n" - "ld1 { v28.s }[0], [x24], #0x4\n" + "ld1 { v27.s }[0], [x20], #0x4\n" "tbz x1, #1, 40f\n" - "ld1 { v28.h }[2], [x24], #0x2\n" + "ld1 { v27.h }[2], [x20], #0x2\n" "tbz x1, #0, 43f\n" - "ld1 { v28.b }[6], [x24]\n" + "ld1 { v27.b }[6], [x20]\n" "b 43f\n" "40:" // Oddments: Load (2, 5): Bit 2: Bit 1: Unset "tbz x1, #0, 43f\n" - "ld1 { v28.b }[4], [x24]\n" + "ld1 { v27.b }[4], [x20]\n" "b 43f\n" "41:" // Oddments: Load (2, 5): Bit 2: Unset "tbz x1, #1, 42f\n" - "ld1 { v28.h }[0], [x24], #0x2\n" + "ld1 { v27.h }[0], [x20], #0x2\n" "tbz x1, #0, 43f\n" - "ld1 { v28.b }[2], [x24]\n" + "ld1 { v27.b }[2], [x20]\n" "b 43f\n" "42:" // Oddments: Load (2, 5): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 43f\n" - "ld1 { v28.b }[0], [x24]\n" + "ld1 { v27.b }[0], [x20]\n" "43:" // Oddments: Load (2, 5): Bit 2: End - "ldr d0, [x3, #0x50]\n" - "usubl v28.8h, v28.8b, v9.8b\n" - "usubl v0.8h, v0.8b, v15.8b\n" - "ldr x15, [x4, #0x90]\n" - "smlal v17.4s, v28.4h, v4.4h\n" - "smlal2 v21.4s, v28.8h, v4.8h\n" - "add x15, x15, x0\n" - "smlal v13.4s, v22.4h, v0.4h\n" - "smlal2 v19.4s, v22.8h, v0.8h\n" - "smlal v20.4s, v25.4h, v0.4h\n" - "smlal2 v10.4s, v25.8h, v0.8h\n" + "ldr d21, [x6, #0x50]\n" + "usubl v27.8h, v27.8b, v18.8b\n" + "usubl v21.8h, v21.8b, v13.8b\n" + "ldr x20, [x5, #0x90]\n" + "smlal v23.4s, v27.4h, v16.4h\n" + "smlal2 v19.4s, v27.8h, v16.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v28.4h, v21.4h\n" + "smlal2 v15.4s, v28.8h, v21.8h\n" + "smlal v20.4s, v25.4h, v21.4h\n" + "smlal2 v5.4s, v25.8h, v21.8h\n" "tbz x1, #2, 45f\n" - "ld1 { v31.s }[0], [x15], #0x4\n" + "ld1 { v31.s }[0], [x20], #0x4\n" "tbz x1, #1, 44f\n" - "ld1 { v31.h }[2], [x15], #0x2\n" + "ld1 { v31.h }[2], [x20], #0x2\n" "tbz x1, #0, 47f\n" - "ld1 { v31.b }[6], [x15]\n" + "ld1 { v31.b }[6], [x20]\n" "b 47f\n" "44:" // Oddments: Load (3, 0): Bit 2: Bit 1: Unset "tbz x1, #0, 47f\n" - "ld1 { v31.b }[4], [x15]\n" + "ld1 { v31.b }[4], [x20]\n" "b 47f\n" "45:" // Oddments: Load (3, 0): Bit 2: Unset "tbz x1, #1, 46f\n" - "ld1 { v31.h }[0], [x15], #0x2\n" + "ld1 { v31.h }[0], [x20], #0x2\n" "tbz x1, #0, 47f\n" - "ld1 { v31.b }[2], [x15]\n" + "ld1 { v31.b }[2], [x20]\n" "b 47f\n" "46:" // Oddments: Load (3, 0): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 47f\n" - "ld1 { v31.b }[0], [x15]\n" + "ld1 { v31.b }[0], [x20]\n" "47:" // Oddments: Load (3, 0): Bit 2: End - "usubl v31.8h, v31.8b, v9.8b\n" - "ldr x21, [x4, #0x98]\n" - "smlal v8.4s, v31.4h, v0.4h\n" - "smlal2 v7.4s, v31.8h, v0.8h\n" - "add x21, x21, x0\n" + "usubl v31.8h, v31.8b, v18.8b\n" + "ldr x20, [x5, #0x98]\n" + "smlal v24.4s, v31.4h, v21.4h\n" + "smlal2 v22.4s, v31.8h, v21.8h\n" + "add x20, x20, x3\n" "tbz x1, #2, 49f\n" - "ld1 { v30.s }[0], [x21], #0x4\n" + "ld1 { v28.s }[0], [x20], #0x4\n" "tbz x1, #1, 48f\n" - "ld1 { v30.h }[2], [x21], #0x2\n" + "ld1 { v28.h }[2], [x20], #0x2\n" "tbz x1, #0, 51f\n" - "ld1 { v30.b }[6], [x21]\n" + "ld1 { v28.b }[6], [x20]\n" "b 51f\n" "48:" // Oddments: Load (3, 1): Bit 2: Bit 1: Unset "tbz x1, #0, 51f\n" - "ld1 { v30.b }[4], [x21]\n" + "ld1 { v28.b }[4], [x20]\n" "b 51f\n" "49:" // Oddments: Load (3, 1): Bit 2: Unset "tbz x1, #1, 50f\n" - "ld1 { v30.h }[0], [x21], #0x2\n" + "ld1 { v28.h }[0], [x20], #0x2\n" "tbz x1, #0, 51f\n" - "ld1 { v30.b }[2], [x21]\n" + "ld1 { v28.b }[2], [x20]\n" "b 51f\n" "50:" // Oddments: Load (3, 1): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 51f\n" - "ld1 { v30.b }[0], [x21]\n" + "ld1 { v28.b }[0], [x20]\n" "51:" // Oddments: Load (3, 1): Bit 2: End - "ldr d1, [x3, #0x58]\n" - "usubl v30.8h, v30.8b, v9.8b\n" - "usubl v1.8h, v1.8b, v15.8b\n" - "ldr x14, [x4, #0xa0]\n" - "smlal v17.4s, v30.4h, v0.4h\n" - "smlal2 v21.4s, v30.8h, v0.8h\n" - "add x14, x14, x0\n" - "smlal v13.4s, v25.4h, v1.4h\n" - "smlal2 v19.4s, v25.8h, v1.8h\n" - "smlal v20.4s, v24.4h, v1.4h\n" - "smlal2 v10.4s, v24.8h, v1.8h\n" - "smlal v8.4s, v30.4h, v1.4h\n" - "smlal2 v7.4s, v30.8h, v1.8h\n" + "ldr d2, [x6, #0x58]\n" + "usubl v28.8h, v28.8b, v18.8b\n" + "usubl v2.8h, v2.8b, v13.8b\n" + "ldr x20, [x5, #0xa0]\n" + "smlal v23.4s, v28.4h, v21.4h\n" + "smlal2 v19.4s, v28.8h, v21.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v25.4h, v2.4h\n" + "smlal2 v15.4s, v25.8h, v2.8h\n" + "smlal v20.4s, v10.4h, v2.4h\n" + "smlal2 v5.4s, v10.8h, v2.8h\n" + "smlal v24.4s, v28.4h, v2.4h\n" + "smlal2 v22.4s, v28.8h, v2.8h\n" "tbz x1, #2, 53f\n" - "ld1 { v26.s }[0], [x14], #0x4\n" + "ld1 { v21.s }[0], [x20], #0x4\n" "tbz x1, #1, 52f\n" - "ld1 { v26.h }[2], [x14], #0x2\n" + "ld1 { v21.h }[2], [x20], #0x2\n" "tbz x1, #0, 55f\n" - "ld1 { v26.b }[6], [x14]\n" + "ld1 { v21.b }[6], [x20]\n" "b 55f\n" "52:" // Oddments: Load (3, 2): Bit 2: Bit 1: Unset "tbz x1, #0, 55f\n" - "ld1 { v26.b }[4], [x14]\n" + "ld1 { v21.b }[4], [x20]\n" "b 55f\n" "53:" // Oddments: Load (3, 2): Bit 2: Unset "tbz x1, #1, 54f\n" - "ld1 { v26.h }[0], [x14], #0x2\n" + "ld1 { v21.h }[0], [x20], #0x2\n" "tbz x1, #0, 55f\n" - "ld1 { v26.b }[2], [x14]\n" + "ld1 { v21.b }[2], [x20]\n" "b 55f\n" "54:" // Oddments: Load (3, 2): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 55f\n" - "ld1 { v26.b }[0], [x14]\n" + "ld1 { v21.b }[0], [x20]\n" "55:" // Oddments: Load (3, 2): Bit 2: End - "ldr d2, [x3, #0x60]\n" - "usubl v26.8h, v26.8b, v9.8b\n" - "usubl v2.8h, v2.8b, v15.8b\n" - "ldr x13, [x4, #0xa8]\n" - "smlal v17.4s, v26.4h, v1.4h\n" - "smlal2 v21.4s, v26.8h, v1.8h\n" - "add x13, x13, x0\n" - "smlal v13.4s, v24.4h, v2.4h\n" - "smlal2 v19.4s, v24.8h, v2.8h\n" - "smlal v20.4s, v27.4h, v2.4h\n" - "smlal2 v10.4s, v27.8h, v2.8h\n" - "smlal v8.4s, v26.4h, v2.4h\n" - "smlal2 v7.4s, v26.8h, v2.8h\n" + "ldr d25, [x6, #0x60]\n" + "usubl v21.8h, v21.8b, v18.8b\n" + "usubl v25.8h, v25.8b, v13.8b\n" + "ldr x20, [x5, #0xa8]\n" + "smlal v23.4s, v21.4h, v2.4h\n" + "smlal2 v19.4s, v21.8h, v2.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v10.4h, v25.4h\n" + "smlal2 v15.4s, v10.8h, v25.8h\n" + "smlal v20.4s, v12.4h, v25.4h\n" + "smlal2 v5.4s, v12.8h, v25.8h\n" + "smlal v24.4s, v21.4h, v25.4h\n" + "smlal2 v22.4s, v21.8h, v25.8h\n" "tbz x1, #2, 57f\n" - "ld1 { v25.s }[0], [x13], #0x4\n" + "ld1 { v9.s }[0], [x20], #0x4\n" "tbz x1, #1, 56f\n" - "ld1 { v25.h }[2], [x13], #0x2\n" + "ld1 { v9.h }[2], [x20], #0x2\n" "tbz x1, #0, 59f\n" - "ld1 { v25.b }[6], [x13]\n" + "ld1 { v9.b }[6], [x20]\n" "b 59f\n" "56:" // Oddments: Load (3, 3): Bit 2: Bit 1: Unset "tbz x1, #0, 59f\n" - "ld1 { v25.b }[4], [x13]\n" + "ld1 { v9.b }[4], [x20]\n" "b 59f\n" "57:" // Oddments: Load (3, 3): Bit 2: Unset "tbz x1, #1, 58f\n" - "ld1 { v25.h }[0], [x13], #0x2\n" + "ld1 { v9.h }[0], [x20], #0x2\n" "tbz x1, #0, 59f\n" - "ld1 { v25.b }[2], [x13]\n" + "ld1 { v9.b }[2], [x20]\n" "b 59f\n" "58:" // Oddments: Load (3, 3): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 59f\n" - "ld1 { v25.b }[0], [x13]\n" + "ld1 { v9.b }[0], [x20]\n" "59:" // Oddments: Load (3, 3): Bit 2: End - "ldr d3, [x3, #0x68]\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "usubl v3.8h, v3.8b, v15.8b\n" - "ldr x12, [x4, #0xb0]\n" - "smlal v17.4s, v25.4h, v2.4h\n" - "smlal2 v21.4s, v25.8h, v2.8h\n" - "add x12, x12, x0\n" - "smlal v13.4s, v27.4h, v3.4h\n" - "smlal2 v19.4s, v27.8h, v3.8h\n" - "smlal v20.4s, v23.4h, v3.4h\n" - "smlal2 v10.4s, v23.8h, v3.8h\n" - "smlal v8.4s, v25.4h, v3.4h\n" - "smlal2 v7.4s, v25.8h, v3.8h\n" + "ldr d1, [x6, #0x68]\n" + "usubl v9.8h, v9.8b, v18.8b\n" + "usubl v1.8h, v1.8b, v13.8b\n" + "ldr x20, [x5, #0xb0]\n" + "smlal v23.4s, v9.4h, v25.4h\n" + "smlal2 v19.4s, v9.8h, v25.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v12.4h, v1.4h\n" + "smlal2 v15.4s, v12.8h, v1.8h\n" + "smlal v20.4s, v8.4h, v1.4h\n" + "smlal2 v5.4s, v8.8h, v1.8h\n" + "smlal v24.4s, v9.4h, v1.4h\n" + "smlal2 v22.4s, v9.8h, v1.8h\n" "tbz x1, #2, 61f\n" - "ld1 { v24.s }[0], [x12], #0x4\n" + "ld1 { v3.s }[0], [x20], #0x4\n" "tbz x1, #1, 60f\n" - "ld1 { v24.h }[2], [x12], #0x2\n" + "ld1 { v3.h }[2], [x20], #0x2\n" "tbz x1, #0, 63f\n" - "ld1 { v24.b }[6], [x12]\n" + "ld1 { v3.b }[6], [x20]\n" "b 63f\n" "60:" // Oddments: Load (3, 4): Bit 2: Bit 1: Unset "tbz x1, #0, 63f\n" - "ld1 { v24.b }[4], [x12]\n" + "ld1 { v3.b }[4], [x20]\n" "b 63f\n" "61:" // Oddments: Load (3, 4): Bit 2: Unset "tbz x1, #1, 62f\n" - "ld1 { v24.h }[0], [x12], #0x2\n" + "ld1 { v3.h }[0], [x20], #0x2\n" "tbz x1, #0, 63f\n" - "ld1 { v24.b }[2], [x12]\n" + "ld1 { v3.b }[2], [x20]\n" "b 63f\n" "62:" // Oddments: Load (3, 4): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 63f\n" - "ld1 { v24.b }[0], [x12]\n" + "ld1 { v3.b }[0], [x20]\n" "63:" // Oddments: Load (3, 4): Bit 2: End - "ldr d4, [x3, #0x70]\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "usubl v4.8h, v4.8b, v15.8b\n" - "ldr x20, [x4, #0xb8]\n" - "smlal v17.4s, v24.4h, v3.4h\n" - "smlal2 v21.4s, v24.8h, v3.8h\n" - "add x20, x20, x0\n" - "smlal v13.4s, v23.4h, v4.4h\n" - "smlal2 v19.4s, v23.8h, v4.8h\n" - "smlal v20.4s, v28.4h, v4.4h\n" - "smlal2 v10.4s, v28.8h, v4.8h\n" - "smlal v8.4s, v24.4h, v4.4h\n" - "smlal2 v7.4s, v24.8h, v4.8h\n" + "ldr d16, [x6, #0x70]\n" + "usubl v3.8h, v3.8b, v18.8b\n" + "usubl v16.8h, v16.8b, v13.8b\n" + "ldr x20, [x5, #0xb8]\n" + "smlal v23.4s, v3.4h, v1.4h\n" + "smlal2 v19.4s, v3.8h, v1.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v8.4h, v16.4h\n" + "smlal2 v15.4s, v8.8h, v16.8h\n" + "smlal v20.4s, v27.4h, v16.4h\n" + "smlal2 v5.4s, v27.8h, v16.8h\n" + "smlal v24.4s, v3.4h, v16.4h\n" + "smlal2 v22.4s, v3.8h, v16.8h\n" "tbz x1, #2, 65f\n" - "ld1 { v22.s }[0], [x20], #0x4\n" + "ld1 { v14.s }[0], [x20], #0x4\n" "tbz x1, #1, 64f\n" - "ld1 { v22.h }[2], [x20], #0x2\n" + "ld1 { v14.h }[2], [x20], #0x2\n" "tbz x1, #0, 67f\n" - "ld1 { v22.b }[6], [x20]\n" + "ld1 { v14.b }[6], [x20]\n" "b 67f\n" "64:" // Oddments: Load (3, 5): Bit 2: Bit 1: Unset "tbz x1, #0, 67f\n" - "ld1 { v22.b }[4], [x20]\n" + "ld1 { v14.b }[4], [x20]\n" "b 67f\n" "65:" // Oddments: Load (3, 5): Bit 2: Unset "tbz x1, #1, 66f\n" - "ld1 { v22.h }[0], [x20], #0x2\n" + "ld1 { v14.h }[0], [x20], #0x2\n" "tbz x1, #0, 67f\n" - "ld1 { v22.b }[2], [x20]\n" + "ld1 { v14.b }[2], [x20]\n" "b 67f\n" "66:" // Oddments: Load (3, 5): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 67f\n" - "ld1 { v22.b }[0], [x20]\n" + "ld1 { v14.b }[0], [x20]\n" "67:" // Oddments: Load (3, 5): Bit 2: End - "ldr d0, [x3, #0x78]\n" - "usubl v22.8h, v22.8b, v9.8b\n" - "usubl v0.8h, v0.8b, v15.8b\n" - "ldr x11, [x4, #0xc0]\n" - "smlal v17.4s, v22.4h, v4.4h\n" - "smlal2 v21.4s, v22.8h, v4.8h\n" - "add x11, x11, x0\n" - "smlal v13.4s, v31.4h, v0.4h\n" - "smlal2 v19.4s, v31.8h, v0.8h\n" - "smlal v20.4s, v30.4h, v0.4h\n" - "smlal2 v10.4s, v30.8h, v0.8h\n" + "ldr d17, [x6, #0x78]\n" + "usubl v14.8h, v14.8b, v18.8b\n" + "usubl v17.8h, v17.8b, v13.8b\n" + "ldr x20, [x5, #0xc0]\n" + "smlal v23.4s, v14.4h, v16.4h\n" + "smlal2 v19.4s, v14.8h, v16.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v31.4h, v17.4h\n" + "smlal2 v15.4s, v31.8h, v17.8h\n" + "smlal v20.4s, v28.4h, v17.4h\n" + "smlal2 v5.4s, v28.8h, v17.8h\n" "tbz x1, #2, 69f\n" - "ld1 { v27.s }[0], [x11], #0x4\n" + "ld1 { v1.s }[0], [x20], #0x4\n" "tbz x1, #1, 68f\n" - "ld1 { v27.h }[2], [x11], #0x2\n" + "ld1 { v1.h }[2], [x20], #0x2\n" "tbz x1, #0, 71f\n" - "ld1 { v27.b }[6], [x11]\n" + "ld1 { v1.b }[6], [x20]\n" "b 71f\n" "68:" // Oddments: Load (4, 0): Bit 2: Bit 1: Unset "tbz x1, #0, 71f\n" - "ld1 { v27.b }[4], [x11]\n" + "ld1 { v1.b }[4], [x20]\n" "b 71f\n" "69:" // Oddments: Load (4, 0): Bit 2: Unset "tbz x1, #1, 70f\n" - "ld1 { v27.h }[0], [x11], #0x2\n" + "ld1 { v1.h }[0], [x20], #0x2\n" "tbz x1, #0, 71f\n" - "ld1 { v27.b }[2], [x11]\n" + "ld1 { v1.b }[2], [x20]\n" "b 71f\n" "70:" // Oddments: Load (4, 0): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 71f\n" - "ld1 { v27.b }[0], [x11]\n" + "ld1 { v1.b }[0], [x20]\n" "71:" // Oddments: Load (4, 0): Bit 2: End - "usubl v27.8h, v27.8b, v9.8b\n" - "ldr x22, [x4, #0xc8]\n" - "smlal v8.4s, v27.4h, v0.4h\n" - "smlal2 v7.4s, v27.8h, v0.8h\n" - "add x22, x22, x0\n" + "usubl v1.8h, v1.8b, v18.8b\n" + "ldr x20, [x5, #0xc8]\n" + "smlal v24.4s, v1.4h, v17.4h\n" + "smlal2 v22.4s, v1.8h, v17.8h\n" + "add x20, x20, x3\n" "tbz x1, #2, 73f\n" - "ld1 { v23.s }[0], [x22], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x1, #1, 72f\n" - "ld1 { v23.h }[2], [x22], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x1, #0, 75f\n" - "ld1 { v23.b }[6], [x22]\n" + "ld1 { v16.b }[6], [x20]\n" "b 75f\n" "72:" // Oddments: Load (4, 1): Bit 2: Bit 1: Unset "tbz x1, #0, 75f\n" - "ld1 { v23.b }[4], [x22]\n" + "ld1 { v16.b }[4], [x20]\n" "b 75f\n" "73:" // Oddments: Load (4, 1): Bit 2: Unset "tbz x1, #1, 74f\n" - "ld1 { v23.h }[0], [x22], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x1, #0, 75f\n" - "ld1 { v23.b }[2], [x22]\n" + "ld1 { v16.b }[2], [x20]\n" "b 75f\n" "74:" // Oddments: Load (4, 1): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 75f\n" - "ld1 { v23.b }[0], [x22]\n" + "ld1 { v16.b }[0], [x20]\n" "75:" // Oddments: Load (4, 1): Bit 2: End - "ldr d1, [x3, #0x80]\n" - "usubl v23.8h, v23.8b, v9.8b\n" - "usubl v1.8h, v1.8b, v15.8b\n" - "ldr x9, [x4, #0xd0]\n" - "smlal v17.4s, v23.4h, v0.4h\n" - "smlal2 v21.4s, v23.8h, v0.8h\n" - "add x9, x9, x0\n" - "smlal v13.4s, v30.4h, v1.4h\n" - "smlal2 v19.4s, v30.8h, v1.8h\n" - "smlal v20.4s, v26.4h, v1.4h\n" - "smlal2 v10.4s, v26.8h, v1.8h\n" - "smlal v8.4s, v23.4h, v1.4h\n" - "smlal2 v7.4s, v23.8h, v1.8h\n" + "ldr d29, [x6, #0x80]\n" + "usubl v16.8h, v16.8b, v18.8b\n" + "usubl v29.8h, v29.8b, v13.8b\n" + "ldr x20, [x5, #0xd0]\n" + "smlal v23.4s, v16.4h, v17.4h\n" + "smlal2 v19.4s, v16.8h, v17.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v28.4h, v29.4h\n" + "smlal2 v15.4s, v28.8h, v29.8h\n" + "smlal v20.4s, v21.4h, v29.4h\n" + "smlal2 v5.4s, v21.8h, v29.8h\n" + "smlal v24.4s, v16.4h, v29.4h\n" + "smlal2 v22.4s, v16.8h, v29.8h\n" "tbz x1, #2, 77f\n" - "ld1 { v31.s }[0], [x9], #0x4\n" + "ld1 { v30.s }[0], [x20], #0x4\n" "tbz x1, #1, 76f\n" - "ld1 { v31.h }[2], [x9], #0x2\n" + "ld1 { v30.h }[2], [x20], #0x2\n" "tbz x1, #0, 79f\n" - "ld1 { v31.b }[6], [x9]\n" + "ld1 { v30.b }[6], [x20]\n" "b 79f\n" "76:" // Oddments: Load (4, 2): Bit 2: Bit 1: Unset "tbz x1, #0, 79f\n" - "ld1 { v31.b }[4], [x9]\n" + "ld1 { v30.b }[4], [x20]\n" "b 79f\n" "77:" // Oddments: Load (4, 2): Bit 2: Unset "tbz x1, #1, 78f\n" - "ld1 { v31.h }[0], [x9], #0x2\n" + "ld1 { v30.h }[0], [x20], #0x2\n" "tbz x1, #0, 79f\n" - "ld1 { v31.b }[2], [x9]\n" + "ld1 { v30.b }[2], [x20]\n" "b 79f\n" "78:" // Oddments: Load (4, 2): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 79f\n" - "ld1 { v31.b }[0], [x9]\n" + "ld1 { v30.b }[0], [x20]\n" "79:" // Oddments: Load (4, 2): Bit 2: End - "ldr d2, [x3, #0x88]\n" - "usubl v31.8h, v31.8b, v9.8b\n" - "usubl v2.8h, v2.8b, v15.8b\n" - "ldr x28, [x4, #0xd8]\n" - "smlal v17.4s, v31.4h, v1.4h\n" - "smlal2 v21.4s, v31.8h, v1.8h\n" - "add x28, x28, x0\n" - "smlal v13.4s, v26.4h, v2.4h\n" - "smlal2 v19.4s, v26.8h, v2.8h\n" - "smlal v20.4s, v25.4h, v2.4h\n" - "smlal2 v10.4s, v25.8h, v2.8h\n" - "smlal v8.4s, v31.4h, v2.4h\n" - "smlal2 v7.4s, v31.8h, v2.8h\n" + "ldr d12, [x6, #0x88]\n" + "usubl v30.8h, v30.8b, v18.8b\n" + "usubl v12.8h, v12.8b, v13.8b\n" + "ldr x20, [x5, #0xd8]\n" + "smlal v23.4s, v30.4h, v29.4h\n" + "smlal2 v19.4s, v30.8h, v29.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v21.4h, v12.4h\n" + "smlal2 v15.4s, v21.8h, v12.8h\n" + "smlal v20.4s, v9.4h, v12.4h\n" + "smlal2 v5.4s, v9.8h, v12.8h\n" + "smlal v24.4s, v30.4h, v12.4h\n" + "smlal2 v22.4s, v30.8h, v12.8h\n" "tbz x1, #2, 81f\n" - "ld1 { v30.s }[0], [x28], #0x4\n" + "ld1 { v29.s }[0], [x20], #0x4\n" "tbz x1, #1, 80f\n" - "ld1 { v30.h }[2], [x28], #0x2\n" + "ld1 { v29.h }[2], [x20], #0x2\n" "tbz x1, #0, 83f\n" - "ld1 { v30.b }[6], [x28]\n" + "ld1 { v29.b }[6], [x20]\n" "b 83f\n" "80:" // Oddments: Load (4, 3): Bit 2: Bit 1: Unset "tbz x1, #0, 83f\n" - "ld1 { v30.b }[4], [x28]\n" + "ld1 { v29.b }[4], [x20]\n" "b 83f\n" "81:" // Oddments: Load (4, 3): Bit 2: Unset "tbz x1, #1, 82f\n" - "ld1 { v30.h }[0], [x28], #0x2\n" + "ld1 { v29.h }[0], [x20], #0x2\n" "tbz x1, #0, 83f\n" - "ld1 { v30.b }[2], [x28]\n" + "ld1 { v29.b }[2], [x20]\n" "b 83f\n" "82:" // Oddments: Load (4, 3): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 83f\n" - "ld1 { v30.b }[0], [x28]\n" + "ld1 { v29.b }[0], [x20]\n" "83:" // Oddments: Load (4, 3): Bit 2: End - "ldr d3, [x3, #0x90]\n" - "usubl v30.8h, v30.8b, v9.8b\n" - "usubl v3.8h, v3.8b, v15.8b\n" - "ldr x27, [x4, #0xe0]\n" - "smlal v17.4s, v30.4h, v2.4h\n" - "smlal2 v21.4s, v30.8h, v2.8h\n" - "add x27, x27, x0\n" - "smlal v13.4s, v25.4h, v3.4h\n" - "smlal2 v19.4s, v25.8h, v3.8h\n" - "smlal v20.4s, v24.4h, v3.4h\n" - "smlal2 v10.4s, v24.8h, v3.8h\n" - "smlal v8.4s, v30.4h, v3.4h\n" - "smlal2 v7.4s, v30.8h, v3.8h\n" + "ldr d21, [x6, #0x90]\n" + "usubl v29.8h, v29.8b, v18.8b\n" + "usubl v21.8h, v21.8b, v13.8b\n" + "ldr x20, [x5, #0xe0]\n" + "smlal v23.4s, v29.4h, v12.4h\n" + "smlal2 v19.4s, v29.8h, v12.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v9.4h, v21.4h\n" + "smlal2 v15.4s, v9.8h, v21.8h\n" + "smlal v20.4s, v3.4h, v21.4h\n" + "smlal2 v5.4s, v3.8h, v21.8h\n" + "smlal v24.4s, v29.4h, v21.4h\n" + "smlal2 v22.4s, v29.8h, v21.8h\n" "tbz x1, #2, 85f\n" - "ld1 { v28.s }[0], [x27], #0x4\n" + "ld1 { v25.s }[0], [x20], #0x4\n" "tbz x1, #1, 84f\n" - "ld1 { v28.h }[2], [x27], #0x2\n" + "ld1 { v25.h }[2], [x20], #0x2\n" "tbz x1, #0, 87f\n" - "ld1 { v28.b }[6], [x27]\n" + "ld1 { v25.b }[6], [x20]\n" "b 87f\n" "84:" // Oddments: Load (4, 4): Bit 2: Bit 1: Unset "tbz x1, #0, 87f\n" - "ld1 { v28.b }[4], [x27]\n" + "ld1 { v25.b }[4], [x20]\n" "b 87f\n" "85:" // Oddments: Load (4, 4): Bit 2: Unset "tbz x1, #1, 86f\n" - "ld1 { v28.h }[0], [x27], #0x2\n" + "ld1 { v25.h }[0], [x20], #0x2\n" "tbz x1, #0, 87f\n" - "ld1 { v28.b }[2], [x27]\n" + "ld1 { v25.b }[2], [x20]\n" "b 87f\n" "86:" // Oddments: Load (4, 4): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 87f\n" - "ld1 { v28.b }[0], [x27]\n" + "ld1 { v25.b }[0], [x20]\n" "87:" // Oddments: Load (4, 4): Bit 2: End - "ldr d4, [x3, #0x98]\n" - "usubl v28.8h, v28.8b, v9.8b\n" - "usubl v4.8h, v4.8b, v15.8b\n" - "ldr x26, [x4, #0xe8]\n" - "smlal v17.4s, v28.4h, v3.4h\n" - "smlal2 v21.4s, v28.8h, v3.8h\n" - "add x26, x26, x0\n" - "smlal v13.4s, v24.4h, v4.4h\n" - "smlal2 v19.4s, v24.8h, v4.8h\n" - "smlal v20.4s, v22.4h, v4.4h\n" - "smlal2 v10.4s, v22.8h, v4.8h\n" - "smlal v8.4s, v28.4h, v4.4h\n" - "smlal2 v7.4s, v28.8h, v4.8h\n" + "ldr d8, [x6, #0x98]\n" + "usubl v25.8h, v25.8b, v18.8b\n" + "usubl v8.8h, v8.8b, v13.8b\n" + "ldr x20, [x5, #0xe8]\n" + "smlal v23.4s, v25.4h, v21.4h\n" + "smlal2 v19.4s, v25.8h, v21.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v3.4h, v8.4h\n" + "smlal2 v15.4s, v3.8h, v8.8h\n" + "smlal v20.4s, v14.4h, v8.4h\n" + "smlal2 v5.4s, v14.8h, v8.8h\n" + "smlal v24.4s, v25.4h, v8.4h\n" + "smlal2 v22.4s, v25.8h, v8.8h\n" "tbz x1, #2, 89f\n" - "ld1 { v26.s }[0], [x26], #0x4\n" + "ld1 { v21.s }[0], [x20], #0x4\n" "tbz x1, #1, 88f\n" - "ld1 { v26.h }[2], [x26], #0x2\n" + "ld1 { v21.h }[2], [x20], #0x2\n" "tbz x1, #0, 91f\n" - "ld1 { v26.b }[6], [x26]\n" + "ld1 { v21.b }[6], [x20]\n" "b 91f\n" "88:" // Oddments: Load (4, 5): Bit 2: Bit 1: Unset "tbz x1, #0, 91f\n" - "ld1 { v26.b }[4], [x26]\n" + "ld1 { v21.b }[4], [x20]\n" "b 91f\n" "89:" // Oddments: Load (4, 5): Bit 2: Unset "tbz x1, #1, 90f\n" - "ld1 { v26.h }[0], [x26], #0x2\n" + "ld1 { v21.h }[0], [x20], #0x2\n" "tbz x1, #0, 91f\n" - "ld1 { v26.b }[2], [x26]\n" + "ld1 { v21.b }[2], [x20]\n" "b 91f\n" "90:" // Oddments: Load (4, 5): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 91f\n" - "ld1 { v26.b }[0], [x26]\n" + "ld1 { v21.b }[0], [x20]\n" "91:" // Oddments: Load (4, 5): Bit 2: End - "ldr d0, [x3, #0xa0]\n" - "usubl v26.8h, v26.8b, v9.8b\n" - "usubl v0.8h, v0.8b, v15.8b\n" - "ldr x25, [x4, #0xf0]\n" - "smlal v17.4s, v26.4h, v4.4h\n" - "smlal2 v21.4s, v26.8h, v4.8h\n" - "add x25, x25, x0\n" - "smlal v13.4s, v27.4h, v0.4h\n" - "smlal2 v19.4s, v27.8h, v0.8h\n" - "smlal v20.4s, v23.4h, v0.4h\n" - "smlal2 v10.4s, v23.8h, v0.8h\n" + "ldr d9, [x6, #0xa0]\n" + "usubl v21.8h, v21.8b, v18.8b\n" + "usubl v9.8h, v9.8b, v13.8b\n" + "ldr x20, [x5, #0xf0]\n" + "smlal v23.4s, v21.4h, v8.4h\n" + "smlal2 v19.4s, v21.8h, v8.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v1.4h, v9.4h\n" + "smlal2 v15.4s, v1.8h, v9.8h\n" + "smlal v20.4s, v16.4h, v9.4h\n" + "smlal2 v5.4s, v16.8h, v9.8h\n" "tbz x1, #2, 93f\n" - "ld1 { v25.s }[0], [x25], #0x4\n" + "ld1 { v12.s }[0], [x20], #0x4\n" "tbz x1, #1, 92f\n" - "ld1 { v25.h }[2], [x25], #0x2\n" + "ld1 { v12.h }[2], [x20], #0x2\n" "tbz x1, #0, 95f\n" - "ld1 { v25.b }[6], [x25]\n" + "ld1 { v12.b }[6], [x20]\n" "b 95f\n" "92:" // Oddments: Load (5, 0): Bit 2: Bit 1: Unset "tbz x1, #0, 95f\n" - "ld1 { v25.b }[4], [x25]\n" + "ld1 { v12.b }[4], [x20]\n" "b 95f\n" "93:" // Oddments: Load (5, 0): Bit 2: Unset "tbz x1, #1, 94f\n" - "ld1 { v25.h }[0], [x25], #0x2\n" + "ld1 { v12.h }[0], [x20], #0x2\n" "tbz x1, #0, 95f\n" - "ld1 { v25.b }[2], [x25]\n" + "ld1 { v12.b }[2], [x20]\n" "b 95f\n" "94:" // Oddments: Load (5, 0): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 95f\n" - "ld1 { v25.b }[0], [x25]\n" + "ld1 { v12.b }[0], [x20]\n" "95:" // Oddments: Load (5, 0): Bit 2: End - "usubl v25.8h, v25.8b, v9.8b\n" - "ldr x24, [x4, #0xf8]\n" - "smlal v8.4s, v25.4h, v0.4h\n" - "smlal2 v7.4s, v25.8h, v0.8h\n" - "add x24, x24, x0\n" + "usubl v12.8h, v12.8b, v18.8b\n" + "ldr x20, [x5, #0xf8]\n" + "smlal v24.4s, v12.4h, v9.4h\n" + "smlal2 v22.4s, v12.8h, v9.8h\n" + "add x20, x20, x3\n" "tbz x1, #2, 97f\n" - "ld1 { v24.s }[0], [x24], #0x4\n" + "ld1 { v10.s }[0], [x20], #0x4\n" "tbz x1, #1, 96f\n" - "ld1 { v24.h }[2], [x24], #0x2\n" + "ld1 { v10.h }[2], [x20], #0x2\n" "tbz x1, #0, 99f\n" - "ld1 { v24.b }[6], [x24]\n" + "ld1 { v10.b }[6], [x20]\n" "b 99f\n" "96:" // Oddments: Load (5, 1): Bit 2: Bit 1: Unset "tbz x1, #0, 99f\n" - "ld1 { v24.b }[4], [x24]\n" + "ld1 { v10.b }[4], [x20]\n" "b 99f\n" "97:" // Oddments: Load (5, 1): Bit 2: Unset "tbz x1, #1, 98f\n" - "ld1 { v24.h }[0], [x24], #0x2\n" + "ld1 { v10.h }[0], [x20], #0x2\n" "tbz x1, #0, 99f\n" - "ld1 { v24.b }[2], [x24]\n" + "ld1 { v10.b }[2], [x20]\n" "b 99f\n" "98:" // Oddments: Load (5, 1): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 99f\n" - "ld1 { v24.b }[0], [x24]\n" + "ld1 { v10.b }[0], [x20]\n" "99:" // Oddments: Load (5, 1): Bit 2: End - "ldr d1, [x3, #0xa8]\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "usubl v1.8h, v1.8b, v15.8b\n" - "ldr x23, [x4, #0x100]\n" - "smlal v17.4s, v24.4h, v0.4h\n" - "smlal2 v21.4s, v24.8h, v0.8h\n" - "add x23, x23, x0\n" - "smlal v13.4s, v23.4h, v1.4h\n" - "smlal2 v19.4s, v23.8h, v1.8h\n" - "smlal v20.4s, v31.4h, v1.4h\n" - "smlal2 v10.4s, v31.8h, v1.8h\n" - "smlal v8.4s, v24.4h, v1.4h\n" - "smlal2 v7.4s, v24.8h, v1.8h\n" + "ldr d12, [x6, #0xa8]\n" + "usubl v10.8h, v10.8b, v18.8b\n" + "usubl v12.8h, v12.8b, v13.8b\n" + "ldr x20, [x5, #0x100]\n" + "smlal v23.4s, v10.4h, v9.4h\n" + "smlal2 v19.4s, v10.8h, v9.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v16.4h, v12.4h\n" + "smlal2 v15.4s, v16.8h, v12.8h\n" + "smlal v20.4s, v30.4h, v12.4h\n" + "smlal2 v5.4s, v30.8h, v12.8h\n" + "smlal v24.4s, v10.4h, v12.4h\n" + "smlal2 v22.4s, v10.8h, v12.8h\n" "tbz x1, #2, 101f\n" - "ld1 { v27.s }[0], [x23], #0x4\n" + "ld1 { v9.s }[0], [x20], #0x4\n" "tbz x1, #1, 100f\n" - "ld1 { v27.h }[2], [x23], #0x2\n" + "ld1 { v9.h }[2], [x20], #0x2\n" "tbz x1, #0, 103f\n" - "ld1 { v27.b }[6], [x23]\n" + "ld1 { v9.b }[6], [x20]\n" "b 103f\n" "100:" // Oddments: Load (5, 2): Bit 2: Bit 1: Unset "tbz x1, #0, 103f\n" - "ld1 { v27.b }[4], [x23]\n" + "ld1 { v9.b }[4], [x20]\n" "b 103f\n" "101:" // Oddments: Load (5, 2): Bit 2: Unset "tbz x1, #1, 102f\n" - "ld1 { v27.h }[0], [x23], #0x2\n" + "ld1 { v9.h }[0], [x20], #0x2\n" "tbz x1, #0, 103f\n" - "ld1 { v27.b }[2], [x23]\n" + "ld1 { v9.b }[2], [x20]\n" "b 103f\n" "102:" // Oddments: Load (5, 2): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 103f\n" - "ld1 { v27.b }[0], [x23]\n" + "ld1 { v9.b }[0], [x20]\n" "103:" // Oddments: Load (5, 2): Bit 2: End - "ldr d2, [x3, #0xb0]\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "usubl v2.8h, v2.8b, v15.8b\n" - "ldr x15, [x4, #0x108]\n" - "smlal v17.4s, v27.4h, v1.4h\n" - "smlal2 v21.4s, v27.8h, v1.8h\n" - "add x15, x15, x0\n" - "smlal v13.4s, v31.4h, v2.4h\n" - "smlal2 v19.4s, v31.8h, v2.8h\n" - "smlal v20.4s, v30.4h, v2.4h\n" - "smlal2 v10.4s, v30.8h, v2.8h\n" - "smlal v8.4s, v27.4h, v2.4h\n" - "smlal2 v7.4s, v27.8h, v2.8h\n" + "ldr d28, [x6, #0xb0]\n" + "usubl v9.8h, v9.8b, v18.8b\n" + "usubl v28.8h, v28.8b, v13.8b\n" + "ldr x20, [x5, #0x108]\n" + "smlal v23.4s, v9.4h, v12.4h\n" + "smlal2 v19.4s, v9.8h, v12.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v30.4h, v28.4h\n" + "smlal2 v15.4s, v30.8h, v28.8h\n" + "smlal v20.4s, v29.4h, v28.4h\n" + "smlal2 v5.4s, v29.8h, v28.8h\n" + "smlal v24.4s, v9.4h, v28.4h\n" + "smlal2 v22.4s, v9.8h, v28.8h\n" "tbz x1, #2, 105f\n" - "ld1 { v25.s }[0], [x15], #0x4\n" + "ld1 { v2.s }[0], [x20], #0x4\n" "tbz x1, #1, 104f\n" - "ld1 { v25.h }[2], [x15], #0x2\n" + "ld1 { v2.h }[2], [x20], #0x2\n" "tbz x1, #0, 107f\n" - "ld1 { v25.b }[6], [x15]\n" + "ld1 { v2.b }[6], [x20]\n" "b 107f\n" "104:" // Oddments: Load (5, 3): Bit 2: Bit 1: Unset "tbz x1, #0, 107f\n" - "ld1 { v25.b }[4], [x15]\n" + "ld1 { v2.b }[4], [x20]\n" "b 107f\n" "105:" // Oddments: Load (5, 3): Bit 2: Unset "tbz x1, #1, 106f\n" - "ld1 { v25.h }[0], [x15], #0x2\n" + "ld1 { v2.h }[0], [x20], #0x2\n" "tbz x1, #0, 107f\n" - "ld1 { v25.b }[2], [x15]\n" + "ld1 { v2.b }[2], [x20]\n" "b 107f\n" "106:" // Oddments: Load (5, 3): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 107f\n" - "ld1 { v25.b }[0], [x15]\n" + "ld1 { v2.b }[0], [x20]\n" "107:" // Oddments: Load (5, 3): Bit 2: End - "ldr d3, [x3, #0xb8]\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "usubl v3.8h, v3.8b, v15.8b\n" - "ldr x21, [x4, #0x110]\n" - "smlal v17.4s, v25.4h, v2.4h\n" - "smlal2 v21.4s, v25.8h, v2.8h\n" - "add x21, x21, x0\n" - "smlal v13.4s, v30.4h, v3.4h\n" - "smlal2 v19.4s, v30.8h, v3.8h\n" - "smlal v20.4s, v28.4h, v3.4h\n" - "smlal2 v10.4s, v28.8h, v3.8h\n" - "smlal v8.4s, v25.4h, v3.4h\n" - "smlal2 v7.4s, v25.8h, v3.8h\n" + "ldr d30, [x6, #0xb8]\n" + "usubl v2.8h, v2.8b, v18.8b\n" + "usubl v30.8h, v30.8b, v13.8b\n" + "ldr x20, [x5, #0x110]\n" + "smlal v23.4s, v2.4h, v28.4h\n" + "smlal2 v19.4s, v2.8h, v28.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v29.4h, v30.4h\n" + "smlal2 v15.4s, v29.8h, v30.8h\n" + "smlal v20.4s, v25.4h, v30.4h\n" + "smlal2 v5.4s, v25.8h, v30.8h\n" + "smlal v24.4s, v2.4h, v30.4h\n" + "smlal2 v22.4s, v2.8h, v30.8h\n" "tbz x1, #2, 109f\n" - "ld1 { v24.s }[0], [x21], #0x4\n" + "ld1 { v27.s }[0], [x20], #0x4\n" "tbz x1, #1, 108f\n" - "ld1 { v24.h }[2], [x21], #0x2\n" + "ld1 { v27.h }[2], [x20], #0x2\n" "tbz x1, #0, 111f\n" - "ld1 { v24.b }[6], [x21]\n" + "ld1 { v27.b }[6], [x20]\n" "b 111f\n" "108:" // Oddments: Load (5, 4): Bit 2: Bit 1: Unset "tbz x1, #0, 111f\n" - "ld1 { v24.b }[4], [x21]\n" + "ld1 { v27.b }[4], [x20]\n" "b 111f\n" "109:" // Oddments: Load (5, 4): Bit 2: Unset "tbz x1, #1, 110f\n" - "ld1 { v24.h }[0], [x21], #0x2\n" + "ld1 { v27.h }[0], [x20], #0x2\n" "tbz x1, #0, 111f\n" - "ld1 { v24.b }[2], [x21]\n" + "ld1 { v27.b }[2], [x20]\n" "b 111f\n" "110:" // Oddments: Load (5, 4): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 111f\n" - "ld1 { v24.b }[0], [x21]\n" + "ld1 { v27.b }[0], [x20]\n" "111:" // Oddments: Load (5, 4): Bit 2: End - "ldr d4, [x3, #0xc0]\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "usubl v4.8h, v4.8b, v15.8b\n" - "ldr x20, [x4, #0x118]\n" - "smlal v17.4s, v24.4h, v3.4h\n" - "smlal2 v21.4s, v24.8h, v3.8h\n" - "add x20, x20, x0\n" - "smlal v13.4s, v28.4h, v4.4h\n" - "smlal2 v19.4s, v28.8h, v4.8h\n" - "smlal v20.4s, v26.4h, v4.4h\n" - "smlal2 v10.4s, v26.8h, v4.8h\n" - "smlal v8.4s, v24.4h, v4.4h\n" - "smlal2 v7.4s, v24.8h, v4.8h\n" + "ldr d8, [x6, #0xc0]\n" + "usubl v27.8h, v27.8b, v18.8b\n" + "usubl v8.8h, v8.8b, v13.8b\n" + "ldr x20, [x5, #0x118]\n" + "smlal v23.4s, v27.4h, v30.4h\n" + "smlal2 v19.4s, v27.8h, v30.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v25.4h, v8.4h\n" + "smlal2 v15.4s, v25.8h, v8.8h\n" + "smlal v20.4s, v21.4h, v8.4h\n" + "smlal2 v5.4s, v21.8h, v8.8h\n" + "smlal v24.4s, v27.4h, v8.4h\n" + "smlal2 v22.4s, v27.8h, v8.8h\n" "tbz x1, #2, 113f\n" - "ld1 { v27.s }[0], [x20], #0x4\n" + "ld1 { v9.s }[0], [x20], #0x4\n" "tbz x1, #1, 112f\n" - "ld1 { v27.h }[2], [x20], #0x2\n" + "ld1 { v9.h }[2], [x20], #0x2\n" "tbz x1, #0, 115f\n" - "ld1 { v27.b }[6], [x20]\n" + "ld1 { v9.b }[6], [x20]\n" "b 115f\n" "112:" // Oddments: Load (5, 5): Bit 2: Bit 1: Unset "tbz x1, #0, 115f\n" - "ld1 { v27.b }[4], [x20]\n" + "ld1 { v9.b }[4], [x20]\n" "b 115f\n" "113:" // Oddments: Load (5, 5): Bit 2: Unset "tbz x1, #1, 114f\n" - "ld1 { v27.h }[0], [x20], #0x2\n" + "ld1 { v9.h }[0], [x20], #0x2\n" "tbz x1, #0, 115f\n" - "ld1 { v27.b }[2], [x20]\n" + "ld1 { v9.b }[2], [x20]\n" "b 115f\n" "114:" // Oddments: Load (5, 5): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 115f\n" - "ld1 { v27.b }[0], [x20]\n" + "ld1 { v9.b }[0], [x20]\n" "115:" // Oddments: Load (5, 5): Bit 2: End - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal v17.4s, v27.4h, v4.4h\n" - "smlal2 v21.4s, v27.8h, v4.8h\n" + "usubl v9.8h, v9.8b, v18.8b\n" + "smlal v23.4s, v9.4h, v8.4h\n" + "smlal2 v19.4s, v9.8h, v8.8h\n" "tbz x1, #2, 117f\n" - "ld1 { v18.4s }, [x5], #0x10\n" - "ld1 { v6.4s }, [x8], #0x10\n" + "ld1 { v30.4s }, [x7], #0x10\n" + "ld1 { v12.4s }, [x8], #0x10\n" "tbz x1, #1, 116f\n" - "ld1 { v5.d }[0], [x5], #0x8\n" - "ld1 { v22.d }[0], [x8], #0x8\n" + "ld1 { v14.d }[0], [x7], #0x8\n" + "ld1 { v27.d }[0], [x8], #0x8\n" "tbz x1, #0, 119f\n" - "ld1 { v5.s }[2], [x5]\n" - "ld1 { v22.s }[2], [x8]\n" + "ld1 { v14.s }[2], [x7]\n" + "ld1 { v27.s }[2], [x8]\n" "b 119f\n" "116:" // Oddments: Load requant params: Bit 2: Bit 1: Unset "tbz x1, #0, 119f\n" - "ld1 { v5.s }[0], [x5]\n" - "ld1 { v22.s }[0], [x8]\n" + "ld1 { v14.s }[0], [x7]\n" + "ld1 { v27.s }[0], [x8]\n" "b 119f\n" "117:" // Oddments: Load requant params: Bit 2: Unset "tbz x1, #1, 118f\n" - "ld1 { v18.d }[0], [x5], #0x8\n" - "ld1 { v6.d }[0], [x8], #0x8\n" + "ld1 { v30.d }[0], [x7], #0x8\n" + "ld1 { v12.d }[0], [x8], #0x8\n" "tbz x1, #0, 119f\n" - "ld1 { v18.s }[2], [x5]\n" - "ld1 { v6.s }[2], [x8]\n" + "ld1 { v30.s }[2], [x7]\n" + "ld1 { v12.s }[2], [x8]\n" "b 119f\n" "118:" // Oddments: Load requant params: Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 119f\n" - "ld1 { v18.s }[0], [x5]\n" - "ld1 { v6.s }[0], [x8]\n" + "ld1 { v30.s }[0], [x7]\n" + "ld1 { v12.s }[0], [x8]\n" "119:" // Oddments: Load requant params: Bit 2: End - "sqrdmulh v13.4s, v13.4s, v18.4s\n" - "and v30.16b, v13.16b, v6.16b\n" - "add x17, x17, x10\n" - "add x6, x6, x10\n" - "sqrdmulh v19.4s, v19.4s, v5.4s\n" - "sshr v30.4s, v30.4s, #0x1f\n" - "add x7, x7, x10\n" - "add x16, x16, x10\n" - "and v16.16b, v19.16b, v22.16b\n" - "sqrdmulh v20.4s, v20.4s, v18.4s\n" - "sqrdmulh v8.4s, v8.4s, v18.4s\n" - "sqrdmulh v17.4s, v17.4s, v18.4s\n" - "sqadd v13.4s, v13.4s, v30.4s\n" + "sqrdmulh v7.4s, v7.4s, v30.4s\n" + "and v16.16b, v7.16b, v12.16b\n" + "add x17, x17, x4\n" + "add x16, x16, x4\n" + "sqrdmulh v15.4s, v15.4s, v14.4s\n" "sshr v16.4s, v16.4s, #0x1f\n" - "and v0.16b, v20.16b, v6.16b\n" - "sqrdmulh v10.4s, v10.4s, v5.4s\n" - "and v18.16b, v8.16b, v6.16b\n" - "sqrdmulh v7.4s, v7.4s, v5.4s\n" - "and v30.16b, v17.16b, v6.16b\n" - "sqrdmulh v21.4s, v21.4s, v5.4s\n" - "sqadd v19.4s, v19.4s, v16.4s\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v26.16b, v10.16b, v22.16b\n" + "add x15, x15, x4\n" + "add x14, x14, x4\n" + "and v2.16b, v15.16b, v27.16b\n" + "sqrdmulh v20.4s, v20.4s, v30.4s\n" + "sqrdmulh v24.4s, v24.4s, v30.4s\n" + "sqrdmulh v23.4s, v23.4s, v30.4s\n" + "sqadd v7.4s, v7.4s, v16.4s\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "and v21.16b, v20.16b, v12.16b\n" + "sqrdmulh v5.4s, v5.4s, v14.4s\n" + "and v18.16b, v24.16b, v12.16b\n" + "sqrdmulh v22.4s, v22.4s, v14.4s\n" + "and v31.16b, v23.16b, v12.16b\n" + "sqrdmulh v19.4s, v19.4s, v14.4s\n" + "sqadd v15.4s, v15.4s, v2.4s\n" + "sshr v21.4s, v21.4s, #0x1f\n" + "and v9.16b, v5.16b, v27.16b\n" "sshr v18.4s, v18.4s, #0x1f\n" - "and v23.16b, v7.16b, v22.16b\n" - "sshr v30.4s, v30.4s, #0x1f\n" - "and v16.16b, v21.16b, v22.16b\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "sshr v26.4s, v26.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v18.4s\n" - "sshr v23.4s, v23.4s, #0x1f\n" - "sqadd v17.4s, v17.4s, v30.4s\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "srshl v13.4s, v13.4s, v6.4s\n" - "srshl v20.4s, v20.4s, v6.4s\n" - "sqadd v10.4s, v10.4s, v26.4s\n" - "srshl v8.4s, v8.4s, v6.4s\n" - "sqadd v7.4s, v7.4s, v23.4s\n" - "srshl v17.4s, v17.4s, v6.4s\n" - "sqadd v21.4s, v21.4s, v16.4s\n" - "srshl v19.4s, v19.4s, v22.4s\n" - "sqxtn v13.4h, v13.4s\n" - "srshl v10.4s, v10.4s, v22.4s\n" + "and v4.16b, v22.16b, v27.16b\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "and v28.16b, v19.16b, v27.16b\n" + "sqadd v20.4s, v20.4s, v21.4s\n" + "sshr v9.4s, v9.4s, #0x1f\n" + "sqadd v24.4s, v24.4s, v18.4s\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sqadd v23.4s, v23.4s, v31.4s\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "srshl v7.4s, v7.4s, v12.4s\n" + "srshl v20.4s, v20.4s, v12.4s\n" + "sqadd v5.4s, v5.4s, v9.4s\n" + "srshl v24.4s, v24.4s, v12.4s\n" + "sqadd v22.4s, v22.4s, v4.4s\n" + "srshl v23.4s, v23.4s, v12.4s\n" + "sqadd v19.4s, v19.4s, v28.4s\n" + "srshl v15.4s, v15.4s, v27.4s\n" + "sqxtn v7.4h, v7.4s\n" + "srshl v5.4s, v5.4s, v27.4s\n" "sqxtn v20.4h, v20.4s\n" - "srshl v7.4s, v7.4s, v22.4s\n" - "sqxtn v8.4h, v8.4s\n" - "srshl v21.4s, v21.4s, v22.4s\n" - "sqxtn v17.4h, v17.4s\n" - "sqxtn2 v13.8h, v19.4s\n" - "sqxtn2 v20.8h, v10.4s\n" - "sqxtn2 v8.8h, v7.4s\n" - "sqxtn2 v17.8h, v21.4s\n" - "sqadd v13.8h, v13.8h, v14.8h\n" - "sqadd v20.8h, v20.8h, v14.8h\n" - "sqadd v8.8h, v8.8h, v14.8h\n" - "sqadd v17.8h, v17.8h, v14.8h\n" - "smax v13.8h, v13.8h, v12.8h\n" - "smax v20.8h, v20.8h, v12.8h\n" - "smax v8.8h, v8.8h, v12.8h\n" - "smax v17.8h, v17.8h, v12.8h\n" - "smin v13.8h, v13.8h, v11.8h\n" - "smin v20.8h, v20.8h, v11.8h\n" - "smin v8.8h, v8.8h, v11.8h\n" - "smin v17.8h, v17.8h, v11.8h\n" - "uzp1 v13.16b, v13.16b, v13.16b\n" + "srshl v22.4s, v22.4s, v27.4s\n" + "sqxtn v24.4h, v24.4s\n" + "srshl v19.4s, v19.4s, v27.4s\n" + "sqxtn v23.4h, v23.4s\n" + "sqxtn2 v7.8h, v15.4s\n" + "sqxtn2 v20.8h, v5.4s\n" + "sqxtn2 v24.8h, v22.4s\n" + "sqxtn2 v23.8h, v19.4s\n" + "sqadd v7.8h, v7.8h, v26.8h\n" + "sqadd v20.8h, v20.8h, v26.8h\n" + "sqadd v24.8h, v24.8h, v26.8h\n" + "sqadd v23.8h, v23.8h, v26.8h\n" + "smax v7.8h, v7.8h, v11.8h\n" + "smax v20.8h, v20.8h, v11.8h\n" + "smax v24.8h, v24.8h, v11.8h\n" + "smax v23.8h, v23.8h, v11.8h\n" + "smin v7.8h, v7.8h, v0.8h\n" + "smin v20.8h, v20.8h, v0.8h\n" + "smin v24.8h, v24.8h, v0.8h\n" + "smin v23.8h, v23.8h, v0.8h\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "uzp1 v8.16b, v8.16b, v8.16b\n" - "uzp1 v17.16b, v17.16b, v17.16b\n" + "uzp1 v24.16b, v24.16b, v24.16b\n" + "uzp1 v23.16b, v23.16b, v23.16b\n" "tbz x1, #2, 121f\n" - "st1 { v13.s }[0], [x17], #0x4\n" - "st1 { v20.s }[0], [x6], #0x4\n" - "st1 { v8.s }[0], [x7], #0x4\n" - "st1 { v17.s }[0], [x16], #0x4\n" + "st1 { v7.s }[0], [x17], #0x4\n" + "st1 { v20.s }[0], [x16], #0x4\n" + "st1 { v24.s }[0], [x15], #0x4\n" + "st1 { v23.s }[0], [x14], #0x4\n" "tbz x1, #1, 120f\n" - "st1 { v13.h }[2], [x17], #0x2\n" - "st1 { v20.h }[2], [x6], #0x2\n" - "st1 { v8.h }[2], [x7], #0x2\n" - "st1 { v17.h }[2], [x16], #0x2\n" + "st1 { v7.h }[2], [x17], #0x2\n" + "st1 { v20.h }[2], [x16], #0x2\n" + "st1 { v24.h }[2], [x15], #0x2\n" + "st1 { v23.h }[2], [x14], #0x2\n" "tbz x1, #0, 123f\n" - "st1 { v13.b }[6], [x17], #0x1\n" - "st1 { v20.b }[6], [x6], #0x1\n" - "st1 { v8.b }[6], [x7], #0x1\n" - "st1 { v17.b }[6], [x16], #0x1\n" + "st1 { v7.b }[6], [x17], #0x1\n" + "st1 { v20.b }[6], [x16], #0x1\n" + "st1 { v24.b }[6], [x15], #0x1\n" + "st1 { v23.b }[6], [x14], #0x1\n" "b 123f\n" "120:" // Oddments: Bit 2: Bit 1: Unset "tbz x1, #0, 123f\n" - "st1 { v13.b }[4], [x17], #0x1\n" - "st1 { v20.b }[4], [x6], #0x1\n" - "st1 { v8.b }[4], [x7], #0x1\n" - "st1 { v17.b }[4], [x16], #0x1\n" + "st1 { v7.b }[4], [x17], #0x1\n" + "st1 { v20.b }[4], [x16], #0x1\n" + "st1 { v24.b }[4], [x15], #0x1\n" + "st1 { v23.b }[4], [x14], #0x1\n" "b 123f\n" "121:" // Oddments: Bit 2: Unset "tbz x1, #1, 122f\n" - "st1 { v13.h }[0], [x17], #0x2\n" - "st1 { v20.h }[0], [x6], #0x2\n" - "st1 { v8.h }[0], [x7], #0x2\n" - "st1 { v17.h }[0], [x16], #0x2\n" + "st1 { v7.h }[0], [x17], #0x2\n" + "st1 { v20.h }[0], [x16], #0x2\n" + "st1 { v24.h }[0], [x15], #0x2\n" + "st1 { v23.h }[0], [x14], #0x2\n" "tbz x1, #0, 123f\n" - "st1 { v13.b }[2], [x17], #0x1\n" - "st1 { v20.b }[2], [x6], #0x1\n" - "st1 { v8.b }[2], [x7], #0x1\n" - "st1 { v17.b }[2], [x16], #0x1\n" + "st1 { v7.b }[2], [x17], #0x1\n" + "st1 { v20.b }[2], [x16], #0x1\n" + "st1 { v24.b }[2], [x15], #0x1\n" + "st1 { v23.b }[2], [x14], #0x1\n" "b 123f\n" "122:" // Oddments: Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 123f\n" - "st1 { v13.b }[0], [x17], #0x1\n" - "st1 { v20.b }[0], [x6], #0x1\n" - "st1 { v8.b }[0], [x7], #0x1\n" - "st1 { v17.b }[0], [x16], #0x1\n" + "st1 { v7.b }[0], [x17], #0x1\n" + "st1 { v20.b }[0], [x16], #0x1\n" + "st1 { v24.b }[0], [x15], #0x1\n" + "st1 { v23.b }[0], [x14], #0x1\n" "123:" // Oddments: Bit 2: End "124:" // End : : [offsetof_Params_bias] "I" (offsetof(Params, bias)), [offsetof_Params_inptrs] "I" (offsetof(Params, inptrs)), [offsetof_Params_n_channels] "I" (offsetof(Params, n_channels)), [offsetof_Params_outptrs] "I" (offsetof(Params, outptrs)), [offsetof_Params_requant] "I" (offsetof(Params, requant)), [offsetof_Params_requant_muls] "I" (offsetof(Params, requant_muls)), [offsetof_Params_requant_shifts] "I" (offsetof(Params, requant_shifts)), [offsetof_Params_weights] "I" (offsetof(Params, weights)), [offsetof_Requantize32_a_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [params] "r" (¶ms) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_generic_output9_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_generic_output9_mla_depthfirst.hpp index b859978b1e4faa7e2b7183e546afe94794e9ea88..814efe006e100fd1b62941e6b14b32f4a1eb38dc 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_generic_output9_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_generic_output9_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include @@ -47,4 +47,5 @@ class a64_u8q_nhwc_generic_output9_mla_depthfirst : public GenericDepthfirstKern } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_generic_output9_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_generic_output9_mla_depthfirst/generic.cpp index 39001aa1fde380de00eacb09cdd1b6ce20e2bd99..f7aa889b562c89228de11f88a94eb411d0014500 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_generic_output9_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_generic_output9_mla_depthfirst/generic.cpp @@ -22,12 +22,13 @@ * SOFTWARE. */ -#if defined(__aarch64__) #include "arm_gemm.hpp" #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace depthwise { @@ -41,7 +42,7 @@ void a64_u8q_nhwc_generic_output9_mla_depthfirst_impl( ) { __asm__ __volatile__( - "lsr x12, %x[n_channels], #0x2\n" + "lsr x9, %x[n_channels], #0x2\n" "add x20, %x[qp], %[offsetof_Requantize32_minval]\n" "ld1r { v8.4s }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_maxval]\n" @@ -59,7 +60,7 @@ void a64_u8q_nhwc_generic_output9_mla_depthfirst_impl( "add x20, %x[qp], %[offsetof_Requantize32_per_layer_right_shift]\n" "ld1r { v1.4s }, [x20]\n" "mov x11, #0x0\n" - "cbz x12, 6f\n" + "cbz x9, 6f\n" "1:" // Channel loop "movi v23.4s, #0x0\n" "cbz %x[bias], 2f\n" @@ -67,34 +68,34 @@ void a64_u8q_nhwc_generic_output9_mla_depthfirst_impl( "ldr q23, [%x[bias], x20]\n" "2:" // Channel loop: Load bias: Done "ldr s0, [%x[params]], #0x4\n" - "mov x21, %x[inptrs]\n" - "ldp x10, x9, [x21], #0x10\n" - "subs x20, %x[n_points], #0x1\n" - "ldr s14, [x10, x11]\n" - "ldr s15, [x9, x11]\n" + "mov x25, %x[inptrs]\n" + "ldp x21, x20, [x25], #0x10\n" + "subs x24, %x[n_points], #0x1\n" + "ldr s14, [x21, x11]\n" + "ldr s15, [x20, x11]\n" "mov v24.16b, v23.16b\n" "mov v25.16b, v23.16b\n" - "ldp x28, x27, [x21], #0x10\n" - "ldr s16, [x28, x11]\n" + "ldp x21, x20, [x25], #0x10\n" + "ldr s16, [x21, x11]\n" "mov v26.16b, v23.16b\n" "mov v27.16b, v23.16b\n" - "ldr s17, [x27, x11]\n" - "ldp x26, x25, [x21], #0x10\n" + "ldr s17, [x20, x11]\n" + "ldp x21, x20, [x25], #0x10\n" "mov v28.16b, v23.16b\n" "mov v29.16b, v23.16b\n" - "ldr s18, [x26, x11]\n" - "ldr s19, [x25, x11]\n" + "ldr s18, [x21, x11]\n" + "ldr s19, [x20, x11]\n" "mov v30.16b, v23.16b\n" "mov v31.16b, v23.16b\n" - "ldp x24, x23, [x21], #0x10\n" - "ldr s20, [x24, x11]\n" + "ldp x21, x20, [x25], #0x10\n" + "ldr s20, [x21, x11]\n" "usubl v0.8h, v0.8b, v5.8b\n" "usubl v14.8h, v14.8b, v6.8b\n" - "ldr s21, [x23, x11]\n" - "ldr x22, [x21], #0x8\n" + "ldr s21, [x20, x11]\n" + "ldr x20, [x25], #0x8\n" "usubl v15.8h, v15.8b, v6.8b\n" "usubl v16.8h, v16.8b, v6.8b\n" - "ldr s22, [x22, x11]\n" + "ldr s22, [x20, x11]\n" "usubl v17.8h, v17.8b, v6.8b\n" "usubl v18.8h, v18.8b, v6.8b\n" "usubl v19.8h, v19.8b, v6.8b\n" @@ -103,35 +104,35 @@ void a64_u8q_nhwc_generic_output9_mla_depthfirst_impl( "usubl v22.8h, v22.8b, v6.8b\n" "ble 4f\n" "3:" // Channel loop: Planar loop - "ldp x10, x9, [x21], #0x10\n" - "ldp x28, x27, [x21], #0x10\n" + "ldp x23, x22, [x25], #0x10\n" + "ldp x21, x20, [x25], #0x10\n" "smlal v23.4s, v14.4h, v0.4h\n" "smlal v24.4s, v15.4h, v0.4h\n" - "ldr s14, [x10, x11]\n" - "ldr s15, [x9, x11]\n" + "ldr s14, [x23, x11]\n" + "ldr s15, [x22, x11]\n" "smlal v25.4s, v16.4h, v0.4h\n" "smlal v26.4s, v17.4h, v0.4h\n" - "ldr s16, [x28, x11]\n" - "ldr s17, [x27, x11]\n" + "ldr s16, [x21, x11]\n" + "ldr s17, [x20, x11]\n" "smlal v27.4s, v18.4h, v0.4h\n" "smlal v28.4s, v19.4h, v0.4h\n" - "ldp x26, x25, [x21], #0x10\n" - "ldr s18, [x26, x11]\n" + "ldp x21, x20, [x25], #0x10\n" + "ldr s18, [x21, x11]\n" "smlal v29.4s, v20.4h, v0.4h\n" "smlal v30.4s, v21.4h, v0.4h\n" - "ldr s19, [x25, x11]\n" - "ldp x24, x23, [x21], #0x10\n" + "ldr s19, [x20, x11]\n" + "ldp x21, x20, [x25], #0x10\n" "smlal v31.4s, v22.4h, v0.4h\n" - "subs x20, x20, #0x1\n" + "subs x24, x24, #0x1\n" "ldr s0, [%x[params]], #0x4\n" - "ldr s20, [x24, x11]\n" + "ldr s20, [x21, x11]\n" "usubl v0.8h, v0.8b, v5.8b\n" "usubl v14.8h, v14.8b, v6.8b\n" - "ldr s21, [x23, x11]\n" - "ldr x22, [x21], #0x8\n" + "ldr s21, [x20, x11]\n" + "ldr x20, [x25], #0x8\n" "usubl v15.8h, v15.8b, v6.8b\n" "usubl v16.8h, v16.8b, v6.8b\n" - "ldr s22, [x22, x11]\n" + "ldr s22, [x20, x11]\n" "usubl v17.8h, v17.8b, v6.8b\n" "usubl v18.8h, v18.8b, v6.8b\n" "usubl v19.8h, v19.8b, v6.8b\n" @@ -167,45 +168,45 @@ void a64_u8q_nhwc_generic_output9_mla_depthfirst_impl( "sqrdmulh v24.4s, v24.4s, v2.4s\n" "sqrdmulh v25.4s, v25.4s, v2.4s\n" "ldr x20, [%x[outptrs], #0x40]\n" - "and v21.16b, v23.16b, v1.16b\n" - "and v20.16b, v24.16b, v1.16b\n" - "and v19.16b, v25.16b, v1.16b\n" + "and v18.16b, v23.16b, v1.16b\n" + "and v17.16b, v24.16b, v1.16b\n" + "and v16.16b, v25.16b, v1.16b\n" "sshl v26.4s, v26.4s, v3.4s\n" "sshl v27.4s, v27.4s, v3.4s\n" "sshl v28.4s, v28.4s, v3.4s\n" "sshl v29.4s, v29.4s, v3.4s\n" "sshl v30.4s, v30.4s, v3.4s\n" "sshl v31.4s, v31.4s, v3.4s\n" - "sshr v21.4s, v21.4s, #0x1f\n" - "sshr v20.4s, v20.4s, #0x1f\n" - "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" "sqrdmulh v26.4s, v26.4s, v2.4s\n" "sqrdmulh v27.4s, v27.4s, v2.4s\n" "sqrdmulh v28.4s, v28.4s, v2.4s\n" "sqrdmulh v29.4s, v29.4s, v2.4s\n" "sqrdmulh v30.4s, v30.4s, v2.4s\n" "sqrdmulh v31.4s, v31.4s, v2.4s\n" - "sqadd v23.4s, v23.4s, v21.4s\n" - "sqadd v24.4s, v24.4s, v20.4s\n" - "sqadd v25.4s, v25.4s, v19.4s\n" - "and v18.16b, v26.16b, v1.16b\n" - "and v17.16b, v27.16b, v1.16b\n" - "and v16.16b, v28.16b, v1.16b\n" - "and v21.16b, v29.16b, v1.16b\n" - "and v20.16b, v30.16b, v1.16b\n" - "and v19.16b, v31.16b, v1.16b\n" - "sshr v18.4s, v18.4s, #0x1f\n" - "sshr v17.4s, v17.4s, #0x1f\n" - "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v23.4s, v23.4s, v18.4s\n" + "sqadd v24.4s, v24.4s, v17.4s\n" + "sqadd v25.4s, v25.4s, v16.4s\n" + "and v21.16b, v26.16b, v1.16b\n" + "and v20.16b, v27.16b, v1.16b\n" + "and v19.16b, v28.16b, v1.16b\n" + "and v18.16b, v29.16b, v1.16b\n" + "and v17.16b, v30.16b, v1.16b\n" + "and v16.16b, v31.16b, v1.16b\n" "sshr v21.4s, v21.4s, #0x1f\n" "sshr v20.4s, v20.4s, #0x1f\n" "sshr v19.4s, v19.4s, #0x1f\n" - "sqadd v26.4s, v26.4s, v18.4s\n" - "sqadd v27.4s, v27.4s, v17.4s\n" - "sqadd v28.4s, v28.4s, v16.4s\n" - "sqadd v29.4s, v29.4s, v21.4s\n" - "sqadd v30.4s, v30.4s, v20.4s\n" - "sqadd v31.4s, v31.4s, v19.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v26.4s, v26.4s, v21.4s\n" + "sqadd v27.4s, v27.4s, v20.4s\n" + "sqadd v28.4s, v28.4s, v19.4s\n" + "sqadd v29.4s, v29.4s, v18.4s\n" + "sqadd v30.4s, v30.4s, v17.4s\n" + "sqadd v31.4s, v31.4s, v16.4s\n" "srshl v23.4s, v23.4s, v1.4s\n" "srshl v24.4s, v24.4s, v1.4s\n" "srshl v25.4s, v25.4s, v1.4s\n" @@ -270,7 +271,7 @@ void a64_u8q_nhwc_generic_output9_mla_depthfirst_impl( "str s30, [x21, x11]\n" "str s31, [x20, x11]\n" "add x11, x11, #0x4\n" - "cmp x11, x12, LSL #2\n" + "cmp x11, x9, LSL #2\n" "blt 1b\n" "6:" // Oddments "tst %x[n_channels], #0x3\n" @@ -288,61 +289,61 @@ void a64_u8q_nhwc_generic_output9_mla_depthfirst_impl( "8:" // Oddments: Load bias: Bit 1: End "9:" // Oddments: Load bias: Done "ldr s0, [%x[params]], #0x4\n" - "mov x21, %x[inptrs]\n" - "ldp x10, x9, [x21], #0x10\n" + "mov x10, %x[inptrs]\n" + "ldp x9, x28, [x10], #0x10\n" "mov v24.16b, v23.16b\n" - "ldp x28, x27, [x21], #0x10\n" - "ldp x26, x25, [x21], #0x10\n" + "ldp x27, x26, [x10], #0x10\n" + "ldp x25, x24, [x10], #0x10\n" "mov v25.16b, v23.16b\n" "mov v26.16b, v23.16b\n" - "ldp x24, x23, [x21], #0x10\n" - "ldr x22, [x21], #0x8\n" + "ldp x23, x22, [x10], #0x10\n" + "ldr x21, [x10], #0x8\n" "mov v27.16b, v23.16b\n" "mov v28.16b, v23.16b\n" "mov v29.16b, v23.16b\n" "mov v30.16b, v23.16b\n" - "add x10, x10, x11\n" "add x9, x9, x11\n" + "add x28, x28, x11\n" "mov v31.16b, v23.16b\n" "usubl v0.8h, v0.8b, v5.8b\n" - "add x28, x28, x11\n" "add x27, x27, x11\n" "add x26, x26, x11\n" "add x25, x25, x11\n" "add x24, x24, x11\n" "add x23, x23, x11\n" "add x22, x22, x11\n" + "add x21, x21, x11\n" "tbz %x[n_channels], #1, 10f\n" - "ldr h14, [x10], #0x2\n" - "ldr h15, [x9], #0x2\n" - "ldr h16, [x28], #0x2\n" - "ldr h17, [x27], #0x2\n" - "ldr h18, [x26], #0x2\n" - "ldr h19, [x25], #0x2\n" - "ldr h20, [x24], #0x2\n" - "ldr h21, [x23], #0x2\n" - "ldr h22, [x22], #0x2\n" + "ldr h14, [x9], #0x2\n" + "ldr h15, [x28], #0x2\n" + "ldr h16, [x27], #0x2\n" + "ldr h17, [x26], #0x2\n" + "ldr h18, [x25], #0x2\n" + "ldr h19, [x24], #0x2\n" + "ldr h20, [x23], #0x2\n" + "ldr h21, [x22], #0x2\n" + "ldr h22, [x21], #0x2\n" "tbz %x[n_channels], #0, 11f\n" - "ld1 { v14.b }[2], [x10], #0x1\n" - "ld1 { v15.b }[2], [x9], #0x1\n" - "ld1 { v16.b }[2], [x28], #0x1\n" - "ld1 { v17.b }[2], [x27], #0x1\n" - "ld1 { v18.b }[2], [x26], #0x1\n" - "ld1 { v19.b }[2], [x25], #0x1\n" - "ld1 { v20.b }[2], [x24], #0x1\n" - "ld1 { v21.b }[2], [x23], #0x1\n" - "ld1 { v22.b }[2], [x22], #0x1\n" + "ld1 { v14.b }[2], [x9], #0x1\n" + "ld1 { v15.b }[2], [x28], #0x1\n" + "ld1 { v16.b }[2], [x27], #0x1\n" + "ld1 { v17.b }[2], [x26], #0x1\n" + "ld1 { v18.b }[2], [x25], #0x1\n" + "ld1 { v19.b }[2], [x24], #0x1\n" + "ld1 { v20.b }[2], [x23], #0x1\n" + "ld1 { v21.b }[2], [x22], #0x1\n" + "ld1 { v22.b }[2], [x21], #0x1\n" "b 11f\n" "10:" // Oddments: Load: Bit 1: Unset - "ldr b14, [x10], #0x1\n" - "ldr b15, [x9], #0x1\n" - "ldr b16, [x28], #0x1\n" - "ldr b17, [x27], #0x1\n" - "ldr b18, [x26], #0x1\n" - "ldr b19, [x25], #0x1\n" - "ldr b20, [x24], #0x1\n" - "ldr b21, [x23], #0x1\n" - "ldr b22, [x22], #0x1\n" + "ldr b14, [x9], #0x1\n" + "ldr b15, [x28], #0x1\n" + "ldr b16, [x27], #0x1\n" + "ldr b17, [x26], #0x1\n" + "ldr b18, [x25], #0x1\n" + "ldr b19, [x24], #0x1\n" + "ldr b20, [x23], #0x1\n" + "ldr b21, [x22], #0x1\n" + "ldr b22, [x21], #0x1\n" "11:" // Oddments: Load: Bit 1: End "subs x20, %x[n_points], #0x1\n" "usubl v14.8h, v14.8b, v6.8b\n" @@ -356,62 +357,62 @@ void a64_u8q_nhwc_generic_output9_mla_depthfirst_impl( "usubl v22.8h, v22.8b, v6.8b\n" "ble 15f\n" "12:" // Oddments: Planar loop - "ldp x10, x9, [x21], #0x10\n" - "ldp x28, x27, [x21], #0x10\n" + "ldp x9, x28, [x10], #0x10\n" + "ldp x27, x26, [x10], #0x10\n" "smlal v23.4s, v14.4h, v0.4h\n" "smlal v24.4s, v15.4h, v0.4h\n" - "ldp x26, x25, [x21], #0x10\n" - "ldp x24, x23, [x21], #0x10\n" + "ldp x25, x24, [x10], #0x10\n" + "ldp x23, x22, [x10], #0x10\n" "smlal v25.4s, v16.4h, v0.4h\n" "smlal v26.4s, v17.4h, v0.4h\n" "smlal v27.4s, v18.4h, v0.4h\n" "smlal v28.4s, v19.4h, v0.4h\n" - "ldr x22, [x21], #0x8\n" - "add x10, x10, x11\n" + "ldr x21, [x10], #0x8\n" + "add x9, x9, x11\n" "smlal v29.4s, v20.4h, v0.4h\n" "smlal v30.4s, v21.4h, v0.4h\n" - "add x9, x9, x11\n" "add x28, x28, x11\n" + "add x27, x27, x11\n" "smlal v31.4s, v22.4h, v0.4h\n" "ldr s0, [%x[params]], #0x4\n" "usubl v0.8h, v0.8b, v5.8b\n" - "add x27, x27, x11\n" "add x26, x26, x11\n" "add x25, x25, x11\n" "add x24, x24, x11\n" "add x23, x23, x11\n" "add x22, x22, x11\n" + "add x21, x21, x11\n" "tbz %x[n_channels], #1, 13f\n" - "ldr h14, [x10], #0x2\n" - "ldr h15, [x9], #0x2\n" - "ldr h16, [x28], #0x2\n" - "ldr h17, [x27], #0x2\n" - "ldr h18, [x26], #0x2\n" - "ldr h19, [x25], #0x2\n" - "ldr h20, [x24], #0x2\n" - "ldr h21, [x23], #0x2\n" - "ldr h22, [x22], #0x2\n" + "ldr h14, [x9], #0x2\n" + "ldr h15, [x28], #0x2\n" + "ldr h16, [x27], #0x2\n" + "ldr h17, [x26], #0x2\n" + "ldr h18, [x25], #0x2\n" + "ldr h19, [x24], #0x2\n" + "ldr h20, [x23], #0x2\n" + "ldr h21, [x22], #0x2\n" + "ldr h22, [x21], #0x2\n" "tbz %x[n_channels], #0, 14f\n" - "ld1 { v14.b }[2], [x10], #0x1\n" - "ld1 { v15.b }[2], [x9], #0x1\n" - "ld1 { v16.b }[2], [x28], #0x1\n" - "ld1 { v17.b }[2], [x27], #0x1\n" - "ld1 { v18.b }[2], [x26], #0x1\n" - "ld1 { v19.b }[2], [x25], #0x1\n" - "ld1 { v20.b }[2], [x24], #0x1\n" - "ld1 { v21.b }[2], [x23], #0x1\n" - "ld1 { v22.b }[2], [x22], #0x1\n" + "ld1 { v14.b }[2], [x9], #0x1\n" + "ld1 { v15.b }[2], [x28], #0x1\n" + "ld1 { v16.b }[2], [x27], #0x1\n" + "ld1 { v17.b }[2], [x26], #0x1\n" + "ld1 { v18.b }[2], [x25], #0x1\n" + "ld1 { v19.b }[2], [x24], #0x1\n" + "ld1 { v20.b }[2], [x23], #0x1\n" + "ld1 { v21.b }[2], [x22], #0x1\n" + "ld1 { v22.b }[2], [x21], #0x1\n" "b 14f\n" "13:" // Oddments: Planar loop: Load: Bit 1: Unset - "ldr b14, [x10], #0x1\n" - "ldr b15, [x9], #0x1\n" - "ldr b16, [x28], #0x1\n" - "ldr b17, [x27], #0x1\n" - "ldr b18, [x26], #0x1\n" - "ldr b19, [x25], #0x1\n" - "ldr b20, [x24], #0x1\n" - "ldr b21, [x23], #0x1\n" - "ldr b22, [x22], #0x1\n" + "ldr b14, [x9], #0x1\n" + "ldr b15, [x28], #0x1\n" + "ldr b16, [x27], #0x1\n" + "ldr b17, [x26], #0x1\n" + "ldr b18, [x25], #0x1\n" + "ldr b19, [x24], #0x1\n" + "ldr b20, [x23], #0x1\n" + "ldr b21, [x22], #0x1\n" + "ldr b22, [x21], #0x1\n" "14:" // Oddments: Planar loop: Load: Bit 1: End "subs x20, x20, #0x1\n" "usubl v14.8h, v14.8b, v6.8b\n" @@ -457,9 +458,7 @@ void a64_u8q_nhwc_generic_output9_mla_depthfirst_impl( "cbz %x[rq_left_shift_ptr], 19f\n" "ld1 { v3.s }[0], [x20], #0x4\n" "19:" // Oddments: Load quantisation parameters: Bit 1: Unset: Bit 0: Load left shift: Done - "20:" // Oddments: Load quantisation parameters: Bit 1: End - "21:" // Oddments: Load quantisation parameters: Done "sshl v23.4s, v23.4s, v3.4s\n" "sshl v24.4s, v24.4s, v3.4s\n" @@ -473,11 +472,11 @@ void a64_u8q_nhwc_generic_output9_mla_depthfirst_impl( "sqrdmulh v25.4s, v25.4s, v2.4s\n" "ldr x20, [%x[outptrs], #0x40]\n" "add x28, x28, x11\n" - "and v21.16b, v23.16b, v1.16b\n" - "and v20.16b, v24.16b, v1.16b\n" + "and v18.16b, v23.16b, v1.16b\n" + "and v17.16b, v24.16b, v1.16b\n" "add x27, x27, x11\n" "add x26, x26, x11\n" - "and v19.16b, v25.16b, v1.16b\n" + "and v16.16b, v25.16b, v1.16b\n" "sshl v26.4s, v26.4s, v3.4s\n" "add x25, x25, x11\n" "add x24, x24, x11\n" @@ -490,36 +489,36 @@ void a64_u8q_nhwc_generic_output9_mla_depthfirst_impl( "add x21, x21, x11\n" "add x20, x20, x11\n" "sshl v31.4s, v31.4s, v3.4s\n" - "sshr v21.4s, v21.4s, #0x1f\n" - "sshr v20.4s, v20.4s, #0x1f\n" - "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" "sqrdmulh v26.4s, v26.4s, v2.4s\n" "sqrdmulh v27.4s, v27.4s, v2.4s\n" "sqrdmulh v28.4s, v28.4s, v2.4s\n" "sqrdmulh v29.4s, v29.4s, v2.4s\n" "sqrdmulh v30.4s, v30.4s, v2.4s\n" "sqrdmulh v31.4s, v31.4s, v2.4s\n" - "sqadd v23.4s, v23.4s, v21.4s\n" - "sqadd v24.4s, v24.4s, v20.4s\n" - "sqadd v25.4s, v25.4s, v19.4s\n" - "and v18.16b, v26.16b, v1.16b\n" - "and v17.16b, v27.16b, v1.16b\n" - "and v16.16b, v28.16b, v1.16b\n" - "and v21.16b, v29.16b, v1.16b\n" - "and v20.16b, v30.16b, v1.16b\n" - "and v19.16b, v31.16b, v1.16b\n" - "sshr v18.4s, v18.4s, #0x1f\n" - "sshr v17.4s, v17.4s, #0x1f\n" - "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v23.4s, v23.4s, v18.4s\n" + "sqadd v24.4s, v24.4s, v17.4s\n" + "sqadd v25.4s, v25.4s, v16.4s\n" + "and v21.16b, v26.16b, v1.16b\n" + "and v20.16b, v27.16b, v1.16b\n" + "and v19.16b, v28.16b, v1.16b\n" + "and v18.16b, v29.16b, v1.16b\n" + "and v17.16b, v30.16b, v1.16b\n" + "and v16.16b, v31.16b, v1.16b\n" "sshr v21.4s, v21.4s, #0x1f\n" "sshr v20.4s, v20.4s, #0x1f\n" "sshr v19.4s, v19.4s, #0x1f\n" - "sqadd v26.4s, v26.4s, v18.4s\n" - "sqadd v27.4s, v27.4s, v17.4s\n" - "sqadd v28.4s, v28.4s, v16.4s\n" - "sqadd v29.4s, v29.4s, v21.4s\n" - "sqadd v30.4s, v30.4s, v20.4s\n" - "sqadd v31.4s, v31.4s, v19.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v26.4s, v26.4s, v21.4s\n" + "sqadd v27.4s, v27.4s, v20.4s\n" + "sqadd v28.4s, v28.4s, v19.4s\n" + "sqadd v29.4s, v29.4s, v18.4s\n" + "sqadd v30.4s, v30.4s, v17.4s\n" + "sqadd v31.4s, v31.4s, v16.4s\n" "srshl v23.4s, v23.4s, v1.4s\n" "srshl v24.4s, v24.4s, v1.4s\n" "srshl v25.4s, v25.4s, v1.4s\n" @@ -606,15 +605,14 @@ void a64_u8q_nhwc_generic_output9_mla_depthfirst_impl( "st1 { v30.b }[0], [x21], #0x1\n" "st1 { v31.b }[0], [x20], #0x1\n" "23:" // Oddments: Store: Bit 1: End - "24:" // End - : [params] "+&r" (params) : [bias] "r" (qp.bias), [inptrs] "r" (inptrs), [n_channels] "r" ((uint64_t) n_channels), [n_points] "r" ((uint64_t) n_points), [offsetof_Requantize32_a_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [offsetof_Requantize32_per_layer_left_shift] "I" (offsetof(arm_gemm::Requantize32, per_layer_left_shift)), [offsetof_Requantize32_per_layer_mul] "I" (offsetof(arm_gemm::Requantize32, per_layer_mul)), [offsetof_Requantize32_per_layer_right_shift] "I" (offsetof(arm_gemm::Requantize32, per_layer_right_shift)), [outptrs] "r" (outptrs), [qp] "r" (&qp), [rq_left_shift_ptr] "r" (qp.per_channel_left_shifts), [rq_mul_ptr] "r" (qp.per_channel_muls), [rq_right_shift_ptr] "r" (qp.per_channel_right_shifts) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst.hpp index 134f657fb88aa3f51fbcaf64a75a50f2f4ef80e2..76965606f7cf6592064af7c376085438695a9681 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include @@ -57,4 +57,5 @@ struct a64_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst : } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst/generic.cpp index a6dba90f9eb588673f867e9598b8711dd27d607d..d69f391514315148c0fa3b2d6468ace56186d6db 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst/generic.cpp @@ -22,12 +22,13 @@ * SOFTWARE. */ -#if defined(__aarch64__) #include "arm_gemm.hpp" #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace depthwise { @@ -40,169 +41,169 @@ void a64_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst_impl ) { __asm__ __volatile__( - "ldr q14, [%x[params], #0x0]\n" + "ldr q11, [%x[params], #0x0]\n" "ldr q5, [%x[params], #0x10]\n" - "movi v15.16b, #0x1\n" - "ushr v15.4s, v15.4s, #0x8\n" + "movi v8.16b, #0x1\n" + "ushr v8.4s, v8.4s, #0x8\n" "ldr q6, [%x[params], #0x20]\n" "ldr q7, [%x[params], #0x30]\n" - "movi v26.4s, #0x0\n" - "movi v27.4s, #0x0\n" + "movi v24.4s, #0x0\n" + "movi v25.4s, #0x0\n" "ldr x20, [%x[inptrs], #0x8]\n" "ld1 { v1.16b }, [x20]\n" - "mov v29.16b, v1.16b\n" - "mov v16.16b, v1.16b\n" + "mov v28.16b, v1.16b\n" + "mov v23.16b, v1.16b\n" "ldr x20, [%x[inptrs], #0x10]\n" "ld1 { v2.16b }, [x20]\n" - "mov v28.16b, v1.16b\n" - "mov v22.16b, v2.16b\n" + "mov v30.16b, v1.16b\n" + "mov v21.16b, v2.16b\n" "ldr x20, [%x[inptrs], #0x20]\n" "ld1 { v4.16b }, [x20]\n" - "mov v31.16b, v2.16b\n" - "mov v30.16b, v2.16b\n" + "mov v20.16b, v2.16b\n" + "mov v29.16b, v2.16b\n" "ldr x20, [%x[inptrs], #0x0]\n" "ld1 { v0.16b }, [x20]\n" - "mov v23.16b, v4.16b\n" - "mov v21.16b, v4.16b\n" + "mov v9.16b, v4.16b\n" + "mov v22.16b, v4.16b\n" "ldr x20, [%x[inptrs], #0x18]\n" "ld1 { v3.16b }, [x20]\n" - "mov v20.16b, v4.16b\n" - "ext v29.16b, v29.16b, v29.16b, #0x2\n" - "ext v16.16b, v16.16b, v16.16b, #0x4\n" - "ext v28.16b, v28.16b, v28.16b, #0x6\n" + "mov v31.16b, v4.16b\n" + "ext v28.16b, v28.16b, v28.16b, #0x2\n" + "ext v23.16b, v23.16b, v23.16b, #0x4\n" + "ext v30.16b, v30.16b, v30.16b, #0x6\n" "add x20, %x[qp], %[offsetof_Requantize32_b_offset]\n" - "ld1r { v13.4s }, [x20]\n" - "ext v22.16b, v22.16b, v22.16b, #0x2\n" - "ext v31.16b, v31.16b, v31.16b, #0x4\n" - "add x20, %x[qp], %[offsetof_Requantize32_c_offset]\n" "ld1r { v12.4s }, [x20]\n" - "ext v30.16b, v30.16b, v30.16b, #0x6\n" - "ext v23.16b, v23.16b, v23.16b, #0x2\n" + "ext v21.16b, v21.16b, v21.16b, #0x2\n" + "ext v20.16b, v20.16b, v20.16b, #0x4\n" + "add x20, %x[qp], %[offsetof_Requantize32_c_offset]\n" + "ld1r { v14.4s }, [x20]\n" + "ext v29.16b, v29.16b, v29.16b, #0x6\n" + "ext v9.16b, v9.16b, v9.16b, #0x2\n" "add x20, %x[qp], %[offsetof_Requantize32_minval]\n" - "ld1r { v11.4s }, [x20]\n" - "ext v21.16b, v21.16b, v21.16b, #0x4\n" - "ext v20.16b, v20.16b, v20.16b, #0x6\n" + "ld1r { v13.4s }, [x20]\n" + "ext v22.16b, v22.16b, v22.16b, #0x4\n" + "ext v31.16b, v31.16b, v31.16b, #0x6\n" "add x20, %x[qp], %[offsetof_Requantize32_maxval]\n" - "ld1r { v10.4s }, [x20]\n" - "mov v25.16b, v0.16b\n" + "ld1r { v15.4s }, [x20]\n" + "mov v27.16b, v0.16b\n" "mov v19.16b, v0.16b\n" "cmp %x[n_channels], #0x4\n" "mov x9, #0x0\n" "mov v18.16b, v0.16b\n" - "mov v24.16b, v3.16b\n" + "mov v26.16b, v3.16b\n" "mov x28, #0x0\n" "ldp x27, x26, [%x[outptrs], #0x0]\n" "mov v17.16b, v3.16b\n" - "ext v25.16b, v25.16b, v25.16b, #0x2\n" + "mov v16.16b, v3.16b\n" "ldp x25, x24, [%x[outptrs], #0x10]\n" "ldp x23, x22, [%x[outptrs], #0x20]\n" + "ext v27.16b, v27.16b, v27.16b, #0x2\n" "ext v19.16b, v19.16b, v19.16b, #0x4\n" - "ext v18.16b, v18.16b, v18.16b, #0x6\n" "ldp x21, x20, [%x[outptrs], #0x30]\n" "add %x[params], %x[params], #0x40\n" - "zip1 v1.4s, v1.4s, v16.4s\n" - "mov v16.16b, v3.16b\n" - "zip1 v29.4s, v29.4s, v28.4s\n" - "zip1 v2.4s, v2.4s, v31.4s\n" - "zip1 v22.4s, v22.4s, v30.4s\n" - "ext v24.16b, v24.16b, v24.16b, #0x2\n" + "ext v18.16b, v18.16b, v18.16b, #0x6\n" + "zip1 v1.4s, v1.4s, v23.4s\n" + "zip1 v28.4s, v28.4s, v30.4s\n" + "zip1 v2.4s, v2.4s, v20.4s\n" + "zip1 v21.4s, v21.4s, v29.4s\n" + "ext v26.16b, v26.16b, v26.16b, #0x2\n" "ext v17.16b, v17.16b, v17.16b, #0x4\n" "ext v16.16b, v16.16b, v16.16b, #0x6\n" - "zip1 v4.4s, v4.4s, v21.4s\n" - "zip1 v23.4s, v23.4s, v20.4s\n" + "zip1 v4.4s, v4.4s, v22.4s\n" + "zip1 v9.4s, v9.4s, v31.4s\n" "zip1 v0.4s, v0.4s, v19.4s\n" - "zip1 v25.4s, v25.4s, v18.4s\n" - "zip1 v1.4s, v1.4s, v29.4s\n" - "zip1 v2.4s, v2.4s, v22.4s\n" - ".inst 0x6f81e1fa // udot v26.4s, v15.16b, v1.4b[0]\n" + "zip1 v27.4s, v27.4s, v18.4s\n" + "zip1 v1.4s, v1.4s, v28.4s\n" + "zip1 v2.4s, v2.4s, v21.4s\n" + ".inst 0x6f81e118 // udot v24.4s, v8.16b, v1.4b[0]\n" "zip1 v3.4s, v3.4s, v17.4s\n" - "zip1 v24.4s, v24.4s, v16.4s\n" - ".inst 0x6fa1e1fb // udot v27.4s, v15.16b, v1.4b[1]\n" - "zip1 v4.4s, v4.4s, v23.4s\n" + "zip1 v26.4s, v26.4s, v16.4s\n" + ".inst 0x6fa1e119 // udot v25.4s, v8.16b, v1.4b[1]\n" + "zip1 v4.4s, v4.4s, v9.4s\n" "movi v23.4s, #0x0\n" - ".inst 0x6f81e9f7 // udot v23.4s, v15.16b, v1.4b[2]\n" + ".inst 0x6f81e917 // udot v23.4s, v8.16b, v1.4b[2]\n" "movi v22.4s, #0x0\n" "movi v21.4s, #0x0\n" - ".inst 0x6fa1e9f6 // udot v22.4s, v15.16b, v1.4b[3]\n" - "movi v20.4s, #0x0\n" - "movi v9.4s, #0x0\n" - ".inst 0x6f82e1f5 // udot v21.4s, v15.16b, v2.4b[0]\n" - "movi v8.4s, #0x0\n" + ".inst 0x6fa1e916 // udot v22.4s, v8.16b, v1.4b[3]\n" "movi v19.4s, #0x0\n" - ".inst 0x6fa2e1f4 // udot v20.4s, v15.16b, v2.4b[1]\n" + "movi v9.4s, #0x0\n" + ".inst 0x6f82e115 // udot v21.4s, v8.16b, v2.4b[0]\n" + "movi v10.4s, #0x0\n" + "movi v20.4s, #0x0\n" + ".inst 0x6fa2e113 // udot v19.4s, v8.16b, v2.4b[1]\n" "movi v18.4s, #0x0\n" "movi v17.4s, #0x0\n" - ".inst 0x6f82e9e9 // udot v9.4s, v15.16b, v2.4b[2]\n" + ".inst 0x6f82e909 // udot v9.4s, v8.16b, v2.4b[2]\n" "movi v16.4s, #0x0\n" - "zip1 v0.4s, v0.4s, v25.4s\n" - ".inst 0x6fa2e9e8 // udot v8.4s, v15.16b, v2.4b[3]\n" - "zip1 v3.4s, v3.4s, v24.4s\n" - ".inst 0x6f84e1f3 // udot v19.4s, v15.16b, v4.4b[0]\n" - ".inst 0x6fa4e1f2 // udot v18.4s, v15.16b, v4.4b[1]\n" - ".inst 0x6f84e9f1 // udot v17.4s, v15.16b, v4.4b[2]\n" - ".inst 0x6fa4e9f0 // udot v16.4s, v15.16b, v4.4b[3]\n" + "zip1 v0.4s, v0.4s, v27.4s\n" + ".inst 0x6fa2e90a // udot v10.4s, v8.16b, v2.4b[3]\n" + "zip1 v3.4s, v3.4s, v26.4s\n" + ".inst 0x6f84e114 // udot v20.4s, v8.16b, v4.4b[0]\n" + ".inst 0x6fa4e112 // udot v18.4s, v8.16b, v4.4b[1]\n" + ".inst 0x6f84e911 // udot v17.4s, v8.16b, v4.4b[2]\n" + ".inst 0x6fa4e910 // udot v16.4s, v8.16b, v4.4b[3]\n" "movi v31.4s, #0x0\n" "movi v30.4s, #0x0\n" - "movi v29.4s, #0x0\n" - ".inst 0x6f80e1ff // udot v31.4s, v15.16b, v0.4b[0]\n" + "movi v26.4s, #0x0\n" + ".inst 0x6f80e11f // udot v31.4s, v8.16b, v0.4b[0]\n" + "movi v27.4s, #0x0\n" "movi v28.4s, #0x0\n" - ".inst 0x6fa0e1fe // udot v30.4s, v15.16b, v0.4b[1]\n" - ".inst 0x6f80e9fd // udot v29.4s, v15.16b, v0.4b[2]\n" - ".inst 0x6fa0e9fc // udot v28.4s, v15.16b, v0.4b[3]\n" - "add v24.4s, v26.4s, v21.4s\n" - "add v25.4s, v27.4s, v20.4s\n" - "add v26.4s, v23.4s, v9.4s\n" - "add v27.4s, v22.4s, v8.4s\n" - "add v23.4s, v19.4s, v21.4s\n" - "movi v22.4s, #0x0\n" - ".inst 0x6f83e1f6 // udot v22.4s, v15.16b, v3.4b[0]\n" - "add v21.4s, v18.4s, v20.4s\n" + ".inst 0x6fa0e11e // udot v30.4s, v8.16b, v0.4b[1]\n" + "movi v29.4s, #0x0\n" + ".inst 0x6f80e91a // udot v26.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6fa0e91b // udot v27.4s, v8.16b, v0.4b[3]\n" + ".inst 0x6f83e11c // udot v28.4s, v8.16b, v3.4b[0]\n" + ".inst 0x6fa3e11d // udot v29.4s, v8.16b, v3.4b[1]\n" + "add v24.4s, v24.4s, v21.4s\n" + "add v25.4s, v25.4s, v19.4s\n" + "add v23.4s, v23.4s, v9.4s\n" + "add v22.4s, v22.4s, v10.4s\n" + "add v21.4s, v20.4s, v21.4s\n" "movi v20.4s, #0x0\n" - ".inst 0x6fa3e1f4 // udot v20.4s, v15.16b, v3.4b[1]\n" - "add v19.4s, v17.4s, v9.4s\n" + ".inst 0x6f83e914 // udot v20.4s, v8.16b, v3.4b[2]\n" + "add v19.4s, v18.4s, v19.4s\n" "movi v18.4s, #0x0\n" - ".inst 0x6f83e9f2 // udot v18.4s, v15.16b, v3.4b[2]\n" - "add v17.4s, v16.4s, v8.4s\n" - "movi v16.4s, #0x0\n" - ".inst 0x6fa3e9f0 // udot v16.4s, v15.16b, v3.4b[3]\n" + ".inst 0x6fa3e912 // udot v18.4s, v8.16b, v3.4b[3]\n" + "add v17.4s, v17.4s, v9.4s\n" + "add v16.4s, v16.4s, v10.4s\n" "add v24.4s, v24.4s, v31.4s\n" "add v25.4s, v25.4s, v30.4s\n" - "add v26.4s, v26.4s, v29.4s\n" - "add v27.4s, v27.4s, v28.4s\n" - "add v28.4s, v23.4s, v22.4s\n" - "add v29.4s, v21.4s, v20.4s\n" - "add v30.4s, v19.4s, v18.4s\n" - "add v31.4s, v17.4s, v16.4s\n" - "neg v13.4s, v13.4s\n" - "mul v24.4s, v24.4s, v13.4s\n" - "mul v25.4s, v25.4s, v13.4s\n" - "mul v26.4s, v26.4s, v13.4s\n" - "mul v27.4s, v27.4s, v13.4s\n" - "mul v28.4s, v28.4s, v13.4s\n" - "mul v29.4s, v29.4s, v13.4s\n" - "mul v30.4s, v30.4s, v13.4s\n" - "mul v31.4s, v31.4s, v13.4s\n" + "add v26.4s, v23.4s, v26.4s\n" + "add v27.4s, v22.4s, v27.4s\n" + "add v28.4s, v21.4s, v28.4s\n" + "add v29.4s, v19.4s, v29.4s\n" + "add v30.4s, v17.4s, v20.4s\n" + "add v31.4s, v16.4s, v18.4s\n" + "neg v12.4s, v12.4s\n" + "mul v24.4s, v24.4s, v12.4s\n" + "mul v25.4s, v25.4s, v12.4s\n" + "mul v26.4s, v26.4s, v12.4s\n" + "mul v27.4s, v27.4s, v12.4s\n" + "mul v28.4s, v28.4s, v12.4s\n" + "mul v29.4s, v29.4s, v12.4s\n" + "mul v30.4s, v30.4s, v12.4s\n" + "mul v31.4s, v31.4s, v12.4s\n" "zip1 v19.4s, v24.4s, v26.4s\n" "zip1 v18.4s, v25.4s, v27.4s\n" "zip1 v17.4s, v28.4s, v30.4s\n" "zip1 v16.4s, v29.4s, v31.4s\n" "zip1 v22.4s, v19.4s, v18.4s\n" "zip1 v23.4s, v17.4s, v16.4s\n" - "add v24.4s, v24.4s, v14.4s\n" - "add v25.4s, v25.4s, v14.4s\n" - "add v26.4s, v26.4s, v14.4s\n" - "add v27.4s, v27.4s, v14.4s\n" - "add v28.4s, v28.4s, v14.4s\n" - "add v29.4s, v29.4s, v14.4s\n" - "add v30.4s, v30.4s, v14.4s\n" - "add v31.4s, v31.4s, v14.4s\n" + "add v24.4s, v24.4s, v11.4s\n" + "add v25.4s, v25.4s, v11.4s\n" + "add v26.4s, v26.4s, v11.4s\n" + "add v27.4s, v27.4s, v11.4s\n" + "add v28.4s, v28.4s, v11.4s\n" + "add v29.4s, v29.4s, v11.4s\n" + "add v30.4s, v30.4s, v11.4s\n" + "add v31.4s, v31.4s, v11.4s\n" "ble 2f\n" "1:" // Loop - "ldr q21, [%x[params], #0x0]\n" - "ldr q20, [%x[params], #0x10]\n" + "ldr q8, [%x[params], #0x0]\n" + "ldr q21, [%x[params], #0x10]\n" ".inst 0x6f80e0b8 // udot v24.4s, v5.16b, v0.4b[0]\n" ".inst 0x6fa0e0b9 // udot v25.4s, v5.16b, v0.4b[1]\n" - "ldr q14, [%x[params], #0x20]\n" + "ldr q20, [%x[params], #0x20]\n" ".inst 0x6f80e8ba // udot v26.4s, v5.16b, v0.4b[2]\n" ".inst 0x6fa0e8bb // udot v27.4s, v5.16b, v0.4b[3]\n" "sub %x[n_channels], %x[n_channels], #0x4\n" @@ -219,43 +220,43 @@ void a64_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst_impl "ldr q5, [%x[params], #0x30]\n" ".inst 0x6f82e0f8 // udot v24.4s, v7.16b, v2.4b[0]\n" ".inst 0x6fa2e0f9 // udot v25.4s, v7.16b, v2.4b[1]\n" - "sqrdmulh v24.4s, v24.4s, v21.4s\n" + "sqrdmulh v24.4s, v24.4s, v8.4s\n" ".inst 0x6f82e8fa // udot v26.4s, v7.16b, v2.4b[2]\n" ".inst 0x6fa2e8fb // udot v27.4s, v7.16b, v2.4b[3]\n" - "sqrdmulh v25.4s, v25.4s, v21.4s\n" + "sqrdmulh v25.4s, v25.4s, v8.4s\n" ".inst 0x6f83e0dc // udot v28.4s, v6.16b, v3.4b[0]\n" ".inst 0x6fa3e0dd // udot v29.4s, v6.16b, v3.4b[1]\n" - "sqrdmulh v26.4s, v26.4s, v21.4s\n" + "sqrdmulh v26.4s, v26.4s, v8.4s\n" ".inst 0x6f83e8de // udot v30.4s, v6.16b, v3.4b[2]\n" ".inst 0x6fa3e8df // udot v31.4s, v6.16b, v3.4b[3]\n" "ldr q6, [%x[params], #0x40]\n" - "sqrdmulh v27.4s, v27.4s, v21.4s\n" + "sqrdmulh v27.4s, v27.4s, v8.4s\n" ".inst 0x6f84e0fc // udot v28.4s, v7.16b, v4.4b[0]\n" ".inst 0x6fa4e0fd // udot v29.4s, v7.16b, v4.4b[1]\n" - "and v19.16b, v24.16b, v20.16b\n" + "and v19.16b, v24.16b, v21.16b\n" ".inst 0x6f84e8fe // udot v30.4s, v7.16b, v4.4b[2]\n" ".inst 0x6fa4e8ff // udot v31.4s, v7.16b, v4.4b[3]\n" "ldr q7, [%x[params], #0x50]\n" - "and v18.16b, v25.16b, v20.16b\n" - "and v17.16b, v26.16b, v20.16b\n" - "and v16.16b, v27.16b, v20.16b\n" + "and v18.16b, v25.16b, v21.16b\n" + "and v17.16b, v26.16b, v21.16b\n" + "and v16.16b, v27.16b, v21.16b\n" "add %x[params], %x[params], #0x60\n" "sshr v19.4s, v19.4s, #0x1f\n" "sshr v18.4s, v18.4s, #0x1f\n" "sshr v17.4s, v17.4s, #0x1f\n" "sshr v16.4s, v16.4s, #0x1f\n" - "sqrdmulh v28.4s, v28.4s, v21.4s\n" - "sqrdmulh v29.4s, v29.4s, v21.4s\n" - "sqrdmulh v30.4s, v30.4s, v21.4s\n" - "sqrdmulh v31.4s, v31.4s, v21.4s\n" + "sqrdmulh v28.4s, v28.4s, v8.4s\n" + "sqrdmulh v29.4s, v29.4s, v8.4s\n" + "sqrdmulh v30.4s, v30.4s, v8.4s\n" + "sqrdmulh v31.4s, v31.4s, v8.4s\n" "sqadd v24.4s, v24.4s, v19.4s\n" "sqadd v25.4s, v25.4s, v18.4s\n" "sqadd v26.4s, v26.4s, v17.4s\n" "sqadd v27.4s, v27.4s, v16.4s\n" - "and v19.16b, v28.16b, v20.16b\n" - "and v18.16b, v29.16b, v20.16b\n" - "and v17.16b, v30.16b, v20.16b\n" - "and v16.16b, v31.16b, v20.16b\n" + "and v19.16b, v28.16b, v21.16b\n" + "and v18.16b, v29.16b, v21.16b\n" + "and v17.16b, v30.16b, v21.16b\n" + "and v16.16b, v31.16b, v21.16b\n" "sshr v19.4s, v19.4s, #0x1f\n" "sshr v18.4s, v18.4s, #0x1f\n" "sshr v17.4s, v17.4s, #0x1f\n" @@ -264,38 +265,38 @@ void a64_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst_impl "sqadd v29.4s, v29.4s, v18.4s\n" "sqadd v30.4s, v30.4s, v17.4s\n" "sqadd v31.4s, v31.4s, v16.4s\n" - "srshl v24.4s, v24.4s, v20.4s\n" - "srshl v25.4s, v25.4s, v20.4s\n" - "srshl v26.4s, v26.4s, v20.4s\n" - "srshl v27.4s, v27.4s, v20.4s\n" - "srshl v28.4s, v28.4s, v20.4s\n" - "srshl v29.4s, v29.4s, v20.4s\n" - "srshl v30.4s, v30.4s, v20.4s\n" - "srshl v31.4s, v31.4s, v20.4s\n" - "add v24.4s, v24.4s, v12.4s\n" - "add v25.4s, v25.4s, v12.4s\n" - "add v26.4s, v26.4s, v12.4s\n" - "add v27.4s, v27.4s, v12.4s\n" - "add v28.4s, v28.4s, v12.4s\n" - "add v29.4s, v29.4s, v12.4s\n" - "add v30.4s, v30.4s, v12.4s\n" - "add v31.4s, v31.4s, v12.4s\n" - "smin v24.4s, v24.4s, v10.4s\n" - "smin v25.4s, v25.4s, v10.4s\n" - "smin v26.4s, v26.4s, v10.4s\n" - "smin v27.4s, v27.4s, v10.4s\n" - "smin v28.4s, v28.4s, v10.4s\n" - "smin v29.4s, v29.4s, v10.4s\n" - "smin v30.4s, v30.4s, v10.4s\n" - "smin v31.4s, v31.4s, v10.4s\n" - "smax v24.4s, v24.4s, v11.4s\n" - "smax v25.4s, v25.4s, v11.4s\n" - "smax v26.4s, v26.4s, v11.4s\n" - "smax v27.4s, v27.4s, v11.4s\n" - "smax v28.4s, v28.4s, v11.4s\n" - "smax v29.4s, v29.4s, v11.4s\n" - "smax v30.4s, v30.4s, v11.4s\n" - "smax v31.4s, v31.4s, v11.4s\n" + "srshl v24.4s, v24.4s, v21.4s\n" + "srshl v25.4s, v25.4s, v21.4s\n" + "srshl v26.4s, v26.4s, v21.4s\n" + "srshl v27.4s, v27.4s, v21.4s\n" + "srshl v28.4s, v28.4s, v21.4s\n" + "srshl v29.4s, v29.4s, v21.4s\n" + "srshl v30.4s, v30.4s, v21.4s\n" + "srshl v31.4s, v31.4s, v21.4s\n" + "add v24.4s, v24.4s, v14.4s\n" + "add v25.4s, v25.4s, v14.4s\n" + "add v26.4s, v26.4s, v14.4s\n" + "add v27.4s, v27.4s, v14.4s\n" + "add v28.4s, v28.4s, v14.4s\n" + "add v29.4s, v29.4s, v14.4s\n" + "add v30.4s, v30.4s, v14.4s\n" + "add v31.4s, v31.4s, v14.4s\n" + "smin v24.4s, v24.4s, v15.4s\n" + "smin v25.4s, v25.4s, v15.4s\n" + "smin v26.4s, v26.4s, v15.4s\n" + "smin v27.4s, v27.4s, v15.4s\n" + "smin v28.4s, v28.4s, v15.4s\n" + "smin v29.4s, v29.4s, v15.4s\n" + "smin v30.4s, v30.4s, v15.4s\n" + "smin v31.4s, v31.4s, v15.4s\n" + "smax v24.4s, v24.4s, v13.4s\n" + "smax v25.4s, v25.4s, v13.4s\n" + "smax v26.4s, v26.4s, v13.4s\n" + "smax v27.4s, v27.4s, v13.4s\n" + "smax v28.4s, v28.4s, v13.4s\n" + "smax v29.4s, v29.4s, v13.4s\n" + "smax v30.4s, v30.4s, v13.4s\n" + "smax v31.4s, v31.4s, v13.4s\n" "uzp1 v24.16b, v24.16b, v24.16b\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" @@ -329,14 +330,14 @@ void a64_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst_impl "dup v30.4s, v23.s[2]\n" "dup v31.4s, v23.s[3]\n" "add x28, x28, #0x4\n" - "add v24.4s, v24.4s, v14.4s\n" - "add v25.4s, v25.4s, v14.4s\n" - "add v26.4s, v26.4s, v14.4s\n" - "add v27.4s, v27.4s, v14.4s\n" - "add v28.4s, v28.4s, v14.4s\n" - "add v29.4s, v29.4s, v14.4s\n" - "add v30.4s, v30.4s, v14.4s\n" - "add v31.4s, v31.4s, v14.4s\n" + "add v24.4s, v24.4s, v20.4s\n" + "add v25.4s, v25.4s, v20.4s\n" + "add v26.4s, v26.4s, v20.4s\n" + "add v27.4s, v27.4s, v20.4s\n" + "add v28.4s, v28.4s, v20.4s\n" + "add v29.4s, v29.4s, v20.4s\n" + "add v30.4s, v30.4s, v20.4s\n" + "add v31.4s, v31.4s, v20.4s\n" "bgt 1b\n" "2:" // Tail "ldr q21, [%x[params], #0x0]\n" @@ -415,30 +416,30 @@ void a64_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst_impl "srshl v29.4s, v29.4s, v20.4s\n" "srshl v30.4s, v30.4s, v20.4s\n" "srshl v31.4s, v31.4s, v20.4s\n" - "add v24.4s, v24.4s, v12.4s\n" - "add v25.4s, v25.4s, v12.4s\n" - "add v26.4s, v26.4s, v12.4s\n" - "add v27.4s, v27.4s, v12.4s\n" - "add v28.4s, v28.4s, v12.4s\n" - "add v29.4s, v29.4s, v12.4s\n" - "add v30.4s, v30.4s, v12.4s\n" - "add v31.4s, v31.4s, v12.4s\n" - "smin v24.4s, v24.4s, v10.4s\n" - "smin v25.4s, v25.4s, v10.4s\n" - "smin v26.4s, v26.4s, v10.4s\n" - "smin v27.4s, v27.4s, v10.4s\n" - "smin v28.4s, v28.4s, v10.4s\n" - "smin v29.4s, v29.4s, v10.4s\n" - "smin v30.4s, v30.4s, v10.4s\n" - "smin v31.4s, v31.4s, v10.4s\n" - "smax v24.4s, v24.4s, v11.4s\n" - "smax v25.4s, v25.4s, v11.4s\n" - "smax v26.4s, v26.4s, v11.4s\n" - "smax v27.4s, v27.4s, v11.4s\n" - "smax v28.4s, v28.4s, v11.4s\n" - "smax v29.4s, v29.4s, v11.4s\n" - "smax v30.4s, v30.4s, v11.4s\n" - "smax v31.4s, v31.4s, v11.4s\n" + "add v24.4s, v24.4s, v14.4s\n" + "add v25.4s, v25.4s, v14.4s\n" + "add v26.4s, v26.4s, v14.4s\n" + "add v27.4s, v27.4s, v14.4s\n" + "add v28.4s, v28.4s, v14.4s\n" + "add v29.4s, v29.4s, v14.4s\n" + "add v30.4s, v30.4s, v14.4s\n" + "add v31.4s, v31.4s, v14.4s\n" + "smin v24.4s, v24.4s, v15.4s\n" + "smin v25.4s, v25.4s, v15.4s\n" + "smin v26.4s, v26.4s, v15.4s\n" + "smin v27.4s, v27.4s, v15.4s\n" + "smin v28.4s, v28.4s, v15.4s\n" + "smin v29.4s, v29.4s, v15.4s\n" + "smin v30.4s, v30.4s, v15.4s\n" + "smin v31.4s, v31.4s, v15.4s\n" + "smax v24.4s, v24.4s, v13.4s\n" + "smax v25.4s, v25.4s, v13.4s\n" + "smax v26.4s, v26.4s, v13.4s\n" + "smax v27.4s, v27.4s, v13.4s\n" + "smax v28.4s, v28.4s, v13.4s\n" + "smax v29.4s, v29.4s, v13.4s\n" + "smax v30.4s, v30.4s, v13.4s\n" + "smax v31.4s, v31.4s, v13.4s\n" "uzp1 v24.16b, v24.16b, v24.16b\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" @@ -514,4 +515,5 @@ void a64_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst_impl } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst.hpp index b575a5d169fb0f95467bbd39e7e5e591a34f4f85..4485aaa7356a219567a6f01e32f3d06eaa72500b 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include @@ -57,4 +57,5 @@ struct a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst : } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst/generic.cpp index 027cc9e5a2de483a19b1a8a59bfd57e74583e4a4..61cec2b66d112a04c45065c282c37cdd506e3bb2 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst/generic.cpp @@ -22,12 +22,13 @@ * SOFTWARE. */ -#if defined(__aarch64__) #include "arm_gemm.hpp" #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace depthwise { @@ -42,133 +43,133 @@ void a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl __asm__ __volatile__( "ldr q12, [%x[params], #0x0]\n" "ldr q8, [%x[params], #0x10]\n" - "movi v28.16b, #0x1\n" - "movi v18.4s, #0x0\n" + "movi v30.16b, #0x1\n" + "movi v17.4s, #0x0\n" "ldr q9, [%x[params], #0x20]\n" "ldr q10, [%x[params], #0x30]\n" - "movi v31.4s, #0x0\n" - "movi v24.4s, #0x0\n" + "movi v16.4s, #0x0\n" + "movi v25.4s, #0x0\n" "ldr q11, [%x[params], #0x40]\n" "ldr x20, [%x[inptrs], #0x18]\n" - "movi v30.4s, #0x0\n" - "movi v21.4s, #0x0\n" + "movi v24.4s, #0x0\n" + "movi v31.4s, #0x0\n" "ld1 { v3.16b }, [x20]\n" "ldr x20, [%x[inptrs], #0x20]\n" - "mov v16.16b, v3.16b\n" - "ext v16.16b, v16.16b, v16.16b, #0x1\n" + "mov v26.16b, v3.16b\n" + "ext v26.16b, v26.16b, v26.16b, #0x1\n" "ld1 { v4.16b }, [x20]\n" "ldr x20, [%x[inptrs], #0x10]\n" - "mov v15.16b, v4.16b\n" - "ext v15.16b, v15.16b, v15.16b, #0x1\n" + "mov v21.16b, v4.16b\n" + "ext v21.16b, v21.16b, v21.16b, #0x1\n" "ld1 { v2.16b }, [x20]\n" "ldr x20, [%x[inptrs], #0x8]\n" - "mov v20.16b, v2.16b\n" - "ext v20.16b, v20.16b, v20.16b, #0x1\n" + "mov v27.16b, v2.16b\n" + "ext v27.16b, v27.16b, v27.16b, #0x1\n" "ld1 { v1.16b }, [x20]\n" "ldr x20, [%x[inptrs], #0x28]\n" - "zip1 v3.2d, v3.2d, v16.2d\n" - "zip1 v4.2d, v4.2d, v15.2d\n" + "zip1 v3.2d, v3.2d, v26.2d\n" + "zip1 v4.2d, v4.2d, v21.2d\n" "ld1 { v5.16b }, [x20]\n" "ldr x20, [%x[inptrs], #0x30]\n" "mov v26.16b, v1.16b\n" - "mov v13.16b, v5.16b\n" + "mov v22.16b, v5.16b\n" "ld1 { v6.16b }, [x20]\n" "ldr x20, [%x[inptrs], #0x38]\n" "mov v19.16b, v6.16b\n" "ext v26.16b, v26.16b, v26.16b, #0x1\n" "ld1 { v7.16b }, [x20]\n" "ldr x20, [%x[inptrs], #0x0]\n" - "mov v17.16b, v7.16b\n" - "zip1 v2.2d, v2.2d, v20.2d\n" + "mov v21.16b, v7.16b\n" + "zip1 v2.2d, v2.2d, v27.2d\n" "ld1 { v0.16b }, [x20]\n" - "ext v13.16b, v13.16b, v13.16b, #0x1\n" + "ext v22.16b, v22.16b, v22.16b, #0x1\n" "ext v19.16b, v19.16b, v19.16b, #0x1\n" - ".inst 0x6f83e392 // udot v18.4s, v28.16b, v3.4b[0]\n" - "ext v17.16b, v17.16b, v17.16b, #0x1\n" - ".inst 0x6f83eb9f // udot v31.4s, v28.16b, v3.4b[2]\n" - ".inst 0x6f84e398 // udot v24.4s, v28.16b, v4.4b[0]\n" + ".inst 0x6f83e3d1 // udot v17.4s, v30.16b, v3.4b[0]\n" + "ext v21.16b, v21.16b, v21.16b, #0x1\n" + ".inst 0x6f83ebd0 // udot v16.4s, v30.16b, v3.4b[2]\n" + ".inst 0x6f84e3d9 // udot v25.4s, v30.16b, v4.4b[0]\n" "add x20, %x[qp], %[offsetof_Requantize32_b_offset]\n" "ld1r { v23.4s }, [x20]\n" - ".inst 0x6f84eb9e // udot v30.4s, v28.16b, v4.4b[2]\n" - "mov v16.16b, v0.16b\n" - ".inst 0x6f82e395 // udot v21.4s, v28.16b, v2.4b[0]\n" - "movi v20.4s, #0x0\n" - "movi v29.4s, #0x1\n" - ".inst 0x6f82eb94 // udot v20.4s, v28.16b, v2.4b[2]\n" + ".inst 0x6f84ebd8 // udot v24.4s, v30.16b, v4.4b[2]\n" + "mov v18.16b, v0.16b\n" + ".inst 0x6f82e3df // udot v31.4s, v30.16b, v2.4b[0]\n" + "movi v29.4s, #0x0\n" + "movi v28.4s, #0x1\n" + ".inst 0x6f82ebdd // udot v29.4s, v30.16b, v2.4b[2]\n" "add x20, %x[qp], %[offsetof_Requantize32_c_offset]\n" - "ld1r { v14.4s }, [x20]\n" - "ext v16.16b, v16.16b, v16.16b, #0x1\n" + "ld1r { v13.4s }, [x20]\n" + "ext v18.16b, v18.16b, v18.16b, #0x1\n" "zip1 v1.2d, v1.2d, v26.2d\n" - ".inst 0x6fa3e3b2 // udot v18.4s, v29.16b, v3.4b[1]\n" - "zip1 v5.2d, v5.2d, v13.2d\n" + ".inst 0x6fa3e391 // udot v17.4s, v28.16b, v3.4b[1]\n" + "zip1 v5.2d, v5.2d, v22.2d\n" "zip1 v6.2d, v6.2d, v19.2d\n" - ".inst 0x6fa3ebbf // udot v31.4s, v29.16b, v3.4b[3]\n" + ".inst 0x6fa3eb90 // udot v16.4s, v28.16b, v3.4b[3]\n" "add x20, %x[qp], %[offsetof_Requantize32_minval]\n" - "ld1r { v13.4s }, [x20]\n" - "zip1 v7.2d, v7.2d, v17.2d\n" + "ld1r { v14.4s }, [x20]\n" + "zip1 v7.2d, v7.2d, v21.2d\n" "movi v22.4s, #0x0\n" - ".inst 0x6fa4e3b8 // udot v24.4s, v29.16b, v4.4b[1]\n" - "movi v26.4s, #0x0\n" - ".inst 0x6fa4ebbe // udot v30.4s, v29.16b, v4.4b[3]\n" - ".inst 0x6f81e396 // udot v22.4s, v28.16b, v1.4b[0]\n" + ".inst 0x6fa4e399 // udot v25.4s, v28.16b, v4.4b[1]\n" + "movi v21.4s, #0x0\n" + ".inst 0x6fa4eb98 // udot v24.4s, v28.16b, v4.4b[3]\n" + ".inst 0x6f81e3d6 // udot v22.4s, v30.16b, v1.4b[0]\n" "add x20, %x[qp], %[offsetof_Requantize32_maxval]\n" "ld1r { v15.4s }, [x20]\n" - "movi v25.4s, #0x0\n" + "movi v26.4s, #0x0\n" "movi v27.4s, #0x0\n" - ".inst 0x6f81eb9a // udot v26.4s, v28.16b, v1.4b[2]\n" - "zip1 v0.2d, v0.2d, v16.2d\n" + ".inst 0x6f81ebd5 // udot v21.4s, v30.16b, v1.4b[2]\n" + "movi v20.4s, #0x0\n" "movi v19.4s, #0x0\n" - ".inst 0x6f85e399 // udot v25.4s, v28.16b, v5.4b[0]\n" + ".inst 0x6f85e3da // udot v26.4s, v30.16b, v5.4b[0]\n" "cmp %x[n_channels], #0x4\n" - ".inst 0x6f85eb9b // udot v27.4s, v28.16b, v5.4b[2]\n" - ".inst 0x6f86e393 // udot v19.4s, v28.16b, v6.4b[0]\n" - "add v24.4s, v18.4s, v24.4s\n" - "mov x9, #0x0\n" + "zip1 v0.2d, v0.2d, v18.2d\n" "movi v18.4s, #0x0\n" - ".inst 0x6f86eb92 // udot v18.4s, v28.16b, v6.4b[2]\n" - ".inst 0x6fa2e3b5 // udot v21.4s, v29.16b, v2.4b[1]\n" + ".inst 0x6f85ebdb // udot v27.4s, v30.16b, v5.4b[2]\n" + "mov x9, #0x0\n" + ".inst 0x6f86e3d4 // udot v20.4s, v30.16b, v6.4b[0]\n" + ".inst 0x6f86ebd3 // udot v19.4s, v30.16b, v6.4b[2]\n" + "add v17.4s, v17.4s, v25.4s\n" "mov x28, #0x0\n" - ".inst 0x6fa2ebb4 // udot v20.4s, v29.16b, v2.4b[3]\n" - "add v17.4s, v31.4s, v30.4s\n" - ".inst 0x6fa1e3b6 // udot v22.4s, v29.16b, v1.4b[1]\n" + "movi v25.4s, #0x0\n" + ".inst 0x6f87e3d2 // udot v18.4s, v30.16b, v7.4b[0]\n" + ".inst 0x6f87ebd9 // udot v25.4s, v30.16b, v7.4b[2]\n" "ldp x27, x26, [%x[outptrs], #0x0]\n" - "movi v16.4s, #0x0\n" - ".inst 0x6f87e390 // udot v16.4s, v28.16b, v7.4b[0]\n" - ".inst 0x6fa1ebba // udot v26.4s, v29.16b, v1.4b[3]\n" + ".inst 0x6fa2e39f // udot v31.4s, v28.16b, v2.4b[1]\n" + ".inst 0x6fa2eb9d // udot v29.4s, v28.16b, v2.4b[3]\n" + "add v16.4s, v16.4s, v24.4s\n" "ldp x25, x24, [%x[outptrs], #0x10]\n" - ".inst 0x6fa5e3b9 // udot v25.4s, v29.16b, v5.4b[1]\n" - ".inst 0x6fa5ebbb // udot v27.4s, v29.16b, v5.4b[3]\n" - "add v30.4s, v21.4s, v24.4s\n" + "movi v24.4s, #0x0\n" + ".inst 0x6f80e3d8 // udot v24.4s, v30.16b, v0.4b[0]\n" + ".inst 0x6fa1e396 // udot v22.4s, v28.16b, v1.4b[1]\n" "ldp x23, x22, [%x[outptrs], #0x20]\n" - ".inst 0x6fa6e3b3 // udot v19.4s, v29.16b, v6.4b[1]\n" - ".inst 0x6fa6ebb2 // udot v18.4s, v29.16b, v6.4b[3]\n" - "add v31.4s, v20.4s, v17.4s\n" + ".inst 0x6fa1eb95 // udot v21.4s, v28.16b, v1.4b[3]\n" + ".inst 0x6fa5e39a // udot v26.4s, v28.16b, v5.4b[1]\n" + "add v31.4s, v31.4s, v17.4s\n" "ldp x21, x20, [%x[outptrs], #0x30]\n" - ".inst 0x6fa7e3b0 // udot v16.4s, v29.16b, v7.4b[1]\n" - "add v22.4s, v22.4s, v30.4s\n" + ".inst 0x6fa5eb9b // udot v27.4s, v28.16b, v5.4b[3]\n" + ".inst 0x6fa6e394 // udot v20.4s, v28.16b, v6.4b[1]\n" + "add v29.4s, v29.4s, v16.4s\n" "add %x[params], %x[params], #0x50\n" - "add v21.4s, v26.4s, v31.4s\n" - "add v20.4s, v25.4s, v19.4s\n" - "add v19.4s, v27.4s, v18.4s\n" - "add v18.4s, v16.4s, v24.4s\n" - "movi v16.4s, #0x0\n" - ".inst 0x6f87eb90 // udot v16.4s, v28.16b, v7.4b[2]\n" - ".inst 0x6fa7ebb0 // udot v16.4s, v29.16b, v7.4b[3]\n" - "add v17.4s, v16.4s, v17.4s\n" - "movi v16.4s, #0x0\n" - ".inst 0x6f80e390 // udot v16.4s, v28.16b, v0.4b[0]\n" - ".inst 0x6fa0e3b0 // udot v16.4s, v29.16b, v0.4b[1]\n" - "add v24.4s, v22.4s, v16.4s\n" - "add v26.4s, v22.4s, v25.4s\n" - "movi v16.4s, #0x0\n" - ".inst 0x6f80eb90 // udot v16.4s, v28.16b, v0.4b[2]\n" - ".inst 0x6fa0ebb0 // udot v16.4s, v29.16b, v0.4b[3]\n" - "add v25.4s, v21.4s, v16.4s\n" - "add v27.4s, v21.4s, v27.4s\n" - "add v28.4s, v20.4s, v30.4s\n" - "add v29.4s, v19.4s, v31.4s\n" - "add v30.4s, v18.4s, v20.4s\n" - "add v31.4s, v17.4s, v19.4s\n" + ".inst 0x6fa6eb93 // udot v19.4s, v28.16b, v6.4b[3]\n" + ".inst 0x6fa7e392 // udot v18.4s, v28.16b, v7.4b[1]\n" + "add v22.4s, v22.4s, v31.4s\n" + ".inst 0x6fa7eb99 // udot v25.4s, v28.16b, v7.4b[3]\n" + ".inst 0x6fa0e398 // udot v24.4s, v28.16b, v0.4b[1]\n" + "add v21.4s, v21.4s, v29.4s\n" + "add v20.4s, v26.4s, v20.4s\n" + "add v19.4s, v27.4s, v19.4s\n" + "add v18.4s, v18.4s, v17.4s\n" + "movi v17.4s, #0x0\n" + ".inst 0x6f80ebd1 // udot v17.4s, v30.16b, v0.4b[2]\n" + ".inst 0x6fa0eb91 // udot v17.4s, v28.16b, v0.4b[3]\n" + "add v16.4s, v25.4s, v16.4s\n" + "add v24.4s, v22.4s, v24.4s\n" + "add v25.4s, v21.4s, v17.4s\n" + "add v26.4s, v26.4s, v22.4s\n" + "add v27.4s, v27.4s, v21.4s\n" + "add v28.4s, v20.4s, v31.4s\n" + "add v29.4s, v19.4s, v29.4s\n" + "add v30.4s, v20.4s, v18.4s\n" + "add v31.4s, v19.4s, v16.4s\n" "neg v23.4s, v23.4s\n" "mul v24.4s, v24.4s, v23.4s\n" "mul v25.4s, v25.4s, v23.4s\n" @@ -194,11 +195,11 @@ void a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl "add v31.4s, v31.4s, v12.4s\n" "ble 2f\n" "1:" // Loop - "ldr q21, [%x[params], #0x60]\n" - "ldr q20, [%x[params], #0x70]\n" + "ldr q12, [%x[params], #0x60]\n" + "ldr q21, [%x[params], #0x70]\n" ".inst 0x6f80e118 // udot v24.4s, v8.16b, v0.4b[0]\n" ".inst 0x6f80e919 // udot v25.4s, v8.16b, v0.4b[2]\n" - "ldr q12, [%x[params], #0x80]\n" + "ldr q20, [%x[params], #0x80]\n" ".inst 0x6f81e11a // udot v26.4s, v8.16b, v1.4b[0]\n" ".inst 0x6f81e91b // udot v27.4s, v8.16b, v1.4b[2]\n" "sub %x[n_channels], %x[n_channels], #0x4\n" @@ -212,7 +213,7 @@ void a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl ".inst 0x6f82e91d // udot v29.4s, v8.16b, v2.4b[2]\n" ".inst 0x6f83e11e // udot v30.4s, v8.16b, v3.4b[0]\n" ".inst 0x6f83e91f // udot v31.4s, v8.16b, v3.4b[2]\n" - "ldr q8, [%x[params], #0x0]\n" + "ldr q17, [%x[params], #0x0]\n" ".inst 0x6f81e158 // udot v24.4s, v10.16b, v1.4b[0]\n" ".inst 0x6f81e959 // udot v25.4s, v10.16b, v1.4b[2]\n" ".inst 0x6f82e15a // udot v26.4s, v10.16b, v2.4b[0]\n" @@ -221,7 +222,7 @@ void a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl ".inst 0x6fa2e93d // udot v29.4s, v9.16b, v2.4b[3]\n" ".inst 0x6fa3e13e // udot v30.4s, v9.16b, v3.4b[1]\n" ".inst 0x6fa3e93f // udot v31.4s, v9.16b, v3.4b[3]\n" - "ldr q9, [%x[params], #0x10]\n" + "ldr q16, [%x[params], #0x10]\n" ".inst 0x6fa1e178 // udot v24.4s, v11.16b, v1.4b[1]\n" ".inst 0x6fa1e979 // udot v25.4s, v11.16b, v1.4b[3]\n" ".inst 0x6fa2e17a // udot v26.4s, v11.16b, v2.4b[1]\n" @@ -230,115 +231,115 @@ void a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl ".inst 0x6f83e95d // udot v29.4s, v10.16b, v3.4b[2]\n" ".inst 0x6f84e15e // udot v30.4s, v10.16b, v4.4b[0]\n" ".inst 0x6f84e95f // udot v31.4s, v10.16b, v4.4b[2]\n" - "ldr q10, [%x[params], #0x20]\n" - ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" - ".inst 0x6f82e919 // udot v25.4s, v8.16b, v2.4b[2]\n" - ".inst 0x6f83e11a // udot v26.4s, v8.16b, v3.4b[0]\n" - ".inst 0x6f83e91b // udot v27.4s, v8.16b, v3.4b[2]\n" + "ldr q19, [%x[params], #0x20]\n" + ".inst 0x6f82e238 // udot v24.4s, v17.16b, v2.4b[0]\n" + ".inst 0x6f82ea39 // udot v25.4s, v17.16b, v2.4b[2]\n" + ".inst 0x6f83e23a // udot v26.4s, v17.16b, v3.4b[0]\n" + ".inst 0x6f83ea3b // udot v27.4s, v17.16b, v3.4b[2]\n" ".inst 0x6fa3e17c // udot v28.4s, v11.16b, v3.4b[1]\n" ".inst 0x6fa3e97d // udot v29.4s, v11.16b, v3.4b[3]\n" ".inst 0x6fa4e17e // udot v30.4s, v11.16b, v4.4b[1]\n" ".inst 0x6fa4e97f // udot v31.4s, v11.16b, v4.4b[3]\n" - "ldr q11, [%x[params], #0x30]\n" - ".inst 0x6fa2e138 // udot v24.4s, v9.16b, v2.4b[1]\n" - ".inst 0x6fa2e939 // udot v25.4s, v9.16b, v2.4b[3]\n" - ".inst 0x6fa3e13a // udot v26.4s, v9.16b, v3.4b[1]\n" - ".inst 0x6fa3e93b // udot v27.4s, v9.16b, v3.4b[3]\n" - ".inst 0x6f84e11c // udot v28.4s, v8.16b, v4.4b[0]\n" - ".inst 0x6f84e91d // udot v29.4s, v8.16b, v4.4b[2]\n" - ".inst 0x6f85e11e // udot v30.4s, v8.16b, v5.4b[0]\n" - ".inst 0x6f85e91f // udot v31.4s, v8.16b, v5.4b[2]\n" - "ldr q8, [%x[params], #0x40]\n" - ".inst 0x6f83e158 // udot v24.4s, v10.16b, v3.4b[0]\n" - ".inst 0x6f83e959 // udot v25.4s, v10.16b, v3.4b[2]\n" - ".inst 0x6f84e15a // udot v26.4s, v10.16b, v4.4b[0]\n" - ".inst 0x6f84e95b // udot v27.4s, v10.16b, v4.4b[2]\n" - ".inst 0x6fa4e13c // udot v28.4s, v9.16b, v4.4b[1]\n" - ".inst 0x6fa4e93d // udot v29.4s, v9.16b, v4.4b[3]\n" - ".inst 0x6fa5e13e // udot v30.4s, v9.16b, v5.4b[1]\n" - ".inst 0x6fa5e93f // udot v31.4s, v9.16b, v5.4b[3]\n" - "ldr q9, [%x[params], #0x50]\n" - ".inst 0x6fa3e178 // udot v24.4s, v11.16b, v3.4b[1]\n" - ".inst 0x6fa3e979 // udot v25.4s, v11.16b, v3.4b[3]\n" - ".inst 0x6fa4e17a // udot v26.4s, v11.16b, v4.4b[1]\n" - ".inst 0x6fa4e97b // udot v27.4s, v11.16b, v4.4b[3]\n" - ".inst 0x6f85e15c // udot v28.4s, v10.16b, v5.4b[0]\n" - ".inst 0x6f85e95d // udot v29.4s, v10.16b, v5.4b[2]\n" - ".inst 0x6f86e15e // udot v30.4s, v10.16b, v6.4b[0]\n" - ".inst 0x6f86e95f // udot v31.4s, v10.16b, v6.4b[2]\n" + "ldr q18, [%x[params], #0x30]\n" + ".inst 0x6fa2e218 // udot v24.4s, v16.16b, v2.4b[1]\n" + ".inst 0x6fa2ea19 // udot v25.4s, v16.16b, v2.4b[3]\n" + ".inst 0x6fa3e21a // udot v26.4s, v16.16b, v3.4b[1]\n" + ".inst 0x6fa3ea1b // udot v27.4s, v16.16b, v3.4b[3]\n" + ".inst 0x6f84e23c // udot v28.4s, v17.16b, v4.4b[0]\n" + ".inst 0x6f84ea3d // udot v29.4s, v17.16b, v4.4b[2]\n" + ".inst 0x6f85e23e // udot v30.4s, v17.16b, v5.4b[0]\n" + ".inst 0x6f85ea3f // udot v31.4s, v17.16b, v5.4b[2]\n" + "ldr q17, [%x[params], #0x40]\n" + ".inst 0x6f83e278 // udot v24.4s, v19.16b, v3.4b[0]\n" + ".inst 0x6f83ea79 // udot v25.4s, v19.16b, v3.4b[2]\n" + ".inst 0x6f84e27a // udot v26.4s, v19.16b, v4.4b[0]\n" + ".inst 0x6f84ea7b // udot v27.4s, v19.16b, v4.4b[2]\n" + ".inst 0x6fa4e21c // udot v28.4s, v16.16b, v4.4b[1]\n" + ".inst 0x6fa4ea1d // udot v29.4s, v16.16b, v4.4b[3]\n" + ".inst 0x6fa5e21e // udot v30.4s, v16.16b, v5.4b[1]\n" + ".inst 0x6fa5ea1f // udot v31.4s, v16.16b, v5.4b[3]\n" + "ldr q16, [%x[params], #0x50]\n" + ".inst 0x6fa3e258 // udot v24.4s, v18.16b, v3.4b[1]\n" + ".inst 0x6fa3ea59 // udot v25.4s, v18.16b, v3.4b[3]\n" + ".inst 0x6fa4e25a // udot v26.4s, v18.16b, v4.4b[1]\n" + ".inst 0x6fa4ea5b // udot v27.4s, v18.16b, v4.4b[3]\n" + ".inst 0x6f85e27c // udot v28.4s, v19.16b, v5.4b[0]\n" + ".inst 0x6f85ea7d // udot v29.4s, v19.16b, v5.4b[2]\n" + ".inst 0x6f86e27e // udot v30.4s, v19.16b, v6.4b[0]\n" + ".inst 0x6f86ea7f // udot v31.4s, v19.16b, v6.4b[2]\n" "ldr q10, [%x[params], #0xb0]\n" - ".inst 0x6f84e118 // udot v24.4s, v8.16b, v4.4b[0]\n" - ".inst 0x6f84e919 // udot v25.4s, v8.16b, v4.4b[2]\n" - ".inst 0x6f85e11a // udot v26.4s, v8.16b, v5.4b[0]\n" - ".inst 0x6f85e91b // udot v27.4s, v8.16b, v5.4b[2]\n" - ".inst 0x6fa5e17c // udot v28.4s, v11.16b, v5.4b[1]\n" - ".inst 0x6fa5e97d // udot v29.4s, v11.16b, v5.4b[3]\n" - ".inst 0x6fa6e17e // udot v30.4s, v11.16b, v6.4b[1]\n" - ".inst 0x6fa6e97f // udot v31.4s, v11.16b, v6.4b[3]\n" + ".inst 0x6f84e238 // udot v24.4s, v17.16b, v4.4b[0]\n" + ".inst 0x6f84ea39 // udot v25.4s, v17.16b, v4.4b[2]\n" + ".inst 0x6f85e23a // udot v26.4s, v17.16b, v5.4b[0]\n" + ".inst 0x6f85ea3b // udot v27.4s, v17.16b, v5.4b[2]\n" + ".inst 0x6fa5e25c // udot v28.4s, v18.16b, v5.4b[1]\n" + ".inst 0x6fa5ea5d // udot v29.4s, v18.16b, v5.4b[3]\n" + ".inst 0x6fa6e25e // udot v30.4s, v18.16b, v6.4b[1]\n" + ".inst 0x6fa6ea5f // udot v31.4s, v18.16b, v6.4b[3]\n" "ldr q11, [%x[params], #0xc0]\n" - ".inst 0x6fa4e138 // udot v24.4s, v9.16b, v4.4b[1]\n" - ".inst 0x6fa4e939 // udot v25.4s, v9.16b, v4.4b[3]\n" - "sqrdmulh v24.4s, v24.4s, v21.4s\n" - ".inst 0x6fa5e13a // udot v26.4s, v9.16b, v5.4b[1]\n" - ".inst 0x6fa5e93b // udot v27.4s, v9.16b, v5.4b[3]\n" - "sqrdmulh v25.4s, v25.4s, v21.4s\n" - ".inst 0x6f86e11c // udot v28.4s, v8.16b, v6.4b[0]\n" - ".inst 0x6f86e91d // udot v29.4s, v8.16b, v6.4b[2]\n" - "sqrdmulh v26.4s, v26.4s, v21.4s\n" - ".inst 0x6f87e11e // udot v30.4s, v8.16b, v7.4b[0]\n" - ".inst 0x6f87e91f // udot v31.4s, v8.16b, v7.4b[2]\n" + ".inst 0x6fa4e218 // udot v24.4s, v16.16b, v4.4b[1]\n" + ".inst 0x6fa4ea19 // udot v25.4s, v16.16b, v4.4b[3]\n" + "sqrdmulh v24.4s, v24.4s, v12.4s\n" + ".inst 0x6fa5e21a // udot v26.4s, v16.16b, v5.4b[1]\n" + ".inst 0x6fa5ea1b // udot v27.4s, v16.16b, v5.4b[3]\n" + "sqrdmulh v25.4s, v25.4s, v12.4s\n" + ".inst 0x6f86e23c // udot v28.4s, v17.16b, v6.4b[0]\n" + ".inst 0x6f86ea3d // udot v29.4s, v17.16b, v6.4b[2]\n" + "sqrdmulh v26.4s, v26.4s, v12.4s\n" + ".inst 0x6f87e23e // udot v30.4s, v17.16b, v7.4b[0]\n" + ".inst 0x6f87ea3f // udot v31.4s, v17.16b, v7.4b[2]\n" "ldr q8, [%x[params], #0x90]\n" - "sqrdmulh v27.4s, v27.4s, v21.4s\n" - ".inst 0x6fa6e13c // udot v28.4s, v9.16b, v6.4b[1]\n" - ".inst 0x6fa6e93d // udot v29.4s, v9.16b, v6.4b[3]\n" - "and v19.16b, v24.16b, v20.16b\n" - ".inst 0x6fa7e13e // udot v30.4s, v9.16b, v7.4b[1]\n" - ".inst 0x6fa7e93f // udot v31.4s, v9.16b, v7.4b[3]\n" + "sqrdmulh v27.4s, v27.4s, v12.4s\n" + ".inst 0x6fa6e21c // udot v28.4s, v16.16b, v6.4b[1]\n" + ".inst 0x6fa6ea1d // udot v29.4s, v16.16b, v6.4b[3]\n" + "and v19.16b, v24.16b, v21.16b\n" + ".inst 0x6fa7e21e // udot v30.4s, v16.16b, v7.4b[1]\n" + ".inst 0x6fa7ea1f // udot v31.4s, v16.16b, v7.4b[3]\n" "ldr q9, [%x[params], #0xa0]\n" - "and v18.16b, v25.16b, v20.16b\n" + "and v18.16b, v25.16b, v21.16b\n" + "and v17.16b, v26.16b, v21.16b\n" + "and v16.16b, v27.16b, v21.16b\n" + "add %x[params], %x[params], #0xd0\n" "sshr v19.4s, v19.4s, #0x1f\n" "sshr v18.4s, v18.4s, #0x1f\n" - "add %x[params], %x[params], #0xd0\n" - "sqrdmulh v28.4s, v28.4s, v21.4s\n" - "sqrdmulh v29.4s, v29.4s, v21.4s\n" - "sqrdmulh v30.4s, v30.4s, v21.4s\n" - "sqrdmulh v31.4s, v31.4s, v21.4s\n" - "and v17.16b, v26.16b, v20.16b\n" "sshr v17.4s, v17.4s, #0x1f\n" - "sqadd v24.4s, v24.4s, v19.4s\n" - "and v16.16b, v27.16b, v20.16b\n" "sshr v16.4s, v16.4s, #0x1f\n" + "sqrdmulh v28.4s, v28.4s, v12.4s\n" + "sqrdmulh v29.4s, v29.4s, v12.4s\n" + "sqrdmulh v30.4s, v30.4s, v12.4s\n" + "sqrdmulh v31.4s, v31.4s, v12.4s\n" + "sqadd v24.4s, v24.4s, v19.4s\n" "sqadd v25.4s, v25.4s, v18.4s\n" "sqadd v26.4s, v26.4s, v17.4s\n" "sqadd v27.4s, v27.4s, v16.4s\n" - "and v19.16b, v28.16b, v20.16b\n" - "and v18.16b, v29.16b, v20.16b\n" - "and v17.16b, v30.16b, v20.16b\n" + "and v19.16b, v28.16b, v21.16b\n" + "and v18.16b, v29.16b, v21.16b\n" + "and v17.16b, v30.16b, v21.16b\n" + "and v16.16b, v31.16b, v21.16b\n" "sshr v19.4s, v19.4s, #0x1f\n" "sshr v18.4s, v18.4s, #0x1f\n" "sshr v17.4s, v17.4s, #0x1f\n" - "sqadd v28.4s, v28.4s, v19.4s\n" - "and v16.16b, v31.16b, v20.16b\n" "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v28.4s, v28.4s, v19.4s\n" "sqadd v29.4s, v29.4s, v18.4s\n" "sqadd v30.4s, v30.4s, v17.4s\n" "sqadd v31.4s, v31.4s, v16.4s\n" - "srshl v24.4s, v24.4s, v20.4s\n" - "srshl v25.4s, v25.4s, v20.4s\n" - "srshl v26.4s, v26.4s, v20.4s\n" - "srshl v27.4s, v27.4s, v20.4s\n" - "srshl v28.4s, v28.4s, v20.4s\n" - "srshl v29.4s, v29.4s, v20.4s\n" - "srshl v30.4s, v30.4s, v20.4s\n" - "srshl v31.4s, v31.4s, v20.4s\n" - "add v24.4s, v24.4s, v14.4s\n" - "add v25.4s, v25.4s, v14.4s\n" - "add v26.4s, v26.4s, v14.4s\n" - "add v27.4s, v27.4s, v14.4s\n" - "add v28.4s, v28.4s, v14.4s\n" - "add v29.4s, v29.4s, v14.4s\n" - "add v30.4s, v30.4s, v14.4s\n" - "add v31.4s, v31.4s, v14.4s\n" + "srshl v24.4s, v24.4s, v21.4s\n" + "srshl v25.4s, v25.4s, v21.4s\n" + "srshl v26.4s, v26.4s, v21.4s\n" + "srshl v27.4s, v27.4s, v21.4s\n" + "srshl v28.4s, v28.4s, v21.4s\n" + "srshl v29.4s, v29.4s, v21.4s\n" + "srshl v30.4s, v30.4s, v21.4s\n" + "srshl v31.4s, v31.4s, v21.4s\n" + "add v24.4s, v24.4s, v13.4s\n" + "add v25.4s, v25.4s, v13.4s\n" + "add v26.4s, v26.4s, v13.4s\n" + "add v27.4s, v27.4s, v13.4s\n" + "add v28.4s, v28.4s, v13.4s\n" + "add v29.4s, v29.4s, v13.4s\n" + "add v30.4s, v30.4s, v13.4s\n" + "add v31.4s, v31.4s, v13.4s\n" "smin v24.4s, v24.4s, v15.4s\n" "smin v25.4s, v25.4s, v15.4s\n" "smin v26.4s, v26.4s, v15.4s\n" @@ -347,14 +348,14 @@ void a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl "smin v29.4s, v29.4s, v15.4s\n" "smin v30.4s, v30.4s, v15.4s\n" "smin v31.4s, v31.4s, v15.4s\n" - "smax v24.4s, v24.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v26.4s, v26.4s, v13.4s\n" - "smax v27.4s, v27.4s, v13.4s\n" - "smax v28.4s, v28.4s, v13.4s\n" - "smax v29.4s, v29.4s, v13.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" - "smax v31.4s, v31.4s, v13.4s\n" + "smax v24.4s, v24.4s, v14.4s\n" + "smax v25.4s, v25.4s, v14.4s\n" + "smax v26.4s, v26.4s, v14.4s\n" + "smax v27.4s, v27.4s, v14.4s\n" + "smax v28.4s, v28.4s, v14.4s\n" + "smax v29.4s, v29.4s, v14.4s\n" + "smax v30.4s, v30.4s, v14.4s\n" + "smax v31.4s, v31.4s, v14.4s\n" "uzp1 v24.16b, v24.16b, v24.16b\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" @@ -388,14 +389,14 @@ void a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl "dup v30.4s, v23.s[2]\n" "dup v31.4s, v23.s[3]\n" "add x28, x28, #0x4\n" - "add v24.4s, v24.4s, v12.4s\n" - "add v25.4s, v25.4s, v12.4s\n" - "add v26.4s, v26.4s, v12.4s\n" - "add v27.4s, v27.4s, v12.4s\n" - "add v28.4s, v28.4s, v12.4s\n" - "add v29.4s, v29.4s, v12.4s\n" - "add v30.4s, v30.4s, v12.4s\n" - "add v31.4s, v31.4s, v12.4s\n" + "add v24.4s, v24.4s, v20.4s\n" + "add v25.4s, v25.4s, v20.4s\n" + "add v26.4s, v26.4s, v20.4s\n" + "add v27.4s, v27.4s, v20.4s\n" + "add v28.4s, v28.4s, v20.4s\n" + "add v29.4s, v29.4s, v20.4s\n" + "add v30.4s, v30.4s, v20.4s\n" + "add v31.4s, v31.4s, v20.4s\n" "bgt 1b\n" "2:" // Tail "ldr q21, [%x[params], #0x60]\n" @@ -420,7 +421,7 @@ void a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl "add x21, x21, x28\n" ".inst 0x6f83e11e // udot v30.4s, v8.16b, v3.4b[0]\n" ".inst 0x6f83e91f // udot v31.4s, v8.16b, v3.4b[2]\n" - "ldr q8, [%x[params], #0x0]\n" + "ldr q17, [%x[params], #0x0]\n" "add x20, x20, x28\n" ".inst 0x6f81e158 // udot v24.4s, v10.16b, v1.4b[0]\n" ".inst 0x6f81e959 // udot v25.4s, v10.16b, v1.4b[2]\n" @@ -430,7 +431,7 @@ void a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl ".inst 0x6fa2e93d // udot v29.4s, v9.16b, v2.4b[3]\n" ".inst 0x6fa3e13e // udot v30.4s, v9.16b, v3.4b[1]\n" ".inst 0x6fa3e93f // udot v31.4s, v9.16b, v3.4b[3]\n" - "ldr q9, [%x[params], #0x10]\n" + "ldr q16, [%x[params], #0x10]\n" ".inst 0x6fa1e178 // udot v24.4s, v11.16b, v1.4b[1]\n" ".inst 0x6fa1e979 // udot v25.4s, v11.16b, v1.4b[3]\n" ".inst 0x6fa2e17a // udot v26.4s, v11.16b, v2.4b[1]\n" @@ -439,68 +440,68 @@ void a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl ".inst 0x6f83e95d // udot v29.4s, v10.16b, v3.4b[2]\n" ".inst 0x6f84e15e // udot v30.4s, v10.16b, v4.4b[0]\n" ".inst 0x6f84e95f // udot v31.4s, v10.16b, v4.4b[2]\n" - "ldr q10, [%x[params], #0x20]\n" - ".inst 0x6f82e118 // udot v24.4s, v8.16b, v2.4b[0]\n" - ".inst 0x6f82e919 // udot v25.4s, v8.16b, v2.4b[2]\n" - ".inst 0x6f83e11a // udot v26.4s, v8.16b, v3.4b[0]\n" - ".inst 0x6f83e91b // udot v27.4s, v8.16b, v3.4b[2]\n" + "ldr q19, [%x[params], #0x20]\n" + ".inst 0x6f82e238 // udot v24.4s, v17.16b, v2.4b[0]\n" + ".inst 0x6f82ea39 // udot v25.4s, v17.16b, v2.4b[2]\n" + ".inst 0x6f83e23a // udot v26.4s, v17.16b, v3.4b[0]\n" + ".inst 0x6f83ea3b // udot v27.4s, v17.16b, v3.4b[2]\n" ".inst 0x6fa3e17c // udot v28.4s, v11.16b, v3.4b[1]\n" ".inst 0x6fa3e97d // udot v29.4s, v11.16b, v3.4b[3]\n" ".inst 0x6fa4e17e // udot v30.4s, v11.16b, v4.4b[1]\n" ".inst 0x6fa4e97f // udot v31.4s, v11.16b, v4.4b[3]\n" - "ldr q11, [%x[params], #0x30]\n" - ".inst 0x6fa2e138 // udot v24.4s, v9.16b, v2.4b[1]\n" - ".inst 0x6fa2e939 // udot v25.4s, v9.16b, v2.4b[3]\n" - ".inst 0x6fa3e13a // udot v26.4s, v9.16b, v3.4b[1]\n" - ".inst 0x6fa3e93b // udot v27.4s, v9.16b, v3.4b[3]\n" - ".inst 0x6f84e11c // udot v28.4s, v8.16b, v4.4b[0]\n" - ".inst 0x6f84e91d // udot v29.4s, v8.16b, v4.4b[2]\n" - ".inst 0x6f85e11e // udot v30.4s, v8.16b, v5.4b[0]\n" - ".inst 0x6f85e91f // udot v31.4s, v8.16b, v5.4b[2]\n" - "ldr q8, [%x[params], #0x40]\n" - ".inst 0x6f83e158 // udot v24.4s, v10.16b, v3.4b[0]\n" - ".inst 0x6f83e959 // udot v25.4s, v10.16b, v3.4b[2]\n" - ".inst 0x6f84e15a // udot v26.4s, v10.16b, v4.4b[0]\n" - ".inst 0x6f84e95b // udot v27.4s, v10.16b, v4.4b[2]\n" - ".inst 0x6fa4e13c // udot v28.4s, v9.16b, v4.4b[1]\n" - ".inst 0x6fa4e93d // udot v29.4s, v9.16b, v4.4b[3]\n" - ".inst 0x6fa5e13e // udot v30.4s, v9.16b, v5.4b[1]\n" - ".inst 0x6fa5e93f // udot v31.4s, v9.16b, v5.4b[3]\n" - "ldr q9, [%x[params], #0x50]\n" + "ldr q18, [%x[params], #0x30]\n" + ".inst 0x6fa2e218 // udot v24.4s, v16.16b, v2.4b[1]\n" + ".inst 0x6fa2ea19 // udot v25.4s, v16.16b, v2.4b[3]\n" + ".inst 0x6fa3e21a // udot v26.4s, v16.16b, v3.4b[1]\n" + ".inst 0x6fa3ea1b // udot v27.4s, v16.16b, v3.4b[3]\n" + ".inst 0x6f84e23c // udot v28.4s, v17.16b, v4.4b[0]\n" + ".inst 0x6f84ea3d // udot v29.4s, v17.16b, v4.4b[2]\n" + ".inst 0x6f85e23e // udot v30.4s, v17.16b, v5.4b[0]\n" + ".inst 0x6f85ea3f // udot v31.4s, v17.16b, v5.4b[2]\n" + "ldr q17, [%x[params], #0x40]\n" + ".inst 0x6f83e278 // udot v24.4s, v19.16b, v3.4b[0]\n" + ".inst 0x6f83ea79 // udot v25.4s, v19.16b, v3.4b[2]\n" + ".inst 0x6f84e27a // udot v26.4s, v19.16b, v4.4b[0]\n" + ".inst 0x6f84ea7b // udot v27.4s, v19.16b, v4.4b[2]\n" + ".inst 0x6fa4e21c // udot v28.4s, v16.16b, v4.4b[1]\n" + ".inst 0x6fa4ea1d // udot v29.4s, v16.16b, v4.4b[3]\n" + ".inst 0x6fa5e21e // udot v30.4s, v16.16b, v5.4b[1]\n" + ".inst 0x6fa5ea1f // udot v31.4s, v16.16b, v5.4b[3]\n" + "ldr q16, [%x[params], #0x50]\n" "add %x[params], %x[params], #0x80\n" - ".inst 0x6fa3e178 // udot v24.4s, v11.16b, v3.4b[1]\n" - ".inst 0x6fa3e979 // udot v25.4s, v11.16b, v3.4b[3]\n" - ".inst 0x6fa4e17a // udot v26.4s, v11.16b, v4.4b[1]\n" - ".inst 0x6fa4e97b // udot v27.4s, v11.16b, v4.4b[3]\n" - ".inst 0x6f85e15c // udot v28.4s, v10.16b, v5.4b[0]\n" - ".inst 0x6f85e95d // udot v29.4s, v10.16b, v5.4b[2]\n" - ".inst 0x6f86e15e // udot v30.4s, v10.16b, v6.4b[0]\n" - ".inst 0x6f86e95f // udot v31.4s, v10.16b, v6.4b[2]\n" - ".inst 0x6f84e118 // udot v24.4s, v8.16b, v4.4b[0]\n" - ".inst 0x6f84e919 // udot v25.4s, v8.16b, v4.4b[2]\n" - ".inst 0x6f85e11a // udot v26.4s, v8.16b, v5.4b[0]\n" - ".inst 0x6f85e91b // udot v27.4s, v8.16b, v5.4b[2]\n" - ".inst 0x6fa5e17c // udot v28.4s, v11.16b, v5.4b[1]\n" - ".inst 0x6fa5e97d // udot v29.4s, v11.16b, v5.4b[3]\n" - ".inst 0x6fa6e17e // udot v30.4s, v11.16b, v6.4b[1]\n" - ".inst 0x6fa6e97f // udot v31.4s, v11.16b, v6.4b[3]\n" - ".inst 0x6fa4e138 // udot v24.4s, v9.16b, v4.4b[1]\n" - ".inst 0x6fa4e939 // udot v25.4s, v9.16b, v4.4b[3]\n" + ".inst 0x6fa3e258 // udot v24.4s, v18.16b, v3.4b[1]\n" + ".inst 0x6fa3ea59 // udot v25.4s, v18.16b, v3.4b[3]\n" + ".inst 0x6fa4e25a // udot v26.4s, v18.16b, v4.4b[1]\n" + ".inst 0x6fa4ea5b // udot v27.4s, v18.16b, v4.4b[3]\n" + ".inst 0x6f85e27c // udot v28.4s, v19.16b, v5.4b[0]\n" + ".inst 0x6f85ea7d // udot v29.4s, v19.16b, v5.4b[2]\n" + ".inst 0x6f86e27e // udot v30.4s, v19.16b, v6.4b[0]\n" + ".inst 0x6f86ea7f // udot v31.4s, v19.16b, v6.4b[2]\n" + ".inst 0x6f84e238 // udot v24.4s, v17.16b, v4.4b[0]\n" + ".inst 0x6f84ea39 // udot v25.4s, v17.16b, v4.4b[2]\n" + ".inst 0x6f85e23a // udot v26.4s, v17.16b, v5.4b[0]\n" + ".inst 0x6f85ea3b // udot v27.4s, v17.16b, v5.4b[2]\n" + ".inst 0x6fa5e25c // udot v28.4s, v18.16b, v5.4b[1]\n" + ".inst 0x6fa5ea5d // udot v29.4s, v18.16b, v5.4b[3]\n" + ".inst 0x6fa6e25e // udot v30.4s, v18.16b, v6.4b[1]\n" + ".inst 0x6fa6ea5f // udot v31.4s, v18.16b, v6.4b[3]\n" + ".inst 0x6fa4e218 // udot v24.4s, v16.16b, v4.4b[1]\n" + ".inst 0x6fa4ea19 // udot v25.4s, v16.16b, v4.4b[3]\n" "sqrdmulh v24.4s, v24.4s, v21.4s\n" - ".inst 0x6fa5e13a // udot v26.4s, v9.16b, v5.4b[1]\n" - ".inst 0x6fa5e93b // udot v27.4s, v9.16b, v5.4b[3]\n" + ".inst 0x6fa5e21a // udot v26.4s, v16.16b, v5.4b[1]\n" + ".inst 0x6fa5ea1b // udot v27.4s, v16.16b, v5.4b[3]\n" "sqrdmulh v25.4s, v25.4s, v21.4s\n" - ".inst 0x6f86e11c // udot v28.4s, v8.16b, v6.4b[0]\n" - ".inst 0x6f86e91d // udot v29.4s, v8.16b, v6.4b[2]\n" + ".inst 0x6f86e23c // udot v28.4s, v17.16b, v6.4b[0]\n" + ".inst 0x6f86ea3d // udot v29.4s, v17.16b, v6.4b[2]\n" "sqrdmulh v26.4s, v26.4s, v21.4s\n" - ".inst 0x6f87e11e // udot v30.4s, v8.16b, v7.4b[0]\n" - ".inst 0x6f87e91f // udot v31.4s, v8.16b, v7.4b[2]\n" + ".inst 0x6f87e23e // udot v30.4s, v17.16b, v7.4b[0]\n" + ".inst 0x6f87ea3f // udot v31.4s, v17.16b, v7.4b[2]\n" "sqrdmulh v27.4s, v27.4s, v21.4s\n" - ".inst 0x6fa6e13c // udot v28.4s, v9.16b, v6.4b[1]\n" - ".inst 0x6fa6e93d // udot v29.4s, v9.16b, v6.4b[3]\n" + ".inst 0x6fa6e21c // udot v28.4s, v16.16b, v6.4b[1]\n" + ".inst 0x6fa6ea1d // udot v29.4s, v16.16b, v6.4b[3]\n" "and v19.16b, v24.16b, v20.16b\n" - ".inst 0x6fa7e13e // udot v30.4s, v9.16b, v7.4b[1]\n" - ".inst 0x6fa7e93f // udot v31.4s, v9.16b, v7.4b[3]\n" + ".inst 0x6fa7e21e // udot v30.4s, v16.16b, v7.4b[1]\n" + ".inst 0x6fa7ea1f // udot v31.4s, v16.16b, v7.4b[3]\n" "and v18.16b, v25.16b, v20.16b\n" "and v17.16b, v26.16b, v20.16b\n" "and v16.16b, v27.16b, v20.16b\n" @@ -536,14 +537,14 @@ void a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl "srshl v29.4s, v29.4s, v20.4s\n" "srshl v30.4s, v30.4s, v20.4s\n" "srshl v31.4s, v31.4s, v20.4s\n" - "add v24.4s, v24.4s, v14.4s\n" - "add v25.4s, v25.4s, v14.4s\n" - "add v26.4s, v26.4s, v14.4s\n" - "add v27.4s, v27.4s, v14.4s\n" - "add v28.4s, v28.4s, v14.4s\n" - "add v29.4s, v29.4s, v14.4s\n" - "add v30.4s, v30.4s, v14.4s\n" - "add v31.4s, v31.4s, v14.4s\n" + "add v24.4s, v24.4s, v13.4s\n" + "add v25.4s, v25.4s, v13.4s\n" + "add v26.4s, v26.4s, v13.4s\n" + "add v27.4s, v27.4s, v13.4s\n" + "add v28.4s, v28.4s, v13.4s\n" + "add v29.4s, v29.4s, v13.4s\n" + "add v30.4s, v30.4s, v13.4s\n" + "add v31.4s, v31.4s, v13.4s\n" "smin v24.4s, v24.4s, v15.4s\n" "smin v25.4s, v25.4s, v15.4s\n" "smin v26.4s, v26.4s, v15.4s\n" @@ -552,14 +553,14 @@ void a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl "smin v29.4s, v29.4s, v15.4s\n" "smin v30.4s, v30.4s, v15.4s\n" "smin v31.4s, v31.4s, v15.4s\n" - "smax v24.4s, v24.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v26.4s, v26.4s, v13.4s\n" - "smax v27.4s, v27.4s, v13.4s\n" - "smax v28.4s, v28.4s, v13.4s\n" - "smax v29.4s, v29.4s, v13.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" - "smax v31.4s, v31.4s, v13.4s\n" + "smax v24.4s, v24.4s, v14.4s\n" + "smax v25.4s, v25.4s, v14.4s\n" + "smax v26.4s, v26.4s, v14.4s\n" + "smax v27.4s, v27.4s, v14.4s\n" + "smax v28.4s, v28.4s, v14.4s\n" + "smax v29.4s, v29.4s, v14.4s\n" + "smax v30.4s, v30.4s, v14.4s\n" + "smax v31.4s, v31.4s, v14.4s\n" "uzp1 v24.16b, v24.16b, v24.16b\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" @@ -635,4 +636,5 @@ void a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp index 13f903b95dabd83bc0aea73b806c239fb067a89e..1f2d211be2004a321af4bc3061572d0a5265c2b8 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp index bbb817a883530c686efb368baf9c3b513c79f0fd..0770c126ecc77576edc14270d7c9821a8f1ece88 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp @@ -22,12 +22,13 @@ * SOFTWARE. */ -#if defined(__aarch64__) #include "arm_gemm.hpp" #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace depthwise { @@ -47,21 +48,21 @@ void a64_u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_imp __asm__ __volatile__( "lsr x10, %x[n_output_channels], #0x2\n" "add x20, %x[qp], %[offsetof_Requantize32_minval]\n" - "ld1r { v13.4s }, [x20]\n" + "ld1r { v15.4s }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_maxval]\n" - "ld1r { v11.4s }, [x20]\n" + "ld1r { v14.4s }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_a_offset]\n" - "ld1r { v3.16b }, [x20]\n" + "ld1r { v13.16b }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_b_offset]\n" "ld1r { v12.16b }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_c_offset]\n" - "ld1r { v14.4s }, [x20]\n" + "ld1r { v11.4s }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_per_layer_left_shift]\n" - "ld1r { v15.4s }, [x20]\n" + "ld1r { v10.4s }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_per_layer_mul]\n" "ld1r { v9.4s }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_per_layer_right_shift]\n" - "ld1r { v10.4s }, [x20]\n" + "ld1r { v8.4s }, [x20]\n" "mov x9, #0x0\n" "cbz x10, 9f\n" "1:" // Output channel loop @@ -89,256 +90,256 @@ void a64_u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_imp "cbz %x[rq_mul_ptr], 3f\n" "lsl x20, x9, #0x2\n" "ldr q9, [%x[rq_mul_ptr], x20]\n" - "ldr q10, [%x[rq_right_shift_ptr], x20]\n" + "ldr q8, [%x[rq_right_shift_ptr], x20]\n" "cbz %x[rq_left_shift_ptr], 3f\n" - "ldr q15, [%x[rq_left_shift_ptr], x20]\n" + "ldr q10, [%x[rq_left_shift_ptr], x20]\n" "3:" // Output channel loop: Load quantization parameters: Done - "ldr s8, [%x[weights]], #0x4\n" - "mov x20, %x[inptrs]\n" - "ldp x25, x28, [x20], #0x10\n" - "lsr x21, %x[kernel_points], #0x1\n" - "ldr d2, [x25, #0x0]\n" - "ldr d7, [x28, #0x0]\n" - "usubl v2.8h, v2.8b, v3.8b\n" - "usubl v7.8h, v7.8b, v3.8b\n" - "usubl v8.8h, v8.8b, v12.8b\n" - "cbz x21, 7f\n" - "ldr s6, [%x[weights]], #0x4\n" - "ldp x25, x28, [x20], #0x10\n" - "subs x21, x21, #0x1\n" - "usubl v6.8h, v6.8b, v12.8b\n" - "ldr d1, [x25, #0x0]\n" - "ldr d0, [x28, #0x0]\n" - "usubl v1.8h, v1.8b, v3.8b\n" - "usubl v0.8h, v0.8b, v3.8b\n" + "ldr s5, [%x[weights]], #0x4\n" + "mov x22, %x[inptrs]\n" + "ldp x21, x20, [x22], #0x10\n" + "lsr x23, %x[kernel_points], #0x1\n" + "ldr d0, [x21, #0x0]\n" + "ldr d4, [x20, #0x0]\n" + "usubl v0.8h, v0.8b, v13.8b\n" + "usubl v4.8h, v4.8b, v13.8b\n" + "usubl v5.8h, v5.8b, v12.8b\n" + "cbz x23, 7f\n" + "ldr s7, [%x[weights]], #0x4\n" + "ldp x21, x20, [x22], #0x10\n" + "subs x23, x23, #0x1\n" + "usubl v7.8h, v7.8b, v12.8b\n" + "ldr d3, [x21, #0x0]\n" + "ldr d6, [x20, #0x0]\n" + "usubl v3.8h, v3.8b, v13.8b\n" + "usubl v6.8h, v6.8b, v13.8b\n" "beq 5f\n" "4:" // Output channel loop: Kernel loop - "ldp x25, x28, [x20], #0x10\n" - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "subs x21, x21, #0x1\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "ldr d2, [x25, #0x0]\n" - "usubl v2.8h, v2.8b, v3.8b\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" - "ldr d7, [x28, #0x0]\n" - "ldr s8, [%x[weights]], #0x4\n" - "ldp x25, x28, [x20], #0x10\n" - "smlal v16.4s, v6.4h, v1.h[0]\n" - "smlal v17.4s, v6.4h, v1.h[1]\n" - "usubl v7.8h, v7.8b, v3.8b\n" - "smlal v18.4s, v6.4h, v1.h[2]\n" - "smlal v19.4s, v6.4h, v1.h[3]\n" - "usubl v8.8h, v8.8b, v12.8b\n" - "smlal v20.4s, v6.4h, v1.h[4]\n" - "smlal v21.4s, v6.4h, v1.h[5]\n" - "smlal v22.4s, v6.4h, v1.h[6]\n" - "smlal v23.4s, v6.4h, v1.h[7]\n" - "ldr d1, [x25, #0x0]\n" - "usubl v1.8h, v1.8b, v3.8b\n" - "smlal v24.4s, v6.4h, v0.h[0]\n" - "smlal v25.4s, v6.4h, v0.h[1]\n" - "smlal v26.4s, v6.4h, v0.h[2]\n" - "smlal v27.4s, v6.4h, v0.h[3]\n" - "smlal v28.4s, v6.4h, v0.h[4]\n" - "smlal v29.4s, v6.4h, v0.h[5]\n" - "smlal v30.4s, v6.4h, v0.h[6]\n" - "smlal v31.4s, v6.4h, v0.h[7]\n" - "ldr d0, [x28, #0x0]\n" - "ldr s6, [%x[weights]], #0x4\n" - "usubl v0.8h, v0.8b, v3.8b\n" - "usubl v6.8h, v6.8b, v12.8b\n" + "ldp x21, x20, [x22], #0x10\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "subs x23, x23, #0x1\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" + "ldr d0, [x21, #0x0]\n" + "usubl v0.8h, v0.8b, v13.8b\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" + "ldr d4, [x20, #0x0]\n" + "ldr s5, [%x[weights]], #0x4\n" + "ldp x21, x20, [x22], #0x10\n" + "smlal v16.4s, v7.4h, v3.h[0]\n" + "smlal v17.4s, v7.4h, v3.h[1]\n" + "usubl v4.8h, v4.8b, v13.8b\n" + "smlal v18.4s, v7.4h, v3.h[2]\n" + "smlal v19.4s, v7.4h, v3.h[3]\n" + "usubl v5.8h, v5.8b, v12.8b\n" + "smlal v20.4s, v7.4h, v3.h[4]\n" + "smlal v21.4s, v7.4h, v3.h[5]\n" + "smlal v22.4s, v7.4h, v3.h[6]\n" + "smlal v23.4s, v7.4h, v3.h[7]\n" + "ldr d3, [x21, #0x0]\n" + "usubl v3.8h, v3.8b, v13.8b\n" + "smlal v24.4s, v7.4h, v6.h[0]\n" + "smlal v25.4s, v7.4h, v6.h[1]\n" + "smlal v26.4s, v7.4h, v6.h[2]\n" + "smlal v27.4s, v7.4h, v6.h[3]\n" + "smlal v28.4s, v7.4h, v6.h[4]\n" + "smlal v29.4s, v7.4h, v6.h[5]\n" + "smlal v30.4s, v7.4h, v6.h[6]\n" + "smlal v31.4s, v7.4h, v6.h[7]\n" + "ldr d6, [x20, #0x0]\n" + "ldr s7, [%x[weights]], #0x4\n" + "usubl v6.8h, v6.8b, v13.8b\n" + "usubl v7.8h, v7.8b, v12.8b\n" "bgt 4b\n" "5:" // Output channel loop: Kernel loop tail "tbnz %x[kernel_points], #0, 6f\n" - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "smlal v16.4s, v6.4h, v1.h[0]\n" - "smlal v17.4s, v6.4h, v1.h[1]\n" - "sshl v16.4s, v16.4s, v15.4s\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "smlal v18.4s, v6.4h, v1.h[2]\n" - "smlal v19.4s, v6.4h, v1.h[3]\n" - "sshl v17.4s, v17.4s, v15.4s\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "sshl v18.4s, v18.4s, v15.4s\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "sshl v19.4s, v19.4s, v15.4s\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "smlal v16.4s, v7.4h, v3.h[0]\n" + "smlal v17.4s, v7.4h, v3.h[1]\n" + "sshl v16.4s, v16.4s, v10.4s\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "smlal v18.4s, v7.4h, v3.h[2]\n" + "smlal v19.4s, v7.4h, v3.h[3]\n" + "sshl v17.4s, v17.4s, v10.4s\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" + "sshl v18.4s, v18.4s, v10.4s\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" + "sshl v19.4s, v19.4s, v10.4s\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" "sqrdmulh v16.4s, v16.4s, v9.4s\n" - "smlal v20.4s, v6.4h, v1.h[4]\n" - "smlal v21.4s, v6.4h, v1.h[5]\n" + "smlal v20.4s, v7.4h, v3.h[4]\n" + "smlal v21.4s, v7.4h, v3.h[5]\n" "sqrdmulh v17.4s, v17.4s, v9.4s\n" - "smlal v22.4s, v6.4h, v1.h[6]\n" - "smlal v23.4s, v6.4h, v1.h[7]\n" + "smlal v22.4s, v7.4h, v3.h[6]\n" + "smlal v23.4s, v7.4h, v3.h[7]\n" "sqrdmulh v18.4s, v18.4s, v9.4s\n" - "smlal v24.4s, v6.4h, v0.h[0]\n" - "smlal v25.4s, v6.4h, v0.h[1]\n" + "smlal v24.4s, v7.4h, v6.h[0]\n" + "smlal v25.4s, v7.4h, v6.h[1]\n" "sqrdmulh v19.4s, v19.4s, v9.4s\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "and v5.16b, v16.16b, v10.16b\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "and v4.16b, v17.16b, v10.16b\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" - "and v2.16b, v18.16b, v10.16b\n" - "and v1.16b, v19.16b, v10.16b\n" - "sshl v20.4s, v20.4s, v15.4s\n" - "smlal v26.4s, v6.4h, v0.h[2]\n" - "sshl v21.4s, v21.4s, v15.4s\n" - "sshl v22.4s, v22.4s, v15.4s\n" - "smlal v27.4s, v6.4h, v0.h[3]\n" - "sshl v23.4s, v23.4s, v15.4s\n" - "sshl v24.4s, v24.4s, v15.4s\n" - "smlal v28.4s, v6.4h, v0.h[4]\n" - "sshl v25.4s, v25.4s, v15.4s\n" - "smlal v29.4s, v6.4h, v0.h[5]\n" - "smlal v30.4s, v6.4h, v0.h[6]\n" - "smlal v31.4s, v6.4h, v0.h[7]\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "and v3.16b, v16.16b, v8.16b\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "and v2.16b, v17.16b, v8.16b\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" + "and v1.16b, v18.16b, v8.16b\n" + "and v0.16b, v19.16b, v8.16b\n" + "sshl v20.4s, v20.4s, v10.4s\n" + "smlal v26.4s, v7.4h, v6.h[2]\n" + "sshl v21.4s, v21.4s, v10.4s\n" + "sshl v22.4s, v22.4s, v10.4s\n" + "smlal v27.4s, v7.4h, v6.h[3]\n" + "sshl v23.4s, v23.4s, v10.4s\n" + "sshl v24.4s, v24.4s, v10.4s\n" + "smlal v28.4s, v7.4h, v6.h[4]\n" + "sshl v25.4s, v25.4s, v10.4s\n" + "smlal v29.4s, v7.4h, v6.h[5]\n" + "smlal v30.4s, v7.4h, v6.h[6]\n" + "smlal v31.4s, v7.4h, v6.h[7]\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v0.4s, v0.4s, #0x1f\n" "sqrdmulh v20.4s, v20.4s, v9.4s\n" "sqrdmulh v21.4s, v21.4s, v9.4s\n" "sqrdmulh v22.4s, v22.4s, v9.4s\n" "sqrdmulh v23.4s, v23.4s, v9.4s\n" "sqrdmulh v24.4s, v24.4s, v9.4s\n" "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqadd v16.4s, v16.4s, v5.4s\n" - "sqadd v17.4s, v17.4s, v4.4s\n" - "sqadd v18.4s, v18.4s, v2.4s\n" - "sqadd v19.4s, v19.4s, v1.4s\n" - "and v8.16b, v20.16b, v10.16b\n" - "and v0.16b, v21.16b, v10.16b\n" - "and v5.16b, v22.16b, v10.16b\n" - "and v4.16b, v23.16b, v10.16b\n" - "and v2.16b, v24.16b, v10.16b\n" - "and v1.16b, v25.16b, v10.16b\n" - "sshl v26.4s, v26.4s, v15.4s\n" - "sshl v27.4s, v27.4s, v15.4s\n" - "sshl v28.4s, v28.4s, v15.4s\n" - "sshl v29.4s, v29.4s, v15.4s\n" - "sshl v30.4s, v30.4s, v15.4s\n" - "sshl v31.4s, v31.4s, v15.4s\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v3.4s\n" + "sqadd v17.4s, v17.4s, v2.4s\n" + "sqadd v18.4s, v18.4s, v1.4s\n" + "sqadd v19.4s, v19.4s, v0.4s\n" + "and v5.16b, v20.16b, v8.16b\n" + "and v4.16b, v21.16b, v8.16b\n" + "and v3.16b, v22.16b, v8.16b\n" + "and v2.16b, v23.16b, v8.16b\n" + "and v1.16b, v24.16b, v8.16b\n" + "and v0.16b, v25.16b, v8.16b\n" + "sshl v26.4s, v26.4s, v10.4s\n" + "sshl v27.4s, v27.4s, v10.4s\n" + "sshl v28.4s, v28.4s, v10.4s\n" + "sshl v29.4s, v29.4s, v10.4s\n" + "sshl v30.4s, v30.4s, v10.4s\n" + "sshl v31.4s, v31.4s, v10.4s\n" "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v0.4s, v0.4s, #0x1f\n" "sqrdmulh v26.4s, v26.4s, v9.4s\n" "sqrdmulh v27.4s, v27.4s, v9.4s\n" "sqrdmulh v28.4s, v28.4s, v9.4s\n" "sqrdmulh v29.4s, v29.4s, v9.4s\n" "sqrdmulh v30.4s, v30.4s, v9.4s\n" "sqrdmulh v31.4s, v31.4s, v9.4s\n" - "sqadd v20.4s, v20.4s, v8.4s\n" - "sqadd v21.4s, v21.4s, v0.4s\n" - "sqadd v22.4s, v22.4s, v5.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v24.4s, v24.4s, v2.4s\n" - "sqadd v25.4s, v25.4s, v1.4s\n" - "and v8.16b, v26.16b, v10.16b\n" - "and v0.16b, v27.16b, v10.16b\n" - "and v5.16b, v28.16b, v10.16b\n" - "and v4.16b, v29.16b, v10.16b\n" - "and v2.16b, v30.16b, v10.16b\n" - "and v1.16b, v31.16b, v10.16b\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v5.4s\n" + "sqadd v21.4s, v21.4s, v4.4s\n" + "sqadd v22.4s, v22.4s, v3.4s\n" + "sqadd v23.4s, v23.4s, v2.4s\n" + "sqadd v24.4s, v24.4s, v1.4s\n" + "sqadd v25.4s, v25.4s, v0.4s\n" + "and v5.16b, v26.16b, v8.16b\n" + "and v4.16b, v27.16b, v8.16b\n" + "and v3.16b, v28.16b, v8.16b\n" + "and v2.16b, v29.16b, v8.16b\n" + "and v1.16b, v30.16b, v8.16b\n" + "and v0.16b, v31.16b, v8.16b\n" "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" - "srshl v16.4s, v16.4s, v10.4s\n" - "srshl v17.4s, v17.4s, v10.4s\n" - "srshl v18.4s, v18.4s, v10.4s\n" - "srshl v19.4s, v19.4s, v10.4s\n" - "srshl v20.4s, v20.4s, v10.4s\n" - "srshl v21.4s, v21.4s, v10.4s\n" - "srshl v22.4s, v22.4s, v10.4s\n" - "srshl v23.4s, v23.4s, v10.4s\n" - "sqadd v26.4s, v26.4s, v8.4s\n" - "sqadd v27.4s, v27.4s, v0.4s\n" - "sqadd v28.4s, v28.4s, v5.4s\n" - "sqadd v29.4s, v29.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v2.4s\n" - "sqadd v31.4s, v31.4s, v1.4s\n" - "add v16.4s, v16.4s, v14.4s\n" - "add v17.4s, v17.4s, v14.4s\n" - "add v18.4s, v18.4s, v14.4s\n" - "add v19.4s, v19.4s, v14.4s\n" - "add v20.4s, v20.4s, v14.4s\n" - "add v21.4s, v21.4s, v14.4s\n" - "add v22.4s, v22.4s, v14.4s\n" - "add v23.4s, v23.4s, v14.4s\n" - "srshl v24.4s, v24.4s, v10.4s\n" - "srshl v25.4s, v25.4s, v10.4s\n" - "srshl v26.4s, v26.4s, v10.4s\n" - "srshl v27.4s, v27.4s, v10.4s\n" - "srshl v28.4s, v28.4s, v10.4s\n" - "srshl v29.4s, v29.4s, v10.4s\n" - "srshl v30.4s, v30.4s, v10.4s\n" - "srshl v31.4s, v31.4s, v10.4s\n" - "smin v16.4s, v16.4s, v11.4s\n" - "smin v17.4s, v17.4s, v11.4s\n" - "smin v18.4s, v18.4s, v11.4s\n" - "smin v19.4s, v19.4s, v11.4s\n" - "smin v20.4s, v20.4s, v11.4s\n" - "smin v21.4s, v21.4s, v11.4s\n" - "smin v22.4s, v22.4s, v11.4s\n" - "smin v23.4s, v23.4s, v11.4s\n" - "add v24.4s, v24.4s, v14.4s\n" - "add v25.4s, v25.4s, v14.4s\n" - "add v26.4s, v26.4s, v14.4s\n" - "add v27.4s, v27.4s, v14.4s\n" - "add v28.4s, v28.4s, v14.4s\n" - "add v29.4s, v29.4s, v14.4s\n" - "add v30.4s, v30.4s, v14.4s\n" - "add v31.4s, v31.4s, v14.4s\n" - "smax v16.4s, v16.4s, v13.4s\n" - "smax v17.4s, v17.4s, v13.4s\n" - "smax v18.4s, v18.4s, v13.4s\n" - "smax v19.4s, v19.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smax v21.4s, v21.4s, v13.4s\n" - "smax v22.4s, v22.4s, v13.4s\n" - "smax v23.4s, v23.4s, v13.4s\n" - "smin v24.4s, v24.4s, v11.4s\n" - "smin v25.4s, v25.4s, v11.4s\n" - "smin v26.4s, v26.4s, v11.4s\n" - "smin v27.4s, v27.4s, v11.4s\n" - "smin v28.4s, v28.4s, v11.4s\n" - "smin v29.4s, v29.4s, v11.4s\n" - "smin v30.4s, v30.4s, v11.4s\n" - "smin v31.4s, v31.4s, v11.4s\n" + "sshr v0.4s, v0.4s, #0x1f\n" + "srshl v16.4s, v16.4s, v8.4s\n" + "srshl v17.4s, v17.4s, v8.4s\n" + "srshl v18.4s, v18.4s, v8.4s\n" + "srshl v19.4s, v19.4s, v8.4s\n" + "srshl v20.4s, v20.4s, v8.4s\n" + "srshl v21.4s, v21.4s, v8.4s\n" + "srshl v22.4s, v22.4s, v8.4s\n" + "srshl v23.4s, v23.4s, v8.4s\n" + "sqadd v26.4s, v26.4s, v5.4s\n" + "sqadd v27.4s, v27.4s, v4.4s\n" + "sqadd v28.4s, v28.4s, v3.4s\n" + "sqadd v29.4s, v29.4s, v2.4s\n" + "sqadd v30.4s, v30.4s, v1.4s\n" + "sqadd v31.4s, v31.4s, v0.4s\n" + "add v16.4s, v16.4s, v11.4s\n" + "add v17.4s, v17.4s, v11.4s\n" + "add v18.4s, v18.4s, v11.4s\n" + "add v19.4s, v19.4s, v11.4s\n" + "add v20.4s, v20.4s, v11.4s\n" + "add v21.4s, v21.4s, v11.4s\n" + "add v22.4s, v22.4s, v11.4s\n" + "add v23.4s, v23.4s, v11.4s\n" + "srshl v24.4s, v24.4s, v8.4s\n" + "srshl v25.4s, v25.4s, v8.4s\n" + "srshl v26.4s, v26.4s, v8.4s\n" + "srshl v27.4s, v27.4s, v8.4s\n" + "srshl v28.4s, v28.4s, v8.4s\n" + "srshl v29.4s, v29.4s, v8.4s\n" + "srshl v30.4s, v30.4s, v8.4s\n" + "srshl v31.4s, v31.4s, v8.4s\n" + "smin v16.4s, v16.4s, v14.4s\n" + "smin v17.4s, v17.4s, v14.4s\n" + "smin v18.4s, v18.4s, v14.4s\n" + "smin v19.4s, v19.4s, v14.4s\n" + "smin v20.4s, v20.4s, v14.4s\n" + "smin v21.4s, v21.4s, v14.4s\n" + "smin v22.4s, v22.4s, v14.4s\n" + "smin v23.4s, v23.4s, v14.4s\n" + "add v24.4s, v24.4s, v11.4s\n" + "add v25.4s, v25.4s, v11.4s\n" + "add v26.4s, v26.4s, v11.4s\n" + "add v27.4s, v27.4s, v11.4s\n" + "add v28.4s, v28.4s, v11.4s\n" + "add v29.4s, v29.4s, v11.4s\n" + "add v30.4s, v30.4s, v11.4s\n" + "add v31.4s, v31.4s, v11.4s\n" + "smax v16.4s, v16.4s, v15.4s\n" + "smax v17.4s, v17.4s, v15.4s\n" + "smax v18.4s, v18.4s, v15.4s\n" + "smax v19.4s, v19.4s, v15.4s\n" + "smax v20.4s, v20.4s, v15.4s\n" + "smax v21.4s, v21.4s, v15.4s\n" + "smax v22.4s, v22.4s, v15.4s\n" + "smax v23.4s, v23.4s, v15.4s\n" + "smin v24.4s, v24.4s, v14.4s\n" + "smin v25.4s, v25.4s, v14.4s\n" + "smin v26.4s, v26.4s, v14.4s\n" + "smin v27.4s, v27.4s, v14.4s\n" + "smin v28.4s, v28.4s, v14.4s\n" + "smin v29.4s, v29.4s, v14.4s\n" + "smin v30.4s, v30.4s, v14.4s\n" + "smin v31.4s, v31.4s, v14.4s\n" "uzp1 v16.16b, v16.16b, v16.16b\n" "uzp1 v17.16b, v17.16b, v17.16b\n" "uzp1 v18.16b, v18.16b, v18.16b\n" @@ -347,263 +348,263 @@ void a64_u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_imp "uzp1 v21.16b, v21.16b, v21.16b\n" "uzp1 v22.16b, v22.16b, v22.16b\n" "uzp1 v23.16b, v23.16b, v23.16b\n" - "smax v24.4s, v24.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v26.4s, v26.4s, v13.4s\n" - "smax v27.4s, v27.4s, v13.4s\n" - "smax v28.4s, v28.4s, v13.4s\n" - "smax v29.4s, v29.4s, v13.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" - "smax v31.4s, v31.4s, v13.4s\n" + "smax v24.4s, v24.4s, v15.4s\n" + "smax v25.4s, v25.4s, v15.4s\n" + "smax v26.4s, v26.4s, v15.4s\n" + "smax v27.4s, v27.4s, v15.4s\n" + "smax v28.4s, v28.4s, v15.4s\n" + "smax v29.4s, v29.4s, v15.4s\n" + "smax v30.4s, v30.4s, v15.4s\n" + "smax v31.4s, v31.4s, v15.4s\n" "uzp1 v16.16b, v16.16b, v16.16b\n" - "str s16, [x20, x9]\n" - "ldr x20, [%x[outptrs], #0x40]\n" + "str s16, [x27, x9]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "uzp1 v17.16b, v17.16b, v17.16b\n" "uzp1 v18.16b, v18.16b, v18.16b\n" - "str s17, [x21, x9]\n" - "ldr x21, [%x[outptrs], #0x48]\n" + "str s17, [x26, x9]\n" + "ldr x26, [%x[outptrs], #0x48]\n" "uzp1 v19.16b, v19.16b, v19.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s18, [x22, x9]\n" - "ldr x22, [%x[outptrs], #0x50]\n" + "str s18, [x25, x9]\n" + "ldr x25, [%x[outptrs], #0x50]\n" "uzp1 v21.16b, v21.16b, v21.16b\n" "uzp1 v22.16b, v22.16b, v22.16b\n" - "str s19, [x23, x9]\n" - "ldr x23, [%x[outptrs], #0x58]\n" + "str s19, [x24, x9]\n" + "ldr x24, [%x[outptrs], #0x58]\n" "uzp1 v23.16b, v23.16b, v23.16b\n" "uzp1 v24.16b, v24.16b, v24.16b\n" - "str s20, [x24, x9]\n" - "ldr x24, [%x[outptrs], #0x60]\n" + "str s20, [x23, x9]\n" + "ldr x23, [%x[outptrs], #0x60]\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s21, [x25, x9]\n" - "ldr x25, [%x[outptrs], #0x68]\n" + "str s21, [x22, x9]\n" + "ldr x22, [%x[outptrs], #0x68]\n" "uzp1 v27.16b, v27.16b, v27.16b\n" "uzp1 v28.16b, v28.16b, v28.16b\n" - "str s22, [x26, x9]\n" - "ldr x26, [%x[outptrs], #0x70]\n" + "str s22, [x21, x9]\n" + "ldr x21, [%x[outptrs], #0x70]\n" "uzp1 v29.16b, v29.16b, v29.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s23, [x27, x9]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "str s23, [x20, x9]\n" + "ldr x20, [%x[outptrs], #0x78]\n" "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v24.16b, v24.16b, v24.16b\n" - "str s24, [x20, x9]\n" + "str s24, [x27, x9]\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s25, [x21, x9]\n" + "str s25, [x26, x9]\n" "uzp1 v27.16b, v27.16b, v27.16b\n" "uzp1 v28.16b, v28.16b, v28.16b\n" - "str s26, [x22, x9]\n" + "str s26, [x25, x9]\n" "uzp1 v29.16b, v29.16b, v29.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s27, [x23, x9]\n" + "str s27, [x24, x9]\n" "uzp1 v31.16b, v31.16b, v31.16b\n" - "str s28, [x24, x9]\n" - "str s29, [x25, x9]\n" - "str s30, [x26, x9]\n" - "str s31, [x27, x9]\n" + "str s28, [x23, x9]\n" + "str s29, [x22, x9]\n" + "str s30, [x21, x9]\n" + "str s31, [x20, x9]\n" "b 8f\n" "6:" // Output channel loop: Odd tail - "ldp x25, x28, [x20], #0x10\n" - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "ldr d2, [x25, #0x0]\n" - "usubl v2.8h, v2.8b, v3.8b\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" - "ldr s8, [%x[weights]], #0x4\n" - "ldr d7, [x28, #0x0]\n" - "smlal v16.4s, v6.4h, v1.h[0]\n" - "smlal v17.4s, v6.4h, v1.h[1]\n" - "usubl v8.8h, v8.8b, v12.8b\n" - "smlal v18.4s, v6.4h, v1.h[2]\n" - "smlal v19.4s, v6.4h, v1.h[3]\n" - "usubl v7.8h, v7.8b, v3.8b\n" - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "sshl v16.4s, v16.4s, v15.4s\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "sshl v17.4s, v17.4s, v15.4s\n" - "smlal v20.4s, v6.4h, v1.h[4]\n" - "smlal v21.4s, v6.4h, v1.h[5]\n" - "sshl v18.4s, v18.4s, v15.4s\n" - "smlal v22.4s, v6.4h, v1.h[6]\n" - "smlal v23.4s, v6.4h, v1.h[7]\n" - "sshl v19.4s, v19.4s, v15.4s\n" - "smlal v24.4s, v6.4h, v0.h[0]\n" - "smlal v25.4s, v6.4h, v0.h[1]\n" + "ldp x20, x28, [x22], #0x10\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "ldr x26, [%x[outptrs], #0x8]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" + "ldr d0, [x20, #0x0]\n" + "usubl v0.8h, v0.8b, v13.8b\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" + "ldr s5, [%x[weights]], #0x4\n" + "ldr d4, [x28, #0x0]\n" + "smlal v16.4s, v7.4h, v3.h[0]\n" + "smlal v17.4s, v7.4h, v3.h[1]\n" + "usubl v5.8h, v5.8b, v12.8b\n" + "smlal v18.4s, v7.4h, v3.h[2]\n" + "smlal v19.4s, v7.4h, v3.h[3]\n" + "usubl v4.8h, v4.8b, v13.8b\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "sshl v16.4s, v16.4s, v10.4s\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "sshl v17.4s, v17.4s, v10.4s\n" + "smlal v20.4s, v7.4h, v3.h[4]\n" + "smlal v21.4s, v7.4h, v3.h[5]\n" + "sshl v18.4s, v18.4s, v10.4s\n" + "smlal v22.4s, v7.4h, v3.h[6]\n" + "smlal v23.4s, v7.4h, v3.h[7]\n" + "sshl v19.4s, v19.4s, v10.4s\n" + "smlal v24.4s, v7.4h, v6.h[0]\n" + "smlal v25.4s, v7.4h, v6.h[1]\n" "sqrdmulh v16.4s, v16.4s, v9.4s\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" "sqrdmulh v17.4s, v17.4s, v9.4s\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" "sqrdmulh v18.4s, v18.4s, v9.4s\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" "sqrdmulh v19.4s, v19.4s, v9.4s\n" - "smlal v26.4s, v6.4h, v0.h[2]\n" - "smlal v27.4s, v6.4h, v0.h[3]\n" - "and v5.16b, v16.16b, v10.16b\n" - "smlal v28.4s, v6.4h, v0.h[4]\n" - "smlal v29.4s, v6.4h, v0.h[5]\n" - "and v4.16b, v17.16b, v10.16b\n" - "smlal v30.4s, v6.4h, v0.h[6]\n" - "smlal v31.4s, v6.4h, v0.h[7]\n" - "and v2.16b, v18.16b, v10.16b\n" - "and v1.16b, v19.16b, v10.16b\n" - "sshl v20.4s, v20.4s, v15.4s\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "sshl v21.4s, v21.4s, v15.4s\n" - "sshl v22.4s, v22.4s, v15.4s\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "sshl v23.4s, v23.4s, v15.4s\n" - "sshl v24.4s, v24.4s, v15.4s\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "sshl v25.4s, v25.4s, v15.4s\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" + "smlal v26.4s, v7.4h, v6.h[2]\n" + "smlal v27.4s, v7.4h, v6.h[3]\n" + "and v3.16b, v16.16b, v8.16b\n" + "smlal v28.4s, v7.4h, v6.h[4]\n" + "smlal v29.4s, v7.4h, v6.h[5]\n" + "and v2.16b, v17.16b, v8.16b\n" + "smlal v30.4s, v7.4h, v6.h[6]\n" + "smlal v31.4s, v7.4h, v6.h[7]\n" + "and v1.16b, v18.16b, v8.16b\n" + "and v0.16b, v19.16b, v8.16b\n" + "sshl v20.4s, v20.4s, v10.4s\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "sshl v21.4s, v21.4s, v10.4s\n" + "sshl v22.4s, v22.4s, v10.4s\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "sshl v23.4s, v23.4s, v10.4s\n" + "sshl v24.4s, v24.4s, v10.4s\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "sshl v25.4s, v25.4s, v10.4s\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v0.4s, v0.4s, #0x1f\n" "sqrdmulh v20.4s, v20.4s, v9.4s\n" "sqrdmulh v21.4s, v21.4s, v9.4s\n" "sqrdmulh v22.4s, v22.4s, v9.4s\n" "sqrdmulh v23.4s, v23.4s, v9.4s\n" "sqrdmulh v24.4s, v24.4s, v9.4s\n" "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqadd v16.4s, v16.4s, v5.4s\n" - "sqadd v17.4s, v17.4s, v4.4s\n" - "sqadd v18.4s, v18.4s, v2.4s\n" - "sqadd v19.4s, v19.4s, v1.4s\n" - "and v8.16b, v20.16b, v10.16b\n" - "and v0.16b, v21.16b, v10.16b\n" - "and v5.16b, v22.16b, v10.16b\n" - "and v4.16b, v23.16b, v10.16b\n" - "and v2.16b, v24.16b, v10.16b\n" - "and v1.16b, v25.16b, v10.16b\n" - "sshl v26.4s, v26.4s, v15.4s\n" - "sshl v27.4s, v27.4s, v15.4s\n" - "sshl v28.4s, v28.4s, v15.4s\n" - "sshl v29.4s, v29.4s, v15.4s\n" - "sshl v30.4s, v30.4s, v15.4s\n" - "sshl v31.4s, v31.4s, v15.4s\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v3.4s\n" + "sqadd v17.4s, v17.4s, v2.4s\n" + "sqadd v18.4s, v18.4s, v1.4s\n" + "sqadd v19.4s, v19.4s, v0.4s\n" + "and v5.16b, v20.16b, v8.16b\n" + "and v4.16b, v21.16b, v8.16b\n" + "and v3.16b, v22.16b, v8.16b\n" + "and v2.16b, v23.16b, v8.16b\n" + "and v1.16b, v24.16b, v8.16b\n" + "and v0.16b, v25.16b, v8.16b\n" + "sshl v26.4s, v26.4s, v10.4s\n" + "sshl v27.4s, v27.4s, v10.4s\n" + "sshl v28.4s, v28.4s, v10.4s\n" + "sshl v29.4s, v29.4s, v10.4s\n" + "sshl v30.4s, v30.4s, v10.4s\n" + "sshl v31.4s, v31.4s, v10.4s\n" "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v0.4s, v0.4s, #0x1f\n" "sqrdmulh v26.4s, v26.4s, v9.4s\n" "sqrdmulh v27.4s, v27.4s, v9.4s\n" "sqrdmulh v28.4s, v28.4s, v9.4s\n" "sqrdmulh v29.4s, v29.4s, v9.4s\n" "sqrdmulh v30.4s, v30.4s, v9.4s\n" "sqrdmulh v31.4s, v31.4s, v9.4s\n" - "sqadd v20.4s, v20.4s, v8.4s\n" - "sqadd v21.4s, v21.4s, v0.4s\n" - "sqadd v22.4s, v22.4s, v5.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v24.4s, v24.4s, v2.4s\n" - "sqadd v25.4s, v25.4s, v1.4s\n" - "and v8.16b, v26.16b, v10.16b\n" - "and v0.16b, v27.16b, v10.16b\n" - "and v5.16b, v28.16b, v10.16b\n" - "and v4.16b, v29.16b, v10.16b\n" - "and v2.16b, v30.16b, v10.16b\n" - "and v1.16b, v31.16b, v10.16b\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v5.4s\n" + "sqadd v21.4s, v21.4s, v4.4s\n" + "sqadd v22.4s, v22.4s, v3.4s\n" + "sqadd v23.4s, v23.4s, v2.4s\n" + "sqadd v24.4s, v24.4s, v1.4s\n" + "sqadd v25.4s, v25.4s, v0.4s\n" + "and v5.16b, v26.16b, v8.16b\n" + "and v4.16b, v27.16b, v8.16b\n" + "and v3.16b, v28.16b, v8.16b\n" + "and v2.16b, v29.16b, v8.16b\n" + "and v1.16b, v30.16b, v8.16b\n" + "and v0.16b, v31.16b, v8.16b\n" "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" - "srshl v16.4s, v16.4s, v10.4s\n" - "srshl v17.4s, v17.4s, v10.4s\n" - "srshl v18.4s, v18.4s, v10.4s\n" - "srshl v19.4s, v19.4s, v10.4s\n" - "srshl v20.4s, v20.4s, v10.4s\n" - "srshl v21.4s, v21.4s, v10.4s\n" - "srshl v22.4s, v22.4s, v10.4s\n" - "srshl v23.4s, v23.4s, v10.4s\n" - "sqadd v26.4s, v26.4s, v8.4s\n" - "sqadd v27.4s, v27.4s, v0.4s\n" - "sqadd v28.4s, v28.4s, v5.4s\n" - "sqadd v29.4s, v29.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v2.4s\n" - "sqadd v31.4s, v31.4s, v1.4s\n" - "add v16.4s, v16.4s, v14.4s\n" - "add v17.4s, v17.4s, v14.4s\n" - "add v18.4s, v18.4s, v14.4s\n" - "add v19.4s, v19.4s, v14.4s\n" - "add v20.4s, v20.4s, v14.4s\n" - "add v21.4s, v21.4s, v14.4s\n" - "add v22.4s, v22.4s, v14.4s\n" - "add v23.4s, v23.4s, v14.4s\n" - "srshl v24.4s, v24.4s, v10.4s\n" - "srshl v25.4s, v25.4s, v10.4s\n" - "srshl v26.4s, v26.4s, v10.4s\n" - "srshl v27.4s, v27.4s, v10.4s\n" - "srshl v28.4s, v28.4s, v10.4s\n" - "srshl v29.4s, v29.4s, v10.4s\n" - "srshl v30.4s, v30.4s, v10.4s\n" - "srshl v31.4s, v31.4s, v10.4s\n" - "smin v16.4s, v16.4s, v11.4s\n" - "smin v17.4s, v17.4s, v11.4s\n" - "smin v18.4s, v18.4s, v11.4s\n" - "smin v19.4s, v19.4s, v11.4s\n" - "smin v20.4s, v20.4s, v11.4s\n" - "smin v21.4s, v21.4s, v11.4s\n" - "smin v22.4s, v22.4s, v11.4s\n" - "smin v23.4s, v23.4s, v11.4s\n" - "add v24.4s, v24.4s, v14.4s\n" - "add v25.4s, v25.4s, v14.4s\n" - "add v26.4s, v26.4s, v14.4s\n" - "add v27.4s, v27.4s, v14.4s\n" - "add v28.4s, v28.4s, v14.4s\n" - "add v29.4s, v29.4s, v14.4s\n" - "add v30.4s, v30.4s, v14.4s\n" - "add v31.4s, v31.4s, v14.4s\n" - "smax v16.4s, v16.4s, v13.4s\n" - "smax v17.4s, v17.4s, v13.4s\n" - "smax v18.4s, v18.4s, v13.4s\n" - "smax v19.4s, v19.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smax v21.4s, v21.4s, v13.4s\n" - "smax v22.4s, v22.4s, v13.4s\n" - "smax v23.4s, v23.4s, v13.4s\n" - "smin v24.4s, v24.4s, v11.4s\n" - "smin v25.4s, v25.4s, v11.4s\n" - "smin v26.4s, v26.4s, v11.4s\n" - "smin v27.4s, v27.4s, v11.4s\n" - "smin v28.4s, v28.4s, v11.4s\n" - "smin v29.4s, v29.4s, v11.4s\n" - "smin v30.4s, v30.4s, v11.4s\n" - "smin v31.4s, v31.4s, v11.4s\n" + "sshr v0.4s, v0.4s, #0x1f\n" + "srshl v16.4s, v16.4s, v8.4s\n" + "srshl v17.4s, v17.4s, v8.4s\n" + "srshl v18.4s, v18.4s, v8.4s\n" + "srshl v19.4s, v19.4s, v8.4s\n" + "srshl v20.4s, v20.4s, v8.4s\n" + "srshl v21.4s, v21.4s, v8.4s\n" + "srshl v22.4s, v22.4s, v8.4s\n" + "srshl v23.4s, v23.4s, v8.4s\n" + "sqadd v26.4s, v26.4s, v5.4s\n" + "sqadd v27.4s, v27.4s, v4.4s\n" + "sqadd v28.4s, v28.4s, v3.4s\n" + "sqadd v29.4s, v29.4s, v2.4s\n" + "sqadd v30.4s, v30.4s, v1.4s\n" + "sqadd v31.4s, v31.4s, v0.4s\n" + "add v16.4s, v16.4s, v11.4s\n" + "add v17.4s, v17.4s, v11.4s\n" + "add v18.4s, v18.4s, v11.4s\n" + "add v19.4s, v19.4s, v11.4s\n" + "add v20.4s, v20.4s, v11.4s\n" + "add v21.4s, v21.4s, v11.4s\n" + "add v22.4s, v22.4s, v11.4s\n" + "add v23.4s, v23.4s, v11.4s\n" + "srshl v24.4s, v24.4s, v8.4s\n" + "srshl v25.4s, v25.4s, v8.4s\n" + "srshl v26.4s, v26.4s, v8.4s\n" + "srshl v27.4s, v27.4s, v8.4s\n" + "srshl v28.4s, v28.4s, v8.4s\n" + "srshl v29.4s, v29.4s, v8.4s\n" + "srshl v30.4s, v30.4s, v8.4s\n" + "srshl v31.4s, v31.4s, v8.4s\n" + "smin v16.4s, v16.4s, v14.4s\n" + "smin v17.4s, v17.4s, v14.4s\n" + "smin v18.4s, v18.4s, v14.4s\n" + "smin v19.4s, v19.4s, v14.4s\n" + "smin v20.4s, v20.4s, v14.4s\n" + "smin v21.4s, v21.4s, v14.4s\n" + "smin v22.4s, v22.4s, v14.4s\n" + "smin v23.4s, v23.4s, v14.4s\n" + "add v24.4s, v24.4s, v11.4s\n" + "add v25.4s, v25.4s, v11.4s\n" + "add v26.4s, v26.4s, v11.4s\n" + "add v27.4s, v27.4s, v11.4s\n" + "add v28.4s, v28.4s, v11.4s\n" + "add v29.4s, v29.4s, v11.4s\n" + "add v30.4s, v30.4s, v11.4s\n" + "add v31.4s, v31.4s, v11.4s\n" + "smax v16.4s, v16.4s, v15.4s\n" + "smax v17.4s, v17.4s, v15.4s\n" + "smax v18.4s, v18.4s, v15.4s\n" + "smax v19.4s, v19.4s, v15.4s\n" + "smax v20.4s, v20.4s, v15.4s\n" + "smax v21.4s, v21.4s, v15.4s\n" + "smax v22.4s, v22.4s, v15.4s\n" + "smax v23.4s, v23.4s, v15.4s\n" + "smin v24.4s, v24.4s, v14.4s\n" + "smin v25.4s, v25.4s, v14.4s\n" + "smin v26.4s, v26.4s, v14.4s\n" + "smin v27.4s, v27.4s, v14.4s\n" + "smin v28.4s, v28.4s, v14.4s\n" + "smin v29.4s, v29.4s, v14.4s\n" + "smin v30.4s, v30.4s, v14.4s\n" + "smin v31.4s, v31.4s, v14.4s\n" "uzp1 v16.16b, v16.16b, v16.16b\n" "uzp1 v17.16b, v17.16b, v17.16b\n" "uzp1 v18.16b, v18.16b, v18.16b\n" @@ -612,224 +613,224 @@ void a64_u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_imp "uzp1 v21.16b, v21.16b, v21.16b\n" "uzp1 v22.16b, v22.16b, v22.16b\n" "uzp1 v23.16b, v23.16b, v23.16b\n" - "smax v24.4s, v24.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v26.4s, v26.4s, v13.4s\n" - "smax v27.4s, v27.4s, v13.4s\n" - "smax v28.4s, v28.4s, v13.4s\n" - "smax v29.4s, v29.4s, v13.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" - "smax v31.4s, v31.4s, v13.4s\n" + "smax v24.4s, v24.4s, v15.4s\n" + "smax v25.4s, v25.4s, v15.4s\n" + "smax v26.4s, v26.4s, v15.4s\n" + "smax v27.4s, v27.4s, v15.4s\n" + "smax v28.4s, v28.4s, v15.4s\n" + "smax v29.4s, v29.4s, v15.4s\n" + "smax v30.4s, v30.4s, v15.4s\n" + "smax v31.4s, v31.4s, v15.4s\n" "uzp1 v16.16b, v16.16b, v16.16b\n" - "str s16, [x20, x9]\n" - "ldr x20, [%x[outptrs], #0x40]\n" + "str s16, [x27, x9]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "uzp1 v17.16b, v17.16b, v17.16b\n" "uzp1 v18.16b, v18.16b, v18.16b\n" - "str s17, [x21, x9]\n" - "ldr x21, [%x[outptrs], #0x48]\n" + "str s17, [x26, x9]\n" + "ldr x26, [%x[outptrs], #0x48]\n" "uzp1 v19.16b, v19.16b, v19.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s18, [x22, x9]\n" - "ldr x22, [%x[outptrs], #0x50]\n" + "str s18, [x25, x9]\n" + "ldr x25, [%x[outptrs], #0x50]\n" "uzp1 v21.16b, v21.16b, v21.16b\n" "uzp1 v22.16b, v22.16b, v22.16b\n" - "str s19, [x23, x9]\n" - "ldr x23, [%x[outptrs], #0x58]\n" + "str s19, [x24, x9]\n" + "ldr x24, [%x[outptrs], #0x58]\n" "uzp1 v23.16b, v23.16b, v23.16b\n" "uzp1 v24.16b, v24.16b, v24.16b\n" - "str s20, [x24, x9]\n" - "ldr x24, [%x[outptrs], #0x60]\n" + "str s20, [x23, x9]\n" + "ldr x23, [%x[outptrs], #0x60]\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s21, [x25, x9]\n" - "ldr x25, [%x[outptrs], #0x68]\n" + "str s21, [x22, x9]\n" + "ldr x22, [%x[outptrs], #0x68]\n" "uzp1 v27.16b, v27.16b, v27.16b\n" "uzp1 v28.16b, v28.16b, v28.16b\n" - "str s22, [x26, x9]\n" - "ldr x26, [%x[outptrs], #0x70]\n" + "str s22, [x21, x9]\n" + "ldr x21, [%x[outptrs], #0x70]\n" "uzp1 v29.16b, v29.16b, v29.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s23, [x27, x9]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "str s23, [x20, x9]\n" + "ldr x20, [%x[outptrs], #0x78]\n" "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v24.16b, v24.16b, v24.16b\n" - "str s24, [x20, x9]\n" + "str s24, [x27, x9]\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s25, [x21, x9]\n" + "str s25, [x26, x9]\n" "uzp1 v27.16b, v27.16b, v27.16b\n" "uzp1 v28.16b, v28.16b, v28.16b\n" - "str s26, [x22, x9]\n" + "str s26, [x25, x9]\n" "uzp1 v29.16b, v29.16b, v29.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s27, [x23, x9]\n" + "str s27, [x24, x9]\n" "uzp1 v31.16b, v31.16b, v31.16b\n" - "str s28, [x24, x9]\n" - "str s29, [x25, x9]\n" - "str s30, [x26, x9]\n" - "str s31, [x27, x9]\n" + "str s28, [x23, x9]\n" + "str s29, [x22, x9]\n" + "str s30, [x21, x9]\n" + "str s31, [x20, x9]\n" "b 8f\n" "7:" // Output channel loop: Single kernel point - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "sshl v16.4s, v16.4s, v15.4s\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "sshl v17.4s, v17.4s, v15.4s\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "sshl v18.4s, v18.4s, v15.4s\n" - "sshl v19.4s, v19.4s, v15.4s\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "sshl v16.4s, v16.4s, v10.4s\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "sshl v17.4s, v17.4s, v10.4s\n" + "ldr x26, [%x[outptrs], #0x8]\n" + "sshl v18.4s, v18.4s, v10.4s\n" + "sshl v19.4s, v19.4s, v10.4s\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" "sqrdmulh v16.4s, v16.4s, v9.4s\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" "sqrdmulh v17.4s, v17.4s, v9.4s\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" "sqrdmulh v18.4s, v18.4s, v9.4s\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "ldr x25, [%x[outptrs], #0x28]\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "ldr x22, [%x[outptrs], #0x28]\n" "sqrdmulh v19.4s, v19.4s, v9.4s\n" - "and v5.16b, v16.16b, v10.16b\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "and v4.16b, v17.16b, v10.16b\n" - "and v2.16b, v18.16b, v10.16b\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "and v1.16b, v19.16b, v10.16b\n" - "sshl v20.4s, v20.4s, v15.4s\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "sshl v21.4s, v21.4s, v15.4s\n" - "sshl v22.4s, v22.4s, v15.4s\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "sshl v23.4s, v23.4s, v15.4s\n" - "sshl v24.4s, v24.4s, v15.4s\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" - "sshl v25.4s, v25.4s, v15.4s\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" + "and v3.16b, v16.16b, v8.16b\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "and v2.16b, v17.16b, v8.16b\n" + "and v1.16b, v18.16b, v8.16b\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "and v0.16b, v19.16b, v8.16b\n" + "sshl v20.4s, v20.4s, v10.4s\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "sshl v21.4s, v21.4s, v10.4s\n" + "sshl v22.4s, v22.4s, v10.4s\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "sshl v23.4s, v23.4s, v10.4s\n" + "sshl v24.4s, v24.4s, v10.4s\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" + "sshl v25.4s, v25.4s, v10.4s\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v0.4s, v0.4s, #0x1f\n" "sqrdmulh v20.4s, v20.4s, v9.4s\n" "sqrdmulh v21.4s, v21.4s, v9.4s\n" "sqrdmulh v22.4s, v22.4s, v9.4s\n" "sqrdmulh v23.4s, v23.4s, v9.4s\n" "sqrdmulh v24.4s, v24.4s, v9.4s\n" "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqadd v16.4s, v16.4s, v5.4s\n" - "sqadd v17.4s, v17.4s, v4.4s\n" - "sqadd v18.4s, v18.4s, v2.4s\n" - "sqadd v19.4s, v19.4s, v1.4s\n" - "and v8.16b, v20.16b, v10.16b\n" - "and v0.16b, v21.16b, v10.16b\n" - "and v5.16b, v22.16b, v10.16b\n" - "and v4.16b, v23.16b, v10.16b\n" - "and v2.16b, v24.16b, v10.16b\n" - "and v1.16b, v25.16b, v10.16b\n" - "sshl v26.4s, v26.4s, v15.4s\n" - "sshl v27.4s, v27.4s, v15.4s\n" - "sshl v28.4s, v28.4s, v15.4s\n" - "sshl v29.4s, v29.4s, v15.4s\n" - "sshl v30.4s, v30.4s, v15.4s\n" - "sshl v31.4s, v31.4s, v15.4s\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v3.4s\n" + "sqadd v17.4s, v17.4s, v2.4s\n" + "sqadd v18.4s, v18.4s, v1.4s\n" + "sqadd v19.4s, v19.4s, v0.4s\n" + "and v5.16b, v20.16b, v8.16b\n" + "and v4.16b, v21.16b, v8.16b\n" + "and v3.16b, v22.16b, v8.16b\n" + "and v2.16b, v23.16b, v8.16b\n" + "and v1.16b, v24.16b, v8.16b\n" + "and v0.16b, v25.16b, v8.16b\n" + "sshl v26.4s, v26.4s, v10.4s\n" + "sshl v27.4s, v27.4s, v10.4s\n" + "sshl v28.4s, v28.4s, v10.4s\n" + "sshl v29.4s, v29.4s, v10.4s\n" + "sshl v30.4s, v30.4s, v10.4s\n" + "sshl v31.4s, v31.4s, v10.4s\n" "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v0.4s, v0.4s, #0x1f\n" "sqrdmulh v26.4s, v26.4s, v9.4s\n" "sqrdmulh v27.4s, v27.4s, v9.4s\n" "sqrdmulh v28.4s, v28.4s, v9.4s\n" "sqrdmulh v29.4s, v29.4s, v9.4s\n" "sqrdmulh v30.4s, v30.4s, v9.4s\n" "sqrdmulh v31.4s, v31.4s, v9.4s\n" - "sqadd v20.4s, v20.4s, v8.4s\n" - "sqadd v21.4s, v21.4s, v0.4s\n" - "sqadd v22.4s, v22.4s, v5.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v24.4s, v24.4s, v2.4s\n" - "sqadd v25.4s, v25.4s, v1.4s\n" - "and v8.16b, v26.16b, v10.16b\n" - "and v0.16b, v27.16b, v10.16b\n" - "and v5.16b, v28.16b, v10.16b\n" - "and v4.16b, v29.16b, v10.16b\n" - "and v2.16b, v30.16b, v10.16b\n" - "and v1.16b, v31.16b, v10.16b\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v5.4s\n" + "sqadd v21.4s, v21.4s, v4.4s\n" + "sqadd v22.4s, v22.4s, v3.4s\n" + "sqadd v23.4s, v23.4s, v2.4s\n" + "sqadd v24.4s, v24.4s, v1.4s\n" + "sqadd v25.4s, v25.4s, v0.4s\n" + "and v5.16b, v26.16b, v8.16b\n" + "and v4.16b, v27.16b, v8.16b\n" + "and v3.16b, v28.16b, v8.16b\n" + "and v2.16b, v29.16b, v8.16b\n" + "and v1.16b, v30.16b, v8.16b\n" + "and v0.16b, v31.16b, v8.16b\n" "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" - "srshl v16.4s, v16.4s, v10.4s\n" - "srshl v17.4s, v17.4s, v10.4s\n" - "srshl v18.4s, v18.4s, v10.4s\n" - "srshl v19.4s, v19.4s, v10.4s\n" - "srshl v20.4s, v20.4s, v10.4s\n" - "srshl v21.4s, v21.4s, v10.4s\n" - "srshl v22.4s, v22.4s, v10.4s\n" - "srshl v23.4s, v23.4s, v10.4s\n" - "sqadd v26.4s, v26.4s, v8.4s\n" - "sqadd v27.4s, v27.4s, v0.4s\n" - "sqadd v28.4s, v28.4s, v5.4s\n" - "sqadd v29.4s, v29.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v2.4s\n" - "sqadd v31.4s, v31.4s, v1.4s\n" - "add v16.4s, v16.4s, v14.4s\n" - "add v17.4s, v17.4s, v14.4s\n" - "add v18.4s, v18.4s, v14.4s\n" - "add v19.4s, v19.4s, v14.4s\n" - "add v20.4s, v20.4s, v14.4s\n" - "add v21.4s, v21.4s, v14.4s\n" - "add v22.4s, v22.4s, v14.4s\n" - "add v23.4s, v23.4s, v14.4s\n" - "srshl v24.4s, v24.4s, v10.4s\n" - "srshl v25.4s, v25.4s, v10.4s\n" - "srshl v26.4s, v26.4s, v10.4s\n" - "srshl v27.4s, v27.4s, v10.4s\n" - "srshl v28.4s, v28.4s, v10.4s\n" - "srshl v29.4s, v29.4s, v10.4s\n" - "srshl v30.4s, v30.4s, v10.4s\n" - "srshl v31.4s, v31.4s, v10.4s\n" - "smin v16.4s, v16.4s, v11.4s\n" - "smin v17.4s, v17.4s, v11.4s\n" - "smin v18.4s, v18.4s, v11.4s\n" - "smin v19.4s, v19.4s, v11.4s\n" - "smin v20.4s, v20.4s, v11.4s\n" - "smin v21.4s, v21.4s, v11.4s\n" - "smin v22.4s, v22.4s, v11.4s\n" - "smin v23.4s, v23.4s, v11.4s\n" - "add v24.4s, v24.4s, v14.4s\n" - "add v25.4s, v25.4s, v14.4s\n" - "add v26.4s, v26.4s, v14.4s\n" - "add v27.4s, v27.4s, v14.4s\n" - "add v28.4s, v28.4s, v14.4s\n" - "add v29.4s, v29.4s, v14.4s\n" - "add v30.4s, v30.4s, v14.4s\n" - "add v31.4s, v31.4s, v14.4s\n" - "smax v16.4s, v16.4s, v13.4s\n" - "smax v17.4s, v17.4s, v13.4s\n" - "smax v18.4s, v18.4s, v13.4s\n" - "smax v19.4s, v19.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smax v21.4s, v21.4s, v13.4s\n" - "smax v22.4s, v22.4s, v13.4s\n" - "smax v23.4s, v23.4s, v13.4s\n" - "smin v24.4s, v24.4s, v11.4s\n" - "smin v25.4s, v25.4s, v11.4s\n" - "smin v26.4s, v26.4s, v11.4s\n" - "smin v27.4s, v27.4s, v11.4s\n" - "smin v28.4s, v28.4s, v11.4s\n" - "smin v29.4s, v29.4s, v11.4s\n" - "smin v30.4s, v30.4s, v11.4s\n" - "smin v31.4s, v31.4s, v11.4s\n" + "sshr v0.4s, v0.4s, #0x1f\n" + "srshl v16.4s, v16.4s, v8.4s\n" + "srshl v17.4s, v17.4s, v8.4s\n" + "srshl v18.4s, v18.4s, v8.4s\n" + "srshl v19.4s, v19.4s, v8.4s\n" + "srshl v20.4s, v20.4s, v8.4s\n" + "srshl v21.4s, v21.4s, v8.4s\n" + "srshl v22.4s, v22.4s, v8.4s\n" + "srshl v23.4s, v23.4s, v8.4s\n" + "sqadd v26.4s, v26.4s, v5.4s\n" + "sqadd v27.4s, v27.4s, v4.4s\n" + "sqadd v28.4s, v28.4s, v3.4s\n" + "sqadd v29.4s, v29.4s, v2.4s\n" + "sqadd v30.4s, v30.4s, v1.4s\n" + "sqadd v31.4s, v31.4s, v0.4s\n" + "add v16.4s, v16.4s, v11.4s\n" + "add v17.4s, v17.4s, v11.4s\n" + "add v18.4s, v18.4s, v11.4s\n" + "add v19.4s, v19.4s, v11.4s\n" + "add v20.4s, v20.4s, v11.4s\n" + "add v21.4s, v21.4s, v11.4s\n" + "add v22.4s, v22.4s, v11.4s\n" + "add v23.4s, v23.4s, v11.4s\n" + "srshl v24.4s, v24.4s, v8.4s\n" + "srshl v25.4s, v25.4s, v8.4s\n" + "srshl v26.4s, v26.4s, v8.4s\n" + "srshl v27.4s, v27.4s, v8.4s\n" + "srshl v28.4s, v28.4s, v8.4s\n" + "srshl v29.4s, v29.4s, v8.4s\n" + "srshl v30.4s, v30.4s, v8.4s\n" + "srshl v31.4s, v31.4s, v8.4s\n" + "smin v16.4s, v16.4s, v14.4s\n" + "smin v17.4s, v17.4s, v14.4s\n" + "smin v18.4s, v18.4s, v14.4s\n" + "smin v19.4s, v19.4s, v14.4s\n" + "smin v20.4s, v20.4s, v14.4s\n" + "smin v21.4s, v21.4s, v14.4s\n" + "smin v22.4s, v22.4s, v14.4s\n" + "smin v23.4s, v23.4s, v14.4s\n" + "add v24.4s, v24.4s, v11.4s\n" + "add v25.4s, v25.4s, v11.4s\n" + "add v26.4s, v26.4s, v11.4s\n" + "add v27.4s, v27.4s, v11.4s\n" + "add v28.4s, v28.4s, v11.4s\n" + "add v29.4s, v29.4s, v11.4s\n" + "add v30.4s, v30.4s, v11.4s\n" + "add v31.4s, v31.4s, v11.4s\n" + "smax v16.4s, v16.4s, v15.4s\n" + "smax v17.4s, v17.4s, v15.4s\n" + "smax v18.4s, v18.4s, v15.4s\n" + "smax v19.4s, v19.4s, v15.4s\n" + "smax v20.4s, v20.4s, v15.4s\n" + "smax v21.4s, v21.4s, v15.4s\n" + "smax v22.4s, v22.4s, v15.4s\n" + "smax v23.4s, v23.4s, v15.4s\n" + "smin v24.4s, v24.4s, v14.4s\n" + "smin v25.4s, v25.4s, v14.4s\n" + "smin v26.4s, v26.4s, v14.4s\n" + "smin v27.4s, v27.4s, v14.4s\n" + "smin v28.4s, v28.4s, v14.4s\n" + "smin v29.4s, v29.4s, v14.4s\n" + "smin v30.4s, v30.4s, v14.4s\n" + "smin v31.4s, v31.4s, v14.4s\n" "uzp1 v16.16b, v16.16b, v16.16b\n" "uzp1 v17.16b, v17.16b, v17.16b\n" "uzp1 v18.16b, v18.16b, v18.16b\n" @@ -838,62 +839,62 @@ void a64_u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_imp "uzp1 v21.16b, v21.16b, v21.16b\n" "uzp1 v22.16b, v22.16b, v22.16b\n" "uzp1 v23.16b, v23.16b, v23.16b\n" - "smax v24.4s, v24.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v26.4s, v26.4s, v13.4s\n" - "smax v27.4s, v27.4s, v13.4s\n" - "smax v28.4s, v28.4s, v13.4s\n" - "smax v29.4s, v29.4s, v13.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" - "smax v31.4s, v31.4s, v13.4s\n" + "smax v24.4s, v24.4s, v15.4s\n" + "smax v25.4s, v25.4s, v15.4s\n" + "smax v26.4s, v26.4s, v15.4s\n" + "smax v27.4s, v27.4s, v15.4s\n" + "smax v28.4s, v28.4s, v15.4s\n" + "smax v29.4s, v29.4s, v15.4s\n" + "smax v30.4s, v30.4s, v15.4s\n" + "smax v31.4s, v31.4s, v15.4s\n" "uzp1 v16.16b, v16.16b, v16.16b\n" - "str s16, [x20, x9]\n" - "ldr x20, [%x[outptrs], #0x40]\n" + "str s16, [x27, x9]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "uzp1 v17.16b, v17.16b, v17.16b\n" "uzp1 v18.16b, v18.16b, v18.16b\n" - "str s17, [x21, x9]\n" - "ldr x21, [%x[outptrs], #0x48]\n" + "str s17, [x26, x9]\n" + "ldr x26, [%x[outptrs], #0x48]\n" "uzp1 v19.16b, v19.16b, v19.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s18, [x22, x9]\n" - "ldr x22, [%x[outptrs], #0x50]\n" + "str s18, [x25, x9]\n" + "ldr x25, [%x[outptrs], #0x50]\n" "uzp1 v21.16b, v21.16b, v21.16b\n" "uzp1 v22.16b, v22.16b, v22.16b\n" - "str s19, [x23, x9]\n" - "ldr x23, [%x[outptrs], #0x58]\n" + "str s19, [x24, x9]\n" + "ldr x24, [%x[outptrs], #0x58]\n" "uzp1 v23.16b, v23.16b, v23.16b\n" "uzp1 v24.16b, v24.16b, v24.16b\n" - "str s20, [x24, x9]\n" - "ldr x24, [%x[outptrs], #0x60]\n" + "str s20, [x23, x9]\n" + "ldr x23, [%x[outptrs], #0x60]\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s21, [x25, x9]\n" - "ldr x25, [%x[outptrs], #0x68]\n" + "str s21, [x22, x9]\n" + "ldr x22, [%x[outptrs], #0x68]\n" "uzp1 v27.16b, v27.16b, v27.16b\n" "uzp1 v28.16b, v28.16b, v28.16b\n" - "str s22, [x26, x9]\n" - "ldr x26, [%x[outptrs], #0x70]\n" + "str s22, [x21, x9]\n" + "ldr x21, [%x[outptrs], #0x70]\n" "uzp1 v29.16b, v29.16b, v29.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s23, [x27, x9]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "str s23, [x20, x9]\n" + "ldr x20, [%x[outptrs], #0x78]\n" "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v24.16b, v24.16b, v24.16b\n" - "str s24, [x20, x9]\n" + "str s24, [x27, x9]\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s25, [x21, x9]\n" + "str s25, [x26, x9]\n" "uzp1 v27.16b, v27.16b, v27.16b\n" "uzp1 v28.16b, v28.16b, v28.16b\n" - "str s26, [x22, x9]\n" + "str s26, [x25, x9]\n" "uzp1 v29.16b, v29.16b, v29.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s27, [x23, x9]\n" + "str s27, [x24, x9]\n" "uzp1 v31.16b, v31.16b, v31.16b\n" - "str s28, [x24, x9]\n" - "str s29, [x25, x9]\n" - "str s30, [x26, x9]\n" - "str s31, [x27, x9]\n" + "str s28, [x23, x9]\n" + "str s29, [x22, x9]\n" + "str s30, [x21, x9]\n" + "str s31, [x20, x9]\n" "8:" // Output channel loop: Done "add x9, x9, #0x4\n" "cmp x9, x10, LSL #2\n" @@ -936,354 +937,354 @@ void a64_u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_imp "cbz %x[rq_left_shift_ptr], 15f\n" "tbz %x[n_output_channels], #1, 13f\n" "ld1 { v9.d }[0], [x22], #0x8\n" - "ld1 { v10.d }[0], [x21], #0x8\n" - "ld1 { v15.d }[0], [x20], #0x8\n" + "ld1 { v8.d }[0], [x21], #0x8\n" + "ld1 { v10.d }[0], [x20], #0x8\n" "tbz %x[n_output_channels], #0, 14f\n" "ld1 { v9.s }[2], [x22], #0x4\n" - "ld1 { v10.s }[2], [x21], #0x4\n" - "ld1 { v15.s }[2], [x20], #0x4\n" + "ld1 { v8.s }[2], [x21], #0x4\n" + "ld1 { v10.s }[2], [x20], #0x4\n" "b 14f\n" "13:" // Output channel oddments: Load quantization parameters: With left shift: Bit 1: Unset "ld1 { v9.s }[0], [x22], #0x4\n" - "ld1 { v10.s }[0], [x21], #0x4\n" - "ld1 { v15.s }[0], [x20], #0x4\n" + "ld1 { v8.s }[0], [x21], #0x4\n" + "ld1 { v10.s }[0], [x20], #0x4\n" "14:" // Output channel oddments: Load quantization parameters: With left shift: Bit 1: End "b 18f\n" "15:" // Output channel oddments: Load quantization parameters: No left shift "tbz %x[n_output_channels], #1, 16f\n" "ld1 { v9.d }[0], [x22], #0x8\n" - "ld1 { v10.d }[0], [x21], #0x8\n" + "ld1 { v8.d }[0], [x21], #0x8\n" "tbz %x[n_output_channels], #0, 17f\n" "ld1 { v9.s }[2], [x22], #0x4\n" - "ld1 { v10.s }[2], [x21], #0x4\n" + "ld1 { v8.s }[2], [x21], #0x4\n" "b 17f\n" "16:" // Output channel oddments: Load quantization parameters: No left shift: Bit 1: Unset "ld1 { v9.s }[0], [x22], #0x4\n" - "ld1 { v10.s }[0], [x21], #0x4\n" + "ld1 { v8.s }[0], [x21], #0x4\n" "17:" // Output channel oddments: Load quantization parameters: No left shift: Bit 1: End "18:" // Output channel oddments: Load quantization parameters: Done - "ldr s8, [%x[weights]], #0x4\n" - "mov x20, %x[inptrs]\n" - "ldp x25, x28, [x20], #0x10\n" - "lsr x21, %x[kernel_points], #0x1\n" - "ldr d2, [x25, #0x0]\n" - "ldr d7, [x28, #0x0]\n" - "usubl v2.8h, v2.8b, v3.8b\n" - "usubl v7.8h, v7.8b, v3.8b\n" - "usubl v8.8h, v8.8b, v12.8b\n" - "cbz x21, 22f\n" - "ldr s6, [%x[weights]], #0x4\n" - "ldp x25, x28, [x20], #0x10\n" - "subs x21, x21, #0x1\n" - "usubl v6.8h, v6.8b, v12.8b\n" - "ldr d1, [x25, #0x0]\n" - "ldr d0, [x28, #0x0]\n" - "usubl v1.8h, v1.8b, v3.8b\n" - "usubl v0.8h, v0.8b, v3.8b\n" + "ldr s5, [%x[weights]], #0x4\n" + "mov x22, %x[inptrs]\n" + "ldp x21, x20, [x22], #0x10\n" + "lsr x23, %x[kernel_points], #0x1\n" + "ldr d0, [x21, #0x0]\n" + "ldr d4, [x20, #0x0]\n" + "usubl v0.8h, v0.8b, v13.8b\n" + "usubl v4.8h, v4.8b, v13.8b\n" + "usubl v5.8h, v5.8b, v12.8b\n" + "cbz x23, 22f\n" + "ldr s7, [%x[weights]], #0x4\n" + "ldp x21, x20, [x22], #0x10\n" + "subs x23, x23, #0x1\n" + "usubl v7.8h, v7.8b, v12.8b\n" + "ldr d3, [x21, #0x0]\n" + "ldr d6, [x20, #0x0]\n" + "usubl v3.8h, v3.8b, v13.8b\n" + "usubl v6.8h, v6.8b, v13.8b\n" "beq 20f\n" "19:" // Output channel oddments: Kernel loop - "ldp x25, x28, [x20], #0x10\n" - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "subs x21, x21, #0x1\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "ldr d2, [x25, #0x0]\n" - "usubl v2.8h, v2.8b, v3.8b\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" - "ldr d7, [x28, #0x0]\n" - "ldr s8, [%x[weights]], #0x4\n" - "ldp x25, x28, [x20], #0x10\n" - "smlal v16.4s, v6.4h, v1.h[0]\n" - "smlal v17.4s, v6.4h, v1.h[1]\n" - "usubl v7.8h, v7.8b, v3.8b\n" - "smlal v18.4s, v6.4h, v1.h[2]\n" - "smlal v19.4s, v6.4h, v1.h[3]\n" - "usubl v8.8h, v8.8b, v12.8b\n" - "smlal v20.4s, v6.4h, v1.h[4]\n" - "smlal v21.4s, v6.4h, v1.h[5]\n" - "smlal v22.4s, v6.4h, v1.h[6]\n" - "smlal v23.4s, v6.4h, v1.h[7]\n" - "ldr d1, [x25, #0x0]\n" - "usubl v1.8h, v1.8b, v3.8b\n" - "smlal v24.4s, v6.4h, v0.h[0]\n" - "smlal v25.4s, v6.4h, v0.h[1]\n" - "smlal v26.4s, v6.4h, v0.h[2]\n" - "smlal v27.4s, v6.4h, v0.h[3]\n" - "smlal v28.4s, v6.4h, v0.h[4]\n" - "smlal v29.4s, v6.4h, v0.h[5]\n" - "smlal v30.4s, v6.4h, v0.h[6]\n" - "smlal v31.4s, v6.4h, v0.h[7]\n" - "ldr d0, [x28, #0x0]\n" - "ldr s6, [%x[weights]], #0x4\n" - "usubl v0.8h, v0.8b, v3.8b\n" - "usubl v6.8h, v6.8b, v12.8b\n" + "ldp x21, x20, [x22], #0x10\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "subs x23, x23, #0x1\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" + "ldr d0, [x21, #0x0]\n" + "usubl v0.8h, v0.8b, v13.8b\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" + "ldr d4, [x20, #0x0]\n" + "ldr s5, [%x[weights]], #0x4\n" + "ldp x21, x20, [x22], #0x10\n" + "smlal v16.4s, v7.4h, v3.h[0]\n" + "smlal v17.4s, v7.4h, v3.h[1]\n" + "usubl v4.8h, v4.8b, v13.8b\n" + "smlal v18.4s, v7.4h, v3.h[2]\n" + "smlal v19.4s, v7.4h, v3.h[3]\n" + "usubl v5.8h, v5.8b, v12.8b\n" + "smlal v20.4s, v7.4h, v3.h[4]\n" + "smlal v21.4s, v7.4h, v3.h[5]\n" + "smlal v22.4s, v7.4h, v3.h[6]\n" + "smlal v23.4s, v7.4h, v3.h[7]\n" + "ldr d3, [x21, #0x0]\n" + "usubl v3.8h, v3.8b, v13.8b\n" + "smlal v24.4s, v7.4h, v6.h[0]\n" + "smlal v25.4s, v7.4h, v6.h[1]\n" + "smlal v26.4s, v7.4h, v6.h[2]\n" + "smlal v27.4s, v7.4h, v6.h[3]\n" + "smlal v28.4s, v7.4h, v6.h[4]\n" + "smlal v29.4s, v7.4h, v6.h[5]\n" + "smlal v30.4s, v7.4h, v6.h[6]\n" + "smlal v31.4s, v7.4h, v6.h[7]\n" + "ldr d6, [x20, #0x0]\n" + "ldr s7, [%x[weights]], #0x4\n" + "usubl v6.8h, v6.8b, v13.8b\n" + "usubl v7.8h, v7.8b, v12.8b\n" "bgt 19b\n" "20:" // Output channel oddments: Kernel loop tail "tbnz %x[kernel_points], #0, 21f\n" - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" - "smlal v16.4s, v6.4h, v1.h[0]\n" - "smlal v17.4s, v6.4h, v1.h[1]\n" - "smlal v18.4s, v6.4h, v1.h[2]\n" - "smlal v19.4s, v6.4h, v1.h[3]\n" - "smlal v20.4s, v6.4h, v1.h[4]\n" - "smlal v21.4s, v6.4h, v1.h[5]\n" - "smlal v22.4s, v6.4h, v1.h[6]\n" - "smlal v23.4s, v6.4h, v1.h[7]\n" - "smlal v24.4s, v6.4h, v0.h[0]\n" - "smlal v25.4s, v6.4h, v0.h[1]\n" - "smlal v26.4s, v6.4h, v0.h[2]\n" - "smlal v27.4s, v6.4h, v0.h[3]\n" - "smlal v28.4s, v6.4h, v0.h[4]\n" - "smlal v29.4s, v6.4h, v0.h[5]\n" - "smlal v30.4s, v6.4h, v0.h[6]\n" - "smlal v31.4s, v6.4h, v0.h[7]\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" + "smlal v16.4s, v7.4h, v3.h[0]\n" + "smlal v17.4s, v7.4h, v3.h[1]\n" + "smlal v18.4s, v7.4h, v3.h[2]\n" + "smlal v19.4s, v7.4h, v3.h[3]\n" + "smlal v20.4s, v7.4h, v3.h[4]\n" + "smlal v21.4s, v7.4h, v3.h[5]\n" + "smlal v22.4s, v7.4h, v3.h[6]\n" + "smlal v23.4s, v7.4h, v3.h[7]\n" + "smlal v24.4s, v7.4h, v6.h[0]\n" + "smlal v25.4s, v7.4h, v6.h[1]\n" + "smlal v26.4s, v7.4h, v6.h[2]\n" + "smlal v27.4s, v7.4h, v6.h[3]\n" + "smlal v28.4s, v7.4h, v6.h[4]\n" + "smlal v29.4s, v7.4h, v6.h[5]\n" + "smlal v30.4s, v7.4h, v6.h[6]\n" + "smlal v31.4s, v7.4h, v6.h[7]\n" "b 23f\n" "21:" // Output channel oddments: Odd tail - "ldp x25, x28, [x20], #0x10\n" - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "ldr d2, [x25, #0x0]\n" - "usubl v2.8h, v2.8b, v3.8b\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" - "ldr d7, [x28, #0x0]\n" - "ldr s8, [%x[weights]], #0x4\n" - "smlal v16.4s, v6.4h, v1.h[0]\n" - "smlal v17.4s, v6.4h, v1.h[1]\n" - "usubl v7.8h, v7.8b, v3.8b\n" - "smlal v18.4s, v6.4h, v1.h[2]\n" - "smlal v19.4s, v6.4h, v1.h[3]\n" - "usubl v8.8h, v8.8b, v12.8b\n" - "smlal v20.4s, v6.4h, v1.h[4]\n" - "smlal v21.4s, v6.4h, v1.h[5]\n" - "smlal v22.4s, v6.4h, v1.h[6]\n" - "smlal v23.4s, v6.4h, v1.h[7]\n" - "smlal v24.4s, v6.4h, v0.h[0]\n" - "smlal v25.4s, v6.4h, v0.h[1]\n" - "smlal v26.4s, v6.4h, v0.h[2]\n" - "smlal v27.4s, v6.4h, v0.h[3]\n" - "smlal v28.4s, v6.4h, v0.h[4]\n" - "smlal v29.4s, v6.4h, v0.h[5]\n" - "smlal v30.4s, v6.4h, v0.h[6]\n" - "smlal v31.4s, v6.4h, v0.h[7]\n" - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" + "ldp x21, x20, [x22], #0x10\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" + "ldr d2, [x21, #0x0]\n" + "usubl v2.8h, v2.8b, v13.8b\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" + "ldr d1, [x20, #0x0]\n" + "ldr s0, [%x[weights]], #0x4\n" + "smlal v16.4s, v7.4h, v3.h[0]\n" + "smlal v17.4s, v7.4h, v3.h[1]\n" + "usubl v1.8h, v1.8b, v13.8b\n" + "smlal v18.4s, v7.4h, v3.h[2]\n" + "smlal v19.4s, v7.4h, v3.h[3]\n" + "usubl v0.8h, v0.8b, v12.8b\n" + "smlal v20.4s, v7.4h, v3.h[4]\n" + "smlal v21.4s, v7.4h, v3.h[5]\n" + "smlal v22.4s, v7.4h, v3.h[6]\n" + "smlal v23.4s, v7.4h, v3.h[7]\n" + "smlal v24.4s, v7.4h, v6.h[0]\n" + "smlal v25.4s, v7.4h, v6.h[1]\n" + "smlal v26.4s, v7.4h, v6.h[2]\n" + "smlal v27.4s, v7.4h, v6.h[3]\n" + "smlal v28.4s, v7.4h, v6.h[4]\n" + "smlal v29.4s, v7.4h, v6.h[5]\n" + "smlal v30.4s, v7.4h, v6.h[6]\n" + "smlal v31.4s, v7.4h, v6.h[7]\n" + "smlal v16.4s, v0.4h, v2.h[0]\n" + "smlal v17.4s, v0.4h, v2.h[1]\n" + "smlal v18.4s, v0.4h, v2.h[2]\n" + "smlal v19.4s, v0.4h, v2.h[3]\n" + "smlal v20.4s, v0.4h, v2.h[4]\n" + "smlal v21.4s, v0.4h, v2.h[5]\n" + "smlal v22.4s, v0.4h, v2.h[6]\n" + "smlal v23.4s, v0.4h, v2.h[7]\n" + "smlal v24.4s, v0.4h, v1.h[0]\n" + "smlal v25.4s, v0.4h, v1.h[1]\n" + "smlal v26.4s, v0.4h, v1.h[2]\n" + "smlal v27.4s, v0.4h, v1.h[3]\n" + "smlal v28.4s, v0.4h, v1.h[4]\n" + "smlal v29.4s, v0.4h, v1.h[5]\n" + "smlal v30.4s, v0.4h, v1.h[6]\n" + "smlal v31.4s, v0.4h, v1.h[7]\n" "b 23f\n" "22:" // Output channel oddments: Single kernel point - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" "23:" // Output channel oddments: Done - "sshl v16.4s, v16.4s, v15.4s\n" - "sshl v17.4s, v17.4s, v15.4s\n" - "sshl v18.4s, v18.4s, v15.4s\n" - "sshl v19.4s, v19.4s, v15.4s\n" + "sshl v16.4s, v16.4s, v10.4s\n" + "sshl v17.4s, v17.4s, v10.4s\n" + "sshl v18.4s, v18.4s, v10.4s\n" + "sshl v19.4s, v19.4s, v10.4s\n" "sqrdmulh v16.4s, v16.4s, v9.4s\n" "sqrdmulh v17.4s, v17.4s, v9.4s\n" "sqrdmulh v18.4s, v18.4s, v9.4s\n" "sqrdmulh v19.4s, v19.4s, v9.4s\n" - "and v5.16b, v16.16b, v10.16b\n" - "and v4.16b, v17.16b, v10.16b\n" - "and v2.16b, v18.16b, v10.16b\n" - "and v1.16b, v19.16b, v10.16b\n" - "sshl v20.4s, v20.4s, v15.4s\n" - "sshl v21.4s, v21.4s, v15.4s\n" - "sshl v22.4s, v22.4s, v15.4s\n" - "sshl v23.4s, v23.4s, v15.4s\n" - "sshl v24.4s, v24.4s, v15.4s\n" - "sshl v25.4s, v25.4s, v15.4s\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" + "and v3.16b, v16.16b, v8.16b\n" + "and v2.16b, v17.16b, v8.16b\n" + "and v1.16b, v18.16b, v8.16b\n" + "and v0.16b, v19.16b, v8.16b\n" + "sshl v20.4s, v20.4s, v10.4s\n" + "sshl v21.4s, v21.4s, v10.4s\n" + "sshl v22.4s, v22.4s, v10.4s\n" + "sshl v23.4s, v23.4s, v10.4s\n" + "sshl v24.4s, v24.4s, v10.4s\n" + "sshl v25.4s, v25.4s, v10.4s\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v0.4s, v0.4s, #0x1f\n" "sqrdmulh v20.4s, v20.4s, v9.4s\n" "sqrdmulh v21.4s, v21.4s, v9.4s\n" "sqrdmulh v22.4s, v22.4s, v9.4s\n" "sqrdmulh v23.4s, v23.4s, v9.4s\n" "sqrdmulh v24.4s, v24.4s, v9.4s\n" "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqadd v16.4s, v16.4s, v5.4s\n" - "sqadd v17.4s, v17.4s, v4.4s\n" - "sqadd v18.4s, v18.4s, v2.4s\n" - "sqadd v19.4s, v19.4s, v1.4s\n" - "and v8.16b, v20.16b, v10.16b\n" - "and v0.16b, v21.16b, v10.16b\n" - "and v5.16b, v22.16b, v10.16b\n" - "and v4.16b, v23.16b, v10.16b\n" - "and v2.16b, v24.16b, v10.16b\n" - "and v1.16b, v25.16b, v10.16b\n" - "sshl v26.4s, v26.4s, v15.4s\n" - "sshl v27.4s, v27.4s, v15.4s\n" - "sshl v28.4s, v28.4s, v15.4s\n" - "sshl v29.4s, v29.4s, v15.4s\n" - "sshl v30.4s, v30.4s, v15.4s\n" - "sshl v31.4s, v31.4s, v15.4s\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v3.4s\n" + "sqadd v17.4s, v17.4s, v2.4s\n" + "sqadd v18.4s, v18.4s, v1.4s\n" + "sqadd v19.4s, v19.4s, v0.4s\n" + "and v5.16b, v20.16b, v8.16b\n" + "and v4.16b, v21.16b, v8.16b\n" + "and v3.16b, v22.16b, v8.16b\n" + "and v2.16b, v23.16b, v8.16b\n" + "and v1.16b, v24.16b, v8.16b\n" + "and v0.16b, v25.16b, v8.16b\n" + "sshl v26.4s, v26.4s, v10.4s\n" + "sshl v27.4s, v27.4s, v10.4s\n" + "sshl v28.4s, v28.4s, v10.4s\n" + "sshl v29.4s, v29.4s, v10.4s\n" + "sshl v30.4s, v30.4s, v10.4s\n" + "sshl v31.4s, v31.4s, v10.4s\n" "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v0.4s, v0.4s, #0x1f\n" "sqrdmulh v26.4s, v26.4s, v9.4s\n" "sqrdmulh v27.4s, v27.4s, v9.4s\n" "sqrdmulh v28.4s, v28.4s, v9.4s\n" "sqrdmulh v29.4s, v29.4s, v9.4s\n" "sqrdmulh v30.4s, v30.4s, v9.4s\n" "sqrdmulh v31.4s, v31.4s, v9.4s\n" - "sqadd v20.4s, v20.4s, v8.4s\n" - "sqadd v21.4s, v21.4s, v0.4s\n" - "sqadd v22.4s, v22.4s, v5.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v24.4s, v24.4s, v2.4s\n" - "sqadd v25.4s, v25.4s, v1.4s\n" - "and v8.16b, v26.16b, v10.16b\n" - "and v0.16b, v27.16b, v10.16b\n" - "and v5.16b, v28.16b, v10.16b\n" - "and v4.16b, v29.16b, v10.16b\n" - "and v2.16b, v30.16b, v10.16b\n" - "and v1.16b, v31.16b, v10.16b\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v5.4s\n" + "sqadd v21.4s, v21.4s, v4.4s\n" + "sqadd v22.4s, v22.4s, v3.4s\n" + "sqadd v23.4s, v23.4s, v2.4s\n" + "sqadd v24.4s, v24.4s, v1.4s\n" + "sqadd v25.4s, v25.4s, v0.4s\n" + "and v5.16b, v26.16b, v8.16b\n" + "and v4.16b, v27.16b, v8.16b\n" + "and v3.16b, v28.16b, v8.16b\n" + "and v2.16b, v29.16b, v8.16b\n" + "and v1.16b, v30.16b, v8.16b\n" + "and v0.16b, v31.16b, v8.16b\n" "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" - "sqadd v26.4s, v26.4s, v8.4s\n" - "sqadd v27.4s, v27.4s, v0.4s\n" - "sqadd v28.4s, v28.4s, v5.4s\n" - "sqadd v29.4s, v29.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v2.4s\n" - "sqadd v31.4s, v31.4s, v1.4s\n" - "srshl v16.4s, v16.4s, v10.4s\n" - "srshl v17.4s, v17.4s, v10.4s\n" - "srshl v18.4s, v18.4s, v10.4s\n" - "srshl v19.4s, v19.4s, v10.4s\n" - "srshl v20.4s, v20.4s, v10.4s\n" - "srshl v21.4s, v21.4s, v10.4s\n" - "srshl v22.4s, v22.4s, v10.4s\n" - "srshl v23.4s, v23.4s, v10.4s\n" - "srshl v24.4s, v24.4s, v10.4s\n" - "srshl v25.4s, v25.4s, v10.4s\n" - "srshl v26.4s, v26.4s, v10.4s\n" - "srshl v27.4s, v27.4s, v10.4s\n" - "srshl v28.4s, v28.4s, v10.4s\n" - "srshl v29.4s, v29.4s, v10.4s\n" - "srshl v30.4s, v30.4s, v10.4s\n" - "srshl v31.4s, v31.4s, v10.4s\n" - "add v16.4s, v16.4s, v14.4s\n" - "add v17.4s, v17.4s, v14.4s\n" - "add v18.4s, v18.4s, v14.4s\n" - "add v19.4s, v19.4s, v14.4s\n" - "add v20.4s, v20.4s, v14.4s\n" - "add v21.4s, v21.4s, v14.4s\n" - "add v22.4s, v22.4s, v14.4s\n" - "add v23.4s, v23.4s, v14.4s\n" - "add v24.4s, v24.4s, v14.4s\n" - "add v25.4s, v25.4s, v14.4s\n" - "add v26.4s, v26.4s, v14.4s\n" - "add v27.4s, v27.4s, v14.4s\n" - "add v28.4s, v28.4s, v14.4s\n" - "add v29.4s, v29.4s, v14.4s\n" - "add v30.4s, v30.4s, v14.4s\n" - "add v31.4s, v31.4s, v14.4s\n" - "smin v16.4s, v16.4s, v11.4s\n" - "smin v17.4s, v17.4s, v11.4s\n" - "smin v18.4s, v18.4s, v11.4s\n" - "smin v19.4s, v19.4s, v11.4s\n" - "smin v20.4s, v20.4s, v11.4s\n" - "smin v21.4s, v21.4s, v11.4s\n" - "smin v22.4s, v22.4s, v11.4s\n" - "smin v23.4s, v23.4s, v11.4s\n" - "smin v24.4s, v24.4s, v11.4s\n" - "smin v25.4s, v25.4s, v11.4s\n" - "smin v26.4s, v26.4s, v11.4s\n" - "smin v27.4s, v27.4s, v11.4s\n" - "smin v28.4s, v28.4s, v11.4s\n" - "smin v29.4s, v29.4s, v11.4s\n" - "smin v30.4s, v30.4s, v11.4s\n" - "smin v31.4s, v31.4s, v11.4s\n" - "smax v16.4s, v16.4s, v13.4s\n" - "smax v17.4s, v17.4s, v13.4s\n" - "smax v18.4s, v18.4s, v13.4s\n" - "smax v19.4s, v19.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smax v21.4s, v21.4s, v13.4s\n" - "smax v22.4s, v22.4s, v13.4s\n" - "smax v23.4s, v23.4s, v13.4s\n" - "smax v24.4s, v24.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v26.4s, v26.4s, v13.4s\n" - "smax v27.4s, v27.4s, v13.4s\n" - "smax v28.4s, v28.4s, v13.4s\n" - "smax v29.4s, v29.4s, v13.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" - "smax v31.4s, v31.4s, v13.4s\n" + "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v26.4s, v26.4s, v5.4s\n" + "sqadd v27.4s, v27.4s, v4.4s\n" + "sqadd v28.4s, v28.4s, v3.4s\n" + "sqadd v29.4s, v29.4s, v2.4s\n" + "sqadd v30.4s, v30.4s, v1.4s\n" + "sqadd v31.4s, v31.4s, v0.4s\n" + "srshl v16.4s, v16.4s, v8.4s\n" + "srshl v17.4s, v17.4s, v8.4s\n" + "srshl v18.4s, v18.4s, v8.4s\n" + "srshl v19.4s, v19.4s, v8.4s\n" + "srshl v20.4s, v20.4s, v8.4s\n" + "srshl v21.4s, v21.4s, v8.4s\n" + "srshl v22.4s, v22.4s, v8.4s\n" + "srshl v23.4s, v23.4s, v8.4s\n" + "srshl v24.4s, v24.4s, v8.4s\n" + "srshl v25.4s, v25.4s, v8.4s\n" + "srshl v26.4s, v26.4s, v8.4s\n" + "srshl v27.4s, v27.4s, v8.4s\n" + "srshl v28.4s, v28.4s, v8.4s\n" + "srshl v29.4s, v29.4s, v8.4s\n" + "srshl v30.4s, v30.4s, v8.4s\n" + "srshl v31.4s, v31.4s, v8.4s\n" + "add v16.4s, v16.4s, v11.4s\n" + "add v17.4s, v17.4s, v11.4s\n" + "add v18.4s, v18.4s, v11.4s\n" + "add v19.4s, v19.4s, v11.4s\n" + "add v20.4s, v20.4s, v11.4s\n" + "add v21.4s, v21.4s, v11.4s\n" + "add v22.4s, v22.4s, v11.4s\n" + "add v23.4s, v23.4s, v11.4s\n" + "add v24.4s, v24.4s, v11.4s\n" + "add v25.4s, v25.4s, v11.4s\n" + "add v26.4s, v26.4s, v11.4s\n" + "add v27.4s, v27.4s, v11.4s\n" + "add v28.4s, v28.4s, v11.4s\n" + "add v29.4s, v29.4s, v11.4s\n" + "add v30.4s, v30.4s, v11.4s\n" + "add v31.4s, v31.4s, v11.4s\n" + "smin v16.4s, v16.4s, v14.4s\n" + "smin v17.4s, v17.4s, v14.4s\n" + "smin v18.4s, v18.4s, v14.4s\n" + "smin v19.4s, v19.4s, v14.4s\n" + "smin v20.4s, v20.4s, v14.4s\n" + "smin v21.4s, v21.4s, v14.4s\n" + "smin v22.4s, v22.4s, v14.4s\n" + "smin v23.4s, v23.4s, v14.4s\n" + "smin v24.4s, v24.4s, v14.4s\n" + "smin v25.4s, v25.4s, v14.4s\n" + "smin v26.4s, v26.4s, v14.4s\n" + "smin v27.4s, v27.4s, v14.4s\n" + "smin v28.4s, v28.4s, v14.4s\n" + "smin v29.4s, v29.4s, v14.4s\n" + "smin v30.4s, v30.4s, v14.4s\n" + "smin v31.4s, v31.4s, v14.4s\n" + "smax v16.4s, v16.4s, v15.4s\n" + "smax v17.4s, v17.4s, v15.4s\n" + "smax v18.4s, v18.4s, v15.4s\n" + "smax v19.4s, v19.4s, v15.4s\n" + "smax v20.4s, v20.4s, v15.4s\n" + "smax v21.4s, v21.4s, v15.4s\n" + "smax v22.4s, v22.4s, v15.4s\n" + "smax v23.4s, v23.4s, v15.4s\n" + "smax v24.4s, v24.4s, v15.4s\n" + "smax v25.4s, v25.4s, v15.4s\n" + "smax v26.4s, v26.4s, v15.4s\n" + "smax v27.4s, v27.4s, v15.4s\n" + "smax v28.4s, v28.4s, v15.4s\n" + "smax v29.4s, v29.4s, v15.4s\n" + "smax v30.4s, v30.4s, v15.4s\n" + "smax v31.4s, v31.4s, v15.4s\n" "uzp1 v16.16b, v16.16b, v16.16b\n" "uzp1 v17.16b, v17.16b, v17.16b\n" "uzp1 v18.16b, v18.16b, v18.16b\n" @@ -1317,158 +1318,156 @@ void a64_u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_imp "uzp1 v30.16b, v30.16b, v30.16b\n" "uzp1 v31.16b, v31.16b, v31.16b\n" "tbz %x[n_output_channels], #1, 24f\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "add x20, x20, x9\n" - "add x21, x21, x9\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "add x22, x22, x9\n" - "add x23, x23, x9\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "add x24, x24, x9\n" - "add x25, x25, x9\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "add x26, x26, x9\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" "add x27, x27, x9\n" - "st1 { v16.h }[0], [x20]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "add x20, x20, x9\n" - "st1 { v17.h }[0], [x21]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "add x21, x21, x9\n" - "st1 { v18.h }[0], [x22]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "add x22, x22, x9\n" - "st1 { v19.h }[0], [x23]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "add x23, x23, x9\n" - "st1 { v20.h }[0], [x24]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "add x24, x24, x9\n" - "st1 { v21.h }[0], [x25]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "add x25, x25, x9\n" - "st1 { v22.h }[0], [x26]\n" - "ldr x26, [%x[outptrs], #0x70]\n" "add x26, x26, x9\n" - "st1 { v23.h }[0], [x27]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "add x25, x25, x9\n" + "add x24, x24, x9\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "add x23, x23, x9\n" + "add x22, x22, x9\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "add x21, x21, x9\n" + "add x20, x20, x9\n" + "st1 { v16.h }[0], [x27]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "add x27, x27, x9\n" + "st1 { v17.h }[0], [x26]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "add x26, x26, x9\n" + "st1 { v18.h }[0], [x25]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "add x25, x25, x9\n" + "st1 { v19.h }[0], [x24]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "add x24, x24, x9\n" + "st1 { v20.h }[0], [x23]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "add x23, x23, x9\n" + "st1 { v21.h }[0], [x22]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "add x22, x22, x9\n" + "st1 { v22.h }[0], [x21]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "add x21, x21, x9\n" + "st1 { v23.h }[0], [x20]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "add x20, x20, x9\n" "add x9, x9, #0x2\n" - "st1 { v24.h }[0], [x20]\n" - "st1 { v25.h }[0], [x21]\n" - "st1 { v26.h }[0], [x22]\n" - "st1 { v27.h }[0], [x23]\n" - "st1 { v28.h }[0], [x24]\n" - "st1 { v29.h }[0], [x25]\n" - "st1 { v30.h }[0], [x26]\n" - "st1 { v31.h }[0], [x27]\n" + "st1 { v24.h }[0], [x27]\n" + "st1 { v25.h }[0], [x26]\n" + "st1 { v26.h }[0], [x25]\n" + "st1 { v27.h }[0], [x24]\n" + "st1 { v28.h }[0], [x23]\n" + "st1 { v29.h }[0], [x22]\n" + "st1 { v30.h }[0], [x21]\n" + "st1 { v31.h }[0], [x20]\n" "tbz %x[n_output_channels], #0, 25f\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "add x20, x20, x9\n" - "add x21, x21, x9\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "add x22, x22, x9\n" - "add x23, x23, x9\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "add x24, x24, x9\n" - "add x25, x25, x9\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "add x26, x26, x9\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" "add x27, x27, x9\n" - "st1 { v16.b }[2], [x20]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "add x20, x20, x9\n" - "st1 { v17.b }[2], [x21]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "add x21, x21, x9\n" - "st1 { v18.b }[2], [x22]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "add x22, x22, x9\n" - "st1 { v19.b }[2], [x23]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "add x23, x23, x9\n" - "st1 { v20.b }[2], [x24]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "add x24, x24, x9\n" - "st1 { v21.b }[2], [x25]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "add x25, x25, x9\n" - "st1 { v22.b }[2], [x26]\n" - "ldr x26, [%x[outptrs], #0x70]\n" "add x26, x26, x9\n" - "st1 { v23.b }[2], [x27]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "add x25, x25, x9\n" + "add x24, x24, x9\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "add x23, x23, x9\n" + "add x22, x22, x9\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "add x21, x21, x9\n" + "add x20, x20, x9\n" + "st1 { v16.b }[2], [x27]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "add x27, x27, x9\n" - "st1 { v24.b }[2], [x20]\n" - "st1 { v25.b }[2], [x21]\n" - "st1 { v26.b }[2], [x22]\n" - "st1 { v27.b }[2], [x23]\n" - "st1 { v28.b }[2], [x24]\n" - "st1 { v29.b }[2], [x25]\n" - "st1 { v30.b }[2], [x26]\n" - "st1 { v31.b }[2], [x27]\n" + "st1 { v17.b }[2], [x26]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "add x26, x26, x9\n" + "st1 { v18.b }[2], [x25]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "add x25, x25, x9\n" + "st1 { v19.b }[2], [x24]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "add x24, x24, x9\n" + "st1 { v20.b }[2], [x23]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "add x23, x23, x9\n" + "st1 { v21.b }[2], [x22]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "add x22, x22, x9\n" + "st1 { v22.b }[2], [x21]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "add x21, x21, x9\n" + "st1 { v23.b }[2], [x20]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "add x20, x20, x9\n" + "st1 { v24.b }[2], [x27]\n" + "st1 { v25.b }[2], [x26]\n" + "st1 { v26.b }[2], [x25]\n" + "st1 { v27.b }[2], [x24]\n" + "st1 { v28.b }[2], [x23]\n" + "st1 { v29.b }[2], [x22]\n" + "st1 { v30.b }[2], [x21]\n" + "st1 { v31.b }[2], [x20]\n" "b 25f\n" "24:" // Output channel oddments: Done: Store: Bit 1: Unset - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "add x20, x20, x9\n" - "add x21, x21, x9\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "add x22, x22, x9\n" - "add x23, x23, x9\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "add x24, x24, x9\n" - "add x25, x25, x9\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "add x26, x26, x9\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" "add x27, x27, x9\n" - "st1 { v16.b }[0], [x20]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "add x20, x20, x9\n" - "st1 { v17.b }[0], [x21]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "add x21, x21, x9\n" - "st1 { v18.b }[0], [x22]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "add x22, x22, x9\n" - "st1 { v19.b }[0], [x23]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "add x23, x23, x9\n" - "st1 { v20.b }[0], [x24]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "add x24, x24, x9\n" - "st1 { v21.b }[0], [x25]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "add x25, x25, x9\n" - "st1 { v22.b }[0], [x26]\n" - "ldr x26, [%x[outptrs], #0x70]\n" "add x26, x26, x9\n" - "st1 { v23.b }[0], [x27]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "add x25, x25, x9\n" + "add x24, x24, x9\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "add x23, x23, x9\n" + "add x22, x22, x9\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "add x21, x21, x9\n" + "add x20, x20, x9\n" + "st1 { v16.b }[0], [x27]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "add x27, x27, x9\n" - "st1 { v24.b }[0], [x20]\n" - "st1 { v25.b }[0], [x21]\n" - "st1 { v26.b }[0], [x22]\n" - "st1 { v27.b }[0], [x23]\n" - "st1 { v28.b }[0], [x24]\n" - "st1 { v29.b }[0], [x25]\n" - "st1 { v30.b }[0], [x26]\n" - "st1 { v31.b }[0], [x27]\n" + "st1 { v17.b }[0], [x26]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "add x26, x26, x9\n" + "st1 { v18.b }[0], [x25]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "add x25, x25, x9\n" + "st1 { v19.b }[0], [x24]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "add x24, x24, x9\n" + "st1 { v20.b }[0], [x23]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "add x23, x23, x9\n" + "st1 { v21.b }[0], [x22]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "add x22, x22, x9\n" + "st1 { v22.b }[0], [x21]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "add x21, x21, x9\n" + "st1 { v23.b }[0], [x20]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "add x20, x20, x9\n" + "st1 { v24.b }[0], [x27]\n" + "st1 { v25.b }[0], [x26]\n" + "st1 { v26.b }[0], [x25]\n" + "st1 { v27.b }[0], [x24]\n" + "st1 { v28.b }[0], [x23]\n" + "st1 { v29.b }[0], [x22]\n" + "st1 { v30.b }[0], [x21]\n" + "st1 { v31.b }[0], [x20]\n" "25:" // Output channel oddments: Done: Store: Bit 1: End - "26:" // Done - : [weights] "+&r" (weights) : [bias] "r" (bias), [inptrs] "r" (inptrs), [kernel_points] "r" ((uint64_t) kernel_points), [n_output_channels] "r" ((uint64_t) n_output_channels), [offsetof_Requantize32_a_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [offsetof_Requantize32_per_layer_left_shift] "I" (offsetof(arm_gemm::Requantize32, per_layer_left_shift)), [offsetof_Requantize32_per_layer_mul] "I" (offsetof(arm_gemm::Requantize32, per_layer_mul)), [offsetof_Requantize32_per_layer_right_shift] "I" (offsetof(arm_gemm::Requantize32, per_layer_right_shift)), [outptrs] "r" (outptrs), [qp] "r" (&qp), [rq_left_shift_ptr] "r" (per_channel_left_shifts), [rq_mul_ptr] "r" (per_channel_muls), [rq_right_shift_ptr] "r" (per_channel_right_shifts) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" @@ -1477,4 +1476,5 @@ void a64_u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_imp } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp index 55731060f4646111fb77d3aa01e3519d7f3e3d2b..20a37b157fe9d76b23282069dddb85380d83656d 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp @@ -22,8 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" - +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include @@ -35,16 +34,7 @@ namespace arm_conv { namespace depthwise { -void a64_u8qa_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( - const unsigned int, - const uint8_t *const *const, - const uint8_t *const, - const int32_t *const, - const arm_gemm::Requantize32 &, - const int32_t *const, - const int32_t *const, - uint8_t *const *const -); +void a64_u8qa_nhwc_3x3_s1_output2x2_mla_depthfirst_impl(unsigned int, const uint8_t *const *, const uint8_t *, const int32_t *, const arm_gemm::Requantize32 &, const int32_t *, const int32_t *, uint8_t *const *); class a64_u8qa_nhwc_3x3_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirstStrategy { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp index afc6695ff18773fae6ad0779aa9a13f20f0ec08b..d1872c90f81562b02760b5c69ac7eab56998f4ea 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp @@ -91,1070 +91,1070 @@ void a64_u8qa_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( requant_muls, requant_shifts, outptrs); __asm__ __volatile__( - "ldr x7, [%x[params], %[offsetof_Params_n_channels]]\n" + "ldr x16, [%x[params], %[offsetof_Params_n_channels]]\n" "ldr x23, [%x[params], %[offsetof_Params_requant]]\n" - "lsr x8, x7, #0x3\n" + "lsr x15, x16, #0x3\n" "add x20, x23, %[offsetof_Requantize32_b_offset]\n" - "ld1r { v12.16b }, [x20]\n" + "ld1r { v18.16b }, [x20]\n" "ldr x22, [%x[params], %[offsetof_Params_outptrs]]\n" "add x21, x23, %[offsetof_Requantize32_c_offset]\n" "add x20, x23, %[offsetof_Requantize32_minval]\n" - "ld1r { v20.8h }, [x21]\n" - "ld1r { v15.8h }, [x20]\n" + "ld1r { v5.8h }, [x21]\n" + "ld1r { v14.8h }, [x20]\n" "add x20, x23, %[offsetof_Requantize32_maxval]\n" - "mov x17, #0x0\n" - "ld1r { v13.8h }, [x20]\n" - "mov x16, #0x0\n" - "add x15, %x[params], %[offsetof_Params_inptrs]\n" - "ldr x14, [%x[params], %[offsetof_Params_weights]]\n" - "ldr x13, [%x[params], %[offsetof_Params_requant_muls]]\n" - "ldr x12, [%x[params], %[offsetof_Params_requant_shifts]]\n" - "ldp x11, x10, [x22, #0x0]\n" - "ldp x9, x28, [x22, #0x10]\n" - "cbz x8, 3f\n" - "ldr d0, [x14, #0x0]\n" - "ldr d1, [x14, #0x8]\n" - "subs x8, x8, #0x1\n" - "usubl v0.8h, v0.8b, v12.8b\n" - "ldr d2, [x14, #0x10]\n" - "ldr d3, [x14, #0x18]\n" - "usubl v1.8h, v1.8b, v12.8b\n" - "usubl v2.8h, v2.8b, v12.8b\n" - "ldr d4, [x14, #0x20]\n" - "ldr d5, [x14, #0x28]\n" - "usubl v3.8h, v3.8b, v12.8b\n" - "usubl v4.8h, v4.8b, v12.8b\n" - "ldr d6, [x14, #0x30]\n" - "ldr d7, [x14, #0x38]\n" - "usubl v5.8h, v5.8b, v12.8b\n" - "usubl v6.8h, v6.8b, v12.8b\n" - "ldr d8, [x14, #0x40]\n" - "ldr x27, [%x[params], %[offsetof_Params_bias]]\n" - "usubl v7.8h, v7.8b, v12.8b\n" - "usubl v8.8h, v8.8b, v12.8b\n" - "ldr q14, [x27, #0x0]\n" - "ldr q11, [x27, #0x10]\n" - "add x27, x27, #0x20\n" - "str x27, [%x[params], %[offsetof_Params_bias]]\n" - "ldp x24, x23, [x15, #0x0]\n" - "ldp x22, x21, [x15, #0x10]\n" - "mov v16.16b, v14.16b\n" - "mov v9.16b, v11.16b\n" - "ldr d31, [x24, x17]\n" - "ldr d30, [x23, x17]\n" - "mov v24.16b, v14.16b\n" - "mov v17.16b, v11.16b\n" - "ldr d29, [x22, x17]\n" - "ldr d28, [x21, x17]\n" - "mov v23.16b, v14.16b\n" - "mov v25.16b, v11.16b\n" - "ldr x20, [x15, #0x20]\n" - "ldr d27, [x20, x17]\n" - "ushll v31.8h, v31.8b, #0x0\n" - "ushll v30.8h, v30.8b, #0x0\n" - "ushll v29.8h, v29.8b, #0x0\n" - "ushll v28.8h, v28.8b, #0x0\n" + "mov x14, #0x0\n" + "ld1r { v12.8h }, [x20]\n" + "mov x13, #0x0\n" + "add x12, %x[params], %[offsetof_Params_inptrs]\n" + "ldr x11, [%x[params], %[offsetof_Params_weights]]\n" + "ldr x10, [%x[params], %[offsetof_Params_requant_muls]]\n" + "ldr x9, [%x[params], %[offsetof_Params_requant_shifts]]\n" + "ldp x28, x27, [x22, #0x0]\n" + "ldp x26, x25, [x22, #0x10]\n" + "cbz x15, 3f\n" + "ldr d19, [x11, #0x0]\n" + "ldr d7, [x11, #0x8]\n" + "subs x15, x15, #0x1\n" + "usubl v19.8h, v19.8b, v18.8b\n" + "ldr d1, [x11, #0x10]\n" + "ldr d17, [x11, #0x18]\n" + "usubl v7.8h, v7.8b, v18.8b\n" + "usubl v1.8h, v1.8b, v18.8b\n" + "ldr d8, [x11, #0x20]\n" + "ldr d31, [x11, #0x28]\n" + "usubl v17.8h, v17.8b, v18.8b\n" + "usubl v8.8h, v8.8b, v18.8b\n" + "ldr d29, [x11, #0x30]\n" + "ldr d16, [x11, #0x38]\n" + "usubl v31.8h, v31.8b, v18.8b\n" + "usubl v29.8h, v29.8b, v18.8b\n" + "ldr d4, [x11, #0x40]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "usubl v16.8h, v16.8b, v18.8b\n" + "usubl v4.8h, v4.8b, v18.8b\n" + "ldr q28, [x20, #0x0]\n" + "ldr q9, [x20, #0x10]\n" + "add x20, x20, #0x20\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" + "ldp x23, x22, [x12, #0x0]\n" + "ldp x21, x20, [x12, #0x10]\n" + "mov v3.16b, v28.16b\n" + "mov v30.16b, v9.16b\n" + "ldr d23, [x23, x14]\n" + "ldr d10, [x22, x14]\n" + "mov v0.16b, v28.16b\n" + "mov v22.16b, v9.16b\n" + "ldr d11, [x21, x14]\n" + "ldr d13, [x20, x14]\n" + "mov v6.16b, v28.16b\n" + "mov v2.16b, v9.16b\n" + "ldr x20, [x12, #0x20]\n" + "ldr d27, [x20, x14]\n" + "ushll v23.8h, v23.8b, #0x0\n" + "ushll v10.8h, v10.8b, #0x0\n" + "ushll v11.8h, v11.8b, #0x0\n" + "ushll v13.8h, v13.8b, #0x0\n" "ushll v27.8h, v27.8b, #0x0\n" "beq 2f\n" "1:" // Loop - "ldr q22, [x13, #0x0]\n" - "ldr q10, [x12, #0x0]\n" - "smlal v14.4s, v31.4h, v4.4h\n" - "smlal2 v11.4s, v31.8h, v4.8h\n" - "ldr q18, [x13, #0x10]\n" - "ldr q26, [x12, #0x10]\n" - "smlal v16.4s, v31.4h, v3.4h\n" - "smlal2 v9.4s, v31.8h, v3.8h\n" - "smlal v14.4s, v30.4h, v0.4h\n" - "smlal2 v11.4s, v30.8h, v0.8h\n" - "ldr x22, [x15, #0x28]\n" - "ldr x21, [x15, #0x38]\n" - "smlal v16.4s, v29.4h, v2.4h\n" - "smlal2 v9.4s, v29.8h, v2.8h\n" - "ldr x20, [x15, #0x30]\n" - "ldr d29, [x20, x17]\n" - "smlal v24.4s, v31.4h, v1.4h\n" - "smlal2 v17.4s, v31.8h, v1.8h\n" - "ldr x26, [x15, #0x40]\n" - "ldr x20, [x15, #0x48]\n" - "ldr d30, [x20, x17]\n" - "smlal v23.4s, v31.4h, v0.4h\n" - "smlal2 v25.4s, v31.8h, v0.8h\n" - "ldr d31, [x22, x17]\n" - "smlal v14.4s, v28.4h, v5.4h\n" - "smlal2 v11.4s, v28.8h, v5.8h\n" - "ushll v31.8h, v31.8b, #0x0\n" - "ldr x25, [x15, #0x50]\n" - "smlal v16.4s, v28.4h, v4.4h\n" - "smlal2 v9.4s, v28.8h, v4.8h\n" - "ushll v29.8h, v29.8b, #0x0\n" - "ldr x24, [x15, #0x58]\n" - "smlal v24.4s, v28.4h, v2.4h\n" - "smlal2 v17.4s, v28.8h, v2.8h\n" - "ushll v30.8h, v30.8b, #0x0\n" - "ldr x23, [x15, #0x60]\n" - "smlal v23.4s, v28.4h, v1.4h\n" - "smlal2 v25.4s, v28.8h, v1.8h\n" - "ldr d28, [x21, x17]\n" - "ushll v28.8h, v28.8b, #0x0\n" - "smlal v14.4s, v27.4h, v7.4h\n" - "smlal2 v11.4s, v27.8h, v7.8h\n" - "ldr x22, [x15, #0x68]\n" - "ldr x21, [x15, #0x70]\n" - "smlal v16.4s, v27.4h, v6.4h\n" - "smlal2 v9.4s, v27.8h, v6.8h\n" - "ldr x20, [x15, #0x78]\n" - "ldr x27, [%x[params], %[offsetof_Params_bias]]\n" - "smlal v24.4s, v31.4h, v6.4h\n" - "smlal2 v17.4s, v31.8h, v6.8h\n" - "ldr d31, [x26, x17]\n" - "ushll v31.8h, v31.8b, #0x0\n" - "smlal v23.4s, v27.4h, v3.4h\n" - "smlal2 v25.4s, v27.8h, v3.8h\n" - "add x14, x14, #0x48\n" - "subs x8, x8, #0x1\n" - "smlal v14.4s, v28.4h, v1.4h\n" - "smlal2 v11.4s, v28.8h, v1.8h\n" - "add x13, x13, #0x20\n" - "add x12, x12, #0x20\n" - "smlal v16.4s, v28.4h, v0.4h\n" - "smlal2 v9.4s, v28.8h, v0.8h\n" - "ldr d28, [x24, x17]\n" - "ushll v28.8h, v28.8b, #0x0\n" - "smlal v24.4s, v27.4h, v4.4h\n" - "smlal v23.4s, v29.4h, v8.4h\n" - "smlal2 v17.4s, v27.8h, v4.8h\n" - "smlal2 v25.4s, v29.8h, v8.8h\n" - "ldr d29, [x25, x17]\n" - "ushll v29.8h, v29.8b, #0x0\n" - "smlal v14.4s, v31.4h, v2.4h\n" - "smlal2 v11.4s, v31.8h, v2.8h\n" - "smlal v16.4s, v31.4h, v1.4h\n" - "smlal2 v9.4s, v31.8h, v1.8h\n" - "ldr d31, [x23, x17]\n" - "ushll v31.8h, v31.8b, #0x0\n" - "smlal v24.4s, v30.4h, v5.4h\n" - "smlal v23.4s, v30.4h, v4.4h\n" - "smlal v14.4s, v30.4h, v8.4h\n" - "smlal2 v11.4s, v30.8h, v8.8h\n" - "smlal v16.4s, v30.4h, v7.4h\n" - "smlal2 v9.4s, v30.8h, v7.8h\n" - "smlal2 v17.4s, v30.8h, v5.8h\n" - "smlal2 v25.4s, v30.8h, v4.8h\n" - "ldr d30, [x22, x17]\n" - "ushll v30.8h, v30.8b, #0x0\n" - "smlal v24.4s, v29.4h, v0.4h\n" - "smlal v23.4s, v28.4h, v2.4h\n" - "smlal v14.4s, v29.4h, v3.4h\n" - "smlal2 v11.4s, v29.8h, v3.8h\n" - "smlal2 v17.4s, v29.8h, v0.8h\n" - "ldr d29, [x21, x17]\n" - "smlal2 v25.4s, v28.8h, v2.8h\n" - "ushll v29.8h, v29.8b, #0x0\n" - "smlal v24.4s, v31.4h, v3.4h\n" - "smlal v23.4s, v30.4h, v5.4h\n" - "smlal v16.4s, v28.4h, v5.4h\n" - "smlal2 v9.4s, v28.8h, v5.8h\n" - "ldr d28, [x20, x17]\n" - "ushll v28.8h, v28.8b, #0x0\n" - "smlal v14.4s, v31.4h, v6.4h\n" - "smlal2 v17.4s, v31.8h, v3.8h\n" - "sqrdmulh v14.4s, v14.4s, v22.4s\n" - "add x17, x17, #0x8\n" - "smlal2 v25.4s, v30.8h, v5.8h\n" - "smlal v24.4s, v29.4h, v7.4h\n" - "and v21.16b, v14.16b, v10.16b\n" - "smlal v23.4s, v29.4h, v6.4h\n" - "smlal2 v11.4s, v31.8h, v6.8h\n" - "sqrdmulh v11.4s, v11.4s, v18.4s\n" - "smlal2 v17.4s, v29.8h, v7.8h\n" - "smlal2 v25.4s, v29.8h, v6.8h\n" - "sshr v21.4s, v21.4s, #0x1f\n" - "smlal v16.4s, v30.4h, v8.4h\n" - "smlal v24.4s, v28.4h, v8.4h\n" - "and v4.16b, v11.16b, v26.16b\n" - "smlal v23.4s, v28.4h, v7.4h\n" - "smlal2 v9.4s, v30.8h, v8.8h\n" - "sqrdmulh v16.4s, v16.4s, v22.4s\n" - "smlal2 v17.4s, v28.8h, v8.8h\n" - "smlal2 v25.4s, v28.8h, v7.8h\n" - "sqrdmulh v24.4s, v24.4s, v22.4s\n" - "sqrdmulh v23.4s, v23.4s, v22.4s\n" - "sqadd v14.4s, v14.4s, v21.4s\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "and v19.16b, v16.16b, v10.16b\n" - "sqrdmulh v9.4s, v9.4s, v18.4s\n" - "and v3.16b, v24.16b, v10.16b\n" - "sqrdmulh v17.4s, v17.4s, v18.4s\n" - "and v21.16b, v23.16b, v10.16b\n" - "sqrdmulh v25.4s, v25.4s, v18.4s\n" - "sqadd v11.4s, v11.4s, v4.4s\n" - "sshr v19.4s, v19.4s, #0x1f\n" - "and v27.16b, v9.16b, v26.16b\n" - "sshr v3.4s, v3.4s, #0x1f\n" - "and v5.16b, v17.16b, v26.16b\n" + "ldr q24, [x10, #0x0]\n" + "ldr q25, [x9, #0x0]\n" + "smlal v28.4s, v23.4h, v8.4h\n" + "smlal2 v9.4s, v23.8h, v8.8h\n" + "ldr q20, [x10, #0x10]\n" + "ldr q26, [x9, #0x10]\n" + "smlal v28.4s, v10.4h, v19.4h\n" + "smlal v3.4s, v23.4h, v17.4h\n" + "ldr x20, [x12, #0x28]\n" + "ldr d21, [x20, x14]\n" + "smlal v0.4s, v23.4h, v7.4h\n" + "smlal v6.4s, v23.4h, v19.4h\n" + "smlal2 v9.4s, v10.8h, v19.8h\n" + "ldr x20, [x12, #0x38]\n" + "ldr d10, [x20, x14]\n" + "smlal v28.4s, v13.4h, v31.4h\n" + "smlal2 v30.4s, v23.8h, v17.8h\n" + "smlal2 v22.4s, v23.8h, v7.8h\n" + "ldr x20, [x12, #0x30]\n" + "ldr d15, [x20, x14]\n" + "smlal2 v2.4s, v23.8h, v19.8h\n" + "smlal v3.4s, v11.4h, v1.4h\n" + "ushll v21.8h, v21.8b, #0x0\n" + "ldr x20, [x12, #0x40]\n" + "ldr d23, [x20, x14]\n" + "smlal v0.4s, v13.4h, v1.4h\n" + "smlal v6.4s, v13.4h, v7.4h\n" + "ushll v10.8h, v10.8b, #0x0\n" + "smlal2 v9.4s, v13.8h, v31.8h\n" + "smlal v28.4s, v27.4h, v16.4h\n" + "ldr x20, [x12, #0x48]\n" + "ushll v15.8h, v15.8b, #0x0\n" + "smlal2 v30.4s, v11.8h, v1.8h\n" + "ldr d11, [x20, x14]\n" + "smlal2 v22.4s, v13.8h, v1.8h\n" + "ushll v23.8h, v23.8b, #0x0\n" + "smlal2 v2.4s, v13.8h, v7.8h\n" + "smlal v3.4s, v13.4h, v8.4h\n" + "ldr x21, [x12, #0x50]\n" + "ldr x20, [x12, #0x58]\n" + "smlal v0.4s, v21.4h, v29.4h\n" + "smlal v6.4s, v27.4h, v17.4h\n" + "ushll v11.8h, v11.8b, #0x0\n" + "ldr x24, [x12, #0x60]\n" + "smlal2 v9.4s, v27.8h, v16.8h\n" + "smlal v28.4s, v10.4h, v7.4h\n" + "ldr x23, [x12, #0x68]\n" + "ldr x22, [x12, #0x70]\n" + "smlal2 v30.4s, v13.8h, v8.8h\n" + "ldr d13, [x21, x14]\n" + "smlal2 v22.4s, v21.8h, v29.8h\n" + "ldr d21, [x20, x14]\n" + "smlal2 v2.4s, v27.8h, v17.8h\n" + "smlal v3.4s, v27.4h, v29.4h\n" + "ushll v13.8h, v13.8b, #0x0\n" + "ldr x21, [x12, #0x78]\n" + "smlal v0.4s, v27.4h, v8.4h\n" + "smlal v6.4s, v15.4h, v4.4h\n" + "ushll v21.8h, v21.8b, #0x0\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "smlal2 v9.4s, v10.8h, v7.8h\n" + "smlal v28.4s, v23.4h, v1.4h\n" + "add x11, x11, #0x48\n" + "subs x15, x15, #0x1\n" + "smlal2 v30.4s, v27.8h, v29.8h\n" + "smlal2 v22.4s, v27.8h, v8.8h\n" + "ldr d27, [x24, x14]\n" + "ushll v27.8h, v27.8b, #0x0\n" + "smlal2 v2.4s, v15.8h, v4.8h\n" + "ldr d15, [x23, x14]\n" + "smlal v3.4s, v10.4h, v19.4h\n" + "ushll v15.8h, v15.8b, #0x0\n" + "smlal v0.4s, v11.4h, v31.4h\n" + "smlal v6.4s, v11.4h, v8.4h\n" + "add x10, x10, #0x20\n" + "add x9, x9, #0x20\n" + "smlal2 v9.4s, v23.8h, v1.8h\n" + "smlal v28.4s, v11.4h, v4.4h\n" + "smlal2 v30.4s, v10.8h, v19.8h\n" + "ldr d10, [x22, x14]\n" + "smlal2 v22.4s, v11.8h, v31.8h\n" + "ushll v10.8h, v10.8b, #0x0\n" + "smlal2 v2.4s, v11.8h, v8.8h\n" + "ldr d8, [x21, x14]\n" + "smlal v3.4s, v23.4h, v7.4h\n" + "ushll v8.8h, v8.8b, #0x0\n" + "smlal v0.4s, v13.4h, v19.4h\n" + "smlal v6.4s, v21.4h, v1.4h\n" + "add x14, x14, #0x8\n" + "smlal2 v9.4s, v11.8h, v4.8h\n" + "smlal v28.4s, v13.4h, v17.4h\n" + "smlal2 v30.4s, v23.8h, v7.8h\n" + "smlal2 v22.4s, v13.8h, v19.8h\n" + "smlal2 v2.4s, v21.8h, v1.8h\n" + "smlal v3.4s, v11.4h, v16.4h\n" + "smlal v0.4s, v27.4h, v17.4h\n" + "smlal v6.4s, v15.4h, v31.4h\n" + "smlal2 v9.4s, v13.8h, v17.8h\n" + "smlal v28.4s, v27.4h, v29.4h\n" + "sqrdmulh v28.4s, v28.4s, v24.4s\n" + "smlal2 v30.4s, v11.8h, v16.8h\n" + "smlal2 v22.4s, v27.8h, v17.8h\n" + "and v17.16b, v28.16b, v25.16b\n" + "smlal2 v2.4s, v15.8h, v31.8h\n" + "smlal v3.4s, v21.4h, v31.4h\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "smlal v0.4s, v10.4h, v16.4h\n" + "smlal v6.4s, v10.4h, v29.4h\n" + "sqadd v28.4s, v28.4s, v17.4s\n" + "smlal2 v9.4s, v27.8h, v29.8h\n" + "smlal2 v30.4s, v21.8h, v31.8h\n" + "sqrdmulh v9.4s, v9.4s, v20.4s\n" + "smlal2 v22.4s, v10.8h, v16.8h\n" + "smlal2 v2.4s, v10.8h, v29.8h\n" + "and v23.16b, v9.16b, v26.16b\n" + "smlal v3.4s, v15.4h, v4.4h\n" + "smlal v0.4s, v8.4h, v4.4h\n" + "sqrdmulh v3.4s, v3.4s, v24.4s\n" + "smlal v6.4s, v8.4h, v16.4h\n" + "smlal2 v30.4s, v15.8h, v4.8h\n" + "sqrdmulh v0.4s, v0.4s, v24.4s\n" + "smlal2 v22.4s, v8.8h, v4.8h\n" + "smlal2 v2.4s, v8.8h, v16.8h\n" + "sqrdmulh v6.4s, v6.4s, v24.4s\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "and v8.16b, v3.16b, v25.16b\n" + "sqrdmulh v30.4s, v30.4s, v20.4s\n" + "and v11.16b, v0.16b, v25.16b\n" + "sqrdmulh v22.4s, v22.4s, v20.4s\n" + "and v29.16b, v6.16b, v25.16b\n" + "sqrdmulh v2.4s, v2.4s, v20.4s\n" + "sqadd v9.4s, v9.4s, v23.4s\n" + "sshr v8.4s, v8.4s, #0x1f\n" + "and v13.16b, v30.16b, v26.16b\n" + "sshr v11.4s, v11.4s, #0x1f\n" + "and v21.16b, v22.16b, v26.16b\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "and v23.16b, v2.16b, v26.16b\n" + "sqadd v3.4s, v3.4s, v8.4s\n" + "sshr v13.4s, v13.4s, #0x1f\n" + "sqadd v0.4s, v0.4s, v11.4s\n" "sshr v21.4s, v21.4s, #0x1f\n" - "and v4.16b, v25.16b, v26.16b\n" - "sqadd v16.4s, v16.4s, v19.4s\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "sqadd v24.4s, v24.4s, v3.4s\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v23.4s, v23.4s, v21.4s\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "srshl v14.4s, v14.4s, v10.4s\n" - "srshl v16.4s, v16.4s, v10.4s\n" - "sqadd v9.4s, v9.4s, v27.4s\n" - "srshl v24.4s, v24.4s, v10.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "srshl v23.4s, v23.4s, v10.4s\n" - "sqadd v25.4s, v25.4s, v4.4s\n" - "srshl v11.4s, v11.4s, v26.4s\n" - "sqxtn v14.4h, v14.4s\n" + "sqadd v6.4s, v6.4s, v29.4s\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "srshl v28.4s, v28.4s, v25.4s\n" + "srshl v3.4s, v3.4s, v25.4s\n" + "sqadd v30.4s, v30.4s, v13.4s\n" + "srshl v0.4s, v0.4s, v25.4s\n" + "sqadd v22.4s, v22.4s, v21.4s\n" + "srshl v6.4s, v6.4s, v25.4s\n" + "sqadd v2.4s, v2.4s, v23.4s\n" "srshl v9.4s, v9.4s, v26.4s\n" - "sqxtn v16.4h, v16.4s\n" - "srshl v17.4s, v17.4s, v26.4s\n" - "sqxtn v24.4h, v24.4s\n" - "srshl v25.4s, v25.4s, v26.4s\n" - "sqxtn v23.4h, v23.4s\n" - "sqxtn2 v14.8h, v11.4s\n" - "sqxtn2 v16.8h, v9.4s\n" - "sqxtn2 v24.8h, v17.4s\n" - "sqxtn2 v23.8h, v25.4s\n" - "sqadd v14.8h, v14.8h, v20.8h\n" - "sqadd v16.8h, v16.8h, v20.8h\n" - "sqadd v24.8h, v24.8h, v20.8h\n" - "sqadd v23.8h, v23.8h, v20.8h\n" - "smax v14.8h, v14.8h, v15.8h\n" - "smax v16.8h, v16.8h, v15.8h\n" - "smax v24.8h, v24.8h, v15.8h\n" - "smax v23.8h, v23.8h, v15.8h\n" - "smin v14.8h, v14.8h, v13.8h\n" - "smin v16.8h, v16.8h, v13.8h\n" - "smin v24.8h, v24.8h, v13.8h\n" - "smin v23.8h, v23.8h, v13.8h\n" - "uzp1 v14.16b, v14.16b, v14.16b\n" - "str d14, [x11, x16]\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "uzp1 v24.16b, v24.16b, v24.16b\n" - "str d16, [x10, x16]\n" - "uzp1 v23.16b, v23.16b, v23.16b\n" - "str d24, [x9, x16]\n" - "str d23, [x28, x16]\n" - "ldr q14, [x27, #0x0]\n" - "ldr q11, [x27, #0x10]\n" - "add x27, x27, #0x20\n" - "ldr d0, [x14, #0x0]\n" - "ldr d1, [x14, #0x8]\n" - "add x16, x16, #0x8\n" - "str x27, [%x[params], %[offsetof_Params_bias]]\n" - "ldr d2, [x14, #0x10]\n" - "ldr d3, [x14, #0x18]\n" - "mov v16.16b, v14.16b\n" - "mov v9.16b, v11.16b\n" - "ldr d4, [x14, #0x20]\n" - "ldr d5, [x14, #0x28]\n" - "mov v24.16b, v14.16b\n" - "mov v17.16b, v11.16b\n" - "ldr d6, [x14, #0x30]\n" - "ldr d7, [x14, #0x38]\n" - "mov v23.16b, v14.16b\n" - "mov v25.16b, v11.16b\n" - "ldr d8, [x14, #0x40]\n" - "ldp x24, x23, [x15, #0x0]\n" - "usubl v0.8h, v0.8b, v12.8b\n" - "usubl v1.8h, v1.8b, v12.8b\n" - "ldp x22, x21, [x15, #0x10]\n" - "ldr d31, [x24, x17]\n" - "usubl v2.8h, v2.8b, v12.8b\n" - "usubl v3.8h, v3.8b, v12.8b\n" - "ldr d30, [x23, x17]\n" - "ldr d29, [x22, x17]\n" - "usubl v4.8h, v4.8b, v12.8b\n" - "usubl v5.8h, v5.8b, v12.8b\n" - "ldr d28, [x21, x17]\n" - "ldr x20, [x15, #0x20]\n" - "usubl v6.8h, v6.8b, v12.8b\n" - "usubl v7.8h, v7.8b, v12.8b\n" - "ldr d27, [x20, x17]\n" - "usubl v8.8h, v8.8b, v12.8b\n" - "ushll v31.8h, v31.8b, #0x0\n" - "ushll v30.8h, v30.8b, #0x0\n" - "ushll v29.8h, v29.8b, #0x0\n" - "ushll v28.8h, v28.8b, #0x0\n" + "sqxtn v28.4h, v28.4s\n" + "srshl v30.4s, v30.4s, v26.4s\n" + "sqxtn v3.4h, v3.4s\n" + "srshl v22.4s, v22.4s, v26.4s\n" + "sqxtn v0.4h, v0.4s\n" + "srshl v2.4s, v2.4s, v26.4s\n" + "sqxtn v6.4h, v6.4s\n" + "sqxtn2 v28.8h, v9.4s\n" + "sqxtn2 v3.8h, v30.4s\n" + "sqxtn2 v0.8h, v22.4s\n" + "sqxtn2 v6.8h, v2.4s\n" + "sqadd v28.8h, v28.8h, v5.8h\n" + "sqadd v3.8h, v3.8h, v5.8h\n" + "sqadd v0.8h, v0.8h, v5.8h\n" + "sqadd v6.8h, v6.8h, v5.8h\n" + "smax v28.8h, v28.8h, v14.8h\n" + "smax v3.8h, v3.8h, v14.8h\n" + "smax v0.8h, v0.8h, v14.8h\n" + "smax v6.8h, v6.8h, v14.8h\n" + "smin v28.8h, v28.8h, v12.8h\n" + "smin v3.8h, v3.8h, v12.8h\n" + "smin v0.8h, v0.8h, v12.8h\n" + "smin v6.8h, v6.8h, v12.8h\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + "str d28, [x28, x13]\n" + "uzp1 v3.16b, v3.16b, v3.16b\n" + "uzp1 v0.16b, v0.16b, v0.16b\n" + "str d3, [x27, x13]\n" + "uzp1 v6.16b, v6.16b, v6.16b\n" + "str d0, [x26, x13]\n" + "str d6, [x25, x13]\n" + "ldr q28, [x20, #0x0]\n" + "ldr q9, [x20, #0x10]\n" + "add x20, x20, #0x20\n" + "ldr d19, [x11, #0x0]\n" + "ldr d7, [x11, #0x8]\n" + "add x13, x13, #0x8\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" + "ldr d1, [x11, #0x10]\n" + "ldr d17, [x11, #0x18]\n" + "mov v3.16b, v28.16b\n" + "mov v30.16b, v9.16b\n" + "ldr d8, [x11, #0x20]\n" + "ldr d31, [x11, #0x28]\n" + "mov v0.16b, v28.16b\n" + "mov v22.16b, v9.16b\n" + "ldr d29, [x11, #0x30]\n" + "ldr d16, [x11, #0x38]\n" + "mov v6.16b, v28.16b\n" + "mov v2.16b, v9.16b\n" + "ldr d4, [x11, #0x40]\n" + "ldp x23, x22, [x12, #0x0]\n" + "usubl v19.8h, v19.8b, v18.8b\n" + "usubl v7.8h, v7.8b, v18.8b\n" + "ldp x21, x20, [x12, #0x10]\n" + "ldr d23, [x23, x14]\n" + "usubl v1.8h, v1.8b, v18.8b\n" + "usubl v17.8h, v17.8b, v18.8b\n" + "ldr d10, [x22, x14]\n" + "ldr d11, [x21, x14]\n" + "usubl v8.8h, v8.8b, v18.8b\n" + "usubl v31.8h, v31.8b, v18.8b\n" + "ldr d13, [x20, x14]\n" + "ldr x20, [x12, #0x20]\n" + "usubl v29.8h, v29.8b, v18.8b\n" + "usubl v16.8h, v16.8b, v18.8b\n" + "ldr d27, [x20, x14]\n" + "usubl v4.8h, v4.8b, v18.8b\n" + "ushll v23.8h, v23.8b, #0x0\n" + "ushll v10.8h, v10.8b, #0x0\n" + "ushll v11.8h, v11.8b, #0x0\n" + "ushll v13.8h, v13.8b, #0x0\n" "ushll v27.8h, v27.8b, #0x0\n" "bgt 1b\n" "2:" // Tail - "ldr q22, [x13, #0x0]\n" - "ldr q10, [x12, #0x0]\n" - "smlal v14.4s, v31.4h, v4.4h\n" - "smlal2 v11.4s, v31.8h, v4.8h\n" - "ldr q18, [x13, #0x10]\n" - "ldr q26, [x12, #0x10]\n" - "smlal v16.4s, v31.4h, v3.4h\n" - "smlal2 v9.4s, v31.8h, v3.8h\n" - "smlal v14.4s, v30.4h, v0.4h\n" - "smlal2 v11.4s, v30.8h, v0.8h\n" - "ldr x22, [x15, #0x28]\n" - "ldr x21, [x15, #0x38]\n" - "smlal v16.4s, v29.4h, v2.4h\n" - "smlal2 v9.4s, v29.8h, v2.8h\n" - "ldr x20, [x15, #0x30]\n" - "ldr d29, [x20, x17]\n" - "smlal v24.4s, v31.4h, v1.4h\n" - "smlal2 v17.4s, v31.8h, v1.8h\n" - "ldr x26, [x15, #0x40]\n" - "ldr x20, [x15, #0x48]\n" - "ldr d30, [x20, x17]\n" - "smlal v23.4s, v31.4h, v0.4h\n" - "smlal2 v25.4s, v31.8h, v0.8h\n" - "ldr d31, [x22, x17]\n" - "smlal v14.4s, v28.4h, v5.4h\n" - "smlal2 v11.4s, v28.8h, v5.8h\n" - "ushll v31.8h, v31.8b, #0x0\n" - "ldr x25, [x15, #0x50]\n" - "smlal v16.4s, v28.4h, v4.4h\n" - "smlal2 v9.4s, v28.8h, v4.8h\n" - "ushll v29.8h, v29.8b, #0x0\n" - "ldr x24, [x15, #0x58]\n" - "smlal v24.4s, v28.4h, v2.4h\n" - "smlal2 v17.4s, v28.8h, v2.8h\n" - "ushll v30.8h, v30.8b, #0x0\n" - "ldr x23, [x15, #0x60]\n" - "smlal v23.4s, v28.4h, v1.4h\n" - "smlal2 v25.4s, v28.8h, v1.8h\n" - "ldr d28, [x21, x17]\n" - "ushll v28.8h, v28.8b, #0x0\n" - "smlal v14.4s, v27.4h, v7.4h\n" - "smlal2 v11.4s, v27.8h, v7.8h\n" - "ldr x22, [x15, #0x68]\n" - "ldr x21, [x15, #0x70]\n" - "smlal v16.4s, v27.4h, v6.4h\n" - "smlal2 v9.4s, v27.8h, v6.8h\n" - "ldr x20, [x15, #0x78]\n" - "tst x7, #0x7\n" - "smlal v24.4s, v31.4h, v6.4h\n" - "smlal2 v17.4s, v31.8h, v6.8h\n" - "ldr d31, [x26, x17]\n" - "ushll v31.8h, v31.8b, #0x0\n" - "smlal v23.4s, v27.4h, v3.4h\n" - "smlal2 v25.4s, v27.8h, v3.8h\n" - "add x13, x13, #0x20\n" - "add x12, x12, #0x20\n" - "smlal v14.4s, v28.4h, v1.4h\n" - "smlal2 v11.4s, v28.8h, v1.8h\n" - "smlal v16.4s, v28.4h, v0.4h\n" - "smlal2 v9.4s, v28.8h, v0.8h\n" - "ldr d28, [x24, x17]\n" - "ushll v28.8h, v28.8b, #0x0\n" - "smlal v24.4s, v27.4h, v4.4h\n" - "smlal v23.4s, v29.4h, v8.4h\n" - "smlal2 v17.4s, v27.8h, v4.8h\n" - "smlal2 v25.4s, v29.8h, v8.8h\n" - "ldr d29, [x25, x17]\n" - "ushll v29.8h, v29.8b, #0x0\n" - "smlal v14.4s, v31.4h, v2.4h\n" - "smlal2 v11.4s, v31.8h, v2.8h\n" - "smlal v16.4s, v31.4h, v1.4h\n" - "smlal2 v9.4s, v31.8h, v1.8h\n" - "ldr d31, [x23, x17]\n" - "ushll v31.8h, v31.8b, #0x0\n" - "smlal v24.4s, v30.4h, v5.4h\n" - "smlal v23.4s, v30.4h, v4.4h\n" - "smlal v14.4s, v30.4h, v8.4h\n" - "smlal2 v11.4s, v30.8h, v8.8h\n" - "smlal v16.4s, v30.4h, v7.4h\n" - "smlal2 v9.4s, v30.8h, v7.8h\n" - "smlal2 v17.4s, v30.8h, v5.8h\n" - "smlal2 v25.4s, v30.8h, v4.8h\n" - "ldr d30, [x22, x17]\n" - "ushll v30.8h, v30.8b, #0x0\n" - "smlal v24.4s, v29.4h, v0.4h\n" - "smlal v23.4s, v28.4h, v2.4h\n" - "smlal v14.4s, v29.4h, v3.4h\n" - "smlal2 v11.4s, v29.8h, v3.8h\n" - "smlal2 v17.4s, v29.8h, v0.8h\n" - "ldr d29, [x21, x17]\n" - "smlal2 v25.4s, v28.8h, v2.8h\n" - "ushll v29.8h, v29.8b, #0x0\n" - "smlal v24.4s, v31.4h, v3.4h\n" - "smlal v23.4s, v30.4h, v5.4h\n" - "smlal v16.4s, v28.4h, v5.4h\n" - "smlal2 v9.4s, v28.8h, v5.8h\n" - "ldr d28, [x20, x17]\n" - "ushll v28.8h, v28.8b, #0x0\n" - "smlal v14.4s, v31.4h, v6.4h\n" - "smlal2 v17.4s, v31.8h, v3.8h\n" - "sqrdmulh v14.4s, v14.4s, v22.4s\n" - "add x17, x17, #0x8\n" - "smlal2 v25.4s, v30.8h, v5.8h\n" - "smlal v24.4s, v29.4h, v7.4h\n" - "and v21.16b, v14.16b, v10.16b\n" - "smlal v23.4s, v29.4h, v6.4h\n" - "smlal2 v11.4s, v31.8h, v6.8h\n" - "sqrdmulh v11.4s, v11.4s, v18.4s\n" - "smlal2 v17.4s, v29.8h, v7.8h\n" - "smlal2 v25.4s, v29.8h, v6.8h\n" - "sshr v21.4s, v21.4s, #0x1f\n" - "smlal v16.4s, v30.4h, v8.4h\n" - "smlal v24.4s, v28.4h, v8.4h\n" - "and v4.16b, v11.16b, v26.16b\n" - "smlal v23.4s, v28.4h, v7.4h\n" - "smlal2 v9.4s, v30.8h, v8.8h\n" - "sqrdmulh v16.4s, v16.4s, v22.4s\n" - "smlal2 v17.4s, v28.8h, v8.8h\n" - "smlal2 v25.4s, v28.8h, v7.8h\n" - "sqrdmulh v24.4s, v24.4s, v22.4s\n" - "sqrdmulh v23.4s, v23.4s, v22.4s\n" - "sqadd v14.4s, v14.4s, v21.4s\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "and v19.16b, v16.16b, v10.16b\n" - "sqrdmulh v9.4s, v9.4s, v18.4s\n" - "and v3.16b, v24.16b, v10.16b\n" - "sqrdmulh v17.4s, v17.4s, v18.4s\n" - "and v21.16b, v23.16b, v10.16b\n" - "sqrdmulh v25.4s, v25.4s, v18.4s\n" - "sqadd v11.4s, v11.4s, v4.4s\n" - "sshr v19.4s, v19.4s, #0x1f\n" - "and v27.16b, v9.16b, v26.16b\n" - "sshr v3.4s, v3.4s, #0x1f\n" - "and v5.16b, v17.16b, v26.16b\n" - "sshr v21.4s, v21.4s, #0x1f\n" - "and v4.16b, v25.16b, v26.16b\n" - "sqadd v16.4s, v16.4s, v19.4s\n" + "ldr q26, [x10, #0x0]\n" + "ldr q25, [x9, #0x0]\n" + "smlal v28.4s, v23.4h, v8.4h\n" + "smlal2 v9.4s, v23.8h, v8.8h\n" + "ldr q24, [x10, #0x10]\n" + "ldr q20, [x9, #0x10]\n" + "smlal v28.4s, v10.4h, v19.4h\n" + "smlal v3.4s, v23.4h, v17.4h\n" + "ldr x20, [x12, #0x28]\n" + "ldr d21, [x20, x14]\n" + "smlal v0.4s, v23.4h, v7.4h\n" + "smlal v6.4s, v23.4h, v19.4h\n" + "smlal2 v9.4s, v10.8h, v19.8h\n" + "ldr x20, [x12, #0x38]\n" + "ldr d15, [x20, x14]\n" + "smlal v28.4s, v13.4h, v31.4h\n" + "smlal2 v30.4s, v23.8h, v17.8h\n" + "smlal2 v22.4s, v23.8h, v7.8h\n" + "ldr x20, [x12, #0x30]\n" + "ldr d10, [x20, x14]\n" + "smlal2 v2.4s, v23.8h, v19.8h\n" + "smlal v3.4s, v11.4h, v1.4h\n" + "ushll v21.8h, v21.8b, #0x0\n" + "ldr x20, [x12, #0x40]\n" + "ldr d23, [x20, x14]\n" + "smlal v0.4s, v13.4h, v1.4h\n" + "smlal v6.4s, v13.4h, v7.4h\n" + "ushll v15.8h, v15.8b, #0x0\n" + "smlal2 v9.4s, v13.8h, v31.8h\n" + "smlal v28.4s, v27.4h, v16.4h\n" + "ldr x20, [x12, #0x48]\n" + "ushll v10.8h, v10.8b, #0x0\n" + "smlal2 v30.4s, v11.8h, v1.8h\n" + "ldr d11, [x20, x14]\n" + "smlal2 v22.4s, v13.8h, v1.8h\n" + "ushll v23.8h, v23.8b, #0x0\n" + "smlal2 v2.4s, v13.8h, v7.8h\n" + "smlal v3.4s, v13.4h, v8.4h\n" + "ldr x24, [x12, #0x50]\n" + "ldr x20, [x12, #0x58]\n" + "smlal v0.4s, v21.4h, v29.4h\n" + "smlal v6.4s, v27.4h, v17.4h\n" + "ushll v11.8h, v11.8b, #0x0\n" + "ldr x23, [x12, #0x60]\n" + "smlal2 v9.4s, v27.8h, v16.8h\n" + "smlal v28.4s, v15.4h, v7.4h\n" + "ldr x22, [x12, #0x68]\n" + "ldr x21, [x12, #0x70]\n" + "smlal2 v30.4s, v13.8h, v8.8h\n" + "ldr d13, [x24, x14]\n" + "smlal2 v22.4s, v21.8h, v29.8h\n" + "ldr d21, [x20, x14]\n" + "smlal2 v2.4s, v27.8h, v17.8h\n" + "smlal v3.4s, v27.4h, v29.4h\n" + "ushll v13.8h, v13.8b, #0x0\n" + "ldr x20, [x12, #0x78]\n" + "smlal v0.4s, v27.4h, v8.4h\n" + "smlal v6.4s, v10.4h, v4.4h\n" + "ushll v21.8h, v21.8b, #0x0\n" + "tst x16, #0x7\n" + "smlal2 v9.4s, v15.8h, v7.8h\n" + "smlal v28.4s, v23.4h, v1.4h\n" + "add x10, x10, #0x20\n" + "add x9, x9, #0x20\n" + "smlal2 v30.4s, v27.8h, v29.8h\n" + "smlal2 v22.4s, v27.8h, v8.8h\n" + "ldr d27, [x23, x14]\n" + "ushll v27.8h, v27.8b, #0x0\n" + "smlal2 v2.4s, v10.8h, v4.8h\n" + "ldr d10, [x22, x14]\n" + "smlal v3.4s, v15.4h, v19.4h\n" + "ushll v10.8h, v10.8b, #0x0\n" + "smlal v0.4s, v11.4h, v31.4h\n" + "smlal v6.4s, v11.4h, v8.4h\n" + "smlal2 v9.4s, v23.8h, v1.8h\n" + "smlal v28.4s, v11.4h, v4.4h\n" + "smlal2 v30.4s, v15.8h, v19.8h\n" + "ldr d15, [x21, x14]\n" + "smlal2 v22.4s, v11.8h, v31.8h\n" + "ushll v15.8h, v15.8b, #0x0\n" + "smlal2 v2.4s, v11.8h, v8.8h\n" + "ldr d8, [x20, x14]\n" + "smlal v3.4s, v23.4h, v7.4h\n" + "ushll v8.8h, v8.8b, #0x0\n" + "smlal v0.4s, v13.4h, v19.4h\n" + "smlal v6.4s, v21.4h, v1.4h\n" + "add x14, x14, #0x8\n" + "smlal2 v9.4s, v11.8h, v4.8h\n" + "smlal v28.4s, v13.4h, v17.4h\n" + "smlal2 v30.4s, v23.8h, v7.8h\n" + "smlal2 v22.4s, v13.8h, v19.8h\n" + "smlal2 v2.4s, v21.8h, v1.8h\n" + "smlal v3.4s, v11.4h, v16.4h\n" + "smlal v0.4s, v27.4h, v17.4h\n" + "smlal v6.4s, v10.4h, v31.4h\n" + "smlal2 v9.4s, v13.8h, v17.8h\n" + "smlal v28.4s, v27.4h, v29.4h\n" + "sqrdmulh v28.4s, v28.4s, v26.4s\n" + "smlal2 v30.4s, v11.8h, v16.8h\n" + "smlal2 v22.4s, v27.8h, v17.8h\n" + "and v1.16b, v28.16b, v25.16b\n" + "smlal2 v2.4s, v10.8h, v31.8h\n" + "smlal v3.4s, v21.4h, v31.4h\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "smlal v0.4s, v15.4h, v16.4h\n" + "smlal v6.4s, v15.4h, v29.4h\n" + "sqadd v28.4s, v28.4s, v1.4s\n" + "smlal2 v9.4s, v27.8h, v29.8h\n" + "smlal2 v30.4s, v21.8h, v31.8h\n" + "sqrdmulh v9.4s, v9.4s, v24.4s\n" + "smlal2 v22.4s, v15.8h, v16.8h\n" + "smlal2 v2.4s, v15.8h, v29.8h\n" + "and v27.16b, v9.16b, v20.16b\n" + "smlal v3.4s, v10.4h, v4.4h\n" + "smlal v0.4s, v8.4h, v4.4h\n" + "sqrdmulh v3.4s, v3.4s, v26.4s\n" + "smlal v6.4s, v8.4h, v16.4h\n" + "smlal2 v30.4s, v10.8h, v4.8h\n" + "sqrdmulh v0.4s, v0.4s, v26.4s\n" + "smlal2 v22.4s, v8.8h, v4.8h\n" + "smlal2 v2.4s, v8.8h, v16.8h\n" + "sqrdmulh v6.4s, v6.4s, v26.4s\n" "sshr v27.4s, v27.4s, #0x1f\n" - "sqadd v24.4s, v24.4s, v3.4s\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v23.4s, v23.4s, v21.4s\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "srshl v14.4s, v14.4s, v10.4s\n" - "srshl v16.4s, v16.4s, v10.4s\n" + "and v16.16b, v3.16b, v25.16b\n" + "sqrdmulh v30.4s, v30.4s, v24.4s\n" + "and v4.16b, v0.16b, v25.16b\n" + "sqrdmulh v22.4s, v22.4s, v24.4s\n" + "and v17.16b, v6.16b, v25.16b\n" + "sqrdmulh v2.4s, v2.4s, v24.4s\n" "sqadd v9.4s, v9.4s, v27.4s\n" - "srshl v24.4s, v24.4s, v10.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "srshl v23.4s, v23.4s, v10.4s\n" - "sqadd v25.4s, v25.4s, v4.4s\n" - "srshl v11.4s, v11.4s, v26.4s\n" - "sqxtn v14.4h, v14.4s\n" - "srshl v9.4s, v9.4s, v26.4s\n" - "sqxtn v16.4h, v16.4s\n" - "srshl v17.4s, v17.4s, v26.4s\n" - "sqxtn v24.4h, v24.4s\n" - "srshl v25.4s, v25.4s, v26.4s\n" - "sqxtn v23.4h, v23.4s\n" - "sqxtn2 v14.8h, v11.4s\n" - "sqxtn2 v16.8h, v9.4s\n" - "sqxtn2 v24.8h, v17.4s\n" - "sqxtn2 v23.8h, v25.4s\n" - "sqadd v14.8h, v14.8h, v20.8h\n" - "sqadd v16.8h, v16.8h, v20.8h\n" - "sqadd v24.8h, v24.8h, v20.8h\n" - "sqadd v23.8h, v23.8h, v20.8h\n" - "smax v14.8h, v14.8h, v15.8h\n" - "smax v16.8h, v16.8h, v15.8h\n" - "smax v24.8h, v24.8h, v15.8h\n" - "smax v23.8h, v23.8h, v15.8h\n" - "smin v14.8h, v14.8h, v13.8h\n" - "smin v16.8h, v16.8h, v13.8h\n" - "smin v24.8h, v24.8h, v13.8h\n" - "smin v23.8h, v23.8h, v13.8h\n" - "uzp1 v14.16b, v14.16b, v14.16b\n" - "str d14, [x11, x16]\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "uzp1 v24.16b, v24.16b, v24.16b\n" - "str d16, [x10, x16]\n" - "uzp1 v23.16b, v23.16b, v23.16b\n" - "str d24, [x9, x16]\n" - "str d23, [x28, x16]\n" - "add x16, x16, #0x8\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "and v8.16b, v30.16b, v20.16b\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "and v26.16b, v22.16b, v20.16b\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "and v11.16b, v2.16b, v20.16b\n" + "sqadd v3.4s, v3.4s, v16.4s\n" + "sshr v8.4s, v8.4s, #0x1f\n" + "sqadd v0.4s, v0.4s, v4.4s\n" + "sshr v26.4s, v26.4s, #0x1f\n" + "sqadd v6.4s, v6.4s, v17.4s\n" + "sshr v11.4s, v11.4s, #0x1f\n" + "srshl v28.4s, v28.4s, v25.4s\n" + "srshl v3.4s, v3.4s, v25.4s\n" + "sqadd v30.4s, v30.4s, v8.4s\n" + "srshl v0.4s, v0.4s, v25.4s\n" + "sqadd v22.4s, v22.4s, v26.4s\n" + "srshl v6.4s, v6.4s, v25.4s\n" + "sqadd v2.4s, v2.4s, v11.4s\n" + "srshl v9.4s, v9.4s, v20.4s\n" + "sqxtn v28.4h, v28.4s\n" + "srshl v30.4s, v30.4s, v20.4s\n" + "sqxtn v3.4h, v3.4s\n" + "srshl v22.4s, v22.4s, v20.4s\n" + "sqxtn v0.4h, v0.4s\n" + "srshl v2.4s, v2.4s, v20.4s\n" + "sqxtn v6.4h, v6.4s\n" + "sqxtn2 v28.8h, v9.4s\n" + "sqxtn2 v3.8h, v30.4s\n" + "sqxtn2 v0.8h, v22.4s\n" + "sqxtn2 v6.8h, v2.4s\n" + "sqadd v28.8h, v28.8h, v5.8h\n" + "sqadd v3.8h, v3.8h, v5.8h\n" + "sqadd v0.8h, v0.8h, v5.8h\n" + "sqadd v6.8h, v6.8h, v5.8h\n" + "smax v28.8h, v28.8h, v14.8h\n" + "smax v3.8h, v3.8h, v14.8h\n" + "smax v0.8h, v0.8h, v14.8h\n" + "smax v6.8h, v6.8h, v14.8h\n" + "smin v28.8h, v28.8h, v12.8h\n" + "smin v3.8h, v3.8h, v12.8h\n" + "smin v0.8h, v0.8h, v12.8h\n" + "smin v6.8h, v6.8h, v12.8h\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + "str d28, [x28, x13]\n" + "uzp1 v3.16b, v3.16b, v3.16b\n" + "uzp1 v0.16b, v0.16b, v0.16b\n" + "str d3, [x27, x13]\n" + "uzp1 v6.16b, v6.16b, v6.16b\n" + "str d0, [x26, x13]\n" + "str d6, [x25, x13]\n" + "add x13, x13, #0x8\n" "beq 64f\n" - "add x14, x14, #0x48\n" + "add x11, x11, #0x48\n" "3:" // Oddments - "ldr x27, [%x[params], %[offsetof_Params_bias]]\n" - "tbz x7, #2, 5f\n" - "ld1 { v14.4s }, [x27], #0x10\n" - "tbz x7, #1, 4f\n" - "ld1 { v11.d }[0], [x27], #0x8\n" - "tbz x7, #0, 7f\n" - "ld1 { v11.s }[2], [x27]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "tbz x16, #2, 5f\n" + "ld1 { v28.4s }, [x20], #0x10\n" + "tbz x16, #1, 4f\n" + "ld1 { v9.d }[0], [x20], #0x8\n" + "tbz x16, #0, 7f\n" + "ld1 { v9.s }[2], [x20]\n" "b 7f\n" "4:" // Oddments: Load bias: Bit 2: Bit 1: Unset - "tbz x7, #0, 7f\n" - "ld1 { v11.s }[0], [x27]\n" + "tbz x16, #0, 7f\n" + "ld1 { v9.s }[0], [x20]\n" "b 7f\n" "5:" // Oddments: Load bias: Bit 2: Unset - "tbz x7, #1, 6f\n" - "ld1 { v14.d }[0], [x27], #0x8\n" - "tbz x7, #0, 7f\n" - "ld1 { v14.s }[2], [x27]\n" + "tbz x16, #1, 6f\n" + "ld1 { v28.d }[0], [x20], #0x8\n" + "tbz x16, #0, 7f\n" + "ld1 { v28.s }[2], [x20]\n" "b 7f\n" "6:" // Oddments: Load bias: Bit 2: Unset: Bit 1: Unset - "tbz x7, #0, 7f\n" - "ld1 { v14.s }[0], [x27]\n" + "tbz x16, #0, 7f\n" + "ld1 { v28.s }[0], [x20]\n" "7:" // Oddments: Load bias: Bit 2: End - "ldr d0, [x14, #0x0]\n" - "ldr d1, [x14, #0x8]\n" - "mov v16.16b, v14.16b\n" - "mov v9.16b, v11.16b\n" - "ldr d2, [x14, #0x10]\n" - "ldr d3, [x14, #0x18]\n" - "mov v24.16b, v14.16b\n" - "mov v17.16b, v11.16b\n" - "ldr d4, [x14, #0x20]\n" - "ldr d5, [x14, #0x28]\n" - "mov v23.16b, v14.16b\n" - "mov v25.16b, v11.16b\n" - "ldr d6, [x14, #0x30]\n" - "ldr d7, [x14, #0x38]\n" - "usubl v0.8h, v0.8b, v12.8b\n" - "usubl v1.8h, v1.8b, v12.8b\n" - "ldr d8, [x14, #0x40]\n" - "ldp x24, x23, [x15, #0x0]\n" - "usubl v2.8h, v2.8b, v12.8b\n" - "usubl v3.8h, v3.8b, v12.8b\n" - "ldp x22, x21, [x15, #0x10]\n" - "ldr x20, [x15, #0x20]\n" - "usubl v4.8h, v4.8b, v12.8b\n" - "usubl v5.8h, v5.8b, v12.8b\n" - "usubl v6.8h, v6.8b, v12.8b\n" - "usubl v7.8h, v7.8b, v12.8b\n" - "usubl v8.8h, v8.8b, v12.8b\n" - "add x24, x24, x17\n" - "add x23, x23, x17\n" - "add x22, x22, x17\n" - "add x21, x21, x17\n" - "add x20, x20, x17\n" - "tbz x7, #2, 9f\n" - "ld1 { v31.s }[0], [x24], #0x4\n" - "ld1 { v30.s }[0], [x23], #0x4\n" - "ld1 { v29.s }[0], [x22], #0x4\n" - "ld1 { v28.s }[0], [x21], #0x4\n" + "ldr d19, [x11, #0x0]\n" + "ldr d7, [x11, #0x8]\n" + "mov v3.16b, v28.16b\n" + "mov v30.16b, v9.16b\n" + "ldr d1, [x11, #0x10]\n" + "ldr d17, [x11, #0x18]\n" + "mov v0.16b, v28.16b\n" + "mov v22.16b, v9.16b\n" + "ldr d8, [x11, #0x20]\n" + "ldr d31, [x11, #0x28]\n" + "mov v6.16b, v28.16b\n" + "mov v2.16b, v9.16b\n" + "ldr d29, [x11, #0x30]\n" + "ldr d16, [x11, #0x38]\n" + "usubl v19.8h, v19.8b, v18.8b\n" + "usubl v7.8h, v7.8b, v18.8b\n" + "ldr d4, [x11, #0x40]\n" + "ldp x24, x23, [x12, #0x0]\n" + "usubl v1.8h, v1.8b, v18.8b\n" + "usubl v17.8h, v17.8b, v18.8b\n" + "ldp x22, x21, [x12, #0x10]\n" + "ldr x20, [x12, #0x20]\n" + "usubl v8.8h, v8.8b, v18.8b\n" + "usubl v31.8h, v31.8b, v18.8b\n" + "usubl v29.8h, v29.8b, v18.8b\n" + "usubl v16.8h, v16.8b, v18.8b\n" + "usubl v4.8h, v4.8b, v18.8b\n" + "add x24, x24, x14\n" + "add x23, x23, x14\n" + "add x22, x22, x14\n" + "add x21, x21, x14\n" + "add x20, x20, x14\n" + "tbz x16, #2, 9f\n" + "ld1 { v23.s }[0], [x24], #0x4\n" + "ld1 { v10.s }[0], [x23], #0x4\n" + "ld1 { v11.s }[0], [x22], #0x4\n" + "ld1 { v13.s }[0], [x21], #0x4\n" "ld1 { v27.s }[0], [x20], #0x4\n" - "tbz x7, #1, 8f\n" - "ld1 { v31.h }[2], [x24], #0x2\n" - "ld1 { v30.h }[2], [x23], #0x2\n" - "ld1 { v29.h }[2], [x22], #0x2\n" - "ld1 { v28.h }[2], [x21], #0x2\n" + "tbz x16, #1, 8f\n" + "ld1 { v23.h }[2], [x24], #0x2\n" + "ld1 { v10.h }[2], [x23], #0x2\n" + "ld1 { v11.h }[2], [x22], #0x2\n" + "ld1 { v13.h }[2], [x21], #0x2\n" "ld1 { v27.h }[2], [x20], #0x2\n" - "tbz x7, #0, 11f\n" - "ld1 { v31.b }[6], [x24]\n" - "ld1 { v30.b }[6], [x23]\n" - "ld1 { v29.b }[6], [x22]\n" - "ld1 { v28.b }[6], [x21]\n" + "tbz x16, #0, 11f\n" + "ld1 { v23.b }[6], [x24]\n" + "ld1 { v10.b }[6], [x23]\n" + "ld1 { v11.b }[6], [x22]\n" + "ld1 { v13.b }[6], [x21]\n" "ld1 { v27.b }[6], [x20]\n" "b 11f\n" "8:" // Oddments: Initial loads: Bit 2: Bit 1: Unset - "tbz x7, #0, 11f\n" - "ld1 { v31.b }[4], [x24]\n" - "ld1 { v30.b }[4], [x23]\n" - "ld1 { v29.b }[4], [x22]\n" - "ld1 { v28.b }[4], [x21]\n" + "tbz x16, #0, 11f\n" + "ld1 { v23.b }[4], [x24]\n" + "ld1 { v10.b }[4], [x23]\n" + "ld1 { v11.b }[4], [x22]\n" + "ld1 { v13.b }[4], [x21]\n" "ld1 { v27.b }[4], [x20]\n" "b 11f\n" "9:" // Oddments: Initial loads: Bit 2: Unset - "tbz x7, #1, 10f\n" - "ld1 { v31.h }[0], [x24], #0x2\n" - "ld1 { v30.h }[0], [x23], #0x2\n" - "ld1 { v29.h }[0], [x22], #0x2\n" - "ld1 { v28.h }[0], [x21], #0x2\n" + "tbz x16, #1, 10f\n" + "ld1 { v23.h }[0], [x24], #0x2\n" + "ld1 { v10.h }[0], [x23], #0x2\n" + "ld1 { v11.h }[0], [x22], #0x2\n" + "ld1 { v13.h }[0], [x21], #0x2\n" "ld1 { v27.h }[0], [x20], #0x2\n" - "tbz x7, #0, 11f\n" - "ld1 { v31.b }[2], [x24]\n" - "ld1 { v30.b }[2], [x23]\n" - "ld1 { v29.b }[2], [x22]\n" - "ld1 { v28.b }[2], [x21]\n" + "tbz x16, #0, 11f\n" + "ld1 { v23.b }[2], [x24]\n" + "ld1 { v10.b }[2], [x23]\n" + "ld1 { v11.b }[2], [x22]\n" + "ld1 { v13.b }[2], [x21]\n" "ld1 { v27.b }[2], [x20]\n" "b 11f\n" "10:" // Oddments: Initial loads: Bit 2: Unset: Bit 1: Unset - "tbz x7, #0, 11f\n" - "ld1 { v31.b }[0], [x24]\n" - "ld1 { v30.b }[0], [x23]\n" - "ld1 { v29.b }[0], [x22]\n" - "ld1 { v28.b }[0], [x21]\n" + "tbz x16, #0, 11f\n" + "ld1 { v23.b }[0], [x24]\n" + "ld1 { v10.b }[0], [x23]\n" + "ld1 { v11.b }[0], [x22]\n" + "ld1 { v13.b }[0], [x21]\n" "ld1 { v27.b }[0], [x20]\n" "11:" // Oddments: Initial loads: Bit 2: End - "ushll v31.8h, v31.8b, #0x0\n" - "smlal v14.4s, v31.4h, v4.4h\n" - "smlal2 v11.4s, v31.8h, v4.8h\n" - "ldr x22, [x15, #0x28]\n" - "smlal v16.4s, v31.4h, v3.4h\n" - "smlal2 v9.4s, v31.8h, v3.8h\n" - "ushll v30.8h, v30.8b, #0x0\n" - "add x22, x22, x17\n" - "ushll v29.8h, v29.8b, #0x0\n" - "smlal v24.4s, v31.4h, v1.4h\n" - "smlal2 v17.4s, v31.8h, v1.8h\n" - "smlal v23.4s, v31.4h, v0.4h\n" - "smlal2 v25.4s, v31.8h, v0.8h\n" - "ushll v28.8h, v28.8b, #0x0\n" - "smlal v14.4s, v30.4h, v0.4h\n" - "smlal2 v11.4s, v30.8h, v0.8h\n" + "ushll v23.8h, v23.8b, #0x0\n" + "smlal v28.4s, v23.4h, v8.4h\n" + "smlal2 v9.4s, v23.8h, v8.8h\n" + "ldr x20, [x12, #0x28]\n" + "smlal v3.4s, v23.4h, v17.4h\n" + "smlal2 v30.4s, v23.8h, v17.8h\n" + "ushll v10.8h, v10.8b, #0x0\n" + "ushll v11.8h, v11.8b, #0x0\n" + "smlal v0.4s, v23.4h, v7.4h\n" + "smlal2 v22.4s, v23.8h, v7.8h\n" + "add x20, x20, x14\n" + "smlal v6.4s, v23.4h, v19.4h\n" + "smlal2 v2.4s, v23.8h, v19.8h\n" + "ushll v13.8h, v13.8b, #0x0\n" + "smlal v28.4s, v10.4h, v19.4h\n" + "smlal2 v9.4s, v10.8h, v19.8h\n" "ushll v27.8h, v27.8b, #0x0\n" - "smlal v16.4s, v29.4h, v2.4h\n" - "smlal2 v9.4s, v29.8h, v2.8h\n" - "smlal v14.4s, v28.4h, v5.4h\n" - "smlal2 v11.4s, v28.8h, v5.8h\n" - "smlal v16.4s, v28.4h, v4.4h\n" - "smlal2 v9.4s, v28.8h, v4.8h\n" - "smlal v24.4s, v28.4h, v2.4h\n" - "smlal2 v17.4s, v28.8h, v2.8h\n" - "smlal v23.4s, v28.4h, v1.4h\n" - "smlal2 v25.4s, v28.8h, v1.8h\n" - "tbz x7, #2, 13f\n" - "ld1 { v31.s }[0], [x22], #0x4\n" - "tbz x7, #1, 12f\n" - "ld1 { v31.h }[2], [x22], #0x2\n" - "tbz x7, #0, 15f\n" - "ld1 { v31.b }[6], [x22]\n" + "smlal v3.4s, v11.4h, v1.4h\n" + "smlal2 v30.4s, v11.8h, v1.8h\n" + "smlal v28.4s, v13.4h, v31.4h\n" + "smlal2 v9.4s, v13.8h, v31.8h\n" + "smlal v3.4s, v13.4h, v8.4h\n" + "smlal2 v30.4s, v13.8h, v8.8h\n" + "smlal v0.4s, v13.4h, v1.4h\n" + "smlal2 v22.4s, v13.8h, v1.8h\n" + "smlal v6.4s, v13.4h, v7.4h\n" + "smlal2 v2.4s, v13.8h, v7.8h\n" + "tbz x16, #2, 13f\n" + "ld1 { v26.s }[0], [x20], #0x4\n" + "tbz x16, #1, 12f\n" + "ld1 { v26.h }[2], [x20], #0x2\n" + "tbz x16, #0, 15f\n" + "ld1 { v26.b }[6], [x20]\n" "b 15f\n" "12:" // Oddments: Load (3, 0): Bit 2: Bit 1: Unset - "tbz x7, #0, 15f\n" - "ld1 { v31.b }[4], [x22]\n" + "tbz x16, #0, 15f\n" + "ld1 { v26.b }[4], [x20]\n" "b 15f\n" "13:" // Oddments: Load (3, 0): Bit 2: Unset - "tbz x7, #1, 14f\n" - "ld1 { v31.h }[0], [x22], #0x2\n" - "tbz x7, #0, 15f\n" - "ld1 { v31.b }[2], [x22]\n" + "tbz x16, #1, 14f\n" + "ld1 { v26.h }[0], [x20], #0x2\n" + "tbz x16, #0, 15f\n" + "ld1 { v26.b }[2], [x20]\n" "b 15f\n" "14:" // Oddments: Load (3, 0): Bit 2: Unset: Bit 1: Unset - "tbz x7, #0, 15f\n" - "ld1 { v31.b }[0], [x22]\n" + "tbz x16, #0, 15f\n" + "ld1 { v26.b }[0], [x20]\n" "15:" // Oddments: Load (3, 0): Bit 2: End - "ushll v31.8h, v31.8b, #0x0\n" - "smlal v24.4s, v31.4h, v6.4h\n" - "smlal2 v17.4s, v31.8h, v6.8h\n" - "ldr x20, [x15, #0x30]\n" - "smlal v14.4s, v27.4h, v7.4h\n" - "smlal2 v11.4s, v27.8h, v7.8h\n" - "add x20, x20, x17\n" - "smlal v16.4s, v27.4h, v6.4h\n" - "smlal2 v9.4s, v27.8h, v6.8h\n" - "smlal v24.4s, v27.4h, v4.4h\n" - "smlal2 v17.4s, v27.8h, v4.8h\n" - "smlal v23.4s, v27.4h, v3.4h\n" - "smlal2 v25.4s, v27.8h, v3.8h\n" - "tbz x7, #2, 17f\n" - "ld1 { v29.s }[0], [x20], #0x4\n" - "tbz x7, #1, 16f\n" - "ld1 { v29.h }[2], [x20], #0x2\n" - "tbz x7, #0, 19f\n" - "ld1 { v29.b }[6], [x20]\n" + "ushll v26.8h, v26.8b, #0x0\n" + "smlal v0.4s, v26.4h, v29.4h\n" + "smlal2 v22.4s, v26.8h, v29.8h\n" + "ldr x20, [x12, #0x30]\n" + "smlal v28.4s, v27.4h, v16.4h\n" + "smlal2 v9.4s, v27.8h, v16.8h\n" + "add x20, x20, x14\n" + "smlal v3.4s, v27.4h, v29.4h\n" + "smlal2 v30.4s, v27.8h, v29.8h\n" + "smlal v0.4s, v27.4h, v8.4h\n" + "smlal2 v22.4s, v27.8h, v8.8h\n" + "smlal v6.4s, v27.4h, v17.4h\n" + "smlal2 v2.4s, v27.8h, v17.8h\n" + "tbz x16, #2, 17f\n" + "ld1 { v23.s }[0], [x20], #0x4\n" + "tbz x16, #1, 16f\n" + "ld1 { v23.h }[2], [x20], #0x2\n" + "tbz x16, #0, 19f\n" + "ld1 { v23.b }[6], [x20]\n" "b 19f\n" "16:" // Oddments: Load (3, 3): Bit 2: Bit 1: Unset - "tbz x7, #0, 19f\n" - "ld1 { v29.b }[4], [x20]\n" + "tbz x16, #0, 19f\n" + "ld1 { v23.b }[4], [x20]\n" "b 19f\n" "17:" // Oddments: Load (3, 3): Bit 2: Unset - "tbz x7, #1, 18f\n" - "ld1 { v29.h }[0], [x20], #0x2\n" - "tbz x7, #0, 19f\n" - "ld1 { v29.b }[2], [x20]\n" + "tbz x16, #1, 18f\n" + "ld1 { v23.h }[0], [x20], #0x2\n" + "tbz x16, #0, 19f\n" + "ld1 { v23.b }[2], [x20]\n" "b 19f\n" "18:" // Oddments: Load (3, 3): Bit 2: Unset: Bit 1: Unset - "tbz x7, #0, 19f\n" - "ld1 { v29.b }[0], [x20]\n" + "tbz x16, #0, 19f\n" + "ld1 { v23.b }[0], [x20]\n" "19:" // Oddments: Load (3, 3): Bit 2: End - "ushll v29.8h, v29.8b, #0x0\n" - "ldr x21, [x15, #0x38]\n" - "smlal v23.4s, v29.4h, v8.4h\n" - "smlal2 v25.4s, v29.8h, v8.8h\n" - "add x21, x21, x17\n" - "tbz x7, #2, 21f\n" - "ld1 { v28.s }[0], [x21], #0x4\n" - "tbz x7, #1, 20f\n" - "ld1 { v28.h }[2], [x21], #0x2\n" - "tbz x7, #0, 23f\n" - "ld1 { v28.b }[6], [x21]\n" + "ushll v23.8h, v23.8b, #0x0\n" + "ldr x20, [x12, #0x38]\n" + "smlal v6.4s, v23.4h, v4.4h\n" + "smlal2 v2.4s, v23.8h, v4.8h\n" + "add x20, x20, x14\n" + "tbz x16, #2, 21f\n" + "ld1 { v21.s }[0], [x20], #0x4\n" + "tbz x16, #1, 20f\n" + "ld1 { v21.h }[2], [x20], #0x2\n" + "tbz x16, #0, 23f\n" + "ld1 { v21.b }[6], [x20]\n" "b 23f\n" "20:" // Oddments: Load (0, 1): Bit 2: Bit 1: Unset - "tbz x7, #0, 23f\n" - "ld1 { v28.b }[4], [x21]\n" + "tbz x16, #0, 23f\n" + "ld1 { v21.b }[4], [x20]\n" "b 23f\n" "21:" // Oddments: Load (0, 1): Bit 2: Unset - "tbz x7, #1, 22f\n" - "ld1 { v28.h }[0], [x21], #0x2\n" - "tbz x7, #0, 23f\n" - "ld1 { v28.b }[2], [x21]\n" + "tbz x16, #1, 22f\n" + "ld1 { v21.h }[0], [x20], #0x2\n" + "tbz x16, #0, 23f\n" + "ld1 { v21.b }[2], [x20]\n" "b 23f\n" "22:" // Oddments: Load (0, 1): Bit 2: Unset: Bit 1: Unset - "tbz x7, #0, 23f\n" - "ld1 { v28.b }[0], [x21]\n" + "tbz x16, #0, 23f\n" + "ld1 { v21.b }[0], [x20]\n" "23:" // Oddments: Load (0, 1): Bit 2: End - "ushll v28.8h, v28.8b, #0x0\n" - "ldr x26, [x15, #0x40]\n" - "smlal v14.4s, v28.4h, v1.4h\n" - "smlal2 v11.4s, v28.8h, v1.8h\n" - "smlal v16.4s, v28.4h, v0.4h\n" - "smlal2 v9.4s, v28.8h, v0.8h\n" - "add x26, x26, x17\n" - "tbz x7, #2, 25f\n" - "ld1 { v31.s }[0], [x26], #0x4\n" - "tbz x7, #1, 24f\n" - "ld1 { v31.h }[2], [x26], #0x2\n" - "tbz x7, #0, 27f\n" - "ld1 { v31.b }[6], [x26]\n" + "ushll v21.8h, v21.8b, #0x0\n" + "ldr x20, [x12, #0x40]\n" + "smlal v28.4s, v21.4h, v7.4h\n" + "smlal2 v9.4s, v21.8h, v7.8h\n" + "smlal v3.4s, v21.4h, v19.4h\n" + "smlal2 v30.4s, v21.8h, v19.8h\n" + "add x20, x20, x14\n" + "tbz x16, #2, 25f\n" + "ld1 { v18.s }[0], [x20], #0x4\n" + "tbz x16, #1, 24f\n" + "ld1 { v18.h }[2], [x20], #0x2\n" + "tbz x16, #0, 27f\n" + "ld1 { v18.b }[6], [x20]\n" "b 27f\n" "24:" // Oddments: Load (0, 2): Bit 2: Bit 1: Unset - "tbz x7, #0, 27f\n" - "ld1 { v31.b }[4], [x26]\n" + "tbz x16, #0, 27f\n" + "ld1 { v18.b }[4], [x20]\n" "b 27f\n" "25:" // Oddments: Load (0, 2): Bit 2: Unset - "tbz x7, #1, 26f\n" - "ld1 { v31.h }[0], [x26], #0x2\n" - "tbz x7, #0, 27f\n" - "ld1 { v31.b }[2], [x26]\n" + "tbz x16, #1, 26f\n" + "ld1 { v18.h }[0], [x20], #0x2\n" + "tbz x16, #0, 27f\n" + "ld1 { v18.b }[2], [x20]\n" "b 27f\n" "26:" // Oddments: Load (0, 2): Bit 2: Unset: Bit 1: Unset - "tbz x7, #0, 27f\n" - "ld1 { v31.b }[0], [x26]\n" + "tbz x16, #0, 27f\n" + "ld1 { v18.b }[0], [x20]\n" "27:" // Oddments: Load (0, 2): Bit 2: End - "ushll v31.8h, v31.8b, #0x0\n" - "ldr x20, [x15, #0x48]\n" - "smlal v14.4s, v31.4h, v2.4h\n" - "smlal2 v11.4s, v31.8h, v2.8h\n" - "smlal v16.4s, v31.4h, v1.4h\n" - "smlal2 v9.4s, v31.8h, v1.8h\n" - "add x20, x20, x17\n" - "tbz x7, #2, 29f\n" - "ld1 { v30.s }[0], [x20], #0x4\n" - "tbz x7, #1, 28f\n" - "ld1 { v30.h }[2], [x20], #0x2\n" - "tbz x7, #0, 31f\n" - "ld1 { v30.b }[6], [x20]\n" + "ushll v18.8h, v18.8b, #0x0\n" + "ldr x20, [x12, #0x48]\n" + "smlal v28.4s, v18.4h, v1.4h\n" + "smlal2 v9.4s, v18.8h, v1.8h\n" + "smlal v3.4s, v18.4h, v7.4h\n" + "smlal2 v30.4s, v18.8h, v7.8h\n" + "add x20, x20, x14\n" + "tbz x16, #2, 29f\n" + "ld1 { v15.s }[0], [x20], #0x4\n" + "tbz x16, #1, 28f\n" + "ld1 { v15.h }[2], [x20], #0x2\n" + "tbz x16, #0, 31f\n" + "ld1 { v15.b }[6], [x20]\n" "b 31f\n" "28:" // Oddments: Load (2, 2): Bit 2: Bit 1: Unset - "tbz x7, #0, 31f\n" - "ld1 { v30.b }[4], [x20]\n" + "tbz x16, #0, 31f\n" + "ld1 { v15.b }[4], [x20]\n" "b 31f\n" "29:" // Oddments: Load (2, 2): Bit 2: Unset - "tbz x7, #1, 30f\n" - "ld1 { v30.h }[0], [x20], #0x2\n" - "tbz x7, #0, 31f\n" - "ld1 { v30.b }[2], [x20]\n" + "tbz x16, #1, 30f\n" + "ld1 { v15.h }[0], [x20], #0x2\n" + "tbz x16, #0, 31f\n" + "ld1 { v15.b }[2], [x20]\n" "b 31f\n" "30:" // Oddments: Load (2, 2): Bit 2: Unset: Bit 1: Unset - "tbz x7, #0, 31f\n" - "ld1 { v30.b }[0], [x20]\n" + "tbz x16, #0, 31f\n" + "ld1 { v15.b }[0], [x20]\n" "31:" // Oddments: Load (2, 2): Bit 2: End - "ushll v30.8h, v30.8b, #0x0\n" - "ldr x25, [x15, #0x50]\n" - "smlal v14.4s, v30.4h, v8.4h\n" - "smlal2 v11.4s, v30.8h, v8.8h\n" - "smlal v16.4s, v30.4h, v7.4h\n" - "smlal2 v9.4s, v30.8h, v7.8h\n" - "add x25, x25, x17\n" - "smlal v24.4s, v30.4h, v5.4h\n" - "smlal2 v17.4s, v30.8h, v5.8h\n" - "smlal v23.4s, v30.4h, v4.4h\n" - "smlal2 v25.4s, v30.8h, v4.8h\n" - "tbz x7, #2, 33f\n" - "ld1 { v29.s }[0], [x25], #0x4\n" - "tbz x7, #1, 32f\n" - "ld1 { v29.h }[2], [x25], #0x2\n" - "tbz x7, #0, 35f\n" - "ld1 { v29.b }[6], [x25]\n" + "ushll v15.8h, v15.8b, #0x0\n" + "ldr x20, [x12, #0x50]\n" + "smlal v28.4s, v15.4h, v4.4h\n" + "smlal2 v9.4s, v15.8h, v4.8h\n" + "smlal v3.4s, v15.4h, v16.4h\n" + "smlal2 v30.4s, v15.8h, v16.8h\n" + "add x20, x20, x14\n" + "smlal v0.4s, v15.4h, v31.4h\n" + "smlal2 v22.4s, v15.8h, v31.8h\n" + "smlal v6.4s, v15.4h, v8.4h\n" + "smlal2 v2.4s, v15.8h, v8.8h\n" + "tbz x16, #2, 33f\n" + "ld1 { v20.s }[0], [x20], #0x4\n" + "tbz x16, #1, 32f\n" + "ld1 { v20.h }[2], [x20], #0x2\n" + "tbz x16, #0, 35f\n" + "ld1 { v20.b }[6], [x20]\n" "b 35f\n" "32:" // Oddments: Load (1, 0): Bit 2: Bit 1: Unset - "tbz x7, #0, 35f\n" - "ld1 { v29.b }[4], [x25]\n" + "tbz x16, #0, 35f\n" + "ld1 { v20.b }[4], [x20]\n" "b 35f\n" "33:" // Oddments: Load (1, 0): Bit 2: Unset - "tbz x7, #1, 34f\n" - "ld1 { v29.h }[0], [x25], #0x2\n" - "tbz x7, #0, 35f\n" - "ld1 { v29.b }[2], [x25]\n" + "tbz x16, #1, 34f\n" + "ld1 { v20.h }[0], [x20], #0x2\n" + "tbz x16, #0, 35f\n" + "ld1 { v20.b }[2], [x20]\n" "b 35f\n" "34:" // Oddments: Load (1, 0): Bit 2: Unset: Bit 1: Unset - "tbz x7, #0, 35f\n" - "ld1 { v29.b }[0], [x25]\n" + "tbz x16, #0, 35f\n" + "ld1 { v20.b }[0], [x20]\n" "35:" // Oddments: Load (1, 0): Bit 2: End - "ushll v29.8h, v29.8b, #0x0\n" - "ldr x24, [x15, #0x58]\n" - "smlal v14.4s, v29.4h, v3.4h\n" - "smlal2 v11.4s, v29.8h, v3.8h\n" - "smlal v24.4s, v29.4h, v0.4h\n" - "smlal2 v17.4s, v29.8h, v0.8h\n" - "add x24, x24, x17\n" - "tbz x7, #2, 37f\n" - "ld1 { v28.s }[0], [x24], #0x4\n" - "tbz x7, #1, 36f\n" - "ld1 { v28.h }[2], [x24], #0x2\n" - "tbz x7, #0, 39f\n" - "ld1 { v28.b }[6], [x24]\n" + "ushll v20.8h, v20.8b, #0x0\n" + "ldr x20, [x12, #0x58]\n" + "smlal v28.4s, v20.4h, v17.4h\n" + "smlal2 v9.4s, v20.8h, v17.8h\n" + "smlal v0.4s, v20.4h, v19.4h\n" + "smlal2 v22.4s, v20.8h, v19.8h\n" + "add x20, x20, x14\n" + "tbz x16, #2, 37f\n" + "ld1 { v11.s }[0], [x20], #0x4\n" + "tbz x16, #1, 36f\n" + "ld1 { v11.h }[2], [x20], #0x2\n" + "tbz x16, #0, 39f\n" + "ld1 { v11.b }[6], [x20]\n" "b 39f\n" "36:" // Oddments: Load (1, 3): Bit 2: Bit 1: Unset - "tbz x7, #0, 39f\n" - "ld1 { v28.b }[4], [x24]\n" + "tbz x16, #0, 39f\n" + "ld1 { v11.b }[4], [x20]\n" "b 39f\n" "37:" // Oddments: Load (1, 3): Bit 2: Unset - "tbz x7, #1, 38f\n" - "ld1 { v28.h }[0], [x24], #0x2\n" - "tbz x7, #0, 39f\n" - "ld1 { v28.b }[2], [x24]\n" + "tbz x16, #1, 38f\n" + "ld1 { v11.h }[0], [x20], #0x2\n" + "tbz x16, #0, 39f\n" + "ld1 { v11.b }[2], [x20]\n" "b 39f\n" "38:" // Oddments: Load (1, 3): Bit 2: Unset: Bit 1: Unset - "tbz x7, #0, 39f\n" - "ld1 { v28.b }[0], [x24]\n" + "tbz x16, #0, 39f\n" + "ld1 { v11.b }[0], [x20]\n" "39:" // Oddments: Load (1, 3): Bit 2: End - "ushll v28.8h, v28.8b, #0x0\n" - "ldr x23, [x15, #0x60]\n" - "smlal v16.4s, v28.4h, v5.4h\n" - "smlal2 v9.4s, v28.8h, v5.8h\n" - "smlal v23.4s, v28.4h, v2.4h\n" - "smlal2 v25.4s, v28.8h, v2.8h\n" - "add x23, x23, x17\n" - "tbz x7, #2, 41f\n" - "ld1 { v31.s }[0], [x23], #0x4\n" - "tbz x7, #1, 40f\n" - "ld1 { v31.h }[2], [x23], #0x2\n" - "tbz x7, #0, 43f\n" - "ld1 { v31.b }[6], [x23]\n" + "ushll v11.8h, v11.8b, #0x0\n" + "ldr x20, [x12, #0x60]\n" + "smlal v3.4s, v11.4h, v31.4h\n" + "smlal2 v30.4s, v11.8h, v31.8h\n" + "smlal v6.4s, v11.4h, v1.4h\n" + "smlal2 v2.4s, v11.8h, v1.8h\n" + "add x20, x20, x14\n" + "tbz x16, #2, 41f\n" + "ld1 { v23.s }[0], [x20], #0x4\n" + "tbz x16, #1, 40f\n" + "ld1 { v23.h }[2], [x20], #0x2\n" + "tbz x16, #0, 43f\n" + "ld1 { v23.b }[6], [x20]\n" "b 43f\n" "40:" // Oddments: Load (2, 0): Bit 2: Bit 1: Unset - "tbz x7, #0, 43f\n" - "ld1 { v31.b }[4], [x23]\n" + "tbz x16, #0, 43f\n" + "ld1 { v23.b }[4], [x20]\n" "b 43f\n" "41:" // Oddments: Load (2, 0): Bit 2: Unset - "tbz x7, #1, 42f\n" - "ld1 { v31.h }[0], [x23], #0x2\n" - "tbz x7, #0, 43f\n" - "ld1 { v31.b }[2], [x23]\n" + "tbz x16, #1, 42f\n" + "ld1 { v23.h }[0], [x20], #0x2\n" + "tbz x16, #0, 43f\n" + "ld1 { v23.b }[2], [x20]\n" "b 43f\n" "42:" // Oddments: Load (2, 0): Bit 2: Unset: Bit 1: Unset - "tbz x7, #0, 43f\n" - "ld1 { v31.b }[0], [x23]\n" + "tbz x16, #0, 43f\n" + "ld1 { v23.b }[0], [x20]\n" "43:" // Oddments: Load (2, 0): Bit 2: End - "ushll v31.8h, v31.8b, #0x0\n" - "ldr x22, [x15, #0x68]\n" - "smlal v14.4s, v31.4h, v6.4h\n" - "smlal2 v11.4s, v31.8h, v6.8h\n" - "smlal v24.4s, v31.4h, v3.4h\n" - "smlal2 v17.4s, v31.8h, v3.8h\n" - "add x22, x22, x17\n" - "tbz x7, #2, 45f\n" - "ld1 { v30.s }[0], [x22], #0x4\n" - "tbz x7, #1, 44f\n" - "ld1 { v30.h }[2], [x22], #0x2\n" - "tbz x7, #0, 47f\n" - "ld1 { v30.b }[6], [x22]\n" + "ushll v23.8h, v23.8b, #0x0\n" + "ldr x20, [x12, #0x68]\n" + "smlal v28.4s, v23.4h, v29.4h\n" + "smlal2 v9.4s, v23.8h, v29.8h\n" + "smlal v0.4s, v23.4h, v17.4h\n" + "smlal2 v22.4s, v23.8h, v17.8h\n" + "add x20, x20, x14\n" + "tbz x16, #2, 45f\n" + "ld1 { v20.s }[0], [x20], #0x4\n" + "tbz x16, #1, 44f\n" + "ld1 { v20.h }[2], [x20], #0x2\n" + "tbz x16, #0, 47f\n" + "ld1 { v20.b }[6], [x20]\n" "b 47f\n" "44:" // Oddments: Load (2, 3): Bit 2: Bit 1: Unset - "tbz x7, #0, 47f\n" - "ld1 { v30.b }[4], [x22]\n" + "tbz x16, #0, 47f\n" + "ld1 { v20.b }[4], [x20]\n" "b 47f\n" "45:" // Oddments: Load (2, 3): Bit 2: Unset - "tbz x7, #1, 46f\n" - "ld1 { v30.h }[0], [x22], #0x2\n" - "tbz x7, #0, 47f\n" - "ld1 { v30.b }[2], [x22]\n" + "tbz x16, #1, 46f\n" + "ld1 { v20.h }[0], [x20], #0x2\n" + "tbz x16, #0, 47f\n" + "ld1 { v20.b }[2], [x20]\n" "b 47f\n" "46:" // Oddments: Load (2, 3): Bit 2: Unset: Bit 1: Unset - "tbz x7, #0, 47f\n" - "ld1 { v30.b }[0], [x22]\n" + "tbz x16, #0, 47f\n" + "ld1 { v20.b }[0], [x20]\n" "47:" // Oddments: Load (2, 3): Bit 2: End - "ushll v30.8h, v30.8b, #0x0\n" - "ldr x21, [x15, #0x70]\n" - "smlal v16.4s, v30.4h, v8.4h\n" - "smlal2 v9.4s, v30.8h, v8.8h\n" - "smlal v23.4s, v30.4h, v5.4h\n" - "smlal2 v25.4s, v30.8h, v5.8h\n" - "add x21, x21, x17\n" - "tbz x7, #2, 49f\n" - "ld1 { v29.s }[0], [x21], #0x4\n" - "tbz x7, #1, 48f\n" - "ld1 { v29.h }[2], [x21], #0x2\n" - "tbz x7, #0, 51f\n" - "ld1 { v29.b }[6], [x21]\n" + "ushll v20.8h, v20.8b, #0x0\n" + "ldr x20, [x12, #0x70]\n" + "smlal v3.4s, v20.4h, v4.4h\n" + "smlal2 v30.4s, v20.8h, v4.8h\n" + "smlal v6.4s, v20.4h, v31.4h\n" + "smlal2 v2.4s, v20.8h, v31.8h\n" + "add x20, x20, x14\n" + "tbz x16, #2, 49f\n" + "ld1 { v8.s }[0], [x20], #0x4\n" + "tbz x16, #1, 48f\n" + "ld1 { v8.h }[2], [x20], #0x2\n" + "tbz x16, #0, 51f\n" + "ld1 { v8.b }[6], [x20]\n" "b 51f\n" "48:" // Oddments: Load (3, 1): Bit 2: Bit 1: Unset - "tbz x7, #0, 51f\n" - "ld1 { v29.b }[4], [x21]\n" + "tbz x16, #0, 51f\n" + "ld1 { v8.b }[4], [x20]\n" "b 51f\n" "49:" // Oddments: Load (3, 1): Bit 2: Unset - "tbz x7, #1, 50f\n" - "ld1 { v29.h }[0], [x21], #0x2\n" - "tbz x7, #0, 51f\n" - "ld1 { v29.b }[2], [x21]\n" + "tbz x16, #1, 50f\n" + "ld1 { v8.h }[0], [x20], #0x2\n" + "tbz x16, #0, 51f\n" + "ld1 { v8.b }[2], [x20]\n" "b 51f\n" "50:" // Oddments: Load (3, 1): Bit 2: Unset: Bit 1: Unset - "tbz x7, #0, 51f\n" - "ld1 { v29.b }[0], [x21]\n" + "tbz x16, #0, 51f\n" + "ld1 { v8.b }[0], [x20]\n" "51:" // Oddments: Load (3, 1): Bit 2: End - "ushll v29.8h, v29.8b, #0x0\n" - "ldr x20, [x15, #0x78]\n" - "smlal v24.4s, v29.4h, v7.4h\n" - "smlal2 v17.4s, v29.8h, v7.8h\n" - "smlal v23.4s, v29.4h, v6.4h\n" - "smlal2 v25.4s, v29.8h, v6.8h\n" - "add x20, x20, x17\n" - "tbz x7, #2, 53f\n" - "ld1 { v28.s }[0], [x20], #0x4\n" - "tbz x7, #1, 52f\n" - "ld1 { v28.h }[2], [x20], #0x2\n" - "tbz x7, #0, 55f\n" - "ld1 { v28.b }[6], [x20]\n" + "ushll v8.8h, v8.8b, #0x0\n" + "ldr x20, [x12, #0x78]\n" + "smlal v0.4s, v8.4h, v16.4h\n" + "smlal2 v22.4s, v8.8h, v16.8h\n" + "smlal v6.4s, v8.4h, v29.4h\n" + "smlal2 v2.4s, v8.8h, v29.8h\n" + "add x20, x20, x14\n" + "tbz x16, #2, 53f\n" + "ld1 { v8.s }[0], [x20], #0x4\n" + "tbz x16, #1, 52f\n" + "ld1 { v8.h }[2], [x20], #0x2\n" + "tbz x16, #0, 55f\n" + "ld1 { v8.b }[6], [x20]\n" "b 55f\n" "52:" // Oddments: Load (3, 2): Bit 2: Bit 1: Unset - "tbz x7, #0, 55f\n" - "ld1 { v28.b }[4], [x20]\n" + "tbz x16, #0, 55f\n" + "ld1 { v8.b }[4], [x20]\n" "b 55f\n" "53:" // Oddments: Load (3, 2): Bit 2: Unset - "tbz x7, #1, 54f\n" - "ld1 { v28.h }[0], [x20], #0x2\n" - "tbz x7, #0, 55f\n" - "ld1 { v28.b }[2], [x20]\n" + "tbz x16, #1, 54f\n" + "ld1 { v8.h }[0], [x20], #0x2\n" + "tbz x16, #0, 55f\n" + "ld1 { v8.b }[2], [x20]\n" "b 55f\n" "54:" // Oddments: Load (3, 2): Bit 2: Unset: Bit 1: Unset - "tbz x7, #0, 55f\n" - "ld1 { v28.b }[0], [x20]\n" + "tbz x16, #0, 55f\n" + "ld1 { v8.b }[0], [x20]\n" "55:" // Oddments: Load (3, 2): Bit 2: End - "ushll v28.8h, v28.8b, #0x0\n" - "smlal v24.4s, v28.4h, v8.4h\n" - "smlal2 v17.4s, v28.8h, v8.8h\n" - "smlal v23.4s, v28.4h, v7.4h\n" - "smlal2 v25.4s, v28.8h, v7.8h\n" - "tbz x7, #2, 57f\n" - "ld1 { v22.4s }, [x13], #0x10\n" - "ld1 { v10.4s }, [x12], #0x10\n" - "tbz x7, #1, 56f\n" - "ld1 { v18.d }[0], [x13], #0x8\n" - "ld1 { v26.d }[0], [x12], #0x8\n" - "tbz x7, #0, 59f\n" - "ld1 { v18.s }[2], [x13]\n" - "ld1 { v26.s }[2], [x12]\n" + "ushll v8.8h, v8.8b, #0x0\n" + "smlal v0.4s, v8.4h, v4.4h\n" + "smlal2 v22.4s, v8.8h, v4.8h\n" + "smlal v6.4s, v8.4h, v16.4h\n" + "smlal2 v2.4s, v8.8h, v16.8h\n" + "tbz x16, #2, 57f\n" + "ld1 { v7.4s }, [x10], #0x10\n" + "ld1 { v23.4s }, [x9], #0x10\n" + "tbz x16, #1, 56f\n" + "ld1 { v11.d }[0], [x10], #0x8\n" + "ld1 { v27.d }[0], [x9], #0x8\n" + "tbz x16, #0, 59f\n" + "ld1 { v11.s }[2], [x10]\n" + "ld1 { v27.s }[2], [x9]\n" "b 59f\n" "56:" // Oddments: Load requant params: Bit 2: Bit 1: Unset - "tbz x7, #0, 59f\n" - "ld1 { v18.s }[0], [x13]\n" - "ld1 { v26.s }[0], [x12]\n" + "tbz x16, #0, 59f\n" + "ld1 { v11.s }[0], [x10]\n" + "ld1 { v27.s }[0], [x9]\n" "b 59f\n" "57:" // Oddments: Load requant params: Bit 2: Unset - "tbz x7, #1, 58f\n" - "ld1 { v22.d }[0], [x13], #0x8\n" - "ld1 { v10.d }[0], [x12], #0x8\n" - "tbz x7, #0, 59f\n" - "ld1 { v22.s }[2], [x13]\n" - "ld1 { v10.s }[2], [x12]\n" + "tbz x16, #1, 58f\n" + "ld1 { v7.d }[0], [x10], #0x8\n" + "ld1 { v23.d }[0], [x9], #0x8\n" + "tbz x16, #0, 59f\n" + "ld1 { v7.s }[2], [x10]\n" + "ld1 { v23.s }[2], [x9]\n" "b 59f\n" "58:" // Oddments: Load requant params: Bit 2: Unset: Bit 1: Unset - "tbz x7, #0, 59f\n" - "ld1 { v22.s }[0], [x13]\n" - "ld1 { v10.s }[0], [x12]\n" + "tbz x16, #0, 59f\n" + "ld1 { v7.s }[0], [x10]\n" + "ld1 { v23.s }[0], [x9]\n" "59:" // Oddments: Load requant params: Bit 2: End - "sqrdmulh v14.4s, v14.4s, v22.4s\n" - "and v21.16b, v14.16b, v10.16b\n" - "add x11, x11, x16\n" - "add x10, x10, x16\n" - "sqrdmulh v11.4s, v11.4s, v18.4s\n" - "sshr v21.4s, v21.4s, #0x1f\n" - "add x9, x9, x16\n" - "add x28, x28, x16\n" - "and v4.16b, v11.16b, v26.16b\n" - "sqrdmulh v16.4s, v16.4s, v22.4s\n" - "sqrdmulh v24.4s, v24.4s, v22.4s\n" - "sqrdmulh v23.4s, v23.4s, v22.4s\n" - "sqadd v14.4s, v14.4s, v21.4s\n" + "sqrdmulh v28.4s, v28.4s, v7.4s\n" + "and v20.16b, v28.16b, v23.16b\n" + "add x28, x28, x13\n" + "add x27, x27, x13\n" + "sqrdmulh v9.4s, v9.4s, v11.4s\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "add x26, x26, x13\n" + "add x25, x25, x13\n" + "and v4.16b, v9.16b, v27.16b\n" + "sqrdmulh v3.4s, v3.4s, v7.4s\n" + "sqrdmulh v0.4s, v0.4s, v7.4s\n" + "sqrdmulh v6.4s, v6.4s, v7.4s\n" + "sqadd v28.4s, v28.4s, v20.4s\n" "sshr v4.4s, v4.4s, #0x1f\n" - "and v19.16b, v16.16b, v10.16b\n" - "sqrdmulh v9.4s, v9.4s, v18.4s\n" - "and v3.16b, v24.16b, v10.16b\n" - "sqrdmulh v17.4s, v17.4s, v18.4s\n" - "and v21.16b, v23.16b, v10.16b\n" - "sqrdmulh v25.4s, v25.4s, v18.4s\n" - "sqadd v11.4s, v11.4s, v4.4s\n" + "and v19.16b, v3.16b, v23.16b\n" + "sqrdmulh v30.4s, v30.4s, v11.4s\n" + "and v29.16b, v0.16b, v23.16b\n" + "sqrdmulh v22.4s, v22.4s, v11.4s\n" + "and v26.16b, v6.16b, v23.16b\n" + "sqrdmulh v2.4s, v2.4s, v11.4s\n" + "sqadd v9.4s, v9.4s, v4.4s\n" "sshr v19.4s, v19.4s, #0x1f\n" - "and v27.16b, v9.16b, v26.16b\n" - "sshr v3.4s, v3.4s, #0x1f\n" - "and v5.16b, v17.16b, v26.16b\n" - "sshr v21.4s, v21.4s, #0x1f\n" - "and v4.16b, v25.16b, v26.16b\n" - "sqadd v16.4s, v16.4s, v19.4s\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "sqadd v24.4s, v24.4s, v3.4s\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v23.4s, v23.4s, v21.4s\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "srshl v14.4s, v14.4s, v10.4s\n" - "srshl v16.4s, v16.4s, v10.4s\n" - "sqadd v9.4s, v9.4s, v27.4s\n" - "srshl v24.4s, v24.4s, v10.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "srshl v23.4s, v23.4s, v10.4s\n" - "sqadd v25.4s, v25.4s, v4.4s\n" - "srshl v11.4s, v11.4s, v26.4s\n" - "sqxtn v14.4h, v14.4s\n" - "srshl v9.4s, v9.4s, v26.4s\n" - "sqxtn v16.4h, v16.4s\n" - "srshl v17.4s, v17.4s, v26.4s\n" - "sqxtn v24.4h, v24.4s\n" - "srshl v25.4s, v25.4s, v26.4s\n" - "sqxtn v23.4h, v23.4s\n" - "sqxtn2 v14.8h, v11.4s\n" - "sqxtn2 v16.8h, v9.4s\n" - "sqxtn2 v24.8h, v17.4s\n" - "sqxtn2 v23.8h, v25.4s\n" - "sqadd v14.8h, v14.8h, v20.8h\n" - "sqadd v16.8h, v16.8h, v20.8h\n" - "sqadd v24.8h, v24.8h, v20.8h\n" - "sqadd v23.8h, v23.8h, v20.8h\n" - "smax v14.8h, v14.8h, v15.8h\n" - "smax v16.8h, v16.8h, v15.8h\n" - "smax v24.8h, v24.8h, v15.8h\n" - "smax v23.8h, v23.8h, v15.8h\n" - "smin v14.8h, v14.8h, v13.8h\n" - "smin v16.8h, v16.8h, v13.8h\n" - "smin v24.8h, v24.8h, v13.8h\n" - "smin v23.8h, v23.8h, v13.8h\n" - "uzp1 v14.16b, v14.16b, v14.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "uzp1 v24.16b, v24.16b, v24.16b\n" - "uzp1 v23.16b, v23.16b, v23.16b\n" - "tbz x7, #2, 61f\n" - "st1 { v14.s }[0], [x11], #0x4\n" - "st1 { v16.s }[0], [x10], #0x4\n" - "st1 { v24.s }[0], [x9], #0x4\n" - "st1 { v23.s }[0], [x28], #0x4\n" - "tbz x7, #1, 60f\n" - "st1 { v14.h }[2], [x11], #0x2\n" - "st1 { v16.h }[2], [x10], #0x2\n" - "st1 { v24.h }[2], [x9], #0x2\n" - "st1 { v23.h }[2], [x28], #0x2\n" - "tbz x7, #0, 63f\n" - "st1 { v14.b }[6], [x11], #0x1\n" - "st1 { v16.b }[6], [x10], #0x1\n" - "st1 { v24.b }[6], [x9], #0x1\n" - "st1 { v23.b }[6], [x28], #0x1\n" + "and v17.16b, v30.16b, v27.16b\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "and v8.16b, v22.16b, v27.16b\n" + "sshr v26.4s, v26.4s, #0x1f\n" + "and v13.16b, v2.16b, v27.16b\n" + "sqadd v3.4s, v3.4s, v19.4s\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqadd v0.4s, v0.4s, v29.4s\n" + "sshr v8.4s, v8.4s, #0x1f\n" + "sqadd v6.4s, v6.4s, v26.4s\n" + "sshr v13.4s, v13.4s, #0x1f\n" + "srshl v28.4s, v28.4s, v23.4s\n" + "srshl v3.4s, v3.4s, v23.4s\n" + "sqadd v30.4s, v30.4s, v17.4s\n" + "srshl v0.4s, v0.4s, v23.4s\n" + "sqadd v22.4s, v22.4s, v8.4s\n" + "srshl v6.4s, v6.4s, v23.4s\n" + "sqadd v2.4s, v2.4s, v13.4s\n" + "srshl v9.4s, v9.4s, v27.4s\n" + "sqxtn v28.4h, v28.4s\n" + "srshl v30.4s, v30.4s, v27.4s\n" + "sqxtn v3.4h, v3.4s\n" + "srshl v22.4s, v22.4s, v27.4s\n" + "sqxtn v0.4h, v0.4s\n" + "srshl v2.4s, v2.4s, v27.4s\n" + "sqxtn v6.4h, v6.4s\n" + "sqxtn2 v28.8h, v9.4s\n" + "sqxtn2 v3.8h, v30.4s\n" + "sqxtn2 v0.8h, v22.4s\n" + "sqxtn2 v6.8h, v2.4s\n" + "sqadd v28.8h, v28.8h, v5.8h\n" + "sqadd v3.8h, v3.8h, v5.8h\n" + "sqadd v0.8h, v0.8h, v5.8h\n" + "sqadd v6.8h, v6.8h, v5.8h\n" + "smax v28.8h, v28.8h, v14.8h\n" + "smax v3.8h, v3.8h, v14.8h\n" + "smax v0.8h, v0.8h, v14.8h\n" + "smax v6.8h, v6.8h, v14.8h\n" + "smin v28.8h, v28.8h, v12.8h\n" + "smin v3.8h, v3.8h, v12.8h\n" + "smin v0.8h, v0.8h, v12.8h\n" + "smin v6.8h, v6.8h, v12.8h\n" + "uzp1 v28.16b, v28.16b, v28.16b\n" + "uzp1 v3.16b, v3.16b, v3.16b\n" + "uzp1 v0.16b, v0.16b, v0.16b\n" + "uzp1 v6.16b, v6.16b, v6.16b\n" + "tbz x16, #2, 61f\n" + "st1 { v28.s }[0], [x28], #0x4\n" + "st1 { v3.s }[0], [x27], #0x4\n" + "st1 { v0.s }[0], [x26], #0x4\n" + "st1 { v6.s }[0], [x25], #0x4\n" + "tbz x16, #1, 60f\n" + "st1 { v28.h }[2], [x28], #0x2\n" + "st1 { v3.h }[2], [x27], #0x2\n" + "st1 { v0.h }[2], [x26], #0x2\n" + "st1 { v6.h }[2], [x25], #0x2\n" + "tbz x16, #0, 63f\n" + "st1 { v28.b }[6], [x28], #0x1\n" + "st1 { v3.b }[6], [x27], #0x1\n" + "st1 { v0.b }[6], [x26], #0x1\n" + "st1 { v6.b }[6], [x25], #0x1\n" "b 63f\n" "60:" // Oddments: Bit 2: Bit 1: Unset - "tbz x7, #0, 63f\n" - "st1 { v14.b }[4], [x11], #0x1\n" - "st1 { v16.b }[4], [x10], #0x1\n" - "st1 { v24.b }[4], [x9], #0x1\n" - "st1 { v23.b }[4], [x28], #0x1\n" + "tbz x16, #0, 63f\n" + "st1 { v28.b }[4], [x28], #0x1\n" + "st1 { v3.b }[4], [x27], #0x1\n" + "st1 { v0.b }[4], [x26], #0x1\n" + "st1 { v6.b }[4], [x25], #0x1\n" "b 63f\n" "61:" // Oddments: Bit 2: Unset - "tbz x7, #1, 62f\n" - "st1 { v14.h }[0], [x11], #0x2\n" - "st1 { v16.h }[0], [x10], #0x2\n" - "st1 { v24.h }[0], [x9], #0x2\n" - "st1 { v23.h }[0], [x28], #0x2\n" - "tbz x7, #0, 63f\n" - "st1 { v14.b }[2], [x11], #0x1\n" - "st1 { v16.b }[2], [x10], #0x1\n" - "st1 { v24.b }[2], [x9], #0x1\n" - "st1 { v23.b }[2], [x28], #0x1\n" + "tbz x16, #1, 62f\n" + "st1 { v28.h }[0], [x28], #0x2\n" + "st1 { v3.h }[0], [x27], #0x2\n" + "st1 { v0.h }[0], [x26], #0x2\n" + "st1 { v6.h }[0], [x25], #0x2\n" + "tbz x16, #0, 63f\n" + "st1 { v28.b }[2], [x28], #0x1\n" + "st1 { v3.b }[2], [x27], #0x1\n" + "st1 { v0.b }[2], [x26], #0x1\n" + "st1 { v6.b }[2], [x25], #0x1\n" "b 63f\n" "62:" // Oddments: Bit 2: Unset: Bit 1: Unset - "tbz x7, #0, 63f\n" - "st1 { v14.b }[0], [x11], #0x1\n" - "st1 { v16.b }[0], [x10], #0x1\n" - "st1 { v24.b }[0], [x9], #0x1\n" - "st1 { v23.b }[0], [x28], #0x1\n" + "tbz x16, #0, 63f\n" + "st1 { v28.b }[0], [x28], #0x1\n" + "st1 { v3.b }[0], [x27], #0x1\n" + "st1 { v0.b }[0], [x26], #0x1\n" + "st1 { v6.b }[0], [x25], #0x1\n" "63:" // Oddments: Bit 2: End "64:" // End : : [offsetof_Params_bias] "I" (offsetof(Params, bias)), [offsetof_Params_inptrs] "I" (offsetof(Params, inptrs)), [offsetof_Params_n_channels] "I" (offsetof(Params, n_channels)), [offsetof_Params_outptrs] "I" (offsetof(Params, outptrs)), [offsetof_Params_requant] "I" (offsetof(Params, requant)), [offsetof_Params_requant_muls] "I" (offsetof(Params, requant_muls)), [offsetof_Params_requant_shifts] "I" (offsetof(Params, requant_shifts)), [offsetof_Params_weights] "I" (offsetof(Params, weights)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [params] "r" (¶ms) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp index b27e8687e0591b9fcc02f9f84b4dccd16fafe554..50778e9cbb7fb38e76e23c3993c1d57ab0ca40b3 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp @@ -22,8 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" - +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include @@ -35,15 +34,7 @@ namespace arm_conv { namespace depthwise { -void a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( - const unsigned int, - const uint8_t *const *const, - const uint8_t *const, - const int32_t *const, - const arm_gemm::Requantize32 &, - const int32_t *const, - const int32_t *const, - uint8_t *const *const); +void a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst_impl(unsigned int, const uint8_t *const *, const uint8_t *, const int32_t *, const arm_gemm::Requantize32 &, const int32_t *, const int32_t *, uint8_t *const *); class a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst : public DepthwiseDepthfirstStrategy { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp index a1e5c669b72f599c12ddaf7d0b885da1c698da84..c807cb3adeeeeb07e370ccae6e4ef79a5f451f1b 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp @@ -104,15 +104,15 @@ void a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ldr x23, [%x[params], %[offsetof_Params_requant]]\n" "lsr x8, x7, #0x3\n" "add x20, x23, %[offsetof_Requantize32_b_offset]\n" - "ld1r { v19.16b }, [x20]\n" + "ld1r { v6.16b }, [x20]\n" "ldr x22, [%x[params], %[offsetof_Params_outptrs]]\n" "add x21, x23, %[offsetof_Requantize32_c_offset]\n" "add x20, x23, %[offsetof_Requantize32_minval]\n" - "ld1r { v12.8h }, [x21]\n" - "ld1r { v14.8h }, [x20]\n" + "ld1r { v22.8h }, [x21]\n" + "ld1r { v13.8h }, [x20]\n" "add x20, x23, %[offsetof_Requantize32_maxval]\n" "mov x17, #0x0\n" - "ld1r { v23.8h }, [x20]\n" + "ld1r { v5.8h }, [x20]\n" "mov x16, #0x0\n" "add x15, %x[params], %[offsetof_Params_inptrs]\n" "ldr x14, [%x[params], %[offsetof_Params_weights]]\n" @@ -121,563 +121,563 @@ void a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ldp x11, x10, [x22, #0x0]\n" "ldp x9, x28, [x22, #0x10]\n" "cbz x8, 3f\n" - "ldr d0, [x14, #0x0]\n" - "ldr d1, [x14, #0x8]\n" + "ldr d12, [x14, #0x0]\n" + "ldr d11, [x14, #0x8]\n" "subs x8, x8, #0x1\n" - "usubl v0.8h, v0.8b, v19.8b\n" - "ldr d2, [x14, #0x10]\n" - "ldr d3, [x14, #0x18]\n" - "usubl v1.8h, v1.8b, v19.8b\n" - "usubl v2.8h, v2.8b, v19.8b\n" - "ldr d4, [x14, #0x20]\n" - "ldr d5, [x14, #0x28]\n" - "usubl v3.8h, v3.8b, v19.8b\n" - "usubl v4.8h, v4.8b, v19.8b\n" - "ldr d6, [x14, #0x30]\n" - "ldr d7, [x14, #0x38]\n" - "usubl v5.8h, v5.8b, v19.8b\n" - "usubl v6.8h, v6.8b, v19.8b\n" - "ldr d8, [x14, #0x40]\n" - "ldr x22, [%x[params], %[offsetof_Params_bias]]\n" - "usubl v7.8h, v7.8b, v19.8b\n" - "usubl v8.8h, v8.8b, v19.8b\n" - "ldr q15, [x22, #0x0]\n" - "ldr q13, [x22, #0x10]\n" - "add x22, x22, #0x20\n" - "str x22, [%x[params], %[offsetof_Params_bias]]\n" + "usubl v12.8h, v12.8b, v6.8b\n" + "ldr d25, [x14, #0x10]\n" + "ldr d24, [x14, #0x18]\n" + "usubl v11.8h, v11.8b, v6.8b\n" + "usubl v25.8h, v25.8b, v6.8b\n" + "ldr d23, [x14, #0x20]\n" + "ldr d7, [x14, #0x28]\n" + "usubl v24.8h, v24.8b, v6.8b\n" + "usubl v23.8h, v23.8b, v6.8b\n" + "ldr d3, [x14, #0x30]\n" + "ldr d9, [x14, #0x38]\n" + "usubl v7.8h, v7.8b, v6.8b\n" + "usubl v3.8h, v3.8b, v6.8b\n" + "ldr d30, [x14, #0x40]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "usubl v9.8h, v9.8b, v6.8b\n" + "usubl v30.8h, v30.8b, v6.8b\n" + "ldr q8, [x20, #0x0]\n" + "ldr q2, [x20, #0x10]\n" + "add x20, x20, #0x20\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" "ldp x27, x26, [x15, #0x0]\n" "ldp x25, x24, [x15, #0x10]\n" - "mov v17.16b, v15.16b\n" - "mov v20.16b, v13.16b\n" + "mov v21.16b, v8.16b\n" + "mov v4.16b, v2.16b\n" "ldp x23, x22, [x15, #0x20]\n" "ldp x21, x20, [x15, #0x30]\n" - "mov v11.16b, v15.16b\n" - "mov v10.16b, v13.16b\n" - "ldr d31, [x27, x17]\n" - "ldr d30, [x26, x17]\n" - "mov v9.16b, v15.16b\n" - "mov v22.16b, v13.16b\n" - "ldr d29, [x25, x17]\n" - "ldr d28, [x24, x17]\n" - "ushll v31.8h, v31.8b, #0x0\n" - "ushll v30.8h, v30.8b, #0x0\n" - "ldr d27, [x23, x17]\n" - "ldr d26, [x22, x17]\n" - "ushll v29.8h, v29.8b, #0x0\n" - "ushll v28.8h, v28.8b, #0x0\n" - "ldr d25, [x21, x17]\n" - "ldr d24, [x20, x17]\n" - "ushll v27.8h, v27.8b, #0x0\n" + "mov v20.16b, v8.16b\n" + "mov v1.16b, v2.16b\n" + "ldr d26, [x27, x17]\n" + "ldr d18, [x26, x17]\n" + "mov v16.16b, v8.16b\n" + "mov v14.16b, v2.16b\n" + "ldr d10, [x25, x17]\n" + "ldr d27, [x24, x17]\n" "ushll v26.8h, v26.8b, #0x0\n" - "ushll v25.8h, v25.8b, #0x0\n" - "ushll v24.8h, v24.8b, #0x0\n" + "ushll v18.8h, v18.8b, #0x0\n" + "ldr d17, [x23, x17]\n" + "ldr d19, [x22, x17]\n" + "ushll v10.8h, v10.8b, #0x0\n" + "ushll v27.8h, v27.8b, #0x0\n" + "ldr d15, [x21, x17]\n" + "ldr d28, [x20, x17]\n" + "ushll v17.8h, v17.8b, #0x0\n" + "ushll v19.8h, v19.8b, #0x0\n" + "ushll v15.8h, v15.8b, #0x0\n" + "ushll v28.8h, v28.8b, #0x0\n" "beq 2f\n" "1:" // Loop - "ldr q18, [x13, #0x0]\n" - "smlal v15.4s, v31.4h, v8.4h\n" - "smlal2 v13.4s, v31.8h, v8.8h\n" - "ldr x23, [x15, #0x40]\n" - "smlal v17.4s, v31.4h, v6.4h\n" - "smlal2 v20.4s, v31.8h, v6.8h\n" - "ldr x22, [x15, #0x48]\n" - "ldr x21, [x15, #0x50]\n" - "smlal v15.4s, v30.4h, v0.4h\n" - "smlal2 v13.4s, v30.8h, v0.8h\n" - "ldr q21, [x12, #0x0]\n" - "ldr x20, [x15, #0x58]\n" - "smlal v17.4s, v28.4h, v1.4h\n" - "smlal2 v20.4s, v28.8h, v1.8h\n" - "ldr d28, [x22, x17]\n" - "ushll v28.8h, v28.8b, #0x0\n" - "smlal v15.4s, v29.4h, v1.4h\n" - "smlal2 v13.4s, v29.8h, v1.8h\n" - "ldr d29, [x23, x17]\n" - "ushll v29.8h, v29.8b, #0x0\n" - "smlal v17.4s, v27.4h, v2.4h\n" - "smlal2 v20.4s, v27.8h, v2.8h\n" - "ldr d27, [x21, x17]\n" - "ushll v27.8h, v27.8b, #0x0\n" - "smlal v15.4s, v26.4h, v3.4h\n" - "smlal2 v13.4s, v26.8h, v3.8h\n" + "ldr q31, [x13, #0x0]\n" + "ldr q0, [x12, #0x0]\n" + "smlal v8.4s, v26.4h, v30.4h\n" + "smlal2 v2.4s, v26.8h, v30.8h\n" + "ldr q29, [x13, #0x10]\n" + "ldr x21, [x15, #0x58]\n" + "smlal v8.4s, v18.4h, v12.4h\n" + "smlal v21.4s, v26.4h, v3.4h\n" + "ldr x20, [x15, #0x78]\n" + "ldr x25, [x15, #0x60]\n" + "smlal v20.4s, v26.4h, v25.4h\n" + "smlal v16.4s, v26.4h, v12.4h\n" + "ldr x24, [x15, #0x80]\n" + "smlal2 v2.4s, v18.8h, v12.8h\n" + "ldr d18, [x21, x17]\n" + "ushll v18.8h, v18.8b, #0x0\n" + "smlal v8.4s, v10.4h, v11.4h\n" + "smlal2 v4.4s, v26.8h, v3.8h\n" + "ldr x23, [x15, #0x68]\n" + "ldr x22, [x15, #0x88]\n" + "smlal2 v1.4s, v26.8h, v25.8h\n" + "smlal2 v14.4s, v26.8h, v12.8h\n" "ldr d26, [x20, x17]\n" - "ldr x23, [x15, #0x78]\n" - "smlal v17.4s, v24.4h, v0.4h\n" - "smlal2 v20.4s, v24.8h, v0.8h\n" - "ldr x20, [x15, #0x60]\n" "ushll v26.8h, v26.8b, #0x0\n" - "smlal v15.4s, v25.4h, v4.4h\n" - "smlal2 v13.4s, v25.8h, v4.8h\n" - "ldr d25, [x20, x17]\n" - "ldr x21, [x15, #0x80]\n" - "smlal v17.4s, v29.4h, v4.4h\n" - "smlal2 v20.4s, v29.8h, v4.8h\n" - "ldr q30, [x13, #0x10]\n" - "ldr x20, [x15, #0x68]\n" - "smlal v15.4s, v24.4h, v2.4h\n" - "smlal2 v13.4s, v24.8h, v2.8h\n" - "ldr d29, [x20, x17]\n" - "ushll v25.8h, v25.8b, #0x0\n" - "smlal v17.4s, v28.4h, v5.4h\n" - "smlal2 v20.4s, v28.8h, v5.8h\n" - "ldr d28, [x21, x17]\n" - "ldr x22, [x15, #0x88]\n" - "smlal v11.4s, v31.4h, v2.4h\n" - "smlal2 v10.4s, v31.8h, v2.8h\n" + "smlal v21.4s, v27.4h, v11.4h\n" + "smlal v20.4s, v18.4h, v24.4h\n" + "ldr x21, [x15, #0x40]\n" "ldr x20, [x15, #0x70]\n" - "ushll v28.8h, v28.8b, #0x0\n" - "smlal v15.4s, v27.4h, v5.4h\n" - "smlal2 v13.4s, v27.8h, v5.8h\n" - "ushll v29.8h, v29.8b, #0x0\n" - "ldr x24, [x15, #0x98]\n" - "smlal v17.4s, v27.4h, v3.4h\n" - "smlal2 v20.4s, v27.8h, v3.8h\n" - "ldr d27, [x23, x17]\n" + "smlal v16.4s, v26.4h, v23.4h\n" + "smlal2 v2.4s, v10.8h, v11.8h\n" + "ldr d10, [x25, x17]\n" + "ushll v10.8h, v10.8b, #0x0\n" + "smlal v8.4s, v19.4h, v24.4h\n" + "smlal2 v4.4s, v27.8h, v11.8h\n" + "ldr d27, [x24, x17]\n" "ushll v27.8h, v27.8b, #0x0\n" - "smlal v9.4s, v31.4h, v0.4h\n" - "smlal v11.4s, v26.4h, v3.4h\n" - "ldr x21, [x15, #0x90]\n" - "ldr x23, [x15, #0xa8]\n" - "smlal2 v10.4s, v26.8h, v3.8h\n" + "smlal2 v1.4s, v18.8h, v24.8h\n" + "ldr d18, [x23, x17]\n" + "smlal2 v14.4s, v26.8h, v23.8h\n" "ldr d26, [x22, x17]\n" - "smlal2 v22.4s, v31.8h, v0.8h\n" - "ldr d24, [x20, x17]\n" - "smlal v9.4s, v27.4h, v4.4h\n" - "smlal v11.4s, v25.4h, v0.4h\n" + "ldr x24, [x15, #0x98]\n" + "smlal v21.4s, v17.4h, v25.4h\n" + "smlal v20.4s, v10.4h, v12.4h\n" + "ldr x23, [x15, #0x50]\n" + "smlal v16.4s, v27.4h, v11.4h\n" + "smlal2 v2.4s, v19.8h, v24.8h\n" + "ldr d19, [x21, x17]\n" + "ushll v18.8h, v18.8b, #0x0\n" "ushll v26.8h, v26.8b, #0x0\n" - "ldr x20, [x15, #0xa0]\n" - "smlal2 v10.4s, v25.8h, v0.8h\n" - "ldr q31, [x12, #0x10]\n" - "smlal2 v22.4s, v27.8h, v4.8h\n" - "ldr d27, [x20, x17]\n" - "smlal v9.4s, v28.4h, v1.4h\n" - "smlal v15.4s, v25.4h, v6.4h\n" - "ushll v24.8h, v24.8b, #0x0\n" - "ldr x22, [x15, #0xb0]\n" - "smlal2 v13.4s, v25.8h, v6.8h\n" - "ldr d25, [x21, x17]\n" - "smlal v11.4s, v29.4h, v4.4h\n" - "ushll v25.8h, v25.8b, #0x0\n" - "smlal2 v10.4s, v29.8h, v4.8h\n" - "ldr d29, [x24, x17]\n" - "smlal2 v22.4s, v28.8h, v1.8h\n" - "ushll v29.8h, v29.8b, #0x0\n" - "smlal v9.4s, v26.4h, v5.4h\n" - "smlal v15.4s, v24.4h, v7.4h\n" - "ldr x21, [x15, #0xb8]\n" - "ushll v27.8h, v27.8b, #0x0\n" - "smlal2 v13.4s, v24.8h, v7.8h\n" - "smlal v11.4s, v24.4h, v1.4h\n" - "ldr x20, [x15, #0xc0]\n" - "sqrdmulh v15.4s, v15.4s, v18.4s\n" - "smlal2 v10.4s, v24.8h, v1.8h\n" - "ldr d24, [x23, x17]\n" - "smlal2 v22.4s, v26.8h, v5.8h\n" - "ldr d26, [x22, x17]\n" - "smlal v9.4s, v29.4h, v2.4h\n" - "ushll v24.8h, v24.8b, #0x0\n" - "smlal2 v22.4s, v29.8h, v2.8h\n" - "ldr x22, [%x[params], %[offsetof_Params_bias]]\n" - "smlal v11.4s, v25.4h, v6.4h\n" - "smlal v9.4s, v24.4h, v3.4h\n" + "smlal v8.4s, v15.4h, v23.4h\n" + "smlal2 v4.4s, v17.8h, v25.8h\n" + "ldr d17, [x20, x17]\n" + "ldr x22, [x15, #0x48]\n" + "smlal2 v1.4s, v10.8h, v12.8h\n" + "smlal2 v14.4s, v27.8h, v11.8h\n" + "ldr x21, [x15, #0x90]\n" + "ldr x20, [x15, #0xa8]\n" + "smlal v21.4s, v28.4h, v12.4h\n" + "smlal v20.4s, v18.4h, v23.4h\n" + "ushll v19.8h, v19.8b, #0x0\n" + "smlal v16.4s, v26.4h, v7.4h\n" + "smlal2 v2.4s, v15.8h, v23.8h\n" + "ldr d15, [x24, x17]\n" + "ushll v17.8h, v17.8b, #0x0\n" + "ushll v15.8h, v15.8b, #0x0\n" + "smlal v8.4s, v28.4h, v25.4h\n" + "smlal2 v4.4s, v28.8h, v12.8h\n" + "ldr d12, [x23, x17]\n" + "ushll v12.8h, v12.8b, #0x0\n" + "smlal2 v1.4s, v18.8h, v23.8h\n" + "ldr d18, [x22, x17]\n" + "smlal2 v14.4s, v26.8h, v7.8h\n" + "ldr d26, [x21, x17]\n" + "ldr x22, [x15, #0xa0]\n" + "smlal v21.4s, v19.4h, v23.4h\n" + "smlal v20.4s, v17.4h, v11.4h\n" + "ldr x21, [x15, #0xb0]\n" + "smlal v16.4s, v15.4h, v25.4h\n" + "smlal2 v2.4s, v28.8h, v25.8h\n" + "ldr d28, [x20, x17]\n" + "ushll v18.8h, v18.8b, #0x0\n" "ushll v26.8h, v26.8b, #0x0\n" + "smlal v8.4s, v12.4h, v7.4h\n" + "ldr x20, [x15, #0xb8]\n" + "ushll v28.8h, v28.8b, #0x0\n" + "smlal2 v4.4s, v19.8h, v23.8h\n" + "ldr d23, [x22, x17]\n" + "ldr d19, [x21, x17]\n" + "smlal2 v1.4s, v17.8h, v11.8h\n" + "ldr d11, [x20, x17]\n" + "smlal2 v14.4s, v15.8h, v25.8h\n" + "ldr q25, [x12, #0x10]\n" + "smlal v21.4s, v18.4h, v7.4h\n" + "smlal v20.4s, v26.4h, v3.4h\n" + "ushll v23.8h, v23.8b, #0x0\n" + "ldr x21, [x15, #0xc0]\n" + "smlal v16.4s, v28.4h, v24.4h\n" + "smlal2 v2.4s, v12.8h, v7.8h\n" + "ushll v19.8h, v19.8b, #0x0\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "smlal v8.4s, v10.4h, v3.4h\n" + "smlal2 v4.4s, v18.8h, v7.8h\n" + "ldr d18, [x21, x17]\n" + "ushll v11.8h, v11.8b, #0x0\n" + "smlal2 v1.4s, v26.8h, v3.8h\n" + "smlal2 v14.4s, v28.8h, v24.8h\n" + "ushll v18.8h, v18.8b, #0x0\n" "add x14, x14, #0x48\n" - "smlal v17.4s, v28.4h, v7.4h\n" - "smlal2 v20.4s, v28.8h, v7.8h\n" - "and v2.16b, v15.16b, v21.16b\n" + "smlal v21.4s, v12.4h, v24.4h\n" + "smlal v20.4s, v23.4h, v9.4h\n" + "add x17, x17, #0x8\n" "subs x8, x8, #0x1\n" - "smlal2 v10.4s, v25.8h, v6.8h\n" - "ldr d25, [x21, x17]\n" - "smlal2 v22.4s, v24.8h, v3.8h\n" - "ushll v25.8h, v25.8b, #0x0\n" - "smlal v11.4s, v27.4h, v7.4h\n" - "smlal v9.4s, v26.4h, v7.4h\n" - "sqrdmulh v13.4s, v13.4s, v30.4s\n" + "smlal v16.4s, v19.4h, v9.4h\n" + "smlal2 v2.4s, v10.8h, v3.8h\n" "add x13, x13, #0x20\n" - "smlal v17.4s, v29.4h, v8.4h\n" - "smlal2 v20.4s, v29.8h, v8.8h\n" - "ldr d29, [x20, x17]\n" - "ushll v29.8h, v29.8b, #0x0\n" - "smlal2 v10.4s, v27.8h, v7.8h\n" - "smlal2 v22.4s, v26.8h, v7.8h\n" - "sshr v2.4s, v2.4s, #0x1f\n" - "add x17, x17, #0x8\n" - "smlal v11.4s, v24.4h, v5.4h\n" - "smlal v9.4s, v25.4h, v6.4h\n" - "and v16.16b, v13.16b, v31.16b\n" "add x12, x12, #0x20\n" - "smlal2 v10.4s, v24.8h, v5.8h\n" - "smlal2 v22.4s, v25.8h, v6.8h\n" - "sqrdmulh v17.4s, v17.4s, v18.4s\n" - "smlal v11.4s, v25.4h, v8.4h\n" - "smlal v9.4s, v29.4h, v8.4h\n" - "sqrdmulh v11.4s, v11.4s, v18.4s\n" - "smlal2 v10.4s, v25.8h, v8.8h\n" - "smlal2 v22.4s, v29.8h, v8.8h\n" - "sqrdmulh v9.4s, v9.4s, v18.4s\n" - "sqadd v15.4s, v15.4s, v2.4s\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "and v18.16b, v17.16b, v21.16b\n" - "sqrdmulh v20.4s, v20.4s, v30.4s\n" - "and v28.16b, v11.16b, v21.16b\n" - "sqrdmulh v10.4s, v10.4s, v30.4s\n" - "and v2.16b, v9.16b, v21.16b\n" - "sqrdmulh v22.4s, v22.4s, v30.4s\n" - "sqadd v13.4s, v13.4s, v16.4s\n" + "smlal v8.4s, v17.4h, v9.4h\n" + "smlal2 v4.4s, v12.8h, v24.8h\n" + "sqrdmulh v8.4s, v8.4s, v31.4s\n" + "smlal2 v1.4s, v23.8h, v9.8h\n" + "smlal2 v14.4s, v19.8h, v9.8h\n" + "and v10.16b, v8.16b, v0.16b\n" + "smlal v21.4s, v27.4h, v9.4h\n" + "smlal v20.4s, v28.4h, v7.4h\n" + "sshr v10.4s, v10.4s, #0x1f\n" + "smlal v16.4s, v11.4h, v3.4h\n" + "smlal2 v2.4s, v17.8h, v9.8h\n" + "sqrdmulh v2.4s, v2.4s, v29.4s\n" + "smlal2 v4.4s, v27.8h, v9.8h\n" + "smlal2 v1.4s, v28.8h, v7.8h\n" + "and v12.16b, v2.16b, v25.16b\n" + "smlal2 v14.4s, v11.8h, v3.8h\n" + "smlal v21.4s, v15.4h, v30.4h\n" + "sqrdmulh v21.4s, v21.4s, v31.4s\n" + "smlal v20.4s, v11.4h, v30.4h\n" + "smlal v16.4s, v18.4h, v30.4h\n" + "sqrdmulh v20.4s, v20.4s, v31.4s\n" + "smlal2 v4.4s, v15.8h, v30.8h\n" + "smlal2 v1.4s, v11.8h, v30.8h\n" + "sqrdmulh v16.4s, v16.4s, v31.4s\n" + "smlal2 v14.4s, v18.8h, v30.8h\n" + "sqadd v8.4s, v8.4s, v10.4s\n" + "sshr v12.4s, v12.4s, #0x1f\n" + "and v27.16b, v21.16b, v0.16b\n" + "sqrdmulh v4.4s, v4.4s, v29.4s\n" + "and v24.16b, v20.16b, v0.16b\n" + "sqrdmulh v1.4s, v1.4s, v29.4s\n" + "and v19.16b, v16.16b, v0.16b\n" + "sqrdmulh v14.4s, v14.4s, v29.4s\n" + "sqadd v2.4s, v2.4s, v12.4s\n" + "sshr v27.4s, v27.4s, #0x1f\n" + "and v18.16b, v4.16b, v25.16b\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "and v17.16b, v1.16b, v25.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "and v15.16b, v14.16b, v25.16b\n" + "sqadd v21.4s, v21.4s, v27.4s\n" "sshr v18.4s, v18.4s, #0x1f\n" - "and v4.16b, v20.16b, v31.16b\n" - "sshr v28.4s, v28.4s, #0x1f\n" - "and v3.16b, v10.16b, v31.16b\n" - "sshr v2.4s, v2.4s, #0x1f\n" - "and v16.16b, v22.16b, v31.16b\n" - "sqadd v17.4s, v17.4s, v18.4s\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v11.4s, v11.4s, v28.4s\n" - "sshr v3.4s, v3.4s, #0x1f\n" - "sqadd v9.4s, v9.4s, v2.4s\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "srshl v15.4s, v15.4s, v21.4s\n" - "srshl v17.4s, v17.4s, v21.4s\n" - "sqadd v20.4s, v20.4s, v4.4s\n" - "srshl v11.4s, v11.4s, v21.4s\n" - "sqadd v10.4s, v10.4s, v3.4s\n" - "srshl v9.4s, v9.4s, v21.4s\n" - "sqadd v22.4s, v22.4s, v16.4s\n" - "srshl v13.4s, v13.4s, v31.4s\n" - "sqxtn v15.4h, v15.4s\n" - "srshl v20.4s, v20.4s, v31.4s\n" - "sqxtn v17.4h, v17.4s\n" - "srshl v10.4s, v10.4s, v31.4s\n" - "sqxtn v11.4h, v11.4s\n" - "srshl v22.4s, v22.4s, v31.4s\n" - "sqxtn v9.4h, v9.4s\n" - "sqxtn2 v15.8h, v13.4s\n" - "sqxtn2 v17.8h, v20.4s\n" - "sqxtn2 v11.8h, v10.4s\n" - "sqxtn2 v9.8h, v22.4s\n" - "sqadd v15.8h, v15.8h, v12.8h\n" - "sqadd v17.8h, v17.8h, v12.8h\n" - "sqadd v11.8h, v11.8h, v12.8h\n" - "sqadd v9.8h, v9.8h, v12.8h\n" - "smax v15.8h, v15.8h, v14.8h\n" - "smax v17.8h, v17.8h, v14.8h\n" - "smax v11.8h, v11.8h, v14.8h\n" - "smax v9.8h, v9.8h, v14.8h\n" - "smin v15.8h, v15.8h, v23.8h\n" - "smin v17.8h, v17.8h, v23.8h\n" - "smin v11.8h, v11.8h, v23.8h\n" - "smin v9.8h, v9.8h, v23.8h\n" - "uzp1 v15.16b, v15.16b, v15.16b\n" - "uzp1 v17.16b, v17.16b, v17.16b\n" - "str d15, [x11, x16]\n" - "uzp1 v11.16b, v11.16b, v11.16b\n" - "uzp1 v9.16b, v9.16b, v9.16b\n" - "str d17, [x10, x16]\n" - "str d11, [x9, x16]\n" - "str d9, [x28, x16]\n" - "ldr q15, [x22, #0x0]\n" - "ldr q13, [x22, #0x10]\n" - "add x22, x22, #0x20\n" - "ldr d0, [x14, #0x0]\n" - "ldr d1, [x14, #0x8]\n" + "sqadd v20.4s, v20.4s, v24.4s\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v19.4s\n" + "sshr v15.4s, v15.4s, #0x1f\n" + "srshl v8.4s, v8.4s, v0.4s\n" + "srshl v21.4s, v21.4s, v0.4s\n" + "sqadd v4.4s, v4.4s, v18.4s\n" + "srshl v20.4s, v20.4s, v0.4s\n" + "sqadd v1.4s, v1.4s, v17.4s\n" + "srshl v16.4s, v16.4s, v0.4s\n" + "sqadd v14.4s, v14.4s, v15.4s\n" + "srshl v2.4s, v2.4s, v25.4s\n" + "sqxtn v8.4h, v8.4s\n" + "srshl v4.4s, v4.4s, v25.4s\n" + "sqxtn v21.4h, v21.4s\n" + "srshl v1.4s, v1.4s, v25.4s\n" + "sqxtn v20.4h, v20.4s\n" + "srshl v14.4s, v14.4s, v25.4s\n" + "sqxtn v16.4h, v16.4s\n" + "sqxtn2 v8.8h, v2.4s\n" + "sqxtn2 v21.8h, v4.4s\n" + "sqxtn2 v20.8h, v1.4s\n" + "sqxtn2 v16.8h, v14.4s\n" + "sqadd v8.8h, v8.8h, v22.8h\n" + "sqadd v21.8h, v21.8h, v22.8h\n" + "sqadd v20.8h, v20.8h, v22.8h\n" + "sqadd v16.8h, v16.8h, v22.8h\n" + "smax v8.8h, v8.8h, v13.8h\n" + "smax v21.8h, v21.8h, v13.8h\n" + "smax v20.8h, v20.8h, v13.8h\n" + "smax v16.8h, v16.8h, v13.8h\n" + "smin v8.8h, v8.8h, v5.8h\n" + "smin v21.8h, v21.8h, v5.8h\n" + "smin v20.8h, v20.8h, v5.8h\n" + "smin v16.8h, v16.8h, v5.8h\n" + "uzp1 v8.16b, v8.16b, v8.16b\n" + "str d8, [x11, x16]\n" + "uzp1 v21.16b, v21.16b, v21.16b\n" + "uzp1 v20.16b, v20.16b, v20.16b\n" + "str d21, [x10, x16]\n" + "uzp1 v16.16b, v16.16b, v16.16b\n" + "str d20, [x9, x16]\n" + "str d16, [x28, x16]\n" + "ldr q8, [x20, #0x0]\n" + "ldr q2, [x20, #0x10]\n" + "add x20, x20, #0x20\n" + "ldr d12, [x14, #0x0]\n" + "ldr d11, [x14, #0x8]\n" "add x16, x16, #0x8\n" - "str x22, [%x[params], %[offsetof_Params_bias]]\n" - "ldr d2, [x14, #0x10]\n" - "ldr d3, [x14, #0x18]\n" - "mov v17.16b, v15.16b\n" - "mov v20.16b, v13.16b\n" - "ldr d4, [x14, #0x20]\n" - "ldr d5, [x14, #0x28]\n" - "mov v11.16b, v15.16b\n" - "mov v10.16b, v13.16b\n" - "ldr d6, [x14, #0x30]\n" - "ldr d7, [x14, #0x38]\n" - "mov v9.16b, v15.16b\n" - "mov v22.16b, v13.16b\n" - "ldr d8, [x14, #0x40]\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" + "ldr d25, [x14, #0x10]\n" + "ldr d24, [x14, #0x18]\n" + "mov v21.16b, v8.16b\n" + "mov v4.16b, v2.16b\n" + "ldr d23, [x14, #0x20]\n" + "ldr d7, [x14, #0x28]\n" + "mov v20.16b, v8.16b\n" + "mov v1.16b, v2.16b\n" + "ldr d3, [x14, #0x30]\n" + "ldr d9, [x14, #0x38]\n" + "mov v16.16b, v8.16b\n" + "mov v14.16b, v2.16b\n" + "ldr d30, [x14, #0x40]\n" "ldp x27, x26, [x15, #0x0]\n" - "usubl v0.8h, v0.8b, v19.8b\n" - "usubl v1.8h, v1.8b, v19.8b\n" + "usubl v12.8h, v12.8b, v6.8b\n" + "usubl v11.8h, v11.8b, v6.8b\n" "ldp x25, x24, [x15, #0x10]\n" "ldp x23, x22, [x15, #0x20]\n" - "usubl v2.8h, v2.8b, v19.8b\n" - "usubl v3.8h, v3.8b, v19.8b\n" + "usubl v25.8h, v25.8b, v6.8b\n" + "usubl v24.8h, v24.8b, v6.8b\n" "ldp x21, x20, [x15, #0x30]\n" - "ldr d31, [x27, x17]\n" - "usubl v4.8h, v4.8b, v19.8b\n" - "usubl v5.8h, v5.8b, v19.8b\n" - "ldr d30, [x26, x17]\n" - "ldr d29, [x25, x17]\n" - "usubl v6.8h, v6.8b, v19.8b\n" - "usubl v7.8h, v7.8b, v19.8b\n" - "ldr d28, [x24, x17]\n" - "ldr d27, [x23, x17]\n" - "usubl v8.8h, v8.8b, v19.8b\n" - "ushll v31.8h, v31.8b, #0x0\n" - "ldr d26, [x22, x17]\n" - "ldr d25, [x21, x17]\n" - "ushll v30.8h, v30.8b, #0x0\n" - "ushll v29.8h, v29.8b, #0x0\n" - "ldr d24, [x20, x17]\n" - "ushll v28.8h, v28.8b, #0x0\n" - "ushll v27.8h, v27.8b, #0x0\n" + "ldr d26, [x27, x17]\n" + "usubl v23.8h, v23.8b, v6.8b\n" + "usubl v7.8h, v7.8b, v6.8b\n" + "ldr d18, [x26, x17]\n" + "ldr d10, [x25, x17]\n" + "usubl v3.8h, v3.8b, v6.8b\n" + "usubl v9.8h, v9.8b, v6.8b\n" + "ldr d27, [x24, x17]\n" + "ldr d17, [x23, x17]\n" + "usubl v30.8h, v30.8b, v6.8b\n" "ushll v26.8h, v26.8b, #0x0\n" - "ushll v25.8h, v25.8b, #0x0\n" - "ushll v24.8h, v24.8b, #0x0\n" + "ldr d19, [x22, x17]\n" + "ldr d15, [x21, x17]\n" + "ushll v18.8h, v18.8b, #0x0\n" + "ushll v10.8h, v10.8b, #0x0\n" + "ldr d28, [x20, x17]\n" + "ushll v27.8h, v27.8b, #0x0\n" + "ushll v17.8h, v17.8b, #0x0\n" + "ushll v19.8h, v19.8b, #0x0\n" + "ushll v15.8h, v15.8b, #0x0\n" + "ushll v28.8h, v28.8b, #0x0\n" "bgt 1b\n" "2:" // Tail - "ldr q18, [x13, #0x0]\n" - "smlal v15.4s, v31.4h, v8.4h\n" - "smlal2 v13.4s, v31.8h, v8.8h\n" - "ldr x23, [x15, #0x40]\n" - "smlal v17.4s, v31.4h, v6.4h\n" - "smlal2 v20.4s, v31.8h, v6.8h\n" - "ldr x22, [x15, #0x48]\n" - "ldr x21, [x15, #0x50]\n" - "smlal v15.4s, v30.4h, v0.4h\n" - "smlal2 v13.4s, v30.8h, v0.8h\n" - "ldr q21, [x12, #0x0]\n" - "ldr x20, [x15, #0x58]\n" - "smlal v17.4s, v28.4h, v1.4h\n" - "smlal2 v20.4s, v28.8h, v1.8h\n" - "ldr d28, [x22, x17]\n" - "ushll v28.8h, v28.8b, #0x0\n" - "smlal v15.4s, v29.4h, v1.4h\n" - "smlal2 v13.4s, v29.8h, v1.8h\n" - "ldr d29, [x23, x17]\n" - "ushll v29.8h, v29.8b, #0x0\n" - "smlal v17.4s, v27.4h, v2.4h\n" - "smlal2 v20.4s, v27.8h, v2.8h\n" - "ldr d27, [x21, x17]\n" - "ushll v27.8h, v27.8b, #0x0\n" - "smlal v15.4s, v26.4h, v3.4h\n" - "smlal2 v13.4s, v26.8h, v3.8h\n" + "ldr q0, [x13, #0x0]\n" + "ldr q31, [x12, #0x0]\n" + "smlal v8.4s, v26.4h, v30.4h\n" + "smlal2 v2.4s, v26.8h, v30.8h\n" + "ldr q29, [x13, #0x10]\n" + "ldr x21, [x15, #0x58]\n" + "smlal v8.4s, v18.4h, v12.4h\n" + "smlal v21.4s, v26.4h, v3.4h\n" + "ldr x20, [x15, #0x78]\n" + "ldr x25, [x15, #0x60]\n" + "smlal v20.4s, v26.4h, v25.4h\n" + "smlal v16.4s, v26.4h, v12.4h\n" + "ldr x24, [x15, #0x80]\n" + "smlal2 v2.4s, v18.8h, v12.8h\n" + "ldr d18, [x21, x17]\n" + "ushll v18.8h, v18.8b, #0x0\n" + "smlal v8.4s, v10.4h, v11.4h\n" + "smlal2 v4.4s, v26.8h, v3.8h\n" + "ldr x23, [x15, #0x68]\n" + "ldr x22, [x15, #0x88]\n" + "smlal2 v1.4s, v26.8h, v25.8h\n" + "smlal2 v14.4s, v26.8h, v12.8h\n" "ldr d26, [x20, x17]\n" - "ldr x23, [x15, #0x78]\n" - "smlal v17.4s, v24.4h, v0.4h\n" - "smlal2 v20.4s, v24.8h, v0.8h\n" - "ldr x20, [x15, #0x60]\n" "ushll v26.8h, v26.8b, #0x0\n" - "smlal v15.4s, v25.4h, v4.4h\n" - "smlal2 v13.4s, v25.8h, v4.8h\n" - "ldr d25, [x20, x17]\n" - "ldr x21, [x15, #0x80]\n" - "smlal v17.4s, v29.4h, v4.4h\n" - "smlal2 v20.4s, v29.8h, v4.8h\n" - "ldr q30, [x13, #0x10]\n" - "ldr x20, [x15, #0x68]\n" - "smlal v15.4s, v24.4h, v2.4h\n" - "smlal2 v13.4s, v24.8h, v2.8h\n" - "ldr d29, [x20, x17]\n" - "ushll v25.8h, v25.8b, #0x0\n" - "smlal v17.4s, v28.4h, v5.4h\n" - "smlal2 v20.4s, v28.8h, v5.8h\n" - "ldr d28, [x21, x17]\n" - "ldr x22, [x15, #0x88]\n" - "smlal v11.4s, v31.4h, v2.4h\n" - "smlal2 v10.4s, v31.8h, v2.8h\n" + "smlal v21.4s, v27.4h, v11.4h\n" + "smlal v20.4s, v18.4h, v24.4h\n" + "ldr x21, [x15, #0x40]\n" "ldr x20, [x15, #0x70]\n" - "ushll v28.8h, v28.8b, #0x0\n" - "smlal v15.4s, v27.4h, v5.4h\n" - "smlal2 v13.4s, v27.8h, v5.8h\n" - "ushll v29.8h, v29.8b, #0x0\n" - "ldr x24, [x15, #0x98]\n" - "smlal v17.4s, v27.4h, v3.4h\n" - "smlal2 v20.4s, v27.8h, v3.8h\n" - "ldr d27, [x23, x17]\n" + "smlal v16.4s, v26.4h, v23.4h\n" + "smlal2 v2.4s, v10.8h, v11.8h\n" + "ldr d10, [x25, x17]\n" + "ushll v10.8h, v10.8b, #0x0\n" + "smlal v8.4s, v19.4h, v24.4h\n" + "smlal2 v4.4s, v27.8h, v11.8h\n" + "ldr d27, [x24, x17]\n" "ushll v27.8h, v27.8b, #0x0\n" - "smlal v9.4s, v31.4h, v0.4h\n" - "smlal v11.4s, v26.4h, v3.4h\n" - "ldr x21, [x15, #0x90]\n" - "ldr x23, [x15, #0xa8]\n" - "smlal2 v10.4s, v26.8h, v3.8h\n" + "smlal2 v1.4s, v18.8h, v24.8h\n" + "ldr d18, [x23, x17]\n" + "smlal2 v14.4s, v26.8h, v23.8h\n" "ldr d26, [x22, x17]\n" - "smlal2 v22.4s, v31.8h, v0.8h\n" - "ldr d24, [x20, x17]\n" - "smlal v9.4s, v27.4h, v4.4h\n" - "smlal v11.4s, v25.4h, v0.4h\n" + "ldr x24, [x15, #0x98]\n" + "smlal v21.4s, v17.4h, v25.4h\n" + "smlal v20.4s, v10.4h, v12.4h\n" + "ldr x23, [x15, #0x50]\n" + "smlal v16.4s, v27.4h, v11.4h\n" + "smlal2 v2.4s, v19.8h, v24.8h\n" + "ldr d19, [x21, x17]\n" + "ushll v18.8h, v18.8b, #0x0\n" "ushll v26.8h, v26.8b, #0x0\n" - "ldr x20, [x15, #0xa0]\n" - "smlal2 v10.4s, v25.8h, v0.8h\n" - "ldr q31, [x12, #0x10]\n" - "smlal2 v22.4s, v27.8h, v4.8h\n" - "ldr d27, [x20, x17]\n" - "smlal v9.4s, v28.4h, v1.4h\n" - "smlal v15.4s, v25.4h, v6.4h\n" - "ushll v24.8h, v24.8b, #0x0\n" - "ldr x22, [x15, #0xb0]\n" - "smlal2 v13.4s, v25.8h, v6.8h\n" - "ldr d25, [x21, x17]\n" - "smlal v11.4s, v29.4h, v4.4h\n" - "ushll v25.8h, v25.8b, #0x0\n" - "smlal2 v10.4s, v29.8h, v4.8h\n" - "ldr d29, [x24, x17]\n" - "smlal2 v22.4s, v28.8h, v1.8h\n" - "ushll v29.8h, v29.8b, #0x0\n" - "smlal v9.4s, v26.4h, v5.4h\n" - "smlal v15.4s, v24.4h, v7.4h\n" - "ldr x21, [x15, #0xb8]\n" - "ushll v27.8h, v27.8b, #0x0\n" - "smlal2 v13.4s, v24.8h, v7.8h\n" - "smlal v11.4s, v24.4h, v1.4h\n" + "smlal v8.4s, v15.4h, v23.4h\n" + "smlal2 v4.4s, v17.8h, v25.8h\n" + "ldr d17, [x20, x17]\n" + "ldr x22, [x15, #0x48]\n" + "smlal2 v1.4s, v10.8h, v12.8h\n" + "smlal2 v14.4s, v27.8h, v11.8h\n" + "ldr x21, [x15, #0x90]\n" + "ldr x20, [x15, #0xa8]\n" + "smlal v21.4s, v28.4h, v12.4h\n" + "smlal v20.4s, v18.4h, v23.4h\n" + "ushll v19.8h, v19.8b, #0x0\n" + "smlal v16.4s, v26.4h, v7.4h\n" + "smlal2 v2.4s, v15.8h, v23.8h\n" + "ldr d15, [x24, x17]\n" + "ushll v17.8h, v17.8b, #0x0\n" + "ushll v15.8h, v15.8b, #0x0\n" + "smlal v8.4s, v28.4h, v25.4h\n" + "smlal2 v4.4s, v28.8h, v12.8h\n" + "ldr d12, [x23, x17]\n" + "ushll v12.8h, v12.8b, #0x0\n" + "smlal2 v1.4s, v18.8h, v23.8h\n" + "ldr d18, [x22, x17]\n" + "smlal2 v14.4s, v26.8h, v7.8h\n" + "ldr d26, [x21, x17]\n" + "ldr x22, [x15, #0xa0]\n" + "smlal v21.4s, v19.4h, v23.4h\n" + "smlal v20.4s, v17.4h, v11.4h\n" + "ldr x21, [x15, #0xb0]\n" + "smlal v16.4s, v15.4h, v25.4h\n" + "smlal2 v2.4s, v28.8h, v25.8h\n" + "ldr d28, [x20, x17]\n" + "ushll v18.8h, v18.8b, #0x0\n" + "ushll v26.8h, v26.8b, #0x0\n" + "smlal v8.4s, v12.4h, v7.4h\n" + "ldr x20, [x15, #0xb8]\n" + "ushll v28.8h, v28.8b, #0x0\n" + "smlal2 v4.4s, v19.8h, v23.8h\n" + "ldr d23, [x22, x17]\n" + "ldr d19, [x21, x17]\n" + "smlal2 v1.4s, v17.8h, v11.8h\n" + "ldr d11, [x20, x17]\n" + "smlal2 v14.4s, v15.8h, v25.8h\n" + "ldr q25, [x12, #0x10]\n" + "smlal v21.4s, v18.4h, v7.4h\n" + "smlal v20.4s, v26.4h, v3.4h\n" + "ushll v23.8h, v23.8b, #0x0\n" "ldr x20, [x15, #0xc0]\n" - "sqrdmulh v15.4s, v15.4s, v18.4s\n" - "smlal2 v10.4s, v24.8h, v1.8h\n" - "ldr d24, [x23, x17]\n" - "smlal2 v22.4s, v26.8h, v5.8h\n" - "ldr d26, [x22, x17]\n" - "smlal v9.4s, v29.4h, v2.4h\n" - "ushll v24.8h, v24.8b, #0x0\n" - "smlal2 v22.4s, v29.8h, v2.8h\n" + "smlal v16.4s, v28.4h, v24.4h\n" + "smlal2 v2.4s, v12.8h, v7.8h\n" + "ushll v19.8h, v19.8b, #0x0\n" "tst x7, #0x7\n" - "smlal v11.4s, v25.4h, v6.4h\n" - "smlal v9.4s, v24.4h, v3.4h\n" - "ushll v26.8h, v26.8b, #0x0\n" + "smlal v8.4s, v10.4h, v3.4h\n" + "smlal2 v4.4s, v18.8h, v7.8h\n" + "ldr d18, [x20, x17]\n" + "ushll v11.8h, v11.8b, #0x0\n" + "smlal2 v1.4s, v26.8h, v3.8h\n" + "smlal2 v14.4s, v28.8h, v24.8h\n" + "ushll v18.8h, v18.8b, #0x0\n" + "add x17, x17, #0x8\n" + "smlal v21.4s, v12.4h, v24.4h\n" + "smlal v20.4s, v23.4h, v9.4h\n" "add x13, x13, #0x20\n" - "smlal v17.4s, v28.4h, v7.4h\n" - "smlal2 v20.4s, v28.8h, v7.8h\n" - "and v2.16b, v15.16b, v21.16b\n" "add x12, x12, #0x20\n" - "smlal2 v10.4s, v25.8h, v6.8h\n" - "ldr d25, [x21, x17]\n" - "smlal2 v22.4s, v24.8h, v3.8h\n" - "ushll v25.8h, v25.8b, #0x0\n" - "smlal v11.4s, v27.4h, v7.4h\n" - "smlal v9.4s, v26.4h, v7.4h\n" - "sqrdmulh v13.4s, v13.4s, v30.4s\n" - "smlal v17.4s, v29.4h, v8.4h\n" - "smlal2 v20.4s, v29.8h, v8.8h\n" - "ldr d29, [x20, x17]\n" - "ushll v29.8h, v29.8b, #0x0\n" - "smlal2 v10.4s, v27.8h, v7.8h\n" - "smlal2 v22.4s, v26.8h, v7.8h\n" - "sshr v2.4s, v2.4s, #0x1f\n" - "add x17, x17, #0x8\n" - "smlal v11.4s, v24.4h, v5.4h\n" - "smlal v9.4s, v25.4h, v6.4h\n" - "and v16.16b, v13.16b, v31.16b\n" - "smlal2 v10.4s, v24.8h, v5.8h\n" - "smlal2 v22.4s, v25.8h, v6.8h\n" - "sqrdmulh v17.4s, v17.4s, v18.4s\n" - "smlal v11.4s, v25.4h, v8.4h\n" - "smlal v9.4s, v29.4h, v8.4h\n" - "sqrdmulh v11.4s, v11.4s, v18.4s\n" - "smlal2 v10.4s, v25.8h, v8.8h\n" - "smlal2 v22.4s, v29.8h, v8.8h\n" - "sqrdmulh v9.4s, v9.4s, v18.4s\n" - "sqadd v15.4s, v15.4s, v2.4s\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "and v18.16b, v17.16b, v21.16b\n" - "sqrdmulh v20.4s, v20.4s, v30.4s\n" - "and v28.16b, v11.16b, v21.16b\n" - "sqrdmulh v10.4s, v10.4s, v30.4s\n" - "and v2.16b, v9.16b, v21.16b\n" - "sqrdmulh v22.4s, v22.4s, v30.4s\n" - "sqadd v13.4s, v13.4s, v16.4s\n" + "smlal v16.4s, v19.4h, v9.4h\n" + "smlal2 v2.4s, v10.8h, v3.8h\n" + "smlal v8.4s, v17.4h, v9.4h\n" + "smlal2 v4.4s, v12.8h, v24.8h\n" + "sqrdmulh v8.4s, v8.4s, v0.4s\n" + "smlal2 v1.4s, v23.8h, v9.8h\n" + "smlal2 v14.4s, v19.8h, v9.8h\n" + "and v23.16b, v8.16b, v31.16b\n" + "smlal v21.4s, v27.4h, v9.4h\n" + "smlal v20.4s, v28.4h, v7.4h\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "smlal v16.4s, v11.4h, v3.4h\n" + "smlal2 v2.4s, v17.8h, v9.8h\n" + "sqrdmulh v2.4s, v2.4s, v29.4s\n" + "smlal2 v4.4s, v27.8h, v9.8h\n" + "smlal2 v1.4s, v28.8h, v7.8h\n" + "and v7.16b, v2.16b, v25.16b\n" + "smlal2 v14.4s, v11.8h, v3.8h\n" + "smlal v21.4s, v15.4h, v30.4h\n" + "sqrdmulh v21.4s, v21.4s, v0.4s\n" + "smlal v20.4s, v11.4h, v30.4h\n" + "smlal v16.4s, v18.4h, v30.4h\n" + "sqrdmulh v20.4s, v20.4s, v0.4s\n" + "smlal2 v4.4s, v15.8h, v30.8h\n" + "smlal2 v1.4s, v11.8h, v30.8h\n" + "sqrdmulh v16.4s, v16.4s, v0.4s\n" + "smlal2 v14.4s, v18.8h, v30.8h\n" + "sqadd v8.4s, v8.4s, v23.4s\n" + "sshr v7.4s, v7.4s, #0x1f\n" + "and v23.16b, v21.16b, v31.16b\n" + "sqrdmulh v4.4s, v4.4s, v29.4s\n" + "and v24.16b, v20.16b, v31.16b\n" + "sqrdmulh v1.4s, v1.4s, v29.4s\n" + "and v19.16b, v16.16b, v31.16b\n" + "sqrdmulh v14.4s, v14.4s, v29.4s\n" + "sqadd v2.4s, v2.4s, v7.4s\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "and v18.16b, v4.16b, v25.16b\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "and v17.16b, v1.16b, v25.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "and v15.16b, v14.16b, v25.16b\n" + "sqadd v21.4s, v21.4s, v23.4s\n" "sshr v18.4s, v18.4s, #0x1f\n" - "and v4.16b, v20.16b, v31.16b\n" - "sshr v28.4s, v28.4s, #0x1f\n" - "and v3.16b, v10.16b, v31.16b\n" - "sshr v2.4s, v2.4s, #0x1f\n" - "and v16.16b, v22.16b, v31.16b\n" - "sqadd v17.4s, v17.4s, v18.4s\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v11.4s, v11.4s, v28.4s\n" - "sshr v3.4s, v3.4s, #0x1f\n" - "sqadd v9.4s, v9.4s, v2.4s\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "srshl v15.4s, v15.4s, v21.4s\n" - "srshl v17.4s, v17.4s, v21.4s\n" - "sqadd v20.4s, v20.4s, v4.4s\n" - "srshl v11.4s, v11.4s, v21.4s\n" - "sqadd v10.4s, v10.4s, v3.4s\n" - "srshl v9.4s, v9.4s, v21.4s\n" - "sqadd v22.4s, v22.4s, v16.4s\n" - "srshl v13.4s, v13.4s, v31.4s\n" - "sqxtn v15.4h, v15.4s\n" + "sqadd v20.4s, v20.4s, v24.4s\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v19.4s\n" + "sshr v15.4s, v15.4s, #0x1f\n" + "srshl v8.4s, v8.4s, v31.4s\n" + "srshl v21.4s, v21.4s, v31.4s\n" + "sqadd v4.4s, v4.4s, v18.4s\n" "srshl v20.4s, v20.4s, v31.4s\n" - "sqxtn v17.4h, v17.4s\n" - "srshl v10.4s, v10.4s, v31.4s\n" - "sqxtn v11.4h, v11.4s\n" - "srshl v22.4s, v22.4s, v31.4s\n" - "sqxtn v9.4h, v9.4s\n" - "sqxtn2 v15.8h, v13.4s\n" - "sqxtn2 v17.8h, v20.4s\n" - "sqxtn2 v11.8h, v10.4s\n" - "sqxtn2 v9.8h, v22.4s\n" - "sqadd v15.8h, v15.8h, v12.8h\n" - "sqadd v17.8h, v17.8h, v12.8h\n" - "sqadd v11.8h, v11.8h, v12.8h\n" - "sqadd v9.8h, v9.8h, v12.8h\n" - "smax v15.8h, v15.8h, v14.8h\n" - "smax v17.8h, v17.8h, v14.8h\n" - "smax v11.8h, v11.8h, v14.8h\n" - "smax v9.8h, v9.8h, v14.8h\n" - "smin v15.8h, v15.8h, v23.8h\n" - "smin v17.8h, v17.8h, v23.8h\n" - "smin v11.8h, v11.8h, v23.8h\n" - "smin v9.8h, v9.8h, v23.8h\n" - "uzp1 v15.16b, v15.16b, v15.16b\n" - "uzp1 v17.16b, v17.16b, v17.16b\n" - "str d15, [x11, x16]\n" - "uzp1 v11.16b, v11.16b, v11.16b\n" - "uzp1 v9.16b, v9.16b, v9.16b\n" - "str d17, [x10, x16]\n" - "str d11, [x9, x16]\n" - "str d9, [x28, x16]\n" + "sqadd v1.4s, v1.4s, v17.4s\n" + "srshl v16.4s, v16.4s, v31.4s\n" + "sqadd v14.4s, v14.4s, v15.4s\n" + "srshl v2.4s, v2.4s, v25.4s\n" + "sqxtn v8.4h, v8.4s\n" + "srshl v4.4s, v4.4s, v25.4s\n" + "sqxtn v21.4h, v21.4s\n" + "srshl v1.4s, v1.4s, v25.4s\n" + "sqxtn v20.4h, v20.4s\n" + "srshl v14.4s, v14.4s, v25.4s\n" + "sqxtn v16.4h, v16.4s\n" + "sqxtn2 v8.8h, v2.4s\n" + "sqxtn2 v21.8h, v4.4s\n" + "sqxtn2 v20.8h, v1.4s\n" + "sqxtn2 v16.8h, v14.4s\n" + "sqadd v8.8h, v8.8h, v22.8h\n" + "sqadd v21.8h, v21.8h, v22.8h\n" + "sqadd v20.8h, v20.8h, v22.8h\n" + "sqadd v16.8h, v16.8h, v22.8h\n" + "smax v8.8h, v8.8h, v13.8h\n" + "smax v21.8h, v21.8h, v13.8h\n" + "smax v20.8h, v20.8h, v13.8h\n" + "smax v16.8h, v16.8h, v13.8h\n" + "smin v8.8h, v8.8h, v5.8h\n" + "smin v21.8h, v21.8h, v5.8h\n" + "smin v20.8h, v20.8h, v5.8h\n" + "smin v16.8h, v16.8h, v5.8h\n" + "uzp1 v8.16b, v8.16b, v8.16b\n" + "str d8, [x11, x16]\n" + "uzp1 v21.16b, v21.16b, v21.16b\n" + "uzp1 v20.16b, v20.16b, v20.16b\n" + "str d21, [x10, x16]\n" + "uzp1 v16.16b, v16.16b, v16.16b\n" + "str d20, [x9, x16]\n" + "str d16, [x28, x16]\n" "add x16, x16, #0x8\n" "beq 88f\n" "add x14, x14, #0x48\n" "3:" // Oddments - "ldr x22, [%x[params], %[offsetof_Params_bias]]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" "tbz x7, #2, 5f\n" - "ld1 { v15.4s }, [x22], #0x10\n" + "ld1 { v8.4s }, [x20], #0x10\n" "tbz x7, #1, 4f\n" - "ld1 { v13.d }[0], [x22], #0x8\n" + "ld1 { v2.d }[0], [x20], #0x8\n" "tbz x7, #0, 7f\n" - "ld1 { v13.s }[2], [x22]\n" + "ld1 { v2.s }[2], [x20]\n" "b 7f\n" "4:" // Oddments: Load bias: Bit 2: Bit 1: Unset "tbz x7, #0, 7f\n" - "ld1 { v13.s }[0], [x22]\n" + "ld1 { v2.s }[0], [x20]\n" "b 7f\n" "5:" // Oddments: Load bias: Bit 2: Unset "tbz x7, #1, 6f\n" - "ld1 { v15.d }[0], [x22], #0x8\n" + "ld1 { v8.d }[0], [x20], #0x8\n" "tbz x7, #0, 7f\n" - "ld1 { v15.s }[2], [x22]\n" + "ld1 { v8.s }[2], [x20]\n" "b 7f\n" "6:" // Oddments: Load bias: Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 7f\n" - "ld1 { v15.s }[0], [x22]\n" + "ld1 { v8.s }[0], [x20]\n" "7:" // Oddments: Load bias: Bit 2: End - "ldr d0, [x14, #0x0]\n" - "ldr d1, [x14, #0x8]\n" - "mov v17.16b, v15.16b\n" - "mov v20.16b, v13.16b\n" - "ldr d2, [x14, #0x10]\n" - "ldr d3, [x14, #0x18]\n" - "mov v11.16b, v15.16b\n" - "mov v10.16b, v13.16b\n" - "ldr d4, [x14, #0x20]\n" - "ldr d5, [x14, #0x28]\n" - "mov v9.16b, v15.16b\n" - "mov v22.16b, v13.16b\n" - "ldr d6, [x14, #0x30]\n" - "ldr d7, [x14, #0x38]\n" - "usubl v0.8h, v0.8b, v19.8b\n" - "usubl v1.8h, v1.8b, v19.8b\n" - "ldr d8, [x14, #0x40]\n" + "ldr d12, [x14, #0x0]\n" + "ldr d11, [x14, #0x8]\n" + "mov v21.16b, v8.16b\n" + "mov v4.16b, v2.16b\n" + "ldr d25, [x14, #0x10]\n" + "ldr d24, [x14, #0x18]\n" + "mov v20.16b, v8.16b\n" + "mov v1.16b, v2.16b\n" + "ldr d23, [x14, #0x20]\n" + "ldr d7, [x14, #0x28]\n" + "mov v16.16b, v8.16b\n" + "mov v14.16b, v2.16b\n" + "ldr d3, [x14, #0x30]\n" + "ldr d9, [x14, #0x38]\n" + "usubl v12.8h, v12.8b, v6.8b\n" + "usubl v11.8h, v11.8b, v6.8b\n" + "ldr d30, [x14, #0x40]\n" "ldp x27, x26, [x15, #0x0]\n" - "usubl v2.8h, v2.8b, v19.8b\n" - "usubl v3.8h, v3.8b, v19.8b\n" + "usubl v25.8h, v25.8b, v6.8b\n" + "usubl v24.8h, v24.8b, v6.8b\n" "ldp x25, x24, [x15, #0x10]\n" "ldp x23, x22, [x15, #0x20]\n" - "usubl v4.8h, v4.8b, v19.8b\n" - "usubl v5.8h, v5.8b, v19.8b\n" + "usubl v23.8h, v23.8b, v6.8b\n" + "usubl v7.8h, v7.8b, v6.8b\n" "ldp x21, x20, [x15, #0x30]\n" - "usubl v6.8h, v6.8b, v19.8b\n" - "usubl v7.8h, v7.8b, v19.8b\n" - "usubl v8.8h, v8.8b, v19.8b\n" + "usubl v3.8h, v3.8b, v6.8b\n" + "usubl v9.8h, v9.8b, v6.8b\n" + "usubl v30.8h, v30.8b, v6.8b\n" "add x27, x27, x17\n" "add x26, x26, x17\n" "add x25, x25, x17\n" @@ -687,700 +687,700 @@ void a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "add x21, x21, x17\n" "add x20, x20, x17\n" "tbz x7, #2, 9f\n" - "ld1 { v31.s }[0], [x27], #0x4\n" - "ld1 { v30.s }[0], [x26], #0x4\n" - "ld1 { v29.s }[0], [x25], #0x4\n" - "ld1 { v28.s }[0], [x24], #0x4\n" - "ld1 { v27.s }[0], [x23], #0x4\n" - "ld1 { v26.s }[0], [x22], #0x4\n" - "ld1 { v25.s }[0], [x21], #0x4\n" - "ld1 { v24.s }[0], [x20], #0x4\n" + "ld1 { v26.s }[0], [x27], #0x4\n" + "ld1 { v18.s }[0], [x26], #0x4\n" + "ld1 { v10.s }[0], [x25], #0x4\n" + "ld1 { v27.s }[0], [x24], #0x4\n" + "ld1 { v17.s }[0], [x23], #0x4\n" + "ld1 { v19.s }[0], [x22], #0x4\n" + "ld1 { v15.s }[0], [x21], #0x4\n" + "ld1 { v28.s }[0], [x20], #0x4\n" "tbz x7, #1, 8f\n" - "ld1 { v31.h }[2], [x27], #0x2\n" - "ld1 { v30.h }[2], [x26], #0x2\n" - "ld1 { v29.h }[2], [x25], #0x2\n" - "ld1 { v28.h }[2], [x24], #0x2\n" - "ld1 { v27.h }[2], [x23], #0x2\n" - "ld1 { v26.h }[2], [x22], #0x2\n" - "ld1 { v25.h }[2], [x21], #0x2\n" - "ld1 { v24.h }[2], [x20], #0x2\n" + "ld1 { v26.h }[2], [x27], #0x2\n" + "ld1 { v18.h }[2], [x26], #0x2\n" + "ld1 { v10.h }[2], [x25], #0x2\n" + "ld1 { v27.h }[2], [x24], #0x2\n" + "ld1 { v17.h }[2], [x23], #0x2\n" + "ld1 { v19.h }[2], [x22], #0x2\n" + "ld1 { v15.h }[2], [x21], #0x2\n" + "ld1 { v28.h }[2], [x20], #0x2\n" "tbz x7, #0, 11f\n" - "ld1 { v31.b }[6], [x27]\n" - "ld1 { v30.b }[6], [x26]\n" - "ld1 { v29.b }[6], [x25]\n" - "ld1 { v28.b }[6], [x24]\n" - "ld1 { v27.b }[6], [x23]\n" - "ld1 { v26.b }[6], [x22]\n" - "ld1 { v25.b }[6], [x21]\n" - "ld1 { v24.b }[6], [x20]\n" + "ld1 { v26.b }[6], [x27]\n" + "ld1 { v18.b }[6], [x26]\n" + "ld1 { v10.b }[6], [x25]\n" + "ld1 { v27.b }[6], [x24]\n" + "ld1 { v17.b }[6], [x23]\n" + "ld1 { v19.b }[6], [x22]\n" + "ld1 { v15.b }[6], [x21]\n" + "ld1 { v28.b }[6], [x20]\n" "b 11f\n" "8:" // Oddments: Initial loads: Bit 2: Bit 1: Unset "tbz x7, #0, 11f\n" - "ld1 { v31.b }[4], [x27]\n" - "ld1 { v30.b }[4], [x26]\n" - "ld1 { v29.b }[4], [x25]\n" - "ld1 { v28.b }[4], [x24]\n" - "ld1 { v27.b }[4], [x23]\n" - "ld1 { v26.b }[4], [x22]\n" - "ld1 { v25.b }[4], [x21]\n" - "ld1 { v24.b }[4], [x20]\n" + "ld1 { v26.b }[4], [x27]\n" + "ld1 { v18.b }[4], [x26]\n" + "ld1 { v10.b }[4], [x25]\n" + "ld1 { v27.b }[4], [x24]\n" + "ld1 { v17.b }[4], [x23]\n" + "ld1 { v19.b }[4], [x22]\n" + "ld1 { v15.b }[4], [x21]\n" + "ld1 { v28.b }[4], [x20]\n" "b 11f\n" "9:" // Oddments: Initial loads: Bit 2: Unset "tbz x7, #1, 10f\n" - "ld1 { v31.h }[0], [x27], #0x2\n" - "ld1 { v30.h }[0], [x26], #0x2\n" - "ld1 { v29.h }[0], [x25], #0x2\n" - "ld1 { v28.h }[0], [x24], #0x2\n" - "ld1 { v27.h }[0], [x23], #0x2\n" - "ld1 { v26.h }[0], [x22], #0x2\n" - "ld1 { v25.h }[0], [x21], #0x2\n" - "ld1 { v24.h }[0], [x20], #0x2\n" + "ld1 { v26.h }[0], [x27], #0x2\n" + "ld1 { v18.h }[0], [x26], #0x2\n" + "ld1 { v10.h }[0], [x25], #0x2\n" + "ld1 { v27.h }[0], [x24], #0x2\n" + "ld1 { v17.h }[0], [x23], #0x2\n" + "ld1 { v19.h }[0], [x22], #0x2\n" + "ld1 { v15.h }[0], [x21], #0x2\n" + "ld1 { v28.h }[0], [x20], #0x2\n" "tbz x7, #0, 11f\n" - "ld1 { v31.b }[2], [x27]\n" - "ld1 { v30.b }[2], [x26]\n" - "ld1 { v29.b }[2], [x25]\n" - "ld1 { v28.b }[2], [x24]\n" - "ld1 { v27.b }[2], [x23]\n" - "ld1 { v26.b }[2], [x22]\n" - "ld1 { v25.b }[2], [x21]\n" - "ld1 { v24.b }[2], [x20]\n" + "ld1 { v26.b }[2], [x27]\n" + "ld1 { v18.b }[2], [x26]\n" + "ld1 { v10.b }[2], [x25]\n" + "ld1 { v27.b }[2], [x24]\n" + "ld1 { v17.b }[2], [x23]\n" + "ld1 { v19.b }[2], [x22]\n" + "ld1 { v15.b }[2], [x21]\n" + "ld1 { v28.b }[2], [x20]\n" "b 11f\n" "10:" // Oddments: Initial loads: Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 11f\n" - "ld1 { v31.b }[0], [x27]\n" - "ld1 { v30.b }[0], [x26]\n" - "ld1 { v29.b }[0], [x25]\n" - "ld1 { v28.b }[0], [x24]\n" - "ld1 { v27.b }[0], [x23]\n" - "ld1 { v26.b }[0], [x22]\n" - "ld1 { v25.b }[0], [x21]\n" - "ld1 { v24.b }[0], [x20]\n" + "ld1 { v26.b }[0], [x27]\n" + "ld1 { v18.b }[0], [x26]\n" + "ld1 { v10.b }[0], [x25]\n" + "ld1 { v27.b }[0], [x24]\n" + "ld1 { v17.b }[0], [x23]\n" + "ld1 { v19.b }[0], [x22]\n" + "ld1 { v15.b }[0], [x21]\n" + "ld1 { v28.b }[0], [x20]\n" "11:" // Oddments: Initial loads: Bit 2: End - "ushll v31.8h, v31.8b, #0x0\n" - "smlal v15.4s, v31.4h, v8.4h\n" - "smlal2 v13.4s, v31.8h, v8.8h\n" - "ldr x23, [x15, #0x40]\n" - "ushll v30.8h, v30.8b, #0x0\n" - "smlal v15.4s, v30.4h, v0.4h\n" - "smlal2 v13.4s, v30.8h, v0.8h\n" - "add x23, x23, x17\n" - "ushll v29.8h, v29.8b, #0x0\n" - "smlal v17.4s, v31.4h, v6.4h\n" - "smlal2 v20.4s, v31.8h, v6.8h\n" - "smlal v15.4s, v29.4h, v1.4h\n" - "smlal2 v13.4s, v29.8h, v1.8h\n" - "ushll v28.8h, v28.8b, #0x0\n" "ushll v26.8h, v26.8b, #0x0\n" - "smlal v17.4s, v28.4h, v1.4h\n" - "smlal2 v20.4s, v28.8h, v1.8h\n" - "smlal v15.4s, v26.4h, v3.4h\n" - "smlal2 v13.4s, v26.8h, v3.8h\n" + "smlal v8.4s, v26.4h, v30.4h\n" + "smlal2 v2.4s, v26.8h, v30.8h\n" + "ldr x20, [x15, #0x40]\n" + "ushll v18.8h, v18.8b, #0x0\n" + "smlal v8.4s, v18.4h, v12.4h\n" + "smlal2 v2.4s, v18.8h, v12.8h\n" + "ushll v10.8h, v10.8b, #0x0\n" + "smlal v21.4s, v26.4h, v3.4h\n" + "smlal2 v4.4s, v26.8h, v3.8h\n" + "add x20, x20, x17\n" + "smlal v8.4s, v10.4h, v11.4h\n" + "smlal2 v2.4s, v10.8h, v11.8h\n" "ushll v27.8h, v27.8b, #0x0\n" - "ushll v25.8h, v25.8b, #0x0\n" - "smlal v17.4s, v27.4h, v2.4h\n" - "smlal2 v20.4s, v27.8h, v2.8h\n" - "smlal v15.4s, v25.4h, v4.4h\n" - "smlal2 v13.4s, v25.8h, v4.8h\n" - "ushll v24.8h, v24.8b, #0x0\n" - "smlal v11.4s, v31.4h, v2.4h\n" - "smlal2 v10.4s, v31.8h, v2.8h\n" - "smlal v9.4s, v31.4h, v0.4h\n" - "smlal2 v22.4s, v31.8h, v0.8h\n" - "smlal v15.4s, v24.4h, v2.4h\n" - "smlal2 v13.4s, v24.8h, v2.8h\n" - "smlal v17.4s, v24.4h, v0.4h\n" - "smlal2 v20.4s, v24.8h, v0.8h\n" + "ushll v19.8h, v19.8b, #0x0\n" + "smlal v21.4s, v27.4h, v11.4h\n" + "smlal2 v4.4s, v27.8h, v11.8h\n" + "smlal v8.4s, v19.4h, v24.4h\n" + "smlal2 v2.4s, v19.8h, v24.8h\n" + "ushll v17.8h, v17.8b, #0x0\n" + "ushll v15.8h, v15.8b, #0x0\n" + "smlal v21.4s, v17.4h, v25.4h\n" + "smlal2 v4.4s, v17.8h, v25.8h\n" + "smlal v8.4s, v15.4h, v23.4h\n" + "smlal2 v2.4s, v15.8h, v23.8h\n" + "ushll v28.8h, v28.8b, #0x0\n" + "smlal v20.4s, v26.4h, v25.4h\n" + "smlal2 v1.4s, v26.8h, v25.8h\n" + "smlal v16.4s, v26.4h, v12.4h\n" + "smlal2 v14.4s, v26.8h, v12.8h\n" + "smlal v8.4s, v28.4h, v25.4h\n" + "smlal2 v2.4s, v28.8h, v25.8h\n" + "smlal v21.4s, v28.4h, v12.4h\n" + "smlal2 v4.4s, v28.8h, v12.8h\n" "tbz x7, #2, 13f\n" - "ld1 { v29.s }[0], [x23], #0x4\n" + "ld1 { v31.s }[0], [x20], #0x4\n" "tbz x7, #1, 12f\n" - "ld1 { v29.h }[2], [x23], #0x2\n" + "ld1 { v31.h }[2], [x20], #0x2\n" "tbz x7, #0, 15f\n" - "ld1 { v29.b }[6], [x23]\n" + "ld1 { v31.b }[6], [x20]\n" "b 15f\n" "12:" // Oddments: Load (1, 3): Bit 2: Bit 1: Unset "tbz x7, #0, 15f\n" - "ld1 { v29.b }[4], [x23]\n" + "ld1 { v31.b }[4], [x20]\n" "b 15f\n" "13:" // Oddments: Load (1, 3): Bit 2: Unset "tbz x7, #1, 14f\n" - "ld1 { v29.h }[0], [x23], #0x2\n" + "ld1 { v31.h }[0], [x20], #0x2\n" "tbz x7, #0, 15f\n" - "ld1 { v29.b }[2], [x23]\n" + "ld1 { v31.b }[2], [x20]\n" "b 15f\n" "14:" // Oddments: Load (1, 3): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 15f\n" - "ld1 { v29.b }[0], [x23]\n" + "ld1 { v31.b }[0], [x20]\n" "15:" // Oddments: Load (1, 3): Bit 2: End - "ushll v29.8h, v29.8b, #0x0\n" - "ldr x22, [x15, #0x48]\n" - "smlal v17.4s, v29.4h, v4.4h\n" - "smlal2 v20.4s, v29.8h, v4.8h\n" - "add x22, x22, x17\n" + "ushll v31.8h, v31.8b, #0x0\n" + "ldr x20, [x15, #0x48]\n" + "smlal v21.4s, v31.4h, v23.4h\n" + "smlal2 v4.4s, v31.8h, v23.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 17f\n" - "ld1 { v28.s }[0], [x22], #0x4\n" + "ld1 { v28.s }[0], [x20], #0x4\n" "tbz x7, #1, 16f\n" - "ld1 { v28.h }[2], [x22], #0x2\n" + "ld1 { v28.h }[2], [x20], #0x2\n" "tbz x7, #0, 19f\n" - "ld1 { v28.b }[6], [x22]\n" + "ld1 { v28.b }[6], [x20]\n" "b 19f\n" "16:" // Oddments: Load (1, 4): Bit 2: Bit 1: Unset "tbz x7, #0, 19f\n" - "ld1 { v28.b }[4], [x22]\n" + "ld1 { v28.b }[4], [x20]\n" "b 19f\n" "17:" // Oddments: Load (1, 4): Bit 2: Unset "tbz x7, #1, 18f\n" - "ld1 { v28.h }[0], [x22], #0x2\n" + "ld1 { v28.h }[0], [x20], #0x2\n" "tbz x7, #0, 19f\n" - "ld1 { v28.b }[2], [x22]\n" + "ld1 { v28.b }[2], [x20]\n" "b 19f\n" "18:" // Oddments: Load (1, 4): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 19f\n" - "ld1 { v28.b }[0], [x22]\n" + "ld1 { v28.b }[0], [x20]\n" "19:" // Oddments: Load (1, 4): Bit 2: End "ushll v28.8h, v28.8b, #0x0\n" - "ldr x21, [x15, #0x50]\n" - "smlal v17.4s, v28.4h, v5.4h\n" - "smlal2 v20.4s, v28.8h, v5.8h\n" - "add x21, x21, x17\n" + "ldr x20, [x15, #0x50]\n" + "smlal v21.4s, v28.4h, v7.4h\n" + "smlal2 v4.4s, v28.8h, v7.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 21f\n" - "ld1 { v27.s }[0], [x21], #0x4\n" + "ld1 { v27.s }[0], [x20], #0x4\n" "tbz x7, #1, 20f\n" - "ld1 { v27.h }[2], [x21], #0x2\n" + "ld1 { v27.h }[2], [x20], #0x2\n" "tbz x7, #0, 23f\n" - "ld1 { v27.b }[6], [x21]\n" + "ld1 { v27.b }[6], [x20]\n" "b 23f\n" "20:" // Oddments: Load (1, 2): Bit 2: Bit 1: Unset "tbz x7, #0, 23f\n" - "ld1 { v27.b }[4], [x21]\n" + "ld1 { v27.b }[4], [x20]\n" "b 23f\n" "21:" // Oddments: Load (1, 2): Bit 2: Unset "tbz x7, #1, 22f\n" - "ld1 { v27.h }[0], [x21], #0x2\n" + "ld1 { v27.h }[0], [x20], #0x2\n" "tbz x7, #0, 23f\n" - "ld1 { v27.b }[2], [x21]\n" + "ld1 { v27.b }[2], [x20]\n" "b 23f\n" "22:" // Oddments: Load (1, 2): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 23f\n" - "ld1 { v27.b }[0], [x21]\n" + "ld1 { v27.b }[0], [x20]\n" "23:" // Oddments: Load (1, 2): Bit 2: End "ushll v27.8h, v27.8b, #0x0\n" "ldr x20, [x15, #0x58]\n" - "smlal v15.4s, v27.4h, v5.4h\n" - "smlal2 v13.4s, v27.8h, v5.8h\n" - "smlal v17.4s, v27.4h, v3.4h\n" - "smlal2 v20.4s, v27.8h, v3.8h\n" + "smlal v8.4s, v27.4h, v7.4h\n" + "smlal2 v2.4s, v27.8h, v7.8h\n" + "smlal v21.4s, v27.4h, v24.4h\n" + "smlal2 v4.4s, v27.8h, v24.8h\n" "add x20, x20, x17\n" "tbz x7, #2, 25f\n" - "ld1 { v26.s }[0], [x20], #0x4\n" + "ld1 { v0.s }[0], [x20], #0x4\n" "tbz x7, #1, 24f\n" - "ld1 { v26.h }[2], [x20], #0x2\n" + "ld1 { v0.h }[2], [x20], #0x2\n" "tbz x7, #0, 27f\n" - "ld1 { v26.b }[6], [x20]\n" + "ld1 { v0.b }[6], [x20]\n" "b 27f\n" "24:" // Oddments: Load (3, 0): Bit 2: Bit 1: Unset "tbz x7, #0, 27f\n" - "ld1 { v26.b }[4], [x20]\n" + "ld1 { v0.b }[4], [x20]\n" "b 27f\n" "25:" // Oddments: Load (3, 0): Bit 2: Unset "tbz x7, #1, 26f\n" - "ld1 { v26.h }[0], [x20], #0x2\n" + "ld1 { v0.h }[0], [x20], #0x2\n" "tbz x7, #0, 27f\n" - "ld1 { v26.b }[2], [x20]\n" + "ld1 { v0.b }[2], [x20]\n" "b 27f\n" "26:" // Oddments: Load (3, 0): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 27f\n" - "ld1 { v26.b }[0], [x20]\n" + "ld1 { v0.b }[0], [x20]\n" "27:" // Oddments: Load (3, 0): Bit 2: End - "ushll v26.8h, v26.8b, #0x0\n" + "ushll v0.8h, v0.8b, #0x0\n" "ldr x20, [x15, #0x60]\n" - "smlal v11.4s, v26.4h, v3.4h\n" - "smlal2 v10.4s, v26.8h, v3.8h\n" + "smlal v20.4s, v0.4h, v24.4h\n" + "smlal2 v1.4s, v0.8h, v24.8h\n" "add x20, x20, x17\n" "tbz x7, #2, 29f\n" - "ld1 { v25.s }[0], [x20], #0x4\n" + "ld1 { v15.s }[0], [x20], #0x4\n" "tbz x7, #1, 28f\n" - "ld1 { v25.h }[2], [x20], #0x2\n" + "ld1 { v15.h }[2], [x20], #0x2\n" "tbz x7, #0, 31f\n" - "ld1 { v25.b }[6], [x20]\n" + "ld1 { v15.b }[6], [x20]\n" "b 31f\n" "28:" // Oddments: Load (2, 0): Bit 2: Bit 1: Unset "tbz x7, #0, 31f\n" - "ld1 { v25.b }[4], [x20]\n" + "ld1 { v15.b }[4], [x20]\n" "b 31f\n" "29:" // Oddments: Load (2, 0): Bit 2: Unset "tbz x7, #1, 30f\n" - "ld1 { v25.h }[0], [x20], #0x2\n" + "ld1 { v15.h }[0], [x20], #0x2\n" "tbz x7, #0, 31f\n" - "ld1 { v25.b }[2], [x20]\n" + "ld1 { v15.b }[2], [x20]\n" "b 31f\n" "30:" // Oddments: Load (2, 0): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 31f\n" - "ld1 { v25.b }[0], [x20]\n" + "ld1 { v15.b }[0], [x20]\n" "31:" // Oddments: Load (2, 0): Bit 2: End - "ushll v25.8h, v25.8b, #0x0\n" + "ushll v15.8h, v15.8b, #0x0\n" "ldr x20, [x15, #0x68]\n" - "smlal v15.4s, v25.4h, v6.4h\n" - "smlal2 v13.4s, v25.8h, v6.8h\n" - "smlal v11.4s, v25.4h, v0.4h\n" - "smlal2 v10.4s, v25.8h, v0.8h\n" + "smlal v8.4s, v15.4h, v3.4h\n" + "smlal2 v2.4s, v15.8h, v3.8h\n" + "smlal v20.4s, v15.4h, v12.4h\n" + "smlal2 v1.4s, v15.8h, v12.8h\n" "add x20, x20, x17\n" "tbz x7, #2, 33f\n" - "ld1 { v29.s }[0], [x20], #0x4\n" + "ld1 { v0.s }[0], [x20], #0x4\n" "tbz x7, #1, 32f\n" - "ld1 { v29.h }[2], [x20], #0x2\n" + "ld1 { v0.h }[2], [x20], #0x2\n" "tbz x7, #0, 35f\n" - "ld1 { v29.b }[6], [x20]\n" + "ld1 { v0.b }[6], [x20]\n" "b 35f\n" "32:" // Oddments: Load (3, 1): Bit 2: Bit 1: Unset "tbz x7, #0, 35f\n" - "ld1 { v29.b }[4], [x20]\n" + "ld1 { v0.b }[4], [x20]\n" "b 35f\n" "33:" // Oddments: Load (3, 1): Bit 2: Unset "tbz x7, #1, 34f\n" - "ld1 { v29.h }[0], [x20], #0x2\n" + "ld1 { v0.h }[0], [x20], #0x2\n" "tbz x7, #0, 35f\n" - "ld1 { v29.b }[2], [x20]\n" + "ld1 { v0.b }[2], [x20]\n" "b 35f\n" "34:" // Oddments: Load (3, 1): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 35f\n" - "ld1 { v29.b }[0], [x20]\n" + "ld1 { v0.b }[0], [x20]\n" "35:" // Oddments: Load (3, 1): Bit 2: End - "ushll v29.8h, v29.8b, #0x0\n" + "ushll v0.8h, v0.8b, #0x0\n" "ldr x20, [x15, #0x70]\n" - "smlal v11.4s, v29.4h, v4.4h\n" - "smlal2 v10.4s, v29.8h, v4.8h\n" + "smlal v20.4s, v0.4h, v23.4h\n" + "smlal2 v1.4s, v0.8h, v23.8h\n" "add x20, x20, x17\n" "tbz x7, #2, 37f\n" - "ld1 { v24.s }[0], [x20], #0x4\n" + "ld1 { v6.s }[0], [x20], #0x4\n" "tbz x7, #1, 36f\n" - "ld1 { v24.h }[2], [x20], #0x2\n" + "ld1 { v6.h }[2], [x20], #0x2\n" "tbz x7, #0, 39f\n" - "ld1 { v24.b }[6], [x20]\n" + "ld1 { v6.b }[6], [x20]\n" "b 39f\n" "36:" // Oddments: Load (2, 1): Bit 2: Bit 1: Unset "tbz x7, #0, 39f\n" - "ld1 { v24.b }[4], [x20]\n" + "ld1 { v6.b }[4], [x20]\n" "b 39f\n" "37:" // Oddments: Load (2, 1): Bit 2: Unset "tbz x7, #1, 38f\n" - "ld1 { v24.h }[0], [x20], #0x2\n" + "ld1 { v6.h }[0], [x20], #0x2\n" "tbz x7, #0, 39f\n" - "ld1 { v24.b }[2], [x20]\n" + "ld1 { v6.b }[2], [x20]\n" "b 39f\n" "38:" // Oddments: Load (2, 1): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 39f\n" - "ld1 { v24.b }[0], [x20]\n" + "ld1 { v6.b }[0], [x20]\n" "39:" // Oddments: Load (2, 1): Bit 2: End - "ushll v24.8h, v24.8b, #0x0\n" - "ldr x23, [x15, #0x78]\n" - "smlal v15.4s, v24.4h, v7.4h\n" - "smlal2 v13.4s, v24.8h, v7.8h\n" - "smlal v11.4s, v24.4h, v1.4h\n" - "smlal2 v10.4s, v24.8h, v1.8h\n" - "add x23, x23, x17\n" + "ushll v6.8h, v6.8b, #0x0\n" + "ldr x20, [x15, #0x78]\n" + "smlal v8.4s, v6.4h, v9.4h\n" + "smlal2 v2.4s, v6.8h, v9.8h\n" + "smlal v20.4s, v6.4h, v11.4h\n" + "smlal2 v1.4s, v6.8h, v11.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 41f\n" - "ld1 { v27.s }[0], [x23], #0x4\n" + "ld1 { v27.s }[0], [x20], #0x4\n" "tbz x7, #1, 40f\n" - "ld1 { v27.h }[2], [x23], #0x2\n" + "ld1 { v27.h }[2], [x20], #0x2\n" "tbz x7, #0, 43f\n" - "ld1 { v27.b }[6], [x23]\n" + "ld1 { v27.b }[6], [x20]\n" "b 43f\n" "40:" // Oddments: Load (3, 3): Bit 2: Bit 1: Unset "tbz x7, #0, 43f\n" - "ld1 { v27.b }[4], [x23]\n" + "ld1 { v27.b }[4], [x20]\n" "b 43f\n" "41:" // Oddments: Load (3, 3): Bit 2: Unset "tbz x7, #1, 42f\n" - "ld1 { v27.h }[0], [x23], #0x2\n" + "ld1 { v27.h }[0], [x20], #0x2\n" "tbz x7, #0, 43f\n" - "ld1 { v27.b }[2], [x23]\n" + "ld1 { v27.b }[2], [x20]\n" "b 43f\n" "42:" // Oddments: Load (3, 3): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 43f\n" - "ld1 { v27.b }[0], [x23]\n" + "ld1 { v27.b }[0], [x20]\n" "43:" // Oddments: Load (3, 3): Bit 2: End "ushll v27.8h, v27.8b, #0x0\n" - "ldr x21, [x15, #0x80]\n" - "smlal v9.4s, v27.4h, v4.4h\n" - "smlal2 v22.4s, v27.8h, v4.8h\n" - "add x21, x21, x17\n" + "ldr x20, [x15, #0x80]\n" + "smlal v16.4s, v27.4h, v23.4h\n" + "smlal2 v14.4s, v27.8h, v23.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 45f\n" - "ld1 { v28.s }[0], [x21], #0x4\n" + "ld1 { v10.s }[0], [x20], #0x4\n" "tbz x7, #1, 44f\n" - "ld1 { v28.h }[2], [x21], #0x2\n" + "ld1 { v10.h }[2], [x20], #0x2\n" "tbz x7, #0, 47f\n" - "ld1 { v28.b }[6], [x21]\n" + "ld1 { v10.b }[6], [x20]\n" "b 47f\n" "44:" // Oddments: Load (2, 3): Bit 2: Bit 1: Unset "tbz x7, #0, 47f\n" - "ld1 { v28.b }[4], [x21]\n" + "ld1 { v10.b }[4], [x20]\n" "b 47f\n" "45:" // Oddments: Load (2, 3): Bit 2: Unset "tbz x7, #1, 46f\n" - "ld1 { v28.h }[0], [x21], #0x2\n" + "ld1 { v10.h }[0], [x20], #0x2\n" "tbz x7, #0, 47f\n" - "ld1 { v28.b }[2], [x21]\n" + "ld1 { v10.b }[2], [x20]\n" "b 47f\n" "46:" // Oddments: Load (2, 3): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 47f\n" - "ld1 { v28.b }[0], [x21]\n" + "ld1 { v10.b }[0], [x20]\n" "47:" // Oddments: Load (2, 3): Bit 2: End - "ushll v28.8h, v28.8b, #0x0\n" - "ldr x22, [x15, #0x88]\n" - "smlal v17.4s, v28.4h, v7.4h\n" - "smlal2 v20.4s, v28.8h, v7.8h\n" - "smlal v9.4s, v28.4h, v1.4h\n" - "smlal2 v22.4s, v28.8h, v1.8h\n" - "add x22, x22, x17\n" + "ushll v10.8h, v10.8b, #0x0\n" + "ldr x20, [x15, #0x88]\n" + "smlal v21.4s, v10.4h, v9.4h\n" + "smlal2 v4.4s, v10.8h, v9.8h\n" + "smlal v16.4s, v10.4h, v11.4h\n" + "smlal2 v14.4s, v10.8h, v11.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 49f\n" - "ld1 { v26.s }[0], [x22], #0x4\n" + "ld1 { v28.s }[0], [x20], #0x4\n" "tbz x7, #1, 48f\n" - "ld1 { v26.h }[2], [x22], #0x2\n" + "ld1 { v28.h }[2], [x20], #0x2\n" "tbz x7, #0, 51f\n" - "ld1 { v26.b }[6], [x22]\n" + "ld1 { v28.b }[6], [x20]\n" "b 51f\n" "48:" // Oddments: Load (3, 4): Bit 2: Bit 1: Unset "tbz x7, #0, 51f\n" - "ld1 { v26.b }[4], [x22]\n" + "ld1 { v28.b }[4], [x20]\n" "b 51f\n" "49:" // Oddments: Load (3, 4): Bit 2: Unset "tbz x7, #1, 50f\n" - "ld1 { v26.h }[0], [x22], #0x2\n" + "ld1 { v28.h }[0], [x20], #0x2\n" "tbz x7, #0, 51f\n" - "ld1 { v26.b }[2], [x22]\n" + "ld1 { v28.b }[2], [x20]\n" "b 51f\n" "50:" // Oddments: Load (3, 4): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 51f\n" - "ld1 { v26.b }[0], [x22]\n" + "ld1 { v28.b }[0], [x20]\n" "51:" // Oddments: Load (3, 4): Bit 2: End - "ushll v26.8h, v26.8b, #0x0\n" - "ldr x21, [x15, #0x90]\n" - "smlal v9.4s, v26.4h, v5.4h\n" - "smlal2 v22.4s, v26.8h, v5.8h\n" - "add x21, x21, x17\n" + "ushll v28.8h, v28.8b, #0x0\n" + "ldr x20, [x15, #0x90]\n" + "smlal v16.4s, v28.4h, v7.4h\n" + "smlal2 v14.4s, v28.8h, v7.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 53f\n" - "ld1 { v25.s }[0], [x21], #0x4\n" + "ld1 { v15.s }[0], [x20], #0x4\n" "tbz x7, #1, 52f\n" - "ld1 { v25.h }[2], [x21], #0x2\n" + "ld1 { v15.h }[2], [x20], #0x2\n" "tbz x7, #0, 55f\n" - "ld1 { v25.b }[6], [x21]\n" + "ld1 { v15.b }[6], [x20]\n" "b 55f\n" "52:" // Oddments: Load (4, 0): Bit 2: Bit 1: Unset "tbz x7, #0, 55f\n" - "ld1 { v25.b }[4], [x21]\n" + "ld1 { v15.b }[4], [x20]\n" "b 55f\n" "53:" // Oddments: Load (4, 0): Bit 2: Unset "tbz x7, #1, 54f\n" - "ld1 { v25.h }[0], [x21], #0x2\n" + "ld1 { v15.h }[0], [x20], #0x2\n" "tbz x7, #0, 55f\n" - "ld1 { v25.b }[2], [x21]\n" + "ld1 { v15.b }[2], [x20]\n" "b 55f\n" "54:" // Oddments: Load (4, 0): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 55f\n" - "ld1 { v25.b }[0], [x21]\n" + "ld1 { v15.b }[0], [x20]\n" "55:" // Oddments: Load (4, 0): Bit 2: End - "ushll v25.8h, v25.8b, #0x0\n" - "ldr x24, [x15, #0x98]\n" - "smlal v11.4s, v25.4h, v6.4h\n" - "smlal2 v10.4s, v25.8h, v6.8h\n" - "add x24, x24, x17\n" + "ushll v15.8h, v15.8b, #0x0\n" + "ldr x20, [x15, #0x98]\n" + "smlal v20.4s, v15.4h, v3.4h\n" + "smlal2 v1.4s, v15.8h, v3.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 57f\n" - "ld1 { v29.s }[0], [x24], #0x4\n" + "ld1 { v6.s }[0], [x20], #0x4\n" "tbz x7, #1, 56f\n" - "ld1 { v29.h }[2], [x24], #0x2\n" + "ld1 { v6.h }[2], [x20], #0x2\n" "tbz x7, #0, 59f\n" - "ld1 { v29.b }[6], [x24]\n" + "ld1 { v6.b }[6], [x20]\n" "b 59f\n" "56:" // Oddments: Load (2, 4): Bit 2: Bit 1: Unset "tbz x7, #0, 59f\n" - "ld1 { v29.b }[4], [x24]\n" + "ld1 { v6.b }[4], [x20]\n" "b 59f\n" "57:" // Oddments: Load (2, 4): Bit 2: Unset "tbz x7, #1, 58f\n" - "ld1 { v29.h }[0], [x24], #0x2\n" + "ld1 { v6.h }[0], [x20], #0x2\n" "tbz x7, #0, 59f\n" - "ld1 { v29.b }[2], [x24]\n" + "ld1 { v6.b }[2], [x20]\n" "b 59f\n" "58:" // Oddments: Load (2, 4): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 59f\n" - "ld1 { v29.b }[0], [x24]\n" + "ld1 { v6.b }[0], [x20]\n" "59:" // Oddments: Load (2, 4): Bit 2: End - "ushll v29.8h, v29.8b, #0x0\n" + "ushll v6.8h, v6.8b, #0x0\n" "ldr x20, [x15, #0xa0]\n" - "smlal v17.4s, v29.4h, v8.4h\n" - "smlal2 v20.4s, v29.8h, v8.8h\n" - "smlal v9.4s, v29.4h, v2.4h\n" - "smlal2 v22.4s, v29.8h, v2.8h\n" + "smlal v21.4s, v6.4h, v30.4h\n" + "smlal2 v4.4s, v6.8h, v30.8h\n" + "smlal v16.4s, v6.4h, v25.4h\n" + "smlal2 v14.4s, v6.8h, v25.8h\n" "add x20, x20, x17\n" "tbz x7, #2, 61f\n" - "ld1 { v27.s }[0], [x20], #0x4\n" + "ld1 { v23.s }[0], [x20], #0x4\n" "tbz x7, #1, 60f\n" - "ld1 { v27.h }[2], [x20], #0x2\n" + "ld1 { v23.h }[2], [x20], #0x2\n" "tbz x7, #0, 63f\n" - "ld1 { v27.b }[6], [x20]\n" + "ld1 { v23.b }[6], [x20]\n" "b 63f\n" "60:" // Oddments: Load (4, 1): Bit 2: Bit 1: Unset "tbz x7, #0, 63f\n" - "ld1 { v27.b }[4], [x20]\n" + "ld1 { v23.b }[4], [x20]\n" "b 63f\n" "61:" // Oddments: Load (4, 1): Bit 2: Unset "tbz x7, #1, 62f\n" - "ld1 { v27.h }[0], [x20], #0x2\n" + "ld1 { v23.h }[0], [x20], #0x2\n" "tbz x7, #0, 63f\n" - "ld1 { v27.b }[2], [x20]\n" + "ld1 { v23.b }[2], [x20]\n" "b 63f\n" "62:" // Oddments: Load (4, 1): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 63f\n" - "ld1 { v27.b }[0], [x20]\n" + "ld1 { v23.b }[0], [x20]\n" "63:" // Oddments: Load (4, 1): Bit 2: End - "ushll v27.8h, v27.8b, #0x0\n" - "ldr x23, [x15, #0xa8]\n" - "smlal v11.4s, v27.4h, v7.4h\n" - "smlal2 v10.4s, v27.8h, v7.8h\n" - "add x23, x23, x17\n" + "ushll v23.8h, v23.8b, #0x0\n" + "ldr x20, [x15, #0xa8]\n" + "smlal v20.4s, v23.4h, v9.4h\n" + "smlal2 v1.4s, v23.8h, v9.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 65f\n" - "ld1 { v24.s }[0], [x23], #0x4\n" + "ld1 { v12.s }[0], [x20], #0x4\n" "tbz x7, #1, 64f\n" - "ld1 { v24.h }[2], [x23], #0x2\n" + "ld1 { v12.h }[2], [x20], #0x2\n" "tbz x7, #0, 67f\n" - "ld1 { v24.b }[6], [x23]\n" + "ld1 { v12.b }[6], [x20]\n" "b 67f\n" "64:" // Oddments: Load (3, 2): Bit 2: Bit 1: Unset "tbz x7, #0, 67f\n" - "ld1 { v24.b }[4], [x23]\n" + "ld1 { v12.b }[4], [x20]\n" "b 67f\n" "65:" // Oddments: Load (3, 2): Bit 2: Unset "tbz x7, #1, 66f\n" - "ld1 { v24.h }[0], [x23], #0x2\n" + "ld1 { v12.h }[0], [x20], #0x2\n" "tbz x7, #0, 67f\n" - "ld1 { v24.b }[2], [x23]\n" + "ld1 { v12.b }[2], [x20]\n" "b 67f\n" "66:" // Oddments: Load (3, 2): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 67f\n" - "ld1 { v24.b }[0], [x23]\n" + "ld1 { v12.b }[0], [x20]\n" "67:" // Oddments: Load (3, 2): Bit 2: End - "ushll v24.8h, v24.8b, #0x0\n" - "ldr x22, [x15, #0xb0]\n" - "smlal v11.4s, v24.4h, v5.4h\n" - "smlal2 v10.4s, v24.8h, v5.8h\n" - "smlal v9.4s, v24.4h, v3.4h\n" - "smlal2 v22.4s, v24.8h, v3.8h\n" - "add x22, x22, x17\n" + "ushll v12.8h, v12.8b, #0x0\n" + "ldr x20, [x15, #0xb0]\n" + "smlal v20.4s, v12.4h, v7.4h\n" + "smlal2 v1.4s, v12.8h, v7.8h\n" + "smlal v16.4s, v12.4h, v24.4h\n" + "smlal2 v14.4s, v12.8h, v24.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 69f\n" - "ld1 { v26.s }[0], [x22], #0x4\n" + "ld1 { v10.s }[0], [x20], #0x4\n" "tbz x7, #1, 68f\n" - "ld1 { v26.h }[2], [x22], #0x2\n" + "ld1 { v10.h }[2], [x20], #0x2\n" "tbz x7, #0, 71f\n" - "ld1 { v26.b }[6], [x22]\n" + "ld1 { v10.b }[6], [x20]\n" "b 71f\n" "68:" // Oddments: Load (4, 3): Bit 2: Bit 1: Unset "tbz x7, #0, 71f\n" - "ld1 { v26.b }[4], [x22]\n" + "ld1 { v10.b }[4], [x20]\n" "b 71f\n" "69:" // Oddments: Load (4, 3): Bit 2: Unset "tbz x7, #1, 70f\n" - "ld1 { v26.h }[0], [x22], #0x2\n" + "ld1 { v10.h }[0], [x20], #0x2\n" "tbz x7, #0, 71f\n" - "ld1 { v26.b }[2], [x22]\n" + "ld1 { v10.b }[2], [x20]\n" "b 71f\n" "70:" // Oddments: Load (4, 3): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 71f\n" - "ld1 { v26.b }[0], [x22]\n" + "ld1 { v10.b }[0], [x20]\n" "71:" // Oddments: Load (4, 3): Bit 2: End - "ushll v26.8h, v26.8b, #0x0\n" - "ldr x21, [x15, #0xb8]\n" - "smlal v9.4s, v26.4h, v7.4h\n" - "smlal2 v22.4s, v26.8h, v7.8h\n" - "add x21, x21, x17\n" + "ushll v10.8h, v10.8b, #0x0\n" + "ldr x20, [x15, #0xb8]\n" + "smlal v16.4s, v10.4h, v9.4h\n" + "smlal2 v14.4s, v10.8h, v9.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 73f\n" - "ld1 { v25.s }[0], [x21], #0x4\n" + "ld1 { v15.s }[0], [x20], #0x4\n" "tbz x7, #1, 72f\n" - "ld1 { v25.h }[2], [x21], #0x2\n" + "ld1 { v15.h }[2], [x20], #0x2\n" "tbz x7, #0, 75f\n" - "ld1 { v25.b }[6], [x21]\n" + "ld1 { v15.b }[6], [x20]\n" "b 75f\n" "72:" // Oddments: Load (4, 2): Bit 2: Bit 1: Unset "tbz x7, #0, 75f\n" - "ld1 { v25.b }[4], [x21]\n" + "ld1 { v15.b }[4], [x20]\n" "b 75f\n" "73:" // Oddments: Load (4, 2): Bit 2: Unset "tbz x7, #1, 74f\n" - "ld1 { v25.h }[0], [x21], #0x2\n" + "ld1 { v15.h }[0], [x20], #0x2\n" "tbz x7, #0, 75f\n" - "ld1 { v25.b }[2], [x21]\n" + "ld1 { v15.b }[2], [x20]\n" "b 75f\n" "74:" // Oddments: Load (4, 2): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 75f\n" - "ld1 { v25.b }[0], [x21]\n" + "ld1 { v15.b }[0], [x20]\n" "75:" // Oddments: Load (4, 2): Bit 2: End - "ushll v25.8h, v25.8b, #0x0\n" + "ushll v15.8h, v15.8b, #0x0\n" "ldr x20, [x15, #0xc0]\n" - "smlal v11.4s, v25.4h, v8.4h\n" - "smlal2 v10.4s, v25.8h, v8.8h\n" - "smlal v9.4s, v25.4h, v6.4h\n" - "smlal2 v22.4s, v25.8h, v6.8h\n" + "smlal v20.4s, v15.4h, v30.4h\n" + "smlal2 v1.4s, v15.8h, v30.8h\n" + "smlal v16.4s, v15.4h, v3.4h\n" + "smlal2 v14.4s, v15.8h, v3.8h\n" "add x20, x20, x17\n" "tbz x7, #2, 77f\n" - "ld1 { v29.s }[0], [x20], #0x4\n" + "ld1 { v28.s }[0], [x20], #0x4\n" "tbz x7, #1, 76f\n" - "ld1 { v29.h }[2], [x20], #0x2\n" + "ld1 { v28.h }[2], [x20], #0x2\n" "tbz x7, #0, 79f\n" - "ld1 { v29.b }[6], [x20]\n" + "ld1 { v28.b }[6], [x20]\n" "b 79f\n" "76:" // Oddments: Load (4, 4): Bit 2: Bit 1: Unset "tbz x7, #0, 79f\n" - "ld1 { v29.b }[4], [x20]\n" + "ld1 { v28.b }[4], [x20]\n" "b 79f\n" "77:" // Oddments: Load (4, 4): Bit 2: Unset "tbz x7, #1, 78f\n" - "ld1 { v29.h }[0], [x20], #0x2\n" + "ld1 { v28.h }[0], [x20], #0x2\n" "tbz x7, #0, 79f\n" - "ld1 { v29.b }[2], [x20]\n" + "ld1 { v28.b }[2], [x20]\n" "b 79f\n" "78:" // Oddments: Load (4, 4): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 79f\n" - "ld1 { v29.b }[0], [x20]\n" + "ld1 { v28.b }[0], [x20]\n" "79:" // Oddments: Load (4, 4): Bit 2: End - "ushll v29.8h, v29.8b, #0x0\n" - "smlal v9.4s, v29.4h, v8.4h\n" - "smlal2 v22.4s, v29.8h, v8.8h\n" + "ushll v28.8h, v28.8b, #0x0\n" + "smlal v16.4s, v28.4h, v30.4h\n" + "smlal2 v14.4s, v28.8h, v30.8h\n" "tbz x7, #2, 81f\n" - "ld1 { v18.4s }, [x13], #0x10\n" - "ld1 { v21.4s }, [x12], #0x10\n" + "ld1 { v19.4s }, [x13], #0x10\n" + "ld1 { v23.4s }, [x12], #0x10\n" "tbz x7, #1, 80f\n" - "ld1 { v30.d }[0], [x13], #0x8\n" - "ld1 { v31.d }[0], [x12], #0x8\n" + "ld1 { v18.d }[0], [x13], #0x8\n" + "ld1 { v24.d }[0], [x12], #0x8\n" "tbz x7, #0, 83f\n" - "ld1 { v30.s }[2], [x13]\n" - "ld1 { v31.s }[2], [x12]\n" + "ld1 { v18.s }[2], [x13]\n" + "ld1 { v24.s }[2], [x12]\n" "b 83f\n" "80:" // Oddments: Load requant params: Bit 2: Bit 1: Unset "tbz x7, #0, 83f\n" - "ld1 { v30.s }[0], [x13]\n" - "ld1 { v31.s }[0], [x12]\n" + "ld1 { v18.s }[0], [x13]\n" + "ld1 { v24.s }[0], [x12]\n" "b 83f\n" "81:" // Oddments: Load requant params: Bit 2: Unset "tbz x7, #1, 82f\n" - "ld1 { v18.d }[0], [x13], #0x8\n" - "ld1 { v21.d }[0], [x12], #0x8\n" + "ld1 { v19.d }[0], [x13], #0x8\n" + "ld1 { v23.d }[0], [x12], #0x8\n" "tbz x7, #0, 83f\n" - "ld1 { v18.s }[2], [x13]\n" - "ld1 { v21.s }[2], [x12]\n" + "ld1 { v19.s }[2], [x13]\n" + "ld1 { v23.s }[2], [x12]\n" "b 83f\n" "82:" // Oddments: Load requant params: Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 83f\n" - "ld1 { v18.s }[0], [x13]\n" - "ld1 { v21.s }[0], [x12]\n" + "ld1 { v19.s }[0], [x13]\n" + "ld1 { v23.s }[0], [x12]\n" "83:" // Oddments: Load requant params: Bit 2: End - "sqrdmulh v15.4s, v15.4s, v18.4s\n" - "and v2.16b, v15.16b, v21.16b\n" + "sqrdmulh v8.4s, v8.4s, v19.4s\n" + "and v17.16b, v8.16b, v23.16b\n" "add x11, x11, x16\n" "add x10, x10, x16\n" - "sqrdmulh v13.4s, v13.4s, v30.4s\n" - "sshr v2.4s, v2.4s, #0x1f\n" + "sqrdmulh v2.4s, v2.4s, v18.4s\n" + "sshr v17.4s, v17.4s, #0x1f\n" "add x9, x9, x16\n" "add x28, x28, x16\n" - "and v16.16b, v13.16b, v31.16b\n" - "sqrdmulh v17.4s, v17.4s, v18.4s\n" - "sqrdmulh v11.4s, v11.4s, v18.4s\n" - "sqrdmulh v9.4s, v9.4s, v18.4s\n" - "sqadd v15.4s, v15.4s, v2.4s\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "and v18.16b, v17.16b, v21.16b\n" - "sqrdmulh v20.4s, v20.4s, v30.4s\n" - "and v28.16b, v11.16b, v21.16b\n" - "sqrdmulh v10.4s, v10.4s, v30.4s\n" - "and v2.16b, v9.16b, v21.16b\n" - "sqrdmulh v22.4s, v22.4s, v30.4s\n" - "sqadd v13.4s, v13.4s, v16.4s\n" - "sshr v18.4s, v18.4s, #0x1f\n" - "and v4.16b, v20.16b, v31.16b\n" + "and v11.16b, v2.16b, v24.16b\n" + "sqrdmulh v21.4s, v21.4s, v19.4s\n" + "sqrdmulh v20.4s, v20.4s, v19.4s\n" + "sqrdmulh v16.4s, v16.4s, v19.4s\n" + "sqadd v8.4s, v8.4s, v17.4s\n" + "sshr v11.4s, v11.4s, #0x1f\n" + "and v28.16b, v21.16b, v23.16b\n" + "sqrdmulh v4.4s, v4.4s, v18.4s\n" + "and v17.16b, v20.16b, v23.16b\n" + "sqrdmulh v1.4s, v1.4s, v18.4s\n" + "and v19.16b, v16.16b, v23.16b\n" + "sqrdmulh v14.4s, v14.4s, v18.4s\n" + "sqadd v2.4s, v2.4s, v11.4s\n" "sshr v28.4s, v28.4s, #0x1f\n" - "and v3.16b, v10.16b, v31.16b\n" - "sshr v2.4s, v2.4s, #0x1f\n" - "and v16.16b, v22.16b, v31.16b\n" - "sqadd v17.4s, v17.4s, v18.4s\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v11.4s, v11.4s, v28.4s\n" - "sshr v3.4s, v3.4s, #0x1f\n" - "sqadd v9.4s, v9.4s, v2.4s\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "srshl v15.4s, v15.4s, v21.4s\n" - "srshl v17.4s, v17.4s, v21.4s\n" - "sqadd v20.4s, v20.4s, v4.4s\n" - "srshl v11.4s, v11.4s, v21.4s\n" - "sqadd v10.4s, v10.4s, v3.4s\n" - "srshl v9.4s, v9.4s, v21.4s\n" - "sqadd v22.4s, v22.4s, v16.4s\n" - "srshl v13.4s, v13.4s, v31.4s\n" - "sqxtn v15.4h, v15.4s\n" - "srshl v20.4s, v20.4s, v31.4s\n" - "sqxtn v17.4h, v17.4s\n" - "srshl v10.4s, v10.4s, v31.4s\n" - "sqxtn v11.4h, v11.4s\n" - "srshl v22.4s, v22.4s, v31.4s\n" - "sqxtn v9.4h, v9.4s\n" - "sqxtn2 v15.8h, v13.4s\n" - "sqxtn2 v17.8h, v20.4s\n" - "sqxtn2 v11.8h, v10.4s\n" - "sqxtn2 v9.8h, v22.4s\n" - "sqadd v15.8h, v15.8h, v12.8h\n" - "sqadd v17.8h, v17.8h, v12.8h\n" - "sqadd v11.8h, v11.8h, v12.8h\n" - "sqadd v9.8h, v9.8h, v12.8h\n" - "smax v15.8h, v15.8h, v14.8h\n" - "smax v17.8h, v17.8h, v14.8h\n" - "smax v11.8h, v11.8h, v14.8h\n" - "smax v9.8h, v9.8h, v14.8h\n" - "smin v15.8h, v15.8h, v23.8h\n" - "smin v17.8h, v17.8h, v23.8h\n" - "smin v11.8h, v11.8h, v23.8h\n" - "smin v9.8h, v9.8h, v23.8h\n" - "uzp1 v15.16b, v15.16b, v15.16b\n" - "uzp1 v17.16b, v17.16b, v17.16b\n" - "uzp1 v11.16b, v11.16b, v11.16b\n" - "uzp1 v9.16b, v9.16b, v9.16b\n" + "and v18.16b, v4.16b, v24.16b\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "and v12.16b, v1.16b, v24.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "and v25.16b, v14.16b, v24.16b\n" + "sqadd v21.4s, v21.4s, v28.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v17.4s\n" + "sshr v12.4s, v12.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v19.4s\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "srshl v8.4s, v8.4s, v23.4s\n" + "srshl v21.4s, v21.4s, v23.4s\n" + "sqadd v4.4s, v4.4s, v18.4s\n" + "srshl v20.4s, v20.4s, v23.4s\n" + "sqadd v1.4s, v1.4s, v12.4s\n" + "srshl v16.4s, v16.4s, v23.4s\n" + "sqadd v14.4s, v14.4s, v25.4s\n" + "srshl v2.4s, v2.4s, v24.4s\n" + "sqxtn v8.4h, v8.4s\n" + "srshl v4.4s, v4.4s, v24.4s\n" + "sqxtn v21.4h, v21.4s\n" + "srshl v1.4s, v1.4s, v24.4s\n" + "sqxtn v20.4h, v20.4s\n" + "srshl v14.4s, v14.4s, v24.4s\n" + "sqxtn v16.4h, v16.4s\n" + "sqxtn2 v8.8h, v2.4s\n" + "sqxtn2 v21.8h, v4.4s\n" + "sqxtn2 v20.8h, v1.4s\n" + "sqxtn2 v16.8h, v14.4s\n" + "sqadd v8.8h, v8.8h, v22.8h\n" + "sqadd v21.8h, v21.8h, v22.8h\n" + "sqadd v20.8h, v20.8h, v22.8h\n" + "sqadd v16.8h, v16.8h, v22.8h\n" + "smax v8.8h, v8.8h, v13.8h\n" + "smax v21.8h, v21.8h, v13.8h\n" + "smax v20.8h, v20.8h, v13.8h\n" + "smax v16.8h, v16.8h, v13.8h\n" + "smin v8.8h, v8.8h, v5.8h\n" + "smin v21.8h, v21.8h, v5.8h\n" + "smin v20.8h, v20.8h, v5.8h\n" + "smin v16.8h, v16.8h, v5.8h\n" + "uzp1 v8.16b, v8.16b, v8.16b\n" + "uzp1 v21.16b, v21.16b, v21.16b\n" + "uzp1 v20.16b, v20.16b, v20.16b\n" + "uzp1 v16.16b, v16.16b, v16.16b\n" "tbz x7, #2, 85f\n" - "st1 { v15.s }[0], [x11], #0x4\n" - "st1 { v17.s }[0], [x10], #0x4\n" - "st1 { v11.s }[0], [x9], #0x4\n" - "st1 { v9.s }[0], [x28], #0x4\n" + "st1 { v8.s }[0], [x11], #0x4\n" + "st1 { v21.s }[0], [x10], #0x4\n" + "st1 { v20.s }[0], [x9], #0x4\n" + "st1 { v16.s }[0], [x28], #0x4\n" "tbz x7, #1, 84f\n" - "st1 { v15.h }[2], [x11], #0x2\n" - "st1 { v17.h }[2], [x10], #0x2\n" - "st1 { v11.h }[2], [x9], #0x2\n" - "st1 { v9.h }[2], [x28], #0x2\n" + "st1 { v8.h }[2], [x11], #0x2\n" + "st1 { v21.h }[2], [x10], #0x2\n" + "st1 { v20.h }[2], [x9], #0x2\n" + "st1 { v16.h }[2], [x28], #0x2\n" "tbz x7, #0, 87f\n" - "st1 { v15.b }[6], [x11], #0x1\n" - "st1 { v17.b }[6], [x10], #0x1\n" - "st1 { v11.b }[6], [x9], #0x1\n" - "st1 { v9.b }[6], [x28], #0x1\n" + "st1 { v8.b }[6], [x11], #0x1\n" + "st1 { v21.b }[6], [x10], #0x1\n" + "st1 { v20.b }[6], [x9], #0x1\n" + "st1 { v16.b }[6], [x28], #0x1\n" "b 87f\n" "84:" // Oddments: Bit 2: Bit 1: Unset "tbz x7, #0, 87f\n" - "st1 { v15.b }[4], [x11], #0x1\n" - "st1 { v17.b }[4], [x10], #0x1\n" - "st1 { v11.b }[4], [x9], #0x1\n" - "st1 { v9.b }[4], [x28], #0x1\n" + "st1 { v8.b }[4], [x11], #0x1\n" + "st1 { v21.b }[4], [x10], #0x1\n" + "st1 { v20.b }[4], [x9], #0x1\n" + "st1 { v16.b }[4], [x28], #0x1\n" "b 87f\n" "85:" // Oddments: Bit 2: Unset "tbz x7, #1, 86f\n" - "st1 { v15.h }[0], [x11], #0x2\n" - "st1 { v17.h }[0], [x10], #0x2\n" - "st1 { v11.h }[0], [x9], #0x2\n" - "st1 { v9.h }[0], [x28], #0x2\n" + "st1 { v8.h }[0], [x11], #0x2\n" + "st1 { v21.h }[0], [x10], #0x2\n" + "st1 { v20.h }[0], [x9], #0x2\n" + "st1 { v16.h }[0], [x28], #0x2\n" "tbz x7, #0, 87f\n" - "st1 { v15.b }[2], [x11], #0x1\n" - "st1 { v17.b }[2], [x10], #0x1\n" - "st1 { v11.b }[2], [x9], #0x1\n" - "st1 { v9.b }[2], [x28], #0x1\n" + "st1 { v8.b }[2], [x11], #0x1\n" + "st1 { v21.b }[2], [x10], #0x1\n" + "st1 { v20.b }[2], [x9], #0x1\n" + "st1 { v16.b }[2], [x28], #0x1\n" "b 87f\n" "86:" // Oddments: Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 87f\n" - "st1 { v15.b }[0], [x11], #0x1\n" - "st1 { v17.b }[0], [x10], #0x1\n" - "st1 { v11.b }[0], [x9], #0x1\n" - "st1 { v9.b }[0], [x28], #0x1\n" + "st1 { v8.b }[0], [x11], #0x1\n" + "st1 { v21.b }[0], [x10], #0x1\n" + "st1 { v20.b }[0], [x9], #0x1\n" + "st1 { v16.b }[0], [x28], #0x1\n" "87:" // Oddments: Bit 2: End "88:" // End : diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp index 7075f58f92dd61e7089dbebe4cc38a6e3c3f8f13..f2ab5831d86522a46b5e79f3127701f0872ad3a6 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp @@ -22,8 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" - +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include @@ -35,15 +34,7 @@ namespace arm_conv { namespace depthwise { -void a64_u8qa_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( - const unsigned int, - const uint8_t *const *const, - const uint8_t *const, - const int32_t *const, - const arm_gemm::Requantize32 &, - const int32_t *const, - const int32_t *const, - uint8_t *const *const); +void a64_u8qa_nhwc_5x5_s1_output2x2_mla_depthfirst_impl(unsigned int, const uint8_t *const *, const uint8_t *, const int32_t *, const arm_gemm::Requantize32 &, const int32_t *, const int32_t *, uint8_t *const *); class a64_u8qa_nhwc_5x5_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirstStrategy { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp index eec3ba590092261e9b50e55a1e715020db1360ec..c8fe567e77ab8312e93d833366701a365acd089f 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp @@ -111,2071 +111,2071 @@ void a64_u8qa_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( requant_muls, requant_shifts, outptrs); __asm__ __volatile__( - "ldr x4, [%x[params], %[offsetof_Params_n_channels]]\n" - "ldr x22, [%x[params], %[offsetof_Params_requant]]\n" - "lsr x9, x4, #0x3\n" - "add x24, x22, %[offsetof_Requantize32_b_offset]\n" - "ld1r { v9.16b }, [x24]\n" - "ldr x25, [%x[params], %[offsetof_Params_outptrs]]\n" - "add x28, x22, %[offsetof_Requantize32_c_offset]\n" - "add x24, x22, %[offsetof_Requantize32_minval]\n" - "ld1r { v15.8h }, [x28]\n" - "ld1r { v14.8h }, [x24]\n" - "add x20, x22, %[offsetof_Requantize32_maxval]\n" - "mov x3, #0x0\n" + "ldr x2, [%x[params], %[offsetof_Params_n_channels]]\n" + "ldr x23, [%x[params], %[offsetof_Params_requant]]\n" + "lsr x3, x2, #0x3\n" + "add x20, x23, %[offsetof_Requantize32_b_offset]\n" + "ld1r { v2.16b }, [x20]\n" + "ldr x22, [%x[params], %[offsetof_Params_outptrs]]\n" + "add x21, x23, %[offsetof_Requantize32_c_offset]\n" + "add x20, x23, %[offsetof_Requantize32_minval]\n" + "ld1r { v25.8h }, [x21]\n" "ld1r { v12.8h }, [x20]\n" - "mov x1, #0x0\n" - "add x2, %x[params], %[offsetof_Params_inptrs]\n" - "ldr x0, [%x[params], %[offsetof_Params_weights]]\n" - "ldr x6, [%x[params], %[offsetof_Params_requant_muls]]\n" - "ldr x5, [%x[params], %[offsetof_Params_requant_shifts]]\n" - "ldp x7, x8, [x25, #0x0]\n" - "ldp x17, x16, [x25, #0x10]\n" - "cbz x9, 3f\n" - "ldr d0, [x0, #0x0]\n" - "ldr d1, [x0, #0x8]\n" - "subs x9, x9, #0x1\n" - "usubl v0.8h, v0.8b, v9.8b\n" - "ldr d2, [x0, #0x10]\n" - "ldr d3, [x0, #0x18]\n" - "usubl v1.8h, v1.8b, v9.8b\n" - "usubl v2.8h, v2.8b, v9.8b\n" - "ldr d4, [x0, #0x20]\n" - "ldr x13, [%x[params], %[offsetof_Params_bias]]\n" - "usubl v3.8h, v3.8b, v9.8b\n" - "usubl v4.8h, v4.8b, v9.8b\n" - "ldr q11, [x13, #0x0]\n" - "ldr q13, [x13, #0x10]\n" - "add x13, x13, #0x20\n" - "str x13, [%x[params], %[offsetof_Params_bias]]\n" - "ldp x10, x28, [x2, #0x0]\n" - "ldp x27, x26, [x2, #0x10]\n" - "mov v20.16b, v11.16b\n" - "mov v19.16b, v13.16b\n" - "ldp x25, x24, [x2, #0x20]\n" - "ldp x23, x22, [x2, #0x30]\n" - "mov v8.16b, v11.16b\n" + "add x20, x23, %[offsetof_Requantize32_maxval]\n" + "mov x4, #0x0\n" + "ld1r { v26.8h }, [x20]\n" + "mov x5, #0x0\n" + "add x6, %x[params], %[offsetof_Params_inptrs]\n" + "ldr x7, [%x[params], %[offsetof_Params_weights]]\n" + "ldr x8, [%x[params], %[offsetof_Params_requant_muls]]\n" + "ldr x17, [%x[params], %[offsetof_Params_requant_shifts]]\n" + "ldp x16, x15, [x22, #0x0]\n" + "ldp x14, x13, [x22, #0x10]\n" + "cbz x3, 3f\n" + "ldr d21, [x7, #0x0]\n" + "ldr d15, [x7, #0x8]\n" + "subs x3, x3, #0x1\n" + "usubl v21.8h, v21.8b, v2.8b\n" + "ldr d29, [x7, #0x10]\n" + "ldr d18, [x7, #0x18]\n" + "usubl v15.8h, v15.8b, v2.8b\n" + "usubl v29.8h, v29.8b, v2.8b\n" + "ldr d3, [x7, #0x20]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "usubl v18.8h, v18.8b, v2.8b\n" + "usubl v3.8h, v3.8b, v2.8b\n" + "ldr q13, [x20, #0x0]\n" + "ldr q24, [x20, #0x10]\n" + "add x20, x20, #0x20\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" + "ldp x9, x28, [x6, #0x0]\n" + "ldp x27, x26, [x6, #0x10]\n" "mov v7.16b, v13.16b\n" - "ldp x21, x20, [x2, #0x40]\n" - "ldr d31, [x10, x3]\n" - "mov v6.16b, v11.16b\n" - "mov v5.16b, v13.16b\n" - "ldr d30, [x28, x3]\n" - "ldr d29, [x27, x3]\n" - "ushll v31.8h, v31.8b, #0x0\n" + "mov v14.16b, v24.16b\n" + "ldp x25, x24, [x6, #0x20]\n" + "ldp x23, x22, [x6, #0x30]\n" + "mov v27.16b, v13.16b\n" + "mov v22.16b, v24.16b\n" + "ldp x21, x20, [x6, #0x40]\n" + "ldr d10, [x9, x4]\n" + "mov v8.16b, v13.16b\n" + "mov v17.16b, v24.16b\n" + "ldr d16, [x28, x4]\n" + "ldr d23, [x27, x4]\n" + "ushll v10.8h, v10.8b, #0x0\n" + "ushll v16.8h, v16.8b, #0x0\n" + "ldr d30, [x26, x4]\n" + "ldr d4, [x25, x4]\n" + "ushll v23.8h, v23.8b, #0x0\n" "ushll v30.8h, v30.8b, #0x0\n" - "ldr d28, [x26, x3]\n" - "ldr d27, [x25, x3]\n" - "ushll v29.8h, v29.8b, #0x0\n" + "ldr d28, [x24, x4]\n" + "ldr d31, [x23, x4]\n" + "ushll v4.8h, v4.8b, #0x0\n" "ushll v28.8h, v28.8b, #0x0\n" - "ldr d23, [x24, x3]\n" - "ldr d25, [x23, x3]\n" - "ushll v27.8h, v27.8b, #0x0\n" - "ushll v23.8h, v23.8b, #0x0\n" - "ldr d24, [x22, x3]\n" - "ldr d26, [x21, x3]\n" - "ushll v25.8h, v25.8b, #0x0\n" - "ushll v24.8h, v24.8b, #0x0\n" - "ldr d22, [x20, x3]\n" - "ushll v26.8h, v26.8b, #0x0\n" - "ushll v22.8h, v22.8b, #0x0\n" + "ldr d1, [x22, x4]\n" + "ldr d9, [x21, x4]\n" + "ushll v31.8h, v31.8b, #0x0\n" + "ushll v1.8h, v1.8b, #0x0\n" + "ldr d11, [x20, x4]\n" + "ushll v9.8h, v9.8b, #0x0\n" + "ushll v11.8h, v11.8b, #0x0\n" "beq 2f\n" "1:" // Loop - "ldr q18, [x6, #0x0]\n" - "ldr q21, [x5, #0x0]\n" - "smlal v11.4s, v31.4h, v0.4h\n" - "smlal2 v13.4s, v31.8h, v0.8h\n" - "ldr q16, [x6, #0x10]\n" - "ldr q10, [x5, #0x10]\n" - "smlal v11.4s, v30.4h, v1.4h\n" - "smlal v20.4s, v30.4h, v0.4h\n" - "ldr x20, [x2, #0x50]\n" - "smlal v8.4s, v29.4h, v0.4h\n" - "smlal v6.4s, v28.4h, v0.4h\n" - "ldr x22, [x2, #0x58]\n" - "smlal2 v13.4s, v30.8h, v1.8h\n" - "smlal2 v19.4s, v30.8h, v0.8h\n" - "ldr d31, [x20, x3]\n" + "ldr d5, [x7, #0x28]\n" + "ldr d6, [x7, #0x30]\n" + "smlal v13.4s, v10.4h, v21.4h\n" + "smlal2 v24.4s, v10.8h, v21.8h\n" + "ldr d19, [x7, #0x38]\n" + "ldr d0, [x7, #0x40]\n" + "smlal v13.4s, v16.4h, v15.4h\n" + "smlal v7.4s, v16.4h, v21.4h\n" + "ldr d10, [x7, #0x48]\n" + "ldr d20, [x7, #0x50]\n" + "smlal v27.4s, v23.4h, v21.4h\n" + "smlal v8.4s, v30.4h, v21.4h\n" + "ldr x21, [x6, #0x50]\n" + "smlal2 v24.4s, v16.8h, v15.8h\n" + "smlal v13.4s, v4.4h, v29.4h\n" + "ldr x20, [x6, #0x58]\n" + "smlal2 v14.4s, v16.8h, v21.8h\n" + "ldr d16, [x21, x4]\n" + "smlal2 v22.4s, v23.8h, v21.8h\n" + "ushll v16.8h, v16.8b, #0x0\n" + "smlal2 v17.4s, v30.8h, v21.8h\n" + "ldr d21, [x20, x4]\n" + "smlal v7.4s, v4.4h, v15.4h\n" + "ldr x22, [x6, #0x60]\n" + "smlal v27.4s, v30.4h, v15.4h\n" + "smlal v8.4s, v28.4h, v15.4h\n" + "ushll v21.8h, v21.8b, #0x0\n" + "ldr x20, [x6, #0x68]\n" + "smlal2 v24.4s, v4.8h, v29.8h\n" + "smlal v13.4s, v31.4h, v18.4h\n" + "usubl v5.8h, v5.8b, v2.8b\n" + "ldr x21, [x6, #0x70]\n" + "smlal2 v14.4s, v4.8h, v15.8h\n" + "ldr d4, [x22, x4]\n" + "smlal2 v22.4s, v30.8h, v15.8h\n" + "ushll v4.8h, v4.8b, #0x0\n" + "smlal2 v17.4s, v28.8h, v15.8h\n" + "ldr d15, [x20, x4]\n" + "smlal v7.4s, v31.4h, v29.4h\n" + "usubl v6.8h, v6.8b, v2.8b\n" + "smlal v27.4s, v28.4h, v29.4h\n" + "smlal v8.4s, v16.4h, v29.4h\n" + "ushll v15.8h, v15.8b, #0x0\n" + "ldr x20, [x6, #0x78]\n" + "smlal2 v24.4s, v31.8h, v18.8h\n" + "smlal v13.4s, v1.4h, v3.4h\n" + "usubl v19.8h, v19.8b, v2.8b\n" + "ldr x22, [x6, #0x80]\n" + "smlal2 v14.4s, v31.8h, v29.8h\n" + "ldr d31, [x21, x4]\n" + "smlal2 v22.4s, v28.8h, v29.8h\n" "ushll v31.8h, v31.8b, #0x0\n" - "smlal2 v7.4s, v29.8h, v0.8h\n" - "smlal v11.4s, v27.4h, v2.4h\n" - "ldr x21, [x2, #0x60]\n" - "ldr x20, [x2, #0x68]\n" - "smlal2 v5.4s, v28.8h, v0.8h\n" - "ldr d30, [x22, x3]\n" - "smlal v20.4s, v27.4h, v1.4h\n" - "ushll v30.8h, v30.8b, #0x0\n" - "smlal v8.4s, v28.4h, v1.4h\n" - "smlal v6.4s, v23.4h, v1.4h\n" - "ldr x25, [x2, #0x70]\n" - "ldr x26, [x2, #0x78]\n" - "smlal2 v13.4s, v27.8h, v2.8h\n" - "smlal2 v19.4s, v27.8h, v1.8h\n" - "ldr d0, [x0, #0x28]\n" - "usubl v0.8h, v0.8b, v9.8b\n" - "smlal2 v7.4s, v28.8h, v1.8h\n" - "smlal v11.4s, v25.4h, v3.4h\n" - "ldr x23, [x2, #0x80]\n" - "ldr x24, [x2, #0x88]\n" - "smlal2 v5.4s, v23.8h, v1.8h\n" - "ldr d27, [x21, x3]\n" - "smlal v20.4s, v25.4h, v2.4h\n" - "ushll v27.8h, v27.8b, #0x0\n" - "smlal v8.4s, v23.4h, v2.4h\n" - "smlal v6.4s, v31.4h, v2.4h\n" - "ldr x15, [x2, #0x90]\n" - "ldr x21, [x2, #0x98]\n" - "smlal2 v13.4s, v25.8h, v3.8h\n" - "smlal2 v19.4s, v25.8h, v2.8h\n" - "ldr d1, [x0, #0x30]\n" - "usubl v1.8h, v1.8b, v9.8b\n" - "smlal2 v7.4s, v23.8h, v2.8h\n" - "smlal v11.4s, v24.4h, v4.4h\n" - "ldr x14, [x2, #0xa0]\n" - "ldr x13, [x2, #0xa8]\n" - "smlal2 v5.4s, v31.8h, v2.8h\n" - "ldr d25, [x20, x3]\n" - "smlal v20.4s, v24.4h, v3.4h\n" - "ushll v25.8h, v25.8b, #0x0\n" - "smlal v8.4s, v31.4h, v3.4h\n" - "smlal v6.4s, v30.4h, v3.4h\n" - "ldr x12, [x2, #0xb0]\n" - "ldr x20, [x2, #0xb8]\n" - "smlal2 v13.4s, v24.8h, v4.8h\n" - "smlal2 v19.4s, v24.8h, v3.8h\n" - "ldr d2, [x0, #0x38]\n" - "usubl v2.8h, v2.8b, v9.8b\n" - "smlal2 v7.4s, v31.8h, v3.8h\n" - "smlal v11.4s, v29.4h, v0.4h\n" - "ldr x11, [x2, #0xc0]\n" - "ldr x10, [x2, #0xc8]\n" - "smlal2 v5.4s, v30.8h, v3.8h\n" - "ldr d24, [x25, x3]\n" - "smlal v20.4s, v27.4h, v4.4h\n" - "ushll v24.8h, v24.8b, #0x0\n" - "smlal v8.4s, v30.4h, v4.4h\n" - "smlal v6.4s, v26.4h, v4.4h\n" - "ldr x22, [x2, #0xd0]\n" - "ldr x28, [x2, #0xd8]\n" - "smlal2 v13.4s, v29.8h, v0.8h\n" - "ldr d3, [x0, #0x40]\n" - "smlal2 v19.4s, v27.8h, v4.8h\n" - "ldr d27, [x26, x3]\n" - "smlal2 v7.4s, v30.8h, v4.8h\n" - "smlal v11.4s, v28.4h, v1.4h\n" - "usubl v3.8h, v3.8b, v9.8b\n" - "ldr x27, [x2, #0xe0]\n" - "smlal2 v5.4s, v26.8h, v4.8h\n" - "ldr d4, [x0, #0x48]\n" - "smlal v20.4s, v28.4h, v0.4h\n" - "ushll v27.8h, v27.8b, #0x0\n" - "smlal v8.4s, v22.4h, v0.4h\n" - "smlal v6.4s, v25.4h, v0.4h\n" - "usubl v4.8h, v4.8b, v9.8b\n" - "ldr x26, [x2, #0xe8]\n" - "smlal2 v13.4s, v28.8h, v1.8h\n" - "smlal2 v19.4s, v28.8h, v0.8h\n" - "ldr d28, [x24, x3]\n" - "ushll v28.8h, v28.8b, #0x0\n" - "smlal2 v7.4s, v22.8h, v0.8h\n" - "smlal v11.4s, v23.4h, v2.4h\n" - "ldr x25, [x2, #0xf0]\n" - "subs x9, x9, #0x1\n" - "smlal2 v5.4s, v25.8h, v0.8h\n" - "ldr d0, [x0, #0x50]\n" - "smlal v20.4s, v23.4h, v1.4h\n" - "usubl v0.8h, v0.8b, v9.8b\n" - "smlal v8.4s, v25.4h, v1.4h\n" - "smlal v6.4s, v24.4h, v1.4h\n" - "add x6, x6, #0x20\n" - "add x5, x5, #0x20\n" - "smlal2 v13.4s, v23.8h, v2.8h\n" - "smlal2 v19.4s, v23.8h, v1.8h\n" - "ldr d23, [x23, x3]\n" + "smlal2 v17.4s, v16.8h, v29.8h\n" + "ldr d29, [x20, x4]\n" + "smlal v7.4s, v1.4h, v18.4h\n" + "usubl v0.8h, v0.8b, v2.8b\n" + "smlal v27.4s, v16.4h, v18.4h\n" + "smlal v8.4s, v21.4h, v18.4h\n" + "ushll v29.8h, v29.8b, #0x0\n" + "ldr x20, [x6, #0x88]\n" + "smlal2 v24.4s, v1.8h, v3.8h\n" + "smlal v13.4s, v23.4h, v5.4h\n" + "usubl v10.8h, v10.8b, v2.8b\n" + "ldr x21, [x6, #0x90]\n" + "smlal2 v14.4s, v1.8h, v18.8h\n" + "ldr d1, [x22, x4]\n" + "smlal2 v22.4s, v16.8h, v18.8h\n" + "ushll v1.8h, v1.8b, #0x0\n" + "smlal2 v17.4s, v21.8h, v18.8h\n" + "ldr d18, [x20, x4]\n" + "smlal v7.4s, v4.4h, v3.4h\n" + "usubl v20.8h, v20.8b, v2.8b\n" + "smlal v27.4s, v21.4h, v3.4h\n" + "smlal v8.4s, v9.4h, v3.4h\n" + "ldr x20, [x6, #0x98]\n" + "ushll v18.8h, v18.8b, #0x0\n" + "smlal2 v24.4s, v23.8h, v5.8h\n" + "ldr d23, [x7, #0x58]\n" + "smlal v13.4s, v30.4h, v6.4h\n" + "usubl v23.8h, v23.8b, v2.8b\n" + "smlal2 v14.4s, v4.8h, v3.8h\n" + "ldr d4, [x21, x4]\n" + "smlal2 v22.4s, v21.8h, v3.8h\n" + "ldr x23, [x6, #0xa0]\n" + "smlal2 v17.4s, v9.8h, v3.8h\n" + "ldr d3, [x20, x4]\n" + "smlal v7.4s, v30.4h, v5.4h\n" + "ushll v4.8h, v4.8b, #0x0\n" + "smlal v27.4s, v11.4h, v5.4h\n" + "smlal v8.4s, v15.4h, v5.4h\n" + "ushll v3.8h, v3.8b, #0x0\n" + "ldr x22, [x6, #0xa8]\n" + "smlal2 v24.4s, v30.8h, v6.8h\n" + "smlal v13.4s, v28.4h, v19.4h\n" + "ldr x21, [x6, #0xb0]\n" + "ldr x20, [x6, #0xb8]\n" + "smlal2 v14.4s, v30.8h, v5.8h\n" + "ldr d30, [x7, #0x60]\n" + "smlal2 v22.4s, v11.8h, v5.8h\n" + "usubl v30.8h, v30.8b, v2.8b\n" + "smlal2 v17.4s, v15.8h, v5.8h\n" + "ldr d5, [x23, x4]\n" + "smlal v7.4s, v28.4h, v6.4h\n" + "ushll v5.8h, v5.8b, #0x0\n" + "smlal v27.4s, v15.4h, v6.4h\n" + "smlal v8.4s, v31.4h, v6.4h\n" + "ldr x12, [x6, #0xc0]\n" + "ldr x11, [x6, #0xc8]\n" + "smlal2 v24.4s, v28.8h, v19.8h\n" + "smlal v13.4s, v16.4h, v0.4h\n" + "ldr x10, [x6, #0xd0]\n" + "ldr x9, [x6, #0xd8]\n" + "smlal2 v14.4s, v28.8h, v6.8h\n" + "ldr d28, [x7, #0x68]\n" + "smlal2 v22.4s, v15.8h, v6.8h\n" + "usubl v28.8h, v28.8b, v2.8b\n" + "smlal2 v17.4s, v31.8h, v6.8h\n" + "ldr d6, [x22, x4]\n" + "smlal v7.4s, v16.4h, v19.4h\n" + "ushll v6.8h, v6.8b, #0x0\n" + "smlal v27.4s, v31.4h, v19.4h\n" + "smlal v8.4s, v29.4h, v19.4h\n" + "ldr x28, [x6, #0xe0]\n" + "ldr x27, [x6, #0xe8]\n" + "smlal2 v24.4s, v16.8h, v0.8h\n" + "smlal v13.4s, v21.4h, v10.4h\n" + "ldr x26, [x6, #0xf0]\n" + "ldr x25, [x6, #0xf8]\n" + "smlal2 v14.4s, v16.8h, v19.8h\n" + "ldr d16, [x7, #0x70]\n" + "smlal2 v22.4s, v31.8h, v19.8h\n" + "usubl v16.8h, v16.8b, v2.8b\n" + "smlal2 v17.4s, v29.8h, v19.8h\n" + "ldr d19, [x21, x4]\n" + "smlal v7.4s, v21.4h, v0.4h\n" + "ushll v19.8h, v19.8b, #0x0\n" + "smlal v27.4s, v29.4h, v0.4h\n" + "smlal v8.4s, v1.4h, v0.4h\n" + "ldr x24, [x6, #0x100]\n" + "ldr x23, [x6, #0x108]\n" + "smlal2 v24.4s, v21.8h, v10.8h\n" + "smlal v13.4s, v11.4h, v20.4h\n" + "ldr x22, [x6, #0x110]\n" + "ldr x21, [x6, #0x118]\n" + "smlal2 v14.4s, v21.8h, v0.8h\n" + "ldr d21, [x7, #0x78]\n" + "smlal2 v22.4s, v29.8h, v0.8h\n" + "usubl v21.8h, v21.8b, v2.8b\n" + "smlal2 v17.4s, v1.8h, v0.8h\n" + "ldr d0, [x20, x4]\n" + "smlal v7.4s, v9.4h, v10.4h\n" + "ushll v0.8h, v0.8b, #0x0\n" + "smlal v27.4s, v1.4h, v10.4h\n" + "smlal v8.4s, v18.4h, v10.4h\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "subs x3, x3, #0x1\n" + "smlal2 v24.4s, v11.8h, v20.8h\n" + "ldr d11, [x7, #0x80]\n" + "smlal v13.4s, v15.4h, v23.4h\n" + "usubl v11.8h, v11.8b, v2.8b\n" + "smlal2 v14.4s, v9.8h, v10.8h\n" + "ldr d9, [x12, x4]\n" + "smlal2 v22.4s, v1.8h, v10.8h\n" + "ushll v9.8h, v9.8b, #0x0\n" + "smlal2 v17.4s, v18.8h, v10.8h\n" + "ldr d10, [x11, x4]\n" + "smlal v7.4s, v15.4h, v20.4h\n" + "ushll v10.8h, v10.8b, #0x0\n" + "smlal v27.4s, v4.4h, v20.4h\n" + "smlal v8.4s, v3.4h, v20.4h\n" + "smlal2 v24.4s, v15.8h, v23.8h\n" + "smlal v13.4s, v31.4h, v30.4h\n" + "smlal2 v14.4s, v15.8h, v20.8h\n" + "ldr d15, [x7, #0x88]\n" + "smlal2 v22.4s, v4.8h, v20.8h\n" + "usubl v15.8h, v15.8b, v2.8b\n" + "smlal2 v17.4s, v3.8h, v20.8h\n" + "ldr d20, [x10, x4]\n" + "smlal v7.4s, v31.4h, v23.4h\n" + "ushll v20.8h, v20.8b, #0x0\n" + "smlal v27.4s, v3.4h, v23.4h\n" + "smlal v8.4s, v5.4h, v23.4h\n" + "smlal2 v24.4s, v31.8h, v30.8h\n" + "smlal v13.4s, v29.4h, v28.4h\n" + "smlal2 v14.4s, v31.8h, v23.8h\n" + "ldr d31, [x7, #0x90]\n" + "smlal2 v22.4s, v3.8h, v23.8h\n" + "usubl v31.8h, v31.8b, v2.8b\n" + "smlal2 v17.4s, v5.8h, v23.8h\n" + "ldr d23, [x9, x4]\n" + "smlal v7.4s, v29.4h, v30.4h\n" "ushll v23.8h, v23.8b, #0x0\n" - "smlal2 v7.4s, v25.8h, v1.8h\n" - "smlal v11.4s, v31.4h, v3.4h\n" - "ldr x24, [x2, #0xf8]\n" - "smlal2 v5.4s, v24.8h, v1.8h\n" - "ldr d1, [x0, #0x58]\n" - "smlal v20.4s, v31.4h, v2.4h\n" - "usubl v1.8h, v1.8b, v9.8b\n" - "smlal v8.4s, v24.4h, v2.4h\n" - "smlal v6.4s, v27.4h, v2.4h\n" - "smlal2 v13.4s, v31.8h, v3.8h\n" - "smlal2 v19.4s, v31.8h, v2.8h\n" - "ldr d31, [x15, x3]\n" - "ushll v31.8h, v31.8b, #0x0\n" - "smlal2 v7.4s, v24.8h, v2.8h\n" - "smlal v11.4s, v30.4h, v4.4h\n" - "ldr x23, [x2, #0x100]\n" - "smlal2 v5.4s, v27.8h, v2.8h\n" - "ldr d2, [x0, #0x60]\n" - "smlal v20.4s, v30.4h, v3.4h\n" - "usubl v2.8h, v2.8b, v9.8b\n" - "smlal v8.4s, v27.4h, v3.4h\n" - "smlal v6.4s, v23.4h, v3.4h\n" - "smlal2 v13.4s, v30.8h, v4.8h\n" - "smlal2 v19.4s, v30.8h, v3.8h\n" - "ldr d30, [x21, x3]\n" + "smlal v27.4s, v5.4h, v30.4h\n" + "smlal v8.4s, v6.4h, v30.4h\n" + "smlal2 v24.4s, v29.8h, v28.8h\n" + "smlal v13.4s, v1.4h, v16.4h\n" + "smlal2 v14.4s, v29.8h, v30.8h\n" + "ldr d29, [x7, #0x98]\n" + "smlal2 v22.4s, v5.8h, v30.8h\n" + "usubl v29.8h, v29.8b, v2.8b\n" + "smlal2 v17.4s, v6.8h, v30.8h\n" + "ldr d30, [x28, x4]\n" + "smlal v7.4s, v1.4h, v28.4h\n" "ushll v30.8h, v30.8b, #0x0\n" - "smlal2 v7.4s, v27.8h, v3.8h\n" - "smlal v11.4s, v22.4h, v0.4h\n" - "ldr x15, [x2, #0x108]\n" - "smlal2 v5.4s, v23.8h, v3.8h\n" - "ldr d3, [x0, #0x68]\n" - "smlal v20.4s, v26.4h, v4.4h\n" - "usubl v3.8h, v3.8b, v9.8b\n" - "smlal v8.4s, v23.4h, v4.4h\n" - "smlal v6.4s, v28.4h, v4.4h\n" - "smlal2 v13.4s, v22.8h, v0.8h\n" - "ldr d22, [x20, x3]\n" - "smlal2 v19.4s, v26.8h, v4.8h\n" - "ldr d26, [x14, x3]\n" - "smlal2 v7.4s, v23.8h, v4.8h\n" - "smlal v11.4s, v25.4h, v1.4h\n" - "ushll v26.8h, v26.8b, #0x0\n" - "ldr x21, [x2, #0x110]\n" - "smlal2 v5.4s, v28.8h, v4.8h\n" - "ldr d4, [x0, #0x70]\n" - "smlal v20.4s, v25.4h, v0.4h\n" - "usubl v4.8h, v4.8b, v9.8b\n" - "smlal v8.4s, v31.4h, v0.4h\n" - "smlal v6.4s, v30.4h, v0.4h\n" - "ushll v22.8h, v22.8b, #0x0\n" - "ldr x20, [x2, #0x118]\n" - "smlal2 v13.4s, v25.8h, v1.8h\n" - "smlal2 v19.4s, v25.8h, v0.8h\n" - "ldr d25, [x13, x3]\n" - "ushll v25.8h, v25.8b, #0x0\n" - "smlal2 v7.4s, v31.8h, v0.8h\n" - "smlal v11.4s, v24.4h, v2.4h\n" - "ldr x13, [%x[params], %[offsetof_Params_bias]]\n" - "smlal2 v5.4s, v30.8h, v0.8h\n" - "ldr d0, [x0, #0x78]\n" - "smlal v20.4s, v24.4h, v1.4h\n" - "usubl v0.8h, v0.8b, v9.8b\n" - "smlal v8.4s, v30.4h, v1.4h\n" - "smlal v6.4s, v26.4h, v1.4h\n" - "smlal2 v13.4s, v24.8h, v2.8h\n" - "smlal2 v19.4s, v24.8h, v1.8h\n" - "ldr d24, [x12, x3]\n" - "ushll v24.8h, v24.8b, #0x0\n" - "smlal2 v7.4s, v30.8h, v1.8h\n" - "smlal v11.4s, v27.4h, v3.4h\n" - "smlal2 v5.4s, v26.8h, v1.8h\n" - "ldr d1, [x0, #0x80]\n" - "smlal v20.4s, v27.4h, v2.4h\n" - "usubl v1.8h, v1.8b, v9.8b\n" - "smlal v8.4s, v26.4h, v2.4h\n" - "smlal v6.4s, v25.4h, v2.4h\n" - "smlal2 v13.4s, v27.8h, v3.8h\n" - "smlal2 v19.4s, v27.8h, v2.8h\n" - "ldr d27, [x11, x3]\n" - "ushll v27.8h, v27.8b, #0x0\n" - "smlal2 v7.4s, v26.8h, v2.8h\n" - "smlal v11.4s, v23.4h, v4.4h\n" - "smlal2 v5.4s, v25.8h, v2.8h\n" - "ldr d2, [x0, #0x88]\n" - "smlal v20.4s, v23.4h, v3.4h\n" - "usubl v2.8h, v2.8b, v9.8b\n" - "smlal v8.4s, v25.4h, v3.4h\n" - "smlal v6.4s, v24.4h, v3.4h\n" - "smlal2 v13.4s, v23.8h, v4.8h\n" - "smlal2 v19.4s, v23.8h, v3.8h\n" - "ldr d23, [x10, x3]\n" - "ushll v23.8h, v23.8b, #0x0\n" - "smlal2 v7.4s, v25.8h, v3.8h\n" - "smlal v11.4s, v31.4h, v0.4h\n" - "smlal2 v5.4s, v24.8h, v3.8h\n" - "ldr d3, [x0, #0x90]\n" - "smlal v20.4s, v28.4h, v4.4h\n" - "usubl v3.8h, v3.8b, v9.8b\n" - "smlal v8.4s, v24.4h, v4.4h\n" - "smlal v6.4s, v22.4h, v4.4h\n" - "smlal2 v13.4s, v31.8h, v0.8h\n" - "ldr d31, [x22, x3]\n" - "smlal2 v19.4s, v28.8h, v4.8h\n" - "ldr d28, [x27, x3]\n" - "smlal2 v7.4s, v24.8h, v4.8h\n" - "smlal v11.4s, v30.4h, v1.4h\n" - "ushll v31.8h, v31.8b, #0x0\n" - "smlal2 v5.4s, v22.8h, v4.8h\n" - "ldr d4, [x0, #0x98]\n" - "smlal v20.4s, v30.4h, v0.4h\n" - "usubl v4.8h, v4.8b, v9.8b\n" - "smlal v8.4s, v27.4h, v0.4h\n" - "smlal v6.4s, v23.4h, v0.4h\n" + "smlal v27.4s, v6.4h, v28.4h\n" + "smlal v8.4s, v19.4h, v28.4h\n" + "smlal2 v24.4s, v1.8h, v16.8h\n" + "smlal v13.4s, v4.4h, v21.4h\n" + "smlal2 v14.4s, v1.8h, v28.8h\n" + "ldr d1, [x7, #0xa0]\n" + "smlal2 v22.4s, v6.8h, v28.8h\n" + "usubl v1.8h, v1.8b, v2.8b\n" + "smlal2 v17.4s, v19.8h, v28.8h\n" + "ldr d28, [x27, x4]\n" + "smlal v7.4s, v18.4h, v16.4h\n" "ushll v28.8h, v28.8b, #0x0\n" - "smlal2 v13.4s, v30.8h, v1.8h\n" - "smlal2 v19.4s, v30.8h, v0.8h\n" - "ldr d30, [x28, x3]\n" - "ushll v30.8h, v30.8b, #0x0\n" - "smlal2 v7.4s, v27.8h, v0.8h\n" - "smlal v11.4s, v26.4h, v2.4h\n" - "smlal2 v5.4s, v23.8h, v0.8h\n" - "ldr d0, [x0, #0xa0]\n" - "smlal v20.4s, v26.4h, v1.4h\n" - "usubl v0.8h, v0.8b, v9.8b\n" - "smlal v8.4s, v23.4h, v1.4h\n" - "smlal v6.4s, v31.4h, v1.4h\n" - "smlal2 v13.4s, v26.8h, v2.8h\n" - "smlal2 v19.4s, v26.8h, v1.8h\n" - "ldr d26, [x26, x3]\n" - "ushll v26.8h, v26.8b, #0x0\n" - "smlal2 v7.4s, v23.8h, v1.8h\n" - "smlal v11.4s, v25.4h, v3.4h\n" - "smlal2 v5.4s, v31.8h, v1.8h\n" - "ldr d1, [x0, #0xa8]\n" - "smlal v20.4s, v25.4h, v2.4h\n" - "usubl v1.8h, v1.8b, v9.8b\n" - "smlal v8.4s, v31.4h, v2.4h\n" - "smlal v6.4s, v30.4h, v2.4h\n" - "smlal2 v13.4s, v25.8h, v3.8h\n" - "smlal2 v19.4s, v25.8h, v2.8h\n" - "ldr d25, [x25, x3]\n" - "ushll v25.8h, v25.8b, #0x0\n" - "smlal2 v7.4s, v31.8h, v2.8h\n" - "smlal v11.4s, v24.4h, v4.4h\n" - "smlal2 v5.4s, v30.8h, v2.8h\n" - "ldr d2, [x0, #0xb0]\n" - "smlal v20.4s, v24.4h, v3.4h\n" - "usubl v2.8h, v2.8b, v9.8b\n" - "smlal v8.4s, v30.4h, v3.4h\n" - "smlal v6.4s, v28.4h, v3.4h\n" - "smlal2 v13.4s, v24.8h, v4.8h\n" - "smlal2 v19.4s, v24.8h, v3.8h\n" - "ldr d24, [x24, x3]\n" - "ushll v24.8h, v24.8b, #0x0\n" - "smlal2 v7.4s, v30.8h, v3.8h\n" - "smlal v11.4s, v27.4h, v0.4h\n" - "smlal2 v5.4s, v28.8h, v3.8h\n" - "ldr d3, [x0, #0xb8]\n" - "smlal v20.4s, v22.4h, v4.4h\n" - "usubl v3.8h, v3.8b, v9.8b\n" - "smlal v8.4s, v28.4h, v4.4h\n" - "smlal v6.4s, v26.4h, v4.4h\n" - "smlal2 v13.4s, v27.8h, v0.8h\n" - "ldr d27, [x23, x3]\n" - "smlal2 v7.4s, v28.8h, v4.8h\n" - "ushll v27.8h, v27.8b, #0x0\n" - "smlal v11.4s, v23.4h, v1.4h\n" - "smlal2 v19.4s, v22.8h, v4.8h\n" - "smlal2 v5.4s, v26.8h, v4.8h\n" - "ldr d4, [x0, #0xc0]\n" - "smlal v20.4s, v23.4h, v0.4h\n" - "usubl v4.8h, v4.8b, v9.8b\n" - "smlal v8.4s, v25.4h, v0.4h\n" - "smlal v6.4s, v24.4h, v0.4h\n" - "add x0, x0, #0xc8\n" - "smlal2 v13.4s, v23.8h, v1.8h\n" - "smlal2 v7.4s, v25.8h, v0.8h\n" - "ldr d25, [x15, x3]\n" - "ushll v25.8h, v25.8b, #0x0\n" - "smlal v11.4s, v31.4h, v2.4h\n" - "smlal2 v19.4s, v23.8h, v0.8h\n" - "smlal2 v5.4s, v24.8h, v0.8h\n" - "smlal v20.4s, v31.4h, v1.4h\n" - "smlal v8.4s, v24.4h, v1.4h\n" - "smlal v6.4s, v27.4h, v1.4h\n" - "smlal2 v13.4s, v31.8h, v2.8h\n" - "smlal2 v7.4s, v24.8h, v1.8h\n" - "ldr d24, [x21, x3]\n" - "ushll v24.8h, v24.8b, #0x0\n" - "smlal v11.4s, v30.4h, v3.4h\n" - "smlal2 v19.4s, v31.8h, v1.8h\n" - "smlal2 v5.4s, v27.8h, v1.8h\n" - "smlal v20.4s, v30.4h, v2.4h\n" - "smlal v8.4s, v27.4h, v2.4h\n" - "smlal v6.4s, v25.4h, v2.4h\n" - "smlal2 v13.4s, v30.8h, v3.8h\n" - "smlal2 v7.4s, v27.8h, v2.8h\n" - "ldr d27, [x20, x3]\n" - "ushll v27.8h, v27.8b, #0x0\n" - "smlal v11.4s, v28.4h, v4.4h\n" - "smlal2 v19.4s, v30.8h, v2.8h\n" - "sqrdmulh v11.4s, v11.4s, v18.4s\n" - "add x3, x3, #0x8\n" - "smlal2 v5.4s, v25.8h, v2.8h\n" - "smlal v20.4s, v28.4h, v3.4h\n" - "and v31.16b, v11.16b, v21.16b\n" - "smlal v8.4s, v25.4h, v3.4h\n" - "smlal v6.4s, v24.4h, v3.4h\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "smlal2 v13.4s, v28.8h, v4.8h\n" - "smlal2 v19.4s, v28.8h, v3.8h\n" - "sqrdmulh v13.4s, v13.4s, v16.4s\n" - "smlal2 v7.4s, v25.8h, v3.8h\n" - "smlal2 v5.4s, v24.8h, v3.8h\n" - "and v17.16b, v13.16b, v10.16b\n" - "smlal v20.4s, v26.4h, v4.4h\n" - "smlal v8.4s, v24.4h, v4.4h\n" - "sqrdmulh v20.4s, v20.4s, v18.4s\n" - "smlal v6.4s, v27.4h, v4.4h\n" - "smlal2 v19.4s, v26.8h, v4.8h\n" - "sqrdmulh v8.4s, v8.4s, v18.4s\n" - "smlal2 v7.4s, v24.8h, v4.8h\n" - "smlal2 v5.4s, v27.8h, v4.8h\n" - "sqrdmulh v6.4s, v6.4s, v18.4s\n" - "sqadd v11.4s, v11.4s, v31.4s\n" - "sshr v17.4s, v17.4s, #0x1f\n" - "and v26.16b, v20.16b, v21.16b\n" - "sqrdmulh v19.4s, v19.4s, v16.4s\n" - "and v18.16b, v8.16b, v21.16b\n" - "sqrdmulh v7.4s, v7.4s, v16.4s\n" - "and v31.16b, v6.16b, v21.16b\n" - "sqrdmulh v5.4s, v5.4s, v16.4s\n" - "sqadd v13.4s, v13.4s, v17.4s\n" - "sshr v26.4s, v26.4s, #0x1f\n" - "and v27.16b, v19.16b, v10.16b\n" + "smlal v27.4s, v19.4h, v16.4h\n" + "smlal v8.4s, v0.4h, v16.4h\n" + "smlal2 v24.4s, v4.8h, v21.8h\n" + "ldr d4, [x7, #0xa8]\n" + "smlal v13.4s, v3.4h, v11.4h\n" + "usubl v4.8h, v4.8b, v2.8b\n" + "smlal2 v14.4s, v18.8h, v16.8h\n" + "ldr d18, [x26, x4]\n" + "smlal2 v22.4s, v19.8h, v16.8h\n" + "ushll v18.8h, v18.8b, #0x0\n" + "smlal2 v17.4s, v0.8h, v16.8h\n" + "ldr d16, [x25, x4]\n" + "smlal v7.4s, v3.4h, v21.4h\n" + "ushll v16.8h, v16.8b, #0x0\n" + "smlal v27.4s, v9.4h, v21.4h\n" + "smlal v8.4s, v10.4h, v21.4h\n" + "smlal2 v24.4s, v3.8h, v11.8h\n" + "smlal v13.4s, v5.4h, v15.4h\n" + "smlal2 v14.4s, v3.8h, v21.8h\n" + "ldr d3, [x7, #0xb0]\n" + "smlal2 v22.4s, v9.8h, v21.8h\n" + "usubl v3.8h, v3.8b, v2.8b\n" + "smlal2 v17.4s, v10.8h, v21.8h\n" + "ldr d21, [x24, x4]\n" + "smlal v7.4s, v5.4h, v11.4h\n" + "ushll v21.8h, v21.8b, #0x0\n" + "smlal v27.4s, v10.4h, v11.4h\n" + "smlal v8.4s, v20.4h, v11.4h\n" + "smlal2 v24.4s, v5.8h, v15.8h\n" + "smlal v13.4s, v6.4h, v31.4h\n" + "smlal2 v14.4s, v5.8h, v11.8h\n" + "ldr d5, [x7, #0xb8]\n" + "smlal2 v22.4s, v10.8h, v11.8h\n" + "usubl v5.8h, v5.8b, v2.8b\n" + "smlal2 v17.4s, v20.8h, v11.8h\n" + "ldr d11, [x23, x4]\n" + "smlal v7.4s, v6.4h, v15.4h\n" + "ushll v11.8h, v11.8b, #0x0\n" + "smlal v27.4s, v20.4h, v15.4h\n" + "smlal v8.4s, v23.4h, v15.4h\n" + "smlal2 v24.4s, v6.8h, v31.8h\n" + "smlal v13.4s, v19.4h, v29.4h\n" + "smlal2 v14.4s, v6.8h, v15.8h\n" + "ldr d6, [x7, #0xc0]\n" + "smlal2 v22.4s, v20.8h, v15.8h\n" + "usubl v6.8h, v6.8b, v2.8b\n" + "smlal2 v17.4s, v23.8h, v15.8h\n" + "ldr d15, [x22, x4]\n" + "smlal v7.4s, v19.4h, v31.4h\n" + "ushll v15.8h, v15.8b, #0x0\n" + "smlal v27.4s, v23.4h, v31.4h\n" + "smlal v8.4s, v30.4h, v31.4h\n" + "add x7, x7, #0xc8\n" + "smlal2 v24.4s, v19.8h, v29.8h\n" + "smlal v13.4s, v9.4h, v1.4h\n" + "smlal2 v14.4s, v19.8h, v31.8h\n" + "ldr d19, [x21, x4]\n" + "smlal2 v22.4s, v23.8h, v31.8h\n" + "ushll v19.8h, v19.8b, #0x0\n" + "smlal2 v17.4s, v30.8h, v31.8h\n" + "ldr q31, [x8, #0x0]\n" + "smlal v7.4s, v0.4h, v29.4h\n" + "add x4, x4, #0x8\n" + "smlal v27.4s, v30.4h, v29.4h\n" + "smlal v8.4s, v28.4h, v29.4h\n" + "smlal2 v24.4s, v9.8h, v1.8h\n" + "ldr q9, [x17, #0x0]\n" + "smlal v13.4s, v10.4h, v4.4h\n" + "smlal2 v14.4s, v0.8h, v29.8h\n" + "ldr q0, [x8, #0x10]\n" + "smlal2 v22.4s, v30.8h, v29.8h\n" + "add x8, x8, #0x20\n" + "smlal2 v17.4s, v28.8h, v29.8h\n" + "ldr q29, [x17, #0x10]\n" + "smlal v7.4s, v10.4h, v1.4h\n" + "add x17, x17, #0x20\n" + "smlal v27.4s, v18.4h, v1.4h\n" + "smlal v8.4s, v16.4h, v1.4h\n" + "smlal2 v24.4s, v10.8h, v4.8h\n" + "smlal v13.4s, v20.4h, v3.4h\n" + "smlal2 v14.4s, v10.8h, v1.8h\n" + "smlal2 v22.4s, v18.8h, v1.8h\n" + "smlal2 v17.4s, v16.8h, v1.8h\n" + "smlal v7.4s, v20.4h, v4.4h\n" + "smlal v27.4s, v16.4h, v4.4h\n" + "smlal v8.4s, v21.4h, v4.4h\n" + "smlal2 v24.4s, v20.8h, v3.8h\n" + "smlal v13.4s, v23.4h, v5.4h\n" + "smlal2 v14.4s, v20.8h, v4.8h\n" + "smlal2 v22.4s, v16.8h, v4.8h\n" + "smlal2 v17.4s, v21.8h, v4.8h\n" + "smlal v7.4s, v23.4h, v3.4h\n" + "smlal v27.4s, v21.4h, v3.4h\n" + "smlal v8.4s, v11.4h, v3.4h\n" + "smlal2 v24.4s, v23.8h, v5.8h\n" + "smlal v13.4s, v30.4h, v6.4h\n" + "sqrdmulh v13.4s, v13.4s, v31.4s\n" + "smlal2 v14.4s, v23.8h, v3.8h\n" + "smlal2 v22.4s, v21.8h, v3.8h\n" + "and v23.16b, v13.16b, v9.16b\n" + "smlal2 v17.4s, v11.8h, v3.8h\n" + "smlal v7.4s, v30.4h, v5.4h\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "smlal v27.4s, v11.4h, v5.4h\n" + "smlal v8.4s, v15.4h, v5.4h\n" + "sqadd v13.4s, v13.4s, v23.4s\n" + "smlal2 v24.4s, v30.8h, v6.8h\n" + "smlal2 v14.4s, v30.8h, v5.8h\n" + "sqrdmulh v24.4s, v24.4s, v0.4s\n" + "smlal2 v22.4s, v11.8h, v5.8h\n" + "smlal2 v17.4s, v15.8h, v5.8h\n" + "and v10.16b, v24.16b, v29.16b\n" + "smlal v7.4s, v28.4h, v6.4h\n" + "smlal v27.4s, v15.4h, v6.4h\n" + "sqrdmulh v7.4s, v7.4s, v31.4s\n" + "smlal v8.4s, v19.4h, v6.4h\n" + "smlal2 v14.4s, v28.8h, v6.8h\n" + "sqrdmulh v27.4s, v27.4s, v31.4s\n" + "smlal2 v22.4s, v15.8h, v6.8h\n" + "smlal2 v17.4s, v19.8h, v6.8h\n" + "sqrdmulh v8.4s, v8.4s, v31.4s\n" + "sshr v10.4s, v10.4s, #0x1f\n" + "and v28.16b, v7.16b, v9.16b\n" + "sqrdmulh v14.4s, v14.4s, v0.4s\n" + "and v20.16b, v27.16b, v9.16b\n" + "sqrdmulh v22.4s, v22.4s, v0.4s\n" + "and v23.16b, v8.16b, v9.16b\n" + "sqrdmulh v17.4s, v17.4s, v0.4s\n" + "sqadd v24.4s, v24.4s, v10.4s\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "and v18.16b, v14.16b, v29.16b\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "and v30.16b, v22.16b, v29.16b\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "and v5.16b, v17.16b, v29.16b\n" + "sqadd v7.4s, v7.4s, v28.4s\n" "sshr v18.4s, v18.4s, #0x1f\n" - "and v25.16b, v7.16b, v10.16b\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "and v17.16b, v5.16b, v10.16b\n" - "sqadd v20.4s, v20.4s, v26.4s\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v18.4s\n" - "sshr v25.4s, v25.4s, #0x1f\n" - "sqadd v6.4s, v6.4s, v31.4s\n" - "sshr v17.4s, v17.4s, #0x1f\n" - "srshl v11.4s, v11.4s, v21.4s\n" - "srshl v20.4s, v20.4s, v21.4s\n" - "sqadd v19.4s, v19.4s, v27.4s\n" - "srshl v8.4s, v8.4s, v21.4s\n" - "sqadd v7.4s, v7.4s, v25.4s\n" - "srshl v6.4s, v6.4s, v21.4s\n" - "sqadd v5.4s, v5.4s, v17.4s\n" - "srshl v13.4s, v13.4s, v10.4s\n" - "sqxtn v11.4h, v11.4s\n" - "srshl v19.4s, v19.4s, v10.4s\n" - "sqxtn v20.4h, v20.4s\n" - "srshl v7.4s, v7.4s, v10.4s\n" + "sqadd v27.4s, v27.4s, v20.4s\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sqadd v8.4s, v8.4s, v23.4s\n" + "sshr v5.4s, v5.4s, #0x1f\n" + "srshl v13.4s, v13.4s, v9.4s\n" + "srshl v7.4s, v7.4s, v9.4s\n" + "sqadd v14.4s, v14.4s, v18.4s\n" + "srshl v27.4s, v27.4s, v9.4s\n" + "sqadd v22.4s, v22.4s, v30.4s\n" + "srshl v8.4s, v8.4s, v9.4s\n" + "sqadd v17.4s, v17.4s, v5.4s\n" + "srshl v24.4s, v24.4s, v29.4s\n" + "sqxtn v13.4h, v13.4s\n" + "srshl v14.4s, v14.4s, v29.4s\n" + "sqxtn v7.4h, v7.4s\n" + "srshl v22.4s, v22.4s, v29.4s\n" + "sqxtn v27.4h, v27.4s\n" + "srshl v17.4s, v17.4s, v29.4s\n" "sqxtn v8.4h, v8.4s\n" - "srshl v5.4s, v5.4s, v10.4s\n" - "sqxtn v6.4h, v6.4s\n" - "sqxtn2 v11.8h, v13.4s\n" - "sqxtn2 v20.8h, v19.4s\n" - "sqxtn2 v8.8h, v7.4s\n" - "sqxtn2 v6.8h, v5.4s\n" - "sqadd v11.8h, v11.8h, v15.8h\n" - "sqadd v20.8h, v20.8h, v15.8h\n" - "sqadd v8.8h, v8.8h, v15.8h\n" - "sqadd v6.8h, v6.8h, v15.8h\n" - "smax v11.8h, v11.8h, v14.8h\n" - "smax v20.8h, v20.8h, v14.8h\n" - "smax v8.8h, v8.8h, v14.8h\n" - "smax v6.8h, v6.8h, v14.8h\n" - "smin v11.8h, v11.8h, v12.8h\n" - "smin v20.8h, v20.8h, v12.8h\n" - "smin v8.8h, v8.8h, v12.8h\n" - "smin v6.8h, v6.8h, v12.8h\n" - "uzp1 v11.16b, v11.16b, v11.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "str d11, [x7, x1]\n" + "sqxtn2 v13.8h, v24.4s\n" + "sqxtn2 v7.8h, v14.4s\n" + "sqxtn2 v27.8h, v22.4s\n" + "sqxtn2 v8.8h, v17.4s\n" + "sqadd v13.8h, v13.8h, v25.8h\n" + "sqadd v7.8h, v7.8h, v25.8h\n" + "sqadd v27.8h, v27.8h, v25.8h\n" + "sqadd v8.8h, v8.8h, v25.8h\n" + "smax v13.8h, v13.8h, v12.8h\n" + "smax v7.8h, v7.8h, v12.8h\n" + "smax v27.8h, v27.8h, v12.8h\n" + "smax v8.8h, v8.8h, v12.8h\n" + "smin v13.8h, v13.8h, v26.8h\n" + "smin v7.8h, v7.8h, v26.8h\n" + "smin v27.8h, v27.8h, v26.8h\n" + "smin v8.8h, v8.8h, v26.8h\n" + "uzp1 v13.16b, v13.16b, v13.16b\n" + "str d13, [x16, x5]\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" + "uzp1 v27.16b, v27.16b, v27.16b\n" + "str d7, [x15, x5]\n" "uzp1 v8.16b, v8.16b, v8.16b\n" - "uzp1 v6.16b, v6.16b, v6.16b\n" - "str d20, [x8, x1]\n" - "str d8, [x17, x1]\n" - "str d6, [x16, x1]\n" - "ldr q11, [x13, #0x0]\n" - "ldr q13, [x13, #0x10]\n" - "add x13, x13, #0x20\n" - "ldr d0, [x0, #0x0]\n" - "ldr d1, [x0, #0x8]\n" - "add x1, x1, #0x8\n" - "str x13, [%x[params], %[offsetof_Params_bias]]\n" - "ldr d2, [x0, #0x10]\n" - "ldr d3, [x0, #0x18]\n" - "mov v20.16b, v11.16b\n" - "mov v19.16b, v13.16b\n" - "ldr d4, [x0, #0x20]\n" - "ldp x10, x28, [x2, #0x0]\n" - "mov v8.16b, v11.16b\n" + "str d27, [x14, x5]\n" + "str d8, [x13, x5]\n" + "ldr q13, [x20, #0x0]\n" + "ldr q24, [x20, #0x10]\n" + "add x20, x20, #0x20\n" + "ldr d21, [x7, #0x0]\n" + "ldr d15, [x7, #0x8]\n" + "add x5, x5, #0x8\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" + "ldr d29, [x7, #0x10]\n" + "ldr d18, [x7, #0x18]\n" "mov v7.16b, v13.16b\n" - "ldp x27, x26, [x2, #0x10]\n" - "ldp x25, x24, [x2, #0x20]\n" - "mov v6.16b, v11.16b\n" - "mov v5.16b, v13.16b\n" - "ldp x23, x22, [x2, #0x30]\n" - "ldp x21, x20, [x2, #0x40]\n" - "usubl v0.8h, v0.8b, v9.8b\n" - "usubl v1.8h, v1.8b, v9.8b\n" - "ldr d31, [x10, x3]\n" - "ldr d30, [x28, x3]\n" - "usubl v2.8h, v2.8b, v9.8b\n" - "usubl v3.8h, v3.8b, v9.8b\n" - "ldr d29, [x27, x3]\n" - "ldr d28, [x26, x3]\n" - "usubl v4.8h, v4.8b, v9.8b\n" - "ushll v31.8h, v31.8b, #0x0\n" - "ldr d27, [x25, x3]\n" - "ldr d23, [x24, x3]\n" + "mov v14.16b, v24.16b\n" + "ldr d3, [x7, #0x20]\n" + "ldp x9, x28, [x6, #0x0]\n" + "mov v27.16b, v13.16b\n" + "mov v22.16b, v24.16b\n" + "ldp x27, x26, [x6, #0x10]\n" + "ldp x25, x24, [x6, #0x20]\n" + "mov v8.16b, v13.16b\n" + "mov v17.16b, v24.16b\n" + "ldp x23, x22, [x6, #0x30]\n" + "ldp x21, x20, [x6, #0x40]\n" + "usubl v21.8h, v21.8b, v2.8b\n" + "usubl v15.8h, v15.8b, v2.8b\n" + "ldr d10, [x9, x4]\n" + "ldr d16, [x28, x4]\n" + "usubl v29.8h, v29.8b, v2.8b\n" + "usubl v18.8h, v18.8b, v2.8b\n" + "ldr d23, [x27, x4]\n" + "ldr d30, [x26, x4]\n" + "usubl v3.8h, v3.8b, v2.8b\n" + "ushll v10.8h, v10.8b, #0x0\n" + "ldr d4, [x25, x4]\n" + "ldr d28, [x24, x4]\n" + "ushll v16.8h, v16.8b, #0x0\n" + "ushll v23.8h, v23.8b, #0x0\n" + "ldr d31, [x23, x4]\n" + "ldr d1, [x22, x4]\n" "ushll v30.8h, v30.8b, #0x0\n" - "ushll v29.8h, v29.8b, #0x0\n" - "ldr d25, [x23, x3]\n" - "ldr d24, [x22, x3]\n" + "ushll v4.8h, v4.8b, #0x0\n" + "ldr d9, [x21, x4]\n" + "ldr d11, [x20, x4]\n" "ushll v28.8h, v28.8b, #0x0\n" - "ushll v27.8h, v27.8b, #0x0\n" - "ldr d26, [x21, x3]\n" - "ldr d22, [x20, x3]\n" - "ushll v23.8h, v23.8b, #0x0\n" - "ushll v25.8h, v25.8b, #0x0\n" - "ushll v24.8h, v24.8b, #0x0\n" - "ushll v26.8h, v26.8b, #0x0\n" - "ushll v22.8h, v22.8b, #0x0\n" + "ushll v31.8h, v31.8b, #0x0\n" + "ushll v1.8h, v1.8b, #0x0\n" + "ushll v9.8h, v9.8b, #0x0\n" + "ushll v11.8h, v11.8b, #0x0\n" "bgt 1b\n" "2:" // Tail - "ldr q18, [x6, #0x0]\n" - "ldr q21, [x5, #0x0]\n" - "smlal v11.4s, v31.4h, v0.4h\n" - "smlal2 v13.4s, v31.8h, v0.8h\n" - "ldr q16, [x6, #0x10]\n" - "ldr q10, [x5, #0x10]\n" - "smlal v11.4s, v30.4h, v1.4h\n" - "smlal v20.4s, v30.4h, v0.4h\n" - "ldr x20, [x2, #0x50]\n" - "smlal v8.4s, v29.4h, v0.4h\n" - "smlal v6.4s, v28.4h, v0.4h\n" - "ldr x22, [x2, #0x58]\n" - "smlal2 v13.4s, v30.8h, v1.8h\n" - "smlal2 v19.4s, v30.8h, v0.8h\n" - "ldr d31, [x20, x3]\n" + "ldr d0, [x7, #0x28]\n" + "ldr d20, [x7, #0x30]\n" + "smlal v13.4s, v10.4h, v21.4h\n" + "smlal2 v24.4s, v10.8h, v21.8h\n" + "ldr d6, [x7, #0x38]\n" + "ldr d19, [x7, #0x40]\n" + "smlal v13.4s, v16.4h, v15.4h\n" + "smlal v7.4s, v16.4h, v21.4h\n" + "ldr d10, [x7, #0x48]\n" + "ldr d5, [x7, #0x50]\n" + "smlal v27.4s, v23.4h, v21.4h\n" + "smlal v8.4s, v30.4h, v21.4h\n" + "ldr x21, [x6, #0x50]\n" + "smlal2 v24.4s, v16.8h, v15.8h\n" + "smlal v13.4s, v4.4h, v29.4h\n" + "ldr x20, [x6, #0x58]\n" + "smlal2 v14.4s, v16.8h, v21.8h\n" + "ldr d16, [x21, x4]\n" + "smlal2 v22.4s, v23.8h, v21.8h\n" + "ushll v16.8h, v16.8b, #0x0\n" + "smlal2 v17.4s, v30.8h, v21.8h\n" + "ldr d21, [x20, x4]\n" + "smlal v7.4s, v4.4h, v15.4h\n" + "ldr x22, [x6, #0x60]\n" + "smlal v27.4s, v30.4h, v15.4h\n" + "smlal v8.4s, v28.4h, v15.4h\n" + "ushll v21.8h, v21.8b, #0x0\n" + "ldr x20, [x6, #0x68]\n" + "smlal2 v24.4s, v4.8h, v29.8h\n" + "smlal v13.4s, v31.4h, v18.4h\n" + "usubl v0.8h, v0.8b, v2.8b\n" + "ldr x21, [x6, #0x70]\n" + "smlal2 v14.4s, v4.8h, v15.8h\n" + "ldr d4, [x22, x4]\n" + "smlal2 v22.4s, v30.8h, v15.8h\n" + "ushll v4.8h, v4.8b, #0x0\n" + "smlal2 v17.4s, v28.8h, v15.8h\n" + "ldr d15, [x20, x4]\n" + "smlal v7.4s, v31.4h, v29.4h\n" + "usubl v20.8h, v20.8b, v2.8b\n" + "smlal v27.4s, v28.4h, v29.4h\n" + "smlal v8.4s, v16.4h, v29.4h\n" + "ushll v15.8h, v15.8b, #0x0\n" + "ldr x20, [x6, #0x78]\n" + "smlal2 v24.4s, v31.8h, v18.8h\n" + "smlal v13.4s, v1.4h, v3.4h\n" + "usubl v6.8h, v6.8b, v2.8b\n" + "ldr x22, [x6, #0x80]\n" + "smlal2 v14.4s, v31.8h, v29.8h\n" + "ldr d31, [x21, x4]\n" + "smlal2 v22.4s, v28.8h, v29.8h\n" "ushll v31.8h, v31.8b, #0x0\n" - "smlal2 v7.4s, v29.8h, v0.8h\n" - "smlal v11.4s, v27.4h, v2.4h\n" - "ldr x21, [x2, #0x60]\n" - "ldr x20, [x2, #0x68]\n" - "smlal2 v5.4s, v28.8h, v0.8h\n" - "ldr d30, [x22, x3]\n" - "smlal v20.4s, v27.4h, v1.4h\n" - "ushll v30.8h, v30.8b, #0x0\n" - "smlal v8.4s, v28.4h, v1.4h\n" - "smlal v6.4s, v23.4h, v1.4h\n" - "ldr x25, [x2, #0x70]\n" - "ldr x26, [x2, #0x78]\n" - "smlal2 v13.4s, v27.8h, v2.8h\n" - "smlal2 v19.4s, v27.8h, v1.8h\n" - "ldr d0, [x0, #0x28]\n" - "usubl v0.8h, v0.8b, v9.8b\n" - "smlal2 v7.4s, v28.8h, v1.8h\n" - "smlal v11.4s, v25.4h, v3.4h\n" - "ldr x23, [x2, #0x80]\n" - "ldr x24, [x2, #0x88]\n" - "smlal2 v5.4s, v23.8h, v1.8h\n" - "ldr d27, [x21, x3]\n" - "smlal v20.4s, v25.4h, v2.4h\n" - "ushll v27.8h, v27.8b, #0x0\n" - "smlal v8.4s, v23.4h, v2.4h\n" - "smlal v6.4s, v31.4h, v2.4h\n" - "ldr x15, [x2, #0x90]\n" - "ldr x21, [x2, #0x98]\n" - "smlal2 v13.4s, v25.8h, v3.8h\n" - "smlal2 v19.4s, v25.8h, v2.8h\n" - "ldr d1, [x0, #0x30]\n" - "usubl v1.8h, v1.8b, v9.8b\n" - "smlal2 v7.4s, v23.8h, v2.8h\n" - "smlal v11.4s, v24.4h, v4.4h\n" - "ldr x14, [x2, #0xa0]\n" - "ldr x13, [x2, #0xa8]\n" - "smlal2 v5.4s, v31.8h, v2.8h\n" - "ldr d25, [x20, x3]\n" - "smlal v20.4s, v24.4h, v3.4h\n" - "ushll v25.8h, v25.8b, #0x0\n" - "smlal v8.4s, v31.4h, v3.4h\n" - "smlal v6.4s, v30.4h, v3.4h\n" - "ldr x12, [x2, #0xb0]\n" - "ldr x20, [x2, #0xb8]\n" - "smlal2 v13.4s, v24.8h, v4.8h\n" - "smlal2 v19.4s, v24.8h, v3.8h\n" - "ldr d2, [x0, #0x38]\n" - "usubl v2.8h, v2.8b, v9.8b\n" - "smlal2 v7.4s, v31.8h, v3.8h\n" - "smlal v11.4s, v29.4h, v0.4h\n" - "ldr x11, [x2, #0xc0]\n" - "ldr x10, [x2, #0xc8]\n" - "smlal2 v5.4s, v30.8h, v3.8h\n" - "ldr d24, [x25, x3]\n" - "smlal v20.4s, v27.4h, v4.4h\n" - "ushll v24.8h, v24.8b, #0x0\n" - "smlal v8.4s, v30.4h, v4.4h\n" - "smlal v6.4s, v26.4h, v4.4h\n" - "ldr x22, [x2, #0xd0]\n" - "ldr x28, [x2, #0xd8]\n" - "smlal2 v13.4s, v29.8h, v0.8h\n" - "ldr d3, [x0, #0x40]\n" - "smlal2 v19.4s, v27.8h, v4.8h\n" - "ldr d27, [x26, x3]\n" - "smlal2 v7.4s, v30.8h, v4.8h\n" - "smlal v11.4s, v28.4h, v1.4h\n" - "usubl v3.8h, v3.8b, v9.8b\n" - "ldr x27, [x2, #0xe0]\n" - "smlal2 v5.4s, v26.8h, v4.8h\n" - "ldr d4, [x0, #0x48]\n" - "smlal v20.4s, v28.4h, v0.4h\n" - "ushll v27.8h, v27.8b, #0x0\n" - "smlal v8.4s, v22.4h, v0.4h\n" - "smlal v6.4s, v25.4h, v0.4h\n" - "usubl v4.8h, v4.8b, v9.8b\n" - "ldr x26, [x2, #0xe8]\n" - "smlal2 v13.4s, v28.8h, v1.8h\n" - "smlal2 v19.4s, v28.8h, v0.8h\n" - "ldr d28, [x24, x3]\n" - "ushll v28.8h, v28.8b, #0x0\n" - "smlal2 v7.4s, v22.8h, v0.8h\n" - "smlal v11.4s, v23.4h, v2.4h\n" - "ldr x25, [x2, #0xf0]\n" - "ldr x24, [x2, #0xf8]\n" - "smlal2 v5.4s, v25.8h, v0.8h\n" - "ldr d0, [x0, #0x50]\n" - "smlal v20.4s, v23.4h, v1.4h\n" - "usubl v0.8h, v0.8b, v9.8b\n" - "smlal v8.4s, v25.4h, v1.4h\n" - "smlal v6.4s, v24.4h, v1.4h\n" - "tst x4, #0x7\n" - "add x6, x6, #0x20\n" - "smlal2 v13.4s, v23.8h, v2.8h\n" - "smlal2 v19.4s, v23.8h, v1.8h\n" - "ldr d23, [x23, x3]\n" + "smlal2 v17.4s, v16.8h, v29.8h\n" + "ldr d29, [x20, x4]\n" + "smlal v7.4s, v1.4h, v18.4h\n" + "usubl v19.8h, v19.8b, v2.8b\n" + "smlal v27.4s, v16.4h, v18.4h\n" + "smlal v8.4s, v21.4h, v18.4h\n" + "ushll v29.8h, v29.8b, #0x0\n" + "ldr x20, [x6, #0x88]\n" + "smlal2 v24.4s, v1.8h, v3.8h\n" + "smlal v13.4s, v23.4h, v0.4h\n" + "usubl v10.8h, v10.8b, v2.8b\n" + "ldr x21, [x6, #0x90]\n" + "smlal2 v14.4s, v1.8h, v18.8h\n" + "ldr d1, [x22, x4]\n" + "smlal2 v22.4s, v16.8h, v18.8h\n" + "ushll v1.8h, v1.8b, #0x0\n" + "smlal2 v17.4s, v21.8h, v18.8h\n" + "ldr d18, [x20, x4]\n" + "smlal v7.4s, v4.4h, v3.4h\n" + "usubl v5.8h, v5.8b, v2.8b\n" + "smlal v27.4s, v21.4h, v3.4h\n" + "smlal v8.4s, v9.4h, v3.4h\n" + "ldr x20, [x6, #0x98]\n" + "ushll v18.8h, v18.8b, #0x0\n" + "smlal2 v24.4s, v23.8h, v0.8h\n" + "ldr d23, [x7, #0x58]\n" + "smlal v13.4s, v30.4h, v20.4h\n" + "usubl v23.8h, v23.8b, v2.8b\n" + "smlal2 v14.4s, v4.8h, v3.8h\n" + "ldr d4, [x21, x4]\n" + "smlal2 v22.4s, v21.8h, v3.8h\n" + "ldr x22, [x6, #0xa0]\n" + "smlal2 v17.4s, v9.8h, v3.8h\n" + "ldr d3, [x20, x4]\n" + "smlal v7.4s, v30.4h, v0.4h\n" + "ushll v4.8h, v4.8b, #0x0\n" + "smlal v27.4s, v11.4h, v0.4h\n" + "smlal v8.4s, v15.4h, v0.4h\n" + "ushll v3.8h, v3.8b, #0x0\n" + "ldr x21, [x6, #0xa8]\n" + "smlal2 v24.4s, v30.8h, v20.8h\n" + "smlal v13.4s, v28.4h, v6.4h\n" + "ldr x20, [x6, #0xb0]\n" + "ldr x12, [x6, #0xb8]\n" + "smlal2 v14.4s, v30.8h, v0.8h\n" + "ldr d30, [x7, #0x60]\n" + "smlal2 v22.4s, v11.8h, v0.8h\n" + "usubl v30.8h, v30.8b, v2.8b\n" + "smlal2 v17.4s, v15.8h, v0.8h\n" + "ldr d0, [x22, x4]\n" + "smlal v7.4s, v28.4h, v20.4h\n" + "ushll v0.8h, v0.8b, #0x0\n" + "smlal v27.4s, v15.4h, v20.4h\n" + "smlal v8.4s, v31.4h, v20.4h\n" + "ldr x11, [x6, #0xc0]\n" + "ldr x10, [x6, #0xc8]\n" + "smlal2 v24.4s, v28.8h, v6.8h\n" + "smlal v13.4s, v16.4h, v19.4h\n" + "ldr x9, [x6, #0xd0]\n" + "ldr x28, [x6, #0xd8]\n" + "smlal2 v14.4s, v28.8h, v20.8h\n" + "ldr d28, [x7, #0x68]\n" + "smlal2 v22.4s, v15.8h, v20.8h\n" + "usubl v28.8h, v28.8b, v2.8b\n" + "smlal2 v17.4s, v31.8h, v20.8h\n" + "ldr d20, [x21, x4]\n" + "smlal v7.4s, v16.4h, v6.4h\n" + "ushll v20.8h, v20.8b, #0x0\n" + "smlal v27.4s, v31.4h, v6.4h\n" + "smlal v8.4s, v29.4h, v6.4h\n" + "ldr x27, [x6, #0xe0]\n" + "ldr x26, [x6, #0xe8]\n" + "smlal2 v24.4s, v16.8h, v19.8h\n" + "smlal v13.4s, v21.4h, v10.4h\n" + "ldr x25, [x6, #0xf0]\n" + "ldr x24, [x6, #0xf8]\n" + "smlal2 v14.4s, v16.8h, v6.8h\n" + "ldr d16, [x7, #0x70]\n" + "smlal2 v22.4s, v31.8h, v6.8h\n" + "usubl v16.8h, v16.8b, v2.8b\n" + "smlal2 v17.4s, v29.8h, v6.8h\n" + "ldr d6, [x20, x4]\n" + "smlal v7.4s, v21.4h, v19.4h\n" + "ushll v6.8h, v6.8b, #0x0\n" + "smlal v27.4s, v29.4h, v19.4h\n" + "smlal v8.4s, v1.4h, v19.4h\n" + "ldr x23, [x6, #0x100]\n" + "ldr x22, [x6, #0x108]\n" + "smlal2 v24.4s, v21.8h, v10.8h\n" + "smlal v13.4s, v11.4h, v5.4h\n" + "ldr x21, [x6, #0x110]\n" + "ldr x20, [x6, #0x118]\n" + "smlal2 v14.4s, v21.8h, v19.8h\n" + "ldr d21, [x7, #0x78]\n" + "smlal2 v22.4s, v29.8h, v19.8h\n" + "usubl v21.8h, v21.8b, v2.8b\n" + "smlal2 v17.4s, v1.8h, v19.8h\n" + "ldr d19, [x12, x4]\n" + "smlal v7.4s, v9.4h, v10.4h\n" + "ushll v19.8h, v19.8b, #0x0\n" + "smlal v27.4s, v1.4h, v10.4h\n" + "smlal v8.4s, v18.4h, v10.4h\n" + "tst x2, #0x7\n" + "smlal2 v24.4s, v11.8h, v5.8h\n" + "ldr d11, [x7, #0x80]\n" + "smlal v13.4s, v15.4h, v23.4h\n" + "usubl v11.8h, v11.8b, v2.8b\n" + "smlal2 v14.4s, v9.8h, v10.8h\n" + "ldr d9, [x11, x4]\n" + "smlal2 v22.4s, v1.8h, v10.8h\n" + "ushll v9.8h, v9.8b, #0x0\n" + "smlal2 v17.4s, v18.8h, v10.8h\n" + "ldr d10, [x10, x4]\n" + "smlal v7.4s, v15.4h, v5.4h\n" + "ushll v10.8h, v10.8b, #0x0\n" + "smlal v27.4s, v4.4h, v5.4h\n" + "smlal v8.4s, v3.4h, v5.4h\n" + "smlal2 v24.4s, v15.8h, v23.8h\n" + "smlal v13.4s, v31.4h, v30.4h\n" + "smlal2 v14.4s, v15.8h, v5.8h\n" + "ldr d15, [x7, #0x88]\n" + "smlal2 v22.4s, v4.8h, v5.8h\n" + "usubl v15.8h, v15.8b, v2.8b\n" + "smlal2 v17.4s, v3.8h, v5.8h\n" + "ldr d5, [x9, x4]\n" + "smlal v7.4s, v31.4h, v23.4h\n" + "ushll v5.8h, v5.8b, #0x0\n" + "smlal v27.4s, v3.4h, v23.4h\n" + "smlal v8.4s, v0.4h, v23.4h\n" + "smlal2 v24.4s, v31.8h, v30.8h\n" + "smlal v13.4s, v29.4h, v28.4h\n" + "smlal2 v14.4s, v31.8h, v23.8h\n" + "ldr d31, [x7, #0x90]\n" + "smlal2 v22.4s, v3.8h, v23.8h\n" + "usubl v31.8h, v31.8b, v2.8b\n" + "smlal2 v17.4s, v0.8h, v23.8h\n" + "ldr d23, [x28, x4]\n" + "smlal v7.4s, v29.4h, v30.4h\n" "ushll v23.8h, v23.8b, #0x0\n" - "smlal2 v7.4s, v25.8h, v1.8h\n" - "smlal v11.4s, v31.4h, v3.4h\n" - "ldr x23, [x2, #0x100]\n" - "add x5, x5, #0x20\n" - "smlal2 v5.4s, v24.8h, v1.8h\n" - "ldr d1, [x0, #0x58]\n" - "smlal v20.4s, v31.4h, v2.4h\n" - "usubl v1.8h, v1.8b, v9.8b\n" - "smlal v8.4s, v24.4h, v2.4h\n" - "smlal v6.4s, v27.4h, v2.4h\n" - "smlal2 v13.4s, v31.8h, v3.8h\n" - "smlal2 v19.4s, v31.8h, v2.8h\n" - "ldr d31, [x15, x3]\n" - "ushll v31.8h, v31.8b, #0x0\n" - "smlal2 v7.4s, v24.8h, v2.8h\n" - "smlal v11.4s, v30.4h, v4.4h\n" - "ldr x15, [x2, #0x108]\n" - "smlal2 v5.4s, v27.8h, v2.8h\n" - "ldr d2, [x0, #0x60]\n" - "smlal v20.4s, v30.4h, v3.4h\n" - "usubl v2.8h, v2.8b, v9.8b\n" - "smlal v8.4s, v27.4h, v3.4h\n" - "smlal v6.4s, v23.4h, v3.4h\n" - "smlal2 v13.4s, v30.8h, v4.8h\n" - "smlal2 v19.4s, v30.8h, v3.8h\n" - "ldr d30, [x21, x3]\n" + "smlal v27.4s, v0.4h, v30.4h\n" + "smlal v8.4s, v20.4h, v30.4h\n" + "smlal2 v24.4s, v29.8h, v28.8h\n" + "smlal v13.4s, v1.4h, v16.4h\n" + "smlal2 v14.4s, v29.8h, v30.8h\n" + "ldr d29, [x7, #0x98]\n" + "smlal2 v22.4s, v0.8h, v30.8h\n" + "usubl v29.8h, v29.8b, v2.8b\n" + "smlal2 v17.4s, v20.8h, v30.8h\n" + "ldr d30, [x27, x4]\n" + "smlal v7.4s, v1.4h, v28.4h\n" "ushll v30.8h, v30.8b, #0x0\n" - "smlal2 v7.4s, v27.8h, v3.8h\n" - "smlal v11.4s, v22.4h, v0.4h\n" - "ldr x21, [x2, #0x110]\n" - "smlal2 v5.4s, v23.8h, v3.8h\n" - "ldr d3, [x0, #0x68]\n" - "smlal v20.4s, v26.4h, v4.4h\n" - "usubl v3.8h, v3.8b, v9.8b\n" - "smlal v8.4s, v23.4h, v4.4h\n" - "smlal v6.4s, v28.4h, v4.4h\n" - "smlal2 v13.4s, v22.8h, v0.8h\n" - "ldr d22, [x20, x3]\n" - "smlal2 v19.4s, v26.8h, v4.8h\n" - "ldr d26, [x14, x3]\n" - "smlal2 v7.4s, v23.8h, v4.8h\n" - "smlal v11.4s, v25.4h, v1.4h\n" - "ushll v26.8h, v26.8b, #0x0\n" - "ldr x20, [x2, #0x118]\n" - "smlal2 v5.4s, v28.8h, v4.8h\n" - "ldr d4, [x0, #0x70]\n" - "smlal v20.4s, v25.4h, v0.4h\n" - "usubl v4.8h, v4.8b, v9.8b\n" - "smlal v8.4s, v31.4h, v0.4h\n" - "smlal v6.4s, v30.4h, v0.4h\n" - "ushll v22.8h, v22.8b, #0x0\n" - "smlal2 v13.4s, v25.8h, v1.8h\n" - "smlal2 v19.4s, v25.8h, v0.8h\n" - "ldr d25, [x13, x3]\n" - "ushll v25.8h, v25.8b, #0x0\n" - "smlal2 v7.4s, v31.8h, v0.8h\n" - "smlal v11.4s, v24.4h, v2.4h\n" - "smlal2 v5.4s, v30.8h, v0.8h\n" - "ldr d0, [x0, #0x78]\n" - "smlal v20.4s, v24.4h, v1.4h\n" - "usubl v0.8h, v0.8b, v9.8b\n" - "smlal v8.4s, v30.4h, v1.4h\n" - "smlal v6.4s, v26.4h, v1.4h\n" - "smlal2 v13.4s, v24.8h, v2.8h\n" - "smlal2 v19.4s, v24.8h, v1.8h\n" - "ldr d24, [x12, x3]\n" - "ushll v24.8h, v24.8b, #0x0\n" - "smlal2 v7.4s, v30.8h, v1.8h\n" - "smlal v11.4s, v27.4h, v3.4h\n" - "smlal2 v5.4s, v26.8h, v1.8h\n" - "ldr d1, [x0, #0x80]\n" - "smlal v20.4s, v27.4h, v2.4h\n" - "usubl v1.8h, v1.8b, v9.8b\n" - "smlal v8.4s, v26.4h, v2.4h\n" - "smlal v6.4s, v25.4h, v2.4h\n" - "smlal2 v13.4s, v27.8h, v3.8h\n" - "smlal2 v19.4s, v27.8h, v2.8h\n" - "ldr d27, [x11, x3]\n" - "ushll v27.8h, v27.8b, #0x0\n" - "smlal2 v7.4s, v26.8h, v2.8h\n" - "smlal v11.4s, v23.4h, v4.4h\n" - "smlal2 v5.4s, v25.8h, v2.8h\n" - "ldr d2, [x0, #0x88]\n" - "smlal v20.4s, v23.4h, v3.4h\n" - "usubl v2.8h, v2.8b, v9.8b\n" - "smlal v8.4s, v25.4h, v3.4h\n" - "smlal v6.4s, v24.4h, v3.4h\n" - "smlal2 v13.4s, v23.8h, v4.8h\n" - "smlal2 v19.4s, v23.8h, v3.8h\n" - "ldr d23, [x10, x3]\n" - "ushll v23.8h, v23.8b, #0x0\n" - "smlal2 v7.4s, v25.8h, v3.8h\n" - "smlal v11.4s, v31.4h, v0.4h\n" - "smlal2 v5.4s, v24.8h, v3.8h\n" - "ldr d3, [x0, #0x90]\n" - "smlal v20.4s, v28.4h, v4.4h\n" - "usubl v3.8h, v3.8b, v9.8b\n" - "smlal v8.4s, v24.4h, v4.4h\n" - "smlal v6.4s, v22.4h, v4.4h\n" - "smlal2 v13.4s, v31.8h, v0.8h\n" - "ldr d31, [x22, x3]\n" - "smlal2 v19.4s, v28.8h, v4.8h\n" - "ldr d28, [x27, x3]\n" - "smlal2 v7.4s, v24.8h, v4.8h\n" - "smlal v11.4s, v30.4h, v1.4h\n" - "ushll v31.8h, v31.8b, #0x0\n" - "smlal2 v5.4s, v22.8h, v4.8h\n" - "ldr d4, [x0, #0x98]\n" - "smlal v20.4s, v30.4h, v0.4h\n" - "usubl v4.8h, v4.8b, v9.8b\n" - "smlal v8.4s, v27.4h, v0.4h\n" - "smlal v6.4s, v23.4h, v0.4h\n" + "smlal v27.4s, v20.4h, v28.4h\n" + "smlal v8.4s, v6.4h, v28.4h\n" + "smlal2 v24.4s, v1.8h, v16.8h\n" + "smlal v13.4s, v4.4h, v21.4h\n" + "smlal2 v14.4s, v1.8h, v28.8h\n" + "ldr d1, [x7, #0xa0]\n" + "smlal2 v22.4s, v20.8h, v28.8h\n" + "usubl v1.8h, v1.8b, v2.8b\n" + "smlal2 v17.4s, v6.8h, v28.8h\n" + "ldr d28, [x26, x4]\n" + "smlal v7.4s, v18.4h, v16.4h\n" "ushll v28.8h, v28.8b, #0x0\n" - "smlal2 v13.4s, v30.8h, v1.8h\n" - "smlal2 v19.4s, v30.8h, v0.8h\n" - "ldr d30, [x28, x3]\n" - "ushll v30.8h, v30.8b, #0x0\n" - "smlal2 v7.4s, v27.8h, v0.8h\n" - "smlal v11.4s, v26.4h, v2.4h\n" - "smlal2 v5.4s, v23.8h, v0.8h\n" - "ldr d0, [x0, #0xa0]\n" - "smlal v20.4s, v26.4h, v1.4h\n" - "usubl v0.8h, v0.8b, v9.8b\n" - "smlal v8.4s, v23.4h, v1.4h\n" - "smlal v6.4s, v31.4h, v1.4h\n" - "smlal2 v13.4s, v26.8h, v2.8h\n" - "smlal2 v19.4s, v26.8h, v1.8h\n" - "ldr d26, [x26, x3]\n" - "ushll v26.8h, v26.8b, #0x0\n" - "smlal2 v7.4s, v23.8h, v1.8h\n" - "smlal v11.4s, v25.4h, v3.4h\n" - "smlal2 v5.4s, v31.8h, v1.8h\n" - "ldr d1, [x0, #0xa8]\n" - "smlal v20.4s, v25.4h, v2.4h\n" - "usubl v1.8h, v1.8b, v9.8b\n" - "smlal v8.4s, v31.4h, v2.4h\n" - "smlal v6.4s, v30.4h, v2.4h\n" - "smlal2 v13.4s, v25.8h, v3.8h\n" - "smlal2 v19.4s, v25.8h, v2.8h\n" - "ldr d25, [x25, x3]\n" - "ushll v25.8h, v25.8b, #0x0\n" - "smlal2 v7.4s, v31.8h, v2.8h\n" - "smlal v11.4s, v24.4h, v4.4h\n" - "smlal2 v5.4s, v30.8h, v2.8h\n" - "ldr d2, [x0, #0xb0]\n" - "smlal v20.4s, v24.4h, v3.4h\n" - "usubl v2.8h, v2.8b, v9.8b\n" - "smlal v8.4s, v30.4h, v3.4h\n" - "smlal v6.4s, v28.4h, v3.4h\n" - "smlal2 v13.4s, v24.8h, v4.8h\n" - "smlal2 v19.4s, v24.8h, v3.8h\n" - "ldr d24, [x24, x3]\n" - "ushll v24.8h, v24.8b, #0x0\n" - "smlal2 v7.4s, v30.8h, v3.8h\n" - "smlal v11.4s, v27.4h, v0.4h\n" - "smlal2 v5.4s, v28.8h, v3.8h\n" - "ldr d3, [x0, #0xb8]\n" - "smlal v20.4s, v22.4h, v4.4h\n" - "usubl v3.8h, v3.8b, v9.8b\n" - "smlal v8.4s, v28.4h, v4.4h\n" - "smlal v6.4s, v26.4h, v4.4h\n" - "smlal2 v13.4s, v27.8h, v0.8h\n" - "ldr d27, [x23, x3]\n" - "smlal2 v7.4s, v28.8h, v4.8h\n" - "ushll v27.8h, v27.8b, #0x0\n" - "smlal v11.4s, v23.4h, v1.4h\n" - "smlal2 v19.4s, v22.8h, v4.8h\n" - "smlal2 v5.4s, v26.8h, v4.8h\n" - "ldr d4, [x0, #0xc0]\n" - "smlal v20.4s, v23.4h, v0.4h\n" - "usubl v4.8h, v4.8b, v9.8b\n" - "smlal v8.4s, v25.4h, v0.4h\n" - "smlal v6.4s, v24.4h, v0.4h\n" - "smlal2 v13.4s, v23.8h, v1.8h\n" - "smlal2 v7.4s, v25.8h, v0.8h\n" - "ldr d25, [x15, x3]\n" - "ushll v25.8h, v25.8b, #0x0\n" - "smlal v11.4s, v31.4h, v2.4h\n" - "smlal2 v19.4s, v23.8h, v0.8h\n" - "smlal2 v5.4s, v24.8h, v0.8h\n" - "smlal v20.4s, v31.4h, v1.4h\n" - "smlal v8.4s, v24.4h, v1.4h\n" - "smlal v6.4s, v27.4h, v1.4h\n" - "smlal2 v13.4s, v31.8h, v2.8h\n" - "smlal2 v7.4s, v24.8h, v1.8h\n" - "ldr d24, [x21, x3]\n" - "ushll v24.8h, v24.8b, #0x0\n" - "smlal v11.4s, v30.4h, v3.4h\n" - "smlal2 v19.4s, v31.8h, v1.8h\n" - "smlal2 v5.4s, v27.8h, v1.8h\n" - "smlal v20.4s, v30.4h, v2.4h\n" - "smlal v8.4s, v27.4h, v2.4h\n" - "smlal v6.4s, v25.4h, v2.4h\n" - "smlal2 v13.4s, v30.8h, v3.8h\n" - "smlal2 v7.4s, v27.8h, v2.8h\n" - "ldr d27, [x20, x3]\n" - "ushll v27.8h, v27.8b, #0x0\n" - "smlal v11.4s, v28.4h, v4.4h\n" - "smlal2 v19.4s, v30.8h, v2.8h\n" - "sqrdmulh v11.4s, v11.4s, v18.4s\n" - "add x3, x3, #0x8\n" - "smlal2 v5.4s, v25.8h, v2.8h\n" - "smlal v20.4s, v28.4h, v3.4h\n" - "and v31.16b, v11.16b, v21.16b\n" - "smlal v8.4s, v25.4h, v3.4h\n" - "smlal v6.4s, v24.4h, v3.4h\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "smlal2 v13.4s, v28.8h, v4.8h\n" - "smlal2 v19.4s, v28.8h, v3.8h\n" - "sqrdmulh v13.4s, v13.4s, v16.4s\n" - "smlal2 v7.4s, v25.8h, v3.8h\n" - "smlal2 v5.4s, v24.8h, v3.8h\n" - "and v17.16b, v13.16b, v10.16b\n" - "smlal v20.4s, v26.4h, v4.4h\n" - "smlal v8.4s, v24.4h, v4.4h\n" - "sqrdmulh v20.4s, v20.4s, v18.4s\n" - "smlal v6.4s, v27.4h, v4.4h\n" - "smlal2 v19.4s, v26.8h, v4.8h\n" - "sqrdmulh v8.4s, v8.4s, v18.4s\n" - "smlal2 v7.4s, v24.8h, v4.8h\n" - "smlal2 v5.4s, v27.8h, v4.8h\n" - "sqrdmulh v6.4s, v6.4s, v18.4s\n" - "sqadd v11.4s, v11.4s, v31.4s\n" - "sshr v17.4s, v17.4s, #0x1f\n" - "and v26.16b, v20.16b, v21.16b\n" - "sqrdmulh v19.4s, v19.4s, v16.4s\n" - "and v18.16b, v8.16b, v21.16b\n" - "sqrdmulh v7.4s, v7.4s, v16.4s\n" - "and v31.16b, v6.16b, v21.16b\n" - "sqrdmulh v5.4s, v5.4s, v16.4s\n" - "sqadd v13.4s, v13.4s, v17.4s\n" - "sshr v26.4s, v26.4s, #0x1f\n" - "and v27.16b, v19.16b, v10.16b\n" + "smlal v27.4s, v6.4h, v16.4h\n" + "smlal v8.4s, v19.4h, v16.4h\n" + "smlal2 v24.4s, v4.8h, v21.8h\n" + "ldr d4, [x7, #0xa8]\n" + "smlal v13.4s, v3.4h, v11.4h\n" + "usubl v4.8h, v4.8b, v2.8b\n" + "smlal2 v14.4s, v18.8h, v16.8h\n" + "ldr d18, [x25, x4]\n" + "smlal2 v22.4s, v6.8h, v16.8h\n" + "ushll v18.8h, v18.8b, #0x0\n" + "smlal2 v17.4s, v19.8h, v16.8h\n" + "ldr d16, [x24, x4]\n" + "smlal v7.4s, v3.4h, v21.4h\n" + "ushll v16.8h, v16.8b, #0x0\n" + "smlal v27.4s, v9.4h, v21.4h\n" + "smlal v8.4s, v10.4h, v21.4h\n" + "smlal2 v24.4s, v3.8h, v11.8h\n" + "smlal v13.4s, v0.4h, v15.4h\n" + "smlal2 v14.4s, v3.8h, v21.8h\n" + "ldr d3, [x7, #0xb0]\n" + "smlal2 v22.4s, v9.8h, v21.8h\n" + "usubl v3.8h, v3.8b, v2.8b\n" + "smlal2 v17.4s, v10.8h, v21.8h\n" + "ldr d21, [x23, x4]\n" + "smlal v7.4s, v0.4h, v11.4h\n" + "ushll v21.8h, v21.8b, #0x0\n" + "smlal v27.4s, v10.4h, v11.4h\n" + "smlal v8.4s, v5.4h, v11.4h\n" + "smlal2 v24.4s, v0.8h, v15.8h\n" + "smlal v13.4s, v20.4h, v31.4h\n" + "smlal2 v14.4s, v0.8h, v11.8h\n" + "ldr d0, [x7, #0xb8]\n" + "smlal2 v22.4s, v10.8h, v11.8h\n" + "usubl v0.8h, v0.8b, v2.8b\n" + "smlal2 v17.4s, v5.8h, v11.8h\n" + "ldr d11, [x22, x4]\n" + "smlal v7.4s, v20.4h, v15.4h\n" + "ushll v11.8h, v11.8b, #0x0\n" + "smlal v27.4s, v5.4h, v15.4h\n" + "smlal v8.4s, v23.4h, v15.4h\n" + "smlal2 v24.4s, v20.8h, v31.8h\n" + "smlal v13.4s, v6.4h, v29.4h\n" + "smlal2 v14.4s, v20.8h, v15.8h\n" + "ldr d20, [x7, #0xc0]\n" + "smlal2 v22.4s, v5.8h, v15.8h\n" + "usubl v20.8h, v20.8b, v2.8b\n" + "smlal2 v17.4s, v23.8h, v15.8h\n" + "ldr d15, [x21, x4]\n" + "smlal v7.4s, v6.4h, v31.4h\n" + "ushll v15.8h, v15.8b, #0x0\n" + "smlal v27.4s, v23.4h, v31.4h\n" + "smlal v8.4s, v30.4h, v31.4h\n" + "smlal2 v24.4s, v6.8h, v29.8h\n" + "smlal v13.4s, v9.4h, v1.4h\n" + "smlal2 v14.4s, v6.8h, v31.8h\n" + "ldr d6, [x20, x4]\n" + "smlal2 v22.4s, v23.8h, v31.8h\n" + "ushll v6.8h, v6.8b, #0x0\n" + "smlal2 v17.4s, v30.8h, v31.8h\n" + "ldr q31, [x8, #0x0]\n" + "smlal v7.4s, v19.4h, v29.4h\n" + "add x4, x4, #0x8\n" + "smlal v27.4s, v30.4h, v29.4h\n" + "smlal v8.4s, v28.4h, v29.4h\n" + "smlal2 v24.4s, v9.8h, v1.8h\n" + "ldr q9, [x17, #0x0]\n" + "smlal v13.4s, v10.4h, v4.4h\n" + "smlal2 v14.4s, v19.8h, v29.8h\n" + "ldr q19, [x8, #0x10]\n" + "smlal2 v22.4s, v30.8h, v29.8h\n" + "add x8, x8, #0x20\n" + "smlal2 v17.4s, v28.8h, v29.8h\n" + "ldr q29, [x17, #0x10]\n" + "smlal v7.4s, v10.4h, v1.4h\n" + "add x17, x17, #0x20\n" + "smlal v27.4s, v18.4h, v1.4h\n" + "smlal v8.4s, v16.4h, v1.4h\n" + "smlal2 v24.4s, v10.8h, v4.8h\n" + "smlal v13.4s, v5.4h, v3.4h\n" + "smlal2 v14.4s, v10.8h, v1.8h\n" + "smlal2 v22.4s, v18.8h, v1.8h\n" + "smlal2 v17.4s, v16.8h, v1.8h\n" + "smlal v7.4s, v5.4h, v4.4h\n" + "smlal v27.4s, v16.4h, v4.4h\n" + "smlal v8.4s, v21.4h, v4.4h\n" + "smlal2 v24.4s, v5.8h, v3.8h\n" + "smlal v13.4s, v23.4h, v0.4h\n" + "smlal2 v14.4s, v5.8h, v4.8h\n" + "smlal2 v22.4s, v16.8h, v4.8h\n" + "smlal2 v17.4s, v21.8h, v4.8h\n" + "smlal v7.4s, v23.4h, v3.4h\n" + "smlal v27.4s, v21.4h, v3.4h\n" + "smlal v8.4s, v11.4h, v3.4h\n" + "smlal2 v24.4s, v23.8h, v0.8h\n" + "smlal v13.4s, v30.4h, v20.4h\n" + "sqrdmulh v13.4s, v13.4s, v31.4s\n" + "smlal2 v14.4s, v23.8h, v3.8h\n" + "smlal2 v22.4s, v21.8h, v3.8h\n" + "and v21.16b, v13.16b, v9.16b\n" + "smlal2 v17.4s, v11.8h, v3.8h\n" + "smlal v7.4s, v30.4h, v0.4h\n" + "sshr v21.4s, v21.4s, #0x1f\n" + "smlal v27.4s, v11.4h, v0.4h\n" + "smlal v8.4s, v15.4h, v0.4h\n" + "sqadd v13.4s, v13.4s, v21.4s\n" + "smlal2 v24.4s, v30.8h, v20.8h\n" + "smlal2 v14.4s, v30.8h, v0.8h\n" + "sqrdmulh v24.4s, v24.4s, v19.4s\n" + "smlal2 v22.4s, v11.8h, v0.8h\n" + "smlal2 v17.4s, v15.8h, v0.8h\n" + "and v16.16b, v24.16b, v29.16b\n" + "smlal v7.4s, v28.4h, v20.4h\n" + "smlal v27.4s, v15.4h, v20.4h\n" + "sqrdmulh v7.4s, v7.4s, v31.4s\n" + "smlal v8.4s, v6.4h, v20.4h\n" + "smlal2 v14.4s, v28.8h, v20.8h\n" + "sqrdmulh v27.4s, v27.4s, v31.4s\n" + "smlal2 v22.4s, v15.8h, v20.8h\n" + "smlal2 v17.4s, v6.8h, v20.8h\n" + "sqrdmulh v8.4s, v8.4s, v31.4s\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "and v23.16b, v7.16b, v9.16b\n" + "sqrdmulh v14.4s, v14.4s, v19.4s\n" + "and v20.16b, v27.16b, v9.16b\n" + "sqrdmulh v22.4s, v22.4s, v19.4s\n" + "and v3.16b, v8.16b, v9.16b\n" + "sqrdmulh v17.4s, v17.4s, v19.4s\n" + "sqadd v24.4s, v24.4s, v16.4s\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "and v18.16b, v14.16b, v29.16b\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "and v19.16b, v22.16b, v29.16b\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "and v30.16b, v17.16b, v29.16b\n" + "sqadd v7.4s, v7.4s, v23.4s\n" "sshr v18.4s, v18.4s, #0x1f\n" - "and v25.16b, v7.16b, v10.16b\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "and v17.16b, v5.16b, v10.16b\n" - "sqadd v20.4s, v20.4s, v26.4s\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v18.4s\n" - "sshr v25.4s, v25.4s, #0x1f\n" - "sqadd v6.4s, v6.4s, v31.4s\n" - "sshr v17.4s, v17.4s, #0x1f\n" - "srshl v11.4s, v11.4s, v21.4s\n" - "srshl v20.4s, v20.4s, v21.4s\n" - "sqadd v19.4s, v19.4s, v27.4s\n" - "srshl v8.4s, v8.4s, v21.4s\n" - "sqadd v7.4s, v7.4s, v25.4s\n" - "srshl v6.4s, v6.4s, v21.4s\n" - "sqadd v5.4s, v5.4s, v17.4s\n" - "srshl v13.4s, v13.4s, v10.4s\n" - "sqxtn v11.4h, v11.4s\n" - "srshl v19.4s, v19.4s, v10.4s\n" - "sqxtn v20.4h, v20.4s\n" - "srshl v7.4s, v7.4s, v10.4s\n" + "sqadd v27.4s, v27.4s, v20.4s\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "sqadd v8.4s, v8.4s, v3.4s\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "srshl v13.4s, v13.4s, v9.4s\n" + "srshl v7.4s, v7.4s, v9.4s\n" + "sqadd v14.4s, v14.4s, v18.4s\n" + "srshl v27.4s, v27.4s, v9.4s\n" + "sqadd v22.4s, v22.4s, v19.4s\n" + "srshl v8.4s, v8.4s, v9.4s\n" + "sqadd v17.4s, v17.4s, v30.4s\n" + "srshl v24.4s, v24.4s, v29.4s\n" + "sqxtn v13.4h, v13.4s\n" + "srshl v14.4s, v14.4s, v29.4s\n" + "sqxtn v7.4h, v7.4s\n" + "srshl v22.4s, v22.4s, v29.4s\n" + "sqxtn v27.4h, v27.4s\n" + "srshl v17.4s, v17.4s, v29.4s\n" "sqxtn v8.4h, v8.4s\n" - "srshl v5.4s, v5.4s, v10.4s\n" - "sqxtn v6.4h, v6.4s\n" - "sqxtn2 v11.8h, v13.4s\n" - "sqxtn2 v20.8h, v19.4s\n" - "sqxtn2 v8.8h, v7.4s\n" - "sqxtn2 v6.8h, v5.4s\n" - "sqadd v11.8h, v11.8h, v15.8h\n" - "sqadd v20.8h, v20.8h, v15.8h\n" - "sqadd v8.8h, v8.8h, v15.8h\n" - "sqadd v6.8h, v6.8h, v15.8h\n" - "smax v11.8h, v11.8h, v14.8h\n" - "smax v20.8h, v20.8h, v14.8h\n" - "smax v8.8h, v8.8h, v14.8h\n" - "smax v6.8h, v6.8h, v14.8h\n" - "smin v11.8h, v11.8h, v12.8h\n" - "smin v20.8h, v20.8h, v12.8h\n" - "smin v8.8h, v8.8h, v12.8h\n" - "smin v6.8h, v6.8h, v12.8h\n" - "uzp1 v11.16b, v11.16b, v11.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" - "str d11, [x7, x1]\n" + "sqxtn2 v13.8h, v24.4s\n" + "sqxtn2 v7.8h, v14.4s\n" + "sqxtn2 v27.8h, v22.4s\n" + "sqxtn2 v8.8h, v17.4s\n" + "sqadd v13.8h, v13.8h, v25.8h\n" + "sqadd v7.8h, v7.8h, v25.8h\n" + "sqadd v27.8h, v27.8h, v25.8h\n" + "sqadd v8.8h, v8.8h, v25.8h\n" + "smax v13.8h, v13.8h, v12.8h\n" + "smax v7.8h, v7.8h, v12.8h\n" + "smax v27.8h, v27.8h, v12.8h\n" + "smax v8.8h, v8.8h, v12.8h\n" + "smin v13.8h, v13.8h, v26.8h\n" + "smin v7.8h, v7.8h, v26.8h\n" + "smin v27.8h, v27.8h, v26.8h\n" + "smin v8.8h, v8.8h, v26.8h\n" + "uzp1 v13.16b, v13.16b, v13.16b\n" + "str d13, [x16, x5]\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" + "uzp1 v27.16b, v27.16b, v27.16b\n" + "str d7, [x15, x5]\n" "uzp1 v8.16b, v8.16b, v8.16b\n" - "uzp1 v6.16b, v6.16b, v6.16b\n" - "str d20, [x8, x1]\n" - "str d8, [x17, x1]\n" - "str d6, [x16, x1]\n" - "add x1, x1, #0x8\n" + "str d27, [x14, x5]\n" + "str d8, [x13, x5]\n" + "add x5, x5, #0x8\n" "beq 124f\n" - "add x0, x0, #0xc8\n" + "add x7, x7, #0xc8\n" "3:" // Oddments - "ldr x13, [%x[params], %[offsetof_Params_bias]]\n" - "tbz x4, #2, 5f\n" - "ld1 { v11.4s }, [x13], #0x10\n" - "tbz x4, #1, 4f\n" - "ld1 { v13.d }[0], [x13], #0x8\n" - "tbz x4, #0, 7f\n" - "ld1 { v13.s }[2], [x13]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "tbz x2, #2, 5f\n" + "ld1 { v13.4s }, [x20], #0x10\n" + "tbz x2, #1, 4f\n" + "ld1 { v24.d }[0], [x20], #0x8\n" + "tbz x2, #0, 7f\n" + "ld1 { v24.s }[2], [x20]\n" "b 7f\n" "4:" // Oddments: Load bias: Bit 2: Bit 1: Unset - "tbz x4, #0, 7f\n" - "ld1 { v13.s }[0], [x13]\n" + "tbz x2, #0, 7f\n" + "ld1 { v24.s }[0], [x20]\n" "b 7f\n" "5:" // Oddments: Load bias: Bit 2: Unset - "tbz x4, #1, 6f\n" - "ld1 { v11.d }[0], [x13], #0x8\n" - "tbz x4, #0, 7f\n" - "ld1 { v11.s }[2], [x13]\n" + "tbz x2, #1, 6f\n" + "ld1 { v13.d }[0], [x20], #0x8\n" + "tbz x2, #0, 7f\n" + "ld1 { v13.s }[2], [x20]\n" "b 7f\n" "6:" // Oddments: Load bias: Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 7f\n" - "ld1 { v11.s }[0], [x13]\n" + "tbz x2, #0, 7f\n" + "ld1 { v13.s }[0], [x20]\n" "7:" // Oddments: Load bias: Bit 2: End - "ldr d0, [x0, #0x0]\n" - "ldr d1, [x0, #0x8]\n" - "mov v20.16b, v11.16b\n" - "mov v19.16b, v13.16b\n" - "ldr d2, [x0, #0x10]\n" - "ldr d3, [x0, #0x18]\n" - "mov v8.16b, v11.16b\n" + "ldr d21, [x7, #0x0]\n" + "ldr d15, [x7, #0x8]\n" "mov v7.16b, v13.16b\n" - "ldr d4, [x0, #0x20]\n" - "ldp x10, x28, [x2, #0x0]\n" - "mov v6.16b, v11.16b\n" - "mov v5.16b, v13.16b\n" - "ldp x27, x26, [x2, #0x10]\n" - "ldp x25, x24, [x2, #0x20]\n" - "usubl v0.8h, v0.8b, v9.8b\n" - "usubl v1.8h, v1.8b, v9.8b\n" - "ldp x23, x22, [x2, #0x30]\n" - "ldp x21, x20, [x2, #0x40]\n" - "usubl v2.8h, v2.8b, v9.8b\n" - "usubl v3.8h, v3.8b, v9.8b\n" - "usubl v4.8h, v4.8b, v9.8b\n" - "add x10, x10, x3\n" - "add x28, x28, x3\n" - "add x27, x27, x3\n" - "add x26, x26, x3\n" - "add x25, x25, x3\n" - "add x24, x24, x3\n" - "add x23, x23, x3\n" - "add x22, x22, x3\n" - "add x21, x21, x3\n" - "add x20, x20, x3\n" - "tbz x4, #2, 9f\n" - "ld1 { v31.s }[0], [x10], #0x4\n" - "ld1 { v30.s }[0], [x28], #0x4\n" - "ld1 { v29.s }[0], [x27], #0x4\n" - "ld1 { v28.s }[0], [x26], #0x4\n" - "ld1 { v27.s }[0], [x25], #0x4\n" - "ld1 { v23.s }[0], [x24], #0x4\n" - "ld1 { v25.s }[0], [x23], #0x4\n" - "ld1 { v24.s }[0], [x22], #0x4\n" - "ld1 { v26.s }[0], [x21], #0x4\n" - "ld1 { v22.s }[0], [x20], #0x4\n" - "tbz x4, #1, 8f\n" - "ld1 { v31.h }[2], [x10], #0x2\n" - "ld1 { v30.h }[2], [x28], #0x2\n" - "ld1 { v29.h }[2], [x27], #0x2\n" - "ld1 { v28.h }[2], [x26], #0x2\n" - "ld1 { v27.h }[2], [x25], #0x2\n" - "ld1 { v23.h }[2], [x24], #0x2\n" - "ld1 { v25.h }[2], [x23], #0x2\n" - "ld1 { v24.h }[2], [x22], #0x2\n" - "ld1 { v26.h }[2], [x21], #0x2\n" - "ld1 { v22.h }[2], [x20], #0x2\n" - "tbz x4, #0, 11f\n" - "ld1 { v31.b }[6], [x10]\n" - "ld1 { v30.b }[6], [x28]\n" - "ld1 { v29.b }[6], [x27]\n" - "ld1 { v28.b }[6], [x26]\n" - "ld1 { v27.b }[6], [x25]\n" - "ld1 { v23.b }[6], [x24]\n" - "ld1 { v25.b }[6], [x23]\n" - "ld1 { v24.b }[6], [x22]\n" - "ld1 { v26.b }[6], [x21]\n" - "ld1 { v22.b }[6], [x20]\n" + "mov v14.16b, v24.16b\n" + "ldr d29, [x7, #0x10]\n" + "ldr d18, [x7, #0x18]\n" + "mov v27.16b, v13.16b\n" + "mov v22.16b, v24.16b\n" + "ldr d3, [x7, #0x20]\n" + "ldp x9, x28, [x6, #0x0]\n" + "mov v8.16b, v13.16b\n" + "mov v17.16b, v24.16b\n" + "ldp x27, x26, [x6, #0x10]\n" + "ldp x25, x24, [x6, #0x20]\n" + "usubl v21.8h, v21.8b, v2.8b\n" + "usubl v15.8h, v15.8b, v2.8b\n" + "ldp x23, x22, [x6, #0x30]\n" + "ldp x21, x20, [x6, #0x40]\n" + "usubl v29.8h, v29.8b, v2.8b\n" + "usubl v18.8h, v18.8b, v2.8b\n" + "usubl v3.8h, v3.8b, v2.8b\n" + "add x9, x9, x4\n" + "add x28, x28, x4\n" + "add x27, x27, x4\n" + "add x26, x26, x4\n" + "add x25, x25, x4\n" + "add x24, x24, x4\n" + "add x23, x23, x4\n" + "add x22, x22, x4\n" + "add x21, x21, x4\n" + "add x20, x20, x4\n" + "tbz x2, #2, 9f\n" + "ld1 { v10.s }[0], [x9], #0x4\n" + "ld1 { v16.s }[0], [x28], #0x4\n" + "ld1 { v23.s }[0], [x27], #0x4\n" + "ld1 { v30.s }[0], [x26], #0x4\n" + "ld1 { v4.s }[0], [x25], #0x4\n" + "ld1 { v28.s }[0], [x24], #0x4\n" + "ld1 { v31.s }[0], [x23], #0x4\n" + "ld1 { v1.s }[0], [x22], #0x4\n" + "ld1 { v9.s }[0], [x21], #0x4\n" + "ld1 { v11.s }[0], [x20], #0x4\n" + "tbz x2, #1, 8f\n" + "ld1 { v10.h }[2], [x9], #0x2\n" + "ld1 { v16.h }[2], [x28], #0x2\n" + "ld1 { v23.h }[2], [x27], #0x2\n" + "ld1 { v30.h }[2], [x26], #0x2\n" + "ld1 { v4.h }[2], [x25], #0x2\n" + "ld1 { v28.h }[2], [x24], #0x2\n" + "ld1 { v31.h }[2], [x23], #0x2\n" + "ld1 { v1.h }[2], [x22], #0x2\n" + "ld1 { v9.h }[2], [x21], #0x2\n" + "ld1 { v11.h }[2], [x20], #0x2\n" + "tbz x2, #0, 11f\n" + "ld1 { v10.b }[6], [x9]\n" + "ld1 { v16.b }[6], [x28]\n" + "ld1 { v23.b }[6], [x27]\n" + "ld1 { v30.b }[6], [x26]\n" + "ld1 { v4.b }[6], [x25]\n" + "ld1 { v28.b }[6], [x24]\n" + "ld1 { v31.b }[6], [x23]\n" + "ld1 { v1.b }[6], [x22]\n" + "ld1 { v9.b }[6], [x21]\n" + "ld1 { v11.b }[6], [x20]\n" "b 11f\n" "8:" // Oddments: Initial loads: Bit 2: Bit 1: Unset - "tbz x4, #0, 11f\n" - "ld1 { v31.b }[4], [x10]\n" - "ld1 { v30.b }[4], [x28]\n" - "ld1 { v29.b }[4], [x27]\n" - "ld1 { v28.b }[4], [x26]\n" - "ld1 { v27.b }[4], [x25]\n" - "ld1 { v23.b }[4], [x24]\n" - "ld1 { v25.b }[4], [x23]\n" - "ld1 { v24.b }[4], [x22]\n" - "ld1 { v26.b }[4], [x21]\n" - "ld1 { v22.b }[4], [x20]\n" + "tbz x2, #0, 11f\n" + "ld1 { v10.b }[4], [x9]\n" + "ld1 { v16.b }[4], [x28]\n" + "ld1 { v23.b }[4], [x27]\n" + "ld1 { v30.b }[4], [x26]\n" + "ld1 { v4.b }[4], [x25]\n" + "ld1 { v28.b }[4], [x24]\n" + "ld1 { v31.b }[4], [x23]\n" + "ld1 { v1.b }[4], [x22]\n" + "ld1 { v9.b }[4], [x21]\n" + "ld1 { v11.b }[4], [x20]\n" "b 11f\n" "9:" // Oddments: Initial loads: Bit 2: Unset - "tbz x4, #1, 10f\n" - "ld1 { v31.h }[0], [x10], #0x2\n" - "ld1 { v30.h }[0], [x28], #0x2\n" - "ld1 { v29.h }[0], [x27], #0x2\n" - "ld1 { v28.h }[0], [x26], #0x2\n" - "ld1 { v27.h }[0], [x25], #0x2\n" - "ld1 { v23.h }[0], [x24], #0x2\n" - "ld1 { v25.h }[0], [x23], #0x2\n" - "ld1 { v24.h }[0], [x22], #0x2\n" - "ld1 { v26.h }[0], [x21], #0x2\n" - "ld1 { v22.h }[0], [x20], #0x2\n" - "tbz x4, #0, 11f\n" - "ld1 { v31.b }[2], [x10]\n" - "ld1 { v30.b }[2], [x28]\n" - "ld1 { v29.b }[2], [x27]\n" - "ld1 { v28.b }[2], [x26]\n" - "ld1 { v27.b }[2], [x25]\n" - "ld1 { v23.b }[2], [x24]\n" - "ld1 { v25.b }[2], [x23]\n" - "ld1 { v24.b }[2], [x22]\n" - "ld1 { v26.b }[2], [x21]\n" - "ld1 { v22.b }[2], [x20]\n" + "tbz x2, #1, 10f\n" + "ld1 { v10.h }[0], [x9], #0x2\n" + "ld1 { v16.h }[0], [x28], #0x2\n" + "ld1 { v23.h }[0], [x27], #0x2\n" + "ld1 { v30.h }[0], [x26], #0x2\n" + "ld1 { v4.h }[0], [x25], #0x2\n" + "ld1 { v28.h }[0], [x24], #0x2\n" + "ld1 { v31.h }[0], [x23], #0x2\n" + "ld1 { v1.h }[0], [x22], #0x2\n" + "ld1 { v9.h }[0], [x21], #0x2\n" + "ld1 { v11.h }[0], [x20], #0x2\n" + "tbz x2, #0, 11f\n" + "ld1 { v10.b }[2], [x9]\n" + "ld1 { v16.b }[2], [x28]\n" + "ld1 { v23.b }[2], [x27]\n" + "ld1 { v30.b }[2], [x26]\n" + "ld1 { v4.b }[2], [x25]\n" + "ld1 { v28.b }[2], [x24]\n" + "ld1 { v31.b }[2], [x23]\n" + "ld1 { v1.b }[2], [x22]\n" + "ld1 { v9.b }[2], [x21]\n" + "ld1 { v11.b }[2], [x20]\n" "b 11f\n" "10:" // Oddments: Initial loads: Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 11f\n" - "ld1 { v31.b }[0], [x10]\n" - "ld1 { v30.b }[0], [x28]\n" - "ld1 { v29.b }[0], [x27]\n" - "ld1 { v28.b }[0], [x26]\n" - "ld1 { v27.b }[0], [x25]\n" - "ld1 { v23.b }[0], [x24]\n" - "ld1 { v25.b }[0], [x23]\n" - "ld1 { v24.b }[0], [x22]\n" - "ld1 { v26.b }[0], [x21]\n" - "ld1 { v22.b }[0], [x20]\n" + "tbz x2, #0, 11f\n" + "ld1 { v10.b }[0], [x9]\n" + "ld1 { v16.b }[0], [x28]\n" + "ld1 { v23.b }[0], [x27]\n" + "ld1 { v30.b }[0], [x26]\n" + "ld1 { v4.b }[0], [x25]\n" + "ld1 { v28.b }[0], [x24]\n" + "ld1 { v31.b }[0], [x23]\n" + "ld1 { v1.b }[0], [x22]\n" + "ld1 { v9.b }[0], [x21]\n" + "ld1 { v11.b }[0], [x20]\n" "11:" // Oddments: Initial loads: Bit 2: End - "ushll v31.8h, v31.8b, #0x0\n" + "ushll v10.8h, v10.8b, #0x0\n" + "ushll v16.8h, v16.8b, #0x0\n" + "smlal v13.4s, v10.4h, v21.4h\n" + "ldr x20, [x6, #0x50]\n" + "ushll v23.8h, v23.8b, #0x0\n" + "smlal2 v24.4s, v10.8h, v21.8h\n" + "smlal v7.4s, v16.4h, v21.4h\n" + "smlal2 v14.4s, v16.8h, v21.8h\n" + "smlal v27.4s, v23.4h, v21.4h\n" "ushll v30.8h, v30.8b, #0x0\n" - "smlal v11.4s, v31.4h, v0.4h\n" - "ldr x20, [x2, #0x50]\n" - "ushll v29.8h, v29.8b, #0x0\n" - "smlal2 v13.4s, v31.8h, v0.8h\n" - "smlal v20.4s, v30.4h, v0.4h\n" - "smlal2 v19.4s, v30.8h, v0.8h\n" - "smlal v8.4s, v29.4h, v0.4h\n" + "add x20, x20, x4\n" + "smlal2 v22.4s, v23.8h, v21.8h\n" + "ushll v4.8h, v4.8b, #0x0\n" + "smlal v8.4s, v30.4h, v21.4h\n" + "smlal2 v17.4s, v30.8h, v21.8h\n" + "smlal v13.4s, v16.4h, v15.4h\n" "ushll v28.8h, v28.8b, #0x0\n" - "add x20, x20, x3\n" - "smlal2 v7.4s, v29.8h, v0.8h\n" - "ushll v27.8h, v27.8b, #0x0\n" - "smlal v6.4s, v28.4h, v0.4h\n" - "smlal2 v5.4s, v28.8h, v0.8h\n" - "smlal v11.4s, v30.4h, v1.4h\n" - "ushll v23.8h, v23.8b, #0x0\n" - "smlal2 v13.4s, v30.8h, v1.8h\n" - "smlal v20.4s, v27.4h, v1.4h\n" - "ushll v25.8h, v25.8b, #0x0\n" - "smlal2 v19.4s, v27.8h, v1.8h\n" - "smlal v8.4s, v28.4h, v1.4h\n" - "ushll v24.8h, v24.8b, #0x0\n" - "smlal2 v7.4s, v28.8h, v1.8h\n" - "ushll v26.8h, v26.8b, #0x0\n" - "smlal v6.4s, v23.4h, v1.4h\n" - "ushll v22.8h, v22.8b, #0x0\n" - "smlal2 v5.4s, v23.8h, v1.8h\n" - "smlal v11.4s, v27.4h, v2.4h\n" - "smlal2 v13.4s, v27.8h, v2.8h\n" - "smlal v20.4s, v25.4h, v2.4h\n" - "smlal2 v19.4s, v25.8h, v2.8h\n" - "smlal v8.4s, v23.4h, v2.4h\n" - "smlal2 v7.4s, v23.8h, v2.8h\n" - "tbz x4, #2, 13f\n" - "ld1 { v31.s }[0], [x20], #0x4\n" - "tbz x4, #1, 12f\n" - "ld1 { v31.h }[2], [x20], #0x2\n" - "tbz x4, #0, 15f\n" - "ld1 { v31.b }[6], [x20]\n" + "smlal2 v24.4s, v16.8h, v15.8h\n" + "smlal v7.4s, v4.4h, v15.4h\n" + "ushll v31.8h, v31.8b, #0x0\n" + "smlal2 v14.4s, v4.8h, v15.8h\n" + "smlal v27.4s, v30.4h, v15.4h\n" + "ushll v1.8h, v1.8b, #0x0\n" + "smlal2 v22.4s, v30.8h, v15.8h\n" + "ushll v9.8h, v9.8b, #0x0\n" + "smlal v8.4s, v28.4h, v15.4h\n" + "ushll v11.8h, v11.8b, #0x0\n" + "smlal2 v17.4s, v28.8h, v15.8h\n" + "smlal v13.4s, v4.4h, v29.4h\n" + "smlal2 v24.4s, v4.8h, v29.8h\n" + "smlal v7.4s, v31.4h, v29.4h\n" + "smlal2 v14.4s, v31.8h, v29.8h\n" + "smlal v27.4s, v28.4h, v29.4h\n" + "smlal2 v22.4s, v28.8h, v29.8h\n" + "tbz x2, #2, 13f\n" + "ld1 { v5.s }[0], [x20], #0x4\n" + "tbz x2, #1, 12f\n" + "ld1 { v5.h }[2], [x20], #0x2\n" + "tbz x2, #0, 15f\n" + "ld1 { v5.b }[6], [x20]\n" "b 15f\n" "12:" // Oddments: Load (1, 3): Bit 2: Bit 1: Unset - "tbz x4, #0, 15f\n" - "ld1 { v31.b }[4], [x20]\n" + "tbz x2, #0, 15f\n" + "ld1 { v5.b }[4], [x20]\n" "b 15f\n" "13:" // Oddments: Load (1, 3): Bit 2: Unset - "tbz x4, #1, 14f\n" - "ld1 { v31.h }[0], [x20], #0x2\n" - "tbz x4, #0, 15f\n" - "ld1 { v31.b }[2], [x20]\n" + "tbz x2, #1, 14f\n" + "ld1 { v5.h }[0], [x20], #0x2\n" + "tbz x2, #0, 15f\n" + "ld1 { v5.b }[2], [x20]\n" "b 15f\n" "14:" // Oddments: Load (1, 3): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 15f\n" - "ld1 { v31.b }[0], [x20]\n" + "tbz x2, #0, 15f\n" + "ld1 { v5.b }[0], [x20]\n" "15:" // Oddments: Load (1, 3): Bit 2: End - "ushll v31.8h, v31.8b, #0x0\n" - "ldr x22, [x2, #0x58]\n" - "smlal v6.4s, v31.4h, v2.4h\n" - "smlal2 v5.4s, v31.8h, v2.8h\n" - "smlal v11.4s, v25.4h, v3.4h\n" - "smlal2 v13.4s, v25.8h, v3.8h\n" - "add x22, x22, x3\n" - "smlal v20.4s, v24.4h, v3.4h\n" - "smlal2 v19.4s, v24.8h, v3.8h\n" - "smlal v8.4s, v31.4h, v3.4h\n" - "smlal2 v7.4s, v31.8h, v3.8h\n" - "tbz x4, #2, 17f\n" - "ld1 { v30.s }[0], [x22], #0x4\n" - "tbz x4, #1, 16f\n" - "ld1 { v30.h }[2], [x22], #0x2\n" - "tbz x4, #0, 19f\n" - "ld1 { v30.b }[6], [x22]\n" + "ushll v5.8h, v5.8b, #0x0\n" + "ldr x20, [x6, #0x58]\n" + "smlal v8.4s, v5.4h, v29.4h\n" + "smlal2 v17.4s, v5.8h, v29.8h\n" + "smlal v13.4s, v31.4h, v18.4h\n" + "smlal2 v24.4s, v31.8h, v18.8h\n" + "add x20, x20, x4\n" + "smlal v7.4s, v1.4h, v18.4h\n" + "smlal2 v14.4s, v1.8h, v18.8h\n" + "smlal v27.4s, v5.4h, v18.4h\n" + "smlal2 v22.4s, v5.8h, v18.8h\n" + "tbz x2, #2, 17f\n" + "ld1 { v10.s }[0], [x20], #0x4\n" + "tbz x2, #1, 16f\n" + "ld1 { v10.h }[2], [x20], #0x2\n" + "tbz x2, #0, 19f\n" + "ld1 { v10.b }[6], [x20]\n" "b 19f\n" "16:" // Oddments: Load (1, 4): Bit 2: Bit 1: Unset - "tbz x4, #0, 19f\n" - "ld1 { v30.b }[4], [x22]\n" + "tbz x2, #0, 19f\n" + "ld1 { v10.b }[4], [x20]\n" "b 19f\n" "17:" // Oddments: Load (1, 4): Bit 2: Unset - "tbz x4, #1, 18f\n" - "ld1 { v30.h }[0], [x22], #0x2\n" - "tbz x4, #0, 19f\n" - "ld1 { v30.b }[2], [x22]\n" + "tbz x2, #1, 18f\n" + "ld1 { v10.h }[0], [x20], #0x2\n" + "tbz x2, #0, 19f\n" + "ld1 { v10.b }[2], [x20]\n" "b 19f\n" "18:" // Oddments: Load (1, 4): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 19f\n" - "ld1 { v30.b }[0], [x22]\n" + "tbz x2, #0, 19f\n" + "ld1 { v10.b }[0], [x20]\n" "19:" // Oddments: Load (1, 4): Bit 2: End - "ushll v30.8h, v30.8b, #0x0\n" - "ldr x21, [x2, #0x60]\n" - "smlal v6.4s, v30.4h, v3.4h\n" - "smlal2 v5.4s, v30.8h, v3.8h\n" - "smlal v11.4s, v24.4h, v4.4h\n" - "smlal2 v13.4s, v24.8h, v4.8h\n" - "add x21, x21, x3\n" - "tbz x4, #2, 21f\n" - "ld1 { v27.s }[0], [x21], #0x4\n" - "tbz x4, #1, 20f\n" - "ld1 { v27.h }[2], [x21], #0x2\n" - "tbz x4, #0, 23f\n" - "ld1 { v27.b }[6], [x21]\n" + "ushll v10.8h, v10.8b, #0x0\n" + "ldr x20, [x6, #0x60]\n" + "smlal v8.4s, v10.4h, v18.4h\n" + "smlal2 v17.4s, v10.8h, v18.8h\n" + "smlal v13.4s, v1.4h, v3.4h\n" + "smlal2 v24.4s, v1.8h, v3.8h\n" + "add x20, x20, x4\n" + "tbz x2, #2, 21f\n" + "ld1 { v15.s }[0], [x20], #0x4\n" + "tbz x2, #1, 20f\n" + "ld1 { v15.h }[2], [x20], #0x2\n" + "tbz x2, #0, 23f\n" + "ld1 { v15.b }[6], [x20]\n" "b 23f\n" "20:" // Oddments: Load (0, 5): Bit 2: Bit 1: Unset - "tbz x4, #0, 23f\n" - "ld1 { v27.b }[4], [x21]\n" + "tbz x2, #0, 23f\n" + "ld1 { v15.b }[4], [x20]\n" "b 23f\n" "21:" // Oddments: Load (0, 5): Bit 2: Unset - "tbz x4, #1, 22f\n" - "ld1 { v27.h }[0], [x21], #0x2\n" - "tbz x4, #0, 23f\n" - "ld1 { v27.b }[2], [x21]\n" + "tbz x2, #1, 22f\n" + "ld1 { v15.h }[0], [x20], #0x2\n" + "tbz x2, #0, 23f\n" + "ld1 { v15.b }[2], [x20]\n" "b 23f\n" "22:" // Oddments: Load (0, 5): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 23f\n" - "ld1 { v27.b }[0], [x21]\n" + "tbz x2, #0, 23f\n" + "ld1 { v15.b }[0], [x20]\n" "23:" // Oddments: Load (0, 5): Bit 2: End - "ldr d0, [x0, #0x28]\n" - "ushll v27.8h, v27.8b, #0x0\n" - "smlal v20.4s, v27.4h, v4.4h\n" - "smlal2 v19.4s, v27.8h, v4.8h\n" - "smlal v8.4s, v30.4h, v4.4h\n" - "smlal2 v7.4s, v30.8h, v4.8h\n" - "usubl v0.8h, v0.8b, v9.8b\n" - "ldr x20, [x2, #0x68]\n" - "smlal v6.4s, v26.4h, v4.4h\n" - "smlal2 v5.4s, v26.8h, v4.8h\n" - "add x20, x20, x3\n" - "smlal v11.4s, v29.4h, v0.4h\n" - "smlal2 v13.4s, v29.8h, v0.8h\n" - "smlal v20.4s, v28.4h, v0.4h\n" - "smlal2 v19.4s, v28.8h, v0.8h\n" - "smlal v8.4s, v22.4h, v0.4h\n" - "smlal2 v7.4s, v22.8h, v0.8h\n" - "tbz x4, #2, 25f\n" - "ld1 { v25.s }[0], [x20], #0x4\n" - "tbz x4, #1, 24f\n" - "ld1 { v25.h }[2], [x20], #0x2\n" - "tbz x4, #0, 27f\n" - "ld1 { v25.b }[6], [x20]\n" + "ldr d6, [x7, #0x28]\n" + "ushll v15.8h, v15.8b, #0x0\n" + "smlal v7.4s, v15.4h, v3.4h\n" + "smlal2 v14.4s, v15.8h, v3.8h\n" + "smlal v27.4s, v10.4h, v3.4h\n" + "smlal2 v22.4s, v10.8h, v3.8h\n" + "usubl v6.8h, v6.8b, v2.8b\n" + "ldr x20, [x6, #0x68]\n" + "smlal v8.4s, v9.4h, v3.4h\n" + "smlal2 v17.4s, v9.8h, v3.8h\n" + "add x20, x20, x4\n" + "smlal v13.4s, v23.4h, v6.4h\n" + "smlal2 v24.4s, v23.8h, v6.8h\n" + "smlal v7.4s, v30.4h, v6.4h\n" + "smlal2 v14.4s, v30.8h, v6.8h\n" + "smlal v27.4s, v11.4h, v6.4h\n" + "smlal2 v22.4s, v11.8h, v6.8h\n" + "tbz x2, #2, 25f\n" + "ld1 { v20.s }[0], [x20], #0x4\n" + "tbz x2, #1, 24f\n" + "ld1 { v20.h }[2], [x20], #0x2\n" + "tbz x2, #0, 27f\n" + "ld1 { v20.b }[6], [x20]\n" "b 27f\n" "24:" // Oddments: Load (2, 1): Bit 2: Bit 1: Unset - "tbz x4, #0, 27f\n" - "ld1 { v25.b }[4], [x20]\n" + "tbz x2, #0, 27f\n" + "ld1 { v20.b }[4], [x20]\n" "b 27f\n" "25:" // Oddments: Load (2, 1): Bit 2: Unset - "tbz x4, #1, 26f\n" - "ld1 { v25.h }[0], [x20], #0x2\n" - "tbz x4, #0, 27f\n" - "ld1 { v25.b }[2], [x20]\n" + "tbz x2, #1, 26f\n" + "ld1 { v20.h }[0], [x20], #0x2\n" + "tbz x2, #0, 27f\n" + "ld1 { v20.b }[2], [x20]\n" "b 27f\n" "26:" // Oddments: Load (2, 1): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 27f\n" - "ld1 { v25.b }[0], [x20]\n" + "tbz x2, #0, 27f\n" + "ld1 { v20.b }[0], [x20]\n" "27:" // Oddments: Load (2, 1): Bit 2: End - "ldr d1, [x0, #0x30]\n" - "ushll v25.8h, v25.8b, #0x0\n" - "usubl v1.8h, v1.8b, v9.8b\n" - "ldr x25, [x2, #0x70]\n" - "smlal v6.4s, v25.4h, v0.4h\n" - "smlal2 v5.4s, v25.8h, v0.8h\n" - "add x25, x25, x3\n" - "smlal v11.4s, v28.4h, v1.4h\n" - "smlal2 v13.4s, v28.8h, v1.8h\n" - "smlal v20.4s, v23.4h, v1.4h\n" - "smlal2 v19.4s, v23.8h, v1.8h\n" - "smlal v8.4s, v25.4h, v1.4h\n" - "smlal2 v7.4s, v25.8h, v1.8h\n" - "tbz x4, #2, 29f\n" - "ld1 { v24.s }[0], [x25], #0x4\n" - "tbz x4, #1, 28f\n" - "ld1 { v24.h }[2], [x25], #0x2\n" - "tbz x4, #0, 31f\n" - "ld1 { v24.b }[6], [x25]\n" + "ldr d4, [x7, #0x30]\n" + "ushll v20.8h, v20.8b, #0x0\n" + "usubl v4.8h, v4.8b, v2.8b\n" + "ldr x20, [x6, #0x70]\n" + "smlal v8.4s, v20.4h, v6.4h\n" + "smlal2 v17.4s, v20.8h, v6.8h\n" + "add x20, x20, x4\n" + "smlal v13.4s, v30.4h, v4.4h\n" + "smlal2 v24.4s, v30.8h, v4.8h\n" + "smlal v7.4s, v28.4h, v4.4h\n" + "smlal2 v14.4s, v28.8h, v4.8h\n" + "smlal v27.4s, v20.4h, v4.4h\n" + "smlal2 v22.4s, v20.8h, v4.8h\n" + "tbz x2, #2, 29f\n" + "ld1 { v23.s }[0], [x20], #0x4\n" + "tbz x2, #1, 28f\n" + "ld1 { v23.h }[2], [x20], #0x2\n" + "tbz x2, #0, 31f\n" + "ld1 { v23.b }[6], [x20]\n" "b 31f\n" "28:" // Oddments: Load (2, 2): Bit 2: Bit 1: Unset - "tbz x4, #0, 31f\n" - "ld1 { v24.b }[4], [x25]\n" + "tbz x2, #0, 31f\n" + "ld1 { v23.b }[4], [x20]\n" "b 31f\n" "29:" // Oddments: Load (2, 2): Bit 2: Unset - "tbz x4, #1, 30f\n" - "ld1 { v24.h }[0], [x25], #0x2\n" - "tbz x4, #0, 31f\n" - "ld1 { v24.b }[2], [x25]\n" + "tbz x2, #1, 30f\n" + "ld1 { v23.h }[0], [x20], #0x2\n" + "tbz x2, #0, 31f\n" + "ld1 { v23.b }[2], [x20]\n" "b 31f\n" "30:" // Oddments: Load (2, 2): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 31f\n" - "ld1 { v24.b }[0], [x25]\n" + "tbz x2, #0, 31f\n" + "ld1 { v23.b }[0], [x20]\n" "31:" // Oddments: Load (2, 2): Bit 2: End - "ldr d2, [x0, #0x38]\n" - "ushll v24.8h, v24.8b, #0x0\n" - "usubl v2.8h, v2.8b, v9.8b\n" - "ldr x26, [x2, #0x78]\n" - "smlal v6.4s, v24.4h, v1.4h\n" - "smlal2 v5.4s, v24.8h, v1.8h\n" - "add x26, x26, x3\n" - "smlal v11.4s, v23.4h, v2.4h\n" - "smlal2 v13.4s, v23.8h, v2.8h\n" - "smlal v20.4s, v31.4h, v2.4h\n" - "smlal2 v19.4s, v31.8h, v2.8h\n" - "smlal v8.4s, v24.4h, v2.4h\n" - "smlal2 v7.4s, v24.8h, v2.8h\n" - "tbz x4, #2, 33f\n" - "ld1 { v27.s }[0], [x26], #0x4\n" - "tbz x4, #1, 32f\n" - "ld1 { v27.h }[2], [x26], #0x2\n" - "tbz x4, #0, 35f\n" - "ld1 { v27.b }[6], [x26]\n" + "ldr d30, [x7, #0x38]\n" + "ushll v23.8h, v23.8b, #0x0\n" + "usubl v30.8h, v30.8b, v2.8b\n" + "ldr x20, [x6, #0x78]\n" + "smlal v8.4s, v23.4h, v4.4h\n" + "smlal2 v17.4s, v23.8h, v4.8h\n" + "add x20, x20, x4\n" + "smlal v13.4s, v28.4h, v30.4h\n" + "smlal2 v24.4s, v28.8h, v30.8h\n" + "smlal v7.4s, v5.4h, v30.4h\n" + "smlal2 v14.4s, v5.8h, v30.8h\n" + "smlal v27.4s, v23.4h, v30.4h\n" + "smlal2 v22.4s, v23.8h, v30.8h\n" + "tbz x2, #2, 33f\n" + "ld1 { v3.s }[0], [x20], #0x4\n" + "tbz x2, #1, 32f\n" + "ld1 { v3.h }[2], [x20], #0x2\n" + "tbz x2, #0, 35f\n" + "ld1 { v3.b }[6], [x20]\n" "b 35f\n" "32:" // Oddments: Load (2, 3): Bit 2: Bit 1: Unset - "tbz x4, #0, 35f\n" - "ld1 { v27.b }[4], [x26]\n" + "tbz x2, #0, 35f\n" + "ld1 { v3.b }[4], [x20]\n" "b 35f\n" "33:" // Oddments: Load (2, 3): Bit 2: Unset - "tbz x4, #1, 34f\n" - "ld1 { v27.h }[0], [x26], #0x2\n" - "tbz x4, #0, 35f\n" - "ld1 { v27.b }[2], [x26]\n" + "tbz x2, #1, 34f\n" + "ld1 { v3.h }[0], [x20], #0x2\n" + "tbz x2, #0, 35f\n" + "ld1 { v3.b }[2], [x20]\n" "b 35f\n" "34:" // Oddments: Load (2, 3): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 35f\n" - "ld1 { v27.b }[0], [x26]\n" + "tbz x2, #0, 35f\n" + "ld1 { v3.b }[0], [x20]\n" "35:" // Oddments: Load (2, 3): Bit 2: End - "ldr d3, [x0, #0x40]\n" - "ushll v27.8h, v27.8b, #0x0\n" - "usubl v3.8h, v3.8b, v9.8b\n" - "ldr x23, [x2, #0x80]\n" - "smlal v6.4s, v27.4h, v2.4h\n" - "smlal2 v5.4s, v27.8h, v2.8h\n" - "add x23, x23, x3\n" - "smlal v11.4s, v31.4h, v3.4h\n" - "smlal2 v13.4s, v31.8h, v3.8h\n" - "smlal v20.4s, v30.4h, v3.4h\n" - "smlal2 v19.4s, v30.8h, v3.8h\n" - "smlal v8.4s, v27.4h, v3.4h\n" - "smlal2 v7.4s, v27.8h, v3.8h\n" - "tbz x4, #2, 37f\n" - "ld1 { v23.s }[0], [x23], #0x4\n" - "tbz x4, #1, 36f\n" - "ld1 { v23.h }[2], [x23], #0x2\n" - "tbz x4, #0, 39f\n" - "ld1 { v23.b }[6], [x23]\n" + "ldr d16, [x7, #0x40]\n" + "ushll v3.8h, v3.8b, #0x0\n" + "usubl v16.8h, v16.8b, v2.8b\n" + "ldr x20, [x6, #0x80]\n" + "smlal v8.4s, v3.4h, v30.4h\n" + "smlal2 v17.4s, v3.8h, v30.8h\n" + "add x20, x20, x4\n" + "smlal v13.4s, v5.4h, v16.4h\n" + "smlal2 v24.4s, v5.8h, v16.8h\n" + "smlal v7.4s, v10.4h, v16.4h\n" + "smlal2 v14.4s, v10.8h, v16.8h\n" + "smlal v27.4s, v3.4h, v16.4h\n" + "smlal2 v22.4s, v3.8h, v16.8h\n" + "tbz x2, #2, 37f\n" + "ld1 { v6.s }[0], [x20], #0x4\n" + "tbz x2, #1, 36f\n" + "ld1 { v6.h }[2], [x20], #0x2\n" + "tbz x2, #0, 39f\n" + "ld1 { v6.b }[6], [x20]\n" "b 39f\n" "36:" // Oddments: Load (2, 4): Bit 2: Bit 1: Unset - "tbz x4, #0, 39f\n" - "ld1 { v23.b }[4], [x23]\n" + "tbz x2, #0, 39f\n" + "ld1 { v6.b }[4], [x20]\n" "b 39f\n" "37:" // Oddments: Load (2, 4): Bit 2: Unset - "tbz x4, #1, 38f\n" - "ld1 { v23.h }[0], [x23], #0x2\n" - "tbz x4, #0, 39f\n" - "ld1 { v23.b }[2], [x23]\n" + "tbz x2, #1, 38f\n" + "ld1 { v6.h }[0], [x20], #0x2\n" + "tbz x2, #0, 39f\n" + "ld1 { v6.b }[2], [x20]\n" "b 39f\n" "38:" // Oddments: Load (2, 4): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 39f\n" - "ld1 { v23.b }[0], [x23]\n" + "tbz x2, #0, 39f\n" + "ld1 { v6.b }[0], [x20]\n" "39:" // Oddments: Load (2, 4): Bit 2: End - "ldr d4, [x0, #0x48]\n" - "ushll v23.8h, v23.8b, #0x0\n" - "usubl v4.8h, v4.8b, v9.8b\n" - "ldr x24, [x2, #0x88]\n" - "smlal v6.4s, v23.4h, v3.4h\n" - "smlal2 v5.4s, v23.8h, v3.8h\n" - "add x24, x24, x3\n" - "smlal v11.4s, v30.4h, v4.4h\n" - "smlal2 v13.4s, v30.8h, v4.8h\n" - "smlal v20.4s, v26.4h, v4.4h\n" - "smlal2 v19.4s, v26.8h, v4.8h\n" - "smlal v8.4s, v23.4h, v4.4h\n" - "smlal2 v7.4s, v23.8h, v4.8h\n" - "tbz x4, #2, 41f\n" - "ld1 { v28.s }[0], [x24], #0x4\n" - "tbz x4, #1, 40f\n" - "ld1 { v28.h }[2], [x24], #0x2\n" - "tbz x4, #0, 43f\n" - "ld1 { v28.b }[6], [x24]\n" + "ldr d1, [x7, #0x48]\n" + "ushll v6.8h, v6.8b, #0x0\n" + "usubl v1.8h, v1.8b, v2.8b\n" + "ldr x20, [x6, #0x88]\n" + "smlal v8.4s, v6.4h, v16.4h\n" + "smlal2 v17.4s, v6.8h, v16.8h\n" + "add x20, x20, x4\n" + "smlal v13.4s, v10.4h, v1.4h\n" + "smlal2 v24.4s, v10.8h, v1.8h\n" + "smlal v7.4s, v9.4h, v1.4h\n" + "smlal2 v14.4s, v9.8h, v1.8h\n" + "smlal v27.4s, v6.4h, v1.4h\n" + "smlal2 v22.4s, v6.8h, v1.8h\n" + "tbz x2, #2, 41f\n" + "ld1 { v18.s }[0], [x20], #0x4\n" + "tbz x2, #1, 40f\n" + "ld1 { v18.h }[2], [x20], #0x2\n" + "tbz x2, #0, 43f\n" + "ld1 { v18.b }[6], [x20]\n" "b 43f\n" "40:" // Oddments: Load (2, 5): Bit 2: Bit 1: Unset - "tbz x4, #0, 43f\n" - "ld1 { v28.b }[4], [x24]\n" + "tbz x2, #0, 43f\n" + "ld1 { v18.b }[4], [x20]\n" "b 43f\n" "41:" // Oddments: Load (2, 5): Bit 2: Unset - "tbz x4, #1, 42f\n" - "ld1 { v28.h }[0], [x24], #0x2\n" - "tbz x4, #0, 43f\n" - "ld1 { v28.b }[2], [x24]\n" + "tbz x2, #1, 42f\n" + "ld1 { v18.h }[0], [x20], #0x2\n" + "tbz x2, #0, 43f\n" + "ld1 { v18.b }[2], [x20]\n" "b 43f\n" "42:" // Oddments: Load (2, 5): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 43f\n" - "ld1 { v28.b }[0], [x24]\n" + "tbz x2, #0, 43f\n" + "ld1 { v18.b }[0], [x20]\n" "43:" // Oddments: Load (2, 5): Bit 2: End - "ldr d0, [x0, #0x50]\n" - "ushll v28.8h, v28.8b, #0x0\n" - "usubl v0.8h, v0.8b, v9.8b\n" - "ldr x15, [x2, #0x90]\n" - "smlal v6.4s, v28.4h, v4.4h\n" - "smlal2 v5.4s, v28.8h, v4.8h\n" - "add x15, x15, x3\n" - "smlal v11.4s, v22.4h, v0.4h\n" - "smlal2 v13.4s, v22.8h, v0.8h\n" - "smlal v20.4s, v25.4h, v0.4h\n" - "smlal2 v19.4s, v25.8h, v0.8h\n" - "tbz x4, #2, 45f\n" - "ld1 { v31.s }[0], [x15], #0x4\n" - "tbz x4, #1, 44f\n" - "ld1 { v31.h }[2], [x15], #0x2\n" - "tbz x4, #0, 47f\n" - "ld1 { v31.b }[6], [x15]\n" + "ldr d28, [x7, #0x50]\n" + "ushll v18.8h, v18.8b, #0x0\n" + "usubl v28.8h, v28.8b, v2.8b\n" + "ldr x20, [x6, #0x90]\n" + "smlal v8.4s, v18.4h, v1.4h\n" + "smlal2 v17.4s, v18.8h, v1.8h\n" + "add x20, x20, x4\n" + "smlal v13.4s, v11.4h, v28.4h\n" + "smlal2 v24.4s, v11.8h, v28.8h\n" + "smlal v7.4s, v20.4h, v28.4h\n" + "smlal2 v14.4s, v20.8h, v28.8h\n" + "tbz x2, #2, 45f\n" + "ld1 { v30.s }[0], [x20], #0x4\n" + "tbz x2, #1, 44f\n" + "ld1 { v30.h }[2], [x20], #0x2\n" + "tbz x2, #0, 47f\n" + "ld1 { v30.b }[6], [x20]\n" "b 47f\n" "44:" // Oddments: Load (3, 0): Bit 2: Bit 1: Unset - "tbz x4, #0, 47f\n" - "ld1 { v31.b }[4], [x15]\n" + "tbz x2, #0, 47f\n" + "ld1 { v30.b }[4], [x20]\n" "b 47f\n" "45:" // Oddments: Load (3, 0): Bit 2: Unset - "tbz x4, #1, 46f\n" - "ld1 { v31.h }[0], [x15], #0x2\n" - "tbz x4, #0, 47f\n" - "ld1 { v31.b }[2], [x15]\n" + "tbz x2, #1, 46f\n" + "ld1 { v30.h }[0], [x20], #0x2\n" + "tbz x2, #0, 47f\n" + "ld1 { v30.b }[2], [x20]\n" "b 47f\n" "46:" // Oddments: Load (3, 0): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 47f\n" - "ld1 { v31.b }[0], [x15]\n" + "tbz x2, #0, 47f\n" + "ld1 { v30.b }[0], [x20]\n" "47:" // Oddments: Load (3, 0): Bit 2: End - "ushll v31.8h, v31.8b, #0x0\n" - "ldr x21, [x2, #0x98]\n" - "smlal v8.4s, v31.4h, v0.4h\n" - "smlal2 v7.4s, v31.8h, v0.8h\n" - "add x21, x21, x3\n" - "tbz x4, #2, 49f\n" - "ld1 { v30.s }[0], [x21], #0x4\n" - "tbz x4, #1, 48f\n" - "ld1 { v30.h }[2], [x21], #0x2\n" - "tbz x4, #0, 51f\n" - "ld1 { v30.b }[6], [x21]\n" + "ushll v30.8h, v30.8b, #0x0\n" + "ldr x20, [x6, #0x98]\n" + "smlal v27.4s, v30.4h, v28.4h\n" + "smlal2 v22.4s, v30.8h, v28.8h\n" + "add x20, x20, x4\n" + "tbz x2, #2, 49f\n" + "ld1 { v19.s }[0], [x20], #0x4\n" + "tbz x2, #1, 48f\n" + "ld1 { v19.h }[2], [x20], #0x2\n" + "tbz x2, #0, 51f\n" + "ld1 { v19.b }[6], [x20]\n" "b 51f\n" "48:" // Oddments: Load (3, 1): Bit 2: Bit 1: Unset - "tbz x4, #0, 51f\n" - "ld1 { v30.b }[4], [x21]\n" + "tbz x2, #0, 51f\n" + "ld1 { v19.b }[4], [x20]\n" "b 51f\n" "49:" // Oddments: Load (3, 1): Bit 2: Unset - "tbz x4, #1, 50f\n" - "ld1 { v30.h }[0], [x21], #0x2\n" - "tbz x4, #0, 51f\n" - "ld1 { v30.b }[2], [x21]\n" + "tbz x2, #1, 50f\n" + "ld1 { v19.h }[0], [x20], #0x2\n" + "tbz x2, #0, 51f\n" + "ld1 { v19.b }[2], [x20]\n" "b 51f\n" "50:" // Oddments: Load (3, 1): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 51f\n" - "ld1 { v30.b }[0], [x21]\n" + "tbz x2, #0, 51f\n" + "ld1 { v19.b }[0], [x20]\n" "51:" // Oddments: Load (3, 1): Bit 2: End - "ldr d1, [x0, #0x58]\n" - "ushll v30.8h, v30.8b, #0x0\n" - "usubl v1.8h, v1.8b, v9.8b\n" - "ldr x14, [x2, #0xa0]\n" - "smlal v6.4s, v30.4h, v0.4h\n" - "smlal2 v5.4s, v30.8h, v0.8h\n" - "add x14, x14, x3\n" - "smlal v11.4s, v25.4h, v1.4h\n" - "smlal2 v13.4s, v25.8h, v1.8h\n" - "smlal v20.4s, v24.4h, v1.4h\n" - "smlal2 v19.4s, v24.8h, v1.8h\n" - "smlal v8.4s, v30.4h, v1.4h\n" - "smlal2 v7.4s, v30.8h, v1.8h\n" - "tbz x4, #2, 53f\n" - "ld1 { v26.s }[0], [x14], #0x4\n" - "tbz x4, #1, 52f\n" - "ld1 { v26.h }[2], [x14], #0x2\n" - "tbz x4, #0, 55f\n" - "ld1 { v26.b }[6], [x14]\n" + "ldr d0, [x7, #0x58]\n" + "ushll v19.8h, v19.8b, #0x0\n" + "usubl v0.8h, v0.8b, v2.8b\n" + "ldr x20, [x6, #0xa0]\n" + "smlal v8.4s, v19.4h, v28.4h\n" + "smlal2 v17.4s, v19.8h, v28.8h\n" + "add x20, x20, x4\n" + "smlal v13.4s, v20.4h, v0.4h\n" + "smlal2 v24.4s, v20.8h, v0.8h\n" + "smlal v7.4s, v23.4h, v0.4h\n" + "smlal2 v14.4s, v23.8h, v0.8h\n" + "smlal v27.4s, v19.4h, v0.4h\n" + "smlal2 v22.4s, v19.8h, v0.8h\n" + "tbz x2, #2, 53f\n" + "ld1 { v9.s }[0], [x20], #0x4\n" + "tbz x2, #1, 52f\n" + "ld1 { v9.h }[2], [x20], #0x2\n" + "tbz x2, #0, 55f\n" + "ld1 { v9.b }[6], [x20]\n" "b 55f\n" "52:" // Oddments: Load (3, 2): Bit 2: Bit 1: Unset - "tbz x4, #0, 55f\n" - "ld1 { v26.b }[4], [x14]\n" + "tbz x2, #0, 55f\n" + "ld1 { v9.b }[4], [x20]\n" "b 55f\n" "53:" // Oddments: Load (3, 2): Bit 2: Unset - "tbz x4, #1, 54f\n" - "ld1 { v26.h }[0], [x14], #0x2\n" - "tbz x4, #0, 55f\n" - "ld1 { v26.b }[2], [x14]\n" + "tbz x2, #1, 54f\n" + "ld1 { v9.h }[0], [x20], #0x2\n" + "tbz x2, #0, 55f\n" + "ld1 { v9.b }[2], [x20]\n" "b 55f\n" "54:" // Oddments: Load (3, 2): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 55f\n" - "ld1 { v26.b }[0], [x14]\n" + "tbz x2, #0, 55f\n" + "ld1 { v9.b }[0], [x20]\n" "55:" // Oddments: Load (3, 2): Bit 2: End - "ldr d2, [x0, #0x60]\n" - "ushll v26.8h, v26.8b, #0x0\n" - "usubl v2.8h, v2.8b, v9.8b\n" - "ldr x13, [x2, #0xa8]\n" - "smlal v6.4s, v26.4h, v1.4h\n" - "smlal2 v5.4s, v26.8h, v1.8h\n" - "add x13, x13, x3\n" - "smlal v11.4s, v24.4h, v2.4h\n" - "smlal2 v13.4s, v24.8h, v2.8h\n" - "smlal v20.4s, v27.4h, v2.4h\n" - "smlal2 v19.4s, v27.8h, v2.8h\n" - "smlal v8.4s, v26.4h, v2.4h\n" - "smlal2 v7.4s, v26.8h, v2.8h\n" - "tbz x4, #2, 57f\n" - "ld1 { v25.s }[0], [x13], #0x4\n" - "tbz x4, #1, 56f\n" - "ld1 { v25.h }[2], [x13], #0x2\n" - "tbz x4, #0, 59f\n" - "ld1 { v25.b }[6], [x13]\n" + "ldr d10, [x7, #0x60]\n" + "ushll v9.8h, v9.8b, #0x0\n" + "usubl v10.8h, v10.8b, v2.8b\n" + "ldr x20, [x6, #0xa8]\n" + "smlal v8.4s, v9.4h, v0.4h\n" + "smlal2 v17.4s, v9.8h, v0.8h\n" + "add x20, x20, x4\n" + "smlal v13.4s, v23.4h, v10.4h\n" + "smlal2 v24.4s, v23.8h, v10.8h\n" + "smlal v7.4s, v3.4h, v10.4h\n" + "smlal2 v14.4s, v3.8h, v10.8h\n" + "smlal v27.4s, v9.4h, v10.4h\n" + "smlal2 v22.4s, v9.8h, v10.8h\n" + "tbz x2, #2, 57f\n" + "ld1 { v20.s }[0], [x20], #0x4\n" + "tbz x2, #1, 56f\n" + "ld1 { v20.h }[2], [x20], #0x2\n" + "tbz x2, #0, 59f\n" + "ld1 { v20.b }[6], [x20]\n" "b 59f\n" "56:" // Oddments: Load (3, 3): Bit 2: Bit 1: Unset - "tbz x4, #0, 59f\n" - "ld1 { v25.b }[4], [x13]\n" + "tbz x2, #0, 59f\n" + "ld1 { v20.b }[4], [x20]\n" "b 59f\n" "57:" // Oddments: Load (3, 3): Bit 2: Unset - "tbz x4, #1, 58f\n" - "ld1 { v25.h }[0], [x13], #0x2\n" - "tbz x4, #0, 59f\n" - "ld1 { v25.b }[2], [x13]\n" + "tbz x2, #1, 58f\n" + "ld1 { v20.h }[0], [x20], #0x2\n" + "tbz x2, #0, 59f\n" + "ld1 { v20.b }[2], [x20]\n" "b 59f\n" "58:" // Oddments: Load (3, 3): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 59f\n" - "ld1 { v25.b }[0], [x13]\n" + "tbz x2, #0, 59f\n" + "ld1 { v20.b }[0], [x20]\n" "59:" // Oddments: Load (3, 3): Bit 2: End - "ldr d3, [x0, #0x68]\n" - "ushll v25.8h, v25.8b, #0x0\n" - "usubl v3.8h, v3.8b, v9.8b\n" - "ldr x12, [x2, #0xb0]\n" - "smlal v6.4s, v25.4h, v2.4h\n" - "smlal2 v5.4s, v25.8h, v2.8h\n" - "add x12, x12, x3\n" - "smlal v11.4s, v27.4h, v3.4h\n" - "smlal2 v13.4s, v27.8h, v3.8h\n" - "smlal v20.4s, v23.4h, v3.4h\n" - "smlal2 v19.4s, v23.8h, v3.8h\n" - "smlal v8.4s, v25.4h, v3.4h\n" - "smlal2 v7.4s, v25.8h, v3.8h\n" - "tbz x4, #2, 61f\n" - "ld1 { v24.s }[0], [x12], #0x4\n" - "tbz x4, #1, 60f\n" - "ld1 { v24.h }[2], [x12], #0x2\n" - "tbz x4, #0, 63f\n" - "ld1 { v24.b }[6], [x12]\n" + "ldr d28, [x7, #0x68]\n" + "ushll v20.8h, v20.8b, #0x0\n" + "usubl v28.8h, v28.8b, v2.8b\n" + "ldr x20, [x6, #0xb0]\n" + "smlal v8.4s, v20.4h, v10.4h\n" + "smlal2 v17.4s, v20.8h, v10.8h\n" + "add x20, x20, x4\n" + "smlal v13.4s, v3.4h, v28.4h\n" + "smlal2 v24.4s, v3.8h, v28.8h\n" + "smlal v7.4s, v6.4h, v28.4h\n" + "smlal2 v14.4s, v6.8h, v28.8h\n" + "smlal v27.4s, v20.4h, v28.4h\n" + "smlal2 v22.4s, v20.8h, v28.8h\n" + "tbz x2, #2, 61f\n" + "ld1 { v5.s }[0], [x20], #0x4\n" + "tbz x2, #1, 60f\n" + "ld1 { v5.h }[2], [x20], #0x2\n" + "tbz x2, #0, 63f\n" + "ld1 { v5.b }[6], [x20]\n" "b 63f\n" "60:" // Oddments: Load (3, 4): Bit 2: Bit 1: Unset - "tbz x4, #0, 63f\n" - "ld1 { v24.b }[4], [x12]\n" + "tbz x2, #0, 63f\n" + "ld1 { v5.b }[4], [x20]\n" "b 63f\n" "61:" // Oddments: Load (3, 4): Bit 2: Unset - "tbz x4, #1, 62f\n" - "ld1 { v24.h }[0], [x12], #0x2\n" - "tbz x4, #0, 63f\n" - "ld1 { v24.b }[2], [x12]\n" + "tbz x2, #1, 62f\n" + "ld1 { v5.h }[0], [x20], #0x2\n" + "tbz x2, #0, 63f\n" + "ld1 { v5.b }[2], [x20]\n" "b 63f\n" "62:" // Oddments: Load (3, 4): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 63f\n" - "ld1 { v24.b }[0], [x12]\n" + "tbz x2, #0, 63f\n" + "ld1 { v5.b }[0], [x20]\n" "63:" // Oddments: Load (3, 4): Bit 2: End - "ldr d4, [x0, #0x70]\n" - "ushll v24.8h, v24.8b, #0x0\n" - "usubl v4.8h, v4.8b, v9.8b\n" - "ldr x20, [x2, #0xb8]\n" - "smlal v6.4s, v24.4h, v3.4h\n" - "smlal2 v5.4s, v24.8h, v3.8h\n" - "add x20, x20, x3\n" - "smlal v11.4s, v23.4h, v4.4h\n" - "smlal2 v13.4s, v23.8h, v4.8h\n" - "smlal v20.4s, v28.4h, v4.4h\n" - "smlal2 v19.4s, v28.8h, v4.8h\n" - "smlal v8.4s, v24.4h, v4.4h\n" - "smlal2 v7.4s, v24.8h, v4.8h\n" - "tbz x4, #2, 65f\n" - "ld1 { v22.s }[0], [x20], #0x4\n" - "tbz x4, #1, 64f\n" - "ld1 { v22.h }[2], [x20], #0x2\n" - "tbz x4, #0, 67f\n" - "ld1 { v22.b }[6], [x20]\n" + "ldr d23, [x7, #0x70]\n" + "ushll v5.8h, v5.8b, #0x0\n" + "usubl v23.8h, v23.8b, v2.8b\n" + "ldr x20, [x6, #0xb8]\n" + "smlal v8.4s, v5.4h, v28.4h\n" + "smlal2 v17.4s, v5.8h, v28.8h\n" + "add x20, x20, x4\n" + "smlal v13.4s, v6.4h, v23.4h\n" + "smlal2 v24.4s, v6.8h, v23.8h\n" + "smlal v7.4s, v18.4h, v23.4h\n" + "smlal2 v14.4s, v18.8h, v23.8h\n" + "smlal v27.4s, v5.4h, v23.4h\n" + "smlal2 v22.4s, v5.8h, v23.8h\n" + "tbz x2, #2, 65f\n" + "ld1 { v29.s }[0], [x20], #0x4\n" + "tbz x2, #1, 64f\n" + "ld1 { v29.h }[2], [x20], #0x2\n" + "tbz x2, #0, 67f\n" + "ld1 { v29.b }[6], [x20]\n" "b 67f\n" "64:" // Oddments: Load (3, 5): Bit 2: Bit 1: Unset - "tbz x4, #0, 67f\n" - "ld1 { v22.b }[4], [x20]\n" + "tbz x2, #0, 67f\n" + "ld1 { v29.b }[4], [x20]\n" "b 67f\n" "65:" // Oddments: Load (3, 5): Bit 2: Unset - "tbz x4, #1, 66f\n" - "ld1 { v22.h }[0], [x20], #0x2\n" - "tbz x4, #0, 67f\n" - "ld1 { v22.b }[2], [x20]\n" + "tbz x2, #1, 66f\n" + "ld1 { v29.h }[0], [x20], #0x2\n" + "tbz x2, #0, 67f\n" + "ld1 { v29.b }[2], [x20]\n" "b 67f\n" "66:" // Oddments: Load (3, 5): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 67f\n" - "ld1 { v22.b }[0], [x20]\n" + "tbz x2, #0, 67f\n" + "ld1 { v29.b }[0], [x20]\n" "67:" // Oddments: Load (3, 5): Bit 2: End - "ldr d0, [x0, #0x78]\n" - "ushll v22.8h, v22.8b, #0x0\n" - "usubl v0.8h, v0.8b, v9.8b\n" - "ldr x11, [x2, #0xc0]\n" - "smlal v6.4s, v22.4h, v4.4h\n" - "smlal2 v5.4s, v22.8h, v4.8h\n" - "add x11, x11, x3\n" - "smlal v11.4s, v31.4h, v0.4h\n" - "smlal2 v13.4s, v31.8h, v0.8h\n" - "smlal v20.4s, v30.4h, v0.4h\n" - "smlal2 v19.4s, v30.8h, v0.8h\n" - "tbz x4, #2, 69f\n" - "ld1 { v27.s }[0], [x11], #0x4\n" - "tbz x4, #1, 68f\n" - "ld1 { v27.h }[2], [x11], #0x2\n" - "tbz x4, #0, 71f\n" - "ld1 { v27.b }[6], [x11]\n" + "ldr d4, [x7, #0x78]\n" + "ushll v29.8h, v29.8b, #0x0\n" + "usubl v4.8h, v4.8b, v2.8b\n" + "ldr x20, [x6, #0xc0]\n" + "smlal v8.4s, v29.4h, v23.4h\n" + "smlal2 v17.4s, v29.8h, v23.8h\n" + "add x20, x20, x4\n" + "smlal v13.4s, v30.4h, v4.4h\n" + "smlal2 v24.4s, v30.8h, v4.8h\n" + "smlal v7.4s, v19.4h, v4.4h\n" + "smlal2 v14.4s, v19.8h, v4.8h\n" + "tbz x2, #2, 69f\n" + "ld1 { v18.s }[0], [x20], #0x4\n" + "tbz x2, #1, 68f\n" + "ld1 { v18.h }[2], [x20], #0x2\n" + "tbz x2, #0, 71f\n" + "ld1 { v18.b }[6], [x20]\n" "b 71f\n" "68:" // Oddments: Load (4, 0): Bit 2: Bit 1: Unset - "tbz x4, #0, 71f\n" - "ld1 { v27.b }[4], [x11]\n" + "tbz x2, #0, 71f\n" + "ld1 { v18.b }[4], [x20]\n" "b 71f\n" "69:" // Oddments: Load (4, 0): Bit 2: Unset - "tbz x4, #1, 70f\n" - "ld1 { v27.h }[0], [x11], #0x2\n" - "tbz x4, #0, 71f\n" - "ld1 { v27.b }[2], [x11]\n" + "tbz x2, #1, 70f\n" + "ld1 { v18.h }[0], [x20], #0x2\n" + "tbz x2, #0, 71f\n" + "ld1 { v18.b }[2], [x20]\n" "b 71f\n" "70:" // Oddments: Load (4, 0): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 71f\n" - "ld1 { v27.b }[0], [x11]\n" + "tbz x2, #0, 71f\n" + "ld1 { v18.b }[0], [x20]\n" "71:" // Oddments: Load (4, 0): Bit 2: End - "ushll v27.8h, v27.8b, #0x0\n" - "ldr x10, [x2, #0xc8]\n" - "smlal v8.4s, v27.4h, v0.4h\n" - "smlal2 v7.4s, v27.8h, v0.8h\n" - "add x10, x10, x3\n" - "tbz x4, #2, 73f\n" - "ld1 { v23.s }[0], [x10], #0x4\n" - "tbz x4, #1, 72f\n" - "ld1 { v23.h }[2], [x10], #0x2\n" - "tbz x4, #0, 75f\n" - "ld1 { v23.b }[6], [x10]\n" + "ushll v18.8h, v18.8b, #0x0\n" + "ldr x20, [x6, #0xc8]\n" + "smlal v27.4s, v18.4h, v4.4h\n" + "smlal2 v22.4s, v18.8h, v4.8h\n" + "add x20, x20, x4\n" + "tbz x2, #2, 73f\n" + "ld1 { v1.s }[0], [x20], #0x4\n" + "tbz x2, #1, 72f\n" + "ld1 { v1.h }[2], [x20], #0x2\n" + "tbz x2, #0, 75f\n" + "ld1 { v1.b }[6], [x20]\n" "b 75f\n" "72:" // Oddments: Load (4, 1): Bit 2: Bit 1: Unset - "tbz x4, #0, 75f\n" - "ld1 { v23.b }[4], [x10]\n" + "tbz x2, #0, 75f\n" + "ld1 { v1.b }[4], [x20]\n" "b 75f\n" "73:" // Oddments: Load (4, 1): Bit 2: Unset - "tbz x4, #1, 74f\n" - "ld1 { v23.h }[0], [x10], #0x2\n" - "tbz x4, #0, 75f\n" - "ld1 { v23.b }[2], [x10]\n" + "tbz x2, #1, 74f\n" + "ld1 { v1.h }[0], [x20], #0x2\n" + "tbz x2, #0, 75f\n" + "ld1 { v1.b }[2], [x20]\n" "b 75f\n" "74:" // Oddments: Load (4, 1): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 75f\n" - "ld1 { v23.b }[0], [x10]\n" + "tbz x2, #0, 75f\n" + "ld1 { v1.b }[0], [x20]\n" "75:" // Oddments: Load (4, 1): Bit 2: End - "ldr d1, [x0, #0x80]\n" - "ushll v23.8h, v23.8b, #0x0\n" - "usubl v1.8h, v1.8b, v9.8b\n" - "ldr x22, [x2, #0xd0]\n" - "smlal v6.4s, v23.4h, v0.4h\n" - "smlal2 v5.4s, v23.8h, v0.8h\n" - "add x22, x22, x3\n" - "smlal v11.4s, v30.4h, v1.4h\n" - "smlal2 v13.4s, v30.8h, v1.8h\n" - "smlal v20.4s, v26.4h, v1.4h\n" - "smlal2 v19.4s, v26.8h, v1.8h\n" - "smlal v8.4s, v23.4h, v1.4h\n" - "smlal2 v7.4s, v23.8h, v1.8h\n" - "tbz x4, #2, 77f\n" - "ld1 { v31.s }[0], [x22], #0x4\n" - "tbz x4, #1, 76f\n" - "ld1 { v31.h }[2], [x22], #0x2\n" - "tbz x4, #0, 79f\n" - "ld1 { v31.b }[6], [x22]\n" + "ldr d23, [x7, #0x80]\n" + "ushll v1.8h, v1.8b, #0x0\n" + "usubl v23.8h, v23.8b, v2.8b\n" + "ldr x20, [x6, #0xd0]\n" + "smlal v8.4s, v1.4h, v4.4h\n" + "smlal2 v17.4s, v1.8h, v4.8h\n" + "add x20, x20, x4\n" + "smlal v13.4s, v19.4h, v23.4h\n" + "smlal2 v24.4s, v19.8h, v23.8h\n" + "smlal v7.4s, v9.4h, v23.4h\n" + "smlal2 v14.4s, v9.8h, v23.8h\n" + "smlal v27.4s, v1.4h, v23.4h\n" + "smlal2 v22.4s, v1.8h, v23.8h\n" + "tbz x2, #2, 77f\n" + "ld1 { v4.s }[0], [x20], #0x4\n" + "tbz x2, #1, 76f\n" + "ld1 { v4.h }[2], [x20], #0x2\n" + "tbz x2, #0, 79f\n" + "ld1 { v4.b }[6], [x20]\n" "b 79f\n" "76:" // Oddments: Load (4, 2): Bit 2: Bit 1: Unset - "tbz x4, #0, 79f\n" - "ld1 { v31.b }[4], [x22]\n" + "tbz x2, #0, 79f\n" + "ld1 { v4.b }[4], [x20]\n" "b 79f\n" "77:" // Oddments: Load (4, 2): Bit 2: Unset - "tbz x4, #1, 78f\n" - "ld1 { v31.h }[0], [x22], #0x2\n" - "tbz x4, #0, 79f\n" - "ld1 { v31.b }[2], [x22]\n" + "tbz x2, #1, 78f\n" + "ld1 { v4.h }[0], [x20], #0x2\n" + "tbz x2, #0, 79f\n" + "ld1 { v4.b }[2], [x20]\n" "b 79f\n" "78:" // Oddments: Load (4, 2): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 79f\n" - "ld1 { v31.b }[0], [x22]\n" + "tbz x2, #0, 79f\n" + "ld1 { v4.b }[0], [x20]\n" "79:" // Oddments: Load (4, 2): Bit 2: End - "ldr d2, [x0, #0x88]\n" - "ushll v31.8h, v31.8b, #0x0\n" - "usubl v2.8h, v2.8b, v9.8b\n" - "ldr x28, [x2, #0xd8]\n" - "smlal v6.4s, v31.4h, v1.4h\n" - "smlal2 v5.4s, v31.8h, v1.8h\n" - "add x28, x28, x3\n" - "smlal v11.4s, v26.4h, v2.4h\n" - "smlal2 v13.4s, v26.8h, v2.8h\n" - "smlal v20.4s, v25.4h, v2.4h\n" - "smlal2 v19.4s, v25.8h, v2.8h\n" - "smlal v8.4s, v31.4h, v2.4h\n" - "smlal2 v7.4s, v31.8h, v2.8h\n" - "tbz x4, #2, 81f\n" - "ld1 { v30.s }[0], [x28], #0x4\n" - "tbz x4, #1, 80f\n" - "ld1 { v30.h }[2], [x28], #0x2\n" - "tbz x4, #0, 83f\n" - "ld1 { v30.b }[6], [x28]\n" + "ldr d30, [x7, #0x88]\n" + "ushll v4.8h, v4.8b, #0x0\n" + "usubl v30.8h, v30.8b, v2.8b\n" + "ldr x20, [x6, #0xd8]\n" + "smlal v8.4s, v4.4h, v23.4h\n" + "smlal2 v17.4s, v4.8h, v23.8h\n" + "add x20, x20, x4\n" + "smlal v13.4s, v9.4h, v30.4h\n" + "smlal2 v24.4s, v9.8h, v30.8h\n" + "smlal v7.4s, v20.4h, v30.4h\n" + "smlal2 v14.4s, v20.8h, v30.8h\n" + "smlal v27.4s, v4.4h, v30.4h\n" + "smlal2 v22.4s, v4.8h, v30.8h\n" + "tbz x2, #2, 81f\n" + "ld1 { v21.s }[0], [x20], #0x4\n" + "tbz x2, #1, 80f\n" + "ld1 { v21.h }[2], [x20], #0x2\n" + "tbz x2, #0, 83f\n" + "ld1 { v21.b }[6], [x20]\n" "b 83f\n" "80:" // Oddments: Load (4, 3): Bit 2: Bit 1: Unset - "tbz x4, #0, 83f\n" - "ld1 { v30.b }[4], [x28]\n" + "tbz x2, #0, 83f\n" + "ld1 { v21.b }[4], [x20]\n" "b 83f\n" "81:" // Oddments: Load (4, 3): Bit 2: Unset - "tbz x4, #1, 82f\n" - "ld1 { v30.h }[0], [x28], #0x2\n" - "tbz x4, #0, 83f\n" - "ld1 { v30.b }[2], [x28]\n" + "tbz x2, #1, 82f\n" + "ld1 { v21.h }[0], [x20], #0x2\n" + "tbz x2, #0, 83f\n" + "ld1 { v21.b }[2], [x20]\n" "b 83f\n" "82:" // Oddments: Load (4, 3): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 83f\n" - "ld1 { v30.b }[0], [x28]\n" + "tbz x2, #0, 83f\n" + "ld1 { v21.b }[0], [x20]\n" "83:" // Oddments: Load (4, 3): Bit 2: End - "ldr d3, [x0, #0x90]\n" - "ushll v30.8h, v30.8b, #0x0\n" - "usubl v3.8h, v3.8b, v9.8b\n" - "ldr x27, [x2, #0xe0]\n" - "smlal v6.4s, v30.4h, v2.4h\n" - "smlal2 v5.4s, v30.8h, v2.8h\n" - "add x27, x27, x3\n" - "smlal v11.4s, v25.4h, v3.4h\n" - "smlal2 v13.4s, v25.8h, v3.8h\n" - "smlal v20.4s, v24.4h, v3.4h\n" - "smlal2 v19.4s, v24.8h, v3.8h\n" - "smlal v8.4s, v30.4h, v3.4h\n" - "smlal2 v7.4s, v30.8h, v3.8h\n" - "tbz x4, #2, 85f\n" - "ld1 { v28.s }[0], [x27], #0x4\n" - "tbz x4, #1, 84f\n" - "ld1 { v28.h }[2], [x27], #0x2\n" - "tbz x4, #0, 87f\n" - "ld1 { v28.b }[6], [x27]\n" + "ldr d3, [x7, #0x90]\n" + "ushll v21.8h, v21.8b, #0x0\n" + "usubl v3.8h, v3.8b, v2.8b\n" + "ldr x20, [x6, #0xe0]\n" + "smlal v8.4s, v21.4h, v30.4h\n" + "smlal2 v17.4s, v21.8h, v30.8h\n" + "add x20, x20, x4\n" + "smlal v13.4s, v20.4h, v3.4h\n" + "smlal2 v24.4s, v20.8h, v3.8h\n" + "smlal v7.4s, v5.4h, v3.4h\n" + "smlal2 v14.4s, v5.8h, v3.8h\n" + "smlal v27.4s, v21.4h, v3.4h\n" + "smlal2 v22.4s, v21.8h, v3.8h\n" + "tbz x2, #2, 85f\n" + "ld1 { v30.s }[0], [x20], #0x4\n" + "tbz x2, #1, 84f\n" + "ld1 { v30.h }[2], [x20], #0x2\n" + "tbz x2, #0, 87f\n" + "ld1 { v30.b }[6], [x20]\n" "b 87f\n" "84:" // Oddments: Load (4, 4): Bit 2: Bit 1: Unset - "tbz x4, #0, 87f\n" - "ld1 { v28.b }[4], [x27]\n" + "tbz x2, #0, 87f\n" + "ld1 { v30.b }[4], [x20]\n" "b 87f\n" "85:" // Oddments: Load (4, 4): Bit 2: Unset - "tbz x4, #1, 86f\n" - "ld1 { v28.h }[0], [x27], #0x2\n" - "tbz x4, #0, 87f\n" - "ld1 { v28.b }[2], [x27]\n" + "tbz x2, #1, 86f\n" + "ld1 { v30.h }[0], [x20], #0x2\n" + "tbz x2, #0, 87f\n" + "ld1 { v30.b }[2], [x20]\n" "b 87f\n" "86:" // Oddments: Load (4, 4): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 87f\n" - "ld1 { v28.b }[0], [x27]\n" + "tbz x2, #0, 87f\n" + "ld1 { v30.b }[0], [x20]\n" "87:" // Oddments: Load (4, 4): Bit 2: End - "ldr d4, [x0, #0x98]\n" - "ushll v28.8h, v28.8b, #0x0\n" - "usubl v4.8h, v4.8b, v9.8b\n" - "ldr x26, [x2, #0xe8]\n" - "smlal v6.4s, v28.4h, v3.4h\n" - "smlal2 v5.4s, v28.8h, v3.8h\n" - "add x26, x26, x3\n" - "smlal v11.4s, v24.4h, v4.4h\n" - "smlal2 v13.4s, v24.8h, v4.8h\n" - "smlal v20.4s, v22.4h, v4.4h\n" - "smlal2 v19.4s, v22.8h, v4.8h\n" - "smlal v8.4s, v28.4h, v4.4h\n" - "smlal2 v7.4s, v28.8h, v4.8h\n" - "tbz x4, #2, 89f\n" - "ld1 { v26.s }[0], [x26], #0x4\n" - "tbz x4, #1, 88f\n" - "ld1 { v26.h }[2], [x26], #0x2\n" - "tbz x4, #0, 91f\n" - "ld1 { v26.b }[6], [x26]\n" + "ldr d19, [x7, #0x98]\n" + "ushll v30.8h, v30.8b, #0x0\n" + "usubl v19.8h, v19.8b, v2.8b\n" + "ldr x20, [x6, #0xe8]\n" + "smlal v8.4s, v30.4h, v3.4h\n" + "smlal2 v17.4s, v30.8h, v3.8h\n" + "add x20, x20, x4\n" + "smlal v13.4s, v5.4h, v19.4h\n" + "smlal2 v24.4s, v5.8h, v19.8h\n" + "smlal v7.4s, v29.4h, v19.4h\n" + "smlal2 v14.4s, v29.8h, v19.8h\n" + "smlal v27.4s, v30.4h, v19.4h\n" + "smlal2 v22.4s, v30.8h, v19.8h\n" + "tbz x2, #2, 89f\n" + "ld1 { v20.s }[0], [x20], #0x4\n" + "tbz x2, #1, 88f\n" + "ld1 { v20.h }[2], [x20], #0x2\n" + "tbz x2, #0, 91f\n" + "ld1 { v20.b }[6], [x20]\n" "b 91f\n" "88:" // Oddments: Load (4, 5): Bit 2: Bit 1: Unset - "tbz x4, #0, 91f\n" - "ld1 { v26.b }[4], [x26]\n" + "tbz x2, #0, 91f\n" + "ld1 { v20.b }[4], [x20]\n" "b 91f\n" "89:" // Oddments: Load (4, 5): Bit 2: Unset - "tbz x4, #1, 90f\n" - "ld1 { v26.h }[0], [x26], #0x2\n" - "tbz x4, #0, 91f\n" - "ld1 { v26.b }[2], [x26]\n" + "tbz x2, #1, 90f\n" + "ld1 { v20.h }[0], [x20], #0x2\n" + "tbz x2, #0, 91f\n" + "ld1 { v20.b }[2], [x20]\n" "b 91f\n" "90:" // Oddments: Load (4, 5): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 91f\n" - "ld1 { v26.b }[0], [x26]\n" + "tbz x2, #0, 91f\n" + "ld1 { v20.b }[0], [x20]\n" "91:" // Oddments: Load (4, 5): Bit 2: End - "ldr d0, [x0, #0xa0]\n" - "ushll v26.8h, v26.8b, #0x0\n" - "usubl v0.8h, v0.8b, v9.8b\n" - "ldr x25, [x2, #0xf0]\n" - "smlal v6.4s, v26.4h, v4.4h\n" - "smlal2 v5.4s, v26.8h, v4.8h\n" - "add x25, x25, x3\n" - "smlal v11.4s, v27.4h, v0.4h\n" - "smlal2 v13.4s, v27.8h, v0.8h\n" - "smlal v20.4s, v23.4h, v0.4h\n" - "smlal2 v19.4s, v23.8h, v0.8h\n" - "tbz x4, #2, 93f\n" - "ld1 { v25.s }[0], [x25], #0x4\n" - "tbz x4, #1, 92f\n" - "ld1 { v25.h }[2], [x25], #0x2\n" - "tbz x4, #0, 95f\n" - "ld1 { v25.b }[6], [x25]\n" + "ldr d23, [x7, #0xa0]\n" + "ushll v20.8h, v20.8b, #0x0\n" + "usubl v23.8h, v23.8b, v2.8b\n" + "ldr x20, [x6, #0xf0]\n" + "smlal v8.4s, v20.4h, v19.4h\n" + "smlal2 v17.4s, v20.8h, v19.8h\n" + "add x20, x20, x4\n" + "smlal v13.4s, v18.4h, v23.4h\n" + "smlal2 v24.4s, v18.8h, v23.8h\n" + "smlal v7.4s, v1.4h, v23.4h\n" + "smlal2 v14.4s, v1.8h, v23.8h\n" + "tbz x2, #2, 93f\n" + "ld1 { v10.s }[0], [x20], #0x4\n" + "tbz x2, #1, 92f\n" + "ld1 { v10.h }[2], [x20], #0x2\n" + "tbz x2, #0, 95f\n" + "ld1 { v10.b }[6], [x20]\n" "b 95f\n" "92:" // Oddments: Load (5, 0): Bit 2: Bit 1: Unset - "tbz x4, #0, 95f\n" - "ld1 { v25.b }[4], [x25]\n" + "tbz x2, #0, 95f\n" + "ld1 { v10.b }[4], [x20]\n" "b 95f\n" "93:" // Oddments: Load (5, 0): Bit 2: Unset - "tbz x4, #1, 94f\n" - "ld1 { v25.h }[0], [x25], #0x2\n" - "tbz x4, #0, 95f\n" - "ld1 { v25.b }[2], [x25]\n" + "tbz x2, #1, 94f\n" + "ld1 { v10.h }[0], [x20], #0x2\n" + "tbz x2, #0, 95f\n" + "ld1 { v10.b }[2], [x20]\n" "b 95f\n" "94:" // Oddments: Load (5, 0): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 95f\n" - "ld1 { v25.b }[0], [x25]\n" + "tbz x2, #0, 95f\n" + "ld1 { v10.b }[0], [x20]\n" "95:" // Oddments: Load (5, 0): Bit 2: End - "ushll v25.8h, v25.8b, #0x0\n" - "ldr x24, [x2, #0xf8]\n" - "smlal v8.4s, v25.4h, v0.4h\n" - "smlal2 v7.4s, v25.8h, v0.8h\n" - "add x24, x24, x3\n" - "tbz x4, #2, 97f\n" - "ld1 { v24.s }[0], [x24], #0x4\n" - "tbz x4, #1, 96f\n" - "ld1 { v24.h }[2], [x24], #0x2\n" - "tbz x4, #0, 99f\n" - "ld1 { v24.b }[6], [x24]\n" + "ushll v10.8h, v10.8b, #0x0\n" + "ldr x20, [x6, #0xf8]\n" + "smlal v27.4s, v10.4h, v23.4h\n" + "smlal2 v22.4s, v10.8h, v23.8h\n" + "add x20, x20, x4\n" + "tbz x2, #2, 97f\n" + "ld1 { v18.s }[0], [x20], #0x4\n" + "tbz x2, #1, 96f\n" + "ld1 { v18.h }[2], [x20], #0x2\n" + "tbz x2, #0, 99f\n" + "ld1 { v18.b }[6], [x20]\n" "b 99f\n" "96:" // Oddments: Load (5, 1): Bit 2: Bit 1: Unset - "tbz x4, #0, 99f\n" - "ld1 { v24.b }[4], [x24]\n" + "tbz x2, #0, 99f\n" + "ld1 { v18.b }[4], [x20]\n" "b 99f\n" "97:" // Oddments: Load (5, 1): Bit 2: Unset - "tbz x4, #1, 98f\n" - "ld1 { v24.h }[0], [x24], #0x2\n" - "tbz x4, #0, 99f\n" - "ld1 { v24.b }[2], [x24]\n" + "tbz x2, #1, 98f\n" + "ld1 { v18.h }[0], [x20], #0x2\n" + "tbz x2, #0, 99f\n" + "ld1 { v18.b }[2], [x20]\n" "b 99f\n" "98:" // Oddments: Load (5, 1): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 99f\n" - "ld1 { v24.b }[0], [x24]\n" + "tbz x2, #0, 99f\n" + "ld1 { v18.b }[0], [x20]\n" "99:" // Oddments: Load (5, 1): Bit 2: End - "ldr d1, [x0, #0xa8]\n" - "ushll v24.8h, v24.8b, #0x0\n" - "usubl v1.8h, v1.8b, v9.8b\n" - "ldr x23, [x2, #0x100]\n" - "smlal v6.4s, v24.4h, v0.4h\n" - "smlal2 v5.4s, v24.8h, v0.8h\n" - "add x23, x23, x3\n" - "smlal v11.4s, v23.4h, v1.4h\n" - "smlal2 v13.4s, v23.8h, v1.8h\n" - "smlal v20.4s, v31.4h, v1.4h\n" - "smlal2 v19.4s, v31.8h, v1.8h\n" - "smlal v8.4s, v24.4h, v1.4h\n" - "smlal2 v7.4s, v24.8h, v1.8h\n" - "tbz x4, #2, 101f\n" - "ld1 { v27.s }[0], [x23], #0x4\n" - "tbz x4, #1, 100f\n" - "ld1 { v27.h }[2], [x23], #0x2\n" - "tbz x4, #0, 103f\n" - "ld1 { v27.b }[6], [x23]\n" + "ldr d5, [x7, #0xa8]\n" + "ushll v18.8h, v18.8b, #0x0\n" + "usubl v5.8h, v5.8b, v2.8b\n" + "ldr x20, [x6, #0x100]\n" + "smlal v8.4s, v18.4h, v23.4h\n" + "smlal2 v17.4s, v18.8h, v23.8h\n" + "add x20, x20, x4\n" + "smlal v13.4s, v1.4h, v5.4h\n" + "smlal2 v24.4s, v1.8h, v5.8h\n" + "smlal v7.4s, v4.4h, v5.4h\n" + "smlal2 v14.4s, v4.8h, v5.8h\n" + "smlal v27.4s, v18.4h, v5.4h\n" + "smlal2 v22.4s, v18.8h, v5.8h\n" + "tbz x2, #2, 101f\n" + "ld1 { v9.s }[0], [x20], #0x4\n" + "tbz x2, #1, 100f\n" + "ld1 { v9.h }[2], [x20], #0x2\n" + "tbz x2, #0, 103f\n" + "ld1 { v9.b }[6], [x20]\n" "b 103f\n" "100:" // Oddments: Load (5, 2): Bit 2: Bit 1: Unset - "tbz x4, #0, 103f\n" - "ld1 { v27.b }[4], [x23]\n" + "tbz x2, #0, 103f\n" + "ld1 { v9.b }[4], [x20]\n" "b 103f\n" "101:" // Oddments: Load (5, 2): Bit 2: Unset - "tbz x4, #1, 102f\n" - "ld1 { v27.h }[0], [x23], #0x2\n" - "tbz x4, #0, 103f\n" - "ld1 { v27.b }[2], [x23]\n" + "tbz x2, #1, 102f\n" + "ld1 { v9.h }[0], [x20], #0x2\n" + "tbz x2, #0, 103f\n" + "ld1 { v9.b }[2], [x20]\n" "b 103f\n" "102:" // Oddments: Load (5, 2): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 103f\n" - "ld1 { v27.b }[0], [x23]\n" + "tbz x2, #0, 103f\n" + "ld1 { v9.b }[0], [x20]\n" "103:" // Oddments: Load (5, 2): Bit 2: End - "ldr d2, [x0, #0xb0]\n" - "ushll v27.8h, v27.8b, #0x0\n" - "usubl v2.8h, v2.8b, v9.8b\n" - "ldr x15, [x2, #0x108]\n" - "smlal v6.4s, v27.4h, v1.4h\n" - "smlal2 v5.4s, v27.8h, v1.8h\n" - "add x15, x15, x3\n" - "smlal v11.4s, v31.4h, v2.4h\n" - "smlal2 v13.4s, v31.8h, v2.8h\n" - "smlal v20.4s, v30.4h, v2.4h\n" - "smlal2 v19.4s, v30.8h, v2.8h\n" - "smlal v8.4s, v27.4h, v2.4h\n" - "smlal2 v7.4s, v27.8h, v2.8h\n" - "tbz x4, #2, 105f\n" - "ld1 { v25.s }[0], [x15], #0x4\n" - "tbz x4, #1, 104f\n" - "ld1 { v25.h }[2], [x15], #0x2\n" - "tbz x4, #0, 107f\n" - "ld1 { v25.b }[6], [x15]\n" + "ldr d18, [x7, #0xb0]\n" + "ushll v9.8h, v9.8b, #0x0\n" + "usubl v18.8h, v18.8b, v2.8b\n" + "ldr x20, [x6, #0x108]\n" + "smlal v8.4s, v9.4h, v5.4h\n" + "smlal2 v17.4s, v9.8h, v5.8h\n" + "add x20, x20, x4\n" + "smlal v13.4s, v4.4h, v18.4h\n" + "smlal2 v24.4s, v4.8h, v18.8h\n" + "smlal v7.4s, v21.4h, v18.4h\n" + "smlal2 v14.4s, v21.8h, v18.8h\n" + "smlal v27.4s, v9.4h, v18.4h\n" + "smlal2 v22.4s, v9.8h, v18.8h\n" + "tbz x2, #2, 105f\n" + "ld1 { v5.s }[0], [x20], #0x4\n" + "tbz x2, #1, 104f\n" + "ld1 { v5.h }[2], [x20], #0x2\n" + "tbz x2, #0, 107f\n" + "ld1 { v5.b }[6], [x20]\n" "b 107f\n" "104:" // Oddments: Load (5, 3): Bit 2: Bit 1: Unset - "tbz x4, #0, 107f\n" - "ld1 { v25.b }[4], [x15]\n" + "tbz x2, #0, 107f\n" + "ld1 { v5.b }[4], [x20]\n" "b 107f\n" "105:" // Oddments: Load (5, 3): Bit 2: Unset - "tbz x4, #1, 106f\n" - "ld1 { v25.h }[0], [x15], #0x2\n" - "tbz x4, #0, 107f\n" - "ld1 { v25.b }[2], [x15]\n" + "tbz x2, #1, 106f\n" + "ld1 { v5.h }[0], [x20], #0x2\n" + "tbz x2, #0, 107f\n" + "ld1 { v5.b }[2], [x20]\n" "b 107f\n" "106:" // Oddments: Load (5, 3): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 107f\n" - "ld1 { v25.b }[0], [x15]\n" + "tbz x2, #0, 107f\n" + "ld1 { v5.b }[0], [x20]\n" "107:" // Oddments: Load (5, 3): Bit 2: End - "ldr d3, [x0, #0xb8]\n" - "ushll v25.8h, v25.8b, #0x0\n" - "usubl v3.8h, v3.8b, v9.8b\n" - "ldr x21, [x2, #0x110]\n" - "smlal v6.4s, v25.4h, v2.4h\n" - "smlal2 v5.4s, v25.8h, v2.8h\n" - "add x21, x21, x3\n" - "smlal v11.4s, v30.4h, v3.4h\n" - "smlal2 v13.4s, v30.8h, v3.8h\n" - "smlal v20.4s, v28.4h, v3.4h\n" - "smlal2 v19.4s, v28.8h, v3.8h\n" - "smlal v8.4s, v25.4h, v3.4h\n" - "smlal2 v7.4s, v25.8h, v3.8h\n" - "tbz x4, #2, 109f\n" - "ld1 { v24.s }[0], [x21], #0x4\n" - "tbz x4, #1, 108f\n" - "ld1 { v24.h }[2], [x21], #0x2\n" - "tbz x4, #0, 111f\n" - "ld1 { v24.b }[6], [x21]\n" + "ldr d11, [x7, #0xb8]\n" + "ushll v5.8h, v5.8b, #0x0\n" + "usubl v11.8h, v11.8b, v2.8b\n" + "ldr x20, [x6, #0x110]\n" + "smlal v8.4s, v5.4h, v18.4h\n" + "smlal2 v17.4s, v5.8h, v18.8h\n" + "add x20, x20, x4\n" + "smlal v13.4s, v21.4h, v11.4h\n" + "smlal2 v24.4s, v21.8h, v11.8h\n" + "smlal v7.4s, v30.4h, v11.4h\n" + "smlal2 v14.4s, v30.8h, v11.8h\n" + "smlal v27.4s, v5.4h, v11.4h\n" + "smlal2 v22.4s, v5.8h, v11.8h\n" + "tbz x2, #2, 109f\n" + "ld1 { v18.s }[0], [x20], #0x4\n" + "tbz x2, #1, 108f\n" + "ld1 { v18.h }[2], [x20], #0x2\n" + "tbz x2, #0, 111f\n" + "ld1 { v18.b }[6], [x20]\n" "b 111f\n" "108:" // Oddments: Load (5, 4): Bit 2: Bit 1: Unset - "tbz x4, #0, 111f\n" - "ld1 { v24.b }[4], [x21]\n" + "tbz x2, #0, 111f\n" + "ld1 { v18.b }[4], [x20]\n" "b 111f\n" "109:" // Oddments: Load (5, 4): Bit 2: Unset - "tbz x4, #1, 110f\n" - "ld1 { v24.h }[0], [x21], #0x2\n" - "tbz x4, #0, 111f\n" - "ld1 { v24.b }[2], [x21]\n" + "tbz x2, #1, 110f\n" + "ld1 { v18.h }[0], [x20], #0x2\n" + "tbz x2, #0, 111f\n" + "ld1 { v18.b }[2], [x20]\n" "b 111f\n" "110:" // Oddments: Load (5, 4): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 111f\n" - "ld1 { v24.b }[0], [x21]\n" + "tbz x2, #0, 111f\n" + "ld1 { v18.b }[0], [x20]\n" "111:" // Oddments: Load (5, 4): Bit 2: End - "ldr d4, [x0, #0xc0]\n" - "ushll v24.8h, v24.8b, #0x0\n" - "usubl v4.8h, v4.8b, v9.8b\n" - "ldr x20, [x2, #0x118]\n" - "smlal v6.4s, v24.4h, v3.4h\n" - "smlal2 v5.4s, v24.8h, v3.8h\n" - "add x20, x20, x3\n" - "smlal v11.4s, v28.4h, v4.4h\n" - "smlal2 v13.4s, v28.8h, v4.8h\n" - "smlal v20.4s, v26.4h, v4.4h\n" - "smlal2 v19.4s, v26.8h, v4.8h\n" - "smlal v8.4s, v24.4h, v4.4h\n" - "smlal2 v7.4s, v24.8h, v4.8h\n" - "tbz x4, #2, 113f\n" - "ld1 { v27.s }[0], [x20], #0x4\n" - "tbz x4, #1, 112f\n" - "ld1 { v27.h }[2], [x20], #0x2\n" - "tbz x4, #0, 115f\n" - "ld1 { v27.b }[6], [x20]\n" + "ldr d16, [x7, #0xc0]\n" + "ushll v18.8h, v18.8b, #0x0\n" + "usubl v16.8h, v16.8b, v2.8b\n" + "ldr x20, [x6, #0x118]\n" + "smlal v8.4s, v18.4h, v11.4h\n" + "smlal2 v17.4s, v18.8h, v11.8h\n" + "add x20, x20, x4\n" + "smlal v13.4s, v30.4h, v16.4h\n" + "smlal2 v24.4s, v30.8h, v16.8h\n" + "smlal v7.4s, v20.4h, v16.4h\n" + "smlal2 v14.4s, v20.8h, v16.8h\n" + "smlal v27.4s, v18.4h, v16.4h\n" + "smlal2 v22.4s, v18.8h, v16.8h\n" + "tbz x2, #2, 113f\n" + "ld1 { v21.s }[0], [x20], #0x4\n" + "tbz x2, #1, 112f\n" + "ld1 { v21.h }[2], [x20], #0x2\n" + "tbz x2, #0, 115f\n" + "ld1 { v21.b }[6], [x20]\n" "b 115f\n" "112:" // Oddments: Load (5, 5): Bit 2: Bit 1: Unset - "tbz x4, #0, 115f\n" - "ld1 { v27.b }[4], [x20]\n" + "tbz x2, #0, 115f\n" + "ld1 { v21.b }[4], [x20]\n" "b 115f\n" "113:" // Oddments: Load (5, 5): Bit 2: Unset - "tbz x4, #1, 114f\n" - "ld1 { v27.h }[0], [x20], #0x2\n" - "tbz x4, #0, 115f\n" - "ld1 { v27.b }[2], [x20]\n" + "tbz x2, #1, 114f\n" + "ld1 { v21.h }[0], [x20], #0x2\n" + "tbz x2, #0, 115f\n" + "ld1 { v21.b }[2], [x20]\n" "b 115f\n" "114:" // Oddments: Load (5, 5): Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 115f\n" - "ld1 { v27.b }[0], [x20]\n" + "tbz x2, #0, 115f\n" + "ld1 { v21.b }[0], [x20]\n" "115:" // Oddments: Load (5, 5): Bit 2: End - "ushll v27.8h, v27.8b, #0x0\n" - "smlal v6.4s, v27.4h, v4.4h\n" - "smlal2 v5.4s, v27.8h, v4.8h\n" - "tbz x4, #2, 117f\n" - "ld1 { v18.4s }, [x6], #0x10\n" - "ld1 { v21.4s }, [x5], #0x10\n" - "tbz x4, #1, 116f\n" - "ld1 { v16.d }[0], [x6], #0x8\n" - "ld1 { v10.d }[0], [x5], #0x8\n" - "tbz x4, #0, 119f\n" - "ld1 { v16.s }[2], [x6]\n" - "ld1 { v10.s }[2], [x5]\n" + "ushll v21.8h, v21.8b, #0x0\n" + "smlal v8.4s, v21.4h, v16.4h\n" + "smlal2 v17.4s, v21.8h, v16.8h\n" + "tbz x2, #2, 117f\n" + "ld1 { v16.4s }, [x8], #0x10\n" + "ld1 { v21.4s }, [x17], #0x10\n" + "tbz x2, #1, 116f\n" + "ld1 { v18.d }[0], [x8], #0x8\n" + "ld1 { v0.d }[0], [x17], #0x8\n" + "tbz x2, #0, 119f\n" + "ld1 { v18.s }[2], [x8]\n" + "ld1 { v0.s }[2], [x17]\n" "b 119f\n" "116:" // Oddments: Load requant params: Bit 2: Bit 1: Unset - "tbz x4, #0, 119f\n" - "ld1 { v16.s }[0], [x6]\n" - "ld1 { v10.s }[0], [x5]\n" + "tbz x2, #0, 119f\n" + "ld1 { v18.s }[0], [x8]\n" + "ld1 { v0.s }[0], [x17]\n" "b 119f\n" "117:" // Oddments: Load requant params: Bit 2: Unset - "tbz x4, #1, 118f\n" - "ld1 { v18.d }[0], [x6], #0x8\n" - "ld1 { v21.d }[0], [x5], #0x8\n" - "tbz x4, #0, 119f\n" - "ld1 { v18.s }[2], [x6]\n" - "ld1 { v21.s }[2], [x5]\n" + "tbz x2, #1, 118f\n" + "ld1 { v16.d }[0], [x8], #0x8\n" + "ld1 { v21.d }[0], [x17], #0x8\n" + "tbz x2, #0, 119f\n" + "ld1 { v16.s }[2], [x8]\n" + "ld1 { v21.s }[2], [x17]\n" "b 119f\n" "118:" // Oddments: Load requant params: Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 119f\n" - "ld1 { v18.s }[0], [x6]\n" - "ld1 { v21.s }[0], [x5]\n" + "tbz x2, #0, 119f\n" + "ld1 { v16.s }[0], [x8]\n" + "ld1 { v21.s }[0], [x17]\n" "119:" // Oddments: Load requant params: Bit 2: End - "sqrdmulh v11.4s, v11.4s, v18.4s\n" - "and v31.16b, v11.16b, v21.16b\n" - "add x7, x7, x1\n" - "add x8, x8, x1\n" "sqrdmulh v13.4s, v13.4s, v16.4s\n" - "sshr v31.4s, v31.4s, #0x1f\n" - "add x17, x17, x1\n" - "add x16, x16, x1\n" - "and v17.16b, v13.16b, v10.16b\n" - "sqrdmulh v20.4s, v20.4s, v18.4s\n" - "sqrdmulh v8.4s, v8.4s, v18.4s\n" - "sqrdmulh v6.4s, v6.4s, v18.4s\n" - "sqadd v11.4s, v11.4s, v31.4s\n" - "sshr v17.4s, v17.4s, #0x1f\n" - "and v26.16b, v20.16b, v21.16b\n" - "sqrdmulh v19.4s, v19.4s, v16.4s\n" - "and v18.16b, v8.16b, v21.16b\n" + "and v5.16b, v13.16b, v21.16b\n" + "add x16, x16, x5\n" + "add x15, x15, x5\n" + "sqrdmulh v24.4s, v24.4s, v18.4s\n" + "sshr v5.4s, v5.4s, #0x1f\n" + "add x14, x14, x5\n" + "add x13, x13, x5\n" + "and v2.16b, v24.16b, v0.16b\n" "sqrdmulh v7.4s, v7.4s, v16.4s\n" - "and v31.16b, v6.16b, v21.16b\n" - "sqrdmulh v5.4s, v5.4s, v16.4s\n" - "sqadd v13.4s, v13.4s, v17.4s\n" - "sshr v26.4s, v26.4s, #0x1f\n" - "and v27.16b, v19.16b, v10.16b\n" - "sshr v18.4s, v18.4s, #0x1f\n" - "and v25.16b, v7.16b, v10.16b\n" + "sqrdmulh v27.4s, v27.4s, v16.4s\n" + "sqrdmulh v8.4s, v8.4s, v16.4s\n" + "sqadd v13.4s, v13.4s, v5.4s\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "and v23.16b, v7.16b, v21.16b\n" + "sqrdmulh v14.4s, v14.4s, v18.4s\n" + "and v20.16b, v27.16b, v21.16b\n" + "sqrdmulh v22.4s, v22.4s, v18.4s\n" + "and v31.16b, v8.16b, v21.16b\n" + "sqrdmulh v17.4s, v17.4s, v18.4s\n" + "sqadd v24.4s, v24.4s, v2.4s\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "and v18.16b, v14.16b, v0.16b\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "and v11.16b, v22.16b, v0.16b\n" "sshr v31.4s, v31.4s, #0x1f\n" - "and v17.16b, v5.16b, v10.16b\n" - "sqadd v20.4s, v20.4s, v26.4s\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v18.4s\n" - "sshr v25.4s, v25.4s, #0x1f\n" - "sqadd v6.4s, v6.4s, v31.4s\n" - "sshr v17.4s, v17.4s, #0x1f\n" - "srshl v11.4s, v11.4s, v21.4s\n" - "srshl v20.4s, v20.4s, v21.4s\n" - "sqadd v19.4s, v19.4s, v27.4s\n" + "and v10.16b, v17.16b, v0.16b\n" + "sqadd v7.4s, v7.4s, v23.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sqadd v27.4s, v27.4s, v20.4s\n" + "sshr v11.4s, v11.4s, #0x1f\n" + "sqadd v8.4s, v8.4s, v31.4s\n" + "sshr v10.4s, v10.4s, #0x1f\n" + "srshl v13.4s, v13.4s, v21.4s\n" + "srshl v7.4s, v7.4s, v21.4s\n" + "sqadd v14.4s, v14.4s, v18.4s\n" + "srshl v27.4s, v27.4s, v21.4s\n" + "sqadd v22.4s, v22.4s, v11.4s\n" "srshl v8.4s, v8.4s, v21.4s\n" - "sqadd v7.4s, v7.4s, v25.4s\n" - "srshl v6.4s, v6.4s, v21.4s\n" - "sqadd v5.4s, v5.4s, v17.4s\n" - "srshl v13.4s, v13.4s, v10.4s\n" - "sqxtn v11.4h, v11.4s\n" - "srshl v19.4s, v19.4s, v10.4s\n" - "sqxtn v20.4h, v20.4s\n" - "srshl v7.4s, v7.4s, v10.4s\n" + "sqadd v17.4s, v17.4s, v10.4s\n" + "srshl v24.4s, v24.4s, v0.4s\n" + "sqxtn v13.4h, v13.4s\n" + "srshl v14.4s, v14.4s, v0.4s\n" + "sqxtn v7.4h, v7.4s\n" + "srshl v22.4s, v22.4s, v0.4s\n" + "sqxtn v27.4h, v27.4s\n" + "srshl v17.4s, v17.4s, v0.4s\n" "sqxtn v8.4h, v8.4s\n" - "srshl v5.4s, v5.4s, v10.4s\n" - "sqxtn v6.4h, v6.4s\n" - "sqxtn2 v11.8h, v13.4s\n" - "sqxtn2 v20.8h, v19.4s\n" - "sqxtn2 v8.8h, v7.4s\n" - "sqxtn2 v6.8h, v5.4s\n" - "sqadd v11.8h, v11.8h, v15.8h\n" - "sqadd v20.8h, v20.8h, v15.8h\n" - "sqadd v8.8h, v8.8h, v15.8h\n" - "sqadd v6.8h, v6.8h, v15.8h\n" - "smax v11.8h, v11.8h, v14.8h\n" - "smax v20.8h, v20.8h, v14.8h\n" - "smax v8.8h, v8.8h, v14.8h\n" - "smax v6.8h, v6.8h, v14.8h\n" - "smin v11.8h, v11.8h, v12.8h\n" - "smin v20.8h, v20.8h, v12.8h\n" - "smin v8.8h, v8.8h, v12.8h\n" - "smin v6.8h, v6.8h, v12.8h\n" - "uzp1 v11.16b, v11.16b, v11.16b\n" - "uzp1 v20.16b, v20.16b, v20.16b\n" + "sqxtn2 v13.8h, v24.4s\n" + "sqxtn2 v7.8h, v14.4s\n" + "sqxtn2 v27.8h, v22.4s\n" + "sqxtn2 v8.8h, v17.4s\n" + "sqadd v13.8h, v13.8h, v25.8h\n" + "sqadd v7.8h, v7.8h, v25.8h\n" + "sqadd v27.8h, v27.8h, v25.8h\n" + "sqadd v8.8h, v8.8h, v25.8h\n" + "smax v13.8h, v13.8h, v12.8h\n" + "smax v7.8h, v7.8h, v12.8h\n" + "smax v27.8h, v27.8h, v12.8h\n" + "smax v8.8h, v8.8h, v12.8h\n" + "smin v13.8h, v13.8h, v26.8h\n" + "smin v7.8h, v7.8h, v26.8h\n" + "smin v27.8h, v27.8h, v26.8h\n" + "smin v8.8h, v8.8h, v26.8h\n" + "uzp1 v13.16b, v13.16b, v13.16b\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" + "uzp1 v27.16b, v27.16b, v27.16b\n" "uzp1 v8.16b, v8.16b, v8.16b\n" - "uzp1 v6.16b, v6.16b, v6.16b\n" - "tbz x4, #2, 121f\n" - "st1 { v11.s }[0], [x7], #0x4\n" - "st1 { v20.s }[0], [x8], #0x4\n" - "st1 { v8.s }[0], [x17], #0x4\n" - "st1 { v6.s }[0], [x16], #0x4\n" - "tbz x4, #1, 120f\n" - "st1 { v11.h }[2], [x7], #0x2\n" - "st1 { v20.h }[2], [x8], #0x2\n" - "st1 { v8.h }[2], [x17], #0x2\n" - "st1 { v6.h }[2], [x16], #0x2\n" - "tbz x4, #0, 123f\n" - "st1 { v11.b }[6], [x7], #0x1\n" - "st1 { v20.b }[6], [x8], #0x1\n" - "st1 { v8.b }[6], [x17], #0x1\n" - "st1 { v6.b }[6], [x16], #0x1\n" + "tbz x2, #2, 121f\n" + "st1 { v13.s }[0], [x16], #0x4\n" + "st1 { v7.s }[0], [x15], #0x4\n" + "st1 { v27.s }[0], [x14], #0x4\n" + "st1 { v8.s }[0], [x13], #0x4\n" + "tbz x2, #1, 120f\n" + "st1 { v13.h }[2], [x16], #0x2\n" + "st1 { v7.h }[2], [x15], #0x2\n" + "st1 { v27.h }[2], [x14], #0x2\n" + "st1 { v8.h }[2], [x13], #0x2\n" + "tbz x2, #0, 123f\n" + "st1 { v13.b }[6], [x16], #0x1\n" + "st1 { v7.b }[6], [x15], #0x1\n" + "st1 { v27.b }[6], [x14], #0x1\n" + "st1 { v8.b }[6], [x13], #0x1\n" "b 123f\n" "120:" // Oddments: Bit 2: Bit 1: Unset - "tbz x4, #0, 123f\n" - "st1 { v11.b }[4], [x7], #0x1\n" - "st1 { v20.b }[4], [x8], #0x1\n" - "st1 { v8.b }[4], [x17], #0x1\n" - "st1 { v6.b }[4], [x16], #0x1\n" + "tbz x2, #0, 123f\n" + "st1 { v13.b }[4], [x16], #0x1\n" + "st1 { v7.b }[4], [x15], #0x1\n" + "st1 { v27.b }[4], [x14], #0x1\n" + "st1 { v8.b }[4], [x13], #0x1\n" "b 123f\n" "121:" // Oddments: Bit 2: Unset - "tbz x4, #1, 122f\n" - "st1 { v11.h }[0], [x7], #0x2\n" - "st1 { v20.h }[0], [x8], #0x2\n" - "st1 { v8.h }[0], [x17], #0x2\n" - "st1 { v6.h }[0], [x16], #0x2\n" - "tbz x4, #0, 123f\n" - "st1 { v11.b }[2], [x7], #0x1\n" - "st1 { v20.b }[2], [x8], #0x1\n" - "st1 { v8.b }[2], [x17], #0x1\n" - "st1 { v6.b }[2], [x16], #0x1\n" + "tbz x2, #1, 122f\n" + "st1 { v13.h }[0], [x16], #0x2\n" + "st1 { v7.h }[0], [x15], #0x2\n" + "st1 { v27.h }[0], [x14], #0x2\n" + "st1 { v8.h }[0], [x13], #0x2\n" + "tbz x2, #0, 123f\n" + "st1 { v13.b }[2], [x16], #0x1\n" + "st1 { v7.b }[2], [x15], #0x1\n" + "st1 { v27.b }[2], [x14], #0x1\n" + "st1 { v8.b }[2], [x13], #0x1\n" "b 123f\n" "122:" // Oddments: Bit 2: Unset: Bit 1: Unset - "tbz x4, #0, 123f\n" - "st1 { v11.b }[0], [x7], #0x1\n" - "st1 { v20.b }[0], [x8], #0x1\n" - "st1 { v8.b }[0], [x17], #0x1\n" - "st1 { v6.b }[0], [x16], #0x1\n" + "tbz x2, #0, 123f\n" + "st1 { v13.b }[0], [x16], #0x1\n" + "st1 { v7.b }[0], [x15], #0x1\n" + "st1 { v27.b }[0], [x14], #0x1\n" + "st1 { v8.b }[0], [x13], #0x1\n" "123:" // Oddments: Bit 2: End "124:" // End : : [offsetof_Params_bias] "I" (offsetof(Params, bias)), [offsetof_Params_inptrs] "I" (offsetof(Params, inptrs)), [offsetof_Params_n_channels] "I" (offsetof(Params, n_channels)), [offsetof_Params_outptrs] "I" (offsetof(Params, outptrs)), [offsetof_Params_requant] "I" (offsetof(Params, requant)), [offsetof_Params_requant_muls] "I" (offsetof(Params, requant_muls)), [offsetof_Params_requant_shifts] "I" (offsetof(Params, requant_shifts)), [offsetof_Params_weights] "I" (offsetof(Params, weights)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [params] "r" (¶ms) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp index cf655cbe780e45a38d1a5dcd669df6345555fdda..7b0b414517613711adb958fde7b455b62c5f8401 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include @@ -34,16 +34,7 @@ namespace arm_conv { namespace depthwise { -void a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( - const unsigned int, - const uint8_t *const *const, - const int8_t *const, - const int32_t *const, - const arm_gemm::Requantize32 &, - const int32_t *const, - const int32_t *const, - uint8_t *const *const -); +void a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl(unsigned int, const uint8_t *const *, const int8_t *, const int32_t *, const arm_gemm::Requantize32 &, const int32_t *, const int32_t *, uint8_t *const *); class a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirstStrategy { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp index 4419048793705b5e73b46aa8450412557c14ba29..89253ba6704c61a841b3a3ff270193cacd7c523e 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp @@ -91,1072 +91,1072 @@ void a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( requant_muls, requant_shifts, outptrs); __asm__ __volatile__( - "ldr x6, [%x[params], %[offsetof_Params_n_channels]]\n" + "ldr x7, [%x[params], %[offsetof_Params_n_channels]]\n" "ldr x23, [%x[params], %[offsetof_Params_requant]]\n" - "lsr x7, x6, #0x3\n" + "lsr x8, x7, #0x3\n" "add x20, x23, %[offsetof_Requantize32_a_offset]\n" - "ld1r { v24.16b }, [x20]\n" + "ld1r { v14.16b }, [x20]\n" "ldr x22, [%x[params], %[offsetof_Params_outptrs]]\n" "add x21, x23, %[offsetof_Requantize32_b_offset]\n" "add x20, x23, %[offsetof_Requantize32_c_offset]\n" - "ld1r { v15.16b }, [x21]\n" - "ld1r { v14.8h }, [x20]\n" + "ld1r { v19.16b }, [x21]\n" + "ld1r { v13.8h }, [x20]\n" "add x21, x23, %[offsetof_Requantize32_minval]\n" "add x20, x23, %[offsetof_Requantize32_maxval]\n" - "ld1r { v12.8h }, [x21]\n" - "ld1r { v11.8h }, [x20]\n" - "mov x8, #0x0\n" + "ld1r { v29.8h }, [x21]\n" + "ld1r { v12.8h }, [x20]\n" "mov x17, #0x0\n" - "add x16, %x[params], %[offsetof_Params_inptrs]\n" - "ldr x15, [%x[params], %[offsetof_Params_weights]]\n" - "ldr x14, [%x[params], %[offsetof_Params_requant_muls]]\n" - "ldr x13, [%x[params], %[offsetof_Params_requant_shifts]]\n" - "ldp x12, x11, [x22, #0x0]\n" - "ldp x10, x9, [x22, #0x10]\n" - "cbz x7, 3f\n" - "ldr d0, [x15, #0x0]\n" - "ldr d1, [x15, #0x8]\n" - "subs x7, x7, #0x1\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "ldr d2, [x15, #0x10]\n" - "ldr d3, [x15, #0x18]\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "ldr d4, [x15, #0x20]\n" - "ldr d5, [x15, #0x28]\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "ldr d6, [x15, #0x30]\n" - "ldr d7, [x15, #0x38]\n" - "ssubl v5.8h, v5.8b, v15.8b\n" - "ssubl v6.8h, v6.8b, v15.8b\n" - "ldr d8, [x15, #0x40]\n" - "ldr x28, [%x[params], %[offsetof_Params_bias]]\n" - "ssubl v7.8h, v7.8b, v15.8b\n" - "ssubl v8.8h, v8.8b, v15.8b\n" - "ldr q13, [x28, #0x0]\n" - "ldr q20, [x28, #0x10]\n" - "add x28, x28, #0x20\n" - "str x28, [%x[params], %[offsetof_Params_bias]]\n" - "ldp x24, x23, [x16, #0x0]\n" - "ldp x22, x21, [x16, #0x10]\n" - "mov v9.16b, v13.16b\n" - "mov v18.16b, v20.16b\n" - "ldr d31, [x24, x8]\n" - "ldr d30, [x23, x8]\n" - "mov v16.16b, v13.16b\n" - "mov v26.16b, v20.16b\n" - "ldr d29, [x22, x8]\n" - "ldr d28, [x21, x8]\n" - "mov v25.16b, v13.16b\n" - "mov v10.16b, v20.16b\n" - "ldr x20, [x16, #0x20]\n" - "ldr d27, [x20, x8]\n" - "usubl v31.8h, v31.8b, v24.8b\n" - "usubl v30.8h, v30.8b, v24.8b\n" - "usubl v29.8h, v29.8b, v24.8b\n" - "usubl v28.8h, v28.8b, v24.8b\n" - "usubl v27.8h, v27.8b, v24.8b\n" + "mov x16, #0x0\n" + "add x15, %x[params], %[offsetof_Params_inptrs]\n" + "ldr x14, [%x[params], %[offsetof_Params_weights]]\n" + "ldr x13, [%x[params], %[offsetof_Params_requant_muls]]\n" + "ldr x12, [%x[params], %[offsetof_Params_requant_shifts]]\n" + "ldp x11, x10, [x22, #0x0]\n" + "ldp x9, x28, [x22, #0x10]\n" + "cbz x8, 3f\n" + "ldr d23, [x14, #0x0]\n" + "ldr d16, [x14, #0x8]\n" + "subs x8, x8, #0x1\n" + "ssubl v23.8h, v23.8b, v19.8b\n" + "ldr d1, [x14, #0x10]\n" + "ldr d5, [x14, #0x18]\n" + "ssubl v16.8h, v16.8b, v19.8b\n" + "ssubl v1.8h, v1.8b, v19.8b\n" + "ldr d26, [x14, #0x20]\n" + "ldr d18, [x14, #0x28]\n" + "ssubl v5.8h, v5.8b, v19.8b\n" + "ssubl v26.8h, v26.8b, v19.8b\n" + "ldr d31, [x14, #0x30]\n" + "ldr d25, [x14, #0x38]\n" + "ssubl v18.8h, v18.8b, v19.8b\n" + "ssubl v31.8h, v31.8b, v19.8b\n" + "ldr d20, [x14, #0x40]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "ssubl v25.8h, v25.8b, v19.8b\n" + "ssubl v20.8h, v20.8b, v19.8b\n" + "ldr q9, [x20, #0x0]\n" + "ldr q24, [x20, #0x10]\n" + "add x20, x20, #0x20\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" + "ldp x23, x22, [x15, #0x0]\n" + "ldp x21, x20, [x15, #0x10]\n" + "mov v7.16b, v9.16b\n" + "mov v0.16b, v24.16b\n" + "ldr d22, [x23, x17]\n" + "ldr d4, [x22, x17]\n" + "mov v2.16b, v9.16b\n" + "mov v30.16b, v24.16b\n" + "ldr d8, [x21, x17]\n" + "ldr d27, [x20, x17]\n" + "mov v10.16b, v9.16b\n" + "mov v6.16b, v24.16b\n" + "ldr x20, [x15, #0x20]\n" + "ldr d15, [x20, x17]\n" + "usubl v22.8h, v22.8b, v14.8b\n" + "usubl v4.8h, v4.8b, v14.8b\n" + "usubl v8.8h, v8.8b, v14.8b\n" + "usubl v27.8h, v27.8b, v14.8b\n" + "usubl v15.8h, v15.8b, v14.8b\n" "beq 2f\n" "1:" // Loop - "ldr q17, [x14, #0x0]\n" - "ldr q22, [x13, #0x0]\n" - "smlal v13.4s, v31.4h, v4.4h\n" - "smlal2 v20.4s, v31.8h, v4.8h\n" - "ldr q23, [x14, #0x10]\n" - "smlal v9.4s, v31.4h, v3.4h\n" - "smlal2 v18.4s, v31.8h, v3.8h\n" - "ldr x21, [x16, #0x28]\n" - "smlal v13.4s, v30.4h, v0.4h\n" - "smlal2 v20.4s, v30.8h, v0.8h\n" - "ldr q19, [x13, #0x10]\n" - "ldr x28, [x16, #0x38]\n" - "smlal v9.4s, v29.4h, v2.4h\n" - "smlal2 v18.4s, v29.8h, v2.8h\n" - "ldr x20, [x16, #0x30]\n" - "ldr d29, [x20, x8]\n" - "smlal v16.4s, v31.4h, v1.4h\n" - "smlal2 v26.4s, v31.8h, v1.8h\n" - "ldr x27, [x16, #0x40]\n" - "ldr x26, [x16, #0x48]\n" - "smlal v25.4s, v31.4h, v0.4h\n" - "smlal2 v10.4s, v31.8h, v0.8h\n" - "ldr d31, [x21, x8]\n" - "usubl v31.8h, v31.8b, v24.8b\n" - "smlal v13.4s, v28.4h, v5.4h\n" - "smlal2 v20.4s, v28.8h, v5.8h\n" - "usubl v29.8h, v29.8b, v24.8b\n" - "ldr x25, [x16, #0x50]\n" - "smlal v9.4s, v28.4h, v4.4h\n" - "smlal2 v18.4s, v28.8h, v4.8h\n" - "ldr x24, [x16, #0x58]\n" - "ldr x23, [x16, #0x60]\n" - "smlal v16.4s, v28.4h, v2.4h\n" - "smlal2 v26.4s, v28.8h, v2.8h\n" - "ldr x22, [x16, #0x68]\n" - "ldr x21, [x16, #0x70]\n" - "smlal v25.4s, v28.4h, v1.4h\n" - "smlal2 v10.4s, v28.8h, v1.8h\n" - "ldr d28, [x28, x8]\n" - "usubl v28.8h, v28.8b, v24.8b\n" - "smlal v13.4s, v27.4h, v7.4h\n" - "smlal2 v20.4s, v27.8h, v7.8h\n" - "ldr x20, [x16, #0x78]\n" - "ldr x28, [%x[params], %[offsetof_Params_bias]]\n" - "smlal v9.4s, v27.4h, v6.4h\n" - "smlal2 v18.4s, v27.8h, v6.8h\n" - "add x15, x15, #0x48\n" - "subs x7, x7, #0x1\n" - "smlal v16.4s, v31.4h, v6.4h\n" - "smlal2 v26.4s, v31.8h, v6.8h\n" - "ldr d31, [x27, x8]\n" - "usubl v31.8h, v31.8b, v24.8b\n" - "smlal v25.4s, v27.4h, v3.4h\n" - "smlal2 v10.4s, v27.8h, v3.8h\n" - "add x14, x14, #0x20\n" + "ldr q3, [x13, #0x0]\n" + "ldr q17, [x12, #0x0]\n" + "smlal v9.4s, v22.4h, v26.4h\n" + "smlal2 v24.4s, v22.8h, v26.8h\n" + "ldr q21, [x13, #0x10]\n" + "ldr q28, [x12, #0x10]\n" + "smlal v9.4s, v4.4h, v23.4h\n" + "smlal v7.4s, v22.4h, v5.4h\n" + "ldr x20, [x15, #0x28]\n" + "ldr d11, [x20, x17]\n" + "smlal v2.4s, v22.4h, v16.4h\n" + "smlal v10.4s, v22.4h, v23.4h\n" + "smlal2 v24.4s, v4.8h, v23.8h\n" + "ldr x20, [x15, #0x38]\n" + "ldr d4, [x20, x17]\n" + "smlal v9.4s, v27.4h, v18.4h\n" + "smlal2 v0.4s, v22.8h, v5.8h\n" + "smlal2 v30.4s, v22.8h, v16.8h\n" + "ldr x20, [x15, #0x30]\n" + "usubl v11.8h, v11.8b, v14.8b\n" + "smlal2 v6.4s, v22.8h, v23.8h\n" + "ldr d22, [x20, x17]\n" + "smlal v7.4s, v8.4h, v1.4h\n" + "ldr x20, [x15, #0x40]\n" + "smlal v2.4s, v27.4h, v1.4h\n" + "smlal v10.4s, v27.4h, v16.4h\n" + "usubl v4.8h, v4.8b, v14.8b\n" + "ldr x27, [x15, #0x48]\n" + "smlal2 v24.4s, v27.8h, v18.8h\n" + "smlal v9.4s, v15.4h, v25.4h\n" + "usubl v22.8h, v22.8b, v14.8b\n" + "ldr x26, [x15, #0x50]\n" + "smlal2 v0.4s, v8.8h, v1.8h\n" + "ldr d8, [x20, x17]\n" + "smlal2 v30.4s, v27.8h, v1.8h\n" + "usubl v8.8h, v8.8b, v14.8b\n" + "smlal2 v6.4s, v27.8h, v16.8h\n" + "smlal v7.4s, v27.4h, v26.4h\n" + "ldr x25, [x15, #0x58]\n" + "ldr x24, [x15, #0x60]\n" + "smlal v2.4s, v11.4h, v31.4h\n" + "smlal v10.4s, v15.4h, v5.4h\n" + "ldr x23, [x15, #0x68]\n" + "ldr x22, [x15, #0x70]\n" + "smlal2 v24.4s, v15.8h, v25.8h\n" + "smlal v9.4s, v4.4h, v16.4h\n" + "ldr x21, [x15, #0x78]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "smlal2 v0.4s, v27.8h, v26.8h\n" + "ldr d27, [x27, x17]\n" + "smlal2 v30.4s, v11.8h, v31.8h\n" + "ldr d11, [x26, x17]\n" + "smlal2 v6.4s, v15.8h, v5.8h\n" + "smlal v7.4s, v15.4h, v31.4h\n" + "usubl v27.8h, v27.8b, v14.8b\n" + "add x14, x14, #0x48\n" + "smlal v2.4s, v15.4h, v26.4h\n" + "smlal v10.4s, v22.4h, v20.4h\n" + "usubl v11.8h, v11.8b, v14.8b\n" + "subs x8, x8, #0x1\n" + "smlal2 v24.4s, v4.8h, v16.8h\n" + "smlal v9.4s, v8.4h, v1.4h\n" "add x13, x13, #0x20\n" - "smlal v13.4s, v28.4h, v1.4h\n" - "smlal2 v20.4s, v28.8h, v1.8h\n" - "smlal v9.4s, v28.4h, v0.4h\n" - "smlal2 v18.4s, v28.8h, v0.8h\n" - "ldr d30, [x26, x8]\n" - "usubl v30.8h, v30.8b, v24.8b\n" - "smlal v16.4s, v27.4h, v4.4h\n" - "smlal v25.4s, v29.4h, v8.4h\n" - "smlal2 v26.4s, v27.8h, v4.8h\n" - "ldr d28, [x24, x8]\n" - "smlal2 v10.4s, v29.8h, v8.8h\n" - "ldr d29, [x25, x8]\n" - "smlal v13.4s, v31.4h, v2.4h\n" - "smlal2 v20.4s, v31.8h, v2.8h\n" - "usubl v29.8h, v29.8b, v24.8b\n" - "smlal v9.4s, v31.4h, v1.4h\n" - "smlal2 v18.4s, v31.8h, v1.8h\n" - "ldr d31, [x23, x8]\n" - "usubl v28.8h, v28.8b, v24.8b\n" - "smlal v16.4s, v30.4h, v5.4h\n" - "smlal v25.4s, v30.4h, v4.4h\n" - "usubl v31.8h, v31.8b, v24.8b\n" - "smlal v13.4s, v30.4h, v8.4h\n" - "smlal2 v20.4s, v30.8h, v8.8h\n" - "smlal v9.4s, v30.4h, v7.4h\n" - "smlal2 v18.4s, v30.8h, v7.8h\n" - "smlal2 v26.4s, v30.8h, v5.8h\n" - "smlal2 v10.4s, v30.8h, v4.8h\n" - "ldr d30, [x22, x8]\n" - "usubl v30.8h, v30.8b, v24.8b\n" - "smlal v16.4s, v29.4h, v0.4h\n" - "smlal v25.4s, v28.4h, v2.4h\n" - "smlal v13.4s, v29.4h, v3.4h\n" - "smlal2 v20.4s, v29.8h, v3.8h\n" - "smlal2 v26.4s, v29.8h, v0.8h\n" - "ldr d29, [x21, x8]\n" - "smlal2 v10.4s, v28.8h, v2.8h\n" - "usubl v29.8h, v29.8b, v24.8b\n" - "smlal v16.4s, v31.4h, v3.4h\n" - "smlal v25.4s, v30.4h, v5.4h\n" - "smlal v9.4s, v28.4h, v5.4h\n" - "smlal2 v18.4s, v28.8h, v5.8h\n" - "ldr d28, [x20, x8]\n" - "usubl v28.8h, v28.8b, v24.8b\n" - "smlal v13.4s, v31.4h, v6.4h\n" - "smlal2 v26.4s, v31.8h, v3.8h\n" - "sqrdmulh v13.4s, v13.4s, v17.4s\n" - "add x8, x8, #0x8\n" - "smlal2 v10.4s, v30.8h, v5.8h\n" - "smlal v16.4s, v29.4h, v7.4h\n" - "and v21.16b, v13.16b, v22.16b\n" - "smlal v25.4s, v29.4h, v6.4h\n" - "smlal2 v20.4s, v31.8h, v6.8h\n" - "sqrdmulh v20.4s, v20.4s, v23.4s\n" - "smlal2 v26.4s, v29.8h, v7.8h\n" - "smlal2 v10.4s, v29.8h, v6.8h\n" - "sshr v21.4s, v21.4s, #0x1f\n" - "smlal v9.4s, v30.4h, v8.4h\n" - "smlal v16.4s, v28.4h, v8.4h\n" - "and v29.16b, v20.16b, v19.16b\n" - "smlal v25.4s, v28.4h, v7.4h\n" - "smlal2 v18.4s, v30.8h, v8.8h\n" - "sqrdmulh v9.4s, v9.4s, v17.4s\n" - "smlal2 v26.4s, v28.8h, v8.8h\n" - "smlal2 v10.4s, v28.8h, v7.8h\n" - "sqrdmulh v16.4s, v16.4s, v17.4s\n" - "sqrdmulh v25.4s, v25.4s, v17.4s\n" - "sqadd v13.4s, v13.4s, v21.4s\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "and v0.16b, v9.16b, v22.16b\n" - "sqrdmulh v18.4s, v18.4s, v23.4s\n" - "and v27.16b, v16.16b, v22.16b\n" - "sqrdmulh v26.4s, v26.4s, v23.4s\n" - "and v21.16b, v25.16b, v22.16b\n" - "sqrdmulh v10.4s, v10.4s, v23.4s\n" - "sqadd v20.4s, v20.4s, v29.4s\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v17.16b, v18.16b, v19.16b\n" + "add x12, x12, #0x20\n" + "smlal2 v0.4s, v15.8h, v31.8h\n" + "smlal2 v30.4s, v15.8h, v26.8h\n" + "ldr d15, [x25, x17]\n" + "usubl v15.8h, v15.8b, v14.8b\n" + "smlal2 v6.4s, v22.8h, v20.8h\n" + "ldr d22, [x24, x17]\n" + "smlal v7.4s, v4.4h, v23.4h\n" + "usubl v22.8h, v22.8b, v14.8b\n" + "smlal v2.4s, v27.4h, v18.4h\n" + "smlal v10.4s, v27.4h, v26.4h\n" + "smlal2 v24.4s, v8.8h, v1.8h\n" + "smlal v9.4s, v27.4h, v20.4h\n" + "smlal2 v0.4s, v4.8h, v23.8h\n" + "ldr d4, [x23, x17]\n" + "smlal2 v30.4s, v27.8h, v18.8h\n" + "usubl v4.8h, v4.8b, v14.8b\n" + "smlal2 v6.4s, v27.8h, v26.8h\n" + "ldr d26, [x22, x17]\n" + "smlal v7.4s, v8.4h, v16.4h\n" + "usubl v26.8h, v26.8b, v14.8b\n" + "smlal v2.4s, v11.4h, v23.4h\n" + "smlal v10.4s, v15.4h, v1.4h\n" + "smlal2 v24.4s, v27.8h, v20.8h\n" + "smlal v9.4s, v11.4h, v5.4h\n" + "smlal2 v0.4s, v8.8h, v16.8h\n" + "ldr d8, [x21, x17]\n" + "smlal2 v30.4s, v11.8h, v23.8h\n" + "usubl v8.8h, v8.8b, v14.8b\n" + "smlal2 v6.4s, v15.8h, v1.8h\n" + "smlal v7.4s, v27.4h, v25.4h\n" + "add x17, x17, #0x8\n" + "smlal v2.4s, v22.4h, v5.4h\n" + "smlal v10.4s, v4.4h, v18.4h\n" + "smlal2 v24.4s, v11.8h, v5.8h\n" + "smlal v9.4s, v22.4h, v31.4h\n" + "sqrdmulh v9.4s, v9.4s, v3.4s\n" + "smlal2 v0.4s, v27.8h, v25.8h\n" + "smlal2 v30.4s, v22.8h, v5.8h\n" + "and v27.16b, v9.16b, v17.16b\n" + "smlal2 v6.4s, v4.8h, v18.8h\n" + "smlal v7.4s, v15.4h, v18.4h\n" "sshr v27.4s, v27.4s, #0x1f\n" - "and v7.16b, v26.16b, v19.16b\n" - "sshr v21.4s, v21.4s, #0x1f\n" - "and v29.16b, v10.16b, v19.16b\n" - "sqadd v9.4s, v9.4s, v0.4s\n" - "sshr v17.4s, v17.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v27.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v25.4s, v25.4s, v21.4s\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "srshl v13.4s, v13.4s, v22.4s\n" - "srshl v9.4s, v9.4s, v22.4s\n" - "sqadd v18.4s, v18.4s, v17.4s\n" - "srshl v16.4s, v16.4s, v22.4s\n" - "sqadd v26.4s, v26.4s, v7.4s\n" - "srshl v25.4s, v25.4s, v22.4s\n" - "sqadd v10.4s, v10.4s, v29.4s\n" - "srshl v20.4s, v20.4s, v19.4s\n" - "sqxtn v13.4h, v13.4s\n" - "srshl v18.4s, v18.4s, v19.4s\n" + "smlal v2.4s, v26.4h, v25.4h\n" + "smlal v10.4s, v26.4h, v31.4h\n" + "sqadd v9.4s, v9.4s, v27.4s\n" + "smlal2 v24.4s, v22.8h, v31.8h\n" + "smlal2 v0.4s, v15.8h, v18.8h\n" + "sqrdmulh v24.4s, v24.4s, v21.4s\n" + "smlal2 v30.4s, v26.8h, v25.8h\n" + "smlal2 v6.4s, v26.8h, v31.8h\n" + "and v31.16b, v24.16b, v28.16b\n" + "smlal v7.4s, v4.4h, v20.4h\n" + "smlal v2.4s, v8.4h, v20.4h\n" + "sqrdmulh v7.4s, v7.4s, v3.4s\n" + "smlal v10.4s, v8.4h, v25.4h\n" + "smlal2 v0.4s, v4.8h, v20.8h\n" + "sqrdmulh v2.4s, v2.4s, v3.4s\n" + "smlal2 v30.4s, v8.8h, v20.8h\n" + "smlal2 v6.4s, v8.8h, v25.8h\n" + "sqrdmulh v10.4s, v10.4s, v3.4s\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "and v22.16b, v7.16b, v17.16b\n" + "sqrdmulh v0.4s, v0.4s, v21.4s\n" + "and v3.16b, v2.16b, v17.16b\n" + "sqrdmulh v30.4s, v30.4s, v21.4s\n" + "and v11.16b, v10.16b, v17.16b\n" + "sqrdmulh v6.4s, v6.4s, v21.4s\n" + "sqadd v24.4s, v24.4s, v31.4s\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "and v20.16b, v0.16b, v28.16b\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "and v31.16b, v30.16b, v28.16b\n" + "sshr v11.4s, v11.4s, #0x1f\n" + "and v18.16b, v6.16b, v28.16b\n" + "sqadd v7.4s, v7.4s, v22.4s\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "sqadd v2.4s, v2.4s, v3.4s\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "sqadd v10.4s, v10.4s, v11.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "srshl v9.4s, v9.4s, v17.4s\n" + "srshl v7.4s, v7.4s, v17.4s\n" + "sqadd v0.4s, v0.4s, v20.4s\n" + "srshl v2.4s, v2.4s, v17.4s\n" + "sqadd v30.4s, v30.4s, v31.4s\n" + "srshl v10.4s, v10.4s, v17.4s\n" + "sqadd v6.4s, v6.4s, v18.4s\n" + "srshl v24.4s, v24.4s, v28.4s\n" "sqxtn v9.4h, v9.4s\n" - "srshl v26.4s, v26.4s, v19.4s\n" - "sqxtn v16.4h, v16.4s\n" - "srshl v10.4s, v10.4s, v19.4s\n" - "sqxtn v25.4h, v25.4s\n" - "sqxtn2 v13.8h, v20.4s\n" - "sqxtn2 v9.8h, v18.4s\n" - "sqxtn2 v16.8h, v26.4s\n" - "sqxtn2 v25.8h, v10.4s\n" - "sqadd v13.8h, v13.8h, v14.8h\n" - "sqadd v9.8h, v9.8h, v14.8h\n" - "sqadd v16.8h, v16.8h, v14.8h\n" - "sqadd v25.8h, v25.8h, v14.8h\n" - "smax v13.8h, v13.8h, v12.8h\n" - "smax v9.8h, v9.8h, v12.8h\n" - "smax v16.8h, v16.8h, v12.8h\n" - "smax v25.8h, v25.8h, v12.8h\n" - "smin v13.8h, v13.8h, v11.8h\n" - "smin v9.8h, v9.8h, v11.8h\n" - "smin v16.8h, v16.8h, v11.8h\n" - "smin v25.8h, v25.8h, v11.8h\n" - "uzp1 v13.16b, v13.16b, v13.16b\n" - "str d13, [x12, x17]\n" + "srshl v0.4s, v0.4s, v28.4s\n" + "sqxtn v7.4h, v7.4s\n" + "srshl v30.4s, v30.4s, v28.4s\n" + "sqxtn v2.4h, v2.4s\n" + "srshl v6.4s, v6.4s, v28.4s\n" + "sqxtn v10.4h, v10.4s\n" + "sqxtn2 v9.8h, v24.4s\n" + "sqxtn2 v7.8h, v0.4s\n" + "sqxtn2 v2.8h, v30.4s\n" + "sqxtn2 v10.8h, v6.4s\n" + "sqadd v9.8h, v9.8h, v13.8h\n" + "sqadd v7.8h, v7.8h, v13.8h\n" + "sqadd v2.8h, v2.8h, v13.8h\n" + "sqadd v10.8h, v10.8h, v13.8h\n" + "smax v9.8h, v9.8h, v29.8h\n" + "smax v7.8h, v7.8h, v29.8h\n" + "smax v2.8h, v2.8h, v29.8h\n" + "smax v10.8h, v10.8h, v29.8h\n" + "smin v9.8h, v9.8h, v12.8h\n" + "smin v7.8h, v7.8h, v12.8h\n" + "smin v2.8h, v2.8h, v12.8h\n" + "smin v10.8h, v10.8h, v12.8h\n" "uzp1 v9.16b, v9.16b, v9.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "str d9, [x11, x17]\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "str d16, [x10, x17]\n" - "str d25, [x9, x17]\n" - "ldr q13, [x28, #0x0]\n" - "ldr q20, [x28, #0x10]\n" - "add x28, x28, #0x20\n" - "ldr d0, [x15, #0x0]\n" - "ldr d1, [x15, #0x8]\n" - "add x17, x17, #0x8\n" - "str x28, [%x[params], %[offsetof_Params_bias]]\n" - "ldr d2, [x15, #0x10]\n" - "ldr d3, [x15, #0x18]\n" - "mov v9.16b, v13.16b\n" - "mov v18.16b, v20.16b\n" - "ldr d4, [x15, #0x20]\n" - "ldr d5, [x15, #0x28]\n" - "mov v16.16b, v13.16b\n" - "mov v26.16b, v20.16b\n" - "ldr d6, [x15, #0x30]\n" - "ldr d7, [x15, #0x38]\n" - "mov v25.16b, v13.16b\n" - "mov v10.16b, v20.16b\n" - "ldr d8, [x15, #0x40]\n" - "ldp x24, x23, [x16, #0x0]\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "ldp x22, x21, [x16, #0x10]\n" - "ldr d31, [x24, x8]\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "ldr d30, [x23, x8]\n" - "ldr d29, [x22, x8]\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "ssubl v5.8h, v5.8b, v15.8b\n" - "ldr d28, [x21, x8]\n" - "ldr x20, [x16, #0x20]\n" - "ssubl v6.8h, v6.8b, v15.8b\n" - "ssubl v7.8h, v7.8b, v15.8b\n" - "ldr d27, [x20, x8]\n" - "ssubl v8.8h, v8.8b, v15.8b\n" - "usubl v31.8h, v31.8b, v24.8b\n" - "usubl v30.8h, v30.8b, v24.8b\n" - "usubl v29.8h, v29.8b, v24.8b\n" - "usubl v28.8h, v28.8b, v24.8b\n" - "usubl v27.8h, v27.8b, v24.8b\n" + "str d9, [x11, x16]\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" + "uzp1 v2.16b, v2.16b, v2.16b\n" + "str d7, [x10, x16]\n" + "uzp1 v10.16b, v10.16b, v10.16b\n" + "str d2, [x9, x16]\n" + "str d10, [x28, x16]\n" + "ldr q9, [x20, #0x0]\n" + "ldr q24, [x20, #0x10]\n" + "add x20, x20, #0x20\n" + "ldr d23, [x14, #0x0]\n" + "ldr d16, [x14, #0x8]\n" + "add x16, x16, #0x8\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" + "ldr d1, [x14, #0x10]\n" + "ldr d5, [x14, #0x18]\n" + "mov v7.16b, v9.16b\n" + "mov v0.16b, v24.16b\n" + "ldr d26, [x14, #0x20]\n" + "ldr d18, [x14, #0x28]\n" + "mov v2.16b, v9.16b\n" + "mov v30.16b, v24.16b\n" + "ldr d31, [x14, #0x30]\n" + "ldr d25, [x14, #0x38]\n" + "mov v10.16b, v9.16b\n" + "mov v6.16b, v24.16b\n" + "ldr d20, [x14, #0x40]\n" + "ldp x23, x22, [x15, #0x0]\n" + "ssubl v23.8h, v23.8b, v19.8b\n" + "ssubl v16.8h, v16.8b, v19.8b\n" + "ldp x21, x20, [x15, #0x10]\n" + "ldr d22, [x23, x17]\n" + "ssubl v1.8h, v1.8b, v19.8b\n" + "ssubl v5.8h, v5.8b, v19.8b\n" + "ldr d4, [x22, x17]\n" + "ldr d8, [x21, x17]\n" + "ssubl v26.8h, v26.8b, v19.8b\n" + "ssubl v18.8h, v18.8b, v19.8b\n" + "ldr d27, [x20, x17]\n" + "ldr x20, [x15, #0x20]\n" + "ssubl v31.8h, v31.8b, v19.8b\n" + "ssubl v25.8h, v25.8b, v19.8b\n" + "ldr d15, [x20, x17]\n" + "ssubl v20.8h, v20.8b, v19.8b\n" + "usubl v22.8h, v22.8b, v14.8b\n" + "usubl v4.8h, v4.8b, v14.8b\n" + "usubl v8.8h, v8.8b, v14.8b\n" + "usubl v27.8h, v27.8b, v14.8b\n" + "usubl v15.8h, v15.8b, v14.8b\n" "bgt 1b\n" "2:" // Tail - "ldr q17, [x14, #0x0]\n" - "ldr q22, [x13, #0x0]\n" - "smlal v13.4s, v31.4h, v4.4h\n" - "smlal2 v20.4s, v31.8h, v4.8h\n" - "ldr q23, [x14, #0x10]\n" - "smlal v9.4s, v31.4h, v3.4h\n" - "smlal2 v18.4s, v31.8h, v3.8h\n" - "ldr x21, [x16, #0x28]\n" - "smlal v13.4s, v30.4h, v0.4h\n" - "smlal2 v20.4s, v30.8h, v0.8h\n" - "ldr q19, [x13, #0x10]\n" - "ldr x28, [x16, #0x38]\n" - "smlal v9.4s, v29.4h, v2.4h\n" - "smlal2 v18.4s, v29.8h, v2.8h\n" - "ldr x20, [x16, #0x30]\n" - "ldr d29, [x20, x8]\n" - "smlal v16.4s, v31.4h, v1.4h\n" - "smlal2 v26.4s, v31.8h, v1.8h\n" - "ldr x27, [x16, #0x40]\n" - "ldr x26, [x16, #0x48]\n" - "smlal v25.4s, v31.4h, v0.4h\n" - "smlal2 v10.4s, v31.8h, v0.8h\n" - "ldr d31, [x21, x8]\n" - "usubl v31.8h, v31.8b, v24.8b\n" - "smlal v13.4s, v28.4h, v5.4h\n" - "smlal2 v20.4s, v28.8h, v5.8h\n" - "usubl v29.8h, v29.8b, v24.8b\n" - "ldr x25, [x16, #0x50]\n" - "smlal v9.4s, v28.4h, v4.4h\n" - "smlal2 v18.4s, v28.8h, v4.8h\n" - "ldr x24, [x16, #0x58]\n" - "ldr x23, [x16, #0x60]\n" - "smlal v16.4s, v28.4h, v2.4h\n" - "smlal2 v26.4s, v28.8h, v2.8h\n" - "ldr x22, [x16, #0x68]\n" - "ldr x21, [x16, #0x70]\n" - "smlal v25.4s, v28.4h, v1.4h\n" - "smlal2 v10.4s, v28.8h, v1.8h\n" - "ldr d28, [x28, x8]\n" - "usubl v28.8h, v28.8b, v24.8b\n" - "smlal v13.4s, v27.4h, v7.4h\n" - "smlal2 v20.4s, v27.8h, v7.8h\n" - "ldr x20, [x16, #0x78]\n" - "tst x6, #0x7\n" - "smlal v9.4s, v27.4h, v6.4h\n" - "smlal2 v18.4s, v27.8h, v6.8h\n" - "add x14, x14, #0x20\n" + "ldr q28, [x13, #0x0]\n" + "ldr q17, [x12, #0x0]\n" + "smlal v9.4s, v22.4h, v26.4h\n" + "smlal2 v24.4s, v22.8h, v26.8h\n" + "ldr q21, [x13, #0x10]\n" + "ldr q3, [x12, #0x10]\n" + "smlal v9.4s, v4.4h, v23.4h\n" + "smlal v7.4s, v22.4h, v5.4h\n" + "ldr x20, [x15, #0x28]\n" + "ldr d11, [x20, x17]\n" + "smlal v2.4s, v22.4h, v16.4h\n" + "smlal v10.4s, v22.4h, v23.4h\n" + "smlal2 v24.4s, v4.8h, v23.8h\n" + "ldr x20, [x15, #0x38]\n" + "ldr d4, [x20, x17]\n" + "smlal v9.4s, v27.4h, v18.4h\n" + "smlal2 v0.4s, v22.8h, v5.8h\n" + "smlal2 v30.4s, v22.8h, v16.8h\n" + "ldr x20, [x15, #0x30]\n" + "usubl v11.8h, v11.8b, v14.8b\n" + "smlal2 v6.4s, v22.8h, v23.8h\n" + "ldr d22, [x20, x17]\n" + "smlal v7.4s, v8.4h, v1.4h\n" + "ldr x20, [x15, #0x40]\n" + "smlal v2.4s, v27.4h, v1.4h\n" + "smlal v10.4s, v27.4h, v16.4h\n" + "usubl v4.8h, v4.8b, v14.8b\n" + "ldr x26, [x15, #0x48]\n" + "smlal2 v24.4s, v27.8h, v18.8h\n" + "smlal v9.4s, v15.4h, v25.4h\n" + "usubl v22.8h, v22.8b, v14.8b\n" + "ldr x25, [x15, #0x50]\n" + "smlal2 v0.4s, v8.8h, v1.8h\n" + "ldr d8, [x20, x17]\n" + "smlal2 v30.4s, v27.8h, v1.8h\n" + "usubl v8.8h, v8.8b, v14.8b\n" + "smlal2 v6.4s, v27.8h, v16.8h\n" + "smlal v7.4s, v27.4h, v26.4h\n" + "ldr x24, [x15, #0x58]\n" + "ldr x23, [x15, #0x60]\n" + "smlal v2.4s, v11.4h, v31.4h\n" + "smlal v10.4s, v15.4h, v5.4h\n" + "ldr x22, [x15, #0x68]\n" + "ldr x21, [x15, #0x70]\n" + "smlal2 v24.4s, v15.8h, v25.8h\n" + "smlal v9.4s, v4.4h, v16.4h\n" + "ldr x20, [x15, #0x78]\n" + "tst x7, #0x7\n" + "smlal2 v0.4s, v27.8h, v26.8h\n" + "ldr d27, [x26, x17]\n" + "smlal2 v30.4s, v11.8h, v31.8h\n" + "ldr d11, [x25, x17]\n" + "smlal2 v6.4s, v15.8h, v5.8h\n" + "smlal v7.4s, v15.4h, v31.4h\n" + "usubl v27.8h, v27.8b, v14.8b\n" "add x13, x13, #0x20\n" - "smlal v16.4s, v31.4h, v6.4h\n" - "smlal2 v26.4s, v31.8h, v6.8h\n" - "ldr d31, [x27, x8]\n" - "usubl v31.8h, v31.8b, v24.8b\n" - "smlal v25.4s, v27.4h, v3.4h\n" - "smlal2 v10.4s, v27.8h, v3.8h\n" - "smlal v13.4s, v28.4h, v1.4h\n" - "smlal2 v20.4s, v28.8h, v1.8h\n" - "smlal v9.4s, v28.4h, v0.4h\n" - "smlal2 v18.4s, v28.8h, v0.8h\n" - "ldr d30, [x26, x8]\n" - "usubl v30.8h, v30.8b, v24.8b\n" - "smlal v16.4s, v27.4h, v4.4h\n" - "smlal v25.4s, v29.4h, v8.4h\n" - "smlal2 v26.4s, v27.8h, v4.8h\n" - "ldr d28, [x24, x8]\n" - "smlal2 v10.4s, v29.8h, v8.8h\n" - "ldr d29, [x25, x8]\n" - "smlal v13.4s, v31.4h, v2.4h\n" - "smlal2 v20.4s, v31.8h, v2.8h\n" - "usubl v29.8h, v29.8b, v24.8b\n" - "smlal v9.4s, v31.4h, v1.4h\n" - "smlal2 v18.4s, v31.8h, v1.8h\n" - "ldr d31, [x23, x8]\n" - "usubl v28.8h, v28.8b, v24.8b\n" - "smlal v16.4s, v30.4h, v5.4h\n" - "smlal v25.4s, v30.4h, v4.4h\n" - "usubl v31.8h, v31.8b, v24.8b\n" - "smlal v13.4s, v30.4h, v8.4h\n" - "smlal2 v20.4s, v30.8h, v8.8h\n" - "smlal v9.4s, v30.4h, v7.4h\n" - "smlal2 v18.4s, v30.8h, v7.8h\n" - "smlal2 v26.4s, v30.8h, v5.8h\n" - "smlal2 v10.4s, v30.8h, v4.8h\n" - "ldr d30, [x22, x8]\n" - "usubl v30.8h, v30.8b, v24.8b\n" - "smlal v16.4s, v29.4h, v0.4h\n" - "smlal v25.4s, v28.4h, v2.4h\n" - "smlal v13.4s, v29.4h, v3.4h\n" - "smlal2 v20.4s, v29.8h, v3.8h\n" - "smlal2 v26.4s, v29.8h, v0.8h\n" - "ldr d29, [x21, x8]\n" - "smlal2 v10.4s, v28.8h, v2.8h\n" - "usubl v29.8h, v29.8b, v24.8b\n" - "smlal v16.4s, v31.4h, v3.4h\n" - "smlal v25.4s, v30.4h, v5.4h\n" - "smlal v9.4s, v28.4h, v5.4h\n" - "smlal2 v18.4s, v28.8h, v5.8h\n" - "ldr d28, [x20, x8]\n" - "usubl v28.8h, v28.8b, v24.8b\n" - "smlal v13.4s, v31.4h, v6.4h\n" - "smlal2 v26.4s, v31.8h, v3.8h\n" - "sqrdmulh v13.4s, v13.4s, v17.4s\n" - "add x8, x8, #0x8\n" - "smlal2 v10.4s, v30.8h, v5.8h\n" - "smlal v16.4s, v29.4h, v7.4h\n" - "and v21.16b, v13.16b, v22.16b\n" - "smlal v25.4s, v29.4h, v6.4h\n" - "smlal2 v20.4s, v31.8h, v6.8h\n" - "sqrdmulh v20.4s, v20.4s, v23.4s\n" - "smlal2 v26.4s, v29.8h, v7.8h\n" - "smlal2 v10.4s, v29.8h, v6.8h\n" - "sshr v21.4s, v21.4s, #0x1f\n" - "smlal v9.4s, v30.4h, v8.4h\n" - "smlal v16.4s, v28.4h, v8.4h\n" - "and v29.16b, v20.16b, v19.16b\n" - "smlal v25.4s, v28.4h, v7.4h\n" - "smlal2 v18.4s, v30.8h, v8.8h\n" - "sqrdmulh v9.4s, v9.4s, v17.4s\n" - "smlal2 v26.4s, v28.8h, v8.8h\n" - "smlal2 v10.4s, v28.8h, v7.8h\n" - "sqrdmulh v16.4s, v16.4s, v17.4s\n" - "sqrdmulh v25.4s, v25.4s, v17.4s\n" - "sqadd v13.4s, v13.4s, v21.4s\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "and v0.16b, v9.16b, v22.16b\n" - "sqrdmulh v18.4s, v18.4s, v23.4s\n" - "and v27.16b, v16.16b, v22.16b\n" - "sqrdmulh v26.4s, v26.4s, v23.4s\n" - "and v21.16b, v25.16b, v22.16b\n" - "sqrdmulh v10.4s, v10.4s, v23.4s\n" - "sqadd v20.4s, v20.4s, v29.4s\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v17.16b, v18.16b, v19.16b\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "and v7.16b, v26.16b, v19.16b\n" + "smlal v2.4s, v15.4h, v26.4h\n" + "smlal v10.4s, v22.4h, v20.4h\n" + "usubl v11.8h, v11.8b, v14.8b\n" + "add x12, x12, #0x20\n" + "smlal2 v24.4s, v4.8h, v16.8h\n" + "smlal v9.4s, v8.4h, v1.4h\n" + "smlal2 v0.4s, v15.8h, v31.8h\n" + "smlal2 v30.4s, v15.8h, v26.8h\n" + "ldr d15, [x24, x17]\n" + "usubl v15.8h, v15.8b, v14.8b\n" + "smlal2 v6.4s, v22.8h, v20.8h\n" + "ldr d22, [x23, x17]\n" + "smlal v7.4s, v4.4h, v23.4h\n" + "usubl v22.8h, v22.8b, v14.8b\n" + "smlal v2.4s, v27.4h, v18.4h\n" + "smlal v10.4s, v27.4h, v26.4h\n" + "smlal2 v24.4s, v8.8h, v1.8h\n" + "smlal v9.4s, v27.4h, v20.4h\n" + "smlal2 v0.4s, v4.8h, v23.8h\n" + "ldr d4, [x22, x17]\n" + "smlal2 v30.4s, v27.8h, v18.8h\n" + "usubl v4.8h, v4.8b, v14.8b\n" + "smlal2 v6.4s, v27.8h, v26.8h\n" + "ldr d26, [x21, x17]\n" + "smlal v7.4s, v8.4h, v16.4h\n" + "usubl v26.8h, v26.8b, v14.8b\n" + "smlal v2.4s, v11.4h, v23.4h\n" + "smlal v10.4s, v15.4h, v1.4h\n" + "smlal2 v24.4s, v27.8h, v20.8h\n" + "smlal v9.4s, v11.4h, v5.4h\n" + "smlal2 v0.4s, v8.8h, v16.8h\n" + "ldr d16, [x20, x17]\n" + "smlal2 v30.4s, v11.8h, v23.8h\n" + "usubl v16.8h, v16.8b, v14.8b\n" + "smlal2 v6.4s, v15.8h, v1.8h\n" + "smlal v7.4s, v27.4h, v25.4h\n" + "add x17, x17, #0x8\n" + "smlal v2.4s, v22.4h, v5.4h\n" + "smlal v10.4s, v4.4h, v18.4h\n" + "smlal2 v24.4s, v11.8h, v5.8h\n" + "smlal v9.4s, v22.4h, v31.4h\n" + "sqrdmulh v9.4s, v9.4s, v28.4s\n" + "smlal2 v0.4s, v27.8h, v25.8h\n" + "smlal2 v30.4s, v22.8h, v5.8h\n" + "and v1.16b, v9.16b, v17.16b\n" + "smlal2 v6.4s, v4.8h, v18.8h\n" + "smlal v7.4s, v15.4h, v18.4h\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "smlal v2.4s, v26.4h, v25.4h\n" + "smlal v10.4s, v26.4h, v31.4h\n" + "sqadd v9.4s, v9.4s, v1.4s\n" + "smlal2 v24.4s, v22.8h, v31.8h\n" + "smlal2 v0.4s, v15.8h, v18.8h\n" + "sqrdmulh v24.4s, v24.4s, v21.4s\n" + "smlal2 v30.4s, v26.8h, v25.8h\n" + "smlal2 v6.4s, v26.8h, v31.8h\n" + "and v31.16b, v24.16b, v3.16b\n" + "smlal v7.4s, v4.4h, v20.4h\n" + "smlal v2.4s, v16.4h, v20.4h\n" + "sqrdmulh v7.4s, v7.4s, v28.4s\n" + "smlal v10.4s, v16.4h, v25.4h\n" + "smlal2 v0.4s, v4.8h, v20.8h\n" + "sqrdmulh v2.4s, v2.4s, v28.4s\n" + "smlal2 v30.4s, v16.8h, v20.8h\n" + "smlal2 v6.4s, v16.8h, v25.8h\n" + "sqrdmulh v10.4s, v10.4s, v28.4s\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "and v22.16b, v7.16b, v17.16b\n" + "sqrdmulh v0.4s, v0.4s, v21.4s\n" + "and v15.16b, v2.16b, v17.16b\n" + "sqrdmulh v30.4s, v30.4s, v21.4s\n" + "and v11.16b, v10.16b, v17.16b\n" + "sqrdmulh v6.4s, v6.4s, v21.4s\n" + "sqadd v24.4s, v24.4s, v31.4s\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "and v18.16b, v0.16b, v3.16b\n" + "sshr v15.4s, v15.4s, #0x1f\n" + "and v23.16b, v30.16b, v3.16b\n" + "sshr v11.4s, v11.4s, #0x1f\n" + "and v21.16b, v6.16b, v3.16b\n" + "sqadd v7.4s, v7.4s, v22.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sqadd v2.4s, v2.4s, v15.4s\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "sqadd v10.4s, v10.4s, v11.4s\n" "sshr v21.4s, v21.4s, #0x1f\n" - "and v29.16b, v10.16b, v19.16b\n" - "sqadd v9.4s, v9.4s, v0.4s\n" - "sshr v17.4s, v17.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v27.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v25.4s, v25.4s, v21.4s\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "srshl v13.4s, v13.4s, v22.4s\n" - "srshl v9.4s, v9.4s, v22.4s\n" - "sqadd v18.4s, v18.4s, v17.4s\n" - "srshl v16.4s, v16.4s, v22.4s\n" - "sqadd v26.4s, v26.4s, v7.4s\n" - "srshl v25.4s, v25.4s, v22.4s\n" - "sqadd v10.4s, v10.4s, v29.4s\n" - "srshl v20.4s, v20.4s, v19.4s\n" - "sqxtn v13.4h, v13.4s\n" - "srshl v18.4s, v18.4s, v19.4s\n" + "srshl v9.4s, v9.4s, v17.4s\n" + "srshl v7.4s, v7.4s, v17.4s\n" + "sqadd v0.4s, v0.4s, v18.4s\n" + "srshl v2.4s, v2.4s, v17.4s\n" + "sqadd v30.4s, v30.4s, v23.4s\n" + "srshl v10.4s, v10.4s, v17.4s\n" + "sqadd v6.4s, v6.4s, v21.4s\n" + "srshl v24.4s, v24.4s, v3.4s\n" "sqxtn v9.4h, v9.4s\n" - "srshl v26.4s, v26.4s, v19.4s\n" - "sqxtn v16.4h, v16.4s\n" - "srshl v10.4s, v10.4s, v19.4s\n" - "sqxtn v25.4h, v25.4s\n" - "sqxtn2 v13.8h, v20.4s\n" - "sqxtn2 v9.8h, v18.4s\n" - "sqxtn2 v16.8h, v26.4s\n" - "sqxtn2 v25.8h, v10.4s\n" - "sqadd v13.8h, v13.8h, v14.8h\n" - "sqadd v9.8h, v9.8h, v14.8h\n" - "sqadd v16.8h, v16.8h, v14.8h\n" - "sqadd v25.8h, v25.8h, v14.8h\n" - "smax v13.8h, v13.8h, v12.8h\n" - "smax v9.8h, v9.8h, v12.8h\n" - "smax v16.8h, v16.8h, v12.8h\n" - "smax v25.8h, v25.8h, v12.8h\n" - "smin v13.8h, v13.8h, v11.8h\n" - "smin v9.8h, v9.8h, v11.8h\n" - "smin v16.8h, v16.8h, v11.8h\n" - "smin v25.8h, v25.8h, v11.8h\n" - "uzp1 v13.16b, v13.16b, v13.16b\n" - "str d13, [x12, x17]\n" + "srshl v0.4s, v0.4s, v3.4s\n" + "sqxtn v7.4h, v7.4s\n" + "srshl v30.4s, v30.4s, v3.4s\n" + "sqxtn v2.4h, v2.4s\n" + "srshl v6.4s, v6.4s, v3.4s\n" + "sqxtn v10.4h, v10.4s\n" + "sqxtn2 v9.8h, v24.4s\n" + "sqxtn2 v7.8h, v0.4s\n" + "sqxtn2 v2.8h, v30.4s\n" + "sqxtn2 v10.8h, v6.4s\n" + "sqadd v9.8h, v9.8h, v13.8h\n" + "sqadd v7.8h, v7.8h, v13.8h\n" + "sqadd v2.8h, v2.8h, v13.8h\n" + "sqadd v10.8h, v10.8h, v13.8h\n" + "smax v9.8h, v9.8h, v29.8h\n" + "smax v7.8h, v7.8h, v29.8h\n" + "smax v2.8h, v2.8h, v29.8h\n" + "smax v10.8h, v10.8h, v29.8h\n" + "smin v9.8h, v9.8h, v12.8h\n" + "smin v7.8h, v7.8h, v12.8h\n" + "smin v2.8h, v2.8h, v12.8h\n" + "smin v10.8h, v10.8h, v12.8h\n" "uzp1 v9.16b, v9.16b, v9.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "str d9, [x11, x17]\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "str d16, [x10, x17]\n" - "str d25, [x9, x17]\n" - "add x17, x17, #0x8\n" + "str d9, [x11, x16]\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" + "uzp1 v2.16b, v2.16b, v2.16b\n" + "str d7, [x10, x16]\n" + "uzp1 v10.16b, v10.16b, v10.16b\n" + "str d2, [x9, x16]\n" + "str d10, [x28, x16]\n" + "add x16, x16, #0x8\n" "beq 64f\n" - "add x15, x15, #0x48\n" + "add x14, x14, #0x48\n" "3:" // Oddments - "ldr x28, [%x[params], %[offsetof_Params_bias]]\n" - "tbz x6, #2, 5f\n" - "ld1 { v13.4s }, [x28], #0x10\n" - "tbz x6, #1, 4f\n" - "ld1 { v20.d }[0], [x28], #0x8\n" - "tbz x6, #0, 7f\n" - "ld1 { v20.s }[2], [x28]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "tbz x7, #2, 5f\n" + "ld1 { v9.4s }, [x20], #0x10\n" + "tbz x7, #1, 4f\n" + "ld1 { v24.d }[0], [x20], #0x8\n" + "tbz x7, #0, 7f\n" + "ld1 { v24.s }[2], [x20]\n" "b 7f\n" "4:" // Oddments: Load bias: Bit 2: Bit 1: Unset - "tbz x6, #0, 7f\n" - "ld1 { v20.s }[0], [x28]\n" + "tbz x7, #0, 7f\n" + "ld1 { v24.s }[0], [x20]\n" "b 7f\n" "5:" // Oddments: Load bias: Bit 2: Unset - "tbz x6, #1, 6f\n" - "ld1 { v13.d }[0], [x28], #0x8\n" - "tbz x6, #0, 7f\n" - "ld1 { v13.s }[2], [x28]\n" + "tbz x7, #1, 6f\n" + "ld1 { v9.d }[0], [x20], #0x8\n" + "tbz x7, #0, 7f\n" + "ld1 { v9.s }[2], [x20]\n" "b 7f\n" "6:" // Oddments: Load bias: Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 7f\n" - "ld1 { v13.s }[0], [x28]\n" + "tbz x7, #0, 7f\n" + "ld1 { v9.s }[0], [x20]\n" "7:" // Oddments: Load bias: Bit 2: End - "ldr d0, [x15, #0x0]\n" - "ldr d1, [x15, #0x8]\n" - "mov v9.16b, v13.16b\n" - "mov v18.16b, v20.16b\n" - "ldr d2, [x15, #0x10]\n" - "ldr d3, [x15, #0x18]\n" - "mov v16.16b, v13.16b\n" - "mov v26.16b, v20.16b\n" - "ldr d4, [x15, #0x20]\n" - "ldr d5, [x15, #0x28]\n" - "mov v25.16b, v13.16b\n" - "mov v10.16b, v20.16b\n" - "ldr d6, [x15, #0x30]\n" - "ldr d7, [x15, #0x38]\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "ldr d8, [x15, #0x40]\n" - "ldp x24, x23, [x16, #0x0]\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "ldp x22, x21, [x16, #0x10]\n" - "ldr x20, [x16, #0x20]\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "ssubl v5.8h, v5.8b, v15.8b\n" - "ssubl v6.8h, v6.8b, v15.8b\n" - "ssubl v7.8h, v7.8b, v15.8b\n" - "ssubl v8.8h, v8.8b, v15.8b\n" - "add x24, x24, x8\n" - "add x23, x23, x8\n" - "add x22, x22, x8\n" - "add x21, x21, x8\n" - "add x20, x20, x8\n" - "tbz x6, #2, 9f\n" - "ld1 { v31.s }[0], [x24], #0x4\n" - "ld1 { v30.s }[0], [x23], #0x4\n" - "ld1 { v29.s }[0], [x22], #0x4\n" - "ld1 { v28.s }[0], [x21], #0x4\n" - "ld1 { v27.s }[0], [x20], #0x4\n" - "tbz x6, #1, 8f\n" - "ld1 { v31.h }[2], [x24], #0x2\n" - "ld1 { v30.h }[2], [x23], #0x2\n" - "ld1 { v29.h }[2], [x22], #0x2\n" - "ld1 { v28.h }[2], [x21], #0x2\n" - "ld1 { v27.h }[2], [x20], #0x2\n" - "tbz x6, #0, 11f\n" - "ld1 { v31.b }[6], [x24]\n" - "ld1 { v30.b }[6], [x23]\n" - "ld1 { v29.b }[6], [x22]\n" - "ld1 { v28.b }[6], [x21]\n" - "ld1 { v27.b }[6], [x20]\n" + "ldr d23, [x14, #0x0]\n" + "ldr d16, [x14, #0x8]\n" + "mov v7.16b, v9.16b\n" + "mov v0.16b, v24.16b\n" + "ldr d1, [x14, #0x10]\n" + "ldr d5, [x14, #0x18]\n" + "mov v2.16b, v9.16b\n" + "mov v30.16b, v24.16b\n" + "ldr d26, [x14, #0x20]\n" + "ldr d18, [x14, #0x28]\n" + "mov v10.16b, v9.16b\n" + "mov v6.16b, v24.16b\n" + "ldr d31, [x14, #0x30]\n" + "ldr d25, [x14, #0x38]\n" + "ssubl v23.8h, v23.8b, v19.8b\n" + "ssubl v16.8h, v16.8b, v19.8b\n" + "ldr d20, [x14, #0x40]\n" + "ldp x24, x23, [x15, #0x0]\n" + "ssubl v1.8h, v1.8b, v19.8b\n" + "ssubl v5.8h, v5.8b, v19.8b\n" + "ldp x22, x21, [x15, #0x10]\n" + "ldr x20, [x15, #0x20]\n" + "ssubl v26.8h, v26.8b, v19.8b\n" + "ssubl v18.8h, v18.8b, v19.8b\n" + "ssubl v31.8h, v31.8b, v19.8b\n" + "ssubl v25.8h, v25.8b, v19.8b\n" + "ssubl v20.8h, v20.8b, v19.8b\n" + "add x24, x24, x17\n" + "add x23, x23, x17\n" + "add x22, x22, x17\n" + "add x21, x21, x17\n" + "add x20, x20, x17\n" + "tbz x7, #2, 9f\n" + "ld1 { v22.s }[0], [x24], #0x4\n" + "ld1 { v4.s }[0], [x23], #0x4\n" + "ld1 { v8.s }[0], [x22], #0x4\n" + "ld1 { v27.s }[0], [x21], #0x4\n" + "ld1 { v15.s }[0], [x20], #0x4\n" + "tbz x7, #1, 8f\n" + "ld1 { v22.h }[2], [x24], #0x2\n" + "ld1 { v4.h }[2], [x23], #0x2\n" + "ld1 { v8.h }[2], [x22], #0x2\n" + "ld1 { v27.h }[2], [x21], #0x2\n" + "ld1 { v15.h }[2], [x20], #0x2\n" + "tbz x7, #0, 11f\n" + "ld1 { v22.b }[6], [x24]\n" + "ld1 { v4.b }[6], [x23]\n" + "ld1 { v8.b }[6], [x22]\n" + "ld1 { v27.b }[6], [x21]\n" + "ld1 { v15.b }[6], [x20]\n" "b 11f\n" "8:" // Oddments: Initial loads: Bit 2: Bit 1: Unset - "tbz x6, #0, 11f\n" - "ld1 { v31.b }[4], [x24]\n" - "ld1 { v30.b }[4], [x23]\n" - "ld1 { v29.b }[4], [x22]\n" - "ld1 { v28.b }[4], [x21]\n" - "ld1 { v27.b }[4], [x20]\n" + "tbz x7, #0, 11f\n" + "ld1 { v22.b }[4], [x24]\n" + "ld1 { v4.b }[4], [x23]\n" + "ld1 { v8.b }[4], [x22]\n" + "ld1 { v27.b }[4], [x21]\n" + "ld1 { v15.b }[4], [x20]\n" "b 11f\n" "9:" // Oddments: Initial loads: Bit 2: Unset - "tbz x6, #1, 10f\n" - "ld1 { v31.h }[0], [x24], #0x2\n" - "ld1 { v30.h }[0], [x23], #0x2\n" - "ld1 { v29.h }[0], [x22], #0x2\n" - "ld1 { v28.h }[0], [x21], #0x2\n" - "ld1 { v27.h }[0], [x20], #0x2\n" - "tbz x6, #0, 11f\n" - "ld1 { v31.b }[2], [x24]\n" - "ld1 { v30.b }[2], [x23]\n" - "ld1 { v29.b }[2], [x22]\n" - "ld1 { v28.b }[2], [x21]\n" - "ld1 { v27.b }[2], [x20]\n" + "tbz x7, #1, 10f\n" + "ld1 { v22.h }[0], [x24], #0x2\n" + "ld1 { v4.h }[0], [x23], #0x2\n" + "ld1 { v8.h }[0], [x22], #0x2\n" + "ld1 { v27.h }[0], [x21], #0x2\n" + "ld1 { v15.h }[0], [x20], #0x2\n" + "tbz x7, #0, 11f\n" + "ld1 { v22.b }[2], [x24]\n" + "ld1 { v4.b }[2], [x23]\n" + "ld1 { v8.b }[2], [x22]\n" + "ld1 { v27.b }[2], [x21]\n" + "ld1 { v15.b }[2], [x20]\n" "b 11f\n" "10:" // Oddments: Initial loads: Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 11f\n" - "ld1 { v31.b }[0], [x24]\n" - "ld1 { v30.b }[0], [x23]\n" - "ld1 { v29.b }[0], [x22]\n" - "ld1 { v28.b }[0], [x21]\n" - "ld1 { v27.b }[0], [x20]\n" + "tbz x7, #0, 11f\n" + "ld1 { v22.b }[0], [x24]\n" + "ld1 { v4.b }[0], [x23]\n" + "ld1 { v8.b }[0], [x22]\n" + "ld1 { v27.b }[0], [x21]\n" + "ld1 { v15.b }[0], [x20]\n" "11:" // Oddments: Initial loads: Bit 2: End - "usubl v31.8h, v31.8b, v24.8b\n" - "smlal v13.4s, v31.4h, v4.4h\n" - "smlal2 v20.4s, v31.8h, v4.8h\n" - "ldr x21, [x16, #0x28]\n" - "smlal v9.4s, v31.4h, v3.4h\n" - "smlal2 v18.4s, v31.8h, v3.8h\n" - "usubl v30.8h, v30.8b, v24.8b\n" - "add x21, x21, x8\n" - "usubl v29.8h, v29.8b, v24.8b\n" - "smlal v16.4s, v31.4h, v1.4h\n" - "smlal2 v26.4s, v31.8h, v1.8h\n" - "smlal v25.4s, v31.4h, v0.4h\n" - "smlal2 v10.4s, v31.8h, v0.8h\n" - "usubl v28.8h, v28.8b, v24.8b\n" - "smlal v13.4s, v30.4h, v0.4h\n" - "smlal2 v20.4s, v30.8h, v0.8h\n" - "usubl v27.8h, v27.8b, v24.8b\n" - "smlal v9.4s, v29.4h, v2.4h\n" - "smlal2 v18.4s, v29.8h, v2.8h\n" - "smlal v13.4s, v28.4h, v5.4h\n" - "smlal2 v20.4s, v28.8h, v5.8h\n" - "smlal v9.4s, v28.4h, v4.4h\n" - "smlal2 v18.4s, v28.8h, v4.8h\n" - "smlal v16.4s, v28.4h, v2.4h\n" - "smlal2 v26.4s, v28.8h, v2.8h\n" - "smlal v25.4s, v28.4h, v1.4h\n" - "smlal2 v10.4s, v28.8h, v1.8h\n" - "tbz x6, #2, 13f\n" - "ld1 { v31.s }[0], [x21], #0x4\n" - "tbz x6, #1, 12f\n" - "ld1 { v31.h }[2], [x21], #0x2\n" - "tbz x6, #0, 15f\n" - "ld1 { v31.b }[6], [x21]\n" + "usubl v22.8h, v22.8b, v14.8b\n" + "smlal v9.4s, v22.4h, v26.4h\n" + "smlal2 v24.4s, v22.8h, v26.8h\n" + "ldr x20, [x15, #0x28]\n" + "smlal v7.4s, v22.4h, v5.4h\n" + "smlal2 v0.4s, v22.8h, v5.8h\n" + "usubl v4.8h, v4.8b, v14.8b\n" + "usubl v8.8h, v8.8b, v14.8b\n" + "smlal v2.4s, v22.4h, v16.4h\n" + "smlal2 v30.4s, v22.8h, v16.8h\n" + "add x20, x20, x17\n" + "smlal v10.4s, v22.4h, v23.4h\n" + "smlal2 v6.4s, v22.8h, v23.8h\n" + "usubl v27.8h, v27.8b, v14.8b\n" + "smlal v9.4s, v4.4h, v23.4h\n" + "smlal2 v24.4s, v4.8h, v23.8h\n" + "usubl v15.8h, v15.8b, v14.8b\n" + "smlal v7.4s, v8.4h, v1.4h\n" + "smlal2 v0.4s, v8.8h, v1.8h\n" + "smlal v9.4s, v27.4h, v18.4h\n" + "smlal2 v24.4s, v27.8h, v18.8h\n" + "smlal v7.4s, v27.4h, v26.4h\n" + "smlal2 v0.4s, v27.8h, v26.8h\n" + "smlal v2.4s, v27.4h, v1.4h\n" + "smlal2 v30.4s, v27.8h, v1.8h\n" + "smlal v10.4s, v27.4h, v16.4h\n" + "smlal2 v6.4s, v27.8h, v16.8h\n" + "tbz x7, #2, 13f\n" + "ld1 { v21.s }[0], [x20], #0x4\n" + "tbz x7, #1, 12f\n" + "ld1 { v21.h }[2], [x20], #0x2\n" + "tbz x7, #0, 15f\n" + "ld1 { v21.b }[6], [x20]\n" "b 15f\n" "12:" // Oddments: Load (3, 0): Bit 2: Bit 1: Unset - "tbz x6, #0, 15f\n" - "ld1 { v31.b }[4], [x21]\n" + "tbz x7, #0, 15f\n" + "ld1 { v21.b }[4], [x20]\n" "b 15f\n" "13:" // Oddments: Load (3, 0): Bit 2: Unset - "tbz x6, #1, 14f\n" - "ld1 { v31.h }[0], [x21], #0x2\n" - "tbz x6, #0, 15f\n" - "ld1 { v31.b }[2], [x21]\n" + "tbz x7, #1, 14f\n" + "ld1 { v21.h }[0], [x20], #0x2\n" + "tbz x7, #0, 15f\n" + "ld1 { v21.b }[2], [x20]\n" "b 15f\n" "14:" // Oddments: Load (3, 0): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 15f\n" - "ld1 { v31.b }[0], [x21]\n" + "tbz x7, #0, 15f\n" + "ld1 { v21.b }[0], [x20]\n" "15:" // Oddments: Load (3, 0): Bit 2: End - "usubl v31.8h, v31.8b, v24.8b\n" - "smlal v16.4s, v31.4h, v6.4h\n" - "smlal2 v26.4s, v31.8h, v6.8h\n" - "ldr x20, [x16, #0x30]\n" - "smlal v13.4s, v27.4h, v7.4h\n" - "smlal2 v20.4s, v27.8h, v7.8h\n" - "add x20, x20, x8\n" - "smlal v9.4s, v27.4h, v6.4h\n" - "smlal2 v18.4s, v27.8h, v6.8h\n" - "smlal v16.4s, v27.4h, v4.4h\n" - "smlal2 v26.4s, v27.8h, v4.8h\n" - "smlal v25.4s, v27.4h, v3.4h\n" - "smlal2 v10.4s, v27.8h, v3.8h\n" - "tbz x6, #2, 17f\n" - "ld1 { v29.s }[0], [x20], #0x4\n" - "tbz x6, #1, 16f\n" - "ld1 { v29.h }[2], [x20], #0x2\n" - "tbz x6, #0, 19f\n" - "ld1 { v29.b }[6], [x20]\n" + "usubl v21.8h, v21.8b, v14.8b\n" + "smlal v2.4s, v21.4h, v31.4h\n" + "smlal2 v30.4s, v21.8h, v31.8h\n" + "ldr x20, [x15, #0x30]\n" + "smlal v9.4s, v15.4h, v25.4h\n" + "smlal2 v24.4s, v15.8h, v25.8h\n" + "add x20, x20, x17\n" + "smlal v7.4s, v15.4h, v31.4h\n" + "smlal2 v0.4s, v15.8h, v31.8h\n" + "smlal v2.4s, v15.4h, v26.4h\n" + "smlal2 v30.4s, v15.8h, v26.8h\n" + "smlal v10.4s, v15.4h, v5.4h\n" + "smlal2 v6.4s, v15.8h, v5.8h\n" + "tbz x7, #2, 17f\n" + "ld1 { v28.s }[0], [x20], #0x4\n" + "tbz x7, #1, 16f\n" + "ld1 { v28.h }[2], [x20], #0x2\n" + "tbz x7, #0, 19f\n" + "ld1 { v28.b }[6], [x20]\n" "b 19f\n" "16:" // Oddments: Load (3, 3): Bit 2: Bit 1: Unset - "tbz x6, #0, 19f\n" - "ld1 { v29.b }[4], [x20]\n" + "tbz x7, #0, 19f\n" + "ld1 { v28.b }[4], [x20]\n" "b 19f\n" "17:" // Oddments: Load (3, 3): Bit 2: Unset - "tbz x6, #1, 18f\n" - "ld1 { v29.h }[0], [x20], #0x2\n" - "tbz x6, #0, 19f\n" - "ld1 { v29.b }[2], [x20]\n" + "tbz x7, #1, 18f\n" + "ld1 { v28.h }[0], [x20], #0x2\n" + "tbz x7, #0, 19f\n" + "ld1 { v28.b }[2], [x20]\n" "b 19f\n" "18:" // Oddments: Load (3, 3): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 19f\n" - "ld1 { v29.b }[0], [x20]\n" + "tbz x7, #0, 19f\n" + "ld1 { v28.b }[0], [x20]\n" "19:" // Oddments: Load (3, 3): Bit 2: End - "usubl v29.8h, v29.8b, v24.8b\n" - "ldr x28, [x16, #0x38]\n" - "smlal v25.4s, v29.4h, v8.4h\n" - "smlal2 v10.4s, v29.8h, v8.8h\n" - "add x28, x28, x8\n" - "tbz x6, #2, 21f\n" - "ld1 { v28.s }[0], [x28], #0x4\n" - "tbz x6, #1, 20f\n" - "ld1 { v28.h }[2], [x28], #0x2\n" - "tbz x6, #0, 23f\n" - "ld1 { v28.b }[6], [x28]\n" + "usubl v28.8h, v28.8b, v14.8b\n" + "ldr x20, [x15, #0x38]\n" + "smlal v10.4s, v28.4h, v20.4h\n" + "smlal2 v6.4s, v28.8h, v20.8h\n" + "add x20, x20, x17\n" + "tbz x7, #2, 21f\n" + "ld1 { v22.s }[0], [x20], #0x4\n" + "tbz x7, #1, 20f\n" + "ld1 { v22.h }[2], [x20], #0x2\n" + "tbz x7, #0, 23f\n" + "ld1 { v22.b }[6], [x20]\n" "b 23f\n" "20:" // Oddments: Load (0, 1): Bit 2: Bit 1: Unset - "tbz x6, #0, 23f\n" - "ld1 { v28.b }[4], [x28]\n" + "tbz x7, #0, 23f\n" + "ld1 { v22.b }[4], [x20]\n" "b 23f\n" "21:" // Oddments: Load (0, 1): Bit 2: Unset - "tbz x6, #1, 22f\n" - "ld1 { v28.h }[0], [x28], #0x2\n" - "tbz x6, #0, 23f\n" - "ld1 { v28.b }[2], [x28]\n" + "tbz x7, #1, 22f\n" + "ld1 { v22.h }[0], [x20], #0x2\n" + "tbz x7, #0, 23f\n" + "ld1 { v22.b }[2], [x20]\n" "b 23f\n" "22:" // Oddments: Load (0, 1): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 23f\n" - "ld1 { v28.b }[0], [x28]\n" + "tbz x7, #0, 23f\n" + "ld1 { v22.b }[0], [x20]\n" "23:" // Oddments: Load (0, 1): Bit 2: End - "usubl v28.8h, v28.8b, v24.8b\n" - "ldr x27, [x16, #0x40]\n" - "smlal v13.4s, v28.4h, v1.4h\n" - "smlal2 v20.4s, v28.8h, v1.8h\n" - "smlal v9.4s, v28.4h, v0.4h\n" - "smlal2 v18.4s, v28.8h, v0.8h\n" - "add x27, x27, x8\n" - "tbz x6, #2, 25f\n" - "ld1 { v31.s }[0], [x27], #0x4\n" - "tbz x6, #1, 24f\n" - "ld1 { v31.h }[2], [x27], #0x2\n" - "tbz x6, #0, 27f\n" - "ld1 { v31.b }[6], [x27]\n" + "usubl v22.8h, v22.8b, v14.8b\n" + "ldr x20, [x15, #0x40]\n" + "smlal v9.4s, v22.4h, v16.4h\n" + "smlal2 v24.4s, v22.8h, v16.8h\n" + "smlal v7.4s, v22.4h, v23.4h\n" + "smlal2 v0.4s, v22.8h, v23.8h\n" + "add x20, x20, x17\n" + "tbz x7, #2, 25f\n" + "ld1 { v21.s }[0], [x20], #0x4\n" + "tbz x7, #1, 24f\n" + "ld1 { v21.h }[2], [x20], #0x2\n" + "tbz x7, #0, 27f\n" + "ld1 { v21.b }[6], [x20]\n" "b 27f\n" "24:" // Oddments: Load (0, 2): Bit 2: Bit 1: Unset - "tbz x6, #0, 27f\n" - "ld1 { v31.b }[4], [x27]\n" + "tbz x7, #0, 27f\n" + "ld1 { v21.b }[4], [x20]\n" "b 27f\n" "25:" // Oddments: Load (0, 2): Bit 2: Unset - "tbz x6, #1, 26f\n" - "ld1 { v31.h }[0], [x27], #0x2\n" - "tbz x6, #0, 27f\n" - "ld1 { v31.b }[2], [x27]\n" + "tbz x7, #1, 26f\n" + "ld1 { v21.h }[0], [x20], #0x2\n" + "tbz x7, #0, 27f\n" + "ld1 { v21.b }[2], [x20]\n" "b 27f\n" "26:" // Oddments: Load (0, 2): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 27f\n" - "ld1 { v31.b }[0], [x27]\n" + "tbz x7, #0, 27f\n" + "ld1 { v21.b }[0], [x20]\n" "27:" // Oddments: Load (0, 2): Bit 2: End - "usubl v31.8h, v31.8b, v24.8b\n" - "ldr x26, [x16, #0x48]\n" - "smlal v13.4s, v31.4h, v2.4h\n" - "smlal2 v20.4s, v31.8h, v2.8h\n" - "smlal v9.4s, v31.4h, v1.4h\n" - "smlal2 v18.4s, v31.8h, v1.8h\n" - "add x26, x26, x8\n" - "tbz x6, #2, 29f\n" - "ld1 { v30.s }[0], [x26], #0x4\n" - "tbz x6, #1, 28f\n" - "ld1 { v30.h }[2], [x26], #0x2\n" - "tbz x6, #0, 31f\n" - "ld1 { v30.b }[6], [x26]\n" + "usubl v21.8h, v21.8b, v14.8b\n" + "ldr x20, [x15, #0x48]\n" + "smlal v9.4s, v21.4h, v1.4h\n" + "smlal2 v24.4s, v21.8h, v1.8h\n" + "smlal v7.4s, v21.4h, v16.4h\n" + "smlal2 v0.4s, v21.8h, v16.8h\n" + "add x20, x20, x17\n" + "tbz x7, #2, 29f\n" + "ld1 { v28.s }[0], [x20], #0x4\n" + "tbz x7, #1, 28f\n" + "ld1 { v28.h }[2], [x20], #0x2\n" + "tbz x7, #0, 31f\n" + "ld1 { v28.b }[6], [x20]\n" "b 31f\n" "28:" // Oddments: Load (2, 2): Bit 2: Bit 1: Unset - "tbz x6, #0, 31f\n" - "ld1 { v30.b }[4], [x26]\n" + "tbz x7, #0, 31f\n" + "ld1 { v28.b }[4], [x20]\n" "b 31f\n" "29:" // Oddments: Load (2, 2): Bit 2: Unset - "tbz x6, #1, 30f\n" - "ld1 { v30.h }[0], [x26], #0x2\n" - "tbz x6, #0, 31f\n" - "ld1 { v30.b }[2], [x26]\n" + "tbz x7, #1, 30f\n" + "ld1 { v28.h }[0], [x20], #0x2\n" + "tbz x7, #0, 31f\n" + "ld1 { v28.b }[2], [x20]\n" "b 31f\n" "30:" // Oddments: Load (2, 2): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 31f\n" - "ld1 { v30.b }[0], [x26]\n" + "tbz x7, #0, 31f\n" + "ld1 { v28.b }[0], [x20]\n" "31:" // Oddments: Load (2, 2): Bit 2: End - "usubl v30.8h, v30.8b, v24.8b\n" - "ldr x25, [x16, #0x50]\n" - "smlal v13.4s, v30.4h, v8.4h\n" - "smlal2 v20.4s, v30.8h, v8.8h\n" - "smlal v9.4s, v30.4h, v7.4h\n" - "smlal2 v18.4s, v30.8h, v7.8h\n" - "add x25, x25, x8\n" - "smlal v16.4s, v30.4h, v5.4h\n" - "smlal2 v26.4s, v30.8h, v5.8h\n" - "smlal v25.4s, v30.4h, v4.4h\n" - "smlal2 v10.4s, v30.8h, v4.8h\n" - "tbz x6, #2, 33f\n" - "ld1 { v29.s }[0], [x25], #0x4\n" - "tbz x6, #1, 32f\n" - "ld1 { v29.h }[2], [x25], #0x2\n" - "tbz x6, #0, 35f\n" - "ld1 { v29.b }[6], [x25]\n" + "usubl v28.8h, v28.8b, v14.8b\n" + "ldr x20, [x15, #0x50]\n" + "smlal v9.4s, v28.4h, v20.4h\n" + "smlal2 v24.4s, v28.8h, v20.8h\n" + "smlal v7.4s, v28.4h, v25.4h\n" + "smlal2 v0.4s, v28.8h, v25.8h\n" + "add x20, x20, x17\n" + "smlal v2.4s, v28.4h, v18.4h\n" + "smlal2 v30.4s, v28.8h, v18.8h\n" + "smlal v10.4s, v28.4h, v26.4h\n" + "smlal2 v6.4s, v28.8h, v26.8h\n" + "tbz x7, #2, 33f\n" + "ld1 { v8.s }[0], [x20], #0x4\n" + "tbz x7, #1, 32f\n" + "ld1 { v8.h }[2], [x20], #0x2\n" + "tbz x7, #0, 35f\n" + "ld1 { v8.b }[6], [x20]\n" "b 35f\n" "32:" // Oddments: Load (1, 0): Bit 2: Bit 1: Unset - "tbz x6, #0, 35f\n" - "ld1 { v29.b }[4], [x25]\n" + "tbz x7, #0, 35f\n" + "ld1 { v8.b }[4], [x20]\n" "b 35f\n" "33:" // Oddments: Load (1, 0): Bit 2: Unset - "tbz x6, #1, 34f\n" - "ld1 { v29.h }[0], [x25], #0x2\n" - "tbz x6, #0, 35f\n" - "ld1 { v29.b }[2], [x25]\n" + "tbz x7, #1, 34f\n" + "ld1 { v8.h }[0], [x20], #0x2\n" + "tbz x7, #0, 35f\n" + "ld1 { v8.b }[2], [x20]\n" "b 35f\n" "34:" // Oddments: Load (1, 0): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 35f\n" - "ld1 { v29.b }[0], [x25]\n" + "tbz x7, #0, 35f\n" + "ld1 { v8.b }[0], [x20]\n" "35:" // Oddments: Load (1, 0): Bit 2: End - "usubl v29.8h, v29.8b, v24.8b\n" - "ldr x24, [x16, #0x58]\n" - "smlal v13.4s, v29.4h, v3.4h\n" - "smlal2 v20.4s, v29.8h, v3.8h\n" - "smlal v16.4s, v29.4h, v0.4h\n" - "smlal2 v26.4s, v29.8h, v0.8h\n" - "add x24, x24, x8\n" - "tbz x6, #2, 37f\n" - "ld1 { v28.s }[0], [x24], #0x4\n" - "tbz x6, #1, 36f\n" - "ld1 { v28.h }[2], [x24], #0x2\n" - "tbz x6, #0, 39f\n" - "ld1 { v28.b }[6], [x24]\n" + "usubl v8.8h, v8.8b, v14.8b\n" + "ldr x20, [x15, #0x58]\n" + "smlal v9.4s, v8.4h, v5.4h\n" + "smlal2 v24.4s, v8.8h, v5.8h\n" + "smlal v2.4s, v8.4h, v23.4h\n" + "smlal2 v30.4s, v8.8h, v23.8h\n" + "add x20, x20, x17\n" + "tbz x7, #2, 37f\n" + "ld1 { v8.s }[0], [x20], #0x4\n" + "tbz x7, #1, 36f\n" + "ld1 { v8.h }[2], [x20], #0x2\n" + "tbz x7, #0, 39f\n" + "ld1 { v8.b }[6], [x20]\n" "b 39f\n" "36:" // Oddments: Load (1, 3): Bit 2: Bit 1: Unset - "tbz x6, #0, 39f\n" - "ld1 { v28.b }[4], [x24]\n" + "tbz x7, #0, 39f\n" + "ld1 { v8.b }[4], [x20]\n" "b 39f\n" "37:" // Oddments: Load (1, 3): Bit 2: Unset - "tbz x6, #1, 38f\n" - "ld1 { v28.h }[0], [x24], #0x2\n" - "tbz x6, #0, 39f\n" - "ld1 { v28.b }[2], [x24]\n" + "tbz x7, #1, 38f\n" + "ld1 { v8.h }[0], [x20], #0x2\n" + "tbz x7, #0, 39f\n" + "ld1 { v8.b }[2], [x20]\n" "b 39f\n" "38:" // Oddments: Load (1, 3): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 39f\n" - "ld1 { v28.b }[0], [x24]\n" + "tbz x7, #0, 39f\n" + "ld1 { v8.b }[0], [x20]\n" "39:" // Oddments: Load (1, 3): Bit 2: End - "usubl v28.8h, v28.8b, v24.8b\n" - "ldr x23, [x16, #0x60]\n" - "smlal v9.4s, v28.4h, v5.4h\n" - "smlal2 v18.4s, v28.8h, v5.8h\n" - "smlal v25.4s, v28.4h, v2.4h\n" - "smlal2 v10.4s, v28.8h, v2.8h\n" - "add x23, x23, x8\n" - "tbz x6, #2, 41f\n" - "ld1 { v31.s }[0], [x23], #0x4\n" - "tbz x6, #1, 40f\n" - "ld1 { v31.h }[2], [x23], #0x2\n" - "tbz x6, #0, 43f\n" - "ld1 { v31.b }[6], [x23]\n" + "usubl v8.8h, v8.8b, v14.8b\n" + "ldr x20, [x15, #0x60]\n" + "smlal v7.4s, v8.4h, v18.4h\n" + "smlal2 v0.4s, v8.8h, v18.8h\n" + "smlal v10.4s, v8.4h, v1.4h\n" + "smlal2 v6.4s, v8.8h, v1.8h\n" + "add x20, x20, x17\n" + "tbz x7, #2, 41f\n" + "ld1 { v17.s }[0], [x20], #0x4\n" + "tbz x7, #1, 40f\n" + "ld1 { v17.h }[2], [x20], #0x2\n" + "tbz x7, #0, 43f\n" + "ld1 { v17.b }[6], [x20]\n" "b 43f\n" "40:" // Oddments: Load (2, 0): Bit 2: Bit 1: Unset - "tbz x6, #0, 43f\n" - "ld1 { v31.b }[4], [x23]\n" + "tbz x7, #0, 43f\n" + "ld1 { v17.b }[4], [x20]\n" "b 43f\n" "41:" // Oddments: Load (2, 0): Bit 2: Unset - "tbz x6, #1, 42f\n" - "ld1 { v31.h }[0], [x23], #0x2\n" - "tbz x6, #0, 43f\n" - "ld1 { v31.b }[2], [x23]\n" + "tbz x7, #1, 42f\n" + "ld1 { v17.h }[0], [x20], #0x2\n" + "tbz x7, #0, 43f\n" + "ld1 { v17.b }[2], [x20]\n" "b 43f\n" "42:" // Oddments: Load (2, 0): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 43f\n" - "ld1 { v31.b }[0], [x23]\n" + "tbz x7, #0, 43f\n" + "ld1 { v17.b }[0], [x20]\n" "43:" // Oddments: Load (2, 0): Bit 2: End - "usubl v31.8h, v31.8b, v24.8b\n" - "ldr x22, [x16, #0x68]\n" - "smlal v13.4s, v31.4h, v6.4h\n" - "smlal2 v20.4s, v31.8h, v6.8h\n" - "smlal v16.4s, v31.4h, v3.4h\n" - "smlal2 v26.4s, v31.8h, v3.8h\n" - "add x22, x22, x8\n" - "tbz x6, #2, 45f\n" - "ld1 { v30.s }[0], [x22], #0x4\n" - "tbz x6, #1, 44f\n" - "ld1 { v30.h }[2], [x22], #0x2\n" - "tbz x6, #0, 47f\n" - "ld1 { v30.b }[6], [x22]\n" + "usubl v17.8h, v17.8b, v14.8b\n" + "ldr x20, [x15, #0x68]\n" + "smlal v9.4s, v17.4h, v31.4h\n" + "smlal2 v24.4s, v17.8h, v31.8h\n" + "smlal v2.4s, v17.4h, v5.4h\n" + "smlal2 v30.4s, v17.8h, v5.8h\n" + "add x20, x20, x17\n" + "tbz x7, #2, 45f\n" + "ld1 { v23.s }[0], [x20], #0x4\n" + "tbz x7, #1, 44f\n" + "ld1 { v23.h }[2], [x20], #0x2\n" + "tbz x7, #0, 47f\n" + "ld1 { v23.b }[6], [x20]\n" "b 47f\n" "44:" // Oddments: Load (2, 3): Bit 2: Bit 1: Unset - "tbz x6, #0, 47f\n" - "ld1 { v30.b }[4], [x22]\n" + "tbz x7, #0, 47f\n" + "ld1 { v23.b }[4], [x20]\n" "b 47f\n" "45:" // Oddments: Load (2, 3): Bit 2: Unset - "tbz x6, #1, 46f\n" - "ld1 { v30.h }[0], [x22], #0x2\n" - "tbz x6, #0, 47f\n" - "ld1 { v30.b }[2], [x22]\n" + "tbz x7, #1, 46f\n" + "ld1 { v23.h }[0], [x20], #0x2\n" + "tbz x7, #0, 47f\n" + "ld1 { v23.b }[2], [x20]\n" "b 47f\n" "46:" // Oddments: Load (2, 3): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 47f\n" - "ld1 { v30.b }[0], [x22]\n" + "tbz x7, #0, 47f\n" + "ld1 { v23.b }[0], [x20]\n" "47:" // Oddments: Load (2, 3): Bit 2: End - "usubl v30.8h, v30.8b, v24.8b\n" - "ldr x21, [x16, #0x70]\n" - "smlal v9.4s, v30.4h, v8.4h\n" - "smlal2 v18.4s, v30.8h, v8.8h\n" - "smlal v25.4s, v30.4h, v5.4h\n" - "smlal2 v10.4s, v30.8h, v5.8h\n" - "add x21, x21, x8\n" - "tbz x6, #2, 49f\n" - "ld1 { v29.s }[0], [x21], #0x4\n" - "tbz x6, #1, 48f\n" - "ld1 { v29.h }[2], [x21], #0x2\n" - "tbz x6, #0, 51f\n" - "ld1 { v29.b }[6], [x21]\n" + "usubl v23.8h, v23.8b, v14.8b\n" + "ldr x20, [x15, #0x70]\n" + "smlal v7.4s, v23.4h, v20.4h\n" + "smlal2 v0.4s, v23.8h, v20.8h\n" + "smlal v10.4s, v23.4h, v18.4h\n" + "smlal2 v6.4s, v23.8h, v18.8h\n" + "add x20, x20, x17\n" + "tbz x7, #2, 49f\n" + "ld1 { v5.s }[0], [x20], #0x4\n" + "tbz x7, #1, 48f\n" + "ld1 { v5.h }[2], [x20], #0x2\n" + "tbz x7, #0, 51f\n" + "ld1 { v5.b }[6], [x20]\n" "b 51f\n" "48:" // Oddments: Load (3, 1): Bit 2: Bit 1: Unset - "tbz x6, #0, 51f\n" - "ld1 { v29.b }[4], [x21]\n" + "tbz x7, #0, 51f\n" + "ld1 { v5.b }[4], [x20]\n" "b 51f\n" "49:" // Oddments: Load (3, 1): Bit 2: Unset - "tbz x6, #1, 50f\n" - "ld1 { v29.h }[0], [x21], #0x2\n" - "tbz x6, #0, 51f\n" - "ld1 { v29.b }[2], [x21]\n" + "tbz x7, #1, 50f\n" + "ld1 { v5.h }[0], [x20], #0x2\n" + "tbz x7, #0, 51f\n" + "ld1 { v5.b }[2], [x20]\n" "b 51f\n" "50:" // Oddments: Load (3, 1): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 51f\n" - "ld1 { v29.b }[0], [x21]\n" + "tbz x7, #0, 51f\n" + "ld1 { v5.b }[0], [x20]\n" "51:" // Oddments: Load (3, 1): Bit 2: End - "usubl v29.8h, v29.8b, v24.8b\n" - "ldr x20, [x16, #0x78]\n" - "smlal v16.4s, v29.4h, v7.4h\n" - "smlal2 v26.4s, v29.8h, v7.8h\n" - "smlal v25.4s, v29.4h, v6.4h\n" - "smlal2 v10.4s, v29.8h, v6.8h\n" - "add x20, x20, x8\n" - "tbz x6, #2, 53f\n" - "ld1 { v28.s }[0], [x20], #0x4\n" - "tbz x6, #1, 52f\n" - "ld1 { v28.h }[2], [x20], #0x2\n" - "tbz x6, #0, 55f\n" - "ld1 { v28.b }[6], [x20]\n" + "usubl v5.8h, v5.8b, v14.8b\n" + "ldr x20, [x15, #0x78]\n" + "smlal v2.4s, v5.4h, v25.4h\n" + "smlal2 v30.4s, v5.8h, v25.8h\n" + "smlal v10.4s, v5.4h, v31.4h\n" + "smlal2 v6.4s, v5.8h, v31.8h\n" + "add x20, x20, x17\n" + "tbz x7, #2, 53f\n" + "ld1 { v23.s }[0], [x20], #0x4\n" + "tbz x7, #1, 52f\n" + "ld1 { v23.h }[2], [x20], #0x2\n" + "tbz x7, #0, 55f\n" + "ld1 { v23.b }[6], [x20]\n" "b 55f\n" "52:" // Oddments: Load (3, 2): Bit 2: Bit 1: Unset - "tbz x6, #0, 55f\n" - "ld1 { v28.b }[4], [x20]\n" + "tbz x7, #0, 55f\n" + "ld1 { v23.b }[4], [x20]\n" "b 55f\n" "53:" // Oddments: Load (3, 2): Bit 2: Unset - "tbz x6, #1, 54f\n" - "ld1 { v28.h }[0], [x20], #0x2\n" - "tbz x6, #0, 55f\n" - "ld1 { v28.b }[2], [x20]\n" + "tbz x7, #1, 54f\n" + "ld1 { v23.h }[0], [x20], #0x2\n" + "tbz x7, #0, 55f\n" + "ld1 { v23.b }[2], [x20]\n" "b 55f\n" "54:" // Oddments: Load (3, 2): Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 55f\n" - "ld1 { v28.b }[0], [x20]\n" + "tbz x7, #0, 55f\n" + "ld1 { v23.b }[0], [x20]\n" "55:" // Oddments: Load (3, 2): Bit 2: End - "usubl v28.8h, v28.8b, v24.8b\n" - "smlal v16.4s, v28.4h, v8.4h\n" - "smlal2 v26.4s, v28.8h, v8.8h\n" - "smlal v25.4s, v28.4h, v7.4h\n" - "smlal2 v10.4s, v28.8h, v7.8h\n" - "tbz x6, #2, 57f\n" - "ld1 { v17.4s }, [x14], #0x10\n" - "ld1 { v22.4s }, [x13], #0x10\n" - "tbz x6, #1, 56f\n" - "ld1 { v23.d }[0], [x14], #0x8\n" - "ld1 { v19.d }[0], [x13], #0x8\n" - "tbz x6, #0, 59f\n" - "ld1 { v23.s }[2], [x14]\n" - "ld1 { v19.s }[2], [x13]\n" + "usubl v23.8h, v23.8b, v14.8b\n" + "smlal v2.4s, v23.4h, v20.4h\n" + "smlal2 v30.4s, v23.8h, v20.8h\n" + "smlal v10.4s, v23.4h, v25.4h\n" + "smlal2 v6.4s, v23.8h, v25.8h\n" + "tbz x7, #2, 57f\n" + "ld1 { v15.4s }, [x13], #0x10\n" + "ld1 { v19.4s }, [x12], #0x10\n" + "tbz x7, #1, 56f\n" + "ld1 { v18.d }[0], [x13], #0x8\n" + "ld1 { v22.d }[0], [x12], #0x8\n" + "tbz x7, #0, 59f\n" + "ld1 { v18.s }[2], [x13]\n" + "ld1 { v22.s }[2], [x12]\n" "b 59f\n" "56:" // Oddments: Load requant params: Bit 2: Bit 1: Unset - "tbz x6, #0, 59f\n" - "ld1 { v23.s }[0], [x14]\n" - "ld1 { v19.s }[0], [x13]\n" + "tbz x7, #0, 59f\n" + "ld1 { v18.s }[0], [x13]\n" + "ld1 { v22.s }[0], [x12]\n" "b 59f\n" "57:" // Oddments: Load requant params: Bit 2: Unset - "tbz x6, #1, 58f\n" - "ld1 { v17.d }[0], [x14], #0x8\n" - "ld1 { v22.d }[0], [x13], #0x8\n" - "tbz x6, #0, 59f\n" - "ld1 { v17.s }[2], [x14]\n" - "ld1 { v22.s }[2], [x13]\n" + "tbz x7, #1, 58f\n" + "ld1 { v15.d }[0], [x13], #0x8\n" + "ld1 { v19.d }[0], [x12], #0x8\n" + "tbz x7, #0, 59f\n" + "ld1 { v15.s }[2], [x13]\n" + "ld1 { v19.s }[2], [x12]\n" "b 59f\n" "58:" // Oddments: Load requant params: Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 59f\n" - "ld1 { v17.s }[0], [x14]\n" - "ld1 { v22.s }[0], [x13]\n" + "tbz x7, #0, 59f\n" + "ld1 { v15.s }[0], [x13]\n" + "ld1 { v19.s }[0], [x12]\n" "59:" // Oddments: Load requant params: Bit 2: End - "sqrdmulh v13.4s, v13.4s, v17.4s\n" - "and v21.16b, v13.16b, v22.16b\n" - "add x12, x12, x17\n" - "add x11, x11, x17\n" - "sqrdmulh v20.4s, v20.4s, v23.4s\n" - "sshr v21.4s, v21.4s, #0x1f\n" - "add x10, x10, x17\n" - "add x9, x9, x17\n" - "and v29.16b, v20.16b, v19.16b\n" - "sqrdmulh v9.4s, v9.4s, v17.4s\n" - "sqrdmulh v16.4s, v16.4s, v17.4s\n" - "sqrdmulh v25.4s, v25.4s, v17.4s\n" - "sqadd v13.4s, v13.4s, v21.4s\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "and v0.16b, v9.16b, v22.16b\n" - "sqrdmulh v18.4s, v18.4s, v23.4s\n" - "and v27.16b, v16.16b, v22.16b\n" - "sqrdmulh v26.4s, v26.4s, v23.4s\n" - "and v21.16b, v25.16b, v22.16b\n" - "sqrdmulh v10.4s, v10.4s, v23.4s\n" - "sqadd v20.4s, v20.4s, v29.4s\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v17.16b, v18.16b, v19.16b\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "and v7.16b, v26.16b, v19.16b\n" + "sqrdmulh v9.4s, v9.4s, v15.4s\n" + "and v17.16b, v9.16b, v19.16b\n" + "add x11, x11, x16\n" + "add x10, x10, x16\n" + "sqrdmulh v24.4s, v24.4s, v18.4s\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "add x9, x9, x16\n" + "add x28, x28, x16\n" + "and v20.16b, v24.16b, v22.16b\n" + "sqrdmulh v7.4s, v7.4s, v15.4s\n" + "sqrdmulh v2.4s, v2.4s, v15.4s\n" + "sqrdmulh v10.4s, v10.4s, v15.4s\n" + "sqadd v9.4s, v9.4s, v17.4s\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "and v21.16b, v7.16b, v19.16b\n" + "sqrdmulh v0.4s, v0.4s, v18.4s\n" + "and v15.16b, v2.16b, v19.16b\n" + "sqrdmulh v30.4s, v30.4s, v18.4s\n" + "and v23.16b, v10.16b, v19.16b\n" + "sqrdmulh v6.4s, v6.4s, v18.4s\n" + "sqadd v24.4s, v24.4s, v20.4s\n" "sshr v21.4s, v21.4s, #0x1f\n" - "and v29.16b, v10.16b, v19.16b\n" - "sqadd v9.4s, v9.4s, v0.4s\n" + "and v18.16b, v0.16b, v22.16b\n" + "sshr v15.4s, v15.4s, #0x1f\n" + "and v17.16b, v30.16b, v22.16b\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "and v28.16b, v6.16b, v22.16b\n" + "sqadd v7.4s, v7.4s, v21.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sqadd v2.4s, v2.4s, v15.4s\n" "sshr v17.4s, v17.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v27.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v25.4s, v25.4s, v21.4s\n" - "sshr v29.4s, v29.4s, #0x1f\n" - "srshl v13.4s, v13.4s, v22.4s\n" - "srshl v9.4s, v9.4s, v22.4s\n" - "sqadd v18.4s, v18.4s, v17.4s\n" - "srshl v16.4s, v16.4s, v22.4s\n" - "sqadd v26.4s, v26.4s, v7.4s\n" - "srshl v25.4s, v25.4s, v22.4s\n" - "sqadd v10.4s, v10.4s, v29.4s\n" - "srshl v20.4s, v20.4s, v19.4s\n" - "sqxtn v13.4h, v13.4s\n" - "srshl v18.4s, v18.4s, v19.4s\n" - "sqxtn v9.4h, v9.4s\n" - "srshl v26.4s, v26.4s, v19.4s\n" - "sqxtn v16.4h, v16.4s\n" + "sqadd v10.4s, v10.4s, v23.4s\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "srshl v9.4s, v9.4s, v19.4s\n" + "srshl v7.4s, v7.4s, v19.4s\n" + "sqadd v0.4s, v0.4s, v18.4s\n" + "srshl v2.4s, v2.4s, v19.4s\n" + "sqadd v30.4s, v30.4s, v17.4s\n" "srshl v10.4s, v10.4s, v19.4s\n" - "sqxtn v25.4h, v25.4s\n" - "sqxtn2 v13.8h, v20.4s\n" - "sqxtn2 v9.8h, v18.4s\n" - "sqxtn2 v16.8h, v26.4s\n" - "sqxtn2 v25.8h, v10.4s\n" - "sqadd v13.8h, v13.8h, v14.8h\n" - "sqadd v9.8h, v9.8h, v14.8h\n" - "sqadd v16.8h, v16.8h, v14.8h\n" - "sqadd v25.8h, v25.8h, v14.8h\n" - "smax v13.8h, v13.8h, v12.8h\n" - "smax v9.8h, v9.8h, v12.8h\n" - "smax v16.8h, v16.8h, v12.8h\n" - "smax v25.8h, v25.8h, v12.8h\n" - "smin v13.8h, v13.8h, v11.8h\n" - "smin v9.8h, v9.8h, v11.8h\n" - "smin v16.8h, v16.8h, v11.8h\n" - "smin v25.8h, v25.8h, v11.8h\n" - "uzp1 v13.16b, v13.16b, v13.16b\n" + "sqadd v6.4s, v6.4s, v28.4s\n" + "srshl v24.4s, v24.4s, v22.4s\n" + "sqxtn v9.4h, v9.4s\n" + "srshl v0.4s, v0.4s, v22.4s\n" + "sqxtn v7.4h, v7.4s\n" + "srshl v30.4s, v30.4s, v22.4s\n" + "sqxtn v2.4h, v2.4s\n" + "srshl v6.4s, v6.4s, v22.4s\n" + "sqxtn v10.4h, v10.4s\n" + "sqxtn2 v9.8h, v24.4s\n" + "sqxtn2 v7.8h, v0.4s\n" + "sqxtn2 v2.8h, v30.4s\n" + "sqxtn2 v10.8h, v6.4s\n" + "sqadd v9.8h, v9.8h, v13.8h\n" + "sqadd v7.8h, v7.8h, v13.8h\n" + "sqadd v2.8h, v2.8h, v13.8h\n" + "sqadd v10.8h, v10.8h, v13.8h\n" + "smax v9.8h, v9.8h, v29.8h\n" + "smax v7.8h, v7.8h, v29.8h\n" + "smax v2.8h, v2.8h, v29.8h\n" + "smax v10.8h, v10.8h, v29.8h\n" + "smin v9.8h, v9.8h, v12.8h\n" + "smin v7.8h, v7.8h, v12.8h\n" + "smin v2.8h, v2.8h, v12.8h\n" + "smin v10.8h, v10.8h, v12.8h\n" "uzp1 v9.16b, v9.16b, v9.16b\n" - "uzp1 v16.16b, v16.16b, v16.16b\n" - "uzp1 v25.16b, v25.16b, v25.16b\n" - "tbz x6, #2, 61f\n" - "st1 { v13.s }[0], [x12], #0x4\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" + "uzp1 v2.16b, v2.16b, v2.16b\n" + "uzp1 v10.16b, v10.16b, v10.16b\n" + "tbz x7, #2, 61f\n" "st1 { v9.s }[0], [x11], #0x4\n" - "st1 { v16.s }[0], [x10], #0x4\n" - "st1 { v25.s }[0], [x9], #0x4\n" - "tbz x6, #1, 60f\n" - "st1 { v13.h }[2], [x12], #0x2\n" + "st1 { v7.s }[0], [x10], #0x4\n" + "st1 { v2.s }[0], [x9], #0x4\n" + "st1 { v10.s }[0], [x28], #0x4\n" + "tbz x7, #1, 60f\n" "st1 { v9.h }[2], [x11], #0x2\n" - "st1 { v16.h }[2], [x10], #0x2\n" - "st1 { v25.h }[2], [x9], #0x2\n" - "tbz x6, #0, 63f\n" - "st1 { v13.b }[6], [x12], #0x1\n" + "st1 { v7.h }[2], [x10], #0x2\n" + "st1 { v2.h }[2], [x9], #0x2\n" + "st1 { v10.h }[2], [x28], #0x2\n" + "tbz x7, #0, 63f\n" "st1 { v9.b }[6], [x11], #0x1\n" - "st1 { v16.b }[6], [x10], #0x1\n" - "st1 { v25.b }[6], [x9], #0x1\n" + "st1 { v7.b }[6], [x10], #0x1\n" + "st1 { v2.b }[6], [x9], #0x1\n" + "st1 { v10.b }[6], [x28], #0x1\n" "b 63f\n" "60:" // Oddments: Bit 2: Bit 1: Unset - "tbz x6, #0, 63f\n" - "st1 { v13.b }[4], [x12], #0x1\n" + "tbz x7, #0, 63f\n" "st1 { v9.b }[4], [x11], #0x1\n" - "st1 { v16.b }[4], [x10], #0x1\n" - "st1 { v25.b }[4], [x9], #0x1\n" + "st1 { v7.b }[4], [x10], #0x1\n" + "st1 { v2.b }[4], [x9], #0x1\n" + "st1 { v10.b }[4], [x28], #0x1\n" "b 63f\n" "61:" // Oddments: Bit 2: Unset - "tbz x6, #1, 62f\n" - "st1 { v13.h }[0], [x12], #0x2\n" + "tbz x7, #1, 62f\n" "st1 { v9.h }[0], [x11], #0x2\n" - "st1 { v16.h }[0], [x10], #0x2\n" - "st1 { v25.h }[0], [x9], #0x2\n" - "tbz x6, #0, 63f\n" - "st1 { v13.b }[2], [x12], #0x1\n" + "st1 { v7.h }[0], [x10], #0x2\n" + "st1 { v2.h }[0], [x9], #0x2\n" + "st1 { v10.h }[0], [x28], #0x2\n" + "tbz x7, #0, 63f\n" "st1 { v9.b }[2], [x11], #0x1\n" - "st1 { v16.b }[2], [x10], #0x1\n" - "st1 { v25.b }[2], [x9], #0x1\n" + "st1 { v7.b }[2], [x10], #0x1\n" + "st1 { v2.b }[2], [x9], #0x1\n" + "st1 { v10.b }[2], [x28], #0x1\n" "b 63f\n" "62:" // Oddments: Bit 2: Unset: Bit 1: Unset - "tbz x6, #0, 63f\n" - "st1 { v13.b }[0], [x12], #0x1\n" + "tbz x7, #0, 63f\n" "st1 { v9.b }[0], [x11], #0x1\n" - "st1 { v16.b }[0], [x10], #0x1\n" - "st1 { v25.b }[0], [x9], #0x1\n" + "st1 { v7.b }[0], [x10], #0x1\n" + "st1 { v2.b }[0], [x9], #0x1\n" + "st1 { v10.b }[0], [x28], #0x1\n" "63:" // Oddments: Bit 2: End "64:" // End : : [offsetof_Params_bias] "I" (offsetof(Params, bias)), [offsetof_Params_inptrs] "I" (offsetof(Params, inptrs)), [offsetof_Params_n_channels] "I" (offsetof(Params, n_channels)), [offsetof_Params_outptrs] "I" (offsetof(Params, outptrs)), [offsetof_Params_requant] "I" (offsetof(Params, requant)), [offsetof_Params_requant_muls] "I" (offsetof(Params, requant_muls)), [offsetof_Params_requant_shifts] "I" (offsetof(Params, requant_shifts)), [offsetof_Params_weights] "I" (offsetof(Params, weights)), [offsetof_Requantize32_a_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [params] "r" (¶ms) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp index 9e80fbfc07406592245549aea032751ae81a5de2..5d6fbac4bd0a2fffb3aa8c19c65334ffa2a5b247 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include @@ -34,15 +34,7 @@ namespace arm_conv { namespace depthwise { -void a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( - const unsigned int, - const uint8_t *const *const, - const int8_t *const, - const int32_t *const, - const arm_gemm::Requantize32 &, - const int32_t *const, - const int32_t *const, - uint8_t *const *const); +void a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl(unsigned int, const uint8_t *const *, const int8_t *, const int32_t *, const arm_gemm::Requantize32 &, const int32_t *, const int32_t *, uint8_t *const *); class a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst : public DepthwiseDepthfirstStrategy { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp index 5124b2c8f3f8ccf41faa0056105417bfa7842c42..2cc802f9e669d5c894c99e0c921718996dcb99b3 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp @@ -104,16 +104,16 @@ void a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ldr x23, [%x[params], %[offsetof_Params_requant]]\n" "lsr x8, x7, #0x3\n" "add x20, x23, %[offsetof_Requantize32_a_offset]\n" - "ld1r { v12.16b }, [x20]\n" + "ld1r { v6.16b }, [x20]\n" "ldr x22, [%x[params], %[offsetof_Params_outptrs]]\n" "add x21, x23, %[offsetof_Requantize32_b_offset]\n" "add x20, x23, %[offsetof_Requantize32_c_offset]\n" - "ld1r { v13.16b }, [x21]\n" - "ld1r { v11.8h }, [x20]\n" + "ld1r { v15.16b }, [x21]\n" + "ld1r { v13.8h }, [x20]\n" "add x21, x23, %[offsetof_Requantize32_minval]\n" "add x20, x23, %[offsetof_Requantize32_maxval]\n" - "ld1r { v16.8h }, [x21]\n" - "ld1r { v14.8h }, [x20]\n" + "ld1r { v17.8h }, [x21]\n" + "ld1r { v24.8h }, [x20]\n" "mov x17, #0x0\n" "mov x16, #0x0\n" "add x15, %x[params], %[offsetof_Params_inptrs]\n" @@ -123,563 +123,563 @@ void a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ldp x11, x10, [x22, #0x0]\n" "ldp x9, x28, [x22, #0x10]\n" "cbz x8, 3f\n" - "ldr d0, [x14, #0x0]\n" - "ldr d1, [x14, #0x8]\n" + "ldr d11, [x14, #0x0]\n" + "ldr d22, [x14, #0x8]\n" "subs x8, x8, #0x1\n" - "ssubl v0.8h, v0.8b, v13.8b\n" - "ldr d2, [x14, #0x10]\n" - "ldr d3, [x14, #0x18]\n" - "ssubl v1.8h, v1.8b, v13.8b\n" - "ssubl v2.8h, v2.8b, v13.8b\n" - "ldr d4, [x14, #0x20]\n" - "ldr d5, [x14, #0x28]\n" - "ssubl v3.8h, v3.8b, v13.8b\n" - "ssubl v4.8h, v4.8b, v13.8b\n" - "ldr d6, [x14, #0x30]\n" + "ssubl v11.8h, v11.8b, v15.8b\n" + "ldr d14, [x14, #0x10]\n" + "ldr d28, [x14, #0x18]\n" + "ssubl v22.8h, v22.8b, v15.8b\n" + "ssubl v14.8h, v14.8b, v15.8b\n" + "ldr d18, [x14, #0x20]\n" + "ldr d9, [x14, #0x28]\n" + "ssubl v28.8h, v28.8b, v15.8b\n" + "ssubl v18.8h, v18.8b, v15.8b\n" + "ldr d26, [x14, #0x30]\n" "ldr d7, [x14, #0x38]\n" - "ssubl v5.8h, v5.8b, v13.8b\n" - "ssubl v6.8h, v6.8b, v13.8b\n" - "ldr d8, [x14, #0x40]\n" - "ldr x24, [%x[params], %[offsetof_Params_bias]]\n" - "ssubl v7.8h, v7.8b, v13.8b\n" - "ssubl v8.8h, v8.8b, v13.8b\n" - "ldr q15, [x24, #0x0]\n" - "ldr q17, [x24, #0x10]\n" - "add x24, x24, #0x20\n" - "str x24, [%x[params], %[offsetof_Params_bias]]\n" + "ssubl v9.8h, v9.8b, v15.8b\n" + "ssubl v26.8h, v26.8b, v15.8b\n" + "ldr d4, [x14, #0x40]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "ssubl v7.8h, v7.8b, v15.8b\n" + "ssubl v4.8h, v4.8b, v15.8b\n" + "ldr q5, [x20, #0x0]\n" + "ldr q3, [x20, #0x10]\n" + "add x20, x20, #0x20\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" "ldp x27, x26, [x15, #0x0]\n" "ldp x25, x24, [x15, #0x10]\n" - "mov v10.16b, v15.16b\n" - "mov v20.16b, v17.16b\n" + "mov v21.16b, v5.16b\n" + "mov v8.16b, v3.16b\n" "ldp x23, x22, [x15, #0x20]\n" "ldp x21, x20, [x15, #0x30]\n" - "mov v9.16b, v15.16b\n" - "mov v23.16b, v17.16b\n" - "ldr d31, [x27, x17]\n" - "ldr d30, [x26, x17]\n" - "mov v21.16b, v15.16b\n" - "mov v22.16b, v17.16b\n" - "ldr d29, [x25, x17]\n" - "ldr d28, [x24, x17]\n" - "usubl v31.8h, v31.8b, v12.8b\n" - "usubl v30.8h, v30.8b, v12.8b\n" - "ldr d27, [x23, x17]\n" - "ldr d26, [x22, x17]\n" - "usubl v29.8h, v29.8b, v12.8b\n" - "usubl v28.8h, v28.8b, v12.8b\n" - "ldr d25, [x21, x17]\n" - "ldr d24, [x20, x17]\n" - "usubl v27.8h, v27.8b, v12.8b\n" - "usubl v26.8h, v26.8b, v12.8b\n" - "usubl v25.8h, v25.8b, v12.8b\n" - "usubl v24.8h, v24.8b, v12.8b\n" + "mov v20.16b, v5.16b\n" + "mov v0.16b, v3.16b\n" + "ldr d25, [x27, x17]\n" + "ldr d27, [x26, x17]\n" + "mov v19.16b, v5.16b\n" + "mov v31.16b, v3.16b\n" + "ldr d1, [x25, x17]\n" + "ldr d2, [x24, x17]\n" + "usubl v25.8h, v25.8b, v6.8b\n" + "usubl v27.8h, v27.8b, v6.8b\n" + "ldr d12, [x23, x17]\n" + "ldr d16, [x22, x17]\n" + "usubl v1.8h, v1.8b, v6.8b\n" + "usubl v2.8h, v2.8b, v6.8b\n" + "ldr d23, [x21, x17]\n" + "ldr d10, [x20, x17]\n" + "usubl v12.8h, v12.8b, v6.8b\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "usubl v23.8h, v23.8b, v6.8b\n" + "usubl v10.8h, v10.8b, v6.8b\n" "beq 2f\n" "1:" // Loop - "smlal v15.4s, v31.4h, v8.4h\n" - "smlal2 v17.4s, v31.8h, v8.8h\n" - "ldr x24, [x15, #0x40]\n" - "ldr x22, [x15, #0x48]\n" - "smlal v10.4s, v31.4h, v6.4h\n" - "smlal2 v20.4s, v31.8h, v6.8h\n" - "ldr x21, [x15, #0x50]\n" - "ldr x20, [x15, #0x58]\n" - "smlal v15.4s, v30.4h, v0.4h\n" - "smlal2 v17.4s, v30.8h, v0.8h\n" - "ldr q19, [x13, #0x0]\n" - "ldr x23, [x15, #0x78]\n" - "smlal v10.4s, v28.4h, v1.4h\n" - "smlal2 v20.4s, v28.8h, v1.8h\n" - "ldr d28, [x22, x17]\n" - "usubl v28.8h, v28.8b, v12.8b\n" - "smlal v15.4s, v29.4h, v1.4h\n" - "smlal2 v17.4s, v29.8h, v1.8h\n" - "ldr d29, [x24, x17]\n" - "usubl v29.8h, v29.8b, v12.8b\n" - "smlal v10.4s, v27.4h, v2.4h\n" - "smlal2 v20.4s, v27.8h, v2.8h\n" + "ldr q30, [x13, #0x0]\n" + "ldr q29, [x12, #0x0]\n" + "smlal v5.4s, v25.4h, v4.4h\n" + "smlal2 v3.4s, v25.8h, v4.8h\n" + "ldr x21, [x15, #0x58]\n" + "ldr x20, [x15, #0x78]\n" + "smlal v5.4s, v27.4h, v11.4h\n" + "smlal v21.4s, v25.4h, v26.4h\n" + "ldr x25, [x15, #0x60]\n" + "ldr x24, [x15, #0x80]\n" + "smlal v20.4s, v25.4h, v14.4h\n" + "smlal v19.4s, v25.4h, v11.4h\n" + "smlal2 v3.4s, v27.8h, v11.8h\n" "ldr d27, [x21, x17]\n" - "usubl v27.8h, v27.8b, v12.8b\n" - "smlal v15.4s, v26.4h, v3.4h\n" - "smlal2 v17.4s, v26.8h, v3.8h\n" - "ldr d26, [x20, x17]\n" - "ldr x20, [x15, #0x60]\n" - "smlal v10.4s, v24.4h, v0.4h\n" - "smlal2 v20.4s, v24.8h, v0.8h\n" - "usubl v26.8h, v26.8b, v12.8b\n" - "ldr x21, [x15, #0x80]\n" - "smlal v15.4s, v25.4h, v4.4h\n" - "smlal2 v17.4s, v25.8h, v4.8h\n" - "ldr d25, [x20, x17]\n" - "ldr x20, [x15, #0x68]\n" - "smlal v10.4s, v29.4h, v4.4h\n" - "smlal2 v20.4s, v29.8h, v4.8h\n" - "ldr d29, [x20, x17]\n" - "usubl v25.8h, v25.8b, v12.8b\n" - "smlal v15.4s, v24.4h, v2.4h\n" - "smlal2 v17.4s, v24.8h, v2.8h\n" - "ldr q18, [x12, #0x0]\n" + "usubl v27.8h, v27.8b, v6.8b\n" + "smlal v5.4s, v1.4h, v22.4h\n" + "smlal2 v8.4s, v25.8h, v26.8h\n" + "smlal2 v0.4s, v25.8h, v14.8h\n" + "ldr x23, [x15, #0x68]\n" "ldr x22, [x15, #0x88]\n" - "smlal v10.4s, v28.4h, v5.4h\n" - "smlal2 v20.4s, v28.8h, v5.8h\n" - "ldr d28, [x21, x17]\n" - "ldr x21, [x15, #0x70]\n" - "smlal v9.4s, v31.4h, v2.4h\n" - "smlal2 v23.4s, v31.8h, v2.8h\n" - "usubl v28.8h, v28.8b, v12.8b\n" + "smlal2 v31.4s, v25.8h, v11.8h\n" + "ldr d25, [x20, x17]\n" + "usubl v25.8h, v25.8b, v6.8b\n" + "smlal v21.4s, v2.4h, v22.4h\n" + "smlal v20.4s, v27.4h, v28.4h\n" + "smlal v19.4s, v25.4h, v18.4h\n" + "ldr x21, [x15, #0x40]\n" + "ldr x20, [x15, #0x70]\n" + "smlal2 v3.4s, v1.8h, v22.8h\n" + "ldr d1, [x25, x17]\n" + "usubl v1.8h, v1.8b, v6.8b\n" + "smlal v5.4s, v16.4h, v28.4h\n" + "smlal2 v8.4s, v2.8h, v22.8h\n" + "ldr d2, [x24, x17]\n" + "usubl v2.8h, v2.8b, v6.8b\n" + "smlal2 v0.4s, v27.8h, v28.8h\n" + "ldr d27, [x23, x17]\n" + "smlal2 v31.4s, v25.8h, v18.8h\n" + "ldr d25, [x22, x17]\n" + "smlal v21.4s, v12.4h, v14.4h\n" "ldr x25, [x15, #0x98]\n" - "smlal v15.4s, v27.4h, v5.4h\n" - "smlal2 v17.4s, v27.8h, v5.8h\n" - "usubl v29.8h, v29.8b, v12.8b\n" - "ldr x24, [x15, #0x90]\n" - "smlal v10.4s, v27.4h, v3.4h\n" - "smlal2 v20.4s, v27.8h, v3.8h\n" + "smlal v20.4s, v1.4h, v11.4h\n" + "smlal v19.4s, v2.4h, v22.4h\n" + "ldr x24, [x15, #0x50]\n" + "smlal2 v3.4s, v16.8h, v28.8h\n" + "ldr d16, [x21, x17]\n" + "usubl v27.8h, v27.8b, v6.8b\n" + "smlal v5.4s, v23.4h, v18.4h\n" + "usubl v25.8h, v25.8b, v6.8b\n" + "smlal2 v8.4s, v12.8h, v14.8h\n" + "ldr d12, [x20, x17]\n" + "ldr x23, [x15, #0x48]\n" + "smlal2 v0.4s, v1.8h, v11.8h\n" + "smlal2 v31.4s, v2.8h, v22.8h\n" + "ldr x21, [x15, #0x90]\n" + "ldr x20, [x15, #0xa8]\n" + "smlal v21.4s, v10.4h, v11.4h\n" + "smlal v20.4s, v27.4h, v18.4h\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "ldr x22, [x15, #0xa0]\n" + "smlal v19.4s, v25.4h, v9.4h\n" + "smlal2 v3.4s, v23.8h, v18.8h\n" + "ldr d23, [x25, x17]\n" + "usubl v12.8h, v12.8b, v6.8b\n" + "usubl v23.8h, v23.8b, v6.8b\n" + "smlal v5.4s, v10.4h, v14.4h\n" + "smlal2 v8.4s, v10.8h, v11.8h\n" + "ldr d11, [x24, x17]\n" + "usubl v11.8h, v11.8b, v6.8b\n" + "smlal2 v0.4s, v27.8h, v18.8h\n" "ldr d27, [x23, x17]\n" - "usubl v27.8h, v27.8b, v12.8b\n" - "smlal v21.4s, v31.4h, v0.4h\n" - "smlal v9.4s, v26.4h, v3.4h\n" - "ldr x23, [x15, #0xa8]\n" - "ldr x20, [x15, #0xa0]\n" - "smlal2 v23.4s, v26.8h, v3.8h\n" - "ldr d26, [x22, x17]\n" - "smlal2 v22.4s, v31.8h, v0.8h\n" - "ldr d24, [x21, x17]\n" - "smlal v21.4s, v27.4h, v4.4h\n" - "smlal v9.4s, v25.4h, v0.4h\n" - "usubl v26.8h, v26.8b, v12.8b\n" - "ldr x22, [x15, #0xb0]\n" - "smlal2 v23.4s, v25.8h, v0.8h\n" - "ldr q30, [x13, #0x10]\n" - "smlal2 v22.4s, v27.8h, v4.8h\n" - "ldr d27, [x20, x17]\n" - "smlal v21.4s, v28.4h, v1.4h\n" - "smlal v15.4s, v25.4h, v6.4h\n" - "usubl v24.8h, v24.8b, v12.8b\n" - "ldr x21, [x15, #0xb8]\n" - "smlal2 v17.4s, v25.8h, v6.8h\n" - "ldr d25, [x24, x17]\n" - "smlal v9.4s, v29.4h, v4.4h\n" - "usubl v25.8h, v25.8b, v12.8b\n" - "smlal2 v23.4s, v29.8h, v4.8h\n" - "ldr d29, [x25, x17]\n" - "ldr q31, [x12, #0x10]\n" - "smlal2 v22.4s, v28.8h, v1.8h\n" - "smlal v21.4s, v26.4h, v5.4h\n" - "usubl v29.8h, v29.8b, v12.8b\n" - "smlal v15.4s, v24.4h, v7.4h\n" - "ldr x20, [x15, #0xc0]\n" - "smlal2 v17.4s, v24.8h, v7.8h\n" - "smlal v9.4s, v24.4h, v1.4h\n" - "usubl v27.8h, v27.8b, v12.8b\n" - "ldr x24, [%x[params], %[offsetof_Params_bias]]\n" - "smlal2 v23.4s, v24.8h, v1.8h\n" - "ldr d24, [x23, x17]\n" - "smlal2 v22.4s, v26.8h, v5.8h\n" - "ldr d26, [x22, x17]\n" - "smlal v21.4s, v29.4h, v2.4h\n" - "usubl v24.8h, v24.8b, v12.8b\n" - "smlal2 v22.4s, v29.8h, v2.8h\n" + "smlal2 v31.4s, v25.8h, v9.8h\n" + "ldr d25, [x21, x17]\n" + "ldr x21, [x15, #0xb0]\n" + "smlal v21.4s, v16.4h, v18.4h\n" + "smlal v20.4s, v12.4h, v22.4h\n" + "smlal v19.4s, v23.4h, v14.4h\n" + "smlal2 v3.4s, v10.8h, v14.8h\n" + "ldr d10, [x20, x17]\n" + "usubl v27.8h, v27.8b, v6.8b\n" + "usubl v25.8h, v25.8b, v6.8b\n" + "usubl v10.8h, v10.8b, v6.8b\n" + "smlal v5.4s, v11.4h, v9.4h\n" + "ldr x20, [x15, #0xb8]\n" + "smlal2 v8.4s, v16.8h, v18.8h\n" + "ldr d18, [x22, x17]\n" + "ldr d16, [x21, x17]\n" + "smlal2 v0.4s, v12.8h, v22.8h\n" + "ldr d22, [x20, x17]\n" + "smlal2 v31.4s, v23.8h, v14.8h\n" + "ldr q14, [x13, #0x10]\n" + "smlal v21.4s, v27.4h, v9.4h\n" + "smlal v20.4s, v25.4h, v26.4h\n" + "smlal v19.4s, v10.4h, v28.4h\n" + "usubl v18.8h, v18.8b, v6.8b\n" + "ldr x21, [x15, #0xc0]\n" + "smlal2 v3.4s, v11.8h, v9.8h\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "smlal v5.4s, v1.4h, v26.4h\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "smlal2 v8.4s, v27.8h, v9.8h\n" + "ldr d27, [x21, x17]\n" + "smlal2 v0.4s, v25.8h, v26.8h\n" + "ldr q25, [x12, #0x10]\n" + "smlal2 v31.4s, v10.8h, v28.8h\n" + "smlal v21.4s, v11.4h, v28.4h\n" + "usubl v22.8h, v22.8b, v6.8b\n" "add x14, x14, #0x48\n" - "smlal v9.4s, v25.4h, v6.4h\n" - "smlal v21.4s, v24.4h, v3.4h\n" - "usubl v26.8h, v26.8b, v12.8b\n" + "smlal v20.4s, v18.4h, v7.4h\n" + "smlal v19.4s, v16.4h, v7.4h\n" + "usubl v27.8h, v27.8b, v6.8b\n" + "add x17, x17, #0x8\n" + "smlal2 v3.4s, v1.8h, v26.8h\n" + "smlal v5.4s, v12.4h, v7.4h\n" + "sqrdmulh v5.4s, v5.4s, v30.4s\n" "subs x8, x8, #0x1\n" - "smlal v10.4s, v28.4h, v7.4h\n" - "smlal2 v20.4s, v28.8h, v7.8h\n" - "sqrdmulh v15.4s, v15.4s, v19.4s\n" + "smlal2 v8.4s, v11.8h, v28.8h\n" + "smlal2 v0.4s, v18.8h, v7.8h\n" + "and v28.16b, v5.16b, v29.16b\n" "add x13, x13, #0x20\n" - "smlal2 v23.4s, v25.8h, v6.8h\n" - "ldr d25, [x21, x17]\n" - "smlal2 v22.4s, v24.8h, v3.8h\n" - "usubl v25.8h, v25.8b, v12.8b\n" - "smlal v9.4s, v27.4h, v7.4h\n" - "smlal v21.4s, v26.4h, v7.4h\n" - "and v0.16b, v15.16b, v18.16b\n" + "smlal2 v31.4s, v16.8h, v7.8h\n" + "smlal v21.4s, v2.4h, v7.4h\n" + "sshr v28.4s, v28.4s, #0x1f\n" "add x12, x12, #0x20\n" - "smlal v10.4s, v29.4h, v8.4h\n" - "smlal2 v20.4s, v29.8h, v8.8h\n" - "ldr d29, [x20, x17]\n" - "usubl v29.8h, v29.8b, v12.8b\n" - "smlal2 v23.4s, v27.8h, v7.8h\n" - "smlal2 v22.4s, v26.8h, v7.8h\n" - "sqrdmulh v17.4s, v17.4s, v30.4s\n" - "add x17, x17, #0x8\n" - "smlal v9.4s, v24.4h, v5.4h\n" - "smlal v21.4s, v25.4h, v6.4h\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "smlal2 v23.4s, v24.8h, v5.8h\n" - "smlal2 v22.4s, v25.8h, v6.8h\n" - "and v7.16b, v17.16b, v31.16b\n" - "smlal v9.4s, v25.4h, v8.4h\n" - "smlal v21.4s, v29.4h, v8.4h\n" - "sqrdmulh v10.4s, v10.4s, v19.4s\n" - "smlal2 v23.4s, v25.8h, v8.8h\n" - "smlal2 v22.4s, v29.8h, v8.8h\n" - "sqrdmulh v9.4s, v9.4s, v19.4s\n" - "sqrdmulh v21.4s, v21.4s, v19.4s\n" - "sqadd v15.4s, v15.4s, v0.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "and v19.16b, v10.16b, v18.16b\n" + "smlal v20.4s, v10.4h, v9.4h\n" + "smlal v19.4s, v22.4h, v26.4h\n" + "sqadd v5.4s, v5.4s, v28.4s\n" + "smlal2 v3.4s, v12.8h, v7.8h\n" + "smlal2 v8.4s, v2.8h, v7.8h\n" + "sqrdmulh v3.4s, v3.4s, v14.4s\n" + "smlal2 v0.4s, v10.8h, v9.8h\n" + "smlal2 v31.4s, v22.8h, v26.8h\n" + "and v16.16b, v3.16b, v25.16b\n" + "smlal v21.4s, v23.4h, v4.4h\n" + "smlal v20.4s, v22.4h, v4.4h\n" + "sqrdmulh v21.4s, v21.4s, v30.4s\n" + "smlal v19.4s, v27.4h, v4.4h\n" + "smlal2 v8.4s, v23.8h, v4.8h\n" "sqrdmulh v20.4s, v20.4s, v30.4s\n" - "and v27.16b, v9.16b, v18.16b\n" - "sqrdmulh v23.4s, v23.4s, v30.4s\n" - "and v0.16b, v21.16b, v18.16b\n" - "sqrdmulh v22.4s, v22.4s, v30.4s\n" - "sqadd v17.4s, v17.4s, v7.4s\n" - "sshr v19.4s, v19.4s, #0x1f\n" - "and v5.16b, v20.16b, v31.16b\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "and v4.16b, v23.16b, v31.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v7.16b, v22.16b, v31.16b\n" - "sqadd v10.4s, v10.4s, v19.4s\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v9.4s, v9.4s, v27.4s\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v21.4s, v21.4s, v0.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "srshl v15.4s, v15.4s, v18.4s\n" - "srshl v10.4s, v10.4s, v18.4s\n" - "sqadd v20.4s, v20.4s, v5.4s\n" - "srshl v9.4s, v9.4s, v18.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "srshl v21.4s, v21.4s, v18.4s\n" - "sqadd v22.4s, v22.4s, v7.4s\n" - "srshl v17.4s, v17.4s, v31.4s\n" - "sqxtn v15.4h, v15.4s\n" - "srshl v20.4s, v20.4s, v31.4s\n" - "sqxtn v10.4h, v10.4s\n" - "srshl v23.4s, v23.4s, v31.4s\n" - "sqxtn v9.4h, v9.4s\n" - "srshl v22.4s, v22.4s, v31.4s\n" + "smlal2 v0.4s, v22.8h, v4.8h\n" + "smlal2 v31.4s, v27.8h, v4.8h\n" + "sqrdmulh v19.4s, v19.4s, v30.4s\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "and v12.16b, v21.16b, v29.16b\n" + "sqrdmulh v8.4s, v8.4s, v14.4s\n" + "and v23.16b, v20.16b, v29.16b\n" + "sqrdmulh v0.4s, v0.4s, v14.4s\n" + "and v9.16b, v19.16b, v29.16b\n" + "sqrdmulh v31.4s, v31.4s, v14.4s\n" + "sqadd v3.4s, v3.4s, v16.4s\n" + "sshr v12.4s, v12.4s, #0x1f\n" + "and v18.16b, v8.16b, v25.16b\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "and v22.16b, v0.16b, v25.16b\n" + "sshr v9.4s, v9.4s, #0x1f\n" + "and v16.16b, v31.16b, v25.16b\n" + "sqadd v21.4s, v21.4s, v12.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v23.4s\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "sqadd v19.4s, v19.4s, v9.4s\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "srshl v5.4s, v5.4s, v29.4s\n" + "srshl v21.4s, v21.4s, v29.4s\n" + "sqadd v8.4s, v8.4s, v18.4s\n" + "srshl v20.4s, v20.4s, v29.4s\n" + "sqadd v0.4s, v0.4s, v22.4s\n" + "srshl v19.4s, v19.4s, v29.4s\n" + "sqadd v31.4s, v31.4s, v16.4s\n" + "srshl v3.4s, v3.4s, v25.4s\n" + "sqxtn v5.4h, v5.4s\n" + "srshl v8.4s, v8.4s, v25.4s\n" "sqxtn v21.4h, v21.4s\n" - "sqxtn2 v15.8h, v17.4s\n" - "sqxtn2 v10.8h, v20.4s\n" - "sqxtn2 v9.8h, v23.4s\n" - "sqxtn2 v21.8h, v22.4s\n" - "sqadd v15.8h, v15.8h, v11.8h\n" - "sqadd v10.8h, v10.8h, v11.8h\n" - "sqadd v9.8h, v9.8h, v11.8h\n" - "sqadd v21.8h, v21.8h, v11.8h\n" - "smax v15.8h, v15.8h, v16.8h\n" - "smax v10.8h, v10.8h, v16.8h\n" - "smax v9.8h, v9.8h, v16.8h\n" - "smax v21.8h, v21.8h, v16.8h\n" - "smin v15.8h, v15.8h, v14.8h\n" - "smin v10.8h, v10.8h, v14.8h\n" - "smin v9.8h, v9.8h, v14.8h\n" - "smin v21.8h, v21.8h, v14.8h\n" - "uzp1 v15.16b, v15.16b, v15.16b\n" - "str d15, [x11, x16]\n" - "uzp1 v10.16b, v10.16b, v10.16b\n" - "uzp1 v9.16b, v9.16b, v9.16b\n" - "str d10, [x10, x16]\n" + "srshl v0.4s, v0.4s, v25.4s\n" + "sqxtn v20.4h, v20.4s\n" + "srshl v31.4s, v31.4s, v25.4s\n" + "sqxtn v19.4h, v19.4s\n" + "sqxtn2 v5.8h, v3.4s\n" + "sqxtn2 v21.8h, v8.4s\n" + "sqxtn2 v20.8h, v0.4s\n" + "sqxtn2 v19.8h, v31.4s\n" + "sqadd v5.8h, v5.8h, v13.8h\n" + "sqadd v21.8h, v21.8h, v13.8h\n" + "sqadd v20.8h, v20.8h, v13.8h\n" + "sqadd v19.8h, v19.8h, v13.8h\n" + "smax v5.8h, v5.8h, v17.8h\n" + "smax v21.8h, v21.8h, v17.8h\n" + "smax v20.8h, v20.8h, v17.8h\n" + "smax v19.8h, v19.8h, v17.8h\n" + "smin v5.8h, v5.8h, v24.8h\n" + "smin v21.8h, v21.8h, v24.8h\n" + "smin v20.8h, v20.8h, v24.8h\n" + "smin v19.8h, v19.8h, v24.8h\n" + "uzp1 v5.16b, v5.16b, v5.16b\n" + "str d5, [x11, x16]\n" "uzp1 v21.16b, v21.16b, v21.16b\n" - "str d9, [x9, x16]\n" - "str d21, [x28, x16]\n" - "ldr q15, [x24, #0x0]\n" - "ldr q17, [x24, #0x10]\n" - "add x24, x24, #0x20\n" - "ldr d0, [x14, #0x0]\n" - "ldr d1, [x14, #0x8]\n" + "uzp1 v20.16b, v20.16b, v20.16b\n" + "str d21, [x10, x16]\n" + "uzp1 v19.16b, v19.16b, v19.16b\n" + "str d20, [x9, x16]\n" + "str d19, [x28, x16]\n" + "ldr q5, [x20, #0x0]\n" + "ldr q3, [x20, #0x10]\n" + "add x20, x20, #0x20\n" + "ldr d11, [x14, #0x0]\n" + "ldr d22, [x14, #0x8]\n" "add x16, x16, #0x8\n" - "str x24, [%x[params], %[offsetof_Params_bias]]\n" - "ldr d2, [x14, #0x10]\n" - "ldr d3, [x14, #0x18]\n" - "mov v10.16b, v15.16b\n" - "mov v20.16b, v17.16b\n" - "ldr d4, [x14, #0x20]\n" - "ldr d5, [x14, #0x28]\n" - "mov v9.16b, v15.16b\n" - "mov v23.16b, v17.16b\n" - "ldr d6, [x14, #0x30]\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" + "ldr d14, [x14, #0x10]\n" + "ldr d28, [x14, #0x18]\n" + "mov v21.16b, v5.16b\n" + "mov v8.16b, v3.16b\n" + "ldr d18, [x14, #0x20]\n" + "ldr d9, [x14, #0x28]\n" + "mov v20.16b, v5.16b\n" + "mov v0.16b, v3.16b\n" + "ldr d26, [x14, #0x30]\n" "ldr d7, [x14, #0x38]\n" - "mov v21.16b, v15.16b\n" - "mov v22.16b, v17.16b\n" - "ldr d8, [x14, #0x40]\n" + "mov v19.16b, v5.16b\n" + "mov v31.16b, v3.16b\n" + "ldr d4, [x14, #0x40]\n" "ldp x27, x26, [x15, #0x0]\n" - "ssubl v0.8h, v0.8b, v13.8b\n" - "ssubl v1.8h, v1.8b, v13.8b\n" + "ssubl v11.8h, v11.8b, v15.8b\n" + "ssubl v22.8h, v22.8b, v15.8b\n" "ldp x25, x24, [x15, #0x10]\n" "ldp x23, x22, [x15, #0x20]\n" - "ssubl v2.8h, v2.8b, v13.8b\n" - "ssubl v3.8h, v3.8b, v13.8b\n" + "ssubl v14.8h, v14.8b, v15.8b\n" + "ssubl v28.8h, v28.8b, v15.8b\n" "ldp x21, x20, [x15, #0x30]\n" - "ldr d31, [x27, x17]\n" - "ssubl v4.8h, v4.8b, v13.8b\n" - "ssubl v5.8h, v5.8b, v13.8b\n" - "ldr d30, [x26, x17]\n" - "ldr d29, [x25, x17]\n" - "ssubl v6.8h, v6.8b, v13.8b\n" - "ssubl v7.8h, v7.8b, v13.8b\n" - "ldr d28, [x24, x17]\n" - "ldr d27, [x23, x17]\n" - "ssubl v8.8h, v8.8b, v13.8b\n" - "usubl v31.8h, v31.8b, v12.8b\n" - "ldr d26, [x22, x17]\n" - "ldr d25, [x21, x17]\n" - "usubl v30.8h, v30.8b, v12.8b\n" - "usubl v29.8h, v29.8b, v12.8b\n" - "ldr d24, [x20, x17]\n" - "usubl v28.8h, v28.8b, v12.8b\n" - "usubl v27.8h, v27.8b, v12.8b\n" - "usubl v26.8h, v26.8b, v12.8b\n" - "usubl v25.8h, v25.8b, v12.8b\n" - "usubl v24.8h, v24.8b, v12.8b\n" + "ldr d25, [x27, x17]\n" + "ssubl v18.8h, v18.8b, v15.8b\n" + "ssubl v9.8h, v9.8b, v15.8b\n" + "ldr d27, [x26, x17]\n" + "ldr d1, [x25, x17]\n" + "ssubl v26.8h, v26.8b, v15.8b\n" + "ssubl v7.8h, v7.8b, v15.8b\n" + "ldr d2, [x24, x17]\n" + "ldr d12, [x23, x17]\n" + "ssubl v4.8h, v4.8b, v15.8b\n" + "usubl v25.8h, v25.8b, v6.8b\n" + "ldr d16, [x22, x17]\n" + "ldr d23, [x21, x17]\n" + "usubl v27.8h, v27.8b, v6.8b\n" + "usubl v1.8h, v1.8b, v6.8b\n" + "ldr d10, [x20, x17]\n" + "usubl v2.8h, v2.8b, v6.8b\n" + "usubl v12.8h, v12.8b, v6.8b\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "usubl v23.8h, v23.8b, v6.8b\n" + "usubl v10.8h, v10.8b, v6.8b\n" "bgt 1b\n" "2:" // Tail - "smlal v15.4s, v31.4h, v8.4h\n" - "smlal2 v17.4s, v31.8h, v8.8h\n" - "ldr x24, [x15, #0x40]\n" - "ldr x22, [x15, #0x48]\n" - "smlal v10.4s, v31.4h, v6.4h\n" - "smlal2 v20.4s, v31.8h, v6.8h\n" - "ldr x21, [x15, #0x50]\n" - "ldr x20, [x15, #0x58]\n" - "smlal v15.4s, v30.4h, v0.4h\n" - "smlal2 v17.4s, v30.8h, v0.8h\n" - "ldr q19, [x13, #0x0]\n" - "ldr x23, [x15, #0x78]\n" - "smlal v10.4s, v28.4h, v1.4h\n" - "smlal2 v20.4s, v28.8h, v1.8h\n" - "ldr d28, [x22, x17]\n" - "usubl v28.8h, v28.8b, v12.8b\n" - "smlal v15.4s, v29.4h, v1.4h\n" - "smlal2 v17.4s, v29.8h, v1.8h\n" - "ldr d29, [x24, x17]\n" - "usubl v29.8h, v29.8b, v12.8b\n" - "smlal v10.4s, v27.4h, v2.4h\n" - "smlal2 v20.4s, v27.8h, v2.8h\n" + "ldr q29, [x13, #0x0]\n" + "ldr q30, [x12, #0x0]\n" + "smlal v5.4s, v25.4h, v4.4h\n" + "smlal2 v3.4s, v25.8h, v4.8h\n" + "ldr x21, [x15, #0x58]\n" + "ldr x20, [x15, #0x78]\n" + "smlal v5.4s, v27.4h, v11.4h\n" + "smlal v21.4s, v25.4h, v26.4h\n" + "ldr x25, [x15, #0x60]\n" + "ldr x24, [x15, #0x80]\n" + "smlal v20.4s, v25.4h, v14.4h\n" + "smlal v19.4s, v25.4h, v11.4h\n" + "smlal2 v3.4s, v27.8h, v11.8h\n" "ldr d27, [x21, x17]\n" - "usubl v27.8h, v27.8b, v12.8b\n" - "smlal v15.4s, v26.4h, v3.4h\n" - "smlal2 v17.4s, v26.8h, v3.8h\n" - "ldr d26, [x20, x17]\n" - "ldr x20, [x15, #0x60]\n" - "smlal v10.4s, v24.4h, v0.4h\n" - "smlal2 v20.4s, v24.8h, v0.8h\n" - "usubl v26.8h, v26.8b, v12.8b\n" - "ldr x21, [x15, #0x80]\n" - "smlal v15.4s, v25.4h, v4.4h\n" - "smlal2 v17.4s, v25.8h, v4.8h\n" - "ldr d25, [x20, x17]\n" - "ldr x20, [x15, #0x68]\n" - "smlal v10.4s, v29.4h, v4.4h\n" - "smlal2 v20.4s, v29.8h, v4.8h\n" - "ldr d29, [x20, x17]\n" - "usubl v25.8h, v25.8b, v12.8b\n" - "smlal v15.4s, v24.4h, v2.4h\n" - "smlal2 v17.4s, v24.8h, v2.8h\n" - "ldr q18, [x12, #0x0]\n" + "usubl v27.8h, v27.8b, v6.8b\n" + "smlal v5.4s, v1.4h, v22.4h\n" + "smlal2 v8.4s, v25.8h, v26.8h\n" + "smlal2 v0.4s, v25.8h, v14.8h\n" + "ldr x23, [x15, #0x68]\n" "ldr x22, [x15, #0x88]\n" - "smlal v10.4s, v28.4h, v5.4h\n" - "smlal2 v20.4s, v28.8h, v5.8h\n" - "ldr d28, [x21, x17]\n" - "ldr x21, [x15, #0x70]\n" - "smlal v9.4s, v31.4h, v2.4h\n" - "smlal2 v23.4s, v31.8h, v2.8h\n" - "usubl v28.8h, v28.8b, v12.8b\n" + "smlal2 v31.4s, v25.8h, v11.8h\n" + "ldr d25, [x20, x17]\n" + "usubl v25.8h, v25.8b, v6.8b\n" + "smlal v21.4s, v2.4h, v22.4h\n" + "smlal v20.4s, v27.4h, v28.4h\n" + "smlal v19.4s, v25.4h, v18.4h\n" + "ldr x21, [x15, #0x40]\n" + "ldr x20, [x15, #0x70]\n" + "smlal2 v3.4s, v1.8h, v22.8h\n" + "ldr d1, [x25, x17]\n" + "usubl v1.8h, v1.8b, v6.8b\n" + "smlal v5.4s, v16.4h, v28.4h\n" + "smlal2 v8.4s, v2.8h, v22.8h\n" + "ldr d2, [x24, x17]\n" + "usubl v2.8h, v2.8b, v6.8b\n" + "smlal2 v0.4s, v27.8h, v28.8h\n" + "ldr d27, [x23, x17]\n" + "smlal2 v31.4s, v25.8h, v18.8h\n" + "ldr d25, [x22, x17]\n" + "smlal v21.4s, v12.4h, v14.4h\n" "ldr x25, [x15, #0x98]\n" - "smlal v15.4s, v27.4h, v5.4h\n" - "smlal2 v17.4s, v27.8h, v5.8h\n" - "usubl v29.8h, v29.8b, v12.8b\n" - "ldr x24, [x15, #0x90]\n" - "smlal v10.4s, v27.4h, v3.4h\n" - "smlal2 v20.4s, v27.8h, v3.8h\n" + "smlal v20.4s, v1.4h, v11.4h\n" + "smlal v19.4s, v2.4h, v22.4h\n" + "ldr x24, [x15, #0x50]\n" + "smlal2 v3.4s, v16.8h, v28.8h\n" + "ldr d16, [x21, x17]\n" + "usubl v27.8h, v27.8b, v6.8b\n" + "smlal v5.4s, v23.4h, v18.4h\n" + "usubl v25.8h, v25.8b, v6.8b\n" + "smlal2 v8.4s, v12.8h, v14.8h\n" + "ldr d12, [x20, x17]\n" + "ldr x23, [x15, #0x48]\n" + "smlal2 v0.4s, v1.8h, v11.8h\n" + "smlal2 v31.4s, v2.8h, v22.8h\n" + "ldr x21, [x15, #0x90]\n" + "ldr x20, [x15, #0xa8]\n" + "smlal v21.4s, v10.4h, v11.4h\n" + "smlal v20.4s, v27.4h, v18.4h\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "ldr x22, [x15, #0xa0]\n" + "smlal v19.4s, v25.4h, v9.4h\n" + "smlal2 v3.4s, v23.8h, v18.8h\n" + "ldr d23, [x25, x17]\n" + "usubl v12.8h, v12.8b, v6.8b\n" + "usubl v23.8h, v23.8b, v6.8b\n" + "smlal v5.4s, v10.4h, v14.4h\n" + "smlal2 v8.4s, v10.8h, v11.8h\n" + "ldr d11, [x24, x17]\n" + "usubl v11.8h, v11.8b, v6.8b\n" + "smlal2 v0.4s, v27.8h, v18.8h\n" "ldr d27, [x23, x17]\n" - "usubl v27.8h, v27.8b, v12.8b\n" - "smlal v21.4s, v31.4h, v0.4h\n" - "smlal v9.4s, v26.4h, v3.4h\n" - "ldr x23, [x15, #0xa8]\n" - "ldr x20, [x15, #0xa0]\n" - "smlal2 v23.4s, v26.8h, v3.8h\n" - "ldr d26, [x22, x17]\n" - "smlal2 v22.4s, v31.8h, v0.8h\n" - "ldr d24, [x21, x17]\n" - "smlal v21.4s, v27.4h, v4.4h\n" - "smlal v9.4s, v25.4h, v0.4h\n" - "usubl v26.8h, v26.8b, v12.8b\n" - "ldr x22, [x15, #0xb0]\n" - "smlal2 v23.4s, v25.8h, v0.8h\n" - "ldr q30, [x13, #0x10]\n" - "smlal2 v22.4s, v27.8h, v4.8h\n" - "ldr d27, [x20, x17]\n" - "smlal v21.4s, v28.4h, v1.4h\n" - "smlal v15.4s, v25.4h, v6.4h\n" - "usubl v24.8h, v24.8b, v12.8b\n" - "ldr x21, [x15, #0xb8]\n" - "smlal2 v17.4s, v25.8h, v6.8h\n" - "ldr d25, [x24, x17]\n" - "smlal v9.4s, v29.4h, v4.4h\n" - "usubl v25.8h, v25.8b, v12.8b\n" - "smlal2 v23.4s, v29.8h, v4.8h\n" - "ldr d29, [x25, x17]\n" - "ldr q31, [x12, #0x10]\n" - "smlal2 v22.4s, v28.8h, v1.8h\n" - "smlal v21.4s, v26.4h, v5.4h\n" - "usubl v29.8h, v29.8b, v12.8b\n" - "smlal v15.4s, v24.4h, v7.4h\n" + "smlal2 v31.4s, v25.8h, v9.8h\n" + "ldr d25, [x21, x17]\n" + "ldr x21, [x15, #0xb0]\n" + "smlal v21.4s, v16.4h, v18.4h\n" + "smlal v20.4s, v12.4h, v22.4h\n" + "smlal v19.4s, v23.4h, v14.4h\n" + "smlal2 v3.4s, v10.8h, v14.8h\n" + "ldr d10, [x20, x17]\n" + "usubl v27.8h, v27.8b, v6.8b\n" + "usubl v25.8h, v25.8b, v6.8b\n" + "usubl v10.8h, v10.8b, v6.8b\n" + "smlal v5.4s, v11.4h, v9.4h\n" + "ldr x20, [x15, #0xb8]\n" + "smlal2 v8.4s, v16.8h, v18.8h\n" + "ldr d16, [x22, x17]\n" + "ldr d18, [x21, x17]\n" + "smlal2 v0.4s, v12.8h, v22.8h\n" + "ldr d22, [x20, x17]\n" + "smlal2 v31.4s, v23.8h, v14.8h\n" + "ldr q14, [x13, #0x10]\n" + "smlal v21.4s, v27.4h, v9.4h\n" + "smlal v20.4s, v25.4h, v26.4h\n" + "smlal v19.4s, v10.4h, v28.4h\n" + "usubl v16.8h, v16.8b, v6.8b\n" "ldr x20, [x15, #0xc0]\n" - "smlal2 v17.4s, v24.8h, v7.8h\n" - "smlal v9.4s, v24.4h, v1.4h\n" - "usubl v27.8h, v27.8b, v12.8b\n" + "smlal2 v3.4s, v11.8h, v9.8h\n" + "usubl v18.8h, v18.8b, v6.8b\n" + "smlal v5.4s, v1.4h, v26.4h\n" "tst x7, #0x7\n" - "smlal2 v23.4s, v24.8h, v1.8h\n" - "ldr d24, [x23, x17]\n" - "smlal2 v22.4s, v26.8h, v5.8h\n" - "ldr d26, [x22, x17]\n" - "smlal v21.4s, v29.4h, v2.4h\n" - "usubl v24.8h, v24.8b, v12.8b\n" - "smlal2 v22.4s, v29.8h, v2.8h\n" + "smlal2 v8.4s, v27.8h, v9.8h\n" + "ldr d27, [x20, x17]\n" + "smlal2 v0.4s, v25.8h, v26.8h\n" + "ldr q25, [x12, #0x10]\n" + "smlal2 v31.4s, v10.8h, v28.8h\n" + "smlal v21.4s, v11.4h, v28.4h\n" + "usubl v22.8h, v22.8b, v6.8b\n" + "add x17, x17, #0x8\n" + "smlal v20.4s, v16.4h, v7.4h\n" + "smlal v19.4s, v18.4h, v7.4h\n" + "usubl v27.8h, v27.8b, v6.8b\n" "add x13, x13, #0x20\n" - "smlal v9.4s, v25.4h, v6.4h\n" - "smlal v21.4s, v24.4h, v3.4h\n" - "usubl v26.8h, v26.8b, v12.8b\n" + "smlal2 v3.4s, v1.8h, v26.8h\n" + "smlal v5.4s, v12.4h, v7.4h\n" + "sqrdmulh v5.4s, v5.4s, v29.4s\n" "add x12, x12, #0x20\n" - "smlal v10.4s, v28.4h, v7.4h\n" - "smlal2 v20.4s, v28.8h, v7.8h\n" - "sqrdmulh v15.4s, v15.4s, v19.4s\n" - "smlal2 v23.4s, v25.8h, v6.8h\n" - "ldr d25, [x21, x17]\n" - "smlal2 v22.4s, v24.8h, v3.8h\n" - "usubl v25.8h, v25.8b, v12.8b\n" - "smlal v9.4s, v27.4h, v7.4h\n" - "smlal v21.4s, v26.4h, v7.4h\n" - "and v0.16b, v15.16b, v18.16b\n" - "smlal v10.4s, v29.4h, v8.4h\n" - "smlal2 v20.4s, v29.8h, v8.8h\n" - "ldr d29, [x20, x17]\n" - "usubl v29.8h, v29.8b, v12.8b\n" - "smlal2 v23.4s, v27.8h, v7.8h\n" - "smlal2 v22.4s, v26.8h, v7.8h\n" - "sqrdmulh v17.4s, v17.4s, v30.4s\n" - "add x17, x17, #0x8\n" - "smlal v9.4s, v24.4h, v5.4h\n" - "smlal v21.4s, v25.4h, v6.4h\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "smlal2 v23.4s, v24.8h, v5.8h\n" - "smlal2 v22.4s, v25.8h, v6.8h\n" - "and v7.16b, v17.16b, v31.16b\n" - "smlal v9.4s, v25.4h, v8.4h\n" - "smlal v21.4s, v29.4h, v8.4h\n" - "sqrdmulh v10.4s, v10.4s, v19.4s\n" - "smlal2 v23.4s, v25.8h, v8.8h\n" - "smlal2 v22.4s, v29.8h, v8.8h\n" - "sqrdmulh v9.4s, v9.4s, v19.4s\n" - "sqrdmulh v21.4s, v21.4s, v19.4s\n" - "sqadd v15.4s, v15.4s, v0.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "and v19.16b, v10.16b, v18.16b\n" - "sqrdmulh v20.4s, v20.4s, v30.4s\n" - "and v27.16b, v9.16b, v18.16b\n" - "sqrdmulh v23.4s, v23.4s, v30.4s\n" - "and v0.16b, v21.16b, v18.16b\n" - "sqrdmulh v22.4s, v22.4s, v30.4s\n" - "sqadd v17.4s, v17.4s, v7.4s\n" - "sshr v19.4s, v19.4s, #0x1f\n" - "and v5.16b, v20.16b, v31.16b\n" + "smlal2 v8.4s, v11.8h, v28.8h\n" + "smlal2 v0.4s, v16.8h, v7.8h\n" + "and v16.16b, v5.16b, v30.16b\n" + "smlal2 v31.4s, v18.8h, v7.8h\n" + "smlal v21.4s, v2.4h, v7.4h\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "smlal v20.4s, v10.4h, v9.4h\n" + "smlal v19.4s, v22.4h, v26.4h\n" + "sqadd v5.4s, v5.4s, v16.4s\n" + "smlal2 v3.4s, v12.8h, v7.8h\n" + "smlal2 v8.4s, v2.8h, v7.8h\n" + "sqrdmulh v3.4s, v3.4s, v14.4s\n" + "smlal2 v0.4s, v10.8h, v9.8h\n" + "smlal2 v31.4s, v22.8h, v26.8h\n" + "and v16.16b, v3.16b, v25.16b\n" + "smlal v21.4s, v23.4h, v4.4h\n" + "smlal v20.4s, v22.4h, v4.4h\n" + "sqrdmulh v21.4s, v21.4s, v29.4s\n" + "smlal v19.4s, v27.4h, v4.4h\n" + "smlal2 v8.4s, v23.8h, v4.8h\n" + "sqrdmulh v20.4s, v20.4s, v29.4s\n" + "smlal2 v0.4s, v22.8h, v4.8h\n" + "smlal2 v31.4s, v27.8h, v4.8h\n" + "sqrdmulh v19.4s, v19.4s, v29.4s\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "and v23.16b, v21.16b, v30.16b\n" + "sqrdmulh v8.4s, v8.4s, v14.4s\n" + "and v27.16b, v20.16b, v30.16b\n" + "sqrdmulh v0.4s, v0.4s, v14.4s\n" + "and v22.16b, v19.16b, v30.16b\n" + "sqrdmulh v31.4s, v31.4s, v14.4s\n" + "sqadd v3.4s, v3.4s, v16.4s\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "and v14.16b, v8.16b, v25.16b\n" "sshr v27.4s, v27.4s, #0x1f\n" - "and v4.16b, v23.16b, v31.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v7.16b, v22.16b, v31.16b\n" - "sqadd v10.4s, v10.4s, v19.4s\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v9.4s, v9.4s, v27.4s\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v21.4s, v21.4s, v0.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "srshl v15.4s, v15.4s, v18.4s\n" - "srshl v10.4s, v10.4s, v18.4s\n" - "sqadd v20.4s, v20.4s, v5.4s\n" - "srshl v9.4s, v9.4s, v18.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "srshl v21.4s, v21.4s, v18.4s\n" - "sqadd v22.4s, v22.4s, v7.4s\n" - "srshl v17.4s, v17.4s, v31.4s\n" - "sqxtn v15.4h, v15.4s\n" - "srshl v20.4s, v20.4s, v31.4s\n" - "sqxtn v10.4h, v10.4s\n" - "srshl v23.4s, v23.4s, v31.4s\n" - "sqxtn v9.4h, v9.4s\n" - "srshl v22.4s, v22.4s, v31.4s\n" + "and v18.16b, v0.16b, v25.16b\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "and v16.16b, v31.16b, v25.16b\n" + "sqadd v21.4s, v21.4s, v23.4s\n" + "sshr v14.4s, v14.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v27.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sqadd v19.4s, v19.4s, v22.4s\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "srshl v5.4s, v5.4s, v30.4s\n" + "srshl v21.4s, v21.4s, v30.4s\n" + "sqadd v8.4s, v8.4s, v14.4s\n" + "srshl v20.4s, v20.4s, v30.4s\n" + "sqadd v0.4s, v0.4s, v18.4s\n" + "srshl v19.4s, v19.4s, v30.4s\n" + "sqadd v31.4s, v31.4s, v16.4s\n" + "srshl v3.4s, v3.4s, v25.4s\n" + "sqxtn v5.4h, v5.4s\n" + "srshl v8.4s, v8.4s, v25.4s\n" "sqxtn v21.4h, v21.4s\n" - "sqxtn2 v15.8h, v17.4s\n" - "sqxtn2 v10.8h, v20.4s\n" - "sqxtn2 v9.8h, v23.4s\n" - "sqxtn2 v21.8h, v22.4s\n" - "sqadd v15.8h, v15.8h, v11.8h\n" - "sqadd v10.8h, v10.8h, v11.8h\n" - "sqadd v9.8h, v9.8h, v11.8h\n" - "sqadd v21.8h, v21.8h, v11.8h\n" - "smax v15.8h, v15.8h, v16.8h\n" - "smax v10.8h, v10.8h, v16.8h\n" - "smax v9.8h, v9.8h, v16.8h\n" - "smax v21.8h, v21.8h, v16.8h\n" - "smin v15.8h, v15.8h, v14.8h\n" - "smin v10.8h, v10.8h, v14.8h\n" - "smin v9.8h, v9.8h, v14.8h\n" - "smin v21.8h, v21.8h, v14.8h\n" - "uzp1 v15.16b, v15.16b, v15.16b\n" - "str d15, [x11, x16]\n" - "uzp1 v10.16b, v10.16b, v10.16b\n" - "uzp1 v9.16b, v9.16b, v9.16b\n" - "str d10, [x10, x16]\n" + "srshl v0.4s, v0.4s, v25.4s\n" + "sqxtn v20.4h, v20.4s\n" + "srshl v31.4s, v31.4s, v25.4s\n" + "sqxtn v19.4h, v19.4s\n" + "sqxtn2 v5.8h, v3.4s\n" + "sqxtn2 v21.8h, v8.4s\n" + "sqxtn2 v20.8h, v0.4s\n" + "sqxtn2 v19.8h, v31.4s\n" + "sqadd v5.8h, v5.8h, v13.8h\n" + "sqadd v21.8h, v21.8h, v13.8h\n" + "sqadd v20.8h, v20.8h, v13.8h\n" + "sqadd v19.8h, v19.8h, v13.8h\n" + "smax v5.8h, v5.8h, v17.8h\n" + "smax v21.8h, v21.8h, v17.8h\n" + "smax v20.8h, v20.8h, v17.8h\n" + "smax v19.8h, v19.8h, v17.8h\n" + "smin v5.8h, v5.8h, v24.8h\n" + "smin v21.8h, v21.8h, v24.8h\n" + "smin v20.8h, v20.8h, v24.8h\n" + "smin v19.8h, v19.8h, v24.8h\n" + "uzp1 v5.16b, v5.16b, v5.16b\n" + "str d5, [x11, x16]\n" "uzp1 v21.16b, v21.16b, v21.16b\n" - "str d9, [x9, x16]\n" - "str d21, [x28, x16]\n" + "uzp1 v20.16b, v20.16b, v20.16b\n" + "str d21, [x10, x16]\n" + "uzp1 v19.16b, v19.16b, v19.16b\n" + "str d20, [x9, x16]\n" + "str d19, [x28, x16]\n" "add x16, x16, #0x8\n" "beq 88f\n" "add x14, x14, #0x48\n" "3:" // Oddments - "ldr x24, [%x[params], %[offsetof_Params_bias]]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" "tbz x7, #2, 5f\n" - "ld1 { v15.4s }, [x24], #0x10\n" + "ld1 { v5.4s }, [x20], #0x10\n" "tbz x7, #1, 4f\n" - "ld1 { v17.d }[0], [x24], #0x8\n" + "ld1 { v3.d }[0], [x20], #0x8\n" "tbz x7, #0, 7f\n" - "ld1 { v17.s }[2], [x24]\n" + "ld1 { v3.s }[2], [x20]\n" "b 7f\n" "4:" // Oddments: Load bias: Bit 2: Bit 1: Unset "tbz x7, #0, 7f\n" - "ld1 { v17.s }[0], [x24]\n" + "ld1 { v3.s }[0], [x20]\n" "b 7f\n" "5:" // Oddments: Load bias: Bit 2: Unset "tbz x7, #1, 6f\n" - "ld1 { v15.d }[0], [x24], #0x8\n" + "ld1 { v5.d }[0], [x20], #0x8\n" "tbz x7, #0, 7f\n" - "ld1 { v15.s }[2], [x24]\n" + "ld1 { v5.s }[2], [x20]\n" "b 7f\n" "6:" // Oddments: Load bias: Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 7f\n" - "ld1 { v15.s }[0], [x24]\n" + "ld1 { v5.s }[0], [x20]\n" "7:" // Oddments: Load bias: Bit 2: End - "ldr d0, [x14, #0x0]\n" - "ldr d1, [x14, #0x8]\n" - "mov v10.16b, v15.16b\n" - "mov v20.16b, v17.16b\n" - "ldr d2, [x14, #0x10]\n" - "ldr d3, [x14, #0x18]\n" - "mov v9.16b, v15.16b\n" - "mov v23.16b, v17.16b\n" - "ldr d4, [x14, #0x20]\n" - "ldr d5, [x14, #0x28]\n" - "mov v21.16b, v15.16b\n" - "mov v22.16b, v17.16b\n" - "ldr d6, [x14, #0x30]\n" + "ldr d11, [x14, #0x0]\n" + "ldr d22, [x14, #0x8]\n" + "mov v21.16b, v5.16b\n" + "mov v8.16b, v3.16b\n" + "ldr d14, [x14, #0x10]\n" + "ldr d28, [x14, #0x18]\n" + "mov v20.16b, v5.16b\n" + "mov v0.16b, v3.16b\n" + "ldr d18, [x14, #0x20]\n" + "ldr d9, [x14, #0x28]\n" + "mov v19.16b, v5.16b\n" + "mov v31.16b, v3.16b\n" + "ldr d26, [x14, #0x30]\n" "ldr d7, [x14, #0x38]\n" - "ssubl v0.8h, v0.8b, v13.8b\n" - "ssubl v1.8h, v1.8b, v13.8b\n" - "ldr d8, [x14, #0x40]\n" + "ssubl v11.8h, v11.8b, v15.8b\n" + "ssubl v22.8h, v22.8b, v15.8b\n" + "ldr d4, [x14, #0x40]\n" "ldp x27, x26, [x15, #0x0]\n" - "ssubl v2.8h, v2.8b, v13.8b\n" - "ssubl v3.8h, v3.8b, v13.8b\n" + "ssubl v14.8h, v14.8b, v15.8b\n" + "ssubl v28.8h, v28.8b, v15.8b\n" "ldp x25, x24, [x15, #0x10]\n" "ldp x23, x22, [x15, #0x20]\n" - "ssubl v4.8h, v4.8b, v13.8b\n" - "ssubl v5.8h, v5.8b, v13.8b\n" + "ssubl v18.8h, v18.8b, v15.8b\n" + "ssubl v9.8h, v9.8b, v15.8b\n" "ldp x21, x20, [x15, #0x30]\n" - "ssubl v6.8h, v6.8b, v13.8b\n" - "ssubl v7.8h, v7.8b, v13.8b\n" - "ssubl v8.8h, v8.8b, v13.8b\n" + "ssubl v26.8h, v26.8b, v15.8b\n" + "ssubl v7.8h, v7.8b, v15.8b\n" + "ssubl v4.8h, v4.8b, v15.8b\n" "add x27, x27, x17\n" "add x26, x26, x17\n" "add x25, x25, x17\n" @@ -689,700 +689,700 @@ void a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "add x21, x21, x17\n" "add x20, x20, x17\n" "tbz x7, #2, 9f\n" - "ld1 { v31.s }[0], [x27], #0x4\n" - "ld1 { v30.s }[0], [x26], #0x4\n" - "ld1 { v29.s }[0], [x25], #0x4\n" - "ld1 { v28.s }[0], [x24], #0x4\n" - "ld1 { v27.s }[0], [x23], #0x4\n" - "ld1 { v26.s }[0], [x22], #0x4\n" - "ld1 { v25.s }[0], [x21], #0x4\n" - "ld1 { v24.s }[0], [x20], #0x4\n" + "ld1 { v25.s }[0], [x27], #0x4\n" + "ld1 { v27.s }[0], [x26], #0x4\n" + "ld1 { v1.s }[0], [x25], #0x4\n" + "ld1 { v2.s }[0], [x24], #0x4\n" + "ld1 { v12.s }[0], [x23], #0x4\n" + "ld1 { v16.s }[0], [x22], #0x4\n" + "ld1 { v23.s }[0], [x21], #0x4\n" + "ld1 { v10.s }[0], [x20], #0x4\n" "tbz x7, #1, 8f\n" - "ld1 { v31.h }[2], [x27], #0x2\n" - "ld1 { v30.h }[2], [x26], #0x2\n" - "ld1 { v29.h }[2], [x25], #0x2\n" - "ld1 { v28.h }[2], [x24], #0x2\n" - "ld1 { v27.h }[2], [x23], #0x2\n" - "ld1 { v26.h }[2], [x22], #0x2\n" - "ld1 { v25.h }[2], [x21], #0x2\n" - "ld1 { v24.h }[2], [x20], #0x2\n" + "ld1 { v25.h }[2], [x27], #0x2\n" + "ld1 { v27.h }[2], [x26], #0x2\n" + "ld1 { v1.h }[2], [x25], #0x2\n" + "ld1 { v2.h }[2], [x24], #0x2\n" + "ld1 { v12.h }[2], [x23], #0x2\n" + "ld1 { v16.h }[2], [x22], #0x2\n" + "ld1 { v23.h }[2], [x21], #0x2\n" + "ld1 { v10.h }[2], [x20], #0x2\n" "tbz x7, #0, 11f\n" - "ld1 { v31.b }[6], [x27]\n" - "ld1 { v30.b }[6], [x26]\n" - "ld1 { v29.b }[6], [x25]\n" - "ld1 { v28.b }[6], [x24]\n" - "ld1 { v27.b }[6], [x23]\n" - "ld1 { v26.b }[6], [x22]\n" - "ld1 { v25.b }[6], [x21]\n" - "ld1 { v24.b }[6], [x20]\n" + "ld1 { v25.b }[6], [x27]\n" + "ld1 { v27.b }[6], [x26]\n" + "ld1 { v1.b }[6], [x25]\n" + "ld1 { v2.b }[6], [x24]\n" + "ld1 { v12.b }[6], [x23]\n" + "ld1 { v16.b }[6], [x22]\n" + "ld1 { v23.b }[6], [x21]\n" + "ld1 { v10.b }[6], [x20]\n" "b 11f\n" "8:" // Oddments: Initial loads: Bit 2: Bit 1: Unset "tbz x7, #0, 11f\n" - "ld1 { v31.b }[4], [x27]\n" - "ld1 { v30.b }[4], [x26]\n" - "ld1 { v29.b }[4], [x25]\n" - "ld1 { v28.b }[4], [x24]\n" - "ld1 { v27.b }[4], [x23]\n" - "ld1 { v26.b }[4], [x22]\n" - "ld1 { v25.b }[4], [x21]\n" - "ld1 { v24.b }[4], [x20]\n" + "ld1 { v25.b }[4], [x27]\n" + "ld1 { v27.b }[4], [x26]\n" + "ld1 { v1.b }[4], [x25]\n" + "ld1 { v2.b }[4], [x24]\n" + "ld1 { v12.b }[4], [x23]\n" + "ld1 { v16.b }[4], [x22]\n" + "ld1 { v23.b }[4], [x21]\n" + "ld1 { v10.b }[4], [x20]\n" "b 11f\n" "9:" // Oddments: Initial loads: Bit 2: Unset "tbz x7, #1, 10f\n" - "ld1 { v31.h }[0], [x27], #0x2\n" - "ld1 { v30.h }[0], [x26], #0x2\n" - "ld1 { v29.h }[0], [x25], #0x2\n" - "ld1 { v28.h }[0], [x24], #0x2\n" - "ld1 { v27.h }[0], [x23], #0x2\n" - "ld1 { v26.h }[0], [x22], #0x2\n" - "ld1 { v25.h }[0], [x21], #0x2\n" - "ld1 { v24.h }[0], [x20], #0x2\n" + "ld1 { v25.h }[0], [x27], #0x2\n" + "ld1 { v27.h }[0], [x26], #0x2\n" + "ld1 { v1.h }[0], [x25], #0x2\n" + "ld1 { v2.h }[0], [x24], #0x2\n" + "ld1 { v12.h }[0], [x23], #0x2\n" + "ld1 { v16.h }[0], [x22], #0x2\n" + "ld1 { v23.h }[0], [x21], #0x2\n" + "ld1 { v10.h }[0], [x20], #0x2\n" "tbz x7, #0, 11f\n" - "ld1 { v31.b }[2], [x27]\n" - "ld1 { v30.b }[2], [x26]\n" - "ld1 { v29.b }[2], [x25]\n" - "ld1 { v28.b }[2], [x24]\n" - "ld1 { v27.b }[2], [x23]\n" - "ld1 { v26.b }[2], [x22]\n" - "ld1 { v25.b }[2], [x21]\n" - "ld1 { v24.b }[2], [x20]\n" + "ld1 { v25.b }[2], [x27]\n" + "ld1 { v27.b }[2], [x26]\n" + "ld1 { v1.b }[2], [x25]\n" + "ld1 { v2.b }[2], [x24]\n" + "ld1 { v12.b }[2], [x23]\n" + "ld1 { v16.b }[2], [x22]\n" + "ld1 { v23.b }[2], [x21]\n" + "ld1 { v10.b }[2], [x20]\n" "b 11f\n" "10:" // Oddments: Initial loads: Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 11f\n" - "ld1 { v31.b }[0], [x27]\n" - "ld1 { v30.b }[0], [x26]\n" - "ld1 { v29.b }[0], [x25]\n" - "ld1 { v28.b }[0], [x24]\n" - "ld1 { v27.b }[0], [x23]\n" - "ld1 { v26.b }[0], [x22]\n" - "ld1 { v25.b }[0], [x21]\n" - "ld1 { v24.b }[0], [x20]\n" + "ld1 { v25.b }[0], [x27]\n" + "ld1 { v27.b }[0], [x26]\n" + "ld1 { v1.b }[0], [x25]\n" + "ld1 { v2.b }[0], [x24]\n" + "ld1 { v12.b }[0], [x23]\n" + "ld1 { v16.b }[0], [x22]\n" + "ld1 { v23.b }[0], [x21]\n" + "ld1 { v10.b }[0], [x20]\n" "11:" // Oddments: Initial loads: Bit 2: End - "usubl v31.8h, v31.8b, v12.8b\n" - "smlal v15.4s, v31.4h, v8.4h\n" - "smlal2 v17.4s, v31.8h, v8.8h\n" - "ldr x24, [x15, #0x40]\n" - "usubl v30.8h, v30.8b, v12.8b\n" - "smlal v15.4s, v30.4h, v0.4h\n" - "smlal2 v17.4s, v30.8h, v0.8h\n" - "add x24, x24, x17\n" - "usubl v29.8h, v29.8b, v12.8b\n" - "smlal v10.4s, v31.4h, v6.4h\n" - "smlal2 v20.4s, v31.8h, v6.8h\n" - "smlal v15.4s, v29.4h, v1.4h\n" - "smlal2 v17.4s, v29.8h, v1.8h\n" - "usubl v28.8h, v28.8b, v12.8b\n" - "usubl v26.8h, v26.8b, v12.8b\n" - "smlal v10.4s, v28.4h, v1.4h\n" - "smlal2 v20.4s, v28.8h, v1.8h\n" - "smlal v15.4s, v26.4h, v3.4h\n" - "smlal2 v17.4s, v26.8h, v3.8h\n" - "usubl v27.8h, v27.8b, v12.8b\n" - "usubl v25.8h, v25.8b, v12.8b\n" - "smlal v10.4s, v27.4h, v2.4h\n" - "smlal2 v20.4s, v27.8h, v2.8h\n" - "smlal v15.4s, v25.4h, v4.4h\n" - "smlal2 v17.4s, v25.8h, v4.8h\n" - "usubl v24.8h, v24.8b, v12.8b\n" - "smlal v9.4s, v31.4h, v2.4h\n" - "smlal2 v23.4s, v31.8h, v2.8h\n" - "smlal v21.4s, v31.4h, v0.4h\n" - "smlal2 v22.4s, v31.8h, v0.8h\n" - "smlal v15.4s, v24.4h, v2.4h\n" - "smlal2 v17.4s, v24.8h, v2.8h\n" - "smlal v10.4s, v24.4h, v0.4h\n" - "smlal2 v20.4s, v24.8h, v0.8h\n" + "usubl v25.8h, v25.8b, v6.8b\n" + "smlal v5.4s, v25.4h, v4.4h\n" + "smlal2 v3.4s, v25.8h, v4.8h\n" + "ldr x20, [x15, #0x40]\n" + "usubl v27.8h, v27.8b, v6.8b\n" + "smlal v5.4s, v27.4h, v11.4h\n" + "smlal2 v3.4s, v27.8h, v11.8h\n" + "usubl v1.8h, v1.8b, v6.8b\n" + "smlal v21.4s, v25.4h, v26.4h\n" + "smlal2 v8.4s, v25.8h, v26.8h\n" + "add x20, x20, x17\n" + "smlal v5.4s, v1.4h, v22.4h\n" + "smlal2 v3.4s, v1.8h, v22.8h\n" + "usubl v2.8h, v2.8b, v6.8b\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "smlal v21.4s, v2.4h, v22.4h\n" + "smlal2 v8.4s, v2.8h, v22.8h\n" + "smlal v5.4s, v16.4h, v28.4h\n" + "smlal2 v3.4s, v16.8h, v28.8h\n" + "usubl v12.8h, v12.8b, v6.8b\n" + "usubl v23.8h, v23.8b, v6.8b\n" + "smlal v21.4s, v12.4h, v14.4h\n" + "smlal2 v8.4s, v12.8h, v14.8h\n" + "smlal v5.4s, v23.4h, v18.4h\n" + "smlal2 v3.4s, v23.8h, v18.8h\n" + "usubl v10.8h, v10.8b, v6.8b\n" + "smlal v20.4s, v25.4h, v14.4h\n" + "smlal2 v0.4s, v25.8h, v14.8h\n" + "smlal v19.4s, v25.4h, v11.4h\n" + "smlal2 v31.4s, v25.8h, v11.8h\n" + "smlal v5.4s, v10.4h, v14.4h\n" + "smlal2 v3.4s, v10.8h, v14.8h\n" + "smlal v21.4s, v10.4h, v11.4h\n" + "smlal2 v8.4s, v10.8h, v11.8h\n" "tbz x7, #2, 13f\n" - "ld1 { v29.s }[0], [x24], #0x4\n" + "ld1 { v15.s }[0], [x20], #0x4\n" "tbz x7, #1, 12f\n" - "ld1 { v29.h }[2], [x24], #0x2\n" + "ld1 { v15.h }[2], [x20], #0x2\n" "tbz x7, #0, 15f\n" - "ld1 { v29.b }[6], [x24]\n" + "ld1 { v15.b }[6], [x20]\n" "b 15f\n" "12:" // Oddments: Load (1, 3): Bit 2: Bit 1: Unset "tbz x7, #0, 15f\n" - "ld1 { v29.b }[4], [x24]\n" + "ld1 { v15.b }[4], [x20]\n" "b 15f\n" "13:" // Oddments: Load (1, 3): Bit 2: Unset "tbz x7, #1, 14f\n" - "ld1 { v29.h }[0], [x24], #0x2\n" + "ld1 { v15.h }[0], [x20], #0x2\n" "tbz x7, #0, 15f\n" - "ld1 { v29.b }[2], [x24]\n" + "ld1 { v15.b }[2], [x20]\n" "b 15f\n" "14:" // Oddments: Load (1, 3): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 15f\n" - "ld1 { v29.b }[0], [x24]\n" + "ld1 { v15.b }[0], [x20]\n" "15:" // Oddments: Load (1, 3): Bit 2: End - "usubl v29.8h, v29.8b, v12.8b\n" - "ldr x22, [x15, #0x48]\n" - "smlal v10.4s, v29.4h, v4.4h\n" - "smlal2 v20.4s, v29.8h, v4.8h\n" - "add x22, x22, x17\n" + "usubl v15.8h, v15.8b, v6.8b\n" + "ldr x20, [x15, #0x48]\n" + "smlal v21.4s, v15.4h, v18.4h\n" + "smlal2 v8.4s, v15.8h, v18.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 17f\n" - "ld1 { v28.s }[0], [x22], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 16f\n" - "ld1 { v28.h }[2], [x22], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 19f\n" - "ld1 { v28.b }[6], [x22]\n" + "ld1 { v16.b }[6], [x20]\n" "b 19f\n" "16:" // Oddments: Load (1, 4): Bit 2: Bit 1: Unset "tbz x7, #0, 19f\n" - "ld1 { v28.b }[4], [x22]\n" + "ld1 { v16.b }[4], [x20]\n" "b 19f\n" "17:" // Oddments: Load (1, 4): Bit 2: Unset "tbz x7, #1, 18f\n" - "ld1 { v28.h }[0], [x22], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 19f\n" - "ld1 { v28.b }[2], [x22]\n" + "ld1 { v16.b }[2], [x20]\n" "b 19f\n" "18:" // Oddments: Load (1, 4): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 19f\n" - "ld1 { v28.b }[0], [x22]\n" + "ld1 { v16.b }[0], [x20]\n" "19:" // Oddments: Load (1, 4): Bit 2: End - "usubl v28.8h, v28.8b, v12.8b\n" - "ldr x21, [x15, #0x50]\n" - "smlal v10.4s, v28.4h, v5.4h\n" - "smlal2 v20.4s, v28.8h, v5.8h\n" - "add x21, x21, x17\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0x50]\n" + "smlal v21.4s, v16.4h, v9.4h\n" + "smlal2 v8.4s, v16.8h, v9.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 21f\n" - "ld1 { v27.s }[0], [x21], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 20f\n" - "ld1 { v27.h }[2], [x21], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 23f\n" - "ld1 { v27.b }[6], [x21]\n" + "ld1 { v16.b }[6], [x20]\n" "b 23f\n" "20:" // Oddments: Load (1, 2): Bit 2: Bit 1: Unset "tbz x7, #0, 23f\n" - "ld1 { v27.b }[4], [x21]\n" + "ld1 { v16.b }[4], [x20]\n" "b 23f\n" "21:" // Oddments: Load (1, 2): Bit 2: Unset "tbz x7, #1, 22f\n" - "ld1 { v27.h }[0], [x21], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 23f\n" - "ld1 { v27.b }[2], [x21]\n" + "ld1 { v16.b }[2], [x20]\n" "b 23f\n" "22:" // Oddments: Load (1, 2): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 23f\n" - "ld1 { v27.b }[0], [x21]\n" + "ld1 { v16.b }[0], [x20]\n" "23:" // Oddments: Load (1, 2): Bit 2: End - "usubl v27.8h, v27.8b, v12.8b\n" + "usubl v16.8h, v16.8b, v6.8b\n" "ldr x20, [x15, #0x58]\n" - "smlal v15.4s, v27.4h, v5.4h\n" - "smlal2 v17.4s, v27.8h, v5.8h\n" - "smlal v10.4s, v27.4h, v3.4h\n" - "smlal2 v20.4s, v27.8h, v3.8h\n" + "smlal v5.4s, v16.4h, v9.4h\n" + "smlal2 v3.4s, v16.8h, v9.8h\n" + "smlal v21.4s, v16.4h, v28.4h\n" + "smlal2 v8.4s, v16.8h, v28.8h\n" "add x20, x20, x17\n" "tbz x7, #2, 25f\n" - "ld1 { v26.s }[0], [x20], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 24f\n" - "ld1 { v26.h }[2], [x20], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 27f\n" - "ld1 { v26.b }[6], [x20]\n" + "ld1 { v16.b }[6], [x20]\n" "b 27f\n" "24:" // Oddments: Load (3, 0): Bit 2: Bit 1: Unset "tbz x7, #0, 27f\n" - "ld1 { v26.b }[4], [x20]\n" + "ld1 { v16.b }[4], [x20]\n" "b 27f\n" "25:" // Oddments: Load (3, 0): Bit 2: Unset "tbz x7, #1, 26f\n" - "ld1 { v26.h }[0], [x20], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 27f\n" - "ld1 { v26.b }[2], [x20]\n" + "ld1 { v16.b }[2], [x20]\n" "b 27f\n" "26:" // Oddments: Load (3, 0): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 27f\n" - "ld1 { v26.b }[0], [x20]\n" + "ld1 { v16.b }[0], [x20]\n" "27:" // Oddments: Load (3, 0): Bit 2: End - "usubl v26.8h, v26.8b, v12.8b\n" + "usubl v16.8h, v16.8b, v6.8b\n" "ldr x20, [x15, #0x60]\n" - "smlal v9.4s, v26.4h, v3.4h\n" - "smlal2 v23.4s, v26.8h, v3.8h\n" + "smlal v20.4s, v16.4h, v28.4h\n" + "smlal2 v0.4s, v16.8h, v28.8h\n" "add x20, x20, x17\n" "tbz x7, #2, 29f\n" - "ld1 { v25.s }[0], [x20], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 28f\n" - "ld1 { v25.h }[2], [x20], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 31f\n" - "ld1 { v25.b }[6], [x20]\n" + "ld1 { v16.b }[6], [x20]\n" "b 31f\n" "28:" // Oddments: Load (2, 0): Bit 2: Bit 1: Unset "tbz x7, #0, 31f\n" - "ld1 { v25.b }[4], [x20]\n" + "ld1 { v16.b }[4], [x20]\n" "b 31f\n" "29:" // Oddments: Load (2, 0): Bit 2: Unset "tbz x7, #1, 30f\n" - "ld1 { v25.h }[0], [x20], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 31f\n" - "ld1 { v25.b }[2], [x20]\n" + "ld1 { v16.b }[2], [x20]\n" "b 31f\n" "30:" // Oddments: Load (2, 0): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 31f\n" - "ld1 { v25.b }[0], [x20]\n" + "ld1 { v16.b }[0], [x20]\n" "31:" // Oddments: Load (2, 0): Bit 2: End - "usubl v25.8h, v25.8b, v12.8b\n" + "usubl v16.8h, v16.8b, v6.8b\n" "ldr x20, [x15, #0x68]\n" - "smlal v15.4s, v25.4h, v6.4h\n" - "smlal2 v17.4s, v25.8h, v6.8h\n" - "smlal v9.4s, v25.4h, v0.4h\n" - "smlal2 v23.4s, v25.8h, v0.8h\n" + "smlal v5.4s, v16.4h, v26.4h\n" + "smlal2 v3.4s, v16.8h, v26.8h\n" + "smlal v20.4s, v16.4h, v11.4h\n" + "smlal2 v0.4s, v16.8h, v11.8h\n" "add x20, x20, x17\n" "tbz x7, #2, 33f\n" - "ld1 { v29.s }[0], [x20], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 32f\n" - "ld1 { v29.h }[2], [x20], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 35f\n" - "ld1 { v29.b }[6], [x20]\n" + "ld1 { v16.b }[6], [x20]\n" "b 35f\n" "32:" // Oddments: Load (3, 1): Bit 2: Bit 1: Unset "tbz x7, #0, 35f\n" - "ld1 { v29.b }[4], [x20]\n" + "ld1 { v16.b }[4], [x20]\n" "b 35f\n" "33:" // Oddments: Load (3, 1): Bit 2: Unset "tbz x7, #1, 34f\n" - "ld1 { v29.h }[0], [x20], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 35f\n" - "ld1 { v29.b }[2], [x20]\n" + "ld1 { v16.b }[2], [x20]\n" "b 35f\n" "34:" // Oddments: Load (3, 1): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 35f\n" - "ld1 { v29.b }[0], [x20]\n" + "ld1 { v16.b }[0], [x20]\n" "35:" // Oddments: Load (3, 1): Bit 2: End - "usubl v29.8h, v29.8b, v12.8b\n" - "ldr x21, [x15, #0x70]\n" - "smlal v9.4s, v29.4h, v4.4h\n" - "smlal2 v23.4s, v29.8h, v4.8h\n" - "add x21, x21, x17\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0x70]\n" + "smlal v20.4s, v16.4h, v18.4h\n" + "smlal2 v0.4s, v16.8h, v18.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 37f\n" - "ld1 { v24.s }[0], [x21], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 36f\n" - "ld1 { v24.h }[2], [x21], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 39f\n" - "ld1 { v24.b }[6], [x21]\n" + "ld1 { v16.b }[6], [x20]\n" "b 39f\n" "36:" // Oddments: Load (2, 1): Bit 2: Bit 1: Unset "tbz x7, #0, 39f\n" - "ld1 { v24.b }[4], [x21]\n" + "ld1 { v16.b }[4], [x20]\n" "b 39f\n" "37:" // Oddments: Load (2, 1): Bit 2: Unset "tbz x7, #1, 38f\n" - "ld1 { v24.h }[0], [x21], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 39f\n" - "ld1 { v24.b }[2], [x21]\n" + "ld1 { v16.b }[2], [x20]\n" "b 39f\n" "38:" // Oddments: Load (2, 1): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 39f\n" - "ld1 { v24.b }[0], [x21]\n" + "ld1 { v16.b }[0], [x20]\n" "39:" // Oddments: Load (2, 1): Bit 2: End - "usubl v24.8h, v24.8b, v12.8b\n" - "ldr x23, [x15, #0x78]\n" - "smlal v15.4s, v24.4h, v7.4h\n" - "smlal2 v17.4s, v24.8h, v7.8h\n" - "smlal v9.4s, v24.4h, v1.4h\n" - "smlal2 v23.4s, v24.8h, v1.8h\n" - "add x23, x23, x17\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0x78]\n" + "smlal v5.4s, v16.4h, v7.4h\n" + "smlal2 v3.4s, v16.8h, v7.8h\n" + "smlal v20.4s, v16.4h, v22.4h\n" + "smlal2 v0.4s, v16.8h, v22.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 41f\n" - "ld1 { v27.s }[0], [x23], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 40f\n" - "ld1 { v27.h }[2], [x23], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 43f\n" - "ld1 { v27.b }[6], [x23]\n" + "ld1 { v16.b }[6], [x20]\n" "b 43f\n" "40:" // Oddments: Load (3, 3): Bit 2: Bit 1: Unset "tbz x7, #0, 43f\n" - "ld1 { v27.b }[4], [x23]\n" + "ld1 { v16.b }[4], [x20]\n" "b 43f\n" "41:" // Oddments: Load (3, 3): Bit 2: Unset "tbz x7, #1, 42f\n" - "ld1 { v27.h }[0], [x23], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 43f\n" - "ld1 { v27.b }[2], [x23]\n" + "ld1 { v16.b }[2], [x20]\n" "b 43f\n" "42:" // Oddments: Load (3, 3): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 43f\n" - "ld1 { v27.b }[0], [x23]\n" + "ld1 { v16.b }[0], [x20]\n" "43:" // Oddments: Load (3, 3): Bit 2: End - "usubl v27.8h, v27.8b, v12.8b\n" - "ldr x21, [x15, #0x80]\n" - "smlal v21.4s, v27.4h, v4.4h\n" - "smlal2 v22.4s, v27.8h, v4.8h\n" - "add x21, x21, x17\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0x80]\n" + "smlal v19.4s, v16.4h, v18.4h\n" + "smlal2 v31.4s, v16.8h, v18.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 45f\n" - "ld1 { v28.s }[0], [x21], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 44f\n" - "ld1 { v28.h }[2], [x21], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 47f\n" - "ld1 { v28.b }[6], [x21]\n" + "ld1 { v16.b }[6], [x20]\n" "b 47f\n" "44:" // Oddments: Load (2, 3): Bit 2: Bit 1: Unset "tbz x7, #0, 47f\n" - "ld1 { v28.b }[4], [x21]\n" + "ld1 { v16.b }[4], [x20]\n" "b 47f\n" "45:" // Oddments: Load (2, 3): Bit 2: Unset "tbz x7, #1, 46f\n" - "ld1 { v28.h }[0], [x21], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 47f\n" - "ld1 { v28.b }[2], [x21]\n" + "ld1 { v16.b }[2], [x20]\n" "b 47f\n" "46:" // Oddments: Load (2, 3): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 47f\n" - "ld1 { v28.b }[0], [x21]\n" + "ld1 { v16.b }[0], [x20]\n" "47:" // Oddments: Load (2, 3): Bit 2: End - "usubl v28.8h, v28.8b, v12.8b\n" - "ldr x22, [x15, #0x88]\n" - "smlal v10.4s, v28.4h, v7.4h\n" - "smlal2 v20.4s, v28.8h, v7.8h\n" - "smlal v21.4s, v28.4h, v1.4h\n" - "smlal2 v22.4s, v28.8h, v1.8h\n" - "add x22, x22, x17\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0x88]\n" + "smlal v21.4s, v16.4h, v7.4h\n" + "smlal2 v8.4s, v16.8h, v7.8h\n" + "smlal v19.4s, v16.4h, v22.4h\n" + "smlal2 v31.4s, v16.8h, v22.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 49f\n" - "ld1 { v26.s }[0], [x22], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 48f\n" - "ld1 { v26.h }[2], [x22], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 51f\n" - "ld1 { v26.b }[6], [x22]\n" + "ld1 { v16.b }[6], [x20]\n" "b 51f\n" "48:" // Oddments: Load (3, 4): Bit 2: Bit 1: Unset "tbz x7, #0, 51f\n" - "ld1 { v26.b }[4], [x22]\n" + "ld1 { v16.b }[4], [x20]\n" "b 51f\n" "49:" // Oddments: Load (3, 4): Bit 2: Unset "tbz x7, #1, 50f\n" - "ld1 { v26.h }[0], [x22], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 51f\n" - "ld1 { v26.b }[2], [x22]\n" + "ld1 { v16.b }[2], [x20]\n" "b 51f\n" "50:" // Oddments: Load (3, 4): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 51f\n" - "ld1 { v26.b }[0], [x22]\n" + "ld1 { v16.b }[0], [x20]\n" "51:" // Oddments: Load (3, 4): Bit 2: End - "usubl v26.8h, v26.8b, v12.8b\n" - "ldr x24, [x15, #0x90]\n" - "smlal v21.4s, v26.4h, v5.4h\n" - "smlal2 v22.4s, v26.8h, v5.8h\n" - "add x24, x24, x17\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0x90]\n" + "smlal v19.4s, v16.4h, v9.4h\n" + "smlal2 v31.4s, v16.8h, v9.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 53f\n" - "ld1 { v25.s }[0], [x24], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 52f\n" - "ld1 { v25.h }[2], [x24], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 55f\n" - "ld1 { v25.b }[6], [x24]\n" + "ld1 { v16.b }[6], [x20]\n" "b 55f\n" "52:" // Oddments: Load (4, 0): Bit 2: Bit 1: Unset "tbz x7, #0, 55f\n" - "ld1 { v25.b }[4], [x24]\n" + "ld1 { v16.b }[4], [x20]\n" "b 55f\n" "53:" // Oddments: Load (4, 0): Bit 2: Unset "tbz x7, #1, 54f\n" - "ld1 { v25.h }[0], [x24], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 55f\n" - "ld1 { v25.b }[2], [x24]\n" + "ld1 { v16.b }[2], [x20]\n" "b 55f\n" "54:" // Oddments: Load (4, 0): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 55f\n" - "ld1 { v25.b }[0], [x24]\n" + "ld1 { v16.b }[0], [x20]\n" "55:" // Oddments: Load (4, 0): Bit 2: End - "usubl v25.8h, v25.8b, v12.8b\n" - "ldr x25, [x15, #0x98]\n" - "smlal v9.4s, v25.4h, v6.4h\n" - "smlal2 v23.4s, v25.8h, v6.8h\n" - "add x25, x25, x17\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0x98]\n" + "smlal v20.4s, v16.4h, v26.4h\n" + "smlal2 v0.4s, v16.8h, v26.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 57f\n" - "ld1 { v29.s }[0], [x25], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 56f\n" - "ld1 { v29.h }[2], [x25], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 59f\n" - "ld1 { v29.b }[6], [x25]\n" + "ld1 { v16.b }[6], [x20]\n" "b 59f\n" "56:" // Oddments: Load (2, 4): Bit 2: Bit 1: Unset "tbz x7, #0, 59f\n" - "ld1 { v29.b }[4], [x25]\n" + "ld1 { v16.b }[4], [x20]\n" "b 59f\n" "57:" // Oddments: Load (2, 4): Bit 2: Unset "tbz x7, #1, 58f\n" - "ld1 { v29.h }[0], [x25], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 59f\n" - "ld1 { v29.b }[2], [x25]\n" + "ld1 { v16.b }[2], [x20]\n" "b 59f\n" "58:" // Oddments: Load (2, 4): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 59f\n" - "ld1 { v29.b }[0], [x25]\n" + "ld1 { v16.b }[0], [x20]\n" "59:" // Oddments: Load (2, 4): Bit 2: End - "usubl v29.8h, v29.8b, v12.8b\n" + "usubl v16.8h, v16.8b, v6.8b\n" "ldr x20, [x15, #0xa0]\n" - "smlal v10.4s, v29.4h, v8.4h\n" - "smlal2 v20.4s, v29.8h, v8.8h\n" - "smlal v21.4s, v29.4h, v2.4h\n" - "smlal2 v22.4s, v29.8h, v2.8h\n" + "smlal v21.4s, v16.4h, v4.4h\n" + "smlal2 v8.4s, v16.8h, v4.8h\n" + "smlal v19.4s, v16.4h, v14.4h\n" + "smlal2 v31.4s, v16.8h, v14.8h\n" "add x20, x20, x17\n" "tbz x7, #2, 61f\n" - "ld1 { v27.s }[0], [x20], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 60f\n" - "ld1 { v27.h }[2], [x20], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 63f\n" - "ld1 { v27.b }[6], [x20]\n" + "ld1 { v16.b }[6], [x20]\n" "b 63f\n" "60:" // Oddments: Load (4, 1): Bit 2: Bit 1: Unset "tbz x7, #0, 63f\n" - "ld1 { v27.b }[4], [x20]\n" + "ld1 { v16.b }[4], [x20]\n" "b 63f\n" "61:" // Oddments: Load (4, 1): Bit 2: Unset "tbz x7, #1, 62f\n" - "ld1 { v27.h }[0], [x20], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 63f\n" - "ld1 { v27.b }[2], [x20]\n" + "ld1 { v16.b }[2], [x20]\n" "b 63f\n" "62:" // Oddments: Load (4, 1): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 63f\n" - "ld1 { v27.b }[0], [x20]\n" + "ld1 { v16.b }[0], [x20]\n" "63:" // Oddments: Load (4, 1): Bit 2: End - "usubl v27.8h, v27.8b, v12.8b\n" - "ldr x23, [x15, #0xa8]\n" - "smlal v9.4s, v27.4h, v7.4h\n" - "smlal2 v23.4s, v27.8h, v7.8h\n" - "add x23, x23, x17\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0xa8]\n" + "smlal v20.4s, v16.4h, v7.4h\n" + "smlal2 v0.4s, v16.8h, v7.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 65f\n" - "ld1 { v24.s }[0], [x23], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 64f\n" - "ld1 { v24.h }[2], [x23], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 67f\n" - "ld1 { v24.b }[6], [x23]\n" + "ld1 { v16.b }[6], [x20]\n" "b 67f\n" "64:" // Oddments: Load (3, 2): Bit 2: Bit 1: Unset "tbz x7, #0, 67f\n" - "ld1 { v24.b }[4], [x23]\n" + "ld1 { v16.b }[4], [x20]\n" "b 67f\n" "65:" // Oddments: Load (3, 2): Bit 2: Unset "tbz x7, #1, 66f\n" - "ld1 { v24.h }[0], [x23], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 67f\n" - "ld1 { v24.b }[2], [x23]\n" + "ld1 { v16.b }[2], [x20]\n" "b 67f\n" "66:" // Oddments: Load (3, 2): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 67f\n" - "ld1 { v24.b }[0], [x23]\n" + "ld1 { v16.b }[0], [x20]\n" "67:" // Oddments: Load (3, 2): Bit 2: End - "usubl v24.8h, v24.8b, v12.8b\n" - "ldr x22, [x15, #0xb0]\n" - "smlal v9.4s, v24.4h, v5.4h\n" - "smlal2 v23.4s, v24.8h, v5.8h\n" - "smlal v21.4s, v24.4h, v3.4h\n" - "smlal2 v22.4s, v24.8h, v3.8h\n" - "add x22, x22, x17\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0xb0]\n" + "smlal v20.4s, v16.4h, v9.4h\n" + "smlal2 v0.4s, v16.8h, v9.8h\n" + "smlal v19.4s, v16.4h, v28.4h\n" + "smlal2 v31.4s, v16.8h, v28.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 69f\n" - "ld1 { v26.s }[0], [x22], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 68f\n" - "ld1 { v26.h }[2], [x22], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 71f\n" - "ld1 { v26.b }[6], [x22]\n" + "ld1 { v16.b }[6], [x20]\n" "b 71f\n" "68:" // Oddments: Load (4, 3): Bit 2: Bit 1: Unset "tbz x7, #0, 71f\n" - "ld1 { v26.b }[4], [x22]\n" + "ld1 { v16.b }[4], [x20]\n" "b 71f\n" "69:" // Oddments: Load (4, 3): Bit 2: Unset "tbz x7, #1, 70f\n" - "ld1 { v26.h }[0], [x22], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 71f\n" - "ld1 { v26.b }[2], [x22]\n" + "ld1 { v16.b }[2], [x20]\n" "b 71f\n" "70:" // Oddments: Load (4, 3): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 71f\n" - "ld1 { v26.b }[0], [x22]\n" + "ld1 { v16.b }[0], [x20]\n" "71:" // Oddments: Load (4, 3): Bit 2: End - "usubl v26.8h, v26.8b, v12.8b\n" - "ldr x21, [x15, #0xb8]\n" - "smlal v21.4s, v26.4h, v7.4h\n" - "smlal2 v22.4s, v26.8h, v7.8h\n" - "add x21, x21, x17\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "ldr x20, [x15, #0xb8]\n" + "smlal v19.4s, v16.4h, v7.4h\n" + "smlal2 v31.4s, v16.8h, v7.8h\n" + "add x20, x20, x17\n" "tbz x7, #2, 73f\n" - "ld1 { v25.s }[0], [x21], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 72f\n" - "ld1 { v25.h }[2], [x21], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 75f\n" - "ld1 { v25.b }[6], [x21]\n" + "ld1 { v16.b }[6], [x20]\n" "b 75f\n" "72:" // Oddments: Load (4, 2): Bit 2: Bit 1: Unset "tbz x7, #0, 75f\n" - "ld1 { v25.b }[4], [x21]\n" + "ld1 { v16.b }[4], [x20]\n" "b 75f\n" "73:" // Oddments: Load (4, 2): Bit 2: Unset "tbz x7, #1, 74f\n" - "ld1 { v25.h }[0], [x21], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 75f\n" - "ld1 { v25.b }[2], [x21]\n" + "ld1 { v16.b }[2], [x20]\n" "b 75f\n" "74:" // Oddments: Load (4, 2): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 75f\n" - "ld1 { v25.b }[0], [x21]\n" + "ld1 { v16.b }[0], [x20]\n" "75:" // Oddments: Load (4, 2): Bit 2: End - "usubl v25.8h, v25.8b, v12.8b\n" + "usubl v16.8h, v16.8b, v6.8b\n" "ldr x20, [x15, #0xc0]\n" - "smlal v9.4s, v25.4h, v8.4h\n" - "smlal2 v23.4s, v25.8h, v8.8h\n" - "smlal v21.4s, v25.4h, v6.4h\n" - "smlal2 v22.4s, v25.8h, v6.8h\n" + "smlal v20.4s, v16.4h, v4.4h\n" + "smlal2 v0.4s, v16.8h, v4.8h\n" + "smlal v19.4s, v16.4h, v26.4h\n" + "smlal2 v31.4s, v16.8h, v26.8h\n" "add x20, x20, x17\n" "tbz x7, #2, 77f\n" - "ld1 { v29.s }[0], [x20], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x7, #1, 76f\n" - "ld1 { v29.h }[2], [x20], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x7, #0, 79f\n" - "ld1 { v29.b }[6], [x20]\n" + "ld1 { v16.b }[6], [x20]\n" "b 79f\n" "76:" // Oddments: Load (4, 4): Bit 2: Bit 1: Unset "tbz x7, #0, 79f\n" - "ld1 { v29.b }[4], [x20]\n" + "ld1 { v16.b }[4], [x20]\n" "b 79f\n" "77:" // Oddments: Load (4, 4): Bit 2: Unset "tbz x7, #1, 78f\n" - "ld1 { v29.h }[0], [x20], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x7, #0, 79f\n" - "ld1 { v29.b }[2], [x20]\n" + "ld1 { v16.b }[2], [x20]\n" "b 79f\n" "78:" // Oddments: Load (4, 4): Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 79f\n" - "ld1 { v29.b }[0], [x20]\n" + "ld1 { v16.b }[0], [x20]\n" "79:" // Oddments: Load (4, 4): Bit 2: End - "usubl v29.8h, v29.8b, v12.8b\n" - "smlal v21.4s, v29.4h, v8.4h\n" - "smlal2 v22.4s, v29.8h, v8.8h\n" + "usubl v16.8h, v16.8b, v6.8b\n" + "smlal v19.4s, v16.4h, v4.4h\n" + "smlal2 v31.4s, v16.8h, v4.8h\n" "tbz x7, #2, 81f\n" - "ld1 { v19.4s }, [x13], #0x10\n" - "ld1 { v18.4s }, [x12], #0x10\n" + "ld1 { v14.4s }, [x13], #0x10\n" + "ld1 { v25.4s }, [x12], #0x10\n" "tbz x7, #1, 80f\n" - "ld1 { v30.d }[0], [x13], #0x8\n" - "ld1 { v31.d }[0], [x12], #0x8\n" + "ld1 { v18.d }[0], [x13], #0x8\n" + "ld1 { v12.d }[0], [x12], #0x8\n" "tbz x7, #0, 83f\n" - "ld1 { v30.s }[2], [x13]\n" - "ld1 { v31.s }[2], [x12]\n" + "ld1 { v18.s }[2], [x13]\n" + "ld1 { v12.s }[2], [x12]\n" "b 83f\n" "80:" // Oddments: Load requant params: Bit 2: Bit 1: Unset "tbz x7, #0, 83f\n" - "ld1 { v30.s }[0], [x13]\n" - "ld1 { v31.s }[0], [x12]\n" + "ld1 { v18.s }[0], [x13]\n" + "ld1 { v12.s }[0], [x12]\n" "b 83f\n" "81:" // Oddments: Load requant params: Bit 2: Unset "tbz x7, #1, 82f\n" - "ld1 { v19.d }[0], [x13], #0x8\n" - "ld1 { v18.d }[0], [x12], #0x8\n" + "ld1 { v14.d }[0], [x13], #0x8\n" + "ld1 { v25.d }[0], [x12], #0x8\n" "tbz x7, #0, 83f\n" - "ld1 { v19.s }[2], [x13]\n" - "ld1 { v18.s }[2], [x12]\n" + "ld1 { v14.s }[2], [x13]\n" + "ld1 { v25.s }[2], [x12]\n" "b 83f\n" "82:" // Oddments: Load requant params: Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 83f\n" - "ld1 { v19.s }[0], [x13]\n" - "ld1 { v18.s }[0], [x12]\n" + "ld1 { v14.s }[0], [x13]\n" + "ld1 { v25.s }[0], [x12]\n" "83:" // Oddments: Load requant params: Bit 2: End - "sqrdmulh v15.4s, v15.4s, v19.4s\n" - "and v0.16b, v15.16b, v18.16b\n" + "sqrdmulh v5.4s, v5.4s, v14.4s\n" + "and v28.16b, v5.16b, v25.16b\n" "add x11, x11, x16\n" "add x10, x10, x16\n" - "sqrdmulh v17.4s, v17.4s, v30.4s\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqrdmulh v3.4s, v3.4s, v18.4s\n" + "sshr v28.4s, v28.4s, #0x1f\n" "add x9, x9, x16\n" "add x28, x28, x16\n" - "and v7.16b, v17.16b, v31.16b\n" - "sqrdmulh v10.4s, v10.4s, v19.4s\n" - "sqrdmulh v9.4s, v9.4s, v19.4s\n" - "sqrdmulh v21.4s, v21.4s, v19.4s\n" - "sqadd v15.4s, v15.4s, v0.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "and v19.16b, v10.16b, v18.16b\n" - "sqrdmulh v20.4s, v20.4s, v30.4s\n" - "and v27.16b, v9.16b, v18.16b\n" - "sqrdmulh v23.4s, v23.4s, v30.4s\n" - "and v0.16b, v21.16b, v18.16b\n" - "sqrdmulh v22.4s, v22.4s, v30.4s\n" - "sqadd v17.4s, v17.4s, v7.4s\n" - "sshr v19.4s, v19.4s, #0x1f\n" - "and v5.16b, v20.16b, v31.16b\n" - "sshr v27.4s, v27.4s, #0x1f\n" - "and v4.16b, v23.16b, v31.16b\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v7.16b, v22.16b, v31.16b\n" - "sqadd v10.4s, v10.4s, v19.4s\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v9.4s, v9.4s, v27.4s\n" + "and v16.16b, v3.16b, v12.16b\n" + "sqrdmulh v21.4s, v21.4s, v14.4s\n" + "sqrdmulh v20.4s, v20.4s, v14.4s\n" + "sqrdmulh v19.4s, v19.4s, v14.4s\n" + "sqadd v5.4s, v5.4s, v28.4s\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "and v14.16b, v21.16b, v25.16b\n" + "sqrdmulh v8.4s, v8.4s, v18.4s\n" + "and v6.16b, v20.16b, v25.16b\n" + "sqrdmulh v0.4s, v0.4s, v18.4s\n" + "and v4.16b, v19.16b, v25.16b\n" + "sqrdmulh v31.4s, v31.4s, v18.4s\n" + "sqadd v3.4s, v3.4s, v16.4s\n" + "sshr v14.4s, v14.4s, #0x1f\n" + "and v18.16b, v8.16b, v12.16b\n" + "sshr v6.4s, v6.4s, #0x1f\n" + "and v7.16b, v0.16b, v12.16b\n" "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v21.4s, v21.4s, v0.4s\n" + "and v16.16b, v31.16b, v12.16b\n" + "sqadd v21.4s, v21.4s, v14.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v6.4s\n" "sshr v7.4s, v7.4s, #0x1f\n" - "srshl v15.4s, v15.4s, v18.4s\n" - "srshl v10.4s, v10.4s, v18.4s\n" - "sqadd v20.4s, v20.4s, v5.4s\n" - "srshl v9.4s, v9.4s, v18.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "srshl v21.4s, v21.4s, v18.4s\n" - "sqadd v22.4s, v22.4s, v7.4s\n" - "srshl v17.4s, v17.4s, v31.4s\n" - "sqxtn v15.4h, v15.4s\n" - "srshl v20.4s, v20.4s, v31.4s\n" - "sqxtn v10.4h, v10.4s\n" - "srshl v23.4s, v23.4s, v31.4s\n" - "sqxtn v9.4h, v9.4s\n" - "srshl v22.4s, v22.4s, v31.4s\n" + "sqadd v19.4s, v19.4s, v4.4s\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "srshl v5.4s, v5.4s, v25.4s\n" + "srshl v21.4s, v21.4s, v25.4s\n" + "sqadd v8.4s, v8.4s, v18.4s\n" + "srshl v20.4s, v20.4s, v25.4s\n" + "sqadd v0.4s, v0.4s, v7.4s\n" + "srshl v19.4s, v19.4s, v25.4s\n" + "sqadd v31.4s, v31.4s, v16.4s\n" + "srshl v3.4s, v3.4s, v12.4s\n" + "sqxtn v5.4h, v5.4s\n" + "srshl v8.4s, v8.4s, v12.4s\n" "sqxtn v21.4h, v21.4s\n" - "sqxtn2 v15.8h, v17.4s\n" - "sqxtn2 v10.8h, v20.4s\n" - "sqxtn2 v9.8h, v23.4s\n" - "sqxtn2 v21.8h, v22.4s\n" - "sqadd v15.8h, v15.8h, v11.8h\n" - "sqadd v10.8h, v10.8h, v11.8h\n" - "sqadd v9.8h, v9.8h, v11.8h\n" - "sqadd v21.8h, v21.8h, v11.8h\n" - "smax v15.8h, v15.8h, v16.8h\n" - "smax v10.8h, v10.8h, v16.8h\n" - "smax v9.8h, v9.8h, v16.8h\n" - "smax v21.8h, v21.8h, v16.8h\n" - "smin v15.8h, v15.8h, v14.8h\n" - "smin v10.8h, v10.8h, v14.8h\n" - "smin v9.8h, v9.8h, v14.8h\n" - "smin v21.8h, v21.8h, v14.8h\n" - "uzp1 v15.16b, v15.16b, v15.16b\n" - "uzp1 v10.16b, v10.16b, v10.16b\n" - "uzp1 v9.16b, v9.16b, v9.16b\n" + "srshl v0.4s, v0.4s, v12.4s\n" + "sqxtn v20.4h, v20.4s\n" + "srshl v31.4s, v31.4s, v12.4s\n" + "sqxtn v19.4h, v19.4s\n" + "sqxtn2 v5.8h, v3.4s\n" + "sqxtn2 v21.8h, v8.4s\n" + "sqxtn2 v20.8h, v0.4s\n" + "sqxtn2 v19.8h, v31.4s\n" + "sqadd v5.8h, v5.8h, v13.8h\n" + "sqadd v21.8h, v21.8h, v13.8h\n" + "sqadd v20.8h, v20.8h, v13.8h\n" + "sqadd v19.8h, v19.8h, v13.8h\n" + "smax v5.8h, v5.8h, v17.8h\n" + "smax v21.8h, v21.8h, v17.8h\n" + "smax v20.8h, v20.8h, v17.8h\n" + "smax v19.8h, v19.8h, v17.8h\n" + "smin v5.8h, v5.8h, v24.8h\n" + "smin v21.8h, v21.8h, v24.8h\n" + "smin v20.8h, v20.8h, v24.8h\n" + "smin v19.8h, v19.8h, v24.8h\n" + "uzp1 v5.16b, v5.16b, v5.16b\n" "uzp1 v21.16b, v21.16b, v21.16b\n" + "uzp1 v20.16b, v20.16b, v20.16b\n" + "uzp1 v19.16b, v19.16b, v19.16b\n" "tbz x7, #2, 85f\n" - "st1 { v15.s }[0], [x11], #0x4\n" - "st1 { v10.s }[0], [x10], #0x4\n" - "st1 { v9.s }[0], [x9], #0x4\n" - "st1 { v21.s }[0], [x28], #0x4\n" + "st1 { v5.s }[0], [x11], #0x4\n" + "st1 { v21.s }[0], [x10], #0x4\n" + "st1 { v20.s }[0], [x9], #0x4\n" + "st1 { v19.s }[0], [x28], #0x4\n" "tbz x7, #1, 84f\n" - "st1 { v15.h }[2], [x11], #0x2\n" - "st1 { v10.h }[2], [x10], #0x2\n" - "st1 { v9.h }[2], [x9], #0x2\n" - "st1 { v21.h }[2], [x28], #0x2\n" + "st1 { v5.h }[2], [x11], #0x2\n" + "st1 { v21.h }[2], [x10], #0x2\n" + "st1 { v20.h }[2], [x9], #0x2\n" + "st1 { v19.h }[2], [x28], #0x2\n" "tbz x7, #0, 87f\n" - "st1 { v15.b }[6], [x11], #0x1\n" - "st1 { v10.b }[6], [x10], #0x1\n" - "st1 { v9.b }[6], [x9], #0x1\n" - "st1 { v21.b }[6], [x28], #0x1\n" + "st1 { v5.b }[6], [x11], #0x1\n" + "st1 { v21.b }[6], [x10], #0x1\n" + "st1 { v20.b }[6], [x9], #0x1\n" + "st1 { v19.b }[6], [x28], #0x1\n" "b 87f\n" "84:" // Oddments: Bit 2: Bit 1: Unset "tbz x7, #0, 87f\n" - "st1 { v15.b }[4], [x11], #0x1\n" - "st1 { v10.b }[4], [x10], #0x1\n" - "st1 { v9.b }[4], [x9], #0x1\n" - "st1 { v21.b }[4], [x28], #0x1\n" + "st1 { v5.b }[4], [x11], #0x1\n" + "st1 { v21.b }[4], [x10], #0x1\n" + "st1 { v20.b }[4], [x9], #0x1\n" + "st1 { v19.b }[4], [x28], #0x1\n" "b 87f\n" "85:" // Oddments: Bit 2: Unset "tbz x7, #1, 86f\n" - "st1 { v15.h }[0], [x11], #0x2\n" - "st1 { v10.h }[0], [x10], #0x2\n" - "st1 { v9.h }[0], [x9], #0x2\n" - "st1 { v21.h }[0], [x28], #0x2\n" + "st1 { v5.h }[0], [x11], #0x2\n" + "st1 { v21.h }[0], [x10], #0x2\n" + "st1 { v20.h }[0], [x9], #0x2\n" + "st1 { v19.h }[0], [x28], #0x2\n" "tbz x7, #0, 87f\n" - "st1 { v15.b }[2], [x11], #0x1\n" - "st1 { v10.b }[2], [x10], #0x1\n" - "st1 { v9.b }[2], [x9], #0x1\n" - "st1 { v21.b }[2], [x28], #0x1\n" + "st1 { v5.b }[2], [x11], #0x1\n" + "st1 { v21.b }[2], [x10], #0x1\n" + "st1 { v20.b }[2], [x9], #0x1\n" + "st1 { v19.b }[2], [x28], #0x1\n" "b 87f\n" "86:" // Oddments: Bit 2: Unset: Bit 1: Unset "tbz x7, #0, 87f\n" - "st1 { v15.b }[0], [x11], #0x1\n" - "st1 { v10.b }[0], [x10], #0x1\n" - "st1 { v9.b }[0], [x9], #0x1\n" - "st1 { v21.b }[0], [x28], #0x1\n" + "st1 { v5.b }[0], [x11], #0x1\n" + "st1 { v21.b }[0], [x10], #0x1\n" + "st1 { v20.b }[0], [x9], #0x1\n" + "st1 { v19.b }[0], [x28], #0x1\n" "87:" // Oddments: Bit 2: End "88:" // End : diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp index 19767e2823857cbd9a09a3b8e4e2a5c459f2ef6a..32117ad1e628c3415f5653bb6ea9220356b9eb95 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include @@ -34,15 +34,7 @@ namespace arm_conv { namespace depthwise { -void a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( - const unsigned int, - const uint8_t *const *const, - const int8_t *const, - const int32_t *const, - const arm_gemm::Requantize32 &, - const int32_t *const, - const int32_t *const, - uint8_t *const *const); +void a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl(unsigned int, const uint8_t *const *, const int8_t *, const int32_t *, const arm_gemm::Requantize32 &, const int32_t *, const int32_t *, uint8_t *const *); class a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirstStrategy { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp index 1ce037b68c72559370c55a640cc45681cd92d772..df955206e2b8d0452571069ce63ddfe9dadd1f6d 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp @@ -112,1188 +112,1188 @@ void a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( __asm__ __volatile__( "ldr x1, [%x[params], %[offsetof_Params_n_channels]]\n" - "ldr x13, [%x[params], %[offsetof_Params_requant]]\n" + "ldr x23, [%x[params], %[offsetof_Params_requant]]\n" "lsr x2, x1, #0x3\n" - "add x3, x13, %[offsetof_Requantize32_a_offset]\n" - "ld1r { v9.16b }, [x3]\n" - "ldr x24, [%x[params], %[offsetof_Params_outptrs]]\n" - "add x11, x13, %[offsetof_Requantize32_b_offset]\n" - "add x5, x13, %[offsetof_Requantize32_c_offset]\n" - "ld1r { v15.16b }, [x11]\n" - "ld1r { v14.8h }, [x5]\n" - "add x3, x13, %[offsetof_Requantize32_minval]\n" - "add x15, x13, %[offsetof_Requantize32_maxval]\n" - "ld1r { v12.8h }, [x3]\n" - "ld1r { v11.8h }, [x15]\n" - "mov x0, #0x0\n" - "mov x10, #0x0\n" - "add x4, %x[params], %[offsetof_Params_inptrs]\n" - "ldr x3, [%x[params], %[offsetof_Params_weights]]\n" - "ldr x5, [%x[params], %[offsetof_Params_requant_muls]]\n" + "add x20, x23, %[offsetof_Requantize32_a_offset]\n" + "ld1r { v18.16b }, [x20]\n" + "ldr x22, [%x[params], %[offsetof_Params_outptrs]]\n" + "add x21, x23, %[offsetof_Requantize32_b_offset]\n" + "add x20, x23, %[offsetof_Requantize32_c_offset]\n" + "ld1r { v13.16b }, [x21]\n" + "ld1r { v26.8h }, [x20]\n" + "add x21, x23, %[offsetof_Requantize32_minval]\n" + "add x20, x23, %[offsetof_Requantize32_maxval]\n" + "ld1r { v11.8h }, [x21]\n" + "ld1r { v0.8h }, [x20]\n" + "mov x3, #0x0\n" + "mov x4, #0x0\n" + "add x5, %x[params], %[offsetof_Params_inptrs]\n" + "ldr x6, [%x[params], %[offsetof_Params_weights]]\n" + "ldr x7, [%x[params], %[offsetof_Params_requant_muls]]\n" "ldr x8, [%x[params], %[offsetof_Params_requant_shifts]]\n" - "ldp x17, x6, [x24, #0x0]\n" - "ldp x7, x16, [x24, #0x10]\n" + "ldp x17, x16, [x22, #0x0]\n" + "ldp x15, x14, [x22, #0x10]\n" "cbz x2, 3f\n" - "ldr d0, [x3, #0x0]\n" - "ldr d1, [x3, #0x8]\n" + "ldr d6, [x6, #0x0]\n" + "ldr d14, [x6, #0x8]\n" "subs x2, x2, #0x1\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "ldr d2, [x3, #0x10]\n" - "ldr d3, [x3, #0x18]\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "ldr d4, [x3, #0x20]\n" - "ldr x13, [%x[params], %[offsetof_Params_bias]]\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "ldr q13, [x13, #0x0]\n" - "ldr q19, [x13, #0x10]\n" - "add x13, x13, #0x20\n" - "str x13, [%x[params], %[offsetof_Params_bias]]\n" - "ldp x9, x28, [x4, #0x0]\n" - "ldp x27, x26, [x4, #0x10]\n" - "mov v20.16b, v13.16b\n" - "mov v10.16b, v19.16b\n" - "ldp x25, x24, [x4, #0x20]\n" - "ldp x23, x22, [x4, #0x30]\n" - "mov v8.16b, v13.16b\n" - "mov v7.16b, v19.16b\n" - "ldp x21, x20, [x4, #0x40]\n" - "ldr d31, [x9, x0]\n" - "mov v17.16b, v13.16b\n" - "mov v21.16b, v19.16b\n" - "ldr d30, [x28, x0]\n" - "ldr d29, [x27, x0]\n" - "usubl v31.8h, v31.8b, v9.8b\n" - "usubl v30.8h, v30.8b, v9.8b\n" - "ldr d28, [x26, x0]\n" - "ldr d27, [x25, x0]\n" - "usubl v29.8h, v29.8b, v9.8b\n" - "usubl v28.8h, v28.8b, v9.8b\n" - "ldr d23, [x24, x0]\n" - "ldr d25, [x23, x0]\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "usubl v23.8h, v23.8b, v9.8b\n" - "ldr d24, [x22, x0]\n" - "ldr d26, [x21, x0]\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "ldr d22, [x20, x0]\n" - "usubl v26.8h, v26.8b, v9.8b\n" - "usubl v22.8h, v22.8b, v9.8b\n" + "ssubl v6.8h, v6.8b, v13.8b\n" + "ldr d10, [x6, #0x10]\n" + "ldr d21, [x6, #0x18]\n" + "ssubl v14.8h, v14.8b, v13.8b\n" + "ssubl v10.8h, v10.8b, v13.8b\n" + "ldr d12, [x6, #0x20]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "ssubl v21.8h, v21.8b, v13.8b\n" + "ssubl v12.8h, v12.8b, v13.8b\n" + "ldr q7, [x20, #0x0]\n" + "ldr q15, [x20, #0x10]\n" + "add x20, x20, #0x20\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" + "ldp x9, x28, [x5, #0x0]\n" + "ldp x27, x26, [x5, #0x10]\n" + "mov v20.16b, v7.16b\n" + "mov v5.16b, v15.16b\n" + "ldp x25, x24, [x5, #0x20]\n" + "ldp x23, x22, [x5, #0x30]\n" + "mov v24.16b, v7.16b\n" + "mov v22.16b, v15.16b\n" + "ldp x21, x20, [x5, #0x40]\n" + "ldr d31, [x9, x3]\n" + "mov v23.16b, v7.16b\n" + "mov v19.16b, v15.16b\n" + "ldr d17, [x28, x3]\n" + "ldr d30, [x27, x3]\n" + "usubl v31.8h, v31.8b, v18.8b\n" + "usubl v17.8h, v17.8b, v18.8b\n" + "ldr d16, [x26, x3]\n" + "ldr d3, [x25, x3]\n" + "usubl v30.8h, v30.8b, v18.8b\n" + "usubl v16.8h, v16.8b, v18.8b\n" + "ldr d4, [x24, x3]\n" + "ldr d25, [x23, x3]\n" + "usubl v3.8h, v3.8b, v18.8b\n" + "usubl v4.8h, v4.8b, v18.8b\n" + "ldr d9, [x22, x3]\n" + "ldr d29, [x21, x3]\n" + "usubl v25.8h, v25.8b, v18.8b\n" + "usubl v9.8h, v9.8b, v18.8b\n" + "ldr d28, [x20, x3]\n" + "usubl v29.8h, v29.8b, v18.8b\n" + "usubl v28.8h, v28.8b, v18.8b\n" "beq 2f\n" "1:" // Loop - "ldr q18, [x5, #0x0]\n" - "ldr q6, [x8, #0x0]\n" - "smlal v13.4s, v31.4h, v0.4h\n" - "smlal2 v19.4s, v31.8h, v0.8h\n" - "ldr q5, [x5, #0x10]\n" - "smlal v13.4s, v30.4h, v1.4h\n" - "ldr x20, [x4, #0x50]\n" - "smlal v20.4s, v30.4h, v0.4h\n" - "smlal v8.4s, v29.4h, v0.4h\n" - "smlal v17.4s, v28.4h, v0.4h\n" - "ldr x22, [x4, #0x58]\n" - "ldr x21, [x4, #0x60]\n" - "smlal2 v19.4s, v30.8h, v1.8h\n" - "smlal2 v10.4s, v30.8h, v0.8h\n" - "ldr d31, [x20, x0]\n" - "usubl v31.8h, v31.8b, v9.8b\n" - "smlal2 v7.4s, v29.8h, v0.8h\n" - "smlal v13.4s, v27.4h, v2.4h\n" - "ldr x20, [x4, #0x68]\n" - "ldr x26, [x4, #0x70]\n" - "smlal2 v21.4s, v28.8h, v0.8h\n" - "ldr d30, [x22, x0]\n" - "smlal v20.4s, v27.4h, v1.4h\n" - "usubl v30.8h, v30.8b, v9.8b\n" - "smlal v8.4s, v28.4h, v1.4h\n" - "smlal v17.4s, v23.4h, v1.4h\n" - "ldr x25, [x4, #0x78]\n" - "ldr x23, [x4, #0x80]\n" - "smlal2 v19.4s, v27.8h, v2.8h\n" - "smlal2 v10.4s, v27.8h, v1.8h\n" - "ldr d0, [x3, #0x28]\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "smlal2 v7.4s, v28.8h, v1.8h\n" - "smlal v13.4s, v25.4h, v3.4h\n" - "ldr x24, [x4, #0x88]\n" - "ldr x15, [x4, #0x90]\n" - "smlal2 v21.4s, v23.8h, v1.8h\n" - "ldr d27, [x21, x0]\n" - "smlal v20.4s, v25.4h, v2.4h\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal v8.4s, v23.4h, v2.4h\n" - "smlal v17.4s, v31.4h, v2.4h\n" - "ldr x21, [x4, #0x98]\n" - "ldr x14, [x4, #0xa0]\n" - "smlal2 v19.4s, v25.8h, v3.8h\n" - "smlal2 v10.4s, v25.8h, v2.8h\n" - "ldr d1, [x3, #0x30]\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "smlal2 v7.4s, v23.8h, v2.8h\n" - "smlal v13.4s, v24.4h, v4.4h\n" - "ldr x13, [x4, #0xa8]\n" - "ldr x12, [x4, #0xb0]\n" - "smlal2 v21.4s, v31.8h, v2.8h\n" - "ldr d25, [x20, x0]\n" - "smlal v20.4s, v24.4h, v3.4h\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "smlal v8.4s, v31.4h, v3.4h\n" - "smlal v17.4s, v30.4h, v3.4h\n" - "ldr x20, [x4, #0xb8]\n" - "ldr x11, [x4, #0xc0]\n" - "smlal2 v19.4s, v24.8h, v4.8h\n" - "smlal2 v10.4s, v24.8h, v3.8h\n" - "ldr d2, [x3, #0x38]\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "smlal2 v7.4s, v31.8h, v3.8h\n" - "smlal v13.4s, v29.4h, v0.4h\n" - "ldr x22, [x4, #0xc8]\n" - "ldr x9, [x4, #0xd0]\n" - "smlal2 v21.4s, v30.8h, v3.8h\n" - "ldr d24, [x26, x0]\n" - "smlal v20.4s, v27.4h, v4.4h\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "smlal v8.4s, v30.4h, v4.4h\n" - "smlal v17.4s, v26.4h, v4.4h\n" - "ldr x28, [x4, #0xd8]\n" - "ldr x27, [x4, #0xe0]\n" - "smlal2 v19.4s, v29.8h, v0.8h\n" - "ldr d3, [x3, #0x40]\n" - "smlal2 v10.4s, v27.8h, v4.8h\n" - "ldr d27, [x25, x0]\n" - "smlal2 v7.4s, v30.8h, v4.8h\n" - "smlal v13.4s, v28.4h, v1.4h\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "ldr x26, [x4, #0xe8]\n" - "smlal2 v21.4s, v26.8h, v4.8h\n" - "ldr d4, [x3, #0x48]\n" - "smlal v20.4s, v28.4h, v0.4h\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal v8.4s, v22.4h, v0.4h\n" - "smlal v17.4s, v25.4h, v0.4h\n" - "ssubl v4.8h, v4.8b, v15.8b\n" + "ldr d2, [x6, #0x28]\n" + "ldr d27, [x6, #0x30]\n" + "smlal v7.4s, v31.4h, v6.4h\n" + "smlal2 v15.4s, v31.8h, v6.8h\n" + "ldr d1, [x6, #0x38]\n" + "ldr d31, [x6, #0x40]\n" + "smlal v7.4s, v17.4h, v14.4h\n" + "smlal v20.4s, v17.4h, v6.4h\n" + "ldr d8, [x6, #0x48]\n" + "ldr x22, [x5, #0x50]\n" + "smlal v24.4s, v30.4h, v6.4h\n" + "smlal v23.4s, v16.4h, v6.4h\n" + "smlal2 v15.4s, v17.8h, v14.8h\n" + "smlal v7.4s, v3.4h, v10.4h\n" + "ldr x20, [x5, #0x58]\n" + "ldr x21, [x5, #0x60]\n" + "smlal2 v5.4s, v17.8h, v6.8h\n" + "ldr d17, [x22, x3]\n" + "smlal2 v22.4s, v30.8h, v6.8h\n" + "usubl v17.8h, v17.8b, v18.8b\n" + "smlal2 v19.4s, v16.8h, v6.8h\n" + "ldr d6, [x20, x3]\n" + "smlal v20.4s, v3.4h, v14.4h\n" + "usubl v6.8h, v6.8b, v18.8b\n" + "smlal v24.4s, v16.4h, v14.4h\n" + "smlal v23.4s, v4.4h, v14.4h\n" + "ssubl v2.8h, v2.8b, v13.8b\n" + "ldr x20, [x5, #0x68]\n" + "smlal2 v15.4s, v3.8h, v10.8h\n" + "smlal v7.4s, v25.4h, v21.4h\n" + "ssubl v27.8h, v27.8b, v13.8b\n" + "ldr x22, [x5, #0x70]\n" + "smlal2 v5.4s, v3.8h, v14.8h\n" + "ldr d3, [x21, x3]\n" + "smlal2 v22.4s, v16.8h, v14.8h\n" + "usubl v3.8h, v3.8b, v18.8b\n" + "smlal2 v19.4s, v4.8h, v14.8h\n" + "ldr d14, [x20, x3]\n" + "smlal v20.4s, v25.4h, v10.4h\n" + "usubl v14.8h, v14.8b, v18.8b\n" + "smlal v24.4s, v4.4h, v10.4h\n" + "smlal v23.4s, v17.4h, v10.4h\n" + "ssubl v1.8h, v1.8b, v13.8b\n" + "ldr x20, [x5, #0x78]\n" + "smlal2 v15.4s, v25.8h, v21.8h\n" + "smlal v7.4s, v9.4h, v12.4h\n" + "ssubl v31.8h, v31.8b, v13.8b\n" + "ldr x21, [x5, #0x80]\n" + "smlal2 v5.4s, v25.8h, v10.8h\n" + "ldr d25, [x22, x3]\n" + "smlal2 v22.4s, v4.8h, v10.8h\n" + "usubl v25.8h, v25.8b, v18.8b\n" + "smlal2 v19.4s, v17.8h, v10.8h\n" + "ldr d10, [x20, x3]\n" + "smlal v20.4s, v9.4h, v21.4h\n" + "usubl v10.8h, v10.8b, v18.8b\n" + "smlal v24.4s, v17.4h, v21.4h\n" + "smlal v23.4s, v6.4h, v21.4h\n" + "ssubl v8.8h, v8.8b, v13.8b\n" + "ldr x24, [x5, #0x88]\n" + "smlal2 v15.4s, v9.8h, v12.8h\n" + "smlal v7.4s, v30.4h, v2.4h\n" + "ldr x20, [x5, #0x90]\n" + "ldr x23, [x5, #0x98]\n" + "smlal2 v5.4s, v9.8h, v21.8h\n" + "ldr d9, [x21, x3]\n" + "smlal2 v22.4s, v17.8h, v21.8h\n" + "usubl v9.8h, v9.8b, v18.8b\n" + "smlal2 v19.4s, v6.8h, v21.8h\n" + "ldr d21, [x6, #0x50]\n" + "smlal v20.4s, v3.4h, v12.4h\n" + "ssubl v21.8h, v21.8b, v13.8b\n" + "smlal v24.4s, v6.4h, v12.4h\n" + "smlal v23.4s, v29.4h, v12.4h\n" + "ldr x22, [x5, #0xa0]\n" + "ldr x21, [x5, #0xa8]\n" + "smlal2 v15.4s, v30.8h, v2.8h\n" + "ldr d30, [x24, x3]\n" + "smlal v7.4s, v16.4h, v27.4h\n" + "usubl v30.8h, v30.8b, v18.8b\n" + "smlal2 v5.4s, v3.8h, v12.8h\n" + "ldr d3, [x6, #0x58]\n" + "smlal2 v22.4s, v6.8h, v12.8h\n" + "ssubl v3.8h, v3.8b, v13.8b\n" + "smlal2 v19.4s, v29.8h, v12.8h\n" + "ldr d12, [x20, x3]\n" + "smlal v20.4s, v16.4h, v2.4h\n" + "usubl v12.8h, v12.8b, v18.8b\n" + "smlal v24.4s, v28.4h, v2.4h\n" + "smlal v23.4s, v14.4h, v2.4h\n" + "ldr x20, [x5, #0xb0]\n" + "ldr x13, [x5, #0xb8]\n" + "smlal2 v15.4s, v16.8h, v27.8h\n" + "smlal v7.4s, v4.4h, v1.4h\n" + "ldr x12, [x5, #0xc0]\n" + "ldr x11, [x5, #0xc8]\n" + "smlal2 v5.4s, v16.8h, v2.8h\n" + "ldr d16, [x23, x3]\n" + "smlal2 v22.4s, v28.8h, v2.8h\n" + "usubl v16.8h, v16.8b, v18.8b\n" + "smlal2 v19.4s, v14.8h, v2.8h\n" + "ldr d2, [x6, #0x60]\n" + "smlal v20.4s, v4.4h, v27.4h\n" + "ssubl v2.8h, v2.8b, v13.8b\n" + "smlal v24.4s, v14.4h, v27.4h\n" + "smlal v23.4s, v25.4h, v27.4h\n" + "ldr x10, [x5, #0xd0]\n" + "ldr x9, [x5, #0xd8]\n" + "smlal2 v15.4s, v4.8h, v1.8h\n" + "smlal v7.4s, v17.4h, v31.4h\n" + "ldr x28, [x5, #0xe0]\n" + "ldr x27, [x5, #0xe8]\n" + "smlal2 v5.4s, v4.8h, v27.8h\n" + "ldr d4, [x22, x3]\n" + "smlal2 v22.4s, v14.8h, v27.8h\n" + "usubl v4.8h, v4.8b, v18.8b\n" + "smlal2 v19.4s, v25.8h, v27.8h\n" + "ldr d27, [x6, #0x68]\n" + "smlal v20.4s, v17.4h, v1.4h\n" + "ssubl v27.8h, v27.8b, v13.8b\n" + "smlal v24.4s, v25.4h, v1.4h\n" + "smlal v23.4s, v10.4h, v1.4h\n" + "ldr x26, [x5, #0xf0]\n" + "ldr x25, [x5, #0xf8]\n" + "smlal2 v15.4s, v17.8h, v31.8h\n" + "smlal v7.4s, v6.4h, v8.4h\n" + "ldr x24, [x5, #0x100]\n" + "ldr x23, [x5, #0x108]\n" + "smlal2 v5.4s, v17.8h, v1.8h\n" + "ldr d17, [x21, x3]\n" + "smlal2 v22.4s, v25.8h, v1.8h\n" + "usubl v17.8h, v17.8b, v18.8b\n" + "smlal2 v19.4s, v10.8h, v1.8h\n" + "ldr d1, [x6, #0x70]\n" + "smlal v20.4s, v6.4h, v31.4h\n" + "ssubl v1.8h, v1.8b, v13.8b\n" + "smlal v24.4s, v10.4h, v31.4h\n" + "smlal v23.4s, v9.4h, v31.4h\n" + "ldr x22, [x5, #0x110]\n" + "ldr x21, [x5, #0x118]\n" + "smlal2 v15.4s, v6.8h, v8.8h\n" + "smlal v7.4s, v28.4h, v21.4h\n" "subs x2, x2, #0x1\n" + "smlal2 v5.4s, v6.8h, v31.8h\n" + "ldr d6, [x20, x3]\n" + "smlal2 v22.4s, v10.8h, v31.8h\n" + "usubl v6.8h, v6.8b, v18.8b\n" + "smlal2 v19.4s, v9.8h, v31.8h\n" + "ldr d31, [x6, #0x78]\n" + "smlal v20.4s, v29.4h, v8.4h\n" + "ssubl v31.8h, v31.8b, v13.8b\n" + "smlal v24.4s, v9.4h, v8.4h\n" + "smlal v23.4s, v30.4h, v8.4h\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + "smlal2 v15.4s, v28.8h, v21.8h\n" + "ldr d28, [x13, x3]\n" + "smlal v7.4s, v14.4h, v3.4h\n" + "usubl v28.8h, v28.8b, v18.8b\n" + "smlal2 v5.4s, v29.8h, v8.8h\n" + "ldr d29, [x6, #0x80]\n" + "smlal2 v22.4s, v9.8h, v8.8h\n" + "ssubl v29.8h, v29.8b, v13.8b\n" + "smlal2 v19.4s, v30.8h, v8.8h\n" + "ldr d8, [x12, x3]\n" + "smlal v20.4s, v14.4h, v21.4h\n" + "usubl v8.8h, v8.8b, v18.8b\n" + "smlal v24.4s, v12.4h, v21.4h\n" + "smlal v23.4s, v16.4h, v21.4h\n" + "smlal2 v15.4s, v14.8h, v3.8h\n" + "smlal v7.4s, v25.4h, v2.4h\n" + "smlal2 v5.4s, v14.8h, v21.8h\n" + "ldr d14, [x11, x3]\n" + "smlal2 v22.4s, v12.8h, v21.8h\n" + "usubl v14.8h, v14.8b, v18.8b\n" + "smlal2 v19.4s, v16.8h, v21.8h\n" + "ldr d21, [x6, #0x88]\n" + "smlal v20.4s, v25.4h, v3.4h\n" + "ssubl v21.8h, v21.8b, v13.8b\n" + "smlal v24.4s, v16.4h, v3.4h\n" + "smlal v23.4s, v4.4h, v3.4h\n" + "smlal2 v15.4s, v25.8h, v2.8h\n" + "smlal v7.4s, v10.4h, v27.4h\n" + "smlal2 v5.4s, v25.8h, v3.8h\n" + "ldr d25, [x10, x3]\n" + "smlal2 v22.4s, v16.8h, v3.8h\n" + "usubl v25.8h, v25.8b, v18.8b\n" + "smlal2 v19.4s, v4.8h, v3.8h\n" + "ldr d3, [x6, #0x90]\n" + "smlal v20.4s, v10.4h, v2.4h\n" + "ssubl v3.8h, v3.8b, v13.8b\n" + "smlal v24.4s, v4.4h, v2.4h\n" + "smlal v23.4s, v17.4h, v2.4h\n" + "smlal2 v15.4s, v10.8h, v27.8h\n" + "smlal v7.4s, v9.4h, v1.4h\n" + "smlal2 v5.4s, v10.8h, v2.8h\n" + "ldr d10, [x9, x3]\n" + "smlal2 v22.4s, v4.8h, v2.8h\n" + "usubl v10.8h, v10.8b, v18.8b\n" + "smlal2 v19.4s, v17.8h, v2.8h\n" + "ldr d2, [x6, #0x98]\n" + "smlal v20.4s, v9.4h, v27.4h\n" + "ssubl v2.8h, v2.8b, v13.8b\n" + "smlal v24.4s, v17.4h, v27.4h\n" + "smlal v23.4s, v6.4h, v27.4h\n" + "smlal2 v15.4s, v9.8h, v1.8h\n" + "smlal v7.4s, v12.4h, v31.4h\n" + "smlal2 v5.4s, v9.8h, v27.8h\n" + "ldr d9, [x28, x3]\n" + "smlal2 v22.4s, v17.8h, v27.8h\n" + "usubl v9.8h, v9.8b, v18.8b\n" + "smlal2 v19.4s, v6.8h, v27.8h\n" + "ldr d27, [x6, #0xa0]\n" + "smlal v20.4s, v30.4h, v1.4h\n" + "ssubl v27.8h, v27.8b, v13.8b\n" + "smlal v24.4s, v6.4h, v1.4h\n" + "smlal v23.4s, v28.4h, v1.4h\n" + "smlal2 v15.4s, v12.8h, v31.8h\n" + "ldr d12, [x27, x3]\n" + "smlal v7.4s, v16.4h, v29.4h\n" + "usubl v12.8h, v12.8b, v18.8b\n" + "smlal2 v5.4s, v30.8h, v1.8h\n" + "ldr d30, [x6, #0xa8]\n" + "smlal2 v22.4s, v6.8h, v1.8h\n" + "ssubl v30.8h, v30.8b, v13.8b\n" "smlal2 v19.4s, v28.8h, v1.8h\n" - "smlal2 v10.4s, v28.8h, v0.8h\n" - "ldr d28, [x24, x0]\n" - "usubl v28.8h, v28.8b, v9.8b\n" - "smlal2 v7.4s, v22.8h, v0.8h\n" - "smlal v13.4s, v23.4h, v2.4h\n" - "ldr x25, [x4, #0xf0]\n" - "add x5, x5, #0x20\n" - "smlal2 v21.4s, v25.8h, v0.8h\n" - "ldr d0, [x3, #0x50]\n" - "smlal v20.4s, v23.4h, v1.4h\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "smlal v8.4s, v25.4h, v1.4h\n" - "smlal v17.4s, v24.4h, v1.4h\n" - "smlal2 v19.4s, v23.8h, v2.8h\n" - "smlal2 v10.4s, v23.8h, v1.8h\n" - "ldr d23, [x23, x0]\n" - "usubl v23.8h, v23.8b, v9.8b\n" - "smlal2 v7.4s, v25.8h, v1.8h\n" - "smlal v13.4s, v31.4h, v3.4h\n" - "ldr x24, [x4, #0xf8]\n" - "smlal2 v21.4s, v24.8h, v1.8h\n" - "ldr d1, [x3, #0x58]\n" - "smlal v20.4s, v31.4h, v2.4h\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "smlal v8.4s, v24.4h, v2.4h\n" - "smlal v17.4s, v27.4h, v2.4h\n" - "smlal2 v19.4s, v31.8h, v3.8h\n" - "smlal2 v10.4s, v31.8h, v2.8h\n" - "ldr d31, [x15, x0]\n" - "usubl v31.8h, v31.8b, v9.8b\n" - "smlal2 v7.4s, v24.8h, v2.8h\n" - "smlal v13.4s, v30.4h, v4.4h\n" - "ldr x23, [x4, #0x100]\n" - "smlal2 v21.4s, v27.8h, v2.8h\n" - "ldr d2, [x3, #0x60]\n" - "smlal v20.4s, v30.4h, v3.4h\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "smlal v8.4s, v27.4h, v3.4h\n" - "smlal v17.4s, v23.4h, v3.4h\n" - "smlal2 v19.4s, v30.8h, v4.8h\n" - "smlal2 v10.4s, v30.8h, v3.8h\n" - "ldr d30, [x21, x0]\n" - "usubl v30.8h, v30.8b, v9.8b\n" - "smlal2 v7.4s, v27.8h, v3.8h\n" - "smlal v13.4s, v22.4h, v0.4h\n" - "ldr x15, [x4, #0x108]\n" - "smlal2 v21.4s, v23.8h, v3.8h\n" - "ldr d3, [x3, #0x68]\n" - "smlal v20.4s, v26.4h, v4.4h\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "smlal v8.4s, v23.4h, v4.4h\n" - "smlal v17.4s, v28.4h, v4.4h\n" - "smlal2 v19.4s, v22.8h, v0.8h\n" - "ldr d22, [x20, x0]\n" - "smlal2 v10.4s, v26.8h, v4.8h\n" - "ldr d26, [x14, x0]\n" - "smlal2 v7.4s, v23.8h, v4.8h\n" - "smlal v13.4s, v25.4h, v1.4h\n" - "usubl v26.8h, v26.8b, v9.8b\n" - "ldr x21, [x4, #0x110]\n" - "smlal2 v21.4s, v28.8h, v4.8h\n" - "ldr d4, [x3, #0x70]\n" - "smlal v20.4s, v25.4h, v0.4h\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "smlal v8.4s, v31.4h, v0.4h\n" - "smlal v17.4s, v30.4h, v0.4h\n" - "usubl v22.8h, v22.8b, v9.8b\n" - "ldr x20, [x4, #0x118]\n" - "smlal2 v19.4s, v25.8h, v1.8h\n" - "smlal2 v10.4s, v25.8h, v0.8h\n" - "ldr d25, [x13, x0]\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "smlal2 v7.4s, v31.8h, v0.8h\n" - "smlal v13.4s, v24.4h, v2.4h\n" - "ldr x13, [%x[params], %[offsetof_Params_bias]]\n" - "smlal2 v21.4s, v30.8h, v0.8h\n" - "ldr d0, [x3, #0x78]\n" - "smlal v20.4s, v24.4h, v1.4h\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "smlal v8.4s, v30.4h, v1.4h\n" - "smlal v17.4s, v26.4h, v1.4h\n" - "smlal2 v19.4s, v24.8h, v2.8h\n" - "smlal2 v10.4s, v24.8h, v1.8h\n" - "ldr d24, [x12, x0]\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "smlal2 v7.4s, v30.8h, v1.8h\n" - "smlal v13.4s, v27.4h, v3.4h\n" - "smlal2 v21.4s, v26.8h, v1.8h\n" - "ldr d1, [x3, #0x80]\n" - "smlal v20.4s, v27.4h, v2.4h\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "smlal v8.4s, v26.4h, v2.4h\n" - "smlal v17.4s, v25.4h, v2.4h\n" - "smlal2 v19.4s, v27.8h, v3.8h\n" - "smlal2 v10.4s, v27.8h, v2.8h\n" - "ldr d27, [x11, x0]\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal2 v7.4s, v26.8h, v2.8h\n" - "smlal v13.4s, v23.4h, v4.4h\n" - "smlal2 v21.4s, v25.8h, v2.8h\n" - "ldr d2, [x3, #0x88]\n" - "smlal v20.4s, v23.4h, v3.4h\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "smlal v8.4s, v25.4h, v3.4h\n" - "smlal v17.4s, v24.4h, v3.4h\n" - "smlal2 v19.4s, v23.8h, v4.8h\n" - "smlal2 v10.4s, v23.8h, v3.8h\n" - "ldr d23, [x22, x0]\n" - "usubl v23.8h, v23.8b, v9.8b\n" - "smlal2 v7.4s, v25.8h, v3.8h\n" - "smlal v13.4s, v31.4h, v0.4h\n" - "smlal2 v21.4s, v24.8h, v3.8h\n" - "ldr d3, [x3, #0x90]\n" - "smlal v20.4s, v28.4h, v4.4h\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "smlal v8.4s, v24.4h, v4.4h\n" - "smlal v17.4s, v22.4h, v4.4h\n" - "smlal2 v19.4s, v31.8h, v0.8h\n" - "ldr d31, [x9, x0]\n" - "smlal2 v10.4s, v28.8h, v4.8h\n" - "ldr d28, [x27, x0]\n" - "smlal2 v7.4s, v24.8h, v4.8h\n" - "smlal v13.4s, v30.4h, v1.4h\n" - "usubl v31.8h, v31.8b, v9.8b\n" - "smlal2 v21.4s, v22.8h, v4.8h\n" - "ldr d4, [x3, #0x98]\n" - "smlal v20.4s, v30.4h, v0.4h\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "smlal v8.4s, v27.4h, v0.4h\n" - "smlal v17.4s, v23.4h, v0.4h\n" - "usubl v28.8h, v28.8b, v9.8b\n" - "smlal2 v19.4s, v30.8h, v1.8h\n" - "smlal2 v10.4s, v30.8h, v0.8h\n" - "ldr d30, [x28, x0]\n" - "usubl v30.8h, v30.8b, v9.8b\n" - "smlal2 v7.4s, v27.8h, v0.8h\n" - "smlal v13.4s, v26.4h, v2.4h\n" - "smlal2 v21.4s, v23.8h, v0.8h\n" - "ldr d0, [x3, #0xa0]\n" - "smlal v20.4s, v26.4h, v1.4h\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "smlal v8.4s, v23.4h, v1.4h\n" - "smlal v17.4s, v31.4h, v1.4h\n" - "smlal2 v19.4s, v26.8h, v2.8h\n" - "smlal2 v10.4s, v26.8h, v1.8h\n" - "ldr d26, [x26, x0]\n" - "usubl v26.8h, v26.8b, v9.8b\n" - "smlal2 v7.4s, v23.8h, v1.8h\n" - "smlal v13.4s, v25.4h, v3.4h\n" - "smlal2 v21.4s, v31.8h, v1.8h\n" - "ldr d1, [x3, #0xa8]\n" - "smlal v20.4s, v25.4h, v2.4h\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "smlal v8.4s, v31.4h, v2.4h\n" - "smlal v17.4s, v30.4h, v2.4h\n" - "smlal2 v19.4s, v25.8h, v3.8h\n" - "smlal2 v10.4s, v25.8h, v2.8h\n" - "ldr d25, [x25, x0]\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "smlal2 v7.4s, v31.8h, v2.8h\n" - "smlal v13.4s, v24.4h, v4.4h\n" - "smlal2 v21.4s, v30.8h, v2.8h\n" - "ldr d2, [x3, #0xb0]\n" - "smlal v20.4s, v24.4h, v3.4h\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "smlal v8.4s, v30.4h, v3.4h\n" - "smlal v17.4s, v28.4h, v3.4h\n" - "smlal2 v19.4s, v24.8h, v4.8h\n" - "smlal2 v10.4s, v24.8h, v3.8h\n" - "ldr d24, [x24, x0]\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "smlal2 v7.4s, v30.8h, v3.8h\n" - "smlal v13.4s, v27.4h, v0.4h\n" - "smlal2 v21.4s, v28.8h, v3.8h\n" - "ldr d3, [x3, #0xb8]\n" - "smlal v20.4s, v22.4h, v4.4h\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "smlal v8.4s, v28.4h, v4.4h\n" - "smlal v17.4s, v26.4h, v4.4h\n" - "smlal2 v19.4s, v27.8h, v0.8h\n" - "ldr d27, [x23, x0]\n" - "smlal2 v7.4s, v28.8h, v4.8h\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal v13.4s, v23.4h, v1.4h\n" - "smlal2 v10.4s, v22.8h, v4.8h\n" - "ldr q22, [x8, #0x10]\n" + "ldr d1, [x26, x3]\n" + "smlal v20.4s, v16.4h, v31.4h\n" + "usubl v1.8h, v1.8b, v18.8b\n" + "smlal v24.4s, v8.4h, v31.4h\n" + "smlal v23.4s, v14.4h, v31.4h\n" + "smlal2 v15.4s, v16.8h, v29.8h\n" + "smlal v7.4s, v4.4h, v21.4h\n" + "smlal2 v5.4s, v16.8h, v31.8h\n" + "ldr d16, [x25, x3]\n" + "smlal2 v22.4s, v8.8h, v31.8h\n" + "usubl v16.8h, v16.8b, v18.8b\n" + "smlal2 v19.4s, v14.8h, v31.8h\n" + "ldr d31, [x6, #0xb0]\n" + "smlal v20.4s, v4.4h, v29.4h\n" + "ssubl v31.8h, v31.8b, v13.8b\n" + "smlal v24.4s, v14.4h, v29.4h\n" + "smlal v23.4s, v25.4h, v29.4h\n" + "smlal2 v15.4s, v4.8h, v21.8h\n" + "smlal v7.4s, v17.4h, v3.4h\n" + "smlal2 v5.4s, v4.8h, v29.8h\n" + "ldr d4, [x24, x3]\n" + "smlal2 v22.4s, v14.8h, v29.8h\n" + "usubl v4.8h, v4.8b, v18.8b\n" + "smlal2 v19.4s, v25.8h, v29.8h\n" + "ldr d29, [x6, #0xb8]\n" + "smlal v20.4s, v17.4h, v21.4h\n" + "ssubl v29.8h, v29.8b, v13.8b\n" + "smlal v24.4s, v25.4h, v21.4h\n" + "smlal v23.4s, v10.4h, v21.4h\n" + "smlal2 v15.4s, v17.8h, v3.8h\n" + "smlal v7.4s, v6.4h, v2.4h\n" + "smlal2 v5.4s, v17.8h, v21.8h\n" + "ldr d17, [x23, x3]\n" + "smlal2 v22.4s, v25.8h, v21.8h\n" + "usubl v17.8h, v17.8b, v18.8b\n" + "smlal2 v19.4s, v10.8h, v21.8h\n" + "ldr d21, [x6, #0xc0]\n" + "smlal v20.4s, v6.4h, v3.4h\n" + "ssubl v21.8h, v21.8b, v13.8b\n" + "smlal v24.4s, v10.4h, v3.4h\n" + "smlal v23.4s, v9.4h, v3.4h\n" + "add x6, x6, #0xc8\n" + "smlal2 v15.4s, v6.8h, v2.8h\n" + "smlal v7.4s, v8.4h, v27.4h\n" + "smlal2 v5.4s, v6.8h, v3.8h\n" + "ldr d6, [x22, x3]\n" + "smlal2 v22.4s, v10.8h, v3.8h\n" + "usubl v6.8h, v6.8b, v18.8b\n" + "smlal2 v19.4s, v9.8h, v3.8h\n" + "ldr d3, [x21, x3]\n" + "smlal v20.4s, v28.4h, v2.4h\n" + "usubl v3.8h, v3.8b, v18.8b\n" + "smlal v24.4s, v9.4h, v2.4h\n" + "smlal v23.4s, v12.4h, v2.4h\n" + "add x3, x3, #0x8\n" + "smlal2 v15.4s, v8.8h, v27.8h\n" + "ldr q8, [x7, #0x0]\n" + "smlal v7.4s, v14.4h, v30.4h\n" + "smlal2 v5.4s, v28.8h, v2.8h\n" + "ldr q28, [x8, #0x0]\n" + "smlal2 v22.4s, v9.8h, v2.8h\n" + "smlal2 v19.4s, v12.8h, v2.8h\n" + "ldr q2, [x7, #0x10]\n" + "smlal v20.4s, v14.4h, v27.4h\n" + "add x7, x7, #0x20\n" + "smlal v24.4s, v1.4h, v27.4h\n" + "smlal v23.4s, v16.4h, v27.4h\n" + "smlal2 v15.4s, v14.8h, v30.8h\n" + "smlal v7.4s, v25.4h, v31.4h\n" + "smlal2 v5.4s, v14.8h, v27.8h\n" + "ldr q14, [x8, #0x10]\n" + "smlal2 v22.4s, v1.8h, v27.8h\n" "add x8, x8, #0x20\n" - "smlal2 v21.4s, v26.8h, v4.8h\n" - "ldr d4, [x3, #0xc0]\n" - "smlal v20.4s, v23.4h, v0.4h\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "smlal v8.4s, v25.4h, v0.4h\n" - "smlal v17.4s, v24.4h, v0.4h\n" - "add x3, x3, #0xc8\n" - "smlal2 v19.4s, v23.8h, v1.8h\n" - "smlal2 v7.4s, v25.8h, v0.8h\n" - "ldr d25, [x15, x0]\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "smlal v13.4s, v31.4h, v2.4h\n" - "smlal2 v10.4s, v23.8h, v0.8h\n" - "smlal2 v21.4s, v24.8h, v0.8h\n" - "smlal v20.4s, v31.4h, v1.4h\n" - "smlal v8.4s, v24.4h, v1.4h\n" - "smlal v17.4s, v27.4h, v1.4h\n" - "smlal2 v19.4s, v31.8h, v2.8h\n" - "smlal2 v7.4s, v24.8h, v1.8h\n" - "ldr d24, [x21, x0]\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "smlal v13.4s, v30.4h, v3.4h\n" - "smlal2 v10.4s, v31.8h, v1.8h\n" - "smlal2 v21.4s, v27.8h, v1.8h\n" - "smlal v20.4s, v30.4h, v2.4h\n" - "smlal v8.4s, v27.4h, v2.4h\n" - "smlal v17.4s, v25.4h, v2.4h\n" - "smlal2 v19.4s, v30.8h, v3.8h\n" - "smlal2 v7.4s, v27.8h, v2.8h\n" - "ldr d27, [x20, x0]\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal v13.4s, v28.4h, v4.4h\n" - "smlal2 v10.4s, v30.8h, v2.8h\n" - "sqrdmulh v13.4s, v13.4s, v18.4s\n" - "add x0, x0, #0x8\n" - "smlal2 v21.4s, v25.8h, v2.8h\n" - "smlal v20.4s, v28.4h, v3.4h\n" - "and v30.16b, v13.16b, v6.16b\n" - "smlal v8.4s, v25.4h, v3.4h\n" - "smlal v17.4s, v24.4h, v3.4h\n" - "sshr v30.4s, v30.4s, #0x1f\n" - "smlal2 v19.4s, v28.8h, v4.8h\n" - "smlal2 v10.4s, v28.8h, v3.8h\n" - "sqrdmulh v19.4s, v19.4s, v5.4s\n" - "smlal2 v7.4s, v25.8h, v3.8h\n" - "smlal2 v21.4s, v24.8h, v3.8h\n" - "and v16.16b, v19.16b, v22.16b\n" - "smlal v20.4s, v26.4h, v4.4h\n" - "smlal v8.4s, v24.4h, v4.4h\n" - "sqrdmulh v20.4s, v20.4s, v18.4s\n" - "smlal v17.4s, v27.4h, v4.4h\n" - "smlal2 v10.4s, v26.8h, v4.8h\n" - "sqrdmulh v8.4s, v8.4s, v18.4s\n" - "smlal2 v7.4s, v24.8h, v4.8h\n" - "smlal2 v21.4s, v27.8h, v4.8h\n" - "sqrdmulh v17.4s, v17.4s, v18.4s\n" - "sqadd v13.4s, v13.4s, v30.4s\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "and v0.16b, v20.16b, v6.16b\n" - "sqrdmulh v10.4s, v10.4s, v5.4s\n" - "and v18.16b, v8.16b, v6.16b\n" - "sqrdmulh v7.4s, v7.4s, v5.4s\n" - "and v30.16b, v17.16b, v6.16b\n" - "sqrdmulh v21.4s, v21.4s, v5.4s\n" - "sqadd v19.4s, v19.4s, v16.4s\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v26.16b, v10.16b, v22.16b\n" - "sshr v18.4s, v18.4s, #0x1f\n" - "and v23.16b, v7.16b, v22.16b\n" - "sshr v30.4s, v30.4s, #0x1f\n" - "and v16.16b, v21.16b, v22.16b\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "sshr v26.4s, v26.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v18.4s\n" - "sshr v23.4s, v23.4s, #0x1f\n" - "sqadd v17.4s, v17.4s, v30.4s\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "srshl v13.4s, v13.4s, v6.4s\n" - "srshl v20.4s, v20.4s, v6.4s\n" - "sqadd v10.4s, v10.4s, v26.4s\n" - "srshl v8.4s, v8.4s, v6.4s\n" - "sqadd v7.4s, v7.4s, v23.4s\n" - "srshl v17.4s, v17.4s, v6.4s\n" - "sqadd v21.4s, v21.4s, v16.4s\n" - "srshl v19.4s, v19.4s, v22.4s\n" - "sqxtn v13.4h, v13.4s\n" - "srshl v10.4s, v10.4s, v22.4s\n" + "smlal2 v19.4s, v16.8h, v27.8h\n" + "smlal v20.4s, v25.4h, v30.4h\n" + "smlal v24.4s, v16.4h, v30.4h\n" + "smlal v23.4s, v4.4h, v30.4h\n" + "smlal2 v15.4s, v25.8h, v31.8h\n" + "smlal v7.4s, v10.4h, v29.4h\n" + "smlal2 v5.4s, v25.8h, v30.8h\n" + "smlal2 v22.4s, v16.8h, v30.8h\n" + "smlal2 v19.4s, v4.8h, v30.8h\n" + "smlal v20.4s, v10.4h, v31.4h\n" + "smlal v24.4s, v4.4h, v31.4h\n" + "smlal v23.4s, v17.4h, v31.4h\n" + "smlal2 v15.4s, v10.8h, v29.8h\n" + "smlal v7.4s, v9.4h, v21.4h\n" + "sqrdmulh v7.4s, v7.4s, v8.4s\n" + "smlal2 v5.4s, v10.8h, v31.8h\n" + "smlal2 v22.4s, v4.8h, v31.8h\n" + "and v27.16b, v7.16b, v28.16b\n" + "smlal2 v19.4s, v17.8h, v31.8h\n" + "smlal v20.4s, v9.4h, v29.4h\n" + "sshr v27.4s, v27.4s, #0x1f\n" + "smlal v24.4s, v17.4h, v29.4h\n" + "smlal v23.4s, v6.4h, v29.4h\n" + "sqadd v7.4s, v7.4s, v27.4s\n" + "smlal2 v15.4s, v9.8h, v21.8h\n" + "smlal2 v5.4s, v9.8h, v29.8h\n" + "sqrdmulh v15.4s, v15.4s, v2.4s\n" + "smlal2 v22.4s, v17.8h, v29.8h\n" + "smlal2 v19.4s, v6.8h, v29.8h\n" + "and v9.16b, v15.16b, v14.16b\n" + "smlal v20.4s, v12.4h, v21.4h\n" + "smlal v24.4s, v6.4h, v21.4h\n" + "sqrdmulh v20.4s, v20.4s, v8.4s\n" + "smlal v23.4s, v3.4h, v21.4h\n" + "smlal2 v5.4s, v12.8h, v21.8h\n" + "sqrdmulh v24.4s, v24.4s, v8.4s\n" + "smlal2 v22.4s, v6.8h, v21.8h\n" + "smlal2 v19.4s, v3.8h, v21.8h\n" + "sqrdmulh v23.4s, v23.4s, v8.4s\n" + "sshr v9.4s, v9.4s, #0x1f\n" + "and v25.16b, v20.16b, v28.16b\n" + "sqrdmulh v5.4s, v5.4s, v2.4s\n" + "and v10.16b, v24.16b, v28.16b\n" + "sqrdmulh v22.4s, v22.4s, v2.4s\n" + "and v21.16b, v23.16b, v28.16b\n" + "sqrdmulh v19.4s, v19.4s, v2.4s\n" + "sqadd v15.4s, v15.4s, v9.4s\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "and v9.16b, v5.16b, v14.16b\n" + "sshr v10.4s, v10.4s, #0x1f\n" + "and v12.16b, v22.16b, v14.16b\n" + "sshr v21.4s, v21.4s, #0x1f\n" + "and v17.16b, v19.16b, v14.16b\n" + "sqadd v20.4s, v20.4s, v25.4s\n" + "sshr v9.4s, v9.4s, #0x1f\n" + "sqadd v24.4s, v24.4s, v10.4s\n" + "sshr v12.4s, v12.4s, #0x1f\n" + "sqadd v23.4s, v23.4s, v21.4s\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "srshl v7.4s, v7.4s, v28.4s\n" + "srshl v20.4s, v20.4s, v28.4s\n" + "sqadd v5.4s, v5.4s, v9.4s\n" + "srshl v24.4s, v24.4s, v28.4s\n" + "sqadd v22.4s, v22.4s, v12.4s\n" + "srshl v23.4s, v23.4s, v28.4s\n" + "sqadd v19.4s, v19.4s, v17.4s\n" + "srshl v15.4s, v15.4s, v14.4s\n" + "sqxtn v7.4h, v7.4s\n" + "srshl v5.4s, v5.4s, v14.4s\n" "sqxtn v20.4h, v20.4s\n" - "srshl v7.4s, v7.4s, v22.4s\n" - "sqxtn v8.4h, v8.4s\n" - "srshl v21.4s, v21.4s, v22.4s\n" - "sqxtn v17.4h, v17.4s\n" - "sqxtn2 v13.8h, v19.4s\n" - "sqxtn2 v20.8h, v10.4s\n" - "sqxtn2 v8.8h, v7.4s\n" - "sqxtn2 v17.8h, v21.4s\n" - "sqadd v13.8h, v13.8h, v14.8h\n" - "sqadd v20.8h, v20.8h, v14.8h\n" - "sqadd v8.8h, v8.8h, v14.8h\n" - "sqadd v17.8h, v17.8h, v14.8h\n" - "smax v13.8h, v13.8h, v12.8h\n" - "smax v20.8h, v20.8h, v12.8h\n" - "smax v8.8h, v8.8h, v12.8h\n" - "smax v17.8h, v17.8h, v12.8h\n" - "smin v13.8h, v13.8h, v11.8h\n" - "smin v20.8h, v20.8h, v11.8h\n" - "smin v8.8h, v8.8h, v11.8h\n" - "smin v17.8h, v17.8h, v11.8h\n" - "uzp1 v13.16b, v13.16b, v13.16b\n" + "srshl v22.4s, v22.4s, v14.4s\n" + "sqxtn v24.4h, v24.4s\n" + "srshl v19.4s, v19.4s, v14.4s\n" + "sqxtn v23.4h, v23.4s\n" + "sqxtn2 v7.8h, v15.4s\n" + "sqxtn2 v20.8h, v5.4s\n" + "sqxtn2 v24.8h, v22.4s\n" + "sqxtn2 v23.8h, v19.4s\n" + "sqadd v7.8h, v7.8h, v26.8h\n" + "sqadd v20.8h, v20.8h, v26.8h\n" + "sqadd v24.8h, v24.8h, v26.8h\n" + "sqadd v23.8h, v23.8h, v26.8h\n" + "smax v7.8h, v7.8h, v11.8h\n" + "smax v20.8h, v20.8h, v11.8h\n" + "smax v24.8h, v24.8h, v11.8h\n" + "smax v23.8h, v23.8h, v11.8h\n" + "smin v7.8h, v7.8h, v0.8h\n" + "smin v20.8h, v20.8h, v0.8h\n" + "smin v24.8h, v24.8h, v0.8h\n" + "smin v23.8h, v23.8h, v0.8h\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" + "str d7, [x17, x4]\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "str d13, [x17, x10]\n" - "uzp1 v8.16b, v8.16b, v8.16b\n" - "uzp1 v17.16b, v17.16b, v17.16b\n" - "str d20, [x6, x10]\n" - "str d8, [x7, x10]\n" - "str d17, [x16, x10]\n" - "ldr q13, [x13, #0x0]\n" - "ldr q19, [x13, #0x10]\n" - "add x13, x13, #0x20\n" - "ldr d0, [x3, #0x0]\n" - "ldr d1, [x3, #0x8]\n" - "add x10, x10, #0x8\n" - "str x13, [%x[params], %[offsetof_Params_bias]]\n" - "ldr d2, [x3, #0x10]\n" - "ldr d3, [x3, #0x18]\n" - "mov v20.16b, v13.16b\n" - "mov v10.16b, v19.16b\n" - "ldr d4, [x3, #0x20]\n" - "ldp x9, x28, [x4, #0x0]\n" - "mov v8.16b, v13.16b\n" - "mov v7.16b, v19.16b\n" - "ldp x27, x26, [x4, #0x10]\n" - "ldp x25, x24, [x4, #0x20]\n" - "mov v17.16b, v13.16b\n" - "mov v21.16b, v19.16b\n" - "ldp x23, x22, [x4, #0x30]\n" - "ldp x21, x20, [x4, #0x40]\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "ldr d31, [x9, x0]\n" - "ldr d30, [x28, x0]\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "ldr d29, [x27, x0]\n" - "ldr d28, [x26, x0]\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "usubl v31.8h, v31.8b, v9.8b\n" - "ldr d27, [x25, x0]\n" - "ldr d23, [x24, x0]\n" - "usubl v30.8h, v30.8b, v9.8b\n" - "usubl v29.8h, v29.8b, v9.8b\n" - "ldr d25, [x23, x0]\n" - "ldr d24, [x22, x0]\n" - "usubl v28.8h, v28.8b, v9.8b\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "ldr d26, [x21, x0]\n" - "ldr d22, [x20, x0]\n" - "usubl v23.8h, v23.8b, v9.8b\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "usubl v26.8h, v26.8b, v9.8b\n" - "usubl v22.8h, v22.8b, v9.8b\n" + "uzp1 v24.16b, v24.16b, v24.16b\n" + "str d20, [x16, x4]\n" + "uzp1 v23.16b, v23.16b, v23.16b\n" + "str d24, [x15, x4]\n" + "str d23, [x14, x4]\n" + "ldr q7, [x20, #0x0]\n" + "ldr q15, [x20, #0x10]\n" + "add x20, x20, #0x20\n" + "ldr d6, [x6, #0x0]\n" + "ldr d14, [x6, #0x8]\n" + "add x4, x4, #0x8\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" + "ldr d10, [x6, #0x10]\n" + "ldr d21, [x6, #0x18]\n" + "mov v20.16b, v7.16b\n" + "mov v5.16b, v15.16b\n" + "ldr d12, [x6, #0x20]\n" + "ldp x9, x28, [x5, #0x0]\n" + "mov v24.16b, v7.16b\n" + "mov v22.16b, v15.16b\n" + "ldp x27, x26, [x5, #0x10]\n" + "ldp x25, x24, [x5, #0x20]\n" + "mov v23.16b, v7.16b\n" + "mov v19.16b, v15.16b\n" + "ldp x23, x22, [x5, #0x30]\n" + "ldp x21, x20, [x5, #0x40]\n" + "ssubl v6.8h, v6.8b, v13.8b\n" + "ssubl v14.8h, v14.8b, v13.8b\n" + "ldr d31, [x9, x3]\n" + "ldr d17, [x28, x3]\n" + "ssubl v10.8h, v10.8b, v13.8b\n" + "ssubl v21.8h, v21.8b, v13.8b\n" + "ldr d30, [x27, x3]\n" + "ldr d16, [x26, x3]\n" + "ssubl v12.8h, v12.8b, v13.8b\n" + "usubl v31.8h, v31.8b, v18.8b\n" + "ldr d3, [x25, x3]\n" + "ldr d4, [x24, x3]\n" + "usubl v17.8h, v17.8b, v18.8b\n" + "usubl v30.8h, v30.8b, v18.8b\n" + "ldr d25, [x23, x3]\n" + "ldr d9, [x22, x3]\n" + "usubl v16.8h, v16.8b, v18.8b\n" + "usubl v3.8h, v3.8b, v18.8b\n" + "ldr d29, [x21, x3]\n" + "ldr d28, [x20, x3]\n" + "usubl v4.8h, v4.8b, v18.8b\n" + "usubl v25.8h, v25.8b, v18.8b\n" + "usubl v9.8h, v9.8b, v18.8b\n" + "usubl v29.8h, v29.8b, v18.8b\n" + "usubl v28.8h, v28.8b, v18.8b\n" "bgt 1b\n" "2:" // Tail - "ldr q18, [x5, #0x0]\n" - "ldr q6, [x8, #0x0]\n" - "smlal v13.4s, v31.4h, v0.4h\n" - "smlal2 v19.4s, v31.8h, v0.8h\n" - "ldr q5, [x5, #0x10]\n" - "smlal v13.4s, v30.4h, v1.4h\n" - "ldr x20, [x4, #0x50]\n" - "smlal v20.4s, v30.4h, v0.4h\n" - "smlal v8.4s, v29.4h, v0.4h\n" - "smlal v17.4s, v28.4h, v0.4h\n" - "ldr x22, [x4, #0x58]\n" - "ldr x21, [x4, #0x60]\n" - "smlal2 v19.4s, v30.8h, v1.8h\n" - "smlal2 v10.4s, v30.8h, v0.8h\n" - "ldr d31, [x20, x0]\n" - "usubl v31.8h, v31.8b, v9.8b\n" - "smlal2 v7.4s, v29.8h, v0.8h\n" - "smlal v13.4s, v27.4h, v2.4h\n" - "ldr x20, [x4, #0x68]\n" - "ldr x26, [x4, #0x70]\n" - "smlal2 v21.4s, v28.8h, v0.8h\n" - "ldr d30, [x22, x0]\n" - "smlal v20.4s, v27.4h, v1.4h\n" - "usubl v30.8h, v30.8b, v9.8b\n" - "smlal v8.4s, v28.4h, v1.4h\n" - "smlal v17.4s, v23.4h, v1.4h\n" - "ldr x25, [x4, #0x78]\n" - "ldr x23, [x4, #0x80]\n" - "smlal2 v19.4s, v27.8h, v2.8h\n" - "smlal2 v10.4s, v27.8h, v1.8h\n" - "ldr d0, [x3, #0x28]\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "smlal2 v7.4s, v28.8h, v1.8h\n" - "smlal v13.4s, v25.4h, v3.4h\n" - "ldr x24, [x4, #0x88]\n" - "ldr x15, [x4, #0x90]\n" - "smlal2 v21.4s, v23.8h, v1.8h\n" - "ldr d27, [x21, x0]\n" - "smlal v20.4s, v25.4h, v2.4h\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal v8.4s, v23.4h, v2.4h\n" - "smlal v17.4s, v31.4h, v2.4h\n" - "ldr x21, [x4, #0x98]\n" - "ldr x14, [x4, #0xa0]\n" - "smlal2 v19.4s, v25.8h, v3.8h\n" - "smlal2 v10.4s, v25.8h, v2.8h\n" - "ldr d1, [x3, #0x30]\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "smlal2 v7.4s, v23.8h, v2.8h\n" - "smlal v13.4s, v24.4h, v4.4h\n" - "ldr x13, [x4, #0xa8]\n" - "ldr x12, [x4, #0xb0]\n" - "smlal2 v21.4s, v31.8h, v2.8h\n" - "ldr d25, [x20, x0]\n" - "smlal v20.4s, v24.4h, v3.4h\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "smlal v8.4s, v31.4h, v3.4h\n" - "smlal v17.4s, v30.4h, v3.4h\n" - "ldr x20, [x4, #0xb8]\n" - "ldr x11, [x4, #0xc0]\n" - "smlal2 v19.4s, v24.8h, v4.8h\n" - "smlal2 v10.4s, v24.8h, v3.8h\n" - "ldr d2, [x3, #0x38]\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "smlal2 v7.4s, v31.8h, v3.8h\n" - "smlal v13.4s, v29.4h, v0.4h\n" - "ldr x22, [x4, #0xc8]\n" - "ldr x9, [x4, #0xd0]\n" - "smlal2 v21.4s, v30.8h, v3.8h\n" - "ldr d24, [x26, x0]\n" - "smlal v20.4s, v27.4h, v4.4h\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "smlal v8.4s, v30.4h, v4.4h\n" - "smlal v17.4s, v26.4h, v4.4h\n" - "ldr x28, [x4, #0xd8]\n" - "ldr x27, [x4, #0xe0]\n" - "smlal2 v19.4s, v29.8h, v0.8h\n" - "ldr d3, [x3, #0x40]\n" - "smlal2 v10.4s, v27.8h, v4.8h\n" - "ldr d27, [x25, x0]\n" - "smlal2 v7.4s, v30.8h, v4.8h\n" - "smlal v13.4s, v28.4h, v1.4h\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "ldr x26, [x4, #0xe8]\n" - "smlal2 v21.4s, v26.8h, v4.8h\n" - "ldr d4, [x3, #0x48]\n" - "smlal v20.4s, v28.4h, v0.4h\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal v8.4s, v22.4h, v0.4h\n" - "smlal v17.4s, v25.4h, v0.4h\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "ldr x25, [x4, #0xf0]\n" - "smlal2 v19.4s, v28.8h, v1.8h\n" - "smlal2 v10.4s, v28.8h, v0.8h\n" - "ldr d28, [x24, x0]\n" - "usubl v28.8h, v28.8b, v9.8b\n" - "smlal2 v7.4s, v22.8h, v0.8h\n" - "smlal v13.4s, v23.4h, v2.4h\n" - "ldr x24, [x4, #0xf8]\n" - "tst x1, #0x7\n" - "smlal2 v21.4s, v25.8h, v0.8h\n" - "ldr d0, [x3, #0x50]\n" - "smlal v20.4s, v23.4h, v1.4h\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "smlal v8.4s, v25.4h, v1.4h\n" - "smlal v17.4s, v24.4h, v1.4h\n" - "add x5, x5, #0x20\n" - "smlal2 v19.4s, v23.8h, v2.8h\n" - "smlal2 v10.4s, v23.8h, v1.8h\n" - "ldr d23, [x23, x0]\n" - "usubl v23.8h, v23.8b, v9.8b\n" - "smlal2 v7.4s, v25.8h, v1.8h\n" - "smlal v13.4s, v31.4h, v3.4h\n" - "ldr x23, [x4, #0x100]\n" - "smlal2 v21.4s, v24.8h, v1.8h\n" - "ldr d1, [x3, #0x58]\n" - "smlal v20.4s, v31.4h, v2.4h\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "smlal v8.4s, v24.4h, v2.4h\n" - "smlal v17.4s, v27.4h, v2.4h\n" - "smlal2 v19.4s, v31.8h, v3.8h\n" - "smlal2 v10.4s, v31.8h, v2.8h\n" - "ldr d31, [x15, x0]\n" - "usubl v31.8h, v31.8b, v9.8b\n" - "smlal2 v7.4s, v24.8h, v2.8h\n" - "smlal v13.4s, v30.4h, v4.4h\n" - "ldr x15, [x4, #0x108]\n" - "smlal2 v21.4s, v27.8h, v2.8h\n" - "ldr d2, [x3, #0x60]\n" - "smlal v20.4s, v30.4h, v3.4h\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "smlal v8.4s, v27.4h, v3.4h\n" - "smlal v17.4s, v23.4h, v3.4h\n" - "smlal2 v19.4s, v30.8h, v4.8h\n" - "smlal2 v10.4s, v30.8h, v3.8h\n" - "ldr d30, [x21, x0]\n" - "usubl v30.8h, v30.8b, v9.8b\n" - "smlal2 v7.4s, v27.8h, v3.8h\n" - "smlal v13.4s, v22.4h, v0.4h\n" - "ldr x21, [x4, #0x110]\n" - "smlal2 v21.4s, v23.8h, v3.8h\n" - "ldr d3, [x3, #0x68]\n" - "smlal v20.4s, v26.4h, v4.4h\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "smlal v8.4s, v23.4h, v4.4h\n" - "smlal v17.4s, v28.4h, v4.4h\n" - "smlal2 v19.4s, v22.8h, v0.8h\n" - "ldr d22, [x20, x0]\n" - "smlal2 v10.4s, v26.8h, v4.8h\n" - "ldr d26, [x14, x0]\n" - "smlal2 v7.4s, v23.8h, v4.8h\n" - "smlal v13.4s, v25.4h, v1.4h\n" - "usubl v26.8h, v26.8b, v9.8b\n" - "ldr x20, [x4, #0x118]\n" - "smlal2 v21.4s, v28.8h, v4.8h\n" - "ldr d4, [x3, #0x70]\n" - "smlal v20.4s, v25.4h, v0.4h\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "smlal v8.4s, v31.4h, v0.4h\n" - "smlal v17.4s, v30.4h, v0.4h\n" - "usubl v22.8h, v22.8b, v9.8b\n" + "ldr d27, [x6, #0x28]\n" + "ldr d1, [x6, #0x30]\n" + "smlal v7.4s, v31.4h, v6.4h\n" + "smlal2 v15.4s, v31.8h, v6.8h\n" + "ldr d2, [x6, #0x38]\n" + "ldr d31, [x6, #0x40]\n" + "smlal v7.4s, v17.4h, v14.4h\n" + "smlal v20.4s, v17.4h, v6.4h\n" + "ldr d8, [x6, #0x48]\n" + "ldr x22, [x5, #0x50]\n" + "smlal v24.4s, v30.4h, v6.4h\n" + "smlal v23.4s, v16.4h, v6.4h\n" + "smlal2 v15.4s, v17.8h, v14.8h\n" + "smlal v7.4s, v3.4h, v10.4h\n" + "ldr x20, [x5, #0x58]\n" + "ldr x21, [x5, #0x60]\n" + "smlal2 v5.4s, v17.8h, v6.8h\n" + "ldr d17, [x22, x3]\n" + "smlal2 v22.4s, v30.8h, v6.8h\n" + "usubl v17.8h, v17.8b, v18.8b\n" + "smlal2 v19.4s, v16.8h, v6.8h\n" + "ldr d6, [x20, x3]\n" + "smlal v20.4s, v3.4h, v14.4h\n" + "usubl v6.8h, v6.8b, v18.8b\n" + "smlal v24.4s, v16.4h, v14.4h\n" + "smlal v23.4s, v4.4h, v14.4h\n" + "ssubl v27.8h, v27.8b, v13.8b\n" + "ldr x20, [x5, #0x68]\n" + "smlal2 v15.4s, v3.8h, v10.8h\n" + "smlal v7.4s, v25.4h, v21.4h\n" + "ssubl v1.8h, v1.8b, v13.8b\n" + "ldr x22, [x5, #0x70]\n" + "smlal2 v5.4s, v3.8h, v14.8h\n" + "ldr d3, [x21, x3]\n" + "smlal2 v22.4s, v16.8h, v14.8h\n" + "usubl v3.8h, v3.8b, v18.8b\n" + "smlal2 v19.4s, v4.8h, v14.8h\n" + "ldr d14, [x20, x3]\n" + "smlal v20.4s, v25.4h, v10.4h\n" + "usubl v14.8h, v14.8b, v18.8b\n" + "smlal v24.4s, v4.4h, v10.4h\n" + "smlal v23.4s, v17.4h, v10.4h\n" + "ssubl v2.8h, v2.8b, v13.8b\n" + "ldr x21, [x5, #0x78]\n" + "smlal2 v15.4s, v25.8h, v21.8h\n" + "smlal v7.4s, v9.4h, v12.4h\n" + "ssubl v31.8h, v31.8b, v13.8b\n" + "ldr x20, [x5, #0x80]\n" + "smlal2 v5.4s, v25.8h, v10.8h\n" + "ldr d25, [x22, x3]\n" + "smlal2 v22.4s, v4.8h, v10.8h\n" + "usubl v25.8h, v25.8b, v18.8b\n" + "smlal2 v19.4s, v17.8h, v10.8h\n" + "ldr d10, [x21, x3]\n" + "smlal v20.4s, v9.4h, v21.4h\n" + "usubl v10.8h, v10.8b, v18.8b\n" + "smlal v24.4s, v17.4h, v21.4h\n" + "smlal v23.4s, v6.4h, v21.4h\n" + "ssubl v8.8h, v8.8b, v13.8b\n" + "ldr x24, [x5, #0x88]\n" + "smlal2 v15.4s, v9.8h, v12.8h\n" + "smlal v7.4s, v30.4h, v27.4h\n" + "ldr x23, [x5, #0x90]\n" + "ldr x22, [x5, #0x98]\n" + "smlal2 v5.4s, v9.8h, v21.8h\n" + "ldr d9, [x20, x3]\n" + "smlal2 v22.4s, v17.8h, v21.8h\n" + "usubl v9.8h, v9.8b, v18.8b\n" + "smlal2 v19.4s, v6.8h, v21.8h\n" + "ldr d21, [x6, #0x50]\n" + "smlal v20.4s, v3.4h, v12.4h\n" + "ssubl v21.8h, v21.8b, v13.8b\n" + "smlal v24.4s, v6.4h, v12.4h\n" + "smlal v23.4s, v29.4h, v12.4h\n" + "ldr x21, [x5, #0xa0]\n" + "ldr x20, [x5, #0xa8]\n" + "smlal2 v15.4s, v30.8h, v27.8h\n" + "ldr d30, [x24, x3]\n" + "smlal v7.4s, v16.4h, v1.4h\n" + "usubl v30.8h, v30.8b, v18.8b\n" + "smlal2 v5.4s, v3.8h, v12.8h\n" + "ldr d3, [x6, #0x58]\n" + "smlal2 v22.4s, v6.8h, v12.8h\n" + "ssubl v3.8h, v3.8b, v13.8b\n" + "smlal2 v19.4s, v29.8h, v12.8h\n" + "ldr d12, [x23, x3]\n" + "smlal v20.4s, v16.4h, v27.4h\n" + "usubl v12.8h, v12.8b, v18.8b\n" + "smlal v24.4s, v28.4h, v27.4h\n" + "smlal v23.4s, v14.4h, v27.4h\n" + "ldr x13, [x5, #0xb0]\n" + "ldr x12, [x5, #0xb8]\n" + "smlal2 v15.4s, v16.8h, v1.8h\n" + "smlal v7.4s, v4.4h, v2.4h\n" + "ldr x11, [x5, #0xc0]\n" + "ldr x10, [x5, #0xc8]\n" + "smlal2 v5.4s, v16.8h, v27.8h\n" + "ldr d16, [x22, x3]\n" + "smlal2 v22.4s, v28.8h, v27.8h\n" + "usubl v16.8h, v16.8b, v18.8b\n" + "smlal2 v19.4s, v14.8h, v27.8h\n" + "ldr d27, [x6, #0x60]\n" + "smlal v20.4s, v4.4h, v1.4h\n" + "ssubl v27.8h, v27.8b, v13.8b\n" + "smlal v24.4s, v14.4h, v1.4h\n" + "smlal v23.4s, v25.4h, v1.4h\n" + "ldr x9, [x5, #0xd0]\n" + "ldr x28, [x5, #0xd8]\n" + "smlal2 v15.4s, v4.8h, v2.8h\n" + "smlal v7.4s, v17.4h, v31.4h\n" + "ldr x27, [x5, #0xe0]\n" + "ldr x26, [x5, #0xe8]\n" + "smlal2 v5.4s, v4.8h, v1.8h\n" + "ldr d4, [x21, x3]\n" + "smlal2 v22.4s, v14.8h, v1.8h\n" + "usubl v4.8h, v4.8b, v18.8b\n" "smlal2 v19.4s, v25.8h, v1.8h\n" - "smlal2 v10.4s, v25.8h, v0.8h\n" - "ldr d25, [x13, x0]\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "smlal2 v7.4s, v31.8h, v0.8h\n" - "smlal v13.4s, v24.4h, v2.4h\n" - "smlal2 v21.4s, v30.8h, v0.8h\n" - "ldr d0, [x3, #0x78]\n" - "smlal v20.4s, v24.4h, v1.4h\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "smlal v8.4s, v30.4h, v1.4h\n" - "smlal v17.4s, v26.4h, v1.4h\n" - "smlal2 v19.4s, v24.8h, v2.8h\n" - "smlal2 v10.4s, v24.8h, v1.8h\n" - "ldr d24, [x12, x0]\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "smlal2 v7.4s, v30.8h, v1.8h\n" - "smlal v13.4s, v27.4h, v3.4h\n" - "smlal2 v21.4s, v26.8h, v1.8h\n" - "ldr d1, [x3, #0x80]\n" - "smlal v20.4s, v27.4h, v2.4h\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "smlal v8.4s, v26.4h, v2.4h\n" - "smlal v17.4s, v25.4h, v2.4h\n" - "smlal2 v19.4s, v27.8h, v3.8h\n" - "smlal2 v10.4s, v27.8h, v2.8h\n" - "ldr d27, [x11, x0]\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal2 v7.4s, v26.8h, v2.8h\n" - "smlal v13.4s, v23.4h, v4.4h\n" - "smlal2 v21.4s, v25.8h, v2.8h\n" - "ldr d2, [x3, #0x88]\n" - "smlal v20.4s, v23.4h, v3.4h\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "smlal v8.4s, v25.4h, v3.4h\n" - "smlal v17.4s, v24.4h, v3.4h\n" - "smlal2 v19.4s, v23.8h, v4.8h\n" - "smlal2 v10.4s, v23.8h, v3.8h\n" - "ldr d23, [x22, x0]\n" - "usubl v23.8h, v23.8b, v9.8b\n" - "smlal2 v7.4s, v25.8h, v3.8h\n" - "smlal v13.4s, v31.4h, v0.4h\n" - "smlal2 v21.4s, v24.8h, v3.8h\n" - "ldr d3, [x3, #0x90]\n" - "smlal v20.4s, v28.4h, v4.4h\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "smlal v8.4s, v24.4h, v4.4h\n" - "smlal v17.4s, v22.4h, v4.4h\n" - "smlal2 v19.4s, v31.8h, v0.8h\n" - "ldr d31, [x9, x0]\n" - "smlal2 v10.4s, v28.8h, v4.8h\n" - "ldr d28, [x27, x0]\n" - "smlal2 v7.4s, v24.8h, v4.8h\n" - "smlal v13.4s, v30.4h, v1.4h\n" - "usubl v31.8h, v31.8b, v9.8b\n" - "smlal2 v21.4s, v22.8h, v4.8h\n" - "ldr d4, [x3, #0x98]\n" - "smlal v20.4s, v30.4h, v0.4h\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "smlal v8.4s, v27.4h, v0.4h\n" - "smlal v17.4s, v23.4h, v0.4h\n" - "usubl v28.8h, v28.8b, v9.8b\n" - "smlal2 v19.4s, v30.8h, v1.8h\n" - "smlal2 v10.4s, v30.8h, v0.8h\n" - "ldr d30, [x28, x0]\n" - "usubl v30.8h, v30.8b, v9.8b\n" - "smlal2 v7.4s, v27.8h, v0.8h\n" - "smlal v13.4s, v26.4h, v2.4h\n" - "smlal2 v21.4s, v23.8h, v0.8h\n" - "ldr d0, [x3, #0xa0]\n" - "smlal v20.4s, v26.4h, v1.4h\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "smlal v8.4s, v23.4h, v1.4h\n" - "smlal v17.4s, v31.4h, v1.4h\n" - "smlal2 v19.4s, v26.8h, v2.8h\n" - "smlal2 v10.4s, v26.8h, v1.8h\n" - "ldr d26, [x26, x0]\n" - "usubl v26.8h, v26.8b, v9.8b\n" - "smlal2 v7.4s, v23.8h, v1.8h\n" - "smlal v13.4s, v25.4h, v3.4h\n" - "smlal2 v21.4s, v31.8h, v1.8h\n" - "ldr d1, [x3, #0xa8]\n" - "smlal v20.4s, v25.4h, v2.4h\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "smlal v8.4s, v31.4h, v2.4h\n" - "smlal v17.4s, v30.4h, v2.4h\n" - "smlal2 v19.4s, v25.8h, v3.8h\n" - "smlal2 v10.4s, v25.8h, v2.8h\n" - "ldr d25, [x25, x0]\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "smlal2 v7.4s, v31.8h, v2.8h\n" - "smlal v13.4s, v24.4h, v4.4h\n" - "smlal2 v21.4s, v30.8h, v2.8h\n" - "ldr d2, [x3, #0xb0]\n" - "smlal v20.4s, v24.4h, v3.4h\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "smlal v8.4s, v30.4h, v3.4h\n" - "smlal v17.4s, v28.4h, v3.4h\n" - "smlal2 v19.4s, v24.8h, v4.8h\n" - "smlal2 v10.4s, v24.8h, v3.8h\n" - "ldr d24, [x24, x0]\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "smlal2 v7.4s, v30.8h, v3.8h\n" - "smlal v13.4s, v27.4h, v0.4h\n" - "smlal2 v21.4s, v28.8h, v3.8h\n" - "ldr d3, [x3, #0xb8]\n" - "smlal v20.4s, v22.4h, v4.4h\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "smlal v8.4s, v28.4h, v4.4h\n" - "smlal v17.4s, v26.4h, v4.4h\n" - "smlal2 v19.4s, v27.8h, v0.8h\n" - "ldr d27, [x23, x0]\n" - "smlal2 v7.4s, v28.8h, v4.8h\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal v13.4s, v23.4h, v1.4h\n" - "smlal2 v10.4s, v22.8h, v4.8h\n" - "ldr q22, [x8, #0x10]\n" - "add x8, x8, #0x20\n" - "smlal2 v21.4s, v26.8h, v4.8h\n" - "ldr d4, [x3, #0xc0]\n" - "smlal v20.4s, v23.4h, v0.4h\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "smlal v8.4s, v25.4h, v0.4h\n" - "smlal v17.4s, v24.4h, v0.4h\n" - "smlal2 v19.4s, v23.8h, v1.8h\n" - "smlal2 v7.4s, v25.8h, v0.8h\n" - "ldr d25, [x15, x0]\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "smlal v13.4s, v31.4h, v2.4h\n" - "smlal2 v10.4s, v23.8h, v0.8h\n" - "smlal2 v21.4s, v24.8h, v0.8h\n" - "smlal v20.4s, v31.4h, v1.4h\n" - "smlal v8.4s, v24.4h, v1.4h\n" - "smlal v17.4s, v27.4h, v1.4h\n" - "smlal2 v19.4s, v31.8h, v2.8h\n" - "smlal2 v7.4s, v24.8h, v1.8h\n" - "ldr d24, [x21, x0]\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "smlal v13.4s, v30.4h, v3.4h\n" - "smlal2 v10.4s, v31.8h, v1.8h\n" - "smlal2 v21.4s, v27.8h, v1.8h\n" + "ldr d1, [x6, #0x68]\n" + "smlal v20.4s, v17.4h, v2.4h\n" + "ssubl v1.8h, v1.8b, v13.8b\n" + "smlal v24.4s, v25.4h, v2.4h\n" + "smlal v23.4s, v10.4h, v2.4h\n" + "ldr x25, [x5, #0xf0]\n" + "ldr x24, [x5, #0xf8]\n" + "smlal2 v15.4s, v17.8h, v31.8h\n" + "smlal v7.4s, v6.4h, v8.4h\n" + "ldr x23, [x5, #0x100]\n" + "ldr x22, [x5, #0x108]\n" + "smlal2 v5.4s, v17.8h, v2.8h\n" + "ldr d17, [x20, x3]\n" + "smlal2 v22.4s, v25.8h, v2.8h\n" + "usubl v17.8h, v17.8b, v18.8b\n" + "smlal2 v19.4s, v10.8h, v2.8h\n" + "ldr d2, [x6, #0x70]\n" + "smlal v20.4s, v6.4h, v31.4h\n" + "ssubl v2.8h, v2.8b, v13.8b\n" + "smlal v24.4s, v10.4h, v31.4h\n" + "smlal v23.4s, v9.4h, v31.4h\n" + "ldr x21, [x5, #0x110]\n" + "ldr x20, [x5, #0x118]\n" + "smlal2 v15.4s, v6.8h, v8.8h\n" + "smlal v7.4s, v28.4h, v21.4h\n" + "tst x1, #0x7\n" + "smlal2 v5.4s, v6.8h, v31.8h\n" + "ldr d6, [x13, x3]\n" + "smlal2 v22.4s, v10.8h, v31.8h\n" + "usubl v6.8h, v6.8b, v18.8b\n" + "smlal2 v19.4s, v9.8h, v31.8h\n" + "ldr d31, [x6, #0x78]\n" + "smlal v20.4s, v29.4h, v8.4h\n" + "ssubl v31.8h, v31.8b, v13.8b\n" + "smlal v24.4s, v9.4h, v8.4h\n" + "smlal v23.4s, v30.4h, v8.4h\n" + "smlal2 v15.4s, v28.8h, v21.8h\n" + "ldr d28, [x12, x3]\n" + "smlal v7.4s, v14.4h, v3.4h\n" + "usubl v28.8h, v28.8b, v18.8b\n" + "smlal2 v5.4s, v29.8h, v8.8h\n" + "ldr d29, [x6, #0x80]\n" + "smlal2 v22.4s, v9.8h, v8.8h\n" + "ssubl v29.8h, v29.8b, v13.8b\n" + "smlal2 v19.4s, v30.8h, v8.8h\n" + "ldr d8, [x11, x3]\n" + "smlal v20.4s, v14.4h, v21.4h\n" + "usubl v8.8h, v8.8b, v18.8b\n" + "smlal v24.4s, v12.4h, v21.4h\n" + "smlal v23.4s, v16.4h, v21.4h\n" + "smlal2 v15.4s, v14.8h, v3.8h\n" + "smlal v7.4s, v25.4h, v27.4h\n" + "smlal2 v5.4s, v14.8h, v21.8h\n" + "ldr d14, [x10, x3]\n" + "smlal2 v22.4s, v12.8h, v21.8h\n" + "usubl v14.8h, v14.8b, v18.8b\n" + "smlal2 v19.4s, v16.8h, v21.8h\n" + "ldr d21, [x6, #0x88]\n" + "smlal v20.4s, v25.4h, v3.4h\n" + "ssubl v21.8h, v21.8b, v13.8b\n" + "smlal v24.4s, v16.4h, v3.4h\n" + "smlal v23.4s, v4.4h, v3.4h\n" + "smlal2 v15.4s, v25.8h, v27.8h\n" + "smlal v7.4s, v10.4h, v1.4h\n" + "smlal2 v5.4s, v25.8h, v3.8h\n" + "ldr d25, [x9, x3]\n" + "smlal2 v22.4s, v16.8h, v3.8h\n" + "usubl v25.8h, v25.8b, v18.8b\n" + "smlal2 v19.4s, v4.8h, v3.8h\n" + "ldr d3, [x6, #0x90]\n" + "smlal v20.4s, v10.4h, v27.4h\n" + "ssubl v3.8h, v3.8b, v13.8b\n" + "smlal v24.4s, v4.4h, v27.4h\n" + "smlal v23.4s, v17.4h, v27.4h\n" + "smlal2 v15.4s, v10.8h, v1.8h\n" + "smlal v7.4s, v9.4h, v2.4h\n" + "smlal2 v5.4s, v10.8h, v27.8h\n" + "ldr d10, [x28, x3]\n" + "smlal2 v22.4s, v4.8h, v27.8h\n" + "usubl v10.8h, v10.8b, v18.8b\n" + "smlal2 v19.4s, v17.8h, v27.8h\n" + "ldr d27, [x6, #0x98]\n" + "smlal v20.4s, v9.4h, v1.4h\n" + "ssubl v27.8h, v27.8b, v13.8b\n" + "smlal v24.4s, v17.4h, v1.4h\n" + "smlal v23.4s, v6.4h, v1.4h\n" + "smlal2 v15.4s, v9.8h, v2.8h\n" + "smlal v7.4s, v12.4h, v31.4h\n" + "smlal2 v5.4s, v9.8h, v1.8h\n" + "ldr d9, [x27, x3]\n" + "smlal2 v22.4s, v17.8h, v1.8h\n" + "usubl v9.8h, v9.8b, v18.8b\n" + "smlal2 v19.4s, v6.8h, v1.8h\n" + "ldr d1, [x6, #0xa0]\n" "smlal v20.4s, v30.4h, v2.4h\n" - "smlal v8.4s, v27.4h, v2.4h\n" - "smlal v17.4s, v25.4h, v2.4h\n" - "smlal2 v19.4s, v30.8h, v3.8h\n" - "smlal2 v7.4s, v27.8h, v2.8h\n" - "ldr d27, [x20, x0]\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal v13.4s, v28.4h, v4.4h\n" - "smlal2 v10.4s, v30.8h, v2.8h\n" - "sqrdmulh v13.4s, v13.4s, v18.4s\n" - "add x0, x0, #0x8\n" - "smlal2 v21.4s, v25.8h, v2.8h\n" - "smlal v20.4s, v28.4h, v3.4h\n" - "and v30.16b, v13.16b, v6.16b\n" - "smlal v8.4s, v25.4h, v3.4h\n" - "smlal v17.4s, v24.4h, v3.4h\n" - "sshr v30.4s, v30.4s, #0x1f\n" - "smlal2 v19.4s, v28.8h, v4.8h\n" - "smlal2 v10.4s, v28.8h, v3.8h\n" - "sqrdmulh v19.4s, v19.4s, v5.4s\n" - "smlal2 v7.4s, v25.8h, v3.8h\n" - "smlal2 v21.4s, v24.8h, v3.8h\n" - "and v16.16b, v19.16b, v22.16b\n" - "smlal v20.4s, v26.4h, v4.4h\n" - "smlal v8.4s, v24.4h, v4.4h\n" - "sqrdmulh v20.4s, v20.4s, v18.4s\n" - "smlal v17.4s, v27.4h, v4.4h\n" - "smlal2 v10.4s, v26.8h, v4.8h\n" - "sqrdmulh v8.4s, v8.4s, v18.4s\n" - "smlal2 v7.4s, v24.8h, v4.8h\n" - "smlal2 v21.4s, v27.8h, v4.8h\n" - "sqrdmulh v17.4s, v17.4s, v18.4s\n" - "sqadd v13.4s, v13.4s, v30.4s\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "and v0.16b, v20.16b, v6.16b\n" - "sqrdmulh v10.4s, v10.4s, v5.4s\n" - "and v18.16b, v8.16b, v6.16b\n" - "sqrdmulh v7.4s, v7.4s, v5.4s\n" - "and v30.16b, v17.16b, v6.16b\n" - "sqrdmulh v21.4s, v21.4s, v5.4s\n" - "sqadd v19.4s, v19.4s, v16.4s\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v26.16b, v10.16b, v22.16b\n" - "sshr v18.4s, v18.4s, #0x1f\n" - "and v23.16b, v7.16b, v22.16b\n" + "ssubl v1.8h, v1.8b, v13.8b\n" + "smlal v24.4s, v6.4h, v2.4h\n" + "smlal v23.4s, v28.4h, v2.4h\n" + "smlal2 v15.4s, v12.8h, v31.8h\n" + "ldr d12, [x26, x3]\n" + "smlal v7.4s, v16.4h, v29.4h\n" + "usubl v12.8h, v12.8b, v18.8b\n" + "smlal2 v5.4s, v30.8h, v2.8h\n" + "ldr d30, [x6, #0xa8]\n" + "smlal2 v22.4s, v6.8h, v2.8h\n" + "ssubl v30.8h, v30.8b, v13.8b\n" + "smlal2 v19.4s, v28.8h, v2.8h\n" + "ldr d2, [x25, x3]\n" + "smlal v20.4s, v16.4h, v31.4h\n" + "usubl v2.8h, v2.8b, v18.8b\n" + "smlal v24.4s, v8.4h, v31.4h\n" + "smlal v23.4s, v14.4h, v31.4h\n" + "smlal2 v15.4s, v16.8h, v29.8h\n" + "smlal v7.4s, v4.4h, v21.4h\n" + "smlal2 v5.4s, v16.8h, v31.8h\n" + "ldr d16, [x24, x3]\n" + "smlal2 v22.4s, v8.8h, v31.8h\n" + "usubl v16.8h, v16.8b, v18.8b\n" + "smlal2 v19.4s, v14.8h, v31.8h\n" + "ldr d31, [x6, #0xb0]\n" + "smlal v20.4s, v4.4h, v29.4h\n" + "ssubl v31.8h, v31.8b, v13.8b\n" + "smlal v24.4s, v14.4h, v29.4h\n" + "smlal v23.4s, v25.4h, v29.4h\n" + "smlal2 v15.4s, v4.8h, v21.8h\n" + "smlal v7.4s, v17.4h, v3.4h\n" + "smlal2 v5.4s, v4.8h, v29.8h\n" + "ldr d4, [x23, x3]\n" + "smlal2 v22.4s, v14.8h, v29.8h\n" + "usubl v4.8h, v4.8b, v18.8b\n" + "smlal2 v19.4s, v25.8h, v29.8h\n" + "ldr d29, [x6, #0xb8]\n" + "smlal v20.4s, v17.4h, v21.4h\n" + "ssubl v29.8h, v29.8b, v13.8b\n" + "smlal v24.4s, v25.4h, v21.4h\n" + "smlal v23.4s, v10.4h, v21.4h\n" + "smlal2 v15.4s, v17.8h, v3.8h\n" + "smlal v7.4s, v6.4h, v27.4h\n" + "smlal2 v5.4s, v17.8h, v21.8h\n" + "ldr d17, [x22, x3]\n" + "smlal2 v22.4s, v25.8h, v21.8h\n" + "usubl v17.8h, v17.8b, v18.8b\n" + "smlal2 v19.4s, v10.8h, v21.8h\n" + "ldr d21, [x6, #0xc0]\n" + "smlal v20.4s, v6.4h, v3.4h\n" + "ssubl v21.8h, v21.8b, v13.8b\n" + "smlal v24.4s, v10.4h, v3.4h\n" + "smlal v23.4s, v9.4h, v3.4h\n" + "smlal2 v15.4s, v6.8h, v27.8h\n" + "smlal v7.4s, v8.4h, v1.4h\n" + "smlal2 v5.4s, v6.8h, v3.8h\n" + "ldr d6, [x21, x3]\n" + "smlal2 v22.4s, v10.8h, v3.8h\n" + "usubl v6.8h, v6.8b, v18.8b\n" + "smlal2 v19.4s, v9.8h, v3.8h\n" + "ldr d3, [x20, x3]\n" + "smlal v20.4s, v28.4h, v27.4h\n" + "usubl v3.8h, v3.8b, v18.8b\n" + "smlal v24.4s, v9.4h, v27.4h\n" + "smlal v23.4s, v12.4h, v27.4h\n" + "add x3, x3, #0x8\n" + "smlal2 v15.4s, v8.8h, v1.8h\n" + "ldr q8, [x7, #0x0]\n" + "smlal v7.4s, v14.4h, v30.4h\n" + "smlal2 v5.4s, v28.8h, v27.8h\n" + "ldr q28, [x8, #0x0]\n" + "smlal2 v22.4s, v9.8h, v27.8h\n" + "smlal2 v19.4s, v12.8h, v27.8h\n" + "ldr q27, [x7, #0x10]\n" + "smlal v20.4s, v14.4h, v1.4h\n" + "add x7, x7, #0x20\n" + "smlal v24.4s, v2.4h, v1.4h\n" + "smlal v23.4s, v16.4h, v1.4h\n" + "smlal2 v15.4s, v14.8h, v30.8h\n" + "smlal v7.4s, v25.4h, v31.4h\n" + "smlal2 v5.4s, v14.8h, v1.8h\n" + "ldr q14, [x8, #0x10]\n" + "smlal2 v22.4s, v2.8h, v1.8h\n" + "add x8, x8, #0x20\n" + "smlal2 v19.4s, v16.8h, v1.8h\n" + "smlal v20.4s, v25.4h, v30.4h\n" + "smlal v24.4s, v16.4h, v30.4h\n" + "smlal v23.4s, v4.4h, v30.4h\n" + "smlal2 v15.4s, v25.8h, v31.8h\n" + "smlal v7.4s, v10.4h, v29.4h\n" + "smlal2 v5.4s, v25.8h, v30.8h\n" + "smlal2 v22.4s, v16.8h, v30.8h\n" + "smlal2 v19.4s, v4.8h, v30.8h\n" + "smlal v20.4s, v10.4h, v31.4h\n" + "smlal v24.4s, v4.4h, v31.4h\n" + "smlal v23.4s, v17.4h, v31.4h\n" + "smlal2 v15.4s, v10.8h, v29.8h\n" + "smlal v7.4s, v9.4h, v21.4h\n" + "sqrdmulh v7.4s, v7.4s, v8.4s\n" + "smlal2 v5.4s, v10.8h, v31.8h\n" + "smlal2 v22.4s, v4.8h, v31.8h\n" + "and v4.16b, v7.16b, v28.16b\n" + "smlal2 v19.4s, v17.8h, v31.8h\n" + "smlal v20.4s, v9.4h, v29.4h\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "smlal v24.4s, v17.4h, v29.4h\n" + "smlal v23.4s, v6.4h, v29.4h\n" + "sqadd v7.4s, v7.4s, v4.4s\n" + "smlal2 v15.4s, v9.8h, v21.8h\n" + "smlal2 v5.4s, v9.8h, v29.8h\n" + "sqrdmulh v15.4s, v15.4s, v27.4s\n" + "smlal2 v22.4s, v17.8h, v29.8h\n" + "smlal2 v19.4s, v6.8h, v29.8h\n" + "and v30.16b, v15.16b, v14.16b\n" + "smlal v20.4s, v12.4h, v21.4h\n" + "smlal v24.4s, v6.4h, v21.4h\n" + "sqrdmulh v20.4s, v20.4s, v8.4s\n" + "smlal v23.4s, v3.4h, v21.4h\n" + "smlal2 v5.4s, v12.8h, v21.8h\n" + "sqrdmulh v24.4s, v24.4s, v8.4s\n" + "smlal2 v22.4s, v6.8h, v21.8h\n" + "smlal2 v19.4s, v3.8h, v21.8h\n" + "sqrdmulh v23.4s, v23.4s, v8.4s\n" "sshr v30.4s, v30.4s, #0x1f\n" - "and v16.16b, v21.16b, v22.16b\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "sshr v26.4s, v26.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v18.4s\n" - "sshr v23.4s, v23.4s, #0x1f\n" - "sqadd v17.4s, v17.4s, v30.4s\n" + "and v3.16b, v20.16b, v28.16b\n" + "sqrdmulh v5.4s, v5.4s, v27.4s\n" + "and v25.16b, v24.16b, v28.16b\n" + "sqrdmulh v22.4s, v22.4s, v27.4s\n" + "and v16.16b, v23.16b, v28.16b\n" + "sqrdmulh v19.4s, v19.4s, v27.4s\n" + "sqadd v15.4s, v15.4s, v30.4s\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "and v4.16b, v5.16b, v14.16b\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "and v10.16b, v22.16b, v14.16b\n" "sshr v16.4s, v16.4s, #0x1f\n" - "srshl v13.4s, v13.4s, v6.4s\n" - "srshl v20.4s, v20.4s, v6.4s\n" - "sqadd v10.4s, v10.4s, v26.4s\n" - "srshl v8.4s, v8.4s, v6.4s\n" - "sqadd v7.4s, v7.4s, v23.4s\n" - "srshl v17.4s, v17.4s, v6.4s\n" - "sqadd v21.4s, v21.4s, v16.4s\n" - "srshl v19.4s, v19.4s, v22.4s\n" - "sqxtn v13.4h, v13.4s\n" - "srshl v10.4s, v10.4s, v22.4s\n" + "and v12.16b, v19.16b, v14.16b\n" + "sqadd v20.4s, v20.4s, v3.4s\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sqadd v24.4s, v24.4s, v25.4s\n" + "sshr v10.4s, v10.4s, #0x1f\n" + "sqadd v23.4s, v23.4s, v16.4s\n" + "sshr v12.4s, v12.4s, #0x1f\n" + "srshl v7.4s, v7.4s, v28.4s\n" + "srshl v20.4s, v20.4s, v28.4s\n" + "sqadd v5.4s, v5.4s, v4.4s\n" + "srshl v24.4s, v24.4s, v28.4s\n" + "sqadd v22.4s, v22.4s, v10.4s\n" + "srshl v23.4s, v23.4s, v28.4s\n" + "sqadd v19.4s, v19.4s, v12.4s\n" + "srshl v15.4s, v15.4s, v14.4s\n" + "sqxtn v7.4h, v7.4s\n" + "srshl v5.4s, v5.4s, v14.4s\n" "sqxtn v20.4h, v20.4s\n" - "srshl v7.4s, v7.4s, v22.4s\n" - "sqxtn v8.4h, v8.4s\n" - "srshl v21.4s, v21.4s, v22.4s\n" - "sqxtn v17.4h, v17.4s\n" - "sqxtn2 v13.8h, v19.4s\n" - "sqxtn2 v20.8h, v10.4s\n" - "sqxtn2 v8.8h, v7.4s\n" - "sqxtn2 v17.8h, v21.4s\n" - "sqadd v13.8h, v13.8h, v14.8h\n" - "sqadd v20.8h, v20.8h, v14.8h\n" - "sqadd v8.8h, v8.8h, v14.8h\n" - "sqadd v17.8h, v17.8h, v14.8h\n" - "smax v13.8h, v13.8h, v12.8h\n" - "smax v20.8h, v20.8h, v12.8h\n" - "smax v8.8h, v8.8h, v12.8h\n" - "smax v17.8h, v17.8h, v12.8h\n" - "smin v13.8h, v13.8h, v11.8h\n" - "smin v20.8h, v20.8h, v11.8h\n" - "smin v8.8h, v8.8h, v11.8h\n" - "smin v17.8h, v17.8h, v11.8h\n" - "uzp1 v13.16b, v13.16b, v13.16b\n" + "srshl v22.4s, v22.4s, v14.4s\n" + "sqxtn v24.4h, v24.4s\n" + "srshl v19.4s, v19.4s, v14.4s\n" + "sqxtn v23.4h, v23.4s\n" + "sqxtn2 v7.8h, v15.4s\n" + "sqxtn2 v20.8h, v5.4s\n" + "sqxtn2 v24.8h, v22.4s\n" + "sqxtn2 v23.8h, v19.4s\n" + "sqadd v7.8h, v7.8h, v26.8h\n" + "sqadd v20.8h, v20.8h, v26.8h\n" + "sqadd v24.8h, v24.8h, v26.8h\n" + "sqadd v23.8h, v23.8h, v26.8h\n" + "smax v7.8h, v7.8h, v11.8h\n" + "smax v20.8h, v20.8h, v11.8h\n" + "smax v24.8h, v24.8h, v11.8h\n" + "smax v23.8h, v23.8h, v11.8h\n" + "smin v7.8h, v7.8h, v0.8h\n" + "smin v20.8h, v20.8h, v0.8h\n" + "smin v24.8h, v24.8h, v0.8h\n" + "smin v23.8h, v23.8h, v0.8h\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" + "str d7, [x17, x4]\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "str d13, [x17, x10]\n" - "uzp1 v8.16b, v8.16b, v8.16b\n" - "uzp1 v17.16b, v17.16b, v17.16b\n" - "str d20, [x6, x10]\n" - "str d8, [x7, x10]\n" - "str d17, [x16, x10]\n" - "add x10, x10, #0x8\n" + "uzp1 v24.16b, v24.16b, v24.16b\n" + "str d20, [x16, x4]\n" + "uzp1 v23.16b, v23.16b, v23.16b\n" + "str d24, [x15, x4]\n" + "str d23, [x14, x4]\n" + "add x4, x4, #0x8\n" "beq 124f\n" - "add x3, x3, #0xc8\n" + "add x6, x6, #0xc8\n" "3:" // Oddments - "ldr x13, [%x[params], %[offsetof_Params_bias]]\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" "tbz x1, #2, 5f\n" - "ld1 { v13.4s }, [x13], #0x10\n" + "ld1 { v7.4s }, [x20], #0x10\n" "tbz x1, #1, 4f\n" - "ld1 { v19.d }[0], [x13], #0x8\n" + "ld1 { v15.d }[0], [x20], #0x8\n" "tbz x1, #0, 7f\n" - "ld1 { v19.s }[2], [x13]\n" + "ld1 { v15.s }[2], [x20]\n" "b 7f\n" "4:" // Oddments: Load bias: Bit 2: Bit 1: Unset "tbz x1, #0, 7f\n" - "ld1 { v19.s }[0], [x13]\n" + "ld1 { v15.s }[0], [x20]\n" "b 7f\n" "5:" // Oddments: Load bias: Bit 2: Unset "tbz x1, #1, 6f\n" - "ld1 { v13.d }[0], [x13], #0x8\n" + "ld1 { v7.d }[0], [x20], #0x8\n" "tbz x1, #0, 7f\n" - "ld1 { v13.s }[2], [x13]\n" + "ld1 { v7.s }[2], [x20]\n" "b 7f\n" "6:" // Oddments: Load bias: Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 7f\n" - "ld1 { v13.s }[0], [x13]\n" + "ld1 { v7.s }[0], [x20]\n" "7:" // Oddments: Load bias: Bit 2: End - "ldr d0, [x3, #0x0]\n" - "ldr d1, [x3, #0x8]\n" - "mov v20.16b, v13.16b\n" - "mov v10.16b, v19.16b\n" - "ldr d2, [x3, #0x10]\n" - "ldr d3, [x3, #0x18]\n" - "mov v8.16b, v13.16b\n" - "mov v7.16b, v19.16b\n" - "ldr d4, [x3, #0x20]\n" - "ldp x9, x28, [x4, #0x0]\n" - "mov v17.16b, v13.16b\n" - "mov v21.16b, v19.16b\n" - "ldp x27, x26, [x4, #0x10]\n" - "ldp x25, x24, [x4, #0x20]\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "ldp x23, x22, [x4, #0x30]\n" - "ldp x21, x20, [x4, #0x40]\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "add x9, x9, x0\n" - "add x28, x28, x0\n" - "add x27, x27, x0\n" - "add x26, x26, x0\n" - "add x25, x25, x0\n" - "add x24, x24, x0\n" - "add x23, x23, x0\n" - "add x22, x22, x0\n" - "add x21, x21, x0\n" - "add x20, x20, x0\n" + "ldr d6, [x6, #0x0]\n" + "ldr d14, [x6, #0x8]\n" + "mov v20.16b, v7.16b\n" + "mov v5.16b, v15.16b\n" + "ldr d10, [x6, #0x10]\n" + "ldr d21, [x6, #0x18]\n" + "mov v24.16b, v7.16b\n" + "mov v22.16b, v15.16b\n" + "ldr d12, [x6, #0x20]\n" + "ldp x9, x28, [x5, #0x0]\n" + "mov v23.16b, v7.16b\n" + "mov v19.16b, v15.16b\n" + "ldp x27, x26, [x5, #0x10]\n" + "ldp x25, x24, [x5, #0x20]\n" + "ssubl v6.8h, v6.8b, v13.8b\n" + "ssubl v14.8h, v14.8b, v13.8b\n" + "ldp x23, x22, [x5, #0x30]\n" + "ldp x21, x20, [x5, #0x40]\n" + "ssubl v10.8h, v10.8b, v13.8b\n" + "ssubl v21.8h, v21.8b, v13.8b\n" + "ssubl v12.8h, v12.8b, v13.8b\n" + "add x9, x9, x3\n" + "add x28, x28, x3\n" + "add x27, x27, x3\n" + "add x26, x26, x3\n" + "add x25, x25, x3\n" + "add x24, x24, x3\n" + "add x23, x23, x3\n" + "add x22, x22, x3\n" + "add x21, x21, x3\n" + "add x20, x20, x3\n" "tbz x1, #2, 9f\n" "ld1 { v31.s }[0], [x9], #0x4\n" - "ld1 { v30.s }[0], [x28], #0x4\n" - "ld1 { v29.s }[0], [x27], #0x4\n" - "ld1 { v28.s }[0], [x26], #0x4\n" - "ld1 { v27.s }[0], [x25], #0x4\n" - "ld1 { v23.s }[0], [x24], #0x4\n" + "ld1 { v17.s }[0], [x28], #0x4\n" + "ld1 { v30.s }[0], [x27], #0x4\n" + "ld1 { v16.s }[0], [x26], #0x4\n" + "ld1 { v3.s }[0], [x25], #0x4\n" + "ld1 { v4.s }[0], [x24], #0x4\n" "ld1 { v25.s }[0], [x23], #0x4\n" - "ld1 { v24.s }[0], [x22], #0x4\n" - "ld1 { v26.s }[0], [x21], #0x4\n" - "ld1 { v22.s }[0], [x20], #0x4\n" + "ld1 { v9.s }[0], [x22], #0x4\n" + "ld1 { v29.s }[0], [x21], #0x4\n" + "ld1 { v28.s }[0], [x20], #0x4\n" "tbz x1, #1, 8f\n" "ld1 { v31.h }[2], [x9], #0x2\n" - "ld1 { v30.h }[2], [x28], #0x2\n" - "ld1 { v29.h }[2], [x27], #0x2\n" - "ld1 { v28.h }[2], [x26], #0x2\n" - "ld1 { v27.h }[2], [x25], #0x2\n" - "ld1 { v23.h }[2], [x24], #0x2\n" + "ld1 { v17.h }[2], [x28], #0x2\n" + "ld1 { v30.h }[2], [x27], #0x2\n" + "ld1 { v16.h }[2], [x26], #0x2\n" + "ld1 { v3.h }[2], [x25], #0x2\n" + "ld1 { v4.h }[2], [x24], #0x2\n" "ld1 { v25.h }[2], [x23], #0x2\n" - "ld1 { v24.h }[2], [x22], #0x2\n" - "ld1 { v26.h }[2], [x21], #0x2\n" - "ld1 { v22.h }[2], [x20], #0x2\n" + "ld1 { v9.h }[2], [x22], #0x2\n" + "ld1 { v29.h }[2], [x21], #0x2\n" + "ld1 { v28.h }[2], [x20], #0x2\n" "tbz x1, #0, 11f\n" "ld1 { v31.b }[6], [x9]\n" - "ld1 { v30.b }[6], [x28]\n" - "ld1 { v29.b }[6], [x27]\n" - "ld1 { v28.b }[6], [x26]\n" - "ld1 { v27.b }[6], [x25]\n" - "ld1 { v23.b }[6], [x24]\n" + "ld1 { v17.b }[6], [x28]\n" + "ld1 { v30.b }[6], [x27]\n" + "ld1 { v16.b }[6], [x26]\n" + "ld1 { v3.b }[6], [x25]\n" + "ld1 { v4.b }[6], [x24]\n" "ld1 { v25.b }[6], [x23]\n" - "ld1 { v24.b }[6], [x22]\n" - "ld1 { v26.b }[6], [x21]\n" - "ld1 { v22.b }[6], [x20]\n" + "ld1 { v9.b }[6], [x22]\n" + "ld1 { v29.b }[6], [x21]\n" + "ld1 { v28.b }[6], [x20]\n" "b 11f\n" "8:" // Oddments: Initial loads: Bit 2: Bit 1: Unset "tbz x1, #0, 11f\n" "ld1 { v31.b }[4], [x9]\n" - "ld1 { v30.b }[4], [x28]\n" - "ld1 { v29.b }[4], [x27]\n" - "ld1 { v28.b }[4], [x26]\n" - "ld1 { v27.b }[4], [x25]\n" - "ld1 { v23.b }[4], [x24]\n" + "ld1 { v17.b }[4], [x28]\n" + "ld1 { v30.b }[4], [x27]\n" + "ld1 { v16.b }[4], [x26]\n" + "ld1 { v3.b }[4], [x25]\n" + "ld1 { v4.b }[4], [x24]\n" "ld1 { v25.b }[4], [x23]\n" - "ld1 { v24.b }[4], [x22]\n" - "ld1 { v26.b }[4], [x21]\n" - "ld1 { v22.b }[4], [x20]\n" + "ld1 { v9.b }[4], [x22]\n" + "ld1 { v29.b }[4], [x21]\n" + "ld1 { v28.b }[4], [x20]\n" "b 11f\n" "9:" // Oddments: Initial loads: Bit 2: Unset "tbz x1, #1, 10f\n" "ld1 { v31.h }[0], [x9], #0x2\n" - "ld1 { v30.h }[0], [x28], #0x2\n" - "ld1 { v29.h }[0], [x27], #0x2\n" - "ld1 { v28.h }[0], [x26], #0x2\n" - "ld1 { v27.h }[0], [x25], #0x2\n" - "ld1 { v23.h }[0], [x24], #0x2\n" + "ld1 { v17.h }[0], [x28], #0x2\n" + "ld1 { v30.h }[0], [x27], #0x2\n" + "ld1 { v16.h }[0], [x26], #0x2\n" + "ld1 { v3.h }[0], [x25], #0x2\n" + "ld1 { v4.h }[0], [x24], #0x2\n" "ld1 { v25.h }[0], [x23], #0x2\n" - "ld1 { v24.h }[0], [x22], #0x2\n" - "ld1 { v26.h }[0], [x21], #0x2\n" - "ld1 { v22.h }[0], [x20], #0x2\n" + "ld1 { v9.h }[0], [x22], #0x2\n" + "ld1 { v29.h }[0], [x21], #0x2\n" + "ld1 { v28.h }[0], [x20], #0x2\n" "tbz x1, #0, 11f\n" "ld1 { v31.b }[2], [x9]\n" - "ld1 { v30.b }[2], [x28]\n" - "ld1 { v29.b }[2], [x27]\n" - "ld1 { v28.b }[2], [x26]\n" - "ld1 { v27.b }[2], [x25]\n" - "ld1 { v23.b }[2], [x24]\n" + "ld1 { v17.b }[2], [x28]\n" + "ld1 { v30.b }[2], [x27]\n" + "ld1 { v16.b }[2], [x26]\n" + "ld1 { v3.b }[2], [x25]\n" + "ld1 { v4.b }[2], [x24]\n" "ld1 { v25.b }[2], [x23]\n" - "ld1 { v24.b }[2], [x22]\n" - "ld1 { v26.b }[2], [x21]\n" - "ld1 { v22.b }[2], [x20]\n" + "ld1 { v9.b }[2], [x22]\n" + "ld1 { v29.b }[2], [x21]\n" + "ld1 { v28.b }[2], [x20]\n" "b 11f\n" "10:" // Oddments: Initial loads: Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 11f\n" "ld1 { v31.b }[0], [x9]\n" - "ld1 { v30.b }[0], [x28]\n" - "ld1 { v29.b }[0], [x27]\n" - "ld1 { v28.b }[0], [x26]\n" - "ld1 { v27.b }[0], [x25]\n" - "ld1 { v23.b }[0], [x24]\n" + "ld1 { v17.b }[0], [x28]\n" + "ld1 { v30.b }[0], [x27]\n" + "ld1 { v16.b }[0], [x26]\n" + "ld1 { v3.b }[0], [x25]\n" + "ld1 { v4.b }[0], [x24]\n" "ld1 { v25.b }[0], [x23]\n" - "ld1 { v24.b }[0], [x22]\n" - "ld1 { v26.b }[0], [x21]\n" - "ld1 { v22.b }[0], [x20]\n" + "ld1 { v9.b }[0], [x22]\n" + "ld1 { v29.b }[0], [x21]\n" + "ld1 { v28.b }[0], [x20]\n" "11:" // Oddments: Initial loads: Bit 2: End - "usubl v31.8h, v31.8b, v9.8b\n" - "usubl v30.8h, v30.8b, v9.8b\n" - "smlal v13.4s, v31.4h, v0.4h\n" - "ldr x20, [x4, #0x50]\n" - "usubl v29.8h, v29.8b, v9.8b\n" - "smlal2 v19.4s, v31.8h, v0.8h\n" - "smlal v20.4s, v30.4h, v0.4h\n" - "smlal2 v10.4s, v30.8h, v0.8h\n" - "smlal v8.4s, v29.4h, v0.4h\n" - "usubl v28.8h, v28.8b, v9.8b\n" - "add x20, x20, x0\n" - "smlal2 v7.4s, v29.8h, v0.8h\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal v17.4s, v28.4h, v0.4h\n" - "smlal2 v21.4s, v28.8h, v0.8h\n" - "smlal v13.4s, v30.4h, v1.4h\n" - "usubl v23.8h, v23.8b, v9.8b\n" - "smlal2 v19.4s, v30.8h, v1.8h\n" - "smlal v20.4s, v27.4h, v1.4h\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "smlal2 v10.4s, v27.8h, v1.8h\n" - "smlal v8.4s, v28.4h, v1.4h\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "smlal2 v7.4s, v28.8h, v1.8h\n" - "usubl v26.8h, v26.8b, v9.8b\n" - "smlal v17.4s, v23.4h, v1.4h\n" - "usubl v22.8h, v22.8b, v9.8b\n" - "smlal2 v21.4s, v23.8h, v1.8h\n" - "smlal v13.4s, v27.4h, v2.4h\n" - "smlal2 v19.4s, v27.8h, v2.8h\n" - "smlal v20.4s, v25.4h, v2.4h\n" - "smlal2 v10.4s, v25.8h, v2.8h\n" - "smlal v8.4s, v23.4h, v2.4h\n" - "smlal2 v7.4s, v23.8h, v2.8h\n" + "usubl v31.8h, v31.8b, v18.8b\n" + "usubl v17.8h, v17.8b, v18.8b\n" + "smlal v7.4s, v31.4h, v6.4h\n" + "ldr x20, [x5, #0x50]\n" + "usubl v30.8h, v30.8b, v18.8b\n" + "smlal2 v15.4s, v31.8h, v6.8h\n" + "smlal v20.4s, v17.4h, v6.4h\n" + "smlal2 v5.4s, v17.8h, v6.8h\n" + "smlal v24.4s, v30.4h, v6.4h\n" + "usubl v16.8h, v16.8b, v18.8b\n" + "add x20, x20, x3\n" + "smlal2 v22.4s, v30.8h, v6.8h\n" + "usubl v3.8h, v3.8b, v18.8b\n" + "smlal v23.4s, v16.4h, v6.4h\n" + "smlal2 v19.4s, v16.8h, v6.8h\n" + "smlal v7.4s, v17.4h, v14.4h\n" + "usubl v4.8h, v4.8b, v18.8b\n" + "smlal2 v15.4s, v17.8h, v14.8h\n" + "smlal v20.4s, v3.4h, v14.4h\n" + "usubl v25.8h, v25.8b, v18.8b\n" + "smlal2 v5.4s, v3.8h, v14.8h\n" + "smlal v24.4s, v16.4h, v14.4h\n" + "usubl v9.8h, v9.8b, v18.8b\n" + "smlal2 v22.4s, v16.8h, v14.8h\n" + "usubl v29.8h, v29.8b, v18.8b\n" + "smlal v23.4s, v4.4h, v14.4h\n" + "usubl v28.8h, v28.8b, v18.8b\n" + "smlal2 v19.4s, v4.8h, v14.8h\n" + "smlal v7.4s, v3.4h, v10.4h\n" + "smlal2 v15.4s, v3.8h, v10.8h\n" + "smlal v20.4s, v25.4h, v10.4h\n" + "smlal2 v5.4s, v25.8h, v10.8h\n" + "smlal v24.4s, v4.4h, v10.4h\n" + "smlal2 v22.4s, v4.8h, v10.8h\n" "tbz x1, #2, 13f\n" - "ld1 { v31.s }[0], [x20], #0x4\n" + "ld1 { v27.s }[0], [x20], #0x4\n" "tbz x1, #1, 12f\n" - "ld1 { v31.h }[2], [x20], #0x2\n" + "ld1 { v27.h }[2], [x20], #0x2\n" "tbz x1, #0, 15f\n" - "ld1 { v31.b }[6], [x20]\n" + "ld1 { v27.b }[6], [x20]\n" "b 15f\n" "12:" // Oddments: Load (1, 3): Bit 2: Bit 1: Unset "tbz x1, #0, 15f\n" - "ld1 { v31.b }[4], [x20]\n" + "ld1 { v27.b }[4], [x20]\n" "b 15f\n" "13:" // Oddments: Load (1, 3): Bit 2: Unset "tbz x1, #1, 14f\n" - "ld1 { v31.h }[0], [x20], #0x2\n" + "ld1 { v27.h }[0], [x20], #0x2\n" "tbz x1, #0, 15f\n" - "ld1 { v31.b }[2], [x20]\n" + "ld1 { v27.b }[2], [x20]\n" "b 15f\n" "14:" // Oddments: Load (1, 3): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 15f\n" - "ld1 { v31.b }[0], [x20]\n" + "ld1 { v27.b }[0], [x20]\n" "15:" // Oddments: Load (1, 3): Bit 2: End - "usubl v31.8h, v31.8b, v9.8b\n" - "ldr x22, [x4, #0x58]\n" - "smlal v17.4s, v31.4h, v2.4h\n" - "smlal2 v21.4s, v31.8h, v2.8h\n" - "smlal v13.4s, v25.4h, v3.4h\n" - "smlal2 v19.4s, v25.8h, v3.8h\n" - "add x22, x22, x0\n" - "smlal v20.4s, v24.4h, v3.4h\n" - "smlal2 v10.4s, v24.8h, v3.8h\n" - "smlal v8.4s, v31.4h, v3.4h\n" - "smlal2 v7.4s, v31.8h, v3.8h\n" + "usubl v27.8h, v27.8b, v18.8b\n" + "ldr x20, [x5, #0x58]\n" + "smlal v23.4s, v27.4h, v10.4h\n" + "smlal2 v19.4s, v27.8h, v10.8h\n" + "smlal v7.4s, v25.4h, v21.4h\n" + "smlal2 v15.4s, v25.8h, v21.8h\n" + "add x20, x20, x3\n" + "smlal v20.4s, v9.4h, v21.4h\n" + "smlal2 v5.4s, v9.8h, v21.8h\n" + "smlal v24.4s, v27.4h, v21.4h\n" + "smlal2 v22.4s, v27.8h, v21.8h\n" "tbz x1, #2, 17f\n" - "ld1 { v30.s }[0], [x22], #0x4\n" + "ld1 { v6.s }[0], [x20], #0x4\n" "tbz x1, #1, 16f\n" - "ld1 { v30.h }[2], [x22], #0x2\n" + "ld1 { v6.h }[2], [x20], #0x2\n" "tbz x1, #0, 19f\n" - "ld1 { v30.b }[6], [x22]\n" + "ld1 { v6.b }[6], [x20]\n" "b 19f\n" "16:" // Oddments: Load (1, 4): Bit 2: Bit 1: Unset "tbz x1, #0, 19f\n" - "ld1 { v30.b }[4], [x22]\n" + "ld1 { v6.b }[4], [x20]\n" "b 19f\n" "17:" // Oddments: Load (1, 4): Bit 2: Unset "tbz x1, #1, 18f\n" - "ld1 { v30.h }[0], [x22], #0x2\n" + "ld1 { v6.h }[0], [x20], #0x2\n" "tbz x1, #0, 19f\n" - "ld1 { v30.b }[2], [x22]\n" + "ld1 { v6.b }[2], [x20]\n" "b 19f\n" "18:" // Oddments: Load (1, 4): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 19f\n" - "ld1 { v30.b }[0], [x22]\n" + "ld1 { v6.b }[0], [x20]\n" "19:" // Oddments: Load (1, 4): Bit 2: End - "usubl v30.8h, v30.8b, v9.8b\n" - "ldr x21, [x4, #0x60]\n" - "smlal v17.4s, v30.4h, v3.4h\n" - "smlal2 v21.4s, v30.8h, v3.8h\n" - "smlal v13.4s, v24.4h, v4.4h\n" - "smlal2 v19.4s, v24.8h, v4.8h\n" - "add x21, x21, x0\n" + "usubl v6.8h, v6.8b, v18.8b\n" + "ldr x20, [x5, #0x60]\n" + "smlal v23.4s, v6.4h, v21.4h\n" + "smlal2 v19.4s, v6.8h, v21.8h\n" + "smlal v7.4s, v9.4h, v12.4h\n" + "smlal2 v15.4s, v9.8h, v12.8h\n" + "add x20, x20, x3\n" "tbz x1, #2, 21f\n" - "ld1 { v27.s }[0], [x21], #0x4\n" + "ld1 { v9.s }[0], [x20], #0x4\n" "tbz x1, #1, 20f\n" - "ld1 { v27.h }[2], [x21], #0x2\n" + "ld1 { v9.h }[2], [x20], #0x2\n" "tbz x1, #0, 23f\n" - "ld1 { v27.b }[6], [x21]\n" + "ld1 { v9.b }[6], [x20]\n" "b 23f\n" "20:" // Oddments: Load (0, 5): Bit 2: Bit 1: Unset "tbz x1, #0, 23f\n" - "ld1 { v27.b }[4], [x21]\n" + "ld1 { v9.b }[4], [x20]\n" "b 23f\n" "21:" // Oddments: Load (0, 5): Bit 2: Unset "tbz x1, #1, 22f\n" - "ld1 { v27.h }[0], [x21], #0x2\n" + "ld1 { v9.h }[0], [x20], #0x2\n" "tbz x1, #0, 23f\n" - "ld1 { v27.b }[2], [x21]\n" + "ld1 { v9.b }[2], [x20]\n" "b 23f\n" "22:" // Oddments: Load (0, 5): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 23f\n" - "ld1 { v27.b }[0], [x21]\n" + "ld1 { v9.b }[0], [x20]\n" "23:" // Oddments: Load (0, 5): Bit 2: End - "ldr d0, [x3, #0x28]\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal v20.4s, v27.4h, v4.4h\n" - "smlal2 v10.4s, v27.8h, v4.8h\n" - "smlal v8.4s, v30.4h, v4.4h\n" - "smlal2 v7.4s, v30.8h, v4.8h\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "ldr x20, [x4, #0x68]\n" - "smlal v17.4s, v26.4h, v4.4h\n" - "smlal2 v21.4s, v26.8h, v4.8h\n" - "add x20, x20, x0\n" - "smlal v13.4s, v29.4h, v0.4h\n" - "smlal2 v19.4s, v29.8h, v0.8h\n" - "smlal v20.4s, v28.4h, v0.4h\n" - "smlal2 v10.4s, v28.8h, v0.8h\n" - "smlal v8.4s, v22.4h, v0.4h\n" - "smlal2 v7.4s, v22.8h, v0.8h\n" + "ldr d14, [x6, #0x28]\n" + "usubl v9.8h, v9.8b, v18.8b\n" + "smlal v20.4s, v9.4h, v12.4h\n" + "smlal2 v5.4s, v9.8h, v12.8h\n" + "smlal v24.4s, v6.4h, v12.4h\n" + "smlal2 v22.4s, v6.8h, v12.8h\n" + "ssubl v14.8h, v14.8b, v13.8b\n" + "ldr x20, [x5, #0x68]\n" + "smlal v23.4s, v29.4h, v12.4h\n" + "smlal2 v19.4s, v29.8h, v12.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v30.4h, v14.4h\n" + "smlal2 v15.4s, v30.8h, v14.8h\n" + "smlal v20.4s, v16.4h, v14.4h\n" + "smlal2 v5.4s, v16.8h, v14.8h\n" + "smlal v24.4s, v28.4h, v14.4h\n" + "smlal2 v22.4s, v28.8h, v14.8h\n" "tbz x1, #2, 25f\n" "ld1 { v25.s }[0], [x20], #0x4\n" "tbz x1, #1, 24f\n" @@ -1315,869 +1315,869 @@ void a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( "tbz x1, #0, 27f\n" "ld1 { v25.b }[0], [x20]\n" "27:" // Oddments: Load (2, 1): Bit 2: End - "ldr d1, [x3, #0x30]\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "ldr x26, [x4, #0x70]\n" - "smlal v17.4s, v25.4h, v0.4h\n" - "smlal2 v21.4s, v25.8h, v0.8h\n" - "add x26, x26, x0\n" - "smlal v13.4s, v28.4h, v1.4h\n" - "smlal2 v19.4s, v28.8h, v1.8h\n" - "smlal v20.4s, v23.4h, v1.4h\n" - "smlal2 v10.4s, v23.8h, v1.8h\n" - "smlal v8.4s, v25.4h, v1.4h\n" - "smlal2 v7.4s, v25.8h, v1.8h\n" + "ldr d21, [x6, #0x30]\n" + "usubl v25.8h, v25.8b, v18.8b\n" + "ssubl v21.8h, v21.8b, v13.8b\n" + "ldr x20, [x5, #0x70]\n" + "smlal v23.4s, v25.4h, v14.4h\n" + "smlal2 v19.4s, v25.8h, v14.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v16.4h, v21.4h\n" + "smlal2 v15.4s, v16.8h, v21.8h\n" + "smlal v20.4s, v4.4h, v21.4h\n" + "smlal2 v5.4s, v4.8h, v21.8h\n" + "smlal v24.4s, v25.4h, v21.4h\n" + "smlal2 v22.4s, v25.8h, v21.8h\n" "tbz x1, #2, 29f\n" - "ld1 { v24.s }[0], [x26], #0x4\n" + "ld1 { v10.s }[0], [x20], #0x4\n" "tbz x1, #1, 28f\n" - "ld1 { v24.h }[2], [x26], #0x2\n" + "ld1 { v10.h }[2], [x20], #0x2\n" "tbz x1, #0, 31f\n" - "ld1 { v24.b }[6], [x26]\n" + "ld1 { v10.b }[6], [x20]\n" "b 31f\n" "28:" // Oddments: Load (2, 2): Bit 2: Bit 1: Unset "tbz x1, #0, 31f\n" - "ld1 { v24.b }[4], [x26]\n" + "ld1 { v10.b }[4], [x20]\n" "b 31f\n" "29:" // Oddments: Load (2, 2): Bit 2: Unset "tbz x1, #1, 30f\n" - "ld1 { v24.h }[0], [x26], #0x2\n" + "ld1 { v10.h }[0], [x20], #0x2\n" "tbz x1, #0, 31f\n" - "ld1 { v24.b }[2], [x26]\n" + "ld1 { v10.b }[2], [x20]\n" "b 31f\n" "30:" // Oddments: Load (2, 2): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 31f\n" - "ld1 { v24.b }[0], [x26]\n" + "ld1 { v10.b }[0], [x20]\n" "31:" // Oddments: Load (2, 2): Bit 2: End - "ldr d2, [x3, #0x38]\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "ldr x25, [x4, #0x78]\n" - "smlal v17.4s, v24.4h, v1.4h\n" - "smlal2 v21.4s, v24.8h, v1.8h\n" - "add x25, x25, x0\n" - "smlal v13.4s, v23.4h, v2.4h\n" - "smlal2 v19.4s, v23.8h, v2.8h\n" - "smlal v20.4s, v31.4h, v2.4h\n" - "smlal2 v10.4s, v31.8h, v2.8h\n" - "smlal v8.4s, v24.4h, v2.4h\n" - "smlal2 v7.4s, v24.8h, v2.8h\n" + "ldr d9, [x6, #0x38]\n" + "usubl v10.8h, v10.8b, v18.8b\n" + "ssubl v9.8h, v9.8b, v13.8b\n" + "ldr x20, [x5, #0x78]\n" + "smlal v23.4s, v10.4h, v21.4h\n" + "smlal2 v19.4s, v10.8h, v21.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v4.4h, v9.4h\n" + "smlal2 v15.4s, v4.8h, v9.8h\n" + "smlal v20.4s, v27.4h, v9.4h\n" + "smlal2 v5.4s, v27.8h, v9.8h\n" + "smlal v24.4s, v10.4h, v9.4h\n" + "smlal2 v22.4s, v10.8h, v9.8h\n" "tbz x1, #2, 33f\n" - "ld1 { v27.s }[0], [x25], #0x4\n" + "ld1 { v12.s }[0], [x20], #0x4\n" "tbz x1, #1, 32f\n" - "ld1 { v27.h }[2], [x25], #0x2\n" + "ld1 { v12.h }[2], [x20], #0x2\n" "tbz x1, #0, 35f\n" - "ld1 { v27.b }[6], [x25]\n" + "ld1 { v12.b }[6], [x20]\n" "b 35f\n" "32:" // Oddments: Load (2, 3): Bit 2: Bit 1: Unset "tbz x1, #0, 35f\n" - "ld1 { v27.b }[4], [x25]\n" + "ld1 { v12.b }[4], [x20]\n" "b 35f\n" "33:" // Oddments: Load (2, 3): Bit 2: Unset "tbz x1, #1, 34f\n" - "ld1 { v27.h }[0], [x25], #0x2\n" + "ld1 { v12.h }[0], [x20], #0x2\n" "tbz x1, #0, 35f\n" - "ld1 { v27.b }[2], [x25]\n" + "ld1 { v12.b }[2], [x20]\n" "b 35f\n" "34:" // Oddments: Load (2, 3): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 35f\n" - "ld1 { v27.b }[0], [x25]\n" + "ld1 { v12.b }[0], [x20]\n" "35:" // Oddments: Load (2, 3): Bit 2: End - "ldr d3, [x3, #0x40]\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "ldr x23, [x4, #0x80]\n" - "smlal v17.4s, v27.4h, v2.4h\n" - "smlal2 v21.4s, v27.8h, v2.8h\n" - "add x23, x23, x0\n" - "smlal v13.4s, v31.4h, v3.4h\n" - "smlal2 v19.4s, v31.8h, v3.8h\n" - "smlal v20.4s, v30.4h, v3.4h\n" - "smlal2 v10.4s, v30.8h, v3.8h\n" - "smlal v8.4s, v27.4h, v3.4h\n" - "smlal2 v7.4s, v27.8h, v3.8h\n" + "ldr d31, [x6, #0x40]\n" + "usubl v12.8h, v12.8b, v18.8b\n" + "ssubl v31.8h, v31.8b, v13.8b\n" + "ldr x20, [x5, #0x80]\n" + "smlal v23.4s, v12.4h, v9.4h\n" + "smlal2 v19.4s, v12.8h, v9.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v27.4h, v31.4h\n" + "smlal2 v15.4s, v27.8h, v31.8h\n" + "smlal v20.4s, v6.4h, v31.4h\n" + "smlal2 v5.4s, v6.8h, v31.8h\n" + "smlal v24.4s, v12.4h, v31.4h\n" + "smlal2 v22.4s, v12.8h, v31.8h\n" "tbz x1, #2, 37f\n" - "ld1 { v23.s }[0], [x23], #0x4\n" + "ld1 { v8.s }[0], [x20], #0x4\n" "tbz x1, #1, 36f\n" - "ld1 { v23.h }[2], [x23], #0x2\n" + "ld1 { v8.h }[2], [x20], #0x2\n" "tbz x1, #0, 39f\n" - "ld1 { v23.b }[6], [x23]\n" + "ld1 { v8.b }[6], [x20]\n" "b 39f\n" "36:" // Oddments: Load (2, 4): Bit 2: Bit 1: Unset "tbz x1, #0, 39f\n" - "ld1 { v23.b }[4], [x23]\n" + "ld1 { v8.b }[4], [x20]\n" "b 39f\n" "37:" // Oddments: Load (2, 4): Bit 2: Unset "tbz x1, #1, 38f\n" - "ld1 { v23.h }[0], [x23], #0x2\n" + "ld1 { v8.h }[0], [x20], #0x2\n" "tbz x1, #0, 39f\n" - "ld1 { v23.b }[2], [x23]\n" + "ld1 { v8.b }[2], [x20]\n" "b 39f\n" "38:" // Oddments: Load (2, 4): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 39f\n" - "ld1 { v23.b }[0], [x23]\n" + "ld1 { v8.b }[0], [x20]\n" "39:" // Oddments: Load (2, 4): Bit 2: End - "ldr d4, [x3, #0x48]\n" - "usubl v23.8h, v23.8b, v9.8b\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "ldr x24, [x4, #0x88]\n" - "smlal v17.4s, v23.4h, v3.4h\n" - "smlal2 v21.4s, v23.8h, v3.8h\n" - "add x24, x24, x0\n" - "smlal v13.4s, v30.4h, v4.4h\n" - "smlal2 v19.4s, v30.8h, v4.8h\n" - "smlal v20.4s, v26.4h, v4.4h\n" - "smlal2 v10.4s, v26.8h, v4.8h\n" - "smlal v8.4s, v23.4h, v4.4h\n" - "smlal2 v7.4s, v23.8h, v4.8h\n" + "ldr d16, [x6, #0x48]\n" + "usubl v8.8h, v8.8b, v18.8b\n" + "ssubl v16.8h, v16.8b, v13.8b\n" + "ldr x20, [x5, #0x88]\n" + "smlal v23.4s, v8.4h, v31.4h\n" + "smlal2 v19.4s, v8.8h, v31.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v6.4h, v16.4h\n" + "smlal2 v15.4s, v6.8h, v16.8h\n" + "smlal v20.4s, v29.4h, v16.4h\n" + "smlal2 v5.4s, v29.8h, v16.8h\n" + "smlal v24.4s, v8.4h, v16.4h\n" + "smlal2 v22.4s, v8.8h, v16.8h\n" "tbz x1, #2, 41f\n" - "ld1 { v28.s }[0], [x24], #0x4\n" + "ld1 { v27.s }[0], [x20], #0x4\n" "tbz x1, #1, 40f\n" - "ld1 { v28.h }[2], [x24], #0x2\n" + "ld1 { v27.h }[2], [x20], #0x2\n" "tbz x1, #0, 43f\n" - "ld1 { v28.b }[6], [x24]\n" + "ld1 { v27.b }[6], [x20]\n" "b 43f\n" "40:" // Oddments: Load (2, 5): Bit 2: Bit 1: Unset "tbz x1, #0, 43f\n" - "ld1 { v28.b }[4], [x24]\n" + "ld1 { v27.b }[4], [x20]\n" "b 43f\n" "41:" // Oddments: Load (2, 5): Bit 2: Unset "tbz x1, #1, 42f\n" - "ld1 { v28.h }[0], [x24], #0x2\n" + "ld1 { v27.h }[0], [x20], #0x2\n" "tbz x1, #0, 43f\n" - "ld1 { v28.b }[2], [x24]\n" + "ld1 { v27.b }[2], [x20]\n" "b 43f\n" "42:" // Oddments: Load (2, 5): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 43f\n" - "ld1 { v28.b }[0], [x24]\n" + "ld1 { v27.b }[0], [x20]\n" "43:" // Oddments: Load (2, 5): Bit 2: End - "ldr d0, [x3, #0x50]\n" - "usubl v28.8h, v28.8b, v9.8b\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "ldr x15, [x4, #0x90]\n" - "smlal v17.4s, v28.4h, v4.4h\n" - "smlal2 v21.4s, v28.8h, v4.8h\n" - "add x15, x15, x0\n" - "smlal v13.4s, v22.4h, v0.4h\n" - "smlal2 v19.4s, v22.8h, v0.8h\n" - "smlal v20.4s, v25.4h, v0.4h\n" - "smlal2 v10.4s, v25.8h, v0.8h\n" + "ldr d21, [x6, #0x50]\n" + "usubl v27.8h, v27.8b, v18.8b\n" + "ssubl v21.8h, v21.8b, v13.8b\n" + "ldr x20, [x5, #0x90]\n" + "smlal v23.4s, v27.4h, v16.4h\n" + "smlal2 v19.4s, v27.8h, v16.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v28.4h, v21.4h\n" + "smlal2 v15.4s, v28.8h, v21.8h\n" + "smlal v20.4s, v25.4h, v21.4h\n" + "smlal2 v5.4s, v25.8h, v21.8h\n" "tbz x1, #2, 45f\n" - "ld1 { v31.s }[0], [x15], #0x4\n" + "ld1 { v31.s }[0], [x20], #0x4\n" "tbz x1, #1, 44f\n" - "ld1 { v31.h }[2], [x15], #0x2\n" + "ld1 { v31.h }[2], [x20], #0x2\n" "tbz x1, #0, 47f\n" - "ld1 { v31.b }[6], [x15]\n" + "ld1 { v31.b }[6], [x20]\n" "b 47f\n" "44:" // Oddments: Load (3, 0): Bit 2: Bit 1: Unset "tbz x1, #0, 47f\n" - "ld1 { v31.b }[4], [x15]\n" + "ld1 { v31.b }[4], [x20]\n" "b 47f\n" "45:" // Oddments: Load (3, 0): Bit 2: Unset "tbz x1, #1, 46f\n" - "ld1 { v31.h }[0], [x15], #0x2\n" + "ld1 { v31.h }[0], [x20], #0x2\n" "tbz x1, #0, 47f\n" - "ld1 { v31.b }[2], [x15]\n" + "ld1 { v31.b }[2], [x20]\n" "b 47f\n" "46:" // Oddments: Load (3, 0): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 47f\n" - "ld1 { v31.b }[0], [x15]\n" + "ld1 { v31.b }[0], [x20]\n" "47:" // Oddments: Load (3, 0): Bit 2: End - "usubl v31.8h, v31.8b, v9.8b\n" - "ldr x21, [x4, #0x98]\n" - "smlal v8.4s, v31.4h, v0.4h\n" - "smlal2 v7.4s, v31.8h, v0.8h\n" - "add x21, x21, x0\n" + "usubl v31.8h, v31.8b, v18.8b\n" + "ldr x20, [x5, #0x98]\n" + "smlal v24.4s, v31.4h, v21.4h\n" + "smlal2 v22.4s, v31.8h, v21.8h\n" + "add x20, x20, x3\n" "tbz x1, #2, 49f\n" - "ld1 { v30.s }[0], [x21], #0x4\n" + "ld1 { v28.s }[0], [x20], #0x4\n" "tbz x1, #1, 48f\n" - "ld1 { v30.h }[2], [x21], #0x2\n" + "ld1 { v28.h }[2], [x20], #0x2\n" "tbz x1, #0, 51f\n" - "ld1 { v30.b }[6], [x21]\n" + "ld1 { v28.b }[6], [x20]\n" "b 51f\n" "48:" // Oddments: Load (3, 1): Bit 2: Bit 1: Unset "tbz x1, #0, 51f\n" - "ld1 { v30.b }[4], [x21]\n" + "ld1 { v28.b }[4], [x20]\n" "b 51f\n" "49:" // Oddments: Load (3, 1): Bit 2: Unset "tbz x1, #1, 50f\n" - "ld1 { v30.h }[0], [x21], #0x2\n" + "ld1 { v28.h }[0], [x20], #0x2\n" "tbz x1, #0, 51f\n" - "ld1 { v30.b }[2], [x21]\n" + "ld1 { v28.b }[2], [x20]\n" "b 51f\n" "50:" // Oddments: Load (3, 1): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 51f\n" - "ld1 { v30.b }[0], [x21]\n" + "ld1 { v28.b }[0], [x20]\n" "51:" // Oddments: Load (3, 1): Bit 2: End - "ldr d1, [x3, #0x58]\n" - "usubl v30.8h, v30.8b, v9.8b\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "ldr x14, [x4, #0xa0]\n" - "smlal v17.4s, v30.4h, v0.4h\n" - "smlal2 v21.4s, v30.8h, v0.8h\n" - "add x14, x14, x0\n" - "smlal v13.4s, v25.4h, v1.4h\n" - "smlal2 v19.4s, v25.8h, v1.8h\n" - "smlal v20.4s, v24.4h, v1.4h\n" - "smlal2 v10.4s, v24.8h, v1.8h\n" - "smlal v8.4s, v30.4h, v1.4h\n" - "smlal2 v7.4s, v30.8h, v1.8h\n" + "ldr d2, [x6, #0x58]\n" + "usubl v28.8h, v28.8b, v18.8b\n" + "ssubl v2.8h, v2.8b, v13.8b\n" + "ldr x20, [x5, #0xa0]\n" + "smlal v23.4s, v28.4h, v21.4h\n" + "smlal2 v19.4s, v28.8h, v21.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v25.4h, v2.4h\n" + "smlal2 v15.4s, v25.8h, v2.8h\n" + "smlal v20.4s, v10.4h, v2.4h\n" + "smlal2 v5.4s, v10.8h, v2.8h\n" + "smlal v24.4s, v28.4h, v2.4h\n" + "smlal2 v22.4s, v28.8h, v2.8h\n" "tbz x1, #2, 53f\n" - "ld1 { v26.s }[0], [x14], #0x4\n" + "ld1 { v21.s }[0], [x20], #0x4\n" "tbz x1, #1, 52f\n" - "ld1 { v26.h }[2], [x14], #0x2\n" + "ld1 { v21.h }[2], [x20], #0x2\n" "tbz x1, #0, 55f\n" - "ld1 { v26.b }[6], [x14]\n" + "ld1 { v21.b }[6], [x20]\n" "b 55f\n" "52:" // Oddments: Load (3, 2): Bit 2: Bit 1: Unset "tbz x1, #0, 55f\n" - "ld1 { v26.b }[4], [x14]\n" + "ld1 { v21.b }[4], [x20]\n" "b 55f\n" "53:" // Oddments: Load (3, 2): Bit 2: Unset "tbz x1, #1, 54f\n" - "ld1 { v26.h }[0], [x14], #0x2\n" + "ld1 { v21.h }[0], [x20], #0x2\n" "tbz x1, #0, 55f\n" - "ld1 { v26.b }[2], [x14]\n" + "ld1 { v21.b }[2], [x20]\n" "b 55f\n" "54:" // Oddments: Load (3, 2): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 55f\n" - "ld1 { v26.b }[0], [x14]\n" + "ld1 { v21.b }[0], [x20]\n" "55:" // Oddments: Load (3, 2): Bit 2: End - "ldr d2, [x3, #0x60]\n" - "usubl v26.8h, v26.8b, v9.8b\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "ldr x13, [x4, #0xa8]\n" - "smlal v17.4s, v26.4h, v1.4h\n" - "smlal2 v21.4s, v26.8h, v1.8h\n" - "add x13, x13, x0\n" - "smlal v13.4s, v24.4h, v2.4h\n" - "smlal2 v19.4s, v24.8h, v2.8h\n" - "smlal v20.4s, v27.4h, v2.4h\n" - "smlal2 v10.4s, v27.8h, v2.8h\n" - "smlal v8.4s, v26.4h, v2.4h\n" - "smlal2 v7.4s, v26.8h, v2.8h\n" + "ldr d25, [x6, #0x60]\n" + "usubl v21.8h, v21.8b, v18.8b\n" + "ssubl v25.8h, v25.8b, v13.8b\n" + "ldr x20, [x5, #0xa8]\n" + "smlal v23.4s, v21.4h, v2.4h\n" + "smlal2 v19.4s, v21.8h, v2.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v10.4h, v25.4h\n" + "smlal2 v15.4s, v10.8h, v25.8h\n" + "smlal v20.4s, v12.4h, v25.4h\n" + "smlal2 v5.4s, v12.8h, v25.8h\n" + "smlal v24.4s, v21.4h, v25.4h\n" + "smlal2 v22.4s, v21.8h, v25.8h\n" "tbz x1, #2, 57f\n" - "ld1 { v25.s }[0], [x13], #0x4\n" + "ld1 { v9.s }[0], [x20], #0x4\n" "tbz x1, #1, 56f\n" - "ld1 { v25.h }[2], [x13], #0x2\n" + "ld1 { v9.h }[2], [x20], #0x2\n" "tbz x1, #0, 59f\n" - "ld1 { v25.b }[6], [x13]\n" + "ld1 { v9.b }[6], [x20]\n" "b 59f\n" "56:" // Oddments: Load (3, 3): Bit 2: Bit 1: Unset "tbz x1, #0, 59f\n" - "ld1 { v25.b }[4], [x13]\n" + "ld1 { v9.b }[4], [x20]\n" "b 59f\n" "57:" // Oddments: Load (3, 3): Bit 2: Unset "tbz x1, #1, 58f\n" - "ld1 { v25.h }[0], [x13], #0x2\n" + "ld1 { v9.h }[0], [x20], #0x2\n" "tbz x1, #0, 59f\n" - "ld1 { v25.b }[2], [x13]\n" + "ld1 { v9.b }[2], [x20]\n" "b 59f\n" "58:" // Oddments: Load (3, 3): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 59f\n" - "ld1 { v25.b }[0], [x13]\n" + "ld1 { v9.b }[0], [x20]\n" "59:" // Oddments: Load (3, 3): Bit 2: End - "ldr d3, [x3, #0x68]\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "ldr x12, [x4, #0xb0]\n" - "smlal v17.4s, v25.4h, v2.4h\n" - "smlal2 v21.4s, v25.8h, v2.8h\n" - "add x12, x12, x0\n" - "smlal v13.4s, v27.4h, v3.4h\n" - "smlal2 v19.4s, v27.8h, v3.8h\n" - "smlal v20.4s, v23.4h, v3.4h\n" - "smlal2 v10.4s, v23.8h, v3.8h\n" - "smlal v8.4s, v25.4h, v3.4h\n" - "smlal2 v7.4s, v25.8h, v3.8h\n" + "ldr d1, [x6, #0x68]\n" + "usubl v9.8h, v9.8b, v18.8b\n" + "ssubl v1.8h, v1.8b, v13.8b\n" + "ldr x20, [x5, #0xb0]\n" + "smlal v23.4s, v9.4h, v25.4h\n" + "smlal2 v19.4s, v9.8h, v25.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v12.4h, v1.4h\n" + "smlal2 v15.4s, v12.8h, v1.8h\n" + "smlal v20.4s, v8.4h, v1.4h\n" + "smlal2 v5.4s, v8.8h, v1.8h\n" + "smlal v24.4s, v9.4h, v1.4h\n" + "smlal2 v22.4s, v9.8h, v1.8h\n" "tbz x1, #2, 61f\n" - "ld1 { v24.s }[0], [x12], #0x4\n" + "ld1 { v3.s }[0], [x20], #0x4\n" "tbz x1, #1, 60f\n" - "ld1 { v24.h }[2], [x12], #0x2\n" + "ld1 { v3.h }[2], [x20], #0x2\n" "tbz x1, #0, 63f\n" - "ld1 { v24.b }[6], [x12]\n" + "ld1 { v3.b }[6], [x20]\n" "b 63f\n" "60:" // Oddments: Load (3, 4): Bit 2: Bit 1: Unset "tbz x1, #0, 63f\n" - "ld1 { v24.b }[4], [x12]\n" + "ld1 { v3.b }[4], [x20]\n" "b 63f\n" "61:" // Oddments: Load (3, 4): Bit 2: Unset "tbz x1, #1, 62f\n" - "ld1 { v24.h }[0], [x12], #0x2\n" + "ld1 { v3.h }[0], [x20], #0x2\n" "tbz x1, #0, 63f\n" - "ld1 { v24.b }[2], [x12]\n" + "ld1 { v3.b }[2], [x20]\n" "b 63f\n" "62:" // Oddments: Load (3, 4): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 63f\n" - "ld1 { v24.b }[0], [x12]\n" + "ld1 { v3.b }[0], [x20]\n" "63:" // Oddments: Load (3, 4): Bit 2: End - "ldr d4, [x3, #0x70]\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "ldr x20, [x4, #0xb8]\n" - "smlal v17.4s, v24.4h, v3.4h\n" - "smlal2 v21.4s, v24.8h, v3.8h\n" - "add x20, x20, x0\n" - "smlal v13.4s, v23.4h, v4.4h\n" - "smlal2 v19.4s, v23.8h, v4.8h\n" - "smlal v20.4s, v28.4h, v4.4h\n" - "smlal2 v10.4s, v28.8h, v4.8h\n" - "smlal v8.4s, v24.4h, v4.4h\n" - "smlal2 v7.4s, v24.8h, v4.8h\n" + "ldr d16, [x6, #0x70]\n" + "usubl v3.8h, v3.8b, v18.8b\n" + "ssubl v16.8h, v16.8b, v13.8b\n" + "ldr x20, [x5, #0xb8]\n" + "smlal v23.4s, v3.4h, v1.4h\n" + "smlal2 v19.4s, v3.8h, v1.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v8.4h, v16.4h\n" + "smlal2 v15.4s, v8.8h, v16.8h\n" + "smlal v20.4s, v27.4h, v16.4h\n" + "smlal2 v5.4s, v27.8h, v16.8h\n" + "smlal v24.4s, v3.4h, v16.4h\n" + "smlal2 v22.4s, v3.8h, v16.8h\n" "tbz x1, #2, 65f\n" - "ld1 { v22.s }[0], [x20], #0x4\n" + "ld1 { v14.s }[0], [x20], #0x4\n" "tbz x1, #1, 64f\n" - "ld1 { v22.h }[2], [x20], #0x2\n" + "ld1 { v14.h }[2], [x20], #0x2\n" "tbz x1, #0, 67f\n" - "ld1 { v22.b }[6], [x20]\n" + "ld1 { v14.b }[6], [x20]\n" "b 67f\n" "64:" // Oddments: Load (3, 5): Bit 2: Bit 1: Unset "tbz x1, #0, 67f\n" - "ld1 { v22.b }[4], [x20]\n" + "ld1 { v14.b }[4], [x20]\n" "b 67f\n" "65:" // Oddments: Load (3, 5): Bit 2: Unset "tbz x1, #1, 66f\n" - "ld1 { v22.h }[0], [x20], #0x2\n" + "ld1 { v14.h }[0], [x20], #0x2\n" "tbz x1, #0, 67f\n" - "ld1 { v22.b }[2], [x20]\n" + "ld1 { v14.b }[2], [x20]\n" "b 67f\n" "66:" // Oddments: Load (3, 5): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 67f\n" - "ld1 { v22.b }[0], [x20]\n" + "ld1 { v14.b }[0], [x20]\n" "67:" // Oddments: Load (3, 5): Bit 2: End - "ldr d0, [x3, #0x78]\n" - "usubl v22.8h, v22.8b, v9.8b\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "ldr x11, [x4, #0xc0]\n" - "smlal v17.4s, v22.4h, v4.4h\n" - "smlal2 v21.4s, v22.8h, v4.8h\n" - "add x11, x11, x0\n" - "smlal v13.4s, v31.4h, v0.4h\n" - "smlal2 v19.4s, v31.8h, v0.8h\n" - "smlal v20.4s, v30.4h, v0.4h\n" - "smlal2 v10.4s, v30.8h, v0.8h\n" + "ldr d17, [x6, #0x78]\n" + "usubl v14.8h, v14.8b, v18.8b\n" + "ssubl v17.8h, v17.8b, v13.8b\n" + "ldr x20, [x5, #0xc0]\n" + "smlal v23.4s, v14.4h, v16.4h\n" + "smlal2 v19.4s, v14.8h, v16.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v31.4h, v17.4h\n" + "smlal2 v15.4s, v31.8h, v17.8h\n" + "smlal v20.4s, v28.4h, v17.4h\n" + "smlal2 v5.4s, v28.8h, v17.8h\n" "tbz x1, #2, 69f\n" - "ld1 { v27.s }[0], [x11], #0x4\n" + "ld1 { v1.s }[0], [x20], #0x4\n" "tbz x1, #1, 68f\n" - "ld1 { v27.h }[2], [x11], #0x2\n" + "ld1 { v1.h }[2], [x20], #0x2\n" "tbz x1, #0, 71f\n" - "ld1 { v27.b }[6], [x11]\n" + "ld1 { v1.b }[6], [x20]\n" "b 71f\n" "68:" // Oddments: Load (4, 0): Bit 2: Bit 1: Unset "tbz x1, #0, 71f\n" - "ld1 { v27.b }[4], [x11]\n" + "ld1 { v1.b }[4], [x20]\n" "b 71f\n" "69:" // Oddments: Load (4, 0): Bit 2: Unset "tbz x1, #1, 70f\n" - "ld1 { v27.h }[0], [x11], #0x2\n" + "ld1 { v1.h }[0], [x20], #0x2\n" "tbz x1, #0, 71f\n" - "ld1 { v27.b }[2], [x11]\n" + "ld1 { v1.b }[2], [x20]\n" "b 71f\n" "70:" // Oddments: Load (4, 0): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 71f\n" - "ld1 { v27.b }[0], [x11]\n" + "ld1 { v1.b }[0], [x20]\n" "71:" // Oddments: Load (4, 0): Bit 2: End - "usubl v27.8h, v27.8b, v9.8b\n" - "ldr x22, [x4, #0xc8]\n" - "smlal v8.4s, v27.4h, v0.4h\n" - "smlal2 v7.4s, v27.8h, v0.8h\n" - "add x22, x22, x0\n" + "usubl v1.8h, v1.8b, v18.8b\n" + "ldr x20, [x5, #0xc8]\n" + "smlal v24.4s, v1.4h, v17.4h\n" + "smlal2 v22.4s, v1.8h, v17.8h\n" + "add x20, x20, x3\n" "tbz x1, #2, 73f\n" - "ld1 { v23.s }[0], [x22], #0x4\n" + "ld1 { v16.s }[0], [x20], #0x4\n" "tbz x1, #1, 72f\n" - "ld1 { v23.h }[2], [x22], #0x2\n" + "ld1 { v16.h }[2], [x20], #0x2\n" "tbz x1, #0, 75f\n" - "ld1 { v23.b }[6], [x22]\n" + "ld1 { v16.b }[6], [x20]\n" "b 75f\n" "72:" // Oddments: Load (4, 1): Bit 2: Bit 1: Unset "tbz x1, #0, 75f\n" - "ld1 { v23.b }[4], [x22]\n" + "ld1 { v16.b }[4], [x20]\n" "b 75f\n" "73:" // Oddments: Load (4, 1): Bit 2: Unset "tbz x1, #1, 74f\n" - "ld1 { v23.h }[0], [x22], #0x2\n" + "ld1 { v16.h }[0], [x20], #0x2\n" "tbz x1, #0, 75f\n" - "ld1 { v23.b }[2], [x22]\n" + "ld1 { v16.b }[2], [x20]\n" "b 75f\n" "74:" // Oddments: Load (4, 1): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 75f\n" - "ld1 { v23.b }[0], [x22]\n" + "ld1 { v16.b }[0], [x20]\n" "75:" // Oddments: Load (4, 1): Bit 2: End - "ldr d1, [x3, #0x80]\n" - "usubl v23.8h, v23.8b, v9.8b\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "ldr x9, [x4, #0xd0]\n" - "smlal v17.4s, v23.4h, v0.4h\n" - "smlal2 v21.4s, v23.8h, v0.8h\n" - "add x9, x9, x0\n" - "smlal v13.4s, v30.4h, v1.4h\n" - "smlal2 v19.4s, v30.8h, v1.8h\n" - "smlal v20.4s, v26.4h, v1.4h\n" - "smlal2 v10.4s, v26.8h, v1.8h\n" - "smlal v8.4s, v23.4h, v1.4h\n" - "smlal2 v7.4s, v23.8h, v1.8h\n" + "ldr d29, [x6, #0x80]\n" + "usubl v16.8h, v16.8b, v18.8b\n" + "ssubl v29.8h, v29.8b, v13.8b\n" + "ldr x20, [x5, #0xd0]\n" + "smlal v23.4s, v16.4h, v17.4h\n" + "smlal2 v19.4s, v16.8h, v17.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v28.4h, v29.4h\n" + "smlal2 v15.4s, v28.8h, v29.8h\n" + "smlal v20.4s, v21.4h, v29.4h\n" + "smlal2 v5.4s, v21.8h, v29.8h\n" + "smlal v24.4s, v16.4h, v29.4h\n" + "smlal2 v22.4s, v16.8h, v29.8h\n" "tbz x1, #2, 77f\n" - "ld1 { v31.s }[0], [x9], #0x4\n" + "ld1 { v30.s }[0], [x20], #0x4\n" "tbz x1, #1, 76f\n" - "ld1 { v31.h }[2], [x9], #0x2\n" + "ld1 { v30.h }[2], [x20], #0x2\n" "tbz x1, #0, 79f\n" - "ld1 { v31.b }[6], [x9]\n" + "ld1 { v30.b }[6], [x20]\n" "b 79f\n" "76:" // Oddments: Load (4, 2): Bit 2: Bit 1: Unset "tbz x1, #0, 79f\n" - "ld1 { v31.b }[4], [x9]\n" + "ld1 { v30.b }[4], [x20]\n" "b 79f\n" "77:" // Oddments: Load (4, 2): Bit 2: Unset "tbz x1, #1, 78f\n" - "ld1 { v31.h }[0], [x9], #0x2\n" + "ld1 { v30.h }[0], [x20], #0x2\n" "tbz x1, #0, 79f\n" - "ld1 { v31.b }[2], [x9]\n" + "ld1 { v30.b }[2], [x20]\n" "b 79f\n" "78:" // Oddments: Load (4, 2): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 79f\n" - "ld1 { v31.b }[0], [x9]\n" + "ld1 { v30.b }[0], [x20]\n" "79:" // Oddments: Load (4, 2): Bit 2: End - "ldr d2, [x3, #0x88]\n" - "usubl v31.8h, v31.8b, v9.8b\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "ldr x28, [x4, #0xd8]\n" - "smlal v17.4s, v31.4h, v1.4h\n" - "smlal2 v21.4s, v31.8h, v1.8h\n" - "add x28, x28, x0\n" - "smlal v13.4s, v26.4h, v2.4h\n" - "smlal2 v19.4s, v26.8h, v2.8h\n" - "smlal v20.4s, v25.4h, v2.4h\n" - "smlal2 v10.4s, v25.8h, v2.8h\n" - "smlal v8.4s, v31.4h, v2.4h\n" - "smlal2 v7.4s, v31.8h, v2.8h\n" + "ldr d12, [x6, #0x88]\n" + "usubl v30.8h, v30.8b, v18.8b\n" + "ssubl v12.8h, v12.8b, v13.8b\n" + "ldr x20, [x5, #0xd8]\n" + "smlal v23.4s, v30.4h, v29.4h\n" + "smlal2 v19.4s, v30.8h, v29.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v21.4h, v12.4h\n" + "smlal2 v15.4s, v21.8h, v12.8h\n" + "smlal v20.4s, v9.4h, v12.4h\n" + "smlal2 v5.4s, v9.8h, v12.8h\n" + "smlal v24.4s, v30.4h, v12.4h\n" + "smlal2 v22.4s, v30.8h, v12.8h\n" "tbz x1, #2, 81f\n" - "ld1 { v30.s }[0], [x28], #0x4\n" + "ld1 { v29.s }[0], [x20], #0x4\n" "tbz x1, #1, 80f\n" - "ld1 { v30.h }[2], [x28], #0x2\n" + "ld1 { v29.h }[2], [x20], #0x2\n" "tbz x1, #0, 83f\n" - "ld1 { v30.b }[6], [x28]\n" + "ld1 { v29.b }[6], [x20]\n" "b 83f\n" "80:" // Oddments: Load (4, 3): Bit 2: Bit 1: Unset "tbz x1, #0, 83f\n" - "ld1 { v30.b }[4], [x28]\n" + "ld1 { v29.b }[4], [x20]\n" "b 83f\n" "81:" // Oddments: Load (4, 3): Bit 2: Unset "tbz x1, #1, 82f\n" - "ld1 { v30.h }[0], [x28], #0x2\n" + "ld1 { v29.h }[0], [x20], #0x2\n" "tbz x1, #0, 83f\n" - "ld1 { v30.b }[2], [x28]\n" + "ld1 { v29.b }[2], [x20]\n" "b 83f\n" "82:" // Oddments: Load (4, 3): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 83f\n" - "ld1 { v30.b }[0], [x28]\n" + "ld1 { v29.b }[0], [x20]\n" "83:" // Oddments: Load (4, 3): Bit 2: End - "ldr d3, [x3, #0x90]\n" - "usubl v30.8h, v30.8b, v9.8b\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "ldr x27, [x4, #0xe0]\n" - "smlal v17.4s, v30.4h, v2.4h\n" - "smlal2 v21.4s, v30.8h, v2.8h\n" - "add x27, x27, x0\n" - "smlal v13.4s, v25.4h, v3.4h\n" - "smlal2 v19.4s, v25.8h, v3.8h\n" - "smlal v20.4s, v24.4h, v3.4h\n" - "smlal2 v10.4s, v24.8h, v3.8h\n" - "smlal v8.4s, v30.4h, v3.4h\n" - "smlal2 v7.4s, v30.8h, v3.8h\n" + "ldr d21, [x6, #0x90]\n" + "usubl v29.8h, v29.8b, v18.8b\n" + "ssubl v21.8h, v21.8b, v13.8b\n" + "ldr x20, [x5, #0xe0]\n" + "smlal v23.4s, v29.4h, v12.4h\n" + "smlal2 v19.4s, v29.8h, v12.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v9.4h, v21.4h\n" + "smlal2 v15.4s, v9.8h, v21.8h\n" + "smlal v20.4s, v3.4h, v21.4h\n" + "smlal2 v5.4s, v3.8h, v21.8h\n" + "smlal v24.4s, v29.4h, v21.4h\n" + "smlal2 v22.4s, v29.8h, v21.8h\n" "tbz x1, #2, 85f\n" - "ld1 { v28.s }[0], [x27], #0x4\n" + "ld1 { v25.s }[0], [x20], #0x4\n" "tbz x1, #1, 84f\n" - "ld1 { v28.h }[2], [x27], #0x2\n" + "ld1 { v25.h }[2], [x20], #0x2\n" "tbz x1, #0, 87f\n" - "ld1 { v28.b }[6], [x27]\n" + "ld1 { v25.b }[6], [x20]\n" "b 87f\n" "84:" // Oddments: Load (4, 4): Bit 2: Bit 1: Unset "tbz x1, #0, 87f\n" - "ld1 { v28.b }[4], [x27]\n" + "ld1 { v25.b }[4], [x20]\n" "b 87f\n" "85:" // Oddments: Load (4, 4): Bit 2: Unset "tbz x1, #1, 86f\n" - "ld1 { v28.h }[0], [x27], #0x2\n" + "ld1 { v25.h }[0], [x20], #0x2\n" "tbz x1, #0, 87f\n" - "ld1 { v28.b }[2], [x27]\n" + "ld1 { v25.b }[2], [x20]\n" "b 87f\n" "86:" // Oddments: Load (4, 4): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 87f\n" - "ld1 { v28.b }[0], [x27]\n" + "ld1 { v25.b }[0], [x20]\n" "87:" // Oddments: Load (4, 4): Bit 2: End - "ldr d4, [x3, #0x98]\n" - "usubl v28.8h, v28.8b, v9.8b\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "ldr x26, [x4, #0xe8]\n" - "smlal v17.4s, v28.4h, v3.4h\n" - "smlal2 v21.4s, v28.8h, v3.8h\n" - "add x26, x26, x0\n" - "smlal v13.4s, v24.4h, v4.4h\n" - "smlal2 v19.4s, v24.8h, v4.8h\n" - "smlal v20.4s, v22.4h, v4.4h\n" - "smlal2 v10.4s, v22.8h, v4.8h\n" - "smlal v8.4s, v28.4h, v4.4h\n" - "smlal2 v7.4s, v28.8h, v4.8h\n" + "ldr d8, [x6, #0x98]\n" + "usubl v25.8h, v25.8b, v18.8b\n" + "ssubl v8.8h, v8.8b, v13.8b\n" + "ldr x20, [x5, #0xe8]\n" + "smlal v23.4s, v25.4h, v21.4h\n" + "smlal2 v19.4s, v25.8h, v21.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v3.4h, v8.4h\n" + "smlal2 v15.4s, v3.8h, v8.8h\n" + "smlal v20.4s, v14.4h, v8.4h\n" + "smlal2 v5.4s, v14.8h, v8.8h\n" + "smlal v24.4s, v25.4h, v8.4h\n" + "smlal2 v22.4s, v25.8h, v8.8h\n" "tbz x1, #2, 89f\n" - "ld1 { v26.s }[0], [x26], #0x4\n" + "ld1 { v21.s }[0], [x20], #0x4\n" "tbz x1, #1, 88f\n" - "ld1 { v26.h }[2], [x26], #0x2\n" + "ld1 { v21.h }[2], [x20], #0x2\n" "tbz x1, #0, 91f\n" - "ld1 { v26.b }[6], [x26]\n" + "ld1 { v21.b }[6], [x20]\n" "b 91f\n" "88:" // Oddments: Load (4, 5): Bit 2: Bit 1: Unset "tbz x1, #0, 91f\n" - "ld1 { v26.b }[4], [x26]\n" + "ld1 { v21.b }[4], [x20]\n" "b 91f\n" "89:" // Oddments: Load (4, 5): Bit 2: Unset "tbz x1, #1, 90f\n" - "ld1 { v26.h }[0], [x26], #0x2\n" + "ld1 { v21.h }[0], [x20], #0x2\n" "tbz x1, #0, 91f\n" - "ld1 { v26.b }[2], [x26]\n" + "ld1 { v21.b }[2], [x20]\n" "b 91f\n" "90:" // Oddments: Load (4, 5): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 91f\n" - "ld1 { v26.b }[0], [x26]\n" + "ld1 { v21.b }[0], [x20]\n" "91:" // Oddments: Load (4, 5): Bit 2: End - "ldr d0, [x3, #0xa0]\n" - "usubl v26.8h, v26.8b, v9.8b\n" - "ssubl v0.8h, v0.8b, v15.8b\n" - "ldr x25, [x4, #0xf0]\n" - "smlal v17.4s, v26.4h, v4.4h\n" - "smlal2 v21.4s, v26.8h, v4.8h\n" - "add x25, x25, x0\n" - "smlal v13.4s, v27.4h, v0.4h\n" - "smlal2 v19.4s, v27.8h, v0.8h\n" - "smlal v20.4s, v23.4h, v0.4h\n" - "smlal2 v10.4s, v23.8h, v0.8h\n" + "ldr d9, [x6, #0xa0]\n" + "usubl v21.8h, v21.8b, v18.8b\n" + "ssubl v9.8h, v9.8b, v13.8b\n" + "ldr x20, [x5, #0xf0]\n" + "smlal v23.4s, v21.4h, v8.4h\n" + "smlal2 v19.4s, v21.8h, v8.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v1.4h, v9.4h\n" + "smlal2 v15.4s, v1.8h, v9.8h\n" + "smlal v20.4s, v16.4h, v9.4h\n" + "smlal2 v5.4s, v16.8h, v9.8h\n" "tbz x1, #2, 93f\n" - "ld1 { v25.s }[0], [x25], #0x4\n" + "ld1 { v12.s }[0], [x20], #0x4\n" "tbz x1, #1, 92f\n" - "ld1 { v25.h }[2], [x25], #0x2\n" + "ld1 { v12.h }[2], [x20], #0x2\n" "tbz x1, #0, 95f\n" - "ld1 { v25.b }[6], [x25]\n" + "ld1 { v12.b }[6], [x20]\n" "b 95f\n" "92:" // Oddments: Load (5, 0): Bit 2: Bit 1: Unset "tbz x1, #0, 95f\n" - "ld1 { v25.b }[4], [x25]\n" + "ld1 { v12.b }[4], [x20]\n" "b 95f\n" "93:" // Oddments: Load (5, 0): Bit 2: Unset "tbz x1, #1, 94f\n" - "ld1 { v25.h }[0], [x25], #0x2\n" + "ld1 { v12.h }[0], [x20], #0x2\n" "tbz x1, #0, 95f\n" - "ld1 { v25.b }[2], [x25]\n" + "ld1 { v12.b }[2], [x20]\n" "b 95f\n" "94:" // Oddments: Load (5, 0): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 95f\n" - "ld1 { v25.b }[0], [x25]\n" + "ld1 { v12.b }[0], [x20]\n" "95:" // Oddments: Load (5, 0): Bit 2: End - "usubl v25.8h, v25.8b, v9.8b\n" - "ldr x24, [x4, #0xf8]\n" - "smlal v8.4s, v25.4h, v0.4h\n" - "smlal2 v7.4s, v25.8h, v0.8h\n" - "add x24, x24, x0\n" + "usubl v12.8h, v12.8b, v18.8b\n" + "ldr x20, [x5, #0xf8]\n" + "smlal v24.4s, v12.4h, v9.4h\n" + "smlal2 v22.4s, v12.8h, v9.8h\n" + "add x20, x20, x3\n" "tbz x1, #2, 97f\n" - "ld1 { v24.s }[0], [x24], #0x4\n" + "ld1 { v10.s }[0], [x20], #0x4\n" "tbz x1, #1, 96f\n" - "ld1 { v24.h }[2], [x24], #0x2\n" + "ld1 { v10.h }[2], [x20], #0x2\n" "tbz x1, #0, 99f\n" - "ld1 { v24.b }[6], [x24]\n" + "ld1 { v10.b }[6], [x20]\n" "b 99f\n" "96:" // Oddments: Load (5, 1): Bit 2: Bit 1: Unset "tbz x1, #0, 99f\n" - "ld1 { v24.b }[4], [x24]\n" + "ld1 { v10.b }[4], [x20]\n" "b 99f\n" "97:" // Oddments: Load (5, 1): Bit 2: Unset "tbz x1, #1, 98f\n" - "ld1 { v24.h }[0], [x24], #0x2\n" + "ld1 { v10.h }[0], [x20], #0x2\n" "tbz x1, #0, 99f\n" - "ld1 { v24.b }[2], [x24]\n" + "ld1 { v10.b }[2], [x20]\n" "b 99f\n" "98:" // Oddments: Load (5, 1): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 99f\n" - "ld1 { v24.b }[0], [x24]\n" + "ld1 { v10.b }[0], [x20]\n" "99:" // Oddments: Load (5, 1): Bit 2: End - "ldr d1, [x3, #0xa8]\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "ssubl v1.8h, v1.8b, v15.8b\n" - "ldr x23, [x4, #0x100]\n" - "smlal v17.4s, v24.4h, v0.4h\n" - "smlal2 v21.4s, v24.8h, v0.8h\n" - "add x23, x23, x0\n" - "smlal v13.4s, v23.4h, v1.4h\n" - "smlal2 v19.4s, v23.8h, v1.8h\n" - "smlal v20.4s, v31.4h, v1.4h\n" - "smlal2 v10.4s, v31.8h, v1.8h\n" - "smlal v8.4s, v24.4h, v1.4h\n" - "smlal2 v7.4s, v24.8h, v1.8h\n" + "ldr d12, [x6, #0xa8]\n" + "usubl v10.8h, v10.8b, v18.8b\n" + "ssubl v12.8h, v12.8b, v13.8b\n" + "ldr x20, [x5, #0x100]\n" + "smlal v23.4s, v10.4h, v9.4h\n" + "smlal2 v19.4s, v10.8h, v9.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v16.4h, v12.4h\n" + "smlal2 v15.4s, v16.8h, v12.8h\n" + "smlal v20.4s, v30.4h, v12.4h\n" + "smlal2 v5.4s, v30.8h, v12.8h\n" + "smlal v24.4s, v10.4h, v12.4h\n" + "smlal2 v22.4s, v10.8h, v12.8h\n" "tbz x1, #2, 101f\n" - "ld1 { v27.s }[0], [x23], #0x4\n" + "ld1 { v9.s }[0], [x20], #0x4\n" "tbz x1, #1, 100f\n" - "ld1 { v27.h }[2], [x23], #0x2\n" + "ld1 { v9.h }[2], [x20], #0x2\n" "tbz x1, #0, 103f\n" - "ld1 { v27.b }[6], [x23]\n" + "ld1 { v9.b }[6], [x20]\n" "b 103f\n" "100:" // Oddments: Load (5, 2): Bit 2: Bit 1: Unset "tbz x1, #0, 103f\n" - "ld1 { v27.b }[4], [x23]\n" + "ld1 { v9.b }[4], [x20]\n" "b 103f\n" "101:" // Oddments: Load (5, 2): Bit 2: Unset "tbz x1, #1, 102f\n" - "ld1 { v27.h }[0], [x23], #0x2\n" + "ld1 { v9.h }[0], [x20], #0x2\n" "tbz x1, #0, 103f\n" - "ld1 { v27.b }[2], [x23]\n" + "ld1 { v9.b }[2], [x20]\n" "b 103f\n" "102:" // Oddments: Load (5, 2): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 103f\n" - "ld1 { v27.b }[0], [x23]\n" + "ld1 { v9.b }[0], [x20]\n" "103:" // Oddments: Load (5, 2): Bit 2: End - "ldr d2, [x3, #0xb0]\n" - "usubl v27.8h, v27.8b, v9.8b\n" - "ssubl v2.8h, v2.8b, v15.8b\n" - "ldr x15, [x4, #0x108]\n" - "smlal v17.4s, v27.4h, v1.4h\n" - "smlal2 v21.4s, v27.8h, v1.8h\n" - "add x15, x15, x0\n" - "smlal v13.4s, v31.4h, v2.4h\n" - "smlal2 v19.4s, v31.8h, v2.8h\n" - "smlal v20.4s, v30.4h, v2.4h\n" - "smlal2 v10.4s, v30.8h, v2.8h\n" - "smlal v8.4s, v27.4h, v2.4h\n" - "smlal2 v7.4s, v27.8h, v2.8h\n" + "ldr d28, [x6, #0xb0]\n" + "usubl v9.8h, v9.8b, v18.8b\n" + "ssubl v28.8h, v28.8b, v13.8b\n" + "ldr x20, [x5, #0x108]\n" + "smlal v23.4s, v9.4h, v12.4h\n" + "smlal2 v19.4s, v9.8h, v12.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v30.4h, v28.4h\n" + "smlal2 v15.4s, v30.8h, v28.8h\n" + "smlal v20.4s, v29.4h, v28.4h\n" + "smlal2 v5.4s, v29.8h, v28.8h\n" + "smlal v24.4s, v9.4h, v28.4h\n" + "smlal2 v22.4s, v9.8h, v28.8h\n" "tbz x1, #2, 105f\n" - "ld1 { v25.s }[0], [x15], #0x4\n" + "ld1 { v2.s }[0], [x20], #0x4\n" "tbz x1, #1, 104f\n" - "ld1 { v25.h }[2], [x15], #0x2\n" + "ld1 { v2.h }[2], [x20], #0x2\n" "tbz x1, #0, 107f\n" - "ld1 { v25.b }[6], [x15]\n" + "ld1 { v2.b }[6], [x20]\n" "b 107f\n" "104:" // Oddments: Load (5, 3): Bit 2: Bit 1: Unset "tbz x1, #0, 107f\n" - "ld1 { v25.b }[4], [x15]\n" + "ld1 { v2.b }[4], [x20]\n" "b 107f\n" "105:" // Oddments: Load (5, 3): Bit 2: Unset "tbz x1, #1, 106f\n" - "ld1 { v25.h }[0], [x15], #0x2\n" + "ld1 { v2.h }[0], [x20], #0x2\n" "tbz x1, #0, 107f\n" - "ld1 { v25.b }[2], [x15]\n" + "ld1 { v2.b }[2], [x20]\n" "b 107f\n" "106:" // Oddments: Load (5, 3): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 107f\n" - "ld1 { v25.b }[0], [x15]\n" + "ld1 { v2.b }[0], [x20]\n" "107:" // Oddments: Load (5, 3): Bit 2: End - "ldr d3, [x3, #0xb8]\n" - "usubl v25.8h, v25.8b, v9.8b\n" - "ssubl v3.8h, v3.8b, v15.8b\n" - "ldr x21, [x4, #0x110]\n" - "smlal v17.4s, v25.4h, v2.4h\n" - "smlal2 v21.4s, v25.8h, v2.8h\n" - "add x21, x21, x0\n" - "smlal v13.4s, v30.4h, v3.4h\n" - "smlal2 v19.4s, v30.8h, v3.8h\n" - "smlal v20.4s, v28.4h, v3.4h\n" - "smlal2 v10.4s, v28.8h, v3.8h\n" - "smlal v8.4s, v25.4h, v3.4h\n" - "smlal2 v7.4s, v25.8h, v3.8h\n" + "ldr d30, [x6, #0xb8]\n" + "usubl v2.8h, v2.8b, v18.8b\n" + "ssubl v30.8h, v30.8b, v13.8b\n" + "ldr x20, [x5, #0x110]\n" + "smlal v23.4s, v2.4h, v28.4h\n" + "smlal2 v19.4s, v2.8h, v28.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v29.4h, v30.4h\n" + "smlal2 v15.4s, v29.8h, v30.8h\n" + "smlal v20.4s, v25.4h, v30.4h\n" + "smlal2 v5.4s, v25.8h, v30.8h\n" + "smlal v24.4s, v2.4h, v30.4h\n" + "smlal2 v22.4s, v2.8h, v30.8h\n" "tbz x1, #2, 109f\n" - "ld1 { v24.s }[0], [x21], #0x4\n" + "ld1 { v27.s }[0], [x20], #0x4\n" "tbz x1, #1, 108f\n" - "ld1 { v24.h }[2], [x21], #0x2\n" + "ld1 { v27.h }[2], [x20], #0x2\n" "tbz x1, #0, 111f\n" - "ld1 { v24.b }[6], [x21]\n" + "ld1 { v27.b }[6], [x20]\n" "b 111f\n" "108:" // Oddments: Load (5, 4): Bit 2: Bit 1: Unset "tbz x1, #0, 111f\n" - "ld1 { v24.b }[4], [x21]\n" + "ld1 { v27.b }[4], [x20]\n" "b 111f\n" "109:" // Oddments: Load (5, 4): Bit 2: Unset "tbz x1, #1, 110f\n" - "ld1 { v24.h }[0], [x21], #0x2\n" + "ld1 { v27.h }[0], [x20], #0x2\n" "tbz x1, #0, 111f\n" - "ld1 { v24.b }[2], [x21]\n" + "ld1 { v27.b }[2], [x20]\n" "b 111f\n" "110:" // Oddments: Load (5, 4): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 111f\n" - "ld1 { v24.b }[0], [x21]\n" + "ld1 { v27.b }[0], [x20]\n" "111:" // Oddments: Load (5, 4): Bit 2: End - "ldr d4, [x3, #0xc0]\n" - "usubl v24.8h, v24.8b, v9.8b\n" - "ssubl v4.8h, v4.8b, v15.8b\n" - "ldr x20, [x4, #0x118]\n" - "smlal v17.4s, v24.4h, v3.4h\n" - "smlal2 v21.4s, v24.8h, v3.8h\n" - "add x20, x20, x0\n" - "smlal v13.4s, v28.4h, v4.4h\n" - "smlal2 v19.4s, v28.8h, v4.8h\n" - "smlal v20.4s, v26.4h, v4.4h\n" - "smlal2 v10.4s, v26.8h, v4.8h\n" - "smlal v8.4s, v24.4h, v4.4h\n" - "smlal2 v7.4s, v24.8h, v4.8h\n" + "ldr d8, [x6, #0xc0]\n" + "usubl v27.8h, v27.8b, v18.8b\n" + "ssubl v8.8h, v8.8b, v13.8b\n" + "ldr x20, [x5, #0x118]\n" + "smlal v23.4s, v27.4h, v30.4h\n" + "smlal2 v19.4s, v27.8h, v30.8h\n" + "add x20, x20, x3\n" + "smlal v7.4s, v25.4h, v8.4h\n" + "smlal2 v15.4s, v25.8h, v8.8h\n" + "smlal v20.4s, v21.4h, v8.4h\n" + "smlal2 v5.4s, v21.8h, v8.8h\n" + "smlal v24.4s, v27.4h, v8.4h\n" + "smlal2 v22.4s, v27.8h, v8.8h\n" "tbz x1, #2, 113f\n" - "ld1 { v27.s }[0], [x20], #0x4\n" + "ld1 { v9.s }[0], [x20], #0x4\n" "tbz x1, #1, 112f\n" - "ld1 { v27.h }[2], [x20], #0x2\n" + "ld1 { v9.h }[2], [x20], #0x2\n" "tbz x1, #0, 115f\n" - "ld1 { v27.b }[6], [x20]\n" + "ld1 { v9.b }[6], [x20]\n" "b 115f\n" "112:" // Oddments: Load (5, 5): Bit 2: Bit 1: Unset "tbz x1, #0, 115f\n" - "ld1 { v27.b }[4], [x20]\n" + "ld1 { v9.b }[4], [x20]\n" "b 115f\n" "113:" // Oddments: Load (5, 5): Bit 2: Unset "tbz x1, #1, 114f\n" - "ld1 { v27.h }[0], [x20], #0x2\n" + "ld1 { v9.h }[0], [x20], #0x2\n" "tbz x1, #0, 115f\n" - "ld1 { v27.b }[2], [x20]\n" + "ld1 { v9.b }[2], [x20]\n" "b 115f\n" "114:" // Oddments: Load (5, 5): Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 115f\n" - "ld1 { v27.b }[0], [x20]\n" + "ld1 { v9.b }[0], [x20]\n" "115:" // Oddments: Load (5, 5): Bit 2: End - "usubl v27.8h, v27.8b, v9.8b\n" - "smlal v17.4s, v27.4h, v4.4h\n" - "smlal2 v21.4s, v27.8h, v4.8h\n" + "usubl v9.8h, v9.8b, v18.8b\n" + "smlal v23.4s, v9.4h, v8.4h\n" + "smlal2 v19.4s, v9.8h, v8.8h\n" "tbz x1, #2, 117f\n" - "ld1 { v18.4s }, [x5], #0x10\n" - "ld1 { v6.4s }, [x8], #0x10\n" + "ld1 { v30.4s }, [x7], #0x10\n" + "ld1 { v12.4s }, [x8], #0x10\n" "tbz x1, #1, 116f\n" - "ld1 { v5.d }[0], [x5], #0x8\n" - "ld1 { v22.d }[0], [x8], #0x8\n" + "ld1 { v14.d }[0], [x7], #0x8\n" + "ld1 { v27.d }[0], [x8], #0x8\n" "tbz x1, #0, 119f\n" - "ld1 { v5.s }[2], [x5]\n" - "ld1 { v22.s }[2], [x8]\n" + "ld1 { v14.s }[2], [x7]\n" + "ld1 { v27.s }[2], [x8]\n" "b 119f\n" "116:" // Oddments: Load requant params: Bit 2: Bit 1: Unset "tbz x1, #0, 119f\n" - "ld1 { v5.s }[0], [x5]\n" - "ld1 { v22.s }[0], [x8]\n" + "ld1 { v14.s }[0], [x7]\n" + "ld1 { v27.s }[0], [x8]\n" "b 119f\n" "117:" // Oddments: Load requant params: Bit 2: Unset "tbz x1, #1, 118f\n" - "ld1 { v18.d }[0], [x5], #0x8\n" - "ld1 { v6.d }[0], [x8], #0x8\n" + "ld1 { v30.d }[0], [x7], #0x8\n" + "ld1 { v12.d }[0], [x8], #0x8\n" "tbz x1, #0, 119f\n" - "ld1 { v18.s }[2], [x5]\n" - "ld1 { v6.s }[2], [x8]\n" + "ld1 { v30.s }[2], [x7]\n" + "ld1 { v12.s }[2], [x8]\n" "b 119f\n" "118:" // Oddments: Load requant params: Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 119f\n" - "ld1 { v18.s }[0], [x5]\n" - "ld1 { v6.s }[0], [x8]\n" + "ld1 { v30.s }[0], [x7]\n" + "ld1 { v12.s }[0], [x8]\n" "119:" // Oddments: Load requant params: Bit 2: End - "sqrdmulh v13.4s, v13.4s, v18.4s\n" - "and v30.16b, v13.16b, v6.16b\n" - "add x17, x17, x10\n" - "add x6, x6, x10\n" - "sqrdmulh v19.4s, v19.4s, v5.4s\n" - "sshr v30.4s, v30.4s, #0x1f\n" - "add x7, x7, x10\n" - "add x16, x16, x10\n" - "and v16.16b, v19.16b, v22.16b\n" - "sqrdmulh v20.4s, v20.4s, v18.4s\n" - "sqrdmulh v8.4s, v8.4s, v18.4s\n" - "sqrdmulh v17.4s, v17.4s, v18.4s\n" - "sqadd v13.4s, v13.4s, v30.4s\n" + "sqrdmulh v7.4s, v7.4s, v30.4s\n" + "and v16.16b, v7.16b, v12.16b\n" + "add x17, x17, x4\n" + "add x16, x16, x4\n" + "sqrdmulh v15.4s, v15.4s, v14.4s\n" "sshr v16.4s, v16.4s, #0x1f\n" - "and v0.16b, v20.16b, v6.16b\n" - "sqrdmulh v10.4s, v10.4s, v5.4s\n" - "and v18.16b, v8.16b, v6.16b\n" - "sqrdmulh v7.4s, v7.4s, v5.4s\n" - "and v30.16b, v17.16b, v6.16b\n" - "sqrdmulh v21.4s, v21.4s, v5.4s\n" - "sqadd v19.4s, v19.4s, v16.4s\n" - "sshr v0.4s, v0.4s, #0x1f\n" - "and v26.16b, v10.16b, v22.16b\n" + "add x15, x15, x4\n" + "add x14, x14, x4\n" + "and v2.16b, v15.16b, v27.16b\n" + "sqrdmulh v20.4s, v20.4s, v30.4s\n" + "sqrdmulh v24.4s, v24.4s, v30.4s\n" + "sqrdmulh v23.4s, v23.4s, v30.4s\n" + "sqadd v7.4s, v7.4s, v16.4s\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "and v21.16b, v20.16b, v12.16b\n" + "sqrdmulh v5.4s, v5.4s, v14.4s\n" + "and v18.16b, v24.16b, v12.16b\n" + "sqrdmulh v22.4s, v22.4s, v14.4s\n" + "and v31.16b, v23.16b, v12.16b\n" + "sqrdmulh v19.4s, v19.4s, v14.4s\n" + "sqadd v15.4s, v15.4s, v2.4s\n" + "sshr v21.4s, v21.4s, #0x1f\n" + "and v9.16b, v5.16b, v27.16b\n" "sshr v18.4s, v18.4s, #0x1f\n" - "and v23.16b, v7.16b, v22.16b\n" - "sshr v30.4s, v30.4s, #0x1f\n" - "and v16.16b, v21.16b, v22.16b\n" - "sqadd v20.4s, v20.4s, v0.4s\n" - "sshr v26.4s, v26.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v18.4s\n" - "sshr v23.4s, v23.4s, #0x1f\n" - "sqadd v17.4s, v17.4s, v30.4s\n" - "sshr v16.4s, v16.4s, #0x1f\n" - "srshl v13.4s, v13.4s, v6.4s\n" - "srshl v20.4s, v20.4s, v6.4s\n" - "sqadd v10.4s, v10.4s, v26.4s\n" - "srshl v8.4s, v8.4s, v6.4s\n" - "sqadd v7.4s, v7.4s, v23.4s\n" - "srshl v17.4s, v17.4s, v6.4s\n" - "sqadd v21.4s, v21.4s, v16.4s\n" - "srshl v19.4s, v19.4s, v22.4s\n" - "sqxtn v13.4h, v13.4s\n" - "srshl v10.4s, v10.4s, v22.4s\n" + "and v4.16b, v22.16b, v27.16b\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "and v28.16b, v19.16b, v27.16b\n" + "sqadd v20.4s, v20.4s, v21.4s\n" + "sshr v9.4s, v9.4s, #0x1f\n" + "sqadd v24.4s, v24.4s, v18.4s\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sqadd v23.4s, v23.4s, v31.4s\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "srshl v7.4s, v7.4s, v12.4s\n" + "srshl v20.4s, v20.4s, v12.4s\n" + "sqadd v5.4s, v5.4s, v9.4s\n" + "srshl v24.4s, v24.4s, v12.4s\n" + "sqadd v22.4s, v22.4s, v4.4s\n" + "srshl v23.4s, v23.4s, v12.4s\n" + "sqadd v19.4s, v19.4s, v28.4s\n" + "srshl v15.4s, v15.4s, v27.4s\n" + "sqxtn v7.4h, v7.4s\n" + "srshl v5.4s, v5.4s, v27.4s\n" "sqxtn v20.4h, v20.4s\n" - "srshl v7.4s, v7.4s, v22.4s\n" - "sqxtn v8.4h, v8.4s\n" - "srshl v21.4s, v21.4s, v22.4s\n" - "sqxtn v17.4h, v17.4s\n" - "sqxtn2 v13.8h, v19.4s\n" - "sqxtn2 v20.8h, v10.4s\n" - "sqxtn2 v8.8h, v7.4s\n" - "sqxtn2 v17.8h, v21.4s\n" - "sqadd v13.8h, v13.8h, v14.8h\n" - "sqadd v20.8h, v20.8h, v14.8h\n" - "sqadd v8.8h, v8.8h, v14.8h\n" - "sqadd v17.8h, v17.8h, v14.8h\n" - "smax v13.8h, v13.8h, v12.8h\n" - "smax v20.8h, v20.8h, v12.8h\n" - "smax v8.8h, v8.8h, v12.8h\n" - "smax v17.8h, v17.8h, v12.8h\n" - "smin v13.8h, v13.8h, v11.8h\n" - "smin v20.8h, v20.8h, v11.8h\n" - "smin v8.8h, v8.8h, v11.8h\n" - "smin v17.8h, v17.8h, v11.8h\n" - "uzp1 v13.16b, v13.16b, v13.16b\n" + "srshl v22.4s, v22.4s, v27.4s\n" + "sqxtn v24.4h, v24.4s\n" + "srshl v19.4s, v19.4s, v27.4s\n" + "sqxtn v23.4h, v23.4s\n" + "sqxtn2 v7.8h, v15.4s\n" + "sqxtn2 v20.8h, v5.4s\n" + "sqxtn2 v24.8h, v22.4s\n" + "sqxtn2 v23.8h, v19.4s\n" + "sqadd v7.8h, v7.8h, v26.8h\n" + "sqadd v20.8h, v20.8h, v26.8h\n" + "sqadd v24.8h, v24.8h, v26.8h\n" + "sqadd v23.8h, v23.8h, v26.8h\n" + "smax v7.8h, v7.8h, v11.8h\n" + "smax v20.8h, v20.8h, v11.8h\n" + "smax v24.8h, v24.8h, v11.8h\n" + "smax v23.8h, v23.8h, v11.8h\n" + "smin v7.8h, v7.8h, v0.8h\n" + "smin v20.8h, v20.8h, v0.8h\n" + "smin v24.8h, v24.8h, v0.8h\n" + "smin v23.8h, v23.8h, v0.8h\n" + "uzp1 v7.16b, v7.16b, v7.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "uzp1 v8.16b, v8.16b, v8.16b\n" - "uzp1 v17.16b, v17.16b, v17.16b\n" + "uzp1 v24.16b, v24.16b, v24.16b\n" + "uzp1 v23.16b, v23.16b, v23.16b\n" "tbz x1, #2, 121f\n" - "st1 { v13.s }[0], [x17], #0x4\n" - "st1 { v20.s }[0], [x6], #0x4\n" - "st1 { v8.s }[0], [x7], #0x4\n" - "st1 { v17.s }[0], [x16], #0x4\n" + "st1 { v7.s }[0], [x17], #0x4\n" + "st1 { v20.s }[0], [x16], #0x4\n" + "st1 { v24.s }[0], [x15], #0x4\n" + "st1 { v23.s }[0], [x14], #0x4\n" "tbz x1, #1, 120f\n" - "st1 { v13.h }[2], [x17], #0x2\n" - "st1 { v20.h }[2], [x6], #0x2\n" - "st1 { v8.h }[2], [x7], #0x2\n" - "st1 { v17.h }[2], [x16], #0x2\n" + "st1 { v7.h }[2], [x17], #0x2\n" + "st1 { v20.h }[2], [x16], #0x2\n" + "st1 { v24.h }[2], [x15], #0x2\n" + "st1 { v23.h }[2], [x14], #0x2\n" "tbz x1, #0, 123f\n" - "st1 { v13.b }[6], [x17], #0x1\n" - "st1 { v20.b }[6], [x6], #0x1\n" - "st1 { v8.b }[6], [x7], #0x1\n" - "st1 { v17.b }[6], [x16], #0x1\n" + "st1 { v7.b }[6], [x17], #0x1\n" + "st1 { v20.b }[6], [x16], #0x1\n" + "st1 { v24.b }[6], [x15], #0x1\n" + "st1 { v23.b }[6], [x14], #0x1\n" "b 123f\n" "120:" // Oddments: Bit 2: Bit 1: Unset "tbz x1, #0, 123f\n" - "st1 { v13.b }[4], [x17], #0x1\n" - "st1 { v20.b }[4], [x6], #0x1\n" - "st1 { v8.b }[4], [x7], #0x1\n" - "st1 { v17.b }[4], [x16], #0x1\n" + "st1 { v7.b }[4], [x17], #0x1\n" + "st1 { v20.b }[4], [x16], #0x1\n" + "st1 { v24.b }[4], [x15], #0x1\n" + "st1 { v23.b }[4], [x14], #0x1\n" "b 123f\n" "121:" // Oddments: Bit 2: Unset "tbz x1, #1, 122f\n" - "st1 { v13.h }[0], [x17], #0x2\n" - "st1 { v20.h }[0], [x6], #0x2\n" - "st1 { v8.h }[0], [x7], #0x2\n" - "st1 { v17.h }[0], [x16], #0x2\n" + "st1 { v7.h }[0], [x17], #0x2\n" + "st1 { v20.h }[0], [x16], #0x2\n" + "st1 { v24.h }[0], [x15], #0x2\n" + "st1 { v23.h }[0], [x14], #0x2\n" "tbz x1, #0, 123f\n" - "st1 { v13.b }[2], [x17], #0x1\n" - "st1 { v20.b }[2], [x6], #0x1\n" - "st1 { v8.b }[2], [x7], #0x1\n" - "st1 { v17.b }[2], [x16], #0x1\n" + "st1 { v7.b }[2], [x17], #0x1\n" + "st1 { v20.b }[2], [x16], #0x1\n" + "st1 { v24.b }[2], [x15], #0x1\n" + "st1 { v23.b }[2], [x14], #0x1\n" "b 123f\n" "122:" // Oddments: Bit 2: Unset: Bit 1: Unset "tbz x1, #0, 123f\n" - "st1 { v13.b }[0], [x17], #0x1\n" - "st1 { v20.b }[0], [x6], #0x1\n" - "st1 { v8.b }[0], [x7], #0x1\n" - "st1 { v17.b }[0], [x16], #0x1\n" + "st1 { v7.b }[0], [x17], #0x1\n" + "st1 { v20.b }[0], [x16], #0x1\n" + "st1 { v24.b }[0], [x15], #0x1\n" + "st1 { v23.b }[0], [x14], #0x1\n" "123:" // Oddments: Bit 2: End "124:" // End : : [offsetof_Params_bias] "I" (offsetof(Params, bias)), [offsetof_Params_inptrs] "I" (offsetof(Params, inptrs)), [offsetof_Params_n_channels] "I" (offsetof(Params, n_channels)), [offsetof_Params_outptrs] "I" (offsetof(Params, outptrs)), [offsetof_Params_requant] "I" (offsetof(Params, requant)), [offsetof_Params_requant_muls] "I" (offsetof(Params, requant_muls)), [offsetof_Params_requant_shifts] "I" (offsetof(Params, requant_shifts)), [offsetof_Params_weights] "I" (offsetof(Params, weights)), [offsetof_Requantize32_a_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [params] "r" (¶ms) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst.hpp index 6bdcca115cc5cd92a21709b07ee295f778f08791..2c677d2f62a4ee5fbf90b83c9f9daedac6fc2b38 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include @@ -47,4 +47,5 @@ class a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst : public GenericDepthfirst } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst/generic.cpp index 1676119bc1828555393fafe6be8a73a587aec852..c2bec4cdab66c845daa393674ec355ac0dcdbb14 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst/generic.cpp @@ -22,12 +22,13 @@ * SOFTWARE. */ -#if defined(__aarch64__) #include "arm_gemm.hpp" #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace depthwise { @@ -41,7 +42,7 @@ void a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst_impl( ) { __asm__ __volatile__( - "lsr x12, %x[n_channels], #0x2\n" + "lsr x9, %x[n_channels], #0x2\n" "add x20, %x[qp], %[offsetof_Requantize32_minval]\n" "ld1r { v8.4s }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_maxval]\n" @@ -59,7 +60,7 @@ void a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst_impl( "add x20, %x[qp], %[offsetof_Requantize32_per_layer_right_shift]\n" "ld1r { v1.4s }, [x20]\n" "mov x11, #0x0\n" - "cbz x12, 6f\n" + "cbz x9, 6f\n" "1:" // Channel loop "movi v23.4s, #0x0\n" "cbz %x[bias], 2f\n" @@ -67,34 +68,34 @@ void a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst_impl( "ldr q23, [%x[bias], x20]\n" "2:" // Channel loop: Load bias: Done "ldr s0, [%x[params]], #0x4\n" - "mov x21, %x[inptrs]\n" - "ldp x10, x9, [x21], #0x10\n" - "subs x20, %x[n_points], #0x1\n" - "ldr s14, [x10, x11]\n" - "ldr s15, [x9, x11]\n" + "mov x25, %x[inptrs]\n" + "ldp x21, x20, [x25], #0x10\n" + "subs x24, %x[n_points], #0x1\n" + "ldr s14, [x21, x11]\n" + "ldr s15, [x20, x11]\n" "mov v24.16b, v23.16b\n" "mov v25.16b, v23.16b\n" - "ldp x28, x27, [x21], #0x10\n" - "ldr s16, [x28, x11]\n" + "ldp x21, x20, [x25], #0x10\n" + "ldr s16, [x21, x11]\n" "mov v26.16b, v23.16b\n" "mov v27.16b, v23.16b\n" - "ldr s17, [x27, x11]\n" - "ldp x26, x25, [x21], #0x10\n" + "ldr s17, [x20, x11]\n" + "ldp x21, x20, [x25], #0x10\n" "mov v28.16b, v23.16b\n" "mov v29.16b, v23.16b\n" - "ldr s18, [x26, x11]\n" - "ldr s19, [x25, x11]\n" + "ldr s18, [x21, x11]\n" + "ldr s19, [x20, x11]\n" "mov v30.16b, v23.16b\n" "mov v31.16b, v23.16b\n" - "ldp x24, x23, [x21], #0x10\n" - "ldr s20, [x24, x11]\n" + "ldp x21, x20, [x25], #0x10\n" + "ldr s20, [x21, x11]\n" "ssubl v0.8h, v0.8b, v5.8b\n" "usubl v14.8h, v14.8b, v6.8b\n" - "ldr s21, [x23, x11]\n" - "ldr x22, [x21], #0x8\n" + "ldr s21, [x20, x11]\n" + "ldr x20, [x25], #0x8\n" "usubl v15.8h, v15.8b, v6.8b\n" "usubl v16.8h, v16.8b, v6.8b\n" - "ldr s22, [x22, x11]\n" + "ldr s22, [x20, x11]\n" "usubl v17.8h, v17.8b, v6.8b\n" "usubl v18.8h, v18.8b, v6.8b\n" "usubl v19.8h, v19.8b, v6.8b\n" @@ -103,35 +104,35 @@ void a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst_impl( "usubl v22.8h, v22.8b, v6.8b\n" "ble 4f\n" "3:" // Channel loop: Planar loop - "ldp x10, x9, [x21], #0x10\n" - "ldp x28, x27, [x21], #0x10\n" + "ldp x23, x22, [x25], #0x10\n" + "ldp x21, x20, [x25], #0x10\n" "smlal v23.4s, v14.4h, v0.4h\n" "smlal v24.4s, v15.4h, v0.4h\n" - "ldr s14, [x10, x11]\n" - "ldr s15, [x9, x11]\n" + "ldr s14, [x23, x11]\n" + "ldr s15, [x22, x11]\n" "smlal v25.4s, v16.4h, v0.4h\n" "smlal v26.4s, v17.4h, v0.4h\n" - "ldr s16, [x28, x11]\n" - "ldr s17, [x27, x11]\n" + "ldr s16, [x21, x11]\n" + "ldr s17, [x20, x11]\n" "smlal v27.4s, v18.4h, v0.4h\n" "smlal v28.4s, v19.4h, v0.4h\n" - "ldp x26, x25, [x21], #0x10\n" - "ldr s18, [x26, x11]\n" + "ldp x21, x20, [x25], #0x10\n" + "ldr s18, [x21, x11]\n" "smlal v29.4s, v20.4h, v0.4h\n" "smlal v30.4s, v21.4h, v0.4h\n" - "ldr s19, [x25, x11]\n" - "ldp x24, x23, [x21], #0x10\n" + "ldr s19, [x20, x11]\n" + "ldp x21, x20, [x25], #0x10\n" "smlal v31.4s, v22.4h, v0.4h\n" - "subs x20, x20, #0x1\n" + "subs x24, x24, #0x1\n" "ldr s0, [%x[params]], #0x4\n" - "ldr s20, [x24, x11]\n" + "ldr s20, [x21, x11]\n" "ssubl v0.8h, v0.8b, v5.8b\n" "usubl v14.8h, v14.8b, v6.8b\n" - "ldr s21, [x23, x11]\n" - "ldr x22, [x21], #0x8\n" + "ldr s21, [x20, x11]\n" + "ldr x20, [x25], #0x8\n" "usubl v15.8h, v15.8b, v6.8b\n" "usubl v16.8h, v16.8b, v6.8b\n" - "ldr s22, [x22, x11]\n" + "ldr s22, [x20, x11]\n" "usubl v17.8h, v17.8b, v6.8b\n" "usubl v18.8h, v18.8b, v6.8b\n" "usubl v19.8h, v19.8b, v6.8b\n" @@ -167,45 +168,45 @@ void a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst_impl( "sqrdmulh v24.4s, v24.4s, v2.4s\n" "sqrdmulh v25.4s, v25.4s, v2.4s\n" "ldr x20, [%x[outptrs], #0x40]\n" - "and v21.16b, v23.16b, v1.16b\n" - "and v20.16b, v24.16b, v1.16b\n" - "and v19.16b, v25.16b, v1.16b\n" + "and v18.16b, v23.16b, v1.16b\n" + "and v17.16b, v24.16b, v1.16b\n" + "and v16.16b, v25.16b, v1.16b\n" "sshl v26.4s, v26.4s, v3.4s\n" "sshl v27.4s, v27.4s, v3.4s\n" "sshl v28.4s, v28.4s, v3.4s\n" "sshl v29.4s, v29.4s, v3.4s\n" "sshl v30.4s, v30.4s, v3.4s\n" "sshl v31.4s, v31.4s, v3.4s\n" - "sshr v21.4s, v21.4s, #0x1f\n" - "sshr v20.4s, v20.4s, #0x1f\n" - "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" "sqrdmulh v26.4s, v26.4s, v2.4s\n" "sqrdmulh v27.4s, v27.4s, v2.4s\n" "sqrdmulh v28.4s, v28.4s, v2.4s\n" "sqrdmulh v29.4s, v29.4s, v2.4s\n" "sqrdmulh v30.4s, v30.4s, v2.4s\n" "sqrdmulh v31.4s, v31.4s, v2.4s\n" - "sqadd v23.4s, v23.4s, v21.4s\n" - "sqadd v24.4s, v24.4s, v20.4s\n" - "sqadd v25.4s, v25.4s, v19.4s\n" - "and v18.16b, v26.16b, v1.16b\n" - "and v17.16b, v27.16b, v1.16b\n" - "and v16.16b, v28.16b, v1.16b\n" - "and v21.16b, v29.16b, v1.16b\n" - "and v20.16b, v30.16b, v1.16b\n" - "and v19.16b, v31.16b, v1.16b\n" - "sshr v18.4s, v18.4s, #0x1f\n" - "sshr v17.4s, v17.4s, #0x1f\n" - "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v23.4s, v23.4s, v18.4s\n" + "sqadd v24.4s, v24.4s, v17.4s\n" + "sqadd v25.4s, v25.4s, v16.4s\n" + "and v21.16b, v26.16b, v1.16b\n" + "and v20.16b, v27.16b, v1.16b\n" + "and v19.16b, v28.16b, v1.16b\n" + "and v18.16b, v29.16b, v1.16b\n" + "and v17.16b, v30.16b, v1.16b\n" + "and v16.16b, v31.16b, v1.16b\n" "sshr v21.4s, v21.4s, #0x1f\n" "sshr v20.4s, v20.4s, #0x1f\n" "sshr v19.4s, v19.4s, #0x1f\n" - "sqadd v26.4s, v26.4s, v18.4s\n" - "sqadd v27.4s, v27.4s, v17.4s\n" - "sqadd v28.4s, v28.4s, v16.4s\n" - "sqadd v29.4s, v29.4s, v21.4s\n" - "sqadd v30.4s, v30.4s, v20.4s\n" - "sqadd v31.4s, v31.4s, v19.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v26.4s, v26.4s, v21.4s\n" + "sqadd v27.4s, v27.4s, v20.4s\n" + "sqadd v28.4s, v28.4s, v19.4s\n" + "sqadd v29.4s, v29.4s, v18.4s\n" + "sqadd v30.4s, v30.4s, v17.4s\n" + "sqadd v31.4s, v31.4s, v16.4s\n" "srshl v23.4s, v23.4s, v1.4s\n" "srshl v24.4s, v24.4s, v1.4s\n" "srshl v25.4s, v25.4s, v1.4s\n" @@ -270,7 +271,7 @@ void a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst_impl( "str s30, [x21, x11]\n" "str s31, [x20, x11]\n" "add x11, x11, #0x4\n" - "cmp x11, x12, LSL #2\n" + "cmp x11, x9, LSL #2\n" "blt 1b\n" "6:" // Oddments "tst %x[n_channels], #0x3\n" @@ -288,61 +289,61 @@ void a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst_impl( "8:" // Oddments: Load bias: Bit 1: End "9:" // Oddments: Load bias: Done "ldr s0, [%x[params]], #0x4\n" - "mov x21, %x[inptrs]\n" - "ldp x10, x9, [x21], #0x10\n" + "mov x10, %x[inptrs]\n" + "ldp x9, x28, [x10], #0x10\n" "mov v24.16b, v23.16b\n" - "ldp x28, x27, [x21], #0x10\n" - "ldp x26, x25, [x21], #0x10\n" + "ldp x27, x26, [x10], #0x10\n" + "ldp x25, x24, [x10], #0x10\n" "mov v25.16b, v23.16b\n" "mov v26.16b, v23.16b\n" - "ldp x24, x23, [x21], #0x10\n" - "ldr x22, [x21], #0x8\n" + "ldp x23, x22, [x10], #0x10\n" + "ldr x21, [x10], #0x8\n" "mov v27.16b, v23.16b\n" "mov v28.16b, v23.16b\n" "mov v29.16b, v23.16b\n" "mov v30.16b, v23.16b\n" - "add x10, x10, x11\n" "add x9, x9, x11\n" + "add x28, x28, x11\n" "mov v31.16b, v23.16b\n" "ssubl v0.8h, v0.8b, v5.8b\n" - "add x28, x28, x11\n" "add x27, x27, x11\n" "add x26, x26, x11\n" "add x25, x25, x11\n" "add x24, x24, x11\n" "add x23, x23, x11\n" "add x22, x22, x11\n" + "add x21, x21, x11\n" "tbz %x[n_channels], #1, 10f\n" - "ldr h14, [x10], #0x2\n" - "ldr h15, [x9], #0x2\n" - "ldr h16, [x28], #0x2\n" - "ldr h17, [x27], #0x2\n" - "ldr h18, [x26], #0x2\n" - "ldr h19, [x25], #0x2\n" - "ldr h20, [x24], #0x2\n" - "ldr h21, [x23], #0x2\n" - "ldr h22, [x22], #0x2\n" + "ldr h14, [x9], #0x2\n" + "ldr h15, [x28], #0x2\n" + "ldr h16, [x27], #0x2\n" + "ldr h17, [x26], #0x2\n" + "ldr h18, [x25], #0x2\n" + "ldr h19, [x24], #0x2\n" + "ldr h20, [x23], #0x2\n" + "ldr h21, [x22], #0x2\n" + "ldr h22, [x21], #0x2\n" "tbz %x[n_channels], #0, 11f\n" - "ld1 { v14.b }[2], [x10], #0x1\n" - "ld1 { v15.b }[2], [x9], #0x1\n" - "ld1 { v16.b }[2], [x28], #0x1\n" - "ld1 { v17.b }[2], [x27], #0x1\n" - "ld1 { v18.b }[2], [x26], #0x1\n" - "ld1 { v19.b }[2], [x25], #0x1\n" - "ld1 { v20.b }[2], [x24], #0x1\n" - "ld1 { v21.b }[2], [x23], #0x1\n" - "ld1 { v22.b }[2], [x22], #0x1\n" + "ld1 { v14.b }[2], [x9], #0x1\n" + "ld1 { v15.b }[2], [x28], #0x1\n" + "ld1 { v16.b }[2], [x27], #0x1\n" + "ld1 { v17.b }[2], [x26], #0x1\n" + "ld1 { v18.b }[2], [x25], #0x1\n" + "ld1 { v19.b }[2], [x24], #0x1\n" + "ld1 { v20.b }[2], [x23], #0x1\n" + "ld1 { v21.b }[2], [x22], #0x1\n" + "ld1 { v22.b }[2], [x21], #0x1\n" "b 11f\n" "10:" // Oddments: Load: Bit 1: Unset - "ldr b14, [x10], #0x1\n" - "ldr b15, [x9], #0x1\n" - "ldr b16, [x28], #0x1\n" - "ldr b17, [x27], #0x1\n" - "ldr b18, [x26], #0x1\n" - "ldr b19, [x25], #0x1\n" - "ldr b20, [x24], #0x1\n" - "ldr b21, [x23], #0x1\n" - "ldr b22, [x22], #0x1\n" + "ldr b14, [x9], #0x1\n" + "ldr b15, [x28], #0x1\n" + "ldr b16, [x27], #0x1\n" + "ldr b17, [x26], #0x1\n" + "ldr b18, [x25], #0x1\n" + "ldr b19, [x24], #0x1\n" + "ldr b20, [x23], #0x1\n" + "ldr b21, [x22], #0x1\n" + "ldr b22, [x21], #0x1\n" "11:" // Oddments: Load: Bit 1: End "subs x20, %x[n_points], #0x1\n" "usubl v14.8h, v14.8b, v6.8b\n" @@ -356,62 +357,62 @@ void a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst_impl( "usubl v22.8h, v22.8b, v6.8b\n" "ble 15f\n" "12:" // Oddments: Planar loop - "ldp x10, x9, [x21], #0x10\n" - "ldp x28, x27, [x21], #0x10\n" + "ldp x9, x28, [x10], #0x10\n" + "ldp x27, x26, [x10], #0x10\n" "smlal v23.4s, v14.4h, v0.4h\n" "smlal v24.4s, v15.4h, v0.4h\n" - "ldp x26, x25, [x21], #0x10\n" - "ldp x24, x23, [x21], #0x10\n" + "ldp x25, x24, [x10], #0x10\n" + "ldp x23, x22, [x10], #0x10\n" "smlal v25.4s, v16.4h, v0.4h\n" "smlal v26.4s, v17.4h, v0.4h\n" "smlal v27.4s, v18.4h, v0.4h\n" "smlal v28.4s, v19.4h, v0.4h\n" - "ldr x22, [x21], #0x8\n" - "add x10, x10, x11\n" + "ldr x21, [x10], #0x8\n" + "add x9, x9, x11\n" "smlal v29.4s, v20.4h, v0.4h\n" "smlal v30.4s, v21.4h, v0.4h\n" - "add x9, x9, x11\n" "add x28, x28, x11\n" + "add x27, x27, x11\n" "smlal v31.4s, v22.4h, v0.4h\n" "ldr s0, [%x[params]], #0x4\n" "ssubl v0.8h, v0.8b, v5.8b\n" - "add x27, x27, x11\n" "add x26, x26, x11\n" "add x25, x25, x11\n" "add x24, x24, x11\n" "add x23, x23, x11\n" "add x22, x22, x11\n" + "add x21, x21, x11\n" "tbz %x[n_channels], #1, 13f\n" - "ldr h14, [x10], #0x2\n" - "ldr h15, [x9], #0x2\n" - "ldr h16, [x28], #0x2\n" - "ldr h17, [x27], #0x2\n" - "ldr h18, [x26], #0x2\n" - "ldr h19, [x25], #0x2\n" - "ldr h20, [x24], #0x2\n" - "ldr h21, [x23], #0x2\n" - "ldr h22, [x22], #0x2\n" + "ldr h14, [x9], #0x2\n" + "ldr h15, [x28], #0x2\n" + "ldr h16, [x27], #0x2\n" + "ldr h17, [x26], #0x2\n" + "ldr h18, [x25], #0x2\n" + "ldr h19, [x24], #0x2\n" + "ldr h20, [x23], #0x2\n" + "ldr h21, [x22], #0x2\n" + "ldr h22, [x21], #0x2\n" "tbz %x[n_channels], #0, 14f\n" - "ld1 { v14.b }[2], [x10], #0x1\n" - "ld1 { v15.b }[2], [x9], #0x1\n" - "ld1 { v16.b }[2], [x28], #0x1\n" - "ld1 { v17.b }[2], [x27], #0x1\n" - "ld1 { v18.b }[2], [x26], #0x1\n" - "ld1 { v19.b }[2], [x25], #0x1\n" - "ld1 { v20.b }[2], [x24], #0x1\n" - "ld1 { v21.b }[2], [x23], #0x1\n" - "ld1 { v22.b }[2], [x22], #0x1\n" + "ld1 { v14.b }[2], [x9], #0x1\n" + "ld1 { v15.b }[2], [x28], #0x1\n" + "ld1 { v16.b }[2], [x27], #0x1\n" + "ld1 { v17.b }[2], [x26], #0x1\n" + "ld1 { v18.b }[2], [x25], #0x1\n" + "ld1 { v19.b }[2], [x24], #0x1\n" + "ld1 { v20.b }[2], [x23], #0x1\n" + "ld1 { v21.b }[2], [x22], #0x1\n" + "ld1 { v22.b }[2], [x21], #0x1\n" "b 14f\n" "13:" // Oddments: Planar loop: Load: Bit 1: Unset - "ldr b14, [x10], #0x1\n" - "ldr b15, [x9], #0x1\n" - "ldr b16, [x28], #0x1\n" - "ldr b17, [x27], #0x1\n" - "ldr b18, [x26], #0x1\n" - "ldr b19, [x25], #0x1\n" - "ldr b20, [x24], #0x1\n" - "ldr b21, [x23], #0x1\n" - "ldr b22, [x22], #0x1\n" + "ldr b14, [x9], #0x1\n" + "ldr b15, [x28], #0x1\n" + "ldr b16, [x27], #0x1\n" + "ldr b17, [x26], #0x1\n" + "ldr b18, [x25], #0x1\n" + "ldr b19, [x24], #0x1\n" + "ldr b20, [x23], #0x1\n" + "ldr b21, [x22], #0x1\n" + "ldr b22, [x21], #0x1\n" "14:" // Oddments: Planar loop: Load: Bit 1: End "subs x20, x20, #0x1\n" "usubl v14.8h, v14.8b, v6.8b\n" @@ -457,9 +458,7 @@ void a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst_impl( "cbz %x[rq_left_shift_ptr], 19f\n" "ld1 { v3.s }[0], [x20], #0x4\n" "19:" // Oddments: Load quantisation parameters: Bit 1: Unset: Bit 0: Load left shift: Done - "20:" // Oddments: Load quantisation parameters: Bit 1: End - "21:" // Oddments: Load quantisation parameters: Done "sshl v23.4s, v23.4s, v3.4s\n" "sshl v24.4s, v24.4s, v3.4s\n" @@ -473,11 +472,11 @@ void a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst_impl( "sqrdmulh v25.4s, v25.4s, v2.4s\n" "ldr x20, [%x[outptrs], #0x40]\n" "add x28, x28, x11\n" - "and v21.16b, v23.16b, v1.16b\n" - "and v20.16b, v24.16b, v1.16b\n" + "and v18.16b, v23.16b, v1.16b\n" + "and v17.16b, v24.16b, v1.16b\n" "add x27, x27, x11\n" "add x26, x26, x11\n" - "and v19.16b, v25.16b, v1.16b\n" + "and v16.16b, v25.16b, v1.16b\n" "sshl v26.4s, v26.4s, v3.4s\n" "add x25, x25, x11\n" "add x24, x24, x11\n" @@ -490,36 +489,36 @@ void a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst_impl( "add x21, x21, x11\n" "add x20, x20, x11\n" "sshl v31.4s, v31.4s, v3.4s\n" - "sshr v21.4s, v21.4s, #0x1f\n" - "sshr v20.4s, v20.4s, #0x1f\n" - "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" "sqrdmulh v26.4s, v26.4s, v2.4s\n" "sqrdmulh v27.4s, v27.4s, v2.4s\n" "sqrdmulh v28.4s, v28.4s, v2.4s\n" "sqrdmulh v29.4s, v29.4s, v2.4s\n" "sqrdmulh v30.4s, v30.4s, v2.4s\n" "sqrdmulh v31.4s, v31.4s, v2.4s\n" - "sqadd v23.4s, v23.4s, v21.4s\n" - "sqadd v24.4s, v24.4s, v20.4s\n" - "sqadd v25.4s, v25.4s, v19.4s\n" - "and v18.16b, v26.16b, v1.16b\n" - "and v17.16b, v27.16b, v1.16b\n" - "and v16.16b, v28.16b, v1.16b\n" - "and v21.16b, v29.16b, v1.16b\n" - "and v20.16b, v30.16b, v1.16b\n" - "and v19.16b, v31.16b, v1.16b\n" - "sshr v18.4s, v18.4s, #0x1f\n" - "sshr v17.4s, v17.4s, #0x1f\n" - "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v23.4s, v23.4s, v18.4s\n" + "sqadd v24.4s, v24.4s, v17.4s\n" + "sqadd v25.4s, v25.4s, v16.4s\n" + "and v21.16b, v26.16b, v1.16b\n" + "and v20.16b, v27.16b, v1.16b\n" + "and v19.16b, v28.16b, v1.16b\n" + "and v18.16b, v29.16b, v1.16b\n" + "and v17.16b, v30.16b, v1.16b\n" + "and v16.16b, v31.16b, v1.16b\n" "sshr v21.4s, v21.4s, #0x1f\n" "sshr v20.4s, v20.4s, #0x1f\n" "sshr v19.4s, v19.4s, #0x1f\n" - "sqadd v26.4s, v26.4s, v18.4s\n" - "sqadd v27.4s, v27.4s, v17.4s\n" - "sqadd v28.4s, v28.4s, v16.4s\n" - "sqadd v29.4s, v29.4s, v21.4s\n" - "sqadd v30.4s, v30.4s, v20.4s\n" - "sqadd v31.4s, v31.4s, v19.4s\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v26.4s, v26.4s, v21.4s\n" + "sqadd v27.4s, v27.4s, v20.4s\n" + "sqadd v28.4s, v28.4s, v19.4s\n" + "sqadd v29.4s, v29.4s, v18.4s\n" + "sqadd v30.4s, v30.4s, v17.4s\n" + "sqadd v31.4s, v31.4s, v16.4s\n" "srshl v23.4s, v23.4s, v1.4s\n" "srshl v24.4s, v24.4s, v1.4s\n" "srshl v25.4s, v25.4s, v1.4s\n" @@ -606,15 +605,14 @@ void a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst_impl( "st1 { v30.b }[0], [x21], #0x1\n" "st1 { v31.b }[0], [x20], #0x1\n" "23:" // Oddments: Store: Bit 1: End - "24:" // End - : [params] "+&r" (params) : [bias] "r" (qp.bias), [inptrs] "r" (inptrs), [n_channels] "r" ((uint64_t) n_channels), [n_points] "r" ((uint64_t) n_points), [offsetof_Requantize32_a_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [offsetof_Requantize32_per_layer_left_shift] "I" (offsetof(arm_gemm::Requantize32, per_layer_left_shift)), [offsetof_Requantize32_per_layer_mul] "I" (offsetof(arm_gemm::Requantize32, per_layer_mul)), [offsetof_Requantize32_per_layer_right_shift] "I" (offsetof(arm_gemm::Requantize32, per_layer_right_shift)), [outptrs] "r" (outptrs), [qp] "r" (&qp), [rq_left_shift_ptr] "r" (qp.per_channel_left_shifts), [rq_mul_ptr] "r" (qp.per_channel_muls), [rq_right_shift_ptr] "r" (qp.per_channel_right_shifts) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp index 394df363da4ded353ceff997f9436f47caf697df..b7ba363b43a6c361d823c701322c798a7629c491 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp index 976434aa28bfc7d8a544c84c3f4725448155246b..ed99f1f642e8dfd45b3381b0998cfc716d45dc8f 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp @@ -22,12 +22,13 @@ * SOFTWARE. */ -#if defined(__aarch64__) #include "arm_gemm.hpp" #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace depthwise { @@ -47,21 +48,21 @@ void a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst __asm__ __volatile__( "lsr x10, %x[n_output_channels], #0x2\n" "add x20, %x[qp], %[offsetof_Requantize32_minval]\n" - "ld1r { v13.4s }, [x20]\n" + "ld1r { v15.4s }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_maxval]\n" - "ld1r { v11.4s }, [x20]\n" + "ld1r { v14.4s }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_a_offset]\n" - "ld1r { v3.16b }, [x20]\n" + "ld1r { v13.16b }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_b_offset]\n" "ld1r { v12.16b }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_c_offset]\n" - "ld1r { v14.4s }, [x20]\n" + "ld1r { v11.4s }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_per_layer_left_shift]\n" - "ld1r { v15.4s }, [x20]\n" + "ld1r { v10.4s }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_per_layer_mul]\n" "ld1r { v9.4s }, [x20]\n" "add x20, %x[qp], %[offsetof_Requantize32_per_layer_right_shift]\n" - "ld1r { v10.4s }, [x20]\n" + "ld1r { v8.4s }, [x20]\n" "mov x9, #0x0\n" "cbz x10, 9f\n" "1:" // Output channel loop @@ -89,256 +90,256 @@ void a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst "cbz %x[rq_mul_ptr], 3f\n" "lsl x20, x9, #0x2\n" "ldr q9, [%x[rq_mul_ptr], x20]\n" - "ldr q10, [%x[rq_right_shift_ptr], x20]\n" + "ldr q8, [%x[rq_right_shift_ptr], x20]\n" "cbz %x[rq_left_shift_ptr], 3f\n" - "ldr q15, [%x[rq_left_shift_ptr], x20]\n" + "ldr q10, [%x[rq_left_shift_ptr], x20]\n" "3:" // Output channel loop: Load quantization parameters: Done - "ldr s8, [%x[weights]], #0x4\n" - "mov x20, %x[inptrs]\n" - "ldp x25, x28, [x20], #0x10\n" - "lsr x21, %x[kernel_points], #0x1\n" - "ldr d2, [x25, #0x0]\n" - "ldr d7, [x28, #0x0]\n" - "usubl v2.8h, v2.8b, v3.8b\n" - "usubl v7.8h, v7.8b, v3.8b\n" - "ssubl v8.8h, v8.8b, v12.8b\n" - "cbz x21, 7f\n" - "ldr s6, [%x[weights]], #0x4\n" - "ldp x25, x28, [x20], #0x10\n" - "subs x21, x21, #0x1\n" - "ssubl v6.8h, v6.8b, v12.8b\n" - "ldr d1, [x25, #0x0]\n" - "ldr d0, [x28, #0x0]\n" - "usubl v1.8h, v1.8b, v3.8b\n" - "usubl v0.8h, v0.8b, v3.8b\n" + "ldr s5, [%x[weights]], #0x4\n" + "mov x22, %x[inptrs]\n" + "ldp x21, x20, [x22], #0x10\n" + "lsr x23, %x[kernel_points], #0x1\n" + "ldr d0, [x21, #0x0]\n" + "ldr d4, [x20, #0x0]\n" + "usubl v0.8h, v0.8b, v13.8b\n" + "usubl v4.8h, v4.8b, v13.8b\n" + "ssubl v5.8h, v5.8b, v12.8b\n" + "cbz x23, 7f\n" + "ldr s7, [%x[weights]], #0x4\n" + "ldp x21, x20, [x22], #0x10\n" + "subs x23, x23, #0x1\n" + "ssubl v7.8h, v7.8b, v12.8b\n" + "ldr d3, [x21, #0x0]\n" + "ldr d6, [x20, #0x0]\n" + "usubl v3.8h, v3.8b, v13.8b\n" + "usubl v6.8h, v6.8b, v13.8b\n" "beq 5f\n" "4:" // Output channel loop: Kernel loop - "ldp x25, x28, [x20], #0x10\n" - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "subs x21, x21, #0x1\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "ldr d2, [x25, #0x0]\n" - "usubl v2.8h, v2.8b, v3.8b\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" - "ldr d7, [x28, #0x0]\n" - "ldr s8, [%x[weights]], #0x4\n" - "ldp x25, x28, [x20], #0x10\n" - "smlal v16.4s, v6.4h, v1.h[0]\n" - "smlal v17.4s, v6.4h, v1.h[1]\n" - "usubl v7.8h, v7.8b, v3.8b\n" - "smlal v18.4s, v6.4h, v1.h[2]\n" - "smlal v19.4s, v6.4h, v1.h[3]\n" - "ssubl v8.8h, v8.8b, v12.8b\n" - "smlal v20.4s, v6.4h, v1.h[4]\n" - "smlal v21.4s, v6.4h, v1.h[5]\n" - "smlal v22.4s, v6.4h, v1.h[6]\n" - "smlal v23.4s, v6.4h, v1.h[7]\n" - "ldr d1, [x25, #0x0]\n" - "usubl v1.8h, v1.8b, v3.8b\n" - "smlal v24.4s, v6.4h, v0.h[0]\n" - "smlal v25.4s, v6.4h, v0.h[1]\n" - "smlal v26.4s, v6.4h, v0.h[2]\n" - "smlal v27.4s, v6.4h, v0.h[3]\n" - "smlal v28.4s, v6.4h, v0.h[4]\n" - "smlal v29.4s, v6.4h, v0.h[5]\n" - "smlal v30.4s, v6.4h, v0.h[6]\n" - "smlal v31.4s, v6.4h, v0.h[7]\n" - "ldr d0, [x28, #0x0]\n" - "ldr s6, [%x[weights]], #0x4\n" - "usubl v0.8h, v0.8b, v3.8b\n" - "ssubl v6.8h, v6.8b, v12.8b\n" + "ldp x21, x20, [x22], #0x10\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "subs x23, x23, #0x1\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" + "ldr d0, [x21, #0x0]\n" + "usubl v0.8h, v0.8b, v13.8b\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" + "ldr d4, [x20, #0x0]\n" + "ldr s5, [%x[weights]], #0x4\n" + "ldp x21, x20, [x22], #0x10\n" + "smlal v16.4s, v7.4h, v3.h[0]\n" + "smlal v17.4s, v7.4h, v3.h[1]\n" + "usubl v4.8h, v4.8b, v13.8b\n" + "smlal v18.4s, v7.4h, v3.h[2]\n" + "smlal v19.4s, v7.4h, v3.h[3]\n" + "ssubl v5.8h, v5.8b, v12.8b\n" + "smlal v20.4s, v7.4h, v3.h[4]\n" + "smlal v21.4s, v7.4h, v3.h[5]\n" + "smlal v22.4s, v7.4h, v3.h[6]\n" + "smlal v23.4s, v7.4h, v3.h[7]\n" + "ldr d3, [x21, #0x0]\n" + "usubl v3.8h, v3.8b, v13.8b\n" + "smlal v24.4s, v7.4h, v6.h[0]\n" + "smlal v25.4s, v7.4h, v6.h[1]\n" + "smlal v26.4s, v7.4h, v6.h[2]\n" + "smlal v27.4s, v7.4h, v6.h[3]\n" + "smlal v28.4s, v7.4h, v6.h[4]\n" + "smlal v29.4s, v7.4h, v6.h[5]\n" + "smlal v30.4s, v7.4h, v6.h[6]\n" + "smlal v31.4s, v7.4h, v6.h[7]\n" + "ldr d6, [x20, #0x0]\n" + "ldr s7, [%x[weights]], #0x4\n" + "usubl v6.8h, v6.8b, v13.8b\n" + "ssubl v7.8h, v7.8b, v12.8b\n" "bgt 4b\n" "5:" // Output channel loop: Kernel loop tail "tbnz %x[kernel_points], #0, 6f\n" - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "smlal v16.4s, v6.4h, v1.h[0]\n" - "smlal v17.4s, v6.4h, v1.h[1]\n" - "sshl v16.4s, v16.4s, v15.4s\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "smlal v18.4s, v6.4h, v1.h[2]\n" - "smlal v19.4s, v6.4h, v1.h[3]\n" - "sshl v17.4s, v17.4s, v15.4s\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "sshl v18.4s, v18.4s, v15.4s\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "sshl v19.4s, v19.4s, v15.4s\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "smlal v16.4s, v7.4h, v3.h[0]\n" + "smlal v17.4s, v7.4h, v3.h[1]\n" + "sshl v16.4s, v16.4s, v10.4s\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "smlal v18.4s, v7.4h, v3.h[2]\n" + "smlal v19.4s, v7.4h, v3.h[3]\n" + "sshl v17.4s, v17.4s, v10.4s\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" + "sshl v18.4s, v18.4s, v10.4s\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" + "sshl v19.4s, v19.4s, v10.4s\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" "sqrdmulh v16.4s, v16.4s, v9.4s\n" - "smlal v20.4s, v6.4h, v1.h[4]\n" - "smlal v21.4s, v6.4h, v1.h[5]\n" + "smlal v20.4s, v7.4h, v3.h[4]\n" + "smlal v21.4s, v7.4h, v3.h[5]\n" "sqrdmulh v17.4s, v17.4s, v9.4s\n" - "smlal v22.4s, v6.4h, v1.h[6]\n" - "smlal v23.4s, v6.4h, v1.h[7]\n" + "smlal v22.4s, v7.4h, v3.h[6]\n" + "smlal v23.4s, v7.4h, v3.h[7]\n" "sqrdmulh v18.4s, v18.4s, v9.4s\n" - "smlal v24.4s, v6.4h, v0.h[0]\n" - "smlal v25.4s, v6.4h, v0.h[1]\n" + "smlal v24.4s, v7.4h, v6.h[0]\n" + "smlal v25.4s, v7.4h, v6.h[1]\n" "sqrdmulh v19.4s, v19.4s, v9.4s\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "and v5.16b, v16.16b, v10.16b\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "and v4.16b, v17.16b, v10.16b\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" - "and v2.16b, v18.16b, v10.16b\n" - "and v1.16b, v19.16b, v10.16b\n" - "sshl v20.4s, v20.4s, v15.4s\n" - "smlal v26.4s, v6.4h, v0.h[2]\n" - "sshl v21.4s, v21.4s, v15.4s\n" - "sshl v22.4s, v22.4s, v15.4s\n" - "smlal v27.4s, v6.4h, v0.h[3]\n" - "sshl v23.4s, v23.4s, v15.4s\n" - "sshl v24.4s, v24.4s, v15.4s\n" - "smlal v28.4s, v6.4h, v0.h[4]\n" - "sshl v25.4s, v25.4s, v15.4s\n" - "smlal v29.4s, v6.4h, v0.h[5]\n" - "smlal v30.4s, v6.4h, v0.h[6]\n" - "smlal v31.4s, v6.4h, v0.h[7]\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "and v3.16b, v16.16b, v8.16b\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "and v2.16b, v17.16b, v8.16b\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" + "and v1.16b, v18.16b, v8.16b\n" + "and v0.16b, v19.16b, v8.16b\n" + "sshl v20.4s, v20.4s, v10.4s\n" + "smlal v26.4s, v7.4h, v6.h[2]\n" + "sshl v21.4s, v21.4s, v10.4s\n" + "sshl v22.4s, v22.4s, v10.4s\n" + "smlal v27.4s, v7.4h, v6.h[3]\n" + "sshl v23.4s, v23.4s, v10.4s\n" + "sshl v24.4s, v24.4s, v10.4s\n" + "smlal v28.4s, v7.4h, v6.h[4]\n" + "sshl v25.4s, v25.4s, v10.4s\n" + "smlal v29.4s, v7.4h, v6.h[5]\n" + "smlal v30.4s, v7.4h, v6.h[6]\n" + "smlal v31.4s, v7.4h, v6.h[7]\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v0.4s, v0.4s, #0x1f\n" "sqrdmulh v20.4s, v20.4s, v9.4s\n" "sqrdmulh v21.4s, v21.4s, v9.4s\n" "sqrdmulh v22.4s, v22.4s, v9.4s\n" "sqrdmulh v23.4s, v23.4s, v9.4s\n" "sqrdmulh v24.4s, v24.4s, v9.4s\n" "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqadd v16.4s, v16.4s, v5.4s\n" - "sqadd v17.4s, v17.4s, v4.4s\n" - "sqadd v18.4s, v18.4s, v2.4s\n" - "sqadd v19.4s, v19.4s, v1.4s\n" - "and v8.16b, v20.16b, v10.16b\n" - "and v0.16b, v21.16b, v10.16b\n" - "and v5.16b, v22.16b, v10.16b\n" - "and v4.16b, v23.16b, v10.16b\n" - "and v2.16b, v24.16b, v10.16b\n" - "and v1.16b, v25.16b, v10.16b\n" - "sshl v26.4s, v26.4s, v15.4s\n" - "sshl v27.4s, v27.4s, v15.4s\n" - "sshl v28.4s, v28.4s, v15.4s\n" - "sshl v29.4s, v29.4s, v15.4s\n" - "sshl v30.4s, v30.4s, v15.4s\n" - "sshl v31.4s, v31.4s, v15.4s\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v3.4s\n" + "sqadd v17.4s, v17.4s, v2.4s\n" + "sqadd v18.4s, v18.4s, v1.4s\n" + "sqadd v19.4s, v19.4s, v0.4s\n" + "and v5.16b, v20.16b, v8.16b\n" + "and v4.16b, v21.16b, v8.16b\n" + "and v3.16b, v22.16b, v8.16b\n" + "and v2.16b, v23.16b, v8.16b\n" + "and v1.16b, v24.16b, v8.16b\n" + "and v0.16b, v25.16b, v8.16b\n" + "sshl v26.4s, v26.4s, v10.4s\n" + "sshl v27.4s, v27.4s, v10.4s\n" + "sshl v28.4s, v28.4s, v10.4s\n" + "sshl v29.4s, v29.4s, v10.4s\n" + "sshl v30.4s, v30.4s, v10.4s\n" + "sshl v31.4s, v31.4s, v10.4s\n" "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v0.4s, v0.4s, #0x1f\n" "sqrdmulh v26.4s, v26.4s, v9.4s\n" "sqrdmulh v27.4s, v27.4s, v9.4s\n" "sqrdmulh v28.4s, v28.4s, v9.4s\n" "sqrdmulh v29.4s, v29.4s, v9.4s\n" "sqrdmulh v30.4s, v30.4s, v9.4s\n" "sqrdmulh v31.4s, v31.4s, v9.4s\n" - "sqadd v20.4s, v20.4s, v8.4s\n" - "sqadd v21.4s, v21.4s, v0.4s\n" - "sqadd v22.4s, v22.4s, v5.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v24.4s, v24.4s, v2.4s\n" - "sqadd v25.4s, v25.4s, v1.4s\n" - "and v8.16b, v26.16b, v10.16b\n" - "and v0.16b, v27.16b, v10.16b\n" - "and v5.16b, v28.16b, v10.16b\n" - "and v4.16b, v29.16b, v10.16b\n" - "and v2.16b, v30.16b, v10.16b\n" - "and v1.16b, v31.16b, v10.16b\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v5.4s\n" + "sqadd v21.4s, v21.4s, v4.4s\n" + "sqadd v22.4s, v22.4s, v3.4s\n" + "sqadd v23.4s, v23.4s, v2.4s\n" + "sqadd v24.4s, v24.4s, v1.4s\n" + "sqadd v25.4s, v25.4s, v0.4s\n" + "and v5.16b, v26.16b, v8.16b\n" + "and v4.16b, v27.16b, v8.16b\n" + "and v3.16b, v28.16b, v8.16b\n" + "and v2.16b, v29.16b, v8.16b\n" + "and v1.16b, v30.16b, v8.16b\n" + "and v0.16b, v31.16b, v8.16b\n" "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" - "srshl v16.4s, v16.4s, v10.4s\n" - "srshl v17.4s, v17.4s, v10.4s\n" - "srshl v18.4s, v18.4s, v10.4s\n" - "srshl v19.4s, v19.4s, v10.4s\n" - "srshl v20.4s, v20.4s, v10.4s\n" - "srshl v21.4s, v21.4s, v10.4s\n" - "srshl v22.4s, v22.4s, v10.4s\n" - "srshl v23.4s, v23.4s, v10.4s\n" - "sqadd v26.4s, v26.4s, v8.4s\n" - "sqadd v27.4s, v27.4s, v0.4s\n" - "sqadd v28.4s, v28.4s, v5.4s\n" - "sqadd v29.4s, v29.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v2.4s\n" - "sqadd v31.4s, v31.4s, v1.4s\n" - "add v16.4s, v16.4s, v14.4s\n" - "add v17.4s, v17.4s, v14.4s\n" - "add v18.4s, v18.4s, v14.4s\n" - "add v19.4s, v19.4s, v14.4s\n" - "add v20.4s, v20.4s, v14.4s\n" - "add v21.4s, v21.4s, v14.4s\n" - "add v22.4s, v22.4s, v14.4s\n" - "add v23.4s, v23.4s, v14.4s\n" - "srshl v24.4s, v24.4s, v10.4s\n" - "srshl v25.4s, v25.4s, v10.4s\n" - "srshl v26.4s, v26.4s, v10.4s\n" - "srshl v27.4s, v27.4s, v10.4s\n" - "srshl v28.4s, v28.4s, v10.4s\n" - "srshl v29.4s, v29.4s, v10.4s\n" - "srshl v30.4s, v30.4s, v10.4s\n" - "srshl v31.4s, v31.4s, v10.4s\n" - "smin v16.4s, v16.4s, v11.4s\n" - "smin v17.4s, v17.4s, v11.4s\n" - "smin v18.4s, v18.4s, v11.4s\n" - "smin v19.4s, v19.4s, v11.4s\n" - "smin v20.4s, v20.4s, v11.4s\n" - "smin v21.4s, v21.4s, v11.4s\n" - "smin v22.4s, v22.4s, v11.4s\n" - "smin v23.4s, v23.4s, v11.4s\n" - "add v24.4s, v24.4s, v14.4s\n" - "add v25.4s, v25.4s, v14.4s\n" - "add v26.4s, v26.4s, v14.4s\n" - "add v27.4s, v27.4s, v14.4s\n" - "add v28.4s, v28.4s, v14.4s\n" - "add v29.4s, v29.4s, v14.4s\n" - "add v30.4s, v30.4s, v14.4s\n" - "add v31.4s, v31.4s, v14.4s\n" - "smax v16.4s, v16.4s, v13.4s\n" - "smax v17.4s, v17.4s, v13.4s\n" - "smax v18.4s, v18.4s, v13.4s\n" - "smax v19.4s, v19.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smax v21.4s, v21.4s, v13.4s\n" - "smax v22.4s, v22.4s, v13.4s\n" - "smax v23.4s, v23.4s, v13.4s\n" - "smin v24.4s, v24.4s, v11.4s\n" - "smin v25.4s, v25.4s, v11.4s\n" - "smin v26.4s, v26.4s, v11.4s\n" - "smin v27.4s, v27.4s, v11.4s\n" - "smin v28.4s, v28.4s, v11.4s\n" - "smin v29.4s, v29.4s, v11.4s\n" - "smin v30.4s, v30.4s, v11.4s\n" - "smin v31.4s, v31.4s, v11.4s\n" + "sshr v0.4s, v0.4s, #0x1f\n" + "srshl v16.4s, v16.4s, v8.4s\n" + "srshl v17.4s, v17.4s, v8.4s\n" + "srshl v18.4s, v18.4s, v8.4s\n" + "srshl v19.4s, v19.4s, v8.4s\n" + "srshl v20.4s, v20.4s, v8.4s\n" + "srshl v21.4s, v21.4s, v8.4s\n" + "srshl v22.4s, v22.4s, v8.4s\n" + "srshl v23.4s, v23.4s, v8.4s\n" + "sqadd v26.4s, v26.4s, v5.4s\n" + "sqadd v27.4s, v27.4s, v4.4s\n" + "sqadd v28.4s, v28.4s, v3.4s\n" + "sqadd v29.4s, v29.4s, v2.4s\n" + "sqadd v30.4s, v30.4s, v1.4s\n" + "sqadd v31.4s, v31.4s, v0.4s\n" + "add v16.4s, v16.4s, v11.4s\n" + "add v17.4s, v17.4s, v11.4s\n" + "add v18.4s, v18.4s, v11.4s\n" + "add v19.4s, v19.4s, v11.4s\n" + "add v20.4s, v20.4s, v11.4s\n" + "add v21.4s, v21.4s, v11.4s\n" + "add v22.4s, v22.4s, v11.4s\n" + "add v23.4s, v23.4s, v11.4s\n" + "srshl v24.4s, v24.4s, v8.4s\n" + "srshl v25.4s, v25.4s, v8.4s\n" + "srshl v26.4s, v26.4s, v8.4s\n" + "srshl v27.4s, v27.4s, v8.4s\n" + "srshl v28.4s, v28.4s, v8.4s\n" + "srshl v29.4s, v29.4s, v8.4s\n" + "srshl v30.4s, v30.4s, v8.4s\n" + "srshl v31.4s, v31.4s, v8.4s\n" + "smin v16.4s, v16.4s, v14.4s\n" + "smin v17.4s, v17.4s, v14.4s\n" + "smin v18.4s, v18.4s, v14.4s\n" + "smin v19.4s, v19.4s, v14.4s\n" + "smin v20.4s, v20.4s, v14.4s\n" + "smin v21.4s, v21.4s, v14.4s\n" + "smin v22.4s, v22.4s, v14.4s\n" + "smin v23.4s, v23.4s, v14.4s\n" + "add v24.4s, v24.4s, v11.4s\n" + "add v25.4s, v25.4s, v11.4s\n" + "add v26.4s, v26.4s, v11.4s\n" + "add v27.4s, v27.4s, v11.4s\n" + "add v28.4s, v28.4s, v11.4s\n" + "add v29.4s, v29.4s, v11.4s\n" + "add v30.4s, v30.4s, v11.4s\n" + "add v31.4s, v31.4s, v11.4s\n" + "smax v16.4s, v16.4s, v15.4s\n" + "smax v17.4s, v17.4s, v15.4s\n" + "smax v18.4s, v18.4s, v15.4s\n" + "smax v19.4s, v19.4s, v15.4s\n" + "smax v20.4s, v20.4s, v15.4s\n" + "smax v21.4s, v21.4s, v15.4s\n" + "smax v22.4s, v22.4s, v15.4s\n" + "smax v23.4s, v23.4s, v15.4s\n" + "smin v24.4s, v24.4s, v14.4s\n" + "smin v25.4s, v25.4s, v14.4s\n" + "smin v26.4s, v26.4s, v14.4s\n" + "smin v27.4s, v27.4s, v14.4s\n" + "smin v28.4s, v28.4s, v14.4s\n" + "smin v29.4s, v29.4s, v14.4s\n" + "smin v30.4s, v30.4s, v14.4s\n" + "smin v31.4s, v31.4s, v14.4s\n" "uzp1 v16.16b, v16.16b, v16.16b\n" "uzp1 v17.16b, v17.16b, v17.16b\n" "uzp1 v18.16b, v18.16b, v18.16b\n" @@ -347,263 +348,263 @@ void a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst "uzp1 v21.16b, v21.16b, v21.16b\n" "uzp1 v22.16b, v22.16b, v22.16b\n" "uzp1 v23.16b, v23.16b, v23.16b\n" - "smax v24.4s, v24.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v26.4s, v26.4s, v13.4s\n" - "smax v27.4s, v27.4s, v13.4s\n" - "smax v28.4s, v28.4s, v13.4s\n" - "smax v29.4s, v29.4s, v13.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" - "smax v31.4s, v31.4s, v13.4s\n" + "smax v24.4s, v24.4s, v15.4s\n" + "smax v25.4s, v25.4s, v15.4s\n" + "smax v26.4s, v26.4s, v15.4s\n" + "smax v27.4s, v27.4s, v15.4s\n" + "smax v28.4s, v28.4s, v15.4s\n" + "smax v29.4s, v29.4s, v15.4s\n" + "smax v30.4s, v30.4s, v15.4s\n" + "smax v31.4s, v31.4s, v15.4s\n" "uzp1 v16.16b, v16.16b, v16.16b\n" - "str s16, [x20, x9]\n" - "ldr x20, [%x[outptrs], #0x40]\n" + "str s16, [x27, x9]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "uzp1 v17.16b, v17.16b, v17.16b\n" "uzp1 v18.16b, v18.16b, v18.16b\n" - "str s17, [x21, x9]\n" - "ldr x21, [%x[outptrs], #0x48]\n" + "str s17, [x26, x9]\n" + "ldr x26, [%x[outptrs], #0x48]\n" "uzp1 v19.16b, v19.16b, v19.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s18, [x22, x9]\n" - "ldr x22, [%x[outptrs], #0x50]\n" + "str s18, [x25, x9]\n" + "ldr x25, [%x[outptrs], #0x50]\n" "uzp1 v21.16b, v21.16b, v21.16b\n" "uzp1 v22.16b, v22.16b, v22.16b\n" - "str s19, [x23, x9]\n" - "ldr x23, [%x[outptrs], #0x58]\n" + "str s19, [x24, x9]\n" + "ldr x24, [%x[outptrs], #0x58]\n" "uzp1 v23.16b, v23.16b, v23.16b\n" "uzp1 v24.16b, v24.16b, v24.16b\n" - "str s20, [x24, x9]\n" - "ldr x24, [%x[outptrs], #0x60]\n" + "str s20, [x23, x9]\n" + "ldr x23, [%x[outptrs], #0x60]\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s21, [x25, x9]\n" - "ldr x25, [%x[outptrs], #0x68]\n" + "str s21, [x22, x9]\n" + "ldr x22, [%x[outptrs], #0x68]\n" "uzp1 v27.16b, v27.16b, v27.16b\n" "uzp1 v28.16b, v28.16b, v28.16b\n" - "str s22, [x26, x9]\n" - "ldr x26, [%x[outptrs], #0x70]\n" + "str s22, [x21, x9]\n" + "ldr x21, [%x[outptrs], #0x70]\n" "uzp1 v29.16b, v29.16b, v29.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s23, [x27, x9]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "str s23, [x20, x9]\n" + "ldr x20, [%x[outptrs], #0x78]\n" "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v24.16b, v24.16b, v24.16b\n" - "str s24, [x20, x9]\n" + "str s24, [x27, x9]\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s25, [x21, x9]\n" + "str s25, [x26, x9]\n" "uzp1 v27.16b, v27.16b, v27.16b\n" "uzp1 v28.16b, v28.16b, v28.16b\n" - "str s26, [x22, x9]\n" + "str s26, [x25, x9]\n" "uzp1 v29.16b, v29.16b, v29.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s27, [x23, x9]\n" + "str s27, [x24, x9]\n" "uzp1 v31.16b, v31.16b, v31.16b\n" - "str s28, [x24, x9]\n" - "str s29, [x25, x9]\n" - "str s30, [x26, x9]\n" - "str s31, [x27, x9]\n" + "str s28, [x23, x9]\n" + "str s29, [x22, x9]\n" + "str s30, [x21, x9]\n" + "str s31, [x20, x9]\n" "b 8f\n" "6:" // Output channel loop: Odd tail - "ldp x25, x28, [x20], #0x10\n" - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "ldr d2, [x25, #0x0]\n" - "usubl v2.8h, v2.8b, v3.8b\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" - "ldr s8, [%x[weights]], #0x4\n" - "ldr d7, [x28, #0x0]\n" - "smlal v16.4s, v6.4h, v1.h[0]\n" - "smlal v17.4s, v6.4h, v1.h[1]\n" - "ssubl v8.8h, v8.8b, v12.8b\n" - "smlal v18.4s, v6.4h, v1.h[2]\n" - "smlal v19.4s, v6.4h, v1.h[3]\n" - "usubl v7.8h, v7.8b, v3.8b\n" - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "sshl v16.4s, v16.4s, v15.4s\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "sshl v17.4s, v17.4s, v15.4s\n" - "smlal v20.4s, v6.4h, v1.h[4]\n" - "smlal v21.4s, v6.4h, v1.h[5]\n" - "sshl v18.4s, v18.4s, v15.4s\n" - "smlal v22.4s, v6.4h, v1.h[6]\n" - "smlal v23.4s, v6.4h, v1.h[7]\n" - "sshl v19.4s, v19.4s, v15.4s\n" - "smlal v24.4s, v6.4h, v0.h[0]\n" - "smlal v25.4s, v6.4h, v0.h[1]\n" + "ldp x20, x28, [x22], #0x10\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "ldr x26, [%x[outptrs], #0x8]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" + "ldr d0, [x20, #0x0]\n" + "usubl v0.8h, v0.8b, v13.8b\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" + "ldr s5, [%x[weights]], #0x4\n" + "ldr d4, [x28, #0x0]\n" + "smlal v16.4s, v7.4h, v3.h[0]\n" + "smlal v17.4s, v7.4h, v3.h[1]\n" + "ssubl v5.8h, v5.8b, v12.8b\n" + "smlal v18.4s, v7.4h, v3.h[2]\n" + "smlal v19.4s, v7.4h, v3.h[3]\n" + "usubl v4.8h, v4.8b, v13.8b\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "sshl v16.4s, v16.4s, v10.4s\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "sshl v17.4s, v17.4s, v10.4s\n" + "smlal v20.4s, v7.4h, v3.h[4]\n" + "smlal v21.4s, v7.4h, v3.h[5]\n" + "sshl v18.4s, v18.4s, v10.4s\n" + "smlal v22.4s, v7.4h, v3.h[6]\n" + "smlal v23.4s, v7.4h, v3.h[7]\n" + "sshl v19.4s, v19.4s, v10.4s\n" + "smlal v24.4s, v7.4h, v6.h[0]\n" + "smlal v25.4s, v7.4h, v6.h[1]\n" "sqrdmulh v16.4s, v16.4s, v9.4s\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" "sqrdmulh v17.4s, v17.4s, v9.4s\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" "sqrdmulh v18.4s, v18.4s, v9.4s\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" "sqrdmulh v19.4s, v19.4s, v9.4s\n" - "smlal v26.4s, v6.4h, v0.h[2]\n" - "smlal v27.4s, v6.4h, v0.h[3]\n" - "and v5.16b, v16.16b, v10.16b\n" - "smlal v28.4s, v6.4h, v0.h[4]\n" - "smlal v29.4s, v6.4h, v0.h[5]\n" - "and v4.16b, v17.16b, v10.16b\n" - "smlal v30.4s, v6.4h, v0.h[6]\n" - "smlal v31.4s, v6.4h, v0.h[7]\n" - "and v2.16b, v18.16b, v10.16b\n" - "and v1.16b, v19.16b, v10.16b\n" - "sshl v20.4s, v20.4s, v15.4s\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "sshl v21.4s, v21.4s, v15.4s\n" - "sshl v22.4s, v22.4s, v15.4s\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "sshl v23.4s, v23.4s, v15.4s\n" - "sshl v24.4s, v24.4s, v15.4s\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "sshl v25.4s, v25.4s, v15.4s\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" + "smlal v26.4s, v7.4h, v6.h[2]\n" + "smlal v27.4s, v7.4h, v6.h[3]\n" + "and v3.16b, v16.16b, v8.16b\n" + "smlal v28.4s, v7.4h, v6.h[4]\n" + "smlal v29.4s, v7.4h, v6.h[5]\n" + "and v2.16b, v17.16b, v8.16b\n" + "smlal v30.4s, v7.4h, v6.h[6]\n" + "smlal v31.4s, v7.4h, v6.h[7]\n" + "and v1.16b, v18.16b, v8.16b\n" + "and v0.16b, v19.16b, v8.16b\n" + "sshl v20.4s, v20.4s, v10.4s\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "sshl v21.4s, v21.4s, v10.4s\n" + "sshl v22.4s, v22.4s, v10.4s\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "sshl v23.4s, v23.4s, v10.4s\n" + "sshl v24.4s, v24.4s, v10.4s\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "sshl v25.4s, v25.4s, v10.4s\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v0.4s, v0.4s, #0x1f\n" "sqrdmulh v20.4s, v20.4s, v9.4s\n" "sqrdmulh v21.4s, v21.4s, v9.4s\n" "sqrdmulh v22.4s, v22.4s, v9.4s\n" "sqrdmulh v23.4s, v23.4s, v9.4s\n" "sqrdmulh v24.4s, v24.4s, v9.4s\n" "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqadd v16.4s, v16.4s, v5.4s\n" - "sqadd v17.4s, v17.4s, v4.4s\n" - "sqadd v18.4s, v18.4s, v2.4s\n" - "sqadd v19.4s, v19.4s, v1.4s\n" - "and v8.16b, v20.16b, v10.16b\n" - "and v0.16b, v21.16b, v10.16b\n" - "and v5.16b, v22.16b, v10.16b\n" - "and v4.16b, v23.16b, v10.16b\n" - "and v2.16b, v24.16b, v10.16b\n" - "and v1.16b, v25.16b, v10.16b\n" - "sshl v26.4s, v26.4s, v15.4s\n" - "sshl v27.4s, v27.4s, v15.4s\n" - "sshl v28.4s, v28.4s, v15.4s\n" - "sshl v29.4s, v29.4s, v15.4s\n" - "sshl v30.4s, v30.4s, v15.4s\n" - "sshl v31.4s, v31.4s, v15.4s\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v3.4s\n" + "sqadd v17.4s, v17.4s, v2.4s\n" + "sqadd v18.4s, v18.4s, v1.4s\n" + "sqadd v19.4s, v19.4s, v0.4s\n" + "and v5.16b, v20.16b, v8.16b\n" + "and v4.16b, v21.16b, v8.16b\n" + "and v3.16b, v22.16b, v8.16b\n" + "and v2.16b, v23.16b, v8.16b\n" + "and v1.16b, v24.16b, v8.16b\n" + "and v0.16b, v25.16b, v8.16b\n" + "sshl v26.4s, v26.4s, v10.4s\n" + "sshl v27.4s, v27.4s, v10.4s\n" + "sshl v28.4s, v28.4s, v10.4s\n" + "sshl v29.4s, v29.4s, v10.4s\n" + "sshl v30.4s, v30.4s, v10.4s\n" + "sshl v31.4s, v31.4s, v10.4s\n" "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v0.4s, v0.4s, #0x1f\n" "sqrdmulh v26.4s, v26.4s, v9.4s\n" "sqrdmulh v27.4s, v27.4s, v9.4s\n" "sqrdmulh v28.4s, v28.4s, v9.4s\n" "sqrdmulh v29.4s, v29.4s, v9.4s\n" "sqrdmulh v30.4s, v30.4s, v9.4s\n" "sqrdmulh v31.4s, v31.4s, v9.4s\n" - "sqadd v20.4s, v20.4s, v8.4s\n" - "sqadd v21.4s, v21.4s, v0.4s\n" - "sqadd v22.4s, v22.4s, v5.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v24.4s, v24.4s, v2.4s\n" - "sqadd v25.4s, v25.4s, v1.4s\n" - "and v8.16b, v26.16b, v10.16b\n" - "and v0.16b, v27.16b, v10.16b\n" - "and v5.16b, v28.16b, v10.16b\n" - "and v4.16b, v29.16b, v10.16b\n" - "and v2.16b, v30.16b, v10.16b\n" - "and v1.16b, v31.16b, v10.16b\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v5.4s\n" + "sqadd v21.4s, v21.4s, v4.4s\n" + "sqadd v22.4s, v22.4s, v3.4s\n" + "sqadd v23.4s, v23.4s, v2.4s\n" + "sqadd v24.4s, v24.4s, v1.4s\n" + "sqadd v25.4s, v25.4s, v0.4s\n" + "and v5.16b, v26.16b, v8.16b\n" + "and v4.16b, v27.16b, v8.16b\n" + "and v3.16b, v28.16b, v8.16b\n" + "and v2.16b, v29.16b, v8.16b\n" + "and v1.16b, v30.16b, v8.16b\n" + "and v0.16b, v31.16b, v8.16b\n" "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" - "srshl v16.4s, v16.4s, v10.4s\n" - "srshl v17.4s, v17.4s, v10.4s\n" - "srshl v18.4s, v18.4s, v10.4s\n" - "srshl v19.4s, v19.4s, v10.4s\n" - "srshl v20.4s, v20.4s, v10.4s\n" - "srshl v21.4s, v21.4s, v10.4s\n" - "srshl v22.4s, v22.4s, v10.4s\n" - "srshl v23.4s, v23.4s, v10.4s\n" - "sqadd v26.4s, v26.4s, v8.4s\n" - "sqadd v27.4s, v27.4s, v0.4s\n" - "sqadd v28.4s, v28.4s, v5.4s\n" - "sqadd v29.4s, v29.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v2.4s\n" - "sqadd v31.4s, v31.4s, v1.4s\n" - "add v16.4s, v16.4s, v14.4s\n" - "add v17.4s, v17.4s, v14.4s\n" - "add v18.4s, v18.4s, v14.4s\n" - "add v19.4s, v19.4s, v14.4s\n" - "add v20.4s, v20.4s, v14.4s\n" - "add v21.4s, v21.4s, v14.4s\n" - "add v22.4s, v22.4s, v14.4s\n" - "add v23.4s, v23.4s, v14.4s\n" - "srshl v24.4s, v24.4s, v10.4s\n" - "srshl v25.4s, v25.4s, v10.4s\n" - "srshl v26.4s, v26.4s, v10.4s\n" - "srshl v27.4s, v27.4s, v10.4s\n" - "srshl v28.4s, v28.4s, v10.4s\n" - "srshl v29.4s, v29.4s, v10.4s\n" - "srshl v30.4s, v30.4s, v10.4s\n" - "srshl v31.4s, v31.4s, v10.4s\n" - "smin v16.4s, v16.4s, v11.4s\n" - "smin v17.4s, v17.4s, v11.4s\n" - "smin v18.4s, v18.4s, v11.4s\n" - "smin v19.4s, v19.4s, v11.4s\n" - "smin v20.4s, v20.4s, v11.4s\n" - "smin v21.4s, v21.4s, v11.4s\n" - "smin v22.4s, v22.4s, v11.4s\n" - "smin v23.4s, v23.4s, v11.4s\n" - "add v24.4s, v24.4s, v14.4s\n" - "add v25.4s, v25.4s, v14.4s\n" - "add v26.4s, v26.4s, v14.4s\n" - "add v27.4s, v27.4s, v14.4s\n" - "add v28.4s, v28.4s, v14.4s\n" - "add v29.4s, v29.4s, v14.4s\n" - "add v30.4s, v30.4s, v14.4s\n" - "add v31.4s, v31.4s, v14.4s\n" - "smax v16.4s, v16.4s, v13.4s\n" - "smax v17.4s, v17.4s, v13.4s\n" - "smax v18.4s, v18.4s, v13.4s\n" - "smax v19.4s, v19.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smax v21.4s, v21.4s, v13.4s\n" - "smax v22.4s, v22.4s, v13.4s\n" - "smax v23.4s, v23.4s, v13.4s\n" - "smin v24.4s, v24.4s, v11.4s\n" - "smin v25.4s, v25.4s, v11.4s\n" - "smin v26.4s, v26.4s, v11.4s\n" - "smin v27.4s, v27.4s, v11.4s\n" - "smin v28.4s, v28.4s, v11.4s\n" - "smin v29.4s, v29.4s, v11.4s\n" - "smin v30.4s, v30.4s, v11.4s\n" - "smin v31.4s, v31.4s, v11.4s\n" + "sshr v0.4s, v0.4s, #0x1f\n" + "srshl v16.4s, v16.4s, v8.4s\n" + "srshl v17.4s, v17.4s, v8.4s\n" + "srshl v18.4s, v18.4s, v8.4s\n" + "srshl v19.4s, v19.4s, v8.4s\n" + "srshl v20.4s, v20.4s, v8.4s\n" + "srshl v21.4s, v21.4s, v8.4s\n" + "srshl v22.4s, v22.4s, v8.4s\n" + "srshl v23.4s, v23.4s, v8.4s\n" + "sqadd v26.4s, v26.4s, v5.4s\n" + "sqadd v27.4s, v27.4s, v4.4s\n" + "sqadd v28.4s, v28.4s, v3.4s\n" + "sqadd v29.4s, v29.4s, v2.4s\n" + "sqadd v30.4s, v30.4s, v1.4s\n" + "sqadd v31.4s, v31.4s, v0.4s\n" + "add v16.4s, v16.4s, v11.4s\n" + "add v17.4s, v17.4s, v11.4s\n" + "add v18.4s, v18.4s, v11.4s\n" + "add v19.4s, v19.4s, v11.4s\n" + "add v20.4s, v20.4s, v11.4s\n" + "add v21.4s, v21.4s, v11.4s\n" + "add v22.4s, v22.4s, v11.4s\n" + "add v23.4s, v23.4s, v11.4s\n" + "srshl v24.4s, v24.4s, v8.4s\n" + "srshl v25.4s, v25.4s, v8.4s\n" + "srshl v26.4s, v26.4s, v8.4s\n" + "srshl v27.4s, v27.4s, v8.4s\n" + "srshl v28.4s, v28.4s, v8.4s\n" + "srshl v29.4s, v29.4s, v8.4s\n" + "srshl v30.4s, v30.4s, v8.4s\n" + "srshl v31.4s, v31.4s, v8.4s\n" + "smin v16.4s, v16.4s, v14.4s\n" + "smin v17.4s, v17.4s, v14.4s\n" + "smin v18.4s, v18.4s, v14.4s\n" + "smin v19.4s, v19.4s, v14.4s\n" + "smin v20.4s, v20.4s, v14.4s\n" + "smin v21.4s, v21.4s, v14.4s\n" + "smin v22.4s, v22.4s, v14.4s\n" + "smin v23.4s, v23.4s, v14.4s\n" + "add v24.4s, v24.4s, v11.4s\n" + "add v25.4s, v25.4s, v11.4s\n" + "add v26.4s, v26.4s, v11.4s\n" + "add v27.4s, v27.4s, v11.4s\n" + "add v28.4s, v28.4s, v11.4s\n" + "add v29.4s, v29.4s, v11.4s\n" + "add v30.4s, v30.4s, v11.4s\n" + "add v31.4s, v31.4s, v11.4s\n" + "smax v16.4s, v16.4s, v15.4s\n" + "smax v17.4s, v17.4s, v15.4s\n" + "smax v18.4s, v18.4s, v15.4s\n" + "smax v19.4s, v19.4s, v15.4s\n" + "smax v20.4s, v20.4s, v15.4s\n" + "smax v21.4s, v21.4s, v15.4s\n" + "smax v22.4s, v22.4s, v15.4s\n" + "smax v23.4s, v23.4s, v15.4s\n" + "smin v24.4s, v24.4s, v14.4s\n" + "smin v25.4s, v25.4s, v14.4s\n" + "smin v26.4s, v26.4s, v14.4s\n" + "smin v27.4s, v27.4s, v14.4s\n" + "smin v28.4s, v28.4s, v14.4s\n" + "smin v29.4s, v29.4s, v14.4s\n" + "smin v30.4s, v30.4s, v14.4s\n" + "smin v31.4s, v31.4s, v14.4s\n" "uzp1 v16.16b, v16.16b, v16.16b\n" "uzp1 v17.16b, v17.16b, v17.16b\n" "uzp1 v18.16b, v18.16b, v18.16b\n" @@ -612,224 +613,224 @@ void a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst "uzp1 v21.16b, v21.16b, v21.16b\n" "uzp1 v22.16b, v22.16b, v22.16b\n" "uzp1 v23.16b, v23.16b, v23.16b\n" - "smax v24.4s, v24.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v26.4s, v26.4s, v13.4s\n" - "smax v27.4s, v27.4s, v13.4s\n" - "smax v28.4s, v28.4s, v13.4s\n" - "smax v29.4s, v29.4s, v13.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" - "smax v31.4s, v31.4s, v13.4s\n" + "smax v24.4s, v24.4s, v15.4s\n" + "smax v25.4s, v25.4s, v15.4s\n" + "smax v26.4s, v26.4s, v15.4s\n" + "smax v27.4s, v27.4s, v15.4s\n" + "smax v28.4s, v28.4s, v15.4s\n" + "smax v29.4s, v29.4s, v15.4s\n" + "smax v30.4s, v30.4s, v15.4s\n" + "smax v31.4s, v31.4s, v15.4s\n" "uzp1 v16.16b, v16.16b, v16.16b\n" - "str s16, [x20, x9]\n" - "ldr x20, [%x[outptrs], #0x40]\n" + "str s16, [x27, x9]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "uzp1 v17.16b, v17.16b, v17.16b\n" "uzp1 v18.16b, v18.16b, v18.16b\n" - "str s17, [x21, x9]\n" - "ldr x21, [%x[outptrs], #0x48]\n" + "str s17, [x26, x9]\n" + "ldr x26, [%x[outptrs], #0x48]\n" "uzp1 v19.16b, v19.16b, v19.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s18, [x22, x9]\n" - "ldr x22, [%x[outptrs], #0x50]\n" + "str s18, [x25, x9]\n" + "ldr x25, [%x[outptrs], #0x50]\n" "uzp1 v21.16b, v21.16b, v21.16b\n" "uzp1 v22.16b, v22.16b, v22.16b\n" - "str s19, [x23, x9]\n" - "ldr x23, [%x[outptrs], #0x58]\n" + "str s19, [x24, x9]\n" + "ldr x24, [%x[outptrs], #0x58]\n" "uzp1 v23.16b, v23.16b, v23.16b\n" "uzp1 v24.16b, v24.16b, v24.16b\n" - "str s20, [x24, x9]\n" - "ldr x24, [%x[outptrs], #0x60]\n" + "str s20, [x23, x9]\n" + "ldr x23, [%x[outptrs], #0x60]\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s21, [x25, x9]\n" - "ldr x25, [%x[outptrs], #0x68]\n" + "str s21, [x22, x9]\n" + "ldr x22, [%x[outptrs], #0x68]\n" "uzp1 v27.16b, v27.16b, v27.16b\n" "uzp1 v28.16b, v28.16b, v28.16b\n" - "str s22, [x26, x9]\n" - "ldr x26, [%x[outptrs], #0x70]\n" + "str s22, [x21, x9]\n" + "ldr x21, [%x[outptrs], #0x70]\n" "uzp1 v29.16b, v29.16b, v29.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s23, [x27, x9]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "str s23, [x20, x9]\n" + "ldr x20, [%x[outptrs], #0x78]\n" "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v24.16b, v24.16b, v24.16b\n" - "str s24, [x20, x9]\n" + "str s24, [x27, x9]\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s25, [x21, x9]\n" + "str s25, [x26, x9]\n" "uzp1 v27.16b, v27.16b, v27.16b\n" "uzp1 v28.16b, v28.16b, v28.16b\n" - "str s26, [x22, x9]\n" + "str s26, [x25, x9]\n" "uzp1 v29.16b, v29.16b, v29.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s27, [x23, x9]\n" + "str s27, [x24, x9]\n" "uzp1 v31.16b, v31.16b, v31.16b\n" - "str s28, [x24, x9]\n" - "str s29, [x25, x9]\n" - "str s30, [x26, x9]\n" - "str s31, [x27, x9]\n" + "str s28, [x23, x9]\n" + "str s29, [x22, x9]\n" + "str s30, [x21, x9]\n" + "str s31, [x20, x9]\n" "b 8f\n" "7:" // Output channel loop: Single kernel point - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "sshl v16.4s, v16.4s, v15.4s\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "sshl v17.4s, v17.4s, v15.4s\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "sshl v18.4s, v18.4s, v15.4s\n" - "sshl v19.4s, v19.4s, v15.4s\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "sshl v16.4s, v16.4s, v10.4s\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "sshl v17.4s, v17.4s, v10.4s\n" + "ldr x26, [%x[outptrs], #0x8]\n" + "sshl v18.4s, v18.4s, v10.4s\n" + "sshl v19.4s, v19.4s, v10.4s\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" "sqrdmulh v16.4s, v16.4s, v9.4s\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" "sqrdmulh v17.4s, v17.4s, v9.4s\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" "sqrdmulh v18.4s, v18.4s, v9.4s\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "ldr x25, [%x[outptrs], #0x28]\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "ldr x22, [%x[outptrs], #0x28]\n" "sqrdmulh v19.4s, v19.4s, v9.4s\n" - "and v5.16b, v16.16b, v10.16b\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "and v4.16b, v17.16b, v10.16b\n" - "and v2.16b, v18.16b, v10.16b\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "and v1.16b, v19.16b, v10.16b\n" - "sshl v20.4s, v20.4s, v15.4s\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "sshl v21.4s, v21.4s, v15.4s\n" - "sshl v22.4s, v22.4s, v15.4s\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "sshl v23.4s, v23.4s, v15.4s\n" - "sshl v24.4s, v24.4s, v15.4s\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" - "sshl v25.4s, v25.4s, v15.4s\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" + "and v3.16b, v16.16b, v8.16b\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "and v2.16b, v17.16b, v8.16b\n" + "and v1.16b, v18.16b, v8.16b\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "and v0.16b, v19.16b, v8.16b\n" + "sshl v20.4s, v20.4s, v10.4s\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "sshl v21.4s, v21.4s, v10.4s\n" + "sshl v22.4s, v22.4s, v10.4s\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "sshl v23.4s, v23.4s, v10.4s\n" + "sshl v24.4s, v24.4s, v10.4s\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" + "sshl v25.4s, v25.4s, v10.4s\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v0.4s, v0.4s, #0x1f\n" "sqrdmulh v20.4s, v20.4s, v9.4s\n" "sqrdmulh v21.4s, v21.4s, v9.4s\n" "sqrdmulh v22.4s, v22.4s, v9.4s\n" "sqrdmulh v23.4s, v23.4s, v9.4s\n" "sqrdmulh v24.4s, v24.4s, v9.4s\n" "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqadd v16.4s, v16.4s, v5.4s\n" - "sqadd v17.4s, v17.4s, v4.4s\n" - "sqadd v18.4s, v18.4s, v2.4s\n" - "sqadd v19.4s, v19.4s, v1.4s\n" - "and v8.16b, v20.16b, v10.16b\n" - "and v0.16b, v21.16b, v10.16b\n" - "and v5.16b, v22.16b, v10.16b\n" - "and v4.16b, v23.16b, v10.16b\n" - "and v2.16b, v24.16b, v10.16b\n" - "and v1.16b, v25.16b, v10.16b\n" - "sshl v26.4s, v26.4s, v15.4s\n" - "sshl v27.4s, v27.4s, v15.4s\n" - "sshl v28.4s, v28.4s, v15.4s\n" - "sshl v29.4s, v29.4s, v15.4s\n" - "sshl v30.4s, v30.4s, v15.4s\n" - "sshl v31.4s, v31.4s, v15.4s\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v3.4s\n" + "sqadd v17.4s, v17.4s, v2.4s\n" + "sqadd v18.4s, v18.4s, v1.4s\n" + "sqadd v19.4s, v19.4s, v0.4s\n" + "and v5.16b, v20.16b, v8.16b\n" + "and v4.16b, v21.16b, v8.16b\n" + "and v3.16b, v22.16b, v8.16b\n" + "and v2.16b, v23.16b, v8.16b\n" + "and v1.16b, v24.16b, v8.16b\n" + "and v0.16b, v25.16b, v8.16b\n" + "sshl v26.4s, v26.4s, v10.4s\n" + "sshl v27.4s, v27.4s, v10.4s\n" + "sshl v28.4s, v28.4s, v10.4s\n" + "sshl v29.4s, v29.4s, v10.4s\n" + "sshl v30.4s, v30.4s, v10.4s\n" + "sshl v31.4s, v31.4s, v10.4s\n" "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v0.4s, v0.4s, #0x1f\n" "sqrdmulh v26.4s, v26.4s, v9.4s\n" "sqrdmulh v27.4s, v27.4s, v9.4s\n" "sqrdmulh v28.4s, v28.4s, v9.4s\n" "sqrdmulh v29.4s, v29.4s, v9.4s\n" "sqrdmulh v30.4s, v30.4s, v9.4s\n" "sqrdmulh v31.4s, v31.4s, v9.4s\n" - "sqadd v20.4s, v20.4s, v8.4s\n" - "sqadd v21.4s, v21.4s, v0.4s\n" - "sqadd v22.4s, v22.4s, v5.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v24.4s, v24.4s, v2.4s\n" - "sqadd v25.4s, v25.4s, v1.4s\n" - "and v8.16b, v26.16b, v10.16b\n" - "and v0.16b, v27.16b, v10.16b\n" - "and v5.16b, v28.16b, v10.16b\n" - "and v4.16b, v29.16b, v10.16b\n" - "and v2.16b, v30.16b, v10.16b\n" - "and v1.16b, v31.16b, v10.16b\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v5.4s\n" + "sqadd v21.4s, v21.4s, v4.4s\n" + "sqadd v22.4s, v22.4s, v3.4s\n" + "sqadd v23.4s, v23.4s, v2.4s\n" + "sqadd v24.4s, v24.4s, v1.4s\n" + "sqadd v25.4s, v25.4s, v0.4s\n" + "and v5.16b, v26.16b, v8.16b\n" + "and v4.16b, v27.16b, v8.16b\n" + "and v3.16b, v28.16b, v8.16b\n" + "and v2.16b, v29.16b, v8.16b\n" + "and v1.16b, v30.16b, v8.16b\n" + "and v0.16b, v31.16b, v8.16b\n" "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" - "srshl v16.4s, v16.4s, v10.4s\n" - "srshl v17.4s, v17.4s, v10.4s\n" - "srshl v18.4s, v18.4s, v10.4s\n" - "srshl v19.4s, v19.4s, v10.4s\n" - "srshl v20.4s, v20.4s, v10.4s\n" - "srshl v21.4s, v21.4s, v10.4s\n" - "srshl v22.4s, v22.4s, v10.4s\n" - "srshl v23.4s, v23.4s, v10.4s\n" - "sqadd v26.4s, v26.4s, v8.4s\n" - "sqadd v27.4s, v27.4s, v0.4s\n" - "sqadd v28.4s, v28.4s, v5.4s\n" - "sqadd v29.4s, v29.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v2.4s\n" - "sqadd v31.4s, v31.4s, v1.4s\n" - "add v16.4s, v16.4s, v14.4s\n" - "add v17.4s, v17.4s, v14.4s\n" - "add v18.4s, v18.4s, v14.4s\n" - "add v19.4s, v19.4s, v14.4s\n" - "add v20.4s, v20.4s, v14.4s\n" - "add v21.4s, v21.4s, v14.4s\n" - "add v22.4s, v22.4s, v14.4s\n" - "add v23.4s, v23.4s, v14.4s\n" - "srshl v24.4s, v24.4s, v10.4s\n" - "srshl v25.4s, v25.4s, v10.4s\n" - "srshl v26.4s, v26.4s, v10.4s\n" - "srshl v27.4s, v27.4s, v10.4s\n" - "srshl v28.4s, v28.4s, v10.4s\n" - "srshl v29.4s, v29.4s, v10.4s\n" - "srshl v30.4s, v30.4s, v10.4s\n" - "srshl v31.4s, v31.4s, v10.4s\n" - "smin v16.4s, v16.4s, v11.4s\n" - "smin v17.4s, v17.4s, v11.4s\n" - "smin v18.4s, v18.4s, v11.4s\n" - "smin v19.4s, v19.4s, v11.4s\n" - "smin v20.4s, v20.4s, v11.4s\n" - "smin v21.4s, v21.4s, v11.4s\n" - "smin v22.4s, v22.4s, v11.4s\n" - "smin v23.4s, v23.4s, v11.4s\n" - "add v24.4s, v24.4s, v14.4s\n" - "add v25.4s, v25.4s, v14.4s\n" - "add v26.4s, v26.4s, v14.4s\n" - "add v27.4s, v27.4s, v14.4s\n" - "add v28.4s, v28.4s, v14.4s\n" - "add v29.4s, v29.4s, v14.4s\n" - "add v30.4s, v30.4s, v14.4s\n" - "add v31.4s, v31.4s, v14.4s\n" - "smax v16.4s, v16.4s, v13.4s\n" - "smax v17.4s, v17.4s, v13.4s\n" - "smax v18.4s, v18.4s, v13.4s\n" - "smax v19.4s, v19.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smax v21.4s, v21.4s, v13.4s\n" - "smax v22.4s, v22.4s, v13.4s\n" - "smax v23.4s, v23.4s, v13.4s\n" - "smin v24.4s, v24.4s, v11.4s\n" - "smin v25.4s, v25.4s, v11.4s\n" - "smin v26.4s, v26.4s, v11.4s\n" - "smin v27.4s, v27.4s, v11.4s\n" - "smin v28.4s, v28.4s, v11.4s\n" - "smin v29.4s, v29.4s, v11.4s\n" - "smin v30.4s, v30.4s, v11.4s\n" - "smin v31.4s, v31.4s, v11.4s\n" + "sshr v0.4s, v0.4s, #0x1f\n" + "srshl v16.4s, v16.4s, v8.4s\n" + "srshl v17.4s, v17.4s, v8.4s\n" + "srshl v18.4s, v18.4s, v8.4s\n" + "srshl v19.4s, v19.4s, v8.4s\n" + "srshl v20.4s, v20.4s, v8.4s\n" + "srshl v21.4s, v21.4s, v8.4s\n" + "srshl v22.4s, v22.4s, v8.4s\n" + "srshl v23.4s, v23.4s, v8.4s\n" + "sqadd v26.4s, v26.4s, v5.4s\n" + "sqadd v27.4s, v27.4s, v4.4s\n" + "sqadd v28.4s, v28.4s, v3.4s\n" + "sqadd v29.4s, v29.4s, v2.4s\n" + "sqadd v30.4s, v30.4s, v1.4s\n" + "sqadd v31.4s, v31.4s, v0.4s\n" + "add v16.4s, v16.4s, v11.4s\n" + "add v17.4s, v17.4s, v11.4s\n" + "add v18.4s, v18.4s, v11.4s\n" + "add v19.4s, v19.4s, v11.4s\n" + "add v20.4s, v20.4s, v11.4s\n" + "add v21.4s, v21.4s, v11.4s\n" + "add v22.4s, v22.4s, v11.4s\n" + "add v23.4s, v23.4s, v11.4s\n" + "srshl v24.4s, v24.4s, v8.4s\n" + "srshl v25.4s, v25.4s, v8.4s\n" + "srshl v26.4s, v26.4s, v8.4s\n" + "srshl v27.4s, v27.4s, v8.4s\n" + "srshl v28.4s, v28.4s, v8.4s\n" + "srshl v29.4s, v29.4s, v8.4s\n" + "srshl v30.4s, v30.4s, v8.4s\n" + "srshl v31.4s, v31.4s, v8.4s\n" + "smin v16.4s, v16.4s, v14.4s\n" + "smin v17.4s, v17.4s, v14.4s\n" + "smin v18.4s, v18.4s, v14.4s\n" + "smin v19.4s, v19.4s, v14.4s\n" + "smin v20.4s, v20.4s, v14.4s\n" + "smin v21.4s, v21.4s, v14.4s\n" + "smin v22.4s, v22.4s, v14.4s\n" + "smin v23.4s, v23.4s, v14.4s\n" + "add v24.4s, v24.4s, v11.4s\n" + "add v25.4s, v25.4s, v11.4s\n" + "add v26.4s, v26.4s, v11.4s\n" + "add v27.4s, v27.4s, v11.4s\n" + "add v28.4s, v28.4s, v11.4s\n" + "add v29.4s, v29.4s, v11.4s\n" + "add v30.4s, v30.4s, v11.4s\n" + "add v31.4s, v31.4s, v11.4s\n" + "smax v16.4s, v16.4s, v15.4s\n" + "smax v17.4s, v17.4s, v15.4s\n" + "smax v18.4s, v18.4s, v15.4s\n" + "smax v19.4s, v19.4s, v15.4s\n" + "smax v20.4s, v20.4s, v15.4s\n" + "smax v21.4s, v21.4s, v15.4s\n" + "smax v22.4s, v22.4s, v15.4s\n" + "smax v23.4s, v23.4s, v15.4s\n" + "smin v24.4s, v24.4s, v14.4s\n" + "smin v25.4s, v25.4s, v14.4s\n" + "smin v26.4s, v26.4s, v14.4s\n" + "smin v27.4s, v27.4s, v14.4s\n" + "smin v28.4s, v28.4s, v14.4s\n" + "smin v29.4s, v29.4s, v14.4s\n" + "smin v30.4s, v30.4s, v14.4s\n" + "smin v31.4s, v31.4s, v14.4s\n" "uzp1 v16.16b, v16.16b, v16.16b\n" "uzp1 v17.16b, v17.16b, v17.16b\n" "uzp1 v18.16b, v18.16b, v18.16b\n" @@ -838,62 +839,62 @@ void a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst "uzp1 v21.16b, v21.16b, v21.16b\n" "uzp1 v22.16b, v22.16b, v22.16b\n" "uzp1 v23.16b, v23.16b, v23.16b\n" - "smax v24.4s, v24.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v26.4s, v26.4s, v13.4s\n" - "smax v27.4s, v27.4s, v13.4s\n" - "smax v28.4s, v28.4s, v13.4s\n" - "smax v29.4s, v29.4s, v13.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" - "smax v31.4s, v31.4s, v13.4s\n" + "smax v24.4s, v24.4s, v15.4s\n" + "smax v25.4s, v25.4s, v15.4s\n" + "smax v26.4s, v26.4s, v15.4s\n" + "smax v27.4s, v27.4s, v15.4s\n" + "smax v28.4s, v28.4s, v15.4s\n" + "smax v29.4s, v29.4s, v15.4s\n" + "smax v30.4s, v30.4s, v15.4s\n" + "smax v31.4s, v31.4s, v15.4s\n" "uzp1 v16.16b, v16.16b, v16.16b\n" - "str s16, [x20, x9]\n" - "ldr x20, [%x[outptrs], #0x40]\n" + "str s16, [x27, x9]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "uzp1 v17.16b, v17.16b, v17.16b\n" "uzp1 v18.16b, v18.16b, v18.16b\n" - "str s17, [x21, x9]\n" - "ldr x21, [%x[outptrs], #0x48]\n" + "str s17, [x26, x9]\n" + "ldr x26, [%x[outptrs], #0x48]\n" "uzp1 v19.16b, v19.16b, v19.16b\n" "uzp1 v20.16b, v20.16b, v20.16b\n" - "str s18, [x22, x9]\n" - "ldr x22, [%x[outptrs], #0x50]\n" + "str s18, [x25, x9]\n" + "ldr x25, [%x[outptrs], #0x50]\n" "uzp1 v21.16b, v21.16b, v21.16b\n" "uzp1 v22.16b, v22.16b, v22.16b\n" - "str s19, [x23, x9]\n" - "ldr x23, [%x[outptrs], #0x58]\n" + "str s19, [x24, x9]\n" + "ldr x24, [%x[outptrs], #0x58]\n" "uzp1 v23.16b, v23.16b, v23.16b\n" "uzp1 v24.16b, v24.16b, v24.16b\n" - "str s20, [x24, x9]\n" - "ldr x24, [%x[outptrs], #0x60]\n" + "str s20, [x23, x9]\n" + "ldr x23, [%x[outptrs], #0x60]\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s21, [x25, x9]\n" - "ldr x25, [%x[outptrs], #0x68]\n" + "str s21, [x22, x9]\n" + "ldr x22, [%x[outptrs], #0x68]\n" "uzp1 v27.16b, v27.16b, v27.16b\n" "uzp1 v28.16b, v28.16b, v28.16b\n" - "str s22, [x26, x9]\n" - "ldr x26, [%x[outptrs], #0x70]\n" + "str s22, [x21, x9]\n" + "ldr x21, [%x[outptrs], #0x70]\n" "uzp1 v29.16b, v29.16b, v29.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s23, [x27, x9]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "str s23, [x20, x9]\n" + "ldr x20, [%x[outptrs], #0x78]\n" "uzp1 v31.16b, v31.16b, v31.16b\n" "uzp1 v24.16b, v24.16b, v24.16b\n" - "str s24, [x20, x9]\n" + "str s24, [x27, x9]\n" "uzp1 v25.16b, v25.16b, v25.16b\n" "uzp1 v26.16b, v26.16b, v26.16b\n" - "str s25, [x21, x9]\n" + "str s25, [x26, x9]\n" "uzp1 v27.16b, v27.16b, v27.16b\n" "uzp1 v28.16b, v28.16b, v28.16b\n" - "str s26, [x22, x9]\n" + "str s26, [x25, x9]\n" "uzp1 v29.16b, v29.16b, v29.16b\n" "uzp1 v30.16b, v30.16b, v30.16b\n" - "str s27, [x23, x9]\n" + "str s27, [x24, x9]\n" "uzp1 v31.16b, v31.16b, v31.16b\n" - "str s28, [x24, x9]\n" - "str s29, [x25, x9]\n" - "str s30, [x26, x9]\n" - "str s31, [x27, x9]\n" + "str s28, [x23, x9]\n" + "str s29, [x22, x9]\n" + "str s30, [x21, x9]\n" + "str s31, [x20, x9]\n" "8:" // Output channel loop: Done "add x9, x9, #0x4\n" "cmp x9, x10, LSL #2\n" @@ -936,354 +937,354 @@ void a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst "cbz %x[rq_left_shift_ptr], 15f\n" "tbz %x[n_output_channels], #1, 13f\n" "ld1 { v9.d }[0], [x22], #0x8\n" - "ld1 { v10.d }[0], [x21], #0x8\n" - "ld1 { v15.d }[0], [x20], #0x8\n" + "ld1 { v8.d }[0], [x21], #0x8\n" + "ld1 { v10.d }[0], [x20], #0x8\n" "tbz %x[n_output_channels], #0, 14f\n" "ld1 { v9.s }[2], [x22], #0x4\n" - "ld1 { v10.s }[2], [x21], #0x4\n" - "ld1 { v15.s }[2], [x20], #0x4\n" + "ld1 { v8.s }[2], [x21], #0x4\n" + "ld1 { v10.s }[2], [x20], #0x4\n" "b 14f\n" "13:" // Output channel oddments: Load quantization parameters: With left shift: Bit 1: Unset "ld1 { v9.s }[0], [x22], #0x4\n" - "ld1 { v10.s }[0], [x21], #0x4\n" - "ld1 { v15.s }[0], [x20], #0x4\n" + "ld1 { v8.s }[0], [x21], #0x4\n" + "ld1 { v10.s }[0], [x20], #0x4\n" "14:" // Output channel oddments: Load quantization parameters: With left shift: Bit 1: End "b 18f\n" "15:" // Output channel oddments: Load quantization parameters: No left shift "tbz %x[n_output_channels], #1, 16f\n" "ld1 { v9.d }[0], [x22], #0x8\n" - "ld1 { v10.d }[0], [x21], #0x8\n" + "ld1 { v8.d }[0], [x21], #0x8\n" "tbz %x[n_output_channels], #0, 17f\n" "ld1 { v9.s }[2], [x22], #0x4\n" - "ld1 { v10.s }[2], [x21], #0x4\n" + "ld1 { v8.s }[2], [x21], #0x4\n" "b 17f\n" "16:" // Output channel oddments: Load quantization parameters: No left shift: Bit 1: Unset "ld1 { v9.s }[0], [x22], #0x4\n" - "ld1 { v10.s }[0], [x21], #0x4\n" + "ld1 { v8.s }[0], [x21], #0x4\n" "17:" // Output channel oddments: Load quantization parameters: No left shift: Bit 1: End "18:" // Output channel oddments: Load quantization parameters: Done - "ldr s8, [%x[weights]], #0x4\n" - "mov x20, %x[inptrs]\n" - "ldp x25, x28, [x20], #0x10\n" - "lsr x21, %x[kernel_points], #0x1\n" - "ldr d2, [x25, #0x0]\n" - "ldr d7, [x28, #0x0]\n" - "usubl v2.8h, v2.8b, v3.8b\n" - "usubl v7.8h, v7.8b, v3.8b\n" - "ssubl v8.8h, v8.8b, v12.8b\n" - "cbz x21, 22f\n" - "ldr s6, [%x[weights]], #0x4\n" - "ldp x25, x28, [x20], #0x10\n" - "subs x21, x21, #0x1\n" - "ssubl v6.8h, v6.8b, v12.8b\n" - "ldr d1, [x25, #0x0]\n" - "ldr d0, [x28, #0x0]\n" - "usubl v1.8h, v1.8b, v3.8b\n" - "usubl v0.8h, v0.8b, v3.8b\n" + "ldr s5, [%x[weights]], #0x4\n" + "mov x22, %x[inptrs]\n" + "ldp x21, x20, [x22], #0x10\n" + "lsr x23, %x[kernel_points], #0x1\n" + "ldr d0, [x21, #0x0]\n" + "ldr d4, [x20, #0x0]\n" + "usubl v0.8h, v0.8b, v13.8b\n" + "usubl v4.8h, v4.8b, v13.8b\n" + "ssubl v5.8h, v5.8b, v12.8b\n" + "cbz x23, 22f\n" + "ldr s7, [%x[weights]], #0x4\n" + "ldp x21, x20, [x22], #0x10\n" + "subs x23, x23, #0x1\n" + "ssubl v7.8h, v7.8b, v12.8b\n" + "ldr d3, [x21, #0x0]\n" + "ldr d6, [x20, #0x0]\n" + "usubl v3.8h, v3.8b, v13.8b\n" + "usubl v6.8h, v6.8b, v13.8b\n" "beq 20f\n" "19:" // Output channel oddments: Kernel loop - "ldp x25, x28, [x20], #0x10\n" - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "subs x21, x21, #0x1\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "ldr d2, [x25, #0x0]\n" - "usubl v2.8h, v2.8b, v3.8b\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" - "ldr d7, [x28, #0x0]\n" - "ldr s8, [%x[weights]], #0x4\n" - "ldp x25, x28, [x20], #0x10\n" - "smlal v16.4s, v6.4h, v1.h[0]\n" - "smlal v17.4s, v6.4h, v1.h[1]\n" - "usubl v7.8h, v7.8b, v3.8b\n" - "smlal v18.4s, v6.4h, v1.h[2]\n" - "smlal v19.4s, v6.4h, v1.h[3]\n" - "ssubl v8.8h, v8.8b, v12.8b\n" - "smlal v20.4s, v6.4h, v1.h[4]\n" - "smlal v21.4s, v6.4h, v1.h[5]\n" - "smlal v22.4s, v6.4h, v1.h[6]\n" - "smlal v23.4s, v6.4h, v1.h[7]\n" - "ldr d1, [x25, #0x0]\n" - "usubl v1.8h, v1.8b, v3.8b\n" - "smlal v24.4s, v6.4h, v0.h[0]\n" - "smlal v25.4s, v6.4h, v0.h[1]\n" - "smlal v26.4s, v6.4h, v0.h[2]\n" - "smlal v27.4s, v6.4h, v0.h[3]\n" - "smlal v28.4s, v6.4h, v0.h[4]\n" - "smlal v29.4s, v6.4h, v0.h[5]\n" - "smlal v30.4s, v6.4h, v0.h[6]\n" - "smlal v31.4s, v6.4h, v0.h[7]\n" - "ldr d0, [x28, #0x0]\n" - "ldr s6, [%x[weights]], #0x4\n" - "usubl v0.8h, v0.8b, v3.8b\n" - "ssubl v6.8h, v6.8b, v12.8b\n" + "ldp x21, x20, [x22], #0x10\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "subs x23, x23, #0x1\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" + "ldr d0, [x21, #0x0]\n" + "usubl v0.8h, v0.8b, v13.8b\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" + "ldr d4, [x20, #0x0]\n" + "ldr s5, [%x[weights]], #0x4\n" + "ldp x21, x20, [x22], #0x10\n" + "smlal v16.4s, v7.4h, v3.h[0]\n" + "smlal v17.4s, v7.4h, v3.h[1]\n" + "usubl v4.8h, v4.8b, v13.8b\n" + "smlal v18.4s, v7.4h, v3.h[2]\n" + "smlal v19.4s, v7.4h, v3.h[3]\n" + "ssubl v5.8h, v5.8b, v12.8b\n" + "smlal v20.4s, v7.4h, v3.h[4]\n" + "smlal v21.4s, v7.4h, v3.h[5]\n" + "smlal v22.4s, v7.4h, v3.h[6]\n" + "smlal v23.4s, v7.4h, v3.h[7]\n" + "ldr d3, [x21, #0x0]\n" + "usubl v3.8h, v3.8b, v13.8b\n" + "smlal v24.4s, v7.4h, v6.h[0]\n" + "smlal v25.4s, v7.4h, v6.h[1]\n" + "smlal v26.4s, v7.4h, v6.h[2]\n" + "smlal v27.4s, v7.4h, v6.h[3]\n" + "smlal v28.4s, v7.4h, v6.h[4]\n" + "smlal v29.4s, v7.4h, v6.h[5]\n" + "smlal v30.4s, v7.4h, v6.h[6]\n" + "smlal v31.4s, v7.4h, v6.h[7]\n" + "ldr d6, [x20, #0x0]\n" + "ldr s7, [%x[weights]], #0x4\n" + "usubl v6.8h, v6.8b, v13.8b\n" + "ssubl v7.8h, v7.8b, v12.8b\n" "bgt 19b\n" "20:" // Output channel oddments: Kernel loop tail "tbnz %x[kernel_points], #0, 21f\n" - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" - "smlal v16.4s, v6.4h, v1.h[0]\n" - "smlal v17.4s, v6.4h, v1.h[1]\n" - "smlal v18.4s, v6.4h, v1.h[2]\n" - "smlal v19.4s, v6.4h, v1.h[3]\n" - "smlal v20.4s, v6.4h, v1.h[4]\n" - "smlal v21.4s, v6.4h, v1.h[5]\n" - "smlal v22.4s, v6.4h, v1.h[6]\n" - "smlal v23.4s, v6.4h, v1.h[7]\n" - "smlal v24.4s, v6.4h, v0.h[0]\n" - "smlal v25.4s, v6.4h, v0.h[1]\n" - "smlal v26.4s, v6.4h, v0.h[2]\n" - "smlal v27.4s, v6.4h, v0.h[3]\n" - "smlal v28.4s, v6.4h, v0.h[4]\n" - "smlal v29.4s, v6.4h, v0.h[5]\n" - "smlal v30.4s, v6.4h, v0.h[6]\n" - "smlal v31.4s, v6.4h, v0.h[7]\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" + "smlal v16.4s, v7.4h, v3.h[0]\n" + "smlal v17.4s, v7.4h, v3.h[1]\n" + "smlal v18.4s, v7.4h, v3.h[2]\n" + "smlal v19.4s, v7.4h, v3.h[3]\n" + "smlal v20.4s, v7.4h, v3.h[4]\n" + "smlal v21.4s, v7.4h, v3.h[5]\n" + "smlal v22.4s, v7.4h, v3.h[6]\n" + "smlal v23.4s, v7.4h, v3.h[7]\n" + "smlal v24.4s, v7.4h, v6.h[0]\n" + "smlal v25.4s, v7.4h, v6.h[1]\n" + "smlal v26.4s, v7.4h, v6.h[2]\n" + "smlal v27.4s, v7.4h, v6.h[3]\n" + "smlal v28.4s, v7.4h, v6.h[4]\n" + "smlal v29.4s, v7.4h, v6.h[5]\n" + "smlal v30.4s, v7.4h, v6.h[6]\n" + "smlal v31.4s, v7.4h, v6.h[7]\n" "b 23f\n" "21:" // Output channel oddments: Odd tail - "ldp x25, x28, [x20], #0x10\n" - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "ldr d2, [x25, #0x0]\n" - "usubl v2.8h, v2.8b, v3.8b\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" - "ldr d7, [x28, #0x0]\n" - "ldr s8, [%x[weights]], #0x4\n" - "smlal v16.4s, v6.4h, v1.h[0]\n" - "smlal v17.4s, v6.4h, v1.h[1]\n" - "usubl v7.8h, v7.8b, v3.8b\n" - "smlal v18.4s, v6.4h, v1.h[2]\n" - "smlal v19.4s, v6.4h, v1.h[3]\n" - "ssubl v8.8h, v8.8b, v12.8b\n" - "smlal v20.4s, v6.4h, v1.h[4]\n" - "smlal v21.4s, v6.4h, v1.h[5]\n" - "smlal v22.4s, v6.4h, v1.h[6]\n" - "smlal v23.4s, v6.4h, v1.h[7]\n" - "smlal v24.4s, v6.4h, v0.h[0]\n" - "smlal v25.4s, v6.4h, v0.h[1]\n" - "smlal v26.4s, v6.4h, v0.h[2]\n" - "smlal v27.4s, v6.4h, v0.h[3]\n" - "smlal v28.4s, v6.4h, v0.h[4]\n" - "smlal v29.4s, v6.4h, v0.h[5]\n" - "smlal v30.4s, v6.4h, v0.h[6]\n" - "smlal v31.4s, v6.4h, v0.h[7]\n" - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" + "ldp x21, x20, [x22], #0x10\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" + "ldr d2, [x21, #0x0]\n" + "usubl v2.8h, v2.8b, v13.8b\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" + "ldr d1, [x20, #0x0]\n" + "ldr s0, [%x[weights]], #0x4\n" + "smlal v16.4s, v7.4h, v3.h[0]\n" + "smlal v17.4s, v7.4h, v3.h[1]\n" + "usubl v1.8h, v1.8b, v13.8b\n" + "smlal v18.4s, v7.4h, v3.h[2]\n" + "smlal v19.4s, v7.4h, v3.h[3]\n" + "ssubl v0.8h, v0.8b, v12.8b\n" + "smlal v20.4s, v7.4h, v3.h[4]\n" + "smlal v21.4s, v7.4h, v3.h[5]\n" + "smlal v22.4s, v7.4h, v3.h[6]\n" + "smlal v23.4s, v7.4h, v3.h[7]\n" + "smlal v24.4s, v7.4h, v6.h[0]\n" + "smlal v25.4s, v7.4h, v6.h[1]\n" + "smlal v26.4s, v7.4h, v6.h[2]\n" + "smlal v27.4s, v7.4h, v6.h[3]\n" + "smlal v28.4s, v7.4h, v6.h[4]\n" + "smlal v29.4s, v7.4h, v6.h[5]\n" + "smlal v30.4s, v7.4h, v6.h[6]\n" + "smlal v31.4s, v7.4h, v6.h[7]\n" + "smlal v16.4s, v0.4h, v2.h[0]\n" + "smlal v17.4s, v0.4h, v2.h[1]\n" + "smlal v18.4s, v0.4h, v2.h[2]\n" + "smlal v19.4s, v0.4h, v2.h[3]\n" + "smlal v20.4s, v0.4h, v2.h[4]\n" + "smlal v21.4s, v0.4h, v2.h[5]\n" + "smlal v22.4s, v0.4h, v2.h[6]\n" + "smlal v23.4s, v0.4h, v2.h[7]\n" + "smlal v24.4s, v0.4h, v1.h[0]\n" + "smlal v25.4s, v0.4h, v1.h[1]\n" + "smlal v26.4s, v0.4h, v1.h[2]\n" + "smlal v27.4s, v0.4h, v1.h[3]\n" + "smlal v28.4s, v0.4h, v1.h[4]\n" + "smlal v29.4s, v0.4h, v1.h[5]\n" + "smlal v30.4s, v0.4h, v1.h[6]\n" + "smlal v31.4s, v0.4h, v1.h[7]\n" "b 23f\n" "22:" // Output channel oddments: Single kernel point - "smlal v16.4s, v8.4h, v2.h[0]\n" - "smlal v17.4s, v8.4h, v2.h[1]\n" - "smlal v18.4s, v8.4h, v2.h[2]\n" - "smlal v19.4s, v8.4h, v2.h[3]\n" - "smlal v20.4s, v8.4h, v2.h[4]\n" - "smlal v21.4s, v8.4h, v2.h[5]\n" - "smlal v22.4s, v8.4h, v2.h[6]\n" - "smlal v23.4s, v8.4h, v2.h[7]\n" - "smlal v24.4s, v8.4h, v7.h[0]\n" - "smlal v25.4s, v8.4h, v7.h[1]\n" - "smlal v26.4s, v8.4h, v7.h[2]\n" - "smlal v27.4s, v8.4h, v7.h[3]\n" - "smlal v28.4s, v8.4h, v7.h[4]\n" - "smlal v29.4s, v8.4h, v7.h[5]\n" - "smlal v30.4s, v8.4h, v7.h[6]\n" - "smlal v31.4s, v8.4h, v7.h[7]\n" + "smlal v16.4s, v5.4h, v0.h[0]\n" + "smlal v17.4s, v5.4h, v0.h[1]\n" + "smlal v18.4s, v5.4h, v0.h[2]\n" + "smlal v19.4s, v5.4h, v0.h[3]\n" + "smlal v20.4s, v5.4h, v0.h[4]\n" + "smlal v21.4s, v5.4h, v0.h[5]\n" + "smlal v22.4s, v5.4h, v0.h[6]\n" + "smlal v23.4s, v5.4h, v0.h[7]\n" + "smlal v24.4s, v5.4h, v4.h[0]\n" + "smlal v25.4s, v5.4h, v4.h[1]\n" + "smlal v26.4s, v5.4h, v4.h[2]\n" + "smlal v27.4s, v5.4h, v4.h[3]\n" + "smlal v28.4s, v5.4h, v4.h[4]\n" + "smlal v29.4s, v5.4h, v4.h[5]\n" + "smlal v30.4s, v5.4h, v4.h[6]\n" + "smlal v31.4s, v5.4h, v4.h[7]\n" "23:" // Output channel oddments: Done - "sshl v16.4s, v16.4s, v15.4s\n" - "sshl v17.4s, v17.4s, v15.4s\n" - "sshl v18.4s, v18.4s, v15.4s\n" - "sshl v19.4s, v19.4s, v15.4s\n" + "sshl v16.4s, v16.4s, v10.4s\n" + "sshl v17.4s, v17.4s, v10.4s\n" + "sshl v18.4s, v18.4s, v10.4s\n" + "sshl v19.4s, v19.4s, v10.4s\n" "sqrdmulh v16.4s, v16.4s, v9.4s\n" "sqrdmulh v17.4s, v17.4s, v9.4s\n" "sqrdmulh v18.4s, v18.4s, v9.4s\n" "sqrdmulh v19.4s, v19.4s, v9.4s\n" - "and v5.16b, v16.16b, v10.16b\n" - "and v4.16b, v17.16b, v10.16b\n" - "and v2.16b, v18.16b, v10.16b\n" - "and v1.16b, v19.16b, v10.16b\n" - "sshl v20.4s, v20.4s, v15.4s\n" - "sshl v21.4s, v21.4s, v15.4s\n" - "sshl v22.4s, v22.4s, v15.4s\n" - "sshl v23.4s, v23.4s, v15.4s\n" - "sshl v24.4s, v24.4s, v15.4s\n" - "sshl v25.4s, v25.4s, v15.4s\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" + "and v3.16b, v16.16b, v8.16b\n" + "and v2.16b, v17.16b, v8.16b\n" + "and v1.16b, v18.16b, v8.16b\n" + "and v0.16b, v19.16b, v8.16b\n" + "sshl v20.4s, v20.4s, v10.4s\n" + "sshl v21.4s, v21.4s, v10.4s\n" + "sshl v22.4s, v22.4s, v10.4s\n" + "sshl v23.4s, v23.4s, v10.4s\n" + "sshl v24.4s, v24.4s, v10.4s\n" + "sshl v25.4s, v25.4s, v10.4s\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v0.4s, v0.4s, #0x1f\n" "sqrdmulh v20.4s, v20.4s, v9.4s\n" "sqrdmulh v21.4s, v21.4s, v9.4s\n" "sqrdmulh v22.4s, v22.4s, v9.4s\n" "sqrdmulh v23.4s, v23.4s, v9.4s\n" "sqrdmulh v24.4s, v24.4s, v9.4s\n" "sqrdmulh v25.4s, v25.4s, v9.4s\n" - "sqadd v16.4s, v16.4s, v5.4s\n" - "sqadd v17.4s, v17.4s, v4.4s\n" - "sqadd v18.4s, v18.4s, v2.4s\n" - "sqadd v19.4s, v19.4s, v1.4s\n" - "and v8.16b, v20.16b, v10.16b\n" - "and v0.16b, v21.16b, v10.16b\n" - "and v5.16b, v22.16b, v10.16b\n" - "and v4.16b, v23.16b, v10.16b\n" - "and v2.16b, v24.16b, v10.16b\n" - "and v1.16b, v25.16b, v10.16b\n" - "sshl v26.4s, v26.4s, v15.4s\n" - "sshl v27.4s, v27.4s, v15.4s\n" - "sshl v28.4s, v28.4s, v15.4s\n" - "sshl v29.4s, v29.4s, v15.4s\n" - "sshl v30.4s, v30.4s, v15.4s\n" - "sshl v31.4s, v31.4s, v15.4s\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v3.4s\n" + "sqadd v17.4s, v17.4s, v2.4s\n" + "sqadd v18.4s, v18.4s, v1.4s\n" + "sqadd v19.4s, v19.4s, v0.4s\n" + "and v5.16b, v20.16b, v8.16b\n" + "and v4.16b, v21.16b, v8.16b\n" + "and v3.16b, v22.16b, v8.16b\n" + "and v2.16b, v23.16b, v8.16b\n" + "and v1.16b, v24.16b, v8.16b\n" + "and v0.16b, v25.16b, v8.16b\n" + "sshl v26.4s, v26.4s, v10.4s\n" + "sshl v27.4s, v27.4s, v10.4s\n" + "sshl v28.4s, v28.4s, v10.4s\n" + "sshl v29.4s, v29.4s, v10.4s\n" + "sshl v30.4s, v30.4s, v10.4s\n" + "sshl v31.4s, v31.4s, v10.4s\n" "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v0.4s, v0.4s, #0x1f\n" "sqrdmulh v26.4s, v26.4s, v9.4s\n" "sqrdmulh v27.4s, v27.4s, v9.4s\n" "sqrdmulh v28.4s, v28.4s, v9.4s\n" "sqrdmulh v29.4s, v29.4s, v9.4s\n" "sqrdmulh v30.4s, v30.4s, v9.4s\n" "sqrdmulh v31.4s, v31.4s, v9.4s\n" - "sqadd v20.4s, v20.4s, v8.4s\n" - "sqadd v21.4s, v21.4s, v0.4s\n" - "sqadd v22.4s, v22.4s, v5.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v24.4s, v24.4s, v2.4s\n" - "sqadd v25.4s, v25.4s, v1.4s\n" - "and v8.16b, v26.16b, v10.16b\n" - "and v0.16b, v27.16b, v10.16b\n" - "and v5.16b, v28.16b, v10.16b\n" - "and v4.16b, v29.16b, v10.16b\n" - "and v2.16b, v30.16b, v10.16b\n" - "and v1.16b, v31.16b, v10.16b\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v5.4s\n" + "sqadd v21.4s, v21.4s, v4.4s\n" + "sqadd v22.4s, v22.4s, v3.4s\n" + "sqadd v23.4s, v23.4s, v2.4s\n" + "sqadd v24.4s, v24.4s, v1.4s\n" + "sqadd v25.4s, v25.4s, v0.4s\n" + "and v5.16b, v26.16b, v8.16b\n" + "and v4.16b, v27.16b, v8.16b\n" + "and v3.16b, v28.16b, v8.16b\n" + "and v2.16b, v29.16b, v8.16b\n" + "and v1.16b, v30.16b, v8.16b\n" + "and v0.16b, v31.16b, v8.16b\n" "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" "sshr v2.4s, v2.4s, #0x1f\n" "sshr v1.4s, v1.4s, #0x1f\n" - "sqadd v26.4s, v26.4s, v8.4s\n" - "sqadd v27.4s, v27.4s, v0.4s\n" - "sqadd v28.4s, v28.4s, v5.4s\n" - "sqadd v29.4s, v29.4s, v4.4s\n" - "sqadd v30.4s, v30.4s, v2.4s\n" - "sqadd v31.4s, v31.4s, v1.4s\n" - "srshl v16.4s, v16.4s, v10.4s\n" - "srshl v17.4s, v17.4s, v10.4s\n" - "srshl v18.4s, v18.4s, v10.4s\n" - "srshl v19.4s, v19.4s, v10.4s\n" - "srshl v20.4s, v20.4s, v10.4s\n" - "srshl v21.4s, v21.4s, v10.4s\n" - "srshl v22.4s, v22.4s, v10.4s\n" - "srshl v23.4s, v23.4s, v10.4s\n" - "srshl v24.4s, v24.4s, v10.4s\n" - "srshl v25.4s, v25.4s, v10.4s\n" - "srshl v26.4s, v26.4s, v10.4s\n" - "srshl v27.4s, v27.4s, v10.4s\n" - "srshl v28.4s, v28.4s, v10.4s\n" - "srshl v29.4s, v29.4s, v10.4s\n" - "srshl v30.4s, v30.4s, v10.4s\n" - "srshl v31.4s, v31.4s, v10.4s\n" - "add v16.4s, v16.4s, v14.4s\n" - "add v17.4s, v17.4s, v14.4s\n" - "add v18.4s, v18.4s, v14.4s\n" - "add v19.4s, v19.4s, v14.4s\n" - "add v20.4s, v20.4s, v14.4s\n" - "add v21.4s, v21.4s, v14.4s\n" - "add v22.4s, v22.4s, v14.4s\n" - "add v23.4s, v23.4s, v14.4s\n" - "add v24.4s, v24.4s, v14.4s\n" - "add v25.4s, v25.4s, v14.4s\n" - "add v26.4s, v26.4s, v14.4s\n" - "add v27.4s, v27.4s, v14.4s\n" - "add v28.4s, v28.4s, v14.4s\n" - "add v29.4s, v29.4s, v14.4s\n" - "add v30.4s, v30.4s, v14.4s\n" - "add v31.4s, v31.4s, v14.4s\n" - "smin v16.4s, v16.4s, v11.4s\n" - "smin v17.4s, v17.4s, v11.4s\n" - "smin v18.4s, v18.4s, v11.4s\n" - "smin v19.4s, v19.4s, v11.4s\n" - "smin v20.4s, v20.4s, v11.4s\n" - "smin v21.4s, v21.4s, v11.4s\n" - "smin v22.4s, v22.4s, v11.4s\n" - "smin v23.4s, v23.4s, v11.4s\n" - "smin v24.4s, v24.4s, v11.4s\n" - "smin v25.4s, v25.4s, v11.4s\n" - "smin v26.4s, v26.4s, v11.4s\n" - "smin v27.4s, v27.4s, v11.4s\n" - "smin v28.4s, v28.4s, v11.4s\n" - "smin v29.4s, v29.4s, v11.4s\n" - "smin v30.4s, v30.4s, v11.4s\n" - "smin v31.4s, v31.4s, v11.4s\n" - "smax v16.4s, v16.4s, v13.4s\n" - "smax v17.4s, v17.4s, v13.4s\n" - "smax v18.4s, v18.4s, v13.4s\n" - "smax v19.4s, v19.4s, v13.4s\n" - "smax v20.4s, v20.4s, v13.4s\n" - "smax v21.4s, v21.4s, v13.4s\n" - "smax v22.4s, v22.4s, v13.4s\n" - "smax v23.4s, v23.4s, v13.4s\n" - "smax v24.4s, v24.4s, v13.4s\n" - "smax v25.4s, v25.4s, v13.4s\n" - "smax v26.4s, v26.4s, v13.4s\n" - "smax v27.4s, v27.4s, v13.4s\n" - "smax v28.4s, v28.4s, v13.4s\n" - "smax v29.4s, v29.4s, v13.4s\n" - "smax v30.4s, v30.4s, v13.4s\n" - "smax v31.4s, v31.4s, v13.4s\n" + "sshr v0.4s, v0.4s, #0x1f\n" + "sqadd v26.4s, v26.4s, v5.4s\n" + "sqadd v27.4s, v27.4s, v4.4s\n" + "sqadd v28.4s, v28.4s, v3.4s\n" + "sqadd v29.4s, v29.4s, v2.4s\n" + "sqadd v30.4s, v30.4s, v1.4s\n" + "sqadd v31.4s, v31.4s, v0.4s\n" + "srshl v16.4s, v16.4s, v8.4s\n" + "srshl v17.4s, v17.4s, v8.4s\n" + "srshl v18.4s, v18.4s, v8.4s\n" + "srshl v19.4s, v19.4s, v8.4s\n" + "srshl v20.4s, v20.4s, v8.4s\n" + "srshl v21.4s, v21.4s, v8.4s\n" + "srshl v22.4s, v22.4s, v8.4s\n" + "srshl v23.4s, v23.4s, v8.4s\n" + "srshl v24.4s, v24.4s, v8.4s\n" + "srshl v25.4s, v25.4s, v8.4s\n" + "srshl v26.4s, v26.4s, v8.4s\n" + "srshl v27.4s, v27.4s, v8.4s\n" + "srshl v28.4s, v28.4s, v8.4s\n" + "srshl v29.4s, v29.4s, v8.4s\n" + "srshl v30.4s, v30.4s, v8.4s\n" + "srshl v31.4s, v31.4s, v8.4s\n" + "add v16.4s, v16.4s, v11.4s\n" + "add v17.4s, v17.4s, v11.4s\n" + "add v18.4s, v18.4s, v11.4s\n" + "add v19.4s, v19.4s, v11.4s\n" + "add v20.4s, v20.4s, v11.4s\n" + "add v21.4s, v21.4s, v11.4s\n" + "add v22.4s, v22.4s, v11.4s\n" + "add v23.4s, v23.4s, v11.4s\n" + "add v24.4s, v24.4s, v11.4s\n" + "add v25.4s, v25.4s, v11.4s\n" + "add v26.4s, v26.4s, v11.4s\n" + "add v27.4s, v27.4s, v11.4s\n" + "add v28.4s, v28.4s, v11.4s\n" + "add v29.4s, v29.4s, v11.4s\n" + "add v30.4s, v30.4s, v11.4s\n" + "add v31.4s, v31.4s, v11.4s\n" + "smin v16.4s, v16.4s, v14.4s\n" + "smin v17.4s, v17.4s, v14.4s\n" + "smin v18.4s, v18.4s, v14.4s\n" + "smin v19.4s, v19.4s, v14.4s\n" + "smin v20.4s, v20.4s, v14.4s\n" + "smin v21.4s, v21.4s, v14.4s\n" + "smin v22.4s, v22.4s, v14.4s\n" + "smin v23.4s, v23.4s, v14.4s\n" + "smin v24.4s, v24.4s, v14.4s\n" + "smin v25.4s, v25.4s, v14.4s\n" + "smin v26.4s, v26.4s, v14.4s\n" + "smin v27.4s, v27.4s, v14.4s\n" + "smin v28.4s, v28.4s, v14.4s\n" + "smin v29.4s, v29.4s, v14.4s\n" + "smin v30.4s, v30.4s, v14.4s\n" + "smin v31.4s, v31.4s, v14.4s\n" + "smax v16.4s, v16.4s, v15.4s\n" + "smax v17.4s, v17.4s, v15.4s\n" + "smax v18.4s, v18.4s, v15.4s\n" + "smax v19.4s, v19.4s, v15.4s\n" + "smax v20.4s, v20.4s, v15.4s\n" + "smax v21.4s, v21.4s, v15.4s\n" + "smax v22.4s, v22.4s, v15.4s\n" + "smax v23.4s, v23.4s, v15.4s\n" + "smax v24.4s, v24.4s, v15.4s\n" + "smax v25.4s, v25.4s, v15.4s\n" + "smax v26.4s, v26.4s, v15.4s\n" + "smax v27.4s, v27.4s, v15.4s\n" + "smax v28.4s, v28.4s, v15.4s\n" + "smax v29.4s, v29.4s, v15.4s\n" + "smax v30.4s, v30.4s, v15.4s\n" + "smax v31.4s, v31.4s, v15.4s\n" "uzp1 v16.16b, v16.16b, v16.16b\n" "uzp1 v17.16b, v17.16b, v17.16b\n" "uzp1 v18.16b, v18.16b, v18.16b\n" @@ -1317,158 +1318,156 @@ void a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst "uzp1 v30.16b, v30.16b, v30.16b\n" "uzp1 v31.16b, v31.16b, v31.16b\n" "tbz %x[n_output_channels], #1, 24f\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "add x20, x20, x9\n" - "add x21, x21, x9\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "add x22, x22, x9\n" - "add x23, x23, x9\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "add x24, x24, x9\n" - "add x25, x25, x9\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "add x26, x26, x9\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" "add x27, x27, x9\n" - "st1 { v16.h }[0], [x20]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "add x20, x20, x9\n" - "st1 { v17.h }[0], [x21]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "add x21, x21, x9\n" - "st1 { v18.h }[0], [x22]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "add x22, x22, x9\n" - "st1 { v19.h }[0], [x23]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "add x23, x23, x9\n" - "st1 { v20.h }[0], [x24]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "add x24, x24, x9\n" - "st1 { v21.h }[0], [x25]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "add x25, x25, x9\n" - "st1 { v22.h }[0], [x26]\n" - "ldr x26, [%x[outptrs], #0x70]\n" "add x26, x26, x9\n" - "st1 { v23.h }[0], [x27]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "add x25, x25, x9\n" + "add x24, x24, x9\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "add x23, x23, x9\n" + "add x22, x22, x9\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "add x21, x21, x9\n" + "add x20, x20, x9\n" + "st1 { v16.h }[0], [x27]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "add x27, x27, x9\n" + "st1 { v17.h }[0], [x26]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "add x26, x26, x9\n" + "st1 { v18.h }[0], [x25]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "add x25, x25, x9\n" + "st1 { v19.h }[0], [x24]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "add x24, x24, x9\n" + "st1 { v20.h }[0], [x23]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "add x23, x23, x9\n" + "st1 { v21.h }[0], [x22]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "add x22, x22, x9\n" + "st1 { v22.h }[0], [x21]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "add x21, x21, x9\n" + "st1 { v23.h }[0], [x20]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "add x20, x20, x9\n" "add x9, x9, #0x2\n" - "st1 { v24.h }[0], [x20]\n" - "st1 { v25.h }[0], [x21]\n" - "st1 { v26.h }[0], [x22]\n" - "st1 { v27.h }[0], [x23]\n" - "st1 { v28.h }[0], [x24]\n" - "st1 { v29.h }[0], [x25]\n" - "st1 { v30.h }[0], [x26]\n" - "st1 { v31.h }[0], [x27]\n" + "st1 { v24.h }[0], [x27]\n" + "st1 { v25.h }[0], [x26]\n" + "st1 { v26.h }[0], [x25]\n" + "st1 { v27.h }[0], [x24]\n" + "st1 { v28.h }[0], [x23]\n" + "st1 { v29.h }[0], [x22]\n" + "st1 { v30.h }[0], [x21]\n" + "st1 { v31.h }[0], [x20]\n" "tbz %x[n_output_channels], #0, 25f\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "add x20, x20, x9\n" - "add x21, x21, x9\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "add x22, x22, x9\n" - "add x23, x23, x9\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "add x24, x24, x9\n" - "add x25, x25, x9\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "add x26, x26, x9\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" "add x27, x27, x9\n" - "st1 { v16.b }[2], [x20]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "add x20, x20, x9\n" - "st1 { v17.b }[2], [x21]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "add x21, x21, x9\n" - "st1 { v18.b }[2], [x22]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "add x22, x22, x9\n" - "st1 { v19.b }[2], [x23]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "add x23, x23, x9\n" - "st1 { v20.b }[2], [x24]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "add x24, x24, x9\n" - "st1 { v21.b }[2], [x25]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "add x25, x25, x9\n" - "st1 { v22.b }[2], [x26]\n" - "ldr x26, [%x[outptrs], #0x70]\n" "add x26, x26, x9\n" - "st1 { v23.b }[2], [x27]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "add x25, x25, x9\n" + "add x24, x24, x9\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "add x23, x23, x9\n" + "add x22, x22, x9\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "add x21, x21, x9\n" + "add x20, x20, x9\n" + "st1 { v16.b }[2], [x27]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "add x27, x27, x9\n" - "st1 { v24.b }[2], [x20]\n" - "st1 { v25.b }[2], [x21]\n" - "st1 { v26.b }[2], [x22]\n" - "st1 { v27.b }[2], [x23]\n" - "st1 { v28.b }[2], [x24]\n" - "st1 { v29.b }[2], [x25]\n" - "st1 { v30.b }[2], [x26]\n" - "st1 { v31.b }[2], [x27]\n" + "st1 { v17.b }[2], [x26]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "add x26, x26, x9\n" + "st1 { v18.b }[2], [x25]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "add x25, x25, x9\n" + "st1 { v19.b }[2], [x24]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "add x24, x24, x9\n" + "st1 { v20.b }[2], [x23]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "add x23, x23, x9\n" + "st1 { v21.b }[2], [x22]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "add x22, x22, x9\n" + "st1 { v22.b }[2], [x21]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "add x21, x21, x9\n" + "st1 { v23.b }[2], [x20]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "add x20, x20, x9\n" + "st1 { v24.b }[2], [x27]\n" + "st1 { v25.b }[2], [x26]\n" + "st1 { v26.b }[2], [x25]\n" + "st1 { v27.b }[2], [x24]\n" + "st1 { v28.b }[2], [x23]\n" + "st1 { v29.b }[2], [x22]\n" + "st1 { v30.b }[2], [x21]\n" + "st1 { v31.b }[2], [x20]\n" "b 25f\n" "24:" // Output channel oddments: Done: Store: Bit 1: Unset - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "add x20, x20, x9\n" - "add x21, x21, x9\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "add x22, x22, x9\n" - "add x23, x23, x9\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "add x24, x24, x9\n" - "add x25, x25, x9\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "add x26, x26, x9\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" "add x27, x27, x9\n" - "st1 { v16.b }[0], [x20]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "add x20, x20, x9\n" - "st1 { v17.b }[0], [x21]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "add x21, x21, x9\n" - "st1 { v18.b }[0], [x22]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "add x22, x22, x9\n" - "st1 { v19.b }[0], [x23]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "add x23, x23, x9\n" - "st1 { v20.b }[0], [x24]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "add x24, x24, x9\n" - "st1 { v21.b }[0], [x25]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "add x25, x25, x9\n" - "st1 { v22.b }[0], [x26]\n" - "ldr x26, [%x[outptrs], #0x70]\n" "add x26, x26, x9\n" - "st1 { v23.b }[0], [x27]\n" - "ldr x27, [%x[outptrs], #0x78]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "add x25, x25, x9\n" + "add x24, x24, x9\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "add x23, x23, x9\n" + "add x22, x22, x9\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "add x21, x21, x9\n" + "add x20, x20, x9\n" + "st1 { v16.b }[0], [x27]\n" + "ldr x27, [%x[outptrs], #0x40]\n" "add x27, x27, x9\n" - "st1 { v24.b }[0], [x20]\n" - "st1 { v25.b }[0], [x21]\n" - "st1 { v26.b }[0], [x22]\n" - "st1 { v27.b }[0], [x23]\n" - "st1 { v28.b }[0], [x24]\n" - "st1 { v29.b }[0], [x25]\n" - "st1 { v30.b }[0], [x26]\n" - "st1 { v31.b }[0], [x27]\n" + "st1 { v17.b }[0], [x26]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "add x26, x26, x9\n" + "st1 { v18.b }[0], [x25]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "add x25, x25, x9\n" + "st1 { v19.b }[0], [x24]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "add x24, x24, x9\n" + "st1 { v20.b }[0], [x23]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "add x23, x23, x9\n" + "st1 { v21.b }[0], [x22]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "add x22, x22, x9\n" + "st1 { v22.b }[0], [x21]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "add x21, x21, x9\n" + "st1 { v23.b }[0], [x20]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "add x20, x20, x9\n" + "st1 { v24.b }[0], [x27]\n" + "st1 { v25.b }[0], [x26]\n" + "st1 { v26.b }[0], [x25]\n" + "st1 { v27.b }[0], [x24]\n" + "st1 { v28.b }[0], [x23]\n" + "st1 { v29.b }[0], [x22]\n" + "st1 { v30.b }[0], [x21]\n" + "st1 { v31.b }[0], [x20]\n" "25:" // Output channel oddments: Done: Store: Bit 1: End - "26:" // Done - : [weights] "+&r" (weights) : [bias] "r" (bias), [inptrs] "r" (inptrs), [kernel_points] "r" ((uint64_t) kernel_points), [n_output_channels] "r" ((uint64_t) n_output_channels), [offsetof_Requantize32_a_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [offsetof_Requantize32_per_layer_left_shift] "I" (offsetof(arm_gemm::Requantize32, per_layer_left_shift)), [offsetof_Requantize32_per_layer_mul] "I" (offsetof(arm_gemm::Requantize32, per_layer_mul)), [offsetof_Requantize32_per_layer_right_shift] "I" (offsetof(arm_gemm::Requantize32, per_layer_right_shift)), [outptrs] "r" (outptrs), [qp] "r" (&qp), [rq_left_shift_ptr] "r" (per_channel_left_shifts), [rq_mul_ptr] "r" (per_channel_muls), [rq_right_shift_ptr] "r" (per_channel_right_shifts) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" @@ -1477,4 +1476,5 @@ void a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp index a200ebf2ccdc9a60e02c4046619423abf87bf94f..25d83f15c3aa7fa6e9279808dfbcdfc749d05934 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,13 +22,13 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include #pragma once -#if defined(__aarch64__) +#if defined(ARM_COMPUTE_ENABLE_SME2) namespace arm_conv { namespace depthwise { @@ -68,4 +68,4 @@ class sme2_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirs } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) +#endif // defined(ARM_COMPUTE_ENABLE_SME2) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp index 2ee961db15eb6699811c22b393397490e277901c..96cfd5e49753c6b28dceb0de550138c737568ec6 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -22,11 +22,11 @@ * SOFTWARE. */ -#if defined(ARM_COMPUTE_ENABLE_SME2) - #include #include +#if defined(ARM_COMPUTE_ENABLE_SME2) + namespace arm_conv { namespace depthwise { @@ -151,7 +151,7 @@ void sme2_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "ldr x22, [%x[params_struct], %[offsetof_args_ld_output_row]]\n" "mul x21, x4, x22\n" // offset = tile_i * ld_output_row "mov x20, #0x2\n" - "ld1w { z18.s }, p3/Z, [x15]\n" + "ld1w { z22.s }, p3/Z, [x15]\n" "ldr x25, [%x[params_struct], %[offsetof_args_ld_output_col]]\n" "madd x21, x5, x25, x21\n" // offset += tile_j * ld_output_col "addvl x15, x15, #1\n" @@ -159,13 +159,13 @@ void sme2_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "ldr x24, [%x[params_struct], %[offsetof_args_outptr]]\n" "mul x21, x21, x20\n" // offset *= output_tile_size "cntw x23\n" - "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rw { z21.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" "addvl x15, x15, #4\n" "add x24, x24, x21, LSL #2\n" // outptrs[0] += offset * sizeof(float) ".inst 0xa040c1e4 // ld1w { z4.s-z7.s }, pn8.b/Z, [x15]\n" "whilelt p2.s, XZR, %x[n_channels]\n" "addvl x15, x15, #4\n" - "ld1rw { z16.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "ld1rw { z14.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "cmp x23, %x[n_channels]\n" "add x22, x24, x22, LSL #2\n" "ld1w { z8.s }, p3/Z, [x15]\n" @@ -179,71 +179,71 @@ void sme2_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "ld1w { z13.s }, p2/Z, [x17, x6, LSL #2]\n" "bge 4f\n" "3:" // Tile loop: Channel loop - "movprfx z28, z18\n fmla z28.s, p3/M, z4.s, z9.s\n" - "movprfx z29, z18\n fmla z29.s, p3/M, z3.s, z9.s\n" + "movprfx z28, z22\n fmla z28.s, p3/M, z4.s, z9.s\n" + "movprfx z29, z22\n fmla z29.s, p3/M, z3.s, z9.s\n" "whilelt p1.s, x23, %x[n_channels]\n" "incw x21\n" - "movprfx z30, z18\n fmla z30.s, p3/M, z1.s, z9.s\n" - "movprfx z31, z18\n fmla z31.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x14]\n" + "movprfx z30, z22\n fmla z30.s, p3/M, z1.s, z9.s\n" + "movprfx z31, z22\n fmla z31.s, p3/M, z0.s, z9.s\n" + "ld1w { z18.s }, p2/Z, [x14]\n" "incw x23\n" "fmla z28.s, p3/M, z0.s, z10.s\n" "fmla z29.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x14, x13, LSL #2]\n" + "ld1w { z17.s }, p2/Z, [x14, x13, LSL #2]\n" "mov p0.b, p2.b\n" "fmla z30.s, p3/M, z2.s, z12.s\n" "fmla z31.s, p3/M, z1.s, z12.s\n" - "ld1w { z10.s }, p2/Z, [x17, x16, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x17, x16, LSL #2]\n" "incw x20\n" "fmla z28.s, p3/M, z5.s, z12.s\n" "fmla z29.s, p3/M, z4.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x7, x6, LSL #2]\n" - "fmla z30.s, p3/M, z6.s, z9.s\n" + "ld1w { z11.s }, p2/Z, [x7, x6, LSL #2]\n" + "fmla z30.s, p3/M, z6.s, z18.s\n" "fmla z31.s, p3/M, z3.s, z13.s\n" - "ld1w { z9.s }, p2/Z, [x7, x16, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x7, x16, LSL #2]\n" "addvl x7, x7, #1\n" "fmla z28.s, p3/M, z7.s, z13.s\n" "fmla z29.s, p3/M, z6.s, z13.s\n" - "ld1w { z18.s }, p3/Z, [x15]\n" + "ld1w { z22.s }, p3/Z, [x15]\n" "addvl x15, x15, #1\n" "fmla z30.s, p3/M, z4.s, z13.s\n" - "fmla z31.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x8]\n" - "fmla z28.s, p3/M, z1.s, z12.s\n" - "fmla z29.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x8, x13, LSL #2]\n" + "fmla z31.s, p3/M, z8.s, z17.s\n" + "ld1w { z9.s }, p2/Z, [x8]\n" + "fmla z28.s, p3/M, z1.s, z11.s\n" + "fmla z29.s, p3/M, z0.s, z11.s\n" + "ld1w { z19.s }, p2/Z, [x8, x13, LSL #2]\n" "addvl x8, x8, #1\n" - "fmla z30.s, p3/M, z5.s, z10.s\n" - "fmla z31.s, p3/M, z4.s, z10.s\n" - "fmla z28.s, p3/M, z2.s, z9.s\n" - "fmla z29.s, p3/M, z1.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x17]\n" - "fmla z30.s, p3/M, z0.s, z11.s\n" - "fmla z31.s, p3/M, z2.s, z12.s\n" - "fmla z28.s, p3/M, z8.s, z10.s\n" - "fmla z29.s, p3/M, z7.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x17, x13, LSL #2]\n" + "fmla z30.s, p3/M, z5.s, z16.s\n" + "fmla z31.s, p3/M, z4.s, z16.s\n" + "fmla z28.s, p3/M, z2.s, z10.s\n" + "fmla z29.s, p3/M, z1.s, z10.s\n" + "ld1w { z18.s }, p2/Z, [x17]\n" + "fmla z30.s, p3/M, z0.s, z9.s\n" + "fmla z31.s, p3/M, z2.s, z19.s\n" + "fmla z28.s, p3/M, z8.s, z16.s\n" + "fmla z29.s, p3/M, z7.s, z16.s\n" + "ld1w { z17.s }, p2/Z, [x17, x13, LSL #2]\n" "addvl x17, x17, #1\n" - "fmla z30.s, p3/M, z3.s, z9.s\n" - "fmla z31.s, p3/M, z5.s, z10.s\n" + "fmla z30.s, p3/M, z3.s, z18.s\n" + "fmla z31.s, p3/M, z5.s, z17.s\n" "ld1w { z13.s }, p1/Z, [x17, x6, LSL #2]\n" - "fmla z28.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x14, x6, LSL #2]\n" - "fmla z29.s, p3/M, z5.s, z12.s\n" - "fmla z30.s, p3/M, z7.s, z11.s\n" - "fmla z31.s, p3/M, z6.s, z11.s\n" - "ld1w { z12.s }, p2/Z, [x14, x16, LSL #2]\n" + "fmla z28.s, p3/M, z3.s, z9.s\n" + "ld1w { z16.s }, p2/Z, [x14, x6, LSL #2]\n" + "fmla z29.s, p3/M, z5.s, z19.s\n" + "fmla z30.s, p3/M, z7.s, z16.s\n" + "fmla z31.s, p3/M, z6.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x14, x16, LSL #2]\n" "whilelt p2.s, x21, %x[n_channels]\n" - "fmla z28.s, p3/M, z6.s, z9.s\n" - "fmla z29.s, p3/M, z8.s, z10.s\n" + "fmla z28.s, p3/M, z6.s, z18.s\n" + "fmla z29.s, p3/M, z8.s, z17.s\n" ".inst 0xa040c1e0 // ld1w { z0.s-z3.s }, pn8.b/Z, [x15]\n" "addvl x15, x15, #4\n" - "fmla z30.s, p3/M, z8.s, z12.s\n" - "fmla z31.s, p3/M, z7.s, z12.s\n" + "fmla z30.s, p3/M, z8.s, z16.s\n" + "fmla z31.s, p3/M, z7.s, z16.s\n" ".inst 0xa040c1e4 // ld1w { z4.s-z7.s }, pn8.b/Z, [x15]\n" "addvl x15, x15, #4\n" "cmp x23, %x[n_channels]\n" - ".inst 0xc1b0ca3c // fclamp { z28.s-z31.s }, z17.s, z16.s\n" + ".inst 0xc1aecabc // fclamp { z28.s-z31.s }, z21.s, z14.s\n" "addvl x14, x14, #1\n" "ld1w { z9.s }, p1/Z, [x8, x6, LSL #2]\n" "ld1w { z10.s }, p1/Z, [x7]\n" @@ -259,69 +259,69 @@ void sme2_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "addvl x15, x15, #1\n" "blt 3b\n" "4:" // Tile loop: Channel tail - "movprfx z28, z18\n fmla z28.s, p3/M, z4.s, z9.s\n" - "movprfx z29, z18\n fmla z29.s, p3/M, z3.s, z9.s\n" + "movprfx z24, z22\n fmla z24.s, p3/M, z4.s, z9.s\n" + "movprfx z25, z22\n fmla z25.s, p3/M, z3.s, z9.s\n" "ldr x5, [%x[params_struct], %[offsetof_args_tile_j]]\n" "add x5, x5, #0x1\n" - "movprfx z30, z18\n fmla z30.s, p3/M, z1.s, z9.s\n" - "movprfx z31, z18\n fmla z31.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x14]\n" + "movprfx z26, z22\n fmla z26.s, p3/M, z1.s, z9.s\n" + "movprfx z27, z22\n fmla z27.s, p3/M, z0.s, z9.s\n" + "ld1w { z17.s }, p2/Z, [x14]\n" "ldr x4, [%x[params_struct], %[offsetof_args_tile_i]]\n" - "fmla z28.s, p3/M, z0.s, z10.s\n" - "fmla z29.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x14, x13, LSL #2]\n" + "fmla z24.s, p3/M, z0.s, z10.s\n" + "fmla z25.s, p3/M, z2.s, z11.s\n" + "ld1w { z16.s }, p2/Z, [x14, x13, LSL #2]\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" - "fmla z30.s, p3/M, z2.s, z12.s\n" - "fmla z31.s, p3/M, z1.s, z12.s\n" - "ld1w { z10.s }, p2/Z, [x17, x16, LSL #2]\n" + "fmla z26.s, p3/M, z2.s, z12.s\n" + "fmla z27.s, p3/M, z1.s, z12.s\n" + "ld1w { z20.s }, p2/Z, [x17, x16, LSL #2]\n" "ldr x21, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" - "fmla z28.s, p3/M, z5.s, z12.s\n" - "fmla z29.s, p3/M, z4.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x7, x6, LSL #2]\n" + "fmla z24.s, p3/M, z5.s, z12.s\n" + "fmla z25.s, p3/M, z4.s, z12.s\n" + "ld1w { z18.s }, p2/Z, [x7, x6, LSL #2]\n" "cmp x5, x20\n" - "fmla z30.s, p3/M, z6.s, z9.s\n" - "fmla z31.s, p3/M, z3.s, z13.s\n" - "ld1w { z9.s }, p2/Z, [x7, x16, LSL #2]\n" + "fmla z26.s, p3/M, z6.s, z17.s\n" + "fmla z27.s, p3/M, z3.s, z13.s\n" + "ld1w { z17.s }, p2/Z, [x7, x16, LSL #2]\n" "add x20, x4, #0x1\n" - "fmla z28.s, p3/M, z7.s, z13.s\n" - "fmla z29.s, p3/M, z6.s, z13.s\n" + "fmla z24.s, p3/M, z7.s, z13.s\n" + "fmla z25.s, p3/M, z6.s, z13.s\n" "csel x4, x4, x20, LT\n" "mov p0.b, p2.b\n" - "fmla z30.s, p3/M, z4.s, z13.s\n" - "fmla z31.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x8]\n" + "fmla z26.s, p3/M, z4.s, z13.s\n" + "fmla z27.s, p3/M, z8.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x8]\n" "csel x5, x5, XZR, LT\n" - "fmla z28.s, p3/M, z1.s, z12.s\n" - "fmla z29.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x8, x13, LSL #2]\n" + "fmla z24.s, p3/M, z1.s, z18.s\n" + "fmla z25.s, p3/M, z0.s, z18.s\n" + "ld1w { z19.s }, p2/Z, [x8, x13, LSL #2]\n" "cmp x4, x21\n" - "fmla z30.s, p3/M, z5.s, z10.s\n" - "fmla z31.s, p3/M, z4.s, z10.s\n" - "fmla z28.s, p3/M, z2.s, z9.s\n" - "fmla z29.s, p3/M, z1.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x17]\n" - "fmla z30.s, p3/M, z0.s, z11.s\n" - "fmla z31.s, p3/M, z2.s, z12.s\n" - "fmla z28.s, p3/M, z8.s, z10.s\n" - "fmla z29.s, p3/M, z7.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x17, x13, LSL #2]\n" - "fmla z30.s, p3/M, z3.s, z9.s\n" - "fmla z31.s, p3/M, z5.s, z10.s\n" - "fmla z28.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x14, x6, LSL #2]\n" - "fmla z29.s, p3/M, z5.s, z12.s\n" - "fmla z30.s, p3/M, z7.s, z11.s\n" - "fmla z31.s, p3/M, z6.s, z11.s\n" - "ld1w { z12.s }, p2/Z, [x14, x16, LSL #2]\n" - "fmla z28.s, p3/M, z6.s, z9.s\n" - "fmla z29.s, p3/M, z8.s, z10.s\n" - "fmla z30.s, p3/M, z8.s, z12.s\n" - "fmla z31.s, p3/M, z7.s, z12.s\n" - ".inst 0xc1b0ca3c // fclamp { z28.s-z31.s }, z17.s, z16.s\n" - "st1w { z28.s }, p0, [x24]\n" - "st1w { z29.s }, p0, [x24, x25, LSL #2]\n" - "st1w { z30.s }, p0, [x22]\n" - "st1w { z31.s }, p0, [x22, x25, LSL #2]\n" + "fmla z26.s, p3/M, z5.s, z20.s\n" + "fmla z27.s, p3/M, z4.s, z20.s\n" + "fmla z24.s, p3/M, z2.s, z17.s\n" + "fmla z25.s, p3/M, z1.s, z17.s\n" + "ld1w { z18.s }, p2/Z, [x17]\n" + "fmla z26.s, p3/M, z0.s, z16.s\n" + "fmla z27.s, p3/M, z2.s, z19.s\n" + "fmla z24.s, p3/M, z8.s, z20.s\n" + "fmla z25.s, p3/M, z7.s, z20.s\n" + "ld1w { z17.s }, p2/Z, [x17, x13, LSL #2]\n" + "fmla z26.s, p3/M, z3.s, z18.s\n" + "fmla z27.s, p3/M, z5.s, z17.s\n" + "fmla z24.s, p3/M, z3.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x14, x6, LSL #2]\n" + "fmla z25.s, p3/M, z5.s, z19.s\n" + "fmla z26.s, p3/M, z7.s, z16.s\n" + "fmla z27.s, p3/M, z6.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x14, x16, LSL #2]\n" + "fmla z24.s, p3/M, z6.s, z18.s\n" + "fmla z25.s, p3/M, z8.s, z17.s\n" + "fmla z26.s, p3/M, z8.s, z16.s\n" + "fmla z27.s, p3/M, z7.s, z16.s\n" + ".inst 0xc1aecab8 // fclamp { z24.s-z27.s }, z21.s, z14.s\n" + "st1w { z24.s }, p0, [x24]\n" + "st1w { z25.s }, p0, [x24, x25, LSL #2]\n" + "st1w { z26.s }, p0, [x22]\n" + "st1w { z27.s }, p0, [x22, x25, LSL #2]\n" "blt 1b\n" ".inst 0xd503467f // SMSTOP\n" : diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp index 079b39c5ecf374196743c7954fe66efba6bc10b1..39f1b3635f596e089d6e0e210896bcb79ed01d9e 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -22,11 +22,11 @@ * SOFTWARE. */ -#if defined(ARM_COMPUTE_ENABLE_SME2) - #include #include +#if defined(ARM_COMPUTE_ENABLE_SME2) + namespace arm_conv { namespace depthwise { @@ -84,7 +84,7 @@ void sme2_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( "ptrue p3.b\n" "ldr x14, [%x[params_struct], %[offsetof_args_params]]\n" ".inst 0x25207810 // ptrue pn8.b\n" - "ld1w { z18.s }, p3/Z, [x14]\n" + "ld1w { z23.s }, p3/Z, [x14]\n" "addvl x14, x14, #1\n" "ldp x13, x12, [x20, #0x0]\n" "cntw x11\n" @@ -94,176 +94,176 @@ void sme2_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( "mov x28, #0x0\n" "whilelt p2.s, XZR, %x[n_channels]\n" ".inst 0xa040c1c4 // ld1w { z4.s-z7.s }, pn8.b/Z, [x14]\n" - "ldp x27, x26, [x15, #0x0]\n" + "ldp x24, x23, [x15, #0x0]\n" "addvl x14, x14, #4\n" "cmp x11, %x[n_channels]\n" - "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "ldp x25, x22, [x15, #0x10]\n" - "ld1rw { z16.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "sub x24, XZR, x11\n" - "ldr x23, [x15, #0x20]\n" + "ld1rw { z22.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ldp x22, x21, [x15, #0x10]\n" + "ld1rw { z15.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "sub x27, XZR, x11\n" + "ldr x20, [x15, #0x20]\n" "ld1w { z8.s }, p3/Z, [x14]\n" "addvl x14, x14, #1\n" - "ld1w { z9.s }, p2/Z, [x27, x28, LSL #2]\n" - "ld1w { z10.s }, p2/Z, [x26, x28, LSL #2]\n" - "ld1w { z11.s }, p2/Z, [x25, x28, LSL #2]\n" - "ld1w { z12.s }, p2/Z, [x22, x28, LSL #2]\n" - "ld1w { z13.s }, p2/Z, [x23, x28, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x24, x28, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x23, x28, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x22, x28, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x21, x28, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x20, x28, LSL #2]\n" "bge 2f\n" "1:" // Channel loop - "movprfx z28, z18\n fmla z28.s, p3/M, z4.s, z9.s\n" - "movprfx z29, z18\n fmla z29.s, p3/M, z3.s, z9.s\n" - "ldr x22, [x15, #0x28]\n" + "movprfx z28, z23\n fmla z28.s, p3/M, z4.s, z9.s\n" + "movprfx z29, z23\n fmla z29.s, p3/M, z3.s, z9.s\n" + "ldr x20, [x15, #0x28]\n" "whilelt p1.s, x11, %x[n_channels]\n" - "movprfx z30, z18\n fmla z30.s, p3/M, z1.s, z9.s\n" - "movprfx z31, z18\n fmla z31.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x22, x28, LSL #2]\n" - "ldr x21, [x15, #0x30]\n" + "movprfx z30, z23\n fmla z30.s, p3/M, z1.s, z9.s\n" + "movprfx z31, z23\n fmla z31.s, p3/M, z0.s, z9.s\n" + "ld1w { z19.s }, p2/Z, [x20, x28, LSL #2]\n" + "ldr x20, [x15, #0x30]\n" "fmla z28.s, p3/M, z0.s, z10.s\n" "fmla z29.s, p3/M, z2.s, z11.s\n" - "ldr x20, [x15, #0x38]\n" - "ld1w { z11.s }, p2/Z, [x21, x28, LSL #2]\n" + "ldr x21, [x15, #0x38]\n" + "ld1w { z18.s }, p2/Z, [x20, x28, LSL #2]\n" "fmla z30.s, p3/M, z2.s, z12.s\n" "fmla z31.s, p3/M, z1.s, z12.s\n" - "ldr x26, [x15, #0x48]\n" - "ld1w { z10.s }, p2/Z, [x26, x28, LSL #2]\n" + "ldr x20, [x15, #0x48]\n" + "ld1w { z17.s }, p2/Z, [x20, x28, LSL #2]\n" "fmla z28.s, p3/M, z5.s, z12.s\n" "fmla z29.s, p3/M, z4.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x20, x28, LSL #2]\n" - "ldr x27, [x15, #0x40]\n" - "fmla z30.s, p3/M, z6.s, z9.s\n" + "ld1w { z16.s }, p2/Z, [x21, x28, LSL #2]\n" + "ldr x20, [x15, #0x40]\n" + "fmla z30.s, p3/M, z6.s, z19.s\n" "fmla z31.s, p3/M, z3.s, z13.s\n" - "ld1w { z9.s }, p2/Z, [x27, x28, LSL #2]\n" - "ldr x25, [x15, #0x50]\n" + "ld1w { z25.s }, p2/Z, [x20, x28, LSL #2]\n" + "ldr x21, [x15, #0x50]\n" "fmla z28.s, p3/M, z7.s, z13.s\n" "fmla z29.s, p3/M, z6.s, z13.s\n" - "ldr x22, [x15, #0x58]\n" - "ld1w { z18.s }, p3/Z, [x14]\n" + "ldr x20, [x15, #0x58]\n" + "ld1w { z23.s }, p3/Z, [x14]\n" "fmla z30.s, p3/M, z4.s, z13.s\n" - "fmla z31.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x25, x28, LSL #2]\n" - "ldr x23, [x15, #0x60]\n" - "fmla z28.s, p3/M, z1.s, z12.s\n" - "fmla z29.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x22, x28, LSL #2]\n" - "ldr x22, [x15, #0x68]\n" - "fmla z30.s, p3/M, z5.s, z10.s\n" - "fmla z31.s, p3/M, z4.s, z10.s\n" - "ldr x21, [x15, #0x70]\n" + "fmla z31.s, p3/M, z8.s, z18.s\n" + "ld1w { z11.s }, p2/Z, [x21, x28, LSL #2]\n" + "ldr x21, [x15, #0x60]\n" + "fmla z28.s, p3/M, z1.s, z16.s\n" + "fmla z29.s, p3/M, z0.s, z16.s\n" + "ld1w { z19.s }, p2/Z, [x20, x28, LSL #2]\n" + "ldr x20, [x15, #0x68]\n" + "fmla z30.s, p3/M, z5.s, z17.s\n" + "fmla z31.s, p3/M, z4.s, z17.s\n" + "ldr x26, [x15, #0x70]\n" "addvl x14, x14, #1\n" - "fmla z28.s, p3/M, z2.s, z9.s\n" - "fmla z29.s, p3/M, z1.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x23, x28, LSL #2]\n" - "ldr x20, [x15, #0x78]\n" + "fmla z28.s, p3/M, z2.s, z25.s\n" + "fmla z29.s, p3/M, z1.s, z25.s\n" + "ld1w { z18.s }, p2/Z, [x21, x28, LSL #2]\n" + "ldr x25, [x15, #0x78]\n" "fmla z30.s, p3/M, z0.s, z11.s\n" - "fmla z31.s, p3/M, z2.s, z12.s\n" - "ldp x27, x26, [x15, #0x0]\n" - "incw x24\n" - "fmla z28.s, p3/M, z8.s, z10.s\n" - "fmla z29.s, p3/M, z7.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x22, x28, LSL #2]\n" - "ldp x25, x22, [x15, #0x10]\n" - "fmla z30.s, p3/M, z3.s, z9.s\n" - "fmla z31.s, p3/M, z5.s, z10.s\n" - "ldr x23, [x15, #0x20]\n" - "ld1w { z13.s }, p1/Z, [x23, x11, LSL #2]\n" + "fmla z31.s, p3/M, z2.s, z19.s\n" + "ldp x24, x23, [x15, #0x0]\n" + "incw x27\n" + "fmla z28.s, p3/M, z8.s, z17.s\n" + "fmla z29.s, p3/M, z7.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x20, x28, LSL #2]\n" + "ldp x22, x21, [x15, #0x10]\n" + "fmla z30.s, p3/M, z3.s, z18.s\n" + "fmla z31.s, p3/M, z5.s, z17.s\n" + "ldr x20, [x15, #0x20]\n" + "ld1w { z13.s }, p1/Z, [x20, x11, LSL #2]\n" "fmla z28.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x21, x28, LSL #2]\n" - "fmla z29.s, p3/M, z5.s, z12.s\n" + "ld1w { z16.s }, p2/Z, [x26, x28, LSL #2]\n" + "fmla z29.s, p3/M, z5.s, z19.s\n" "mov p0.b, p2.b\n" - "fmla z30.s, p3/M, z7.s, z11.s\n" - "fmla z31.s, p3/M, z6.s, z11.s\n" - "ld1w { z12.s }, p2/Z, [x20, x28, LSL #2]\n" + "fmla z30.s, p3/M, z7.s, z16.s\n" + "fmla z31.s, p3/M, z6.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x25, x28, LSL #2]\n" "incw x28\n" - "fmla z28.s, p3/M, z6.s, z9.s\n" - "fmla z29.s, p3/M, z8.s, z10.s\n" - "ld1w { z9.s }, p1/Z, [x27, x11, LSL #2]\n" + "fmla z28.s, p3/M, z6.s, z18.s\n" + "fmla z29.s, p3/M, z8.s, z17.s\n" + "ld1w { z9.s }, p1/Z, [x24, x11, LSL #2]\n" "whilelt p2.s, x28, %x[n_channels]\n" - "fmla z30.s, p3/M, z8.s, z12.s\n" - "fmla z31.s, p3/M, z7.s, z12.s\n" - "ld1w { z10.s }, p1/Z, [x26, x11, LSL #2]\n" - "ld1w { z11.s }, p1/Z, [x25, x11, LSL #2]\n" - ".inst 0xc1b0ca3c // fclamp { z28.s-z31.s }, z17.s, z16.s\n" - "st1w { z28.s }, p0, [x13, x24, LSL #2]\n" - "ld1w { z12.s }, p1/Z, [x22, x11, LSL #2]\n" + "fmla z30.s, p3/M, z8.s, z16.s\n" + "fmla z31.s, p3/M, z7.s, z16.s\n" + "ld1w { z10.s }, p1/Z, [x23, x11, LSL #2]\n" + "ld1w { z11.s }, p1/Z, [x22, x11, LSL #2]\n" + ".inst 0xc1afcadc // fclamp { z28.s-z31.s }, z22.s, z15.s\n" + "st1w { z28.s }, p0, [x13, x27, LSL #2]\n" + "ld1w { z12.s }, p1/Z, [x21, x11, LSL #2]\n" "incw x11\n" "cmp x11, %x[n_channels]\n" - "st1w { z29.s }, p0, [x12, x24, LSL #2]\n" + "st1w { z29.s }, p0, [x12, x27, LSL #2]\n" ".inst 0xa040c1c0 // ld1w { z0.s-z3.s }, pn8.b/Z, [x14]\n" "addvl x14, x14, #4\n" - "st1w { z30.s }, p0, [x10, x24, LSL #2]\n" + "st1w { z30.s }, p0, [x10, x27, LSL #2]\n" ".inst 0xa040c1c4 // ld1w { z4.s-z7.s }, pn8.b/Z, [x14]\n" "addvl x14, x14, #4\n" - "st1w { z31.s }, p0, [x9, x24, LSL #2]\n" + "st1w { z31.s }, p0, [x9, x27, LSL #2]\n" "ld1w { z8.s }, p3/Z, [x14]\n" "addvl x14, x14, #1\n" "blt 1b\n" "2:" // Channel tail - "movprfx z28, z18\n fmla z28.s, p3/M, z4.s, z9.s\n" - "movprfx z29, z18\n fmla z29.s, p3/M, z3.s, z9.s\n" - "ldr x22, [x15, #0x28]\n" - "incw x24\n" - "movprfx z30, z18\n fmla z30.s, p3/M, z1.s, z9.s\n" - "movprfx z31, z18\n fmla z31.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x22, x28, LSL #2]\n" - "ldr x21, [x15, #0x30]\n" + "movprfx z28, z23\n fmla z28.s, p3/M, z4.s, z9.s\n" + "movprfx z29, z23\n fmla z29.s, p3/M, z3.s, z9.s\n" + "ldr x20, [x15, #0x28]\n" + "incw x27\n" + "movprfx z30, z23\n fmla z30.s, p3/M, z1.s, z9.s\n" + "movprfx z31, z23\n fmla z31.s, p3/M, z0.s, z9.s\n" + "ld1w { z17.s }, p2/Z, [x20, x28, LSL #2]\n" + "ldr x20, [x15, #0x30]\n" "fmla z28.s, p3/M, z0.s, z10.s\n" "fmla z29.s, p3/M, z2.s, z11.s\n" - "ldr x20, [x15, #0x38]\n" - "ld1w { z11.s }, p2/Z, [x21, x28, LSL #2]\n" + "ldr x21, [x15, #0x38]\n" + "ld1w { z16.s }, p2/Z, [x20, x28, LSL #2]\n" "fmla z30.s, p3/M, z2.s, z12.s\n" "fmla z31.s, p3/M, z1.s, z12.s\n" - "ldr x26, [x15, #0x48]\n" - "ld1w { z10.s }, p2/Z, [x26, x28, LSL #2]\n" + "ldr x20, [x15, #0x48]\n" + "ld1w { z20.s }, p2/Z, [x20, x28, LSL #2]\n" "fmla z28.s, p3/M, z5.s, z12.s\n" "fmla z29.s, p3/M, z4.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x20, x28, LSL #2]\n" - "ldr x27, [x15, #0x40]\n" - "fmla z30.s, p3/M, z6.s, z9.s\n" + "ld1w { z18.s }, p2/Z, [x21, x28, LSL #2]\n" + "ldr x20, [x15, #0x40]\n" + "fmla z30.s, p3/M, z6.s, z17.s\n" "fmla z31.s, p3/M, z3.s, z13.s\n" - "ld1w { z9.s }, p2/Z, [x27, x28, LSL #2]\n" - "ldr x25, [x15, #0x50]\n" + "ld1w { z17.s }, p2/Z, [x20, x28, LSL #2]\n" + "ldr x20, [x15, #0x50]\n" "fmla z28.s, p3/M, z7.s, z13.s\n" "fmla z29.s, p3/M, z6.s, z13.s\n" - "ldr x22, [x15, #0x58]\n" + "ldr x21, [x15, #0x58]\n" "mov p0.b, p2.b\n" "fmla z30.s, p3/M, z4.s, z13.s\n" - "fmla z31.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x25, x28, LSL #2]\n" - "ldr x23, [x15, #0x60]\n" - "fmla z28.s, p3/M, z1.s, z12.s\n" - "fmla z29.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x22, x28, LSL #2]\n" + "fmla z31.s, p3/M, z8.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x20, x28, LSL #2]\n" + "ldr x20, [x15, #0x60]\n" + "fmla z28.s, p3/M, z1.s, z18.s\n" + "fmla z29.s, p3/M, z0.s, z18.s\n" + "ld1w { z19.s }, p2/Z, [x21, x28, LSL #2]\n" "ldr x22, [x15, #0x68]\n" - "fmla z30.s, p3/M, z5.s, z10.s\n" - "fmla z31.s, p3/M, z4.s, z10.s\n" + "fmla z30.s, p3/M, z5.s, z20.s\n" + "fmla z31.s, p3/M, z4.s, z20.s\n" "ldr x21, [x15, #0x70]\n" - "fmla z28.s, p3/M, z2.s, z9.s\n" - "fmla z29.s, p3/M, z1.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x23, x28, LSL #2]\n" + "fmla z28.s, p3/M, z2.s, z17.s\n" + "fmla z29.s, p3/M, z1.s, z17.s\n" + "ld1w { z18.s }, p2/Z, [x20, x28, LSL #2]\n" "ldr x20, [x15, #0x78]\n" - "fmla z30.s, p3/M, z0.s, z11.s\n" - "fmla z31.s, p3/M, z2.s, z12.s\n" - "fmla z28.s, p3/M, z8.s, z10.s\n" - "fmla z29.s, p3/M, z7.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x22, x28, LSL #2]\n" - "fmla z30.s, p3/M, z3.s, z9.s\n" - "fmla z31.s, p3/M, z5.s, z10.s\n" - "fmla z28.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x21, x28, LSL #2]\n" - "fmla z29.s, p3/M, z5.s, z12.s\n" - "fmla z30.s, p3/M, z7.s, z11.s\n" - "fmla z31.s, p3/M, z6.s, z11.s\n" - "ld1w { z12.s }, p2/Z, [x20, x28, LSL #2]\n" - "fmla z28.s, p3/M, z6.s, z9.s\n" - "fmla z29.s, p3/M, z8.s, z10.s\n" - "fmla z30.s, p3/M, z8.s, z12.s\n" - "fmla z31.s, p3/M, z7.s, z12.s\n" - ".inst 0xc1b0ca3c // fclamp { z28.s-z31.s }, z17.s, z16.s\n" - "st1w { z28.s }, p0, [x13, x24, LSL #2]\n" - "st1w { z29.s }, p0, [x12, x24, LSL #2]\n" - "st1w { z30.s }, p0, [x10, x24, LSL #2]\n" - "st1w { z31.s }, p0, [x9, x24, LSL #2]\n" + "fmla z30.s, p3/M, z0.s, z16.s\n" + "fmla z31.s, p3/M, z2.s, z19.s\n" + "fmla z28.s, p3/M, z8.s, z20.s\n" + "fmla z29.s, p3/M, z7.s, z20.s\n" + "ld1w { z17.s }, p2/Z, [x22, x28, LSL #2]\n" + "fmla z30.s, p3/M, z3.s, z18.s\n" + "fmla z31.s, p3/M, z5.s, z17.s\n" + "fmla z28.s, p3/M, z3.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x21, x28, LSL #2]\n" + "fmla z29.s, p3/M, z5.s, z19.s\n" + "fmla z30.s, p3/M, z7.s, z16.s\n" + "fmla z31.s, p3/M, z6.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x20, x28, LSL #2]\n" + "fmla z28.s, p3/M, z6.s, z18.s\n" + "fmla z29.s, p3/M, z8.s, z17.s\n" + "fmla z30.s, p3/M, z8.s, z16.s\n" + "fmla z31.s, p3/M, z7.s, z16.s\n" + ".inst 0xc1afcadc // fclamp { z28.s-z31.s }, z22.s, z15.s\n" + "st1w { z28.s }, p0, [x13, x27, LSL #2]\n" + "st1w { z29.s }, p0, [x12, x27, LSL #2]\n" + "st1w { z30.s }, p0, [x10, x27, LSL #2]\n" + "st1w { z31.s }, p0, [x9, x27, LSL #2]\n" ".inst 0xd503467f // SMSTOP\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp index cf74f431dffb24c089030174ccc87efc293c64dc..bd330dc21eb136b85e5a262be62562e8743bd6ba 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,13 +22,13 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include #pragma once -#if defined(__aarch64__) +#if defined(ARM_COMPUTE_ENABLE_SME2) namespace arm_conv { namespace depthwise { @@ -68,4 +68,4 @@ class sme2_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst : public DepthwiseDepthfirs } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) +#endif // defined(ARM_COMPUTE_ENABLE_SME2) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp index ce0ae297567029823647ce7375b8f24cec55d214..d15a3a8377c46de7a3aa074e0e4f3f739157fed8 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp @@ -22,11 +22,11 @@ * SOFTWARE. */ -#if defined(ARM_COMPUTE_ENABLE_SME2) - #include #include +#if defined(ARM_COMPUTE_ENABLE_SME2) + namespace arm_conv { namespace depthwise { @@ -170,11 +170,11 @@ void sme2_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "ldr x22, [%x[params_struct], %[offsetof_args_ld_output_row]]\n" "mul x21, x2, x22\n" // offset = tile_i * ld_output_row "mov x20, #0x3\n" - "ld1w { z18.s }, p3/Z, [x17]\n" + "ld1w { z24.s }, p3/Z, [x17]\n" "ldr x27, [%x[params_struct], %[offsetof_args_ld_output_col]]\n" "madd x21, x3, x27, x21\n" // offset += tile_j * ld_output_col "mul x21, x21, x20\n" // offset *= output_tile_size - "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rw { z26.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" "ldr x26, [%x[params_struct], %[offsetof_args_outptr]]\n" "addvl x17, x17, #1\n" "add x26, x26, x21, LSL #2\n" // outptrs[0] += offset * sizeof(float) @@ -184,7 +184,7 @@ void sme2_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( ".inst 0xa040c224 // ld1w { z4.s-z7.s }, pn8.b/Z, [x17]\n" "add x24, x26, x22, LSL #2\n" "whilelt p2.s, XZR, %x[n_channels]\n" - "ld1rw { z16.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "ld1rw { z14.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "addvl x17, x17, #4\n" "cmp x25, %x[n_channels]\n" "ld1w { z8.s }, p3/Z, [x17]\n" @@ -200,275 +200,275 @@ void sme2_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "ld1w { z13.s }, p2/Z, [x6, x8, LSL #2]\n" "bge 4f\n" "3:" // Tile loop: Channel loop - "movprfx z24, z18\n fmla z24.s, p3/M, z7.s, z9.s\n" - "movprfx z23, z18\n fmla z23.s, p3/M, z8.s, z9.s\n" + "movprfx z28, z24\n fmla z28.s, p3/M, z7.s, z9.s\n" + "movprfx z27, z24\n fmla z27.s, p3/M, z8.s, z9.s\n" "whilelt p1.s, x25, %x[n_channels]\n" "incw x21\n" - "movprfx z25, z18\n fmla z25.s, p3/M, z6.s, z9.s\n" - "fmla z24.s, p3/M, z4.s, z13.s\n" + "movprfx z29, z24\n fmla z29.s, p3/M, z6.s, z9.s\n" + "fmla z28.s, p3/M, z4.s, z13.s\n" "incw x25\n" "mov p0.b, p2.b\n" - "movprfx z26, z18\n fmla z26.s, p3/M, z5.s, z9.s\n" - "movprfx z27, z18\n fmla z27.s, p3/M, z4.s, z9.s\n" + "movprfx z30, z24\n fmla z30.s, p3/M, z5.s, z9.s\n" + "movprfx z31, z24\n fmla z31.s, p3/M, z4.s, z9.s\n" "incw x20\n" - "movprfx z28, z18\n fmla z28.s, p3/M, z3.s, z9.s\n" - "fmla z23.s, p3/M, z0.s, z10.s\n" + "movprfx z20, z24\n fmla z20.s, p3/M, z3.s, z9.s\n" + "fmla z27.s, p3/M, z0.s, z10.s\n" "ld1w { z10.s }, p2/Z, [x7, x15, LSL #2]\n" - "fmla z25.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x7, x4, LSL #2]\n" - "movprfx z29, z18\n fmla z29.s, p3/M, z2.s, z9.s\n" - "fmla z24.s, p3/M, z6.s, z11.s\n" - "movprfx z31, z18\n fmla z31.s, p3/M, z0.s, z9.s\n" - "fmla z23.s, p3/M, z5.s, z13.s\n" - "fmla z25.s, p3/M, z3.s, z13.s\n" - "fmla z26.s, p3/M, z2.s, z13.s\n" - "fmla z27.s, p3/M, z1.s, z13.s\n" - "fmla z28.s, p3/M, z0.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x5, x4, LSL #2]\n" - "fmla z29.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x14, x13, LSL #2]\n" - "movprfx z30, z18\n fmla z30.s, p3/M, z1.s, z9.s\n" - "fmla z24.s, p3/M, z0.s, z13.s\n" - "fmla z31.s, p3/M, z8.s, z12.s\n" - "fmla z23.s, p3/M, z7.s, z11.s\n" - "ld1w { z12.s }, p2/Z, [x5, x15, LSL #2]\n" - "fmla z30.s, p3/M, z0.s, z11.s\n" - "fmla z26.s, p3/M, z4.s, z11.s\n" - "ld1w { z18.s }, p3/Z, [x17]\n" + "fmla z29.s, p3/M, z2.s, z11.s\n" + "ld1w { z19.s }, p2/Z, [x7, x4, LSL #2]\n" + "movprfx z21, z24\n fmla z21.s, p3/M, z2.s, z9.s\n" + "fmla z28.s, p3/M, z6.s, z19.s\n" + "movprfx z23, z24\n fmla z23.s, p3/M, z0.s, z9.s\n" + "fmla z27.s, p3/M, z5.s, z13.s\n" + "fmla z29.s, p3/M, z3.s, z13.s\n" + "fmla z30.s, p3/M, z2.s, z13.s\n" + "fmla z31.s, p3/M, z1.s, z13.s\n" + "fmla z20.s, p3/M, z0.s, z13.s\n" + "ld1w { z18.s }, p2/Z, [x5, x4, LSL #2]\n" + "fmla z21.s, p3/M, z6.s, z12.s\n" + "ld1w { z15.s }, p2/Z, [x14, x13, LSL #2]\n" + "movprfx z22, z24\n fmla z22.s, p3/M, z1.s, z9.s\n" + "fmla z28.s, p3/M, z0.s, z18.s\n" + "fmla z23.s, p3/M, z8.s, z15.s\n" + "fmla z27.s, p3/M, z7.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x5, x15, LSL #2]\n" + "fmla z22.s, p3/M, z0.s, z19.s\n" + "fmla z30.s, p3/M, z4.s, z19.s\n" + "ld1w { z24.s }, p3/Z, [x17]\n" "addvl x17, x17, #1\n" - "fmla z27.s, p3/M, z3.s, z11.s\n" - "fmla z29.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x6]\n" - "fmla z24.s, p3/M, z2.s, z12.s\n" - "fmla z25.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x16]\n" - "fmla z28.s, p3/M, z4.s, z10.s\n" - "fmla z23.s, p3/M, z1.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x6, x13, LSL #2]\n" - "fmla z30.s, p3/M, z2.s, z10.s\n" - "fmla z31.s, p3/M, z1.s, z10.s\n" - "fmla z24.s, p3/M, z8.s, z10.s\n" - "fmla z25.s, p3/M, z7.s, z10.s\n" - "fmla z27.s, p3/M, z5.s, z10.s\n" - "fmla z26.s, p3/M, z0.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x16, x8, LSL #2]\n" - "fmla z28.s, p3/M, z2.s, z13.s\n" - "fmla z29.s, p3/M, z3.s, z12.s\n" - "fmla z30.s, p3/M, z4.s, z10.s\n" - "fmla z31.s, p3/M, z3.s, z10.s\n" - "fmla z23.s, p3/M, z3.s, z11.s\n" - "fmla z25.s, p3/M, z5.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x16, x13, LSL #2]\n" - "fmla z26.s, p3/M, z6.s, z12.s\n" - "fmla z27.s, p3/M, z7.s, z10.s\n" - "ld1w { z13.s }, p2/Z, [x14, x4, LSL #2]\n" - "fmla z28.s, p3/M, z6.s, z10.s\n" - "fmla z29.s, p3/M, z5.s, z10.s\n" - "ld1w { z12.s }, p2/Z, [x6, x4, LSL #2]\n" - "fmla z31.s, p3/M, z5.s, z11.s\n" - "fmla z30.s, p3/M, z6.s, z13.s\n" - "fmla z26.s, p3/M, z8.s, z10.s\n" - "fmla z28.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x6, x15, LSL #2]\n" + "fmla z31.s, p3/M, z3.s, z19.s\n" + "fmla z21.s, p3/M, z1.s, z19.s\n" + "ld1w { z17.s }, p2/Z, [x6]\n" + "fmla z28.s, p3/M, z2.s, z16.s\n" + "fmla z29.s, p3/M, z1.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x16]\n" + "fmla z20.s, p3/M, z4.s, z10.s\n" + "fmla z27.s, p3/M, z1.s, z18.s\n" + "ld1w { z9.s }, p2/Z, [x6, x13, LSL #2]\n" + "fmla z22.s, p3/M, z2.s, z10.s\n" + "fmla z23.s, p3/M, z1.s, z10.s\n" + "fmla z28.s, p3/M, z8.s, z10.s\n" + "fmla z29.s, p3/M, z7.s, z10.s\n" + "fmla z31.s, p3/M, z5.s, z10.s\n" + "fmla z30.s, p3/M, z0.s, z17.s\n" + "ld1w { z19.s }, p2/Z, [x16, x8, LSL #2]\n" + "fmla z20.s, p3/M, z2.s, z9.s\n" + "fmla z21.s, p3/M, z3.s, z16.s\n" + "fmla z22.s, p3/M, z4.s, z19.s\n" + "fmla z23.s, p3/M, z3.s, z19.s\n" + "fmla z27.s, p3/M, z3.s, z17.s\n" + "fmla z29.s, p3/M, z5.s, z9.s\n" + "ld1w { z17.s }, p2/Z, [x16, x13, LSL #2]\n" + "fmla z30.s, p3/M, z6.s, z16.s\n" + "fmla z31.s, p3/M, z7.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x14, x4, LSL #2]\n" + "fmla z20.s, p3/M, z6.s, z19.s\n" + "fmla z21.s, p3/M, z5.s, z19.s\n" + "ld1w { z18.s }, p2/Z, [x6, x4, LSL #2]\n" + "fmla z23.s, p3/M, z5.s, z17.s\n" + "fmla z22.s, p3/M, z6.s, z16.s\n" + "fmla z30.s, p3/M, z8.s, z19.s\n" + "fmla z20.s, p3/M, z8.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x6, x15, LSL #2]\n" "addvl x6, x6, #1\n" - "fmla z29.s, p3/M, z7.s, z13.s\n" - "fmla z24.s, p3/M, z3.s, z12.s\n" - "ld1w { z13.s }, p2/Z, [x14, x15, LSL #2]\n" - "fmla z27.s, p3/M, z0.s, z12.s\n" - "fmla z23.s, p3/M, z4.s, z12.s\n" - "fmla z30.s, p3/M, z8.s, z13.s\n" - "fmla z31.s, p3/M, z7.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x16, x15, LSL #2]\n" - "fmla z26.s, p3/M, z1.s, z12.s\n" - "fmla z24.s, p3/M, z5.s, z11.s\n" - "ld1w { z12.s }, p2/Z, [x16, x4, LSL #2]\n" + "fmla z21.s, p3/M, z7.s, z16.s\n" + "fmla z28.s, p3/M, z3.s, z18.s\n" + "ld1w { z16.s }, p2/Z, [x14, x15, LSL #2]\n" + "fmla z31.s, p3/M, z0.s, z18.s\n" + "fmla z27.s, p3/M, z4.s, z18.s\n" + "fmla z22.s, p3/M, z8.s, z16.s\n" + "fmla z23.s, p3/M, z7.s, z16.s\n" + "ld1w { z19.s }, p2/Z, [x16, x15, LSL #2]\n" + "fmla z30.s, p3/M, z1.s, z18.s\n" + "fmla z28.s, p3/M, z5.s, z17.s\n" + "ld1w { z11.s }, p2/Z, [x16, x4, LSL #2]\n" "addvl x16, x16, #1\n" - "fmla z25.s, p3/M, z4.s, z11.s\n" - "fmla z27.s, p3/M, z2.s, z11.s\n" - "fmla z28.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x5, x8, LSL #2]\n" - "fmla z29.s, p3/M, z4.s, z12.s\n" + "fmla z29.s, p3/M, z4.s, z17.s\n" + "fmla z31.s, p3/M, z2.s, z17.s\n" + "fmla z20.s, p3/M, z1.s, z17.s\n" + "ld1w { z16.s }, p2/Z, [x5, x8, LSL #2]\n" + "fmla z21.s, p3/M, z4.s, z11.s\n" "addvl x5, x5, #1\n" - "fmla z30.s, p3/M, z3.s, z12.s\n" - "fmla z23.s, p3/M, z2.s, z11.s\n" + "fmla z22.s, p3/M, z3.s, z11.s\n" + "fmla z27.s, p3/M, z2.s, z16.s\n" "ld1w { z10.s }, p1/Z, [x5]\n" - "fmla z31.s, p3/M, z4.s, z13.s\n" - "fmla z26.s, p3/M, z7.s, z12.s\n" - "fmla z27.s, p3/M, z6.s, z12.s\n" - "fmla z24.s, p3/M, z1.s, z11.s\n" - "ld1w { z12.s }, p2/Z, [x7]\n" - "fmla z25.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x7, x13, LSL #2]\n" - "fmla z28.s, p3/M, z7.s, z13.s\n" + "fmla z23.s, p3/M, z4.s, z19.s\n" + "fmla z30.s, p3/M, z7.s, z11.s\n" + "fmla z31.s, p3/M, z6.s, z11.s\n" + "fmla z28.s, p3/M, z1.s, z16.s\n" + "ld1w { z18.s }, p2/Z, [x7]\n" + "fmla z29.s, p3/M, z0.s, z16.s\n" + "ld1w { z17.s }, p2/Z, [x7, x13, LSL #2]\n" + "fmla z20.s, p3/M, z7.s, z19.s\n" "addvl x7, x7, #1\n" - "fmla z30.s, p3/M, z5.s, z13.s\n" - "fmla z23.s, p3/M, z6.s, z12.s\n" + "fmla z22.s, p3/M, z5.s, z19.s\n" + "fmla z27.s, p3/M, z6.s, z18.s\n" "ld1w { z9.s }, p1/Z, [x7, x8, LSL #2]\n" - "fmla z29.s, p3/M, z0.s, z12.s\n" - "fmla z31.s, p3/M, z2.s, z11.s\n" - "fmla z27.s, p3/M, z8.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x14, x8, LSL #2]\n" - "fmla z26.s, p3/M, z3.s, z12.s\n" + "fmla z21.s, p3/M, z0.s, z18.s\n" + "fmla z23.s, p3/M, z2.s, z17.s\n" + "fmla z31.s, p3/M, z8.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x14, x8, LSL #2]\n" + "fmla z30.s, p3/M, z3.s, z18.s\n" "whilelt p2.s, x21, %x[n_channels]\n" - "fmla z25.s, p3/M, z8.s, z11.s\n" - "fmla z28.s, p3/M, z5.s, z11.s\n" + "fmla z29.s, p3/M, z8.s, z17.s\n" + "fmla z20.s, p3/M, z5.s, z17.s\n" ".inst 0xa040c220 // ld1w { z0.s-z3.s }, pn8.b/Z, [x17]\n" "addvl x17, x17, #4\n" - "fmla z29.s, p3/M, z8.s, z13.s\n" - "fmla z30.s, p3/M, z7.s, z13.s\n" + "fmla z21.s, p3/M, z8.s, z16.s\n" + "fmla z22.s, p3/M, z7.s, z16.s\n" "addvl x14, x14, #1\n" "cmp x25, %x[n_channels]\n" - "fmla z31.s, p3/M, z6.s, z13.s\n" - "fmax z23.s, p3/M, z23.s, z17.s\n" + "fmla z23.s, p3/M, z6.s, z16.s\n" + "fmax z27.s, p3/M, z27.s, z26.s\n" ".inst 0xa040c224 // ld1w { z4.s-z7.s }, pn8.b/Z, [x17]\n" "addvl x17, x17, #4\n" - "fmin z23.s, p3/M, z23.s, z16.s\n" - ".inst 0xc1b0ca38 // fclamp { z24.s-z27.s }, z17.s, z16.s\n" + "fmin z27.s, p3/M, z27.s, z14.s\n" + ".inst 0xc1aecb5c // fclamp { z28.s-z31.s }, z26.s, z14.s\n" "ld1w { z11.s }, p1/Z, [x5, x13, LSL #2]\n" - ".inst 0xc1b0ca3c // fclamp { z28.s-z31.s }, z17.s, z16.s\n" + ".inst 0xc1aecb54 // fclamp { z20.s-z23.s }, z26.s, z14.s\n" "ld1w { z12.s }, p1/Z, [x14]\n" - "st1w { z23.s }, p0, [x26]\n" + "st1w { z27.s }, p0, [x26]\n" "ld1w { z13.s }, p1/Z, [x6, x8, LSL #2]\n" - "st1w { z24.s }, p0, [x26, x27, LSL #2]\n" - "st1w { z25.s }, p0, [x26, x22, LSL #2]\n" + "st1w { z28.s }, p0, [x26, x27, LSL #2]\n" + "st1w { z29.s }, p0, [x26, x22, LSL #2]\n" "addvl x26, x26, #1\n" "ld1w { z8.s }, p3/Z, [x17]\n" "addvl x17, x17, #1\n" - "st1w { z26.s }, p0, [x24]\n" - "st1w { z27.s }, p0, [x24, x27, LSL #2]\n" - "st1w { z28.s }, p0, [x24, x22, LSL #2]\n" + "st1w { z30.s }, p0, [x24]\n" + "st1w { z31.s }, p0, [x24, x27, LSL #2]\n" + "st1w { z20.s }, p0, [x24, x22, LSL #2]\n" "addvl x24, x24, #1\n" - "st1w { z29.s }, p0, [x23]\n" - "st1w { z30.s }, p0, [x23, x27, LSL #2]\n" - "st1w { z31.s }, p0, [x23, x22, LSL #2]\n" + "st1w { z21.s }, p0, [x23]\n" + "st1w { z22.s }, p0, [x23, x27, LSL #2]\n" + "st1w { z23.s }, p0, [x23, x22, LSL #2]\n" "addvl x23, x23, #1\n" "blt 3b\n" "4:" // Tile loop: Channel tail - "movprfx z24, z18\n fmla z24.s, p3/M, z7.s, z9.s\n" - "movprfx z23, z18\n fmla z23.s, p3/M, z8.s, z9.s\n" + "movprfx z28, z24\n fmla z28.s, p3/M, z7.s, z9.s\n" + "movprfx z25, z24\n fmla z25.s, p3/M, z8.s, z9.s\n" "ldr x3, [%x[params_struct], %[offsetof_args_tile_j]]\n" "add x3, x3, #0x1\n" - "movprfx z25, z18\n fmla z25.s, p3/M, z6.s, z9.s\n" - "fmla z24.s, p3/M, z4.s, z13.s\n" + "movprfx z29, z24\n fmla z29.s, p3/M, z6.s, z9.s\n" + "fmla z28.s, p3/M, z4.s, z13.s\n" "ldr x2, [%x[params_struct], %[offsetof_args_tile_i]]\n" "add x21, x2, #0x1\n" - "movprfx z26, z18\n fmla z26.s, p3/M, z5.s, z9.s\n" - "movprfx z27, z18\n fmla z27.s, p3/M, z4.s, z9.s\n" + "movprfx z30, z24\n fmla z30.s, p3/M, z5.s, z9.s\n" + "movprfx z31, z24\n fmla z31.s, p3/M, z4.s, z9.s\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" "cmp x3, x20\n" - "movprfx z28, z18\n fmla z28.s, p3/M, z3.s, z9.s\n" - "fmla z23.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x7, x15, LSL #2]\n" + "movprfx z20, z24\n fmla z20.s, p3/M, z3.s, z9.s\n" + "fmla z25.s, p3/M, z0.s, z10.s\n" + "ld1w { z27.s }, p2/Z, [x7, x15, LSL #2]\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" - "fmla z25.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x7, x4, LSL #2]\n" - "movprfx z29, z18\n fmla z29.s, p3/M, z2.s, z9.s\n" + "fmla z29.s, p3/M, z2.s, z11.s\n" + "ld1w { z17.s }, p2/Z, [x7, x4, LSL #2]\n" + "movprfx z21, z24\n fmla z21.s, p3/M, z2.s, z9.s\n" "csel x2, x2, x21, LT\n" - "fmla z24.s, p3/M, z6.s, z11.s\n" - "movprfx z31, z18\n fmla z31.s, p3/M, z0.s, z9.s\n" + "fmla z28.s, p3/M, z6.s, z17.s\n" + "movprfx z23, z24\n fmla z23.s, p3/M, z0.s, z9.s\n" "mov p0.b, p2.b\n" "csel x3, x3, XZR, LT\n" - "fmla z23.s, p3/M, z5.s, z13.s\n" - "fmla z25.s, p3/M, z3.s, z13.s\n" - "cmp x2, x20\n" - "fmla z26.s, p3/M, z2.s, z13.s\n" - "fmla z27.s, p3/M, z1.s, z13.s\n" - "fmla z28.s, p3/M, z0.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x5, x4, LSL #2]\n" - "fmla z29.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x14, x13, LSL #2]\n" - "movprfx z30, z18\n fmla z30.s, p3/M, z1.s, z9.s\n" - "fmla z24.s, p3/M, z0.s, z13.s\n" - "fmla z31.s, p3/M, z8.s, z12.s\n" - "fmla z23.s, p3/M, z7.s, z11.s\n" - "ld1w { z12.s }, p2/Z, [x5, x15, LSL #2]\n" - "fmla z30.s, p3/M, z0.s, z11.s\n" - "fmla z26.s, p3/M, z4.s, z11.s\n" - "fmla z27.s, p3/M, z3.s, z11.s\n" - "fmla z29.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x6]\n" - "fmla z24.s, p3/M, z2.s, z12.s\n" - "fmla z25.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x16]\n" - "fmla z28.s, p3/M, z4.s, z10.s\n" - "fmla z23.s, p3/M, z1.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x6, x13, LSL #2]\n" - "fmla z30.s, p3/M, z2.s, z10.s\n" - "fmla z31.s, p3/M, z1.s, z10.s\n" - "fmla z24.s, p3/M, z8.s, z10.s\n" - "fmla z25.s, p3/M, z7.s, z10.s\n" - "fmla z27.s, p3/M, z5.s, z10.s\n" - "fmla z26.s, p3/M, z0.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x16, x8, LSL #2]\n" - "fmla z28.s, p3/M, z2.s, z13.s\n" - "fmla z29.s, p3/M, z3.s, z12.s\n" - "fmla z30.s, p3/M, z4.s, z10.s\n" - "fmla z31.s, p3/M, z3.s, z10.s\n" - "fmla z23.s, p3/M, z3.s, z11.s\n" "fmla z25.s, p3/M, z5.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x16, x13, LSL #2]\n" - "fmla z26.s, p3/M, z6.s, z12.s\n" - "fmla z27.s, p3/M, z7.s, z10.s\n" - "ld1w { z13.s }, p2/Z, [x14, x4, LSL #2]\n" - "fmla z28.s, p3/M, z6.s, z10.s\n" - "fmla z29.s, p3/M, z5.s, z10.s\n" - "ld1w { z12.s }, p2/Z, [x6, x4, LSL #2]\n" - "fmla z31.s, p3/M, z5.s, z11.s\n" - "fmla z30.s, p3/M, z6.s, z13.s\n" - "fmla z26.s, p3/M, z8.s, z10.s\n" - "fmla z28.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x6, x15, LSL #2]\n" - "fmla z29.s, p3/M, z7.s, z13.s\n" - "fmla z24.s, p3/M, z3.s, z12.s\n" - "ld1w { z13.s }, p2/Z, [x14, x15, LSL #2]\n" - "fmla z27.s, p3/M, z0.s, z12.s\n" - "fmla z23.s, p3/M, z4.s, z12.s\n" - "fmla z30.s, p3/M, z8.s, z13.s\n" - "fmla z31.s, p3/M, z7.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x16, x15, LSL #2]\n" - "fmla z26.s, p3/M, z1.s, z12.s\n" - "fmla z24.s, p3/M, z5.s, z11.s\n" - "ld1w { z12.s }, p2/Z, [x16, x4, LSL #2]\n" - "fmla z25.s, p3/M, z4.s, z11.s\n" - "fmla z27.s, p3/M, z2.s, z11.s\n" - "fmla z28.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x5, x8, LSL #2]\n" - "fmla z29.s, p3/M, z4.s, z12.s\n" - "fmla z30.s, p3/M, z3.s, z12.s\n" - "fmla z23.s, p3/M, z2.s, z11.s\n" - "fmla z31.s, p3/M, z4.s, z13.s\n" - "fmla z26.s, p3/M, z7.s, z12.s\n" - "fmla z27.s, p3/M, z6.s, z12.s\n" - "fmla z24.s, p3/M, z1.s, z11.s\n" - "ld1w { z12.s }, p2/Z, [x7]\n" - "fmla z25.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x7, x13, LSL #2]\n" - "fmla z28.s, p3/M, z7.s, z13.s\n" - "fmla z30.s, p3/M, z5.s, z13.s\n" - "fmla z23.s, p3/M, z6.s, z12.s\n" - "fmla z29.s, p3/M, z0.s, z12.s\n" - "fmla z31.s, p3/M, z2.s, z11.s\n" - "fmla z27.s, p3/M, z8.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x14, x8, LSL #2]\n" - "fmla z26.s, p3/M, z3.s, z12.s\n" - "fmla z25.s, p3/M, z8.s, z11.s\n" - "fmla z28.s, p3/M, z5.s, z11.s\n" - "fmla z29.s, p3/M, z8.s, z13.s\n" - "fmla z30.s, p3/M, z7.s, z13.s\n" - "fmla z31.s, p3/M, z6.s, z13.s\n" - "fmax z23.s, p3/M, z23.s, z17.s\n" - "fmin z23.s, p3/M, z23.s, z16.s\n" - ".inst 0xc1b0ca38 // fclamp { z24.s-z27.s }, z17.s, z16.s\n" - "st1w { z23.s }, p0, [x26]\n" - ".inst 0xc1b0ca3c // fclamp { z28.s-z31.s }, z17.s, z16.s\n" - "st1w { z24.s }, p0, [x26, x27, LSL #2]\n" - "st1w { z25.s }, p0, [x26, x22, LSL #2]\n" - "st1w { z26.s }, p0, [x24]\n" - "st1w { z27.s }, p0, [x24, x27, LSL #2]\n" - "st1w { z28.s }, p0, [x24, x22, LSL #2]\n" - "st1w { z29.s }, p0, [x23]\n" - "st1w { z30.s }, p0, [x23, x27, LSL #2]\n" - "st1w { z31.s }, p0, [x23, x22, LSL #2]\n" + "fmla z29.s, p3/M, z3.s, z13.s\n" + "cmp x2, x20\n" + "fmla z30.s, p3/M, z2.s, z13.s\n" + "fmla z31.s, p3/M, z1.s, z13.s\n" + "fmla z20.s, p3/M, z0.s, z13.s\n" + "ld1w { z19.s }, p2/Z, [x5, x4, LSL #2]\n" + "fmla z21.s, p3/M, z6.s, z12.s\n" + "ld1w { z16.s }, p2/Z, [x14, x13, LSL #2]\n" + "movprfx z22, z24\n fmla z22.s, p3/M, z1.s, z9.s\n" + "fmla z28.s, p3/M, z0.s, z19.s\n" + "fmla z23.s, p3/M, z8.s, z16.s\n" + "fmla z25.s, p3/M, z7.s, z17.s\n" + "ld1w { z16.s }, p2/Z, [x5, x15, LSL #2]\n" + "fmla z22.s, p3/M, z0.s, z17.s\n" + "fmla z30.s, p3/M, z4.s, z17.s\n" + "fmla z31.s, p3/M, z3.s, z17.s\n" + "fmla z21.s, p3/M, z1.s, z17.s\n" + "ld1w { z18.s }, p2/Z, [x6]\n" + "fmla z28.s, p3/M, z2.s, z16.s\n" + "fmla z29.s, p3/M, z1.s, z16.s\n" + "ld1w { z17.s }, p2/Z, [x16]\n" + "fmla z20.s, p3/M, z4.s, z27.s\n" + "fmla z25.s, p3/M, z1.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x6, x13, LSL #2]\n" + "fmla z22.s, p3/M, z2.s, z27.s\n" + "fmla z23.s, p3/M, z1.s, z27.s\n" + "fmla z28.s, p3/M, z8.s, z27.s\n" + "fmla z29.s, p3/M, z7.s, z27.s\n" + "fmla z31.s, p3/M, z5.s, z27.s\n" + "fmla z30.s, p3/M, z0.s, z18.s\n" + "ld1w { z19.s }, p2/Z, [x16, x8, LSL #2]\n" + "fmla z20.s, p3/M, z2.s, z16.s\n" + "fmla z21.s, p3/M, z3.s, z17.s\n" + "fmla z22.s, p3/M, z4.s, z19.s\n" + "fmla z23.s, p3/M, z3.s, z19.s\n" + "fmla z25.s, p3/M, z3.s, z18.s\n" + "fmla z29.s, p3/M, z5.s, z16.s\n" + "ld1w { z18.s }, p2/Z, [x16, x13, LSL #2]\n" + "fmla z30.s, p3/M, z6.s, z17.s\n" + "fmla z31.s, p3/M, z7.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x14, x4, LSL #2]\n" + "fmla z20.s, p3/M, z6.s, z19.s\n" + "fmla z21.s, p3/M, z5.s, z19.s\n" + "ld1w { z17.s }, p2/Z, [x6, x4, LSL #2]\n" + "fmla z23.s, p3/M, z5.s, z18.s\n" + "fmla z22.s, p3/M, z6.s, z16.s\n" + "fmla z30.s, p3/M, z8.s, z19.s\n" + "fmla z20.s, p3/M, z8.s, z18.s\n" + "ld1w { z18.s }, p2/Z, [x6, x15, LSL #2]\n" + "fmla z21.s, p3/M, z7.s, z16.s\n" + "fmla z28.s, p3/M, z3.s, z17.s\n" + "ld1w { z16.s }, p2/Z, [x14, x15, LSL #2]\n" + "fmla z31.s, p3/M, z0.s, z17.s\n" + "fmla z25.s, p3/M, z4.s, z17.s\n" + "fmla z22.s, p3/M, z8.s, z16.s\n" + "fmla z23.s, p3/M, z7.s, z16.s\n" + "ld1w { z19.s }, p2/Z, [x16, x15, LSL #2]\n" + "fmla z30.s, p3/M, z1.s, z17.s\n" + "fmla z28.s, p3/M, z5.s, z18.s\n" + "ld1w { z17.s }, p2/Z, [x16, x4, LSL #2]\n" + "fmla z29.s, p3/M, z4.s, z18.s\n" + "fmla z31.s, p3/M, z2.s, z18.s\n" + "fmla z20.s, p3/M, z1.s, z18.s\n" + "ld1w { z16.s }, p2/Z, [x5, x8, LSL #2]\n" + "fmla z21.s, p3/M, z4.s, z17.s\n" + "fmla z22.s, p3/M, z3.s, z17.s\n" + "fmla z25.s, p3/M, z2.s, z16.s\n" + "fmla z23.s, p3/M, z4.s, z19.s\n" + "fmla z30.s, p3/M, z7.s, z17.s\n" + "fmla z31.s, p3/M, z6.s, z17.s\n" + "fmla z28.s, p3/M, z1.s, z16.s\n" + "ld1w { z18.s }, p2/Z, [x7]\n" + "fmla z29.s, p3/M, z0.s, z16.s\n" + "ld1w { z17.s }, p2/Z, [x7, x13, LSL #2]\n" + "fmla z20.s, p3/M, z7.s, z19.s\n" + "fmla z22.s, p3/M, z5.s, z19.s\n" + "fmla z25.s, p3/M, z6.s, z18.s\n" + "fmla z21.s, p3/M, z0.s, z18.s\n" + "fmla z23.s, p3/M, z2.s, z17.s\n" + "fmla z31.s, p3/M, z8.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x14, x8, LSL #2]\n" + "fmla z30.s, p3/M, z3.s, z18.s\n" + "fmla z29.s, p3/M, z8.s, z17.s\n" + "fmla z20.s, p3/M, z5.s, z17.s\n" + "fmla z21.s, p3/M, z8.s, z16.s\n" + "fmla z22.s, p3/M, z7.s, z16.s\n" + "fmla z23.s, p3/M, z6.s, z16.s\n" + "fmax z25.s, p3/M, z25.s, z26.s\n" + "fmin z25.s, p3/M, z25.s, z14.s\n" + ".inst 0xc1aecb5c // fclamp { z28.s-z31.s }, z26.s, z14.s\n" + "st1w { z25.s }, p0, [x26]\n" + ".inst 0xc1aecb54 // fclamp { z20.s-z23.s }, z26.s, z14.s\n" + "st1w { z28.s }, p0, [x26, x27, LSL #2]\n" + "st1w { z29.s }, p0, [x26, x22, LSL #2]\n" + "st1w { z30.s }, p0, [x24]\n" + "st1w { z31.s }, p0, [x24, x27, LSL #2]\n" + "st1w { z20.s }, p0, [x24, x22, LSL #2]\n" + "st1w { z21.s }, p0, [x23]\n" + "st1w { z22.s }, p0, [x23, x27, LSL #2]\n" + "st1w { z23.s }, p0, [x23, x22, LSL #2]\n" "blt 1b\n" ".inst 0xd503467f // SMSTOP\n" : diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp index fd648a392fcc157dee55ded6b7b7f0ad64d191dd..2c868b6cf307ea5c4c793d4e117ae30599fb9315 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp @@ -22,11 +22,11 @@ * SOFTWARE. */ -#if defined(ARM_COMPUTE_ENABLE_SME2) - #include #include +#if defined(ARM_COMPUTE_ENABLE_SME2) + namespace arm_conv { namespace depthwise { @@ -87,354 +87,354 @@ void sme2_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( activation_min, activation_max); __asm__ __volatile__( - "ldr x17, [%x[params_struct], %[offsetof_args_params]]\n" + "ldr x8, [%x[params_struct], %[offsetof_args_params]]\n" ".inst 0xd503477f // SMSTART ZA\n" - "add x16, %x[params_struct], %[offsetof_Args_inptrs]\n" + "add x17, %x[params_struct], %[offsetof_Args_inptrs]\n" "ptrue p3.b\n" ".inst 0x25207810 // ptrue pn8.b\n" - "ld1w { z18.s }, p3/Z, [x17]\n" - "addvl x17, x17, #1\n" - "ldp x15, x14, [x16, #0x0]\n" - "ldp x13, x12, [x16, #0x10]\n" - "cntw x11\n" - ".inst 0xa040c220 // ld1w { z0.s-z3.s }, pn8.b/Z, [x17]\n" - "addvl x17, x17, #4\n" - "ldr x10, [x16, #0x20]\n" - "mov x9, #0x0\n" + "ld1w { z20.s }, p3/Z, [x8]\n" + "addvl x8, x8, #1\n" + "ldp x24, x23, [x17, #0x0]\n" + "ldp x22, x21, [x17, #0x10]\n" + "cntw x16\n" + ".inst 0xa040c100 // ld1w { z0.s-z3.s }, pn8.b/Z, [x8]\n" + "addvl x8, x8, #4\n" + "ldr x20, [x17, #0x20]\n" + "mov x15, #0x0\n" "whilelt p2.s, XZR, %x[n_channels]\n" - ".inst 0xa040c224 // ld1w { z4.s-z7.s }, pn8.b/Z, [x17]\n" - "addvl x17, x17, #4\n" - "cmp x11, %x[n_channels]\n" - "ldr x28, [%x[params_struct], %[offsetof_args_outptrs]]\n" - "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "ld1rw { z16.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "sub x27, XZR, x11\n" - "ld1w { z8.s }, p3/Z, [x17]\n" - "addvl x17, x17, #1\n" - "ld1w { z9.s }, p2/Z, [x15, x9, LSL #2]\n" - "ld1w { z10.s }, p2/Z, [x14, x9, LSL #2]\n" - "ld1w { z11.s }, p2/Z, [x13, x9, LSL #2]\n" - "ld1w { z12.s }, p2/Z, [x12, x9, LSL #2]\n" - "ld1w { z13.s }, p2/Z, [x10, x9, LSL #2]\n" + ".inst 0xa040c104 // ld1w { z4.s-z7.s }, pn8.b/Z, [x8]\n" + "addvl x8, x8, #4\n" + "cmp x16, %x[n_channels]\n" + "ldr x14, [%x[params_struct], %[offsetof_args_outptrs]]\n" + "ld1rw { z22.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rw { z14.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "sub x13, XZR, x16\n" + "ld1w { z8.s }, p3/Z, [x8]\n" + "addvl x8, x8, #1\n" + "ld1w { z9.s }, p2/Z, [x24, x15, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x23, x15, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x22, x15, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x21, x15, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x20, x15, LSL #2]\n" "bge 2f\n" "1:" // Channel loop - "movprfx z23, z18\n fmla z23.s, p3/M, z8.s, z9.s\n" - "movprfx z24, z18\n fmla z24.s, p3/M, z7.s, z9.s\n" - "ldr x26, [x16, #0x30]\n" - "incw x27\n" - "movprfx z25, z18\n fmla z25.s, p3/M, z6.s, z9.s\n" - "fmla z23.s, p3/M, z0.s, z10.s\n" - "ldr x25, [x16, #0x38]\n" + "movprfx z21, z20\n fmla z21.s, p3/M, z8.s, z9.s\n" + "movprfx z24, z20\n fmla z24.s, p3/M, z7.s, z9.s\n" + "ldr x22, [x17, #0x30]\n" + "incw x13\n" + "movprfx z25, z20\n fmla z25.s, p3/M, z6.s, z9.s\n" + "fmla z21.s, p3/M, z0.s, z10.s\n" + "ldr x25, [x17, #0x38]\n" "mov p1.b, p2.b\n" "fmla z24.s, p3/M, z4.s, z13.s\n" - "movprfx z26, z18\n fmla z26.s, p3/M, z5.s, z9.s\n" - "ldr x24, [x16, #0x28]\n" - "whilelt p0.s, x11, %x[n_channels]\n" - "movprfx z27, z18\n fmla z27.s, p3/M, z4.s, z9.s\n" - "movprfx z28, z18\n fmla z28.s, p3/M, z3.s, z9.s\n" - "ldr x14, [x16, #0x48]\n" - "ld1w { z10.s }, p2/Z, [x14, x9, LSL #2]\n" + "movprfx z26, z20\n fmla z26.s, p3/M, z5.s, z9.s\n" + "ldr x21, [x17, #0x28]\n" + "whilelt p0.s, x16, %x[n_channels]\n" + "movprfx z27, z20\n fmla z27.s, p3/M, z4.s, z9.s\n" + "movprfx z28, z20\n fmla z28.s, p3/M, z3.s, z9.s\n" + "ldr x20, [x17, #0x48]\n" + "ld1w { z19.s }, p2/Z, [x20, x15, LSL #2]\n" "fmla z25.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x9, LSL #2]\n" - "movprfx z29, z18\n fmla z29.s, p3/M, z2.s, z9.s\n" - "ldr x15, [x16, #0x40]\n" - "fmla z23.s, p3/M, z5.s, z13.s\n" - "fmla z24.s, p3/M, z6.s, z11.s\n" - "ldr x13, [x16, #0x50]\n" - "movprfx z31, z18\n fmla z31.s, p3/M, z0.s, z9.s\n" + "ld1w { z23.s }, p2/Z, [x22, x15, LSL #2]\n" + "movprfx z29, z20\n fmla z29.s, p3/M, z2.s, z9.s\n" + "ldr x20, [x17, #0x40]\n" + "fmla z21.s, p3/M, z5.s, z13.s\n" + "fmla z24.s, p3/M, z6.s, z23.s\n" + "ldr x24, [x17, #0x50]\n" + "movprfx z31, z20\n fmla z31.s, p3/M, z0.s, z9.s\n" "fmla z25.s, p3/M, z3.s, z13.s\n" - "ldr x12, [x16, #0x58]\n" + "ldr x23, [x17, #0x58]\n" "fmla z26.s, p3/M, z2.s, z13.s\n" "fmla z27.s, p3/M, z1.s, z13.s\n" - "ldr x10, [x16, #0x60]\n" + "ldr x22, [x17, #0x60]\n" "fmla z28.s, p3/M, z0.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x25, x9, LSL #2]\n" + "ld1w { z17.s }, p2/Z, [x25, x15, LSL #2]\n" "fmla z29.s, p3/M, z6.s, z12.s\n" - "ldr x26, [x16, #0x70]\n" - "ld1w { z12.s }, p2/Z, [x24, x9, LSL #2]\n" - "movprfx z30, z18\n fmla z30.s, p3/M, z1.s, z9.s\n" - "fmla z23.s, p3/M, z7.s, z11.s\n" - "ldr x24, [x16, #0x68]\n" - "fmla z24.s, p3/M, z0.s, z13.s\n" - "fmla z31.s, p3/M, z8.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x15, x9, LSL #2]\n" - "ldr x25, [x16, #0x78]\n" - "fmla z26.s, p3/M, z4.s, z11.s\n" - "fmla z27.s, p3/M, z3.s, z11.s\n" - "ldr x15, [x16, #0x80]\n" - "ld1w { z18.s }, p3/Z, [x17]\n" - "fmla z30.s, p3/M, z0.s, z11.s\n" - "fmla z28.s, p3/M, z4.s, z10.s\n" - "ldr x14, [x16, #0x88]\n" - "addvl x17, x17, #1\n" - "fmla z29.s, p3/M, z1.s, z11.s\n" - "fmla z23.s, p3/M, z1.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x13, x9, LSL #2]\n" - "ldr x13, [x16, #0x90]\n" - "fmla z24.s, p3/M, z2.s, z12.s\n" - "fmla z25.s, p3/M, z1.s, z12.s\n" - "ld1w { z13.s }, p2/Z, [x12, x9, LSL #2]\n" - "ldr x12, [x16, #0x98]\n" - "ld1w { z12.s }, p2/Z, [x10, x9, LSL #2]\n" - "fmla z27.s, p3/M, z5.s, z10.s\n" - "fmla z30.s, p3/M, z2.s, z10.s\n" - "ldr x10, [x16, #0xa0]\n" - "fmla z26.s, p3/M, z0.s, z11.s\n" - "fmla z28.s, p3/M, z2.s, z13.s\n" - "ldr x23, [x28, #0x0]\n" - "fmla z24.s, p3/M, z8.s, z10.s\n" - "fmla z25.s, p3/M, z7.s, z10.s\n" - "ldr x22, [x28, #0x8]\n" - "fmla z31.s, p3/M, z1.s, z10.s\n" - "fmla z29.s, p3/M, z3.s, z12.s\n" - "ld1w { z10.s }, p2/Z, [x24, x9, LSL #2]\n" - "ldr x24, [x16, #0xa8]\n" - "fmla z26.s, p3/M, z6.s, z12.s\n" - "fmla z27.s, p3/M, z7.s, z10.s\n" - "ld1w { z12.s }, p2/Z, [x15, x9, LSL #2]\n" - "ldr x15, [x16, #0xc0]\n" - "fmla z28.s, p3/M, z6.s, z10.s\n" - "fmla z30.s, p3/M, z4.s, z10.s\n" - "ldr x21, [x28, #0x10]\n" - "fmla z23.s, p3/M, z3.s, z11.s\n" - "fmla z25.s, p3/M, z5.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x26, x9, LSL #2]\n" - "ldr x26, [x16, #0xb0]\n" - "fmla z29.s, p3/M, z5.s, z10.s\n" - "fmla z31.s, p3/M, z3.s, z10.s\n" - "ld1w { z13.s }, p2/Z, [x25, x9, LSL #2]\n" - "ldr x25, [x16, #0xb8]\n" - "fmla z26.s, p3/M, z8.s, z10.s\n" - "fmla z28.s, p3/M, z8.s, z11.s\n" - "ldr x20, [x28, #0x18]\n" - "fmla z30.s, p3/M, z6.s, z13.s\n" - "fmla z24.s, p3/M, z3.s, z12.s\n" - "fmla z27.s, p3/M, z0.s, z12.s\n" - "fmla z31.s, p3/M, z5.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x14, x9, LSL #2]\n" - "fmla z29.s, p3/M, z7.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x13, x9, LSL #2]\n" - "fmla z23.s, p3/M, z4.s, z12.s\n" - "fmla z26.s, p3/M, z1.s, z12.s\n" - "fmla z24.s, p3/M, z5.s, z11.s\n" - "ld1w { z12.s }, p2/Z, [x12, x9, LSL #2]\n" - "fmla z25.s, p3/M, z4.s, z11.s\n" - "fmla z27.s, p3/M, z2.s, z11.s\n" - "fmla z28.s, p3/M, z1.s, z11.s\n" - "fmla z30.s, p3/M, z8.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x10, x9, LSL #2]\n" - "ldr x10, [x16, #0x20]\n" - "fmla z23.s, p3/M, z2.s, z11.s\n" - "fmla z26.s, p3/M, z7.s, z12.s\n" - "fmla z27.s, p3/M, z6.s, z12.s\n" - "fmla z29.s, p3/M, z4.s, z12.s\n" - "fmla z30.s, p3/M, z3.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x26, x9, LSL #2]\n" - "fmla z31.s, p3/M, z7.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x24, x9, LSL #2]\n" - "fmla z23.s, p3/M, z6.s, z12.s\n" - "fmla z31.s, p3/M, z4.s, z13.s\n" - "fmla z24.s, p3/M, z1.s, z11.s\n" - "fmla z25.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x25, x9, LSL #2]\n" - "fmax z23.s, p3/M, z23.s, z17.s\n" - "fmla z28.s, p3/M, z7.s, z13.s\n" - "fmla z30.s, p3/M, z5.s, z13.s\n" - "fmla z29.s, p3/M, z0.s, z12.s\n" - "fmla z31.s, p3/M, z2.s, z11.s\n" - "fmla z27.s, p3/M, z8.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x15, x9, LSL #2]\n" - "ldp x15, x14, [x16, #0x0]\n" - "fmla z26.s, p3/M, z3.s, z12.s\n" - "fmla z25.s, p3/M, z8.s, z11.s\n" - "ldp x13, x12, [x16, #0x10]\n" - "incw x9\n" - "fmin z23.s, p3/M, z23.s, z16.s\n" - "st1w { z23.s }, p1, [x23, x27, LSL #2]\n" - "ldr x23, [x28, #0x20]\n" - "fmla z28.s, p3/M, z5.s, z11.s\n" - "fmla z29.s, p3/M, z8.s, z13.s\n" - "fmla z30.s, p3/M, z7.s, z13.s\n" - "ld1w { z9.s }, p0/Z, [x15, x11, LSL #2]\n" - "whilelt p2.s, x9, %x[n_channels]\n" - "fmla z31.s, p3/M, z6.s, z13.s\n" - ".inst 0xc1b0ca38 // fclamp { z24.s-z27.s }, z17.s, z16.s\n" - "st1w { z24.s }, p1, [x22, x27, LSL #2]\n" - "ldr x22, [x28, #0x28]\n" - "st1w { z25.s }, p1, [x21, x27, LSL #2]\n" - "ldr x21, [x28, #0x30]\n" - "ld1w { z10.s }, p0/Z, [x14, x11, LSL #2]\n" - ".inst 0xc1b0ca3c // fclamp { z28.s-z31.s }, z17.s, z16.s\n" - "st1w { z26.s }, p1, [x20, x27, LSL #2]\n" - "ldr x20, [x28, #0x38]\n" - "ld1w { z11.s }, p0/Z, [x13, x11, LSL #2]\n" - "st1w { z27.s }, p1, [x23, x27, LSL #2]\n" - "ldr x23, [x28, #0x40]\n" - "ld1w { z12.s }, p0/Z, [x12, x11, LSL #2]\n" - "ld1w { z13.s }, p0/Z, [x10, x11, LSL #2]\n" - "incw x11\n" - "cmp x11, %x[n_channels]\n" - "st1w { z28.s }, p1, [x22, x27, LSL #2]\n" - ".inst 0xa040c220 // ld1w { z0.s-z3.s }, pn8.b/Z, [x17]\n" - "addvl x17, x17, #4\n" - "st1w { z29.s }, p1, [x21, x27, LSL #2]\n" - ".inst 0xa040c224 // ld1w { z4.s-z7.s }, pn8.b/Z, [x17]\n" - "addvl x17, x17, #4\n" - "st1w { z30.s }, p1, [x20, x27, LSL #2]\n" - "st1w { z31.s }, p1, [x23, x27, LSL #2]\n" - "ld1w { z8.s }, p3/Z, [x17]\n" - "addvl x17, x17, #1\n" + "ldr x12, [x17, #0x70]\n" + "ld1w { z16.s }, p2/Z, [x21, x15, LSL #2]\n" + "movprfx z30, z20\n fmla z30.s, p3/M, z1.s, z9.s\n" + "fmla z21.s, p3/M, z7.s, z23.s\n" + "ldr x21, [x17, #0x68]\n" + "fmla z24.s, p3/M, z0.s, z17.s\n" + "fmla z31.s, p3/M, z8.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x20, x15, LSL #2]\n" + "ldr x27, [x17, #0x78]\n" + "fmla z26.s, p3/M, z4.s, z23.s\n" + "fmla z27.s, p3/M, z3.s, z23.s\n" + "ldr x20, [x17, #0x80]\n" + "ld1w { z20.s }, p3/Z, [x8]\n" + "fmla z30.s, p3/M, z0.s, z23.s\n" + "fmla z28.s, p3/M, z4.s, z19.s\n" + "ldr x11, [x17, #0x88]\n" + "addvl x8, x8, #1\n" + "fmla z29.s, p3/M, z1.s, z23.s\n" + "fmla z21.s, p3/M, z1.s, z17.s\n" + "ld1w { z18.s }, p2/Z, [x24, x15, LSL #2]\n" + "ldr x26, [x17, #0x90]\n" + "fmla z24.s, p3/M, z2.s, z16.s\n" + "fmla z25.s, p3/M, z1.s, z16.s\n" + "ld1w { z11.s }, p2/Z, [x23, x15, LSL #2]\n" + "ldr x25, [x17, #0x98]\n" + "ld1w { z17.s }, p2/Z, [x22, x15, LSL #2]\n" + "fmla z27.s, p3/M, z5.s, z19.s\n" + "fmla z30.s, p3/M, z2.s, z19.s\n" + "ldr x24, [x17, #0xa0]\n" + "fmla z26.s, p3/M, z0.s, z18.s\n" + "fmla z28.s, p3/M, z2.s, z11.s\n" + "ldr x10, [x14, #0x0]\n" + "fmla z24.s, p3/M, z8.s, z19.s\n" + "fmla z25.s, p3/M, z7.s, z19.s\n" + "ldr x9, [x14, #0x8]\n" + "fmla z31.s, p3/M, z1.s, z19.s\n" + "fmla z29.s, p3/M, z3.s, z17.s\n" + "ld1w { z16.s }, p2/Z, [x21, x15, LSL #2]\n" + "ldr x23, [x17, #0xa8]\n" + "fmla z26.s, p3/M, z6.s, z17.s\n" + "fmla z27.s, p3/M, z7.s, z16.s\n" + "ld1w { z23.s }, p2/Z, [x20, x15, LSL #2]\n" + "ldr x22, [x17, #0xc0]\n" + "fmla z28.s, p3/M, z6.s, z16.s\n" + "fmla z30.s, p3/M, z4.s, z16.s\n" + "ldr x28, [x14, #0x10]\n" + "fmla z21.s, p3/M, z3.s, z18.s\n" + "fmla z25.s, p3/M, z5.s, z11.s\n" + "ld1w { z15.s }, p2/Z, [x12, x15, LSL #2]\n" + "ldr x21, [x17, #0xb0]\n" + "fmla z29.s, p3/M, z5.s, z16.s\n" + "fmla z31.s, p3/M, z3.s, z16.s\n" + "ld1w { z19.s }, p2/Z, [x27, x15, LSL #2]\n" + "ldr x20, [x17, #0xb8]\n" + "fmla z26.s, p3/M, z8.s, z16.s\n" + "fmla z28.s, p3/M, z8.s, z15.s\n" + "ldr x27, [x14, #0x18]\n" + "fmla z30.s, p3/M, z6.s, z19.s\n" + "fmla z24.s, p3/M, z3.s, z23.s\n" + "fmla z27.s, p3/M, z0.s, z23.s\n" + "fmla z31.s, p3/M, z5.s, z15.s\n" + "ld1w { z17.s }, p2/Z, [x11, x15, LSL #2]\n" + "fmla z29.s, p3/M, z7.s, z19.s\n" + "ld1w { z19.s }, p2/Z, [x26, x15, LSL #2]\n" + "fmla z21.s, p3/M, z4.s, z23.s\n" + "fmla z26.s, p3/M, z1.s, z23.s\n" + "fmla z24.s, p3/M, z5.s, z17.s\n" + "ld1w { z16.s }, p2/Z, [x25, x15, LSL #2]\n" + "fmla z25.s, p3/M, z4.s, z17.s\n" + "fmla z27.s, p3/M, z2.s, z17.s\n" + "fmla z28.s, p3/M, z1.s, z17.s\n" + "fmla z30.s, p3/M, z8.s, z19.s\n" + "ld1w { z17.s }, p2/Z, [x24, x15, LSL #2]\n" + "ldr x26, [x17, #0x20]\n" + "fmla z21.s, p3/M, z2.s, z17.s\n" + "fmla z26.s, p3/M, z7.s, z16.s\n" + "fmla z27.s, p3/M, z6.s, z16.s\n" + "fmla z29.s, p3/M, z4.s, z16.s\n" + "fmla z30.s, p3/M, z3.s, z16.s\n" + "ld1w { z18.s }, p2/Z, [x21, x15, LSL #2]\n" + "fmla z31.s, p3/M, z7.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x23, x15, LSL #2]\n" + "fmla z21.s, p3/M, z6.s, z18.s\n" + "fmla z31.s, p3/M, z4.s, z16.s\n" + "fmla z24.s, p3/M, z1.s, z17.s\n" + "fmla z25.s, p3/M, z0.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x20, x15, LSL #2]\n" + "fmax z21.s, p3/M, z21.s, z22.s\n" + "fmla z28.s, p3/M, z7.s, z16.s\n" + "fmla z30.s, p3/M, z5.s, z16.s\n" + "fmla z29.s, p3/M, z0.s, z18.s\n" + "fmla z31.s, p3/M, z2.s, z17.s\n" + "fmla z27.s, p3/M, z8.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x22, x15, LSL #2]\n" + "ldp x22, x21, [x17, #0x0]\n" + "fmla z26.s, p3/M, z3.s, z18.s\n" + "fmla z25.s, p3/M, z8.s, z17.s\n" + "ldp x25, x24, [x17, #0x10]\n" + "incw x15\n" + "fmin z21.s, p3/M, z21.s, z14.s\n" + "st1w { z21.s }, p1, [x10, x13, LSL #2]\n" + "ldr x20, [x14, #0x20]\n" + "fmla z28.s, p3/M, z5.s, z17.s\n" + "fmla z29.s, p3/M, z8.s, z16.s\n" + "fmla z30.s, p3/M, z7.s, z16.s\n" + "ld1w { z9.s }, p0/Z, [x22, x16, LSL #2]\n" + "whilelt p2.s, x15, %x[n_channels]\n" + "fmla z31.s, p3/M, z6.s, z16.s\n" + ".inst 0xc1aecad8 // fclamp { z24.s-z27.s }, z22.s, z14.s\n" + "st1w { z24.s }, p1, [x9, x13, LSL #2]\n" + "ldr x23, [x14, #0x28]\n" + "st1w { z25.s }, p1, [x28, x13, LSL #2]\n" + "ldr x22, [x14, #0x30]\n" + "ld1w { z10.s }, p0/Z, [x21, x16, LSL #2]\n" + ".inst 0xc1aecadc // fclamp { z28.s-z31.s }, z22.s, z14.s\n" + "st1w { z26.s }, p1, [x27, x13, LSL #2]\n" + "ldr x21, [x14, #0x38]\n" + "ld1w { z11.s }, p0/Z, [x25, x16, LSL #2]\n" + "st1w { z27.s }, p1, [x20, x13, LSL #2]\n" + "ldr x20, [x14, #0x40]\n" + "ld1w { z12.s }, p0/Z, [x24, x16, LSL #2]\n" + "ld1w { z13.s }, p0/Z, [x26, x16, LSL #2]\n" + "incw x16\n" + "cmp x16, %x[n_channels]\n" + "st1w { z28.s }, p1, [x23, x13, LSL #2]\n" + ".inst 0xa040c100 // ld1w { z0.s-z3.s }, pn8.b/Z, [x8]\n" + "addvl x8, x8, #4\n" + "st1w { z29.s }, p1, [x22, x13, LSL #2]\n" + ".inst 0xa040c104 // ld1w { z4.s-z7.s }, pn8.b/Z, [x8]\n" + "addvl x8, x8, #4\n" + "st1w { z30.s }, p1, [x21, x13, LSL #2]\n" + "st1w { z31.s }, p1, [x20, x13, LSL #2]\n" + "ld1w { z8.s }, p3/Z, [x8]\n" + "addvl x8, x8, #1\n" "blt 1b\n" "2:" // Channel tail - "movprfx z23, z18\n fmla z23.s, p3/M, z8.s, z9.s\n" - "movprfx z24, z18\n fmla z24.s, p3/M, z7.s, z9.s\n" - "ldr x26, [x16, #0x30]\n" - "incw x27\n" - "movprfx z25, z18\n fmla z25.s, p3/M, z6.s, z9.s\n" - "fmla z23.s, p3/M, z0.s, z10.s\n" - "ldr x25, [x16, #0x38]\n" - "mov p1.b, p2.b\n" + "movprfx z21, z20\n fmla z21.s, p3/M, z8.s, z9.s\n" + "movprfx z24, z20\n fmla z24.s, p3/M, z7.s, z9.s\n" + "ldr x23, [x17, #0x30]\n" + "incw x13\n" + "movprfx z25, z20\n fmla z25.s, p3/M, z6.s, z9.s\n" + "fmla z21.s, p3/M, z0.s, z10.s\n" + "ldr x22, [x17, #0x38]\n" + "mov p0.b, p2.b\n" "fmla z24.s, p3/M, z4.s, z13.s\n" - "movprfx z26, z18\n fmla z26.s, p3/M, z5.s, z9.s\n" - "ldr x24, [x16, #0x28]\n" - "movprfx z27, z18\n fmla z27.s, p3/M, z4.s, z9.s\n" - "movprfx z28, z18\n fmla z28.s, p3/M, z3.s, z9.s\n" - "ldr x14, [x16, #0x48]\n" - "ld1w { z10.s }, p2/Z, [x14, x9, LSL #2]\n" + "movprfx z26, z20\n fmla z26.s, p3/M, z5.s, z9.s\n" + "ldr x21, [x17, #0x28]\n" + "movprfx z27, z20\n fmla z27.s, p3/M, z4.s, z9.s\n" + "movprfx z28, z20\n fmla z28.s, p3/M, z3.s, z9.s\n" + "ldr x20, [x17, #0x48]\n" + "ld1w { z19.s }, p2/Z, [x20, x15, LSL #2]\n" "fmla z25.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x9, LSL #2]\n" - "movprfx z29, z18\n fmla z29.s, p3/M, z2.s, z9.s\n" - "ldr x15, [x16, #0x40]\n" - "fmla z23.s, p3/M, z5.s, z13.s\n" - "fmla z24.s, p3/M, z6.s, z11.s\n" - "ldr x13, [x16, #0x50]\n" - "movprfx z31, z18\n fmla z31.s, p3/M, z0.s, z9.s\n" + "ld1w { z18.s }, p2/Z, [x23, x15, LSL #2]\n" + "movprfx z29, z20\n fmla z29.s, p3/M, z2.s, z9.s\n" + "ldr x20, [x17, #0x40]\n" + "fmla z21.s, p3/M, z5.s, z13.s\n" + "fmla z24.s, p3/M, z6.s, z18.s\n" + "ldr x25, [x17, #0x50]\n" + "movprfx z31, z20\n fmla z31.s, p3/M, z0.s, z9.s\n" "fmla z25.s, p3/M, z3.s, z13.s\n" - "ldr x12, [x16, #0x58]\n" + "ldr x24, [x17, #0x58]\n" "fmla z26.s, p3/M, z2.s, z13.s\n" "fmla z27.s, p3/M, z1.s, z13.s\n" - "ldr x10, [x16, #0x60]\n" + "ldr x23, [x17, #0x60]\n" "fmla z28.s, p3/M, z0.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x25, x9, LSL #2]\n" + "ld1w { z17.s }, p2/Z, [x22, x15, LSL #2]\n" "fmla z29.s, p3/M, z6.s, z12.s\n" - "ldr x26, [x16, #0x70]\n" - "ld1w { z12.s }, p2/Z, [x24, x9, LSL #2]\n" - "movprfx z30, z18\n fmla z30.s, p3/M, z1.s, z9.s\n" - "fmla z23.s, p3/M, z7.s, z11.s\n" - "ldr x24, [x16, #0x68]\n" - "fmla z24.s, p3/M, z0.s, z13.s\n" - "fmla z31.s, p3/M, z8.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x15, x9, LSL #2]\n" - "ldr x25, [x16, #0x78]\n" - "fmla z26.s, p3/M, z4.s, z11.s\n" - "fmla z27.s, p3/M, z3.s, z11.s\n" - "ldr x15, [x16, #0x80]\n" - "fmla z30.s, p3/M, z0.s, z11.s\n" - "fmla z28.s, p3/M, z4.s, z10.s\n" - "ldr x14, [x16, #0x88]\n" - "fmla z29.s, p3/M, z1.s, z11.s\n" - "fmla z23.s, p3/M, z1.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x13, x9, LSL #2]\n" - "ldr x13, [x16, #0x90]\n" - "fmla z24.s, p3/M, z2.s, z12.s\n" - "fmla z25.s, p3/M, z1.s, z12.s\n" - "ld1w { z13.s }, p2/Z, [x12, x9, LSL #2]\n" - "ldr x12, [x16, #0x98]\n" - "ld1w { z12.s }, p2/Z, [x10, x9, LSL #2]\n" - "fmla z27.s, p3/M, z5.s, z10.s\n" - "fmla z30.s, p3/M, z2.s, z10.s\n" - "ldr x10, [x16, #0xa0]\n" - "fmla z26.s, p3/M, z0.s, z11.s\n" - "fmla z28.s, p3/M, z2.s, z13.s\n" - "ldr x23, [x28, #0x0]\n" - "fmla z24.s, p3/M, z8.s, z10.s\n" - "fmla z25.s, p3/M, z7.s, z10.s\n" - "ldr x22, [x28, #0x8]\n" - "fmla z31.s, p3/M, z1.s, z10.s\n" - "fmla z29.s, p3/M, z3.s, z12.s\n" - "ld1w { z10.s }, p2/Z, [x24, x9, LSL #2]\n" - "ldr x24, [x16, #0xa8]\n" - "fmla z26.s, p3/M, z6.s, z12.s\n" - "fmla z27.s, p3/M, z7.s, z10.s\n" - "ld1w { z12.s }, p2/Z, [x15, x9, LSL #2]\n" - "ldr x15, [x16, #0xc0]\n" - "fmla z28.s, p3/M, z6.s, z10.s\n" - "fmla z30.s, p3/M, z4.s, z10.s\n" - "ldr x21, [x28, #0x10]\n" - "fmla z23.s, p3/M, z3.s, z11.s\n" - "fmla z25.s, p3/M, z5.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x26, x9, LSL #2]\n" - "ldr x26, [x16, #0xb0]\n" - "fmla z29.s, p3/M, z5.s, z10.s\n" - "fmla z31.s, p3/M, z3.s, z10.s\n" - "ld1w { z13.s }, p2/Z, [x25, x9, LSL #2]\n" - "ldr x25, [x16, #0xb8]\n" - "fmla z26.s, p3/M, z8.s, z10.s\n" - "fmla z28.s, p3/M, z8.s, z11.s\n" - "ldr x20, [x28, #0x18]\n" - "fmla z30.s, p3/M, z6.s, z13.s\n" - "fmla z24.s, p3/M, z3.s, z12.s\n" - "fmla z27.s, p3/M, z0.s, z12.s\n" - "fmla z31.s, p3/M, z5.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x14, x9, LSL #2]\n" - "fmla z29.s, p3/M, z7.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x13, x9, LSL #2]\n" - "fmla z23.s, p3/M, z4.s, z12.s\n" - "fmla z26.s, p3/M, z1.s, z12.s\n" - "fmla z24.s, p3/M, z5.s, z11.s\n" - "ld1w { z12.s }, p2/Z, [x12, x9, LSL #2]\n" - "fmla z25.s, p3/M, z4.s, z11.s\n" - "fmla z27.s, p3/M, z2.s, z11.s\n" - "fmla z28.s, p3/M, z1.s, z11.s\n" - "fmla z30.s, p3/M, z8.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x10, x9, LSL #2]\n" - "fmla z23.s, p3/M, z2.s, z11.s\n" - "fmla z26.s, p3/M, z7.s, z12.s\n" - "fmla z27.s, p3/M, z6.s, z12.s\n" - "fmla z29.s, p3/M, z4.s, z12.s\n" - "fmla z30.s, p3/M, z3.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x26, x9, LSL #2]\n" - "fmla z31.s, p3/M, z7.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x24, x9, LSL #2]\n" - "fmla z23.s, p3/M, z6.s, z12.s\n" - "fmla z31.s, p3/M, z4.s, z13.s\n" - "fmla z24.s, p3/M, z1.s, z11.s\n" - "fmla z25.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x25, x9, LSL #2]\n" - "fmax z23.s, p3/M, z23.s, z17.s\n" - "fmla z28.s, p3/M, z7.s, z13.s\n" - "fmla z30.s, p3/M, z5.s, z13.s\n" - "fmla z29.s, p3/M, z0.s, z12.s\n" - "fmla z31.s, p3/M, z2.s, z11.s\n" - "fmla z27.s, p3/M, z8.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x15, x9, LSL #2]\n" - "fmla z26.s, p3/M, z3.s, z12.s\n" - "fmla z25.s, p3/M, z8.s, z11.s\n" - "fmin z23.s, p3/M, z23.s, z16.s\n" - "st1w { z23.s }, p1, [x23, x27, LSL #2]\n" - "ldr x23, [x28, #0x20]\n" - "fmla z28.s, p3/M, z5.s, z11.s\n" - "fmla z29.s, p3/M, z8.s, z13.s\n" - "fmla z30.s, p3/M, z7.s, z13.s\n" - "fmla z31.s, p3/M, z6.s, z13.s\n" - ".inst 0xc1b0ca38 // fclamp { z24.s-z27.s }, z17.s, z16.s\n" - "st1w { z24.s }, p1, [x22, x27, LSL #2]\n" - "ldr x22, [x28, #0x28]\n" - "st1w { z25.s }, p1, [x21, x27, LSL #2]\n" - "ldr x21, [x28, #0x30]\n" - ".inst 0xc1b0ca3c // fclamp { z28.s-z31.s }, z17.s, z16.s\n" - "st1w { z26.s }, p1, [x20, x27, LSL #2]\n" - "ldr x20, [x28, #0x38]\n" - "st1w { z27.s }, p1, [x23, x27, LSL #2]\n" - "ldr x23, [x28, #0x40]\n" - "st1w { z28.s }, p1, [x22, x27, LSL #2]\n" - "st1w { z29.s }, p1, [x21, x27, LSL #2]\n" - "st1w { z30.s }, p1, [x20, x27, LSL #2]\n" - "st1w { z31.s }, p1, [x23, x27, LSL #2]\n" + "ldr x12, [x17, #0x70]\n" + "ld1w { z16.s }, p2/Z, [x21, x15, LSL #2]\n" + "movprfx z30, z20\n fmla z30.s, p3/M, z1.s, z9.s\n" + "fmla z21.s, p3/M, z7.s, z18.s\n" + "ldr x22, [x17, #0x68]\n" + "fmla z24.s, p3/M, z0.s, z17.s\n" + "fmla z31.s, p3/M, z8.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x20, x15, LSL #2]\n" + "ldr x21, [x17, #0x78]\n" + "fmla z26.s, p3/M, z4.s, z18.s\n" + "fmla z27.s, p3/M, z3.s, z18.s\n" + "ldr x20, [x17, #0x80]\n" + "fmla z30.s, p3/M, z0.s, z18.s\n" + "fmla z28.s, p3/M, z4.s, z19.s\n" + "ldr x11, [x17, #0x88]\n" + "fmla z29.s, p3/M, z1.s, z18.s\n" + "fmla z21.s, p3/M, z1.s, z17.s\n" + "ld1w { z20.s }, p2/Z, [x25, x15, LSL #2]\n" + "ldr x10, [x17, #0x90]\n" + "fmla z24.s, p3/M, z2.s, z16.s\n" + "fmla z25.s, p3/M, z1.s, z16.s\n" + "ld1w { z17.s }, p2/Z, [x24, x15, LSL #2]\n" + "ldr x9, [x17, #0x98]\n" + "ld1w { z16.s }, p2/Z, [x23, x15, LSL #2]\n" + "fmla z27.s, p3/M, z5.s, z19.s\n" + "fmla z30.s, p3/M, z2.s, z19.s\n" + "ldr x28, [x17, #0xa0]\n" + "fmla z26.s, p3/M, z0.s, z20.s\n" + "fmla z28.s, p3/M, z2.s, z17.s\n" + "ldr x27, [x14, #0x0]\n" + "fmla z24.s, p3/M, z8.s, z19.s\n" + "fmla z25.s, p3/M, z7.s, z19.s\n" + "ldr x26, [x14, #0x8]\n" + "fmla z31.s, p3/M, z1.s, z19.s\n" + "fmla z29.s, p3/M, z3.s, z16.s\n" + "ld1w { z19.s }, p2/Z, [x22, x15, LSL #2]\n" + "ldr x25, [x17, #0xa8]\n" + "fmla z26.s, p3/M, z6.s, z16.s\n" + "fmla z27.s, p3/M, z7.s, z19.s\n" + "ld1w { z18.s }, p2/Z, [x20, x15, LSL #2]\n" + "ldr x23, [x17, #0xc0]\n" + "fmla z28.s, p3/M, z6.s, z19.s\n" + "fmla z30.s, p3/M, z4.s, z19.s\n" + "ldr x24, [x14, #0x10]\n" + "fmla z21.s, p3/M, z3.s, z20.s\n" + "fmla z25.s, p3/M, z5.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x12, x15, LSL #2]\n" + "ldr x22, [x17, #0xb0]\n" + "fmla z29.s, p3/M, z5.s, z19.s\n" + "fmla z31.s, p3/M, z3.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x21, x15, LSL #2]\n" + "ldr x20, [x17, #0xb8]\n" + "fmla z26.s, p3/M, z8.s, z19.s\n" + "fmla z28.s, p3/M, z8.s, z17.s\n" + "ldr x21, [x14, #0x18]\n" + "fmla z30.s, p3/M, z6.s, z16.s\n" + "fmla z24.s, p3/M, z3.s, z18.s\n" + "fmla z27.s, p3/M, z0.s, z18.s\n" + "fmla z31.s, p3/M, z5.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x11, x15, LSL #2]\n" + "fmla z29.s, p3/M, z7.s, z16.s\n" + "ld1w { z19.s }, p2/Z, [x10, x15, LSL #2]\n" + "fmla z21.s, p3/M, z4.s, z18.s\n" + "fmla z26.s, p3/M, z1.s, z18.s\n" + "fmla z24.s, p3/M, z5.s, z17.s\n" + "ld1w { z16.s }, p2/Z, [x9, x15, LSL #2]\n" + "fmla z25.s, p3/M, z4.s, z17.s\n" + "fmla z27.s, p3/M, z2.s, z17.s\n" + "fmla z28.s, p3/M, z1.s, z17.s\n" + "fmla z30.s, p3/M, z8.s, z19.s\n" + "ld1w { z17.s }, p2/Z, [x28, x15, LSL #2]\n" + "fmla z21.s, p3/M, z2.s, z17.s\n" + "fmla z26.s, p3/M, z7.s, z16.s\n" + "fmla z27.s, p3/M, z6.s, z16.s\n" + "fmla z29.s, p3/M, z4.s, z16.s\n" + "fmla z30.s, p3/M, z3.s, z16.s\n" + "ld1w { z18.s }, p2/Z, [x22, x15, LSL #2]\n" + "fmla z31.s, p3/M, z7.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x25, x15, LSL #2]\n" + "fmla z21.s, p3/M, z6.s, z18.s\n" + "fmla z31.s, p3/M, z4.s, z16.s\n" + "fmla z24.s, p3/M, z1.s, z17.s\n" + "fmla z25.s, p3/M, z0.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x20, x15, LSL #2]\n" + "fmax z21.s, p3/M, z21.s, z22.s\n" + "fmla z28.s, p3/M, z7.s, z16.s\n" + "fmla z30.s, p3/M, z5.s, z16.s\n" + "fmla z29.s, p3/M, z0.s, z18.s\n" + "fmla z31.s, p3/M, z2.s, z17.s\n" + "fmla z27.s, p3/M, z8.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x23, x15, LSL #2]\n" + "fmla z26.s, p3/M, z3.s, z18.s\n" + "fmla z25.s, p3/M, z8.s, z17.s\n" + "fmin z21.s, p3/M, z21.s, z14.s\n" + "st1w { z21.s }, p0, [x27, x13, LSL #2]\n" + "ldr x20, [x14, #0x20]\n" + "fmla z28.s, p3/M, z5.s, z17.s\n" + "fmla z29.s, p3/M, z8.s, z16.s\n" + "fmla z30.s, p3/M, z7.s, z16.s\n" + "fmla z31.s, p3/M, z6.s, z16.s\n" + ".inst 0xc1aecad8 // fclamp { z24.s-z27.s }, z22.s, z14.s\n" + "st1w { z24.s }, p0, [x26, x13, LSL #2]\n" + "ldr x23, [x14, #0x28]\n" + "st1w { z25.s }, p0, [x24, x13, LSL #2]\n" + "ldr x22, [x14, #0x30]\n" + ".inst 0xc1aecadc // fclamp { z28.s-z31.s }, z22.s, z14.s\n" + "st1w { z26.s }, p0, [x21, x13, LSL #2]\n" + "ldr x21, [x14, #0x38]\n" + "st1w { z27.s }, p0, [x20, x13, LSL #2]\n" + "ldr x20, [x14, #0x40]\n" + "st1w { z28.s }, p0, [x23, x13, LSL #2]\n" + "st1w { z29.s }, p0, [x22, x13, LSL #2]\n" + "st1w { z30.s }, p0, [x21, x13, LSL #2]\n" + "st1w { z31.s }, p0, [x20, x13, LSL #2]\n" ".inst 0xd503467f // SMSTOP\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp index 9184cc00e48a293a9c68a0e7f7f2b621e03c062a..add666e14ef2e3c193707f717ca16eb9da9cac4b 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,13 +22,13 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include #pragma once -#if defined(__aarch64__) +#if defined(ARM_COMPUTE_ENABLE_SME2) namespace arm_conv { namespace depthwise { @@ -68,4 +68,4 @@ class sme2_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst : public DepthwiseDepthfirs } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) +#endif // defined(ARM_COMPUTE_ENABLE_SME2) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp index 5380567d368fbdf1820f0d98f02cef7c00c3e488..efd37c38ec14631d69f46532b6480ccf2b590b0c 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp @@ -22,11 +22,11 @@ * SOFTWARE. */ -#if defined(ARM_COMPUTE_ENABLE_SME2) - #include #include +#if defined(ARM_COMPUTE_ENABLE_SME2) + namespace arm_conv { namespace depthwise { @@ -193,18 +193,18 @@ void sme2_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "ldr x22, [%x[params_struct], %[offsetof_args_ld_output_row]]\n" "mul x21, x2, x22\n" // offset = tile_i * ld_output_row "mov x20, #0x4\n" - "ld1w { z15.s }, p3/Z, [x17]\n" + "ld1w { z14.s }, p3/Z, [x17]\n" "ldr x9, [%x[params_struct], %[offsetof_args_ld_output_col]]\n" "madd x21, x3, x9, x21\n" // offset += tile_j * ld_output_col "mul x21, x21, x20\n" // offset *= output_tile_size - "ld1rw { z14.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rw { z13.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" "ldr x28, [%x[params_struct], %[offsetof_args_outptr]]\n" "add x28, x28, x21, LSL #2\n" // outptrs[0] += offset * sizeof(float) "addvl x17, x17, #1\n" ".inst 0xa040c220 // ld1w { z0.s-z3.s }, pn8.b/Z, [x17]\n" "add x27, x28, x22, LSL #2\n" "cntw x26\n" - "ld1rw { z13.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "ld1rw { z15.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "addvl x17, x17, #4\n" "add x25, x27, x22, LSL #2\n" ".inst 0xa040c224 // ld1w { z4.s-z7.s }, pn8.b/Z, [x17]\n" @@ -224,440 +224,440 @@ void sme2_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "addvl x17, x17, #1\n" "bge 4f\n" "3:" // Tile loop: Channel loop - "movprfx z21, z15\n fmla z21.s, p3/M, z4.s, z9.s\n" - "movprfx z16, z15\n fmla z16.s, p3/M, z8.s, z9.s\n" + "movprfx z25, z14\n fmla z25.s, p3/M, z4.s, z9.s\n" + "movprfx z28, z14\n fmla z28.s, p3/M, z8.s, z9.s\n" "whilelt p1.s, x26, %x[n_channels]\n" "incw x21\n" - "movprfx z22, z15\n fmla z22.s, p3/M, z3.s, z9.s\n" - "movprfx z25, z15\n fmla z25.s, p3/M, z1.s, z9.s\n" + "movprfx z26, z14\n fmla z26.s, p3/M, z3.s, z9.s\n" + "movprfx z17, z14\n fmla z17.s, p3/M, z1.s, z9.s\n" "incw x26\n" "mov p0.b, p2.b\n" - "movprfx z26, z15\n fmla z26.s, p3/M, z0.s, z9.s\n" - "fmla z21.s, p3/M, z5.s, z12.s\n" + "movprfx z18, z14\n fmla z18.s, p3/M, z0.s, z9.s\n" + "fmla z25.s, p3/M, z5.s, z12.s\n" "incw x20\n" - "movprfx z17, z15\n fmla z17.s, p3/M, z7.s, z9.s\n" - "movprfx z18, z15\n fmla z18.s, p3/M, z6.s, z9.s\n" - "movprfx z20, z15\n fmla z20.s, p3/M, z5.s, z9.s\n" - "movprfx z24, z15\n fmla z24.s, p3/M, z2.s, z9.s\n" + "movprfx z29, z14\n fmla z29.s, p3/M, z7.s, z9.s\n" + "movprfx z30, z14\n fmla z30.s, p3/M, z6.s, z9.s\n" + "movprfx z24, z14\n fmla z24.s, p3/M, z5.s, z9.s\n" + "movprfx z16, z14\n fmla z16.s, p3/M, z2.s, z9.s\n" "ld1w { z9.s }, p2/Z, [x16, x8, LSL #2]\n" - "fmla z16.s, p3/M, z0.s, z10.s\n" - "movprfx z19, z15\n fmla z19.s, p3/M, z2.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x12]\n" - "fmla z22.s, p3/M, z4.s, z12.s\n" - "fmla z25.s, p3/M, z2.s, z12.s\n" - "ld1w { z11.s }, p2/Z, [x12, x11, LSL #2]\n" - "fmla z26.s, p3/M, z1.s, z12.s\n" - "movprfx z28, z15\n fmla z28.s, p3/M, z6.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x16, x15, LSL #2]\n" - "fmla z21.s, p3/M, z7.s, z9.s\n" - "fmla z17.s, p3/M, z8.s, z12.s\n" - "fmla z18.s, p3/M, z7.s, z12.s\n" - "fmla z19.s, p3/M, z6.s, z12.s\n" - "movprfx z23, z15\n fmla z23.s, p3/M, z3.s, z12.s\n" - "movprfx z27, z15\n fmla z27.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x5, x4, LSL #2]\n" - "movprfx z31, z15\n fmla z31.s, p3/M, z8.s, z11.s\n" - "fmla z22.s, p3/M, z6.s, z9.s\n" - "ld1w { z11.s }, p2/Z, [x5, x13, LSL #2]\n" - "fmla z25.s, p3/M, z4.s, z9.s\n" - "fmla z26.s, p3/M, z3.s, z9.s\n" - "fmla z20.s, p3/M, z8.s, z9.s\n" - "fmla z24.s, p3/M, z5.s, z9.s\n" - "fmla z28.s, p3/M, z2.s, z9.s\n" - "fmla z21.s, p3/M, z8.s, z10.s\n" - "fmla z16.s, p3/M, z1.s, z12.s\n" - "fmla z17.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x6, x11, LSL #2]\n" - "fmla z18.s, p3/M, z2.s, z11.s\n" - "fmla z19.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x14]\n" - "fmla z22.s, p3/M, z7.s, z10.s\n" - "fmla z23.s, p3/M, z6.s, z10.s\n" - "fmla z25.s, p3/M, z5.s, z10.s\n" - "fmla z26.s, p3/M, z4.s, z10.s\n" - "fmla z27.s, p3/M, z3.s, z10.s\n" - "fmla z31.s, p3/M, z0.s, z10.s\n" - "fmla z24.s, p3/M, z6.s, z11.s\n" - "fmla z28.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x14, x11, LSL #2]\n" - "fmla z19.s, p3/M, z5.s, z12.s\n" - "fmla z23.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x6, x15, LSL #2]\n" - "fmla z27.s, p3/M, z8.s, z11.s\n" - "fmla z31.s, p3/M, z5.s, z11.s\n" - "movprfx z29, z15\n fmla z29.s, p3/M, z1.s, z9.s\n" - "movprfx z30, z15\n fmla z30.s, p3/M, z0.s, z9.s\n" + "fmla z28.s, p3/M, z0.s, z10.s\n" + "movprfx z31, z14\n fmla z31.s, p3/M, z2.s, z11.s\n" + "ld1w { z19.s }, p2/Z, [x12]\n" + "fmla z26.s, p3/M, z4.s, z12.s\n" + "fmla z17.s, p3/M, z2.s, z12.s\n" + "ld1w { z22.s }, p2/Z, [x12, x11, LSL #2]\n" + "fmla z18.s, p3/M, z1.s, z12.s\n" + "movprfx z20, z14\n fmla z20.s, p3/M, z6.s, z19.s\n" + "ld1w { z11.s }, p2/Z, [x16, x15, LSL #2]\n" + "fmla z25.s, p3/M, z7.s, z9.s\n" + "fmla z29.s, p3/M, z8.s, z12.s\n" + "fmla z30.s, p3/M, z7.s, z12.s\n" + "fmla z31.s, p3/M, z6.s, z12.s\n" + "movprfx z27, z14\n fmla z27.s, p3/M, z3.s, z12.s\n" + "movprfx z19, z14\n fmla z19.s, p3/M, z0.s, z12.s\n" + "ld1w { z10.s }, p2/Z, [x5, x4, LSL #2]\n" + "movprfx z23, z14\n fmla z23.s, p3/M, z8.s, z22.s\n" + "fmla z26.s, p3/M, z6.s, z9.s\n" + "ld1w { z12.s }, p2/Z, [x5, x13, LSL #2]\n" + "fmla z17.s, p3/M, z4.s, z9.s\n" + "fmla z18.s, p3/M, z3.s, z9.s\n" + "movprfx z21, z14\n fmla z21.s, p3/M, z1.s, z9.s\n" + "movprfx z22, z14\n fmla z22.s, p3/M, z0.s, z9.s\n" + "ld1w { z14.s }, p3/Z, [x17]\n" + "addvl x17, x17, #1\n" + "fmla z24.s, p3/M, z8.s, z9.s\n" + "fmla z16.s, p3/M, z5.s, z9.s\n" + "fmla z20.s, p3/M, z2.s, z9.s\n" + "fmla z25.s, p3/M, z8.s, z11.s\n" "ld1w { z9.s }, p2/Z, [x6]\n" - "fmla z29.s, p3/M, z2.s, z10.s\n" - "fmla z30.s, p3/M, z1.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x6, x8, LSL #2]\n" - "fmla z20.s, p3/M, z0.s, z9.s\n" - "fmla z21.s, p3/M, z1.s, z10.s\n" - "fmla z16.s, p3/M, z3.s, z9.s\n" - "fmla z17.s, p3/M, z4.s, z10.s\n" - "ld1w { z11.s }, p2/Z, [x12, x4, LSL #2]\n" - "fmla z18.s, p3/M, z3.s, z10.s\n" - "fmla z22.s, p3/M, z0.s, z10.s\n" - "fmla z20.s, p3/M, z2.s, z10.s\n" - "fmla z21.s, p3/M, z2.s, z12.s\n" - "fmla z16.s, p3/M, z5.s, z10.s\n" - "fmla z17.s, p3/M, z5.s, z12.s\n" - "ld1w { z10.s }, p2/Z, [x7, x4, LSL #2]\n" - "fmla z18.s, p3/M, z4.s, z12.s\n" - "fmla z19.s, p3/M, z3.s, z12.s\n" - "fmla z22.s, p3/M, z1.s, z12.s\n" - "fmla z23.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x7, x13, LSL #2]\n" - "fmla z28.s, p3/M, z7.s, z11.s\n" - "fmla z29.s, p3/M, z6.s, z11.s\n" + "fmla z28.s, p3/M, z1.s, z10.s\n" + "fmla z29.s, p3/M, z0.s, z10.s\n" + "ld1w { z10.s }, p2/Z, [x6, x11, LSL #2]\n" + "fmla z30.s, p3/M, z2.s, z12.s\n" + "fmla z31.s, p3/M, z1.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x14]\n" + "fmla z26.s, p3/M, z7.s, z11.s\n" + "fmla z27.s, p3/M, z6.s, z11.s\n" + "fmla z17.s, p3/M, z5.s, z11.s\n" + "fmla z18.s, p3/M, z4.s, z11.s\n" + "fmla z19.s, p3/M, z3.s, z11.s\n" + "fmla z21.s, p3/M, z2.s, z11.s\n" + "fmla z22.s, p3/M, z1.s, z11.s\n" + "fmla z23.s, p3/M, z0.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x6, x8, LSL #2]\n" + "fmla z24.s, p3/M, z0.s, z9.s\n" + "fmla z16.s, p3/M, z6.s, z12.s\n" + "fmla z20.s, p3/M, z3.s, z12.s\n" + "fmla z25.s, p3/M, z1.s, z11.s\n" + "ld1w { z12.s }, p2/Z, [x14, x11, LSL #2]\n" + "fmla z28.s, p3/M, z3.s, z9.s\n" + "fmla z31.s, p3/M, z5.s, z10.s\n" + "fmla z27.s, p3/M, z2.s, z10.s\n" + "fmla z29.s, p3/M, z4.s, z11.s\n" + "ld1w { z10.s }, p2/Z, [x6, x15, LSL #2]\n" + "fmla z30.s, p3/M, z3.s, z11.s\n" + "fmla z26.s, p3/M, z0.s, z11.s\n" + "fmla z19.s, p3/M, z8.s, z12.s\n" + "fmla z23.s, p3/M, z5.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x12, x4, LSL #2]\n" + "fmla z24.s, p3/M, z2.s, z11.s\n" + "fmla z25.s, p3/M, z2.s, z10.s\n" + "fmla z28.s, p3/M, z5.s, z11.s\n" + "fmla z29.s, p3/M, z5.s, z10.s\n" + "ld1w { z9.s }, p2/Z, [x7, x4, LSL #2]\n" + "fmla z30.s, p3/M, z4.s, z10.s\n" + "fmla z31.s, p3/M, z3.s, z10.s\n" + "fmla z26.s, p3/M, z1.s, z10.s\n" + "fmla z27.s, p3/M, z0.s, z10.s\n" + "ld1w { z10.s }, p2/Z, [x7, x13, LSL #2]\n" + "fmla z20.s, p3/M, z7.s, z12.s\n" + "fmla z21.s, p3/M, z6.s, z12.s\n" "ld1w { z11.s }, p2/Z, [x12, x13, LSL #2]\n" - "fmla z20.s, p3/M, z4.s, z10.s\n" - "fmla z21.s, p3/M, z3.s, z10.s\n" - "fmla z24.s, p3/M, z1.s, z10.s\n" - "fmla z25.s, p3/M, z0.s, z10.s\n" - "fmla z16.s, p3/M, z7.s, z10.s\n" - "fmla z17.s, p3/M, z6.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x5, x8, LSL #2]\n" - "fmla z30.s, p3/M, z8.s, z11.s\n" - "fmla z31.s, p3/M, z7.s, z11.s\n" + "fmla z24.s, p3/M, z4.s, z9.s\n" + "fmla z25.s, p3/M, z3.s, z9.s\n" + "fmla z16.s, p3/M, z1.s, z9.s\n" + "fmla z17.s, p3/M, z0.s, z9.s\n" + "fmla z28.s, p3/M, z7.s, z9.s\n" + "fmla z29.s, p3/M, z6.s, z9.s\n" + "ld1w { z12.s }, p2/Z, [x5, x8, LSL #2]\n" + "fmla z22.s, p3/M, z8.s, z11.s\n" + "fmla z23.s, p3/M, z7.s, z11.s\n" "ld1w { z11.s }, p2/Z, [x16, x4, LSL #2]\n" - "fmla z18.s, p3/M, z8.s, z12.s\n" - "fmla z19.s, p3/M, z7.s, z12.s\n" - "fmla z22.s, p3/M, z5.s, z12.s\n" - "fmla z23.s, p3/M, z4.s, z12.s\n" - "fmla z26.s, p3/M, z2.s, z12.s\n" - "fmla z27.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x5, x15, LSL #2]\n" + "fmla z30.s, p3/M, z8.s, z10.s\n" + "fmla z31.s, p3/M, z7.s, z10.s\n" + "fmla z26.s, p3/M, z5.s, z10.s\n" + "fmla z27.s, p3/M, z4.s, z10.s\n" + "fmla z18.s, p3/M, z2.s, z10.s\n" + "fmla z19.s, p3/M, z1.s, z10.s\n" + "ld1w { z9.s }, p2/Z, [x5, x15, LSL #2]\n" "addvl x5, x5, #1\n" - "fmla z20.s, p3/M, z7.s, z11.s\n" - "fmla z21.s, p3/M, z6.s, z11.s\n" - "fmla z24.s, p3/M, z4.s, z11.s\n" - "fmla z25.s, p3/M, z3.s, z11.s\n" - "fmla z28.s, p3/M, z1.s, z11.s\n" - "fmla z29.s, p3/M, z0.s, z11.s\n" + "fmla z24.s, p3/M, z7.s, z11.s\n" + "fmla z25.s, p3/M, z6.s, z11.s\n" + "fmla z16.s, p3/M, z4.s, z11.s\n" + "fmla z17.s, p3/M, z3.s, z11.s\n" + "fmla z20.s, p3/M, z1.s, z11.s\n" + "fmla z21.s, p3/M, z0.s, z11.s\n" "ld1w { z11.s }, p2/Z, [x16, x13, LSL #2]\n" - "fmla z16.s, p3/M, z2.s, z10.s\n" - "fmla z17.s, p3/M, z1.s, z10.s\n" - "fmla z18.s, p3/M, z0.s, z10.s\n" + "fmla z28.s, p3/M, z2.s, z12.s\n" + "fmla z29.s, p3/M, z1.s, z12.s\n" + "fmla z30.s, p3/M, z0.s, z12.s\n" "ld1w { z10.s }, p2/Z, [x7]\n" - "fmla z30.s, p3/M, z2.s, z11.s\n" - "fmla z19.s, p3/M, z0.s, z12.s\n" - "fmla z20.s, p3/M, z3.s, z10.s\n" - "fmla z24.s, p3/M, z0.s, z10.s\n" - "fmla z22.s, p3/M, z8.s, z11.s\n" - "fmla z23.s, p3/M, z7.s, z11.s\n" - "fmla z26.s, p3/M, z5.s, z11.s\n" - "fmla z27.s, p3/M, z4.s, z11.s\n" - "fmla z31.s, p3/M, z1.s, z11.s\n" + "fmla z22.s, p3/M, z2.s, z11.s\n" + "fmla z31.s, p3/M, z0.s, z9.s\n" + "fmla z24.s, p3/M, z3.s, z10.s\n" + "fmla z16.s, p3/M, z0.s, z10.s\n" + "fmla z26.s, p3/M, z8.s, z11.s\n" + "fmla z27.s, p3/M, z7.s, z11.s\n" + "fmla z18.s, p3/M, z5.s, z11.s\n" + "fmla z19.s, p3/M, z4.s, z11.s\n" + "fmla z23.s, p3/M, z1.s, z11.s\n" "ld1w { z11.s }, p2/Z, [x14, x8, LSL #2]\n" - "fmla z17.s, p3/M, z2.s, z12.s\n" - "fmla z18.s, p3/M, z1.s, z12.s\n" + "fmla z29.s, p3/M, z2.s, z9.s\n" + "fmla z30.s, p3/M, z1.s, z9.s\n" "ld1w { z12.s }, p2/Z, [x7, x11, LSL #2]\n" "addvl x7, x7, #1\n" - "fmla z16.s, p3/M, z6.s, z10.s\n" + "fmla z28.s, p3/M, z6.s, z10.s\n" "ld1w { z10.s }, p2/Z, [x16]\n" - "fmla z29.s, p3/M, z4.s, z11.s\n" - "fmla z30.s, p3/M, z3.s, z11.s\n" - "fmla z19.s, p3/M, z8.s, z12.s\n" - "fmla z23.s, p3/M, z5.s, z12.s\n" - "fmla z27.s, p3/M, z2.s, z12.s\n" + "fmla z21.s, p3/M, z4.s, z11.s\n" + "fmla z22.s, p3/M, z3.s, z11.s\n" + "fmla z31.s, p3/M, z8.s, z12.s\n" + "ld1w { z9.s }, p1/Z, [x7, x8, LSL #2]\n" + "fmla z27.s, p3/M, z5.s, z12.s\n" + "fmla z19.s, p3/M, z2.s, z12.s\n" "ld1w { z12.s }, p2/Z, [x16, x11, LSL #2]\n" "addvl x16, x16, #1\n" - "fmla z20.s, p3/M, z6.s, z10.s\n" - "fmla z24.s, p3/M, z3.s, z10.s\n" - "fmla z28.s, p3/M, z0.s, z10.s\n" + "fmla z24.s, p3/M, z6.s, z10.s\n" + "fmla z16.s, p3/M, z3.s, z10.s\n" + "fmla z20.s, p3/M, z0.s, z10.s\n" "ld1w { z10.s }, p2/Z, [x12, x8, LSL #2]\n" - "fmla z31.s, p3/M, z2.s, z12.s\n" - "fmla z29.s, p3/M, z7.s, z10.s\n" - "fmla z30.s, p3/M, z6.s, z10.s\n" - "fmla z24.s, p3/M, z8.s, z11.s\n" - "fmla z25.s, p3/M, z7.s, z11.s\n" - "fmla z26.s, p3/M, z6.s, z11.s\n" - "fmla z28.s, p3/M, z5.s, z11.s\n" + "fmla z23.s, p3/M, z2.s, z12.s\n" + "fmla z21.s, p3/M, z7.s, z10.s\n" + "fmla z22.s, p3/M, z6.s, z10.s\n" + "fmla z16.s, p3/M, z8.s, z11.s\n" + "fmla z17.s, p3/M, z7.s, z11.s\n" + "fmla z18.s, p3/M, z6.s, z11.s\n" + "fmla z20.s, p3/M, z5.s, z11.s\n" "ld1w { z11.s }, p2/Z, [x14, x15, LSL #2]\n" - "fmla z27.s, p3/M, z5.s, z12.s\n" - "fmla z29.s, p3/M, z5.s, z11.s\n" - "fmla z30.s, p3/M, z4.s, z11.s\n" - "fmla z31.s, p3/M, z3.s, z11.s\n" - "fmla z23.s, p3/M, z8.s, z12.s\n" + "fmla z19.s, p3/M, z5.s, z12.s\n" + "fmla z21.s, p3/M, z5.s, z11.s\n" + "fmla z22.s, p3/M, z4.s, z11.s\n" + "fmla z23.s, p3/M, z3.s, z11.s\n" + "fmla z27.s, p3/M, z8.s, z12.s\n" "ld1w { z12.s }, p2/Z, [x12, x15, LSL #2]\n" - "fmla z28.s, p3/M, z8.s, z10.s\n" + "fmla z20.s, p3/M, z8.s, z10.s\n" "addvl x12, x12, #1\n" "ld1w { z10.s }, p2/Z, [x6, x4, LSL #2]\n" - "fmla z25.s, p3/M, z8.s, z11.s\n" - "fmla z26.s, p3/M, z7.s, z11.s\n" - "fmla z27.s, p3/M, z6.s, z11.s\n" - "fmla z29.s, p3/M, z8.s, z12.s\n" + "fmla z17.s, p3/M, z8.s, z11.s\n" + "fmla z18.s, p3/M, z7.s, z11.s\n" + "fmla z19.s, p3/M, z6.s, z11.s\n" + "fmla z21.s, p3/M, z8.s, z12.s\n" "ld1w { z11.s }, p2/Z, [x6, x13, LSL #2]\n" "addvl x6, x6, #1\n" - "fmla z30.s, p3/M, z7.s, z12.s\n" - "fmla z31.s, p3/M, z6.s, z12.s\n" + "fmla z22.s, p3/M, z7.s, z12.s\n" + "fmla z23.s, p3/M, z6.s, z12.s\n" "ld1w { z12.s }, p2/Z, [x14, x4, LSL #2]\n" - "fmla z16.s, p3/M, z4.s, z10.s\n" - "fmla z17.s, p3/M, z3.s, z10.s\n" - "fmla z20.s, p3/M, z1.s, z10.s\n" - "fmla z21.s, p3/M, z0.s, z10.s\n" + "fmla z28.s, p3/M, z4.s, z10.s\n" + "fmla z29.s, p3/M, z3.s, z10.s\n" + "fmla z24.s, p3/M, z1.s, z10.s\n" + "fmla z25.s, p3/M, z0.s, z10.s\n" "ld1w { z10.s }, p2/Z, [x14, x13, LSL #2]\n" "whilelt p2.s, x21, %x[n_channels]\n" - "fmla z18.s, p3/M, z5.s, z11.s\n" - "fmla z19.s, p3/M, z4.s, z11.s\n" - "ld1w { z15.s }, p3/Z, [x17]\n" - "addvl x17, x17, #1\n" - "fmla z22.s, p3/M, z2.s, z11.s\n" - "fmla z23.s, p3/M, z1.s, z11.s\n" + "fmla z30.s, p3/M, z5.s, z11.s\n" + "fmla z31.s, p3/M, z4.s, z11.s\n" "cmp x26, %x[n_channels]\n" "addvl x14, x14, #1\n" - "fmla z24.s, p3/M, z7.s, z12.s\n" - "fmla z25.s, p3/M, z6.s, z12.s\n" - "ld1w { z9.s }, p1/Z, [x7, x8, LSL #2]\n" - "fmla z28.s, p3/M, z4.s, z12.s\n" - "fmla z29.s, p3/M, z3.s, z12.s\n" + "fmla z26.s, p3/M, z2.s, z11.s\n" + "fmla z27.s, p3/M, z1.s, z11.s\n" + "ld1w { z11.s }, p1/Z, [x5, x11, LSL #2]\n" + "fmla z16.s, p3/M, z7.s, z12.s\n" + "fmla z17.s, p3/M, z6.s, z12.s\n" + "fmla z20.s, p3/M, z4.s, z12.s\n" + "fmla z21.s, p3/M, z3.s, z12.s\n" ".inst 0xa040c220 // ld1w { z0.s-z3.s }, pn8.b/Z, [x17]\n" "addvl x17, x17, #4\n" - "fmla z26.s, p3/M, z8.s, z10.s\n" - "fmla z27.s, p3/M, z7.s, z10.s\n" - "ld1w { z11.s }, p1/Z, [x5, x11, LSL #2]\n" - "fmla z30.s, p3/M, z5.s, z10.s\n" - "fmla z31.s, p3/M, z4.s, z10.s\n" + "fmla z18.s, p3/M, z8.s, z10.s\n" + "fmla z19.s, p3/M, z7.s, z10.s\n" + "ld1w { z12.s }, p1/Z, [x7, x15, LSL #2]\n" + "fmla z22.s, p3/M, z5.s, z10.s\n" + "fmla z23.s, p3/M, z4.s, z10.s\n" ".inst 0xa040c224 // ld1w { z4.s-z7.s }, pn8.b/Z, [x17]\n" "addvl x17, x17, #4\n" - ".inst 0xc1adc9d0 // fclamp { z16.s-z19.s }, z14.s, z13.s\n" - ".inst 0xc1adc9d4 // fclamp { z20.s-z23.s }, z14.s, z13.s\n" + ".inst 0xc1afc9bc // fclamp { z28.s-z31.s }, z13.s, z15.s\n" + ".inst 0xc1afc9b8 // fclamp { z24.s-z27.s }, z13.s, z15.s\n" "ld1w { z10.s }, p1/Z, [x5]\n" - ".inst 0xc1adc9d8 // fclamp { z24.s-z27.s }, z14.s, z13.s\n" - ".inst 0xc1adc9dc // fclamp { z28.s-z31.s }, z14.s, z13.s\n" - "st1w { z16.s }, p0, [x28]\n" - "ld1w { z12.s }, p1/Z, [x7, x15, LSL #2]\n" - "st1w { z17.s }, p0, [x28, x9, LSL #2]\n" - "st1w { z18.s }, p0, [x28, x24, LSL #2]\n" + ".inst 0xc1afc9b0 // fclamp { z16.s-z19.s }, z13.s, z15.s\n" + ".inst 0xc1afc9b4 // fclamp { z20.s-z23.s }, z13.s, z15.s\n" + "st1w { z28.s }, p0, [x28]\n" + "st1w { z29.s }, p0, [x28, x9, LSL #2]\n" "ld1w { z8.s }, p3/Z, [x17]\n" "addvl x17, x17, #1\n" - "st1w { z19.s }, p0, [x28, x22, LSL #2]\n" + "st1w { z30.s }, p0, [x28, x24, LSL #2]\n" + "st1w { z31.s }, p0, [x28, x22, LSL #2]\n" "addvl x28, x28, #1\n" - "st1w { z20.s }, p0, [x27]\n" - "st1w { z21.s }, p0, [x27, x9, LSL #2]\n" - "st1w { z22.s }, p0, [x27, x24, LSL #2]\n" - "st1w { z23.s }, p0, [x27, x22, LSL #2]\n" + "st1w { z24.s }, p0, [x27]\n" + "st1w { z25.s }, p0, [x27, x9, LSL #2]\n" + "st1w { z26.s }, p0, [x27, x24, LSL #2]\n" + "st1w { z27.s }, p0, [x27, x22, LSL #2]\n" "addvl x27, x27, #1\n" - "st1w { z24.s }, p0, [x25]\n" - "st1w { z25.s }, p0, [x25, x9, LSL #2]\n" - "st1w { z26.s }, p0, [x25, x24, LSL #2]\n" - "st1w { z27.s }, p0, [x25, x22, LSL #2]\n" + "st1w { z16.s }, p0, [x25]\n" + "st1w { z17.s }, p0, [x25, x9, LSL #2]\n" + "st1w { z18.s }, p0, [x25, x24, LSL #2]\n" + "st1w { z19.s }, p0, [x25, x22, LSL #2]\n" "addvl x25, x25, #1\n" - "st1w { z28.s }, p0, [x23]\n" - "st1w { z29.s }, p0, [x23, x9, LSL #2]\n" - "st1w { z30.s }, p0, [x23, x24, LSL #2]\n" - "st1w { z31.s }, p0, [x23, x22, LSL #2]\n" + "st1w { z20.s }, p0, [x23]\n" + "st1w { z21.s }, p0, [x23, x9, LSL #2]\n" + "st1w { z22.s }, p0, [x23, x24, LSL #2]\n" + "st1w { z23.s }, p0, [x23, x22, LSL #2]\n" "addvl x23, x23, #1\n" "blt 3b\n" "4:" // Tile loop: Channel tail - "movprfx z21, z15\n fmla z21.s, p3/M, z4.s, z9.s\n" - "movprfx z16, z15\n fmla z16.s, p3/M, z8.s, z9.s\n" + "movprfx z21, z14\n fmla z21.s, p3/M, z4.s, z9.s\n" + "movprfx z24, z14\n fmla z24.s, p3/M, z8.s, z9.s\n" "ldr x3, [%x[params_struct], %[offsetof_args_tile_j]]\n" "add x3, x3, #0x1\n" - "movprfx z22, z15\n fmla z22.s, p3/M, z3.s, z9.s\n" - "movprfx z25, z15\n fmla z25.s, p3/M, z1.s, z9.s\n" + "movprfx z22, z14\n fmla z22.s, p3/M, z3.s, z9.s\n" + "movprfx z29, z14\n fmla z29.s, p3/M, z1.s, z9.s\n" "ldr x2, [%x[params_struct], %[offsetof_args_tile_i]]\n" "add x21, x2, #0x1\n" - "movprfx z26, z15\n fmla z26.s, p3/M, z0.s, z9.s\n" + "movprfx z30, z14\n fmla z30.s, p3/M, z0.s, z9.s\n" "fmla z21.s, p3/M, z5.s, z12.s\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" "cmp x3, x20\n" - "movprfx z17, z15\n fmla z17.s, p3/M, z7.s, z9.s\n" - "movprfx z18, z15\n fmla z18.s, p3/M, z6.s, z9.s\n" + "movprfx z25, z14\n fmla z25.s, p3/M, z7.s, z9.s\n" + "movprfx z26, z14\n fmla z26.s, p3/M, z6.s, z9.s\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" "csel x2, x2, x21, LT\n" - "movprfx z20, z15\n fmla z20.s, p3/M, z5.s, z9.s\n" - "movprfx z24, z15\n fmla z24.s, p3/M, z2.s, z9.s\n" + "movprfx z20, z14\n fmla z20.s, p3/M, z5.s, z9.s\n" + "movprfx z28, z14\n fmla z28.s, p3/M, z2.s, z9.s\n" "ld1w { z9.s }, p2/Z, [x16, x8, LSL #2]\n" "mov p0.b, p2.b\n" - "fmla z16.s, p3/M, z0.s, z10.s\n" - "movprfx z19, z15\n fmla z19.s, p3/M, z2.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x12]\n" + "fmla z24.s, p3/M, z0.s, z10.s\n" + "movprfx z27, z14\n fmla z27.s, p3/M, z2.s, z11.s\n" + "ld1w { z17.s }, p2/Z, [x12]\n" "csel x3, x3, XZR, LT\n" "fmla z22.s, p3/M, z4.s, z12.s\n" - "fmla z25.s, p3/M, z2.s, z12.s\n" - "ld1w { z11.s }, p2/Z, [x12, x11, LSL #2]\n" + "fmla z29.s, p3/M, z2.s, z12.s\n" + "ld1w { z18.s }, p2/Z, [x12, x11, LSL #2]\n" "cmp x2, x20\n" - "fmla z26.s, p3/M, z1.s, z12.s\n" - "movprfx z28, z15\n fmla z28.s, p3/M, z6.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x16, x15, LSL #2]\n" + "fmla z30.s, p3/M, z1.s, z12.s\n" + "movprfx z16, z14\n fmla z16.s, p3/M, z6.s, z17.s\n" + "ld1w { z11.s }, p2/Z, [x16, x15, LSL #2]\n" "fmla z21.s, p3/M, z7.s, z9.s\n" - "fmla z17.s, p3/M, z8.s, z12.s\n" - "fmla z18.s, p3/M, z7.s, z12.s\n" - "fmla z19.s, p3/M, z6.s, z12.s\n" - "movprfx z23, z15\n fmla z23.s, p3/M, z3.s, z12.s\n" - "movprfx z27, z15\n fmla z27.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x5, x4, LSL #2]\n" - "movprfx z31, z15\n fmla z31.s, p3/M, z8.s, z11.s\n" + "fmla z25.s, p3/M, z8.s, z12.s\n" + "fmla z26.s, p3/M, z7.s, z12.s\n" + "fmla z27.s, p3/M, z6.s, z12.s\n" + "movprfx z23, z14\n fmla z23.s, p3/M, z3.s, z12.s\n" + "movprfx z31, z14\n fmla z31.s, p3/M, z0.s, z12.s\n" + "ld1w { z10.s }, p2/Z, [x5, x4, LSL #2]\n" + "movprfx z19, z14\n fmla z19.s, p3/M, z8.s, z18.s\n" "fmla z22.s, p3/M, z6.s, z9.s\n" - "ld1w { z11.s }, p2/Z, [x5, x13, LSL #2]\n" - "fmla z25.s, p3/M, z4.s, z9.s\n" - "fmla z26.s, p3/M, z3.s, z9.s\n" + "ld1w { z12.s }, p2/Z, [x5, x13, LSL #2]\n" + "fmla z29.s, p3/M, z4.s, z9.s\n" + "fmla z30.s, p3/M, z3.s, z9.s\n" + "movprfx z17, z14\n fmla z17.s, p3/M, z1.s, z9.s\n" + "movprfx z18, z14\n fmla z18.s, p3/M, z0.s, z9.s\n" "fmla z20.s, p3/M, z8.s, z9.s\n" - "fmla z24.s, p3/M, z5.s, z9.s\n" - "fmla z28.s, p3/M, z2.s, z9.s\n" - "fmla z21.s, p3/M, z8.s, z10.s\n" - "fmla z16.s, p3/M, z1.s, z12.s\n" - "fmla z17.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x6, x11, LSL #2]\n" - "fmla z18.s, p3/M, z2.s, z11.s\n" - "fmla z19.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x14]\n" - "fmla z22.s, p3/M, z7.s, z10.s\n" - "fmla z23.s, p3/M, z6.s, z10.s\n" - "fmla z25.s, p3/M, z5.s, z10.s\n" - "fmla z26.s, p3/M, z4.s, z10.s\n" - "fmla z27.s, p3/M, z3.s, z10.s\n" - "fmla z31.s, p3/M, z0.s, z10.s\n" - "fmla z24.s, p3/M, z6.s, z11.s\n" - "fmla z28.s, p3/M, z3.s, z11.s\n" + "fmla z28.s, p3/M, z5.s, z9.s\n" + "fmla z16.s, p3/M, z2.s, z9.s\n" + "fmla z21.s, p3/M, z8.s, z11.s\n" + "ld1w { z14.s }, p2/Z, [x6]\n" + "fmla z24.s, p3/M, z1.s, z10.s\n" + "fmla z25.s, p3/M, z0.s, z10.s\n" + "ld1w { z10.s }, p2/Z, [x6, x11, LSL #2]\n" + "fmla z26.s, p3/M, z2.s, z12.s\n" + "fmla z27.s, p3/M, z1.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x14]\n" + "fmla z22.s, p3/M, z7.s, z11.s\n" + "fmla z23.s, p3/M, z6.s, z11.s\n" + "fmla z29.s, p3/M, z5.s, z11.s\n" + "fmla z30.s, p3/M, z4.s, z11.s\n" + "fmla z31.s, p3/M, z3.s, z11.s\n" + "fmla z17.s, p3/M, z2.s, z11.s\n" + "fmla z18.s, p3/M, z1.s, z11.s\n" + "fmla z19.s, p3/M, z0.s, z11.s\n" + "ld1w { z9.s }, p2/Z, [x6, x8, LSL #2]\n" + "fmla z20.s, p3/M, z0.s, z14.s\n" + "fmla z28.s, p3/M, z6.s, z12.s\n" + "fmla z16.s, p3/M, z3.s, z12.s\n" + "fmla z21.s, p3/M, z1.s, z9.s\n" "ld1w { z11.s }, p2/Z, [x14, x11, LSL #2]\n" - "fmla z19.s, p3/M, z5.s, z12.s\n" - "fmla z23.s, p3/M, z2.s, z12.s\n" + "fmla z24.s, p3/M, z3.s, z14.s\n" + "fmla z27.s, p3/M, z5.s, z10.s\n" + "fmla z23.s, p3/M, z2.s, z10.s\n" + "fmla z25.s, p3/M, z4.s, z9.s\n" "ld1w { z12.s }, p2/Z, [x6, x15, LSL #2]\n" - "fmla z27.s, p3/M, z8.s, z11.s\n" - "fmla z31.s, p3/M, z5.s, z11.s\n" - "movprfx z29, z15\n fmla z29.s, p3/M, z1.s, z9.s\n" - "movprfx z30, z15\n fmla z30.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x6]\n" - "fmla z29.s, p3/M, z2.s, z10.s\n" - "fmla z30.s, p3/M, z1.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x6, x8, LSL #2]\n" - "fmla z20.s, p3/M, z0.s, z9.s\n" - "fmla z21.s, p3/M, z1.s, z10.s\n" - "fmla z16.s, p3/M, z3.s, z9.s\n" - "fmla z17.s, p3/M, z4.s, z10.s\n" - "ld1w { z11.s }, p2/Z, [x12, x4, LSL #2]\n" - "fmla z18.s, p3/M, z3.s, z10.s\n" - "fmla z22.s, p3/M, z0.s, z10.s\n" - "fmla z20.s, p3/M, z2.s, z10.s\n" + "fmla z26.s, p3/M, z3.s, z9.s\n" + "fmla z22.s, p3/M, z0.s, z9.s\n" + "fmla z31.s, p3/M, z8.s, z11.s\n" + "fmla z19.s, p3/M, z5.s, z11.s\n" + "ld1w { z10.s }, p2/Z, [x12, x4, LSL #2]\n" + "fmla z20.s, p3/M, z2.s, z9.s\n" "fmla z21.s, p3/M, z2.s, z12.s\n" - "fmla z16.s, p3/M, z5.s, z10.s\n" - "fmla z17.s, p3/M, z5.s, z12.s\n" - "ld1w { z10.s }, p2/Z, [x7, x4, LSL #2]\n" - "fmla z18.s, p3/M, z4.s, z12.s\n" - "fmla z19.s, p3/M, z3.s, z12.s\n" + "fmla z24.s, p3/M, z5.s, z9.s\n" + "fmla z25.s, p3/M, z5.s, z12.s\n" + "ld1w { z9.s }, p2/Z, [x7, x4, LSL #2]\n" + "fmla z26.s, p3/M, z4.s, z12.s\n" + "fmla z27.s, p3/M, z3.s, z12.s\n" "fmla z22.s, p3/M, z1.s, z12.s\n" "fmla z23.s, p3/M, z0.s, z12.s\n" "ld1w { z12.s }, p2/Z, [x7, x13, LSL #2]\n" - "fmla z28.s, p3/M, z7.s, z11.s\n" - "fmla z29.s, p3/M, z6.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x12, x13, LSL #2]\n" - "fmla z20.s, p3/M, z4.s, z10.s\n" - "fmla z21.s, p3/M, z3.s, z10.s\n" - "fmla z24.s, p3/M, z1.s, z10.s\n" - "fmla z25.s, p3/M, z0.s, z10.s\n" "fmla z16.s, p3/M, z7.s, z10.s\n" "fmla z17.s, p3/M, z6.s, z10.s\n" + "ld1w { z11.s }, p2/Z, [x12, x13, LSL #2]\n" + "fmla z20.s, p3/M, z4.s, z9.s\n" + "fmla z21.s, p3/M, z3.s, z9.s\n" + "fmla z28.s, p3/M, z1.s, z9.s\n" + "fmla z29.s, p3/M, z0.s, z9.s\n" + "fmla z24.s, p3/M, z7.s, z9.s\n" + "fmla z25.s, p3/M, z6.s, z9.s\n" "ld1w { z10.s }, p2/Z, [x5, x8, LSL #2]\n" - "fmla z30.s, p3/M, z8.s, z11.s\n" - "fmla z31.s, p3/M, z7.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x16, x4, LSL #2]\n" - "fmla z18.s, p3/M, z8.s, z12.s\n" - "fmla z19.s, p3/M, z7.s, z12.s\n" + "fmla z18.s, p3/M, z8.s, z11.s\n" + "fmla z19.s, p3/M, z7.s, z11.s\n" + "ld1w { z14.s }, p2/Z, [x16, x4, LSL #2]\n" + "fmla z26.s, p3/M, z8.s, z12.s\n" + "fmla z27.s, p3/M, z7.s, z12.s\n" "fmla z22.s, p3/M, z5.s, z12.s\n" "fmla z23.s, p3/M, z4.s, z12.s\n" - "fmla z26.s, p3/M, z2.s, z12.s\n" - "fmla z27.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x5, x15, LSL #2]\n" - "fmla z20.s, p3/M, z7.s, z11.s\n" - "fmla z21.s, p3/M, z6.s, z11.s\n" - "fmla z24.s, p3/M, z4.s, z11.s\n" - "fmla z25.s, p3/M, z3.s, z11.s\n" - "fmla z28.s, p3/M, z1.s, z11.s\n" - "fmla z29.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x16, x13, LSL #2]\n" - "fmla z16.s, p3/M, z2.s, z10.s\n" - "fmla z17.s, p3/M, z1.s, z10.s\n" - "fmla z18.s, p3/M, z0.s, z10.s\n" + "fmla z30.s, p3/M, z2.s, z12.s\n" + "fmla z31.s, p3/M, z1.s, z12.s\n" + "ld1w { z9.s }, p2/Z, [x5, x15, LSL #2]\n" + "fmla z20.s, p3/M, z7.s, z14.s\n" + "fmla z21.s, p3/M, z6.s, z14.s\n" + "fmla z28.s, p3/M, z4.s, z14.s\n" + "fmla z29.s, p3/M, z3.s, z14.s\n" + "fmla z16.s, p3/M, z1.s, z14.s\n" + "fmla z17.s, p3/M, z0.s, z14.s\n" + "ld1w { z14.s }, p2/Z, [x16, x13, LSL #2]\n" + "fmla z24.s, p3/M, z2.s, z10.s\n" + "fmla z25.s, p3/M, z1.s, z10.s\n" + "fmla z26.s, p3/M, z0.s, z10.s\n" "ld1w { z10.s }, p2/Z, [x7]\n" - "fmla z30.s, p3/M, z2.s, z11.s\n" - "fmla z19.s, p3/M, z0.s, z12.s\n" + "fmla z18.s, p3/M, z2.s, z14.s\n" + "fmla z27.s, p3/M, z0.s, z9.s\n" "fmla z20.s, p3/M, z3.s, z10.s\n" - "fmla z24.s, p3/M, z0.s, z10.s\n" - "fmla z22.s, p3/M, z8.s, z11.s\n" - "fmla z23.s, p3/M, z7.s, z11.s\n" - "fmla z26.s, p3/M, z5.s, z11.s\n" - "fmla z27.s, p3/M, z4.s, z11.s\n" - "fmla z31.s, p3/M, z1.s, z11.s\n" + "fmla z28.s, p3/M, z0.s, z10.s\n" + "fmla z22.s, p3/M, z8.s, z14.s\n" + "fmla z23.s, p3/M, z7.s, z14.s\n" + "fmla z30.s, p3/M, z5.s, z14.s\n" + "fmla z31.s, p3/M, z4.s, z14.s\n" + "fmla z19.s, p3/M, z1.s, z14.s\n" "ld1w { z11.s }, p2/Z, [x14, x8, LSL #2]\n" - "fmla z17.s, p3/M, z2.s, z12.s\n" - "fmla z18.s, p3/M, z1.s, z12.s\n" + "fmla z25.s, p3/M, z2.s, z9.s\n" + "fmla z26.s, p3/M, z1.s, z9.s\n" "ld1w { z12.s }, p2/Z, [x7, x11, LSL #2]\n" - "fmla z16.s, p3/M, z6.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x16]\n" - "fmla z29.s, p3/M, z4.s, z11.s\n" - "fmla z30.s, p3/M, z3.s, z11.s\n" - "fmla z19.s, p3/M, z8.s, z12.s\n" + "fmla z24.s, p3/M, z6.s, z10.s\n" + "ld1w { z14.s }, p2/Z, [x16]\n" + "fmla z17.s, p3/M, z4.s, z11.s\n" + "fmla z18.s, p3/M, z3.s, z11.s\n" + "fmla z27.s, p3/M, z8.s, z12.s\n" "fmla z23.s, p3/M, z5.s, z12.s\n" - "fmla z27.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x16, x11, LSL #2]\n" - "fmla z20.s, p3/M, z6.s, z10.s\n" - "fmla z24.s, p3/M, z3.s, z10.s\n" - "fmla z28.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x12, x8, LSL #2]\n" "fmla z31.s, p3/M, z2.s, z12.s\n" - "fmla z29.s, p3/M, z7.s, z10.s\n" - "fmla z30.s, p3/M, z6.s, z10.s\n" - "fmla z24.s, p3/M, z8.s, z11.s\n" - "fmla z25.s, p3/M, z7.s, z11.s\n" - "fmla z26.s, p3/M, z6.s, z11.s\n" - "fmla z28.s, p3/M, z5.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x14, x15, LSL #2]\n" - "fmla z27.s, p3/M, z5.s, z12.s\n" - "fmla z29.s, p3/M, z5.s, z11.s\n" - "fmla z30.s, p3/M, z4.s, z11.s\n" - "fmla z31.s, p3/M, z3.s, z11.s\n" - "fmla z23.s, p3/M, z8.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x12, x15, LSL #2]\n" - "fmla z28.s, p3/M, z8.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x6, x4, LSL #2]\n" - "fmla z25.s, p3/M, z8.s, z11.s\n" - "fmla z26.s, p3/M, z7.s, z11.s\n" - "fmla z27.s, p3/M, z6.s, z11.s\n" - "fmla z29.s, p3/M, z8.s, z12.s\n" + "ld1w { z9.s }, p2/Z, [x16, x11, LSL #2]\n" + "fmla z20.s, p3/M, z6.s, z14.s\n" + "fmla z28.s, p3/M, z3.s, z14.s\n" + "fmla z16.s, p3/M, z0.s, z14.s\n" + "ld1w { z12.s }, p2/Z, [x12, x8, LSL #2]\n" + "fmla z19.s, p3/M, z2.s, z9.s\n" + "fmla z17.s, p3/M, z7.s, z12.s\n" + "fmla z18.s, p3/M, z6.s, z12.s\n" + "fmla z28.s, p3/M, z8.s, z11.s\n" + "fmla z29.s, p3/M, z7.s, z11.s\n" + "fmla z30.s, p3/M, z6.s, z11.s\n" + "fmla z16.s, p3/M, z5.s, z11.s\n" + "ld1w { z10.s }, p2/Z, [x14, x15, LSL #2]\n" + "fmla z31.s, p3/M, z5.s, z9.s\n" + "fmla z17.s, p3/M, z5.s, z10.s\n" + "fmla z18.s, p3/M, z4.s, z10.s\n" + "fmla z19.s, p3/M, z3.s, z10.s\n" + "fmla z23.s, p3/M, z8.s, z9.s\n" + "ld1w { z14.s }, p2/Z, [x12, x15, LSL #2]\n" + "fmla z16.s, p3/M, z8.s, z12.s\n" + "ld1w { z9.s }, p2/Z, [x6, x4, LSL #2]\n" + "fmla z29.s, p3/M, z8.s, z10.s\n" + "fmla z30.s, p3/M, z7.s, z10.s\n" + "fmla z31.s, p3/M, z6.s, z10.s\n" + "fmla z17.s, p3/M, z8.s, z14.s\n" "ld1w { z11.s }, p2/Z, [x6, x13, LSL #2]\n" - "fmla z30.s, p3/M, z7.s, z12.s\n" - "fmla z31.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x14, x4, LSL #2]\n" - "fmla z16.s, p3/M, z4.s, z10.s\n" - "fmla z17.s, p3/M, z3.s, z10.s\n" - "fmla z20.s, p3/M, z1.s, z10.s\n" - "fmla z21.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x14, x13, LSL #2]\n" - "fmla z18.s, p3/M, z5.s, z11.s\n" - "fmla z19.s, p3/M, z4.s, z11.s\n" + "fmla z18.s, p3/M, z7.s, z14.s\n" + "fmla z19.s, p3/M, z6.s, z14.s\n" + "ld1w { z10.s }, p2/Z, [x14, x4, LSL #2]\n" + "fmla z24.s, p3/M, z4.s, z9.s\n" + "fmla z25.s, p3/M, z3.s, z9.s\n" + "fmla z20.s, p3/M, z1.s, z9.s\n" + "fmla z21.s, p3/M, z0.s, z9.s\n" + "ld1w { z12.s }, p2/Z, [x14, x13, LSL #2]\n" + "fmla z26.s, p3/M, z5.s, z11.s\n" + "fmla z27.s, p3/M, z4.s, z11.s\n" "fmla z22.s, p3/M, z2.s, z11.s\n" "fmla z23.s, p3/M, z1.s, z11.s\n" - "fmla z24.s, p3/M, z7.s, z12.s\n" - "fmla z25.s, p3/M, z6.s, z12.s\n" - "fmla z28.s, p3/M, z4.s, z12.s\n" - "fmla z29.s, p3/M, z3.s, z12.s\n" - "fmla z26.s, p3/M, z8.s, z10.s\n" - "fmla z27.s, p3/M, z7.s, z10.s\n" - "fmla z30.s, p3/M, z5.s, z10.s\n" - "fmla z31.s, p3/M, z4.s, z10.s\n" - ".inst 0xc1adc9d0 // fclamp { z16.s-z19.s }, z14.s, z13.s\n" - ".inst 0xc1adc9d4 // fclamp { z20.s-z23.s }, z14.s, z13.s\n" - "st1w { z16.s }, p0, [x28]\n" - ".inst 0xc1adc9d8 // fclamp { z24.s-z27.s }, z14.s, z13.s\n" - ".inst 0xc1adc9dc // fclamp { z28.s-z31.s }, z14.s, z13.s\n" - "st1w { z17.s }, p0, [x28, x9, LSL #2]\n" - "st1w { z18.s }, p0, [x28, x24, LSL #2]\n" - "st1w { z19.s }, p0, [x28, x22, LSL #2]\n" + "fmla z28.s, p3/M, z7.s, z10.s\n" + "fmla z29.s, p3/M, z6.s, z10.s\n" + "fmla z16.s, p3/M, z4.s, z10.s\n" + "fmla z17.s, p3/M, z3.s, z10.s\n" + "fmla z30.s, p3/M, z8.s, z12.s\n" + "fmla z31.s, p3/M, z7.s, z12.s\n" + "fmla z18.s, p3/M, z5.s, z12.s\n" + "fmla z19.s, p3/M, z4.s, z12.s\n" + ".inst 0xc1afc9b8 // fclamp { z24.s-z27.s }, z13.s, z15.s\n" + ".inst 0xc1afc9b4 // fclamp { z20.s-z23.s }, z13.s, z15.s\n" + "st1w { z24.s }, p0, [x28]\n" + ".inst 0xc1afc9bc // fclamp { z28.s-z31.s }, z13.s, z15.s\n" + ".inst 0xc1afc9b0 // fclamp { z16.s-z19.s }, z13.s, z15.s\n" + "st1w { z25.s }, p0, [x28, x9, LSL #2]\n" + "st1w { z26.s }, p0, [x28, x24, LSL #2]\n" + "st1w { z27.s }, p0, [x28, x22, LSL #2]\n" "st1w { z20.s }, p0, [x27]\n" "st1w { z21.s }, p0, [x27, x9, LSL #2]\n" "st1w { z22.s }, p0, [x27, x24, LSL #2]\n" "st1w { z23.s }, p0, [x27, x22, LSL #2]\n" - "st1w { z24.s }, p0, [x25]\n" - "st1w { z25.s }, p0, [x25, x9, LSL #2]\n" - "st1w { z26.s }, p0, [x25, x24, LSL #2]\n" - "st1w { z27.s }, p0, [x25, x22, LSL #2]\n" - "st1w { z28.s }, p0, [x23]\n" - "st1w { z29.s }, p0, [x23, x9, LSL #2]\n" - "st1w { z30.s }, p0, [x23, x24, LSL #2]\n" - "st1w { z31.s }, p0, [x23, x22, LSL #2]\n" + "st1w { z28.s }, p0, [x25]\n" + "st1w { z29.s }, p0, [x25, x9, LSL #2]\n" + "st1w { z30.s }, p0, [x25, x24, LSL #2]\n" + "st1w { z31.s }, p0, [x25, x22, LSL #2]\n" + "st1w { z16.s }, p0, [x23]\n" + "st1w { z17.s }, p0, [x23, x9, LSL #2]\n" + "st1w { z18.s }, p0, [x23, x24, LSL #2]\n" + "st1w { z19.s }, p0, [x23, x22, LSL #2]\n" "blt 1b\n" ".inst 0xd503467f // SMSTOP\n" : diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp index d904f68806b9d9d620562391b6d2637f80506562..2e2a45bab0f4fcae0e20ef48b78059a7307dae28 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp @@ -22,11 +22,11 @@ * SOFTWARE. */ -#if defined(ARM_COMPUTE_ENABLE_SME2) - #include #include +#if defined(ARM_COMPUTE_ENABLE_SME2) + namespace arm_conv { namespace depthwise { @@ -98,552 +98,552 @@ void sme2_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( activation_min, activation_max); __asm__ __volatile__( - "ldr x17, [%x[params_struct], %[offsetof_args_params]]\n" + "ldr x8, [%x[params_struct], %[offsetof_args_params]]\n" ".inst 0xd503477f // SMSTART ZA\n" - "add x16, %x[params_struct], %[offsetof_Args_inptrs]\n" + "add x17, %x[params_struct], %[offsetof_Args_inptrs]\n" "ptrue p3.b\n" ".inst 0x25207810 // ptrue pn8.b\n" - "ld1w { z15.s }, p3/Z, [x17]\n" - "addvl x17, x17, #1\n" - "ldp x15, x14, [x16, #0x0]\n" - "ldp x13, x12, [x16, #0x10]\n" - "cntw x11\n" - ".inst 0xa040c220 // ld1w { z0.s-z3.s }, pn8.b/Z, [x17]\n" - "addvl x17, x17, #4\n" - "mov x10, #0x0\n" + "ld1w { z13.s }, p3/Z, [x8]\n" + "addvl x8, x8, #1\n" + "ldp x23, x22, [x17, #0x0]\n" + "ldp x21, x20, [x17, #0x10]\n" + "cntw x16\n" + ".inst 0xa040c100 // ld1w { z0.s-z3.s }, pn8.b/Z, [x8]\n" + "addvl x8, x8, #4\n" + "mov x15, #0x0\n" "whilelt p2.s, XZR, %x[n_channels]\n" - ".inst 0xa040c224 // ld1w { z4.s-z7.s }, pn8.b/Z, [x17]\n" - "ldr x9, [%x[params_struct], %[offsetof_args_outptrs]]\n" - "addvl x17, x17, #4\n" - "cmp x11, %x[n_channels]\n" + ".inst 0xa040c104 // ld1w { z4.s-z7.s }, pn8.b/Z, [x8]\n" + "ldr x14, [%x[params_struct], %[offsetof_args_outptrs]]\n" + "addvl x8, x8, #4\n" + "cmp x16, %x[n_channels]\n" "ld1rw { z14.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "ld1rw { z13.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "sub x28, XZR, x11\n" - "ld1w { z8.s }, p3/Z, [x17]\n" - "addvl x17, x17, #1\n" - "ld1w { z9.s }, p2/Z, [x15, x10, LSL #2]\n" - "ld1w { z10.s }, p2/Z, [x14, x10, LSL #2]\n" - "ld1w { z11.s }, p2/Z, [x13, x10, LSL #2]\n" - "ld1w { z12.s }, p2/Z, [x12, x10, LSL #2]\n" + "ld1rw { z15.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "sub x13, XZR, x16\n" + "ld1w { z8.s }, p3/Z, [x8]\n" + "addvl x8, x8, #1\n" + "ld1w { z9.s }, p2/Z, [x23, x15, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x22, x15, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x21, x15, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x20, x15, LSL #2]\n" "bge 2f\n" "1:" // Channel loop - "movprfx z21, z15\n fmla z21.s, p3/M, z4.s, z9.s\n" - "movprfx z16, z15\n fmla z16.s, p3/M, z8.s, z9.s\n" - "ldr x27, [x16, #0x20]\n" - "incw x28\n" - "movprfx z22, z15\n fmla z22.s, p3/M, z3.s, z9.s\n" - "movprfx z25, z15\n fmla z25.s, p3/M, z1.s, z9.s\n" - "ldr x26, [x16, #0x30]\n" + "movprfx z29, z13\n fmla z29.s, p3/M, z4.s, z9.s\n" + "movprfx z16, z13\n fmla z16.s, p3/M, z8.s, z9.s\n" + "ldr x24, [x17, #0x20]\n" + "incw x13\n" + "movprfx z30, z13\n fmla z30.s, p3/M, z3.s, z9.s\n" + "movprfx z25, z13\n fmla z25.s, p3/M, z1.s, z9.s\n" + "ldr x20, [x17, #0x30]\n" "mov p1.b, p2.b\n" - "movprfx z26, z15\n fmla z26.s, p3/M, z0.s, z9.s\n" - "ldr x25, [x16, #0x28]\n" - "movprfx z17, z15\n fmla z17.s, p3/M, z7.s, z9.s\n" - "whilelt p0.s, x11, %x[n_channels]\n" - "movprfx z18, z15\n fmla z18.s, p3/M, z6.s, z9.s\n" - "fmla z21.s, p3/M, z5.s, z12.s\n" - "ldr x24, [x16, #0x38]\n" - "movprfx z20, z15\n fmla z20.s, p3/M, z5.s, z9.s\n" - "movprfx z24, z15\n fmla z24.s, p3/M, z2.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x26, x10, LSL #2]\n" - "ldr x15, [x16, #0x40]\n" + "movprfx z26, z13\n fmla z26.s, p3/M, z0.s, z9.s\n" + "ldr x21, [x17, #0x28]\n" + "movprfx z17, z13\n fmla z17.s, p3/M, z7.s, z9.s\n" + "whilelt p0.s, x16, %x[n_channels]\n" + "movprfx z18, z13\n fmla z18.s, p3/M, z6.s, z9.s\n" + "fmla z29.s, p3/M, z5.s, z12.s\n" + "ldr x23, [x17, #0x38]\n" + "movprfx z28, z13\n fmla z28.s, p3/M, z5.s, z9.s\n" + "movprfx z24, z13\n fmla z24.s, p3/M, z2.s, z9.s\n" + "ld1w { z9.s }, p2/Z, [x20, x15, LSL #2]\n" + "ldr x22, [x17, #0x40]\n" "fmla z16.s, p3/M, z0.s, z10.s\n" - "movprfx z19, z15\n fmla z19.s, p3/M, z2.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x27, x10, LSL #2]\n" - "ldr x14, [x16, #0x48]\n" - "fmla z22.s, p3/M, z4.s, z12.s\n" + "movprfx z19, z13\n fmla z19.s, p3/M, z2.s, z11.s\n" + "ld1w { z22.s }, p2/Z, [x24, x15, LSL #2]\n" + "ldr x20, [x17, #0x48]\n" + "fmla z30.s, p3/M, z4.s, z12.s\n" "fmla z25.s, p3/M, z2.s, z12.s\n" - "ld1w { z11.s }, p2/Z, [x25, x10, LSL #2]\n" - "ldr x13, [x16, #0x50]\n" + "ld1w { z21.s }, p2/Z, [x21, x15, LSL #2]\n" + "ldr x27, [x17, #0x50]\n" "fmla z26.s, p3/M, z1.s, z12.s\n" "fmla z17.s, p3/M, z8.s, z12.s\n" - "ldr x27, [x16, #0x60]\n" + "ldr x26, [x17, #0x60]\n" "fmla z18.s, p3/M, z7.s, z12.s\n" - "movprfx z28, z15\n fmla z28.s, p3/M, z6.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x14, x10, LSL #2]\n" - "ldr x25, [x16, #0x68]\n" - "fmla z21.s, p3/M, z7.s, z9.s\n" + "movprfx z20, z13\n fmla z20.s, p3/M, z6.s, z22.s\n" + "ld1w { z11.s }, p2/Z, [x20, x15, LSL #2]\n" + "ldr x25, [x17, #0x68]\n" + "fmla z29.s, p3/M, z7.s, z9.s\n" "fmla z19.s, p3/M, z6.s, z12.s\n" - "ldr x12, [x16, #0x58]\n" - "movprfx z23, z15\n fmla z23.s, p3/M, z3.s, z12.s\n" - "movprfx z27, z15\n fmla z27.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x24, x10, LSL #2]\n" - "ldr x26, [x16, #0x70]\n" - "movprfx z31, z15\n fmla z31.s, p3/M, z8.s, z11.s\n" - "fmla z22.s, p3/M, z6.s, z9.s\n" - "ld1w { z11.s }, p2/Z, [x15, x10, LSL #2]\n" - "ldr x24, [x16, #0x78]\n" + "ldr x21, [x17, #0x58]\n" + "movprfx z31, z13\n fmla z31.s, p3/M, z3.s, z12.s\n" + "movprfx z27, z13\n fmla z27.s, p3/M, z0.s, z12.s\n" + "ld1w { z10.s }, p2/Z, [x23, x15, LSL #2]\n" + "ldr x24, [x17, #0x70]\n" + "movprfx z23, z13\n fmla z23.s, p3/M, z8.s, z21.s\n" + "fmla z30.s, p3/M, z6.s, z9.s\n" + "ld1w { z12.s }, p2/Z, [x22, x15, LSL #2]\n" + "ldr x23, [x17, #0x78]\n" "fmla z25.s, p3/M, z4.s, z9.s\n" "fmla z26.s, p3/M, z3.s, z9.s\n" - "ldr x15, [x16, #0x80]\n" - "fmla z20.s, p3/M, z8.s, z9.s\n" + "ldr x22, [x17, #0x80]\n" + "movprfx z21, z13\n fmla z21.s, p3/M, z1.s, z9.s\n" + "movprfx z22, z13\n fmla z22.s, p3/M, z0.s, z9.s\n" + "ldr x20, [x17, #0x88]\n" + "ld1w { z13.s }, p3/Z, [x8]\n" + "fmla z28.s, p3/M, z8.s, z9.s\n" "fmla z24.s, p3/M, z5.s, z9.s\n" - "ldr x14, [x16, #0x88]\n" - "fmla z28.s, p3/M, z2.s, z9.s\n" - "fmla z16.s, p3/M, z1.s, z12.s\n" - "ldr x23, [x9, #0x0]\n" - "fmla z17.s, p3/M, z0.s, z12.s\n" - "movprfx z29, z15\n fmla z29.s, p3/M, z1.s, z9.s\n" - "ldr x22, [x9, #0x8]\n" - "movprfx z30, z15\n fmla z30.s, p3/M, z0.s, z9.s\n" - "fmla z18.s, p3/M, z2.s, z11.s\n" - "ld1w { z9.s }, p2/Z, [x13, x10, LSL #2]\n" - "ldr x13, [x16, #0x90]\n" - "fmla z21.s, p3/M, z8.s, z10.s\n" - "fmla z19.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x27, x10, LSL #2]\n" - "ldr x27, [x16, #0xa0]\n" - "fmla z22.s, p3/M, z7.s, z10.s\n" - "fmla z23.s, p3/M, z6.s, z10.s\n" - "ldr x21, [x9, #0x10]\n" - "fmla z25.s, p3/M, z5.s, z10.s\n" - "fmla z26.s, p3/M, z4.s, z10.s\n" - "ldr x20, [x9, #0x18]\n" - "fmla z27.s, p3/M, z3.s, z10.s\n" - "fmla z29.s, p3/M, z2.s, z10.s\n" - "fmla z30.s, p3/M, z1.s, z10.s\n" - "fmla z31.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x25, x10, LSL #2]\n" - "ldr x25, [x16, #0xa8]\n" + "ldr x12, [x14, #0x0]\n" + "addvl x8, x8, #1\n" + "fmla z20.s, p3/M, z2.s, z9.s\n" + "fmla z16.s, p3/M, z1.s, z10.s\n" + "ld1w { z9.s }, p2/Z, [x27, x15, LSL #2]\n" + "ldr x27, [x17, #0x90]\n" + "fmla z17.s, p3/M, z0.s, z10.s\n" + "fmla z18.s, p3/M, z2.s, z12.s\n" + "ld1w { z10.s }, p2/Z, [x21, x15, LSL #2]\n" + "ldr x21, [x17, #0x98]\n" + "fmla z29.s, p3/M, z8.s, z11.s\n" + "fmla z19.s, p3/M, z1.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x26, x15, LSL #2]\n" + "ldr x26, [x17, #0xa0]\n" + "fmla z30.s, p3/M, z7.s, z11.s\n" + "fmla z31.s, p3/M, z6.s, z11.s\n" + "ldr x11, [x14, #0x8]\n" + "fmla z25.s, p3/M, z5.s, z11.s\n" + "fmla z26.s, p3/M, z4.s, z11.s\n" + "ldr x10, [x14, #0x10]\n" + "fmla z27.s, p3/M, z3.s, z11.s\n" + "fmla z21.s, p3/M, z2.s, z11.s\n" + "ldr x9, [x14, #0x18]\n" + "fmla z22.s, p3/M, z1.s, z11.s\n" + "fmla z23.s, p3/M, z0.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x25, x15, LSL #2]\n" + "ldr x25, [x17, #0xa8]\n" "fmla z16.s, p3/M, z3.s, z9.s\n" - "fmla z20.s, p3/M, z0.s, z9.s\n" - "ld1w { z12.s }, p2/Z, [x12, x10, LSL #2]\n" - "ldr x12, [x16, #0x98]\n" - "fmla z24.s, p3/M, z6.s, z11.s\n" - "fmla z28.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x10, LSL #2]\n" - "ldr x26, [x16, #0xb0]\n" - "fmla z17.s, p3/M, z4.s, z10.s\n" - "fmla z18.s, p3/M, z3.s, z10.s\n" - "fmla z21.s, p3/M, z1.s, z10.s\n" - "fmla z19.s, p3/M, z5.s, z12.s\n" - "fmla z23.s, p3/M, z2.s, z12.s\n" - "fmla z22.s, p3/M, z0.s, z10.s\n" - "ld1w { z12.s }, p2/Z, [x24, x10, LSL #2]\n" - "ldr x24, [x16, #0xb8]\n" - "fmla z27.s, p3/M, z8.s, z11.s\n" - "fmla z31.s, p3/M, z5.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x15, x10, LSL #2]\n" - "ldr x15, [x16, #0xc0]\n" - "fmla z16.s, p3/M, z5.s, z10.s\n" - "fmla z20.s, p3/M, z2.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x14, x10, LSL #2]\n" - "ldr x14, [x16, #0xc8]\n" + "fmla z28.s, p3/M, z0.s, z9.s\n" + "fmla z24.s, p3/M, z6.s, z12.s\n" + "fmla z20.s, p3/M, z3.s, z12.s\n" + "ld1w { z9.s }, p2/Z, [x24, x15, LSL #2]\n" + "ldr x24, [x17, #0xb0]\n" + "fmla z17.s, p3/M, z4.s, z11.s\n" + "fmla z18.s, p3/M, z3.s, z11.s\n" + "fmla z29.s, p3/M, z1.s, z11.s\n" + "fmla z19.s, p3/M, z5.s, z10.s\n" + "fmla z31.s, p3/M, z2.s, z10.s\n" + "fmla z30.s, p3/M, z0.s, z11.s\n" + "ld1w { z12.s }, p2/Z, [x23, x15, LSL #2]\n" + "ldr x23, [x17, #0xb8]\n" + "fmla z27.s, p3/M, z8.s, z9.s\n" + "fmla z23.s, p3/M, z5.s, z9.s\n" + "ld1w { z10.s }, p2/Z, [x22, x15, LSL #2]\n" + "ldr x22, [x17, #0xc0]\n" + "fmla z16.s, p3/M, z5.s, z11.s\n" + "fmla z28.s, p3/M, z2.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x20, x15, LSL #2]\n" + "ldr x20, [x17, #0xc8]\n" "fmla z17.s, p3/M, z5.s, z12.s\n" "fmla z18.s, p3/M, z4.s, z12.s\n" - "fmla z21.s, p3/M, z2.s, z12.s\n" + "fmla z29.s, p3/M, z2.s, z12.s\n" "fmla z19.s, p3/M, z3.s, z12.s\n" - "fmla z22.s, p3/M, z1.s, z12.s\n" - "fmla z23.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x12, x10, LSL #2]\n" - "ldr x12, [x16, #0xd8]\n" - "fmla z28.s, p3/M, z7.s, z11.s\n" - "fmla z29.s, p3/M, z6.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x13, x10, LSL #2]\n" - "ldr x13, [x16, #0xd0]\n" - "fmla z16.s, p3/M, z7.s, z10.s\n" - "fmla z17.s, p3/M, z6.s, z10.s\n" - "fmla z20.s, p3/M, z4.s, z10.s\n" - "fmla z21.s, p3/M, z3.s, z10.s\n" - "fmla z24.s, p3/M, z1.s, z10.s\n" - "fmla z25.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x27, x10, LSL #2]\n" - "ldr x27, [x16, #0xe0]\n" - "fmla z18.s, p3/M, z8.s, z12.s\n" - "fmla z30.s, p3/M, z8.s, z11.s\n" - "fmla z31.s, p3/M, z7.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x25, x10, LSL #2]\n" - "fmla z27.s, p3/M, z1.s, z12.s\n" - "ldr x25, [x16, #0xe8]\n" - "fmla z19.s, p3/M, z7.s, z12.s\n" - "fmla z22.s, p3/M, z5.s, z12.s\n" - "fmla z23.s, p3/M, z4.s, z12.s\n" - "fmla z26.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x26, x10, LSL #2]\n" - "ldr x26, [x16, #0xf0]\n" - "fmla z16.s, p3/M, z2.s, z10.s\n" - "fmla z17.s, p3/M, z1.s, z10.s\n" - "fmla z18.s, p3/M, z0.s, z10.s\n" - "fmla z20.s, p3/M, z7.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x24, x10, LSL #2]\n" - "ldr x24, [x16, #0xf8]\n" - "fmla z21.s, p3/M, z6.s, z11.s\n" - "fmla z24.s, p3/M, z4.s, z11.s\n" - "fmla z25.s, p3/M, z3.s, z11.s\n" - "fmla z28.s, p3/M, z1.s, z11.s\n" - "fmla z29.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x15, x10, LSL #2]\n" - "fmla z27.s, p3/M, z4.s, z11.s\n" - "ldr x15, [x16, #0x100]\n" - "fmla z30.s, p3/M, z2.s, z11.s\n" - "fmla z17.s, p3/M, z2.s, z12.s\n" - "fmla z18.s, p3/M, z1.s, z12.s\n" - "fmla z19.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x14, x10, LSL #2]\n" - "ldr x14, [x16, #0x108]\n" - "fmla z16.s, p3/M, z6.s, z10.s\n" - "fmla z20.s, p3/M, z3.s, z10.s\n" - "fmla z24.s, p3/M, z0.s, z10.s\n" - "fmla z22.s, p3/M, z8.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x13, x10, LSL #2]\n" - "ldr x13, [x16, #0x110]\n" - "fmla z23.s, p3/M, z7.s, z11.s\n" - "fmla z26.s, p3/M, z5.s, z11.s\n" - "fmla z31.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x12, x10, LSL #2]\n" - "fmla z27.s, p3/M, z2.s, z12.s\n" - "ldr x12, [x16, #0x118]\n" - "fmla z28.s, p3/M, z0.s, z10.s\n" - "fmla z29.s, p3/M, z4.s, z11.s\n" - "fmla z30.s, p3/M, z3.s, z11.s\n" - "fmla z19.s, p3/M, z8.s, z12.s\n" - "fmla z23.s, p3/M, z5.s, z12.s\n" - "fmla z20.s, p3/M, z6.s, z10.s\n" - "ld1w { z12.s }, p2/Z, [x27, x10, LSL #2]\n" - "fmla z24.s, p3/M, z3.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x25, x10, LSL #2]\n" - "fmla z25.s, p3/M, z7.s, z11.s\n" - "fmla z26.s, p3/M, z6.s, z11.s\n" - "fmla z28.s, p3/M, z5.s, z11.s\n" - "fmla z27.s, p3/M, z5.s, z12.s\n" - "fmla z31.s, p3/M, z2.s, z12.s\n" - "fmla z29.s, p3/M, z7.s, z10.s\n" - "fmla z30.s, p3/M, z6.s, z10.s\n" - "fmla z24.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x10, LSL #2]\n" - "fmla z28.s, p3/M, z8.s, z10.s\n" + "fmla z30.s, p3/M, z1.s, z12.s\n" + "fmla z31.s, p3/M, z0.s, z12.s\n" + "ld1w { z9.s }, p2/Z, [x21, x15, LSL #2]\n" + "ldr x28, [x17, #0xd8]\n" + "fmla z20.s, p3/M, z7.s, z10.s\n" + "fmla z21.s, p3/M, z6.s, z10.s\n" + "ld1w { z10.s }, p2/Z, [x27, x15, LSL #2]\n" + "ldr x21, [x17, #0xd0]\n" + "fmla z16.s, p3/M, z7.s, z11.s\n" + "fmla z17.s, p3/M, z6.s, z11.s\n" + "fmla z28.s, p3/M, z4.s, z11.s\n" + "fmla z29.s, p3/M, z3.s, z11.s\n" + "fmla z24.s, p3/M, z1.s, z11.s\n" + "fmla z25.s, p3/M, z0.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x26, x15, LSL #2]\n" + "ldr x27, [x17, #0xe0]\n" + "fmla z18.s, p3/M, z8.s, z9.s\n" + "fmla z22.s, p3/M, z8.s, z10.s\n" + "fmla z23.s, p3/M, z7.s, z10.s\n" + "ld1w { z10.s }, p2/Z, [x25, x15, LSL #2]\n" + "fmla z27.s, p3/M, z1.s, z9.s\n" + "ldr x26, [x17, #0xe8]\n" + "fmla z19.s, p3/M, z7.s, z9.s\n" + "fmla z30.s, p3/M, z5.s, z9.s\n" + "fmla z31.s, p3/M, z4.s, z9.s\n" + "fmla z26.s, p3/M, z2.s, z9.s\n" + "ld1w { z9.s }, p2/Z, [x24, x15, LSL #2]\n" + "ldr x25, [x17, #0xf0]\n" + "fmla z16.s, p3/M, z2.s, z11.s\n" + "fmla z17.s, p3/M, z1.s, z11.s\n" + "fmla z18.s, p3/M, z0.s, z11.s\n" + "fmla z28.s, p3/M, z7.s, z10.s\n" + "ld1w { z11.s }, p2/Z, [x23, x15, LSL #2]\n" + "ldr x24, [x17, #0xf8]\n" + "fmla z29.s, p3/M, z6.s, z10.s\n" + "fmla z24.s, p3/M, z4.s, z10.s\n" + "fmla z25.s, p3/M, z3.s, z10.s\n" + "fmla z20.s, p3/M, z1.s, z10.s\n" + "fmla z21.s, p3/M, z0.s, z10.s\n" + "ld1w { z10.s }, p2/Z, [x22, x15, LSL #2]\n" + "fmla z27.s, p3/M, z4.s, z10.s\n" + "ldr x23, [x17, #0x100]\n" + "fmla z22.s, p3/M, z2.s, z10.s\n" + "fmla z17.s, p3/M, z2.s, z9.s\n" + "fmla z18.s, p3/M, z1.s, z9.s\n" + "fmla z19.s, p3/M, z0.s, z9.s\n" + "ld1w { z9.s }, p2/Z, [x20, x15, LSL #2]\n" + "ldr x20, [x17, #0x108]\n" + "fmla z16.s, p3/M, z6.s, z11.s\n" + "fmla z28.s, p3/M, z3.s, z11.s\n" + "fmla z24.s, p3/M, z0.s, z11.s\n" + "fmla z30.s, p3/M, z8.s, z10.s\n" + "ld1w { z11.s }, p2/Z, [x21, x15, LSL #2]\n" + "ldr x22, [x17, #0x110]\n" + "fmla z31.s, p3/M, z7.s, z10.s\n" + "fmla z26.s, p3/M, z5.s, z10.s\n" + "fmla z23.s, p3/M, z1.s, z10.s\n" + "ld1w { z10.s }, p2/Z, [x28, x15, LSL #2]\n" + "fmla z27.s, p3/M, z2.s, z9.s\n" + "ldr x21, [x17, #0x118]\n" + "fmla z20.s, p3/M, z0.s, z11.s\n" + "fmla z21.s, p3/M, z4.s, z10.s\n" + "fmla z22.s, p3/M, z3.s, z10.s\n" + "fmla z19.s, p3/M, z8.s, z9.s\n" + "fmla z31.s, p3/M, z5.s, z9.s\n" + "fmla z28.s, p3/M, z6.s, z11.s\n" + "ld1w { z9.s }, p2/Z, [x27, x15, LSL #2]\n" + "fmla z24.s, p3/M, z3.s, z11.s\n" + "ld1w { z12.s }, p2/Z, [x26, x15, LSL #2]\n" + "fmla z25.s, p3/M, z7.s, z10.s\n" + "fmla z26.s, p3/M, z6.s, z10.s\n" + "fmla z20.s, p3/M, z5.s, z10.s\n" + "fmla z27.s, p3/M, z5.s, z9.s\n" + "fmla z23.s, p3/M, z2.s, z9.s\n" + "fmla z21.s, p3/M, z7.s, z12.s\n" + "fmla z22.s, p3/M, z6.s, z12.s\n" + "fmla z24.s, p3/M, z8.s, z10.s\n" + "ld1w { z11.s }, p2/Z, [x25, x15, LSL #2]\n" + "fmla z20.s, p3/M, z8.s, z12.s\n" "fmla z25.s, p3/M, z8.s, z11.s\n" "fmla z26.s, p3/M, z7.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x15, x10, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x23, x15, LSL #2]\n" "fmla z27.s, p3/M, z6.s, z11.s\n" - "fmla z29.s, p3/M, z5.s, z11.s\n" - "fmla z30.s, p3/M, z4.s, z11.s\n" - "fmla z31.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x14, x10, LSL #2]\n" - "ldp x15, x14, [x16, #0x0]\n" - "fmla z23.s, p3/M, z8.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x24, x10, LSL #2]\n" + "fmla z21.s, p3/M, z5.s, z11.s\n" + "fmla z22.s, p3/M, z4.s, z11.s\n" + "fmla z23.s, p3/M, z3.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x20, x15, LSL #2]\n" + "ldp x20, x25, [x17, #0x0]\n" + "fmla z31.s, p3/M, z8.s, z9.s\n" + "ld1w { z12.s }, p2/Z, [x24, x15, LSL #2]\n" "fmla z16.s, p3/M, z4.s, z10.s\n" "fmla z17.s, p3/M, z3.s, z10.s\n" "fmla z18.s, p3/M, z5.s, z11.s\n" + "ld1w { z9.s }, p0/Z, [x20, x16, LSL #2]\n" "fmla z19.s, p3/M, z4.s, z11.s\n" - "fmla z29.s, p3/M, z8.s, z12.s\n" - "fmla z30.s, p3/M, z7.s, z12.s\n" - "fmla z31.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x13, x10, LSL #2]\n" - "fmla z20.s, p3/M, z1.s, z10.s\n" - "fmla z21.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x12, x10, LSL #2]\n" - "ldp x13, x12, [x16, #0x10]\n" - "fmla z22.s, p3/M, z2.s, z11.s\n" - "fmla z23.s, p3/M, z1.s, z11.s\n" - "ld1w { z15.s }, p3/Z, [x17]\n" - "addvl x17, x17, #1\n" - ".inst 0xc1adc9d0 // fclamp { z16.s-z19.s }, z14.s, z13.s\n" - "st1w { z16.s }, p1, [x23, x28, LSL #2]\n" - "ldr x23, [x9, #0x20]\n" + "fmla z21.s, p3/M, z8.s, z12.s\n" + "fmla z22.s, p3/M, z7.s, z12.s\n" + "fmla z23.s, p3/M, z6.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x22, x15, LSL #2]\n" + "fmla z28.s, p3/M, z1.s, z10.s\n" + "fmla z29.s, p3/M, z0.s, z10.s\n" + "ld1w { z0.s }, p2/Z, [x21, x15, LSL #2]\n" + "ldp x20, x24, [x17, #0x10]\n" + "fmla z30.s, p3/M, z2.s, z11.s\n" + "fmla z31.s, p3/M, z1.s, z11.s\n" + "incw x15\n" + "ld1w { z11.s }, p0/Z, [x20, x16, LSL #2]\n" + ".inst 0xc1afc9d0 // fclamp { z16.s-z19.s }, z14.s, z15.s\n" + "st1w { z16.s }, p1, [x12, x13, LSL #2]\n" + "ldr x23, [x14, #0x20]\n" "fmla z24.s, p3/M, z7.s, z12.s\n" - "st1w { z17.s }, p1, [x22, x28, LSL #2]\n" - "ldr x22, [x9, #0x28]\n" + "st1w { z17.s }, p1, [x11, x13, LSL #2]\n" + "ldr x22, [x14, #0x28]\n" "fmla z25.s, p3/M, z6.s, z12.s\n" - "fmla z26.s, p3/M, z8.s, z10.s\n" - "st1w { z18.s }, p1, [x21, x28, LSL #2]\n" - "ldr x21, [x9, #0x30]\n" - "fmla z27.s, p3/M, z7.s, z10.s\n" - ".inst 0xc1adc9d4 // fclamp { z20.s-z23.s }, z14.s, z13.s\n" - "st1w { z19.s }, p1, [x20, x28, LSL #2]\n" - "ldr x20, [x9, #0x38]\n" - "fmla z28.s, p3/M, z4.s, z12.s\n" - "fmla z29.s, p3/M, z3.s, z12.s\n" - "st1w { z20.s }, p1, [x23, x28, LSL #2]\n" - "ldr x23, [x9, #0x40]\n" - "fmla z30.s, p3/M, z5.s, z10.s\n" - "fmla z31.s, p3/M, z4.s, z10.s\n" - "st1w { z21.s }, p1, [x22, x28, LSL #2]\n" - "ldr x22, [x9, #0x48]\n" - ".inst 0xc1adc9d8 // fclamp { z24.s-z27.s }, z14.s, z13.s\n" - "incw x10\n" - "st1w { z22.s }, p1, [x21, x28, LSL #2]\n" - "ldr x21, [x9, #0x50]\n" - "ld1w { z9.s }, p0/Z, [x15, x11, LSL #2]\n" - "whilelt p2.s, x10, %x[n_channels]\n" - "st1w { z23.s }, p1, [x20, x28, LSL #2]\n" - "ldr x20, [x9, #0x58]\n" - "ld1w { z10.s }, p0/Z, [x14, x11, LSL #2]\n" - ".inst 0xc1adc9dc // fclamp { z28.s-z31.s }, z14.s, z13.s\n" - "st1w { z24.s }, p1, [x23, x28, LSL #2]\n" - "ldr x23, [x9, #0x60]\n" - "ld1w { z11.s }, p0/Z, [x13, x11, LSL #2]\n" - "st1w { z25.s }, p1, [x22, x28, LSL #2]\n" - "ldr x22, [x9, #0x68]\n" - "ld1w { z12.s }, p0/Z, [x12, x11, LSL #2]\n" - "incw x11\n" - "st1w { z26.s }, p1, [x21, x28, LSL #2]\n" - "ldr x21, [x9, #0x70]\n" - ".inst 0xa040c220 // ld1w { z0.s-z3.s }, pn8.b/Z, [x17]\n" - "addvl x17, x17, #4\n" - "st1w { z27.s }, p1, [x20, x28, LSL #2]\n" - "ldr x20, [x9, #0x78]\n" - ".inst 0xa040c224 // ld1w { z4.s-z7.s }, pn8.b/Z, [x17]\n" - "addvl x17, x17, #4\n" - "cmp x11, %x[n_channels]\n" - "st1w { z28.s }, p1, [x23, x28, LSL #2]\n" - "ld1w { z8.s }, p3/Z, [x17]\n" - "addvl x17, x17, #1\n" - "st1w { z29.s }, p1, [x22, x28, LSL #2]\n" - "st1w { z30.s }, p1, [x21, x28, LSL #2]\n" - "st1w { z31.s }, p1, [x20, x28, LSL #2]\n" + "fmla z26.s, p3/M, z8.s, z0.s\n" + "st1w { z18.s }, p1, [x10, x13, LSL #2]\n" + "ldr x21, [x14, #0x30]\n" + "fmla z27.s, p3/M, z7.s, z0.s\n" + ".inst 0xc1afc9dc // fclamp { z28.s-z31.s }, z14.s, z15.s\n" + "st1w { z19.s }, p1, [x9, x13, LSL #2]\n" + "ldr x20, [x14, #0x38]\n" + "fmla z20.s, p3/M, z4.s, z12.s\n" + "fmla z21.s, p3/M, z3.s, z12.s\n" + "st1w { z28.s }, p1, [x23, x13, LSL #2]\n" + "ldr x23, [x14, #0x40]\n" + "fmla z22.s, p3/M, z5.s, z0.s\n" + "fmla z23.s, p3/M, z4.s, z0.s\n" + "st1w { z29.s }, p1, [x22, x13, LSL #2]\n" + "ldr x22, [x14, #0x48]\n" + ".inst 0xc1afc9d8 // fclamp { z24.s-z27.s }, z14.s, z15.s\n" + "ld1w { z10.s }, p0/Z, [x25, x16, LSL #2]\n" + "st1w { z30.s }, p1, [x21, x13, LSL #2]\n" + "ldr x21, [x14, #0x50]\n" + "ld1w { z12.s }, p0/Z, [x24, x16, LSL #2]\n" + "incw x16\n" + "st1w { z31.s }, p1, [x20, x13, LSL #2]\n" + "ldr x20, [x14, #0x58]\n" + ".inst 0xa040c100 // ld1w { z0.s-z3.s }, pn8.b/Z, [x8]\n" + "addvl x8, x8, #4\n" + "st1w { z24.s }, p1, [x23, x13, LSL #2]\n" + "ldr x23, [x14, #0x60]\n" + "whilelt p2.s, x15, %x[n_channels]\n" + ".inst 0xa040c104 // ld1w { z4.s-z7.s }, pn8.b/Z, [x8]\n" + "st1w { z25.s }, p1, [x22, x13, LSL #2]\n" + "ldr x22, [x14, #0x68]\n" + "addvl x8, x8, #4\n" + "cmp x16, %x[n_channels]\n" + "st1w { z26.s }, p1, [x21, x13, LSL #2]\n" + "ldr x21, [x14, #0x70]\n" + ".inst 0xc1afc9d4 // fclamp { z20.s-z23.s }, z14.s, z15.s\n" + "ld1w { z8.s }, p3/Z, [x8]\n" + "st1w { z27.s }, p1, [x20, x13, LSL #2]\n" + "ldr x20, [x14, #0x78]\n" + "addvl x8, x8, #1\n" + "st1w { z20.s }, p1, [x23, x13, LSL #2]\n" + "st1w { z21.s }, p1, [x22, x13, LSL #2]\n" + "st1w { z22.s }, p1, [x21, x13, LSL #2]\n" + "st1w { z23.s }, p1, [x20, x13, LSL #2]\n" "blt 1b\n" "2:" // Channel tail - "movprfx z21, z15\n fmla z21.s, p3/M, z4.s, z9.s\n" - "movprfx z16, z15\n fmla z16.s, p3/M, z8.s, z9.s\n" - "ldr x27, [x16, #0x20]\n" - "incw x28\n" - "movprfx z22, z15\n fmla z22.s, p3/M, z3.s, z9.s\n" - "movprfx z25, z15\n fmla z25.s, p3/M, z1.s, z9.s\n" - "ldr x26, [x16, #0x30]\n" - "mov p1.b, p2.b\n" - "movprfx z26, z15\n fmla z26.s, p3/M, z0.s, z9.s\n" - "ldr x25, [x16, #0x28]\n" - "movprfx z17, z15\n fmla z17.s, p3/M, z7.s, z9.s\n" - "movprfx z18, z15\n fmla z18.s, p3/M, z6.s, z9.s\n" - "fmla z21.s, p3/M, z5.s, z12.s\n" - "ldr x24, [x16, #0x38]\n" - "movprfx z20, z15\n fmla z20.s, p3/M, z5.s, z9.s\n" - "movprfx z24, z15\n fmla z24.s, p3/M, z2.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x26, x10, LSL #2]\n" - "ldr x15, [x16, #0x40]\n" - "fmla z16.s, p3/M, z0.s, z10.s\n" - "movprfx z19, z15\n fmla z19.s, p3/M, z2.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x27, x10, LSL #2]\n" - "ldr x14, [x16, #0x48]\n" - "fmla z22.s, p3/M, z4.s, z12.s\n" + "movprfx z29, z13\n fmla z29.s, p3/M, z4.s, z9.s\n" + "movprfx z20, z13\n fmla z20.s, p3/M, z8.s, z9.s\n" + "ldr x24, [x17, #0x20]\n" + "incw x13\n" + "movprfx z30, z13\n fmla z30.s, p3/M, z3.s, z9.s\n" + "movprfx z25, z13\n fmla z25.s, p3/M, z1.s, z9.s\n" + "ldr x20, [x17, #0x30]\n" + "mov p0.b, p2.b\n" + "movprfx z26, z13\n fmla z26.s, p3/M, z0.s, z9.s\n" + "ldr x23, [x17, #0x28]\n" + "movprfx z21, z13\n fmla z21.s, p3/M, z7.s, z9.s\n" + "movprfx z22, z13\n fmla z22.s, p3/M, z6.s, z9.s\n" + "fmla z29.s, p3/M, z5.s, z12.s\n" + "ldr x22, [x17, #0x38]\n" + "movprfx z28, z13\n fmla z28.s, p3/M, z5.s, z9.s\n" + "movprfx z24, z13\n fmla z24.s, p3/M, z2.s, z9.s\n" + "ld1w { z9.s }, p2/Z, [x20, x15, LSL #2]\n" + "ldr x21, [x17, #0x40]\n" + "fmla z20.s, p3/M, z0.s, z10.s\n" + "movprfx z23, z13\n fmla z23.s, p3/M, z2.s, z11.s\n" + "ld1w { z19.s }, p2/Z, [x24, x15, LSL #2]\n" + "ldr x20, [x17, #0x48]\n" + "fmla z30.s, p3/M, z4.s, z12.s\n" "fmla z25.s, p3/M, z2.s, z12.s\n" - "ld1w { z11.s }, p2/Z, [x25, x10, LSL #2]\n" - "ldr x13, [x16, #0x50]\n" + "ld1w { z17.s }, p2/Z, [x23, x15, LSL #2]\n" + "ldr x27, [x17, #0x50]\n" "fmla z26.s, p3/M, z1.s, z12.s\n" - "fmla z17.s, p3/M, z8.s, z12.s\n" - "ldr x27, [x16, #0x60]\n" - "fmla z18.s, p3/M, z7.s, z12.s\n" - "movprfx z28, z15\n fmla z28.s, p3/M, z6.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x14, x10, LSL #2]\n" - "ldr x25, [x16, #0x68]\n" - "fmla z21.s, p3/M, z7.s, z9.s\n" - "fmla z19.s, p3/M, z6.s, z12.s\n" - "ldr x12, [x16, #0x58]\n" - "movprfx z23, z15\n fmla z23.s, p3/M, z3.s, z12.s\n" - "movprfx z27, z15\n fmla z27.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x24, x10, LSL #2]\n" - "ldr x26, [x16, #0x70]\n" - "movprfx z31, z15\n fmla z31.s, p3/M, z8.s, z11.s\n" - "fmla z22.s, p3/M, z6.s, z9.s\n" - "ld1w { z11.s }, p2/Z, [x15, x10, LSL #2]\n" - "ldr x24, [x16, #0x78]\n" + "fmla z21.s, p3/M, z8.s, z12.s\n" + "ldr x26, [x17, #0x60]\n" + "fmla z22.s, p3/M, z7.s, z12.s\n" + "movprfx z16, z13\n fmla z16.s, p3/M, z6.s, z19.s\n" + "ld1w { z11.s }, p2/Z, [x20, x15, LSL #2]\n" + "ldr x25, [x17, #0x68]\n" + "fmla z29.s, p3/M, z7.s, z9.s\n" + "fmla z23.s, p3/M, z6.s, z12.s\n" + "ldr x20, [x17, #0x58]\n" + "movprfx z31, z13\n fmla z31.s, p3/M, z3.s, z12.s\n" + "movprfx z27, z13\n fmla z27.s, p3/M, z0.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x22, x15, LSL #2]\n" + "ldr x24, [x17, #0x70]\n" + "movprfx z19, z13\n fmla z19.s, p3/M, z8.s, z17.s\n" + "fmla z30.s, p3/M, z6.s, z9.s\n" + "ld1w { z10.s }, p2/Z, [x21, x15, LSL #2]\n" + "ldr x23, [x17, #0x78]\n" "fmla z25.s, p3/M, z4.s, z9.s\n" "fmla z26.s, p3/M, z3.s, z9.s\n" - "ldr x15, [x16, #0x80]\n" - "fmla z20.s, p3/M, z8.s, z9.s\n" + "ldr x22, [x17, #0x80]\n" + "movprfx z17, z13\n fmla z17.s, p3/M, z1.s, z9.s\n" + "movprfx z18, z13\n fmla z18.s, p3/M, z0.s, z9.s\n" + "ldr x21, [x17, #0x88]\n" + "fmla z28.s, p3/M, z8.s, z9.s\n" "fmla z24.s, p3/M, z5.s, z9.s\n" - "ldr x14, [x16, #0x88]\n" - "fmla z28.s, p3/M, z2.s, z9.s\n" - "fmla z16.s, p3/M, z1.s, z12.s\n" - "ldr x23, [x9, #0x0]\n" - "fmla z17.s, p3/M, z0.s, z12.s\n" - "movprfx z29, z15\n fmla z29.s, p3/M, z1.s, z9.s\n" - "ldr x22, [x9, #0x8]\n" - "movprfx z30, z15\n fmla z30.s, p3/M, z0.s, z9.s\n" - "fmla z18.s, p3/M, z2.s, z11.s\n" - "ld1w { z9.s }, p2/Z, [x13, x10, LSL #2]\n" - "ldr x13, [x16, #0x90]\n" - "fmla z21.s, p3/M, z8.s, z10.s\n" - "fmla z19.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x27, x10, LSL #2]\n" - "ldr x27, [x16, #0xa0]\n" - "fmla z22.s, p3/M, z7.s, z10.s\n" - "fmla z23.s, p3/M, z6.s, z10.s\n" - "ldr x21, [x9, #0x10]\n" - "fmla z25.s, p3/M, z5.s, z10.s\n" - "fmla z26.s, p3/M, z4.s, z10.s\n" - "ldr x20, [x9, #0x18]\n" - "fmla z27.s, p3/M, z3.s, z10.s\n" - "fmla z29.s, p3/M, z2.s, z10.s\n" - "fmla z30.s, p3/M, z1.s, z10.s\n" - "fmla z31.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x25, x10, LSL #2]\n" - "ldr x25, [x16, #0xa8]\n" - "fmla z16.s, p3/M, z3.s, z9.s\n" - "fmla z20.s, p3/M, z0.s, z9.s\n" - "ld1w { z12.s }, p2/Z, [x12, x10, LSL #2]\n" - "ldr x12, [x16, #0x98]\n" - "fmla z24.s, p3/M, z6.s, z11.s\n" - "fmla z28.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x10, LSL #2]\n" - "ldr x26, [x16, #0xb0]\n" - "fmla z17.s, p3/M, z4.s, z10.s\n" - "fmla z18.s, p3/M, z3.s, z10.s\n" - "fmla z21.s, p3/M, z1.s, z10.s\n" + "ldr x12, [x14, #0x0]\n" + "fmla z16.s, p3/M, z2.s, z9.s\n" + "fmla z20.s, p3/M, z1.s, z12.s\n" + "ld1w { z9.s }, p2/Z, [x27, x15, LSL #2]\n" + "ldr x27, [x17, #0x90]\n" + "fmla z21.s, p3/M, z0.s, z12.s\n" + "fmla z22.s, p3/M, z2.s, z10.s\n" + "ld1w { z13.s }, p2/Z, [x20, x15, LSL #2]\n" + "ldr x20, [x17, #0x98]\n" + "fmla z29.s, p3/M, z8.s, z11.s\n" + "fmla z23.s, p3/M, z1.s, z10.s\n" + "ld1w { z12.s }, p2/Z, [x26, x15, LSL #2]\n" + "ldr x26, [x17, #0xa0]\n" + "fmla z30.s, p3/M, z7.s, z11.s\n" + "fmla z31.s, p3/M, z6.s, z11.s\n" + "ldr x11, [x14, #0x8]\n" + "fmla z25.s, p3/M, z5.s, z11.s\n" + "fmla z26.s, p3/M, z4.s, z11.s\n" + "ldr x10, [x14, #0x10]\n" + "fmla z27.s, p3/M, z3.s, z11.s\n" + "fmla z17.s, p3/M, z2.s, z11.s\n" + "ldr x9, [x14, #0x18]\n" + "fmla z18.s, p3/M, z1.s, z11.s\n" + "fmla z19.s, p3/M, z0.s, z11.s\n" + "ld1w { z10.s }, p2/Z, [x25, x15, LSL #2]\n" + "ldr x25, [x17, #0xa8]\n" + "fmla z20.s, p3/M, z3.s, z9.s\n" + "fmla z28.s, p3/M, z0.s, z9.s\n" + "fmla z24.s, p3/M, z6.s, z12.s\n" + "fmla z16.s, p3/M, z3.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x24, x15, LSL #2]\n" + "ldr x24, [x17, #0xb0]\n" + "fmla z21.s, p3/M, z4.s, z10.s\n" + "fmla z22.s, p3/M, z3.s, z10.s\n" + "fmla z29.s, p3/M, z1.s, z10.s\n" + "fmla z23.s, p3/M, z5.s, z13.s\n" + "fmla z31.s, p3/M, z2.s, z13.s\n" + "fmla z30.s, p3/M, z0.s, z10.s\n" + "ld1w { z13.s }, p2/Z, [x23, x15, LSL #2]\n" + "ldr x23, [x17, #0xb8]\n" + "fmla z27.s, p3/M, z8.s, z12.s\n" "fmla z19.s, p3/M, z5.s, z12.s\n" - "fmla z23.s, p3/M, z2.s, z12.s\n" - "fmla z22.s, p3/M, z0.s, z10.s\n" - "ld1w { z12.s }, p2/Z, [x24, x10, LSL #2]\n" - "ldr x24, [x16, #0xb8]\n" - "fmla z27.s, p3/M, z8.s, z11.s\n" - "fmla z31.s, p3/M, z5.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x15, x10, LSL #2]\n" - "ldr x15, [x16, #0xc0]\n" - "fmla z16.s, p3/M, z5.s, z10.s\n" - "fmla z20.s, p3/M, z2.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x14, x10, LSL #2]\n" - "ldr x14, [x16, #0xc8]\n" - "fmla z17.s, p3/M, z5.s, z12.s\n" - "fmla z18.s, p3/M, z4.s, z12.s\n" - "fmla z21.s, p3/M, z2.s, z12.s\n" - "fmla z19.s, p3/M, z3.s, z12.s\n" - "fmla z22.s, p3/M, z1.s, z12.s\n" - "fmla z23.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x12, x10, LSL #2]\n" - "ldr x12, [x16, #0xd8]\n" + "ld1w { z9.s }, p2/Z, [x22, x15, LSL #2]\n" + "ldr x22, [x17, #0xc0]\n" + "fmla z20.s, p3/M, z5.s, z10.s\n" + "fmla z28.s, p3/M, z2.s, z10.s\n" + "ld1w { z12.s }, p2/Z, [x21, x15, LSL #2]\n" + "ldr x21, [x17, #0xc8]\n" + "fmla z21.s, p3/M, z5.s, z13.s\n" + "fmla z22.s, p3/M, z4.s, z13.s\n" + "fmla z29.s, p3/M, z2.s, z13.s\n" + "fmla z23.s, p3/M, z3.s, z13.s\n" + "fmla z30.s, p3/M, z1.s, z13.s\n" + "fmla z31.s, p3/M, z0.s, z13.s\n" + "ld1w { z10.s }, p2/Z, [x20, x15, LSL #2]\n" + "ldr x28, [x17, #0xd8]\n" + "fmla z16.s, p3/M, z7.s, z9.s\n" + "fmla z17.s, p3/M, z6.s, z9.s\n" + "ld1w { z11.s }, p2/Z, [x27, x15, LSL #2]\n" + "ldr x20, [x17, #0xd0]\n" + "fmla z20.s, p3/M, z7.s, z12.s\n" + "fmla z21.s, p3/M, z6.s, z12.s\n" + "fmla z28.s, p3/M, z4.s, z12.s\n" + "fmla z29.s, p3/M, z3.s, z12.s\n" + "fmla z24.s, p3/M, z1.s, z12.s\n" + "fmla z25.s, p3/M, z0.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x26, x15, LSL #2]\n" + "ldr x27, [x17, #0xe0]\n" + "fmla z22.s, p3/M, z8.s, z10.s\n" + "fmla z18.s, p3/M, z8.s, z11.s\n" + "fmla z19.s, p3/M, z7.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x25, x15, LSL #2]\n" + "fmla z27.s, p3/M, z1.s, z10.s\n" + "ldr x26, [x17, #0xe8]\n" + "fmla z23.s, p3/M, z7.s, z10.s\n" + "fmla z30.s, p3/M, z5.s, z10.s\n" + "fmla z31.s, p3/M, z4.s, z10.s\n" + "fmla z26.s, p3/M, z2.s, z10.s\n" + "ld1w { z9.s }, p2/Z, [x24, x15, LSL #2]\n" + "ldr x25, [x17, #0xf0]\n" + "fmla z20.s, p3/M, z2.s, z12.s\n" + "fmla z21.s, p3/M, z1.s, z12.s\n" + "fmla z22.s, p3/M, z0.s, z12.s\n" "fmla z28.s, p3/M, z7.s, z11.s\n" + "ld1w { z12.s }, p2/Z, [x23, x15, LSL #2]\n" + "ldr x24, [x17, #0xf8]\n" "fmla z29.s, p3/M, z6.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x13, x10, LSL #2]\n" - "ldr x13, [x16, #0xd0]\n" - "fmla z16.s, p3/M, z7.s, z10.s\n" - "fmla z17.s, p3/M, z6.s, z10.s\n" - "fmla z20.s, p3/M, z4.s, z10.s\n" - "fmla z21.s, p3/M, z3.s, z10.s\n" - "fmla z24.s, p3/M, z1.s, z10.s\n" - "fmla z25.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x27, x10, LSL #2]\n" - "ldr x27, [x16, #0xe0]\n" - "fmla z18.s, p3/M, z8.s, z12.s\n" - "fmla z30.s, p3/M, z8.s, z11.s\n" - "fmla z31.s, p3/M, z7.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x25, x10, LSL #2]\n" - "fmla z27.s, p3/M, z1.s, z12.s\n" - "ldr x25, [x16, #0xe8]\n" - "fmla z19.s, p3/M, z7.s, z12.s\n" - "fmla z22.s, p3/M, z5.s, z12.s\n" - "fmla z23.s, p3/M, z4.s, z12.s\n" - "fmla z26.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x26, x10, LSL #2]\n" - "ldr x26, [x16, #0xf0]\n" - "fmla z16.s, p3/M, z2.s, z10.s\n" - "fmla z17.s, p3/M, z1.s, z10.s\n" - "fmla z18.s, p3/M, z0.s, z10.s\n" - "fmla z20.s, p3/M, z7.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x24, x10, LSL #2]\n" - "ldr x24, [x16, #0xf8]\n" - "fmla z21.s, p3/M, z6.s, z11.s\n" "fmla z24.s, p3/M, z4.s, z11.s\n" "fmla z25.s, p3/M, z3.s, z11.s\n" - "fmla z28.s, p3/M, z1.s, z11.s\n" - "fmla z29.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x15, x10, LSL #2]\n" - "fmla z27.s, p3/M, z4.s, z11.s\n" - "ldr x15, [x16, #0x100]\n" - "fmla z30.s, p3/M, z2.s, z11.s\n" - "fmla z17.s, p3/M, z2.s, z12.s\n" - "fmla z18.s, p3/M, z1.s, z12.s\n" - "fmla z19.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x14, x10, LSL #2]\n" - "ldr x14, [x16, #0x108]\n" - "fmla z16.s, p3/M, z6.s, z10.s\n" - "fmla z20.s, p3/M, z3.s, z10.s\n" - "fmla z24.s, p3/M, z0.s, z10.s\n" - "fmla z22.s, p3/M, z8.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x13, x10, LSL #2]\n" - "ldr x13, [x16, #0x110]\n" - "fmla z23.s, p3/M, z7.s, z11.s\n" - "fmla z26.s, p3/M, z5.s, z11.s\n" - "fmla z31.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x12, x10, LSL #2]\n" - "fmla z27.s, p3/M, z2.s, z12.s\n" - "ldr x12, [x16, #0x118]\n" - "fmla z28.s, p3/M, z0.s, z10.s\n" - "fmla z29.s, p3/M, z4.s, z11.s\n" - "fmla z30.s, p3/M, z3.s, z11.s\n" - "fmla z19.s, p3/M, z8.s, z12.s\n" - "fmla z23.s, p3/M, z5.s, z12.s\n" - "fmla z20.s, p3/M, z6.s, z10.s\n" - "ld1w { z12.s }, p2/Z, [x27, x10, LSL #2]\n" - "fmla z24.s, p3/M, z3.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x25, x10, LSL #2]\n" - "fmla z25.s, p3/M, z7.s, z11.s\n" - "fmla z26.s, p3/M, z6.s, z11.s\n" - "fmla z28.s, p3/M, z5.s, z11.s\n" - "fmla z27.s, p3/M, z5.s, z12.s\n" - "fmla z31.s, p3/M, z2.s, z12.s\n" - "fmla z29.s, p3/M, z7.s, z10.s\n" - "fmla z30.s, p3/M, z6.s, z10.s\n" - "fmla z24.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x10, LSL #2]\n" - "fmla z28.s, p3/M, z8.s, z10.s\n" + "fmla z16.s, p3/M, z1.s, z11.s\n" + "fmla z17.s, p3/M, z0.s, z11.s\n" + "ld1w { z10.s }, p2/Z, [x22, x15, LSL #2]\n" + "fmla z27.s, p3/M, z4.s, z10.s\n" + "ldr x23, [x17, #0x100]\n" + "fmla z18.s, p3/M, z2.s, z10.s\n" + "fmla z21.s, p3/M, z2.s, z9.s\n" + "fmla z22.s, p3/M, z1.s, z9.s\n" + "fmla z23.s, p3/M, z0.s, z9.s\n" + "ld1w { z11.s }, p2/Z, [x21, x15, LSL #2]\n" + "ldr x22, [x17, #0x108]\n" + "fmla z20.s, p3/M, z6.s, z12.s\n" + "fmla z28.s, p3/M, z3.s, z12.s\n" + "fmla z24.s, p3/M, z0.s, z12.s\n" + "fmla z30.s, p3/M, z8.s, z10.s\n" + "ld1w { z12.s }, p2/Z, [x20, x15, LSL #2]\n" + "ldr x21, [x17, #0x110]\n" + "fmla z31.s, p3/M, z7.s, z10.s\n" + "fmla z26.s, p3/M, z5.s, z10.s\n" + "fmla z19.s, p3/M, z1.s, z10.s\n" + "ld1w { z9.s }, p2/Z, [x28, x15, LSL #2]\n" + "fmla z27.s, p3/M, z2.s, z11.s\n" + "ldr x20, [x17, #0x118]\n" + "fmla z16.s, p3/M, z0.s, z12.s\n" + "fmla z17.s, p3/M, z4.s, z9.s\n" + "fmla z18.s, p3/M, z3.s, z9.s\n" + "fmla z23.s, p3/M, z8.s, z11.s\n" + "fmla z31.s, p3/M, z5.s, z11.s\n" + "fmla z28.s, p3/M, z6.s, z12.s\n" + "ld1w { z10.s }, p2/Z, [x27, x15, LSL #2]\n" + "fmla z24.s, p3/M, z3.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x26, x15, LSL #2]\n" + "fmla z25.s, p3/M, z7.s, z9.s\n" + "fmla z26.s, p3/M, z6.s, z9.s\n" + "fmla z16.s, p3/M, z5.s, z9.s\n" + "fmla z27.s, p3/M, z5.s, z10.s\n" + "fmla z19.s, p3/M, z2.s, z10.s\n" + "fmla z17.s, p3/M, z7.s, z12.s\n" + "fmla z18.s, p3/M, z6.s, z12.s\n" + "fmla z24.s, p3/M, z8.s, z9.s\n" + "ld1w { z11.s }, p2/Z, [x25, x15, LSL #2]\n" + "fmla z16.s, p3/M, z8.s, z12.s\n" "fmla z25.s, p3/M, z8.s, z11.s\n" "fmla z26.s, p3/M, z7.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x15, x10, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x23, x15, LSL #2]\n" "fmla z27.s, p3/M, z6.s, z11.s\n" - "fmla z29.s, p3/M, z5.s, z11.s\n" - "fmla z30.s, p3/M, z4.s, z11.s\n" - "fmla z31.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x14, x10, LSL #2]\n" - "fmla z23.s, p3/M, z8.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x24, x10, LSL #2]\n" - "fmla z16.s, p3/M, z4.s, z10.s\n" - "fmla z17.s, p3/M, z3.s, z10.s\n" - "fmla z18.s, p3/M, z5.s, z11.s\n" - "fmla z19.s, p3/M, z4.s, z11.s\n" - "fmla z29.s, p3/M, z8.s, z12.s\n" - "fmla z30.s, p3/M, z7.s, z12.s\n" - "fmla z31.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x13, x10, LSL #2]\n" - "fmla z20.s, p3/M, z1.s, z10.s\n" - "fmla z21.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x12, x10, LSL #2]\n" - "fmla z22.s, p3/M, z2.s, z11.s\n" - "fmla z23.s, p3/M, z1.s, z11.s\n" - ".inst 0xc1adc9d0 // fclamp { z16.s-z19.s }, z14.s, z13.s\n" - "st1w { z16.s }, p1, [x23, x28, LSL #2]\n" - "ldr x23, [x9, #0x20]\n" - "fmla z24.s, p3/M, z7.s, z12.s\n" - "st1w { z17.s }, p1, [x22, x28, LSL #2]\n" - "ldr x22, [x9, #0x28]\n" - "fmla z25.s, p3/M, z6.s, z12.s\n" - "fmla z26.s, p3/M, z8.s, z10.s\n" - "st1w { z18.s }, p1, [x21, x28, LSL #2]\n" - "ldr x21, [x9, #0x30]\n" - "fmla z27.s, p3/M, z7.s, z10.s\n" - ".inst 0xc1adc9d4 // fclamp { z20.s-z23.s }, z14.s, z13.s\n" - "st1w { z19.s }, p1, [x20, x28, LSL #2]\n" - "ldr x20, [x9, #0x38]\n" - "fmla z28.s, p3/M, z4.s, z12.s\n" - "fmla z29.s, p3/M, z3.s, z12.s\n" - "st1w { z20.s }, p1, [x23, x28, LSL #2]\n" - "ldr x23, [x9, #0x40]\n" - "fmla z30.s, p3/M, z5.s, z10.s\n" - "fmla z31.s, p3/M, z4.s, z10.s\n" - "st1w { z21.s }, p1, [x22, x28, LSL #2]\n" - "ldr x22, [x9, #0x48]\n" - ".inst 0xc1adc9d8 // fclamp { z24.s-z27.s }, z14.s, z13.s\n" - ".inst 0xc1adc9dc // fclamp { z28.s-z31.s }, z14.s, z13.s\n" - "st1w { z22.s }, p1, [x21, x28, LSL #2]\n" - "ldr x21, [x9, #0x50]\n" - "st1w { z23.s }, p1, [x20, x28, LSL #2]\n" - "ldr x20, [x9, #0x58]\n" - "st1w { z24.s }, p1, [x23, x28, LSL #2]\n" - "ldr x23, [x9, #0x60]\n" - "st1w { z25.s }, p1, [x22, x28, LSL #2]\n" - "ldr x22, [x9, #0x68]\n" - "st1w { z26.s }, p1, [x21, x28, LSL #2]\n" - "ldr x21, [x9, #0x70]\n" - "st1w { z27.s }, p1, [x20, x28, LSL #2]\n" - "ldr x20, [x9, #0x78]\n" - "st1w { z28.s }, p1, [x23, x28, LSL #2]\n" - "st1w { z29.s }, p1, [x22, x28, LSL #2]\n" - "st1w { z30.s }, p1, [x21, x28, LSL #2]\n" - "st1w { z31.s }, p1, [x20, x28, LSL #2]\n" + "fmla z17.s, p3/M, z5.s, z11.s\n" + "fmla z18.s, p3/M, z4.s, z11.s\n" + "fmla z19.s, p3/M, z3.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x22, x15, LSL #2]\n" + "fmla z31.s, p3/M, z8.s, z10.s\n" + "ld1w { z12.s }, p2/Z, [x24, x15, LSL #2]\n" + "fmla z20.s, p3/M, z4.s, z9.s\n" + "fmla z21.s, p3/M, z3.s, z9.s\n" + "fmla z22.s, p3/M, z5.s, z11.s\n" + "fmla z23.s, p3/M, z4.s, z11.s\n" + "fmla z17.s, p3/M, z8.s, z12.s\n" + "fmla z18.s, p3/M, z7.s, z12.s\n" + "fmla z19.s, p3/M, z6.s, z12.s\n" + "ld1w { z13.s }, p2/Z, [x21, x15, LSL #2]\n" + "fmla z28.s, p3/M, z1.s, z9.s\n" + "fmla z29.s, p3/M, z0.s, z9.s\n" + "ld1w { z0.s }, p2/Z, [x20, x15, LSL #2]\n" + "fmla z30.s, p3/M, z2.s, z11.s\n" + "fmla z31.s, p3/M, z1.s, z11.s\n" + ".inst 0xc1afc9d4 // fclamp { z20.s-z23.s }, z14.s, z15.s\n" + "st1w { z20.s }, p0, [x12, x13, LSL #2]\n" + "ldr x23, [x14, #0x20]\n" + "fmla z24.s, p3/M, z7.s, z13.s\n" + "st1w { z21.s }, p0, [x11, x13, LSL #2]\n" + "ldr x22, [x14, #0x28]\n" + "fmla z25.s, p3/M, z6.s, z13.s\n" + "fmla z26.s, p3/M, z8.s, z0.s\n" + "st1w { z22.s }, p0, [x10, x13, LSL #2]\n" + "ldr x21, [x14, #0x30]\n" + "fmla z27.s, p3/M, z7.s, z0.s\n" + ".inst 0xc1afc9dc // fclamp { z28.s-z31.s }, z14.s, z15.s\n" + "st1w { z23.s }, p0, [x9, x13, LSL #2]\n" + "ldr x20, [x14, #0x38]\n" + "fmla z16.s, p3/M, z4.s, z13.s\n" + "fmla z17.s, p3/M, z3.s, z13.s\n" + "st1w { z28.s }, p0, [x23, x13, LSL #2]\n" + "ldr x23, [x14, #0x40]\n" + "fmla z18.s, p3/M, z5.s, z0.s\n" + "fmla z19.s, p3/M, z4.s, z0.s\n" + "st1w { z29.s }, p0, [x22, x13, LSL #2]\n" + "ldr x22, [x14, #0x48]\n" + ".inst 0xc1afc9d8 // fclamp { z24.s-z27.s }, z14.s, z15.s\n" + ".inst 0xc1afc9d0 // fclamp { z16.s-z19.s }, z14.s, z15.s\n" + "st1w { z30.s }, p0, [x21, x13, LSL #2]\n" + "ldr x21, [x14, #0x50]\n" + "st1w { z31.s }, p0, [x20, x13, LSL #2]\n" + "ldr x20, [x14, #0x58]\n" + "st1w { z24.s }, p0, [x23, x13, LSL #2]\n" + "ldr x23, [x14, #0x60]\n" + "st1w { z25.s }, p0, [x22, x13, LSL #2]\n" + "ldr x22, [x14, #0x68]\n" + "st1w { z26.s }, p0, [x21, x13, LSL #2]\n" + "ldr x21, [x14, #0x70]\n" + "st1w { z27.s }, p0, [x20, x13, LSL #2]\n" + "ldr x20, [x14, #0x78]\n" + "st1w { z16.s }, p0, [x23, x13, LSL #2]\n" + "st1w { z17.s }, p0, [x22, x13, LSL #2]\n" + "st1w { z18.s }, p0, [x21, x13, LSL #2]\n" + "st1w { z19.s }, p0, [x20, x13, LSL #2]\n" ".inst 0xd503467f // SMSTOP\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp index a4ca907e1b178e39017511b74b90949029feb9d5..dcffffeb218741f0371c99f9abeb3dc990917608 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,13 +22,13 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include #pragma once -#if defined(__aarch64__) +#if defined(ARM_COMPUTE_ENABLE_SME2) namespace arm_conv { namespace depthwise { @@ -68,4 +68,4 @@ class sme2_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst : public DepthwiseDepthfirs } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) +#endif // defined(ARM_COMPUTE_ENABLE_SME2) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp index f7f67855c1244f83c4ba75aa694dcbad4e3bcf13..066b9354865bf3d42f59acfbae919d11a5764f4c 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp @@ -22,11 +22,11 @@ * SOFTWARE. */ -#if defined(ARM_COMPUTE_ENABLE_SME2) - #include #include +#if defined(ARM_COMPUTE_ENABLE_SME2) + namespace arm_conv { namespace depthwise { @@ -170,7 +170,7 @@ void sme2_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "ldr x22, [%x[params_struct], %[offsetof_args_ld_output_row]]\n" "mul x21, x2, x22\n" // offset = tile_i * ld_output_row "mov x20, #0x2\n" - "ld1w { z19.s }, p3/Z, [x17]\n" + "ld1w { z22.s }, p3/Z, [x17]\n" "ldr x25, [%x[params_struct], %[offsetof_args_ld_output_col]]\n" "madd x21, x3, x25, x21\n" // offset += tile_j * ld_output_col "addvl x17, x17, #1\n" @@ -178,13 +178,13 @@ void sme2_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "ldr x24, [%x[params_struct], %[offsetof_args_outptr]]\n" "mul x21, x21, x20\n" // offset *= output_tile_size "cntw x23\n" - "ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rw { z26.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" "addvl x17, x17, #4\n" "add x24, x24, x21, LSL #2\n" // outptrs[0] += offset * sizeof(float) ".inst 0xa040c224 // ld1w { z4.s-z7.s }, pn8.b/Z, [x17]\n" "whilelt p2.s, XZR, %x[n_channels]\n" "addvl x17, x17, #4\n" - "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "ld1rw { z24.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "cmp x23, %x[n_channels]\n" "add x22, x24, x22, LSL #2\n" "ld1w { z8.s }, p3/Z, [x17]\n" @@ -201,73 +201,73 @@ void sme2_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "ld1w { z16.s }, p2/Z, [x5, x8, LSL #2]\n" "bge 4f\n" "3:" // Tile loop: Channel loop - "movprfx z28, z19\n fmla z28.s, p3/M, z8.s, z9.s\n" - "movprfx z29, z19\n fmla z29.s, p3/M, z6.s, z9.s\n" + "movprfx z28, z22\n fmla z28.s, p3/M, z8.s, z9.s\n" + "movprfx z29, z22\n fmla z29.s, p3/M, z6.s, z9.s\n" "whilelt p1.s, x23, %x[n_channels]\n" "incw x21\n" "fmla z28.s, p3/M, z0.s, z10.s\n" "fmla z29.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x6, x13, LSL #2]\n" + "ld1w { z18.s }, p2/Z, [x6, x13, LSL #2]\n" "incw x23\n" "fmla z28.s, p3/M, z1.s, z11.s\n" "fmla z29.s, p3/M, z2.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x6, x15, LSL #2]\n" + "ld1w { z27.s }, p2/Z, [x6, x15, LSL #2]\n" "mov p0.b, p2.b\n" "fmla z28.s, p3/M, z3.s, z14.s\n" "fmla z29.s, p3/M, z0.s, z16.s\n" - "ld1w { z13.s }, p2/Z, [x6, x8, LSL #2]\n" + "ld1w { z17.s }, p2/Z, [x6, x8, LSL #2]\n" "addvl x5, x5, #1\n" "fmla z28.s, p3/M, z4.s, z15.s\n" - "fmla z29.s, p3/M, z4.s, z11.s\n" - "ld1w { z14.s }, p2/Z, [x16]\n" + "fmla z29.s, p3/M, z4.s, z27.s\n" + "ld1w { z25.s }, p2/Z, [x16]\n" "addvl x6, x6, #1\n" "fmla z28.s, p3/M, z2.s, z16.s\n" - "fmla z29.s, p3/M, z5.s, z12.s\n" - "ld1w { z15.s }, p2/Z, [x7]\n" + "fmla z29.s, p3/M, z5.s, z18.s\n" + "ld1w { z12.s }, p2/Z, [x7]\n" "incw x20\n" - "movprfx z30, z19\n fmla z30.s, p3/M, z2.s, z9.s\n" - "movprfx z31, z19\n fmla z31.s, p3/M, z0.s, z9.s\n" - "ld1w { z12.s }, p2/Z, [x7, x15, LSL #2]\n" - "fmla z28.s, p3/M, z5.s, z13.s\n" - "fmla z29.s, p3/M, z3.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x16, x15, LSL #2]\n" - "fmla z30.s, p3/M, z3.s, z14.s\n" - "fmla z31.s, p3/M, z4.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x16, x4, LSL #2]\n" - "fmla z30.s, p3/M, z0.s, z15.s\n" - "fmla z31.s, p3/M, z1.s, z12.s\n" - "ld1w { z14.s }, p2/Z, [x16, x13, LSL #2]\n" - "fmla z30.s, p3/M, z4.s, z11.s\n" - "fmla z31.s, p3/M, z5.s, z14.s\n" + "movprfx z30, z22\n fmla z30.s, p3/M, z2.s, z9.s\n" + "movprfx z31, z22\n fmla z31.s, p3/M, z0.s, z9.s\n" + "ld1w { z18.s }, p2/Z, [x7, x15, LSL #2]\n" + "fmla z28.s, p3/M, z5.s, z17.s\n" + "fmla z29.s, p3/M, z3.s, z17.s\n" + "ld1w { z16.s }, p2/Z, [x16, x15, LSL #2]\n" + "fmla z30.s, p3/M, z3.s, z25.s\n" + "fmla z31.s, p3/M, z4.s, z16.s\n" + "ld1w { z10.s }, p2/Z, [x16, x4, LSL #2]\n" + "fmla z30.s, p3/M, z0.s, z12.s\n" + "fmla z31.s, p3/M, z1.s, z18.s\n" + "ld1w { z16.s }, p2/Z, [x16, x13, LSL #2]\n" + "fmla z30.s, p3/M, z4.s, z10.s\n" + "fmla z31.s, p3/M, z5.s, z16.s\n" "ld1w { z16.s }, p2/Z, [x7, x4, LSL #2]\n" - "fmla z28.s, p3/M, z6.s, z15.s\n" - "ld1w { z11.s }, p2/Z, [x7, x13, LSL #2]\n" + "fmla z28.s, p3/M, z6.s, z12.s\n" + "ld1w { z22.s }, p2/Z, [x7, x13, LSL #2]\n" "fmla z30.s, p3/M, z1.s, z16.s\n" "addvl x7, x7, #1\n" - "fmla z31.s, p3/M, z2.s, z11.s\n" + "fmla z31.s, p3/M, z2.s, z22.s\n" "fmla z28.s, p3/M, z7.s, z16.s\n" - "ld1w { z15.s }, p2/Z, [x14]\n" - "ld1w { z16.s }, p2/Z, [x16, x8, LSL #2]\n" - "fmla z30.s, p3/M, z6.s, z15.s\n" - "fmla z31.s, p3/M, z3.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x14]\n" + "ld1w { z17.s }, p2/Z, [x16, x8, LSL #2]\n" + "fmla z30.s, p3/M, z6.s, z16.s\n" + "fmla z31.s, p3/M, z3.s, z17.s\n" "addvl x16, x16, #1\n" - "ld1w { z13.s }, p2/Z, [x14, x4, LSL #2]\n" - "fmla z30.s, p3/M, z7.s, z13.s\n" - "fmla z29.s, p3/M, z7.s, z12.s\n" - "ld1w { z14.s }, p2/Z, [x14, x15, LSL #2]\n" - "fmla z31.s, p3/M, z7.s, z14.s\n" - "fmla z30.s, p3/M, z5.s, z16.s\n" - "ld1w { z15.s }, p2/Z, [x14, x8, LSL #2]\n" - "fmla z31.s, p3/M, z6.s, z15.s\n" - "fmla z29.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x14, x13, LSL #2]\n" - "fmla z30.s, p3/M, z8.s, z15.s\n" - "fmla z31.s, p3/M, z8.s, z11.s\n" + "ld1w { z16.s }, p2/Z, [x14, x4, LSL #2]\n" + "fmla z30.s, p3/M, z7.s, z16.s\n" + "fmla z29.s, p3/M, z7.s, z18.s\n" + "ld1w { z16.s }, p2/Z, [x14, x15, LSL #2]\n" + "fmla z31.s, p3/M, z7.s, z16.s\n" + "fmla z30.s, p3/M, z5.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x14, x8, LSL #2]\n" + "fmla z31.s, p3/M, z6.s, z17.s\n" + "fmla z29.s, p3/M, z8.s, z22.s\n" + "ld1w { z16.s }, p2/Z, [x14, x13, LSL #2]\n" + "fmla z30.s, p3/M, z8.s, z17.s\n" + "fmla z31.s, p3/M, z8.s, z16.s\n" "whilelt p2.s, x21, %x[n_channels]\n" - "ld1w { z19.s }, p3/Z, [x17]\n" + "ld1w { z22.s }, p3/Z, [x17]\n" "addvl x17, x17, #1\n" "cmp x23, %x[n_channels]\n" - ".inst 0xc1b1ca5c // fclamp { z28.s-z31.s }, z18.s, z17.s\n" + ".inst 0xc1b8cb5c // fclamp { z28.s-z31.s }, z26.s, z24.s\n" ".inst 0xa040c220 // ld1w { z0.s-z3.s }, pn8.b/Z, [x17]\n" "addvl x17, x17, #4\n" "addvl x14, x14, #1\n" @@ -291,71 +291,71 @@ void sme2_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "addvl x17, x17, #1\n" "blt 3b\n" "4:" // Tile loop: Channel tail - "movprfx z28, z19\n fmla z28.s, p3/M, z8.s, z9.s\n" - "movprfx z29, z19\n fmla z29.s, p3/M, z6.s, z9.s\n" + "movprfx z28, z22\n fmla z28.s, p3/M, z8.s, z9.s\n" + "movprfx z29, z22\n fmla z29.s, p3/M, z6.s, z9.s\n" "ldr x3, [%x[params_struct], %[offsetof_args_tile_j]]\n" "add x3, x3, #0x1\n" "fmla z28.s, p3/M, z0.s, z10.s\n" "fmla z29.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x6, x13, LSL #2]\n" + "ld1w { z18.s }, p2/Z, [x6, x13, LSL #2]\n" "ldr x2, [%x[params_struct], %[offsetof_args_tile_i]]\n" "fmla z28.s, p3/M, z1.s, z11.s\n" "fmla z29.s, p3/M, z2.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x6, x15, LSL #2]\n" + "ld1w { z17.s }, p2/Z, [x6, x15, LSL #2]\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" "fmla z28.s, p3/M, z3.s, z14.s\n" "fmla z29.s, p3/M, z0.s, z16.s\n" - "ld1w { z13.s }, p2/Z, [x6, x8, LSL #2]\n" + "ld1w { z20.s }, p2/Z, [x6, x8, LSL #2]\n" "ldr x21, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" "fmla z28.s, p3/M, z4.s, z15.s\n" - "fmla z29.s, p3/M, z4.s, z11.s\n" - "ld1w { z14.s }, p2/Z, [x16]\n" + "fmla z29.s, p3/M, z4.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x16]\n" "cmp x3, x20\n" "fmla z28.s, p3/M, z2.s, z16.s\n" - "fmla z29.s, p3/M, z5.s, z12.s\n" - "ld1w { z15.s }, p2/Z, [x7]\n" + "fmla z29.s, p3/M, z5.s, z18.s\n" + "ld1w { z18.s }, p2/Z, [x7]\n" "add x20, x2, #0x1\n" - "movprfx z30, z19\n fmla z30.s, p3/M, z2.s, z9.s\n" - "movprfx z31, z19\n fmla z31.s, p3/M, z0.s, z9.s\n" - "ld1w { z12.s }, p2/Z, [x7, x15, LSL #2]\n" + "movprfx z30, z22\n fmla z30.s, p3/M, z2.s, z9.s\n" + "movprfx z31, z22\n fmla z31.s, p3/M, z0.s, z9.s\n" + "ld1w { z19.s }, p2/Z, [x7, x15, LSL #2]\n" "csel x2, x2, x20, LT\n" - "fmla z28.s, p3/M, z5.s, z13.s\n" - "fmla z29.s, p3/M, z3.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x16, x15, LSL #2]\n" + "fmla z28.s, p3/M, z5.s, z20.s\n" + "fmla z29.s, p3/M, z3.s, z20.s\n" + "ld1w { z16.s }, p2/Z, [x16, x15, LSL #2]\n" "mov p0.b, p2.b\n" - "fmla z30.s, p3/M, z3.s, z14.s\n" - "fmla z31.s, p3/M, z4.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x16, x4, LSL #2]\n" + "fmla z30.s, p3/M, z3.s, z17.s\n" + "fmla z31.s, p3/M, z4.s, z16.s\n" + "ld1w { z17.s }, p2/Z, [x16, x4, LSL #2]\n" "csel x3, x3, XZR, LT\n" - "fmla z30.s, p3/M, z0.s, z15.s\n" - "fmla z31.s, p3/M, z1.s, z12.s\n" - "ld1w { z14.s }, p2/Z, [x16, x13, LSL #2]\n" + "fmla z30.s, p3/M, z0.s, z18.s\n" + "fmla z31.s, p3/M, z1.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x16, x13, LSL #2]\n" "cmp x2, x21\n" - "fmla z30.s, p3/M, z4.s, z11.s\n" - "fmla z31.s, p3/M, z5.s, z14.s\n" + "fmla z30.s, p3/M, z4.s, z17.s\n" + "fmla z31.s, p3/M, z5.s, z16.s\n" "ld1w { z16.s }, p2/Z, [x7, x4, LSL #2]\n" - "fmla z28.s, p3/M, z6.s, z15.s\n" - "ld1w { z11.s }, p2/Z, [x7, x13, LSL #2]\n" + "fmla z28.s, p3/M, z6.s, z18.s\n" + "ld1w { z18.s }, p2/Z, [x7, x13, LSL #2]\n" "fmla z30.s, p3/M, z1.s, z16.s\n" - "fmla z31.s, p3/M, z2.s, z11.s\n" + "fmla z31.s, p3/M, z2.s, z18.s\n" "fmla z28.s, p3/M, z7.s, z16.s\n" - "ld1w { z15.s }, p2/Z, [x14]\n" - "ld1w { z16.s }, p2/Z, [x16, x8, LSL #2]\n" - "fmla z30.s, p3/M, z6.s, z15.s\n" - "fmla z31.s, p3/M, z3.s, z16.s\n" - "ld1w { z13.s }, p2/Z, [x14, x4, LSL #2]\n" - "fmla z30.s, p3/M, z7.s, z13.s\n" - "fmla z29.s, p3/M, z7.s, z12.s\n" - "ld1w { z14.s }, p2/Z, [x14, x15, LSL #2]\n" - "fmla z31.s, p3/M, z7.s, z14.s\n" - "fmla z30.s, p3/M, z5.s, z16.s\n" - "ld1w { z15.s }, p2/Z, [x14, x8, LSL #2]\n" - "fmla z31.s, p3/M, z6.s, z15.s\n" - "fmla z29.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x14, x13, LSL #2]\n" - "fmla z30.s, p3/M, z8.s, z15.s\n" - "fmla z31.s, p3/M, z8.s, z11.s\n" - ".inst 0xc1b1ca5c // fclamp { z28.s-z31.s }, z18.s, z17.s\n" + "ld1w { z16.s }, p2/Z, [x14]\n" + "ld1w { z17.s }, p2/Z, [x16, x8, LSL #2]\n" + "fmla z30.s, p3/M, z6.s, z16.s\n" + "fmla z31.s, p3/M, z3.s, z17.s\n" + "ld1w { z16.s }, p2/Z, [x14, x4, LSL #2]\n" + "fmla z30.s, p3/M, z7.s, z16.s\n" + "fmla z29.s, p3/M, z7.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x14, x15, LSL #2]\n" + "fmla z31.s, p3/M, z7.s, z16.s\n" + "fmla z30.s, p3/M, z5.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x14, x8, LSL #2]\n" + "fmla z31.s, p3/M, z6.s, z17.s\n" + "fmla z29.s, p3/M, z8.s, z18.s\n" + "ld1w { z16.s }, p2/Z, [x14, x13, LSL #2]\n" + "fmla z30.s, p3/M, z8.s, z17.s\n" + "fmla z31.s, p3/M, z8.s, z16.s\n" + ".inst 0xc1b8cb5c // fclamp { z28.s-z31.s }, z26.s, z24.s\n" "st1w { z28.s }, p0, [x24]\n" "st1w { z29.s }, p0, [x24, x25, LSL #2]\n" "st1w { z30.s }, p0, [x22]\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp index e2ff9a214e8713bee75635b0942b3d1ea01fc227..dc7a40ff54139f7d7f9a9765c1797e2eaa92e122 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp @@ -22,11 +22,11 @@ * SOFTWARE. */ -#if defined(ARM_COMPUTE_ENABLE_SME2) - #include #include +#if defined(ARM_COMPUTE_ENABLE_SME2) + namespace arm_conv { namespace depthwise { @@ -93,7 +93,7 @@ void sme2_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( "ptrue p3.b\n" "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" ".inst 0x25207810 // ptrue pn8.b\n" - "ld1w { z19.s }, p3/Z, [x15]\n" + "ld1w { z26.s }, p3/Z, [x15]\n" "addvl x15, x15, #1\n" "ldp x14, x13, [x20, #0x0]\n" "cntw x12\n" @@ -103,119 +103,119 @@ void sme2_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( "mov x9, #0x0\n" "whilelt p2.s, XZR, %x[n_channels]\n" ".inst 0xa040c1e4 // ld1w { z4.s-z7.s }, pn8.b/Z, [x15]\n" - "ldp x28, x27, [x16, #0x0]\n" + "ldp x28, x26, [x16, #0x0]\n" "addvl x15, x15, #4\n" "cmp x12, %x[n_channels]\n" - "ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "ldp x26, x25, [x16, #0x10]\n" - "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "sub x24, XZR, x12\n" + "ld1rw { z25.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ldp x25, x24, [x16, #0x10]\n" + "ld1rw { z24.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "sub x27, XZR, x12\n" "ldp x23, x22, [x16, #0x20]\n" "ld1w { z8.s }, p3/Z, [x15]\n" "addvl x15, x15, #1\n" "ldp x21, x20, [x16, #0x30]\n" "ld1w { z9.s }, p2/Z, [x28, x9, LSL #2]\n" - "ld1w { z10.s }, p2/Z, [x27, x9, LSL #2]\n" - "ld1w { z11.s }, p2/Z, [x26, x9, LSL #2]\n" - "ld1w { z12.s }, p2/Z, [x25, x9, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x26, x9, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x25, x9, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x24, x9, LSL #2]\n" "ld1w { z13.s }, p2/Z, [x23, x9, LSL #2]\n" "ld1w { z14.s }, p2/Z, [x22, x9, LSL #2]\n" "ld1w { z15.s }, p2/Z, [x21, x9, LSL #2]\n" "ld1w { z16.s }, p2/Z, [x20, x9, LSL #2]\n" "bge 2f\n" "1:" // Channel loop - "movprfx z28, z19\n fmla z28.s, p3/M, z8.s, z9.s\n" - "movprfx z29, z19\n fmla z29.s, p3/M, z6.s, z9.s\n" - "ldr x28, [x16, #0x40]\n" + "movprfx z28, z26\n fmla z28.s, p3/M, z8.s, z9.s\n" + "movprfx z29, z26\n fmla z29.s, p3/M, z6.s, z9.s\n" + "ldr x21, [x16, #0x40]\n" "whilelt p1.s, x12, %x[n_channels]\n" "fmla z28.s, p3/M, z0.s, z10.s\n" "fmla z29.s, p3/M, z1.s, z12.s\n" - "ldr x27, [x16, #0x48]\n" - "ld1w { z12.s }, p2/Z, [x27, x9, LSL #2]\n" + "ldr x20, [x16, #0x48]\n" + "ld1w { z18.s }, p2/Z, [x20, x9, LSL #2]\n" "fmla z28.s, p3/M, z1.s, z11.s\n" "fmla z29.s, p3/M, z2.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x28, x9, LSL #2]\n" - "ldr x26, [x16, #0x50]\n" + "ld1w { z22.s }, p2/Z, [x21, x9, LSL #2]\n" + "ldr x20, [x16, #0x50]\n" "fmla z28.s, p3/M, z3.s, z14.s\n" "fmla z29.s, p3/M, z0.s, z16.s\n" - "ld1w { z13.s }, p2/Z, [x26, x9, LSL #2]\n" - "ldr x25, [x16, #0x58]\n" + "ld1w { z17.s }, p2/Z, [x20, x9, LSL #2]\n" + "ldr x20, [x16, #0x58]\n" "fmla z28.s, p3/M, z4.s, z15.s\n" - "fmla z29.s, p3/M, z4.s, z11.s\n" - "ldr x20, [x16, #0x78]\n" - "ld1w { z14.s }, p2/Z, [x25, x9, LSL #2]\n" + "fmla z29.s, p3/M, z4.s, z22.s\n" + "ldr x21, [x16, #0x78]\n" + "ld1w { z23.s }, p2/Z, [x20, x9, LSL #2]\n" "fmla z28.s, p3/M, z2.s, z16.s\n" - "fmla z29.s, p3/M, z5.s, z12.s\n" - "ldr x23, [x16, #0x60]\n" - "ld1w { z15.s }, p2/Z, [x23, x9, LSL #2]\n" - "movprfx z30, z19\n fmla z30.s, p3/M, z2.s, z9.s\n" - "movprfx z31, z19\n fmla z31.s, p3/M, z0.s, z9.s\n" - "ldr x28, [x16, #0x80]\n" - "ld1w { z12.s }, p2/Z, [x28, x9, LSL #2]\n" - "fmla z28.s, p3/M, z5.s, z13.s\n" - "fmla z29.s, p3/M, z3.s, z13.s\n" + "fmla z29.s, p3/M, z5.s, z18.s\n" + "ldr x20, [x16, #0x60]\n" "ld1w { z13.s }, p2/Z, [x20, x9, LSL #2]\n" - "ldr x22, [x16, #0x68]\n" - "fmla z30.s, p3/M, z3.s, z14.s\n" - "fmla z31.s, p3/M, z4.s, z13.s\n" - "ldr x27, [x16, #0x88]\n" - "ld1w { z11.s }, p2/Z, [x22, x9, LSL #2]\n" - "fmla z30.s, p3/M, z0.s, z15.s\n" - "fmla z31.s, p3/M, z1.s, z12.s\n" - "ld1w { z14.s }, p2/Z, [x27, x9, LSL #2]\n" + "movprfx z30, z26\n fmla z30.s, p3/M, z2.s, z9.s\n" + "movprfx z31, z26\n fmla z31.s, p3/M, z0.s, z9.s\n" + "ldr x20, [x16, #0x80]\n" + "ld1w { z18.s }, p2/Z, [x20, x9, LSL #2]\n" + "fmla z28.s, p3/M, z5.s, z17.s\n" + "fmla z29.s, p3/M, z3.s, z17.s\n" + "ld1w { z16.s }, p2/Z, [x21, x9, LSL #2]\n" + "ldr x21, [x16, #0x68]\n" + "fmla z30.s, p3/M, z3.s, z23.s\n" + "fmla z31.s, p3/M, z4.s, z16.s\n" + "ldr x20, [x16, #0x88]\n" + "ld1w { z17.s }, p2/Z, [x21, x9, LSL #2]\n" + "fmla z30.s, p3/M, z0.s, z13.s\n" + "fmla z31.s, p3/M, z1.s, z18.s\n" + "ld1w { z16.s }, p2/Z, [x20, x9, LSL #2]\n" "ldr x21, [x16, #0x70]\n" - "ldr x25, [x16, #0x98]\n" - "fmla z30.s, p3/M, z4.s, z11.s\n" - "fmla z31.s, p3/M, z5.s, z14.s\n" + "ldr x20, [x16, #0x98]\n" + "fmla z30.s, p3/M, z4.s, z17.s\n" + "fmla z31.s, p3/M, z5.s, z16.s\n" "ld1w { z16.s }, p2/Z, [x21, x9, LSL #2]\n" - "fmla z28.s, p3/M, z6.s, z15.s\n" - "ld1w { z11.s }, p2/Z, [x25, x9, LSL #2]\n" - "ldr x26, [x16, #0x90]\n" + "fmla z28.s, p3/M, z6.s, z13.s\n" + "ld1w { z4.s }, p2/Z, [x20, x9, LSL #2]\n" + "ldr x21, [x16, #0x90]\n" "fmla z30.s, p3/M, z1.s, z16.s\n" - "ldr x22, [x16, #0xa8]\n" - "fmla z31.s, p3/M, z2.s, z11.s\n" + "ldr x20, [x16, #0xa8]\n" + "fmla z31.s, p3/M, z2.s, z4.s\n" "fmla z28.s, p3/M, z7.s, z16.s\n" - "ld1w { z15.s }, p2/Z, [x26, x9, LSL #2]\n" - "ld1w { z16.s }, p2/Z, [x22, x9, LSL #2]\n" - "ldr x23, [x16, #0xa0]\n" - "fmla z30.s, p3/M, z6.s, z15.s\n" - "fmla z31.s, p3/M, z3.s, z16.s\n" - "ldr x21, [x16, #0xb0]\n" - "ld1w { z13.s }, p2/Z, [x23, x9, LSL #2]\n" - "fmla z30.s, p3/M, z7.s, z13.s\n" - "fmla z29.s, p3/M, z7.s, z12.s\n" - "ld1w { z14.s }, p2/Z, [x21, x9, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x21, x9, LSL #2]\n" + "ld1w { z17.s }, p2/Z, [x20, x9, LSL #2]\n" + "ldr x21, [x16, #0xa0]\n" + "fmla z30.s, p3/M, z6.s, z16.s\n" + "fmla z31.s, p3/M, z3.s, z17.s\n" + "ldr x20, [x16, #0xb0]\n" + "ld1w { z16.s }, p2/Z, [x21, x9, LSL #2]\n" + "fmla z30.s, p3/M, z7.s, z16.s\n" + "fmla z29.s, p3/M, z7.s, z18.s\n" + "ld1w { z16.s }, p2/Z, [x20, x9, LSL #2]\n" "ldr x20, [x16, #0xb8]\n" - "fmla z31.s, p3/M, z7.s, z14.s\n" - "fmla z30.s, p3/M, z5.s, z16.s\n" - "ld1w { z15.s }, p2/Z, [x20, x9, LSL #2]\n" - "ldr x28, [x16, #0xc0]\n" - "fmla z31.s, p3/M, z6.s, z15.s\n" - "fmla z29.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x28, x9, LSL #2]\n" - "ldp x28, x27, [x16, #0x0]\n" - "fmla z30.s, p3/M, z8.s, z15.s\n" - "fmla z31.s, p3/M, z8.s, z11.s\n" - "ldp x26, x25, [x16, #0x10]\n" - "ld1w { z19.s }, p3/Z, [x15]\n" + "fmla z31.s, p3/M, z7.s, z16.s\n" + "fmla z30.s, p3/M, z5.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x20, x9, LSL #2]\n" + "ldr x20, [x16, #0xc0]\n" + "fmla z31.s, p3/M, z6.s, z17.s\n" + "fmla z29.s, p3/M, z8.s, z4.s\n" + "ld1w { z16.s }, p2/Z, [x20, x9, LSL #2]\n" + "ldp x20, x26, [x16, #0x0]\n" + "fmla z30.s, p3/M, z8.s, z17.s\n" + "fmla z31.s, p3/M, z8.s, z16.s\n" + "ldp x25, x24, [x16, #0x10]\n" + "ld1w { z26.s }, p3/Z, [x15]\n" "addvl x15, x15, #1\n" "incw x9\n" "ldp x23, x22, [x16, #0x20]\n" - "ld1w { z9.s }, p1/Z, [x28, x12, LSL #2]\n" - "incw x24\n" + "ld1w { z9.s }, p1/Z, [x20, x12, LSL #2]\n" + "incw x27\n" "mov p0.b, p2.b\n" "ldp x21, x20, [x16, #0x30]\n" - "ld1w { z10.s }, p1/Z, [x27, x12, LSL #2]\n" + "ld1w { z10.s }, p1/Z, [x26, x12, LSL #2]\n" "whilelt p2.s, x9, %x[n_channels]\n" - ".inst 0xc1b1ca5c // fclamp { z28.s-z31.s }, z18.s, z17.s\n" - "ld1w { z11.s }, p1/Z, [x26, x12, LSL #2]\n" - "st1w { z28.s }, p0, [x14, x24, LSL #2]\n" - "ld1w { z12.s }, p1/Z, [x25, x12, LSL #2]\n" - "st1w { z29.s }, p0, [x13, x24, LSL #2]\n" + ".inst 0xc1b8cb3c // fclamp { z28.s-z31.s }, z25.s, z24.s\n" + "ld1w { z11.s }, p1/Z, [x25, x12, LSL #2]\n" + "st1w { z28.s }, p0, [x14, x27, LSL #2]\n" + "ld1w { z12.s }, p1/Z, [x24, x12, LSL #2]\n" + "st1w { z29.s }, p0, [x13, x27, LSL #2]\n" "ld1w { z13.s }, p1/Z, [x23, x12, LSL #2]\n" - "st1w { z30.s }, p0, [x11, x24, LSL #2]\n" + "st1w { z30.s }, p0, [x11, x27, LSL #2]\n" "ld1w { z14.s }, p1/Z, [x22, x12, LSL #2]\n" - "st1w { z31.s }, p0, [x10, x24, LSL #2]\n" + "st1w { z31.s }, p0, [x10, x27, LSL #2]\n" "ld1w { z15.s }, p1/Z, [x21, x12, LSL #2]\n" "ld1w { z16.s }, p1/Z, [x20, x12, LSL #2]\n" "incw x12\n" @@ -228,83 +228,83 @@ void sme2_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( "addvl x15, x15, #1\n" "blt 1b\n" "2:" // Channel tail - "movprfx z28, z19\n fmla z28.s, p3/M, z8.s, z9.s\n" - "movprfx z29, z19\n fmla z29.s, p3/M, z6.s, z9.s\n" - "ldr x28, [x16, #0x40]\n" - "incw x24\n" + "movprfx z28, z26\n fmla z28.s, p3/M, z8.s, z9.s\n" + "movprfx z29, z26\n fmla z29.s, p3/M, z6.s, z9.s\n" + "ldr x21, [x16, #0x40]\n" + "incw x27\n" "fmla z28.s, p3/M, z0.s, z10.s\n" "fmla z29.s, p3/M, z1.s, z12.s\n" - "ldr x27, [x16, #0x48]\n" - "ld1w { z12.s }, p2/Z, [x27, x9, LSL #2]\n" + "ldr x20, [x16, #0x48]\n" + "ld1w { z18.s }, p2/Z, [x20, x9, LSL #2]\n" "fmla z28.s, p3/M, z1.s, z11.s\n" "fmla z29.s, p3/M, z2.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x28, x9, LSL #2]\n" - "ldr x26, [x16, #0x50]\n" + "ld1w { z17.s }, p2/Z, [x21, x9, LSL #2]\n" + "ldr x20, [x16, #0x50]\n" "fmla z28.s, p3/M, z3.s, z14.s\n" "fmla z29.s, p3/M, z0.s, z16.s\n" - "ld1w { z13.s }, p2/Z, [x26, x9, LSL #2]\n" - "ldr x25, [x16, #0x58]\n" + "ld1w { z20.s }, p2/Z, [x20, x9, LSL #2]\n" + "ldr x20, [x16, #0x58]\n" "fmla z28.s, p3/M, z4.s, z15.s\n" - "fmla z29.s, p3/M, z4.s, z11.s\n" - "ldr x20, [x16, #0x78]\n" - "ld1w { z14.s }, p2/Z, [x25, x9, LSL #2]\n" + "fmla z29.s, p3/M, z4.s, z17.s\n" + "ldr x21, [x16, #0x78]\n" + "ld1w { z17.s }, p2/Z, [x20, x9, LSL #2]\n" "fmla z28.s, p3/M, z2.s, z16.s\n" - "fmla z29.s, p3/M, z5.s, z12.s\n" - "ldr x23, [x16, #0x60]\n" - "ld1w { z15.s }, p2/Z, [x23, x9, LSL #2]\n" - "movprfx z30, z19\n fmla z30.s, p3/M, z2.s, z9.s\n" - "movprfx z31, z19\n fmla z31.s, p3/M, z0.s, z9.s\n" - "ldr x28, [x16, #0x80]\n" - "ld1w { z12.s }, p2/Z, [x28, x9, LSL #2]\n" - "fmla z28.s, p3/M, z5.s, z13.s\n" - "fmla z29.s, p3/M, z3.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x20, x9, LSL #2]\n" - "ldr x22, [x16, #0x68]\n" - "fmla z30.s, p3/M, z3.s, z14.s\n" - "fmla z31.s, p3/M, z4.s, z13.s\n" - "ldr x27, [x16, #0x88]\n" - "ld1w { z11.s }, p2/Z, [x22, x9, LSL #2]\n" - "fmla z30.s, p3/M, z0.s, z15.s\n" - "fmla z31.s, p3/M, z1.s, z12.s\n" - "ld1w { z14.s }, p2/Z, [x27, x9, LSL #2]\n" + "fmla z29.s, p3/M, z5.s, z18.s\n" + "ldr x20, [x16, #0x60]\n" + "ld1w { z18.s }, p2/Z, [x20, x9, LSL #2]\n" + "movprfx z30, z26\n fmla z30.s, p3/M, z2.s, z9.s\n" + "movprfx z31, z26\n fmla z31.s, p3/M, z0.s, z9.s\n" + "ldr x20, [x16, #0x80]\n" + "ld1w { z19.s }, p2/Z, [x20, x9, LSL #2]\n" + "fmla z28.s, p3/M, z5.s, z20.s\n" + "fmla z29.s, p3/M, z3.s, z20.s\n" + "ld1w { z16.s }, p2/Z, [x21, x9, LSL #2]\n" + "ldr x21, [x16, #0x68]\n" + "fmla z30.s, p3/M, z3.s, z17.s\n" + "fmla z31.s, p3/M, z4.s, z16.s\n" + "ldr x20, [x16, #0x88]\n" + "ld1w { z17.s }, p2/Z, [x21, x9, LSL #2]\n" + "fmla z30.s, p3/M, z0.s, z18.s\n" + "fmla z31.s, p3/M, z1.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x20, x9, LSL #2]\n" "ldr x21, [x16, #0x70]\n" - "ldr x25, [x16, #0x98]\n" - "fmla z30.s, p3/M, z4.s, z11.s\n" - "fmla z31.s, p3/M, z5.s, z14.s\n" + "ldr x20, [x16, #0x98]\n" + "fmla z30.s, p3/M, z4.s, z17.s\n" + "fmla z31.s, p3/M, z5.s, z16.s\n" "ld1w { z16.s }, p2/Z, [x21, x9, LSL #2]\n" - "fmla z28.s, p3/M, z6.s, z15.s\n" - "ld1w { z11.s }, p2/Z, [x25, x9, LSL #2]\n" - "ldr x26, [x16, #0x90]\n" + "fmla z28.s, p3/M, z6.s, z18.s\n" + "ld1w { z18.s }, p2/Z, [x20, x9, LSL #2]\n" + "ldr x21, [x16, #0x90]\n" "fmla z30.s, p3/M, z1.s, z16.s\n" - "ldr x22, [x16, #0xa8]\n" - "fmla z31.s, p3/M, z2.s, z11.s\n" + "ldr x20, [x16, #0xa8]\n" + "fmla z31.s, p3/M, z2.s, z18.s\n" "fmla z28.s, p3/M, z7.s, z16.s\n" - "ld1w { z15.s }, p2/Z, [x26, x9, LSL #2]\n" - "ld1w { z16.s }, p2/Z, [x22, x9, LSL #2]\n" - "ldr x23, [x16, #0xa0]\n" - "fmla z30.s, p3/M, z6.s, z15.s\n" - "fmla z31.s, p3/M, z3.s, z16.s\n" - "ldr x21, [x16, #0xb0]\n" - "ld1w { z13.s }, p2/Z, [x23, x9, LSL #2]\n" - "fmla z30.s, p3/M, z7.s, z13.s\n" - "fmla z29.s, p3/M, z7.s, z12.s\n" - "ld1w { z14.s }, p2/Z, [x21, x9, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x21, x9, LSL #2]\n" + "ld1w { z17.s }, p2/Z, [x20, x9, LSL #2]\n" + "ldr x21, [x16, #0xa0]\n" + "fmla z30.s, p3/M, z6.s, z16.s\n" + "fmla z31.s, p3/M, z3.s, z17.s\n" + "ldr x20, [x16, #0xb0]\n" + "ld1w { z16.s }, p2/Z, [x21, x9, LSL #2]\n" + "fmla z30.s, p3/M, z7.s, z16.s\n" + "fmla z29.s, p3/M, z7.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x20, x9, LSL #2]\n" "ldr x20, [x16, #0xb8]\n" - "fmla z31.s, p3/M, z7.s, z14.s\n" - "fmla z30.s, p3/M, z5.s, z16.s\n" - "ld1w { z15.s }, p2/Z, [x20, x9, LSL #2]\n" - "ldr x28, [x16, #0xc0]\n" - "fmla z31.s, p3/M, z6.s, z15.s\n" - "fmla z29.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x28, x9, LSL #2]\n" - "fmla z30.s, p3/M, z8.s, z15.s\n" - "fmla z31.s, p3/M, z8.s, z11.s\n" + "fmla z31.s, p3/M, z7.s, z16.s\n" + "fmla z30.s, p3/M, z5.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x20, x9, LSL #2]\n" + "ldr x20, [x16, #0xc0]\n" + "fmla z31.s, p3/M, z6.s, z17.s\n" + "fmla z29.s, p3/M, z8.s, z18.s\n" + "ld1w { z16.s }, p2/Z, [x20, x9, LSL #2]\n" + "fmla z30.s, p3/M, z8.s, z17.s\n" + "fmla z31.s, p3/M, z8.s, z16.s\n" "mov p0.b, p2.b\n" - ".inst 0xc1b1ca5c // fclamp { z28.s-z31.s }, z18.s, z17.s\n" - "st1w { z28.s }, p0, [x14, x24, LSL #2]\n" - "st1w { z29.s }, p0, [x13, x24, LSL #2]\n" - "st1w { z30.s }, p0, [x11, x24, LSL #2]\n" - "st1w { z31.s }, p0, [x10, x24, LSL #2]\n" + ".inst 0xc1b8cb3c // fclamp { z28.s-z31.s }, z25.s, z24.s\n" + "st1w { z28.s }, p0, [x14, x27, LSL #2]\n" + "st1w { z29.s }, p0, [x13, x27, LSL #2]\n" + "st1w { z30.s }, p0, [x11, x27, LSL #2]\n" + "st1w { z31.s }, p0, [x10, x27, LSL #2]\n" ".inst 0xd503467f // SMSTOP\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_3x3_s1_4rows_mla_za.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_3x3_s1_4rows_mla_za.hpp index d29d0b5496d16c12be13a255e4ea429758e0b2ac..061b0a1e2e14187181561c1a4ec7d611acfeb882 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_3x3_s1_4rows_mla_za.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_3x3_s1_4rows_mla_za.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,8 @@ * SOFTWARE. */ +#include "src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp" + namespace arm_conv { namespace depthwise { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_3x3_s1_4rows_mla_za/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_3x3_s1_4rows_mla_za/generic.cpp index 4d02d29e4ecd1856fc45d113281b545020ca67f0..a3858931469f97b77e7e2b8364cb7ebec8b1b29b 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_3x3_s1_4rows_mla_za/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_3x3_s1_4rows_mla_za/generic.cpp @@ -69,69 +69,69 @@ void sme2_fp32_planar_3x3_s1_4rows_mla_za_impl( Args args = { inptr, ld_in_vl, pad_top, 6u - std::min(6u, pad_top + valid_input_rows), pad_left, weights, bias, valid_input_cols, output_cols, outptrs, outlds, outvllds, start_channel, valid_channels, act_min, act_max }; __asm__ __volatile__( - "ldr x6, [%x[args], %[offsetof_Args_pad_bottom]]\n" + "ldr x7, [%x[args], %[offsetof_Args_pad_bottom]]\n" "mov x20, #0x6\n" ".inst 0xd503477f // SMSTART ZA\n" - "sub x20, x20, x6\n" - "ldr x7, [%x[args], %[offsetof_Args_pad_top]]\n" + "sub x20, x20, x7\n" + "ldr x17, [%x[args], %[offsetof_Args_pad_top]]\n" "ptrue p2.b\n" ".inst 0x25207812 // ptrue pn10.b\n" - "ld1rw { z5.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_min]]\n" - "ldr x17, [%x[args], %[offsetof_Args_n_channels]]\n" - "whilelt p1.s, XZR, x17\n" + "ld1rw { z2.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_min]]\n" + "ldr x16, [%x[args], %[offsetof_Args_n_channels]]\n" + "whilelt p1.s, XZR, x16\n" "whilelt p9.s, XZR, x20\n" - "ld1rw { z11.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_max]]\n" - "whilelt p8.s, XZR, x7\n" + "ld1rw { z24.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_max]]\n" + "whilelt p8.s, XZR, x17\n" "eor p8.b, p2/Z, p8.b, p9.b\n" - "ldr x16, [%x[args], %[offsetof_Args_current_channel]]\n" + "ldr x15, [%x[args], %[offsetof_Args_current_channel]]\n" "1:" // Channel loop "ldr x20, [%x[args], %[offsetof_Args_bias]]\n" - "fmov z16.s, #0x0\n" + "fmov z20.s, #0x0\n" "cbz x20, 2f\n" - "ld1w { z16.s }, p1/Z, [x20, x16, LSL #2]\n" + "ld1w { z20.s }, p1/Z, [x20, x15, LSL #2]\n" "2:" // Load bias: Done - "ldr x15, [%x[args], %[offsetof_Args_input_cols]]\n" - "sub x20, x15, #0x1\n" + "ldr x14, [%x[args], %[offsetof_Args_input_cols]]\n" + "sub x20, x14, #0x1\n" "orr x24, x20, %x[ld_in_col], LSL #18\n" - "mov z17.d, z16.d\n" + "mov z21.d, z20.d\n" "ldr x23, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xa1404ae0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x23]\n" - "orr x24, x17, x24, LSL #20\n" + ".inst 0xa0404ae6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x23]\n" + "orr x24, x16, x24, LSL #20\n" "mov x22, #0x6\n" - "ldr x14, [%x[args], %[offsetof_Args_inptr]]\n" - "ld1w { z3.s }, p2/Z, [x23, #2, MUL VL]\n" + "ldr x13, [%x[args], %[offsetof_Args_inptr]]\n" + "ld1w { z10.s }, p2/Z, [x23, #2, MUL VL]\n" "addvl x23, x23, #3\n" - "add x21, x7, x6\n" - ".inst 0xa0404ae6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x23]\n" + "add x21, x17, x7\n" + ".inst 0xa1404ae0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x23]\n" "lsl x20, %x[ld_in_row], #0x2\n" - "mov z18.d, z16.d\n" - "mov z19.d, z16.d\n" + "mov z22.d, z20.d\n" + "mov z23.d, z20.d\n" "ld1w { z9.s }, p2/Z, [x23, #2, MUL VL]\n" "addvl x23, x23, #3\n" "mov x8, #0x0\n" - "ldr x13, [%x[args], %[offsetof_Args_output_cols]]\n" - ".inst 0xa1404ae2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x23]\n" + "ldr x11, [%x[args], %[offsetof_Args_output_cols]]\n" + ".inst 0xa0404ae4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x23]\n" "lsl x24, x24, #0x2\n" "sub x22, x22, x21\n" "ld1w { z1.s }, p2/Z, [x23, #2, MUL VL]\n" - "madd x20, x20, x7, x14\n" + "madd x20, x20, x17, x13\n" "3:" // Issue prefetches "subs x22, x22, #0x1\n" ".inst 0xf8b84a9c // rprfm pldstrm, x24, [x20]\n" "add x20, x20, %x[ld_in_col], LSL #2\n" "bgt 3b\n" - "ldr x11, [%x[args], %[offsetof_Args_outptrs]]\n" + "ldr x22, [%x[args], %[offsetof_Args_outptrs]]\n" "lsl x20, %x[ld_in_row], #0x2\n" - "msub x14, x7, x20, x14\n" - ".inst 0xc0040e00 // mova za.d[x8, #0], { z16.d-z19.d }\n" + "msub x13, x17, x20, x13\n" + ".inst 0xc0040e80 // mova za.d[x8, #0], { z20.d-z23.d }\n" "ldr x20, [%x[args], %[offsetof_Args_ld_out_cols]]\n" - ".inst 0xc0040e01 // mova za.d[x8, #1], { z16.d-z19.d }\n" + ".inst 0xc0040e81 // mova za.d[x8, #1], { z20.d-z23.d }\n" "mov x10, #0x2\n" - "ldp x9, x28, [x11], #0x10\n" - ".inst 0xc0040e02 // mova za.d[x8, #2], { z16.d-z19.d }\n" + "ldp x9, x28, [x22], #0x10\n" + ".inst 0xc0040e82 // mova za.d[x8, #2], { z20.d-z23.d }\n" "ldp x27, x26, [x20], #0x10\n" "ldr x21, [%x[args], %[offsetof_Args_pad_left]]\n" - "ldp x25, x24, [x11], #0x10\n" + "ldp x25, x24, [x22], #0x10\n" "ldp x23, x22, [x20], #0x10\n" "cbz x21, 5f\n" "cmp x21, x10\n" @@ -140,8 +140,8 @@ void sme2_fp32_planar_3x3_s1_4rows_mla_za_impl( "sub x10, x10, x20\n" "cbz x21, 5f\n" ".inst 0xc0060c0c // mova { z12.d-z15.d }, za.d[x8, #0]\n" - "sub x13, x13, x21\n" - ".inst 0xc1abc8ac // fclamp { z12.s-z15.s }, z5.s, z11.s\n" + "sub x11, x11, x21\n" + ".inst 0xc1b8c84c // fclamp { z12.s-z15.s }, z2.s, z24.s\n" "4:" // Left padding "subs x21, x21, #0x1\n" "st1w { z12.s }, p1, [x9]\n" @@ -154,298 +154,298 @@ void sme2_fp32_planar_3x3_s1_4rows_mla_za_impl( "add x24, x24, x22, LSL #2\n" "bgt 4b\n" "5:" // Left padding: End - "adds XZR, x7, x6\n" + "adds XZR, x17, x7\n" "bne 10f\n" "cbz x10, 8f\n" "cmp x10, #0x1\n" - "sub x15, x15, x10\n" + "sub x14, x14, x10\n" "beq 7f\n" "6:" // Unpadded: 2 priming loads - "add x20, x14, %x[ld_in_row], LSL #2\n" - "ld1w { z23.s }, p1/Z, [x14]\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - "ld1w { z24.s }, p1/Z, [x20]\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" + "ld1w { z14.s }, p1/Z, [x13]\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + "ld1w { z15.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z25.s }, p1/Z, [x20]\n" + "ld1w { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z26.s }, p1/Z, [x20]\n" + "ld1w { z17.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1301ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z0.s\n" - "ld1w { z27.s }, p1/Z, [x20]\n" + ".inst 0xc13619c0 // fmla za.s[x8, 0], { z14.s-z17.s }, z6.s\n" + "ld1w { z18.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1361b00 // fmla za.s[x8, 0], { z24.s-z27.s }, z6.s\n" - "ld1w { z28.s }, p1/Z, [x20]\n" - ".inst 0xc1321b20 // fmla za.s[x8, 0], { z25.s-z28.s }, z2.s\n" + ".inst 0xc13019e0 // fmla za.s[x8, 0], { z15.s-z18.s }, z0.s\n" + "ld1w { z19.s }, p1/Z, [x20]\n" + ".inst 0xc1341a00 // fmla za.s[x8, 0], { z16.s-z19.s }, z4.s\n" "7:" // Unpadded: 1 priming loads - "add x20, x14, %x[ld_in_row], LSL #2\n" - "ld1w { z23.s }, p1/Z, [x14]\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - "ld1w { z24.s }, p1/Z, [x20]\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" + "ld1w { z13.s }, p1/Z, [x13]\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + "ld1w { z14.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z25.s }, p1/Z, [x20]\n" + "ld1w { z15.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z26.s }, p1/Z, [x20]\n" + "ld1w { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1381ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z8.s\n" - ".inst 0xc1301ae1 // fmla za.s[x8, 1], { z23.s-z26.s }, z0.s\n" - "ld1w { z27.s }, p1/Z, [x20]\n" + ".inst 0xc13719a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z7.s\n" + ".inst 0xc13619a1 // fmla za.s[x8, 1], { z13.s-z16.s }, z6.s\n" + "ld1w { z17.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1371b00 // fmla za.s[x8, 0], { z24.s-z27.s }, z7.s\n" - "ld1w { z28.s }, p1/Z, [x20]\n" - ".inst 0xc1361b01 // fmla za.s[x8, 1], { z24.s-z27.s }, z6.s\n" - ".inst 0xc13a1b20 // fmla za.s[x8, 0], { z25.s-z28.s }, z10.s\n" - ".inst 0xc1321b21 // fmla za.s[x8, 1], { z25.s-z28.s }, z2.s\n" + ".inst 0xc13819c0 // fmla za.s[x8, 0], { z14.s-z17.s }, z8.s\n" + "ld1w { z18.s }, p1/Z, [x20]\n" + ".inst 0xc13019c1 // fmla za.s[x8, 1], { z14.s-z17.s }, z0.s\n" + ".inst 0xc13519e0 // fmla za.s[x8, 0], { z15.s-z18.s }, z5.s\n" + ".inst 0xc13419e1 // fmla za.s[x8, 1], { z15.s-z18.s }, z4.s\n" "8:" // Unpadded: 0 priming loads - "cbz x15, 16f\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" - "ld1w { z23.s }, p1/Z, [x14]\n" - "sub x15, x15, #0x1\n" - "ld1w { z24.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - "sub x13, x13, #0x1\n" - "ld1w { z25.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - "cmp x15, x13\n" + "cbz x14, 16f\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" + "ld1w { z25.s }, p1/Z, [x13]\n" + "sub x14, x14, #0x1\n" "ld1w { z26.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "csel x21, x15, x13, LT\n" + "sub x11, x11, #0x1\n" "ld1w { z27.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" + "cmp x14, x11\n" "ld1w { z28.s }, p1/Z, [x20]\n" - "sub x13, x13, x21\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + "csel x21, x14, x11, LT\n" + "ld1w { z29.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + "ld1w { z30.s }, p1/Z, [x20]\n" + "sub x11, x11, x21\n" "cbz x21, 15f\n" "9:" // Unpadded: Main loop - ".inst 0xc1331ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z3.s\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + ".inst 0xc13a1b20 // fmla za.s[x8, 0], { z25.s-z28.s }, z10.s\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" "subs x21, x21, #0x1\n" - ".inst 0xc1391b00 // fmla za.s[x8, 0], { z24.s-z27.s }, z9.s\n" - ".inst 0xc1381ae1 // fmla za.s[x8, 1], { z23.s-z26.s }, z8.s\n" - ".inst 0xc1301ae2 // fmla za.s[x8, 2], { z23.s-z26.s }, z0.s\n" - "ld1w { z23.s }, p1/Z, [x14]\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xc1311b20 // fmla za.s[x8, 0], { z25.s-z28.s }, z1.s\n" - ".inst 0xc1371b01 // fmla za.s[x8, 1], { z24.s-z27.s }, z7.s\n" - ".inst 0xc1361b02 // fmla za.s[x8, 2], { z24.s-z27.s }, z6.s\n" - "ld1w { z24.s }, p1/Z, [x20]\n" + ".inst 0xc1391b40 // fmla za.s[x8, 0], { z26.s-z29.s }, z9.s\n" + ".inst 0xc1371b21 // fmla za.s[x8, 1], { z25.s-z28.s }, z7.s\n" + ".inst 0xc1361b22 // fmla za.s[x8, 2], { z25.s-z28.s }, z6.s\n" + "ld1w { z25.s }, p1/Z, [x13]\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + ".inst 0xc1311b60 // fmla za.s[x8, 0], { z27.s-z30.s }, z1.s\n" + ".inst 0xc1381b41 // fmla za.s[x8, 1], { z26.s-z29.s }, z8.s\n" + ".inst 0xc1301b42 // fmla za.s[x8, 2], { z26.s-z29.s }, z0.s\n" + "ld1w { z26.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0xc0060c0c // mova { z12.d-z15.d }, za.d[x8, #0]\n" - ".inst 0xc1abc8ac // fclamp { z12.s-z15.s }, z5.s, z11.s\n" + ".inst 0xc1b8c84c // fclamp { z12.s-z15.s }, z2.s, z24.s\n" "st1w { z12.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - ".inst 0xc13a1b21 // fmla za.s[x8, 1], { z25.s-z28.s }, z10.s\n" + ".inst 0xc1351b61 // fmla za.s[x8, 1], { z27.s-z30.s }, z5.s\n" "st1w { z13.s }, p1, [x28]\n" "add x28, x28, x26, LSL #2\n" - ".inst 0xc1321b22 // fmla za.s[x8, 2], { z25.s-z28.s }, z2.s\n" - "ld1w { z25.s }, p1/Z, [x20]\n" + ".inst 0xc1341b62 // fmla za.s[x8, 2], { z27.s-z30.s }, z4.s\n" + "ld1w { z27.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "add x8, x8, #0x1\n" - "ld1w { z26.s }, p1/Z, [x20]\n" + "ld1w { z28.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "st1w { z14.s }, p1, [x25]\n" "add x25, x25, x23, LSL #2\n" - "ld1w { z27.s }, p1/Z, [x20]\n" + "ld1w { z29.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "st1w { z15.s }, p1, [x24]\n" "add x24, x24, x22, LSL #2\n" - ".inst 0xc0040e02 // mova za.d[x8, #2], { z16.d-z19.d }\n" - "ld1w { z28.s }, p1/Z, [x20]\n" + ".inst 0xc0040e82 // mova za.d[x8, #2], { z20.d-z23.d }\n" + "ld1w { z30.s }, p1/Z, [x20]\n" "bgt 9b\n" "b 15f\n" "10:" // Padded "cbz x10, 13f\n" "cmp x10, #0x1\n" - "sub x15, x15, x10\n" + "sub x14, x14, x10\n" "beq 12f\n" "11:" // Padded: 2 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z23.s }, p0/Z, [x14]\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + "ld1w { z11.s }, p0/Z, [x13]\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z24.s }, p0/Z, [x20]\n" + "ld1w { z12.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z25.s }, p0/Z, [x20]\n" + "ld1w { z13.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z26.s }, p0/Z, [x20]\n" + "ld1w { z14.s }, p0/Z, [x20]\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1301ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z0.s\n" + ".inst 0xc1361960 // fmla za.s[x8, 0], { z11.s-z14.s }, z6.s\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z27.s }, p0/Z, [x20]\n" + "ld1w { z15.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1361b00 // fmla za.s[x8, 0], { z24.s-z27.s }, z6.s\n" + ".inst 0xc1301980 // fmla za.s[x8, 0], { z12.s-z15.s }, z0.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z28.s }, p0/Z, [x20]\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xc1321b20 // fmla za.s[x8, 0], { z25.s-z28.s }, z2.s\n" + "ld1w { z16.s }, p0/Z, [x20]\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + ".inst 0xc13419a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z4.s\n" "12:" // Padded: 1 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z23.s }, p0/Z, [x14]\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + "ld1w { z11.s }, p0/Z, [x13]\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z24.s }, p0/Z, [x20]\n" + "ld1w { z12.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z25.s }, p0/Z, [x20]\n" + "ld1w { z13.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z26.s }, p0/Z, [x20]\n" + "ld1w { z14.s }, p0/Z, [x20]\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1381ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z8.s\n" + ".inst 0xc1371960 // fmla za.s[x8, 0], { z11.s-z14.s }, z7.s\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1301ae1 // fmla za.s[x8, 1], { z23.s-z26.s }, z0.s\n" - "ld1w { z27.s }, p0/Z, [x20]\n" + ".inst 0xc1361961 // fmla za.s[x8, 1], { z11.s-z14.s }, z6.s\n" + "ld1w { z15.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0xc1371b00 // fmla za.s[x8, 0], { z24.s-z27.s }, z7.s\n" - "ld1w { z28.s }, p0/Z, [x20]\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xc1361b01 // fmla za.s[x8, 1], { z24.s-z27.s }, z6.s\n" - ".inst 0xc13a1b20 // fmla za.s[x8, 0], { z25.s-z28.s }, z10.s\n" - ".inst 0xc1321b21 // fmla za.s[x8, 1], { z25.s-z28.s }, z2.s\n" + ".inst 0xc1381980 // fmla za.s[x8, 0], { z12.s-z15.s }, z8.s\n" + "ld1w { z16.s }, p0/Z, [x20]\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + ".inst 0xc1301981 // fmla za.s[x8, 1], { z12.s-z15.s }, z0.s\n" + ".inst 0xc13519a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z5.s\n" + ".inst 0xc13419a1 // fmla za.s[x8, 1], { z13.s-z16.s }, z4.s\n" "13:" // Padded: 0 priming loads - "cbz x15, 16f\n" + "cbz x14, 16f\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z23.s }, p0/Z, [x14]\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + "ld1w { z25.s }, p0/Z, [x13]\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z24.s }, p0/Z, [x20]\n" + "ld1w { z26.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z25.s }, p0/Z, [x20]\n" + "ld1w { z27.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z26.s }, p0/Z, [x20]\n" + "ld1w { z28.s }, p0/Z, [x20]\n" "mov x12, #0x4\n" - "sub x15, x15, #0x1\n" - "sub x13, x13, #0x1\n" + "sub x14, x14, #0x1\n" + "sub x11, x11, #0x1\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "cmp x15, x13\n" - "ld1w { z27.s }, p0/Z, [x20]\n" + "cmp x14, x11\n" + "ld1w { z29.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z28.s }, p0/Z, [x20]\n" - "csel x21, x15, x13, LT\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - "sub x13, x13, x21\n" + "ld1w { z30.s }, p0/Z, [x20]\n" + "csel x21, x14, x11, LT\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + "sub x11, x11, x21\n" "cbz x21, 15f\n" "14:" // Padded: Main loop - ".inst 0xc1331ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z3.s\n" + ".inst 0xc13a1b20 // fmla za.s[x8, 0], { z25.s-z28.s }, z10.s\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1391b00 // fmla za.s[x8, 0], { z24.s-z27.s }, z9.s\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + ".inst 0xc1391b40 // fmla za.s[x8, 0], { z26.s-z29.s }, z9.s\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" "subs x21, x21, #0x1\n" - ".inst 0xc1381ae1 // fmla za.s[x8, 1], { z23.s-z26.s }, z8.s\n" - ".inst 0xc1301ae2 // fmla za.s[x8, 2], { z23.s-z26.s }, z0.s\n" - "ld1w { z23.s }, p0/Z, [x14]\n" + ".inst 0xc1371b21 // fmla za.s[x8, 1], { z25.s-z28.s }, z7.s\n" + ".inst 0xc1361b22 // fmla za.s[x8, 2], { z25.s-z28.s }, z6.s\n" + "ld1w { z25.s }, p0/Z, [x13]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xc1311b20 // fmla za.s[x8, 0], { z25.s-z28.s }, z1.s\n" - ".inst 0xc1371b01 // fmla za.s[x8, 1], { z24.s-z27.s }, z7.s\n" - ".inst 0xc1361b02 // fmla za.s[x8, 2], { z24.s-z27.s }, z6.s\n" - "ld1w { z24.s }, p0/Z, [x20]\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + ".inst 0xc1311b60 // fmla za.s[x8, 0], { z27.s-z30.s }, z1.s\n" + ".inst 0xc1381b41 // fmla za.s[x8, 1], { z26.s-z29.s }, z8.s\n" + ".inst 0xc1301b42 // fmla za.s[x8, 2], { z26.s-z29.s }, z0.s\n" + "ld1w { z26.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc0060c0c // mova { z12.d-z15.d }, za.d[x8, #0]\n" - ".inst 0xc1abc8ac // fclamp { z12.s-z15.s }, z5.s, z11.s\n" - "st1w { z12.s }, p1, [x9]\n" + ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" + ".inst 0xc1b8c850 // fclamp { z16.s-z19.s }, z2.s, z24.s\n" + "st1w { z16.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - ".inst 0xc13a1b21 // fmla za.s[x8, 1], { z25.s-z28.s }, z10.s\n" - "st1w { z13.s }, p1, [x28]\n" + ".inst 0xc1351b61 // fmla za.s[x8, 1], { z27.s-z30.s }, z5.s\n" + "st1w { z17.s }, p1, [x28]\n" "add x28, x28, x26, LSL #2\n" - ".inst 0xc1321b22 // fmla za.s[x8, 2], { z25.s-z28.s }, z2.s\n" - "ld1w { z25.s }, p0/Z, [x20]\n" + ".inst 0xc1341b62 // fmla za.s[x8, 2], { z27.s-z30.s }, z4.s\n" + "ld1w { z27.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x4\n" - "ld1w { z26.s }, p0/Z, [x20]\n" + "ld1w { z28.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "st1w { z14.s }, p1, [x25]\n" + "st1w { z18.s }, p1, [x25]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "add x8, x8, #0x1\n" - "ld1w { z27.s }, p0/Z, [x20]\n" - "st1w { z15.s }, p1, [x24]\n" + "ld1w { z29.s }, p0/Z, [x20]\n" + "st1w { z19.s }, p1, [x24]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0xc0040e02 // mova za.d[x8, #2], { z16.d-z19.d }\n" - "ld1w { z28.s }, p0/Z, [x20]\n" + ".inst 0xc0040e82 // mova za.d[x8, #2], { z20.d-z23.d }\n" + "ld1w { z30.s }, p0/Z, [x20]\n" "add x25, x25, x23, LSL #2\n" "add x24, x24, x22, LSL #2\n" "bgt 14b\n" "15:" // Main loop tail - ".inst 0xc1331ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z3.s\n" - ".inst 0xc1391b00 // fmla za.s[x8, 0], { z24.s-z27.s }, z9.s\n" - ".inst 0xc1381ae1 // fmla za.s[x8, 1], { z23.s-z26.s }, z8.s\n" - ".inst 0xc1301ae2 // fmla za.s[x8, 2], { z23.s-z26.s }, z0.s\n" - ".inst 0xc1311b20 // fmla za.s[x8, 0], { z25.s-z28.s }, z1.s\n" - ".inst 0xc1371b01 // fmla za.s[x8, 1], { z24.s-z27.s }, z7.s\n" - ".inst 0xc1361b02 // fmla za.s[x8, 2], { z24.s-z27.s }, z6.s\n" - ".inst 0xc0060c0c // mova { z12.d-z15.d }, za.d[x8, #0]\n" - ".inst 0xc1abc8ac // fclamp { z12.s-z15.s }, z5.s, z11.s\n" - "st1w { z12.s }, p1, [x9]\n" + ".inst 0xc13a1b20 // fmla za.s[x8, 0], { z25.s-z28.s }, z10.s\n" + ".inst 0xc1391b40 // fmla za.s[x8, 0], { z26.s-z29.s }, z9.s\n" + ".inst 0xc1371b21 // fmla za.s[x8, 1], { z25.s-z28.s }, z7.s\n" + ".inst 0xc1361b22 // fmla za.s[x8, 2], { z25.s-z28.s }, z6.s\n" + ".inst 0xc1311b60 // fmla za.s[x8, 0], { z27.s-z30.s }, z1.s\n" + ".inst 0xc1381b41 // fmla za.s[x8, 1], { z26.s-z29.s }, z8.s\n" + ".inst 0xc1301b42 // fmla za.s[x8, 2], { z26.s-z29.s }, z0.s\n" + ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" + ".inst 0xc1b8c850 // fclamp { z16.s-z19.s }, z2.s, z24.s\n" + "st1w { z16.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - ".inst 0xc13a1b21 // fmla za.s[x8, 1], { z25.s-z28.s }, z10.s\n" - "st1w { z13.s }, p1, [x28]\n" + ".inst 0xc1351b61 // fmla za.s[x8, 1], { z27.s-z30.s }, z5.s\n" + "st1w { z17.s }, p1, [x28]\n" "add x28, x28, x26, LSL #2\n" - ".inst 0xc1321b22 // fmla za.s[x8, 2], { z25.s-z28.s }, z2.s\n" + ".inst 0xc1341b62 // fmla za.s[x8, 2], { z27.s-z30.s }, z4.s\n" "add x8, x8, #0x1\n" - "st1w { z14.s }, p1, [x25]\n" + "st1w { z18.s }, p1, [x25]\n" "add x25, x25, x23, LSL #2\n" - "st1w { z15.s }, p1, [x24]\n" + "st1w { z19.s }, p1, [x24]\n" "add x24, x24, x22, LSL #2\n" - ".inst 0xc0040e02 // mova za.d[x8, #2], { z16.d-z19.d }\n" + ".inst 0xc0040e82 // mova za.d[x8, #2], { z20.d-z23.d }\n" "16:" // Main loop skip tail - "cbz x13, 18f\n" + "cbz x11, 18f\n" "17:" // Right padding loop - ".inst 0xc0060c0c // mova { z12.d-z15.d }, za.d[x8, #0]\n" + ".inst 0xc0060c08 // mova { z8.d-z11.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" - "subs x13, x13, #0x1\n" - ".inst 0xc1abc8ac // fclamp { z12.s-z15.s }, z5.s, z11.s\n" - "st1w { z12.s }, p1, [x9]\n" + "subs x11, x11, #0x1\n" + ".inst 0xc1b8c848 // fclamp { z8.s-z11.s }, z2.s, z24.s\n" + "st1w { z8.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - ".inst 0xc0040e02 // mova za.d[x8, #2], { z16.d-z19.d }\n" - "st1w { z13.s }, p1, [x28]\n" + ".inst 0xc0040e82 // mova za.d[x8, #2], { z20.d-z23.d }\n" + "st1w { z9.s }, p1, [x28]\n" "add x28, x28, x26, LSL #2\n" - "st1w { z14.s }, p1, [x25]\n" + "st1w { z10.s }, p1, [x25]\n" "add x25, x25, x23, LSL #2\n" - "st1w { z15.s }, p1, [x24]\n" + "st1w { z11.s }, p1, [x24]\n" "add x24, x24, x22, LSL #2\n" "bgt 17b\n" "18:" // End - "ldr x23, [%x[args], %[offsetof_Args_weights]]\n" - "incb x23, ALL, MUL #9\n" - "str x23, [%x[args], %[offsetof_Args_weights]]\n" - "incw x16\n" - "ldr x20, [%x[args], %[offsetof_Args_ld_in_vl]]\n" - "whilelt p1.s, x16, x17\n" - "ldr x14, [%x[args], %[offsetof_Args_inptr]]\n" - "add x14, x14, x20, LSL #2\n" - "str x14, [%x[args], %[offsetof_Args_inptr]]\n" - "ldr x11, [%x[args], %[offsetof_Args_outptrs]]\n" + "ldr x20, [%x[args], %[offsetof_Args_weights]]\n" + "incb x20, ALL, MUL #9\n" + "str x20, [%x[args], %[offsetof_Args_weights]]\n" + "incw x15\n" + "ldr x21, [%x[args], %[offsetof_Args_ld_in_vl]]\n" + "whilelt p1.s, x15, x16\n" + "ldr x20, [%x[args], %[offsetof_Args_inptr]]\n" + "add x20, x20, x21, LSL #2\n" + "str x20, [%x[args], %[offsetof_Args_inptr]]\n" + "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" "ldr x24, [%x[args], %[offsetof_Args_ld_out_vls]]\n" - "ldp x23, x22, [x11, #0x0]\n" + "ldp x23, x22, [x25, #0x0]\n" "ldp x21, x20, [x24, #0x0]\n" "add x23, x23, x21, LSL #2\n" "add x22, x22, x20, LSL #2\n" - "stp x23, x22, [x11, #0x0]\n" - "ldp x23, x22, [x11, #0x10]\n" + "stp x23, x22, [x25, #0x0]\n" + "ldp x23, x22, [x25, #0x10]\n" "ldp x21, x20, [x24, #0x10]\n" "add x23, x23, x21, LSL #2\n" "add x22, x22, x20, LSL #2\n" - "stp x23, x22, [x11, #0x10]\n" + "stp x23, x22, [x25, #0x10]\n" "b.any 1b\n" ".inst 0xd503467f // SMSTOP\n" : : [args] "r" (&args), [ld_in_col] "r" (ld_in_col), [ld_in_row] "r" (ld_in_row), [offsetof_Args_bias] "I" (offsetof(Args, bias)), [offsetof_Args_clamp_max] "I" (offsetof(Args, clamp_max)), [offsetof_Args_clamp_min] "I" (offsetof(Args, clamp_min)), [offsetof_Args_current_channel] "I" (offsetof(Args, current_channel)), [offsetof_Args_inptr] "I" (offsetof(Args, inptr)), [offsetof_Args_input_cols] "I" (offsetof(Args, input_cols)), [offsetof_Args_ld_in_vl] "I" (offsetof(Args, ld_in_vl)), [offsetof_Args_ld_out_cols] "I" (offsetof(Args, ld_out_cols)), [offsetof_Args_ld_out_vls] "I" (offsetof(Args, ld_out_vls)), [offsetof_Args_n_channels] "I" (offsetof(Args, n_channels)), [offsetof_Args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_Args_output_cols] "I" (offsetof(Args, output_cols)), [offsetof_Args_pad_bottom] "I" (offsetof(Args, pad_bottom)), [offsetof_Args_pad_left] "I" (offsetof(Args, pad_left)), [offsetof_Args_pad_top] "I" (offsetof(Args, pad_top)), [offsetof_Args_weights] "I" (offsetof(Args, weights)) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_3x3_s2_4rows_mla_za.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_3x3_s2_4rows_mla_za.hpp index 18a572954a5cdebc974a8a5f5bbf5f70b52e4aa5..711f7f479a08aca507dc03d67687e26d66f2fe07 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_3x3_s2_4rows_mla_za.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_3x3_s2_4rows_mla_za.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,8 @@ * SOFTWARE. */ +#include "src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp" + namespace arm_conv { namespace depthwise { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_3x3_s2_4rows_mla_za/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_3x3_s2_4rows_mla_za/generic.cpp index 9f6b09ef88831993b1e1f0e5c6604ca03d470e86..26315101b42d1e8dfb42008e7ad191e3f8961318 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_3x3_s2_4rows_mla_za/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_3x3_s2_4rows_mla_za/generic.cpp @@ -69,69 +69,69 @@ void sme2_fp32_planar_3x3_s2_4rows_mla_za_impl( Args args = { inptr, ld_in_vl, pad_top, 9u - std::min(9u, pad_top + valid_input_rows), pad_left, weights, bias, valid_input_cols, output_cols, outptrs, outlds, outvllds, start_channel, valid_channels, act_min, act_max }; __asm__ __volatile__( - "ldr x6, [%x[args], %[offsetof_Args_pad_bottom]]\n" + "ldr x7, [%x[args], %[offsetof_Args_pad_bottom]]\n" "mov x20, #0x9\n" ".inst 0xd503477f // SMSTART ZA\n" - "sub x20, x20, x6\n" - "ldr x7, [%x[args], %[offsetof_Args_pad_top]]\n" + "sub x20, x20, x7\n" + "ldr x17, [%x[args], %[offsetof_Args_pad_top]]\n" "ptrue p2.b\n" ".inst 0x25207812 // ptrue pn10.b\n" - "ld1rw { z28.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_min]]\n" - "ldr x17, [%x[args], %[offsetof_Args_n_channels]]\n" - "whilelt p1.s, XZR, x17\n" + "ld1rw { z7.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_min]]\n" + "ldr x16, [%x[args], %[offsetof_Args_n_channels]]\n" + "whilelt p1.s, XZR, x16\n" "whilelt p9.s, XZR, x20\n" - "ld1rw { z19.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_max]]\n" - "whilelt p8.s, XZR, x7\n" + "ld1rw { z9.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_max]]\n" + "whilelt p8.s, XZR, x17\n" "eor p8.b, p2/Z, p8.b, p9.b\n" - "ldr x16, [%x[args], %[offsetof_Args_current_channel]]\n" + "ldr x15, [%x[args], %[offsetof_Args_current_channel]]\n" "1:" // Channel loop "ldr x20, [%x[args], %[offsetof_Args_bias]]\n" - "fmov z24.s, #0x0\n" + "fmov z12.s, #0x0\n" "cbz x20, 2f\n" - "ld1w { z24.s }, p1/Z, [x20, x16, LSL #2]\n" + "ld1w { z12.s }, p1/Z, [x20, x15, LSL #2]\n" "2:" // Load bias: Done - "ldr x15, [%x[args], %[offsetof_Args_input_cols]]\n" - "sub x20, x15, #0x1\n" + "ldr x14, [%x[args], %[offsetof_Args_input_cols]]\n" + "sub x20, x14, #0x1\n" "orr x24, x20, %x[ld_in_col], LSL #18\n" - "mov z25.d, z24.d\n" + "mov z13.d, z12.d\n" "ldr x23, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xa0404ae2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x23]\n" - "orr x24, x17, x24, LSL #20\n" + ".inst 0xa1404ae2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x23]\n" + "orr x24, x16, x24, LSL #20\n" "mov x22, #0x9\n" - "ldr x14, [%x[args], %[offsetof_Args_inptr]]\n" - "ld1w { z7.s }, p2/Z, [x23, #2, MUL VL]\n" + "ldr x13, [%x[args], %[offsetof_Args_inptr]]\n" + "ld1w { z8.s }, p2/Z, [x23, #2, MUL VL]\n" "addvl x23, x23, #3\n" - "add x21, x7, x6\n" - ".inst 0xa0404ae4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x23]\n" + "add x21, x17, x7\n" + ".inst 0xa0404ae0 // ld1w { z0.s-z1.s }, pn10.b/Z, [x23]\n" "lsl x20, %x[ld_in_row], #0x2\n" - "mov z26.d, z24.d\n" - "mov z27.d, z24.d\n" - "ld1w { z6.s }, p2/Z, [x23, #2, MUL VL]\n" + "mov z14.d, z12.d\n" + "mov z15.d, z12.d\n" + "ld1w { z5.s }, p2/Z, [x23, #2, MUL VL]\n" "addvl x23, x23, #3\n" "mov x8, #0x0\n" - "ldr x13, [%x[args], %[offsetof_Args_output_cols]]\n" - ".inst 0xa1404ae1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x23]\n" + "ldr x11, [%x[args], %[offsetof_Args_output_cols]]\n" + ".inst 0xa1404ae3 // ld1w { z3.s, z11.s }, pn10.b/Z, [x23]\n" "lsl x24, x24, #0x2\n" "sub x22, x22, x21\n" - "ld1w { z8.s }, p2/Z, [x23, #2, MUL VL]\n" - "madd x20, x20, x7, x14\n" + "ld1w { z6.s }, p2/Z, [x23, #2, MUL VL]\n" + "madd x20, x20, x17, x13\n" "3:" // Issue prefetches "subs x22, x22, #0x1\n" ".inst 0xf8b84a9c // rprfm pldstrm, x24, [x20]\n" "add x20, x20, %x[ld_in_col], LSL #2\n" "bgt 3b\n" - "ldr x11, [%x[args], %[offsetof_Args_outptrs]]\n" + "ldr x23, [%x[args], %[offsetof_Args_outptrs]]\n" "lsl x20, %x[ld_in_row], #0x2\n" - "msub x14, x7, x20, x14\n" - ".inst 0xc0040f00 // mova za.d[x8, #0], { z24.d-z27.d }\n" + "msub x13, x17, x20, x13\n" + ".inst 0xc0040d80 // mova za.d[x8, #0], { z12.d-z15.d }\n" "ldr x20, [%x[args], %[offsetof_Args_ld_out_cols]]\n" - ".inst 0xc0040f01 // mova za.d[x8, #1], { z24.d-z27.d }\n" + ".inst 0xc0040d81 // mova za.d[x8, #1], { z12.d-z15.d }\n" "mov x22, #0x2\n" - "ldp x10, x9, [x11], #0x10\n" - ".inst 0xc0040f02 // mova za.d[x8, #2], { z24.d-z27.d }\n" + "ldp x10, x9, [x23], #0x10\n" + ".inst 0xc0040d82 // mova za.d[x8, #2], { z12.d-z15.d }\n" "ldp x28, x27, [x20], #0x10\n" "ldr x21, [%x[args], %[offsetof_Args_pad_left]]\n" - "ldp x26, x25, [x11], #0x10\n" + "ldp x26, x25, [x23], #0x10\n" "ldp x24, x23, [x20], #0x10\n" "cbz x21, 5f\n" "cmp x21, x22\n" @@ -142,9 +142,9 @@ void sme2_fp32_planar_3x3_s2_4rows_mla_za_impl( ".inst 0xc0060c14 // mova { z20.d-z23.d }, za.d[x8, #0]\n" "and x22, x21, #0x1\n" "add x21, x21, #0x1\n" - ".inst 0xc1b3cb94 // fclamp { z20.s-z23.s }, z28.s, z19.s\n" + ".inst 0xc1a9c8f4 // fclamp { z20.s-z23.s }, z7.s, z9.s\n" "lsr x21, x21, #0x1\n" - "sub x13, x13, x21\n" + "sub x11, x11, x21\n" "4:" // Left padding "subs x21, x21, #0x1\n" "st1w { z20.s }, p1, [x10]\n" @@ -157,490 +157,490 @@ void sme2_fp32_planar_3x3_s2_4rows_mla_za_impl( "add x25, x25, x23, LSL #2\n" "bgt 4b\n" "5:" // Left padding: End - "adds XZR, x7, x6\n" + "adds XZR, x17, x7\n" "bne 10f\n" "cbz x22, 8f\n" "cmp x22, #0x1\n" - "sub x15, x15, x22\n" + "sub x14, x14, x22\n" "beq 7f\n" "6:" // Unpadded: 2 priming loads - "add x20, x14, %x[ld_in_row], LSL #2\n" - "ld1w { z12.s }, p1/Z, [x14]\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - "ld1w { z29.s }, p1/Z, [x20]\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" + "ld1w { z19.s }, p1/Z, [x13]\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + "ld1w { z24.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z13.s }, p1/Z, [x20]\n" + "ld1w { z20.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z30.s }, p1/Z, [x20]\n" + "ld1w { z25.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z14.s }, p1/Z, [x20]\n" + "ld1w { z21.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z31.s }, p1/Z, [x20]\n" + "ld1w { z26.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z15.s }, p1/Z, [x20]\n" + "ld1w { z22.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1321980 // fmla za.s[x8, 0], { z12.s-z15.s }, z2.s\n" - "ld1w { z0.s }, p1/Z, [x20]\n" + ".inst 0xc1321a60 // fmla za.s[x8, 0], { z19.s-z22.s }, z2.s\n" + "ld1w { z27.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1341ba0 // fmla za.s[x8, 0], { z29.s-z0.s }, z4.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" - ".inst 0xc13119a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z1.s\n" + ".inst 0xc1301b00 // fmla za.s[x8, 0], { z24.s-z27.s }, z0.s\n" + "ld1w { z23.s }, p1/Z, [x20]\n" + ".inst 0xc1331a80 // fmla za.s[x8, 0], { z20.s-z23.s }, z3.s\n" "7:" // Unpadded: 1 priming loads - "add x20, x14, %x[ld_in_row], LSL #2\n" - "ld1w { z12.s }, p1/Z, [x14]\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - "ld1w { z29.s }, p1/Z, [x20]\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" + "ld1w { z26.s }, p1/Z, [x13]\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + "ld1w { z19.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z13.s }, p1/Z, [x20]\n" + "ld1w { z27.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z30.s }, p1/Z, [x20]\n" + "ld1w { z20.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z14.s }, p1/Z, [x20]\n" + "ld1w { z28.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z31.s }, p1/Z, [x20]\n" + "ld1w { z21.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z15.s }, p1/Z, [x20]\n" + "ld1w { z29.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1331980 // fmla za.s[x8, 0], { z12.s-z15.s }, z3.s\n" - "ld1w { z0.s }, p1/Z, [x20]\n" + ".inst 0xc13a1b40 // fmla za.s[x8, 0], { z26.s-z29.s }, z10.s\n" + "ld1w { z22.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1351ba0 // fmla za.s[x8, 0], { z29.s-z0.s }, z5.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" - ".inst 0xc13919a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z9.s\n" + ".inst 0xc1311a60 // fmla za.s[x8, 0], { z19.s-z22.s }, z1.s\n" + "ld1w { z30.s }, p1/Z, [x20]\n" + ".inst 0xc13b1b60 // fmla za.s[x8, 0], { z27.s-z30.s }, z11.s\n" "8:" // Unpadded: 0 priming loads - "cmp x15, #0x2\n" + "cmp x14, #0x2\n" "blt 16f\n" - "add x21, x14, %x[ld_in_row], LSL #2\n" - "ld1w { z12.s }, p1/Z, [x14]\n" - "sub x15, x15, #0x2\n" - "ld1w { z29.s }, p1/Z, [x21]\n" + "add x21, x13, %x[ld_in_row], LSL #2\n" + "ld1w { z25.s }, p1/Z, [x13]\n" + "sub x14, x14, #0x2\n" + "ld1w { z19.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - "sub x13, x13, #0x1\n" - "ld1w { z13.s }, p1/Z, [x21]\n" + "sub x11, x11, #0x1\n" + "ld1w { z26.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - "lsr x20, x15, #0x1\n" - "ld1w { z30.s }, p1/Z, [x21]\n" + "lsr x20, x14, #0x1\n" + "ld1w { z20.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - "cmp x20, x13\n" - "ld1w { z14.s }, p1/Z, [x21]\n" + "cmp x20, x11\n" + "ld1w { z27.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - "csel x22, x20, x13, LT\n" - "ld1w { z31.s }, p1/Z, [x21]\n" + "csel x22, x20, x11, LT\n" + "ld1w { z21.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - "ld1w { z15.s }, p1/Z, [x21]\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + "ld1w { z28.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - "and x15, x15, #0x1\n" - "ld1w { z0.s }, p1/Z, [x21]\n" + "and x14, x14, #0x1\n" + "ld1w { z22.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - "sub x13, x13, x22\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + "sub x11, x11, x22\n" + "ld1w { z29.s }, p1/Z, [x21]\n" "cbz x22, 15f\n" "9:" // Unpadded: Main loop - ".inst 0xc1371980 // fmla za.s[x8, 0], { z12.s-z15.s }, z7.s\n" - "add x21, x14, %x[ld_in_row], LSL #2\n" + ".inst 0xc1381b20 // fmla za.s[x8, 0], { z25.s-z28.s }, z8.s\n" + "add x21, x13, %x[ld_in_row], LSL #2\n" "subs x22, x22, #0x1\n" - ".inst 0xc1321981 // fmla za.s[x8, 1], { z12.s-z15.s }, z2.s\n" - "ld1w { z12.s }, p1/Z, [x14]\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" - ".inst 0xc1361ba0 // fmla za.s[x8, 0], { z29.s-z0.s }, z6.s\n" - ".inst 0xc1341ba1 // fmla za.s[x8, 1], { z29.s-z0.s }, z4.s\n" - "ld1w { z29.s }, p1/Z, [x21]\n" + ".inst 0xc1321b21 // fmla za.s[x8, 1], { z25.s-z28.s }, z2.s\n" + "ld1w { z25.s }, p1/Z, [x13]\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" + ".inst 0xc1351a60 // fmla za.s[x8, 0], { z19.s-z22.s }, z5.s\n" + ".inst 0xc1301a61 // fmla za.s[x8, 1], { z19.s-z22.s }, z0.s\n" + "ld1w { z18.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0xc13819a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z8.s\n" - ".inst 0xc13119a1 // fmla za.s[x8, 1], { z13.s-z16.s }, z1.s\n" - "ld1w { z13.s }, p1/Z, [x21]\n" + ".inst 0xc1361b40 // fmla za.s[x8, 0], { z26.s-z29.s }, z6.s\n" + ".inst 0xc1331b41 // fmla za.s[x8, 1], { z26.s-z29.s }, z3.s\n" + "ld1w { z26.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - "ld1w { z30.s }, p1/Z, [x21]\n" + "ld1w { z19.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0xc0060c14 // mova { z20.d-z23.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" - "ld1w { z14.s }, p1/Z, [x21]\n" + "ld1w { z27.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0xc1b3cb94 // fclamp { z20.s-z23.s }, z28.s, z19.s\n" + ".inst 0xc1a9c8f4 // fclamp { z20.s-z23.s }, z7.s, z9.s\n" "st1w { z20.s }, p1, [x10]\n" - "ld1w { z31.s }, p1/Z, [x21]\n" + "ld1w { z20.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" "add x10, x10, x28, LSL #2\n" "st1w { z21.s }, p1, [x9]\n" - "ld1w { z15.s }, p1/Z, [x21]\n" + "ld1w { z28.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0xc1331980 // fmla za.s[x8, 0], { z12.s-z15.s }, z3.s\n" + ".inst 0xc13a1b20 // fmla za.s[x8, 0], { z25.s-z28.s }, z10.s\n" "add x9, x9, x27, LSL #2\n" - "ld1w { z0.s }, p1/Z, [x21]\n" + "ld1w { z21.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0xc1351ba0 // fmla za.s[x8, 0], { z29.s-z0.s }, z5.s\n" + ".inst 0xc1311a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z1.s\n" "st1w { z22.s }, p1, [x26]\n" - "ld1w { z16.s }, p1/Z, [x21]\n" - ".inst 0xc13919a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z9.s\n" + "ld1w { z29.s }, p1/Z, [x21]\n" + ".inst 0xc13b1b40 // fmla za.s[x8, 0], { z26.s-z29.s }, z11.s\n" "add x26, x26, x24, LSL #2\n" "st1w { z23.s }, p1, [x25]\n" - "ld1w { z12.s }, p1/Z, [x14]\n" + "ld1w { z25.s }, p1/Z, [x13]\n" "add x25, x25, x23, LSL #2\n" - ".inst 0xc0040f02 // mova za.d[x8, #2], { z24.d-z27.d }\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - "ld1w { z29.s }, p1/Z, [x20]\n" + ".inst 0xc0040d82 // mova za.d[x8, #2], { z12.d-z15.d }\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + "ld1w { z19.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z13.s }, p1/Z, [x20]\n" + "ld1w { z26.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z30.s }, p1/Z, [x20]\n" + "ld1w { z20.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z14.s }, p1/Z, [x20]\n" + "ld1w { z27.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z31.s }, p1/Z, [x20]\n" + "ld1w { z21.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z15.s }, p1/Z, [x20]\n" + "ld1w { z28.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z0.s }, p1/Z, [x20]\n" + "ld1w { z22.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + "ld1w { z29.s }, p1/Z, [x20]\n" "bgt 9b\n" "b 15f\n" "10:" // Padded "cbz x22, 13f\n" "cmp x22, #0x1\n" - "sub x15, x15, x22\n" + "sub x14, x14, x22\n" "beq 12f\n" "11:" // Padded: 2 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z12.s }, p0/Z, [x14]\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + "ld1w { z27.s }, p0/Z, [x13]\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z29.s }, p0/Z, [x20]\n" + "ld1w { z23.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z13.s }, p0/Z, [x20]\n" + "ld1w { z28.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z30.s }, p0/Z, [x20]\n" + "ld1w { z24.s }, p0/Z, [x20]\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z14.s }, p0/Z, [x20]\n" + "ld1w { z29.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z31.s }, p0/Z, [x20]\n" + "ld1w { z25.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z15.s }, p0/Z, [x20]\n" + "ld1w { z30.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x8\n" - ".inst 0xc1321980 // fmla za.s[x8, 0], { z12.s-z15.s }, z2.s\n" - "ld1w { z0.s }, p0/Z, [x20]\n" + ".inst 0xc1321b60 // fmla za.s[x8, 0], { z27.s-z30.s }, z2.s\n" + "ld1w { z26.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1341ba0 // fmla za.s[x8, 0], { z29.s-z0.s }, z4.s\n" - "ld1w { z16.s }, p0/Z, [x20]\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xc13119a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z1.s\n" + ".inst 0xc1301ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z0.s\n" + "ld1w { z31.s }, p0/Z, [x20]\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + ".inst 0xc1331b80 // fmla za.s[x8, 0], { z28.s-z31.s }, z3.s\n" "12:" // Padded: 1 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z12.s }, p0/Z, [x14]\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + "ld1w { z22.s }, p0/Z, [x13]\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z29.s }, p0/Z, [x20]\n" + "ld1w { z27.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z13.s }, p0/Z, [x20]\n" + "ld1w { z23.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z30.s }, p0/Z, [x20]\n" + "ld1w { z28.s }, p0/Z, [x20]\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z14.s }, p0/Z, [x20]\n" + "ld1w { z24.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z31.s }, p0/Z, [x20]\n" + "ld1w { z29.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z15.s }, p0/Z, [x20]\n" + "ld1w { z25.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x8\n" - ".inst 0xc1331980 // fmla za.s[x8, 0], { z12.s-z15.s }, z3.s\n" - "ld1w { z0.s }, p0/Z, [x20]\n" + ".inst 0xc13a1ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z10.s\n" + "ld1w { z30.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1351ba0 // fmla za.s[x8, 0], { z29.s-z0.s }, z5.s\n" - "ld1w { z16.s }, p0/Z, [x20]\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xc13919a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z9.s\n" + ".inst 0xc1311b60 // fmla za.s[x8, 0], { z27.s-z30.s }, z1.s\n" + "ld1w { z26.s }, p0/Z, [x20]\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + ".inst 0xc13b1ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z11.s\n" "13:" // Padded: 0 priming loads - "cmp x15, #0x2\n" + "cmp x14, #0x2\n" "blt 16f\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z12.s }, p0/Z, [x14]\n" - "add x21, x14, %x[ld_in_row], LSL #2\n" + "ld1w { z25.s }, p0/Z, [x13]\n" + "add x21, x13, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z29.s }, p0/Z, [x21]\n" + "ld1w { z19.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z13.s }, p0/Z, [x21]\n" + "ld1w { z26.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z30.s }, p0/Z, [x21]\n" + "ld1w { z20.s }, p0/Z, [x21]\n" "mov x12, #0x4\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z14.s }, p0/Z, [x21]\n" + "ld1w { z27.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "sub x15, x15, #0x2\n" - "ld1w { z31.s }, p0/Z, [x21]\n" + "sub x14, x14, #0x2\n" + "ld1w { z21.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z15.s }, p0/Z, [x21]\n" - "sub x13, x13, #0x1\n" - "lsr x20, x15, #0x1\n" + "ld1w { z28.s }, p0/Z, [x21]\n" + "sub x11, x11, #0x1\n" + "lsr x20, x14, #0x1\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z0.s }, p0/Z, [x21]\n" + "ld1w { z22.s }, p0/Z, [x21]\n" "mov x12, #0x8\n" - "cmp x20, x13\n" + "cmp x20, x11\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x21]\n" - "csel x22, x20, x13, LT\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - "and x15, x15, #0x1\n" - "sub x13, x13, x22\n" + "ld1w { z29.s }, p0/Z, [x21]\n" + "csel x22, x20, x11, LT\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + "and x14, x14, #0x1\n" + "sub x11, x11, x22\n" "cbz x22, 15f\n" "14:" // Padded: Main loop - ".inst 0xc1371980 // fmla za.s[x8, 0], { z12.s-z15.s }, z7.s\n" + ".inst 0xc1381b20 // fmla za.s[x8, 0], { z25.s-z28.s }, z8.s\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1321981 // fmla za.s[x8, 1], { z12.s-z15.s }, z2.s\n" - "ld1w { z12.s }, p0/Z, [x14]\n" - "add x21, x14, %x[ld_in_row], LSL #2\n" + ".inst 0xc1321b21 // fmla za.s[x8, 1], { z25.s-z28.s }, z2.s\n" + "ld1w { z18.s }, p0/Z, [x13]\n" + "add x21, x13, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0xc1361ba0 // fmla za.s[x8, 0], { z29.s-z0.s }, z6.s\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" - ".inst 0xc1341ba1 // fmla za.s[x8, 1], { z29.s-z0.s }, z4.s\n" - "ld1w { z29.s }, p0/Z, [x21]\n" + ".inst 0xc1351a60 // fmla za.s[x8, 0], { z19.s-z22.s }, z5.s\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" + ".inst 0xc1301a61 // fmla za.s[x8, 1], { z19.s-z22.s }, z0.s\n" + "ld1w { z25.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc13819a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z8.s\n" + ".inst 0xc1361b40 // fmla za.s[x8, 0], { z26.s-z29.s }, z6.s\n" "subs x22, x22, #0x1\n" - ".inst 0xc13119a1 // fmla za.s[x8, 1], { z13.s-z16.s }, z1.s\n" - "ld1w { z13.s }, p0/Z, [x21]\n" + ".inst 0xc1331b41 // fmla za.s[x8, 1], { z26.s-z29.s }, z3.s\n" + "ld1w { z19.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x4\n" - "ld1w { z30.s }, p0/Z, [x21]\n" + "ld1w { z26.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0xc0060c14 // mova { z20.d-z23.d }, za.d[x8, #0]\n" + ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z14.s }, p0/Z, [x21]\n" + "ld1w { z20.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0xc1b3cb94 // fclamp { z20.s-z23.s }, z28.s, z19.s\n" + ".inst 0xc1a9c8fc // fclamp { z28.s-z31.s }, z7.s, z9.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z31.s }, p0/Z, [x21]\n" + "ld1w { z27.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - "st1w { z20.s }, p1, [x10]\n" + "st1w { z28.s }, p1, [x10]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z15.s }, p0/Z, [x21]\n" + "ld1w { z21.s }, p0/Z, [x21]\n" "add x8, x8, #0x1\n" - "st1w { z21.s }, p1, [x9]\n" + "st1w { z29.s }, p1, [x9]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z0.s }, p0/Z, [x21]\n" - "st1w { z22.s }, p1, [x26]\n" + "ld1w { z28.s }, p0/Z, [x21]\n" + "st1w { z30.s }, p1, [x26]\n" "mov x12, #0x8\n" - ".inst 0xc1331980 // fmla za.s[x8, 0], { z12.s-z15.s }, z3.s\n" + ".inst 0xc13a1a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z10.s\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - "st1w { z23.s }, p1, [x25]\n" + "st1w { z31.s }, p1, [x25]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1351ba0 // fmla za.s[x8, 0], { z29.s-z0.s }, z5.s\n" + ".inst 0xc1311b20 // fmla za.s[x8, 0], { z25.s-z28.s }, z1.s\n" "mov x12, #0x0\n" - "ld1w { z16.s }, p0/Z, [x21]\n" + "ld1w { z22.s }, p0/Z, [x21]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z12.s }, p0/Z, [x14]\n" + "ld1w { z25.s }, p0/Z, [x13]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0xc13919a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z9.s\n" - "ld1w { z29.s }, p0/Z, [x20]\n" + ".inst 0xc13b1a60 // fmla za.s[x8, 0], { z19.s-z22.s }, z11.s\n" + "ld1w { z19.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc0040f02 // mova za.d[x8, #2], { z24.d-z27.d }\n" - "ld1w { z13.s }, p0/Z, [x20]\n" + ".inst 0xc0040d82 // mova za.d[x8, #2], { z12.d-z15.d }\n" + "ld1w { z26.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x4\n" - "ld1w { z30.s }, p0/Z, [x20]\n" + "ld1w { z20.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z14.s }, p0/Z, [x20]\n" + "ld1w { z27.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z31.s }, p0/Z, [x20]\n" + "ld1w { z21.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z15.s }, p0/Z, [x20]\n" + "ld1w { z28.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x8\n" - "ld1w { z0.s }, p0/Z, [x20]\n" + "ld1w { z22.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z29.s }, p0/Z, [x20]\n" "add x10, x10, x28, LSL #2\n" "add x9, x9, x27, LSL #2\n" "add x26, x26, x24, LSL #2\n" "add x25, x25, x23, LSL #2\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" "bgt 14b\n" "15:" // Main loop tail - ".inst 0xc1371980 // fmla za.s[x8, 0], { z12.s-z15.s }, z7.s\n" + ".inst 0xc1381b20 // fmla za.s[x8, 0], { z25.s-z28.s }, z8.s\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1321981 // fmla za.s[x8, 1], { z12.s-z15.s }, z2.s\n" - "ld1w { z12.s }, p0/Z, [x14]\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + ".inst 0xc1321b21 // fmla za.s[x8, 1], { z25.s-z28.s }, z2.s\n" + "ld1w { z18.s }, p0/Z, [x13]\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0xc1361ba0 // fmla za.s[x8, 0], { z29.s-z0.s }, z6.s\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xc1341ba1 // fmla za.s[x8, 1], { z29.s-z0.s }, z4.s\n" - "ld1w { z29.s }, p0/Z, [x20]\n" + ".inst 0xc1351a60 // fmla za.s[x8, 0], { z19.s-z22.s }, z5.s\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + ".inst 0xc1301a61 // fmla za.s[x8, 1], { z19.s-z22.s }, z0.s\n" + "ld1w { z22.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc13819a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z8.s\n" - ".inst 0xc13119a1 // fmla za.s[x8, 1], { z13.s-z16.s }, z1.s\n" - "ld1w { z13.s }, p0/Z, [x20]\n" + ".inst 0xc1361b40 // fmla za.s[x8, 0], { z26.s-z29.s }, z6.s\n" + ".inst 0xc1331b41 // fmla za.s[x8, 1], { z26.s-z29.s }, z3.s\n" + "ld1w { z19.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x4\n" - "ld1w { z30.s }, p0/Z, [x20]\n" + "ld1w { z23.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc0060c14 // mova { z20.d-z23.d }, za.d[x8, #0]\n" + ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z14.s }, p0/Z, [x20]\n" + "ld1w { z20.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1b3cb94 // fclamp { z20.s-z23.s }, z28.s, z19.s\n" + ".inst 0xc1a9c8fc // fclamp { z28.s-z31.s }, z7.s, z9.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z31.s }, p0/Z, [x20]\n" + "ld1w { z24.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "st1w { z20.s }, p1, [x10]\n" + "st1w { z28.s }, p1, [x10]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z15.s }, p0/Z, [x20]\n" + "ld1w { z21.s }, p0/Z, [x20]\n" "add x8, x8, #0x1\n" - "st1w { z21.s }, p1, [x9]\n" + "st1w { z29.s }, p1, [x9]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z0.s }, p0/Z, [x20]\n" - "st1w { z22.s }, p1, [x26]\n" + "ld1w { z25.s }, p0/Z, [x20]\n" + "st1w { z30.s }, p1, [x26]\n" "mov x12, #0x8\n" - ".inst 0xc1331980 // fmla za.s[x8, 0], { z12.s-z15.s }, z3.s\n" + ".inst 0xc13a1a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z10.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "st1w { z23.s }, p1, [x25]\n" + "st1w { z31.s }, p1, [x25]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1351ba0 // fmla za.s[x8, 0], { z29.s-z0.s }, z5.s\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + ".inst 0xc1311ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z1.s\n" + "ld1w { z22.s }, p0/Z, [x20]\n" "add x10, x10, x28, LSL #2\n" "add x9, x9, x27, LSL #2\n" "add x26, x26, x24, LSL #2\n" - ".inst 0xc0040f02 // mova za.d[x8, #2], { z24.d-z27.d }\n" + ".inst 0xc0040d82 // mova za.d[x8, #2], { z12.d-z15.d }\n" "add x25, x25, x23, LSL #2\n" - ".inst 0xc13919a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z9.s\n" + ".inst 0xc13b1a60 // fmla za.s[x8, 0], { z19.s-z22.s }, z11.s\n" "16:" // Main loop skip tail - "cbz x15, 17f\n" // Skip remainder inputs + "cbz x14, 17f\n" // Skip remainder inputs "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z12.s }, p0/Z, [x14]\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + "ld1w { z21.s }, p0/Z, [x13]\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z29.s }, p0/Z, [x20]\n" + "ld1w { z28.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z13.s }, p0/Z, [x20]\n" + "ld1w { z22.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z30.s }, p0/Z, [x20]\n" + "ld1w { z29.s }, p0/Z, [x20]\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z14.s }, p0/Z, [x20]\n" + "ld1w { z23.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z31.s }, p0/Z, [x20]\n" + "ld1w { z30.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z15.s }, p0/Z, [x20]\n" + "ld1w { z24.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x8\n" - ".inst 0xc1371980 // fmla za.s[x8, 0], { z12.s-z15.s }, z7.s\n" - "ld1w { z0.s }, p0/Z, [x20]\n" + ".inst 0xc1381aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z8.s\n" + "ld1w { z31.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1361ba0 // fmla za.s[x8, 0], { z29.s-z0.s }, z6.s\n" - "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0xc1321981 // fmla za.s[x8, 1], { z12.s-z15.s }, z2.s\n" - "sub x13, x13, #0x1\n" - ".inst 0xc13819a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z8.s\n" - ".inst 0xc1341ba1 // fmla za.s[x8, 1], { z29.s-z0.s }, z4.s\n" - ".inst 0xc0060c14 // mova { z20.d-z23.d }, za.d[x8, #0]\n" - ".inst 0xc1b3cb94 // fclamp { z20.s-z23.s }, z28.s, z19.s\n" - "st1w { z20.s }, p1, [x10]\n" + ".inst 0xc1351b80 // fmla za.s[x8, 0], { z28.s-z31.s }, z5.s\n" + "ld1w { z25.s }, p0/Z, [x20]\n" + ".inst 0xc1321aa1 // fmla za.s[x8, 1], { z21.s-z24.s }, z2.s\n" + "sub x11, x11, #0x1\n" + ".inst 0xc1361ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z6.s\n" + ".inst 0xc1301b81 // fmla za.s[x8, 1], { z28.s-z31.s }, z0.s\n" + ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" + ".inst 0xc1a9c8f0 // fclamp { z16.s-z19.s }, z7.s, z9.s\n" + "st1w { z16.s }, p1, [x10]\n" "add x10, x10, x28, LSL #2\n" - ".inst 0xc13119a1 // fmla za.s[x8, 1], { z13.s-z16.s }, z1.s\n" + ".inst 0xc1331ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z3.s\n" "add x8, x8, #0x1\n" - "st1w { z21.s }, p1, [x9]\n" + "st1w { z17.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - "st1w { z22.s }, p1, [x26]\n" + "st1w { z18.s }, p1, [x26]\n" "add x26, x26, x24, LSL #2\n" - ".inst 0xc0040f02 // mova za.d[x8, #2], { z24.d-z27.d }\n" - "st1w { z23.s }, p1, [x25]\n" + ".inst 0xc0040d82 // mova za.d[x8, #2], { z12.d-z15.d }\n" + "st1w { z19.s }, p1, [x25]\n" "add x25, x25, x23, LSL #2\n" "17:" // Tail input: End - "cbz x13, 19f\n" + "cbz x11, 19f\n" "18:" // Right padding loop - ".inst 0xc0060c14 // mova { z20.d-z23.d }, za.d[x8, #0]\n" + ".inst 0xc0060c00 // mova { z0.d-z3.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" - "subs x13, x13, #0x1\n" - ".inst 0xc1b3cb94 // fclamp { z20.s-z23.s }, z28.s, z19.s\n" - "st1w { z20.s }, p1, [x10]\n" + "subs x11, x11, #0x1\n" + ".inst 0xc1a9c8e0 // fclamp { z0.s-z3.s }, z7.s, z9.s\n" + "st1w { z0.s }, p1, [x10]\n" "add x10, x10, x28, LSL #2\n" - ".inst 0xc0040f02 // mova za.d[x8, #2], { z24.d-z27.d }\n" - "st1w { z21.s }, p1, [x9]\n" + ".inst 0xc0040d82 // mova za.d[x8, #2], { z12.d-z15.d }\n" + "st1w { z1.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - "st1w { z22.s }, p1, [x26]\n" + "st1w { z2.s }, p1, [x26]\n" "add x26, x26, x24, LSL #2\n" - "st1w { z23.s }, p1, [x25]\n" + "st1w { z3.s }, p1, [x25]\n" "add x25, x25, x23, LSL #2\n" "bgt 18b\n" "19:" // End - "ldr x23, [%x[args], %[offsetof_Args_weights]]\n" - "incb x23, ALL, MUL #9\n" - "str x23, [%x[args], %[offsetof_Args_weights]]\n" - "incw x16\n" - "ldr x20, [%x[args], %[offsetof_Args_ld_in_vl]]\n" - "whilelt p1.s, x16, x17\n" - "ldr x14, [%x[args], %[offsetof_Args_inptr]]\n" - "add x14, x14, x20, LSL #2\n" - "str x14, [%x[args], %[offsetof_Args_inptr]]\n" - "ldr x11, [%x[args], %[offsetof_Args_outptrs]]\n" + "ldr x20, [%x[args], %[offsetof_Args_weights]]\n" + "incb x20, ALL, MUL #9\n" + "str x20, [%x[args], %[offsetof_Args_weights]]\n" + "incw x15\n" + "ldr x21, [%x[args], %[offsetof_Args_ld_in_vl]]\n" + "whilelt p1.s, x15, x16\n" + "ldr x20, [%x[args], %[offsetof_Args_inptr]]\n" + "add x20, x20, x21, LSL #2\n" + "str x20, [%x[args], %[offsetof_Args_inptr]]\n" + "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" "ldr x24, [%x[args], %[offsetof_Args_ld_out_vls]]\n" - "ldp x23, x22, [x11, #0x0]\n" + "ldp x23, x22, [x25, #0x0]\n" "ldp x21, x20, [x24, #0x0]\n" "add x23, x23, x21, LSL #2\n" "add x22, x22, x20, LSL #2\n" - "stp x23, x22, [x11, #0x0]\n" - "ldp x23, x22, [x11, #0x10]\n" + "stp x23, x22, [x25, #0x0]\n" + "ldp x23, x22, [x25, #0x10]\n" "ldp x21, x20, [x24, #0x10]\n" "add x23, x23, x21, LSL #2\n" "add x22, x22, x20, LSL #2\n" - "stp x23, x22, [x11, #0x10]\n" + "stp x23, x22, [x25, #0x10]\n" "b.any 1b\n" ".inst 0xd503467f // SMSTOP\n" : : [args] "r" (&args), [ld_in_col] "r" (ld_in_col), [ld_in_row] "r" (ld_in_row), [offsetof_Args_bias] "I" (offsetof(Args, bias)), [offsetof_Args_clamp_max] "I" (offsetof(Args, clamp_max)), [offsetof_Args_clamp_min] "I" (offsetof(Args, clamp_min)), [offsetof_Args_current_channel] "I" (offsetof(Args, current_channel)), [offsetof_Args_inptr] "I" (offsetof(Args, inptr)), [offsetof_Args_input_cols] "I" (offsetof(Args, input_cols)), [offsetof_Args_ld_in_vl] "I" (offsetof(Args, ld_in_vl)), [offsetof_Args_ld_out_cols] "I" (offsetof(Args, ld_out_cols)), [offsetof_Args_ld_out_vls] "I" (offsetof(Args, ld_out_vls)), [offsetof_Args_n_channels] "I" (offsetof(Args, n_channels)), [offsetof_Args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_Args_output_cols] "I" (offsetof(Args, output_cols)), [offsetof_Args_pad_bottom] "I" (offsetof(Args, pad_bottom)), [offsetof_Args_pad_left] "I" (offsetof(Args, pad_left)), [offsetof_Args_pad_top] "I" (offsetof(Args, pad_top)), [offsetof_Args_weights] "I" (offsetof(Args, weights)) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_5x5_s1_4rows_mla_za.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_5x5_s1_4rows_mla_za.hpp index 0fa0300f9f355cdcf4df03645852f61bf8a78d7a..71487e08b6182a49ece6d430d8b82907b8044544 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_5x5_s1_4rows_mla_za.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_5x5_s1_4rows_mla_za.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,8 @@ * SOFTWARE. */ +#include "src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp" + namespace arm_conv { namespace depthwise { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_5x5_s1_4rows_mla_za/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_5x5_s1_4rows_mla_za/generic.cpp index bf12b42ddcaed2c529bca4bf84cb11616c14a27f..3741b973b4001659ee094f1b78dc2e9bd0683169 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_5x5_s1_4rows_mla_za/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_5x5_s1_4rows_mla_za/generic.cpp @@ -69,71 +69,71 @@ void sme2_fp32_planar_5x5_s1_4rows_mla_za_impl( Args args = { inptr, ld_in_vl, pad_top, 8u - std::min(8u, pad_top + valid_input_rows), pad_left, weights, bias, valid_input_cols, output_cols, outptrs, outlds, outvllds, start_channel, valid_channels, act_min, act_max }; __asm__ __volatile__( - "ldr x5, [%x[args], %[offsetof_Args_pad_bottom]]\n" + "ldr x6, [%x[args], %[offsetof_Args_pad_bottom]]\n" "mov x20, #0x8\n" ".inst 0xd503477f // SMSTART ZA\n" - "sub x20, x20, x5\n" - "ldr x6, [%x[args], %[offsetof_Args_pad_top]]\n" + "sub x20, x20, x6\n" + "ldr x7, [%x[args], %[offsetof_Args_pad_top]]\n" "ptrue p2.b\n" ".inst 0x25207812 // ptrue pn10.b\n" - "ld1rw { z22.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_min]]\n" - "ldr x7, [%x[args], %[offsetof_Args_n_channels]]\n" - "whilelt p1.s, XZR, x7\n" + "ld1rw { z16.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_min]]\n" + "ldr x17, [%x[args], %[offsetof_Args_n_channels]]\n" + "whilelt p1.s, XZR, x17\n" "whilelt p9.s, XZR, x20\n" - "ld1rw { z11.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_max]]\n" - "whilelt p8.s, XZR, x6\n" + "ld1rw { z17.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_max]]\n" + "whilelt p8.s, XZR, x7\n" "eor p8.b, p2/Z, p8.b, p9.b\n" - "ldr x17, [%x[args], %[offsetof_Args_current_channel]]\n" + "ldr x16, [%x[args], %[offsetof_Args_current_channel]]\n" "1:" // Channel loop "ldr x20, [%x[args], %[offsetof_Args_bias]]\n" "fmov z28.s, #0x0\n" "cbz x20, 2f\n" - "ld1w { z28.s }, p1/Z, [x20, x17, LSL #2]\n" + "ld1w { z28.s }, p1/Z, [x20, x16, LSL #2]\n" "2:" // Load bias: Done - "ldr x16, [%x[args], %[offsetof_Args_input_cols]]\n" - "sub x20, x16, #0x1\n" + "ldr x15, [%x[args], %[offsetof_Args_input_cols]]\n" + "sub x20, x15, #0x1\n" "orr x23, x20, %x[ld_in_col], LSL #18\n" "mov z29.d, z28.d\n" - "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "orr x23, x7, x23, LSL #20\n" + "ldr x14, [%x[args], %[offsetof_Args_weights]]\n" + ".inst 0xa04049ce // ld1w { z14.s-z15.s }, pn10.b/Z, [x14]\n" + "orr x23, x17, x23, LSL #20\n" "mov x22, #0x8\n" - "ldr x14, [%x[args], %[offsetof_Args_inptr]]\n" - ".inst 0xa04149e2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "add x21, x6, x5\n" + "ldr x13, [%x[args], %[offsetof_Args_inptr]]\n" + ".inst 0xa04149cc // ld1w { z12.s-z13.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + "add x21, x7, x6\n" "lsl x20, %x[ld_in_row], #0x2\n" - "ld1w { z10.s }, p2/Z, [x15, #4, MUL VL]\n" - "addvl x15, x15, #5\n" + "ld1w { z2.s }, p2/Z, [x14, #4, MUL VL]\n" + "addvl x14, x14, #5\n" "mov z30.d, z28.d\n" "mov z31.d, z28.d\n" - ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" + ".inst 0xa04049ca // ld1w { z10.s-z11.s }, pn10.b/Z, [x14]\n" "mov x8, #0x0\n" - "ldr x13, [%x[args], %[offsetof_Args_output_cols]]\n" + "ldr x11, [%x[args], %[offsetof_Args_output_cols]]\n" "lsl x23, x23, #0x2\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xa04149c8 // ld1w { z8.s-z9.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" "sub x22, x22, x21\n" - "madd x20, x20, x6, x14\n" - "ld1w { z1.s }, p2/Z, [x15, #4, MUL VL]\n" - "addvl x15, x15, #5\n" + "madd x20, x20, x7, x13\n" + "ld1w { z3.s }, p2/Z, [x14, #4, MUL VL]\n" + "addvl x14, x14, #5\n" "3:" // Issue prefetches "subs x22, x22, #0x1\n" ".inst 0xf8b74a9c // rprfm pldstrm, x23, [x20]\n" "add x20, x20, %x[ld_in_col], LSL #2\n" "bgt 3b\n" - "ldr x11, [%x[args], %[offsetof_Args_outptrs]]\n" + "ldr x22, [%x[args], %[offsetof_Args_outptrs]]\n" "lsl x20, %x[ld_in_row], #0x2\n" - "msub x14, x6, x20, x14\n" + "msub x13, x7, x20, x13\n" ".inst 0xc0040f80 // mova za.d[x8, #0], { z28.d-z31.d }\n" "ldr x20, [%x[args], %[offsetof_Args_ld_out_cols]]\n" ".inst 0xc0040f81 // mova za.d[x8, #1], { z28.d-z31.d }\n" "mov x10, #0x4\n" - "ldp x9, x28, [x11], #0x10\n" + "ldp x9, x28, [x22], #0x10\n" ".inst 0xc0040f82 // mova za.d[x8, #2], { z28.d-z31.d }\n" "ldp x27, x26, [x20], #0x10\n" ".inst 0xc0040f83 // mova za.d[x8, #3], { z28.d-z31.d }\n" "ldr x21, [%x[args], %[offsetof_Args_pad_left]]\n" ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" - "ldp x25, x24, [x11], #0x10\n" + "ldp x25, x24, [x22], #0x10\n" "ldp x23, x22, [x20], #0x10\n" "cbz x21, 5f\n" "cmp x21, x10\n" @@ -141,308 +141,308 @@ void sme2_fp32_planar_5x5_s1_4rows_mla_za_impl( "sub x21, x21, x20\n" "sub x10, x10, x20\n" "cbz x21, 5f\n" - ".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n" - "sub x13, x13, x21\n" - ".inst 0xc1abcad8 // fclamp { z24.s-z27.s }, z22.s, z11.s\n" + ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" + "sub x11, x11, x21\n" + ".inst 0xc1b1ca04 // fclamp { z4.s-z7.s }, z16.s, z17.s\n" "4:" // Left padding "subs x21, x21, #0x1\n" - "st1w { z24.s }, p1, [x9]\n" + "st1w { z4.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - "st1w { z25.s }, p1, [x28]\n" + "st1w { z5.s }, p1, [x28]\n" "add x28, x28, x26, LSL #2\n" - "st1w { z26.s }, p1, [x25]\n" + "st1w { z6.s }, p1, [x25]\n" "add x25, x25, x23, LSL #2\n" - "st1w { z27.s }, p1, [x24]\n" + "st1w { z7.s }, p1, [x24]\n" "add x24, x24, x22, LSL #2\n" "bgt 4b\n" "5:" // Left padding: End - "adds XZR, x6, x5\n" + "adds XZR, x7, x6\n" "bne 12f\n" "cbz x10, 10f\n" "cmp x10, #0x1\n" - "sub x16, x16, x10\n" + "sub x15, x15, x10\n" "beq 9f\n" "cmp x10, #0x2\n" "beq 8f\n" "cmp x10, #0x3\n" "beq 7f\n" "6:" // Unpadded: 4 priming loads - "add x20, x14, %x[ld_in_row], LSL #2\n" - "ld1w { z14.s }, p1/Z, [x14]\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - "ld1w { z15.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z17.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc13419c0 // fmla za.s[x8, 0], { z14.s-z17.s }, z4.s\n" - "ld1w { z18.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc13019e0 // fmla za.s[x8, 0], { z15.s-z18.s }, z0.s\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" + "ld1w { z18.s }, p1/Z, [x13]\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" "ld1w { z19.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - ".inst 0xc1341a00 // fmla za.s[x8, 0], { z16.s-z19.s }, z4.s\n" "ld1w { z20.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - ".inst 0xc1301a20 // fmla za.s[x8, 0], { z17.s-z20.s }, z0.s\n" "ld1w { z21.s }, p1/Z, [x20]\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xc1341a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - "7:" // Unpadded: 3 priming loads - "add x20, x14, %x[ld_in_row], LSL #2\n" - "ld1w { z14.s }, p1/Z, [x14]\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - "ld1w { z15.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z17.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc13519c0 // fmla za.s[x8, 0], { z14.s-z17.s }, z5.s\n" - ".inst 0xc13419c1 // fmla za.s[x8, 1], { z14.s-z17.s }, z4.s\n" - "ld1w { z18.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc13819e0 // fmla za.s[x8, 0], { z15.s-z18.s }, z8.s\n" - "ld1w { z19.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc13019e1 // fmla za.s[x8, 1], { z15.s-z18.s }, z0.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - ".inst 0xc1351a00 // fmla za.s[x8, 0], { z16.s-z19.s }, z5.s\n" - "ld1w { z20.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1341a01 // fmla za.s[x8, 1], { z16.s-z19.s }, z4.s\n" - ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - ".inst 0xc1381a20 // fmla za.s[x8, 0], { z17.s-z20.s }, z8.s\n" + ".inst 0xc13e1a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z14.s\n" + "ld1w { z22.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xc13a1a60 // fmla za.s[x8, 0], { z19.s-z22.s }, z10.s\n" + "ld1w { z23.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xa14049c5 // ld1w { z5.s, z13.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" + ".inst 0xc1351a80 // fmla za.s[x8, 0], { z20.s-z23.s }, z5.s\n" + "ld1w { z24.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xa04049cc // ld1w { z12.s-z13.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" + ".inst 0xc13c1aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z12.s\n" + "ld1w { z25.s }, p1/Z, [x20]\n" + ".inst 0xa14049c1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x14]\n" + "ldr x14, [%x[args], %[offsetof_Args_weights]]\n" + ".inst 0xc1311ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z1.s\n" + ".inst 0xa04049ce // ld1w { z14.s-z15.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" + ".inst 0xa04049ca // ld1w { z10.s-z11.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" + "7:" // Unpadded: 3 priming loads + "add x20, x13, %x[ld_in_row], LSL #2\n" + "ld1w { z20.s }, p1/Z, [x13]\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" "ld1w { z21.s }, p1/Z, [x20]\n" - ".inst 0xc1301a21 // fmla za.s[x8, 1], { z17.s-z20.s }, z0.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xc1351a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z5.s\n" - ".inst 0xa04149e2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc1341a41 // fmla za.s[x8, 1], { z18.s-z21.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "addvl x15, x15, #5\n" - "8:" // Unpadded: 2 priming loads - "add x20, x14, %x[ld_in_row], LSL #2\n" - "ld1w { z14.s }, p1/Z, [x14]\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - "ld1w { z15.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z17.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc13219c0 // fmla za.s[x8, 0], { z14.s-z17.s }, z2.s\n" - ".inst 0xc13519c1 // fmla za.s[x8, 1], { z14.s-z17.s }, z5.s\n" - "ld1w { z18.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc13419c2 // fmla za.s[x8, 2], { z14.s-z17.s }, z4.s\n" - "ld1w { z19.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc13619e0 // fmla za.s[x8, 0], { z15.s-z18.s }, z6.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xc13819e1 // fmla za.s[x8, 1], { z15.s-z18.s }, z8.s\n" - ".inst 0xa04149e2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "addvl x15, x15, #5\n" - ".inst 0xc13019e2 // fmla za.s[x8, 2], { z15.s-z18.s }, z0.s\n" - "ld1w { z20.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1321a00 // fmla za.s[x8, 0], { z16.s-z19.s }, z2.s\n" - ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" - ".inst 0xc1351a01 // fmla za.s[x8, 1], { z16.s-z19.s }, z5.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "addvl x15, x15, #5\n" - ".inst 0xc1341a02 // fmla za.s[x8, 2], { z16.s-z19.s }, z4.s\n" - "ld1w { z21.s }, p1/Z, [x20]\n" - ".inst 0xc1361a20 // fmla za.s[x8, 0], { z17.s-z20.s }, z6.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xc1381a21 // fmla za.s[x8, 1], { z17.s-z20.s }, z8.s\n" - ".inst 0xa04149e2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xc1301a22 // fmla za.s[x8, 2], { z17.s-z20.s }, z0.s\n" - ".inst 0xc1321a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z2.s\n" - ".inst 0xa04149e2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc1351a41 // fmla za.s[x8, 1], { z18.s-z21.s }, z5.s\n" - ".inst 0xc1341a42 // fmla za.s[x8, 2], { z18.s-z21.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "addvl x15, x15, #5\n" + "ld1w { z22.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + "ld1w { z23.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xc13f1a80 // fmla za.s[x8, 0], { z20.s-z23.s }, z15.s\n" + ".inst 0xc13e1a81 // fmla za.s[x8, 1], { z20.s-z23.s }, z14.s\n" + "ld1w { z24.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xc13b1aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z11.s\n" + "ld1w { z25.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xc13a1aa1 // fmla za.s[x8, 1], { z21.s-z24.s }, z10.s\n" + ".inst 0xa04049c6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" + ".inst 0xc1371ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z7.s\n" + "ld1w { z26.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xc1361ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z6.s\n" + ".inst 0xa14049c5 // ld1w { z5.s, z13.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" + ".inst 0xc13d1ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z13.s\n" + "ld1w { z27.s }, p1/Z, [x20]\n" + ".inst 0xc1351ae1 // fmla za.s[x8, 1], { z23.s-z26.s }, z5.s\n" + ".inst 0xa04049c6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x14]\n" + "ldr x14, [%x[args], %[offsetof_Args_weights]]\n" + ".inst 0xc1371b00 // fmla za.s[x8, 0], { z24.s-z27.s }, z7.s\n" + ".inst 0xa04149cc // ld1w { z12.s-z13.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + ".inst 0xc1361b01 // fmla za.s[x8, 1], { z24.s-z27.s }, z6.s\n" + ".inst 0xa04049ce // ld1w { z14.s-z15.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" + ".inst 0xa04049ca // ld1w { z10.s-z11.s }, pn10.b/Z, [x14]\n" + ".inst 0xa04149c8 // ld1w { z8.s-z9.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + "addvl x14, x14, #5\n" + "8:" // Unpadded: 2 priming loads + "add x20, x13, %x[ld_in_row], LSL #2\n" + "ld1w { z1.s }, p1/Z, [x13]\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + "ld1w { z2.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + "ld1w { z3.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + "ld1w { z4.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xc13c1820 // fmla za.s[x8, 0], { z1.s-z4.s }, z12.s\n" + ".inst 0xc13f1821 // fmla za.s[x8, 1], { z1.s-z4.s }, z15.s\n" + "ld1w { z5.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xc13e1822 // fmla za.s[x8, 2], { z1.s-z4.s }, z14.s\n" + "ld1w { z6.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xc1381840 // fmla za.s[x8, 0], { z2.s-z5.s }, z8.s\n" + ".inst 0xa04049cc // ld1w { z12.s-z13.s }, pn10.b/Z, [x14]\n" + ".inst 0xc13b1841 // fmla za.s[x8, 1], { z2.s-z5.s }, z11.s\n" + ".inst 0xa04149ce // ld1w { z14.s-z15.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + "addvl x14, x14, #5\n" + ".inst 0xc13a1842 // fmla za.s[x8, 2], { z2.s-z5.s }, z10.s\n" + "ld1w { z7.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xc13e1860 // fmla za.s[x8, 0], { z3.s-z6.s }, z14.s\n" + ".inst 0xa04049ce // ld1w { z14.s-z15.s }, pn10.b/Z, [x14]\n" + ".inst 0xc13d1861 // fmla za.s[x8, 1], { z3.s-z6.s }, z13.s\n" + ".inst 0xa14149c0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + "addvl x14, x14, #5\n" + ".inst 0xc13c1862 // fmla za.s[x8, 2], { z3.s-z6.s }, z12.s\n" + "ld1w { z8.s }, p1/Z, [x20]\n" + ".inst 0xc1301880 // fmla za.s[x8, 0], { z4.s-z7.s }, z0.s\n" + ".inst 0xa04049c0 // ld1w { z0.s-z1.s }, pn10.b/Z, [x14]\n" + ".inst 0xc13f1881 // fmla za.s[x8, 1], { z4.s-z7.s }, z15.s\n" + ".inst 0xa04149cc // ld1w { z12.s-z13.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + "ldr x14, [%x[args], %[offsetof_Args_weights]]\n" + ".inst 0xc13e1882 // fmla za.s[x8, 2], { z4.s-z7.s }, z14.s\n" + ".inst 0xc13c18a0 // fmla za.s[x8, 0], { z5.s-z8.s }, z12.s\n" + ".inst 0xa04149cc // ld1w { z12.s-z13.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + ".inst 0xc13118a1 // fmla za.s[x8, 1], { z5.s-z8.s }, z1.s\n" + ".inst 0xc13018a2 // fmla za.s[x8, 2], { z5.s-z8.s }, z0.s\n" + ".inst 0xa04049ce // ld1w { z14.s-z15.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" + ".inst 0xa04049ca // ld1w { z10.s-z11.s }, pn10.b/Z, [x14]\n" + ".inst 0xa04149c8 // ld1w { z8.s-z9.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + "addvl x14, x14, #5\n" "9:" // Unpadded: 1 priming loads - "add x20, x14, %x[ld_in_row], LSL #2\n" - "ld1w { z14.s }, p1/Z, [x14]\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - "ld1w { z15.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z17.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc13319c0 // fmla za.s[x8, 0], { z14.s-z17.s }, z3.s\n" - ".inst 0xc13219c1 // fmla za.s[x8, 1], { z14.s-z17.s }, z2.s\n" - "ld1w { z18.s }, p1/Z, [x20]\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" + "ld1w { z20.s }, p1/Z, [x13]\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + "ld1w { z21.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc13519c2 // fmla za.s[x8, 2], { z14.s-z17.s }, z5.s\n" + "ld1w { z22.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + "ld1w { z23.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xc13d1a80 // fmla za.s[x8, 0], { z20.s-z23.s }, z13.s\n" + ".inst 0xc13c1a81 // fmla za.s[x8, 1], { z20.s-z23.s }, z12.s\n" + "ld1w { z24.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xc13f1a82 // fmla za.s[x8, 2], { z20.s-z23.s }, z15.s\n" + "ld1w { z25.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xc13e1a83 // fmla za.s[x8, 3], { z20.s-z23.s }, z14.s\n" + ".inst 0xa04049c4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x14]\n" + ".inst 0xc1391aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z9.s\n" + ".inst 0xa04149cc // ld1w { z12.s-z13.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + "addvl x14, x14, #5\n" + ".inst 0xc1381aa1 // fmla za.s[x8, 1], { z21.s-z24.s }, z8.s\n" + "ld1w { z26.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xc13b1aa2 // fmla za.s[x8, 2], { z21.s-z24.s }, z11.s\n" + ".inst 0xa14149c6 // ld1w { z6.s, z14.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + ".inst 0xc13a1aa3 // fmla za.s[x8, 3], { z21.s-z24.s }, z10.s\n" + ".inst 0xa14049c1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" + ".inst 0xc13d1ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z13.s\n" + "ld1w { z27.s }, p1/Z, [x20]\n" + ".inst 0xc13c1ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z12.s\n" + ".inst 0xa04149cc // ld1w { z12.s-z13.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + ".inst 0xc1351ac2 // fmla za.s[x8, 2], { z22.s-z25.s }, z5.s\n" + ".inst 0xc1341ac3 // fmla za.s[x8, 3], { z22.s-z25.s }, z4.s\n" + ".inst 0xa04049c4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x14]\n" + "ldr x14, [%x[args], %[offsetof_Args_weights]]\n" + ".inst 0xc13e1ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z14.s\n" + "ld1w { z2.s }, p2/Z, [x14, #4, MUL VL]\n" + ".inst 0xc1361ae1 // fmla za.s[x8, 1], { z23.s-z26.s }, z6.s\n" + ".inst 0xc1391ae2 // fmla za.s[x8, 2], { z23.s-z26.s }, z9.s\n" + ".inst 0xc1311ae3 // fmla za.s[x8, 3], { z23.s-z26.s }, z1.s\n" + ".inst 0xc13d1b00 // fmla za.s[x8, 0], { z24.s-z27.s }, z13.s\n" + ".inst 0xc13c1b01 // fmla za.s[x8, 1], { z24.s-z27.s }, z12.s\n" + ".inst 0xa04149cc // ld1w { z12.s-z13.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + ".inst 0xc1351b02 // fmla za.s[x8, 2], { z24.s-z27.s }, z5.s\n" + ".inst 0xc1341b03 // fmla za.s[x8, 3], { z24.s-z27.s }, z4.s\n" + ".inst 0xa04049ce // ld1w { z14.s-z15.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" + ".inst 0xa04049ca // ld1w { z10.s-z11.s }, pn10.b/Z, [x14]\n" + ".inst 0xa04149c8 // ld1w { z8.s-z9.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + "ld1w { z3.s }, p2/Z, [x14, #4, MUL VL]\n" + "addvl x14, x14, #5\n" + "10:" // Unpadded: 0 priming loads + "cbz x15, 20f\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" + "ld1w { z18.s }, p1/Z, [x13]\n" + "sub x15, x15, #0x1\n" "ld1w { z19.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc13419c3 // fmla za.s[x8, 3], { z14.s-z17.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xc13719e0 // fmla za.s[x8, 0], { z15.s-z18.s }, z7.s\n" - ".inst 0xa04149e2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "addvl x15, x15, #5\n" - ".inst 0xc13619e1 // fmla za.s[x8, 1], { z15.s-z18.s }, z6.s\n" + "sub x11, x11, #0x1\n" "ld1w { z20.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc13819e2 // fmla za.s[x8, 2], { z15.s-z18.s }, z8.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc13019e3 // fmla za.s[x8, 3], { z15.s-z18.s }, z0.s\n" - ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - ".inst 0xc1331a00 // fmla za.s[x8, 0], { z16.s-z19.s }, z3.s\n" + "cmp x15, x11\n" "ld1w { z21.s }, p1/Z, [x20]\n" - ".inst 0xc1321a01 // fmla za.s[x8, 1], { z16.s-z19.s }, z2.s\n" - ".inst 0xa04149e2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc1351a02 // fmla za.s[x8, 2], { z16.s-z19.s }, z5.s\n" - ".inst 0xc1341a03 // fmla za.s[x8, 3], { z16.s-z19.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xc1371a20 // fmla za.s[x8, 0], { z17.s-z20.s }, z7.s\n" - "ld1w { z10.s }, p2/Z, [x15, #4, MUL VL]\n" - ".inst 0xc1361a21 // fmla za.s[x8, 1], { z17.s-z20.s }, z6.s\n" - ".inst 0xc1381a22 // fmla za.s[x8, 2], { z17.s-z20.s }, z8.s\n" - ".inst 0xc1301a23 // fmla za.s[x8, 3], { z17.s-z20.s }, z0.s\n" - ".inst 0xc1331a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z3.s\n" - ".inst 0xc1321a41 // fmla za.s[x8, 1], { z18.s-z21.s }, z2.s\n" - ".inst 0xa04149e2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc1351a42 // fmla za.s[x8, 2], { z18.s-z21.s }, z5.s\n" - ".inst 0xc1341a43 // fmla za.s[x8, 3], { z18.s-z21.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "ld1w { z1.s }, p2/Z, [x15, #4, MUL VL]\n" - "addvl x15, x15, #5\n" - "10:" // Unpadded: 0 priming loads - "cbz x16, 20f\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" - "ld1w { z14.s }, p1/Z, [x14]\n" - "sub x16, x16, #0x1\n" - "ld1w { z15.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "sub x13, x13, #0x1\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + "csel x21, x15, x11, LT\n" + "ld1w { z22.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "cmp x16, x13\n" - "ld1w { z17.s }, p1/Z, [x20]\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + "ld1w { z23.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "csel x21, x16, x13, LT\n" - "ld1w { z18.s }, p1/Z, [x20]\n" + "sub x11, x11, x21\n" + "ld1w { z24.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" + "ld1w { z25.s }, p1/Z, [x20]\n" + "cbz x21, 19f\n" + "11:" // Unpadded: Main loop + ".inst 0xc1321a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z2.s\n" + "ld1w { z6.s }, p2/Z, [x14, #4, MUL VL]\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" + "subs x21, x21, #0x1\n" + ".inst 0xc1331a60 // fmla za.s[x8, 0], { z19.s-z22.s }, z3.s\n" + ".inst 0xc13d1a41 // fmla za.s[x8, 1], { z18.s-z21.s }, z13.s\n" + ".inst 0xc13c1a42 // fmla za.s[x8, 2], { z18.s-z21.s }, z12.s\n" + ".inst 0xa04149cc // ld1w { z12.s-z13.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + ".inst 0xc13f1a43 // fmla za.s[x8, 3], { z18.s-z21.s }, z15.s\n" + ".inst 0xc13e1a44 // fmla za.s[x8, 4], { z18.s-z21.s }, z14.s\n" + ".inst 0xa04049c4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" + ".inst 0xc1361a80 // fmla za.s[x8, 0], { z20.s-z23.s }, z6.s\n" + "ld1w { z6.s }, p2/Z, [x14, #4, MUL VL]\n" + ".inst 0xc1391a61 // fmla za.s[x8, 1], { z19.s-z22.s }, z9.s\n" + "ld1w { z18.s }, p1/Z, [x13]\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + ".inst 0xc1381a62 // fmla za.s[x8, 2], { z19.s-z22.s }, z8.s\n" + ".inst 0xa04149ce // ld1w { z14.s-z15.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + ".inst 0xc13b1a63 // fmla za.s[x8, 3], { z19.s-z22.s }, z11.s\n" + ".inst 0xc13a1a64 // fmla za.s[x8, 4], { z19.s-z22.s }, z10.s\n" + ".inst 0xa14049c0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" + ".inst 0xc1361aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z6.s\n" + "ld1w { z2.s }, p2/Z, [x14, #4, MUL VL]\n" + ".inst 0xc13d1a81 // fmla za.s[x8, 1], { z20.s-z23.s }, z13.s\n" "ld1w { z19.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "sub x13, x13, x21\n" + ".inst 0xc13c1a82 // fmla za.s[x8, 2], { z20.s-z23.s }, z12.s\n" + ".inst 0xa04149c6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + ".inst 0xc1351a83 // fmla za.s[x8, 3], { z20.s-z23.s }, z5.s\n" + ".inst 0xc1341a84 // fmla za.s[x8, 4], { z20.s-z23.s }, z4.s\n" + ".inst 0xa04049c4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x14]\n" + "ldr x14, [%x[args], %[offsetof_Args_weights]]\n" + ".inst 0xc1321ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z2.s\n" "ld1w { z20.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xc13f1aa1 // fmla za.s[x8, 1], { z21.s-z24.s }, z15.s\n" + "ld1w { z2.s }, p2/Z, [x14, #4, MUL VL]\n" + ".inst 0xc13e1aa2 // fmla za.s[x8, 2], { z21.s-z24.s }, z14.s\n" + ".inst 0xc1381aa3 // fmla za.s[x8, 3], { z21.s-z24.s }, z8.s\n" + ".inst 0xc1301aa4 // fmla za.s[x8, 4], { z21.s-z24.s }, z0.s\n" "ld1w { z21.s }, p1/Z, [x20]\n" - "cbz x21, 19f\n" - "11:" // Unpadded: Main loop - ".inst 0xc13a19c0 // fmla za.s[x8, 0], { z14.s-z17.s }, z10.s\n" - "ld1w { z10.s }, p2/Z, [x15, #4, MUL VL]\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" - "subs x21, x21, #0x1\n" - ".inst 0xc13119e0 // fmla za.s[x8, 0], { z15.s-z18.s }, z1.s\n" - ".inst 0xc13319c1 // fmla za.s[x8, 1], { z14.s-z17.s }, z3.s\n" - ".inst 0xc13219c2 // fmla za.s[x8, 2], { z14.s-z17.s }, z2.s\n" - ".inst 0xa04149e2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc13519c3 // fmla za.s[x8, 3], { z14.s-z17.s }, z5.s\n" - ".inst 0xc13419c4 // fmla za.s[x8, 4], { z14.s-z17.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - ".inst 0xc13a1a00 // fmla za.s[x8, 0], { z16.s-z19.s }, z10.s\n" - "ld1w { z1.s }, p2/Z, [x15, #4, MUL VL]\n" - ".inst 0xc13719e1 // fmla za.s[x8, 1], { z15.s-z18.s }, z7.s\n" - "ld1w { z14.s }, p1/Z, [x14]\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xc13619e2 // fmla za.s[x8, 2], { z15.s-z18.s }, z6.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc13819e3 // fmla za.s[x8, 3], { z15.s-z18.s }, z8.s\n" - ".inst 0xc13019e4 // fmla za.s[x8, 4], { z15.s-z18.s }, z0.s\n" - ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - ".inst 0xc1311a20 // fmla za.s[x8, 0], { z17.s-z20.s }, z1.s\n" - "ld1w { z10.s }, p2/Z, [x15, #4, MUL VL]\n" - ".inst 0xc1331a01 // fmla za.s[x8, 1], { z16.s-z19.s }, z3.s\n" - "ld1w { z15.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1321a02 // fmla za.s[x8, 2], { z16.s-z19.s }, z2.s\n" - ".inst 0xa04149e2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc1351a03 // fmla za.s[x8, 3], { z16.s-z19.s }, z5.s\n" - ".inst 0xc1341a04 // fmla za.s[x8, 4], { z16.s-z19.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xc13a1a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z10.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1371a21 // fmla za.s[x8, 1], { z17.s-z20.s }, z7.s\n" - "ld1w { z10.s }, p2/Z, [x15, #4, MUL VL]\n" - ".inst 0xc1361a22 // fmla za.s[x8, 2], { z17.s-z20.s }, z6.s\n" - ".inst 0xc1381a23 // fmla za.s[x8, 3], { z17.s-z20.s }, z8.s\n" - ".inst 0xc1301a24 // fmla za.s[x8, 4], { z17.s-z20.s }, z0.s\n" - "ld1w { z17.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n" - ".inst 0xc1abcad8 // fclamp { z24.s-z27.s }, z22.s, z11.s\n" - "st1w { z24.s }, p1, [x9]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xc0060c0c // mova { z12.d-z15.d }, za.d[x8, #0]\n" + ".inst 0xc1b1ca0c // fclamp { z12.s-z15.s }, z16.s, z17.s\n" + "st1w { z12.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - ".inst 0xc1331a41 // fmla za.s[x8, 1], { z18.s-z21.s }, z3.s\n" - "st1w { z25.s }, p1, [x28]\n" + ".inst 0xc1371ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z7.s\n" + "st1w { z13.s }, p1, [x28]\n" "add x28, x28, x26, LSL #2\n" - ".inst 0xc1321a42 // fmla za.s[x8, 2], { z18.s-z21.s }, z2.s\n" - ".inst 0xa04149e2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "st1w { z26.s }, p1, [x25]\n" + ".inst 0xc1361ac2 // fmla za.s[x8, 2], { z22.s-z25.s }, z6.s\n" + ".inst 0xa04149cc // ld1w { z12.s-z13.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + "st1w { z14.s }, p1, [x25]\n" "add x25, x25, x23, LSL #2\n" - ".inst 0xc1351a43 // fmla za.s[x8, 3], { z18.s-z21.s }, z5.s\n" - "st1w { z27.s }, p1, [x24]\n" + ".inst 0xc1351ac3 // fmla za.s[x8, 3], { z22.s-z25.s }, z5.s\n" + "st1w { z15.s }, p1, [x24]\n" "add x24, x24, x22, LSL #2\n" - ".inst 0xc1341a44 // fmla za.s[x8, 4], { z18.s-z21.s }, z4.s\n" - "ld1w { z18.s }, p1/Z, [x20]\n" + ".inst 0xc1341ac4 // fmla za.s[x8, 4], { z22.s-z25.s }, z4.s\n" + "ld1w { z22.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "add x8, x8, #0x1\n" - "ld1w { z19.s }, p1/Z, [x20]\n" + "ld1w { z23.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - "ld1w { z20.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "ld1w { z1.s }, p2/Z, [x15, #4, MUL VL]\n" - "addvl x15, x15, #5\n" - "ld1w { z21.s }, p1/Z, [x20]\n" + ".inst 0xa04049ce // ld1w { z14.s-z15.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" + "ld1w { z24.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xa04049ca // ld1w { z10.s-z11.s }, pn10.b/Z, [x14]\n" + ".inst 0xa04149c8 // ld1w { z8.s-z9.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + "ld1w { z3.s }, p2/Z, [x14, #4, MUL VL]\n" + "addvl x14, x14, #5\n" + "ld1w { z25.s }, p1/Z, [x20]\n" "bgt 11b\n" "b 19f\n" "12:" // Padded "cbz x10, 17f\n" "cmp x10, #0x1\n" - "sub x16, x16, x10\n" + "sub x15, x15, x10\n" "beq 16f\n" "cmp x10, #0x2\n" "beq 15f\n" @@ -451,429 +451,429 @@ void sme2_fp32_planar_5x5_s1_4rows_mla_za_impl( "13:" // Padded: 4 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z14.s }, p0/Z, [x14]\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + "ld1w { z19.s }, p0/Z, [x13]\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z15.s }, p0/Z, [x20]\n" + "ld1w { z20.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z21.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z17.s }, p0/Z, [x20]\n" + "ld1w { z22.s }, p0/Z, [x20]\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc13419c0 // fmla za.s[x8, 0], { z14.s-z17.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" + ".inst 0xc13e1a60 // fmla za.s[x8, 0], { z19.s-z22.s }, z14.s\n" + ".inst 0xa14049c1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x14]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z18.s }, p0/Z, [x20]\n" + "ld1w { z23.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc13019e0 // fmla za.s[x8, 0], { z15.s-z18.s }, z0.s\n" + ".inst 0xc13a1a80 // fmla za.s[x8, 0], { z20.s-z23.s }, z10.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z19.s }, p0/Z, [x20]\n" + "ld1w { z24.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1341a00 // fmla za.s[x8, 0], { z16.s-z19.s }, z4.s\n" + ".inst 0xc1311aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z1.s\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "addvl x15, x15, #5\n" - "ld1w { z20.s }, p0/Z, [x20]\n" - ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" + "addvl x14, x14, #5\n" + "ld1w { z25.s }, p0/Z, [x20]\n" + ".inst 0xa04049c6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x14]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xc1301a20 // fmla za.s[x8, 0], { z17.s-z20.s }, z0.s\n" - "addvl x15, x15, #5\n" - "ld1w { z21.s }, p0/Z, [x20]\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xc1341a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" + ".inst 0xc1361ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z6.s\n" + "addvl x14, x14, #5\n" + "ld1w { z26.s }, p0/Z, [x20]\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + ".inst 0xa14049c0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x14]\n" + "ldr x14, [%x[args], %[offsetof_Args_weights]]\n" + ".inst 0xc1301ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z0.s\n" + ".inst 0xa04049ce // ld1w { z14.s-z15.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" + ".inst 0xa04049ca // ld1w { z10.s-z11.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" "14:" // Padded: 3 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z14.s }, p0/Z, [x14]\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + "ld1w { z0.s }, p0/Z, [x13]\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z15.s }, p0/Z, [x20]\n" + "ld1w { z1.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z2.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z17.s }, p0/Z, [x20]\n" + "ld1w { z3.s }, p0/Z, [x20]\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc13519c0 // fmla za.s[x8, 0], { z14.s-z17.s }, z5.s\n" + ".inst 0xc13f1800 // fmla za.s[x8, 0], { z0.s-z3.s }, z15.s\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc13419c1 // fmla za.s[x8, 1], { z14.s-z17.s }, z4.s\n" - "ld1w { z18.s }, p0/Z, [x20]\n" + ".inst 0xc13e1801 // fmla za.s[x8, 1], { z0.s-z3.s }, z14.s\n" + "ld1w { z4.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0xc13819e0 // fmla za.s[x8, 0], { z15.s-z18.s }, z8.s\n" - "ld1w { z19.s }, p0/Z, [x20]\n" + ".inst 0xc13b1820 // fmla za.s[x8, 0], { z1.s-z4.s }, z11.s\n" + "ld1w { z5.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc13019e1 // fmla za.s[x8, 1], { z15.s-z18.s }, z0.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" + ".inst 0xc13a1821 // fmla za.s[x8, 1], { z1.s-z4.s }, z10.s\n" + ".inst 0xa04049c8 // ld1w { z8.s-z9.s }, pn10.b/Z, [x14]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "addvl x15, x15, #5\n" - ".inst 0xc1351a00 // fmla za.s[x8, 0], { z16.s-z19.s }, z5.s\n" - "ld1w { z20.s }, p0/Z, [x20]\n" + "addvl x14, x14, #5\n" + ".inst 0xc1391840 // fmla za.s[x8, 0], { z2.s-z5.s }, z9.s\n" + "ld1w { z6.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xc1341a01 // fmla za.s[x8, 1], { z16.s-z19.s }, z4.s\n" - ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xc1381a20 // fmla za.s[x8, 0], { z17.s-z20.s }, z8.s\n" - "ld1w { z21.s }, p0/Z, [x20]\n" - ".inst 0xc1301a21 // fmla za.s[x8, 1], { z17.s-z20.s }, z0.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xc1351a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z5.s\n" - ".inst 0xa04149e2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc1341a41 // fmla za.s[x8, 1], { z18.s-z21.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "addvl x15, x15, #5\n" + ".inst 0xc1381841 // fmla za.s[x8, 1], { z2.s-z5.s }, z8.s\n" + ".inst 0xa04049ce // ld1w { z14.s-z15.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + ".inst 0xc13f1860 // fmla za.s[x8, 0], { z3.s-z6.s }, z15.s\n" + "ld1w { z7.s }, p0/Z, [x20]\n" + ".inst 0xc13e1861 // fmla za.s[x8, 1], { z3.s-z6.s }, z14.s\n" + ".inst 0xa14049c3 // ld1w { z3.s, z11.s }, pn10.b/Z, [x14]\n" + "ldr x14, [%x[args], %[offsetof_Args_weights]]\n" + ".inst 0xc13b1880 // fmla za.s[x8, 0], { z4.s-z7.s }, z11.s\n" + ".inst 0xa04149cc // ld1w { z12.s-z13.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + ".inst 0xc1331881 // fmla za.s[x8, 1], { z4.s-z7.s }, z3.s\n" + ".inst 0xa04049ce // ld1w { z14.s-z15.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" + ".inst 0xa04049ca // ld1w { z10.s-z11.s }, pn10.b/Z, [x14]\n" + ".inst 0xa04149c8 // ld1w { z8.s-z9.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + "addvl x14, x14, #5\n" "15:" // Padded: 2 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z14.s }, p0/Z, [x14]\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + "ld1w { z19.s }, p0/Z, [x13]\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z15.s }, p0/Z, [x20]\n" + "ld1w { z20.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z21.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z17.s }, p0/Z, [x20]\n" + "ld1w { z22.s }, p0/Z, [x20]\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc13219c0 // fmla za.s[x8, 0], { z14.s-z17.s }, z2.s\n" - ".inst 0xa04149e2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xc13c1a60 // fmla za.s[x8, 0], { z19.s-z22.s }, z12.s\n" + ".inst 0xa04149c6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc13519c1 // fmla za.s[x8, 1], { z14.s-z17.s }, z5.s\n" - "ld1w { z18.s }, p0/Z, [x20]\n" + ".inst 0xc13f1a61 // fmla za.s[x8, 1], { z19.s-z22.s }, z15.s\n" + "ld1w { z23.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc13419c2 // fmla za.s[x8, 2], { z14.s-z17.s }, z4.s\n" + ".inst 0xc13e1a62 // fmla za.s[x8, 2], { z19.s-z22.s }, z14.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z19.s }, p0/Z, [x20]\n" + "ld1w { z24.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc13619e0 // fmla za.s[x8, 0], { z15.s-z18.s }, z6.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" + ".inst 0xc1381a80 // fmla za.s[x8, 0], { z20.s-z23.s }, z8.s\n" + ".inst 0xa14049c0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x14]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "addvl x15, x15, #5\n" - ".inst 0xc13819e1 // fmla za.s[x8, 1], { z15.s-z18.s }, z8.s\n" - "ld1w { z20.s }, p0/Z, [x20]\n" + "addvl x14, x14, #5\n" + ".inst 0xc13b1a81 // fmla za.s[x8, 1], { z20.s-z23.s }, z11.s\n" + "ld1w { z25.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xc13019e2 // fmla za.s[x8, 2], { z15.s-z18.s }, z0.s\n" - ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xc1321a00 // fmla za.s[x8, 0], { z16.s-z19.s }, z2.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "addvl x15, x15, #5\n" - ".inst 0xc1351a01 // fmla za.s[x8, 1], { z16.s-z19.s }, z5.s\n" - "ld1w { z21.s }, p0/Z, [x20]\n" - ".inst 0xc1341a02 // fmla za.s[x8, 2], { z16.s-z19.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xc1361a20 // fmla za.s[x8, 0], { z17.s-z20.s }, z6.s\n" - ".inst 0xa04149e2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xc1381a21 // fmla za.s[x8, 1], { z17.s-z20.s }, z8.s\n" - ".inst 0xc1301a22 // fmla za.s[x8, 2], { z17.s-z20.s }, z0.s\n" - ".inst 0xc1321a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z2.s\n" - ".inst 0xa04149e2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc1351a41 // fmla za.s[x8, 1], { z18.s-z21.s }, z5.s\n" - ".inst 0xc1341a42 // fmla za.s[x8, 2], { z18.s-z21.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "addvl x15, x15, #5\n" + ".inst 0xc13a1a82 // fmla za.s[x8, 2], { z20.s-z23.s }, z10.s\n" + ".inst 0xa14049c2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x14]\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + ".inst 0xc1361aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z6.s\n" + ".inst 0xa14149c4 // ld1w { z4.s, z12.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + "addvl x14, x14, #5\n" + ".inst 0xc1381aa1 // fmla za.s[x8, 1], { z21.s-z24.s }, z8.s\n" + "ld1w { z26.s }, p0/Z, [x20]\n" + ".inst 0xc1301aa2 // fmla za.s[x8, 2], { z21.s-z24.s }, z0.s\n" + ".inst 0xa04049c6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x14]\n" + ".inst 0xc1341ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z4.s\n" + ".inst 0xa14149c3 // ld1w { z3.s, z11.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + "ldr x14, [%x[args], %[offsetof_Args_weights]]\n" + ".inst 0xc13a1ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z10.s\n" + ".inst 0xc1321ac2 // fmla za.s[x8, 2], { z22.s-z25.s }, z2.s\n" + ".inst 0xc1331ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z3.s\n" + ".inst 0xa04149cc // ld1w { z12.s-z13.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + ".inst 0xc1371ae1 // fmla za.s[x8, 1], { z23.s-z26.s }, z7.s\n" + ".inst 0xc1361ae2 // fmla za.s[x8, 2], { z23.s-z26.s }, z6.s\n" + ".inst 0xa04049ce // ld1w { z14.s-z15.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" + ".inst 0xa04049ca // ld1w { z10.s-z11.s }, pn10.b/Z, [x14]\n" + ".inst 0xa04149c8 // ld1w { z8.s-z9.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + "addvl x14, x14, #5\n" "16:" // Padded: 1 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z14.s }, p0/Z, [x14]\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + "ld1w { z18.s }, p0/Z, [x13]\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z15.s }, p0/Z, [x20]\n" + "ld1w { z19.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z20.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z17.s }, p0/Z, [x20]\n" + "ld1w { z21.s }, p0/Z, [x20]\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc13319c0 // fmla za.s[x8, 0], { z14.s-z17.s }, z3.s\n" + ".inst 0xc13d1a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z13.s\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc13219c1 // fmla za.s[x8, 1], { z14.s-z17.s }, z2.s\n" - "ld1w { z18.s }, p0/Z, [x20]\n" + ".inst 0xc13c1a41 // fmla za.s[x8, 1], { z18.s-z21.s }, z12.s\n" + "ld1w { z22.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc13519c2 // fmla za.s[x8, 2], { z14.s-z17.s }, z5.s\n" + ".inst 0xc13f1a42 // fmla za.s[x8, 2], { z18.s-z21.s }, z15.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z19.s }, p0/Z, [x20]\n" + "ld1w { z23.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc13419c3 // fmla za.s[x8, 3], { z14.s-z17.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" + ".inst 0xc13e1a43 // fmla za.s[x8, 3], { z18.s-z21.s }, z14.s\n" + ".inst 0xa04049ce // ld1w { z14.s-z15.s }, pn10.b/Z, [x14]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xc13719e0 // fmla za.s[x8, 0], { z15.s-z18.s }, z7.s\n" - ".inst 0xa04149e2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "addvl x15, x15, #5\n" - ".inst 0xc13619e1 // fmla za.s[x8, 1], { z15.s-z18.s }, z6.s\n" - "ld1w { z20.s }, p0/Z, [x20]\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + ".inst 0xc1391a60 // fmla za.s[x8, 0], { z19.s-z22.s }, z9.s\n" + ".inst 0xa04149cc // ld1w { z12.s-z13.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + "addvl x14, x14, #5\n" + ".inst 0xc1381a61 // fmla za.s[x8, 1], { z19.s-z22.s }, z8.s\n" + "ld1w { z24.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xc13819e2 // fmla za.s[x8, 2], { z15.s-z18.s }, z8.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc13019e3 // fmla za.s[x8, 3], { z15.s-z18.s }, z0.s\n" - ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - ".inst 0xc1331a00 // fmla za.s[x8, 0], { z16.s-z19.s }, z3.s\n" - "ld1w { z21.s }, p0/Z, [x20]\n" - ".inst 0xc1321a01 // fmla za.s[x8, 1], { z16.s-z19.s }, z2.s\n" - ".inst 0xa04149e2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc1351a02 // fmla za.s[x8, 2], { z16.s-z19.s }, z5.s\n" - ".inst 0xc1341a03 // fmla za.s[x8, 3], { z16.s-z19.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xc1371a20 // fmla za.s[x8, 0], { z17.s-z20.s }, z7.s\n" - "ld1w { z10.s }, p2/Z, [x15, #4, MUL VL]\n" - ".inst 0xc1361a21 // fmla za.s[x8, 1], { z17.s-z20.s }, z6.s\n" - ".inst 0xc1381a22 // fmla za.s[x8, 2], { z17.s-z20.s }, z8.s\n" - ".inst 0xc1301a23 // fmla za.s[x8, 3], { z17.s-z20.s }, z0.s\n" - ".inst 0xc1331a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z3.s\n" - ".inst 0xc1321a41 // fmla za.s[x8, 1], { z18.s-z21.s }, z2.s\n" - ".inst 0xa04149e2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc1351a42 // fmla za.s[x8, 2], { z18.s-z21.s }, z5.s\n" - ".inst 0xc1341a43 // fmla za.s[x8, 3], { z18.s-z21.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "ld1w { z1.s }, p2/Z, [x15, #4, MUL VL]\n" - "addvl x15, x15, #5\n" + ".inst 0xc13b1a62 // fmla za.s[x8, 2], { z19.s-z22.s }, z11.s\n" + ".inst 0xa14149c0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + ".inst 0xc13a1a63 // fmla za.s[x8, 3], { z19.s-z22.s }, z10.s\n" + ".inst 0xa14049c1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" + ".inst 0xc13d1a80 // fmla za.s[x8, 0], { z20.s-z23.s }, z13.s\n" + "ld1w { z25.s }, p0/Z, [x20]\n" + ".inst 0xc13c1a81 // fmla za.s[x8, 1], { z20.s-z23.s }, z12.s\n" + ".inst 0xa04149cc // ld1w { z12.s-z13.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + ".inst 0xc13f1a82 // fmla za.s[x8, 2], { z20.s-z23.s }, z15.s\n" + ".inst 0xc13e1a83 // fmla za.s[x8, 3], { z20.s-z23.s }, z14.s\n" + ".inst 0xa04049ca // ld1w { z10.s-z11.s }, pn10.b/Z, [x14]\n" + "ldr x14, [%x[args], %[offsetof_Args_weights]]\n" + ".inst 0xc1381aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z8.s\n" + "ld1w { z2.s }, p2/Z, [x14, #4, MUL VL]\n" + ".inst 0xc1301aa1 // fmla za.s[x8, 1], { z21.s-z24.s }, z0.s\n" + ".inst 0xc1391aa2 // fmla za.s[x8, 2], { z21.s-z24.s }, z9.s\n" + ".inst 0xc1311aa3 // fmla za.s[x8, 3], { z21.s-z24.s }, z1.s\n" + ".inst 0xc13d1ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z13.s\n" + ".inst 0xc13c1ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z12.s\n" + ".inst 0xa04149cc // ld1w { z12.s-z13.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + ".inst 0xc13b1ac2 // fmla za.s[x8, 2], { z22.s-z25.s }, z11.s\n" + ".inst 0xc13a1ac3 // fmla za.s[x8, 3], { z22.s-z25.s }, z10.s\n" + ".inst 0xa04049ce // ld1w { z14.s-z15.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" + ".inst 0xa04049ca // ld1w { z10.s-z11.s }, pn10.b/Z, [x14]\n" + ".inst 0xa04149c8 // ld1w { z8.s-z9.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + "ld1w { z3.s }, p2/Z, [x14, #4, MUL VL]\n" + "addvl x14, x14, #5\n" "17:" // Padded: 0 priming loads - "cbz x16, 20f\n" + "cbz x15, 20f\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z14.s }, p0/Z, [x14]\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + "ld1w { z18.s }, p0/Z, [x13]\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z15.s }, p0/Z, [x20]\n" + "ld1w { z19.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z20.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z17.s }, p0/Z, [x20]\n" + "ld1w { z21.s }, p0/Z, [x20]\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z18.s }, p0/Z, [x20]\n" + "ld1w { z22.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "sub x16, x16, #0x1\n" - "ld1w { z19.s }, p0/Z, [x20]\n" - "sub x13, x13, #0x1\n" + "sub x15, x15, #0x1\n" + "ld1w { z23.s }, p0/Z, [x20]\n" + "sub x11, x11, #0x1\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "cmp x16, x13\n" - "ld1w { z20.s }, p0/Z, [x20]\n" + "cmp x15, x11\n" + "ld1w { z24.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z21.s }, p0/Z, [x20]\n" - "csel x21, x16, x13, LT\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - "sub x13, x13, x21\n" + "ld1w { z25.s }, p0/Z, [x20]\n" + "csel x21, x15, x11, LT\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + "sub x11, x11, x21\n" "cbz x21, 19f\n" "18:" // Padded: Main loop - ".inst 0xc13a19c0 // fmla za.s[x8, 0], { z14.s-z17.s }, z10.s\n" - "ld1w { z10.s }, p2/Z, [x15, #4, MUL VL]\n" + ".inst 0xc1321a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z2.s\n" + "ld1w { z0.s }, p2/Z, [x14, #4, MUL VL]\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc13119e0 // fmla za.s[x8, 0], { z15.s-z18.s }, z1.s\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + ".inst 0xc1331a60 // fmla za.s[x8, 0], { z19.s-z22.s }, z3.s\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" "subs x21, x21, #0x1\n" - ".inst 0xc13319c1 // fmla za.s[x8, 1], { z14.s-z17.s }, z3.s\n" - ".inst 0xc13219c2 // fmla za.s[x8, 2], { z14.s-z17.s }, z2.s\n" - ".inst 0xa04149e2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc13519c3 // fmla za.s[x8, 3], { z14.s-z17.s }, z5.s\n" - ".inst 0xc13419c4 // fmla za.s[x8, 4], { z14.s-z17.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - ".inst 0xc13a1a00 // fmla za.s[x8, 0], { z16.s-z19.s }, z10.s\n" - "ld1w { z1.s }, p2/Z, [x15, #4, MUL VL]\n" - ".inst 0xc13719e1 // fmla za.s[x8, 1], { z15.s-z18.s }, z7.s\n" - "ld1w { z14.s }, p0/Z, [x14]\n" + ".inst 0xc13d1a41 // fmla za.s[x8, 1], { z18.s-z21.s }, z13.s\n" + ".inst 0xc13c1a42 // fmla za.s[x8, 2], { z18.s-z21.s }, z12.s\n" + ".inst 0xa04149c2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + ".inst 0xc13f1a43 // fmla za.s[x8, 3], { z18.s-z21.s }, z15.s\n" + ".inst 0xc13e1a44 // fmla za.s[x8, 4], { z18.s-z21.s }, z14.s\n" + ".inst 0xa04049c4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" + ".inst 0xc1301a80 // fmla za.s[x8, 0], { z20.s-z23.s }, z0.s\n" + "ld1w { z12.s }, p2/Z, [x14, #4, MUL VL]\n" + ".inst 0xc1391a61 // fmla za.s[x8, 1], { z19.s-z22.s }, z9.s\n" + "ld1w { z18.s }, p0/Z, [x13]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xc13619e2 // fmla za.s[x8, 2], { z15.s-z18.s }, z6.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc13819e3 // fmla za.s[x8, 3], { z15.s-z18.s }, z8.s\n" - ".inst 0xc13019e4 // fmla za.s[x8, 4], { z15.s-z18.s }, z0.s\n" - ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - ".inst 0xc1311a20 // fmla za.s[x8, 0], { z17.s-z20.s }, z1.s\n" - "ld1w { z10.s }, p2/Z, [x15, #4, MUL VL]\n" - ".inst 0xc1331a01 // fmla za.s[x8, 1], { z16.s-z19.s }, z3.s\n" - "ld1w { z15.s }, p0/Z, [x20]\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + ".inst 0xc1381a62 // fmla za.s[x8, 2], { z19.s-z22.s }, z8.s\n" + ".inst 0xa14149c0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + ".inst 0xc13b1a63 // fmla za.s[x8, 3], { z19.s-z22.s }, z11.s\n" + ".inst 0xc13a1a64 // fmla za.s[x8, 4], { z19.s-z22.s }, z10.s\n" + ".inst 0xa04049c6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" + ".inst 0xc13c1aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z12.s\n" + "ld1w { z12.s }, p2/Z, [x14, #4, MUL VL]\n" + ".inst 0xc1331a81 // fmla za.s[x8, 1], { z20.s-z23.s }, z3.s\n" + "ld1w { z19.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc1321a02 // fmla za.s[x8, 2], { z16.s-z19.s }, z2.s\n" - ".inst 0xa04149e2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc1351a03 // fmla za.s[x8, 3], { z16.s-z19.s }, z5.s\n" - ".inst 0xc1341a04 // fmla za.s[x8, 4], { z16.s-z19.s }, z4.s\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + ".inst 0xc1321a82 // fmla za.s[x8, 2], { z20.s-z23.s }, z2.s\n" + ".inst 0xa14149c3 // ld1w { z3.s, z11.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + ".inst 0xc1351a83 // fmla za.s[x8, 3], { z20.s-z23.s }, z5.s\n" + ".inst 0xc1341a84 // fmla za.s[x8, 4], { z20.s-z23.s }, z4.s\n" + "ld1w { z20.s }, p0/Z, [x20]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x4\n" - ".inst 0xc13a1a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z10.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xc1371a21 // fmla za.s[x8, 1], { z17.s-z20.s }, z7.s\n" - "ld1w { z10.s }, p2/Z, [x15, #4, MUL VL]\n" - ".inst 0xc1361a22 // fmla za.s[x8, 2], { z17.s-z20.s }, z6.s\n" - ".inst 0xc1381a23 // fmla za.s[x8, 3], { z17.s-z20.s }, z8.s\n" - ".inst 0xc1301a24 // fmla za.s[x8, 4], { z17.s-z20.s }, z0.s\n" - "ld1w { z17.s }, p0/Z, [x20]\n" + ".inst 0xc13c1ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z12.s\n" + ".inst 0xa04049ce // ld1w { z14.s-z15.s }, pn10.b/Z, [x14]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + "ldr x14, [%x[args], %[offsetof_Args_weights]]\n" + ".inst 0xc1381aa1 // fmla za.s[x8, 1], { z21.s-z24.s }, z8.s\n" + "ld1w { z2.s }, p2/Z, [x14, #4, MUL VL]\n" + ".inst 0xc1301aa2 // fmla za.s[x8, 2], { z21.s-z24.s }, z0.s\n" + ".inst 0xc1371aa3 // fmla za.s[x8, 3], { z21.s-z24.s }, z7.s\n" + ".inst 0xc1361aa4 // fmla za.s[x8, 4], { z21.s-z24.s }, z6.s\n" + "ld1w { z21.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n" - ".inst 0xc1abcad8 // fclamp { z24.s-z27.s }, z22.s, z11.s\n" - "st1w { z24.s }, p1, [x9]\n" + ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" + ".inst 0xc1b1ca04 // fclamp { z4.s-z7.s }, z16.s, z17.s\n" + "st1w { z4.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - ".inst 0xc1331a41 // fmla za.s[x8, 1], { z18.s-z21.s }, z3.s\n" - "st1w { z25.s }, p1, [x28]\n" + ".inst 0xc13b1ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z11.s\n" + "st1w { z5.s }, p1, [x28]\n" "add x28, x28, x26, LSL #2\n" - ".inst 0xc1321a42 // fmla za.s[x8, 2], { z18.s-z21.s }, z2.s\n" - ".inst 0xa04149e2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "st1w { z26.s }, p1, [x25]\n" + ".inst 0xc1331ac2 // fmla za.s[x8, 2], { z22.s-z25.s }, z3.s\n" + ".inst 0xa04149cc // ld1w { z12.s-z13.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + "st1w { z6.s }, p1, [x25]\n" "add x25, x25, x23, LSL #2\n" - ".inst 0xc1351a43 // fmla za.s[x8, 3], { z18.s-z21.s }, z5.s\n" - "st1w { z27.s }, p1, [x24]\n" + ".inst 0xc13f1ac3 // fmla za.s[x8, 3], { z22.s-z25.s }, z15.s\n" + "st1w { z7.s }, p1, [x24]\n" "add x24, x24, x22, LSL #2\n" - ".inst 0xc1341a44 // fmla za.s[x8, 4], { z18.s-z21.s }, z4.s\n" - "ld1w { z18.s }, p0/Z, [x20]\n" + ".inst 0xc13e1ac4 // fmla za.s[x8, 4], { z22.s-z25.s }, z14.s\n" + "ld1w { z22.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z19.s }, p0/Z, [x20]\n" + "ld1w { z23.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" + ".inst 0xa04049ce // ld1w { z14.s-z15.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" "add x8, x8, #0x1\n" ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" - "ld1w { z20.s }, p0/Z, [x20]\n" + "ld1w { z24.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "ld1w { z1.s }, p2/Z, [x15, #4, MUL VL]\n" - "addvl x15, x15, #5\n" - "ld1w { z21.s }, p0/Z, [x20]\n" + ".inst 0xa04049ca // ld1w { z10.s-z11.s }, pn10.b/Z, [x14]\n" + ".inst 0xa04149c8 // ld1w { z8.s-z9.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + "ld1w { z3.s }, p2/Z, [x14, #4, MUL VL]\n" + "addvl x14, x14, #5\n" + "ld1w { z25.s }, p0/Z, [x20]\n" "bgt 18b\n" "19:" // Main loop tail - ".inst 0xc13a19c0 // fmla za.s[x8, 0], { z14.s-z17.s }, z10.s\n" - "ld1w { z10.s }, p2/Z, [x15, #4, MUL VL]\n" - ".inst 0xc13119e0 // fmla za.s[x8, 0], { z15.s-z18.s }, z1.s\n" - ".inst 0xc13319c1 // fmla za.s[x8, 1], { z14.s-z17.s }, z3.s\n" - ".inst 0xc13219c2 // fmla za.s[x8, 2], { z14.s-z17.s }, z2.s\n" - ".inst 0xa04149e2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc13519c3 // fmla za.s[x8, 3], { z14.s-z17.s }, z5.s\n" - ".inst 0xc13419c4 // fmla za.s[x8, 4], { z14.s-z17.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - ".inst 0xc13a1a00 // fmla za.s[x8, 0], { z16.s-z19.s }, z10.s\n" - "ld1w { z1.s }, p2/Z, [x15, #4, MUL VL]\n" - ".inst 0xc13719e1 // fmla za.s[x8, 1], { z15.s-z18.s }, z7.s\n" - ".inst 0xc13619e2 // fmla za.s[x8, 2], { z15.s-z18.s }, z6.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc13819e3 // fmla za.s[x8, 3], { z15.s-z18.s }, z8.s\n" - ".inst 0xc13019e4 // fmla za.s[x8, 4], { z15.s-z18.s }, z0.s\n" - ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - ".inst 0xc1311a20 // fmla za.s[x8, 0], { z17.s-z20.s }, z1.s\n" - "ld1w { z10.s }, p2/Z, [x15, #4, MUL VL]\n" - ".inst 0xc1331a01 // fmla za.s[x8, 1], { z16.s-z19.s }, z3.s\n" - ".inst 0xc1321a02 // fmla za.s[x8, 2], { z16.s-z19.s }, z2.s\n" - ".inst 0xa04149e2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc1351a03 // fmla za.s[x8, 3], { z16.s-z19.s }, z5.s\n" - ".inst 0xc1341a04 // fmla za.s[x8, 4], { z16.s-z19.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xc13a1a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z10.s\n" - ".inst 0xc1371a21 // fmla za.s[x8, 1], { z17.s-z20.s }, z7.s\n" - ".inst 0xc1361a22 // fmla za.s[x8, 2], { z17.s-z20.s }, z6.s\n" - ".inst 0xc1381a23 // fmla za.s[x8, 3], { z17.s-z20.s }, z8.s\n" - ".inst 0xc1301a24 // fmla za.s[x8, 4], { z17.s-z20.s }, z0.s\n" - ".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n" - ".inst 0xc1abcad8 // fclamp { z24.s-z27.s }, z22.s, z11.s\n" - "st1w { z24.s }, p1, [x9]\n" + ".inst 0xc1321a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z2.s\n" + "ld1w { z6.s }, p2/Z, [x14, #4, MUL VL]\n" + ".inst 0xc1331a60 // fmla za.s[x8, 0], { z19.s-z22.s }, z3.s\n" + ".inst 0xc13d1a41 // fmla za.s[x8, 1], { z18.s-z21.s }, z13.s\n" + ".inst 0xc13c1a42 // fmla za.s[x8, 2], { z18.s-z21.s }, z12.s\n" + ".inst 0xa04149c4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + ".inst 0xc13f1a43 // fmla za.s[x8, 3], { z18.s-z21.s }, z15.s\n" + ".inst 0xc13e1a44 // fmla za.s[x8, 4], { z18.s-z21.s }, z14.s\n" + ".inst 0xa04049c2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" + ".inst 0xc1361a80 // fmla za.s[x8, 0], { z20.s-z23.s }, z6.s\n" + "ld1w { z7.s }, p2/Z, [x14, #4, MUL VL]\n" + ".inst 0xc1391a61 // fmla za.s[x8, 1], { z19.s-z22.s }, z9.s\n" + ".inst 0xc1381a62 // fmla za.s[x8, 2], { z19.s-z22.s }, z8.s\n" + ".inst 0xa14149c0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + ".inst 0xc13b1a63 // fmla za.s[x8, 3], { z19.s-z22.s }, z11.s\n" + ".inst 0xc13a1a64 // fmla za.s[x8, 4], { z19.s-z22.s }, z10.s\n" + ".inst 0xa04049ca // ld1w { z10.s-z11.s }, pn10.b/Z, [x14]\n" + "addvl x14, x14, #5\n" + ".inst 0xc1371aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z7.s\n" + "ld1w { z1.s }, p2/Z, [x14, #4, MUL VL]\n" + ".inst 0xc1351a81 // fmla za.s[x8, 1], { z20.s-z23.s }, z5.s\n" + ".inst 0xc1341a82 // fmla za.s[x8, 2], { z20.s-z23.s }, z4.s\n" + ".inst 0xa04149cc // ld1w { z12.s-z13.s }, pn10.b/Z, [x14, #0x2, MUL VL]\n" + ".inst 0xc1331a83 // fmla za.s[x8, 3], { z20.s-z23.s }, z3.s\n" + ".inst 0xc1321a84 // fmla za.s[x8, 4], { z20.s-z23.s }, z2.s\n" + ".inst 0xa04049c2 // ld1w { z2.s-z3.s }, pn10.b/Z, [x14]\n" + ".inst 0xc1311ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z1.s\n" + ".inst 0xc1381aa1 // fmla za.s[x8, 1], { z21.s-z24.s }, z8.s\n" + ".inst 0xc1301aa2 // fmla za.s[x8, 2], { z21.s-z24.s }, z0.s\n" + ".inst 0xc13b1aa3 // fmla za.s[x8, 3], { z21.s-z24.s }, z11.s\n" + ".inst 0xc13a1aa4 // fmla za.s[x8, 4], { z21.s-z24.s }, z10.s\n" + ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" + ".inst 0xc1b1ca04 // fclamp { z4.s-z7.s }, z16.s, z17.s\n" + "st1w { z4.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - ".inst 0xc1331a41 // fmla za.s[x8, 1], { z18.s-z21.s }, z3.s\n" - "st1w { z25.s }, p1, [x28]\n" + ".inst 0xc13d1ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z13.s\n" + "st1w { z5.s }, p1, [x28]\n" "add x28, x28, x26, LSL #2\n" - ".inst 0xc1321a42 // fmla za.s[x8, 2], { z18.s-z21.s }, z2.s\n" - "st1w { z26.s }, p1, [x25]\n" + ".inst 0xc13c1ac2 // fmla za.s[x8, 2], { z22.s-z25.s }, z12.s\n" + "st1w { z6.s }, p1, [x25]\n" "add x25, x25, x23, LSL #2\n" - ".inst 0xc1351a43 // fmla za.s[x8, 3], { z18.s-z21.s }, z5.s\n" - "st1w { z27.s }, p1, [x24]\n" + ".inst 0xc1331ac3 // fmla za.s[x8, 3], { z22.s-z25.s }, z3.s\n" + "st1w { z7.s }, p1, [x24]\n" "add x24, x24, x22, LSL #2\n" - ".inst 0xc1341a44 // fmla za.s[x8, 4], { z18.s-z21.s }, z4.s\n" + ".inst 0xc1321ac4 // fmla za.s[x8, 4], { z22.s-z25.s }, z2.s\n" "add x8, x8, #0x1\n" ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" "20:" // Main loop skip tail - "cbz x13, 22f\n" + "cbz x11, 22f\n" "21:" // Right padding loop - ".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n" + ".inst 0xc0060c00 // mova { z0.d-z3.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" - "subs x13, x13, #0x1\n" - ".inst 0xc1abcad8 // fclamp { z24.s-z27.s }, z22.s, z11.s\n" - "st1w { z24.s }, p1, [x9]\n" + "subs x11, x11, #0x1\n" + ".inst 0xc1b1ca00 // fclamp { z0.s-z3.s }, z16.s, z17.s\n" + "st1w { z0.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" - "st1w { z25.s }, p1, [x28]\n" + "st1w { z1.s }, p1, [x28]\n" "add x28, x28, x26, LSL #2\n" - "st1w { z26.s }, p1, [x25]\n" + "st1w { z2.s }, p1, [x25]\n" "add x25, x25, x23, LSL #2\n" - "st1w { z27.s }, p1, [x24]\n" + "st1w { z3.s }, p1, [x24]\n" "add x24, x24, x22, LSL #2\n" "bgt 21b\n" "22:" // End - "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - "incb x15, ALL, MUL #16\n" - "incb x15, ALL, MUL #9\n" - "str x15, [%x[args], %[offsetof_Args_weights]]\n" - "ldr x20, [%x[args], %[offsetof_Args_ld_in_vl]]\n" - "incw x17\n" - "whilelt p1.s, x17, x7\n" - "ldr x14, [%x[args], %[offsetof_Args_inptr]]\n" - "add x14, x14, x20, LSL #2\n" - "str x14, [%x[args], %[offsetof_Args_inptr]]\n" - "ldr x11, [%x[args], %[offsetof_Args_outptrs]]\n" + "ldr x20, [%x[args], %[offsetof_Args_weights]]\n" + "incb x20, ALL, MUL #16\n" + "incb x20, ALL, MUL #9\n" + "str x20, [%x[args], %[offsetof_Args_weights]]\n" + "ldr x21, [%x[args], %[offsetof_Args_ld_in_vl]]\n" + "incw x16\n" + "whilelt p1.s, x16, x17\n" + "ldr x20, [%x[args], %[offsetof_Args_inptr]]\n" + "add x20, x20, x21, LSL #2\n" + "str x20, [%x[args], %[offsetof_Args_inptr]]\n" + "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" "ldr x24, [%x[args], %[offsetof_Args_ld_out_vls]]\n" - "ldp x23, x22, [x11, #0x0]\n" + "ldp x23, x22, [x25, #0x0]\n" "ldp x21, x20, [x24, #0x0]\n" "add x23, x23, x21, LSL #2\n" "add x22, x22, x20, LSL #2\n" - "stp x23, x22, [x11, #0x0]\n" - "ldp x23, x22, [x11, #0x10]\n" + "stp x23, x22, [x25, #0x0]\n" + "ldp x23, x22, [x25, #0x10]\n" "ldp x21, x20, [x24, #0x10]\n" "add x23, x23, x21, LSL #2\n" "add x22, x22, x20, LSL #2\n" - "stp x23, x22, [x11, #0x10]\n" + "stp x23, x22, [x25, #0x10]\n" "b.any 1b\n" ".inst 0xd503467f // SMSTOP\n" : : [args] "r" (&args), [ld_in_col] "r" (ld_in_col), [ld_in_row] "r" (ld_in_row), [offsetof_Args_bias] "I" (offsetof(Args, bias)), [offsetof_Args_clamp_max] "I" (offsetof(Args, clamp_max)), [offsetof_Args_clamp_min] "I" (offsetof(Args, clamp_min)), [offsetof_Args_current_channel] "I" (offsetof(Args, current_channel)), [offsetof_Args_inptr] "I" (offsetof(Args, inptr)), [offsetof_Args_input_cols] "I" (offsetof(Args, input_cols)), [offsetof_Args_ld_in_vl] "I" (offsetof(Args, ld_in_vl)), [offsetof_Args_ld_out_cols] "I" (offsetof(Args, ld_out_cols)), [offsetof_Args_ld_out_vls] "I" (offsetof(Args, ld_out_vls)), [offsetof_Args_n_channels] "I" (offsetof(Args, n_channels)), [offsetof_Args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_Args_output_cols] "I" (offsetof(Args, output_cols)), [offsetof_Args_pad_bottom] "I" (offsetof(Args, pad_bottom)), [offsetof_Args_pad_left] "I" (offsetof(Args, pad_left)), [offsetof_Args_pad_top] "I" (offsetof(Args, pad_top)), [offsetof_Args_weights] "I" (offsetof(Args, weights)) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_5x5_s2_4rows_mla_za.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_5x5_s2_4rows_mla_za.hpp index cae4b24e66f1f00527fb1cd00071d850a91ee9b6..7412c7b57ca77ef2089b2ad585b67f89941aefff 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_5x5_s2_4rows_mla_za.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_5x5_s2_4rows_mla_za.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,8 @@ * SOFTWARE. */ +#include "src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp" + namespace arm_conv { namespace depthwise { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_5x5_s2_4rows_mla_za/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_5x5_s2_4rows_mla_za/generic.cpp index 755265835df94251da22c717077938a797a7e9f1..81ad8e583360df6d0d3a3927297278b0ebfbf64f 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_5x5_s2_4rows_mla_za/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_5x5_s2_4rows_mla_za/generic.cpp @@ -76,11 +76,11 @@ void sme2_fp32_planar_5x5_s2_4rows_mla_za_impl( "ldr x6, [%x[args], %[offsetof_Args_pad_top]]\n" "ptrue p2.b\n" ".inst 0x25207812 // ptrue pn10.b\n" - "ld1rw { z0.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_min]]\n" + "ld1rw { z2.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_min]]\n" "ldr x7, [%x[args], %[offsetof_Args_n_channels]]\n" "whilelt p1.s, XZR, x7\n" "whilelt p9.s, XZR, x20\n" - "ld1rw { z17.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_max]]\n" + "ld1rw { z3.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_max]]\n" "whilelt p8.s, XZR, x6\n" "eor p8.b, p2/Z, p8.b, p9.b\n" "ldr x17, [%x[args], %[offsetof_Args_current_channel]]\n" @@ -99,64 +99,64 @@ void sme2_fp32_planar_5x5_s2_4rows_mla_za_impl( "orr x23, x7, x23, LSL #20\n" "mov x22, #0xb\n" "ldr x14, [%x[args], %[offsetof_Args_inptr]]\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xa04149ea // ld1w { z10.s-z11.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" "add x21, x6, x5\n" "lsl x20, %x[ld_in_row], #0x2\n" - "ld1w { z8.s }, p2/Z, [x15, #4, MUL VL]\n" + "ld1w { z9.s }, p2/Z, [x15, #4, MUL VL]\n" "addvl x15, x15, #5\n" "mov z30.d, z28.d\n" "mov z31.d, z28.d\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" + ".inst 0xa14049e7 // ld1w { z7.s, z15.s }, pn10.b/Z, [x15]\n" "mov x8, #0x0\n" "ldr x13, [%x[args], %[offsetof_Args_output_cols]]\n" "lsl x23, x23, #0x2\n" - ".inst 0xa14149e2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xa04149e0 // ld1w { z0.s-z1.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" "sub x22, x22, x21\n" "madd x20, x20, x6, x14\n" - "ld1w { z3.s }, p2/Z, [x15, #4, MUL VL]\n" + "ld1w { z6.s }, p2/Z, [x15, #4, MUL VL]\n" "addvl x15, x15, #5\n" "3:" // Issue prefetches "subs x22, x22, #0x1\n" ".inst 0xf8b74a9c // rprfm pldstrm, x23, [x20]\n" "add x20, x20, %x[ld_in_col], LSL #2\n" "bgt 3b\n" - "ldr x11, [%x[args], %[offsetof_Args_outptrs]]\n" + "ldr x23, [%x[args], %[offsetof_Args_outptrs]]\n" "lsl x20, %x[ld_in_row], #0x2\n" "msub x14, x6, x20, x14\n" ".inst 0xc0040f80 // mova za.d[x8, #0], { z28.d-z31.d }\n" "ldr x20, [%x[args], %[offsetof_Args_ld_out_cols]]\n" ".inst 0xc0040f81 // mova za.d[x8, #1], { z28.d-z31.d }\n" "mov x22, #0x4\n" - "ldp x10, x9, [x11], #0x10\n" + "ldp x11, x10, [x23], #0x10\n" ".inst 0xc0040f82 // mova za.d[x8, #2], { z28.d-z31.d }\n" - "ldp x28, x27, [x20], #0x10\n" + "ldp x9, x28, [x20], #0x10\n" ".inst 0xc0040f83 // mova za.d[x8, #3], { z28.d-z31.d }\n" "ldr x21, [%x[args], %[offsetof_Args_pad_left]]\n" ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" - "ldp x26, x25, [x11], #0x10\n" - "ldp x24, x23, [x20], #0x10\n" + "ldp x27, x26, [x23], #0x10\n" + "ldp x25, x24, [x20], #0x10\n" "cbz x21, 5f\n" "cmp x21, x22\n" "csel x20, x21, x22, LT\n" "sub x21, x21, x20\n" "sub x22, x22, x20\n" "cbz x21, 5f\n" - ".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n" + ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" "and x22, x21, #0x1\n" "add x21, x21, #0x1\n" - ".inst 0xc1b1c818 // fclamp { z24.s-z27.s }, z0.s, z17.s\n" + ".inst 0xc1a3c850 // fclamp { z16.s-z19.s }, z2.s, z3.s\n" "lsr x21, x21, #0x1\n" "sub x13, x13, x21\n" "4:" // Left padding "subs x21, x21, #0x1\n" - "st1w { z24.s }, p1, [x10]\n" + "st1w { z16.s }, p1, [x11]\n" + "add x11, x11, x9, LSL #2\n" + "st1w { z17.s }, p1, [x10]\n" "add x10, x10, x28, LSL #2\n" - "st1w { z25.s }, p1, [x9]\n" - "add x9, x9, x27, LSL #2\n" - "st1w { z26.s }, p1, [x26]\n" + "st1w { z18.s }, p1, [x27]\n" + "add x27, x27, x25, LSL #2\n" + "st1w { z19.s }, p1, [x26]\n" "add x26, x26, x24, LSL #2\n" - "st1w { z27.s }, p1, [x25]\n" - "add x25, x25, x23, LSL #2\n" "bgt 4b\n" "5:" // Left padding: End "adds XZR, x6, x5\n" @@ -171,331 +171,331 @@ void sme2_fp32_planar_5x5_s2_4rows_mla_za_impl( "beq 7f\n" "6:" // Unpadded: 4 priming loads "add x20, x14, %x[ld_in_row], LSL #2\n" - "ld1w { z11.s }, p1/Z, [x14]\n" + "ld1w { z9.s }, p1/Z, [x14]\n" "add x14, x14, %x[ld_in_col], LSL #2\n" + "ld1w { z20.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + "ld1w { z10.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" "ld1w { z21.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z12.s }, p1/Z, [x20]\n" + "ld1w { z11.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "ld1w { z22.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z13.s }, p1/Z, [x20]\n" + "ld1w { z12.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xc1341920 // fmla za.s[x8, 0], { z9.s-z12.s }, z4.s\n" "ld1w { z23.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z14.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1341960 // fmla za.s[x8, 0], { z11.s-z14.s }, z4.s\n" - "ld1w { z24.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1311aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z1.s\n" - "ld1w { z15.s }, p1/Z, [x20]\n" + ".inst 0xc1371a80 // fmla za.s[x8, 0], { z20.s-z23.s }, z7.s\n" + "ld1w { z13.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" "addvl x15, x15, #5\n" - ".inst 0xc1341980 // fmla za.s[x8, 0], { z12.s-z15.s }, z4.s\n" - "ld1w { z25.s }, p1/Z, [x20]\n" + ".inst 0xc1341940 // fmla za.s[x8, 0], { z10.s-z13.s }, z4.s\n" + "ld1w { z24.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" + ".inst 0xa04049e0 // ld1w { z0.s-z1.s }, pn10.b/Z, [x15]\n" "addvl x15, x15, #5\n" - ".inst 0xc1311ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z1.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + ".inst 0xc1301aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z0.s\n" + "ld1w { z14.s }, p1/Z, [x20]\n" ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xc13419a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z4.s\n" + ".inst 0xc1341960 // fmla za.s[x8, 0], { z11.s-z14.s }, z4.s\n" ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" "addvl x15, x15, #5\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" + ".inst 0xa14049e7 // ld1w { z7.s, z15.s }, pn10.b/Z, [x15]\n" "addvl x15, x15, #5\n" "7:" // Unpadded: 3 priming loads "add x20, x14, %x[ld_in_row], LSL #2\n" - "ld1w { z11.s }, p1/Z, [x14]\n" + "ld1w { z22.s }, p1/Z, [x14]\n" "add x14, x14, %x[ld_in_col], LSL #2\n" - "ld1w { z21.s }, p1/Z, [x20]\n" + "ld1w { z7.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z12.s }, p1/Z, [x20]\n" + "ld1w { z23.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z22.s }, p1/Z, [x20]\n" + "ld1w { z8.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z13.s }, p1/Z, [x20]\n" + "ld1w { z24.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z23.s }, p1/Z, [x20]\n" + "ld1w { z9.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z14.s }, p1/Z, [x20]\n" + "ld1w { z25.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1351960 // fmla za.s[x8, 0], { z11.s-z14.s }, z5.s\n" - "ld1w { z24.s }, p1/Z, [x20]\n" + ".inst 0xc1351ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z5.s\n" + "ld1w { z10.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1391aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z9.s\n" - "ld1w { z15.s }, p1/Z, [x20]\n" + ".inst 0xc13f18e0 // fmla za.s[x8, 0], { z7.s-z10.s }, z15.s\n" + "ld1w { z26.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" + ".inst 0xa04049ee // ld1w { z14.s-z15.s }, pn10.b/Z, [x15]\n" "addvl x15, x15, #5\n" - ".inst 0xc1351980 // fmla za.s[x8, 0], { z12.s-z15.s }, z5.s\n" - "ld1w { z25.s }, p1/Z, [x20]\n" + ".inst 0xc13f1ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z15.s\n" + "ld1w { z11.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" + ".inst 0xa04049e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15]\n" "addvl x15, x15, #5\n" - ".inst 0xc1391ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z9.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" + ".inst 0xc1371900 // fmla za.s[x8, 0], { z8.s-z11.s }, z7.s\n" + "ld1w { z27.s }, p1/Z, [x20]\n" + ".inst 0xa04049ea // ld1w { z10.s-z11.s }, pn10.b/Z, [x15]\n" "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xc13519a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z5.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xc13b1b00 // fmla za.s[x8, 0], { z24.s-z27.s }, z11.s\n" + ".inst 0xa14049e4 // ld1w { z4.s, z12.s }, pn10.b/Z, [x15]\n" + ".inst 0xa04149ea // ld1w { z10.s-z11.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" "addvl x15, x15, #5\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" - ".inst 0xa14149e2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xa14049e7 // ld1w { z7.s, z15.s }, pn10.b/Z, [x15]\n" + ".inst 0xa04149e0 // ld1w { z0.s-z1.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" "addvl x15, x15, #5\n" "8:" // Unpadded: 2 priming loads "add x20, x14, %x[ld_in_row], LSL #2\n" - "ld1w { z11.s }, p1/Z, [x14]\n" + "ld1w { z19.s }, p1/Z, [x14]\n" "add x14, x14, %x[ld_in_col], LSL #2\n" - "ld1w { z21.s }, p1/Z, [x20]\n" + "ld1w { z14.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z12.s }, p1/Z, [x20]\n" + "ld1w { z20.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z22.s }, p1/Z, [x20]\n" + "ld1w { z15.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z13.s }, p1/Z, [x20]\n" + "ld1w { z21.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z23.s }, p1/Z, [x20]\n" + "ld1w { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z14.s }, p1/Z, [x20]\n" + "ld1w { z22.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1361960 // fmla za.s[x8, 0], { z11.s-z14.s }, z6.s\n" - ".inst 0xc1341961 // fmla za.s[x8, 1], { z11.s-z14.s }, z4.s\n" - "ld1w { z24.s }, p1/Z, [x20]\n" + ".inst 0xc13a1a60 // fmla za.s[x8, 0], { z19.s-z22.s }, z10.s\n" + ".inst 0xc1341a61 // fmla za.s[x8, 1], { z19.s-z22.s }, z4.s\n" + "ld1w { z17.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1321aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z2.s\n" - "ld1w { z15.s }, p1/Z, [x20]\n" + ".inst 0xc13019c0 // fmla za.s[x8, 0], { z14.s-z17.s }, z0.s\n" + "ld1w { z23.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1311aa1 // fmla za.s[x8, 1], { z21.s-z24.s }, z1.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xc13719c1 // fmla za.s[x8, 1], { z14.s-z17.s }, z7.s\n" + ".inst 0xa04049e8 // ld1w { z8.s-z9.s }, pn10.b/Z, [x15]\n" + ".inst 0xa04149ea // ld1w { z10.s-z11.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" "addvl x15, x15, #5\n" - ".inst 0xc1361980 // fmla za.s[x8, 0], { z12.s-z15.s }, z6.s\n" - ".inst 0xc1341981 // fmla za.s[x8, 1], { z12.s-z15.s }, z4.s\n" - "ld1w { z25.s }, p1/Z, [x20]\n" + ".inst 0xc13a1a80 // fmla za.s[x8, 0], { z20.s-z23.s }, z10.s\n" + ".inst 0xc1381a81 // fmla za.s[x8, 1], { z20.s-z23.s }, z8.s\n" + "ld1w { z18.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" - ".inst 0xc1311ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z1.s\n" - ".inst 0xa14149e2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xa04049e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15]\n" + ".inst 0xc13619e1 // fmla za.s[x8, 1], { z15.s-z18.s }, z6.s\n" + ".inst 0xa04149e8 // ld1w { z8.s-z9.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" "addvl x15, x15, #5\n" - ".inst 0xc1321ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z2.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xc13419a1 // fmla za.s[x8, 1], { z13.s-z16.s }, z4.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xc13819e0 // fmla za.s[x8, 0], { z15.s-z18.s }, z8.s\n" + "ld1w { z24.s }, p1/Z, [x20]\n" + ".inst 0xa04049ee // ld1w { z14.s-z15.s }, pn10.b/Z, [x15]\n" + ".inst 0xc13e1aa1 // fmla za.s[x8, 1], { z21.s-z24.s }, z14.s\n" + ".inst 0xa14149e7 // ld1w { z7.s, z15.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xc13619a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z6.s\n" + ".inst 0xc1371aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z7.s\n" ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xa04149ea // ld1w { z10.s-z11.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" "addvl x15, x15, #5\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" - ".inst 0xa14149e2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xa04049ee // ld1w { z14.s-z15.s }, pn10.b/Z, [x15]\n" + ".inst 0xa04149e0 // ld1w { z0.s-z1.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" "addvl x15, x15, #5\n" "9:" // Unpadded: 1 priming loads "add x20, x14, %x[ld_in_row], LSL #2\n" - "ld1w { z11.s }, p1/Z, [x14]\n" + "ld1w { z7.s }, p1/Z, [x14]\n" "add x14, x14, %x[ld_in_col], LSL #2\n" - "ld1w { z21.s }, p1/Z, [x20]\n" + "ld1w { z18.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z12.s }, p1/Z, [x20]\n" + "ld1w { z8.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z22.s }, p1/Z, [x20]\n" + "ld1w { z19.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z13.s }, p1/Z, [x20]\n" + "ld1w { z9.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z23.s }, p1/Z, [x20]\n" + "ld1w { z20.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z14.s }, p1/Z, [x20]\n" + "ld1w { z10.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1371960 // fmla za.s[x8, 0], { z11.s-z14.s }, z7.s\n" - ".inst 0xc1351961 // fmla za.s[x8, 1], { z11.s-z14.s }, z5.s\n" - "ld1w { z24.s }, p1/Z, [x20]\n" + ".inst 0xc13b18e0 // fmla za.s[x8, 0], { z7.s-z10.s }, z11.s\n" + ".inst 0xc13518e1 // fmla za.s[x8, 1], { z7.s-z10.s }, z5.s\n" + "ld1w { z21.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc13a1aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z10.s\n" - "ld1w { z15.s }, p1/Z, [x20]\n" + ".inst 0xc1311a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z1.s\n" + "ld1w { z11.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1391aa1 // fmla za.s[x8, 1], { z21.s-z24.s }, z9.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xc13f1a41 // fmla za.s[x8, 1], { z18.s-z21.s }, z15.s\n" + ".inst 0xa04049e0 // ld1w { z0.s-z1.s }, pn10.b/Z, [x15]\n" + ".inst 0xa04149ec // ld1w { z12.s-z13.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" "addvl x15, x15, #5\n" - ".inst 0xc1371980 // fmla za.s[x8, 0], { z12.s-z15.s }, z7.s\n" - ".inst 0xc1351981 // fmla za.s[x8, 1], { z12.s-z15.s }, z5.s\n" - "ld1w { z25.s }, p1/Z, [x20]\n" + ".inst 0xc13d1900 // fmla za.s[x8, 0], { z8.s-z11.s }, z13.s\n" + ".inst 0xc1311901 // fmla za.s[x8, 1], { z8.s-z11.s }, z1.s\n" + "ld1w { z22.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" - ".inst 0xc1391ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z9.s\n" - ".inst 0xa14149e2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xa14049e6 // ld1w { z6.s, z14.s }, pn10.b/Z, [x15]\n" + ".inst 0xc13e1a61 // fmla za.s[x8, 1], { z19.s-z22.s }, z14.s\n" + ".inst 0xa14149e6 // ld1w { z6.s, z14.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" "addvl x15, x15, #5\n" - ".inst 0xc13a1ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z10.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xc13519a1 // fmla za.s[x8, 1], { z13.s-z16.s }, z5.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xc13e1a60 // fmla za.s[x8, 0], { z19.s-z22.s }, z14.s\n" + "ld1w { z12.s }, p1/Z, [x20]\n" + ".inst 0xa04049ee // ld1w { z14.s-z15.s }, pn10.b/Z, [x15]\n" + ".inst 0xc13f1921 // fmla za.s[x8, 1], { z9.s-z12.s }, z15.s\n" + ".inst 0xa04149ee // ld1w { z14.s-z15.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xc13719a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z7.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "ld1w { z8.s }, p2/Z, [x15, #4, MUL VL]\n" + ".inst 0xc13f1920 // fmla za.s[x8, 0], { z9.s-z12.s }, z15.s\n" + ".inst 0xa14049e4 // ld1w { z4.s, z12.s }, pn10.b/Z, [x15]\n" + ".inst 0xa04149ea // ld1w { z10.s-z11.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + "ld1w { z9.s }, p2/Z, [x15, #4, MUL VL]\n" "addvl x15, x15, #5\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" - ".inst 0xa14149e2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x15, #4, MUL VL]\n" + ".inst 0xa14049e7 // ld1w { z7.s, z15.s }, pn10.b/Z, [x15]\n" + ".inst 0xa04149e0 // ld1w { z0.s-z1.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + "ld1w { z6.s }, p2/Z, [x15, #4, MUL VL]\n" "addvl x15, x15, #5\n" "10:" // Unpadded: 0 priming loads "cmp x16, #0x2\n" "blt 20f\n" "add x21, x14, %x[ld_in_row], LSL #2\n" - "ld1w { z11.s }, p1/Z, [x14]\n" + "ld1w { z22.s }, p1/Z, [x14]\n" "sub x16, x16, #0x2\n" - "ld1w { z21.s }, p1/Z, [x21]\n" + "ld1w { z16.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" "sub x13, x13, #0x1\n" - "ld1w { z12.s }, p1/Z, [x21]\n" + "ld1w { z23.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" "lsr x20, x16, #0x1\n" - "ld1w { z22.s }, p1/Z, [x21]\n" + "ld1w { z17.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" "cmp x20, x13\n" - "ld1w { z13.s }, p1/Z, [x21]\n" + "ld1w { z24.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - "csel x22, x20, x13, LT\n" - "ld1w { z23.s }, p1/Z, [x21]\n" + "csel x23, x20, x13, LT\n" + "ld1w { z18.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" "add x14, x14, %x[ld_in_col], LSL #2\n" - "ld1w { z14.s }, p1/Z, [x21]\n" + "ld1w { z25.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" "and x16, x16, #0x1\n" - "ld1w { z24.s }, p1/Z, [x21]\n" + "ld1w { z19.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - "sub x13, x13, x22\n" - "ld1w { z15.s }, p1/Z, [x21]\n" + "sub x13, x13, x23\n" + "ld1w { z26.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - "ld1w { z25.s }, p1/Z, [x21]\n" + "ld1w { z20.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x21]\n" - "cbz x22, 19f\n" + "ld1w { z27.s }, p1/Z, [x21]\n" + "cbz x23, 19f\n" "11:" // Unpadded: Main loop - ".inst 0xc1381960 // fmla za.s[x8, 0], { z11.s-z14.s }, z8.s\n" - "ld1w { z8.s }, p2/Z, [x15, #4, MUL VL]\n" - "add x21, x14, %x[ld_in_row], LSL #2\n" - "subs x22, x22, #0x1\n" - ".inst 0xc1361961 // fmla za.s[x8, 1], { z11.s-z14.s }, z6.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc1341962 // fmla za.s[x8, 2], { z11.s-z14.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" + ".inst 0xc1391ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z9.s\n" + "ld1w { z13.s }, p2/Z, [x15, #4, MUL VL]\n" + "add x22, x14, %x[ld_in_row], LSL #2\n" + "subs x23, x23, #0x1\n" + ".inst 0xc13a1ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z10.s\n" + ".inst 0xa14149e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xc1341ac2 // fmla za.s[x8, 2], { z22.s-z25.s }, z4.s\n" + ".inst 0xa04049e8 // ld1w { z8.s-z9.s }, pn10.b/Z, [x15]\n" "addvl x15, x15, #5\n" - ".inst 0xc1331aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z3.s\n" - "ld1w { z3.s }, p2/Z, [x15, #4, MUL VL]\n" - ".inst 0xc1321aa1 // fmla za.s[x8, 1], { z21.s-z24.s }, z2.s\n" - ".inst 0xa14149e2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc1311aa2 // fmla za.s[x8, 2], { z21.s-z24.s }, z1.s\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" + ".inst 0xc1361a00 // fmla za.s[x8, 0], { z16.s-z19.s }, z6.s\n" + "ld1w { z11.s }, p2/Z, [x15, #4, MUL VL]\n" + ".inst 0xc1301a01 // fmla za.s[x8, 1], { z16.s-z19.s }, z0.s\n" + ".inst 0xa04149ee // ld1w { z14.s-z15.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xc1371a02 // fmla za.s[x8, 2], { z16.s-z19.s }, z7.s\n" + ".inst 0xa04049e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15]\n" "addvl x15, x15, #5\n" - ".inst 0xc1381980 // fmla za.s[x8, 0], { z12.s-z15.s }, z8.s\n" - "ld1w { z8.s }, p2/Z, [x15, #4, MUL VL]\n" - ".inst 0xc1361981 // fmla za.s[x8, 1], { z12.s-z15.s }, z6.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc1341982 // fmla za.s[x8, 2], { z12.s-z15.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xc1331ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z3.s\n" - "ld1w { z11.s }, p1/Z, [x14]\n" + ".inst 0xc13d1ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z13.s\n" + "ld1w { z4.s }, p2/Z, [x15, #4, MUL VL]\n" + ".inst 0xc1311ae1 // fmla za.s[x8, 1], { z23.s-z26.s }, z1.s\n" + ".inst 0xa04149e0 // ld1w { z0.s-z1.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xc1381ae2 // fmla za.s[x8, 2], { z23.s-z26.s }, z8.s\n" + ".inst 0xa04049ec // ld1w { z12.s-z13.s }, pn10.b/Z, [x15]\n" + "ldr x21, [%x[args], %[offsetof_Args_weights]]\n" + ".inst 0xc13b1a20 // fmla za.s[x8, 0], { z17.s-z20.s }, z11.s\n" + "ld1w { z15.s }, p1/Z, [x14]\n" "add x14, x14, %x[ld_in_col], LSL #2\n" "add x20, x14, %x[ld_in_row], LSL #2\n" - ".inst 0xc1321ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z2.s\n" - "ld1w { z21.s }, p1/Z, [x21]\n" - "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0xc1311ac2 // fmla za.s[x8, 2], { z22.s-z25.s }, z1.s\n" - "ld1w { z12.s }, p1/Z, [x21]\n" - "add x21, x21, %x[ld_in_row], LSL #2\n" - "ld1w { z22.s }, p1/Z, [x21]\n" - "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0xc13819a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z8.s\n" - ".inst 0xc13619a1 // fmla za.s[x8, 1], { z13.s-z16.s }, z6.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc13419a2 // fmla za.s[x8, 2], { z13.s-z16.s }, z4.s\n" - "ld1w { z13.s }, p1/Z, [x21]\n" - "add x21, x21, %x[ld_in_row], LSL #2\n" - "ld1w { z23.s }, p1/Z, [x21]\n" - "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n" + ".inst 0xc13e1a21 // fmla za.s[x8, 1], { z17.s-z20.s }, z14.s\n" + "ld1w { z22.s }, p1/Z, [x22]\n" + "add x22, x22, %x[ld_in_row], LSL #2\n" + ".inst 0xc1361a22 // fmla za.s[x8, 2], { z17.s-z20.s }, z6.s\n" + "ld1w { z16.s }, p1/Z, [x22]\n" + "add x22, x22, %x[ld_in_row], LSL #2\n" + "ld1w { z23.s }, p1/Z, [x22]\n" + "add x22, x22, %x[ld_in_row], LSL #2\n" + ".inst 0xc1341b00 // fmla za.s[x8, 0], { z24.s-z27.s }, z4.s\n" + ".inst 0xc1301b01 // fmla za.s[x8, 1], { z24.s-z27.s }, z0.s\n" + ".inst 0xa0414aa6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc13c1b02 // fmla za.s[x8, 2], { z24.s-z27.s }, z12.s\n" + "ld1w { z17.s }, p1/Z, [x22]\n" + "add x22, x22, %x[ld_in_row], LSL #2\n" + "ld1w { z24.s }, p1/Z, [x22]\n" + "add x22, x22, %x[ld_in_row], LSL #2\n" + ".inst 0xc0060c08 // mova { z8.d-z11.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - ".inst 0xc1b1c818 // fclamp { z24.s-z27.s }, z0.s, z17.s\n" - "st1w { z24.s }, p1, [x10]\n" - "ld1w { z14.s }, p1/Z, [x21]\n" - "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0xc1371960 // fmla za.s[x8, 0], { z11.s-z14.s }, z7.s\n" + ".inst 0xa1404aa4 // ld1w { z4.s, z12.s }, pn10.b/Z, [x21]\n" + "addvl x21, x21, #5\n" + ".inst 0xc1a3c848 // fclamp { z8.s-z11.s }, z2.s, z3.s\n" + "st1w { z8.s }, p1, [x11]\n" + "ld1w { z18.s }, p1/Z, [x22]\n" + "add x22, x22, %x[ld_in_row], LSL #2\n" + ".inst 0xc13719e0 // fmla za.s[x8, 0], { z15.s-z18.s }, z7.s\n" + "add x11, x11, x9, LSL #2\n" + ".inst 0xc13c19e1 // fmla za.s[x8, 1], { z15.s-z18.s }, z12.s\n" + ".inst 0xa1404aa7 // ld1w { z7.s, z15.s }, pn10.b/Z, [x21]\n" + "st1w { z9.s }, p1, [x10]\n" "add x10, x10, x28, LSL #2\n" - ".inst 0xc1351961 // fmla za.s[x8, 1], { z11.s-z14.s }, z5.s\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" - "st1w { z25.s }, p1, [x9]\n" - "add x9, x9, x27, LSL #2\n" - ".inst 0xa14149e2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "addvl x15, x15, #5\n" - "st1w { z26.s }, p1, [x26]\n" + ".inst 0xa1414aa6 // ld1w { z6.s, z14.s }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + "addvl x21, x21, #5\n" + "st1w { z10.s }, p1, [x27]\n" + "add x27, x27, x25, LSL #2\n" + "ld1w { z25.s }, p1/Z, [x22]\n" + "add x22, x22, %x[ld_in_row], LSL #2\n" + ".inst 0xc13e1ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z14.s\n" + "st1w { z11.s }, p1, [x26]\n" + ".inst 0xc13f1ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z15.s\n" + "ld1w { z19.s }, p1/Z, [x22]\n" + "add x22, x22, %x[ld_in_row], LSL #2\n" "add x26, x26, x24, LSL #2\n" - "ld1w { z24.s }, p1/Z, [x21]\n" - "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0xc13a1aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z10.s\n" - "st1w { z27.s }, p1, [x25]\n" - ".inst 0xc1391aa1 // fmla za.s[x8, 1], { z21.s-z24.s }, z9.s\n" - "ld1w { z15.s }, p1/Z, [x21]\n" - "add x21, x21, %x[ld_in_row], LSL #2\n" - "add x25, x25, x23, LSL #2\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xc1351981 // fmla za.s[x8, 1], { z12.s-z15.s }, z5.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "addvl x15, x15, #5\n" - ".inst 0xc1371980 // fmla za.s[x8, 0], { z12.s-z15.s }, z7.s\n" - "ld1w { z25.s }, p1/Z, [x21]\n" - "add x21, x21, %x[ld_in_row], LSL #2\n" + ".inst 0xa0404aae // ld1w { z14.s-z15.s }, pn10.b/Z, [x21]\n" + ".inst 0xc13f1a01 // fmla za.s[x8, 1], { z16.s-z19.s }, z15.s\n" + ".inst 0xa1414aa4 // ld1w { z4.s, z12.s }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + "addvl x21, x21, #5\n" + ".inst 0xc13c1a00 // fmla za.s[x8, 0], { z16.s-z19.s }, z12.s\n" + "ld1w { z26.s }, p1/Z, [x22]\n" + "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" - ".inst 0xc1391ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z9.s\n" - ".inst 0xa14149e2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "addvl x15, x15, #5\n" - ".inst 0xc13a1ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z10.s\n" - "ld1w { z16.s }, p1/Z, [x21]\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xc13519a1 // fmla za.s[x8, 1], { z13.s-z16.s }, z5.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc13719a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z7.s\n" + ".inst 0xa0404aac // ld1w { z12.s-z13.s }, pn10.b/Z, [x21]\n" + ".inst 0xc13d1ae1 // fmla za.s[x8, 1], { z23.s-z26.s }, z13.s\n" + ".inst 0xa1414aa4 // ld1w { z4.s, z12.s }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + "addvl x21, x21, #5\n" + ".inst 0xc13c1ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z12.s\n" + "ld1w { z20.s }, p1/Z, [x22]\n" + ".inst 0xa1404aa7 // ld1w { z7.s, z15.s }, pn10.b/Z, [x21]\n" + ".inst 0xc13f1a21 // fmla za.s[x8, 1], { z17.s-z20.s }, z15.s\n" + ".inst 0xa0414aaa // ld1w { z10.s-z11.s }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc13b1a20 // fmla za.s[x8, 0], { z17.s-z20.s }, z11.s\n" "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - "ld1w { z11.s }, p1/Z, [x14]\n" + "ld1w { z22.s }, p1/Z, [x14]\n" "add x14, x14, %x[ld_in_col], LSL #2\n" - "ld1w { z21.s }, p1/Z, [x20]\n" + "ld1w { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z12.s }, p1/Z, [x20]\n" + "ld1w { z23.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z22.s }, p1/Z, [x20]\n" + "ld1w { z17.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z13.s }, p1/Z, [x20]\n" + "ld1w { z24.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z23.s }, p1/Z, [x20]\n" + "ld1w { z18.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z14.s }, p1/Z, [x20]\n" + "ld1w { z25.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z24.s }, p1/Z, [x20]\n" + "ld1w { z19.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z15.s }, p1/Z, [x20]\n" + "ld1w { z26.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "ld1w { z8.s }, p2/Z, [x15, #4, MUL VL]\n" + ".inst 0xa04149ea // ld1w { z10.s-z11.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + "ld1w { z9.s }, p2/Z, [x15, #4, MUL VL]\n" "addvl x15, x15, #5\n" - "ld1w { z25.s }, p1/Z, [x20]\n" + "ld1w { z20.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" - ".inst 0xa14149e2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x15, #4, MUL VL]\n" + ".inst 0xa14049e7 // ld1w { z7.s, z15.s }, pn10.b/Z, [x15]\n" + ".inst 0xa04149e0 // ld1w { z0.s-z1.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + "ld1w { z6.s }, p2/Z, [x15, #4, MUL VL]\n" "addvl x15, x15, #5\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + "ld1w { z27.s }, p1/Z, [x20]\n" "bgt 11b\n" "b 19f\n" "12:" // Padded @@ -510,654 +510,654 @@ void sme2_fp32_planar_5x5_s2_4rows_mla_za_impl( "13:" // Padded: 4 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z11.s }, p0/Z, [x14]\n" + "ld1w { z9.s }, p0/Z, [x14]\n" "add x20, x14, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z21.s }, p0/Z, [x20]\n" + "ld1w { z23.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z12.s }, p0/Z, [x20]\n" + "ld1w { z10.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z22.s }, p0/Z, [x20]\n" + "ld1w { z24.s }, p0/Z, [x20]\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z13.s }, p0/Z, [x20]\n" + "ld1w { z11.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z23.s }, p0/Z, [x20]\n" + "ld1w { z25.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z14.s }, p0/Z, [x20]\n" + "ld1w { z12.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x8\n" - ".inst 0xc1341960 // fmla za.s[x8, 0], { z11.s-z14.s }, z4.s\n" - "ld1w { z24.s }, p0/Z, [x20]\n" + ".inst 0xc1341920 // fmla za.s[x8, 0], { z9.s-z12.s }, z4.s\n" + "ld1w { z26.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1311aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z1.s\n" - "ld1w { z15.s }, p0/Z, [x20]\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" + ".inst 0xc1371ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z7.s\n" + "ld1w { z13.s }, p0/Z, [x20]\n" + ".inst 0xa04049e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0xc1341980 // fmla za.s[x8, 0], { z12.s-z15.s }, z4.s\n" + ".inst 0xc1361940 // fmla za.s[x8, 0], { z10.s-z13.s }, z6.s\n" "addvl x15, x15, #5\n" - "ld1w { z25.s }, p0/Z, [x20]\n" + "ld1w { z27.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" + ".inst 0xa14049e6 // ld1w { z6.s, z14.s }, pn10.b/Z, [x15]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "addvl x15, x15, #5\n" - ".inst 0xc1311ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z1.s\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + ".inst 0xc1361b00 // fmla za.s[x8, 0], { z24.s-z27.s }, z6.s\n" + "ld1w { z14.s }, p0/Z, [x20]\n" "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" + ".inst 0xa04049e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15]\n" "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xc13419a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z4.s\n" + ".inst 0xc1361960 // fmla za.s[x8, 0], { z11.s-z14.s }, z6.s\n" ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" "addvl x15, x15, #5\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" + ".inst 0xa14049e7 // ld1w { z7.s, z15.s }, pn10.b/Z, [x15]\n" "addvl x15, x15, #5\n" "14:" // Padded: 3 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z11.s }, p0/Z, [x14]\n" + "ld1w { z22.s }, p0/Z, [x14]\n" "add x20, x14, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z21.s }, p0/Z, [x20]\n" + "ld1w { z9.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z12.s }, p0/Z, [x20]\n" + "ld1w { z23.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z22.s }, p0/Z, [x20]\n" + "ld1w { z10.s }, p0/Z, [x20]\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z13.s }, p0/Z, [x20]\n" + "ld1w { z24.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z23.s }, p0/Z, [x20]\n" + "ld1w { z11.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z14.s }, p0/Z, [x20]\n" + "ld1w { z25.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x8\n" - ".inst 0xc1351960 // fmla za.s[x8, 0], { z11.s-z14.s }, z5.s\n" - "ld1w { z24.s }, p0/Z, [x20]\n" + ".inst 0xc1351ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z5.s\n" + "ld1w { z12.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1391aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z9.s\n" - "ld1w { z15.s }, p0/Z, [x20]\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" + ".inst 0xc13f1920 // fmla za.s[x8, 0], { z9.s-z12.s }, z15.s\n" + "ld1w { z26.s }, p0/Z, [x20]\n" + ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0xc1351980 // fmla za.s[x8, 0], { z12.s-z15.s }, z5.s\n" + ".inst 0xc1381ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z8.s\n" "addvl x15, x15, #5\n" - "ld1w { z25.s }, p0/Z, [x20]\n" + "ld1w { z13.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" + ".inst 0xa14049e7 // ld1w { z7.s, z15.s }, pn10.b/Z, [x15]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "addvl x15, x15, #5\n" - ".inst 0xc1391ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z9.s\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + ".inst 0xc13f1940 // fmla za.s[x8, 0], { z10.s-z13.s }, z15.s\n" + "ld1w { z27.s }, p0/Z, [x20]\n" "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" + ".inst 0xa04049ee // ld1w { z14.s-z15.s }, pn10.b/Z, [x15]\n" "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xc13519a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z5.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xc13f1b00 // fmla za.s[x8, 0], { z24.s-z27.s }, z15.s\n" + ".inst 0xa14049e4 // ld1w { z4.s, z12.s }, pn10.b/Z, [x15]\n" + ".inst 0xa04149ea // ld1w { z10.s-z11.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" "addvl x15, x15, #5\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" - ".inst 0xa14149e2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xa14049e7 // ld1w { z7.s, z15.s }, pn10.b/Z, [x15]\n" + ".inst 0xa04149e0 // ld1w { z0.s-z1.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" "addvl x15, x15, #5\n" "15:" // Padded: 2 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z11.s }, p0/Z, [x14]\n" + "ld1w { z16.s }, p0/Z, [x14]\n" "add x20, x14, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z21.s }, p0/Z, [x20]\n" + "ld1w { z23.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z12.s }, p0/Z, [x20]\n" + "ld1w { z17.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z22.s }, p0/Z, [x20]\n" + "ld1w { z24.s }, p0/Z, [x20]\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z13.s }, p0/Z, [x20]\n" + "ld1w { z18.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z23.s }, p0/Z, [x20]\n" + "ld1w { z25.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z14.s }, p0/Z, [x20]\n" + "ld1w { z19.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x8\n" - ".inst 0xc1361960 // fmla za.s[x8, 0], { z11.s-z14.s }, z6.s\n" - "ld1w { z24.s }, p0/Z, [x20]\n" - ".inst 0xc1341961 // fmla za.s[x8, 1], { z11.s-z14.s }, z4.s\n" + ".inst 0xc13a1a00 // fmla za.s[x8, 0], { z16.s-z19.s }, z10.s\n" + "ld1w { z26.s }, p0/Z, [x20]\n" + ".inst 0xc1341a01 // fmla za.s[x8, 1], { z16.s-z19.s }, z4.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z15.s }, p0/Z, [x20]\n" - ".inst 0xc1321aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z2.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" + "ld1w { z20.s }, p0/Z, [x20]\n" + ".inst 0xc1301ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z0.s\n" + ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0xc1311aa1 // fmla za.s[x8, 1], { z21.s-z24.s }, z1.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xc1371ae1 // fmla za.s[x8, 1], { z23.s-z26.s }, z7.s\n" + ".inst 0xa14149e7 // ld1w { z7.s, z15.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" "addvl x15, x15, #5\n" "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xc1361980 // fmla za.s[x8, 0], { z12.s-z15.s }, z6.s\n" - "ld1w { z25.s }, p0/Z, [x20]\n" + ".inst 0xc1371a20 // fmla za.s[x8, 0], { z17.s-z20.s }, z7.s\n" + "ld1w { z27.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc1341981 // fmla za.s[x8, 1], { z12.s-z15.s }, z4.s\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" - ".inst 0xa14149e2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xc1301a21 // fmla za.s[x8, 1], { z17.s-z20.s }, z0.s\n" + ".inst 0xa14049e5 // ld1w { z5.s, z13.s }, pn10.b/Z, [x15]\n" + ".inst 0xa04149ea // ld1w { z10.s-z11.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" "addvl x15, x15, #5\n" - ".inst 0xc1321ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z2.s\n" - ".inst 0xc1311ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z1.s\n" - "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xc13419a1 // fmla za.s[x8, 1], { z13.s-z16.s }, z4.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xc13a1b00 // fmla za.s[x8, 0], { z24.s-z27.s }, z10.s\n" + ".inst 0xc1351b01 // fmla za.s[x8, 1], { z24.s-z27.s }, z5.s\n" + "ld1w { z21.s }, p0/Z, [x20]\n" + ".inst 0xa14049e5 // ld1w { z5.s, z13.s }, pn10.b/Z, [x15]\n" + ".inst 0xc1351a41 // fmla za.s[x8, 1], { z18.s-z21.s }, z5.s\n" + ".inst 0xa04149e0 // ld1w { z0.s-z1.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xc13619a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z6.s\n" + ".inst 0xc1301a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z0.s\n" ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xa04149ea // ld1w { z10.s-z11.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" "addvl x15, x15, #5\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" - ".inst 0xa14149e2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xa14049e7 // ld1w { z7.s, z15.s }, pn10.b/Z, [x15]\n" + ".inst 0xa04149e0 // ld1w { z0.s-z1.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" "addvl x15, x15, #5\n" "16:" // Padded: 1 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z11.s }, p0/Z, [x14]\n" + "ld1w { z19.s }, p0/Z, [x14]\n" "add x20, x14, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z21.s }, p0/Z, [x20]\n" + "ld1w { z8.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z12.s }, p0/Z, [x20]\n" + "ld1w { z20.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z22.s }, p0/Z, [x20]\n" + "ld1w { z9.s }, p0/Z, [x20]\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z13.s }, p0/Z, [x20]\n" + "ld1w { z21.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z23.s }, p0/Z, [x20]\n" + "ld1w { z10.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z14.s }, p0/Z, [x20]\n" + "ld1w { z22.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x8\n" - ".inst 0xc1371960 // fmla za.s[x8, 0], { z11.s-z14.s }, z7.s\n" - "ld1w { z24.s }, p0/Z, [x20]\n" - ".inst 0xc1351961 // fmla za.s[x8, 1], { z11.s-z14.s }, z5.s\n" + ".inst 0xc13b1a60 // fmla za.s[x8, 0], { z19.s-z22.s }, z11.s\n" + "ld1w { z11.s }, p0/Z, [x20]\n" + ".inst 0xc1351a61 // fmla za.s[x8, 1], { z19.s-z22.s }, z5.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z15.s }, p0/Z, [x20]\n" - ".inst 0xc13a1aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z10.s\n" + "ld1w { z23.s }, p0/Z, [x20]\n" + ".inst 0xc1311900 // fmla za.s[x8, 0], { z8.s-z11.s }, z1.s\n" ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0xc1391aa1 // fmla za.s[x8, 1], { z21.s-z24.s }, z9.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xc13f1901 // fmla za.s[x8, 1], { z8.s-z11.s }, z15.s\n" + ".inst 0xa14149e6 // ld1w { z6.s, z14.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" "addvl x15, x15, #5\n" "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xc1371980 // fmla za.s[x8, 0], { z12.s-z15.s }, z7.s\n" - "ld1w { z25.s }, p0/Z, [x20]\n" + ".inst 0xc13e1a80 // fmla za.s[x8, 0], { z20.s-z23.s }, z14.s\n" + "ld1w { z12.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc1351981 // fmla za.s[x8, 1], { z12.s-z15.s }, z5.s\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" - ".inst 0xa14149e2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xc1351a81 // fmla za.s[x8, 1], { z20.s-z23.s }, z5.s\n" + ".inst 0xa04049e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15]\n" + ".inst 0xa14149e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" "addvl x15, x15, #5\n" - ".inst 0xc13a1ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z10.s\n" - ".inst 0xc1391ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z9.s\n" - "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xc13519a1 // fmla za.s[x8, 1], { z13.s-z16.s }, z5.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xc1381920 // fmla za.s[x8, 0], { z9.s-z12.s }, z8.s\n" + ".inst 0xc1371921 // fmla za.s[x8, 1], { z9.s-z12.s }, z7.s\n" + "ld1w { z24.s }, p0/Z, [x20]\n" + ".inst 0xa14049e0 // ld1w { z0.s, z8.s }, pn10.b/Z, [x15]\n" + ".inst 0xc1381aa1 // fmla za.s[x8, 1], { z21.s-z24.s }, z8.s\n" + ".inst 0xa04149ec // ld1w { z12.s-z13.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xc13719a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z7.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "ld1w { z8.s }, p2/Z, [x15, #4, MUL VL]\n" + ".inst 0xc13d1aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z13.s\n" + ".inst 0xa14049e4 // ld1w { z4.s, z12.s }, pn10.b/Z, [x15]\n" + ".inst 0xa04149ea // ld1w { z10.s-z11.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + "ld1w { z9.s }, p2/Z, [x15, #4, MUL VL]\n" "addvl x15, x15, #5\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" - ".inst 0xa14149e2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x15, #4, MUL VL]\n" + ".inst 0xa14049e7 // ld1w { z7.s, z15.s }, pn10.b/Z, [x15]\n" + ".inst 0xa04149e0 // ld1w { z0.s-z1.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + "ld1w { z6.s }, p2/Z, [x15, #4, MUL VL]\n" "addvl x15, x15, #5\n" "17:" // Padded: 0 priming loads "cmp x16, #0x2\n" "blt 20f\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z11.s }, p0/Z, [x14]\n" + "ld1w { z22.s }, p0/Z, [x14]\n" "add x21, x14, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z21.s }, p0/Z, [x21]\n" + "ld1w { z16.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z12.s }, p0/Z, [x21]\n" + "ld1w { z23.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z22.s }, p0/Z, [x21]\n" + "ld1w { z17.s }, p0/Z, [x21]\n" "mov x12, #0x4\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z13.s }, p0/Z, [x21]\n" + "ld1w { z24.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z23.s }, p0/Z, [x21]\n" + "ld1w { z18.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z14.s }, p0/Z, [x21]\n" + "ld1w { z25.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x8\n" - "ld1w { z24.s }, p0/Z, [x21]\n" + "ld1w { z19.s }, p0/Z, [x21]\n" "sub x16, x16, #0x2\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "sub x13, x13, #0x1\n" - "ld1w { z15.s }, p0/Z, [x21]\n" + "ld1w { z26.s }, p0/Z, [x21]\n" "lsr x20, x16, #0x1\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "cmp x20, x13\n" - "ld1w { z25.s }, p0/Z, [x21]\n" + "ld1w { z20.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x21]\n" - "csel x22, x20, x13, LT\n" + "ld1w { z27.s }, p0/Z, [x21]\n" + "csel x23, x20, x13, LT\n" "add x14, x14, %x[ld_in_col], LSL #2\n" "and x16, x16, #0x1\n" - "sub x13, x13, x22\n" - "cbz x22, 19f\n" + "sub x13, x13, x23\n" + "cbz x23, 19f\n" "18:" // Padded: Main loop - ".inst 0xc1381960 // fmla za.s[x8, 0], { z11.s-z14.s }, z8.s\n" - "ld1w { z8.s }, p2/Z, [x15, #4, MUL VL]\n" + ".inst 0xc1391ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z9.s\n" + "ld1w { z15.s }, p2/Z, [x15, #4, MUL VL]\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1361961 // fmla za.s[x8, 1], { z11.s-z14.s }, z6.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "add x21, x14, %x[ld_in_row], LSL #2\n" - "subs x22, x22, #0x1\n" - ".inst 0xc1341962 // fmla za.s[x8, 2], { z11.s-z14.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" + ".inst 0xc13a1ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z10.s\n" + ".inst 0xa04149ea // ld1w { z10.s-z11.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + "add x22, x14, %x[ld_in_row], LSL #2\n" + "subs x23, x23, #0x1\n" + ".inst 0xc1341ac2 // fmla za.s[x8, 2], { z22.s-z25.s }, z4.s\n" + ".inst 0xa14049e5 // ld1w { z5.s, z13.s }, pn10.b/Z, [x15]\n" "addvl x15, x15, #5\n" - ".inst 0xc1331aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z3.s\n" - "ld1w { z3.s }, p2/Z, [x15, #4, MUL VL]\n" - ".inst 0xc1321aa1 // fmla za.s[x8, 1], { z21.s-z24.s }, z2.s\n" - ".inst 0xa14149e2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc1311aa2 // fmla za.s[x8, 2], { z21.s-z24.s }, z1.s\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" + ".inst 0xc1361a00 // fmla za.s[x8, 0], { z16.s-z19.s }, z6.s\n" + "ld1w { z1.s }, p2/Z, [x15, #4, MUL VL]\n" + ".inst 0xc1301a01 // fmla za.s[x8, 1], { z16.s-z19.s }, z0.s\n" + ".inst 0xa04149ec // ld1w { z12.s-z13.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xc1371a02 // fmla za.s[x8, 2], { z16.s-z19.s }, z7.s\n" + ".inst 0xa14049e6 // ld1w { z6.s, z14.s }, pn10.b/Z, [x15]\n" "addvl x15, x15, #5\n" - ".inst 0xc1381980 // fmla za.s[x8, 0], { z12.s-z15.s }, z8.s\n" - "ld1w { z11.s }, p0/Z, [x14]\n" + ".inst 0xc13f1ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z15.s\n" + "ld1w { z16.s }, p0/Z, [x14]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xc1361981 // fmla za.s[x8, 1], { z12.s-z15.s }, z6.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" - ".inst 0xc1341982 // fmla za.s[x8, 2], { z12.s-z15.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xc1331ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z3.s\n" - "ld1w { z8.s }, p2/Z, [x15, #4, MUL VL]\n" - "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xc1321ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z2.s\n" - "ld1w { z21.s }, p0/Z, [x21]\n" - "add x21, x21, %x[ld_in_row], LSL #2\n" + ".inst 0xc13a1ae1 // fmla za.s[x8, 1], { z23.s-z26.s }, z10.s\n" + ".inst 0xa04149ea // ld1w { z10.s-z11.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + "add x21, x14, %x[ld_in_row], LSL #2\n" + ".inst 0xc1351ae2 // fmla za.s[x8, 2], { z23.s-z26.s }, z5.s\n" + ".inst 0xa04049ee // ld1w { z14.s-z15.s }, pn10.b/Z, [x15]\n" + ".inst 0xc1311a20 // fmla za.s[x8, 0], { z17.s-z20.s }, z1.s\n" + "ld1w { z0.s }, p2/Z, [x15, #4, MUL VL]\n" + "ldr x20, [%x[args], %[offsetof_Args_weights]]\n" + ".inst 0xc13c1a21 // fmla za.s[x8, 1], { z17.s-z20.s }, z12.s\n" + "ld1w { z12.s }, p0/Z, [x22]\n" + "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc1311ac2 // fmla za.s[x8, 2], { z22.s-z25.s }, z1.s\n" - "ld1w { z12.s }, p0/Z, [x21]\n" - "add x21, x21, %x[ld_in_row], LSL #2\n" + ".inst 0xc1361a22 // fmla za.s[x8, 2], { z17.s-z20.s }, z6.s\n" + "ld1w { z17.s }, p0/Z, [x22]\n" + "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x4\n" - "ld1w { z22.s }, p0/Z, [x21]\n" - "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0xc13819a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z8.s\n" + "ld1w { z13.s }, p0/Z, [x22]\n" + "add x22, x22, %x[ld_in_row], LSL #2\n" + ".inst 0xc1301b00 // fmla za.s[x8, 0], { z24.s-z27.s }, z0.s\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc13619a1 // fmla za.s[x8, 1], { z13.s-z16.s }, z6.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc13419a2 // fmla za.s[x8, 2], { z13.s-z16.s }, z4.s\n" - "ld1w { z13.s }, p0/Z, [x21]\n" - "add x21, x21, %x[ld_in_row], LSL #2\n" + ".inst 0xc13a1b01 // fmla za.s[x8, 1], { z24.s-z27.s }, z10.s\n" + ".inst 0xa1414a81 // ld1w { z1.s, z9.s }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc13e1b02 // fmla za.s[x8, 2], { z24.s-z27.s }, z14.s\n" + "ld1w { z18.s }, p0/Z, [x22]\n" + "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z23.s }, p0/Z, [x21]\n" - "add x21, x21, %x[ld_in_row], LSL #2\n" + "ld1w { z14.s }, p0/Z, [x22]\n" + "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" ".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" + ".inst 0xa0404a80 // ld1w { z0.s-z1.s }, pn10.b/Z, [x20]\n" "add x8, x8, #0x1\n" - "addvl x15, x15, #5\n" - ".inst 0xc1b1c818 // fclamp { z24.s-z27.s }, z0.s, z17.s\n" - "ld1w { z14.s }, p0/Z, [x21]\n" - "add x21, x21, %x[ld_in_row], LSL #2\n" + "addvl x20, x20, #5\n" + ".inst 0xc1a3c858 // fclamp { z24.s-z27.s }, z2.s, z3.s\n" + "ld1w { z19.s }, p0/Z, [x22]\n" + "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "st1w { z24.s }, p1, [x10]\n" + "st1w { z24.s }, p1, [x11]\n" "mov x12, #0x8\n" - ".inst 0xc1371960 // fmla za.s[x8, 0], { z11.s-z14.s }, z7.s\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" - "add x10, x10, x28, LSL #2\n" - ".inst 0xc1351961 // fmla za.s[x8, 1], { z11.s-z14.s }, z5.s\n" - ".inst 0xa14149e2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "addvl x15, x15, #5\n" - "st1w { z25.s }, p1, [x9]\n" - "ld1w { z24.s }, p0/Z, [x21]\n" - "add x21, x21, %x[ld_in_row], LSL #2\n" + ".inst 0xc1391a00 // fmla za.s[x8, 0], { z16.s-z19.s }, z9.s\n" + ".inst 0xa0404a88 // ld1w { z8.s-z9.s }, pn10.b/Z, [x20]\n" + "add x11, x11, x9, LSL #2\n" + ".inst 0xc1311a01 // fmla za.s[x8, 1], { z16.s-z19.s }, z1.s\n" + ".inst 0xa0414a80 // ld1w { z0.s-z1.s }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + "addvl x20, x20, #5\n" + "st1w { z25.s }, p1, [x10]\n" + "ld1w { z15.s }, p0/Z, [x22]\n" + "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc13a1aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z10.s\n" - ".inst 0xc1391aa1 // fmla za.s[x8, 1], { z21.s-z24.s }, z9.s\n" - "ld1w { z15.s }, p0/Z, [x21]\n" - "add x21, x21, %x[ld_in_row], LSL #2\n" + ".inst 0xc1311980 // fmla za.s[x8, 0], { z12.s-z15.s }, z1.s\n" + ".inst 0xc1391981 // fmla za.s[x8, 1], { z12.s-z15.s }, z9.s\n" + "ld1w { z20.s }, p0/Z, [x22]\n" + "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xc1351981 // fmla za.s[x8, 1], { z12.s-z15.s }, z5.s\n" - "add x9, x9, x27, LSL #2\n" - "st1w { z26.s }, p1, [x26]\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "addvl x15, x15, #5\n" - ".inst 0xc1371980 // fmla za.s[x8, 0], { z12.s-z15.s }, z7.s\n" - "add x26, x26, x24, LSL #2\n" - "ld1w { z25.s }, p0/Z, [x21]\n" - "add x21, x21, %x[ld_in_row], LSL #2\n" + ".inst 0xa0404a8a // ld1w { z10.s-z11.s }, pn10.b/Z, [x20]\n" + ".inst 0xc13b1a21 // fmla za.s[x8, 1], { z17.s-z20.s }, z11.s\n" + "add x10, x10, x28, LSL #2\n" + "st1w { z26.s }, p1, [x27]\n" + ".inst 0xa1414a80 // ld1w { z0.s, z8.s }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + "addvl x20, x20, #5\n" + ".inst 0xc1381a20 // fmla za.s[x8, 0], { z17.s-z20.s }, z8.s\n" + "add x27, x27, x25, LSL #2\n" + "ld1w { z16.s }, p0/Z, [x22]\n" + "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "st1w { z27.s }, p1, [x25]\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" + "st1w { z27.s }, p1, [x26]\n" + ".inst 0xa0404a88 // ld1w { z8.s-z9.s }, pn10.b/Z, [x20]\n" "mov x12, #0x0\n" - ".inst 0xc1391ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z9.s\n" - "add x25, x25, x23, LSL #2\n" - ".inst 0xa14149e2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "addvl x15, x15, #5\n" - ".inst 0xc13a1ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z10.s\n" - "ld1w { z16.s }, p0/Z, [x21]\n" + ".inst 0xc13919a1 // fmla za.s[x8, 1], { z13.s-z16.s }, z9.s\n" + "add x26, x26, x24, LSL #2\n" + ".inst 0xa1414a81 // ld1w { z1.s, z9.s }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + "addvl x20, x20, #5\n" + ".inst 0xc13919a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z9.s\n" + "ld1w { z21.s }, p0/Z, [x22]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" - "ld1w { z11.s }, p0/Z, [x14]\n" + "ld1w { z22.s }, p0/Z, [x14]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xc13519a1 // fmla za.s[x8, 1], { z13.s-z16.s }, z5.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc13719a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z7.s\n" + ".inst 0xa0404a8e // ld1w { z14.s-z15.s }, pn10.b/Z, [x20]\n" + ".inst 0xc13f1a41 // fmla za.s[x8, 1], { z18.s-z21.s }, z15.s\n" + ".inst 0xa0414a80 // ld1w { z0.s-z1.s }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc1311a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z1.s\n" "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - "ld1w { z21.s }, p0/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" + "ld1w { z16.s }, p0/Z, [x21]\n" + "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z12.s }, p0/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" + "ld1w { z23.s }, p0/Z, [x21]\n" + "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x4\n" - "ld1w { z22.s }, p0/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" + "ld1w { z17.s }, p0/Z, [x21]\n" + "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z13.s }, p0/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" + "ld1w { z24.s }, p0/Z, [x21]\n" + "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z23.s }, p0/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" + "ld1w { z18.s }, p0/Z, [x21]\n" + "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z14.s }, p0/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" + "ld1w { z25.s }, p0/Z, [x21]\n" + "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x8\n" - "ld1w { z24.s }, p0/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" + "ld1w { z19.s }, p0/Z, [x21]\n" + "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z15.s }, p0/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" + "ld1w { z26.s }, p0/Z, [x21]\n" + "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "ld1w { z8.s }, p2/Z, [x15, #4, MUL VL]\n" + ".inst 0xa04149ea // ld1w { z10.s-z11.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + "ld1w { z9.s }, p2/Z, [x15, #4, MUL VL]\n" "addvl x15, x15, #5\n" - "ld1w { z25.s }, p0/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" + "ld1w { z20.s }, p0/Z, [x21]\n" + "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" - ".inst 0xa14149e2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x15, #4, MUL VL]\n" + ".inst 0xa14049e7 // ld1w { z7.s, z15.s }, pn10.b/Z, [x15]\n" + ".inst 0xa04149e0 // ld1w { z0.s-z1.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + "ld1w { z6.s }, p2/Z, [x15, #4, MUL VL]\n" "addvl x15, x15, #5\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z27.s }, p0/Z, [x21]\n" "bgt 18b\n" "19:" // Main loop tail - ".inst 0xc1381960 // fmla za.s[x8, 0], { z11.s-z14.s }, z8.s\n" + ".inst 0xc1391ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z9.s\n" "ld1w { z8.s }, p2/Z, [x15, #4, MUL VL]\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1361961 // fmla za.s[x8, 1], { z11.s-z14.s }, z6.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" - ".inst 0xc1341962 // fmla za.s[x8, 2], { z11.s-z14.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - "addvl x15, x15, #5\n" - ".inst 0xc1331aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z3.s\n" - "ld1w { z3.s }, p2/Z, [x15, #4, MUL VL]\n" - ".inst 0xc1321aa1 // fmla za.s[x8, 1], { z21.s-z24.s }, z2.s\n" - ".inst 0xa14149e2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc1311aa2 // fmla za.s[x8, 2], { z21.s-z24.s }, z1.s\n" + ".inst 0xc13a1ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z10.s\n" + ".inst 0xa04149ea // ld1w { z10.s-z11.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + "add x21, x14, %x[ld_in_row], LSL #2\n" + ".inst 0xc1341ac2 // fmla za.s[x8, 2], { z22.s-z25.s }, z4.s\n" ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" "addvl x15, x15, #5\n" - ".inst 0xc1381980 // fmla za.s[x8, 0], { z12.s-z15.s }, z8.s\n" - "ld1w { z11.s }, p0/Z, [x14]\n" + ".inst 0xc1361a00 // fmla za.s[x8, 0], { z16.s-z19.s }, z6.s\n" + "ld1w { z9.s }, p2/Z, [x15, #4, MUL VL]\n" + ".inst 0xc1301a01 // fmla za.s[x8, 1], { z16.s-z19.s }, z0.s\n" + ".inst 0xa04149ee // ld1w { z14.s-z15.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xc1371a02 // fmla za.s[x8, 2], { z16.s-z19.s }, z7.s\n" + ".inst 0xa04049ec // ld1w { z12.s-z13.s }, pn10.b/Z, [x15]\n" + "addvl x15, x15, #5\n" + ".inst 0xc1381ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z8.s\n" + "ld1w { z16.s }, p0/Z, [x14]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xc1361981 // fmla za.s[x8, 1], { z12.s-z15.s }, z6.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc1341982 // fmla za.s[x8, 2], { z12.s-z15.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xc1331ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z3.s\n" - "ld1w { z8.s }, p2/Z, [x15, #4, MUL VL]\n" - "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xc1321ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z2.s\n" - "ld1w { z21.s }, p0/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xc13a1ae1 // fmla za.s[x8, 1], { z23.s-z26.s }, z10.s\n" + ".inst 0xa14149e5 // ld1w { z5.s, z13.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xc1311ae2 // fmla za.s[x8, 2], { z23.s-z26.s }, z1.s\n" + ".inst 0xa14049e7 // ld1w { z7.s, z15.s }, pn10.b/Z, [x15]\n" + ".inst 0xc1391a20 // fmla za.s[x8, 0], { z17.s-z20.s }, z9.s\n" + "ld1w { z1.s }, p2/Z, [x15, #4, MUL VL]\n" + "ldr x20, [%x[args], %[offsetof_Args_weights]]\n" + ".inst 0xc13e1a21 // fmla za.s[x8, 1], { z17.s-z20.s }, z14.s\n" + "ld1w { z22.s }, p0/Z, [x21]\n" + "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc1311ac2 // fmla za.s[x8, 2], { z22.s-z25.s }, z1.s\n" - "ld1w { z12.s }, p0/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xc13c1a22 // fmla za.s[x8, 2], { z17.s-z20.s }, z12.s\n" + "ld1w { z17.s }, p0/Z, [x21]\n" + "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x4\n" - "ld1w { z22.s }, p0/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc13819a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z8.s\n" + "ld1w { z23.s }, p0/Z, [x21]\n" + "add x21, x21, %x[ld_in_row], LSL #2\n" + ".inst 0xc1311b00 // fmla za.s[x8, 0], { z24.s-z27.s }, z1.s\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc13619a1 // fmla za.s[x8, 1], { z13.s-z16.s }, z6.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - ".inst 0xc13419a2 // fmla za.s[x8, 2], { z13.s-z16.s }, z4.s\n" - "ld1w { z13.s }, p0/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xc1351b01 // fmla za.s[x8, 1], { z24.s-z27.s }, z5.s\n" + ".inst 0xa0414a8e // ld1w { z14.s-z15.s }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc1371b02 // fmla za.s[x8, 2], { z24.s-z27.s }, z7.s\n" + "ld1w { z18.s }, p0/Z, [x21]\n" + "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z23.s }, p0/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" + "ld1w { z24.s }, p0/Z, [x21]\n" + "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" + ".inst 0xc0060c08 // mova { z8.d-z11.d }, za.d[x8, #0]\n" + ".inst 0xa0404a84 // ld1w { z4.s-z5.s }, pn10.b/Z, [x20]\n" "add x8, x8, #0x1\n" - "addvl x15, x15, #5\n" - ".inst 0xc1b1c818 // fclamp { z24.s-z27.s }, z0.s, z17.s\n" - "ld1w { z14.s }, p0/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" + "addvl x20, x20, #5\n" + ".inst 0xc1a3c848 // fclamp { z8.s-z11.s }, z2.s, z3.s\n" + "ld1w { z19.s }, p0/Z, [x21]\n" + "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "st1w { z24.s }, p1, [x10]\n" + "st1w { z8.s }, p1, [x11]\n" "mov x12, #0x8\n" - ".inst 0xc1371960 // fmla za.s[x8, 0], { z11.s-z14.s }, z7.s\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" - "add x10, x10, x28, LSL #2\n" - ".inst 0xc1351961 // fmla za.s[x8, 1], { z11.s-z14.s }, z5.s\n" - ".inst 0xa14149e2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "addvl x15, x15, #5\n" - "st1w { z25.s }, p1, [x9]\n" - "ld1w { z24.s }, p0/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xc13f1a00 // fmla za.s[x8, 0], { z16.s-z19.s }, z15.s\n" + ".inst 0xa0404a80 // ld1w { z0.s-z1.s }, pn10.b/Z, [x20]\n" + "add x11, x11, x9, LSL #2\n" + ".inst 0xc1351a01 // fmla za.s[x8, 1], { z16.s-z19.s }, z5.s\n" + ".inst 0xa1414a80 // ld1w { z0.s, z8.s }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + "addvl x20, x20, #5\n" + "st1w { z9.s }, p1, [x10]\n" + "ld1w { z25.s }, p0/Z, [x21]\n" + "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc13a1aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z10.s\n" - ".inst 0xc1391aa1 // fmla za.s[x8, 1], { z21.s-z24.s }, z9.s\n" - "ld1w { z15.s }, p0/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xc1381ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z8.s\n" + ".inst 0xc1311ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z1.s\n" + "ld1w { z20.s }, p0/Z, [x21]\n" + "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xc1351981 // fmla za.s[x8, 1], { z12.s-z15.s }, z5.s\n" - "add x9, x9, x27, LSL #2\n" - "st1w { z26.s }, p1, [x26]\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "addvl x15, x15, #5\n" - ".inst 0xc1371980 // fmla za.s[x8, 0], { z12.s-z15.s }, z7.s\n" - "add x26, x26, x24, LSL #2\n" - "ld1w { z25.s }, p0/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0xa0404a86 // ld1w { z6.s-z7.s }, pn10.b/Z, [x20]\n" + ".inst 0xc1371a21 // fmla za.s[x8, 1], { z17.s-z20.s }, z7.s\n" + "add x10, x10, x28, LSL #2\n" + "st1w { z10.s }, p1, [x27]\n" + ".inst 0xa1414a81 // ld1w { z1.s, z9.s }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + "addvl x20, x20, #5\n" + ".inst 0xc1391a20 // fmla za.s[x8, 0], { z17.s-z20.s }, z9.s\n" + "add x27, x27, x25, LSL #2\n" + "ld1w { z26.s }, p0/Z, [x21]\n" + "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "st1w { z27.s }, p1, [x25]\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" - ".inst 0xc1391ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z9.s\n" - "add x25, x25, x23, LSL #2\n" - ".inst 0xa14149e2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "addvl x15, x15, #5\n" - ".inst 0xc13a1ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z10.s\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "st1w { z11.s }, p1, [x26]\n" + ".inst 0xa1404a84 // ld1w { z4.s, z12.s }, pn10.b/Z, [x20]\n" + ".inst 0xc13c1ae1 // fmla za.s[x8, 1], { z23.s-z26.s }, z12.s\n" + "add x26, x26, x24, LSL #2\n" + ".inst 0xa1414a84 // ld1w { z4.s, z12.s }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + "addvl x20, x20, #5\n" + ".inst 0xc13c1ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z12.s\n" + "ld1w { z21.s }, p0/Z, [x21]\n" ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xc13519a1 // fmla za.s[x8, 1], { z13.s-z16.s }, z5.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xa0404a80 // ld1w { z0.s-z1.s }, pn10.b/Z, [x20]\n" + ".inst 0xc1311a41 // fmla za.s[x8, 1], { z18.s-z21.s }, z1.s\n" + ".inst 0xa0414a80 // ld1w { z0.s-z1.s }, pn10.b/Z, [x20, #0x2, MUL VL]\n" "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - ".inst 0xc13719a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z7.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "ld1w { z8.s }, p2/Z, [x15, #4, MUL VL]\n" + ".inst 0xc1311a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z1.s\n" + ".inst 0xa14049e4 // ld1w { z4.s, z12.s }, pn10.b/Z, [x15]\n" + ".inst 0xa04149ea // ld1w { z10.s-z11.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + "ld1w { z9.s }, p2/Z, [x15, #4, MUL VL]\n" "addvl x15, x15, #5\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" - ".inst 0xa14149e2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x15, #4, MUL VL]\n" + ".inst 0xa14049e7 // ld1w { z7.s, z15.s }, pn10.b/Z, [x15]\n" + ".inst 0xa04149e0 // ld1w { z0.s-z1.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + "ld1w { z6.s }, p2/Z, [x15, #4, MUL VL]\n" "addvl x15, x15, #5\n" "20:" // Main loop skip tail "cbz x16, 21f\n" // Skip remainder inputs "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z11.s }, p0/Z, [x14]\n" + "ld1w { z16.s }, p0/Z, [x14]\n" "add x20, x14, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z21.s }, p0/Z, [x20]\n" + "ld1w { z23.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z12.s }, p0/Z, [x20]\n" + "ld1w { z17.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z22.s }, p0/Z, [x20]\n" + "ld1w { z24.s }, p0/Z, [x20]\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z13.s }, p0/Z, [x20]\n" + "ld1w { z18.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z23.s }, p0/Z, [x20]\n" + "ld1w { z25.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z14.s }, p0/Z, [x20]\n" + "ld1w { z19.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x8\n" - ".inst 0xc1381960 // fmla za.s[x8, 0], { z11.s-z14.s }, z8.s\n" - "ld1w { z24.s }, p0/Z, [x20]\n" + ".inst 0xc1391a00 // fmla za.s[x8, 0], { z16.s-z19.s }, z9.s\n" + "ld1w { z26.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1331aa0 // fmla za.s[x8, 0], { z21.s-z24.s }, z3.s\n" - "ld1w { z15.s }, p0/Z, [x20]\n" + ".inst 0xc1361ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z6.s\n" + "ld1w { z20.s }, p0/Z, [x20]\n" "ld1w { z8.s }, p2/Z, [x15, #4, MUL VL]\n" - ".inst 0xc1361961 // fmla za.s[x8, 1], { z11.s-z14.s }, z6.s\n" + ".inst 0xc13a1a01 // fmla za.s[x8, 1], { z16.s-z19.s }, z10.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0xc1341962 // fmla za.s[x8, 2], { z11.s-z14.s }, z4.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" + ".inst 0xc1341a02 // fmla za.s[x8, 2], { z16.s-z19.s }, z4.s\n" + ".inst 0xa04049ea // ld1w { z10.s-z11.s }, pn10.b/Z, [x15]\n" "sub x13, x13, #0x1\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xa04149ee // ld1w { z14.s-z15.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" "addvl x15, x15, #5\n" - ".inst 0xc1381980 // fmla za.s[x8, 0], { z12.s-z15.s }, z8.s\n" - "ld1w { z25.s }, p0/Z, [x20]\n" - ".inst 0xc1321aa1 // fmla za.s[x8, 1], { z21.s-z24.s }, z2.s\n" + ".inst 0xc1381a20 // fmla za.s[x8, 0], { z17.s-z20.s }, z8.s\n" + "ld1w { z27.s }, p0/Z, [x20]\n" + ".inst 0xc1301ae1 // fmla za.s[x8, 1], { z23.s-z26.s }, z0.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z3.s }, p2/Z, [x15, #4, MUL VL]\n" - ".inst 0xc1311aa2 // fmla za.s[x8, 2], { z21.s-z24.s }, z1.s\n" - ".inst 0xa14049e1 // ld1w { z1.s, z9.s }, pn10.b/Z, [x15]\n" - ".inst 0xc1331ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z3.s\n" - ".inst 0xa14149e2 // ld1w { z2.s, z10.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + "ld1w { z9.s }, p2/Z, [x15, #4, MUL VL]\n" + ".inst 0xc1371ae2 // fmla za.s[x8, 2], { z23.s-z26.s }, z7.s\n" + ".inst 0xa04049e0 // ld1w { z0.s-z1.s }, pn10.b/Z, [x15]\n" + ".inst 0xc1391b00 // fmla za.s[x8, 0], { z24.s-z27.s }, z9.s\n" + ".inst 0xa14149e5 // ld1w { z5.s, z13.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" "addvl x15, x15, #5\n" - ".inst 0xc1361981 // fmla za.s[x8, 1], { z12.s-z15.s }, z6.s\n" - "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0xc1341982 // fmla za.s[x8, 2], { z12.s-z15.s }, z4.s\n" + ".inst 0xc13e1a21 // fmla za.s[x8, 1], { z17.s-z20.s }, z14.s\n" + "ld1w { z21.s }, p0/Z, [x20]\n" + ".inst 0xc13a1a22 // fmla za.s[x8, 2], { z17.s-z20.s }, z10.s\n" "ld1w { z8.s }, p2/Z, [x15, #4, MUL VL]\n" - ".inst 0xc13819a0 // fmla za.s[x8, 0], { z13.s-z16.s }, z8.s\n" - ".inst 0xc1321ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z2.s\n" - ".inst 0xa04049e4 // ld1w { z4.s-z5.s }, pn10.b/Z, [x15]\n" - ".inst 0xc1311ac2 // fmla za.s[x8, 2], { z22.s-z25.s }, z1.s\n" - ".inst 0xa04149e6 // ld1w { z6.s-z7.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" + ".inst 0xc1381a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z8.s\n" + ".inst 0xc1351b01 // fmla za.s[x8, 1], { z24.s-z27.s }, z5.s\n" + ".inst 0xa04049e8 // ld1w { z8.s-z9.s }, pn10.b/Z, [x15]\n" + ".inst 0xc1301b02 // fmla za.s[x8, 2], { z24.s-z27.s }, z0.s\n" + ".inst 0xa04149e0 // ld1w { z0.s-z1.s }, pn10.b/Z, [x15, #0x2, MUL VL]\n" ".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n" - ".inst 0xc1b1c818 // fclamp { z24.s-z27.s }, z0.s, z17.s\n" - "st1w { z24.s }, p1, [x10]\n" + ".inst 0xc1a3c858 // fclamp { z24.s-z27.s }, z2.s, z3.s\n" + "st1w { z24.s }, p1, [x11]\n" + "add x11, x11, x9, LSL #2\n" + ".inst 0xc1301a41 // fmla za.s[x8, 1], { z18.s-z21.s }, z0.s\n" + "st1w { z25.s }, p1, [x10]\n" "add x10, x10, x28, LSL #2\n" - ".inst 0xc13619a1 // fmla za.s[x8, 1], { z13.s-z16.s }, z6.s\n" - "st1w { z25.s }, p1, [x9]\n" - "add x9, x9, x27, LSL #2\n" - ".inst 0xc13419a2 // fmla za.s[x8, 2], { z13.s-z16.s }, z4.s\n" + ".inst 0xc1381a42 // fmla za.s[x8, 2], { z18.s-z21.s }, z8.s\n" "add x8, x8, #0x1\n" - "st1w { z26.s }, p1, [x26]\n" + "st1w { z26.s }, p1, [x27]\n" + "add x27, x27, x25, LSL #2\n" + "st1w { z27.s }, p1, [x26]\n" "add x26, x26, x24, LSL #2\n" - "st1w { z27.s }, p1, [x25]\n" - "add x25, x25, x23, LSL #2\n" ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" "21:" // Tail input: End "cbz x13, 23f\n" "22:" // Right padding loop - ".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n" + ".inst 0xc0060c08 // mova { z8.d-z11.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" "subs x13, x13, #0x1\n" - ".inst 0xc1b1c818 // fclamp { z24.s-z27.s }, z0.s, z17.s\n" - "st1w { z24.s }, p1, [x10]\n" - "add x10, x10, x28, LSL #2\n" + ".inst 0xc1a3c848 // fclamp { z8.s-z11.s }, z2.s, z3.s\n" + "st1w { z8.s }, p1, [x11]\n" + "add x11, x11, x9, LSL #2\n" ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" - "st1w { z25.s }, p1, [x9]\n" - "add x9, x9, x27, LSL #2\n" - "st1w { z26.s }, p1, [x26]\n" + "st1w { z9.s }, p1, [x10]\n" + "add x10, x10, x28, LSL #2\n" + "st1w { z10.s }, p1, [x27]\n" + "add x27, x27, x25, LSL #2\n" + "st1w { z11.s }, p1, [x26]\n" "add x26, x26, x24, LSL #2\n" - "st1w { z27.s }, p1, [x25]\n" - "add x25, x25, x23, LSL #2\n" "bgt 22b\n" "23:" // End - "ldr x15, [%x[args], %[offsetof_Args_weights]]\n" - "incb x15, ALL, MUL #16\n" - "incb x15, ALL, MUL #9\n" - "str x15, [%x[args], %[offsetof_Args_weights]]\n" - "ldr x20, [%x[args], %[offsetof_Args_ld_in_vl]]\n" + "ldr x20, [%x[args], %[offsetof_Args_weights]]\n" + "incb x20, ALL, MUL #16\n" + "incb x20, ALL, MUL #9\n" + "str x20, [%x[args], %[offsetof_Args_weights]]\n" + "ldr x21, [%x[args], %[offsetof_Args_ld_in_vl]]\n" "incw x17\n" "whilelt p1.s, x17, x7\n" - "ldr x14, [%x[args], %[offsetof_Args_inptr]]\n" - "add x14, x14, x20, LSL #2\n" - "str x14, [%x[args], %[offsetof_Args_inptr]]\n" - "ldr x11, [%x[args], %[offsetof_Args_outptrs]]\n" + "ldr x20, [%x[args], %[offsetof_Args_inptr]]\n" + "add x20, x20, x21, LSL #2\n" + "str x20, [%x[args], %[offsetof_Args_inptr]]\n" + "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" "ldr x24, [%x[args], %[offsetof_Args_ld_out_vls]]\n" - "ldp x23, x22, [x11, #0x0]\n" + "ldp x23, x22, [x25, #0x0]\n" "ldp x21, x20, [x24, #0x0]\n" "add x23, x23, x21, LSL #2\n" "add x22, x22, x20, LSL #2\n" - "stp x23, x22, [x11, #0x0]\n" - "ldp x23, x22, [x11, #0x10]\n" + "stp x23, x22, [x25, #0x0]\n" + "ldp x23, x22, [x25, #0x10]\n" "ldp x21, x20, [x24, #0x10]\n" "add x23, x23, x21, LSL #2\n" "add x22, x22, x20, LSL #2\n" - "stp x23, x22, [x11, #0x10]\n" + "stp x23, x22, [x25, #0x10]\n" "b.any 1b\n" ".inst 0xd503467f // SMSTOP\n" : diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_3x3_s1_4rows_dot_za.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_3x3_s1_4rows_dot_za.hpp index f09c61667f103426f5a8506575655ebf24fcc1b7..50ef6c3815c509987c636d2df27873a44f51f2a1 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_3x3_s1_4rows_dot_za.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_3x3_s1_4rows_dot_za.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,8 @@ * SOFTWARE. */ +#include "src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp" + namespace arm_conv { namespace depthwise { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_3x3_s1_4rows_dot_za/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_3x3_s1_4rows_dot_za/generic.cpp index 5570b27644133c7bf6027b6e82237b1b4a247a49..be82e046139ea967ceecd986c544bdb9b406aa27 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_3x3_s1_4rows_dot_za/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_3x3_s1_4rows_dot_za/generic.cpp @@ -69,102 +69,102 @@ void sme2_fp32bf16fp32_planar_3x3_s1_4rows_dot_za_impl( Args args = { inptr, ld_in_vl, pad_top, 6u - std::min(6u, pad_top + valid_input_rows), pad_left, weights, bias, valid_input_cols, output_cols, outptrs, outlds, outvllds, start_channel, valid_channels, act_min, act_max }; __asm__ __volatile__( - "ldr x6, [%x[args], %[offsetof_Args_pad_bottom]]\n" + "ldr x7, [%x[args], %[offsetof_Args_pad_bottom]]\n" "mov x20, #0x6\n" ".inst 0xd503477f // SMSTART ZA\n" - "sub x20, x20, x6\n" - "ldr x7, [%x[args], %[offsetof_Args_pad_top]]\n" + "sub x20, x20, x7\n" + "ldr x17, [%x[args], %[offsetof_Args_pad_top]]\n" "ptrue p2.b\n" - "ld1rw { z28.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_min]]\n" - "ldr x17, [%x[args], %[offsetof_Args_n_channels]]\n" - "whilelt p1.s, XZR, x17\n" + "ld1rw { z25.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_min]]\n" + "ldr x16, [%x[args], %[offsetof_Args_n_channels]]\n" + "whilelt p1.s, XZR, x16\n" "whilelt p9.s, XZR, x20\n" - "ld1rw { z29.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_max]]\n" - "whilelt p8.s, XZR, x7\n" + "ld1rw { z13.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_max]]\n" + "whilelt p8.s, XZR, x17\n" "eor p8.b, p2/Z, p8.b, p9.b\n" - "ldr x16, [%x[args], %[offsetof_Args_current_channel]]\n" + "ldr x15, [%x[args], %[offsetof_Args_current_channel]]\n" "1:" // Channel loop "ldr x20, [%x[args], %[offsetof_Args_bias]]\n" - "fmov z22.s, #0x0\n" + "fmov z26.s, #0x0\n" "cbz x20, 2f\n" - "ld1w { z22.s }, p1/Z, [x20, x16, LSL #2]\n" + "ld1w { z26.s }, p1/Z, [x20, x15, LSL #2]\n" "2:" // Load bias: Done - "ldr x20, [%x[args], %[offsetof_Args_weights]]\n" - "mov x21, x20\n" - "fmov z9.s, #0x0\n" - "ld1w { z25.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #3\n" - "incb x20\n" - "ld1w { z27.s }, p2/Z, [x21]\n" - ".inst 0x648aab29 // bfcvtnt z9.h, p2/M, z25.s\n" - "incb x21, ALL, MUL #3\n" - "ld1w { z21.s }, p2/Z, [x21]\n" - "mov x21, x20\n" - ".inst 0x658aab28 // bfcvt z8.h, p2/M, z25.s\n" - "ld1w { z25.s }, p2/Z, [x21]\n" - ".inst 0x658aab66 // bfcvt z6.h, p2/M, z27.s\n" - "fmov z2.s, #0x0\n" - "incb x21, ALL, MUL #3\n" - ".inst 0x658aab21 // bfcvt z1.h, p2/M, z25.s\n" - ".inst 0x648aab68 // bfcvtnt z8.h, p2/M, z27.s\n" - "incb x20\n" - "ld1w { z27.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #3\n" - ".inst 0x648aaaa6 // bfcvtnt z6.h, p2/M, z21.s\n" - ".inst 0x658aaaa5 // bfcvt z5.h, p2/M, z21.s\n" - "ld1w { z21.s }, p2/Z, [x21]\n" - "mov x21, x20\n" - ".inst 0x648aab22 // bfcvtnt z2.h, p2/M, z25.s\n" - "ld1w { z25.s }, p2/Z, [x21]\n" + "ldr x21, [%x[args], %[offsetof_Args_weights]]\n" + "mov x20, x21\n" + "fmov z6.s, #0x0\n" + "ld1w { z15.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #3\n" + "incb x21\n" + "ld1w { z29.s }, p2/Z, [x20]\n" + ".inst 0x648aa9e6 // bfcvtnt z6.h, p2/M, z15.s\n" + "incb x20, ALL, MUL #3\n" + "ld1w { z30.s }, p2/Z, [x20]\n" + "mov x20, x21\n" + ".inst 0x658aa9e5 // bfcvt z5.h, p2/M, z15.s\n" + "ld1w { z14.s }, p2/Z, [x20]\n" + ".inst 0x658aaba8 // bfcvt z8.h, p2/M, z29.s\n" + "fmov z11.s, #0x0\n" + "incb x20, ALL, MUL #3\n" + ".inst 0x658aa9ca // bfcvt z10.h, p2/M, z14.s\n" + ".inst 0x648aaba5 // bfcvtnt z5.h, p2/M, z29.s\n" + "incb x21\n" + "ld1w { z24.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #3\n" + ".inst 0x648aabc8 // bfcvtnt z8.h, p2/M, z30.s\n" + ".inst 0x658aabcc // bfcvt z12.h, p2/M, z30.s\n" + "ld1w { z28.s }, p2/Z, [x20]\n" + "mov x21, x21\n" + ".inst 0x648aa9cb // bfcvtnt z11.h, p2/M, z14.s\n" + "ld1w { z20.s }, p2/Z, [x21]\n" "incb x21, ALL, MUL #3\n" - ".inst 0x648aab61 // bfcvtnt z1.h, p2/M, z27.s\n" - ".inst 0x658aab6c // bfcvt z12.h, p2/M, z27.s\n" - "ld1w { z27.s }, p2/Z, [x21]\n" - "ldr x15, [%x[args], %[offsetof_Args_input_cols]]\n" + ".inst 0x648aab0a // bfcvtnt z10.h, p2/M, z24.s\n" + ".inst 0x658aab09 // bfcvt z9.h, p2/M, z24.s\n" + "ld1w { z15.s }, p2/Z, [x21]\n" + "ldr x14, [%x[args], %[offsetof_Args_input_cols]]\n" "incb x21, ALL, MUL #3\n" - "fmov z7.s, #0x0\n" - ".inst 0x658aab24 // bfcvt z4.h, p2/M, z25.s\n" - "ldr x14, [%x[args], %[offsetof_Args_inptr]]\n" - ".inst 0x658aab60 // bfcvt z0.h, p2/M, z27.s\n" - ".inst 0x648aaaac // bfcvtnt z12.h, p2/M, z21.s\n" - "sub x20, x15, #0x1\n" + "fmov z14.s, #0x0\n" + ".inst 0x658aaa81 // bfcvt z1.h, p2/M, z20.s\n" + "ldr x13, [%x[args], %[offsetof_Args_inptr]]\n" + ".inst 0x658aa9e7 // bfcvt z7.h, p2/M, z15.s\n" + ".inst 0x648aab89 // bfcvtnt z9.h, p2/M, z28.s\n" + "sub x20, x14, #0x1\n" "orr x23, x20, %x[ld_in_col], LSL #18\n" - ".inst 0x658aaaaa // bfcvt z10.h, p2/M, z21.s\n" - "ld1w { z21.s }, p2/Z, [x21]\n" - "orr x23, x17, x23, LSL #20\n" + ".inst 0x658aab84 // bfcvt z4.h, p2/M, z28.s\n" + "ld1w { z29.s }, p2/Z, [x21]\n" + "orr x23, x16, x23, LSL #20\n" "mov x22, #0x6\n" - "add x21, x7, x6\n" + "add x21, x17, x7\n" "lsl x20, %x[ld_in_row], #0x2\n" - "mov z23.d, z22.d\n" - ".inst 0x648aab27 // bfcvtnt z7.h, p2/M, z25.s\n" - ".inst 0x648aab64 // bfcvtnt z4.h, p2/M, z27.s\n" - ".inst 0x648aaaa0 // bfcvtnt z0.h, p2/M, z21.s\n" + "mov z27.d, z26.d\n" + ".inst 0x648aaa8e // bfcvtnt z14.h, p2/M, z20.s\n" + ".inst 0x648aa9e1 // bfcvtnt z1.h, p2/M, z15.s\n" + ".inst 0x648aaba7 // bfcvtnt z7.h, p2/M, z29.s\n" "mov x8, #0x0\n" - "ldr x13, [%x[args], %[offsetof_Args_output_cols]]\n" - ".inst 0x658aaaa3 // bfcvt z3.h, p2/M, z21.s\n" + "ldr x11, [%x[args], %[offsetof_Args_output_cols]]\n" + ".inst 0x658aaba2 // bfcvt z2.h, p2/M, z29.s\n" "lsl x23, x23, #0x2\n" "sub x22, x22, x21\n" - "madd x20, x20, x7, x14\n" + "madd x20, x20, x17, x13\n" "3:" // Issue prefetches "subs x22, x22, #0x1\n" ".inst 0xf8b74a9c // rprfm pldstrm, x23, [x20]\n" "add x20, x20, %x[ld_in_col], LSL #2\n" "bgt 3b\n" - "ldr x11, [%x[args], %[offsetof_Args_outptrs]]\n" + "ldr x22, [%x[args], %[offsetof_Args_outptrs]]\n" "lsl x20, %x[ld_in_row], #0x2\n" - "msub x14, x7, x20, x14\n" - ".inst 0xc0040ac0 // mova za.d[x8, #0], { z22.d-z23.d }\n" + "msub x13, x17, x20, x13\n" + ".inst 0xc0040b40 // mova za.d[x8, #0], { z26.d-z27.d }\n" "ldr x20, [%x[args], %[offsetof_Args_ld_out_cols]]\n" - ".inst 0xc0040ac1 // mova za.d[x8, #1], { z22.d-z23.d }\n" + ".inst 0xc0040b41 // mova za.d[x8, #1], { z26.d-z27.d }\n" "mov x10, #0x2\n" - "ldp x9, x28, [x11], #0x10\n" - ".inst 0xc0040ac2 // mova za.d[x8, #2], { z22.d-z23.d }\n" + "ldp x9, x28, [x22], #0x10\n" + ".inst 0xc0040b42 // mova za.d[x8, #2], { z26.d-z27.d }\n" "ldp x27, x26, [x20], #0x10\n" - ".inst 0xc0040ac3 // mova za.d[x8, #3], { z22.d-z23.d }\n" + ".inst 0xc0040b43 // mova za.d[x8, #3], { z26.d-z27.d }\n" "ldr x21, [%x[args], %[offsetof_Args_pad_left]]\n" - ".inst 0xc0040ac4 // mova za.d[x8, #4], { z22.d-z23.d }\n" - "ldp x25, x24, [x11], #0x10\n" - ".inst 0xc0040ac5 // mova za.d[x8, #5], { z22.d-z23.d }\n" + ".inst 0xc0040b44 // mova za.d[x8, #4], { z26.d-z27.d }\n" + "ldp x25, x24, [x22], #0x10\n" + ".inst 0xc0040b45 // mova za.d[x8, #5], { z26.d-z27.d }\n" "ldp x23, x22, [x20], #0x10\n" "cbz x21, 5f\n" "cmp x21, x10\n" @@ -172,389 +172,389 @@ void sme2_fp32bf16fp32_planar_3x3_s1_4rows_dot_za_impl( "sub x21, x21, x20\n" "sub x10, x10, x20\n" "cbz x21, 5f\n" - ".inst 0xc0060818 // mova { z24.d-z25.d }, za.d[x8, #0]\n" - "sub x13, x13, x21\n" - ".inst 0xc006083a // mova { z26.d-z27.d }, za.d[x8, #1]\n" - ".inst 0xc1bdcb98 // fclamp { z24.s-z27.s }, z28.s, z29.s\n" + ".inst 0xc0060814 // mova { z20.d-z21.d }, za.d[x8, #0]\n" + "sub x11, x11, x21\n" + ".inst 0xc0060836 // mova { z22.d-z23.d }, za.d[x8, #1]\n" + ".inst 0xc1adcb34 // fclamp { z20.s-z23.s }, z25.s, z13.s\n" "4:" // Left padding "subs x21, x21, #0x1\n" - "st1w { z24.s }, p1, [x9]\n" + "st1w { z20.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - "st1w { z26.s }, p1, [x28]\n" + "st1w { z22.s }, p1, [x28]\n" "add x28, x28, x26, LSL #2\n" - "st1w { z25.s }, p1, [x25]\n" + "st1w { z21.s }, p1, [x25]\n" "add x25, x25, x23, LSL #2\n" - "st1w { z27.s }, p1, [x24]\n" + "st1w { z23.s }, p1, [x24]\n" "add x24, x24, x22, LSL #2\n" "bgt 4b\n" "5:" // Left padding: End - "adds XZR, x7, x6\n" + "adds XZR, x17, x7\n" "bne 10f\n" "cbz x10, 8f\n" "cmp x10, #0x1\n" - "sub x15, x15, x10\n" + "sub x14, x14, x10\n" "beq 7f\n" "6:" // Unpadded: 2 priming loads - "add x20, x14, %x[ld_in_row], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x14]\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" + "ld1w { z17.s }, p1/Z, [x13]\n" + ".inst 0x658aaa3e // bfcvt z30.h, p2/M, z17.s\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + "ld1w { z28.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + ".inst 0x648aab9e // bfcvtnt z30.h, p2/M, z28.s\n" "ld1w { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + ".inst 0x658aaa1f // bfcvt z31.h, p2/M, z16.s\n" + "ld1w { z15.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" - ".inst 0xc12811b0 // bfdot za.s[x8, 0], { z13.h-z14.h }, z8.h\n" + ".inst 0x648aa9ff // bfcvtnt z31.h, p2/M, z15.s\n" + ".inst 0xc12513d0 // bfdot za.s[x8, 0], { z30.h-z31.h }, z5.h\n" "ld1w { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - ".inst 0xc12911b1 // bfdot za.s[x8, 1], { z13.h-z14.h }, z9.h\n" - "ld1w { z16.s }, p1/Z, [x20]\n" - ".inst 0x648aaa0f // bfcvtnt z15.h, p2/M, z16.s\n" - ".inst 0xc12511d0 // bfdot za.s[x8, 0], { z14.h-z15.h }, z5.h\n" - ".inst 0xc12611d1 // bfdot za.s[x8, 1], { z14.h-z15.h }, z6.h\n" + ".inst 0x658aaa00 // bfcvt z0.h, p2/M, z16.s\n" + ".inst 0xc12613d1 // bfdot za.s[x8, 1], { z30.h-z31.h }, z6.h\n" + "ld1w { z15.s }, p1/Z, [x20]\n" + ".inst 0x648aa9e0 // bfcvtnt z0.h, p2/M, z15.s\n" + ".inst 0xc12c13f0 // bfdot za.s[x8, 0], { z31.h-z0.h }, z12.h\n" + ".inst 0xc12813f1 // bfdot za.s[x8, 1], { z31.h-z0.h }, z8.h\n" "7:" // Unpadded: 1 priming loads - "add x20, x14, %x[ld_in_row], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x14]\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" + "ld1w { z31.s }, p1/Z, [x13]\n" + ".inst 0x658aabef // bfcvt z15.h, p2/M, z31.s\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" "ld1w { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + ".inst 0x648aaa0f // bfcvtnt z15.h, p2/M, z16.s\n" "ld1w { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + ".inst 0x658aaa10 // bfcvt z16.h, p2/M, z16.s\n" + "ld1w { z17.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" - ".inst 0xc12111b0 // bfdot za.s[x8, 0], { z13.h-z14.h }, z1.h\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + ".inst 0x648aaa30 // bfcvtnt z16.h, p2/M, z17.s\n" + ".inst 0xc12a11f0 // bfdot za.s[x8, 0], { z15.h-z16.h }, z10.h\n" + "ld1w { z22.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - ".inst 0xc12211b1 // bfdot za.s[x8, 1], { z13.h-z14.h }, z2.h\n" - "ld1w { z16.s }, p1/Z, [x20]\n" - ".inst 0x648aaa0f // bfcvtnt z15.h, p2/M, z16.s\n" - ".inst 0xc12811b2 // bfdot za.s[x8, 2], { z13.h-z14.h }, z8.h\n" - ".inst 0xc12911b3 // bfdot za.s[x8, 3], { z13.h-z14.h }, z9.h\n" - ".inst 0xc12a11d0 // bfdot za.s[x8, 0], { z14.h-z15.h }, z10.h\n" - ".inst 0xc12c11d1 // bfdot za.s[x8, 1], { z14.h-z15.h }, z12.h\n" - ".inst 0xc12511d2 // bfdot za.s[x8, 2], { z14.h-z15.h }, z5.h\n" - ".inst 0xc12611d3 // bfdot za.s[x8, 3], { z14.h-z15.h }, z6.h\n" + ".inst 0x658aaad1 // bfcvt z17.h, p2/M, z22.s\n" + ".inst 0xc12b11f1 // bfdot za.s[x8, 1], { z15.h-z16.h }, z11.h\n" + "ld1w { z18.s }, p1/Z, [x20]\n" + ".inst 0x648aaa51 // bfcvtnt z17.h, p2/M, z18.s\n" + ".inst 0xc12511f2 // bfdot za.s[x8, 2], { z15.h-z16.h }, z5.h\n" + ".inst 0xc12611f3 // bfdot za.s[x8, 3], { z15.h-z16.h }, z6.h\n" + ".inst 0xc1241210 // bfdot za.s[x8, 0], { z16.h-z17.h }, z4.h\n" + ".inst 0xc1291211 // bfdot za.s[x8, 1], { z16.h-z17.h }, z9.h\n" + ".inst 0xc12c1212 // bfdot za.s[x8, 2], { z16.h-z17.h }, z12.h\n" + ".inst 0xc1281213 // bfdot za.s[x8, 3], { z16.h-z17.h }, z8.h\n" "8:" // Unpadded: 0 priming loads - "cbz x15, 16f\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x14]\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" - "sub x15, x15, #0x1\n" + "cbz x14, 16f\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" + "ld1w { z16.s }, p1/Z, [x13]\n" + ".inst 0x658aaa16 // bfcvt z22.h, p2/M, z16.s\n" + "sub x14, x14, #0x1\n" "ld1w { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "sub x13, x13, #0x1\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + "sub x11, x11, #0x1\n" + ".inst 0x648aaa16 // bfcvtnt z22.h, p2/M, z16.s\n" + "ld1w { z0.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" - "cmp x15, x13\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + ".inst 0x658aa817 // bfcvt z23.h, p2/M, z0.s\n" + "cmp x14, x11\n" + "ld1w { z24.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "csel x21, x15, x13, LT\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + "csel x21, x14, x11, LT\n" + ".inst 0x648aab17 // bfcvtnt z23.h, p2/M, z24.s\n" + "ld1w { z0.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" + ".inst 0x658aa818 // bfcvt z24.h, p2/M, z0.s\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" "ld1w { z16.s }, p1/Z, [x20]\n" - ".inst 0x648aaa0f // bfcvtnt z15.h, p2/M, z16.s\n" - "sub x13, x13, x21\n" + ".inst 0x648aaa18 // bfcvtnt z24.h, p2/M, z16.s\n" + "sub x11, x11, x21\n" "cbz x21, 15f\n" "9:" // Unpadded: Main loop - ".inst 0xc12411b0 // bfdot za.s[x8, 0], { z13.h-z14.h }, z4.h\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" - "ld1w { z21.s }, p1/Z, [x14]\n" + ".inst 0xc12112d0 // bfdot za.s[x8, 0], { z22.h-z23.h }, z1.h\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" + "ld1w { z0.s }, p1/Z, [x13]\n" "subs x21, x21, #0x1\n" - ".inst 0xc12711b1 // bfdot za.s[x8, 1], { z13.h-z14.h }, z7.h\n" + ".inst 0xc12e12d1 // bfdot za.s[x8, 1], { z22.h-z23.h }, z14.h\n" "ld1w { z20.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" "ld1w { z19.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc12311d0 // bfdot za.s[x8, 0], { z14.h-z15.h }, z3.h\n" - ".inst 0xc12011d1 // bfdot za.s[x8, 1], { z14.h-z15.h }, z0.h\n" + ".inst 0xc12212f0 // bfdot za.s[x8, 0], { z23.h-z24.h }, z2.h\n" + ".inst 0xc12712f1 // bfdot za.s[x8, 1], { z23.h-z24.h }, z7.h\n" "ld1w { z18.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc12111b2 // bfdot za.s[x8, 2], { z13.h-z14.h }, z1.h\n" + ".inst 0xc12a12d2 // bfdot za.s[x8, 2], { z22.h-z23.h }, z10.h\n" "ld1w { z17.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc12211b3 // bfdot za.s[x8, 3], { z13.h-z14.h }, z2.h\n" - "ld1w { z16.s }, p1/Z, [x20]\n" - ".inst 0xc12811b4 // bfdot za.s[x8, 4], { z13.h-z14.h }, z8.h\n" - ".inst 0xc12911b5 // bfdot za.s[x8, 5], { z13.h-z14.h }, z9.h\n" - ".inst 0x658aaaad // bfcvt z13.h, p2/M, z21.s\n" - ".inst 0x648aaa8d // bfcvtnt z13.h, p2/M, z20.s\n" - ".inst 0xc12a11d2 // bfdot za.s[x8, 2], { z14.h-z15.h }, z10.h\n" - ".inst 0xc12c11d3 // bfdot za.s[x8, 3], { z14.h-z15.h }, z12.h\n" - ".inst 0xc12511d4 // bfdot za.s[x8, 4], { z14.h-z15.h }, z5.h\n" - ".inst 0xc12611d5 // bfdot za.s[x8, 5], { z14.h-z15.h }, z6.h\n" - ".inst 0x658aaa6e // bfcvt z14.h, p2/M, z19.s\n" - ".inst 0x658aaa2f // bfcvt z15.h, p2/M, z17.s\n" - ".inst 0xc0060818 // mova { z24.d-z25.d }, za.d[x8, #0]\n" - ".inst 0x648aaa4e // bfcvtnt z14.h, p2/M, z18.s\n" - ".inst 0x648aaa0f // bfcvtnt z15.h, p2/M, z16.s\n" - ".inst 0xc006083a // mova { z26.d-z27.d }, za.d[x8, #1]\n" + ".inst 0xc12b12d3 // bfdot za.s[x8, 3], { z22.h-z23.h }, z11.h\n" + "ld1w { z28.s }, p1/Z, [x20]\n" + ".inst 0xc12512d4 // bfdot za.s[x8, 4], { z22.h-z23.h }, z5.h\n" + ".inst 0xc12612d5 // bfdot za.s[x8, 5], { z22.h-z23.h }, z6.h\n" + ".inst 0x658aa816 // bfcvt z22.h, p2/M, z0.s\n" + ".inst 0x648aaa96 // bfcvtnt z22.h, p2/M, z20.s\n" + ".inst 0xc12412f2 // bfdot za.s[x8, 2], { z23.h-z24.h }, z4.h\n" + ".inst 0xc12912f3 // bfdot za.s[x8, 3], { z23.h-z24.h }, z9.h\n" + ".inst 0xc12c12f4 // bfdot za.s[x8, 4], { z23.h-z24.h }, z12.h\n" + ".inst 0xc12812f5 // bfdot za.s[x8, 5], { z23.h-z24.h }, z8.h\n" + ".inst 0x658aaa77 // bfcvt z23.h, p2/M, z19.s\n" + ".inst 0x658aaa38 // bfcvt z24.h, p2/M, z17.s\n" + ".inst 0xc0060810 // mova { z16.d-z17.d }, za.d[x8, #0]\n" + ".inst 0x648aaa57 // bfcvtnt z23.h, p2/M, z18.s\n" + ".inst 0x648aab98 // bfcvtnt z24.h, p2/M, z28.s\n" + ".inst 0xc0060832 // mova { z18.d-z19.d }, za.d[x8, #1]\n" "add x8, x8, #0x2\n" - ".inst 0xc1bdcb98 // fclamp { z24.s-z27.s }, z28.s, z29.s\n" - "st1w { z24.s }, p1, [x9]\n" + ".inst 0xc1adcb30 // fclamp { z16.s-z19.s }, z25.s, z13.s\n" + "st1w { z16.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - "st1w { z26.s }, p1, [x28]\n" + "st1w { z18.s }, p1, [x28]\n" "add x28, x28, x26, LSL #2\n" - ".inst 0xc0040ac4 // mova za.d[x8, #4], { z22.d-z23.d }\n" - "st1w { z25.s }, p1, [x25]\n" + ".inst 0xc0040b44 // mova za.d[x8, #4], { z26.d-z27.d }\n" + "st1w { z17.s }, p1, [x25]\n" "add x25, x25, x23, LSL #2\n" - ".inst 0xc0040ac5 // mova za.d[x8, #5], { z22.d-z23.d }\n" - "st1w { z27.s }, p1, [x24]\n" + ".inst 0xc0040b45 // mova za.d[x8, #5], { z26.d-z27.d }\n" + "st1w { z19.s }, p1, [x24]\n" "add x24, x24, x22, LSL #2\n" "bgt 9b\n" "b 15f\n" "10:" // Padded "cbz x10, 13f\n" "cmp x10, #0x1\n" - "sub x15, x15, x10\n" + "sub x14, x14, x10\n" "beq 12f\n" "11:" // Padded: 2 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x14]\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + "ld1w { z16.s }, p0/Z, [x13]\n" + ".inst 0x658aaa14 // bfcvt z20.h, p2/M, z16.s\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + ".inst 0x648aaa14 // bfcvtnt z20.h, p2/M, z16.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" + ".inst 0x658aaa15 // bfcvt z21.h, p2/M, z16.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" + ".inst 0x648aaa15 // bfcvtnt z21.h, p2/M, z16.s\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc12811b0 // bfdot za.s[x8, 0], { z13.h-z14.h }, z8.h\n" + ".inst 0xc1251290 // bfdot za.s[x8, 0], { z20.h-z21.h }, z5.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z23.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" + ".inst 0x658aaaf6 // bfcvt z22.h, p2/M, z23.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x648aaa0f // bfcvtnt z15.h, p2/M, z16.s\n" - ".inst 0xc12911b1 // bfdot za.s[x8, 1], { z13.h-z14.h }, z9.h\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xc12511d0 // bfdot za.s[x8, 0], { z14.h-z15.h }, z5.h\n" - ".inst 0xc12611d1 // bfdot za.s[x8, 1], { z14.h-z15.h }, z6.h\n" + ".inst 0x648aaa16 // bfcvtnt z22.h, p2/M, z16.s\n" + ".inst 0xc1261291 // bfdot za.s[x8, 1], { z20.h-z21.h }, z6.h\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + ".inst 0xc12c12b0 // bfdot za.s[x8, 0], { z21.h-z22.h }, z12.h\n" + ".inst 0xc12812b1 // bfdot za.s[x8, 1], { z21.h-z22.h }, z8.h\n" "12:" // Padded: 1 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x14]\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + "ld1w { z16.s }, p0/Z, [x13]\n" + ".inst 0x658aaa13 // bfcvt z19.h, p2/M, z16.s\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + ".inst 0x648aaa13 // bfcvtnt z19.h, p2/M, z16.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" + ".inst 0x658aaa14 // bfcvt z20.h, p2/M, z16.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" + ".inst 0x648aaa14 // bfcvtnt z20.h, p2/M, z16.s\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc12111b0 // bfdot za.s[x8, 0], { z13.h-z14.h }, z1.h\n" + ".inst 0xc12a1270 // bfdot za.s[x8, 0], { z19.h-z20.h }, z10.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z15.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" + ".inst 0x658aa9f5 // bfcvt z21.h, p2/M, z15.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x648aaa0f // bfcvtnt z15.h, p2/M, z16.s\n" - ".inst 0xc12211b1 // bfdot za.s[x8, 1], { z13.h-z14.h }, z2.h\n" - ".inst 0xc12811b2 // bfdot za.s[x8, 2], { z13.h-z14.h }, z8.h\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xc12911b3 // bfdot za.s[x8, 3], { z13.h-z14.h }, z9.h\n" - ".inst 0xc12a11d0 // bfdot za.s[x8, 0], { z14.h-z15.h }, z10.h\n" - ".inst 0xc12c11d1 // bfdot za.s[x8, 1], { z14.h-z15.h }, z12.h\n" - ".inst 0xc12511d2 // bfdot za.s[x8, 2], { z14.h-z15.h }, z5.h\n" - ".inst 0xc12611d3 // bfdot za.s[x8, 3], { z14.h-z15.h }, z6.h\n" + ".inst 0x648aaa15 // bfcvtnt z21.h, p2/M, z16.s\n" + ".inst 0xc12b1271 // bfdot za.s[x8, 1], { z19.h-z20.h }, z11.h\n" + ".inst 0xc1251272 // bfdot za.s[x8, 2], { z19.h-z20.h }, z5.h\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + ".inst 0xc1261273 // bfdot za.s[x8, 3], { z19.h-z20.h }, z6.h\n" + ".inst 0xc1241290 // bfdot za.s[x8, 0], { z20.h-z21.h }, z4.h\n" + ".inst 0xc1291291 // bfdot za.s[x8, 1], { z20.h-z21.h }, z9.h\n" + ".inst 0xc12c1292 // bfdot za.s[x8, 2], { z20.h-z21.h }, z12.h\n" + ".inst 0xc1281293 // bfdot za.s[x8, 3], { z20.h-z21.h }, z8.h\n" "13:" // Padded: 0 priming loads - "cbz x15, 16f\n" + "cbz x14, 16f\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x14]\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + "ld1w { z16.s }, p0/Z, [x13]\n" + ".inst 0x658aaa16 // bfcvt z22.h, p2/M, z16.s\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + ".inst 0x648aaa16 // bfcvtnt z22.h, p2/M, z16.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" + ".inst 0x658aaa17 // bfcvt z23.h, p2/M, z16.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" + ".inst 0x648aaa17 // bfcvtnt z23.h, p2/M, z16.s\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1w { z16.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" + ".inst 0x658aaa18 // bfcvt z24.h, p2/M, z16.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - "sub x15, x15, #0x1\n" - ".inst 0x648aaa0f // bfcvtnt z15.h, p2/M, z16.s\n" - "sub x13, x13, #0x1\n" - "cmp x15, x13\n" - "csel x21, x15, x13, LT\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - "sub x13, x13, x21\n" + "sub x14, x14, #0x1\n" + ".inst 0x648aaa18 // bfcvtnt z24.h, p2/M, z16.s\n" + "sub x11, x11, #0x1\n" + "cmp x14, x11\n" + "csel x21, x14, x11, LT\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + "sub x11, x11, x21\n" "cbz x21, 15f\n" "14:" // Padded: Main loop "mov x12, #0x0\n" - ".inst 0xc12411b0 // bfdot za.s[x8, 0], { z13.h-z14.h }, z4.h\n" + ".inst 0xc12112d0 // bfdot za.s[x8, 0], { z22.h-z23.h }, z1.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z21.s }, p0/Z, [x14]\n" - ".inst 0xc12711b1 // bfdot za.s[x8, 1], { z13.h-z14.h }, z7.h\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + "ld1w { z20.s }, p0/Z, [x13]\n" + ".inst 0xc12e12d1 // bfdot za.s[x8, 1], { z22.h-z23.h }, z14.h\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z20.s }, p0/Z, [x20]\n" + "ld1w { z19.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z19.s }, p0/Z, [x20]\n" - ".inst 0xc12311d0 // bfdot za.s[x8, 0], { z14.h-z15.h }, z3.h\n" + "ld1w { z17.s }, p0/Z, [x20]\n" + ".inst 0xc12212f0 // bfdot za.s[x8, 0], { z23.h-z24.h }, z2.h\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xc12011d1 // bfdot za.s[x8, 1], { z14.h-z15.h }, z0.h\n" + ".inst 0xc12712f1 // bfdot za.s[x8, 1], { z23.h-z24.h }, z7.h\n" "ld1w { z18.s }, p0/Z, [x20]\n" "mov x12, #0x4\n" - ".inst 0xc12111b2 // bfdot za.s[x8, 2], { z13.h-z14.h }, z1.h\n" + ".inst 0xc12a12d2 // bfdot za.s[x8, 2], { z22.h-z23.h }, z10.h\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc12211b3 // bfdot za.s[x8, 3], { z13.h-z14.h }, z2.h\n" + ".inst 0xc12b12d3 // bfdot za.s[x8, 3], { z22.h-z23.h }, z11.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z17.s }, p0/Z, [x20]\n" + "ld1w { z16.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc12811b4 // bfdot za.s[x8, 4], { z13.h-z14.h }, z8.h\n" + ".inst 0xc12512d4 // bfdot za.s[x8, 4], { z22.h-z23.h }, z5.h\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z15.s }, p0/Z, [x20]\n" "subs x21, x21, #0x1\n" - ".inst 0xc12911b5 // bfdot za.s[x8, 5], { z13.h-z14.h }, z9.h\n" - ".inst 0x658aaaad // bfcvt z13.h, p2/M, z21.s\n" - ".inst 0x648aaa8d // bfcvtnt z13.h, p2/M, z20.s\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xc12a11d2 // bfdot za.s[x8, 2], { z14.h-z15.h }, z10.h\n" - ".inst 0xc12c11d3 // bfdot za.s[x8, 3], { z14.h-z15.h }, z12.h\n" - ".inst 0xc12511d4 // bfdot za.s[x8, 4], { z14.h-z15.h }, z5.h\n" - ".inst 0xc12611d5 // bfdot za.s[x8, 5], { z14.h-z15.h }, z6.h\n" - ".inst 0x658aaa6e // bfcvt z14.h, p2/M, z19.s\n" - ".inst 0x658aaa2f // bfcvt z15.h, p2/M, z17.s\n" - ".inst 0xc0060818 // mova { z24.d-z25.d }, za.d[x8, #0]\n" - ".inst 0x648aaa4e // bfcvtnt z14.h, p2/M, z18.s\n" - ".inst 0x648aaa0f // bfcvtnt z15.h, p2/M, z16.s\n" - ".inst 0xc006083a // mova { z26.d-z27.d }, za.d[x8, #1]\n" + ".inst 0xc12612d5 // bfdot za.s[x8, 5], { z22.h-z23.h }, z6.h\n" + ".inst 0x658aaa96 // bfcvt z22.h, p2/M, z20.s\n" + ".inst 0x648aaa76 // bfcvtnt z22.h, p2/M, z19.s\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + ".inst 0xc12412f2 // bfdot za.s[x8, 2], { z23.h-z24.h }, z4.h\n" + ".inst 0xc12912f3 // bfdot za.s[x8, 3], { z23.h-z24.h }, z9.h\n" + ".inst 0xc12c12f4 // bfdot za.s[x8, 4], { z23.h-z24.h }, z12.h\n" + ".inst 0xc12812f5 // bfdot za.s[x8, 5], { z23.h-z24.h }, z8.h\n" + ".inst 0x658aaa37 // bfcvt z23.h, p2/M, z17.s\n" + ".inst 0x658aaa18 // bfcvt z24.h, p2/M, z16.s\n" + ".inst 0xc0060810 // mova { z16.d-z17.d }, za.d[x8, #0]\n" + ".inst 0x648aaa57 // bfcvtnt z23.h, p2/M, z18.s\n" + ".inst 0x648aa9f8 // bfcvtnt z24.h, p2/M, z15.s\n" + ".inst 0xc0060832 // mova { z18.d-z19.d }, za.d[x8, #1]\n" "add x8, x8, #0x2\n" - ".inst 0xc1bdcb98 // fclamp { z24.s-z27.s }, z28.s, z29.s\n" - "st1w { z24.s }, p1, [x9]\n" + ".inst 0xc1adcb30 // fclamp { z16.s-z19.s }, z25.s, z13.s\n" + "st1w { z16.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - "st1w { z26.s }, p1, [x28]\n" + "st1w { z18.s }, p1, [x28]\n" "add x28, x28, x26, LSL #2\n" - ".inst 0xc0040ac4 // mova za.d[x8, #4], { z22.d-z23.d }\n" - "st1w { z25.s }, p1, [x25]\n" + ".inst 0xc0040b44 // mova za.d[x8, #4], { z26.d-z27.d }\n" + "st1w { z17.s }, p1, [x25]\n" "add x25, x25, x23, LSL #2\n" - ".inst 0xc0040ac5 // mova za.d[x8, #5], { z22.d-z23.d }\n" - "st1w { z27.s }, p1, [x24]\n" + ".inst 0xc0040b45 // mova za.d[x8, #5], { z26.d-z27.d }\n" + "st1w { z19.s }, p1, [x24]\n" "add x24, x24, x22, LSL #2\n" "bgt 14b\n" "15:" // Main loop tail - ".inst 0xc12411b0 // bfdot za.s[x8, 0], { z13.h-z14.h }, z4.h\n" - ".inst 0xc12711b1 // bfdot za.s[x8, 1], { z13.h-z14.h }, z7.h\n" - ".inst 0xc12311d0 // bfdot za.s[x8, 0], { z14.h-z15.h }, z3.h\n" - ".inst 0xc12011d1 // bfdot za.s[x8, 1], { z14.h-z15.h }, z0.h\n" - ".inst 0xc12111b2 // bfdot za.s[x8, 2], { z13.h-z14.h }, z1.h\n" - ".inst 0xc12211b3 // bfdot za.s[x8, 3], { z13.h-z14.h }, z2.h\n" - ".inst 0xc12811b4 // bfdot za.s[x8, 4], { z13.h-z14.h }, z8.h\n" - ".inst 0xc12911b5 // bfdot za.s[x8, 5], { z13.h-z14.h }, z9.h\n" - ".inst 0xc0060818 // mova { z24.d-z25.d }, za.d[x8, #0]\n" - ".inst 0xc006083a // mova { z26.d-z27.d }, za.d[x8, #1]\n" - ".inst 0xc1bdcb98 // fclamp { z24.s-z27.s }, z28.s, z29.s\n" - "st1w { z24.s }, p1, [x9]\n" + ".inst 0xc12112d0 // bfdot za.s[x8, 0], { z22.h-z23.h }, z1.h\n" + ".inst 0xc12e12d1 // bfdot za.s[x8, 1], { z22.h-z23.h }, z14.h\n" + ".inst 0xc12212f0 // bfdot za.s[x8, 0], { z23.h-z24.h }, z2.h\n" + ".inst 0xc12712f1 // bfdot za.s[x8, 1], { z23.h-z24.h }, z7.h\n" + ".inst 0xc12a12d2 // bfdot za.s[x8, 2], { z22.h-z23.h }, z10.h\n" + ".inst 0xc12b12d3 // bfdot za.s[x8, 3], { z22.h-z23.h }, z11.h\n" + ".inst 0xc12512d4 // bfdot za.s[x8, 4], { z22.h-z23.h }, z5.h\n" + ".inst 0xc12612d5 // bfdot za.s[x8, 5], { z22.h-z23.h }, z6.h\n" + ".inst 0xc0060810 // mova { z16.d-z17.d }, za.d[x8, #0]\n" + ".inst 0xc0060832 // mova { z18.d-z19.d }, za.d[x8, #1]\n" + ".inst 0xc1adcb30 // fclamp { z16.s-z19.s }, z25.s, z13.s\n" + "st1w { z16.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - ".inst 0xc12a11d2 // bfdot za.s[x8, 2], { z14.h-z15.h }, z10.h\n" - "st1w { z26.s }, p1, [x28]\n" + ".inst 0xc12412f2 // bfdot za.s[x8, 2], { z23.h-z24.h }, z4.h\n" + "st1w { z18.s }, p1, [x28]\n" "add x28, x28, x26, LSL #2\n" - ".inst 0xc12c11d3 // bfdot za.s[x8, 3], { z14.h-z15.h }, z12.h\n" - "st1w { z25.s }, p1, [x25]\n" + ".inst 0xc12912f3 // bfdot za.s[x8, 3], { z23.h-z24.h }, z9.h\n" + "st1w { z17.s }, p1, [x25]\n" "add x25, x25, x23, LSL #2\n" - ".inst 0xc12511d4 // bfdot za.s[x8, 4], { z14.h-z15.h }, z5.h\n" - "st1w { z27.s }, p1, [x24]\n" + ".inst 0xc12c12f4 // bfdot za.s[x8, 4], { z23.h-z24.h }, z12.h\n" + "st1w { z19.s }, p1, [x24]\n" "add x24, x24, x22, LSL #2\n" - ".inst 0xc12611d5 // bfdot za.s[x8, 5], { z14.h-z15.h }, z6.h\n" + ".inst 0xc12812f5 // bfdot za.s[x8, 5], { z23.h-z24.h }, z8.h\n" "add x8, x8, #0x2\n" - ".inst 0xc0040ac4 // mova za.d[x8, #4], { z22.d-z23.d }\n" - ".inst 0xc0040ac5 // mova za.d[x8, #5], { z22.d-z23.d }\n" + ".inst 0xc0040b44 // mova za.d[x8, #4], { z26.d-z27.d }\n" + ".inst 0xc0040b45 // mova za.d[x8, #5], { z26.d-z27.d }\n" "16:" // Main loop skip tail - "cbz x13, 18f\n" + "cbz x11, 18f\n" "17:" // Right padding loop - ".inst 0xc0060818 // mova { z24.d-z25.d }, za.d[x8, #0]\n" - "subs x13, x13, #0x1\n" - ".inst 0xc006083a // mova { z26.d-z27.d }, za.d[x8, #1]\n" + ".inst 0xc006081c // mova { z28.d-z29.d }, za.d[x8, #0]\n" + "subs x11, x11, #0x1\n" + ".inst 0xc006083e // mova { z30.d-z31.d }, za.d[x8, #1]\n" "add x8, x8, #0x2\n" - ".inst 0xc1bdcb98 // fclamp { z24.s-z27.s }, z28.s, z29.s\n" - "st1w { z24.s }, p1, [x9]\n" + ".inst 0xc1adcb3c // fclamp { z28.s-z31.s }, z25.s, z13.s\n" + "st1w { z28.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - "st1w { z26.s }, p1, [x28]\n" + "st1w { z30.s }, p1, [x28]\n" "add x28, x28, x26, LSL #2\n" - ".inst 0xc0040ac4 // mova za.d[x8, #4], { z22.d-z23.d }\n" - "st1w { z25.s }, p1, [x25]\n" + ".inst 0xc0040b44 // mova za.d[x8, #4], { z26.d-z27.d }\n" + "st1w { z29.s }, p1, [x25]\n" "add x25, x25, x23, LSL #2\n" - ".inst 0xc0040ac5 // mova za.d[x8, #5], { z22.d-z23.d }\n" - "st1w { z27.s }, p1, [x24]\n" + ".inst 0xc0040b45 // mova za.d[x8, #5], { z26.d-z27.d }\n" + "st1w { z31.s }, p1, [x24]\n" "add x24, x24, x22, LSL #2\n" "bgt 17b\n" "18:" // End "ldr x20, [%x[args], %[offsetof_Args_weights]]\n" "incb x20, ALL, MUL #9\n" "str x20, [%x[args], %[offsetof_Args_weights]]\n" - "incw x16\n" - "ldr x20, [%x[args], %[offsetof_Args_ld_in_vl]]\n" - "whilelt p1.s, x16, x17\n" - "ldr x14, [%x[args], %[offsetof_Args_inptr]]\n" - "add x14, x14, x20, LSL #2\n" - "str x14, [%x[args], %[offsetof_Args_inptr]]\n" - "ldr x11, [%x[args], %[offsetof_Args_outptrs]]\n" + "incw x15\n" + "ldr x21, [%x[args], %[offsetof_Args_ld_in_vl]]\n" + "whilelt p1.s, x15, x16\n" + "ldr x20, [%x[args], %[offsetof_Args_inptr]]\n" + "add x20, x20, x21, LSL #2\n" + "str x20, [%x[args], %[offsetof_Args_inptr]]\n" + "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" "ldr x24, [%x[args], %[offsetof_Args_ld_out_vls]]\n" - "ldp x23, x22, [x11, #0x0]\n" + "ldp x23, x22, [x25, #0x0]\n" "ldp x21, x20, [x24, #0x0]\n" "add x23, x23, x21, LSL #2\n" "add x22, x22, x20, LSL #2\n" - "stp x23, x22, [x11, #0x0]\n" - "ldp x23, x22, [x11, #0x10]\n" + "stp x23, x22, [x25, #0x0]\n" + "ldp x23, x22, [x25, #0x10]\n" "ldp x21, x20, [x24, #0x10]\n" "add x23, x23, x21, LSL #2\n" "add x22, x22, x20, LSL #2\n" - "stp x23, x22, [x11, #0x10]\n" + "stp x23, x22, [x25, #0x10]\n" "b.any 1b\n" ".inst 0xd503467f // SMSTOP\n" : : [args] "r" (&args), [ld_in_col] "r" (ld_in_col), [ld_in_row] "r" (ld_in_row), [offsetof_Args_bias] "I" (offsetof(Args, bias)), [offsetof_Args_clamp_max] "I" (offsetof(Args, clamp_max)), [offsetof_Args_clamp_min] "I" (offsetof(Args, clamp_min)), [offsetof_Args_current_channel] "I" (offsetof(Args, current_channel)), [offsetof_Args_inptr] "I" (offsetof(Args, inptr)), [offsetof_Args_input_cols] "I" (offsetof(Args, input_cols)), [offsetof_Args_ld_in_vl] "I" (offsetof(Args, ld_in_vl)), [offsetof_Args_ld_out_cols] "I" (offsetof(Args, ld_out_cols)), [offsetof_Args_ld_out_vls] "I" (offsetof(Args, ld_out_vls)), [offsetof_Args_n_channels] "I" (offsetof(Args, n_channels)), [offsetof_Args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_Args_output_cols] "I" (offsetof(Args, output_cols)), [offsetof_Args_pad_bottom] "I" (offsetof(Args, pad_bottom)), [offsetof_Args_pad_left] "I" (offsetof(Args, pad_left)), [offsetof_Args_pad_top] "I" (offsetof(Args, pad_top)), [offsetof_Args_weights] "I" (offsetof(Args, weights)) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SME2) +#endif // defined(ARM_COMPUTE_ENABLE_SME2) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_3x3_s2_4rows_dot_za.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_3x3_s2_4rows_dot_za.hpp index 89b9199084bd591c4a8bcdbe9e2a8d22881095d5..e685884762d104f511ca915dcee7c0ff39e1362b 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_3x3_s2_4rows_dot_za.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_3x3_s2_4rows_dot_za.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,8 @@ * SOFTWARE. */ +#include "src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp" + namespace arm_conv { namespace depthwise { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_3x3_s2_4rows_dot_za/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_3x3_s2_4rows_dot_za/generic.cpp index e8c9bfeb2966913d17cb3aff230fa4bc098a3ac9..a3b9ca402ab337d928aed2f8f912c56add13ed88 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_3x3_s2_4rows_dot_za/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_3x3_s2_4rows_dot_za/generic.cpp @@ -69,89 +69,89 @@ void sme2_fp32bf16fp32_planar_3x3_s2_4rows_dot_za_impl( Args args = { inptr, ld_in_vl, pad_top, 9u - std::min(9u, pad_top + valid_input_rows), pad_left, weights, bias, valid_input_cols, output_cols, outptrs, outlds, outvllds, start_channel, valid_channels, act_min, act_max }; __asm__ __volatile__( - "ldr x6, [%x[args], %[offsetof_Args_pad_bottom]]\n" + "ldr x7, [%x[args], %[offsetof_Args_pad_bottom]]\n" "mov x20, #0x9\n" ".inst 0xd503477f // SMSTART ZA\n" - "sub x20, x20, x6\n" - "ldr x7, [%x[args], %[offsetof_Args_pad_top]]\n" + "sub x20, x20, x7\n" + "ldr x17, [%x[args], %[offsetof_Args_pad_top]]\n" "ptrue p2.b\n" - "ld1rw { z27.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_min]]\n" - "ldr x17, [%x[args], %[offsetof_Args_n_channels]]\n" - "whilelt p1.s, XZR, x17\n" + "ld1rw { z4.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_min]]\n" + "ldr x16, [%x[args], %[offsetof_Args_n_channels]]\n" + "whilelt p1.s, XZR, x16\n" "whilelt p9.s, XZR, x20\n" - "ld1rw { z23.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_max]]\n" - "whilelt p8.s, XZR, x7\n" + "ld1rw { z1.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_max]]\n" + "whilelt p8.s, XZR, x17\n" "eor p8.b, p2/Z, p8.b, p9.b\n" - "ldr x16, [%x[args], %[offsetof_Args_current_channel]]\n" + "ldr x15, [%x[args], %[offsetof_Args_current_channel]]\n" "1:" // Channel loop "ldr x20, [%x[args], %[offsetof_Args_bias]]\n" - "fmov z4.s, #0x0\n" + "fmov z24.s, #0x0\n" "cbz x20, 2f\n" - "ld1w { z4.s }, p1/Z, [x20, x16, LSL #2]\n" + "ld1w { z24.s }, p1/Z, [x20, x15, LSL #2]\n" "2:" // Load bias: Done - "ldr x20, [%x[args], %[offsetof_Args_weights]]\n" - "mov x21, x20\n" - "ld1w { z19.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #3\n" - "incb x20\n" - "ld1w { z24.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #3\n" - ".inst 0x658aaa69 // bfcvt z9.h, p2/M, z19.s\n" - "ld1w { z12.s }, p2/Z, [x21]\n" - "mov x21, x20\n" - ".inst 0x648aab09 // bfcvtnt z9.h, p2/M, z24.s\n" - "incb x20\n" - "ld1w { z19.s }, p2/Z, [x21]\n" + "ldr x21, [%x[args], %[offsetof_Args_weights]]\n" + "mov x20, x21\n" + "ld1w { z18.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #3\n" + "incb x21\n" + "ld1w { z23.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #3\n" + ".inst 0x658aaa4e // bfcvt z14.h, p2/M, z18.s\n" + "ld1w { z6.s }, p2/Z, [x20]\n" + "mov x20, x21\n" + ".inst 0x648aaaee // bfcvtnt z14.h, p2/M, z23.s\n" + "incb x21\n" + "ld1w { z28.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #3\n" + ".inst 0x658aa8c3 // bfcvt z3.h, p2/M, z6.s\n" + ".inst 0x658aab88 // bfcvt z8.h, p2/M, z28.s\n" + "ld1w { z10.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #3\n" + "ldr x14, [%x[args], %[offsetof_Args_input_cols]]\n" + ".inst 0x648aa948 // bfcvtnt z8.h, p2/M, z10.s\n" + "ld1w { z2.s }, p2/Z, [x20]\n" + "mov x21, x21\n" + ".inst 0x658aa847 // bfcvt z7.h, p2/M, z2.s\n" + "ldr x13, [%x[args], %[offsetof_Args_inptr]]\n" + "ld1w { z9.s }, p2/Z, [x21]\n" "incb x21, ALL, MUL #3\n" - ".inst 0x658aa983 // bfcvt z3.h, p2/M, z12.s\n" - ".inst 0x658aaa62 // bfcvt z2.h, p2/M, z19.s\n" - "ld1w { z24.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #3\n" - "ldr x15, [%x[args], %[offsetof_Args_input_cols]]\n" - ".inst 0x648aab02 // bfcvtnt z2.h, p2/M, z24.s\n" - "ld1w { z12.s }, p2/Z, [x21]\n" - "mov x21, x20\n" - ".inst 0x658aa980 // bfcvt z0.h, p2/M, z12.s\n" - "ldr x14, [%x[args], %[offsetof_Args_inptr]]\n" - "ld1w { z19.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #3\n" - ".inst 0x658aaa6a // bfcvt z10.h, p2/M, z19.s\n" - "sub x20, x15, #0x1\n" - "ld1w { z24.s }, p2/Z, [x21]\n" + ".inst 0x658aa920 // bfcvt z0.h, p2/M, z9.s\n" + "sub x20, x14, #0x1\n" + "ld1w { z6.s }, p2/Z, [x21]\n" "incb x21, ALL, MUL #3\n" "orr x23, x20, %x[ld_in_col], LSL #18\n" - "mov z5.d, z4.d\n" - "ld1w { z12.s }, p2/Z, [x21]\n" - "orr x23, x17, x23, LSL #20\n" + "mov z25.d, z24.d\n" + "ld1w { z17.s }, p2/Z, [x21]\n" + "orr x23, x16, x23, LSL #20\n" "mov x22, #0x9\n" - "mov z6.d, z4.d\n" - "add x21, x7, x6\n" + "mov z26.d, z24.d\n" + "add x21, x17, x7\n" "lsl x20, %x[ld_in_row], #0x2\n" - "mov z7.d, z4.d\n" - ".inst 0x648aab0a // bfcvtnt z10.h, p2/M, z24.s\n" - ".inst 0x658aa981 // bfcvt z1.h, p2/M, z12.s\n" + "mov z27.d, z24.d\n" + ".inst 0x648aa8c0 // bfcvtnt z0.h, p2/M, z6.s\n" + ".inst 0x658aaa26 // bfcvt z6.h, p2/M, z17.s\n" "mov x8, #0x0\n" - "ldr x13, [%x[args], %[offsetof_Args_output_cols]]\n" + "ldr x11, [%x[args], %[offsetof_Args_output_cols]]\n" "lsl x23, x23, #0x2\n" "sub x22, x22, x21\n" - "madd x20, x20, x7, x14\n" + "madd x20, x20, x17, x13\n" "3:" // Issue prefetches "subs x22, x22, #0x1\n" ".inst 0xf8b74a9c // rprfm pldstrm, x23, [x20]\n" "add x20, x20, %x[ld_in_col], LSL #2\n" "bgt 3b\n" - "ldr x11, [%x[args], %[offsetof_Args_outptrs]]\n" + "ldr x23, [%x[args], %[offsetof_Args_outptrs]]\n" "lsl x20, %x[ld_in_row], #0x2\n" - "msub x14, x7, x20, x14\n" - ".inst 0xc0040c80 // mova za.d[x8, #0], { z4.d-z7.d }\n" + "msub x13, x17, x20, x13\n" + ".inst 0xc0040f00 // mova za.d[x8, #0], { z24.d-z27.d }\n" "ldr x20, [%x[args], %[offsetof_Args_ld_out_cols]]\n" - ".inst 0xc0040c81 // mova za.d[x8, #1], { z4.d-z7.d }\n" + ".inst 0xc0040f01 // mova za.d[x8, #1], { z24.d-z27.d }\n" "mov x22, #0x2\n" - "ldp x10, x9, [x11], #0x10\n" - ".inst 0xc0040c82 // mova za.d[x8, #2], { z4.d-z7.d }\n" + "ldp x10, x9, [x23], #0x10\n" + ".inst 0xc0040f02 // mova za.d[x8, #2], { z24.d-z27.d }\n" "ldp x28, x27, [x20], #0x10\n" "ldr x21, [%x[args], %[offsetof_Args_pad_left]]\n" - "ldp x26, x25, [x11], #0x10\n" + "ldp x26, x25, [x23], #0x10\n" "ldp x24, x23, [x20], #0x10\n" "cbz x21, 5f\n" "cmp x21, x22\n" @@ -159,396 +159,396 @@ void sme2_fp32bf16fp32_planar_3x3_s2_4rows_dot_za_impl( "sub x21, x21, x20\n" "sub x22, x22, x20\n" "cbz x21, 5f\n" - ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" + ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" "and x22, x21, #0x1\n" "add x21, x21, #0x1\n" - ".inst 0xc1b7cb7c // fclamp { z28.s-z31.s }, z27.s, z23.s\n" + ".inst 0xc1a1c890 // fclamp { z16.s-z19.s }, z4.s, z1.s\n" "lsr x21, x21, #0x1\n" - "sub x13, x13, x21\n" + "sub x11, x11, x21\n" "4:" // Left padding "subs x21, x21, #0x1\n" - "st1w { z28.s }, p1, [x10]\n" + "st1w { z16.s }, p1, [x10]\n" "add x10, x10, x28, LSL #2\n" - "st1w { z29.s }, p1, [x9]\n" + "st1w { z17.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - "st1w { z30.s }, p1, [x26]\n" + "st1w { z18.s }, p1, [x26]\n" "add x26, x26, x24, LSL #2\n" - "st1w { z31.s }, p1, [x25]\n" + "st1w { z19.s }, p1, [x25]\n" "add x25, x25, x23, LSL #2\n" "bgt 4b\n" "5:" // Left padding: End - "adds XZR, x7, x6\n" + "adds XZR, x17, x7\n" "bne 10f\n" "cbz x22, 8f\n" "cmp x22, #0x1\n" - "sub x15, x15, x22\n" + "sub x14, x14, x22\n" "beq 7f\n" "6:" // Unpadded: 2 priming loads - "add x20, x14, %x[ld_in_row], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x14]\n" - ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" + "ld1w { z18.s }, p1/Z, [x13]\n" + ".inst 0x658aaa53 // bfcvt z19.h, p2/M, z18.s\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + "ld1w { z12.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0b // bfcvtnt z11.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + ".inst 0x648aa993 // bfcvtnt z19.h, p2/M, z12.s\n" + "ld1w { z23.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + ".inst 0x658aaaf4 // bfcvt z20.h, p2/M, z23.s\n" + "ld1w { z2.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + ".inst 0x648aa854 // bfcvtnt z20.h, p2/M, z2.s\n" + "ld1w { z15.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + ".inst 0x658aa9f5 // bfcvt z21.h, p2/M, z15.s\n" "ld1w { z22.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaace // bfcvt z14.h, p2/M, z22.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + ".inst 0x648aaad5 // bfcvtnt z21.h, p2/M, z22.s\n" + "ld1w { z30.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" - ".inst 0xc1391170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z9.h\n" - "ld1w { z16.s }, p1/Z, [x20]\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - ".inst 0xc1331190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z3.h\n" + ".inst 0x658aabd6 // bfcvt z22.h, p2/M, z30.s\n" + "ld1w { z12.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0x648aa996 // bfcvtnt z22.h, p2/M, z12.s\n" + ".inst 0xc13e1270 // bfdot za.s[x8, 0], { z19.h-z22.h }, z14.h\n" + "ld1w { z31.s }, p1/Z, [x20]\n" + ".inst 0x658aabf7 // bfcvt z23.h, p2/M, z31.s\n" + ".inst 0xc1331290 // bfdot za.s[x8, 0], { z20.h-z23.h }, z3.h\n" "7:" // Unpadded: 1 priming loads - "add x20, x14, %x[ld_in_row], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x14]\n" - ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" + "ld1w { z17.s }, p1/Z, [x13]\n" + ".inst 0x658aaa30 // bfcvt z16.h, p2/M, z17.s\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + "ld1w { z22.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0b // bfcvtnt z11.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + ".inst 0x648aaad0 // bfcvtnt z16.h, p2/M, z22.s\n" + "ld1w { z28.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + ".inst 0x658aab91 // bfcvt z17.h, p2/M, z28.s\n" + "ld1w { z18.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + ".inst 0x648aaa51 // bfcvtnt z17.h, p2/M, z18.s\n" + "ld1w { z2.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + ".inst 0x658aa852 // bfcvt z18.h, p2/M, z2.s\n" + "ld1w { z19.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + ".inst 0x648aaa72 // bfcvtnt z18.h, p2/M, z19.s\n" + "ld1w { z2.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + ".inst 0x658aa853 // bfcvt z19.h, p2/M, z2.s\n" + "ld1w { z23.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" - ".inst 0xc1321170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1w { z16.s }, p1/Z, [x20]\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - ".inst 0xc1301190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z0.h\n" + ".inst 0x648aaaf3 // bfcvtnt z19.h, p2/M, z23.s\n" + ".inst 0xc1381210 // bfdot za.s[x8, 0], { z16.h-z19.h }, z8.h\n" + "ld1w { z10.s }, p1/Z, [x20]\n" + ".inst 0x658aa954 // bfcvt z20.h, p2/M, z10.s\n" + ".inst 0xc1371230 // bfdot za.s[x8, 0], { z17.h-z20.h }, z7.h\n" "8:" // Unpadded: 0 priming loads - "cmp x15, #0x2\n" + "cmp x14, #0x2\n" "blt 16f\n" - "add x21, x14, %x[ld_in_row], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x14]\n" - ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" - "sub x15, x15, #0x2\n" + "add x21, x13, %x[ld_in_row], LSL #2\n" + "ld1w { z16.s }, p1/Z, [x13]\n" + ".inst 0x658aaa09 // bfcvt z9.h, p2/M, z16.s\n" + "sub x14, x14, #0x2\n" "ld1w { z16.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - "sub x13, x13, #0x1\n" - ".inst 0x648aaa0b // bfcvtnt z11.h, p2/M, z16.s\n" + "sub x11, x11, #0x1\n" + ".inst 0x648aaa09 // bfcvtnt z9.h, p2/M, z16.s\n" "ld1w { z16.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" - "lsr x20, x15, #0x1\n" + ".inst 0x658aaa0a // bfcvt z10.h, p2/M, z16.s\n" + "lsr x20, x14, #0x1\n" "ld1w { z16.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - "cmp x20, x13\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + "cmp x20, x11\n" + ".inst 0x648aaa0a // bfcvtnt z10.h, p2/M, z16.s\n" "ld1w { z16.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" - "csel x22, x20, x13, LT\n" + ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" + "csel x22, x20, x11, LT\n" "ld1w { z16.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" + ".inst 0x648aaa0b // bfcvtnt z11.h, p2/M, z16.s\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" "ld1w { z16.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" - "and x15, x15, #0x1\n" + ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + "and x14, x14, #0x1\n" "ld1w { z16.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" - "sub x13, x13, x22\n" + ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + "sub x11, x11, x22\n" "ld1w { z16.s }, p1/Z, [x21]\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" + ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" "cbz x22, 15f\n" "9:" // Unpadded: Main loop - "add x21, x14, %x[ld_in_row], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x14]\n" - ".inst 0xc13a1170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z10.h\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - "ld1w { z18.s }, p1/Z, [x21]\n" + "add x21, x13, %x[ld_in_row], LSL #2\n" + "ld1w { z16.s }, p1/Z, [x13]\n" + ".inst 0xc1301130 // bfdot za.s[x8, 0], { z9.h-z12.h }, z0.h\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + "ld1w { z15.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0xc1391171 // bfdot za.s[x8, 1], { z11.h-z14.h }, z9.h\n" - ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0xc13e1131 // bfdot za.s[x8, 1], { z9.h-z12.h }, z14.h\n" + ".inst 0x658aaa09 // bfcvt z9.h, p2/M, z16.s\n" + "ld1w { z18.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0xc1311190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z1.h\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + ".inst 0xc1361150 // bfdot za.s[x8, 0], { z10.h-z13.h }, z6.h\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" "ld1w { z17.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0xc1331191 // bfdot za.s[x8, 1], { z12.h-z15.h }, z3.h\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0xc1331151 // bfdot za.s[x8, 1], { z10.h-z13.h }, z3.h\n" + ".inst 0x658aaa4a // bfcvt z10.h, p2/M, z18.s\n" + "ld1w { z30.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" - ".inst 0x648aaa4b // bfcvtnt z11.h, p2/M, z18.s\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0x658aabcb // bfcvt z11.h, p2/M, z30.s\n" + ".inst 0x648aa9e9 // bfcvtnt z9.h, p2/M, z15.s\n" + "ld1w { z19.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa2c // bfcvtnt z12.h, p2/M, z17.s\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + ".inst 0x648aaa2a // bfcvtnt z10.h, p2/M, z17.s\n" + ".inst 0x648aaa6b // bfcvtnt z11.h, p2/M, z19.s\n" "ld1w { z16.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" - ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" + "ld1w { z2.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" + ".inst 0x648aa84c // bfcvtnt z12.h, p2/M, z2.s\n" "add x8, x8, #0x1\n" - "ld1w { z16.s }, p1/Z, [x14]\n" - ".inst 0xc1321170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" + "ld1w { z29.s }, p1/Z, [x13]\n" + ".inst 0xc1381130 // bfdot za.s[x8, 0], { z9.h-z12.h }, z8.h\n" + ".inst 0x658aaba9 // bfcvt z9.h, p2/M, z29.s\n" "subs x22, x22, #0x1\n" - "ld1w { z20.s }, p1/Z, [x20]\n" + "ld1w { z22.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1b7cb7c // fclamp { z28.s-z31.s }, z27.s, z23.s\n" - "st1w { z28.s }, p1, [x10]\n" + ".inst 0xc1a1c890 // fclamp { z16.s-z19.s }, z4.s, z1.s\n" + "st1w { z16.s }, p1, [x10]\n" "ld1w { z16.s }, p1/Z, [x21]\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - ".inst 0xc1301190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z0.h\n" + ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0xc1371150 // bfdot za.s[x8, 0], { z10.h-z13.h }, z7.h\n" "add x10, x10, x28, LSL #2\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + "ld1w { z28.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" - "st1w { z29.s }, p1, [x9]\n" - "ld1w { z19.s }, p1/Z, [x20]\n" + ".inst 0x658aab8a // bfcvt z10.h, p2/M, z28.s\n" + "st1w { z17.s }, p1, [x9]\n" + "ld1w { z31.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "add x9, x9, x27, LSL #2\n" - "st1w { z30.s }, p1, [x26]\n" + "st1w { z18.s }, p1, [x26]\n" "ld1w { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" "add x26, x26, x24, LSL #2\n" - "ld1w { z18.s }, p1/Z, [x20]\n" + "ld1w { z17.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "st1w { z31.s }, p1, [x25]\n" + "st1w { z19.s }, p1, [x25]\n" "add x25, x25, x23, LSL #2\n" "ld1w { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" - ".inst 0xc0040c82 // mova za.d[x8, #2], { z4.d-z7.d }\n" - "ld1w { z17.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa8b // bfcvtnt z11.h, p2/M, z20.s\n" - ".inst 0x648aaa6c // bfcvtnt z12.h, p2/M, z19.s\n" + ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + ".inst 0xc0040f02 // mova za.d[x8, #2], { z24.d-z27.d }\n" "ld1w { z16.s }, p1/Z, [x20]\n" - ".inst 0x648aaa4d // bfcvtnt z13.h, p2/M, z18.s\n" - ".inst 0x648aaa2e // bfcvtnt z14.h, p2/M, z17.s\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" + "add x20, x20, %x[ld_in_row], LSL #2\n" + ".inst 0x648aaac9 // bfcvtnt z9.h, p2/M, z22.s\n" + ".inst 0x648aabea // bfcvtnt z10.h, p2/M, z31.s\n" + "ld1w { z31.s }, p1/Z, [x20]\n" + ".inst 0x648aaa2b // bfcvtnt z11.h, p2/M, z17.s\n" + ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + ".inst 0x658aabed // bfcvt z13.h, p2/M, z31.s\n" "bgt 9b\n" "b 15f\n" "10:" // Padded "cbz x22, 13f\n" "cmp x22, #0x1\n" - "sub x15, x15, x22\n" + "sub x14, x14, x22\n" "beq 12f\n" "11:" // Padded: 2 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x14]\n" - ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + "ld1w { z16.s }, p0/Z, [x13]\n" + ".inst 0x658aaa09 // bfcvt z9.h, p2/M, z16.s\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x648aaa0b // bfcvtnt z11.h, p2/M, z16.s\n" + "ld1w { z18.s }, p0/Z, [x20]\n" + ".inst 0x648aaa49 // bfcvtnt z9.h, p2/M, z18.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + "ld1w { z12.s }, p0/Z, [x20]\n" + ".inst 0x658aa98a // bfcvt z10.h, p2/M, z12.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + "ld1w { z12.s }, p0/Z, [x20]\n" + ".inst 0x648aa98a // bfcvtnt z10.h, p2/M, z12.s\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z18.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0x658aaa4b // bfcvt z11.h, p2/M, z18.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1w { z16.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + ".inst 0x648aaa0b // bfcvtnt z11.h, p2/M, z16.s\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1w { z16.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" + ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1w { z16.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" + ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - ".inst 0xc1391170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z9.h\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xc1331190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z3.h\n" + ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0xc13e1130 // bfdot za.s[x8, 0], { z9.h-z12.h }, z14.h\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + ".inst 0xc1331150 // bfdot za.s[x8, 0], { z10.h-z13.h }, z3.h\n" "12:" // Padded: 1 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x14]\n" - ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + "ld1w { z16.s }, p0/Z, [x13]\n" + ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x648aaa0b // bfcvtnt z11.h, p2/M, z16.s\n" + ".inst 0x648aaa0f // bfcvtnt z15.h, p2/M, z16.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + ".inst 0x658aaa10 // bfcvt z16.h, p2/M, z16.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + "ld1w { z19.s }, p0/Z, [x20]\n" + ".inst 0x648aaa70 // bfcvtnt z16.h, p2/M, z19.s\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z13.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0x658aa9b1 // bfcvt z17.h, p2/M, z13.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z12.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + ".inst 0x648aa991 // bfcvtnt z17.h, p2/M, z12.s\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z9.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" + ".inst 0x658aa932 // bfcvt z18.h, p2/M, z9.s\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z11.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" + ".inst 0x648aa972 // bfcvtnt z18.h, p2/M, z11.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - ".inst 0xc1321170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0xc1301190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z0.h\n" + "ld1w { z21.s }, p0/Z, [x20]\n" + ".inst 0x658aaab3 // bfcvt z19.h, p2/M, z21.s\n" + ".inst 0xc13811f0 // bfdot za.s[x8, 0], { z15.h-z18.h }, z8.h\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + ".inst 0xc1371210 // bfdot za.s[x8, 0], { z16.h-z19.h }, z7.h\n" "13:" // Padded: 0 priming loads - "cmp x15, #0x2\n" + "cmp x14, #0x2\n" "blt 16f\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x14]\n" - ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + "ld1w { z16.s }, p0/Z, [x13]\n" + ".inst 0x658aaa09 // bfcvt z9.h, p2/M, z16.s\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x648aaa0b // bfcvtnt z11.h, p2/M, z16.s\n" + ".inst 0x648aaa09 // bfcvtnt z9.h, p2/M, z16.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + ".inst 0x658aaa0a // bfcvt z10.h, p2/M, z16.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + ".inst 0x648aaa0a // bfcvtnt z10.h, p2/M, z16.s\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1w { z16.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1w { z16.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + ".inst 0x648aaa0b // bfcvtnt z11.h, p2/M, z16.s\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1w { z16.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" + ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x8\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" + ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - "sub x15, x15, #0x2\n" - "sub x13, x13, #0x1\n" - "lsr x20, x15, #0x1\n" - "cmp x20, x13\n" - "csel x21, x20, x13, LT\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - "and x15, x15, #0x1\n" - "sub x13, x13, x21\n" + ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + "sub x14, x14, #0x2\n" + "sub x11, x11, #0x1\n" + "lsr x20, x14, #0x1\n" + "cmp x20, x11\n" + "csel x21, x20, x11, LT\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + "and x14, x14, #0x1\n" + "sub x11, x11, x21\n" "cbz x21, 15f\n" "14:" // Padded: Main loop "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z18.s }, p0/Z, [x14]\n" - ".inst 0xc13a1170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z10.h\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + "ld1w { z18.s }, p0/Z, [x13]\n" + ".inst 0xc1301130 // bfdot za.s[x8, 0], { z9.h-z12.h }, z0.h\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z19.s }, p0/Z, [x20]\n" - ".inst 0xc1391171 // bfdot za.s[x8, 1], { z11.h-z14.h }, z9.h\n" + "ld1w { z17.s }, p0/Z, [x20]\n" + ".inst 0xc13e1131 // bfdot za.s[x8, 1], { z9.h-z12.h }, z14.h\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0xc1311190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z1.h\n" + ".inst 0xc1361150 // bfdot za.s[x8, 0], { z10.h-z13.h }, z6.h\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z17.s }, p0/Z, [x20]\n" - ".inst 0xc1331191 // bfdot za.s[x8, 1], { z12.h-z15.h }, z3.h\n" + "ld1w { z19.s }, p0/Z, [x20]\n" + ".inst 0xc1331151 // bfdot za.s[x8, 1], { z10.h-z13.h }, z3.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa4b // bfcvt z11.h, p2/M, z18.s\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + ".inst 0x658aaa49 // bfcvt z9.h, p2/M, z18.s\n" + ".inst 0x658aaa0a // bfcvt z10.h, p2/M, z16.s\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z2.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0x658aa84b // bfcvt z11.h, p2/M, z2.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z18.s }, p0/Z, [x20]\n" + "ld1w { z15.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa6b // bfcvtnt z11.h, p2/M, z19.s\n" + ".inst 0x648aaa29 // bfcvtnt z9.h, p2/M, z17.s\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z28.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" + ".inst 0x658aab8c // bfcvt z12.h, p2/M, z28.s\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x8\n" - "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x648aaa2c // bfcvtnt z12.h, p2/M, z17.s\n" + "ld1w { z17.s }, p0/Z, [x20]\n" + ".inst 0x648aaa6a // bfcvtnt z10.h, p2/M, z19.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z17.s }, p0/Z, [x20]\n" - ".inst 0x648aaa4d // bfcvtnt z13.h, p2/M, z18.s\n" + "ld1w { z13.s }, p0/Z, [x20]\n" + ".inst 0x648aa9eb // bfcvtnt z11.h, p2/M, z15.s\n" "mov x12, #0x0\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + ".inst 0x648aaa2c // bfcvtnt z12.h, p2/M, z17.s\n" ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x14]\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa2f // bfcvt z15.h, p2/M, z17.s\n" + "ld1w { z16.s }, p0/Z, [x13]\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" + ".inst 0x658aa9ad // bfcvt z13.h, p2/M, z13.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1w { z21.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1b7cb7c // fclamp { z28.s-z31.s }, z27.s, z23.s\n" + ".inst 0xc1a1c89c // fclamp { z28.s-z31.s }, z4.s, z1.s\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1w { z17.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" @@ -563,197 +563,197 @@ void sme2_fp32bf16fp32_planar_3x3_s2_4rows_dot_za_impl( "st1w { z30.s }, p1, [x26]\n" "add x8, x8, #0x1\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1321170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" + ".inst 0xc1381130 // bfdot za.s[x8, 0], { z9.h-z12.h }, z8.h\n" + ".inst 0x658aaa09 // bfcvt z9.h, p2/M, z16.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1w { z18.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1301190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z0.h\n" + ".inst 0xc1371150 // bfdot za.s[x8, 0], { z10.h-z13.h }, z7.h\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1w { z16.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa2c // bfcvt z12.h, p2/M, z17.s\n" + ".inst 0x658aaa2a // bfcvt z10.h, p2/M, z17.s\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x8\n" "ld1w { z17.s }, p0/Z, [x20]\n" - ".inst 0x658aaa6d // bfcvt z13.h, p2/M, z19.s\n" + ".inst 0x658aaa6b // bfcvt z11.h, p2/M, z19.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" + ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" "ld1w { z16.s }, p0/Z, [x20]\n" "subs x21, x21, #0x1\n" "add x10, x10, x28, LSL #2\n" "st1w { z31.s }, p1, [x25]\n" - ".inst 0xc0040c82 // mova za.d[x8, #2], { z4.d-z7.d }\n" + ".inst 0xc0040f02 // mova za.d[x8, #2], { z24.d-z27.d }\n" "add x9, x9, x27, LSL #2\n" "add x26, x26, x24, LSL #2\n" - ".inst 0x648aaaab // bfcvtnt z11.h, p2/M, z21.s\n" - ".inst 0x648aaa8c // bfcvtnt z12.h, p2/M, z20.s\n" + ".inst 0x648aaaa9 // bfcvtnt z9.h, p2/M, z21.s\n" + ".inst 0x648aaa8a // bfcvtnt z10.h, p2/M, z20.s\n" "add x25, x25, x23, LSL #2\n" - ".inst 0x648aaa4d // bfcvtnt z13.h, p2/M, z18.s\n" - ".inst 0x648aaa2e // bfcvtnt z14.h, p2/M, z17.s\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" + ".inst 0x648aaa4b // bfcvtnt z11.h, p2/M, z18.s\n" + ".inst 0x648aaa2c // bfcvtnt z12.h, p2/M, z17.s\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" + ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" "bgt 14b\n" "15:" // Main loop tail "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z17.s }, p0/Z, [x14]\n" - ".inst 0xc13a1170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z10.h\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + "ld1w { z17.s }, p0/Z, [x13]\n" + ".inst 0xc1301130 // bfdot za.s[x8, 0], { z9.h-z12.h }, z0.h\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z19.s }, p0/Z, [x20]\n" - ".inst 0xc1391171 // bfdot za.s[x8, 1], { z11.h-z14.h }, z9.h\n" + "ld1w { z2.s }, p0/Z, [x20]\n" + ".inst 0xc13e1131 // bfdot za.s[x8, 1], { z9.h-z12.h }, z14.h\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0xc1311190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z1.h\n" + ".inst 0xc1361150 // bfdot za.s[x8, 0], { z10.h-z13.h }, z6.h\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z18.s }, p0/Z, [x20]\n" - ".inst 0xc1331191 // bfdot za.s[x8, 1], { z12.h-z15.h }, z3.h\n" + "ld1w { z23.s }, p0/Z, [x20]\n" + ".inst 0xc1331151 // bfdot za.s[x8, 1], { z10.h-z13.h }, z3.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa2b // bfcvt z11.h, p2/M, z17.s\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + ".inst 0x658aaa32 // bfcvt z18.h, p2/M, z17.s\n" + ".inst 0x658aaa13 // bfcvt z19.h, p2/M, z16.s\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1w { z16.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0x658aaa14 // bfcvt z20.h, p2/M, z16.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z17.s }, p0/Z, [x20]\n" + "ld1w { z15.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa6b // bfcvtnt z11.h, p2/M, z19.s\n" + ".inst 0x648aa852 // bfcvtnt z18.h, p2/M, z2.s\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1w { z16.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" + ".inst 0x658aaa15 // bfcvt z21.h, p2/M, z16.s\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1w { z16.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - ".inst 0x648aaa4c // bfcvtnt z12.h, p2/M, z18.s\n" + ".inst 0x648aaaf3 // bfcvtnt z19.h, p2/M, z23.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0x648aaa2d // bfcvtnt z13.h, p2/M, z17.s\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" + ".inst 0x648aa9f4 // bfcvtnt z20.h, p2/M, z15.s\n" + ".inst 0x648aaa15 // bfcvtnt z21.h, p2/M, z16.s\n" "ld1w { z16.s }, p0/Z, [x20]\n" ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - ".inst 0xc1321170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - ".inst 0xc1b7cb7c // fclamp { z28.s-z31.s }, z27.s, z23.s\n" + ".inst 0x658aaa16 // bfcvt z22.h, p2/M, z16.s\n" + ".inst 0xc1381250 // bfdot za.s[x8, 0], { z18.h-z21.h }, z8.h\n" + ".inst 0xc1a1c89c // fclamp { z28.s-z31.s }, z4.s, z1.s\n" "st1w { z28.s }, p1, [x10]\n" "add x10, x10, x28, LSL #2\n" "st1w { z29.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - ".inst 0xc0040c82 // mova za.d[x8, #2], { z4.d-z7.d }\n" - "add x14, x14, %x[ld_in_col], LSL #2\n" + ".inst 0xc0040f02 // mova za.d[x8, #2], { z24.d-z27.d }\n" + "add x13, x13, %x[ld_in_col], LSL #2\n" "st1w { z30.s }, p1, [x26]\n" "add x26, x26, x24, LSL #2\n" - ".inst 0xc1301190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z0.h\n" + ".inst 0xc1371270 // bfdot za.s[x8, 0], { z19.h-z22.h }, z7.h\n" "st1w { z31.s }, p1, [x25]\n" "add x25, x25, x23, LSL #2\n" "16:" // Main loop skip tail - "cbz x15, 17f\n" // Skip remainder inputs + "cbz x14, 17f\n" // Skip remainder inputs "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x14]\n" - ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" - "add x20, x14, %x[ld_in_row], LSL #2\n" + "ld1w { z16.s }, p0/Z, [x13]\n" + ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" + "add x20, x13, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x648aaa0b // bfcvtnt z11.h, p2/M, z16.s\n" + ".inst 0x648aaa0f // bfcvtnt z15.h, p2/M, z16.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + ".inst 0x658aaa10 // bfcvt z16.h, p2/M, z16.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + "ld1w { z2.s }, p0/Z, [x20]\n" + ".inst 0x648aa850 // bfcvtnt z16.h, p2/M, z2.s\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z10.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0x658aa951 // bfcvt z17.h, p2/M, z10.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z30.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + ".inst 0x648aabd1 // bfcvtnt z17.h, p2/M, z30.s\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z19.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" + ".inst 0x658aaa72 // bfcvt z18.h, p2/M, z19.s\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z19.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" + ".inst 0x648aaa72 // bfcvtnt z18.h, p2/M, z19.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - ".inst 0xc13a1170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z10.h\n" - "sub x13, x13, #0x1\n" - ".inst 0xc1311190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z1.h\n" - ".inst 0xc1391171 // bfdot za.s[x8, 1], { z11.h-z14.h }, z9.h\n" - ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" - ".inst 0xc1b7cb7c // fclamp { z28.s-z31.s }, z27.s, z23.s\n" - "st1w { z28.s }, p1, [x10]\n" + "ld1w { z19.s }, p0/Z, [x20]\n" + ".inst 0x658aaa73 // bfcvt z19.h, p2/M, z19.s\n" + ".inst 0xc13011f0 // bfdot za.s[x8, 0], { z15.h-z18.h }, z0.h\n" + "sub x11, x11, #0x1\n" + ".inst 0xc1361210 // bfdot za.s[x8, 0], { z16.h-z19.h }, z6.h\n" + ".inst 0xc13e11f1 // bfdot za.s[x8, 1], { z15.h-z18.h }, z14.h\n" + ".inst 0xc0060c08 // mova { z8.d-z11.d }, za.d[x8, #0]\n" + ".inst 0xc1a1c888 // fclamp { z8.s-z11.s }, z4.s, z1.s\n" + "st1w { z8.s }, p1, [x10]\n" "add x10, x10, x28, LSL #2\n" - ".inst 0xc1331191 // bfdot za.s[x8, 1], { z12.h-z15.h }, z3.h\n" + ".inst 0xc1331211 // bfdot za.s[x8, 1], { z16.h-z19.h }, z3.h\n" "add x8, x8, #0x1\n" - "st1w { z29.s }, p1, [x9]\n" + "st1w { z9.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - "st1w { z30.s }, p1, [x26]\n" + "st1w { z10.s }, p1, [x26]\n" "add x26, x26, x24, LSL #2\n" - ".inst 0xc0040c82 // mova za.d[x8, #2], { z4.d-z7.d }\n" - "st1w { z31.s }, p1, [x25]\n" + ".inst 0xc0040f02 // mova za.d[x8, #2], { z24.d-z27.d }\n" + "st1w { z11.s }, p1, [x25]\n" "add x25, x25, x23, LSL #2\n" "17:" // Tail input: End - "cbz x13, 19f\n" + "cbz x11, 19f\n" "18:" // Right padding loop - ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" + ".inst 0xc0060c08 // mova { z8.d-z11.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" - "subs x13, x13, #0x1\n" - ".inst 0xc1b7cb7c // fclamp { z28.s-z31.s }, z27.s, z23.s\n" - "st1w { z28.s }, p1, [x10]\n" + "subs x11, x11, #0x1\n" + ".inst 0xc1a1c888 // fclamp { z8.s-z11.s }, z4.s, z1.s\n" + "st1w { z8.s }, p1, [x10]\n" "add x10, x10, x28, LSL #2\n" - ".inst 0xc0040c82 // mova za.d[x8, #2], { z4.d-z7.d }\n" - "st1w { z29.s }, p1, [x9]\n" + ".inst 0xc0040f02 // mova za.d[x8, #2], { z24.d-z27.d }\n" + "st1w { z9.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - "st1w { z30.s }, p1, [x26]\n" + "st1w { z10.s }, p1, [x26]\n" "add x26, x26, x24, LSL #2\n" - "st1w { z31.s }, p1, [x25]\n" + "st1w { z11.s }, p1, [x25]\n" "add x25, x25, x23, LSL #2\n" "bgt 18b\n" "19:" // End "ldr x20, [%x[args], %[offsetof_Args_weights]]\n" "incb x20, ALL, MUL #9\n" "str x20, [%x[args], %[offsetof_Args_weights]]\n" - "incw x16\n" - "ldr x20, [%x[args], %[offsetof_Args_ld_in_vl]]\n" - "whilelt p1.s, x16, x17\n" - "ldr x14, [%x[args], %[offsetof_Args_inptr]]\n" - "add x14, x14, x20, LSL #2\n" - "str x14, [%x[args], %[offsetof_Args_inptr]]\n" - "ldr x11, [%x[args], %[offsetof_Args_outptrs]]\n" + "incw x15\n" + "ldr x21, [%x[args], %[offsetof_Args_ld_in_vl]]\n" + "whilelt p1.s, x15, x16\n" + "ldr x20, [%x[args], %[offsetof_Args_inptr]]\n" + "add x20, x20, x21, LSL #2\n" + "str x20, [%x[args], %[offsetof_Args_inptr]]\n" + "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" "ldr x24, [%x[args], %[offsetof_Args_ld_out_vls]]\n" - "ldp x23, x22, [x11, #0x0]\n" + "ldp x23, x22, [x25, #0x0]\n" "ldp x21, x20, [x24, #0x0]\n" "add x23, x23, x21, LSL #2\n" "add x22, x22, x20, LSL #2\n" - "stp x23, x22, [x11, #0x0]\n" - "ldp x23, x22, [x11, #0x10]\n" + "stp x23, x22, [x25, #0x0]\n" + "ldp x23, x22, [x25, #0x10]\n" "ldp x21, x20, [x24, #0x10]\n" "add x23, x23, x21, LSL #2\n" "add x22, x22, x20, LSL #2\n" - "stp x23, x22, [x11, #0x10]\n" + "stp x23, x22, [x25, #0x10]\n" "b.any 1b\n" ".inst 0xd503467f // SMSTOP\n" : : [args] "r" (&args), [ld_in_col] "r" (ld_in_col), [ld_in_row] "r" (ld_in_row), [offsetof_Args_bias] "I" (offsetof(Args, bias)), [offsetof_Args_clamp_max] "I" (offsetof(Args, clamp_max)), [offsetof_Args_clamp_min] "I" (offsetof(Args, clamp_min)), [offsetof_Args_current_channel] "I" (offsetof(Args, current_channel)), [offsetof_Args_inptr] "I" (offsetof(Args, inptr)), [offsetof_Args_input_cols] "I" (offsetof(Args, input_cols)), [offsetof_Args_ld_in_vl] "I" (offsetof(Args, ld_in_vl)), [offsetof_Args_ld_out_cols] "I" (offsetof(Args, ld_out_cols)), [offsetof_Args_ld_out_vls] "I" (offsetof(Args, ld_out_vls)), [offsetof_Args_n_channels] "I" (offsetof(Args, n_channels)), [offsetof_Args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_Args_output_cols] "I" (offsetof(Args, output_cols)), [offsetof_Args_pad_bottom] "I" (offsetof(Args, pad_bottom)), [offsetof_Args_pad_left] "I" (offsetof(Args, pad_left)), [offsetof_Args_pad_top] "I" (offsetof(Args, pad_top)), [offsetof_Args_weights] "I" (offsetof(Args, weights)) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_5x5_s1_4rows_dot_za.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_5x5_s1_4rows_dot_za.hpp index c2d439fe78bf043510806fb03bb3e17247512bde..5215ccaf399889dd5dc1de21711fce63d89f7cad 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_5x5_s1_4rows_dot_za.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_5x5_s1_4rows_dot_za.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,8 @@ * SOFTWARE. */ +#include "src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp" + namespace arm_conv { namespace depthwise { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_5x5_s1_4rows_dot_za/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_5x5_s1_4rows_dot_za/generic.cpp index 2b3a2476869e71d5978899904fb97fb60f4e5a96..b72042558d9d5dbf62a7ff93491dee8e60817aeb 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_5x5_s1_4rows_dot_za/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_5x5_s1_4rows_dot_za/generic.cpp @@ -73,237 +73,237 @@ void sme2_fp32bf16fp32_planar_5x5_s1_4rows_dot_za_impl( "mov x20, #0x8\n" ".inst 0xd503477f // SMSTART ZA\n" "sub x20, x20, x4\n" - "ldr x5, [%x[args], %[offsetof_Args_pad_top]]\n" + "ldr x6, [%x[args], %[offsetof_Args_pad_top]]\n" "ptrue p2.b\n" ".inst 0x25207812 // ptrue pn10.b\n" - "ld1rw { z26.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_min]]\n" - "ldr x6, [%x[args], %[offsetof_Args_n_channels]]\n" - "whilelt p1.s, XZR, x6\n" + "ld1rw { z29.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_min]]\n" + "ldr x7, [%x[args], %[offsetof_Args_n_channels]]\n" + "whilelt p1.s, XZR, x7\n" "whilelt p9.s, XZR, x20\n" - "ld1rw { z31.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_max]]\n" - "whilelt p8.s, XZR, x5\n" + "ld1rw { z28.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_max]]\n" + "whilelt p8.s, XZR, x6\n" "addvl SP, SP, #-30\n" - "ldr x7, [%x[args], %[offsetof_Args_current_channel]]\n" + "ldr x17, [%x[args], %[offsetof_Args_current_channel]]\n" "eor p8.b, p2/Z, p8.b, p9.b\n" "1:" // Channel loop - "ldr x20, [%x[args], %[offsetof_Args_bias]]\n" - "fmov z24.s, #0x0\n" - "cbz x20, 2f\n" - "ld1w { z24.s }, p1/Z, [x20, x7, LSL #2]\n" + "ldr x21, [%x[args], %[offsetof_Args_bias]]\n" + "fmov z30.s, #0x0\n" + "cbz x21, 2f\n" + "ld1w { z30.s }, p1/Z, [x21, x17, LSL #2]\n" "2:" // Load bias: Done - "ldr x20, [%x[args], %[offsetof_Args_weights]]\n" - "mov x21, x20\n" - "ld1w { z18.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - "ld1w { z11.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - "fmov z4.s, #0x0\n" - "incb x20\n" - "ld1w { z3.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - ".inst 0x658aaa45 // bfcvt z5.h, p2/M, z18.s\n" - ".inst 0x658aa966 // bfcvt z6.h, p2/M, z11.s\n" - "ld1w { z17.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" + "ldr x21, [%x[args], %[offsetof_Args_weights]]\n" + "mov x20, x21\n" + "ld1w { z12.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" + "ld1w { z24.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" + "fmov z11.s, #0x0\n" + "incb x21\n" + "ld1w { z3.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" + ".inst 0x658aa99a // bfcvt z26.h, p2/M, z12.s\n" + ".inst 0x658aab10 // bfcvt z16.h, p2/M, z24.s\n" + "ld1w { z20.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" "addvl x24, SP, #30\n" - ".inst 0x648aaa44 // bfcvtnt z4.h, p2/M, z18.s\n" - "ld1w { z16.s }, p2/Z, [x21]\n" - "mov x21, x20\n" - ".inst 0x658aa867 // bfcvt z7.h, p2/M, z3.s\n" + ".inst 0x648aa98b // bfcvtnt z11.h, p2/M, z12.s\n" + "ld1w { z25.s }, p2/Z, [x20]\n" + "mov x20, x21\n" + ".inst 0x658aa875 // bfcvt z21.h, p2/M, z3.s\n" "addvl x24, x24, #-6\n" - "ld1w { z18.s }, p2/Z, [x21]\n" - ".inst 0x658aaa28 // bfcvt z8.h, p2/M, z17.s\n" - "incb x21, ALL, MUL #5\n" - "st1h { z4.h }, p2, [x24]\n" - ".inst 0x648aa965 // bfcvtnt z5.h, p2/M, z11.s\n" - "ld1w { z11.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - "fmov z4.s, #0x0\n" - "st1h { z5.h }, p2, [x24, #1, MUL VL]\n" - ".inst 0x648aa866 // bfcvtnt z6.h, p2/M, z3.s\n" - "ld1w { z3.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - ".inst 0x658aaa45 // bfcvt z5.h, p2/M, z18.s\n" - ".inst 0x648aaa27 // bfcvtnt z7.h, p2/M, z17.s\n" - "incb x20\n" - "ld1w { z17.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - "st1h { z6.h }, p2, [x24, #2, MUL VL]\n" - ".inst 0x648aaa08 // bfcvtnt z8.h, p2/M, z16.s\n" - ".inst 0x658aaa09 // bfcvt z9.h, p2/M, z16.s\n" - "ld1w { z16.s }, p2/Z, [x21]\n" - ".inst 0x658aa966 // bfcvt z6.h, p2/M, z11.s\n" - "mov x21, x20\n" - "st1h { z7.h }, p2, [x24, #3, MUL VL]\n" - ".inst 0x648aaa44 // bfcvtnt z4.h, p2/M, z18.s\n" - ".inst 0x658aa867 // bfcvt z7.h, p2/M, z3.s\n" - "ld1w { z18.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - "st1h { z8.h }, p2, [x24, #4, MUL VL]\n" - ".inst 0x648aa965 // bfcvtnt z5.h, p2/M, z11.s\n" - ".inst 0x658aaa28 // bfcvt z8.h, p2/M, z17.s\n" - "incb x20\n" - "st1h { z9.h }, p2, [x24, #5, MUL VL]\n" + "ld1w { z6.s }, p2/Z, [x20]\n" + ".inst 0x658aaa9b // bfcvt z27.h, p2/M, z20.s\n" + "incb x20, ALL, MUL #5\n" + "st1h { z11.h }, p2, [x24]\n" + ".inst 0x648aab1a // bfcvtnt z26.h, p2/M, z24.s\n" + "ld1w { z14.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" + "fmov z11.s, #0x0\n" + "st1h { z26.h }, p2, [x24, #1, MUL VL]\n" + ".inst 0x648aa870 // bfcvtnt z16.h, p2/M, z3.s\n" + "ld1w { z19.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" + ".inst 0x658aa8c9 // bfcvt z9.h, p2/M, z6.s\n" + ".inst 0x648aaa95 // bfcvtnt z21.h, p2/M, z20.s\n" + "incb x21\n" + "ld1w { z12.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" + "st1h { z16.h }, p2, [x24, #2, MUL VL]\n" + ".inst 0x648aab3b // bfcvtnt z27.h, p2/M, z25.s\n" + ".inst 0x658aab37 // bfcvt z23.h, p2/M, z25.s\n" + "ld1w { z5.s }, p2/Z, [x20]\n" + ".inst 0x658aa9c8 // bfcvt z8.h, p2/M, z14.s\n" + "mov x23, x21\n" + "st1h { z21.h }, p2, [x24, #3, MUL VL]\n" + ".inst 0x648aa8cb // bfcvtnt z11.h, p2/M, z6.s\n" + ".inst 0x658aaa79 // bfcvt z25.h, p2/M, z19.s\n" + "ld1w { z4.s }, p2/Z, [x23]\n" + "incb x23, ALL, MUL #5\n" + "st1h { z27.h }, p2, [x24, #4, MUL VL]\n" + ".inst 0x648aa9c9 // bfcvtnt z9.h, p2/M, z14.s\n" + ".inst 0x658aa991 // bfcvt z17.h, p2/M, z12.s\n" + "incb x21\n" + "st1h { z23.h }, p2, [x24, #5, MUL VL]\n" "addvl x24, x24, #-6\n" - "ld1w { z11.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - "st1h { z4.h }, p2, [x24]\n" - "fmov z4.s, #0x0\n" - ".inst 0x648aa866 // bfcvtnt z6.h, p2/M, z3.s\n" - "ldr x17, [%x[args], %[offsetof_Args_input_cols]]\n" - "st1h { z5.h }, p2, [x24, #1, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - ".inst 0x658aaa45 // bfcvt z5.h, p2/M, z18.s\n" - "st1h { z6.h }, p2, [x24, #2, MUL VL]\n" - ".inst 0x648aaa27 // bfcvtnt z7.h, p2/M, z17.s\n" - "ld1w { z17.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - ".inst 0x658aa966 // bfcvt z6.h, p2/M, z11.s\n" - ".inst 0x648aaa08 // bfcvtnt z8.h, p2/M, z16.s\n" - "st1h { z7.h }, p2, [x24, #3, MUL VL]\n" + "ld1w { z26.s }, p2/Z, [x23]\n" + "incb x23, ALL, MUL #5\n" + "st1h { z11.h }, p2, [x24]\n" + "fmov z2.s, #0x0\n" + ".inst 0x648aaa68 // bfcvtnt z8.h, p2/M, z19.s\n" + "ldr x25, [%x[args], %[offsetof_Args_input_cols]]\n" + "st1h { z9.h }, p2, [x24, #1, MUL VL]\n" + "ld1w { z27.s }, p2/Z, [x23]\n" + "incb x23, ALL, MUL #5\n" + ".inst 0x658aa893 // bfcvt z19.h, p2/M, z4.s\n" + "st1h { z8.h }, p2, [x24, #2, MUL VL]\n" + ".inst 0x648aa999 // bfcvtnt z25.h, p2/M, z12.s\n" + "ld1w { z7.s }, p2/Z, [x23]\n" + "incb x23, ALL, MUL #5\n" + ".inst 0x658aab4e // bfcvt z14.h, p2/M, z26.s\n" + ".inst 0x648aa8b1 // bfcvtnt z17.h, p2/M, z5.s\n" + "st1h { z25.h }, p2, [x24, #3, MUL VL]\n" "ldr x16, [%x[args], %[offsetof_Args_inptr]]\n" - ".inst 0x658aaa09 // bfcvt z9.h, p2/M, z16.s\n" - "ld1w { z16.s }, p2/Z, [x21]\n" - "mov x21, x20\n" - ".inst 0x648aaa44 // bfcvtnt z4.h, p2/M, z18.s\n" - ".inst 0x658aa867 // bfcvt z7.h, p2/M, z3.s\n" - "ld1w { z18.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - "st1h { z8.h }, p2, [x24, #4, MUL VL]\n" - "st1h { z9.h }, p2, [x24, #5, MUL VL]\n" + ".inst 0x658aa8ab // bfcvt z11.h, p2/M, z5.s\n" + "ld1w { z18.s }, p2/Z, [x23]\n" + "mov x20, x21\n" + ".inst 0x648aa882 // bfcvtnt z2.h, p2/M, z4.s\n" + ".inst 0x658aab66 // bfcvt z6.h, p2/M, z27.s\n" + "ld1w { z15.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" + "st1h { z17.h }, p2, [x24, #4, MUL VL]\n" + "st1h { z11.h }, p2, [x24, #5, MUL VL]\n" "addvl x24, x24, #-6\n" - ".inst 0x648aa965 // bfcvtnt z5.h, p2/M, z11.s\n" - ".inst 0x658aaa28 // bfcvt z8.h, p2/M, z17.s\n" - "ld1w { z11.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - "st1h { z4.h }, p2, [x24]\n" - ".inst 0x648aa866 // bfcvtnt z6.h, p2/M, z3.s\n" - "ld1w { z3.s }, p2/Z, [x21]\n" - "fmov z4.s, #0x0\n" - "st1h { z5.h }, p2, [x24, #1, MUL VL]\n" - "incb x21, ALL, MUL #5\n" + ".inst 0x648aab53 // bfcvtnt z19.h, p2/M, z26.s\n" + ".inst 0x658aa8fa // bfcvt z26.h, p2/M, z7.s\n" + "ld1w { z11.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" + "st1h { z2.h }, p2, [x24]\n" + ".inst 0x648aab6e // bfcvtnt z14.h, p2/M, z27.s\n" + "ld1w { z4.s }, p2/Z, [x20]\n" + "fmov z21.s, #0x0\n" + "st1h { z19.h }, p2, [x24, #1, MUL VL]\n" + "incb x20, ALL, MUL #5\n" + ".inst 0x658aa9ea // bfcvt z10.h, p2/M, z15.s\n" + "st1h { z14.h }, p2, [x24, #2, MUL VL]\n" + ".inst 0x648aa8e6 // bfcvtnt z6.h, p2/M, z7.s\n" + "incb x21\n" + "ld1w { z17.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" + ".inst 0x658aa973 // bfcvt z19.h, p2/M, z11.s\n" + "st1h { z6.h }, p2, [x24, #3, MUL VL]\n" + ".inst 0x648aaa5a // bfcvtnt z26.h, p2/M, z18.s\n" ".inst 0x658aaa45 // bfcvt z5.h, p2/M, z18.s\n" - "st1h { z6.h }, p2, [x24, #2, MUL VL]\n" - ".inst 0x648aaa27 // bfcvtnt z7.h, p2/M, z17.s\n" - "incb x20\n" - "ld1w { z17.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - ".inst 0x658aa966 // bfcvt z6.h, p2/M, z11.s\n" - "st1h { z7.h }, p2, [x24, #3, MUL VL]\n" - ".inst 0x648aaa08 // bfcvtnt z8.h, p2/M, z16.s\n" - ".inst 0x658aaa09 // bfcvt z9.h, p2/M, z16.s\n" - "ld1w { z16.s }, p2/Z, [x21]\n" - "mov x21, x20\n" - ".inst 0x658aa867 // bfcvt z7.h, p2/M, z3.s\n" - ".inst 0x648aaa44 // bfcvtnt z4.h, p2/M, z18.s\n" - "ld1w { z18.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - ".inst 0x648aa965 // bfcvtnt z5.h, p2/M, z11.s\n" - "ld1w { z11.s }, p2/Z, [x21]\n" + "ld1w { z12.s }, p2/Z, [x20]\n" + "mov x21, x21\n" + ".inst 0x658aa897 // bfcvt z23.h, p2/M, z4.s\n" + ".inst 0x648aa9f5 // bfcvtnt z21.h, p2/M, z15.s\n" + "ld1w { z24.s }, p2/Z, [x21]\n" "incb x21, ALL, MUL #5\n" - "st1h { z8.h }, p2, [x24, #4, MUL VL]\n" - ".inst 0x648aa866 // bfcvtnt z6.h, p2/M, z3.s\n" - ".inst 0x658aaa28 // bfcvt z8.h, p2/M, z17.s\n" + ".inst 0x648aa96a // bfcvtnt z10.h, p2/M, z11.s\n" "ld1w { z3.s }, p2/Z, [x21]\n" "incb x21, ALL, MUL #5\n" - ".inst 0x648aaa27 // bfcvtnt z7.h, p2/M, z17.s\n" - "ld1w { z17.s }, p2/Z, [x21]\n" - "st1h { z9.h }, p2, [x24, #5, MUL VL]\n" + "st1h { z26.h }, p2, [x24, #4, MUL VL]\n" + ".inst 0x648aa893 // bfcvtnt z19.h, p2/M, z4.s\n" + ".inst 0x658aaa30 // bfcvt z16.h, p2/M, z17.s\n" + "ld1w { z2.s }, p2/Z, [x21]\n" + "incb x21, ALL, MUL #5\n" + ".inst 0x648aaa37 // bfcvtnt z23.h, p2/M, z17.s\n" + "ld1w { z26.s }, p2/Z, [x21]\n" + "st1h { z5.h }, p2, [x24, #5, MUL VL]\n" "addvl x24, x24, #-6\n" - "st1h { z4.h }, p2, [x24]\n" - ".inst 0x648aaa08 // bfcvtnt z8.h, p2/M, z16.s\n" + "st1h { z21.h }, p2, [x24]\n" + ".inst 0x648aa990 // bfcvtnt z16.h, p2/M, z12.s\n" "incb x21, ALL, MUL #5\n" - "fmov z4.s, #0x0\n" - "st1h { z5.h }, p2, [x24, #1, MUL VL]\n" - ".inst 0x658aaa45 // bfcvt z5.h, p2/M, z18.s\n" - ".inst 0x658aaa09 // bfcvt z9.h, p2/M, z16.s\n" - "sub x20, x17, #0x1\n" - "st1h { z6.h }, p2, [x24, #2, MUL VL]\n" - ".inst 0x658aa966 // bfcvt z6.h, p2/M, z11.s\n" - "ld1w { z16.s }, p2/Z, [x21]\n" + "fmov z8.s, #0x0\n" + "st1h { z10.h }, p2, [x24, #1, MUL VL]\n" + ".inst 0x658aab04 // bfcvt z4.h, p2/M, z24.s\n" + ".inst 0x658aa985 // bfcvt z5.h, p2/M, z12.s\n" + "sub x20, x25, #0x1\n" + "st1h { z19.h }, p2, [x24, #2, MUL VL]\n" + ".inst 0x658aa871 // bfcvt z17.h, p2/M, z3.s\n" + "ld1w { z25.s }, p2/Z, [x21]\n" "orr x23, x20, %x[ld_in_col], LSL #18\n" - "st1h { z7.h }, p2, [x24, #3, MUL VL]\n" - ".inst 0x658aa867 // bfcvt z7.h, p2/M, z3.s\n" - "orr x23, x6, x23, LSL #20\n" + "st1h { z23.h }, p2, [x24, #3, MUL VL]\n" + ".inst 0x658aa857 // bfcvt z23.h, p2/M, z2.s\n" + "orr x23, x7, x23, LSL #20\n" "mov x22, #0x8\n" - "st1h { z8.h }, p2, [x24, #4, MUL VL]\n" - ".inst 0x658aaa28 // bfcvt z8.h, p2/M, z17.s\n" - "add x21, x5, x4\n" + "st1h { z16.h }, p2, [x24, #4, MUL VL]\n" + ".inst 0x658aab4e // bfcvt z14.h, p2/M, z26.s\n" + "add x21, x6, x4\n" "lsl x20, %x[ld_in_row], #0x2\n" - "st1h { z9.h }, p2, [x24, #5, MUL VL]\n" + "st1h { z5.h }, p2, [x24, #5, MUL VL]\n" "addvl x24, x24, #-6\n" - "mov z25.d, z24.d\n" - ".inst 0x648aaa44 // bfcvtnt z4.h, p2/M, z18.s\n" - "st1h { z4.h }, p2, [x24]\n" - ".inst 0x648aa965 // bfcvtnt z5.h, p2/M, z11.s\n" - ".inst 0x648aa866 // bfcvtnt z6.h, p2/M, z3.s\n" + "mov z31.d, z30.d\n" + ".inst 0x648aab08 // bfcvtnt z8.h, p2/M, z24.s\n" + "st1h { z8.h }, p2, [x24]\n" + ".inst 0x648aa864 // bfcvtnt z4.h, p2/M, z3.s\n" + ".inst 0x648aa851 // bfcvtnt z17.h, p2/M, z2.s\n" "mov x11, #0x0\n" - "st1h { z5.h }, p2, [x24, #1, MUL VL]\n" - ".inst 0x648aaa27 // bfcvtnt z7.h, p2/M, z17.s\n" - ".inst 0x648aaa08 // bfcvtnt z8.h, p2/M, z16.s\n" + "st1h { z4.h }, p2, [x24, #1, MUL VL]\n" + ".inst 0x648aab57 // bfcvtnt z23.h, p2/M, z26.s\n" + ".inst 0x648aab2e // bfcvtnt z14.h, p2/M, z25.s\n" "mov x8, #0x8\n" - "st1h { z6.h }, p2, [x24, #2, MUL VL]\n" - ".inst 0x658aaa09 // bfcvt z9.h, p2/M, z16.s\n" + "st1h { z17.h }, p2, [x24, #2, MUL VL]\n" + ".inst 0x658aab26 // bfcvt z6.h, p2/M, z25.s\n" "ldr x15, [%x[args], %[offsetof_Args_output_cols]]\n" "lsl x23, x23, #0x2\n" - "st1h { z7.h }, p2, [x24, #3, MUL VL]\n" + "st1h { z23.h }, p2, [x24, #3, MUL VL]\n" "sub x22, x22, x21\n" - "madd x20, x20, x5, x16\n" - "st1h { z8.h }, p2, [x24, #4, MUL VL]\n" - "st1h { z9.h }, p2, [x24, #5, MUL VL]\n" + "madd x20, x20, x6, x16\n" + "st1h { z14.h }, p2, [x24, #4, MUL VL]\n" + "st1h { z6.h }, p2, [x24, #5, MUL VL]\n" "3:" // Issue prefetches "subs x22, x22, #0x1\n" ".inst 0xf8b74a9c // rprfm pldstrm, x23, [x20]\n" "add x20, x20, %x[ld_in_col], LSL #2\n" "bgt 3b\n" - "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" + "ldr x23, [%x[args], %[offsetof_Args_outptrs]]\n" "lsl x20, %x[ld_in_row], #0x2\n" - "msub x16, x5, x20, x16\n" - ".inst 0xc0046b00 // mova za.d[x11, #0], { z24.d-z25.d }\n" + "msub x16, x6, x20, x16\n" + ".inst 0xc0046bc0 // mova za.d[x11, #0], { z30.d-z31.d }\n" "ldr x20, [%x[args], %[offsetof_Args_ld_out_cols]]\n" - ".inst 0xc0046b01 // mova za.d[x11, #1], { z24.d-z25.d }\n" + ".inst 0xc0046bc1 // mova za.d[x11, #1], { z30.d-z31.d }\n" "mov x22, #0x4\n" - "ldp x14, x13, [x25], #0x10\n" - ".inst 0xc0046b02 // mova za.d[x11, #2], { z24.d-z25.d }\n" - "ldp x0, x10, [x20], #0x10\n" - ".inst 0xc0046b03 // mova za.d[x11, #3], { z24.d-z25.d }\n" + "ldp x14, x13, [x23], #0x10\n" + ".inst 0xc0046bc2 // mova za.d[x11, #2], { z30.d-z31.d }\n" + "ldp x5, x10, [x20], #0x10\n" + ".inst 0xc0046bc3 // mova za.d[x11, #3], { z30.d-z31.d }\n" "ldr x21, [%x[args], %[offsetof_Args_pad_left]]\n" - ".inst 0xc0046b04 // mova za.d[x11, #4], { z24.d-z25.d }\n" - "ldp x9, x28, [x25], #0x10\n" - ".inst 0xc0046b05 // mova za.d[x11, #5], { z24.d-z25.d }\n" + ".inst 0xc0046bc4 // mova za.d[x11, #4], { z30.d-z31.d }\n" + "ldp x9, x28, [x23], #0x10\n" + ".inst 0xc0046bc5 // mova za.d[x11, #5], { z30.d-z31.d }\n" "ldp x27, x26, [x20], #0x10\n" - ".inst 0xc0046b06 // mova za.d[x11, #6], { z24.d-z25.d }\n" - ".inst 0xc0046b07 // mova za.d[x11, #7], { z24.d-z25.d }\n" - ".inst 0xc0040b00 // mova za.d[x8, #0], { z24.d-z25.d }\n" - ".inst 0xc0040b01 // mova za.d[x8, #1], { z24.d-z25.d }\n" + ".inst 0xc0046bc6 // mova za.d[x11, #6], { z30.d-z31.d }\n" + ".inst 0xc0046bc7 // mova za.d[x11, #7], { z30.d-z31.d }\n" + ".inst 0xc0040bc0 // mova za.d[x8, #0], { z30.d-z31.d }\n" + ".inst 0xc0040bc1 // mova za.d[x8, #1], { z30.d-z31.d }\n" "cbz x21, 5f\n" "cmp x21, x22\n" "csel x20, x21, x22, LT\n" "sub x21, x21, x20\n" "sub x22, x22, x20\n" "cbz x21, 5f\n" - ".inst 0xc0066800 // mova { z0.d-z1.d }, za.d[x11, #0]\n" + ".inst 0xc0066804 // mova { z4.d-z5.d }, za.d[x11, #0]\n" "sub x15, x15, x21\n" - ".inst 0xc0066822 // mova { z2.d-z3.d }, za.d[x11, #1]\n" - ".inst 0xc1bfcb40 // fclamp { z0.s-z3.s }, z26.s, z31.s\n" + ".inst 0xc0066826 // mova { z6.d-z7.d }, za.d[x11, #1]\n" + ".inst 0xc1bccba4 // fclamp { z4.s-z7.s }, z29.s, z28.s\n" "4:" // Left padding "subs x21, x21, #0x1\n" - "st1w { z0.s }, p1, [x14]\n" - "add x14, x14, x0, LSL #2\n" - "st1w { z2.s }, p1, [x13]\n" + "st1w { z4.s }, p1, [x14]\n" + "add x14, x14, x5, LSL #2\n" + "st1w { z6.s }, p1, [x13]\n" "add x13, x13, x10, LSL #2\n" - "st1w { z1.s }, p1, [x9]\n" + "st1w { z5.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - "st1w { z3.s }, p1, [x28]\n" + "st1w { z7.s }, p1, [x28]\n" "add x28, x28, x26, LSL #2\n" "bgt 4b\n" "5:" // Left padding: End - "adds XZR, x5, x4\n" + "adds XZR, x6, x4\n" "bne 12f\n" "cbz x22, 10f\n" "cmp x22, #0x1\n" - "sub x17, x17, x22\n" + "sub x25, x25, x22\n" "beq 9f\n" "cmp x22, #0x2\n" "beq 8f\n" @@ -311,335 +311,335 @@ void sme2_fp32bf16fp32_planar_5x5_s1_4rows_dot_za_impl( "beq 7f\n" "6:" // Unpadded: 4 priming loads "add x21, x16, %x[ld_in_row], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x16]\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + "ld1w { z21.s }, p1/Z, [x16]\n" + ".inst 0x658aaab2 // bfcvt z18.h, p2/M, z21.s\n" "addvl x20, SP, #24\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + "ld1w { z11.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + ".inst 0x648aa972 // bfcvtnt z18.h, p2/M, z11.s\n" "add x16, x16, %x[ld_in_col], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + "ld1w { z17.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0x658aaa33 // bfcvt z19.h, p2/M, z17.s\n" + "ld1w { z12.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0x648aa993 // bfcvtnt z19.h, p2/M, z12.s\n" + "ld1w { z7.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0x658aa8f4 // bfcvt z20.h, p2/M, z7.s\n" + "ld1w { z12.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" - ".inst 0xa0402a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1257190 // bfdot za.s[x11, 0], { z12.h-z13.h }, z5.h\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0x648aa994 // bfcvtnt z20.h, p2/M, z12.s\n" + ".inst 0xa0402a8c // ld1h { z12.h-z13.h }, pn10.b/Z, [x20]\n" + ".inst 0xc12d7250 // bfdot za.s[x11, 0], { z18.h-z19.h }, z13.h\n" + "ld1w { z6.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - ".inst 0xc1247191 // bfdot za.s[x11, 1], { z12.h-z13.h }, z4.h\n" - ".inst 0xa0412a86 // ld1h { z6.h-z7.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc12771b0 // bfdot za.s[x11, 0], { z13.h-z14.h }, z7.h\n" - "ld1w { z16.s }, p1/Z, [x21]\n" - ".inst 0x648aaa0f // bfcvtnt z15.h, p2/M, z16.s\n" - ".inst 0xc12671b1 // bfdot za.s[x11, 1], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0422a88 // ld1h { z8.h-z9.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc12971d0 // bfdot za.s[x11, 0], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d1 // bfdot za.s[x11, 1], { z14.h-z15.h }, z8.h\n" + ".inst 0x658aa8d5 // bfcvt z21.h, p2/M, z6.s\n" + ".inst 0xc12c7251 // bfdot za.s[x11, 1], { z18.h-z19.h }, z12.h\n" + ".inst 0xa0412a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc12b7270 // bfdot za.s[x11, 0], { z19.h-z20.h }, z11.h\n" + "ld1w { z27.s }, p1/Z, [x21]\n" + ".inst 0x648aab75 // bfcvtnt z21.h, p2/M, z27.s\n" + ".inst 0xc12a7271 // bfdot za.s[x11, 1], { z19.h-z20.h }, z10.h\n" + ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc12b7290 // bfdot za.s[x11, 0], { z20.h-z21.h }, z11.h\n" + ".inst 0xc12a7291 // bfdot za.s[x11, 1], { z20.h-z21.h }, z10.h\n" "7:" // Unpadded: 3 priming loads "add x22, x16, %x[ld_in_row], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x16]\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + "ld1w { z6.s }, p1/Z, [x16]\n" + ".inst 0x658aa8d7 // bfcvt z23.h, p2/M, z6.s\n" "addvl x21, SP, #18\n" - "ld1w { z16.s }, p1/Z, [x22]\n" + "ld1w { z1.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + ".inst 0x648aa837 // bfcvtnt z23.h, p2/M, z1.s\n" "addvl x20, SP, #24\n" - "ld1w { z16.s }, p1/Z, [x22]\n" + "ld1w { z15.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0x658aa9f8 // bfcvt z24.h, p2/M, z15.s\n" "add x16, x16, %x[ld_in_col], LSL #2\n" "ld1w { z16.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x22]\n" + ".inst 0x648aaa18 // bfcvtnt z24.h, p2/M, z16.s\n" + "ld1w { z1.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x22]\n" + ".inst 0x658aa839 // bfcvt z25.h, p2/M, z1.s\n" + "ld1w { z9.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" - ".inst 0xa0402aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1257190 // bfdot za.s[x11, 0], { z12.h-z13.h }, z5.h\n" + ".inst 0x648aa939 // bfcvtnt z25.h, p2/M, z9.s\n" + ".inst 0xa1402aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21]\n" + ".inst 0xc12972f0 // bfdot za.s[x11, 0], { z23.h-z24.h }, z9.h\n" "ld1w { z16.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - ".inst 0xc1247191 // bfdot za.s[x11, 1], { z12.h-z13.h }, z4.h\n" - ".inst 0xa0402a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1257192 // bfdot za.s[x11, 2], { z12.h-z13.h }, z5.h\n" - ".inst 0xa0412aa6 // ld1h { z6.h-z7.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc1247193 // bfdot za.s[x11, 3], { z12.h-z13.h }, z4.h\n" + ".inst 0x658aaa1a // bfcvt z26.h, p2/M, z16.s\n" + ".inst 0xc12172f1 // bfdot za.s[x11, 1], { z23.h-z24.h }, z1.h\n" + ".inst 0xa1402a87 // ld1h { z7.h, z15.h }, pn10.b/Z, [x20]\n" + ".inst 0xc12f72f2 // bfdot za.s[x11, 2], { z23.h-z24.h }, z15.h\n" + ".inst 0xa1412aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc12772f3 // bfdot za.s[x11, 3], { z23.h-z24.h }, z7.h\n" "ld1w { z16.s }, p1/Z, [x22]\n" - ".inst 0x648aaa0f // bfcvtnt z15.h, p2/M, z16.s\n" - ".inst 0xc12771b0 // bfdot za.s[x11, 0], { z13.h-z14.h }, z7.h\n" - ".inst 0xc12671b1 // bfdot za.s[x11, 1], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0412a86 // ld1h { z6.h-z7.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xa0422aa8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc12771b2 // bfdot za.s[x11, 2], { z13.h-z14.h }, z7.h\n" - ".inst 0xc12671b3 // bfdot za.s[x11, 3], { z13.h-z14.h }, z6.h\n" - ".inst 0xc12971d0 // bfdot za.s[x11, 0], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d1 // bfdot za.s[x11, 1], { z14.h-z15.h }, z8.h\n" - ".inst 0xa0422a88 // ld1h { z8.h-z9.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc12971d2 // bfdot za.s[x11, 2], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d3 // bfdot za.s[x11, 3], { z14.h-z15.h }, z8.h\n" + ".inst 0x648aaa1a // bfcvtnt z26.h, p2/M, z16.s\n" + ".inst 0xc1297310 // bfdot za.s[x11, 0], { z24.h-z25.h }, z9.h\n" + ".inst 0xc1217311 // bfdot za.s[x11, 1], { z24.h-z25.h }, z1.h\n" + ".inst 0xa1412a87 // ld1h { z7.h, z15.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xa1422aa3 // ld1h { z3.h, z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc12f7312 // bfdot za.s[x11, 2], { z24.h-z25.h }, z15.h\n" + ".inst 0xc1277313 // bfdot za.s[x11, 3], { z24.h-z25.h }, z7.h\n" + ".inst 0xc12b7330 // bfdot za.s[x11, 0], { z25.h-z26.h }, z11.h\n" + ".inst 0xc1237331 // bfdot za.s[x11, 1], { z25.h-z26.h }, z3.h\n" + ".inst 0xa0422a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc1237332 // bfdot za.s[x11, 2], { z25.h-z26.h }, z3.h\n" + ".inst 0xc1227333 // bfdot za.s[x11, 3], { z25.h-z26.h }, z2.h\n" "8:" // Unpadded: 2 priming loads "add x23, x16, %x[ld_in_row], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x16]\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + "ld1w { z24.s }, p1/Z, [x16]\n" + ".inst 0x658aab02 // bfcvt z2.h, p2/M, z24.s\n" "addvl x22, SP, #12\n" "ld1w { z16.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + ".inst 0x648aaa02 // bfcvtnt z2.h, p2/M, z16.s\n" "addvl x21, SP, #18\n" "ld1w { z16.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0x658aaa03 // bfcvt z3.h, p2/M, z16.s\n" "addvl x20, SP, #24\n" "ld1w { z16.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + ".inst 0x648aaa03 // bfcvtnt z3.h, p2/M, z16.s\n" "add x16, x16, %x[ld_in_col], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x23]\n" + "ld1w { z1.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x23]\n" + ".inst 0x658aa824 // bfcvt z4.h, p2/M, z1.s\n" + "ld1w { z19.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" - ".inst 0xa0402ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22]\n" - ".inst 0xc1257190 // bfdot za.s[x11, 0], { z12.h-z13.h }, z5.h\n" - "ld1w { z16.s }, p1/Z, [x23]\n" + ".inst 0x648aaa64 // bfcvtnt z4.h, p2/M, z19.s\n" + ".inst 0xa1402ac7 // ld1h { z7.h, z15.h }, pn10.b/Z, [x22]\n" + ".inst 0xc12f7050 // bfdot za.s[x11, 0], { z2.h-z3.h }, z15.h\n" + "ld1w { z0.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - ".inst 0xc1247191 // bfdot za.s[x11, 1], { z12.h-z13.h }, z4.h\n" - ".inst 0xa0402aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1257192 // bfdot za.s[x11, 2], { z12.h-z13.h }, z5.h\n" - ".inst 0xa0412ac6 // ld1h { z6.h-z7.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xc1247193 // bfdot za.s[x11, 3], { z12.h-z13.h }, z4.h\n" - "ld1w { z16.s }, p1/Z, [x23]\n" - ".inst 0x648aaa0f // bfcvtnt z15.h, p2/M, z16.s\n" - ".inst 0xc12771b0 // bfdot za.s[x11, 0], { z13.h-z14.h }, z7.h\n" - ".inst 0xa0402a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20]\n" - ".inst 0xc12671b1 // bfdot za.s[x11, 1], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0412aa6 // ld1h { z6.h-z7.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc1257194 // bfdot za.s[x11, 4], { z12.h-z13.h }, z5.h\n" - ".inst 0xa0422ac8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc1247195 // bfdot za.s[x11, 5], { z12.h-z13.h }, z4.h\n" - ".inst 0xc12771b2 // bfdot za.s[x11, 2], { z13.h-z14.h }, z7.h\n" - ".inst 0xc12671b3 // bfdot za.s[x11, 3], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0412a86 // ld1h { z6.h-z7.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc12971d0 // bfdot za.s[x11, 0], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d1 // bfdot za.s[x11, 1], { z14.h-z15.h }, z8.h\n" - ".inst 0xa0422aa8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc12771b4 // bfdot za.s[x11, 4], { z13.h-z14.h }, z7.h\n" - ".inst 0xc12671b5 // bfdot za.s[x11, 5], { z13.h-z14.h }, z6.h\n" - ".inst 0xc12971d2 // bfdot za.s[x11, 2], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d3 // bfdot za.s[x11, 3], { z14.h-z15.h }, z8.h\n" - ".inst 0xa0422a88 // ld1h { z8.h-z9.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc12971d4 // bfdot za.s[x11, 4], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d5 // bfdot za.s[x11, 5], { z14.h-z15.h }, z8.h\n" + ".inst 0x658aa805 // bfcvt z5.h, p2/M, z0.s\n" + ".inst 0xc1277051 // bfdot za.s[x11, 1], { z2.h-z3.h }, z7.h\n" + ".inst 0xa1402aa7 // ld1h { z7.h, z15.h }, pn10.b/Z, [x21]\n" + ".inst 0xc12f7052 // bfdot za.s[x11, 2], { z2.h-z3.h }, z15.h\n" + ".inst 0xa1412ac6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc1277053 // bfdot za.s[x11, 3], { z2.h-z3.h }, z7.h\n" + "ld1w { z10.s }, p1/Z, [x23]\n" + ".inst 0x648aa945 // bfcvtnt z5.h, p2/M, z10.s\n" + ".inst 0xc12e7070 // bfdot za.s[x11, 0], { z3.h-z4.h }, z14.h\n" + ".inst 0xa1402a87 // ld1h { z7.h, z15.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1267071 // bfdot za.s[x11, 1], { z3.h-z4.h }, z6.h\n" + ".inst 0xa0412aac // ld1h { z12.h-z13.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc12f7054 // bfdot za.s[x11, 4], { z2.h-z3.h }, z15.h\n" + ".inst 0xa1422ac0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc1277055 // bfdot za.s[x11, 5], { z2.h-z3.h }, z7.h\n" + ".inst 0xc12d7072 // bfdot za.s[x11, 2], { z3.h-z4.h }, z13.h\n" + ".inst 0xc12c7073 // bfdot za.s[x11, 3], { z3.h-z4.h }, z12.h\n" + ".inst 0xa0412a8e // ld1h { z14.h-z15.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc1287090 // bfdot za.s[x11, 0], { z4.h-z5.h }, z8.h\n" + ".inst 0xc1207091 // bfdot za.s[x11, 1], { z4.h-z5.h }, z0.h\n" + ".inst 0xa0422aa6 // ld1h { z6.h-z7.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc12f7074 // bfdot za.s[x11, 4], { z3.h-z4.h }, z15.h\n" + ".inst 0xc12e7075 // bfdot za.s[x11, 5], { z3.h-z4.h }, z14.h\n" + ".inst 0xc1277092 // bfdot za.s[x11, 2], { z4.h-z5.h }, z7.h\n" + ".inst 0xc1267093 // bfdot za.s[x11, 3], { z4.h-z5.h }, z6.h\n" + ".inst 0xa1422a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc1287094 // bfdot za.s[x11, 4], { z4.h-z5.h }, z8.h\n" + ".inst 0xc1207095 // bfdot za.s[x11, 5], { z4.h-z5.h }, z0.h\n" "9:" // Unpadded: 1 priming loads "add x24, x16, %x[ld_in_row], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x16]\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + "ld1w { z18.s }, p1/Z, [x16]\n" + ".inst 0x658aaa4c // bfcvt z12.h, p2/M, z18.s\n" "addvl x23, SP, #6\n" - "ld1w { z16.s }, p1/Z, [x24]\n" + "ld1w { z7.s }, p1/Z, [x24]\n" "add x24, x24, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + ".inst 0x648aa8ec // bfcvtnt z12.h, p2/M, z7.s\n" "addvl x22, SP, #12\n" - "ld1w { z16.s }, p1/Z, [x24]\n" + "ld1w { z20.s }, p1/Z, [x24]\n" "add x24, x24, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0x658aaa8d // bfcvt z13.h, p2/M, z20.s\n" "addvl x21, SP, #18\n" - "ld1w { z16.s }, p1/Z, [x24]\n" + "ld1w { z0.s }, p1/Z, [x24]\n" "add x24, x24, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + ".inst 0x648aa80d // bfcvtnt z13.h, p2/M, z0.s\n" "addvl x20, SP, #24\n" - "ld1w { z16.s }, p1/Z, [x24]\n" + "ld1w { z10.s }, p1/Z, [x24]\n" "add x24, x24, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" + ".inst 0x658aa94e // bfcvt z14.h, p2/M, z10.s\n" "add x16, x16, %x[ld_in_col], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x24]\n" + "ld1w { z0.s }, p1/Z, [x24]\n" "add x24, x24, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" - ".inst 0xa0402ae4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x23]\n" - ".inst 0xc1257190 // bfdot za.s[x11, 0], { z12.h-z13.h }, z5.h\n" - "ld1w { z16.s }, p1/Z, [x24]\n" + ".inst 0x648aa80e // bfcvtnt z14.h, p2/M, z0.s\n" + ".inst 0xa0402ae0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x23]\n" + ".inst 0xc1217190 // bfdot za.s[x11, 0], { z12.h-z13.h }, z1.h\n" + "ld1w { z17.s }, p1/Z, [x24]\n" "add x24, x24, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - ".inst 0xc1247191 // bfdot za.s[x11, 1], { z12.h-z13.h }, z4.h\n" - ".inst 0xa0402ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22]\n" - ".inst 0xc1257192 // bfdot za.s[x11, 2], { z12.h-z13.h }, z5.h\n" - ".inst 0xa0412ae6 // ld1h { z6.h-z7.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0xc1247193 // bfdot za.s[x11, 3], { z12.h-z13.h }, z4.h\n" - "ld1w { z16.s }, p1/Z, [x24]\n" - ".inst 0x648aaa0f // bfcvtnt z15.h, p2/M, z16.s\n" - ".inst 0xc12771b0 // bfdot za.s[x11, 0], { z13.h-z14.h }, z7.h\n" - ".inst 0xa0402aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21]\n" - ".inst 0xc12671b1 // bfdot za.s[x11, 1], { z13.h-z14.h }, z6.h\n" + ".inst 0x658aaa2f // bfcvt z15.h, p2/M, z17.s\n" + ".inst 0xc1207191 // bfdot za.s[x11, 1], { z12.h-z13.h }, z0.h\n" + ".inst 0xa0402aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22]\n" + ".inst 0xc12b7192 // bfdot za.s[x11, 2], { z12.h-z13.h }, z11.h\n" + ".inst 0xa0412ae0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xc12a7193 // bfdot za.s[x11, 3], { z12.h-z13.h }, z10.h\n" + "ld1w { z18.s }, p1/Z, [x24]\n" + ".inst 0x648aaa4f // bfcvtnt z15.h, p2/M, z18.s\n" + ".inst 0xc12171b0 // bfdot za.s[x11, 0], { z13.h-z14.h }, z1.h\n" + ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" + ".inst 0xc12071b1 // bfdot za.s[x11, 1], { z13.h-z14.h }, z0.h\n" ".inst 0xa0412ac6 // ld1h { z6.h-z7.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xc1257194 // bfdot za.s[x11, 4], { z12.h-z13.h }, z5.h\n" - ".inst 0xa0422ae8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xc1247195 // bfdot za.s[x11, 5], { z12.h-z13.h }, z4.h\n" - ".inst 0xa0402a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20]\n" + ".inst 0xc12a7194 // bfdot za.s[x11, 4], { z12.h-z13.h }, z10.h\n" + ".inst 0xa0422aea // ld1h { z10.h-z11.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xc1227195 // bfdot za.s[x11, 5], { z12.h-z13.h }, z2.h\n" + ".inst 0xa0402a88 // ld1h { z8.h-z9.h }, pn10.b/Z, [x20]\n" ".inst 0xc12771b2 // bfdot za.s[x11, 2], { z13.h-z14.h }, z7.h\n" ".inst 0xc12671b3 // bfdot za.s[x11, 3], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0412aa6 // ld1h { z6.h-z7.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc12971d0 // bfdot za.s[x11, 0], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d1 // bfdot za.s[x11, 1], { z14.h-z15.h }, z8.h\n" - ".inst 0xa0422ac8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc1257196 // bfdot za.s[x11, 6], { z12.h-z13.h }, z5.h\n" - ".inst 0xc1247197 // bfdot za.s[x11, 7], { z12.h-z13.h }, z4.h\n" - ".inst 0xc12771b4 // bfdot za.s[x11, 4], { z13.h-z14.h }, z7.h\n" - ".inst 0xc12671b5 // bfdot za.s[x11, 5], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0412a86 // ld1h { z6.h-z7.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc12971d2 // bfdot za.s[x11, 2], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d3 // bfdot za.s[x11, 3], { z14.h-z15.h }, z8.h\n" - ".inst 0xa0422aa8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc12771b6 // bfdot za.s[x11, 6], { z13.h-z14.h }, z7.h\n" - ".inst 0xc12671b7 // bfdot za.s[x11, 7], { z13.h-z14.h }, z6.h\n" - ".inst 0xc12971d4 // bfdot za.s[x11, 4], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d5 // bfdot za.s[x11, 5], { z14.h-z15.h }, z8.h\n" - ".inst 0xa0422a88 // ld1h { z8.h-z9.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc12971d6 // bfdot za.s[x11, 6], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d7 // bfdot za.s[x11, 7], { z14.h-z15.h }, z8.h\n" + ".inst 0xa0412aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc12b71d0 // bfdot za.s[x11, 0], { z14.h-z15.h }, z11.h\n" + ".inst 0xc12a71d1 // bfdot za.s[x11, 1], { z14.h-z15.h }, z10.h\n" + ".inst 0xa1422ac2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc1297196 // bfdot za.s[x11, 6], { z12.h-z13.h }, z9.h\n" + ".inst 0xc1287197 // bfdot za.s[x11, 7], { z12.h-z13.h }, z8.h\n" + ".inst 0xc12171b4 // bfdot za.s[x11, 4], { z13.h-z14.h }, z1.h\n" + ".inst 0xc12071b5 // bfdot za.s[x11, 5], { z13.h-z14.h }, z0.h\n" + ".inst 0xa1412a83 // ld1h { z3.h, z11.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc12a71d2 // bfdot za.s[x11, 2], { z14.h-z15.h }, z10.h\n" + ".inst 0xc12271d3 // bfdot za.s[x11, 3], { z14.h-z15.h }, z2.h\n" + ".inst 0xa0422aa6 // ld1h { z6.h-z7.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc12b71b6 // bfdot za.s[x11, 6], { z13.h-z14.h }, z11.h\n" + ".inst 0xc12371b7 // bfdot za.s[x11, 7], { z13.h-z14.h }, z3.h\n" + ".inst 0xc12771d4 // bfdot za.s[x11, 4], { z14.h-z15.h }, z7.h\n" + ".inst 0xc12671d5 // bfdot za.s[x11, 5], { z14.h-z15.h }, z6.h\n" + ".inst 0xa0422a86 // ld1h { z6.h-z7.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc12771d6 // bfdot za.s[x11, 6], { z14.h-z15.h }, z7.h\n" + ".inst 0xc12671d7 // bfdot za.s[x11, 7], { z14.h-z15.h }, z6.h\n" "10:" // Unpadded: 0 priming loads - ".inst 0xa0402be4 // ld1h { z4.h-z5.h }, pn10.b/Z, [SP]\n" - ".inst 0xa0412be6 // ld1h { z6.h-z7.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" - ".inst 0xa0422be8 // ld1h { z8.h-z9.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" - "cbz x17, 20f\n" + ".inst 0xa1402be6 // ld1h { z6.h, z14.h }, pn10.b/Z, [SP]\n" + ".inst 0xa1412be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + ".inst 0xa1422be4 // ld1h { z4.h, z12.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" + "cbz x25, 20f\n" "add x20, x16, %x[ld_in_row], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x16]\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" - "sub x17, x17, #0x1\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + "ld1w { z1.s }, p1/Z, [x16]\n" + ".inst 0x658aa834 // bfcvt z20.h, p2/M, z1.s\n" + "sub x25, x25, #0x1\n" + "ld1w { z10.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "sub x15, x15, #0x1\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + ".inst 0x648aa954 // bfcvtnt z20.h, p2/M, z10.s\n" "ld1w { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" - "cmp x17, x15\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + ".inst 0x658aaa15 // bfcvt z21.h, p2/M, z16.s\n" + "cmp x25, x15\n" + "ld1w { z19.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "csel x25, x17, x15, LT\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + "csel x25, x25, x15, LT\n" + ".inst 0x648aaa75 // bfcvtnt z21.h, p2/M, z19.s\n" + "ld1w { z23.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" + ".inst 0x658aaaf6 // bfcvt z22.h, p2/M, z23.s\n" "add x16, x16, %x[ld_in_col], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + "ld1w { z27.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" + ".inst 0x648aab76 // bfcvtnt z22.h, p2/M, z27.s\n" "sub x15, x15, x25\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + "ld1w { z15.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" + ".inst 0x658aa9f7 // bfcvt z23.h, p2/M, z15.s\n" "ld1w { z16.s }, p1/Z, [x20]\n" - ".inst 0x648aaa0f // bfcvtnt z15.h, p2/M, z16.s\n" + ".inst 0x648aaa17 // bfcvtnt z23.h, p2/M, z16.s\n" "cbz x25, 19f\n" "11:" // Unpadded: Main loop "addvl x24, SP, #6\n" - ".inst 0xc1257190 // bfdot za.s[x11, 0], { z12.h-z13.h }, z5.h\n" + ".inst 0xc12e7290 // bfdot za.s[x11, 0], { z20.h-z21.h }, z14.h\n" "addvl x23, SP, #12\n" - "ld1w { z23.s }, p1/Z, [x16]\n" - ".inst 0xc1247191 // bfdot za.s[x11, 1], { z12.h-z13.h }, z4.h\n" - ".inst 0xa0402b04 // ld1h { z4.h-z5.h }, pn10.b/Z, [x24]\n" + "ld1w { z27.s }, p1/Z, [x16]\n" + ".inst 0xc1267291 // bfdot za.s[x11, 1], { z20.h-z21.h }, z6.h\n" + ".inst 0xa1402b01 // ld1h { z1.h, z9.h }, pn10.b/Z, [x24]\n" "addvl x22, SP, #18\n" "addvl x21, SP, #24\n" - ".inst 0xc1257192 // bfdot za.s[x11, 2], { z12.h-z13.h }, z5.h\n" + ".inst 0xc1297292 // bfdot za.s[x11, 2], { z20.h-z21.h }, z9.h\n" "add x20, x16, %x[ld_in_row], LSL #2\n" - "ld1w { z22.s }, p1/Z, [x20]\n" + "ld1w { z26.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1247193 // bfdot za.s[x11, 3], { z12.h-z13.h }, z4.h\n" - ".inst 0xa0402ae4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x23]\n" + ".inst 0xc1217293 // bfdot za.s[x11, 3], { z20.h-z21.h }, z1.h\n" + ".inst 0xa1402ae6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x23]\n" "subs x25, x25, #0x1\n" "add x16, x16, %x[ld_in_col], LSL #2\n" - ".inst 0xc12771b0 // bfdot za.s[x11, 0], { z13.h-z14.h }, z7.h\n" - "ld1w { z21.s }, p1/Z, [x20]\n" + ".inst 0xc12d72b0 // bfdot za.s[x11, 0], { z21.h-z22.h }, z13.h\n" + "ld1w { z25.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc12671b1 // bfdot za.s[x11, 1], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0412b06 // ld1h { z6.h-z7.h }, pn10.b/Z, [x24, #0x2, MUL VL]\n" - ".inst 0xc1257194 // bfdot za.s[x11, 4], { z12.h-z13.h }, z5.h\n" - "ld1w { z20.s }, p1/Z, [x20]\n" + ".inst 0xc12572b1 // bfdot za.s[x11, 1], { z21.h-z22.h }, z5.h\n" + ".inst 0xa1412b07 // ld1h { z7.h, z15.h }, pn10.b/Z, [x24, #0x2, MUL VL]\n" + ".inst 0xc12e7294 // bfdot za.s[x11, 4], { z20.h-z21.h }, z14.h\n" + "ld1w { z24.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1247195 // bfdot za.s[x11, 5], { z12.h-z13.h }, z4.h\n" - ".inst 0xa0402ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22]\n" - ".inst 0xc12771b2 // bfdot za.s[x11, 2], { z13.h-z14.h }, z7.h\n" + ".inst 0xc1267295 // bfdot za.s[x11, 5], { z20.h-z21.h }, z6.h\n" + ".inst 0xa1402ac5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x22]\n" + ".inst 0xc12f72b2 // bfdot za.s[x11, 2], { z21.h-z22.h }, z15.h\n" "ld1w { z19.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc12671b3 // bfdot za.s[x11, 3], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0412ae6 // ld1h { z6.h-z7.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0xc12971d0 // bfdot za.s[x11, 0], { z14.h-z15.h }, z9.h\n" + ".inst 0xc12772b3 // bfdot za.s[x11, 3], { z21.h-z22.h }, z7.h\n" + ".inst 0xa1412ae6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xc12c72d0 // bfdot za.s[x11, 0], { z22.h-z23.h }, z12.h\n" "ld1w { z18.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc12871d1 // bfdot za.s[x11, 1], { z14.h-z15.h }, z8.h\n" - ".inst 0xa0422b08 // ld1h { z8.h-z9.h }, pn10.b/Z, [x24, #0x4, MUL VL]\n" - ".inst 0xc1257196 // bfdot za.s[x11, 6], { z12.h-z13.h }, z5.h\n" + ".inst 0xc12472d1 // bfdot za.s[x11, 1], { z22.h-z23.h }, z4.h\n" + ".inst 0xa1422b07 // ld1h { z7.h, z15.h }, pn10.b/Z, [x24, #0x4, MUL VL]\n" + ".inst 0xc12d7296 // bfdot za.s[x11, 6], { z20.h-z21.h }, z13.h\n" "ld1w { z17.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc1247197 // bfdot za.s[x11, 7], { z12.h-z13.h }, z4.h\n" - ".inst 0xa0402aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21]\n" - ".inst 0xc12771b4 // bfdot za.s[x11, 4], { z13.h-z14.h }, z7.h\n" + ".inst 0xc1257297 // bfdot za.s[x11, 7], { z20.h-z21.h }, z5.h\n" + ".inst 0xa1402aa4 // ld1h { z4.h, z12.h }, pn10.b/Z, [x21]\n" + ".inst 0xc12e72b4 // bfdot za.s[x11, 4], { z21.h-z22.h }, z14.h\n" "ld1w { z16.s }, p1/Z, [x20]\n" - ".inst 0xc12671b5 // bfdot za.s[x11, 5], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0412ac6 // ld1h { z6.h-z7.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xc12971d2 // bfdot za.s[x11, 2], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d3 // bfdot za.s[x11, 3], { z14.h-z15.h }, z8.h\n" - ".inst 0xa0422ae8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xc12771b6 // bfdot za.s[x11, 6], { z13.h-z14.h }, z7.h\n" - ".inst 0xc12671b7 // bfdot za.s[x11, 7], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0412aa6 // ld1h { z6.h-z7.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc12971d4 // bfdot za.s[x11, 4], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d5 // bfdot za.s[x11, 5], { z14.h-z15.h }, z8.h\n" - ".inst 0xa0422ac8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc12971d6 // bfdot za.s[x11, 6], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d7 // bfdot za.s[x11, 7], { z14.h-z15.h }, z8.h\n" - ".inst 0xa0422aa8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc1251190 // bfdot za.s[x8, 0], { z12.h-z13.h }, z5.h\n" - ".inst 0xc1241191 // bfdot za.s[x8, 1], { z12.h-z13.h }, z4.h\n" - ".inst 0x658aaaec // bfcvt z12.h, p2/M, z23.s\n" - ".inst 0xa0402be4 // ld1h { z4.h-z5.h }, pn10.b/Z, [SP]\n" - ".inst 0xc12711b0 // bfdot za.s[x8, 0], { z13.h-z14.h }, z7.h\n" - ".inst 0x648aaacc // bfcvtnt z12.h, p2/M, z22.s\n" - ".inst 0xc12611b1 // bfdot za.s[x8, 1], { z13.h-z14.h }, z6.h\n" - ".inst 0x658aaaad // bfcvt z13.h, p2/M, z21.s\n" - ".inst 0xa0412be6 // ld1h { z6.h-z7.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" - ".inst 0xc12911d0 // bfdot za.s[x8, 0], { z14.h-z15.h }, z9.h\n" - ".inst 0x648aaa8d // bfcvtnt z13.h, p2/M, z20.s\n" - ".inst 0xc12811d1 // bfdot za.s[x8, 1], { z14.h-z15.h }, z8.h\n" - ".inst 0x658aaa6e // bfcvt z14.h, p2/M, z19.s\n" - ".inst 0x658aaa2f // bfcvt z15.h, p2/M, z17.s\n" + ".inst 0xc12672b5 // bfdot za.s[x11, 5], { z21.h-z22.h }, z6.h\n" + ".inst 0xa1412ac6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc12f72d2 // bfdot za.s[x11, 2], { z22.h-z23.h }, z15.h\n" + ".inst 0xc12772d3 // bfdot za.s[x11, 3], { z22.h-z23.h }, z7.h\n" + ".inst 0xa1422ae7 // ld1h { z7.h, z15.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xc12e72b6 // bfdot za.s[x11, 6], { z21.h-z22.h }, z14.h\n" + ".inst 0xc12672b7 // bfdot za.s[x11, 7], { z21.h-z22.h }, z6.h\n" + ".inst 0xa1412aa5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc12f72d4 // bfdot za.s[x11, 4], { z22.h-z23.h }, z15.h\n" + ".inst 0xc12772d5 // bfdot za.s[x11, 5], { z22.h-z23.h }, z7.h\n" + ".inst 0xa0422ace // ld1h { z14.h-z15.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc12f72d6 // bfdot za.s[x11, 6], { z22.h-z23.h }, z15.h\n" + ".inst 0xc12e72d7 // bfdot za.s[x11, 7], { z22.h-z23.h }, z14.h\n" + ".inst 0xa1422aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc12c1290 // bfdot za.s[x8, 0], { z20.h-z21.h }, z12.h\n" + ".inst 0xc1241291 // bfdot za.s[x8, 1], { z20.h-z21.h }, z4.h\n" + ".inst 0x658aab74 // bfcvt z20.h, p2/M, z27.s\n" + ".inst 0xa1402be6 // ld1h { z6.h, z14.h }, pn10.b/Z, [SP]\n" + ".inst 0xc12d12b0 // bfdot za.s[x8, 0], { z21.h-z22.h }, z13.h\n" + ".inst 0x648aab54 // bfcvtnt z20.h, p2/M, z26.s\n" + ".inst 0xc12512b1 // bfdot za.s[x8, 1], { z21.h-z22.h }, z5.h\n" + ".inst 0x658aab35 // bfcvt z21.h, p2/M, z25.s\n" + ".inst 0xa1412be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + ".inst 0xc12912d0 // bfdot za.s[x8, 0], { z22.h-z23.h }, z9.h\n" + ".inst 0x648aab15 // bfcvtnt z21.h, p2/M, z24.s\n" + ".inst 0xc12112d1 // bfdot za.s[x8, 1], { z22.h-z23.h }, z1.h\n" + ".inst 0x658aaa76 // bfcvt z22.h, p2/M, z19.s\n" + ".inst 0x658aaa37 // bfcvt z23.h, p2/M, z17.s\n" "add x8, x8, #0x2\n" - ".inst 0xc0066800 // mova { z0.d-z1.d }, za.d[x11, #0]\n" - ".inst 0xa0422be8 // ld1h { z8.h-z9.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" - ".inst 0x648aaa4e // bfcvtnt z14.h, p2/M, z18.s\n" - ".inst 0xc0066822 // mova { z2.d-z3.d }, za.d[x11, #1]\n" - ".inst 0xc1bfcb40 // fclamp { z0.s-z3.s }, z26.s, z31.s\n" - "st1w { z0.s }, p1, [x14]\n" - "add x14, x14, x0, LSL #2\n" - "st1w { z2.s }, p1, [x13]\n" + ".inst 0xc0066808 // mova { z8.d-z9.d }, za.d[x11, #0]\n" + ".inst 0xa1422be4 // ld1h { z4.h, z12.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" + ".inst 0x648aaa56 // bfcvtnt z22.h, p2/M, z18.s\n" + ".inst 0xc006682a // mova { z10.d-z11.d }, za.d[x11, #1]\n" + ".inst 0xc1bccba8 // fclamp { z8.s-z11.s }, z29.s, z28.s\n" + "st1w { z8.s }, p1, [x14]\n" + "add x14, x14, x5, LSL #2\n" + "st1w { z10.s }, p1, [x13]\n" "add x13, x13, x10, LSL #2\n" "add x11, x11, #0x2\n" - ".inst 0xc0040b00 // mova za.d[x8, #0], { z24.d-z25.d }\n" - "st1w { z1.s }, p1, [x9]\n" + ".inst 0xc0040bc0 // mova za.d[x8, #0], { z30.d-z31.d }\n" + "st1w { z9.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - ".inst 0xc0040b01 // mova za.d[x8, #1], { z24.d-z25.d }\n" - ".inst 0x648aaa0f // bfcvtnt z15.h, p2/M, z16.s\n" - "st1w { z3.s }, p1, [x28]\n" + ".inst 0xc0040bc1 // mova za.d[x8, #1], { z30.d-z31.d }\n" + ".inst 0x648aaa17 // bfcvtnt z23.h, p2/M, z16.s\n" + "st1w { z11.s }, p1, [x28]\n" "add x28, x28, x26, LSL #2\n" "bgt 11b\n" "b 19f\n" "12:" // Padded "cbz x22, 17f\n" "cmp x22, #0x1\n" - "sub x17, x17, x22\n" + "sub x25, x25, x22\n" "beq 16f\n" "cmp x22, #0x2\n" "beq 15f\n" @@ -649,449 +649,449 @@ void sme2_fp32bf16fp32_planar_5x5_s1_4rows_dot_za_impl( "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1w { z16.s }, p0/Z, [x16]\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + ".inst 0x658aaa06 // bfcvt z6.h, p2/M, z16.s\n" "add x21, x16, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1w { z16.s }, p0/Z, [x21]\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + ".inst 0x648aaa06 // bfcvtnt z6.h, p2/M, z16.s\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1w { z16.s }, p0/Z, [x21]\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0x658aaa07 // bfcvt z7.h, p2/M, z16.s\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1w { z16.s }, p0/Z, [x21]\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + ".inst 0x648aaa07 // bfcvtnt z7.h, p2/M, z16.s\n" "mov x12, #0x4\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1w { z16.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" + ".inst 0x658aaa08 // bfcvt z8.h, p2/M, z16.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "addvl x20, SP, #24\n" "ld1w { z16.s }, p0/Z, [x21]\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" + ".inst 0x648aaa08 // bfcvtnt z8.h, p2/M, z16.s\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xa0402a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1257190 // bfdot za.s[x11, 0], { z12.h-z13.h }, z5.h\n" - "ld1w { z16.s }, p0/Z, [x21]\n" + ".inst 0xa0402a8e // ld1h { z14.h-z15.h }, pn10.b/Z, [x20]\n" + ".inst 0xc12f70d0 // bfdot za.s[x11, 0], { z6.h-z7.h }, z15.h\n" + "ld1w { z9.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - ".inst 0xc1247191 // bfdot za.s[x11, 1], { z12.h-z13.h }, z4.h\n" - ".inst 0xa0412a86 // ld1h { z6.h-z7.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0x658aa929 // bfcvt z9.h, p2/M, z9.s\n" + ".inst 0xc12e70d1 // bfdot za.s[x11, 1], { z6.h-z7.h }, z14.h\n" + ".inst 0xa0412a8e // ld1h { z14.h-z15.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" "add x16, x16, %x[ld_in_col], LSL #2\n" "ld1w { z16.s }, p0/Z, [x21]\n" - ".inst 0x648aaa0f // bfcvtnt z15.h, p2/M, z16.s\n" - ".inst 0xc12771b0 // bfdot za.s[x11, 0], { z13.h-z14.h }, z7.h\n" - ".inst 0xc12671b1 // bfdot za.s[x11, 1], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0422a88 // ld1h { z8.h-z9.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc12971d0 // bfdot za.s[x11, 0], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d1 // bfdot za.s[x11, 1], { z14.h-z15.h }, z8.h\n" + ".inst 0x648aaa09 // bfcvtnt z9.h, p2/M, z16.s\n" + ".inst 0xc12f70f0 // bfdot za.s[x11, 0], { z7.h-z8.h }, z15.h\n" + ".inst 0xc12e70f1 // bfdot za.s[x11, 1], { z7.h-z8.h }, z14.h\n" + ".inst 0xa0422a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc1237110 // bfdot za.s[x11, 0], { z8.h-z9.h }, z3.h\n" + ".inst 0xc1227111 // bfdot za.s[x11, 1], { z8.h-z9.h }, z2.h\n" "14:" // Padded: 3 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1w { z16.s }, p0/Z, [x16]\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + ".inst 0x658aaa09 // bfcvt z9.h, p2/M, z16.s\n" "add x22, x16, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + ".inst 0x648aaa09 // bfcvtnt z9.h, p2/M, z16.s\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0x658aaa0a // bfcvt z10.h, p2/M, z16.s\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + ".inst 0x648aaa0a // bfcvtnt z10.h, p2/M, z16.s\n" "mov x12, #0x4\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1w { z16.s }, p0/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" + ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "addvl x21, SP, #18\n" "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" + ".inst 0x648aaa0b // bfcvtnt z11.h, p2/M, z16.s\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xa0402aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1257190 // bfdot za.s[x11, 0], { z12.h-z13.h }, z5.h\n" + ".inst 0xa1402aa7 // ld1h { z7.h, z15.h }, pn10.b/Z, [x21]\n" + ".inst 0xc12f7130 // bfdot za.s[x11, 0], { z9.h-z10.h }, z15.h\n" "ld1w { z16.s }, p0/Z, [x22]\n" "addvl x20, SP, #24\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" + ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xc1247191 // bfdot za.s[x11, 1], { z12.h-z13.h }, z4.h\n" - ".inst 0xa0402a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1277131 // bfdot za.s[x11, 1], { z9.h-z10.h }, z7.h\n" + ".inst 0xa1402a86 // ld1h { z6.h, z14.h }, pn10.b/Z, [x20]\n" "add x16, x16, %x[ld_in_col], LSL #2\n" - ".inst 0xa0412aa6 // ld1h { z6.h-z7.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc1257192 // bfdot za.s[x11, 2], { z12.h-z13.h }, z5.h\n" + ".inst 0xa1412aa7 // ld1h { z7.h, z15.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc12e7132 // bfdot za.s[x11, 2], { z9.h-z10.h }, z14.h\n" "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0xc1247193 // bfdot za.s[x11, 3], { z12.h-z13.h }, z4.h\n" - ".inst 0x648aaa0f // bfcvtnt z15.h, p2/M, z16.s\n" - ".inst 0xc12771b0 // bfdot za.s[x11, 0], { z13.h-z14.h }, z7.h\n" - ".inst 0xa0422aa8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc12671b1 // bfdot za.s[x11, 1], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0412a86 // ld1h { z6.h-z7.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc12771b2 // bfdot za.s[x11, 2], { z13.h-z14.h }, z7.h\n" - ".inst 0xc12671b3 // bfdot za.s[x11, 3], { z13.h-z14.h }, z6.h\n" - ".inst 0xc12971d0 // bfdot za.s[x11, 0], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d1 // bfdot za.s[x11, 1], { z14.h-z15.h }, z8.h\n" - ".inst 0xa0422a88 // ld1h { z8.h-z9.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc12971d2 // bfdot za.s[x11, 2], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d3 // bfdot za.s[x11, 3], { z14.h-z15.h }, z8.h\n" + ".inst 0xc1267133 // bfdot za.s[x11, 3], { z9.h-z10.h }, z6.h\n" + ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + ".inst 0xc12f7150 // bfdot za.s[x11, 0], { z10.h-z11.h }, z15.h\n" + ".inst 0xa1422aa5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc1277151 // bfdot za.s[x11, 1], { z10.h-z11.h }, z7.h\n" + ".inst 0xa0412a8e // ld1h { z14.h-z15.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc12f7152 // bfdot za.s[x11, 2], { z10.h-z11.h }, z15.h\n" + ".inst 0xc12e7153 // bfdot za.s[x11, 3], { z10.h-z11.h }, z14.h\n" + ".inst 0xc12d7170 // bfdot za.s[x11, 0], { z11.h-z12.h }, z13.h\n" + ".inst 0xc1257171 // bfdot za.s[x11, 1], { z11.h-z12.h }, z5.h\n" + ".inst 0xa0422a8e // ld1h { z14.h-z15.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc12f7172 // bfdot za.s[x11, 2], { z11.h-z12.h }, z15.h\n" + ".inst 0xc12e7173 // bfdot za.s[x11, 3], { z11.h-z12.h }, z14.h\n" "15:" // Padded: 2 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1w { z16.s }, p0/Z, [x16]\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + ".inst 0x658aaa12 // bfcvt z18.h, p2/M, z16.s\n" "add x23, x16, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1w { z16.s }, p0/Z, [x23]\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + ".inst 0x648aaa12 // bfcvtnt z18.h, p2/M, z16.s\n" "add x23, x23, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1w { z16.s }, p0/Z, [x23]\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0x658aaa13 // bfcvt z19.h, p2/M, z16.s\n" "add x23, x23, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1w { z16.s }, p0/Z, [x23]\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + ".inst 0x648aaa13 // bfcvtnt z19.h, p2/M, z16.s\n" "mov x12, #0x4\n" "add x23, x23, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1w { z16.s }, p0/Z, [x23]\n" "add x23, x23, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" + ".inst 0x658aaa14 // bfcvt z20.h, p2/M, z16.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "addvl x22, SP, #12\n" "ld1w { z16.s }, p0/Z, [x23]\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" + ".inst 0x648aaa14 // bfcvtnt z20.h, p2/M, z16.s\n" "add x23, x23, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xa0402ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22]\n" - ".inst 0xc1257190 // bfdot za.s[x11, 0], { z12.h-z13.h }, z5.h\n" - "ld1w { z16.s }, p0/Z, [x23]\n" + ".inst 0xa1402ac1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x22]\n" + ".inst 0xc1297250 // bfdot za.s[x11, 0], { z18.h-z19.h }, z9.h\n" + "ld1w { z26.s }, p0/Z, [x23]\n" "addvl x21, SP, #18\n" "add x23, x23, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" + ".inst 0x658aab55 // bfcvt z21.h, p2/M, z26.s\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xc1247191 // bfdot za.s[x11, 1], { z12.h-z13.h }, z4.h\n" - ".inst 0xa0402aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21]\n" + ".inst 0xc1217251 // bfdot za.s[x11, 1], { z18.h-z19.h }, z1.h\n" + ".inst 0xa1402aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21]\n" "addvl x20, SP, #24\n" - ".inst 0xa0412ac6 // ld1h { z6.h-z7.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xc1257192 // bfdot za.s[x11, 2], { z12.h-z13.h }, z5.h\n" + ".inst 0xa1412ac7 // ld1h { z7.h, z15.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc12e7252 // bfdot za.s[x11, 2], { z18.h-z19.h }, z14.h\n" "add x16, x16, %x[ld_in_col], LSL #2\n" "ld1w { z16.s }, p0/Z, [x23]\n" - ".inst 0xc1247193 // bfdot za.s[x11, 3], { z12.h-z13.h }, z4.h\n" - ".inst 0x648aaa0f // bfcvtnt z15.h, p2/M, z16.s\n" - ".inst 0xa0402a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20]\n" - ".inst 0xc12771b0 // bfdot za.s[x11, 0], { z13.h-z14.h }, z7.h\n" - ".inst 0xc12671b1 // bfdot za.s[x11, 1], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0412aa6 // ld1h { z6.h-z7.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xa0422ac8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc1257194 // bfdot za.s[x11, 4], { z12.h-z13.h }, z5.h\n" - ".inst 0xc1247195 // bfdot za.s[x11, 5], { z12.h-z13.h }, z4.h\n" - ".inst 0xc12771b2 // bfdot za.s[x11, 2], { z13.h-z14.h }, z7.h\n" - ".inst 0xc12671b3 // bfdot za.s[x11, 3], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0412a86 // ld1h { z6.h-z7.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc12971d0 // bfdot za.s[x11, 0], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d1 // bfdot za.s[x11, 1], { z14.h-z15.h }, z8.h\n" - ".inst 0xa0422aa8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc12771b4 // bfdot za.s[x11, 4], { z13.h-z14.h }, z7.h\n" - ".inst 0xc12671b5 // bfdot za.s[x11, 5], { z13.h-z14.h }, z6.h\n" - ".inst 0xc12971d2 // bfdot za.s[x11, 2], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d3 // bfdot za.s[x11, 3], { z14.h-z15.h }, z8.h\n" - ".inst 0xa0422a88 // ld1h { z8.h-z9.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc12971d4 // bfdot za.s[x11, 4], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d5 // bfdot za.s[x11, 5], { z14.h-z15.h }, z8.h\n" + ".inst 0xc1267253 // bfdot za.s[x11, 3], { z18.h-z19.h }, z6.h\n" + ".inst 0x648aaa15 // bfcvtnt z21.h, p2/M, z16.s\n" + ".inst 0xa1402a85 // ld1h { z5.h, z13.h }, pn10.b/Z, [x20]\n" + ".inst 0xc12f7270 // bfdot za.s[x11, 0], { z19.h-z20.h }, z15.h\n" + ".inst 0xc1277271 // bfdot za.s[x11, 1], { z19.h-z20.h }, z7.h\n" + ".inst 0xa1412aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xa1422ac7 // ld1h { z7.h, z15.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc12d7254 // bfdot za.s[x11, 4], { z18.h-z19.h }, z13.h\n" + ".inst 0xc1257255 // bfdot za.s[x11, 5], { z18.h-z19.h }, z5.h\n" + ".inst 0xc12e7272 // bfdot za.s[x11, 2], { z19.h-z20.h }, z14.h\n" + ".inst 0xc1267273 // bfdot za.s[x11, 3], { z19.h-z20.h }, z6.h\n" + ".inst 0xa1412a85 // ld1h { z5.h, z13.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc12f7290 // bfdot za.s[x11, 0], { z20.h-z21.h }, z15.h\n" + ".inst 0xc1277291 // bfdot za.s[x11, 1], { z20.h-z21.h }, z7.h\n" + ".inst 0xa0422aae // ld1h { z14.h-z15.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc12d7274 // bfdot za.s[x11, 4], { z19.h-z20.h }, z13.h\n" + ".inst 0xc1257275 // bfdot za.s[x11, 5], { z19.h-z20.h }, z5.h\n" + ".inst 0xc12f7292 // bfdot za.s[x11, 2], { z20.h-z21.h }, z15.h\n" + ".inst 0xc12e7293 // bfdot za.s[x11, 3], { z20.h-z21.h }, z14.h\n" + ".inst 0xa0422a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc1237294 // bfdot za.s[x11, 4], { z20.h-z21.h }, z3.h\n" + ".inst 0xc1227295 // bfdot za.s[x11, 5], { z20.h-z21.h }, z2.h\n" "16:" // Padded: 1 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1w { z16.s }, p0/Z, [x16]\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + ".inst 0x658aaa09 // bfcvt z9.h, p2/M, z16.s\n" "add x24, x16, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1w { z16.s }, p0/Z, [x24]\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + ".inst 0x648aaa09 // bfcvtnt z9.h, p2/M, z16.s\n" "add x24, x24, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1w { z16.s }, p0/Z, [x24]\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0x658aaa0a // bfcvt z10.h, p2/M, z16.s\n" "add x24, x24, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1w { z16.s }, p0/Z, [x24]\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + ".inst 0x648aaa0a // bfcvtnt z10.h, p2/M, z16.s\n" "mov x12, #0x4\n" "add x24, x24, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1w { z16.s }, p0/Z, [x24]\n" "add x24, x24, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" + ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "addvl x23, SP, #6\n" "ld1w { z16.s }, p0/Z, [x24]\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" + ".inst 0x648aaa0b // bfcvtnt z11.h, p2/M, z16.s\n" "add x24, x24, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xa0402ae4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x23]\n" - ".inst 0xc1257190 // bfdot za.s[x11, 0], { z12.h-z13.h }, z5.h\n" + ".inst 0xa1402ae7 // ld1h { z7.h, z15.h }, pn10.b/Z, [x23]\n" + ".inst 0xc12f7130 // bfdot za.s[x11, 0], { z9.h-z10.h }, z15.h\n" "ld1w { z16.s }, p0/Z, [x24]\n" "addvl x22, SP, #12\n" "add x24, x24, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" + ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xc1247191 // bfdot za.s[x11, 1], { z12.h-z13.h }, z4.h\n" - ".inst 0xa0402ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22]\n" + ".inst 0xc1277131 // bfdot za.s[x11, 1], { z9.h-z10.h }, z7.h\n" + ".inst 0xa1402ac6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x22]\n" "addvl x21, SP, #18\n" - ".inst 0xa0412ae6 // ld1h { z6.h-z7.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0xc1257192 // bfdot za.s[x11, 2], { z12.h-z13.h }, z5.h\n" + ".inst 0xa1412ae7 // ld1h { z7.h, z15.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xc12e7132 // bfdot za.s[x11, 2], { z9.h-z10.h }, z14.h\n" "addvl x20, SP, #24\n" "add x16, x16, %x[ld_in_col], LSL #2\n" "ld1w { z16.s }, p0/Z, [x24]\n" - ".inst 0xc1247193 // bfdot za.s[x11, 3], { z12.h-z13.h }, z4.h\n" - ".inst 0x648aaa0f // bfcvtnt z15.h, p2/M, z16.s\n" - ".inst 0xa0402aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21]\n" - ".inst 0xc12771b0 // bfdot za.s[x11, 0], { z13.h-z14.h }, z7.h\n" - ".inst 0xc12671b1 // bfdot za.s[x11, 1], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0412ac6 // ld1h { z6.h-z7.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xa0422ae8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xc1257194 // bfdot za.s[x11, 4], { z12.h-z13.h }, z5.h\n" - ".inst 0xc1247195 // bfdot za.s[x11, 5], { z12.h-z13.h }, z4.h\n" - ".inst 0xa0402a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20]\n" - ".inst 0xc12771b2 // bfdot za.s[x11, 2], { z13.h-z14.h }, z7.h\n" - ".inst 0xc12671b3 // bfdot za.s[x11, 3], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0412aa6 // ld1h { z6.h-z7.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc12971d0 // bfdot za.s[x11, 0], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d1 // bfdot za.s[x11, 1], { z14.h-z15.h }, z8.h\n" - ".inst 0xa0422ac8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc1257196 // bfdot za.s[x11, 6], { z12.h-z13.h }, z5.h\n" - ".inst 0xc1247197 // bfdot za.s[x11, 7], { z12.h-z13.h }, z4.h\n" - ".inst 0xc12771b4 // bfdot za.s[x11, 4], { z13.h-z14.h }, z7.h\n" - ".inst 0xc12671b5 // bfdot za.s[x11, 5], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0412a86 // ld1h { z6.h-z7.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc12971d2 // bfdot za.s[x11, 2], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d3 // bfdot za.s[x11, 3], { z14.h-z15.h }, z8.h\n" - ".inst 0xa0422aa8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc12771b6 // bfdot za.s[x11, 6], { z13.h-z14.h }, z7.h\n" - ".inst 0xc12671b7 // bfdot za.s[x11, 7], { z13.h-z14.h }, z6.h\n" - ".inst 0xc12971d4 // bfdot za.s[x11, 4], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d5 // bfdot za.s[x11, 5], { z14.h-z15.h }, z8.h\n" - ".inst 0xa0422a88 // ld1h { z8.h-z9.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc12971d6 // bfdot za.s[x11, 6], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d7 // bfdot za.s[x11, 7], { z14.h-z15.h }, z8.h\n" + ".inst 0xc1267133 // bfdot za.s[x11, 3], { z9.h-z10.h }, z6.h\n" + ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + ".inst 0xa1402aa5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x21]\n" + ".inst 0xc12f7150 // bfdot za.s[x11, 0], { z10.h-z11.h }, z15.h\n" + ".inst 0xc1277151 // bfdot za.s[x11, 1], { z10.h-z11.h }, z7.h\n" + ".inst 0xa1412ac7 // ld1h { z7.h, z15.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xa1422ae6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xc12d7134 // bfdot za.s[x11, 4], { z9.h-z10.h }, z13.h\n" + ".inst 0xc1257135 // bfdot za.s[x11, 5], { z9.h-z10.h }, z5.h\n" + ".inst 0xa1402a85 // ld1h { z5.h, z13.h }, pn10.b/Z, [x20]\n" + ".inst 0xc12f7152 // bfdot za.s[x11, 2], { z10.h-z11.h }, z15.h\n" + ".inst 0xc1277153 // bfdot za.s[x11, 3], { z10.h-z11.h }, z7.h\n" + ".inst 0xa1412aa7 // ld1h { z7.h, z15.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc12e7170 // bfdot za.s[x11, 0], { z11.h-z12.h }, z14.h\n" + ".inst 0xc1267171 // bfdot za.s[x11, 1], { z11.h-z12.h }, z6.h\n" + ".inst 0xa1422ac6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc12d7136 // bfdot za.s[x11, 6], { z9.h-z10.h }, z13.h\n" + ".inst 0xc1257137 // bfdot za.s[x11, 7], { z9.h-z10.h }, z5.h\n" + ".inst 0xc12f7154 // bfdot za.s[x11, 4], { z10.h-z11.h }, z15.h\n" + ".inst 0xc1277155 // bfdot za.s[x11, 5], { z10.h-z11.h }, z7.h\n" + ".inst 0xa1412a87 // ld1h { z7.h, z15.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc12e7172 // bfdot za.s[x11, 2], { z11.h-z12.h }, z14.h\n" + ".inst 0xc1267173 // bfdot za.s[x11, 3], { z11.h-z12.h }, z6.h\n" + ".inst 0xa1422aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc12f7156 // bfdot za.s[x11, 6], { z10.h-z11.h }, z15.h\n" + ".inst 0xc1277157 // bfdot za.s[x11, 7], { z10.h-z11.h }, z7.h\n" + ".inst 0xc1297174 // bfdot za.s[x11, 4], { z11.h-z12.h }, z9.h\n" + ".inst 0xc1217175 // bfdot za.s[x11, 5], { z11.h-z12.h }, z1.h\n" + ".inst 0xa0422a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc1217176 // bfdot za.s[x11, 6], { z11.h-z12.h }, z1.h\n" + ".inst 0xc1207177 // bfdot za.s[x11, 7], { z11.h-z12.h }, z0.h\n" "17:" // Padded: 0 priming loads - ".inst 0xa0402be4 // ld1h { z4.h-z5.h }, pn10.b/Z, [SP]\n" - ".inst 0xa0412be6 // ld1h { z6.h-z7.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" - ".inst 0xa0422be8 // ld1h { z8.h-z9.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" - "cbz x17, 20f\n" + ".inst 0xa1402be6 // ld1h { z6.h, z14.h }, pn10.b/Z, [SP]\n" + ".inst 0xa1412be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + ".inst 0xa1422be4 // ld1h { z4.h, z12.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" + "cbz x25, 20f\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1w { z16.s }, p0/Z, [x16]\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + ".inst 0x658aaa14 // bfcvt z20.h, p2/M, z16.s\n" "add x20, x16, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + ".inst 0x648aaa14 // bfcvtnt z20.h, p2/M, z16.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0x658aaa15 // bfcvt z21.h, p2/M, z16.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + ".inst 0x648aaa15 // bfcvtnt z21.h, p2/M, z16.s\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1w { z16.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" + ".inst 0x658aaa16 // bfcvt z22.h, p2/M, z16.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1w { z16.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" + ".inst 0x648aaa16 // bfcvtnt z22.h, p2/M, z16.s\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1w { z16.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" + ".inst 0x658aaa17 // bfcvt z23.h, p2/M, z16.s\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - "sub x17, x17, #0x1\n" - ".inst 0x648aaa0f // bfcvtnt z15.h, p2/M, z16.s\n" + "sub x25, x25, #0x1\n" + ".inst 0x648aaa17 // bfcvtnt z23.h, p2/M, z16.s\n" "sub x15, x15, #0x1\n" - "cmp x17, x15\n" - "csel x25, x17, x15, LT\n" + "cmp x25, x15\n" + "csel x25, x25, x15, LT\n" "add x16, x16, %x[ld_in_col], LSL #2\n" "sub x15, x15, x25\n" "cbz x25, 19f\n" "18:" // Padded: Main loop "addvl x24, SP, #6\n" - ".inst 0xc1257190 // bfdot za.s[x11, 0], { z12.h-z13.h }, z5.h\n" + ".inst 0xc12e7290 // bfdot za.s[x11, 0], { z20.h-z21.h }, z14.h\n" "addvl x23, SP, #12\n" - ".inst 0xc1247191 // bfdot za.s[x11, 1], { z12.h-z13.h }, z4.h\n" - ".inst 0xa0402b04 // ld1h { z4.h-z5.h }, pn10.b/Z, [x24]\n" + ".inst 0xc1267291 // bfdot za.s[x11, 1], { z20.h-z21.h }, z6.h\n" + ".inst 0xa0402b02 // ld1h { z2.h-z3.h }, pn10.b/Z, [x24]\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1257192 // bfdot za.s[x11, 2], { z12.h-z13.h }, z5.h\n" - "ld1w { z23.s }, p0/Z, [x16]\n" + ".inst 0xc1237292 // bfdot za.s[x11, 2], { z20.h-z21.h }, z3.h\n" + "ld1w { z16.s }, p0/Z, [x16]\n" "add x22, x16, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0xc1247193 // bfdot za.s[x11, 3], { z12.h-z13.h }, z4.h\n" - ".inst 0xa0402ae4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x23]\n" + ".inst 0xc1227293 // bfdot za.s[x11, 3], { z20.h-z21.h }, z2.h\n" + ".inst 0xa1402ae6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x23]\n" "addvl x21, SP, #18\n" "addvl x20, SP, #24\n" - ".inst 0xc12771b0 // bfdot za.s[x11, 0], { z13.h-z14.h }, z7.h\n" - "ld1w { z22.s }, p0/Z, [x22]\n" + ".inst 0xc12d72b0 // bfdot za.s[x11, 0], { z21.h-z22.h }, z13.h\n" + "ld1w { z19.s }, p0/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc12671b1 // bfdot za.s[x11, 1], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0412b06 // ld1h { z6.h-z7.h }, pn10.b/Z, [x24, #0x2, MUL VL]\n" + ".inst 0xc12572b1 // bfdot za.s[x11, 1], { z21.h-z22.h }, z5.h\n" + ".inst 0xa1412b07 // ld1h { z7.h, z15.h }, pn10.b/Z, [x24, #0x2, MUL VL]\n" "subs x25, x25, #0x1\n" "add x16, x16, %x[ld_in_col], LSL #2\n" - ".inst 0xc1257194 // bfdot za.s[x11, 4], { z12.h-z13.h }, z5.h\n" - "ld1w { z21.s }, p0/Z, [x22]\n" + ".inst 0xc12e7294 // bfdot za.s[x11, 4], { z20.h-z21.h }, z14.h\n" + "ld1w { z17.s }, p0/Z, [x22]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x4\n" - ".inst 0xc1247195 // bfdot za.s[x11, 5], { z12.h-z13.h }, z4.h\n" - ".inst 0xa0402aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21]\n" + ".inst 0xc1267295 // bfdot za.s[x11, 5], { z20.h-z21.h }, z6.h\n" + ".inst 0xa1402aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0xc12771b2 // bfdot za.s[x11, 2], { z13.h-z14.h }, z7.h\n" - "ld1w { z20.s }, p0/Z, [x22]\n" + ".inst 0xc12f72b2 // bfdot za.s[x11, 2], { z21.h-z22.h }, z15.h\n" + "ld1w { z27.s }, p0/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc12671b3 // bfdot za.s[x11, 3], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0412ae6 // ld1h { z6.h-z7.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0xc12971d0 // bfdot za.s[x11, 0], { z14.h-z15.h }, z9.h\n" - "ld1w { z19.s }, p0/Z, [x22]\n" + ".inst 0xc12772b3 // bfdot za.s[x11, 3], { z21.h-z22.h }, z7.h\n" + ".inst 0xa1412ae7 // ld1h { z7.h, z15.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xc12c72d0 // bfdot za.s[x11, 0], { z22.h-z23.h }, z12.h\n" + "ld1w { z10.s }, p0/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0xc12871d1 // bfdot za.s[x11, 1], { z14.h-z15.h }, z8.h\n" - ".inst 0xa0422b08 // ld1h { z8.h-z9.h }, pn10.b/Z, [x24, #0x4, MUL VL]\n" - ".inst 0xc1257196 // bfdot za.s[x11, 6], { z12.h-z13.h }, z5.h\n" - "ld1w { z18.s }, p0/Z, [x22]\n" + ".inst 0xc12472d1 // bfdot za.s[x11, 1], { z22.h-z23.h }, z4.h\n" + ".inst 0xa1422b04 // ld1h { z4.h, z12.h }, pn10.b/Z, [x24, #0x4, MUL VL]\n" + ".inst 0xc12e7296 // bfdot za.s[x11, 6], { z20.h-z21.h }, z14.h\n" + "ld1w { z8.s }, p0/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc1247197 // bfdot za.s[x11, 7], { z12.h-z13.h }, z4.h\n" - ".inst 0xa0402a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20]\n" - ".inst 0xc12771b4 // bfdot za.s[x11, 4], { z13.h-z14.h }, z7.h\n" - "ld1w { z17.s }, p0/Z, [x22]\n" + ".inst 0xc1267297 // bfdot za.s[x11, 7], { z20.h-z21.h }, z6.h\n" + ".inst 0xa1402a85 // ld1h { z5.h, z13.h }, pn10.b/Z, [x20]\n" + ".inst 0xc12f72b4 // bfdot za.s[x11, 4], { z21.h-z22.h }, z15.h\n" + "ld1w { z11.s }, p0/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xc12671b5 // bfdot za.s[x11, 5], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0412aa6 // ld1h { z6.h-z7.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc12971d2 // bfdot za.s[x11, 2], { z14.h-z15.h }, z9.h\n" - "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0xc12871d3 // bfdot za.s[x11, 3], { z14.h-z15.h }, z8.h\n" - ".inst 0xa0422ae8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xc12771b6 // bfdot za.s[x11, 6], { z13.h-z14.h }, z7.h\n" - ".inst 0xc12671b7 // bfdot za.s[x11, 7], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0412a86 // ld1h { z6.h-z7.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc12971d4 // bfdot za.s[x11, 4], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d5 // bfdot za.s[x11, 5], { z14.h-z15.h }, z8.h\n" - ".inst 0xa0422aa8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc12971d6 // bfdot za.s[x11, 6], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d7 // bfdot za.s[x11, 7], { z14.h-z15.h }, z8.h\n" - ".inst 0xa0422a88 // ld1h { z8.h-z9.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc1251190 // bfdot za.s[x8, 0], { z12.h-z13.h }, z5.h\n" - ".inst 0xc1241191 // bfdot za.s[x8, 1], { z12.h-z13.h }, z4.h\n" - ".inst 0x658aaaec // bfcvt z12.h, p2/M, z23.s\n" - ".inst 0xa0402be4 // ld1h { z4.h-z5.h }, pn10.b/Z, [SP]\n" - ".inst 0xc12711b0 // bfdot za.s[x8, 0], { z13.h-z14.h }, z7.h\n" - ".inst 0x648aaacc // bfcvtnt z12.h, p2/M, z22.s\n" - ".inst 0xc12611b1 // bfdot za.s[x8, 1], { z13.h-z14.h }, z6.h\n" - ".inst 0x658aaaad // bfcvt z13.h, p2/M, z21.s\n" - ".inst 0xa0412be6 // ld1h { z6.h-z7.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" - ".inst 0xc12911d0 // bfdot za.s[x8, 0], { z14.h-z15.h }, z9.h\n" - ".inst 0x648aaa8d // bfcvtnt z13.h, p2/M, z20.s\n" - ".inst 0xc12811d1 // bfdot za.s[x8, 1], { z14.h-z15.h }, z8.h\n" - ".inst 0x658aaa6e // bfcvt z14.h, p2/M, z19.s\n" - ".inst 0x658aaa2f // bfcvt z15.h, p2/M, z17.s\n" + ".inst 0xc12772b5 // bfdot za.s[x11, 5], { z21.h-z22.h }, z7.h\n" + ".inst 0xa0412aae // ld1h { z14.h-z15.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc12c72d2 // bfdot za.s[x11, 2], { z22.h-z23.h }, z12.h\n" + "ld1w { z18.s }, p0/Z, [x22]\n" + ".inst 0xc12472d3 // bfdot za.s[x11, 3], { z22.h-z23.h }, z4.h\n" + ".inst 0xa1422ae4 // ld1h { z4.h, z12.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xc12f72b6 // bfdot za.s[x11, 6], { z21.h-z22.h }, z15.h\n" + ".inst 0xc12e72b7 // bfdot za.s[x11, 7], { z21.h-z22.h }, z14.h\n" + ".inst 0xa1412a87 // ld1h { z7.h, z15.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc12c72d4 // bfdot za.s[x11, 4], { z22.h-z23.h }, z12.h\n" + ".inst 0xc12472d5 // bfdot za.s[x11, 5], { z22.h-z23.h }, z4.h\n" + ".inst 0xa0422aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc12172d6 // bfdot za.s[x11, 6], { z22.h-z23.h }, z1.h\n" + ".inst 0xc12072d7 // bfdot za.s[x11, 7], { z22.h-z23.h }, z0.h\n" + ".inst 0xa0422a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc12d1290 // bfdot za.s[x8, 0], { z20.h-z21.h }, z13.h\n" + ".inst 0xc1251291 // bfdot za.s[x8, 1], { z20.h-z21.h }, z5.h\n" + ".inst 0x658aaa14 // bfcvt z20.h, p2/M, z16.s\n" + ".inst 0xa1402be6 // ld1h { z6.h, z14.h }, pn10.b/Z, [SP]\n" + ".inst 0xc12f12b0 // bfdot za.s[x8, 0], { z21.h-z22.h }, z15.h\n" + ".inst 0x648aaa74 // bfcvtnt z20.h, p2/M, z19.s\n" + ".inst 0xc12712b1 // bfdot za.s[x8, 1], { z21.h-z22.h }, z7.h\n" + ".inst 0x658aaa35 // bfcvt z21.h, p2/M, z17.s\n" + ".inst 0xa1412be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + ".inst 0xc12112d0 // bfdot za.s[x8, 0], { z22.h-z23.h }, z1.h\n" + ".inst 0x648aab75 // bfcvtnt z21.h, p2/M, z27.s\n" + ".inst 0xc12012d1 // bfdot za.s[x8, 1], { z22.h-z23.h }, z0.h\n" + ".inst 0x658aa956 // bfcvt z22.h, p2/M, z10.s\n" + ".inst 0x658aa977 // bfcvt z23.h, p2/M, z11.s\n" "add x8, x8, #0x2\n" ".inst 0xc0066800 // mova { z0.d-z1.d }, za.d[x11, #0]\n" - ".inst 0xa0422be8 // ld1h { z8.h-z9.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" - ".inst 0x648aaa4e // bfcvtnt z14.h, p2/M, z18.s\n" + ".inst 0xa1422be4 // ld1h { z4.h, z12.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" + ".inst 0x648aa916 // bfcvtnt z22.h, p2/M, z8.s\n" ".inst 0xc0066822 // mova { z2.d-z3.d }, za.d[x11, #1]\n" - ".inst 0xc1bfcb40 // fclamp { z0.s-z3.s }, z26.s, z31.s\n" + ".inst 0xc1bccba0 // fclamp { z0.s-z3.s }, z29.s, z28.s\n" "st1w { z0.s }, p1, [x14]\n" - "add x14, x14, x0, LSL #2\n" + "add x14, x14, x5, LSL #2\n" "st1w { z2.s }, p1, [x13]\n" "add x13, x13, x10, LSL #2\n" "add x11, x11, #0x2\n" - ".inst 0xc0040b00 // mova za.d[x8, #0], { z24.d-z25.d }\n" + ".inst 0xc0040bc0 // mova za.d[x8, #0], { z30.d-z31.d }\n" "st1w { z1.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - ".inst 0xc0040b01 // mova za.d[x8, #1], { z24.d-z25.d }\n" - ".inst 0x648aaa0f // bfcvtnt z15.h, p2/M, z16.s\n" + ".inst 0xc0040bc1 // mova za.d[x8, #1], { z30.d-z31.d }\n" + ".inst 0x648aaa57 // bfcvtnt z23.h, p2/M, z18.s\n" "st1w { z3.s }, p1, [x28]\n" "add x28, x28, x26, LSL #2\n" "bgt 18b\n" "19:" // Main loop tail "addvl x23, SP, #6\n" - ".inst 0xc1257190 // bfdot za.s[x11, 0], { z12.h-z13.h }, z5.h\n" + ".inst 0xc12e7290 // bfdot za.s[x11, 0], { z20.h-z21.h }, z14.h\n" "addvl x22, SP, #12\n" - ".inst 0xc1247191 // bfdot za.s[x11, 1], { z12.h-z13.h }, z4.h\n" - ".inst 0xa0402ae4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x23]\n" + ".inst 0xc1267291 // bfdot za.s[x11, 1], { z20.h-z21.h }, z6.h\n" + ".inst 0xa0402ae0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x23]\n" "addvl x21, SP, #18\n" "addvl x20, SP, #24\n" - ".inst 0xc1257192 // bfdot za.s[x11, 2], { z12.h-z13.h }, z5.h\n" - ".inst 0xc1247193 // bfdot za.s[x11, 3], { z12.h-z13.h }, z4.h\n" - ".inst 0xa0402ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22]\n" - ".inst 0xc12771b0 // bfdot za.s[x11, 0], { z13.h-z14.h }, z7.h\n" - ".inst 0xc12671b1 // bfdot za.s[x11, 1], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0412ae6 // ld1h { z6.h-z7.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0xc1257194 // bfdot za.s[x11, 4], { z12.h-z13.h }, z5.h\n" - ".inst 0xc1247195 // bfdot za.s[x11, 5], { z12.h-z13.h }, z4.h\n" - ".inst 0xa0402aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21]\n" - ".inst 0xc12771b2 // bfdot za.s[x11, 2], { z13.h-z14.h }, z7.h\n" - ".inst 0xc12671b3 // bfdot za.s[x11, 3], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0412ac6 // ld1h { z6.h-z7.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xc12971d0 // bfdot za.s[x11, 0], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d1 // bfdot za.s[x11, 1], { z14.h-z15.h }, z8.h\n" - ".inst 0xa0422ae8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xc1257196 // bfdot za.s[x11, 6], { z12.h-z13.h }, z5.h\n" - ".inst 0xc1247197 // bfdot za.s[x11, 7], { z12.h-z13.h }, z4.h\n" - ".inst 0xa0402a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20]\n" - ".inst 0xc12771b4 // bfdot za.s[x11, 4], { z13.h-z14.h }, z7.h\n" - ".inst 0xc12671b5 // bfdot za.s[x11, 5], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0412aa6 // ld1h { z6.h-z7.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc12971d2 // bfdot za.s[x11, 2], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d3 // bfdot za.s[x11, 3], { z14.h-z15.h }, z8.h\n" - ".inst 0xa0422ac8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc12771b6 // bfdot za.s[x11, 6], { z13.h-z14.h }, z7.h\n" - ".inst 0xc12671b7 // bfdot za.s[x11, 7], { z13.h-z14.h }, z6.h\n" - ".inst 0xa0412a86 // ld1h { z6.h-z7.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc12971d4 // bfdot za.s[x11, 4], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d5 // bfdot za.s[x11, 5], { z14.h-z15.h }, z8.h\n" - ".inst 0xa0422aa8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc12971d6 // bfdot za.s[x11, 6], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12871d7 // bfdot za.s[x11, 7], { z14.h-z15.h }, z8.h\n" - ".inst 0xa0422a88 // ld1h { z8.h-z9.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc1251190 // bfdot za.s[x8, 0], { z12.h-z13.h }, z5.h\n" - ".inst 0xc1241191 // bfdot za.s[x8, 1], { z12.h-z13.h }, z4.h\n" - ".inst 0xc12711b0 // bfdot za.s[x8, 0], { z13.h-z14.h }, z7.h\n" - ".inst 0xc12611b1 // bfdot za.s[x8, 1], { z13.h-z14.h }, z6.h\n" - ".inst 0xc12911d0 // bfdot za.s[x8, 0], { z14.h-z15.h }, z9.h\n" - ".inst 0xc12811d1 // bfdot za.s[x8, 1], { z14.h-z15.h }, z8.h\n" + ".inst 0xc1217292 // bfdot za.s[x11, 2], { z20.h-z21.h }, z1.h\n" + ".inst 0xc1207293 // bfdot za.s[x11, 3], { z20.h-z21.h }, z0.h\n" + ".inst 0xa1402ac6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x22]\n" + ".inst 0xc12d72b0 // bfdot za.s[x11, 0], { z21.h-z22.h }, z13.h\n" + ".inst 0xc12572b1 // bfdot za.s[x11, 1], { z21.h-z22.h }, z5.h\n" + ".inst 0xa1412ae7 // ld1h { z7.h, z15.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xc12e7294 // bfdot za.s[x11, 4], { z20.h-z21.h }, z14.h\n" + ".inst 0xc1267295 // bfdot za.s[x11, 5], { z20.h-z21.h }, z6.h\n" + ".inst 0xa1402aa5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x21]\n" + ".inst 0xc12f72b2 // bfdot za.s[x11, 2], { z21.h-z22.h }, z15.h\n" + ".inst 0xc12772b3 // bfdot za.s[x11, 3], { z21.h-z22.h }, z7.h\n" + ".inst 0xa1412ac7 // ld1h { z7.h, z15.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc12c72d0 // bfdot za.s[x11, 0], { z22.h-z23.h }, z12.h\n" + ".inst 0xc12472d1 // bfdot za.s[x11, 1], { z22.h-z23.h }, z4.h\n" + ".inst 0xa1422ae6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xc12d7296 // bfdot za.s[x11, 6], { z20.h-z21.h }, z13.h\n" + ".inst 0xc1257297 // bfdot za.s[x11, 7], { z20.h-z21.h }, z5.h\n" + ".inst 0xa1402a85 // ld1h { z5.h, z13.h }, pn10.b/Z, [x20]\n" + ".inst 0xc12f72b4 // bfdot za.s[x11, 4], { z21.h-z22.h }, z15.h\n" + ".inst 0xc12772b5 // bfdot za.s[x11, 5], { z21.h-z22.h }, z7.h\n" + ".inst 0xa1412aa7 // ld1h { z7.h, z15.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc12e72d2 // bfdot za.s[x11, 2], { z22.h-z23.h }, z14.h\n" + ".inst 0xc12672d3 // bfdot za.s[x11, 3], { z22.h-z23.h }, z6.h\n" + ".inst 0xa1422ac6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc12f72b6 // bfdot za.s[x11, 6], { z21.h-z22.h }, z15.h\n" + ".inst 0xc12772b7 // bfdot za.s[x11, 7], { z21.h-z22.h }, z7.h\n" + ".inst 0xa1412a87 // ld1h { z7.h, z15.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc12e72d4 // bfdot za.s[x11, 4], { z22.h-z23.h }, z14.h\n" + ".inst 0xc12672d5 // bfdot za.s[x11, 5], { z22.h-z23.h }, z6.h\n" + ".inst 0xa1422aa3 // ld1h { z3.h, z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc12b72d6 // bfdot za.s[x11, 6], { z22.h-z23.h }, z11.h\n" + ".inst 0xc12372d7 // bfdot za.s[x11, 7], { z22.h-z23.h }, z3.h\n" + ".inst 0xa0422a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc12d1290 // bfdot za.s[x8, 0], { z20.h-z21.h }, z13.h\n" + ".inst 0xc1251291 // bfdot za.s[x8, 1], { z20.h-z21.h }, z5.h\n" + ".inst 0xc12f12b0 // bfdot za.s[x8, 0], { z21.h-z22.h }, z15.h\n" + ".inst 0xc12712b1 // bfdot za.s[x8, 1], { z21.h-z22.h }, z7.h\n" + ".inst 0xc12312d0 // bfdot za.s[x8, 0], { z22.h-z23.h }, z3.h\n" + ".inst 0xc12212d1 // bfdot za.s[x8, 1], { z22.h-z23.h }, z2.h\n" "add x8, x8, #0x2\n" - ".inst 0xc0066800 // mova { z0.d-z1.d }, za.d[x11, #0]\n" - ".inst 0xc0066822 // mova { z2.d-z3.d }, za.d[x11, #1]\n" - ".inst 0xc1bfcb40 // fclamp { z0.s-z3.s }, z26.s, z31.s\n" - "st1w { z0.s }, p1, [x14]\n" - "add x14, x14, x0, LSL #2\n" - "st1w { z2.s }, p1, [x13]\n" + ".inst 0xc0066814 // mova { z20.d-z21.d }, za.d[x11, #0]\n" + ".inst 0xc0066836 // mova { z22.d-z23.d }, za.d[x11, #1]\n" + ".inst 0xc1bccbb4 // fclamp { z20.s-z23.s }, z29.s, z28.s\n" + "st1w { z20.s }, p1, [x14]\n" + "add x14, x14, x5, LSL #2\n" + "st1w { z22.s }, p1, [x13]\n" "add x13, x13, x10, LSL #2\n" "add x11, x11, #0x2\n" - ".inst 0xc0040b00 // mova za.d[x8, #0], { z24.d-z25.d }\n" - "st1w { z1.s }, p1, [x9]\n" + ".inst 0xc0040bc0 // mova za.d[x8, #0], { z30.d-z31.d }\n" + "st1w { z21.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - ".inst 0xc0040b01 // mova za.d[x8, #1], { z24.d-z25.d }\n" - "st1w { z3.s }, p1, [x28]\n" + ".inst 0xc0040bc1 // mova za.d[x8, #1], { z30.d-z31.d }\n" + "st1w { z23.s }, p1, [x28]\n" "add x28, x28, x26, LSL #2\n" "20:" // Main loop skip tail "cbz x15, 22f\n" @@ -1100,16 +1100,16 @@ void sme2_fp32bf16fp32_planar_5x5_s1_4rows_dot_za_impl( "add x8, x8, #0x2\n" "subs x15, x15, #0x1\n" ".inst 0xc0066822 // mova { z2.d-z3.d }, za.d[x11, #1]\n" - ".inst 0xc1bfcb40 // fclamp { z0.s-z3.s }, z26.s, z31.s\n" + ".inst 0xc1bccba0 // fclamp { z0.s-z3.s }, z29.s, z28.s\n" "st1w { z0.s }, p1, [x14]\n" - "add x14, x14, x0, LSL #2\n" + "add x14, x14, x5, LSL #2\n" "st1w { z2.s }, p1, [x13]\n" "add x13, x13, x10, LSL #2\n" "add x11, x11, #0x2\n" - ".inst 0xc0040b00 // mova za.d[x8, #0], { z24.d-z25.d }\n" + ".inst 0xc0040bc0 // mova za.d[x8, #0], { z30.d-z31.d }\n" "st1w { z1.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - ".inst 0xc0040b01 // mova za.d[x8, #1], { z24.d-z25.d }\n" + ".inst 0xc0040bc1 // mova za.d[x8, #1], { z30.d-z31.d }\n" "st1w { z3.s }, p1, [x28]\n" "add x28, x28, x26, LSL #2\n" "bgt 21b\n" @@ -1118,12 +1118,12 @@ void sme2_fp32bf16fp32_planar_5x5_s1_4rows_dot_za_impl( "incb x20, ALL, MUL #16\n" "incb x20, ALL, MUL #9\n" "str x20, [%x[args], %[offsetof_Args_weights]]\n" - "ldr x20, [%x[args], %[offsetof_Args_ld_in_vl]]\n" - "incw x7\n" - "whilelt p1.s, x7, x6\n" - "ldr x16, [%x[args], %[offsetof_Args_inptr]]\n" - "add x16, x16, x20, LSL #2\n" - "str x16, [%x[args], %[offsetof_Args_inptr]]\n" + "ldr x21, [%x[args], %[offsetof_Args_ld_in_vl]]\n" + "incw x17\n" + "whilelt p1.s, x17, x7\n" + "ldr x20, [%x[args], %[offsetof_Args_inptr]]\n" + "add x20, x20, x21, LSL #2\n" + "str x20, [%x[args], %[offsetof_Args_inptr]]\n" "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" "ldr x24, [%x[args], %[offsetof_Args_ld_out_vls]]\n" "ldp x23, x22, [x25, #0x0]\n" @@ -1141,7 +1141,7 @@ void sme2_fp32bf16fp32_planar_5x5_s1_4rows_dot_za_impl( ".inst 0xd503467f // SMSTOP\n" : : [args] "r" (&args), [ld_in_col] "r" (ld_in_col), [ld_in_row] "r" (ld_in_row), [offsetof_Args_bias] "I" (offsetof(Args, bias)), [offsetof_Args_clamp_max] "I" (offsetof(Args, clamp_max)), [offsetof_Args_clamp_min] "I" (offsetof(Args, clamp_min)), [offsetof_Args_current_channel] "I" (offsetof(Args, current_channel)), [offsetof_Args_inptr] "I" (offsetof(Args, inptr)), [offsetof_Args_input_cols] "I" (offsetof(Args, input_cols)), [offsetof_Args_ld_in_vl] "I" (offsetof(Args, ld_in_vl)), [offsetof_Args_ld_out_cols] "I" (offsetof(Args, ld_out_cols)), [offsetof_Args_ld_out_vls] "I" (offsetof(Args, ld_out_vls)), [offsetof_Args_n_channels] "I" (offsetof(Args, n_channels)), [offsetof_Args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_Args_output_cols] "I" (offsetof(Args, output_cols)), [offsetof_Args_pad_bottom] "I" (offsetof(Args, pad_bottom)), [offsetof_Args_pad_left] "I" (offsetof(Args, pad_left)), [offsetof_Args_pad_top] "I" (offsetof(Args, pad_top)), [offsetof_Args_weights] "I" (offsetof(Args, weights)) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x0", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_5x5_s2_4rows_dot_za.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_5x5_s2_4rows_dot_za.hpp index c99cf51da4d89ac86dc8a7a4572b620d65981262..53e596418bd90ba0cae42037e9ac4ea332a2140c 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_5x5_s2_4rows_dot_za.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_5x5_s2_4rows_dot_za.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,8 @@ * SOFTWARE. */ +#include "src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp" + namespace arm_conv { namespace depthwise { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_5x5_s2_4rows_dot_za/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_5x5_s2_4rows_dot_za/generic.cpp index 01f689a0b4cd3ed6198d601b5922eab4c0a0cd12..3a56e69d26cb2e3e374ad913d3ff061bc9046735 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_5x5_s2_4rows_dot_za/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32bf16fp32_planar_5x5_s2_4rows_dot_za/generic.cpp @@ -76,134 +76,134 @@ void sme2_fp32bf16fp32_planar_5x5_s2_4rows_dot_za_impl( "ldr x4, [%x[args], %[offsetof_Args_pad_top]]\n" "ptrue p2.b\n" ".inst 0x25207812 // ptrue pn10.b\n" - "ld1rw { z30.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_min]]\n" + "ld1rw { z13.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_min]]\n" "ldr x5, [%x[args], %[offsetof_Args_n_channels]]\n" "whilelt p1.s, XZR, x5\n" "whilelt p9.s, XZR, x20\n" - "ld1rw { z22.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_max]]\n" + "ld1rw { z12.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_max]]\n" "whilelt p8.s, XZR, x4\n" "addvl SP, SP, #-15\n" "ldr x6, [%x[args], %[offsetof_Args_current_channel]]\n" "eor p8.b, p2/Z, p8.b, p9.b\n" "1:" // Channel loop "ldr x20, [%x[args], %[offsetof_Args_bias]]\n" - "fmov z4.s, #0x0\n" + "fmov z16.s, #0x0\n" "cbz x20, 2f\n" - "ld1w { z4.s }, p1/Z, [x20, x6, LSL #2]\n" + "ld1w { z16.s }, p1/Z, [x20, x6, LSL #2]\n" "2:" // Load bias: Done - "ldr x20, [%x[args], %[offsetof_Args_weights]]\n" - "mov x21, x20\n" - "ld1w { z31.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - "ld1w { z16.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - ".inst 0x658aabe1 // bfcvt z1.h, p2/M, z31.s\n" - "incb x20\n" - "ld1w { z13.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - ".inst 0x658aa9a9 // bfcvt z9.h, p2/M, z13.s\n" + "ldr x21, [%x[args], %[offsetof_Args_weights]]\n" + "mov x20, x21\n" + "ld1w { z31.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" + "ld1w { z8.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" + ".inst 0x658aabef // bfcvt z15.h, p2/M, z31.s\n" + "incb x21\n" + "ld1w { z18.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" + ".inst 0x658aaa4e // bfcvt z14.h, p2/M, z18.s\n" "addvl x24, SP, #15\n" - "ld1w { z18.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - ".inst 0x648aaa01 // bfcvtnt z1.h, p2/M, z16.s\n" + "ld1w { z17.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" + ".inst 0x648aa90f // bfcvtnt z15.h, p2/M, z8.s\n" "addvl x24, x24, #-3\n" - "ld1w { z15.s }, p2/Z, [x21]\n" - "mov x21, x20\n" - "st1h { z1.h }, p2, [x24]\n" - ".inst 0x648aaa49 // bfcvtnt z9.h, p2/M, z18.s\n" - "ld1w { z31.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - ".inst 0x658aabe1 // bfcvt z1.h, p2/M, z31.s\n" - "incb x20\n" - "ld1w { z16.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - "st1h { z9.h }, p2, [x24, #1, MUL VL]\n" - ".inst 0x658aa9e2 // bfcvt z2.h, p2/M, z15.s\n" - "ld1w { z13.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - ".inst 0x658aa9a9 // bfcvt z9.h, p2/M, z13.s\n" - ".inst 0x648aaa01 // bfcvtnt z1.h, p2/M, z16.s\n" - "ld1w { z18.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - "st1h { z2.h }, p2, [x24, #2, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x20]\n" + "mov x20, x21\n" + "st1h { z15.h }, p2, [x24]\n" + ".inst 0x648aaa2e // bfcvtnt z14.h, p2/M, z17.s\n" + "ld1w { z29.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" + ".inst 0x658aabb5 // bfcvt z21.h, p2/M, z29.s\n" + "incb x21\n" + "ld1w { z17.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" + "st1h { z14.h }, p2, [x24, #1, MUL VL]\n" + ".inst 0x658aaa58 // bfcvt z24.h, p2/M, z18.s\n" + "ld1w { z26.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" + ".inst 0x658aab41 // bfcvt z1.h, p2/M, z26.s\n" + ".inst 0x648aaa35 // bfcvtnt z21.h, p2/M, z17.s\n" + "ld1w { z17.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" + "st1h { z24.h }, p2, [x24, #2, MUL VL]\n" "addvl x24, x24, #-3\n" - "ld1w { z15.s }, p2/Z, [x21]\n" - "mov x21, x20\n" - "st1h { z1.h }, p2, [x24]\n" - ".inst 0x648aaa49 // bfcvtnt z9.h, p2/M, z18.s\n" - "ld1w { z31.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - "incb x20\n" - ".inst 0x658aabe1 // bfcvt z1.h, p2/M, z31.s\n" - "ld1w { z16.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - ".inst 0x658aa9e2 // bfcvt z2.h, p2/M, z15.s\n" - "st1h { z9.h }, p2, [x24, #1, MUL VL]\n" - "ld1w { z13.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - ".inst 0x658aa9a9 // bfcvt z9.h, p2/M, z13.s\n" - "st1h { z2.h }, p2, [x24, #2, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" + "ld1w { z9.s }, p2/Z, [x20]\n" + "mov x20, x21\n" + "st1h { z21.h }, p2, [x24]\n" + ".inst 0x648aaa21 // bfcvtnt z1.h, p2/M, z17.s\n" + "ld1w { z3.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" + "incb x21\n" + ".inst 0x658aa864 // bfcvt z4.h, p2/M, z3.s\n" + "ld1w { z31.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" + ".inst 0x658aa92b // bfcvt z11.h, p2/M, z9.s\n" + "st1h { z1.h }, p2, [x24, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" + ".inst 0x658aaa46 // bfcvt z6.h, p2/M, z18.s\n" + "st1h { z11.h }, p2, [x24, #2, MUL VL]\n" + "ld1w { z5.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" "addvl x24, x24, #-3\n" - ".inst 0x648aaa01 // bfcvtnt z1.h, p2/M, z16.s\n" - "ld1w { z15.s }, p2/Z, [x21]\n" - "mov x21, x20\n" - "st1h { z1.h }, p2, [x24]\n" - ".inst 0x648aaa49 // bfcvtnt z9.h, p2/M, z18.s\n" - "ld1w { z31.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - ".inst 0x658aabe1 // bfcvt z1.h, p2/M, z31.s\n" - "incb x20\n" - "ld1w { z16.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - ".inst 0x658aa9e2 // bfcvt z2.h, p2/M, z15.s\n" - "st1h { z9.h }, p2, [x24, #1, MUL VL]\n" - "ld1w { z13.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" - ".inst 0x648aaa01 // bfcvtnt z1.h, p2/M, z16.s\n" - ".inst 0x658aa9a9 // bfcvt z9.h, p2/M, z13.s\n" - "ld1w { z18.s }, p2/Z, [x21]\n" - "incb x21, ALL, MUL #5\n" + ".inst 0x648aabe4 // bfcvtnt z4.h, p2/M, z31.s\n" + "ld1w { z27.s }, p2/Z, [x20]\n" + "mov x20, x21\n" + "st1h { z4.h }, p2, [x24]\n" + ".inst 0x648aa8a6 // bfcvtnt z6.h, p2/M, z5.s\n" + "ld1w { z9.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" + ".inst 0x658aa938 // bfcvt z24.h, p2/M, z9.s\n" + "incb x21\n" + "ld1w { z17.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" + ".inst 0x658aab75 // bfcvt z21.h, p2/M, z27.s\n" + "st1h { z6.h }, p2, [x24, #1, MUL VL]\n" + "ld1w { z31.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" + ".inst 0x648aaa38 // bfcvtnt z24.h, p2/M, z17.s\n" + ".inst 0x658aabf9 // bfcvt z25.h, p2/M, z31.s\n" + "ld1w { z18.s }, p2/Z, [x20]\n" + "incb x20, ALL, MUL #5\n" "ldr x7, [%x[args], %[offsetof_Args_input_cols]]\n" - "st1h { z2.h }, p2, [x24, #2, MUL VL]\n" - "ld1w { z15.s }, p2/Z, [x21]\n" - "mov x21, x20\n" + "st1h { z21.h }, p2, [x24, #2, MUL VL]\n" + "ld1w { z11.s }, p2/Z, [x20]\n" + "mov x21, x21\n" "addvl x24, x24, #-3\n" - "st1h { z1.h }, p2, [x24]\n" - "ld1w { z31.s }, p2/Z, [x21]\n" + "st1h { z24.h }, p2, [x24]\n" + "ld1w { z17.s }, p2/Z, [x21]\n" "incb x21, ALL, MUL #5\n" - ".inst 0x648aaa49 // bfcvtnt z9.h, p2/M, z18.s\n" - "st1h { z9.h }, p2, [x24, #1, MUL VL]\n" - "ld1w { z16.s }, p2/Z, [x21]\n" + ".inst 0x648aaa59 // bfcvtnt z25.h, p2/M, z18.s\n" + "st1h { z25.h }, p2, [x24, #1, MUL VL]\n" + "ld1w { z8.s }, p2/Z, [x21]\n" "incb x21, ALL, MUL #5\n" - ".inst 0x658aabe1 // bfcvt z1.h, p2/M, z31.s\n" - ".inst 0x658aa9e2 // bfcvt z2.h, p2/M, z15.s\n" - "ld1w { z13.s }, p2/Z, [x21]\n" + ".inst 0x658aaa29 // bfcvt z9.h, p2/M, z17.s\n" + ".inst 0x658aa976 // bfcvt z22.h, p2/M, z11.s\n" + "ld1w { z28.s }, p2/Z, [x21]\n" "incb x21, ALL, MUL #5\n" - ".inst 0x658aa9a9 // bfcvt z9.h, p2/M, z13.s\n" + ".inst 0x658aab85 // bfcvt z5.h, p2/M, z28.s\n" "ldr x17, [%x[args], %[offsetof_Args_inptr]]\n" - "ld1w { z18.s }, p2/Z, [x21]\n" + "ld1w { z25.s }, p2/Z, [x21]\n" "incb x21, ALL, MUL #5\n" "sub x20, x7, #0x1\n" - "st1h { z2.h }, p2, [x24, #2, MUL VL]\n" - "ld1w { z15.s }, p2/Z, [x21]\n" + "st1h { z22.h }, p2, [x24, #2, MUL VL]\n" + "ld1w { z11.s }, p2/Z, [x21]\n" "orr x23, x20, %x[ld_in_col], LSL #18\n" "addvl x24, x24, #-3\n" - "mov z5.d, z4.d\n" + "mov z17.d, z16.d\n" "orr x23, x5, x23, LSL #20\n" "mov x22, #0xb\n" - "mov z6.d, z4.d\n" - "mov z7.d, z4.d\n" + "mov z18.d, z16.d\n" + "mov z19.d, z16.d\n" "add x21, x4, x3\n" "lsl x20, %x[ld_in_row], #0x2\n" - ".inst 0x648aaa01 // bfcvtnt z1.h, p2/M, z16.s\n" - "st1h { z1.h }, p2, [x24]\n" - ".inst 0x648aaa49 // bfcvtnt z9.h, p2/M, z18.s\n" - "st1h { z9.h }, p2, [x24, #1, MUL VL]\n" - ".inst 0x658aa9e2 // bfcvt z2.h, p2/M, z15.s\n" + ".inst 0x648aa909 // bfcvtnt z9.h, p2/M, z8.s\n" + "st1h { z9.h }, p2, [x24]\n" + ".inst 0x648aab25 // bfcvtnt z5.h, p2/M, z25.s\n" + "st1h { z5.h }, p2, [x24, #1, MUL VL]\n" + ".inst 0x658aa97b // bfcvt z27.h, p2/M, z11.s\n" "mov x8, #0x0\n" - "st1h { z2.h }, p2, [x24, #2, MUL VL]\n" + "st1h { z27.h }, p2, [x24, #2, MUL VL]\n" "ldr x16, [%x[args], %[offsetof_Args_output_cols]]\n" "lsl x23, x23, #0x2\n" "sub x22, x22, x21\n" @@ -213,20 +213,20 @@ void sme2_fp32bf16fp32_planar_5x5_s2_4rows_dot_za_impl( ".inst 0xf8b74a9c // rprfm pldstrm, x23, [x20]\n" "add x20, x20, %x[ld_in_col], LSL #2\n" "bgt 3b\n" - "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" + "ldr x23, [%x[args], %[offsetof_Args_outptrs]]\n" "lsl x20, %x[ld_in_row], #0x2\n" "msub x17, x4, x20, x17\n" - ".inst 0xc0040c80 // mova za.d[x8, #0], { z4.d-z7.d }\n" + ".inst 0xc0040e00 // mova za.d[x8, #0], { z16.d-z19.d }\n" "ldr x20, [%x[args], %[offsetof_Args_ld_out_cols]]\n" - ".inst 0xc0040c81 // mova za.d[x8, #1], { z4.d-z7.d }\n" + ".inst 0xc0040e01 // mova za.d[x8, #1], { z16.d-z19.d }\n" "mov x22, #0x4\n" - "ldp x15, x14, [x25], #0x10\n" - ".inst 0xc0040c82 // mova za.d[x8, #2], { z4.d-z7.d }\n" + "ldp x15, x14, [x23], #0x10\n" + ".inst 0xc0040e02 // mova za.d[x8, #2], { z16.d-z19.d }\n" "ldp x13, x11, [x20], #0x10\n" - ".inst 0xc0040c83 // mova za.d[x8, #3], { z4.d-z7.d }\n" + ".inst 0xc0040e03 // mova za.d[x8, #3], { z16.d-z19.d }\n" "ldr x21, [%x[args], %[offsetof_Args_pad_left]]\n" - ".inst 0xc0040c84 // mova za.d[x8, #4], { z4.d-z7.d }\n" - "ldp x10, x9, [x25], #0x10\n" + ".inst 0xc0040e04 // mova za.d[x8, #4], { z16.d-z19.d }\n" + "ldp x10, x9, [x23], #0x10\n" "ldp x28, x27, [x20], #0x10\n" "cbz x21, 5f\n" "cmp x21, x22\n" @@ -234,21 +234,21 @@ void sme2_fp32bf16fp32_planar_5x5_s2_4rows_dot_za_impl( "sub x21, x21, x20\n" "sub x22, x22, x20\n" "cbz x21, 5f\n" - ".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n" + ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" "and x22, x21, #0x1\n" "add x21, x21, #0x1\n" - ".inst 0xc1b6cbd8 // fclamp { z24.s-z27.s }, z30.s, z22.s\n" + ".inst 0xc1acc9a4 // fclamp { z4.s-z7.s }, z13.s, z12.s\n" "lsr x21, x21, #0x1\n" "sub x16, x16, x21\n" "4:" // Left padding "subs x21, x21, #0x1\n" - "st1w { z24.s }, p1, [x15]\n" + "st1w { z4.s }, p1, [x15]\n" "add x15, x15, x13, LSL #2\n" - "st1w { z25.s }, p1, [x14]\n" + "st1w { z5.s }, p1, [x14]\n" "add x14, x14, x11, LSL #2\n" - "st1w { z26.s }, p1, [x10]\n" + "st1w { z6.s }, p1, [x10]\n" "add x10, x10, x28, LSL #2\n" - "st1w { z27.s }, p1, [x9]\n" + "st1w { z7.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" "bgt 4b\n" "5:" // Left padding: End @@ -264,331 +264,331 @@ void sme2_fp32bf16fp32_planar_5x5_s2_4rows_dot_za_impl( "beq 7f\n" "6:" // Unpadded: 4 priming loads "add x21, x17, %x[ld_in_row], LSL #2\n" - "ld1w { z23.s }, p1/Z, [x17]\n" - ".inst 0x658aaaea // bfcvt z10.h, p2/M, z23.s\n" + "ld1w { z0.s }, p1/Z, [x17]\n" + ".inst 0x658aa816 // bfcvt z22.h, p2/M, z0.s\n" "addvl x20, SP, #12\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + "ld1w { z9.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0a // bfcvtnt z10.h, p2/M, z16.s\n" + ".inst 0x648aa936 // bfcvtnt z22.h, p2/M, z9.s\n" "add x17, x17, %x[ld_in_col], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + "ld1w { z28.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0x658aab97 // bfcvt z23.h, p2/M, z28.s\n" + "ld1w { z20.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0b // bfcvtnt z11.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0x648aaa97 // bfcvtnt z23.h, p2/M, z20.s\n" + "ld1w { z20.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0x658aaa98 // bfcvt z24.h, p2/M, z20.s\n" + "ld1w { z29.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0x648aabb8 // bfcvtnt z24.h, p2/M, z29.s\n" + "ld1w { z30.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0x658aabd9 // bfcvt z25.h, p2/M, z30.s\n" + "ld1w { z9.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0x648aa939 // bfcvtnt z25.h, p2/M, z9.s\n" + "ld1w { z26.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" - ".inst 0xa1402a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1311150 // bfdot za.s[x8, 0], { z10.h-z13.h }, z1.h\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0x658aab5a // bfcvt z26.h, p2/M, z26.s\n" + ".inst 0xa1402a83 // ld1h { z3.h, z11.h }, pn10.b/Z, [x20]\n" + ".inst 0xc13312d0 // bfdot za.s[x8, 0], { z22.h-z25.h }, z3.h\n" + "ld1w { z9.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" - ".inst 0xc1391170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z9.h\n" - "ld1w { z16.s }, p1/Z, [x21]\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - "ld1h { z2.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc1321190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z2.h\n" + ".inst 0x648aa93a // bfcvtnt z26.h, p2/M, z9.s\n" + ".inst 0xc13b12f0 // bfdot za.s[x8, 0], { z23.h-z26.h }, z11.h\n" + "ld1w { z9.s }, p1/Z, [x21]\n" + ".inst 0x658aa93b // bfcvt z27.h, p2/M, z9.s\n" + "ld1h { z9.h }, p2/Z, [x20, #2, MUL VL]\n" + ".inst 0xc1391310 // bfdot za.s[x8, 0], { z24.h-z27.h }, z9.h\n" "7:" // Unpadded: 3 priming loads "add x21, x17, %x[ld_in_row], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x17]\n" - ".inst 0x658aaa0a // bfcvt z10.h, p2/M, z16.s\n" + "ld1w { z27.s }, p1/Z, [x17]\n" + ".inst 0x658aab7d // bfcvt z29.h, p2/M, z27.s\n" "addvl x20, SP, #9\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + "ld1w { z26.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0a // bfcvtnt z10.h, p2/M, z16.s\n" + ".inst 0x648aab5d // bfcvtnt z29.h, p2/M, z26.s\n" "add x17, x17, %x[ld_in_col], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + "ld1w { z9.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0x658aa93e // bfcvt z30.h, p2/M, z9.s\n" + "ld1w { z20.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0b // bfcvtnt z11.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0x648aaa9e // bfcvtnt z30.h, p2/M, z20.s\n" + "ld1w { z25.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0x658aab3f // bfcvt z31.h, p2/M, z25.s\n" + "ld1w { z26.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0x648aab5f // bfcvtnt z31.h, p2/M, z26.s\n" + "ld1w { z27.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0x658aab60 // bfcvt z0.h, p2/M, z27.s\n" + "ld1w { z9.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0x648aa920 // bfcvtnt z0.h, p2/M, z9.s\n" + "ld1w { z23.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" - ".inst 0xa1402a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1311150 // bfdot za.s[x8, 0], { z10.h-z13.h }, z1.h\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0x658aaae1 // bfcvt z1.h, p2/M, z23.s\n" + ".inst 0xa0402a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20]\n" + ".inst 0xc13413b0 // bfdot za.s[x8, 0], { z29.h-z0.h }, z4.h\n" + "ld1w { z9.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" - ".inst 0xc1391170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z9.h\n" - "ld1w { z16.s }, p1/Z, [x21]\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - "ld1h { z2.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc1321190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z2.h\n" + ".inst 0x648aa921 // bfcvtnt z1.h, p2/M, z9.s\n" + ".inst 0xc13513d0 // bfdot za.s[x8, 0], { z30.h-z1.h }, z5.h\n" + "ld1w { z29.s }, p1/Z, [x21]\n" + ".inst 0x658aaba2 // bfcvt z2.h, p2/M, z29.s\n" + "ld1h { z9.h }, p2/Z, [x20, #2, MUL VL]\n" + ".inst 0xc13913f0 // bfdot za.s[x8, 0], { z31.h-z2.h }, z9.h\n" "8:" // Unpadded: 2 priming loads "add x22, x17, %x[ld_in_row], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x17]\n" - ".inst 0x658aaa0a // bfcvt z10.h, p2/M, z16.s\n" + "ld1w { z27.s }, p1/Z, [x17]\n" + ".inst 0x658aab7a // bfcvt z26.h, p2/M, z27.s\n" "addvl x21, SP, #6\n" - "ld1w { z16.s }, p1/Z, [x22]\n" + "ld1w { z21.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0a // bfcvtnt z10.h, p2/M, z16.s\n" + ".inst 0x648aaaba // bfcvtnt z26.h, p2/M, z21.s\n" "addvl x20, SP, #12\n" - "ld1w { z16.s }, p1/Z, [x22]\n" + "ld1w { z25.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" + ".inst 0x658aab3b // bfcvt z27.h, p2/M, z25.s\n" "add x17, x17, %x[ld_in_col], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x22]\n" + "ld1w { z4.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0b // bfcvtnt z11.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x22]\n" + ".inst 0x648aa89b // bfcvtnt z27.h, p2/M, z4.s\n" + "ld1w { z10.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x22]\n" + ".inst 0x658aa95c // bfcvt z28.h, p2/M, z10.s\n" + "ld1w { z4.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x22]\n" + ".inst 0x648aa89c // bfcvtnt z28.h, p2/M, z4.s\n" + "ld1w { z5.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x22]\n" + ".inst 0x658aa8bd // bfcvt z29.h, p2/M, z5.s\n" + "ld1w { z5.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x22]\n" + ".inst 0x648aa8bd // bfcvtnt z29.h, p2/M, z5.s\n" + "ld1w { z5.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" - ".inst 0xa1402aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1311150 // bfdot za.s[x8, 0], { z10.h-z13.h }, z1.h\n" - "ld1w { z16.s }, p1/Z, [x22]\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" + ".inst 0x658aa8be // bfcvt z30.h, p2/M, z5.s\n" + ".inst 0xa0402aae // ld1h { z14.h-z15.h }, pn10.b/Z, [x21]\n" + ".inst 0xc13e1350 // bfdot za.s[x8, 0], { z26.h-z29.h }, z14.h\n" + "ld1w { z5.s }, p1/Z, [x22]\n" + ".inst 0x648aa8be // bfcvtnt z30.h, p2/M, z5.s\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0xc1391170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z9.h\n" - ".inst 0xa1402a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1311151 // bfdot za.s[x8, 1], { z10.h-z13.h }, z1.h\n" - "ld1w { z16.s }, p1/Z, [x22]\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - ".inst 0xc1391171 // bfdot za.s[x8, 1], { z11.h-z14.h }, z9.h\n" - "ld1h { z2.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc1321190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z2.h\n" - "ld1h { z2.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc1321191 // bfdot za.s[x8, 1], { z12.h-z15.h }, z2.h\n" + ".inst 0xc13f1370 // bfdot za.s[x8, 0], { z27.h-z30.h }, z15.h\n" + ".inst 0xa0402a88 // ld1h { z8.h-z9.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1381351 // bfdot za.s[x8, 1], { z26.h-z29.h }, z8.h\n" + "ld1w { z23.s }, p1/Z, [x22]\n" + ".inst 0x658aaaff // bfcvt z31.h, p2/M, z23.s\n" + ".inst 0xc1391371 // bfdot za.s[x8, 1], { z27.h-z30.h }, z9.h\n" + "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" + ".inst 0xc1301390 // bfdot za.s[x8, 0], { z28.h-z31.h }, z0.h\n" + "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" + ".inst 0xc1301391 // bfdot za.s[x8, 1], { z28.h-z31.h }, z0.h\n" "9:" // Unpadded: 1 priming loads "add x22, x17, %x[ld_in_row], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x17]\n" - ".inst 0x658aaa0a // bfcvt z10.h, p2/M, z16.s\n" + "ld1w { z27.s }, p1/Z, [x17]\n" + ".inst 0x658aab77 // bfcvt z23.h, p2/M, z27.s\n" "addvl x21, SP, #3\n" - "ld1w { z16.s }, p1/Z, [x22]\n" + "ld1w { z24.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0a // bfcvtnt z10.h, p2/M, z16.s\n" + ".inst 0x648aab17 // bfcvtnt z23.h, p2/M, z24.s\n" "addvl x20, SP, #9\n" - "ld1w { z16.s }, p1/Z, [x22]\n" + "ld1w { z31.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" + ".inst 0x658aabf8 // bfcvt z24.h, p2/M, z31.s\n" "add x17, x17, %x[ld_in_col], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x22]\n" + "ld1w { z6.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0b // bfcvtnt z11.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x22]\n" + ".inst 0x648aa8d8 // bfcvtnt z24.h, p2/M, z6.s\n" + "ld1w { z28.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x22]\n" + ".inst 0x658aab99 // bfcvt z25.h, p2/M, z28.s\n" + "ld1w { z26.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x22]\n" + ".inst 0x648aab59 // bfcvtnt z25.h, p2/M, z26.s\n" + "ld1w { z28.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x22]\n" + ".inst 0x658aab9a // bfcvt z26.h, p2/M, z28.s\n" + "ld1w { z4.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x22]\n" + ".inst 0x648aa89a // bfcvtnt z26.h, p2/M, z4.s\n" + "ld1w { z20.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" - ".inst 0xa1402aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1311150 // bfdot za.s[x8, 0], { z10.h-z13.h }, z1.h\n" - "ld1w { z16.s }, p1/Z, [x22]\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" + ".inst 0x658aaa9b // bfcvt z27.h, p2/M, z20.s\n" + ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" + ".inst 0xc13012f0 // bfdot za.s[x8, 0], { z23.h-z26.h }, z0.h\n" + "ld1w { z20.s }, p1/Z, [x22]\n" + ".inst 0x648aaa9b // bfcvtnt z27.h, p2/M, z20.s\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0xc1391170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z9.h\n" - ".inst 0xa1402a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1311151 // bfdot za.s[x8, 1], { z10.h-z13.h }, z1.h\n" - "ld1w { z16.s }, p1/Z, [x22]\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - ".inst 0xc1391171 // bfdot za.s[x8, 1], { z11.h-z14.h }, z9.h\n" - "ld1h { z2.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc1321190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z2.h\n" - "ld1h { z2.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc1321191 // bfdot za.s[x8, 1], { z12.h-z15.h }, z2.h\n" + ".inst 0xc1381310 // bfdot za.s[x8, 0], { z24.h-z27.h }, z8.h\n" + ".inst 0xa0402a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20]\n" + ".inst 0xc13212f1 // bfdot za.s[x8, 1], { z23.h-z26.h }, z2.h\n" + "ld1w { z11.s }, p1/Z, [x22]\n" + ".inst 0x658aa97c // bfcvt z28.h, p2/M, z11.s\n" + ".inst 0xc1331311 // bfdot za.s[x8, 1], { z24.h-z27.h }, z3.h\n" + "ld1h { z4.h }, p2/Z, [x21, #2, MUL VL]\n" + ".inst 0xc1341330 // bfdot za.s[x8, 0], { z25.h-z28.h }, z4.h\n" + "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" + ".inst 0xc1301331 // bfdot za.s[x8, 1], { z25.h-z28.h }, z0.h\n" "10:" // Unpadded: 0 priming loads "cmp x7, #0x2\n" - ".inst 0xa1402be1 // ld1h { z1.h, z9.h }, pn10.b/Z, [SP]\n" - "ld1h { z2.h }, p2/Z, [SP, #2, MUL VL]\n" + ".inst 0xa1402be3 // ld1h { z3.h, z11.h }, pn10.b/Z, [SP]\n" + "ld1h { z7.h }, p2/Z, [SP, #2, MUL VL]\n" "blt 20f\n" "add x21, x17, %x[ld_in_row], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x17]\n" - ".inst 0x658aaa0a // bfcvt z10.h, p2/M, z16.s\n" + "ld1w { z27.s }, p1/Z, [x17]\n" + ".inst 0x658aab75 // bfcvt z21.h, p2/M, z27.s\n" "sub x7, x7, #0x2\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + "ld1w { z26.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" "sub x16, x16, #0x1\n" - ".inst 0x648aaa0a // bfcvtnt z10.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0x648aab55 // bfcvtnt z21.h, p2/M, z26.s\n" + "ld1w { z26.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" + ".inst 0x658aab56 // bfcvt z22.h, p2/M, z26.s\n" "lsr x20, x7, #0x1\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + "ld1w { z26.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" "cmp x20, x16\n" - ".inst 0x648aaa0b // bfcvtnt z11.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0x648aab56 // bfcvtnt z22.h, p2/M, z26.s\n" + "ld1w { z8.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + ".inst 0x658aa917 // bfcvt z23.h, p2/M, z8.s\n" "csel x26, x20, x16, LT\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + "ld1w { z2.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + ".inst 0x648aa857 // bfcvtnt z23.h, p2/M, z2.s\n" "add x17, x17, %x[ld_in_col], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + "ld1w { z6.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0x658aa8d8 // bfcvt z24.h, p2/M, z6.s\n" "and x7, x7, #0x1\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + "ld1w { z15.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + ".inst 0x648aa9f8 // bfcvtnt z24.h, p2/M, z15.s\n" "sub x16, x16, x26\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + "ld1w { z27.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x21]\n" + ".inst 0x658aab79 // bfcvt z25.h, p2/M, z27.s\n" + "ld1w { z26.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x21]\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" + ".inst 0x648aab59 // bfcvtnt z25.h, p2/M, z26.s\n" + "ld1w { z27.s }, p1/Z, [x21]\n" + ".inst 0x658aab7a // bfcvt z26.h, p2/M, z27.s\n" "cbz x26, 19f\n" "11:" // Unpadded: Main loop - ".inst 0xc1311150 // bfdot za.s[x8, 0], { z10.h-z13.h }, z1.h\n" + ".inst 0xc13312b0 // bfdot za.s[x8, 0], { z21.h-z24.h }, z3.h\n" "addvl x25, SP, #6\n" "addvl x24, SP, #12\n" - "ld1w { z18.s }, p1/Z, [x17]\n" - ".inst 0xc1391170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z9.h\n" - ".inst 0xa1402b21 // ld1h { z1.h, z9.h }, pn10.b/Z, [x25]\n" + "ld1w { z14.s }, p1/Z, [x17]\n" + ".inst 0xc13b12d0 // bfdot za.s[x8, 0], { z22.h-z25.h }, z11.h\n" + ".inst 0xa1402b20 // ld1h { z0.h, z8.h }, pn10.b/Z, [x25]\n" "add x23, x17, %x[ld_in_row], LSL #2\n" "addvl x22, SP, #3\n" - ".inst 0xc1311151 // bfdot za.s[x8, 1], { z10.h-z13.h }, z1.h\n" - "ld1w { z17.s }, p1/Z, [x23]\n" + ".inst 0xc13012b1 // bfdot za.s[x8, 1], { z21.h-z24.h }, z0.h\n" + "ld1w { z27.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row], LSL #2\n" "add x17, x17, %x[ld_in_col], LSL #2\n" - ".inst 0xc1391171 // bfdot za.s[x8, 1], { z11.h-z14.h }, z9.h\n" - ".inst 0xa1402b01 // ld1h { z1.h, z9.h }, pn10.b/Z, [x24]\n" + ".inst 0xc13812d1 // bfdot za.s[x8, 1], { z22.h-z25.h }, z8.h\n" + ".inst 0xa1402b00 // ld1h { z0.h, z8.h }, pn10.b/Z, [x24]\n" "addvl x21, SP, #9\n" "add x20, x17, %x[ld_in_row], LSL #2\n" - ".inst 0xc1311152 // bfdot za.s[x8, 2], { z10.h-z13.h }, z1.h\n" - "ld1w { z16.s }, p1/Z, [x23]\n" + ".inst 0xc13012b2 // bfdot za.s[x8, 2], { z21.h-z24.h }, z0.h\n" + "ld1w { z2.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa4a // bfcvt z10.h, p2/M, z18.s\n" - ".inst 0xc1321190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z2.h\n" - "ld1h { z2.h }, p2/Z, [x25, #2, MUL VL]\n" - ".inst 0x648aaa2a // bfcvtnt z10.h, p2/M, z17.s\n" + ".inst 0x658aa9d5 // bfcvt z21.h, p2/M, z14.s\n" + ".inst 0xc13712f0 // bfdot za.s[x8, 0], { z23.h-z26.h }, z7.h\n" + "ld1h { z11.h }, p2/Z, [x25, #2, MUL VL]\n" + ".inst 0x648aab75 // bfcvtnt z21.h, p2/M, z27.s\n" "subs x26, x26, #0x1\n" - "ld1w { z17.s }, p1/Z, [x23]\n" + "ld1w { z14.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row], LSL #2\n" - ".inst 0xc1391172 // bfdot za.s[x8, 2], { z11.h-z14.h }, z9.h\n" - ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x23]\n" + ".inst 0xc13812d2 // bfdot za.s[x8, 2], { z22.h-z25.h }, z8.h\n" + ".inst 0x658aa856 // bfcvt z22.h, p2/M, z2.s\n" + "ld1w { z7.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row], LSL #2\n" - ".inst 0xc1321191 // bfdot za.s[x8, 1], { z12.h-z15.h }, z2.h\n" - ".inst 0x648aaa2b // bfcvtnt z11.h, p2/M, z17.s\n" - "ld1w { z17.s }, p1/Z, [x23]\n" + ".inst 0xc13b12f1 // bfdot za.s[x8, 1], { z23.h-z26.h }, z11.h\n" + ".inst 0x648aa9d6 // bfcvtnt z22.h, p2/M, z14.s\n" + "ld1w { z31.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row], LSL #2\n" - ".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n" - ".inst 0xc1b6cbd8 // fclamp { z24.s-z27.s }, z30.s, z22.s\n" - "ld1h { z2.h }, p2/Z, [x24, #2, MUL VL]\n" - ".inst 0xc1321192 // bfdot za.s[x8, 2], { z12.h-z15.h }, z2.h\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + ".inst 0xc0060c08 // mova { z8.d-z11.d }, za.d[x8, #0]\n" + ".inst 0xc1acc9a8 // fclamp { z8.s-z11.s }, z13.s, z12.s\n" + "ld1h { z0.h }, p2/Z, [x24, #2, MUL VL]\n" + ".inst 0xc13012f2 // bfdot za.s[x8, 2], { z23.h-z26.h }, z0.h\n" + ".inst 0x658aa8f7 // bfcvt z23.h, p2/M, z7.s\n" "add x8, x8, #0x1\n" - "ld1w { z16.s }, p1/Z, [x23]\n" + "ld1w { z26.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" - ".inst 0x648aaa2c // bfcvtnt z12.h, p2/M, z17.s\n" - "ld1w { z16.s }, p1/Z, [x23]\n" + ".inst 0x658aab58 // bfcvt z24.h, p2/M, z26.s\n" + ".inst 0x648aabf7 // bfcvtnt z23.h, p2/M, z31.s\n" + "ld1w { z2.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" - "st1w { z24.s }, p1, [x15]\n" - "ld1w { z16.s }, p1/Z, [x23]\n" + ".inst 0x648aa858 // bfcvtnt z24.h, p2/M, z2.s\n" + "st1w { z8.s }, p1, [x15]\n" + "ld1w { z0.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" + ".inst 0x658aa819 // bfcvt z25.h, p2/M, z0.s\n" "add x15, x15, x13, LSL #2\n" - ".inst 0xa1402ac1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x22]\n" - ".inst 0xc1311150 // bfdot za.s[x8, 0], { z10.h-z13.h }, z1.h\n" - "st1w { z25.s }, p1, [x14]\n" + ".inst 0xa0402ac2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x22]\n" + ".inst 0xc13212b0 // bfdot za.s[x8, 0], { z21.h-z24.h }, z2.h\n" + "st1w { z9.s }, p1, [x14]\n" "add x14, x14, x11, LSL #2\n" - "ld1w { z16.s }, p1/Z, [x23]\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" + "ld1w { z26.s }, p1/Z, [x23]\n" + ".inst 0x648aab59 // bfcvtnt z25.h, p2/M, z26.s\n" "add x23, x23, %x[ld_in_row], LSL #2\n" - ".inst 0xc1391170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z9.h\n" + ".inst 0xc13312d0 // bfdot za.s[x8, 0], { z22.h-z25.h }, z3.h\n" ".inst 0xa1402aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1311151 // bfdot za.s[x8, 1], { z10.h-z13.h }, z1.h\n" - "st1w { z26.s }, p1, [x10]\n" + ".inst 0xc13112b1 // bfdot za.s[x8, 1], { z21.h-z24.h }, z1.h\n" + "st1w { z10.s }, p1, [x10]\n" "add x10, x10, x28, LSL #2\n" - "ld1w { z16.s }, p1/Z, [x23]\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - ".inst 0xc1391171 // bfdot za.s[x8, 1], { z11.h-z14.h }, z9.h\n" - "ld1w { z16.s }, p1/Z, [x17]\n" - ".inst 0x658aaa0a // bfcvt z10.h, p2/M, z16.s\n" - "st1w { z27.s }, p1, [x9]\n" + "ld1w { z26.s }, p1/Z, [x23]\n" + ".inst 0x658aab5a // bfcvt z26.h, p2/M, z26.s\n" + ".inst 0xc13912d1 // bfdot za.s[x8, 1], { z22.h-z25.h }, z9.h\n" + "ld1w { z31.s }, p1/Z, [x17]\n" + ".inst 0x658aabf5 // bfcvt z21.h, p2/M, z31.s\n" + "st1w { z11.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + "ld1w { z30.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0xc0040c84 // mova za.d[x8, #4], { z4.d-z7.d }\n" - ".inst 0x648aaa0a // bfcvtnt z10.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + ".inst 0xc0040e04 // mova za.d[x8, #4], { z16.d-z19.d }\n" + ".inst 0x648aabd5 // bfcvtnt z21.h, p2/M, z30.s\n" + "ld1w { z0.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" + ".inst 0x658aa816 // bfcvt z22.h, p2/M, z0.s\n" "add x17, x17, %x[ld_in_col], LSL #2\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + "ld1w { z1.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0b // bfcvtnt z11.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + ".inst 0x648aa836 // bfcvtnt z22.h, p2/M, z1.s\n" + "ld1w { z11.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "ld1h { z2.h }, p2/Z, [x22, #2, MUL VL]\n" - ".inst 0xc1321190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z2.h\n" - "ld1w { z19.s }, p1/Z, [x20]\n" + ".inst 0xc13212f0 // bfdot za.s[x8, 0], { z23.h-z26.h }, z2.h\n" + "ld1w { z28.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z17.s }, p1/Z, [x20]\n" + "ld1w { z14.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1w { z18.s }, p1/Z, [x20]\n" + "ld1w { z27.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - "ld1h { z2.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc1321191 // bfdot za.s[x8, 1], { z12.h-z15.h }, z2.h\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" + "ld1h { z4.h }, p2/Z, [x21, #2, MUL VL]\n" + ".inst 0xc13412f1 // bfdot za.s[x8, 1], { z23.h-z26.h }, z4.h\n" + ".inst 0x658aa977 // bfcvt z23.h, p2/M, z11.s\n" + "ld1w { z29.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa2d // bfcvt z13.h, p2/M, z17.s\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" - "ld1w { z17.s }, p1/Z, [x20]\n" + ".inst 0x658aa9d8 // bfcvt z24.h, p2/M, z14.s\n" + ".inst 0x658aabb9 // bfcvt z25.h, p2/M, z29.s\n" + "ld1w { z5.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa6c // bfcvtnt z12.h, p2/M, z19.s\n" - ".inst 0x648aaa4d // bfcvtnt z13.h, p2/M, z18.s\n" - "ld1w { z16.s }, p1/Z, [x20]\n" - ".inst 0x648aaa2e // bfcvtnt z14.h, p2/M, z17.s\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - ".inst 0xa1402be1 // ld1h { z1.h, z9.h }, pn10.b/Z, [SP]\n" - "ld1h { z2.h }, p2/Z, [SP, #2, MUL VL]\n" + ".inst 0x648aab97 // bfcvtnt z23.h, p2/M, z28.s\n" + ".inst 0x648aab78 // bfcvtnt z24.h, p2/M, z27.s\n" + "ld1w { z11.s }, p1/Z, [x20]\n" + ".inst 0x648aa8b9 // bfcvtnt z25.h, p2/M, z5.s\n" + ".inst 0x658aa97a // bfcvt z26.h, p2/M, z11.s\n" + ".inst 0xa1402be3 // ld1h { z3.h, z11.h }, pn10.b/Z, [SP]\n" + "ld1h { z7.h }, p2/Z, [SP, #2, MUL VL]\n" "bgt 11b\n" "b 19f\n" "12:" // Padded @@ -603,282 +603,282 @@ void sme2_fp32bf16fp32_planar_5x5_s2_4rows_dot_za_impl( "13:" // Padded: 4 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x17]\n" - ".inst 0x658aaa0a // bfcvt z10.h, p2/M, z16.s\n" + "ld1w { z1.s }, p0/Z, [x17]\n" + ".inst 0x658aa837 // bfcvt z23.h, p2/M, z1.s\n" "add x21, x17, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z16.s }, p0/Z, [x21]\n" - ".inst 0x648aaa0a // bfcvtnt z10.h, p2/M, z16.s\n" + "ld1w { z29.s }, p0/Z, [x21]\n" + ".inst 0x648aabb7 // bfcvtnt z23.h, p2/M, z29.s\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x21]\n" - ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" + "ld1w { z30.s }, p0/Z, [x21]\n" + ".inst 0x658aabd8 // bfcvt z24.h, p2/M, z30.s\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z16.s }, p0/Z, [x21]\n" - ".inst 0x648aaa0b // bfcvtnt z11.h, p2/M, z16.s\n" + "ld1w { z15.s }, p0/Z, [x21]\n" + ".inst 0x648aa9f8 // bfcvtnt z24.h, p2/M, z15.s\n" "mov x12, #0x4\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x21]\n" + "ld1w { z27.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + ".inst 0x658aab79 // bfcvt z25.h, p2/M, z27.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z16.s }, p0/Z, [x21]\n" + "ld1w { z20.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + ".inst 0x648aaa99 // bfcvtnt z25.h, p2/M, z20.s\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x21]\n" + "ld1w { z10.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0x658aa95a // bfcvt z26.h, p2/M, z10.s\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x8\n" - "ld1w { z16.s }, p0/Z, [x21]\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + "ld1w { z8.s }, p0/Z, [x21]\n" + ".inst 0x648aa91a // bfcvtnt z26.h, p2/M, z8.s\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x21]\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" + "ld1w { z28.s }, p0/Z, [x21]\n" + ".inst 0x658aab9b // bfcvt z27.h, p2/M, z28.s\n" "addvl x20, SP, #12\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0xa1402a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1311150 // bfdot za.s[x8, 0], { z10.h-z13.h }, z1.h\n" + ".inst 0xc13112f0 // bfdot za.s[x8, 0], { z23.h-z26.h }, z1.h\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z16.s }, p0/Z, [x21]\n" + "ld1w { z28.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" + ".inst 0x648aab9b // bfcvtnt z27.h, p2/M, z28.s\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x21]\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - ".inst 0xc1391170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z9.h\n" - "ld1h { z2.h }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z0.s }, p0/Z, [x21]\n" + ".inst 0x658aa81c // bfcvt z28.h, p2/M, z0.s\n" + ".inst 0xc1391310 // bfdot za.s[x8, 0], { z24.h-z27.h }, z9.h\n" + "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" "add x17, x17, %x[ld_in_col], LSL #2\n" - ".inst 0xc1321190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z2.h\n" + ".inst 0xc1301330 // bfdot za.s[x8, 0], { z25.h-z28.h }, z0.h\n" "14:" // Padded: 3 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x17]\n" - ".inst 0x658aaa0a // bfcvt z10.h, p2/M, z16.s\n" + "ld1w { z21.s }, p0/Z, [x17]\n" + ".inst 0x658aaab4 // bfcvt z20.h, p2/M, z21.s\n" "add x21, x17, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z16.s }, p0/Z, [x21]\n" - ".inst 0x648aaa0a // bfcvtnt z10.h, p2/M, z16.s\n" + "ld1w { z27.s }, p0/Z, [x21]\n" + ".inst 0x648aab74 // bfcvtnt z20.h, p2/M, z27.s\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x21]\n" - ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" + "ld1w { z27.s }, p0/Z, [x21]\n" + ".inst 0x658aab75 // bfcvt z21.h, p2/M, z27.s\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z16.s }, p0/Z, [x21]\n" - ".inst 0x648aaa0b // bfcvtnt z11.h, p2/M, z16.s\n" + "ld1w { z27.s }, p0/Z, [x21]\n" + ".inst 0x648aab75 // bfcvtnt z21.h, p2/M, z27.s\n" "mov x12, #0x4\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x21]\n" + "ld1w { z29.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + ".inst 0x658aabb6 // bfcvt z22.h, p2/M, z29.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z16.s }, p0/Z, [x21]\n" + "ld1w { z27.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + ".inst 0x648aab76 // bfcvtnt z22.h, p2/M, z27.s\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x21]\n" + "ld1w { z27.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0x658aab77 // bfcvt z23.h, p2/M, z27.s\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x8\n" - "ld1w { z16.s }, p0/Z, [x21]\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + "ld1w { z8.s }, p0/Z, [x21]\n" + ".inst 0x648aa917 // bfcvtnt z23.h, p2/M, z8.s\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x21]\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" + "ld1w { z28.s }, p0/Z, [x21]\n" + ".inst 0x658aab98 // bfcvt z24.h, p2/M, z28.s\n" "addvl x20, SP, #9\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0xa1402a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1311150 // bfdot za.s[x8, 0], { z10.h-z13.h }, z1.h\n" + ".inst 0xc1311290 // bfdot za.s[x8, 0], { z20.h-z23.h }, z1.h\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z16.s }, p0/Z, [x21]\n" + "ld1w { z0.s }, p0/Z, [x21]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" + ".inst 0x648aa818 // bfcvtnt z24.h, p2/M, z0.s\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x21]\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - ".inst 0xc1391170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z9.h\n" - "ld1h { z2.h }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z1.s }, p0/Z, [x21]\n" + ".inst 0x658aa839 // bfcvt z25.h, p2/M, z1.s\n" + ".inst 0xc13912b0 // bfdot za.s[x8, 0], { z21.h-z24.h }, z9.h\n" + "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" "add x17, x17, %x[ld_in_col], LSL #2\n" - ".inst 0xc1321190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z2.h\n" + ".inst 0xc13012d0 // bfdot za.s[x8, 0], { z22.h-z25.h }, z0.h\n" "15:" // Padded: 2 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x17]\n" - ".inst 0x658aaa0a // bfcvt z10.h, p2/M, z16.s\n" + "ld1w { z6.s }, p0/Z, [x17]\n" + ".inst 0x658aa8da // bfcvt z26.h, p2/M, z6.s\n" "add x22, x17, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0x648aaa0a // bfcvtnt z10.h, p2/M, z16.s\n" + "ld1w { z29.s }, p0/Z, [x22]\n" + ".inst 0x648aabba // bfcvtnt z26.h, p2/M, z29.s\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" + "ld1w { z28.s }, p0/Z, [x22]\n" + ".inst 0x658aab9b // bfcvt z27.h, p2/M, z28.s\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0x648aaa0b // bfcvtnt z11.h, p2/M, z16.s\n" + "ld1w { z14.s }, p0/Z, [x22]\n" + ".inst 0x648aa9db // bfcvtnt z27.h, p2/M, z14.s\n" "mov x12, #0x4\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" + "ld1w { z24.s }, p0/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + ".inst 0x658aab1c // bfcvt z28.h, p2/M, z24.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" + "ld1w { z1.s }, p0/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + ".inst 0x648aa83c // bfcvtnt z28.h, p2/M, z1.s\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" + "ld1w { z3.s }, p0/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0x658aa87d // bfcvt z29.h, p2/M, z3.s\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x8\n" - "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + "ld1w { z0.s }, p0/Z, [x22]\n" + ".inst 0x648aa81d // bfcvtnt z29.h, p2/M, z0.s\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" + "ld1w { z24.s }, p0/Z, [x22]\n" + ".inst 0x658aab1e // bfcvt z30.h, p2/M, z24.s\n" "addvl x21, SP, #6\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0xa1402aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1311150 // bfdot za.s[x8, 0], { z10.h-z13.h }, z1.h\n" + ".inst 0xc1311350 // bfdot za.s[x8, 0], { z26.h-z29.h }, z1.h\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" + "ld1w { z23.s }, p0/Z, [x22]\n" + ".inst 0x648aaafe // bfcvtnt z30.h, p2/M, z23.s\n" "addvl x20, SP, #12\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc1391170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z9.h\n" - ".inst 0xa1402a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0xc1311151 // bfdot za.s[x8, 1], { z10.h-z13.h }, z1.h\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" + ".inst 0xc1391370 // bfdot za.s[x8, 0], { z27.h-z30.h }, z9.h\n" + ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" + "ld1w { z31.s }, p0/Z, [x22]\n" + ".inst 0xc1301351 // bfdot za.s[x8, 1], { z26.h-z29.h }, z0.h\n" + ".inst 0x658aabff // bfcvt z31.h, p2/M, z31.s\n" "add x17, x17, %x[ld_in_col], LSL #2\n" - "ld1h { z2.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc1391171 // bfdot za.s[x8, 1], { z11.h-z14.h }, z9.h\n" - ".inst 0xc1321190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z2.h\n" - "ld1h { z2.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc1321191 // bfdot za.s[x8, 1], { z12.h-z15.h }, z2.h\n" + "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" + ".inst 0xc1311371 // bfdot za.s[x8, 1], { z27.h-z30.h }, z1.h\n" + ".inst 0xc1301390 // bfdot za.s[x8, 0], { z28.h-z31.h }, z0.h\n" + "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" + ".inst 0xc1301391 // bfdot za.s[x8, 1], { z28.h-z31.h }, z0.h\n" "16:" // Padded: 1 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x17]\n" - ".inst 0x658aaa0a // bfcvt z10.h, p2/M, z16.s\n" + "ld1w { z22.s }, p0/Z, [x17]\n" + ".inst 0x658aaad5 // bfcvt z21.h, p2/M, z22.s\n" "add x22, x17, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0x648aaa0a // bfcvtnt z10.h, p2/M, z16.s\n" + "ld1w { z3.s }, p0/Z, [x22]\n" + ".inst 0x648aa875 // bfcvtnt z21.h, p2/M, z3.s\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" + "ld1w { z20.s }, p0/Z, [x22]\n" + ".inst 0x658aaa96 // bfcvt z22.h, p2/M, z20.s\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0x648aaa0b // bfcvtnt z11.h, p2/M, z16.s\n" + "ld1w { z25.s }, p0/Z, [x22]\n" + ".inst 0x648aab36 // bfcvtnt z22.h, p2/M, z25.s\n" "mov x12, #0x4\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" + "ld1w { z24.s }, p0/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + ".inst 0x658aab17 // bfcvt z23.h, p2/M, z24.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" + "ld1w { z0.s }, p0/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + ".inst 0x648aa817 // bfcvtnt z23.h, p2/M, z0.s\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" + "ld1w { z7.s }, p0/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0x658aa8f8 // bfcvt z24.h, p2/M, z7.s\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x8\n" - "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + "ld1w { z28.s }, p0/Z, [x22]\n" + ".inst 0x648aab98 // bfcvtnt z24.h, p2/M, z28.s\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" + "ld1w { z6.s }, p0/Z, [x22]\n" + ".inst 0x658aa8d9 // bfcvt z25.h, p2/M, z6.s\n" "addvl x21, SP, #3\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0xa1402aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1311150 // bfdot za.s[x8, 0], { z10.h-z13.h }, z1.h\n" + ".inst 0xc13112b0 // bfdot za.s[x8, 0], { z21.h-z24.h }, z1.h\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" + "ld1w { z6.s }, p0/Z, [x22]\n" + ".inst 0x648aa8d9 // bfcvtnt z25.h, p2/M, z6.s\n" "addvl x20, SP, #9\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc1391170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z9.h\n" - ".inst 0xa1402a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0xc1311151 // bfdot za.s[x8, 1], { z10.h-z13.h }, z1.h\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" + ".inst 0xc13912d0 // bfdot za.s[x8, 0], { z22.h-z25.h }, z9.h\n" + ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" + "ld1w { z3.s }, p0/Z, [x22]\n" + ".inst 0xc13012b1 // bfdot za.s[x8, 1], { z21.h-z24.h }, z0.h\n" + ".inst 0x658aa87a // bfcvt z26.h, p2/M, z3.s\n" "add x17, x17, %x[ld_in_col], LSL #2\n" - "ld1h { z2.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc1391171 // bfdot za.s[x8, 1], { z11.h-z14.h }, z9.h\n" - ".inst 0xc1321190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z2.h\n" - "ld1h { z2.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc1321191 // bfdot za.s[x8, 1], { z12.h-z15.h }, z2.h\n" + "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" + ".inst 0xc13112d1 // bfdot za.s[x8, 1], { z22.h-z25.h }, z1.h\n" + ".inst 0xc13012f0 // bfdot za.s[x8, 0], { z23.h-z26.h }, z0.h\n" + "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" + ".inst 0xc13012f1 // bfdot za.s[x8, 1], { z23.h-z26.h }, z0.h\n" "17:" // Padded: 0 priming loads "cmp x7, #0x2\n" - ".inst 0xa1402be1 // ld1h { z1.h, z9.h }, pn10.b/Z, [SP]\n" - "ld1h { z2.h }, p2/Z, [SP, #2, MUL VL]\n" + ".inst 0xa1402be3 // ld1h { z3.h, z11.h }, pn10.b/Z, [SP]\n" + "ld1h { z7.h }, p2/Z, [SP, #2, MUL VL]\n" "blt 20f\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x17]\n" - ".inst 0x658aaa0a // bfcvt z10.h, p2/M, z16.s\n" + "ld1w { z25.s }, p0/Z, [x17]\n" + ".inst 0x658aab35 // bfcvt z21.h, p2/M, z25.s\n" "add x20, x17, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x648aaa0a // bfcvtnt z10.h, p2/M, z16.s\n" + "ld1w { z27.s }, p0/Z, [x20]\n" + ".inst 0x648aab75 // bfcvtnt z21.h, p2/M, z27.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" + "ld1w { z27.s }, p0/Z, [x20]\n" + ".inst 0x658aab76 // bfcvt z22.h, p2/M, z27.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x648aaa0b // bfcvtnt z11.h, p2/M, z16.s\n" + "ld1w { z27.s }, p0/Z, [x20]\n" + ".inst 0x648aab76 // bfcvtnt z22.h, p2/M, z27.s\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z27.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + ".inst 0x658aab77 // bfcvt z23.h, p2/M, z27.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z25.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + ".inst 0x648aab37 // bfcvtnt z23.h, p2/M, z25.s\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z26.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0x658aab58 // bfcvt z24.h, p2/M, z26.s\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x8\n" - "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + "ld1w { z27.s }, p0/Z, [x20]\n" + ".inst 0x648aab78 // bfcvtnt z24.h, p2/M, z27.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" + "ld1w { z27.s }, p0/Z, [x20]\n" + ".inst 0x658aab79 // bfcvt z25.h, p2/M, z27.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" + "ld1w { z26.s }, p0/Z, [x20]\n" + ".inst 0x648aab59 // bfcvtnt z25.h, p2/M, z26.s\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" + "ld1w { z27.s }, p0/Z, [x20]\n" + ".inst 0x658aab7a // bfcvt z26.h, p2/M, z27.s\n" "sub x7, x7, #0x2\n" "sub x16, x16, #0x1\n" "lsr x20, x7, #0x1\n" @@ -889,323 +889,323 @@ void sme2_fp32bf16fp32_planar_5x5_s2_4rows_dot_za_impl( "sub x16, x16, x24\n" "cbz x24, 19f\n" "18:" // Padded: Main loop - ".inst 0xc1311150 // bfdot za.s[x8, 0], { z10.h-z13.h }, z1.h\n" + ".inst 0xc13312b0 // bfdot za.s[x8, 0], { z21.h-z24.h }, z3.h\n" "addvl x23, SP, #6\n" "addvl x21, SP, #12\n" - ".inst 0xc1391170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z9.h\n" - ".inst 0xa1402ae1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x23]\n" + ".inst 0xc13b12d0 // bfdot za.s[x8, 0], { z22.h-z25.h }, z11.h\n" + ".inst 0xa0402ae0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x23]\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1311151 // bfdot za.s[x8, 1], { z10.h-z13.h }, z1.h\n" - "ld1w { z16.s }, p0/Z, [x17]\n" + ".inst 0xc13012b1 // bfdot za.s[x8, 1], { z21.h-z24.h }, z0.h\n" + "ld1w { z9.s }, p0/Z, [x17]\n" "add x20, x17, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0xc1391171 // bfdot za.s[x8, 1], { z11.h-z14.h }, z9.h\n" - ".inst 0xa1402aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21]\n" + ".inst 0xc13112d1 // bfdot za.s[x8, 1], { z22.h-z25.h }, z1.h\n" + ".inst 0xa0402aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21]\n" "addvl x22, SP, #3\n" "add x17, x17, %x[ld_in_col], LSL #2\n" - ".inst 0xc1311152 // bfdot za.s[x8, 2], { z10.h-z13.h }, z1.h\n" - "ld1w { z20.s }, p0/Z, [x20]\n" + ".inst 0xc13012b2 // bfdot za.s[x8, 2], { z21.h-z24.h }, z0.h\n" + "ld1w { z14.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z17.s }, p0/Z, [x20]\n" + "ld1w { z27.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xc1321190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z2.h\n" + ".inst 0xc13712f0 // bfdot za.s[x8, 0], { z23.h-z26.h }, z7.h\n" "mov x12, #0x4\n" - "ld1h { z2.h }, p2/Z, [x23, #2, MUL VL]\n" - ".inst 0xc1391172 // bfdot za.s[x8, 2], { z11.h-z14.h }, z9.h\n" - ".inst 0x658aaa0a // bfcvt z10.h, p2/M, z16.s\n" - "ld1w { z19.s }, p0/Z, [x20]\n" + "ld1h { z0.h }, p2/Z, [x23, #2, MUL VL]\n" + ".inst 0xc13112d2 // bfdot za.s[x8, 2], { z22.h-z25.h }, z1.h\n" + ".inst 0x658aa921 // bfcvt z1.h, p2/M, z9.s\n" + "ld1w { z15.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1321191 // bfdot za.s[x8, 1], { z12.h-z15.h }, z2.h\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + ".inst 0xc13012f1 // bfdot za.s[x8, 1], { z23.h-z26.h }, z0.h\n" + "ld1w { z9.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0x658aaa2b // bfcvt z11.h, p2/M, z17.s\n" - "ld1w { z18.s }, p0/Z, [x20]\n" + ".inst 0x658aab62 // bfcvt z2.h, p2/M, z27.s\n" + "ld1w { z27.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0x648aaa8a // bfcvtnt z10.h, p2/M, z20.s\n" - "ld1h { z2.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc1321192 // bfdot za.s[x8, 2], { z12.h-z15.h }, z2.h\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + ".inst 0x648aa9c1 // bfcvtnt z1.h, p2/M, z14.s\n" + "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" + ".inst 0xc13012f2 // bfdot za.s[x8, 2], { z23.h-z26.h }, z0.h\n" + ".inst 0x658aa923 // bfcvt z3.h, p2/M, z9.s\n" "addvl x21, SP, #9\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z9.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0x658aa924 // bfcvt z4.h, p2/M, z9.s\n" "mov x12, #0x8\n" - "ld1w { z17.s }, p0/Z, [x20]\n" + "ld1w { z24.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa6b // bfcvtnt z11.h, p2/M, z19.s\n" + ".inst 0x648aa9e2 // bfcvtnt z2.h, p2/M, z15.s\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z9.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa4c // bfcvtnt z12.h, p2/M, z18.s\n" + ".inst 0x648aab63 // bfcvtnt z3.h, p2/M, z27.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0x648aaa2d // bfcvtnt z13.h, p2/M, z17.s\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" - ".inst 0xa1402ac1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x22]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + ".inst 0x648aab04 // bfcvtnt z4.h, p2/M, z24.s\n" + ".inst 0x658aa925 // bfcvt z5.h, p2/M, z9.s\n" + ".inst 0xa1402ac0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x22]\n" + "ld1w { z30.s }, p0/Z, [x20]\n" ".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" - ".inst 0xc1311150 // bfdot za.s[x8, 0], { z10.h-z13.h }, z1.h\n" + ".inst 0x648aabc5 // bfcvtnt z5.h, p2/M, z30.s\n" + ".inst 0xc1301030 // bfdot za.s[x8, 0], { z1.h-z4.h }, z0.h\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z17.s }, p0/Z, [x20]\n" + "ld1w { z29.s }, p0/Z, [x20]\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1391170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z9.h\n" - ".inst 0xa1402aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21]\n" - "ld1w { z16.s }, p0/Z, [x17]\n" + ".inst 0xc1381050 // bfdot za.s[x8, 0], { z2.h-z5.h }, z8.h\n" + ".inst 0xa1402aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21]\n" + "ld1w { z0.s }, p0/Z, [x17]\n" "add x20, x17, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0xc1311151 // bfdot za.s[x8, 1], { z10.h-z13.h }, z1.h\n" - "ld1w { z21.s }, p0/Z, [x20]\n" + ".inst 0xc1361031 // bfdot za.s[x8, 1], { z1.h-z4.h }, z6.h\n" + "ld1w { z10.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0x658aaa2f // bfcvt z15.h, p2/M, z17.s\n" - "ld1w { z17.s }, p0/Z, [x20]\n" + ".inst 0x658aaba6 // bfcvt z6.h, p2/M, z29.s\n" + "ld1w { z9.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xc1391171 // bfdot za.s[x8, 1], { z11.h-z14.h }, z9.h\n" + ".inst 0xc13e1051 // bfdot za.s[x8, 1], { z2.h-z5.h }, z14.h\n" "mov x12, #0x4\n" - "ld1w { z20.s }, p0/Z, [x20]\n" + "ld1w { z29.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0a // bfcvt z10.h, p2/M, z16.s\n" + ".inst 0x658aa815 // bfcvt z21.h, p2/M, z0.s\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1w { z31.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa2b // bfcvt z11.h, p2/M, z17.s\n" + ".inst 0x658aa936 // bfcvt z22.h, p2/M, z9.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1h { z2.h }, p2/Z, [x22, #2, MUL VL]\n" - ".inst 0xc1321190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z2.h\n" + "ld1h { z0.h }, p2/Z, [x22, #2, MUL VL]\n" + ".inst 0xc1301070 // bfdot za.s[x8, 0], { z3.h-z6.h }, z0.h\n" "subs x24, x24, #0x1\n" - "ld1w { z19.s }, p0/Z, [x20]\n" + "ld1w { z15.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc1b6cbd8 // fclamp { z24.s-z27.s }, z30.s, z22.s\n" - "ld1w { z17.s }, p0/Z, [x20]\n" + ".inst 0xc1acc9b8 // fclamp { z24.s-z27.s }, z13.s, z12.s\n" + "ld1w { z30.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "st1w { z24.s }, p1, [x15]\n" "mov x12, #0x8\n" - "ld1w { z18.s }, p0/Z, [x20]\n" + "ld1w { z14.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "st1w { z25.s }, p1, [x14]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1h { z2.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc1321191 // bfdot za.s[x8, 1], { z12.h-z15.h }, z2.h\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" + ".inst 0xc1301071 // bfdot za.s[x8, 1], { z3.h-z6.h }, z0.h\n" + ".inst 0x658aabf7 // bfcvt z23.h, p2/M, z31.s\n" + "ld1w { z8.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0x658aaa2d // bfcvt z13.h, p2/M, z17.s\n" - "ld1w { z17.s }, p0/Z, [x20]\n" + ".inst 0x658aabd8 // bfcvt z24.h, p2/M, z30.s\n" + "ld1w { z4.s }, p0/Z, [x20]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" - "ld1w { z16.s }, p0/Z, [x20]\n" + ".inst 0x658aa919 // bfcvt z25.h, p2/M, z8.s\n" + "ld1w { z5.s }, p0/Z, [x20]\n" "add x15, x15, x13, LSL #2\n" "add x14, x14, x11, LSL #2\n" "st1w { z26.s }, p1, [x10]\n" "add x10, x10, x28, LSL #2\n" "st1w { z27.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - ".inst 0xc0040c84 // mova za.d[x8, #4], { z4.d-z7.d }\n" - ".inst 0xa1402be1 // ld1h { z1.h, z9.h }, pn10.b/Z, [SP]\n" - ".inst 0x648aaaaa // bfcvtnt z10.h, p2/M, z21.s\n" - ".inst 0x648aaa8b // bfcvtnt z11.h, p2/M, z20.s\n" + ".inst 0xc0040e04 // mova za.d[x8, #4], { z16.d-z19.d }\n" + ".inst 0xa1402be3 // ld1h { z3.h, z11.h }, pn10.b/Z, [SP]\n" + ".inst 0x648aa955 // bfcvtnt z21.h, p2/M, z10.s\n" + ".inst 0x648aabb6 // bfcvtnt z22.h, p2/M, z29.s\n" "add x17, x17, %x[ld_in_col], LSL #2\n" - "ld1h { z2.h }, p2/Z, [SP, #2, MUL VL]\n" - ".inst 0x648aaa6c // bfcvtnt z12.h, p2/M, z19.s\n" - ".inst 0x648aaa4d // bfcvtnt z13.h, p2/M, z18.s\n" - ".inst 0x648aaa2e // bfcvtnt z14.h, p2/M, z17.s\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" + "ld1h { z7.h }, p2/Z, [SP, #2, MUL VL]\n" + ".inst 0x648aa9f7 // bfcvtnt z23.h, p2/M, z15.s\n" + ".inst 0x648aa9d8 // bfcvtnt z24.h, p2/M, z14.s\n" + ".inst 0x648aa899 // bfcvtnt z25.h, p2/M, z4.s\n" + ".inst 0x658aa8ba // bfcvt z26.h, p2/M, z5.s\n" "bgt 18b\n" "19:" // Main loop tail - ".inst 0xc1311150 // bfdot za.s[x8, 0], { z10.h-z13.h }, z1.h\n" + ".inst 0xc13312b0 // bfdot za.s[x8, 0], { z21.h-z24.h }, z3.h\n" "addvl x24, SP, #6\n" "addvl x23, SP, #12\n" - ".inst 0xc1391170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z9.h\n" - ".inst 0xa1402b01 // ld1h { z1.h, z9.h }, pn10.b/Z, [x24]\n" + ".inst 0xc13b12d0 // bfdot za.s[x8, 0], { z22.h-z25.h }, z11.h\n" + ".inst 0xa0402b00 // ld1h { z0.h-z1.h }, pn10.b/Z, [x24]\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1311151 // bfdot za.s[x8, 1], { z10.h-z13.h }, z1.h\n" - "ld1w { z16.s }, p0/Z, [x17]\n" + ".inst 0xc13012b1 // bfdot za.s[x8, 1], { z21.h-z24.h }, z0.h\n" + "ld1w { z5.s }, p0/Z, [x17]\n" "add x22, x17, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0xc1391171 // bfdot za.s[x8, 1], { z11.h-z14.h }, z9.h\n" - ".inst 0xa1402ae1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x23]\n" + ".inst 0xc13112d1 // bfdot za.s[x8, 1], { z22.h-z25.h }, z1.h\n" + ".inst 0xa0402ae0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x23]\n" "addvl x21, SP, #3\n" "addvl x20, SP, #9\n" - ".inst 0xc1311152 // bfdot za.s[x8, 2], { z10.h-z13.h }, z1.h\n" - "ld1w { z20.s }, p0/Z, [x22]\n" + ".inst 0xc13012b2 // bfdot za.s[x8, 2], { z21.h-z24.h }, z0.h\n" + "ld1w { z29.s }, p0/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z17.s }, p0/Z, [x22]\n" + "ld1w { z2.s }, p0/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xc1321190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z2.h\n" + ".inst 0xc13712f0 // bfdot za.s[x8, 0], { z23.h-z26.h }, z7.h\n" "mov x12, #0x4\n" - "ld1h { z2.h }, p2/Z, [x24, #2, MUL VL]\n" - ".inst 0xc1391172 // bfdot za.s[x8, 2], { z11.h-z14.h }, z9.h\n" - ".inst 0x658aaa0a // bfcvt z10.h, p2/M, z16.s\n" - "ld1w { z19.s }, p0/Z, [x22]\n" + "ld1h { z0.h }, p2/Z, [x24, #2, MUL VL]\n" + ".inst 0xc13112d2 // bfdot za.s[x8, 2], { z22.h-z25.h }, z1.h\n" + ".inst 0x658aa8bb // bfcvt z27.h, p2/M, z5.s\n" + "ld1w { z20.s }, p0/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1321191 // bfdot za.s[x8, 1], { z12.h-z15.h }, z2.h\n" - "ld1w { z16.s }, p0/Z, [x22]\n" + ".inst 0xc13012f1 // bfdot za.s[x8, 1], { z23.h-z26.h }, z0.h\n" + "ld1w { z1.s }, p0/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0x658aaa2b // bfcvt z11.h, p2/M, z17.s\n" - "ld1w { z18.s }, p0/Z, [x22]\n" + ".inst 0x658aa85c // bfcvt z28.h, p2/M, z2.s\n" + "ld1w { z14.s }, p0/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0x648aaa8a // bfcvtnt z10.h, p2/M, z20.s\n" - "ld1h { z2.h }, p2/Z, [x23, #2, MUL VL]\n" - ".inst 0xc1321192 // bfdot za.s[x8, 2], { z12.h-z15.h }, z2.h\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + ".inst 0x648aabbb // bfcvtnt z27.h, p2/M, z29.s\n" + "ld1h { z0.h }, p2/Z, [x23, #2, MUL VL]\n" + ".inst 0xc13012f2 // bfdot za.s[x8, 2], { z23.h-z26.h }, z0.h\n" + ".inst 0x658aa83d // bfcvt z29.h, p2/M, z1.s\n" "add x17, x17, %x[ld_in_col], LSL #2\n" - "ld1w { z16.s }, p0/Z, [x22]\n" + "ld1w { z1.s }, p0/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0x658aa83e // bfcvt z30.h, p2/M, z1.s\n" "mov x12, #0x8\n" - "ld1w { z17.s }, p0/Z, [x22]\n" + "ld1w { z31.s }, p0/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa6b // bfcvtnt z11.h, p2/M, z19.s\n" + ".inst 0x648aaa9c // bfcvtnt z28.h, p2/M, z20.s\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" + "ld1w { z26.s }, p0/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa4c // bfcvtnt z12.h, p2/M, z18.s\n" + ".inst 0x648aa9dd // bfcvtnt z29.h, p2/M, z14.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0x648aaa2d // bfcvtnt z13.h, p2/M, z17.s\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" - ".inst 0xa1402aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n" + ".inst 0x648aabfe // bfcvtnt z30.h, p2/M, z31.s\n" + ".inst 0x658aab5f // bfcvt z31.h, p2/M, z26.s\n" + ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" + "ld1w { z9.s }, p0/Z, [x22]\n" + ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" - ".inst 0xc1311150 // bfdot za.s[x8, 0], { z10.h-z13.h }, z1.h\n" + ".inst 0x648aa93f // bfcvtnt z31.h, p2/M, z9.s\n" + ".inst 0xc1321370 // bfdot za.s[x8, 0], { z27.h-z30.h }, z2.h\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0xc1391170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z9.h\n" - ".inst 0xa1402a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20]\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" - ".inst 0xc1311151 // bfdot za.s[x8, 1], { z10.h-z13.h }, z1.h\n" - "ld1h { z2.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc1b6cbd8 // fclamp { z24.s-z27.s }, z30.s, z22.s\n" - ".inst 0xc1391171 // bfdot za.s[x8, 1], { z11.h-z14.h }, z9.h\n" - "st1w { z24.s }, p1, [x15]\n" + "ld1w { z26.s }, p0/Z, [x22]\n" + ".inst 0xc13a1390 // bfdot za.s[x8, 0], { z28.h-z31.h }, z10.h\n" + ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" + ".inst 0x658aab40 // bfcvt z0.h, p2/M, z26.s\n" + ".inst 0xc1321371 // bfdot za.s[x8, 1], { z27.h-z30.h }, z2.h\n" + "ld1h { z9.h }, p2/Z, [x21, #2, MUL VL]\n" + ".inst 0xc1acc9a4 // fclamp { z4.s-z7.s }, z13.s, z12.s\n" + ".inst 0xc13a1391 // bfdot za.s[x8, 1], { z28.h-z31.h }, z10.h\n" + "st1w { z4.s }, p1, [x15]\n" "add x15, x15, x13, LSL #2\n" - ".inst 0xa1402be1 // ld1h { z1.h, z9.h }, pn10.b/Z, [SP]\n" - ".inst 0xc1321190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z2.h\n" - "ld1h { z2.h }, p2/Z, [x20, #2, MUL VL]\n" - "st1w { z25.s }, p1, [x14]\n" + ".inst 0xa1402be3 // ld1h { z3.h, z11.h }, pn10.b/Z, [SP]\n" + ".inst 0xc13913b0 // bfdot za.s[x8, 0], { z29.h-z0.h }, z9.h\n" + "ld1h { z9.h }, p2/Z, [x20, #2, MUL VL]\n" + "st1w { z5.s }, p1, [x14]\n" "add x14, x14, x11, LSL #2\n" - "st1w { z26.s }, p1, [x10]\n" + "st1w { z6.s }, p1, [x10]\n" "add x10, x10, x28, LSL #2\n" - ".inst 0xc0040c84 // mova za.d[x8, #4], { z4.d-z7.d }\n" - "st1w { z27.s }, p1, [x9]\n" + ".inst 0xc0040e04 // mova za.d[x8, #4], { z16.d-z19.d }\n" + "st1w { z7.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" - ".inst 0xc1321191 // bfdot za.s[x8, 1], { z12.h-z15.h }, z2.h\n" - "ld1h { z2.h }, p2/Z, [SP, #2, MUL VL]\n" + ".inst 0xc13913b1 // bfdot za.s[x8, 1], { z29.h-z0.h }, z9.h\n" + "ld1h { z7.h }, p2/Z, [SP, #2, MUL VL]\n" "20:" // Main loop skip tail "cbz x7, 21f\n" // Skip remainder inputs "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x17]\n" - ".inst 0x658aaa0a // bfcvt z10.h, p2/M, z16.s\n" + "ld1w { z25.s }, p0/Z, [x17]\n" + ".inst 0x658aab3d // bfcvt z29.h, p2/M, z25.s\n" "add x22, x17, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0x648aaa0a // bfcvtnt z10.h, p2/M, z16.s\n" + "ld1w { z26.s }, p0/Z, [x22]\n" + ".inst 0x648aab5d // bfcvtnt z29.h, p2/M, z26.s\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0x658aaa0b // bfcvt z11.h, p2/M, z16.s\n" + "ld1w { z25.s }, p0/Z, [x22]\n" + ".inst 0x658aab3e // bfcvt z30.h, p2/M, z25.s\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0x648aaa0b // bfcvtnt z11.h, p2/M, z16.s\n" + "ld1w { z24.s }, p0/Z, [x22]\n" + ".inst 0x648aab1e // bfcvtnt z30.h, p2/M, z24.s\n" "mov x12, #0x4\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" + "ld1w { z26.s }, p0/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0c // bfcvt z12.h, p2/M, z16.s\n" + ".inst 0x658aab5f // bfcvt z31.h, p2/M, z26.s\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" + "ld1w { z9.s }, p0/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x648aaa0c // bfcvtnt z12.h, p2/M, z16.s\n" + ".inst 0x648aa93f // bfcvtnt z31.h, p2/M, z9.s\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" + "ld1w { z9.s }, p0/Z, [x22]\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0x658aaa0d // bfcvt z13.h, p2/M, z16.s\n" + ".inst 0x658aa920 // bfcvt z0.h, p2/M, z9.s\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x8\n" - "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0x648aaa0d // bfcvtnt z13.h, p2/M, z16.s\n" + "ld1w { z24.s }, p0/Z, [x22]\n" + ".inst 0x648aab00 // bfcvtnt z0.h, p2/M, z24.s\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0x658aaa0e // bfcvt z14.h, p2/M, z16.s\n" + "ld1w { z9.s }, p0/Z, [x22]\n" + ".inst 0x658aa921 // bfcvt z1.h, p2/M, z9.s\n" "add x22, x22, %x[ld_in_row], LSL #2\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0x648aaa0e // bfcvtnt z14.h, p2/M, z16.s\n" - ".inst 0xc1311150 // bfdot za.s[x8, 0], { z10.h-z13.h }, z1.h\n" + "ld1w { z25.s }, p0/Z, [x22]\n" + ".inst 0x648aab21 // bfcvtnt z1.h, p2/M, z25.s\n" + ".inst 0xc13313b0 // bfdot za.s[x8, 0], { z29.h-z0.h }, z3.h\n" "addvl x21, SP, #6\n" "add x22, x22, %x[ld_in_row], LSL #2\n" - ".inst 0xc1391170 // bfdot za.s[x8, 0], { z11.h-z14.h }, z9.h\n" - ".inst 0xa1402aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21]\n" + ".inst 0xc13b13d0 // bfdot za.s[x8, 0], { z30.h-z1.h }, z11.h\n" + ".inst 0xa0402aae // ld1h { z14.h-z15.h }, pn10.b/Z, [x21]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "addvl x20, SP, #12\n" - ".inst 0xc1311151 // bfdot za.s[x8, 1], { z10.h-z13.h }, z1.h\n" - "ld1w { z16.s }, p0/Z, [x22]\n" - ".inst 0x658aaa0f // bfcvt z15.h, p2/M, z16.s\n" + ".inst 0xc13e13b1 // bfdot za.s[x8, 1], { z29.h-z0.h }, z14.h\n" + "ld1w { z25.s }, p0/Z, [x22]\n" + ".inst 0x658aab22 // bfcvt z2.h, p2/M, z25.s\n" "sub x16, x16, #0x1\n" - ".inst 0xc1391171 // bfdot za.s[x8, 1], { z11.h-z14.h }, z9.h\n" - ".inst 0xa1402a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1311152 // bfdot za.s[x8, 2], { z10.h-z13.h }, z1.h\n" - ".inst 0xc1321190 // bfdot za.s[x8, 0], { z12.h-z15.h }, z2.h\n" - "ld1h { z2.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc1391172 // bfdot za.s[x8, 2], { z11.h-z14.h }, z9.h\n" - ".inst 0xc1321191 // bfdot za.s[x8, 1], { z12.h-z15.h }, z2.h\n" - "ld1h { z2.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n" - ".inst 0xc1b6cbd8 // fclamp { z24.s-z27.s }, z30.s, z22.s\n" - "st1w { z24.s }, p1, [x15]\n" + ".inst 0xc13f13d1 // bfdot za.s[x8, 1], { z30.h-z1.h }, z15.h\n" + ".inst 0xa0402a8e // ld1h { z14.h-z15.h }, pn10.b/Z, [x20]\n" + ".inst 0xc13e13b2 // bfdot za.s[x8, 2], { z29.h-z0.h }, z14.h\n" + ".inst 0xc13713f0 // bfdot za.s[x8, 0], { z31.h-z2.h }, z7.h\n" + "ld1h { z4.h }, p2/Z, [x21, #2, MUL VL]\n" + ".inst 0xc13f13d2 // bfdot za.s[x8, 2], { z30.h-z1.h }, z15.h\n" + ".inst 0xc13413f1 // bfdot za.s[x8, 1], { z31.h-z2.h }, z4.h\n" + "ld1h { z9.h }, p2/Z, [x20, #2, MUL VL]\n" + ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" + ".inst 0xc1acc9a4 // fclamp { z4.s-z7.s }, z13.s, z12.s\n" + "st1w { z4.s }, p1, [x15]\n" "add x15, x15, x13, LSL #2\n" - ".inst 0xc1321192 // bfdot za.s[x8, 2], { z12.h-z15.h }, z2.h\n" + ".inst 0xc13913f2 // bfdot za.s[x8, 2], { z31.h-z2.h }, z9.h\n" "add x8, x8, #0x1\n" - "st1w { z25.s }, p1, [x14]\n" + "st1w { z5.s }, p1, [x14]\n" "add x14, x14, x11, LSL #2\n" - "st1w { z26.s }, p1, [x10]\n" + "st1w { z6.s }, p1, [x10]\n" "add x10, x10, x28, LSL #2\n" - ".inst 0xc0040c84 // mova za.d[x8, #4], { z4.d-z7.d }\n" - "st1w { z27.s }, p1, [x9]\n" + ".inst 0xc0040e04 // mova za.d[x8, #4], { z16.d-z19.d }\n" + "st1w { z7.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" "21:" // Tail input: End "cbz x16, 23f\n" "22:" // Right padding loop - ".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n" + ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" "subs x16, x16, #0x1\n" - ".inst 0xc1b6cbd8 // fclamp { z24.s-z27.s }, z30.s, z22.s\n" - "st1w { z24.s }, p1, [x15]\n" + ".inst 0xc1acc9a4 // fclamp { z4.s-z7.s }, z13.s, z12.s\n" + "st1w { z4.s }, p1, [x15]\n" "add x15, x15, x13, LSL #2\n" - ".inst 0xc0040c84 // mova za.d[x8, #4], { z4.d-z7.d }\n" - "st1w { z25.s }, p1, [x14]\n" + ".inst 0xc0040e04 // mova za.d[x8, #4], { z16.d-z19.d }\n" + "st1w { z5.s }, p1, [x14]\n" "add x14, x14, x11, LSL #2\n" - "st1w { z26.s }, p1, [x10]\n" + "st1w { z6.s }, p1, [x10]\n" "add x10, x10, x28, LSL #2\n" - "st1w { z27.s }, p1, [x9]\n" + "st1w { z7.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" "bgt 22b\n" "23:" // End @@ -1213,12 +1213,12 @@ void sme2_fp32bf16fp32_planar_5x5_s2_4rows_dot_za_impl( "incb x20, ALL, MUL #16\n" "incb x20, ALL, MUL #9\n" "str x20, [%x[args], %[offsetof_Args_weights]]\n" - "ldr x20, [%x[args], %[offsetof_Args_ld_in_vl]]\n" + "ldr x21, [%x[args], %[offsetof_Args_ld_in_vl]]\n" "incw x6\n" "whilelt p1.s, x6, x5\n" - "ldr x17, [%x[args], %[offsetof_Args_inptr]]\n" - "add x17, x17, x20, LSL #2\n" - "str x17, [%x[args], %[offsetof_Args_inptr]]\n" + "ldr x20, [%x[args], %[offsetof_Args_inptr]]\n" + "add x20, x20, x21, LSL #2\n" + "str x20, [%x[args], %[offsetof_Args_inptr]]\n" "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" "ldr x24, [%x[args], %[offsetof_Args_ld_out_vls]]\n" "ldp x23, x22, [x25, #0x0]\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_3x3_s1_4rows_dot_za.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_3x3_s1_4rows_dot_za.hpp index be4f02fc309f1210765dbe2551e0114e20fe9160..de3eadac8a727783b0fd2152246a28b71834b311 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_3x3_s1_4rows_dot_za.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_3x3_s1_4rows_dot_za.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,8 @@ * SOFTWARE. */ +#include "src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp" + namespace arm_conv { namespace depthwise { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_3x3_s1_4rows_dot_za/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_3x3_s1_4rows_dot_za/generic.cpp index 6c42c76683691ff64e1ebf6bd0bfd605f92d0765..845f37692640df91ce3021a9c7a36554105cf6b4 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_3x3_s1_4rows_dot_za/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_3x3_s1_4rows_dot_za/generic.cpp @@ -73,96 +73,96 @@ void sme2_s8q_planar_3x3_s1_4rows_dot_za_impl( "ptrue p2.b\n" "mov x20, #0x6\n" "ldr x7, [%x[args], %[offsetof_Args_pad_top]]\n" - "ld1rh { z24.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_a_offset]]\n" + "ld1rh { z21.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_a_offset]]\n" "sub x20, x20, x6\n" ".inst 0x25207812 // ptrue pn10.b\n" "ldr x17, [%x[args], %[offsetof_Args_n_channels]]\n" "whilelt p1.s, XZR, x17\n" "whilelt p9.s, XZR, x20\n" - "ld1rw { z12.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" + "ld1rw { z15.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" "whilelt p8.s, XZR, x7\n" "addvl SP, SP, #-12\n" "ldr x16, [%x[args], %[offsetof_Args_current_channel]]\n" - "neg z24.h, p2/M, z24.h\n" + "neg z21.h, p2/M, z21.h\n" "eor p8.b, p2/Z, p8.b, p9.b\n" - "ld1rw { z10.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_mul]]\n" - "ld1rw { z11.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_right_shift]]\n" - "ld1rw { z22.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" - "ld1rw { z26.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" + "ld1rw { z14.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z12.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z29.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" + "ld1rw { z28.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" "1:" // Channel loop "ldr x20, [%x[qp], %[offsetof_Requantize32_bias]]\n" - "mov z8.s, #0x0\n" + "mov z30.s, #0x0\n" "cbz x20, 2f\n" - "ld1w { z8.s }, p1/Z, [x20, x16, LSL #2]\n" + "ld1w { z30.s }, p1/Z, [x20, x16, LSL #2]\n" "2:" // Load bias: Done "ldr x22, [%x[args], %[offsetof_Args_weights]]\n" "mov x20, x22\n" - "ld1sb { z27.s }, p2/Z, [x20]\n" + "ld1sb { z10.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "ld1rh { z21.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" - "mov z20.h, #0x0\n" - "sub z27.h, z27.h, z21.h\n" + "ld1rh { z31.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" + "mov z7.h, #0x0\n" + "sub z10.h, z10.h, z31.h\n" "incw x22\n" - "ld1sb { z23.s }, p2/Z, [x20]\n" - "incw x20, ALL, MUL #3\n" - "sub z23.h, z23.h, z21.h\n" - "trn1 z0.h, z20.h, z27.h\n" "ld1sb { z16.s }, p2/Z, [x20]\n" - "sub z16.h, z16.h, z21.h\n" + "incw x20, ALL, MUL #3\n" + "sub z16.h, z16.h, z31.h\n" + "trn1 z20.h, z7.h, z10.h\n" + "ld1sb { z11.s }, p2/Z, [x20]\n" + "sub z11.h, z11.h, z31.h\n" "mov x20, x22\n" - "trn1 z1.h, z27.h, z23.h\n" - "ld1sb { z27.s }, p2/Z, [x20]\n" + "trn1 z19.h, z10.h, z16.h\n" + "ld1sb { z24.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "trn1 z2.h, z23.h, z16.h\n" - "trn1 z3.h, z16.h, z20.h\n" - "ld1sb { z23.s }, p2/Z, [x20]\n" + "trn1 z26.h, z16.h, z11.h\n" + "trn1 z13.h, z11.h, z7.h\n" + "ld1sb { z11.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "sub z27.h, z27.h, z21.h\n" - "sub z23.h, z23.h, z21.h\n" - "ld1sb { z16.s }, p2/Z, [x20]\n" - "sub z16.h, z16.h, z21.h\n" + "sub z24.h, z24.h, z31.h\n" + "sub z11.h, z11.h, z31.h\n" + "ld1sb { z2.s }, p2/Z, [x20]\n" + "sub z2.h, z2.h, z31.h\n" "addvl x21, SP, #12\n" "incw x22\n" "addvl x21, x21, #-4\n" "mov x20, x22\n" - "st1h { z0.h }, p2, [x21]\n" - "trn1 z0.h, z20.h, z27.h\n" - "st1h { z1.h }, p2, [x21, #1, MUL VL]\n" - "trn1 z1.h, z27.h, z23.h\n" - "ld1sb { z27.s }, p2/Z, [x20]\n" + "st1h { z20.h }, p2, [x21]\n" + "trn1 z22.h, z7.h, z24.h\n" + "st1h { z19.h }, p2, [x21, #1, MUL VL]\n" + "trn1 z1.h, z24.h, z11.h\n" + "ld1sb { z16.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "st1h { z2.h }, p2, [x21, #2, MUL VL]\n" - "trn1 z2.h, z23.h, z16.h\n" - "ld1sb { z23.s }, p2/Z, [x20]\n" + "st1h { z26.h }, p2, [x21, #2, MUL VL]\n" + "trn1 z3.h, z11.h, z2.h\n" + "ld1sb { z0.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "st1h { z3.h }, p2, [x21, #3, MUL VL]\n" - "trn1 z3.h, z16.h, z20.h\n" - "ld1sb { z16.s }, p2/Z, [x20]\n" + "st1h { z13.h }, p2, [x21, #3, MUL VL]\n" + "trn1 z25.h, z2.h, z7.h\n" + "ld1sb { z4.s }, p2/Z, [x20]\n" "ldr x20, [%x[qp], %[offsetof_Requantize32_per_channel_muls]]\n" - "sub z27.h, z27.h, z21.h\n" - "sub z23.h, z23.h, z21.h\n" + "sub z16.h, z16.h, z31.h\n" + "sub z0.h, z0.h, z31.h\n" "addvl x21, x21, #-4\n" - "st1h { z0.h }, p2, [x21]\n" - "sub z16.h, z16.h, z21.h\n" + "st1h { z22.h }, p2, [x21]\n" + "sub z4.h, z4.h, z31.h\n" "st1h { z1.h }, p2, [x21, #1, MUL VL]\n" - "mov z9.d, z8.d\n" - "st1h { z2.h }, p2, [x21, #2, MUL VL]\n" - "trn1 z0.h, z20.h, z27.h\n" - "trn1 z1.h, z27.h, z23.h\n" - "st1h { z3.h }, p2, [x21, #3, MUL VL]\n" + "mov z31.d, z30.d\n" + "st1h { z3.h }, p2, [x21, #2, MUL VL]\n" + "trn1 z24.h, z7.h, z16.h\n" + "trn1 z18.h, z16.h, z0.h\n" + "st1h { z25.h }, p2, [x21, #3, MUL VL]\n" "addvl x21, x21, #-4\n" - "trn1 z2.h, z23.h, z16.h\n" - "trn1 z3.h, z16.h, z20.h\n" - "st1h { z0.h }, p2, [x21]\n" - "st1h { z1.h }, p2, [x21, #1, MUL VL]\n" - "st1h { z2.h }, p2, [x21, #2, MUL VL]\n" - "st1h { z3.h }, p2, [x21, #3, MUL VL]\n" + "trn1 z0.h, z0.h, z4.h\n" + "trn1 z1.h, z4.h, z7.h\n" + "st1h { z24.h }, p2, [x21]\n" + "st1h { z18.h }, p2, [x21, #1, MUL VL]\n" + "st1h { z0.h }, p2, [x21, #2, MUL VL]\n" + "st1h { z1.h }, p2, [x21, #3, MUL VL]\n" "cbz x20, 3f\n" - "ld1w { z10.s }, p1/Z, [x20, x16, LSL #2]\n" + "ld1w { z14.s }, p1/Z, [x20, x16, LSL #2]\n" "3:" // Load mul: End "ldr x20, [%x[qp], %[offsetof_Requantize32_per_channel_right_shifts]]\n" "cbz x20, 4f\n" - "ld1w { z11.s }, p1/Z, [x20, x16, LSL #2]\n" + "ld1w { z12.s }, p1/Z, [x20, x16, LSL #2]\n" "4:" // Load right_shift: End "ldr x15, [%x[args], %[offsetof_Args_input_cols]]\n" "sub x20, x15, #0x1\n" @@ -182,21 +182,21 @@ void sme2_s8q_planar_3x3_s1_4rows_dot_za_impl( ".inst 0xf8b74a9c // rprfm pldstrm, x23, [x20]\n" "add x20, x20, %x[ld_in_col]\n" "bgt 5b\n" - "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" + "ldr x23, [%x[args], %[offsetof_Args_outptrs]]\n" "lsl x20, %x[ld_in_row], #0x0\n" "msub x14, x7, x20, x14\n" - ".inst 0xc0040900 // mova za.d[x8, #0], { z8.d-z9.d }\n" + ".inst 0xc0040bc0 // mova za.d[x8, #0], { z30.d-z31.d }\n" "ldr x20, [%x[args], %[offsetof_Args_ld_out_cols]]\n" - ".inst 0xc0040901 // mova za.d[x8, #1], { z8.d-z9.d }\n" + ".inst 0xc0040bc1 // mova za.d[x8, #1], { z30.d-z31.d }\n" "mov x22, #0x2\n" - "ldp x11, x10, [x25], #0x10\n" - ".inst 0xc0040902 // mova za.d[x8, #2], { z8.d-z9.d }\n" + "ldp x11, x10, [x23], #0x10\n" + ".inst 0xc0040bc2 // mova za.d[x8, #2], { z30.d-z31.d }\n" "ldp x9, x28, [x20], #0x10\n" - ".inst 0xc0040903 // mova za.d[x8, #3], { z8.d-z9.d }\n" + ".inst 0xc0040bc3 // mova za.d[x8, #3], { z30.d-z31.d }\n" "ldr x21, [%x[args], %[offsetof_Args_pad_left]]\n" - ".inst 0xc0040904 // mova za.d[x8, #4], { z8.d-z9.d }\n" - "ldp x27, x26, [x25], #0x10\n" - ".inst 0xc0040905 // mova za.d[x8, #5], { z8.d-z9.d }\n" + ".inst 0xc0040bc4 // mova za.d[x8, #4], { z30.d-z31.d }\n" + "ldp x27, x26, [x23], #0x10\n" + ".inst 0xc0040bc5 // mova za.d[x8, #5], { z30.d-z31.d }\n" "ldp x25, x24, [x20], #0x10\n" "cbz x21, 7f\n" "cmp x21, x22\n" @@ -204,22 +204,22 @@ void sme2_s8q_planar_3x3_s1_4rows_dot_za_impl( "sub x21, x21, x20\n" "sub x22, x22, x20\n" "cbz x21, 7f\n" - ".inst 0xc0060804 // mova { z4.d-z5.d }, za.d[x8, #0]\n" + ".inst 0xc0060818 // mova { z24.d-z25.d }, za.d[x8, #0]\n" "sub x13, x13, x21\n" - ".inst 0xc0060826 // mova { z6.d-z7.d }, za.d[x8, #1]\n" - ".inst 0xc1aaac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z10.s\n" - ".inst 0xc1abaa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z11.s\n" - ".inst 0xc1acab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z12.s\n" - ".inst 0xc1bacec4 // sclamp { z4.s-z7.s }, z22.s, z26.s\n" + ".inst 0xc006083a // mova { z26.d-z27.d }, za.d[x8, #1]\n" + ".inst 0xc1aeac18 // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z14.s\n" + ".inst 0xc1acaa38 // srshl { z24.s-z27.s }, { z24.s-z27.s }, z12.s\n" + ".inst 0xc1afab18 // add { z24.s-z27.s }, { z24.s-z27.s }, z15.s\n" + ".inst 0xc1bccfb8 // sclamp { z24.s-z27.s }, z29.s, z28.s\n" "6:" // Left padding "subs x21, x21, #0x1\n" - "st1b { z4.s }, p1, [x11]\n" + "st1b { z24.s }, p1, [x11]\n" "add x11, x11, x9\n" - "st1b { z6.s }, p1, [x10]\n" + "st1b { z26.s }, p1, [x10]\n" "add x10, x10, x28\n" - "st1b { z5.s }, p1, [x27]\n" + "st1b { z25.s }, p1, [x27]\n" "add x27, x27, x25\n" - "st1b { z7.s }, p1, [x26]\n" + "st1b { z27.s }, p1, [x26]\n" "add x26, x26, x24\n" "bgt 6b\n" "7:" // Left padding: End @@ -231,148 +231,148 @@ void sme2_s8q_planar_3x3_s1_4rows_dot_za_impl( "beq 9f\n" "8:" // Unpadded: 2 priming loads "add x21, x14, %x[ld_in_row]\n" - "ld1sb { z17.s }, p1/Z, [x14]\n" + "ld1sb { z20.s }, p1/Z, [x14]\n" "addvl x20, SP, #8\n" "ld1sb { z16.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z17.h, z16.h\n" - "add z13.h, z13.h, z24.h\n" - "ld1sb { z17.s }, p1/Z, [x21]\n" + "trn1 z4.h, z20.h, z16.h\n" + "add z4.h, z4.h, z21.h\n" + "ld1sb { z23.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "add x14, x14, %x[ld_in_col]\n" - "ld1sb { z16.s }, p1/Z, [x21]\n" + "ld1sb { z22.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z14.h, z17.h, z16.h\n" - "add z14.h, z14.h, z24.h\n" + "trn1 z5.h, z23.h, z22.h\n" + "add z5.h, z5.h, z21.h\n" "ld1sb { z17.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "ld1sb { z16.s }, p1/Z, [x21]\n" - "trn1 z15.h, z17.h, z16.h\n" - "add z15.h, z15.h, z24.h\n" - ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" - ".inst 0xc16115a8 // sdot za.s[x8, 0], { z13.h-z14.h }, z1.h\n" - ".inst 0xc16015a9 // sdot za.s[x8, 1], { z13.h-z14.h }, z0.h\n" - ".inst 0xa0412a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16315c8 // sdot za.s[x8, 0], { z14.h-z15.h }, z3.h\n" - ".inst 0xc16215c9 // sdot za.s[x8, 1], { z14.h-z15.h }, z2.h\n" + "trn1 z6.h, z17.h, z16.h\n" + "add z6.h, z6.h, z21.h\n" + ".inst 0xa1402a83 // ld1h { z3.h, z11.h }, pn10.b/Z, [x20]\n" + ".inst 0xc16b1488 // sdot za.s[x8, 0], { z4.h-z5.h }, z11.h\n" + ".inst 0xc1631489 // sdot za.s[x8, 1], { z4.h-z5.h }, z3.h\n" + ".inst 0xa1412a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc16814a8 // sdot za.s[x8, 0], { z5.h-z6.h }, z8.h\n" + ".inst 0xc16014a9 // sdot za.s[x8, 1], { z5.h-z6.h }, z0.h\n" "9:" // Unpadded: 1 priming loads "add x22, x14, %x[ld_in_row]\n" - "ld1sb { z17.s }, p1/Z, [x14]\n" + "ld1sb { z25.s }, p1/Z, [x14]\n" "addvl x21, SP, #4\n" - "ld1sb { z16.s }, p1/Z, [x22]\n" + "ld1sb { z6.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z13.h, z17.h, z16.h\n" - "add z13.h, z13.h, z24.h\n" - "ld1sb { z17.s }, p1/Z, [x22]\n" + "trn1 z3.h, z25.h, z6.h\n" + "add z3.h, z3.h, z21.h\n" + "ld1sb { z18.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "addvl x20, SP, #8\n" - "ld1sb { z16.s }, p1/Z, [x22]\n" + "ld1sb { z26.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z14.h, z17.h, z16.h\n" - "add z14.h, z14.h, z24.h\n" - "ld1sb { z17.s }, p1/Z, [x22]\n" + "trn1 z4.h, z18.h, z26.h\n" + "add z4.h, z4.h, z21.h\n" + "ld1sb { z2.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "add x14, x14, %x[ld_in_col]\n" - "ld1sb { z16.s }, p1/Z, [x22]\n" - "trn1 z15.h, z17.h, z16.h\n" - "add z15.h, z15.h, z24.h\n" + "ld1sb { z5.s }, p1/Z, [x22]\n" + "trn1 z5.h, z2.h, z5.h\n" + "add z5.h, z5.h, z21.h\n" ".inst 0xa0402aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21]\n" - ".inst 0xc16115a8 // sdot za.s[x8, 0], { z13.h-z14.h }, z1.h\n" - ".inst 0xc16015a9 // sdot za.s[x8, 1], { z13.h-z14.h }, z0.h\n" - ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" - ".inst 0xa0412aa2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc16115aa // sdot za.s[x8, 2], { z13.h-z14.h }, z1.h\n" - ".inst 0xc16015ab // sdot za.s[x8, 3], { z13.h-z14.h }, z0.h\n" - ".inst 0xc16315c8 // sdot za.s[x8, 0], { z14.h-z15.h }, z3.h\n" - ".inst 0xc16215c9 // sdot za.s[x8, 1], { z14.h-z15.h }, z2.h\n" - ".inst 0xa0412a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16315ca // sdot za.s[x8, 2], { z14.h-z15.h }, z3.h\n" - ".inst 0xc16215cb // sdot za.s[x8, 3], { z14.h-z15.h }, z2.h\n" + ".inst 0xc1611468 // sdot za.s[x8, 0], { z3.h-z4.h }, z1.h\n" + ".inst 0xc1601469 // sdot za.s[x8, 1], { z3.h-z4.h }, z0.h\n" + ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" + ".inst 0xa0412aa8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc16a146a // sdot za.s[x8, 2], { z3.h-z4.h }, z10.h\n" + ".inst 0xc162146b // sdot za.s[x8, 3], { z3.h-z4.h }, z2.h\n" + ".inst 0xc1691488 // sdot za.s[x8, 0], { z4.h-z5.h }, z9.h\n" + ".inst 0xc1681489 // sdot za.s[x8, 1], { z4.h-z5.h }, z8.h\n" + ".inst 0xa1412a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc16a148a // sdot za.s[x8, 2], { z4.h-z5.h }, z10.h\n" + ".inst 0xc162148b // sdot za.s[x8, 3], { z4.h-z5.h }, z2.h\n" "10:" // Unpadded: 0 priming loads - ".inst 0xa0402be0 // ld1h { z0.h-z1.h }, pn10.b/Z, [SP]\n" - ".inst 0xa0412be2 // ld1h { z2.h-z3.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + ".inst 0xa1402be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP]\n" + ".inst 0xa0412bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" "cbz x15, 18f\n" "add x20, x14, %x[ld_in_row]\n" "ld1sb { z17.s }, p1/Z, [x14]\n" "sub x15, x15, #0x1\n" - "ld1sb { z16.s }, p1/Z, [x20]\n" + "ld1sb { z9.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z13.h, z17.h, z16.h\n" + "trn1 z6.h, z17.h, z9.h\n" "sub x13, x13, #0x1\n" "ld1sb { z17.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" "cmp x15, x13\n" - "add z13.h, z13.h, z24.h\n" - "ld1sb { z16.s }, p1/Z, [x20]\n" + "add z6.h, z6.h, z21.h\n" + "ld1sb { z7.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z14.h, z17.h, z16.h\n" + "trn1 z7.h, z17.h, z7.h\n" "csel x23, x15, x13, LT\n" "ld1sb { z17.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "add z14.h, z14.h, z24.h\n" + "add z7.h, z7.h, z21.h\n" "add x14, x14, %x[ld_in_col]\n" - "ld1sb { z16.s }, p1/Z, [x20]\n" - "trn1 z15.h, z17.h, z16.h\n" - "add z15.h, z15.h, z24.h\n" + "ld1sb { z1.s }, p1/Z, [x20]\n" + "trn1 z8.h, z17.h, z1.h\n" + "add z8.h, z8.h, z21.h\n" "sub x13, x13, x23\n" "cbz x23, 17f\n" "11:" // Unpadded: Main loop - ".inst 0xc16115a8 // sdot za.s[x8, 0], { z13.h-z14.h }, z1.h\n" + ".inst 0xc16d14c8 // sdot za.s[x8, 0], { z6.h-z7.h }, z13.h\n" "addvl x22, SP, #4\n" "addvl x21, SP, #8\n" - "ld1sb { z21.s }, p1/Z, [x14]\n" - ".inst 0xc16015a9 // sdot za.s[x8, 1], { z13.h-z14.h }, z0.h\n" - ".inst 0xa0402ac0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x22]\n" + "ld1sb { z2.s }, p1/Z, [x14]\n" + ".inst 0xc16514c9 // sdot za.s[x8, 1], { z6.h-z7.h }, z5.h\n" + ".inst 0xa1402ac5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x22]\n" "add x20, x14, %x[ld_in_row]\n" "subs x23, x23, #0x1\n" - ".inst 0xc16315c8 // sdot za.s[x8, 0], { z14.h-z15.h }, z3.h\n" - "ld1sb { z20.s }, p1/Z, [x20]\n" + ".inst 0xc16b14e8 // sdot za.s[x8, 0], { z7.h-z8.h }, z11.h\n" + "ld1sb { z19.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" "add x14, x14, %x[ld_in_col]\n" - ".inst 0xc16215c9 // sdot za.s[x8, 1], { z14.h-z15.h }, z2.h\n" - ".inst 0xa0412ac2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xc0060804 // mova { z4.d-z5.d }, za.d[x8, #0]\n" - "ld1sb { z19.s }, p1/Z, [x20]\n" + ".inst 0xc16a14e9 // sdot za.s[x8, 1], { z7.h-z8.h }, z10.h\n" + ".inst 0xa1412ac3 // ld1h { z3.h, z11.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc0060818 // mova { z24.d-z25.d }, za.d[x8, #0]\n" + "ld1sb { z23.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc0060826 // mova { z6.d-z7.d }, za.d[x8, #1]\n" - ".inst 0xc1aaac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z10.s\n" - ".inst 0xc16115aa // sdot za.s[x8, 2], { z13.h-z14.h }, z1.h\n" + ".inst 0xc006083a // mova { z26.d-z27.d }, za.d[x8, #1]\n" + ".inst 0xc1aeac18 // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z14.s\n" + ".inst 0xc16d14ca // sdot za.s[x8, 2], { z6.h-z7.h }, z13.h\n" "ld1sb { z18.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc16015ab // sdot za.s[x8, 3], { z13.h-z14.h }, z0.h\n" - ".inst 0xa0402aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1abaa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z11.s\n" - ".inst 0xc16115ac // sdot za.s[x8, 4], { z13.h-z14.h }, z1.h\n" + ".inst 0xc16514cb // sdot za.s[x8, 3], { z6.h-z7.h }, z5.h\n" + ".inst 0xa1402aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21]\n" + ".inst 0xc1acaa38 // srshl { z24.s-z27.s }, { z24.s-z27.s }, z12.s\n" + ".inst 0xc16914cc // sdot za.s[x8, 4], { z6.h-z7.h }, z9.h\n" "ld1sb { z17.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc16015ad // sdot za.s[x8, 5], { z13.h-z14.h }, z0.h\n" + ".inst 0xc16114cd // sdot za.s[x8, 5], { z6.h-z7.h }, z1.h\n" "ld1sb { z16.s }, p1/Z, [x20]\n" - ".inst 0xc1acab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z12.s\n" - ".inst 0xc16315ca // sdot za.s[x8, 2], { z14.h-z15.h }, z3.h\n" - "trn1 z13.h, z21.h, z20.h\n" - ".inst 0xa0402be0 // ld1h { z0.h-z1.h }, pn10.b/Z, [SP]\n" - ".inst 0xc16215cb // sdot za.s[x8, 3], { z14.h-z15.h }, z2.h\n" - ".inst 0xa0412aa2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc1bacec4 // sclamp { z4.s-z7.s }, z22.s, z26.s\n" - ".inst 0xc16315cc // sdot za.s[x8, 4], { z14.h-z15.h }, z3.h\n" - "st1b { z4.s }, p1, [x11]\n" + ".inst 0xc1afab18 // add { z24.s-z27.s }, { z24.s-z27.s }, z15.s\n" + ".inst 0xc16b14ea // sdot za.s[x8, 2], { z7.h-z8.h }, z11.h\n" + "trn1 z6.h, z2.h, z19.h\n" + ".inst 0xa1402be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP]\n" + ".inst 0xc16314eb // sdot za.s[x8, 3], { z7.h-z8.h }, z3.h\n" + ".inst 0xa1412aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc1bccfb8 // sclamp { z24.s-z27.s }, z29.s, z28.s\n" + ".inst 0xc16914ec // sdot za.s[x8, 4], { z7.h-z8.h }, z9.h\n" + "st1b { z24.s }, p1, [x11]\n" "add x11, x11, x9\n" - "add z13.h, z13.h, z24.h\n" - ".inst 0xc16215cd // sdot za.s[x8, 5], { z14.h-z15.h }, z2.h\n" - "trn1 z14.h, z19.h, z18.h\n" - "trn1 z15.h, z17.h, z16.h\n" + "add z6.h, z6.h, z21.h\n" + ".inst 0xc16114ed // sdot za.s[x8, 5], { z7.h-z8.h }, z1.h\n" + "trn1 z7.h, z23.h, z18.h\n" + "trn1 z8.h, z17.h, z16.h\n" "add x8, x8, #0x2\n" - ".inst 0xa0412be2 // ld1h { z2.h-z3.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" - "st1b { z6.s }, p1, [x10]\n" + ".inst 0xa0412bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + "st1b { z26.s }, p1, [x10]\n" "add x10, x10, x28\n" - ".inst 0xc0040904 // mova za.d[x8, #4], { z8.d-z9.d }\n" - "st1b { z5.s }, p1, [x27]\n" + ".inst 0xc0040bc4 // mova za.d[x8, #4], { z30.d-z31.d }\n" + "st1b { z25.s }, p1, [x27]\n" "add x27, x27, x25\n" - ".inst 0xc0040905 // mova za.d[x8, #5], { z8.d-z9.d }\n" - "add z14.h, z14.h, z24.h\n" - "st1b { z7.s }, p1, [x26]\n" + ".inst 0xc0040bc5 // mova za.d[x8, #5], { z30.d-z31.d }\n" + "add z7.h, z7.h, z21.h\n" + "st1b { z27.s }, p1, [x26]\n" "add x26, x26, x24\n" - "add z15.h, z15.h, z24.h\n" + "add z8.h, z8.h, z21.h\n" "bgt 11b\n" "b 17f\n" "12:" // Padded @@ -384,118 +384,118 @@ void sme2_s8q_planar_3x3_s1_4rows_dot_za_impl( "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1sb { z19.s }, p0/Z, [x14]\n" - "add z19.h, p0/M, z19.h, z24.h\n" + "add z19.h, p0/M, z19.h, z21.h\n" "add x20, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1sb { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z24.h\n" + "add z18.h, p0/M, z18.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1sb { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z24.h\n" + "add z17.h, p0/M, z17.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1sb { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z24.h\n" + "add z16.h, p0/M, z16.h, z21.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z13.h, z19.h, z18.h\n" - "trn1 z14.h, z17.h, z16.h\n" + "trn1 z7.h, z19.h, z18.h\n" + "trn1 z8.h, z17.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1sb { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z24.h\n" + "add z17.h, p0/M, z17.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1sb { z16.s }, p0/Z, [x20]\n" "addvl x20, SP, #8\n" - "add z16.h, p0/M, z16.h, z24.h\n" - ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" - "trn1 z15.h, z17.h, z16.h\n" - ".inst 0xc16115a8 // sdot za.s[x8, 0], { z13.h-z14.h }, z1.h\n" + "add z16.h, p0/M, z16.h, z21.h\n" + ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" + "trn1 z9.h, z17.h, z16.h\n" + ".inst 0xc16a14e8 // sdot za.s[x8, 0], { z7.h-z8.h }, z10.h\n" "add x14, x14, %x[ld_in_col]\n" - ".inst 0xc16015a9 // sdot za.s[x8, 1], { z13.h-z14.h }, z0.h\n" - ".inst 0xa0412a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16315c8 // sdot za.s[x8, 0], { z14.h-z15.h }, z3.h\n" - ".inst 0xc16215c9 // sdot za.s[x8, 1], { z14.h-z15.h }, z2.h\n" + ".inst 0xc16214e9 // sdot za.s[x8, 1], { z7.h-z8.h }, z2.h\n" + ".inst 0xa1412a85 // ld1h { z5.h, z13.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc16d1508 // sdot za.s[x8, 0], { z8.h-z9.h }, z13.h\n" + ".inst 0xc1651509 // sdot za.s[x8, 1], { z8.h-z9.h }, z5.h\n" "14:" // Padded: 1 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1sb { z19.s }, p0/Z, [x14]\n" - "add z19.h, p0/M, z19.h, z24.h\n" + "add z19.h, p0/M, z19.h, z21.h\n" "add x20, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1sb { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z24.h\n" + "add z18.h, p0/M, z18.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1sb { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z24.h\n" + "add z17.h, p0/M, z17.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1sb { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z24.h\n" + "add z16.h, p0/M, z16.h, z21.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z13.h, z19.h, z18.h\n" - "trn1 z14.h, z17.h, z16.h\n" + "trn1 z22.h, z19.h, z18.h\n" + "trn1 z23.h, z17.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1sb { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z24.h\n" + "add z17.h, p0/M, z17.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1sb { z16.s }, p0/Z, [x20]\n" "addvl x21, SP, #4\n" - "add z16.h, p0/M, z16.h, z24.h\n" + "add z16.h, p0/M, z16.h, z21.h\n" ".inst 0xa0402aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21]\n" "addvl x20, SP, #8\n" - "trn1 z15.h, z17.h, z16.h\n" - ".inst 0xc16115a8 // sdot za.s[x8, 0], { z13.h-z14.h }, z1.h\n" - ".inst 0xc16015a9 // sdot za.s[x8, 1], { z13.h-z14.h }, z0.h\n" - ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" + "trn1 z24.h, z17.h, z16.h\n" + ".inst 0xc16116c8 // sdot za.s[x8, 0], { z22.h-z23.h }, z1.h\n" + ".inst 0xc16016c9 // sdot za.s[x8, 1], { z22.h-z23.h }, z0.h\n" + ".inst 0xa1402a85 // ld1h { z5.h, z13.h }, pn10.b/Z, [x20]\n" "add x14, x14, %x[ld_in_col]\n" - ".inst 0xa0412aa2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc16115aa // sdot za.s[x8, 2], { z13.h-z14.h }, z1.h\n" - ".inst 0xc16015ab // sdot za.s[x8, 3], { z13.h-z14.h }, z0.h\n" - ".inst 0xc16315c8 // sdot za.s[x8, 0], { z14.h-z15.h }, z3.h\n" - ".inst 0xc16215c9 // sdot za.s[x8, 1], { z14.h-z15.h }, z2.h\n" - ".inst 0xa0412a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16315ca // sdot za.s[x8, 2], { z14.h-z15.h }, z3.h\n" - ".inst 0xc16215cb // sdot za.s[x8, 3], { z14.h-z15.h }, z2.h\n" + ".inst 0xa0412aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc16d16ca // sdot za.s[x8, 2], { z22.h-z23.h }, z13.h\n" + ".inst 0xc16516cb // sdot za.s[x8, 3], { z22.h-z23.h }, z5.h\n" + ".inst 0xc16116e8 // sdot za.s[x8, 0], { z23.h-z24.h }, z1.h\n" + ".inst 0xc16016e9 // sdot za.s[x8, 1], { z23.h-z24.h }, z0.h\n" + ".inst 0xa0412a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc16116ea // sdot za.s[x8, 2], { z23.h-z24.h }, z1.h\n" + ".inst 0xc16016eb // sdot za.s[x8, 3], { z23.h-z24.h }, z0.h\n" "15:" // Padded: 0 priming loads - ".inst 0xa0402be0 // ld1h { z0.h-z1.h }, pn10.b/Z, [SP]\n" - ".inst 0xa0412be2 // ld1h { z2.h-z3.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + ".inst 0xa1402be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP]\n" + ".inst 0xa0412bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" "cbz x15, 18f\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1sb { z19.s }, p0/Z, [x14]\n" - "add z19.h, p0/M, z19.h, z24.h\n" + "add z19.h, p0/M, z19.h, z21.h\n" "add x20, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1sb { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z24.h\n" + "add z18.h, p0/M, z18.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1sb { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z24.h\n" + "add z17.h, p0/M, z17.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1sb { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z24.h\n" + "add z16.h, p0/M, z16.h, z21.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z13.h, z19.h, z18.h\n" - "trn1 z14.h, z17.h, z16.h\n" + "trn1 z6.h, z19.h, z18.h\n" + "trn1 z7.h, z17.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1sb { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z24.h\n" + "add z17.h, p0/M, z17.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1sb { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z24.h\n" + "add z16.h, p0/M, z16.h, z21.h\n" "sub x15, x15, #0x1\n" "sub x13, x13, #0x1\n" "cmp x15, x13\n" - "trn1 z15.h, z17.h, z16.h\n" + "trn1 z8.h, z17.h, z16.h\n" "csel x23, x15, x13, LT\n" "add x14, x14, %x[ld_in_col]\n" "sub x13, x13, x23\n" @@ -503,121 +503,121 @@ void sme2_s8q_planar_3x3_s1_4rows_dot_za_impl( "16:" // Padded: Main loop "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z21.s }, p0/Z, [x14]\n" - ".inst 0xc16115a8 // sdot za.s[x8, 0], { z13.h-z14.h }, z1.h\n" - ".inst 0xc16015a9 // sdot za.s[x8, 1], { z13.h-z14.h }, z0.h\n" - "add z21.h, p0/M, z21.h, z24.h\n" + "ld1sb { z9.s }, p0/Z, [x14]\n" + ".inst 0xc16d14c8 // sdot za.s[x8, 0], { z6.h-z7.h }, z13.h\n" + ".inst 0xc16514c9 // sdot za.s[x8, 1], { z6.h-z7.h }, z5.h\n" + "add z9.h, p0/M, z9.h, z21.h\n" "add x22, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z20.s }, p0/Z, [x22]\n" - ".inst 0xc16315c8 // sdot za.s[x8, 0], { z14.h-z15.h }, z3.h\n" - "add z20.h, p0/M, z20.h, z24.h\n" + "ld1sb { z19.s }, p0/Z, [x22]\n" + ".inst 0xc16b14e8 // sdot za.s[x8, 0], { z7.h-z8.h }, z11.h\n" + "add z19.h, p0/M, z19.h, z21.h\n" "add x22, x22, %x[ld_in_row]\n" - ".inst 0xc16215c9 // sdot za.s[x8, 1], { z14.h-z15.h }, z2.h\n" + ".inst 0xc16a14e9 // sdot za.s[x8, 1], { z7.h-z8.h }, z10.h\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z19.s }, p0/Z, [x22]\n" - "add z19.h, p0/M, z19.h, z24.h\n" - ".inst 0xc0060804 // mova { z4.d-z5.d }, za.d[x8, #0]\n" + "ld1sb { z18.s }, p0/Z, [x22]\n" + "add z18.h, p0/M, z18.h, z21.h\n" + ".inst 0xc0060818 // mova { z24.d-z25.d }, za.d[x8, #0]\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z18.s }, p0/Z, [x22]\n" - ".inst 0xc0060826 // mova { z6.d-z7.d }, za.d[x8, #1]\n" + "ld1sb { z16.s }, p0/Z, [x22]\n" + ".inst 0xc006083a // mova { z26.d-z27.d }, za.d[x8, #1]\n" "mov x12, #0x4\n" "addvl x21, SP, #4\n" - "add z18.h, p0/M, z18.h, z24.h\n" - ".inst 0xc1aaac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z10.s\n" + "add z16.h, p0/M, z16.h, z21.h\n" + ".inst 0xc1aeac18 // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z14.s\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xa0402aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21]\n" + ".inst 0xa1402aa3 // ld1h { z3.h, z11.h }, pn10.b/Z, [x21]\n" "addvl x20, SP, #8\n" - ".inst 0xc16115aa // sdot za.s[x8, 2], { z13.h-z14.h }, z1.h\n" + ".inst 0xc16b14ca // sdot za.s[x8, 2], { z6.h-z7.h }, z11.h\n" "subs x23, x23, #0x1\n" "ld1sb { z17.s }, p0/Z, [x22]\n" - ".inst 0xc16015ab // sdot za.s[x8, 3], { z13.h-z14.h }, z0.h\n" - ".inst 0xc1abaa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z11.s\n" - ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z24.h\n" + ".inst 0xc16314cb // sdot za.s[x8, 3], { z6.h-z7.h }, z3.h\n" + ".inst 0xc1acaa38 // srshl { z24.s-z27.s }, { z24.s-z27.s }, z12.s\n" + ".inst 0xa1402a85 // ld1h { z5.h, z13.h }, pn10.b/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z21.h\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0xa0412aa2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc16115ac // sdot za.s[x8, 4], { z13.h-z14.h }, z1.h\n" - ".inst 0xc1acab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z12.s\n" - "ld1sb { z16.s }, p0/Z, [x22]\n" - ".inst 0xc16015ad // sdot za.s[x8, 5], { z13.h-z14.h }, z0.h\n" - "add z16.h, p0/M, z16.h, z24.h\n" + ".inst 0xa0412aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc16d14cc // sdot za.s[x8, 4], { z6.h-z7.h }, z13.h\n" + ".inst 0xc1afab18 // add { z24.s-z27.s }, { z24.s-z27.s }, z15.s\n" + "ld1sb { z2.s }, p0/Z, [x22]\n" + ".inst 0xc16514cd // sdot za.s[x8, 5], { z6.h-z7.h }, z5.h\n" + "add z2.h, p0/M, z2.h, z21.h\n" "add x14, x14, %x[ld_in_col]\n" - ".inst 0xc16315ca // sdot za.s[x8, 2], { z14.h-z15.h }, z3.h\n" - ".inst 0xa0402be0 // ld1h { z0.h-z1.h }, pn10.b/Z, [SP]\n" - ".inst 0xc1bacec4 // sclamp { z4.s-z7.s }, z22.s, z26.s\n" - ".inst 0xc16215cb // sdot za.s[x8, 3], { z14.h-z15.h }, z2.h\n" - ".inst 0xa0412a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - "st1b { z4.s }, p1, [x11]\n" + ".inst 0xc16b14ea // sdot za.s[x8, 2], { z7.h-z8.h }, z11.h\n" + ".inst 0xa1402be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP]\n" + ".inst 0xc1bccfb8 // sclamp { z24.s-z27.s }, z29.s, z28.s\n" + ".inst 0xc16a14eb // sdot za.s[x8, 3], { z7.h-z8.h }, z10.h\n" + ".inst 0xa1412a83 // ld1h { z3.h, z11.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + "st1b { z24.s }, p1, [x11]\n" "add x11, x11, x9\n" - ".inst 0xc16315cc // sdot za.s[x8, 4], { z14.h-z15.h }, z3.h\n" - "st1b { z6.s }, p1, [x10]\n" + ".inst 0xc16b14ec // sdot za.s[x8, 4], { z7.h-z8.h }, z11.h\n" + "st1b { z26.s }, p1, [x10]\n" "add x10, x10, x28\n" - "trn1 z13.h, z21.h, z20.h\n" - ".inst 0xc16215cd // sdot za.s[x8, 5], { z14.h-z15.h }, z2.h\n" + "trn1 z6.h, z9.h, z19.h\n" + ".inst 0xc16314ed // sdot za.s[x8, 5], { z7.h-z8.h }, z3.h\n" "add x8, x8, #0x2\n" - ".inst 0xa0412be2 // ld1h { z2.h-z3.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" - "st1b { z5.s }, p1, [x27]\n" + ".inst 0xa0412bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + "st1b { z25.s }, p1, [x27]\n" "add x27, x27, x25\n" - "st1b { z7.s }, p1, [x26]\n" + "st1b { z27.s }, p1, [x26]\n" "add x26, x26, x24\n" - ".inst 0xc0040904 // mova za.d[x8, #4], { z8.d-z9.d }\n" - ".inst 0xc0040905 // mova za.d[x8, #5], { z8.d-z9.d }\n" - "trn1 z14.h, z19.h, z18.h\n" - "trn1 z15.h, z17.h, z16.h\n" + ".inst 0xc0040bc4 // mova za.d[x8, #4], { z30.d-z31.d }\n" + ".inst 0xc0040bc5 // mova za.d[x8, #5], { z30.d-z31.d }\n" + "trn1 z7.h, z18.h, z16.h\n" + "trn1 z8.h, z17.h, z2.h\n" "bgt 16b\n" "17:" // Main loop tail - ".inst 0xc16115a8 // sdot za.s[x8, 0], { z13.h-z14.h }, z1.h\n" + ".inst 0xc16d14c8 // sdot za.s[x8, 0], { z6.h-z7.h }, z13.h\n" "addvl x21, SP, #4\n" "addvl x20, SP, #8\n" - ".inst 0xc16015a9 // sdot za.s[x8, 1], { z13.h-z14.h }, z0.h\n" + ".inst 0xc16514c9 // sdot za.s[x8, 1], { z6.h-z7.h }, z5.h\n" ".inst 0xa0402aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21]\n" - ".inst 0xc16315c8 // sdot za.s[x8, 0], { z14.h-z15.h }, z3.h\n" - ".inst 0xc16215c9 // sdot za.s[x8, 1], { z14.h-z15.h }, z2.h\n" + ".inst 0xc16b14e8 // sdot za.s[x8, 0], { z7.h-z8.h }, z11.h\n" + ".inst 0xc16a14e9 // sdot za.s[x8, 1], { z7.h-z8.h }, z10.h\n" ".inst 0xa0412aa2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc0060804 // mova { z4.d-z5.d }, za.d[x8, #0]\n" - ".inst 0xc0060826 // mova { z6.d-z7.d }, za.d[x8, #1]\n" - ".inst 0xc1aaac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z10.s\n" - ".inst 0xc16115aa // sdot za.s[x8, 2], { z13.h-z14.h }, z1.h\n" - ".inst 0xc1abaa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z11.s\n" - ".inst 0xc16015ab // sdot za.s[x8, 3], { z13.h-z14.h }, z0.h\n" - ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1acab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z12.s\n" - ".inst 0xc16115ac // sdot za.s[x8, 4], { z13.h-z14.h }, z1.h\n" - ".inst 0xc1bacec4 // sclamp { z4.s-z7.s }, z22.s, z26.s\n" - "st1b { z4.s }, p1, [x11]\n" + ".inst 0xc0060818 // mova { z24.d-z25.d }, za.d[x8, #0]\n" + ".inst 0xc006083a // mova { z26.d-z27.d }, za.d[x8, #1]\n" + ".inst 0xc1aeac18 // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z14.s\n" + ".inst 0xc16114ca // sdot za.s[x8, 2], { z6.h-z7.h }, z1.h\n" + ".inst 0xc1acaa38 // srshl { z24.s-z27.s }, { z24.s-z27.s }, z12.s\n" + ".inst 0xc16014cb // sdot za.s[x8, 3], { z6.h-z7.h }, z0.h\n" + ".inst 0xa1402a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1afab18 // add { z24.s-z27.s }, { z24.s-z27.s }, z15.s\n" + ".inst 0xc16914cc // sdot za.s[x8, 4], { z6.h-z7.h }, z9.h\n" + ".inst 0xc1bccfb8 // sclamp { z24.s-z27.s }, z29.s, z28.s\n" + "st1b { z24.s }, p1, [x11]\n" "add x11, x11, x9\n" - ".inst 0xc16015ad // sdot za.s[x8, 5], { z13.h-z14.h }, z0.h\n" - "st1b { z6.s }, p1, [x10]\n" + ".inst 0xc16114cd // sdot za.s[x8, 5], { z6.h-z7.h }, z1.h\n" + "st1b { z26.s }, p1, [x10]\n" "add x10, x10, x28\n" - ".inst 0xc16315ca // sdot za.s[x8, 2], { z14.h-z15.h }, z3.h\n" - "st1b { z5.s }, p1, [x27]\n" + ".inst 0xc16314ea // sdot za.s[x8, 2], { z7.h-z8.h }, z3.h\n" + "st1b { z25.s }, p1, [x27]\n" "add x27, x27, x25\n" - ".inst 0xc16215cb // sdot za.s[x8, 3], { z14.h-z15.h }, z2.h\n" - ".inst 0xa0412a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - "st1b { z7.s }, p1, [x26]\n" + ".inst 0xc16214eb // sdot za.s[x8, 3], { z7.h-z8.h }, z2.h\n" + ".inst 0xa0412a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + "st1b { z27.s }, p1, [x26]\n" "add x26, x26, x24\n" - ".inst 0xc16315cc // sdot za.s[x8, 4], { z14.h-z15.h }, z3.h\n" - ".inst 0xc16215cd // sdot za.s[x8, 5], { z14.h-z15.h }, z2.h\n" + ".inst 0xc16114ec // sdot za.s[x8, 4], { z7.h-z8.h }, z1.h\n" + ".inst 0xc16014ed // sdot za.s[x8, 5], { z7.h-z8.h }, z0.h\n" "add x8, x8, #0x2\n" - ".inst 0xc0040904 // mova za.d[x8, #4], { z8.d-z9.d }\n" - ".inst 0xc0040905 // mova za.d[x8, #5], { z8.d-z9.d }\n" + ".inst 0xc0040bc4 // mova za.d[x8, #4], { z30.d-z31.d }\n" + ".inst 0xc0040bc5 // mova za.d[x8, #5], { z30.d-z31.d }\n" "18:" // Main loop skip tail "cbz x13, 20f\n" "19:" // Right padding loop ".inst 0xc0060804 // mova { z4.d-z5.d }, za.d[x8, #0]\n" "subs x13, x13, #0x1\n" ".inst 0xc0060826 // mova { z6.d-z7.d }, za.d[x8, #1]\n" - ".inst 0xc1aaac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z10.s\n" + ".inst 0xc1aeac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z14.s\n" "add x8, x8, #0x2\n" - ".inst 0xc1abaa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z11.s\n" - ".inst 0xc0040904 // mova za.d[x8, #4], { z8.d-z9.d }\n" - ".inst 0xc1acab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z12.s\n" - ".inst 0xc0040905 // mova za.d[x8, #5], { z8.d-z9.d }\n" - ".inst 0xc1bacec4 // sclamp { z4.s-z7.s }, z22.s, z26.s\n" + ".inst 0xc1acaa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z12.s\n" + ".inst 0xc0040bc4 // mova za.d[x8, #4], { z30.d-z31.d }\n" + ".inst 0xc1afab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z15.s\n" + ".inst 0xc0040bc5 // mova za.d[x8, #5], { z30.d-z31.d }\n" + ".inst 0xc1bccfa4 // sclamp { z4.s-z7.s }, z29.s, z28.s\n" "st1b { z4.s }, p1, [x11]\n" "add x11, x11, x9\n" "st1b { z6.s }, p1, [x10]\n" @@ -628,15 +628,15 @@ void sme2_s8q_planar_3x3_s1_4rows_dot_za_impl( "add x26, x26, x24\n" "bgt 19b\n" "20:" // End - "ldr x22, [%x[args], %[offsetof_Args_weights]]\n" - "incw x22, ALL, MUL #9\n" - "str x22, [%x[args], %[offsetof_Args_weights]]\n" + "ldr x20, [%x[args], %[offsetof_Args_weights]]\n" + "incw x20, ALL, MUL #9\n" + "str x20, [%x[args], %[offsetof_Args_weights]]\n" "incw x16\n" - "ldr x20, [%x[args], %[offsetof_Args_ld_in_vl]]\n" + "ldr x21, [%x[args], %[offsetof_Args_ld_in_vl]]\n" "whilelt p1.s, x16, x17\n" - "ldr x14, [%x[args], %[offsetof_Args_inptr]]\n" - "add x14, x14, x20\n" - "str x14, [%x[args], %[offsetof_Args_inptr]]\n" + "ldr x20, [%x[args], %[offsetof_Args_inptr]]\n" + "add x20, x20, x21\n" + "str x20, [%x[args], %[offsetof_Args_inptr]]\n" "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" "ldr x24, [%x[args], %[offsetof_Args_ld_out_vls]]\n" "ldp x23, x22, [x25, #0x0]\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_3x3_s2_4rows_dot_za.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_3x3_s2_4rows_dot_za.hpp index d14d662240597498150e911599035e61e082a212..56fb127aa0fb5f9c385e6ff7bb0b980170d7a429 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_3x3_s2_4rows_dot_za.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_3x3_s2_4rows_dot_za.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,8 @@ * SOFTWARE. */ +#include "src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp" + namespace arm_conv { namespace depthwise { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_3x3_s2_4rows_dot_za/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_3x3_s2_4rows_dot_za/generic.cpp index 03575aa7998297b925ac5c57b22ede6e5c522926..1d0efc6bc1a2ab0fbf3205f96efaf7b30754b492 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_3x3_s2_4rows_dot_za/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_3x3_s2_4rows_dot_za/generic.cpp @@ -73,86 +73,86 @@ void sme2_s8q_planar_3x3_s2_4rows_dot_za_impl( "ptrue p2.b\n" "mov x20, #0x9\n" "ldr x7, [%x[args], %[offsetof_Args_pad_top]]\n" - "ld1rh { z5.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_a_offset]]\n" + "ld1rh { z11.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_a_offset]]\n" "sub x20, x20, x6\n" ".inst 0x25207812 // ptrue pn10.b\n" "ldr x17, [%x[args], %[offsetof_Args_n_channels]]\n" "whilelt p1.s, XZR, x17\n" "whilelt p9.s, XZR, x20\n" - "ld1rw { z4.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" + "ld1rw { z13.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" "whilelt p8.s, XZR, x7\n" "addvl SP, SP, #-6\n" "ldr x16, [%x[args], %[offsetof_Args_current_channel]]\n" - "neg z5.h, p2/M, z5.h\n" + "neg z11.h, p2/M, z11.h\n" "eor p8.b, p2/Z, p8.b, p9.b\n" - "ld1rw { z8.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_mul]]\n" - "ld1rw { z7.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_right_shift]]\n" - "ld1rw { z27.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" - "ld1rw { z23.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" + "ld1rw { z6.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z9.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z10.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" + "ld1rw { z7.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" "1:" // Channel loop "ldr x20, [%x[qp], %[offsetof_Requantize32_bias]]\n" - "mov z0.s, #0x0\n" + "mov z28.s, #0x0\n" "cbz x20, 2f\n" - "ld1w { z0.s }, p1/Z, [x20, x16, LSL #2]\n" + "ld1w { z28.s }, p1/Z, [x20, x16, LSL #2]\n" "2:" // Load bias: Done "ldr x22, [%x[args], %[offsetof_Args_weights]]\n" "mov x20, x22\n" - "ld1sb { z24.s }, p2/Z, [x20]\n" + "ld1sb { z26.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "ld1rh { z13.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" - "sub z24.h, z24.h, z13.h\n" + "ld1rh { z16.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" + "sub z26.h, z26.h, z16.h\n" "incw x22\n" - "mov z17.h, #0x0\n" - "ld1sb { z25.s }, p2/Z, [x20]\n" + "mov z24.h, #0x0\n" + "ld1sb { z3.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "sub z25.h, z25.h, z13.h\n" - "trn1 z10.h, z24.h, z25.h\n" - "ld1sb { z16.s }, p2/Z, [x20]\n" - "sub z16.h, z16.h, z13.h\n" + "sub z3.h, z3.h, z16.h\n" + "trn1 z31.h, z26.h, z3.h\n" + "ld1sb { z21.s }, p2/Z, [x20]\n" + "sub z21.h, z21.h, z16.h\n" "mov x20, x22\n" - "trn1 z11.h, z16.h, z17.h\n" - "ld1sb { z24.s }, p2/Z, [x20]\n" + "trn1 z14.h, z21.h, z24.h\n" + "ld1sb { z2.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "sub z24.h, z24.h, z13.h\n" + "sub z2.h, z2.h, z16.h\n" "addvl x21, SP, #6\n" "ld1sb { z25.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "sub z25.h, z25.h, z13.h\n" + "sub z25.h, z25.h, z16.h\n" "incw x22\n" - "ld1sb { z16.s }, p2/Z, [x20]\n" - "sub z16.h, z16.h, z13.h\n" + "ld1sb { z27.s }, p2/Z, [x20]\n" + "sub z27.h, z27.h, z16.h\n" "addvl x21, x21, #-2\n" "mov x20, x22\n" - "st1h { z10.h }, p2, [x21]\n" - "trn1 z10.h, z24.h, z25.h\n" - "ld1sb { z24.s }, p2/Z, [x20]\n" + "st1h { z31.h }, p2, [x21]\n" + "trn1 z4.h, z2.h, z25.h\n" + "ld1sb { z26.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "ld1sb { z25.s }, p2/Z, [x20]\n" + "ld1sb { z23.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "st1h { z11.h }, p2, [x21, #1, MUL VL]\n" - "trn1 z11.h, z16.h, z17.h\n" - "ld1sb { z16.s }, p2/Z, [x20]\n" - "sub z24.h, z24.h, z13.h\n" - "sub z25.h, z25.h, z13.h\n" + "st1h { z14.h }, p2, [x21, #1, MUL VL]\n" + "trn1 z12.h, z27.h, z24.h\n" + "ld1sb { z20.s }, p2/Z, [x20]\n" + "sub z26.h, z26.h, z16.h\n" + "sub z23.h, z23.h, z16.h\n" "ldr x20, [%x[qp], %[offsetof_Requantize32_per_channel_muls]]\n" - "sub z16.h, z16.h, z13.h\n" + "sub z20.h, z20.h, z16.h\n" "addvl x21, x21, #-2\n" - "st1h { z10.h }, p2, [x21]\n" - "mov z1.d, z0.d\n" - "st1h { z11.h }, p2, [x21, #1, MUL VL]\n" + "st1h { z4.h }, p2, [x21]\n" + "mov z29.d, z28.d\n" + "st1h { z12.h }, p2, [x21, #1, MUL VL]\n" "addvl x21, x21, #-2\n" - "mov z2.d, z0.d\n" - "mov z3.d, z0.d\n" - "trn1 z10.h, z24.h, z25.h\n" - "st1h { z10.h }, p2, [x21]\n" - "trn1 z11.h, z16.h, z17.h\n" - "st1h { z11.h }, p2, [x21, #1, MUL VL]\n" + "mov z30.d, z28.d\n" + "mov z31.d, z28.d\n" + "trn1 z25.h, z26.h, z23.h\n" + "st1h { z25.h }, p2, [x21]\n" + "trn1 z3.h, z20.h, z24.h\n" + "st1h { z3.h }, p2, [x21, #1, MUL VL]\n" "cbz x20, 3f\n" - "ld1w { z8.s }, p1/Z, [x20, x16, LSL #2]\n" + "ld1w { z6.s }, p1/Z, [x20, x16, LSL #2]\n" "3:" // Load mul: End "ldr x20, [%x[qp], %[offsetof_Requantize32_per_channel_right_shifts]]\n" "cbz x20, 4f\n" - "ld1w { z7.s }, p1/Z, [x20, x16, LSL #2]\n" + "ld1w { z9.s }, p1/Z, [x20, x16, LSL #2]\n" "4:" // Load right_shift: End "ldr x15, [%x[args], %[offsetof_Args_input_cols]]\n" "sub x20, x15, #0x1\n" @@ -172,18 +172,18 @@ void sme2_s8q_planar_3x3_s2_4rows_dot_za_impl( ".inst 0xf8b74a9c // rprfm pldstrm, x23, [x20]\n" "add x20, x20, %x[ld_in_col]\n" "bgt 5b\n" - "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" + "ldr x23, [%x[args], %[offsetof_Args_outptrs]]\n" "lsl x20, %x[ld_in_row], #0x0\n" "msub x14, x7, x20, x14\n" - ".inst 0xc0040c00 // mova za.d[x8, #0], { z0.d-z3.d }\n" + ".inst 0xc0040f80 // mova za.d[x8, #0], { z28.d-z31.d }\n" "ldr x20, [%x[args], %[offsetof_Args_ld_out_cols]]\n" - ".inst 0xc0040c01 // mova za.d[x8, #1], { z0.d-z3.d }\n" + ".inst 0xc0040f81 // mova za.d[x8, #1], { z28.d-z31.d }\n" "mov x22, #0x2\n" - "ldp x11, x10, [x25], #0x10\n" - ".inst 0xc0040c02 // mova za.d[x8, #2], { z0.d-z3.d }\n" + "ldp x11, x10, [x23], #0x10\n" + ".inst 0xc0040f82 // mova za.d[x8, #2], { z28.d-z31.d }\n" "ldp x9, x28, [x20], #0x10\n" "ldr x21, [%x[args], %[offsetof_Args_pad_left]]\n" - "ldp x27, x26, [x25], #0x10\n" + "ldp x27, x26, [x23], #0x10\n" "ldp x25, x24, [x20], #0x10\n" "cbz x21, 7f\n" "cmp x21, x22\n" @@ -191,24 +191,24 @@ void sme2_s8q_planar_3x3_s2_4rows_dot_za_impl( "sub x21, x21, x20\n" "sub x22, x22, x20\n" "cbz x21, 7f\n" - ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" - ".inst 0xc1a8ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z8.s\n" + ".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n" + ".inst 0xc1a6ac18 // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z6.s\n" "and x22, x21, #0x1\n" - ".inst 0xc1a7aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z7.s\n" + ".inst 0xc1a9aa38 // srshl { z24.s-z27.s }, { z24.s-z27.s }, z9.s\n" "add x21, x21, #0x1\n" "lsr x21, x21, #0x1\n" - ".inst 0xc1a4ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" + ".inst 0xc1adab18 // add { z24.s-z27.s }, { z24.s-z27.s }, z13.s\n" "sub x13, x13, x21\n" - ".inst 0xc1b7cf7c // sclamp { z28.s-z31.s }, z27.s, z23.s\n" + ".inst 0xc1a7cd58 // sclamp { z24.s-z27.s }, z10.s, z7.s\n" "6:" // Left padding "subs x21, x21, #0x1\n" - "st1b { z28.s }, p1, [x11]\n" + "st1b { z24.s }, p1, [x11]\n" "add x11, x11, x9\n" - "st1b { z29.s }, p1, [x10]\n" + "st1b { z25.s }, p1, [x10]\n" "add x10, x10, x28\n" - "st1b { z30.s }, p1, [x27]\n" + "st1b { z26.s }, p1, [x27]\n" "add x27, x27, x25\n" - "st1b { z31.s }, p1, [x26]\n" + "st1b { z27.s }, p1, [x26]\n" "add x26, x26, x24\n" "bgt 6b\n" "7:" // Left padding: End @@ -220,194 +220,194 @@ void sme2_s8q_planar_3x3_s2_4rows_dot_za_impl( "beq 9f\n" "8:" // Unpadded: 2 priming loads "add x21, x14, %x[ld_in_row]\n" - "ld1sb { z12.s }, p1/Z, [x14]\n" + "ld1sb { z1.s }, p1/Z, [x14]\n" "addvl x20, SP, #4\n" - "ld1sb { z20.s }, p1/Z, [x21]\n" + "ld1sb { z21.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "add z12.h, z12.h, z5.h\n" - "ld1sb { z13.s }, p1/Z, [x21]\n" + "trn1 z1.h, z1.h, z21.h\n" + "add z1.h, z1.h, z11.h\n" + "ld1sb { z2.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "add x14, x14, %x[ld_in_col]\n" - "ld1sb { z19.s }, p1/Z, [x21]\n" + "ld1sb { z15.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z5.h\n" - "ld1sb { z14.s }, p1/Z, [x21]\n" + "trn1 z2.h, z2.h, z15.h\n" + "add z2.h, z2.h, z11.h\n" + "ld1sb { z3.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1sb { z18.s }, p1/Z, [x21]\n" + "ld1sb { z21.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z5.h\n" - "ld1sb { z15.s }, p1/Z, [x21]\n" + "trn1 z3.h, z3.h, z21.h\n" + "add z3.h, z3.h, z11.h\n" + "ld1sb { z4.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1sb { z17.s }, p1/Z, [x21]\n" + "ld1sb { z19.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z15.h, z15.h, z5.h\n" - "ld1sb { z16.s }, p1/Z, [x21]\n" - "mov z16.d, z16.d\n" - "add z16.h, z16.h, z5.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" + "trn1 z4.h, z4.h, z19.h\n" + "add z4.h, z4.h, z11.h\n" + "ld1sb { z8.s }, p1/Z, [x21]\n" + "mov z5.d, z8.d\n" + "add z5.h, z5.h, z11.h\n" + ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1701428 // sdot za.s[x8, 0], { z1.h-z4.h }, z0.h\n" + ".inst 0xc1781448 // sdot za.s[x8, 0], { z2.h-z5.h }, z8.h\n" "9:" // Unpadded: 1 priming loads "add x21, x14, %x[ld_in_row]\n" - "ld1sb { z12.s }, p1/Z, [x14]\n" + "ld1sb { z1.s }, p1/Z, [x14]\n" "addvl x20, SP, #2\n" - "ld1sb { z20.s }, p1/Z, [x21]\n" + "ld1sb { z21.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "add z12.h, z12.h, z5.h\n" - "ld1sb { z13.s }, p1/Z, [x21]\n" + "trn1 z1.h, z1.h, z21.h\n" + "add z1.h, z1.h, z11.h\n" + "ld1sb { z2.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "add x14, x14, %x[ld_in_col]\n" - "ld1sb { z19.s }, p1/Z, [x21]\n" + "ld1sb { z12.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z5.h\n" - "ld1sb { z14.s }, p1/Z, [x21]\n" + "trn1 z2.h, z2.h, z12.h\n" + "add z2.h, z2.h, z11.h\n" + "ld1sb { z3.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1sb { z18.s }, p1/Z, [x21]\n" + "ld1sb { z8.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z5.h\n" - "ld1sb { z15.s }, p1/Z, [x21]\n" + "trn1 z3.h, z3.h, z8.h\n" + "add z3.h, z3.h, z11.h\n" + "ld1sb { z4.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1sb { z17.s }, p1/Z, [x21]\n" + "ld1sb { z5.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z15.h, z15.h, z5.h\n" - "ld1sb { z16.s }, p1/Z, [x21]\n" - "mov z16.d, z16.d\n" - "add z16.h, z16.h, z5.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" + "trn1 z4.h, z4.h, z5.h\n" + "add z4.h, z4.h, z11.h\n" + "ld1sb { z5.s }, p1/Z, [x21]\n" + "mov z5.d, z5.d\n" + "add z5.h, z5.h, z11.h\n" + ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1701428 // sdot za.s[x8, 0], { z1.h-z4.h }, z0.h\n" + ".inst 0xc1781448 // sdot za.s[x8, 0], { z2.h-z5.h }, z8.h\n" "10:" // Unpadded: 0 priming loads "cmp x15, #0x2\n" - ".inst 0xa0402bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP]\n" + ".inst 0xa0402bee // ld1h { z14.h-z15.h }, pn10.b/Z, [SP]\n" "blt 18f\n" "add x21, x14, %x[ld_in_row]\n" - "ld1sb { z12.s }, p1/Z, [x14]\n" + "ld1sb { z21.s }, p1/Z, [x14]\n" "sub x15, x15, #0x2\n" - "ld1sb { z20.s }, p1/Z, [x21]\n" + "ld1sb { z8.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z21.h, z21.h, z8.h\n" "sub x13, x13, #0x1\n" - "ld1sb { z13.s }, p1/Z, [x21]\n" + "ld1sb { z22.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "lsr x20, x15, #0x1\n" - "add z12.h, z12.h, z5.h\n" - "ld1sb { z19.s }, p1/Z, [x21]\n" + "add z21.h, z21.h, z11.h\n" + "ld1sb { z25.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" + "trn1 z22.h, z22.h, z25.h\n" "cmp x20, x13\n" - "ld1sb { z14.s }, p1/Z, [x21]\n" + "ld1sb { z23.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "csel x23, x20, x13, LT\n" - "add z13.h, z13.h, z5.h\n" + "add z22.h, z22.h, z11.h\n" "ld1sb { z18.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z5.h\n" - "ld1sb { z15.s }, p1/Z, [x21]\n" + "trn1 z23.h, z23.h, z18.h\n" + "add z23.h, z23.h, z11.h\n" + "ld1sb { z24.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "add x14, x14, %x[ld_in_col]\n" - "ld1sb { z17.s }, p1/Z, [x21]\n" + "ld1sb { z19.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z15.h, z15.h, z5.h\n" - "ld1sb { z16.s }, p1/Z, [x21]\n" - "mov z16.d, z16.d\n" - "add z16.h, z16.h, z5.h\n" + "trn1 z24.h, z24.h, z19.h\n" + "add z24.h, z24.h, z11.h\n" + "ld1sb { z8.s }, p1/Z, [x21]\n" + "mov z25.d, z8.d\n" + "add z25.h, z25.h, z11.h\n" "and x15, x15, #0x1\n" "sub x13, x13, x23\n" "cbz x23, 17f\n" "11:" // Unpadded: Main loop - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" + ".inst 0xc17e16a8 // sdot za.s[x8, 0], { z21.h-z24.h }, z14.h\n" "addvl x20, SP, #4\n" "add x22, x14, %x[ld_in_row]\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" + ".inst 0xc17f16c8 // sdot za.s[x8, 0], { z22.h-z25.h }, z15.h\n" + ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" "addvl x21, SP, #2\n" "subs x23, x23, #0x1\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - "ld1sb { z12.s }, p1/Z, [x14]\n" + ".inst 0xc17016a9 // sdot za.s[x8, 1], { z21.h-z24.h }, z0.h\n" + "ld1sb { z21.s }, p1/Z, [x14]\n" "add x14, x14, %x[ld_in_col]\n" "add x20, x14, %x[ld_in_row]\n" - "ld1sb { z20.s }, p1/Z, [x22]\n" + "ld1sb { z18.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - ".inst 0xc17b15a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z11.h\n" - "trn1 z12.h, z12.h, z20.h\n" - "ld1sb { z13.s }, p1/Z, [x22]\n" + ".inst 0xc17116c9 // sdot za.s[x8, 1], { z22.h-z25.h }, z1.h\n" + "trn1 z21.h, z21.h, z18.h\n" + "ld1sb { z22.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "add z12.h, z12.h, z5.h\n" - ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" - "ld1sb { z19.s }, p1/Z, [x22]\n" + "add z21.h, z21.h, z11.h\n" + ".inst 0xc0060c00 // mova { z0.d-z3.d }, za.d[x8, #0]\n" + "ld1sb { z8.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z5.h\n" - "ld1sb { z14.s }, p1/Z, [x22]\n" + "trn1 z22.h, z22.h, z8.h\n" + "add z22.h, z22.h, z11.h\n" + "ld1sb { z23.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "add x8, x8, #0x1\n" - ".inst 0xc0040c02 // mova za.d[x8, #2], { z0.d-z3.d }\n" - "ld1sb { z18.s }, p1/Z, [x22]\n" + ".inst 0xc0040f82 // mova za.d[x8, #2], { z28.d-z31.d }\n" + "ld1sb { z27.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z5.h\n" - "ld1sb { z15.s }, p1/Z, [x22]\n" + "trn1 z23.h, z23.h, z27.h\n" + "add z23.h, z23.h, z11.h\n" + "ld1sb { z24.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - ".inst 0xc1a8ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z8.s\n" - "ld1sb { z17.s }, p1/Z, [x22]\n" + ".inst 0xc1a6ac00 // sqdmulh { z0.s-z3.s }, { z0.s-z3.s }, z6.s\n" + "ld1sb { z8.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z15.h, z15.h, z5.h\n" - "ld1sb { z16.s }, p1/Z, [x22]\n" - "mov z16.d, z16.d\n" - "add z16.h, z16.h, z5.h\n" - ".inst 0xa0402aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21]\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xc1a7aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z7.s\n" - "ld1sb { z12.s }, p1/Z, [x14]\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" - ".inst 0xc1a4ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" - "ld1sb { z20.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - ".inst 0xc1b7cf7c // sclamp { z28.s-z31.s }, z27.s, z23.s\n" - "ld1sb { z13.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row]\n" - "st1b { z28.s }, p1, [x11]\n" + "trn1 z24.h, z24.h, z8.h\n" + "add z24.h, z24.h, z11.h\n" + "ld1sb { z4.s }, p1/Z, [x22]\n" + "mov z25.d, z4.d\n" + "add z25.h, z25.h, z11.h\n" + ".inst 0xa1402aa4 // ld1h { z4.h, z12.h }, pn10.b/Z, [x21]\n" + ".inst 0xc17416a8 // sdot za.s[x8, 0], { z21.h-z24.h }, z4.h\n" + ".inst 0xc1a9aa20 // srshl { z0.s-z3.s }, { z0.s-z3.s }, z9.s\n" + "ld1sb { z21.s }, p1/Z, [x14]\n" + ".inst 0xc17c16c8 // sdot za.s[x8, 0], { z22.h-z25.h }, z12.h\n" + ".inst 0xc1adab00 // add { z0.s-z3.s }, { z0.s-z3.s }, z13.s\n" + "ld1sb { z12.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row]\n" + "trn1 z21.h, z21.h, z12.h\n" + ".inst 0xc1a7cd40 // sclamp { z0.s-z3.s }, z10.s, z7.s\n" + "ld1sb { z22.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row]\n" + "st1b { z0.s }, p1, [x11]\n" "add x11, x11, x9\n" - "ld1sb { z19.s }, p1/Z, [x20]\n" + "ld1sb { z20.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "st1b { z29.s }, p1, [x10]\n" - "ld1sb { z14.s }, p1/Z, [x20]\n" + "trn1 z22.h, z22.h, z20.h\n" + "st1b { z1.s }, p1, [x10]\n" + "ld1sb { z23.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" "add x10, x10, x28\n" - "st1b { z30.s }, p1, [x27]\n" - "ld1sb { z18.s }, p1/Z, [x20]\n" + "st1b { z2.s }, p1, [x27]\n" + "ld1sb { z24.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" + "trn1 z23.h, z23.h, z24.h\n" "add x27, x27, x25\n" - "ld1sb { z15.s }, p1/Z, [x20]\n" + "ld1sb { z24.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "st1b { z31.s }, p1, [x26]\n" + "st1b { z3.s }, p1, [x26]\n" "add x26, x26, x24\n" - "ld1sb { z17.s }, p1/Z, [x20]\n" + "ld1sb { z3.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z12.h, z12.h, z5.h\n" - "ld1sb { z16.s }, p1/Z, [x20]\n" - "mov z16.d, z16.d\n" - "add z13.h, z13.h, z5.h\n" + "trn1 z24.h, z24.h, z3.h\n" + "add z21.h, z21.h, z11.h\n" + "ld1sb { z3.s }, p1/Z, [x20]\n" + "mov z25.d, z3.d\n" + "add z22.h, z22.h, z11.h\n" "add x14, x14, %x[ld_in_col]\n" - ".inst 0xa0402bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP]\n" - "add z14.h, z14.h, z5.h\n" - "add z15.h, z15.h, z5.h\n" - "add z16.h, z16.h, z5.h\n" + ".inst 0xa0402bee // ld1h { z14.h-z15.h }, pn10.b/Z, [SP]\n" + "add z23.h, z23.h, z11.h\n" + "add z24.h, z24.h, z11.h\n" + "add z25.h, z25.h, z11.h\n" "bgt 11b\n" "b 17f\n" "12:" // Padded @@ -418,442 +418,442 @@ void sme2_s8q_planar_3x3_s2_4rows_dot_za_impl( "13:" // Padded: 2 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z12.s }, p0/Z, [x14]\n" - "add z12.h, p0/M, z12.h, z5.h\n" + "ld1sb { z22.s }, p0/Z, [x14]\n" + "add z22.h, p0/M, z22.h, z11.h\n" "add x20, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z5.h\n" + "ld1sb { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z5.h\n" + "ld1sb { z23.s }, p0/Z, [x20]\n" + "add z23.h, p0/M, z23.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z5.h\n" + "ld1sb { z4.s }, p0/Z, [x20]\n" + "add z4.h, p0/M, z4.h, z11.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "trn1 z13.h, z13.h, z19.h\n" + "trn1 z22.h, z22.h, z17.h\n" + "trn1 z23.h, z23.h, z4.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z5.h\n" + "ld1sb { z24.s }, p0/Z, [x20]\n" + "add z24.h, p0/M, z24.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1sb { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z5.h\n" + "add z18.h, p0/M, z18.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z5.h\n" + "ld1sb { z25.s }, p0/Z, [x20]\n" + "add z25.h, p0/M, z25.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1sb { z17.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z17.h, p0/M, z17.h, z5.h\n" + "add z17.h, p0/M, z17.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z5.h\n" + "ld1sb { z1.s }, p0/Z, [x20]\n" + "add z1.h, p0/M, z1.h, z11.h\n" "addvl x20, SP, #4\n" - "trn1 z14.h, z14.h, z18.h\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" - "mov z16.d, z16.d\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" + "trn1 z24.h, z24.h, z18.h\n" + "trn1 z25.h, z25.h, z17.h\n" + ".inst 0xa1402a84 // ld1h { z4.h, z12.h }, pn10.b/Z, [x20]\n" + "mov z26.d, z1.d\n" + ".inst 0xc17416c8 // sdot za.s[x8, 0], { z22.h-z25.h }, z4.h\n" "add x14, x14, %x[ld_in_col]\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" + ".inst 0xc17c16e8 // sdot za.s[x8, 0], { z23.h-z26.h }, z12.h\n" "14:" // Padded: 1 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z12.s }, p0/Z, [x14]\n" - "add z12.h, p0/M, z12.h, z5.h\n" + "ld1sb { z22.s }, p0/Z, [x14]\n" + "add z22.h, p0/M, z22.h, z11.h\n" "add x20, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z5.h\n" + "ld1sb { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z5.h\n" + "ld1sb { z23.s }, p0/Z, [x20]\n" + "add z23.h, p0/M, z23.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z5.h\n" + "ld1sb { z5.s }, p0/Z, [x20]\n" + "add z5.h, p0/M, z5.h, z11.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "trn1 z13.h, z13.h, z19.h\n" + "trn1 z22.h, z22.h, z17.h\n" + "trn1 z23.h, z23.h, z5.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z5.h\n" + "ld1sb { z24.s }, p0/Z, [x20]\n" + "add z24.h, p0/M, z24.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1sb { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z5.h\n" + "add z18.h, p0/M, z18.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z5.h\n" + "ld1sb { z25.s }, p0/Z, [x20]\n" + "add z25.h, p0/M, z25.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1sb { z17.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z17.h, p0/M, z17.h, z5.h\n" + "add z17.h, p0/M, z17.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z5.h\n" + "ld1sb { z15.s }, p0/Z, [x20]\n" + "add z15.h, p0/M, z15.h, z11.h\n" "addvl x20, SP, #2\n" - "trn1 z14.h, z14.h, z18.h\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" - "mov z16.d, z16.d\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" + "trn1 z24.h, z24.h, z18.h\n" + "trn1 z25.h, z25.h, z17.h\n" + ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" + "mov z26.d, z15.d\n" + ".inst 0xc17016c8 // sdot za.s[x8, 0], { z22.h-z25.h }, z0.h\n" "add x14, x14, %x[ld_in_col]\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" + ".inst 0xc17116e8 // sdot za.s[x8, 0], { z23.h-z26.h }, z1.h\n" "15:" // Padded: 0 priming loads "cmp x15, #0x2\n" - ".inst 0xa0402bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP]\n" + ".inst 0xa0402bee // ld1h { z14.h-z15.h }, pn10.b/Z, [SP]\n" "blt 18f\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z12.s }, p0/Z, [x14]\n" - "add z12.h, p0/M, z12.h, z5.h\n" + "ld1sb { z21.s }, p0/Z, [x14]\n" + "add z21.h, p0/M, z21.h, z11.h\n" "add x20, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z5.h\n" + "ld1sb { z18.s }, p0/Z, [x20]\n" + "add z18.h, p0/M, z18.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z5.h\n" + "ld1sb { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z5.h\n" + "ld1sb { z3.s }, p0/Z, [x20]\n" + "add z3.h, p0/M, z3.h, z11.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "trn1 z13.h, z13.h, z19.h\n" + "trn1 z21.h, z21.h, z18.h\n" + "trn1 z22.h, z22.h, z3.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z5.h\n" + "ld1sb { z23.s }, p0/Z, [x20]\n" + "add z23.h, p0/M, z23.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z5.h\n" + "ld1sb { z19.s }, p0/Z, [x20]\n" + "add z19.h, p0/M, z19.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z5.h\n" + "ld1sb { z24.s }, p0/Z, [x20]\n" + "add z24.h, p0/M, z24.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z17.s }, p0/Z, [x20]\n" + "ld1sb { z20.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z17.h, p0/M, z17.h, z5.h\n" + "add z20.h, p0/M, z20.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z5.h\n" + "ld1sb { z3.s }, p0/Z, [x20]\n" + "add z3.h, p0/M, z3.h, z11.h\n" "sub x15, x15, #0x2\n" "sub x13, x13, #0x1\n" - "trn1 z14.h, z14.h, z18.h\n" - "trn1 z15.h, z15.h, z17.h\n" + "trn1 z23.h, z23.h, z19.h\n" + "trn1 z24.h, z24.h, z20.h\n" "lsr x20, x15, #0x1\n" "cmp x20, x13\n" - "mov z16.d, z16.d\n" + "mov z25.d, z3.d\n" "csel x22, x20, x13, LT\n" "add x14, x14, %x[ld_in_col]\n" "and x15, x15, #0x1\n" "sub x13, x13, x22\n" "cbz x22, 17f\n" "16:" // Padded: Main loop - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" + ".inst 0xc17e16a8 // sdot za.s[x8, 0], { z21.h-z24.h }, z14.h\n" "addvl x20, SP, #4\n" "mov x12, #0x0\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" + ".inst 0xc17f16c8 // sdot za.s[x8, 0], { z22.h-z25.h }, z15.h\n" + ".inst 0xa1402a84 // ld1h { z4.h, z12.h }, pn10.b/Z, [x20]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "add x21, x14, %x[ld_in_row]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - "ld1sb { z12.s }, p0/Z, [x14]\n" - "add z12.h, p0/M, z12.h, z5.h\n" + ".inst 0xc17416a9 // sdot za.s[x8, 1], { z21.h-z24.h }, z4.h\n" + "ld1sb { z21.s }, p0/Z, [x14]\n" + "add z21.h, p0/M, z21.h, z11.h\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z20.s }, p0/Z, [x21]\n" - "add z20.h, p0/M, z20.h, z5.h\n" + "ld1sb { z14.s }, p0/Z, [x21]\n" + "add z14.h, p0/M, z14.h, z11.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc17b15a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z11.h\n" - "ld1sb { z13.s }, p0/Z, [x21]\n" - "add z13.h, p0/M, z13.h, z5.h\n" + ".inst 0xc17c16c9 // sdot za.s[x8, 1], { z22.h-z25.h }, z12.h\n" + "ld1sb { z22.s }, p0/Z, [x21]\n" + "add z22.h, p0/M, z22.h, z11.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z19.s }, p0/Z, [x21]\n" + "ld1sb { z15.s }, p0/Z, [x21]\n" "mov x12, #0x4\n" - "add z19.h, p0/M, z19.h, z5.h\n" + "add z15.h, p0/M, z15.h, z11.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z14.s }, p0/Z, [x21]\n" - "add z14.h, p0/M, z14.h, z5.h\n" + "ld1sb { z23.s }, p0/Z, [x21]\n" + "add z23.h, p0/M, z23.h, z11.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z18.s }, p0/Z, [x21]\n" - "add z18.h, p0/M, z18.h, z5.h\n" + "ld1sb { z17.s }, p0/Z, [x21]\n" + "add z17.h, p0/M, z17.h, z11.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z15.s }, p0/Z, [x21]\n" - "add z15.h, p0/M, z15.h, z5.h\n" + "ld1sb { z24.s }, p0/Z, [x21]\n" + "add z24.h, p0/M, z24.h, z11.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z17.s }, p0/Z, [x21]\n" - "add z17.h, p0/M, z17.h, z5.h\n" + "ld1sb { z4.s }, p0/Z, [x21]\n" + "add z4.h, p0/M, z4.h, z11.h\n" "mov x12, #0x8\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "trn1 z13.h, z13.h, z19.h\n" + "trn1 z21.h, z21.h, z14.h\n" + "trn1 z22.h, z22.h, z15.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "addvl x20, SP, #2\n" - "ld1sb { z16.s }, p0/Z, [x21]\n" - "trn1 z14.h, z14.h, z18.h\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" + "ld1sb { z2.s }, p0/Z, [x21]\n" + "trn1 z23.h, z23.h, z17.h\n" + "trn1 z24.h, z24.h, z4.h\n" + ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" "mov x12, #0x0\n" - ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" + ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" - "add z16.h, p0/M, z16.h, z5.h\n" + "add z2.h, p0/M, z2.h, z11.h\n" "add x14, x14, %x[ld_in_col]\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" + ".inst 0xc17016a8 // sdot za.s[x8, 0], { z21.h-z24.h }, z0.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z12.s }, p0/Z, [x14]\n" - "add z12.h, p0/M, z12.h, z5.h\n" + "ld1sb { z21.s }, p0/Z, [x14]\n" + "add z21.h, p0/M, z21.h, z11.h\n" "add x20, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "mov z16.d, z16.d\n" + "mov z25.d, z2.d\n" "ld1sb { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z5.h\n" + "add z20.h, p0/M, z20.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" - "ld1sb { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z5.h\n" + ".inst 0xc17116c8 // sdot za.s[x8, 0], { z22.h-z25.h }, z1.h\n" + "ld1sb { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z19.s }, p0/Z, [x20]\n" + "ld1sb { z4.s }, p0/Z, [x20]\n" "mov x12, #0x4\n" - "add z19.h, p0/M, z19.h, z5.h\n" + "add z4.h, p0/M, z4.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc0040c02 // mova za.d[x8, #2], { z0.d-z3.d }\n" + ".inst 0xc0040f82 // mova za.d[x8, #2], { z28.d-z31.d }\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z5.h\n" + "ld1sb { z23.s }, p0/Z, [x20]\n" + "add z23.h, p0/M, z23.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z5.h\n" + "ld1sb { z27.s }, p0/Z, [x20]\n" + "add z27.h, p0/M, z27.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z15.s }, p0/Z, [x20]\n" - ".inst 0xc1a8ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z8.s\n" - "add z15.h, p0/M, z15.h, z5.h\n" + "ld1sb { z24.s }, p0/Z, [x20]\n" + ".inst 0xc1a6ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n" + "add z24.h, p0/M, z24.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z17.s }, p0/Z, [x20]\n" + "ld1sb { z12.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - ".inst 0xc1a7aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z7.s\n" + ".inst 0xc1a9aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z9.s\n" "add x20, x20, %x[ld_in_row]\n" - "add z17.h, p0/M, z17.h, z5.h\n" + "add z12.h, p0/M, z12.h, z11.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z5.h\n" - ".inst 0xc1a4ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" + "ld1sb { z8.s }, p0/Z, [x20]\n" + "add z8.h, p0/M, z8.h, z11.h\n" + ".inst 0xc1adab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z13.s\n" "subs x22, x22, #0x1\n" - ".inst 0xa0402bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP]\n" - ".inst 0xc1b7cf7c // sclamp { z28.s-z31.s }, z27.s, z23.s\n" - "st1b { z28.s }, p1, [x11]\n" + ".inst 0xa0402bee // ld1h { z14.h-z15.h }, pn10.b/Z, [SP]\n" + ".inst 0xc1a7cd50 // sclamp { z16.s-z19.s }, z10.s, z7.s\n" + "st1b { z16.s }, p1, [x11]\n" "add x11, x11, x9\n" - "trn1 z12.h, z12.h, z20.h\n" - "st1b { z29.s }, p1, [x10]\n" + "trn1 z21.h, z21.h, z20.h\n" + "st1b { z17.s }, p1, [x10]\n" "add x10, x10, x28\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" - "st1b { z30.s }, p1, [x27]\n" + "trn1 z22.h, z22.h, z4.h\n" + "trn1 z23.h, z23.h, z27.h\n" + "st1b { z18.s }, p1, [x27]\n" "add x27, x27, x25\n" - "trn1 z15.h, z15.h, z17.h\n" - "mov z16.d, z16.d\n" - "st1b { z31.s }, p1, [x26]\n" + "trn1 z24.h, z24.h, z12.h\n" + "mov z25.d, z8.d\n" + "st1b { z19.s }, p1, [x26]\n" "add x26, x26, x24\n" "add x14, x14, %x[ld_in_col]\n" "bgt 16b\n" "17:" // Main loop tail - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" + ".inst 0xc17e16a8 // sdot za.s[x8, 0], { z21.h-z24.h }, z14.h\n" "addvl x20, SP, #4\n" "mov x12, #0x0\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" + ".inst 0xc17f16c8 // sdot za.s[x8, 0], { z22.h-z25.h }, z15.h\n" + ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "add x20, x14, %x[ld_in_row]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - "ld1sb { z12.s }, p0/Z, [x14]\n" - "add z12.h, p0/M, z12.h, z5.h\n" + ".inst 0xc17016a9 // sdot za.s[x8, 1], { z21.h-z24.h }, z0.h\n" + "ld1sb { z0.s }, p0/Z, [x14]\n" + "add z0.h, p0/M, z0.h, z11.h\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z5.h\n" + "ld1sb { z14.s }, p0/Z, [x20]\n" + "add z14.h, p0/M, z14.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc17b15a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z11.h\n" - "ld1sb { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z5.h\n" + ".inst 0xc17116c9 // sdot za.s[x8, 1], { z22.h-z25.h }, z1.h\n" + "ld1sb { z1.s }, p0/Z, [x20]\n" + "add z1.h, p0/M, z1.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z19.s }, p0/Z, [x20]\n" + "ld1sb { z12.s }, p0/Z, [x20]\n" "mov x12, #0x4\n" - "add z19.h, p0/M, z19.h, z5.h\n" + "add z12.h, p0/M, z12.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z5.h\n" + "ld1sb { z2.s }, p0/Z, [x20]\n" + "add z2.h, p0/M, z2.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z5.h\n" + "ld1sb { z21.s }, p0/Z, [x20]\n" + "add z21.h, p0/M, z21.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z15.s }, p0/Z, [x20]\n" - ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" - "add z15.h, p0/M, z15.h, z5.h\n" + "ld1sb { z3.s }, p0/Z, [x20]\n" + ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" + "add z3.h, p0/M, z3.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z17.s }, p0/Z, [x20]\n" + "ld1sb { z25.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - ".inst 0xc1a8ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z8.s\n" + ".inst 0xc1a6ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n" "add x20, x20, %x[ld_in_row]\n" - "add z17.h, p0/M, z17.h, z5.h\n" + "add z25.h, p0/M, z25.h, z11.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z16.s }, p0/Z, [x20]\n" + "ld1sb { z27.s }, p0/Z, [x20]\n" "addvl x20, SP, #2\n" - ".inst 0xc1a7aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z7.s\n" - "trn1 z12.h, z12.h, z20.h\n" + ".inst 0xc1a9aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z9.s\n" + "trn1 z0.h, z0.h, z14.h\n" "add x8, x8, #0x1\n" - "add z16.h, p0/M, z16.h, z5.h\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" + "add z27.h, p0/M, z27.h, z11.h\n" + "trn1 z1.h, z1.h, z12.h\n" + "trn1 z2.h, z2.h, z21.h\n" "add x14, x14, %x[ld_in_col]\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1a4ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" - "mov z16.d, z16.d\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xc1b7cf7c // sclamp { z28.s-z31.s }, z27.s, z23.s\n" - "st1b { z28.s }, p1, [x11]\n" + "trn1 z3.h, z3.h, z25.h\n" + ".inst 0xa0402a8e // ld1h { z14.h-z15.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1adab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z13.s\n" + "mov z4.d, z27.d\n" + ".inst 0xc17e1408 // sdot za.s[x8, 0], { z0.h-z3.h }, z14.h\n" + ".inst 0xc1a7cd50 // sclamp { z16.s-z19.s }, z10.s, z7.s\n" + "st1b { z16.s }, p1, [x11]\n" "add x11, x11, x9\n" - ".inst 0xc0040c02 // mova za.d[x8, #2], { z0.d-z3.d }\n" - "st1b { z29.s }, p1, [x10]\n" + ".inst 0xc0040f82 // mova za.d[x8, #2], { z28.d-z31.d }\n" + "st1b { z17.s }, p1, [x10]\n" "add x10, x10, x28\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" - ".inst 0xa0402bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP]\n" - "st1b { z30.s }, p1, [x27]\n" + ".inst 0xc17f1428 // sdot za.s[x8, 0], { z1.h-z4.h }, z15.h\n" + ".inst 0xa0402bee // ld1h { z14.h-z15.h }, pn10.b/Z, [SP]\n" + "st1b { z18.s }, p1, [x27]\n" "add x27, x27, x25\n" - "st1b { z31.s }, p1, [x26]\n" + "st1b { z19.s }, p1, [x26]\n" "add x26, x26, x24\n" "18:" // Main loop skip tail "cbz x15, 19f\n" // Skip remainder inputs "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z12.s }, p0/Z, [x14]\n" - "add z12.h, p0/M, z12.h, z5.h\n" + "ld1sb { z21.s }, p0/Z, [x14]\n" + "add z21.h, p0/M, z21.h, z11.h\n" "add x20, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z5.h\n" + "ld1sb { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z5.h\n" + "ld1sb { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z5.h\n" + "ld1sb { z0.s }, p0/Z, [x20]\n" + "add z0.h, p0/M, z0.h, z11.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "trn1 z13.h, z13.h, z19.h\n" + "trn1 z21.h, z21.h, z17.h\n" + "trn1 z22.h, z22.h, z0.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z5.h\n" + "ld1sb { z23.s }, p0/Z, [x20]\n" + "add z23.h, p0/M, z23.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z5.h\n" + "ld1sb { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z5.h\n" + "ld1sb { z24.s }, p0/Z, [x20]\n" + "add z24.h, p0/M, z24.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z17.s }, p0/Z, [x20]\n" + "ld1sb { z5.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z17.h, p0/M, z17.h, z5.h\n" + "add z5.h, p0/M, z5.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z5.h\n" - "trn1 z14.h, z14.h, z18.h\n" - "trn1 z15.h, z15.h, z17.h\n" - "mov z16.d, z16.d\n" + "ld1sb { z4.s }, p0/Z, [x20]\n" + "add z4.h, p0/M, z4.h, z11.h\n" + "trn1 z23.h, z23.h, z17.h\n" + "trn1 z24.h, z24.h, z5.h\n" + "mov z25.d, z4.d\n" "addvl x20, SP, #4\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" + ".inst 0xc17e16a8 // sdot za.s[x8, 0], { z21.h-z24.h }, z14.h\n" "sub x13, x13, #0x1\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" - ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" - ".inst 0xc1a8ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z8.s\n" - ".inst 0xc1a7aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z7.s\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xc1a4ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" - ".inst 0xc17b15a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z11.h\n" + ".inst 0xc17f16c8 // sdot za.s[x8, 0], { z22.h-z25.h }, z15.h\n" + ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" + ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" + ".inst 0xc1a6ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n" + ".inst 0xc1a9aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z9.s\n" + ".inst 0xc17016a9 // sdot za.s[x8, 1], { z21.h-z24.h }, z0.h\n" + ".inst 0xc1adab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z13.s\n" + ".inst 0xc17116c9 // sdot za.s[x8, 1], { z22.h-z25.h }, z1.h\n" "add x8, x8, #0x1\n" - ".inst 0xc1b7cf7c // sclamp { z28.s-z31.s }, z27.s, z23.s\n" - "st1b { z28.s }, p1, [x11]\n" + ".inst 0xc1a7cd50 // sclamp { z16.s-z19.s }, z10.s, z7.s\n" + "st1b { z16.s }, p1, [x11]\n" "add x11, x11, x9\n" - ".inst 0xc0040c02 // mova za.d[x8, #2], { z0.d-z3.d }\n" - "st1b { z29.s }, p1, [x10]\n" + ".inst 0xc0040f82 // mova za.d[x8, #2], { z28.d-z31.d }\n" + "st1b { z17.s }, p1, [x10]\n" "add x10, x10, x28\n" - "st1b { z30.s }, p1, [x27]\n" + "st1b { z18.s }, p1, [x27]\n" "add x27, x27, x25\n" - "st1b { z31.s }, p1, [x26]\n" + "st1b { z19.s }, p1, [x26]\n" "add x26, x26, x24\n" "19:" // Tail input: End "cbz x13, 21f\n" "20:" // Right padding loop - ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" - ".inst 0xc1a8ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z8.s\n" + ".inst 0xc0060c00 // mova { z0.d-z3.d }, za.d[x8, #0]\n" + ".inst 0xc1a6ac00 // sqdmulh { z0.s-z3.s }, { z0.s-z3.s }, z6.s\n" "add x8, x8, #0x1\n" - ".inst 0xc1a7aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z7.s\n" + ".inst 0xc1a9aa20 // srshl { z0.s-z3.s }, { z0.s-z3.s }, z9.s\n" "subs x13, x13, #0x1\n" - ".inst 0xc0040c02 // mova za.d[x8, #2], { z0.d-z3.d }\n" - ".inst 0xc1a4ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" - ".inst 0xc1b7cf7c // sclamp { z28.s-z31.s }, z27.s, z23.s\n" - "st1b { z28.s }, p1, [x11]\n" + ".inst 0xc0040f82 // mova za.d[x8, #2], { z28.d-z31.d }\n" + ".inst 0xc1adab00 // add { z0.s-z3.s }, { z0.s-z3.s }, z13.s\n" + ".inst 0xc1a7cd40 // sclamp { z0.s-z3.s }, z10.s, z7.s\n" + "st1b { z0.s }, p1, [x11]\n" "add x11, x11, x9\n" - "st1b { z29.s }, p1, [x10]\n" + "st1b { z1.s }, p1, [x10]\n" "add x10, x10, x28\n" - "st1b { z30.s }, p1, [x27]\n" + "st1b { z2.s }, p1, [x27]\n" "add x27, x27, x25\n" - "st1b { z31.s }, p1, [x26]\n" + "st1b { z3.s }, p1, [x26]\n" "add x26, x26, x24\n" "bgt 20b\n" "21:" // End - "ldr x22, [%x[args], %[offsetof_Args_weights]]\n" - "incw x22, ALL, MUL #9\n" - "str x22, [%x[args], %[offsetof_Args_weights]]\n" + "ldr x20, [%x[args], %[offsetof_Args_weights]]\n" + "incw x20, ALL, MUL #9\n" + "str x20, [%x[args], %[offsetof_Args_weights]]\n" "incw x16\n" - "ldr x20, [%x[args], %[offsetof_Args_ld_in_vl]]\n" + "ldr x21, [%x[args], %[offsetof_Args_ld_in_vl]]\n" "whilelt p1.s, x16, x17\n" - "ldr x14, [%x[args], %[offsetof_Args_inptr]]\n" - "add x14, x14, x20\n" - "str x14, [%x[args], %[offsetof_Args_inptr]]\n" + "ldr x20, [%x[args], %[offsetof_Args_inptr]]\n" + "add x20, x20, x21\n" + "str x20, [%x[args], %[offsetof_Args_inptr]]\n" "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" "ldr x24, [%x[args], %[offsetof_Args_ld_out_vls]]\n" "ldp x23, x22, [x25, #0x0]\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_5x5_s1_4rows_dot_za.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_5x5_s1_4rows_dot_za.hpp index 6f3290fd3c0fdf7ecd230eff8afd15df616b50a4..40fa7182669f609edd1fdd157afd489cb499b93a 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_5x5_s1_4rows_dot_za.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_5x5_s1_4rows_dot_za.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,8 @@ * SOFTWARE. */ +#include "src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp" + namespace arm_conv { namespace depthwise { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_5x5_s1_4rows_dot_za/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_5x5_s1_4rows_dot_za/generic.cpp index d366b3c8d50f6aee71169185274a39fc2e46d313..bb68733a456dcadb5e3ff125c60a33f2b289f487 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_5x5_s1_4rows_dot_za/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_5x5_s1_4rows_dot_za/generic.cpp @@ -69,196 +69,196 @@ void sme2_s8q_planar_5x5_s1_4rows_dot_za_impl( __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" - "ldr x4, [%x[args], %[offsetof_Args_pad_bottom]]\n" + "ldr x5, [%x[args], %[offsetof_Args_pad_bottom]]\n" "ptrue p2.b\n" "mov x20, #0x8\n" "ldr x6, [%x[args], %[offsetof_Args_pad_top]]\n" - "ld1rh { z25.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_a_offset]]\n" - "sub x20, x20, x4\n" + "ld1rh { z17.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_a_offset]]\n" + "sub x20, x20, x5\n" ".inst 0x25207812 // ptrue pn10.b\n" "ldr x7, [%x[args], %[offsetof_Args_n_channels]]\n" "whilelt p1.s, XZR, x7\n" "whilelt p9.s, XZR, x20\n" - "ld1rw { z9.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" + "ld1rw { z12.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" "whilelt p8.s, XZR, x6\n" "addvl SP, SP, #-30\n" - "ldr x5, [%x[args], %[offsetof_Args_current_channel]]\n" - "neg z25.h, p2/M, z25.h\n" + "ldr x17, [%x[args], %[offsetof_Args_current_channel]]\n" + "neg z17.h, p2/M, z17.h\n" "eor p8.b, p2/Z, p8.b, p9.b\n" - "ld1rw { z3.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_mul]]\n" - "ld1rw { z1.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z7.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z4.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_right_shift]]\n" "ld1rw { z24.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" - "ld1rw { z31.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" + "ld1rw { z16.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" "1:" // Channel loop "ldr x20, [%x[qp], %[offsetof_Requantize32_bias]]\n" - "mov z6.s, #0x0\n" + "mov z18.s, #0x0\n" "cbz x20, 2f\n" - "ld1w { z6.s }, p1/Z, [x20, x5, LSL #2]\n" + "ld1w { z18.s }, p1/Z, [x20, x17, LSL #2]\n" "2:" // Load bias: Done "ldr x23, [%x[args], %[offsetof_Args_weights]]\n" - "mov x22, x23\n" - "ld1sb { z18.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "ld1rh { z12.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" - "mov z2.h, #0x0\n" - "sub z18.h, z18.h, z12.h\n" + "mov x20, x23\n" + "ld1sb { z2.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "ld1rh { z3.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" + "mov z15.h, #0x0\n" + "sub z2.h, z2.h, z3.h\n" "incw x23\n" - "ld1sb { z17.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "sub z17.h, z17.h, z12.h\n" - "trn1 z0.h, z2.h, z18.h\n" - "ld1sb { z21.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "sub z21.h, z21.h, z12.h\n" - "trn1 z8.h, z18.h, z17.h\n" - "ld1sb { z16.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "sub z16.h, z16.h, z12.h\n" - "trn1 z4.h, z17.h, z21.h\n" - "ld1sb { z15.s }, p2/Z, [x22]\n" - "sub z15.h, z15.h, z12.h\n" - "mov x22, x23\n" - "trn1 z5.h, z21.h, z16.h\n" - "ld1sb { z18.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "trn1 z10.h, z16.h, z15.h\n" + "ld1sb { z13.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "sub z13.h, z13.h, z3.h\n" "trn1 z11.h, z15.h, z2.h\n" - "ld1sb { z17.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "sub z18.h, z18.h, z12.h\n" - "sub z17.h, z17.h, z12.h\n" - "ld1sb { z21.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "sub z21.h, z21.h, z12.h\n" - "addvl x21, SP, #30\n" - "ld1sb { z16.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" + "ld1sb { z27.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "sub z27.h, z27.h, z3.h\n" + "trn1 z0.h, z2.h, z13.h\n" + "ld1sb { z19.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "sub z19.h, z19.h, z3.h\n" + "trn1 z26.h, z13.h, z27.h\n" + "ld1sb { z14.s }, p2/Z, [x20]\n" + "sub z14.h, z14.h, z3.h\n" + "mov x20, x23\n" + "trn1 z10.h, z27.h, z19.h\n" + "ld1sb { z9.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "trn1 z19.h, z19.h, z14.h\n" + "trn1 z1.h, z14.h, z15.h\n" + "ld1sb { z5.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "sub z9.h, z9.h, z3.h\n" + "sub z5.h, z5.h, z3.h\n" + "ld1sb { z29.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "sub z29.h, z29.h, z3.h\n" + "addvl x22, SP, #30\n" + "ld1sb { z2.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" "incw x23\n" - "sub z16.h, z16.h, z12.h\n" - "ld1sb { z15.s }, p2/Z, [x22]\n" - "addvl x21, x21, #-6\n" - "sub z15.h, z15.h, z12.h\n" - "mov x22, x23\n" - "st1h { z0.h }, p2, [x21]\n" - "trn1 z0.h, z2.h, z18.h\n" + "sub z2.h, z2.h, z3.h\n" + "ld1sb { z23.s }, p2/Z, [x20]\n" + "addvl x22, x22, #-6\n" + "sub z23.h, z23.h, z3.h\n" + "mov x20, x23\n" + "st1h { z11.h }, p2, [x22]\n" + "trn1 z20.h, z15.h, z9.h\n" "incw x23\n" - "ldr x20, [%x[qp], %[offsetof_Requantize32_per_channel_muls]]\n" - "st1h { z8.h }, p2, [x21, #1, MUL VL]\n" - "trn1 z8.h, z18.h, z17.h\n" - "ld1sb { z18.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z4.h }, p2, [x21, #2, MUL VL]\n" - "trn1 z4.h, z17.h, z21.h\n" - "ld1sb { z17.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z5.h }, p2, [x21, #3, MUL VL]\n" - "trn1 z5.h, z21.h, z16.h\n" - "ld1sb { z21.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z10.h }, p2, [x21, #4, MUL VL]\n" - "trn1 z10.h, z16.h, z15.h\n" - "ld1sb { z16.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z11.h }, p2, [x21, #5, MUL VL]\n" - "trn1 z11.h, z15.h, z2.h\n" - "sub z18.h, z18.h, z12.h\n" - "addvl x21, x21, #-6\n" - "sub z17.h, z17.h, z12.h\n" - "ld1sb { z15.s }, p2/Z, [x22]\n" - "sub z21.h, z21.h, z12.h\n" - "mov x22, x23\n" - "sub z16.h, z16.h, z12.h\n" - "sub z15.h, z15.h, z12.h\n" - "st1h { z0.h }, p2, [x21]\n" + "ldr x21, [%x[qp], %[offsetof_Requantize32_per_channel_muls]]\n" + "st1h { z0.h }, p2, [x22, #1, MUL VL]\n" + "trn1 z22.h, z9.h, z5.h\n" + "ld1sb { z25.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z26.h }, p2, [x22, #2, MUL VL]\n" + "trn1 z9.h, z5.h, z29.h\n" + "ld1sb { z21.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z10.h }, p2, [x22, #3, MUL VL]\n" + "trn1 z26.h, z29.h, z2.h\n" + "ld1sb { z0.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z19.h }, p2, [x22, #4, MUL VL]\n" + "trn1 z28.h, z2.h, z23.h\n" + "ld1sb { z19.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z1.h }, p2, [x22, #5, MUL VL]\n" + "trn1 z2.h, z23.h, z15.h\n" + "sub z25.h, z25.h, z3.h\n" + "addvl x22, x22, #-6\n" + "sub z21.h, z21.h, z3.h\n" + "ld1sb { z6.s }, p2/Z, [x20]\n" + "sub z0.h, z0.h, z3.h\n" + "mov x20, x23\n" + "sub z19.h, z19.h, z3.h\n" + "sub z6.h, z6.h, z3.h\n" + "st1h { z20.h }, p2, [x22]\n" "incw x23\n" - "st1h { z8.h }, p2, [x21, #1, MUL VL]\n" - "trn1 z0.h, z2.h, z18.h\n" - "trn1 z8.h, z18.h, z17.h\n" - "ld1sb { z18.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z4.h }, p2, [x21, #2, MUL VL]\n" - "trn1 z4.h, z17.h, z21.h\n" - "ld1sb { z17.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z5.h }, p2, [x21, #3, MUL VL]\n" - "trn1 z5.h, z21.h, z16.h\n" - "ld1sb { z21.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z10.h }, p2, [x21, #4, MUL VL]\n" - "trn1 z10.h, z16.h, z15.h\n" - "ld1sb { z16.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z11.h }, p2, [x21, #5, MUL VL]\n" - "trn1 z11.h, z15.h, z2.h\n" - "sub z18.h, z18.h, z12.h\n" - "sub z17.h, z17.h, z12.h\n" - "ld1sb { z15.s }, p2/Z, [x22]\n" - "addvl x21, x21, #-6\n" - "sub z21.h, z21.h, z12.h\n" - "sub z16.h, z16.h, z12.h\n" - "mov x22, x23\n" - "st1h { z0.h }, p2, [x21]\n" - "sub z15.h, z15.h, z12.h\n" - "st1h { z8.h }, p2, [x21, #1, MUL VL]\n" - "trn1 z0.h, z2.h, z18.h\n" - "trn1 z8.h, z18.h, z17.h\n" - "ld1sb { z18.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z4.h }, p2, [x21, #2, MUL VL]\n" - "trn1 z4.h, z17.h, z21.h\n" - "ld1sb { z17.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z5.h }, p2, [x21, #3, MUL VL]\n" - "trn1 z5.h, z21.h, z16.h\n" - "ld1sb { z21.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z10.h }, p2, [x21, #4, MUL VL]\n" - "trn1 z10.h, z16.h, z15.h\n" - "ld1sb { z16.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z11.h }, p2, [x21, #5, MUL VL]\n" - "trn1 z11.h, z15.h, z2.h\n" - "ld1sb { z15.s }, p2/Z, [x22]\n" - "sub z18.h, z18.h, z12.h\n" - "addvl x21, x21, #-6\n" - "sub z17.h, z17.h, z12.h\n" - "sub z21.h, z21.h, z12.h\n" - "st1h { z0.h }, p2, [x21]\n" - "sub z16.h, z16.h, z12.h\n" - "sub z15.h, z15.h, z12.h\n" - "st1h { z8.h }, p2, [x21, #1, MUL VL]\n" - "st1h { z4.h }, p2, [x21, #2, MUL VL]\n" - "mov z7.d, z6.d\n" - "trn1 z0.h, z2.h, z18.h\n" - "st1h { z5.h }, p2, [x21, #3, MUL VL]\n" - "trn1 z8.h, z18.h, z17.h\n" - "trn1 z4.h, z17.h, z21.h\n" - "st1h { z10.h }, p2, [x21, #4, MUL VL]\n" - "trn1 z5.h, z21.h, z16.h\n" - "trn1 z10.h, z16.h, z15.h\n" - "st1h { z11.h }, p2, [x21, #5, MUL VL]\n" - "addvl x21, x21, #-6\n" - "trn1 z11.h, z15.h, z2.h\n" - "st1h { z0.h }, p2, [x21]\n" - "st1h { z8.h }, p2, [x21, #1, MUL VL]\n" - "st1h { z4.h }, p2, [x21, #2, MUL VL]\n" - "st1h { z5.h }, p2, [x21, #3, MUL VL]\n" - "st1h { z10.h }, p2, [x21, #4, MUL VL]\n" - "st1h { z11.h }, p2, [x21, #5, MUL VL]\n" - "cbz x20, 3f\n" - "ld1w { z3.s }, p1/Z, [x20, x5, LSL #2]\n" + "st1h { z22.h }, p2, [x22, #1, MUL VL]\n" + "trn1 z11.h, z15.h, z25.h\n" + "trn1 z10.h, z25.h, z21.h\n" + "ld1sb { z5.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z9.h }, p2, [x22, #2, MUL VL]\n" + "trn1 z14.h, z21.h, z0.h\n" + "ld1sb { z23.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z26.h }, p2, [x22, #3, MUL VL]\n" + "trn1 z21.h, z0.h, z19.h\n" + "ld1sb { z27.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z28.h }, p2, [x22, #4, MUL VL]\n" + "trn1 z19.h, z19.h, z6.h\n" + "ld1sb { z29.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z2.h }, p2, [x22, #5, MUL VL]\n" + "trn1 z13.h, z6.h, z15.h\n" + "sub z5.h, z5.h, z3.h\n" + "sub z23.h, z23.h, z3.h\n" + "ld1sb { z1.s }, p2/Z, [x20]\n" + "addvl x22, x22, #-6\n" + "sub z27.h, z27.h, z3.h\n" + "sub z29.h, z29.h, z3.h\n" + "mov x20, x23\n" + "st1h { z11.h }, p2, [x22]\n" + "sub z1.h, z1.h, z3.h\n" + "st1h { z10.h }, p2, [x22, #1, MUL VL]\n" + "trn1 z30.h, z15.h, z5.h\n" + "trn1 z26.h, z5.h, z23.h\n" + "ld1sb { z11.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z14.h }, p2, [x22, #2, MUL VL]\n" + "trn1 z22.h, z23.h, z27.h\n" + "ld1sb { z5.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z21.h }, p2, [x22, #3, MUL VL]\n" + "trn1 z28.h, z27.h, z29.h\n" + "ld1sb { z8.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z19.h }, p2, [x22, #4, MUL VL]\n" + "trn1 z27.h, z29.h, z1.h\n" + "ld1sb { z9.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z13.h }, p2, [x22, #5, MUL VL]\n" + "trn1 z2.h, z1.h, z15.h\n" + "ld1sb { z14.s }, p2/Z, [x20]\n" + "sub z11.h, z11.h, z3.h\n" + "addvl x22, x22, #-6\n" + "sub z5.h, z5.h, z3.h\n" + "sub z8.h, z8.h, z3.h\n" + "st1h { z30.h }, p2, [x22]\n" + "sub z9.h, z9.h, z3.h\n" + "sub z14.h, z14.h, z3.h\n" + "st1h { z26.h }, p2, [x22, #1, MUL VL]\n" + "st1h { z22.h }, p2, [x22, #2, MUL VL]\n" + "mov z19.d, z18.d\n" + "trn1 z22.h, z15.h, z11.h\n" + "st1h { z28.h }, p2, [x22, #3, MUL VL]\n" + "trn1 z1.h, z11.h, z5.h\n" + "trn1 z31.h, z5.h, z8.h\n" + "st1h { z27.h }, p2, [x22, #4, MUL VL]\n" + "trn1 z8.h, z8.h, z9.h\n" + "trn1 z21.h, z9.h, z14.h\n" + "st1h { z2.h }, p2, [x22, #5, MUL VL]\n" + "addvl x22, x22, #-6\n" + "trn1 z15.h, z14.h, z15.h\n" + "st1h { z22.h }, p2, [x22]\n" + "st1h { z1.h }, p2, [x22, #1, MUL VL]\n" + "st1h { z31.h }, p2, [x22, #2, MUL VL]\n" + "st1h { z8.h }, p2, [x22, #3, MUL VL]\n" + "st1h { z21.h }, p2, [x22, #4, MUL VL]\n" + "st1h { z15.h }, p2, [x22, #5, MUL VL]\n" + "cbz x21, 3f\n" + "ld1w { z7.s }, p1/Z, [x21, x17, LSL #2]\n" "3:" // Load mul: End "ldr x20, [%x[qp], %[offsetof_Requantize32_per_channel_right_shifts]]\n" "cbz x20, 4f\n" - "ld1w { z1.s }, p1/Z, [x20, x5, LSL #2]\n" + "ld1w { z4.s }, p1/Z, [x20, x17, LSL #2]\n" "4:" // Load right_shift: End - "ldr x17, [%x[args], %[offsetof_Args_input_cols]]\n" - "sub x20, x17, #0x1\n" + "ldr x25, [%x[args], %[offsetof_Args_input_cols]]\n" + "sub x20, x25, #0x1\n" "orr x23, x20, %x[ld_in_col], LSL #16\n" "ldr x16, [%x[args], %[offsetof_Args_inptr]]\n" "orr x23, x7, x23, LSL #22\n" "mov x22, #0x8\n" - "add x21, x6, x4\n" + "add x21, x6, x5\n" "lsl x20, %x[ld_in_row], #0x0\n" "ldr x15, [%x[args], %[offsetof_Args_output_cols]]\n" "mov x11, #0x0\n" @@ -271,56 +271,56 @@ void sme2_s8q_planar_5x5_s1_4rows_dot_za_impl( ".inst 0xf8b74a9c // rprfm pldstrm, x23, [x20]\n" "add x20, x20, %x[ld_in_col]\n" "bgt 5b\n" - "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" + "ldr x23, [%x[args], %[offsetof_Args_outptrs]]\n" "lsl x20, %x[ld_in_row], #0x0\n" "msub x16, x6, x20, x16\n" - ".inst 0xc00468c0 // mova za.d[x11, #0], { z6.d-z7.d }\n" + ".inst 0xc0046a40 // mova za.d[x11, #0], { z18.d-z19.d }\n" "ldr x20, [%x[args], %[offsetof_Args_ld_out_cols]]\n" - ".inst 0xc00468c1 // mova za.d[x11, #1], { z6.d-z7.d }\n" + ".inst 0xc0046a41 // mova za.d[x11, #1], { z18.d-z19.d }\n" "mov x22, #0x4\n" - "ldp x14, x13, [x25], #0x10\n" - ".inst 0xc00468c2 // mova za.d[x11, #2], { z6.d-z7.d }\n" - "ldp x3, x10, [x20], #0x10\n" - ".inst 0xc00468c3 // mova za.d[x11, #3], { z6.d-z7.d }\n" + "ldp x14, x13, [x23], #0x10\n" + ".inst 0xc0046a42 // mova za.d[x11, #2], { z18.d-z19.d }\n" + "ldp x4, x10, [x20], #0x10\n" + ".inst 0xc0046a43 // mova za.d[x11, #3], { z18.d-z19.d }\n" "ldr x21, [%x[args], %[offsetof_Args_pad_left]]\n" - ".inst 0xc00468c4 // mova za.d[x11, #4], { z6.d-z7.d }\n" - "ldp x9, x28, [x25], #0x10\n" - ".inst 0xc00468c5 // mova za.d[x11, #5], { z6.d-z7.d }\n" + ".inst 0xc0046a44 // mova za.d[x11, #4], { z18.d-z19.d }\n" + "ldp x9, x28, [x23], #0x10\n" + ".inst 0xc0046a45 // mova za.d[x11, #5], { z18.d-z19.d }\n" "ldp x27, x26, [x20], #0x10\n" - ".inst 0xc00468c6 // mova za.d[x11, #6], { z6.d-z7.d }\n" - ".inst 0xc00468c7 // mova za.d[x11, #7], { z6.d-z7.d }\n" - ".inst 0xc00408c0 // mova za.d[x8, #0], { z6.d-z7.d }\n" - ".inst 0xc00408c1 // mova za.d[x8, #1], { z6.d-z7.d }\n" + ".inst 0xc0046a46 // mova za.d[x11, #6], { z18.d-z19.d }\n" + ".inst 0xc0046a47 // mova za.d[x11, #7], { z18.d-z19.d }\n" + ".inst 0xc0040a40 // mova za.d[x8, #0], { z18.d-z19.d }\n" + ".inst 0xc0040a41 // mova za.d[x8, #1], { z18.d-z19.d }\n" "cbz x21, 7f\n" "cmp x21, x22\n" "csel x20, x21, x22, LT\n" "sub x21, x21, x20\n" "sub x22, x22, x20\n" "cbz x21, 7f\n" - ".inst 0xc006680c // mova { z12.d-z13.d }, za.d[x11, #0]\n" + ".inst 0xc0066814 // mova { z20.d-z21.d }, za.d[x11, #0]\n" "sub x15, x15, x21\n" - ".inst 0xc006682e // mova { z14.d-z15.d }, za.d[x11, #1]\n" - ".inst 0xc1a3ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z3.s\n" - ".inst 0xc1a1aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n" - ".inst 0xc1a9ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z9.s\n" - ".inst 0xc1bfcf0c // sclamp { z12.s-z15.s }, z24.s, z31.s\n" + ".inst 0xc0066836 // mova { z22.d-z23.d }, za.d[x11, #1]\n" + ".inst 0xc1a7ac14 // sqdmulh { z20.s-z23.s }, { z20.s-z23.s }, z7.s\n" + ".inst 0xc1a4aa34 // srshl { z20.s-z23.s }, { z20.s-z23.s }, z4.s\n" + ".inst 0xc1acab14 // add { z20.s-z23.s }, { z20.s-z23.s }, z12.s\n" + ".inst 0xc1b0cf14 // sclamp { z20.s-z23.s }, z24.s, z16.s\n" "6:" // Left padding "subs x21, x21, #0x1\n" - "st1b { z12.s }, p1, [x14]\n" - "add x14, x14, x3\n" - "st1b { z14.s }, p1, [x13]\n" + "st1b { z20.s }, p1, [x14]\n" + "add x14, x14, x4\n" + "st1b { z22.s }, p1, [x13]\n" "add x13, x13, x10\n" - "st1b { z13.s }, p1, [x9]\n" + "st1b { z21.s }, p1, [x9]\n" "add x9, x9, x27\n" - "st1b { z15.s }, p1, [x28]\n" + "st1b { z23.s }, p1, [x28]\n" "add x28, x28, x26\n" "bgt 6b\n" "7:" // Left padding: End - "adds XZR, x6, x4\n" + "adds XZR, x6, x5\n" "bne 14f\n" "cbz x22, 12f\n" "cmp x22, #0x1\n" - "sub x17, x17, x22\n" + "sub x25, x25, x22\n" "beq 11f\n" "cmp x22, #0x2\n" "beq 10f\n" @@ -328,338 +328,338 @@ void sme2_s8q_planar_5x5_s1_4rows_dot_za_impl( "beq 9f\n" "8:" // Unpadded: 4 priming loads "add x21, x16, %x[ld_in_row]\n" - "ld1sb { z17.s }, p1/Z, [x16]\n" + "ld1sb { z1.s }, p1/Z, [x16]\n" "addvl x20, SP, #24\n" - "ld1sb { z16.s }, p1/Z, [x21]\n" + "ld1sb { z28.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z27.h, z17.h, z16.h\n" - "add z27.h, z27.h, z25.h\n" - "ld1sb { z17.s }, p1/Z, [x21]\n" + "trn1 z27.h, z1.h, z28.h\n" + "add z27.h, z27.h, z17.h\n" + "ld1sb { z1.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "add x16, x16, %x[ld_in_col]\n" - "ld1sb { z16.s }, p1/Z, [x21]\n" + "ld1sb { z2.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z28.h, z17.h, z16.h\n" - "add z28.h, z28.h, z25.h\n" - "ld1sb { z16.s }, p1/Z, [x21]\n" + "trn1 z28.h, z1.h, z2.h\n" + "add z28.h, z28.h, z17.h\n" + "ld1sb { z13.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1sb { z29.s }, p1/Z, [x21]\n" + "ld1sb { z6.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z29.h, z16.h, z29.h\n" - "add z29.h, z29.h, z25.h\n" - "ld1sb { z17.s }, p1/Z, [x21]\n" + "trn1 z29.h, z13.h, z6.h\n" + "add z29.h, z29.h, z17.h\n" + "ld1sb { z30.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" - "ld1sb { z16.s }, p1/Z, [x21]\n" - "trn1 z30.h, z17.h, z16.h\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - "add z30.h, z30.h, z25.h\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" + ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" + ".inst 0xc16a7768 // sdot za.s[x11, 0], { z27.h-z28.h }, z10.h\n" + "ld1sb { z20.s }, p1/Z, [x21]\n" + "trn1 z30.h, z30.h, z20.h\n" + ".inst 0xc1627769 // sdot za.s[x11, 1], { z27.h-z28.h }, z2.h\n" + ".inst 0xa1412a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + "add z30.h, z30.h, z17.h\n" + ".inst 0xc1697788 // sdot za.s[x11, 0], { z28.h-z29.h }, z9.h\n" + ".inst 0xc1617789 // sdot za.s[x11, 1], { z28.h-z29.h }, z1.h\n" ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" "9:" // Unpadded: 3 priming loads "add x22, x16, %x[ld_in_row]\n" - "ld1sb { z17.s }, p1/Z, [x16]\n" + "ld1sb { z2.s }, p1/Z, [x16]\n" "addvl x21, SP, #18\n" - "ld1sb { z16.s }, p1/Z, [x22]\n" + "ld1sb { z28.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z27.h, z17.h, z16.h\n" - "add z27.h, z27.h, z25.h\n" - "ld1sb { z17.s }, p1/Z, [x22]\n" + "trn1 z20.h, z2.h, z28.h\n" + "add z20.h, z20.h, z17.h\n" + "ld1sb { z31.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "addvl x20, SP, #24\n" - "ld1sb { z16.s }, p1/Z, [x22]\n" + "ld1sb { z11.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z28.h, z17.h, z16.h\n" - "add z28.h, z28.h, z25.h\n" - "ld1sb { z17.s }, p1/Z, [x22]\n" + "trn1 z21.h, z31.h, z11.h\n" + "add z21.h, z21.h, z17.h\n" + "ld1sb { z25.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "add x16, x16, %x[ld_in_col]\n" - "ld1sb { z16.s }, p1/Z, [x22]\n" + "ld1sb { z8.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z29.h, z17.h, z16.h\n" - "add z29.h, z29.h, z25.h\n" - "ld1sb { z17.s }, p1/Z, [x22]\n" + "trn1 z22.h, z25.h, z8.h\n" + "add z22.h, z22.h, z17.h\n" + "ld1sb { z8.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" - "ld1sb { z16.s }, p1/Z, [x22]\n" - "trn1 z30.h, z17.h, z16.h\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" - "add z30.h, z30.h, z25.h\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" + ".inst 0xa1402aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21]\n" + ".inst 0xc16e7688 // sdot za.s[x11, 0], { z20.h-z21.h }, z14.h\n" + "ld1sb { z3.s }, p1/Z, [x22]\n" + "trn1 z23.h, z8.h, z3.h\n" + ".inst 0xc1667689 // sdot za.s[x11, 1], { z20.h-z21.h }, z6.h\n" + ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" + ".inst 0xc161768a // sdot za.s[x11, 2], { z20.h-z21.h }, z1.h\n" + "add z23.h, z23.h, z17.h\n" + ".inst 0xa1412aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc160768b // sdot za.s[x11, 3], { z20.h-z21.h }, z0.h\n" + ".inst 0xc16976a8 // sdot za.s[x11, 0], { z21.h-z22.h }, z9.h\n" + ".inst 0xa0422aae // ld1h { z14.h-z15.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc16176a9 // sdot za.s[x11, 1], { z21.h-z22.h }, z1.h\n" + ".inst 0xa1412a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc16976aa // sdot za.s[x11, 2], { z21.h-z22.h }, z9.h\n" + ".inst 0xc16176ab // sdot za.s[x11, 3], { z21.h-z22.h }, z1.h\n" + ".inst 0xc16f76c8 // sdot za.s[x11, 0], { z22.h-z23.h }, z15.h\n" + ".inst 0xc16e76c9 // sdot za.s[x11, 1], { z22.h-z23.h }, z14.h\n" ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" + ".inst 0xc16b76ca // sdot za.s[x11, 2], { z22.h-z23.h }, z11.h\n" + ".inst 0xc16a76cb // sdot za.s[x11, 3], { z22.h-z23.h }, z10.h\n" "10:" // Unpadded: 2 priming loads "add x23, x16, %x[ld_in_row]\n" - "ld1sb { z17.s }, p1/Z, [x16]\n" + "ld1sb { z2.s }, p1/Z, [x16]\n" "addvl x22, SP, #12\n" - "ld1sb { z16.s }, p1/Z, [x23]\n" + "ld1sb { z22.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - "trn1 z27.h, z17.h, z16.h\n" - "add z27.h, z27.h, z25.h\n" - "ld1sb { z17.s }, p1/Z, [x23]\n" + "trn1 z0.h, z2.h, z22.h\n" + "add z0.h, z0.h, z17.h\n" + "ld1sb { z14.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" "addvl x21, SP, #18\n" - "ld1sb { z16.s }, p1/Z, [x23]\n" + "ld1sb { z6.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - "trn1 z28.h, z17.h, z16.h\n" - "add z28.h, z28.h, z25.h\n" - "ld1sb { z17.s }, p1/Z, [x23]\n" + "trn1 z1.h, z14.h, z6.h\n" + "add z1.h, z1.h, z17.h\n" + "ld1sb { z15.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" "addvl x20, SP, #24\n" - "ld1sb { z16.s }, p1/Z, [x23]\n" + "ld1sb { z6.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - "trn1 z29.h, z17.h, z16.h\n" - "add z29.h, z29.h, z25.h\n" - "ld1sb { z17.s }, p1/Z, [x23]\n" + "trn1 z2.h, z15.h, z6.h\n" + "add z2.h, z2.h, z17.h\n" + "ld1sb { z21.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" "add x16, x16, %x[ld_in_col]\n" - ".inst 0xa1402ac0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x22]\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" - "ld1sb { z16.s }, p1/Z, [x23]\n" - "trn1 z30.h, z17.h, z16.h\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" - "add z30.h, z30.h, z25.h\n" - ".inst 0xa0412ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xa0422aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc168776c // sdot za.s[x11, 4], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776d // sdot za.s[x11, 5], { z27.h-z28.h }, z0.h\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" + ".inst 0xa0402ace // ld1h { z14.h-z15.h }, pn10.b/Z, [x22]\n" + ".inst 0xc16f7408 // sdot za.s[x11, 0], { z0.h-z1.h }, z15.h\n" + "ld1sb { z30.s }, p1/Z, [x23]\n" + "trn1 z3.h, z21.h, z30.h\n" + ".inst 0xc16e7409 // sdot za.s[x11, 1], { z0.h-z1.h }, z14.h\n" + ".inst 0xa1402aa5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x21]\n" + ".inst 0xc16d740a // sdot za.s[x11, 2], { z0.h-z1.h }, z13.h\n" + "add z3.h, z3.h, z17.h\n" + ".inst 0xa0412ace // ld1h { z14.h-z15.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc165740b // sdot za.s[x11, 3], { z0.h-z1.h }, z5.h\n" + ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" + ".inst 0xc16f7428 // sdot za.s[x11, 0], { z1.h-z2.h }, z15.h\n" + ".inst 0xc16e7429 // sdot za.s[x11, 1], { z1.h-z2.h }, z14.h\n" + ".inst 0xa0412aae // ld1h { z14.h-z15.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xa0422ac8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc16b740c // sdot za.s[x11, 4], { z0.h-z1.h }, z11.h\n" + ".inst 0xc16a740d // sdot za.s[x11, 5], { z0.h-z1.h }, z10.h\n" + ".inst 0xc16f742a // sdot za.s[x11, 2], { z1.h-z2.h }, z15.h\n" + ".inst 0xc16e742b // sdot za.s[x11, 3], { z1.h-z2.h }, z14.h\n" + ".inst 0xa0412a8e // ld1h { z14.h-z15.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc1697448 // sdot za.s[x11, 0], { z2.h-z3.h }, z9.h\n" + ".inst 0xc1687449 // sdot za.s[x11, 1], { z2.h-z3.h }, z8.h\n" ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc165778c // sdot za.s[x11, 4], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778d // sdot za.s[x11, 5], { z28.h-z29.h }, z4.h\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc16b77ac // sdot za.s[x11, 4], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ad // sdot za.s[x11, 5], { z29.h-z30.h }, z10.h\n" + ".inst 0xc16f742c // sdot za.s[x11, 4], { z1.h-z2.h }, z15.h\n" + ".inst 0xc16e742d // sdot za.s[x11, 5], { z1.h-z2.h }, z14.h\n" + ".inst 0xc16b744a // sdot za.s[x11, 2], { z2.h-z3.h }, z11.h\n" + ".inst 0xc16a744b // sdot za.s[x11, 3], { z2.h-z3.h }, z10.h\n" + ".inst 0xa0422a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc161744c // sdot za.s[x11, 4], { z2.h-z3.h }, z1.h\n" + ".inst 0xc160744d // sdot za.s[x11, 5], { z2.h-z3.h }, z0.h\n" "11:" // Unpadded: 1 priming loads "add x24, x16, %x[ld_in_row]\n" - "ld1sb { z17.s }, p1/Z, [x16]\n" + "ld1sb { z0.s }, p1/Z, [x16]\n" "addvl x23, SP, #6\n" - "ld1sb { z16.s }, p1/Z, [x24]\n" + "ld1sb { z3.s }, p1/Z, [x24]\n" "add x24, x24, %x[ld_in_row]\n" - "trn1 z27.h, z17.h, z16.h\n" - "add z27.h, z27.h, z25.h\n" - "ld1sb { z17.s }, p1/Z, [x24]\n" + "trn1 z28.h, z0.h, z3.h\n" + "add z28.h, z28.h, z17.h\n" + "ld1sb { z6.s }, p1/Z, [x24]\n" "add x24, x24, %x[ld_in_row]\n" "addvl x22, SP, #12\n" - "ld1sb { z16.s }, p1/Z, [x24]\n" + "ld1sb { z30.s }, p1/Z, [x24]\n" "add x24, x24, %x[ld_in_row]\n" - "trn1 z28.h, z17.h, z16.h\n" - "add z28.h, z28.h, z25.h\n" - "ld1sb { z17.s }, p1/Z, [x24]\n" + "trn1 z29.h, z6.h, z30.h\n" + "add z29.h, z29.h, z17.h\n" + "ld1sb { z1.s }, p1/Z, [x24]\n" "add x24, x24, %x[ld_in_row]\n" "addvl x21, SP, #18\n" - "ld1sb { z16.s }, p1/Z, [x24]\n" + "ld1sb { z25.s }, p1/Z, [x24]\n" "add x24, x24, %x[ld_in_row]\n" - "trn1 z29.h, z17.h, z16.h\n" - "add z29.h, z29.h, z25.h\n" - "ld1sb { z17.s }, p1/Z, [x24]\n" + "trn1 z30.h, z1.h, z25.h\n" + "add z30.h, z30.h, z17.h\n" + "ld1sb { z3.s }, p1/Z, [x24]\n" "add x24, x24, %x[ld_in_row]\n" "addvl x20, SP, #24\n" - ".inst 0xa1402ae0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x23]\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" + ".inst 0xa0402ae0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x23]\n" + ".inst 0xc1617788 // sdot za.s[x11, 0], { z28.h-z29.h }, z1.h\n" "add x16, x16, %x[ld_in_col]\n" - "ld1sb { z16.s }, p1/Z, [x24]\n" - "trn1 z30.h, z17.h, z16.h\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402ac0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x22]\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" - "add z30.h, z30.h, z25.h\n" - ".inst 0xa0412ae4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xa0422aea // ld1h { z10.h-z11.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xc168776c // sdot za.s[x11, 4], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776d // sdot za.s[x11, 5], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc168776e // sdot za.s[x11, 6], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776f // sdot za.s[x11, 7], { z27.h-z28.h }, z0.h\n" - ".inst 0xc165778c // sdot za.s[x11, 4], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778d // sdot za.s[x11, 5], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc165778e // sdot za.s[x11, 6], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778f // sdot za.s[x11, 7], { z28.h-z29.h }, z4.h\n" - ".inst 0xc16b77ac // sdot za.s[x11, 4], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ad // sdot za.s[x11, 5], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc16b77ae // sdot za.s[x11, 6], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77af // sdot za.s[x11, 7], { z29.h-z30.h }, z10.h\n" + "ld1sb { z5.s }, p1/Z, [x24]\n" + "trn1 z31.h, z3.h, z5.h\n" + ".inst 0xc1607789 // sdot za.s[x11, 1], { z28.h-z29.h }, z0.h\n" + ".inst 0xa1402ac6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x22]\n" + ".inst 0xc16e778a // sdot za.s[x11, 2], { z28.h-z29.h }, z14.h\n" + "add z31.h, z31.h, z17.h\n" + ".inst 0xa1412ae2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xc166778b // sdot za.s[x11, 3], { z28.h-z29.h }, z6.h\n" + ".inst 0xa0402aae // ld1h { z14.h-z15.h }, pn10.b/Z, [x21]\n" + ".inst 0xc16a77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z10.h\n" + ".inst 0xc16277a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z2.h\n" + ".inst 0xa0412ac8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xa1422ae2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xc16f778c // sdot za.s[x11, 4], { z28.h-z29.h }, z15.h\n" + ".inst 0xc16e778d // sdot za.s[x11, 5], { z28.h-z29.h }, z14.h\n" + ".inst 0xa1402a86 // ld1h { z6.h, z14.h }, pn10.b/Z, [x20]\n" + ".inst 0xc16977aa // sdot za.s[x11, 2], { z29.h-z30.h }, z9.h\n" + ".inst 0xc16877ab // sdot za.s[x11, 3], { z29.h-z30.h }, z8.h\n" + ".inst 0xa1412aa5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc16a77c8 // sdot za.s[x11, 0], { z30.h-z31.h }, z10.h\n" + ".inst 0xc16277c9 // sdot za.s[x11, 1], { z30.h-z31.h }, z2.h\n" + ".inst 0xa1422ac2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc16e778e // sdot za.s[x11, 6], { z28.h-z29.h }, z14.h\n" + ".inst 0xc166778f // sdot za.s[x11, 7], { z28.h-z29.h }, z6.h\n" + ".inst 0xc16d77ac // sdot za.s[x11, 4], { z29.h-z30.h }, z13.h\n" + ".inst 0xc16577ad // sdot za.s[x11, 5], { z29.h-z30.h }, z5.h\n" + ".inst 0xa1412a86 // ld1h { z6.h, z14.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc16a77ca // sdot za.s[x11, 2], { z30.h-z31.h }, z10.h\n" + ".inst 0xc16277cb // sdot za.s[x11, 3], { z30.h-z31.h }, z2.h\n" + ".inst 0xa0422aa8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc16e77ae // sdot za.s[x11, 6], { z29.h-z30.h }, z14.h\n" + ".inst 0xc16677af // sdot za.s[x11, 7], { z29.h-z30.h }, z6.h\n" + ".inst 0xc16977cc // sdot za.s[x11, 4], { z30.h-z31.h }, z9.h\n" + ".inst 0xc16877cd // sdot za.s[x11, 5], { z30.h-z31.h }, z8.h\n" + ".inst 0xa1422a86 // ld1h { z6.h, z14.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc16e77ce // sdot za.s[x11, 6], { z30.h-z31.h }, z14.h\n" + ".inst 0xc16677cf // sdot za.s[x11, 7], { z30.h-z31.h }, z6.h\n" "12:" // Unpadded: 0 priming loads - ".inst 0xa1402be0 // ld1h { z0.h, z8.h }, pn10.b/Z, [SP]\n" - ".inst 0xa0412be4 // ld1h { z4.h-z5.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" - ".inst 0xa0422bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" - "cbz x17, 22f\n" + ".inst 0xa0402be0 // ld1h { z0.h-z1.h }, pn10.b/Z, [SP]\n" + ".inst 0xa1412be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + ".inst 0xa0422be2 // ld1h { z2.h-z3.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" + "cbz x25, 22f\n" "add x20, x16, %x[ld_in_row]\n" - "ld1sb { z17.s }, p1/Z, [x16]\n" - "sub x17, x17, #0x1\n" - "ld1sb { z16.s }, p1/Z, [x20]\n" + "ld1sb { z26.s }, p1/Z, [x16]\n" + "sub x25, x25, #0x1\n" + "ld1sb { z28.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z27.h, z17.h, z16.h\n" + "trn1 z25.h, z26.h, z28.h\n" "sub x15, x15, #0x1\n" - "ld1sb { z17.s }, p1/Z, [x20]\n" + "ld1sb { z31.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "cmp x17, x15\n" - "add z27.h, z27.h, z25.h\n" - "ld1sb { z16.s }, p1/Z, [x20]\n" + "cmp x25, x15\n" + "add z25.h, z25.h, z17.h\n" + "ld1sb { z15.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z28.h, z17.h, z16.h\n" - "csel x25, x17, x15, LT\n" - "ld1sb { z17.s }, p1/Z, [x20]\n" + "trn1 z26.h, z31.h, z15.h\n" + "csel x25, x25, x15, LT\n" + "ld1sb { z22.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "add z28.h, z28.h, z25.h\n" + "add z26.h, z26.h, z17.h\n" "add x16, x16, %x[ld_in_col]\n" - "ld1sb { z16.s }, p1/Z, [x20]\n" + "ld1sb { z8.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z29.h, z17.h, z16.h\n" - "add z29.h, z29.h, z25.h\n" - "ld1sb { z17.s }, p1/Z, [x20]\n" + "trn1 z27.h, z22.h, z8.h\n" + "add z27.h, z27.h, z17.h\n" + "ld1sb { z21.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" "sub x15, x15, x25\n" - "ld1sb { z16.s }, p1/Z, [x20]\n" - "trn1 z30.h, z17.h, z16.h\n" - "add z30.h, z30.h, z25.h\n" + "ld1sb { z20.s }, p1/Z, [x20]\n" + "trn1 z28.h, z21.h, z20.h\n" + "add z28.h, z28.h, z17.h\n" "cbz x25, 21f\n" "13:" // Unpadded: Main loop "addvl x24, SP, #6\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" + ".inst 0xc1617728 // sdot za.s[x11, 0], { z25.h-z26.h }, z1.h\n" "addvl x23, SP, #12\n" - "ld1sb { z23.s }, p1/Z, [x16]\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402b00 // ld1h { z0.h, z8.h }, pn10.b/Z, [x24]\n" + "ld1sb { z21.s }, p1/Z, [x16]\n" + ".inst 0xc1607729 // sdot za.s[x11, 1], { z25.h-z26.h }, z0.h\n" + ".inst 0xa0402b0e // ld1h { z14.h-z15.h }, pn10.b/Z, [x24]\n" "addvl x22, SP, #18\n" "addvl x21, SP, #24\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" + ".inst 0xc16f772a // sdot za.s[x11, 2], { z25.h-z26.h }, z15.h\n" "add x20, x16, %x[ld_in_row]\n" - "ld1sb { z22.s }, p1/Z, [x20]\n" + "ld1sb { z0.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402ae0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x23]\n" + ".inst 0xc16e772b // sdot za.s[x11, 3], { z25.h-z26.h }, z14.h\n" + ".inst 0xa1402ae6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x23]\n" "subs x25, x25, #0x1\n" "add x16, x16, %x[ld_in_col]\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - "ld1sb { z21.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412b04 // ld1h { z4.h-z5.h }, pn10.b/Z, [x24, #0x2, MUL VL]\n" - ".inst 0xc168776c // sdot za.s[x11, 4], { z27.h-z28.h }, z8.h\n" + ".inst 0xc16d7748 // sdot za.s[x11, 0], { z26.h-z27.h }, z13.h\n" "ld1sb { z20.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc160776d // sdot za.s[x11, 5], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402ac0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x22]\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - "ld1sb { z19.s }, p1/Z, [x20]\n" + ".inst 0xc1657749 // sdot za.s[x11, 1], { z26.h-z27.h }, z5.h\n" + ".inst 0xa1412b05 // ld1h { z5.h, z13.h }, pn10.b/Z, [x24, #0x2, MUL VL]\n" + ".inst 0xc16e772c // sdot za.s[x11, 4], { z25.h-z26.h }, z14.h\n" + "ld1sb { z31.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412ae4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - "ld1sb { z18.s }, p1/Z, [x20]\n" + ".inst 0xc166772d // sdot za.s[x11, 5], { z25.h-z26.h }, z6.h\n" + ".inst 0xa0402ace // ld1h { z14.h-z15.h }, pn10.b/Z, [x22]\n" + ".inst 0xc16d774a // sdot za.s[x11, 2], { z26.h-z27.h }, z13.h\n" + "ld1sb { z29.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422b0a // ld1h { z10.h-z11.h }, pn10.b/Z, [x24, #0x4, MUL VL]\n" - ".inst 0xc168776e // sdot za.s[x11, 6], { z27.h-z28.h }, z8.h\n" - "ld1sb { z17.s }, p1/Z, [x20]\n" + ".inst 0xc165774b // sdot za.s[x11, 3], { z26.h-z27.h }, z5.h\n" + ".inst 0xa1412ae5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xc1637768 // sdot za.s[x11, 0], { z27.h-z28.h }, z3.h\n" + "ld1sb { z22.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc160776f // sdot za.s[x11, 7], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" - ".inst 0xc165778c // sdot za.s[x11, 4], { z28.h-z29.h }, z5.h\n" - "ld1sb { z16.s }, p1/Z, [x20]\n" - ".inst 0xc164778d // sdot za.s[x11, 5], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aea // ld1h { z10.h-z11.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xc165778e // sdot za.s[x11, 6], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778f // sdot za.s[x11, 7], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc16b77ac // sdot za.s[x11, 4], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ad // sdot za.s[x11, 5], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc16b77ae // sdot za.s[x11, 6], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77af // sdot za.s[x11, 7], { z29.h-z30.h }, z10.h\n" + ".inst 0xc1627769 // sdot za.s[x11, 1], { z27.h-z28.h }, z2.h\n" + ".inst 0xa1422b02 // ld1h { z2.h, z10.h }, pn10.b/Z, [x24, #0x4, MUL VL]\n" + ".inst 0xc16f772e // sdot za.s[x11, 6], { z25.h-z26.h }, z15.h\n" + "ld1sb { z30.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row]\n" + ".inst 0xc16e772f // sdot za.s[x11, 7], { z25.h-z26.h }, z14.h\n" + ".inst 0xa0402aae // ld1h { z14.h-z15.h }, pn10.b/Z, [x21]\n" + ".inst 0xc16d774c // sdot za.s[x11, 4], { z26.h-z27.h }, z13.h\n" + "ld1sb { z6.s }, p1/Z, [x20]\n" + ".inst 0xc165774d // sdot za.s[x11, 5], { z26.h-z27.h }, z5.h\n" + ".inst 0xa1412ac5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc16a776a // sdot za.s[x11, 2], { z27.h-z28.h }, z10.h\n" + ".inst 0xc162776b // sdot za.s[x11, 3], { z27.h-z28.h }, z2.h\n" + ".inst 0xa1422ae2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xc16d774e // sdot za.s[x11, 6], { z26.h-z27.h }, z13.h\n" + ".inst 0xc165774f // sdot za.s[x11, 7], { z26.h-z27.h }, z5.h\n" + ".inst 0xa1412aa5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc16a776c // sdot za.s[x11, 4], { z27.h-z28.h }, z10.h\n" + ".inst 0xc162776d // sdot za.s[x11, 5], { z27.h-z28.h }, z2.h\n" + ".inst 0xa1422ac1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc169776e // sdot za.s[x11, 6], { z27.h-z28.h }, z9.h\n" + ".inst 0xc161776f // sdot za.s[x11, 7], { z27.h-z28.h }, z1.h\n" ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc1681768 // sdot za.s[x8, 0], { z27.h-z28.h }, z8.h\n" - ".inst 0xc1601769 // sdot za.s[x8, 1], { z27.h-z28.h }, z0.h\n" - "trn1 z27.h, z23.h, z22.h\n" - ".inst 0xa1402be0 // ld1h { z0.h, z8.h }, pn10.b/Z, [SP]\n" - ".inst 0xc1651788 // sdot za.s[x8, 0], { z28.h-z29.h }, z5.h\n" - "add z27.h, z27.h, z25.h\n" - ".inst 0xc1641789 // sdot za.s[x8, 1], { z28.h-z29.h }, z4.h\n" - "trn1 z28.h, z21.h, z20.h\n" - ".inst 0xa0412be4 // ld1h { z4.h-z5.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" - ".inst 0xc16b17a8 // sdot za.s[x8, 0], { z29.h-z30.h }, z11.h\n" - "add z28.h, z28.h, z25.h\n" - ".inst 0xc16a17a9 // sdot za.s[x8, 1], { z29.h-z30.h }, z10.h\n" - "trn1 z29.h, z19.h, z18.h\n" - "trn1 z30.h, z17.h, z16.h\n" + ".inst 0xc16f1728 // sdot za.s[x8, 0], { z25.h-z26.h }, z15.h\n" + ".inst 0xc16e1729 // sdot za.s[x8, 1], { z25.h-z26.h }, z14.h\n" + "trn1 z25.h, z21.h, z0.h\n" + ".inst 0xa0402be0 // ld1h { z0.h-z1.h }, pn10.b/Z, [SP]\n" + ".inst 0xc16d1748 // sdot za.s[x8, 0], { z26.h-z27.h }, z13.h\n" + "add z25.h, z25.h, z17.h\n" + ".inst 0xc1651749 // sdot za.s[x8, 1], { z26.h-z27.h }, z5.h\n" + "trn1 z26.h, z20.h, z31.h\n" + ".inst 0xa1412be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + ".inst 0xc16b1768 // sdot za.s[x8, 0], { z27.h-z28.h }, z11.h\n" + "add z26.h, z26.h, z17.h\n" + ".inst 0xc16a1769 // sdot za.s[x8, 1], { z27.h-z28.h }, z10.h\n" + "trn1 z27.h, z29.h, z22.h\n" + "trn1 z28.h, z30.h, z6.h\n" "add x8, x8, #0x2\n" - ".inst 0xc006680c // mova { z12.d-z13.d }, za.d[x11, #0]\n" - ".inst 0xa0422bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" - "add z29.h, z29.h, z25.h\n" - ".inst 0xc006682e // mova { z14.d-z15.d }, za.d[x11, #1]\n" - ".inst 0xc1a3ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z3.s\n" + ".inst 0xc0066808 // mova { z8.d-z9.d }, za.d[x11, #0]\n" + ".inst 0xa0422be2 // ld1h { z2.h-z3.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" + "add z27.h, z27.h, z17.h\n" + ".inst 0xc006682a // mova { z10.d-z11.d }, za.d[x11, #1]\n" + ".inst 0xc1a7ac08 // sqdmulh { z8.s-z11.s }, { z8.s-z11.s }, z7.s\n" "add x11, x11, #0x2\n" - ".inst 0xc1a1aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n" - ".inst 0xc00408c0 // mova za.d[x8, #0], { z6.d-z7.d }\n" - ".inst 0xc1a9ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z9.s\n" - ".inst 0xc00408c1 // mova za.d[x8, #1], { z6.d-z7.d }\n" - ".inst 0xc1bfcf0c // sclamp { z12.s-z15.s }, z24.s, z31.s\n" - "st1b { z12.s }, p1, [x14]\n" - "add x14, x14, x3\n" - "add z30.h, z30.h, z25.h\n" - "st1b { z14.s }, p1, [x13]\n" + ".inst 0xc1a4aa28 // srshl { z8.s-z11.s }, { z8.s-z11.s }, z4.s\n" + ".inst 0xc0040a40 // mova za.d[x8, #0], { z18.d-z19.d }\n" + ".inst 0xc1acab08 // add { z8.s-z11.s }, { z8.s-z11.s }, z12.s\n" + ".inst 0xc0040a41 // mova za.d[x8, #1], { z18.d-z19.d }\n" + ".inst 0xc1b0cf08 // sclamp { z8.s-z11.s }, z24.s, z16.s\n" + "st1b { z8.s }, p1, [x14]\n" + "add x14, x14, x4\n" + "add z28.h, z28.h, z17.h\n" + "st1b { z10.s }, p1, [x13]\n" "add x13, x13, x10\n" - "st1b { z13.s }, p1, [x9]\n" + "st1b { z9.s }, p1, [x9]\n" "add x9, x9, x27\n" - "st1b { z15.s }, p1, [x28]\n" + "st1b { z11.s }, p1, [x28]\n" "add x28, x28, x26\n" "bgt 13b\n" "b 21f\n" "14:" // Padded "cbz x22, 19f\n" "cmp x22, #0x1\n" - "sub x17, x17, x22\n" + "sub x25, x25, x22\n" "beq 18f\n" "cmp x22, #0x2\n" "beq 17f\n" @@ -668,515 +668,515 @@ void sme2_s8q_planar_5x5_s1_4rows_dot_za_impl( "15:" // Padded: 4 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z19.s }, p0/Z, [x16]\n" - "add z19.h, p0/M, z19.h, z25.h\n" + "ld1sb { z9.s }, p0/Z, [x16]\n" + "add z9.h, p0/M, z9.h, z17.h\n" "add x21, x16, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z18.s }, p0/Z, [x21]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1sb { z22.s }, p0/Z, [x21]\n" + "add z22.h, p0/M, z22.h, z17.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z17.s }, p0/Z, [x21]\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "ld1sb { z21.s }, p0/Z, [x21]\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z16.s }, p0/Z, [x21]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1sb { z20.s }, p0/Z, [x21]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "mov x12, #0x4\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z27.h, z19.h, z18.h\n" - "trn1 z28.h, z17.h, z16.h\n" + "trn1 z31.h, z9.h, z22.h\n" + "trn1 z0.h, z21.h, z20.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z18.s }, p0/Z, [x21]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1sb { z22.s }, p0/Z, [x21]\n" + "add z22.h, p0/M, z22.h, z17.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z16.s }, p0/Z, [x21]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1sb { z20.s }, p0/Z, [x21]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z17.s }, p0/Z, [x21]\n" + "ld1sb { z21.s }, p0/Z, [x21]\n" "addvl x20, SP, #24\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - "trn1 z29.h, z18.h, z16.h\n" - "ld1sb { z16.s }, p0/Z, [x21]\n" - "add z16.h, p0/M, z16.h, z25.h\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" + ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" + "trn1 z1.h, z22.h, z20.h\n" + "ld1sb { z20.s }, p0/Z, [x21]\n" + "add z20.h, p0/M, z20.h, z17.h\n" + ".inst 0xc16a77e8 // sdot za.s[x11, 0], { z31.h-z0.h }, z10.h\n" "add x16, x16, %x[ld_in_col]\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - "trn1 z30.h, z17.h, z16.h\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" + ".inst 0xc16277e9 // sdot za.s[x11, 1], { z31.h-z0.h }, z2.h\n" + ".inst 0xa1412a85 // ld1h { z5.h, z13.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + "trn1 z2.h, z21.h, z20.h\n" + ".inst 0xc16d7408 // sdot za.s[x11, 0], { z0.h-z1.h }, z13.h\n" + ".inst 0xa0422a88 // ld1h { z8.h-z9.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc1657409 // sdot za.s[x11, 1], { z0.h-z1.h }, z5.h\n" + ".inst 0xc1697428 // sdot za.s[x11, 0], { z1.h-z2.h }, z9.h\n" + ".inst 0xc1687429 // sdot za.s[x11, 1], { z1.h-z2.h }, z8.h\n" "16:" // Padded: 3 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z19.s }, p0/Z, [x16]\n" - "add z19.h, p0/M, z19.h, z25.h\n" + "ld1sb { z5.s }, p0/Z, [x16]\n" + "add z5.h, p0/M, z5.h, z17.h\n" "add x20, x16, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1sb { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "ld1sb { z21.s }, p0/Z, [x20]\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1sb { z20.s }, p0/Z, [x20]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z27.h, z19.h, z18.h\n" - "trn1 z28.h, z17.h, z16.h\n" + "trn1 z28.h, z5.h, z22.h\n" + "trn1 z29.h, z21.h, z20.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1sb { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1sb { z20.s }, p0/Z, [x20]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z17.s }, p0/Z, [x20]\n" + "ld1sb { z21.s }, p0/Z, [x20]\n" "addvl x21, SP, #18\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" - "trn1 z29.h, z18.h, z16.h\n" - "ld1sb { z16.s }, p0/Z, [x20]\n" + ".inst 0xa0402aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21]\n" + "trn1 z30.h, z22.h, z20.h\n" + "ld1sb { z20.s }, p0/Z, [x20]\n" "addvl x20, SP, #24\n" - "add z16.h, p0/M, z16.h, z25.h\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - "trn1 z30.h, z17.h, z16.h\n" + "add z20.h, p0/M, z20.h, z17.h\n" + ".inst 0xc1617788 // sdot za.s[x11, 0], { z28.h-z29.h }, z1.h\n" + ".inst 0xc1607789 // sdot za.s[x11, 1], { z28.h-z29.h }, z0.h\n" + ".inst 0xa1402a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20]\n" + "trn1 z31.h, z21.h, z20.h\n" "add x16, x16, %x[ld_in_col]\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" + ".inst 0xa0412aae // ld1h { z14.h-z15.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc169778a // sdot za.s[x11, 2], { z28.h-z29.h }, z9.h\n" + ".inst 0xc161778b // sdot za.s[x11, 3], { z28.h-z29.h }, z1.h\n" + ".inst 0xa1422aa3 // ld1h { z3.h, z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc16f77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z15.h\n" + ".inst 0xc16e77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z14.h\n" + ".inst 0xa1412a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc16977aa // sdot za.s[x11, 2], { z29.h-z30.h }, z9.h\n" + ".inst 0xc16177ab // sdot za.s[x11, 3], { z29.h-z30.h }, z1.h\n" + ".inst 0xc16b77c8 // sdot za.s[x11, 0], { z30.h-z31.h }, z11.h\n" + ".inst 0xc16377c9 // sdot za.s[x11, 1], { z30.h-z31.h }, z3.h\n" + ".inst 0xa0422a8e // ld1h { z14.h-z15.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc16f77ca // sdot za.s[x11, 2], { z30.h-z31.h }, z15.h\n" + ".inst 0xc16e77cb // sdot za.s[x11, 3], { z30.h-z31.h }, z14.h\n" "17:" // Padded: 2 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z19.s }, p0/Z, [x16]\n" - "add z19.h, p0/M, z19.h, z25.h\n" + "ld1sb { z29.s }, p0/Z, [x16]\n" + "add z29.h, p0/M, z29.h, z17.h\n" "add x20, x16, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1sb { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "ld1sb { z21.s }, p0/Z, [x20]\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1sb { z20.s }, p0/Z, [x20]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z27.h, z19.h, z18.h\n" - "trn1 z28.h, z17.h, z16.h\n" + "trn1 z8.h, z29.h, z22.h\n" + "trn1 z9.h, z21.h, z20.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1sb { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1sb { z20.s }, p0/Z, [x20]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z17.s }, p0/Z, [x20]\n" + "ld1sb { z21.s }, p0/Z, [x20]\n" "addvl x22, SP, #12\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xa1402ac0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x22]\n" - "trn1 z29.h, z18.h, z16.h\n" - "ld1sb { z16.s }, p0/Z, [x20]\n" + ".inst 0xa0402ace // ld1h { z14.h-z15.h }, pn10.b/Z, [x22]\n" + "trn1 z10.h, z22.h, z20.h\n" + "ld1sb { z20.s }, p0/Z, [x20]\n" "addvl x21, SP, #18\n" - "add z16.h, p0/M, z16.h, z25.h\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" + "add z20.h, p0/M, z20.h, z17.h\n" + ".inst 0xc16f7508 // sdot za.s[x11, 0], { z8.h-z9.h }, z15.h\n" + ".inst 0xc16e7509 // sdot za.s[x11, 1], { z8.h-z9.h }, z14.h\n" + ".inst 0xa1402aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21]\n" "addvl x20, SP, #24\n" - "trn1 z30.h, z17.h, z16.h\n" - ".inst 0xa0412ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" + "trn1 z11.h, z21.h, z20.h\n" + ".inst 0xa1412ac5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc16e750a // sdot za.s[x11, 2], { z8.h-z9.h }, z14.h\n" "add x16, x16, %x[ld_in_col]\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xa0422aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc168776c // sdot za.s[x11, 4], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776d // sdot za.s[x11, 5], { z27.h-z28.h }, z0.h\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc165778c // sdot za.s[x11, 4], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778d // sdot za.s[x11, 5], { z28.h-z29.h }, z4.h\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc16b77ac // sdot za.s[x11, 4], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ad // sdot za.s[x11, 5], { z29.h-z30.h }, z10.h\n" + ".inst 0xc166750b // sdot za.s[x11, 3], { z8.h-z9.h }, z6.h\n" + ".inst 0xa0402a8e // ld1h { z14.h-z15.h }, pn10.b/Z, [x20]\n" + ".inst 0xc16d7528 // sdot za.s[x11, 0], { z9.h-z10.h }, z13.h\n" + ".inst 0xa0422ac0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc1657529 // sdot za.s[x11, 1], { z9.h-z10.h }, z5.h\n" + ".inst 0xa1412aa5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc16f750c // sdot za.s[x11, 4], { z8.h-z9.h }, z15.h\n" + ".inst 0xc16e750d // sdot za.s[x11, 5], { z8.h-z9.h }, z14.h\n" + ".inst 0xc16d752a // sdot za.s[x11, 2], { z9.h-z10.h }, z13.h\n" + ".inst 0xc165752b // sdot za.s[x11, 3], { z9.h-z10.h }, z5.h\n" + ".inst 0xa1412a86 // ld1h { z6.h, z14.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc1617548 // sdot za.s[x11, 0], { z10.h-z11.h }, z1.h\n" + ".inst 0xc1607549 // sdot za.s[x11, 1], { z10.h-z11.h }, z0.h\n" + ".inst 0xa0422aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc16e752c // sdot za.s[x11, 4], { z9.h-z10.h }, z14.h\n" + ".inst 0xc166752d // sdot za.s[x11, 5], { z9.h-z10.h }, z6.h\n" + ".inst 0xc161754a // sdot za.s[x11, 2], { z10.h-z11.h }, z1.h\n" + ".inst 0xc160754b // sdot za.s[x11, 3], { z10.h-z11.h }, z0.h\n" + ".inst 0xa0422a8e // ld1h { z14.h-z15.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc16f754c // sdot za.s[x11, 4], { z10.h-z11.h }, z15.h\n" + ".inst 0xc16e754d // sdot za.s[x11, 5], { z10.h-z11.h }, z14.h\n" "18:" // Padded: 1 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z19.s }, p0/Z, [x16]\n" - "add z19.h, p0/M, z19.h, z25.h\n" + "ld1sb { z1.s }, p0/Z, [x16]\n" + "add z1.h, p0/M, z1.h, z17.h\n" "add x20, x16, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1sb { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "ld1sb { z21.s }, p0/Z, [x20]\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1sb { z20.s }, p0/Z, [x20]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z27.h, z19.h, z18.h\n" - "trn1 z28.h, z17.h, z16.h\n" + "trn1 z26.h, z1.h, z22.h\n" + "trn1 z27.h, z21.h, z20.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1sb { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1sb { z20.s }, p0/Z, [x20]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z17.s }, p0/Z, [x20]\n" + "ld1sb { z21.s }, p0/Z, [x20]\n" "addvl x23, SP, #6\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xa1402ae0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x23]\n" - "trn1 z29.h, z18.h, z16.h\n" - "ld1sb { z16.s }, p0/Z, [x20]\n" + ".inst 0xa0402aee // ld1h { z14.h-z15.h }, pn10.b/Z, [x23]\n" + "trn1 z28.h, z22.h, z20.h\n" + "ld1sb { z20.s }, p0/Z, [x20]\n" "addvl x22, SP, #12\n" - "add z16.h, p0/M, z16.h, z25.h\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402ac0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x22]\n" + "add z20.h, p0/M, z20.h, z17.h\n" + ".inst 0xc16f7748 // sdot za.s[x11, 0], { z26.h-z27.h }, z15.h\n" + ".inst 0xc16e7749 // sdot za.s[x11, 1], { z26.h-z27.h }, z14.h\n" + ".inst 0xa0402ac0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x22]\n" "addvl x21, SP, #18\n" - "trn1 z30.h, z17.h, z16.h\n" - ".inst 0xa0412ae4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" + "trn1 z29.h, z21.h, z20.h\n" + ".inst 0xa0412aea // ld1h { z10.h-z11.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xc161774a // sdot za.s[x11, 2], { z26.h-z27.h }, z1.h\n" "addvl x20, SP, #24\n" "add x16, x16, %x[ld_in_col]\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xa0422aea // ld1h { z10.h-z11.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xc168776c // sdot za.s[x11, 4], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776d // sdot za.s[x11, 5], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc168776e // sdot za.s[x11, 6], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776f // sdot za.s[x11, 7], { z27.h-z28.h }, z0.h\n" - ".inst 0xc165778c // sdot za.s[x11, 4], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778d // sdot za.s[x11, 5], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc165778e // sdot za.s[x11, 6], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778f // sdot za.s[x11, 7], { z28.h-z29.h }, z4.h\n" - ".inst 0xc16b77ac // sdot za.s[x11, 4], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ad // sdot za.s[x11, 5], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc16b77ae // sdot za.s[x11, 6], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77af // sdot za.s[x11, 7], { z29.h-z30.h }, z10.h\n" + ".inst 0xc160774b // sdot za.s[x11, 3], { z26.h-z27.h }, z0.h\n" + ".inst 0xa1402aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21]\n" + ".inst 0xc16b7768 // sdot za.s[x11, 0], { z27.h-z28.h }, z11.h\n" + ".inst 0xa0422ae8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xc16a7769 // sdot za.s[x11, 1], { z27.h-z28.h }, z10.h\n" + ".inst 0xa0412aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc16e774c // sdot za.s[x11, 4], { z26.h-z27.h }, z14.h\n" + ".inst 0xc166774d // sdot za.s[x11, 5], { z26.h-z27.h }, z6.h\n" + ".inst 0xa1402a85 // ld1h { z5.h, z13.h }, pn10.b/Z, [x20]\n" + ".inst 0xc16b776a // sdot za.s[x11, 2], { z27.h-z28.h }, z11.h\n" + ".inst 0xc16a776b // sdot za.s[x11, 3], { z27.h-z28.h }, z10.h\n" + ".inst 0xa1412aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc1697788 // sdot za.s[x11, 0], { z28.h-z29.h }, z9.h\n" + ".inst 0xc1687789 // sdot za.s[x11, 1], { z28.h-z29.h }, z8.h\n" + ".inst 0xa1422ac2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc16d774e // sdot za.s[x11, 6], { z26.h-z27.h }, z13.h\n" + ".inst 0xc165774f // sdot za.s[x11, 7], { z26.h-z27.h }, z5.h\n" + ".inst 0xc16e776c // sdot za.s[x11, 4], { z27.h-z28.h }, z14.h\n" + ".inst 0xc166776d // sdot za.s[x11, 5], { z27.h-z28.h }, z6.h\n" + ".inst 0xa1412a86 // ld1h { z6.h, z14.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc16a778a // sdot za.s[x11, 2], { z28.h-z29.h }, z10.h\n" + ".inst 0xc162778b // sdot za.s[x11, 3], { z28.h-z29.h }, z2.h\n" + ".inst 0xa0422aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc16e776e // sdot za.s[x11, 6], { z27.h-z28.h }, z14.h\n" + ".inst 0xc166776f // sdot za.s[x11, 7], { z27.h-z28.h }, z6.h\n" + ".inst 0xc161778c // sdot za.s[x11, 4], { z28.h-z29.h }, z1.h\n" + ".inst 0xc160778d // sdot za.s[x11, 5], { z28.h-z29.h }, z0.h\n" + ".inst 0xa1422a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc16a778e // sdot za.s[x11, 6], { z28.h-z29.h }, z10.h\n" + ".inst 0xc162778f // sdot za.s[x11, 7], { z28.h-z29.h }, z2.h\n" "19:" // Padded: 0 priming loads - ".inst 0xa1402be0 // ld1h { z0.h, z8.h }, pn10.b/Z, [SP]\n" - ".inst 0xa0412be4 // ld1h { z4.h-z5.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" - ".inst 0xa0422bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" - "cbz x17, 22f\n" + ".inst 0xa0402be0 // ld1h { z0.h-z1.h }, pn10.b/Z, [SP]\n" + ".inst 0xa1412be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + ".inst 0xa0422be2 // ld1h { z2.h-z3.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" + "cbz x25, 22f\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z19.s }, p0/Z, [x16]\n" - "add z19.h, p0/M, z19.h, z25.h\n" + "ld1sb { z6.s }, p0/Z, [x16]\n" + "add z6.h, p0/M, z6.h, z17.h\n" "add x20, x16, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1sb { z30.s }, p0/Z, [x20]\n" + "add z30.h, p0/M, z30.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "ld1sb { z27.s }, p0/Z, [x20]\n" + "add z27.h, p0/M, z27.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1sb { z26.s }, p0/Z, [x20]\n" + "add z26.h, p0/M, z26.h, z17.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z27.h, z19.h, z18.h\n" - "trn1 z28.h, z17.h, z16.h\n" + "trn1 z25.h, z6.h, z30.h\n" + "trn1 z26.h, z27.h, z26.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z25.h\n" + "ld1sb { z8.s }, p0/Z, [x20]\n" + "add z8.h, p0/M, z8.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1sb { z9.s }, p0/Z, [x20]\n" + "add z9.h, p0/M, z9.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "ld1sb { z21.s }, p0/Z, [x20]\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z25.h\n" - "sub x17, x17, #0x1\n" + "ld1sb { z29.s }, p0/Z, [x20]\n" + "add z29.h, p0/M, z29.h, z17.h\n" + "sub x25, x25, #0x1\n" "sub x15, x15, #0x1\n" - "cmp x17, x15\n" - "trn1 z29.h, z19.h, z18.h\n" - "trn1 z30.h, z17.h, z16.h\n" - "csel x25, x17, x15, LT\n" + "cmp x25, x15\n" + "trn1 z27.h, z8.h, z9.h\n" + "trn1 z28.h, z21.h, z29.h\n" + "csel x25, x25, x15, LT\n" "add x16, x16, %x[ld_in_col]\n" "sub x15, x15, x25\n" "cbz x25, 21f\n" "20:" // Padded: Main loop "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z23.s }, p0/Z, [x16]\n" - "add z23.h, p0/M, z23.h, z25.h\n" + "ld1sb { z8.s }, p0/Z, [x16]\n" + "add z8.h, p0/M, z8.h, z17.h\n" "add x24, x16, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z22.s }, p0/Z, [x24]\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" + "ld1sb { z21.s }, p0/Z, [x24]\n" + ".inst 0xc1617728 // sdot za.s[x11, 0], { z25.h-z26.h }, z1.h\n" "addvl x23, SP, #6\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402ae0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x23]\n" + ".inst 0xc1607729 // sdot za.s[x11, 1], { z25.h-z26.h }, z0.h\n" + ".inst 0xa0402ae0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x23]\n" "addvl x22, SP, #12\n" - "add z22.h, p0/M, z22.h, z25.h\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x24, x24, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402ac0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x22]\n" + ".inst 0xc161772a // sdot za.s[x11, 2], { z25.h-z26.h }, z1.h\n" + ".inst 0xc160772b // sdot za.s[x11, 3], { z25.h-z26.h }, z0.h\n" + ".inst 0xa1402ac6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x22]\n" "addvl x21, SP, #18\n" "addvl x20, SP, #24\n" - "ld1sb { z21.s }, p0/Z, [x24]\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - "add z21.h, p0/M, z21.h, z25.h\n" + "ld1sb { z29.s }, p0/Z, [x24]\n" + ".inst 0xc16d7748 // sdot za.s[x11, 0], { z26.h-z27.h }, z13.h\n" + "add z29.h, p0/M, z29.h, z17.h\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412ae4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xc1657749 // sdot za.s[x11, 1], { z26.h-z27.h }, z5.h\n" + ".inst 0xa1412ae5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" "mov x12, #0x4\n" "add x24, x24, %x[ld_in_row]\n" - ".inst 0xc168776c // sdot za.s[x11, 4], { z27.h-z28.h }, z8.h\n" - "ld1sb { z20.s }, p0/Z, [x24]\n" - "add z20.h, p0/M, z20.h, z25.h\n" + ".inst 0xc16e772c // sdot za.s[x11, 4], { z25.h-z26.h }, z14.h\n" + "ld1sb { z30.s }, p0/Z, [x24]\n" + "add z30.h, p0/M, z30.h, z17.h\n" "add x24, x24, %x[ld_in_row]\n" - ".inst 0xc160776d // sdot za.s[x11, 5], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" + ".inst 0xc166772d // sdot za.s[x11, 5], { z25.h-z26.h }, z6.h\n" + ".inst 0xa1402aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "subs x25, x25, #0x1\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - "ld1sb { z19.s }, p0/Z, [x24]\n" - "add z19.h, p0/M, z19.h, z25.h\n" + ".inst 0xc16d774a // sdot za.s[x11, 2], { z26.h-z27.h }, z13.h\n" + "ld1sb { z15.s }, p0/Z, [x24]\n" + "add z15.h, p0/M, z15.h, z17.h\n" "add x24, x24, %x[ld_in_row]\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc165774b // sdot za.s[x11, 3], { z26.h-z27.h }, z5.h\n" + ".inst 0xa0412aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "add x16, x16, %x[ld_in_col]\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - "ld1sb { z18.s }, p0/Z, [x24]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + ".inst 0xc1637768 // sdot za.s[x11, 0], { z27.h-z28.h }, z3.h\n" + "ld1sb { z20.s }, p0/Z, [x24]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "add x24, x24, %x[ld_in_row]\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aea // ld1h { z10.h-z11.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xc1627769 // sdot za.s[x11, 1], { z27.h-z28.h }, z2.h\n" + ".inst 0xa1422ae1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc168776e // sdot za.s[x11, 6], { z27.h-z28.h }, z8.h\n" - "ld1sb { z17.s }, p0/Z, [x24]\n" - "add z17.h, p0/M, z17.h, z25.h\n" + ".inst 0xc16e772e // sdot za.s[x11, 6], { z25.h-z26.h }, z14.h\n" + "ld1sb { z31.s }, p0/Z, [x24]\n" + "add z31.h, p0/M, z31.h, z17.h\n" "add x24, x24, %x[ld_in_row]\n" - ".inst 0xc160776f // sdot za.s[x11, 7], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" + ".inst 0xc166772f // sdot za.s[x11, 7], { z25.h-z26.h }, z6.h\n" + ".inst 0xa0402a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xc165778c // sdot za.s[x11, 4], { z28.h-z29.h }, z5.h\n" - "ld1sb { z16.s }, p0/Z, [x24]\n" - "add z16.h, p0/M, z16.h, z25.h\n" - ".inst 0xc164778d // sdot za.s[x11, 5], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc165778e // sdot za.s[x11, 6], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778f // sdot za.s[x11, 7], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16b77ac // sdot za.s[x11, 4], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ad // sdot za.s[x11, 5], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc16b77ae // sdot za.s[x11, 6], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77af // sdot za.s[x11, 7], { z29.h-z30.h }, z10.h\n" + ".inst 0xc16b774c // sdot za.s[x11, 4], { z26.h-z27.h }, z11.h\n" + "ld1sb { z22.s }, p0/Z, [x24]\n" + "add z22.h, p0/M, z22.h, z17.h\n" + ".inst 0xc16a774d // sdot za.s[x11, 5], { z26.h-z27.h }, z10.h\n" + ".inst 0xa1412aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc169776a // sdot za.s[x11, 2], { z27.h-z28.h }, z9.h\n" + ".inst 0xc161776b // sdot za.s[x11, 3], { z27.h-z28.h }, z1.h\n" + ".inst 0xa0422ac0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc16e774e // sdot za.s[x11, 6], { z26.h-z27.h }, z14.h\n" + ".inst 0xc166774f // sdot za.s[x11, 7], { z26.h-z27.h }, z6.h\n" + ".inst 0xa1412a86 // ld1h { z6.h, z14.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc161776c // sdot za.s[x11, 4], { z27.h-z28.h }, z1.h\n" + ".inst 0xc160776d // sdot za.s[x11, 5], { z27.h-z28.h }, z0.h\n" + ".inst 0xa1422aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc169776e // sdot za.s[x11, 6], { z27.h-z28.h }, z9.h\n" + ".inst 0xc161776f // sdot za.s[x11, 7], { z27.h-z28.h }, z1.h\n" ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc1681768 // sdot za.s[x8, 0], { z27.h-z28.h }, z8.h\n" - ".inst 0xc1601769 // sdot za.s[x8, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402be0 // ld1h { z0.h, z8.h }, pn10.b/Z, [SP]\n" - "trn1 z27.h, z23.h, z22.h\n" - ".inst 0xc1651788 // sdot za.s[x8, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xc1641789 // sdot za.s[x8, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412be4 // ld1h { z4.h-z5.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" - "trn1 z28.h, z21.h, z20.h\n" - ".inst 0xc16b17a8 // sdot za.s[x8, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a17a9 // sdot za.s[x8, 1], { z29.h-z30.h }, z10.h\n" + ".inst 0xc1631728 // sdot za.s[x8, 0], { z25.h-z26.h }, z3.h\n" + ".inst 0xc1621729 // sdot za.s[x8, 1], { z25.h-z26.h }, z2.h\n" + ".inst 0xa0402be0 // ld1h { z0.h-z1.h }, pn10.b/Z, [SP]\n" + "trn1 z25.h, z8.h, z21.h\n" + ".inst 0xc16e1748 // sdot za.s[x8, 0], { z26.h-z27.h }, z14.h\n" + ".inst 0xc1661749 // sdot za.s[x8, 1], { z26.h-z27.h }, z6.h\n" + ".inst 0xa1412be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + "trn1 z26.h, z29.h, z30.h\n" + ".inst 0xc16b1768 // sdot za.s[x8, 0], { z27.h-z28.h }, z11.h\n" + ".inst 0xc16a1769 // sdot za.s[x8, 1], { z27.h-z28.h }, z10.h\n" "add x8, x8, #0x2\n" - ".inst 0xa0422bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" - "trn1 z29.h, z19.h, z18.h\n" - ".inst 0xc006680c // mova { z12.d-z13.d }, za.d[x11, #0]\n" - "trn1 z30.h, z17.h, z16.h\n" - ".inst 0xc006682e // mova { z14.d-z15.d }, za.d[x11, #1]\n" - ".inst 0xc1a3ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z3.s\n" + ".inst 0xa0422be2 // ld1h { z2.h-z3.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" + "trn1 z27.h, z15.h, z20.h\n" + ".inst 0xc0066808 // mova { z8.d-z9.d }, za.d[x11, #0]\n" + "trn1 z28.h, z31.h, z22.h\n" + ".inst 0xc006682a // mova { z10.d-z11.d }, za.d[x11, #1]\n" + ".inst 0xc1a7ac08 // sqdmulh { z8.s-z11.s }, { z8.s-z11.s }, z7.s\n" "add x11, x11, #0x2\n" - ".inst 0xc1a1aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n" - ".inst 0xc00408c0 // mova za.d[x8, #0], { z6.d-z7.d }\n" - ".inst 0xc1a9ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z9.s\n" - ".inst 0xc00408c1 // mova za.d[x8, #1], { z6.d-z7.d }\n" - ".inst 0xc1bfcf0c // sclamp { z12.s-z15.s }, z24.s, z31.s\n" - "st1b { z12.s }, p1, [x14]\n" - "add x14, x14, x3\n" - "st1b { z14.s }, p1, [x13]\n" + ".inst 0xc1a4aa28 // srshl { z8.s-z11.s }, { z8.s-z11.s }, z4.s\n" + ".inst 0xc0040a40 // mova za.d[x8, #0], { z18.d-z19.d }\n" + ".inst 0xc1acab08 // add { z8.s-z11.s }, { z8.s-z11.s }, z12.s\n" + ".inst 0xc0040a41 // mova za.d[x8, #1], { z18.d-z19.d }\n" + ".inst 0xc1b0cf08 // sclamp { z8.s-z11.s }, z24.s, z16.s\n" + "st1b { z8.s }, p1, [x14]\n" + "add x14, x14, x4\n" + "st1b { z10.s }, p1, [x13]\n" "add x13, x13, x10\n" - "st1b { z13.s }, p1, [x9]\n" + "st1b { z9.s }, p1, [x9]\n" "add x9, x9, x27\n" - "st1b { z15.s }, p1, [x28]\n" + "st1b { z11.s }, p1, [x28]\n" "add x28, x28, x26\n" "bgt 20b\n" "21:" // Main loop tail "addvl x23, SP, #6\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" + ".inst 0xc1617728 // sdot za.s[x11, 0], { z25.h-z26.h }, z1.h\n" "addvl x22, SP, #12\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402ae0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x23]\n" + ".inst 0xc1607729 // sdot za.s[x11, 1], { z25.h-z26.h }, z0.h\n" + ".inst 0xa0402ae0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x23]\n" "addvl x21, SP, #18\n" "addvl x20, SP, #24\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402ac0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x22]\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412ae4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0xc168776c // sdot za.s[x11, 4], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776d // sdot za.s[x11, 5], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" + ".inst 0xc161772a // sdot za.s[x11, 2], { z25.h-z26.h }, z1.h\n" + ".inst 0xc160772b // sdot za.s[x11, 3], { z25.h-z26.h }, z0.h\n" + ".inst 0xa1402ac6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x22]\n" + ".inst 0xc16d7748 // sdot za.s[x11, 0], { z26.h-z27.h }, z13.h\n" + ".inst 0xc1657749 // sdot za.s[x11, 1], { z26.h-z27.h }, z5.h\n" + ".inst 0xa1412ae1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xc16e772c // sdot za.s[x11, 4], { z25.h-z26.h }, z14.h\n" + ".inst 0xc166772d // sdot za.s[x11, 5], { z25.h-z26.h }, z6.h\n" + ".inst 0xa1402aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21]\n" + ".inst 0xc169774a // sdot za.s[x11, 2], { z26.h-z27.h }, z9.h\n" + ".inst 0xc161774b // sdot za.s[x11, 3], { z26.h-z27.h }, z1.h\n" + ".inst 0xa1412ac1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc1637768 // sdot za.s[x11, 0], { z27.h-z28.h }, z3.h\n" + ".inst 0xc1627769 // sdot za.s[x11, 1], { z27.h-z28.h }, z2.h\n" ".inst 0xa0422aea // ld1h { z10.h-z11.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xc168776e // sdot za.s[x11, 6], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776f // sdot za.s[x11, 7], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - ".inst 0xc165778c // sdot za.s[x11, 4], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778d // sdot za.s[x11, 5], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc165778e // sdot za.s[x11, 6], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778f // sdot za.s[x11, 7], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16b77ac // sdot za.s[x11, 4], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ad // sdot za.s[x11, 5], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc16b77ae // sdot za.s[x11, 6], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77af // sdot za.s[x11, 7], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc1681768 // sdot za.s[x8, 0], { z27.h-z28.h }, z8.h\n" + ".inst 0xc16e772e // sdot za.s[x11, 6], { z25.h-z26.h }, z14.h\n" + ".inst 0xc166772f // sdot za.s[x11, 7], { z25.h-z26.h }, z6.h\n" + ".inst 0xa0402a8e // ld1h { z14.h-z15.h }, pn10.b/Z, [x20]\n" + ".inst 0xc169774c // sdot za.s[x11, 4], { z26.h-z27.h }, z9.h\n" + ".inst 0xc161774d // sdot za.s[x11, 5], { z26.h-z27.h }, z1.h\n" + ".inst 0xa1412aa5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc16b776a // sdot za.s[x11, 2], { z27.h-z28.h }, z11.h\n" + ".inst 0xc16a776b // sdot za.s[x11, 3], { z27.h-z28.h }, z10.h\n" + ".inst 0xa0422ac2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc16d774e // sdot za.s[x11, 6], { z26.h-z27.h }, z13.h\n" + ".inst 0xc165774f // sdot za.s[x11, 7], { z26.h-z27.h }, z5.h\n" + ".inst 0xa0412a88 // ld1h { z8.h-z9.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc163776c // sdot za.s[x11, 4], { z27.h-z28.h }, z3.h\n" + ".inst 0xc162776d // sdot za.s[x11, 5], { z27.h-z28.h }, z2.h\n" + ".inst 0xa1422aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc16a776e // sdot za.s[x11, 6], { z27.h-z28.h }, z10.h\n" + ".inst 0xc162776f // sdot za.s[x11, 7], { z27.h-z28.h }, z2.h\n" + ".inst 0xa0422a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc16f1728 // sdot za.s[x8, 0], { z25.h-z26.h }, z15.h\n" + ".inst 0xc16e1729 // sdot za.s[x8, 1], { z25.h-z26.h }, z14.h\n" + ".inst 0xc1691748 // sdot za.s[x8, 0], { z26.h-z27.h }, z9.h\n" + ".inst 0xc1681749 // sdot za.s[x8, 1], { z26.h-z27.h }, z8.h\n" + ".inst 0xc1611768 // sdot za.s[x8, 0], { z27.h-z28.h }, z1.h\n" ".inst 0xc1601769 // sdot za.s[x8, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xc1651788 // sdot za.s[x8, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xc1641789 // sdot za.s[x8, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xc16b17a8 // sdot za.s[x8, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a17a9 // sdot za.s[x8, 1], { z29.h-z30.h }, z10.h\n" "add x8, x8, #0x2\n" - ".inst 0xc006680c // mova { z12.d-z13.d }, za.d[x11, #0]\n" - ".inst 0xc006682e // mova { z14.d-z15.d }, za.d[x11, #1]\n" - ".inst 0xc1a3ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z3.s\n" + ".inst 0xc0066808 // mova { z8.d-z9.d }, za.d[x11, #0]\n" + ".inst 0xc006682a // mova { z10.d-z11.d }, za.d[x11, #1]\n" + ".inst 0xc1a7ac08 // sqdmulh { z8.s-z11.s }, { z8.s-z11.s }, z7.s\n" "add x11, x11, #0x2\n" - ".inst 0xc1a1aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n" - ".inst 0xc00408c0 // mova za.d[x8, #0], { z6.d-z7.d }\n" - ".inst 0xc1a9ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z9.s\n" - ".inst 0xc00408c1 // mova za.d[x8, #1], { z6.d-z7.d }\n" - ".inst 0xc1bfcf0c // sclamp { z12.s-z15.s }, z24.s, z31.s\n" - "st1b { z12.s }, p1, [x14]\n" - "add x14, x14, x3\n" - "st1b { z14.s }, p1, [x13]\n" + ".inst 0xc1a4aa28 // srshl { z8.s-z11.s }, { z8.s-z11.s }, z4.s\n" + ".inst 0xc0040a40 // mova za.d[x8, #0], { z18.d-z19.d }\n" + ".inst 0xc1acab08 // add { z8.s-z11.s }, { z8.s-z11.s }, z12.s\n" + ".inst 0xc0040a41 // mova za.d[x8, #1], { z18.d-z19.d }\n" + ".inst 0xc1b0cf08 // sclamp { z8.s-z11.s }, z24.s, z16.s\n" + "st1b { z8.s }, p1, [x14]\n" + "add x14, x14, x4\n" + "st1b { z10.s }, p1, [x13]\n" "add x13, x13, x10\n" - "st1b { z13.s }, p1, [x9]\n" + "st1b { z9.s }, p1, [x9]\n" "add x9, x9, x27\n" - "st1b { z15.s }, p1, [x28]\n" + "st1b { z11.s }, p1, [x28]\n" "add x28, x28, x26\n" "22:" // Main loop skip tail "cbz x15, 24f\n" "23:" // Right padding loop - ".inst 0xc006680c // mova { z12.d-z13.d }, za.d[x11, #0]\n" + ".inst 0xc0066808 // mova { z8.d-z9.d }, za.d[x11, #0]\n" "add x8, x8, #0x2\n" "subs x15, x15, #0x1\n" - ".inst 0xc006682e // mova { z14.d-z15.d }, za.d[x11, #1]\n" - ".inst 0xc1a3ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z3.s\n" + ".inst 0xc006682a // mova { z10.d-z11.d }, za.d[x11, #1]\n" + ".inst 0xc1a7ac08 // sqdmulh { z8.s-z11.s }, { z8.s-z11.s }, z7.s\n" "add x11, x11, #0x2\n" - ".inst 0xc1a1aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n" - ".inst 0xc00408c0 // mova za.d[x8, #0], { z6.d-z7.d }\n" - ".inst 0xc1a9ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z9.s\n" - ".inst 0xc00408c1 // mova za.d[x8, #1], { z6.d-z7.d }\n" - ".inst 0xc1bfcf0c // sclamp { z12.s-z15.s }, z24.s, z31.s\n" - "st1b { z12.s }, p1, [x14]\n" - "add x14, x14, x3\n" - "st1b { z14.s }, p1, [x13]\n" + ".inst 0xc1a4aa28 // srshl { z8.s-z11.s }, { z8.s-z11.s }, z4.s\n" + ".inst 0xc0040a40 // mova za.d[x8, #0], { z18.d-z19.d }\n" + ".inst 0xc1acab08 // add { z8.s-z11.s }, { z8.s-z11.s }, z12.s\n" + ".inst 0xc0040a41 // mova za.d[x8, #1], { z18.d-z19.d }\n" + ".inst 0xc1b0cf08 // sclamp { z8.s-z11.s }, z24.s, z16.s\n" + "st1b { z8.s }, p1, [x14]\n" + "add x14, x14, x4\n" + "st1b { z10.s }, p1, [x13]\n" "add x13, x13, x10\n" - "st1b { z13.s }, p1, [x9]\n" + "st1b { z9.s }, p1, [x9]\n" "add x9, x9, x27\n" - "st1b { z15.s }, p1, [x28]\n" + "st1b { z11.s }, p1, [x28]\n" "add x28, x28, x26\n" "bgt 23b\n" "24:" // End - "ldr x23, [%x[args], %[offsetof_Args_weights]]\n" - "incw x23, ALL, MUL #16\n" - "incw x23, ALL, MUL #9\n" - "str x23, [%x[args], %[offsetof_Args_weights]]\n" - "ldr x20, [%x[args], %[offsetof_Args_ld_in_vl]]\n" - "incw x5\n" - "whilelt p1.s, x5, x7\n" - "ldr x16, [%x[args], %[offsetof_Args_inptr]]\n" - "add x16, x16, x20\n" - "str x16, [%x[args], %[offsetof_Args_inptr]]\n" + "ldr x20, [%x[args], %[offsetof_Args_weights]]\n" + "incw x20, ALL, MUL #16\n" + "incw x20, ALL, MUL #9\n" + "str x20, [%x[args], %[offsetof_Args_weights]]\n" + "ldr x21, [%x[args], %[offsetof_Args_ld_in_vl]]\n" + "incw x17\n" + "whilelt p1.s, x17, x7\n" + "ldr x20, [%x[args], %[offsetof_Args_inptr]]\n" + "add x20, x20, x21\n" + "str x20, [%x[args], %[offsetof_Args_inptr]]\n" "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" "ldr x24, [%x[args], %[offsetof_Args_ld_out_vls]]\n" "ldp x23, x22, [x25, #0x0]\n" @@ -1194,7 +1194,7 @@ void sme2_s8q_planar_5x5_s1_4rows_dot_za_impl( ".inst 0xd503467f // SMSTOP\n" : : [args] "r" (&args), [ld_in_col] "r" (ld_in_col), [ld_in_row] "r" (ld_in_row), [offsetof_Args_current_channel] "I" (offsetof(Args, current_channel)), [offsetof_Args_inptr] "I" (offsetof(Args, inptr)), [offsetof_Args_input_cols] "I" (offsetof(Args, input_cols)), [offsetof_Args_ld_in_vl] "I" (offsetof(Args, ld_in_vl)), [offsetof_Args_ld_out_cols] "I" (offsetof(Args, ld_out_cols)), [offsetof_Args_ld_out_vls] "I" (offsetof(Args, ld_out_vls)), [offsetof_Args_n_channels] "I" (offsetof(Args, n_channels)), [offsetof_Args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_Args_output_cols] "I" (offsetof(Args, output_cols)), [offsetof_Args_pad_bottom] "I" (offsetof(Args, pad_bottom)), [offsetof_Args_pad_left] "I" (offsetof(Args, pad_left)), [offsetof_Args_pad_top] "I" (offsetof(Args, pad_top)), [offsetof_Args_weights] "I" (offsetof(Args, weights)), [offsetof_Requantize32_a_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_bias] "I" (offsetof(arm_gemm::Requantize32, bias)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [offsetof_Requantize32_per_channel_muls] "I" (offsetof(arm_gemm::Requantize32, per_channel_muls)), [offsetof_Requantize32_per_channel_right_shifts] "I" (offsetof(arm_gemm::Requantize32, per_channel_right_shifts)), [offsetof_Requantize32_per_layer_mul] "I" (offsetof(arm_gemm::Requantize32, per_layer_mul)), [offsetof_Requantize32_per_layer_right_shift] "I" (offsetof(arm_gemm::Requantize32, per_layer_right_shift)), [qp] "r" (&qp) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_5x5_s2_4rows_dot_za.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_5x5_s2_4rows_dot_za.hpp index e7a781d072772f9cf7873f727d12d530128d2cd5..8bffc05e1fea3a8467d824fc9abe22cbfe9bba3c 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_5x5_s2_4rows_dot_za.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_5x5_s2_4rows_dot_za.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,8 @@ * SOFTWARE. */ +#include "src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp" + namespace arm_conv { namespace depthwise { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_5x5_s2_4rows_dot_za/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_5x5_s2_4rows_dot_za/generic.cpp index 3e8510392f19c322e3e28ba53f56abbc67338a0b..3da0d14d742983f5b2166a6eaf9a8d0e7a456a98 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_5x5_s2_4rows_dot_za/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_s8q_planar_5x5_s2_4rows_dot_za/generic.cpp @@ -73,156 +73,156 @@ void sme2_s8q_planar_5x5_s2_4rows_dot_za_impl( "ptrue p2.b\n" "mov x20, #0xb\n" "ldr x4, [%x[args], %[offsetof_Args_pad_top]]\n" - "ld1rh { z9.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_a_offset]]\n" + "ld1rh { z7.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_a_offset]]\n" "sub x20, x20, x3\n" ".inst 0x25207812 // ptrue pn10.b\n" "ldr x5, [%x[args], %[offsetof_Args_n_channels]]\n" "whilelt p1.s, XZR, x5\n" "whilelt p9.s, XZR, x20\n" - "ld1rw { z8.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" + "ld1rw { z10.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" "whilelt p8.s, XZR, x4\n" "addvl SP, SP, #-15\n" "ldr x6, [%x[args], %[offsetof_Args_current_channel]]\n" - "neg z9.h, p2/M, z9.h\n" + "neg z7.h, p2/M, z7.h\n" "eor p8.b, p2/Z, p8.b, p9.b\n" - "ld1rw { z3.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_mul]]\n" - "ld1rw { z1.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_right_shift]]\n" - "ld1rw { z26.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" - "ld1rw { z23.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" + "ld1rw { z6.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z4.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z5.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" + "ld1rw { z21.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" "1:" // Channel loop "ldr x20, [%x[qp], %[offsetof_Requantize32_bias]]\n" - "mov z28.s, #0x0\n" + "mov z12.s, #0x0\n" "cbz x20, 2f\n" - "ld1w { z28.s }, p1/Z, [x20, x6, LSL #2]\n" + "ld1w { z12.s }, p1/Z, [x20, x6, LSL #2]\n" "2:" // Load bias: Done "ldr x22, [%x[args], %[offsetof_Args_weights]]\n" "mov x20, x22\n" - "ld1sb { z12.s }, p2/Z, [x20]\n" + "ld1sb { z13.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "ld1rh { z18.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" - "sub z12.h, z12.h, z18.h\n" + "ld1rh { z28.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" + "sub z13.h, z13.h, z28.h\n" "incw x22\n" - "mov z14.h, #0x0\n" - "ld1sb { z25.s }, p2/Z, [x20]\n" + "mov z26.h, #0x0\n" + "ld1sb { z22.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z25.h, z25.h, z18.h\n" - "trn1 z2.h, z12.h, z25.h\n" - "ld1sb { z24.s }, p2/Z, [x20]\n" + "sub z22.h, z22.h, z28.h\n" + "trn1 z17.h, z13.h, z22.h\n" + "ld1sb { z20.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z24.h, z24.h, z18.h\n" + "sub z20.h, z20.h, z28.h\n" "addvl x21, SP, #15\n" - "ld1sb { z17.s }, p2/Z, [x20]\n" + "ld1sb { z1.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z17.h, z17.h, z18.h\n" - "trn1 z10.h, z24.h, z17.h\n" - "ld1sb { z16.s }, p2/Z, [x20]\n" + "sub z1.h, z1.h, z28.h\n" + "trn1 z29.h, z20.h, z1.h\n" + "ld1sb { z27.s }, p2/Z, [x20]\n" "mov x20, x22\n" - "sub z16.h, z16.h, z18.h\n" + "sub z27.h, z27.h, z28.h\n" "incw x22\n" - "ld1sb { z12.s }, p2/Z, [x20]\n" + "ld1sb { z14.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z12.h, z12.h, z18.h\n" + "sub z14.h, z14.h, z28.h\n" "addvl x21, x21, #-3\n" - "ld1sb { z25.s }, p2/Z, [x20]\n" + "ld1sb { z18.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z25.h, z25.h, z18.h\n" - "trn1 z0.h, z16.h, z14.h\n" - "ld1sb { z24.s }, p2/Z, [x20]\n" + "sub z18.h, z18.h, z28.h\n" + "trn1 z22.h, z27.h, z26.h\n" + "ld1sb { z23.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z24.h, z24.h, z18.h\n" - "st1h { z2.h }, p2, [x21]\n" - "ld1sb { z17.s }, p2/Z, [x20]\n" + "sub z23.h, z23.h, z28.h\n" + "st1h { z17.h }, p2, [x21]\n" + "ld1sb { z30.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z17.h, z17.h, z18.h\n" - "trn1 z2.h, z12.h, z25.h\n" - "ld1sb { z16.s }, p2/Z, [x20]\n" + "sub z30.h, z30.h, z28.h\n" + "trn1 z8.h, z14.h, z18.h\n" + "ld1sb { z15.s }, p2/Z, [x20]\n" "mov x20, x22\n" - "st1h { z10.h }, p2, [x21, #1, MUL VL]\n" - "sub z16.h, z16.h, z18.h\n" - "ld1sb { z12.s }, p2/Z, [x20]\n" - "incw x20, ALL, MUL #5\n" - "trn1 z10.h, z24.h, z17.h\n" - "sub z12.h, z12.h, z18.h\n" - "ld1sb { z25.s }, p2/Z, [x20]\n" + "st1h { z29.h }, p2, [x21, #1, MUL VL]\n" + "sub z15.h, z15.h, z28.h\n" + "ld1sb { z20.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z25.h, z25.h, z18.h\n" - "st1h { z0.h }, p2, [x21, #2, MUL VL]\n" + "trn1 z23.h, z23.h, z30.h\n" + "sub z20.h, z20.h, z28.h\n" "ld1sb { z24.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "trn1 z0.h, z16.h, z14.h\n" + "sub z24.h, z24.h, z28.h\n" + "st1h { z22.h }, p2, [x21, #2, MUL VL]\n" + "ld1sb { z16.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "trn1 z0.h, z15.h, z26.h\n" "incw x22\n" - "ld1sb { z17.s }, p2/Z, [x20]\n" + "ld1sb { z13.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z24.h, z24.h, z18.h\n" - "sub z17.h, z17.h, z18.h\n" - "ld1sb { z16.s }, p2/Z, [x20]\n" + "sub z16.h, z16.h, z28.h\n" + "sub z13.h, z13.h, z28.h\n" + "ld1sb { z11.s }, p2/Z, [x20]\n" "addvl x21, x21, #-3\n" "mov x20, x22\n" - "st1h { z2.h }, p2, [x21]\n" - "trn1 z2.h, z12.h, z25.h\n" - "ld1sb { z12.s }, p2/Z, [x20]\n" + "st1h { z8.h }, p2, [x21]\n" + "trn1 z27.h, z20.h, z24.h\n" + "ld1sb { z22.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z16.h, z16.h, z18.h\n" - "ld1sb { z25.s }, p2/Z, [x20]\n" + "sub z11.h, z11.h, z28.h\n" + "ld1sb { z3.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "st1h { z10.h }, p2, [x21, #1, MUL VL]\n" - "trn1 z10.h, z24.h, z17.h\n" - "ld1sb { z24.s }, p2/Z, [x20]\n" + "st1h { z23.h }, p2, [x21, #1, MUL VL]\n" + "trn1 z20.h, z16.h, z13.h\n" + "ld1sb { z13.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z12.h, z12.h, z18.h\n" - "sub z25.h, z25.h, z18.h\n" - "ld1sb { z17.s }, p2/Z, [x20]\n" + "sub z22.h, z22.h, z28.h\n" + "sub z3.h, z3.h, z28.h\n" + "ld1sb { z15.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" "st1h { z0.h }, p2, [x21, #2, MUL VL]\n" - "trn1 z0.h, z16.h, z14.h\n" + "trn1 z29.h, z11.h, z26.h\n" "ld1sb { z16.s }, p2/Z, [x20]\n" "incw x22\n" - "sub z24.h, z24.h, z18.h\n" - "sub z17.h, z17.h, z18.h\n" + "sub z13.h, z13.h, z28.h\n" + "sub z15.h, z15.h, z28.h\n" "addvl x21, x21, #-3\n" "mov x20, x22\n" - "st1h { z2.h }, p2, [x21]\n" - "sub z16.h, z16.h, z18.h\n" - "trn1 z2.h, z12.h, z25.h\n" - "ld1sb { z12.s }, p2/Z, [x20]\n" + "st1h { z27.h }, p2, [x21]\n" + "sub z16.h, z16.h, z28.h\n" + "trn1 z19.h, z22.h, z3.h\n" + "ld1sb { z17.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "st1h { z10.h }, p2, [x21, #1, MUL VL]\n" - "ld1sb { z25.s }, p2/Z, [x20]\n" + "st1h { z20.h }, p2, [x21, #1, MUL VL]\n" + "ld1sb { z0.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "trn1 z10.h, z24.h, z17.h\n" - "st1h { z0.h }, p2, [x21, #2, MUL VL]\n" - "ld1sb { z24.s }, p2/Z, [x20]\n" + "trn1 z31.h, z13.h, z15.h\n" + "st1h { z29.h }, p2, [x21, #2, MUL VL]\n" + "ld1sb { z18.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "trn1 z0.h, z16.h, z14.h\n" - "sub z12.h, z12.h, z18.h\n" - "ld1sb { z17.s }, p2/Z, [x20]\n" + "trn1 z16.h, z16.h, z26.h\n" + "sub z17.h, z17.h, z28.h\n" + "ld1sb { z22.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z25.h, z25.h, z18.h\n" - "sub z24.h, z24.h, z18.h\n" - "ld1sb { z16.s }, p2/Z, [x20]\n" - "sub z17.h, z17.h, z18.h\n" - "sub z16.h, z16.h, z18.h\n" + "sub z0.h, z0.h, z28.h\n" + "sub z18.h, z18.h, z28.h\n" + "ld1sb { z1.s }, p2/Z, [x20]\n" + "sub z22.h, z22.h, z28.h\n" + "sub z1.h, z1.h, z28.h\n" "ldr x20, [%x[qp], %[offsetof_Requantize32_per_channel_muls]]\n" "addvl x21, x21, #-3\n" - "st1h { z2.h }, p2, [x21]\n" - "mov z29.d, z28.d\n" - "mov z30.d, z28.d\n" - "st1h { z10.h }, p2, [x21, #1, MUL VL]\n" - "mov z31.d, z28.d\n" - "trn1 z2.h, z12.h, z25.h\n" - "st1h { z0.h }, p2, [x21, #2, MUL VL]\n" + "st1h { z19.h }, p2, [x21]\n" + "mov z13.d, z12.d\n" + "mov z14.d, z12.d\n" + "st1h { z31.h }, p2, [x21, #1, MUL VL]\n" + "mov z15.d, z12.d\n" + "trn1 z8.h, z17.h, z0.h\n" + "st1h { z16.h }, p2, [x21, #2, MUL VL]\n" "addvl x21, x21, #-3\n" - "trn1 z10.h, z24.h, z17.h\n" - "trn1 z0.h, z16.h, z14.h\n" - "st1h { z2.h }, p2, [x21]\n" - "st1h { z10.h }, p2, [x21, #1, MUL VL]\n" - "st1h { z0.h }, p2, [x21, #2, MUL VL]\n" + "trn1 z31.h, z18.h, z22.h\n" + "trn1 z29.h, z1.h, z26.h\n" + "st1h { z8.h }, p2, [x21]\n" + "st1h { z31.h }, p2, [x21, #1, MUL VL]\n" + "st1h { z29.h }, p2, [x21, #2, MUL VL]\n" "cbz x20, 3f\n" - "ld1w { z3.s }, p1/Z, [x20, x6, LSL #2]\n" + "ld1w { z6.s }, p1/Z, [x20, x6, LSL #2]\n" "3:" // Load mul: End "ldr x20, [%x[qp], %[offsetof_Requantize32_per_channel_right_shifts]]\n" "cbz x20, 4f\n" - "ld1w { z1.s }, p1/Z, [x20, x6, LSL #2]\n" + "ld1w { z4.s }, p1/Z, [x20, x6, LSL #2]\n" "4:" // Load right_shift: End "ldr x7, [%x[args], %[offsetof_Args_input_cols]]\n" "sub x20, x7, #0x1\n" @@ -242,20 +242,20 @@ void sme2_s8q_planar_5x5_s2_4rows_dot_za_impl( ".inst 0xf8b74a9c // rprfm pldstrm, x23, [x20]\n" "add x20, x20, %x[ld_in_col]\n" "bgt 5b\n" - "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" + "ldr x23, [%x[args], %[offsetof_Args_outptrs]]\n" "lsl x20, %x[ld_in_row], #0x0\n" "msub x17, x4, x20, x17\n" - ".inst 0xc0040f80 // mova za.d[x8, #0], { z28.d-z31.d }\n" + ".inst 0xc0040d80 // mova za.d[x8, #0], { z12.d-z15.d }\n" "ldr x20, [%x[args], %[offsetof_Args_ld_out_cols]]\n" - ".inst 0xc0040f81 // mova za.d[x8, #1], { z28.d-z31.d }\n" + ".inst 0xc0040d81 // mova za.d[x8, #1], { z12.d-z15.d }\n" "mov x22, #0x4\n" - "ldp x15, x14, [x25], #0x10\n" - ".inst 0xc0040f82 // mova za.d[x8, #2], { z28.d-z31.d }\n" + "ldp x15, x14, [x23], #0x10\n" + ".inst 0xc0040d82 // mova za.d[x8, #2], { z12.d-z15.d }\n" "ldp x13, x11, [x20], #0x10\n" - ".inst 0xc0040f83 // mova za.d[x8, #3], { z28.d-z31.d }\n" + ".inst 0xc0040d83 // mova za.d[x8, #3], { z12.d-z15.d }\n" "ldr x21, [%x[args], %[offsetof_Args_pad_left]]\n" - ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" - "ldp x10, x9, [x25], #0x10\n" + ".inst 0xc0040d84 // mova za.d[x8, #4], { z12.d-z15.d }\n" + "ldp x10, x9, [x23], #0x10\n" "ldp x28, x27, [x20], #0x10\n" "cbz x21, 7f\n" "cmp x21, x22\n" @@ -263,24 +263,24 @@ void sme2_s8q_planar_5x5_s2_4rows_dot_za_impl( "sub x21, x21, x20\n" "sub x22, x22, x20\n" "cbz x21, 7f\n" - ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" - ".inst 0xc1a3ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z3.s\n" + ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" + ".inst 0xc1a6ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z6.s\n" "and x22, x21, #0x1\n" - ".inst 0xc1a1aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" + ".inst 0xc1a4aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" "add x21, x21, #0x1\n" "lsr x21, x21, #0x1\n" - ".inst 0xc1a8ab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z8.s\n" + ".inst 0xc1aaab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z10.s\n" "sub x16, x16, x21\n" - ".inst 0xc1b7cf44 // sclamp { z4.s-z7.s }, z26.s, z23.s\n" + ".inst 0xc1b5ccbc // sclamp { z28.s-z31.s }, z5.s, z21.s\n" "6:" // Left padding "subs x21, x21, #0x1\n" - "st1b { z4.s }, p1, [x15]\n" + "st1b { z28.s }, p1, [x15]\n" "add x15, x15, x13\n" - "st1b { z5.s }, p1, [x14]\n" + "st1b { z29.s }, p1, [x14]\n" "add x14, x14, x11\n" - "st1b { z6.s }, p1, [x10]\n" + "st1b { z30.s }, p1, [x10]\n" "add x10, x10, x28\n" - "st1b { z7.s }, p1, [x9]\n" + "st1b { z31.s }, p1, [x9]\n" "add x9, x9, x27\n" "bgt 6b\n" "7:" // Left padding: End @@ -296,341 +296,341 @@ void sme2_s8q_planar_5x5_s2_4rows_dot_za_impl( "beq 9f\n" "8:" // Unpadded: 4 priming loads "add x21, x17, %x[ld_in_row]\n" - "ld1sb { z11.s }, p1/Z, [x17]\n" + "ld1sb { z27.s }, p1/Z, [x17]\n" "addvl x20, SP, #12\n" - "ld1sb { z21.s }, p1/Z, [x21]\n" + "ld1sb { z0.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "add z11.h, z11.h, z9.h\n" - "ld1sb { z12.s }, p1/Z, [x21]\n" + "trn1 z27.h, z27.h, z0.h\n" + "add z27.h, z27.h, z7.h\n" + "ld1sb { z28.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "add x17, x17, %x[ld_in_col]\n" - "ld1sb { z20.s }, p1/Z, [x21]\n" + "ld1sb { z11.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "add z12.h, z12.h, z9.h\n" - "ld1sb { z13.s }, p1/Z, [x21]\n" + "trn1 z28.h, z28.h, z11.h\n" + "add z28.h, z28.h, z7.h\n" + "ld1sb { z29.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1sb { z19.s }, p1/Z, [x21]\n" + "ld1sb { z8.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z9.h\n" - "ld1sb { z14.s }, p1/Z, [x21]\n" + "trn1 z29.h, z29.h, z8.h\n" + "add z29.h, z29.h, z7.h\n" + "ld1sb { z30.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1sb { z18.s }, p1/Z, [x21]\n" + "ld1sb { z17.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z9.h\n" - "ld1sb { z15.s }, p1/Z, [x21]\n" + "trn1 z30.h, z30.h, z17.h\n" + "add z30.h, z30.h, z7.h\n" + "ld1sb { z31.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1sb { z17.s }, p1/Z, [x21]\n" + "ld1sb { z26.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z15.h, z15.h, z9.h\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1sb { z16.s }, p1/Z, [x21]\n" - "mov z16.d, z16.d\n" - "add z16.h, z16.h, z9.h\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + "trn1 z31.h, z31.h, z26.h\n" + "add z31.h, z31.h, z7.h\n" + ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1701768 // sdot za.s[x8, 0], { z27.h-z30.h }, z0.h\n" + "ld1sb { z20.s }, p1/Z, [x21]\n" + "mov z0.d, z20.d\n" + "add z0.h, z0.h, z7.h\n" + ".inst 0xc1781788 // sdot za.s[x8, 0], { z28.h-z31.h }, z8.h\n" + "ld1h { z8.h }, p2/Z, [x20, #2, MUL VL]\n" + ".inst 0xc17817a8 // sdot za.s[x8, 0], { z29.h-z0.h }, z8.h\n" "9:" // Unpadded: 3 priming loads "add x21, x17, %x[ld_in_row]\n" - "ld1sb { z11.s }, p1/Z, [x17]\n" + "ld1sb { z29.s }, p1/Z, [x17]\n" "addvl x20, SP, #9\n" - "ld1sb { z21.s }, p1/Z, [x21]\n" + "ld1sb { z17.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "add z11.h, z11.h, z9.h\n" - "ld1sb { z12.s }, p1/Z, [x21]\n" + "trn1 z29.h, z29.h, z17.h\n" + "add z29.h, z29.h, z7.h\n" + "ld1sb { z30.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "add x17, x17, %x[ld_in_col]\n" - "ld1sb { z20.s }, p1/Z, [x21]\n" + "ld1sb { z0.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "add z12.h, z12.h, z9.h\n" - "ld1sb { z13.s }, p1/Z, [x21]\n" + "trn1 z30.h, z30.h, z0.h\n" + "add z30.h, z30.h, z7.h\n" + "ld1sb { z31.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1sb { z19.s }, p1/Z, [x21]\n" + "ld1sb { z16.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z9.h\n" - "ld1sb { z14.s }, p1/Z, [x21]\n" + "trn1 z31.h, z31.h, z16.h\n" + "add z31.h, z31.h, z7.h\n" + "ld1sb { z0.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1sb { z18.s }, p1/Z, [x21]\n" + "ld1sb { z16.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z9.h\n" - "ld1sb { z15.s }, p1/Z, [x21]\n" + "trn1 z0.h, z0.h, z16.h\n" + "add z0.h, z0.h, z7.h\n" + "ld1sb { z1.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1sb { z17.s }, p1/Z, [x21]\n" + "ld1sb { z16.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z15.h, z15.h, z9.h\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" + "trn1 z1.h, z1.h, z16.h\n" + "add z1.h, z1.h, z7.h\n" + ".inst 0xa0402a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20]\n" + ".inst 0xc17217a8 // sdot za.s[x8, 0], { z29.h-z0.h }, z2.h\n" "ld1sb { z16.s }, p1/Z, [x21]\n" - "mov z16.d, z16.d\n" - "add z16.h, z16.h, z9.h\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + "mov z2.d, z16.d\n" + "add z2.h, z2.h, z7.h\n" + ".inst 0xc17317c8 // sdot za.s[x8, 0], { z30.h-z1.h }, z3.h\n" + "ld1h { z8.h }, p2/Z, [x20, #2, MUL VL]\n" + ".inst 0xc17817e8 // sdot za.s[x8, 0], { z31.h-z2.h }, z8.h\n" "10:" // Unpadded: 2 priming loads "add x22, x17, %x[ld_in_row]\n" - "ld1sb { z11.s }, p1/Z, [x17]\n" + "ld1sb { z26.s }, p1/Z, [x17]\n" "addvl x21, SP, #6\n" - "ld1sb { z21.s }, p1/Z, [x22]\n" + "ld1sb { z16.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "add z11.h, z11.h, z9.h\n" - "ld1sb { z12.s }, p1/Z, [x22]\n" + "trn1 z26.h, z26.h, z16.h\n" + "add z26.h, z26.h, z7.h\n" + "ld1sb { z27.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "addvl x20, SP, #12\n" - "ld1sb { z20.s }, p1/Z, [x22]\n" + "ld1sb { z16.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "add z12.h, z12.h, z9.h\n" - "ld1sb { z13.s }, p1/Z, [x22]\n" + "trn1 z27.h, z27.h, z16.h\n" + "add z27.h, z27.h, z7.h\n" + "ld1sb { z28.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "add x17, x17, %x[ld_in_col]\n" - "ld1sb { z19.s }, p1/Z, [x22]\n" + "ld1sb { z29.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z9.h\n" - "ld1sb { z14.s }, p1/Z, [x22]\n" + "trn1 z28.h, z28.h, z29.h\n" + "add z28.h, z28.h, z7.h\n" + "ld1sb { z29.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "ld1sb { z18.s }, p1/Z, [x22]\n" + "ld1sb { z19.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z9.h\n" - "ld1sb { z15.s }, p1/Z, [x22]\n" + "trn1 z29.h, z29.h, z19.h\n" + "add z29.h, z29.h, z7.h\n" + "ld1sb { z30.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "ld1sb { z17.s }, p1/Z, [x22]\n" - "trn1 z15.h, z15.h, z17.h\n" + "ld1sb { z23.s }, p1/Z, [x22]\n" + "trn1 z30.h, z30.h, z23.h\n" "add x22, x22, %x[ld_in_row]\n" - "add z15.h, z15.h, z9.h\n" - ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1sb { z16.s }, p1/Z, [x22]\n" - "mov z16.d, z16.d\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" - "add z16.h, z16.h, z9.h\n" - "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + "add z30.h, z30.h, z7.h\n" + ".inst 0xa0402aa2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x21]\n" + ".inst 0xc1721748 // sdot za.s[x8, 0], { z26.h-z29.h }, z2.h\n" + "ld1sb { z22.s }, p1/Z, [x22]\n" + "mov z31.d, z22.d\n" + ".inst 0xc1731768 // sdot za.s[x8, 0], { z27.h-z30.h }, z3.h\n" + ".inst 0xa1402a83 // ld1h { z3.h, z11.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1731749 // sdot za.s[x8, 1], { z26.h-z29.h }, z3.h\n" + "add z31.h, z31.h, z7.h\n" + "ld1h { z3.h }, p2/Z, [x21, #2, MUL VL]\n" + ".inst 0xc17b1769 // sdot za.s[x8, 1], { z27.h-z30.h }, z11.h\n" + ".inst 0xc1731788 // sdot za.s[x8, 0], { z28.h-z31.h }, z3.h\n" "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1701789 // sdot za.s[x8, 1], { z28.h-z31.h }, z0.h\n" "11:" // Unpadded: 1 priming loads "add x22, x17, %x[ld_in_row]\n" - "ld1sb { z11.s }, p1/Z, [x17]\n" + "ld1sb { z29.s }, p1/Z, [x17]\n" "addvl x21, SP, #3\n" - "ld1sb { z21.s }, p1/Z, [x22]\n" + "ld1sb { z22.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "add z11.h, z11.h, z9.h\n" - "ld1sb { z12.s }, p1/Z, [x22]\n" + "trn1 z29.h, z29.h, z22.h\n" + "add z29.h, z29.h, z7.h\n" + "ld1sb { z30.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "addvl x20, SP, #9\n" - "ld1sb { z20.s }, p1/Z, [x22]\n" + "ld1sb { z25.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "add z12.h, z12.h, z9.h\n" - "ld1sb { z13.s }, p1/Z, [x22]\n" + "trn1 z30.h, z30.h, z25.h\n" + "add z30.h, z30.h, z7.h\n" + "ld1sb { z31.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "add x17, x17, %x[ld_in_col]\n" - "ld1sb { z19.s }, p1/Z, [x22]\n" + "ld1sb { z16.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z9.h\n" - "ld1sb { z14.s }, p1/Z, [x22]\n" + "trn1 z31.h, z31.h, z16.h\n" + "add z31.h, z31.h, z7.h\n" + "ld1sb { z0.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "ld1sb { z18.s }, p1/Z, [x22]\n" + "ld1sb { z16.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z9.h\n" - "ld1sb { z15.s }, p1/Z, [x22]\n" + "trn1 z0.h, z0.h, z16.h\n" + "add z0.h, z0.h, z7.h\n" + "ld1sb { z1.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "ld1sb { z17.s }, p1/Z, [x22]\n" - "trn1 z15.h, z15.h, z17.h\n" + "ld1sb { z2.s }, p1/Z, [x22]\n" + "trn1 z1.h, z1.h, z2.h\n" "add x22, x22, %x[ld_in_row]\n" - "add z15.h, z15.h, z9.h\n" - ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1sb { z16.s }, p1/Z, [x22]\n" - "mov z16.d, z16.d\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" - "add z16.h, z16.h, z9.h\n" - "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" - "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" + "add z1.h, z1.h, z7.h\n" + ".inst 0xa0402aa2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x21]\n" + ".inst 0xc17217a8 // sdot za.s[x8, 0], { z29.h-z0.h }, z2.h\n" + "ld1sb { z24.s }, p1/Z, [x22]\n" + "mov z2.d, z24.d\n" + ".inst 0xc17317c8 // sdot za.s[x8, 0], { z30.h-z1.h }, z3.h\n" + ".inst 0xa0402a88 // ld1h { z8.h-z9.h }, pn10.b/Z, [x20]\n" + ".inst 0xc17817a9 // sdot za.s[x8, 1], { z29.h-z0.h }, z8.h\n" + "add z2.h, z2.h, z7.h\n" + "ld1h { z3.h }, p2/Z, [x21, #2, MUL VL]\n" + ".inst 0xc17917c9 // sdot za.s[x8, 1], { z30.h-z1.h }, z9.h\n" + ".inst 0xc17317e8 // sdot za.s[x8, 0], { z31.h-z2.h }, z3.h\n" + "ld1h { z3.h }, p2/Z, [x20, #2, MUL VL]\n" + ".inst 0xc17317e9 // sdot za.s[x8, 1], { z31.h-z2.h }, z3.h\n" "12:" // Unpadded: 0 priming loads "cmp x7, #0x2\n" - ".inst 0xa1402be2 // ld1h { z2.h, z10.h }, pn10.b/Z, [SP]\n" - "ld1h { z0.h }, p2/Z, [SP, #2, MUL VL]\n" + ".inst 0xa1402be3 // ld1h { z3.h, z11.h }, pn10.b/Z, [SP]\n" + "ld1h { z2.h }, p2/Z, [SP, #2, MUL VL]\n" "blt 22f\n" "add x21, x17, %x[ld_in_row]\n" - "ld1sb { z11.s }, p1/Z, [x17]\n" + "ld1sb { z23.s }, p1/Z, [x17]\n" "sub x7, x7, #0x2\n" - "ld1sb { z21.s }, p1/Z, [x21]\n" + "ld1sb { z25.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" + "trn1 z23.h, z23.h, z25.h\n" "sub x16, x16, #0x1\n" - "ld1sb { z12.s }, p1/Z, [x21]\n" + "ld1sb { z24.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "lsr x20, x7, #0x1\n" - "add z11.h, z11.h, z9.h\n" - "ld1sb { z20.s }, p1/Z, [x21]\n" + "add z23.h, z23.h, z7.h\n" + "ld1sb { z30.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z24.h, z24.h, z30.h\n" "cmp x20, x16\n" - "ld1sb { z13.s }, p1/Z, [x21]\n" + "ld1sb { z25.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "csel x26, x20, x16, LT\n" - "add z12.h, z12.h, z9.h\n" - "ld1sb { z19.s }, p1/Z, [x21]\n" + "add z24.h, z24.h, z7.h\n" + "ld1sb { z22.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z9.h\n" - "ld1sb { z14.s }, p1/Z, [x21]\n" + "trn1 z25.h, z25.h, z22.h\n" + "add z25.h, z25.h, z7.h\n" + "ld1sb { z26.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "add x17, x17, %x[ld_in_col]\n" - "ld1sb { z18.s }, p1/Z, [x21]\n" + "ld1sb { z22.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z9.h\n" - "ld1sb { z15.s }, p1/Z, [x21]\n" + "trn1 z26.h, z26.h, z22.h\n" + "add z26.h, z26.h, z7.h\n" + "ld1sb { z27.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "and x7, x7, #0x1\n" - "ld1sb { z17.s }, p1/Z, [x21]\n" + "ld1sb { z30.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z15.h, z15.h, z9.h\n" - "ld1sb { z16.s }, p1/Z, [x21]\n" - "mov z16.d, z16.d\n" - "add z16.h, z16.h, z9.h\n" + "trn1 z27.h, z27.h, z30.h\n" + "add z27.h, z27.h, z7.h\n" + "ld1sb { z28.s }, p1/Z, [x21]\n" + "mov z28.d, z28.d\n" + "add z28.h, z28.h, z7.h\n" "sub x16, x16, x26\n" "cbz x26, 21f\n" "13:" // Unpadded: Main loop - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" + ".inst 0xc17316e8 // sdot za.s[x8, 0], { z23.h-z26.h }, z3.h\n" "addvl x25, SP, #6\n" "addvl x24, SP, #12\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402b22 // ld1h { z2.h, z10.h }, pn10.b/Z, [x25]\n" + ".inst 0xc17b1708 // sdot za.s[x8, 0], { z24.h-z27.h }, z11.h\n" + ".inst 0xa0402b20 // ld1h { z0.h-z1.h }, pn10.b/Z, [x25]\n" "add x23, x17, %x[ld_in_row]\n" "addvl x22, SP, #3\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" + ".inst 0xc17016e9 // sdot za.s[x8, 1], { z23.h-z26.h }, z0.h\n" "addvl x21, SP, #9\n" "subs x26, x26, #0x1\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402b02 // ld1h { z2.h, z10.h }, pn10.b/Z, [x24]\n" - ".inst 0xc172156a // sdot za.s[x8, 2], { z11.h-z14.h }, z2.h\n" - "ld1sb { z11.s }, p1/Z, [x17]\n" + ".inst 0xc1711709 // sdot za.s[x8, 1], { z24.h-z27.h }, z1.h\n" + ".inst 0xa0402b08 // ld1h { z8.h-z9.h }, pn10.b/Z, [x24]\n" + ".inst 0xc17816ea // sdot za.s[x8, 2], { z23.h-z26.h }, z8.h\n" + "ld1sb { z23.s }, p1/Z, [x17]\n" "add x17, x17, %x[ld_in_col]\n" "add x20, x17, %x[ld_in_row]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1721728 // sdot za.s[x8, 0], { z25.h-z28.h }, z2.h\n" "ld1h { z0.h }, p2/Z, [x25, #2, MUL VL]\n" - ".inst 0xc17a158a // sdot za.s[x8, 2], { z12.h-z15.h }, z10.h\n" - "ld1sb { z21.s }, p1/Z, [x23]\n" + ".inst 0xc179170a // sdot za.s[x8, 2], { z24.h-z27.h }, z9.h\n" + "ld1sb { z16.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" - "ld1h { z0.h }, p2/Z, [x24, #2, MUL VL]\n" - "add z11.h, z11.h, z9.h\n" - "ld1sb { z12.s }, p1/Z, [x23]\n" + "trn1 z23.h, z23.h, z16.h\n" + ".inst 0xc1701729 // sdot za.s[x8, 1], { z25.h-z28.h }, z0.h\n" + "ld1h { z9.h }, p2/Z, [x24, #2, MUL VL]\n" + "add z23.h, z23.h, z7.h\n" + "ld1sb { z24.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - ".inst 0xc17015aa // sdot za.s[x8, 2], { z13.h-z16.h }, z0.h\n" - "ld1sb { z20.s }, p1/Z, [x23]\n" + ".inst 0xc179172a // sdot za.s[x8, 2], { z25.h-z28.h }, z9.h\n" + "ld1sb { z18.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "add z12.h, z12.h, z9.h\n" - "ld1sb { z13.s }, p1/Z, [x23]\n" + "trn1 z24.h, z24.h, z18.h\n" + "add z24.h, z24.h, z7.h\n" + "ld1sb { z25.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" + ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" - "ld1sb { z19.s }, p1/Z, [x23]\n" + "ld1sb { z8.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z9.h\n" - "ld1sb { z14.s }, p1/Z, [x23]\n" + "trn1 z25.h, z25.h, z8.h\n" + "add z25.h, z25.h, z7.h\n" + "ld1sb { z26.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - ".inst 0xc1a3ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z3.s\n" - "ld1sb { z18.s }, p1/Z, [x23]\n" + ".inst 0xc1a6ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n" + "ld1sb { z28.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z9.h\n" - "ld1sb { z15.s }, p1/Z, [x23]\n" + "trn1 z26.h, z26.h, z28.h\n" + "add z26.h, z26.h, z7.h\n" + "ld1sb { z27.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - ".inst 0xc1a1aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" - "ld1sb { z17.s }, p1/Z, [x23]\n" - "trn1 z15.h, z15.h, z17.h\n" + ".inst 0xc1a4aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z4.s\n" + "ld1sb { z28.s }, p1/Z, [x23]\n" + "trn1 z27.h, z27.h, z28.h\n" "add x23, x23, %x[ld_in_row]\n" - "add z15.h, z15.h, z9.h\n" - ".inst 0xa1402ac2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x22]\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - ".inst 0xc1a8ab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z8.s\n" - "ld1sb { z16.s }, p1/Z, [x23]\n" - "mov z16.d, z16.d\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" - "add z16.h, z16.h, z9.h\n" + "add z27.h, z27.h, z7.h\n" + ".inst 0xa0402ac2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x22]\n" + ".inst 0xc17216e8 // sdot za.s[x8, 0], { z23.h-z26.h }, z2.h\n" + ".inst 0xc1aaab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z10.s\n" + "ld1sb { z20.s }, p1/Z, [x23]\n" + "mov z28.d, z20.d\n" + ".inst 0xc1731708 // sdot za.s[x8, 0], { z24.h-z27.h }, z3.h\n" + ".inst 0xa0402aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21]\n" + ".inst 0xc17016e9 // sdot za.s[x8, 1], { z23.h-z26.h }, z0.h\n" + "add z28.h, z28.h, z7.h\n" "ld1h { z0.h }, p2/Z, [x22, #2, MUL VL]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xc1b7cf44 // sclamp { z4.s-z7.s }, z26.s, z23.s\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1711709 // sdot za.s[x8, 1], { z24.h-z27.h }, z1.h\n" + ".inst 0xc1b5ccb0 // sclamp { z16.s-z19.s }, z5.s, z21.s\n" + ".inst 0xc1701728 // sdot za.s[x8, 0], { z25.h-z28.h }, z0.h\n" "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" - "st1b { z4.s }, p1, [x15]\n" + "st1b { z16.s }, p1, [x15]\n" "add x15, x15, x13\n" - "ld1sb { z11.s }, p1/Z, [x17]\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" - "st1b { z5.s }, p1, [x14]\n" + "ld1sb { z23.s }, p1/Z, [x17]\n" + ".inst 0xc1701729 // sdot za.s[x8, 1], { z25.h-z28.h }, z0.h\n" + "st1b { z17.s }, p1, [x14]\n" "add x14, x14, x11\n" - "ld1sb { z21.s }, p1/Z, [x20]\n" + "ld1sb { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "st1b { z6.s }, p1, [x10]\n" - "ld1sb { z12.s }, p1/Z, [x20]\n" + "trn1 z23.h, z23.h, z16.h\n" + "st1b { z18.s }, p1, [x10]\n" + "ld1sb { z24.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" "add x10, x10, x28\n" - "st1b { z7.s }, p1, [x9]\n" - "ld1sb { z20.s }, p1/Z, [x20]\n" + "st1b { z19.s }, p1, [x9]\n" + "ld1sb { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z24.h, z24.h, z16.h\n" "add x9, x9, x27\n" - "ld1sb { z13.s }, p1/Z, [x20]\n" + "ld1sb { z25.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" - "add z11.h, z11.h, z9.h\n" - "ld1sb { z19.s }, p1/Z, [x20]\n" + ".inst 0xc0040d84 // mova za.d[x8, #4], { z12.d-z15.d }\n" + "add z23.h, z23.h, z7.h\n" + "ld1sb { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z12.h, z12.h, z9.h\n" - "ld1sb { z14.s }, p1/Z, [x20]\n" + "trn1 z25.h, z25.h, z16.h\n" + "add z24.h, z24.h, z7.h\n" + "ld1sb { z26.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "add z13.h, z13.h, z9.h\n" + "add z25.h, z25.h, z7.h\n" "add x17, x17, %x[ld_in_col]\n" - "ld1sb { z18.s }, p1/Z, [x20]\n" + "ld1sb { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z9.h\n" - "ld1sb { z15.s }, p1/Z, [x20]\n" + "trn1 z26.h, z26.h, z16.h\n" + "add z26.h, z26.h, z7.h\n" + "ld1sb { z27.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "ld1sb { z17.s }, p1/Z, [x20]\n" + "ld1sb { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z15.h, z15.h, z9.h\n" + "trn1 z27.h, z27.h, z16.h\n" + "add z27.h, z27.h, z7.h\n" "ld1sb { z16.s }, p1/Z, [x20]\n" - "mov z16.d, z16.d\n" - "add z16.h, z16.h, z9.h\n" - ".inst 0xa1402be2 // ld1h { z2.h, z10.h }, pn10.b/Z, [SP]\n" - "ld1h { z0.h }, p2/Z, [SP, #2, MUL VL]\n" + "mov z28.d, z16.d\n" + "add z28.h, z28.h, z7.h\n" + ".inst 0xa1402be3 // ld1h { z3.h, z11.h }, pn10.b/Z, [SP]\n" + "ld1h { z2.h }, p2/Z, [SP, #2, MUL VL]\n" "bgt 13b\n" "b 21f\n" "14:" // Padded @@ -645,688 +645,688 @@ void sme2_s8q_planar_5x5_s2_4rows_dot_za_impl( "15:" // Padded: 4 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + "ld1sb { z27.s }, p0/Z, [x17]\n" + "add z27.h, p0/M, z27.h, z7.h\n" "add x21, x17, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z21.s }, p0/Z, [x21]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1sb { z17.s }, p0/Z, [x21]\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z12.s }, p0/Z, [x21]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + "ld1sb { z28.s }, p0/Z, [x21]\n" + "add z28.h, p0/M, z28.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z20.s }, p0/Z, [x21]\n" - "add z20.h, p0/M, z20.h, z9.h\n" + "ld1sb { z16.s }, p0/Z, [x21]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "mov x12, #0x4\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z27.h, z27.h, z17.h\n" + "trn1 z28.h, z28.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z13.s }, p0/Z, [x21]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + "ld1sb { z29.s }, p0/Z, [x21]\n" + "add z29.h, p0/M, z29.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z19.s }, p0/Z, [x21]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "ld1sb { z18.s }, p0/Z, [x21]\n" + "add z18.h, p0/M, z18.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z14.s }, p0/Z, [x21]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1sb { z30.s }, p0/Z, [x21]\n" + "add z30.h, p0/M, z30.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z18.s }, p0/Z, [x21]\n" + "ld1sb { z17.s }, p0/Z, [x21]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z15.s }, p0/Z, [x21]\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1sb { z31.s }, p0/Z, [x21]\n" + "add z31.h, p0/M, z31.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z17.s }, p0/Z, [x21]\n" - "add z17.h, p0/M, z17.h, z9.h\n" + "ld1sb { z16.s }, p0/Z, [x21]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "addvl x20, SP, #12\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" + "trn1 z29.h, z29.h, z18.h\n" + "trn1 z30.h, z30.h, z17.h\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1sb { z16.s }, p0/Z, [x21]\n" - "add z16.h, p0/M, z16.h, z9.h\n" - "mov z16.d, z16.d\n" + ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" + "trn1 z31.h, z31.h, z16.h\n" + ".inst 0xc1701768 // sdot za.s[x8, 0], { z27.h-z30.h }, z0.h\n" + "ld1sb { z20.s }, p0/Z, [x21]\n" + "add z20.h, p0/M, z20.h, z7.h\n" + "mov z0.d, z20.d\n" "add x17, x17, %x[ld_in_col]\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1711788 // sdot za.s[x8, 0], { z28.h-z31.h }, z1.h\n" + "ld1h { z1.h }, p2/Z, [x20, #2, MUL VL]\n" + ".inst 0xc17117a8 // sdot za.s[x8, 0], { z29.h-z0.h }, z1.h\n" "16:" // Padded: 3 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + "ld1sb { z24.s }, p0/Z, [x17]\n" + "add z24.h, p0/M, z24.h, z7.h\n" "add x21, x17, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z21.s }, p0/Z, [x21]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1sb { z17.s }, p0/Z, [x21]\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z12.s }, p0/Z, [x21]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + "ld1sb { z25.s }, p0/Z, [x21]\n" + "add z25.h, p0/M, z25.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z20.s }, p0/Z, [x21]\n" - "add z20.h, p0/M, z20.h, z9.h\n" + "ld1sb { z16.s }, p0/Z, [x21]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "mov x12, #0x4\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z24.h, z24.h, z17.h\n" + "trn1 z25.h, z25.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z13.s }, p0/Z, [x21]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + "ld1sb { z26.s }, p0/Z, [x21]\n" + "add z26.h, p0/M, z26.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z19.s }, p0/Z, [x21]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "ld1sb { z18.s }, p0/Z, [x21]\n" + "add z18.h, p0/M, z18.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z14.s }, p0/Z, [x21]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1sb { z27.s }, p0/Z, [x21]\n" + "add z27.h, p0/M, z27.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z18.s }, p0/Z, [x21]\n" + "ld1sb { z17.s }, p0/Z, [x21]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z15.s }, p0/Z, [x21]\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1sb { z28.s }, p0/Z, [x21]\n" + "add z28.h, p0/M, z28.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z17.s }, p0/Z, [x21]\n" - "add z17.h, p0/M, z17.h, z9.h\n" + "ld1sb { z16.s }, p0/Z, [x21]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "addvl x20, SP, #9\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" + "trn1 z26.h, z26.h, z18.h\n" + "trn1 z27.h, z27.h, z17.h\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1sb { z16.s }, p0/Z, [x21]\n" - "add z16.h, p0/M, z16.h, z9.h\n" - "mov z16.d, z16.d\n" + ".inst 0xa0402a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20]\n" + "trn1 z28.h, z28.h, z16.h\n" + ".inst 0xc1721708 // sdot za.s[x8, 0], { z24.h-z27.h }, z2.h\n" + "ld1sb { z11.s }, p0/Z, [x21]\n" + "add z11.h, p0/M, z11.h, z7.h\n" + "mov z29.d, z11.d\n" "add x17, x17, %x[ld_in_col]\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" + ".inst 0xc1731728 // sdot za.s[x8, 0], { z25.h-z28.h }, z3.h\n" "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1701748 // sdot za.s[x8, 0], { z26.h-z29.h }, z0.h\n" "17:" // Padded: 2 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + "ld1sb { z25.s }, p0/Z, [x17]\n" + "add z25.h, p0/M, z25.h, z7.h\n" "add x20, x17, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z21.s }, p0/Z, [x20]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1sb { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z12.s }, p0/Z, [x20]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + "ld1sb { z26.s }, p0/Z, [x20]\n" + "add z26.h, p0/M, z26.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z9.h\n" + "ld1sb { z16.s }, p0/Z, [x20]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z25.h, z25.h, z17.h\n" + "trn1 z26.h, z26.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + "ld1sb { z27.s }, p0/Z, [x20]\n" + "add z27.h, p0/M, z27.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "ld1sb { z18.s }, p0/Z, [x20]\n" + "add z18.h, p0/M, z18.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1sb { z28.s }, p0/Z, [x20]\n" + "add z28.h, p0/M, z28.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z18.s }, p0/Z, [x20]\n" + "ld1sb { z17.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1sb { z29.s }, p0/Z, [x20]\n" + "add z29.h, p0/M, z29.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z9.h\n" + "ld1sb { z16.s }, p0/Z, [x20]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "addvl x21, SP, #6\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" - ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" + "trn1 z27.h, z27.h, z18.h\n" + "trn1 z28.h, z28.h, z17.h\n" + ".inst 0xa1402aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21]\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1sb { z16.s }, p0/Z, [x20]\n" + "trn1 z29.h, z29.h, z16.h\n" + ".inst 0xc1711728 // sdot za.s[x8, 0], { z25.h-z28.h }, z1.h\n" + "ld1sb { z1.s }, p0/Z, [x20]\n" "addvl x20, SP, #12\n" - "add z16.h, p0/M, z16.h, z9.h\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" - "mov z16.d, z16.d\n" + "add z1.h, p0/M, z1.h, z7.h\n" + ".inst 0xc1791748 // sdot za.s[x8, 0], { z26.h-z29.h }, z9.h\n" + ".inst 0xa0402a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1721729 // sdot za.s[x8, 1], { z25.h-z28.h }, z2.h\n" + "mov z30.d, z1.d\n" "add x17, x17, %x[ld_in_col]\n" - "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + "ld1h { z9.h }, p2/Z, [x21, #2, MUL VL]\n" + ".inst 0xc1731749 // sdot za.s[x8, 1], { z26.h-z29.h }, z3.h\n" + ".inst 0xc1791768 // sdot za.s[x8, 0], { z27.h-z30.h }, z9.h\n" "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1701769 // sdot za.s[x8, 1], { z27.h-z30.h }, z0.h\n" "18:" // Padded: 1 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + "ld1sb { z25.s }, p0/Z, [x17]\n" + "add z25.h, p0/M, z25.h, z7.h\n" "add x20, x17, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z21.s }, p0/Z, [x20]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1sb { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z12.s }, p0/Z, [x20]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + "ld1sb { z26.s }, p0/Z, [x20]\n" + "add z26.h, p0/M, z26.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z9.h\n" + "ld1sb { z16.s }, p0/Z, [x20]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z25.h, z25.h, z17.h\n" + "trn1 z26.h, z26.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + "ld1sb { z27.s }, p0/Z, [x20]\n" + "add z27.h, p0/M, z27.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "ld1sb { z18.s }, p0/Z, [x20]\n" + "add z18.h, p0/M, z18.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1sb { z28.s }, p0/Z, [x20]\n" + "add z28.h, p0/M, z28.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z18.s }, p0/Z, [x20]\n" + "ld1sb { z17.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1sb { z29.s }, p0/Z, [x20]\n" + "add z29.h, p0/M, z29.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z9.h\n" + "ld1sb { z16.s }, p0/Z, [x20]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "addvl x21, SP, #3\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" - ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" + "trn1 z27.h, z27.h, z18.h\n" + "trn1 z28.h, z28.h, z17.h\n" + ".inst 0xa1402aa3 // ld1h { z3.h, z11.h }, pn10.b/Z, [x21]\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1sb { z16.s }, p0/Z, [x20]\n" + "trn1 z29.h, z29.h, z16.h\n" + ".inst 0xc1731728 // sdot za.s[x8, 0], { z25.h-z28.h }, z3.h\n" + "ld1sb { z0.s }, p0/Z, [x20]\n" "addvl x20, SP, #9\n" - "add z16.h, p0/M, z16.h, z9.h\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" - "mov z16.d, z16.d\n" + "add z0.h, p0/M, z0.h, z7.h\n" + ".inst 0xc17b1748 // sdot za.s[x8, 0], { z26.h-z29.h }, z11.h\n" + ".inst 0xa0402a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1721729 // sdot za.s[x8, 1], { z25.h-z28.h }, z2.h\n" + "mov z30.d, z0.d\n" "add x17, x17, %x[ld_in_col]\n" "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1731749 // sdot za.s[x8, 1], { z26.h-z29.h }, z3.h\n" + ".inst 0xc1701768 // sdot za.s[x8, 0], { z27.h-z30.h }, z0.h\n" "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1701769 // sdot za.s[x8, 1], { z27.h-z30.h }, z0.h\n" "19:" // Padded: 0 priming loads "cmp x7, #0x2\n" - ".inst 0xa1402be2 // ld1h { z2.h, z10.h }, pn10.b/Z, [SP]\n" - "ld1h { z0.h }, p2/Z, [SP, #2, MUL VL]\n" + ".inst 0xa1402be3 // ld1h { z3.h, z11.h }, pn10.b/Z, [SP]\n" + "ld1h { z2.h }, p2/Z, [SP, #2, MUL VL]\n" "blt 22f\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + "ld1sb { z23.s }, p0/Z, [x17]\n" + "add z23.h, p0/M, z23.h, z7.h\n" "add x20, x17, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z21.s }, p0/Z, [x20]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1sb { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z12.s }, p0/Z, [x20]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + "ld1sb { z24.s }, p0/Z, [x20]\n" + "add z24.h, p0/M, z24.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z9.h\n" + "ld1sb { z16.s }, p0/Z, [x20]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z23.h, z23.h, z17.h\n" + "trn1 z24.h, z24.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + "ld1sb { z25.s }, p0/Z, [x20]\n" + "add z25.h, p0/M, z25.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1sb { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "add z19.h, p0/M, z19.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1sb { z26.s }, p0/Z, [x20]\n" + "add z26.h, p0/M, z26.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1sb { z18.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z18.h, p0/M, z18.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1sb { z27.s }, p0/Z, [x20]\n" + "add z27.h, p0/M, z27.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1sb { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z9.h\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1sb { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z9.h\n" + "add z16.h, p0/M, z16.h, z7.h\n" "sub x7, x7, #0x2\n" "sub x16, x16, #0x1\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" + "trn1 z25.h, z25.h, z19.h\n" + "trn1 z26.h, z26.h, z18.h\n" "lsr x20, x7, #0x1\n" "cmp x20, x16\n" - "trn1 z15.h, z15.h, z17.h\n" - "mov z16.d, z16.d\n" + "trn1 z27.h, z27.h, z17.h\n" + "mov z28.d, z16.d\n" "csel x25, x20, x16, LT\n" "add x17, x17, %x[ld_in_col]\n" "and x7, x7, #0x1\n" "sub x16, x16, x25\n" "cbz x25, 21f\n" "20:" // Padded: Main loop - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" + ".inst 0xc17316e8 // sdot za.s[x8, 0], { z23.h-z26.h }, z3.h\n" "addvl x24, SP, #6\n" "addvl x23, SP, #12\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402b02 // ld1h { z2.h, z10.h }, pn10.b/Z, [x24]\n" + ".inst 0xc17b1708 // sdot za.s[x8, 0], { z24.h-z27.h }, z11.h\n" + ".inst 0xa1402b00 // ld1h { z0.h, z8.h }, pn10.b/Z, [x24]\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" + ".inst 0xc17016e9 // sdot za.s[x8, 1], { z23.h-z26.h }, z0.h\n" "add x20, x17, %x[ld_in_row]\n" "addvl x22, SP, #3\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402ae2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x23]\n" + ".inst 0xc1781709 // sdot za.s[x8, 1], { z24.h-z27.h }, z8.h\n" + ".inst 0xa1402ae3 // ld1h { z3.h, z11.h }, pn10.b/Z, [x23]\n" "addvl x21, SP, #9\n" "subs x25, x25, #0x1\n" - ".inst 0xc172156a // sdot za.s[x8, 2], { z11.h-z14.h }, z2.h\n" - "ld1sb { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + ".inst 0xc17316ea // sdot za.s[x8, 2], { z23.h-z26.h }, z3.h\n" + "ld1sb { z23.s }, p0/Z, [x17]\n" + "add z23.h, p0/M, z23.h, z7.h\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z21.s }, p0/Z, [x20]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1sb { z16.s }, p0/Z, [x20]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc17a158a // sdot za.s[x8, 2], { z12.h-z15.h }, z10.h\n" - "ld1sb { z12.s }, p0/Z, [x20]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + ".inst 0xc17b170a // sdot za.s[x8, 2], { z24.h-z27.h }, z11.h\n" + "ld1sb { z24.s }, p0/Z, [x20]\n" + "add z24.h, p0/M, z24.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1721728 // sdot za.s[x8, 0], { z25.h-z28.h }, z2.h\n" "ld1h { z0.h }, p2/Z, [x24, #2, MUL VL]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x4\n" - "ld1sb { z20.s }, p0/Z, [x20]\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" - "add z20.h, p0/M, z20.h, z9.h\n" + "ld1sb { z1.s }, p0/Z, [x20]\n" + ".inst 0xc1701729 // sdot za.s[x8, 1], { z25.h-z28.h }, z0.h\n" + "add z1.h, p0/M, z1.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" - "ld1h { z0.h }, p2/Z, [x23, #2, MUL VL]\n" + "ld1h { z3.h }, p2/Z, [x23, #2, MUL VL]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc17015aa // sdot za.s[x8, 2], { z13.h-z16.h }, z0.h\n" - "trn1 z11.h, z11.h, z21.h\n" - "ld1sb { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + ".inst 0xc173172a // sdot za.s[x8, 2], { z25.h-z28.h }, z3.h\n" + "trn1 z23.h, z23.h, z16.h\n" + "ld1sb { z25.s }, p0/Z, [x20]\n" + "add z25.h, p0/M, z25.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "ld1sb { z3.s }, p0/Z, [x20]\n" + "add z3.h, p0/M, z3.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1sb { z26.s }, p0/Z, [x20]\n" + "add z26.h, p0/M, z26.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z18.s }, p0/Z, [x20]\n" + "ld1sb { z30.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z30.h, p0/M, z30.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1sb { z27.s }, p0/Z, [x20]\n" + "add z27.h, p0/M, z27.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z9.h\n" - "trn1 z12.h, z12.h, z20.h\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" - ".inst 0xa1402ac2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x22]\n" + "ld1sb { z29.s }, p0/Z, [x20]\n" + "add z29.h, p0/M, z29.h, z7.h\n" + "trn1 z24.h, z24.h, z1.h\n" + "trn1 z25.h, z25.h, z3.h\n" + "trn1 z26.h, z26.h, z30.h\n" + ".inst 0xa0402ac2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x22]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" + ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" - "trn1 z15.h, z15.h, z17.h\n" + "trn1 z27.h, z27.h, z29.h\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1sb { z16.s }, p0/Z, [x20]\n" + ".inst 0xc17216e8 // sdot za.s[x8, 0], { z23.h-z26.h }, z2.h\n" + "ld1sb { z20.s }, p0/Z, [x20]\n" "mov x12, #0x0\n" - "add z16.h, p0/M, z16.h, z9.h\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" + "add z20.h, p0/M, z20.h, z7.h\n" + ".inst 0xc1731708 // sdot za.s[x8, 0], { z24.h-z27.h }, z3.h\n" + ".inst 0xa0402aa2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x21]\n" "add x17, x17, %x[ld_in_col]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" - "ld1sb { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + ".inst 0xc17216e9 // sdot za.s[x8, 1], { z23.h-z26.h }, z2.h\n" + "ld1sb { z23.s }, p0/Z, [x17]\n" + "add z23.h, p0/M, z23.h, z7.h\n" "add x20, x17, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z21.s }, p0/Z, [x20]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1sb { z8.s }, p0/Z, [x20]\n" + "add z8.h, p0/M, z8.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - "ld1sb { z12.s }, p0/Z, [x20]\n" - "mov z16.d, z16.d\n" - "ld1h { z0.h }, p2/Z, [x22, #2, MUL VL]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + ".inst 0xc1731709 // sdot za.s[x8, 1], { z24.h-z27.h }, z3.h\n" + "ld1sb { z24.s }, p0/Z, [x20]\n" + "mov z28.d, z20.d\n" + "ld1h { z1.h }, p2/Z, [x22, #2, MUL VL]\n" + "add z24.h, p0/M, z24.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z20.s }, p0/Z, [x20]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + "ld1sb { z22.s }, p0/Z, [x20]\n" + ".inst 0xc1711728 // sdot za.s[x8, 0], { z25.h-z28.h }, z1.h\n" "mov x12, #0x4\n" - "add z20.h, p0/M, z20.h, z9.h\n" - "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" + "add z22.h, p0/M, z22.h, z7.h\n" + "ld1h { z1.h }, p2/Z, [x21, #2, MUL VL]\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" - "ld1sb { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + ".inst 0xc1711729 // sdot za.s[x8, 1], { z25.h-z28.h }, z1.h\n" + "ld1sb { z25.s }, p0/Z, [x20]\n" + "add z25.h, p0/M, z25.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "ld1sb { z28.s }, p0/Z, [x20]\n" + "add z28.h, p0/M, z28.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1sb { z26.s }, p0/Z, [x20]\n" + "add z26.h, p0/M, z26.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z18.s }, p0/Z, [x20]\n" + "ld1sb { z20.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z20.h, p0/M, z20.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z15.s }, p0/Z, [x20]\n" - ".inst 0xc1a3ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z3.s\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1sb { z27.s }, p0/Z, [x20]\n" + ".inst 0xc1a6ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n" + "add z27.h, p0/M, z27.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z17.s }, p0/Z, [x20]\n" - ".inst 0xc1a1aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" + "ld1sb { z31.s }, p0/Z, [x20]\n" + ".inst 0xc1a4aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z4.s\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" - "add z17.h, p0/M, z17.h, z9.h\n" + ".inst 0xc0040d84 // mova za.d[x8, #4], { z12.d-z15.d }\n" + "add z31.h, p0/M, z31.h, z7.h\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z9.h\n" - ".inst 0xc1a8ab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z8.s\n" - ".inst 0xa1402be2 // ld1h { z2.h, z10.h }, pn10.b/Z, [SP]\n" + "ld1sb { z1.s }, p0/Z, [x20]\n" + "add z1.h, p0/M, z1.h, z7.h\n" + ".inst 0xc1aaab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z10.s\n" + ".inst 0xa1402be3 // ld1h { z3.h, z11.h }, pn10.b/Z, [SP]\n" "add x17, x17, %x[ld_in_col]\n" - ".inst 0xc1b7cf44 // sclamp { z4.s-z7.s }, z26.s, z23.s\n" - "st1b { z4.s }, p1, [x15]\n" + ".inst 0xc1b5ccb0 // sclamp { z16.s-z19.s }, z5.s, z21.s\n" + "st1b { z16.s }, p1, [x15]\n" "add x15, x15, x13\n" - "ld1h { z0.h }, p2/Z, [SP, #2, MUL VL]\n" - "st1b { z5.s }, p1, [x14]\n" + "ld1h { z2.h }, p2/Z, [SP, #2, MUL VL]\n" + "st1b { z17.s }, p1, [x14]\n" "add x14, x14, x11\n" - "trn1 z11.h, z11.h, z21.h\n" - "trn1 z12.h, z12.h, z20.h\n" - "st1b { z6.s }, p1, [x10]\n" + "trn1 z23.h, z23.h, z8.h\n" + "trn1 z24.h, z24.h, z22.h\n" + "st1b { z18.s }, p1, [x10]\n" "add x10, x10, x28\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" - "st1b { z7.s }, p1, [x9]\n" + "trn1 z25.h, z25.h, z28.h\n" + "trn1 z26.h, z26.h, z20.h\n" + "st1b { z19.s }, p1, [x9]\n" "add x9, x9, x27\n" - "trn1 z15.h, z15.h, z17.h\n" - "mov z16.d, z16.d\n" + "trn1 z27.h, z27.h, z31.h\n" + "mov z28.d, z1.d\n" "bgt 20b\n" "21:" // Main loop tail - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" + ".inst 0xc17316e8 // sdot za.s[x8, 0], { z23.h-z26.h }, z3.h\n" "addvl x24, SP, #6\n" "addvl x23, SP, #12\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402b02 // ld1h { z2.h, z10.h }, pn10.b/Z, [x24]\n" + ".inst 0xc17b1708 // sdot za.s[x8, 0], { z24.h-z27.h }, z11.h\n" + ".inst 0xa0402b08 // ld1h { z8.h-z9.h }, pn10.b/Z, [x24]\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" + ".inst 0xc17816e9 // sdot za.s[x8, 1], { z23.h-z26.h }, z8.h\n" "add x22, x17, %x[ld_in_row]\n" "addvl x21, SP, #3\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402ae2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x23]\n" + ".inst 0xc1791709 // sdot za.s[x8, 1], { z24.h-z27.h }, z9.h\n" + ".inst 0xa1402ae3 // ld1h { z3.h, z11.h }, pn10.b/Z, [x23]\n" "addvl x20, SP, #9\n" - ".inst 0xc172156a // sdot za.s[x8, 2], { z11.h-z14.h }, z2.h\n" - "ld1sb { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + ".inst 0xc17316ea // sdot za.s[x8, 2], { z23.h-z26.h }, z3.h\n" + "ld1sb { z29.s }, p0/Z, [x17]\n" + "add z29.h, p0/M, z29.h, z7.h\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z21.s }, p0/Z, [x22]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1sb { z8.s }, p0/Z, [x22]\n" + "add z8.h, p0/M, z8.h, z7.h\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc17a158a // sdot za.s[x8, 2], { z12.h-z15.h }, z10.h\n" - "ld1sb { z12.s }, p0/Z, [x22]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + ".inst 0xc17b170a // sdot za.s[x8, 2], { z24.h-z27.h }, z11.h\n" + "ld1sb { z30.s }, p0/Z, [x22]\n" + "add z30.h, p0/M, z30.h, z7.h\n" "add x22, x22, %x[ld_in_row]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1721728 // sdot za.s[x8, 0], { z25.h-z28.h }, z2.h\n" "ld1h { z0.h }, p2/Z, [x24, #2, MUL VL]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x4\n" "ld1sb { z20.s }, p0/Z, [x22]\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" - "add z20.h, p0/M, z20.h, z9.h\n" + ".inst 0xc1701729 // sdot za.s[x8, 1], { z25.h-z28.h }, z0.h\n" + "add z20.h, p0/M, z20.h, z7.h\n" "add x22, x22, %x[ld_in_row]\n" - "ld1h { z0.h }, p2/Z, [x23, #2, MUL VL]\n" + "ld1h { z2.h }, p2/Z, [x23, #2, MUL VL]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc17015aa // sdot za.s[x8, 2], { z13.h-z16.h }, z0.h\n" - "trn1 z11.h, z11.h, z21.h\n" - "ld1sb { z13.s }, p0/Z, [x22]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + ".inst 0xc172172a // sdot za.s[x8, 2], { z25.h-z28.h }, z2.h\n" + "trn1 z29.h, z29.h, z8.h\n" + "ld1sb { z31.s }, p0/Z, [x22]\n" + "add z31.h, p0/M, z31.h, z7.h\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z19.s }, p0/Z, [x22]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "ld1sb { z25.s }, p0/Z, [x22]\n" + "add z25.h, p0/M, z25.h, z7.h\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z14.s }, p0/Z, [x22]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1sb { z0.s }, p0/Z, [x22]\n" + "add z0.h, p0/M, z0.h, z7.h\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z18.s }, p0/Z, [x22]\n" + "ld1sb { z17.s }, p0/Z, [x22]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z15.s }, p0/Z, [x22]\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1sb { z1.s }, p0/Z, [x22]\n" + "add z1.h, p0/M, z1.h, z7.h\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z17.s }, p0/Z, [x22]\n" - "add z17.h, p0/M, z17.h, z9.h\n" - "trn1 z12.h, z12.h, z20.h\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" - ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" + "ld1sb { z28.s }, p0/Z, [x22]\n" + "add z28.h, p0/M, z28.h, z7.h\n" + "trn1 z30.h, z30.h, z20.h\n" + "trn1 z31.h, z31.h, z25.h\n" + "trn1 z0.h, z0.h, z17.h\n" + ".inst 0xa1402aa3 // ld1h { z3.h, z11.h }, pn10.b/Z, [x21]\n" "add x22, x22, %x[ld_in_row]\n" - ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" + ".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1sb { z16.s }, p0/Z, [x22]\n" - ".inst 0xc1a3ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z3.s\n" - "add z16.h, p0/M, z16.h, z9.h\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" + "trn1 z1.h, z1.h, z28.h\n" + ".inst 0xc17317a8 // sdot za.s[x8, 0], { z29.h-z0.h }, z3.h\n" + "ld1sb { z22.s }, p0/Z, [x22]\n" + ".inst 0xc1a6ac18 // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z6.s\n" + "add z22.h, p0/M, z22.h, z7.h\n" + ".inst 0xc17b17c8 // sdot za.s[x8, 0], { z30.h-z1.h }, z11.h\n" + ".inst 0xa1402a83 // ld1h { z3.h, z11.h }, pn10.b/Z, [x20]\n" "add x17, x17, %x[ld_in_col]\n" - ".inst 0xc1a1aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" - "mov z16.d, z16.d\n" - "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xc1a8ab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z8.s\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" - "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc1b7cf44 // sclamp { z4.s-z7.s }, z26.s, z23.s\n" - "st1b { z4.s }, p1, [x15]\n" + ".inst 0xc1a4aa38 // srshl { z24.s-z27.s }, { z24.s-z27.s }, z4.s\n" + ".inst 0xc17317a9 // sdot za.s[x8, 1], { z29.h-z0.h }, z3.h\n" + "mov z2.d, z22.d\n" + "ld1h { z9.h }, p2/Z, [x21, #2, MUL VL]\n" + ".inst 0xc17b17c9 // sdot za.s[x8, 1], { z30.h-z1.h }, z11.h\n" + ".inst 0xc1aaab18 // add { z24.s-z27.s }, { z24.s-z27.s }, z10.s\n" + ".inst 0xc17917e8 // sdot za.s[x8, 0], { z31.h-z2.h }, z9.h\n" + "ld1h { z8.h }, p2/Z, [x20, #2, MUL VL]\n" + ".inst 0xc1b5ccb8 // sclamp { z24.s-z27.s }, z5.s, z21.s\n" + "st1b { z24.s }, p1, [x15]\n" "add x15, x15, x13\n" - "st1b { z5.s }, p1, [x14]\n" + "st1b { z25.s }, p1, [x14]\n" "add x14, x14, x11\n" - ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" - ".inst 0xa1402be2 // ld1h { z2.h, z10.h }, pn10.b/Z, [SP]\n" - "st1b { z6.s }, p1, [x10]\n" + ".inst 0xc0040d84 // mova za.d[x8, #4], { z12.d-z15.d }\n" + ".inst 0xa1402be3 // ld1h { z3.h, z11.h }, pn10.b/Z, [SP]\n" + "st1b { z26.s }, p1, [x10]\n" "add x10, x10, x28\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" - "ld1h { z0.h }, p2/Z, [SP, #2, MUL VL]\n" - "st1b { z7.s }, p1, [x9]\n" + ".inst 0xc17817e9 // sdot za.s[x8, 1], { z31.h-z2.h }, z8.h\n" + "ld1h { z2.h }, p2/Z, [SP, #2, MUL VL]\n" + "st1b { z27.s }, p1, [x9]\n" "add x9, x9, x27\n" "22:" // Main loop skip tail "cbz x7, 23f\n" // Skip remainder inputs "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + "ld1sb { z24.s }, p0/Z, [x17]\n" + "add z24.h, p0/M, z24.h, z7.h\n" "add x20, x17, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z21.s }, p0/Z, [x20]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1sb { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z12.s }, p0/Z, [x20]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + "ld1sb { z25.s }, p0/Z, [x20]\n" + "add z25.h, p0/M, z25.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z9.h\n" + "ld1sb { z16.s }, p0/Z, [x20]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z24.h, z24.h, z17.h\n" + "trn1 z25.h, z25.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + "ld1sb { z26.s }, p0/Z, [x20]\n" + "add z26.h, p0/M, z26.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "ld1sb { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1sb { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1sb { z27.s }, p0/Z, [x20]\n" + "add z27.h, p0/M, z27.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1sb { z18.s }, p0/Z, [x20]\n" + "ld1sb { z16.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z16.h, p0/M, z16.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1sb { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1sb { z28.s }, p0/Z, [x20]\n" + "add z28.h, p0/M, z28.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1sb { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z9.h\n" + "ld1sb { z31.s }, p0/Z, [x20]\n" + "add z31.h, p0/M, z31.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" - "ld1sb { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z9.h\n" - "trn1 z15.h, z15.h, z17.h\n" + "trn1 z26.h, z26.h, z17.h\n" + "trn1 z27.h, z27.h, z16.h\n" + "ld1sb { z0.s }, p0/Z, [x20]\n" + "add z0.h, p0/M, z0.h, z7.h\n" + "trn1 z28.h, z28.h, z31.h\n" "addvl x21, SP, #6\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "mov z16.d, z16.d\n" + ".inst 0xc1731708 // sdot za.s[x8, 0], { z24.h-z27.h }, z3.h\n" + "mov z29.d, z0.d\n" "addvl x20, SP, #12\n" "sub x16, x16, #0x1\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" - "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" - ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" - ".inst 0xc1a3ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z3.s\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1a1aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" - ".inst 0xc172156a // sdot za.s[x8, 2], { z11.h-z14.h }, z2.h\n" - ".inst 0xc1a8ab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z8.s\n" - ".inst 0xc17a158a // sdot za.s[x8, 2], { z12.h-z15.h }, z10.h\n" - ".inst 0xc1b7cf44 // sclamp { z4.s-z7.s }, z26.s, z23.s\n" - "st1b { z4.s }, p1, [x15]\n" + ".inst 0xc17b1728 // sdot za.s[x8, 0], { z25.h-z28.h }, z11.h\n" + ".inst 0xa0402aa8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x21]\n" + ".inst 0xc1721748 // sdot za.s[x8, 0], { z26.h-z29.h }, z2.h\n" + "ld1h { z2.h }, p2/Z, [x21, #2, MUL VL]\n" + ".inst 0xc1781709 // sdot za.s[x8, 1], { z24.h-z27.h }, z8.h\n" + ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" + ".inst 0xc1a6ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n" + ".inst 0xc1791729 // sdot za.s[x8, 1], { z25.h-z28.h }, z9.h\n" + ".inst 0xa1402a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1a4aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z4.s\n" + ".inst 0xc171170a // sdot za.s[x8, 2], { z24.h-z27.h }, z1.h\n" + ".inst 0xc1aaab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z10.s\n" + ".inst 0xc179172a // sdot za.s[x8, 2], { z25.h-z28.h }, z9.h\n" + ".inst 0xc1b5ccb0 // sclamp { z16.s-z19.s }, z5.s, z21.s\n" + "st1b { z16.s }, p1, [x15]\n" "add x15, x15, x13\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" - "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - "st1b { z5.s }, p1, [x14]\n" + ".inst 0xc1721749 // sdot za.s[x8, 1], { z26.h-z29.h }, z2.h\n" + "ld1h { z3.h }, p2/Z, [x20, #2, MUL VL]\n" + "st1b { z17.s }, p1, [x14]\n" "add x14, x14, x11\n" - ".inst 0xc17015aa // sdot za.s[x8, 2], { z13.h-z16.h }, z0.h\n" + ".inst 0xc173174a // sdot za.s[x8, 2], { z26.h-z29.h }, z3.h\n" "add x8, x8, #0x1\n" - "st1b { z6.s }, p1, [x10]\n" + "st1b { z18.s }, p1, [x10]\n" "add x10, x10, x28\n" - "st1b { z7.s }, p1, [x9]\n" + "st1b { z19.s }, p1, [x9]\n" "add x9, x9, x27\n" - ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" + ".inst 0xc0040d84 // mova za.d[x8, #4], { z12.d-z15.d }\n" "23:" // Tail input: End "cbz x16, 25f\n" "24:" // Right padding loop - ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" - ".inst 0xc1a3ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z3.s\n" + ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" + ".inst 0xc1a6ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z6.s\n" "add x8, x8, #0x1\n" - ".inst 0xc1a1aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" + ".inst 0xc1a4aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" "subs x16, x16, #0x1\n" - ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" - ".inst 0xc1a8ab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z8.s\n" - ".inst 0xc1b7cf44 // sclamp { z4.s-z7.s }, z26.s, z23.s\n" - "st1b { z4.s }, p1, [x15]\n" + ".inst 0xc0040d84 // mova za.d[x8, #4], { z12.d-z15.d }\n" + ".inst 0xc1aaab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z10.s\n" + ".inst 0xc1b5ccbc // sclamp { z28.s-z31.s }, z5.s, z21.s\n" + "st1b { z28.s }, p1, [x15]\n" "add x15, x15, x13\n" - "st1b { z5.s }, p1, [x14]\n" + "st1b { z29.s }, p1, [x14]\n" "add x14, x14, x11\n" - "st1b { z6.s }, p1, [x10]\n" + "st1b { z30.s }, p1, [x10]\n" "add x10, x10, x28\n" - "st1b { z7.s }, p1, [x9]\n" + "st1b { z31.s }, p1, [x9]\n" "add x9, x9, x27\n" "bgt 24b\n" "25:" // End - "ldr x22, [%x[args], %[offsetof_Args_weights]]\n" - "incw x22, ALL, MUL #16\n" - "incw x22, ALL, MUL #9\n" - "str x22, [%x[args], %[offsetof_Args_weights]]\n" - "ldr x20, [%x[args], %[offsetof_Args_ld_in_vl]]\n" + "ldr x20, [%x[args], %[offsetof_Args_weights]]\n" + "incw x20, ALL, MUL #16\n" + "incw x20, ALL, MUL #9\n" + "str x20, [%x[args], %[offsetof_Args_weights]]\n" + "ldr x21, [%x[args], %[offsetof_Args_ld_in_vl]]\n" "incw x6\n" "whilelt p1.s, x6, x5\n" - "ldr x17, [%x[args], %[offsetof_Args_inptr]]\n" - "add x17, x17, x20\n" - "str x17, [%x[args], %[offsetof_Args_inptr]]\n" + "ldr x20, [%x[args], %[offsetof_Args_inptr]]\n" + "add x20, x20, x21\n" + "str x20, [%x[args], %[offsetof_Args_inptr]]\n" "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" "ldr x24, [%x[args], %[offsetof_Args_ld_out_vls]]\n" "ldp x23, x22, [x25, #0x0]\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_3x3_s1_4rows_dot_za.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_3x3_s1_4rows_dot_za.hpp index 875a9f8294838ce536f5a7308aaef0838c911971..2e40c75d6b7a7465243094b9245c198b105ce7f2 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_3x3_s1_4rows_dot_za.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_3x3_s1_4rows_dot_za.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,8 @@ * SOFTWARE. */ +#include "src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp" + namespace arm_conv { namespace depthwise { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_3x3_s1_4rows_dot_za/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_3x3_s1_4rows_dot_za/generic.cpp index a7ef5568401eb5cab79abc70b606cfbd44a11def..60c3a1e63222accc8434061de6b8e7d4ae9a370f 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_3x3_s1_4rows_dot_za/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_3x3_s1_4rows_dot_za/generic.cpp @@ -73,96 +73,96 @@ void sme2_u8q_planar_3x3_s1_4rows_dot_za_impl( "ptrue p2.b\n" "mov x20, #0x6\n" "ldr x7, [%x[args], %[offsetof_Args_pad_top]]\n" - "ld1rh { z24.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_a_offset]]\n" + "ld1rh { z21.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_a_offset]]\n" "sub x20, x20, x6\n" ".inst 0x25207812 // ptrue pn10.b\n" "ldr x17, [%x[args], %[offsetof_Args_n_channels]]\n" "whilelt p1.s, XZR, x17\n" "whilelt p9.s, XZR, x20\n" - "ld1rw { z12.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" + "ld1rw { z15.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" "whilelt p8.s, XZR, x7\n" "addvl SP, SP, #-12\n" "ldr x16, [%x[args], %[offsetof_Args_current_channel]]\n" - "neg z24.h, p2/M, z24.h\n" + "neg z21.h, p2/M, z21.h\n" "eor p8.b, p2/Z, p8.b, p9.b\n" - "ld1rw { z10.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_mul]]\n" - "ld1rw { z11.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_right_shift]]\n" - "ld1rw { z22.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" - "ld1rw { z26.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" + "ld1rw { z14.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z12.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z29.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" + "ld1rw { z28.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" "1:" // Channel loop "ldr x20, [%x[qp], %[offsetof_Requantize32_bias]]\n" - "mov z8.s, #0x0\n" + "mov z30.s, #0x0\n" "cbz x20, 2f\n" - "ld1w { z8.s }, p1/Z, [x20, x16, LSL #2]\n" + "ld1w { z30.s }, p1/Z, [x20, x16, LSL #2]\n" "2:" // Load bias: Done "ldr x22, [%x[args], %[offsetof_Args_weights]]\n" "mov x20, x22\n" - "ld1b { z27.s }, p2/Z, [x20]\n" + "ld1b { z10.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "ld1rh { z21.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" - "mov z20.h, #0x0\n" - "sub z27.h, z27.h, z21.h\n" + "ld1rh { z31.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" + "mov z7.h, #0x0\n" + "sub z10.h, z10.h, z31.h\n" "incw x22\n" - "ld1b { z23.s }, p2/Z, [x20]\n" - "incw x20, ALL, MUL #3\n" - "sub z23.h, z23.h, z21.h\n" - "trn1 z0.h, z20.h, z27.h\n" "ld1b { z16.s }, p2/Z, [x20]\n" - "sub z16.h, z16.h, z21.h\n" + "incw x20, ALL, MUL #3\n" + "sub z16.h, z16.h, z31.h\n" + "trn1 z20.h, z7.h, z10.h\n" + "ld1b { z11.s }, p2/Z, [x20]\n" + "sub z11.h, z11.h, z31.h\n" "mov x20, x22\n" - "trn1 z1.h, z27.h, z23.h\n" - "ld1b { z27.s }, p2/Z, [x20]\n" + "trn1 z19.h, z10.h, z16.h\n" + "ld1b { z24.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "trn1 z2.h, z23.h, z16.h\n" - "trn1 z3.h, z16.h, z20.h\n" - "ld1b { z23.s }, p2/Z, [x20]\n" + "trn1 z26.h, z16.h, z11.h\n" + "trn1 z13.h, z11.h, z7.h\n" + "ld1b { z11.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "sub z27.h, z27.h, z21.h\n" - "sub z23.h, z23.h, z21.h\n" - "ld1b { z16.s }, p2/Z, [x20]\n" - "sub z16.h, z16.h, z21.h\n" + "sub z24.h, z24.h, z31.h\n" + "sub z11.h, z11.h, z31.h\n" + "ld1b { z2.s }, p2/Z, [x20]\n" + "sub z2.h, z2.h, z31.h\n" "addvl x21, SP, #12\n" "incw x22\n" "addvl x21, x21, #-4\n" "mov x20, x22\n" - "st1h { z0.h }, p2, [x21]\n" - "trn1 z0.h, z20.h, z27.h\n" - "st1h { z1.h }, p2, [x21, #1, MUL VL]\n" - "trn1 z1.h, z27.h, z23.h\n" - "ld1b { z27.s }, p2/Z, [x20]\n" + "st1h { z20.h }, p2, [x21]\n" + "trn1 z22.h, z7.h, z24.h\n" + "st1h { z19.h }, p2, [x21, #1, MUL VL]\n" + "trn1 z1.h, z24.h, z11.h\n" + "ld1b { z16.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "st1h { z2.h }, p2, [x21, #2, MUL VL]\n" - "trn1 z2.h, z23.h, z16.h\n" - "ld1b { z23.s }, p2/Z, [x20]\n" + "st1h { z26.h }, p2, [x21, #2, MUL VL]\n" + "trn1 z3.h, z11.h, z2.h\n" + "ld1b { z0.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "st1h { z3.h }, p2, [x21, #3, MUL VL]\n" - "trn1 z3.h, z16.h, z20.h\n" - "ld1b { z16.s }, p2/Z, [x20]\n" + "st1h { z13.h }, p2, [x21, #3, MUL VL]\n" + "trn1 z25.h, z2.h, z7.h\n" + "ld1b { z4.s }, p2/Z, [x20]\n" "ldr x20, [%x[qp], %[offsetof_Requantize32_per_channel_muls]]\n" - "sub z27.h, z27.h, z21.h\n" - "sub z23.h, z23.h, z21.h\n" + "sub z16.h, z16.h, z31.h\n" + "sub z0.h, z0.h, z31.h\n" "addvl x21, x21, #-4\n" - "st1h { z0.h }, p2, [x21]\n" - "sub z16.h, z16.h, z21.h\n" + "st1h { z22.h }, p2, [x21]\n" + "sub z4.h, z4.h, z31.h\n" "st1h { z1.h }, p2, [x21, #1, MUL VL]\n" - "mov z9.d, z8.d\n" - "st1h { z2.h }, p2, [x21, #2, MUL VL]\n" - "trn1 z0.h, z20.h, z27.h\n" - "trn1 z1.h, z27.h, z23.h\n" - "st1h { z3.h }, p2, [x21, #3, MUL VL]\n" + "mov z31.d, z30.d\n" + "st1h { z3.h }, p2, [x21, #2, MUL VL]\n" + "trn1 z24.h, z7.h, z16.h\n" + "trn1 z18.h, z16.h, z0.h\n" + "st1h { z25.h }, p2, [x21, #3, MUL VL]\n" "addvl x21, x21, #-4\n" - "trn1 z2.h, z23.h, z16.h\n" - "trn1 z3.h, z16.h, z20.h\n" - "st1h { z0.h }, p2, [x21]\n" - "st1h { z1.h }, p2, [x21, #1, MUL VL]\n" - "st1h { z2.h }, p2, [x21, #2, MUL VL]\n" - "st1h { z3.h }, p2, [x21, #3, MUL VL]\n" + "trn1 z0.h, z0.h, z4.h\n" + "trn1 z1.h, z4.h, z7.h\n" + "st1h { z24.h }, p2, [x21]\n" + "st1h { z18.h }, p2, [x21, #1, MUL VL]\n" + "st1h { z0.h }, p2, [x21, #2, MUL VL]\n" + "st1h { z1.h }, p2, [x21, #3, MUL VL]\n" "cbz x20, 3f\n" - "ld1w { z10.s }, p1/Z, [x20, x16, LSL #2]\n" + "ld1w { z14.s }, p1/Z, [x20, x16, LSL #2]\n" "3:" // Load mul: End "ldr x20, [%x[qp], %[offsetof_Requantize32_per_channel_right_shifts]]\n" "cbz x20, 4f\n" - "ld1w { z11.s }, p1/Z, [x20, x16, LSL #2]\n" + "ld1w { z12.s }, p1/Z, [x20, x16, LSL #2]\n" "4:" // Load right_shift: End "ldr x15, [%x[args], %[offsetof_Args_input_cols]]\n" "sub x20, x15, #0x1\n" @@ -182,21 +182,21 @@ void sme2_u8q_planar_3x3_s1_4rows_dot_za_impl( ".inst 0xf8b74a9c // rprfm pldstrm, x23, [x20]\n" "add x20, x20, %x[ld_in_col]\n" "bgt 5b\n" - "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" + "ldr x23, [%x[args], %[offsetof_Args_outptrs]]\n" "lsl x20, %x[ld_in_row], #0x0\n" "msub x14, x7, x20, x14\n" - ".inst 0xc0040900 // mova za.d[x8, #0], { z8.d-z9.d }\n" + ".inst 0xc0040bc0 // mova za.d[x8, #0], { z30.d-z31.d }\n" "ldr x20, [%x[args], %[offsetof_Args_ld_out_cols]]\n" - ".inst 0xc0040901 // mova za.d[x8, #1], { z8.d-z9.d }\n" + ".inst 0xc0040bc1 // mova za.d[x8, #1], { z30.d-z31.d }\n" "mov x22, #0x2\n" - "ldp x11, x10, [x25], #0x10\n" - ".inst 0xc0040902 // mova za.d[x8, #2], { z8.d-z9.d }\n" + "ldp x11, x10, [x23], #0x10\n" + ".inst 0xc0040bc2 // mova za.d[x8, #2], { z30.d-z31.d }\n" "ldp x9, x28, [x20], #0x10\n" - ".inst 0xc0040903 // mova za.d[x8, #3], { z8.d-z9.d }\n" + ".inst 0xc0040bc3 // mova za.d[x8, #3], { z30.d-z31.d }\n" "ldr x21, [%x[args], %[offsetof_Args_pad_left]]\n" - ".inst 0xc0040904 // mova za.d[x8, #4], { z8.d-z9.d }\n" - "ldp x27, x26, [x25], #0x10\n" - ".inst 0xc0040905 // mova za.d[x8, #5], { z8.d-z9.d }\n" + ".inst 0xc0040bc4 // mova za.d[x8, #4], { z30.d-z31.d }\n" + "ldp x27, x26, [x23], #0x10\n" + ".inst 0xc0040bc5 // mova za.d[x8, #5], { z30.d-z31.d }\n" "ldp x25, x24, [x20], #0x10\n" "cbz x21, 7f\n" "cmp x21, x22\n" @@ -204,22 +204,22 @@ void sme2_u8q_planar_3x3_s1_4rows_dot_za_impl( "sub x21, x21, x20\n" "sub x22, x22, x20\n" "cbz x21, 7f\n" - ".inst 0xc0060804 // mova { z4.d-z5.d }, za.d[x8, #0]\n" + ".inst 0xc0060818 // mova { z24.d-z25.d }, za.d[x8, #0]\n" "sub x13, x13, x21\n" - ".inst 0xc0060826 // mova { z6.d-z7.d }, za.d[x8, #1]\n" - ".inst 0xc1aaac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z10.s\n" - ".inst 0xc1abaa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z11.s\n" - ".inst 0xc1acab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z12.s\n" - ".inst 0xc1bacec4 // sclamp { z4.s-z7.s }, z22.s, z26.s\n" + ".inst 0xc006083a // mova { z26.d-z27.d }, za.d[x8, #1]\n" + ".inst 0xc1aeac18 // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z14.s\n" + ".inst 0xc1acaa38 // srshl { z24.s-z27.s }, { z24.s-z27.s }, z12.s\n" + ".inst 0xc1afab18 // add { z24.s-z27.s }, { z24.s-z27.s }, z15.s\n" + ".inst 0xc1bccfb8 // sclamp { z24.s-z27.s }, z29.s, z28.s\n" "6:" // Left padding "subs x21, x21, #0x1\n" - "st1b { z4.s }, p1, [x11]\n" + "st1b { z24.s }, p1, [x11]\n" "add x11, x11, x9\n" - "st1b { z6.s }, p1, [x10]\n" + "st1b { z26.s }, p1, [x10]\n" "add x10, x10, x28\n" - "st1b { z5.s }, p1, [x27]\n" + "st1b { z25.s }, p1, [x27]\n" "add x27, x27, x25\n" - "st1b { z7.s }, p1, [x26]\n" + "st1b { z27.s }, p1, [x26]\n" "add x26, x26, x24\n" "bgt 6b\n" "7:" // Left padding: End @@ -231,148 +231,148 @@ void sme2_u8q_planar_3x3_s1_4rows_dot_za_impl( "beq 9f\n" "8:" // Unpadded: 2 priming loads "add x21, x14, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x14]\n" + "ld1b { z20.s }, p1/Z, [x14]\n" "addvl x20, SP, #8\n" "ld1b { z16.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z17.h, z16.h\n" - "add z13.h, z13.h, z24.h\n" - "ld1b { z17.s }, p1/Z, [x21]\n" + "trn1 z4.h, z20.h, z16.h\n" + "add z4.h, z4.h, z21.h\n" + "ld1b { z23.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "add x14, x14, %x[ld_in_col]\n" - "ld1b { z16.s }, p1/Z, [x21]\n" + "ld1b { z22.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z14.h, z17.h, z16.h\n" - "add z14.h, z14.h, z24.h\n" + "trn1 z5.h, z23.h, z22.h\n" + "add z5.h, z5.h, z21.h\n" "ld1b { z17.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "ld1b { z16.s }, p1/Z, [x21]\n" - "trn1 z15.h, z17.h, z16.h\n" - "add z15.h, z15.h, z24.h\n" - ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" - ".inst 0xc16115a8 // sdot za.s[x8, 0], { z13.h-z14.h }, z1.h\n" - ".inst 0xc16015a9 // sdot za.s[x8, 1], { z13.h-z14.h }, z0.h\n" - ".inst 0xa0412a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16315c8 // sdot za.s[x8, 0], { z14.h-z15.h }, z3.h\n" - ".inst 0xc16215c9 // sdot za.s[x8, 1], { z14.h-z15.h }, z2.h\n" + "trn1 z6.h, z17.h, z16.h\n" + "add z6.h, z6.h, z21.h\n" + ".inst 0xa1402a83 // ld1h { z3.h, z11.h }, pn10.b/Z, [x20]\n" + ".inst 0xc16b1488 // sdot za.s[x8, 0], { z4.h-z5.h }, z11.h\n" + ".inst 0xc1631489 // sdot za.s[x8, 1], { z4.h-z5.h }, z3.h\n" + ".inst 0xa1412a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc16814a8 // sdot za.s[x8, 0], { z5.h-z6.h }, z8.h\n" + ".inst 0xc16014a9 // sdot za.s[x8, 1], { z5.h-z6.h }, z0.h\n" "9:" // Unpadded: 1 priming loads "add x22, x14, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x14]\n" + "ld1b { z25.s }, p1/Z, [x14]\n" "addvl x21, SP, #4\n" - "ld1b { z16.s }, p1/Z, [x22]\n" + "ld1b { z6.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z13.h, z17.h, z16.h\n" - "add z13.h, z13.h, z24.h\n" - "ld1b { z17.s }, p1/Z, [x22]\n" + "trn1 z3.h, z25.h, z6.h\n" + "add z3.h, z3.h, z21.h\n" + "ld1b { z18.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "addvl x20, SP, #8\n" - "ld1b { z16.s }, p1/Z, [x22]\n" + "ld1b { z26.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z14.h, z17.h, z16.h\n" - "add z14.h, z14.h, z24.h\n" - "ld1b { z17.s }, p1/Z, [x22]\n" + "trn1 z4.h, z18.h, z26.h\n" + "add z4.h, z4.h, z21.h\n" + "ld1b { z2.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "add x14, x14, %x[ld_in_col]\n" - "ld1b { z16.s }, p1/Z, [x22]\n" - "trn1 z15.h, z17.h, z16.h\n" - "add z15.h, z15.h, z24.h\n" + "ld1b { z5.s }, p1/Z, [x22]\n" + "trn1 z5.h, z2.h, z5.h\n" + "add z5.h, z5.h, z21.h\n" ".inst 0xa0402aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21]\n" - ".inst 0xc16115a8 // sdot za.s[x8, 0], { z13.h-z14.h }, z1.h\n" - ".inst 0xc16015a9 // sdot za.s[x8, 1], { z13.h-z14.h }, z0.h\n" - ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" - ".inst 0xa0412aa2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc16115aa // sdot za.s[x8, 2], { z13.h-z14.h }, z1.h\n" - ".inst 0xc16015ab // sdot za.s[x8, 3], { z13.h-z14.h }, z0.h\n" - ".inst 0xc16315c8 // sdot za.s[x8, 0], { z14.h-z15.h }, z3.h\n" - ".inst 0xc16215c9 // sdot za.s[x8, 1], { z14.h-z15.h }, z2.h\n" - ".inst 0xa0412a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16315ca // sdot za.s[x8, 2], { z14.h-z15.h }, z3.h\n" - ".inst 0xc16215cb // sdot za.s[x8, 3], { z14.h-z15.h }, z2.h\n" + ".inst 0xc1611468 // sdot za.s[x8, 0], { z3.h-z4.h }, z1.h\n" + ".inst 0xc1601469 // sdot za.s[x8, 1], { z3.h-z4.h }, z0.h\n" + ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" + ".inst 0xa0412aa8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc16a146a // sdot za.s[x8, 2], { z3.h-z4.h }, z10.h\n" + ".inst 0xc162146b // sdot za.s[x8, 3], { z3.h-z4.h }, z2.h\n" + ".inst 0xc1691488 // sdot za.s[x8, 0], { z4.h-z5.h }, z9.h\n" + ".inst 0xc1681489 // sdot za.s[x8, 1], { z4.h-z5.h }, z8.h\n" + ".inst 0xa1412a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc16a148a // sdot za.s[x8, 2], { z4.h-z5.h }, z10.h\n" + ".inst 0xc162148b // sdot za.s[x8, 3], { z4.h-z5.h }, z2.h\n" "10:" // Unpadded: 0 priming loads - ".inst 0xa0402be0 // ld1h { z0.h-z1.h }, pn10.b/Z, [SP]\n" - ".inst 0xa0412be2 // ld1h { z2.h-z3.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + ".inst 0xa1402be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP]\n" + ".inst 0xa0412bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" "cbz x15, 18f\n" "add x20, x14, %x[ld_in_row]\n" "ld1b { z17.s }, p1/Z, [x14]\n" "sub x15, x15, #0x1\n" - "ld1b { z16.s }, p1/Z, [x20]\n" + "ld1b { z9.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z13.h, z17.h, z16.h\n" + "trn1 z6.h, z17.h, z9.h\n" "sub x13, x13, #0x1\n" "ld1b { z17.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" "cmp x15, x13\n" - "add z13.h, z13.h, z24.h\n" - "ld1b { z16.s }, p1/Z, [x20]\n" + "add z6.h, z6.h, z21.h\n" + "ld1b { z7.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z14.h, z17.h, z16.h\n" + "trn1 z7.h, z17.h, z7.h\n" "csel x23, x15, x13, LT\n" "ld1b { z17.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "add z14.h, z14.h, z24.h\n" + "add z7.h, z7.h, z21.h\n" "add x14, x14, %x[ld_in_col]\n" - "ld1b { z16.s }, p1/Z, [x20]\n" - "trn1 z15.h, z17.h, z16.h\n" - "add z15.h, z15.h, z24.h\n" + "ld1b { z1.s }, p1/Z, [x20]\n" + "trn1 z8.h, z17.h, z1.h\n" + "add z8.h, z8.h, z21.h\n" "sub x13, x13, x23\n" "cbz x23, 17f\n" "11:" // Unpadded: Main loop - ".inst 0xc16115a8 // sdot za.s[x8, 0], { z13.h-z14.h }, z1.h\n" + ".inst 0xc16d14c8 // sdot za.s[x8, 0], { z6.h-z7.h }, z13.h\n" "addvl x22, SP, #4\n" "addvl x21, SP, #8\n" - "ld1b { z21.s }, p1/Z, [x14]\n" - ".inst 0xc16015a9 // sdot za.s[x8, 1], { z13.h-z14.h }, z0.h\n" - ".inst 0xa0402ac0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x22]\n" + "ld1b { z2.s }, p1/Z, [x14]\n" + ".inst 0xc16514c9 // sdot za.s[x8, 1], { z6.h-z7.h }, z5.h\n" + ".inst 0xa1402ac5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x22]\n" "add x20, x14, %x[ld_in_row]\n" "subs x23, x23, #0x1\n" - ".inst 0xc16315c8 // sdot za.s[x8, 0], { z14.h-z15.h }, z3.h\n" - "ld1b { z20.s }, p1/Z, [x20]\n" + ".inst 0xc16b14e8 // sdot za.s[x8, 0], { z7.h-z8.h }, z11.h\n" + "ld1b { z19.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" "add x14, x14, %x[ld_in_col]\n" - ".inst 0xc16215c9 // sdot za.s[x8, 1], { z14.h-z15.h }, z2.h\n" - ".inst 0xa0412ac2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xc0060804 // mova { z4.d-z5.d }, za.d[x8, #0]\n" - "ld1b { z19.s }, p1/Z, [x20]\n" + ".inst 0xc16a14e9 // sdot za.s[x8, 1], { z7.h-z8.h }, z10.h\n" + ".inst 0xa1412ac3 // ld1h { z3.h, z11.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc0060818 // mova { z24.d-z25.d }, za.d[x8, #0]\n" + "ld1b { z23.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc0060826 // mova { z6.d-z7.d }, za.d[x8, #1]\n" - ".inst 0xc1aaac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z10.s\n" - ".inst 0xc16115aa // sdot za.s[x8, 2], { z13.h-z14.h }, z1.h\n" + ".inst 0xc006083a // mova { z26.d-z27.d }, za.d[x8, #1]\n" + ".inst 0xc1aeac18 // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z14.s\n" + ".inst 0xc16d14ca // sdot za.s[x8, 2], { z6.h-z7.h }, z13.h\n" "ld1b { z18.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc16015ab // sdot za.s[x8, 3], { z13.h-z14.h }, z0.h\n" - ".inst 0xa0402aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1abaa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z11.s\n" - ".inst 0xc16115ac // sdot za.s[x8, 4], { z13.h-z14.h }, z1.h\n" + ".inst 0xc16514cb // sdot za.s[x8, 3], { z6.h-z7.h }, z5.h\n" + ".inst 0xa1402aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21]\n" + ".inst 0xc1acaa38 // srshl { z24.s-z27.s }, { z24.s-z27.s }, z12.s\n" + ".inst 0xc16914cc // sdot za.s[x8, 4], { z6.h-z7.h }, z9.h\n" "ld1b { z17.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc16015ad // sdot za.s[x8, 5], { z13.h-z14.h }, z0.h\n" + ".inst 0xc16114cd // sdot za.s[x8, 5], { z6.h-z7.h }, z1.h\n" "ld1b { z16.s }, p1/Z, [x20]\n" - ".inst 0xc1acab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z12.s\n" - ".inst 0xc16315ca // sdot za.s[x8, 2], { z14.h-z15.h }, z3.h\n" - "trn1 z13.h, z21.h, z20.h\n" - ".inst 0xa0402be0 // ld1h { z0.h-z1.h }, pn10.b/Z, [SP]\n" - ".inst 0xc16215cb // sdot za.s[x8, 3], { z14.h-z15.h }, z2.h\n" - ".inst 0xa0412aa2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc1bacec4 // sclamp { z4.s-z7.s }, z22.s, z26.s\n" - ".inst 0xc16315cc // sdot za.s[x8, 4], { z14.h-z15.h }, z3.h\n" - "st1b { z4.s }, p1, [x11]\n" + ".inst 0xc1afab18 // add { z24.s-z27.s }, { z24.s-z27.s }, z15.s\n" + ".inst 0xc16b14ea // sdot za.s[x8, 2], { z7.h-z8.h }, z11.h\n" + "trn1 z6.h, z2.h, z19.h\n" + ".inst 0xa1402be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP]\n" + ".inst 0xc16314eb // sdot za.s[x8, 3], { z7.h-z8.h }, z3.h\n" + ".inst 0xa1412aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc1bccfb8 // sclamp { z24.s-z27.s }, z29.s, z28.s\n" + ".inst 0xc16914ec // sdot za.s[x8, 4], { z7.h-z8.h }, z9.h\n" + "st1b { z24.s }, p1, [x11]\n" "add x11, x11, x9\n" - "add z13.h, z13.h, z24.h\n" - ".inst 0xc16215cd // sdot za.s[x8, 5], { z14.h-z15.h }, z2.h\n" - "trn1 z14.h, z19.h, z18.h\n" - "trn1 z15.h, z17.h, z16.h\n" + "add z6.h, z6.h, z21.h\n" + ".inst 0xc16114ed // sdot za.s[x8, 5], { z7.h-z8.h }, z1.h\n" + "trn1 z7.h, z23.h, z18.h\n" + "trn1 z8.h, z17.h, z16.h\n" "add x8, x8, #0x2\n" - ".inst 0xa0412be2 // ld1h { z2.h-z3.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" - "st1b { z6.s }, p1, [x10]\n" + ".inst 0xa0412bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + "st1b { z26.s }, p1, [x10]\n" "add x10, x10, x28\n" - ".inst 0xc0040904 // mova za.d[x8, #4], { z8.d-z9.d }\n" - "st1b { z5.s }, p1, [x27]\n" + ".inst 0xc0040bc4 // mova za.d[x8, #4], { z30.d-z31.d }\n" + "st1b { z25.s }, p1, [x27]\n" "add x27, x27, x25\n" - ".inst 0xc0040905 // mova za.d[x8, #5], { z8.d-z9.d }\n" - "add z14.h, z14.h, z24.h\n" - "st1b { z7.s }, p1, [x26]\n" + ".inst 0xc0040bc5 // mova za.d[x8, #5], { z30.d-z31.d }\n" + "add z7.h, z7.h, z21.h\n" + "st1b { z27.s }, p1, [x26]\n" "add x26, x26, x24\n" - "add z15.h, z15.h, z24.h\n" + "add z8.h, z8.h, z21.h\n" "bgt 11b\n" "b 17f\n" "12:" // Padded @@ -384,118 +384,118 @@ void sme2_u8q_planar_3x3_s1_4rows_dot_za_impl( "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1b { z19.s }, p0/Z, [x14]\n" - "add z19.h, p0/M, z19.h, z24.h\n" + "add z19.h, p0/M, z19.h, z21.h\n" "add x20, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z24.h\n" + "add z18.h, p0/M, z18.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z24.h\n" + "add z17.h, p0/M, z17.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z24.h\n" + "add z16.h, p0/M, z16.h, z21.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z13.h, z19.h, z18.h\n" - "trn1 z14.h, z17.h, z16.h\n" + "trn1 z7.h, z19.h, z18.h\n" + "trn1 z8.h, z17.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z24.h\n" + "add z17.h, p0/M, z17.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1b { z16.s }, p0/Z, [x20]\n" "addvl x20, SP, #8\n" - "add z16.h, p0/M, z16.h, z24.h\n" - ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" - "trn1 z15.h, z17.h, z16.h\n" - ".inst 0xc16115a8 // sdot za.s[x8, 0], { z13.h-z14.h }, z1.h\n" + "add z16.h, p0/M, z16.h, z21.h\n" + ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" + "trn1 z9.h, z17.h, z16.h\n" + ".inst 0xc16a14e8 // sdot za.s[x8, 0], { z7.h-z8.h }, z10.h\n" "add x14, x14, %x[ld_in_col]\n" - ".inst 0xc16015a9 // sdot za.s[x8, 1], { z13.h-z14.h }, z0.h\n" - ".inst 0xa0412a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16315c8 // sdot za.s[x8, 0], { z14.h-z15.h }, z3.h\n" - ".inst 0xc16215c9 // sdot za.s[x8, 1], { z14.h-z15.h }, z2.h\n" + ".inst 0xc16214e9 // sdot za.s[x8, 1], { z7.h-z8.h }, z2.h\n" + ".inst 0xa1412a85 // ld1h { z5.h, z13.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc16d1508 // sdot za.s[x8, 0], { z8.h-z9.h }, z13.h\n" + ".inst 0xc1651509 // sdot za.s[x8, 1], { z8.h-z9.h }, z5.h\n" "14:" // Padded: 1 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1b { z19.s }, p0/Z, [x14]\n" - "add z19.h, p0/M, z19.h, z24.h\n" + "add z19.h, p0/M, z19.h, z21.h\n" "add x20, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z24.h\n" + "add z18.h, p0/M, z18.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z24.h\n" + "add z17.h, p0/M, z17.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z24.h\n" + "add z16.h, p0/M, z16.h, z21.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z13.h, z19.h, z18.h\n" - "trn1 z14.h, z17.h, z16.h\n" + "trn1 z22.h, z19.h, z18.h\n" + "trn1 z23.h, z17.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z24.h\n" + "add z17.h, p0/M, z17.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1b { z16.s }, p0/Z, [x20]\n" "addvl x21, SP, #4\n" - "add z16.h, p0/M, z16.h, z24.h\n" + "add z16.h, p0/M, z16.h, z21.h\n" ".inst 0xa0402aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21]\n" "addvl x20, SP, #8\n" - "trn1 z15.h, z17.h, z16.h\n" - ".inst 0xc16115a8 // sdot za.s[x8, 0], { z13.h-z14.h }, z1.h\n" - ".inst 0xc16015a9 // sdot za.s[x8, 1], { z13.h-z14.h }, z0.h\n" - ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" + "trn1 z24.h, z17.h, z16.h\n" + ".inst 0xc16116c8 // sdot za.s[x8, 0], { z22.h-z23.h }, z1.h\n" + ".inst 0xc16016c9 // sdot za.s[x8, 1], { z22.h-z23.h }, z0.h\n" + ".inst 0xa1402a85 // ld1h { z5.h, z13.h }, pn10.b/Z, [x20]\n" "add x14, x14, %x[ld_in_col]\n" - ".inst 0xa0412aa2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc16115aa // sdot za.s[x8, 2], { z13.h-z14.h }, z1.h\n" - ".inst 0xc16015ab // sdot za.s[x8, 3], { z13.h-z14.h }, z0.h\n" - ".inst 0xc16315c8 // sdot za.s[x8, 0], { z14.h-z15.h }, z3.h\n" - ".inst 0xc16215c9 // sdot za.s[x8, 1], { z14.h-z15.h }, z2.h\n" - ".inst 0xa0412a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16315ca // sdot za.s[x8, 2], { z14.h-z15.h }, z3.h\n" - ".inst 0xc16215cb // sdot za.s[x8, 3], { z14.h-z15.h }, z2.h\n" + ".inst 0xa0412aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc16d16ca // sdot za.s[x8, 2], { z22.h-z23.h }, z13.h\n" + ".inst 0xc16516cb // sdot za.s[x8, 3], { z22.h-z23.h }, z5.h\n" + ".inst 0xc16116e8 // sdot za.s[x8, 0], { z23.h-z24.h }, z1.h\n" + ".inst 0xc16016e9 // sdot za.s[x8, 1], { z23.h-z24.h }, z0.h\n" + ".inst 0xa0412a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc16116ea // sdot za.s[x8, 2], { z23.h-z24.h }, z1.h\n" + ".inst 0xc16016eb // sdot za.s[x8, 3], { z23.h-z24.h }, z0.h\n" "15:" // Padded: 0 priming loads - ".inst 0xa0402be0 // ld1h { z0.h-z1.h }, pn10.b/Z, [SP]\n" - ".inst 0xa0412be2 // ld1h { z2.h-z3.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + ".inst 0xa1402be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP]\n" + ".inst 0xa0412bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" "cbz x15, 18f\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1b { z19.s }, p0/Z, [x14]\n" - "add z19.h, p0/M, z19.h, z24.h\n" + "add z19.h, p0/M, z19.h, z21.h\n" "add x20, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z24.h\n" + "add z18.h, p0/M, z18.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z24.h\n" + "add z17.h, p0/M, z17.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z24.h\n" + "add z16.h, p0/M, z16.h, z21.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z13.h, z19.h, z18.h\n" - "trn1 z14.h, z17.h, z16.h\n" + "trn1 z6.h, z19.h, z18.h\n" + "trn1 z7.h, z17.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z24.h\n" + "add z17.h, p0/M, z17.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z24.h\n" + "add z16.h, p0/M, z16.h, z21.h\n" "sub x15, x15, #0x1\n" "sub x13, x13, #0x1\n" "cmp x15, x13\n" - "trn1 z15.h, z17.h, z16.h\n" + "trn1 z8.h, z17.h, z16.h\n" "csel x23, x15, x13, LT\n" "add x14, x14, %x[ld_in_col]\n" "sub x13, x13, x23\n" @@ -503,121 +503,121 @@ void sme2_u8q_planar_3x3_s1_4rows_dot_za_impl( "16:" // Padded: Main loop "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z21.s }, p0/Z, [x14]\n" - ".inst 0xc16115a8 // sdot za.s[x8, 0], { z13.h-z14.h }, z1.h\n" - ".inst 0xc16015a9 // sdot za.s[x8, 1], { z13.h-z14.h }, z0.h\n" - "add z21.h, p0/M, z21.h, z24.h\n" + "ld1b { z9.s }, p0/Z, [x14]\n" + ".inst 0xc16d14c8 // sdot za.s[x8, 0], { z6.h-z7.h }, z13.h\n" + ".inst 0xc16514c9 // sdot za.s[x8, 1], { z6.h-z7.h }, z5.h\n" + "add z9.h, p0/M, z9.h, z21.h\n" "add x22, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z20.s }, p0/Z, [x22]\n" - ".inst 0xc16315c8 // sdot za.s[x8, 0], { z14.h-z15.h }, z3.h\n" - "add z20.h, p0/M, z20.h, z24.h\n" + "ld1b { z19.s }, p0/Z, [x22]\n" + ".inst 0xc16b14e8 // sdot za.s[x8, 0], { z7.h-z8.h }, z11.h\n" + "add z19.h, p0/M, z19.h, z21.h\n" "add x22, x22, %x[ld_in_row]\n" - ".inst 0xc16215c9 // sdot za.s[x8, 1], { z14.h-z15.h }, z2.h\n" + ".inst 0xc16a14e9 // sdot za.s[x8, 1], { z7.h-z8.h }, z10.h\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z19.s }, p0/Z, [x22]\n" - "add z19.h, p0/M, z19.h, z24.h\n" - ".inst 0xc0060804 // mova { z4.d-z5.d }, za.d[x8, #0]\n" + "ld1b { z18.s }, p0/Z, [x22]\n" + "add z18.h, p0/M, z18.h, z21.h\n" + ".inst 0xc0060818 // mova { z24.d-z25.d }, za.d[x8, #0]\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z18.s }, p0/Z, [x22]\n" - ".inst 0xc0060826 // mova { z6.d-z7.d }, za.d[x8, #1]\n" + "ld1b { z16.s }, p0/Z, [x22]\n" + ".inst 0xc006083a // mova { z26.d-z27.d }, za.d[x8, #1]\n" "mov x12, #0x4\n" "addvl x21, SP, #4\n" - "add z18.h, p0/M, z18.h, z24.h\n" - ".inst 0xc1aaac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z10.s\n" + "add z16.h, p0/M, z16.h, z21.h\n" + ".inst 0xc1aeac18 // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z14.s\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xa0402aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21]\n" + ".inst 0xa1402aa3 // ld1h { z3.h, z11.h }, pn10.b/Z, [x21]\n" "addvl x20, SP, #8\n" - ".inst 0xc16115aa // sdot za.s[x8, 2], { z13.h-z14.h }, z1.h\n" + ".inst 0xc16b14ca // sdot za.s[x8, 2], { z6.h-z7.h }, z11.h\n" "subs x23, x23, #0x1\n" "ld1b { z17.s }, p0/Z, [x22]\n" - ".inst 0xc16015ab // sdot za.s[x8, 3], { z13.h-z14.h }, z0.h\n" - ".inst 0xc1abaa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z11.s\n" - ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z24.h\n" + ".inst 0xc16314cb // sdot za.s[x8, 3], { z6.h-z7.h }, z3.h\n" + ".inst 0xc1acaa38 // srshl { z24.s-z27.s }, { z24.s-z27.s }, z12.s\n" + ".inst 0xa1402a85 // ld1h { z5.h, z13.h }, pn10.b/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z21.h\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0xa0412aa2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc16115ac // sdot za.s[x8, 4], { z13.h-z14.h }, z1.h\n" - ".inst 0xc1acab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z12.s\n" - "ld1b { z16.s }, p0/Z, [x22]\n" - ".inst 0xc16015ad // sdot za.s[x8, 5], { z13.h-z14.h }, z0.h\n" - "add z16.h, p0/M, z16.h, z24.h\n" + ".inst 0xa0412aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc16d14cc // sdot za.s[x8, 4], { z6.h-z7.h }, z13.h\n" + ".inst 0xc1afab18 // add { z24.s-z27.s }, { z24.s-z27.s }, z15.s\n" + "ld1b { z2.s }, p0/Z, [x22]\n" + ".inst 0xc16514cd // sdot za.s[x8, 5], { z6.h-z7.h }, z5.h\n" + "add z2.h, p0/M, z2.h, z21.h\n" "add x14, x14, %x[ld_in_col]\n" - ".inst 0xc16315ca // sdot za.s[x8, 2], { z14.h-z15.h }, z3.h\n" - ".inst 0xa0402be0 // ld1h { z0.h-z1.h }, pn10.b/Z, [SP]\n" - ".inst 0xc1bacec4 // sclamp { z4.s-z7.s }, z22.s, z26.s\n" - ".inst 0xc16215cb // sdot za.s[x8, 3], { z14.h-z15.h }, z2.h\n" - ".inst 0xa0412a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - "st1b { z4.s }, p1, [x11]\n" + ".inst 0xc16b14ea // sdot za.s[x8, 2], { z7.h-z8.h }, z11.h\n" + ".inst 0xa1402be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP]\n" + ".inst 0xc1bccfb8 // sclamp { z24.s-z27.s }, z29.s, z28.s\n" + ".inst 0xc16a14eb // sdot za.s[x8, 3], { z7.h-z8.h }, z10.h\n" + ".inst 0xa1412a83 // ld1h { z3.h, z11.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + "st1b { z24.s }, p1, [x11]\n" "add x11, x11, x9\n" - ".inst 0xc16315cc // sdot za.s[x8, 4], { z14.h-z15.h }, z3.h\n" - "st1b { z6.s }, p1, [x10]\n" + ".inst 0xc16b14ec // sdot za.s[x8, 4], { z7.h-z8.h }, z11.h\n" + "st1b { z26.s }, p1, [x10]\n" "add x10, x10, x28\n" - "trn1 z13.h, z21.h, z20.h\n" - ".inst 0xc16215cd // sdot za.s[x8, 5], { z14.h-z15.h }, z2.h\n" + "trn1 z6.h, z9.h, z19.h\n" + ".inst 0xc16314ed // sdot za.s[x8, 5], { z7.h-z8.h }, z3.h\n" "add x8, x8, #0x2\n" - ".inst 0xa0412be2 // ld1h { z2.h-z3.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" - "st1b { z5.s }, p1, [x27]\n" + ".inst 0xa0412bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + "st1b { z25.s }, p1, [x27]\n" "add x27, x27, x25\n" - "st1b { z7.s }, p1, [x26]\n" + "st1b { z27.s }, p1, [x26]\n" "add x26, x26, x24\n" - ".inst 0xc0040904 // mova za.d[x8, #4], { z8.d-z9.d }\n" - ".inst 0xc0040905 // mova za.d[x8, #5], { z8.d-z9.d }\n" - "trn1 z14.h, z19.h, z18.h\n" - "trn1 z15.h, z17.h, z16.h\n" + ".inst 0xc0040bc4 // mova za.d[x8, #4], { z30.d-z31.d }\n" + ".inst 0xc0040bc5 // mova za.d[x8, #5], { z30.d-z31.d }\n" + "trn1 z7.h, z18.h, z16.h\n" + "trn1 z8.h, z17.h, z2.h\n" "bgt 16b\n" "17:" // Main loop tail - ".inst 0xc16115a8 // sdot za.s[x8, 0], { z13.h-z14.h }, z1.h\n" + ".inst 0xc16d14c8 // sdot za.s[x8, 0], { z6.h-z7.h }, z13.h\n" "addvl x21, SP, #4\n" "addvl x20, SP, #8\n" - ".inst 0xc16015a9 // sdot za.s[x8, 1], { z13.h-z14.h }, z0.h\n" + ".inst 0xc16514c9 // sdot za.s[x8, 1], { z6.h-z7.h }, z5.h\n" ".inst 0xa0402aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21]\n" - ".inst 0xc16315c8 // sdot za.s[x8, 0], { z14.h-z15.h }, z3.h\n" - ".inst 0xc16215c9 // sdot za.s[x8, 1], { z14.h-z15.h }, z2.h\n" + ".inst 0xc16b14e8 // sdot za.s[x8, 0], { z7.h-z8.h }, z11.h\n" + ".inst 0xc16a14e9 // sdot za.s[x8, 1], { z7.h-z8.h }, z10.h\n" ".inst 0xa0412aa2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc0060804 // mova { z4.d-z5.d }, za.d[x8, #0]\n" - ".inst 0xc0060826 // mova { z6.d-z7.d }, za.d[x8, #1]\n" - ".inst 0xc1aaac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z10.s\n" - ".inst 0xc16115aa // sdot za.s[x8, 2], { z13.h-z14.h }, z1.h\n" - ".inst 0xc1abaa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z11.s\n" - ".inst 0xc16015ab // sdot za.s[x8, 3], { z13.h-z14.h }, z0.h\n" - ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1acab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z12.s\n" - ".inst 0xc16115ac // sdot za.s[x8, 4], { z13.h-z14.h }, z1.h\n" - ".inst 0xc1bacec4 // sclamp { z4.s-z7.s }, z22.s, z26.s\n" - "st1b { z4.s }, p1, [x11]\n" + ".inst 0xc0060818 // mova { z24.d-z25.d }, za.d[x8, #0]\n" + ".inst 0xc006083a // mova { z26.d-z27.d }, za.d[x8, #1]\n" + ".inst 0xc1aeac18 // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z14.s\n" + ".inst 0xc16114ca // sdot za.s[x8, 2], { z6.h-z7.h }, z1.h\n" + ".inst 0xc1acaa38 // srshl { z24.s-z27.s }, { z24.s-z27.s }, z12.s\n" + ".inst 0xc16014cb // sdot za.s[x8, 3], { z6.h-z7.h }, z0.h\n" + ".inst 0xa1402a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1afab18 // add { z24.s-z27.s }, { z24.s-z27.s }, z15.s\n" + ".inst 0xc16914cc // sdot za.s[x8, 4], { z6.h-z7.h }, z9.h\n" + ".inst 0xc1bccfb8 // sclamp { z24.s-z27.s }, z29.s, z28.s\n" + "st1b { z24.s }, p1, [x11]\n" "add x11, x11, x9\n" - ".inst 0xc16015ad // sdot za.s[x8, 5], { z13.h-z14.h }, z0.h\n" - "st1b { z6.s }, p1, [x10]\n" + ".inst 0xc16114cd // sdot za.s[x8, 5], { z6.h-z7.h }, z1.h\n" + "st1b { z26.s }, p1, [x10]\n" "add x10, x10, x28\n" - ".inst 0xc16315ca // sdot za.s[x8, 2], { z14.h-z15.h }, z3.h\n" - "st1b { z5.s }, p1, [x27]\n" + ".inst 0xc16314ea // sdot za.s[x8, 2], { z7.h-z8.h }, z3.h\n" + "st1b { z25.s }, p1, [x27]\n" "add x27, x27, x25\n" - ".inst 0xc16215cb // sdot za.s[x8, 3], { z14.h-z15.h }, z2.h\n" - ".inst 0xa0412a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - "st1b { z7.s }, p1, [x26]\n" + ".inst 0xc16214eb // sdot za.s[x8, 3], { z7.h-z8.h }, z2.h\n" + ".inst 0xa0412a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + "st1b { z27.s }, p1, [x26]\n" "add x26, x26, x24\n" - ".inst 0xc16315cc // sdot za.s[x8, 4], { z14.h-z15.h }, z3.h\n" - ".inst 0xc16215cd // sdot za.s[x8, 5], { z14.h-z15.h }, z2.h\n" + ".inst 0xc16114ec // sdot za.s[x8, 4], { z7.h-z8.h }, z1.h\n" + ".inst 0xc16014ed // sdot za.s[x8, 5], { z7.h-z8.h }, z0.h\n" "add x8, x8, #0x2\n" - ".inst 0xc0040904 // mova za.d[x8, #4], { z8.d-z9.d }\n" - ".inst 0xc0040905 // mova za.d[x8, #5], { z8.d-z9.d }\n" + ".inst 0xc0040bc4 // mova za.d[x8, #4], { z30.d-z31.d }\n" + ".inst 0xc0040bc5 // mova za.d[x8, #5], { z30.d-z31.d }\n" "18:" // Main loop skip tail "cbz x13, 20f\n" "19:" // Right padding loop ".inst 0xc0060804 // mova { z4.d-z5.d }, za.d[x8, #0]\n" "subs x13, x13, #0x1\n" ".inst 0xc0060826 // mova { z6.d-z7.d }, za.d[x8, #1]\n" - ".inst 0xc1aaac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z10.s\n" + ".inst 0xc1aeac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z14.s\n" "add x8, x8, #0x2\n" - ".inst 0xc1abaa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z11.s\n" - ".inst 0xc0040904 // mova za.d[x8, #4], { z8.d-z9.d }\n" - ".inst 0xc1acab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z12.s\n" - ".inst 0xc0040905 // mova za.d[x8, #5], { z8.d-z9.d }\n" - ".inst 0xc1bacec4 // sclamp { z4.s-z7.s }, z22.s, z26.s\n" + ".inst 0xc1acaa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z12.s\n" + ".inst 0xc0040bc4 // mova za.d[x8, #4], { z30.d-z31.d }\n" + ".inst 0xc1afab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z15.s\n" + ".inst 0xc0040bc5 // mova za.d[x8, #5], { z30.d-z31.d }\n" + ".inst 0xc1bccfa4 // sclamp { z4.s-z7.s }, z29.s, z28.s\n" "st1b { z4.s }, p1, [x11]\n" "add x11, x11, x9\n" "st1b { z6.s }, p1, [x10]\n" @@ -628,15 +628,15 @@ void sme2_u8q_planar_3x3_s1_4rows_dot_za_impl( "add x26, x26, x24\n" "bgt 19b\n" "20:" // End - "ldr x22, [%x[args], %[offsetof_Args_weights]]\n" - "incw x22, ALL, MUL #9\n" - "str x22, [%x[args], %[offsetof_Args_weights]]\n" + "ldr x20, [%x[args], %[offsetof_Args_weights]]\n" + "incw x20, ALL, MUL #9\n" + "str x20, [%x[args], %[offsetof_Args_weights]]\n" "incw x16\n" - "ldr x20, [%x[args], %[offsetof_Args_ld_in_vl]]\n" + "ldr x21, [%x[args], %[offsetof_Args_ld_in_vl]]\n" "whilelt p1.s, x16, x17\n" - "ldr x14, [%x[args], %[offsetof_Args_inptr]]\n" - "add x14, x14, x20\n" - "str x14, [%x[args], %[offsetof_Args_inptr]]\n" + "ldr x20, [%x[args], %[offsetof_Args_inptr]]\n" + "add x20, x20, x21\n" + "str x20, [%x[args], %[offsetof_Args_inptr]]\n" "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" "ldr x24, [%x[args], %[offsetof_Args_ld_out_vls]]\n" "ldp x23, x22, [x25, #0x0]\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_3x3_s2_4rows_dot_za.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_3x3_s2_4rows_dot_za.hpp index b878914ce8fb108d57697ec0b3e38010f8e2e4d9..f852e12de16e084f7190cca6d868be6bf758eaf6 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_3x3_s2_4rows_dot_za.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_3x3_s2_4rows_dot_za.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,8 @@ * SOFTWARE. */ +#include "src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp" + namespace arm_conv { namespace depthwise { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_3x3_s2_4rows_dot_za/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_3x3_s2_4rows_dot_za/generic.cpp index 630d8704332d2fb7892df5f5c0fb47502449130e..e4ce6c74fb3cc0b84a746bf06b5cede25df81920 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_3x3_s2_4rows_dot_za/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_3x3_s2_4rows_dot_za/generic.cpp @@ -73,86 +73,86 @@ void sme2_u8q_planar_3x3_s2_4rows_dot_za_impl( "ptrue p2.b\n" "mov x20, #0x9\n" "ldr x7, [%x[args], %[offsetof_Args_pad_top]]\n" - "ld1rh { z5.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_a_offset]]\n" + "ld1rh { z11.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_a_offset]]\n" "sub x20, x20, x6\n" ".inst 0x25207812 // ptrue pn10.b\n" "ldr x17, [%x[args], %[offsetof_Args_n_channels]]\n" "whilelt p1.s, XZR, x17\n" "whilelt p9.s, XZR, x20\n" - "ld1rw { z4.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" + "ld1rw { z13.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" "whilelt p8.s, XZR, x7\n" "addvl SP, SP, #-6\n" "ldr x16, [%x[args], %[offsetof_Args_current_channel]]\n" - "neg z5.h, p2/M, z5.h\n" + "neg z11.h, p2/M, z11.h\n" "eor p8.b, p2/Z, p8.b, p9.b\n" - "ld1rw { z8.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_mul]]\n" - "ld1rw { z7.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_right_shift]]\n" - "ld1rw { z27.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" - "ld1rw { z23.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" + "ld1rw { z6.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z9.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z10.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" + "ld1rw { z7.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" "1:" // Channel loop "ldr x20, [%x[qp], %[offsetof_Requantize32_bias]]\n" - "mov z0.s, #0x0\n" + "mov z28.s, #0x0\n" "cbz x20, 2f\n" - "ld1w { z0.s }, p1/Z, [x20, x16, LSL #2]\n" + "ld1w { z28.s }, p1/Z, [x20, x16, LSL #2]\n" "2:" // Load bias: Done "ldr x22, [%x[args], %[offsetof_Args_weights]]\n" "mov x20, x22\n" - "ld1b { z24.s }, p2/Z, [x20]\n" + "ld1b { z26.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "ld1rh { z13.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" - "sub z24.h, z24.h, z13.h\n" + "ld1rh { z16.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" + "sub z26.h, z26.h, z16.h\n" "incw x22\n" - "mov z17.h, #0x0\n" - "ld1b { z25.s }, p2/Z, [x20]\n" + "mov z24.h, #0x0\n" + "ld1b { z3.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "sub z25.h, z25.h, z13.h\n" - "trn1 z10.h, z24.h, z25.h\n" - "ld1b { z16.s }, p2/Z, [x20]\n" - "sub z16.h, z16.h, z13.h\n" + "sub z3.h, z3.h, z16.h\n" + "trn1 z31.h, z26.h, z3.h\n" + "ld1b { z21.s }, p2/Z, [x20]\n" + "sub z21.h, z21.h, z16.h\n" "mov x20, x22\n" - "trn1 z11.h, z16.h, z17.h\n" - "ld1b { z24.s }, p2/Z, [x20]\n" + "trn1 z14.h, z21.h, z24.h\n" + "ld1b { z2.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "sub z24.h, z24.h, z13.h\n" + "sub z2.h, z2.h, z16.h\n" "addvl x21, SP, #6\n" "ld1b { z25.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "sub z25.h, z25.h, z13.h\n" + "sub z25.h, z25.h, z16.h\n" "incw x22\n" - "ld1b { z16.s }, p2/Z, [x20]\n" - "sub z16.h, z16.h, z13.h\n" + "ld1b { z27.s }, p2/Z, [x20]\n" + "sub z27.h, z27.h, z16.h\n" "addvl x21, x21, #-2\n" "mov x20, x22\n" - "st1h { z10.h }, p2, [x21]\n" - "trn1 z10.h, z24.h, z25.h\n" - "ld1b { z24.s }, p2/Z, [x20]\n" + "st1h { z31.h }, p2, [x21]\n" + "trn1 z4.h, z2.h, z25.h\n" + "ld1b { z26.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "ld1b { z25.s }, p2/Z, [x20]\n" + "ld1b { z23.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "st1h { z11.h }, p2, [x21, #1, MUL VL]\n" - "trn1 z11.h, z16.h, z17.h\n" - "ld1b { z16.s }, p2/Z, [x20]\n" - "sub z24.h, z24.h, z13.h\n" - "sub z25.h, z25.h, z13.h\n" + "st1h { z14.h }, p2, [x21, #1, MUL VL]\n" + "trn1 z12.h, z27.h, z24.h\n" + "ld1b { z20.s }, p2/Z, [x20]\n" + "sub z26.h, z26.h, z16.h\n" + "sub z23.h, z23.h, z16.h\n" "ldr x20, [%x[qp], %[offsetof_Requantize32_per_channel_muls]]\n" - "sub z16.h, z16.h, z13.h\n" + "sub z20.h, z20.h, z16.h\n" "addvl x21, x21, #-2\n" - "st1h { z10.h }, p2, [x21]\n" - "mov z1.d, z0.d\n" - "st1h { z11.h }, p2, [x21, #1, MUL VL]\n" + "st1h { z4.h }, p2, [x21]\n" + "mov z29.d, z28.d\n" + "st1h { z12.h }, p2, [x21, #1, MUL VL]\n" "addvl x21, x21, #-2\n" - "mov z2.d, z0.d\n" - "mov z3.d, z0.d\n" - "trn1 z10.h, z24.h, z25.h\n" - "st1h { z10.h }, p2, [x21]\n" - "trn1 z11.h, z16.h, z17.h\n" - "st1h { z11.h }, p2, [x21, #1, MUL VL]\n" + "mov z30.d, z28.d\n" + "mov z31.d, z28.d\n" + "trn1 z25.h, z26.h, z23.h\n" + "st1h { z25.h }, p2, [x21]\n" + "trn1 z3.h, z20.h, z24.h\n" + "st1h { z3.h }, p2, [x21, #1, MUL VL]\n" "cbz x20, 3f\n" - "ld1w { z8.s }, p1/Z, [x20, x16, LSL #2]\n" + "ld1w { z6.s }, p1/Z, [x20, x16, LSL #2]\n" "3:" // Load mul: End "ldr x20, [%x[qp], %[offsetof_Requantize32_per_channel_right_shifts]]\n" "cbz x20, 4f\n" - "ld1w { z7.s }, p1/Z, [x20, x16, LSL #2]\n" + "ld1w { z9.s }, p1/Z, [x20, x16, LSL #2]\n" "4:" // Load right_shift: End "ldr x15, [%x[args], %[offsetof_Args_input_cols]]\n" "sub x20, x15, #0x1\n" @@ -172,18 +172,18 @@ void sme2_u8q_planar_3x3_s2_4rows_dot_za_impl( ".inst 0xf8b74a9c // rprfm pldstrm, x23, [x20]\n" "add x20, x20, %x[ld_in_col]\n" "bgt 5b\n" - "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" + "ldr x23, [%x[args], %[offsetof_Args_outptrs]]\n" "lsl x20, %x[ld_in_row], #0x0\n" "msub x14, x7, x20, x14\n" - ".inst 0xc0040c00 // mova za.d[x8, #0], { z0.d-z3.d }\n" + ".inst 0xc0040f80 // mova za.d[x8, #0], { z28.d-z31.d }\n" "ldr x20, [%x[args], %[offsetof_Args_ld_out_cols]]\n" - ".inst 0xc0040c01 // mova za.d[x8, #1], { z0.d-z3.d }\n" + ".inst 0xc0040f81 // mova za.d[x8, #1], { z28.d-z31.d }\n" "mov x22, #0x2\n" - "ldp x11, x10, [x25], #0x10\n" - ".inst 0xc0040c02 // mova za.d[x8, #2], { z0.d-z3.d }\n" + "ldp x11, x10, [x23], #0x10\n" + ".inst 0xc0040f82 // mova za.d[x8, #2], { z28.d-z31.d }\n" "ldp x9, x28, [x20], #0x10\n" "ldr x21, [%x[args], %[offsetof_Args_pad_left]]\n" - "ldp x27, x26, [x25], #0x10\n" + "ldp x27, x26, [x23], #0x10\n" "ldp x25, x24, [x20], #0x10\n" "cbz x21, 7f\n" "cmp x21, x22\n" @@ -191,24 +191,24 @@ void sme2_u8q_planar_3x3_s2_4rows_dot_za_impl( "sub x21, x21, x20\n" "sub x22, x22, x20\n" "cbz x21, 7f\n" - ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" - ".inst 0xc1a8ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z8.s\n" + ".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n" + ".inst 0xc1a6ac18 // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z6.s\n" "and x22, x21, #0x1\n" - ".inst 0xc1a7aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z7.s\n" + ".inst 0xc1a9aa38 // srshl { z24.s-z27.s }, { z24.s-z27.s }, z9.s\n" "add x21, x21, #0x1\n" "lsr x21, x21, #0x1\n" - ".inst 0xc1a4ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" + ".inst 0xc1adab18 // add { z24.s-z27.s }, { z24.s-z27.s }, z13.s\n" "sub x13, x13, x21\n" - ".inst 0xc1b7cf7c // sclamp { z28.s-z31.s }, z27.s, z23.s\n" + ".inst 0xc1a7cd58 // sclamp { z24.s-z27.s }, z10.s, z7.s\n" "6:" // Left padding "subs x21, x21, #0x1\n" - "st1b { z28.s }, p1, [x11]\n" + "st1b { z24.s }, p1, [x11]\n" "add x11, x11, x9\n" - "st1b { z29.s }, p1, [x10]\n" + "st1b { z25.s }, p1, [x10]\n" "add x10, x10, x28\n" - "st1b { z30.s }, p1, [x27]\n" + "st1b { z26.s }, p1, [x27]\n" "add x27, x27, x25\n" - "st1b { z31.s }, p1, [x26]\n" + "st1b { z27.s }, p1, [x26]\n" "add x26, x26, x24\n" "bgt 6b\n" "7:" // Left padding: End @@ -220,194 +220,194 @@ void sme2_u8q_planar_3x3_s2_4rows_dot_za_impl( "beq 9f\n" "8:" // Unpadded: 2 priming loads "add x21, x14, %x[ld_in_row]\n" - "ld1b { z12.s }, p1/Z, [x14]\n" + "ld1b { z1.s }, p1/Z, [x14]\n" "addvl x20, SP, #4\n" - "ld1b { z20.s }, p1/Z, [x21]\n" + "ld1b { z21.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "add z12.h, z12.h, z5.h\n" - "ld1b { z13.s }, p1/Z, [x21]\n" + "trn1 z1.h, z1.h, z21.h\n" + "add z1.h, z1.h, z11.h\n" + "ld1b { z2.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "add x14, x14, %x[ld_in_col]\n" - "ld1b { z19.s }, p1/Z, [x21]\n" + "ld1b { z15.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z5.h\n" - "ld1b { z14.s }, p1/Z, [x21]\n" + "trn1 z2.h, z2.h, z15.h\n" + "add z2.h, z2.h, z11.h\n" + "ld1b { z3.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1b { z18.s }, p1/Z, [x21]\n" + "ld1b { z21.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z5.h\n" - "ld1b { z15.s }, p1/Z, [x21]\n" + "trn1 z3.h, z3.h, z21.h\n" + "add z3.h, z3.h, z11.h\n" + "ld1b { z4.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x21]\n" + "ld1b { z19.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z15.h, z15.h, z5.h\n" - "ld1b { z16.s }, p1/Z, [x21]\n" - "mov z16.d, z16.d\n" - "add z16.h, z16.h, z5.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" + "trn1 z4.h, z4.h, z19.h\n" + "add z4.h, z4.h, z11.h\n" + "ld1b { z8.s }, p1/Z, [x21]\n" + "mov z5.d, z8.d\n" + "add z5.h, z5.h, z11.h\n" + ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1701428 // sdot za.s[x8, 0], { z1.h-z4.h }, z0.h\n" + ".inst 0xc1781448 // sdot za.s[x8, 0], { z2.h-z5.h }, z8.h\n" "9:" // Unpadded: 1 priming loads "add x21, x14, %x[ld_in_row]\n" - "ld1b { z12.s }, p1/Z, [x14]\n" + "ld1b { z1.s }, p1/Z, [x14]\n" "addvl x20, SP, #2\n" - "ld1b { z20.s }, p1/Z, [x21]\n" + "ld1b { z21.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "add z12.h, z12.h, z5.h\n" - "ld1b { z13.s }, p1/Z, [x21]\n" + "trn1 z1.h, z1.h, z21.h\n" + "add z1.h, z1.h, z11.h\n" + "ld1b { z2.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "add x14, x14, %x[ld_in_col]\n" - "ld1b { z19.s }, p1/Z, [x21]\n" + "ld1b { z12.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z5.h\n" - "ld1b { z14.s }, p1/Z, [x21]\n" + "trn1 z2.h, z2.h, z12.h\n" + "add z2.h, z2.h, z11.h\n" + "ld1b { z3.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1b { z18.s }, p1/Z, [x21]\n" + "ld1b { z8.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z5.h\n" - "ld1b { z15.s }, p1/Z, [x21]\n" + "trn1 z3.h, z3.h, z8.h\n" + "add z3.h, z3.h, z11.h\n" + "ld1b { z4.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x21]\n" + "ld1b { z5.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z15.h, z15.h, z5.h\n" - "ld1b { z16.s }, p1/Z, [x21]\n" - "mov z16.d, z16.d\n" - "add z16.h, z16.h, z5.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" + "trn1 z4.h, z4.h, z5.h\n" + "add z4.h, z4.h, z11.h\n" + "ld1b { z5.s }, p1/Z, [x21]\n" + "mov z5.d, z5.d\n" + "add z5.h, z5.h, z11.h\n" + ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1701428 // sdot za.s[x8, 0], { z1.h-z4.h }, z0.h\n" + ".inst 0xc1781448 // sdot za.s[x8, 0], { z2.h-z5.h }, z8.h\n" "10:" // Unpadded: 0 priming loads "cmp x15, #0x2\n" - ".inst 0xa0402bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP]\n" + ".inst 0xa0402bee // ld1h { z14.h-z15.h }, pn10.b/Z, [SP]\n" "blt 18f\n" "add x21, x14, %x[ld_in_row]\n" - "ld1b { z12.s }, p1/Z, [x14]\n" + "ld1b { z21.s }, p1/Z, [x14]\n" "sub x15, x15, #0x2\n" - "ld1b { z20.s }, p1/Z, [x21]\n" + "ld1b { z8.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z21.h, z21.h, z8.h\n" "sub x13, x13, #0x1\n" - "ld1b { z13.s }, p1/Z, [x21]\n" + "ld1b { z22.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "lsr x20, x15, #0x1\n" - "add z12.h, z12.h, z5.h\n" - "ld1b { z19.s }, p1/Z, [x21]\n" + "add z21.h, z21.h, z11.h\n" + "ld1b { z25.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" + "trn1 z22.h, z22.h, z25.h\n" "cmp x20, x13\n" - "ld1b { z14.s }, p1/Z, [x21]\n" + "ld1b { z23.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "csel x23, x20, x13, LT\n" - "add z13.h, z13.h, z5.h\n" + "add z22.h, z22.h, z11.h\n" "ld1b { z18.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z5.h\n" - "ld1b { z15.s }, p1/Z, [x21]\n" + "trn1 z23.h, z23.h, z18.h\n" + "add z23.h, z23.h, z11.h\n" + "ld1b { z24.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "add x14, x14, %x[ld_in_col]\n" - "ld1b { z17.s }, p1/Z, [x21]\n" + "ld1b { z19.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z15.h, z15.h, z5.h\n" - "ld1b { z16.s }, p1/Z, [x21]\n" - "mov z16.d, z16.d\n" - "add z16.h, z16.h, z5.h\n" + "trn1 z24.h, z24.h, z19.h\n" + "add z24.h, z24.h, z11.h\n" + "ld1b { z8.s }, p1/Z, [x21]\n" + "mov z25.d, z8.d\n" + "add z25.h, z25.h, z11.h\n" "and x15, x15, #0x1\n" "sub x13, x13, x23\n" "cbz x23, 17f\n" "11:" // Unpadded: Main loop - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" + ".inst 0xc17e16a8 // sdot za.s[x8, 0], { z21.h-z24.h }, z14.h\n" "addvl x20, SP, #4\n" "add x22, x14, %x[ld_in_row]\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" + ".inst 0xc17f16c8 // sdot za.s[x8, 0], { z22.h-z25.h }, z15.h\n" + ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" "addvl x21, SP, #2\n" "subs x23, x23, #0x1\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - "ld1b { z12.s }, p1/Z, [x14]\n" + ".inst 0xc17016a9 // sdot za.s[x8, 1], { z21.h-z24.h }, z0.h\n" + "ld1b { z21.s }, p1/Z, [x14]\n" "add x14, x14, %x[ld_in_col]\n" "add x20, x14, %x[ld_in_row]\n" - "ld1b { z20.s }, p1/Z, [x22]\n" + "ld1b { z18.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - ".inst 0xc17b15a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z11.h\n" - "trn1 z12.h, z12.h, z20.h\n" - "ld1b { z13.s }, p1/Z, [x22]\n" + ".inst 0xc17116c9 // sdot za.s[x8, 1], { z22.h-z25.h }, z1.h\n" + "trn1 z21.h, z21.h, z18.h\n" + "ld1b { z22.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "add z12.h, z12.h, z5.h\n" - ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" - "ld1b { z19.s }, p1/Z, [x22]\n" + "add z21.h, z21.h, z11.h\n" + ".inst 0xc0060c00 // mova { z0.d-z3.d }, za.d[x8, #0]\n" + "ld1b { z8.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z5.h\n" - "ld1b { z14.s }, p1/Z, [x22]\n" + "trn1 z22.h, z22.h, z8.h\n" + "add z22.h, z22.h, z11.h\n" + "ld1b { z23.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "add x8, x8, #0x1\n" - ".inst 0xc0040c02 // mova za.d[x8, #2], { z0.d-z3.d }\n" - "ld1b { z18.s }, p1/Z, [x22]\n" + ".inst 0xc0040f82 // mova za.d[x8, #2], { z28.d-z31.d }\n" + "ld1b { z27.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z5.h\n" - "ld1b { z15.s }, p1/Z, [x22]\n" + "trn1 z23.h, z23.h, z27.h\n" + "add z23.h, z23.h, z11.h\n" + "ld1b { z24.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - ".inst 0xc1a8ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z8.s\n" - "ld1b { z17.s }, p1/Z, [x22]\n" + ".inst 0xc1a6ac00 // sqdmulh { z0.s-z3.s }, { z0.s-z3.s }, z6.s\n" + "ld1b { z8.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z15.h, z15.h, z5.h\n" - "ld1b { z16.s }, p1/Z, [x22]\n" - "mov z16.d, z16.d\n" - "add z16.h, z16.h, z5.h\n" - ".inst 0xa0402aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21]\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xc1a7aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z7.s\n" - "ld1b { z12.s }, p1/Z, [x14]\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" - ".inst 0xc1a4ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" - "ld1b { z20.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - ".inst 0xc1b7cf7c // sclamp { z28.s-z31.s }, z27.s, z23.s\n" - "ld1b { z13.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row]\n" - "st1b { z28.s }, p1, [x11]\n" + "trn1 z24.h, z24.h, z8.h\n" + "add z24.h, z24.h, z11.h\n" + "ld1b { z4.s }, p1/Z, [x22]\n" + "mov z25.d, z4.d\n" + "add z25.h, z25.h, z11.h\n" + ".inst 0xa1402aa4 // ld1h { z4.h, z12.h }, pn10.b/Z, [x21]\n" + ".inst 0xc17416a8 // sdot za.s[x8, 0], { z21.h-z24.h }, z4.h\n" + ".inst 0xc1a9aa20 // srshl { z0.s-z3.s }, { z0.s-z3.s }, z9.s\n" + "ld1b { z21.s }, p1/Z, [x14]\n" + ".inst 0xc17c16c8 // sdot za.s[x8, 0], { z22.h-z25.h }, z12.h\n" + ".inst 0xc1adab00 // add { z0.s-z3.s }, { z0.s-z3.s }, z13.s\n" + "ld1b { z12.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row]\n" + "trn1 z21.h, z21.h, z12.h\n" + ".inst 0xc1a7cd40 // sclamp { z0.s-z3.s }, z10.s, z7.s\n" + "ld1b { z22.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row]\n" + "st1b { z0.s }, p1, [x11]\n" "add x11, x11, x9\n" - "ld1b { z19.s }, p1/Z, [x20]\n" + "ld1b { z20.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "st1b { z29.s }, p1, [x10]\n" - "ld1b { z14.s }, p1/Z, [x20]\n" + "trn1 z22.h, z22.h, z20.h\n" + "st1b { z1.s }, p1, [x10]\n" + "ld1b { z23.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" "add x10, x10, x28\n" - "st1b { z30.s }, p1, [x27]\n" - "ld1b { z18.s }, p1/Z, [x20]\n" + "st1b { z2.s }, p1, [x27]\n" + "ld1b { z24.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" + "trn1 z23.h, z23.h, z24.h\n" "add x27, x27, x25\n" - "ld1b { z15.s }, p1/Z, [x20]\n" + "ld1b { z24.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "st1b { z31.s }, p1, [x26]\n" + "st1b { z3.s }, p1, [x26]\n" "add x26, x26, x24\n" - "ld1b { z17.s }, p1/Z, [x20]\n" + "ld1b { z3.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z12.h, z12.h, z5.h\n" - "ld1b { z16.s }, p1/Z, [x20]\n" - "mov z16.d, z16.d\n" - "add z13.h, z13.h, z5.h\n" + "trn1 z24.h, z24.h, z3.h\n" + "add z21.h, z21.h, z11.h\n" + "ld1b { z3.s }, p1/Z, [x20]\n" + "mov z25.d, z3.d\n" + "add z22.h, z22.h, z11.h\n" "add x14, x14, %x[ld_in_col]\n" - ".inst 0xa0402bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP]\n" - "add z14.h, z14.h, z5.h\n" - "add z15.h, z15.h, z5.h\n" - "add z16.h, z16.h, z5.h\n" + ".inst 0xa0402bee // ld1h { z14.h-z15.h }, pn10.b/Z, [SP]\n" + "add z23.h, z23.h, z11.h\n" + "add z24.h, z24.h, z11.h\n" + "add z25.h, z25.h, z11.h\n" "bgt 11b\n" "b 17f\n" "12:" // Padded @@ -418,442 +418,442 @@ void sme2_u8q_planar_3x3_s2_4rows_dot_za_impl( "13:" // Padded: 2 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z12.s }, p0/Z, [x14]\n" - "add z12.h, p0/M, z12.h, z5.h\n" + "ld1b { z22.s }, p0/Z, [x14]\n" + "add z22.h, p0/M, z22.h, z11.h\n" "add x20, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z5.h\n" + "ld1b { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z5.h\n" + "ld1b { z23.s }, p0/Z, [x20]\n" + "add z23.h, p0/M, z23.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z5.h\n" + "ld1b { z4.s }, p0/Z, [x20]\n" + "add z4.h, p0/M, z4.h, z11.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "trn1 z13.h, z13.h, z19.h\n" + "trn1 z22.h, z22.h, z17.h\n" + "trn1 z23.h, z23.h, z4.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z5.h\n" + "ld1b { z24.s }, p0/Z, [x20]\n" + "add z24.h, p0/M, z24.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z5.h\n" + "add z18.h, p0/M, z18.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z5.h\n" + "ld1b { z25.s }, p0/Z, [x20]\n" + "add z25.h, p0/M, z25.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1b { z17.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z17.h, p0/M, z17.h, z5.h\n" + "add z17.h, p0/M, z17.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z5.h\n" + "ld1b { z1.s }, p0/Z, [x20]\n" + "add z1.h, p0/M, z1.h, z11.h\n" "addvl x20, SP, #4\n" - "trn1 z14.h, z14.h, z18.h\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" - "mov z16.d, z16.d\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" + "trn1 z24.h, z24.h, z18.h\n" + "trn1 z25.h, z25.h, z17.h\n" + ".inst 0xa1402a84 // ld1h { z4.h, z12.h }, pn10.b/Z, [x20]\n" + "mov z26.d, z1.d\n" + ".inst 0xc17416c8 // sdot za.s[x8, 0], { z22.h-z25.h }, z4.h\n" "add x14, x14, %x[ld_in_col]\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" + ".inst 0xc17c16e8 // sdot za.s[x8, 0], { z23.h-z26.h }, z12.h\n" "14:" // Padded: 1 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z12.s }, p0/Z, [x14]\n" - "add z12.h, p0/M, z12.h, z5.h\n" + "ld1b { z22.s }, p0/Z, [x14]\n" + "add z22.h, p0/M, z22.h, z11.h\n" "add x20, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z5.h\n" + "ld1b { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z5.h\n" + "ld1b { z23.s }, p0/Z, [x20]\n" + "add z23.h, p0/M, z23.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z5.h\n" + "ld1b { z5.s }, p0/Z, [x20]\n" + "add z5.h, p0/M, z5.h, z11.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "trn1 z13.h, z13.h, z19.h\n" + "trn1 z22.h, z22.h, z17.h\n" + "trn1 z23.h, z23.h, z5.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z5.h\n" + "ld1b { z24.s }, p0/Z, [x20]\n" + "add z24.h, p0/M, z24.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z5.h\n" + "add z18.h, p0/M, z18.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z5.h\n" + "ld1b { z25.s }, p0/Z, [x20]\n" + "add z25.h, p0/M, z25.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1b { z17.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z17.h, p0/M, z17.h, z5.h\n" + "add z17.h, p0/M, z17.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z5.h\n" + "ld1b { z15.s }, p0/Z, [x20]\n" + "add z15.h, p0/M, z15.h, z11.h\n" "addvl x20, SP, #2\n" - "trn1 z14.h, z14.h, z18.h\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" - "mov z16.d, z16.d\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" + "trn1 z24.h, z24.h, z18.h\n" + "trn1 z25.h, z25.h, z17.h\n" + ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" + "mov z26.d, z15.d\n" + ".inst 0xc17016c8 // sdot za.s[x8, 0], { z22.h-z25.h }, z0.h\n" "add x14, x14, %x[ld_in_col]\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" + ".inst 0xc17116e8 // sdot za.s[x8, 0], { z23.h-z26.h }, z1.h\n" "15:" // Padded: 0 priming loads "cmp x15, #0x2\n" - ".inst 0xa0402bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP]\n" + ".inst 0xa0402bee // ld1h { z14.h-z15.h }, pn10.b/Z, [SP]\n" "blt 18f\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z12.s }, p0/Z, [x14]\n" - "add z12.h, p0/M, z12.h, z5.h\n" + "ld1b { z21.s }, p0/Z, [x14]\n" + "add z21.h, p0/M, z21.h, z11.h\n" "add x20, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z5.h\n" + "ld1b { z18.s }, p0/Z, [x20]\n" + "add z18.h, p0/M, z18.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z5.h\n" + "ld1b { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z5.h\n" + "ld1b { z3.s }, p0/Z, [x20]\n" + "add z3.h, p0/M, z3.h, z11.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "trn1 z13.h, z13.h, z19.h\n" + "trn1 z21.h, z21.h, z18.h\n" + "trn1 z22.h, z22.h, z3.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z5.h\n" + "ld1b { z23.s }, p0/Z, [x20]\n" + "add z23.h, p0/M, z23.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z5.h\n" + "ld1b { z19.s }, p0/Z, [x20]\n" + "add z19.h, p0/M, z19.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z5.h\n" + "ld1b { z24.s }, p0/Z, [x20]\n" + "add z24.h, p0/M, z24.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" + "ld1b { z20.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z17.h, p0/M, z17.h, z5.h\n" + "add z20.h, p0/M, z20.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z5.h\n" + "ld1b { z3.s }, p0/Z, [x20]\n" + "add z3.h, p0/M, z3.h, z11.h\n" "sub x15, x15, #0x2\n" "sub x13, x13, #0x1\n" - "trn1 z14.h, z14.h, z18.h\n" - "trn1 z15.h, z15.h, z17.h\n" + "trn1 z23.h, z23.h, z19.h\n" + "trn1 z24.h, z24.h, z20.h\n" "lsr x20, x15, #0x1\n" "cmp x20, x13\n" - "mov z16.d, z16.d\n" + "mov z25.d, z3.d\n" "csel x22, x20, x13, LT\n" "add x14, x14, %x[ld_in_col]\n" "and x15, x15, #0x1\n" "sub x13, x13, x22\n" "cbz x22, 17f\n" "16:" // Padded: Main loop - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" + ".inst 0xc17e16a8 // sdot za.s[x8, 0], { z21.h-z24.h }, z14.h\n" "addvl x20, SP, #4\n" "mov x12, #0x0\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" + ".inst 0xc17f16c8 // sdot za.s[x8, 0], { z22.h-z25.h }, z15.h\n" + ".inst 0xa1402a84 // ld1h { z4.h, z12.h }, pn10.b/Z, [x20]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "add x21, x14, %x[ld_in_row]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - "ld1b { z12.s }, p0/Z, [x14]\n" - "add z12.h, p0/M, z12.h, z5.h\n" + ".inst 0xc17416a9 // sdot za.s[x8, 1], { z21.h-z24.h }, z4.h\n" + "ld1b { z21.s }, p0/Z, [x14]\n" + "add z21.h, p0/M, z21.h, z11.h\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z20.s }, p0/Z, [x21]\n" - "add z20.h, p0/M, z20.h, z5.h\n" + "ld1b { z14.s }, p0/Z, [x21]\n" + "add z14.h, p0/M, z14.h, z11.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc17b15a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z11.h\n" - "ld1b { z13.s }, p0/Z, [x21]\n" - "add z13.h, p0/M, z13.h, z5.h\n" + ".inst 0xc17c16c9 // sdot za.s[x8, 1], { z22.h-z25.h }, z12.h\n" + "ld1b { z22.s }, p0/Z, [x21]\n" + "add z22.h, p0/M, z22.h, z11.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z19.s }, p0/Z, [x21]\n" + "ld1b { z15.s }, p0/Z, [x21]\n" "mov x12, #0x4\n" - "add z19.h, p0/M, z19.h, z5.h\n" + "add z15.h, p0/M, z15.h, z11.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z14.s }, p0/Z, [x21]\n" - "add z14.h, p0/M, z14.h, z5.h\n" + "ld1b { z23.s }, p0/Z, [x21]\n" + "add z23.h, p0/M, z23.h, z11.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z18.s }, p0/Z, [x21]\n" - "add z18.h, p0/M, z18.h, z5.h\n" + "ld1b { z17.s }, p0/Z, [x21]\n" + "add z17.h, p0/M, z17.h, z11.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z15.s }, p0/Z, [x21]\n" - "add z15.h, p0/M, z15.h, z5.h\n" + "ld1b { z24.s }, p0/Z, [x21]\n" + "add z24.h, p0/M, z24.h, z11.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z17.s }, p0/Z, [x21]\n" - "add z17.h, p0/M, z17.h, z5.h\n" + "ld1b { z4.s }, p0/Z, [x21]\n" + "add z4.h, p0/M, z4.h, z11.h\n" "mov x12, #0x8\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "trn1 z13.h, z13.h, z19.h\n" + "trn1 z21.h, z21.h, z14.h\n" + "trn1 z22.h, z22.h, z15.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "addvl x20, SP, #2\n" - "ld1b { z16.s }, p0/Z, [x21]\n" - "trn1 z14.h, z14.h, z18.h\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" + "ld1b { z2.s }, p0/Z, [x21]\n" + "trn1 z23.h, z23.h, z17.h\n" + "trn1 z24.h, z24.h, z4.h\n" + ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" "mov x12, #0x0\n" - ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" + ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" - "add z16.h, p0/M, z16.h, z5.h\n" + "add z2.h, p0/M, z2.h, z11.h\n" "add x14, x14, %x[ld_in_col]\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" + ".inst 0xc17016a8 // sdot za.s[x8, 0], { z21.h-z24.h }, z0.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z12.s }, p0/Z, [x14]\n" - "add z12.h, p0/M, z12.h, z5.h\n" + "ld1b { z21.s }, p0/Z, [x14]\n" + "add z21.h, p0/M, z21.h, z11.h\n" "add x20, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "mov z16.d, z16.d\n" + "mov z25.d, z2.d\n" "ld1b { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z5.h\n" + "add z20.h, p0/M, z20.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" - "ld1b { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z5.h\n" + ".inst 0xc17116c8 // sdot za.s[x8, 0], { z22.h-z25.h }, z1.h\n" + "ld1b { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z19.s }, p0/Z, [x20]\n" + "ld1b { z4.s }, p0/Z, [x20]\n" "mov x12, #0x4\n" - "add z19.h, p0/M, z19.h, z5.h\n" + "add z4.h, p0/M, z4.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc0040c02 // mova za.d[x8, #2], { z0.d-z3.d }\n" + ".inst 0xc0040f82 // mova za.d[x8, #2], { z28.d-z31.d }\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z5.h\n" + "ld1b { z23.s }, p0/Z, [x20]\n" + "add z23.h, p0/M, z23.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z5.h\n" + "ld1b { z27.s }, p0/Z, [x20]\n" + "add z27.h, p0/M, z27.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z15.s }, p0/Z, [x20]\n" - ".inst 0xc1a8ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z8.s\n" - "add z15.h, p0/M, z15.h, z5.h\n" + "ld1b { z24.s }, p0/Z, [x20]\n" + ".inst 0xc1a6ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n" + "add z24.h, p0/M, z24.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" + "ld1b { z12.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - ".inst 0xc1a7aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z7.s\n" + ".inst 0xc1a9aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z9.s\n" "add x20, x20, %x[ld_in_row]\n" - "add z17.h, p0/M, z17.h, z5.h\n" + "add z12.h, p0/M, z12.h, z11.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z5.h\n" - ".inst 0xc1a4ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" + "ld1b { z8.s }, p0/Z, [x20]\n" + "add z8.h, p0/M, z8.h, z11.h\n" + ".inst 0xc1adab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z13.s\n" "subs x22, x22, #0x1\n" - ".inst 0xa0402bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP]\n" - ".inst 0xc1b7cf7c // sclamp { z28.s-z31.s }, z27.s, z23.s\n" - "st1b { z28.s }, p1, [x11]\n" + ".inst 0xa0402bee // ld1h { z14.h-z15.h }, pn10.b/Z, [SP]\n" + ".inst 0xc1a7cd50 // sclamp { z16.s-z19.s }, z10.s, z7.s\n" + "st1b { z16.s }, p1, [x11]\n" "add x11, x11, x9\n" - "trn1 z12.h, z12.h, z20.h\n" - "st1b { z29.s }, p1, [x10]\n" + "trn1 z21.h, z21.h, z20.h\n" + "st1b { z17.s }, p1, [x10]\n" "add x10, x10, x28\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" - "st1b { z30.s }, p1, [x27]\n" + "trn1 z22.h, z22.h, z4.h\n" + "trn1 z23.h, z23.h, z27.h\n" + "st1b { z18.s }, p1, [x27]\n" "add x27, x27, x25\n" - "trn1 z15.h, z15.h, z17.h\n" - "mov z16.d, z16.d\n" - "st1b { z31.s }, p1, [x26]\n" + "trn1 z24.h, z24.h, z12.h\n" + "mov z25.d, z8.d\n" + "st1b { z19.s }, p1, [x26]\n" "add x26, x26, x24\n" "add x14, x14, %x[ld_in_col]\n" "bgt 16b\n" "17:" // Main loop tail - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" + ".inst 0xc17e16a8 // sdot za.s[x8, 0], { z21.h-z24.h }, z14.h\n" "addvl x20, SP, #4\n" "mov x12, #0x0\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" + ".inst 0xc17f16c8 // sdot za.s[x8, 0], { z22.h-z25.h }, z15.h\n" + ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "add x20, x14, %x[ld_in_row]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - "ld1b { z12.s }, p0/Z, [x14]\n" - "add z12.h, p0/M, z12.h, z5.h\n" + ".inst 0xc17016a9 // sdot za.s[x8, 1], { z21.h-z24.h }, z0.h\n" + "ld1b { z0.s }, p0/Z, [x14]\n" + "add z0.h, p0/M, z0.h, z11.h\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z5.h\n" + "ld1b { z14.s }, p0/Z, [x20]\n" + "add z14.h, p0/M, z14.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc17b15a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z11.h\n" - "ld1b { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z5.h\n" + ".inst 0xc17116c9 // sdot za.s[x8, 1], { z22.h-z25.h }, z1.h\n" + "ld1b { z1.s }, p0/Z, [x20]\n" + "add z1.h, p0/M, z1.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z19.s }, p0/Z, [x20]\n" + "ld1b { z12.s }, p0/Z, [x20]\n" "mov x12, #0x4\n" - "add z19.h, p0/M, z19.h, z5.h\n" + "add z12.h, p0/M, z12.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z5.h\n" + "ld1b { z2.s }, p0/Z, [x20]\n" + "add z2.h, p0/M, z2.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z5.h\n" + "ld1b { z21.s }, p0/Z, [x20]\n" + "add z21.h, p0/M, z21.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z15.s }, p0/Z, [x20]\n" - ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" - "add z15.h, p0/M, z15.h, z5.h\n" + "ld1b { z3.s }, p0/Z, [x20]\n" + ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" + "add z3.h, p0/M, z3.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" + "ld1b { z25.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - ".inst 0xc1a8ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z8.s\n" + ".inst 0xc1a6ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n" "add x20, x20, %x[ld_in_row]\n" - "add z17.h, p0/M, z17.h, z5.h\n" + "add z25.h, p0/M, z25.h, z11.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" + "ld1b { z27.s }, p0/Z, [x20]\n" "addvl x20, SP, #2\n" - ".inst 0xc1a7aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z7.s\n" - "trn1 z12.h, z12.h, z20.h\n" + ".inst 0xc1a9aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z9.s\n" + "trn1 z0.h, z0.h, z14.h\n" "add x8, x8, #0x1\n" - "add z16.h, p0/M, z16.h, z5.h\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" + "add z27.h, p0/M, z27.h, z11.h\n" + "trn1 z1.h, z1.h, z12.h\n" + "trn1 z2.h, z2.h, z21.h\n" "add x14, x14, %x[ld_in_col]\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1a4ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" - "mov z16.d, z16.d\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xc1b7cf7c // sclamp { z28.s-z31.s }, z27.s, z23.s\n" - "st1b { z28.s }, p1, [x11]\n" + "trn1 z3.h, z3.h, z25.h\n" + ".inst 0xa0402a8e // ld1h { z14.h-z15.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1adab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z13.s\n" + "mov z4.d, z27.d\n" + ".inst 0xc17e1408 // sdot za.s[x8, 0], { z0.h-z3.h }, z14.h\n" + ".inst 0xc1a7cd50 // sclamp { z16.s-z19.s }, z10.s, z7.s\n" + "st1b { z16.s }, p1, [x11]\n" "add x11, x11, x9\n" - ".inst 0xc0040c02 // mova za.d[x8, #2], { z0.d-z3.d }\n" - "st1b { z29.s }, p1, [x10]\n" + ".inst 0xc0040f82 // mova za.d[x8, #2], { z28.d-z31.d }\n" + "st1b { z17.s }, p1, [x10]\n" "add x10, x10, x28\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" - ".inst 0xa0402bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP]\n" - "st1b { z30.s }, p1, [x27]\n" + ".inst 0xc17f1428 // sdot za.s[x8, 0], { z1.h-z4.h }, z15.h\n" + ".inst 0xa0402bee // ld1h { z14.h-z15.h }, pn10.b/Z, [SP]\n" + "st1b { z18.s }, p1, [x27]\n" "add x27, x27, x25\n" - "st1b { z31.s }, p1, [x26]\n" + "st1b { z19.s }, p1, [x26]\n" "add x26, x26, x24\n" "18:" // Main loop skip tail "cbz x15, 19f\n" // Skip remainder inputs "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z12.s }, p0/Z, [x14]\n" - "add z12.h, p0/M, z12.h, z5.h\n" + "ld1b { z21.s }, p0/Z, [x14]\n" + "add z21.h, p0/M, z21.h, z11.h\n" "add x20, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z5.h\n" + "ld1b { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z5.h\n" + "ld1b { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z5.h\n" + "ld1b { z0.s }, p0/Z, [x20]\n" + "add z0.h, p0/M, z0.h, z11.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "trn1 z13.h, z13.h, z19.h\n" + "trn1 z21.h, z21.h, z17.h\n" + "trn1 z22.h, z22.h, z0.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z5.h\n" + "ld1b { z23.s }, p0/Z, [x20]\n" + "add z23.h, p0/M, z23.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z5.h\n" + "ld1b { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z5.h\n" + "ld1b { z24.s }, p0/Z, [x20]\n" + "add z24.h, p0/M, z24.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" + "ld1b { z5.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z17.h, p0/M, z17.h, z5.h\n" + "add z5.h, p0/M, z5.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z5.h\n" - "trn1 z14.h, z14.h, z18.h\n" - "trn1 z15.h, z15.h, z17.h\n" - "mov z16.d, z16.d\n" + "ld1b { z4.s }, p0/Z, [x20]\n" + "add z4.h, p0/M, z4.h, z11.h\n" + "trn1 z23.h, z23.h, z17.h\n" + "trn1 z24.h, z24.h, z5.h\n" + "mov z25.d, z4.d\n" "addvl x20, SP, #4\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" + ".inst 0xc17e16a8 // sdot za.s[x8, 0], { z21.h-z24.h }, z14.h\n" "sub x13, x13, #0x1\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" - ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" - ".inst 0xc1a8ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z8.s\n" - ".inst 0xc1a7aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z7.s\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xc1a4ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" - ".inst 0xc17b15a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z11.h\n" + ".inst 0xc17f16c8 // sdot za.s[x8, 0], { z22.h-z25.h }, z15.h\n" + ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" + ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" + ".inst 0xc1a6ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n" + ".inst 0xc1a9aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z9.s\n" + ".inst 0xc17016a9 // sdot za.s[x8, 1], { z21.h-z24.h }, z0.h\n" + ".inst 0xc1adab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z13.s\n" + ".inst 0xc17116c9 // sdot za.s[x8, 1], { z22.h-z25.h }, z1.h\n" "add x8, x8, #0x1\n" - ".inst 0xc1b7cf7c // sclamp { z28.s-z31.s }, z27.s, z23.s\n" - "st1b { z28.s }, p1, [x11]\n" + ".inst 0xc1a7cd50 // sclamp { z16.s-z19.s }, z10.s, z7.s\n" + "st1b { z16.s }, p1, [x11]\n" "add x11, x11, x9\n" - ".inst 0xc0040c02 // mova za.d[x8, #2], { z0.d-z3.d }\n" - "st1b { z29.s }, p1, [x10]\n" + ".inst 0xc0040f82 // mova za.d[x8, #2], { z28.d-z31.d }\n" + "st1b { z17.s }, p1, [x10]\n" "add x10, x10, x28\n" - "st1b { z30.s }, p1, [x27]\n" + "st1b { z18.s }, p1, [x27]\n" "add x27, x27, x25\n" - "st1b { z31.s }, p1, [x26]\n" + "st1b { z19.s }, p1, [x26]\n" "add x26, x26, x24\n" "19:" // Tail input: End "cbz x13, 21f\n" "20:" // Right padding loop - ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" - ".inst 0xc1a8ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z8.s\n" + ".inst 0xc0060c00 // mova { z0.d-z3.d }, za.d[x8, #0]\n" + ".inst 0xc1a6ac00 // sqdmulh { z0.s-z3.s }, { z0.s-z3.s }, z6.s\n" "add x8, x8, #0x1\n" - ".inst 0xc1a7aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z7.s\n" + ".inst 0xc1a9aa20 // srshl { z0.s-z3.s }, { z0.s-z3.s }, z9.s\n" "subs x13, x13, #0x1\n" - ".inst 0xc0040c02 // mova za.d[x8, #2], { z0.d-z3.d }\n" - ".inst 0xc1a4ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" - ".inst 0xc1b7cf7c // sclamp { z28.s-z31.s }, z27.s, z23.s\n" - "st1b { z28.s }, p1, [x11]\n" + ".inst 0xc0040f82 // mova za.d[x8, #2], { z28.d-z31.d }\n" + ".inst 0xc1adab00 // add { z0.s-z3.s }, { z0.s-z3.s }, z13.s\n" + ".inst 0xc1a7cd40 // sclamp { z0.s-z3.s }, z10.s, z7.s\n" + "st1b { z0.s }, p1, [x11]\n" "add x11, x11, x9\n" - "st1b { z29.s }, p1, [x10]\n" + "st1b { z1.s }, p1, [x10]\n" "add x10, x10, x28\n" - "st1b { z30.s }, p1, [x27]\n" + "st1b { z2.s }, p1, [x27]\n" "add x27, x27, x25\n" - "st1b { z31.s }, p1, [x26]\n" + "st1b { z3.s }, p1, [x26]\n" "add x26, x26, x24\n" "bgt 20b\n" "21:" // End - "ldr x22, [%x[args], %[offsetof_Args_weights]]\n" - "incw x22, ALL, MUL #9\n" - "str x22, [%x[args], %[offsetof_Args_weights]]\n" + "ldr x20, [%x[args], %[offsetof_Args_weights]]\n" + "incw x20, ALL, MUL #9\n" + "str x20, [%x[args], %[offsetof_Args_weights]]\n" "incw x16\n" - "ldr x20, [%x[args], %[offsetof_Args_ld_in_vl]]\n" + "ldr x21, [%x[args], %[offsetof_Args_ld_in_vl]]\n" "whilelt p1.s, x16, x17\n" - "ldr x14, [%x[args], %[offsetof_Args_inptr]]\n" - "add x14, x14, x20\n" - "str x14, [%x[args], %[offsetof_Args_inptr]]\n" + "ldr x20, [%x[args], %[offsetof_Args_inptr]]\n" + "add x20, x20, x21\n" + "str x20, [%x[args], %[offsetof_Args_inptr]]\n" "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" "ldr x24, [%x[args], %[offsetof_Args_ld_out_vls]]\n" "ldp x23, x22, [x25, #0x0]\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_5x5_s1_4rows_dot_za.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_5x5_s1_4rows_dot_za.hpp index db0750eb08d1a1d9f854eee60456817d45d989d9..d8b87dcd55f36b5ab8fcc7a0f8553a8b81dae6d7 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_5x5_s1_4rows_dot_za.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_5x5_s1_4rows_dot_za.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,8 @@ * SOFTWARE. */ +#include "src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp" + namespace arm_conv { namespace depthwise { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_5x5_s1_4rows_dot_za/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_5x5_s1_4rows_dot_za/generic.cpp index 2c19e232f808738c82d4a27d2889bbcc5a20e749..d33ef764ef9002ce22618e65d5895d020be74a0f 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_5x5_s1_4rows_dot_za/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_5x5_s1_4rows_dot_za/generic.cpp @@ -69,196 +69,196 @@ void sme2_u8q_planar_5x5_s1_4rows_dot_za_impl( __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" - "ldr x4, [%x[args], %[offsetof_Args_pad_bottom]]\n" + "ldr x5, [%x[args], %[offsetof_Args_pad_bottom]]\n" "ptrue p2.b\n" "mov x20, #0x8\n" "ldr x6, [%x[args], %[offsetof_Args_pad_top]]\n" - "ld1rh { z25.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_a_offset]]\n" - "sub x20, x20, x4\n" + "ld1rh { z17.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_a_offset]]\n" + "sub x20, x20, x5\n" ".inst 0x25207812 // ptrue pn10.b\n" "ldr x7, [%x[args], %[offsetof_Args_n_channels]]\n" "whilelt p1.s, XZR, x7\n" "whilelt p9.s, XZR, x20\n" - "ld1rw { z9.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" + "ld1rw { z12.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" "whilelt p8.s, XZR, x6\n" "addvl SP, SP, #-30\n" - "ldr x5, [%x[args], %[offsetof_Args_current_channel]]\n" - "neg z25.h, p2/M, z25.h\n" + "ldr x17, [%x[args], %[offsetof_Args_current_channel]]\n" + "neg z17.h, p2/M, z17.h\n" "eor p8.b, p2/Z, p8.b, p9.b\n" - "ld1rw { z3.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_mul]]\n" - "ld1rw { z1.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z7.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z4.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_right_shift]]\n" "ld1rw { z24.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" - "ld1rw { z31.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" + "ld1rw { z16.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" "1:" // Channel loop "ldr x20, [%x[qp], %[offsetof_Requantize32_bias]]\n" - "mov z6.s, #0x0\n" + "mov z18.s, #0x0\n" "cbz x20, 2f\n" - "ld1w { z6.s }, p1/Z, [x20, x5, LSL #2]\n" + "ld1w { z18.s }, p1/Z, [x20, x17, LSL #2]\n" "2:" // Load bias: Done "ldr x23, [%x[args], %[offsetof_Args_weights]]\n" - "mov x22, x23\n" - "ld1b { z18.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "ld1rh { z12.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" - "mov z2.h, #0x0\n" - "sub z18.h, z18.h, z12.h\n" + "mov x20, x23\n" + "ld1b { z2.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "ld1rh { z3.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" + "mov z15.h, #0x0\n" + "sub z2.h, z2.h, z3.h\n" "incw x23\n" - "ld1b { z17.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "sub z17.h, z17.h, z12.h\n" - "trn1 z0.h, z2.h, z18.h\n" - "ld1b { z21.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "sub z21.h, z21.h, z12.h\n" - "trn1 z8.h, z18.h, z17.h\n" - "ld1b { z16.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "sub z16.h, z16.h, z12.h\n" - "trn1 z4.h, z17.h, z21.h\n" - "ld1b { z15.s }, p2/Z, [x22]\n" - "sub z15.h, z15.h, z12.h\n" - "mov x22, x23\n" - "trn1 z5.h, z21.h, z16.h\n" - "ld1b { z18.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "trn1 z10.h, z16.h, z15.h\n" + "ld1b { z13.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "sub z13.h, z13.h, z3.h\n" "trn1 z11.h, z15.h, z2.h\n" - "ld1b { z17.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "sub z18.h, z18.h, z12.h\n" - "sub z17.h, z17.h, z12.h\n" - "ld1b { z21.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "sub z21.h, z21.h, z12.h\n" - "addvl x21, SP, #30\n" - "ld1b { z16.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" + "ld1b { z27.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "sub z27.h, z27.h, z3.h\n" + "trn1 z0.h, z2.h, z13.h\n" + "ld1b { z19.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "sub z19.h, z19.h, z3.h\n" + "trn1 z26.h, z13.h, z27.h\n" + "ld1b { z14.s }, p2/Z, [x20]\n" + "sub z14.h, z14.h, z3.h\n" + "mov x20, x23\n" + "trn1 z10.h, z27.h, z19.h\n" + "ld1b { z9.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "trn1 z19.h, z19.h, z14.h\n" + "trn1 z1.h, z14.h, z15.h\n" + "ld1b { z5.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "sub z9.h, z9.h, z3.h\n" + "sub z5.h, z5.h, z3.h\n" + "ld1b { z29.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "sub z29.h, z29.h, z3.h\n" + "addvl x22, SP, #30\n" + "ld1b { z2.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" "incw x23\n" - "sub z16.h, z16.h, z12.h\n" - "ld1b { z15.s }, p2/Z, [x22]\n" - "addvl x21, x21, #-6\n" - "sub z15.h, z15.h, z12.h\n" - "mov x22, x23\n" - "st1h { z0.h }, p2, [x21]\n" - "trn1 z0.h, z2.h, z18.h\n" + "sub z2.h, z2.h, z3.h\n" + "ld1b { z23.s }, p2/Z, [x20]\n" + "addvl x22, x22, #-6\n" + "sub z23.h, z23.h, z3.h\n" + "mov x20, x23\n" + "st1h { z11.h }, p2, [x22]\n" + "trn1 z20.h, z15.h, z9.h\n" "incw x23\n" - "ldr x20, [%x[qp], %[offsetof_Requantize32_per_channel_muls]]\n" - "st1h { z8.h }, p2, [x21, #1, MUL VL]\n" - "trn1 z8.h, z18.h, z17.h\n" - "ld1b { z18.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z4.h }, p2, [x21, #2, MUL VL]\n" - "trn1 z4.h, z17.h, z21.h\n" - "ld1b { z17.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z5.h }, p2, [x21, #3, MUL VL]\n" - "trn1 z5.h, z21.h, z16.h\n" - "ld1b { z21.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z10.h }, p2, [x21, #4, MUL VL]\n" - "trn1 z10.h, z16.h, z15.h\n" - "ld1b { z16.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z11.h }, p2, [x21, #5, MUL VL]\n" - "trn1 z11.h, z15.h, z2.h\n" - "sub z18.h, z18.h, z12.h\n" - "addvl x21, x21, #-6\n" - "sub z17.h, z17.h, z12.h\n" - "ld1b { z15.s }, p2/Z, [x22]\n" - "sub z21.h, z21.h, z12.h\n" - "mov x22, x23\n" - "sub z16.h, z16.h, z12.h\n" - "sub z15.h, z15.h, z12.h\n" - "st1h { z0.h }, p2, [x21]\n" + "ldr x21, [%x[qp], %[offsetof_Requantize32_per_channel_muls]]\n" + "st1h { z0.h }, p2, [x22, #1, MUL VL]\n" + "trn1 z22.h, z9.h, z5.h\n" + "ld1b { z25.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z26.h }, p2, [x22, #2, MUL VL]\n" + "trn1 z9.h, z5.h, z29.h\n" + "ld1b { z21.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z10.h }, p2, [x22, #3, MUL VL]\n" + "trn1 z26.h, z29.h, z2.h\n" + "ld1b { z0.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z19.h }, p2, [x22, #4, MUL VL]\n" + "trn1 z28.h, z2.h, z23.h\n" + "ld1b { z19.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z1.h }, p2, [x22, #5, MUL VL]\n" + "trn1 z2.h, z23.h, z15.h\n" + "sub z25.h, z25.h, z3.h\n" + "addvl x22, x22, #-6\n" + "sub z21.h, z21.h, z3.h\n" + "ld1b { z6.s }, p2/Z, [x20]\n" + "sub z0.h, z0.h, z3.h\n" + "mov x20, x23\n" + "sub z19.h, z19.h, z3.h\n" + "sub z6.h, z6.h, z3.h\n" + "st1h { z20.h }, p2, [x22]\n" "incw x23\n" - "st1h { z8.h }, p2, [x21, #1, MUL VL]\n" - "trn1 z0.h, z2.h, z18.h\n" - "trn1 z8.h, z18.h, z17.h\n" - "ld1b { z18.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z4.h }, p2, [x21, #2, MUL VL]\n" - "trn1 z4.h, z17.h, z21.h\n" - "ld1b { z17.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z5.h }, p2, [x21, #3, MUL VL]\n" - "trn1 z5.h, z21.h, z16.h\n" - "ld1b { z21.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z10.h }, p2, [x21, #4, MUL VL]\n" - "trn1 z10.h, z16.h, z15.h\n" - "ld1b { z16.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z11.h }, p2, [x21, #5, MUL VL]\n" - "trn1 z11.h, z15.h, z2.h\n" - "sub z18.h, z18.h, z12.h\n" - "sub z17.h, z17.h, z12.h\n" - "ld1b { z15.s }, p2/Z, [x22]\n" - "addvl x21, x21, #-6\n" - "sub z21.h, z21.h, z12.h\n" - "sub z16.h, z16.h, z12.h\n" - "mov x22, x23\n" - "st1h { z0.h }, p2, [x21]\n" - "sub z15.h, z15.h, z12.h\n" - "st1h { z8.h }, p2, [x21, #1, MUL VL]\n" - "trn1 z0.h, z2.h, z18.h\n" - "trn1 z8.h, z18.h, z17.h\n" - "ld1b { z18.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z4.h }, p2, [x21, #2, MUL VL]\n" - "trn1 z4.h, z17.h, z21.h\n" - "ld1b { z17.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z5.h }, p2, [x21, #3, MUL VL]\n" - "trn1 z5.h, z21.h, z16.h\n" - "ld1b { z21.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z10.h }, p2, [x21, #4, MUL VL]\n" - "trn1 z10.h, z16.h, z15.h\n" - "ld1b { z16.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z11.h }, p2, [x21, #5, MUL VL]\n" - "trn1 z11.h, z15.h, z2.h\n" - "ld1b { z15.s }, p2/Z, [x22]\n" - "sub z18.h, z18.h, z12.h\n" - "addvl x21, x21, #-6\n" - "sub z17.h, z17.h, z12.h\n" - "sub z21.h, z21.h, z12.h\n" - "st1h { z0.h }, p2, [x21]\n" - "sub z16.h, z16.h, z12.h\n" - "sub z15.h, z15.h, z12.h\n" - "st1h { z8.h }, p2, [x21, #1, MUL VL]\n" - "st1h { z4.h }, p2, [x21, #2, MUL VL]\n" - "mov z7.d, z6.d\n" - "trn1 z0.h, z2.h, z18.h\n" - "st1h { z5.h }, p2, [x21, #3, MUL VL]\n" - "trn1 z8.h, z18.h, z17.h\n" - "trn1 z4.h, z17.h, z21.h\n" - "st1h { z10.h }, p2, [x21, #4, MUL VL]\n" - "trn1 z5.h, z21.h, z16.h\n" - "trn1 z10.h, z16.h, z15.h\n" - "st1h { z11.h }, p2, [x21, #5, MUL VL]\n" - "addvl x21, x21, #-6\n" - "trn1 z11.h, z15.h, z2.h\n" - "st1h { z0.h }, p2, [x21]\n" - "st1h { z8.h }, p2, [x21, #1, MUL VL]\n" - "st1h { z4.h }, p2, [x21, #2, MUL VL]\n" - "st1h { z5.h }, p2, [x21, #3, MUL VL]\n" - "st1h { z10.h }, p2, [x21, #4, MUL VL]\n" - "st1h { z11.h }, p2, [x21, #5, MUL VL]\n" - "cbz x20, 3f\n" - "ld1w { z3.s }, p1/Z, [x20, x5, LSL #2]\n" + "st1h { z22.h }, p2, [x22, #1, MUL VL]\n" + "trn1 z11.h, z15.h, z25.h\n" + "trn1 z10.h, z25.h, z21.h\n" + "ld1b { z5.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z9.h }, p2, [x22, #2, MUL VL]\n" + "trn1 z14.h, z21.h, z0.h\n" + "ld1b { z23.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z26.h }, p2, [x22, #3, MUL VL]\n" + "trn1 z21.h, z0.h, z19.h\n" + "ld1b { z27.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z28.h }, p2, [x22, #4, MUL VL]\n" + "trn1 z19.h, z19.h, z6.h\n" + "ld1b { z29.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z2.h }, p2, [x22, #5, MUL VL]\n" + "trn1 z13.h, z6.h, z15.h\n" + "sub z5.h, z5.h, z3.h\n" + "sub z23.h, z23.h, z3.h\n" + "ld1b { z1.s }, p2/Z, [x20]\n" + "addvl x22, x22, #-6\n" + "sub z27.h, z27.h, z3.h\n" + "sub z29.h, z29.h, z3.h\n" + "mov x20, x23\n" + "st1h { z11.h }, p2, [x22]\n" + "sub z1.h, z1.h, z3.h\n" + "st1h { z10.h }, p2, [x22, #1, MUL VL]\n" + "trn1 z30.h, z15.h, z5.h\n" + "trn1 z26.h, z5.h, z23.h\n" + "ld1b { z11.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z14.h }, p2, [x22, #2, MUL VL]\n" + "trn1 z22.h, z23.h, z27.h\n" + "ld1b { z5.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z21.h }, p2, [x22, #3, MUL VL]\n" + "trn1 z28.h, z27.h, z29.h\n" + "ld1b { z8.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z19.h }, p2, [x22, #4, MUL VL]\n" + "trn1 z27.h, z29.h, z1.h\n" + "ld1b { z9.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z13.h }, p2, [x22, #5, MUL VL]\n" + "trn1 z2.h, z1.h, z15.h\n" + "ld1b { z14.s }, p2/Z, [x20]\n" + "sub z11.h, z11.h, z3.h\n" + "addvl x22, x22, #-6\n" + "sub z5.h, z5.h, z3.h\n" + "sub z8.h, z8.h, z3.h\n" + "st1h { z30.h }, p2, [x22]\n" + "sub z9.h, z9.h, z3.h\n" + "sub z14.h, z14.h, z3.h\n" + "st1h { z26.h }, p2, [x22, #1, MUL VL]\n" + "st1h { z22.h }, p2, [x22, #2, MUL VL]\n" + "mov z19.d, z18.d\n" + "trn1 z22.h, z15.h, z11.h\n" + "st1h { z28.h }, p2, [x22, #3, MUL VL]\n" + "trn1 z1.h, z11.h, z5.h\n" + "trn1 z31.h, z5.h, z8.h\n" + "st1h { z27.h }, p2, [x22, #4, MUL VL]\n" + "trn1 z8.h, z8.h, z9.h\n" + "trn1 z21.h, z9.h, z14.h\n" + "st1h { z2.h }, p2, [x22, #5, MUL VL]\n" + "addvl x22, x22, #-6\n" + "trn1 z15.h, z14.h, z15.h\n" + "st1h { z22.h }, p2, [x22]\n" + "st1h { z1.h }, p2, [x22, #1, MUL VL]\n" + "st1h { z31.h }, p2, [x22, #2, MUL VL]\n" + "st1h { z8.h }, p2, [x22, #3, MUL VL]\n" + "st1h { z21.h }, p2, [x22, #4, MUL VL]\n" + "st1h { z15.h }, p2, [x22, #5, MUL VL]\n" + "cbz x21, 3f\n" + "ld1w { z7.s }, p1/Z, [x21, x17, LSL #2]\n" "3:" // Load mul: End "ldr x20, [%x[qp], %[offsetof_Requantize32_per_channel_right_shifts]]\n" "cbz x20, 4f\n" - "ld1w { z1.s }, p1/Z, [x20, x5, LSL #2]\n" + "ld1w { z4.s }, p1/Z, [x20, x17, LSL #2]\n" "4:" // Load right_shift: End - "ldr x17, [%x[args], %[offsetof_Args_input_cols]]\n" - "sub x20, x17, #0x1\n" + "ldr x25, [%x[args], %[offsetof_Args_input_cols]]\n" + "sub x20, x25, #0x1\n" "orr x23, x20, %x[ld_in_col], LSL #16\n" "ldr x16, [%x[args], %[offsetof_Args_inptr]]\n" "orr x23, x7, x23, LSL #22\n" "mov x22, #0x8\n" - "add x21, x6, x4\n" + "add x21, x6, x5\n" "lsl x20, %x[ld_in_row], #0x0\n" "ldr x15, [%x[args], %[offsetof_Args_output_cols]]\n" "mov x11, #0x0\n" @@ -271,56 +271,56 @@ void sme2_u8q_planar_5x5_s1_4rows_dot_za_impl( ".inst 0xf8b74a9c // rprfm pldstrm, x23, [x20]\n" "add x20, x20, %x[ld_in_col]\n" "bgt 5b\n" - "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" + "ldr x23, [%x[args], %[offsetof_Args_outptrs]]\n" "lsl x20, %x[ld_in_row], #0x0\n" "msub x16, x6, x20, x16\n" - ".inst 0xc00468c0 // mova za.d[x11, #0], { z6.d-z7.d }\n" + ".inst 0xc0046a40 // mova za.d[x11, #0], { z18.d-z19.d }\n" "ldr x20, [%x[args], %[offsetof_Args_ld_out_cols]]\n" - ".inst 0xc00468c1 // mova za.d[x11, #1], { z6.d-z7.d }\n" + ".inst 0xc0046a41 // mova za.d[x11, #1], { z18.d-z19.d }\n" "mov x22, #0x4\n" - "ldp x14, x13, [x25], #0x10\n" - ".inst 0xc00468c2 // mova za.d[x11, #2], { z6.d-z7.d }\n" - "ldp x3, x10, [x20], #0x10\n" - ".inst 0xc00468c3 // mova za.d[x11, #3], { z6.d-z7.d }\n" + "ldp x14, x13, [x23], #0x10\n" + ".inst 0xc0046a42 // mova za.d[x11, #2], { z18.d-z19.d }\n" + "ldp x4, x10, [x20], #0x10\n" + ".inst 0xc0046a43 // mova za.d[x11, #3], { z18.d-z19.d }\n" "ldr x21, [%x[args], %[offsetof_Args_pad_left]]\n" - ".inst 0xc00468c4 // mova za.d[x11, #4], { z6.d-z7.d }\n" - "ldp x9, x28, [x25], #0x10\n" - ".inst 0xc00468c5 // mova za.d[x11, #5], { z6.d-z7.d }\n" + ".inst 0xc0046a44 // mova za.d[x11, #4], { z18.d-z19.d }\n" + "ldp x9, x28, [x23], #0x10\n" + ".inst 0xc0046a45 // mova za.d[x11, #5], { z18.d-z19.d }\n" "ldp x27, x26, [x20], #0x10\n" - ".inst 0xc00468c6 // mova za.d[x11, #6], { z6.d-z7.d }\n" - ".inst 0xc00468c7 // mova za.d[x11, #7], { z6.d-z7.d }\n" - ".inst 0xc00408c0 // mova za.d[x8, #0], { z6.d-z7.d }\n" - ".inst 0xc00408c1 // mova za.d[x8, #1], { z6.d-z7.d }\n" + ".inst 0xc0046a46 // mova za.d[x11, #6], { z18.d-z19.d }\n" + ".inst 0xc0046a47 // mova za.d[x11, #7], { z18.d-z19.d }\n" + ".inst 0xc0040a40 // mova za.d[x8, #0], { z18.d-z19.d }\n" + ".inst 0xc0040a41 // mova za.d[x8, #1], { z18.d-z19.d }\n" "cbz x21, 7f\n" "cmp x21, x22\n" "csel x20, x21, x22, LT\n" "sub x21, x21, x20\n" "sub x22, x22, x20\n" "cbz x21, 7f\n" - ".inst 0xc006680c // mova { z12.d-z13.d }, za.d[x11, #0]\n" + ".inst 0xc0066814 // mova { z20.d-z21.d }, za.d[x11, #0]\n" "sub x15, x15, x21\n" - ".inst 0xc006682e // mova { z14.d-z15.d }, za.d[x11, #1]\n" - ".inst 0xc1a3ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z3.s\n" - ".inst 0xc1a1aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n" - ".inst 0xc1a9ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z9.s\n" - ".inst 0xc1bfcf0c // sclamp { z12.s-z15.s }, z24.s, z31.s\n" + ".inst 0xc0066836 // mova { z22.d-z23.d }, za.d[x11, #1]\n" + ".inst 0xc1a7ac14 // sqdmulh { z20.s-z23.s }, { z20.s-z23.s }, z7.s\n" + ".inst 0xc1a4aa34 // srshl { z20.s-z23.s }, { z20.s-z23.s }, z4.s\n" + ".inst 0xc1acab14 // add { z20.s-z23.s }, { z20.s-z23.s }, z12.s\n" + ".inst 0xc1b0cf14 // sclamp { z20.s-z23.s }, z24.s, z16.s\n" "6:" // Left padding "subs x21, x21, #0x1\n" - "st1b { z12.s }, p1, [x14]\n" - "add x14, x14, x3\n" - "st1b { z14.s }, p1, [x13]\n" + "st1b { z20.s }, p1, [x14]\n" + "add x14, x14, x4\n" + "st1b { z22.s }, p1, [x13]\n" "add x13, x13, x10\n" - "st1b { z13.s }, p1, [x9]\n" + "st1b { z21.s }, p1, [x9]\n" "add x9, x9, x27\n" - "st1b { z15.s }, p1, [x28]\n" + "st1b { z23.s }, p1, [x28]\n" "add x28, x28, x26\n" "bgt 6b\n" "7:" // Left padding: End - "adds XZR, x6, x4\n" + "adds XZR, x6, x5\n" "bne 14f\n" "cbz x22, 12f\n" "cmp x22, #0x1\n" - "sub x17, x17, x22\n" + "sub x25, x25, x22\n" "beq 11f\n" "cmp x22, #0x2\n" "beq 10f\n" @@ -328,338 +328,338 @@ void sme2_u8q_planar_5x5_s1_4rows_dot_za_impl( "beq 9f\n" "8:" // Unpadded: 4 priming loads "add x21, x16, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x16]\n" + "ld1b { z1.s }, p1/Z, [x16]\n" "addvl x20, SP, #24\n" - "ld1b { z16.s }, p1/Z, [x21]\n" + "ld1b { z28.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z27.h, z17.h, z16.h\n" - "add z27.h, z27.h, z25.h\n" - "ld1b { z17.s }, p1/Z, [x21]\n" + "trn1 z27.h, z1.h, z28.h\n" + "add z27.h, z27.h, z17.h\n" + "ld1b { z1.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "add x16, x16, %x[ld_in_col]\n" - "ld1b { z16.s }, p1/Z, [x21]\n" + "ld1b { z2.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z28.h, z17.h, z16.h\n" - "add z28.h, z28.h, z25.h\n" - "ld1b { z16.s }, p1/Z, [x21]\n" + "trn1 z28.h, z1.h, z2.h\n" + "add z28.h, z28.h, z17.h\n" + "ld1b { z13.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1b { z29.s }, p1/Z, [x21]\n" + "ld1b { z6.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z29.h, z16.h, z29.h\n" - "add z29.h, z29.h, z25.h\n" - "ld1b { z17.s }, p1/Z, [x21]\n" + "trn1 z29.h, z13.h, z6.h\n" + "add z29.h, z29.h, z17.h\n" + "ld1b { z30.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" - "ld1b { z16.s }, p1/Z, [x21]\n" - "trn1 z30.h, z17.h, z16.h\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - "add z30.h, z30.h, z25.h\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" + ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" + ".inst 0xc16a7768 // sdot za.s[x11, 0], { z27.h-z28.h }, z10.h\n" + "ld1b { z20.s }, p1/Z, [x21]\n" + "trn1 z30.h, z30.h, z20.h\n" + ".inst 0xc1627769 // sdot za.s[x11, 1], { z27.h-z28.h }, z2.h\n" + ".inst 0xa1412a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + "add z30.h, z30.h, z17.h\n" + ".inst 0xc1697788 // sdot za.s[x11, 0], { z28.h-z29.h }, z9.h\n" + ".inst 0xc1617789 // sdot za.s[x11, 1], { z28.h-z29.h }, z1.h\n" ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" "9:" // Unpadded: 3 priming loads "add x22, x16, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x16]\n" + "ld1b { z2.s }, p1/Z, [x16]\n" "addvl x21, SP, #18\n" - "ld1b { z16.s }, p1/Z, [x22]\n" + "ld1b { z28.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z27.h, z17.h, z16.h\n" - "add z27.h, z27.h, z25.h\n" - "ld1b { z17.s }, p1/Z, [x22]\n" + "trn1 z20.h, z2.h, z28.h\n" + "add z20.h, z20.h, z17.h\n" + "ld1b { z31.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "addvl x20, SP, #24\n" - "ld1b { z16.s }, p1/Z, [x22]\n" + "ld1b { z11.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z28.h, z17.h, z16.h\n" - "add z28.h, z28.h, z25.h\n" - "ld1b { z17.s }, p1/Z, [x22]\n" + "trn1 z21.h, z31.h, z11.h\n" + "add z21.h, z21.h, z17.h\n" + "ld1b { z25.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "add x16, x16, %x[ld_in_col]\n" - "ld1b { z16.s }, p1/Z, [x22]\n" + "ld1b { z8.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z29.h, z17.h, z16.h\n" - "add z29.h, z29.h, z25.h\n" - "ld1b { z17.s }, p1/Z, [x22]\n" + "trn1 z22.h, z25.h, z8.h\n" + "add z22.h, z22.h, z17.h\n" + "ld1b { z8.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" - "ld1b { z16.s }, p1/Z, [x22]\n" - "trn1 z30.h, z17.h, z16.h\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" - "add z30.h, z30.h, z25.h\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" + ".inst 0xa1402aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21]\n" + ".inst 0xc16e7688 // sdot za.s[x11, 0], { z20.h-z21.h }, z14.h\n" + "ld1b { z3.s }, p1/Z, [x22]\n" + "trn1 z23.h, z8.h, z3.h\n" + ".inst 0xc1667689 // sdot za.s[x11, 1], { z20.h-z21.h }, z6.h\n" + ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" + ".inst 0xc161768a // sdot za.s[x11, 2], { z20.h-z21.h }, z1.h\n" + "add z23.h, z23.h, z17.h\n" + ".inst 0xa1412aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc160768b // sdot za.s[x11, 3], { z20.h-z21.h }, z0.h\n" + ".inst 0xc16976a8 // sdot za.s[x11, 0], { z21.h-z22.h }, z9.h\n" + ".inst 0xa0422aae // ld1h { z14.h-z15.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc16176a9 // sdot za.s[x11, 1], { z21.h-z22.h }, z1.h\n" + ".inst 0xa1412a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc16976aa // sdot za.s[x11, 2], { z21.h-z22.h }, z9.h\n" + ".inst 0xc16176ab // sdot za.s[x11, 3], { z21.h-z22.h }, z1.h\n" + ".inst 0xc16f76c8 // sdot za.s[x11, 0], { z22.h-z23.h }, z15.h\n" + ".inst 0xc16e76c9 // sdot za.s[x11, 1], { z22.h-z23.h }, z14.h\n" ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" + ".inst 0xc16b76ca // sdot za.s[x11, 2], { z22.h-z23.h }, z11.h\n" + ".inst 0xc16a76cb // sdot za.s[x11, 3], { z22.h-z23.h }, z10.h\n" "10:" // Unpadded: 2 priming loads "add x23, x16, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x16]\n" + "ld1b { z2.s }, p1/Z, [x16]\n" "addvl x22, SP, #12\n" - "ld1b { z16.s }, p1/Z, [x23]\n" + "ld1b { z22.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - "trn1 z27.h, z17.h, z16.h\n" - "add z27.h, z27.h, z25.h\n" - "ld1b { z17.s }, p1/Z, [x23]\n" + "trn1 z0.h, z2.h, z22.h\n" + "add z0.h, z0.h, z17.h\n" + "ld1b { z14.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" "addvl x21, SP, #18\n" - "ld1b { z16.s }, p1/Z, [x23]\n" + "ld1b { z6.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - "trn1 z28.h, z17.h, z16.h\n" - "add z28.h, z28.h, z25.h\n" - "ld1b { z17.s }, p1/Z, [x23]\n" + "trn1 z1.h, z14.h, z6.h\n" + "add z1.h, z1.h, z17.h\n" + "ld1b { z15.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" "addvl x20, SP, #24\n" - "ld1b { z16.s }, p1/Z, [x23]\n" + "ld1b { z6.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - "trn1 z29.h, z17.h, z16.h\n" - "add z29.h, z29.h, z25.h\n" - "ld1b { z17.s }, p1/Z, [x23]\n" + "trn1 z2.h, z15.h, z6.h\n" + "add z2.h, z2.h, z17.h\n" + "ld1b { z21.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" "add x16, x16, %x[ld_in_col]\n" - ".inst 0xa1402ac0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x22]\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" - "ld1b { z16.s }, p1/Z, [x23]\n" - "trn1 z30.h, z17.h, z16.h\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" - "add z30.h, z30.h, z25.h\n" - ".inst 0xa0412ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xa0422aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc168776c // sdot za.s[x11, 4], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776d // sdot za.s[x11, 5], { z27.h-z28.h }, z0.h\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" + ".inst 0xa0402ace // ld1h { z14.h-z15.h }, pn10.b/Z, [x22]\n" + ".inst 0xc16f7408 // sdot za.s[x11, 0], { z0.h-z1.h }, z15.h\n" + "ld1b { z30.s }, p1/Z, [x23]\n" + "trn1 z3.h, z21.h, z30.h\n" + ".inst 0xc16e7409 // sdot za.s[x11, 1], { z0.h-z1.h }, z14.h\n" + ".inst 0xa1402aa5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x21]\n" + ".inst 0xc16d740a // sdot za.s[x11, 2], { z0.h-z1.h }, z13.h\n" + "add z3.h, z3.h, z17.h\n" + ".inst 0xa0412ace // ld1h { z14.h-z15.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc165740b // sdot za.s[x11, 3], { z0.h-z1.h }, z5.h\n" + ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" + ".inst 0xc16f7428 // sdot za.s[x11, 0], { z1.h-z2.h }, z15.h\n" + ".inst 0xc16e7429 // sdot za.s[x11, 1], { z1.h-z2.h }, z14.h\n" + ".inst 0xa0412aae // ld1h { z14.h-z15.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xa0422ac8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc16b740c // sdot za.s[x11, 4], { z0.h-z1.h }, z11.h\n" + ".inst 0xc16a740d // sdot za.s[x11, 5], { z0.h-z1.h }, z10.h\n" + ".inst 0xc16f742a // sdot za.s[x11, 2], { z1.h-z2.h }, z15.h\n" + ".inst 0xc16e742b // sdot za.s[x11, 3], { z1.h-z2.h }, z14.h\n" + ".inst 0xa0412a8e // ld1h { z14.h-z15.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc1697448 // sdot za.s[x11, 0], { z2.h-z3.h }, z9.h\n" + ".inst 0xc1687449 // sdot za.s[x11, 1], { z2.h-z3.h }, z8.h\n" ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc165778c // sdot za.s[x11, 4], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778d // sdot za.s[x11, 5], { z28.h-z29.h }, z4.h\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc16b77ac // sdot za.s[x11, 4], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ad // sdot za.s[x11, 5], { z29.h-z30.h }, z10.h\n" + ".inst 0xc16f742c // sdot za.s[x11, 4], { z1.h-z2.h }, z15.h\n" + ".inst 0xc16e742d // sdot za.s[x11, 5], { z1.h-z2.h }, z14.h\n" + ".inst 0xc16b744a // sdot za.s[x11, 2], { z2.h-z3.h }, z11.h\n" + ".inst 0xc16a744b // sdot za.s[x11, 3], { z2.h-z3.h }, z10.h\n" + ".inst 0xa0422a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc161744c // sdot za.s[x11, 4], { z2.h-z3.h }, z1.h\n" + ".inst 0xc160744d // sdot za.s[x11, 5], { z2.h-z3.h }, z0.h\n" "11:" // Unpadded: 1 priming loads "add x24, x16, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x16]\n" + "ld1b { z0.s }, p1/Z, [x16]\n" "addvl x23, SP, #6\n" - "ld1b { z16.s }, p1/Z, [x24]\n" + "ld1b { z3.s }, p1/Z, [x24]\n" "add x24, x24, %x[ld_in_row]\n" - "trn1 z27.h, z17.h, z16.h\n" - "add z27.h, z27.h, z25.h\n" - "ld1b { z17.s }, p1/Z, [x24]\n" + "trn1 z28.h, z0.h, z3.h\n" + "add z28.h, z28.h, z17.h\n" + "ld1b { z6.s }, p1/Z, [x24]\n" "add x24, x24, %x[ld_in_row]\n" "addvl x22, SP, #12\n" - "ld1b { z16.s }, p1/Z, [x24]\n" + "ld1b { z30.s }, p1/Z, [x24]\n" "add x24, x24, %x[ld_in_row]\n" - "trn1 z28.h, z17.h, z16.h\n" - "add z28.h, z28.h, z25.h\n" - "ld1b { z17.s }, p1/Z, [x24]\n" + "trn1 z29.h, z6.h, z30.h\n" + "add z29.h, z29.h, z17.h\n" + "ld1b { z1.s }, p1/Z, [x24]\n" "add x24, x24, %x[ld_in_row]\n" "addvl x21, SP, #18\n" - "ld1b { z16.s }, p1/Z, [x24]\n" + "ld1b { z25.s }, p1/Z, [x24]\n" "add x24, x24, %x[ld_in_row]\n" - "trn1 z29.h, z17.h, z16.h\n" - "add z29.h, z29.h, z25.h\n" - "ld1b { z17.s }, p1/Z, [x24]\n" + "trn1 z30.h, z1.h, z25.h\n" + "add z30.h, z30.h, z17.h\n" + "ld1b { z3.s }, p1/Z, [x24]\n" "add x24, x24, %x[ld_in_row]\n" "addvl x20, SP, #24\n" - ".inst 0xa1402ae0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x23]\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" + ".inst 0xa0402ae0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x23]\n" + ".inst 0xc1617788 // sdot za.s[x11, 0], { z28.h-z29.h }, z1.h\n" "add x16, x16, %x[ld_in_col]\n" - "ld1b { z16.s }, p1/Z, [x24]\n" - "trn1 z30.h, z17.h, z16.h\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402ac0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x22]\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" - "add z30.h, z30.h, z25.h\n" - ".inst 0xa0412ae4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xa0422aea // ld1h { z10.h-z11.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xc168776c // sdot za.s[x11, 4], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776d // sdot za.s[x11, 5], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc168776e // sdot za.s[x11, 6], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776f // sdot za.s[x11, 7], { z27.h-z28.h }, z0.h\n" - ".inst 0xc165778c // sdot za.s[x11, 4], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778d // sdot za.s[x11, 5], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc165778e // sdot za.s[x11, 6], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778f // sdot za.s[x11, 7], { z28.h-z29.h }, z4.h\n" - ".inst 0xc16b77ac // sdot za.s[x11, 4], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ad // sdot za.s[x11, 5], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc16b77ae // sdot za.s[x11, 6], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77af // sdot za.s[x11, 7], { z29.h-z30.h }, z10.h\n" + "ld1b { z5.s }, p1/Z, [x24]\n" + "trn1 z31.h, z3.h, z5.h\n" + ".inst 0xc1607789 // sdot za.s[x11, 1], { z28.h-z29.h }, z0.h\n" + ".inst 0xa1402ac6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x22]\n" + ".inst 0xc16e778a // sdot za.s[x11, 2], { z28.h-z29.h }, z14.h\n" + "add z31.h, z31.h, z17.h\n" + ".inst 0xa1412ae2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xc166778b // sdot za.s[x11, 3], { z28.h-z29.h }, z6.h\n" + ".inst 0xa0402aae // ld1h { z14.h-z15.h }, pn10.b/Z, [x21]\n" + ".inst 0xc16a77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z10.h\n" + ".inst 0xc16277a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z2.h\n" + ".inst 0xa0412ac8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xa1422ae2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xc16f778c // sdot za.s[x11, 4], { z28.h-z29.h }, z15.h\n" + ".inst 0xc16e778d // sdot za.s[x11, 5], { z28.h-z29.h }, z14.h\n" + ".inst 0xa1402a86 // ld1h { z6.h, z14.h }, pn10.b/Z, [x20]\n" + ".inst 0xc16977aa // sdot za.s[x11, 2], { z29.h-z30.h }, z9.h\n" + ".inst 0xc16877ab // sdot za.s[x11, 3], { z29.h-z30.h }, z8.h\n" + ".inst 0xa1412aa5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc16a77c8 // sdot za.s[x11, 0], { z30.h-z31.h }, z10.h\n" + ".inst 0xc16277c9 // sdot za.s[x11, 1], { z30.h-z31.h }, z2.h\n" + ".inst 0xa1422ac2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc16e778e // sdot za.s[x11, 6], { z28.h-z29.h }, z14.h\n" + ".inst 0xc166778f // sdot za.s[x11, 7], { z28.h-z29.h }, z6.h\n" + ".inst 0xc16d77ac // sdot za.s[x11, 4], { z29.h-z30.h }, z13.h\n" + ".inst 0xc16577ad // sdot za.s[x11, 5], { z29.h-z30.h }, z5.h\n" + ".inst 0xa1412a86 // ld1h { z6.h, z14.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc16a77ca // sdot za.s[x11, 2], { z30.h-z31.h }, z10.h\n" + ".inst 0xc16277cb // sdot za.s[x11, 3], { z30.h-z31.h }, z2.h\n" + ".inst 0xa0422aa8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc16e77ae // sdot za.s[x11, 6], { z29.h-z30.h }, z14.h\n" + ".inst 0xc16677af // sdot za.s[x11, 7], { z29.h-z30.h }, z6.h\n" + ".inst 0xc16977cc // sdot za.s[x11, 4], { z30.h-z31.h }, z9.h\n" + ".inst 0xc16877cd // sdot za.s[x11, 5], { z30.h-z31.h }, z8.h\n" + ".inst 0xa1422a86 // ld1h { z6.h, z14.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc16e77ce // sdot za.s[x11, 6], { z30.h-z31.h }, z14.h\n" + ".inst 0xc16677cf // sdot za.s[x11, 7], { z30.h-z31.h }, z6.h\n" "12:" // Unpadded: 0 priming loads - ".inst 0xa1402be0 // ld1h { z0.h, z8.h }, pn10.b/Z, [SP]\n" - ".inst 0xa0412be4 // ld1h { z4.h-z5.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" - ".inst 0xa0422bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" - "cbz x17, 22f\n" + ".inst 0xa0402be0 // ld1h { z0.h-z1.h }, pn10.b/Z, [SP]\n" + ".inst 0xa1412be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + ".inst 0xa0422be2 // ld1h { z2.h-z3.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" + "cbz x25, 22f\n" "add x20, x16, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x16]\n" - "sub x17, x17, #0x1\n" - "ld1b { z16.s }, p1/Z, [x20]\n" + "ld1b { z26.s }, p1/Z, [x16]\n" + "sub x25, x25, #0x1\n" + "ld1b { z28.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z27.h, z17.h, z16.h\n" + "trn1 z25.h, z26.h, z28.h\n" "sub x15, x15, #0x1\n" - "ld1b { z17.s }, p1/Z, [x20]\n" + "ld1b { z31.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "cmp x17, x15\n" - "add z27.h, z27.h, z25.h\n" - "ld1b { z16.s }, p1/Z, [x20]\n" + "cmp x25, x15\n" + "add z25.h, z25.h, z17.h\n" + "ld1b { z15.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z28.h, z17.h, z16.h\n" - "csel x25, x17, x15, LT\n" - "ld1b { z17.s }, p1/Z, [x20]\n" + "trn1 z26.h, z31.h, z15.h\n" + "csel x25, x25, x15, LT\n" + "ld1b { z22.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "add z28.h, z28.h, z25.h\n" + "add z26.h, z26.h, z17.h\n" "add x16, x16, %x[ld_in_col]\n" - "ld1b { z16.s }, p1/Z, [x20]\n" + "ld1b { z8.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z29.h, z17.h, z16.h\n" - "add z29.h, z29.h, z25.h\n" - "ld1b { z17.s }, p1/Z, [x20]\n" + "trn1 z27.h, z22.h, z8.h\n" + "add z27.h, z27.h, z17.h\n" + "ld1b { z21.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" "sub x15, x15, x25\n" - "ld1b { z16.s }, p1/Z, [x20]\n" - "trn1 z30.h, z17.h, z16.h\n" - "add z30.h, z30.h, z25.h\n" + "ld1b { z20.s }, p1/Z, [x20]\n" + "trn1 z28.h, z21.h, z20.h\n" + "add z28.h, z28.h, z17.h\n" "cbz x25, 21f\n" "13:" // Unpadded: Main loop "addvl x24, SP, #6\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" + ".inst 0xc1617728 // sdot za.s[x11, 0], { z25.h-z26.h }, z1.h\n" "addvl x23, SP, #12\n" - "ld1b { z23.s }, p1/Z, [x16]\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402b00 // ld1h { z0.h, z8.h }, pn10.b/Z, [x24]\n" + "ld1b { z21.s }, p1/Z, [x16]\n" + ".inst 0xc1607729 // sdot za.s[x11, 1], { z25.h-z26.h }, z0.h\n" + ".inst 0xa0402b0e // ld1h { z14.h-z15.h }, pn10.b/Z, [x24]\n" "addvl x22, SP, #18\n" "addvl x21, SP, #24\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" + ".inst 0xc16f772a // sdot za.s[x11, 2], { z25.h-z26.h }, z15.h\n" "add x20, x16, %x[ld_in_row]\n" - "ld1b { z22.s }, p1/Z, [x20]\n" + "ld1b { z0.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402ae0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x23]\n" + ".inst 0xc16e772b // sdot za.s[x11, 3], { z25.h-z26.h }, z14.h\n" + ".inst 0xa1402ae6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x23]\n" "subs x25, x25, #0x1\n" "add x16, x16, %x[ld_in_col]\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - "ld1b { z21.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412b04 // ld1h { z4.h-z5.h }, pn10.b/Z, [x24, #0x2, MUL VL]\n" - ".inst 0xc168776c // sdot za.s[x11, 4], { z27.h-z28.h }, z8.h\n" + ".inst 0xc16d7748 // sdot za.s[x11, 0], { z26.h-z27.h }, z13.h\n" "ld1b { z20.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc160776d // sdot za.s[x11, 5], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402ac0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x22]\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - "ld1b { z19.s }, p1/Z, [x20]\n" + ".inst 0xc1657749 // sdot za.s[x11, 1], { z26.h-z27.h }, z5.h\n" + ".inst 0xa1412b05 // ld1h { z5.h, z13.h }, pn10.b/Z, [x24, #0x2, MUL VL]\n" + ".inst 0xc16e772c // sdot za.s[x11, 4], { z25.h-z26.h }, z14.h\n" + "ld1b { z31.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412ae4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - "ld1b { z18.s }, p1/Z, [x20]\n" + ".inst 0xc166772d // sdot za.s[x11, 5], { z25.h-z26.h }, z6.h\n" + ".inst 0xa0402ace // ld1h { z14.h-z15.h }, pn10.b/Z, [x22]\n" + ".inst 0xc16d774a // sdot za.s[x11, 2], { z26.h-z27.h }, z13.h\n" + "ld1b { z29.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422b0a // ld1h { z10.h-z11.h }, pn10.b/Z, [x24, #0x4, MUL VL]\n" - ".inst 0xc168776e // sdot za.s[x11, 6], { z27.h-z28.h }, z8.h\n" - "ld1b { z17.s }, p1/Z, [x20]\n" + ".inst 0xc165774b // sdot za.s[x11, 3], { z26.h-z27.h }, z5.h\n" + ".inst 0xa1412ae5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xc1637768 // sdot za.s[x11, 0], { z27.h-z28.h }, z3.h\n" + "ld1b { z22.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc160776f // sdot za.s[x11, 7], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" - ".inst 0xc165778c // sdot za.s[x11, 4], { z28.h-z29.h }, z5.h\n" - "ld1b { z16.s }, p1/Z, [x20]\n" - ".inst 0xc164778d // sdot za.s[x11, 5], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aea // ld1h { z10.h-z11.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xc165778e // sdot za.s[x11, 6], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778f // sdot za.s[x11, 7], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc16b77ac // sdot za.s[x11, 4], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ad // sdot za.s[x11, 5], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc16b77ae // sdot za.s[x11, 6], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77af // sdot za.s[x11, 7], { z29.h-z30.h }, z10.h\n" + ".inst 0xc1627769 // sdot za.s[x11, 1], { z27.h-z28.h }, z2.h\n" + ".inst 0xa1422b02 // ld1h { z2.h, z10.h }, pn10.b/Z, [x24, #0x4, MUL VL]\n" + ".inst 0xc16f772e // sdot za.s[x11, 6], { z25.h-z26.h }, z15.h\n" + "ld1b { z30.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row]\n" + ".inst 0xc16e772f // sdot za.s[x11, 7], { z25.h-z26.h }, z14.h\n" + ".inst 0xa0402aae // ld1h { z14.h-z15.h }, pn10.b/Z, [x21]\n" + ".inst 0xc16d774c // sdot za.s[x11, 4], { z26.h-z27.h }, z13.h\n" + "ld1b { z6.s }, p1/Z, [x20]\n" + ".inst 0xc165774d // sdot za.s[x11, 5], { z26.h-z27.h }, z5.h\n" + ".inst 0xa1412ac5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc16a776a // sdot za.s[x11, 2], { z27.h-z28.h }, z10.h\n" + ".inst 0xc162776b // sdot za.s[x11, 3], { z27.h-z28.h }, z2.h\n" + ".inst 0xa1422ae2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xc16d774e // sdot za.s[x11, 6], { z26.h-z27.h }, z13.h\n" + ".inst 0xc165774f // sdot za.s[x11, 7], { z26.h-z27.h }, z5.h\n" + ".inst 0xa1412aa5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc16a776c // sdot za.s[x11, 4], { z27.h-z28.h }, z10.h\n" + ".inst 0xc162776d // sdot za.s[x11, 5], { z27.h-z28.h }, z2.h\n" + ".inst 0xa1422ac1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc169776e // sdot za.s[x11, 6], { z27.h-z28.h }, z9.h\n" + ".inst 0xc161776f // sdot za.s[x11, 7], { z27.h-z28.h }, z1.h\n" ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc1681768 // sdot za.s[x8, 0], { z27.h-z28.h }, z8.h\n" - ".inst 0xc1601769 // sdot za.s[x8, 1], { z27.h-z28.h }, z0.h\n" - "trn1 z27.h, z23.h, z22.h\n" - ".inst 0xa1402be0 // ld1h { z0.h, z8.h }, pn10.b/Z, [SP]\n" - ".inst 0xc1651788 // sdot za.s[x8, 0], { z28.h-z29.h }, z5.h\n" - "add z27.h, z27.h, z25.h\n" - ".inst 0xc1641789 // sdot za.s[x8, 1], { z28.h-z29.h }, z4.h\n" - "trn1 z28.h, z21.h, z20.h\n" - ".inst 0xa0412be4 // ld1h { z4.h-z5.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" - ".inst 0xc16b17a8 // sdot za.s[x8, 0], { z29.h-z30.h }, z11.h\n" - "add z28.h, z28.h, z25.h\n" - ".inst 0xc16a17a9 // sdot za.s[x8, 1], { z29.h-z30.h }, z10.h\n" - "trn1 z29.h, z19.h, z18.h\n" - "trn1 z30.h, z17.h, z16.h\n" + ".inst 0xc16f1728 // sdot za.s[x8, 0], { z25.h-z26.h }, z15.h\n" + ".inst 0xc16e1729 // sdot za.s[x8, 1], { z25.h-z26.h }, z14.h\n" + "trn1 z25.h, z21.h, z0.h\n" + ".inst 0xa0402be0 // ld1h { z0.h-z1.h }, pn10.b/Z, [SP]\n" + ".inst 0xc16d1748 // sdot za.s[x8, 0], { z26.h-z27.h }, z13.h\n" + "add z25.h, z25.h, z17.h\n" + ".inst 0xc1651749 // sdot za.s[x8, 1], { z26.h-z27.h }, z5.h\n" + "trn1 z26.h, z20.h, z31.h\n" + ".inst 0xa1412be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + ".inst 0xc16b1768 // sdot za.s[x8, 0], { z27.h-z28.h }, z11.h\n" + "add z26.h, z26.h, z17.h\n" + ".inst 0xc16a1769 // sdot za.s[x8, 1], { z27.h-z28.h }, z10.h\n" + "trn1 z27.h, z29.h, z22.h\n" + "trn1 z28.h, z30.h, z6.h\n" "add x8, x8, #0x2\n" - ".inst 0xc006680c // mova { z12.d-z13.d }, za.d[x11, #0]\n" - ".inst 0xa0422bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" - "add z29.h, z29.h, z25.h\n" - ".inst 0xc006682e // mova { z14.d-z15.d }, za.d[x11, #1]\n" - ".inst 0xc1a3ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z3.s\n" + ".inst 0xc0066808 // mova { z8.d-z9.d }, za.d[x11, #0]\n" + ".inst 0xa0422be2 // ld1h { z2.h-z3.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" + "add z27.h, z27.h, z17.h\n" + ".inst 0xc006682a // mova { z10.d-z11.d }, za.d[x11, #1]\n" + ".inst 0xc1a7ac08 // sqdmulh { z8.s-z11.s }, { z8.s-z11.s }, z7.s\n" "add x11, x11, #0x2\n" - ".inst 0xc1a1aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n" - ".inst 0xc00408c0 // mova za.d[x8, #0], { z6.d-z7.d }\n" - ".inst 0xc1a9ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z9.s\n" - ".inst 0xc00408c1 // mova za.d[x8, #1], { z6.d-z7.d }\n" - ".inst 0xc1bfcf0c // sclamp { z12.s-z15.s }, z24.s, z31.s\n" - "st1b { z12.s }, p1, [x14]\n" - "add x14, x14, x3\n" - "add z30.h, z30.h, z25.h\n" - "st1b { z14.s }, p1, [x13]\n" + ".inst 0xc1a4aa28 // srshl { z8.s-z11.s }, { z8.s-z11.s }, z4.s\n" + ".inst 0xc0040a40 // mova za.d[x8, #0], { z18.d-z19.d }\n" + ".inst 0xc1acab08 // add { z8.s-z11.s }, { z8.s-z11.s }, z12.s\n" + ".inst 0xc0040a41 // mova za.d[x8, #1], { z18.d-z19.d }\n" + ".inst 0xc1b0cf08 // sclamp { z8.s-z11.s }, z24.s, z16.s\n" + "st1b { z8.s }, p1, [x14]\n" + "add x14, x14, x4\n" + "add z28.h, z28.h, z17.h\n" + "st1b { z10.s }, p1, [x13]\n" "add x13, x13, x10\n" - "st1b { z13.s }, p1, [x9]\n" + "st1b { z9.s }, p1, [x9]\n" "add x9, x9, x27\n" - "st1b { z15.s }, p1, [x28]\n" + "st1b { z11.s }, p1, [x28]\n" "add x28, x28, x26\n" "bgt 13b\n" "b 21f\n" "14:" // Padded "cbz x22, 19f\n" "cmp x22, #0x1\n" - "sub x17, x17, x22\n" + "sub x25, x25, x22\n" "beq 18f\n" "cmp x22, #0x2\n" "beq 17f\n" @@ -668,515 +668,515 @@ void sme2_u8q_planar_5x5_s1_4rows_dot_za_impl( "15:" // Padded: 4 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z19.s }, p0/Z, [x16]\n" - "add z19.h, p0/M, z19.h, z25.h\n" + "ld1b { z9.s }, p0/Z, [x16]\n" + "add z9.h, p0/M, z9.h, z17.h\n" "add x21, x16, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z18.s }, p0/Z, [x21]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1b { z22.s }, p0/Z, [x21]\n" + "add z22.h, p0/M, z22.h, z17.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z17.s }, p0/Z, [x21]\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "ld1b { z21.s }, p0/Z, [x21]\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z16.s }, p0/Z, [x21]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1b { z20.s }, p0/Z, [x21]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "mov x12, #0x4\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z27.h, z19.h, z18.h\n" - "trn1 z28.h, z17.h, z16.h\n" + "trn1 z31.h, z9.h, z22.h\n" + "trn1 z0.h, z21.h, z20.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z18.s }, p0/Z, [x21]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1b { z22.s }, p0/Z, [x21]\n" + "add z22.h, p0/M, z22.h, z17.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z16.s }, p0/Z, [x21]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1b { z20.s }, p0/Z, [x21]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z17.s }, p0/Z, [x21]\n" + "ld1b { z21.s }, p0/Z, [x21]\n" "addvl x20, SP, #24\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - "trn1 z29.h, z18.h, z16.h\n" - "ld1b { z16.s }, p0/Z, [x21]\n" - "add z16.h, p0/M, z16.h, z25.h\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" + ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" + "trn1 z1.h, z22.h, z20.h\n" + "ld1b { z20.s }, p0/Z, [x21]\n" + "add z20.h, p0/M, z20.h, z17.h\n" + ".inst 0xc16a77e8 // sdot za.s[x11, 0], { z31.h-z0.h }, z10.h\n" "add x16, x16, %x[ld_in_col]\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - "trn1 z30.h, z17.h, z16.h\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" + ".inst 0xc16277e9 // sdot za.s[x11, 1], { z31.h-z0.h }, z2.h\n" + ".inst 0xa1412a85 // ld1h { z5.h, z13.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + "trn1 z2.h, z21.h, z20.h\n" + ".inst 0xc16d7408 // sdot za.s[x11, 0], { z0.h-z1.h }, z13.h\n" + ".inst 0xa0422a88 // ld1h { z8.h-z9.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc1657409 // sdot za.s[x11, 1], { z0.h-z1.h }, z5.h\n" + ".inst 0xc1697428 // sdot za.s[x11, 0], { z1.h-z2.h }, z9.h\n" + ".inst 0xc1687429 // sdot za.s[x11, 1], { z1.h-z2.h }, z8.h\n" "16:" // Padded: 3 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z19.s }, p0/Z, [x16]\n" - "add z19.h, p0/M, z19.h, z25.h\n" + "ld1b { z5.s }, p0/Z, [x16]\n" + "add z5.h, p0/M, z5.h, z17.h\n" "add x20, x16, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1b { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "ld1b { z21.s }, p0/Z, [x20]\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1b { z20.s }, p0/Z, [x20]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z27.h, z19.h, z18.h\n" - "trn1 z28.h, z17.h, z16.h\n" + "trn1 z28.h, z5.h, z22.h\n" + "trn1 z29.h, z21.h, z20.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1b { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1b { z20.s }, p0/Z, [x20]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" + "ld1b { z21.s }, p0/Z, [x20]\n" "addvl x21, SP, #18\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" - "trn1 z29.h, z18.h, z16.h\n" - "ld1b { z16.s }, p0/Z, [x20]\n" + ".inst 0xa0402aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21]\n" + "trn1 z30.h, z22.h, z20.h\n" + "ld1b { z20.s }, p0/Z, [x20]\n" "addvl x20, SP, #24\n" - "add z16.h, p0/M, z16.h, z25.h\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - "trn1 z30.h, z17.h, z16.h\n" + "add z20.h, p0/M, z20.h, z17.h\n" + ".inst 0xc1617788 // sdot za.s[x11, 0], { z28.h-z29.h }, z1.h\n" + ".inst 0xc1607789 // sdot za.s[x11, 1], { z28.h-z29.h }, z0.h\n" + ".inst 0xa1402a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20]\n" + "trn1 z31.h, z21.h, z20.h\n" "add x16, x16, %x[ld_in_col]\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" + ".inst 0xa0412aae // ld1h { z14.h-z15.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc169778a // sdot za.s[x11, 2], { z28.h-z29.h }, z9.h\n" + ".inst 0xc161778b // sdot za.s[x11, 3], { z28.h-z29.h }, z1.h\n" + ".inst 0xa1422aa3 // ld1h { z3.h, z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc16f77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z15.h\n" + ".inst 0xc16e77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z14.h\n" + ".inst 0xa1412a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc16977aa // sdot za.s[x11, 2], { z29.h-z30.h }, z9.h\n" + ".inst 0xc16177ab // sdot za.s[x11, 3], { z29.h-z30.h }, z1.h\n" + ".inst 0xc16b77c8 // sdot za.s[x11, 0], { z30.h-z31.h }, z11.h\n" + ".inst 0xc16377c9 // sdot za.s[x11, 1], { z30.h-z31.h }, z3.h\n" + ".inst 0xa0422a8e // ld1h { z14.h-z15.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc16f77ca // sdot za.s[x11, 2], { z30.h-z31.h }, z15.h\n" + ".inst 0xc16e77cb // sdot za.s[x11, 3], { z30.h-z31.h }, z14.h\n" "17:" // Padded: 2 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z19.s }, p0/Z, [x16]\n" - "add z19.h, p0/M, z19.h, z25.h\n" + "ld1b { z29.s }, p0/Z, [x16]\n" + "add z29.h, p0/M, z29.h, z17.h\n" "add x20, x16, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1b { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "ld1b { z21.s }, p0/Z, [x20]\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1b { z20.s }, p0/Z, [x20]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z27.h, z19.h, z18.h\n" - "trn1 z28.h, z17.h, z16.h\n" + "trn1 z8.h, z29.h, z22.h\n" + "trn1 z9.h, z21.h, z20.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1b { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1b { z20.s }, p0/Z, [x20]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" + "ld1b { z21.s }, p0/Z, [x20]\n" "addvl x22, SP, #12\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xa1402ac0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x22]\n" - "trn1 z29.h, z18.h, z16.h\n" - "ld1b { z16.s }, p0/Z, [x20]\n" + ".inst 0xa0402ace // ld1h { z14.h-z15.h }, pn10.b/Z, [x22]\n" + "trn1 z10.h, z22.h, z20.h\n" + "ld1b { z20.s }, p0/Z, [x20]\n" "addvl x21, SP, #18\n" - "add z16.h, p0/M, z16.h, z25.h\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" + "add z20.h, p0/M, z20.h, z17.h\n" + ".inst 0xc16f7508 // sdot za.s[x11, 0], { z8.h-z9.h }, z15.h\n" + ".inst 0xc16e7509 // sdot za.s[x11, 1], { z8.h-z9.h }, z14.h\n" + ".inst 0xa1402aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21]\n" "addvl x20, SP, #24\n" - "trn1 z30.h, z17.h, z16.h\n" - ".inst 0xa0412ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" + "trn1 z11.h, z21.h, z20.h\n" + ".inst 0xa1412ac5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc16e750a // sdot za.s[x11, 2], { z8.h-z9.h }, z14.h\n" "add x16, x16, %x[ld_in_col]\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xa0422aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc168776c // sdot za.s[x11, 4], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776d // sdot za.s[x11, 5], { z27.h-z28.h }, z0.h\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc165778c // sdot za.s[x11, 4], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778d // sdot za.s[x11, 5], { z28.h-z29.h }, z4.h\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc16b77ac // sdot za.s[x11, 4], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ad // sdot za.s[x11, 5], { z29.h-z30.h }, z10.h\n" + ".inst 0xc166750b // sdot za.s[x11, 3], { z8.h-z9.h }, z6.h\n" + ".inst 0xa0402a8e // ld1h { z14.h-z15.h }, pn10.b/Z, [x20]\n" + ".inst 0xc16d7528 // sdot za.s[x11, 0], { z9.h-z10.h }, z13.h\n" + ".inst 0xa0422ac0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc1657529 // sdot za.s[x11, 1], { z9.h-z10.h }, z5.h\n" + ".inst 0xa1412aa5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc16f750c // sdot za.s[x11, 4], { z8.h-z9.h }, z15.h\n" + ".inst 0xc16e750d // sdot za.s[x11, 5], { z8.h-z9.h }, z14.h\n" + ".inst 0xc16d752a // sdot za.s[x11, 2], { z9.h-z10.h }, z13.h\n" + ".inst 0xc165752b // sdot za.s[x11, 3], { z9.h-z10.h }, z5.h\n" + ".inst 0xa1412a86 // ld1h { z6.h, z14.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc1617548 // sdot za.s[x11, 0], { z10.h-z11.h }, z1.h\n" + ".inst 0xc1607549 // sdot za.s[x11, 1], { z10.h-z11.h }, z0.h\n" + ".inst 0xa0422aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc16e752c // sdot za.s[x11, 4], { z9.h-z10.h }, z14.h\n" + ".inst 0xc166752d // sdot za.s[x11, 5], { z9.h-z10.h }, z6.h\n" + ".inst 0xc161754a // sdot za.s[x11, 2], { z10.h-z11.h }, z1.h\n" + ".inst 0xc160754b // sdot za.s[x11, 3], { z10.h-z11.h }, z0.h\n" + ".inst 0xa0422a8e // ld1h { z14.h-z15.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc16f754c // sdot za.s[x11, 4], { z10.h-z11.h }, z15.h\n" + ".inst 0xc16e754d // sdot za.s[x11, 5], { z10.h-z11.h }, z14.h\n" "18:" // Padded: 1 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z19.s }, p0/Z, [x16]\n" - "add z19.h, p0/M, z19.h, z25.h\n" + "ld1b { z1.s }, p0/Z, [x16]\n" + "add z1.h, p0/M, z1.h, z17.h\n" "add x20, x16, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1b { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "ld1b { z21.s }, p0/Z, [x20]\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1b { z20.s }, p0/Z, [x20]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z27.h, z19.h, z18.h\n" - "trn1 z28.h, z17.h, z16.h\n" + "trn1 z26.h, z1.h, z22.h\n" + "trn1 z27.h, z21.h, z20.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1b { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1b { z20.s }, p0/Z, [x20]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" + "ld1b { z21.s }, p0/Z, [x20]\n" "addvl x23, SP, #6\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xa1402ae0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x23]\n" - "trn1 z29.h, z18.h, z16.h\n" - "ld1b { z16.s }, p0/Z, [x20]\n" + ".inst 0xa0402aee // ld1h { z14.h-z15.h }, pn10.b/Z, [x23]\n" + "trn1 z28.h, z22.h, z20.h\n" + "ld1b { z20.s }, p0/Z, [x20]\n" "addvl x22, SP, #12\n" - "add z16.h, p0/M, z16.h, z25.h\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402ac0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x22]\n" + "add z20.h, p0/M, z20.h, z17.h\n" + ".inst 0xc16f7748 // sdot za.s[x11, 0], { z26.h-z27.h }, z15.h\n" + ".inst 0xc16e7749 // sdot za.s[x11, 1], { z26.h-z27.h }, z14.h\n" + ".inst 0xa0402ac0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x22]\n" "addvl x21, SP, #18\n" - "trn1 z30.h, z17.h, z16.h\n" - ".inst 0xa0412ae4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" + "trn1 z29.h, z21.h, z20.h\n" + ".inst 0xa0412aea // ld1h { z10.h-z11.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xc161774a // sdot za.s[x11, 2], { z26.h-z27.h }, z1.h\n" "addvl x20, SP, #24\n" "add x16, x16, %x[ld_in_col]\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xa0422aea // ld1h { z10.h-z11.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xc168776c // sdot za.s[x11, 4], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776d // sdot za.s[x11, 5], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc168776e // sdot za.s[x11, 6], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776f // sdot za.s[x11, 7], { z27.h-z28.h }, z0.h\n" - ".inst 0xc165778c // sdot za.s[x11, 4], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778d // sdot za.s[x11, 5], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc165778e // sdot za.s[x11, 6], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778f // sdot za.s[x11, 7], { z28.h-z29.h }, z4.h\n" - ".inst 0xc16b77ac // sdot za.s[x11, 4], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ad // sdot za.s[x11, 5], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc16b77ae // sdot za.s[x11, 6], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77af // sdot za.s[x11, 7], { z29.h-z30.h }, z10.h\n" + ".inst 0xc160774b // sdot za.s[x11, 3], { z26.h-z27.h }, z0.h\n" + ".inst 0xa1402aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21]\n" + ".inst 0xc16b7768 // sdot za.s[x11, 0], { z27.h-z28.h }, z11.h\n" + ".inst 0xa0422ae8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xc16a7769 // sdot za.s[x11, 1], { z27.h-z28.h }, z10.h\n" + ".inst 0xa0412aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc16e774c // sdot za.s[x11, 4], { z26.h-z27.h }, z14.h\n" + ".inst 0xc166774d // sdot za.s[x11, 5], { z26.h-z27.h }, z6.h\n" + ".inst 0xa1402a85 // ld1h { z5.h, z13.h }, pn10.b/Z, [x20]\n" + ".inst 0xc16b776a // sdot za.s[x11, 2], { z27.h-z28.h }, z11.h\n" + ".inst 0xc16a776b // sdot za.s[x11, 3], { z27.h-z28.h }, z10.h\n" + ".inst 0xa1412aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc1697788 // sdot za.s[x11, 0], { z28.h-z29.h }, z9.h\n" + ".inst 0xc1687789 // sdot za.s[x11, 1], { z28.h-z29.h }, z8.h\n" + ".inst 0xa1422ac2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc16d774e // sdot za.s[x11, 6], { z26.h-z27.h }, z13.h\n" + ".inst 0xc165774f // sdot za.s[x11, 7], { z26.h-z27.h }, z5.h\n" + ".inst 0xc16e776c // sdot za.s[x11, 4], { z27.h-z28.h }, z14.h\n" + ".inst 0xc166776d // sdot za.s[x11, 5], { z27.h-z28.h }, z6.h\n" + ".inst 0xa1412a86 // ld1h { z6.h, z14.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc16a778a // sdot za.s[x11, 2], { z28.h-z29.h }, z10.h\n" + ".inst 0xc162778b // sdot za.s[x11, 3], { z28.h-z29.h }, z2.h\n" + ".inst 0xa0422aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc16e776e // sdot za.s[x11, 6], { z27.h-z28.h }, z14.h\n" + ".inst 0xc166776f // sdot za.s[x11, 7], { z27.h-z28.h }, z6.h\n" + ".inst 0xc161778c // sdot za.s[x11, 4], { z28.h-z29.h }, z1.h\n" + ".inst 0xc160778d // sdot za.s[x11, 5], { z28.h-z29.h }, z0.h\n" + ".inst 0xa1422a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc16a778e // sdot za.s[x11, 6], { z28.h-z29.h }, z10.h\n" + ".inst 0xc162778f // sdot za.s[x11, 7], { z28.h-z29.h }, z2.h\n" "19:" // Padded: 0 priming loads - ".inst 0xa1402be0 // ld1h { z0.h, z8.h }, pn10.b/Z, [SP]\n" - ".inst 0xa0412be4 // ld1h { z4.h-z5.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" - ".inst 0xa0422bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" - "cbz x17, 22f\n" + ".inst 0xa0402be0 // ld1h { z0.h-z1.h }, pn10.b/Z, [SP]\n" + ".inst 0xa1412be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + ".inst 0xa0422be2 // ld1h { z2.h-z3.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" + "cbz x25, 22f\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z19.s }, p0/Z, [x16]\n" - "add z19.h, p0/M, z19.h, z25.h\n" + "ld1b { z6.s }, p0/Z, [x16]\n" + "add z6.h, p0/M, z6.h, z17.h\n" "add x20, x16, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1b { z30.s }, p0/Z, [x20]\n" + "add z30.h, p0/M, z30.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "ld1b { z27.s }, p0/Z, [x20]\n" + "add z27.h, p0/M, z27.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1b { z26.s }, p0/Z, [x20]\n" + "add z26.h, p0/M, z26.h, z17.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z27.h, z19.h, z18.h\n" - "trn1 z28.h, z17.h, z16.h\n" + "trn1 z25.h, z6.h, z30.h\n" + "trn1 z26.h, z27.h, z26.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z25.h\n" + "ld1b { z8.s }, p0/Z, [x20]\n" + "add z8.h, p0/M, z8.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1b { z9.s }, p0/Z, [x20]\n" + "add z9.h, p0/M, z9.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "ld1b { z21.s }, p0/Z, [x20]\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z25.h\n" - "sub x17, x17, #0x1\n" + "ld1b { z29.s }, p0/Z, [x20]\n" + "add z29.h, p0/M, z29.h, z17.h\n" + "sub x25, x25, #0x1\n" "sub x15, x15, #0x1\n" - "cmp x17, x15\n" - "trn1 z29.h, z19.h, z18.h\n" - "trn1 z30.h, z17.h, z16.h\n" - "csel x25, x17, x15, LT\n" + "cmp x25, x15\n" + "trn1 z27.h, z8.h, z9.h\n" + "trn1 z28.h, z21.h, z29.h\n" + "csel x25, x25, x15, LT\n" "add x16, x16, %x[ld_in_col]\n" "sub x15, x15, x25\n" "cbz x25, 21f\n" "20:" // Padded: Main loop "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z23.s }, p0/Z, [x16]\n" - "add z23.h, p0/M, z23.h, z25.h\n" + "ld1b { z8.s }, p0/Z, [x16]\n" + "add z8.h, p0/M, z8.h, z17.h\n" "add x24, x16, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z22.s }, p0/Z, [x24]\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" + "ld1b { z21.s }, p0/Z, [x24]\n" + ".inst 0xc1617728 // sdot za.s[x11, 0], { z25.h-z26.h }, z1.h\n" "addvl x23, SP, #6\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402ae0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x23]\n" + ".inst 0xc1607729 // sdot za.s[x11, 1], { z25.h-z26.h }, z0.h\n" + ".inst 0xa0402ae0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x23]\n" "addvl x22, SP, #12\n" - "add z22.h, p0/M, z22.h, z25.h\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x24, x24, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402ac0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x22]\n" + ".inst 0xc161772a // sdot za.s[x11, 2], { z25.h-z26.h }, z1.h\n" + ".inst 0xc160772b // sdot za.s[x11, 3], { z25.h-z26.h }, z0.h\n" + ".inst 0xa1402ac6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x22]\n" "addvl x21, SP, #18\n" "addvl x20, SP, #24\n" - "ld1b { z21.s }, p0/Z, [x24]\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - "add z21.h, p0/M, z21.h, z25.h\n" + "ld1b { z29.s }, p0/Z, [x24]\n" + ".inst 0xc16d7748 // sdot za.s[x11, 0], { z26.h-z27.h }, z13.h\n" + "add z29.h, p0/M, z29.h, z17.h\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412ae4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xc1657749 // sdot za.s[x11, 1], { z26.h-z27.h }, z5.h\n" + ".inst 0xa1412ae5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" "mov x12, #0x4\n" "add x24, x24, %x[ld_in_row]\n" - ".inst 0xc168776c // sdot za.s[x11, 4], { z27.h-z28.h }, z8.h\n" - "ld1b { z20.s }, p0/Z, [x24]\n" - "add z20.h, p0/M, z20.h, z25.h\n" + ".inst 0xc16e772c // sdot za.s[x11, 4], { z25.h-z26.h }, z14.h\n" + "ld1b { z30.s }, p0/Z, [x24]\n" + "add z30.h, p0/M, z30.h, z17.h\n" "add x24, x24, %x[ld_in_row]\n" - ".inst 0xc160776d // sdot za.s[x11, 5], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" + ".inst 0xc166772d // sdot za.s[x11, 5], { z25.h-z26.h }, z6.h\n" + ".inst 0xa1402aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "subs x25, x25, #0x1\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - "ld1b { z19.s }, p0/Z, [x24]\n" - "add z19.h, p0/M, z19.h, z25.h\n" + ".inst 0xc16d774a // sdot za.s[x11, 2], { z26.h-z27.h }, z13.h\n" + "ld1b { z15.s }, p0/Z, [x24]\n" + "add z15.h, p0/M, z15.h, z17.h\n" "add x24, x24, %x[ld_in_row]\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc165774b // sdot za.s[x11, 3], { z26.h-z27.h }, z5.h\n" + ".inst 0xa0412aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "add x16, x16, %x[ld_in_col]\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - "ld1b { z18.s }, p0/Z, [x24]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + ".inst 0xc1637768 // sdot za.s[x11, 0], { z27.h-z28.h }, z3.h\n" + "ld1b { z20.s }, p0/Z, [x24]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "add x24, x24, %x[ld_in_row]\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aea // ld1h { z10.h-z11.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xc1627769 // sdot za.s[x11, 1], { z27.h-z28.h }, z2.h\n" + ".inst 0xa1422ae1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc168776e // sdot za.s[x11, 6], { z27.h-z28.h }, z8.h\n" - "ld1b { z17.s }, p0/Z, [x24]\n" - "add z17.h, p0/M, z17.h, z25.h\n" + ".inst 0xc16e772e // sdot za.s[x11, 6], { z25.h-z26.h }, z14.h\n" + "ld1b { z31.s }, p0/Z, [x24]\n" + "add z31.h, p0/M, z31.h, z17.h\n" "add x24, x24, %x[ld_in_row]\n" - ".inst 0xc160776f // sdot za.s[x11, 7], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" + ".inst 0xc166772f // sdot za.s[x11, 7], { z25.h-z26.h }, z6.h\n" + ".inst 0xa0402a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xc165778c // sdot za.s[x11, 4], { z28.h-z29.h }, z5.h\n" - "ld1b { z16.s }, p0/Z, [x24]\n" - "add z16.h, p0/M, z16.h, z25.h\n" - ".inst 0xc164778d // sdot za.s[x11, 5], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc165778e // sdot za.s[x11, 6], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778f // sdot za.s[x11, 7], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16b77ac // sdot za.s[x11, 4], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ad // sdot za.s[x11, 5], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc16b77ae // sdot za.s[x11, 6], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77af // sdot za.s[x11, 7], { z29.h-z30.h }, z10.h\n" + ".inst 0xc16b774c // sdot za.s[x11, 4], { z26.h-z27.h }, z11.h\n" + "ld1b { z22.s }, p0/Z, [x24]\n" + "add z22.h, p0/M, z22.h, z17.h\n" + ".inst 0xc16a774d // sdot za.s[x11, 5], { z26.h-z27.h }, z10.h\n" + ".inst 0xa1412aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc169776a // sdot za.s[x11, 2], { z27.h-z28.h }, z9.h\n" + ".inst 0xc161776b // sdot za.s[x11, 3], { z27.h-z28.h }, z1.h\n" + ".inst 0xa0422ac0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc16e774e // sdot za.s[x11, 6], { z26.h-z27.h }, z14.h\n" + ".inst 0xc166774f // sdot za.s[x11, 7], { z26.h-z27.h }, z6.h\n" + ".inst 0xa1412a86 // ld1h { z6.h, z14.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc161776c // sdot za.s[x11, 4], { z27.h-z28.h }, z1.h\n" + ".inst 0xc160776d // sdot za.s[x11, 5], { z27.h-z28.h }, z0.h\n" + ".inst 0xa1422aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc169776e // sdot za.s[x11, 6], { z27.h-z28.h }, z9.h\n" + ".inst 0xc161776f // sdot za.s[x11, 7], { z27.h-z28.h }, z1.h\n" ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc1681768 // sdot za.s[x8, 0], { z27.h-z28.h }, z8.h\n" - ".inst 0xc1601769 // sdot za.s[x8, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402be0 // ld1h { z0.h, z8.h }, pn10.b/Z, [SP]\n" - "trn1 z27.h, z23.h, z22.h\n" - ".inst 0xc1651788 // sdot za.s[x8, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xc1641789 // sdot za.s[x8, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412be4 // ld1h { z4.h-z5.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" - "trn1 z28.h, z21.h, z20.h\n" - ".inst 0xc16b17a8 // sdot za.s[x8, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a17a9 // sdot za.s[x8, 1], { z29.h-z30.h }, z10.h\n" + ".inst 0xc1631728 // sdot za.s[x8, 0], { z25.h-z26.h }, z3.h\n" + ".inst 0xc1621729 // sdot za.s[x8, 1], { z25.h-z26.h }, z2.h\n" + ".inst 0xa0402be0 // ld1h { z0.h-z1.h }, pn10.b/Z, [SP]\n" + "trn1 z25.h, z8.h, z21.h\n" + ".inst 0xc16e1748 // sdot za.s[x8, 0], { z26.h-z27.h }, z14.h\n" + ".inst 0xc1661749 // sdot za.s[x8, 1], { z26.h-z27.h }, z6.h\n" + ".inst 0xa1412be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + "trn1 z26.h, z29.h, z30.h\n" + ".inst 0xc16b1768 // sdot za.s[x8, 0], { z27.h-z28.h }, z11.h\n" + ".inst 0xc16a1769 // sdot za.s[x8, 1], { z27.h-z28.h }, z10.h\n" "add x8, x8, #0x2\n" - ".inst 0xa0422bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" - "trn1 z29.h, z19.h, z18.h\n" - ".inst 0xc006680c // mova { z12.d-z13.d }, za.d[x11, #0]\n" - "trn1 z30.h, z17.h, z16.h\n" - ".inst 0xc006682e // mova { z14.d-z15.d }, za.d[x11, #1]\n" - ".inst 0xc1a3ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z3.s\n" + ".inst 0xa0422be2 // ld1h { z2.h-z3.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" + "trn1 z27.h, z15.h, z20.h\n" + ".inst 0xc0066808 // mova { z8.d-z9.d }, za.d[x11, #0]\n" + "trn1 z28.h, z31.h, z22.h\n" + ".inst 0xc006682a // mova { z10.d-z11.d }, za.d[x11, #1]\n" + ".inst 0xc1a7ac08 // sqdmulh { z8.s-z11.s }, { z8.s-z11.s }, z7.s\n" "add x11, x11, #0x2\n" - ".inst 0xc1a1aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n" - ".inst 0xc00408c0 // mova za.d[x8, #0], { z6.d-z7.d }\n" - ".inst 0xc1a9ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z9.s\n" - ".inst 0xc00408c1 // mova za.d[x8, #1], { z6.d-z7.d }\n" - ".inst 0xc1bfcf0c // sclamp { z12.s-z15.s }, z24.s, z31.s\n" - "st1b { z12.s }, p1, [x14]\n" - "add x14, x14, x3\n" - "st1b { z14.s }, p1, [x13]\n" + ".inst 0xc1a4aa28 // srshl { z8.s-z11.s }, { z8.s-z11.s }, z4.s\n" + ".inst 0xc0040a40 // mova za.d[x8, #0], { z18.d-z19.d }\n" + ".inst 0xc1acab08 // add { z8.s-z11.s }, { z8.s-z11.s }, z12.s\n" + ".inst 0xc0040a41 // mova za.d[x8, #1], { z18.d-z19.d }\n" + ".inst 0xc1b0cf08 // sclamp { z8.s-z11.s }, z24.s, z16.s\n" + "st1b { z8.s }, p1, [x14]\n" + "add x14, x14, x4\n" + "st1b { z10.s }, p1, [x13]\n" "add x13, x13, x10\n" - "st1b { z13.s }, p1, [x9]\n" + "st1b { z9.s }, p1, [x9]\n" "add x9, x9, x27\n" - "st1b { z15.s }, p1, [x28]\n" + "st1b { z11.s }, p1, [x28]\n" "add x28, x28, x26\n" "bgt 20b\n" "21:" // Main loop tail "addvl x23, SP, #6\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" + ".inst 0xc1617728 // sdot za.s[x11, 0], { z25.h-z26.h }, z1.h\n" "addvl x22, SP, #12\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402ae0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x23]\n" + ".inst 0xc1607729 // sdot za.s[x11, 1], { z25.h-z26.h }, z0.h\n" + ".inst 0xa0402ae0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x23]\n" "addvl x21, SP, #18\n" "addvl x20, SP, #24\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402ac0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x22]\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412ae4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0xc168776c // sdot za.s[x11, 4], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776d // sdot za.s[x11, 5], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" + ".inst 0xc161772a // sdot za.s[x11, 2], { z25.h-z26.h }, z1.h\n" + ".inst 0xc160772b // sdot za.s[x11, 3], { z25.h-z26.h }, z0.h\n" + ".inst 0xa1402ac6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x22]\n" + ".inst 0xc16d7748 // sdot za.s[x11, 0], { z26.h-z27.h }, z13.h\n" + ".inst 0xc1657749 // sdot za.s[x11, 1], { z26.h-z27.h }, z5.h\n" + ".inst 0xa1412ae1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xc16e772c // sdot za.s[x11, 4], { z25.h-z26.h }, z14.h\n" + ".inst 0xc166772d // sdot za.s[x11, 5], { z25.h-z26.h }, z6.h\n" + ".inst 0xa1402aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21]\n" + ".inst 0xc169774a // sdot za.s[x11, 2], { z26.h-z27.h }, z9.h\n" + ".inst 0xc161774b // sdot za.s[x11, 3], { z26.h-z27.h }, z1.h\n" + ".inst 0xa1412ac1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc1637768 // sdot za.s[x11, 0], { z27.h-z28.h }, z3.h\n" + ".inst 0xc1627769 // sdot za.s[x11, 1], { z27.h-z28.h }, z2.h\n" ".inst 0xa0422aea // ld1h { z10.h-z11.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xc168776e // sdot za.s[x11, 6], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776f // sdot za.s[x11, 7], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - ".inst 0xc165778c // sdot za.s[x11, 4], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778d // sdot za.s[x11, 5], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc165778e // sdot za.s[x11, 6], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778f // sdot za.s[x11, 7], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16b77ac // sdot za.s[x11, 4], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ad // sdot za.s[x11, 5], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc16b77ae // sdot za.s[x11, 6], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77af // sdot za.s[x11, 7], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc1681768 // sdot za.s[x8, 0], { z27.h-z28.h }, z8.h\n" + ".inst 0xc16e772e // sdot za.s[x11, 6], { z25.h-z26.h }, z14.h\n" + ".inst 0xc166772f // sdot za.s[x11, 7], { z25.h-z26.h }, z6.h\n" + ".inst 0xa0402a8e // ld1h { z14.h-z15.h }, pn10.b/Z, [x20]\n" + ".inst 0xc169774c // sdot za.s[x11, 4], { z26.h-z27.h }, z9.h\n" + ".inst 0xc161774d // sdot za.s[x11, 5], { z26.h-z27.h }, z1.h\n" + ".inst 0xa1412aa5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc16b776a // sdot za.s[x11, 2], { z27.h-z28.h }, z11.h\n" + ".inst 0xc16a776b // sdot za.s[x11, 3], { z27.h-z28.h }, z10.h\n" + ".inst 0xa0422ac2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc16d774e // sdot za.s[x11, 6], { z26.h-z27.h }, z13.h\n" + ".inst 0xc165774f // sdot za.s[x11, 7], { z26.h-z27.h }, z5.h\n" + ".inst 0xa0412a88 // ld1h { z8.h-z9.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc163776c // sdot za.s[x11, 4], { z27.h-z28.h }, z3.h\n" + ".inst 0xc162776d // sdot za.s[x11, 5], { z27.h-z28.h }, z2.h\n" + ".inst 0xa1422aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc16a776e // sdot za.s[x11, 6], { z27.h-z28.h }, z10.h\n" + ".inst 0xc162776f // sdot za.s[x11, 7], { z27.h-z28.h }, z2.h\n" + ".inst 0xa0422a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc16f1728 // sdot za.s[x8, 0], { z25.h-z26.h }, z15.h\n" + ".inst 0xc16e1729 // sdot za.s[x8, 1], { z25.h-z26.h }, z14.h\n" + ".inst 0xc1691748 // sdot za.s[x8, 0], { z26.h-z27.h }, z9.h\n" + ".inst 0xc1681749 // sdot za.s[x8, 1], { z26.h-z27.h }, z8.h\n" + ".inst 0xc1611768 // sdot za.s[x8, 0], { z27.h-z28.h }, z1.h\n" ".inst 0xc1601769 // sdot za.s[x8, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xc1651788 // sdot za.s[x8, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xc1641789 // sdot za.s[x8, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xc16b17a8 // sdot za.s[x8, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a17a9 // sdot za.s[x8, 1], { z29.h-z30.h }, z10.h\n" "add x8, x8, #0x2\n" - ".inst 0xc006680c // mova { z12.d-z13.d }, za.d[x11, #0]\n" - ".inst 0xc006682e // mova { z14.d-z15.d }, za.d[x11, #1]\n" - ".inst 0xc1a3ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z3.s\n" + ".inst 0xc0066808 // mova { z8.d-z9.d }, za.d[x11, #0]\n" + ".inst 0xc006682a // mova { z10.d-z11.d }, za.d[x11, #1]\n" + ".inst 0xc1a7ac08 // sqdmulh { z8.s-z11.s }, { z8.s-z11.s }, z7.s\n" "add x11, x11, #0x2\n" - ".inst 0xc1a1aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n" - ".inst 0xc00408c0 // mova za.d[x8, #0], { z6.d-z7.d }\n" - ".inst 0xc1a9ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z9.s\n" - ".inst 0xc00408c1 // mova za.d[x8, #1], { z6.d-z7.d }\n" - ".inst 0xc1bfcf0c // sclamp { z12.s-z15.s }, z24.s, z31.s\n" - "st1b { z12.s }, p1, [x14]\n" - "add x14, x14, x3\n" - "st1b { z14.s }, p1, [x13]\n" + ".inst 0xc1a4aa28 // srshl { z8.s-z11.s }, { z8.s-z11.s }, z4.s\n" + ".inst 0xc0040a40 // mova za.d[x8, #0], { z18.d-z19.d }\n" + ".inst 0xc1acab08 // add { z8.s-z11.s }, { z8.s-z11.s }, z12.s\n" + ".inst 0xc0040a41 // mova za.d[x8, #1], { z18.d-z19.d }\n" + ".inst 0xc1b0cf08 // sclamp { z8.s-z11.s }, z24.s, z16.s\n" + "st1b { z8.s }, p1, [x14]\n" + "add x14, x14, x4\n" + "st1b { z10.s }, p1, [x13]\n" "add x13, x13, x10\n" - "st1b { z13.s }, p1, [x9]\n" + "st1b { z9.s }, p1, [x9]\n" "add x9, x9, x27\n" - "st1b { z15.s }, p1, [x28]\n" + "st1b { z11.s }, p1, [x28]\n" "add x28, x28, x26\n" "22:" // Main loop skip tail "cbz x15, 24f\n" "23:" // Right padding loop - ".inst 0xc006680c // mova { z12.d-z13.d }, za.d[x11, #0]\n" + ".inst 0xc0066808 // mova { z8.d-z9.d }, za.d[x11, #0]\n" "add x8, x8, #0x2\n" "subs x15, x15, #0x1\n" - ".inst 0xc006682e // mova { z14.d-z15.d }, za.d[x11, #1]\n" - ".inst 0xc1a3ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z3.s\n" + ".inst 0xc006682a // mova { z10.d-z11.d }, za.d[x11, #1]\n" + ".inst 0xc1a7ac08 // sqdmulh { z8.s-z11.s }, { z8.s-z11.s }, z7.s\n" "add x11, x11, #0x2\n" - ".inst 0xc1a1aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n" - ".inst 0xc00408c0 // mova za.d[x8, #0], { z6.d-z7.d }\n" - ".inst 0xc1a9ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z9.s\n" - ".inst 0xc00408c1 // mova za.d[x8, #1], { z6.d-z7.d }\n" - ".inst 0xc1bfcf0c // sclamp { z12.s-z15.s }, z24.s, z31.s\n" - "st1b { z12.s }, p1, [x14]\n" - "add x14, x14, x3\n" - "st1b { z14.s }, p1, [x13]\n" + ".inst 0xc1a4aa28 // srshl { z8.s-z11.s }, { z8.s-z11.s }, z4.s\n" + ".inst 0xc0040a40 // mova za.d[x8, #0], { z18.d-z19.d }\n" + ".inst 0xc1acab08 // add { z8.s-z11.s }, { z8.s-z11.s }, z12.s\n" + ".inst 0xc0040a41 // mova za.d[x8, #1], { z18.d-z19.d }\n" + ".inst 0xc1b0cf08 // sclamp { z8.s-z11.s }, z24.s, z16.s\n" + "st1b { z8.s }, p1, [x14]\n" + "add x14, x14, x4\n" + "st1b { z10.s }, p1, [x13]\n" "add x13, x13, x10\n" - "st1b { z13.s }, p1, [x9]\n" + "st1b { z9.s }, p1, [x9]\n" "add x9, x9, x27\n" - "st1b { z15.s }, p1, [x28]\n" + "st1b { z11.s }, p1, [x28]\n" "add x28, x28, x26\n" "bgt 23b\n" "24:" // End - "ldr x23, [%x[args], %[offsetof_Args_weights]]\n" - "incw x23, ALL, MUL #16\n" - "incw x23, ALL, MUL #9\n" - "str x23, [%x[args], %[offsetof_Args_weights]]\n" - "ldr x20, [%x[args], %[offsetof_Args_ld_in_vl]]\n" - "incw x5\n" - "whilelt p1.s, x5, x7\n" - "ldr x16, [%x[args], %[offsetof_Args_inptr]]\n" - "add x16, x16, x20\n" - "str x16, [%x[args], %[offsetof_Args_inptr]]\n" + "ldr x20, [%x[args], %[offsetof_Args_weights]]\n" + "incw x20, ALL, MUL #16\n" + "incw x20, ALL, MUL #9\n" + "str x20, [%x[args], %[offsetof_Args_weights]]\n" + "ldr x21, [%x[args], %[offsetof_Args_ld_in_vl]]\n" + "incw x17\n" + "whilelt p1.s, x17, x7\n" + "ldr x20, [%x[args], %[offsetof_Args_inptr]]\n" + "add x20, x20, x21\n" + "str x20, [%x[args], %[offsetof_Args_inptr]]\n" "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" "ldr x24, [%x[args], %[offsetof_Args_ld_out_vls]]\n" "ldp x23, x22, [x25, #0x0]\n" @@ -1194,7 +1194,7 @@ void sme2_u8q_planar_5x5_s1_4rows_dot_za_impl( ".inst 0xd503467f // SMSTOP\n" : : [args] "r" (&args), [ld_in_col] "r" (ld_in_col), [ld_in_row] "r" (ld_in_row), [offsetof_Args_current_channel] "I" (offsetof(Args, current_channel)), [offsetof_Args_inptr] "I" (offsetof(Args, inptr)), [offsetof_Args_input_cols] "I" (offsetof(Args, input_cols)), [offsetof_Args_ld_in_vl] "I" (offsetof(Args, ld_in_vl)), [offsetof_Args_ld_out_cols] "I" (offsetof(Args, ld_out_cols)), [offsetof_Args_ld_out_vls] "I" (offsetof(Args, ld_out_vls)), [offsetof_Args_n_channels] "I" (offsetof(Args, n_channels)), [offsetof_Args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_Args_output_cols] "I" (offsetof(Args, output_cols)), [offsetof_Args_pad_bottom] "I" (offsetof(Args, pad_bottom)), [offsetof_Args_pad_left] "I" (offsetof(Args, pad_left)), [offsetof_Args_pad_top] "I" (offsetof(Args, pad_top)), [offsetof_Args_weights] "I" (offsetof(Args, weights)), [offsetof_Requantize32_a_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_bias] "I" (offsetof(arm_gemm::Requantize32, bias)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [offsetof_Requantize32_per_channel_muls] "I" (offsetof(arm_gemm::Requantize32, per_channel_muls)), [offsetof_Requantize32_per_channel_right_shifts] "I" (offsetof(arm_gemm::Requantize32, per_channel_right_shifts)), [offsetof_Requantize32_per_layer_mul] "I" (offsetof(arm_gemm::Requantize32, per_layer_mul)), [offsetof_Requantize32_per_layer_right_shift] "I" (offsetof(arm_gemm::Requantize32, per_layer_right_shift)), [qp] "r" (&qp) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_5x5_s2_4rows_dot_za.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_5x5_s2_4rows_dot_za.hpp index 9fa295b20edde0ba327039c6ac1a683b10f680b7..05aad19c09c697146b4eacce41fe799088265985 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_5x5_s2_4rows_dot_za.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_5x5_s2_4rows_dot_za.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,8 @@ * SOFTWARE. */ +#include "src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp" + namespace arm_conv { namespace depthwise { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_5x5_s2_4rows_dot_za/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_5x5_s2_4rows_dot_za/generic.cpp index 468e6778a4540d625f11cd0b395102e1171871b5..6c144afa77a5d0da8aadc6924426b62a2bb41941 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_5x5_s2_4rows_dot_za/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8q_planar_5x5_s2_4rows_dot_za/generic.cpp @@ -73,156 +73,156 @@ void sme2_u8q_planar_5x5_s2_4rows_dot_za_impl( "ptrue p2.b\n" "mov x20, #0xb\n" "ldr x4, [%x[args], %[offsetof_Args_pad_top]]\n" - "ld1rh { z9.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_a_offset]]\n" + "ld1rh { z7.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_a_offset]]\n" "sub x20, x20, x3\n" ".inst 0x25207812 // ptrue pn10.b\n" "ldr x5, [%x[args], %[offsetof_Args_n_channels]]\n" "whilelt p1.s, XZR, x5\n" "whilelt p9.s, XZR, x20\n" - "ld1rw { z8.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" + "ld1rw { z10.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" "whilelt p8.s, XZR, x4\n" "addvl SP, SP, #-15\n" "ldr x6, [%x[args], %[offsetof_Args_current_channel]]\n" - "neg z9.h, p2/M, z9.h\n" + "neg z7.h, p2/M, z7.h\n" "eor p8.b, p2/Z, p8.b, p9.b\n" - "ld1rw { z3.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_mul]]\n" - "ld1rw { z1.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_right_shift]]\n" - "ld1rw { z26.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" - "ld1rw { z23.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" + "ld1rw { z6.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z4.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z5.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" + "ld1rw { z21.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" "1:" // Channel loop "ldr x20, [%x[qp], %[offsetof_Requantize32_bias]]\n" - "mov z28.s, #0x0\n" + "mov z12.s, #0x0\n" "cbz x20, 2f\n" - "ld1w { z28.s }, p1/Z, [x20, x6, LSL #2]\n" + "ld1w { z12.s }, p1/Z, [x20, x6, LSL #2]\n" "2:" // Load bias: Done "ldr x22, [%x[args], %[offsetof_Args_weights]]\n" "mov x20, x22\n" - "ld1b { z12.s }, p2/Z, [x20]\n" + "ld1b { z13.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "ld1rh { z18.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" - "sub z12.h, z12.h, z18.h\n" + "ld1rh { z28.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" + "sub z13.h, z13.h, z28.h\n" "incw x22\n" - "mov z14.h, #0x0\n" - "ld1b { z25.s }, p2/Z, [x20]\n" + "mov z26.h, #0x0\n" + "ld1b { z22.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z25.h, z25.h, z18.h\n" - "trn1 z2.h, z12.h, z25.h\n" - "ld1b { z24.s }, p2/Z, [x20]\n" + "sub z22.h, z22.h, z28.h\n" + "trn1 z17.h, z13.h, z22.h\n" + "ld1b { z20.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z24.h, z24.h, z18.h\n" + "sub z20.h, z20.h, z28.h\n" "addvl x21, SP, #15\n" - "ld1b { z17.s }, p2/Z, [x20]\n" + "ld1b { z1.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z17.h, z17.h, z18.h\n" - "trn1 z10.h, z24.h, z17.h\n" - "ld1b { z16.s }, p2/Z, [x20]\n" + "sub z1.h, z1.h, z28.h\n" + "trn1 z29.h, z20.h, z1.h\n" + "ld1b { z27.s }, p2/Z, [x20]\n" "mov x20, x22\n" - "sub z16.h, z16.h, z18.h\n" + "sub z27.h, z27.h, z28.h\n" "incw x22\n" - "ld1b { z12.s }, p2/Z, [x20]\n" + "ld1b { z14.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z12.h, z12.h, z18.h\n" + "sub z14.h, z14.h, z28.h\n" "addvl x21, x21, #-3\n" - "ld1b { z25.s }, p2/Z, [x20]\n" + "ld1b { z18.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z25.h, z25.h, z18.h\n" - "trn1 z0.h, z16.h, z14.h\n" - "ld1b { z24.s }, p2/Z, [x20]\n" + "sub z18.h, z18.h, z28.h\n" + "trn1 z22.h, z27.h, z26.h\n" + "ld1b { z23.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z24.h, z24.h, z18.h\n" - "st1h { z2.h }, p2, [x21]\n" - "ld1b { z17.s }, p2/Z, [x20]\n" + "sub z23.h, z23.h, z28.h\n" + "st1h { z17.h }, p2, [x21]\n" + "ld1b { z30.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z17.h, z17.h, z18.h\n" - "trn1 z2.h, z12.h, z25.h\n" - "ld1b { z16.s }, p2/Z, [x20]\n" + "sub z30.h, z30.h, z28.h\n" + "trn1 z8.h, z14.h, z18.h\n" + "ld1b { z15.s }, p2/Z, [x20]\n" "mov x20, x22\n" - "st1h { z10.h }, p2, [x21, #1, MUL VL]\n" - "sub z16.h, z16.h, z18.h\n" - "ld1b { z12.s }, p2/Z, [x20]\n" - "incw x20, ALL, MUL #5\n" - "trn1 z10.h, z24.h, z17.h\n" - "sub z12.h, z12.h, z18.h\n" - "ld1b { z25.s }, p2/Z, [x20]\n" + "st1h { z29.h }, p2, [x21, #1, MUL VL]\n" + "sub z15.h, z15.h, z28.h\n" + "ld1b { z20.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z25.h, z25.h, z18.h\n" - "st1h { z0.h }, p2, [x21, #2, MUL VL]\n" + "trn1 z23.h, z23.h, z30.h\n" + "sub z20.h, z20.h, z28.h\n" "ld1b { z24.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "trn1 z0.h, z16.h, z14.h\n" + "sub z24.h, z24.h, z28.h\n" + "st1h { z22.h }, p2, [x21, #2, MUL VL]\n" + "ld1b { z16.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "trn1 z0.h, z15.h, z26.h\n" "incw x22\n" - "ld1b { z17.s }, p2/Z, [x20]\n" + "ld1b { z13.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z24.h, z24.h, z18.h\n" - "sub z17.h, z17.h, z18.h\n" - "ld1b { z16.s }, p2/Z, [x20]\n" + "sub z16.h, z16.h, z28.h\n" + "sub z13.h, z13.h, z28.h\n" + "ld1b { z11.s }, p2/Z, [x20]\n" "addvl x21, x21, #-3\n" "mov x20, x22\n" - "st1h { z2.h }, p2, [x21]\n" - "trn1 z2.h, z12.h, z25.h\n" - "ld1b { z12.s }, p2/Z, [x20]\n" + "st1h { z8.h }, p2, [x21]\n" + "trn1 z27.h, z20.h, z24.h\n" + "ld1b { z22.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z16.h, z16.h, z18.h\n" - "ld1b { z25.s }, p2/Z, [x20]\n" + "sub z11.h, z11.h, z28.h\n" + "ld1b { z3.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "st1h { z10.h }, p2, [x21, #1, MUL VL]\n" - "trn1 z10.h, z24.h, z17.h\n" - "ld1b { z24.s }, p2/Z, [x20]\n" + "st1h { z23.h }, p2, [x21, #1, MUL VL]\n" + "trn1 z20.h, z16.h, z13.h\n" + "ld1b { z13.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z12.h, z12.h, z18.h\n" - "sub z25.h, z25.h, z18.h\n" - "ld1b { z17.s }, p2/Z, [x20]\n" + "sub z22.h, z22.h, z28.h\n" + "sub z3.h, z3.h, z28.h\n" + "ld1b { z15.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" "st1h { z0.h }, p2, [x21, #2, MUL VL]\n" - "trn1 z0.h, z16.h, z14.h\n" + "trn1 z29.h, z11.h, z26.h\n" "ld1b { z16.s }, p2/Z, [x20]\n" "incw x22\n" - "sub z24.h, z24.h, z18.h\n" - "sub z17.h, z17.h, z18.h\n" + "sub z13.h, z13.h, z28.h\n" + "sub z15.h, z15.h, z28.h\n" "addvl x21, x21, #-3\n" "mov x20, x22\n" - "st1h { z2.h }, p2, [x21]\n" - "sub z16.h, z16.h, z18.h\n" - "trn1 z2.h, z12.h, z25.h\n" - "ld1b { z12.s }, p2/Z, [x20]\n" + "st1h { z27.h }, p2, [x21]\n" + "sub z16.h, z16.h, z28.h\n" + "trn1 z19.h, z22.h, z3.h\n" + "ld1b { z17.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "st1h { z10.h }, p2, [x21, #1, MUL VL]\n" - "ld1b { z25.s }, p2/Z, [x20]\n" + "st1h { z20.h }, p2, [x21, #1, MUL VL]\n" + "ld1b { z0.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "trn1 z10.h, z24.h, z17.h\n" - "st1h { z0.h }, p2, [x21, #2, MUL VL]\n" - "ld1b { z24.s }, p2/Z, [x20]\n" + "trn1 z31.h, z13.h, z15.h\n" + "st1h { z29.h }, p2, [x21, #2, MUL VL]\n" + "ld1b { z18.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "trn1 z0.h, z16.h, z14.h\n" - "sub z12.h, z12.h, z18.h\n" - "ld1b { z17.s }, p2/Z, [x20]\n" + "trn1 z16.h, z16.h, z26.h\n" + "sub z17.h, z17.h, z28.h\n" + "ld1b { z22.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z25.h, z25.h, z18.h\n" - "sub z24.h, z24.h, z18.h\n" - "ld1b { z16.s }, p2/Z, [x20]\n" - "sub z17.h, z17.h, z18.h\n" - "sub z16.h, z16.h, z18.h\n" + "sub z0.h, z0.h, z28.h\n" + "sub z18.h, z18.h, z28.h\n" + "ld1b { z1.s }, p2/Z, [x20]\n" + "sub z22.h, z22.h, z28.h\n" + "sub z1.h, z1.h, z28.h\n" "ldr x20, [%x[qp], %[offsetof_Requantize32_per_channel_muls]]\n" "addvl x21, x21, #-3\n" - "st1h { z2.h }, p2, [x21]\n" - "mov z29.d, z28.d\n" - "mov z30.d, z28.d\n" - "st1h { z10.h }, p2, [x21, #1, MUL VL]\n" - "mov z31.d, z28.d\n" - "trn1 z2.h, z12.h, z25.h\n" - "st1h { z0.h }, p2, [x21, #2, MUL VL]\n" + "st1h { z19.h }, p2, [x21]\n" + "mov z13.d, z12.d\n" + "mov z14.d, z12.d\n" + "st1h { z31.h }, p2, [x21, #1, MUL VL]\n" + "mov z15.d, z12.d\n" + "trn1 z8.h, z17.h, z0.h\n" + "st1h { z16.h }, p2, [x21, #2, MUL VL]\n" "addvl x21, x21, #-3\n" - "trn1 z10.h, z24.h, z17.h\n" - "trn1 z0.h, z16.h, z14.h\n" - "st1h { z2.h }, p2, [x21]\n" - "st1h { z10.h }, p2, [x21, #1, MUL VL]\n" - "st1h { z0.h }, p2, [x21, #2, MUL VL]\n" + "trn1 z31.h, z18.h, z22.h\n" + "trn1 z29.h, z1.h, z26.h\n" + "st1h { z8.h }, p2, [x21]\n" + "st1h { z31.h }, p2, [x21, #1, MUL VL]\n" + "st1h { z29.h }, p2, [x21, #2, MUL VL]\n" "cbz x20, 3f\n" - "ld1w { z3.s }, p1/Z, [x20, x6, LSL #2]\n" + "ld1w { z6.s }, p1/Z, [x20, x6, LSL #2]\n" "3:" // Load mul: End "ldr x20, [%x[qp], %[offsetof_Requantize32_per_channel_right_shifts]]\n" "cbz x20, 4f\n" - "ld1w { z1.s }, p1/Z, [x20, x6, LSL #2]\n" + "ld1w { z4.s }, p1/Z, [x20, x6, LSL #2]\n" "4:" // Load right_shift: End "ldr x7, [%x[args], %[offsetof_Args_input_cols]]\n" "sub x20, x7, #0x1\n" @@ -242,20 +242,20 @@ void sme2_u8q_planar_5x5_s2_4rows_dot_za_impl( ".inst 0xf8b74a9c // rprfm pldstrm, x23, [x20]\n" "add x20, x20, %x[ld_in_col]\n" "bgt 5b\n" - "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" + "ldr x23, [%x[args], %[offsetof_Args_outptrs]]\n" "lsl x20, %x[ld_in_row], #0x0\n" "msub x17, x4, x20, x17\n" - ".inst 0xc0040f80 // mova za.d[x8, #0], { z28.d-z31.d }\n" + ".inst 0xc0040d80 // mova za.d[x8, #0], { z12.d-z15.d }\n" "ldr x20, [%x[args], %[offsetof_Args_ld_out_cols]]\n" - ".inst 0xc0040f81 // mova za.d[x8, #1], { z28.d-z31.d }\n" + ".inst 0xc0040d81 // mova za.d[x8, #1], { z12.d-z15.d }\n" "mov x22, #0x4\n" - "ldp x15, x14, [x25], #0x10\n" - ".inst 0xc0040f82 // mova za.d[x8, #2], { z28.d-z31.d }\n" + "ldp x15, x14, [x23], #0x10\n" + ".inst 0xc0040d82 // mova za.d[x8, #2], { z12.d-z15.d }\n" "ldp x13, x11, [x20], #0x10\n" - ".inst 0xc0040f83 // mova za.d[x8, #3], { z28.d-z31.d }\n" + ".inst 0xc0040d83 // mova za.d[x8, #3], { z12.d-z15.d }\n" "ldr x21, [%x[args], %[offsetof_Args_pad_left]]\n" - ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" - "ldp x10, x9, [x25], #0x10\n" + ".inst 0xc0040d84 // mova za.d[x8, #4], { z12.d-z15.d }\n" + "ldp x10, x9, [x23], #0x10\n" "ldp x28, x27, [x20], #0x10\n" "cbz x21, 7f\n" "cmp x21, x22\n" @@ -263,24 +263,24 @@ void sme2_u8q_planar_5x5_s2_4rows_dot_za_impl( "sub x21, x21, x20\n" "sub x22, x22, x20\n" "cbz x21, 7f\n" - ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" - ".inst 0xc1a3ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z3.s\n" + ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" + ".inst 0xc1a6ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z6.s\n" "and x22, x21, #0x1\n" - ".inst 0xc1a1aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" + ".inst 0xc1a4aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" "add x21, x21, #0x1\n" "lsr x21, x21, #0x1\n" - ".inst 0xc1a8ab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z8.s\n" + ".inst 0xc1aaab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z10.s\n" "sub x16, x16, x21\n" - ".inst 0xc1b7cf44 // sclamp { z4.s-z7.s }, z26.s, z23.s\n" + ".inst 0xc1b5ccbc // sclamp { z28.s-z31.s }, z5.s, z21.s\n" "6:" // Left padding "subs x21, x21, #0x1\n" - "st1b { z4.s }, p1, [x15]\n" + "st1b { z28.s }, p1, [x15]\n" "add x15, x15, x13\n" - "st1b { z5.s }, p1, [x14]\n" + "st1b { z29.s }, p1, [x14]\n" "add x14, x14, x11\n" - "st1b { z6.s }, p1, [x10]\n" + "st1b { z30.s }, p1, [x10]\n" "add x10, x10, x28\n" - "st1b { z7.s }, p1, [x9]\n" + "st1b { z31.s }, p1, [x9]\n" "add x9, x9, x27\n" "bgt 6b\n" "7:" // Left padding: End @@ -296,341 +296,341 @@ void sme2_u8q_planar_5x5_s2_4rows_dot_za_impl( "beq 9f\n" "8:" // Unpadded: 4 priming loads "add x21, x17, %x[ld_in_row]\n" - "ld1b { z11.s }, p1/Z, [x17]\n" + "ld1b { z27.s }, p1/Z, [x17]\n" "addvl x20, SP, #12\n" - "ld1b { z21.s }, p1/Z, [x21]\n" + "ld1b { z0.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "add z11.h, z11.h, z9.h\n" - "ld1b { z12.s }, p1/Z, [x21]\n" + "trn1 z27.h, z27.h, z0.h\n" + "add z27.h, z27.h, z7.h\n" + "ld1b { z28.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "add x17, x17, %x[ld_in_col]\n" - "ld1b { z20.s }, p1/Z, [x21]\n" + "ld1b { z11.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "add z12.h, z12.h, z9.h\n" - "ld1b { z13.s }, p1/Z, [x21]\n" + "trn1 z28.h, z28.h, z11.h\n" + "add z28.h, z28.h, z7.h\n" + "ld1b { z29.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1b { z19.s }, p1/Z, [x21]\n" + "ld1b { z8.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z9.h\n" - "ld1b { z14.s }, p1/Z, [x21]\n" + "trn1 z29.h, z29.h, z8.h\n" + "add z29.h, z29.h, z7.h\n" + "ld1b { z30.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1b { z18.s }, p1/Z, [x21]\n" + "ld1b { z17.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z9.h\n" - "ld1b { z15.s }, p1/Z, [x21]\n" + "trn1 z30.h, z30.h, z17.h\n" + "add z30.h, z30.h, z7.h\n" + "ld1b { z31.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x21]\n" + "ld1b { z26.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z15.h, z15.h, z9.h\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1b { z16.s }, p1/Z, [x21]\n" - "mov z16.d, z16.d\n" - "add z16.h, z16.h, z9.h\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + "trn1 z31.h, z31.h, z26.h\n" + "add z31.h, z31.h, z7.h\n" + ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1701768 // sdot za.s[x8, 0], { z27.h-z30.h }, z0.h\n" + "ld1b { z20.s }, p1/Z, [x21]\n" + "mov z0.d, z20.d\n" + "add z0.h, z0.h, z7.h\n" + ".inst 0xc1781788 // sdot za.s[x8, 0], { z28.h-z31.h }, z8.h\n" + "ld1h { z8.h }, p2/Z, [x20, #2, MUL VL]\n" + ".inst 0xc17817a8 // sdot za.s[x8, 0], { z29.h-z0.h }, z8.h\n" "9:" // Unpadded: 3 priming loads "add x21, x17, %x[ld_in_row]\n" - "ld1b { z11.s }, p1/Z, [x17]\n" + "ld1b { z29.s }, p1/Z, [x17]\n" "addvl x20, SP, #9\n" - "ld1b { z21.s }, p1/Z, [x21]\n" + "ld1b { z17.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "add z11.h, z11.h, z9.h\n" - "ld1b { z12.s }, p1/Z, [x21]\n" + "trn1 z29.h, z29.h, z17.h\n" + "add z29.h, z29.h, z7.h\n" + "ld1b { z30.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "add x17, x17, %x[ld_in_col]\n" - "ld1b { z20.s }, p1/Z, [x21]\n" + "ld1b { z0.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "add z12.h, z12.h, z9.h\n" - "ld1b { z13.s }, p1/Z, [x21]\n" + "trn1 z30.h, z30.h, z0.h\n" + "add z30.h, z30.h, z7.h\n" + "ld1b { z31.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1b { z19.s }, p1/Z, [x21]\n" + "ld1b { z16.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z9.h\n" - "ld1b { z14.s }, p1/Z, [x21]\n" + "trn1 z31.h, z31.h, z16.h\n" + "add z31.h, z31.h, z7.h\n" + "ld1b { z0.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1b { z18.s }, p1/Z, [x21]\n" + "ld1b { z16.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z9.h\n" - "ld1b { z15.s }, p1/Z, [x21]\n" + "trn1 z0.h, z0.h, z16.h\n" + "add z0.h, z0.h, z7.h\n" + "ld1b { z1.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x21]\n" + "ld1b { z16.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z15.h, z15.h, z9.h\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" + "trn1 z1.h, z1.h, z16.h\n" + "add z1.h, z1.h, z7.h\n" + ".inst 0xa0402a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20]\n" + ".inst 0xc17217a8 // sdot za.s[x8, 0], { z29.h-z0.h }, z2.h\n" "ld1b { z16.s }, p1/Z, [x21]\n" - "mov z16.d, z16.d\n" - "add z16.h, z16.h, z9.h\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + "mov z2.d, z16.d\n" + "add z2.h, z2.h, z7.h\n" + ".inst 0xc17317c8 // sdot za.s[x8, 0], { z30.h-z1.h }, z3.h\n" + "ld1h { z8.h }, p2/Z, [x20, #2, MUL VL]\n" + ".inst 0xc17817e8 // sdot za.s[x8, 0], { z31.h-z2.h }, z8.h\n" "10:" // Unpadded: 2 priming loads "add x22, x17, %x[ld_in_row]\n" - "ld1b { z11.s }, p1/Z, [x17]\n" + "ld1b { z26.s }, p1/Z, [x17]\n" "addvl x21, SP, #6\n" - "ld1b { z21.s }, p1/Z, [x22]\n" + "ld1b { z16.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "add z11.h, z11.h, z9.h\n" - "ld1b { z12.s }, p1/Z, [x22]\n" + "trn1 z26.h, z26.h, z16.h\n" + "add z26.h, z26.h, z7.h\n" + "ld1b { z27.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "addvl x20, SP, #12\n" - "ld1b { z20.s }, p1/Z, [x22]\n" + "ld1b { z16.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "add z12.h, z12.h, z9.h\n" - "ld1b { z13.s }, p1/Z, [x22]\n" + "trn1 z27.h, z27.h, z16.h\n" + "add z27.h, z27.h, z7.h\n" + "ld1b { z28.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "add x17, x17, %x[ld_in_col]\n" - "ld1b { z19.s }, p1/Z, [x22]\n" + "ld1b { z29.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z9.h\n" - "ld1b { z14.s }, p1/Z, [x22]\n" + "trn1 z28.h, z28.h, z29.h\n" + "add z28.h, z28.h, z7.h\n" + "ld1b { z29.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "ld1b { z18.s }, p1/Z, [x22]\n" + "ld1b { z19.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z9.h\n" - "ld1b { z15.s }, p1/Z, [x22]\n" + "trn1 z29.h, z29.h, z19.h\n" + "add z29.h, z29.h, z7.h\n" + "ld1b { z30.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x22]\n" - "trn1 z15.h, z15.h, z17.h\n" + "ld1b { z23.s }, p1/Z, [x22]\n" + "trn1 z30.h, z30.h, z23.h\n" "add x22, x22, %x[ld_in_row]\n" - "add z15.h, z15.h, z9.h\n" - ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1b { z16.s }, p1/Z, [x22]\n" - "mov z16.d, z16.d\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" - "add z16.h, z16.h, z9.h\n" - "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + "add z30.h, z30.h, z7.h\n" + ".inst 0xa0402aa2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x21]\n" + ".inst 0xc1721748 // sdot za.s[x8, 0], { z26.h-z29.h }, z2.h\n" + "ld1b { z22.s }, p1/Z, [x22]\n" + "mov z31.d, z22.d\n" + ".inst 0xc1731768 // sdot za.s[x8, 0], { z27.h-z30.h }, z3.h\n" + ".inst 0xa1402a83 // ld1h { z3.h, z11.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1731749 // sdot za.s[x8, 1], { z26.h-z29.h }, z3.h\n" + "add z31.h, z31.h, z7.h\n" + "ld1h { z3.h }, p2/Z, [x21, #2, MUL VL]\n" + ".inst 0xc17b1769 // sdot za.s[x8, 1], { z27.h-z30.h }, z11.h\n" + ".inst 0xc1731788 // sdot za.s[x8, 0], { z28.h-z31.h }, z3.h\n" "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1701789 // sdot za.s[x8, 1], { z28.h-z31.h }, z0.h\n" "11:" // Unpadded: 1 priming loads "add x22, x17, %x[ld_in_row]\n" - "ld1b { z11.s }, p1/Z, [x17]\n" + "ld1b { z29.s }, p1/Z, [x17]\n" "addvl x21, SP, #3\n" - "ld1b { z21.s }, p1/Z, [x22]\n" + "ld1b { z22.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "add z11.h, z11.h, z9.h\n" - "ld1b { z12.s }, p1/Z, [x22]\n" + "trn1 z29.h, z29.h, z22.h\n" + "add z29.h, z29.h, z7.h\n" + "ld1b { z30.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "addvl x20, SP, #9\n" - "ld1b { z20.s }, p1/Z, [x22]\n" + "ld1b { z25.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "add z12.h, z12.h, z9.h\n" - "ld1b { z13.s }, p1/Z, [x22]\n" + "trn1 z30.h, z30.h, z25.h\n" + "add z30.h, z30.h, z7.h\n" + "ld1b { z31.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "add x17, x17, %x[ld_in_col]\n" - "ld1b { z19.s }, p1/Z, [x22]\n" + "ld1b { z16.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z9.h\n" - "ld1b { z14.s }, p1/Z, [x22]\n" + "trn1 z31.h, z31.h, z16.h\n" + "add z31.h, z31.h, z7.h\n" + "ld1b { z0.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "ld1b { z18.s }, p1/Z, [x22]\n" + "ld1b { z16.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z9.h\n" - "ld1b { z15.s }, p1/Z, [x22]\n" + "trn1 z0.h, z0.h, z16.h\n" + "add z0.h, z0.h, z7.h\n" + "ld1b { z1.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x22]\n" - "trn1 z15.h, z15.h, z17.h\n" + "ld1b { z2.s }, p1/Z, [x22]\n" + "trn1 z1.h, z1.h, z2.h\n" "add x22, x22, %x[ld_in_row]\n" - "add z15.h, z15.h, z9.h\n" - ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1b { z16.s }, p1/Z, [x22]\n" - "mov z16.d, z16.d\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" - "add z16.h, z16.h, z9.h\n" - "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" - "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" + "add z1.h, z1.h, z7.h\n" + ".inst 0xa0402aa2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x21]\n" + ".inst 0xc17217a8 // sdot za.s[x8, 0], { z29.h-z0.h }, z2.h\n" + "ld1b { z24.s }, p1/Z, [x22]\n" + "mov z2.d, z24.d\n" + ".inst 0xc17317c8 // sdot za.s[x8, 0], { z30.h-z1.h }, z3.h\n" + ".inst 0xa0402a88 // ld1h { z8.h-z9.h }, pn10.b/Z, [x20]\n" + ".inst 0xc17817a9 // sdot za.s[x8, 1], { z29.h-z0.h }, z8.h\n" + "add z2.h, z2.h, z7.h\n" + "ld1h { z3.h }, p2/Z, [x21, #2, MUL VL]\n" + ".inst 0xc17917c9 // sdot za.s[x8, 1], { z30.h-z1.h }, z9.h\n" + ".inst 0xc17317e8 // sdot za.s[x8, 0], { z31.h-z2.h }, z3.h\n" + "ld1h { z3.h }, p2/Z, [x20, #2, MUL VL]\n" + ".inst 0xc17317e9 // sdot za.s[x8, 1], { z31.h-z2.h }, z3.h\n" "12:" // Unpadded: 0 priming loads "cmp x7, #0x2\n" - ".inst 0xa1402be2 // ld1h { z2.h, z10.h }, pn10.b/Z, [SP]\n" - "ld1h { z0.h }, p2/Z, [SP, #2, MUL VL]\n" + ".inst 0xa1402be3 // ld1h { z3.h, z11.h }, pn10.b/Z, [SP]\n" + "ld1h { z2.h }, p2/Z, [SP, #2, MUL VL]\n" "blt 22f\n" "add x21, x17, %x[ld_in_row]\n" - "ld1b { z11.s }, p1/Z, [x17]\n" + "ld1b { z23.s }, p1/Z, [x17]\n" "sub x7, x7, #0x2\n" - "ld1b { z21.s }, p1/Z, [x21]\n" + "ld1b { z25.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" + "trn1 z23.h, z23.h, z25.h\n" "sub x16, x16, #0x1\n" - "ld1b { z12.s }, p1/Z, [x21]\n" + "ld1b { z24.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "lsr x20, x7, #0x1\n" - "add z11.h, z11.h, z9.h\n" - "ld1b { z20.s }, p1/Z, [x21]\n" + "add z23.h, z23.h, z7.h\n" + "ld1b { z30.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z24.h, z24.h, z30.h\n" "cmp x20, x16\n" - "ld1b { z13.s }, p1/Z, [x21]\n" + "ld1b { z25.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "csel x26, x20, x16, LT\n" - "add z12.h, z12.h, z9.h\n" - "ld1b { z19.s }, p1/Z, [x21]\n" + "add z24.h, z24.h, z7.h\n" + "ld1b { z22.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z9.h\n" - "ld1b { z14.s }, p1/Z, [x21]\n" + "trn1 z25.h, z25.h, z22.h\n" + "add z25.h, z25.h, z7.h\n" + "ld1b { z26.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "add x17, x17, %x[ld_in_col]\n" - "ld1b { z18.s }, p1/Z, [x21]\n" + "ld1b { z22.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z9.h\n" - "ld1b { z15.s }, p1/Z, [x21]\n" + "trn1 z26.h, z26.h, z22.h\n" + "add z26.h, z26.h, z7.h\n" + "ld1b { z27.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "and x7, x7, #0x1\n" - "ld1b { z17.s }, p1/Z, [x21]\n" + "ld1b { z30.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z15.h, z15.h, z9.h\n" - "ld1b { z16.s }, p1/Z, [x21]\n" - "mov z16.d, z16.d\n" - "add z16.h, z16.h, z9.h\n" + "trn1 z27.h, z27.h, z30.h\n" + "add z27.h, z27.h, z7.h\n" + "ld1b { z28.s }, p1/Z, [x21]\n" + "mov z28.d, z28.d\n" + "add z28.h, z28.h, z7.h\n" "sub x16, x16, x26\n" "cbz x26, 21f\n" "13:" // Unpadded: Main loop - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" + ".inst 0xc17316e8 // sdot za.s[x8, 0], { z23.h-z26.h }, z3.h\n" "addvl x25, SP, #6\n" "addvl x24, SP, #12\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402b22 // ld1h { z2.h, z10.h }, pn10.b/Z, [x25]\n" + ".inst 0xc17b1708 // sdot za.s[x8, 0], { z24.h-z27.h }, z11.h\n" + ".inst 0xa0402b20 // ld1h { z0.h-z1.h }, pn10.b/Z, [x25]\n" "add x23, x17, %x[ld_in_row]\n" "addvl x22, SP, #3\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" + ".inst 0xc17016e9 // sdot za.s[x8, 1], { z23.h-z26.h }, z0.h\n" "addvl x21, SP, #9\n" "subs x26, x26, #0x1\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402b02 // ld1h { z2.h, z10.h }, pn10.b/Z, [x24]\n" - ".inst 0xc172156a // sdot za.s[x8, 2], { z11.h-z14.h }, z2.h\n" - "ld1b { z11.s }, p1/Z, [x17]\n" + ".inst 0xc1711709 // sdot za.s[x8, 1], { z24.h-z27.h }, z1.h\n" + ".inst 0xa0402b08 // ld1h { z8.h-z9.h }, pn10.b/Z, [x24]\n" + ".inst 0xc17816ea // sdot za.s[x8, 2], { z23.h-z26.h }, z8.h\n" + "ld1b { z23.s }, p1/Z, [x17]\n" "add x17, x17, %x[ld_in_col]\n" "add x20, x17, %x[ld_in_row]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1721728 // sdot za.s[x8, 0], { z25.h-z28.h }, z2.h\n" "ld1h { z0.h }, p2/Z, [x25, #2, MUL VL]\n" - ".inst 0xc17a158a // sdot za.s[x8, 2], { z12.h-z15.h }, z10.h\n" - "ld1b { z21.s }, p1/Z, [x23]\n" + ".inst 0xc179170a // sdot za.s[x8, 2], { z24.h-z27.h }, z9.h\n" + "ld1b { z16.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" - "ld1h { z0.h }, p2/Z, [x24, #2, MUL VL]\n" - "add z11.h, z11.h, z9.h\n" - "ld1b { z12.s }, p1/Z, [x23]\n" + "trn1 z23.h, z23.h, z16.h\n" + ".inst 0xc1701729 // sdot za.s[x8, 1], { z25.h-z28.h }, z0.h\n" + "ld1h { z9.h }, p2/Z, [x24, #2, MUL VL]\n" + "add z23.h, z23.h, z7.h\n" + "ld1b { z24.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - ".inst 0xc17015aa // sdot za.s[x8, 2], { z13.h-z16.h }, z0.h\n" - "ld1b { z20.s }, p1/Z, [x23]\n" + ".inst 0xc179172a // sdot za.s[x8, 2], { z25.h-z28.h }, z9.h\n" + "ld1b { z18.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "add z12.h, z12.h, z9.h\n" - "ld1b { z13.s }, p1/Z, [x23]\n" + "trn1 z24.h, z24.h, z18.h\n" + "add z24.h, z24.h, z7.h\n" + "ld1b { z25.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" + ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" - "ld1b { z19.s }, p1/Z, [x23]\n" + "ld1b { z8.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z9.h\n" - "ld1b { z14.s }, p1/Z, [x23]\n" + "trn1 z25.h, z25.h, z8.h\n" + "add z25.h, z25.h, z7.h\n" + "ld1b { z26.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - ".inst 0xc1a3ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z3.s\n" - "ld1b { z18.s }, p1/Z, [x23]\n" + ".inst 0xc1a6ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n" + "ld1b { z28.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z9.h\n" - "ld1b { z15.s }, p1/Z, [x23]\n" + "trn1 z26.h, z26.h, z28.h\n" + "add z26.h, z26.h, z7.h\n" + "ld1b { z27.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - ".inst 0xc1a1aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" - "ld1b { z17.s }, p1/Z, [x23]\n" - "trn1 z15.h, z15.h, z17.h\n" + ".inst 0xc1a4aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z4.s\n" + "ld1b { z28.s }, p1/Z, [x23]\n" + "trn1 z27.h, z27.h, z28.h\n" "add x23, x23, %x[ld_in_row]\n" - "add z15.h, z15.h, z9.h\n" - ".inst 0xa1402ac2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x22]\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - ".inst 0xc1a8ab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z8.s\n" - "ld1b { z16.s }, p1/Z, [x23]\n" - "mov z16.d, z16.d\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" - "add z16.h, z16.h, z9.h\n" + "add z27.h, z27.h, z7.h\n" + ".inst 0xa0402ac2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x22]\n" + ".inst 0xc17216e8 // sdot za.s[x8, 0], { z23.h-z26.h }, z2.h\n" + ".inst 0xc1aaab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z10.s\n" + "ld1b { z20.s }, p1/Z, [x23]\n" + "mov z28.d, z20.d\n" + ".inst 0xc1731708 // sdot za.s[x8, 0], { z24.h-z27.h }, z3.h\n" + ".inst 0xa0402aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21]\n" + ".inst 0xc17016e9 // sdot za.s[x8, 1], { z23.h-z26.h }, z0.h\n" + "add z28.h, z28.h, z7.h\n" "ld1h { z0.h }, p2/Z, [x22, #2, MUL VL]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xc1b7cf44 // sclamp { z4.s-z7.s }, z26.s, z23.s\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1711709 // sdot za.s[x8, 1], { z24.h-z27.h }, z1.h\n" + ".inst 0xc1b5ccb0 // sclamp { z16.s-z19.s }, z5.s, z21.s\n" + ".inst 0xc1701728 // sdot za.s[x8, 0], { z25.h-z28.h }, z0.h\n" "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" - "st1b { z4.s }, p1, [x15]\n" + "st1b { z16.s }, p1, [x15]\n" "add x15, x15, x13\n" - "ld1b { z11.s }, p1/Z, [x17]\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" - "st1b { z5.s }, p1, [x14]\n" + "ld1b { z23.s }, p1/Z, [x17]\n" + ".inst 0xc1701729 // sdot za.s[x8, 1], { z25.h-z28.h }, z0.h\n" + "st1b { z17.s }, p1, [x14]\n" "add x14, x14, x11\n" - "ld1b { z21.s }, p1/Z, [x20]\n" + "ld1b { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "st1b { z6.s }, p1, [x10]\n" - "ld1b { z12.s }, p1/Z, [x20]\n" + "trn1 z23.h, z23.h, z16.h\n" + "st1b { z18.s }, p1, [x10]\n" + "ld1b { z24.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" "add x10, x10, x28\n" - "st1b { z7.s }, p1, [x9]\n" - "ld1b { z20.s }, p1/Z, [x20]\n" + "st1b { z19.s }, p1, [x9]\n" + "ld1b { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z24.h, z24.h, z16.h\n" "add x9, x9, x27\n" - "ld1b { z13.s }, p1/Z, [x20]\n" + "ld1b { z25.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" - "add z11.h, z11.h, z9.h\n" - "ld1b { z19.s }, p1/Z, [x20]\n" + ".inst 0xc0040d84 // mova za.d[x8, #4], { z12.d-z15.d }\n" + "add z23.h, z23.h, z7.h\n" + "ld1b { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z12.h, z12.h, z9.h\n" - "ld1b { z14.s }, p1/Z, [x20]\n" + "trn1 z25.h, z25.h, z16.h\n" + "add z24.h, z24.h, z7.h\n" + "ld1b { z26.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "add z13.h, z13.h, z9.h\n" + "add z25.h, z25.h, z7.h\n" "add x17, x17, %x[ld_in_col]\n" - "ld1b { z18.s }, p1/Z, [x20]\n" + "ld1b { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z9.h\n" - "ld1b { z15.s }, p1/Z, [x20]\n" + "trn1 z26.h, z26.h, z16.h\n" + "add z26.h, z26.h, z7.h\n" + "ld1b { z27.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x20]\n" + "ld1b { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z15.h, z15.h, z9.h\n" + "trn1 z27.h, z27.h, z16.h\n" + "add z27.h, z27.h, z7.h\n" "ld1b { z16.s }, p1/Z, [x20]\n" - "mov z16.d, z16.d\n" - "add z16.h, z16.h, z9.h\n" - ".inst 0xa1402be2 // ld1h { z2.h, z10.h }, pn10.b/Z, [SP]\n" - "ld1h { z0.h }, p2/Z, [SP, #2, MUL VL]\n" + "mov z28.d, z16.d\n" + "add z28.h, z28.h, z7.h\n" + ".inst 0xa1402be3 // ld1h { z3.h, z11.h }, pn10.b/Z, [SP]\n" + "ld1h { z2.h }, p2/Z, [SP, #2, MUL VL]\n" "bgt 13b\n" "b 21f\n" "14:" // Padded @@ -645,688 +645,688 @@ void sme2_u8q_planar_5x5_s2_4rows_dot_za_impl( "15:" // Padded: 4 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + "ld1b { z27.s }, p0/Z, [x17]\n" + "add z27.h, p0/M, z27.h, z7.h\n" "add x21, x17, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z21.s }, p0/Z, [x21]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1b { z17.s }, p0/Z, [x21]\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z12.s }, p0/Z, [x21]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + "ld1b { z28.s }, p0/Z, [x21]\n" + "add z28.h, p0/M, z28.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z20.s }, p0/Z, [x21]\n" - "add z20.h, p0/M, z20.h, z9.h\n" + "ld1b { z16.s }, p0/Z, [x21]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "mov x12, #0x4\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z27.h, z27.h, z17.h\n" + "trn1 z28.h, z28.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z13.s }, p0/Z, [x21]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + "ld1b { z29.s }, p0/Z, [x21]\n" + "add z29.h, p0/M, z29.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z19.s }, p0/Z, [x21]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "ld1b { z18.s }, p0/Z, [x21]\n" + "add z18.h, p0/M, z18.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z14.s }, p0/Z, [x21]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1b { z30.s }, p0/Z, [x21]\n" + "add z30.h, p0/M, z30.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z18.s }, p0/Z, [x21]\n" + "ld1b { z17.s }, p0/Z, [x21]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z15.s }, p0/Z, [x21]\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1b { z31.s }, p0/Z, [x21]\n" + "add z31.h, p0/M, z31.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z17.s }, p0/Z, [x21]\n" - "add z17.h, p0/M, z17.h, z9.h\n" + "ld1b { z16.s }, p0/Z, [x21]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "addvl x20, SP, #12\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" + "trn1 z29.h, z29.h, z18.h\n" + "trn1 z30.h, z30.h, z17.h\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1b { z16.s }, p0/Z, [x21]\n" - "add z16.h, p0/M, z16.h, z9.h\n" - "mov z16.d, z16.d\n" + ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" + "trn1 z31.h, z31.h, z16.h\n" + ".inst 0xc1701768 // sdot za.s[x8, 0], { z27.h-z30.h }, z0.h\n" + "ld1b { z20.s }, p0/Z, [x21]\n" + "add z20.h, p0/M, z20.h, z7.h\n" + "mov z0.d, z20.d\n" "add x17, x17, %x[ld_in_col]\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1711788 // sdot za.s[x8, 0], { z28.h-z31.h }, z1.h\n" + "ld1h { z1.h }, p2/Z, [x20, #2, MUL VL]\n" + ".inst 0xc17117a8 // sdot za.s[x8, 0], { z29.h-z0.h }, z1.h\n" "16:" // Padded: 3 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + "ld1b { z24.s }, p0/Z, [x17]\n" + "add z24.h, p0/M, z24.h, z7.h\n" "add x21, x17, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z21.s }, p0/Z, [x21]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1b { z17.s }, p0/Z, [x21]\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z12.s }, p0/Z, [x21]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + "ld1b { z25.s }, p0/Z, [x21]\n" + "add z25.h, p0/M, z25.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z20.s }, p0/Z, [x21]\n" - "add z20.h, p0/M, z20.h, z9.h\n" + "ld1b { z16.s }, p0/Z, [x21]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "mov x12, #0x4\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z24.h, z24.h, z17.h\n" + "trn1 z25.h, z25.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z13.s }, p0/Z, [x21]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + "ld1b { z26.s }, p0/Z, [x21]\n" + "add z26.h, p0/M, z26.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z19.s }, p0/Z, [x21]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "ld1b { z18.s }, p0/Z, [x21]\n" + "add z18.h, p0/M, z18.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z14.s }, p0/Z, [x21]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1b { z27.s }, p0/Z, [x21]\n" + "add z27.h, p0/M, z27.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z18.s }, p0/Z, [x21]\n" + "ld1b { z17.s }, p0/Z, [x21]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z15.s }, p0/Z, [x21]\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1b { z28.s }, p0/Z, [x21]\n" + "add z28.h, p0/M, z28.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z17.s }, p0/Z, [x21]\n" - "add z17.h, p0/M, z17.h, z9.h\n" + "ld1b { z16.s }, p0/Z, [x21]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "addvl x20, SP, #9\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" + "trn1 z26.h, z26.h, z18.h\n" + "trn1 z27.h, z27.h, z17.h\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1b { z16.s }, p0/Z, [x21]\n" - "add z16.h, p0/M, z16.h, z9.h\n" - "mov z16.d, z16.d\n" + ".inst 0xa0402a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20]\n" + "trn1 z28.h, z28.h, z16.h\n" + ".inst 0xc1721708 // sdot za.s[x8, 0], { z24.h-z27.h }, z2.h\n" + "ld1b { z11.s }, p0/Z, [x21]\n" + "add z11.h, p0/M, z11.h, z7.h\n" + "mov z29.d, z11.d\n" "add x17, x17, %x[ld_in_col]\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" + ".inst 0xc1731728 // sdot za.s[x8, 0], { z25.h-z28.h }, z3.h\n" "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1701748 // sdot za.s[x8, 0], { z26.h-z29.h }, z0.h\n" "17:" // Padded: 2 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + "ld1b { z25.s }, p0/Z, [x17]\n" + "add z25.h, p0/M, z25.h, z7.h\n" "add x20, x17, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z21.s }, p0/Z, [x20]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1b { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z12.s }, p0/Z, [x20]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + "ld1b { z26.s }, p0/Z, [x20]\n" + "add z26.h, p0/M, z26.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z9.h\n" + "ld1b { z16.s }, p0/Z, [x20]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z25.h, z25.h, z17.h\n" + "trn1 z26.h, z26.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + "ld1b { z27.s }, p0/Z, [x20]\n" + "add z27.h, p0/M, z27.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "ld1b { z18.s }, p0/Z, [x20]\n" + "add z18.h, p0/M, z18.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1b { z28.s }, p0/Z, [x20]\n" + "add z28.h, p0/M, z28.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" + "ld1b { z17.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1b { z29.s }, p0/Z, [x20]\n" + "add z29.h, p0/M, z29.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z9.h\n" + "ld1b { z16.s }, p0/Z, [x20]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "addvl x21, SP, #6\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" - ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" + "trn1 z27.h, z27.h, z18.h\n" + "trn1 z28.h, z28.h, z17.h\n" + ".inst 0xa1402aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21]\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1b { z16.s }, p0/Z, [x20]\n" + "trn1 z29.h, z29.h, z16.h\n" + ".inst 0xc1711728 // sdot za.s[x8, 0], { z25.h-z28.h }, z1.h\n" + "ld1b { z1.s }, p0/Z, [x20]\n" "addvl x20, SP, #12\n" - "add z16.h, p0/M, z16.h, z9.h\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" - "mov z16.d, z16.d\n" + "add z1.h, p0/M, z1.h, z7.h\n" + ".inst 0xc1791748 // sdot za.s[x8, 0], { z26.h-z29.h }, z9.h\n" + ".inst 0xa0402a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1721729 // sdot za.s[x8, 1], { z25.h-z28.h }, z2.h\n" + "mov z30.d, z1.d\n" "add x17, x17, %x[ld_in_col]\n" - "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + "ld1h { z9.h }, p2/Z, [x21, #2, MUL VL]\n" + ".inst 0xc1731749 // sdot za.s[x8, 1], { z26.h-z29.h }, z3.h\n" + ".inst 0xc1791768 // sdot za.s[x8, 0], { z27.h-z30.h }, z9.h\n" "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1701769 // sdot za.s[x8, 1], { z27.h-z30.h }, z0.h\n" "18:" // Padded: 1 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + "ld1b { z25.s }, p0/Z, [x17]\n" + "add z25.h, p0/M, z25.h, z7.h\n" "add x20, x17, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z21.s }, p0/Z, [x20]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1b { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z12.s }, p0/Z, [x20]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + "ld1b { z26.s }, p0/Z, [x20]\n" + "add z26.h, p0/M, z26.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z9.h\n" + "ld1b { z16.s }, p0/Z, [x20]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z25.h, z25.h, z17.h\n" + "trn1 z26.h, z26.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + "ld1b { z27.s }, p0/Z, [x20]\n" + "add z27.h, p0/M, z27.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "ld1b { z18.s }, p0/Z, [x20]\n" + "add z18.h, p0/M, z18.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1b { z28.s }, p0/Z, [x20]\n" + "add z28.h, p0/M, z28.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" + "ld1b { z17.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1b { z29.s }, p0/Z, [x20]\n" + "add z29.h, p0/M, z29.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z9.h\n" + "ld1b { z16.s }, p0/Z, [x20]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "addvl x21, SP, #3\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" - ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" + "trn1 z27.h, z27.h, z18.h\n" + "trn1 z28.h, z28.h, z17.h\n" + ".inst 0xa1402aa3 // ld1h { z3.h, z11.h }, pn10.b/Z, [x21]\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1b { z16.s }, p0/Z, [x20]\n" + "trn1 z29.h, z29.h, z16.h\n" + ".inst 0xc1731728 // sdot za.s[x8, 0], { z25.h-z28.h }, z3.h\n" + "ld1b { z0.s }, p0/Z, [x20]\n" "addvl x20, SP, #9\n" - "add z16.h, p0/M, z16.h, z9.h\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" - "mov z16.d, z16.d\n" + "add z0.h, p0/M, z0.h, z7.h\n" + ".inst 0xc17b1748 // sdot za.s[x8, 0], { z26.h-z29.h }, z11.h\n" + ".inst 0xa0402a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1721729 // sdot za.s[x8, 1], { z25.h-z28.h }, z2.h\n" + "mov z30.d, z0.d\n" "add x17, x17, %x[ld_in_col]\n" "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1731749 // sdot za.s[x8, 1], { z26.h-z29.h }, z3.h\n" + ".inst 0xc1701768 // sdot za.s[x8, 0], { z27.h-z30.h }, z0.h\n" "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1701769 // sdot za.s[x8, 1], { z27.h-z30.h }, z0.h\n" "19:" // Padded: 0 priming loads "cmp x7, #0x2\n" - ".inst 0xa1402be2 // ld1h { z2.h, z10.h }, pn10.b/Z, [SP]\n" - "ld1h { z0.h }, p2/Z, [SP, #2, MUL VL]\n" + ".inst 0xa1402be3 // ld1h { z3.h, z11.h }, pn10.b/Z, [SP]\n" + "ld1h { z2.h }, p2/Z, [SP, #2, MUL VL]\n" "blt 22f\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + "ld1b { z23.s }, p0/Z, [x17]\n" + "add z23.h, p0/M, z23.h, z7.h\n" "add x20, x17, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z21.s }, p0/Z, [x20]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1b { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z12.s }, p0/Z, [x20]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + "ld1b { z24.s }, p0/Z, [x20]\n" + "add z24.h, p0/M, z24.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z9.h\n" + "ld1b { z16.s }, p0/Z, [x20]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z23.h, z23.h, z17.h\n" + "trn1 z24.h, z24.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + "ld1b { z25.s }, p0/Z, [x20]\n" + "add z25.h, p0/M, z25.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1b { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "add z19.h, p0/M, z19.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1b { z26.s }, p0/Z, [x20]\n" + "add z26.h, p0/M, z26.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1b { z18.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z18.h, p0/M, z18.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1b { z27.s }, p0/Z, [x20]\n" + "add z27.h, p0/M, z27.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z9.h\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z9.h\n" + "add z16.h, p0/M, z16.h, z7.h\n" "sub x7, x7, #0x2\n" "sub x16, x16, #0x1\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" + "trn1 z25.h, z25.h, z19.h\n" + "trn1 z26.h, z26.h, z18.h\n" "lsr x20, x7, #0x1\n" "cmp x20, x16\n" - "trn1 z15.h, z15.h, z17.h\n" - "mov z16.d, z16.d\n" + "trn1 z27.h, z27.h, z17.h\n" + "mov z28.d, z16.d\n" "csel x25, x20, x16, LT\n" "add x17, x17, %x[ld_in_col]\n" "and x7, x7, #0x1\n" "sub x16, x16, x25\n" "cbz x25, 21f\n" "20:" // Padded: Main loop - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" + ".inst 0xc17316e8 // sdot za.s[x8, 0], { z23.h-z26.h }, z3.h\n" "addvl x24, SP, #6\n" "addvl x23, SP, #12\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402b02 // ld1h { z2.h, z10.h }, pn10.b/Z, [x24]\n" + ".inst 0xc17b1708 // sdot za.s[x8, 0], { z24.h-z27.h }, z11.h\n" + ".inst 0xa1402b00 // ld1h { z0.h, z8.h }, pn10.b/Z, [x24]\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" + ".inst 0xc17016e9 // sdot za.s[x8, 1], { z23.h-z26.h }, z0.h\n" "add x20, x17, %x[ld_in_row]\n" "addvl x22, SP, #3\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402ae2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x23]\n" + ".inst 0xc1781709 // sdot za.s[x8, 1], { z24.h-z27.h }, z8.h\n" + ".inst 0xa1402ae3 // ld1h { z3.h, z11.h }, pn10.b/Z, [x23]\n" "addvl x21, SP, #9\n" "subs x25, x25, #0x1\n" - ".inst 0xc172156a // sdot za.s[x8, 2], { z11.h-z14.h }, z2.h\n" - "ld1b { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + ".inst 0xc17316ea // sdot za.s[x8, 2], { z23.h-z26.h }, z3.h\n" + "ld1b { z23.s }, p0/Z, [x17]\n" + "add z23.h, p0/M, z23.h, z7.h\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z21.s }, p0/Z, [x20]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1b { z16.s }, p0/Z, [x20]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc17a158a // sdot za.s[x8, 2], { z12.h-z15.h }, z10.h\n" - "ld1b { z12.s }, p0/Z, [x20]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + ".inst 0xc17b170a // sdot za.s[x8, 2], { z24.h-z27.h }, z11.h\n" + "ld1b { z24.s }, p0/Z, [x20]\n" + "add z24.h, p0/M, z24.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1721728 // sdot za.s[x8, 0], { z25.h-z28.h }, z2.h\n" "ld1h { z0.h }, p2/Z, [x24, #2, MUL VL]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x4\n" - "ld1b { z20.s }, p0/Z, [x20]\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" - "add z20.h, p0/M, z20.h, z9.h\n" + "ld1b { z1.s }, p0/Z, [x20]\n" + ".inst 0xc1701729 // sdot za.s[x8, 1], { z25.h-z28.h }, z0.h\n" + "add z1.h, p0/M, z1.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" - "ld1h { z0.h }, p2/Z, [x23, #2, MUL VL]\n" + "ld1h { z3.h }, p2/Z, [x23, #2, MUL VL]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc17015aa // sdot za.s[x8, 2], { z13.h-z16.h }, z0.h\n" - "trn1 z11.h, z11.h, z21.h\n" - "ld1b { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + ".inst 0xc173172a // sdot za.s[x8, 2], { z25.h-z28.h }, z3.h\n" + "trn1 z23.h, z23.h, z16.h\n" + "ld1b { z25.s }, p0/Z, [x20]\n" + "add z25.h, p0/M, z25.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "ld1b { z3.s }, p0/Z, [x20]\n" + "add z3.h, p0/M, z3.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1b { z26.s }, p0/Z, [x20]\n" + "add z26.h, p0/M, z26.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" + "ld1b { z30.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z30.h, p0/M, z30.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1b { z27.s }, p0/Z, [x20]\n" + "add z27.h, p0/M, z27.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z9.h\n" - "trn1 z12.h, z12.h, z20.h\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" - ".inst 0xa1402ac2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x22]\n" + "ld1b { z29.s }, p0/Z, [x20]\n" + "add z29.h, p0/M, z29.h, z7.h\n" + "trn1 z24.h, z24.h, z1.h\n" + "trn1 z25.h, z25.h, z3.h\n" + "trn1 z26.h, z26.h, z30.h\n" + ".inst 0xa0402ac2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x22]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" + ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" - "trn1 z15.h, z15.h, z17.h\n" + "trn1 z27.h, z27.h, z29.h\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1b { z16.s }, p0/Z, [x20]\n" + ".inst 0xc17216e8 // sdot za.s[x8, 0], { z23.h-z26.h }, z2.h\n" + "ld1b { z20.s }, p0/Z, [x20]\n" "mov x12, #0x0\n" - "add z16.h, p0/M, z16.h, z9.h\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" + "add z20.h, p0/M, z20.h, z7.h\n" + ".inst 0xc1731708 // sdot za.s[x8, 0], { z24.h-z27.h }, z3.h\n" + ".inst 0xa0402aa2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x21]\n" "add x17, x17, %x[ld_in_col]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" - "ld1b { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + ".inst 0xc17216e9 // sdot za.s[x8, 1], { z23.h-z26.h }, z2.h\n" + "ld1b { z23.s }, p0/Z, [x17]\n" + "add z23.h, p0/M, z23.h, z7.h\n" "add x20, x17, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z21.s }, p0/Z, [x20]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1b { z8.s }, p0/Z, [x20]\n" + "add z8.h, p0/M, z8.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - "ld1b { z12.s }, p0/Z, [x20]\n" - "mov z16.d, z16.d\n" - "ld1h { z0.h }, p2/Z, [x22, #2, MUL VL]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + ".inst 0xc1731709 // sdot za.s[x8, 1], { z24.h-z27.h }, z3.h\n" + "ld1b { z24.s }, p0/Z, [x20]\n" + "mov z28.d, z20.d\n" + "ld1h { z1.h }, p2/Z, [x22, #2, MUL VL]\n" + "add z24.h, p0/M, z24.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z20.s }, p0/Z, [x20]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + "ld1b { z22.s }, p0/Z, [x20]\n" + ".inst 0xc1711728 // sdot za.s[x8, 0], { z25.h-z28.h }, z1.h\n" "mov x12, #0x4\n" - "add z20.h, p0/M, z20.h, z9.h\n" - "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" + "add z22.h, p0/M, z22.h, z7.h\n" + "ld1h { z1.h }, p2/Z, [x21, #2, MUL VL]\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" - "ld1b { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + ".inst 0xc1711729 // sdot za.s[x8, 1], { z25.h-z28.h }, z1.h\n" + "ld1b { z25.s }, p0/Z, [x20]\n" + "add z25.h, p0/M, z25.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "ld1b { z28.s }, p0/Z, [x20]\n" + "add z28.h, p0/M, z28.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1b { z26.s }, p0/Z, [x20]\n" + "add z26.h, p0/M, z26.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" + "ld1b { z20.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z20.h, p0/M, z20.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z15.s }, p0/Z, [x20]\n" - ".inst 0xc1a3ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z3.s\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1b { z27.s }, p0/Z, [x20]\n" + ".inst 0xc1a6ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n" + "add z27.h, p0/M, z27.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" - ".inst 0xc1a1aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" + "ld1b { z31.s }, p0/Z, [x20]\n" + ".inst 0xc1a4aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z4.s\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" - "add z17.h, p0/M, z17.h, z9.h\n" + ".inst 0xc0040d84 // mova za.d[x8, #4], { z12.d-z15.d }\n" + "add z31.h, p0/M, z31.h, z7.h\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z9.h\n" - ".inst 0xc1a8ab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z8.s\n" - ".inst 0xa1402be2 // ld1h { z2.h, z10.h }, pn10.b/Z, [SP]\n" + "ld1b { z1.s }, p0/Z, [x20]\n" + "add z1.h, p0/M, z1.h, z7.h\n" + ".inst 0xc1aaab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z10.s\n" + ".inst 0xa1402be3 // ld1h { z3.h, z11.h }, pn10.b/Z, [SP]\n" "add x17, x17, %x[ld_in_col]\n" - ".inst 0xc1b7cf44 // sclamp { z4.s-z7.s }, z26.s, z23.s\n" - "st1b { z4.s }, p1, [x15]\n" + ".inst 0xc1b5ccb0 // sclamp { z16.s-z19.s }, z5.s, z21.s\n" + "st1b { z16.s }, p1, [x15]\n" "add x15, x15, x13\n" - "ld1h { z0.h }, p2/Z, [SP, #2, MUL VL]\n" - "st1b { z5.s }, p1, [x14]\n" + "ld1h { z2.h }, p2/Z, [SP, #2, MUL VL]\n" + "st1b { z17.s }, p1, [x14]\n" "add x14, x14, x11\n" - "trn1 z11.h, z11.h, z21.h\n" - "trn1 z12.h, z12.h, z20.h\n" - "st1b { z6.s }, p1, [x10]\n" + "trn1 z23.h, z23.h, z8.h\n" + "trn1 z24.h, z24.h, z22.h\n" + "st1b { z18.s }, p1, [x10]\n" "add x10, x10, x28\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" - "st1b { z7.s }, p1, [x9]\n" + "trn1 z25.h, z25.h, z28.h\n" + "trn1 z26.h, z26.h, z20.h\n" + "st1b { z19.s }, p1, [x9]\n" "add x9, x9, x27\n" - "trn1 z15.h, z15.h, z17.h\n" - "mov z16.d, z16.d\n" + "trn1 z27.h, z27.h, z31.h\n" + "mov z28.d, z1.d\n" "bgt 20b\n" "21:" // Main loop tail - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" + ".inst 0xc17316e8 // sdot za.s[x8, 0], { z23.h-z26.h }, z3.h\n" "addvl x24, SP, #6\n" "addvl x23, SP, #12\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402b02 // ld1h { z2.h, z10.h }, pn10.b/Z, [x24]\n" + ".inst 0xc17b1708 // sdot za.s[x8, 0], { z24.h-z27.h }, z11.h\n" + ".inst 0xa0402b08 // ld1h { z8.h-z9.h }, pn10.b/Z, [x24]\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" + ".inst 0xc17816e9 // sdot za.s[x8, 1], { z23.h-z26.h }, z8.h\n" "add x22, x17, %x[ld_in_row]\n" "addvl x21, SP, #3\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402ae2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x23]\n" + ".inst 0xc1791709 // sdot za.s[x8, 1], { z24.h-z27.h }, z9.h\n" + ".inst 0xa1402ae3 // ld1h { z3.h, z11.h }, pn10.b/Z, [x23]\n" "addvl x20, SP, #9\n" - ".inst 0xc172156a // sdot za.s[x8, 2], { z11.h-z14.h }, z2.h\n" - "ld1b { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + ".inst 0xc17316ea // sdot za.s[x8, 2], { z23.h-z26.h }, z3.h\n" + "ld1b { z29.s }, p0/Z, [x17]\n" + "add z29.h, p0/M, z29.h, z7.h\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z21.s }, p0/Z, [x22]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1b { z8.s }, p0/Z, [x22]\n" + "add z8.h, p0/M, z8.h, z7.h\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc17a158a // sdot za.s[x8, 2], { z12.h-z15.h }, z10.h\n" - "ld1b { z12.s }, p0/Z, [x22]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + ".inst 0xc17b170a // sdot za.s[x8, 2], { z24.h-z27.h }, z11.h\n" + "ld1b { z30.s }, p0/Z, [x22]\n" + "add z30.h, p0/M, z30.h, z7.h\n" "add x22, x22, %x[ld_in_row]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1721728 // sdot za.s[x8, 0], { z25.h-z28.h }, z2.h\n" "ld1h { z0.h }, p2/Z, [x24, #2, MUL VL]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x4\n" "ld1b { z20.s }, p0/Z, [x22]\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" - "add z20.h, p0/M, z20.h, z9.h\n" + ".inst 0xc1701729 // sdot za.s[x8, 1], { z25.h-z28.h }, z0.h\n" + "add z20.h, p0/M, z20.h, z7.h\n" "add x22, x22, %x[ld_in_row]\n" - "ld1h { z0.h }, p2/Z, [x23, #2, MUL VL]\n" + "ld1h { z2.h }, p2/Z, [x23, #2, MUL VL]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc17015aa // sdot za.s[x8, 2], { z13.h-z16.h }, z0.h\n" - "trn1 z11.h, z11.h, z21.h\n" - "ld1b { z13.s }, p0/Z, [x22]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + ".inst 0xc172172a // sdot za.s[x8, 2], { z25.h-z28.h }, z2.h\n" + "trn1 z29.h, z29.h, z8.h\n" + "ld1b { z31.s }, p0/Z, [x22]\n" + "add z31.h, p0/M, z31.h, z7.h\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z19.s }, p0/Z, [x22]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "ld1b { z25.s }, p0/Z, [x22]\n" + "add z25.h, p0/M, z25.h, z7.h\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z14.s }, p0/Z, [x22]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1b { z0.s }, p0/Z, [x22]\n" + "add z0.h, p0/M, z0.h, z7.h\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z18.s }, p0/Z, [x22]\n" + "ld1b { z17.s }, p0/Z, [x22]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z15.s }, p0/Z, [x22]\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1b { z1.s }, p0/Z, [x22]\n" + "add z1.h, p0/M, z1.h, z7.h\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z17.s }, p0/Z, [x22]\n" - "add z17.h, p0/M, z17.h, z9.h\n" - "trn1 z12.h, z12.h, z20.h\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" - ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" + "ld1b { z28.s }, p0/Z, [x22]\n" + "add z28.h, p0/M, z28.h, z7.h\n" + "trn1 z30.h, z30.h, z20.h\n" + "trn1 z31.h, z31.h, z25.h\n" + "trn1 z0.h, z0.h, z17.h\n" + ".inst 0xa1402aa3 // ld1h { z3.h, z11.h }, pn10.b/Z, [x21]\n" "add x22, x22, %x[ld_in_row]\n" - ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" + ".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1b { z16.s }, p0/Z, [x22]\n" - ".inst 0xc1a3ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z3.s\n" - "add z16.h, p0/M, z16.h, z9.h\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" + "trn1 z1.h, z1.h, z28.h\n" + ".inst 0xc17317a8 // sdot za.s[x8, 0], { z29.h-z0.h }, z3.h\n" + "ld1b { z22.s }, p0/Z, [x22]\n" + ".inst 0xc1a6ac18 // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z6.s\n" + "add z22.h, p0/M, z22.h, z7.h\n" + ".inst 0xc17b17c8 // sdot za.s[x8, 0], { z30.h-z1.h }, z11.h\n" + ".inst 0xa1402a83 // ld1h { z3.h, z11.h }, pn10.b/Z, [x20]\n" "add x17, x17, %x[ld_in_col]\n" - ".inst 0xc1a1aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" - "mov z16.d, z16.d\n" - "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xc1a8ab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z8.s\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" - "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc1b7cf44 // sclamp { z4.s-z7.s }, z26.s, z23.s\n" - "st1b { z4.s }, p1, [x15]\n" + ".inst 0xc1a4aa38 // srshl { z24.s-z27.s }, { z24.s-z27.s }, z4.s\n" + ".inst 0xc17317a9 // sdot za.s[x8, 1], { z29.h-z0.h }, z3.h\n" + "mov z2.d, z22.d\n" + "ld1h { z9.h }, p2/Z, [x21, #2, MUL VL]\n" + ".inst 0xc17b17c9 // sdot za.s[x8, 1], { z30.h-z1.h }, z11.h\n" + ".inst 0xc1aaab18 // add { z24.s-z27.s }, { z24.s-z27.s }, z10.s\n" + ".inst 0xc17917e8 // sdot za.s[x8, 0], { z31.h-z2.h }, z9.h\n" + "ld1h { z8.h }, p2/Z, [x20, #2, MUL VL]\n" + ".inst 0xc1b5ccb8 // sclamp { z24.s-z27.s }, z5.s, z21.s\n" + "st1b { z24.s }, p1, [x15]\n" "add x15, x15, x13\n" - "st1b { z5.s }, p1, [x14]\n" + "st1b { z25.s }, p1, [x14]\n" "add x14, x14, x11\n" - ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" - ".inst 0xa1402be2 // ld1h { z2.h, z10.h }, pn10.b/Z, [SP]\n" - "st1b { z6.s }, p1, [x10]\n" + ".inst 0xc0040d84 // mova za.d[x8, #4], { z12.d-z15.d }\n" + ".inst 0xa1402be3 // ld1h { z3.h, z11.h }, pn10.b/Z, [SP]\n" + "st1b { z26.s }, p1, [x10]\n" "add x10, x10, x28\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" - "ld1h { z0.h }, p2/Z, [SP, #2, MUL VL]\n" - "st1b { z7.s }, p1, [x9]\n" + ".inst 0xc17817e9 // sdot za.s[x8, 1], { z31.h-z2.h }, z8.h\n" + "ld1h { z2.h }, p2/Z, [SP, #2, MUL VL]\n" + "st1b { z27.s }, p1, [x9]\n" "add x9, x9, x27\n" "22:" // Main loop skip tail "cbz x7, 23f\n" // Skip remainder inputs "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + "ld1b { z24.s }, p0/Z, [x17]\n" + "add z24.h, p0/M, z24.h, z7.h\n" "add x20, x17, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z21.s }, p0/Z, [x20]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1b { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z12.s }, p0/Z, [x20]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + "ld1b { z25.s }, p0/Z, [x20]\n" + "add z25.h, p0/M, z25.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z9.h\n" + "ld1b { z16.s }, p0/Z, [x20]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z24.h, z24.h, z17.h\n" + "trn1 z25.h, z25.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + "ld1b { z26.s }, p0/Z, [x20]\n" + "add z26.h, p0/M, z26.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "ld1b { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1b { z27.s }, p0/Z, [x20]\n" + "add z27.h, p0/M, z27.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" + "ld1b { z16.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z16.h, p0/M, z16.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1b { z28.s }, p0/Z, [x20]\n" + "add z28.h, p0/M, z28.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z9.h\n" + "ld1b { z31.s }, p0/Z, [x20]\n" + "add z31.h, p0/M, z31.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z9.h\n" - "trn1 z15.h, z15.h, z17.h\n" + "trn1 z26.h, z26.h, z17.h\n" + "trn1 z27.h, z27.h, z16.h\n" + "ld1b { z0.s }, p0/Z, [x20]\n" + "add z0.h, p0/M, z0.h, z7.h\n" + "trn1 z28.h, z28.h, z31.h\n" "addvl x21, SP, #6\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "mov z16.d, z16.d\n" + ".inst 0xc1731708 // sdot za.s[x8, 0], { z24.h-z27.h }, z3.h\n" + "mov z29.d, z0.d\n" "addvl x20, SP, #12\n" "sub x16, x16, #0x1\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" - "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" - ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" - ".inst 0xc1a3ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z3.s\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1a1aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" - ".inst 0xc172156a // sdot za.s[x8, 2], { z11.h-z14.h }, z2.h\n" - ".inst 0xc1a8ab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z8.s\n" - ".inst 0xc17a158a // sdot za.s[x8, 2], { z12.h-z15.h }, z10.h\n" - ".inst 0xc1b7cf44 // sclamp { z4.s-z7.s }, z26.s, z23.s\n" - "st1b { z4.s }, p1, [x15]\n" + ".inst 0xc17b1728 // sdot za.s[x8, 0], { z25.h-z28.h }, z11.h\n" + ".inst 0xa0402aa8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x21]\n" + ".inst 0xc1721748 // sdot za.s[x8, 0], { z26.h-z29.h }, z2.h\n" + "ld1h { z2.h }, p2/Z, [x21, #2, MUL VL]\n" + ".inst 0xc1781709 // sdot za.s[x8, 1], { z24.h-z27.h }, z8.h\n" + ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" + ".inst 0xc1a6ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n" + ".inst 0xc1791729 // sdot za.s[x8, 1], { z25.h-z28.h }, z9.h\n" + ".inst 0xa1402a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1a4aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z4.s\n" + ".inst 0xc171170a // sdot za.s[x8, 2], { z24.h-z27.h }, z1.h\n" + ".inst 0xc1aaab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z10.s\n" + ".inst 0xc179172a // sdot za.s[x8, 2], { z25.h-z28.h }, z9.h\n" + ".inst 0xc1b5ccb0 // sclamp { z16.s-z19.s }, z5.s, z21.s\n" + "st1b { z16.s }, p1, [x15]\n" "add x15, x15, x13\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" - "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - "st1b { z5.s }, p1, [x14]\n" + ".inst 0xc1721749 // sdot za.s[x8, 1], { z26.h-z29.h }, z2.h\n" + "ld1h { z3.h }, p2/Z, [x20, #2, MUL VL]\n" + "st1b { z17.s }, p1, [x14]\n" "add x14, x14, x11\n" - ".inst 0xc17015aa // sdot za.s[x8, 2], { z13.h-z16.h }, z0.h\n" + ".inst 0xc173174a // sdot za.s[x8, 2], { z26.h-z29.h }, z3.h\n" "add x8, x8, #0x1\n" - "st1b { z6.s }, p1, [x10]\n" + "st1b { z18.s }, p1, [x10]\n" "add x10, x10, x28\n" - "st1b { z7.s }, p1, [x9]\n" + "st1b { z19.s }, p1, [x9]\n" "add x9, x9, x27\n" - ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" + ".inst 0xc0040d84 // mova za.d[x8, #4], { z12.d-z15.d }\n" "23:" // Tail input: End "cbz x16, 25f\n" "24:" // Right padding loop - ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" - ".inst 0xc1a3ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z3.s\n" + ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" + ".inst 0xc1a6ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z6.s\n" "add x8, x8, #0x1\n" - ".inst 0xc1a1aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" + ".inst 0xc1a4aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" "subs x16, x16, #0x1\n" - ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" - ".inst 0xc1a8ab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z8.s\n" - ".inst 0xc1b7cf44 // sclamp { z4.s-z7.s }, z26.s, z23.s\n" - "st1b { z4.s }, p1, [x15]\n" + ".inst 0xc0040d84 // mova za.d[x8, #4], { z12.d-z15.d }\n" + ".inst 0xc1aaab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z10.s\n" + ".inst 0xc1b5ccbc // sclamp { z28.s-z31.s }, z5.s, z21.s\n" + "st1b { z28.s }, p1, [x15]\n" "add x15, x15, x13\n" - "st1b { z5.s }, p1, [x14]\n" + "st1b { z29.s }, p1, [x14]\n" "add x14, x14, x11\n" - "st1b { z6.s }, p1, [x10]\n" + "st1b { z30.s }, p1, [x10]\n" "add x10, x10, x28\n" - "st1b { z7.s }, p1, [x9]\n" + "st1b { z31.s }, p1, [x9]\n" "add x9, x9, x27\n" "bgt 24b\n" "25:" // End - "ldr x22, [%x[args], %[offsetof_Args_weights]]\n" - "incw x22, ALL, MUL #16\n" - "incw x22, ALL, MUL #9\n" - "str x22, [%x[args], %[offsetof_Args_weights]]\n" - "ldr x20, [%x[args], %[offsetof_Args_ld_in_vl]]\n" + "ldr x20, [%x[args], %[offsetof_Args_weights]]\n" + "incw x20, ALL, MUL #16\n" + "incw x20, ALL, MUL #9\n" + "str x20, [%x[args], %[offsetof_Args_weights]]\n" + "ldr x21, [%x[args], %[offsetof_Args_ld_in_vl]]\n" "incw x6\n" "whilelt p1.s, x6, x5\n" - "ldr x17, [%x[args], %[offsetof_Args_inptr]]\n" - "add x17, x17, x20\n" - "str x17, [%x[args], %[offsetof_Args_inptr]]\n" + "ldr x20, [%x[args], %[offsetof_Args_inptr]]\n" + "add x20, x20, x21\n" + "str x20, [%x[args], %[offsetof_Args_inptr]]\n" "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" "ldr x24, [%x[args], %[offsetof_Args_ld_out_vls]]\n" "ldp x23, x22, [x25, #0x0]\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_3x3_s1_4rows_dot_za.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_3x3_s1_4rows_dot_za.hpp index de574fff9aa7ea3eff366c33fa7840304e1ac2ef..a4345097b5f8c48ec4a799d004b720800f9271b3 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_3x3_s1_4rows_dot_za.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_3x3_s1_4rows_dot_za.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,8 @@ * SOFTWARE. */ +#include "src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp" + namespace arm_conv { namespace depthwise { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_3x3_s1_4rows_dot_za/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_3x3_s1_4rows_dot_za/generic.cpp index 1636225b31373bb1c38c3a25d8145e25673e47f0..612beb342a0d25852c1c1c7cd73ddaa450a36aa9 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_3x3_s1_4rows_dot_za/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_3x3_s1_4rows_dot_za/generic.cpp @@ -73,96 +73,96 @@ void sme2_u8s8u8q_planar_3x3_s1_4rows_dot_za_impl( "ptrue p2.b\n" "mov x20, #0x6\n" "ldr x7, [%x[args], %[offsetof_Args_pad_top]]\n" - "ld1rh { z24.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_a_offset]]\n" + "ld1rh { z21.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_a_offset]]\n" "sub x20, x20, x6\n" ".inst 0x25207812 // ptrue pn10.b\n" "ldr x17, [%x[args], %[offsetof_Args_n_channels]]\n" "whilelt p1.s, XZR, x17\n" "whilelt p9.s, XZR, x20\n" - "ld1rw { z12.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" + "ld1rw { z15.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" "whilelt p8.s, XZR, x7\n" "addvl SP, SP, #-12\n" "ldr x16, [%x[args], %[offsetof_Args_current_channel]]\n" - "neg z24.h, p2/M, z24.h\n" + "neg z21.h, p2/M, z21.h\n" "eor p8.b, p2/Z, p8.b, p9.b\n" - "ld1rw { z10.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_mul]]\n" - "ld1rw { z11.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_right_shift]]\n" - "ld1rw { z22.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" - "ld1rw { z26.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" + "ld1rw { z14.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z12.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z29.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" + "ld1rw { z28.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" "1:" // Channel loop "ldr x20, [%x[qp], %[offsetof_Requantize32_bias]]\n" - "mov z8.s, #0x0\n" + "mov z30.s, #0x0\n" "cbz x20, 2f\n" - "ld1w { z8.s }, p1/Z, [x20, x16, LSL #2]\n" + "ld1w { z30.s }, p1/Z, [x20, x16, LSL #2]\n" "2:" // Load bias: Done "ldr x22, [%x[args], %[offsetof_Args_weights]]\n" "mov x20, x22\n" - "ld1sb { z27.s }, p2/Z, [x20]\n" + "ld1sb { z10.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "ld1rh { z21.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" - "mov z20.h, #0x0\n" - "sub z27.h, z27.h, z21.h\n" + "ld1rh { z31.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" + "mov z7.h, #0x0\n" + "sub z10.h, z10.h, z31.h\n" "incw x22\n" - "ld1sb { z23.s }, p2/Z, [x20]\n" - "incw x20, ALL, MUL #3\n" - "sub z23.h, z23.h, z21.h\n" - "trn1 z0.h, z20.h, z27.h\n" "ld1sb { z16.s }, p2/Z, [x20]\n" - "sub z16.h, z16.h, z21.h\n" + "incw x20, ALL, MUL #3\n" + "sub z16.h, z16.h, z31.h\n" + "trn1 z20.h, z7.h, z10.h\n" + "ld1sb { z11.s }, p2/Z, [x20]\n" + "sub z11.h, z11.h, z31.h\n" "mov x20, x22\n" - "trn1 z1.h, z27.h, z23.h\n" - "ld1sb { z27.s }, p2/Z, [x20]\n" + "trn1 z19.h, z10.h, z16.h\n" + "ld1sb { z24.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "trn1 z2.h, z23.h, z16.h\n" - "trn1 z3.h, z16.h, z20.h\n" - "ld1sb { z23.s }, p2/Z, [x20]\n" + "trn1 z26.h, z16.h, z11.h\n" + "trn1 z13.h, z11.h, z7.h\n" + "ld1sb { z11.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "sub z27.h, z27.h, z21.h\n" - "sub z23.h, z23.h, z21.h\n" - "ld1sb { z16.s }, p2/Z, [x20]\n" - "sub z16.h, z16.h, z21.h\n" + "sub z24.h, z24.h, z31.h\n" + "sub z11.h, z11.h, z31.h\n" + "ld1sb { z2.s }, p2/Z, [x20]\n" + "sub z2.h, z2.h, z31.h\n" "addvl x21, SP, #12\n" "incw x22\n" "addvl x21, x21, #-4\n" "mov x20, x22\n" - "st1h { z0.h }, p2, [x21]\n" - "trn1 z0.h, z20.h, z27.h\n" - "st1h { z1.h }, p2, [x21, #1, MUL VL]\n" - "trn1 z1.h, z27.h, z23.h\n" - "ld1sb { z27.s }, p2/Z, [x20]\n" + "st1h { z20.h }, p2, [x21]\n" + "trn1 z22.h, z7.h, z24.h\n" + "st1h { z19.h }, p2, [x21, #1, MUL VL]\n" + "trn1 z1.h, z24.h, z11.h\n" + "ld1sb { z16.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "st1h { z2.h }, p2, [x21, #2, MUL VL]\n" - "trn1 z2.h, z23.h, z16.h\n" - "ld1sb { z23.s }, p2/Z, [x20]\n" + "st1h { z26.h }, p2, [x21, #2, MUL VL]\n" + "trn1 z3.h, z11.h, z2.h\n" + "ld1sb { z0.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "st1h { z3.h }, p2, [x21, #3, MUL VL]\n" - "trn1 z3.h, z16.h, z20.h\n" - "ld1sb { z16.s }, p2/Z, [x20]\n" + "st1h { z13.h }, p2, [x21, #3, MUL VL]\n" + "trn1 z25.h, z2.h, z7.h\n" + "ld1sb { z4.s }, p2/Z, [x20]\n" "ldr x20, [%x[qp], %[offsetof_Requantize32_per_channel_muls]]\n" - "sub z27.h, z27.h, z21.h\n" - "sub z23.h, z23.h, z21.h\n" + "sub z16.h, z16.h, z31.h\n" + "sub z0.h, z0.h, z31.h\n" "addvl x21, x21, #-4\n" - "st1h { z0.h }, p2, [x21]\n" - "sub z16.h, z16.h, z21.h\n" + "st1h { z22.h }, p2, [x21]\n" + "sub z4.h, z4.h, z31.h\n" "st1h { z1.h }, p2, [x21, #1, MUL VL]\n" - "mov z9.d, z8.d\n" - "st1h { z2.h }, p2, [x21, #2, MUL VL]\n" - "trn1 z0.h, z20.h, z27.h\n" - "trn1 z1.h, z27.h, z23.h\n" - "st1h { z3.h }, p2, [x21, #3, MUL VL]\n" + "mov z31.d, z30.d\n" + "st1h { z3.h }, p2, [x21, #2, MUL VL]\n" + "trn1 z24.h, z7.h, z16.h\n" + "trn1 z18.h, z16.h, z0.h\n" + "st1h { z25.h }, p2, [x21, #3, MUL VL]\n" "addvl x21, x21, #-4\n" - "trn1 z2.h, z23.h, z16.h\n" - "trn1 z3.h, z16.h, z20.h\n" - "st1h { z0.h }, p2, [x21]\n" - "st1h { z1.h }, p2, [x21, #1, MUL VL]\n" - "st1h { z2.h }, p2, [x21, #2, MUL VL]\n" - "st1h { z3.h }, p2, [x21, #3, MUL VL]\n" + "trn1 z0.h, z0.h, z4.h\n" + "trn1 z1.h, z4.h, z7.h\n" + "st1h { z24.h }, p2, [x21]\n" + "st1h { z18.h }, p2, [x21, #1, MUL VL]\n" + "st1h { z0.h }, p2, [x21, #2, MUL VL]\n" + "st1h { z1.h }, p2, [x21, #3, MUL VL]\n" "cbz x20, 3f\n" - "ld1w { z10.s }, p1/Z, [x20, x16, LSL #2]\n" + "ld1w { z14.s }, p1/Z, [x20, x16, LSL #2]\n" "3:" // Load mul: End "ldr x20, [%x[qp], %[offsetof_Requantize32_per_channel_right_shifts]]\n" "cbz x20, 4f\n" - "ld1w { z11.s }, p1/Z, [x20, x16, LSL #2]\n" + "ld1w { z12.s }, p1/Z, [x20, x16, LSL #2]\n" "4:" // Load right_shift: End "ldr x15, [%x[args], %[offsetof_Args_input_cols]]\n" "sub x20, x15, #0x1\n" @@ -182,21 +182,21 @@ void sme2_u8s8u8q_planar_3x3_s1_4rows_dot_za_impl( ".inst 0xf8b74a9c // rprfm pldstrm, x23, [x20]\n" "add x20, x20, %x[ld_in_col]\n" "bgt 5b\n" - "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" + "ldr x23, [%x[args], %[offsetof_Args_outptrs]]\n" "lsl x20, %x[ld_in_row], #0x0\n" "msub x14, x7, x20, x14\n" - ".inst 0xc0040900 // mova za.d[x8, #0], { z8.d-z9.d }\n" + ".inst 0xc0040bc0 // mova za.d[x8, #0], { z30.d-z31.d }\n" "ldr x20, [%x[args], %[offsetof_Args_ld_out_cols]]\n" - ".inst 0xc0040901 // mova za.d[x8, #1], { z8.d-z9.d }\n" + ".inst 0xc0040bc1 // mova za.d[x8, #1], { z30.d-z31.d }\n" "mov x22, #0x2\n" - "ldp x11, x10, [x25], #0x10\n" - ".inst 0xc0040902 // mova za.d[x8, #2], { z8.d-z9.d }\n" + "ldp x11, x10, [x23], #0x10\n" + ".inst 0xc0040bc2 // mova za.d[x8, #2], { z30.d-z31.d }\n" "ldp x9, x28, [x20], #0x10\n" - ".inst 0xc0040903 // mova za.d[x8, #3], { z8.d-z9.d }\n" + ".inst 0xc0040bc3 // mova za.d[x8, #3], { z30.d-z31.d }\n" "ldr x21, [%x[args], %[offsetof_Args_pad_left]]\n" - ".inst 0xc0040904 // mova za.d[x8, #4], { z8.d-z9.d }\n" - "ldp x27, x26, [x25], #0x10\n" - ".inst 0xc0040905 // mova za.d[x8, #5], { z8.d-z9.d }\n" + ".inst 0xc0040bc4 // mova za.d[x8, #4], { z30.d-z31.d }\n" + "ldp x27, x26, [x23], #0x10\n" + ".inst 0xc0040bc5 // mova za.d[x8, #5], { z30.d-z31.d }\n" "ldp x25, x24, [x20], #0x10\n" "cbz x21, 7f\n" "cmp x21, x22\n" @@ -204,22 +204,22 @@ void sme2_u8s8u8q_planar_3x3_s1_4rows_dot_za_impl( "sub x21, x21, x20\n" "sub x22, x22, x20\n" "cbz x21, 7f\n" - ".inst 0xc0060804 // mova { z4.d-z5.d }, za.d[x8, #0]\n" + ".inst 0xc0060818 // mova { z24.d-z25.d }, za.d[x8, #0]\n" "sub x13, x13, x21\n" - ".inst 0xc0060826 // mova { z6.d-z7.d }, za.d[x8, #1]\n" - ".inst 0xc1aaac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z10.s\n" - ".inst 0xc1abaa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z11.s\n" - ".inst 0xc1acab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z12.s\n" - ".inst 0xc1bacec4 // sclamp { z4.s-z7.s }, z22.s, z26.s\n" + ".inst 0xc006083a // mova { z26.d-z27.d }, za.d[x8, #1]\n" + ".inst 0xc1aeac18 // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z14.s\n" + ".inst 0xc1acaa38 // srshl { z24.s-z27.s }, { z24.s-z27.s }, z12.s\n" + ".inst 0xc1afab18 // add { z24.s-z27.s }, { z24.s-z27.s }, z15.s\n" + ".inst 0xc1bccfb8 // sclamp { z24.s-z27.s }, z29.s, z28.s\n" "6:" // Left padding "subs x21, x21, #0x1\n" - "st1b { z4.s }, p1, [x11]\n" + "st1b { z24.s }, p1, [x11]\n" "add x11, x11, x9\n" - "st1b { z6.s }, p1, [x10]\n" + "st1b { z26.s }, p1, [x10]\n" "add x10, x10, x28\n" - "st1b { z5.s }, p1, [x27]\n" + "st1b { z25.s }, p1, [x27]\n" "add x27, x27, x25\n" - "st1b { z7.s }, p1, [x26]\n" + "st1b { z27.s }, p1, [x26]\n" "add x26, x26, x24\n" "bgt 6b\n" "7:" // Left padding: End @@ -231,148 +231,148 @@ void sme2_u8s8u8q_planar_3x3_s1_4rows_dot_za_impl( "beq 9f\n" "8:" // Unpadded: 2 priming loads "add x21, x14, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x14]\n" + "ld1b { z20.s }, p1/Z, [x14]\n" "addvl x20, SP, #8\n" "ld1b { z16.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z17.h, z16.h\n" - "add z13.h, z13.h, z24.h\n" - "ld1b { z17.s }, p1/Z, [x21]\n" + "trn1 z4.h, z20.h, z16.h\n" + "add z4.h, z4.h, z21.h\n" + "ld1b { z23.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "add x14, x14, %x[ld_in_col]\n" - "ld1b { z16.s }, p1/Z, [x21]\n" + "ld1b { z22.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z14.h, z17.h, z16.h\n" - "add z14.h, z14.h, z24.h\n" + "trn1 z5.h, z23.h, z22.h\n" + "add z5.h, z5.h, z21.h\n" "ld1b { z17.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "ld1b { z16.s }, p1/Z, [x21]\n" - "trn1 z15.h, z17.h, z16.h\n" - "add z15.h, z15.h, z24.h\n" - ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" - ".inst 0xc16115a8 // sdot za.s[x8, 0], { z13.h-z14.h }, z1.h\n" - ".inst 0xc16015a9 // sdot za.s[x8, 1], { z13.h-z14.h }, z0.h\n" - ".inst 0xa0412a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16315c8 // sdot za.s[x8, 0], { z14.h-z15.h }, z3.h\n" - ".inst 0xc16215c9 // sdot za.s[x8, 1], { z14.h-z15.h }, z2.h\n" + "trn1 z6.h, z17.h, z16.h\n" + "add z6.h, z6.h, z21.h\n" + ".inst 0xa1402a83 // ld1h { z3.h, z11.h }, pn10.b/Z, [x20]\n" + ".inst 0xc16b1488 // sdot za.s[x8, 0], { z4.h-z5.h }, z11.h\n" + ".inst 0xc1631489 // sdot za.s[x8, 1], { z4.h-z5.h }, z3.h\n" + ".inst 0xa1412a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc16814a8 // sdot za.s[x8, 0], { z5.h-z6.h }, z8.h\n" + ".inst 0xc16014a9 // sdot za.s[x8, 1], { z5.h-z6.h }, z0.h\n" "9:" // Unpadded: 1 priming loads "add x22, x14, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x14]\n" + "ld1b { z25.s }, p1/Z, [x14]\n" "addvl x21, SP, #4\n" - "ld1b { z16.s }, p1/Z, [x22]\n" + "ld1b { z6.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z13.h, z17.h, z16.h\n" - "add z13.h, z13.h, z24.h\n" - "ld1b { z17.s }, p1/Z, [x22]\n" + "trn1 z3.h, z25.h, z6.h\n" + "add z3.h, z3.h, z21.h\n" + "ld1b { z18.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "addvl x20, SP, #8\n" - "ld1b { z16.s }, p1/Z, [x22]\n" + "ld1b { z26.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z14.h, z17.h, z16.h\n" - "add z14.h, z14.h, z24.h\n" - "ld1b { z17.s }, p1/Z, [x22]\n" + "trn1 z4.h, z18.h, z26.h\n" + "add z4.h, z4.h, z21.h\n" + "ld1b { z2.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "add x14, x14, %x[ld_in_col]\n" - "ld1b { z16.s }, p1/Z, [x22]\n" - "trn1 z15.h, z17.h, z16.h\n" - "add z15.h, z15.h, z24.h\n" + "ld1b { z5.s }, p1/Z, [x22]\n" + "trn1 z5.h, z2.h, z5.h\n" + "add z5.h, z5.h, z21.h\n" ".inst 0xa0402aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21]\n" - ".inst 0xc16115a8 // sdot za.s[x8, 0], { z13.h-z14.h }, z1.h\n" - ".inst 0xc16015a9 // sdot za.s[x8, 1], { z13.h-z14.h }, z0.h\n" - ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" - ".inst 0xa0412aa2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc16115aa // sdot za.s[x8, 2], { z13.h-z14.h }, z1.h\n" - ".inst 0xc16015ab // sdot za.s[x8, 3], { z13.h-z14.h }, z0.h\n" - ".inst 0xc16315c8 // sdot za.s[x8, 0], { z14.h-z15.h }, z3.h\n" - ".inst 0xc16215c9 // sdot za.s[x8, 1], { z14.h-z15.h }, z2.h\n" - ".inst 0xa0412a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16315ca // sdot za.s[x8, 2], { z14.h-z15.h }, z3.h\n" - ".inst 0xc16215cb // sdot za.s[x8, 3], { z14.h-z15.h }, z2.h\n" + ".inst 0xc1611468 // sdot za.s[x8, 0], { z3.h-z4.h }, z1.h\n" + ".inst 0xc1601469 // sdot za.s[x8, 1], { z3.h-z4.h }, z0.h\n" + ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" + ".inst 0xa0412aa8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc16a146a // sdot za.s[x8, 2], { z3.h-z4.h }, z10.h\n" + ".inst 0xc162146b // sdot za.s[x8, 3], { z3.h-z4.h }, z2.h\n" + ".inst 0xc1691488 // sdot za.s[x8, 0], { z4.h-z5.h }, z9.h\n" + ".inst 0xc1681489 // sdot za.s[x8, 1], { z4.h-z5.h }, z8.h\n" + ".inst 0xa1412a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc16a148a // sdot za.s[x8, 2], { z4.h-z5.h }, z10.h\n" + ".inst 0xc162148b // sdot za.s[x8, 3], { z4.h-z5.h }, z2.h\n" "10:" // Unpadded: 0 priming loads - ".inst 0xa0402be0 // ld1h { z0.h-z1.h }, pn10.b/Z, [SP]\n" - ".inst 0xa0412be2 // ld1h { z2.h-z3.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + ".inst 0xa1402be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP]\n" + ".inst 0xa0412bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" "cbz x15, 18f\n" "add x20, x14, %x[ld_in_row]\n" "ld1b { z17.s }, p1/Z, [x14]\n" "sub x15, x15, #0x1\n" - "ld1b { z16.s }, p1/Z, [x20]\n" + "ld1b { z9.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z13.h, z17.h, z16.h\n" + "trn1 z6.h, z17.h, z9.h\n" "sub x13, x13, #0x1\n" "ld1b { z17.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" "cmp x15, x13\n" - "add z13.h, z13.h, z24.h\n" - "ld1b { z16.s }, p1/Z, [x20]\n" + "add z6.h, z6.h, z21.h\n" + "ld1b { z7.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z14.h, z17.h, z16.h\n" + "trn1 z7.h, z17.h, z7.h\n" "csel x23, x15, x13, LT\n" "ld1b { z17.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "add z14.h, z14.h, z24.h\n" + "add z7.h, z7.h, z21.h\n" "add x14, x14, %x[ld_in_col]\n" - "ld1b { z16.s }, p1/Z, [x20]\n" - "trn1 z15.h, z17.h, z16.h\n" - "add z15.h, z15.h, z24.h\n" + "ld1b { z1.s }, p1/Z, [x20]\n" + "trn1 z8.h, z17.h, z1.h\n" + "add z8.h, z8.h, z21.h\n" "sub x13, x13, x23\n" "cbz x23, 17f\n" "11:" // Unpadded: Main loop - ".inst 0xc16115a8 // sdot za.s[x8, 0], { z13.h-z14.h }, z1.h\n" + ".inst 0xc16d14c8 // sdot za.s[x8, 0], { z6.h-z7.h }, z13.h\n" "addvl x22, SP, #4\n" "addvl x21, SP, #8\n" - "ld1b { z21.s }, p1/Z, [x14]\n" - ".inst 0xc16015a9 // sdot za.s[x8, 1], { z13.h-z14.h }, z0.h\n" - ".inst 0xa0402ac0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x22]\n" + "ld1b { z2.s }, p1/Z, [x14]\n" + ".inst 0xc16514c9 // sdot za.s[x8, 1], { z6.h-z7.h }, z5.h\n" + ".inst 0xa1402ac5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x22]\n" "add x20, x14, %x[ld_in_row]\n" "subs x23, x23, #0x1\n" - ".inst 0xc16315c8 // sdot za.s[x8, 0], { z14.h-z15.h }, z3.h\n" - "ld1b { z20.s }, p1/Z, [x20]\n" + ".inst 0xc16b14e8 // sdot za.s[x8, 0], { z7.h-z8.h }, z11.h\n" + "ld1b { z19.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" "add x14, x14, %x[ld_in_col]\n" - ".inst 0xc16215c9 // sdot za.s[x8, 1], { z14.h-z15.h }, z2.h\n" - ".inst 0xa0412ac2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xc0060804 // mova { z4.d-z5.d }, za.d[x8, #0]\n" - "ld1b { z19.s }, p1/Z, [x20]\n" + ".inst 0xc16a14e9 // sdot za.s[x8, 1], { z7.h-z8.h }, z10.h\n" + ".inst 0xa1412ac3 // ld1h { z3.h, z11.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc0060818 // mova { z24.d-z25.d }, za.d[x8, #0]\n" + "ld1b { z23.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc0060826 // mova { z6.d-z7.d }, za.d[x8, #1]\n" - ".inst 0xc1aaac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z10.s\n" - ".inst 0xc16115aa // sdot za.s[x8, 2], { z13.h-z14.h }, z1.h\n" + ".inst 0xc006083a // mova { z26.d-z27.d }, za.d[x8, #1]\n" + ".inst 0xc1aeac18 // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z14.s\n" + ".inst 0xc16d14ca // sdot za.s[x8, 2], { z6.h-z7.h }, z13.h\n" "ld1b { z18.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc16015ab // sdot za.s[x8, 3], { z13.h-z14.h }, z0.h\n" - ".inst 0xa0402aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1abaa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z11.s\n" - ".inst 0xc16115ac // sdot za.s[x8, 4], { z13.h-z14.h }, z1.h\n" + ".inst 0xc16514cb // sdot za.s[x8, 3], { z6.h-z7.h }, z5.h\n" + ".inst 0xa1402aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21]\n" + ".inst 0xc1acaa38 // srshl { z24.s-z27.s }, { z24.s-z27.s }, z12.s\n" + ".inst 0xc16914cc // sdot za.s[x8, 4], { z6.h-z7.h }, z9.h\n" "ld1b { z17.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc16015ad // sdot za.s[x8, 5], { z13.h-z14.h }, z0.h\n" + ".inst 0xc16114cd // sdot za.s[x8, 5], { z6.h-z7.h }, z1.h\n" "ld1b { z16.s }, p1/Z, [x20]\n" - ".inst 0xc1acab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z12.s\n" - ".inst 0xc16315ca // sdot za.s[x8, 2], { z14.h-z15.h }, z3.h\n" - "trn1 z13.h, z21.h, z20.h\n" - ".inst 0xa0402be0 // ld1h { z0.h-z1.h }, pn10.b/Z, [SP]\n" - ".inst 0xc16215cb // sdot za.s[x8, 3], { z14.h-z15.h }, z2.h\n" - ".inst 0xa0412aa2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc1bacec4 // sclamp { z4.s-z7.s }, z22.s, z26.s\n" - ".inst 0xc16315cc // sdot za.s[x8, 4], { z14.h-z15.h }, z3.h\n" - "st1b { z4.s }, p1, [x11]\n" + ".inst 0xc1afab18 // add { z24.s-z27.s }, { z24.s-z27.s }, z15.s\n" + ".inst 0xc16b14ea // sdot za.s[x8, 2], { z7.h-z8.h }, z11.h\n" + "trn1 z6.h, z2.h, z19.h\n" + ".inst 0xa1402be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP]\n" + ".inst 0xc16314eb // sdot za.s[x8, 3], { z7.h-z8.h }, z3.h\n" + ".inst 0xa1412aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc1bccfb8 // sclamp { z24.s-z27.s }, z29.s, z28.s\n" + ".inst 0xc16914ec // sdot za.s[x8, 4], { z7.h-z8.h }, z9.h\n" + "st1b { z24.s }, p1, [x11]\n" "add x11, x11, x9\n" - "add z13.h, z13.h, z24.h\n" - ".inst 0xc16215cd // sdot za.s[x8, 5], { z14.h-z15.h }, z2.h\n" - "trn1 z14.h, z19.h, z18.h\n" - "trn1 z15.h, z17.h, z16.h\n" + "add z6.h, z6.h, z21.h\n" + ".inst 0xc16114ed // sdot za.s[x8, 5], { z7.h-z8.h }, z1.h\n" + "trn1 z7.h, z23.h, z18.h\n" + "trn1 z8.h, z17.h, z16.h\n" "add x8, x8, #0x2\n" - ".inst 0xa0412be2 // ld1h { z2.h-z3.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" - "st1b { z6.s }, p1, [x10]\n" + ".inst 0xa0412bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + "st1b { z26.s }, p1, [x10]\n" "add x10, x10, x28\n" - ".inst 0xc0040904 // mova za.d[x8, #4], { z8.d-z9.d }\n" - "st1b { z5.s }, p1, [x27]\n" + ".inst 0xc0040bc4 // mova za.d[x8, #4], { z30.d-z31.d }\n" + "st1b { z25.s }, p1, [x27]\n" "add x27, x27, x25\n" - ".inst 0xc0040905 // mova za.d[x8, #5], { z8.d-z9.d }\n" - "add z14.h, z14.h, z24.h\n" - "st1b { z7.s }, p1, [x26]\n" + ".inst 0xc0040bc5 // mova za.d[x8, #5], { z30.d-z31.d }\n" + "add z7.h, z7.h, z21.h\n" + "st1b { z27.s }, p1, [x26]\n" "add x26, x26, x24\n" - "add z15.h, z15.h, z24.h\n" + "add z8.h, z8.h, z21.h\n" "bgt 11b\n" "b 17f\n" "12:" // Padded @@ -384,118 +384,118 @@ void sme2_u8s8u8q_planar_3x3_s1_4rows_dot_za_impl( "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1b { z19.s }, p0/Z, [x14]\n" - "add z19.h, p0/M, z19.h, z24.h\n" + "add z19.h, p0/M, z19.h, z21.h\n" "add x20, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z24.h\n" + "add z18.h, p0/M, z18.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z24.h\n" + "add z17.h, p0/M, z17.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z24.h\n" + "add z16.h, p0/M, z16.h, z21.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z13.h, z19.h, z18.h\n" - "trn1 z14.h, z17.h, z16.h\n" + "trn1 z7.h, z19.h, z18.h\n" + "trn1 z8.h, z17.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z24.h\n" + "add z17.h, p0/M, z17.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1b { z16.s }, p0/Z, [x20]\n" "addvl x20, SP, #8\n" - "add z16.h, p0/M, z16.h, z24.h\n" - ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" - "trn1 z15.h, z17.h, z16.h\n" - ".inst 0xc16115a8 // sdot za.s[x8, 0], { z13.h-z14.h }, z1.h\n" + "add z16.h, p0/M, z16.h, z21.h\n" + ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" + "trn1 z9.h, z17.h, z16.h\n" + ".inst 0xc16a14e8 // sdot za.s[x8, 0], { z7.h-z8.h }, z10.h\n" "add x14, x14, %x[ld_in_col]\n" - ".inst 0xc16015a9 // sdot za.s[x8, 1], { z13.h-z14.h }, z0.h\n" - ".inst 0xa0412a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16315c8 // sdot za.s[x8, 0], { z14.h-z15.h }, z3.h\n" - ".inst 0xc16215c9 // sdot za.s[x8, 1], { z14.h-z15.h }, z2.h\n" + ".inst 0xc16214e9 // sdot za.s[x8, 1], { z7.h-z8.h }, z2.h\n" + ".inst 0xa1412a85 // ld1h { z5.h, z13.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc16d1508 // sdot za.s[x8, 0], { z8.h-z9.h }, z13.h\n" + ".inst 0xc1651509 // sdot za.s[x8, 1], { z8.h-z9.h }, z5.h\n" "14:" // Padded: 1 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1b { z19.s }, p0/Z, [x14]\n" - "add z19.h, p0/M, z19.h, z24.h\n" + "add z19.h, p0/M, z19.h, z21.h\n" "add x20, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z24.h\n" + "add z18.h, p0/M, z18.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z24.h\n" + "add z17.h, p0/M, z17.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z24.h\n" + "add z16.h, p0/M, z16.h, z21.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z13.h, z19.h, z18.h\n" - "trn1 z14.h, z17.h, z16.h\n" + "trn1 z22.h, z19.h, z18.h\n" + "trn1 z23.h, z17.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z24.h\n" + "add z17.h, p0/M, z17.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1b { z16.s }, p0/Z, [x20]\n" "addvl x21, SP, #4\n" - "add z16.h, p0/M, z16.h, z24.h\n" + "add z16.h, p0/M, z16.h, z21.h\n" ".inst 0xa0402aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21]\n" "addvl x20, SP, #8\n" - "trn1 z15.h, z17.h, z16.h\n" - ".inst 0xc16115a8 // sdot za.s[x8, 0], { z13.h-z14.h }, z1.h\n" - ".inst 0xc16015a9 // sdot za.s[x8, 1], { z13.h-z14.h }, z0.h\n" - ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" + "trn1 z24.h, z17.h, z16.h\n" + ".inst 0xc16116c8 // sdot za.s[x8, 0], { z22.h-z23.h }, z1.h\n" + ".inst 0xc16016c9 // sdot za.s[x8, 1], { z22.h-z23.h }, z0.h\n" + ".inst 0xa1402a85 // ld1h { z5.h, z13.h }, pn10.b/Z, [x20]\n" "add x14, x14, %x[ld_in_col]\n" - ".inst 0xa0412aa2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc16115aa // sdot za.s[x8, 2], { z13.h-z14.h }, z1.h\n" - ".inst 0xc16015ab // sdot za.s[x8, 3], { z13.h-z14.h }, z0.h\n" - ".inst 0xc16315c8 // sdot za.s[x8, 0], { z14.h-z15.h }, z3.h\n" - ".inst 0xc16215c9 // sdot za.s[x8, 1], { z14.h-z15.h }, z2.h\n" - ".inst 0xa0412a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16315ca // sdot za.s[x8, 2], { z14.h-z15.h }, z3.h\n" - ".inst 0xc16215cb // sdot za.s[x8, 3], { z14.h-z15.h }, z2.h\n" + ".inst 0xa0412aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc16d16ca // sdot za.s[x8, 2], { z22.h-z23.h }, z13.h\n" + ".inst 0xc16516cb // sdot za.s[x8, 3], { z22.h-z23.h }, z5.h\n" + ".inst 0xc16116e8 // sdot za.s[x8, 0], { z23.h-z24.h }, z1.h\n" + ".inst 0xc16016e9 // sdot za.s[x8, 1], { z23.h-z24.h }, z0.h\n" + ".inst 0xa0412a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc16116ea // sdot za.s[x8, 2], { z23.h-z24.h }, z1.h\n" + ".inst 0xc16016eb // sdot za.s[x8, 3], { z23.h-z24.h }, z0.h\n" "15:" // Padded: 0 priming loads - ".inst 0xa0402be0 // ld1h { z0.h-z1.h }, pn10.b/Z, [SP]\n" - ".inst 0xa0412be2 // ld1h { z2.h-z3.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + ".inst 0xa1402be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP]\n" + ".inst 0xa0412bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" "cbz x15, 18f\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1b { z19.s }, p0/Z, [x14]\n" - "add z19.h, p0/M, z19.h, z24.h\n" + "add z19.h, p0/M, z19.h, z21.h\n" "add x20, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z24.h\n" + "add z18.h, p0/M, z18.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z24.h\n" + "add z17.h, p0/M, z17.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z24.h\n" + "add z16.h, p0/M, z16.h, z21.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z13.h, z19.h, z18.h\n" - "trn1 z14.h, z17.h, z16.h\n" + "trn1 z6.h, z19.h, z18.h\n" + "trn1 z7.h, z17.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z24.h\n" + "add z17.h, p0/M, z17.h, z21.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z24.h\n" + "add z16.h, p0/M, z16.h, z21.h\n" "sub x15, x15, #0x1\n" "sub x13, x13, #0x1\n" "cmp x15, x13\n" - "trn1 z15.h, z17.h, z16.h\n" + "trn1 z8.h, z17.h, z16.h\n" "csel x23, x15, x13, LT\n" "add x14, x14, %x[ld_in_col]\n" "sub x13, x13, x23\n" @@ -503,121 +503,121 @@ void sme2_u8s8u8q_planar_3x3_s1_4rows_dot_za_impl( "16:" // Padded: Main loop "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z21.s }, p0/Z, [x14]\n" - ".inst 0xc16115a8 // sdot za.s[x8, 0], { z13.h-z14.h }, z1.h\n" - ".inst 0xc16015a9 // sdot za.s[x8, 1], { z13.h-z14.h }, z0.h\n" - "add z21.h, p0/M, z21.h, z24.h\n" + "ld1b { z9.s }, p0/Z, [x14]\n" + ".inst 0xc16d14c8 // sdot za.s[x8, 0], { z6.h-z7.h }, z13.h\n" + ".inst 0xc16514c9 // sdot za.s[x8, 1], { z6.h-z7.h }, z5.h\n" + "add z9.h, p0/M, z9.h, z21.h\n" "add x22, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z20.s }, p0/Z, [x22]\n" - ".inst 0xc16315c8 // sdot za.s[x8, 0], { z14.h-z15.h }, z3.h\n" - "add z20.h, p0/M, z20.h, z24.h\n" + "ld1b { z19.s }, p0/Z, [x22]\n" + ".inst 0xc16b14e8 // sdot za.s[x8, 0], { z7.h-z8.h }, z11.h\n" + "add z19.h, p0/M, z19.h, z21.h\n" "add x22, x22, %x[ld_in_row]\n" - ".inst 0xc16215c9 // sdot za.s[x8, 1], { z14.h-z15.h }, z2.h\n" + ".inst 0xc16a14e9 // sdot za.s[x8, 1], { z7.h-z8.h }, z10.h\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z19.s }, p0/Z, [x22]\n" - "add z19.h, p0/M, z19.h, z24.h\n" - ".inst 0xc0060804 // mova { z4.d-z5.d }, za.d[x8, #0]\n" + "ld1b { z18.s }, p0/Z, [x22]\n" + "add z18.h, p0/M, z18.h, z21.h\n" + ".inst 0xc0060818 // mova { z24.d-z25.d }, za.d[x8, #0]\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z18.s }, p0/Z, [x22]\n" - ".inst 0xc0060826 // mova { z6.d-z7.d }, za.d[x8, #1]\n" + "ld1b { z16.s }, p0/Z, [x22]\n" + ".inst 0xc006083a // mova { z26.d-z27.d }, za.d[x8, #1]\n" "mov x12, #0x4\n" "addvl x21, SP, #4\n" - "add z18.h, p0/M, z18.h, z24.h\n" - ".inst 0xc1aaac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z10.s\n" + "add z16.h, p0/M, z16.h, z21.h\n" + ".inst 0xc1aeac18 // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z14.s\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xa0402aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21]\n" + ".inst 0xa1402aa3 // ld1h { z3.h, z11.h }, pn10.b/Z, [x21]\n" "addvl x20, SP, #8\n" - ".inst 0xc16115aa // sdot za.s[x8, 2], { z13.h-z14.h }, z1.h\n" + ".inst 0xc16b14ca // sdot za.s[x8, 2], { z6.h-z7.h }, z11.h\n" "subs x23, x23, #0x1\n" "ld1b { z17.s }, p0/Z, [x22]\n" - ".inst 0xc16015ab // sdot za.s[x8, 3], { z13.h-z14.h }, z0.h\n" - ".inst 0xc1abaa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z11.s\n" - ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z24.h\n" + ".inst 0xc16314cb // sdot za.s[x8, 3], { z6.h-z7.h }, z3.h\n" + ".inst 0xc1acaa38 // srshl { z24.s-z27.s }, { z24.s-z27.s }, z12.s\n" + ".inst 0xa1402a85 // ld1h { z5.h, z13.h }, pn10.b/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z21.h\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - ".inst 0xa0412aa2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc16115ac // sdot za.s[x8, 4], { z13.h-z14.h }, z1.h\n" - ".inst 0xc1acab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z12.s\n" - "ld1b { z16.s }, p0/Z, [x22]\n" - ".inst 0xc16015ad // sdot za.s[x8, 5], { z13.h-z14.h }, z0.h\n" - "add z16.h, p0/M, z16.h, z24.h\n" + ".inst 0xa0412aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc16d14cc // sdot za.s[x8, 4], { z6.h-z7.h }, z13.h\n" + ".inst 0xc1afab18 // add { z24.s-z27.s }, { z24.s-z27.s }, z15.s\n" + "ld1b { z2.s }, p0/Z, [x22]\n" + ".inst 0xc16514cd // sdot za.s[x8, 5], { z6.h-z7.h }, z5.h\n" + "add z2.h, p0/M, z2.h, z21.h\n" "add x14, x14, %x[ld_in_col]\n" - ".inst 0xc16315ca // sdot za.s[x8, 2], { z14.h-z15.h }, z3.h\n" - ".inst 0xa0402be0 // ld1h { z0.h-z1.h }, pn10.b/Z, [SP]\n" - ".inst 0xc1bacec4 // sclamp { z4.s-z7.s }, z22.s, z26.s\n" - ".inst 0xc16215cb // sdot za.s[x8, 3], { z14.h-z15.h }, z2.h\n" - ".inst 0xa0412a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - "st1b { z4.s }, p1, [x11]\n" + ".inst 0xc16b14ea // sdot za.s[x8, 2], { z7.h-z8.h }, z11.h\n" + ".inst 0xa1402be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP]\n" + ".inst 0xc1bccfb8 // sclamp { z24.s-z27.s }, z29.s, z28.s\n" + ".inst 0xc16a14eb // sdot za.s[x8, 3], { z7.h-z8.h }, z10.h\n" + ".inst 0xa1412a83 // ld1h { z3.h, z11.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + "st1b { z24.s }, p1, [x11]\n" "add x11, x11, x9\n" - ".inst 0xc16315cc // sdot za.s[x8, 4], { z14.h-z15.h }, z3.h\n" - "st1b { z6.s }, p1, [x10]\n" + ".inst 0xc16b14ec // sdot za.s[x8, 4], { z7.h-z8.h }, z11.h\n" + "st1b { z26.s }, p1, [x10]\n" "add x10, x10, x28\n" - "trn1 z13.h, z21.h, z20.h\n" - ".inst 0xc16215cd // sdot za.s[x8, 5], { z14.h-z15.h }, z2.h\n" + "trn1 z6.h, z9.h, z19.h\n" + ".inst 0xc16314ed // sdot za.s[x8, 5], { z7.h-z8.h }, z3.h\n" "add x8, x8, #0x2\n" - ".inst 0xa0412be2 // ld1h { z2.h-z3.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" - "st1b { z5.s }, p1, [x27]\n" + ".inst 0xa0412bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + "st1b { z25.s }, p1, [x27]\n" "add x27, x27, x25\n" - "st1b { z7.s }, p1, [x26]\n" + "st1b { z27.s }, p1, [x26]\n" "add x26, x26, x24\n" - ".inst 0xc0040904 // mova za.d[x8, #4], { z8.d-z9.d }\n" - ".inst 0xc0040905 // mova za.d[x8, #5], { z8.d-z9.d }\n" - "trn1 z14.h, z19.h, z18.h\n" - "trn1 z15.h, z17.h, z16.h\n" + ".inst 0xc0040bc4 // mova za.d[x8, #4], { z30.d-z31.d }\n" + ".inst 0xc0040bc5 // mova za.d[x8, #5], { z30.d-z31.d }\n" + "trn1 z7.h, z18.h, z16.h\n" + "trn1 z8.h, z17.h, z2.h\n" "bgt 16b\n" "17:" // Main loop tail - ".inst 0xc16115a8 // sdot za.s[x8, 0], { z13.h-z14.h }, z1.h\n" + ".inst 0xc16d14c8 // sdot za.s[x8, 0], { z6.h-z7.h }, z13.h\n" "addvl x21, SP, #4\n" "addvl x20, SP, #8\n" - ".inst 0xc16015a9 // sdot za.s[x8, 1], { z13.h-z14.h }, z0.h\n" + ".inst 0xc16514c9 // sdot za.s[x8, 1], { z6.h-z7.h }, z5.h\n" ".inst 0xa0402aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21]\n" - ".inst 0xc16315c8 // sdot za.s[x8, 0], { z14.h-z15.h }, z3.h\n" - ".inst 0xc16215c9 // sdot za.s[x8, 1], { z14.h-z15.h }, z2.h\n" + ".inst 0xc16b14e8 // sdot za.s[x8, 0], { z7.h-z8.h }, z11.h\n" + ".inst 0xc16a14e9 // sdot za.s[x8, 1], { z7.h-z8.h }, z10.h\n" ".inst 0xa0412aa2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc0060804 // mova { z4.d-z5.d }, za.d[x8, #0]\n" - ".inst 0xc0060826 // mova { z6.d-z7.d }, za.d[x8, #1]\n" - ".inst 0xc1aaac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z10.s\n" - ".inst 0xc16115aa // sdot za.s[x8, 2], { z13.h-z14.h }, z1.h\n" - ".inst 0xc1abaa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z11.s\n" - ".inst 0xc16015ab // sdot za.s[x8, 3], { z13.h-z14.h }, z0.h\n" - ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1acab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z12.s\n" - ".inst 0xc16115ac // sdot za.s[x8, 4], { z13.h-z14.h }, z1.h\n" - ".inst 0xc1bacec4 // sclamp { z4.s-z7.s }, z22.s, z26.s\n" - "st1b { z4.s }, p1, [x11]\n" + ".inst 0xc0060818 // mova { z24.d-z25.d }, za.d[x8, #0]\n" + ".inst 0xc006083a // mova { z26.d-z27.d }, za.d[x8, #1]\n" + ".inst 0xc1aeac18 // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z14.s\n" + ".inst 0xc16114ca // sdot za.s[x8, 2], { z6.h-z7.h }, z1.h\n" + ".inst 0xc1acaa38 // srshl { z24.s-z27.s }, { z24.s-z27.s }, z12.s\n" + ".inst 0xc16014cb // sdot za.s[x8, 3], { z6.h-z7.h }, z0.h\n" + ".inst 0xa1402a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1afab18 // add { z24.s-z27.s }, { z24.s-z27.s }, z15.s\n" + ".inst 0xc16914cc // sdot za.s[x8, 4], { z6.h-z7.h }, z9.h\n" + ".inst 0xc1bccfb8 // sclamp { z24.s-z27.s }, z29.s, z28.s\n" + "st1b { z24.s }, p1, [x11]\n" "add x11, x11, x9\n" - ".inst 0xc16015ad // sdot za.s[x8, 5], { z13.h-z14.h }, z0.h\n" - "st1b { z6.s }, p1, [x10]\n" + ".inst 0xc16114cd // sdot za.s[x8, 5], { z6.h-z7.h }, z1.h\n" + "st1b { z26.s }, p1, [x10]\n" "add x10, x10, x28\n" - ".inst 0xc16315ca // sdot za.s[x8, 2], { z14.h-z15.h }, z3.h\n" - "st1b { z5.s }, p1, [x27]\n" + ".inst 0xc16314ea // sdot za.s[x8, 2], { z7.h-z8.h }, z3.h\n" + "st1b { z25.s }, p1, [x27]\n" "add x27, x27, x25\n" - ".inst 0xc16215cb // sdot za.s[x8, 3], { z14.h-z15.h }, z2.h\n" - ".inst 0xa0412a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - "st1b { z7.s }, p1, [x26]\n" + ".inst 0xc16214eb // sdot za.s[x8, 3], { z7.h-z8.h }, z2.h\n" + ".inst 0xa0412a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + "st1b { z27.s }, p1, [x26]\n" "add x26, x26, x24\n" - ".inst 0xc16315cc // sdot za.s[x8, 4], { z14.h-z15.h }, z3.h\n" - ".inst 0xc16215cd // sdot za.s[x8, 5], { z14.h-z15.h }, z2.h\n" + ".inst 0xc16114ec // sdot za.s[x8, 4], { z7.h-z8.h }, z1.h\n" + ".inst 0xc16014ed // sdot za.s[x8, 5], { z7.h-z8.h }, z0.h\n" "add x8, x8, #0x2\n" - ".inst 0xc0040904 // mova za.d[x8, #4], { z8.d-z9.d }\n" - ".inst 0xc0040905 // mova za.d[x8, #5], { z8.d-z9.d }\n" + ".inst 0xc0040bc4 // mova za.d[x8, #4], { z30.d-z31.d }\n" + ".inst 0xc0040bc5 // mova za.d[x8, #5], { z30.d-z31.d }\n" "18:" // Main loop skip tail "cbz x13, 20f\n" "19:" // Right padding loop ".inst 0xc0060804 // mova { z4.d-z5.d }, za.d[x8, #0]\n" "subs x13, x13, #0x1\n" ".inst 0xc0060826 // mova { z6.d-z7.d }, za.d[x8, #1]\n" - ".inst 0xc1aaac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z10.s\n" + ".inst 0xc1aeac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z14.s\n" "add x8, x8, #0x2\n" - ".inst 0xc1abaa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z11.s\n" - ".inst 0xc0040904 // mova za.d[x8, #4], { z8.d-z9.d }\n" - ".inst 0xc1acab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z12.s\n" - ".inst 0xc0040905 // mova za.d[x8, #5], { z8.d-z9.d }\n" - ".inst 0xc1bacec4 // sclamp { z4.s-z7.s }, z22.s, z26.s\n" + ".inst 0xc1acaa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z12.s\n" + ".inst 0xc0040bc4 // mova za.d[x8, #4], { z30.d-z31.d }\n" + ".inst 0xc1afab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z15.s\n" + ".inst 0xc0040bc5 // mova za.d[x8, #5], { z30.d-z31.d }\n" + ".inst 0xc1bccfa4 // sclamp { z4.s-z7.s }, z29.s, z28.s\n" "st1b { z4.s }, p1, [x11]\n" "add x11, x11, x9\n" "st1b { z6.s }, p1, [x10]\n" @@ -628,15 +628,15 @@ void sme2_u8s8u8q_planar_3x3_s1_4rows_dot_za_impl( "add x26, x26, x24\n" "bgt 19b\n" "20:" // End - "ldr x22, [%x[args], %[offsetof_Args_weights]]\n" - "incw x22, ALL, MUL #9\n" - "str x22, [%x[args], %[offsetof_Args_weights]]\n" + "ldr x20, [%x[args], %[offsetof_Args_weights]]\n" + "incw x20, ALL, MUL #9\n" + "str x20, [%x[args], %[offsetof_Args_weights]]\n" "incw x16\n" - "ldr x20, [%x[args], %[offsetof_Args_ld_in_vl]]\n" + "ldr x21, [%x[args], %[offsetof_Args_ld_in_vl]]\n" "whilelt p1.s, x16, x17\n" - "ldr x14, [%x[args], %[offsetof_Args_inptr]]\n" - "add x14, x14, x20\n" - "str x14, [%x[args], %[offsetof_Args_inptr]]\n" + "ldr x20, [%x[args], %[offsetof_Args_inptr]]\n" + "add x20, x20, x21\n" + "str x20, [%x[args], %[offsetof_Args_inptr]]\n" "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" "ldr x24, [%x[args], %[offsetof_Args_ld_out_vls]]\n" "ldp x23, x22, [x25, #0x0]\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_3x3_s2_4rows_dot_za.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_3x3_s2_4rows_dot_za.hpp index e412216af33eb71c0e4dcfe3c00b11d37fc78c72..104c11fc9d4900699b17d77e0b331c0b23f5ae50 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_3x3_s2_4rows_dot_za.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_3x3_s2_4rows_dot_za.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,8 @@ * SOFTWARE. */ +#include "src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp" + namespace arm_conv { namespace depthwise { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_3x3_s2_4rows_dot_za/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_3x3_s2_4rows_dot_za/generic.cpp index 2848a015db7ee083f327096c75557fa4e763dcf6..8ce04fb8c2e71c05b8b95a99d2d863556953bcba 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_3x3_s2_4rows_dot_za/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_3x3_s2_4rows_dot_za/generic.cpp @@ -73,86 +73,86 @@ void sme2_u8s8u8q_planar_3x3_s2_4rows_dot_za_impl( "ptrue p2.b\n" "mov x20, #0x9\n" "ldr x7, [%x[args], %[offsetof_Args_pad_top]]\n" - "ld1rh { z5.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_a_offset]]\n" + "ld1rh { z11.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_a_offset]]\n" "sub x20, x20, x6\n" ".inst 0x25207812 // ptrue pn10.b\n" "ldr x17, [%x[args], %[offsetof_Args_n_channels]]\n" "whilelt p1.s, XZR, x17\n" "whilelt p9.s, XZR, x20\n" - "ld1rw { z4.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" + "ld1rw { z13.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" "whilelt p8.s, XZR, x7\n" "addvl SP, SP, #-6\n" "ldr x16, [%x[args], %[offsetof_Args_current_channel]]\n" - "neg z5.h, p2/M, z5.h\n" + "neg z11.h, p2/M, z11.h\n" "eor p8.b, p2/Z, p8.b, p9.b\n" - "ld1rw { z8.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_mul]]\n" - "ld1rw { z7.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_right_shift]]\n" - "ld1rw { z27.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" - "ld1rw { z23.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" + "ld1rw { z6.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z9.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z10.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" + "ld1rw { z7.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" "1:" // Channel loop "ldr x20, [%x[qp], %[offsetof_Requantize32_bias]]\n" - "mov z0.s, #0x0\n" + "mov z28.s, #0x0\n" "cbz x20, 2f\n" - "ld1w { z0.s }, p1/Z, [x20, x16, LSL #2]\n" + "ld1w { z28.s }, p1/Z, [x20, x16, LSL #2]\n" "2:" // Load bias: Done "ldr x22, [%x[args], %[offsetof_Args_weights]]\n" "mov x20, x22\n" - "ld1sb { z24.s }, p2/Z, [x20]\n" + "ld1sb { z26.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "ld1rh { z13.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" - "sub z24.h, z24.h, z13.h\n" + "ld1rh { z16.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" + "sub z26.h, z26.h, z16.h\n" "incw x22\n" - "mov z17.h, #0x0\n" - "ld1sb { z25.s }, p2/Z, [x20]\n" + "mov z24.h, #0x0\n" + "ld1sb { z3.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "sub z25.h, z25.h, z13.h\n" - "trn1 z10.h, z24.h, z25.h\n" - "ld1sb { z16.s }, p2/Z, [x20]\n" - "sub z16.h, z16.h, z13.h\n" + "sub z3.h, z3.h, z16.h\n" + "trn1 z31.h, z26.h, z3.h\n" + "ld1sb { z21.s }, p2/Z, [x20]\n" + "sub z21.h, z21.h, z16.h\n" "mov x20, x22\n" - "trn1 z11.h, z16.h, z17.h\n" - "ld1sb { z24.s }, p2/Z, [x20]\n" + "trn1 z14.h, z21.h, z24.h\n" + "ld1sb { z2.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "sub z24.h, z24.h, z13.h\n" + "sub z2.h, z2.h, z16.h\n" "addvl x21, SP, #6\n" "ld1sb { z25.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "sub z25.h, z25.h, z13.h\n" + "sub z25.h, z25.h, z16.h\n" "incw x22\n" - "ld1sb { z16.s }, p2/Z, [x20]\n" - "sub z16.h, z16.h, z13.h\n" + "ld1sb { z27.s }, p2/Z, [x20]\n" + "sub z27.h, z27.h, z16.h\n" "addvl x21, x21, #-2\n" "mov x20, x22\n" - "st1h { z10.h }, p2, [x21]\n" - "trn1 z10.h, z24.h, z25.h\n" - "ld1sb { z24.s }, p2/Z, [x20]\n" + "st1h { z31.h }, p2, [x21]\n" + "trn1 z4.h, z2.h, z25.h\n" + "ld1sb { z26.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "ld1sb { z25.s }, p2/Z, [x20]\n" + "ld1sb { z23.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #3\n" - "st1h { z11.h }, p2, [x21, #1, MUL VL]\n" - "trn1 z11.h, z16.h, z17.h\n" - "ld1sb { z16.s }, p2/Z, [x20]\n" - "sub z24.h, z24.h, z13.h\n" - "sub z25.h, z25.h, z13.h\n" + "st1h { z14.h }, p2, [x21, #1, MUL VL]\n" + "trn1 z12.h, z27.h, z24.h\n" + "ld1sb { z20.s }, p2/Z, [x20]\n" + "sub z26.h, z26.h, z16.h\n" + "sub z23.h, z23.h, z16.h\n" "ldr x20, [%x[qp], %[offsetof_Requantize32_per_channel_muls]]\n" - "sub z16.h, z16.h, z13.h\n" + "sub z20.h, z20.h, z16.h\n" "addvl x21, x21, #-2\n" - "st1h { z10.h }, p2, [x21]\n" - "mov z1.d, z0.d\n" - "st1h { z11.h }, p2, [x21, #1, MUL VL]\n" + "st1h { z4.h }, p2, [x21]\n" + "mov z29.d, z28.d\n" + "st1h { z12.h }, p2, [x21, #1, MUL VL]\n" "addvl x21, x21, #-2\n" - "mov z2.d, z0.d\n" - "mov z3.d, z0.d\n" - "trn1 z10.h, z24.h, z25.h\n" - "st1h { z10.h }, p2, [x21]\n" - "trn1 z11.h, z16.h, z17.h\n" - "st1h { z11.h }, p2, [x21, #1, MUL VL]\n" + "mov z30.d, z28.d\n" + "mov z31.d, z28.d\n" + "trn1 z25.h, z26.h, z23.h\n" + "st1h { z25.h }, p2, [x21]\n" + "trn1 z3.h, z20.h, z24.h\n" + "st1h { z3.h }, p2, [x21, #1, MUL VL]\n" "cbz x20, 3f\n" - "ld1w { z8.s }, p1/Z, [x20, x16, LSL #2]\n" + "ld1w { z6.s }, p1/Z, [x20, x16, LSL #2]\n" "3:" // Load mul: End "ldr x20, [%x[qp], %[offsetof_Requantize32_per_channel_right_shifts]]\n" "cbz x20, 4f\n" - "ld1w { z7.s }, p1/Z, [x20, x16, LSL #2]\n" + "ld1w { z9.s }, p1/Z, [x20, x16, LSL #2]\n" "4:" // Load right_shift: End "ldr x15, [%x[args], %[offsetof_Args_input_cols]]\n" "sub x20, x15, #0x1\n" @@ -172,18 +172,18 @@ void sme2_u8s8u8q_planar_3x3_s2_4rows_dot_za_impl( ".inst 0xf8b74a9c // rprfm pldstrm, x23, [x20]\n" "add x20, x20, %x[ld_in_col]\n" "bgt 5b\n" - "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" + "ldr x23, [%x[args], %[offsetof_Args_outptrs]]\n" "lsl x20, %x[ld_in_row], #0x0\n" "msub x14, x7, x20, x14\n" - ".inst 0xc0040c00 // mova za.d[x8, #0], { z0.d-z3.d }\n" + ".inst 0xc0040f80 // mova za.d[x8, #0], { z28.d-z31.d }\n" "ldr x20, [%x[args], %[offsetof_Args_ld_out_cols]]\n" - ".inst 0xc0040c01 // mova za.d[x8, #1], { z0.d-z3.d }\n" + ".inst 0xc0040f81 // mova za.d[x8, #1], { z28.d-z31.d }\n" "mov x22, #0x2\n" - "ldp x11, x10, [x25], #0x10\n" - ".inst 0xc0040c02 // mova za.d[x8, #2], { z0.d-z3.d }\n" + "ldp x11, x10, [x23], #0x10\n" + ".inst 0xc0040f82 // mova za.d[x8, #2], { z28.d-z31.d }\n" "ldp x9, x28, [x20], #0x10\n" "ldr x21, [%x[args], %[offsetof_Args_pad_left]]\n" - "ldp x27, x26, [x25], #0x10\n" + "ldp x27, x26, [x23], #0x10\n" "ldp x25, x24, [x20], #0x10\n" "cbz x21, 7f\n" "cmp x21, x22\n" @@ -191,24 +191,24 @@ void sme2_u8s8u8q_planar_3x3_s2_4rows_dot_za_impl( "sub x21, x21, x20\n" "sub x22, x22, x20\n" "cbz x21, 7f\n" - ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" - ".inst 0xc1a8ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z8.s\n" + ".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n" + ".inst 0xc1a6ac18 // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z6.s\n" "and x22, x21, #0x1\n" - ".inst 0xc1a7aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z7.s\n" + ".inst 0xc1a9aa38 // srshl { z24.s-z27.s }, { z24.s-z27.s }, z9.s\n" "add x21, x21, #0x1\n" "lsr x21, x21, #0x1\n" - ".inst 0xc1a4ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" + ".inst 0xc1adab18 // add { z24.s-z27.s }, { z24.s-z27.s }, z13.s\n" "sub x13, x13, x21\n" - ".inst 0xc1b7cf7c // sclamp { z28.s-z31.s }, z27.s, z23.s\n" + ".inst 0xc1a7cd58 // sclamp { z24.s-z27.s }, z10.s, z7.s\n" "6:" // Left padding "subs x21, x21, #0x1\n" - "st1b { z28.s }, p1, [x11]\n" + "st1b { z24.s }, p1, [x11]\n" "add x11, x11, x9\n" - "st1b { z29.s }, p1, [x10]\n" + "st1b { z25.s }, p1, [x10]\n" "add x10, x10, x28\n" - "st1b { z30.s }, p1, [x27]\n" + "st1b { z26.s }, p1, [x27]\n" "add x27, x27, x25\n" - "st1b { z31.s }, p1, [x26]\n" + "st1b { z27.s }, p1, [x26]\n" "add x26, x26, x24\n" "bgt 6b\n" "7:" // Left padding: End @@ -220,194 +220,194 @@ void sme2_u8s8u8q_planar_3x3_s2_4rows_dot_za_impl( "beq 9f\n" "8:" // Unpadded: 2 priming loads "add x21, x14, %x[ld_in_row]\n" - "ld1b { z12.s }, p1/Z, [x14]\n" + "ld1b { z1.s }, p1/Z, [x14]\n" "addvl x20, SP, #4\n" - "ld1b { z20.s }, p1/Z, [x21]\n" + "ld1b { z21.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "add z12.h, z12.h, z5.h\n" - "ld1b { z13.s }, p1/Z, [x21]\n" + "trn1 z1.h, z1.h, z21.h\n" + "add z1.h, z1.h, z11.h\n" + "ld1b { z2.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "add x14, x14, %x[ld_in_col]\n" - "ld1b { z19.s }, p1/Z, [x21]\n" + "ld1b { z15.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z5.h\n" - "ld1b { z14.s }, p1/Z, [x21]\n" + "trn1 z2.h, z2.h, z15.h\n" + "add z2.h, z2.h, z11.h\n" + "ld1b { z3.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1b { z18.s }, p1/Z, [x21]\n" + "ld1b { z21.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z5.h\n" - "ld1b { z15.s }, p1/Z, [x21]\n" + "trn1 z3.h, z3.h, z21.h\n" + "add z3.h, z3.h, z11.h\n" + "ld1b { z4.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x21]\n" + "ld1b { z19.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z15.h, z15.h, z5.h\n" - "ld1b { z16.s }, p1/Z, [x21]\n" - "mov z16.d, z16.d\n" - "add z16.h, z16.h, z5.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" + "trn1 z4.h, z4.h, z19.h\n" + "add z4.h, z4.h, z11.h\n" + "ld1b { z8.s }, p1/Z, [x21]\n" + "mov z5.d, z8.d\n" + "add z5.h, z5.h, z11.h\n" + ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1701428 // sdot za.s[x8, 0], { z1.h-z4.h }, z0.h\n" + ".inst 0xc1781448 // sdot za.s[x8, 0], { z2.h-z5.h }, z8.h\n" "9:" // Unpadded: 1 priming loads "add x21, x14, %x[ld_in_row]\n" - "ld1b { z12.s }, p1/Z, [x14]\n" + "ld1b { z1.s }, p1/Z, [x14]\n" "addvl x20, SP, #2\n" - "ld1b { z20.s }, p1/Z, [x21]\n" + "ld1b { z21.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "add z12.h, z12.h, z5.h\n" - "ld1b { z13.s }, p1/Z, [x21]\n" + "trn1 z1.h, z1.h, z21.h\n" + "add z1.h, z1.h, z11.h\n" + "ld1b { z2.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "add x14, x14, %x[ld_in_col]\n" - "ld1b { z19.s }, p1/Z, [x21]\n" + "ld1b { z12.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z5.h\n" - "ld1b { z14.s }, p1/Z, [x21]\n" + "trn1 z2.h, z2.h, z12.h\n" + "add z2.h, z2.h, z11.h\n" + "ld1b { z3.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1b { z18.s }, p1/Z, [x21]\n" + "ld1b { z8.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z5.h\n" - "ld1b { z15.s }, p1/Z, [x21]\n" + "trn1 z3.h, z3.h, z8.h\n" + "add z3.h, z3.h, z11.h\n" + "ld1b { z4.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x21]\n" + "ld1b { z5.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z15.h, z15.h, z5.h\n" - "ld1b { z16.s }, p1/Z, [x21]\n" - "mov z16.d, z16.d\n" - "add z16.h, z16.h, z5.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" + "trn1 z4.h, z4.h, z5.h\n" + "add z4.h, z4.h, z11.h\n" + "ld1b { z5.s }, p1/Z, [x21]\n" + "mov z5.d, z5.d\n" + "add z5.h, z5.h, z11.h\n" + ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1701428 // sdot za.s[x8, 0], { z1.h-z4.h }, z0.h\n" + ".inst 0xc1781448 // sdot za.s[x8, 0], { z2.h-z5.h }, z8.h\n" "10:" // Unpadded: 0 priming loads "cmp x15, #0x2\n" - ".inst 0xa0402bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP]\n" + ".inst 0xa0402bee // ld1h { z14.h-z15.h }, pn10.b/Z, [SP]\n" "blt 18f\n" "add x21, x14, %x[ld_in_row]\n" - "ld1b { z12.s }, p1/Z, [x14]\n" + "ld1b { z21.s }, p1/Z, [x14]\n" "sub x15, x15, #0x2\n" - "ld1b { z20.s }, p1/Z, [x21]\n" + "ld1b { z8.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z21.h, z21.h, z8.h\n" "sub x13, x13, #0x1\n" - "ld1b { z13.s }, p1/Z, [x21]\n" + "ld1b { z22.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "lsr x20, x15, #0x1\n" - "add z12.h, z12.h, z5.h\n" - "ld1b { z19.s }, p1/Z, [x21]\n" + "add z21.h, z21.h, z11.h\n" + "ld1b { z25.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" + "trn1 z22.h, z22.h, z25.h\n" "cmp x20, x13\n" - "ld1b { z14.s }, p1/Z, [x21]\n" + "ld1b { z23.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "csel x23, x20, x13, LT\n" - "add z13.h, z13.h, z5.h\n" + "add z22.h, z22.h, z11.h\n" "ld1b { z18.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z5.h\n" - "ld1b { z15.s }, p1/Z, [x21]\n" + "trn1 z23.h, z23.h, z18.h\n" + "add z23.h, z23.h, z11.h\n" + "ld1b { z24.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "add x14, x14, %x[ld_in_col]\n" - "ld1b { z17.s }, p1/Z, [x21]\n" + "ld1b { z19.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z15.h, z15.h, z5.h\n" - "ld1b { z16.s }, p1/Z, [x21]\n" - "mov z16.d, z16.d\n" - "add z16.h, z16.h, z5.h\n" + "trn1 z24.h, z24.h, z19.h\n" + "add z24.h, z24.h, z11.h\n" + "ld1b { z8.s }, p1/Z, [x21]\n" + "mov z25.d, z8.d\n" + "add z25.h, z25.h, z11.h\n" "and x15, x15, #0x1\n" "sub x13, x13, x23\n" "cbz x23, 17f\n" "11:" // Unpadded: Main loop - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" + ".inst 0xc17e16a8 // sdot za.s[x8, 0], { z21.h-z24.h }, z14.h\n" "addvl x20, SP, #4\n" "add x22, x14, %x[ld_in_row]\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" + ".inst 0xc17f16c8 // sdot za.s[x8, 0], { z22.h-z25.h }, z15.h\n" + ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" "addvl x21, SP, #2\n" "subs x23, x23, #0x1\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - "ld1b { z12.s }, p1/Z, [x14]\n" + ".inst 0xc17016a9 // sdot za.s[x8, 1], { z21.h-z24.h }, z0.h\n" + "ld1b { z21.s }, p1/Z, [x14]\n" "add x14, x14, %x[ld_in_col]\n" "add x20, x14, %x[ld_in_row]\n" - "ld1b { z20.s }, p1/Z, [x22]\n" + "ld1b { z18.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - ".inst 0xc17b15a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z11.h\n" - "trn1 z12.h, z12.h, z20.h\n" - "ld1b { z13.s }, p1/Z, [x22]\n" + ".inst 0xc17116c9 // sdot za.s[x8, 1], { z22.h-z25.h }, z1.h\n" + "trn1 z21.h, z21.h, z18.h\n" + "ld1b { z22.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "add z12.h, z12.h, z5.h\n" - ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" - "ld1b { z19.s }, p1/Z, [x22]\n" + "add z21.h, z21.h, z11.h\n" + ".inst 0xc0060c00 // mova { z0.d-z3.d }, za.d[x8, #0]\n" + "ld1b { z8.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z5.h\n" - "ld1b { z14.s }, p1/Z, [x22]\n" + "trn1 z22.h, z22.h, z8.h\n" + "add z22.h, z22.h, z11.h\n" + "ld1b { z23.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "add x8, x8, #0x1\n" - ".inst 0xc0040c02 // mova za.d[x8, #2], { z0.d-z3.d }\n" - "ld1b { z18.s }, p1/Z, [x22]\n" + ".inst 0xc0040f82 // mova za.d[x8, #2], { z28.d-z31.d }\n" + "ld1b { z27.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z5.h\n" - "ld1b { z15.s }, p1/Z, [x22]\n" + "trn1 z23.h, z23.h, z27.h\n" + "add z23.h, z23.h, z11.h\n" + "ld1b { z24.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - ".inst 0xc1a8ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z8.s\n" - "ld1b { z17.s }, p1/Z, [x22]\n" + ".inst 0xc1a6ac00 // sqdmulh { z0.s-z3.s }, { z0.s-z3.s }, z6.s\n" + "ld1b { z8.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z15.h, z15.h, z5.h\n" - "ld1b { z16.s }, p1/Z, [x22]\n" - "mov z16.d, z16.d\n" - "add z16.h, z16.h, z5.h\n" - ".inst 0xa0402aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21]\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xc1a7aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z7.s\n" - "ld1b { z12.s }, p1/Z, [x14]\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" - ".inst 0xc1a4ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" - "ld1b { z20.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - ".inst 0xc1b7cf7c // sclamp { z28.s-z31.s }, z27.s, z23.s\n" - "ld1b { z13.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row]\n" - "st1b { z28.s }, p1, [x11]\n" + "trn1 z24.h, z24.h, z8.h\n" + "add z24.h, z24.h, z11.h\n" + "ld1b { z4.s }, p1/Z, [x22]\n" + "mov z25.d, z4.d\n" + "add z25.h, z25.h, z11.h\n" + ".inst 0xa1402aa4 // ld1h { z4.h, z12.h }, pn10.b/Z, [x21]\n" + ".inst 0xc17416a8 // sdot za.s[x8, 0], { z21.h-z24.h }, z4.h\n" + ".inst 0xc1a9aa20 // srshl { z0.s-z3.s }, { z0.s-z3.s }, z9.s\n" + "ld1b { z21.s }, p1/Z, [x14]\n" + ".inst 0xc17c16c8 // sdot za.s[x8, 0], { z22.h-z25.h }, z12.h\n" + ".inst 0xc1adab00 // add { z0.s-z3.s }, { z0.s-z3.s }, z13.s\n" + "ld1b { z12.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row]\n" + "trn1 z21.h, z21.h, z12.h\n" + ".inst 0xc1a7cd40 // sclamp { z0.s-z3.s }, z10.s, z7.s\n" + "ld1b { z22.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row]\n" + "st1b { z0.s }, p1, [x11]\n" "add x11, x11, x9\n" - "ld1b { z19.s }, p1/Z, [x20]\n" + "ld1b { z20.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "st1b { z29.s }, p1, [x10]\n" - "ld1b { z14.s }, p1/Z, [x20]\n" + "trn1 z22.h, z22.h, z20.h\n" + "st1b { z1.s }, p1, [x10]\n" + "ld1b { z23.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" "add x10, x10, x28\n" - "st1b { z30.s }, p1, [x27]\n" - "ld1b { z18.s }, p1/Z, [x20]\n" + "st1b { z2.s }, p1, [x27]\n" + "ld1b { z24.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" + "trn1 z23.h, z23.h, z24.h\n" "add x27, x27, x25\n" - "ld1b { z15.s }, p1/Z, [x20]\n" + "ld1b { z24.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "st1b { z31.s }, p1, [x26]\n" + "st1b { z3.s }, p1, [x26]\n" "add x26, x26, x24\n" - "ld1b { z17.s }, p1/Z, [x20]\n" + "ld1b { z3.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z12.h, z12.h, z5.h\n" - "ld1b { z16.s }, p1/Z, [x20]\n" - "mov z16.d, z16.d\n" - "add z13.h, z13.h, z5.h\n" + "trn1 z24.h, z24.h, z3.h\n" + "add z21.h, z21.h, z11.h\n" + "ld1b { z3.s }, p1/Z, [x20]\n" + "mov z25.d, z3.d\n" + "add z22.h, z22.h, z11.h\n" "add x14, x14, %x[ld_in_col]\n" - ".inst 0xa0402bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP]\n" - "add z14.h, z14.h, z5.h\n" - "add z15.h, z15.h, z5.h\n" - "add z16.h, z16.h, z5.h\n" + ".inst 0xa0402bee // ld1h { z14.h-z15.h }, pn10.b/Z, [SP]\n" + "add z23.h, z23.h, z11.h\n" + "add z24.h, z24.h, z11.h\n" + "add z25.h, z25.h, z11.h\n" "bgt 11b\n" "b 17f\n" "12:" // Padded @@ -418,442 +418,442 @@ void sme2_u8s8u8q_planar_3x3_s2_4rows_dot_za_impl( "13:" // Padded: 2 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z12.s }, p0/Z, [x14]\n" - "add z12.h, p0/M, z12.h, z5.h\n" + "ld1b { z22.s }, p0/Z, [x14]\n" + "add z22.h, p0/M, z22.h, z11.h\n" "add x20, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z5.h\n" + "ld1b { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z5.h\n" + "ld1b { z23.s }, p0/Z, [x20]\n" + "add z23.h, p0/M, z23.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z5.h\n" + "ld1b { z4.s }, p0/Z, [x20]\n" + "add z4.h, p0/M, z4.h, z11.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "trn1 z13.h, z13.h, z19.h\n" + "trn1 z22.h, z22.h, z17.h\n" + "trn1 z23.h, z23.h, z4.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z5.h\n" + "ld1b { z24.s }, p0/Z, [x20]\n" + "add z24.h, p0/M, z24.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z5.h\n" + "add z18.h, p0/M, z18.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z5.h\n" + "ld1b { z25.s }, p0/Z, [x20]\n" + "add z25.h, p0/M, z25.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1b { z17.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z17.h, p0/M, z17.h, z5.h\n" + "add z17.h, p0/M, z17.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z5.h\n" + "ld1b { z1.s }, p0/Z, [x20]\n" + "add z1.h, p0/M, z1.h, z11.h\n" "addvl x20, SP, #4\n" - "trn1 z14.h, z14.h, z18.h\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" - "mov z16.d, z16.d\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" + "trn1 z24.h, z24.h, z18.h\n" + "trn1 z25.h, z25.h, z17.h\n" + ".inst 0xa1402a84 // ld1h { z4.h, z12.h }, pn10.b/Z, [x20]\n" + "mov z26.d, z1.d\n" + ".inst 0xc17416c8 // sdot za.s[x8, 0], { z22.h-z25.h }, z4.h\n" "add x14, x14, %x[ld_in_col]\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" + ".inst 0xc17c16e8 // sdot za.s[x8, 0], { z23.h-z26.h }, z12.h\n" "14:" // Padded: 1 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z12.s }, p0/Z, [x14]\n" - "add z12.h, p0/M, z12.h, z5.h\n" + "ld1b { z22.s }, p0/Z, [x14]\n" + "add z22.h, p0/M, z22.h, z11.h\n" "add x20, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z5.h\n" + "ld1b { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z5.h\n" + "ld1b { z23.s }, p0/Z, [x20]\n" + "add z23.h, p0/M, z23.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z5.h\n" + "ld1b { z5.s }, p0/Z, [x20]\n" + "add z5.h, p0/M, z5.h, z11.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "trn1 z13.h, z13.h, z19.h\n" + "trn1 z22.h, z22.h, z17.h\n" + "trn1 z23.h, z23.h, z5.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z5.h\n" + "ld1b { z24.s }, p0/Z, [x20]\n" + "add z24.h, p0/M, z24.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z5.h\n" + "add z18.h, p0/M, z18.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z5.h\n" + "ld1b { z25.s }, p0/Z, [x20]\n" + "add z25.h, p0/M, z25.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1b { z17.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z17.h, p0/M, z17.h, z5.h\n" + "add z17.h, p0/M, z17.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z5.h\n" + "ld1b { z15.s }, p0/Z, [x20]\n" + "add z15.h, p0/M, z15.h, z11.h\n" "addvl x20, SP, #2\n" - "trn1 z14.h, z14.h, z18.h\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" - "mov z16.d, z16.d\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" + "trn1 z24.h, z24.h, z18.h\n" + "trn1 z25.h, z25.h, z17.h\n" + ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" + "mov z26.d, z15.d\n" + ".inst 0xc17016c8 // sdot za.s[x8, 0], { z22.h-z25.h }, z0.h\n" "add x14, x14, %x[ld_in_col]\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" + ".inst 0xc17116e8 // sdot za.s[x8, 0], { z23.h-z26.h }, z1.h\n" "15:" // Padded: 0 priming loads "cmp x15, #0x2\n" - ".inst 0xa0402bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP]\n" + ".inst 0xa0402bee // ld1h { z14.h-z15.h }, pn10.b/Z, [SP]\n" "blt 18f\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z12.s }, p0/Z, [x14]\n" - "add z12.h, p0/M, z12.h, z5.h\n" + "ld1b { z21.s }, p0/Z, [x14]\n" + "add z21.h, p0/M, z21.h, z11.h\n" "add x20, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z5.h\n" + "ld1b { z18.s }, p0/Z, [x20]\n" + "add z18.h, p0/M, z18.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z5.h\n" + "ld1b { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z5.h\n" + "ld1b { z3.s }, p0/Z, [x20]\n" + "add z3.h, p0/M, z3.h, z11.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "trn1 z13.h, z13.h, z19.h\n" + "trn1 z21.h, z21.h, z18.h\n" + "trn1 z22.h, z22.h, z3.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z5.h\n" + "ld1b { z23.s }, p0/Z, [x20]\n" + "add z23.h, p0/M, z23.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z5.h\n" + "ld1b { z19.s }, p0/Z, [x20]\n" + "add z19.h, p0/M, z19.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z5.h\n" + "ld1b { z24.s }, p0/Z, [x20]\n" + "add z24.h, p0/M, z24.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" + "ld1b { z20.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z17.h, p0/M, z17.h, z5.h\n" + "add z20.h, p0/M, z20.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z5.h\n" + "ld1b { z3.s }, p0/Z, [x20]\n" + "add z3.h, p0/M, z3.h, z11.h\n" "sub x15, x15, #0x2\n" "sub x13, x13, #0x1\n" - "trn1 z14.h, z14.h, z18.h\n" - "trn1 z15.h, z15.h, z17.h\n" + "trn1 z23.h, z23.h, z19.h\n" + "trn1 z24.h, z24.h, z20.h\n" "lsr x20, x15, #0x1\n" "cmp x20, x13\n" - "mov z16.d, z16.d\n" + "mov z25.d, z3.d\n" "csel x22, x20, x13, LT\n" "add x14, x14, %x[ld_in_col]\n" "and x15, x15, #0x1\n" "sub x13, x13, x22\n" "cbz x22, 17f\n" "16:" // Padded: Main loop - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" + ".inst 0xc17e16a8 // sdot za.s[x8, 0], { z21.h-z24.h }, z14.h\n" "addvl x20, SP, #4\n" "mov x12, #0x0\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" + ".inst 0xc17f16c8 // sdot za.s[x8, 0], { z22.h-z25.h }, z15.h\n" + ".inst 0xa1402a84 // ld1h { z4.h, z12.h }, pn10.b/Z, [x20]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "add x21, x14, %x[ld_in_row]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - "ld1b { z12.s }, p0/Z, [x14]\n" - "add z12.h, p0/M, z12.h, z5.h\n" + ".inst 0xc17416a9 // sdot za.s[x8, 1], { z21.h-z24.h }, z4.h\n" + "ld1b { z21.s }, p0/Z, [x14]\n" + "add z21.h, p0/M, z21.h, z11.h\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z20.s }, p0/Z, [x21]\n" - "add z20.h, p0/M, z20.h, z5.h\n" + "ld1b { z14.s }, p0/Z, [x21]\n" + "add z14.h, p0/M, z14.h, z11.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc17b15a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z11.h\n" - "ld1b { z13.s }, p0/Z, [x21]\n" - "add z13.h, p0/M, z13.h, z5.h\n" + ".inst 0xc17c16c9 // sdot za.s[x8, 1], { z22.h-z25.h }, z12.h\n" + "ld1b { z22.s }, p0/Z, [x21]\n" + "add z22.h, p0/M, z22.h, z11.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z19.s }, p0/Z, [x21]\n" + "ld1b { z15.s }, p0/Z, [x21]\n" "mov x12, #0x4\n" - "add z19.h, p0/M, z19.h, z5.h\n" + "add z15.h, p0/M, z15.h, z11.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z14.s }, p0/Z, [x21]\n" - "add z14.h, p0/M, z14.h, z5.h\n" + "ld1b { z23.s }, p0/Z, [x21]\n" + "add z23.h, p0/M, z23.h, z11.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z18.s }, p0/Z, [x21]\n" - "add z18.h, p0/M, z18.h, z5.h\n" + "ld1b { z17.s }, p0/Z, [x21]\n" + "add z17.h, p0/M, z17.h, z11.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z15.s }, p0/Z, [x21]\n" - "add z15.h, p0/M, z15.h, z5.h\n" + "ld1b { z24.s }, p0/Z, [x21]\n" + "add z24.h, p0/M, z24.h, z11.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z17.s }, p0/Z, [x21]\n" - "add z17.h, p0/M, z17.h, z5.h\n" + "ld1b { z4.s }, p0/Z, [x21]\n" + "add z4.h, p0/M, z4.h, z11.h\n" "mov x12, #0x8\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "trn1 z13.h, z13.h, z19.h\n" + "trn1 z21.h, z21.h, z14.h\n" + "trn1 z22.h, z22.h, z15.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "addvl x20, SP, #2\n" - "ld1b { z16.s }, p0/Z, [x21]\n" - "trn1 z14.h, z14.h, z18.h\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" + "ld1b { z2.s }, p0/Z, [x21]\n" + "trn1 z23.h, z23.h, z17.h\n" + "trn1 z24.h, z24.h, z4.h\n" + ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" "mov x12, #0x0\n" - ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" + ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" - "add z16.h, p0/M, z16.h, z5.h\n" + "add z2.h, p0/M, z2.h, z11.h\n" "add x14, x14, %x[ld_in_col]\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" + ".inst 0xc17016a8 // sdot za.s[x8, 0], { z21.h-z24.h }, z0.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z12.s }, p0/Z, [x14]\n" - "add z12.h, p0/M, z12.h, z5.h\n" + "ld1b { z21.s }, p0/Z, [x14]\n" + "add z21.h, p0/M, z21.h, z11.h\n" "add x20, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "mov z16.d, z16.d\n" + "mov z25.d, z2.d\n" "ld1b { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z5.h\n" + "add z20.h, p0/M, z20.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" - "ld1b { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z5.h\n" + ".inst 0xc17116c8 // sdot za.s[x8, 0], { z22.h-z25.h }, z1.h\n" + "ld1b { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z19.s }, p0/Z, [x20]\n" + "ld1b { z4.s }, p0/Z, [x20]\n" "mov x12, #0x4\n" - "add z19.h, p0/M, z19.h, z5.h\n" + "add z4.h, p0/M, z4.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc0040c02 // mova za.d[x8, #2], { z0.d-z3.d }\n" + ".inst 0xc0040f82 // mova za.d[x8, #2], { z28.d-z31.d }\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z5.h\n" + "ld1b { z23.s }, p0/Z, [x20]\n" + "add z23.h, p0/M, z23.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z5.h\n" + "ld1b { z27.s }, p0/Z, [x20]\n" + "add z27.h, p0/M, z27.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z15.s }, p0/Z, [x20]\n" - ".inst 0xc1a8ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z8.s\n" - "add z15.h, p0/M, z15.h, z5.h\n" + "ld1b { z24.s }, p0/Z, [x20]\n" + ".inst 0xc1a6ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n" + "add z24.h, p0/M, z24.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" + "ld1b { z12.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - ".inst 0xc1a7aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z7.s\n" + ".inst 0xc1a9aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z9.s\n" "add x20, x20, %x[ld_in_row]\n" - "add z17.h, p0/M, z17.h, z5.h\n" + "add z12.h, p0/M, z12.h, z11.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z5.h\n" - ".inst 0xc1a4ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" + "ld1b { z8.s }, p0/Z, [x20]\n" + "add z8.h, p0/M, z8.h, z11.h\n" + ".inst 0xc1adab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z13.s\n" "subs x22, x22, #0x1\n" - ".inst 0xa0402bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP]\n" - ".inst 0xc1b7cf7c // sclamp { z28.s-z31.s }, z27.s, z23.s\n" - "st1b { z28.s }, p1, [x11]\n" + ".inst 0xa0402bee // ld1h { z14.h-z15.h }, pn10.b/Z, [SP]\n" + ".inst 0xc1a7cd50 // sclamp { z16.s-z19.s }, z10.s, z7.s\n" + "st1b { z16.s }, p1, [x11]\n" "add x11, x11, x9\n" - "trn1 z12.h, z12.h, z20.h\n" - "st1b { z29.s }, p1, [x10]\n" + "trn1 z21.h, z21.h, z20.h\n" + "st1b { z17.s }, p1, [x10]\n" "add x10, x10, x28\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" - "st1b { z30.s }, p1, [x27]\n" + "trn1 z22.h, z22.h, z4.h\n" + "trn1 z23.h, z23.h, z27.h\n" + "st1b { z18.s }, p1, [x27]\n" "add x27, x27, x25\n" - "trn1 z15.h, z15.h, z17.h\n" - "mov z16.d, z16.d\n" - "st1b { z31.s }, p1, [x26]\n" + "trn1 z24.h, z24.h, z12.h\n" + "mov z25.d, z8.d\n" + "st1b { z19.s }, p1, [x26]\n" "add x26, x26, x24\n" "add x14, x14, %x[ld_in_col]\n" "bgt 16b\n" "17:" // Main loop tail - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" + ".inst 0xc17e16a8 // sdot za.s[x8, 0], { z21.h-z24.h }, z14.h\n" "addvl x20, SP, #4\n" "mov x12, #0x0\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" + ".inst 0xc17f16c8 // sdot za.s[x8, 0], { z22.h-z25.h }, z15.h\n" + ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "add x20, x14, %x[ld_in_row]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - "ld1b { z12.s }, p0/Z, [x14]\n" - "add z12.h, p0/M, z12.h, z5.h\n" + ".inst 0xc17016a9 // sdot za.s[x8, 1], { z21.h-z24.h }, z0.h\n" + "ld1b { z0.s }, p0/Z, [x14]\n" + "add z0.h, p0/M, z0.h, z11.h\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z5.h\n" + "ld1b { z14.s }, p0/Z, [x20]\n" + "add z14.h, p0/M, z14.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc17b15a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z11.h\n" - "ld1b { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z5.h\n" + ".inst 0xc17116c9 // sdot za.s[x8, 1], { z22.h-z25.h }, z1.h\n" + "ld1b { z1.s }, p0/Z, [x20]\n" + "add z1.h, p0/M, z1.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z19.s }, p0/Z, [x20]\n" + "ld1b { z12.s }, p0/Z, [x20]\n" "mov x12, #0x4\n" - "add z19.h, p0/M, z19.h, z5.h\n" + "add z12.h, p0/M, z12.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z5.h\n" + "ld1b { z2.s }, p0/Z, [x20]\n" + "add z2.h, p0/M, z2.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z5.h\n" + "ld1b { z21.s }, p0/Z, [x20]\n" + "add z21.h, p0/M, z21.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z15.s }, p0/Z, [x20]\n" - ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" - "add z15.h, p0/M, z15.h, z5.h\n" + "ld1b { z3.s }, p0/Z, [x20]\n" + ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" + "add z3.h, p0/M, z3.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" + "ld1b { z25.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - ".inst 0xc1a8ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z8.s\n" + ".inst 0xc1a6ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n" "add x20, x20, %x[ld_in_row]\n" - "add z17.h, p0/M, z17.h, z5.h\n" + "add z25.h, p0/M, z25.h, z11.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" + "ld1b { z27.s }, p0/Z, [x20]\n" "addvl x20, SP, #2\n" - ".inst 0xc1a7aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z7.s\n" - "trn1 z12.h, z12.h, z20.h\n" + ".inst 0xc1a9aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z9.s\n" + "trn1 z0.h, z0.h, z14.h\n" "add x8, x8, #0x1\n" - "add z16.h, p0/M, z16.h, z5.h\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" + "add z27.h, p0/M, z27.h, z11.h\n" + "trn1 z1.h, z1.h, z12.h\n" + "trn1 z2.h, z2.h, z21.h\n" "add x14, x14, %x[ld_in_col]\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1a4ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" - "mov z16.d, z16.d\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xc1b7cf7c // sclamp { z28.s-z31.s }, z27.s, z23.s\n" - "st1b { z28.s }, p1, [x11]\n" + "trn1 z3.h, z3.h, z25.h\n" + ".inst 0xa0402a8e // ld1h { z14.h-z15.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1adab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z13.s\n" + "mov z4.d, z27.d\n" + ".inst 0xc17e1408 // sdot za.s[x8, 0], { z0.h-z3.h }, z14.h\n" + ".inst 0xc1a7cd50 // sclamp { z16.s-z19.s }, z10.s, z7.s\n" + "st1b { z16.s }, p1, [x11]\n" "add x11, x11, x9\n" - ".inst 0xc0040c02 // mova za.d[x8, #2], { z0.d-z3.d }\n" - "st1b { z29.s }, p1, [x10]\n" + ".inst 0xc0040f82 // mova za.d[x8, #2], { z28.d-z31.d }\n" + "st1b { z17.s }, p1, [x10]\n" "add x10, x10, x28\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" - ".inst 0xa0402bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP]\n" - "st1b { z30.s }, p1, [x27]\n" + ".inst 0xc17f1428 // sdot za.s[x8, 0], { z1.h-z4.h }, z15.h\n" + ".inst 0xa0402bee // ld1h { z14.h-z15.h }, pn10.b/Z, [SP]\n" + "st1b { z18.s }, p1, [x27]\n" "add x27, x27, x25\n" - "st1b { z31.s }, p1, [x26]\n" + "st1b { z19.s }, p1, [x26]\n" "add x26, x26, x24\n" "18:" // Main loop skip tail "cbz x15, 19f\n" // Skip remainder inputs "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z12.s }, p0/Z, [x14]\n" - "add z12.h, p0/M, z12.h, z5.h\n" + "ld1b { z21.s }, p0/Z, [x14]\n" + "add z21.h, p0/M, z21.h, z11.h\n" "add x20, x14, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z5.h\n" + "ld1b { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z5.h\n" + "ld1b { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z5.h\n" + "ld1b { z0.s }, p0/Z, [x20]\n" + "add z0.h, p0/M, z0.h, z11.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "trn1 z13.h, z13.h, z19.h\n" + "trn1 z21.h, z21.h, z17.h\n" + "trn1 z22.h, z22.h, z0.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z5.h\n" + "ld1b { z23.s }, p0/Z, [x20]\n" + "add z23.h, p0/M, z23.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z5.h\n" + "ld1b { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z5.h\n" + "ld1b { z24.s }, p0/Z, [x20]\n" + "add z24.h, p0/M, z24.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" + "ld1b { z5.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z17.h, p0/M, z17.h, z5.h\n" + "add z5.h, p0/M, z5.h, z11.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z5.h\n" - "trn1 z14.h, z14.h, z18.h\n" - "trn1 z15.h, z15.h, z17.h\n" - "mov z16.d, z16.d\n" + "ld1b { z4.s }, p0/Z, [x20]\n" + "add z4.h, p0/M, z4.h, z11.h\n" + "trn1 z23.h, z23.h, z17.h\n" + "trn1 z24.h, z24.h, z5.h\n" + "mov z25.d, z4.d\n" "addvl x20, SP, #4\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" + ".inst 0xc17e16a8 // sdot za.s[x8, 0], { z21.h-z24.h }, z14.h\n" "sub x13, x13, #0x1\n" - ".inst 0xc17b15a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z11.h\n" - ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" - ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" - ".inst 0xc1a8ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z8.s\n" - ".inst 0xc1a7aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z7.s\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xc1a4ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" - ".inst 0xc17b15a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z11.h\n" + ".inst 0xc17f16c8 // sdot za.s[x8, 0], { z22.h-z25.h }, z15.h\n" + ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" + ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" + ".inst 0xc1a6ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n" + ".inst 0xc1a9aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z9.s\n" + ".inst 0xc17016a9 // sdot za.s[x8, 1], { z21.h-z24.h }, z0.h\n" + ".inst 0xc1adab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z13.s\n" + ".inst 0xc17116c9 // sdot za.s[x8, 1], { z22.h-z25.h }, z1.h\n" "add x8, x8, #0x1\n" - ".inst 0xc1b7cf7c // sclamp { z28.s-z31.s }, z27.s, z23.s\n" - "st1b { z28.s }, p1, [x11]\n" + ".inst 0xc1a7cd50 // sclamp { z16.s-z19.s }, z10.s, z7.s\n" + "st1b { z16.s }, p1, [x11]\n" "add x11, x11, x9\n" - ".inst 0xc0040c02 // mova za.d[x8, #2], { z0.d-z3.d }\n" - "st1b { z29.s }, p1, [x10]\n" + ".inst 0xc0040f82 // mova za.d[x8, #2], { z28.d-z31.d }\n" + "st1b { z17.s }, p1, [x10]\n" "add x10, x10, x28\n" - "st1b { z30.s }, p1, [x27]\n" + "st1b { z18.s }, p1, [x27]\n" "add x27, x27, x25\n" - "st1b { z31.s }, p1, [x26]\n" + "st1b { z19.s }, p1, [x26]\n" "add x26, x26, x24\n" "19:" // Tail input: End "cbz x13, 21f\n" "20:" // Right padding loop - ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" - ".inst 0xc1a8ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z8.s\n" + ".inst 0xc0060c00 // mova { z0.d-z3.d }, za.d[x8, #0]\n" + ".inst 0xc1a6ac00 // sqdmulh { z0.s-z3.s }, { z0.s-z3.s }, z6.s\n" "add x8, x8, #0x1\n" - ".inst 0xc1a7aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z7.s\n" + ".inst 0xc1a9aa20 // srshl { z0.s-z3.s }, { z0.s-z3.s }, z9.s\n" "subs x13, x13, #0x1\n" - ".inst 0xc0040c02 // mova za.d[x8, #2], { z0.d-z3.d }\n" - ".inst 0xc1a4ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" - ".inst 0xc1b7cf7c // sclamp { z28.s-z31.s }, z27.s, z23.s\n" - "st1b { z28.s }, p1, [x11]\n" + ".inst 0xc0040f82 // mova za.d[x8, #2], { z28.d-z31.d }\n" + ".inst 0xc1adab00 // add { z0.s-z3.s }, { z0.s-z3.s }, z13.s\n" + ".inst 0xc1a7cd40 // sclamp { z0.s-z3.s }, z10.s, z7.s\n" + "st1b { z0.s }, p1, [x11]\n" "add x11, x11, x9\n" - "st1b { z29.s }, p1, [x10]\n" + "st1b { z1.s }, p1, [x10]\n" "add x10, x10, x28\n" - "st1b { z30.s }, p1, [x27]\n" + "st1b { z2.s }, p1, [x27]\n" "add x27, x27, x25\n" - "st1b { z31.s }, p1, [x26]\n" + "st1b { z3.s }, p1, [x26]\n" "add x26, x26, x24\n" "bgt 20b\n" "21:" // End - "ldr x22, [%x[args], %[offsetof_Args_weights]]\n" - "incw x22, ALL, MUL #9\n" - "str x22, [%x[args], %[offsetof_Args_weights]]\n" + "ldr x20, [%x[args], %[offsetof_Args_weights]]\n" + "incw x20, ALL, MUL #9\n" + "str x20, [%x[args], %[offsetof_Args_weights]]\n" "incw x16\n" - "ldr x20, [%x[args], %[offsetof_Args_ld_in_vl]]\n" + "ldr x21, [%x[args], %[offsetof_Args_ld_in_vl]]\n" "whilelt p1.s, x16, x17\n" - "ldr x14, [%x[args], %[offsetof_Args_inptr]]\n" - "add x14, x14, x20\n" - "str x14, [%x[args], %[offsetof_Args_inptr]]\n" + "ldr x20, [%x[args], %[offsetof_Args_inptr]]\n" + "add x20, x20, x21\n" + "str x20, [%x[args], %[offsetof_Args_inptr]]\n" "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" "ldr x24, [%x[args], %[offsetof_Args_ld_out_vls]]\n" "ldp x23, x22, [x25, #0x0]\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_5x5_s1_4rows_dot_za.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_5x5_s1_4rows_dot_za.hpp index 6071197340f414a96726deafc1077cb9ca245eed..52173b8551501eefe358e64e02771685ae06c8d7 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_5x5_s1_4rows_dot_za.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_5x5_s1_4rows_dot_za.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,8 @@ * SOFTWARE. */ +#include "src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp" + namespace arm_conv { namespace depthwise { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_5x5_s1_4rows_dot_za/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_5x5_s1_4rows_dot_za/generic.cpp index 3e77c75ad74816ad1e889ecc5479dca9572c092f..64023eeaff655fdc8b7c893fc8feccb2246835e8 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_5x5_s1_4rows_dot_za/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_5x5_s1_4rows_dot_za/generic.cpp @@ -69,196 +69,196 @@ void sme2_u8s8u8q_planar_5x5_s1_4rows_dot_za_impl( __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" - "ldr x4, [%x[args], %[offsetof_Args_pad_bottom]]\n" + "ldr x5, [%x[args], %[offsetof_Args_pad_bottom]]\n" "ptrue p2.b\n" "mov x20, #0x8\n" "ldr x6, [%x[args], %[offsetof_Args_pad_top]]\n" - "ld1rh { z25.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_a_offset]]\n" - "sub x20, x20, x4\n" + "ld1rh { z17.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_a_offset]]\n" + "sub x20, x20, x5\n" ".inst 0x25207812 // ptrue pn10.b\n" "ldr x7, [%x[args], %[offsetof_Args_n_channels]]\n" "whilelt p1.s, XZR, x7\n" "whilelt p9.s, XZR, x20\n" - "ld1rw { z9.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" + "ld1rw { z12.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" "whilelt p8.s, XZR, x6\n" "addvl SP, SP, #-30\n" - "ldr x5, [%x[args], %[offsetof_Args_current_channel]]\n" - "neg z25.h, p2/M, z25.h\n" + "ldr x17, [%x[args], %[offsetof_Args_current_channel]]\n" + "neg z17.h, p2/M, z17.h\n" "eor p8.b, p2/Z, p8.b, p9.b\n" - "ld1rw { z3.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_mul]]\n" - "ld1rw { z1.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z7.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z4.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_right_shift]]\n" "ld1rw { z24.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" - "ld1rw { z31.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" + "ld1rw { z16.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" "1:" // Channel loop "ldr x20, [%x[qp], %[offsetof_Requantize32_bias]]\n" - "mov z6.s, #0x0\n" + "mov z18.s, #0x0\n" "cbz x20, 2f\n" - "ld1w { z6.s }, p1/Z, [x20, x5, LSL #2]\n" + "ld1w { z18.s }, p1/Z, [x20, x17, LSL #2]\n" "2:" // Load bias: Done "ldr x23, [%x[args], %[offsetof_Args_weights]]\n" - "mov x22, x23\n" - "ld1sb { z18.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "ld1rh { z12.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" - "mov z2.h, #0x0\n" - "sub z18.h, z18.h, z12.h\n" + "mov x20, x23\n" + "ld1sb { z2.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "ld1rh { z3.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" + "mov z15.h, #0x0\n" + "sub z2.h, z2.h, z3.h\n" "incw x23\n" - "ld1sb { z17.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "sub z17.h, z17.h, z12.h\n" - "trn1 z0.h, z2.h, z18.h\n" - "ld1sb { z21.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "sub z21.h, z21.h, z12.h\n" - "trn1 z8.h, z18.h, z17.h\n" - "ld1sb { z16.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "sub z16.h, z16.h, z12.h\n" - "trn1 z4.h, z17.h, z21.h\n" - "ld1sb { z15.s }, p2/Z, [x22]\n" - "sub z15.h, z15.h, z12.h\n" - "mov x22, x23\n" - "trn1 z5.h, z21.h, z16.h\n" - "ld1sb { z18.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "trn1 z10.h, z16.h, z15.h\n" + "ld1sb { z13.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "sub z13.h, z13.h, z3.h\n" "trn1 z11.h, z15.h, z2.h\n" - "ld1sb { z17.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "sub z18.h, z18.h, z12.h\n" - "sub z17.h, z17.h, z12.h\n" - "ld1sb { z21.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "sub z21.h, z21.h, z12.h\n" - "addvl x21, SP, #30\n" - "ld1sb { z16.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" + "ld1sb { z27.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "sub z27.h, z27.h, z3.h\n" + "trn1 z0.h, z2.h, z13.h\n" + "ld1sb { z19.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "sub z19.h, z19.h, z3.h\n" + "trn1 z26.h, z13.h, z27.h\n" + "ld1sb { z14.s }, p2/Z, [x20]\n" + "sub z14.h, z14.h, z3.h\n" + "mov x20, x23\n" + "trn1 z10.h, z27.h, z19.h\n" + "ld1sb { z9.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "trn1 z19.h, z19.h, z14.h\n" + "trn1 z1.h, z14.h, z15.h\n" + "ld1sb { z5.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "sub z9.h, z9.h, z3.h\n" + "sub z5.h, z5.h, z3.h\n" + "ld1sb { z29.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "sub z29.h, z29.h, z3.h\n" + "addvl x22, SP, #30\n" + "ld1sb { z2.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" "incw x23\n" - "sub z16.h, z16.h, z12.h\n" - "ld1sb { z15.s }, p2/Z, [x22]\n" - "addvl x21, x21, #-6\n" - "sub z15.h, z15.h, z12.h\n" - "mov x22, x23\n" - "st1h { z0.h }, p2, [x21]\n" - "trn1 z0.h, z2.h, z18.h\n" + "sub z2.h, z2.h, z3.h\n" + "ld1sb { z23.s }, p2/Z, [x20]\n" + "addvl x22, x22, #-6\n" + "sub z23.h, z23.h, z3.h\n" + "mov x20, x23\n" + "st1h { z11.h }, p2, [x22]\n" + "trn1 z20.h, z15.h, z9.h\n" "incw x23\n" - "ldr x20, [%x[qp], %[offsetof_Requantize32_per_channel_muls]]\n" - "st1h { z8.h }, p2, [x21, #1, MUL VL]\n" - "trn1 z8.h, z18.h, z17.h\n" - "ld1sb { z18.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z4.h }, p2, [x21, #2, MUL VL]\n" - "trn1 z4.h, z17.h, z21.h\n" - "ld1sb { z17.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z5.h }, p2, [x21, #3, MUL VL]\n" - "trn1 z5.h, z21.h, z16.h\n" - "ld1sb { z21.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z10.h }, p2, [x21, #4, MUL VL]\n" - "trn1 z10.h, z16.h, z15.h\n" - "ld1sb { z16.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z11.h }, p2, [x21, #5, MUL VL]\n" - "trn1 z11.h, z15.h, z2.h\n" - "sub z18.h, z18.h, z12.h\n" - "addvl x21, x21, #-6\n" - "sub z17.h, z17.h, z12.h\n" - "ld1sb { z15.s }, p2/Z, [x22]\n" - "sub z21.h, z21.h, z12.h\n" - "mov x22, x23\n" - "sub z16.h, z16.h, z12.h\n" - "sub z15.h, z15.h, z12.h\n" - "st1h { z0.h }, p2, [x21]\n" + "ldr x21, [%x[qp], %[offsetof_Requantize32_per_channel_muls]]\n" + "st1h { z0.h }, p2, [x22, #1, MUL VL]\n" + "trn1 z22.h, z9.h, z5.h\n" + "ld1sb { z25.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z26.h }, p2, [x22, #2, MUL VL]\n" + "trn1 z9.h, z5.h, z29.h\n" + "ld1sb { z21.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z10.h }, p2, [x22, #3, MUL VL]\n" + "trn1 z26.h, z29.h, z2.h\n" + "ld1sb { z0.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z19.h }, p2, [x22, #4, MUL VL]\n" + "trn1 z28.h, z2.h, z23.h\n" + "ld1sb { z19.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z1.h }, p2, [x22, #5, MUL VL]\n" + "trn1 z2.h, z23.h, z15.h\n" + "sub z25.h, z25.h, z3.h\n" + "addvl x22, x22, #-6\n" + "sub z21.h, z21.h, z3.h\n" + "ld1sb { z6.s }, p2/Z, [x20]\n" + "sub z0.h, z0.h, z3.h\n" + "mov x20, x23\n" + "sub z19.h, z19.h, z3.h\n" + "sub z6.h, z6.h, z3.h\n" + "st1h { z20.h }, p2, [x22]\n" "incw x23\n" - "st1h { z8.h }, p2, [x21, #1, MUL VL]\n" - "trn1 z0.h, z2.h, z18.h\n" - "trn1 z8.h, z18.h, z17.h\n" - "ld1sb { z18.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z4.h }, p2, [x21, #2, MUL VL]\n" - "trn1 z4.h, z17.h, z21.h\n" - "ld1sb { z17.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z5.h }, p2, [x21, #3, MUL VL]\n" - "trn1 z5.h, z21.h, z16.h\n" - "ld1sb { z21.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z10.h }, p2, [x21, #4, MUL VL]\n" - "trn1 z10.h, z16.h, z15.h\n" - "ld1sb { z16.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z11.h }, p2, [x21, #5, MUL VL]\n" - "trn1 z11.h, z15.h, z2.h\n" - "sub z18.h, z18.h, z12.h\n" - "sub z17.h, z17.h, z12.h\n" - "ld1sb { z15.s }, p2/Z, [x22]\n" - "addvl x21, x21, #-6\n" - "sub z21.h, z21.h, z12.h\n" - "sub z16.h, z16.h, z12.h\n" - "mov x22, x23\n" - "st1h { z0.h }, p2, [x21]\n" - "sub z15.h, z15.h, z12.h\n" - "st1h { z8.h }, p2, [x21, #1, MUL VL]\n" - "trn1 z0.h, z2.h, z18.h\n" - "trn1 z8.h, z18.h, z17.h\n" - "ld1sb { z18.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z4.h }, p2, [x21, #2, MUL VL]\n" - "trn1 z4.h, z17.h, z21.h\n" - "ld1sb { z17.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z5.h }, p2, [x21, #3, MUL VL]\n" - "trn1 z5.h, z21.h, z16.h\n" - "ld1sb { z21.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z10.h }, p2, [x21, #4, MUL VL]\n" - "trn1 z10.h, z16.h, z15.h\n" - "ld1sb { z16.s }, p2/Z, [x22]\n" - "incw x22, ALL, MUL #5\n" - "st1h { z11.h }, p2, [x21, #5, MUL VL]\n" - "trn1 z11.h, z15.h, z2.h\n" - "ld1sb { z15.s }, p2/Z, [x22]\n" - "sub z18.h, z18.h, z12.h\n" - "addvl x21, x21, #-6\n" - "sub z17.h, z17.h, z12.h\n" - "sub z21.h, z21.h, z12.h\n" - "st1h { z0.h }, p2, [x21]\n" - "sub z16.h, z16.h, z12.h\n" - "sub z15.h, z15.h, z12.h\n" - "st1h { z8.h }, p2, [x21, #1, MUL VL]\n" - "st1h { z4.h }, p2, [x21, #2, MUL VL]\n" - "mov z7.d, z6.d\n" - "trn1 z0.h, z2.h, z18.h\n" - "st1h { z5.h }, p2, [x21, #3, MUL VL]\n" - "trn1 z8.h, z18.h, z17.h\n" - "trn1 z4.h, z17.h, z21.h\n" - "st1h { z10.h }, p2, [x21, #4, MUL VL]\n" - "trn1 z5.h, z21.h, z16.h\n" - "trn1 z10.h, z16.h, z15.h\n" - "st1h { z11.h }, p2, [x21, #5, MUL VL]\n" - "addvl x21, x21, #-6\n" - "trn1 z11.h, z15.h, z2.h\n" - "st1h { z0.h }, p2, [x21]\n" - "st1h { z8.h }, p2, [x21, #1, MUL VL]\n" - "st1h { z4.h }, p2, [x21, #2, MUL VL]\n" - "st1h { z5.h }, p2, [x21, #3, MUL VL]\n" - "st1h { z10.h }, p2, [x21, #4, MUL VL]\n" - "st1h { z11.h }, p2, [x21, #5, MUL VL]\n" - "cbz x20, 3f\n" - "ld1w { z3.s }, p1/Z, [x20, x5, LSL #2]\n" + "st1h { z22.h }, p2, [x22, #1, MUL VL]\n" + "trn1 z11.h, z15.h, z25.h\n" + "trn1 z10.h, z25.h, z21.h\n" + "ld1sb { z5.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z9.h }, p2, [x22, #2, MUL VL]\n" + "trn1 z14.h, z21.h, z0.h\n" + "ld1sb { z23.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z26.h }, p2, [x22, #3, MUL VL]\n" + "trn1 z21.h, z0.h, z19.h\n" + "ld1sb { z27.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z28.h }, p2, [x22, #4, MUL VL]\n" + "trn1 z19.h, z19.h, z6.h\n" + "ld1sb { z29.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z2.h }, p2, [x22, #5, MUL VL]\n" + "trn1 z13.h, z6.h, z15.h\n" + "sub z5.h, z5.h, z3.h\n" + "sub z23.h, z23.h, z3.h\n" + "ld1sb { z1.s }, p2/Z, [x20]\n" + "addvl x22, x22, #-6\n" + "sub z27.h, z27.h, z3.h\n" + "sub z29.h, z29.h, z3.h\n" + "mov x20, x23\n" + "st1h { z11.h }, p2, [x22]\n" + "sub z1.h, z1.h, z3.h\n" + "st1h { z10.h }, p2, [x22, #1, MUL VL]\n" + "trn1 z30.h, z15.h, z5.h\n" + "trn1 z26.h, z5.h, z23.h\n" + "ld1sb { z11.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z14.h }, p2, [x22, #2, MUL VL]\n" + "trn1 z22.h, z23.h, z27.h\n" + "ld1sb { z5.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z21.h }, p2, [x22, #3, MUL VL]\n" + "trn1 z28.h, z27.h, z29.h\n" + "ld1sb { z8.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z19.h }, p2, [x22, #4, MUL VL]\n" + "trn1 z27.h, z29.h, z1.h\n" + "ld1sb { z9.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "st1h { z13.h }, p2, [x22, #5, MUL VL]\n" + "trn1 z2.h, z1.h, z15.h\n" + "ld1sb { z14.s }, p2/Z, [x20]\n" + "sub z11.h, z11.h, z3.h\n" + "addvl x22, x22, #-6\n" + "sub z5.h, z5.h, z3.h\n" + "sub z8.h, z8.h, z3.h\n" + "st1h { z30.h }, p2, [x22]\n" + "sub z9.h, z9.h, z3.h\n" + "sub z14.h, z14.h, z3.h\n" + "st1h { z26.h }, p2, [x22, #1, MUL VL]\n" + "st1h { z22.h }, p2, [x22, #2, MUL VL]\n" + "mov z19.d, z18.d\n" + "trn1 z22.h, z15.h, z11.h\n" + "st1h { z28.h }, p2, [x22, #3, MUL VL]\n" + "trn1 z1.h, z11.h, z5.h\n" + "trn1 z31.h, z5.h, z8.h\n" + "st1h { z27.h }, p2, [x22, #4, MUL VL]\n" + "trn1 z8.h, z8.h, z9.h\n" + "trn1 z21.h, z9.h, z14.h\n" + "st1h { z2.h }, p2, [x22, #5, MUL VL]\n" + "addvl x22, x22, #-6\n" + "trn1 z15.h, z14.h, z15.h\n" + "st1h { z22.h }, p2, [x22]\n" + "st1h { z1.h }, p2, [x22, #1, MUL VL]\n" + "st1h { z31.h }, p2, [x22, #2, MUL VL]\n" + "st1h { z8.h }, p2, [x22, #3, MUL VL]\n" + "st1h { z21.h }, p2, [x22, #4, MUL VL]\n" + "st1h { z15.h }, p2, [x22, #5, MUL VL]\n" + "cbz x21, 3f\n" + "ld1w { z7.s }, p1/Z, [x21, x17, LSL #2]\n" "3:" // Load mul: End "ldr x20, [%x[qp], %[offsetof_Requantize32_per_channel_right_shifts]]\n" "cbz x20, 4f\n" - "ld1w { z1.s }, p1/Z, [x20, x5, LSL #2]\n" + "ld1w { z4.s }, p1/Z, [x20, x17, LSL #2]\n" "4:" // Load right_shift: End - "ldr x17, [%x[args], %[offsetof_Args_input_cols]]\n" - "sub x20, x17, #0x1\n" + "ldr x25, [%x[args], %[offsetof_Args_input_cols]]\n" + "sub x20, x25, #0x1\n" "orr x23, x20, %x[ld_in_col], LSL #16\n" "ldr x16, [%x[args], %[offsetof_Args_inptr]]\n" "orr x23, x7, x23, LSL #22\n" "mov x22, #0x8\n" - "add x21, x6, x4\n" + "add x21, x6, x5\n" "lsl x20, %x[ld_in_row], #0x0\n" "ldr x15, [%x[args], %[offsetof_Args_output_cols]]\n" "mov x11, #0x0\n" @@ -271,56 +271,56 @@ void sme2_u8s8u8q_planar_5x5_s1_4rows_dot_za_impl( ".inst 0xf8b74a9c // rprfm pldstrm, x23, [x20]\n" "add x20, x20, %x[ld_in_col]\n" "bgt 5b\n" - "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" + "ldr x23, [%x[args], %[offsetof_Args_outptrs]]\n" "lsl x20, %x[ld_in_row], #0x0\n" "msub x16, x6, x20, x16\n" - ".inst 0xc00468c0 // mova za.d[x11, #0], { z6.d-z7.d }\n" + ".inst 0xc0046a40 // mova za.d[x11, #0], { z18.d-z19.d }\n" "ldr x20, [%x[args], %[offsetof_Args_ld_out_cols]]\n" - ".inst 0xc00468c1 // mova za.d[x11, #1], { z6.d-z7.d }\n" + ".inst 0xc0046a41 // mova za.d[x11, #1], { z18.d-z19.d }\n" "mov x22, #0x4\n" - "ldp x14, x13, [x25], #0x10\n" - ".inst 0xc00468c2 // mova za.d[x11, #2], { z6.d-z7.d }\n" - "ldp x3, x10, [x20], #0x10\n" - ".inst 0xc00468c3 // mova za.d[x11, #3], { z6.d-z7.d }\n" + "ldp x14, x13, [x23], #0x10\n" + ".inst 0xc0046a42 // mova za.d[x11, #2], { z18.d-z19.d }\n" + "ldp x4, x10, [x20], #0x10\n" + ".inst 0xc0046a43 // mova za.d[x11, #3], { z18.d-z19.d }\n" "ldr x21, [%x[args], %[offsetof_Args_pad_left]]\n" - ".inst 0xc00468c4 // mova za.d[x11, #4], { z6.d-z7.d }\n" - "ldp x9, x28, [x25], #0x10\n" - ".inst 0xc00468c5 // mova za.d[x11, #5], { z6.d-z7.d }\n" + ".inst 0xc0046a44 // mova za.d[x11, #4], { z18.d-z19.d }\n" + "ldp x9, x28, [x23], #0x10\n" + ".inst 0xc0046a45 // mova za.d[x11, #5], { z18.d-z19.d }\n" "ldp x27, x26, [x20], #0x10\n" - ".inst 0xc00468c6 // mova za.d[x11, #6], { z6.d-z7.d }\n" - ".inst 0xc00468c7 // mova za.d[x11, #7], { z6.d-z7.d }\n" - ".inst 0xc00408c0 // mova za.d[x8, #0], { z6.d-z7.d }\n" - ".inst 0xc00408c1 // mova za.d[x8, #1], { z6.d-z7.d }\n" + ".inst 0xc0046a46 // mova za.d[x11, #6], { z18.d-z19.d }\n" + ".inst 0xc0046a47 // mova za.d[x11, #7], { z18.d-z19.d }\n" + ".inst 0xc0040a40 // mova za.d[x8, #0], { z18.d-z19.d }\n" + ".inst 0xc0040a41 // mova za.d[x8, #1], { z18.d-z19.d }\n" "cbz x21, 7f\n" "cmp x21, x22\n" "csel x20, x21, x22, LT\n" "sub x21, x21, x20\n" "sub x22, x22, x20\n" "cbz x21, 7f\n" - ".inst 0xc006680c // mova { z12.d-z13.d }, za.d[x11, #0]\n" + ".inst 0xc0066814 // mova { z20.d-z21.d }, za.d[x11, #0]\n" "sub x15, x15, x21\n" - ".inst 0xc006682e // mova { z14.d-z15.d }, za.d[x11, #1]\n" - ".inst 0xc1a3ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z3.s\n" - ".inst 0xc1a1aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n" - ".inst 0xc1a9ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z9.s\n" - ".inst 0xc1bfcf0c // sclamp { z12.s-z15.s }, z24.s, z31.s\n" + ".inst 0xc0066836 // mova { z22.d-z23.d }, za.d[x11, #1]\n" + ".inst 0xc1a7ac14 // sqdmulh { z20.s-z23.s }, { z20.s-z23.s }, z7.s\n" + ".inst 0xc1a4aa34 // srshl { z20.s-z23.s }, { z20.s-z23.s }, z4.s\n" + ".inst 0xc1acab14 // add { z20.s-z23.s }, { z20.s-z23.s }, z12.s\n" + ".inst 0xc1b0cf14 // sclamp { z20.s-z23.s }, z24.s, z16.s\n" "6:" // Left padding "subs x21, x21, #0x1\n" - "st1b { z12.s }, p1, [x14]\n" - "add x14, x14, x3\n" - "st1b { z14.s }, p1, [x13]\n" + "st1b { z20.s }, p1, [x14]\n" + "add x14, x14, x4\n" + "st1b { z22.s }, p1, [x13]\n" "add x13, x13, x10\n" - "st1b { z13.s }, p1, [x9]\n" + "st1b { z21.s }, p1, [x9]\n" "add x9, x9, x27\n" - "st1b { z15.s }, p1, [x28]\n" + "st1b { z23.s }, p1, [x28]\n" "add x28, x28, x26\n" "bgt 6b\n" "7:" // Left padding: End - "adds XZR, x6, x4\n" + "adds XZR, x6, x5\n" "bne 14f\n" "cbz x22, 12f\n" "cmp x22, #0x1\n" - "sub x17, x17, x22\n" + "sub x25, x25, x22\n" "beq 11f\n" "cmp x22, #0x2\n" "beq 10f\n" @@ -328,338 +328,338 @@ void sme2_u8s8u8q_planar_5x5_s1_4rows_dot_za_impl( "beq 9f\n" "8:" // Unpadded: 4 priming loads "add x21, x16, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x16]\n" + "ld1b { z1.s }, p1/Z, [x16]\n" "addvl x20, SP, #24\n" - "ld1b { z16.s }, p1/Z, [x21]\n" + "ld1b { z28.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z27.h, z17.h, z16.h\n" - "add z27.h, z27.h, z25.h\n" - "ld1b { z17.s }, p1/Z, [x21]\n" + "trn1 z27.h, z1.h, z28.h\n" + "add z27.h, z27.h, z17.h\n" + "ld1b { z1.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "add x16, x16, %x[ld_in_col]\n" - "ld1b { z16.s }, p1/Z, [x21]\n" + "ld1b { z2.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z28.h, z17.h, z16.h\n" - "add z28.h, z28.h, z25.h\n" - "ld1b { z16.s }, p1/Z, [x21]\n" + "trn1 z28.h, z1.h, z2.h\n" + "add z28.h, z28.h, z17.h\n" + "ld1b { z13.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1b { z29.s }, p1/Z, [x21]\n" + "ld1b { z6.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z29.h, z16.h, z29.h\n" - "add z29.h, z29.h, z25.h\n" - "ld1b { z17.s }, p1/Z, [x21]\n" + "trn1 z29.h, z13.h, z6.h\n" + "add z29.h, z29.h, z17.h\n" + "ld1b { z30.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" - "ld1b { z16.s }, p1/Z, [x21]\n" - "trn1 z30.h, z17.h, z16.h\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - "add z30.h, z30.h, z25.h\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" + ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" + ".inst 0xc16a7768 // sdot za.s[x11, 0], { z27.h-z28.h }, z10.h\n" + "ld1b { z20.s }, p1/Z, [x21]\n" + "trn1 z30.h, z30.h, z20.h\n" + ".inst 0xc1627769 // sdot za.s[x11, 1], { z27.h-z28.h }, z2.h\n" + ".inst 0xa1412a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + "add z30.h, z30.h, z17.h\n" + ".inst 0xc1697788 // sdot za.s[x11, 0], { z28.h-z29.h }, z9.h\n" + ".inst 0xc1617789 // sdot za.s[x11, 1], { z28.h-z29.h }, z1.h\n" ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" "9:" // Unpadded: 3 priming loads "add x22, x16, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x16]\n" + "ld1b { z2.s }, p1/Z, [x16]\n" "addvl x21, SP, #18\n" - "ld1b { z16.s }, p1/Z, [x22]\n" + "ld1b { z28.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z27.h, z17.h, z16.h\n" - "add z27.h, z27.h, z25.h\n" - "ld1b { z17.s }, p1/Z, [x22]\n" + "trn1 z20.h, z2.h, z28.h\n" + "add z20.h, z20.h, z17.h\n" + "ld1b { z31.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "addvl x20, SP, #24\n" - "ld1b { z16.s }, p1/Z, [x22]\n" + "ld1b { z11.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z28.h, z17.h, z16.h\n" - "add z28.h, z28.h, z25.h\n" - "ld1b { z17.s }, p1/Z, [x22]\n" + "trn1 z21.h, z31.h, z11.h\n" + "add z21.h, z21.h, z17.h\n" + "ld1b { z25.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "add x16, x16, %x[ld_in_col]\n" - "ld1b { z16.s }, p1/Z, [x22]\n" + "ld1b { z8.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z29.h, z17.h, z16.h\n" - "add z29.h, z29.h, z25.h\n" - "ld1b { z17.s }, p1/Z, [x22]\n" + "trn1 z22.h, z25.h, z8.h\n" + "add z22.h, z22.h, z17.h\n" + "ld1b { z8.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" - "ld1b { z16.s }, p1/Z, [x22]\n" - "trn1 z30.h, z17.h, z16.h\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" - "add z30.h, z30.h, z25.h\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" + ".inst 0xa1402aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21]\n" + ".inst 0xc16e7688 // sdot za.s[x11, 0], { z20.h-z21.h }, z14.h\n" + "ld1b { z3.s }, p1/Z, [x22]\n" + "trn1 z23.h, z8.h, z3.h\n" + ".inst 0xc1667689 // sdot za.s[x11, 1], { z20.h-z21.h }, z6.h\n" + ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" + ".inst 0xc161768a // sdot za.s[x11, 2], { z20.h-z21.h }, z1.h\n" + "add z23.h, z23.h, z17.h\n" + ".inst 0xa1412aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc160768b // sdot za.s[x11, 3], { z20.h-z21.h }, z0.h\n" + ".inst 0xc16976a8 // sdot za.s[x11, 0], { z21.h-z22.h }, z9.h\n" + ".inst 0xa0422aae // ld1h { z14.h-z15.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc16176a9 // sdot za.s[x11, 1], { z21.h-z22.h }, z1.h\n" + ".inst 0xa1412a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc16976aa // sdot za.s[x11, 2], { z21.h-z22.h }, z9.h\n" + ".inst 0xc16176ab // sdot za.s[x11, 3], { z21.h-z22.h }, z1.h\n" + ".inst 0xc16f76c8 // sdot za.s[x11, 0], { z22.h-z23.h }, z15.h\n" + ".inst 0xc16e76c9 // sdot za.s[x11, 1], { z22.h-z23.h }, z14.h\n" ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" + ".inst 0xc16b76ca // sdot za.s[x11, 2], { z22.h-z23.h }, z11.h\n" + ".inst 0xc16a76cb // sdot za.s[x11, 3], { z22.h-z23.h }, z10.h\n" "10:" // Unpadded: 2 priming loads "add x23, x16, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x16]\n" + "ld1b { z2.s }, p1/Z, [x16]\n" "addvl x22, SP, #12\n" - "ld1b { z16.s }, p1/Z, [x23]\n" + "ld1b { z22.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - "trn1 z27.h, z17.h, z16.h\n" - "add z27.h, z27.h, z25.h\n" - "ld1b { z17.s }, p1/Z, [x23]\n" + "trn1 z0.h, z2.h, z22.h\n" + "add z0.h, z0.h, z17.h\n" + "ld1b { z14.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" "addvl x21, SP, #18\n" - "ld1b { z16.s }, p1/Z, [x23]\n" + "ld1b { z6.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - "trn1 z28.h, z17.h, z16.h\n" - "add z28.h, z28.h, z25.h\n" - "ld1b { z17.s }, p1/Z, [x23]\n" + "trn1 z1.h, z14.h, z6.h\n" + "add z1.h, z1.h, z17.h\n" + "ld1b { z15.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" "addvl x20, SP, #24\n" - "ld1b { z16.s }, p1/Z, [x23]\n" + "ld1b { z6.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - "trn1 z29.h, z17.h, z16.h\n" - "add z29.h, z29.h, z25.h\n" - "ld1b { z17.s }, p1/Z, [x23]\n" + "trn1 z2.h, z15.h, z6.h\n" + "add z2.h, z2.h, z17.h\n" + "ld1b { z21.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" "add x16, x16, %x[ld_in_col]\n" - ".inst 0xa1402ac0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x22]\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" - "ld1b { z16.s }, p1/Z, [x23]\n" - "trn1 z30.h, z17.h, z16.h\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" - "add z30.h, z30.h, z25.h\n" - ".inst 0xa0412ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xa0422aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc168776c // sdot za.s[x11, 4], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776d // sdot za.s[x11, 5], { z27.h-z28.h }, z0.h\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" + ".inst 0xa0402ace // ld1h { z14.h-z15.h }, pn10.b/Z, [x22]\n" + ".inst 0xc16f7408 // sdot za.s[x11, 0], { z0.h-z1.h }, z15.h\n" + "ld1b { z30.s }, p1/Z, [x23]\n" + "trn1 z3.h, z21.h, z30.h\n" + ".inst 0xc16e7409 // sdot za.s[x11, 1], { z0.h-z1.h }, z14.h\n" + ".inst 0xa1402aa5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x21]\n" + ".inst 0xc16d740a // sdot za.s[x11, 2], { z0.h-z1.h }, z13.h\n" + "add z3.h, z3.h, z17.h\n" + ".inst 0xa0412ace // ld1h { z14.h-z15.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc165740b // sdot za.s[x11, 3], { z0.h-z1.h }, z5.h\n" + ".inst 0xa0402a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20]\n" + ".inst 0xc16f7428 // sdot za.s[x11, 0], { z1.h-z2.h }, z15.h\n" + ".inst 0xc16e7429 // sdot za.s[x11, 1], { z1.h-z2.h }, z14.h\n" + ".inst 0xa0412aae // ld1h { z14.h-z15.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xa0422ac8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc16b740c // sdot za.s[x11, 4], { z0.h-z1.h }, z11.h\n" + ".inst 0xc16a740d // sdot za.s[x11, 5], { z0.h-z1.h }, z10.h\n" + ".inst 0xc16f742a // sdot za.s[x11, 2], { z1.h-z2.h }, z15.h\n" + ".inst 0xc16e742b // sdot za.s[x11, 3], { z1.h-z2.h }, z14.h\n" + ".inst 0xa0412a8e // ld1h { z14.h-z15.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc1697448 // sdot za.s[x11, 0], { z2.h-z3.h }, z9.h\n" + ".inst 0xc1687449 // sdot za.s[x11, 1], { z2.h-z3.h }, z8.h\n" ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc165778c // sdot za.s[x11, 4], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778d // sdot za.s[x11, 5], { z28.h-z29.h }, z4.h\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc16b77ac // sdot za.s[x11, 4], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ad // sdot za.s[x11, 5], { z29.h-z30.h }, z10.h\n" + ".inst 0xc16f742c // sdot za.s[x11, 4], { z1.h-z2.h }, z15.h\n" + ".inst 0xc16e742d // sdot za.s[x11, 5], { z1.h-z2.h }, z14.h\n" + ".inst 0xc16b744a // sdot za.s[x11, 2], { z2.h-z3.h }, z11.h\n" + ".inst 0xc16a744b // sdot za.s[x11, 3], { z2.h-z3.h }, z10.h\n" + ".inst 0xa0422a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc161744c // sdot za.s[x11, 4], { z2.h-z3.h }, z1.h\n" + ".inst 0xc160744d // sdot za.s[x11, 5], { z2.h-z3.h }, z0.h\n" "11:" // Unpadded: 1 priming loads "add x24, x16, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x16]\n" + "ld1b { z0.s }, p1/Z, [x16]\n" "addvl x23, SP, #6\n" - "ld1b { z16.s }, p1/Z, [x24]\n" + "ld1b { z3.s }, p1/Z, [x24]\n" "add x24, x24, %x[ld_in_row]\n" - "trn1 z27.h, z17.h, z16.h\n" - "add z27.h, z27.h, z25.h\n" - "ld1b { z17.s }, p1/Z, [x24]\n" + "trn1 z28.h, z0.h, z3.h\n" + "add z28.h, z28.h, z17.h\n" + "ld1b { z6.s }, p1/Z, [x24]\n" "add x24, x24, %x[ld_in_row]\n" "addvl x22, SP, #12\n" - "ld1b { z16.s }, p1/Z, [x24]\n" + "ld1b { z30.s }, p1/Z, [x24]\n" "add x24, x24, %x[ld_in_row]\n" - "trn1 z28.h, z17.h, z16.h\n" - "add z28.h, z28.h, z25.h\n" - "ld1b { z17.s }, p1/Z, [x24]\n" + "trn1 z29.h, z6.h, z30.h\n" + "add z29.h, z29.h, z17.h\n" + "ld1b { z1.s }, p1/Z, [x24]\n" "add x24, x24, %x[ld_in_row]\n" "addvl x21, SP, #18\n" - "ld1b { z16.s }, p1/Z, [x24]\n" + "ld1b { z25.s }, p1/Z, [x24]\n" "add x24, x24, %x[ld_in_row]\n" - "trn1 z29.h, z17.h, z16.h\n" - "add z29.h, z29.h, z25.h\n" - "ld1b { z17.s }, p1/Z, [x24]\n" + "trn1 z30.h, z1.h, z25.h\n" + "add z30.h, z30.h, z17.h\n" + "ld1b { z3.s }, p1/Z, [x24]\n" "add x24, x24, %x[ld_in_row]\n" "addvl x20, SP, #24\n" - ".inst 0xa1402ae0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x23]\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" + ".inst 0xa0402ae0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x23]\n" + ".inst 0xc1617788 // sdot za.s[x11, 0], { z28.h-z29.h }, z1.h\n" "add x16, x16, %x[ld_in_col]\n" - "ld1b { z16.s }, p1/Z, [x24]\n" - "trn1 z30.h, z17.h, z16.h\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402ac0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x22]\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" - "add z30.h, z30.h, z25.h\n" - ".inst 0xa0412ae4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xa0422aea // ld1h { z10.h-z11.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xc168776c // sdot za.s[x11, 4], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776d // sdot za.s[x11, 5], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc168776e // sdot za.s[x11, 6], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776f // sdot za.s[x11, 7], { z27.h-z28.h }, z0.h\n" - ".inst 0xc165778c // sdot za.s[x11, 4], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778d // sdot za.s[x11, 5], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc165778e // sdot za.s[x11, 6], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778f // sdot za.s[x11, 7], { z28.h-z29.h }, z4.h\n" - ".inst 0xc16b77ac // sdot za.s[x11, 4], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ad // sdot za.s[x11, 5], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc16b77ae // sdot za.s[x11, 6], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77af // sdot za.s[x11, 7], { z29.h-z30.h }, z10.h\n" + "ld1b { z5.s }, p1/Z, [x24]\n" + "trn1 z31.h, z3.h, z5.h\n" + ".inst 0xc1607789 // sdot za.s[x11, 1], { z28.h-z29.h }, z0.h\n" + ".inst 0xa1402ac6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x22]\n" + ".inst 0xc16e778a // sdot za.s[x11, 2], { z28.h-z29.h }, z14.h\n" + "add z31.h, z31.h, z17.h\n" + ".inst 0xa1412ae2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xc166778b // sdot za.s[x11, 3], { z28.h-z29.h }, z6.h\n" + ".inst 0xa0402aae // ld1h { z14.h-z15.h }, pn10.b/Z, [x21]\n" + ".inst 0xc16a77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z10.h\n" + ".inst 0xc16277a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z2.h\n" + ".inst 0xa0412ac8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xa1422ae2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xc16f778c // sdot za.s[x11, 4], { z28.h-z29.h }, z15.h\n" + ".inst 0xc16e778d // sdot za.s[x11, 5], { z28.h-z29.h }, z14.h\n" + ".inst 0xa1402a86 // ld1h { z6.h, z14.h }, pn10.b/Z, [x20]\n" + ".inst 0xc16977aa // sdot za.s[x11, 2], { z29.h-z30.h }, z9.h\n" + ".inst 0xc16877ab // sdot za.s[x11, 3], { z29.h-z30.h }, z8.h\n" + ".inst 0xa1412aa5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc16a77c8 // sdot za.s[x11, 0], { z30.h-z31.h }, z10.h\n" + ".inst 0xc16277c9 // sdot za.s[x11, 1], { z30.h-z31.h }, z2.h\n" + ".inst 0xa1422ac2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc16e778e // sdot za.s[x11, 6], { z28.h-z29.h }, z14.h\n" + ".inst 0xc166778f // sdot za.s[x11, 7], { z28.h-z29.h }, z6.h\n" + ".inst 0xc16d77ac // sdot za.s[x11, 4], { z29.h-z30.h }, z13.h\n" + ".inst 0xc16577ad // sdot za.s[x11, 5], { z29.h-z30.h }, z5.h\n" + ".inst 0xa1412a86 // ld1h { z6.h, z14.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc16a77ca // sdot za.s[x11, 2], { z30.h-z31.h }, z10.h\n" + ".inst 0xc16277cb // sdot za.s[x11, 3], { z30.h-z31.h }, z2.h\n" + ".inst 0xa0422aa8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc16e77ae // sdot za.s[x11, 6], { z29.h-z30.h }, z14.h\n" + ".inst 0xc16677af // sdot za.s[x11, 7], { z29.h-z30.h }, z6.h\n" + ".inst 0xc16977cc // sdot za.s[x11, 4], { z30.h-z31.h }, z9.h\n" + ".inst 0xc16877cd // sdot za.s[x11, 5], { z30.h-z31.h }, z8.h\n" + ".inst 0xa1422a86 // ld1h { z6.h, z14.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc16e77ce // sdot za.s[x11, 6], { z30.h-z31.h }, z14.h\n" + ".inst 0xc16677cf // sdot za.s[x11, 7], { z30.h-z31.h }, z6.h\n" "12:" // Unpadded: 0 priming loads - ".inst 0xa1402be0 // ld1h { z0.h, z8.h }, pn10.b/Z, [SP]\n" - ".inst 0xa0412be4 // ld1h { z4.h-z5.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" - ".inst 0xa0422bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" - "cbz x17, 22f\n" + ".inst 0xa0402be0 // ld1h { z0.h-z1.h }, pn10.b/Z, [SP]\n" + ".inst 0xa1412be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + ".inst 0xa0422be2 // ld1h { z2.h-z3.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" + "cbz x25, 22f\n" "add x20, x16, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x16]\n" - "sub x17, x17, #0x1\n" - "ld1b { z16.s }, p1/Z, [x20]\n" + "ld1b { z26.s }, p1/Z, [x16]\n" + "sub x25, x25, #0x1\n" + "ld1b { z28.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z27.h, z17.h, z16.h\n" + "trn1 z25.h, z26.h, z28.h\n" "sub x15, x15, #0x1\n" - "ld1b { z17.s }, p1/Z, [x20]\n" + "ld1b { z31.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "cmp x17, x15\n" - "add z27.h, z27.h, z25.h\n" - "ld1b { z16.s }, p1/Z, [x20]\n" + "cmp x25, x15\n" + "add z25.h, z25.h, z17.h\n" + "ld1b { z15.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z28.h, z17.h, z16.h\n" - "csel x25, x17, x15, LT\n" - "ld1b { z17.s }, p1/Z, [x20]\n" + "trn1 z26.h, z31.h, z15.h\n" + "csel x25, x25, x15, LT\n" + "ld1b { z22.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "add z28.h, z28.h, z25.h\n" + "add z26.h, z26.h, z17.h\n" "add x16, x16, %x[ld_in_col]\n" - "ld1b { z16.s }, p1/Z, [x20]\n" + "ld1b { z8.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z29.h, z17.h, z16.h\n" - "add z29.h, z29.h, z25.h\n" - "ld1b { z17.s }, p1/Z, [x20]\n" + "trn1 z27.h, z22.h, z8.h\n" + "add z27.h, z27.h, z17.h\n" + "ld1b { z21.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" "sub x15, x15, x25\n" - "ld1b { z16.s }, p1/Z, [x20]\n" - "trn1 z30.h, z17.h, z16.h\n" - "add z30.h, z30.h, z25.h\n" + "ld1b { z20.s }, p1/Z, [x20]\n" + "trn1 z28.h, z21.h, z20.h\n" + "add z28.h, z28.h, z17.h\n" "cbz x25, 21f\n" "13:" // Unpadded: Main loop "addvl x24, SP, #6\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" + ".inst 0xc1617728 // sdot za.s[x11, 0], { z25.h-z26.h }, z1.h\n" "addvl x23, SP, #12\n" - "ld1b { z23.s }, p1/Z, [x16]\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402b00 // ld1h { z0.h, z8.h }, pn10.b/Z, [x24]\n" + "ld1b { z21.s }, p1/Z, [x16]\n" + ".inst 0xc1607729 // sdot za.s[x11, 1], { z25.h-z26.h }, z0.h\n" + ".inst 0xa0402b0e // ld1h { z14.h-z15.h }, pn10.b/Z, [x24]\n" "addvl x22, SP, #18\n" "addvl x21, SP, #24\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" + ".inst 0xc16f772a // sdot za.s[x11, 2], { z25.h-z26.h }, z15.h\n" "add x20, x16, %x[ld_in_row]\n" - "ld1b { z22.s }, p1/Z, [x20]\n" + "ld1b { z0.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402ae0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x23]\n" + ".inst 0xc16e772b // sdot za.s[x11, 3], { z25.h-z26.h }, z14.h\n" + ".inst 0xa1402ae6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x23]\n" "subs x25, x25, #0x1\n" "add x16, x16, %x[ld_in_col]\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - "ld1b { z21.s }, p1/Z, [x20]\n" - "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412b04 // ld1h { z4.h-z5.h }, pn10.b/Z, [x24, #0x2, MUL VL]\n" - ".inst 0xc168776c // sdot za.s[x11, 4], { z27.h-z28.h }, z8.h\n" + ".inst 0xc16d7748 // sdot za.s[x11, 0], { z26.h-z27.h }, z13.h\n" "ld1b { z20.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc160776d // sdot za.s[x11, 5], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402ac0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x22]\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - "ld1b { z19.s }, p1/Z, [x20]\n" + ".inst 0xc1657749 // sdot za.s[x11, 1], { z26.h-z27.h }, z5.h\n" + ".inst 0xa1412b05 // ld1h { z5.h, z13.h }, pn10.b/Z, [x24, #0x2, MUL VL]\n" + ".inst 0xc16e772c // sdot za.s[x11, 4], { z25.h-z26.h }, z14.h\n" + "ld1b { z31.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412ae4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - "ld1b { z18.s }, p1/Z, [x20]\n" + ".inst 0xc166772d // sdot za.s[x11, 5], { z25.h-z26.h }, z6.h\n" + ".inst 0xa0402ace // ld1h { z14.h-z15.h }, pn10.b/Z, [x22]\n" + ".inst 0xc16d774a // sdot za.s[x11, 2], { z26.h-z27.h }, z13.h\n" + "ld1b { z29.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422b0a // ld1h { z10.h-z11.h }, pn10.b/Z, [x24, #0x4, MUL VL]\n" - ".inst 0xc168776e // sdot za.s[x11, 6], { z27.h-z28.h }, z8.h\n" - "ld1b { z17.s }, p1/Z, [x20]\n" + ".inst 0xc165774b // sdot za.s[x11, 3], { z26.h-z27.h }, z5.h\n" + ".inst 0xa1412ae5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xc1637768 // sdot za.s[x11, 0], { z27.h-z28.h }, z3.h\n" + "ld1b { z22.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc160776f // sdot za.s[x11, 7], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" - ".inst 0xc165778c // sdot za.s[x11, 4], { z28.h-z29.h }, z5.h\n" - "ld1b { z16.s }, p1/Z, [x20]\n" - ".inst 0xc164778d // sdot za.s[x11, 5], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aea // ld1h { z10.h-z11.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xc165778e // sdot za.s[x11, 6], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778f // sdot za.s[x11, 7], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc16b77ac // sdot za.s[x11, 4], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ad // sdot za.s[x11, 5], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc16b77ae // sdot za.s[x11, 6], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77af // sdot za.s[x11, 7], { z29.h-z30.h }, z10.h\n" + ".inst 0xc1627769 // sdot za.s[x11, 1], { z27.h-z28.h }, z2.h\n" + ".inst 0xa1422b02 // ld1h { z2.h, z10.h }, pn10.b/Z, [x24, #0x4, MUL VL]\n" + ".inst 0xc16f772e // sdot za.s[x11, 6], { z25.h-z26.h }, z15.h\n" + "ld1b { z30.s }, p1/Z, [x20]\n" + "add x20, x20, %x[ld_in_row]\n" + ".inst 0xc16e772f // sdot za.s[x11, 7], { z25.h-z26.h }, z14.h\n" + ".inst 0xa0402aae // ld1h { z14.h-z15.h }, pn10.b/Z, [x21]\n" + ".inst 0xc16d774c // sdot za.s[x11, 4], { z26.h-z27.h }, z13.h\n" + "ld1b { z6.s }, p1/Z, [x20]\n" + ".inst 0xc165774d // sdot za.s[x11, 5], { z26.h-z27.h }, z5.h\n" + ".inst 0xa1412ac5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc16a776a // sdot za.s[x11, 2], { z27.h-z28.h }, z10.h\n" + ".inst 0xc162776b // sdot za.s[x11, 3], { z27.h-z28.h }, z2.h\n" + ".inst 0xa1422ae2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xc16d774e // sdot za.s[x11, 6], { z26.h-z27.h }, z13.h\n" + ".inst 0xc165774f // sdot za.s[x11, 7], { z26.h-z27.h }, z5.h\n" + ".inst 0xa1412aa5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc16a776c // sdot za.s[x11, 4], { z27.h-z28.h }, z10.h\n" + ".inst 0xc162776d // sdot za.s[x11, 5], { z27.h-z28.h }, z2.h\n" + ".inst 0xa1422ac1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc169776e // sdot za.s[x11, 6], { z27.h-z28.h }, z9.h\n" + ".inst 0xc161776f // sdot za.s[x11, 7], { z27.h-z28.h }, z1.h\n" ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc1681768 // sdot za.s[x8, 0], { z27.h-z28.h }, z8.h\n" - ".inst 0xc1601769 // sdot za.s[x8, 1], { z27.h-z28.h }, z0.h\n" - "trn1 z27.h, z23.h, z22.h\n" - ".inst 0xa1402be0 // ld1h { z0.h, z8.h }, pn10.b/Z, [SP]\n" - ".inst 0xc1651788 // sdot za.s[x8, 0], { z28.h-z29.h }, z5.h\n" - "add z27.h, z27.h, z25.h\n" - ".inst 0xc1641789 // sdot za.s[x8, 1], { z28.h-z29.h }, z4.h\n" - "trn1 z28.h, z21.h, z20.h\n" - ".inst 0xa0412be4 // ld1h { z4.h-z5.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" - ".inst 0xc16b17a8 // sdot za.s[x8, 0], { z29.h-z30.h }, z11.h\n" - "add z28.h, z28.h, z25.h\n" - ".inst 0xc16a17a9 // sdot za.s[x8, 1], { z29.h-z30.h }, z10.h\n" - "trn1 z29.h, z19.h, z18.h\n" - "trn1 z30.h, z17.h, z16.h\n" + ".inst 0xc16f1728 // sdot za.s[x8, 0], { z25.h-z26.h }, z15.h\n" + ".inst 0xc16e1729 // sdot za.s[x8, 1], { z25.h-z26.h }, z14.h\n" + "trn1 z25.h, z21.h, z0.h\n" + ".inst 0xa0402be0 // ld1h { z0.h-z1.h }, pn10.b/Z, [SP]\n" + ".inst 0xc16d1748 // sdot za.s[x8, 0], { z26.h-z27.h }, z13.h\n" + "add z25.h, z25.h, z17.h\n" + ".inst 0xc1651749 // sdot za.s[x8, 1], { z26.h-z27.h }, z5.h\n" + "trn1 z26.h, z20.h, z31.h\n" + ".inst 0xa1412be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + ".inst 0xc16b1768 // sdot za.s[x8, 0], { z27.h-z28.h }, z11.h\n" + "add z26.h, z26.h, z17.h\n" + ".inst 0xc16a1769 // sdot za.s[x8, 1], { z27.h-z28.h }, z10.h\n" + "trn1 z27.h, z29.h, z22.h\n" + "trn1 z28.h, z30.h, z6.h\n" "add x8, x8, #0x2\n" - ".inst 0xc006680c // mova { z12.d-z13.d }, za.d[x11, #0]\n" - ".inst 0xa0422bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" - "add z29.h, z29.h, z25.h\n" - ".inst 0xc006682e // mova { z14.d-z15.d }, za.d[x11, #1]\n" - ".inst 0xc1a3ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z3.s\n" + ".inst 0xc0066808 // mova { z8.d-z9.d }, za.d[x11, #0]\n" + ".inst 0xa0422be2 // ld1h { z2.h-z3.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" + "add z27.h, z27.h, z17.h\n" + ".inst 0xc006682a // mova { z10.d-z11.d }, za.d[x11, #1]\n" + ".inst 0xc1a7ac08 // sqdmulh { z8.s-z11.s }, { z8.s-z11.s }, z7.s\n" "add x11, x11, #0x2\n" - ".inst 0xc1a1aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n" - ".inst 0xc00408c0 // mova za.d[x8, #0], { z6.d-z7.d }\n" - ".inst 0xc1a9ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z9.s\n" - ".inst 0xc00408c1 // mova za.d[x8, #1], { z6.d-z7.d }\n" - ".inst 0xc1bfcf0c // sclamp { z12.s-z15.s }, z24.s, z31.s\n" - "st1b { z12.s }, p1, [x14]\n" - "add x14, x14, x3\n" - "add z30.h, z30.h, z25.h\n" - "st1b { z14.s }, p1, [x13]\n" + ".inst 0xc1a4aa28 // srshl { z8.s-z11.s }, { z8.s-z11.s }, z4.s\n" + ".inst 0xc0040a40 // mova za.d[x8, #0], { z18.d-z19.d }\n" + ".inst 0xc1acab08 // add { z8.s-z11.s }, { z8.s-z11.s }, z12.s\n" + ".inst 0xc0040a41 // mova za.d[x8, #1], { z18.d-z19.d }\n" + ".inst 0xc1b0cf08 // sclamp { z8.s-z11.s }, z24.s, z16.s\n" + "st1b { z8.s }, p1, [x14]\n" + "add x14, x14, x4\n" + "add z28.h, z28.h, z17.h\n" + "st1b { z10.s }, p1, [x13]\n" "add x13, x13, x10\n" - "st1b { z13.s }, p1, [x9]\n" + "st1b { z9.s }, p1, [x9]\n" "add x9, x9, x27\n" - "st1b { z15.s }, p1, [x28]\n" + "st1b { z11.s }, p1, [x28]\n" "add x28, x28, x26\n" "bgt 13b\n" "b 21f\n" "14:" // Padded "cbz x22, 19f\n" "cmp x22, #0x1\n" - "sub x17, x17, x22\n" + "sub x25, x25, x22\n" "beq 18f\n" "cmp x22, #0x2\n" "beq 17f\n" @@ -668,515 +668,515 @@ void sme2_u8s8u8q_planar_5x5_s1_4rows_dot_za_impl( "15:" // Padded: 4 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z19.s }, p0/Z, [x16]\n" - "add z19.h, p0/M, z19.h, z25.h\n" + "ld1b { z9.s }, p0/Z, [x16]\n" + "add z9.h, p0/M, z9.h, z17.h\n" "add x21, x16, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z18.s }, p0/Z, [x21]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1b { z22.s }, p0/Z, [x21]\n" + "add z22.h, p0/M, z22.h, z17.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z17.s }, p0/Z, [x21]\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "ld1b { z21.s }, p0/Z, [x21]\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z16.s }, p0/Z, [x21]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1b { z20.s }, p0/Z, [x21]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "mov x12, #0x4\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z27.h, z19.h, z18.h\n" - "trn1 z28.h, z17.h, z16.h\n" + "trn1 z31.h, z9.h, z22.h\n" + "trn1 z0.h, z21.h, z20.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z18.s }, p0/Z, [x21]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1b { z22.s }, p0/Z, [x21]\n" + "add z22.h, p0/M, z22.h, z17.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z16.s }, p0/Z, [x21]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1b { z20.s }, p0/Z, [x21]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z17.s }, p0/Z, [x21]\n" + "ld1b { z21.s }, p0/Z, [x21]\n" "addvl x20, SP, #24\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - "trn1 z29.h, z18.h, z16.h\n" - "ld1b { z16.s }, p0/Z, [x21]\n" - "add z16.h, p0/M, z16.h, z25.h\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" + ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" + "trn1 z1.h, z22.h, z20.h\n" + "ld1b { z20.s }, p0/Z, [x21]\n" + "add z20.h, p0/M, z20.h, z17.h\n" + ".inst 0xc16a77e8 // sdot za.s[x11, 0], { z31.h-z0.h }, z10.h\n" "add x16, x16, %x[ld_in_col]\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - "trn1 z30.h, z17.h, z16.h\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" + ".inst 0xc16277e9 // sdot za.s[x11, 1], { z31.h-z0.h }, z2.h\n" + ".inst 0xa1412a85 // ld1h { z5.h, z13.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + "trn1 z2.h, z21.h, z20.h\n" + ".inst 0xc16d7408 // sdot za.s[x11, 0], { z0.h-z1.h }, z13.h\n" + ".inst 0xa0422a88 // ld1h { z8.h-z9.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc1657409 // sdot za.s[x11, 1], { z0.h-z1.h }, z5.h\n" + ".inst 0xc1697428 // sdot za.s[x11, 0], { z1.h-z2.h }, z9.h\n" + ".inst 0xc1687429 // sdot za.s[x11, 1], { z1.h-z2.h }, z8.h\n" "16:" // Padded: 3 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z19.s }, p0/Z, [x16]\n" - "add z19.h, p0/M, z19.h, z25.h\n" + "ld1b { z5.s }, p0/Z, [x16]\n" + "add z5.h, p0/M, z5.h, z17.h\n" "add x20, x16, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1b { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "ld1b { z21.s }, p0/Z, [x20]\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1b { z20.s }, p0/Z, [x20]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z27.h, z19.h, z18.h\n" - "trn1 z28.h, z17.h, z16.h\n" + "trn1 z28.h, z5.h, z22.h\n" + "trn1 z29.h, z21.h, z20.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1b { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1b { z20.s }, p0/Z, [x20]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" + "ld1b { z21.s }, p0/Z, [x20]\n" "addvl x21, SP, #18\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" - "trn1 z29.h, z18.h, z16.h\n" - "ld1b { z16.s }, p0/Z, [x20]\n" + ".inst 0xa0402aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21]\n" + "trn1 z30.h, z22.h, z20.h\n" + "ld1b { z20.s }, p0/Z, [x20]\n" "addvl x20, SP, #24\n" - "add z16.h, p0/M, z16.h, z25.h\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - "trn1 z30.h, z17.h, z16.h\n" + "add z20.h, p0/M, z20.h, z17.h\n" + ".inst 0xc1617788 // sdot za.s[x11, 0], { z28.h-z29.h }, z1.h\n" + ".inst 0xc1607789 // sdot za.s[x11, 1], { z28.h-z29.h }, z0.h\n" + ".inst 0xa1402a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20]\n" + "trn1 z31.h, z21.h, z20.h\n" "add x16, x16, %x[ld_in_col]\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" + ".inst 0xa0412aae // ld1h { z14.h-z15.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc169778a // sdot za.s[x11, 2], { z28.h-z29.h }, z9.h\n" + ".inst 0xc161778b // sdot za.s[x11, 3], { z28.h-z29.h }, z1.h\n" + ".inst 0xa1422aa3 // ld1h { z3.h, z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc16f77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z15.h\n" + ".inst 0xc16e77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z14.h\n" + ".inst 0xa1412a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc16977aa // sdot za.s[x11, 2], { z29.h-z30.h }, z9.h\n" + ".inst 0xc16177ab // sdot za.s[x11, 3], { z29.h-z30.h }, z1.h\n" + ".inst 0xc16b77c8 // sdot za.s[x11, 0], { z30.h-z31.h }, z11.h\n" + ".inst 0xc16377c9 // sdot za.s[x11, 1], { z30.h-z31.h }, z3.h\n" + ".inst 0xa0422a8e // ld1h { z14.h-z15.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc16f77ca // sdot za.s[x11, 2], { z30.h-z31.h }, z15.h\n" + ".inst 0xc16e77cb // sdot za.s[x11, 3], { z30.h-z31.h }, z14.h\n" "17:" // Padded: 2 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z19.s }, p0/Z, [x16]\n" - "add z19.h, p0/M, z19.h, z25.h\n" + "ld1b { z29.s }, p0/Z, [x16]\n" + "add z29.h, p0/M, z29.h, z17.h\n" "add x20, x16, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1b { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "ld1b { z21.s }, p0/Z, [x20]\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1b { z20.s }, p0/Z, [x20]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z27.h, z19.h, z18.h\n" - "trn1 z28.h, z17.h, z16.h\n" + "trn1 z8.h, z29.h, z22.h\n" + "trn1 z9.h, z21.h, z20.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1b { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1b { z20.s }, p0/Z, [x20]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" + "ld1b { z21.s }, p0/Z, [x20]\n" "addvl x22, SP, #12\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xa1402ac0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x22]\n" - "trn1 z29.h, z18.h, z16.h\n" - "ld1b { z16.s }, p0/Z, [x20]\n" + ".inst 0xa0402ace // ld1h { z14.h-z15.h }, pn10.b/Z, [x22]\n" + "trn1 z10.h, z22.h, z20.h\n" + "ld1b { z20.s }, p0/Z, [x20]\n" "addvl x21, SP, #18\n" - "add z16.h, p0/M, z16.h, z25.h\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" + "add z20.h, p0/M, z20.h, z17.h\n" + ".inst 0xc16f7508 // sdot za.s[x11, 0], { z8.h-z9.h }, z15.h\n" + ".inst 0xc16e7509 // sdot za.s[x11, 1], { z8.h-z9.h }, z14.h\n" + ".inst 0xa1402aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21]\n" "addvl x20, SP, #24\n" - "trn1 z30.h, z17.h, z16.h\n" - ".inst 0xa0412ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" + "trn1 z11.h, z21.h, z20.h\n" + ".inst 0xa1412ac5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc16e750a // sdot za.s[x11, 2], { z8.h-z9.h }, z14.h\n" "add x16, x16, %x[ld_in_col]\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xa0422aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc168776c // sdot za.s[x11, 4], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776d // sdot za.s[x11, 5], { z27.h-z28.h }, z0.h\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc165778c // sdot za.s[x11, 4], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778d // sdot za.s[x11, 5], { z28.h-z29.h }, z4.h\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc16b77ac // sdot za.s[x11, 4], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ad // sdot za.s[x11, 5], { z29.h-z30.h }, z10.h\n" + ".inst 0xc166750b // sdot za.s[x11, 3], { z8.h-z9.h }, z6.h\n" + ".inst 0xa0402a8e // ld1h { z14.h-z15.h }, pn10.b/Z, [x20]\n" + ".inst 0xc16d7528 // sdot za.s[x11, 0], { z9.h-z10.h }, z13.h\n" + ".inst 0xa0422ac0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc1657529 // sdot za.s[x11, 1], { z9.h-z10.h }, z5.h\n" + ".inst 0xa1412aa5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc16f750c // sdot za.s[x11, 4], { z8.h-z9.h }, z15.h\n" + ".inst 0xc16e750d // sdot za.s[x11, 5], { z8.h-z9.h }, z14.h\n" + ".inst 0xc16d752a // sdot za.s[x11, 2], { z9.h-z10.h }, z13.h\n" + ".inst 0xc165752b // sdot za.s[x11, 3], { z9.h-z10.h }, z5.h\n" + ".inst 0xa1412a86 // ld1h { z6.h, z14.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc1617548 // sdot za.s[x11, 0], { z10.h-z11.h }, z1.h\n" + ".inst 0xc1607549 // sdot za.s[x11, 1], { z10.h-z11.h }, z0.h\n" + ".inst 0xa0422aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc16e752c // sdot za.s[x11, 4], { z9.h-z10.h }, z14.h\n" + ".inst 0xc166752d // sdot za.s[x11, 5], { z9.h-z10.h }, z6.h\n" + ".inst 0xc161754a // sdot za.s[x11, 2], { z10.h-z11.h }, z1.h\n" + ".inst 0xc160754b // sdot za.s[x11, 3], { z10.h-z11.h }, z0.h\n" + ".inst 0xa0422a8e // ld1h { z14.h-z15.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc16f754c // sdot za.s[x11, 4], { z10.h-z11.h }, z15.h\n" + ".inst 0xc16e754d // sdot za.s[x11, 5], { z10.h-z11.h }, z14.h\n" "18:" // Padded: 1 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z19.s }, p0/Z, [x16]\n" - "add z19.h, p0/M, z19.h, z25.h\n" + "ld1b { z1.s }, p0/Z, [x16]\n" + "add z1.h, p0/M, z1.h, z17.h\n" "add x20, x16, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1b { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "ld1b { z21.s }, p0/Z, [x20]\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1b { z20.s }, p0/Z, [x20]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z27.h, z19.h, z18.h\n" - "trn1 z28.h, z17.h, z16.h\n" + "trn1 z26.h, z1.h, z22.h\n" + "trn1 z27.h, z21.h, z20.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1b { z22.s }, p0/Z, [x20]\n" + "add z22.h, p0/M, z22.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1b { z20.s }, p0/Z, [x20]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" + "ld1b { z21.s }, p0/Z, [x20]\n" "addvl x23, SP, #6\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xa1402ae0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x23]\n" - "trn1 z29.h, z18.h, z16.h\n" - "ld1b { z16.s }, p0/Z, [x20]\n" + ".inst 0xa0402aee // ld1h { z14.h-z15.h }, pn10.b/Z, [x23]\n" + "trn1 z28.h, z22.h, z20.h\n" + "ld1b { z20.s }, p0/Z, [x20]\n" "addvl x22, SP, #12\n" - "add z16.h, p0/M, z16.h, z25.h\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402ac0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x22]\n" + "add z20.h, p0/M, z20.h, z17.h\n" + ".inst 0xc16f7748 // sdot za.s[x11, 0], { z26.h-z27.h }, z15.h\n" + ".inst 0xc16e7749 // sdot za.s[x11, 1], { z26.h-z27.h }, z14.h\n" + ".inst 0xa0402ac0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x22]\n" "addvl x21, SP, #18\n" - "trn1 z30.h, z17.h, z16.h\n" - ".inst 0xa0412ae4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" + "trn1 z29.h, z21.h, z20.h\n" + ".inst 0xa0412aea // ld1h { z10.h-z11.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xc161774a // sdot za.s[x11, 2], { z26.h-z27.h }, z1.h\n" "addvl x20, SP, #24\n" "add x16, x16, %x[ld_in_col]\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xa0422aea // ld1h { z10.h-z11.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xc168776c // sdot za.s[x11, 4], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776d // sdot za.s[x11, 5], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc168776e // sdot za.s[x11, 6], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776f // sdot za.s[x11, 7], { z27.h-z28.h }, z0.h\n" - ".inst 0xc165778c // sdot za.s[x11, 4], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778d // sdot za.s[x11, 5], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc165778e // sdot za.s[x11, 6], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778f // sdot za.s[x11, 7], { z28.h-z29.h }, z4.h\n" - ".inst 0xc16b77ac // sdot za.s[x11, 4], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ad // sdot za.s[x11, 5], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc16b77ae // sdot za.s[x11, 6], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77af // sdot za.s[x11, 7], { z29.h-z30.h }, z10.h\n" + ".inst 0xc160774b // sdot za.s[x11, 3], { z26.h-z27.h }, z0.h\n" + ".inst 0xa1402aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21]\n" + ".inst 0xc16b7768 // sdot za.s[x11, 0], { z27.h-z28.h }, z11.h\n" + ".inst 0xa0422ae8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xc16a7769 // sdot za.s[x11, 1], { z27.h-z28.h }, z10.h\n" + ".inst 0xa0412aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc16e774c // sdot za.s[x11, 4], { z26.h-z27.h }, z14.h\n" + ".inst 0xc166774d // sdot za.s[x11, 5], { z26.h-z27.h }, z6.h\n" + ".inst 0xa1402a85 // ld1h { z5.h, z13.h }, pn10.b/Z, [x20]\n" + ".inst 0xc16b776a // sdot za.s[x11, 2], { z27.h-z28.h }, z11.h\n" + ".inst 0xc16a776b // sdot za.s[x11, 3], { z27.h-z28.h }, z10.h\n" + ".inst 0xa1412aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc1697788 // sdot za.s[x11, 0], { z28.h-z29.h }, z9.h\n" + ".inst 0xc1687789 // sdot za.s[x11, 1], { z28.h-z29.h }, z8.h\n" + ".inst 0xa1422ac2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc16d774e // sdot za.s[x11, 6], { z26.h-z27.h }, z13.h\n" + ".inst 0xc165774f // sdot za.s[x11, 7], { z26.h-z27.h }, z5.h\n" + ".inst 0xc16e776c // sdot za.s[x11, 4], { z27.h-z28.h }, z14.h\n" + ".inst 0xc166776d // sdot za.s[x11, 5], { z27.h-z28.h }, z6.h\n" + ".inst 0xa1412a86 // ld1h { z6.h, z14.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc16a778a // sdot za.s[x11, 2], { z28.h-z29.h }, z10.h\n" + ".inst 0xc162778b // sdot za.s[x11, 3], { z28.h-z29.h }, z2.h\n" + ".inst 0xa0422aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc16e776e // sdot za.s[x11, 6], { z27.h-z28.h }, z14.h\n" + ".inst 0xc166776f // sdot za.s[x11, 7], { z27.h-z28.h }, z6.h\n" + ".inst 0xc161778c // sdot za.s[x11, 4], { z28.h-z29.h }, z1.h\n" + ".inst 0xc160778d // sdot za.s[x11, 5], { z28.h-z29.h }, z0.h\n" + ".inst 0xa1422a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc16a778e // sdot za.s[x11, 6], { z28.h-z29.h }, z10.h\n" + ".inst 0xc162778f // sdot za.s[x11, 7], { z28.h-z29.h }, z2.h\n" "19:" // Padded: 0 priming loads - ".inst 0xa1402be0 // ld1h { z0.h, z8.h }, pn10.b/Z, [SP]\n" - ".inst 0xa0412be4 // ld1h { z4.h-z5.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" - ".inst 0xa0422bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" - "cbz x17, 22f\n" + ".inst 0xa0402be0 // ld1h { z0.h-z1.h }, pn10.b/Z, [SP]\n" + ".inst 0xa1412be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + ".inst 0xa0422be2 // ld1h { z2.h-z3.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" + "cbz x25, 22f\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z19.s }, p0/Z, [x16]\n" - "add z19.h, p0/M, z19.h, z25.h\n" + "ld1b { z6.s }, p0/Z, [x16]\n" + "add z6.h, p0/M, z6.h, z17.h\n" "add x20, x16, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1b { z30.s }, p0/Z, [x20]\n" + "add z30.h, p0/M, z30.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "ld1b { z27.s }, p0/Z, [x20]\n" + "add z27.h, p0/M, z27.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z25.h\n" + "ld1b { z26.s }, p0/Z, [x20]\n" + "add z26.h, p0/M, z26.h, z17.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z27.h, z19.h, z18.h\n" - "trn1 z28.h, z17.h, z16.h\n" + "trn1 z25.h, z6.h, z30.h\n" + "trn1 z26.h, z27.h, z26.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z25.h\n" + "ld1b { z8.s }, p0/Z, [x20]\n" + "add z8.h, p0/M, z8.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + "ld1b { z9.s }, p0/Z, [x20]\n" + "add z9.h, p0/M, z9.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z25.h\n" + "ld1b { z21.s }, p0/Z, [x20]\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z25.h\n" - "sub x17, x17, #0x1\n" + "ld1b { z29.s }, p0/Z, [x20]\n" + "add z29.h, p0/M, z29.h, z17.h\n" + "sub x25, x25, #0x1\n" "sub x15, x15, #0x1\n" - "cmp x17, x15\n" - "trn1 z29.h, z19.h, z18.h\n" - "trn1 z30.h, z17.h, z16.h\n" - "csel x25, x17, x15, LT\n" + "cmp x25, x15\n" + "trn1 z27.h, z8.h, z9.h\n" + "trn1 z28.h, z21.h, z29.h\n" + "csel x25, x25, x15, LT\n" "add x16, x16, %x[ld_in_col]\n" "sub x15, x15, x25\n" "cbz x25, 21f\n" "20:" // Padded: Main loop "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z23.s }, p0/Z, [x16]\n" - "add z23.h, p0/M, z23.h, z25.h\n" + "ld1b { z8.s }, p0/Z, [x16]\n" + "add z8.h, p0/M, z8.h, z17.h\n" "add x24, x16, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z22.s }, p0/Z, [x24]\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" + "ld1b { z21.s }, p0/Z, [x24]\n" + ".inst 0xc1617728 // sdot za.s[x11, 0], { z25.h-z26.h }, z1.h\n" "addvl x23, SP, #6\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402ae0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x23]\n" + ".inst 0xc1607729 // sdot za.s[x11, 1], { z25.h-z26.h }, z0.h\n" + ".inst 0xa0402ae0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x23]\n" "addvl x22, SP, #12\n" - "add z22.h, p0/M, z22.h, z25.h\n" + "add z21.h, p0/M, z21.h, z17.h\n" "add x24, x24, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402ac0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x22]\n" + ".inst 0xc161772a // sdot za.s[x11, 2], { z25.h-z26.h }, z1.h\n" + ".inst 0xc160772b // sdot za.s[x11, 3], { z25.h-z26.h }, z0.h\n" + ".inst 0xa1402ac6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x22]\n" "addvl x21, SP, #18\n" "addvl x20, SP, #24\n" - "ld1b { z21.s }, p0/Z, [x24]\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - "add z21.h, p0/M, z21.h, z25.h\n" + "ld1b { z29.s }, p0/Z, [x24]\n" + ".inst 0xc16d7748 // sdot za.s[x11, 0], { z26.h-z27.h }, z13.h\n" + "add z29.h, p0/M, z29.h, z17.h\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412ae4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xc1657749 // sdot za.s[x11, 1], { z26.h-z27.h }, z5.h\n" + ".inst 0xa1412ae5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" "mov x12, #0x4\n" "add x24, x24, %x[ld_in_row]\n" - ".inst 0xc168776c // sdot za.s[x11, 4], { z27.h-z28.h }, z8.h\n" - "ld1b { z20.s }, p0/Z, [x24]\n" - "add z20.h, p0/M, z20.h, z25.h\n" + ".inst 0xc16e772c // sdot za.s[x11, 4], { z25.h-z26.h }, z14.h\n" + "ld1b { z30.s }, p0/Z, [x24]\n" + "add z30.h, p0/M, z30.h, z17.h\n" "add x24, x24, %x[ld_in_row]\n" - ".inst 0xc160776d // sdot za.s[x11, 5], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" + ".inst 0xc166772d // sdot za.s[x11, 5], { z25.h-z26.h }, z6.h\n" + ".inst 0xa1402aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" "subs x25, x25, #0x1\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - "ld1b { z19.s }, p0/Z, [x24]\n" - "add z19.h, p0/M, z19.h, z25.h\n" + ".inst 0xc16d774a // sdot za.s[x11, 2], { z26.h-z27.h }, z13.h\n" + "ld1b { z15.s }, p0/Z, [x24]\n" + "add z15.h, p0/M, z15.h, z17.h\n" "add x24, x24, %x[ld_in_row]\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc165774b // sdot za.s[x11, 3], { z26.h-z27.h }, z5.h\n" + ".inst 0xa0412aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "add x16, x16, %x[ld_in_col]\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - "ld1b { z18.s }, p0/Z, [x24]\n" - "add z18.h, p0/M, z18.h, z25.h\n" + ".inst 0xc1637768 // sdot za.s[x11, 0], { z27.h-z28.h }, z3.h\n" + "ld1b { z20.s }, p0/Z, [x24]\n" + "add z20.h, p0/M, z20.h, z17.h\n" "add x24, x24, %x[ld_in_row]\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aea // ld1h { z10.h-z11.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xc1627769 // sdot za.s[x11, 1], { z27.h-z28.h }, z2.h\n" + ".inst 0xa1422ae1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc168776e // sdot za.s[x11, 6], { z27.h-z28.h }, z8.h\n" - "ld1b { z17.s }, p0/Z, [x24]\n" - "add z17.h, p0/M, z17.h, z25.h\n" + ".inst 0xc16e772e // sdot za.s[x11, 6], { z25.h-z26.h }, z14.h\n" + "ld1b { z31.s }, p0/Z, [x24]\n" + "add z31.h, p0/M, z31.h, z17.h\n" "add x24, x24, %x[ld_in_row]\n" - ".inst 0xc160776f // sdot za.s[x11, 7], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" + ".inst 0xc166772f // sdot za.s[x11, 7], { z25.h-z26.h }, z6.h\n" + ".inst 0xa0402a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - ".inst 0xc165778c // sdot za.s[x11, 4], { z28.h-z29.h }, z5.h\n" - "ld1b { z16.s }, p0/Z, [x24]\n" - "add z16.h, p0/M, z16.h, z25.h\n" - ".inst 0xc164778d // sdot za.s[x11, 5], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc165778e // sdot za.s[x11, 6], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778f // sdot za.s[x11, 7], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16b77ac // sdot za.s[x11, 4], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ad // sdot za.s[x11, 5], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc16b77ae // sdot za.s[x11, 6], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77af // sdot za.s[x11, 7], { z29.h-z30.h }, z10.h\n" + ".inst 0xc16b774c // sdot za.s[x11, 4], { z26.h-z27.h }, z11.h\n" + "ld1b { z22.s }, p0/Z, [x24]\n" + "add z22.h, p0/M, z22.h, z17.h\n" + ".inst 0xc16a774d // sdot za.s[x11, 5], { z26.h-z27.h }, z10.h\n" + ".inst 0xa1412aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc169776a // sdot za.s[x11, 2], { z27.h-z28.h }, z9.h\n" + ".inst 0xc161776b // sdot za.s[x11, 3], { z27.h-z28.h }, z1.h\n" + ".inst 0xa0422ac0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc16e774e // sdot za.s[x11, 6], { z26.h-z27.h }, z14.h\n" + ".inst 0xc166774f // sdot za.s[x11, 7], { z26.h-z27.h }, z6.h\n" + ".inst 0xa1412a86 // ld1h { z6.h, z14.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc161776c // sdot za.s[x11, 4], { z27.h-z28.h }, z1.h\n" + ".inst 0xc160776d // sdot za.s[x11, 5], { z27.h-z28.h }, z0.h\n" + ".inst 0xa1422aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc169776e // sdot za.s[x11, 6], { z27.h-z28.h }, z9.h\n" + ".inst 0xc161776f // sdot za.s[x11, 7], { z27.h-z28.h }, z1.h\n" ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc1681768 // sdot za.s[x8, 0], { z27.h-z28.h }, z8.h\n" - ".inst 0xc1601769 // sdot za.s[x8, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402be0 // ld1h { z0.h, z8.h }, pn10.b/Z, [SP]\n" - "trn1 z27.h, z23.h, z22.h\n" - ".inst 0xc1651788 // sdot za.s[x8, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xc1641789 // sdot za.s[x8, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412be4 // ld1h { z4.h-z5.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" - "trn1 z28.h, z21.h, z20.h\n" - ".inst 0xc16b17a8 // sdot za.s[x8, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a17a9 // sdot za.s[x8, 1], { z29.h-z30.h }, z10.h\n" + ".inst 0xc1631728 // sdot za.s[x8, 0], { z25.h-z26.h }, z3.h\n" + ".inst 0xc1621729 // sdot za.s[x8, 1], { z25.h-z26.h }, z2.h\n" + ".inst 0xa0402be0 // ld1h { z0.h-z1.h }, pn10.b/Z, [SP]\n" + "trn1 z25.h, z8.h, z21.h\n" + ".inst 0xc16e1748 // sdot za.s[x8, 0], { z26.h-z27.h }, z14.h\n" + ".inst 0xc1661749 // sdot za.s[x8, 1], { z26.h-z27.h }, z6.h\n" + ".inst 0xa1412be5 // ld1h { z5.h, z13.h }, pn10.b/Z, [SP, #0x2, MUL VL]\n" + "trn1 z26.h, z29.h, z30.h\n" + ".inst 0xc16b1768 // sdot za.s[x8, 0], { z27.h-z28.h }, z11.h\n" + ".inst 0xc16a1769 // sdot za.s[x8, 1], { z27.h-z28.h }, z10.h\n" "add x8, x8, #0x2\n" - ".inst 0xa0422bea // ld1h { z10.h-z11.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" - "trn1 z29.h, z19.h, z18.h\n" - ".inst 0xc006680c // mova { z12.d-z13.d }, za.d[x11, #0]\n" - "trn1 z30.h, z17.h, z16.h\n" - ".inst 0xc006682e // mova { z14.d-z15.d }, za.d[x11, #1]\n" - ".inst 0xc1a3ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z3.s\n" + ".inst 0xa0422be2 // ld1h { z2.h-z3.h }, pn10.b/Z, [SP, #0x4, MUL VL]\n" + "trn1 z27.h, z15.h, z20.h\n" + ".inst 0xc0066808 // mova { z8.d-z9.d }, za.d[x11, #0]\n" + "trn1 z28.h, z31.h, z22.h\n" + ".inst 0xc006682a // mova { z10.d-z11.d }, za.d[x11, #1]\n" + ".inst 0xc1a7ac08 // sqdmulh { z8.s-z11.s }, { z8.s-z11.s }, z7.s\n" "add x11, x11, #0x2\n" - ".inst 0xc1a1aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n" - ".inst 0xc00408c0 // mova za.d[x8, #0], { z6.d-z7.d }\n" - ".inst 0xc1a9ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z9.s\n" - ".inst 0xc00408c1 // mova za.d[x8, #1], { z6.d-z7.d }\n" - ".inst 0xc1bfcf0c // sclamp { z12.s-z15.s }, z24.s, z31.s\n" - "st1b { z12.s }, p1, [x14]\n" - "add x14, x14, x3\n" - "st1b { z14.s }, p1, [x13]\n" + ".inst 0xc1a4aa28 // srshl { z8.s-z11.s }, { z8.s-z11.s }, z4.s\n" + ".inst 0xc0040a40 // mova za.d[x8, #0], { z18.d-z19.d }\n" + ".inst 0xc1acab08 // add { z8.s-z11.s }, { z8.s-z11.s }, z12.s\n" + ".inst 0xc0040a41 // mova za.d[x8, #1], { z18.d-z19.d }\n" + ".inst 0xc1b0cf08 // sclamp { z8.s-z11.s }, z24.s, z16.s\n" + "st1b { z8.s }, p1, [x14]\n" + "add x14, x14, x4\n" + "st1b { z10.s }, p1, [x13]\n" "add x13, x13, x10\n" - "st1b { z13.s }, p1, [x9]\n" + "st1b { z9.s }, p1, [x9]\n" "add x9, x9, x27\n" - "st1b { z15.s }, p1, [x28]\n" + "st1b { z11.s }, p1, [x28]\n" "add x28, x28, x26\n" "bgt 20b\n" "21:" // Main loop tail "addvl x23, SP, #6\n" - ".inst 0xc1687768 // sdot za.s[x11, 0], { z27.h-z28.h }, z8.h\n" + ".inst 0xc1617728 // sdot za.s[x11, 0], { z25.h-z26.h }, z1.h\n" "addvl x22, SP, #12\n" - ".inst 0xc1607769 // sdot za.s[x11, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402ae0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x23]\n" + ".inst 0xc1607729 // sdot za.s[x11, 1], { z25.h-z26.h }, z0.h\n" + ".inst 0xa0402ae0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x23]\n" "addvl x21, SP, #18\n" "addvl x20, SP, #24\n" - ".inst 0xc168776a // sdot za.s[x11, 2], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776b // sdot za.s[x11, 3], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402ac0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x22]\n" - ".inst 0xc1657788 // sdot za.s[x11, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xc1647789 // sdot za.s[x11, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412ae4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0xc168776c // sdot za.s[x11, 4], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776d // sdot za.s[x11, 5], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402aa0 // ld1h { z0.h, z8.h }, pn10.b/Z, [x21]\n" - ".inst 0xc165778a // sdot za.s[x11, 2], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778b // sdot za.s[x11, 3], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412ac4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" - ".inst 0xc16b77a8 // sdot za.s[x11, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77a9 // sdot za.s[x11, 1], { z29.h-z30.h }, z10.h\n" + ".inst 0xc161772a // sdot za.s[x11, 2], { z25.h-z26.h }, z1.h\n" + ".inst 0xc160772b // sdot za.s[x11, 3], { z25.h-z26.h }, z0.h\n" + ".inst 0xa1402ac6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x22]\n" + ".inst 0xc16d7748 // sdot za.s[x11, 0], { z26.h-z27.h }, z13.h\n" + ".inst 0xc1657749 // sdot za.s[x11, 1], { z26.h-z27.h }, z5.h\n" + ".inst 0xa1412ae1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xc16e772c // sdot za.s[x11, 4], { z25.h-z26.h }, z14.h\n" + ".inst 0xc166772d // sdot za.s[x11, 5], { z25.h-z26.h }, z6.h\n" + ".inst 0xa1402aa6 // ld1h { z6.h, z14.h }, pn10.b/Z, [x21]\n" + ".inst 0xc169774a // sdot za.s[x11, 2], { z26.h-z27.h }, z9.h\n" + ".inst 0xc161774b // sdot za.s[x11, 3], { z26.h-z27.h }, z1.h\n" + ".inst 0xa1412ac1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x22, #0x2, MUL VL]\n" + ".inst 0xc1637768 // sdot za.s[x11, 0], { z27.h-z28.h }, z3.h\n" + ".inst 0xc1627769 // sdot za.s[x11, 1], { z27.h-z28.h }, z2.h\n" ".inst 0xa0422aea // ld1h { z10.h-z11.h }, pn10.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xc168776e // sdot za.s[x11, 6], { z27.h-z28.h }, z8.h\n" - ".inst 0xc160776f // sdot za.s[x11, 7], { z27.h-z28.h }, z0.h\n" - ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" - ".inst 0xc165778c // sdot za.s[x11, 4], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778d // sdot za.s[x11, 5], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412aa4 // ld1h { z4.h-z5.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xc16b77aa // sdot za.s[x11, 2], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ab // sdot za.s[x11, 3], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aca // ld1h { z10.h-z11.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" - ".inst 0xc165778e // sdot za.s[x11, 6], { z28.h-z29.h }, z5.h\n" - ".inst 0xc164778f // sdot za.s[x11, 7], { z28.h-z29.h }, z4.h\n" - ".inst 0xa0412a84 // ld1h { z4.h-z5.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" - ".inst 0xc16b77ac // sdot za.s[x11, 4], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77ad // sdot za.s[x11, 5], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422aaa // ld1h { z10.h-z11.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xc16b77ae // sdot za.s[x11, 6], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a77af // sdot za.s[x11, 7], { z29.h-z30.h }, z10.h\n" - ".inst 0xa0422a8a // ld1h { z10.h-z11.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" - ".inst 0xc1681768 // sdot za.s[x8, 0], { z27.h-z28.h }, z8.h\n" + ".inst 0xc16e772e // sdot za.s[x11, 6], { z25.h-z26.h }, z14.h\n" + ".inst 0xc166772f // sdot za.s[x11, 7], { z25.h-z26.h }, z6.h\n" + ".inst 0xa0402a8e // ld1h { z14.h-z15.h }, pn10.b/Z, [x20]\n" + ".inst 0xc169774c // sdot za.s[x11, 4], { z26.h-z27.h }, z9.h\n" + ".inst 0xc161774d // sdot za.s[x11, 5], { z26.h-z27.h }, z1.h\n" + ".inst 0xa1412aa5 // ld1h { z5.h, z13.h }, pn10.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xc16b776a // sdot za.s[x11, 2], { z27.h-z28.h }, z11.h\n" + ".inst 0xc16a776b // sdot za.s[x11, 3], { z27.h-z28.h }, z10.h\n" + ".inst 0xa0422ac2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x22, #0x4, MUL VL]\n" + ".inst 0xc16d774e // sdot za.s[x11, 6], { z26.h-z27.h }, z13.h\n" + ".inst 0xc165774f // sdot za.s[x11, 7], { z26.h-z27.h }, z5.h\n" + ".inst 0xa0412a88 // ld1h { z8.h-z9.h }, pn10.b/Z, [x20, #0x2, MUL VL]\n" + ".inst 0xc163776c // sdot za.s[x11, 4], { z27.h-z28.h }, z3.h\n" + ".inst 0xc162776d // sdot za.s[x11, 5], { z27.h-z28.h }, z2.h\n" + ".inst 0xa1422aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xc16a776e // sdot za.s[x11, 6], { z27.h-z28.h }, z10.h\n" + ".inst 0xc162776f // sdot za.s[x11, 7], { z27.h-z28.h }, z2.h\n" + ".inst 0xa0422a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20, #0x4, MUL VL]\n" + ".inst 0xc16f1728 // sdot za.s[x8, 0], { z25.h-z26.h }, z15.h\n" + ".inst 0xc16e1729 // sdot za.s[x8, 1], { z25.h-z26.h }, z14.h\n" + ".inst 0xc1691748 // sdot za.s[x8, 0], { z26.h-z27.h }, z9.h\n" + ".inst 0xc1681749 // sdot za.s[x8, 1], { z26.h-z27.h }, z8.h\n" + ".inst 0xc1611768 // sdot za.s[x8, 0], { z27.h-z28.h }, z1.h\n" ".inst 0xc1601769 // sdot za.s[x8, 1], { z27.h-z28.h }, z0.h\n" - ".inst 0xc1651788 // sdot za.s[x8, 0], { z28.h-z29.h }, z5.h\n" - ".inst 0xc1641789 // sdot za.s[x8, 1], { z28.h-z29.h }, z4.h\n" - ".inst 0xc16b17a8 // sdot za.s[x8, 0], { z29.h-z30.h }, z11.h\n" - ".inst 0xc16a17a9 // sdot za.s[x8, 1], { z29.h-z30.h }, z10.h\n" "add x8, x8, #0x2\n" - ".inst 0xc006680c // mova { z12.d-z13.d }, za.d[x11, #0]\n" - ".inst 0xc006682e // mova { z14.d-z15.d }, za.d[x11, #1]\n" - ".inst 0xc1a3ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z3.s\n" + ".inst 0xc0066808 // mova { z8.d-z9.d }, za.d[x11, #0]\n" + ".inst 0xc006682a // mova { z10.d-z11.d }, za.d[x11, #1]\n" + ".inst 0xc1a7ac08 // sqdmulh { z8.s-z11.s }, { z8.s-z11.s }, z7.s\n" "add x11, x11, #0x2\n" - ".inst 0xc1a1aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n" - ".inst 0xc00408c0 // mova za.d[x8, #0], { z6.d-z7.d }\n" - ".inst 0xc1a9ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z9.s\n" - ".inst 0xc00408c1 // mova za.d[x8, #1], { z6.d-z7.d }\n" - ".inst 0xc1bfcf0c // sclamp { z12.s-z15.s }, z24.s, z31.s\n" - "st1b { z12.s }, p1, [x14]\n" - "add x14, x14, x3\n" - "st1b { z14.s }, p1, [x13]\n" + ".inst 0xc1a4aa28 // srshl { z8.s-z11.s }, { z8.s-z11.s }, z4.s\n" + ".inst 0xc0040a40 // mova za.d[x8, #0], { z18.d-z19.d }\n" + ".inst 0xc1acab08 // add { z8.s-z11.s }, { z8.s-z11.s }, z12.s\n" + ".inst 0xc0040a41 // mova za.d[x8, #1], { z18.d-z19.d }\n" + ".inst 0xc1b0cf08 // sclamp { z8.s-z11.s }, z24.s, z16.s\n" + "st1b { z8.s }, p1, [x14]\n" + "add x14, x14, x4\n" + "st1b { z10.s }, p1, [x13]\n" "add x13, x13, x10\n" - "st1b { z13.s }, p1, [x9]\n" + "st1b { z9.s }, p1, [x9]\n" "add x9, x9, x27\n" - "st1b { z15.s }, p1, [x28]\n" + "st1b { z11.s }, p1, [x28]\n" "add x28, x28, x26\n" "22:" // Main loop skip tail "cbz x15, 24f\n" "23:" // Right padding loop - ".inst 0xc006680c // mova { z12.d-z13.d }, za.d[x11, #0]\n" + ".inst 0xc0066808 // mova { z8.d-z9.d }, za.d[x11, #0]\n" "add x8, x8, #0x2\n" "subs x15, x15, #0x1\n" - ".inst 0xc006682e // mova { z14.d-z15.d }, za.d[x11, #1]\n" - ".inst 0xc1a3ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z3.s\n" + ".inst 0xc006682a // mova { z10.d-z11.d }, za.d[x11, #1]\n" + ".inst 0xc1a7ac08 // sqdmulh { z8.s-z11.s }, { z8.s-z11.s }, z7.s\n" "add x11, x11, #0x2\n" - ".inst 0xc1a1aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n" - ".inst 0xc00408c0 // mova za.d[x8, #0], { z6.d-z7.d }\n" - ".inst 0xc1a9ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z9.s\n" - ".inst 0xc00408c1 // mova za.d[x8, #1], { z6.d-z7.d }\n" - ".inst 0xc1bfcf0c // sclamp { z12.s-z15.s }, z24.s, z31.s\n" - "st1b { z12.s }, p1, [x14]\n" - "add x14, x14, x3\n" - "st1b { z14.s }, p1, [x13]\n" + ".inst 0xc1a4aa28 // srshl { z8.s-z11.s }, { z8.s-z11.s }, z4.s\n" + ".inst 0xc0040a40 // mova za.d[x8, #0], { z18.d-z19.d }\n" + ".inst 0xc1acab08 // add { z8.s-z11.s }, { z8.s-z11.s }, z12.s\n" + ".inst 0xc0040a41 // mova za.d[x8, #1], { z18.d-z19.d }\n" + ".inst 0xc1b0cf08 // sclamp { z8.s-z11.s }, z24.s, z16.s\n" + "st1b { z8.s }, p1, [x14]\n" + "add x14, x14, x4\n" + "st1b { z10.s }, p1, [x13]\n" "add x13, x13, x10\n" - "st1b { z13.s }, p1, [x9]\n" + "st1b { z9.s }, p1, [x9]\n" "add x9, x9, x27\n" - "st1b { z15.s }, p1, [x28]\n" + "st1b { z11.s }, p1, [x28]\n" "add x28, x28, x26\n" "bgt 23b\n" "24:" // End - "ldr x23, [%x[args], %[offsetof_Args_weights]]\n" - "incw x23, ALL, MUL #16\n" - "incw x23, ALL, MUL #9\n" - "str x23, [%x[args], %[offsetof_Args_weights]]\n" - "ldr x20, [%x[args], %[offsetof_Args_ld_in_vl]]\n" - "incw x5\n" - "whilelt p1.s, x5, x7\n" - "ldr x16, [%x[args], %[offsetof_Args_inptr]]\n" - "add x16, x16, x20\n" - "str x16, [%x[args], %[offsetof_Args_inptr]]\n" + "ldr x20, [%x[args], %[offsetof_Args_weights]]\n" + "incw x20, ALL, MUL #16\n" + "incw x20, ALL, MUL #9\n" + "str x20, [%x[args], %[offsetof_Args_weights]]\n" + "ldr x21, [%x[args], %[offsetof_Args_ld_in_vl]]\n" + "incw x17\n" + "whilelt p1.s, x17, x7\n" + "ldr x20, [%x[args], %[offsetof_Args_inptr]]\n" + "add x20, x20, x21\n" + "str x20, [%x[args], %[offsetof_Args_inptr]]\n" "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" "ldr x24, [%x[args], %[offsetof_Args_ld_out_vls]]\n" "ldp x23, x22, [x25, #0x0]\n" @@ -1194,7 +1194,7 @@ void sme2_u8s8u8q_planar_5x5_s1_4rows_dot_za_impl( ".inst 0xd503467f // SMSTOP\n" : : [args] "r" (&args), [ld_in_col] "r" (ld_in_col), [ld_in_row] "r" (ld_in_row), [offsetof_Args_current_channel] "I" (offsetof(Args, current_channel)), [offsetof_Args_inptr] "I" (offsetof(Args, inptr)), [offsetof_Args_input_cols] "I" (offsetof(Args, input_cols)), [offsetof_Args_ld_in_vl] "I" (offsetof(Args, ld_in_vl)), [offsetof_Args_ld_out_cols] "I" (offsetof(Args, ld_out_cols)), [offsetof_Args_ld_out_vls] "I" (offsetof(Args, ld_out_vls)), [offsetof_Args_n_channels] "I" (offsetof(Args, n_channels)), [offsetof_Args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_Args_output_cols] "I" (offsetof(Args, output_cols)), [offsetof_Args_pad_bottom] "I" (offsetof(Args, pad_bottom)), [offsetof_Args_pad_left] "I" (offsetof(Args, pad_left)), [offsetof_Args_pad_top] "I" (offsetof(Args, pad_top)), [offsetof_Args_weights] "I" (offsetof(Args, weights)), [offsetof_Requantize32_a_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_bias] "I" (offsetof(arm_gemm::Requantize32, bias)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [offsetof_Requantize32_per_channel_muls] "I" (offsetof(arm_gemm::Requantize32, per_channel_muls)), [offsetof_Requantize32_per_channel_right_shifts] "I" (offsetof(arm_gemm::Requantize32, per_channel_right_shifts)), [offsetof_Requantize32_per_layer_mul] "I" (offsetof(arm_gemm::Requantize32, per_layer_mul)), [offsetof_Requantize32_per_layer_right_shift] "I" (offsetof(arm_gemm::Requantize32, per_layer_right_shift)), [qp] "r" (&qp) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_5x5_s2_4rows_dot_za.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_5x5_s2_4rows_dot_za.hpp index 6949e69e39ddd5bc4d08d6c70bd0ed11cd444e81..ad820709129e95b021bf2feef3218032aab4ae17 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_5x5_s2_4rows_dot_za.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_5x5_s2_4rows_dot_za.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,8 @@ * SOFTWARE. */ +#include "src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp" + namespace arm_conv { namespace depthwise { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_5x5_s2_4rows_dot_za/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_5x5_s2_4rows_dot_za/generic.cpp index 33bb4eb8ec49cec7a205a6fb9178bb4d6fb0eeb5..d8dc69127eaf8b88ed4d72ca8bdc6ac8fb13e01d 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_5x5_s2_4rows_dot_za/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_u8s8u8q_planar_5x5_s2_4rows_dot_za/generic.cpp @@ -73,156 +73,156 @@ void sme2_u8s8u8q_planar_5x5_s2_4rows_dot_za_impl( "ptrue p2.b\n" "mov x20, #0xb\n" "ldr x4, [%x[args], %[offsetof_Args_pad_top]]\n" - "ld1rh { z9.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_a_offset]]\n" + "ld1rh { z7.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_a_offset]]\n" "sub x20, x20, x3\n" ".inst 0x25207812 // ptrue pn10.b\n" "ldr x5, [%x[args], %[offsetof_Args_n_channels]]\n" "whilelt p1.s, XZR, x5\n" "whilelt p9.s, XZR, x20\n" - "ld1rw { z8.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" + "ld1rw { z10.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" "whilelt p8.s, XZR, x4\n" "addvl SP, SP, #-15\n" "ldr x6, [%x[args], %[offsetof_Args_current_channel]]\n" - "neg z9.h, p2/M, z9.h\n" + "neg z7.h, p2/M, z7.h\n" "eor p8.b, p2/Z, p8.b, p9.b\n" - "ld1rw { z3.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_mul]]\n" - "ld1rw { z1.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_right_shift]]\n" - "ld1rw { z26.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" - "ld1rw { z23.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" + "ld1rw { z6.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z4.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z5.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" + "ld1rw { z21.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" "1:" // Channel loop "ldr x20, [%x[qp], %[offsetof_Requantize32_bias]]\n" - "mov z28.s, #0x0\n" + "mov z12.s, #0x0\n" "cbz x20, 2f\n" - "ld1w { z28.s }, p1/Z, [x20, x6, LSL #2]\n" + "ld1w { z12.s }, p1/Z, [x20, x6, LSL #2]\n" "2:" // Load bias: Done "ldr x22, [%x[args], %[offsetof_Args_weights]]\n" "mov x20, x22\n" - "ld1sb { z12.s }, p2/Z, [x20]\n" + "ld1sb { z13.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "ld1rh { z18.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" - "sub z12.h, z12.h, z18.h\n" + "ld1rh { z28.h }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" + "sub z13.h, z13.h, z28.h\n" "incw x22\n" - "mov z14.h, #0x0\n" - "ld1sb { z25.s }, p2/Z, [x20]\n" + "mov z26.h, #0x0\n" + "ld1sb { z22.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z25.h, z25.h, z18.h\n" - "trn1 z2.h, z12.h, z25.h\n" - "ld1sb { z24.s }, p2/Z, [x20]\n" + "sub z22.h, z22.h, z28.h\n" + "trn1 z17.h, z13.h, z22.h\n" + "ld1sb { z20.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z24.h, z24.h, z18.h\n" + "sub z20.h, z20.h, z28.h\n" "addvl x21, SP, #15\n" - "ld1sb { z17.s }, p2/Z, [x20]\n" + "ld1sb { z1.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z17.h, z17.h, z18.h\n" - "trn1 z10.h, z24.h, z17.h\n" - "ld1sb { z16.s }, p2/Z, [x20]\n" + "sub z1.h, z1.h, z28.h\n" + "trn1 z29.h, z20.h, z1.h\n" + "ld1sb { z27.s }, p2/Z, [x20]\n" "mov x20, x22\n" - "sub z16.h, z16.h, z18.h\n" + "sub z27.h, z27.h, z28.h\n" "incw x22\n" - "ld1sb { z12.s }, p2/Z, [x20]\n" + "ld1sb { z14.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z12.h, z12.h, z18.h\n" + "sub z14.h, z14.h, z28.h\n" "addvl x21, x21, #-3\n" - "ld1sb { z25.s }, p2/Z, [x20]\n" + "ld1sb { z18.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z25.h, z25.h, z18.h\n" - "trn1 z0.h, z16.h, z14.h\n" - "ld1sb { z24.s }, p2/Z, [x20]\n" + "sub z18.h, z18.h, z28.h\n" + "trn1 z22.h, z27.h, z26.h\n" + "ld1sb { z23.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z24.h, z24.h, z18.h\n" - "st1h { z2.h }, p2, [x21]\n" - "ld1sb { z17.s }, p2/Z, [x20]\n" + "sub z23.h, z23.h, z28.h\n" + "st1h { z17.h }, p2, [x21]\n" + "ld1sb { z30.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z17.h, z17.h, z18.h\n" - "trn1 z2.h, z12.h, z25.h\n" - "ld1sb { z16.s }, p2/Z, [x20]\n" + "sub z30.h, z30.h, z28.h\n" + "trn1 z8.h, z14.h, z18.h\n" + "ld1sb { z15.s }, p2/Z, [x20]\n" "mov x20, x22\n" - "st1h { z10.h }, p2, [x21, #1, MUL VL]\n" - "sub z16.h, z16.h, z18.h\n" - "ld1sb { z12.s }, p2/Z, [x20]\n" - "incw x20, ALL, MUL #5\n" - "trn1 z10.h, z24.h, z17.h\n" - "sub z12.h, z12.h, z18.h\n" - "ld1sb { z25.s }, p2/Z, [x20]\n" + "st1h { z29.h }, p2, [x21, #1, MUL VL]\n" + "sub z15.h, z15.h, z28.h\n" + "ld1sb { z20.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z25.h, z25.h, z18.h\n" - "st1h { z0.h }, p2, [x21, #2, MUL VL]\n" + "trn1 z23.h, z23.h, z30.h\n" + "sub z20.h, z20.h, z28.h\n" "ld1sb { z24.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "trn1 z0.h, z16.h, z14.h\n" + "sub z24.h, z24.h, z28.h\n" + "st1h { z22.h }, p2, [x21, #2, MUL VL]\n" + "ld1sb { z16.s }, p2/Z, [x20]\n" + "incw x20, ALL, MUL #5\n" + "trn1 z0.h, z15.h, z26.h\n" "incw x22\n" - "ld1sb { z17.s }, p2/Z, [x20]\n" + "ld1sb { z13.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z24.h, z24.h, z18.h\n" - "sub z17.h, z17.h, z18.h\n" - "ld1sb { z16.s }, p2/Z, [x20]\n" + "sub z16.h, z16.h, z28.h\n" + "sub z13.h, z13.h, z28.h\n" + "ld1sb { z11.s }, p2/Z, [x20]\n" "addvl x21, x21, #-3\n" "mov x20, x22\n" - "st1h { z2.h }, p2, [x21]\n" - "trn1 z2.h, z12.h, z25.h\n" - "ld1sb { z12.s }, p2/Z, [x20]\n" + "st1h { z8.h }, p2, [x21]\n" + "trn1 z27.h, z20.h, z24.h\n" + "ld1sb { z22.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z16.h, z16.h, z18.h\n" - "ld1sb { z25.s }, p2/Z, [x20]\n" + "sub z11.h, z11.h, z28.h\n" + "ld1sb { z3.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "st1h { z10.h }, p2, [x21, #1, MUL VL]\n" - "trn1 z10.h, z24.h, z17.h\n" - "ld1sb { z24.s }, p2/Z, [x20]\n" + "st1h { z23.h }, p2, [x21, #1, MUL VL]\n" + "trn1 z20.h, z16.h, z13.h\n" + "ld1sb { z13.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z12.h, z12.h, z18.h\n" - "sub z25.h, z25.h, z18.h\n" - "ld1sb { z17.s }, p2/Z, [x20]\n" + "sub z22.h, z22.h, z28.h\n" + "sub z3.h, z3.h, z28.h\n" + "ld1sb { z15.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" "st1h { z0.h }, p2, [x21, #2, MUL VL]\n" - "trn1 z0.h, z16.h, z14.h\n" + "trn1 z29.h, z11.h, z26.h\n" "ld1sb { z16.s }, p2/Z, [x20]\n" "incw x22\n" - "sub z24.h, z24.h, z18.h\n" - "sub z17.h, z17.h, z18.h\n" + "sub z13.h, z13.h, z28.h\n" + "sub z15.h, z15.h, z28.h\n" "addvl x21, x21, #-3\n" "mov x20, x22\n" - "st1h { z2.h }, p2, [x21]\n" - "sub z16.h, z16.h, z18.h\n" - "trn1 z2.h, z12.h, z25.h\n" - "ld1sb { z12.s }, p2/Z, [x20]\n" + "st1h { z27.h }, p2, [x21]\n" + "sub z16.h, z16.h, z28.h\n" + "trn1 z19.h, z22.h, z3.h\n" + "ld1sb { z17.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "st1h { z10.h }, p2, [x21, #1, MUL VL]\n" - "ld1sb { z25.s }, p2/Z, [x20]\n" + "st1h { z20.h }, p2, [x21, #1, MUL VL]\n" + "ld1sb { z0.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "trn1 z10.h, z24.h, z17.h\n" - "st1h { z0.h }, p2, [x21, #2, MUL VL]\n" - "ld1sb { z24.s }, p2/Z, [x20]\n" + "trn1 z31.h, z13.h, z15.h\n" + "st1h { z29.h }, p2, [x21, #2, MUL VL]\n" + "ld1sb { z18.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "trn1 z0.h, z16.h, z14.h\n" - "sub z12.h, z12.h, z18.h\n" - "ld1sb { z17.s }, p2/Z, [x20]\n" + "trn1 z16.h, z16.h, z26.h\n" + "sub z17.h, z17.h, z28.h\n" + "ld1sb { z22.s }, p2/Z, [x20]\n" "incw x20, ALL, MUL #5\n" - "sub z25.h, z25.h, z18.h\n" - "sub z24.h, z24.h, z18.h\n" - "ld1sb { z16.s }, p2/Z, [x20]\n" - "sub z17.h, z17.h, z18.h\n" - "sub z16.h, z16.h, z18.h\n" + "sub z0.h, z0.h, z28.h\n" + "sub z18.h, z18.h, z28.h\n" + "ld1sb { z1.s }, p2/Z, [x20]\n" + "sub z22.h, z22.h, z28.h\n" + "sub z1.h, z1.h, z28.h\n" "ldr x20, [%x[qp], %[offsetof_Requantize32_per_channel_muls]]\n" "addvl x21, x21, #-3\n" - "st1h { z2.h }, p2, [x21]\n" - "mov z29.d, z28.d\n" - "mov z30.d, z28.d\n" - "st1h { z10.h }, p2, [x21, #1, MUL VL]\n" - "mov z31.d, z28.d\n" - "trn1 z2.h, z12.h, z25.h\n" - "st1h { z0.h }, p2, [x21, #2, MUL VL]\n" + "st1h { z19.h }, p2, [x21]\n" + "mov z13.d, z12.d\n" + "mov z14.d, z12.d\n" + "st1h { z31.h }, p2, [x21, #1, MUL VL]\n" + "mov z15.d, z12.d\n" + "trn1 z8.h, z17.h, z0.h\n" + "st1h { z16.h }, p2, [x21, #2, MUL VL]\n" "addvl x21, x21, #-3\n" - "trn1 z10.h, z24.h, z17.h\n" - "trn1 z0.h, z16.h, z14.h\n" - "st1h { z2.h }, p2, [x21]\n" - "st1h { z10.h }, p2, [x21, #1, MUL VL]\n" - "st1h { z0.h }, p2, [x21, #2, MUL VL]\n" + "trn1 z31.h, z18.h, z22.h\n" + "trn1 z29.h, z1.h, z26.h\n" + "st1h { z8.h }, p2, [x21]\n" + "st1h { z31.h }, p2, [x21, #1, MUL VL]\n" + "st1h { z29.h }, p2, [x21, #2, MUL VL]\n" "cbz x20, 3f\n" - "ld1w { z3.s }, p1/Z, [x20, x6, LSL #2]\n" + "ld1w { z6.s }, p1/Z, [x20, x6, LSL #2]\n" "3:" // Load mul: End "ldr x20, [%x[qp], %[offsetof_Requantize32_per_channel_right_shifts]]\n" "cbz x20, 4f\n" - "ld1w { z1.s }, p1/Z, [x20, x6, LSL #2]\n" + "ld1w { z4.s }, p1/Z, [x20, x6, LSL #2]\n" "4:" // Load right_shift: End "ldr x7, [%x[args], %[offsetof_Args_input_cols]]\n" "sub x20, x7, #0x1\n" @@ -242,20 +242,20 @@ void sme2_u8s8u8q_planar_5x5_s2_4rows_dot_za_impl( ".inst 0xf8b74a9c // rprfm pldstrm, x23, [x20]\n" "add x20, x20, %x[ld_in_col]\n" "bgt 5b\n" - "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" + "ldr x23, [%x[args], %[offsetof_Args_outptrs]]\n" "lsl x20, %x[ld_in_row], #0x0\n" "msub x17, x4, x20, x17\n" - ".inst 0xc0040f80 // mova za.d[x8, #0], { z28.d-z31.d }\n" + ".inst 0xc0040d80 // mova za.d[x8, #0], { z12.d-z15.d }\n" "ldr x20, [%x[args], %[offsetof_Args_ld_out_cols]]\n" - ".inst 0xc0040f81 // mova za.d[x8, #1], { z28.d-z31.d }\n" + ".inst 0xc0040d81 // mova za.d[x8, #1], { z12.d-z15.d }\n" "mov x22, #0x4\n" - "ldp x15, x14, [x25], #0x10\n" - ".inst 0xc0040f82 // mova za.d[x8, #2], { z28.d-z31.d }\n" + "ldp x15, x14, [x23], #0x10\n" + ".inst 0xc0040d82 // mova za.d[x8, #2], { z12.d-z15.d }\n" "ldp x13, x11, [x20], #0x10\n" - ".inst 0xc0040f83 // mova za.d[x8, #3], { z28.d-z31.d }\n" + ".inst 0xc0040d83 // mova za.d[x8, #3], { z12.d-z15.d }\n" "ldr x21, [%x[args], %[offsetof_Args_pad_left]]\n" - ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" - "ldp x10, x9, [x25], #0x10\n" + ".inst 0xc0040d84 // mova za.d[x8, #4], { z12.d-z15.d }\n" + "ldp x10, x9, [x23], #0x10\n" "ldp x28, x27, [x20], #0x10\n" "cbz x21, 7f\n" "cmp x21, x22\n" @@ -263,24 +263,24 @@ void sme2_u8s8u8q_planar_5x5_s2_4rows_dot_za_impl( "sub x21, x21, x20\n" "sub x22, x22, x20\n" "cbz x21, 7f\n" - ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" - ".inst 0xc1a3ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z3.s\n" + ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" + ".inst 0xc1a6ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z6.s\n" "and x22, x21, #0x1\n" - ".inst 0xc1a1aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" + ".inst 0xc1a4aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" "add x21, x21, #0x1\n" "lsr x21, x21, #0x1\n" - ".inst 0xc1a8ab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z8.s\n" + ".inst 0xc1aaab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z10.s\n" "sub x16, x16, x21\n" - ".inst 0xc1b7cf44 // sclamp { z4.s-z7.s }, z26.s, z23.s\n" + ".inst 0xc1b5ccbc // sclamp { z28.s-z31.s }, z5.s, z21.s\n" "6:" // Left padding "subs x21, x21, #0x1\n" - "st1b { z4.s }, p1, [x15]\n" + "st1b { z28.s }, p1, [x15]\n" "add x15, x15, x13\n" - "st1b { z5.s }, p1, [x14]\n" + "st1b { z29.s }, p1, [x14]\n" "add x14, x14, x11\n" - "st1b { z6.s }, p1, [x10]\n" + "st1b { z30.s }, p1, [x10]\n" "add x10, x10, x28\n" - "st1b { z7.s }, p1, [x9]\n" + "st1b { z31.s }, p1, [x9]\n" "add x9, x9, x27\n" "bgt 6b\n" "7:" // Left padding: End @@ -296,341 +296,341 @@ void sme2_u8s8u8q_planar_5x5_s2_4rows_dot_za_impl( "beq 9f\n" "8:" // Unpadded: 4 priming loads "add x21, x17, %x[ld_in_row]\n" - "ld1b { z11.s }, p1/Z, [x17]\n" + "ld1b { z27.s }, p1/Z, [x17]\n" "addvl x20, SP, #12\n" - "ld1b { z21.s }, p1/Z, [x21]\n" + "ld1b { z0.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "add z11.h, z11.h, z9.h\n" - "ld1b { z12.s }, p1/Z, [x21]\n" + "trn1 z27.h, z27.h, z0.h\n" + "add z27.h, z27.h, z7.h\n" + "ld1b { z28.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "add x17, x17, %x[ld_in_col]\n" - "ld1b { z20.s }, p1/Z, [x21]\n" + "ld1b { z11.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "add z12.h, z12.h, z9.h\n" - "ld1b { z13.s }, p1/Z, [x21]\n" + "trn1 z28.h, z28.h, z11.h\n" + "add z28.h, z28.h, z7.h\n" + "ld1b { z29.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1b { z19.s }, p1/Z, [x21]\n" + "ld1b { z8.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z9.h\n" - "ld1b { z14.s }, p1/Z, [x21]\n" + "trn1 z29.h, z29.h, z8.h\n" + "add z29.h, z29.h, z7.h\n" + "ld1b { z30.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1b { z18.s }, p1/Z, [x21]\n" + "ld1b { z17.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z9.h\n" - "ld1b { z15.s }, p1/Z, [x21]\n" + "trn1 z30.h, z30.h, z17.h\n" + "add z30.h, z30.h, z7.h\n" + "ld1b { z31.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x21]\n" + "ld1b { z26.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z15.h, z15.h, z9.h\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1b { z16.s }, p1/Z, [x21]\n" - "mov z16.d, z16.d\n" - "add z16.h, z16.h, z9.h\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + "trn1 z31.h, z31.h, z26.h\n" + "add z31.h, z31.h, z7.h\n" + ".inst 0xa1402a80 // ld1h { z0.h, z8.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1701768 // sdot za.s[x8, 0], { z27.h-z30.h }, z0.h\n" + "ld1b { z20.s }, p1/Z, [x21]\n" + "mov z0.d, z20.d\n" + "add z0.h, z0.h, z7.h\n" + ".inst 0xc1781788 // sdot za.s[x8, 0], { z28.h-z31.h }, z8.h\n" + "ld1h { z8.h }, p2/Z, [x20, #2, MUL VL]\n" + ".inst 0xc17817a8 // sdot za.s[x8, 0], { z29.h-z0.h }, z8.h\n" "9:" // Unpadded: 3 priming loads "add x21, x17, %x[ld_in_row]\n" - "ld1b { z11.s }, p1/Z, [x17]\n" + "ld1b { z29.s }, p1/Z, [x17]\n" "addvl x20, SP, #9\n" - "ld1b { z21.s }, p1/Z, [x21]\n" + "ld1b { z17.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "add z11.h, z11.h, z9.h\n" - "ld1b { z12.s }, p1/Z, [x21]\n" + "trn1 z29.h, z29.h, z17.h\n" + "add z29.h, z29.h, z7.h\n" + "ld1b { z30.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "add x17, x17, %x[ld_in_col]\n" - "ld1b { z20.s }, p1/Z, [x21]\n" + "ld1b { z0.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "add z12.h, z12.h, z9.h\n" - "ld1b { z13.s }, p1/Z, [x21]\n" + "trn1 z30.h, z30.h, z0.h\n" + "add z30.h, z30.h, z7.h\n" + "ld1b { z31.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1b { z19.s }, p1/Z, [x21]\n" + "ld1b { z16.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z9.h\n" - "ld1b { z14.s }, p1/Z, [x21]\n" + "trn1 z31.h, z31.h, z16.h\n" + "add z31.h, z31.h, z7.h\n" + "ld1b { z0.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1b { z18.s }, p1/Z, [x21]\n" + "ld1b { z16.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z9.h\n" - "ld1b { z15.s }, p1/Z, [x21]\n" + "trn1 z0.h, z0.h, z16.h\n" + "add z0.h, z0.h, z7.h\n" + "ld1b { z1.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x21]\n" + "ld1b { z16.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z15.h, z15.h, z9.h\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" + "trn1 z1.h, z1.h, z16.h\n" + "add z1.h, z1.h, z7.h\n" + ".inst 0xa0402a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20]\n" + ".inst 0xc17217a8 // sdot za.s[x8, 0], { z29.h-z0.h }, z2.h\n" "ld1b { z16.s }, p1/Z, [x21]\n" - "mov z16.d, z16.d\n" - "add z16.h, z16.h, z9.h\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + "mov z2.d, z16.d\n" + "add z2.h, z2.h, z7.h\n" + ".inst 0xc17317c8 // sdot za.s[x8, 0], { z30.h-z1.h }, z3.h\n" + "ld1h { z8.h }, p2/Z, [x20, #2, MUL VL]\n" + ".inst 0xc17817e8 // sdot za.s[x8, 0], { z31.h-z2.h }, z8.h\n" "10:" // Unpadded: 2 priming loads "add x22, x17, %x[ld_in_row]\n" - "ld1b { z11.s }, p1/Z, [x17]\n" + "ld1b { z26.s }, p1/Z, [x17]\n" "addvl x21, SP, #6\n" - "ld1b { z21.s }, p1/Z, [x22]\n" + "ld1b { z16.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "add z11.h, z11.h, z9.h\n" - "ld1b { z12.s }, p1/Z, [x22]\n" + "trn1 z26.h, z26.h, z16.h\n" + "add z26.h, z26.h, z7.h\n" + "ld1b { z27.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "addvl x20, SP, #12\n" - "ld1b { z20.s }, p1/Z, [x22]\n" + "ld1b { z16.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "add z12.h, z12.h, z9.h\n" - "ld1b { z13.s }, p1/Z, [x22]\n" + "trn1 z27.h, z27.h, z16.h\n" + "add z27.h, z27.h, z7.h\n" + "ld1b { z28.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "add x17, x17, %x[ld_in_col]\n" - "ld1b { z19.s }, p1/Z, [x22]\n" + "ld1b { z29.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z9.h\n" - "ld1b { z14.s }, p1/Z, [x22]\n" + "trn1 z28.h, z28.h, z29.h\n" + "add z28.h, z28.h, z7.h\n" + "ld1b { z29.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "ld1b { z18.s }, p1/Z, [x22]\n" + "ld1b { z19.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z9.h\n" - "ld1b { z15.s }, p1/Z, [x22]\n" + "trn1 z29.h, z29.h, z19.h\n" + "add z29.h, z29.h, z7.h\n" + "ld1b { z30.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x22]\n" - "trn1 z15.h, z15.h, z17.h\n" + "ld1b { z23.s }, p1/Z, [x22]\n" + "trn1 z30.h, z30.h, z23.h\n" "add x22, x22, %x[ld_in_row]\n" - "add z15.h, z15.h, z9.h\n" - ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1b { z16.s }, p1/Z, [x22]\n" - "mov z16.d, z16.d\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" - "add z16.h, z16.h, z9.h\n" - "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + "add z30.h, z30.h, z7.h\n" + ".inst 0xa0402aa2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x21]\n" + ".inst 0xc1721748 // sdot za.s[x8, 0], { z26.h-z29.h }, z2.h\n" + "ld1b { z22.s }, p1/Z, [x22]\n" + "mov z31.d, z22.d\n" + ".inst 0xc1731768 // sdot za.s[x8, 0], { z27.h-z30.h }, z3.h\n" + ".inst 0xa1402a83 // ld1h { z3.h, z11.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1731749 // sdot za.s[x8, 1], { z26.h-z29.h }, z3.h\n" + "add z31.h, z31.h, z7.h\n" + "ld1h { z3.h }, p2/Z, [x21, #2, MUL VL]\n" + ".inst 0xc17b1769 // sdot za.s[x8, 1], { z27.h-z30.h }, z11.h\n" + ".inst 0xc1731788 // sdot za.s[x8, 0], { z28.h-z31.h }, z3.h\n" "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1701789 // sdot za.s[x8, 1], { z28.h-z31.h }, z0.h\n" "11:" // Unpadded: 1 priming loads "add x22, x17, %x[ld_in_row]\n" - "ld1b { z11.s }, p1/Z, [x17]\n" + "ld1b { z29.s }, p1/Z, [x17]\n" "addvl x21, SP, #3\n" - "ld1b { z21.s }, p1/Z, [x22]\n" + "ld1b { z22.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "add z11.h, z11.h, z9.h\n" - "ld1b { z12.s }, p1/Z, [x22]\n" + "trn1 z29.h, z29.h, z22.h\n" + "add z29.h, z29.h, z7.h\n" + "ld1b { z30.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "addvl x20, SP, #9\n" - "ld1b { z20.s }, p1/Z, [x22]\n" + "ld1b { z25.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "add z12.h, z12.h, z9.h\n" - "ld1b { z13.s }, p1/Z, [x22]\n" + "trn1 z30.h, z30.h, z25.h\n" + "add z30.h, z30.h, z7.h\n" + "ld1b { z31.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" "add x17, x17, %x[ld_in_col]\n" - "ld1b { z19.s }, p1/Z, [x22]\n" + "ld1b { z16.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z9.h\n" - "ld1b { z14.s }, p1/Z, [x22]\n" + "trn1 z31.h, z31.h, z16.h\n" + "add z31.h, z31.h, z7.h\n" + "ld1b { z0.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "ld1b { z18.s }, p1/Z, [x22]\n" + "ld1b { z16.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z9.h\n" - "ld1b { z15.s }, p1/Z, [x22]\n" + "trn1 z0.h, z0.h, z16.h\n" + "add z0.h, z0.h, z7.h\n" + "ld1b { z1.s }, p1/Z, [x22]\n" "add x22, x22, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x22]\n" - "trn1 z15.h, z15.h, z17.h\n" + "ld1b { z2.s }, p1/Z, [x22]\n" + "trn1 z1.h, z1.h, z2.h\n" "add x22, x22, %x[ld_in_row]\n" - "add z15.h, z15.h, z9.h\n" - ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1b { z16.s }, p1/Z, [x22]\n" - "mov z16.d, z16.d\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" - "add z16.h, z16.h, z9.h\n" - "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" - "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" + "add z1.h, z1.h, z7.h\n" + ".inst 0xa0402aa2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x21]\n" + ".inst 0xc17217a8 // sdot za.s[x8, 0], { z29.h-z0.h }, z2.h\n" + "ld1b { z24.s }, p1/Z, [x22]\n" + "mov z2.d, z24.d\n" + ".inst 0xc17317c8 // sdot za.s[x8, 0], { z30.h-z1.h }, z3.h\n" + ".inst 0xa0402a88 // ld1h { z8.h-z9.h }, pn10.b/Z, [x20]\n" + ".inst 0xc17817a9 // sdot za.s[x8, 1], { z29.h-z0.h }, z8.h\n" + "add z2.h, z2.h, z7.h\n" + "ld1h { z3.h }, p2/Z, [x21, #2, MUL VL]\n" + ".inst 0xc17917c9 // sdot za.s[x8, 1], { z30.h-z1.h }, z9.h\n" + ".inst 0xc17317e8 // sdot za.s[x8, 0], { z31.h-z2.h }, z3.h\n" + "ld1h { z3.h }, p2/Z, [x20, #2, MUL VL]\n" + ".inst 0xc17317e9 // sdot za.s[x8, 1], { z31.h-z2.h }, z3.h\n" "12:" // Unpadded: 0 priming loads "cmp x7, #0x2\n" - ".inst 0xa1402be2 // ld1h { z2.h, z10.h }, pn10.b/Z, [SP]\n" - "ld1h { z0.h }, p2/Z, [SP, #2, MUL VL]\n" + ".inst 0xa1402be3 // ld1h { z3.h, z11.h }, pn10.b/Z, [SP]\n" + "ld1h { z2.h }, p2/Z, [SP, #2, MUL VL]\n" "blt 22f\n" "add x21, x17, %x[ld_in_row]\n" - "ld1b { z11.s }, p1/Z, [x17]\n" + "ld1b { z23.s }, p1/Z, [x17]\n" "sub x7, x7, #0x2\n" - "ld1b { z21.s }, p1/Z, [x21]\n" + "ld1b { z25.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" + "trn1 z23.h, z23.h, z25.h\n" "sub x16, x16, #0x1\n" - "ld1b { z12.s }, p1/Z, [x21]\n" + "ld1b { z24.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "lsr x20, x7, #0x1\n" - "add z11.h, z11.h, z9.h\n" - "ld1b { z20.s }, p1/Z, [x21]\n" + "add z23.h, z23.h, z7.h\n" + "ld1b { z30.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z24.h, z24.h, z30.h\n" "cmp x20, x16\n" - "ld1b { z13.s }, p1/Z, [x21]\n" + "ld1b { z25.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "csel x26, x20, x16, LT\n" - "add z12.h, z12.h, z9.h\n" - "ld1b { z19.s }, p1/Z, [x21]\n" + "add z24.h, z24.h, z7.h\n" + "ld1b { z22.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z9.h\n" - "ld1b { z14.s }, p1/Z, [x21]\n" + "trn1 z25.h, z25.h, z22.h\n" + "add z25.h, z25.h, z7.h\n" + "ld1b { z26.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "add x17, x17, %x[ld_in_col]\n" - "ld1b { z18.s }, p1/Z, [x21]\n" + "ld1b { z22.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z9.h\n" - "ld1b { z15.s }, p1/Z, [x21]\n" + "trn1 z26.h, z26.h, z22.h\n" + "add z26.h, z26.h, z7.h\n" + "ld1b { z27.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" "and x7, x7, #0x1\n" - "ld1b { z17.s }, p1/Z, [x21]\n" + "ld1b { z30.s }, p1/Z, [x21]\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z15.h, z15.h, z9.h\n" - "ld1b { z16.s }, p1/Z, [x21]\n" - "mov z16.d, z16.d\n" - "add z16.h, z16.h, z9.h\n" + "trn1 z27.h, z27.h, z30.h\n" + "add z27.h, z27.h, z7.h\n" + "ld1b { z28.s }, p1/Z, [x21]\n" + "mov z28.d, z28.d\n" + "add z28.h, z28.h, z7.h\n" "sub x16, x16, x26\n" "cbz x26, 21f\n" "13:" // Unpadded: Main loop - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" + ".inst 0xc17316e8 // sdot za.s[x8, 0], { z23.h-z26.h }, z3.h\n" "addvl x25, SP, #6\n" "addvl x24, SP, #12\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402b22 // ld1h { z2.h, z10.h }, pn10.b/Z, [x25]\n" + ".inst 0xc17b1708 // sdot za.s[x8, 0], { z24.h-z27.h }, z11.h\n" + ".inst 0xa0402b20 // ld1h { z0.h-z1.h }, pn10.b/Z, [x25]\n" "add x23, x17, %x[ld_in_row]\n" "addvl x22, SP, #3\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" + ".inst 0xc17016e9 // sdot za.s[x8, 1], { z23.h-z26.h }, z0.h\n" "addvl x21, SP, #9\n" "subs x26, x26, #0x1\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402b02 // ld1h { z2.h, z10.h }, pn10.b/Z, [x24]\n" - ".inst 0xc172156a // sdot za.s[x8, 2], { z11.h-z14.h }, z2.h\n" - "ld1b { z11.s }, p1/Z, [x17]\n" + ".inst 0xc1711709 // sdot za.s[x8, 1], { z24.h-z27.h }, z1.h\n" + ".inst 0xa0402b08 // ld1h { z8.h-z9.h }, pn10.b/Z, [x24]\n" + ".inst 0xc17816ea // sdot za.s[x8, 2], { z23.h-z26.h }, z8.h\n" + "ld1b { z23.s }, p1/Z, [x17]\n" "add x17, x17, %x[ld_in_col]\n" "add x20, x17, %x[ld_in_row]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1721728 // sdot za.s[x8, 0], { z25.h-z28.h }, z2.h\n" "ld1h { z0.h }, p2/Z, [x25, #2, MUL VL]\n" - ".inst 0xc17a158a // sdot za.s[x8, 2], { z12.h-z15.h }, z10.h\n" - "ld1b { z21.s }, p1/Z, [x23]\n" + ".inst 0xc179170a // sdot za.s[x8, 2], { z24.h-z27.h }, z9.h\n" + "ld1b { z16.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" - "ld1h { z0.h }, p2/Z, [x24, #2, MUL VL]\n" - "add z11.h, z11.h, z9.h\n" - "ld1b { z12.s }, p1/Z, [x23]\n" + "trn1 z23.h, z23.h, z16.h\n" + ".inst 0xc1701729 // sdot za.s[x8, 1], { z25.h-z28.h }, z0.h\n" + "ld1h { z9.h }, p2/Z, [x24, #2, MUL VL]\n" + "add z23.h, z23.h, z7.h\n" + "ld1b { z24.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - ".inst 0xc17015aa // sdot za.s[x8, 2], { z13.h-z16.h }, z0.h\n" - "ld1b { z20.s }, p1/Z, [x23]\n" + ".inst 0xc179172a // sdot za.s[x8, 2], { z25.h-z28.h }, z9.h\n" + "ld1b { z18.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" - "add z12.h, z12.h, z9.h\n" - "ld1b { z13.s }, p1/Z, [x23]\n" + "trn1 z24.h, z24.h, z18.h\n" + "add z24.h, z24.h, z7.h\n" + "ld1b { z25.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" + ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" - "ld1b { z19.s }, p1/Z, [x23]\n" + "ld1b { z8.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z13.h, z13.h, z9.h\n" - "ld1b { z14.s }, p1/Z, [x23]\n" + "trn1 z25.h, z25.h, z8.h\n" + "add z25.h, z25.h, z7.h\n" + "ld1b { z26.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - ".inst 0xc1a3ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z3.s\n" - "ld1b { z18.s }, p1/Z, [x23]\n" + ".inst 0xc1a6ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n" + "ld1b { z28.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z9.h\n" - "ld1b { z15.s }, p1/Z, [x23]\n" + "trn1 z26.h, z26.h, z28.h\n" + "add z26.h, z26.h, z7.h\n" + "ld1b { z27.s }, p1/Z, [x23]\n" "add x23, x23, %x[ld_in_row]\n" - ".inst 0xc1a1aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" - "ld1b { z17.s }, p1/Z, [x23]\n" - "trn1 z15.h, z15.h, z17.h\n" + ".inst 0xc1a4aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z4.s\n" + "ld1b { z28.s }, p1/Z, [x23]\n" + "trn1 z27.h, z27.h, z28.h\n" "add x23, x23, %x[ld_in_row]\n" - "add z15.h, z15.h, z9.h\n" - ".inst 0xa1402ac2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x22]\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - ".inst 0xc1a8ab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z8.s\n" - "ld1b { z16.s }, p1/Z, [x23]\n" - "mov z16.d, z16.d\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" - "add z16.h, z16.h, z9.h\n" + "add z27.h, z27.h, z7.h\n" + ".inst 0xa0402ac2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x22]\n" + ".inst 0xc17216e8 // sdot za.s[x8, 0], { z23.h-z26.h }, z2.h\n" + ".inst 0xc1aaab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z10.s\n" + "ld1b { z20.s }, p1/Z, [x23]\n" + "mov z28.d, z20.d\n" + ".inst 0xc1731708 // sdot za.s[x8, 0], { z24.h-z27.h }, z3.h\n" + ".inst 0xa0402aa0 // ld1h { z0.h-z1.h }, pn10.b/Z, [x21]\n" + ".inst 0xc17016e9 // sdot za.s[x8, 1], { z23.h-z26.h }, z0.h\n" + "add z28.h, z28.h, z7.h\n" "ld1h { z0.h }, p2/Z, [x22, #2, MUL VL]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xc1b7cf44 // sclamp { z4.s-z7.s }, z26.s, z23.s\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1711709 // sdot za.s[x8, 1], { z24.h-z27.h }, z1.h\n" + ".inst 0xc1b5ccb0 // sclamp { z16.s-z19.s }, z5.s, z21.s\n" + ".inst 0xc1701728 // sdot za.s[x8, 0], { z25.h-z28.h }, z0.h\n" "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" - "st1b { z4.s }, p1, [x15]\n" + "st1b { z16.s }, p1, [x15]\n" "add x15, x15, x13\n" - "ld1b { z11.s }, p1/Z, [x17]\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" - "st1b { z5.s }, p1, [x14]\n" + "ld1b { z23.s }, p1/Z, [x17]\n" + ".inst 0xc1701729 // sdot za.s[x8, 1], { z25.h-z28.h }, z0.h\n" + "st1b { z17.s }, p1, [x14]\n" "add x14, x14, x11\n" - "ld1b { z21.s }, p1/Z, [x20]\n" + "ld1b { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "st1b { z6.s }, p1, [x10]\n" - "ld1b { z12.s }, p1/Z, [x20]\n" + "trn1 z23.h, z23.h, z16.h\n" + "st1b { z18.s }, p1, [x10]\n" + "ld1b { z24.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" "add x10, x10, x28\n" - "st1b { z7.s }, p1, [x9]\n" - "ld1b { z20.s }, p1/Z, [x20]\n" + "st1b { z19.s }, p1, [x9]\n" + "ld1b { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z24.h, z24.h, z16.h\n" "add x9, x9, x27\n" - "ld1b { z13.s }, p1/Z, [x20]\n" + "ld1b { z25.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" - "add z11.h, z11.h, z9.h\n" - "ld1b { z19.s }, p1/Z, [x20]\n" + ".inst 0xc0040d84 // mova za.d[x8, #4], { z12.d-z15.d }\n" + "add z23.h, z23.h, z7.h\n" + "ld1b { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "add z12.h, z12.h, z9.h\n" - "ld1b { z14.s }, p1/Z, [x20]\n" + "trn1 z25.h, z25.h, z16.h\n" + "add z24.h, z24.h, z7.h\n" + "ld1b { z26.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "add z13.h, z13.h, z9.h\n" + "add z25.h, z25.h, z7.h\n" "add x17, x17, %x[ld_in_col]\n" - "ld1b { z18.s }, p1/Z, [x20]\n" + "ld1b { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z14.h, z14.h, z18.h\n" - "add z14.h, z14.h, z9.h\n" - "ld1b { z15.s }, p1/Z, [x20]\n" + "trn1 z26.h, z26.h, z16.h\n" + "add z26.h, z26.h, z7.h\n" + "ld1b { z27.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "ld1b { z17.s }, p1/Z, [x20]\n" + "ld1b { z16.s }, p1/Z, [x20]\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z15.h, z15.h, z17.h\n" - "add z15.h, z15.h, z9.h\n" + "trn1 z27.h, z27.h, z16.h\n" + "add z27.h, z27.h, z7.h\n" "ld1b { z16.s }, p1/Z, [x20]\n" - "mov z16.d, z16.d\n" - "add z16.h, z16.h, z9.h\n" - ".inst 0xa1402be2 // ld1h { z2.h, z10.h }, pn10.b/Z, [SP]\n" - "ld1h { z0.h }, p2/Z, [SP, #2, MUL VL]\n" + "mov z28.d, z16.d\n" + "add z28.h, z28.h, z7.h\n" + ".inst 0xa1402be3 // ld1h { z3.h, z11.h }, pn10.b/Z, [SP]\n" + "ld1h { z2.h }, p2/Z, [SP, #2, MUL VL]\n" "bgt 13b\n" "b 21f\n" "14:" // Padded @@ -645,688 +645,688 @@ void sme2_u8s8u8q_planar_5x5_s2_4rows_dot_za_impl( "15:" // Padded: 4 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + "ld1b { z27.s }, p0/Z, [x17]\n" + "add z27.h, p0/M, z27.h, z7.h\n" "add x21, x17, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z21.s }, p0/Z, [x21]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1b { z17.s }, p0/Z, [x21]\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z12.s }, p0/Z, [x21]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + "ld1b { z28.s }, p0/Z, [x21]\n" + "add z28.h, p0/M, z28.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z20.s }, p0/Z, [x21]\n" - "add z20.h, p0/M, z20.h, z9.h\n" + "ld1b { z16.s }, p0/Z, [x21]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "mov x12, #0x4\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z27.h, z27.h, z17.h\n" + "trn1 z28.h, z28.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z13.s }, p0/Z, [x21]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + "ld1b { z29.s }, p0/Z, [x21]\n" + "add z29.h, p0/M, z29.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z19.s }, p0/Z, [x21]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "ld1b { z18.s }, p0/Z, [x21]\n" + "add z18.h, p0/M, z18.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z14.s }, p0/Z, [x21]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1b { z30.s }, p0/Z, [x21]\n" + "add z30.h, p0/M, z30.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z18.s }, p0/Z, [x21]\n" + "ld1b { z17.s }, p0/Z, [x21]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z15.s }, p0/Z, [x21]\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1b { z31.s }, p0/Z, [x21]\n" + "add z31.h, p0/M, z31.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z17.s }, p0/Z, [x21]\n" - "add z17.h, p0/M, z17.h, z9.h\n" + "ld1b { z16.s }, p0/Z, [x21]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "addvl x20, SP, #12\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" + "trn1 z29.h, z29.h, z18.h\n" + "trn1 z30.h, z30.h, z17.h\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1b { z16.s }, p0/Z, [x21]\n" - "add z16.h, p0/M, z16.h, z9.h\n" - "mov z16.d, z16.d\n" + ".inst 0xa0402a80 // ld1h { z0.h-z1.h }, pn10.b/Z, [x20]\n" + "trn1 z31.h, z31.h, z16.h\n" + ".inst 0xc1701768 // sdot za.s[x8, 0], { z27.h-z30.h }, z0.h\n" + "ld1b { z20.s }, p0/Z, [x21]\n" + "add z20.h, p0/M, z20.h, z7.h\n" + "mov z0.d, z20.d\n" "add x17, x17, %x[ld_in_col]\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1711788 // sdot za.s[x8, 0], { z28.h-z31.h }, z1.h\n" + "ld1h { z1.h }, p2/Z, [x20, #2, MUL VL]\n" + ".inst 0xc17117a8 // sdot za.s[x8, 0], { z29.h-z0.h }, z1.h\n" "16:" // Padded: 3 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + "ld1b { z24.s }, p0/Z, [x17]\n" + "add z24.h, p0/M, z24.h, z7.h\n" "add x21, x17, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z21.s }, p0/Z, [x21]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1b { z17.s }, p0/Z, [x21]\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z12.s }, p0/Z, [x21]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + "ld1b { z25.s }, p0/Z, [x21]\n" + "add z25.h, p0/M, z25.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z20.s }, p0/Z, [x21]\n" - "add z20.h, p0/M, z20.h, z9.h\n" + "ld1b { z16.s }, p0/Z, [x21]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "mov x12, #0x4\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z24.h, z24.h, z17.h\n" + "trn1 z25.h, z25.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z13.s }, p0/Z, [x21]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + "ld1b { z26.s }, p0/Z, [x21]\n" + "add z26.h, p0/M, z26.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z19.s }, p0/Z, [x21]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "ld1b { z18.s }, p0/Z, [x21]\n" + "add z18.h, p0/M, z18.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z14.s }, p0/Z, [x21]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1b { z27.s }, p0/Z, [x21]\n" + "add z27.h, p0/M, z27.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z18.s }, p0/Z, [x21]\n" + "ld1b { z17.s }, p0/Z, [x21]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z15.s }, p0/Z, [x21]\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1b { z28.s }, p0/Z, [x21]\n" + "add z28.h, p0/M, z28.h, z7.h\n" "add x21, x21, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z17.s }, p0/Z, [x21]\n" - "add z17.h, p0/M, z17.h, z9.h\n" + "ld1b { z16.s }, p0/Z, [x21]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "addvl x20, SP, #9\n" "add x21, x21, %x[ld_in_row]\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" + "trn1 z26.h, z26.h, z18.h\n" + "trn1 z27.h, z27.h, z17.h\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1b { z16.s }, p0/Z, [x21]\n" - "add z16.h, p0/M, z16.h, z9.h\n" - "mov z16.d, z16.d\n" + ".inst 0xa0402a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20]\n" + "trn1 z28.h, z28.h, z16.h\n" + ".inst 0xc1721708 // sdot za.s[x8, 0], { z24.h-z27.h }, z2.h\n" + "ld1b { z11.s }, p0/Z, [x21]\n" + "add z11.h, p0/M, z11.h, z7.h\n" + "mov z29.d, z11.d\n" "add x17, x17, %x[ld_in_col]\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" + ".inst 0xc1731728 // sdot za.s[x8, 0], { z25.h-z28.h }, z3.h\n" "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1701748 // sdot za.s[x8, 0], { z26.h-z29.h }, z0.h\n" "17:" // Padded: 2 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + "ld1b { z25.s }, p0/Z, [x17]\n" + "add z25.h, p0/M, z25.h, z7.h\n" "add x20, x17, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z21.s }, p0/Z, [x20]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1b { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z12.s }, p0/Z, [x20]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + "ld1b { z26.s }, p0/Z, [x20]\n" + "add z26.h, p0/M, z26.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z9.h\n" + "ld1b { z16.s }, p0/Z, [x20]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z25.h, z25.h, z17.h\n" + "trn1 z26.h, z26.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + "ld1b { z27.s }, p0/Z, [x20]\n" + "add z27.h, p0/M, z27.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "ld1b { z18.s }, p0/Z, [x20]\n" + "add z18.h, p0/M, z18.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1b { z28.s }, p0/Z, [x20]\n" + "add z28.h, p0/M, z28.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" + "ld1b { z17.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1b { z29.s }, p0/Z, [x20]\n" + "add z29.h, p0/M, z29.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z9.h\n" + "ld1b { z16.s }, p0/Z, [x20]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "addvl x21, SP, #6\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" - ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" + "trn1 z27.h, z27.h, z18.h\n" + "trn1 z28.h, z28.h, z17.h\n" + ".inst 0xa1402aa1 // ld1h { z1.h, z9.h }, pn10.b/Z, [x21]\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1b { z16.s }, p0/Z, [x20]\n" + "trn1 z29.h, z29.h, z16.h\n" + ".inst 0xc1711728 // sdot za.s[x8, 0], { z25.h-z28.h }, z1.h\n" + "ld1b { z1.s }, p0/Z, [x20]\n" "addvl x20, SP, #12\n" - "add z16.h, p0/M, z16.h, z9.h\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" - "mov z16.d, z16.d\n" + "add z1.h, p0/M, z1.h, z7.h\n" + ".inst 0xc1791748 // sdot za.s[x8, 0], { z26.h-z29.h }, z9.h\n" + ".inst 0xa0402a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1721729 // sdot za.s[x8, 1], { z25.h-z28.h }, z2.h\n" + "mov z30.d, z1.d\n" "add x17, x17, %x[ld_in_col]\n" - "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + "ld1h { z9.h }, p2/Z, [x21, #2, MUL VL]\n" + ".inst 0xc1731749 // sdot za.s[x8, 1], { z26.h-z29.h }, z3.h\n" + ".inst 0xc1791768 // sdot za.s[x8, 0], { z27.h-z30.h }, z9.h\n" "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1701769 // sdot za.s[x8, 1], { z27.h-z30.h }, z0.h\n" "18:" // Padded: 1 priming loads "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + "ld1b { z25.s }, p0/Z, [x17]\n" + "add z25.h, p0/M, z25.h, z7.h\n" "add x20, x17, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z21.s }, p0/Z, [x20]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1b { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z12.s }, p0/Z, [x20]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + "ld1b { z26.s }, p0/Z, [x20]\n" + "add z26.h, p0/M, z26.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z9.h\n" + "ld1b { z16.s }, p0/Z, [x20]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z25.h, z25.h, z17.h\n" + "trn1 z26.h, z26.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + "ld1b { z27.s }, p0/Z, [x20]\n" + "add z27.h, p0/M, z27.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "ld1b { z18.s }, p0/Z, [x20]\n" + "add z18.h, p0/M, z18.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1b { z28.s }, p0/Z, [x20]\n" + "add z28.h, p0/M, z28.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" + "ld1b { z17.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1b { z29.s }, p0/Z, [x20]\n" + "add z29.h, p0/M, z29.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z9.h\n" + "ld1b { z16.s }, p0/Z, [x20]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "addvl x21, SP, #3\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" - ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" + "trn1 z27.h, z27.h, z18.h\n" + "trn1 z28.h, z28.h, z17.h\n" + ".inst 0xa1402aa3 // ld1h { z3.h, z11.h }, pn10.b/Z, [x21]\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1b { z16.s }, p0/Z, [x20]\n" + "trn1 z29.h, z29.h, z16.h\n" + ".inst 0xc1731728 // sdot za.s[x8, 0], { z25.h-z28.h }, z3.h\n" + "ld1b { z0.s }, p0/Z, [x20]\n" "addvl x20, SP, #9\n" - "add z16.h, p0/M, z16.h, z9.h\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" - "mov z16.d, z16.d\n" + "add z0.h, p0/M, z0.h, z7.h\n" + ".inst 0xc17b1748 // sdot za.s[x8, 0], { z26.h-z29.h }, z11.h\n" + ".inst 0xa0402a82 // ld1h { z2.h-z3.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1721729 // sdot za.s[x8, 1], { z25.h-z28.h }, z2.h\n" + "mov z30.d, z0.d\n" "add x17, x17, %x[ld_in_col]\n" "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1731749 // sdot za.s[x8, 1], { z26.h-z29.h }, z3.h\n" + ".inst 0xc1701768 // sdot za.s[x8, 0], { z27.h-z30.h }, z0.h\n" "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1701769 // sdot za.s[x8, 1], { z27.h-z30.h }, z0.h\n" "19:" // Padded: 0 priming loads "cmp x7, #0x2\n" - ".inst 0xa1402be2 // ld1h { z2.h, z10.h }, pn10.b/Z, [SP]\n" - "ld1h { z0.h }, p2/Z, [SP, #2, MUL VL]\n" + ".inst 0xa1402be3 // ld1h { z3.h, z11.h }, pn10.b/Z, [SP]\n" + "ld1h { z2.h }, p2/Z, [SP, #2, MUL VL]\n" "blt 22f\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + "ld1b { z23.s }, p0/Z, [x17]\n" + "add z23.h, p0/M, z23.h, z7.h\n" "add x20, x17, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z21.s }, p0/Z, [x20]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1b { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z12.s }, p0/Z, [x20]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + "ld1b { z24.s }, p0/Z, [x20]\n" + "add z24.h, p0/M, z24.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z9.h\n" + "ld1b { z16.s }, p0/Z, [x20]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z23.h, z23.h, z17.h\n" + "trn1 z24.h, z24.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + "ld1b { z25.s }, p0/Z, [x20]\n" + "add z25.h, p0/M, z25.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1b { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "add z19.h, p0/M, z19.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1b { z26.s }, p0/Z, [x20]\n" + "add z26.h, p0/M, z26.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "ld1b { z18.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z18.h, p0/M, z18.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1b { z27.s }, p0/Z, [x20]\n" + "add z27.h, p0/M, z27.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z9.h\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z9.h\n" + "add z16.h, p0/M, z16.h, z7.h\n" "sub x7, x7, #0x2\n" "sub x16, x16, #0x1\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" + "trn1 z25.h, z25.h, z19.h\n" + "trn1 z26.h, z26.h, z18.h\n" "lsr x20, x7, #0x1\n" "cmp x20, x16\n" - "trn1 z15.h, z15.h, z17.h\n" - "mov z16.d, z16.d\n" + "trn1 z27.h, z27.h, z17.h\n" + "mov z28.d, z16.d\n" "csel x25, x20, x16, LT\n" "add x17, x17, %x[ld_in_col]\n" "and x7, x7, #0x1\n" "sub x16, x16, x25\n" "cbz x25, 21f\n" "20:" // Padded: Main loop - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" + ".inst 0xc17316e8 // sdot za.s[x8, 0], { z23.h-z26.h }, z3.h\n" "addvl x24, SP, #6\n" "addvl x23, SP, #12\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402b02 // ld1h { z2.h, z10.h }, pn10.b/Z, [x24]\n" + ".inst 0xc17b1708 // sdot za.s[x8, 0], { z24.h-z27.h }, z11.h\n" + ".inst 0xa1402b00 // ld1h { z0.h, z8.h }, pn10.b/Z, [x24]\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" + ".inst 0xc17016e9 // sdot za.s[x8, 1], { z23.h-z26.h }, z0.h\n" "add x20, x17, %x[ld_in_row]\n" "addvl x22, SP, #3\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402ae2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x23]\n" + ".inst 0xc1781709 // sdot za.s[x8, 1], { z24.h-z27.h }, z8.h\n" + ".inst 0xa1402ae3 // ld1h { z3.h, z11.h }, pn10.b/Z, [x23]\n" "addvl x21, SP, #9\n" "subs x25, x25, #0x1\n" - ".inst 0xc172156a // sdot za.s[x8, 2], { z11.h-z14.h }, z2.h\n" - "ld1b { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + ".inst 0xc17316ea // sdot za.s[x8, 2], { z23.h-z26.h }, z3.h\n" + "ld1b { z23.s }, p0/Z, [x17]\n" + "add z23.h, p0/M, z23.h, z7.h\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z21.s }, p0/Z, [x20]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1b { z16.s }, p0/Z, [x20]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc17a158a // sdot za.s[x8, 2], { z12.h-z15.h }, z10.h\n" - "ld1b { z12.s }, p0/Z, [x20]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + ".inst 0xc17b170a // sdot za.s[x8, 2], { z24.h-z27.h }, z11.h\n" + "ld1b { z24.s }, p0/Z, [x20]\n" + "add z24.h, p0/M, z24.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1721728 // sdot za.s[x8, 0], { z25.h-z28.h }, z2.h\n" "ld1h { z0.h }, p2/Z, [x24, #2, MUL VL]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x4\n" - "ld1b { z20.s }, p0/Z, [x20]\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" - "add z20.h, p0/M, z20.h, z9.h\n" + "ld1b { z1.s }, p0/Z, [x20]\n" + ".inst 0xc1701729 // sdot za.s[x8, 1], { z25.h-z28.h }, z0.h\n" + "add z1.h, p0/M, z1.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" - "ld1h { z0.h }, p2/Z, [x23, #2, MUL VL]\n" + "ld1h { z3.h }, p2/Z, [x23, #2, MUL VL]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc17015aa // sdot za.s[x8, 2], { z13.h-z16.h }, z0.h\n" - "trn1 z11.h, z11.h, z21.h\n" - "ld1b { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + ".inst 0xc173172a // sdot za.s[x8, 2], { z25.h-z28.h }, z3.h\n" + "trn1 z23.h, z23.h, z16.h\n" + "ld1b { z25.s }, p0/Z, [x20]\n" + "add z25.h, p0/M, z25.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "ld1b { z3.s }, p0/Z, [x20]\n" + "add z3.h, p0/M, z3.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1b { z26.s }, p0/Z, [x20]\n" + "add z26.h, p0/M, z26.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" + "ld1b { z30.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z30.h, p0/M, z30.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1b { z27.s }, p0/Z, [x20]\n" + "add z27.h, p0/M, z27.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z9.h\n" - "trn1 z12.h, z12.h, z20.h\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" - ".inst 0xa1402ac2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x22]\n" + "ld1b { z29.s }, p0/Z, [x20]\n" + "add z29.h, p0/M, z29.h, z7.h\n" + "trn1 z24.h, z24.h, z1.h\n" + "trn1 z25.h, z25.h, z3.h\n" + "trn1 z26.h, z26.h, z30.h\n" + ".inst 0xa0402ac2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x22]\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" + ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" - "trn1 z15.h, z15.h, z17.h\n" + "trn1 z27.h, z27.h, z29.h\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1b { z16.s }, p0/Z, [x20]\n" + ".inst 0xc17216e8 // sdot za.s[x8, 0], { z23.h-z26.h }, z2.h\n" + "ld1b { z20.s }, p0/Z, [x20]\n" "mov x12, #0x0\n" - "add z16.h, p0/M, z16.h, z9.h\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" + "add z20.h, p0/M, z20.h, z7.h\n" + ".inst 0xc1731708 // sdot za.s[x8, 0], { z24.h-z27.h }, z3.h\n" + ".inst 0xa0402aa2 // ld1h { z2.h-z3.h }, pn10.b/Z, [x21]\n" "add x17, x17, %x[ld_in_col]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" - "ld1b { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + ".inst 0xc17216e9 // sdot za.s[x8, 1], { z23.h-z26.h }, z2.h\n" + "ld1b { z23.s }, p0/Z, [x17]\n" + "add z23.h, p0/M, z23.h, z7.h\n" "add x20, x17, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z21.s }, p0/Z, [x20]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1b { z8.s }, p0/Z, [x20]\n" + "add z8.h, p0/M, z8.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - "ld1b { z12.s }, p0/Z, [x20]\n" - "mov z16.d, z16.d\n" - "ld1h { z0.h }, p2/Z, [x22, #2, MUL VL]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + ".inst 0xc1731709 // sdot za.s[x8, 1], { z24.h-z27.h }, z3.h\n" + "ld1b { z24.s }, p0/Z, [x20]\n" + "mov z28.d, z20.d\n" + "ld1h { z1.h }, p2/Z, [x22, #2, MUL VL]\n" + "add z24.h, p0/M, z24.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z20.s }, p0/Z, [x20]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + "ld1b { z22.s }, p0/Z, [x20]\n" + ".inst 0xc1711728 // sdot za.s[x8, 0], { z25.h-z28.h }, z1.h\n" "mov x12, #0x4\n" - "add z20.h, p0/M, z20.h, z9.h\n" - "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" + "add z22.h, p0/M, z22.h, z7.h\n" + "ld1h { z1.h }, p2/Z, [x21, #2, MUL VL]\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" - "ld1b { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + ".inst 0xc1711729 // sdot za.s[x8, 1], { z25.h-z28.h }, z1.h\n" + "ld1b { z25.s }, p0/Z, [x20]\n" + "add z25.h, p0/M, z25.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "ld1b { z28.s }, p0/Z, [x20]\n" + "add z28.h, p0/M, z28.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1b { z26.s }, p0/Z, [x20]\n" + "add z26.h, p0/M, z26.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" + "ld1b { z20.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z20.h, p0/M, z20.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z15.s }, p0/Z, [x20]\n" - ".inst 0xc1a3ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z3.s\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1b { z27.s }, p0/Z, [x20]\n" + ".inst 0xc1a6ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n" + "add z27.h, p0/M, z27.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" - ".inst 0xc1a1aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" + "ld1b { z31.s }, p0/Z, [x20]\n" + ".inst 0xc1a4aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z4.s\n" "add x20, x20, %x[ld_in_row]\n" - ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" - "add z17.h, p0/M, z17.h, z9.h\n" + ".inst 0xc0040d84 // mova za.d[x8, #4], { z12.d-z15.d }\n" + "add z31.h, p0/M, z31.h, z7.h\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z9.h\n" - ".inst 0xc1a8ab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z8.s\n" - ".inst 0xa1402be2 // ld1h { z2.h, z10.h }, pn10.b/Z, [SP]\n" + "ld1b { z1.s }, p0/Z, [x20]\n" + "add z1.h, p0/M, z1.h, z7.h\n" + ".inst 0xc1aaab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z10.s\n" + ".inst 0xa1402be3 // ld1h { z3.h, z11.h }, pn10.b/Z, [SP]\n" "add x17, x17, %x[ld_in_col]\n" - ".inst 0xc1b7cf44 // sclamp { z4.s-z7.s }, z26.s, z23.s\n" - "st1b { z4.s }, p1, [x15]\n" + ".inst 0xc1b5ccb0 // sclamp { z16.s-z19.s }, z5.s, z21.s\n" + "st1b { z16.s }, p1, [x15]\n" "add x15, x15, x13\n" - "ld1h { z0.h }, p2/Z, [SP, #2, MUL VL]\n" - "st1b { z5.s }, p1, [x14]\n" + "ld1h { z2.h }, p2/Z, [SP, #2, MUL VL]\n" + "st1b { z17.s }, p1, [x14]\n" "add x14, x14, x11\n" - "trn1 z11.h, z11.h, z21.h\n" - "trn1 z12.h, z12.h, z20.h\n" - "st1b { z6.s }, p1, [x10]\n" + "trn1 z23.h, z23.h, z8.h\n" + "trn1 z24.h, z24.h, z22.h\n" + "st1b { z18.s }, p1, [x10]\n" "add x10, x10, x28\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" - "st1b { z7.s }, p1, [x9]\n" + "trn1 z25.h, z25.h, z28.h\n" + "trn1 z26.h, z26.h, z20.h\n" + "st1b { z19.s }, p1, [x9]\n" "add x9, x9, x27\n" - "trn1 z15.h, z15.h, z17.h\n" - "mov z16.d, z16.d\n" + "trn1 z27.h, z27.h, z31.h\n" + "mov z28.d, z1.d\n" "bgt 20b\n" "21:" // Main loop tail - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" + ".inst 0xc17316e8 // sdot za.s[x8, 0], { z23.h-z26.h }, z3.h\n" "addvl x24, SP, #6\n" "addvl x23, SP, #12\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402b02 // ld1h { z2.h, z10.h }, pn10.b/Z, [x24]\n" + ".inst 0xc17b1708 // sdot za.s[x8, 0], { z24.h-z27.h }, z11.h\n" + ".inst 0xa0402b08 // ld1h { z8.h-z9.h }, pn10.b/Z, [x24]\n" "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" + ".inst 0xc17816e9 // sdot za.s[x8, 1], { z23.h-z26.h }, z8.h\n" "add x22, x17, %x[ld_in_row]\n" "addvl x21, SP, #3\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402ae2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x23]\n" + ".inst 0xc1791709 // sdot za.s[x8, 1], { z24.h-z27.h }, z9.h\n" + ".inst 0xa1402ae3 // ld1h { z3.h, z11.h }, pn10.b/Z, [x23]\n" "addvl x20, SP, #9\n" - ".inst 0xc172156a // sdot za.s[x8, 2], { z11.h-z14.h }, z2.h\n" - "ld1b { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + ".inst 0xc17316ea // sdot za.s[x8, 2], { z23.h-z26.h }, z3.h\n" + "ld1b { z29.s }, p0/Z, [x17]\n" + "add z29.h, p0/M, z29.h, z7.h\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z21.s }, p0/Z, [x22]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1b { z8.s }, p0/Z, [x22]\n" + "add z8.h, p0/M, z8.h, z7.h\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - ".inst 0xc17a158a // sdot za.s[x8, 2], { z12.h-z15.h }, z10.h\n" - "ld1b { z12.s }, p0/Z, [x22]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + ".inst 0xc17b170a // sdot za.s[x8, 2], { z24.h-z27.h }, z11.h\n" + "ld1b { z30.s }, p0/Z, [x22]\n" + "add z30.h, p0/M, z30.h, z7.h\n" "add x22, x22, %x[ld_in_row]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" + ".inst 0xc1721728 // sdot za.s[x8, 0], { z25.h-z28.h }, z2.h\n" "ld1h { z0.h }, p2/Z, [x24, #2, MUL VL]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" "mov x12, #0x4\n" "ld1b { z20.s }, p0/Z, [x22]\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" - "add z20.h, p0/M, z20.h, z9.h\n" + ".inst 0xc1701729 // sdot za.s[x8, 1], { z25.h-z28.h }, z0.h\n" + "add z20.h, p0/M, z20.h, z7.h\n" "add x22, x22, %x[ld_in_row]\n" - "ld1h { z0.h }, p2/Z, [x23, #2, MUL VL]\n" + "ld1h { z2.h }, p2/Z, [x23, #2, MUL VL]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - ".inst 0xc17015aa // sdot za.s[x8, 2], { z13.h-z16.h }, z0.h\n" - "trn1 z11.h, z11.h, z21.h\n" - "ld1b { z13.s }, p0/Z, [x22]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + ".inst 0xc172172a // sdot za.s[x8, 2], { z25.h-z28.h }, z2.h\n" + "trn1 z29.h, z29.h, z8.h\n" + "ld1b { z31.s }, p0/Z, [x22]\n" + "add z31.h, p0/M, z31.h, z7.h\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z19.s }, p0/Z, [x22]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "ld1b { z25.s }, p0/Z, [x22]\n" + "add z25.h, p0/M, z25.h, z7.h\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z14.s }, p0/Z, [x22]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1b { z0.s }, p0/Z, [x22]\n" + "add z0.h, p0/M, z0.h, z7.h\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z18.s }, p0/Z, [x22]\n" + "ld1b { z17.s }, p0/Z, [x22]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z15.s }, p0/Z, [x22]\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1b { z1.s }, p0/Z, [x22]\n" + "add z1.h, p0/M, z1.h, z7.h\n" "add x22, x22, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z17.s }, p0/Z, [x22]\n" - "add z17.h, p0/M, z17.h, z9.h\n" - "trn1 z12.h, z12.h, z20.h\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" - ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" + "ld1b { z28.s }, p0/Z, [x22]\n" + "add z28.h, p0/M, z28.h, z7.h\n" + "trn1 z30.h, z30.h, z20.h\n" + "trn1 z31.h, z31.h, z25.h\n" + "trn1 z0.h, z0.h, z17.h\n" + ".inst 0xa1402aa3 // ld1h { z3.h, z11.h }, pn10.b/Z, [x21]\n" "add x22, x22, %x[ld_in_row]\n" - ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" + ".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "trn1 z15.h, z15.h, z17.h\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "ld1b { z16.s }, p0/Z, [x22]\n" - ".inst 0xc1a3ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z3.s\n" - "add z16.h, p0/M, z16.h, z9.h\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" + "trn1 z1.h, z1.h, z28.h\n" + ".inst 0xc17317a8 // sdot za.s[x8, 0], { z29.h-z0.h }, z3.h\n" + "ld1b { z22.s }, p0/Z, [x22]\n" + ".inst 0xc1a6ac18 // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z6.s\n" + "add z22.h, p0/M, z22.h, z7.h\n" + ".inst 0xc17b17c8 // sdot za.s[x8, 0], { z30.h-z1.h }, z11.h\n" + ".inst 0xa1402a83 // ld1h { z3.h, z11.h }, pn10.b/Z, [x20]\n" "add x17, x17, %x[ld_in_col]\n" - ".inst 0xc1a1aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" - "mov z16.d, z16.d\n" - "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xc1a8ab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z8.s\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" - "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - ".inst 0xc1b7cf44 // sclamp { z4.s-z7.s }, z26.s, z23.s\n" - "st1b { z4.s }, p1, [x15]\n" + ".inst 0xc1a4aa38 // srshl { z24.s-z27.s }, { z24.s-z27.s }, z4.s\n" + ".inst 0xc17317a9 // sdot za.s[x8, 1], { z29.h-z0.h }, z3.h\n" + "mov z2.d, z22.d\n" + "ld1h { z9.h }, p2/Z, [x21, #2, MUL VL]\n" + ".inst 0xc17b17c9 // sdot za.s[x8, 1], { z30.h-z1.h }, z11.h\n" + ".inst 0xc1aaab18 // add { z24.s-z27.s }, { z24.s-z27.s }, z10.s\n" + ".inst 0xc17917e8 // sdot za.s[x8, 0], { z31.h-z2.h }, z9.h\n" + "ld1h { z8.h }, p2/Z, [x20, #2, MUL VL]\n" + ".inst 0xc1b5ccb8 // sclamp { z24.s-z27.s }, z5.s, z21.s\n" + "st1b { z24.s }, p1, [x15]\n" "add x15, x15, x13\n" - "st1b { z5.s }, p1, [x14]\n" + "st1b { z25.s }, p1, [x14]\n" "add x14, x14, x11\n" - ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" - ".inst 0xa1402be2 // ld1h { z2.h, z10.h }, pn10.b/Z, [SP]\n" - "st1b { z6.s }, p1, [x10]\n" + ".inst 0xc0040d84 // mova za.d[x8, #4], { z12.d-z15.d }\n" + ".inst 0xa1402be3 // ld1h { z3.h, z11.h }, pn10.b/Z, [SP]\n" + "st1b { z26.s }, p1, [x10]\n" "add x10, x10, x28\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" - "ld1h { z0.h }, p2/Z, [SP, #2, MUL VL]\n" - "st1b { z7.s }, p1, [x9]\n" + ".inst 0xc17817e9 // sdot za.s[x8, 1], { z31.h-z2.h }, z8.h\n" + "ld1h { z2.h }, p2/Z, [SP, #2, MUL VL]\n" + "st1b { z27.s }, p1, [x9]\n" "add x9, x9, x27\n" "22:" // Main loop skip tail "cbz x7, 23f\n" // Skip remainder inputs "mov x12, #0x0\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z11.s }, p0/Z, [x17]\n" - "add z11.h, p0/M, z11.h, z9.h\n" + "ld1b { z24.s }, p0/Z, [x17]\n" + "add z24.h, p0/M, z24.h, z7.h\n" "add x20, x17, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z21.s }, p0/Z, [x20]\n" - "add z21.h, p0/M, z21.h, z9.h\n" + "ld1b { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z12.s }, p0/Z, [x20]\n" - "add z12.h, p0/M, z12.h, z9.h\n" + "ld1b { z25.s }, p0/Z, [x20]\n" + "add z25.h, p0/M, z25.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z20.s }, p0/Z, [x20]\n" - "add z20.h, p0/M, z20.h, z9.h\n" + "ld1b { z16.s }, p0/Z, [x20]\n" + "add z16.h, p0/M, z16.h, z7.h\n" "mov x12, #0x4\n" "add x20, x20, %x[ld_in_row]\n" - "trn1 z11.h, z11.h, z21.h\n" - "trn1 z12.h, z12.h, z20.h\n" + "trn1 z24.h, z24.h, z17.h\n" + "trn1 z25.h, z25.h, z16.h\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z13.s }, p0/Z, [x20]\n" - "add z13.h, p0/M, z13.h, z9.h\n" + "ld1b { z26.s }, p0/Z, [x20]\n" + "add z26.h, p0/M, z26.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z19.s }, p0/Z, [x20]\n" - "add z19.h, p0/M, z19.h, z9.h\n" + "ld1b { z17.s }, p0/Z, [x20]\n" + "add z17.h, p0/M, z17.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "ld1b { z14.s }, p0/Z, [x20]\n" - "add z14.h, p0/M, z14.h, z9.h\n" + "ld1b { z27.s }, p0/Z, [x20]\n" + "add z27.h, p0/M, z27.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" - "ld1b { z18.s }, p0/Z, [x20]\n" + "ld1b { z16.s }, p0/Z, [x20]\n" "mov x12, #0x8\n" - "add z18.h, p0/M, z18.h, z9.h\n" + "add z16.h, p0/M, z16.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" - "ld1b { z15.s }, p0/Z, [x20]\n" - "add z15.h, p0/M, z15.h, z9.h\n" + "ld1b { z28.s }, p0/Z, [x20]\n" + "add z28.h, p0/M, z28.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" - "ld1b { z17.s }, p0/Z, [x20]\n" - "add z17.h, p0/M, z17.h, z9.h\n" + "ld1b { z31.s }, p0/Z, [x20]\n" + "add z31.h, p0/M, z31.h, z7.h\n" "add x20, x20, %x[ld_in_row]\n" ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" - "trn1 z13.h, z13.h, z19.h\n" - "trn1 z14.h, z14.h, z18.h\n" - "ld1b { z16.s }, p0/Z, [x20]\n" - "add z16.h, p0/M, z16.h, z9.h\n" - "trn1 z15.h, z15.h, z17.h\n" + "trn1 z26.h, z26.h, z17.h\n" + "trn1 z27.h, z27.h, z16.h\n" + "ld1b { z0.s }, p0/Z, [x20]\n" + "add z0.h, p0/M, z0.h, z7.h\n" + "trn1 z28.h, z28.h, z31.h\n" "addvl x21, SP, #6\n" - ".inst 0xc1721568 // sdot za.s[x8, 0], { z11.h-z14.h }, z2.h\n" - "mov z16.d, z16.d\n" + ".inst 0xc1731708 // sdot za.s[x8, 0], { z24.h-z27.h }, z3.h\n" + "mov z29.d, z0.d\n" "addvl x20, SP, #12\n" "sub x16, x16, #0x1\n" - ".inst 0xc17a1588 // sdot za.s[x8, 0], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402aa2 // ld1h { z2.h, z10.h }, pn10.b/Z, [x21]\n" - ".inst 0xc17015a8 // sdot za.s[x8, 0], { z13.h-z16.h }, z0.h\n" - "ld1h { z0.h }, p2/Z, [x21, #2, MUL VL]\n" - ".inst 0xc1721569 // sdot za.s[x8, 1], { z11.h-z14.h }, z2.h\n" - ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" - ".inst 0xc1a3ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z3.s\n" - ".inst 0xc17a1589 // sdot za.s[x8, 1], { z12.h-z15.h }, z10.h\n" - ".inst 0xa1402a82 // ld1h { z2.h, z10.h }, pn10.b/Z, [x20]\n" - ".inst 0xc1a1aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" - ".inst 0xc172156a // sdot za.s[x8, 2], { z11.h-z14.h }, z2.h\n" - ".inst 0xc1a8ab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z8.s\n" - ".inst 0xc17a158a // sdot za.s[x8, 2], { z12.h-z15.h }, z10.h\n" - ".inst 0xc1b7cf44 // sclamp { z4.s-z7.s }, z26.s, z23.s\n" - "st1b { z4.s }, p1, [x15]\n" + ".inst 0xc17b1728 // sdot za.s[x8, 0], { z25.h-z28.h }, z11.h\n" + ".inst 0xa0402aa8 // ld1h { z8.h-z9.h }, pn10.b/Z, [x21]\n" + ".inst 0xc1721748 // sdot za.s[x8, 0], { z26.h-z29.h }, z2.h\n" + "ld1h { z2.h }, p2/Z, [x21, #2, MUL VL]\n" + ".inst 0xc1781709 // sdot za.s[x8, 1], { z24.h-z27.h }, z8.h\n" + ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" + ".inst 0xc1a6ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n" + ".inst 0xc1791729 // sdot za.s[x8, 1], { z25.h-z28.h }, z9.h\n" + ".inst 0xa1402a81 // ld1h { z1.h, z9.h }, pn10.b/Z, [x20]\n" + ".inst 0xc1a4aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z4.s\n" + ".inst 0xc171170a // sdot za.s[x8, 2], { z24.h-z27.h }, z1.h\n" + ".inst 0xc1aaab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z10.s\n" + ".inst 0xc179172a // sdot za.s[x8, 2], { z25.h-z28.h }, z9.h\n" + ".inst 0xc1b5ccb0 // sclamp { z16.s-z19.s }, z5.s, z21.s\n" + "st1b { z16.s }, p1, [x15]\n" "add x15, x15, x13\n" - ".inst 0xc17015a9 // sdot za.s[x8, 1], { z13.h-z16.h }, z0.h\n" - "ld1h { z0.h }, p2/Z, [x20, #2, MUL VL]\n" - "st1b { z5.s }, p1, [x14]\n" + ".inst 0xc1721749 // sdot za.s[x8, 1], { z26.h-z29.h }, z2.h\n" + "ld1h { z3.h }, p2/Z, [x20, #2, MUL VL]\n" + "st1b { z17.s }, p1, [x14]\n" "add x14, x14, x11\n" - ".inst 0xc17015aa // sdot za.s[x8, 2], { z13.h-z16.h }, z0.h\n" + ".inst 0xc173174a // sdot za.s[x8, 2], { z26.h-z29.h }, z3.h\n" "add x8, x8, #0x1\n" - "st1b { z6.s }, p1, [x10]\n" + "st1b { z18.s }, p1, [x10]\n" "add x10, x10, x28\n" - "st1b { z7.s }, p1, [x9]\n" + "st1b { z19.s }, p1, [x9]\n" "add x9, x9, x27\n" - ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" + ".inst 0xc0040d84 // mova za.d[x8, #4], { z12.d-z15.d }\n" "23:" // Tail input: End "cbz x16, 25f\n" "24:" // Right padding loop - ".inst 0xc0060c04 // mova { z4.d-z7.d }, za.d[x8, #0]\n" - ".inst 0xc1a3ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z3.s\n" + ".inst 0xc0060c1c // mova { z28.d-z31.d }, za.d[x8, #0]\n" + ".inst 0xc1a6ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z6.s\n" "add x8, x8, #0x1\n" - ".inst 0xc1a1aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" + ".inst 0xc1a4aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" "subs x16, x16, #0x1\n" - ".inst 0xc0040f84 // mova za.d[x8, #4], { z28.d-z31.d }\n" - ".inst 0xc1a8ab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z8.s\n" - ".inst 0xc1b7cf44 // sclamp { z4.s-z7.s }, z26.s, z23.s\n" - "st1b { z4.s }, p1, [x15]\n" + ".inst 0xc0040d84 // mova za.d[x8, #4], { z12.d-z15.d }\n" + ".inst 0xc1aaab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z10.s\n" + ".inst 0xc1b5ccbc // sclamp { z28.s-z31.s }, z5.s, z21.s\n" + "st1b { z28.s }, p1, [x15]\n" "add x15, x15, x13\n" - "st1b { z5.s }, p1, [x14]\n" + "st1b { z29.s }, p1, [x14]\n" "add x14, x14, x11\n" - "st1b { z6.s }, p1, [x10]\n" + "st1b { z30.s }, p1, [x10]\n" "add x10, x10, x28\n" - "st1b { z7.s }, p1, [x9]\n" + "st1b { z31.s }, p1, [x9]\n" "add x9, x9, x27\n" "bgt 24b\n" "25:" // End - "ldr x22, [%x[args], %[offsetof_Args_weights]]\n" - "incw x22, ALL, MUL #16\n" - "incw x22, ALL, MUL #9\n" - "str x22, [%x[args], %[offsetof_Args_weights]]\n" - "ldr x20, [%x[args], %[offsetof_Args_ld_in_vl]]\n" + "ldr x20, [%x[args], %[offsetof_Args_weights]]\n" + "incw x20, ALL, MUL #16\n" + "incw x20, ALL, MUL #9\n" + "str x20, [%x[args], %[offsetof_Args_weights]]\n" + "ldr x21, [%x[args], %[offsetof_Args_ld_in_vl]]\n" "incw x6\n" "whilelt p1.s, x6, x5\n" - "ldr x17, [%x[args], %[offsetof_Args_inptr]]\n" - "add x17, x17, x20\n" - "str x17, [%x[args], %[offsetof_Args_inptr]]\n" + "ldr x20, [%x[args], %[offsetof_Args_inptr]]\n" + "add x20, x20, x21\n" + "str x20, [%x[args], %[offsetof_Args_inptr]]\n" "ldr x25, [%x[args], %[offsetof_Args_outptrs]]\n" "ldr x24, [%x[args], %[offsetof_Args_ld_out_vls]]\n" "ldp x23, x22, [x25, #0x0]\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp index 1c1fb25e1f99610f37bcb48bef0aac7b98052381..edee21e9417037d04e299a33fa7a3231f1524491 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,19 +22,19 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include #pragma once -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { -void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); -void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); +void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const input_ptrs, __fp16 *const *const outptrs, const void *params, unsigned int n_channels, const __fp16 activation_min, const __fp16 activation_max); +void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int n_tile_rows, const unsigned int n_tile_cols, const __fp16 *inptr, int64_t ld_input_row, int64_t ld_input_col, __fp16 *outptr, int64_t ld_output_row, int64_t ld_output_col, const void *params, unsigned int n_channels, const __fp16 activation_min, const __fp16 activation_max); class sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirstStrategy<__fp16, __fp16, __fp16, __fp16> { @@ -57,7 +57,7 @@ class sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirst constexpr static unsigned int output_cols = 2; sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst(const CPUInfo *) - : DepthwiseDepthfirstStrategy<__fp16, __fp16, __fp16, __fp16>(2, 3, 1) {} + : Parent(output_rows, output_cols, kernel_rows, kernel_cols, stride_rows, stride_cols) {} arm_gemm::VLType get_vl_type(void) const override { return vl_type; } @@ -68,4 +68,4 @@ class sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirst } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp index 9fd220abf867df4a9bed1d48ef7c0ac99ad47bda..d807856ccb42bfe0a1cec9e37ef3c076cc54738e 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -108,10 +108,10 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "whilelt p2.h, XZR, %x[n_channels]\n" "madd x20, x14, x12, x20\n" // offset += tile_j * ld_output_col "ldr x28, [%x[params_struct], %[offsetof_args_outptr]]\n" - "ld1h { z18.h }, p3/Z, [x10]\n" + "ld1h { z27.h }, p3/Z, [x10]\n" "add x27, x13, x13\n" "mul x21, x21, x25\n" // offset *= kernel_stride * output_size - "add x9, x9, x21, LSL #1\n" // inptr[0] += offset * sizeof(__fp16) + "add x9, x9, x21, LSL #1\n" // inptr[0] += offset * sizeof(__fp16) "ld1h { z0.h }, p3/Z, [x10, #1, MUL VL]\n" "ld1h { z1.h }, p3/Z, [x10, #2, MUL VL]\n" "mul x20, x20, x24\n" // offset *= output_tile_size @@ -125,10 +125,10 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "ld1h { z6.h }, p3/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" "add x28, x28, x20, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16) - "ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rh { z26.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" "cmp x11, %x[n_channels]\n" "add x23, x25, x23, LSL #1\n" - "ld1rh { z16.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "ld1rh { z25.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "ld1h { z7.h }, p3/Z, [x10, #-8, MUL VL]\n" "add x22, x28, x22, LSL #1\n" "mov x21, #0x0\n" @@ -142,175 +142,175 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "ld1h { z13.h }, p2/Z, [x25, x13, LSL #1]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "movprfx z28, z18\n fmla z28.h, p3/M, z4.h, z9.h\n" - "movprfx z29, z18\n fmla z29.h, p3/M, z3.h, z9.h\n" + "movprfx z24, z27\n fmla z24.h, p3/M, z4.h, z9.h\n" + "movprfx z23, z27\n fmla z23.h, p3/M, z3.h, z9.h\n" "whilelt p1.h, x11, %x[n_channels]\n" "inch x21\n" - "movprfx z30, z18\n fmla z30.h, p3/M, z1.h, z9.h\n" - "movprfx z31, z18\n fmla z31.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x23]\n" + "movprfx z22, z27\n fmla z22.h, p3/M, z1.h, z9.h\n" + "movprfx z21, z27\n fmla z21.h, p3/M, z0.h, z9.h\n" + "ld1h { z18.h }, p2/Z, [x23]\n" "inch x11\n" - "fmla z28.h, p3/M, z0.h, z10.h\n" - "fmla z29.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x23, x24, LSL #1]\n" - "ld1h { z10.h }, p2/Z, [x25, x27, LSL #1]\n" - "fmla z30.h, p3/M, z2.h, z12.h\n" - "fmla z31.h, p3/M, z1.h, z12.h\n" + "fmla z24.h, p3/M, z0.h, z10.h\n" + "fmla z23.h, p3/M, z2.h, z11.h\n" + "ld1h { z17.h }, p2/Z, [x23, x24, LSL #1]\n" + "ld1h { z20.h }, p2/Z, [x25, x27, LSL #1]\n" + "fmla z22.h, p3/M, z2.h, z12.h\n" + "fmla z21.h, p3/M, z1.h, z12.h\n" "mov p0.b, p2.b\n" - "ld1h { z18.h }, p3/Z, [x10]\n" - "fmla z28.h, p3/M, z5.h, z12.h\n" - "fmla z29.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9, x13, LSL #1]\n" + "ld1h { z27.h }, p3/Z, [x10]\n" + "fmla z24.h, p3/M, z5.h, z12.h\n" + "fmla z23.h, p3/M, z4.h, z12.h\n" + "ld1h { z16.h }, p2/Z, [x9, x13, LSL #1]\n" "inch x20\n" - "fmla z30.h, p3/M, z6.h, z9.h\n" - "fmla z31.h, p3/M, z3.h, z13.h\n" - "ld1h { z9.h }, p2/Z, [x9, x27, LSL #1]\n" + "fmla z22.h, p3/M, z6.h, z18.h\n" + "fmla z21.h, p3/M, z3.h, z13.h\n" + "ld1h { z18.h }, p2/Z, [x9, x27, LSL #1]\n" "addvl x9, x9, #1\n" - "fmla z28.h, p3/M, z7.h, z13.h\n" - "fmla z29.h, p3/M, z6.h, z13.h\n" - "fmla z30.h, p3/M, z4.h, z13.h\n" - "fmla z31.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x26]\n" - "fmla z28.h, p3/M, z1.h, z12.h\n" - "fmla z29.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x26, x24, LSL #1]\n" + "fmla z24.h, p3/M, z7.h, z13.h\n" + "fmla z23.h, p3/M, z6.h, z13.h\n" + "fmla z22.h, p3/M, z4.h, z13.h\n" + "fmla z21.h, p3/M, z8.h, z17.h\n" + "ld1h { z17.h }, p2/Z, [x26]\n" + "fmla z24.h, p3/M, z1.h, z16.h\n" + "fmla z23.h, p3/M, z0.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x26, x24, LSL #1]\n" "addvl x26, x26, #1\n" - "fmla z30.h, p3/M, z5.h, z10.h\n" - "fmla z31.h, p3/M, z4.h, z10.h\n" + "fmla z22.h, p3/M, z5.h, z20.h\n" + "fmla z21.h, p3/M, z4.h, z20.h\n" "ld1h { z4.h }, p3/Z, [x10, #5, MUL VL]\n" - "fmla z28.h, p3/M, z2.h, z9.h\n" - "fmla z29.h, p3/M, z1.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x25]\n" + "fmla z24.h, p3/M, z2.h, z18.h\n" + "fmla z23.h, p3/M, z1.h, z18.h\n" + "ld1h { z19.h }, p2/Z, [x25]\n" "ld1h { z1.h }, p3/Z, [x10, #2, MUL VL]\n" - "fmla z30.h, p3/M, z0.h, z11.h\n" - "fmla z31.h, p3/M, z2.h, z12.h\n" + "fmla z22.h, p3/M, z0.h, z17.h\n" + "fmla z21.h, p3/M, z2.h, z16.h\n" "ld1h { z0.h }, p3/Z, [x10, #1, MUL VL]\n" "ld1h { z2.h }, p3/Z, [x10, #3, MUL VL]\n" - "fmla z28.h, p3/M, z8.h, z10.h\n" - "fmla z29.h, p3/M, z7.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x25, x24, LSL #1]\n" + "fmla z24.h, p3/M, z8.h, z20.h\n" + "fmla z23.h, p3/M, z7.h, z20.h\n" + "ld1h { z18.h }, p2/Z, [x25, x24, LSL #1]\n" "addvl x25, x25, #1\n" - "fmla z30.h, p3/M, z3.h, z9.h\n" - "fmla z31.h, p3/M, z5.h, z10.h\n" + "fmla z22.h, p3/M, z3.h, z19.h\n" + "fmla z21.h, p3/M, z5.h, z18.h\n" "ld1h { z13.h }, p1/Z, [x25, x13, LSL #1]\n" - "fmla z28.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x23, x13, LSL #1]\n" - "fmla z29.h, p3/M, z5.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x23, x27, LSL #1]\n" - "fmla z30.h, p3/M, z7.h, z11.h\n" - "fmla z31.h, p3/M, z6.h, z11.h\n" + "fmla z24.h, p3/M, z3.h, z17.h\n" + "ld1h { z17.h }, p2/Z, [x23, x13, LSL #1]\n" + "fmla z23.h, p3/M, z5.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x23, x27, LSL #1]\n" + "fmla z22.h, p3/M, z7.h, z17.h\n" + "fmla z21.h, p3/M, z6.h, z17.h\n" "ld1h { z3.h }, p3/Z, [x10, #4, MUL VL]\n" "ld1h { z5.h }, p3/Z, [x10, #6, MUL VL]\n" - "fmla z28.h, p3/M, z6.h, z9.h\n" - "fmla z29.h, p3/M, z8.h, z10.h\n" - "fmax z28.h, p3/M, z28.h, z17.h\n" - "fmax z29.h, p3/M, z29.h, z17.h\n" - "fmla z30.h, p3/M, z8.h, z12.h\n" - "fmla z31.h, p3/M, z7.h, z12.h\n" - "fmax z30.h, p3/M, z30.h, z17.h\n" - "fmax z31.h, p3/M, z31.h, z17.h\n" + "fmla z24.h, p3/M, z6.h, z19.h\n" + "fmla z23.h, p3/M, z8.h, z18.h\n" + "fmax z24.h, p3/M, z24.h, z26.h\n" + "fmax z23.h, p3/M, z23.h, z26.h\n" + "fmla z22.h, p3/M, z8.h, z16.h\n" + "fmla z21.h, p3/M, z7.h, z16.h\n" + "fmax z22.h, p3/M, z22.h, z26.h\n" + "fmax z21.h, p3/M, z21.h, z26.h\n" "ld1h { z6.h }, p3/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" "whilelt p2.h, x21, %x[n_channels]\n" "ld1h { z9.h }, p1/Z, [x26, x13, LSL #1]\n" "cmp x11, %x[n_channels]\n" - "fmin z28.h, p3/M, z28.h, z16.h\n" + "fmin z24.h, p3/M, z24.h, z25.h\n" "ld1h { z10.h }, p1/Z, [x9]\n" "ld1h { z11.h }, p1/Z, [x9, x24, LSL #1]\n" - "fmin z29.h, p3/M, z29.h, z16.h\n" - "fmin z30.h, p3/M, z30.h, z16.h\n" + "fmin z23.h, p3/M, z23.h, z25.h\n" + "fmin z22.h, p3/M, z22.h, z25.h\n" "ld1h { z12.h }, p1/Z, [x26, x27, LSL #1]\n" - "st1h { z28.h }, p0, [x28]\n" - "fmin z31.h, p3/M, z31.h, z16.h\n" + "st1h { z24.h }, p0, [x28]\n" + "fmin z21.h, p3/M, z21.h, z25.h\n" "addvl x23, x23, #1\n" - "st1h { z29.h }, p0, [x28, x12, LSL #1]\n" + "st1h { z23.h }, p0, [x28, x12, LSL #1]\n" "ld1h { z7.h }, p3/Z, [x10, #-8, MUL VL]\n" - "st1h { z30.h }, p0, [x22]\n" + "st1h { z22.h }, p0, [x22]\n" "addvl x28, x28, #1\n" "ld1h { z8.h }, p3/Z, [x10, #-7, MUL VL]\n" "addvl x10, x10, #-6\n" - "st1h { z31.h }, p0, [x22, x12, LSL #1]\n" + "st1h { z21.h }, p0, [x22, x12, LSL #1]\n" "addvl x22, x22, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "movprfx z28, z18\n fmla z28.h, p3/M, z4.h, z9.h\n" - "movprfx z29, z18\n fmla z29.h, p3/M, z3.h, z9.h\n" + "movprfx z24, z27\n fmla z24.h, p3/M, z4.h, z9.h\n" + "movprfx z23, z27\n fmla z23.h, p3/M, z3.h, z9.h\n" "ldr x14, [%x[params_struct], %[offsetof_args_tile_j]]\n" "ldr x10, [%x[params_struct], %[offsetof_args_tile_i]]\n" - "movprfx z30, z18\n fmla z30.h, p3/M, z1.h, z9.h\n" - "movprfx z31, z18\n fmla z31.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x23]\n" + "movprfx z22, z27\n fmla z22.h, p3/M, z1.h, z9.h\n" + "movprfx z21, z27\n fmla z21.h, p3/M, z0.h, z9.h\n" + "ld1h { z18.h }, p2/Z, [x23]\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" - "fmla z28.h, p3/M, z0.h, z10.h\n" - "fmla z29.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x23, x24, LSL #1]\n" - "ld1h { z10.h }, p2/Z, [x25, x27, LSL #1]\n" - "fmla z30.h, p3/M, z2.h, z12.h\n" - "fmla z31.h, p3/M, z1.h, z12.h\n" + "fmla z24.h, p3/M, z0.h, z10.h\n" + "fmla z23.h, p3/M, z2.h, z11.h\n" + "ld1h { z17.h }, p2/Z, [x23, x24, LSL #1]\n" + "ld1h { z20.h }, p2/Z, [x25, x27, LSL #1]\n" + "fmla z22.h, p3/M, z2.h, z12.h\n" + "fmla z21.h, p3/M, z1.h, z12.h\n" "add x14, x14, #0x1\n" "cmp x14, x20\n" - "fmla z28.h, p3/M, z5.h, z12.h\n" - "fmla z29.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9, x13, LSL #1]\n" + "fmla z24.h, p3/M, z5.h, z12.h\n" + "fmla z23.h, p3/M, z4.h, z12.h\n" + "ld1h { z16.h }, p2/Z, [x9, x13, LSL #1]\n" "add x21, x10, #0x1\n" - "fmla z30.h, p3/M, z6.h, z9.h\n" - "fmla z31.h, p3/M, z3.h, z13.h\n" - "ld1h { z9.h }, p2/Z, [x9, x27, LSL #1]\n" + "fmla z22.h, p3/M, z6.h, z18.h\n" + "fmla z21.h, p3/M, z3.h, z13.h\n" + "ld1h { z18.h }, p2/Z, [x9, x27, LSL #1]\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" - "fmla z28.h, p3/M, z7.h, z13.h\n" - "fmla z29.h, p3/M, z6.h, z13.h\n" + "fmla z24.h, p3/M, z7.h, z13.h\n" + "fmla z23.h, p3/M, z6.h, z13.h\n" "csel x10, x10, x21, LT\n" "mov p0.b, p2.b\n" - "fmla z30.h, p3/M, z4.h, z13.h\n" - "fmla z31.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x26]\n" + "fmla z22.h, p3/M, z4.h, z13.h\n" + "fmla z21.h, p3/M, z8.h, z17.h\n" + "ld1h { z17.h }, p2/Z, [x26]\n" "csel x14, x14, XZR, LT\n" - "fmla z28.h, p3/M, z1.h, z12.h\n" - "fmla z29.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x26, x24, LSL #1]\n" + "fmla z24.h, p3/M, z1.h, z16.h\n" + "fmla z23.h, p3/M, z0.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x26, x24, LSL #1]\n" "cmp x10, x20\n" - "fmla z30.h, p3/M, z5.h, z10.h\n" - "fmla z31.h, p3/M, z4.h, z10.h\n" - "fmla z28.h, p3/M, z2.h, z9.h\n" - "fmla z29.h, p3/M, z1.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x25]\n" - "fmla z30.h, p3/M, z0.h, z11.h\n" - "fmla z31.h, p3/M, z2.h, z12.h\n" - "fmla z28.h, p3/M, z8.h, z10.h\n" - "fmla z29.h, p3/M, z7.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x25, x24, LSL #1]\n" - "fmla z30.h, p3/M, z3.h, z9.h\n" - "fmla z31.h, p3/M, z5.h, z10.h\n" - "fmla z28.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x23, x13, LSL #1]\n" - "fmla z29.h, p3/M, z5.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x23, x27, LSL #1]\n" - "fmla z30.h, p3/M, z7.h, z11.h\n" - "fmla z31.h, p3/M, z6.h, z11.h\n" - "fmla z28.h, p3/M, z6.h, z9.h\n" - "fmla z29.h, p3/M, z8.h, z10.h\n" - "fmax z28.h, p3/M, z28.h, z17.h\n" - "fmax z29.h, p3/M, z29.h, z17.h\n" - "fmla z30.h, p3/M, z8.h, z12.h\n" - "fmla z31.h, p3/M, z7.h, z12.h\n" - "fmax z30.h, p3/M, z30.h, z17.h\n" - "fmax z31.h, p3/M, z31.h, z17.h\n" - "fmin z28.h, p3/M, z28.h, z16.h\n" - "fmin z29.h, p3/M, z29.h, z16.h\n" - "st1h { z28.h }, p0, [x28]\n" - "fmin z30.h, p3/M, z30.h, z16.h\n" - "fmin z31.h, p3/M, z31.h, z16.h\n" - "st1h { z29.h }, p0, [x28, x12, LSL #1]\n" - "st1h { z30.h }, p0, [x22]\n" - "st1h { z31.h }, p0, [x22, x12, LSL #1]\n" + "fmla z22.h, p3/M, z5.h, z20.h\n" + "fmla z21.h, p3/M, z4.h, z20.h\n" + "fmla z24.h, p3/M, z2.h, z18.h\n" + "fmla z23.h, p3/M, z1.h, z18.h\n" + "ld1h { z19.h }, p2/Z, [x25]\n" + "fmla z22.h, p3/M, z0.h, z17.h\n" + "fmla z21.h, p3/M, z2.h, z16.h\n" + "fmla z24.h, p3/M, z8.h, z20.h\n" + "fmla z23.h, p3/M, z7.h, z20.h\n" + "ld1h { z18.h }, p2/Z, [x25, x24, LSL #1]\n" + "fmla z22.h, p3/M, z3.h, z19.h\n" + "fmla z21.h, p3/M, z5.h, z18.h\n" + "fmla z24.h, p3/M, z3.h, z17.h\n" + "ld1h { z17.h }, p2/Z, [x23, x13, LSL #1]\n" + "fmla z23.h, p3/M, z5.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x23, x27, LSL #1]\n" + "fmla z22.h, p3/M, z7.h, z17.h\n" + "fmla z21.h, p3/M, z6.h, z17.h\n" + "fmla z24.h, p3/M, z6.h, z19.h\n" + "fmla z23.h, p3/M, z8.h, z18.h\n" + "fmax z24.h, p3/M, z24.h, z26.h\n" + "fmax z23.h, p3/M, z23.h, z26.h\n" + "fmla z22.h, p3/M, z8.h, z16.h\n" + "fmla z21.h, p3/M, z7.h, z16.h\n" + "fmax z22.h, p3/M, z22.h, z26.h\n" + "fmax z21.h, p3/M, z21.h, z26.h\n" + "fmin z24.h, p3/M, z24.h, z25.h\n" + "fmin z23.h, p3/M, z23.h, z25.h\n" + "st1h { z24.h }, p0, [x28]\n" + "fmin z22.h, p3/M, z22.h, z25.h\n" + "fmin z21.h, p3/M, z21.h, z25.h\n" + "st1h { z23.h }, p0, [x28, x12, LSL #1]\n" + "st1h { z22.h }, p0, [x22]\n" + "st1h { z21.h }, p0, [x22, x12, LSL #1]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp index 9242b470c3ef25bcfd95bdd7978ef694492ba539..90982b6990ae872c90e9ee567522d04bb5a2e227 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -87,7 +87,7 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( "ldp x11, x10, [x20, #0x10]\n" "mov x9, #0x0\n" "whilelt p2.h, XZR, %x[n_channels]\n" - "ld1h { z18.h }, p3/Z, [x16]\n" + "ld1h { z20.h }, p3/Z, [x16]\n" "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n" "cmp x14, %x[n_channels]\n" "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n" @@ -98,99 +98,99 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n" "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n" "addvl x16, x16, #16\n" - "ldp x27, x26, [x15, #0x0]\n" - "ldp x25, x24, [x15, #0x10]\n" - "ldr x23, [x15, #0x20]\n" - "ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "ld1rh { z16.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "ldp x24, x23, [x15, #0x0]\n" + "ldp x22, x21, [x15, #0x10]\n" + "ldr x20, [x15, #0x20]\n" + "ld1rh { z26.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rh { z25.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n" "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n" - "ld1h { z9.h }, p2/Z, [x27, x9, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x24, x9, LSL #1]\n" "addvl x16, x16, #-6\n" - "ld1h { z10.h }, p2/Z, [x26, x9, LSL #1]\n" - "ld1h { z11.h }, p2/Z, [x25, x9, LSL #1]\n" - "ld1h { z12.h }, p2/Z, [x24, x9, LSL #1]\n" - "ld1h { z13.h }, p2/Z, [x23, x9, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x23, x9, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x22, x9, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x21, x9, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x20, x9, LSL #1]\n" "bge 2f\n" "1:" // Channel loop - "movprfx z28, z18\n fmla z28.h, p3/M, z4.h, z9.h\n" - "movprfx z29, z18\n fmla z29.h, p3/M, z3.h, z9.h\n" - "ldr x22, [x15, #0x28]\n" - "ldr x21, [x15, #0x30]\n" - "movprfx z30, z18\n fmla z30.h, p3/M, z1.h, z9.h\n" - "movprfx z31, z18\n fmla z31.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x22, x9, LSL #1]\n" - "ldr x20, [x15, #0x38]\n" - "fmla z28.h, p3/M, z0.h, z10.h\n" - "fmla z29.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x21, x9, LSL #1]\n" - "ldr x26, [x15, #0x48]\n" - "fmla z30.h, p3/M, z2.h, z12.h\n" - "fmla z31.h, p3/M, z1.h, z12.h\n" - "ldr x27, [x15, #0x40]\n" - "ld1h { z10.h }, p2/Z, [x26, x9, LSL #1]\n" - "fmla z28.h, p3/M, z5.h, z12.h\n" - "fmla z29.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x20, x9, LSL #1]\n" - "ldr x25, [x15, #0x50]\n" - "fmla z30.h, p3/M, z6.h, z9.h\n" - "fmla z31.h, p3/M, z3.h, z13.h\n" - "ld1h { z9.h }, p2/Z, [x27, x9, LSL #1]\n" - "ldr x24, [x15, #0x58]\n" - "fmla z28.h, p3/M, z7.h, z13.h\n" - "fmla z29.h, p3/M, z6.h, z13.h\n" - "ldr x23, [x15, #0x60]\n" - "ldr x22, [x15, #0x68]\n" - "fmla z30.h, p3/M, z4.h, z13.h\n" - "fmla z31.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x25, x9, LSL #1]\n" - "ldr x21, [x15, #0x70]\n" - "fmla z28.h, p3/M, z1.h, z12.h\n" - "fmla z29.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x24, x9, LSL #1]\n" - "ldr x20, [x15, #0x78]\n" - "fmla z30.h, p3/M, z5.h, z10.h\n" - "fmla z31.h, p3/M, z4.h, z10.h\n" + "movprfx z24, z20\n fmla z24.h, p3/M, z4.h, z9.h\n" + "movprfx z23, z20\n fmla z23.h, p3/M, z3.h, z9.h\n" + "ldr x21, [x15, #0x28]\n" + "ldr x20, [x15, #0x30]\n" + "movprfx z22, z20\n fmla z22.h, p3/M, z1.h, z9.h\n" + "movprfx z21, z20\n fmla z21.h, p3/M, z0.h, z9.h\n" + "ld1h { z18.h }, p2/Z, [x21, x9, LSL #1]\n" + "ldr x22, [x15, #0x38]\n" + "fmla z24.h, p3/M, z0.h, z10.h\n" + "fmla z23.h, p3/M, z2.h, z11.h\n" + "ld1h { z17.h }, p2/Z, [x20, x9, LSL #1]\n" + "ldr x21, [x15, #0x48]\n" + "fmla z22.h, p3/M, z2.h, z12.h\n" + "fmla z21.h, p3/M, z1.h, z12.h\n" + "ldr x20, [x15, #0x40]\n" + "ld1h { z20.h }, p2/Z, [x21, x9, LSL #1]\n" + "fmla z24.h, p3/M, z5.h, z12.h\n" + "fmla z23.h, p3/M, z4.h, z12.h\n" + "ld1h { z16.h }, p2/Z, [x22, x9, LSL #1]\n" + "ldr x22, [x15, #0x50]\n" + "fmla z22.h, p3/M, z6.h, z18.h\n" + "fmla z21.h, p3/M, z3.h, z13.h\n" + "ld1h { z18.h }, p2/Z, [x20, x9, LSL #1]\n" + "ldr x21, [x15, #0x58]\n" + "fmla z24.h, p3/M, z7.h, z13.h\n" + "fmla z23.h, p3/M, z6.h, z13.h\n" + "ldr x20, [x15, #0x60]\n" + "ldr x27, [x15, #0x68]\n" + "fmla z22.h, p3/M, z4.h, z13.h\n" + "fmla z21.h, p3/M, z8.h, z17.h\n" + "ld1h { z17.h }, p2/Z, [x22, x9, LSL #1]\n" + "ldr x26, [x15, #0x70]\n" + "fmla z24.h, p3/M, z1.h, z16.h\n" + "fmla z23.h, p3/M, z0.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x21, x9, LSL #1]\n" + "ldr x25, [x15, #0x78]\n" + "fmla z22.h, p3/M, z5.h, z20.h\n" + "fmla z21.h, p3/M, z4.h, z20.h\n" "whilelt p1.h, x14, %x[n_channels]\n" - "ldp x27, x26, [x15, #0x0]\n" - "fmla z28.h, p3/M, z2.h, z9.h\n" - "fmla z29.h, p3/M, z1.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x23, x9, LSL #1]\n" - "ldp x25, x24, [x15, #0x10]\n" - "fmla z30.h, p3/M, z0.h, z11.h\n" - "fmla z31.h, p3/M, z2.h, z12.h\n" - "ldr x23, [x15, #0x20]\n" - "ld1h { z13.h }, p1/Z, [x23, x14, LSL #1]\n" - "fmla z28.h, p3/M, z8.h, z10.h\n" - "fmla z29.h, p3/M, z7.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x22, x9, LSL #1]\n" + "ldp x24, x23, [x15, #0x0]\n" + "fmla z24.h, p3/M, z2.h, z18.h\n" + "fmla z23.h, p3/M, z1.h, z18.h\n" + "ld1h { z19.h }, p2/Z, [x20, x9, LSL #1]\n" + "ldp x22, x21, [x15, #0x10]\n" + "fmla z22.h, p3/M, z0.h, z17.h\n" + "fmla z21.h, p3/M, z2.h, z16.h\n" + "ldr x20, [x15, #0x20]\n" + "ld1h { z13.h }, p1/Z, [x20, x14, LSL #1]\n" + "fmla z24.h, p3/M, z8.h, z20.h\n" + "fmla z23.h, p3/M, z7.h, z20.h\n" + "ld1h { z18.h }, p2/Z, [x27, x9, LSL #1]\n" "inch x28\n" - "fmla z30.h, p3/M, z3.h, z9.h\n" - "fmla z31.h, p3/M, z5.h, z10.h\n" + "fmla z22.h, p3/M, z3.h, z19.h\n" + "fmla z21.h, p3/M, z5.h, z18.h\n" "mov p0.b, p2.b\n" - "ld1h { z18.h }, p3/Z, [x16]\n" - "fmla z28.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x21, x9, LSL #1]\n" - "fmla z29.h, p3/M, z5.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x20, x9, LSL #1]\n" - "fmla z30.h, p3/M, z7.h, z11.h\n" - "fmla z31.h, p3/M, z6.h, z11.h\n" + "ld1h { z20.h }, p3/Z, [x16]\n" + "fmla z24.h, p3/M, z3.h, z17.h\n" + "ld1h { z17.h }, p2/Z, [x26, x9, LSL #1]\n" + "fmla z23.h, p3/M, z5.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x25, x9, LSL #1]\n" + "fmla z22.h, p3/M, z7.h, z17.h\n" + "fmla z21.h, p3/M, z6.h, z17.h\n" "inch x9\n" - "ld1h { z11.h }, p1/Z, [x25, x14, LSL #1]\n" - "fmla z28.h, p3/M, z6.h, z9.h\n" - "fmla z29.h, p3/M, z8.h, z10.h\n" - "ld1h { z9.h }, p1/Z, [x27, x14, LSL #1]\n" - "ld1h { z10.h }, p1/Z, [x26, x14, LSL #1]\n" - "fmla z30.h, p3/M, z8.h, z12.h\n" - "fmla z31.h, p3/M, z7.h, z12.h\n" - "ld1h { z12.h }, p1/Z, [x24, x14, LSL #1]\n" + "ld1h { z11.h }, p1/Z, [x22, x14, LSL #1]\n" + "fmla z24.h, p3/M, z6.h, z19.h\n" + "fmla z23.h, p3/M, z8.h, z18.h\n" + "ld1h { z9.h }, p1/Z, [x24, x14, LSL #1]\n" + "ld1h { z10.h }, p1/Z, [x23, x14, LSL #1]\n" + "fmla z22.h, p3/M, z8.h, z16.h\n" + "fmla z21.h, p3/M, z7.h, z16.h\n" + "ld1h { z12.h }, p1/Z, [x21, x14, LSL #1]\n" "inch x14\n" - "fmax z28.h, p3/M, z28.h, z17.h\n" - "fmax z29.h, p3/M, z29.h, z17.h\n" + "fmax z24.h, p3/M, z24.h, z26.h\n" + "fmax z23.h, p3/M, z23.h, z26.h\n" "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n" "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n" - "fmax z30.h, p3/M, z30.h, z17.h\n" - "fmax z31.h, p3/M, z31.h, z17.h\n" + "fmax z22.h, p3/M, z22.h, z26.h\n" + "fmax z21.h, p3/M, z21.h, z26.h\n" "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n" "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n" "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n" @@ -199,98 +199,98 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( "cmp x14, %x[n_channels]\n" "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n" "addvl x16, x16, #16\n" - "fmin z28.h, p3/M, z28.h, z16.h\n" - "st1h { z28.h }, p0, [x13, x28, LSL #1]\n" - "fmin z29.h, p3/M, z29.h, z16.h\n" - "fmin z30.h, p3/M, z30.h, z16.h\n" - "st1h { z29.h }, p0, [x12, x28, LSL #1]\n" + "fmin z24.h, p3/M, z24.h, z25.h\n" + "st1h { z24.h }, p0, [x13, x28, LSL #1]\n" + "fmin z23.h, p3/M, z23.h, z25.h\n" + "fmin z22.h, p3/M, z22.h, z25.h\n" + "st1h { z23.h }, p0, [x12, x28, LSL #1]\n" "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n" - "fmin z31.h, p3/M, z31.h, z16.h\n" - "st1h { z30.h }, p0, [x11, x28, LSL #1]\n" + "fmin z21.h, p3/M, z21.h, z25.h\n" + "st1h { z22.h }, p0, [x11, x28, LSL #1]\n" "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n" "addvl x16, x16, #-6\n" - "st1h { z31.h }, p0, [x10, x28, LSL #1]\n" + "st1h { z21.h }, p0, [x10, x28, LSL #1]\n" "blt 1b\n" "2:" // Channel tail - "movprfx z28, z18\n fmla z28.h, p3/M, z4.h, z9.h\n" - "movprfx z29, z18\n fmla z29.h, p3/M, z3.h, z9.h\n" - "ldr x22, [x15, #0x28]\n" - "ldr x21, [x15, #0x30]\n" - "movprfx z30, z18\n fmla z30.h, p3/M, z1.h, z9.h\n" - "movprfx z31, z18\n fmla z31.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x22, x9, LSL #1]\n" - "ldr x20, [x15, #0x38]\n" - "fmla z28.h, p3/M, z0.h, z10.h\n" - "fmla z29.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x21, x9, LSL #1]\n" - "ldr x26, [x15, #0x48]\n" - "fmla z30.h, p3/M, z2.h, z12.h\n" - "fmla z31.h, p3/M, z1.h, z12.h\n" - "ldr x27, [x15, #0x40]\n" - "ld1h { z10.h }, p2/Z, [x26, x9, LSL #1]\n" - "fmla z28.h, p3/M, z5.h, z12.h\n" - "fmla z29.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x20, x9, LSL #1]\n" - "ldr x25, [x15, #0x50]\n" - "fmla z30.h, p3/M, z6.h, z9.h\n" - "fmla z31.h, p3/M, z3.h, z13.h\n" - "ld1h { z9.h }, p2/Z, [x27, x9, LSL #1]\n" - "ldr x24, [x15, #0x58]\n" - "fmla z28.h, p3/M, z7.h, z13.h\n" - "fmla z29.h, p3/M, z6.h, z13.h\n" + "movprfx z24, z20\n fmla z24.h, p3/M, z4.h, z9.h\n" + "movprfx z23, z20\n fmla z23.h, p3/M, z3.h, z9.h\n" + "ldr x21, [x15, #0x28]\n" + "ldr x20, [x15, #0x30]\n" + "movprfx z22, z20\n fmla z22.h, p3/M, z1.h, z9.h\n" + "movprfx z21, z20\n fmla z21.h, p3/M, z0.h, z9.h\n" + "ld1h { z18.h }, p2/Z, [x21, x9, LSL #1]\n" + "ldr x22, [x15, #0x38]\n" + "fmla z24.h, p3/M, z0.h, z10.h\n" + "fmla z23.h, p3/M, z2.h, z11.h\n" + "ld1h { z17.h }, p2/Z, [x20, x9, LSL #1]\n" + "ldr x21, [x15, #0x48]\n" + "fmla z22.h, p3/M, z2.h, z12.h\n" + "fmla z21.h, p3/M, z1.h, z12.h\n" + "ldr x20, [x15, #0x40]\n" + "ld1h { z20.h }, p2/Z, [x21, x9, LSL #1]\n" + "fmla z24.h, p3/M, z5.h, z12.h\n" + "fmla z23.h, p3/M, z4.h, z12.h\n" + "ld1h { z16.h }, p2/Z, [x22, x9, LSL #1]\n" + "ldr x21, [x15, #0x50]\n" + "fmla z22.h, p3/M, z6.h, z18.h\n" + "fmla z21.h, p3/M, z3.h, z13.h\n" + "ld1h { z18.h }, p2/Z, [x20, x9, LSL #1]\n" + "ldr x20, [x15, #0x58]\n" + "fmla z24.h, p3/M, z7.h, z13.h\n" + "fmla z23.h, p3/M, z6.h, z13.h\n" "ldr x23, [x15, #0x60]\n" "ldr x22, [x15, #0x68]\n" - "fmla z30.h, p3/M, z4.h, z13.h\n" - "fmla z31.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x25, x9, LSL #1]\n" + "fmla z22.h, p3/M, z4.h, z13.h\n" + "fmla z21.h, p3/M, z8.h, z17.h\n" + "ld1h { z17.h }, p2/Z, [x21, x9, LSL #1]\n" "ldr x21, [x15, #0x70]\n" - "fmla z28.h, p3/M, z1.h, z12.h\n" - "fmla z29.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x24, x9, LSL #1]\n" + "fmla z24.h, p3/M, z1.h, z16.h\n" + "fmla z23.h, p3/M, z0.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x20, x9, LSL #1]\n" "ldr x20, [x15, #0x78]\n" - "fmla z30.h, p3/M, z5.h, z10.h\n" - "fmla z31.h, p3/M, z4.h, z10.h\n" + "fmla z22.h, p3/M, z5.h, z20.h\n" + "fmla z21.h, p3/M, z4.h, z20.h\n" "inch x28\n" "mov p0.b, p2.b\n" - "fmla z28.h, p3/M, z2.h, z9.h\n" - "fmla z29.h, p3/M, z1.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x23, x9, LSL #1]\n" - "fmla z30.h, p3/M, z0.h, z11.h\n" - "fmla z31.h, p3/M, z2.h, z12.h\n" - "fmla z28.h, p3/M, z8.h, z10.h\n" - "fmla z29.h, p3/M, z7.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x22, x9, LSL #1]\n" - "fmla z30.h, p3/M, z3.h, z9.h\n" - "fmla z31.h, p3/M, z5.h, z10.h\n" - "fmla z28.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x21, x9, LSL #1]\n" - "fmla z29.h, p3/M, z5.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x20, x9, LSL #1]\n" - "fmla z30.h, p3/M, z7.h, z11.h\n" - "fmla z31.h, p3/M, z6.h, z11.h\n" - "fmla z28.h, p3/M, z6.h, z9.h\n" - "fmla z29.h, p3/M, z8.h, z10.h\n" - "fmax z28.h, p3/M, z28.h, z17.h\n" - "fmax z29.h, p3/M, z29.h, z17.h\n" - "fmla z30.h, p3/M, z8.h, z12.h\n" - "fmla z31.h, p3/M, z7.h, z12.h\n" - "fmax z30.h, p3/M, z30.h, z17.h\n" - "fmax z31.h, p3/M, z31.h, z17.h\n" - "fmin z28.h, p3/M, z28.h, z16.h\n" - "fmin z29.h, p3/M, z29.h, z16.h\n" - "st1h { z28.h }, p0, [x13, x28, LSL #1]\n" - "fmin z30.h, p3/M, z30.h, z16.h\n" - "fmin z31.h, p3/M, z31.h, z16.h\n" - "st1h { z29.h }, p0, [x12, x28, LSL #1]\n" - "st1h { z30.h }, p0, [x11, x28, LSL #1]\n" - "st1h { z31.h }, p0, [x10, x28, LSL #1]\n" + "fmla z24.h, p3/M, z2.h, z18.h\n" + "fmla z23.h, p3/M, z1.h, z18.h\n" + "ld1h { z19.h }, p2/Z, [x23, x9, LSL #1]\n" + "fmla z22.h, p3/M, z0.h, z17.h\n" + "fmla z21.h, p3/M, z2.h, z16.h\n" + "fmla z24.h, p3/M, z8.h, z20.h\n" + "fmla z23.h, p3/M, z7.h, z20.h\n" + "ld1h { z18.h }, p2/Z, [x22, x9, LSL #1]\n" + "fmla z22.h, p3/M, z3.h, z19.h\n" + "fmla z21.h, p3/M, z5.h, z18.h\n" + "fmla z24.h, p3/M, z3.h, z17.h\n" + "ld1h { z17.h }, p2/Z, [x21, x9, LSL #1]\n" + "fmla z23.h, p3/M, z5.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x20, x9, LSL #1]\n" + "fmla z22.h, p3/M, z7.h, z17.h\n" + "fmla z21.h, p3/M, z6.h, z17.h\n" + "fmla z24.h, p3/M, z6.h, z19.h\n" + "fmla z23.h, p3/M, z8.h, z18.h\n" + "fmax z24.h, p3/M, z24.h, z26.h\n" + "fmax z23.h, p3/M, z23.h, z26.h\n" + "fmla z22.h, p3/M, z8.h, z16.h\n" + "fmla z21.h, p3/M, z7.h, z16.h\n" + "fmax z22.h, p3/M, z22.h, z26.h\n" + "fmax z21.h, p3/M, z21.h, z26.h\n" + "fmin z24.h, p3/M, z24.h, z25.h\n" + "fmin z23.h, p3/M, z23.h, z25.h\n" + "st1h { z24.h }, p0, [x13, x28, LSL #1]\n" + "fmin z22.h, p3/M, z22.h, z25.h\n" + "fmin z21.h, p3/M, z21.h, z25.h\n" + "st1h { z23.h }, p0, [x12, x28, LSL #1]\n" + "st1h { z22.h }, p0, [x11, x28, LSL #1]\n" + "st1h { z21.h }, p0, [x10, x28, LSL #1]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp index d49b14eeaf2e4f2d7e37cc9fed6b8aa7e4c2df60..da2ef72a30de65ca38d91c84dbf46f3a4cb40fb0 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,19 +22,19 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include #pragma once -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { -void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); -void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); +void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(const __fp16 *const *const input_ptrs, __fp16 *const *const outptrs, const void *params, unsigned int n_channels, const __fp16 activation_min, const __fp16 activation_max); +void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(const unsigned int n_tile_rows, const unsigned int n_tile_cols, const __fp16 *inptr, int64_t ld_input_row, int64_t ld_input_col, __fp16 *outptr, int64_t ld_output_row, int64_t ld_output_col, const void *params, unsigned int n_channels, const __fp16 activation_min, const __fp16 activation_max); class sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst : public DepthwiseDepthfirstStrategy<__fp16, __fp16, __fp16, __fp16> { @@ -57,7 +57,7 @@ class sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst : public DepthwiseDepthfirst constexpr static unsigned int output_cols = 3; sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst(const CPUInfo *) - : DepthwiseDepthfirstStrategy<__fp16, __fp16, __fp16, __fp16>(3, 3, 1) {} + : Parent(output_rows, output_cols, kernel_rows, kernel_cols, stride_rows, stride_cols) {} arm_gemm::VLType get_vl_type(void) const override { return vl_type; } @@ -68,4 +68,4 @@ class sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst : public DepthwiseDepthfirst } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp index d2dae8408970d1937a5e5a122030ce0d629e4bc9..a22ab39d6f2f8172e13406992fb5dfa8cb5e81dd 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -113,7 +113,7 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "madd x20, x8, x16, x20\n" // offset += tile_j * ld_output_col "add x9, x10, x23, LSL #1\n" "whilelt p2.h, XZR, %x[n_channels]\n" - "ld1h { z18.h }, p3/Z, [x13]\n" + "ld1h { z14.h }, p3/Z, [x13]\n" "mul x20, x20, x24\n" // offset *= output_tile_size "ld1h { z0.h }, p3/Z, [x13, #1, MUL VL]\n" "ld1h { z1.h }, p3/Z, [x13, #2, MUL VL]\n" @@ -129,10 +129,10 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "ld1h { z6.h }, p3/Z, [x13, #7, MUL VL]\n" "addvl x13, x13, #16\n" "add x24, x11, x21, LSL #1\n" - "ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rh { z31.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" "cmp x15, %x[n_channels]\n" "add x23, x24, x21, LSL #1\n" - "ld1rh { z16.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "ld1rh { z30.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "ld1h { z7.h }, p3/Z, [x13, #-8, MUL VL]\n" "add x22, x16, x16\n" "mov x21, #0x0\n" @@ -146,131 +146,131 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "ld1h { z13.h }, p2/Z, [x10, x12, LSL #1]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "movprfx z24, z18\n fmla z24.h, p3/M, z7.h, z9.h\n" - "movprfx z23, z18\n fmla z23.h, p3/M, z8.h, z9.h\n" + "movprfx z29, z14\n fmla z29.h, p3/M, z7.h, z9.h\n" + "movprfx z28, z14\n fmla z28.h, p3/M, z8.h, z9.h\n" "whilelt p1.h, x15, %x[n_channels]\n" "inch x21\n" - "movprfx z25, z18\n fmla z25.h, p3/M, z6.h, z9.h\n" - "fmla z24.h, p3/M, z4.h, z13.h\n" + "movprfx z27, z14\n fmla z27.h, p3/M, z6.h, z9.h\n" + "fmla z29.h, p3/M, z4.h, z13.h\n" "inch x15\n" "mov p0.b, p2.b\n" - "movprfx z26, z18\n fmla z26.h, p3/M, z5.h, z9.h\n" - "movprfx z27, z18\n fmla z27.h, p3/M, z4.h, z9.h\n" + "movprfx z26, z14\n fmla z26.h, p3/M, z5.h, z9.h\n" + "movprfx z25, z14\n fmla z25.h, p3/M, z4.h, z9.h\n" "inch x20\n" - "movprfx z28, z18\n fmla z28.h, p3/M, z3.h, z9.h\n" - "fmla z23.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x9, x27, LSL #1]\n" - "fmla z25.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x9, x17, LSL #1]\n" - "movprfx z29, z18\n fmla z29.h, p3/M, z2.h, z9.h\n" - "fmla z24.h, p3/M, z6.h, z11.h\n" - "movprfx z31, z18\n fmla z31.h, p3/M, z0.h, z9.h\n" - "fmla z23.h, p3/M, z5.h, z13.h\n" - "fmla z25.h, p3/M, z3.h, z13.h\n" + "movprfx z24, z14\n fmla z24.h, p3/M, z3.h, z9.h\n" + "fmla z28.h, p3/M, z0.h, z10.h\n" + "ld1h { z23.h }, p2/Z, [x9, x27, LSL #1]\n" + "fmla z27.h, p3/M, z2.h, z11.h\n" + "ld1h { z18.h }, p2/Z, [x9, x17, LSL #1]\n" + "movprfx z22, z14\n fmla z22.h, p3/M, z2.h, z9.h\n" + "fmla z29.h, p3/M, z6.h, z18.h\n" + "movprfx z21, z14\n fmla z21.h, p3/M, z0.h, z9.h\n" + "fmla z28.h, p3/M, z5.h, z13.h\n" + "fmla z27.h, p3/M, z3.h, z13.h\n" "fmla z26.h, p3/M, z2.h, z13.h\n" - "fmla z27.h, p3/M, z1.h, z13.h\n" - "fmla z28.h, p3/M, z0.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x14, x17, LSL #1]\n" - "fmla z29.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x26, x25, LSL #1]\n" - "movprfx z30, z18\n fmla z30.h, p3/M, z1.h, z9.h\n" + "fmla z25.h, p3/M, z1.h, z13.h\n" "fmla z24.h, p3/M, z0.h, z13.h\n" - "ld1h { z18.h }, p3/Z, [x13]\n" - "fmla z31.h, p3/M, z8.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x14, x27, LSL #1]\n" - "fmla z23.h, p3/M, z7.h, z11.h\n" - "fmla z30.h, p3/M, z0.h, z11.h\n" - "fmla z26.h, p3/M, z4.h, z11.h\n" - "fmla z27.h, p3/M, z3.h, z11.h\n" - "fmla z29.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10]\n" - "fmla z24.h, p3/M, z2.h, z12.h\n" - "fmla z25.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x28]\n" - "fmla z28.h, p3/M, z4.h, z10.h\n" - "fmla z23.h, p3/M, z1.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x10, x25, LSL #1]\n" - "fmla z30.h, p3/M, z2.h, z10.h\n" - "fmla z31.h, p3/M, z1.h, z10.h\n" - "fmla z24.h, p3/M, z8.h, z10.h\n" - "fmla z25.h, p3/M, z7.h, z10.h\n" - "fmla z27.h, p3/M, z5.h, z10.h\n" - "fmla z26.h, p3/M, z0.h, z11.h\n" - "ld1h { z10.h }, p2/Z, [x28, x12, LSL #1]\n" - "fmla z29.h, p3/M, z3.h, z12.h\n" - "fmla z28.h, p3/M, z2.h, z13.h\n" - "fmla z30.h, p3/M, z4.h, z10.h\n" - "fmla z31.h, p3/M, z3.h, z10.h\n" - "fmla z23.h, p3/M, z3.h, z11.h\n" - "fmla z25.h, p3/M, z5.h, z13.h\n" - "ld1h { z11.h }, p2/Z, [x28, x25, LSL #1]\n" - "ld1h { z13.h }, p2/Z, [x26, x17, LSL #1]\n" - "fmla z26.h, p3/M, z6.h, z12.h\n" - "fmla z27.h, p3/M, z7.h, z10.h\n" - "ld1h { z12.h }, p2/Z, [x10, x17, LSL #1]\n" - "fmla z29.h, p3/M, z5.h, z10.h\n" - "fmla z28.h, p3/M, z6.h, z10.h\n" - "fmla z31.h, p3/M, z5.h, z11.h\n" - "fmla z30.h, p3/M, z6.h, z13.h\n" - "fmla z26.h, p3/M, z8.h, z10.h\n" - "fmla z29.h, p3/M, z7.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x26, x27, LSL #1]\n" - "fmla z24.h, p3/M, z3.h, z12.h\n" - "fmla z27.h, p3/M, z0.h, z12.h\n" - "fmla z28.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x27, LSL #1]\n" - "fmla z30.h, p3/M, z8.h, z13.h\n" + "ld1h { z17.h }, p2/Z, [x14, x17, LSL #1]\n" + "fmla z22.h, p3/M, z6.h, z12.h\n" + "ld1h { z16.h }, p2/Z, [x26, x25, LSL #1]\n" + "movprfx z20, z14\n fmla z20.h, p3/M, z1.h, z9.h\n" + "fmla z29.h, p3/M, z0.h, z17.h\n" + "ld1h { z14.h }, p3/Z, [x13]\n" + "fmla z21.h, p3/M, z8.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x14, x27, LSL #1]\n" + "fmla z28.h, p3/M, z7.h, z18.h\n" + "fmla z20.h, p3/M, z0.h, z18.h\n" + "fmla z26.h, p3/M, z4.h, z18.h\n" + "fmla z25.h, p3/M, z3.h, z18.h\n" + "fmla z22.h, p3/M, z1.h, z18.h\n" + "ld1h { z19.h }, p2/Z, [x10]\n" + "fmla z29.h, p3/M, z2.h, z16.h\n" + "fmla z27.h, p3/M, z1.h, z16.h\n" + "ld1h { z18.h }, p2/Z, [x28]\n" + "fmla z24.h, p3/M, z4.h, z23.h\n" + "fmla z28.h, p3/M, z1.h, z17.h\n" + "ld1h { z16.h }, p2/Z, [x10, x25, LSL #1]\n" + "fmla z20.h, p3/M, z2.h, z23.h\n" + "fmla z21.h, p3/M, z1.h, z23.h\n" + "fmla z29.h, p3/M, z8.h, z23.h\n" + "fmla z27.h, p3/M, z7.h, z23.h\n" + "fmla z25.h, p3/M, z5.h, z23.h\n" + "fmla z26.h, p3/M, z0.h, z19.h\n" + "ld1h { z17.h }, p2/Z, [x28, x12, LSL #1]\n" + "fmla z22.h, p3/M, z3.h, z18.h\n" + "fmla z24.h, p3/M, z2.h, z16.h\n" + "fmla z20.h, p3/M, z4.h, z17.h\n" + "fmla z21.h, p3/M, z3.h, z17.h\n" + "fmla z28.h, p3/M, z3.h, z19.h\n" + "fmla z27.h, p3/M, z5.h, z16.h\n" + "ld1h { z19.h }, p2/Z, [x28, x25, LSL #1]\n" + "ld1h { z16.h }, p2/Z, [x26, x17, LSL #1]\n" + "fmla z26.h, p3/M, z6.h, z18.h\n" + "fmla z25.h, p3/M, z7.h, z17.h\n" + "ld1h { z18.h }, p2/Z, [x10, x17, LSL #1]\n" + "fmla z22.h, p3/M, z5.h, z17.h\n" + "fmla z24.h, p3/M, z6.h, z17.h\n" + "fmla z21.h, p3/M, z5.h, z19.h\n" + "fmla z20.h, p3/M, z6.h, z16.h\n" + "fmla z26.h, p3/M, z8.h, z17.h\n" + "fmla z22.h, p3/M, z7.h, z16.h\n" + "ld1h { z17.h }, p2/Z, [x26, x27, LSL #1]\n" + "fmla z29.h, p3/M, z3.h, z18.h\n" + "fmla z25.h, p3/M, z0.h, z18.h\n" + "fmla z24.h, p3/M, z8.h, z19.h\n" + "ld1h { z16.h }, p2/Z, [x10, x27, LSL #1]\n" + "fmla z20.h, p3/M, z8.h, z17.h\n" "addvl x10, x10, #1\n" - "fmla z31.h, p3/M, z7.h, z13.h\n" - "fmla z23.h, p3/M, z4.h, z12.h\n" - "ld1h { z13.h }, p2/Z, [x28, x27, LSL #1]\n" - "fmla z26.h, p3/M, z1.h, z12.h\n" - "fmla z24.h, p3/M, z5.h, z11.h\n" - "ld1h { z12.h }, p2/Z, [x28, x17, LSL #1]\n" + "fmla z21.h, p3/M, z7.h, z17.h\n" + "fmla z28.h, p3/M, z4.h, z18.h\n" + "ld1h { z19.h }, p2/Z, [x28, x27, LSL #1]\n" + "fmla z26.h, p3/M, z1.h, z18.h\n" + "fmla z29.h, p3/M, z5.h, z16.h\n" + "ld1h { z17.h }, p2/Z, [x28, x17, LSL #1]\n" "addvl x28, x28, #1\n" - "fmla z25.h, p3/M, z4.h, z11.h\n" - "fmla z27.h, p3/M, z2.h, z11.h\n" - "fmla z28.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x14, x12, LSL #1]\n" - "fmla z29.h, p3/M, z4.h, z12.h\n" + "fmla z27.h, p3/M, z4.h, z16.h\n" + "fmla z25.h, p3/M, z2.h, z16.h\n" + "fmla z24.h, p3/M, z1.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x14, x12, LSL #1]\n" + "fmla z22.h, p3/M, z4.h, z17.h\n" "addvl x14, x14, #1\n" - "fmla z30.h, p3/M, z3.h, z12.h\n" - "fmla z31.h, p3/M, z4.h, z13.h\n" + "fmla z20.h, p3/M, z3.h, z17.h\n" + "fmla z21.h, p3/M, z4.h, z19.h\n" "ld1h { z4.h }, p3/Z, [x13, #5, MUL VL]\n" "ld1h { z10.h }, p1/Z, [x14]\n" - "fmla z26.h, p3/M, z7.h, z12.h\n" - "fmla z27.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9]\n" - "fmla z23.h, p3/M, z2.h, z11.h\n" - "fmla z24.h, p3/M, z1.h, z11.h\n" - "fmax z24.h, p3/M, z24.h, z17.h\n" + "fmla z26.h, p3/M, z7.h, z17.h\n" + "fmla z25.h, p3/M, z6.h, z17.h\n" + "ld1h { z18.h }, p2/Z, [x9]\n" + "fmla z28.h, p3/M, z2.h, z16.h\n" + "fmla z29.h, p3/M, z1.h, z16.h\n" + "fmax z29.h, p3/M, z29.h, z31.h\n" "ld1h { z1.h }, p3/Z, [x13, #2, MUL VL]\n" - "fmla z25.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" - "fmla z28.h, p3/M, z7.h, z13.h\n" + "fmla z27.h, p3/M, z0.h, z16.h\n" + "ld1h { z17.h }, p2/Z, [x9, x25, LSL #1]\n" + "fmla z24.h, p3/M, z7.h, z19.h\n" "addvl x9, x9, #1\n" - "fmla z30.h, p3/M, z5.h, z13.h\n" - "fmla z29.h, p3/M, z0.h, z12.h\n" + "fmla z20.h, p3/M, z5.h, z19.h\n" + "fmla z22.h, p3/M, z0.h, z18.h\n" "ld1h { z0.h }, p3/Z, [x13, #1, MUL VL]\n" - "fmin z24.h, p3/M, z24.h, z16.h\n" - "fmla z31.h, p3/M, z2.h, z11.h\n" - "fmla z27.h, p3/M, z8.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x26, x12, LSL #1]\n" - "fmax z27.h, p3/M, z27.h, z17.h\n" - "fmla z23.h, p3/M, z6.h, z12.h\n" - "fmla z26.h, p3/M, z3.h, z12.h\n" - "fmax z23.h, p3/M, z23.h, z17.h\n" - "fmax z26.h, p3/M, z26.h, z17.h\n" - "fmla z25.h, p3/M, z8.h, z11.h\n" - "fmla z28.h, p3/M, z5.h, z11.h\n" - "fmax z25.h, p3/M, z25.h, z17.h\n" - "fmax z28.h, p3/M, z28.h, z17.h\n" - "fmla z29.h, p3/M, z8.h, z13.h\n" - "fmla z30.h, p3/M, z7.h, z13.h\n" - "fmax z29.h, p3/M, z29.h, z17.h\n" - "fmax z30.h, p3/M, z30.h, z17.h\n" - "fmla z31.h, p3/M, z6.h, z13.h\n" - "fmax z31.h, p3/M, z31.h, z17.h\n" + "fmin z29.h, p3/M, z29.h, z30.h\n" + "fmla z21.h, p3/M, z2.h, z17.h\n" + "fmla z25.h, p3/M, z8.h, z19.h\n" + "ld1h { z16.h }, p2/Z, [x26, x12, LSL #1]\n" + "fmax z25.h, p3/M, z25.h, z31.h\n" + "fmla z28.h, p3/M, z6.h, z18.h\n" + "fmla z26.h, p3/M, z3.h, z18.h\n" + "fmax z28.h, p3/M, z28.h, z31.h\n" + "fmax z26.h, p3/M, z26.h, z31.h\n" + "fmla z27.h, p3/M, z8.h, z17.h\n" + "fmla z24.h, p3/M, z5.h, z17.h\n" + "fmax z27.h, p3/M, z27.h, z31.h\n" + "fmax z24.h, p3/M, z24.h, z31.h\n" + "fmla z22.h, p3/M, z8.h, z16.h\n" + "fmla z20.h, p3/M, z7.h, z16.h\n" + "fmax z22.h, p3/M, z22.h, z31.h\n" + "fmax z20.h, p3/M, z20.h, z31.h\n" + "fmla z21.h, p3/M, z6.h, z16.h\n" + "fmax z21.h, p3/M, z21.h, z31.h\n" "addvl x26, x26, #1\n" "ld1h { z2.h }, p3/Z, [x13, #3, MUL VL]\n" "ld1h { z3.h }, p3/Z, [x13, #4, MUL VL]\n" @@ -279,182 +279,182 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "cmp x15, %x[n_channels]\n" "ld1h { z6.h }, p3/Z, [x13, #7, MUL VL]\n" "addvl x13, x13, #16\n" - "fmin z23.h, p3/M, z23.h, z16.h\n" + "fmin z28.h, p3/M, z28.h, z30.h\n" "ld1h { z9.h }, p1/Z, [x9, x12, LSL #1]\n" - "fmin z25.h, p3/M, z25.h, z16.h\n" - "fmin z26.h, p3/M, z26.h, z16.h\n" + "fmin z27.h, p3/M, z27.h, z30.h\n" + "fmin z26.h, p3/M, z26.h, z30.h\n" "ld1h { z11.h }, p1/Z, [x14, x25, LSL #1]\n" "ld1h { z12.h }, p1/Z, [x26]\n" - "fmin z27.h, p3/M, z27.h, z16.h\n" - "fmin z28.h, p3/M, z28.h, z16.h\n" + "fmin z25.h, p3/M, z25.h, z30.h\n" + "fmin z24.h, p3/M, z24.h, z30.h\n" "ld1h { z13.h }, p1/Z, [x10, x12, LSL #1]\n" - "st1h { z23.h }, p0, [x11]\n" - "fmin z29.h, p3/M, z29.h, z16.h\n" - "fmin z30.h, p3/M, z30.h, z16.h\n" - "st1h { z24.h }, p0, [x11, x16, LSL #1]\n" + "st1h { z28.h }, p0, [x11]\n" + "fmin z22.h, p3/M, z22.h, z30.h\n" + "fmin z20.h, p3/M, z20.h, z30.h\n" + "st1h { z29.h }, p0, [x11, x16, LSL #1]\n" "ld1h { z7.h }, p3/Z, [x13, #-8, MUL VL]\n" - "fmin z31.h, p3/M, z31.h, z16.h\n" - "st1h { z25.h }, p0, [x11, x22, LSL #1]\n" + "fmin z21.h, p3/M, z21.h, z30.h\n" + "st1h { z27.h }, p0, [x11, x22, LSL #1]\n" "addvl x11, x11, #1\n" "ld1h { z8.h }, p3/Z, [x13, #-7, MUL VL]\n" "st1h { z26.h }, p0, [x24]\n" "addvl x13, x13, #-6\n" - "st1h { z27.h }, p0, [x24, x16, LSL #1]\n" - "st1h { z28.h }, p0, [x24, x22, LSL #1]\n" + "st1h { z25.h }, p0, [x24, x16, LSL #1]\n" + "st1h { z24.h }, p0, [x24, x22, LSL #1]\n" "addvl x24, x24, #1\n" - "st1h { z29.h }, p0, [x23]\n" - "st1h { z30.h }, p0, [x23, x16, LSL #1]\n" - "st1h { z31.h }, p0, [x23, x22, LSL #1]\n" + "st1h { z22.h }, p0, [x23]\n" + "st1h { z20.h }, p0, [x23, x16, LSL #1]\n" + "st1h { z21.h }, p0, [x23, x22, LSL #1]\n" "addvl x23, x23, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "movprfx z24, z18\n fmla z24.h, p3/M, z7.h, z9.h\n" - "movprfx z23, z18\n fmla z23.h, p3/M, z8.h, z9.h\n" + "movprfx z29, z14\n fmla z29.h, p3/M, z7.h, z9.h\n" + "movprfx z28, z14\n fmla z28.h, p3/M, z8.h, z9.h\n" "ldr x8, [%x[params_struct], %[offsetof_args_tile_j]]\n" "ldr x13, [%x[params_struct], %[offsetof_args_tile_i]]\n" - "movprfx z25, z18\n fmla z25.h, p3/M, z6.h, z9.h\n" - "fmla z24.h, p3/M, z4.h, z13.h\n" + "movprfx z27, z14\n fmla z27.h, p3/M, z6.h, z9.h\n" + "fmla z29.h, p3/M, z4.h, z13.h\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" "add x8, x8, #0x1\n" - "movprfx z26, z18\n fmla z26.h, p3/M, z5.h, z9.h\n" - "movprfx z27, z18\n fmla z27.h, p3/M, z4.h, z9.h\n" + "movprfx z26, z14\n fmla z26.h, p3/M, z5.h, z9.h\n" + "movprfx z25, z14\n fmla z25.h, p3/M, z4.h, z9.h\n" "cmp x8, x20\n" "add x21, x13, #0x1\n" - "movprfx z28, z18\n fmla z28.h, p3/M, z3.h, z9.h\n" - "fmla z23.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x9, x27, LSL #1]\n" + "movprfx z24, z14\n fmla z24.h, p3/M, z3.h, z9.h\n" + "fmla z28.h, p3/M, z0.h, z10.h\n" + "ld1h { z23.h }, p2/Z, [x9, x27, LSL #1]\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" - "fmla z25.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x9, x17, LSL #1]\n" - "movprfx z29, z18\n fmla z29.h, p3/M, z2.h, z9.h\n" + "fmla z27.h, p3/M, z2.h, z11.h\n" + "ld1h { z18.h }, p2/Z, [x9, x17, LSL #1]\n" + "movprfx z22, z14\n fmla z22.h, p3/M, z2.h, z9.h\n" "csel x13, x13, x21, LT\n" - "fmla z24.h, p3/M, z6.h, z11.h\n" - "movprfx z31, z18\n fmla z31.h, p3/M, z0.h, z9.h\n" + "fmla z29.h, p3/M, z6.h, z18.h\n" + "movprfx z21, z14\n fmla z21.h, p3/M, z0.h, z9.h\n" "mov p0.b, p2.b\n" "csel x8, x8, XZR, LT\n" - "fmla z23.h, p3/M, z5.h, z13.h\n" - "fmla z25.h, p3/M, z3.h, z13.h\n" + "fmla z28.h, p3/M, z5.h, z13.h\n" + "fmla z27.h, p3/M, z3.h, z13.h\n" "cmp x13, x20\n" "fmla z26.h, p3/M, z2.h, z13.h\n" - "fmla z27.h, p3/M, z1.h, z13.h\n" - "fmla z28.h, p3/M, z0.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x14, x17, LSL #1]\n" - "fmla z29.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x26, x25, LSL #1]\n" - "movprfx z30, z18\n fmla z30.h, p3/M, z1.h, z9.h\n" + "fmla z25.h, p3/M, z1.h, z13.h\n" "fmla z24.h, p3/M, z0.h, z13.h\n" - "fmla z31.h, p3/M, z8.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x14, x27, LSL #1]\n" - "fmla z23.h, p3/M, z7.h, z11.h\n" - "fmla z30.h, p3/M, z0.h, z11.h\n" - "fmla z26.h, p3/M, z4.h, z11.h\n" - "fmla z27.h, p3/M, z3.h, z11.h\n" - "fmla z29.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10]\n" - "fmla z24.h, p3/M, z2.h, z12.h\n" - "fmla z25.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x28]\n" - "fmla z28.h, p3/M, z4.h, z10.h\n" - "fmla z23.h, p3/M, z1.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x10, x25, LSL #1]\n" - "fmla z30.h, p3/M, z2.h, z10.h\n" - "fmla z31.h, p3/M, z1.h, z10.h\n" - "fmla z24.h, p3/M, z8.h, z10.h\n" - "fmla z25.h, p3/M, z7.h, z10.h\n" - "fmla z27.h, p3/M, z5.h, z10.h\n" - "fmla z26.h, p3/M, z0.h, z11.h\n" - "ld1h { z10.h }, p2/Z, [x28, x12, LSL #1]\n" - "fmla z29.h, p3/M, z3.h, z12.h\n" - "fmla z28.h, p3/M, z2.h, z13.h\n" - "fmla z30.h, p3/M, z4.h, z10.h\n" - "fmla z31.h, p3/M, z3.h, z10.h\n" - "fmla z23.h, p3/M, z3.h, z11.h\n" - "fmla z25.h, p3/M, z5.h, z13.h\n" - "ld1h { z11.h }, p2/Z, [x28, x25, LSL #1]\n" - "ld1h { z13.h }, p2/Z, [x26, x17, LSL #1]\n" - "fmla z26.h, p3/M, z6.h, z12.h\n" - "fmla z27.h, p3/M, z7.h, z10.h\n" - "ld1h { z12.h }, p2/Z, [x10, x17, LSL #1]\n" - "fmla z29.h, p3/M, z5.h, z10.h\n" - "fmla z28.h, p3/M, z6.h, z10.h\n" - "fmla z31.h, p3/M, z5.h, z11.h\n" - "fmla z30.h, p3/M, z6.h, z13.h\n" - "fmla z26.h, p3/M, z8.h, z10.h\n" - "fmla z29.h, p3/M, z7.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x26, x27, LSL #1]\n" - "fmla z24.h, p3/M, z3.h, z12.h\n" - "fmla z27.h, p3/M, z0.h, z12.h\n" - "fmla z28.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x27, LSL #1]\n" - "fmla z30.h, p3/M, z8.h, z13.h\n" - "fmla z31.h, p3/M, z7.h, z13.h\n" - "fmla z23.h, p3/M, z4.h, z12.h\n" - "ld1h { z13.h }, p2/Z, [x28, x27, LSL #1]\n" - "fmla z26.h, p3/M, z1.h, z12.h\n" - "fmla z24.h, p3/M, z5.h, z11.h\n" - "ld1h { z12.h }, p2/Z, [x28, x17, LSL #1]\n" - "fmla z25.h, p3/M, z4.h, z11.h\n" - "fmla z27.h, p3/M, z2.h, z11.h\n" - "fmla z28.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x14, x12, LSL #1]\n" - "fmla z29.h, p3/M, z4.h, z12.h\n" - "fmla z30.h, p3/M, z3.h, z12.h\n" - "fmla z31.h, p3/M, z4.h, z13.h\n" - "fmla z26.h, p3/M, z7.h, z12.h\n" - "fmla z27.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9]\n" - "fmla z23.h, p3/M, z2.h, z11.h\n" - "fmla z24.h, p3/M, z1.h, z11.h\n" - "fmax z24.h, p3/M, z24.h, z17.h\n" - "fmin z24.h, p3/M, z24.h, z16.h\n" - "fmla z25.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" - "fmla z28.h, p3/M, z7.h, z13.h\n" - "fmla z30.h, p3/M, z5.h, z13.h\n" - "fmla z29.h, p3/M, z0.h, z12.h\n" - "fmla z31.h, p3/M, z2.h, z11.h\n" - "fmla z27.h, p3/M, z8.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x26, x12, LSL #1]\n" - "fmax z27.h, p3/M, z27.h, z17.h\n" - "fmla z23.h, p3/M, z6.h, z12.h\n" - "fmla z26.h, p3/M, z3.h, z12.h\n" - "fmax z23.h, p3/M, z23.h, z17.h\n" - "fmax z26.h, p3/M, z26.h, z17.h\n" - "fmla z25.h, p3/M, z8.h, z11.h\n" - "fmla z28.h, p3/M, z5.h, z11.h\n" - "fmax z25.h, p3/M, z25.h, z17.h\n" - "fmax z28.h, p3/M, z28.h, z17.h\n" - "fmla z29.h, p3/M, z8.h, z13.h\n" - "fmla z30.h, p3/M, z7.h, z13.h\n" - "fmax z29.h, p3/M, z29.h, z17.h\n" - "fmax z30.h, p3/M, z30.h, z17.h\n" - "fmla z31.h, p3/M, z6.h, z13.h\n" - "fmax z31.h, p3/M, z31.h, z17.h\n" - "fmin z23.h, p3/M, z23.h, z16.h\n" - "st1h { z23.h }, p0, [x11]\n" - "fmin z25.h, p3/M, z25.h, z16.h\n" - "fmin z26.h, p3/M, z26.h, z16.h\n" - "st1h { z24.h }, p0, [x11, x16, LSL #1]\n" - "fmin z27.h, p3/M, z27.h, z16.h\n" - "fmin z28.h, p3/M, z28.h, z16.h\n" - "st1h { z25.h }, p0, [x11, x22, LSL #1]\n" - "fmin z29.h, p3/M, z29.h, z16.h\n" - "fmin z30.h, p3/M, z30.h, z16.h\n" + "ld1h { z17.h }, p2/Z, [x14, x17, LSL #1]\n" + "fmla z22.h, p3/M, z6.h, z12.h\n" + "ld1h { z16.h }, p2/Z, [x26, x25, LSL #1]\n" + "movprfx z20, z14\n fmla z20.h, p3/M, z1.h, z9.h\n" + "fmla z29.h, p3/M, z0.h, z17.h\n" + "fmla z21.h, p3/M, z8.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x14, x27, LSL #1]\n" + "fmla z28.h, p3/M, z7.h, z18.h\n" + "fmla z20.h, p3/M, z0.h, z18.h\n" + "fmla z26.h, p3/M, z4.h, z18.h\n" + "fmla z25.h, p3/M, z3.h, z18.h\n" + "fmla z22.h, p3/M, z1.h, z18.h\n" + "ld1h { z19.h }, p2/Z, [x10]\n" + "fmla z29.h, p3/M, z2.h, z16.h\n" + "fmla z27.h, p3/M, z1.h, z16.h\n" + "ld1h { z18.h }, p2/Z, [x28]\n" + "fmla z24.h, p3/M, z4.h, z23.h\n" + "fmla z28.h, p3/M, z1.h, z17.h\n" + "ld1h { z16.h }, p2/Z, [x10, x25, LSL #1]\n" + "fmla z20.h, p3/M, z2.h, z23.h\n" + "fmla z21.h, p3/M, z1.h, z23.h\n" + "fmla z29.h, p3/M, z8.h, z23.h\n" + "fmla z27.h, p3/M, z7.h, z23.h\n" + "fmla z25.h, p3/M, z5.h, z23.h\n" + "fmla z26.h, p3/M, z0.h, z19.h\n" + "ld1h { z17.h }, p2/Z, [x28, x12, LSL #1]\n" + "fmla z22.h, p3/M, z3.h, z18.h\n" + "fmla z24.h, p3/M, z2.h, z16.h\n" + "fmla z20.h, p3/M, z4.h, z17.h\n" + "fmla z21.h, p3/M, z3.h, z17.h\n" + "fmla z28.h, p3/M, z3.h, z19.h\n" + "fmla z27.h, p3/M, z5.h, z16.h\n" + "ld1h { z19.h }, p2/Z, [x28, x25, LSL #1]\n" + "ld1h { z16.h }, p2/Z, [x26, x17, LSL #1]\n" + "fmla z26.h, p3/M, z6.h, z18.h\n" + "fmla z25.h, p3/M, z7.h, z17.h\n" + "ld1h { z18.h }, p2/Z, [x10, x17, LSL #1]\n" + "fmla z22.h, p3/M, z5.h, z17.h\n" + "fmla z24.h, p3/M, z6.h, z17.h\n" + "fmla z21.h, p3/M, z5.h, z19.h\n" + "fmla z20.h, p3/M, z6.h, z16.h\n" + "fmla z26.h, p3/M, z8.h, z17.h\n" + "fmla z22.h, p3/M, z7.h, z16.h\n" + "ld1h { z17.h }, p2/Z, [x26, x27, LSL #1]\n" + "fmla z29.h, p3/M, z3.h, z18.h\n" + "fmla z25.h, p3/M, z0.h, z18.h\n" + "fmla z24.h, p3/M, z8.h, z19.h\n" + "ld1h { z16.h }, p2/Z, [x10, x27, LSL #1]\n" + "fmla z20.h, p3/M, z8.h, z17.h\n" + "fmla z21.h, p3/M, z7.h, z17.h\n" + "fmla z28.h, p3/M, z4.h, z18.h\n" + "ld1h { z19.h }, p2/Z, [x28, x27, LSL #1]\n" + "fmla z26.h, p3/M, z1.h, z18.h\n" + "fmla z29.h, p3/M, z5.h, z16.h\n" + "ld1h { z17.h }, p2/Z, [x28, x17, LSL #1]\n" + "fmla z27.h, p3/M, z4.h, z16.h\n" + "fmla z25.h, p3/M, z2.h, z16.h\n" + "fmla z24.h, p3/M, z1.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x14, x12, LSL #1]\n" + "fmla z22.h, p3/M, z4.h, z17.h\n" + "fmla z20.h, p3/M, z3.h, z17.h\n" + "fmla z21.h, p3/M, z4.h, z19.h\n" + "fmla z26.h, p3/M, z7.h, z17.h\n" + "fmla z25.h, p3/M, z6.h, z17.h\n" + "ld1h { z18.h }, p2/Z, [x9]\n" + "fmla z28.h, p3/M, z2.h, z16.h\n" + "fmla z29.h, p3/M, z1.h, z16.h\n" + "fmax z29.h, p3/M, z29.h, z31.h\n" + "fmin z29.h, p3/M, z29.h, z30.h\n" + "fmla z27.h, p3/M, z0.h, z16.h\n" + "ld1h { z17.h }, p2/Z, [x9, x25, LSL #1]\n" + "fmla z24.h, p3/M, z7.h, z19.h\n" + "fmla z20.h, p3/M, z5.h, z19.h\n" + "fmla z22.h, p3/M, z0.h, z18.h\n" + "fmla z21.h, p3/M, z2.h, z17.h\n" + "fmla z25.h, p3/M, z8.h, z19.h\n" + "ld1h { z16.h }, p2/Z, [x26, x12, LSL #1]\n" + "fmax z25.h, p3/M, z25.h, z31.h\n" + "fmla z28.h, p3/M, z6.h, z18.h\n" + "fmla z26.h, p3/M, z3.h, z18.h\n" + "fmax z28.h, p3/M, z28.h, z31.h\n" + "fmax z26.h, p3/M, z26.h, z31.h\n" + "fmla z27.h, p3/M, z8.h, z17.h\n" + "fmla z24.h, p3/M, z5.h, z17.h\n" + "fmax z27.h, p3/M, z27.h, z31.h\n" + "fmax z24.h, p3/M, z24.h, z31.h\n" + "fmla z22.h, p3/M, z8.h, z16.h\n" + "fmla z20.h, p3/M, z7.h, z16.h\n" + "fmax z22.h, p3/M, z22.h, z31.h\n" + "fmax z20.h, p3/M, z20.h, z31.h\n" + "fmla z21.h, p3/M, z6.h, z16.h\n" + "fmax z21.h, p3/M, z21.h, z31.h\n" + "fmin z28.h, p3/M, z28.h, z30.h\n" + "st1h { z28.h }, p0, [x11]\n" + "fmin z27.h, p3/M, z27.h, z30.h\n" + "fmin z26.h, p3/M, z26.h, z30.h\n" + "st1h { z29.h }, p0, [x11, x16, LSL #1]\n" + "fmin z25.h, p3/M, z25.h, z30.h\n" + "fmin z24.h, p3/M, z24.h, z30.h\n" + "st1h { z27.h }, p0, [x11, x22, LSL #1]\n" + "fmin z22.h, p3/M, z22.h, z30.h\n" + "fmin z20.h, p3/M, z20.h, z30.h\n" "st1h { z26.h }, p0, [x24]\n" - "fmin z31.h, p3/M, z31.h, z16.h\n" - "st1h { z27.h }, p0, [x24, x16, LSL #1]\n" - "st1h { z28.h }, p0, [x24, x22, LSL #1]\n" - "st1h { z29.h }, p0, [x23]\n" - "st1h { z30.h }, p0, [x23, x16, LSL #1]\n" - "st1h { z31.h }, p0, [x23, x22, LSL #1]\n" + "fmin z21.h, p3/M, z21.h, z30.h\n" + "st1h { z25.h }, p0, [x24, x16, LSL #1]\n" + "st1h { z24.h }, p0, [x24, x22, LSL #1]\n" + "st1h { z22.h }, p0, [x23]\n" + "st1h { z20.h }, p0, [x23, x16, LSL #1]\n" + "st1h { z21.h }, p0, [x23, x22, LSL #1]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp index 59c0e0cf0b0874a5aba453c4e0b4b4b8aeb58e81..4f8368acd5adfc6632771fdc4765f31ec13c0c79 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -88,390 +88,390 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( __asm__ __volatile__( "ptrue p3.b\n" - "ldr x17, [%x[params_struct], %[offsetof_args_params]]\n" - "add x16, %x[params_struct], %[offsetof_Args_inptrs]\n" - "ld1h { z18.h }, p3/Z, [x17]\n" - "cnth x15\n" - "mov x14, #0x0\n" - "ld1h { z0.h }, p3/Z, [x17, #1, MUL VL]\n" - "ld1h { z1.h }, p3/Z, [x17, #2, MUL VL]\n" + "ldr x8, [%x[params_struct], %[offsetof_args_params]]\n" + "add x17, %x[params_struct], %[offsetof_Args_inptrs]\n" + "ld1h { z14.h }, p3/Z, [x8]\n" + "cnth x16\n" + "mov x15, #0x0\n" + "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n" + "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n" "whilelt p2.h, XZR, %x[n_channels]\n" - "ld1h { z2.h }, p3/Z, [x17, #3, MUL VL]\n" - "ld1h { z3.h }, p3/Z, [x17, #4, MUL VL]\n" - "cmp x15, %x[n_channels]\n" - "ld1h { z4.h }, p3/Z, [x17, #5, MUL VL]\n" - "ld1h { z5.h }, p3/Z, [x17, #6, MUL VL]\n" - "sub x13, XZR, x15\n" - "ld1h { z6.h }, p3/Z, [x17, #7, MUL VL]\n" - "addvl x17, x17, #16\n" - "ldp x12, x11, [x16, #0x0]\n" - "ldp x10, x9, [x16, #0x10]\n" - "ldr x28, [x16, #0x20]\n" - "ldr x27, [%x[params_struct], %[offsetof_args_outptrs]]\n" - "ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "ld1rh { z16.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "ld1h { z7.h }, p3/Z, [x17, #-8, MUL VL]\n" - "ld1h { z8.h }, p3/Z, [x17, #-7, MUL VL]\n" - "ld1h { z9.h }, p2/Z, [x12, x14, LSL #1]\n" - "addvl x17, x17, #-6\n" - "ld1h { z10.h }, p2/Z, [x11, x14, LSL #1]\n" - "ld1h { z11.h }, p2/Z, [x10, x14, LSL #1]\n" - "ld1h { z12.h }, p2/Z, [x9, x14, LSL #1]\n" - "ld1h { z13.h }, p2/Z, [x28, x14, LSL #1]\n" + "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n" + "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n" + "cmp x16, %x[n_channels]\n" + "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n" + "ld1h { z5.h }, p3/Z, [x8, #6, MUL VL]\n" + "sub x14, XZR, x16\n" + "ld1h { z6.h }, p3/Z, [x8, #7, MUL VL]\n" + "addvl x8, x8, #16\n" + "ldp x24, x23, [x17, #0x0]\n" + "ldp x22, x21, [x17, #0x10]\n" + "ldr x20, [x17, #0x20]\n" + "ldr x13, [%x[params_struct], %[offsetof_args_outptrs]]\n" + "ld1rh { z31.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rh { z30.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "ld1h { z7.h }, p3/Z, [x8, #-8, MUL VL]\n" + "ld1h { z8.h }, p3/Z, [x8, #-7, MUL VL]\n" + "ld1h { z9.h }, p2/Z, [x24, x15, LSL #1]\n" + "addvl x8, x8, #-6\n" + "ld1h { z10.h }, p2/Z, [x23, x15, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x22, x15, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x21, x15, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x20, x15, LSL #1]\n" "bge 2f\n" "1:" // Channel loop - "movprfx z23, z18\n fmla z23.h, p3/M, z8.h, z9.h\n" - "movprfx z24, z18\n fmla z24.h, p3/M, z7.h, z9.h\n" - "ldr x26, [x16, #0x30]\n" - "ldr x25, [x16, #0x38]\n" - "movprfx z25, z18\n fmla z25.h, p3/M, z6.h, z9.h\n" - "fmla z23.h, p3/M, z0.h, z10.h\n" - "ldr x24, [x16, #0x28]\n" - "ldr x11, [x16, #0x48]\n" - "fmla z24.h, p3/M, z4.h, z13.h\n" - "movprfx z26, z18\n fmla z26.h, p3/M, z5.h, z9.h\n" - "ldr x12, [x16, #0x40]\n" - "ld1h { z10.h }, p2/Z, [x11, x14, LSL #1]\n" - "movprfx z27, z18\n fmla z27.h, p3/M, z4.h, z9.h\n" - "movprfx z28, z18\n fmla z28.h, p3/M, z3.h, z9.h\n" - "ldr x10, [x16, #0x50]\n" - "ldr x9, [x16, #0x58]\n" - "fmla z25.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x26, x14, LSL #1]\n" - "movprfx z29, z18\n fmla z29.h, p3/M, z2.h, z9.h\n" - "ldr x28, [x16, #0x60]\n" - "fmla z23.h, p3/M, z5.h, z13.h\n" - "fmla z24.h, p3/M, z6.h, z11.h\n" - "ldr x26, [x16, #0x70]\n" - "ldr x11, [x16, #0x88]\n" - "movprfx z31, z18\n fmla z31.h, p3/M, z0.h, z9.h\n" - "fmla z25.h, p3/M, z3.h, z13.h\n" - "inch x13\n" + "movprfx z29, z14\n fmla z29.h, p3/M, z8.h, z9.h\n" + "movprfx z28, z14\n fmla z28.h, p3/M, z7.h, z9.h\n" + "ldr x23, [x17, #0x30]\n" + "ldr x26, [x17, #0x38]\n" + "movprfx z27, z14\n fmla z27.h, p3/M, z6.h, z9.h\n" + "fmla z29.h, p3/M, z0.h, z10.h\n" + "ldr x22, [x17, #0x28]\n" + "ldr x21, [x17, #0x48]\n" + "fmla z28.h, p3/M, z4.h, z13.h\n" + "movprfx z26, z14\n fmla z26.h, p3/M, z5.h, z9.h\n" + "ldr x20, [x17, #0x40]\n" + "ld1h { z19.h }, p2/Z, [x21, x15, LSL #1]\n" + "movprfx z25, z14\n fmla z25.h, p3/M, z4.h, z9.h\n" + "movprfx z24, z14\n fmla z24.h, p3/M, z3.h, z9.h\n" + "ldr x25, [x17, #0x50]\n" + "ldr x24, [x17, #0x58]\n" + "fmla z27.h, p3/M, z2.h, z11.h\n" + "ld1h { z18.h }, p2/Z, [x23, x15, LSL #1]\n" + "movprfx z23, z14\n fmla z23.h, p3/M, z2.h, z9.h\n" + "ldr x23, [x17, #0x60]\n" + "fmla z29.h, p3/M, z5.h, z13.h\n" + "fmla z28.h, p3/M, z6.h, z18.h\n" + "ldr x12, [x17, #0x70]\n" + "ldr x11, [x17, #0x88]\n" + "movprfx z22, z14\n fmla z22.h, p3/M, z0.h, z9.h\n" + "fmla z27.h, p3/M, z3.h, z13.h\n" + "inch x14\n" "mov p1.b, p2.b\n" "fmla z26.h, p3/M, z2.h, z13.h\n" - "fmla z27.h, p3/M, z1.h, z13.h\n" - "ldr x23, [x27, #0x0]\n" - "whilelt p0.h, x15, %x[n_channels]\n" - "fmla z28.h, p3/M, z0.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x25, x14, LSL #1]\n" - "fmla z29.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x24, x14, LSL #1]\n" - "movprfx z30, z18\n fmla z30.h, p3/M, z1.h, z9.h\n" - "fmla z23.h, p3/M, z7.h, z11.h\n" - "ldr x24, [x16, #0x68]\n" - "ldr x25, [x16, #0x78]\n" + "fmla z25.h, p3/M, z1.h, z13.h\n" + "ldr x10, [x13, #0x0]\n" + "whilelt p0.h, x16, %x[n_channels]\n" "fmla z24.h, p3/M, z0.h, z13.h\n" - "fmla z31.h, p3/M, z8.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x12, x14, LSL #1]\n" - "ldr x12, [x16, #0x80]\n" - "fmla z26.h, p3/M, z4.h, z11.h\n" - "fmla z27.h, p3/M, z3.h, z11.h\n" - "ldr x22, [x27, #0x8]\n" - "ldr x21, [x27, #0x10]\n" - "fmla z30.h, p3/M, z0.h, z11.h\n" - "fmla z28.h, p3/M, z4.h, z10.h\n" - "ldr x20, [x27, #0x18]\n" - "ld1h { z18.h }, p3/Z, [x17]\n" - "fmla z29.h, p3/M, z1.h, z11.h\n" - "fmla z23.h, p3/M, z1.h, z13.h\n" - "ld1h { z11.h }, p2/Z, [x10, x14, LSL #1]\n" - "ld1h { z13.h }, p2/Z, [x9, x14, LSL #1]\n" - "fmla z24.h, p3/M, z2.h, z12.h\n" - "fmla z25.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x28, x14, LSL #1]\n" - "ldr x10, [x16, #0x90]\n" - "fmla z27.h, p3/M, z5.h, z10.h\n" - "fmla z30.h, p3/M, z2.h, z10.h\n" - "ldr x28, [x16, #0xa0]\n" - "ldr x9, [x16, #0x98]\n" - "fmla z26.h, p3/M, z0.h, z11.h\n" - "fmla z28.h, p3/M, z2.h, z13.h\n" - "fmla z24.h, p3/M, z8.h, z10.h\n" - "fmla z25.h, p3/M, z7.h, z10.h\n" - "fmla z31.h, p3/M, z1.h, z10.h\n" - "fmla z29.h, p3/M, z3.h, z12.h\n" - "ld1h { z10.h }, p2/Z, [x24, x14, LSL #1]\n" - "ldr x24, [x16, #0xa8]\n" - "fmla z26.h, p3/M, z6.h, z12.h\n" - "fmla z27.h, p3/M, z7.h, z10.h\n" - "ld1h { z12.h }, p2/Z, [x12, x14, LSL #1]\n" - "ldr x12, [x16, #0xc0]\n" - "fmla z28.h, p3/M, z6.h, z10.h\n" - "fmla z30.h, p3/M, z4.h, z10.h\n" - "fmla z23.h, p3/M, z3.h, z11.h\n" - "fmla z25.h, p3/M, z5.h, z13.h\n" - "ld1h { z11.h }, p2/Z, [x26, x14, LSL #1]\n" - "ld1h { z13.h }, p2/Z, [x25, x14, LSL #1]\n" - "fmla z29.h, p3/M, z5.h, z10.h\n" - "fmla z31.h, p3/M, z3.h, z10.h\n" - "ldr x26, [x16, #0xb0]\n" - "ldr x25, [x16, #0xb8]\n" - "fmla z26.h, p3/M, z8.h, z10.h\n" - "fmla z28.h, p3/M, z8.h, z11.h\n" - "fmla z30.h, p3/M, z6.h, z13.h\n" - "fmla z24.h, p3/M, z3.h, z12.h\n" - "fmla z27.h, p3/M, z0.h, z12.h\n" - "fmla z31.h, p3/M, z5.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x14, LSL #1]\n" - "fmla z29.h, p3/M, z7.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x10, x14, LSL #1]\n" - "fmla z23.h, p3/M, z4.h, z12.h\n" - "fmla z26.h, p3/M, z1.h, z12.h\n" - "fmla z24.h, p3/M, z5.h, z11.h\n" - "ld1h { z12.h }, p2/Z, [x9, x14, LSL #1]\n" - "fmla z25.h, p3/M, z4.h, z11.h\n" - "fmla z27.h, p3/M, z2.h, z11.h\n" - "fmla z28.h, p3/M, z1.h, z11.h\n" - "fmla z30.h, p3/M, z8.h, z13.h\n" - "ld1h { z11.h }, p2/Z, [x28, x14, LSL #1]\n" - "ldr x28, [x16, #0x20]\n" - "fmla z31.h, p3/M, z7.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x24, x14, LSL #1]\n" - "fmla z23.h, p3/M, z2.h, z11.h\n" - "fmla z26.h, p3/M, z7.h, z12.h\n" - "fmla z27.h, p3/M, z6.h, z12.h\n" - "fmla z29.h, p3/M, z4.h, z12.h\n" - "fmla z30.h, p3/M, z3.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x26, x14, LSL #1]\n" - "fmla z31.h, p3/M, z4.h, z13.h\n" - "fmla z24.h, p3/M, z1.h, z11.h\n" - "fmax z24.h, p3/M, z24.h, z17.h\n" - "fmin z24.h, p3/M, z24.h, z16.h\n" - "fmla z25.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x25, x14, LSL #1]\n" + "ld1h { z17.h }, p2/Z, [x26, x15, LSL #1]\n" "fmla z23.h, p3/M, z6.h, z12.h\n" - "fmax z23.h, p3/M, z23.h, z17.h\n" - "fmla z28.h, p3/M, z7.h, z13.h\n" - "fmla z30.h, p3/M, z5.h, z13.h\n" - "fmin z23.h, p3/M, z23.h, z16.h\n" - "st1h { z23.h }, p1, [x23, x13, LSL #1]\n" - "fmla z29.h, p3/M, z0.h, z12.h\n" - "fmla z31.h, p3/M, z2.h, z11.h\n" - "ldr x23, [x27, #0x20]\n" - "st1h { z24.h }, p1, [x22, x13, LSL #1]\n" - "fmla z27.h, p3/M, z8.h, z13.h\n" - "fmla z26.h, p3/M, z3.h, z12.h\n" - "ld1h { z13.h }, p2/Z, [x12, x14, LSL #1]\n" - "ldp x12, x11, [x16, #0x0]\n" - "fmla z25.h, p3/M, z8.h, z11.h\n" - "fmla z28.h, p3/M, z5.h, z11.h\n" - "ldp x10, x9, [x16, #0x10]\n" - "fmax z25.h, p3/M, z25.h, z17.h\n" - "fmla z29.h, p3/M, z8.h, z13.h\n" - "fmla z30.h, p3/M, z7.h, z13.h\n" - "fmax z26.h, p3/M, z26.h, z17.h\n" - "fmax z27.h, p3/M, z27.h, z17.h\n" - "fmla z31.h, p3/M, z6.h, z13.h\n" - "inch x14\n" - "ld1h { z9.h }, p0/Z, [x12, x15, LSL #1]\n" - "ld1h { z10.h }, p0/Z, [x11, x15, LSL #1]\n" - "ld1h { z11.h }, p0/Z, [x10, x15, LSL #1]\n" - "ld1h { z12.h }, p0/Z, [x9, x15, LSL #1]\n" - "fmin z25.h, p3/M, z25.h, z16.h\n" - "fmin z26.h, p3/M, z26.h, z16.h\n" - "ld1h { z13.h }, p0/Z, [x28, x15, LSL #1]\n" + "ld1h { z16.h }, p2/Z, [x22, x15, LSL #1]\n" + "movprfx z21, z14\n fmla z21.h, p3/M, z1.h, z9.h\n" + "fmla z29.h, p3/M, z7.h, z18.h\n" + "ldr x22, [x17, #0x68]\n" + "ldr x21, [x17, #0x78]\n" + "fmla z28.h, p3/M, z0.h, z17.h\n" + "fmla z22.h, p3/M, z8.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x20, x15, LSL #1]\n" + "ldr x20, [x17, #0x80]\n" + "fmla z26.h, p3/M, z4.h, z18.h\n" + "fmla z25.h, p3/M, z3.h, z18.h\n" + "ldr x9, [x13, #0x8]\n" + "ldr x28, [x13, #0x10]\n" + "fmla z21.h, p3/M, z0.h, z18.h\n" + "fmla z24.h, p3/M, z4.h, z19.h\n" + "ldr x27, [x13, #0x18]\n" + "ld1h { z14.h }, p3/Z, [x8]\n" + "fmla z23.h, p3/M, z1.h, z18.h\n" + "fmla z29.h, p3/M, z1.h, z17.h\n" + "ld1h { z20.h }, p2/Z, [x25, x15, LSL #1]\n" + "ld1h { z17.h }, p2/Z, [x24, x15, LSL #1]\n" + "fmla z28.h, p3/M, z2.h, z16.h\n" + "fmla z27.h, p3/M, z1.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x23, x15, LSL #1]\n" + "ldr x26, [x17, #0x90]\n" + "fmla z25.h, p3/M, z5.h, z19.h\n" + "fmla z21.h, p3/M, z2.h, z19.h\n" + "ldr x25, [x17, #0xa0]\n" + "ldr x24, [x17, #0x98]\n" + "fmla z26.h, p3/M, z0.h, z20.h\n" + "fmla z24.h, p3/M, z2.h, z17.h\n" + "fmla z28.h, p3/M, z8.h, z19.h\n" + "fmla z27.h, p3/M, z7.h, z19.h\n" + "fmla z22.h, p3/M, z1.h, z19.h\n" + "fmla z23.h, p3/M, z3.h, z16.h\n" + "ld1h { z18.h }, p2/Z, [x22, x15, LSL #1]\n" + "ldr x23, [x17, #0xa8]\n" + "fmla z26.h, p3/M, z6.h, z16.h\n" + "fmla z25.h, p3/M, z7.h, z18.h\n" + "ld1h { z19.h }, p2/Z, [x20, x15, LSL #1]\n" + "ldr x22, [x17, #0xc0]\n" + "fmla z24.h, p3/M, z6.h, z18.h\n" + "fmla z21.h, p3/M, z4.h, z18.h\n" + "fmla z29.h, p3/M, z3.h, z20.h\n" + "fmla z27.h, p3/M, z5.h, z17.h\n" + "ld1h { z17.h }, p2/Z, [x12, x15, LSL #1]\n" + "ld1h { z16.h }, p2/Z, [x21, x15, LSL #1]\n" + "fmla z23.h, p3/M, z5.h, z18.h\n" + "fmla z22.h, p3/M, z3.h, z18.h\n" + "ldr x21, [x17, #0xb0]\n" + "ldr x20, [x17, #0xb8]\n" + "fmla z26.h, p3/M, z8.h, z18.h\n" + "fmla z24.h, p3/M, z8.h, z17.h\n" + "fmla z21.h, p3/M, z6.h, z16.h\n" + "fmla z28.h, p3/M, z3.h, z19.h\n" + "fmla z25.h, p3/M, z0.h, z19.h\n" + "fmla z22.h, p3/M, z5.h, z17.h\n" + "ld1h { z17.h }, p2/Z, [x11, x15, LSL #1]\n" + "fmla z23.h, p3/M, z7.h, z16.h\n" + "ld1h { z18.h }, p2/Z, [x26, x15, LSL #1]\n" + "fmla z29.h, p3/M, z4.h, z19.h\n" + "fmla z26.h, p3/M, z1.h, z19.h\n" + "fmla z28.h, p3/M, z5.h, z17.h\n" + "ld1h { z16.h }, p2/Z, [x24, x15, LSL #1]\n" + "fmla z27.h, p3/M, z4.h, z17.h\n" + "fmla z25.h, p3/M, z2.h, z17.h\n" + "fmla z24.h, p3/M, z1.h, z17.h\n" + "fmla z21.h, p3/M, z8.h, z18.h\n" + "ld1h { z17.h }, p2/Z, [x25, x15, LSL #1]\n" + "ldr x25, [x17, #0x20]\n" + "fmla z22.h, p3/M, z7.h, z18.h\n" + "ld1h { z18.h }, p2/Z, [x23, x15, LSL #1]\n" + "fmla z29.h, p3/M, z2.h, z17.h\n" + "fmla z26.h, p3/M, z7.h, z16.h\n" + "fmla z25.h, p3/M, z6.h, z16.h\n" + "fmla z23.h, p3/M, z4.h, z16.h\n" + "fmla z21.h, p3/M, z3.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x21, x15, LSL #1]\n" + "fmla z22.h, p3/M, z4.h, z18.h\n" + "fmla z28.h, p3/M, z1.h, z17.h\n" + "fmax z28.h, p3/M, z28.h, z31.h\n" + "fmin z28.h, p3/M, z28.h, z30.h\n" + "fmla z27.h, p3/M, z0.h, z17.h\n" + "ld1h { z17.h }, p2/Z, [x20, x15, LSL #1]\n" + "fmla z29.h, p3/M, z6.h, z16.h\n" + "fmax z29.h, p3/M, z29.h, z31.h\n" + "fmla z24.h, p3/M, z7.h, z18.h\n" + "fmla z21.h, p3/M, z5.h, z18.h\n" + "fmin z29.h, p3/M, z29.h, z30.h\n" + "st1h { z29.h }, p1, [x10, x14, LSL #1]\n" + "fmla z23.h, p3/M, z0.h, z16.h\n" + "fmla z22.h, p3/M, z2.h, z17.h\n" + "ldr x24, [x13, #0x20]\n" + "st1h { z28.h }, p1, [x9, x14, LSL #1]\n" + "fmla z25.h, p3/M, z8.h, z18.h\n" + "fmla z26.h, p3/M, z3.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x22, x15, LSL #1]\n" + "ldp x23, x22, [x17, #0x0]\n" + "fmla z27.h, p3/M, z8.h, z17.h\n" + "fmla z24.h, p3/M, z5.h, z17.h\n" + "ldp x21, x20, [x17, #0x10]\n" + "fmax z27.h, p3/M, z27.h, z31.h\n" + "fmla z23.h, p3/M, z8.h, z16.h\n" + "fmla z21.h, p3/M, z7.h, z16.h\n" + "fmax z26.h, p3/M, z26.h, z31.h\n" + "fmax z25.h, p3/M, z25.h, z31.h\n" + "fmla z22.h, p3/M, z6.h, z16.h\n" "inch x15\n" - "fmin z27.h, p3/M, z27.h, z16.h\n" - "st1h { z25.h }, p1, [x21, x13, LSL #1]\n" - "fmax z28.h, p3/M, z28.h, z17.h\n" - "fmax z29.h, p3/M, z29.h, z17.h\n" - "st1h { z26.h }, p1, [x20, x13, LSL #1]\n" - "ldr x22, [x27, #0x28]\n" - "fmax z30.h, p3/M, z30.h, z17.h\n" - "fmax z31.h, p3/M, z31.h, z17.h\n" - "st1h { z27.h }, p1, [x23, x13, LSL #1]\n" - "ldr x21, [x27, #0x30]\n" - "ldr x20, [x27, #0x38]\n" - "ldr x23, [x27, #0x40]\n" - "whilelt p2.h, x14, %x[n_channels]\n" - "cmp x15, %x[n_channels]\n" - "ld1h { z0.h }, p3/Z, [x17, #1, MUL VL]\n" - "ld1h { z1.h }, p3/Z, [x17, #2, MUL VL]\n" - "fmin z28.h, p3/M, z28.h, z16.h\n" - "fmin z29.h, p3/M, z29.h, z16.h\n" - "ld1h { z2.h }, p3/Z, [x17, #3, MUL VL]\n" - "ld1h { z3.h }, p3/Z, [x17, #4, MUL VL]\n" - "fmin z30.h, p3/M, z30.h, z16.h\n" - "fmin z31.h, p3/M, z31.h, z16.h\n" - "ld1h { z4.h }, p3/Z, [x17, #5, MUL VL]\n" - "ld1h { z5.h }, p3/Z, [x17, #6, MUL VL]\n" - "st1h { z28.h }, p1, [x22, x13, LSL #1]\n" - "ld1h { z6.h }, p3/Z, [x17, #7, MUL VL]\n" - "addvl x17, x17, #16\n" - "st1h { z29.h }, p1, [x21, x13, LSL #1]\n" - "ld1h { z7.h }, p3/Z, [x17, #-8, MUL VL]\n" - "st1h { z30.h }, p1, [x20, x13, LSL #1]\n" - "ld1h { z8.h }, p3/Z, [x17, #-7, MUL VL]\n" - "addvl x17, x17, #-6\n" - "st1h { z31.h }, p1, [x23, x13, LSL #1]\n" + "ld1h { z9.h }, p0/Z, [x23, x16, LSL #1]\n" + "ld1h { z10.h }, p0/Z, [x22, x16, LSL #1]\n" + "ld1h { z11.h }, p0/Z, [x21, x16, LSL #1]\n" + "ld1h { z12.h }, p0/Z, [x20, x16, LSL #1]\n" + "fmin z27.h, p3/M, z27.h, z30.h\n" + "fmin z26.h, p3/M, z26.h, z30.h\n" + "ld1h { z13.h }, p0/Z, [x25, x16, LSL #1]\n" + "inch x16\n" + "fmin z25.h, p3/M, z25.h, z30.h\n" + "st1h { z27.h }, p1, [x28, x14, LSL #1]\n" + "fmax z24.h, p3/M, z24.h, z31.h\n" + "fmax z23.h, p3/M, z23.h, z31.h\n" + "st1h { z26.h }, p1, [x27, x14, LSL #1]\n" + "ldr x23, [x13, #0x28]\n" + "fmax z21.h, p3/M, z21.h, z31.h\n" + "fmax z22.h, p3/M, z22.h, z31.h\n" + "st1h { z25.h }, p1, [x24, x14, LSL #1]\n" + "ldr x22, [x13, #0x30]\n" + "ldr x21, [x13, #0x38]\n" + "ldr x20, [x13, #0x40]\n" + "whilelt p2.h, x15, %x[n_channels]\n" + "cmp x16, %x[n_channels]\n" + "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n" + "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n" + "fmin z24.h, p3/M, z24.h, z30.h\n" + "fmin z23.h, p3/M, z23.h, z30.h\n" + "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n" + "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n" + "fmin z21.h, p3/M, z21.h, z30.h\n" + "fmin z22.h, p3/M, z22.h, z30.h\n" + "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n" + "ld1h { z5.h }, p3/Z, [x8, #6, MUL VL]\n" + "st1h { z24.h }, p1, [x23, x14, LSL #1]\n" + "ld1h { z6.h }, p3/Z, [x8, #7, MUL VL]\n" + "addvl x8, x8, #16\n" + "st1h { z23.h }, p1, [x22, x14, LSL #1]\n" + "ld1h { z7.h }, p3/Z, [x8, #-8, MUL VL]\n" + "st1h { z21.h }, p1, [x21, x14, LSL #1]\n" + "ld1h { z8.h }, p3/Z, [x8, #-7, MUL VL]\n" + "addvl x8, x8, #-6\n" + "st1h { z22.h }, p1, [x20, x14, LSL #1]\n" "blt 1b\n" "2:" // Channel tail - "movprfx z23, z18\n fmla z23.h, p3/M, z8.h, z9.h\n" - "movprfx z24, z18\n fmla z24.h, p3/M, z7.h, z9.h\n" - "ldr x26, [x16, #0x30]\n" - "ldr x25, [x16, #0x38]\n" - "movprfx z25, z18\n fmla z25.h, p3/M, z6.h, z9.h\n" - "fmla z23.h, p3/M, z0.h, z10.h\n" - "ldr x24, [x16, #0x28]\n" - "ldr x11, [x16, #0x48]\n" - "fmla z24.h, p3/M, z4.h, z13.h\n" - "movprfx z26, z18\n fmla z26.h, p3/M, z5.h, z9.h\n" - "ldr x12, [x16, #0x40]\n" - "ld1h { z10.h }, p2/Z, [x11, x14, LSL #1]\n" - "movprfx z27, z18\n fmla z27.h, p3/M, z4.h, z9.h\n" - "movprfx z28, z18\n fmla z28.h, p3/M, z3.h, z9.h\n" - "ldr x10, [x16, #0x50]\n" - "ldr x9, [x16, #0x58]\n" - "fmla z25.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x26, x14, LSL #1]\n" - "movprfx z29, z18\n fmla z29.h, p3/M, z2.h, z9.h\n" - "ldr x28, [x16, #0x60]\n" - "fmla z23.h, p3/M, z5.h, z13.h\n" - "fmla z24.h, p3/M, z6.h, z11.h\n" - "ldr x26, [x16, #0x70]\n" - "ldr x11, [x16, #0x88]\n" - "movprfx z31, z18\n fmla z31.h, p3/M, z0.h, z9.h\n" - "fmla z25.h, p3/M, z3.h, z13.h\n" - "inch x13\n" - "mov p1.b, p2.b\n" + "movprfx z29, z14\n fmla z29.h, p3/M, z8.h, z9.h\n" + "movprfx z28, z14\n fmla z28.h, p3/M, z7.h, z9.h\n" + "ldr x23, [x17, #0x30]\n" + "ldr x26, [x17, #0x38]\n" + "movprfx z27, z14\n fmla z27.h, p3/M, z6.h, z9.h\n" + "fmla z29.h, p3/M, z0.h, z10.h\n" + "ldr x22, [x17, #0x28]\n" + "ldr x21, [x17, #0x48]\n" + "fmla z28.h, p3/M, z4.h, z13.h\n" + "movprfx z26, z14\n fmla z26.h, p3/M, z5.h, z9.h\n" + "ldr x20, [x17, #0x40]\n" + "ld1h { z19.h }, p2/Z, [x21, x15, LSL #1]\n" + "movprfx z25, z14\n fmla z25.h, p3/M, z4.h, z9.h\n" + "movprfx z24, z14\n fmla z24.h, p3/M, z3.h, z9.h\n" + "ldr x25, [x17, #0x50]\n" + "ldr x24, [x17, #0x58]\n" + "fmla z27.h, p3/M, z2.h, z11.h\n" + "ld1h { z18.h }, p2/Z, [x23, x15, LSL #1]\n" + "movprfx z23, z14\n fmla z23.h, p3/M, z2.h, z9.h\n" + "ldr x23, [x17, #0x60]\n" + "fmla z29.h, p3/M, z5.h, z13.h\n" + "fmla z28.h, p3/M, z6.h, z18.h\n" + "ldr x12, [x17, #0x70]\n" + "ldr x11, [x17, #0x88]\n" + "movprfx z22, z14\n fmla z22.h, p3/M, z0.h, z9.h\n" + "fmla z27.h, p3/M, z3.h, z13.h\n" + "inch x14\n" + "mov p0.b, p2.b\n" "fmla z26.h, p3/M, z2.h, z13.h\n" - "fmla z27.h, p3/M, z1.h, z13.h\n" - "ldr x23, [x27, #0x0]\n" - "ldr x22, [x27, #0x8]\n" - "fmla z28.h, p3/M, z0.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x25, x14, LSL #1]\n" - "fmla z29.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x24, x14, LSL #1]\n" - "movprfx z30, z18\n fmla z30.h, p3/M, z1.h, z9.h\n" - "fmla z23.h, p3/M, z7.h, z11.h\n" - "ldr x24, [x16, #0x68]\n" - "ldr x25, [x16, #0x78]\n" + "fmla z25.h, p3/M, z1.h, z13.h\n" + "ldr x10, [x13, #0x0]\n" + "ldr x9, [x13, #0x8]\n" "fmla z24.h, p3/M, z0.h, z13.h\n" - "fmla z31.h, p3/M, z8.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x12, x14, LSL #1]\n" - "ldr x12, [x16, #0x80]\n" - "fmla z26.h, p3/M, z4.h, z11.h\n" - "fmla z27.h, p3/M, z3.h, z11.h\n" - "ldr x21, [x27, #0x10]\n" - "ldr x20, [x27, #0x18]\n" - "fmla z30.h, p3/M, z0.h, z11.h\n" - "fmla z28.h, p3/M, z4.h, z10.h\n" - "fmla z29.h, p3/M, z1.h, z11.h\n" - "fmla z23.h, p3/M, z1.h, z13.h\n" - "ld1h { z11.h }, p2/Z, [x10, x14, LSL #1]\n" - "ld1h { z13.h }, p2/Z, [x9, x14, LSL #1]\n" - "fmla z24.h, p3/M, z2.h, z12.h\n" - "fmla z25.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x28, x14, LSL #1]\n" - "ldr x10, [x16, #0x90]\n" - "fmla z27.h, p3/M, z5.h, z10.h\n" - "fmla z30.h, p3/M, z2.h, z10.h\n" - "ldr x28, [x16, #0xa0]\n" - "ldr x9, [x16, #0x98]\n" - "fmla z26.h, p3/M, z0.h, z11.h\n" - "fmla z28.h, p3/M, z2.h, z13.h\n" - "fmla z24.h, p3/M, z8.h, z10.h\n" - "fmla z25.h, p3/M, z7.h, z10.h\n" - "fmla z31.h, p3/M, z1.h, z10.h\n" - "fmla z29.h, p3/M, z3.h, z12.h\n" - "ld1h { z10.h }, p2/Z, [x24, x14, LSL #1]\n" - "ldr x24, [x16, #0xa8]\n" - "fmla z26.h, p3/M, z6.h, z12.h\n" - "fmla z27.h, p3/M, z7.h, z10.h\n" - "ld1h { z12.h }, p2/Z, [x12, x14, LSL #1]\n" - "ldr x12, [x16, #0xc0]\n" - "fmla z28.h, p3/M, z6.h, z10.h\n" - "fmla z30.h, p3/M, z4.h, z10.h\n" - "fmla z23.h, p3/M, z3.h, z11.h\n" - "fmla z25.h, p3/M, z5.h, z13.h\n" - "ld1h { z11.h }, p2/Z, [x26, x14, LSL #1]\n" - "ld1h { z13.h }, p2/Z, [x25, x14, LSL #1]\n" - "fmla z29.h, p3/M, z5.h, z10.h\n" - "fmla z31.h, p3/M, z3.h, z10.h\n" - "ldr x26, [x16, #0xb0]\n" - "ldr x25, [x16, #0xb8]\n" - "fmla z26.h, p3/M, z8.h, z10.h\n" - "fmla z28.h, p3/M, z8.h, z11.h\n" - "fmla z30.h, p3/M, z6.h, z13.h\n" - "fmla z24.h, p3/M, z3.h, z12.h\n" - "fmla z27.h, p3/M, z0.h, z12.h\n" - "fmla z31.h, p3/M, z5.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x14, LSL #1]\n" - "fmla z29.h, p3/M, z7.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x10, x14, LSL #1]\n" - "fmla z23.h, p3/M, z4.h, z12.h\n" - "fmla z26.h, p3/M, z1.h, z12.h\n" - "fmla z24.h, p3/M, z5.h, z11.h\n" - "ld1h { z12.h }, p2/Z, [x9, x14, LSL #1]\n" - "fmla z25.h, p3/M, z4.h, z11.h\n" - "fmla z27.h, p3/M, z2.h, z11.h\n" - "fmla z28.h, p3/M, z1.h, z11.h\n" - "fmla z30.h, p3/M, z8.h, z13.h\n" - "ld1h { z11.h }, p2/Z, [x28, x14, LSL #1]\n" - "fmla z31.h, p3/M, z7.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x24, x14, LSL #1]\n" - "fmla z23.h, p3/M, z2.h, z11.h\n" - "fmla z26.h, p3/M, z7.h, z12.h\n" - "fmla z27.h, p3/M, z6.h, z12.h\n" - "fmla z29.h, p3/M, z4.h, z12.h\n" - "fmla z30.h, p3/M, z3.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x26, x14, LSL #1]\n" - "fmla z31.h, p3/M, z4.h, z13.h\n" - "fmla z24.h, p3/M, z1.h, z11.h\n" - "fmax z24.h, p3/M, z24.h, z17.h\n" - "fmin z24.h, p3/M, z24.h, z16.h\n" - "fmla z25.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x25, x14, LSL #1]\n" + "ld1h { z17.h }, p2/Z, [x26, x15, LSL #1]\n" "fmla z23.h, p3/M, z6.h, z12.h\n" - "fmax z23.h, p3/M, z23.h, z17.h\n" - "fmla z28.h, p3/M, z7.h, z13.h\n" - "fmla z30.h, p3/M, z5.h, z13.h\n" - "fmin z23.h, p3/M, z23.h, z16.h\n" - "st1h { z23.h }, p1, [x23, x13, LSL #1]\n" - "fmla z29.h, p3/M, z0.h, z12.h\n" - "fmla z31.h, p3/M, z2.h, z11.h\n" - "ldr x23, [x27, #0x20]\n" - "st1h { z24.h }, p1, [x22, x13, LSL #1]\n" - "fmla z27.h, p3/M, z8.h, z13.h\n" - "fmla z26.h, p3/M, z3.h, z12.h\n" - "ld1h { z13.h }, p2/Z, [x12, x14, LSL #1]\n" - "fmax z26.h, p3/M, z26.h, z17.h\n" - "fmla z25.h, p3/M, z8.h, z11.h\n" - "fmla z28.h, p3/M, z5.h, z11.h\n" - "fmax z25.h, p3/M, z25.h, z17.h\n" - "fmax z27.h, p3/M, z27.h, z17.h\n" - "fmla z29.h, p3/M, z8.h, z13.h\n" - "fmla z30.h, p3/M, z7.h, z13.h\n" - "fmin z25.h, p3/M, z25.h, z16.h\n" - "fmin z26.h, p3/M, z26.h, z16.h\n" - "fmla z31.h, p3/M, z6.h, z13.h\n" - "fmin z27.h, p3/M, z27.h, z16.h\n" - "fmax z28.h, p3/M, z28.h, z17.h\n" - "st1h { z25.h }, p1, [x21, x13, LSL #1]\n" - "fmax z29.h, p3/M, z29.h, z17.h\n" - "fmax z30.h, p3/M, z30.h, z17.h\n" - "st1h { z26.h }, p1, [x20, x13, LSL #1]\n" - "ldr x22, [x27, #0x28]\n" - "fmax z31.h, p3/M, z31.h, z17.h\n" - "st1h { z27.h }, p1, [x23, x13, LSL #1]\n" - "ldr x21, [x27, #0x30]\n" - "ldr x20, [x27, #0x38]\n" - "ldr x23, [x27, #0x40]\n" - "fmin z28.h, p3/M, z28.h, z16.h\n" - "fmin z29.h, p3/M, z29.h, z16.h\n" - "st1h { z28.h }, p1, [x22, x13, LSL #1]\n" - "fmin z30.h, p3/M, z30.h, z16.h\n" - "fmin z31.h, p3/M, z31.h, z16.h\n" - "st1h { z29.h }, p1, [x21, x13, LSL #1]\n" - "st1h { z30.h }, p1, [x20, x13, LSL #1]\n" - "st1h { z31.h }, p1, [x23, x13, LSL #1]\n" + "ld1h { z16.h }, p2/Z, [x22, x15, LSL #1]\n" + "movprfx z21, z14\n fmla z21.h, p3/M, z1.h, z9.h\n" + "fmla z29.h, p3/M, z7.h, z18.h\n" + "ldr x22, [x17, #0x68]\n" + "ldr x21, [x17, #0x78]\n" + "fmla z28.h, p3/M, z0.h, z17.h\n" + "fmla z22.h, p3/M, z8.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x20, x15, LSL #1]\n" + "ldr x20, [x17, #0x80]\n" + "fmla z26.h, p3/M, z4.h, z18.h\n" + "fmla z25.h, p3/M, z3.h, z18.h\n" + "ldr x28, [x13, #0x10]\n" + "ldr x27, [x13, #0x18]\n" + "fmla z21.h, p3/M, z0.h, z18.h\n" + "fmla z24.h, p3/M, z4.h, z19.h\n" + "fmla z23.h, p3/M, z1.h, z18.h\n" + "fmla z29.h, p3/M, z1.h, z17.h\n" + "ld1h { z20.h }, p2/Z, [x25, x15, LSL #1]\n" + "ld1h { z17.h }, p2/Z, [x24, x15, LSL #1]\n" + "fmla z28.h, p3/M, z2.h, z16.h\n" + "fmla z27.h, p3/M, z1.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x23, x15, LSL #1]\n" + "ldr x26, [x17, #0x90]\n" + "fmla z25.h, p3/M, z5.h, z19.h\n" + "fmla z21.h, p3/M, z2.h, z19.h\n" + "ldr x25, [x17, #0xa0]\n" + "ldr x24, [x17, #0x98]\n" + "fmla z26.h, p3/M, z0.h, z20.h\n" + "fmla z24.h, p3/M, z2.h, z17.h\n" + "fmla z28.h, p3/M, z8.h, z19.h\n" + "fmla z27.h, p3/M, z7.h, z19.h\n" + "fmla z22.h, p3/M, z1.h, z19.h\n" + "fmla z23.h, p3/M, z3.h, z16.h\n" + "ld1h { z18.h }, p2/Z, [x22, x15, LSL #1]\n" + "ldr x23, [x17, #0xa8]\n" + "fmla z26.h, p3/M, z6.h, z16.h\n" + "fmla z25.h, p3/M, z7.h, z18.h\n" + "ld1h { z19.h }, p2/Z, [x20, x15, LSL #1]\n" + "ldr x22, [x17, #0xc0]\n" + "fmla z24.h, p3/M, z6.h, z18.h\n" + "fmla z21.h, p3/M, z4.h, z18.h\n" + "fmla z29.h, p3/M, z3.h, z20.h\n" + "fmla z27.h, p3/M, z5.h, z17.h\n" + "ld1h { z17.h }, p2/Z, [x12, x15, LSL #1]\n" + "ld1h { z16.h }, p2/Z, [x21, x15, LSL #1]\n" + "fmla z23.h, p3/M, z5.h, z18.h\n" + "fmla z22.h, p3/M, z3.h, z18.h\n" + "ldr x21, [x17, #0xb0]\n" + "ldr x20, [x17, #0xb8]\n" + "fmla z26.h, p3/M, z8.h, z18.h\n" + "fmla z24.h, p3/M, z8.h, z17.h\n" + "fmla z21.h, p3/M, z6.h, z16.h\n" + "fmla z28.h, p3/M, z3.h, z19.h\n" + "fmla z25.h, p3/M, z0.h, z19.h\n" + "fmla z22.h, p3/M, z5.h, z17.h\n" + "ld1h { z17.h }, p2/Z, [x11, x15, LSL #1]\n" + "fmla z23.h, p3/M, z7.h, z16.h\n" + "ld1h { z18.h }, p2/Z, [x26, x15, LSL #1]\n" + "fmla z29.h, p3/M, z4.h, z19.h\n" + "fmla z26.h, p3/M, z1.h, z19.h\n" + "fmla z28.h, p3/M, z5.h, z17.h\n" + "ld1h { z16.h }, p2/Z, [x24, x15, LSL #1]\n" + "fmla z27.h, p3/M, z4.h, z17.h\n" + "fmla z25.h, p3/M, z2.h, z17.h\n" + "fmla z24.h, p3/M, z1.h, z17.h\n" + "fmla z21.h, p3/M, z8.h, z18.h\n" + "ld1h { z17.h }, p2/Z, [x25, x15, LSL #1]\n" + "fmla z22.h, p3/M, z7.h, z18.h\n" + "ld1h { z18.h }, p2/Z, [x23, x15, LSL #1]\n" + "fmla z29.h, p3/M, z2.h, z17.h\n" + "fmla z26.h, p3/M, z7.h, z16.h\n" + "fmla z25.h, p3/M, z6.h, z16.h\n" + "fmla z23.h, p3/M, z4.h, z16.h\n" + "fmla z21.h, p3/M, z3.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x21, x15, LSL #1]\n" + "fmla z22.h, p3/M, z4.h, z18.h\n" + "fmla z28.h, p3/M, z1.h, z17.h\n" + "fmax z28.h, p3/M, z28.h, z31.h\n" + "fmin z28.h, p3/M, z28.h, z30.h\n" + "fmla z27.h, p3/M, z0.h, z17.h\n" + "ld1h { z17.h }, p2/Z, [x20, x15, LSL #1]\n" + "fmla z29.h, p3/M, z6.h, z16.h\n" + "fmax z29.h, p3/M, z29.h, z31.h\n" + "fmla z24.h, p3/M, z7.h, z18.h\n" + "fmla z21.h, p3/M, z5.h, z18.h\n" + "fmin z29.h, p3/M, z29.h, z30.h\n" + "st1h { z29.h }, p0, [x10, x14, LSL #1]\n" + "fmla z23.h, p3/M, z0.h, z16.h\n" + "fmla z22.h, p3/M, z2.h, z17.h\n" + "ldr x20, [x13, #0x20]\n" + "st1h { z28.h }, p0, [x9, x14, LSL #1]\n" + "fmla z25.h, p3/M, z8.h, z18.h\n" + "fmla z26.h, p3/M, z3.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x22, x15, LSL #1]\n" + "fmax z26.h, p3/M, z26.h, z31.h\n" + "fmla z27.h, p3/M, z8.h, z17.h\n" + "fmla z24.h, p3/M, z5.h, z17.h\n" + "fmax z27.h, p3/M, z27.h, z31.h\n" + "fmax z25.h, p3/M, z25.h, z31.h\n" + "fmla z23.h, p3/M, z8.h, z16.h\n" + "fmla z21.h, p3/M, z7.h, z16.h\n" + "fmin z27.h, p3/M, z27.h, z30.h\n" + "fmin z26.h, p3/M, z26.h, z30.h\n" + "fmla z22.h, p3/M, z6.h, z16.h\n" + "fmin z25.h, p3/M, z25.h, z30.h\n" + "fmax z24.h, p3/M, z24.h, z31.h\n" + "st1h { z27.h }, p0, [x28, x14, LSL #1]\n" + "fmax z23.h, p3/M, z23.h, z31.h\n" + "fmax z21.h, p3/M, z21.h, z31.h\n" + "st1h { z26.h }, p0, [x27, x14, LSL #1]\n" + "ldr x23, [x13, #0x28]\n" + "fmax z22.h, p3/M, z22.h, z31.h\n" + "st1h { z25.h }, p0, [x20, x14, LSL #1]\n" + "ldr x22, [x13, #0x30]\n" + "ldr x21, [x13, #0x38]\n" + "ldr x20, [x13, #0x40]\n" + "fmin z24.h, p3/M, z24.h, z30.h\n" + "fmin z23.h, p3/M, z23.h, z30.h\n" + "st1h { z24.h }, p0, [x23, x14, LSL #1]\n" + "fmin z21.h, p3/M, z21.h, z30.h\n" + "fmin z22.h, p3/M, z22.h, z30.h\n" + "st1h { z23.h }, p0, [x22, x14, LSL #1]\n" + "st1h { z21.h }, p0, [x21, x14, LSL #1]\n" + "st1h { z22.h }, p0, [x20, x14, LSL #1]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp index ac6ae284fd84a92d76bd8545263b99f61966d85d..af5ee740c978d8be9cc22afb6aa92fa3e3ea9031 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,19 +22,19 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include #pragma once -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { -void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); -void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); +void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(const __fp16 *const *const input_ptrs, __fp16 *const *const outptrs, const void *params, unsigned int n_channels, const __fp16 activation_min, const __fp16 activation_max); +void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(const unsigned int n_tile_rows, const unsigned int n_tile_cols, const __fp16 *inptr, int64_t ld_input_row, int64_t ld_input_col, __fp16 *outptr, int64_t ld_output_row, int64_t ld_output_col, const void *params, unsigned int n_channels, const __fp16 activation_min, const __fp16 activation_max); class sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst : public DepthwiseDepthfirstStrategy<__fp16, __fp16, __fp16, __fp16> { @@ -57,7 +57,7 @@ class sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst : public DepthwiseDepthfirst constexpr static unsigned int output_cols = 4; sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst(const CPUInfo *) - : DepthwiseDepthfirstStrategy<__fp16, __fp16, __fp16, __fp16>(4, 3, 1) {} + : Parent(output_rows, output_cols, kernel_rows, kernel_cols, stride_rows, stride_cols) {} arm_gemm::VLType get_vl_type(void) const override { return vl_type; } @@ -68,4 +68,4 @@ class sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst : public DepthwiseDepthfirst } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp index c0b9137f6bffa291b56dd8387cae5645413d2f4d..41eaa4f18c79a33cebcd82f4208e9e1ab366d600 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -113,7 +113,7 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "mul x21, x21, x25\n" // offset *= kernel_stride * output_size "add x8, x8, x21, LSL #1\n" // inptr[0] += offset * sizeof(__fp16) "add x13, x8, x23, LSL #1\n" - "ld1h { z15.h }, p3/Z, [x17]\n" + "ld1h { z19.h }, p3/Z, [x17]\n" "mul x20, x20, x24\n" // offset *= output_tile_size "add x12, x13, x23, LSL #1\n" "add x15, x15, x20, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16) @@ -132,8 +132,8 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "add x27, x10, x5\n" "add x26, x9, x22, LSL #1\n" "add x25, x6, x6\n" - "ld1rh { z14.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "ld1rh { z13.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "ld1rh { z15.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rh { z16.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "cmp x16, %x[n_channels]\n" "add x24, x28, x23, LSL #1\n" "ld1h { z7.h }, p3/Z, [x17, #-8, MUL VL]\n" @@ -149,500 +149,500 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "addvl x17, x17, #-6\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "movprfx z21, z15\n fmla z21.h, p3/M, z4.h, z9.h\n" - "movprfx z16, z15\n fmla z16.h, p3/M, z8.h, z9.h\n" + "movprfx z14, z19\n fmla z14.h, p3/M, z4.h, z9.h\n" + "movprfx z31, z19\n fmla z31.h, p3/M, z8.h, z9.h\n" "whilelt p1.h, x16, %x[n_channels]\n" "inch x21\n" - "movprfx z22, z15\n fmla z22.h, p3/M, z3.h, z9.h\n" - "movprfx z25, z15\n fmla z25.h, p3/M, z1.h, z9.h\n" + "movprfx z21, z19\n fmla z21.h, p3/M, z3.h, z9.h\n" + "movprfx z22, z19\n fmla z22.h, p3/M, z1.h, z9.h\n" "inch x16\n" "mov p0.b, p2.b\n" - "movprfx z26, z15\n fmla z26.h, p3/M, z0.h, z9.h\n" - "fmla z21.h, p3/M, z5.h, z12.h\n" + "movprfx z20, z19\n fmla z20.h, p3/M, z0.h, z9.h\n" + "fmla z14.h, p3/M, z5.h, z12.h\n" "inch x20\n" - "movprfx z17, z15\n fmla z17.h, p3/M, z7.h, z9.h\n" - "movprfx z18, z15\n fmla z18.h, p3/M, z6.h, z9.h\n" - "movprfx z20, z15\n fmla z20.h, p3/M, z5.h, z9.h\n" - "movprfx z24, z15\n fmla z24.h, p3/M, z2.h, z9.h\n" + "movprfx z13, z19\n fmla z13.h, p3/M, z7.h, z9.h\n" + "movprfx z17, z19\n fmla z17.h, p3/M, z6.h, z9.h\n" + "movprfx z27, z19\n fmla z27.h, p3/M, z5.h, z9.h\n" + "movprfx z18, z19\n fmla z18.h, p3/M, z2.h, z9.h\n" "ld1h { z9.h }, p2/Z, [x11, x7, LSL #1]\n" - "fmla z16.h, p3/M, z0.h, z10.h\n" - "movprfx z19, z15\n fmla z19.h, p3/M, z2.h, z11.h\n" - "ld1h { z10.h }, p2/Z, [x24]\n" + "fmla z31.h, p3/M, z0.h, z10.h\n" + "movprfx z30, z19\n fmla z30.h, p3/M, z2.h, z11.h\n" + "ld1h { z29.h }, p2/Z, [x24]\n" "ld1h { z11.h }, p2/Z, [x24, x27, LSL #1]\n" - "fmla z22.h, p3/M, z4.h, z12.h\n" - "fmla z25.h, p3/M, z2.h, z12.h\n" - "fmla z26.h, p3/M, z1.h, z12.h\n" - "movprfx z28, z15\n fmla z28.h, p3/M, z6.h, z10.h\n" + "fmla z21.h, p3/M, z4.h, z12.h\n" + "fmla z22.h, p3/M, z2.h, z12.h\n" + "fmla z20.h, p3/M, z1.h, z12.h\n" + "movprfx z23, z19\n fmla z23.h, p3/M, z6.h, z29.h\n" "ld1h { z10.h }, p2/Z, [x11, x14, LSL #1]\n" - "fmla z21.h, p3/M, z7.h, z9.h\n" - "fmla z17.h, p3/M, z8.h, z12.h\n" - "fmla z18.h, p3/M, z7.h, z12.h\n" - "fmla z19.h, p3/M, z6.h, z12.h\n" - "movprfx z23, z15\n fmla z23.h, p3/M, z3.h, z12.h\n" - "movprfx z27, z15\n fmla z27.h, p3/M, z0.h, z12.h\n" + "fmla z14.h, p3/M, z7.h, z9.h\n" + "fmla z13.h, p3/M, z8.h, z12.h\n" + "fmla z17.h, p3/M, z7.h, z12.h\n" + "fmla z30.h, p3/M, z6.h, z12.h\n" + "movprfx z26, z19\n fmla z26.h, p3/M, z3.h, z12.h\n" + "movprfx z28, z19\n fmla z28.h, p3/M, z0.h, z12.h\n" "ld1h { z12.h }, p2/Z, [x8, x5, LSL #1]\n" - "movprfx z31, z15\n fmla z31.h, p3/M, z8.h, z11.h\n" - "fmla z22.h, p3/M, z6.h, z9.h\n" + "movprfx z24, z19\n fmla z24.h, p3/M, z8.h, z11.h\n" + "fmla z21.h, p3/M, z6.h, z9.h\n" "ld1h { z11.h }, p2/Z, [x8, x10, LSL #1]\n" - "fmla z25.h, p3/M, z4.h, z9.h\n" - "fmla z26.h, p3/M, z3.h, z9.h\n" - "fmla z20.h, p3/M, z8.h, z9.h\n" - "fmla z24.h, p3/M, z5.h, z9.h\n" - "fmla z28.h, p3/M, z2.h, z9.h\n" - "fmla z21.h, p3/M, z8.h, z10.h\n" - "fmla z16.h, p3/M, z1.h, z12.h\n" - "fmla z17.h, p3/M, z0.h, z12.h\n" + "fmla z22.h, p3/M, z4.h, z9.h\n" + "fmla z20.h, p3/M, z3.h, z9.h\n" + "movprfx z25, z19\n fmla z25.h, p3/M, z1.h, z9.h\n" + "movprfx z29, z19\n fmla z29.h, p3/M, z0.h, z9.h\n" + "ld1h { z19.h }, p3/Z, [x17]\n" + "fmla z27.h, p3/M, z8.h, z9.h\n" + "fmla z18.h, p3/M, z5.h, z9.h\n" + "fmla z23.h, p3/M, z2.h, z9.h\n" + "fmla z14.h, p3/M, z8.h, z10.h\n" + "ld1h { z9.h }, p2/Z, [x13]\n" + "fmla z31.h, p3/M, z1.h, z12.h\n" + "fmla z13.h, p3/M, z0.h, z12.h\n" "ld1h { z12.h }, p2/Z, [x13, x27, LSL #1]\n" - "fmla z18.h, p3/M, z2.h, z11.h\n" - "fmla z19.h, p3/M, z1.h, z11.h\n" + "fmla z17.h, p3/M, z2.h, z11.h\n" + "fmla z30.h, p3/M, z1.h, z11.h\n" "ld1h { z11.h }, p2/Z, [x28]\n" - "fmla z22.h, p3/M, z7.h, z10.h\n" - "fmla z23.h, p3/M, z6.h, z10.h\n" - "fmla z25.h, p3/M, z5.h, z10.h\n" - "fmla z26.h, p3/M, z4.h, z10.h\n" - "fmla z27.h, p3/M, z3.h, z10.h\n" - "fmla z31.h, p3/M, z0.h, z10.h\n" - "fmla z24.h, p3/M, z6.h, z11.h\n" - "fmla z28.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x28, x27, LSL #1]\n" - "fmla z19.h, p3/M, z5.h, z12.h\n" - "fmla z23.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x13, x14, LSL #1]\n" - "fmla z27.h, p3/M, z8.h, z11.h\n" - "fmla z31.h, p3/M, z5.h, z11.h\n" - "movprfx z29, z15\n fmla z29.h, p3/M, z1.h, z9.h\n" - "movprfx z30, z15\n fmla z30.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x13]\n" - "fmla z29.h, p3/M, z2.h, z10.h\n" - "fmla z30.h, p3/M, z1.h, z10.h\n" + "fmla z21.h, p3/M, z7.h, z10.h\n" + "fmla z26.h, p3/M, z6.h, z10.h\n" + "fmla z22.h, p3/M, z5.h, z10.h\n" + "fmla z20.h, p3/M, z4.h, z10.h\n" + "fmla z28.h, p3/M, z3.h, z10.h\n" + "fmla z25.h, p3/M, z2.h, z10.h\n" + "fmla z29.h, p3/M, z1.h, z10.h\n" + "fmla z24.h, p3/M, z0.h, z10.h\n" "ld1h { z10.h }, p2/Z, [x13, x7, LSL #1]\n" - "fmla z20.h, p3/M, z0.h, z9.h\n" - "fmla z21.h, p3/M, z1.h, z10.h\n" - "fmla z16.h, p3/M, z3.h, z9.h\n" - "fmla z17.h, p3/M, z4.h, z10.h\n" - "ld1h { z11.h }, p2/Z, [x24, x5, LSL #1]\n" - "fmla z18.h, p3/M, z3.h, z10.h\n" - "fmla z22.h, p3/M, z0.h, z10.h\n" + "fmla z27.h, p3/M, z0.h, z9.h\n" + "fmla z18.h, p3/M, z6.h, z11.h\n" + "fmla z23.h, p3/M, z3.h, z11.h\n" + "fmla z14.h, p3/M, z1.h, z10.h\n" + "ld1h { z11.h }, p2/Z, [x28, x27, LSL #1]\n" + "fmla z31.h, p3/M, z3.h, z9.h\n" + "fmla z30.h, p3/M, z5.h, z12.h\n" + "fmla z26.h, p3/M, z2.h, z12.h\n" + "fmla z13.h, p3/M, z4.h, z10.h\n" + "ld1h { z9.h }, p2/Z, [x13, x14, LSL #1]\n" + "fmla z17.h, p3/M, z3.h, z10.h\n" + "fmla z21.h, p3/M, z0.h, z10.h\n" + "fmla z28.h, p3/M, z8.h, z11.h\n" + "fmla z24.h, p3/M, z5.h, z11.h\n" + "ld1h { z12.h }, p2/Z, [x24, x5, LSL #1]\n" + "fmla z27.h, p3/M, z2.h, z10.h\n" + "fmla z14.h, p3/M, z2.h, z9.h\n" + "fmla z31.h, p3/M, z5.h, z10.h\n" + "fmla z13.h, p3/M, z5.h, z9.h\n" + "ld1h { z11.h }, p2/Z, [x12, x5, LSL #1]\n" + "fmla z17.h, p3/M, z4.h, z9.h\n" + "fmla z30.h, p3/M, z3.h, z9.h\n" + "fmla z21.h, p3/M, z1.h, z9.h\n" + "fmla z26.h, p3/M, z0.h, z9.h\n" + "ld1h { z10.h }, p2/Z, [x12, x10, LSL #1]\n" + "fmla z23.h, p3/M, z7.h, z12.h\n" + "fmla z25.h, p3/M, z6.h, z12.h\n" + "ld1h { z12.h }, p2/Z, [x24, x10, LSL #1]\n" + "fmla z27.h, p3/M, z4.h, z11.h\n" + "fmla z14.h, p3/M, z3.h, z11.h\n" + "fmla z18.h, p3/M, z1.h, z11.h\n" + "fmla z22.h, p3/M, z0.h, z11.h\n" + "fmla z31.h, p3/M, z7.h, z11.h\n" + "fmla z13.h, p3/M, z6.h, z11.h\n" + "ld1h { z9.h }, p2/Z, [x8, x7, LSL #1]\n" + "fmla z29.h, p3/M, z8.h, z12.h\n" + "fmla z24.h, p3/M, z7.h, z12.h\n" + "ld1h { z12.h }, p2/Z, [x11, x5, LSL #1]\n" + "fmla z17.h, p3/M, z8.h, z10.h\n" + "fmla z30.h, p3/M, z7.h, z10.h\n" + "fmla z21.h, p3/M, z5.h, z10.h\n" + "fmla z26.h, p3/M, z4.h, z10.h\n" "fmla z20.h, p3/M, z2.h, z10.h\n" - "fmla z21.h, p3/M, z2.h, z12.h\n" - "fmla z16.h, p3/M, z5.h, z10.h\n" - "fmla z17.h, p3/M, z5.h, z12.h\n" - "ld1h { z10.h }, p2/Z, [x12, x5, LSL #1]\n" + "fmla z28.h, p3/M, z1.h, z10.h\n" + "ld1h { z11.h }, p2/Z, [x8, x14, LSL #1]\n" + "addvl x8, x8, #1\n" + "fmla z27.h, p3/M, z7.h, z12.h\n" + "fmla z14.h, p3/M, z6.h, z12.h\n" "fmla z18.h, p3/M, z4.h, z12.h\n" - "fmla z19.h, p3/M, z3.h, z12.h\n" - "fmla z22.h, p3/M, z1.h, z12.h\n" - "fmla z23.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x12, x10, LSL #1]\n" - "fmla z28.h, p3/M, z7.h, z11.h\n" - "fmla z29.h, p3/M, z6.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x24, x10, LSL #1]\n" - "fmla z20.h, p3/M, z4.h, z10.h\n" - "fmla z21.h, p3/M, z3.h, z10.h\n" - "fmla z24.h, p3/M, z1.h, z10.h\n" - "fmla z25.h, p3/M, z0.h, z10.h\n" - "fmla z16.h, p3/M, z7.h, z10.h\n" - "fmla z17.h, p3/M, z6.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x8, x7, LSL #1]\n" + "fmla z22.h, p3/M, z3.h, z12.h\n" + "fmla z23.h, p3/M, z1.h, z12.h\n" + "fmla z25.h, p3/M, z0.h, z12.h\n" + "ld1h { z12.h }, p2/Z, [x11, x10, LSL #1]\n" + "fmla z31.h, p3/M, z2.h, z9.h\n" + "fmla z13.h, p3/M, z1.h, z9.h\n" + "fmla z17.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x12]\n" + "fmla z29.h, p3/M, z2.h, z12.h\n" + "fmla z30.h, p3/M, z0.h, z11.h\n" + "fmla z27.h, p3/M, z3.h, z9.h\n" + "fmla z18.h, p3/M, z0.h, z9.h\n" + "fmla z21.h, p3/M, z8.h, z12.h\n" + "fmla z26.h, p3/M, z7.h, z12.h\n" + "fmla z20.h, p3/M, z5.h, z12.h\n" + "fmla z28.h, p3/M, z4.h, z12.h\n" + "fmla z24.h, p3/M, z1.h, z12.h\n" + "ld1h { z10.h }, p2/Z, [x28, x7, LSL #1]\n" + "fmla z13.h, p3/M, z2.h, z11.h\n" + "fmla z17.h, p3/M, z1.h, z11.h\n" + "ld1h { z11.h }, p2/Z, [x12, x27, LSL #1]\n" + "addvl x12, x12, #1\n" + "fmla z31.h, p3/M, z6.h, z9.h\n" + "ld1h { z12.h }, p2/Z, [x11]\n" + "fmla z25.h, p3/M, z4.h, z10.h\n" + "ld1h { z9.h }, p1/Z, [x12, x7, LSL #1]\n" + "fmla z29.h, p3/M, z3.h, z10.h\n" "fmla z30.h, p3/M, z8.h, z11.h\n" - "fmla z31.h, p3/M, z7.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x5, LSL #1]\n" - "fmla z18.h, p3/M, z8.h, z12.h\n" - "fmla z19.h, p3/M, z7.h, z12.h\n" - "fmla z22.h, p3/M, z5.h, z12.h\n" - "fmla z23.h, p3/M, z4.h, z12.h\n" - "fmla z26.h, p3/M, z2.h, z12.h\n" - "fmla z27.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x8, x14, LSL #1]\n" - "addvl x8, x8, #1\n" - "fmla z20.h, p3/M, z7.h, z11.h\n" - "fmla z21.h, p3/M, z6.h, z11.h\n" - "fmla z24.h, p3/M, z4.h, z11.h\n" - "fmla z25.h, p3/M, z3.h, z11.h\n" - "fmla z28.h, p3/M, z1.h, z11.h\n" - "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x10, LSL #1]\n" - "fmla z16.h, p3/M, z2.h, z10.h\n" - "fmla z17.h, p3/M, z1.h, z10.h\n" - "fmla z18.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x12]\n" - "fmla z30.h, p3/M, z2.h, z11.h\n" - "fmla z19.h, p3/M, z0.h, z12.h\n" - "fmla z20.h, p3/M, z3.h, z10.h\n" - "fmla z24.h, p3/M, z0.h, z10.h\n" - "fmla z22.h, p3/M, z8.h, z11.h\n" - "fmla z23.h, p3/M, z7.h, z11.h\n" "fmla z26.h, p3/M, z5.h, z11.h\n" - "fmla z27.h, p3/M, z4.h, z11.h\n" - "fmla z31.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x28, x7, LSL #1]\n" - "fmla z17.h, p3/M, z2.h, z12.h\n" - "fmla z18.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x12, x27, LSL #1]\n" - "addvl x12, x12, #1\n" - "fmla z16.h, p3/M, z6.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x11]\n" - "fmla z29.h, p3/M, z4.h, z11.h\n" - "fmla z30.h, p3/M, z3.h, z11.h\n" - "fmla z19.h, p3/M, z8.h, z12.h\n" - "fmla z23.h, p3/M, z5.h, z12.h\n" - "fmla z27.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x11, x27, LSL #1]\n" + "fmla z28.h, p3/M, z2.h, z11.h\n" + "ld1h { z11.h }, p2/Z, [x11, x27, LSL #1]\n" "addvl x11, x11, #1\n" + "fmla z27.h, p3/M, z6.h, z12.h\n" + "fmla z18.h, p3/M, z3.h, z12.h\n" + "fmla z23.h, p3/M, z0.h, z12.h\n" + "ld1h { z12.h }, p2/Z, [x24, x7, LSL #1]\n" + "fmla z24.h, p3/M, z2.h, z11.h\n" + "fmla z25.h, p3/M, z7.h, z12.h\n" + "fmla z29.h, p3/M, z6.h, z12.h\n" + "fmla z18.h, p3/M, z8.h, z10.h\n" + "fmla z22.h, p3/M, z7.h, z10.h\n" "fmla z20.h, p3/M, z6.h, z10.h\n" - "fmla z24.h, p3/M, z3.h, z10.h\n" - "fmla z28.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x24, x7, LSL #1]\n" - "fmla z31.h, p3/M, z2.h, z12.h\n" - "fmla z29.h, p3/M, z7.h, z10.h\n" - "fmla z30.h, p3/M, z6.h, z10.h\n" - "fmla z24.h, p3/M, z8.h, z11.h\n" - "fmla z25.h, p3/M, z7.h, z11.h\n" - "fmla z26.h, p3/M, z6.h, z11.h\n" + "fmla z23.h, p3/M, z5.h, z10.h\n" + "ld1h { z10.h }, p2/Z, [x28, x14, LSL #1]\n" "fmla z28.h, p3/M, z5.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x28, x14, LSL #1]\n" - "fmla z27.h, p3/M, z5.h, z12.h\n" - "fmla z29.h, p3/M, z5.h, z11.h\n" - "fmla z30.h, p3/M, z4.h, z11.h\n" - "fmla z31.h, p3/M, z3.h, z11.h\n" + "fmla z25.h, p3/M, z5.h, z10.h\n" + "fmla z29.h, p3/M, z4.h, z10.h\n" + "fmla z24.h, p3/M, z3.h, z10.h\n" + "fmla z26.h, p3/M, z8.h, z11.h\n" + "ld1h { z11.h }, p2/Z, [x24, x14, LSL #1]\n" "fmla z23.h, p3/M, z8.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x24, x14, LSL #1]\n" - "fmla z28.h, p3/M, z8.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x13, x5, LSL #1]\n" - "fmla z25.h, p3/M, z8.h, z11.h\n" - "fmla z26.h, p3/M, z7.h, z11.h\n" + "ld1h { z12.h }, p2/Z, [x13, x5, LSL #1]\n" + "fmla z22.h, p3/M, z8.h, z10.h\n" + "fmla z20.h, p3/M, z7.h, z10.h\n" "addvl x24, x24, #1\n" - "fmla z27.h, p3/M, z6.h, z11.h\n" - "fmla z29.h, p3/M, z8.h, z12.h\n" - "ld1h { z11.h }, p2/Z, [x13, x10, LSL #1]\n" + "fmla z28.h, p3/M, z6.h, z10.h\n" + "fmla z25.h, p3/M, z8.h, z11.h\n" + "ld1h { z10.h }, p2/Z, [x13, x10, LSL #1]\n" "addvl x13, x13, #1\n" - "fmla z30.h, p3/M, z7.h, z12.h\n" - "fmla z31.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x28, x5, LSL #1]\n" - "fmla z16.h, p3/M, z4.h, z10.h\n" - "fmla z17.h, p3/M, z3.h, z10.h\n" - "fmax z16.h, p3/M, z16.h, z14.h\n" - "fmax z17.h, p3/M, z17.h, z14.h\n" - "fmla z20.h, p3/M, z1.h, z10.h\n" - "fmla z21.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x28, x10, LSL #1]\n" - "fmax z20.h, p3/M, z20.h, z14.h\n" - "fmla z18.h, p3/M, z5.h, z11.h\n" - "fmla z19.h, p3/M, z4.h, z11.h\n" - "fmax z18.h, p3/M, z18.h, z14.h\n" - "fmax z19.h, p3/M, z19.h, z14.h\n" - "fmla z22.h, p3/M, z2.h, z11.h\n" - "fmla z23.h, p3/M, z1.h, z11.h\n" - "fmax z21.h, p3/M, z21.h, z14.h\n" - "fmax z22.h, p3/M, z22.h, z14.h\n" - "fmla z24.h, p3/M, z7.h, z12.h\n" - "fmla z25.h, p3/M, z6.h, z12.h\n" - "fmax z23.h, p3/M, z23.h, z14.h\n" - "fmax z24.h, p3/M, z24.h, z14.h\n" - "fmla z28.h, p3/M, z4.h, z12.h\n" - "fmla z29.h, p3/M, z3.h, z12.h\n" - "fmax z25.h, p3/M, z25.h, z14.h\n" - "fmax z28.h, p3/M, z28.h, z14.h\n" - "fmla z26.h, p3/M, z8.h, z10.h\n" - "fmla z27.h, p3/M, z7.h, z10.h\n" - "fmax z26.h, p3/M, z26.h, z14.h\n" - "fmax z27.h, p3/M, z27.h, z14.h\n" - "fmla z30.h, p3/M, z5.h, z10.h\n" - "fmla z31.h, p3/M, z4.h, z10.h\n" - "fmax z29.h, p3/M, z29.h, z14.h\n" - "fmax z30.h, p3/M, z30.h, z14.h\n" - "fmax z31.h, p3/M, z31.h, z14.h\n" - "ld1h { z15.h }, p3/Z, [x17]\n" + "fmla z29.h, p3/M, z7.h, z11.h\n" + "fmla z24.h, p3/M, z6.h, z11.h\n" + "ld1h { z11.h }, p2/Z, [x28, x5, LSL #1]\n" + "fmla z31.h, p3/M, z4.h, z12.h\n" + "fmla z13.h, p3/M, z3.h, z12.h\n" + "fmax z31.h, p3/M, z31.h, z15.h\n" + "fmax z13.h, p3/M, z13.h, z15.h\n" + "fmla z27.h, p3/M, z1.h, z12.h\n" + "fmla z14.h, p3/M, z0.h, z12.h\n" + "ld1h { z0.h }, p2/Z, [x28, x10, LSL #1]\n" + "fmax z27.h, p3/M, z27.h, z15.h\n" + "fmla z17.h, p3/M, z5.h, z10.h\n" + "fmla z30.h, p3/M, z4.h, z10.h\n" + "fmax z17.h, p3/M, z17.h, z15.h\n" + "fmax z30.h, p3/M, z30.h, z15.h\n" + "fmla z21.h, p3/M, z2.h, z10.h\n" + "fmla z26.h, p3/M, z1.h, z10.h\n" + "fmax z14.h, p3/M, z14.h, z15.h\n" + "fmax z21.h, p3/M, z21.h, z15.h\n" + "fmla z18.h, p3/M, z7.h, z11.h\n" + "fmla z22.h, p3/M, z6.h, z11.h\n" + "fmax z26.h, p3/M, z26.h, z15.h\n" + "fmax z18.h, p3/M, z18.h, z15.h\n" + "fmla z23.h, p3/M, z4.h, z11.h\n" + "fmla z25.h, p3/M, z3.h, z11.h\n" + "fmax z22.h, p3/M, z22.h, z15.h\n" + "fmax z23.h, p3/M, z23.h, z15.h\n" + "fmla z20.h, p3/M, z8.h, z0.h\n" + "fmla z28.h, p3/M, z7.h, z0.h\n" + "fmax z20.h, p3/M, z20.h, z15.h\n" + "fmax z28.h, p3/M, z28.h, z15.h\n" + "fmla z29.h, p3/M, z5.h, z0.h\n" + "fmla z24.h, p3/M, z4.h, z0.h\n" + "fmax z25.h, p3/M, z25.h, z15.h\n" + "fmax z29.h, p3/M, z29.h, z15.h\n" + "fmax z24.h, p3/M, z24.h, z15.h\n" "ld1h { z0.h }, p3/Z, [x17, #1, MUL VL]\n" - "whilelt p2.h, x21, %x[n_channels]\n" "ld1h { z1.h }, p3/Z, [x17, #2, MUL VL]\n" + "whilelt p2.h, x21, %x[n_channels]\n" "ld1h { z2.h }, p3/Z, [x17, #3, MUL VL]\n" - "cmp x16, %x[n_channels]\n" - "fmin z16.h, p3/M, z16.h, z13.h\n" "ld1h { z3.h }, p3/Z, [x17, #4, MUL VL]\n" + "cmp x16, %x[n_channels]\n" + "fmin z31.h, p3/M, z31.h, z16.h\n" "ld1h { z4.h }, p3/Z, [x17, #5, MUL VL]\n" - "fmin z17.h, p3/M, z17.h, z13.h\n" - "fmin z18.h, p3/M, z18.h, z13.h\n" "ld1h { z5.h }, p3/Z, [x17, #6, MUL VL]\n" + "fmin z13.h, p3/M, z13.h, z16.h\n" + "fmin z17.h, p3/M, z17.h, z16.h\n" "ld1h { z6.h }, p3/Z, [x17, #7, MUL VL]\n" "addvl x17, x17, #16\n" - "fmin z19.h, p3/M, z19.h, z13.h\n" - "fmin z20.h, p3/M, z20.h, z13.h\n" - "fmin z21.h, p3/M, z21.h, z13.h\n" - "ld1h { z9.h }, p1/Z, [x12, x7, LSL #1]\n" + "fmin z30.h, p3/M, z30.h, z16.h\n" "ld1h { z10.h }, p1/Z, [x8]\n" - "fmin z22.h, p3/M, z22.h, z13.h\n" - "fmin z23.h, p3/M, z23.h, z13.h\n" + "fmin z27.h, p3/M, z27.h, z16.h\n" + "fmin z14.h, p3/M, z14.h, z16.h\n" "ld1h { z11.h }, p1/Z, [x8, x27, LSL #1]\n" "ld1h { z12.h }, p1/Z, [x12, x14, LSL #1]\n" - "fmin z24.h, p3/M, z24.h, z13.h\n" - "fmin z25.h, p3/M, z25.h, z13.h\n" - "st1h { z16.h }, p0, [x15]\n" + "fmin z21.h, p3/M, z21.h, z16.h\n" + "fmin z26.h, p3/M, z26.h, z16.h\n" + "st1h { z31.h }, p0, [x15]\n" "ld1h { z7.h }, p3/Z, [x17, #-8, MUL VL]\n" - "fmin z26.h, p3/M, z26.h, z13.h\n" - "fmin z27.h, p3/M, z27.h, z13.h\n" - "st1h { z17.h }, p0, [x15, x6, LSL #1]\n" + "fmin z18.h, p3/M, z18.h, z16.h\n" + "fmin z22.h, p3/M, z22.h, z16.h\n" + "st1h { z13.h }, p0, [x15, x6, LSL #1]\n" "ld1h { z8.h }, p3/Z, [x17, #-7, MUL VL]\n" - "fmin z28.h, p3/M, z28.h, z13.h\n" - "fmin z29.h, p3/M, z29.h, z13.h\n" - "st1h { z18.h }, p0, [x15, x25, LSL #1]\n" - "fmin z30.h, p3/M, z30.h, z13.h\n" - "fmin z31.h, p3/M, z31.h, z13.h\n" - "st1h { z19.h }, p0, [x15, x22, LSL #1]\n" + "fmin z20.h, p3/M, z20.h, z16.h\n" + "fmin z28.h, p3/M, z28.h, z16.h\n" + "st1h { z17.h }, p0, [x15, x25, LSL #1]\n" + "fmin z23.h, p3/M, z23.h, z16.h\n" + "fmin z25.h, p3/M, z25.h, z16.h\n" + "st1h { z30.h }, p0, [x15, x22, LSL #1]\n" + "fmin z29.h, p3/M, z29.h, z16.h\n" + "fmin z24.h, p3/M, z24.h, z16.h\n" + "st1h { z27.h }, p0, [x9]\n" "addvl x28, x28, #1\n" - "st1h { z20.h }, p0, [x9]\n" + "st1h { z14.h }, p0, [x9, x6, LSL #1]\n" "addvl x15, x15, #1\n" - "st1h { z21.h }, p0, [x9, x6, LSL #1]\n" + "st1h { z21.h }, p0, [x9, x25, LSL #1]\n" "addvl x17, x17, #-6\n" - "st1h { z22.h }, p0, [x9, x25, LSL #1]\n" - "st1h { z23.h }, p0, [x9, x22, LSL #1]\n" + "st1h { z26.h }, p0, [x9, x22, LSL #1]\n" "addvl x9, x9, #1\n" - "st1h { z24.h }, p0, [x26]\n" - "st1h { z25.h }, p0, [x26, x6, LSL #1]\n" - "st1h { z26.h }, p0, [x26, x25, LSL #1]\n" - "st1h { z27.h }, p0, [x26, x22, LSL #1]\n" + "st1h { z18.h }, p0, [x26]\n" + "st1h { z22.h }, p0, [x26, x6, LSL #1]\n" + "st1h { z20.h }, p0, [x26, x25, LSL #1]\n" + "st1h { z28.h }, p0, [x26, x22, LSL #1]\n" "addvl x26, x26, #1\n" - "st1h { z28.h }, p0, [x23]\n" - "st1h { z29.h }, p0, [x23, x6, LSL #1]\n" - "st1h { z30.h }, p0, [x23, x25, LSL #1]\n" - "st1h { z31.h }, p0, [x23, x22, LSL #1]\n" + "st1h { z23.h }, p0, [x23]\n" + "st1h { z25.h }, p0, [x23, x6, LSL #1]\n" + "st1h { z29.h }, p0, [x23, x25, LSL #1]\n" + "st1h { z24.h }, p0, [x23, x22, LSL #1]\n" "addvl x23, x23, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "movprfx z21, z15\n fmla z21.h, p3/M, z4.h, z9.h\n" - "movprfx z16, z15\n fmla z16.h, p3/M, z8.h, z9.h\n" + "movprfx z14, z19\n fmla z14.h, p3/M, z4.h, z9.h\n" + "movprfx z31, z19\n fmla z31.h, p3/M, z8.h, z9.h\n" "ldr x4, [%x[params_struct], %[offsetof_args_tile_j]]\n" "ldr x16, [%x[params_struct], %[offsetof_args_tile_i]]\n" - "movprfx z22, z15\n fmla z22.h, p3/M, z3.h, z9.h\n" - "movprfx z25, z15\n fmla z25.h, p3/M, z1.h, z9.h\n" + "movprfx z30, z19\n fmla z30.h, p3/M, z3.h, z9.h\n" + "movprfx z13, z19\n fmla z13.h, p3/M, z1.h, z9.h\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" "add x4, x4, #0x1\n" - "movprfx z26, z15\n fmla z26.h, p3/M, z0.h, z9.h\n" - "fmla z21.h, p3/M, z5.h, z12.h\n" + "movprfx z20, z19\n fmla z20.h, p3/M, z0.h, z9.h\n" + "fmla z14.h, p3/M, z5.h, z12.h\n" "cmp x4, x20\n" "add x21, x16, #0x1\n" - "movprfx z17, z15\n fmla z17.h, p3/M, z7.h, z9.h\n" - "movprfx z18, z15\n fmla z18.h, p3/M, z6.h, z9.h\n" + "movprfx z18, z19\n fmla z18.h, p3/M, z7.h, z9.h\n" + "movprfx z28, z19\n fmla z28.h, p3/M, z6.h, z9.h\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" "csel x16, x16, x21, LT\n" - "movprfx z20, z15\n fmla z20.h, p3/M, z5.h, z9.h\n" - "movprfx z24, z15\n fmla z24.h, p3/M, z2.h, z9.h\n" + "movprfx z17, z19\n fmla z17.h, p3/M, z5.h, z9.h\n" + "movprfx z26, z19\n fmla z26.h, p3/M, z2.h, z9.h\n" "ld1h { z9.h }, p2/Z, [x11, x7, LSL #1]\n" "mov p0.b, p2.b\n" - "fmla z16.h, p3/M, z0.h, z10.h\n" - "movprfx z19, z15\n fmla z19.h, p3/M, z2.h, z11.h\n" - "ld1h { z10.h }, p2/Z, [x24]\n" - "ld1h { z11.h }, p2/Z, [x24, x27, LSL #1]\n" - "fmla z22.h, p3/M, z4.h, z12.h\n" - "fmla z25.h, p3/M, z2.h, z12.h\n" + "fmla z31.h, p3/M, z0.h, z10.h\n" + "movprfx z27, z19\n fmla z27.h, p3/M, z2.h, z11.h\n" + "ld1h { z29.h }, p2/Z, [x24]\n" + "ld1h { z21.h }, p2/Z, [x24, x27, LSL #1]\n" + "fmla z30.h, p3/M, z4.h, z12.h\n" + "fmla z13.h, p3/M, z2.h, z12.h\n" "csel x4, x4, XZR, LT\n" "cmp x16, x20\n" - "fmla z26.h, p3/M, z1.h, z12.h\n" - "movprfx z28, z15\n fmla z28.h, p3/M, z6.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x11, x14, LSL #1]\n" - "fmla z21.h, p3/M, z7.h, z9.h\n" - "fmla z17.h, p3/M, z8.h, z12.h\n" - "fmla z18.h, p3/M, z7.h, z12.h\n" - "fmla z19.h, p3/M, z6.h, z12.h\n" - "movprfx z23, z15\n fmla z23.h, p3/M, z3.h, z12.h\n" - "movprfx z27, z15\n fmla z27.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x8, x5, LSL #1]\n" - "movprfx z31, z15\n fmla z31.h, p3/M, z8.h, z11.h\n" - "fmla z22.h, p3/M, z6.h, z9.h\n" - "ld1h { z11.h }, p2/Z, [x8, x10, LSL #1]\n" - "fmla z25.h, p3/M, z4.h, z9.h\n" - "fmla z26.h, p3/M, z3.h, z9.h\n" - "fmla z20.h, p3/M, z8.h, z9.h\n" - "fmla z24.h, p3/M, z5.h, z9.h\n" - "fmla z28.h, p3/M, z2.h, z9.h\n" - "fmla z21.h, p3/M, z8.h, z10.h\n" - "fmla z16.h, p3/M, z1.h, z12.h\n" - "fmla z17.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x13, x27, LSL #1]\n" - "fmla z18.h, p3/M, z2.h, z11.h\n" - "fmla z19.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x28]\n" - "fmla z22.h, p3/M, z7.h, z10.h\n" - "fmla z23.h, p3/M, z6.h, z10.h\n" - "fmla z25.h, p3/M, z5.h, z10.h\n" - "fmla z26.h, p3/M, z4.h, z10.h\n" - "fmla z27.h, p3/M, z3.h, z10.h\n" - "fmla z31.h, p3/M, z0.h, z10.h\n" - "fmla z24.h, p3/M, z6.h, z11.h\n" - "fmla z28.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x28, x27, LSL #1]\n" - "fmla z19.h, p3/M, z5.h, z12.h\n" - "fmla z23.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x13, x14, LSL #1]\n" - "fmla z27.h, p3/M, z8.h, z11.h\n" - "fmla z31.h, p3/M, z5.h, z11.h\n" - "movprfx z29, z15\n fmla z29.h, p3/M, z1.h, z9.h\n" - "movprfx z30, z15\n fmla z30.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x13]\n" - "fmla z29.h, p3/M, z2.h, z10.h\n" - "fmla z30.h, p3/M, z1.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x13, x7, LSL #1]\n" - "fmla z20.h, p3/M, z0.h, z9.h\n" - "fmla z21.h, p3/M, z1.h, z10.h\n" - "fmla z16.h, p3/M, z3.h, z9.h\n" - "fmla z17.h, p3/M, z4.h, z10.h\n" - "ld1h { z11.h }, p2/Z, [x24, x5, LSL #1]\n" - "fmla z18.h, p3/M, z3.h, z10.h\n" - "fmla z22.h, p3/M, z0.h, z10.h\n" - "fmla z20.h, p3/M, z2.h, z10.h\n" - "fmla z21.h, p3/M, z2.h, z12.h\n" - "fmla z16.h, p3/M, z5.h, z10.h\n" - "fmla z17.h, p3/M, z5.h, z12.h\n" - "ld1h { z10.h }, p2/Z, [x12, x5, LSL #1]\n" - "fmla z18.h, p3/M, z4.h, z12.h\n" - "fmla z19.h, p3/M, z3.h, z12.h\n" - "fmla z22.h, p3/M, z1.h, z12.h\n" - "fmla z23.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x12, x10, LSL #1]\n" - "fmla z28.h, p3/M, z7.h, z11.h\n" - "fmla z29.h, p3/M, z6.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x24, x10, LSL #1]\n" - "fmla z20.h, p3/M, z4.h, z10.h\n" - "fmla z21.h, p3/M, z3.h, z10.h\n" - "fmla z24.h, p3/M, z1.h, z10.h\n" - "fmla z25.h, p3/M, z0.h, z10.h\n" - "fmla z16.h, p3/M, z7.h, z10.h\n" - "fmla z17.h, p3/M, z6.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x8, x7, LSL #1]\n" - "fmla z30.h, p3/M, z8.h, z11.h\n" - "fmla z31.h, p3/M, z7.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x5, LSL #1]\n" + "fmla z20.h, p3/M, z1.h, z12.h\n" + "movprfx z10, z19\n fmla z10.h, p3/M, z6.h, z29.h\n" + "ld1h { z29.h }, p2/Z, [x11, x14, LSL #1]\n" + "fmla z14.h, p3/M, z7.h, z9.h\n" "fmla z18.h, p3/M, z8.h, z12.h\n" - "fmla z19.h, p3/M, z7.h, z12.h\n" - "fmla z22.h, p3/M, z5.h, z12.h\n" - "fmla z23.h, p3/M, z4.h, z12.h\n" - "fmla z26.h, p3/M, z2.h, z12.h\n" - "fmla z27.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x8, x14, LSL #1]\n" - "fmla z20.h, p3/M, z7.h, z11.h\n" - "fmla z21.h, p3/M, z6.h, z11.h\n" - "fmla z24.h, p3/M, z4.h, z11.h\n" - "fmla z25.h, p3/M, z3.h, z11.h\n" - "fmla z28.h, p3/M, z1.h, z11.h\n" - "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x10, LSL #1]\n" - "fmla z16.h, p3/M, z2.h, z10.h\n" - "fmla z17.h, p3/M, z1.h, z10.h\n" - "fmla z18.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x12]\n" - "fmla z30.h, p3/M, z2.h, z11.h\n" - "fmla z19.h, p3/M, z0.h, z12.h\n" - "fmla z20.h, p3/M, z3.h, z10.h\n" - "fmla z24.h, p3/M, z0.h, z10.h\n" - "fmla z22.h, p3/M, z8.h, z11.h\n" - "fmla z23.h, p3/M, z7.h, z11.h\n" - "fmla z26.h, p3/M, z5.h, z11.h\n" - "fmla z27.h, p3/M, z4.h, z11.h\n" - "fmla z31.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x28, x7, LSL #1]\n" - "fmla z17.h, p3/M, z2.h, z12.h\n" - "fmla z18.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x12, x27, LSL #1]\n" - "fmla z16.h, p3/M, z6.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x11]\n" - "fmla z29.h, p3/M, z4.h, z11.h\n" - "fmla z30.h, p3/M, z3.h, z11.h\n" - "fmla z19.h, p3/M, z8.h, z12.h\n" - "fmla z23.h, p3/M, z5.h, z12.h\n" - "fmla z27.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x11, x27, LSL #1]\n" - "fmla z20.h, p3/M, z6.h, z10.h\n" - "fmla z24.h, p3/M, z3.h, z10.h\n" - "fmla z28.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x24, x7, LSL #1]\n" - "fmla z31.h, p3/M, z2.h, z12.h\n" - "fmla z29.h, p3/M, z7.h, z10.h\n" - "fmla z30.h, p3/M, z6.h, z10.h\n" - "fmla z24.h, p3/M, z8.h, z11.h\n" - "fmla z25.h, p3/M, z7.h, z11.h\n" - "fmla z26.h, p3/M, z6.h, z11.h\n" - "fmla z28.h, p3/M, z5.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x28, x14, LSL #1]\n" - "fmla z27.h, p3/M, z5.h, z12.h\n" - "fmla z29.h, p3/M, z5.h, z11.h\n" - "fmla z30.h, p3/M, z4.h, z11.h\n" - "fmla z31.h, p3/M, z3.h, z11.h\n" - "fmla z23.h, p3/M, z8.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x24, x14, LSL #1]\n" - "fmla z28.h, p3/M, z8.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x13, x5, LSL #1]\n" - "fmla z25.h, p3/M, z8.h, z11.h\n" - "fmla z26.h, p3/M, z7.h, z11.h\n" - "fmla z27.h, p3/M, z6.h, z11.h\n" - "fmla z29.h, p3/M, z8.h, z12.h\n" - "ld1h { z11.h }, p2/Z, [x13, x10, LSL #1]\n" - "fmla z30.h, p3/M, z7.h, z12.h\n" - "fmla z31.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x28, x5, LSL #1]\n" - "fmla z16.h, p3/M, z4.h, z10.h\n" - "fmla z17.h, p3/M, z3.h, z10.h\n" - "fmax z16.h, p3/M, z16.h, z14.h\n" - "fmax z17.h, p3/M, z17.h, z14.h\n" - "fmla z20.h, p3/M, z1.h, z10.h\n" - "fmla z21.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x28, x10, LSL #1]\n" - "fmax z20.h, p3/M, z20.h, z14.h\n" - "fmla z18.h, p3/M, z5.h, z11.h\n" - "fmla z19.h, p3/M, z4.h, z11.h\n" - "fmax z18.h, p3/M, z18.h, z14.h\n" - "fmax z19.h, p3/M, z19.h, z14.h\n" - "fmla z22.h, p3/M, z2.h, z11.h\n" - "fmla z23.h, p3/M, z1.h, z11.h\n" - "fmax z21.h, p3/M, z21.h, z14.h\n" - "fmax z22.h, p3/M, z22.h, z14.h\n" - "fmla z24.h, p3/M, z7.h, z12.h\n" - "fmla z25.h, p3/M, z6.h, z12.h\n" - "fmax z23.h, p3/M, z23.h, z14.h\n" - "fmax z24.h, p3/M, z24.h, z14.h\n" - "fmla z28.h, p3/M, z4.h, z12.h\n" - "fmla z29.h, p3/M, z3.h, z12.h\n" - "fmax z25.h, p3/M, z25.h, z14.h\n" - "fmax z28.h, p3/M, z28.h, z14.h\n" - "fmla z26.h, p3/M, z8.h, z10.h\n" - "fmla z27.h, p3/M, z7.h, z10.h\n" - "fmax z26.h, p3/M, z26.h, z14.h\n" - "fmax z27.h, p3/M, z27.h, z14.h\n" - "fmla z30.h, p3/M, z5.h, z10.h\n" - "fmla z31.h, p3/M, z4.h, z10.h\n" - "fmax z29.h, p3/M, z29.h, z14.h\n" - "fmax z30.h, p3/M, z30.h, z14.h\n" - "fmax z31.h, p3/M, z31.h, z14.h\n" - "fmin z16.h, p3/M, z16.h, z13.h\n" - "st1h { z16.h }, p0, [x15]\n" - "fmin z17.h, p3/M, z17.h, z13.h\n" - "fmin z18.h, p3/M, z18.h, z13.h\n" - "st1h { z17.h }, p0, [x15, x6, LSL #1]\n" - "fmin z19.h, p3/M, z19.h, z13.h\n" - "fmin z20.h, p3/M, z20.h, z13.h\n" - "st1h { z18.h }, p0, [x15, x25, LSL #1]\n" - "fmin z21.h, p3/M, z21.h, z13.h\n" - "fmin z22.h, p3/M, z22.h, z13.h\n" - "st1h { z19.h }, p0, [x15, x22, LSL #1]\n" - "fmin z23.h, p3/M, z23.h, z13.h\n" - "fmin z24.h, p3/M, z24.h, z13.h\n" - "st1h { z20.h }, p0, [x9]\n" - "fmin z25.h, p3/M, z25.h, z13.h\n" - "fmin z26.h, p3/M, z26.h, z13.h\n" - "st1h { z21.h }, p0, [x9, x6, LSL #1]\n" - "fmin z27.h, p3/M, z27.h, z13.h\n" - "fmin z28.h, p3/M, z28.h, z13.h\n" - "st1h { z22.h }, p0, [x9, x25, LSL #1]\n" - "fmin z29.h, p3/M, z29.h, z13.h\n" - "fmin z30.h, p3/M, z30.h, z13.h\n" - "st1h { z23.h }, p0, [x9, x22, LSL #1]\n" - "fmin z31.h, p3/M, z31.h, z13.h\n" - "st1h { z24.h }, p0, [x26]\n" - "st1h { z25.h }, p0, [x26, x6, LSL #1]\n" - "st1h { z26.h }, p0, [x26, x25, LSL #1]\n" - "st1h { z27.h }, p0, [x26, x22, LSL #1]\n" - "st1h { z28.h }, p0, [x23]\n" - "st1h { z29.h }, p0, [x23, x6, LSL #1]\n" - "st1h { z30.h }, p0, [x23, x25, LSL #1]\n" - "st1h { z31.h }, p0, [x23, x22, LSL #1]\n" + "fmla z28.h, p3/M, z7.h, z12.h\n" + "fmla z27.h, p3/M, z6.h, z12.h\n" + "movprfx z11, z19\n fmla z11.h, p3/M, z3.h, z12.h\n" + "movprfx z25, z19\n fmla z25.h, p3/M, z0.h, z12.h\n" + "ld1h { z22.h }, p2/Z, [x8, x5, LSL #1]\n" + "movprfx z24, z19\n fmla z24.h, p3/M, z8.h, z21.h\n" + "fmla z30.h, p3/M, z6.h, z9.h\n" + "ld1h { z21.h }, p2/Z, [x8, x10, LSL #1]\n" + "fmla z13.h, p3/M, z4.h, z9.h\n" + "fmla z20.h, p3/M, z3.h, z9.h\n" + "movprfx z12, z19\n fmla z12.h, p3/M, z1.h, z9.h\n" + "movprfx z23, z19\n fmla z23.h, p3/M, z0.h, z9.h\n" + "fmla z17.h, p3/M, z8.h, z9.h\n" + "fmla z26.h, p3/M, z5.h, z9.h\n" + "fmla z10.h, p3/M, z2.h, z9.h\n" + "fmla z14.h, p3/M, z8.h, z29.h\n" + "ld1h { z9.h }, p2/Z, [x13]\n" + "fmla z31.h, p3/M, z1.h, z22.h\n" + "fmla z18.h, p3/M, z0.h, z22.h\n" + "ld1h { z22.h }, p2/Z, [x13, x27, LSL #1]\n" + "fmla z28.h, p3/M, z2.h, z21.h\n" + "fmla z27.h, p3/M, z1.h, z21.h\n" + "ld1h { z19.h }, p2/Z, [x28]\n" + "fmla z30.h, p3/M, z7.h, z29.h\n" + "fmla z11.h, p3/M, z6.h, z29.h\n" + "fmla z13.h, p3/M, z5.h, z29.h\n" + "fmla z20.h, p3/M, z4.h, z29.h\n" + "fmla z25.h, p3/M, z3.h, z29.h\n" + "fmla z12.h, p3/M, z2.h, z29.h\n" + "fmla z23.h, p3/M, z1.h, z29.h\n" + "fmla z24.h, p3/M, z0.h, z29.h\n" + "ld1h { z21.h }, p2/Z, [x13, x7, LSL #1]\n" + "fmla z17.h, p3/M, z0.h, z9.h\n" + "fmla z26.h, p3/M, z6.h, z19.h\n" + "fmla z10.h, p3/M, z3.h, z19.h\n" + "fmla z14.h, p3/M, z1.h, z21.h\n" + "ld1h { z19.h }, p2/Z, [x28, x27, LSL #1]\n" + "fmla z31.h, p3/M, z3.h, z9.h\n" + "fmla z27.h, p3/M, z5.h, z22.h\n" + "fmla z11.h, p3/M, z2.h, z22.h\n" + "fmla z18.h, p3/M, z4.h, z21.h\n" + "ld1h { z29.h }, p2/Z, [x13, x14, LSL #1]\n" + "fmla z28.h, p3/M, z3.h, z21.h\n" + "fmla z30.h, p3/M, z0.h, z21.h\n" + "fmla z25.h, p3/M, z8.h, z19.h\n" + "fmla z24.h, p3/M, z5.h, z19.h\n" + "ld1h { z19.h }, p2/Z, [x24, x5, LSL #1]\n" + "fmla z17.h, p3/M, z2.h, z21.h\n" + "fmla z14.h, p3/M, z2.h, z29.h\n" + "fmla z31.h, p3/M, z5.h, z21.h\n" + "fmla z18.h, p3/M, z5.h, z29.h\n" + "ld1h { z22.h }, p2/Z, [x12, x5, LSL #1]\n" + "fmla z28.h, p3/M, z4.h, z29.h\n" + "fmla z27.h, p3/M, z3.h, z29.h\n" + "fmla z30.h, p3/M, z1.h, z29.h\n" + "fmla z11.h, p3/M, z0.h, z29.h\n" + "ld1h { z21.h }, p2/Z, [x12, x10, LSL #1]\n" + "fmla z10.h, p3/M, z7.h, z19.h\n" + "fmla z12.h, p3/M, z6.h, z19.h\n" + "ld1h { z19.h }, p2/Z, [x24, x10, LSL #1]\n" + "fmla z17.h, p3/M, z4.h, z22.h\n" + "fmla z14.h, p3/M, z3.h, z22.h\n" + "fmla z26.h, p3/M, z1.h, z22.h\n" + "fmla z13.h, p3/M, z0.h, z22.h\n" + "fmla z31.h, p3/M, z7.h, z22.h\n" + "fmla z18.h, p3/M, z6.h, z22.h\n" + "ld1h { z29.h }, p2/Z, [x8, x7, LSL #1]\n" + "fmla z23.h, p3/M, z8.h, z19.h\n" + "fmla z24.h, p3/M, z7.h, z19.h\n" + "ld1h { z19.h }, p2/Z, [x11, x5, LSL #1]\n" + "fmla z28.h, p3/M, z8.h, z21.h\n" + "fmla z27.h, p3/M, z7.h, z21.h\n" + "fmla z30.h, p3/M, z5.h, z21.h\n" + "fmla z11.h, p3/M, z4.h, z21.h\n" + "fmla z20.h, p3/M, z2.h, z21.h\n" + "fmla z25.h, p3/M, z1.h, z21.h\n" + "ld1h { z22.h }, p2/Z, [x8, x14, LSL #1]\n" + "fmla z17.h, p3/M, z7.h, z19.h\n" + "fmla z14.h, p3/M, z6.h, z19.h\n" + "fmla z26.h, p3/M, z4.h, z19.h\n" + "fmla z13.h, p3/M, z3.h, z19.h\n" + "fmla z10.h, p3/M, z1.h, z19.h\n" + "fmla z12.h, p3/M, z0.h, z19.h\n" + "ld1h { z21.h }, p2/Z, [x11, x10, LSL #1]\n" + "fmla z31.h, p3/M, z2.h, z29.h\n" + "fmla z18.h, p3/M, z1.h, z29.h\n" + "fmla z28.h, p3/M, z0.h, z29.h\n" + "ld1h { z29.h }, p2/Z, [x12]\n" + "fmla z23.h, p3/M, z2.h, z21.h\n" + "fmla z27.h, p3/M, z0.h, z22.h\n" + "fmla z17.h, p3/M, z3.h, z29.h\n" + "fmla z26.h, p3/M, z0.h, z29.h\n" + "fmla z30.h, p3/M, z8.h, z21.h\n" + "fmla z11.h, p3/M, z7.h, z21.h\n" + "fmla z20.h, p3/M, z5.h, z21.h\n" + "fmla z25.h, p3/M, z4.h, z21.h\n" + "fmla z24.h, p3/M, z1.h, z21.h\n" + "ld1h { z19.h }, p2/Z, [x28, x7, LSL #1]\n" + "fmla z18.h, p3/M, z2.h, z22.h\n" + "fmla z28.h, p3/M, z1.h, z22.h\n" + "ld1h { z21.h }, p2/Z, [x12, x27, LSL #1]\n" + "fmla z31.h, p3/M, z6.h, z29.h\n" + "ld1h { z29.h }, p2/Z, [x11]\n" + "fmla z12.h, p3/M, z4.h, z19.h\n" + "fmla z23.h, p3/M, z3.h, z19.h\n" + "fmla z27.h, p3/M, z8.h, z21.h\n" + "fmla z11.h, p3/M, z5.h, z21.h\n" + "fmla z25.h, p3/M, z2.h, z21.h\n" + "ld1h { z9.h }, p2/Z, [x11, x27, LSL #1]\n" + "fmla z17.h, p3/M, z6.h, z29.h\n" + "fmla z26.h, p3/M, z3.h, z29.h\n" + "fmla z10.h, p3/M, z0.h, z29.h\n" + "ld1h { z22.h }, p2/Z, [x24, x7, LSL #1]\n" + "fmla z24.h, p3/M, z2.h, z9.h\n" + "fmla z12.h, p3/M, z7.h, z22.h\n" + "fmla z23.h, p3/M, z6.h, z22.h\n" + "fmla z26.h, p3/M, z8.h, z19.h\n" + "fmla z13.h, p3/M, z7.h, z19.h\n" + "fmla z20.h, p3/M, z6.h, z19.h\n" + "fmla z10.h, p3/M, z5.h, z19.h\n" + "ld1h { z21.h }, p2/Z, [x28, x14, LSL #1]\n" + "fmla z25.h, p3/M, z5.h, z9.h\n" + "fmla z12.h, p3/M, z5.h, z21.h\n" + "fmla z23.h, p3/M, z4.h, z21.h\n" + "fmla z24.h, p3/M, z3.h, z21.h\n" + "fmla z11.h, p3/M, z8.h, z9.h\n" + "ld1h { z19.h }, p2/Z, [x24, x14, LSL #1]\n" + "fmla z10.h, p3/M, z8.h, z22.h\n" + "ld1h { z22.h }, p2/Z, [x13, x5, LSL #1]\n" + "fmla z13.h, p3/M, z8.h, z21.h\n" + "fmla z20.h, p3/M, z7.h, z21.h\n" + "fmla z25.h, p3/M, z6.h, z21.h\n" + "fmla z12.h, p3/M, z8.h, z19.h\n" + "ld1h { z29.h }, p2/Z, [x13, x10, LSL #1]\n" + "fmla z23.h, p3/M, z7.h, z19.h\n" + "fmla z24.h, p3/M, z6.h, z19.h\n" + "ld1h { z21.h }, p2/Z, [x28, x5, LSL #1]\n" + "fmla z31.h, p3/M, z4.h, z22.h\n" + "fmla z18.h, p3/M, z3.h, z22.h\n" + "fmax z31.h, p3/M, z31.h, z15.h\n" + "fmax z18.h, p3/M, z18.h, z15.h\n" + "fmla z17.h, p3/M, z1.h, z22.h\n" + "fmla z14.h, p3/M, z0.h, z22.h\n" + "ld1h { z9.h }, p2/Z, [x28, x10, LSL #1]\n" + "fmax z17.h, p3/M, z17.h, z15.h\n" + "fmla z28.h, p3/M, z5.h, z29.h\n" + "fmla z27.h, p3/M, z4.h, z29.h\n" + "fmax z28.h, p3/M, z28.h, z15.h\n" + "fmax z27.h, p3/M, z27.h, z15.h\n" + "fmla z30.h, p3/M, z2.h, z29.h\n" + "fmla z11.h, p3/M, z1.h, z29.h\n" + "fmax z14.h, p3/M, z14.h, z15.h\n" + "fmax z30.h, p3/M, z30.h, z15.h\n" + "fmla z26.h, p3/M, z7.h, z21.h\n" + "fmla z13.h, p3/M, z6.h, z21.h\n" + "fmax z11.h, p3/M, z11.h, z15.h\n" + "fmax z26.h, p3/M, z26.h, z15.h\n" + "fmla z10.h, p3/M, z4.h, z21.h\n" + "fmla z12.h, p3/M, z3.h, z21.h\n" + "fmax z13.h, p3/M, z13.h, z15.h\n" + "fmax z10.h, p3/M, z10.h, z15.h\n" + "fmla z20.h, p3/M, z8.h, z9.h\n" + "fmla z25.h, p3/M, z7.h, z9.h\n" + "fmax z20.h, p3/M, z20.h, z15.h\n" + "fmax z25.h, p3/M, z25.h, z15.h\n" + "fmla z23.h, p3/M, z5.h, z9.h\n" + "fmla z24.h, p3/M, z4.h, z9.h\n" + "fmax z12.h, p3/M, z12.h, z15.h\n" + "fmax z23.h, p3/M, z23.h, z15.h\n" + "fmax z24.h, p3/M, z24.h, z15.h\n" + "fmin z31.h, p3/M, z31.h, z16.h\n" + "st1h { z31.h }, p0, [x15]\n" + "fmin z18.h, p3/M, z18.h, z16.h\n" + "fmin z28.h, p3/M, z28.h, z16.h\n" + "st1h { z18.h }, p0, [x15, x6, LSL #1]\n" + "fmin z27.h, p3/M, z27.h, z16.h\n" + "fmin z17.h, p3/M, z17.h, z16.h\n" + "st1h { z28.h }, p0, [x15, x25, LSL #1]\n" + "fmin z14.h, p3/M, z14.h, z16.h\n" + "fmin z30.h, p3/M, z30.h, z16.h\n" + "st1h { z27.h }, p0, [x15, x22, LSL #1]\n" + "fmin z11.h, p3/M, z11.h, z16.h\n" + "fmin z26.h, p3/M, z26.h, z16.h\n" + "st1h { z17.h }, p0, [x9]\n" + "fmin z13.h, p3/M, z13.h, z16.h\n" + "fmin z20.h, p3/M, z20.h, z16.h\n" + "st1h { z14.h }, p0, [x9, x6, LSL #1]\n" + "fmin z25.h, p3/M, z25.h, z16.h\n" + "fmin z10.h, p3/M, z10.h, z16.h\n" + "st1h { z30.h }, p0, [x9, x25, LSL #1]\n" + "fmin z12.h, p3/M, z12.h, z16.h\n" + "fmin z23.h, p3/M, z23.h, z16.h\n" + "st1h { z11.h }, p0, [x9, x22, LSL #1]\n" + "fmin z24.h, p3/M, z24.h, z16.h\n" + "st1h { z26.h }, p0, [x26]\n" + "st1h { z13.h }, p0, [x26, x6, LSL #1]\n" + "st1h { z20.h }, p0, [x26, x25, LSL #1]\n" + "st1h { z25.h }, p0, [x26, x22, LSL #1]\n" + "st1h { z10.h }, p0, [x23]\n" + "st1h { z12.h }, p0, [x23, x6, LSL #1]\n" + "st1h { z23.h }, p0, [x23, x25, LSL #1]\n" + "st1h { z24.h }, p0, [x23, x22, LSL #1]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) @@ -653,4 +653,4 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp index 972b78b6d5e08995c2731b80ddac8d365ee58980..c0be293cd7d68e00e6304c104f840d2ee9de8e8d 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -99,616 +99,616 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( __asm__ __volatile__( "ptrue p3.b\n" - "ldr x17, [%x[params_struct], %[offsetof_args_params]]\n" - "add x16, %x[params_struct], %[offsetof_Args_inptrs]\n" - "ld1h { z15.h }, p3/Z, [x17]\n" - "cnth x15\n" - "mov x14, #0x0\n" - "ld1h { z0.h }, p3/Z, [x17, #1, MUL VL]\n" - "ld1h { z1.h }, p3/Z, [x17, #2, MUL VL]\n" + "ldr x7, [%x[params_struct], %[offsetof_args_params]]\n" + "add x8, %x[params_struct], %[offsetof_Args_inptrs]\n" + "ld1h { z17.h }, p3/Z, [x7]\n" + "cnth x17\n" + "mov x16, #0x0\n" + "ld1h { z0.h }, p3/Z, [x7, #1, MUL VL]\n" + "ld1h { z1.h }, p3/Z, [x7, #2, MUL VL]\n" "whilelt p2.h, XZR, %x[n_channels]\n" - "ld1h { z2.h }, p3/Z, [x17, #3, MUL VL]\n" - "ld1h { z3.h }, p3/Z, [x17, #4, MUL VL]\n" - "cmp x15, %x[n_channels]\n" - "ld1h { z4.h }, p3/Z, [x17, #5, MUL VL]\n" - "ld1h { z5.h }, p3/Z, [x17, #6, MUL VL]\n" - "sub x13, XZR, x15\n" - "ld1h { z6.h }, p3/Z, [x17, #7, MUL VL]\n" - "addvl x17, x17, #16\n" - "ldp x12, x11, [x16, #0x0]\n" - "ldp x10, x9, [x16, #0x10]\n" - "ldr x28, [%x[params_struct], %[offsetof_args_outptrs]]\n" - "ld1rh { z14.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "ld1rh { z13.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "ld1h { z7.h }, p3/Z, [x17, #-8, MUL VL]\n" - "ld1h { z8.h }, p3/Z, [x17, #-7, MUL VL]\n" - "addvl x17, x17, #-6\n" - "ld1h { z9.h }, p2/Z, [x12, x14, LSL #1]\n" - "ld1h { z10.h }, p2/Z, [x11, x14, LSL #1]\n" - "ld1h { z11.h }, p2/Z, [x10, x14, LSL #1]\n" - "ld1h { z12.h }, p2/Z, [x9, x14, LSL #1]\n" + "ld1h { z2.h }, p3/Z, [x7, #3, MUL VL]\n" + "ld1h { z3.h }, p3/Z, [x7, #4, MUL VL]\n" + "cmp x17, %x[n_channels]\n" + "ld1h { z4.h }, p3/Z, [x7, #5, MUL VL]\n" + "ld1h { z5.h }, p3/Z, [x7, #6, MUL VL]\n" + "sub x15, XZR, x17\n" + "ld1h { z6.h }, p3/Z, [x7, #7, MUL VL]\n" + "addvl x7, x7, #16\n" + "ldp x23, x22, [x8, #0x0]\n" + "ldp x21, x20, [x8, #0x10]\n" + "ldr x14, [%x[params_struct], %[offsetof_args_outptrs]]\n" + "ld1rh { z16.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rh { z19.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "ld1h { z7.h }, p3/Z, [x7, #-8, MUL VL]\n" + "ld1h { z8.h }, p3/Z, [x7, #-7, MUL VL]\n" + "addvl x7, x7, #-6\n" + "ld1h { z9.h }, p2/Z, [x23, x16, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x22, x16, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x21, x16, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x20, x16, LSL #1]\n" "bge 2f\n" "1:" // Channel loop - "movprfx z21, z15\n fmla z21.h, p3/M, z4.h, z9.h\n" - "movprfx z16, z15\n fmla z16.h, p3/M, z8.h, z9.h\n" - "ldr x27, [x16, #0x20]\n" - "ldr x26, [x16, #0x30]\n" - "movprfx z22, z15\n fmla z22.h, p3/M, z3.h, z9.h\n" - "movprfx z25, z15\n fmla z25.h, p3/M, z1.h, z9.h\n" - "ldr x25, [x16, #0x28]\n" - "ldr x24, [x16, #0x38]\n" - "movprfx z26, z15\n fmla z26.h, p3/M, z0.h, z9.h\n" - "movprfx z17, z15\n fmla z17.h, p3/M, z7.h, z9.h\n" - "ldr x12, [x16, #0x40]\n" - "ldr x11, [x16, #0x48]\n" - "movprfx z18, z15\n fmla z18.h, p3/M, z6.h, z9.h\n" - "fmla z21.h, p3/M, z5.h, z12.h\n" - "ldr x10, [x16, #0x50]\n" - "ldr x9, [x16, #0x58]\n" - "movprfx z20, z15\n fmla z20.h, p3/M, z5.h, z9.h\n" - "movprfx z24, z15\n fmla z24.h, p3/M, z2.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x26, x14, LSL #1]\n" - "ldr x26, [x16, #0x70]\n" - "fmla z16.h, p3/M, z0.h, z10.h\n" - "movprfx z19, z15\n fmla z19.h, p3/M, z2.h, z11.h\n" - "ld1h { z10.h }, p2/Z, [x27, x14, LSL #1]\n" - "ld1h { z11.h }, p2/Z, [x25, x14, LSL #1]\n" - "fmla z22.h, p3/M, z4.h, z12.h\n" - "fmla z25.h, p3/M, z2.h, z12.h\n" - "ldr x27, [x16, #0x60]\n" - "ldr x25, [x16, #0x68]\n" - "fmla z26.h, p3/M, z1.h, z12.h\n" - "fmla z17.h, p3/M, z8.h, z12.h\n" - "inch x13\n" + "movprfx z20, z17\n fmla z20.h, p3/M, z4.h, z9.h\n" + "movprfx z26, z17\n fmla z26.h, p3/M, z8.h, z9.h\n" + "ldr x27, [x8, #0x20]\n" + "ldr x24, [x8, #0x30]\n" + "movprfx z24, z17\n fmla z24.h, p3/M, z3.h, z9.h\n" + "movprfx z30, z17\n fmla z30.h, p3/M, z1.h, z9.h\n" + "ldr x23, [x8, #0x28]\n" + "ldr x22, [x8, #0x38]\n" + "movprfx z31, z17\n fmla z31.h, p3/M, z0.h, z9.h\n" + "movprfx z22, z17\n fmla z22.h, p3/M, z7.h, z9.h\n" + "ldr x26, [x8, #0x40]\n" + "ldr x21, [x8, #0x48]\n" + "movprfx z27, z17\n fmla z27.h, p3/M, z6.h, z9.h\n" + "fmla z20.h, p3/M, z5.h, z12.h\n" + "ldr x25, [x8, #0x50]\n" + "ldr x20, [x8, #0x58]\n" + "movprfx z14, z17\n fmla z14.h, p3/M, z5.h, z9.h\n" + "movprfx z23, z17\n fmla z23.h, p3/M, z2.h, z9.h\n" + "ld1h { z25.h }, p2/Z, [x24, x16, LSL #1]\n" + "ldr x13, [x8, #0x70]\n" + "fmla z26.h, p3/M, z0.h, z10.h\n" + "movprfx z9, z17\n fmla z9.h, p3/M, z2.h, z11.h\n" + "ld1h { z28.h }, p2/Z, [x27, x16, LSL #1]\n" + "ld1h { z21.h }, p2/Z, [x23, x16, LSL #1]\n" + "fmla z24.h, p3/M, z4.h, z12.h\n" + "fmla z30.h, p3/M, z2.h, z12.h\n" + "ldr x24, [x8, #0x60]\n" + "ldr x23, [x8, #0x68]\n" + "fmla z31.h, p3/M, z1.h, z12.h\n" + "fmla z22.h, p3/M, z8.h, z12.h\n" + "inch x15\n" "mov p1.b, p2.b\n" - "fmla z18.h, p3/M, z7.h, z12.h\n" - "movprfx z28, z15\n fmla z28.h, p3/M, z6.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x11, x14, LSL #1]\n" - "ldr x11, [x16, #0x88]\n" - "fmla z21.h, p3/M, z7.h, z9.h\n" - "fmla z19.h, p3/M, z6.h, z12.h\n" - "ldr x23, [x28, #0x0]\n" - "ldr x22, [x28, #0x8]\n" - "movprfx z23, z15\n fmla z23.h, p3/M, z3.h, z12.h\n" - "movprfx z27, z15\n fmla z27.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x24, x14, LSL #1]\n" - "ldr x24, [x16, #0x78]\n" - "movprfx z31, z15\n fmla z31.h, p3/M, z8.h, z11.h\n" - "fmla z22.h, p3/M, z6.h, z9.h\n" - "ld1h { z11.h }, p2/Z, [x12, x14, LSL #1]\n" - "ldr x12, [x16, #0x80]\n" - "fmla z25.h, p3/M, z4.h, z9.h\n" - "fmla z26.h, p3/M, z3.h, z9.h\n" - "ldr x21, [x28, #0x10]\n" - "ldr x20, [x28, #0x18]\n" - "fmla z20.h, p3/M, z8.h, z9.h\n" - "fmla z24.h, p3/M, z5.h, z9.h\n" - "whilelt p0.h, x15, %x[n_channels]\n" - "fmla z28.h, p3/M, z2.h, z9.h\n" - "fmla z16.h, p3/M, z1.h, z12.h\n" - "fmla z17.h, p3/M, z0.h, z12.h\n" - "movprfx z29, z15\n fmla z29.h, p3/M, z1.h, z9.h\n" - "movprfx z30, z15\n fmla z30.h, p3/M, z0.h, z9.h\n" - "fmla z18.h, p3/M, z2.h, z11.h\n" - "ld1h { z9.h }, p2/Z, [x10, x14, LSL #1]\n" - "ldr x10, [x16, #0x90]\n" - "fmla z21.h, p3/M, z8.h, z10.h\n" - "fmla z19.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27, x14, LSL #1]\n" - "ldr x27, [x16, #0xa0]\n" - "fmla z22.h, p3/M, z7.h, z10.h\n" - "fmla z23.h, p3/M, z6.h, z10.h\n" - "fmla z25.h, p3/M, z5.h, z10.h\n" - "fmla z26.h, p3/M, z4.h, z10.h\n" - "fmla z27.h, p3/M, z3.h, z10.h\n" - "fmla z29.h, p3/M, z2.h, z10.h\n" - "fmla z30.h, p3/M, z1.h, z10.h\n" - "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x25, x14, LSL #1]\n" - "ldr x25, [x16, #0xa8]\n" - "fmla z16.h, p3/M, z3.h, z9.h\n" - "fmla z20.h, p3/M, z0.h, z9.h\n" - "ld1h { z12.h }, p2/Z, [x9, x14, LSL #1]\n" - "ldr x9, [x16, #0x98]\n" - "fmla z24.h, p3/M, z6.h, z11.h\n" - "fmla z28.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x26, x14, LSL #1]\n" - "ldr x26, [x16, #0xb0]\n" - "fmla z17.h, p3/M, z4.h, z10.h\n" - "fmla z18.h, p3/M, z3.h, z10.h\n" + "fmla z27.h, p3/M, z7.h, z12.h\n" + "movprfx z15, z17\n fmla z15.h, p3/M, z6.h, z28.h\n" + "ld1h { z10.h }, p2/Z, [x21, x16, LSL #1]\n" + "ldr x28, [x8, #0x88]\n" + "fmla z20.h, p3/M, z7.h, z25.h\n" + "fmla z9.h, p3/M, z6.h, z12.h\n" + "ldr x12, [x14, #0x0]\n" + "ldr x11, [x14, #0x8]\n" + "movprfx z11, z17\n fmla z11.h, p3/M, z3.h, z12.h\n" + "movprfx z13, z17\n fmla z13.h, p3/M, z0.h, z12.h\n" + "ld1h { z12.h }, p2/Z, [x22, x16, LSL #1]\n" + "ldr x22, [x8, #0x78]\n" + "movprfx z28, z17\n fmla z28.h, p3/M, z8.h, z21.h\n" + "fmla z24.h, p3/M, z6.h, z25.h\n" + "ld1h { z29.h }, p2/Z, [x26, x16, LSL #1]\n" + "ldr x21, [x8, #0x80]\n" + "fmla z30.h, p3/M, z4.h, z25.h\n" + "fmla z31.h, p3/M, z3.h, z25.h\n" + "ldr x10, [x14, #0x10]\n" + "ldr x9, [x14, #0x18]\n" + "movprfx z18, z17\n fmla z18.h, p3/M, z1.h, z25.h\n" + "movprfx z21, z17\n fmla z21.h, p3/M, z0.h, z25.h\n" + "whilelt p0.h, x17, %x[n_channels]\n" + "ld1h { z17.h }, p3/Z, [x7]\n" + "fmla z14.h, p3/M, z8.h, z25.h\n" + "fmla z23.h, p3/M, z5.h, z25.h\n" + "fmla z15.h, p3/M, z2.h, z25.h\n" + "fmla z26.h, p3/M, z1.h, z12.h\n" + "ld1h { z25.h }, p2/Z, [x25, x16, LSL #1]\n" + "ldr x27, [x8, #0x90]\n" + "fmla z22.h, p3/M, z0.h, z12.h\n" + "fmla z27.h, p3/M, z2.h, z29.h\n" + "ld1h { z12.h }, p2/Z, [x20, x16, LSL #1]\n" + "ldr x20, [x8, #0x98]\n" + "fmla z20.h, p3/M, z8.h, z10.h\n" + "fmla z9.h, p3/M, z1.h, z29.h\n" + "ld1h { z29.h }, p2/Z, [x24, x16, LSL #1]\n" + "ldr x26, [x8, #0xa0]\n" + "fmla z24.h, p3/M, z7.h, z10.h\n" + "fmla z11.h, p3/M, z6.h, z10.h\n" + "fmla z30.h, p3/M, z5.h, z10.h\n" + "fmla z31.h, p3/M, z4.h, z10.h\n" + "fmla z13.h, p3/M, z3.h, z10.h\n" + "fmla z18.h, p3/M, z2.h, z10.h\n" "fmla z21.h, p3/M, z1.h, z10.h\n" - "fmla z19.h, p3/M, z5.h, z12.h\n" - "fmla z23.h, p3/M, z2.h, z12.h\n" - "fmla z22.h, p3/M, z0.h, z10.h\n" - "ld1h { z12.h }, p2/Z, [x24, x14, LSL #1]\n" - "ldr x24, [x16, #0xb8]\n" - "fmla z27.h, p3/M, z8.h, z11.h\n" - "fmla z31.h, p3/M, z5.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x12, x14, LSL #1]\n" - "ldr x12, [x16, #0xc0]\n" - "fmla z16.h, p3/M, z5.h, z10.h\n" - "fmla z20.h, p3/M, z2.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x11, x14, LSL #1]\n" - "ldr x11, [x16, #0xc8]\n" - "fmla z17.h, p3/M, z5.h, z12.h\n" - "fmla z18.h, p3/M, z4.h, z12.h\n" - "fmla z21.h, p3/M, z2.h, z12.h\n" - "fmla z19.h, p3/M, z3.h, z12.h\n" - "fmla z22.h, p3/M, z1.h, z12.h\n" - "fmla z23.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9, x14, LSL #1]\n" - "ldr x9, [x16, #0xd8]\n" - "fmla z28.h, p3/M, z7.h, z11.h\n" - "fmla z29.h, p3/M, z6.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x14, LSL #1]\n" - "ldr x10, [x16, #0xd0]\n" - "fmla z16.h, p3/M, z7.h, z10.h\n" - "fmla z17.h, p3/M, z6.h, z10.h\n" - "fmla z20.h, p3/M, z4.h, z10.h\n" - "fmla z21.h, p3/M, z3.h, z10.h\n" - "fmla z24.h, p3/M, z1.h, z10.h\n" - "fmla z25.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x27, x14, LSL #1]\n" - "ldr x27, [x16, #0xe0]\n" - "fmla z18.h, p3/M, z8.h, z12.h\n" - "fmla z30.h, p3/M, z8.h, z11.h\n" - "fmla z31.h, p3/M, z7.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x25, x14, LSL #1]\n" - "fmla z27.h, p3/M, z1.h, z12.h\n" - "ldr x25, [x16, #0xe8]\n" - "fmla z19.h, p3/M, z7.h, z12.h\n" - "fmla z22.h, p3/M, z5.h, z12.h\n" - "fmla z23.h, p3/M, z4.h, z12.h\n" - "fmla z26.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x26, x14, LSL #1]\n" - "ldr x26, [x16, #0xf0]\n" - "fmla z16.h, p3/M, z2.h, z10.h\n" - "fmla z17.h, p3/M, z1.h, z10.h\n" - "fmla z18.h, p3/M, z0.h, z10.h\n" - "fmla z20.h, p3/M, z7.h, z11.h\n" - "ld1h { z10.h }, p2/Z, [x24, x14, LSL #1]\n" - "ldr x24, [x16, #0xf8]\n" - "fmla z21.h, p3/M, z6.h, z11.h\n" - "fmla z24.h, p3/M, z4.h, z11.h\n" - "fmla z25.h, p3/M, z3.h, z11.h\n" - "fmla z28.h, p3/M, z1.h, z11.h\n" - "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x12, x14, LSL #1]\n" - "fmla z27.h, p3/M, z4.h, z11.h\n" - "ldr x12, [x16, #0x100]\n" - "fmla z30.h, p3/M, z2.h, z11.h\n" - "fmla z17.h, p3/M, z2.h, z12.h\n" - "fmla z18.h, p3/M, z1.h, z12.h\n" - "fmla z19.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x11, x14, LSL #1]\n" - "ldr x11, [x16, #0x108]\n" - "fmla z16.h, p3/M, z6.h, z10.h\n" - "fmla z20.h, p3/M, z3.h, z10.h\n" - "fmla z24.h, p3/M, z0.h, z10.h\n" - "fmla z22.h, p3/M, z8.h, z11.h\n" - "ld1h { z10.h }, p2/Z, [x10, x14, LSL #1]\n" - "ldr x10, [x16, #0x110]\n" - "fmla z23.h, p3/M, z7.h, z11.h\n" - "fmla z26.h, p3/M, z5.h, z11.h\n" - "fmla z31.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x9, x14, LSL #1]\n" - "fmla z27.h, p3/M, z2.h, z12.h\n" - "ldr x9, [x16, #0x118]\n" "fmla z28.h, p3/M, z0.h, z10.h\n" - "fmla z29.h, p3/M, z4.h, z11.h\n" - "fmla z30.h, p3/M, z3.h, z11.h\n" - "fmla z19.h, p3/M, z8.h, z12.h\n" - "fmla z23.h, p3/M, z5.h, z12.h\n" - "fmla z20.h, p3/M, z6.h, z10.h\n" - "ld1h { z12.h }, p2/Z, [x27, x14, LSL #1]\n" - "fmla z24.h, p3/M, z3.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x25, x14, LSL #1]\n" - "fmla z25.h, p3/M, z7.h, z11.h\n" - "fmla z26.h, p3/M, z6.h, z11.h\n" - "fmla z28.h, p3/M, z5.h, z11.h\n" - "fmla z27.h, p3/M, z5.h, z12.h\n" - "fmla z31.h, p3/M, z2.h, z12.h\n" - "fmla z29.h, p3/M, z7.h, z10.h\n" - "fmla z30.h, p3/M, z6.h, z10.h\n" - "fmla z24.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x26, x14, LSL #1]\n" - "fmla z28.h, p3/M, z8.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x12, x14, LSL #1]\n" - "fmla z25.h, p3/M, z8.h, z11.h\n" - "fmla z26.h, p3/M, z7.h, z11.h\n" - "fmla z27.h, p3/M, z6.h, z11.h\n" - "fmla z29.h, p3/M, z5.h, z11.h\n" - "fmla z30.h, p3/M, z4.h, z11.h\n" - "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x14, LSL #1]\n" - "ldp x12, x11, [x16, #0x0]\n" - "fmla z23.h, p3/M, z8.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x24, x14, LSL #1]\n" - "fmla z16.h, p3/M, z4.h, z10.h\n" - "fmax z16.h, p3/M, z16.h, z14.h\n" - "fmla z17.h, p3/M, z3.h, z10.h\n" - "fmla z18.h, p3/M, z5.h, z11.h\n" - "fmax z17.h, p3/M, z17.h, z14.h\n" - "fmax z18.h, p3/M, z18.h, z14.h\n" - "fmla z19.h, p3/M, z4.h, z11.h\n" - "fmla z29.h, p3/M, z8.h, z12.h\n" - "fmax z19.h, p3/M, z19.h, z14.h\n" - "fmin z16.h, p3/M, z16.h, z13.h\n" - "fmla z30.h, p3/M, z7.h, z12.h\n" - "fmla z31.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x10, x14, LSL #1]\n" - "fmin z17.h, p3/M, z17.h, z13.h\n" + "ld1h { z10.h }, p2/Z, [x23, x16, LSL #1]\n" + "ldr x25, [x8, #0xa8]\n" + "fmla z26.h, p3/M, z3.h, z25.h\n" + "fmla z14.h, p3/M, z0.h, z25.h\n" + "fmla z23.h, p3/M, z6.h, z29.h\n" + "fmla z15.h, p3/M, z3.h, z29.h\n" + "ld1h { z25.h }, p2/Z, [x13, x16, LSL #1]\n" + "ldr x24, [x8, #0xb0]\n" + "fmla z22.h, p3/M, z4.h, z10.h\n" + "fmla z27.h, p3/M, z3.h, z10.h\n" "fmla z20.h, p3/M, z1.h, z10.h\n" - "fmla z21.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x9, x14, LSL #1]\n" - "fmin z18.h, p3/M, z18.h, z13.h\n" - "fmla z22.h, p3/M, z2.h, z11.h\n" - "fmla z23.h, p3/M, z1.h, z11.h\n" - "fmin z19.h, p3/M, z19.h, z13.h\n" - "fmax z20.h, p3/M, z20.h, z14.h\n" - "fmla z24.h, p3/M, z7.h, z12.h\n" - "fmla z25.h, p3/M, z6.h, z12.h\n" - "fmax z21.h, p3/M, z21.h, z14.h\n" - "fmax z22.h, p3/M, z22.h, z14.h\n" - "fmla z26.h, p3/M, z8.h, z10.h\n" - "fmla z27.h, p3/M, z7.h, z10.h\n" - "fmax z23.h, p3/M, z23.h, z14.h\n" - "st1h { z16.h }, p1, [x23, x13, LSL #1]\n" - "st1h { z17.h }, p1, [x22, x13, LSL #1]\n" - "ldr x23, [x28, #0x20]\n" - "ldr x22, [x28, #0x28]\n" + "fmla z9.h, p3/M, z5.h, z12.h\n" + "fmla z11.h, p3/M, z2.h, z12.h\n" + "fmla z24.h, p3/M, z0.h, z10.h\n" + "ld1h { z12.h }, p2/Z, [x22, x16, LSL #1]\n" + "ldr x23, [x8, #0xb8]\n" + "fmla z13.h, p3/M, z8.h, z25.h\n" + "fmla z28.h, p3/M, z5.h, z25.h\n" + "ld1h { z25.h }, p2/Z, [x21, x16, LSL #1]\n" + "ldr x22, [x8, #0xc0]\n" + "fmla z26.h, p3/M, z5.h, z10.h\n" + "fmla z14.h, p3/M, z2.h, z10.h\n" + "ld1h { z29.h }, p2/Z, [x28, x16, LSL #1]\n" + "ldr x21, [x8, #0xc8]\n" + "fmla z22.h, p3/M, z5.h, z12.h\n" + "fmla z27.h, p3/M, z4.h, z12.h\n" + "fmla z20.h, p3/M, z2.h, z12.h\n" + "fmla z9.h, p3/M, z3.h, z12.h\n" + "fmla z24.h, p3/M, z1.h, z12.h\n" + "fmla z11.h, p3/M, z0.h, z12.h\n" + "ld1h { z10.h }, p2/Z, [x20, x16, LSL #1]\n" + "ldr x28, [x8, #0xd8]\n" + "fmla z15.h, p3/M, z7.h, z25.h\n" + "fmla z18.h, p3/M, z6.h, z25.h\n" + "ld1h { z25.h }, p2/Z, [x27, x16, LSL #1]\n" + "ldr x20, [x8, #0xd0]\n" + "fmla z26.h, p3/M, z7.h, z29.h\n" + "fmla z22.h, p3/M, z6.h, z29.h\n" + "fmla z14.h, p3/M, z4.h, z29.h\n" + "fmla z20.h, p3/M, z3.h, z29.h\n" + "fmla z23.h, p3/M, z1.h, z29.h\n" + "fmla z30.h, p3/M, z0.h, z29.h\n" + "ld1h { z29.h }, p2/Z, [x26, x16, LSL #1]\n" + "ldr x27, [x8, #0xe0]\n" + "fmla z27.h, p3/M, z8.h, z10.h\n" + "fmla z21.h, p3/M, z8.h, z25.h\n" + "fmla z28.h, p3/M, z7.h, z25.h\n" + "ld1h { z25.h }, p2/Z, [x25, x16, LSL #1]\n" + "fmla z13.h, p3/M, z1.h, z10.h\n" + "ldr x26, [x8, #0xe8]\n" + "fmla z9.h, p3/M, z7.h, z10.h\n" + "fmla z24.h, p3/M, z5.h, z10.h\n" + "fmla z11.h, p3/M, z4.h, z10.h\n" + "fmla z31.h, p3/M, z2.h, z10.h\n" + "ld1h { z10.h }, p2/Z, [x24, x16, LSL #1]\n" + "ldr x25, [x8, #0xf0]\n" + "fmla z26.h, p3/M, z2.h, z29.h\n" + "fmla z22.h, p3/M, z1.h, z29.h\n" + "fmla z27.h, p3/M, z0.h, z29.h\n" + "fmla z14.h, p3/M, z7.h, z25.h\n" + "ld1h { z29.h }, p2/Z, [x23, x16, LSL #1]\n" + "ldr x24, [x8, #0xf8]\n" + "fmla z20.h, p3/M, z6.h, z25.h\n" + "fmla z23.h, p3/M, z4.h, z25.h\n" + "fmla z30.h, p3/M, z3.h, z25.h\n" + "fmla z15.h, p3/M, z1.h, z25.h\n" + "fmla z18.h, p3/M, z0.h, z25.h\n" + "ld1h { z25.h }, p2/Z, [x22, x16, LSL #1]\n" + "fmla z13.h, p3/M, z4.h, z25.h\n" + "ldr x23, [x8, #0x100]\n" + "fmla z21.h, p3/M, z2.h, z25.h\n" + "fmla z22.h, p3/M, z2.h, z10.h\n" + "fmla z27.h, p3/M, z1.h, z10.h\n" + "fmla z9.h, p3/M, z0.h, z10.h\n" + "ld1h { z12.h }, p2/Z, [x21, x16, LSL #1]\n" + "ldr x22, [x8, #0x108]\n" + "fmla z26.h, p3/M, z6.h, z29.h\n" + "fmla z14.h, p3/M, z3.h, z29.h\n" + "fmla z23.h, p3/M, z0.h, z29.h\n" + "fmla z24.h, p3/M, z8.h, z25.h\n" + "ld1h { z10.h }, p2/Z, [x20, x16, LSL #1]\n" + "ldr x21, [x8, #0x110]\n" + "fmla z11.h, p3/M, z7.h, z25.h\n" + "fmla z31.h, p3/M, z5.h, z25.h\n" + "fmla z28.h, p3/M, z1.h, z25.h\n" + "ld1h { z25.h }, p2/Z, [x28, x16, LSL #1]\n" + "fmla z13.h, p3/M, z2.h, z12.h\n" + "ldr x20, [x8, #0x118]\n" + "fmla z15.h, p3/M, z0.h, z10.h\n" + "fmla z18.h, p3/M, z4.h, z25.h\n" + "fmla z21.h, p3/M, z3.h, z25.h\n" + "fmla z9.h, p3/M, z8.h, z12.h\n" + "fmla z11.h, p3/M, z5.h, z12.h\n" + "fmla z14.h, p3/M, z6.h, z10.h\n" + "ld1h { z12.h }, p2/Z, [x27, x16, LSL #1]\n" + "fmla z23.h, p3/M, z3.h, z10.h\n" + "ld1h { z29.h }, p2/Z, [x26, x16, LSL #1]\n" + "fmla z30.h, p3/M, z7.h, z25.h\n" + "fmla z31.h, p3/M, z6.h, z25.h\n" + "fmla z15.h, p3/M, z5.h, z25.h\n" + "fmla z13.h, p3/M, z5.h, z12.h\n" + "fmla z28.h, p3/M, z2.h, z12.h\n" + "fmla z18.h, p3/M, z7.h, z29.h\n" + "fmla z21.h, p3/M, z6.h, z29.h\n" + "fmla z23.h, p3/M, z8.h, z25.h\n" + "ld1h { z25.h }, p2/Z, [x25, x16, LSL #1]\n" + "fmla z15.h, p3/M, z8.h, z29.h\n" + "ld1h { z29.h }, p2/Z, [x23, x16, LSL #1]\n" + "fmla z30.h, p3/M, z8.h, z25.h\n" + "fmla z31.h, p3/M, z7.h, z25.h\n" + "fmla z13.h, p3/M, z6.h, z25.h\n" + "fmla z18.h, p3/M, z5.h, z25.h\n" + "fmla z21.h, p3/M, z4.h, z25.h\n" + "fmla z28.h, p3/M, z3.h, z25.h\n" + "ld1h { z25.h }, p2/Z, [x22, x16, LSL #1]\n" + "ldp x27, x26, [x8, #0x0]\n" + "fmla z11.h, p3/M, z8.h, z12.h\n" + "ld1h { z12.h }, p2/Z, [x24, x16, LSL #1]\n" + "fmla z26.h, p3/M, z4.h, z29.h\n" + "fmax z26.h, p3/M, z26.h, z16.h\n" + "fmla z22.h, p3/M, z3.h, z29.h\n" + "fmla z27.h, p3/M, z5.h, z25.h\n" + "fmax z22.h, p3/M, z22.h, z16.h\n" + "fmax z27.h, p3/M, z27.h, z16.h\n" + "fmla z9.h, p3/M, z4.h, z25.h\n" + "fmla z18.h, p3/M, z8.h, z12.h\n" + "fmax z9.h, p3/M, z9.h, z16.h\n" + "fmin z26.h, p3/M, z26.h, z19.h\n" + "fmla z21.h, p3/M, z7.h, z12.h\n" + "fmla z28.h, p3/M, z6.h, z12.h\n" + "ld1h { z10.h }, p2/Z, [x21, x16, LSL #1]\n" + "fmin z22.h, p3/M, z22.h, z19.h\n" + "fmla z14.h, p3/M, z1.h, z29.h\n" + "fmla z20.h, p3/M, z0.h, z29.h\n" + "ld1h { z12.h }, p2/Z, [x20, x16, LSL #1]\n" + "fmin z27.h, p3/M, z27.h, z19.h\n" + "fmla z24.h, p3/M, z2.h, z25.h\n" + "fmla z11.h, p3/M, z1.h, z25.h\n" + "fmin z9.h, p3/M, z9.h, z19.h\n" + "fmax z14.h, p3/M, z14.h, z16.h\n" + "fmla z23.h, p3/M, z7.h, z10.h\n" + "fmla z30.h, p3/M, z6.h, z10.h\n" + "fmax z20.h, p3/M, z20.h, z16.h\n" + "fmax z24.h, p3/M, z24.h, z16.h\n" + "fmla z31.h, p3/M, z8.h, z12.h\n" + "fmla z13.h, p3/M, z7.h, z12.h\n" + "fmax z11.h, p3/M, z11.h, z16.h\n" + "st1h { z26.h }, p1, [x12, x15, LSL #1]\n" + "st1h { z22.h }, p1, [x11, x15, LSL #1]\n" + "ldr x23, [x14, #0x20]\n" + "ldr x22, [x14, #0x28]\n" + "fmla z15.h, p3/M, z4.h, z10.h\n" + "st1h { z27.h }, p1, [x10, x15, LSL #1]\n" + "ldr x21, [x14, #0x30]\n" + "fmla z18.h, p3/M, z3.h, z10.h\n" + "fmla z21.h, p3/M, z5.h, z12.h\n" + "st1h { z9.h }, p1, [x9, x15, LSL #1]\n" + "ldr x20, [x14, #0x38]\n" "fmla z28.h, p3/M, z4.h, z12.h\n" - "st1h { z18.h }, p1, [x21, x13, LSL #1]\n" - "ldr x21, [x28, #0x30]\n" - "fmla z29.h, p3/M, z3.h, z12.h\n" - "fmla z30.h, p3/M, z5.h, z10.h\n" - "st1h { z19.h }, p1, [x20, x13, LSL #1]\n" - "ldr x20, [x28, #0x38]\n" - "fmla z31.h, p3/M, z4.h, z10.h\n" - "ldp x10, x9, [x16, #0x10]\n" - "fmin z20.h, p3/M, z20.h, z13.h\n" - "fmin z21.h, p3/M, z21.h, z13.h\n" - "st1h { z20.h }, p1, [x23, x13, LSL #1]\n" - "ldr x23, [x28, #0x40]\n" - "fmin z22.h, p3/M, z22.h, z13.h\n" - "fmin z23.h, p3/M, z23.h, z13.h\n" - "st1h { z21.h }, p1, [x22, x13, LSL #1]\n" - "ldr x22, [x28, #0x48]\n" - "fmax z24.h, p3/M, z24.h, z14.h\n" - "fmax z25.h, p3/M, z25.h, z14.h\n" - "st1h { z22.h }, p1, [x21, x13, LSL #1]\n" - "ldr x21, [x28, #0x50]\n" - "fmax z26.h, p3/M, z26.h, z14.h\n" - "fmax z27.h, p3/M, z27.h, z14.h\n" - "st1h { z23.h }, p1, [x20, x13, LSL #1]\n" - "ldr x20, [x28, #0x58]\n" - "inch x14\n" - "ld1h { z9.h }, p0/Z, [x12, x15, LSL #1]\n" - "ld1h { z10.h }, p0/Z, [x11, x15, LSL #1]\n" - "fmin z24.h, p3/M, z24.h, z13.h\n" - "ld1h { z11.h }, p0/Z, [x10, x15, LSL #1]\n" - "ld1h { z12.h }, p0/Z, [x9, x15, LSL #1]\n" - "inch x15\n" - "fmin z25.h, p3/M, z25.h, z13.h\n" - "fmin z26.h, p3/M, z26.h, z13.h\n" - "fmin z27.h, p3/M, z27.h, z13.h\n" - "st1h { z24.h }, p1, [x23, x13, LSL #1]\n" - "ldr x23, [x28, #0x60]\n" - "fmax z28.h, p3/M, z28.h, z14.h\n" - "fmax z29.h, p3/M, z29.h, z14.h\n" - "st1h { z25.h }, p1, [x22, x13, LSL #1]\n" - "ldr x22, [x28, #0x68]\n" - "fmax z30.h, p3/M, z30.h, z14.h\n" - "fmax z31.h, p3/M, z31.h, z14.h\n" - "st1h { z26.h }, p1, [x21, x13, LSL #1]\n" - "ldr x21, [x28, #0x70]\n" - "st1h { z27.h }, p1, [x20, x13, LSL #1]\n" - "ldr x20, [x28, #0x78]\n" - "ld1h { z15.h }, p3/Z, [x17]\n" - "whilelt p2.h, x14, %x[n_channels]\n" - "ld1h { z0.h }, p3/Z, [x17, #1, MUL VL]\n" - "ld1h { z1.h }, p3/Z, [x17, #2, MUL VL]\n" - "cmp x15, %x[n_channels]\n" - "fmin z28.h, p3/M, z28.h, z13.h\n" - "ld1h { z2.h }, p3/Z, [x17, #3, MUL VL]\n" - "ld1h { z3.h }, p3/Z, [x17, #4, MUL VL]\n" - "fmin z29.h, p3/M, z29.h, z13.h\n" - "fmin z30.h, p3/M, z30.h, z13.h\n" - "ld1h { z4.h }, p3/Z, [x17, #5, MUL VL]\n" - "ld1h { z5.h }, p3/Z, [x17, #6, MUL VL]\n" - "fmin z31.h, p3/M, z31.h, z13.h\n" - "st1h { z28.h }, p1, [x23, x13, LSL #1]\n" - "ld1h { z6.h }, p3/Z, [x17, #7, MUL VL]\n" - "addvl x17, x17, #16\n" - "st1h { z29.h }, p1, [x22, x13, LSL #1]\n" - "ld1h { z7.h }, p3/Z, [x17, #-8, MUL VL]\n" - "st1h { z30.h }, p1, [x21, x13, LSL #1]\n" - "ld1h { z8.h }, p3/Z, [x17, #-7, MUL VL]\n" - "addvl x17, x17, #-6\n" - "st1h { z31.h }, p1, [x20, x13, LSL #1]\n" + "ldp x25, x24, [x8, #0x10]\n" + "fmin z14.h, p3/M, z14.h, z19.h\n" + "fmin z20.h, p3/M, z20.h, z19.h\n" + "st1h { z14.h }, p1, [x23, x15, LSL #1]\n" + "ldr x23, [x14, #0x40]\n" + "fmin z24.h, p3/M, z24.h, z19.h\n" + "fmin z11.h, p3/M, z11.h, z19.h\n" + "st1h { z20.h }, p1, [x22, x15, LSL #1]\n" + "ldr x22, [x14, #0x48]\n" + "fmax z23.h, p3/M, z23.h, z16.h\n" + "fmax z30.h, p3/M, z30.h, z16.h\n" + "st1h { z24.h }, p1, [x21, x15, LSL #1]\n" + "ldr x21, [x14, #0x50]\n" + "fmax z31.h, p3/M, z31.h, z16.h\n" + "fmax z13.h, p3/M, z13.h, z16.h\n" + "st1h { z11.h }, p1, [x20, x15, LSL #1]\n" + "ldr x20, [x14, #0x58]\n" + "inch x16\n" + "ld1h { z9.h }, p0/Z, [x27, x17, LSL #1]\n" + "ld1h { z10.h }, p0/Z, [x26, x17, LSL #1]\n" + "fmin z23.h, p3/M, z23.h, z19.h\n" + "ld1h { z11.h }, p0/Z, [x25, x17, LSL #1]\n" + "ld1h { z12.h }, p0/Z, [x24, x17, LSL #1]\n" + "inch x17\n" + "fmin z30.h, p3/M, z30.h, z19.h\n" + "fmin z31.h, p3/M, z31.h, z19.h\n" + "fmin z13.h, p3/M, z13.h, z19.h\n" + "st1h { z23.h }, p1, [x23, x15, LSL #1]\n" + "ldr x23, [x14, #0x60]\n" + "fmax z15.h, p3/M, z15.h, z16.h\n" + "fmax z18.h, p3/M, z18.h, z16.h\n" + "st1h { z30.h }, p1, [x22, x15, LSL #1]\n" + "ldr x22, [x14, #0x68]\n" + "fmax z21.h, p3/M, z21.h, z16.h\n" + "fmax z28.h, p3/M, z28.h, z16.h\n" + "st1h { z31.h }, p1, [x21, x15, LSL #1]\n" + "ldr x21, [x14, #0x70]\n" + "st1h { z13.h }, p1, [x20, x15, LSL #1]\n" + "ldr x20, [x14, #0x78]\n" + "ld1h { z0.h }, p3/Z, [x7, #1, MUL VL]\n" + "whilelt p2.h, x16, %x[n_channels]\n" + "ld1h { z1.h }, p3/Z, [x7, #2, MUL VL]\n" + "ld1h { z2.h }, p3/Z, [x7, #3, MUL VL]\n" + "cmp x17, %x[n_channels]\n" + "fmin z15.h, p3/M, z15.h, z19.h\n" + "ld1h { z3.h }, p3/Z, [x7, #4, MUL VL]\n" + "ld1h { z4.h }, p3/Z, [x7, #5, MUL VL]\n" + "fmin z18.h, p3/M, z18.h, z19.h\n" + "fmin z21.h, p3/M, z21.h, z19.h\n" + "ld1h { z5.h }, p3/Z, [x7, #6, MUL VL]\n" + "ld1h { z6.h }, p3/Z, [x7, #7, MUL VL]\n" + "addvl x7, x7, #16\n" + "fmin z28.h, p3/M, z28.h, z19.h\n" + "st1h { z15.h }, p1, [x23, x15, LSL #1]\n" + "ld1h { z7.h }, p3/Z, [x7, #-8, MUL VL]\n" + "ld1h { z8.h }, p3/Z, [x7, #-7, MUL VL]\n" + "addvl x7, x7, #-6\n" + "st1h { z18.h }, p1, [x22, x15, LSL #1]\n" + "st1h { z21.h }, p1, [x21, x15, LSL #1]\n" + "st1h { z28.h }, p1, [x20, x15, LSL #1]\n" "blt 1b\n" "2:" // Channel tail - "movprfx z21, z15\n fmla z21.h, p3/M, z4.h, z9.h\n" - "movprfx z16, z15\n fmla z16.h, p3/M, z8.h, z9.h\n" - "ldr x27, [x16, #0x20]\n" - "ldr x26, [x16, #0x30]\n" - "movprfx z22, z15\n fmla z22.h, p3/M, z3.h, z9.h\n" - "movprfx z25, z15\n fmla z25.h, p3/M, z1.h, z9.h\n" - "ldr x25, [x16, #0x28]\n" - "ldr x24, [x16, #0x38]\n" - "movprfx z26, z15\n fmla z26.h, p3/M, z0.h, z9.h\n" - "movprfx z17, z15\n fmla z17.h, p3/M, z7.h, z9.h\n" - "ldr x12, [x16, #0x40]\n" - "ldr x11, [x16, #0x48]\n" - "movprfx z18, z15\n fmla z18.h, p3/M, z6.h, z9.h\n" - "fmla z21.h, p3/M, z5.h, z12.h\n" - "ldr x10, [x16, #0x50]\n" - "ldr x9, [x16, #0x58]\n" - "movprfx z20, z15\n fmla z20.h, p3/M, z5.h, z9.h\n" - "movprfx z24, z15\n fmla z24.h, p3/M, z2.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x26, x14, LSL #1]\n" - "ldr x26, [x16, #0x70]\n" - "fmla z16.h, p3/M, z0.h, z10.h\n" - "movprfx z19, z15\n fmla z19.h, p3/M, z2.h, z11.h\n" - "ld1h { z10.h }, p2/Z, [x27, x14, LSL #1]\n" - "ld1h { z11.h }, p2/Z, [x25, x14, LSL #1]\n" - "fmla z22.h, p3/M, z4.h, z12.h\n" - "fmla z25.h, p3/M, z2.h, z12.h\n" - "ldr x27, [x16, #0x60]\n" - "ldr x25, [x16, #0x68]\n" - "fmla z26.h, p3/M, z1.h, z12.h\n" - "fmla z17.h, p3/M, z8.h, z12.h\n" - "inch x13\n" - "mov p1.b, p2.b\n" - "fmla z18.h, p3/M, z7.h, z12.h\n" - "movprfx z28, z15\n fmla z28.h, p3/M, z6.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x11, x14, LSL #1]\n" - "ldr x11, [x16, #0x88]\n" - "fmla z21.h, p3/M, z7.h, z9.h\n" - "fmla z19.h, p3/M, z6.h, z12.h\n" - "ldr x23, [x28, #0x0]\n" - "ldr x22, [x28, #0x8]\n" - "movprfx z23, z15\n fmla z23.h, p3/M, z3.h, z12.h\n" - "movprfx z27, z15\n fmla z27.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x24, x14, LSL #1]\n" - "ldr x24, [x16, #0x78]\n" - "movprfx z31, z15\n fmla z31.h, p3/M, z8.h, z11.h\n" - "fmla z22.h, p3/M, z6.h, z9.h\n" - "ld1h { z11.h }, p2/Z, [x12, x14, LSL #1]\n" - "ldr x12, [x16, #0x80]\n" - "fmla z25.h, p3/M, z4.h, z9.h\n" - "fmla z26.h, p3/M, z3.h, z9.h\n" - "ldr x21, [x28, #0x10]\n" - "ldr x20, [x28, #0x18]\n" - "fmla z20.h, p3/M, z8.h, z9.h\n" - "fmla z24.h, p3/M, z5.h, z9.h\n" - "fmla z28.h, p3/M, z2.h, z9.h\n" - "fmla z16.h, p3/M, z1.h, z12.h\n" - "fmla z17.h, p3/M, z0.h, z12.h\n" - "movprfx z29, z15\n fmla z29.h, p3/M, z1.h, z9.h\n" - "movprfx z30, z15\n fmla z30.h, p3/M, z0.h, z9.h\n" - "fmla z18.h, p3/M, z2.h, z11.h\n" - "ld1h { z9.h }, p2/Z, [x10, x14, LSL #1]\n" - "ldr x10, [x16, #0x90]\n" - "fmla z21.h, p3/M, z8.h, z10.h\n" - "fmla z19.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27, x14, LSL #1]\n" - "ldr x27, [x16, #0xa0]\n" - "fmla z22.h, p3/M, z7.h, z10.h\n" - "fmla z23.h, p3/M, z6.h, z10.h\n" - "fmla z25.h, p3/M, z5.h, z10.h\n" - "fmla z26.h, p3/M, z4.h, z10.h\n" - "fmla z27.h, p3/M, z3.h, z10.h\n" - "fmla z29.h, p3/M, z2.h, z10.h\n" - "fmla z30.h, p3/M, z1.h, z10.h\n" - "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x25, x14, LSL #1]\n" - "ldr x25, [x16, #0xa8]\n" - "fmla z16.h, p3/M, z3.h, z9.h\n" - "fmla z20.h, p3/M, z0.h, z9.h\n" - "ld1h { z12.h }, p2/Z, [x9, x14, LSL #1]\n" - "ldr x9, [x16, #0x98]\n" - "fmla z24.h, p3/M, z6.h, z11.h\n" - "fmla z28.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x26, x14, LSL #1]\n" - "ldr x26, [x16, #0xb0]\n" - "fmla z17.h, p3/M, z4.h, z10.h\n" - "fmla z18.h, p3/M, z3.h, z10.h\n" - "fmla z21.h, p3/M, z1.h, z10.h\n" - "fmla z19.h, p3/M, z5.h, z12.h\n" - "fmla z23.h, p3/M, z2.h, z12.h\n" - "fmla z22.h, p3/M, z0.h, z10.h\n" - "ld1h { z12.h }, p2/Z, [x24, x14, LSL #1]\n" - "ldr x24, [x16, #0xb8]\n" - "fmla z27.h, p3/M, z8.h, z11.h\n" - "fmla z31.h, p3/M, z5.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x12, x14, LSL #1]\n" - "ldr x12, [x16, #0xc0]\n" - "fmla z16.h, p3/M, z5.h, z10.h\n" - "fmla z20.h, p3/M, z2.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x11, x14, LSL #1]\n" - "ldr x11, [x16, #0xc8]\n" - "fmla z17.h, p3/M, z5.h, z12.h\n" - "fmla z18.h, p3/M, z4.h, z12.h\n" - "fmla z21.h, p3/M, z2.h, z12.h\n" - "fmla z19.h, p3/M, z3.h, z12.h\n" - "fmla z22.h, p3/M, z1.h, z12.h\n" - "fmla z23.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9, x14, LSL #1]\n" - "ldr x9, [x16, #0xd8]\n" - "fmla z28.h, p3/M, z7.h, z11.h\n" - "fmla z29.h, p3/M, z6.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x14, LSL #1]\n" - "ldr x10, [x16, #0xd0]\n" - "fmla z16.h, p3/M, z7.h, z10.h\n" - "fmla z17.h, p3/M, z6.h, z10.h\n" - "fmla z20.h, p3/M, z4.h, z10.h\n" - "fmla z21.h, p3/M, z3.h, z10.h\n" - "fmla z24.h, p3/M, z1.h, z10.h\n" - "fmla z25.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x27, x14, LSL #1]\n" - "ldr x27, [x16, #0xe0]\n" - "fmla z18.h, p3/M, z8.h, z12.h\n" - "fmla z30.h, p3/M, z8.h, z11.h\n" - "fmla z31.h, p3/M, z7.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x25, x14, LSL #1]\n" - "fmla z27.h, p3/M, z1.h, z12.h\n" - "ldr x25, [x16, #0xe8]\n" - "fmla z19.h, p3/M, z7.h, z12.h\n" - "fmla z22.h, p3/M, z5.h, z12.h\n" - "fmla z23.h, p3/M, z4.h, z12.h\n" - "fmla z26.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x26, x14, LSL #1]\n" - "ldr x26, [x16, #0xf0]\n" - "fmla z16.h, p3/M, z2.h, z10.h\n" - "fmla z17.h, p3/M, z1.h, z10.h\n" + "movprfx z14, z17\n fmla z14.h, p3/M, z4.h, z9.h\n" + "movprfx z18, z17\n fmla z18.h, p3/M, z8.h, z9.h\n" + "ldr x27, [x8, #0x20]\n" + "ldr x24, [x8, #0x30]\n" + "movprfx z15, z17\n fmla z15.h, p3/M, z3.h, z9.h\n" + "movprfx z30, z17\n fmla z30.h, p3/M, z1.h, z9.h\n" + "ldr x23, [x8, #0x28]\n" + "ldr x22, [x8, #0x38]\n" + "movprfx z20, z17\n fmla z20.h, p3/M, z0.h, z9.h\n" + "movprfx z13, z17\n fmla z13.h, p3/M, z7.h, z9.h\n" + "ldr x26, [x8, #0x40]\n" + "ldr x21, [x8, #0x48]\n" + "movprfx z22, z17\n fmla z22.h, p3/M, z6.h, z9.h\n" + "fmla z14.h, p3/M, z5.h, z12.h\n" + "ldr x25, [x8, #0x50]\n" + "ldr x20, [x8, #0x58]\n" + "movprfx z27, z17\n fmla z27.h, p3/M, z5.h, z9.h\n" + "movprfx z31, z17\n fmla z31.h, p3/M, z2.h, z9.h\n" + "ld1h { z23.h }, p2/Z, [x24, x16, LSL #1]\n" + "ldr x13, [x8, #0x70]\n" "fmla z18.h, p3/M, z0.h, z10.h\n" - "fmla z20.h, p3/M, z7.h, z11.h\n" - "ld1h { z10.h }, p2/Z, [x24, x14, LSL #1]\n" - "ldr x24, [x16, #0xf8]\n" - "fmla z21.h, p3/M, z6.h, z11.h\n" - "fmla z24.h, p3/M, z4.h, z11.h\n" - "fmla z25.h, p3/M, z3.h, z11.h\n" - "fmla z28.h, p3/M, z1.h, z11.h\n" - "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x12, x14, LSL #1]\n" - "fmla z27.h, p3/M, z4.h, z11.h\n" - "ldr x12, [x16, #0x100]\n" - "fmla z30.h, p3/M, z2.h, z11.h\n" - "fmla z17.h, p3/M, z2.h, z12.h\n" + "movprfx z9, z17\n fmla z9.h, p3/M, z2.h, z11.h\n" + "ld1h { z21.h }, p2/Z, [x27, x16, LSL #1]\n" + "ld1h { z25.h }, p2/Z, [x23, x16, LSL #1]\n" + "fmla z15.h, p3/M, z4.h, z12.h\n" + "fmla z30.h, p3/M, z2.h, z12.h\n" + "ldr x24, [x8, #0x60]\n" + "ldr x23, [x8, #0x68]\n" + "fmla z20.h, p3/M, z1.h, z12.h\n" + "fmla z13.h, p3/M, z8.h, z12.h\n" + "inch x15\n" + "mov p0.b, p2.b\n" + "fmla z22.h, p3/M, z7.h, z12.h\n" + "movprfx z28, z17\n fmla z28.h, p3/M, z6.h, z21.h\n" + "ld1h { z29.h }, p2/Z, [x21, x16, LSL #1]\n" + "ldr x28, [x8, #0x88]\n" + "fmla z14.h, p3/M, z7.h, z23.h\n" + "fmla z9.h, p3/M, z6.h, z12.h\n" + "ldr x12, [x14, #0x0]\n" + "ldr x11, [x14, #0x8]\n" + "movprfx z11, z17\n fmla z11.h, p3/M, z3.h, z12.h\n" + "movprfx z10, z17\n fmla z10.h, p3/M, z0.h, z12.h\n" + "ld1h { z12.h }, p2/Z, [x22, x16, LSL #1]\n" + "ldr x22, [x8, #0x78]\n" + "movprfx z26, z17\n fmla z26.h, p3/M, z8.h, z25.h\n" + "fmla z15.h, p3/M, z6.h, z23.h\n" + "ld1h { z21.h }, p2/Z, [x26, x16, LSL #1]\n" + "ldr x21, [x8, #0x80]\n" + "fmla z30.h, p3/M, z4.h, z23.h\n" + "fmla z20.h, p3/M, z3.h, z23.h\n" + "ldr x10, [x14, #0x10]\n" + "ldr x9, [x14, #0x18]\n" + "movprfx z25, z17\n fmla z25.h, p3/M, z1.h, z23.h\n" + "movprfx z24, z17\n fmla z24.h, p3/M, z0.h, z23.h\n" + "fmla z27.h, p3/M, z8.h, z23.h\n" + "fmla z31.h, p3/M, z5.h, z23.h\n" + "fmla z28.h, p3/M, z2.h, z23.h\n" "fmla z18.h, p3/M, z1.h, z12.h\n" - "fmla z19.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x11, x14, LSL #1]\n" - "ldr x11, [x16, #0x108]\n" - "fmla z16.h, p3/M, z6.h, z10.h\n" - "fmla z20.h, p3/M, z3.h, z10.h\n" - "fmla z24.h, p3/M, z0.h, z10.h\n" - "fmla z22.h, p3/M, z8.h, z11.h\n" - "ld1h { z10.h }, p2/Z, [x10, x14, LSL #1]\n" - "ldr x10, [x16, #0x110]\n" - "fmla z23.h, p3/M, z7.h, z11.h\n" - "fmla z26.h, p3/M, z5.h, z11.h\n" - "fmla z31.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x9, x14, LSL #1]\n" - "fmla z27.h, p3/M, z2.h, z12.h\n" - "ldr x9, [x16, #0x118]\n" - "fmla z28.h, p3/M, z0.h, z10.h\n" - "fmla z29.h, p3/M, z4.h, z11.h\n" - "fmla z30.h, p3/M, z3.h, z11.h\n" - "fmla z19.h, p3/M, z8.h, z12.h\n" - "fmla z23.h, p3/M, z5.h, z12.h\n" - "fmla z20.h, p3/M, z6.h, z10.h\n" - "ld1h { z12.h }, p2/Z, [x27, x14, LSL #1]\n" - "fmla z24.h, p3/M, z3.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x25, x14, LSL #1]\n" - "fmla z25.h, p3/M, z7.h, z11.h\n" - "fmla z26.h, p3/M, z6.h, z11.h\n" - "fmla z28.h, p3/M, z5.h, z11.h\n" - "fmla z27.h, p3/M, z5.h, z12.h\n" - "fmla z31.h, p3/M, z2.h, z12.h\n" - "fmla z29.h, p3/M, z7.h, z10.h\n" - "fmla z30.h, p3/M, z6.h, z10.h\n" - "fmla z24.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x26, x14, LSL #1]\n" - "fmla z28.h, p3/M, z8.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x12, x14, LSL #1]\n" - "fmla z25.h, p3/M, z8.h, z11.h\n" - "fmla z26.h, p3/M, z7.h, z11.h\n" - "fmla z27.h, p3/M, z6.h, z11.h\n" - "fmla z29.h, p3/M, z5.h, z11.h\n" - "fmla z30.h, p3/M, z4.h, z11.h\n" - "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x14, LSL #1]\n" - "fmla z23.h, p3/M, z8.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x24, x14, LSL #1]\n" - "fmla z16.h, p3/M, z4.h, z10.h\n" - "fmax z16.h, p3/M, z16.h, z14.h\n" - "fmla z17.h, p3/M, z3.h, z10.h\n" - "fmla z18.h, p3/M, z5.h, z11.h\n" - "fmax z17.h, p3/M, z17.h, z14.h\n" - "fmax z18.h, p3/M, z18.h, z14.h\n" - "fmla z19.h, p3/M, z4.h, z11.h\n" - "fmla z29.h, p3/M, z8.h, z12.h\n" - "fmax z19.h, p3/M, z19.h, z14.h\n" - "fmin z16.h, p3/M, z16.h, z13.h\n" - "fmla z30.h, p3/M, z7.h, z12.h\n" - "fmla z31.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x10, x14, LSL #1]\n" - "fmin z17.h, p3/M, z17.h, z13.h\n" - "fmla z20.h, p3/M, z1.h, z10.h\n" - "fmla z21.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x9, x14, LSL #1]\n" - "fmin z18.h, p3/M, z18.h, z13.h\n" - "fmla z22.h, p3/M, z2.h, z11.h\n" - "fmla z23.h, p3/M, z1.h, z11.h\n" - "fmin z19.h, p3/M, z19.h, z13.h\n" - "fmax z20.h, p3/M, z20.h, z14.h\n" - "fmla z24.h, p3/M, z7.h, z12.h\n" - "fmla z25.h, p3/M, z6.h, z12.h\n" - "fmax z21.h, p3/M, z21.h, z14.h\n" - "fmax z22.h, p3/M, z22.h, z14.h\n" - "fmla z26.h, p3/M, z8.h, z10.h\n" - "fmla z27.h, p3/M, z7.h, z10.h\n" - "fmax z23.h, p3/M, z23.h, z14.h\n" - "st1h { z16.h }, p1, [x23, x13, LSL #1]\n" - "st1h { z17.h }, p1, [x22, x13, LSL #1]\n" - "ldr x23, [x28, #0x20]\n" - "ldr x22, [x28, #0x28]\n" - "fmla z28.h, p3/M, z4.h, z12.h\n" - "st1h { z18.h }, p1, [x21, x13, LSL #1]\n" - "ldr x21, [x28, #0x30]\n" - "fmla z29.h, p3/M, z3.h, z12.h\n" - "fmla z30.h, p3/M, z5.h, z10.h\n" - "st1h { z19.h }, p1, [x20, x13, LSL #1]\n" - "ldr x20, [x28, #0x38]\n" - "fmla z31.h, p3/M, z4.h, z10.h\n" - "fmin z20.h, p3/M, z20.h, z13.h\n" - "fmin z21.h, p3/M, z21.h, z13.h\n" - "fmin z22.h, p3/M, z22.h, z13.h\n" - "st1h { z20.h }, p1, [x23, x13, LSL #1]\n" - "ldr x23, [x28, #0x40]\n" - "fmin z23.h, p3/M, z23.h, z13.h\n" - "fmax z24.h, p3/M, z24.h, z14.h\n" - "st1h { z21.h }, p1, [x22, x13, LSL #1]\n" - "ldr x22, [x28, #0x48]\n" - "fmax z25.h, p3/M, z25.h, z14.h\n" - "fmax z26.h, p3/M, z26.h, z14.h\n" - "st1h { z22.h }, p1, [x21, x13, LSL #1]\n" - "ldr x21, [x28, #0x50]\n" - "fmax z27.h, p3/M, z27.h, z14.h\n" - "st1h { z23.h }, p1, [x20, x13, LSL #1]\n" - "ldr x20, [x28, #0x58]\n" - "fmin z24.h, p3/M, z24.h, z13.h\n" - "fmin z25.h, p3/M, z25.h, z13.h\n" - "fmin z26.h, p3/M, z26.h, z13.h\n" - "st1h { z24.h }, p1, [x23, x13, LSL #1]\n" - "ldr x23, [x28, #0x60]\n" - "fmin z27.h, p3/M, z27.h, z13.h\n" - "fmax z28.h, p3/M, z28.h, z14.h\n" - "st1h { z25.h }, p1, [x22, x13, LSL #1]\n" - "ldr x22, [x28, #0x68]\n" - "fmax z29.h, p3/M, z29.h, z14.h\n" - "fmax z30.h, p3/M, z30.h, z14.h\n" - "st1h { z26.h }, p1, [x21, x13, LSL #1]\n" - "ldr x21, [x28, #0x70]\n" - "fmax z31.h, p3/M, z31.h, z14.h\n" - "st1h { z27.h }, p1, [x20, x13, LSL #1]\n" - "ldr x20, [x28, #0x78]\n" - "fmin z28.h, p3/M, z28.h, z13.h\n" - "fmin z29.h, p3/M, z29.h, z13.h\n" - "fmin z30.h, p3/M, z30.h, z13.h\n" - "st1h { z28.h }, p1, [x23, x13, LSL #1]\n" - "fmin z31.h, p3/M, z31.h, z13.h\n" - "st1h { z29.h }, p1, [x22, x13, LSL #1]\n" - "st1h { z30.h }, p1, [x21, x13, LSL #1]\n" - "st1h { z31.h }, p1, [x20, x13, LSL #1]\n" + "ld1h { z23.h }, p2/Z, [x25, x16, LSL #1]\n" + "ldr x27, [x8, #0x90]\n" + "fmla z13.h, p3/M, z0.h, z12.h\n" + "fmla z22.h, p3/M, z2.h, z21.h\n" + "ld1h { z12.h }, p2/Z, [x20, x16, LSL #1]\n" + "ldr x20, [x8, #0x98]\n" + "fmla z14.h, p3/M, z8.h, z29.h\n" + "fmla z9.h, p3/M, z1.h, z21.h\n" + "ld1h { z21.h }, p2/Z, [x24, x16, LSL #1]\n" + "ldr x26, [x8, #0xa0]\n" + "fmla z15.h, p3/M, z7.h, z29.h\n" + "fmla z11.h, p3/M, z6.h, z29.h\n" + "fmla z30.h, p3/M, z5.h, z29.h\n" + "fmla z20.h, p3/M, z4.h, z29.h\n" + "fmla z10.h, p3/M, z3.h, z29.h\n" + "fmla z25.h, p3/M, z2.h, z29.h\n" + "fmla z24.h, p3/M, z1.h, z29.h\n" + "fmla z26.h, p3/M, z0.h, z29.h\n" + "ld1h { z29.h }, p2/Z, [x23, x16, LSL #1]\n" + "ldr x25, [x8, #0xa8]\n" + "fmla z18.h, p3/M, z3.h, z23.h\n" + "fmla z27.h, p3/M, z0.h, z23.h\n" + "fmla z31.h, p3/M, z6.h, z21.h\n" + "fmla z28.h, p3/M, z3.h, z21.h\n" + "ld1h { z21.h }, p2/Z, [x13, x16, LSL #1]\n" + "ldr x24, [x8, #0xb0]\n" + "fmla z13.h, p3/M, z4.h, z29.h\n" + "fmla z22.h, p3/M, z3.h, z29.h\n" + "fmla z14.h, p3/M, z1.h, z29.h\n" + "fmla z9.h, p3/M, z5.h, z12.h\n" + "fmla z11.h, p3/M, z2.h, z12.h\n" + "fmla z15.h, p3/M, z0.h, z29.h\n" + "ld1h { z17.h }, p2/Z, [x22, x16, LSL #1]\n" + "ldr x23, [x8, #0xb8]\n" + "fmla z10.h, p3/M, z8.h, z21.h\n" + "fmla z26.h, p3/M, z5.h, z21.h\n" + "ld1h { z23.h }, p2/Z, [x21, x16, LSL #1]\n" + "ldr x22, [x8, #0xc0]\n" + "fmla z18.h, p3/M, z5.h, z29.h\n" + "fmla z27.h, p3/M, z2.h, z29.h\n" + "ld1h { z21.h }, p2/Z, [x28, x16, LSL #1]\n" + "ldr x21, [x8, #0xc8]\n" + "fmla z13.h, p3/M, z5.h, z17.h\n" + "fmla z22.h, p3/M, z4.h, z17.h\n" + "fmla z14.h, p3/M, z2.h, z17.h\n" + "fmla z9.h, p3/M, z3.h, z17.h\n" + "fmla z15.h, p3/M, z1.h, z17.h\n" + "fmla z11.h, p3/M, z0.h, z17.h\n" + "ld1h { z29.h }, p2/Z, [x20, x16, LSL #1]\n" + "ldr x28, [x8, #0xd8]\n" + "fmla z28.h, p3/M, z7.h, z23.h\n" + "fmla z25.h, p3/M, z6.h, z23.h\n" + "ld1h { z23.h }, p2/Z, [x27, x16, LSL #1]\n" + "ldr x20, [x8, #0xd0]\n" + "fmla z18.h, p3/M, z7.h, z21.h\n" + "fmla z13.h, p3/M, z6.h, z21.h\n" + "fmla z27.h, p3/M, z4.h, z21.h\n" + "fmla z14.h, p3/M, z3.h, z21.h\n" + "fmla z31.h, p3/M, z1.h, z21.h\n" + "fmla z30.h, p3/M, z0.h, z21.h\n" + "ld1h { z21.h }, p2/Z, [x26, x16, LSL #1]\n" + "ldr x27, [x8, #0xe0]\n" + "fmla z22.h, p3/M, z8.h, z29.h\n" + "fmla z24.h, p3/M, z8.h, z23.h\n" + "fmla z26.h, p3/M, z7.h, z23.h\n" + "ld1h { z23.h }, p2/Z, [x25, x16, LSL #1]\n" + "fmla z10.h, p3/M, z1.h, z29.h\n" + "ldr x26, [x8, #0xe8]\n" + "fmla z9.h, p3/M, z7.h, z29.h\n" + "fmla z15.h, p3/M, z5.h, z29.h\n" + "fmla z11.h, p3/M, z4.h, z29.h\n" + "fmla z20.h, p3/M, z2.h, z29.h\n" + "ld1h { z29.h }, p2/Z, [x24, x16, LSL #1]\n" + "ldr x25, [x8, #0xf0]\n" + "fmla z18.h, p3/M, z2.h, z21.h\n" + "fmla z13.h, p3/M, z1.h, z21.h\n" + "fmla z22.h, p3/M, z0.h, z21.h\n" + "fmla z27.h, p3/M, z7.h, z23.h\n" + "ld1h { z21.h }, p2/Z, [x23, x16, LSL #1]\n" + "ldr x24, [x8, #0xf8]\n" + "fmla z14.h, p3/M, z6.h, z23.h\n" + "fmla z31.h, p3/M, z4.h, z23.h\n" + "fmla z30.h, p3/M, z3.h, z23.h\n" + "fmla z28.h, p3/M, z1.h, z23.h\n" + "fmla z25.h, p3/M, z0.h, z23.h\n" + "ld1h { z17.h }, p2/Z, [x22, x16, LSL #1]\n" + "fmla z10.h, p3/M, z4.h, z17.h\n" + "ldr x23, [x8, #0x100]\n" + "fmla z24.h, p3/M, z2.h, z17.h\n" + "fmla z13.h, p3/M, z2.h, z29.h\n" + "fmla z22.h, p3/M, z1.h, z29.h\n" + "fmla z9.h, p3/M, z0.h, z29.h\n" + "ld1h { z23.h }, p2/Z, [x21, x16, LSL #1]\n" + "ldr x22, [x8, #0x108]\n" + "fmla z18.h, p3/M, z6.h, z21.h\n" + "fmla z27.h, p3/M, z3.h, z21.h\n" + "fmla z31.h, p3/M, z0.h, z21.h\n" + "fmla z15.h, p3/M, z8.h, z17.h\n" + "ld1h { z29.h }, p2/Z, [x20, x16, LSL #1]\n" + "ldr x21, [x8, #0x110]\n" + "fmla z11.h, p3/M, z7.h, z17.h\n" + "fmla z20.h, p3/M, z5.h, z17.h\n" + "fmla z26.h, p3/M, z1.h, z17.h\n" + "ld1h { z21.h }, p2/Z, [x28, x16, LSL #1]\n" + "fmla z10.h, p3/M, z2.h, z23.h\n" + "ldr x20, [x8, #0x118]\n" + "fmla z28.h, p3/M, z0.h, z29.h\n" + "fmla z25.h, p3/M, z4.h, z21.h\n" + "fmla z24.h, p3/M, z3.h, z21.h\n" + "fmla z9.h, p3/M, z8.h, z23.h\n" + "fmla z11.h, p3/M, z5.h, z23.h\n" + "fmla z27.h, p3/M, z6.h, z29.h\n" + "ld1h { z23.h }, p2/Z, [x27, x16, LSL #1]\n" + "fmla z31.h, p3/M, z3.h, z29.h\n" + "ld1h { z17.h }, p2/Z, [x26, x16, LSL #1]\n" + "fmla z30.h, p3/M, z7.h, z21.h\n" + "fmla z20.h, p3/M, z6.h, z21.h\n" + "fmla z28.h, p3/M, z5.h, z21.h\n" + "fmla z10.h, p3/M, z5.h, z23.h\n" + "fmla z26.h, p3/M, z2.h, z23.h\n" + "fmla z25.h, p3/M, z7.h, z17.h\n" + "fmla z24.h, p3/M, z6.h, z17.h\n" + "fmla z31.h, p3/M, z8.h, z21.h\n" + "ld1h { z21.h }, p2/Z, [x25, x16, LSL #1]\n" + "fmla z28.h, p3/M, z8.h, z17.h\n" + "ld1h { z12.h }, p2/Z, [x23, x16, LSL #1]\n" + "fmla z30.h, p3/M, z8.h, z21.h\n" + "fmla z20.h, p3/M, z7.h, z21.h\n" + "fmla z10.h, p3/M, z6.h, z21.h\n" + "fmla z25.h, p3/M, z5.h, z21.h\n" + "fmla z24.h, p3/M, z4.h, z21.h\n" + "fmla z26.h, p3/M, z3.h, z21.h\n" + "ld1h { z21.h }, p2/Z, [x22, x16, LSL #1]\n" + "fmla z11.h, p3/M, z8.h, z23.h\n" + "ld1h { z29.h }, p2/Z, [x24, x16, LSL #1]\n" + "fmla z18.h, p3/M, z4.h, z12.h\n" + "fmax z18.h, p3/M, z18.h, z16.h\n" + "fmla z13.h, p3/M, z3.h, z12.h\n" + "fmla z22.h, p3/M, z5.h, z21.h\n" + "fmax z13.h, p3/M, z13.h, z16.h\n" + "fmax z22.h, p3/M, z22.h, z16.h\n" + "fmla z9.h, p3/M, z4.h, z21.h\n" + "fmla z25.h, p3/M, z8.h, z29.h\n" + "fmax z9.h, p3/M, z9.h, z16.h\n" + "fmin z18.h, p3/M, z18.h, z19.h\n" + "fmla z24.h, p3/M, z7.h, z29.h\n" + "fmla z26.h, p3/M, z6.h, z29.h\n" + "ld1h { z23.h }, p2/Z, [x21, x16, LSL #1]\n" + "fmin z13.h, p3/M, z13.h, z19.h\n" + "fmla z27.h, p3/M, z1.h, z12.h\n" + "fmla z14.h, p3/M, z0.h, z12.h\n" + "ld1h { z29.h }, p2/Z, [x20, x16, LSL #1]\n" + "fmin z22.h, p3/M, z22.h, z19.h\n" + "fmla z15.h, p3/M, z2.h, z21.h\n" + "fmla z11.h, p3/M, z1.h, z21.h\n" + "fmin z9.h, p3/M, z9.h, z19.h\n" + "fmax z27.h, p3/M, z27.h, z16.h\n" + "fmla z31.h, p3/M, z7.h, z23.h\n" + "fmla z30.h, p3/M, z6.h, z23.h\n" + "fmax z14.h, p3/M, z14.h, z16.h\n" + "fmax z15.h, p3/M, z15.h, z16.h\n" + "fmla z20.h, p3/M, z8.h, z29.h\n" + "fmla z10.h, p3/M, z7.h, z29.h\n" + "fmax z11.h, p3/M, z11.h, z16.h\n" + "st1h { z18.h }, p0, [x12, x15, LSL #1]\n" + "st1h { z13.h }, p0, [x11, x15, LSL #1]\n" + "ldr x23, [x14, #0x20]\n" + "ldr x22, [x14, #0x28]\n" + "fmla z28.h, p3/M, z4.h, z23.h\n" + "st1h { z22.h }, p0, [x10, x15, LSL #1]\n" + "ldr x21, [x14, #0x30]\n" + "fmla z25.h, p3/M, z3.h, z23.h\n" + "fmla z24.h, p3/M, z5.h, z29.h\n" + "st1h { z9.h }, p0, [x9, x15, LSL #1]\n" + "ldr x20, [x14, #0x38]\n" + "fmla z26.h, p3/M, z4.h, z29.h\n" + "fmin z27.h, p3/M, z27.h, z19.h\n" + "fmin z14.h, p3/M, z14.h, z19.h\n" + "fmin z15.h, p3/M, z15.h, z19.h\n" + "st1h { z27.h }, p0, [x23, x15, LSL #1]\n" + "ldr x23, [x14, #0x40]\n" + "fmin z11.h, p3/M, z11.h, z19.h\n" + "fmax z31.h, p3/M, z31.h, z16.h\n" + "st1h { z14.h }, p0, [x22, x15, LSL #1]\n" + "ldr x22, [x14, #0x48]\n" + "fmax z30.h, p3/M, z30.h, z16.h\n" + "fmax z20.h, p3/M, z20.h, z16.h\n" + "st1h { z15.h }, p0, [x21, x15, LSL #1]\n" + "ldr x21, [x14, #0x50]\n" + "fmax z10.h, p3/M, z10.h, z16.h\n" + "st1h { z11.h }, p0, [x20, x15, LSL #1]\n" + "ldr x20, [x14, #0x58]\n" + "fmin z31.h, p3/M, z31.h, z19.h\n" + "fmin z30.h, p3/M, z30.h, z19.h\n" + "fmin z20.h, p3/M, z20.h, z19.h\n" + "st1h { z31.h }, p0, [x23, x15, LSL #1]\n" + "ldr x23, [x14, #0x60]\n" + "fmin z10.h, p3/M, z10.h, z19.h\n" + "fmax z28.h, p3/M, z28.h, z16.h\n" + "st1h { z30.h }, p0, [x22, x15, LSL #1]\n" + "ldr x22, [x14, #0x68]\n" + "fmax z25.h, p3/M, z25.h, z16.h\n" + "fmax z24.h, p3/M, z24.h, z16.h\n" + "st1h { z20.h }, p0, [x21, x15, LSL #1]\n" + "ldr x21, [x14, #0x70]\n" + "fmax z26.h, p3/M, z26.h, z16.h\n" + "st1h { z10.h }, p0, [x20, x15, LSL #1]\n" + "ldr x20, [x14, #0x78]\n" + "fmin z28.h, p3/M, z28.h, z19.h\n" + "fmin z25.h, p3/M, z25.h, z19.h\n" + "fmin z24.h, p3/M, z24.h, z19.h\n" + "st1h { z28.h }, p0, [x23, x15, LSL #1]\n" + "fmin z26.h, p3/M, z26.h, z19.h\n" + "st1h { z25.h }, p0, [x22, x15, LSL #1]\n" + "st1h { z24.h }, p0, [x21, x15, LSL #1]\n" + "st1h { z26.h }, p0, [x20, x15, LSL #1]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp index 82173ee71f669be137bfc2316e81b51c7043f093..d8a25666bd138bfa989415285f2e265296cdcb6f 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,19 +22,19 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include #pragma once -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { -void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); -void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); +void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const input_ptrs, __fp16 *const *const outptrs, const void *params, unsigned int n_channels, const __fp16 activation_min, const __fp16 activation_max); +void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(const unsigned int n_tile_rows, const unsigned int n_tile_cols, const __fp16 *inptr, int64_t ld_input_row, int64_t ld_input_col, __fp16 *outptr, int64_t ld_output_row, int64_t ld_output_col, const void *params, unsigned int n_channels, const __fp16 activation_min, const __fp16 activation_max); class sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst : public DepthwiseDepthfirstStrategy<__fp16, __fp16, __fp16, __fp16> { @@ -57,7 +57,7 @@ class sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst : public DepthwiseDepthfirst constexpr static unsigned int output_cols = 2; sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst(const CPUInfo *) - : DepthwiseDepthfirstStrategy<__fp16, __fp16, __fp16, __fp16>(2, 3, 2) {} + : Parent(output_rows, output_cols, kernel_rows, kernel_cols, stride_rows, stride_cols) {} arm_gemm::VLType get_vl_type(void) const override { return vl_type; } @@ -68,4 +68,4 @@ class sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst : public DepthwiseDepthfirst } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp index 6a9b354c0265a2ba26cbc945238d8653a0cd7165..58decdba1cb2d8b5a70fc83280685d4f5026fe9b 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -112,7 +112,7 @@ void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "add x28, x12, x23, LSL #1\n" "madd x20, x16, x14, x20\n" // offset += tile_j * ld_output_col "whilelt p2.h, XZR, %x[n_channels]\n" - "ld1h { z19.h }, p3/Z, [x11]\n" + "ld1h { z30.h }, p3/Z, [x11]\n" "ld1h { z0.h }, p3/Z, [x11, #1, MUL VL]\n" "mul x20, x20, x24\n" // offset *= output_tile_size "ld1h { z1.h }, p3/Z, [x11, #2, MUL VL]\n" @@ -128,8 +128,8 @@ void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "add x24, x26, x15\n" "add x9, x9, x20, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16) "cmp x13, %x[n_channels]\n" - "ld1rh { z18.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "ld1rh { z29.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rh { z28.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "add x23, x25, x23, LSL #1\n" "add x22, x9, x21, LSL #1\n" "ld1h { z7.h }, p3/Z, [x11, #-8, MUL VL]\n" @@ -147,191 +147,191 @@ void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "ld1h { z16.h }, p2/Z, [x12, x10, LSL #1]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "movprfx z28, z19\n fmla z28.h, p3/M, z8.h, z9.h\n" - "movprfx z29, z19\n fmla z29.h, p3/M, z6.h, z9.h\n" + "movprfx z27, z30\n fmla z27.h, p3/M, z8.h, z9.h\n" + "movprfx z26, z30\n fmla z26.h, p3/M, z6.h, z9.h\n" "whilelt p1.h, x13, %x[n_channels]\n" "inch x21\n" - "fmla z28.h, p3/M, z0.h, z10.h\n" - "fmla z29.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x28, x24, LSL #1]\n" + "fmla z27.h, p3/M, z0.h, z10.h\n" + "fmla z26.h, p3/M, z1.h, z12.h\n" + "ld1h { z20.h }, p2/Z, [x28, x24, LSL #1]\n" "inch x13\n" - "fmla z28.h, p3/M, z1.h, z11.h\n" - "fmla z29.h, p3/M, z2.h, z13.h\n" - "ld1h { z11.h }, p2/Z, [x28, x26, LSL #1]\n" - "ld1h { z13.h }, p2/Z, [x28, x10, LSL #1]\n" - "fmla z28.h, p3/M, z3.h, z14.h\n" - "fmla z29.h, p3/M, z0.h, z16.h\n" - "ld1h { z14.h }, p2/Z, [x25]\n" + "fmla z27.h, p3/M, z1.h, z11.h\n" + "fmla z26.h, p3/M, z2.h, z13.h\n" + "ld1h { z17.h }, p2/Z, [x28, x26, LSL #1]\n" + "ld1h { z19.h }, p2/Z, [x28, x10, LSL #1]\n" + "fmla z27.h, p3/M, z3.h, z14.h\n" + "fmla z26.h, p3/M, z0.h, z16.h\n" + "ld1h { z18.h }, p2/Z, [x25]\n" "mov p0.b, p2.b\n" - "fmla z28.h, p3/M, z4.h, z15.h\n" - "fmla z29.h, p3/M, z4.h, z11.h\n" - "ld1h { z15.h }, p2/Z, [x27]\n" - "ld1h { z11.h }, p2/Z, [x25, x15, LSL #1]\n" - "fmla z28.h, p3/M, z2.h, z16.h\n" - "fmla z29.h, p3/M, z5.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x26, LSL #1]\n" - "ld1h { z16.h }, p2/Z, [x27, x15, LSL #1]\n" - "movprfx z30, z19\n fmla z30.h, p3/M, z2.h, z9.h\n" - "movprfx z31, z19\n fmla z31.h, p3/M, z0.h, z9.h\n" + "fmla z27.h, p3/M, z4.h, z15.h\n" + "fmla z26.h, p3/M, z4.h, z17.h\n" + "ld1h { z25.h }, p2/Z, [x27]\n" + "ld1h { z17.h }, p2/Z, [x25, x15, LSL #1]\n" + "fmla z27.h, p3/M, z2.h, z16.h\n" + "fmla z26.h, p3/M, z5.h, z20.h\n" + "ld1h { z24.h }, p2/Z, [x27, x26, LSL #1]\n" + "ld1h { z23.h }, p2/Z, [x27, x15, LSL #1]\n" + "movprfx z22, z30\n fmla z22.h, p3/M, z2.h, z9.h\n" + "movprfx z21, z30\n fmla z21.h, p3/M, z0.h, z9.h\n" "addvl x12, x12, #1\n" "addvl x28, x28, #1\n" - "fmla z28.h, p3/M, z5.h, z13.h\n" - "fmla z29.h, p3/M, z3.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x25, x26, LSL #1]\n" - "ld1h { z19.h }, p3/Z, [x11]\n" - "fmla z30.h, p3/M, z3.h, z14.h\n" - "fmla z31.h, p3/M, z4.h, z13.h\n" - "ld1h { z14.h }, p2/Z, [x25, x24, LSL #1]\n" - "ld1h { z13.h }, p2/Z, [x23, x15, LSL #1]\n" - "fmla z30.h, p3/M, z0.h, z15.h\n" - "fmla z31.h, p3/M, z1.h, z12.h\n" + "fmla z27.h, p3/M, z5.h, z19.h\n" + "fmla z26.h, p3/M, z3.h, z19.h\n" + "ld1h { z16.h }, p2/Z, [x25, x26, LSL #1]\n" + "ld1h { z30.h }, p3/Z, [x11]\n" + "fmla z22.h, p3/M, z3.h, z18.h\n" + "fmla z21.h, p3/M, z4.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x25, x24, LSL #1]\n" + "ld1h { z20.h }, p2/Z, [x23, x15, LSL #1]\n" + "fmla z22.h, p3/M, z0.h, z25.h\n" + "fmla z21.h, p3/M, z1.h, z24.h\n" "ld1h { z0.h }, p3/Z, [x11, #1, MUL VL]\n" "inch x20\n" - "fmla z30.h, p3/M, z4.h, z11.h\n" - "fmla z31.h, p3/M, z5.h, z14.h\n" - "ld1h { z11.h }, p2/Z, [x27, x24, LSL #1]\n" - "ld1h { z14.h }, p2/Z, [x23, x26, LSL #1]\n" - "fmla z28.h, p3/M, z6.h, z15.h\n" - "fmla z30.h, p3/M, z1.h, z16.h\n" - "ld1h { z15.h }, p2/Z, [x23]\n" + "fmla z22.h, p3/M, z4.h, z17.h\n" + "fmla z21.h, p3/M, z5.h, z16.h\n" + "ld1h { z19.h }, p2/Z, [x27, x24, LSL #1]\n" + "ld1h { z18.h }, p2/Z, [x23, x26, LSL #1]\n" + "fmla z27.h, p3/M, z6.h, z25.h\n" + "fmla z22.h, p3/M, z1.h, z23.h\n" + "ld1h { z17.h }, p2/Z, [x23]\n" "addvl x27, x27, #1\n" - "fmla z31.h, p3/M, z2.h, z11.h\n" - "fmla z28.h, p3/M, z7.h, z16.h\n" + "fmla z21.h, p3/M, z2.h, z19.h\n" + "fmla z27.h, p3/M, z7.h, z23.h\n" "ld1h { z16.h }, p2/Z, [x25, x10, LSL #1]\n" - "fmax z28.h, p3/M, z28.h, z18.h\n" - "fmla z30.h, p3/M, z6.h, z15.h\n" - "fmla z31.h, p3/M, z3.h, z16.h\n" - "ld1h { z15.h }, p2/Z, [x23, x10, LSL #1]\n" + "fmax z27.h, p3/M, z27.h, z29.h\n" + "fmla z22.h, p3/M, z6.h, z17.h\n" + "fmla z21.h, p3/M, z3.h, z16.h\n" + "ld1h { z17.h }, p2/Z, [x23, x10, LSL #1]\n" "ld1h { z1.h }, p3/Z, [x11, #2, MUL VL]\n" - "fmla z30.h, p3/M, z7.h, z13.h\n" - "fmla z31.h, p3/M, z7.h, z14.h\n" + "fmla z22.h, p3/M, z7.h, z20.h\n" + "fmla z21.h, p3/M, z7.h, z18.h\n" "ld1h { z2.h }, p3/Z, [x11, #3, MUL VL]\n" "ld1h { z3.h }, p3/Z, [x11, #4, MUL VL]\n" - "fmla z29.h, p3/M, z7.h, z12.h\n" - "fmla z30.h, p3/M, z5.h, z16.h\n" + "fmla z26.h, p3/M, z7.h, z24.h\n" + "fmla z22.h, p3/M, z5.h, z16.h\n" "ld1h { z4.h }, p3/Z, [x11, #5, MUL VL]\n" "ld1h { z5.h }, p3/Z, [x11, #6, MUL VL]\n" - "fmla z31.h, p3/M, z6.h, z15.h\n" - "fmla z29.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x23, x24, LSL #1]\n" - "fmax z29.h, p3/M, z29.h, z18.h\n" - "fmla z30.h, p3/M, z8.h, z15.h\n" - "fmla z31.h, p3/M, z8.h, z11.h\n" - "fmax z30.h, p3/M, z30.h, z18.h\n" - "fmax z31.h, p3/M, z31.h, z18.h\n" + "fmla z21.h, p3/M, z6.h, z17.h\n" + "fmla z26.h, p3/M, z8.h, z19.h\n" + "ld1h { z16.h }, p2/Z, [x23, x24, LSL #1]\n" + "fmax z26.h, p3/M, z26.h, z29.h\n" + "fmla z22.h, p3/M, z8.h, z17.h\n" + "fmla z21.h, p3/M, z8.h, z16.h\n" + "fmax z22.h, p3/M, z22.h, z29.h\n" + "fmax z21.h, p3/M, z21.h, z29.h\n" "ld1h { z6.h }, p3/Z, [x11, #7, MUL VL]\n" "addvl x11, x11, #16\n" "whilelt p2.h, x21, %x[n_channels]\n" "ld1h { z9.h }, p1/Z, [x27, x10, LSL #1]\n" "cmp x13, %x[n_channels]\n" - "fmin z28.h, p3/M, z28.h, z17.h\n" + "fmin z27.h, p3/M, z27.h, z28.h\n" "ld1h { z10.h }, p1/Z, [x12]\n" "ld1h { z11.h }, p1/Z, [x12, x15, LSL #1]\n" - "fmin z29.h, p3/M, z29.h, z17.h\n" - "fmin z30.h, p3/M, z30.h, z17.h\n" + "fmin z26.h, p3/M, z26.h, z28.h\n" + "fmin z22.h, p3/M, z22.h, z28.h\n" "ld1h { z12.h }, p1/Z, [x12, x26, LSL #1]\n" "ld1h { z13.h }, p1/Z, [x12, x24, LSL #1]\n" - "fmin z31.h, p3/M, z31.h, z17.h\n" + "fmin z21.h, p3/M, z21.h, z28.h\n" "addvl x25, x25, #1\n" "ld1h { z14.h }, p1/Z, [x28]\n" "ld1h { z15.h }, p1/Z, [x28, x15, LSL #1]\n" "addvl x23, x23, #1\n" "ld1h { z16.h }, p1/Z, [x12, x10, LSL #1]\n" - "st1h { z28.h }, p0, [x9]\n" + "st1h { z27.h }, p0, [x9]\n" "ld1h { z7.h }, p3/Z, [x11, #-8, MUL VL]\n" - "st1h { z29.h }, p0, [x9, x14, LSL #1]\n" + "st1h { z26.h }, p0, [x9, x14, LSL #1]\n" "addvl x9, x9, #1\n" "ld1h { z8.h }, p3/Z, [x11, #-7, MUL VL]\n" "addvl x11, x11, #-6\n" - "st1h { z30.h }, p0, [x22]\n" - "st1h { z31.h }, p0, [x22, x14, LSL #1]\n" + "st1h { z22.h }, p0, [x22]\n" + "st1h { z21.h }, p0, [x22, x14, LSL #1]\n" "addvl x22, x22, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "movprfx z28, z19\n fmla z28.h, p3/M, z8.h, z9.h\n" - "movprfx z29, z19\n fmla z29.h, p3/M, z6.h, z9.h\n" + "movprfx z27, z30\n fmla z27.h, p3/M, z8.h, z9.h\n" + "movprfx z26, z30\n fmla z26.h, p3/M, z6.h, z9.h\n" "ldr x16, [%x[params_struct], %[offsetof_args_tile_j]]\n" "ldr x11, [%x[params_struct], %[offsetof_args_tile_i]]\n" - "fmla z28.h, p3/M, z0.h, z10.h\n" - "fmla z29.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x28, x24, LSL #1]\n" + "fmla z27.h, p3/M, z0.h, z10.h\n" + "fmla z26.h, p3/M, z1.h, z12.h\n" + "ld1h { z20.h }, p2/Z, [x28, x24, LSL #1]\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" - "fmla z28.h, p3/M, z1.h, z11.h\n" - "fmla z29.h, p3/M, z2.h, z13.h\n" - "ld1h { z11.h }, p2/Z, [x28, x26, LSL #1]\n" - "ld1h { z13.h }, p2/Z, [x28, x10, LSL #1]\n" - "fmla z28.h, p3/M, z3.h, z14.h\n" - "fmla z29.h, p3/M, z0.h, z16.h\n" - "ld1h { z14.h }, p2/Z, [x25]\n" + "fmla z27.h, p3/M, z1.h, z11.h\n" + "fmla z26.h, p3/M, z2.h, z13.h\n" + "ld1h { z17.h }, p2/Z, [x28, x26, LSL #1]\n" + "ld1h { z19.h }, p2/Z, [x28, x10, LSL #1]\n" + "fmla z27.h, p3/M, z3.h, z14.h\n" + "fmla z26.h, p3/M, z0.h, z16.h\n" + "ld1h { z18.h }, p2/Z, [x25]\n" "add x16, x16, #0x1\n" - "fmla z28.h, p3/M, z4.h, z15.h\n" - "fmla z29.h, p3/M, z4.h, z11.h\n" - "ld1h { z15.h }, p2/Z, [x27]\n" - "ld1h { z11.h }, p2/Z, [x25, x15, LSL #1]\n" - "fmla z28.h, p3/M, z2.h, z16.h\n" - "fmla z29.h, p3/M, z5.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x26, LSL #1]\n" - "ld1h { z16.h }, p2/Z, [x27, x15, LSL #1]\n" - "movprfx z30, z19\n fmla z30.h, p3/M, z2.h, z9.h\n" - "movprfx z31, z19\n fmla z31.h, p3/M, z0.h, z9.h\n" + "fmla z27.h, p3/M, z4.h, z15.h\n" + "fmla z26.h, p3/M, z4.h, z17.h\n" + "ld1h { z25.h }, p2/Z, [x27]\n" + "ld1h { z17.h }, p2/Z, [x25, x15, LSL #1]\n" + "fmla z27.h, p3/M, z2.h, z16.h\n" + "fmla z26.h, p3/M, z5.h, z20.h\n" + "ld1h { z24.h }, p2/Z, [x27, x26, LSL #1]\n" + "ld1h { z23.h }, p2/Z, [x27, x15, LSL #1]\n" + "movprfx z22, z30\n fmla z22.h, p3/M, z2.h, z9.h\n" + "movprfx z21, z30\n fmla z21.h, p3/M, z0.h, z9.h\n" "cmp x16, x20\n" "add x21, x11, #0x1\n" - "fmla z28.h, p3/M, z5.h, z13.h\n" - "fmla z29.h, p3/M, z3.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x25, x26, LSL #1]\n" + "fmla z27.h, p3/M, z5.h, z19.h\n" + "fmla z26.h, p3/M, z3.h, z19.h\n" + "ld1h { z16.h }, p2/Z, [x25, x26, LSL #1]\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" - "fmla z30.h, p3/M, z3.h, z14.h\n" - "fmla z31.h, p3/M, z4.h, z13.h\n" - "ld1h { z14.h }, p2/Z, [x25, x24, LSL #1]\n" - "ld1h { z13.h }, p2/Z, [x23, x15, LSL #1]\n" - "fmla z30.h, p3/M, z0.h, z15.h\n" - "fmla z31.h, p3/M, z1.h, z12.h\n" + "fmla z22.h, p3/M, z3.h, z18.h\n" + "fmla z21.h, p3/M, z4.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x25, x24, LSL #1]\n" + "ld1h { z20.h }, p2/Z, [x23, x15, LSL #1]\n" + "fmla z22.h, p3/M, z0.h, z25.h\n" + "fmla z21.h, p3/M, z1.h, z24.h\n" "csel x11, x11, x21, LT\n" "mov p0.b, p2.b\n" - "fmla z30.h, p3/M, z4.h, z11.h\n" - "fmla z31.h, p3/M, z5.h, z14.h\n" - "ld1h { z11.h }, p2/Z, [x27, x24, LSL #1]\n" - "ld1h { z14.h }, p2/Z, [x23, x26, LSL #1]\n" - "fmla z28.h, p3/M, z6.h, z15.h\n" - "fmla z30.h, p3/M, z1.h, z16.h\n" - "ld1h { z15.h }, p2/Z, [x23]\n" + "fmla z22.h, p3/M, z4.h, z17.h\n" + "fmla z21.h, p3/M, z5.h, z16.h\n" + "ld1h { z19.h }, p2/Z, [x27, x24, LSL #1]\n" + "ld1h { z18.h }, p2/Z, [x23, x26, LSL #1]\n" + "fmla z27.h, p3/M, z6.h, z25.h\n" + "fmla z22.h, p3/M, z1.h, z23.h\n" + "ld1h { z17.h }, p2/Z, [x23]\n" "csel x16, x16, XZR, LT\n" - "fmla z31.h, p3/M, z2.h, z11.h\n" - "fmla z28.h, p3/M, z7.h, z16.h\n" + "fmla z21.h, p3/M, z2.h, z19.h\n" + "fmla z27.h, p3/M, z7.h, z23.h\n" "ld1h { z16.h }, p2/Z, [x25, x10, LSL #1]\n" - "fmax z28.h, p3/M, z28.h, z18.h\n" - "fmla z30.h, p3/M, z6.h, z15.h\n" - "fmla z31.h, p3/M, z3.h, z16.h\n" - "ld1h { z15.h }, p2/Z, [x23, x10, LSL #1]\n" + "fmax z27.h, p3/M, z27.h, z29.h\n" + "fmla z22.h, p3/M, z6.h, z17.h\n" + "fmla z21.h, p3/M, z3.h, z16.h\n" + "ld1h { z17.h }, p2/Z, [x23, x10, LSL #1]\n" "cmp x11, x20\n" - "fmla z30.h, p3/M, z7.h, z13.h\n" - "fmla z31.h, p3/M, z7.h, z14.h\n" - "fmin z28.h, p3/M, z28.h, z17.h\n" - "st1h { z28.h }, p0, [x9]\n" - "fmla z29.h, p3/M, z7.h, z12.h\n" - "fmla z30.h, p3/M, z5.h, z16.h\n" - "fmla z31.h, p3/M, z6.h, z15.h\n" - "fmla z29.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x23, x24, LSL #1]\n" - "fmax z29.h, p3/M, z29.h, z18.h\n" - "fmla z30.h, p3/M, z8.h, z15.h\n" - "fmla z31.h, p3/M, z8.h, z11.h\n" - "fmax z30.h, p3/M, z30.h, z18.h\n" - "fmax z31.h, p3/M, z31.h, z18.h\n" - "fmin z29.h, p3/M, z29.h, z17.h\n" - "fmin z30.h, p3/M, z30.h, z17.h\n" - "st1h { z29.h }, p0, [x9, x14, LSL #1]\n" - "fmin z31.h, p3/M, z31.h, z17.h\n" - "st1h { z30.h }, p0, [x22]\n" - "st1h { z31.h }, p0, [x22, x14, LSL #1]\n" + "fmla z22.h, p3/M, z7.h, z20.h\n" + "fmla z21.h, p3/M, z7.h, z18.h\n" + "fmin z27.h, p3/M, z27.h, z28.h\n" + "st1h { z27.h }, p0, [x9]\n" + "fmla z26.h, p3/M, z7.h, z24.h\n" + "fmla z22.h, p3/M, z5.h, z16.h\n" + "fmla z21.h, p3/M, z6.h, z17.h\n" + "fmla z26.h, p3/M, z8.h, z19.h\n" + "ld1h { z16.h }, p2/Z, [x23, x24, LSL #1]\n" + "fmax z26.h, p3/M, z26.h, z29.h\n" + "fmla z22.h, p3/M, z8.h, z17.h\n" + "fmla z21.h, p3/M, z8.h, z16.h\n" + "fmax z22.h, p3/M, z22.h, z29.h\n" + "fmax z21.h, p3/M, z21.h, z29.h\n" + "fmin z26.h, p3/M, z26.h, z28.h\n" + "fmin z22.h, p3/M, z22.h, z28.h\n" + "st1h { z26.h }, p0, [x9, x14, LSL #1]\n" + "fmin z21.h, p3/M, z21.h, z28.h\n" + "st1h { z22.h }, p0, [x22]\n" + "st1h { z21.h }, p0, [x22, x14, LSL #1]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp index ff97b51e28bef4de8d2f3cc11417266e2a717aa8..d5fbb6baee0e391fae9106e755224d1cb035d4e4 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -96,7 +96,7 @@ void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( "ldp x11, x10, [x20, #0x10]\n" "mov x9, #0x0\n" "whilelt p2.h, XZR, %x[n_channels]\n" - "ld1h { z19.h }, p3/Z, [x16]\n" + "ld1h { z20.h }, p3/Z, [x16]\n" "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n" "cmp x14, %x[n_channels]\n" "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n" @@ -111,8 +111,8 @@ void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( "ldp x25, x24, [x15, #0x10]\n" "ldp x23, x22, [x15, #0x20]\n" "ldp x21, x20, [x15, #0x30]\n" - "ld1rh { z18.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "ld1rh { z26.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rh { z25.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n" "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n" "addvl x16, x16, #-6\n" @@ -126,89 +126,89 @@ void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( "ld1h { z16.h }, p2/Z, [x20, x9, LSL #1]\n" "bge 2f\n" "1:" // Channel loop - "movprfx z28, z19\n fmla z28.h, p3/M, z8.h, z9.h\n" - "movprfx z29, z19\n fmla z29.h, p3/M, z6.h, z9.h\n" - "ldr x27, [x15, #0x40]\n" - "ldr x26, [x15, #0x48]\n" - "fmla z28.h, p3/M, z0.h, z10.h\n" - "fmla z29.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x26, x9, LSL #1]\n" - "ldr x25, [x15, #0x50]\n" - "fmla z28.h, p3/M, z1.h, z11.h\n" - "fmla z29.h, p3/M, z2.h, z13.h\n" - "ld1h { z11.h }, p2/Z, [x27, x9, LSL #1]\n" - "ld1h { z13.h }, p2/Z, [x25, x9, LSL #1]\n" - "fmla z28.h, p3/M, z3.h, z14.h\n" - "fmla z29.h, p3/M, z0.h, z16.h\n" - "ldr x24, [x15, #0x58]\n" - "ldr x20, [x15, #0x78]\n" - "fmla z28.h, p3/M, z4.h, z15.h\n" - "fmla z29.h, p3/M, z4.h, z11.h\n" - "ld1h { z14.h }, p2/Z, [x24, x9, LSL #1]\n" - "ldr x23, [x15, #0x60]\n" - "fmla z28.h, p3/M, z2.h, z16.h\n" - "fmla z29.h, p3/M, z5.h, z12.h\n" - "ldr x27, [x15, #0x80]\n" - "ld1h { z15.h }, p2/Z, [x23, x9, LSL #1]\n" - "movprfx z30, z19\n fmla z30.h, p3/M, z2.h, z9.h\n" - "movprfx z31, z19\n fmla z31.h, p3/M, z0.h, z9.h\n" - "ld1h { z12.h }, p2/Z, [x27, x9, LSL #1]\n" - "ldr x22, [x15, #0x68]\n" - "fmla z28.h, p3/M, z5.h, z13.h\n" - "fmla z29.h, p3/M, z3.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x20, x9, LSL #1]\n" - "ldr x26, [x15, #0x88]\n" - "fmla z30.h, p3/M, z3.h, z14.h\n" - "fmla z31.h, p3/M, z4.h, z13.h\n" - "ld1h { z11.h }, p2/Z, [x22, x9, LSL #1]\n" - "ld1h { z14.h }, p2/Z, [x26, x9, LSL #1]\n" - "fmla z30.h, p3/M, z0.h, z15.h\n" - "fmla z31.h, p3/M, z1.h, z12.h\n" + "movprfx z24, z20\n fmla z24.h, p3/M, z8.h, z9.h\n" + "movprfx z23, z20\n fmla z23.h, p3/M, z6.h, z9.h\n" + "ldr x21, [x15, #0x40]\n" + "ldr x20, [x15, #0x48]\n" + "fmla z24.h, p3/M, z0.h, z10.h\n" + "fmla z23.h, p3/M, z1.h, z12.h\n" + "ld1h { z18.h }, p2/Z, [x20, x9, LSL #1]\n" + "ldr x20, [x15, #0x50]\n" + "fmla z24.h, p3/M, z1.h, z11.h\n" + "fmla z23.h, p3/M, z2.h, z13.h\n" + "ld1h { z17.h }, p2/Z, [x21, x9, LSL #1]\n" + "ld1h { z19.h }, p2/Z, [x20, x9, LSL #1]\n" + "fmla z24.h, p3/M, z3.h, z14.h\n" + "fmla z23.h, p3/M, z0.h, z16.h\n" + "ldr x20, [x15, #0x58]\n" + "ldr x22, [x15, #0x78]\n" + "fmla z24.h, p3/M, z4.h, z15.h\n" + "fmla z23.h, p3/M, z4.h, z17.h\n" + "ld1h { z17.h }, p2/Z, [x20, x9, LSL #1]\n" + "ldr x21, [x15, #0x60]\n" + "fmla z24.h, p3/M, z2.h, z16.h\n" + "fmla z23.h, p3/M, z5.h, z18.h\n" + "ldr x20, [x15, #0x80]\n" + "ld1h { z18.h }, p2/Z, [x21, x9, LSL #1]\n" + "movprfx z22, z20\n fmla z22.h, p3/M, z2.h, z9.h\n" + "movprfx z21, z20\n fmla z21.h, p3/M, z0.h, z9.h\n" + "ld1h { z20.h }, p2/Z, [x20, x9, LSL #1]\n" + "ldr x21, [x15, #0x68]\n" + "fmla z24.h, p3/M, z5.h, z19.h\n" + "fmla z23.h, p3/M, z3.h, z19.h\n" + "ld1h { z16.h }, p2/Z, [x22, x9, LSL #1]\n" + "ldr x20, [x15, #0x88]\n" + "fmla z22.h, p3/M, z3.h, z17.h\n" + "fmla z21.h, p3/M, z4.h, z16.h\n" + "ld1h { z17.h }, p2/Z, [x21, x9, LSL #1]\n" + "ld1h { z16.h }, p2/Z, [x20, x9, LSL #1]\n" + "fmla z22.h, p3/M, z0.h, z18.h\n" + "fmla z21.h, p3/M, z1.h, z20.h\n" "ldr x21, [x15, #0x70]\n" - "ldr x24, [x15, #0x98]\n" - "fmla z30.h, p3/M, z4.h, z11.h\n" - "fmla z31.h, p3/M, z5.h, z14.h\n" + "ldr x20, [x15, #0x98]\n" + "fmla z22.h, p3/M, z4.h, z17.h\n" + "fmla z21.h, p3/M, z5.h, z16.h\n" "ld1h { z16.h }, p2/Z, [x21, x9, LSL #1]\n" - "ld1h { z11.h }, p2/Z, [x24, x9, LSL #1]\n" - "fmla z28.h, p3/M, z6.h, z15.h\n" - "ldr x25, [x15, #0x90]\n" - "ldr x22, [x15, #0xa8]\n" - "fmla z30.h, p3/M, z1.h, z16.h\n" - "fmla z31.h, p3/M, z2.h, z11.h\n" - "fmla z28.h, p3/M, z7.h, z16.h\n" - "ld1h { z15.h }, p2/Z, [x25, x9, LSL #1]\n" - "ld1h { z16.h }, p2/Z, [x22, x9, LSL #1]\n" - "ldr x23, [x15, #0xa0]\n" - "ldr x21, [x15, #0xb0]\n" - "fmla z30.h, p3/M, z6.h, z15.h\n" - "fmla z31.h, p3/M, z3.h, z16.h\n" - "ld1h { z13.h }, p2/Z, [x23, x9, LSL #1]\n" - "ld1h { z14.h }, p2/Z, [x21, x9, LSL #1]\n" - "fmla z30.h, p3/M, z7.h, z13.h\n" - "fmla z31.h, p3/M, z7.h, z14.h\n" + "ld1h { z19.h }, p2/Z, [x20, x9, LSL #1]\n" + "fmla z24.h, p3/M, z6.h, z18.h\n" + "ldr x21, [x15, #0x90]\n" + "ldr x20, [x15, #0xa8]\n" + "fmla z22.h, p3/M, z1.h, z16.h\n" + "fmla z21.h, p3/M, z2.h, z19.h\n" + "fmla z24.h, p3/M, z7.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x21, x9, LSL #1]\n" + "ld1h { z18.h }, p2/Z, [x20, x9, LSL #1]\n" + "ldr x21, [x15, #0xa0]\n" + "ldr x20, [x15, #0xb0]\n" + "fmla z22.h, p3/M, z6.h, z16.h\n" + "fmla z21.h, p3/M, z3.h, z18.h\n" + "ld1h { z17.h }, p2/Z, [x21, x9, LSL #1]\n" + "ld1h { z16.h }, p2/Z, [x20, x9, LSL #1]\n" + "fmla z22.h, p3/M, z7.h, z17.h\n" + "fmla z21.h, p3/M, z7.h, z16.h\n" "ldr x20, [x15, #0xb8]\n" - "fmla z29.h, p3/M, z7.h, z12.h\n" - "ld1h { z15.h }, p2/Z, [x20, x9, LSL #1]\n" - "fmla z30.h, p3/M, z5.h, z16.h\n" - "ldr x27, [x15, #0xc0]\n" - "fmla z31.h, p3/M, z6.h, z15.h\n" - "fmla z29.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27, x9, LSL #1]\n" - "fmla z30.h, p3/M, z8.h, z15.h\n" - "fmla z31.h, p3/M, z8.h, z11.h\n" + "fmla z23.h, p3/M, z7.h, z20.h\n" + "ld1h { z17.h }, p2/Z, [x20, x9, LSL #1]\n" + "fmla z22.h, p3/M, z5.h, z18.h\n" + "ldr x20, [x15, #0xc0]\n" + "fmla z21.h, p3/M, z6.h, z17.h\n" + "fmla z23.h, p3/M, z8.h, z19.h\n" + "ld1h { z16.h }, p2/Z, [x20, x9, LSL #1]\n" + "fmla z22.h, p3/M, z8.h, z17.h\n" + "fmla z21.h, p3/M, z8.h, z16.h\n" "whilelt p1.h, x14, %x[n_channels]\n" "ldp x27, x26, [x15, #0x0]\n" "ldp x25, x24, [x15, #0x10]\n" "ldp x23, x22, [x15, #0x20]\n" "inch x9\n" - "fmax z28.h, p3/M, z28.h, z18.h\n" + "fmax z24.h, p3/M, z24.h, z26.h\n" "ldp x21, x20, [x15, #0x30]\n" "ld1h { z9.h }, p1/Z, [x27, x14, LSL #1]\n" - "fmax z29.h, p3/M, z29.h, z18.h\n" - "fmax z30.h, p3/M, z30.h, z18.h\n" + "fmax z23.h, p3/M, z23.h, z26.h\n" + "fmax z22.h, p3/M, z22.h, z26.h\n" "ld1h { z10.h }, p1/Z, [x26, x14, LSL #1]\n" "ld1h { z11.h }, p1/Z, [x25, x14, LSL #1]\n" - "fmax z31.h, p3/M, z31.h, z18.h\n" + "fmax z21.h, p3/M, z21.h, z26.h\n" "inch x28\n" "ld1h { z12.h }, p1/Z, [x24, x14, LSL #1]\n" "ld1h { z13.h }, p1/Z, [x23, x14, LSL #1]\n" @@ -216,122 +216,122 @@ void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( "whilelt p2.h, x9, %x[n_channels]\n" "ld1h { z14.h }, p1/Z, [x22, x14, LSL #1]\n" "ld1h { z15.h }, p1/Z, [x21, x14, LSL #1]\n" - "fmin z28.h, p3/M, z28.h, z17.h\n" - "fmin z29.h, p3/M, z29.h, z17.h\n" + "fmin z24.h, p3/M, z24.h, z25.h\n" + "fmin z23.h, p3/M, z23.h, z25.h\n" "ld1h { z16.h }, p1/Z, [x20, x14, LSL #1]\n" "inch x14\n" - "ld1h { z19.h }, p3/Z, [x16]\n" + "ld1h { z20.h }, p3/Z, [x16]\n" "cmp x14, %x[n_channels]\n" "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n" "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n" - "fmin z30.h, p3/M, z30.h, z17.h\n" - "fmin z31.h, p3/M, z31.h, z17.h\n" + "fmin z22.h, p3/M, z22.h, z25.h\n" + "fmin z21.h, p3/M, z21.h, z25.h\n" "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n" "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n" - "st1h { z28.h }, p0, [x13, x28, LSL #1]\n" + "st1h { z24.h }, p0, [x13, x28, LSL #1]\n" "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n" "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n" - "st1h { z29.h }, p0, [x12, x28, LSL #1]\n" + "st1h { z23.h }, p0, [x12, x28, LSL #1]\n" "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n" "addvl x16, x16, #16\n" - "st1h { z30.h }, p0, [x11, x28, LSL #1]\n" + "st1h { z22.h }, p0, [x11, x28, LSL #1]\n" "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n" - "st1h { z31.h }, p0, [x10, x28, LSL #1]\n" + "st1h { z21.h }, p0, [x10, x28, LSL #1]\n" "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n" "addvl x16, x16, #-6\n" "blt 1b\n" "2:" // Channel tail - "movprfx z28, z19\n fmla z28.h, p3/M, z8.h, z9.h\n" - "movprfx z29, z19\n fmla z29.h, p3/M, z6.h, z9.h\n" - "ldr x27, [x15, #0x40]\n" - "ldr x26, [x15, #0x48]\n" - "fmla z28.h, p3/M, z0.h, z10.h\n" - "fmla z29.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x26, x9, LSL #1]\n" - "ldr x25, [x15, #0x50]\n" - "fmla z28.h, p3/M, z1.h, z11.h\n" - "fmla z29.h, p3/M, z2.h, z13.h\n" - "ld1h { z11.h }, p2/Z, [x27, x9, LSL #1]\n" - "ld1h { z13.h }, p2/Z, [x25, x9, LSL #1]\n" - "fmla z28.h, p3/M, z3.h, z14.h\n" - "fmla z29.h, p3/M, z0.h, z16.h\n" - "ldr x24, [x15, #0x58]\n" - "ldr x20, [x15, #0x78]\n" - "fmla z28.h, p3/M, z4.h, z15.h\n" - "fmla z29.h, p3/M, z4.h, z11.h\n" - "ld1h { z14.h }, p2/Z, [x24, x9, LSL #1]\n" - "ldr x23, [x15, #0x60]\n" - "fmla z28.h, p3/M, z2.h, z16.h\n" - "fmla z29.h, p3/M, z5.h, z12.h\n" - "ldr x27, [x15, #0x80]\n" - "ld1h { z15.h }, p2/Z, [x23, x9, LSL #1]\n" - "movprfx z30, z19\n fmla z30.h, p3/M, z2.h, z9.h\n" - "movprfx z31, z19\n fmla z31.h, p3/M, z0.h, z9.h\n" - "ld1h { z12.h }, p2/Z, [x27, x9, LSL #1]\n" - "ldr x22, [x15, #0x68]\n" - "fmla z28.h, p3/M, z5.h, z13.h\n" - "fmla z29.h, p3/M, z3.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x20, x9, LSL #1]\n" - "ldr x26, [x15, #0x88]\n" - "fmla z30.h, p3/M, z3.h, z14.h\n" - "fmla z31.h, p3/M, z4.h, z13.h\n" - "ld1h { z11.h }, p2/Z, [x22, x9, LSL #1]\n" - "ld1h { z14.h }, p2/Z, [x26, x9, LSL #1]\n" - "fmla z30.h, p3/M, z0.h, z15.h\n" - "fmla z31.h, p3/M, z1.h, z12.h\n" + "movprfx z24, z20\n fmla z24.h, p3/M, z8.h, z9.h\n" + "movprfx z23, z20\n fmla z23.h, p3/M, z6.h, z9.h\n" + "ldr x21, [x15, #0x40]\n" + "ldr x20, [x15, #0x48]\n" + "fmla z24.h, p3/M, z0.h, z10.h\n" + "fmla z23.h, p3/M, z1.h, z12.h\n" + "ld1h { z18.h }, p2/Z, [x20, x9, LSL #1]\n" + "ldr x20, [x15, #0x50]\n" + "fmla z24.h, p3/M, z1.h, z11.h\n" + "fmla z23.h, p3/M, z2.h, z13.h\n" + "ld1h { z17.h }, p2/Z, [x21, x9, LSL #1]\n" + "ld1h { z19.h }, p2/Z, [x20, x9, LSL #1]\n" + "fmla z24.h, p3/M, z3.h, z14.h\n" + "fmla z23.h, p3/M, z0.h, z16.h\n" + "ldr x20, [x15, #0x58]\n" + "ldr x22, [x15, #0x78]\n" + "fmla z24.h, p3/M, z4.h, z15.h\n" + "fmla z23.h, p3/M, z4.h, z17.h\n" + "ld1h { z17.h }, p2/Z, [x20, x9, LSL #1]\n" + "ldr x21, [x15, #0x60]\n" + "fmla z24.h, p3/M, z2.h, z16.h\n" + "fmla z23.h, p3/M, z5.h, z18.h\n" + "ldr x20, [x15, #0x80]\n" + "ld1h { z18.h }, p2/Z, [x21, x9, LSL #1]\n" + "movprfx z22, z20\n fmla z22.h, p3/M, z2.h, z9.h\n" + "movprfx z21, z20\n fmla z21.h, p3/M, z0.h, z9.h\n" + "ld1h { z20.h }, p2/Z, [x20, x9, LSL #1]\n" + "ldr x21, [x15, #0x68]\n" + "fmla z24.h, p3/M, z5.h, z19.h\n" + "fmla z23.h, p3/M, z3.h, z19.h\n" + "ld1h { z16.h }, p2/Z, [x22, x9, LSL #1]\n" + "ldr x20, [x15, #0x88]\n" + "fmla z22.h, p3/M, z3.h, z17.h\n" + "fmla z21.h, p3/M, z4.h, z16.h\n" + "ld1h { z17.h }, p2/Z, [x21, x9, LSL #1]\n" + "ld1h { z16.h }, p2/Z, [x20, x9, LSL #1]\n" + "fmla z22.h, p3/M, z0.h, z18.h\n" + "fmla z21.h, p3/M, z1.h, z20.h\n" "ldr x21, [x15, #0x70]\n" - "ldr x24, [x15, #0x98]\n" - "fmla z30.h, p3/M, z4.h, z11.h\n" - "fmla z31.h, p3/M, z5.h, z14.h\n" + "ldr x20, [x15, #0x98]\n" + "fmla z22.h, p3/M, z4.h, z17.h\n" + "fmla z21.h, p3/M, z5.h, z16.h\n" "ld1h { z16.h }, p2/Z, [x21, x9, LSL #1]\n" - "ld1h { z11.h }, p2/Z, [x24, x9, LSL #1]\n" - "fmla z28.h, p3/M, z6.h, z15.h\n" - "ldr x25, [x15, #0x90]\n" - "ldr x22, [x15, #0xa8]\n" - "fmla z30.h, p3/M, z1.h, z16.h\n" - "fmla z31.h, p3/M, z2.h, z11.h\n" - "fmla z28.h, p3/M, z7.h, z16.h\n" - "ld1h { z15.h }, p2/Z, [x25, x9, LSL #1]\n" - "ld1h { z16.h }, p2/Z, [x22, x9, LSL #1]\n" - "ldr x23, [x15, #0xa0]\n" - "ldr x21, [x15, #0xb0]\n" - "fmla z30.h, p3/M, z6.h, z15.h\n" - "fmla z31.h, p3/M, z3.h, z16.h\n" - "ld1h { z13.h }, p2/Z, [x23, x9, LSL #1]\n" - "ld1h { z14.h }, p2/Z, [x21, x9, LSL #1]\n" - "fmla z30.h, p3/M, z7.h, z13.h\n" - "fmla z31.h, p3/M, z7.h, z14.h\n" + "ld1h { z19.h }, p2/Z, [x20, x9, LSL #1]\n" + "fmla z24.h, p3/M, z6.h, z18.h\n" + "ldr x21, [x15, #0x90]\n" + "ldr x20, [x15, #0xa8]\n" + "fmla z22.h, p3/M, z1.h, z16.h\n" + "fmla z21.h, p3/M, z2.h, z19.h\n" + "fmla z24.h, p3/M, z7.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x21, x9, LSL #1]\n" + "ld1h { z18.h }, p2/Z, [x20, x9, LSL #1]\n" + "ldr x21, [x15, #0xa0]\n" + "ldr x20, [x15, #0xb0]\n" + "fmla z22.h, p3/M, z6.h, z16.h\n" + "fmla z21.h, p3/M, z3.h, z18.h\n" + "ld1h { z17.h }, p2/Z, [x21, x9, LSL #1]\n" + "ld1h { z16.h }, p2/Z, [x20, x9, LSL #1]\n" + "fmla z22.h, p3/M, z7.h, z17.h\n" + "fmla z21.h, p3/M, z7.h, z16.h\n" "ldr x20, [x15, #0xb8]\n" - "fmla z29.h, p3/M, z7.h, z12.h\n" - "ld1h { z15.h }, p2/Z, [x20, x9, LSL #1]\n" - "fmla z30.h, p3/M, z5.h, z16.h\n" - "ldr x27, [x15, #0xc0]\n" - "fmla z31.h, p3/M, z6.h, z15.h\n" - "fmla z29.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27, x9, LSL #1]\n" - "fmla z30.h, p3/M, z8.h, z15.h\n" - "fmla z31.h, p3/M, z8.h, z11.h\n" + "fmla z23.h, p3/M, z7.h, z20.h\n" + "ld1h { z17.h }, p2/Z, [x20, x9, LSL #1]\n" + "fmla z22.h, p3/M, z5.h, z18.h\n" + "ldr x20, [x15, #0xc0]\n" + "fmla z21.h, p3/M, z6.h, z17.h\n" + "fmla z23.h, p3/M, z8.h, z19.h\n" + "ld1h { z16.h }, p2/Z, [x20, x9, LSL #1]\n" + "fmla z22.h, p3/M, z8.h, z17.h\n" + "fmla z21.h, p3/M, z8.h, z16.h\n" "inch x28\n" "mov p0.b, p2.b\n" - "fmax z28.h, p3/M, z28.h, z18.h\n" - "fmax z29.h, p3/M, z29.h, z18.h\n" - "fmax z30.h, p3/M, z30.h, z18.h\n" - "fmax z31.h, p3/M, z31.h, z18.h\n" - "fmin z28.h, p3/M, z28.h, z17.h\n" - "fmin z29.h, p3/M, z29.h, z17.h\n" - "st1h { z28.h }, p0, [x13, x28, LSL #1]\n" - "fmin z30.h, p3/M, z30.h, z17.h\n" - "fmin z31.h, p3/M, z31.h, z17.h\n" - "st1h { z29.h }, p0, [x12, x28, LSL #1]\n" - "st1h { z30.h }, p0, [x11, x28, LSL #1]\n" - "st1h { z31.h }, p0, [x10, x28, LSL #1]\n" + "fmax z24.h, p3/M, z24.h, z26.h\n" + "fmax z23.h, p3/M, z23.h, z26.h\n" + "fmax z22.h, p3/M, z22.h, z26.h\n" + "fmax z21.h, p3/M, z21.h, z26.h\n" + "fmin z24.h, p3/M, z24.h, z25.h\n" + "fmin z23.h, p3/M, z23.h, z25.h\n" + "st1h { z24.h }, p0, [x13, x28, LSL #1]\n" + "fmin z22.h, p3/M, z22.h, z25.h\n" + "fmin z21.h, p3/M, z21.h, z25.h\n" + "st1h { z23.h }, p0, [x12, x28, LSL #1]\n" + "st1h { z22.h }, p0, [x11, x28, LSL #1]\n" + "st1h { z21.h }, p0, [x10, x28, LSL #1]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp index f5d4189a4711eed623405d2a233c138a7700c644..abdfac5a3f8e0674704f557c9ad0a179f753cd8f 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,19 +22,19 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include #pragma once -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { -void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); -void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); +void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const input_ptrs, __fp16 *const *const outptrs, const void *params, unsigned int n_channels, const __fp16 activation_min, const __fp16 activation_max); +void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int n_tile_rows, const unsigned int n_tile_cols, const __fp16 *inptr, int64_t ld_input_row, int64_t ld_input_col, __fp16 *outptr, int64_t ld_output_row, int64_t ld_output_col, const void *params, unsigned int n_channels, const __fp16 activation_min, const __fp16 activation_max); class sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirstStrategy<__fp16, __fp16, __fp16, __fp16> { @@ -57,7 +57,7 @@ class sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirst constexpr static unsigned int output_cols = 2; sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst(const CPUInfo *) - : DepthwiseDepthfirstStrategy<__fp16, __fp16, __fp16, __fp16>(2, 5, 1) {} + : Parent(output_rows, output_cols, kernel_rows, kernel_cols, stride_rows, stride_cols) {} arm_gemm::VLType get_vl_type(void) const override { return vl_type; } @@ -68,4 +68,4 @@ class sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirst } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp index e6bfea17904ee8bc68331584ec952ae1cd258dd8..fdbee679262f15b92b28edcb51615a14d27138da 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -113,14 +113,14 @@ void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "madd x20, x8, x16, x20\n" // offset += tile_j * ld_output_col "add x9, x11, x23, LSL #1\n" "add x28, x15, x17\n" - "ld1rh { z18.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rh { z15.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" "mul x20, x20, x24\n" // offset *= output_tile_size "whilelt p2.h, XZR, %x[n_channels]\n" "add x27, x9, x23, LSL #1\n" - "ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "ld1rh { z28.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "add x26, x28, x17\n" "add x25, x27, x23, LSL #1\n" - "ld1h { z16.h }, p3/Z, [x10]\n" + "ld1h { z29.h }, p3/Z, [x10]\n" "ld1h { z0.h }, p3/Z, [x10, #1, MUL VL]\n" "add x24, x26, x17\n" "add x13, x13, x20, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16) @@ -146,378 +146,378 @@ void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "ld1h { z14.h }, p2/Z, [x9]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z5.h\n" - "movprfx z29, z16\n fmla z29.h, p3/M, z0.h, z6.h\n" - "ld1h { z5.h }, p2/Z, [x11, x28, LSL #1]\n" + "movprfx z27, z29\n fmla z27.h, p3/M, z0.h, z5.h\n" + "movprfx z31, z29\n fmla z31.h, p3/M, z0.h, z6.h\n" + "ld1h { z24.h }, p2/Z, [x11, x28, LSL #1]\n" "whilelt p1.h, x12, %x[n_channels]\n" - "movprfx z30, z16\n fmla z30.h, p3/M, z0.h, z7.h\n" - "movprfx z31, z16\n fmla z31.h, p3/M, z0.h, z8.h\n" - "ld1h { z0.h }, p3/Z, [x10]\n" + "movprfx z26, z29\n fmla z26.h, p3/M, z0.h, z7.h\n" + "movprfx z30, z29\n fmla z30.h, p3/M, z0.h, z8.h\n" + "ld1h { z18.h }, p3/Z, [x10]\n" "inch x21\n" - "fmla z28.h, p3/M, z1.h, z6.h\n" - "fmla z29.h, p3/M, z1.h, z9.h\n" - "ld1h { z6.h }, p2/Z, [x11, x26, LSL #1]\n" + "fmla z27.h, p3/M, z1.h, z6.h\n" + "fmla z31.h, p3/M, z1.h, z9.h\n" + "ld1h { z23.h }, p2/Z, [x11, x26, LSL #1]\n" "inch x12\n" - "fmla z30.h, p3/M, z1.h, z8.h\n" - "fmla z31.h, p3/M, z1.h, z13.h\n" - "ld1h { z1.h }, p3/Z, [x10, #1, MUL VL]\n" + "fmla z26.h, p3/M, z1.h, z8.h\n" + "fmla z30.h, p3/M, z1.h, z13.h\n" + "ld1h { z22.h }, p3/Z, [x10, #1, MUL VL]\n" "mov p0.b, p2.b\n" - "fmla z28.h, p3/M, z2.h, z9.h\n" - "fmla z29.h, p3/M, z2.h, z11.h\n" - "ld1h { z9.h }, p2/Z, [x14, x24, LSL #1]\n" + "fmla z27.h, p3/M, z2.h, z9.h\n" + "fmla z31.h, p3/M, z2.h, z11.h\n" + "ld1h { z16.h }, p2/Z, [x14, x24, LSL #1]\n" "addvl x14, x14, #1\n" - "fmla z30.h, p3/M, z2.h, z13.h\n" - "fmla z31.h, p3/M, z2.h, z5.h\n" - "ld1h { z2.h }, p3/Z, [x10, #2, MUL VL]\n" + "fmla z26.h, p3/M, z2.h, z13.h\n" + "fmla z30.h, p3/M, z2.h, z24.h\n" + "ld1h { z20.h }, p3/Z, [x10, #2, MUL VL]\n" "addvl x11, x11, #1\n" - "fmla z28.h, p3/M, z3.h, z11.h\n" - "fmla z29.h, p3/M, z3.h, z12.h\n" - "ld1h { z11.h }, p2/Z, [x9, x17, LSL #1]\n" + "fmla z27.h, p3/M, z3.h, z11.h\n" + "fmla z31.h, p3/M, z3.h, z12.h\n" + "ld1h { z0.h }, p2/Z, [x9, x17, LSL #1]\n" "inch x20\n" - "fmla z30.h, p3/M, z3.h, z5.h\n" - "fmla z31.h, p3/M, z3.h, z6.h\n" - "ld1h { z3.h }, p3/Z, [x10, #3, MUL VL]\n" - "fmla z28.h, p3/M, z4.h, z12.h\n" - "fmla z29.h, p3/M, z4.h, z9.h\n" - "ld1h { z12.h }, p2/Z, [x9, x15, LSL #1]\n" - "ld1h { z9.h }, p2/Z, [x9, x28, LSL #1]\n" - "fmla z30.h, p3/M, z4.h, z6.h\n" - "fmla z31.h, p3/M, z4.h, z10.h\n" - "ld1h { z4.h }, p3/Z, [x10, #4, MUL VL]\n" - "fmla z28.h, p3/M, z0.h, z7.h\n" - "fmla z29.h, p3/M, z0.h, z8.h\n" + "fmla z26.h, p3/M, z3.h, z24.h\n" + "fmla z30.h, p3/M, z3.h, z23.h\n" + "ld1h { z17.h }, p3/Z, [x10, #3, MUL VL]\n" + "fmla z27.h, p3/M, z4.h, z12.h\n" + "fmla z31.h, p3/M, z4.h, z16.h\n" + "ld1h { z19.h }, p2/Z, [x9, x15, LSL #1]\n" + "ld1h { z5.h }, p2/Z, [x9, x28, LSL #1]\n" + "fmla z26.h, p3/M, z4.h, z23.h\n" + "fmla z30.h, p3/M, z4.h, z10.h\n" + "ld1h { z21.h }, p3/Z, [x10, #4, MUL VL]\n" + "fmla z27.h, p3/M, z18.h, z7.h\n" + "fmla z31.h, p3/M, z18.h, z8.h\n" "ld1h { z7.h }, p1/Z, [x11]\n" - "fmla z30.h, p3/M, z0.h, z14.h\n" - "fmla z31.h, p3/M, z0.h, z11.h\n" - "ld1h { z0.h }, p3/Z, [x10, #5, MUL VL]\n" - "fmla z28.h, p3/M, z1.h, z8.h\n" - "fmla z29.h, p3/M, z1.h, z13.h\n" - "ld1h { z8.h }, p2/Z, [x9, x24, LSL #1]\n" - "fmla z30.h, p3/M, z1.h, z11.h\n" - "fmla z31.h, p3/M, z1.h, z12.h\n" - "ld1h { z1.h }, p3/Z, [x10, #6, MUL VL]\n" - "fmla z28.h, p3/M, z2.h, z13.h\n" - "fmla z29.h, p3/M, z2.h, z5.h\n" - "ld1h { z13.h }, p2/Z, [x9, x26, LSL #1]\n" + "fmla z26.h, p3/M, z18.h, z14.h\n" + "fmla z30.h, p3/M, z18.h, z0.h\n" + "ld1h { z18.h }, p3/Z, [x10, #5, MUL VL]\n" + "fmla z27.h, p3/M, z22.h, z8.h\n" + "fmla z31.h, p3/M, z22.h, z13.h\n" + "ld1h { z3.h }, p2/Z, [x9, x24, LSL #1]\n" + "fmla z26.h, p3/M, z22.h, z0.h\n" + "fmla z30.h, p3/M, z22.h, z19.h\n" + "ld1h { z8.h }, p3/Z, [x10, #6, MUL VL]\n" + "fmla z27.h, p3/M, z20.h, z13.h\n" + "fmla z31.h, p3/M, z20.h, z24.h\n" + "ld1h { z2.h }, p2/Z, [x9, x26, LSL #1]\n" "addvl x9, x9, #1\n" - "fmla z30.h, p3/M, z2.h, z12.h\n" - "fmla z31.h, p3/M, z2.h, z9.h\n" - "ld1h { z2.h }, p3/Z, [x10, #7, MUL VL]\n" + "fmla z26.h, p3/M, z20.h, z19.h\n" + "fmla z30.h, p3/M, z20.h, z5.h\n" + "ld1h { z16.h }, p3/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "fmla z28.h, p3/M, z3.h, z5.h\n" - "fmla z29.h, p3/M, z3.h, z6.h\n" - "ld1h { z5.h }, p2/Z, [x27]\n" - "ld1h { z16.h }, p3/Z, [x10, #4, MUL VL]\n" - "fmla z30.h, p3/M, z3.h, z9.h\n" - "fmla z31.h, p3/M, z3.h, z13.h\n" - "ld1h { z3.h }, p3/Z, [x10, #-8, MUL VL]\n" - "fmla z28.h, p3/M, z4.h, z6.h\n" - "fmla z29.h, p3/M, z4.h, z10.h\n" - "ld1h { z6.h }, p2/Z, [x27, x17, LSL #1]\n" - "ld1h { z10.h }, p2/Z, [x27, x15, LSL #1]\n" - "fmla z30.h, p3/M, z4.h, z13.h\n" - "fmla z31.h, p3/M, z4.h, z8.h\n" - "ld1h { z4.h }, p3/Z, [x10, #-7, MUL VL]\n" - "fmla z28.h, p3/M, z0.h, z14.h\n" - "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z14.h }, p2/Z, [x27, x24, LSL #1]\n" - "fmla z30.h, p3/M, z0.h, z5.h\n" - "fmla z31.h, p3/M, z0.h, z6.h\n" - "ld1h { z0.h }, p3/Z, [x10, #-6, MUL VL]\n" - "fmla z28.h, p3/M, z1.h, z11.h\n" - "fmla z29.h, p3/M, z1.h, z12.h\n" - "ld1h { z11.h }, p2/Z, [x27, x28, LSL #1]\n" - "fmla z30.h, p3/M, z1.h, z6.h\n" - "fmla z31.h, p3/M, z1.h, z10.h\n" - "ld1h { z1.h }, p3/Z, [x10, #-5, MUL VL]\n" - "fmla z28.h, p3/M, z2.h, z12.h\n" - "fmla z29.h, p3/M, z2.h, z9.h\n" - "ld1h { z12.h }, p2/Z, [x27, x26, LSL #1]\n" + "fmla z27.h, p3/M, z17.h, z24.h\n" + "fmla z31.h, p3/M, z17.h, z23.h\n" + "ld1h { z25.h }, p2/Z, [x27]\n" + "ld1h { z29.h }, p3/Z, [x10, #4, MUL VL]\n" + "fmla z26.h, p3/M, z17.h, z5.h\n" + "fmla z30.h, p3/M, z17.h, z2.h\n" + "ld1h { z17.h }, p3/Z, [x10, #-8, MUL VL]\n" + "fmla z27.h, p3/M, z21.h, z23.h\n" + "fmla z31.h, p3/M, z21.h, z10.h\n" + "ld1h { z24.h }, p2/Z, [x27, x17, LSL #1]\n" + "ld1h { z22.h }, p2/Z, [x27, x15, LSL #1]\n" + "fmla z26.h, p3/M, z21.h, z2.h\n" + "fmla z30.h, p3/M, z21.h, z3.h\n" + "ld1h { z21.h }, p3/Z, [x10, #-7, MUL VL]\n" + "fmla z27.h, p3/M, z18.h, z14.h\n" + "fmla z31.h, p3/M, z18.h, z0.h\n" + "ld1h { z1.h }, p2/Z, [x27, x24, LSL #1]\n" + "fmla z26.h, p3/M, z18.h, z25.h\n" + "fmla z30.h, p3/M, z18.h, z24.h\n" + "ld1h { z23.h }, p3/Z, [x10, #-6, MUL VL]\n" + "fmla z27.h, p3/M, z8.h, z0.h\n" + "fmla z31.h, p3/M, z8.h, z19.h\n" + "ld1h { z0.h }, p2/Z, [x27, x28, LSL #1]\n" + "fmla z26.h, p3/M, z8.h, z24.h\n" + "fmla z30.h, p3/M, z8.h, z22.h\n" + "ld1h { z20.h }, p3/Z, [x10, #-5, MUL VL]\n" + "fmla z27.h, p3/M, z16.h, z19.h\n" + "fmla z31.h, p3/M, z16.h, z5.h\n" + "ld1h { z19.h }, p2/Z, [x27, x26, LSL #1]\n" "addvl x27, x27, #1\n" - "fmla z30.h, p3/M, z2.h, z10.h\n" - "fmla z31.h, p3/M, z2.h, z11.h\n" - "ld1h { z2.h }, p3/Z, [x10, #-4, MUL VL]\n" - "fmla z28.h, p3/M, z3.h, z9.h\n" - "fmla z29.h, p3/M, z3.h, z13.h\n" - "ld1h { z9.h }, p2/Z, [x25]\n" - "fmla z30.h, p3/M, z3.h, z11.h\n" - "fmla z31.h, p3/M, z3.h, z12.h\n" - "ld1h { z3.h }, p3/Z, [x10, #-3, MUL VL]\n" - "fmla z28.h, p3/M, z4.h, z13.h\n" - "fmla z29.h, p3/M, z4.h, z8.h\n" - "ld1h { z13.h }, p2/Z, [x25, x17, LSL #1]\n" + "fmla z26.h, p3/M, z16.h, z22.h\n" + "fmla z30.h, p3/M, z16.h, z0.h\n" + "ld1h { z18.h }, p3/Z, [x10, #-4, MUL VL]\n" + "fmla z27.h, p3/M, z17.h, z5.h\n" + "fmla z31.h, p3/M, z17.h, z2.h\n" + "ld1h { z16.h }, p2/Z, [x25]\n" + "fmla z26.h, p3/M, z17.h, z0.h\n" + "fmla z30.h, p3/M, z17.h, z19.h\n" + "ld1h { z17.h }, p3/Z, [x10, #-3, MUL VL]\n" + "fmla z27.h, p3/M, z21.h, z2.h\n" + "fmla z31.h, p3/M, z21.h, z3.h\n" + "ld1h { z4.h }, p2/Z, [x25, x17, LSL #1]\n" "ld1h { z8.h }, p2/Z, [x25, x26, LSL #1]\n" - "fmla z30.h, p3/M, z4.h, z12.h\n" - "fmla z31.h, p3/M, z4.h, z14.h\n" - "ld1h { z4.h }, p3/Z, [x10, #-2, MUL VL]\n" - "fmla z28.h, p3/M, z0.h, z5.h\n" - "fmla z29.h, p3/M, z0.h, z6.h\n" - "ld1h { z5.h }, p2/Z, [x25, x15, LSL #1]\n" - "fmla z30.h, p3/M, z0.h, z9.h\n" - "fmla z31.h, p3/M, z0.h, z13.h\n" - "ld1h { z0.h }, p3/Z, [x10, #-1, MUL VL]\n" - "fmla z28.h, p3/M, z1.h, z6.h\n" - "fmla z29.h, p3/M, z1.h, z10.h\n" - "ld1h { z6.h }, p2/Z, [x25, x28, LSL #1]\n" - "fmla z30.h, p3/M, z1.h, z13.h\n" - "fmla z31.h, p3/M, z1.h, z5.h\n" - "ld1h { z1.h }, p3/Z, [x10]\n" - "fmla z28.h, p3/M, z2.h, z10.h\n" - "fmla z29.h, p3/M, z2.h, z11.h\n" - "ld1h { z10.h }, p2/Z, [x25, x24, LSL #1]\n" + "fmla z26.h, p3/M, z21.h, z19.h\n" + "fmla z30.h, p3/M, z21.h, z1.h\n" + "ld1h { z13.h }, p3/Z, [x10, #-2, MUL VL]\n" + "fmla z27.h, p3/M, z23.h, z25.h\n" + "fmla z31.h, p3/M, z23.h, z24.h\n" + "ld1h { z25.h }, p2/Z, [x25, x15, LSL #1]\n" + "fmla z26.h, p3/M, z23.h, z16.h\n" + "fmla z30.h, p3/M, z23.h, z4.h\n" + "ld1h { z5.h }, p3/Z, [x10, #-1, MUL VL]\n" + "fmla z27.h, p3/M, z20.h, z24.h\n" + "fmla z31.h, p3/M, z20.h, z22.h\n" + "ld1h { z24.h }, p2/Z, [x25, x28, LSL #1]\n" + "fmla z26.h, p3/M, z20.h, z4.h\n" + "fmla z30.h, p3/M, z20.h, z25.h\n" + "ld1h { z23.h }, p3/Z, [x10]\n" + "fmla z27.h, p3/M, z18.h, z22.h\n" + "fmla z31.h, p3/M, z18.h, z0.h\n" + "ld1h { z22.h }, p2/Z, [x25, x24, LSL #1]\n" "addvl x25, x25, #1\n" - "fmla z30.h, p3/M, z2.h, z5.h\n" - "fmla z31.h, p3/M, z2.h, z6.h\n" - "ld1h { z2.h }, p3/Z, [x10, #1, MUL VL]\n" - "fmla z28.h, p3/M, z3.h, z11.h\n" - "fmla z29.h, p3/M, z3.h, z12.h\n" - "ld1h { z11.h }, p2/Z, [x23]\n" - "fmla z30.h, p3/M, z3.h, z6.h\n" - "fmla z31.h, p3/M, z3.h, z8.h\n" - "ld1h { z3.h }, p3/Z, [x10, #2, MUL VL]\n" - "fmla z28.h, p3/M, z4.h, z12.h\n" - "fmla z29.h, p3/M, z4.h, z14.h\n" - "ld1h { z12.h }, p2/Z, [x23, x17, LSL #1]\n" + "fmla z26.h, p3/M, z18.h, z25.h\n" + "fmla z30.h, p3/M, z18.h, z24.h\n" + "ld1h { z21.h }, p3/Z, [x10, #1, MUL VL]\n" + "fmla z27.h, p3/M, z17.h, z0.h\n" + "fmla z31.h, p3/M, z17.h, z19.h\n" + "ld1h { z18.h }, p2/Z, [x23]\n" + "fmla z26.h, p3/M, z17.h, z24.h\n" + "fmla z30.h, p3/M, z17.h, z8.h\n" + "ld1h { z20.h }, p3/Z, [x10, #2, MUL VL]\n" + "fmla z27.h, p3/M, z13.h, z19.h\n" + "fmla z31.h, p3/M, z13.h, z1.h\n" + "ld1h { z17.h }, p2/Z, [x23, x17, LSL #1]\n" "ld1h { z14.h }, p1/Z, [x9]\n" - "fmla z30.h, p3/M, z4.h, z8.h\n" - "fmla z31.h, p3/M, z4.h, z10.h\n" - "ld1h { z4.h }, p3/Z, [x10, #3, MUL VL]\n" - "fmla z28.h, p3/M, z0.h, z9.h\n" - "fmla z29.h, p3/M, z0.h, z13.h\n" - "ld1h { z9.h }, p2/Z, [x23, x15, LSL #1]\n" - "fmla z30.h, p3/M, z0.h, z11.h\n" - "fmla z31.h, p3/M, z0.h, z12.h\n" - "ld1h { z11.h }, p2/Z, [x23, x28, LSL #1]\n" + "fmla z26.h, p3/M, z13.h, z8.h\n" + "fmla z30.h, p3/M, z13.h, z22.h\n" + "ld1h { z19.h }, p3/Z, [x10, #3, MUL VL]\n" + "fmla z27.h, p3/M, z5.h, z16.h\n" + "fmla z31.h, p3/M, z5.h, z4.h\n" + "ld1h { z16.h }, p2/Z, [x23, x15, LSL #1]\n" + "fmla z26.h, p3/M, z5.h, z18.h\n" + "fmla z30.h, p3/M, z5.h, z17.h\n" + "ld1h { z18.h }, p2/Z, [x23, x28, LSL #1]\n" "ld1h { z0.h }, p3/Z, [x10, #5, MUL VL]\n" - "fmla z28.h, p3/M, z1.h, z13.h\n" - "fmla z29.h, p3/M, z1.h, z5.h\n" + "fmla z27.h, p3/M, z23.h, z4.h\n" + "fmla z31.h, p3/M, z23.h, z25.h\n" "ld1h { z13.h }, p1/Z, [x11, x15, LSL #1]\n" - "fmla z30.h, p3/M, z1.h, z12.h\n" - "fmla z31.h, p3/M, z1.h, z9.h\n" - "ld1h { z12.h }, p2/Z, [x23, x26, LSL #1]\n" + "fmla z26.h, p3/M, z23.h, z17.h\n" + "fmla z30.h, p3/M, z23.h, z16.h\n" + "ld1h { z17.h }, p2/Z, [x23, x26, LSL #1]\n" "ld1h { z1.h }, p3/Z, [x10, #6, MUL VL]\n" - "fmla z28.h, p3/M, z2.h, z5.h\n" - "fmla z29.h, p3/M, z2.h, z6.h\n" + "fmla z27.h, p3/M, z21.h, z25.h\n" + "fmla z31.h, p3/M, z21.h, z24.h\n" "ld1h { z5.h }, p1/Z, [x14]\n" - "fmla z30.h, p3/M, z2.h, z9.h\n" - "fmla z31.h, p3/M, z2.h, z11.h\n" - "ld1h { z9.h }, p2/Z, [x23, x24, LSL #1]\n" + "fmla z26.h, p3/M, z21.h, z16.h\n" + "fmla z30.h, p3/M, z21.h, z18.h\n" + "ld1h { z16.h }, p2/Z, [x23, x24, LSL #1]\n" "ld1h { z2.h }, p3/Z, [x10, #7, MUL VL]\n" - "fmla z28.h, p3/M, z3.h, z6.h\n" - "fmla z29.h, p3/M, z3.h, z8.h\n" + "fmla z27.h, p3/M, z20.h, z24.h\n" + "fmla z31.h, p3/M, z20.h, z8.h\n" "addvl x10, x10, #16\n" "whilelt p2.h, x21, %x[n_channels]\n" - "fmla z30.h, p3/M, z3.h, z11.h\n" - "fmla z31.h, p3/M, z3.h, z12.h\n" + "fmla z26.h, p3/M, z20.h, z18.h\n" + "fmla z30.h, p3/M, z20.h, z17.h\n" "cmp x12, %x[n_channels]\n" "addvl x23, x23, #1\n" - "fmla z28.h, p3/M, z4.h, z8.h\n" - "fmla z29.h, p3/M, z4.h, z10.h\n" - "fmax z28.h, p3/M, z28.h, z18.h\n" - "fmax z29.h, p3/M, z29.h, z18.h\n" - "fmla z30.h, p3/M, z4.h, z12.h\n" - "fmla z31.h, p3/M, z4.h, z9.h\n" - "fmax z30.h, p3/M, z30.h, z18.h\n" - "fmax z31.h, p3/M, z31.h, z18.h\n" - "fmin z28.h, p3/M, z28.h, z17.h\n" - "fmin z29.h, p3/M, z29.h, z17.h\n" + "fmla z27.h, p3/M, z19.h, z8.h\n" + "fmla z31.h, p3/M, z19.h, z22.h\n" + "fmax z27.h, p3/M, z27.h, z15.h\n" + "fmax z31.h, p3/M, z31.h, z15.h\n" + "fmla z26.h, p3/M, z19.h, z17.h\n" + "fmla z30.h, p3/M, z19.h, z16.h\n" + "fmax z26.h, p3/M, z26.h, z15.h\n" + "fmax z30.h, p3/M, z30.h, z15.h\n" + "fmin z27.h, p3/M, z27.h, z28.h\n" + "fmin z31.h, p3/M, z31.h, z28.h\n" "ld1h { z6.h }, p1/Z, [x14, x17, LSL #1]\n" "ld1h { z8.h }, p1/Z, [x11, x17, LSL #1]\n" - "fmin z30.h, p3/M, z30.h, z17.h\n" - "fmin z31.h, p3/M, z31.h, z17.h\n" + "fmin z26.h, p3/M, z26.h, z28.h\n" + "fmin z30.h, p3/M, z30.h, z28.h\n" "ld1h { z9.h }, p1/Z, [x14, x15, LSL #1]\n" "ld1h { z11.h }, p1/Z, [x14, x28, LSL #1]\n" "ld1h { z12.h }, p1/Z, [x14, x26, LSL #1]\n" "ld1h { z10.h }, p1/Z, [x11, x24, LSL #1]\n" - "st1h { z28.h }, p0, [x13]\n" - "st1h { z29.h }, p0, [x13, x16, LSL #1]\n" + "st1h { z27.h }, p0, [x13]\n" + "st1h { z31.h }, p0, [x13, x16, LSL #1]\n" "addvl x13, x13, #1\n" "ld1h { z3.h }, p3/Z, [x10, #-8, MUL VL]\n" "ld1h { z4.h }, p3/Z, [x10, #-7, MUL VL]\n" - "st1h { z30.h }, p0, [x22]\n" + "st1h { z26.h }, p0, [x22]\n" "addvl x10, x10, #-6\n" - "st1h { z31.h }, p0, [x22, x16, LSL #1]\n" + "st1h { z30.h }, p0, [x22, x16, LSL #1]\n" "addvl x22, x22, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z5.h\n" - "movprfx z29, z16\n fmla z29.h, p3/M, z0.h, z6.h\n" - "ld1h { z5.h }, p2/Z, [x11, x28, LSL #1]\n" + "movprfx z30, z29\n fmla z30.h, p3/M, z0.h, z5.h\n" + "movprfx z31, z29\n fmla z31.h, p3/M, z0.h, z6.h\n" + "ld1h { z22.h }, p2/Z, [x11, x28, LSL #1]\n" "ldr x8, [%x[params_struct], %[offsetof_args_tile_j]]\n" - "movprfx z30, z16\n fmla z30.h, p3/M, z0.h, z7.h\n" - "movprfx z31, z16\n fmla z31.h, p3/M, z0.h, z8.h\n" - "ld1h { z0.h }, p3/Z, [x10]\n" + "movprfx z5, z29\n fmla z5.h, p3/M, z0.h, z7.h\n" + "fmla z29.h, p3/M, z0.h, z8.h\n" + "ld1h { z20.h }, p3/Z, [x10]\n" "ldr x12, [%x[params_struct], %[offsetof_args_tile_i]]\n" - "fmla z28.h, p3/M, z1.h, z6.h\n" - "fmla z29.h, p3/M, z1.h, z9.h\n" + "fmla z30.h, p3/M, z1.h, z6.h\n" + "fmla z31.h, p3/M, z1.h, z9.h\n" "ld1h { z6.h }, p2/Z, [x11, x26, LSL #1]\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" - "fmla z30.h, p3/M, z1.h, z8.h\n" - "fmla z31.h, p3/M, z1.h, z13.h\n" - "ld1h { z1.h }, p3/Z, [x10, #1, MUL VL]\n" + "fmla z5.h, p3/M, z1.h, z8.h\n" + "fmla z29.h, p3/M, z1.h, z13.h\n" + "ld1h { z19.h }, p3/Z, [x10, #1, MUL VL]\n" "add x8, x8, #0x1\n" - "fmla z28.h, p3/M, z2.h, z9.h\n" - "fmla z29.h, p3/M, z2.h, z11.h\n" - "ld1h { z9.h }, p2/Z, [x14, x24, LSL #1]\n" + "fmla z30.h, p3/M, z2.h, z9.h\n" + "fmla z31.h, p3/M, z2.h, z11.h\n" + "ld1h { z16.h }, p2/Z, [x14, x24, LSL #1]\n" "cmp x8, x20\n" - "fmla z30.h, p3/M, z2.h, z13.h\n" - "fmla z31.h, p3/M, z2.h, z5.h\n" - "ld1h { z2.h }, p3/Z, [x10, #2, MUL VL]\n" + "fmla z5.h, p3/M, z2.h, z13.h\n" + "fmla z29.h, p3/M, z2.h, z22.h\n" + "ld1h { z18.h }, p3/Z, [x10, #2, MUL VL]\n" "add x21, x12, #0x1\n" - "fmla z28.h, p3/M, z3.h, z11.h\n" - "fmla z29.h, p3/M, z3.h, z12.h\n" - "ld1h { z11.h }, p2/Z, [x9, x17, LSL #1]\n" + "fmla z30.h, p3/M, z3.h, z11.h\n" + "fmla z31.h, p3/M, z3.h, z12.h\n" + "ld1h { z1.h }, p2/Z, [x9, x17, LSL #1]\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" - "fmla z30.h, p3/M, z3.h, z5.h\n" - "fmla z31.h, p3/M, z3.h, z6.h\n" - "ld1h { z3.h }, p3/Z, [x10, #3, MUL VL]\n" + "fmla z5.h, p3/M, z3.h, z22.h\n" + "fmla z29.h, p3/M, z3.h, z6.h\n" + "ld1h { z17.h }, p3/Z, [x10, #3, MUL VL]\n" "csel x12, x12, x21, LT\n" - "fmla z28.h, p3/M, z4.h, z12.h\n" - "fmla z29.h, p3/M, z4.h, z9.h\n" - "ld1h { z12.h }, p2/Z, [x9, x15, LSL #1]\n" - "ld1h { z9.h }, p2/Z, [x9, x28, LSL #1]\n" - "fmla z30.h, p3/M, z4.h, z6.h\n" - "fmla z31.h, p3/M, z4.h, z10.h\n" - "ld1h { z4.h }, p3/Z, [x10, #4, MUL VL]\n" + "fmla z30.h, p3/M, z4.h, z12.h\n" + "fmla z31.h, p3/M, z4.h, z16.h\n" + "ld1h { z0.h }, p2/Z, [x9, x15, LSL #1]\n" + "ld1h { z27.h }, p2/Z, [x9, x28, LSL #1]\n" + "fmla z5.h, p3/M, z4.h, z6.h\n" + "fmla z29.h, p3/M, z4.h, z10.h\n" + "ld1h { z16.h }, p3/Z, [x10, #4, MUL VL]\n" "mov p0.b, p2.b\n" - "fmla z28.h, p3/M, z0.h, z7.h\n" - "fmla z29.h, p3/M, z0.h, z8.h\n" + "fmla z30.h, p3/M, z20.h, z7.h\n" + "fmla z31.h, p3/M, z20.h, z8.h\n" "csel x8, x8, XZR, LT\n" "cmp x12, x20\n" - "fmla z30.h, p3/M, z0.h, z14.h\n" - "fmla z31.h, p3/M, z0.h, z11.h\n" - "ld1h { z0.h }, p3/Z, [x10, #5, MUL VL]\n" - "fmla z28.h, p3/M, z1.h, z8.h\n" - "fmla z29.h, p3/M, z1.h, z13.h\n" - "ld1h { z8.h }, p2/Z, [x9, x24, LSL #1]\n" - "fmla z30.h, p3/M, z1.h, z11.h\n" - "fmla z31.h, p3/M, z1.h, z12.h\n" - "ld1h { z1.h }, p3/Z, [x10, #6, MUL VL]\n" - "fmla z28.h, p3/M, z2.h, z13.h\n" - "fmla z29.h, p3/M, z2.h, z5.h\n" - "ld1h { z13.h }, p2/Z, [x9, x26, LSL #1]\n" - "fmla z30.h, p3/M, z2.h, z12.h\n" - "fmla z31.h, p3/M, z2.h, z9.h\n" - "ld1h { z2.h }, p3/Z, [x10, #7, MUL VL]\n" + "fmla z5.h, p3/M, z20.h, z14.h\n" + "fmla z29.h, p3/M, z20.h, z1.h\n" + "ld1h { z21.h }, p3/Z, [x10, #5, MUL VL]\n" + "fmla z30.h, p3/M, z19.h, z8.h\n" + "fmla z31.h, p3/M, z19.h, z13.h\n" + "ld1h { z26.h }, p2/Z, [x9, x24, LSL #1]\n" + "fmla z5.h, p3/M, z19.h, z1.h\n" + "fmla z29.h, p3/M, z19.h, z0.h\n" + "ld1h { z25.h }, p3/Z, [x10, #6, MUL VL]\n" + "fmla z30.h, p3/M, z18.h, z13.h\n" + "fmla z31.h, p3/M, z18.h, z22.h\n" + "ld1h { z24.h }, p2/Z, [x9, x26, LSL #1]\n" + "fmla z5.h, p3/M, z18.h, z0.h\n" + "fmla z29.h, p3/M, z18.h, z27.h\n" + "ld1h { z23.h }, p3/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "fmla z28.h, p3/M, z3.h, z5.h\n" - "fmla z29.h, p3/M, z3.h, z6.h\n" - "ld1h { z5.h }, p2/Z, [x27]\n" - "fmla z30.h, p3/M, z3.h, z9.h\n" - "fmla z31.h, p3/M, z3.h, z13.h\n" - "ld1h { z3.h }, p3/Z, [x10, #-8, MUL VL]\n" - "fmla z28.h, p3/M, z4.h, z6.h\n" - "fmla z29.h, p3/M, z4.h, z10.h\n" - "ld1h { z6.h }, p2/Z, [x27, x17, LSL #1]\n" - "ld1h { z10.h }, p2/Z, [x27, x15, LSL #1]\n" - "fmla z30.h, p3/M, z4.h, z13.h\n" - "fmla z31.h, p3/M, z4.h, z8.h\n" - "ld1h { z4.h }, p3/Z, [x10, #-7, MUL VL]\n" - "fmla z28.h, p3/M, z0.h, z14.h\n" - "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z14.h }, p2/Z, [x27, x24, LSL #1]\n" - "fmla z30.h, p3/M, z0.h, z5.h\n" - "fmla z31.h, p3/M, z0.h, z6.h\n" - "ld1h { z0.h }, p3/Z, [x10, #-6, MUL VL]\n" - "fmla z28.h, p3/M, z1.h, z11.h\n" - "fmla z29.h, p3/M, z1.h, z12.h\n" - "ld1h { z11.h }, p2/Z, [x27, x28, LSL #1]\n" - "fmla z30.h, p3/M, z1.h, z6.h\n" - "fmla z31.h, p3/M, z1.h, z10.h\n" - "ld1h { z1.h }, p3/Z, [x10, #-5, MUL VL]\n" - "fmla z28.h, p3/M, z2.h, z12.h\n" - "fmla z29.h, p3/M, z2.h, z9.h\n" - "ld1h { z12.h }, p2/Z, [x27, x26, LSL #1]\n" - "fmla z30.h, p3/M, z2.h, z10.h\n" - "fmla z31.h, p3/M, z2.h, z11.h\n" - "ld1h { z2.h }, p3/Z, [x10, #-4, MUL VL]\n" - "fmla z28.h, p3/M, z3.h, z9.h\n" - "fmla z29.h, p3/M, z3.h, z13.h\n" - "ld1h { z9.h }, p2/Z, [x25]\n" - "fmla z30.h, p3/M, z3.h, z11.h\n" - "fmla z31.h, p3/M, z3.h, z12.h\n" - "ld1h { z3.h }, p3/Z, [x10, #-3, MUL VL]\n" - "fmla z28.h, p3/M, z4.h, z13.h\n" - "fmla z29.h, p3/M, z4.h, z8.h\n" - "ld1h { z13.h }, p2/Z, [x25, x17, LSL #1]\n" - "ld1h { z8.h }, p2/Z, [x25, x26, LSL #1]\n" - "fmla z30.h, p3/M, z4.h, z12.h\n" - "fmla z31.h, p3/M, z4.h, z14.h\n" - "ld1h { z4.h }, p3/Z, [x10, #-2, MUL VL]\n" - "fmla z28.h, p3/M, z0.h, z5.h\n" - "fmla z29.h, p3/M, z0.h, z6.h\n" - "ld1h { z5.h }, p2/Z, [x25, x15, LSL #1]\n" - "fmla z30.h, p3/M, z0.h, z9.h\n" - "fmla z31.h, p3/M, z0.h, z13.h\n" - "ld1h { z0.h }, p3/Z, [x10, #-1, MUL VL]\n" - "fmla z28.h, p3/M, z1.h, z6.h\n" - "fmla z29.h, p3/M, z1.h, z10.h\n" - "ld1h { z6.h }, p2/Z, [x25, x28, LSL #1]\n" - "fmla z30.h, p3/M, z1.h, z13.h\n" - "fmla z31.h, p3/M, z1.h, z5.h\n" - "ld1h { z1.h }, p3/Z, [x10]\n" - "fmla z28.h, p3/M, z2.h, z10.h\n" - "fmla z29.h, p3/M, z2.h, z11.h\n" - "ld1h { z10.h }, p2/Z, [x25, x24, LSL #1]\n" - "fmla z30.h, p3/M, z2.h, z5.h\n" - "fmla z31.h, p3/M, z2.h, z6.h\n" - "ld1h { z2.h }, p3/Z, [x10, #1, MUL VL]\n" - "fmla z28.h, p3/M, z3.h, z11.h\n" - "fmla z29.h, p3/M, z3.h, z12.h\n" - "ld1h { z11.h }, p2/Z, [x23]\n" - "fmla z30.h, p3/M, z3.h, z6.h\n" - "fmla z31.h, p3/M, z3.h, z8.h\n" - "ld1h { z3.h }, p3/Z, [x10, #2, MUL VL]\n" - "fmla z28.h, p3/M, z4.h, z12.h\n" - "fmla z29.h, p3/M, z4.h, z14.h\n" - "ld1h { z12.h }, p2/Z, [x23, x17, LSL #1]\n" - "fmla z30.h, p3/M, z4.h, z8.h\n" - "fmla z31.h, p3/M, z4.h, z10.h\n" - "ld1h { z4.h }, p3/Z, [x10, #3, MUL VL]\n" - "fmla z28.h, p3/M, z0.h, z9.h\n" - "fmla z29.h, p3/M, z0.h, z13.h\n" - "ld1h { z9.h }, p2/Z, [x23, x15, LSL #1]\n" - "fmla z30.h, p3/M, z0.h, z11.h\n" - "fmla z31.h, p3/M, z0.h, z12.h\n" - "ld1h { z11.h }, p2/Z, [x23, x28, LSL #1]\n" - "fmla z28.h, p3/M, z1.h, z13.h\n" - "fmla z29.h, p3/M, z1.h, z5.h\n" - "fmla z30.h, p3/M, z1.h, z12.h\n" - "fmla z31.h, p3/M, z1.h, z9.h\n" - "ld1h { z12.h }, p2/Z, [x23, x26, LSL #1]\n" - "fmla z28.h, p3/M, z2.h, z5.h\n" - "fmla z29.h, p3/M, z2.h, z6.h\n" - "fmla z30.h, p3/M, z2.h, z9.h\n" - "fmla z31.h, p3/M, z2.h, z11.h\n" - "ld1h { z9.h }, p2/Z, [x23, x24, LSL #1]\n" - "fmla z28.h, p3/M, z3.h, z6.h\n" - "fmla z29.h, p3/M, z3.h, z8.h\n" - "fmla z30.h, p3/M, z3.h, z11.h\n" - "fmla z31.h, p3/M, z3.h, z12.h\n" - "fmla z28.h, p3/M, z4.h, z8.h\n" - "fmla z29.h, p3/M, z4.h, z10.h\n" - "fmax z28.h, p3/M, z28.h, z18.h\n" - "fmax z29.h, p3/M, z29.h, z18.h\n" - "fmla z30.h, p3/M, z4.h, z12.h\n" - "fmla z31.h, p3/M, z4.h, z9.h\n" - "fmax z30.h, p3/M, z30.h, z18.h\n" - "fmax z31.h, p3/M, z31.h, z18.h\n" - "fmin z28.h, p3/M, z28.h, z17.h\n" - "fmin z29.h, p3/M, z29.h, z17.h\n" - "st1h { z28.h }, p0, [x13]\n" - "fmin z30.h, p3/M, z30.h, z17.h\n" - "fmin z31.h, p3/M, z31.h, z17.h\n" - "st1h { z29.h }, p0, [x13, x16, LSL #1]\n" - "st1h { z30.h }, p0, [x22]\n" - "st1h { z31.h }, p0, [x22, x16, LSL #1]\n" + "fmla z30.h, p3/M, z17.h, z22.h\n" + "fmla z31.h, p3/M, z17.h, z6.h\n" + "ld1h { z22.h }, p2/Z, [x27]\n" + "fmla z5.h, p3/M, z17.h, z27.h\n" + "fmla z29.h, p3/M, z17.h, z24.h\n" + "ld1h { z20.h }, p3/Z, [x10, #-8, MUL VL]\n" + "fmla z30.h, p3/M, z16.h, z6.h\n" + "fmla z31.h, p3/M, z16.h, z10.h\n" + "ld1h { z19.h }, p2/Z, [x27, x17, LSL #1]\n" + "ld1h { z18.h }, p2/Z, [x27, x15, LSL #1]\n" + "fmla z5.h, p3/M, z16.h, z24.h\n" + "fmla z29.h, p3/M, z16.h, z26.h\n" + "ld1h { z16.h }, p3/Z, [x10, #-7, MUL VL]\n" + "fmla z30.h, p3/M, z21.h, z14.h\n" + "fmla z31.h, p3/M, z21.h, z1.h\n" + "ld1h { z17.h }, p2/Z, [x27, x24, LSL #1]\n" + "fmla z5.h, p3/M, z21.h, z22.h\n" + "fmla z29.h, p3/M, z21.h, z19.h\n" + "ld1h { z21.h }, p3/Z, [x10, #-6, MUL VL]\n" + "fmla z30.h, p3/M, z25.h, z1.h\n" + "fmla z31.h, p3/M, z25.h, z0.h\n" + "ld1h { z7.h }, p2/Z, [x27, x28, LSL #1]\n" + "fmla z5.h, p3/M, z25.h, z19.h\n" + "fmla z29.h, p3/M, z25.h, z18.h\n" + "ld1h { z10.h }, p3/Z, [x10, #-5, MUL VL]\n" + "fmla z30.h, p3/M, z23.h, z0.h\n" + "fmla z31.h, p3/M, z23.h, z27.h\n" + "ld1h { z11.h }, p2/Z, [x27, x26, LSL #1]\n" + "fmla z5.h, p3/M, z23.h, z18.h\n" + "fmla z29.h, p3/M, z23.h, z7.h\n" + "ld1h { z6.h }, p3/Z, [x10, #-4, MUL VL]\n" + "fmla z30.h, p3/M, z20.h, z27.h\n" + "fmla z31.h, p3/M, z20.h, z24.h\n" + "ld1h { z0.h }, p2/Z, [x25]\n" + "fmla z5.h, p3/M, z20.h, z7.h\n" + "fmla z29.h, p3/M, z20.h, z11.h\n" + "ld1h { z9.h }, p3/Z, [x10, #-3, MUL VL]\n" + "fmla z30.h, p3/M, z16.h, z24.h\n" + "fmla z31.h, p3/M, z16.h, z26.h\n" + "ld1h { z3.h }, p2/Z, [x25, x17, LSL #1]\n" + "ld1h { z27.h }, p2/Z, [x25, x26, LSL #1]\n" + "fmla z5.h, p3/M, z16.h, z11.h\n" + "fmla z29.h, p3/M, z16.h, z17.h\n" + "ld1h { z16.h }, p3/Z, [x10, #-2, MUL VL]\n" + "fmla z30.h, p3/M, z21.h, z22.h\n" + "fmla z31.h, p3/M, z21.h, z19.h\n" + "ld1h { z26.h }, p2/Z, [x25, x15, LSL #1]\n" + "fmla z5.h, p3/M, z21.h, z0.h\n" + "fmla z29.h, p3/M, z21.h, z3.h\n" + "ld1h { z25.h }, p3/Z, [x10, #-1, MUL VL]\n" + "fmla z30.h, p3/M, z10.h, z19.h\n" + "fmla z31.h, p3/M, z10.h, z18.h\n" + "ld1h { z24.h }, p2/Z, [x25, x28, LSL #1]\n" + "fmla z5.h, p3/M, z10.h, z3.h\n" + "fmla z29.h, p3/M, z10.h, z26.h\n" + "ld1h { z23.h }, p3/Z, [x10]\n" + "fmla z30.h, p3/M, z6.h, z18.h\n" + "fmla z31.h, p3/M, z6.h, z7.h\n" + "ld1h { z22.h }, p2/Z, [x25, x24, LSL #1]\n" + "fmla z5.h, p3/M, z6.h, z26.h\n" + "fmla z29.h, p3/M, z6.h, z24.h\n" + "ld1h { z21.h }, p3/Z, [x10, #1, MUL VL]\n" + "fmla z30.h, p3/M, z9.h, z7.h\n" + "fmla z31.h, p3/M, z9.h, z11.h\n" + "ld1h { z18.h }, p2/Z, [x23]\n" + "fmla z5.h, p3/M, z9.h, z24.h\n" + "fmla z29.h, p3/M, z9.h, z27.h\n" + "ld1h { z20.h }, p3/Z, [x10, #2, MUL VL]\n" + "fmla z30.h, p3/M, z16.h, z11.h\n" + "fmla z31.h, p3/M, z16.h, z17.h\n" + "ld1h { z17.h }, p2/Z, [x23, x17, LSL #1]\n" + "fmla z5.h, p3/M, z16.h, z27.h\n" + "fmla z29.h, p3/M, z16.h, z22.h\n" + "ld1h { z19.h }, p3/Z, [x10, #3, MUL VL]\n" + "fmla z30.h, p3/M, z25.h, z0.h\n" + "fmla z31.h, p3/M, z25.h, z3.h\n" + "ld1h { z16.h }, p2/Z, [x23, x15, LSL #1]\n" + "fmla z5.h, p3/M, z25.h, z18.h\n" + "fmla z29.h, p3/M, z25.h, z17.h\n" + "ld1h { z18.h }, p2/Z, [x23, x28, LSL #1]\n" + "fmla z30.h, p3/M, z23.h, z3.h\n" + "fmla z31.h, p3/M, z23.h, z26.h\n" + "fmla z5.h, p3/M, z23.h, z17.h\n" + "fmla z29.h, p3/M, z23.h, z16.h\n" + "ld1h { z17.h }, p2/Z, [x23, x26, LSL #1]\n" + "fmla z30.h, p3/M, z21.h, z26.h\n" + "fmla z31.h, p3/M, z21.h, z24.h\n" + "fmla z5.h, p3/M, z21.h, z16.h\n" + "fmla z29.h, p3/M, z21.h, z18.h\n" + "ld1h { z16.h }, p2/Z, [x23, x24, LSL #1]\n" + "fmla z30.h, p3/M, z20.h, z24.h\n" + "fmla z31.h, p3/M, z20.h, z27.h\n" + "fmla z5.h, p3/M, z20.h, z18.h\n" + "fmla z29.h, p3/M, z20.h, z17.h\n" + "fmla z30.h, p3/M, z19.h, z27.h\n" + "fmla z31.h, p3/M, z19.h, z22.h\n" + "fmax z30.h, p3/M, z30.h, z15.h\n" + "fmax z31.h, p3/M, z31.h, z15.h\n" + "fmla z5.h, p3/M, z19.h, z17.h\n" + "fmla z29.h, p3/M, z19.h, z16.h\n" + "fmax z5.h, p3/M, z5.h, z15.h\n" + "fmax z29.h, p3/M, z29.h, z15.h\n" + "fmin z30.h, p3/M, z30.h, z28.h\n" + "fmin z31.h, p3/M, z31.h, z28.h\n" + "st1h { z30.h }, p0, [x13]\n" + "fmin z5.h, p3/M, z5.h, z28.h\n" + "fmin z29.h, p3/M, z29.h, z28.h\n" + "st1h { z31.h }, p0, [x13, x16, LSL #1]\n" + "st1h { z5.h }, p0, [x22]\n" + "st1h { z29.h }, p0, [x22, x16, LSL #1]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp index 2e20b524d8b267ffa23fc0c770593b35fdfcbbc6..1ec0cb2cbf99661ac5dd7456163c72867121ad69 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -104,448 +104,448 @@ void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl( "mov x13, #0x0\n" "ldp x12, x11, [x20, #0x10]\n" "whilelt p3.h, XZR, %x[n_channels]\n" - "ldp x10, x9, [x16, #0x0]\n" - "cnth x28\n" + "ldp x21, x20, [x16, #0x0]\n" + "cnth x10\n" "ptrue p2.b\n" - "ldr x27, [%x[params_struct], %[offsetof_args_params]]\n" - "ld1h { z5.h }, p3/Z, [x10, x13, LSL #1]\n" - "cmp x28, %x[n_channels]\n" - "ld1h { z6.h }, p3/Z, [x9, x13, LSL #1]\n" - "ldp x26, x25, [x16, #0x10]\n" - "sub x24, XZR, x28\n" - "ldp x23, x22, [x16, #0x20]\n" - "ldp x21, x20, [x16, #0x30]\n" - "ldp x10, x9, [x16, #0x40]\n" - "ld1rh { z18.h }, p2/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "ld1rh { z17.h }, p2/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "ld1h { z16.h }, p2/Z, [x27]\n" - "ld1h { z0.h }, p2/Z, [x27, #1, MUL VL]\n" - "ld1h { z1.h }, p2/Z, [x27, #2, MUL VL]\n" - "ld1h { z2.h }, p2/Z, [x27, #3, MUL VL]\n" - "ld1h { z3.h }, p2/Z, [x27, #4, MUL VL]\n" - "ld1h { z4.h }, p2/Z, [x27, #5, MUL VL]\n" - "ld1h { z7.h }, p3/Z, [x26, x13, LSL #1]\n" - "addvl x27, x27, #6\n" - "ld1h { z8.h }, p3/Z, [x25, x13, LSL #1]\n" - "ld1h { z9.h }, p3/Z, [x23, x13, LSL #1]\n" - "ld1h { z13.h }, p3/Z, [x22, x13, LSL #1]\n" - "ld1h { z11.h }, p3/Z, [x21, x13, LSL #1]\n" - "ld1h { z12.h }, p3/Z, [x20, x13, LSL #1]\n" - "ld1h { z10.h }, p3/Z, [x10, x13, LSL #1]\n" - "ld1h { z14.h }, p3/Z, [x9, x13, LSL #1]\n" + "ldr x9, [%x[params_struct], %[offsetof_args_params]]\n" + "ld1h { z5.h }, p3/Z, [x21, x13, LSL #1]\n" + "cmp x10, %x[n_channels]\n" + "ld1h { z6.h }, p3/Z, [x20, x13, LSL #1]\n" + "ldp x27, x26, [x16, #0x10]\n" + "sub x28, XZR, x10\n" + "ldp x25, x24, [x16, #0x20]\n" + "ldp x23, x22, [x16, #0x30]\n" + "ldp x21, x20, [x16, #0x40]\n" + "ld1rh { z15.h }, p2/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rh { z28.h }, p2/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "ld1h { z29.h }, p2/Z, [x9]\n" + "ld1h { z0.h }, p2/Z, [x9, #1, MUL VL]\n" + "ld1h { z1.h }, p2/Z, [x9, #2, MUL VL]\n" + "ld1h { z2.h }, p2/Z, [x9, #3, MUL VL]\n" + "ld1h { z3.h }, p2/Z, [x9, #4, MUL VL]\n" + "ld1h { z4.h }, p2/Z, [x9, #5, MUL VL]\n" + "ld1h { z7.h }, p3/Z, [x27, x13, LSL #1]\n" + "addvl x9, x9, #6\n" + "ld1h { z8.h }, p3/Z, [x26, x13, LSL #1]\n" + "ld1h { z9.h }, p3/Z, [x25, x13, LSL #1]\n" + "ld1h { z13.h }, p3/Z, [x24, x13, LSL #1]\n" + "ld1h { z11.h }, p3/Z, [x23, x13, LSL #1]\n" + "ld1h { z12.h }, p3/Z, [x22, x13, LSL #1]\n" + "ld1h { z10.h }, p3/Z, [x21, x13, LSL #1]\n" + "ld1h { z14.h }, p3/Z, [x20, x13, LSL #1]\n" "bge 2f\n" "1:" // Channel loop - "movprfx z28, z16\n fmla z28.h, p2/M, z0.h, z5.h\n" - "movprfx z29, z16\n fmla z29.h, p2/M, z0.h, z6.h\n" - "ldr x26, [x16, #0x50]\n" - "ld1h { z5.h }, p3/Z, [x26, x13, LSL #1]\n" - "movprfx z30, z16\n fmla z30.h, p2/M, z0.h, z7.h\n" - "movprfx z31, z16\n fmla z31.h, p2/M, z0.h, z8.h\n" - "ldr x25, [x16, #0x58]\n" - "ldr x23, [x16, #0x60]\n" - "fmla z28.h, p2/M, z1.h, z6.h\n" - "fmla z29.h, p2/M, z1.h, z9.h\n" - "ld1h { z6.h }, p3/Z, [x25, x13, LSL #1]\n" - "ldr x22, [x16, #0x68]\n" - "fmla z30.h, p2/M, z1.h, z8.h\n" - "fmla z31.h, p2/M, z1.h, z13.h\n" - "ld1h { z0.h }, p2/Z, [x27]\n" - "ldr x21, [x16, #0x70]\n" - "fmla z28.h, p2/M, z2.h, z9.h\n" - "fmla z29.h, p2/M, z2.h, z11.h\n" - "ld1h { z9.h }, p3/Z, [x23, x13, LSL #1]\n" - "ld1h { z1.h }, p2/Z, [x27, #1, MUL VL]\n" - "fmla z30.h, p2/M, z2.h, z13.h\n" - "fmla z31.h, p2/M, z2.h, z5.h\n" - "ldr x20, [x16, #0x78]\n" - "ld1h { z2.h }, p2/Z, [x27, #2, MUL VL]\n" - "fmla z28.h, p2/M, z3.h, z11.h\n" - "fmla z29.h, p2/M, z3.h, z12.h\n" - "ld1h { z11.h }, p3/Z, [x22, x13, LSL #1]\n" - "ldr x10, [x16, #0x80]\n" - "fmla z30.h, p2/M, z3.h, z5.h\n" - "fmla z31.h, p2/M, z3.h, z6.h\n" - "ld1h { z3.h }, p2/Z, [x27, #3, MUL VL]\n" - "ldr x9, [x16, #0x88]\n" - "fmla z28.h, p2/M, z4.h, z12.h\n" - "fmla z29.h, p2/M, z4.h, z9.h\n" - "ld1h { z12.h }, p3/Z, [x21, x13, LSL #1]\n" - "ld1h { z9.h }, p3/Z, [x20, x13, LSL #1]\n" - "fmla z30.h, p2/M, z4.h, z6.h\n" - "fmla z31.h, p2/M, z4.h, z10.h\n" - "ld1h { z4.h }, p2/Z, [x27, #4, MUL VL]\n" - "ldr x26, [x16, #0x90]\n" - "fmla z28.h, p2/M, z0.h, z7.h\n" - "fmla z29.h, p2/M, z0.h, z8.h\n" - "ldr x25, [x16, #0x98]\n" - "ldr x23, [x16, #0xa0]\n" - "fmla z30.h, p2/M, z0.h, z14.h\n" - "fmla z31.h, p2/M, z0.h, z11.h\n" - "ld1h { z0.h }, p2/Z, [x27, #5, MUL VL]\n" - "ldr x22, [x16, #0xa8]\n" - "fmla z28.h, p2/M, z1.h, z8.h\n" - "fmla z29.h, p2/M, z1.h, z13.h\n" - "ld1h { z8.h }, p3/Z, [x9, x13, LSL #1]\n" - "ldr x21, [x16, #0xb0]\n" - "fmla z30.h, p2/M, z1.h, z11.h\n" - "fmla z31.h, p2/M, z1.h, z12.h\n" - "ld1h { z1.h }, p2/Z, [x27, #6, MUL VL]\n" - "ldr x20, [x16, #0xb8]\n" - "fmla z28.h, p2/M, z2.h, z13.h\n" - "fmla z29.h, p2/M, z2.h, z5.h\n" - "ld1h { z13.h }, p3/Z, [x10, x13, LSL #1]\n" - "ldr x10, [x16, #0xc0]\n" - "fmla z30.h, p2/M, z2.h, z12.h\n" - "fmla z31.h, p2/M, z2.h, z9.h\n" - "ld1h { z2.h }, p2/Z, [x27, #7, MUL VL]\n" - "addvl x27, x27, #16\n" - "fmla z28.h, p2/M, z3.h, z5.h\n" - "fmla z29.h, p2/M, z3.h, z6.h\n" - "ld1h { z5.h }, p3/Z, [x26, x13, LSL #1]\n" - "ldr x9, [x16, #0xc8]\n" - "fmla z30.h, p2/M, z3.h, z9.h\n" - "fmla z31.h, p2/M, z3.h, z13.h\n" - "ld1h { z3.h }, p2/Z, [x27, #-8, MUL VL]\n" - "ldr x26, [x16, #0xd0]\n" - "fmla z28.h, p2/M, z4.h, z6.h\n" - "fmla z29.h, p2/M, z4.h, z10.h\n" - "ld1h { z6.h }, p3/Z, [x25, x13, LSL #1]\n" - "ld1h { z10.h }, p3/Z, [x23, x13, LSL #1]\n" - "fmla z30.h, p2/M, z4.h, z13.h\n" - "fmla z31.h, p2/M, z4.h, z8.h\n" - "ld1h { z4.h }, p2/Z, [x27, #-7, MUL VL]\n" - "ldr x25, [x16, #0xd8]\n" - "fmla z28.h, p2/M, z0.h, z14.h\n" - "fmla z29.h, p2/M, z0.h, z11.h\n" - "ld1h { z14.h }, p3/Z, [x20, x13, LSL #1]\n" - "ldr x23, [x16, #0xe0]\n" - "fmla z30.h, p2/M, z0.h, z5.h\n" - "fmla z31.h, p2/M, z0.h, z6.h\n" - "ld1h { z0.h }, p2/Z, [x27, #-6, MUL VL]\n" - "ldr x20, [x16, #0xf8]\n" - "fmla z28.h, p2/M, z1.h, z11.h\n" - "fmla z29.h, p2/M, z1.h, z12.h\n" - "ld1h { z11.h }, p3/Z, [x22, x13, LSL #1]\n" - "ldr x22, [x16, #0xe8]\n" + "movprfx z30, z29\n fmla z30.h, p2/M, z0.h, z5.h\n" + "movprfx z27, z29\n fmla z27.h, p2/M, z0.h, z6.h\n" + "ldr x20, [x16, #0x50]\n" + "ld1h { z5.h }, p3/Z, [x20, x13, LSL #1]\n" + "movprfx z31, z29\n fmla z31.h, p2/M, z0.h, z7.h\n" + "movprfx z26, z29\n fmla z26.h, p2/M, z0.h, z8.h\n" + "ldr x20, [x16, #0x58]\n" + "ldr x21, [x16, #0x60]\n" "fmla z30.h, p2/M, z1.h, z6.h\n" - "fmla z31.h, p2/M, z1.h, z10.h\n" - "ld1h { z1.h }, p2/Z, [x27, #-5, MUL VL]\n" - "whilelt p1.h, x28, %x[n_channels]\n" - "fmla z28.h, p2/M, z2.h, z12.h\n" - "fmla z29.h, p2/M, z2.h, z9.h\n" - "ld1h { z12.h }, p3/Z, [x21, x13, LSL #1]\n" - "ldr x21, [x16, #0xf0]\n" - "fmla z30.h, p2/M, z2.h, z10.h\n" - "fmla z31.h, p2/M, z2.h, z11.h\n" - "ld1h { z2.h }, p2/Z, [x27, #-4, MUL VL]\n" - "inch x24\n" - "fmla z28.h, p2/M, z3.h, z9.h\n" - "fmla z29.h, p2/M, z3.h, z13.h\n" - "ld1h { z9.h }, p3/Z, [x10, x13, LSL #1]\n" - "ldr x10, [x16, #0x100]\n" - "fmla z30.h, p2/M, z3.h, z11.h\n" - "fmla z31.h, p2/M, z3.h, z12.h\n" - "ld1h { z3.h }, p2/Z, [x27, #-3, MUL VL]\n" - "mov p0.b, p3.b\n" - "fmla z28.h, p2/M, z4.h, z13.h\n" - "fmla z29.h, p2/M, z4.h, z8.h\n" - "ld1h { z13.h }, p3/Z, [x9, x13, LSL #1]\n" - "ld1h { z8.h }, p3/Z, [x23, x13, LSL #1]\n" - "fmla z30.h, p2/M, z4.h, z12.h\n" - "fmla z31.h, p2/M, z4.h, z14.h\n" - "ld1h { z4.h }, p2/Z, [x27, #-2, MUL VL]\n" - "ldr x9, [x16, #0x108]\n" - "fmla z28.h, p2/M, z0.h, z5.h\n" - "fmla z29.h, p2/M, z0.h, z6.h\n" - "ld1h { z5.h }, p3/Z, [x26, x13, LSL #1]\n" - "ldr x26, [x16, #0x110]\n" - "fmla z30.h, p2/M, z0.h, z9.h\n" - "fmla z31.h, p2/M, z0.h, z13.h\n" - "ld1h { z0.h }, p2/Z, [x27, #-1, MUL VL]\n" - "ld1h { z16.h }, p2/Z, [x27, #4, MUL VL]\n" - "fmla z28.h, p2/M, z1.h, z6.h\n" - "fmla z29.h, p2/M, z1.h, z10.h\n" - "ld1h { z6.h }, p3/Z, [x25, x13, LSL #1]\n" - "ldr x25, [x16, #0x118]\n" - "fmla z30.h, p2/M, z1.h, z13.h\n" - "fmla z31.h, p2/M, z1.h, z5.h\n" - "ld1h { z1.h }, p2/Z, [x27]\n" - "fmla z28.h, p2/M, z2.h, z10.h\n" - "fmla z29.h, p2/M, z2.h, z11.h\n" - "ld1h { z10.h }, p3/Z, [x22, x13, LSL #1]\n" - "fmla z30.h, p2/M, z2.h, z5.h\n" - "fmla z31.h, p2/M, z2.h, z6.h\n" - "ld1h { z2.h }, p2/Z, [x27, #1, MUL VL]\n" - "fmla z28.h, p2/M, z3.h, z11.h\n" - "fmla z29.h, p2/M, z3.h, z12.h\n" - "ld1h { z11.h }, p3/Z, [x21, x13, LSL #1]\n" - "fmla z30.h, p2/M, z3.h, z6.h\n" - "fmla z31.h, p2/M, z3.h, z8.h\n" - "ld1h { z3.h }, p2/Z, [x27, #2, MUL VL]\n" - "fmla z28.h, p2/M, z4.h, z12.h\n" - "fmla z29.h, p2/M, z4.h, z14.h\n" - "ld1h { z12.h }, p3/Z, [x20, x13, LSL #1]\n" - "fmla z30.h, p2/M, z4.h, z8.h\n" - "fmla z31.h, p2/M, z4.h, z10.h\n" - "ld1h { z4.h }, p2/Z, [x27, #3, MUL VL]\n" - "fmla z28.h, p2/M, z0.h, z9.h\n" - "fmla z29.h, p2/M, z0.h, z13.h\n" - "ld1h { z9.h }, p3/Z, [x10, x13, LSL #1]\n" - "fmla z30.h, p2/M, z0.h, z11.h\n" - "fmla z31.h, p2/M, z0.h, z12.h\n" - "ld1h { z11.h }, p3/Z, [x9, x13, LSL #1]\n" - "ldp x10, x9, [x16, #0x0]\n" - "fmla z28.h, p2/M, z1.h, z13.h\n" - "fmla z29.h, p2/M, z1.h, z5.h\n" - "ld1h { z0.h }, p2/Z, [x27, #5, MUL VL]\n" - "fmla z30.h, p2/M, z1.h, z12.h\n" - "fmla z31.h, p2/M, z1.h, z9.h\n" - "ld1h { z12.h }, p3/Z, [x26, x13, LSL #1]\n" - "ld1h { z1.h }, p2/Z, [x27, #6, MUL VL]\n" - "fmla z28.h, p2/M, z2.h, z5.h\n" - "fmla z29.h, p2/M, z2.h, z6.h\n" - "ld1h { z5.h }, p1/Z, [x10, x28, LSL #1]\n" + "fmla z27.h, p2/M, z1.h, z9.h\n" + "ld1h { z22.h }, p3/Z, [x20, x13, LSL #1]\n" + "ldr x20, [x16, #0x68]\n" + "fmla z31.h, p2/M, z1.h, z8.h\n" + "fmla z26.h, p2/M, z1.h, z13.h\n" + "ld1h { z21.h }, p2/Z, [x9]\n" + "ldr x23, [x16, #0x70]\n" "fmla z30.h, p2/M, z2.h, z9.h\n" - "fmla z31.h, p2/M, z2.h, z11.h\n" - "ld1h { z9.h }, p3/Z, [x25, x13, LSL #1]\n" - "ldp x26, x25, [x16, #0x10]\n" - "fmla z28.h, p2/M, z3.h, z6.h\n" - "fmla z29.h, p2/M, z3.h, z8.h\n" - "ld1h { z6.h }, p1/Z, [x9, x28, LSL #1]\n" - "ldp x23, x22, [x16, #0x20]\n" + "fmla z27.h, p2/M, z2.h, z11.h\n" + "ld1h { z20.h }, p3/Z, [x21, x13, LSL #1]\n" + "ld1h { z18.h }, p2/Z, [x9, #1, MUL VL]\n" + "fmla z31.h, p2/M, z2.h, z13.h\n" + "fmla z26.h, p2/M, z2.h, z5.h\n" + "ldr x22, [x16, #0x78]\n" + "ld1h { z17.h }, p2/Z, [x9, #2, MUL VL]\n" "fmla z30.h, p2/M, z3.h, z11.h\n" - "fmla z31.h, p2/M, z3.h, z12.h\n" - "ldp x21, x20, [x16, #0x30]\n" - "ldp x10, x9, [x16, #0x40]\n" - "fmla z28.h, p2/M, z4.h, z8.h\n" - "fmla z29.h, p2/M, z4.h, z10.h\n" - "inch x13\n" - "ld1h { z7.h }, p1/Z, [x26, x28, LSL #1]\n" + "fmla z27.h, p2/M, z3.h, z12.h\n" + "ld1h { z11.h }, p3/Z, [x20, x13, LSL #1]\n" + "ldr x21, [x16, #0x80]\n" + "fmla z31.h, p2/M, z3.h, z5.h\n" + "fmla z26.h, p2/M, z3.h, z22.h\n" + "ld1h { z16.h }, p2/Z, [x9, #3, MUL VL]\n" + "ldr x20, [x16, #0x88]\n" "fmla z30.h, p2/M, z4.h, z12.h\n" - "fmla z31.h, p2/M, z4.h, z9.h\n" - "ld1h { z8.h }, p1/Z, [x25, x28, LSL #1]\n" - "ld1h { z9.h }, p1/Z, [x23, x28, LSL #1]\n" - "ld1h { z13.h }, p1/Z, [x22, x28, LSL #1]\n" - "ld1h { z11.h }, p1/Z, [x21, x28, LSL #1]\n" - "fmax z28.h, p2/M, z28.h, z18.h\n" - "fmax z29.h, p2/M, z29.h, z18.h\n" - "ld1h { z12.h }, p1/Z, [x20, x28, LSL #1]\n" - "ld1h { z10.h }, p1/Z, [x10, x28, LSL #1]\n" - "fmax z30.h, p2/M, z30.h, z18.h\n" - "fmax z31.h, p2/M, z31.h, z18.h\n" - "ld1h { z14.h }, p1/Z, [x9, x28, LSL #1]\n" + "fmla z27.h, p2/M, z4.h, z20.h\n" + "ld1h { z0.h }, p3/Z, [x23, x13, LSL #1]\n" + "ld1h { z29.h }, p3/Z, [x22, x13, LSL #1]\n" + "fmla z31.h, p2/M, z4.h, z22.h\n" + "fmla z26.h, p2/M, z4.h, z10.h\n" + "ld1h { z19.h }, p2/Z, [x9, #4, MUL VL]\n" + "ldr x23, [x16, #0x90]\n" + "fmla z30.h, p2/M, z21.h, z7.h\n" + "fmla z27.h, p2/M, z21.h, z8.h\n" + "ldr x26, [x16, #0x98]\n" + "ldr x22, [x16, #0xa0]\n" + "fmla z31.h, p2/M, z21.h, z14.h\n" + "fmla z26.h, p2/M, z21.h, z11.h\n" + "ld1h { z25.h }, p2/Z, [x9, #5, MUL VL]\n" + "ldr x25, [x16, #0xa8]\n" + "fmla z30.h, p2/M, z18.h, z8.h\n" + "fmla z27.h, p2/M, z18.h, z13.h\n" + "ld1h { z24.h }, p3/Z, [x20, x13, LSL #1]\n" + "ldr x24, [x16, #0xb0]\n" + "fmla z31.h, p2/M, z18.h, z11.h\n" + "fmla z26.h, p2/M, z18.h, z0.h\n" + "ld1h { z18.h }, p2/Z, [x9, #6, MUL VL]\n" + "ldr x20, [x16, #0xb8]\n" + "fmla z30.h, p2/M, z17.h, z13.h\n" + "fmla z27.h, p2/M, z17.h, z5.h\n" + "ld1h { z3.h }, p3/Z, [x21, x13, LSL #1]\n" + "ldr x21, [x16, #0xc0]\n" + "fmla z31.h, p2/M, z17.h, z0.h\n" + "fmla z26.h, p2/M, z17.h, z29.h\n" + "ld1h { z17.h }, p2/Z, [x9, #7, MUL VL]\n" + "addvl x9, x9, #16\n" + "fmla z30.h, p2/M, z16.h, z5.h\n" + "fmla z27.h, p2/M, z16.h, z22.h\n" + "ld1h { z6.h }, p3/Z, [x23, x13, LSL #1]\n" + "ldr x27, [x16, #0xc8]\n" + "fmla z31.h, p2/M, z16.h, z29.h\n" + "fmla z26.h, p2/M, z16.h, z3.h\n" + "ld1h { z16.h }, p2/Z, [x9, #-8, MUL VL]\n" + "ldr x23, [x16, #0xd0]\n" + "fmla z30.h, p2/M, z19.h, z22.h\n" + "fmla z27.h, p2/M, z19.h, z10.h\n" + "ld1h { z23.h }, p3/Z, [x26, x13, LSL #1]\n" + "ld1h { z22.h }, p3/Z, [x22, x13, LSL #1]\n" + "fmla z31.h, p2/M, z19.h, z3.h\n" + "fmla z26.h, p2/M, z19.h, z24.h\n" + "ld1h { z21.h }, p2/Z, [x9, #-7, MUL VL]\n" + "ldr x22, [x16, #0xd8]\n" + "fmla z30.h, p2/M, z25.h, z14.h\n" + "fmla z27.h, p2/M, z25.h, z11.h\n" + "ld1h { z1.h }, p3/Z, [x20, x13, LSL #1]\n" + "ldr x20, [x16, #0xe0]\n" + "fmla z31.h, p2/M, z25.h, z6.h\n" + "fmla z26.h, p2/M, z25.h, z23.h\n" + "ld1h { z20.h }, p2/Z, [x9, #-6, MUL VL]\n" + "ldr x26, [x16, #0xf8]\n" + "fmla z30.h, p2/M, z18.h, z11.h\n" + "fmla z27.h, p2/M, z18.h, z0.h\n" + "ld1h { z7.h }, p3/Z, [x25, x13, LSL #1]\n" + "ldr x25, [x16, #0xe8]\n" + "fmla z31.h, p2/M, z18.h, z23.h\n" + "fmla z26.h, p2/M, z18.h, z22.h\n" + "ld1h { z18.h }, p2/Z, [x9, #-5, MUL VL]\n" + "whilelt p1.h, x10, %x[n_channels]\n" + "fmla z30.h, p2/M, z17.h, z0.h\n" + "fmla z27.h, p2/M, z17.h, z29.h\n" + "ld1h { z19.h }, p3/Z, [x24, x13, LSL #1]\n" + "ldr x24, [x16, #0xf0]\n" + "fmla z31.h, p2/M, z17.h, z22.h\n" + "fmla z26.h, p2/M, z17.h, z7.h\n" + "ld1h { z17.h }, p2/Z, [x9, #-4, MUL VL]\n" "inch x28\n" - "ld1h { z2.h }, p2/Z, [x27, #7, MUL VL]\n" - "addvl x27, x27, #16\n" + "fmla z30.h, p2/M, z16.h, z29.h\n" + "fmla z27.h, p2/M, z16.h, z3.h\n" + "ld1h { z0.h }, p3/Z, [x21, x13, LSL #1]\n" + "ldr x21, [x16, #0x100]\n" + "fmla z31.h, p2/M, z16.h, z7.h\n" + "fmla z26.h, p2/M, z16.h, z19.h\n" + "ld1h { z16.h }, p2/Z, [x9, #-3, MUL VL]\n" + "mov p0.b, p3.b\n" + "fmla z30.h, p2/M, z21.h, z3.h\n" + "fmla z27.h, p2/M, z21.h, z24.h\n" + "ld1h { z11.h }, p3/Z, [x27, x13, LSL #1]\n" + "ld1h { z13.h }, p3/Z, [x20, x13, LSL #1]\n" + "fmla z31.h, p2/M, z21.h, z19.h\n" + "fmla z26.h, p2/M, z21.h, z1.h\n" + "ld1h { z10.h }, p2/Z, [x9, #-2, MUL VL]\n" + "ldr x20, [x16, #0x108]\n" + "fmla z30.h, p2/M, z20.h, z6.h\n" + "fmla z27.h, p2/M, z20.h, z23.h\n" + "ld1h { z25.h }, p3/Z, [x23, x13, LSL #1]\n" + "ldr x23, [x16, #0x110]\n" + "fmla z31.h, p2/M, z20.h, z0.h\n" + "fmla z26.h, p2/M, z20.h, z11.h\n" + "ld1h { z8.h }, p2/Z, [x9, #-1, MUL VL]\n" + "ld1h { z29.h }, p2/Z, [x9, #4, MUL VL]\n" + "fmla z30.h, p2/M, z18.h, z23.h\n" + "fmla z27.h, p2/M, z18.h, z22.h\n" + "ld1h { z24.h }, p3/Z, [x22, x13, LSL #1]\n" + "ldr x22, [x16, #0x118]\n" + "fmla z31.h, p2/M, z18.h, z11.h\n" + "fmla z26.h, p2/M, z18.h, z25.h\n" + "ld1h { z23.h }, p2/Z, [x9]\n" + "fmla z30.h, p2/M, z17.h, z22.h\n" + "fmla z27.h, p2/M, z17.h, z7.h\n" + "ld1h { z22.h }, p3/Z, [x25, x13, LSL #1]\n" + "fmla z31.h, p2/M, z17.h, z25.h\n" + "fmla z26.h, p2/M, z17.h, z24.h\n" + "ld1h { z21.h }, p2/Z, [x9, #1, MUL VL]\n" + "fmla z30.h, p2/M, z16.h, z7.h\n" + "fmla z27.h, p2/M, z16.h, z19.h\n" + "ld1h { z18.h }, p3/Z, [x24, x13, LSL #1]\n" + "fmla z31.h, p2/M, z16.h, z24.h\n" + "fmla z26.h, p2/M, z16.h, z13.h\n" + "ld1h { z20.h }, p2/Z, [x9, #2, MUL VL]\n" + "fmla z30.h, p2/M, z10.h, z19.h\n" + "fmla z27.h, p2/M, z10.h, z1.h\n" + "ld1h { z17.h }, p3/Z, [x26, x13, LSL #1]\n" + "fmla z31.h, p2/M, z10.h, z13.h\n" + "fmla z26.h, p2/M, z10.h, z22.h\n" + "ld1h { z19.h }, p2/Z, [x9, #3, MUL VL]\n" + "fmla z30.h, p2/M, z8.h, z0.h\n" + "fmla z27.h, p2/M, z8.h, z11.h\n" + "ld1h { z16.h }, p3/Z, [x21, x13, LSL #1]\n" + "fmla z31.h, p2/M, z8.h, z18.h\n" + "fmla z26.h, p2/M, z8.h, z17.h\n" + "ld1h { z18.h }, p3/Z, [x20, x13, LSL #1]\n" + "ldp x21, x20, [x16, #0x0]\n" + "fmla z30.h, p2/M, z23.h, z11.h\n" + "fmla z27.h, p2/M, z23.h, z25.h\n" + "ld1h { z0.h }, p2/Z, [x9, #5, MUL VL]\n" + "fmla z31.h, p2/M, z23.h, z17.h\n" + "fmla z26.h, p2/M, z23.h, z16.h\n" + "ld1h { z17.h }, p3/Z, [x23, x13, LSL #1]\n" + "ld1h { z1.h }, p2/Z, [x9, #6, MUL VL]\n" + "fmla z30.h, p2/M, z21.h, z25.h\n" + "fmla z27.h, p2/M, z21.h, z24.h\n" + "ld1h { z5.h }, p1/Z, [x21, x10, LSL #1]\n" + "fmla z31.h, p2/M, z21.h, z16.h\n" + "fmla z26.h, p2/M, z21.h, z18.h\n" + "ld1h { z16.h }, p3/Z, [x22, x13, LSL #1]\n" + "ldp x27, x26, [x16, #0x10]\n" + "fmla z30.h, p2/M, z20.h, z24.h\n" + "fmla z27.h, p2/M, z20.h, z13.h\n" + "ld1h { z6.h }, p1/Z, [x20, x10, LSL #1]\n" + "ldp x25, x24, [x16, #0x20]\n" + "fmla z31.h, p2/M, z20.h, z18.h\n" + "fmla z26.h, p2/M, z20.h, z17.h\n" + "ldp x23, x22, [x16, #0x30]\n" + "ldp x21, x20, [x16, #0x40]\n" + "fmla z30.h, p2/M, z19.h, z13.h\n" + "fmla z27.h, p2/M, z19.h, z22.h\n" + "inch x13\n" + "ld1h { z7.h }, p1/Z, [x27, x10, LSL #1]\n" + "fmla z31.h, p2/M, z19.h, z17.h\n" + "fmla z26.h, p2/M, z19.h, z16.h\n" + "ld1h { z8.h }, p1/Z, [x26, x10, LSL #1]\n" + "ld1h { z9.h }, p1/Z, [x25, x10, LSL #1]\n" + "ld1h { z13.h }, p1/Z, [x24, x10, LSL #1]\n" + "ld1h { z11.h }, p1/Z, [x23, x10, LSL #1]\n" + "fmax z30.h, p2/M, z30.h, z15.h\n" + "fmax z27.h, p2/M, z27.h, z15.h\n" + "ld1h { z12.h }, p1/Z, [x22, x10, LSL #1]\n" + "ld1h { z10.h }, p1/Z, [x21, x10, LSL #1]\n" + "fmax z31.h, p2/M, z31.h, z15.h\n" + "fmax z26.h, p2/M, z26.h, z15.h\n" + "ld1h { z14.h }, p1/Z, [x20, x10, LSL #1]\n" + "inch x10\n" + "ld1h { z2.h }, p2/Z, [x9, #7, MUL VL]\n" + "addvl x9, x9, #16\n" "whilelt p3.h, x13, %x[n_channels]\n" - "cmp x28, %x[n_channels]\n" - "ld1h { z3.h }, p2/Z, [x27, #-8, MUL VL]\n" - "ld1h { z4.h }, p2/Z, [x27, #-7, MUL VL]\n" - "fmin z28.h, p2/M, z28.h, z17.h\n" - "fmin z29.h, p2/M, z29.h, z17.h\n" - "st1h { z28.h }, p0, [x15, x24, LSL #1]\n" - "fmin z30.h, p2/M, z30.h, z17.h\n" - "fmin z31.h, p2/M, z31.h, z17.h\n" - "st1h { z29.h }, p0, [x14, x24, LSL #1]\n" - "st1h { z30.h }, p0, [x12, x24, LSL #1]\n" - "addvl x27, x27, #-6\n" - "st1h { z31.h }, p0, [x11, x24, LSL #1]\n" + "cmp x10, %x[n_channels]\n" + "ld1h { z3.h }, p2/Z, [x9, #-8, MUL VL]\n" + "ld1h { z4.h }, p2/Z, [x9, #-7, MUL VL]\n" + "fmin z30.h, p2/M, z30.h, z28.h\n" + "fmin z27.h, p2/M, z27.h, z28.h\n" + "st1h { z30.h }, p0, [x15, x28, LSL #1]\n" + "fmin z31.h, p2/M, z31.h, z28.h\n" + "fmin z26.h, p2/M, z26.h, z28.h\n" + "st1h { z27.h }, p0, [x14, x28, LSL #1]\n" + "st1h { z31.h }, p0, [x12, x28, LSL #1]\n" + "addvl x9, x9, #-6\n" + "st1h { z26.h }, p0, [x11, x28, LSL #1]\n" "blt 1b\n" "2:" // Channel tail - "movprfx z28, z16\n fmla z28.h, p2/M, z0.h, z5.h\n" - "movprfx z29, z16\n fmla z29.h, p2/M, z0.h, z6.h\n" - "ldr x26, [x16, #0x50]\n" - "ld1h { z5.h }, p3/Z, [x26, x13, LSL #1]\n" - "movprfx z30, z16\n fmla z30.h, p2/M, z0.h, z7.h\n" - "movprfx z31, z16\n fmla z31.h, p2/M, z0.h, z8.h\n" - "ldr x25, [x16, #0x58]\n" - "ldr x23, [x16, #0x60]\n" - "fmla z28.h, p2/M, z1.h, z6.h\n" - "fmla z29.h, p2/M, z1.h, z9.h\n" - "ld1h { z6.h }, p3/Z, [x25, x13, LSL #1]\n" - "ldr x22, [x16, #0x68]\n" - "fmla z30.h, p2/M, z1.h, z8.h\n" - "fmla z31.h, p2/M, z1.h, z13.h\n" - "ld1h { z0.h }, p2/Z, [x27]\n" - "ldr x21, [x16, #0x70]\n" - "fmla z28.h, p2/M, z2.h, z9.h\n" - "fmla z29.h, p2/M, z2.h, z11.h\n" - "ld1h { z9.h }, p3/Z, [x23, x13, LSL #1]\n" - "ld1h { z1.h }, p2/Z, [x27, #1, MUL VL]\n" - "fmla z30.h, p2/M, z2.h, z13.h\n" - "fmla z31.h, p2/M, z2.h, z5.h\n" - "ldr x20, [x16, #0x78]\n" - "ld1h { z2.h }, p2/Z, [x27, #2, MUL VL]\n" - "fmla z28.h, p2/M, z3.h, z11.h\n" - "fmla z29.h, p2/M, z3.h, z12.h\n" - "ld1h { z11.h }, p3/Z, [x22, x13, LSL #1]\n" - "ldr x10, [x16, #0x80]\n" - "fmla z30.h, p2/M, z3.h, z5.h\n" - "fmla z31.h, p2/M, z3.h, z6.h\n" - "ld1h { z3.h }, p2/Z, [x27, #3, MUL VL]\n" - "ldr x9, [x16, #0x88]\n" - "fmla z28.h, p2/M, z4.h, z12.h\n" - "fmla z29.h, p2/M, z4.h, z9.h\n" - "ld1h { z12.h }, p3/Z, [x21, x13, LSL #1]\n" - "ld1h { z9.h }, p3/Z, [x20, x13, LSL #1]\n" - "fmla z30.h, p2/M, z4.h, z6.h\n" - "fmla z31.h, p2/M, z4.h, z10.h\n" - "ld1h { z4.h }, p2/Z, [x27, #4, MUL VL]\n" - "ldr x26, [x16, #0x90]\n" - "fmla z28.h, p2/M, z0.h, z7.h\n" + "movprfx z30, z29\n fmla z30.h, p2/M, z0.h, z5.h\n" + "movprfx z31, z29\n fmla z31.h, p2/M, z0.h, z6.h\n" + "ldr x20, [x16, #0x50]\n" + "ld1h { z22.h }, p3/Z, [x20, x13, LSL #1]\n" + "movprfx z5, z29\n fmla z5.h, p2/M, z0.h, z7.h\n" "fmla z29.h, p2/M, z0.h, z8.h\n" - "ldr x25, [x16, #0x98]\n" - "ldr x23, [x16, #0xa0]\n" - "fmla z30.h, p2/M, z0.h, z14.h\n" - "fmla z31.h, p2/M, z0.h, z11.h\n" - "ld1h { z0.h }, p2/Z, [x27, #5, MUL VL]\n" - "ldr x22, [x16, #0xa8]\n" - "fmla z28.h, p2/M, z1.h, z8.h\n" - "fmla z29.h, p2/M, z1.h, z13.h\n" - "ld1h { z8.h }, p3/Z, [x9, x13, LSL #1]\n" - "ldr x21, [x16, #0xb0]\n" - "fmla z30.h, p2/M, z1.h, z11.h\n" - "fmla z31.h, p2/M, z1.h, z12.h\n" - "ld1h { z1.h }, p2/Z, [x27, #6, MUL VL]\n" - "ldr x20, [x16, #0xb8]\n" - "fmla z28.h, p2/M, z2.h, z13.h\n" - "fmla z29.h, p2/M, z2.h, z5.h\n" - "ld1h { z13.h }, p3/Z, [x10, x13, LSL #1]\n" - "ldr x10, [x16, #0xc0]\n" - "fmla z30.h, p2/M, z2.h, z12.h\n" - "fmla z31.h, p2/M, z2.h, z9.h\n" - "ld1h { z2.h }, p2/Z, [x27, #7, MUL VL]\n" - "addvl x27, x27, #16\n" - "fmla z28.h, p2/M, z3.h, z5.h\n" - "fmla z29.h, p2/M, z3.h, z6.h\n" - "ld1h { z5.h }, p3/Z, [x26, x13, LSL #1]\n" - "ldr x9, [x16, #0xc8]\n" - "fmla z30.h, p2/M, z3.h, z9.h\n" - "fmla z31.h, p2/M, z3.h, z13.h\n" - "ld1h { z3.h }, p2/Z, [x27, #-8, MUL VL]\n" - "ldr x26, [x16, #0xd0]\n" - "fmla z28.h, p2/M, z4.h, z6.h\n" - "fmla z29.h, p2/M, z4.h, z10.h\n" - "ld1h { z6.h }, p3/Z, [x25, x13, LSL #1]\n" - "ld1h { z10.h }, p3/Z, [x23, x13, LSL #1]\n" - "fmla z30.h, p2/M, z4.h, z13.h\n" - "fmla z31.h, p2/M, z4.h, z8.h\n" - "ld1h { z4.h }, p2/Z, [x27, #-7, MUL VL]\n" - "ldr x25, [x16, #0xd8]\n" - "fmla z28.h, p2/M, z0.h, z14.h\n" - "fmla z29.h, p2/M, z0.h, z11.h\n" - "ld1h { z14.h }, p3/Z, [x20, x13, LSL #1]\n" - "ldr x23, [x16, #0xe0]\n" - "fmla z30.h, p2/M, z0.h, z5.h\n" - "fmla z31.h, p2/M, z0.h, z6.h\n" - "ld1h { z0.h }, p2/Z, [x27, #-6, MUL VL]\n" - "ldr x20, [x16, #0xf8]\n" - "fmla z28.h, p2/M, z1.h, z11.h\n" - "fmla z29.h, p2/M, z1.h, z12.h\n" - "ld1h { z11.h }, p3/Z, [x22, x13, LSL #1]\n" - "ldr x22, [x16, #0xe8]\n" + "ldr x20, [x16, #0x58]\n" + "ldr x21, [x16, #0x60]\n" "fmla z30.h, p2/M, z1.h, z6.h\n" - "fmla z31.h, p2/M, z1.h, z10.h\n" - "ld1h { z1.h }, p2/Z, [x27, #-5, MUL VL]\n" - "inch x24\n" - "fmla z28.h, p2/M, z2.h, z12.h\n" - "fmla z29.h, p2/M, z2.h, z9.h\n" - "ld1h { z12.h }, p3/Z, [x21, x13, LSL #1]\n" - "ldr x21, [x16, #0xf0]\n" - "fmla z30.h, p2/M, z2.h, z10.h\n" - "fmla z31.h, p2/M, z2.h, z11.h\n" - "ld1h { z2.h }, p2/Z, [x27, #-4, MUL VL]\n" - "mov p0.b, p3.b\n" - "fmla z28.h, p2/M, z3.h, z9.h\n" - "fmla z29.h, p2/M, z3.h, z13.h\n" - "ld1h { z9.h }, p3/Z, [x10, x13, LSL #1]\n" - "ldr x10, [x16, #0x100]\n" - "fmla z30.h, p2/M, z3.h, z11.h\n" - "fmla z31.h, p2/M, z3.h, z12.h\n" - "ld1h { z3.h }, p2/Z, [x27, #-3, MUL VL]\n" - "fmla z28.h, p2/M, z4.h, z13.h\n" - "fmla z29.h, p2/M, z4.h, z8.h\n" - "ld1h { z13.h }, p3/Z, [x9, x13, LSL #1]\n" - "ld1h { z8.h }, p3/Z, [x23, x13, LSL #1]\n" - "fmla z30.h, p2/M, z4.h, z12.h\n" - "fmla z31.h, p2/M, z4.h, z14.h\n" - "ld1h { z4.h }, p2/Z, [x27, #-2, MUL VL]\n" - "ldr x9, [x16, #0x108]\n" - "fmla z28.h, p2/M, z0.h, z5.h\n" - "fmla z29.h, p2/M, z0.h, z6.h\n" - "ld1h { z5.h }, p3/Z, [x26, x13, LSL #1]\n" - "ldr x26, [x16, #0x110]\n" - "fmla z30.h, p2/M, z0.h, z9.h\n" - "fmla z31.h, p2/M, z0.h, z13.h\n" - "ld1h { z0.h }, p2/Z, [x27, #-1, MUL VL]\n" - "fmla z28.h, p2/M, z1.h, z6.h\n" - "fmla z29.h, p2/M, z1.h, z10.h\n" - "ld1h { z6.h }, p3/Z, [x25, x13, LSL #1]\n" - "ldr x25, [x16, #0x118]\n" - "fmla z30.h, p2/M, z1.h, z13.h\n" - "fmla z31.h, p2/M, z1.h, z5.h\n" - "ld1h { z1.h }, p2/Z, [x27]\n" - "fmla z28.h, p2/M, z2.h, z10.h\n" - "fmla z29.h, p2/M, z2.h, z11.h\n" - "ld1h { z10.h }, p3/Z, [x22, x13, LSL #1]\n" - "fmla z30.h, p2/M, z2.h, z5.h\n" - "fmla z31.h, p2/M, z2.h, z6.h\n" - "ld1h { z2.h }, p2/Z, [x27, #1, MUL VL]\n" - "fmla z28.h, p2/M, z3.h, z11.h\n" - "fmla z29.h, p2/M, z3.h, z12.h\n" - "ld1h { z11.h }, p3/Z, [x21, x13, LSL #1]\n" - "fmla z30.h, p2/M, z3.h, z6.h\n" - "fmla z31.h, p2/M, z3.h, z8.h\n" - "ld1h { z3.h }, p2/Z, [x27, #2, MUL VL]\n" - "fmla z28.h, p2/M, z4.h, z12.h\n" - "fmla z29.h, p2/M, z4.h, z14.h\n" - "ld1h { z12.h }, p3/Z, [x20, x13, LSL #1]\n" - "fmla z30.h, p2/M, z4.h, z8.h\n" - "fmla z31.h, p2/M, z4.h, z10.h\n" - "ld1h { z4.h }, p2/Z, [x27, #3, MUL VL]\n" - "fmla z28.h, p2/M, z0.h, z9.h\n" - "fmla z29.h, p2/M, z0.h, z13.h\n" - "ld1h { z9.h }, p3/Z, [x10, x13, LSL #1]\n" - "fmla z30.h, p2/M, z0.h, z11.h\n" - "fmla z31.h, p2/M, z0.h, z12.h\n" - "ld1h { z11.h }, p3/Z, [x9, x13, LSL #1]\n" - "fmla z28.h, p2/M, z1.h, z13.h\n" - "fmla z29.h, p2/M, z1.h, z5.h\n" - "fmla z30.h, p2/M, z1.h, z12.h\n" "fmla z31.h, p2/M, z1.h, z9.h\n" - "ld1h { z12.h }, p3/Z, [x26, x13, LSL #1]\n" - "fmla z28.h, p2/M, z2.h, z5.h\n" - "fmla z29.h, p2/M, z2.h, z6.h\n" + "ld1h { z6.h }, p3/Z, [x20, x13, LSL #1]\n" + "ldr x20, [x16, #0x68]\n" + "fmla z5.h, p2/M, z1.h, z8.h\n" + "fmla z29.h, p2/M, z1.h, z13.h\n" + "ld1h { z20.h }, p2/Z, [x9]\n" + "ldr x23, [x16, #0x70]\n" "fmla z30.h, p2/M, z2.h, z9.h\n" "fmla z31.h, p2/M, z2.h, z11.h\n" - "ld1h { z9.h }, p3/Z, [x25, x13, LSL #1]\n" - "fmla z28.h, p2/M, z3.h, z6.h\n" - "fmla z29.h, p2/M, z3.h, z8.h\n" + "ld1h { z16.h }, p3/Z, [x21, x13, LSL #1]\n" + "ld1h { z19.h }, p2/Z, [x9, #1, MUL VL]\n" + "fmla z5.h, p2/M, z2.h, z13.h\n" + "fmla z29.h, p2/M, z2.h, z22.h\n" + "ldr x21, [x16, #0x78]\n" + "ld1h { z18.h }, p2/Z, [x9, #2, MUL VL]\n" "fmla z30.h, p2/M, z3.h, z11.h\n" "fmla z31.h, p2/M, z3.h, z12.h\n" - "fmla z28.h, p2/M, z4.h, z8.h\n" - "fmla z29.h, p2/M, z4.h, z10.h\n" - "fmax z28.h, p2/M, z28.h, z18.h\n" - "fmax z29.h, p2/M, z29.h, z18.h\n" + "ld1h { z1.h }, p3/Z, [x20, x13, LSL #1]\n" + "ldr x22, [x16, #0x80]\n" + "fmla z5.h, p2/M, z3.h, z22.h\n" + "fmla z29.h, p2/M, z3.h, z6.h\n" + "ld1h { z17.h }, p2/Z, [x9, #3, MUL VL]\n" + "ldr x20, [x16, #0x88]\n" "fmla z30.h, p2/M, z4.h, z12.h\n" - "fmla z31.h, p2/M, z4.h, z9.h\n" - "fmax z30.h, p2/M, z30.h, z18.h\n" - "fmax z31.h, p2/M, z31.h, z18.h\n" - "fmin z28.h, p2/M, z28.h, z17.h\n" - "fmin z29.h, p2/M, z29.h, z17.h\n" - "st1h { z28.h }, p0, [x15, x24, LSL #1]\n" - "fmin z30.h, p2/M, z30.h, z17.h\n" - "fmin z31.h, p2/M, z31.h, z17.h\n" - "st1h { z29.h }, p0, [x14, x24, LSL #1]\n" - "st1h { z30.h }, p0, [x12, x24, LSL #1]\n" - "st1h { z31.h }, p0, [x11, x24, LSL #1]\n" + "fmla z31.h, p2/M, z4.h, z16.h\n" + "ld1h { z0.h }, p3/Z, [x23, x13, LSL #1]\n" + "ld1h { z27.h }, p3/Z, [x21, x13, LSL #1]\n" + "fmla z5.h, p2/M, z4.h, z6.h\n" + "fmla z29.h, p2/M, z4.h, z10.h\n" + "ld1h { z16.h }, p2/Z, [x9, #4, MUL VL]\n" + "ldr x21, [x16, #0x90]\n" + "fmla z30.h, p2/M, z20.h, z7.h\n" + "fmla z31.h, p2/M, z20.h, z8.h\n" + "ldr x27, [x16, #0x98]\n" + "ldr x26, [x16, #0xa0]\n" + "fmla z5.h, p2/M, z20.h, z14.h\n" + "fmla z29.h, p2/M, z20.h, z1.h\n" + "ld1h { z21.h }, p2/Z, [x9, #5, MUL VL]\n" + "ldr x25, [x16, #0xa8]\n" + "fmla z30.h, p2/M, z19.h, z8.h\n" + "fmla z31.h, p2/M, z19.h, z13.h\n" + "ld1h { z26.h }, p3/Z, [x20, x13, LSL #1]\n" + "ldr x24, [x16, #0xb0]\n" + "fmla z5.h, p2/M, z19.h, z1.h\n" + "fmla z29.h, p2/M, z19.h, z0.h\n" + "ld1h { z25.h }, p2/Z, [x9, #6, MUL VL]\n" + "ldr x20, [x16, #0xb8]\n" + "fmla z30.h, p2/M, z18.h, z13.h\n" + "fmla z31.h, p2/M, z18.h, z22.h\n" + "ld1h { z24.h }, p3/Z, [x22, x13, LSL #1]\n" + "ldr x23, [x16, #0xc0]\n" + "fmla z5.h, p2/M, z18.h, z0.h\n" + "fmla z29.h, p2/M, z18.h, z27.h\n" + "ld1h { z23.h }, p2/Z, [x9, #7, MUL VL]\n" + "addvl x9, x9, #16\n" + "fmla z30.h, p2/M, z17.h, z22.h\n" + "fmla z31.h, p2/M, z17.h, z6.h\n" + "ld1h { z22.h }, p3/Z, [x21, x13, LSL #1]\n" + "ldr x22, [x16, #0xc8]\n" + "fmla z5.h, p2/M, z17.h, z27.h\n" + "fmla z29.h, p2/M, z17.h, z24.h\n" + "ld1h { z20.h }, p2/Z, [x9, #-8, MUL VL]\n" + "ldr x21, [x16, #0xd0]\n" + "fmla z30.h, p2/M, z16.h, z6.h\n" + "fmla z31.h, p2/M, z16.h, z10.h\n" + "ld1h { z19.h }, p3/Z, [x27, x13, LSL #1]\n" + "ld1h { z18.h }, p3/Z, [x26, x13, LSL #1]\n" + "fmla z5.h, p2/M, z16.h, z24.h\n" + "fmla z29.h, p2/M, z16.h, z26.h\n" + "ld1h { z16.h }, p2/Z, [x9, #-7, MUL VL]\n" + "ldr x27, [x16, #0xd8]\n" + "fmla z30.h, p2/M, z21.h, z14.h\n" + "fmla z31.h, p2/M, z21.h, z1.h\n" + "ld1h { z17.h }, p3/Z, [x20, x13, LSL #1]\n" + "ldr x20, [x16, #0xe0]\n" + "fmla z5.h, p2/M, z21.h, z22.h\n" + "fmla z29.h, p2/M, z21.h, z19.h\n" + "ld1h { z21.h }, p2/Z, [x9, #-6, MUL VL]\n" + "ldr x26, [x16, #0xf8]\n" + "fmla z30.h, p2/M, z25.h, z1.h\n" + "fmla z31.h, p2/M, z25.h, z0.h\n" + "ld1h { z9.h }, p3/Z, [x25, x13, LSL #1]\n" + "ldr x25, [x16, #0xe8]\n" + "fmla z5.h, p2/M, z25.h, z19.h\n" + "fmla z29.h, p2/M, z25.h, z18.h\n" + "ld1h { z4.h }, p2/Z, [x9, #-5, MUL VL]\n" + "inch x28\n" + "fmla z30.h, p2/M, z23.h, z0.h\n" + "fmla z31.h, p2/M, z23.h, z27.h\n" + "ld1h { z8.h }, p3/Z, [x24, x13, LSL #1]\n" + "ldr x24, [x16, #0xf0]\n" + "fmla z5.h, p2/M, z23.h, z18.h\n" + "fmla z29.h, p2/M, z23.h, z9.h\n" + "ld1h { z6.h }, p2/Z, [x9, #-4, MUL VL]\n" + "mov p0.b, p3.b\n" + "fmla z30.h, p2/M, z20.h, z27.h\n" + "fmla z31.h, p2/M, z20.h, z24.h\n" + "ld1h { z10.h }, p3/Z, [x23, x13, LSL #1]\n" + "ldr x23, [x16, #0x100]\n" + "fmla z5.h, p2/M, z20.h, z9.h\n" + "fmla z29.h, p2/M, z20.h, z8.h\n" + "ld1h { z11.h }, p2/Z, [x9, #-3, MUL VL]\n" + "fmla z30.h, p2/M, z16.h, z24.h\n" + "fmla z31.h, p2/M, z16.h, z26.h\n" + "ld1h { z0.h }, p3/Z, [x22, x13, LSL #1]\n" + "ld1h { z27.h }, p3/Z, [x20, x13, LSL #1]\n" + "fmla z5.h, p2/M, z16.h, z8.h\n" + "fmla z29.h, p2/M, z16.h, z17.h\n" + "ld1h { z16.h }, p2/Z, [x9, #-2, MUL VL]\n" + "ldr x22, [x16, #0x108]\n" + "fmla z30.h, p2/M, z21.h, z22.h\n" + "fmla z31.h, p2/M, z21.h, z19.h\n" + "ld1h { z26.h }, p3/Z, [x21, x13, LSL #1]\n" + "ldr x21, [x16, #0x110]\n" + "fmla z5.h, p2/M, z21.h, z10.h\n" + "fmla z29.h, p2/M, z21.h, z0.h\n" + "ld1h { z25.h }, p2/Z, [x9, #-1, MUL VL]\n" + "fmla z30.h, p2/M, z4.h, z19.h\n" + "fmla z31.h, p2/M, z4.h, z18.h\n" + "ld1h { z24.h }, p3/Z, [x27, x13, LSL #1]\n" + "ldr x20, [x16, #0x118]\n" + "fmla z5.h, p2/M, z4.h, z0.h\n" + "fmla z29.h, p2/M, z4.h, z26.h\n" + "ld1h { z23.h }, p2/Z, [x9]\n" + "fmla z30.h, p2/M, z6.h, z18.h\n" + "fmla z31.h, p2/M, z6.h, z9.h\n" + "ld1h { z22.h }, p3/Z, [x25, x13, LSL #1]\n" + "fmla z5.h, p2/M, z6.h, z26.h\n" + "fmla z29.h, p2/M, z6.h, z24.h\n" + "ld1h { z21.h }, p2/Z, [x9, #1, MUL VL]\n" + "fmla z30.h, p2/M, z11.h, z9.h\n" + "fmla z31.h, p2/M, z11.h, z8.h\n" + "ld1h { z18.h }, p3/Z, [x24, x13, LSL #1]\n" + "fmla z5.h, p2/M, z11.h, z24.h\n" + "fmla z29.h, p2/M, z11.h, z27.h\n" + "ld1h { z20.h }, p2/Z, [x9, #2, MUL VL]\n" + "fmla z30.h, p2/M, z16.h, z8.h\n" + "fmla z31.h, p2/M, z16.h, z17.h\n" + "ld1h { z17.h }, p3/Z, [x26, x13, LSL #1]\n" + "fmla z5.h, p2/M, z16.h, z27.h\n" + "fmla z29.h, p2/M, z16.h, z22.h\n" + "ld1h { z19.h }, p2/Z, [x9, #3, MUL VL]\n" + "fmla z30.h, p2/M, z25.h, z10.h\n" + "fmla z31.h, p2/M, z25.h, z0.h\n" + "ld1h { z16.h }, p3/Z, [x23, x13, LSL #1]\n" + "fmla z5.h, p2/M, z25.h, z18.h\n" + "fmla z29.h, p2/M, z25.h, z17.h\n" + "ld1h { z18.h }, p3/Z, [x22, x13, LSL #1]\n" + "fmla z30.h, p2/M, z23.h, z0.h\n" + "fmla z31.h, p2/M, z23.h, z26.h\n" + "fmla z5.h, p2/M, z23.h, z17.h\n" + "fmla z29.h, p2/M, z23.h, z16.h\n" + "ld1h { z17.h }, p3/Z, [x21, x13, LSL #1]\n" + "fmla z30.h, p2/M, z21.h, z26.h\n" + "fmla z31.h, p2/M, z21.h, z24.h\n" + "fmla z5.h, p2/M, z21.h, z16.h\n" + "fmla z29.h, p2/M, z21.h, z18.h\n" + "ld1h { z16.h }, p3/Z, [x20, x13, LSL #1]\n" + "fmla z30.h, p2/M, z20.h, z24.h\n" + "fmla z31.h, p2/M, z20.h, z27.h\n" + "fmla z5.h, p2/M, z20.h, z18.h\n" + "fmla z29.h, p2/M, z20.h, z17.h\n" + "fmla z30.h, p2/M, z19.h, z27.h\n" + "fmla z31.h, p2/M, z19.h, z22.h\n" + "fmax z30.h, p2/M, z30.h, z15.h\n" + "fmax z31.h, p2/M, z31.h, z15.h\n" + "fmla z5.h, p2/M, z19.h, z17.h\n" + "fmla z29.h, p2/M, z19.h, z16.h\n" + "fmax z5.h, p2/M, z5.h, z15.h\n" + "fmax z29.h, p2/M, z29.h, z15.h\n" + "fmin z30.h, p2/M, z30.h, z28.h\n" + "fmin z31.h, p2/M, z31.h, z28.h\n" + "st1h { z30.h }, p0, [x15, x28, LSL #1]\n" + "fmin z5.h, p2/M, z5.h, z28.h\n" + "fmin z29.h, p2/M, z29.h, z28.h\n" + "st1h { z31.h }, p0, [x14, x28, LSL #1]\n" + "st1h { z5.h }, p0, [x12, x28, LSL #1]\n" + "st1h { z29.h }, p0, [x11, x28, LSL #1]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp index d7b1de20620ffc07fab20ca7ef595bbdbe0fdef2..16b96fdb8edcba6d998ed33a6b27acd8a8f7c0a4 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,19 +22,19 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include #pragma once -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { -void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); -void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); +void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(const float *const *const input_ptrs, float *const *const outptrs, const void *params, unsigned int n_channels, const float activation_min, const float activation_max); +void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int n_tile_rows, const unsigned int n_tile_cols, const float *inptr, int64_t ld_input_row, int64_t ld_input_col, float *outptr, int64_t ld_output_row, int64_t ld_output_col, const void *params, unsigned int n_channels, const float activation_min, const float activation_max); class sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirstStrategy { @@ -57,7 +57,7 @@ class sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirst constexpr static unsigned int output_cols = 2; sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst(const CPUInfo *) - : DepthwiseDepthfirstStrategy(2, 3, 1) {} + : Parent(output_rows, output_cols, kernel_rows, kernel_cols, stride_rows, stride_cols) {} arm_gemm::VLType get_vl_type(void) const override { return vl_type; } @@ -68,4 +68,4 @@ class sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirst } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp index a570c5aa6a44fcd274fd834abef377a912cfe2b5..1bdef8527409cdd88c4caaef40ef407438aa84fe 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -108,10 +108,10 @@ void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "whilelt p2.s, XZR, %x[n_channels]\n" "madd x20, x14, x12, x20\n" // offset += tile_j * ld_output_col "ldr x28, [%x[params_struct], %[offsetof_args_outptr]]\n" - "ld1w { z18.s }, p3/Z, [x10]\n" + "ld1w { z27.s }, p3/Z, [x10]\n" "add x27, x13, x13\n" "mul x21, x21, x25\n" // offset *= kernel_stride * output_size - "add x9, x9, x21, LSL #2\n" // inptr[0] += offset * sizeof(float) + "add x9, x9, x21, LSL #2\n" // inptr[0] += offset * sizeof(float) "ld1w { z0.s }, p3/Z, [x10, #1, MUL VL]\n" "ld1w { z1.s }, p3/Z, [x10, #2, MUL VL]\n" "mul x20, x20, x24\n" // offset *= output_tile_size @@ -125,10 +125,10 @@ void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "ld1w { z6.s }, p3/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" "add x28, x28, x20, LSL #2\n" // outptrs[0] += offset * sizeof(float) - "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rw { z26.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" "cmp x11, %x[n_channels]\n" "add x23, x25, x23, LSL #2\n" - "ld1rw { z16.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "ld1rw { z25.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "ld1w { z7.s }, p3/Z, [x10, #-8, MUL VL]\n" "add x22, x28, x22, LSL #2\n" "mov x21, #0x0\n" @@ -142,175 +142,175 @@ void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "ld1w { z13.s }, p2/Z, [x25, x13, LSL #2]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "movprfx z28, z18\n fmla z28.s, p3/M, z4.s, z9.s\n" - "movprfx z29, z18\n fmla z29.s, p3/M, z3.s, z9.s\n" + "movprfx z24, z27\n fmla z24.s, p3/M, z4.s, z9.s\n" + "movprfx z23, z27\n fmla z23.s, p3/M, z3.s, z9.s\n" "whilelt p1.s, x11, %x[n_channels]\n" "incw x21\n" - "movprfx z30, z18\n fmla z30.s, p3/M, z1.s, z9.s\n" - "movprfx z31, z18\n fmla z31.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x23]\n" + "movprfx z22, z27\n fmla z22.s, p3/M, z1.s, z9.s\n" + "movprfx z21, z27\n fmla z21.s, p3/M, z0.s, z9.s\n" + "ld1w { z18.s }, p2/Z, [x23]\n" "incw x11\n" - "fmla z28.s, p3/M, z0.s, z10.s\n" - "fmla z29.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x23, x24, LSL #2]\n" - "ld1w { z10.s }, p2/Z, [x25, x27, LSL #2]\n" - "fmla z30.s, p3/M, z2.s, z12.s\n" - "fmla z31.s, p3/M, z1.s, z12.s\n" + "fmla z24.s, p3/M, z0.s, z10.s\n" + "fmla z23.s, p3/M, z2.s, z11.s\n" + "ld1w { z17.s }, p2/Z, [x23, x24, LSL #2]\n" + "ld1w { z20.s }, p2/Z, [x25, x27, LSL #2]\n" + "fmla z22.s, p3/M, z2.s, z12.s\n" + "fmla z21.s, p3/M, z1.s, z12.s\n" "mov p0.b, p2.b\n" - "ld1w { z18.s }, p3/Z, [x10]\n" - "fmla z28.s, p3/M, z5.s, z12.s\n" - "fmla z29.s, p3/M, z4.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x9, x13, LSL #2]\n" + "ld1w { z27.s }, p3/Z, [x10]\n" + "fmla z24.s, p3/M, z5.s, z12.s\n" + "fmla z23.s, p3/M, z4.s, z12.s\n" + "ld1w { z16.s }, p2/Z, [x9, x13, LSL #2]\n" "incw x20\n" - "fmla z30.s, p3/M, z6.s, z9.s\n" - "fmla z31.s, p3/M, z3.s, z13.s\n" - "ld1w { z9.s }, p2/Z, [x9, x27, LSL #2]\n" + "fmla z22.s, p3/M, z6.s, z18.s\n" + "fmla z21.s, p3/M, z3.s, z13.s\n" + "ld1w { z18.s }, p2/Z, [x9, x27, LSL #2]\n" "addvl x9, x9, #1\n" - "fmla z28.s, p3/M, z7.s, z13.s\n" - "fmla z29.s, p3/M, z6.s, z13.s\n" - "fmla z30.s, p3/M, z4.s, z13.s\n" - "fmla z31.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26]\n" - "fmla z28.s, p3/M, z1.s, z12.s\n" - "fmla z29.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x26, x24, LSL #2]\n" + "fmla z24.s, p3/M, z7.s, z13.s\n" + "fmla z23.s, p3/M, z6.s, z13.s\n" + "fmla z22.s, p3/M, z4.s, z13.s\n" + "fmla z21.s, p3/M, z8.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x26]\n" + "fmla z24.s, p3/M, z1.s, z16.s\n" + "fmla z23.s, p3/M, z0.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x26, x24, LSL #2]\n" "addvl x26, x26, #1\n" - "fmla z30.s, p3/M, z5.s, z10.s\n" - "fmla z31.s, p3/M, z4.s, z10.s\n" + "fmla z22.s, p3/M, z5.s, z20.s\n" + "fmla z21.s, p3/M, z4.s, z20.s\n" "ld1w { z4.s }, p3/Z, [x10, #5, MUL VL]\n" - "fmla z28.s, p3/M, z2.s, z9.s\n" - "fmla z29.s, p3/M, z1.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x25]\n" + "fmla z24.s, p3/M, z2.s, z18.s\n" + "fmla z23.s, p3/M, z1.s, z18.s\n" + "ld1w { z19.s }, p2/Z, [x25]\n" "ld1w { z1.s }, p3/Z, [x10, #2, MUL VL]\n" - "fmla z30.s, p3/M, z0.s, z11.s\n" - "fmla z31.s, p3/M, z2.s, z12.s\n" + "fmla z22.s, p3/M, z0.s, z17.s\n" + "fmla z21.s, p3/M, z2.s, z16.s\n" "ld1w { z0.s }, p3/Z, [x10, #1, MUL VL]\n" "ld1w { z2.s }, p3/Z, [x10, #3, MUL VL]\n" - "fmla z28.s, p3/M, z8.s, z10.s\n" - "fmla z29.s, p3/M, z7.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x25, x24, LSL #2]\n" + "fmla z24.s, p3/M, z8.s, z20.s\n" + "fmla z23.s, p3/M, z7.s, z20.s\n" + "ld1w { z18.s }, p2/Z, [x25, x24, LSL #2]\n" "addvl x25, x25, #1\n" - "fmla z30.s, p3/M, z3.s, z9.s\n" - "fmla z31.s, p3/M, z5.s, z10.s\n" + "fmla z22.s, p3/M, z3.s, z19.s\n" + "fmla z21.s, p3/M, z5.s, z18.s\n" "ld1w { z13.s }, p1/Z, [x25, x13, LSL #2]\n" - "fmla z28.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n" - "fmla z29.s, p3/M, z5.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x23, x27, LSL #2]\n" - "fmla z30.s, p3/M, z7.s, z11.s\n" - "fmla z31.s, p3/M, z6.s, z11.s\n" + "fmla z24.s, p3/M, z3.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x23, x13, LSL #2]\n" + "fmla z23.s, p3/M, z5.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x23, x27, LSL #2]\n" + "fmla z22.s, p3/M, z7.s, z17.s\n" + "fmla z21.s, p3/M, z6.s, z17.s\n" "ld1w { z3.s }, p3/Z, [x10, #4, MUL VL]\n" "ld1w { z5.s }, p3/Z, [x10, #6, MUL VL]\n" - "fmla z28.s, p3/M, z6.s, z9.s\n" - "fmla z29.s, p3/M, z8.s, z10.s\n" - "fmax z28.s, p3/M, z28.s, z17.s\n" - "fmax z29.s, p3/M, z29.s, z17.s\n" - "fmla z30.s, p3/M, z8.s, z12.s\n" - "fmla z31.s, p3/M, z7.s, z12.s\n" - "fmax z30.s, p3/M, z30.s, z17.s\n" - "fmax z31.s, p3/M, z31.s, z17.s\n" + "fmla z24.s, p3/M, z6.s, z19.s\n" + "fmla z23.s, p3/M, z8.s, z18.s\n" + "fmax z24.s, p3/M, z24.s, z26.s\n" + "fmax z23.s, p3/M, z23.s, z26.s\n" + "fmla z22.s, p3/M, z8.s, z16.s\n" + "fmla z21.s, p3/M, z7.s, z16.s\n" + "fmax z22.s, p3/M, z22.s, z26.s\n" + "fmax z21.s, p3/M, z21.s, z26.s\n" "ld1w { z6.s }, p3/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" "whilelt p2.s, x21, %x[n_channels]\n" "ld1w { z9.s }, p1/Z, [x26, x13, LSL #2]\n" "cmp x11, %x[n_channels]\n" - "fmin z28.s, p3/M, z28.s, z16.s\n" + "fmin z24.s, p3/M, z24.s, z25.s\n" "ld1w { z10.s }, p1/Z, [x9]\n" "ld1w { z11.s }, p1/Z, [x9, x24, LSL #2]\n" - "fmin z29.s, p3/M, z29.s, z16.s\n" - "fmin z30.s, p3/M, z30.s, z16.s\n" + "fmin z23.s, p3/M, z23.s, z25.s\n" + "fmin z22.s, p3/M, z22.s, z25.s\n" "ld1w { z12.s }, p1/Z, [x26, x27, LSL #2]\n" - "st1w { z28.s }, p0, [x28]\n" - "fmin z31.s, p3/M, z31.s, z16.s\n" + "st1w { z24.s }, p0, [x28]\n" + "fmin z21.s, p3/M, z21.s, z25.s\n" "addvl x23, x23, #1\n" - "st1w { z29.s }, p0, [x28, x12, LSL #2]\n" + "st1w { z23.s }, p0, [x28, x12, LSL #2]\n" "ld1w { z7.s }, p3/Z, [x10, #-8, MUL VL]\n" - "st1w { z30.s }, p0, [x22]\n" + "st1w { z22.s }, p0, [x22]\n" "addvl x28, x28, #1\n" "ld1w { z8.s }, p3/Z, [x10, #-7, MUL VL]\n" "addvl x10, x10, #-6\n" - "st1w { z31.s }, p0, [x22, x12, LSL #2]\n" + "st1w { z21.s }, p0, [x22, x12, LSL #2]\n" "addvl x22, x22, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "movprfx z28, z18\n fmla z28.s, p3/M, z4.s, z9.s\n" - "movprfx z29, z18\n fmla z29.s, p3/M, z3.s, z9.s\n" + "movprfx z24, z27\n fmla z24.s, p3/M, z4.s, z9.s\n" + "movprfx z23, z27\n fmla z23.s, p3/M, z3.s, z9.s\n" "ldr x14, [%x[params_struct], %[offsetof_args_tile_j]]\n" "ldr x10, [%x[params_struct], %[offsetof_args_tile_i]]\n" - "movprfx z30, z18\n fmla z30.s, p3/M, z1.s, z9.s\n" - "movprfx z31, z18\n fmla z31.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x23]\n" + "movprfx z22, z27\n fmla z22.s, p3/M, z1.s, z9.s\n" + "movprfx z21, z27\n fmla z21.s, p3/M, z0.s, z9.s\n" + "ld1w { z18.s }, p2/Z, [x23]\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" - "fmla z28.s, p3/M, z0.s, z10.s\n" - "fmla z29.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x23, x24, LSL #2]\n" - "ld1w { z10.s }, p2/Z, [x25, x27, LSL #2]\n" - "fmla z30.s, p3/M, z2.s, z12.s\n" - "fmla z31.s, p3/M, z1.s, z12.s\n" + "fmla z24.s, p3/M, z0.s, z10.s\n" + "fmla z23.s, p3/M, z2.s, z11.s\n" + "ld1w { z17.s }, p2/Z, [x23, x24, LSL #2]\n" + "ld1w { z20.s }, p2/Z, [x25, x27, LSL #2]\n" + "fmla z22.s, p3/M, z2.s, z12.s\n" + "fmla z21.s, p3/M, z1.s, z12.s\n" "add x14, x14, #0x1\n" "cmp x14, x20\n" - "fmla z28.s, p3/M, z5.s, z12.s\n" - "fmla z29.s, p3/M, z4.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x9, x13, LSL #2]\n" + "fmla z24.s, p3/M, z5.s, z12.s\n" + "fmla z23.s, p3/M, z4.s, z12.s\n" + "ld1w { z16.s }, p2/Z, [x9, x13, LSL #2]\n" "add x21, x10, #0x1\n" - "fmla z30.s, p3/M, z6.s, z9.s\n" - "fmla z31.s, p3/M, z3.s, z13.s\n" - "ld1w { z9.s }, p2/Z, [x9, x27, LSL #2]\n" + "fmla z22.s, p3/M, z6.s, z18.s\n" + "fmla z21.s, p3/M, z3.s, z13.s\n" + "ld1w { z18.s }, p2/Z, [x9, x27, LSL #2]\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" - "fmla z28.s, p3/M, z7.s, z13.s\n" - "fmla z29.s, p3/M, z6.s, z13.s\n" + "fmla z24.s, p3/M, z7.s, z13.s\n" + "fmla z23.s, p3/M, z6.s, z13.s\n" "csel x10, x10, x21, LT\n" "mov p0.b, p2.b\n" - "fmla z30.s, p3/M, z4.s, z13.s\n" - "fmla z31.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26]\n" + "fmla z22.s, p3/M, z4.s, z13.s\n" + "fmla z21.s, p3/M, z8.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x26]\n" "csel x14, x14, XZR, LT\n" - "fmla z28.s, p3/M, z1.s, z12.s\n" - "fmla z29.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x26, x24, LSL #2]\n" + "fmla z24.s, p3/M, z1.s, z16.s\n" + "fmla z23.s, p3/M, z0.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x26, x24, LSL #2]\n" "cmp x10, x20\n" - "fmla z30.s, p3/M, z5.s, z10.s\n" - "fmla z31.s, p3/M, z4.s, z10.s\n" - "fmla z28.s, p3/M, z2.s, z9.s\n" - "fmla z29.s, p3/M, z1.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x25]\n" - "fmla z30.s, p3/M, z0.s, z11.s\n" - "fmla z31.s, p3/M, z2.s, z12.s\n" - "fmla z28.s, p3/M, z8.s, z10.s\n" - "fmla z29.s, p3/M, z7.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x25, x24, LSL #2]\n" - "fmla z30.s, p3/M, z3.s, z9.s\n" - "fmla z31.s, p3/M, z5.s, z10.s\n" - "fmla z28.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n" - "fmla z29.s, p3/M, z5.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x23, x27, LSL #2]\n" - "fmla z30.s, p3/M, z7.s, z11.s\n" - "fmla z31.s, p3/M, z6.s, z11.s\n" - "fmla z28.s, p3/M, z6.s, z9.s\n" - "fmla z29.s, p3/M, z8.s, z10.s\n" - "fmax z28.s, p3/M, z28.s, z17.s\n" - "fmax z29.s, p3/M, z29.s, z17.s\n" - "fmla z30.s, p3/M, z8.s, z12.s\n" - "fmla z31.s, p3/M, z7.s, z12.s\n" - "fmax z30.s, p3/M, z30.s, z17.s\n" - "fmax z31.s, p3/M, z31.s, z17.s\n" - "fmin z28.s, p3/M, z28.s, z16.s\n" - "fmin z29.s, p3/M, z29.s, z16.s\n" - "st1w { z28.s }, p0, [x28]\n" - "fmin z30.s, p3/M, z30.s, z16.s\n" - "fmin z31.s, p3/M, z31.s, z16.s\n" - "st1w { z29.s }, p0, [x28, x12, LSL #2]\n" - "st1w { z30.s }, p0, [x22]\n" - "st1w { z31.s }, p0, [x22, x12, LSL #2]\n" + "fmla z22.s, p3/M, z5.s, z20.s\n" + "fmla z21.s, p3/M, z4.s, z20.s\n" + "fmla z24.s, p3/M, z2.s, z18.s\n" + "fmla z23.s, p3/M, z1.s, z18.s\n" + "ld1w { z19.s }, p2/Z, [x25]\n" + "fmla z22.s, p3/M, z0.s, z17.s\n" + "fmla z21.s, p3/M, z2.s, z16.s\n" + "fmla z24.s, p3/M, z8.s, z20.s\n" + "fmla z23.s, p3/M, z7.s, z20.s\n" + "ld1w { z18.s }, p2/Z, [x25, x24, LSL #2]\n" + "fmla z22.s, p3/M, z3.s, z19.s\n" + "fmla z21.s, p3/M, z5.s, z18.s\n" + "fmla z24.s, p3/M, z3.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x23, x13, LSL #2]\n" + "fmla z23.s, p3/M, z5.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x23, x27, LSL #2]\n" + "fmla z22.s, p3/M, z7.s, z17.s\n" + "fmla z21.s, p3/M, z6.s, z17.s\n" + "fmla z24.s, p3/M, z6.s, z19.s\n" + "fmla z23.s, p3/M, z8.s, z18.s\n" + "fmax z24.s, p3/M, z24.s, z26.s\n" + "fmax z23.s, p3/M, z23.s, z26.s\n" + "fmla z22.s, p3/M, z8.s, z16.s\n" + "fmla z21.s, p3/M, z7.s, z16.s\n" + "fmax z22.s, p3/M, z22.s, z26.s\n" + "fmax z21.s, p3/M, z21.s, z26.s\n" + "fmin z24.s, p3/M, z24.s, z25.s\n" + "fmin z23.s, p3/M, z23.s, z25.s\n" + "st1w { z24.s }, p0, [x28]\n" + "fmin z22.s, p3/M, z22.s, z25.s\n" + "fmin z21.s, p3/M, z21.s, z25.s\n" + "st1w { z23.s }, p0, [x28, x12, LSL #2]\n" + "st1w { z22.s }, p0, [x22]\n" + "st1w { z21.s }, p0, [x22, x12, LSL #2]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp index 903de0d309a9af275314ee1ca10a01967ce7230d..873b4736ff73035e40b03b6ded0eea23de72b885 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -87,7 +87,7 @@ void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( "ldp x11, x10, [x20, #0x10]\n" "mov x9, #0x0\n" "whilelt p2.s, XZR, %x[n_channels]\n" - "ld1w { z18.s }, p3/Z, [x16]\n" + "ld1w { z20.s }, p3/Z, [x16]\n" "ld1w { z0.s }, p3/Z, [x16, #1, MUL VL]\n" "cmp x14, %x[n_channels]\n" "ld1w { z1.s }, p3/Z, [x16, #2, MUL VL]\n" @@ -98,99 +98,99 @@ void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( "ld1w { z5.s }, p3/Z, [x16, #6, MUL VL]\n" "ld1w { z6.s }, p3/Z, [x16, #7, MUL VL]\n" "addvl x16, x16, #16\n" - "ldp x27, x26, [x15, #0x0]\n" - "ldp x25, x24, [x15, #0x10]\n" - "ldr x23, [x15, #0x20]\n" - "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "ld1rw { z16.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "ldp x24, x23, [x15, #0x0]\n" + "ldp x22, x21, [x15, #0x10]\n" + "ldr x20, [x15, #0x20]\n" + "ld1rw { z26.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rw { z25.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "ld1w { z7.s }, p3/Z, [x16, #-8, MUL VL]\n" "ld1w { z8.s }, p3/Z, [x16, #-7, MUL VL]\n" - "ld1w { z9.s }, p2/Z, [x27, x9, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x24, x9, LSL #2]\n" "addvl x16, x16, #-6\n" - "ld1w { z10.s }, p2/Z, [x26, x9, LSL #2]\n" - "ld1w { z11.s }, p2/Z, [x25, x9, LSL #2]\n" - "ld1w { z12.s }, p2/Z, [x24, x9, LSL #2]\n" - "ld1w { z13.s }, p2/Z, [x23, x9, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x23, x9, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x22, x9, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x21, x9, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x20, x9, LSL #2]\n" "bge 2f\n" "1:" // Channel loop - "movprfx z28, z18\n fmla z28.s, p3/M, z4.s, z9.s\n" - "movprfx z29, z18\n fmla z29.s, p3/M, z3.s, z9.s\n" - "ldr x22, [x15, #0x28]\n" - "ldr x21, [x15, #0x30]\n" - "movprfx z30, z18\n fmla z30.s, p3/M, z1.s, z9.s\n" - "movprfx z31, z18\n fmla z31.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x22, x9, LSL #2]\n" - "ldr x20, [x15, #0x38]\n" - "fmla z28.s, p3/M, z0.s, z10.s\n" - "fmla z29.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x21, x9, LSL #2]\n" - "ldr x26, [x15, #0x48]\n" - "fmla z30.s, p3/M, z2.s, z12.s\n" - "fmla z31.s, p3/M, z1.s, z12.s\n" - "ldr x27, [x15, #0x40]\n" - "ld1w { z10.s }, p2/Z, [x26, x9, LSL #2]\n" - "fmla z28.s, p3/M, z5.s, z12.s\n" - "fmla z29.s, p3/M, z4.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x20, x9, LSL #2]\n" - "ldr x25, [x15, #0x50]\n" - "fmla z30.s, p3/M, z6.s, z9.s\n" - "fmla z31.s, p3/M, z3.s, z13.s\n" - "ld1w { z9.s }, p2/Z, [x27, x9, LSL #2]\n" - "ldr x24, [x15, #0x58]\n" - "fmla z28.s, p3/M, z7.s, z13.s\n" - "fmla z29.s, p3/M, z6.s, z13.s\n" - "ldr x23, [x15, #0x60]\n" - "ldr x22, [x15, #0x68]\n" - "fmla z30.s, p3/M, z4.s, z13.s\n" - "fmla z31.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x25, x9, LSL #2]\n" - "ldr x21, [x15, #0x70]\n" - "fmla z28.s, p3/M, z1.s, z12.s\n" - "fmla z29.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x24, x9, LSL #2]\n" - "ldr x20, [x15, #0x78]\n" - "fmla z30.s, p3/M, z5.s, z10.s\n" - "fmla z31.s, p3/M, z4.s, z10.s\n" + "movprfx z24, z20\n fmla z24.s, p3/M, z4.s, z9.s\n" + "movprfx z23, z20\n fmla z23.s, p3/M, z3.s, z9.s\n" + "ldr x21, [x15, #0x28]\n" + "ldr x20, [x15, #0x30]\n" + "movprfx z22, z20\n fmla z22.s, p3/M, z1.s, z9.s\n" + "movprfx z21, z20\n fmla z21.s, p3/M, z0.s, z9.s\n" + "ld1w { z18.s }, p2/Z, [x21, x9, LSL #2]\n" + "ldr x22, [x15, #0x38]\n" + "fmla z24.s, p3/M, z0.s, z10.s\n" + "fmla z23.s, p3/M, z2.s, z11.s\n" + "ld1w { z17.s }, p2/Z, [x20, x9, LSL #2]\n" + "ldr x21, [x15, #0x48]\n" + "fmla z22.s, p3/M, z2.s, z12.s\n" + "fmla z21.s, p3/M, z1.s, z12.s\n" + "ldr x20, [x15, #0x40]\n" + "ld1w { z20.s }, p2/Z, [x21, x9, LSL #2]\n" + "fmla z24.s, p3/M, z5.s, z12.s\n" + "fmla z23.s, p3/M, z4.s, z12.s\n" + "ld1w { z16.s }, p2/Z, [x22, x9, LSL #2]\n" + "ldr x22, [x15, #0x50]\n" + "fmla z22.s, p3/M, z6.s, z18.s\n" + "fmla z21.s, p3/M, z3.s, z13.s\n" + "ld1w { z18.s }, p2/Z, [x20, x9, LSL #2]\n" + "ldr x21, [x15, #0x58]\n" + "fmla z24.s, p3/M, z7.s, z13.s\n" + "fmla z23.s, p3/M, z6.s, z13.s\n" + "ldr x20, [x15, #0x60]\n" + "ldr x27, [x15, #0x68]\n" + "fmla z22.s, p3/M, z4.s, z13.s\n" + "fmla z21.s, p3/M, z8.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x22, x9, LSL #2]\n" + "ldr x26, [x15, #0x70]\n" + "fmla z24.s, p3/M, z1.s, z16.s\n" + "fmla z23.s, p3/M, z0.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x21, x9, LSL #2]\n" + "ldr x25, [x15, #0x78]\n" + "fmla z22.s, p3/M, z5.s, z20.s\n" + "fmla z21.s, p3/M, z4.s, z20.s\n" "whilelt p1.s, x14, %x[n_channels]\n" - "ldp x27, x26, [x15, #0x0]\n" - "fmla z28.s, p3/M, z2.s, z9.s\n" - "fmla z29.s, p3/M, z1.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x23, x9, LSL #2]\n" - "ldp x25, x24, [x15, #0x10]\n" - "fmla z30.s, p3/M, z0.s, z11.s\n" - "fmla z31.s, p3/M, z2.s, z12.s\n" - "ldr x23, [x15, #0x20]\n" - "ld1w { z13.s }, p1/Z, [x23, x14, LSL #2]\n" - "fmla z28.s, p3/M, z8.s, z10.s\n" - "fmla z29.s, p3/M, z7.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x22, x9, LSL #2]\n" + "ldp x24, x23, [x15, #0x0]\n" + "fmla z24.s, p3/M, z2.s, z18.s\n" + "fmla z23.s, p3/M, z1.s, z18.s\n" + "ld1w { z19.s }, p2/Z, [x20, x9, LSL #2]\n" + "ldp x22, x21, [x15, #0x10]\n" + "fmla z22.s, p3/M, z0.s, z17.s\n" + "fmla z21.s, p3/M, z2.s, z16.s\n" + "ldr x20, [x15, #0x20]\n" + "ld1w { z13.s }, p1/Z, [x20, x14, LSL #2]\n" + "fmla z24.s, p3/M, z8.s, z20.s\n" + "fmla z23.s, p3/M, z7.s, z20.s\n" + "ld1w { z18.s }, p2/Z, [x27, x9, LSL #2]\n" "incw x28\n" - "fmla z30.s, p3/M, z3.s, z9.s\n" - "fmla z31.s, p3/M, z5.s, z10.s\n" + "fmla z22.s, p3/M, z3.s, z19.s\n" + "fmla z21.s, p3/M, z5.s, z18.s\n" "mov p0.b, p2.b\n" - "ld1w { z18.s }, p3/Z, [x16]\n" - "fmla z28.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x21, x9, LSL #2]\n" - "fmla z29.s, p3/M, z5.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x20, x9, LSL #2]\n" - "fmla z30.s, p3/M, z7.s, z11.s\n" - "fmla z31.s, p3/M, z6.s, z11.s\n" + "ld1w { z20.s }, p3/Z, [x16]\n" + "fmla z24.s, p3/M, z3.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x26, x9, LSL #2]\n" + "fmla z23.s, p3/M, z5.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x25, x9, LSL #2]\n" + "fmla z22.s, p3/M, z7.s, z17.s\n" + "fmla z21.s, p3/M, z6.s, z17.s\n" "incw x9\n" - "ld1w { z11.s }, p1/Z, [x25, x14, LSL #2]\n" - "fmla z28.s, p3/M, z6.s, z9.s\n" - "fmla z29.s, p3/M, z8.s, z10.s\n" - "ld1w { z9.s }, p1/Z, [x27, x14, LSL #2]\n" - "ld1w { z10.s }, p1/Z, [x26, x14, LSL #2]\n" - "fmla z30.s, p3/M, z8.s, z12.s\n" - "fmla z31.s, p3/M, z7.s, z12.s\n" - "ld1w { z12.s }, p1/Z, [x24, x14, LSL #2]\n" + "ld1w { z11.s }, p1/Z, [x22, x14, LSL #2]\n" + "fmla z24.s, p3/M, z6.s, z19.s\n" + "fmla z23.s, p3/M, z8.s, z18.s\n" + "ld1w { z9.s }, p1/Z, [x24, x14, LSL #2]\n" + "ld1w { z10.s }, p1/Z, [x23, x14, LSL #2]\n" + "fmla z22.s, p3/M, z8.s, z16.s\n" + "fmla z21.s, p3/M, z7.s, z16.s\n" + "ld1w { z12.s }, p1/Z, [x21, x14, LSL #2]\n" "incw x14\n" - "fmax z28.s, p3/M, z28.s, z17.s\n" - "fmax z29.s, p3/M, z29.s, z17.s\n" + "fmax z24.s, p3/M, z24.s, z26.s\n" + "fmax z23.s, p3/M, z23.s, z26.s\n" "ld1w { z0.s }, p3/Z, [x16, #1, MUL VL]\n" "ld1w { z1.s }, p3/Z, [x16, #2, MUL VL]\n" - "fmax z30.s, p3/M, z30.s, z17.s\n" - "fmax z31.s, p3/M, z31.s, z17.s\n" + "fmax z22.s, p3/M, z22.s, z26.s\n" + "fmax z21.s, p3/M, z21.s, z26.s\n" "ld1w { z2.s }, p3/Z, [x16, #3, MUL VL]\n" "ld1w { z3.s }, p3/Z, [x16, #4, MUL VL]\n" "ld1w { z4.s }, p3/Z, [x16, #5, MUL VL]\n" @@ -199,98 +199,98 @@ void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( "cmp x14, %x[n_channels]\n" "ld1w { z6.s }, p3/Z, [x16, #7, MUL VL]\n" "addvl x16, x16, #16\n" - "fmin z28.s, p3/M, z28.s, z16.s\n" - "st1w { z28.s }, p0, [x13, x28, LSL #2]\n" - "fmin z29.s, p3/M, z29.s, z16.s\n" - "fmin z30.s, p3/M, z30.s, z16.s\n" - "st1w { z29.s }, p0, [x12, x28, LSL #2]\n" + "fmin z24.s, p3/M, z24.s, z25.s\n" + "st1w { z24.s }, p0, [x13, x28, LSL #2]\n" + "fmin z23.s, p3/M, z23.s, z25.s\n" + "fmin z22.s, p3/M, z22.s, z25.s\n" + "st1w { z23.s }, p0, [x12, x28, LSL #2]\n" "ld1w { z7.s }, p3/Z, [x16, #-8, MUL VL]\n" - "fmin z31.s, p3/M, z31.s, z16.s\n" - "st1w { z30.s }, p0, [x11, x28, LSL #2]\n" + "fmin z21.s, p3/M, z21.s, z25.s\n" + "st1w { z22.s }, p0, [x11, x28, LSL #2]\n" "ld1w { z8.s }, p3/Z, [x16, #-7, MUL VL]\n" "addvl x16, x16, #-6\n" - "st1w { z31.s }, p0, [x10, x28, LSL #2]\n" + "st1w { z21.s }, p0, [x10, x28, LSL #2]\n" "blt 1b\n" "2:" // Channel tail - "movprfx z28, z18\n fmla z28.s, p3/M, z4.s, z9.s\n" - "movprfx z29, z18\n fmla z29.s, p3/M, z3.s, z9.s\n" - "ldr x22, [x15, #0x28]\n" - "ldr x21, [x15, #0x30]\n" - "movprfx z30, z18\n fmla z30.s, p3/M, z1.s, z9.s\n" - "movprfx z31, z18\n fmla z31.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x22, x9, LSL #2]\n" - "ldr x20, [x15, #0x38]\n" - "fmla z28.s, p3/M, z0.s, z10.s\n" - "fmla z29.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x21, x9, LSL #2]\n" - "ldr x26, [x15, #0x48]\n" - "fmla z30.s, p3/M, z2.s, z12.s\n" - "fmla z31.s, p3/M, z1.s, z12.s\n" - "ldr x27, [x15, #0x40]\n" - "ld1w { z10.s }, p2/Z, [x26, x9, LSL #2]\n" - "fmla z28.s, p3/M, z5.s, z12.s\n" - "fmla z29.s, p3/M, z4.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x20, x9, LSL #2]\n" - "ldr x25, [x15, #0x50]\n" - "fmla z30.s, p3/M, z6.s, z9.s\n" - "fmla z31.s, p3/M, z3.s, z13.s\n" - "ld1w { z9.s }, p2/Z, [x27, x9, LSL #2]\n" - "ldr x24, [x15, #0x58]\n" - "fmla z28.s, p3/M, z7.s, z13.s\n" - "fmla z29.s, p3/M, z6.s, z13.s\n" + "movprfx z24, z20\n fmla z24.s, p3/M, z4.s, z9.s\n" + "movprfx z23, z20\n fmla z23.s, p3/M, z3.s, z9.s\n" + "ldr x21, [x15, #0x28]\n" + "ldr x20, [x15, #0x30]\n" + "movprfx z22, z20\n fmla z22.s, p3/M, z1.s, z9.s\n" + "movprfx z21, z20\n fmla z21.s, p3/M, z0.s, z9.s\n" + "ld1w { z18.s }, p2/Z, [x21, x9, LSL #2]\n" + "ldr x22, [x15, #0x38]\n" + "fmla z24.s, p3/M, z0.s, z10.s\n" + "fmla z23.s, p3/M, z2.s, z11.s\n" + "ld1w { z17.s }, p2/Z, [x20, x9, LSL #2]\n" + "ldr x21, [x15, #0x48]\n" + "fmla z22.s, p3/M, z2.s, z12.s\n" + "fmla z21.s, p3/M, z1.s, z12.s\n" + "ldr x20, [x15, #0x40]\n" + "ld1w { z20.s }, p2/Z, [x21, x9, LSL #2]\n" + "fmla z24.s, p3/M, z5.s, z12.s\n" + "fmla z23.s, p3/M, z4.s, z12.s\n" + "ld1w { z16.s }, p2/Z, [x22, x9, LSL #2]\n" + "ldr x21, [x15, #0x50]\n" + "fmla z22.s, p3/M, z6.s, z18.s\n" + "fmla z21.s, p3/M, z3.s, z13.s\n" + "ld1w { z18.s }, p2/Z, [x20, x9, LSL #2]\n" + "ldr x20, [x15, #0x58]\n" + "fmla z24.s, p3/M, z7.s, z13.s\n" + "fmla z23.s, p3/M, z6.s, z13.s\n" "ldr x23, [x15, #0x60]\n" "ldr x22, [x15, #0x68]\n" - "fmla z30.s, p3/M, z4.s, z13.s\n" - "fmla z31.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x25, x9, LSL #2]\n" + "fmla z22.s, p3/M, z4.s, z13.s\n" + "fmla z21.s, p3/M, z8.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x21, x9, LSL #2]\n" "ldr x21, [x15, #0x70]\n" - "fmla z28.s, p3/M, z1.s, z12.s\n" - "fmla z29.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x24, x9, LSL #2]\n" + "fmla z24.s, p3/M, z1.s, z16.s\n" + "fmla z23.s, p3/M, z0.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x20, x9, LSL #2]\n" "ldr x20, [x15, #0x78]\n" - "fmla z30.s, p3/M, z5.s, z10.s\n" - "fmla z31.s, p3/M, z4.s, z10.s\n" + "fmla z22.s, p3/M, z5.s, z20.s\n" + "fmla z21.s, p3/M, z4.s, z20.s\n" "incw x28\n" "mov p0.b, p2.b\n" - "fmla z28.s, p3/M, z2.s, z9.s\n" - "fmla z29.s, p3/M, z1.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x23, x9, LSL #2]\n" - "fmla z30.s, p3/M, z0.s, z11.s\n" - "fmla z31.s, p3/M, z2.s, z12.s\n" - "fmla z28.s, p3/M, z8.s, z10.s\n" - "fmla z29.s, p3/M, z7.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x22, x9, LSL #2]\n" - "fmla z30.s, p3/M, z3.s, z9.s\n" - "fmla z31.s, p3/M, z5.s, z10.s\n" - "fmla z28.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x21, x9, LSL #2]\n" - "fmla z29.s, p3/M, z5.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x20, x9, LSL #2]\n" - "fmla z30.s, p3/M, z7.s, z11.s\n" - "fmla z31.s, p3/M, z6.s, z11.s\n" - "fmla z28.s, p3/M, z6.s, z9.s\n" - "fmla z29.s, p3/M, z8.s, z10.s\n" - "fmax z28.s, p3/M, z28.s, z17.s\n" - "fmax z29.s, p3/M, z29.s, z17.s\n" - "fmla z30.s, p3/M, z8.s, z12.s\n" - "fmla z31.s, p3/M, z7.s, z12.s\n" - "fmax z30.s, p3/M, z30.s, z17.s\n" - "fmax z31.s, p3/M, z31.s, z17.s\n" - "fmin z28.s, p3/M, z28.s, z16.s\n" - "fmin z29.s, p3/M, z29.s, z16.s\n" - "st1w { z28.s }, p0, [x13, x28, LSL #2]\n" - "fmin z30.s, p3/M, z30.s, z16.s\n" - "fmin z31.s, p3/M, z31.s, z16.s\n" - "st1w { z29.s }, p0, [x12, x28, LSL #2]\n" - "st1w { z30.s }, p0, [x11, x28, LSL #2]\n" - "st1w { z31.s }, p0, [x10, x28, LSL #2]\n" + "fmla z24.s, p3/M, z2.s, z18.s\n" + "fmla z23.s, p3/M, z1.s, z18.s\n" + "ld1w { z19.s }, p2/Z, [x23, x9, LSL #2]\n" + "fmla z22.s, p3/M, z0.s, z17.s\n" + "fmla z21.s, p3/M, z2.s, z16.s\n" + "fmla z24.s, p3/M, z8.s, z20.s\n" + "fmla z23.s, p3/M, z7.s, z20.s\n" + "ld1w { z18.s }, p2/Z, [x22, x9, LSL #2]\n" + "fmla z22.s, p3/M, z3.s, z19.s\n" + "fmla z21.s, p3/M, z5.s, z18.s\n" + "fmla z24.s, p3/M, z3.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x21, x9, LSL #2]\n" + "fmla z23.s, p3/M, z5.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x20, x9, LSL #2]\n" + "fmla z22.s, p3/M, z7.s, z17.s\n" + "fmla z21.s, p3/M, z6.s, z17.s\n" + "fmla z24.s, p3/M, z6.s, z19.s\n" + "fmla z23.s, p3/M, z8.s, z18.s\n" + "fmax z24.s, p3/M, z24.s, z26.s\n" + "fmax z23.s, p3/M, z23.s, z26.s\n" + "fmla z22.s, p3/M, z8.s, z16.s\n" + "fmla z21.s, p3/M, z7.s, z16.s\n" + "fmax z22.s, p3/M, z22.s, z26.s\n" + "fmax z21.s, p3/M, z21.s, z26.s\n" + "fmin z24.s, p3/M, z24.s, z25.s\n" + "fmin z23.s, p3/M, z23.s, z25.s\n" + "st1w { z24.s }, p0, [x13, x28, LSL #2]\n" + "fmin z22.s, p3/M, z22.s, z25.s\n" + "fmin z21.s, p3/M, z21.s, z25.s\n" + "st1w { z23.s }, p0, [x12, x28, LSL #2]\n" + "st1w { z22.s }, p0, [x11, x28, LSL #2]\n" + "st1w { z21.s }, p0, [x10, x28, LSL #2]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp index 41ad19336458fa1d3a3c8c397d3be37920dc5e10..e4f432c9ed5434fea0990af20cc8d5b5fa28d8e5 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,19 +22,19 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include #pragma once -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { -void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); -void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); +void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(const float *const *const input_ptrs, float *const *const outptrs, const void *params, unsigned int n_channels, const float activation_min, const float activation_max); +void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(const unsigned int n_tile_rows, const unsigned int n_tile_cols, const float *inptr, int64_t ld_input_row, int64_t ld_input_col, float *outptr, int64_t ld_output_row, int64_t ld_output_col, const void *params, unsigned int n_channels, const float activation_min, const float activation_max); class sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst : public DepthwiseDepthfirstStrategy { @@ -57,7 +57,7 @@ class sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst : public DepthwiseDepthfirst constexpr static unsigned int output_cols = 3; sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst(const CPUInfo *) - : DepthwiseDepthfirstStrategy(3, 3, 1) {} + : Parent(output_rows, output_cols, kernel_rows, kernel_cols, stride_rows, stride_cols) {} arm_gemm::VLType get_vl_type(void) const override { return vl_type; } @@ -68,4 +68,4 @@ class sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst : public DepthwiseDepthfirst } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp index cda34358f504cc20eb3d02b6cac89fe3aad08989..015d0e63c28de88932ab81fb098878846dffbc4a 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -113,7 +113,7 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "madd x20, x8, x16, x20\n" // offset += tile_j * ld_output_col "add x9, x10, x23, LSL #2\n" "whilelt p2.s, XZR, %x[n_channels]\n" - "ld1w { z18.s }, p3/Z, [x13]\n" + "ld1w { z14.s }, p3/Z, [x13]\n" "mul x20, x20, x24\n" // offset *= output_tile_size "ld1w { z0.s }, p3/Z, [x13, #1, MUL VL]\n" "ld1w { z1.s }, p3/Z, [x13, #2, MUL VL]\n" @@ -129,10 +129,10 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "ld1w { z6.s }, p3/Z, [x13, #7, MUL VL]\n" "addvl x13, x13, #16\n" "add x24, x11, x21, LSL #2\n" - "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rw { z31.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" "cmp x15, %x[n_channels]\n" "add x23, x24, x21, LSL #2\n" - "ld1rw { z16.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "ld1rw { z30.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "ld1w { z7.s }, p3/Z, [x13, #-8, MUL VL]\n" "add x22, x16, x16\n" "mov x21, #0x0\n" @@ -146,131 +146,131 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "ld1w { z13.s }, p2/Z, [x10, x12, LSL #2]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "movprfx z24, z18\n fmla z24.s, p3/M, z7.s, z9.s\n" - "movprfx z23, z18\n fmla z23.s, p3/M, z8.s, z9.s\n" + "movprfx z29, z14\n fmla z29.s, p3/M, z7.s, z9.s\n" + "movprfx z28, z14\n fmla z28.s, p3/M, z8.s, z9.s\n" "whilelt p1.s, x15, %x[n_channels]\n" "incw x21\n" - "movprfx z25, z18\n fmla z25.s, p3/M, z6.s, z9.s\n" - "fmla z24.s, p3/M, z4.s, z13.s\n" + "movprfx z27, z14\n fmla z27.s, p3/M, z6.s, z9.s\n" + "fmla z29.s, p3/M, z4.s, z13.s\n" "incw x15\n" "mov p0.b, p2.b\n" - "movprfx z26, z18\n fmla z26.s, p3/M, z5.s, z9.s\n" - "movprfx z27, z18\n fmla z27.s, p3/M, z4.s, z9.s\n" + "movprfx z26, z14\n fmla z26.s, p3/M, z5.s, z9.s\n" + "movprfx z25, z14\n fmla z25.s, p3/M, z4.s, z9.s\n" "incw x20\n" - "movprfx z28, z18\n fmla z28.s, p3/M, z3.s, z9.s\n" - "fmla z23.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x9, x27, LSL #2]\n" - "fmla z25.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x9, x17, LSL #2]\n" - "movprfx z29, z18\n fmla z29.s, p3/M, z2.s, z9.s\n" - "fmla z24.s, p3/M, z6.s, z11.s\n" - "movprfx z31, z18\n fmla z31.s, p3/M, z0.s, z9.s\n" - "fmla z23.s, p3/M, z5.s, z13.s\n" - "fmla z25.s, p3/M, z3.s, z13.s\n" + "movprfx z24, z14\n fmla z24.s, p3/M, z3.s, z9.s\n" + "fmla z28.s, p3/M, z0.s, z10.s\n" + "ld1w { z23.s }, p2/Z, [x9, x27, LSL #2]\n" + "fmla z27.s, p3/M, z2.s, z11.s\n" + "ld1w { z18.s }, p2/Z, [x9, x17, LSL #2]\n" + "movprfx z22, z14\n fmla z22.s, p3/M, z2.s, z9.s\n" + "fmla z29.s, p3/M, z6.s, z18.s\n" + "movprfx z21, z14\n fmla z21.s, p3/M, z0.s, z9.s\n" + "fmla z28.s, p3/M, z5.s, z13.s\n" + "fmla z27.s, p3/M, z3.s, z13.s\n" "fmla z26.s, p3/M, z2.s, z13.s\n" - "fmla z27.s, p3/M, z1.s, z13.s\n" - "fmla z28.s, p3/M, z0.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x14, x17, LSL #2]\n" - "fmla z29.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x26, x25, LSL #2]\n" - "movprfx z30, z18\n fmla z30.s, p3/M, z1.s, z9.s\n" + "fmla z25.s, p3/M, z1.s, z13.s\n" "fmla z24.s, p3/M, z0.s, z13.s\n" - "ld1w { z18.s }, p3/Z, [x13]\n" - "fmla z31.s, p3/M, z8.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x14, x27, LSL #2]\n" - "fmla z23.s, p3/M, z7.s, z11.s\n" - "fmla z30.s, p3/M, z0.s, z11.s\n" - "fmla z26.s, p3/M, z4.s, z11.s\n" - "fmla z27.s, p3/M, z3.s, z11.s\n" - "fmla z29.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x10]\n" - "fmla z24.s, p3/M, z2.s, z12.s\n" - "fmla z25.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x28]\n" - "fmla z28.s, p3/M, z4.s, z10.s\n" - "fmla z23.s, p3/M, z1.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x10, x25, LSL #2]\n" - "fmla z30.s, p3/M, z2.s, z10.s\n" - "fmla z31.s, p3/M, z1.s, z10.s\n" - "fmla z24.s, p3/M, z8.s, z10.s\n" - "fmla z25.s, p3/M, z7.s, z10.s\n" - "fmla z27.s, p3/M, z5.s, z10.s\n" - "fmla z26.s, p3/M, z0.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x28, x12, LSL #2]\n" - "fmla z29.s, p3/M, z3.s, z12.s\n" - "fmla z28.s, p3/M, z2.s, z13.s\n" - "fmla z30.s, p3/M, z4.s, z10.s\n" - "fmla z31.s, p3/M, z3.s, z10.s\n" - "fmla z23.s, p3/M, z3.s, z11.s\n" - "fmla z25.s, p3/M, z5.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x28, x25, LSL #2]\n" - "ld1w { z13.s }, p2/Z, [x26, x17, LSL #2]\n" - "fmla z26.s, p3/M, z6.s, z12.s\n" - "fmla z27.s, p3/M, z7.s, z10.s\n" - "ld1w { z12.s }, p2/Z, [x10, x17, LSL #2]\n" - "fmla z29.s, p3/M, z5.s, z10.s\n" - "fmla z28.s, p3/M, z6.s, z10.s\n" - "fmla z31.s, p3/M, z5.s, z11.s\n" - "fmla z30.s, p3/M, z6.s, z13.s\n" - "fmla z26.s, p3/M, z8.s, z10.s\n" - "fmla z29.s, p3/M, z7.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x26, x27, LSL #2]\n" - "fmla z24.s, p3/M, z3.s, z12.s\n" - "fmla z27.s, p3/M, z0.s, z12.s\n" - "fmla z28.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x10, x27, LSL #2]\n" - "fmla z30.s, p3/M, z8.s, z13.s\n" + "ld1w { z17.s }, p2/Z, [x14, x17, LSL #2]\n" + "fmla z22.s, p3/M, z6.s, z12.s\n" + "ld1w { z16.s }, p2/Z, [x26, x25, LSL #2]\n" + "movprfx z20, z14\n fmla z20.s, p3/M, z1.s, z9.s\n" + "fmla z29.s, p3/M, z0.s, z17.s\n" + "ld1w { z14.s }, p3/Z, [x13]\n" + "fmla z21.s, p3/M, z8.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x14, x27, LSL #2]\n" + "fmla z28.s, p3/M, z7.s, z18.s\n" + "fmla z20.s, p3/M, z0.s, z18.s\n" + "fmla z26.s, p3/M, z4.s, z18.s\n" + "fmla z25.s, p3/M, z3.s, z18.s\n" + "fmla z22.s, p3/M, z1.s, z18.s\n" + "ld1w { z19.s }, p2/Z, [x10]\n" + "fmla z29.s, p3/M, z2.s, z16.s\n" + "fmla z27.s, p3/M, z1.s, z16.s\n" + "ld1w { z18.s }, p2/Z, [x28]\n" + "fmla z24.s, p3/M, z4.s, z23.s\n" + "fmla z28.s, p3/M, z1.s, z17.s\n" + "ld1w { z16.s }, p2/Z, [x10, x25, LSL #2]\n" + "fmla z20.s, p3/M, z2.s, z23.s\n" + "fmla z21.s, p3/M, z1.s, z23.s\n" + "fmla z29.s, p3/M, z8.s, z23.s\n" + "fmla z27.s, p3/M, z7.s, z23.s\n" + "fmla z25.s, p3/M, z5.s, z23.s\n" + "fmla z26.s, p3/M, z0.s, z19.s\n" + "ld1w { z17.s }, p2/Z, [x28, x12, LSL #2]\n" + "fmla z22.s, p3/M, z3.s, z18.s\n" + "fmla z24.s, p3/M, z2.s, z16.s\n" + "fmla z20.s, p3/M, z4.s, z17.s\n" + "fmla z21.s, p3/M, z3.s, z17.s\n" + "fmla z28.s, p3/M, z3.s, z19.s\n" + "fmla z27.s, p3/M, z5.s, z16.s\n" + "ld1w { z19.s }, p2/Z, [x28, x25, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x26, x17, LSL #2]\n" + "fmla z26.s, p3/M, z6.s, z18.s\n" + "fmla z25.s, p3/M, z7.s, z17.s\n" + "ld1w { z18.s }, p2/Z, [x10, x17, LSL #2]\n" + "fmla z22.s, p3/M, z5.s, z17.s\n" + "fmla z24.s, p3/M, z6.s, z17.s\n" + "fmla z21.s, p3/M, z5.s, z19.s\n" + "fmla z20.s, p3/M, z6.s, z16.s\n" + "fmla z26.s, p3/M, z8.s, z17.s\n" + "fmla z22.s, p3/M, z7.s, z16.s\n" + "ld1w { z17.s }, p2/Z, [x26, x27, LSL #2]\n" + "fmla z29.s, p3/M, z3.s, z18.s\n" + "fmla z25.s, p3/M, z0.s, z18.s\n" + "fmla z24.s, p3/M, z8.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x10, x27, LSL #2]\n" + "fmla z20.s, p3/M, z8.s, z17.s\n" "addvl x10, x10, #1\n" - "fmla z31.s, p3/M, z7.s, z13.s\n" - "fmla z23.s, p3/M, z4.s, z12.s\n" - "ld1w { z13.s }, p2/Z, [x28, x27, LSL #2]\n" - "fmla z26.s, p3/M, z1.s, z12.s\n" - "fmla z24.s, p3/M, z5.s, z11.s\n" - "ld1w { z12.s }, p2/Z, [x28, x17, LSL #2]\n" + "fmla z21.s, p3/M, z7.s, z17.s\n" + "fmla z28.s, p3/M, z4.s, z18.s\n" + "ld1w { z19.s }, p2/Z, [x28, x27, LSL #2]\n" + "fmla z26.s, p3/M, z1.s, z18.s\n" + "fmla z29.s, p3/M, z5.s, z16.s\n" + "ld1w { z17.s }, p2/Z, [x28, x17, LSL #2]\n" "addvl x28, x28, #1\n" - "fmla z25.s, p3/M, z4.s, z11.s\n" - "fmla z27.s, p3/M, z2.s, z11.s\n" - "fmla z28.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x14, x12, LSL #2]\n" - "fmla z29.s, p3/M, z4.s, z12.s\n" + "fmla z27.s, p3/M, z4.s, z16.s\n" + "fmla z25.s, p3/M, z2.s, z16.s\n" + "fmla z24.s, p3/M, z1.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x14, x12, LSL #2]\n" + "fmla z22.s, p3/M, z4.s, z17.s\n" "addvl x14, x14, #1\n" - "fmla z30.s, p3/M, z3.s, z12.s\n" - "fmla z31.s, p3/M, z4.s, z13.s\n" + "fmla z20.s, p3/M, z3.s, z17.s\n" + "fmla z21.s, p3/M, z4.s, z19.s\n" "ld1w { z4.s }, p3/Z, [x13, #5, MUL VL]\n" "ld1w { z10.s }, p1/Z, [x14]\n" - "fmla z26.s, p3/M, z7.s, z12.s\n" - "fmla z27.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x9]\n" - "fmla z23.s, p3/M, z2.s, z11.s\n" - "fmla z24.s, p3/M, z1.s, z11.s\n" - "fmax z24.s, p3/M, z24.s, z17.s\n" + "fmla z26.s, p3/M, z7.s, z17.s\n" + "fmla z25.s, p3/M, z6.s, z17.s\n" + "ld1w { z18.s }, p2/Z, [x9]\n" + "fmla z28.s, p3/M, z2.s, z16.s\n" + "fmla z29.s, p3/M, z1.s, z16.s\n" + "fmax z29.s, p3/M, z29.s, z31.s\n" "ld1w { z1.s }, p3/Z, [x13, #2, MUL VL]\n" - "fmla z25.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x9, x25, LSL #2]\n" - "fmla z28.s, p3/M, z7.s, z13.s\n" + "fmla z27.s, p3/M, z0.s, z16.s\n" + "ld1w { z17.s }, p2/Z, [x9, x25, LSL #2]\n" + "fmla z24.s, p3/M, z7.s, z19.s\n" "addvl x9, x9, #1\n" - "fmla z30.s, p3/M, z5.s, z13.s\n" - "fmla z29.s, p3/M, z0.s, z12.s\n" + "fmla z20.s, p3/M, z5.s, z19.s\n" + "fmla z22.s, p3/M, z0.s, z18.s\n" "ld1w { z0.s }, p3/Z, [x13, #1, MUL VL]\n" - "fmin z24.s, p3/M, z24.s, z16.s\n" - "fmla z31.s, p3/M, z2.s, z11.s\n" - "fmla z27.s, p3/M, z8.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x26, x12, LSL #2]\n" - "fmax z27.s, p3/M, z27.s, z17.s\n" - "fmla z23.s, p3/M, z6.s, z12.s\n" - "fmla z26.s, p3/M, z3.s, z12.s\n" - "fmax z23.s, p3/M, z23.s, z17.s\n" - "fmax z26.s, p3/M, z26.s, z17.s\n" - "fmla z25.s, p3/M, z8.s, z11.s\n" - "fmla z28.s, p3/M, z5.s, z11.s\n" - "fmax z25.s, p3/M, z25.s, z17.s\n" - "fmax z28.s, p3/M, z28.s, z17.s\n" - "fmla z29.s, p3/M, z8.s, z13.s\n" - "fmla z30.s, p3/M, z7.s, z13.s\n" - "fmax z29.s, p3/M, z29.s, z17.s\n" - "fmax z30.s, p3/M, z30.s, z17.s\n" - "fmla z31.s, p3/M, z6.s, z13.s\n" - "fmax z31.s, p3/M, z31.s, z17.s\n" + "fmin z29.s, p3/M, z29.s, z30.s\n" + "fmla z21.s, p3/M, z2.s, z17.s\n" + "fmla z25.s, p3/M, z8.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x26, x12, LSL #2]\n" + "fmax z25.s, p3/M, z25.s, z31.s\n" + "fmla z28.s, p3/M, z6.s, z18.s\n" + "fmla z26.s, p3/M, z3.s, z18.s\n" + "fmax z28.s, p3/M, z28.s, z31.s\n" + "fmax z26.s, p3/M, z26.s, z31.s\n" + "fmla z27.s, p3/M, z8.s, z17.s\n" + "fmla z24.s, p3/M, z5.s, z17.s\n" + "fmax z27.s, p3/M, z27.s, z31.s\n" + "fmax z24.s, p3/M, z24.s, z31.s\n" + "fmla z22.s, p3/M, z8.s, z16.s\n" + "fmla z20.s, p3/M, z7.s, z16.s\n" + "fmax z22.s, p3/M, z22.s, z31.s\n" + "fmax z20.s, p3/M, z20.s, z31.s\n" + "fmla z21.s, p3/M, z6.s, z16.s\n" + "fmax z21.s, p3/M, z21.s, z31.s\n" "addvl x26, x26, #1\n" "ld1w { z2.s }, p3/Z, [x13, #3, MUL VL]\n" "ld1w { z3.s }, p3/Z, [x13, #4, MUL VL]\n" @@ -279,182 +279,182 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "cmp x15, %x[n_channels]\n" "ld1w { z6.s }, p3/Z, [x13, #7, MUL VL]\n" "addvl x13, x13, #16\n" - "fmin z23.s, p3/M, z23.s, z16.s\n" + "fmin z28.s, p3/M, z28.s, z30.s\n" "ld1w { z9.s }, p1/Z, [x9, x12, LSL #2]\n" - "fmin z25.s, p3/M, z25.s, z16.s\n" - "fmin z26.s, p3/M, z26.s, z16.s\n" + "fmin z27.s, p3/M, z27.s, z30.s\n" + "fmin z26.s, p3/M, z26.s, z30.s\n" "ld1w { z11.s }, p1/Z, [x14, x25, LSL #2]\n" "ld1w { z12.s }, p1/Z, [x26]\n" - "fmin z27.s, p3/M, z27.s, z16.s\n" - "fmin z28.s, p3/M, z28.s, z16.s\n" + "fmin z25.s, p3/M, z25.s, z30.s\n" + "fmin z24.s, p3/M, z24.s, z30.s\n" "ld1w { z13.s }, p1/Z, [x10, x12, LSL #2]\n" - "st1w { z23.s }, p0, [x11]\n" - "fmin z29.s, p3/M, z29.s, z16.s\n" - "fmin z30.s, p3/M, z30.s, z16.s\n" - "st1w { z24.s }, p0, [x11, x16, LSL #2]\n" + "st1w { z28.s }, p0, [x11]\n" + "fmin z22.s, p3/M, z22.s, z30.s\n" + "fmin z20.s, p3/M, z20.s, z30.s\n" + "st1w { z29.s }, p0, [x11, x16, LSL #2]\n" "ld1w { z7.s }, p3/Z, [x13, #-8, MUL VL]\n" - "fmin z31.s, p3/M, z31.s, z16.s\n" - "st1w { z25.s }, p0, [x11, x22, LSL #2]\n" + "fmin z21.s, p3/M, z21.s, z30.s\n" + "st1w { z27.s }, p0, [x11, x22, LSL #2]\n" "addvl x11, x11, #1\n" "ld1w { z8.s }, p3/Z, [x13, #-7, MUL VL]\n" "st1w { z26.s }, p0, [x24]\n" "addvl x13, x13, #-6\n" - "st1w { z27.s }, p0, [x24, x16, LSL #2]\n" - "st1w { z28.s }, p0, [x24, x22, LSL #2]\n" + "st1w { z25.s }, p0, [x24, x16, LSL #2]\n" + "st1w { z24.s }, p0, [x24, x22, LSL #2]\n" "addvl x24, x24, #1\n" - "st1w { z29.s }, p0, [x23]\n" - "st1w { z30.s }, p0, [x23, x16, LSL #2]\n" - "st1w { z31.s }, p0, [x23, x22, LSL #2]\n" + "st1w { z22.s }, p0, [x23]\n" + "st1w { z20.s }, p0, [x23, x16, LSL #2]\n" + "st1w { z21.s }, p0, [x23, x22, LSL #2]\n" "addvl x23, x23, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "movprfx z24, z18\n fmla z24.s, p3/M, z7.s, z9.s\n" - "movprfx z23, z18\n fmla z23.s, p3/M, z8.s, z9.s\n" + "movprfx z29, z14\n fmla z29.s, p3/M, z7.s, z9.s\n" + "movprfx z28, z14\n fmla z28.s, p3/M, z8.s, z9.s\n" "ldr x8, [%x[params_struct], %[offsetof_args_tile_j]]\n" "ldr x13, [%x[params_struct], %[offsetof_args_tile_i]]\n" - "movprfx z25, z18\n fmla z25.s, p3/M, z6.s, z9.s\n" - "fmla z24.s, p3/M, z4.s, z13.s\n" + "movprfx z27, z14\n fmla z27.s, p3/M, z6.s, z9.s\n" + "fmla z29.s, p3/M, z4.s, z13.s\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" "add x8, x8, #0x1\n" - "movprfx z26, z18\n fmla z26.s, p3/M, z5.s, z9.s\n" - "movprfx z27, z18\n fmla z27.s, p3/M, z4.s, z9.s\n" + "movprfx z26, z14\n fmla z26.s, p3/M, z5.s, z9.s\n" + "movprfx z25, z14\n fmla z25.s, p3/M, z4.s, z9.s\n" "cmp x8, x20\n" "add x21, x13, #0x1\n" - "movprfx z28, z18\n fmla z28.s, p3/M, z3.s, z9.s\n" - "fmla z23.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x9, x27, LSL #2]\n" + "movprfx z24, z14\n fmla z24.s, p3/M, z3.s, z9.s\n" + "fmla z28.s, p3/M, z0.s, z10.s\n" + "ld1w { z23.s }, p2/Z, [x9, x27, LSL #2]\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" - "fmla z25.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x9, x17, LSL #2]\n" - "movprfx z29, z18\n fmla z29.s, p3/M, z2.s, z9.s\n" + "fmla z27.s, p3/M, z2.s, z11.s\n" + "ld1w { z18.s }, p2/Z, [x9, x17, LSL #2]\n" + "movprfx z22, z14\n fmla z22.s, p3/M, z2.s, z9.s\n" "csel x13, x13, x21, LT\n" - "fmla z24.s, p3/M, z6.s, z11.s\n" - "movprfx z31, z18\n fmla z31.s, p3/M, z0.s, z9.s\n" + "fmla z29.s, p3/M, z6.s, z18.s\n" + "movprfx z21, z14\n fmla z21.s, p3/M, z0.s, z9.s\n" "mov p0.b, p2.b\n" "csel x8, x8, XZR, LT\n" - "fmla z23.s, p3/M, z5.s, z13.s\n" - "fmla z25.s, p3/M, z3.s, z13.s\n" + "fmla z28.s, p3/M, z5.s, z13.s\n" + "fmla z27.s, p3/M, z3.s, z13.s\n" "cmp x13, x20\n" "fmla z26.s, p3/M, z2.s, z13.s\n" - "fmla z27.s, p3/M, z1.s, z13.s\n" - "fmla z28.s, p3/M, z0.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x14, x17, LSL #2]\n" - "fmla z29.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x26, x25, LSL #2]\n" - "movprfx z30, z18\n fmla z30.s, p3/M, z1.s, z9.s\n" + "fmla z25.s, p3/M, z1.s, z13.s\n" "fmla z24.s, p3/M, z0.s, z13.s\n" - "fmla z31.s, p3/M, z8.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x14, x27, LSL #2]\n" - "fmla z23.s, p3/M, z7.s, z11.s\n" - "fmla z30.s, p3/M, z0.s, z11.s\n" - "fmla z26.s, p3/M, z4.s, z11.s\n" - "fmla z27.s, p3/M, z3.s, z11.s\n" - "fmla z29.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x10]\n" - "fmla z24.s, p3/M, z2.s, z12.s\n" - "fmla z25.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x28]\n" - "fmla z28.s, p3/M, z4.s, z10.s\n" - "fmla z23.s, p3/M, z1.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x10, x25, LSL #2]\n" - "fmla z30.s, p3/M, z2.s, z10.s\n" - "fmla z31.s, p3/M, z1.s, z10.s\n" - "fmla z24.s, p3/M, z8.s, z10.s\n" - "fmla z25.s, p3/M, z7.s, z10.s\n" - "fmla z27.s, p3/M, z5.s, z10.s\n" - "fmla z26.s, p3/M, z0.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x28, x12, LSL #2]\n" - "fmla z29.s, p3/M, z3.s, z12.s\n" - "fmla z28.s, p3/M, z2.s, z13.s\n" - "fmla z30.s, p3/M, z4.s, z10.s\n" - "fmla z31.s, p3/M, z3.s, z10.s\n" - "fmla z23.s, p3/M, z3.s, z11.s\n" - "fmla z25.s, p3/M, z5.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x28, x25, LSL #2]\n" - "ld1w { z13.s }, p2/Z, [x26, x17, LSL #2]\n" - "fmla z26.s, p3/M, z6.s, z12.s\n" - "fmla z27.s, p3/M, z7.s, z10.s\n" - "ld1w { z12.s }, p2/Z, [x10, x17, LSL #2]\n" - "fmla z29.s, p3/M, z5.s, z10.s\n" - "fmla z28.s, p3/M, z6.s, z10.s\n" - "fmla z31.s, p3/M, z5.s, z11.s\n" - "fmla z30.s, p3/M, z6.s, z13.s\n" - "fmla z26.s, p3/M, z8.s, z10.s\n" - "fmla z29.s, p3/M, z7.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x26, x27, LSL #2]\n" - "fmla z24.s, p3/M, z3.s, z12.s\n" - "fmla z27.s, p3/M, z0.s, z12.s\n" - "fmla z28.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x10, x27, LSL #2]\n" - "fmla z30.s, p3/M, z8.s, z13.s\n" - "fmla z31.s, p3/M, z7.s, z13.s\n" - "fmla z23.s, p3/M, z4.s, z12.s\n" - "ld1w { z13.s }, p2/Z, [x28, x27, LSL #2]\n" - "fmla z26.s, p3/M, z1.s, z12.s\n" - "fmla z24.s, p3/M, z5.s, z11.s\n" - "ld1w { z12.s }, p2/Z, [x28, x17, LSL #2]\n" - "fmla z25.s, p3/M, z4.s, z11.s\n" - "fmla z27.s, p3/M, z2.s, z11.s\n" - "fmla z28.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x14, x12, LSL #2]\n" - "fmla z29.s, p3/M, z4.s, z12.s\n" - "fmla z30.s, p3/M, z3.s, z12.s\n" - "fmla z31.s, p3/M, z4.s, z13.s\n" - "fmla z26.s, p3/M, z7.s, z12.s\n" - "fmla z27.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x9]\n" - "fmla z23.s, p3/M, z2.s, z11.s\n" - "fmla z24.s, p3/M, z1.s, z11.s\n" - "fmax z24.s, p3/M, z24.s, z17.s\n" - "fmin z24.s, p3/M, z24.s, z16.s\n" - "fmla z25.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x9, x25, LSL #2]\n" - "fmla z28.s, p3/M, z7.s, z13.s\n" - "fmla z30.s, p3/M, z5.s, z13.s\n" - "fmla z29.s, p3/M, z0.s, z12.s\n" - "fmla z31.s, p3/M, z2.s, z11.s\n" - "fmla z27.s, p3/M, z8.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x26, x12, LSL #2]\n" - "fmax z27.s, p3/M, z27.s, z17.s\n" - "fmla z23.s, p3/M, z6.s, z12.s\n" - "fmla z26.s, p3/M, z3.s, z12.s\n" - "fmax z23.s, p3/M, z23.s, z17.s\n" - "fmax z26.s, p3/M, z26.s, z17.s\n" - "fmla z25.s, p3/M, z8.s, z11.s\n" - "fmla z28.s, p3/M, z5.s, z11.s\n" - "fmax z25.s, p3/M, z25.s, z17.s\n" - "fmax z28.s, p3/M, z28.s, z17.s\n" - "fmla z29.s, p3/M, z8.s, z13.s\n" - "fmla z30.s, p3/M, z7.s, z13.s\n" - "fmax z29.s, p3/M, z29.s, z17.s\n" - "fmax z30.s, p3/M, z30.s, z17.s\n" - "fmla z31.s, p3/M, z6.s, z13.s\n" - "fmax z31.s, p3/M, z31.s, z17.s\n" - "fmin z23.s, p3/M, z23.s, z16.s\n" - "st1w { z23.s }, p0, [x11]\n" - "fmin z25.s, p3/M, z25.s, z16.s\n" - "fmin z26.s, p3/M, z26.s, z16.s\n" - "st1w { z24.s }, p0, [x11, x16, LSL #2]\n" - "fmin z27.s, p3/M, z27.s, z16.s\n" - "fmin z28.s, p3/M, z28.s, z16.s\n" - "st1w { z25.s }, p0, [x11, x22, LSL #2]\n" - "fmin z29.s, p3/M, z29.s, z16.s\n" - "fmin z30.s, p3/M, z30.s, z16.s\n" + "ld1w { z17.s }, p2/Z, [x14, x17, LSL #2]\n" + "fmla z22.s, p3/M, z6.s, z12.s\n" + "ld1w { z16.s }, p2/Z, [x26, x25, LSL #2]\n" + "movprfx z20, z14\n fmla z20.s, p3/M, z1.s, z9.s\n" + "fmla z29.s, p3/M, z0.s, z17.s\n" + "fmla z21.s, p3/M, z8.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x14, x27, LSL #2]\n" + "fmla z28.s, p3/M, z7.s, z18.s\n" + "fmla z20.s, p3/M, z0.s, z18.s\n" + "fmla z26.s, p3/M, z4.s, z18.s\n" + "fmla z25.s, p3/M, z3.s, z18.s\n" + "fmla z22.s, p3/M, z1.s, z18.s\n" + "ld1w { z19.s }, p2/Z, [x10]\n" + "fmla z29.s, p3/M, z2.s, z16.s\n" + "fmla z27.s, p3/M, z1.s, z16.s\n" + "ld1w { z18.s }, p2/Z, [x28]\n" + "fmla z24.s, p3/M, z4.s, z23.s\n" + "fmla z28.s, p3/M, z1.s, z17.s\n" + "ld1w { z16.s }, p2/Z, [x10, x25, LSL #2]\n" + "fmla z20.s, p3/M, z2.s, z23.s\n" + "fmla z21.s, p3/M, z1.s, z23.s\n" + "fmla z29.s, p3/M, z8.s, z23.s\n" + "fmla z27.s, p3/M, z7.s, z23.s\n" + "fmla z25.s, p3/M, z5.s, z23.s\n" + "fmla z26.s, p3/M, z0.s, z19.s\n" + "ld1w { z17.s }, p2/Z, [x28, x12, LSL #2]\n" + "fmla z22.s, p3/M, z3.s, z18.s\n" + "fmla z24.s, p3/M, z2.s, z16.s\n" + "fmla z20.s, p3/M, z4.s, z17.s\n" + "fmla z21.s, p3/M, z3.s, z17.s\n" + "fmla z28.s, p3/M, z3.s, z19.s\n" + "fmla z27.s, p3/M, z5.s, z16.s\n" + "ld1w { z19.s }, p2/Z, [x28, x25, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x26, x17, LSL #2]\n" + "fmla z26.s, p3/M, z6.s, z18.s\n" + "fmla z25.s, p3/M, z7.s, z17.s\n" + "ld1w { z18.s }, p2/Z, [x10, x17, LSL #2]\n" + "fmla z22.s, p3/M, z5.s, z17.s\n" + "fmla z24.s, p3/M, z6.s, z17.s\n" + "fmla z21.s, p3/M, z5.s, z19.s\n" + "fmla z20.s, p3/M, z6.s, z16.s\n" + "fmla z26.s, p3/M, z8.s, z17.s\n" + "fmla z22.s, p3/M, z7.s, z16.s\n" + "ld1w { z17.s }, p2/Z, [x26, x27, LSL #2]\n" + "fmla z29.s, p3/M, z3.s, z18.s\n" + "fmla z25.s, p3/M, z0.s, z18.s\n" + "fmla z24.s, p3/M, z8.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x10, x27, LSL #2]\n" + "fmla z20.s, p3/M, z8.s, z17.s\n" + "fmla z21.s, p3/M, z7.s, z17.s\n" + "fmla z28.s, p3/M, z4.s, z18.s\n" + "ld1w { z19.s }, p2/Z, [x28, x27, LSL #2]\n" + "fmla z26.s, p3/M, z1.s, z18.s\n" + "fmla z29.s, p3/M, z5.s, z16.s\n" + "ld1w { z17.s }, p2/Z, [x28, x17, LSL #2]\n" + "fmla z27.s, p3/M, z4.s, z16.s\n" + "fmla z25.s, p3/M, z2.s, z16.s\n" + "fmla z24.s, p3/M, z1.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x14, x12, LSL #2]\n" + "fmla z22.s, p3/M, z4.s, z17.s\n" + "fmla z20.s, p3/M, z3.s, z17.s\n" + "fmla z21.s, p3/M, z4.s, z19.s\n" + "fmla z26.s, p3/M, z7.s, z17.s\n" + "fmla z25.s, p3/M, z6.s, z17.s\n" + "ld1w { z18.s }, p2/Z, [x9]\n" + "fmla z28.s, p3/M, z2.s, z16.s\n" + "fmla z29.s, p3/M, z1.s, z16.s\n" + "fmax z29.s, p3/M, z29.s, z31.s\n" + "fmin z29.s, p3/M, z29.s, z30.s\n" + "fmla z27.s, p3/M, z0.s, z16.s\n" + "ld1w { z17.s }, p2/Z, [x9, x25, LSL #2]\n" + "fmla z24.s, p3/M, z7.s, z19.s\n" + "fmla z20.s, p3/M, z5.s, z19.s\n" + "fmla z22.s, p3/M, z0.s, z18.s\n" + "fmla z21.s, p3/M, z2.s, z17.s\n" + "fmla z25.s, p3/M, z8.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x26, x12, LSL #2]\n" + "fmax z25.s, p3/M, z25.s, z31.s\n" + "fmla z28.s, p3/M, z6.s, z18.s\n" + "fmla z26.s, p3/M, z3.s, z18.s\n" + "fmax z28.s, p3/M, z28.s, z31.s\n" + "fmax z26.s, p3/M, z26.s, z31.s\n" + "fmla z27.s, p3/M, z8.s, z17.s\n" + "fmla z24.s, p3/M, z5.s, z17.s\n" + "fmax z27.s, p3/M, z27.s, z31.s\n" + "fmax z24.s, p3/M, z24.s, z31.s\n" + "fmla z22.s, p3/M, z8.s, z16.s\n" + "fmla z20.s, p3/M, z7.s, z16.s\n" + "fmax z22.s, p3/M, z22.s, z31.s\n" + "fmax z20.s, p3/M, z20.s, z31.s\n" + "fmla z21.s, p3/M, z6.s, z16.s\n" + "fmax z21.s, p3/M, z21.s, z31.s\n" + "fmin z28.s, p3/M, z28.s, z30.s\n" + "st1w { z28.s }, p0, [x11]\n" + "fmin z27.s, p3/M, z27.s, z30.s\n" + "fmin z26.s, p3/M, z26.s, z30.s\n" + "st1w { z29.s }, p0, [x11, x16, LSL #2]\n" + "fmin z25.s, p3/M, z25.s, z30.s\n" + "fmin z24.s, p3/M, z24.s, z30.s\n" + "st1w { z27.s }, p0, [x11, x22, LSL #2]\n" + "fmin z22.s, p3/M, z22.s, z30.s\n" + "fmin z20.s, p3/M, z20.s, z30.s\n" "st1w { z26.s }, p0, [x24]\n" - "fmin z31.s, p3/M, z31.s, z16.s\n" - "st1w { z27.s }, p0, [x24, x16, LSL #2]\n" - "st1w { z28.s }, p0, [x24, x22, LSL #2]\n" - "st1w { z29.s }, p0, [x23]\n" - "st1w { z30.s }, p0, [x23, x16, LSL #2]\n" - "st1w { z31.s }, p0, [x23, x22, LSL #2]\n" + "fmin z21.s, p3/M, z21.s, z30.s\n" + "st1w { z25.s }, p0, [x24, x16, LSL #2]\n" + "st1w { z24.s }, p0, [x24, x22, LSL #2]\n" + "st1w { z22.s }, p0, [x23]\n" + "st1w { z20.s }, p0, [x23, x16, LSL #2]\n" + "st1w { z21.s }, p0, [x23, x22, LSL #2]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp index 2eed8cb0c4ea5230feee624f7be42a04323258fc..4809b0c45ca9cca5fe542f8795527c851de9535a 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -88,390 +88,390 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( __asm__ __volatile__( "ptrue p3.b\n" - "ldr x17, [%x[params_struct], %[offsetof_args_params]]\n" - "add x16, %x[params_struct], %[offsetof_Args_inptrs]\n" - "ld1w { z18.s }, p3/Z, [x17]\n" - "cntw x15\n" - "mov x14, #0x0\n" - "ld1w { z0.s }, p3/Z, [x17, #1, MUL VL]\n" - "ld1w { z1.s }, p3/Z, [x17, #2, MUL VL]\n" + "ldr x8, [%x[params_struct], %[offsetof_args_params]]\n" + "add x17, %x[params_struct], %[offsetof_Args_inptrs]\n" + "ld1w { z14.s }, p3/Z, [x8]\n" + "cntw x16\n" + "mov x15, #0x0\n" + "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n" + "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n" "whilelt p2.s, XZR, %x[n_channels]\n" - "ld1w { z2.s }, p3/Z, [x17, #3, MUL VL]\n" - "ld1w { z3.s }, p3/Z, [x17, #4, MUL VL]\n" - "cmp x15, %x[n_channels]\n" - "ld1w { z4.s }, p3/Z, [x17, #5, MUL VL]\n" - "ld1w { z5.s }, p3/Z, [x17, #6, MUL VL]\n" - "sub x13, XZR, x15\n" - "ld1w { z6.s }, p3/Z, [x17, #7, MUL VL]\n" - "addvl x17, x17, #16\n" - "ldp x12, x11, [x16, #0x0]\n" - "ldp x10, x9, [x16, #0x10]\n" - "ldr x28, [x16, #0x20]\n" - "ldr x27, [%x[params_struct], %[offsetof_args_outptrs]]\n" - "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "ld1rw { z16.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "ld1w { z7.s }, p3/Z, [x17, #-8, MUL VL]\n" - "ld1w { z8.s }, p3/Z, [x17, #-7, MUL VL]\n" - "ld1w { z9.s }, p2/Z, [x12, x14, LSL #2]\n" - "addvl x17, x17, #-6\n" - "ld1w { z10.s }, p2/Z, [x11, x14, LSL #2]\n" - "ld1w { z11.s }, p2/Z, [x10, x14, LSL #2]\n" - "ld1w { z12.s }, p2/Z, [x9, x14, LSL #2]\n" - "ld1w { z13.s }, p2/Z, [x28, x14, LSL #2]\n" + "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n" + "cmp x16, %x[n_channels]\n" + "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n" + "ld1w { z5.s }, p3/Z, [x8, #6, MUL VL]\n" + "sub x14, XZR, x16\n" + "ld1w { z6.s }, p3/Z, [x8, #7, MUL VL]\n" + "addvl x8, x8, #16\n" + "ldp x24, x23, [x17, #0x0]\n" + "ldp x22, x21, [x17, #0x10]\n" + "ldr x20, [x17, #0x20]\n" + "ldr x13, [%x[params_struct], %[offsetof_args_outptrs]]\n" + "ld1rw { z31.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rw { z30.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "ld1w { z7.s }, p3/Z, [x8, #-8, MUL VL]\n" + "ld1w { z8.s }, p3/Z, [x8, #-7, MUL VL]\n" + "ld1w { z9.s }, p2/Z, [x24, x15, LSL #2]\n" + "addvl x8, x8, #-6\n" + "ld1w { z10.s }, p2/Z, [x23, x15, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x22, x15, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x21, x15, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x20, x15, LSL #2]\n" "bge 2f\n" "1:" // Channel loop - "movprfx z23, z18\n fmla z23.s, p3/M, z8.s, z9.s\n" - "movprfx z24, z18\n fmla z24.s, p3/M, z7.s, z9.s\n" - "ldr x26, [x16, #0x30]\n" - "ldr x25, [x16, #0x38]\n" - "movprfx z25, z18\n fmla z25.s, p3/M, z6.s, z9.s\n" - "fmla z23.s, p3/M, z0.s, z10.s\n" - "ldr x24, [x16, #0x28]\n" - "ldr x11, [x16, #0x48]\n" - "fmla z24.s, p3/M, z4.s, z13.s\n" - "movprfx z26, z18\n fmla z26.s, p3/M, z5.s, z9.s\n" - "ldr x12, [x16, #0x40]\n" - "ld1w { z10.s }, p2/Z, [x11, x14, LSL #2]\n" - "movprfx z27, z18\n fmla z27.s, p3/M, z4.s, z9.s\n" - "movprfx z28, z18\n fmla z28.s, p3/M, z3.s, z9.s\n" - "ldr x10, [x16, #0x50]\n" - "ldr x9, [x16, #0x58]\n" - "fmla z25.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x14, LSL #2]\n" - "movprfx z29, z18\n fmla z29.s, p3/M, z2.s, z9.s\n" - "ldr x28, [x16, #0x60]\n" - "fmla z23.s, p3/M, z5.s, z13.s\n" - "fmla z24.s, p3/M, z6.s, z11.s\n" - "ldr x26, [x16, #0x70]\n" - "ldr x11, [x16, #0x88]\n" - "movprfx z31, z18\n fmla z31.s, p3/M, z0.s, z9.s\n" - "fmla z25.s, p3/M, z3.s, z13.s\n" - "incw x13\n" + "movprfx z29, z14\n fmla z29.s, p3/M, z8.s, z9.s\n" + "movprfx z28, z14\n fmla z28.s, p3/M, z7.s, z9.s\n" + "ldr x23, [x17, #0x30]\n" + "ldr x26, [x17, #0x38]\n" + "movprfx z27, z14\n fmla z27.s, p3/M, z6.s, z9.s\n" + "fmla z29.s, p3/M, z0.s, z10.s\n" + "ldr x22, [x17, #0x28]\n" + "ldr x21, [x17, #0x48]\n" + "fmla z28.s, p3/M, z4.s, z13.s\n" + "movprfx z26, z14\n fmla z26.s, p3/M, z5.s, z9.s\n" + "ldr x20, [x17, #0x40]\n" + "ld1w { z19.s }, p2/Z, [x21, x15, LSL #2]\n" + "movprfx z25, z14\n fmla z25.s, p3/M, z4.s, z9.s\n" + "movprfx z24, z14\n fmla z24.s, p3/M, z3.s, z9.s\n" + "ldr x25, [x17, #0x50]\n" + "ldr x24, [x17, #0x58]\n" + "fmla z27.s, p3/M, z2.s, z11.s\n" + "ld1w { z18.s }, p2/Z, [x23, x15, LSL #2]\n" + "movprfx z23, z14\n fmla z23.s, p3/M, z2.s, z9.s\n" + "ldr x23, [x17, #0x60]\n" + "fmla z29.s, p3/M, z5.s, z13.s\n" + "fmla z28.s, p3/M, z6.s, z18.s\n" + "ldr x12, [x17, #0x70]\n" + "ldr x11, [x17, #0x88]\n" + "movprfx z22, z14\n fmla z22.s, p3/M, z0.s, z9.s\n" + "fmla z27.s, p3/M, z3.s, z13.s\n" + "incw x14\n" "mov p1.b, p2.b\n" "fmla z26.s, p3/M, z2.s, z13.s\n" - "fmla z27.s, p3/M, z1.s, z13.s\n" - "ldr x23, [x27, #0x0]\n" - "whilelt p0.s, x15, %x[n_channels]\n" - "fmla z28.s, p3/M, z0.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x25, x14, LSL #2]\n" - "fmla z29.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x24, x14, LSL #2]\n" - "movprfx z30, z18\n fmla z30.s, p3/M, z1.s, z9.s\n" - "fmla z23.s, p3/M, z7.s, z11.s\n" - "ldr x24, [x16, #0x68]\n" - "ldr x25, [x16, #0x78]\n" + "fmla z25.s, p3/M, z1.s, z13.s\n" + "ldr x10, [x13, #0x0]\n" + "whilelt p0.s, x16, %x[n_channels]\n" "fmla z24.s, p3/M, z0.s, z13.s\n" - "fmla z31.s, p3/M, z8.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x12, x14, LSL #2]\n" - "ldr x12, [x16, #0x80]\n" - "fmla z26.s, p3/M, z4.s, z11.s\n" - "fmla z27.s, p3/M, z3.s, z11.s\n" - "ldr x22, [x27, #0x8]\n" - "ldr x21, [x27, #0x10]\n" - "fmla z30.s, p3/M, z0.s, z11.s\n" - "fmla z28.s, p3/M, z4.s, z10.s\n" - "ldr x20, [x27, #0x18]\n" - "ld1w { z18.s }, p3/Z, [x17]\n" - "fmla z29.s, p3/M, z1.s, z11.s\n" - "fmla z23.s, p3/M, z1.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x10, x14, LSL #2]\n" - "ld1w { z13.s }, p2/Z, [x9, x14, LSL #2]\n" - "fmla z24.s, p3/M, z2.s, z12.s\n" - "fmla z25.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x28, x14, LSL #2]\n" - "ldr x10, [x16, #0x90]\n" - "fmla z27.s, p3/M, z5.s, z10.s\n" - "fmla z30.s, p3/M, z2.s, z10.s\n" - "ldr x28, [x16, #0xa0]\n" - "ldr x9, [x16, #0x98]\n" - "fmla z26.s, p3/M, z0.s, z11.s\n" - "fmla z28.s, p3/M, z2.s, z13.s\n" - "fmla z24.s, p3/M, z8.s, z10.s\n" - "fmla z25.s, p3/M, z7.s, z10.s\n" - "fmla z31.s, p3/M, z1.s, z10.s\n" - "fmla z29.s, p3/M, z3.s, z12.s\n" - "ld1w { z10.s }, p2/Z, [x24, x14, LSL #2]\n" - "ldr x24, [x16, #0xa8]\n" - "fmla z26.s, p3/M, z6.s, z12.s\n" - "fmla z27.s, p3/M, z7.s, z10.s\n" - "ld1w { z12.s }, p2/Z, [x12, x14, LSL #2]\n" - "ldr x12, [x16, #0xc0]\n" - "fmla z28.s, p3/M, z6.s, z10.s\n" - "fmla z30.s, p3/M, z4.s, z10.s\n" - "fmla z23.s, p3/M, z3.s, z11.s\n" - "fmla z25.s, p3/M, z5.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x26, x14, LSL #2]\n" - "ld1w { z13.s }, p2/Z, [x25, x14, LSL #2]\n" - "fmla z29.s, p3/M, z5.s, z10.s\n" - "fmla z31.s, p3/M, z3.s, z10.s\n" - "ldr x26, [x16, #0xb0]\n" - "ldr x25, [x16, #0xb8]\n" - "fmla z26.s, p3/M, z8.s, z10.s\n" - "fmla z28.s, p3/M, z8.s, z11.s\n" - "fmla z30.s, p3/M, z6.s, z13.s\n" - "fmla z24.s, p3/M, z3.s, z12.s\n" - "fmla z27.s, p3/M, z0.s, z12.s\n" - "fmla z31.s, p3/M, z5.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x11, x14, LSL #2]\n" - "fmla z29.s, p3/M, z7.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x10, x14, LSL #2]\n" - "fmla z23.s, p3/M, z4.s, z12.s\n" - "fmla z26.s, p3/M, z1.s, z12.s\n" - "fmla z24.s, p3/M, z5.s, z11.s\n" - "ld1w { z12.s }, p2/Z, [x9, x14, LSL #2]\n" - "fmla z25.s, p3/M, z4.s, z11.s\n" - "fmla z27.s, p3/M, z2.s, z11.s\n" - "fmla z28.s, p3/M, z1.s, z11.s\n" - "fmla z30.s, p3/M, z8.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x28, x14, LSL #2]\n" - "ldr x28, [x16, #0x20]\n" - "fmla z31.s, p3/M, z7.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x24, x14, LSL #2]\n" - "fmla z23.s, p3/M, z2.s, z11.s\n" - "fmla z26.s, p3/M, z7.s, z12.s\n" - "fmla z27.s, p3/M, z6.s, z12.s\n" - "fmla z29.s, p3/M, z4.s, z12.s\n" - "fmla z30.s, p3/M, z3.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x26, x14, LSL #2]\n" - "fmla z31.s, p3/M, z4.s, z13.s\n" - "fmla z24.s, p3/M, z1.s, z11.s\n" - "fmax z24.s, p3/M, z24.s, z17.s\n" - "fmin z24.s, p3/M, z24.s, z16.s\n" - "fmla z25.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x25, x14, LSL #2]\n" + "ld1w { z17.s }, p2/Z, [x26, x15, LSL #2]\n" "fmla z23.s, p3/M, z6.s, z12.s\n" - "fmax z23.s, p3/M, z23.s, z17.s\n" - "fmla z28.s, p3/M, z7.s, z13.s\n" - "fmla z30.s, p3/M, z5.s, z13.s\n" - "fmin z23.s, p3/M, z23.s, z16.s\n" - "st1w { z23.s }, p1, [x23, x13, LSL #2]\n" - "fmla z29.s, p3/M, z0.s, z12.s\n" - "fmla z31.s, p3/M, z2.s, z11.s\n" - "ldr x23, [x27, #0x20]\n" - "st1w { z24.s }, p1, [x22, x13, LSL #2]\n" - "fmla z27.s, p3/M, z8.s, z13.s\n" - "fmla z26.s, p3/M, z3.s, z12.s\n" - "ld1w { z13.s }, p2/Z, [x12, x14, LSL #2]\n" - "ldp x12, x11, [x16, #0x0]\n" - "fmla z25.s, p3/M, z8.s, z11.s\n" - "fmla z28.s, p3/M, z5.s, z11.s\n" - "ldp x10, x9, [x16, #0x10]\n" - "fmax z25.s, p3/M, z25.s, z17.s\n" - "fmla z29.s, p3/M, z8.s, z13.s\n" - "fmla z30.s, p3/M, z7.s, z13.s\n" - "fmax z26.s, p3/M, z26.s, z17.s\n" - "fmax z27.s, p3/M, z27.s, z17.s\n" - "fmla z31.s, p3/M, z6.s, z13.s\n" - "incw x14\n" - "ld1w { z9.s }, p0/Z, [x12, x15, LSL #2]\n" - "ld1w { z10.s }, p0/Z, [x11, x15, LSL #2]\n" - "ld1w { z11.s }, p0/Z, [x10, x15, LSL #2]\n" - "ld1w { z12.s }, p0/Z, [x9, x15, LSL #2]\n" - "fmin z25.s, p3/M, z25.s, z16.s\n" - "fmin z26.s, p3/M, z26.s, z16.s\n" - "ld1w { z13.s }, p0/Z, [x28, x15, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x22, x15, LSL #2]\n" + "movprfx z21, z14\n fmla z21.s, p3/M, z1.s, z9.s\n" + "fmla z29.s, p3/M, z7.s, z18.s\n" + "ldr x22, [x17, #0x68]\n" + "ldr x21, [x17, #0x78]\n" + "fmla z28.s, p3/M, z0.s, z17.s\n" + "fmla z22.s, p3/M, z8.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x20, x15, LSL #2]\n" + "ldr x20, [x17, #0x80]\n" + "fmla z26.s, p3/M, z4.s, z18.s\n" + "fmla z25.s, p3/M, z3.s, z18.s\n" + "ldr x9, [x13, #0x8]\n" + "ldr x28, [x13, #0x10]\n" + "fmla z21.s, p3/M, z0.s, z18.s\n" + "fmla z24.s, p3/M, z4.s, z19.s\n" + "ldr x27, [x13, #0x18]\n" + "ld1w { z14.s }, p3/Z, [x8]\n" + "fmla z23.s, p3/M, z1.s, z18.s\n" + "fmla z29.s, p3/M, z1.s, z17.s\n" + "ld1w { z20.s }, p2/Z, [x25, x15, LSL #2]\n" + "ld1w { z17.s }, p2/Z, [x24, x15, LSL #2]\n" + "fmla z28.s, p3/M, z2.s, z16.s\n" + "fmla z27.s, p3/M, z1.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x23, x15, LSL #2]\n" + "ldr x26, [x17, #0x90]\n" + "fmla z25.s, p3/M, z5.s, z19.s\n" + "fmla z21.s, p3/M, z2.s, z19.s\n" + "ldr x25, [x17, #0xa0]\n" + "ldr x24, [x17, #0x98]\n" + "fmla z26.s, p3/M, z0.s, z20.s\n" + "fmla z24.s, p3/M, z2.s, z17.s\n" + "fmla z28.s, p3/M, z8.s, z19.s\n" + "fmla z27.s, p3/M, z7.s, z19.s\n" + "fmla z22.s, p3/M, z1.s, z19.s\n" + "fmla z23.s, p3/M, z3.s, z16.s\n" + "ld1w { z18.s }, p2/Z, [x22, x15, LSL #2]\n" + "ldr x23, [x17, #0xa8]\n" + "fmla z26.s, p3/M, z6.s, z16.s\n" + "fmla z25.s, p3/M, z7.s, z18.s\n" + "ld1w { z19.s }, p2/Z, [x20, x15, LSL #2]\n" + "ldr x22, [x17, #0xc0]\n" + "fmla z24.s, p3/M, z6.s, z18.s\n" + "fmla z21.s, p3/M, z4.s, z18.s\n" + "fmla z29.s, p3/M, z3.s, z20.s\n" + "fmla z27.s, p3/M, z5.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x12, x15, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x21, x15, LSL #2]\n" + "fmla z23.s, p3/M, z5.s, z18.s\n" + "fmla z22.s, p3/M, z3.s, z18.s\n" + "ldr x21, [x17, #0xb0]\n" + "ldr x20, [x17, #0xb8]\n" + "fmla z26.s, p3/M, z8.s, z18.s\n" + "fmla z24.s, p3/M, z8.s, z17.s\n" + "fmla z21.s, p3/M, z6.s, z16.s\n" + "fmla z28.s, p3/M, z3.s, z19.s\n" + "fmla z25.s, p3/M, z0.s, z19.s\n" + "fmla z22.s, p3/M, z5.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x11, x15, LSL #2]\n" + "fmla z23.s, p3/M, z7.s, z16.s\n" + "ld1w { z18.s }, p2/Z, [x26, x15, LSL #2]\n" + "fmla z29.s, p3/M, z4.s, z19.s\n" + "fmla z26.s, p3/M, z1.s, z19.s\n" + "fmla z28.s, p3/M, z5.s, z17.s\n" + "ld1w { z16.s }, p2/Z, [x24, x15, LSL #2]\n" + "fmla z27.s, p3/M, z4.s, z17.s\n" + "fmla z25.s, p3/M, z2.s, z17.s\n" + "fmla z24.s, p3/M, z1.s, z17.s\n" + "fmla z21.s, p3/M, z8.s, z18.s\n" + "ld1w { z17.s }, p2/Z, [x25, x15, LSL #2]\n" + "ldr x25, [x17, #0x20]\n" + "fmla z22.s, p3/M, z7.s, z18.s\n" + "ld1w { z18.s }, p2/Z, [x23, x15, LSL #2]\n" + "fmla z29.s, p3/M, z2.s, z17.s\n" + "fmla z26.s, p3/M, z7.s, z16.s\n" + "fmla z25.s, p3/M, z6.s, z16.s\n" + "fmla z23.s, p3/M, z4.s, z16.s\n" + "fmla z21.s, p3/M, z3.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x21, x15, LSL #2]\n" + "fmla z22.s, p3/M, z4.s, z18.s\n" + "fmla z28.s, p3/M, z1.s, z17.s\n" + "fmax z28.s, p3/M, z28.s, z31.s\n" + "fmin z28.s, p3/M, z28.s, z30.s\n" + "fmla z27.s, p3/M, z0.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x20, x15, LSL #2]\n" + "fmla z29.s, p3/M, z6.s, z16.s\n" + "fmax z29.s, p3/M, z29.s, z31.s\n" + "fmla z24.s, p3/M, z7.s, z18.s\n" + "fmla z21.s, p3/M, z5.s, z18.s\n" + "fmin z29.s, p3/M, z29.s, z30.s\n" + "st1w { z29.s }, p1, [x10, x14, LSL #2]\n" + "fmla z23.s, p3/M, z0.s, z16.s\n" + "fmla z22.s, p3/M, z2.s, z17.s\n" + "ldr x24, [x13, #0x20]\n" + "st1w { z28.s }, p1, [x9, x14, LSL #2]\n" + "fmla z25.s, p3/M, z8.s, z18.s\n" + "fmla z26.s, p3/M, z3.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x22, x15, LSL #2]\n" + "ldp x23, x22, [x17, #0x0]\n" + "fmla z27.s, p3/M, z8.s, z17.s\n" + "fmla z24.s, p3/M, z5.s, z17.s\n" + "ldp x21, x20, [x17, #0x10]\n" + "fmax z27.s, p3/M, z27.s, z31.s\n" + "fmla z23.s, p3/M, z8.s, z16.s\n" + "fmla z21.s, p3/M, z7.s, z16.s\n" + "fmax z26.s, p3/M, z26.s, z31.s\n" + "fmax z25.s, p3/M, z25.s, z31.s\n" + "fmla z22.s, p3/M, z6.s, z16.s\n" "incw x15\n" - "fmin z27.s, p3/M, z27.s, z16.s\n" - "st1w { z25.s }, p1, [x21, x13, LSL #2]\n" - "fmax z28.s, p3/M, z28.s, z17.s\n" - "fmax z29.s, p3/M, z29.s, z17.s\n" - "st1w { z26.s }, p1, [x20, x13, LSL #2]\n" - "ldr x22, [x27, #0x28]\n" - "fmax z30.s, p3/M, z30.s, z17.s\n" - "fmax z31.s, p3/M, z31.s, z17.s\n" - "st1w { z27.s }, p1, [x23, x13, LSL #2]\n" - "ldr x21, [x27, #0x30]\n" - "ldr x20, [x27, #0x38]\n" - "ldr x23, [x27, #0x40]\n" - "whilelt p2.s, x14, %x[n_channels]\n" - "cmp x15, %x[n_channels]\n" - "ld1w { z0.s }, p3/Z, [x17, #1, MUL VL]\n" - "ld1w { z1.s }, p3/Z, [x17, #2, MUL VL]\n" - "fmin z28.s, p3/M, z28.s, z16.s\n" - "fmin z29.s, p3/M, z29.s, z16.s\n" - "ld1w { z2.s }, p3/Z, [x17, #3, MUL VL]\n" - "ld1w { z3.s }, p3/Z, [x17, #4, MUL VL]\n" - "fmin z30.s, p3/M, z30.s, z16.s\n" - "fmin z31.s, p3/M, z31.s, z16.s\n" - "ld1w { z4.s }, p3/Z, [x17, #5, MUL VL]\n" - "ld1w { z5.s }, p3/Z, [x17, #6, MUL VL]\n" - "st1w { z28.s }, p1, [x22, x13, LSL #2]\n" - "ld1w { z6.s }, p3/Z, [x17, #7, MUL VL]\n" - "addvl x17, x17, #16\n" - "st1w { z29.s }, p1, [x21, x13, LSL #2]\n" - "ld1w { z7.s }, p3/Z, [x17, #-8, MUL VL]\n" - "st1w { z30.s }, p1, [x20, x13, LSL #2]\n" - "ld1w { z8.s }, p3/Z, [x17, #-7, MUL VL]\n" - "addvl x17, x17, #-6\n" - "st1w { z31.s }, p1, [x23, x13, LSL #2]\n" + "ld1w { z9.s }, p0/Z, [x23, x16, LSL #2]\n" + "ld1w { z10.s }, p0/Z, [x22, x16, LSL #2]\n" + "ld1w { z11.s }, p0/Z, [x21, x16, LSL #2]\n" + "ld1w { z12.s }, p0/Z, [x20, x16, LSL #2]\n" + "fmin z27.s, p3/M, z27.s, z30.s\n" + "fmin z26.s, p3/M, z26.s, z30.s\n" + "ld1w { z13.s }, p0/Z, [x25, x16, LSL #2]\n" + "incw x16\n" + "fmin z25.s, p3/M, z25.s, z30.s\n" + "st1w { z27.s }, p1, [x28, x14, LSL #2]\n" + "fmax z24.s, p3/M, z24.s, z31.s\n" + "fmax z23.s, p3/M, z23.s, z31.s\n" + "st1w { z26.s }, p1, [x27, x14, LSL #2]\n" + "ldr x23, [x13, #0x28]\n" + "fmax z21.s, p3/M, z21.s, z31.s\n" + "fmax z22.s, p3/M, z22.s, z31.s\n" + "st1w { z25.s }, p1, [x24, x14, LSL #2]\n" + "ldr x22, [x13, #0x30]\n" + "ldr x21, [x13, #0x38]\n" + "ldr x20, [x13, #0x40]\n" + "whilelt p2.s, x15, %x[n_channels]\n" + "cmp x16, %x[n_channels]\n" + "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n" + "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n" + "fmin z24.s, p3/M, z24.s, z30.s\n" + "fmin z23.s, p3/M, z23.s, z30.s\n" + "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n" + "fmin z21.s, p3/M, z21.s, z30.s\n" + "fmin z22.s, p3/M, z22.s, z30.s\n" + "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n" + "ld1w { z5.s }, p3/Z, [x8, #6, MUL VL]\n" + "st1w { z24.s }, p1, [x23, x14, LSL #2]\n" + "ld1w { z6.s }, p3/Z, [x8, #7, MUL VL]\n" + "addvl x8, x8, #16\n" + "st1w { z23.s }, p1, [x22, x14, LSL #2]\n" + "ld1w { z7.s }, p3/Z, [x8, #-8, MUL VL]\n" + "st1w { z21.s }, p1, [x21, x14, LSL #2]\n" + "ld1w { z8.s }, p3/Z, [x8, #-7, MUL VL]\n" + "addvl x8, x8, #-6\n" + "st1w { z22.s }, p1, [x20, x14, LSL #2]\n" "blt 1b\n" "2:" // Channel tail - "movprfx z23, z18\n fmla z23.s, p3/M, z8.s, z9.s\n" - "movprfx z24, z18\n fmla z24.s, p3/M, z7.s, z9.s\n" - "ldr x26, [x16, #0x30]\n" - "ldr x25, [x16, #0x38]\n" - "movprfx z25, z18\n fmla z25.s, p3/M, z6.s, z9.s\n" - "fmla z23.s, p3/M, z0.s, z10.s\n" - "ldr x24, [x16, #0x28]\n" - "ldr x11, [x16, #0x48]\n" - "fmla z24.s, p3/M, z4.s, z13.s\n" - "movprfx z26, z18\n fmla z26.s, p3/M, z5.s, z9.s\n" - "ldr x12, [x16, #0x40]\n" - "ld1w { z10.s }, p2/Z, [x11, x14, LSL #2]\n" - "movprfx z27, z18\n fmla z27.s, p3/M, z4.s, z9.s\n" - "movprfx z28, z18\n fmla z28.s, p3/M, z3.s, z9.s\n" - "ldr x10, [x16, #0x50]\n" - "ldr x9, [x16, #0x58]\n" - "fmla z25.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x14, LSL #2]\n" - "movprfx z29, z18\n fmla z29.s, p3/M, z2.s, z9.s\n" - "ldr x28, [x16, #0x60]\n" - "fmla z23.s, p3/M, z5.s, z13.s\n" - "fmla z24.s, p3/M, z6.s, z11.s\n" - "ldr x26, [x16, #0x70]\n" - "ldr x11, [x16, #0x88]\n" - "movprfx z31, z18\n fmla z31.s, p3/M, z0.s, z9.s\n" - "fmla z25.s, p3/M, z3.s, z13.s\n" - "incw x13\n" - "mov p1.b, p2.b\n" + "movprfx z29, z14\n fmla z29.s, p3/M, z8.s, z9.s\n" + "movprfx z28, z14\n fmla z28.s, p3/M, z7.s, z9.s\n" + "ldr x23, [x17, #0x30]\n" + "ldr x26, [x17, #0x38]\n" + "movprfx z27, z14\n fmla z27.s, p3/M, z6.s, z9.s\n" + "fmla z29.s, p3/M, z0.s, z10.s\n" + "ldr x22, [x17, #0x28]\n" + "ldr x21, [x17, #0x48]\n" + "fmla z28.s, p3/M, z4.s, z13.s\n" + "movprfx z26, z14\n fmla z26.s, p3/M, z5.s, z9.s\n" + "ldr x20, [x17, #0x40]\n" + "ld1w { z19.s }, p2/Z, [x21, x15, LSL #2]\n" + "movprfx z25, z14\n fmla z25.s, p3/M, z4.s, z9.s\n" + "movprfx z24, z14\n fmla z24.s, p3/M, z3.s, z9.s\n" + "ldr x25, [x17, #0x50]\n" + "ldr x24, [x17, #0x58]\n" + "fmla z27.s, p3/M, z2.s, z11.s\n" + "ld1w { z18.s }, p2/Z, [x23, x15, LSL #2]\n" + "movprfx z23, z14\n fmla z23.s, p3/M, z2.s, z9.s\n" + "ldr x23, [x17, #0x60]\n" + "fmla z29.s, p3/M, z5.s, z13.s\n" + "fmla z28.s, p3/M, z6.s, z18.s\n" + "ldr x12, [x17, #0x70]\n" + "ldr x11, [x17, #0x88]\n" + "movprfx z22, z14\n fmla z22.s, p3/M, z0.s, z9.s\n" + "fmla z27.s, p3/M, z3.s, z13.s\n" + "incw x14\n" + "mov p0.b, p2.b\n" "fmla z26.s, p3/M, z2.s, z13.s\n" - "fmla z27.s, p3/M, z1.s, z13.s\n" - "ldr x23, [x27, #0x0]\n" - "ldr x22, [x27, #0x8]\n" - "fmla z28.s, p3/M, z0.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x25, x14, LSL #2]\n" - "fmla z29.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x24, x14, LSL #2]\n" - "movprfx z30, z18\n fmla z30.s, p3/M, z1.s, z9.s\n" - "fmla z23.s, p3/M, z7.s, z11.s\n" - "ldr x24, [x16, #0x68]\n" - "ldr x25, [x16, #0x78]\n" + "fmla z25.s, p3/M, z1.s, z13.s\n" + "ldr x10, [x13, #0x0]\n" + "ldr x9, [x13, #0x8]\n" "fmla z24.s, p3/M, z0.s, z13.s\n" - "fmla z31.s, p3/M, z8.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x12, x14, LSL #2]\n" - "ldr x12, [x16, #0x80]\n" - "fmla z26.s, p3/M, z4.s, z11.s\n" - "fmla z27.s, p3/M, z3.s, z11.s\n" - "ldr x21, [x27, #0x10]\n" - "ldr x20, [x27, #0x18]\n" - "fmla z30.s, p3/M, z0.s, z11.s\n" - "fmla z28.s, p3/M, z4.s, z10.s\n" - "fmla z29.s, p3/M, z1.s, z11.s\n" - "fmla z23.s, p3/M, z1.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x10, x14, LSL #2]\n" - "ld1w { z13.s }, p2/Z, [x9, x14, LSL #2]\n" - "fmla z24.s, p3/M, z2.s, z12.s\n" - "fmla z25.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x28, x14, LSL #2]\n" - "ldr x10, [x16, #0x90]\n" - "fmla z27.s, p3/M, z5.s, z10.s\n" - "fmla z30.s, p3/M, z2.s, z10.s\n" - "ldr x28, [x16, #0xa0]\n" - "ldr x9, [x16, #0x98]\n" - "fmla z26.s, p3/M, z0.s, z11.s\n" - "fmla z28.s, p3/M, z2.s, z13.s\n" - "fmla z24.s, p3/M, z8.s, z10.s\n" - "fmla z25.s, p3/M, z7.s, z10.s\n" - "fmla z31.s, p3/M, z1.s, z10.s\n" - "fmla z29.s, p3/M, z3.s, z12.s\n" - "ld1w { z10.s }, p2/Z, [x24, x14, LSL #2]\n" - "ldr x24, [x16, #0xa8]\n" - "fmla z26.s, p3/M, z6.s, z12.s\n" - "fmla z27.s, p3/M, z7.s, z10.s\n" - "ld1w { z12.s }, p2/Z, [x12, x14, LSL #2]\n" - "ldr x12, [x16, #0xc0]\n" - "fmla z28.s, p3/M, z6.s, z10.s\n" - "fmla z30.s, p3/M, z4.s, z10.s\n" - "fmla z23.s, p3/M, z3.s, z11.s\n" - "fmla z25.s, p3/M, z5.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x26, x14, LSL #2]\n" - "ld1w { z13.s }, p2/Z, [x25, x14, LSL #2]\n" - "fmla z29.s, p3/M, z5.s, z10.s\n" - "fmla z31.s, p3/M, z3.s, z10.s\n" - "ldr x26, [x16, #0xb0]\n" - "ldr x25, [x16, #0xb8]\n" - "fmla z26.s, p3/M, z8.s, z10.s\n" - "fmla z28.s, p3/M, z8.s, z11.s\n" - "fmla z30.s, p3/M, z6.s, z13.s\n" - "fmla z24.s, p3/M, z3.s, z12.s\n" - "fmla z27.s, p3/M, z0.s, z12.s\n" - "fmla z31.s, p3/M, z5.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x11, x14, LSL #2]\n" - "fmla z29.s, p3/M, z7.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x10, x14, LSL #2]\n" - "fmla z23.s, p3/M, z4.s, z12.s\n" - "fmla z26.s, p3/M, z1.s, z12.s\n" - "fmla z24.s, p3/M, z5.s, z11.s\n" - "ld1w { z12.s }, p2/Z, [x9, x14, LSL #2]\n" - "fmla z25.s, p3/M, z4.s, z11.s\n" - "fmla z27.s, p3/M, z2.s, z11.s\n" - "fmla z28.s, p3/M, z1.s, z11.s\n" - "fmla z30.s, p3/M, z8.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x28, x14, LSL #2]\n" - "fmla z31.s, p3/M, z7.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x24, x14, LSL #2]\n" - "fmla z23.s, p3/M, z2.s, z11.s\n" - "fmla z26.s, p3/M, z7.s, z12.s\n" - "fmla z27.s, p3/M, z6.s, z12.s\n" - "fmla z29.s, p3/M, z4.s, z12.s\n" - "fmla z30.s, p3/M, z3.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x26, x14, LSL #2]\n" - "fmla z31.s, p3/M, z4.s, z13.s\n" - "fmla z24.s, p3/M, z1.s, z11.s\n" - "fmax z24.s, p3/M, z24.s, z17.s\n" - "fmin z24.s, p3/M, z24.s, z16.s\n" - "fmla z25.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x25, x14, LSL #2]\n" + "ld1w { z17.s }, p2/Z, [x26, x15, LSL #2]\n" "fmla z23.s, p3/M, z6.s, z12.s\n" - "fmax z23.s, p3/M, z23.s, z17.s\n" - "fmla z28.s, p3/M, z7.s, z13.s\n" - "fmla z30.s, p3/M, z5.s, z13.s\n" - "fmin z23.s, p3/M, z23.s, z16.s\n" - "st1w { z23.s }, p1, [x23, x13, LSL #2]\n" - "fmla z29.s, p3/M, z0.s, z12.s\n" - "fmla z31.s, p3/M, z2.s, z11.s\n" - "ldr x23, [x27, #0x20]\n" - "st1w { z24.s }, p1, [x22, x13, LSL #2]\n" - "fmla z27.s, p3/M, z8.s, z13.s\n" - "fmla z26.s, p3/M, z3.s, z12.s\n" - "ld1w { z13.s }, p2/Z, [x12, x14, LSL #2]\n" - "fmax z26.s, p3/M, z26.s, z17.s\n" - "fmla z25.s, p3/M, z8.s, z11.s\n" - "fmla z28.s, p3/M, z5.s, z11.s\n" - "fmax z25.s, p3/M, z25.s, z17.s\n" - "fmax z27.s, p3/M, z27.s, z17.s\n" - "fmla z29.s, p3/M, z8.s, z13.s\n" - "fmla z30.s, p3/M, z7.s, z13.s\n" - "fmin z25.s, p3/M, z25.s, z16.s\n" - "fmin z26.s, p3/M, z26.s, z16.s\n" - "fmla z31.s, p3/M, z6.s, z13.s\n" - "fmin z27.s, p3/M, z27.s, z16.s\n" - "fmax z28.s, p3/M, z28.s, z17.s\n" - "st1w { z25.s }, p1, [x21, x13, LSL #2]\n" - "fmax z29.s, p3/M, z29.s, z17.s\n" - "fmax z30.s, p3/M, z30.s, z17.s\n" - "st1w { z26.s }, p1, [x20, x13, LSL #2]\n" - "ldr x22, [x27, #0x28]\n" - "fmax z31.s, p3/M, z31.s, z17.s\n" - "st1w { z27.s }, p1, [x23, x13, LSL #2]\n" - "ldr x21, [x27, #0x30]\n" - "ldr x20, [x27, #0x38]\n" - "ldr x23, [x27, #0x40]\n" - "fmin z28.s, p3/M, z28.s, z16.s\n" - "fmin z29.s, p3/M, z29.s, z16.s\n" - "st1w { z28.s }, p1, [x22, x13, LSL #2]\n" - "fmin z30.s, p3/M, z30.s, z16.s\n" - "fmin z31.s, p3/M, z31.s, z16.s\n" - "st1w { z29.s }, p1, [x21, x13, LSL #2]\n" - "st1w { z30.s }, p1, [x20, x13, LSL #2]\n" - "st1w { z31.s }, p1, [x23, x13, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x22, x15, LSL #2]\n" + "movprfx z21, z14\n fmla z21.s, p3/M, z1.s, z9.s\n" + "fmla z29.s, p3/M, z7.s, z18.s\n" + "ldr x22, [x17, #0x68]\n" + "ldr x21, [x17, #0x78]\n" + "fmla z28.s, p3/M, z0.s, z17.s\n" + "fmla z22.s, p3/M, z8.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x20, x15, LSL #2]\n" + "ldr x20, [x17, #0x80]\n" + "fmla z26.s, p3/M, z4.s, z18.s\n" + "fmla z25.s, p3/M, z3.s, z18.s\n" + "ldr x28, [x13, #0x10]\n" + "ldr x27, [x13, #0x18]\n" + "fmla z21.s, p3/M, z0.s, z18.s\n" + "fmla z24.s, p3/M, z4.s, z19.s\n" + "fmla z23.s, p3/M, z1.s, z18.s\n" + "fmla z29.s, p3/M, z1.s, z17.s\n" + "ld1w { z20.s }, p2/Z, [x25, x15, LSL #2]\n" + "ld1w { z17.s }, p2/Z, [x24, x15, LSL #2]\n" + "fmla z28.s, p3/M, z2.s, z16.s\n" + "fmla z27.s, p3/M, z1.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x23, x15, LSL #2]\n" + "ldr x26, [x17, #0x90]\n" + "fmla z25.s, p3/M, z5.s, z19.s\n" + "fmla z21.s, p3/M, z2.s, z19.s\n" + "ldr x25, [x17, #0xa0]\n" + "ldr x24, [x17, #0x98]\n" + "fmla z26.s, p3/M, z0.s, z20.s\n" + "fmla z24.s, p3/M, z2.s, z17.s\n" + "fmla z28.s, p3/M, z8.s, z19.s\n" + "fmla z27.s, p3/M, z7.s, z19.s\n" + "fmla z22.s, p3/M, z1.s, z19.s\n" + "fmla z23.s, p3/M, z3.s, z16.s\n" + "ld1w { z18.s }, p2/Z, [x22, x15, LSL #2]\n" + "ldr x23, [x17, #0xa8]\n" + "fmla z26.s, p3/M, z6.s, z16.s\n" + "fmla z25.s, p3/M, z7.s, z18.s\n" + "ld1w { z19.s }, p2/Z, [x20, x15, LSL #2]\n" + "ldr x22, [x17, #0xc0]\n" + "fmla z24.s, p3/M, z6.s, z18.s\n" + "fmla z21.s, p3/M, z4.s, z18.s\n" + "fmla z29.s, p3/M, z3.s, z20.s\n" + "fmla z27.s, p3/M, z5.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x12, x15, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x21, x15, LSL #2]\n" + "fmla z23.s, p3/M, z5.s, z18.s\n" + "fmla z22.s, p3/M, z3.s, z18.s\n" + "ldr x21, [x17, #0xb0]\n" + "ldr x20, [x17, #0xb8]\n" + "fmla z26.s, p3/M, z8.s, z18.s\n" + "fmla z24.s, p3/M, z8.s, z17.s\n" + "fmla z21.s, p3/M, z6.s, z16.s\n" + "fmla z28.s, p3/M, z3.s, z19.s\n" + "fmla z25.s, p3/M, z0.s, z19.s\n" + "fmla z22.s, p3/M, z5.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x11, x15, LSL #2]\n" + "fmla z23.s, p3/M, z7.s, z16.s\n" + "ld1w { z18.s }, p2/Z, [x26, x15, LSL #2]\n" + "fmla z29.s, p3/M, z4.s, z19.s\n" + "fmla z26.s, p3/M, z1.s, z19.s\n" + "fmla z28.s, p3/M, z5.s, z17.s\n" + "ld1w { z16.s }, p2/Z, [x24, x15, LSL #2]\n" + "fmla z27.s, p3/M, z4.s, z17.s\n" + "fmla z25.s, p3/M, z2.s, z17.s\n" + "fmla z24.s, p3/M, z1.s, z17.s\n" + "fmla z21.s, p3/M, z8.s, z18.s\n" + "ld1w { z17.s }, p2/Z, [x25, x15, LSL #2]\n" + "fmla z22.s, p3/M, z7.s, z18.s\n" + "ld1w { z18.s }, p2/Z, [x23, x15, LSL #2]\n" + "fmla z29.s, p3/M, z2.s, z17.s\n" + "fmla z26.s, p3/M, z7.s, z16.s\n" + "fmla z25.s, p3/M, z6.s, z16.s\n" + "fmla z23.s, p3/M, z4.s, z16.s\n" + "fmla z21.s, p3/M, z3.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x21, x15, LSL #2]\n" + "fmla z22.s, p3/M, z4.s, z18.s\n" + "fmla z28.s, p3/M, z1.s, z17.s\n" + "fmax z28.s, p3/M, z28.s, z31.s\n" + "fmin z28.s, p3/M, z28.s, z30.s\n" + "fmla z27.s, p3/M, z0.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x20, x15, LSL #2]\n" + "fmla z29.s, p3/M, z6.s, z16.s\n" + "fmax z29.s, p3/M, z29.s, z31.s\n" + "fmla z24.s, p3/M, z7.s, z18.s\n" + "fmla z21.s, p3/M, z5.s, z18.s\n" + "fmin z29.s, p3/M, z29.s, z30.s\n" + "st1w { z29.s }, p0, [x10, x14, LSL #2]\n" + "fmla z23.s, p3/M, z0.s, z16.s\n" + "fmla z22.s, p3/M, z2.s, z17.s\n" + "ldr x20, [x13, #0x20]\n" + "st1w { z28.s }, p0, [x9, x14, LSL #2]\n" + "fmla z25.s, p3/M, z8.s, z18.s\n" + "fmla z26.s, p3/M, z3.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x22, x15, LSL #2]\n" + "fmax z26.s, p3/M, z26.s, z31.s\n" + "fmla z27.s, p3/M, z8.s, z17.s\n" + "fmla z24.s, p3/M, z5.s, z17.s\n" + "fmax z27.s, p3/M, z27.s, z31.s\n" + "fmax z25.s, p3/M, z25.s, z31.s\n" + "fmla z23.s, p3/M, z8.s, z16.s\n" + "fmla z21.s, p3/M, z7.s, z16.s\n" + "fmin z27.s, p3/M, z27.s, z30.s\n" + "fmin z26.s, p3/M, z26.s, z30.s\n" + "fmla z22.s, p3/M, z6.s, z16.s\n" + "fmin z25.s, p3/M, z25.s, z30.s\n" + "fmax z24.s, p3/M, z24.s, z31.s\n" + "st1w { z27.s }, p0, [x28, x14, LSL #2]\n" + "fmax z23.s, p3/M, z23.s, z31.s\n" + "fmax z21.s, p3/M, z21.s, z31.s\n" + "st1w { z26.s }, p0, [x27, x14, LSL #2]\n" + "ldr x23, [x13, #0x28]\n" + "fmax z22.s, p3/M, z22.s, z31.s\n" + "st1w { z25.s }, p0, [x20, x14, LSL #2]\n" + "ldr x22, [x13, #0x30]\n" + "ldr x21, [x13, #0x38]\n" + "ldr x20, [x13, #0x40]\n" + "fmin z24.s, p3/M, z24.s, z30.s\n" + "fmin z23.s, p3/M, z23.s, z30.s\n" + "st1w { z24.s }, p0, [x23, x14, LSL #2]\n" + "fmin z21.s, p3/M, z21.s, z30.s\n" + "fmin z22.s, p3/M, z22.s, z30.s\n" + "st1w { z23.s }, p0, [x22, x14, LSL #2]\n" + "st1w { z21.s }, p0, [x21, x14, LSL #2]\n" + "st1w { z22.s }, p0, [x20, x14, LSL #2]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp index 6073b2ba7dd8e344dd1680ffaf75750039135721..38b377509e41869dcec46a59849f86de41144f43 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,19 +22,19 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include #pragma once -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { -void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); -void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); +void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(const float *const *const input_ptrs, float *const *const outptrs, const void *params, unsigned int n_channels, const float activation_min, const float activation_max); +void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(const unsigned int n_tile_rows, const unsigned int n_tile_cols, const float *inptr, int64_t ld_input_row, int64_t ld_input_col, float *outptr, int64_t ld_output_row, int64_t ld_output_col, const void *params, unsigned int n_channels, const float activation_min, const float activation_max); class sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst : public DepthwiseDepthfirstStrategy { @@ -57,7 +57,7 @@ class sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst : public DepthwiseDepthfirst constexpr static unsigned int output_cols = 4; sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst(const CPUInfo *) - : DepthwiseDepthfirstStrategy(4, 3, 1) {} + : Parent(output_rows, output_cols, kernel_rows, kernel_cols, stride_rows, stride_cols) {} arm_gemm::VLType get_vl_type(void) const override { return vl_type; } @@ -68,4 +68,4 @@ class sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst : public DepthwiseDepthfirst } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp index cdf77a1cf090d285bdb79289d13538d7569c1706..35445595f8d26e9edb7474de3dbf7cdedeb23cc6 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -113,7 +113,7 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "mul x21, x21, x25\n" // offset *= kernel_stride * output_size "add x8, x8, x21, LSL #2\n" // inptr[0] += offset * sizeof(float) "add x13, x8, x23, LSL #2\n" - "ld1w { z15.s }, p3/Z, [x17]\n" + "ld1w { z19.s }, p3/Z, [x17]\n" "mul x20, x20, x24\n" // offset *= output_tile_size "add x12, x13, x23, LSL #2\n" "add x15, x15, x20, LSL #2\n" // outptrs[0] += offset * sizeof(float) @@ -132,8 +132,8 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "add x27, x10, x5\n" "add x26, x9, x22, LSL #2\n" "add x25, x6, x6\n" - "ld1rw { z14.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "ld1rw { z13.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "ld1rw { z15.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rw { z16.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "cmp x16, %x[n_channels]\n" "add x24, x28, x23, LSL #2\n" "ld1w { z7.s }, p3/Z, [x17, #-8, MUL VL]\n" @@ -149,500 +149,500 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "addvl x17, x17, #-6\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "movprfx z21, z15\n fmla z21.s, p3/M, z4.s, z9.s\n" - "movprfx z16, z15\n fmla z16.s, p3/M, z8.s, z9.s\n" + "movprfx z14, z19\n fmla z14.s, p3/M, z4.s, z9.s\n" + "movprfx z31, z19\n fmla z31.s, p3/M, z8.s, z9.s\n" "whilelt p1.s, x16, %x[n_channels]\n" "incw x21\n" - "movprfx z22, z15\n fmla z22.s, p3/M, z3.s, z9.s\n" - "movprfx z25, z15\n fmla z25.s, p3/M, z1.s, z9.s\n" + "movprfx z21, z19\n fmla z21.s, p3/M, z3.s, z9.s\n" + "movprfx z22, z19\n fmla z22.s, p3/M, z1.s, z9.s\n" "incw x16\n" "mov p0.b, p2.b\n" - "movprfx z26, z15\n fmla z26.s, p3/M, z0.s, z9.s\n" - "fmla z21.s, p3/M, z5.s, z12.s\n" + "movprfx z20, z19\n fmla z20.s, p3/M, z0.s, z9.s\n" + "fmla z14.s, p3/M, z5.s, z12.s\n" "incw x20\n" - "movprfx z17, z15\n fmla z17.s, p3/M, z7.s, z9.s\n" - "movprfx z18, z15\n fmla z18.s, p3/M, z6.s, z9.s\n" - "movprfx z20, z15\n fmla z20.s, p3/M, z5.s, z9.s\n" - "movprfx z24, z15\n fmla z24.s, p3/M, z2.s, z9.s\n" + "movprfx z13, z19\n fmla z13.s, p3/M, z7.s, z9.s\n" + "movprfx z17, z19\n fmla z17.s, p3/M, z6.s, z9.s\n" + "movprfx z27, z19\n fmla z27.s, p3/M, z5.s, z9.s\n" + "movprfx z18, z19\n fmla z18.s, p3/M, z2.s, z9.s\n" "ld1w { z9.s }, p2/Z, [x11, x7, LSL #2]\n" - "fmla z16.s, p3/M, z0.s, z10.s\n" - "movprfx z19, z15\n fmla z19.s, p3/M, z2.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x24]\n" + "fmla z31.s, p3/M, z0.s, z10.s\n" + "movprfx z30, z19\n fmla z30.s, p3/M, z2.s, z11.s\n" + "ld1w { z29.s }, p2/Z, [x24]\n" "ld1w { z11.s }, p2/Z, [x24, x27, LSL #2]\n" - "fmla z22.s, p3/M, z4.s, z12.s\n" - "fmla z25.s, p3/M, z2.s, z12.s\n" - "fmla z26.s, p3/M, z1.s, z12.s\n" - "movprfx z28, z15\n fmla z28.s, p3/M, z6.s, z10.s\n" + "fmla z21.s, p3/M, z4.s, z12.s\n" + "fmla z22.s, p3/M, z2.s, z12.s\n" + "fmla z20.s, p3/M, z1.s, z12.s\n" + "movprfx z23, z19\n fmla z23.s, p3/M, z6.s, z29.s\n" "ld1w { z10.s }, p2/Z, [x11, x14, LSL #2]\n" - "fmla z21.s, p3/M, z7.s, z9.s\n" - "fmla z17.s, p3/M, z8.s, z12.s\n" - "fmla z18.s, p3/M, z7.s, z12.s\n" - "fmla z19.s, p3/M, z6.s, z12.s\n" - "movprfx z23, z15\n fmla z23.s, p3/M, z3.s, z12.s\n" - "movprfx z27, z15\n fmla z27.s, p3/M, z0.s, z12.s\n" + "fmla z14.s, p3/M, z7.s, z9.s\n" + "fmla z13.s, p3/M, z8.s, z12.s\n" + "fmla z17.s, p3/M, z7.s, z12.s\n" + "fmla z30.s, p3/M, z6.s, z12.s\n" + "movprfx z26, z19\n fmla z26.s, p3/M, z3.s, z12.s\n" + "movprfx z28, z19\n fmla z28.s, p3/M, z0.s, z12.s\n" "ld1w { z12.s }, p2/Z, [x8, x5, LSL #2]\n" - "movprfx z31, z15\n fmla z31.s, p3/M, z8.s, z11.s\n" - "fmla z22.s, p3/M, z6.s, z9.s\n" + "movprfx z24, z19\n fmla z24.s, p3/M, z8.s, z11.s\n" + "fmla z21.s, p3/M, z6.s, z9.s\n" "ld1w { z11.s }, p2/Z, [x8, x10, LSL #2]\n" - "fmla z25.s, p3/M, z4.s, z9.s\n" - "fmla z26.s, p3/M, z3.s, z9.s\n" - "fmla z20.s, p3/M, z8.s, z9.s\n" - "fmla z24.s, p3/M, z5.s, z9.s\n" - "fmla z28.s, p3/M, z2.s, z9.s\n" - "fmla z21.s, p3/M, z8.s, z10.s\n" - "fmla z16.s, p3/M, z1.s, z12.s\n" - "fmla z17.s, p3/M, z0.s, z12.s\n" + "fmla z22.s, p3/M, z4.s, z9.s\n" + "fmla z20.s, p3/M, z3.s, z9.s\n" + "movprfx z25, z19\n fmla z25.s, p3/M, z1.s, z9.s\n" + "movprfx z29, z19\n fmla z29.s, p3/M, z0.s, z9.s\n" + "ld1w { z19.s }, p3/Z, [x17]\n" + "fmla z27.s, p3/M, z8.s, z9.s\n" + "fmla z18.s, p3/M, z5.s, z9.s\n" + "fmla z23.s, p3/M, z2.s, z9.s\n" + "fmla z14.s, p3/M, z8.s, z10.s\n" + "ld1w { z9.s }, p2/Z, [x13]\n" + "fmla z31.s, p3/M, z1.s, z12.s\n" + "fmla z13.s, p3/M, z0.s, z12.s\n" "ld1w { z12.s }, p2/Z, [x13, x27, LSL #2]\n" - "fmla z18.s, p3/M, z2.s, z11.s\n" - "fmla z19.s, p3/M, z1.s, z11.s\n" + "fmla z17.s, p3/M, z2.s, z11.s\n" + "fmla z30.s, p3/M, z1.s, z11.s\n" "ld1w { z11.s }, p2/Z, [x28]\n" - "fmla z22.s, p3/M, z7.s, z10.s\n" - "fmla z23.s, p3/M, z6.s, z10.s\n" - "fmla z25.s, p3/M, z5.s, z10.s\n" - "fmla z26.s, p3/M, z4.s, z10.s\n" - "fmla z27.s, p3/M, z3.s, z10.s\n" - "fmla z31.s, p3/M, z0.s, z10.s\n" - "fmla z24.s, p3/M, z6.s, z11.s\n" - "fmla z28.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x28, x27, LSL #2]\n" - "fmla z19.s, p3/M, z5.s, z12.s\n" - "fmla z23.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x13, x14, LSL #2]\n" - "fmla z27.s, p3/M, z8.s, z11.s\n" - "fmla z31.s, p3/M, z5.s, z11.s\n" - "movprfx z29, z15\n fmla z29.s, p3/M, z1.s, z9.s\n" - "movprfx z30, z15\n fmla z30.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x13]\n" - "fmla z29.s, p3/M, z2.s, z10.s\n" - "fmla z30.s, p3/M, z1.s, z10.s\n" + "fmla z21.s, p3/M, z7.s, z10.s\n" + "fmla z26.s, p3/M, z6.s, z10.s\n" + "fmla z22.s, p3/M, z5.s, z10.s\n" + "fmla z20.s, p3/M, z4.s, z10.s\n" + "fmla z28.s, p3/M, z3.s, z10.s\n" + "fmla z25.s, p3/M, z2.s, z10.s\n" + "fmla z29.s, p3/M, z1.s, z10.s\n" + "fmla z24.s, p3/M, z0.s, z10.s\n" "ld1w { z10.s }, p2/Z, [x13, x7, LSL #2]\n" - "fmla z20.s, p3/M, z0.s, z9.s\n" - "fmla z21.s, p3/M, z1.s, z10.s\n" - "fmla z16.s, p3/M, z3.s, z9.s\n" - "fmla z17.s, p3/M, z4.s, z10.s\n" - "ld1w { z11.s }, p2/Z, [x24, x5, LSL #2]\n" - "fmla z18.s, p3/M, z3.s, z10.s\n" - "fmla z22.s, p3/M, z0.s, z10.s\n" + "fmla z27.s, p3/M, z0.s, z9.s\n" + "fmla z18.s, p3/M, z6.s, z11.s\n" + "fmla z23.s, p3/M, z3.s, z11.s\n" + "fmla z14.s, p3/M, z1.s, z10.s\n" + "ld1w { z11.s }, p2/Z, [x28, x27, LSL #2]\n" + "fmla z31.s, p3/M, z3.s, z9.s\n" + "fmla z30.s, p3/M, z5.s, z12.s\n" + "fmla z26.s, p3/M, z2.s, z12.s\n" + "fmla z13.s, p3/M, z4.s, z10.s\n" + "ld1w { z9.s }, p2/Z, [x13, x14, LSL #2]\n" + "fmla z17.s, p3/M, z3.s, z10.s\n" + "fmla z21.s, p3/M, z0.s, z10.s\n" + "fmla z28.s, p3/M, z8.s, z11.s\n" + "fmla z24.s, p3/M, z5.s, z11.s\n" + "ld1w { z12.s }, p2/Z, [x24, x5, LSL #2]\n" + "fmla z27.s, p3/M, z2.s, z10.s\n" + "fmla z14.s, p3/M, z2.s, z9.s\n" + "fmla z31.s, p3/M, z5.s, z10.s\n" + "fmla z13.s, p3/M, z5.s, z9.s\n" + "ld1w { z11.s }, p2/Z, [x12, x5, LSL #2]\n" + "fmla z17.s, p3/M, z4.s, z9.s\n" + "fmla z30.s, p3/M, z3.s, z9.s\n" + "fmla z21.s, p3/M, z1.s, z9.s\n" + "fmla z26.s, p3/M, z0.s, z9.s\n" + "ld1w { z10.s }, p2/Z, [x12, x10, LSL #2]\n" + "fmla z23.s, p3/M, z7.s, z12.s\n" + "fmla z25.s, p3/M, z6.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x24, x10, LSL #2]\n" + "fmla z27.s, p3/M, z4.s, z11.s\n" + "fmla z14.s, p3/M, z3.s, z11.s\n" + "fmla z18.s, p3/M, z1.s, z11.s\n" + "fmla z22.s, p3/M, z0.s, z11.s\n" + "fmla z31.s, p3/M, z7.s, z11.s\n" + "fmla z13.s, p3/M, z6.s, z11.s\n" + "ld1w { z9.s }, p2/Z, [x8, x7, LSL #2]\n" + "fmla z29.s, p3/M, z8.s, z12.s\n" + "fmla z24.s, p3/M, z7.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x11, x5, LSL #2]\n" + "fmla z17.s, p3/M, z8.s, z10.s\n" + "fmla z30.s, p3/M, z7.s, z10.s\n" + "fmla z21.s, p3/M, z5.s, z10.s\n" + "fmla z26.s, p3/M, z4.s, z10.s\n" "fmla z20.s, p3/M, z2.s, z10.s\n" - "fmla z21.s, p3/M, z2.s, z12.s\n" - "fmla z16.s, p3/M, z5.s, z10.s\n" - "fmla z17.s, p3/M, z5.s, z12.s\n" - "ld1w { z10.s }, p2/Z, [x12, x5, LSL #2]\n" + "fmla z28.s, p3/M, z1.s, z10.s\n" + "ld1w { z11.s }, p2/Z, [x8, x14, LSL #2]\n" + "addvl x8, x8, #1\n" + "fmla z27.s, p3/M, z7.s, z12.s\n" + "fmla z14.s, p3/M, z6.s, z12.s\n" "fmla z18.s, p3/M, z4.s, z12.s\n" - "fmla z19.s, p3/M, z3.s, z12.s\n" - "fmla z22.s, p3/M, z1.s, z12.s\n" - "fmla z23.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x12, x10, LSL #2]\n" - "fmla z28.s, p3/M, z7.s, z11.s\n" - "fmla z29.s, p3/M, z6.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x24, x10, LSL #2]\n" - "fmla z20.s, p3/M, z4.s, z10.s\n" - "fmla z21.s, p3/M, z3.s, z10.s\n" - "fmla z24.s, p3/M, z1.s, z10.s\n" - "fmla z25.s, p3/M, z0.s, z10.s\n" - "fmla z16.s, p3/M, z7.s, z10.s\n" - "fmla z17.s, p3/M, z6.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x8, x7, LSL #2]\n" + "fmla z22.s, p3/M, z3.s, z12.s\n" + "fmla z23.s, p3/M, z1.s, z12.s\n" + "fmla z25.s, p3/M, z0.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x11, x10, LSL #2]\n" + "fmla z31.s, p3/M, z2.s, z9.s\n" + "fmla z13.s, p3/M, z1.s, z9.s\n" + "fmla z17.s, p3/M, z0.s, z9.s\n" + "ld1w { z9.s }, p2/Z, [x12]\n" + "fmla z29.s, p3/M, z2.s, z12.s\n" + "fmla z30.s, p3/M, z0.s, z11.s\n" + "fmla z27.s, p3/M, z3.s, z9.s\n" + "fmla z18.s, p3/M, z0.s, z9.s\n" + "fmla z21.s, p3/M, z8.s, z12.s\n" + "fmla z26.s, p3/M, z7.s, z12.s\n" + "fmla z20.s, p3/M, z5.s, z12.s\n" + "fmla z28.s, p3/M, z4.s, z12.s\n" + "fmla z24.s, p3/M, z1.s, z12.s\n" + "ld1w { z10.s }, p2/Z, [x28, x7, LSL #2]\n" + "fmla z13.s, p3/M, z2.s, z11.s\n" + "fmla z17.s, p3/M, z1.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x12, x27, LSL #2]\n" + "addvl x12, x12, #1\n" + "fmla z31.s, p3/M, z6.s, z9.s\n" + "ld1w { z12.s }, p2/Z, [x11]\n" + "fmla z25.s, p3/M, z4.s, z10.s\n" + "ld1w { z9.s }, p1/Z, [x12, x7, LSL #2]\n" + "fmla z29.s, p3/M, z3.s, z10.s\n" "fmla z30.s, p3/M, z8.s, z11.s\n" - "fmla z31.s, p3/M, z7.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x11, x5, LSL #2]\n" - "fmla z18.s, p3/M, z8.s, z12.s\n" - "fmla z19.s, p3/M, z7.s, z12.s\n" - "fmla z22.s, p3/M, z5.s, z12.s\n" - "fmla z23.s, p3/M, z4.s, z12.s\n" - "fmla z26.s, p3/M, z2.s, z12.s\n" - "fmla z27.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x8, x14, LSL #2]\n" - "addvl x8, x8, #1\n" - "fmla z20.s, p3/M, z7.s, z11.s\n" - "fmla z21.s, p3/M, z6.s, z11.s\n" - "fmla z24.s, p3/M, z4.s, z11.s\n" - "fmla z25.s, p3/M, z3.s, z11.s\n" - "fmla z28.s, p3/M, z1.s, z11.s\n" - "fmla z29.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x11, x10, LSL #2]\n" - "fmla z16.s, p3/M, z2.s, z10.s\n" - "fmla z17.s, p3/M, z1.s, z10.s\n" - "fmla z18.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x12]\n" - "fmla z30.s, p3/M, z2.s, z11.s\n" - "fmla z19.s, p3/M, z0.s, z12.s\n" - "fmla z20.s, p3/M, z3.s, z10.s\n" - "fmla z24.s, p3/M, z0.s, z10.s\n" - "fmla z22.s, p3/M, z8.s, z11.s\n" - "fmla z23.s, p3/M, z7.s, z11.s\n" "fmla z26.s, p3/M, z5.s, z11.s\n" - "fmla z27.s, p3/M, z4.s, z11.s\n" - "fmla z31.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x28, x7, LSL #2]\n" - "fmla z17.s, p3/M, z2.s, z12.s\n" - "fmla z18.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x12, x27, LSL #2]\n" - "addvl x12, x12, #1\n" - "fmla z16.s, p3/M, z6.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x11]\n" - "fmla z29.s, p3/M, z4.s, z11.s\n" - "fmla z30.s, p3/M, z3.s, z11.s\n" - "fmla z19.s, p3/M, z8.s, z12.s\n" - "fmla z23.s, p3/M, z5.s, z12.s\n" - "fmla z27.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11, x27, LSL #2]\n" + "fmla z28.s, p3/M, z2.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x11, x27, LSL #2]\n" "addvl x11, x11, #1\n" + "fmla z27.s, p3/M, z6.s, z12.s\n" + "fmla z18.s, p3/M, z3.s, z12.s\n" + "fmla z23.s, p3/M, z0.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x24, x7, LSL #2]\n" + "fmla z24.s, p3/M, z2.s, z11.s\n" + "fmla z25.s, p3/M, z7.s, z12.s\n" + "fmla z29.s, p3/M, z6.s, z12.s\n" + "fmla z18.s, p3/M, z8.s, z10.s\n" + "fmla z22.s, p3/M, z7.s, z10.s\n" "fmla z20.s, p3/M, z6.s, z10.s\n" - "fmla z24.s, p3/M, z3.s, z10.s\n" - "fmla z28.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x24, x7, LSL #2]\n" - "fmla z31.s, p3/M, z2.s, z12.s\n" - "fmla z29.s, p3/M, z7.s, z10.s\n" - "fmla z30.s, p3/M, z6.s, z10.s\n" - "fmla z24.s, p3/M, z8.s, z11.s\n" - "fmla z25.s, p3/M, z7.s, z11.s\n" - "fmla z26.s, p3/M, z6.s, z11.s\n" + "fmla z23.s, p3/M, z5.s, z10.s\n" + "ld1w { z10.s }, p2/Z, [x28, x14, LSL #2]\n" "fmla z28.s, p3/M, z5.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x28, x14, LSL #2]\n" - "fmla z27.s, p3/M, z5.s, z12.s\n" - "fmla z29.s, p3/M, z5.s, z11.s\n" - "fmla z30.s, p3/M, z4.s, z11.s\n" - "fmla z31.s, p3/M, z3.s, z11.s\n" + "fmla z25.s, p3/M, z5.s, z10.s\n" + "fmla z29.s, p3/M, z4.s, z10.s\n" + "fmla z24.s, p3/M, z3.s, z10.s\n" + "fmla z26.s, p3/M, z8.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x24, x14, LSL #2]\n" "fmla z23.s, p3/M, z8.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x24, x14, LSL #2]\n" - "fmla z28.s, p3/M, z8.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x13, x5, LSL #2]\n" - "fmla z25.s, p3/M, z8.s, z11.s\n" - "fmla z26.s, p3/M, z7.s, z11.s\n" + "ld1w { z12.s }, p2/Z, [x13, x5, LSL #2]\n" + "fmla z22.s, p3/M, z8.s, z10.s\n" + "fmla z20.s, p3/M, z7.s, z10.s\n" "addvl x24, x24, #1\n" - "fmla z27.s, p3/M, z6.s, z11.s\n" - "fmla z29.s, p3/M, z8.s, z12.s\n" - "ld1w { z11.s }, p2/Z, [x13, x10, LSL #2]\n" + "fmla z28.s, p3/M, z6.s, z10.s\n" + "fmla z25.s, p3/M, z8.s, z11.s\n" + "ld1w { z10.s }, p2/Z, [x13, x10, LSL #2]\n" "addvl x13, x13, #1\n" - "fmla z30.s, p3/M, z7.s, z12.s\n" - "fmla z31.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x28, x5, LSL #2]\n" - "fmla z16.s, p3/M, z4.s, z10.s\n" - "fmla z17.s, p3/M, z3.s, z10.s\n" - "fmax z16.s, p3/M, z16.s, z14.s\n" - "fmax z17.s, p3/M, z17.s, z14.s\n" - "fmla z20.s, p3/M, z1.s, z10.s\n" - "fmla z21.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x28, x10, LSL #2]\n" - "fmax z20.s, p3/M, z20.s, z14.s\n" - "fmla z18.s, p3/M, z5.s, z11.s\n" - "fmla z19.s, p3/M, z4.s, z11.s\n" - "fmax z18.s, p3/M, z18.s, z14.s\n" - "fmax z19.s, p3/M, z19.s, z14.s\n" - "fmla z22.s, p3/M, z2.s, z11.s\n" - "fmla z23.s, p3/M, z1.s, z11.s\n" - "fmax z21.s, p3/M, z21.s, z14.s\n" - "fmax z22.s, p3/M, z22.s, z14.s\n" - "fmla z24.s, p3/M, z7.s, z12.s\n" - "fmla z25.s, p3/M, z6.s, z12.s\n" - "fmax z23.s, p3/M, z23.s, z14.s\n" - "fmax z24.s, p3/M, z24.s, z14.s\n" - "fmla z28.s, p3/M, z4.s, z12.s\n" - "fmla z29.s, p3/M, z3.s, z12.s\n" - "fmax z25.s, p3/M, z25.s, z14.s\n" - "fmax z28.s, p3/M, z28.s, z14.s\n" - "fmla z26.s, p3/M, z8.s, z10.s\n" - "fmla z27.s, p3/M, z7.s, z10.s\n" - "fmax z26.s, p3/M, z26.s, z14.s\n" - "fmax z27.s, p3/M, z27.s, z14.s\n" - "fmla z30.s, p3/M, z5.s, z10.s\n" - "fmla z31.s, p3/M, z4.s, z10.s\n" - "fmax z29.s, p3/M, z29.s, z14.s\n" - "fmax z30.s, p3/M, z30.s, z14.s\n" - "fmax z31.s, p3/M, z31.s, z14.s\n" - "ld1w { z15.s }, p3/Z, [x17]\n" + "fmla z29.s, p3/M, z7.s, z11.s\n" + "fmla z24.s, p3/M, z6.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x28, x5, LSL #2]\n" + "fmla z31.s, p3/M, z4.s, z12.s\n" + "fmla z13.s, p3/M, z3.s, z12.s\n" + "fmax z31.s, p3/M, z31.s, z15.s\n" + "fmax z13.s, p3/M, z13.s, z15.s\n" + "fmla z27.s, p3/M, z1.s, z12.s\n" + "fmla z14.s, p3/M, z0.s, z12.s\n" + "ld1w { z0.s }, p2/Z, [x28, x10, LSL #2]\n" + "fmax z27.s, p3/M, z27.s, z15.s\n" + "fmla z17.s, p3/M, z5.s, z10.s\n" + "fmla z30.s, p3/M, z4.s, z10.s\n" + "fmax z17.s, p3/M, z17.s, z15.s\n" + "fmax z30.s, p3/M, z30.s, z15.s\n" + "fmla z21.s, p3/M, z2.s, z10.s\n" + "fmla z26.s, p3/M, z1.s, z10.s\n" + "fmax z14.s, p3/M, z14.s, z15.s\n" + "fmax z21.s, p3/M, z21.s, z15.s\n" + "fmla z18.s, p3/M, z7.s, z11.s\n" + "fmla z22.s, p3/M, z6.s, z11.s\n" + "fmax z26.s, p3/M, z26.s, z15.s\n" + "fmax z18.s, p3/M, z18.s, z15.s\n" + "fmla z23.s, p3/M, z4.s, z11.s\n" + "fmla z25.s, p3/M, z3.s, z11.s\n" + "fmax z22.s, p3/M, z22.s, z15.s\n" + "fmax z23.s, p3/M, z23.s, z15.s\n" + "fmla z20.s, p3/M, z8.s, z0.s\n" + "fmla z28.s, p3/M, z7.s, z0.s\n" + "fmax z20.s, p3/M, z20.s, z15.s\n" + "fmax z28.s, p3/M, z28.s, z15.s\n" + "fmla z29.s, p3/M, z5.s, z0.s\n" + "fmla z24.s, p3/M, z4.s, z0.s\n" + "fmax z25.s, p3/M, z25.s, z15.s\n" + "fmax z29.s, p3/M, z29.s, z15.s\n" + "fmax z24.s, p3/M, z24.s, z15.s\n" "ld1w { z0.s }, p3/Z, [x17, #1, MUL VL]\n" - "whilelt p2.s, x21, %x[n_channels]\n" "ld1w { z1.s }, p3/Z, [x17, #2, MUL VL]\n" + "whilelt p2.s, x21, %x[n_channels]\n" "ld1w { z2.s }, p3/Z, [x17, #3, MUL VL]\n" - "cmp x16, %x[n_channels]\n" - "fmin z16.s, p3/M, z16.s, z13.s\n" "ld1w { z3.s }, p3/Z, [x17, #4, MUL VL]\n" + "cmp x16, %x[n_channels]\n" + "fmin z31.s, p3/M, z31.s, z16.s\n" "ld1w { z4.s }, p3/Z, [x17, #5, MUL VL]\n" - "fmin z17.s, p3/M, z17.s, z13.s\n" - "fmin z18.s, p3/M, z18.s, z13.s\n" "ld1w { z5.s }, p3/Z, [x17, #6, MUL VL]\n" + "fmin z13.s, p3/M, z13.s, z16.s\n" + "fmin z17.s, p3/M, z17.s, z16.s\n" "ld1w { z6.s }, p3/Z, [x17, #7, MUL VL]\n" "addvl x17, x17, #16\n" - "fmin z19.s, p3/M, z19.s, z13.s\n" - "fmin z20.s, p3/M, z20.s, z13.s\n" - "fmin z21.s, p3/M, z21.s, z13.s\n" - "ld1w { z9.s }, p1/Z, [x12, x7, LSL #2]\n" + "fmin z30.s, p3/M, z30.s, z16.s\n" "ld1w { z10.s }, p1/Z, [x8]\n" - "fmin z22.s, p3/M, z22.s, z13.s\n" - "fmin z23.s, p3/M, z23.s, z13.s\n" + "fmin z27.s, p3/M, z27.s, z16.s\n" + "fmin z14.s, p3/M, z14.s, z16.s\n" "ld1w { z11.s }, p1/Z, [x8, x27, LSL #2]\n" "ld1w { z12.s }, p1/Z, [x12, x14, LSL #2]\n" - "fmin z24.s, p3/M, z24.s, z13.s\n" - "fmin z25.s, p3/M, z25.s, z13.s\n" - "st1w { z16.s }, p0, [x15]\n" + "fmin z21.s, p3/M, z21.s, z16.s\n" + "fmin z26.s, p3/M, z26.s, z16.s\n" + "st1w { z31.s }, p0, [x15]\n" "ld1w { z7.s }, p3/Z, [x17, #-8, MUL VL]\n" - "fmin z26.s, p3/M, z26.s, z13.s\n" - "fmin z27.s, p3/M, z27.s, z13.s\n" - "st1w { z17.s }, p0, [x15, x6, LSL #2]\n" + "fmin z18.s, p3/M, z18.s, z16.s\n" + "fmin z22.s, p3/M, z22.s, z16.s\n" + "st1w { z13.s }, p0, [x15, x6, LSL #2]\n" "ld1w { z8.s }, p3/Z, [x17, #-7, MUL VL]\n" - "fmin z28.s, p3/M, z28.s, z13.s\n" - "fmin z29.s, p3/M, z29.s, z13.s\n" - "st1w { z18.s }, p0, [x15, x25, LSL #2]\n" - "fmin z30.s, p3/M, z30.s, z13.s\n" - "fmin z31.s, p3/M, z31.s, z13.s\n" - "st1w { z19.s }, p0, [x15, x22, LSL #2]\n" + "fmin z20.s, p3/M, z20.s, z16.s\n" + "fmin z28.s, p3/M, z28.s, z16.s\n" + "st1w { z17.s }, p0, [x15, x25, LSL #2]\n" + "fmin z23.s, p3/M, z23.s, z16.s\n" + "fmin z25.s, p3/M, z25.s, z16.s\n" + "st1w { z30.s }, p0, [x15, x22, LSL #2]\n" + "fmin z29.s, p3/M, z29.s, z16.s\n" + "fmin z24.s, p3/M, z24.s, z16.s\n" + "st1w { z27.s }, p0, [x9]\n" "addvl x28, x28, #1\n" - "st1w { z20.s }, p0, [x9]\n" + "st1w { z14.s }, p0, [x9, x6, LSL #2]\n" "addvl x15, x15, #1\n" - "st1w { z21.s }, p0, [x9, x6, LSL #2]\n" + "st1w { z21.s }, p0, [x9, x25, LSL #2]\n" "addvl x17, x17, #-6\n" - "st1w { z22.s }, p0, [x9, x25, LSL #2]\n" - "st1w { z23.s }, p0, [x9, x22, LSL #2]\n" + "st1w { z26.s }, p0, [x9, x22, LSL #2]\n" "addvl x9, x9, #1\n" - "st1w { z24.s }, p0, [x26]\n" - "st1w { z25.s }, p0, [x26, x6, LSL #2]\n" - "st1w { z26.s }, p0, [x26, x25, LSL #2]\n" - "st1w { z27.s }, p0, [x26, x22, LSL #2]\n" + "st1w { z18.s }, p0, [x26]\n" + "st1w { z22.s }, p0, [x26, x6, LSL #2]\n" + "st1w { z20.s }, p0, [x26, x25, LSL #2]\n" + "st1w { z28.s }, p0, [x26, x22, LSL #2]\n" "addvl x26, x26, #1\n" - "st1w { z28.s }, p0, [x23]\n" - "st1w { z29.s }, p0, [x23, x6, LSL #2]\n" - "st1w { z30.s }, p0, [x23, x25, LSL #2]\n" - "st1w { z31.s }, p0, [x23, x22, LSL #2]\n" + "st1w { z23.s }, p0, [x23]\n" + "st1w { z25.s }, p0, [x23, x6, LSL #2]\n" + "st1w { z29.s }, p0, [x23, x25, LSL #2]\n" + "st1w { z24.s }, p0, [x23, x22, LSL #2]\n" "addvl x23, x23, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "movprfx z21, z15\n fmla z21.s, p3/M, z4.s, z9.s\n" - "movprfx z16, z15\n fmla z16.s, p3/M, z8.s, z9.s\n" + "movprfx z14, z19\n fmla z14.s, p3/M, z4.s, z9.s\n" + "movprfx z31, z19\n fmla z31.s, p3/M, z8.s, z9.s\n" "ldr x4, [%x[params_struct], %[offsetof_args_tile_j]]\n" "ldr x16, [%x[params_struct], %[offsetof_args_tile_i]]\n" - "movprfx z22, z15\n fmla z22.s, p3/M, z3.s, z9.s\n" - "movprfx z25, z15\n fmla z25.s, p3/M, z1.s, z9.s\n" + "movprfx z30, z19\n fmla z30.s, p3/M, z3.s, z9.s\n" + "movprfx z13, z19\n fmla z13.s, p3/M, z1.s, z9.s\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" "add x4, x4, #0x1\n" - "movprfx z26, z15\n fmla z26.s, p3/M, z0.s, z9.s\n" - "fmla z21.s, p3/M, z5.s, z12.s\n" + "movprfx z20, z19\n fmla z20.s, p3/M, z0.s, z9.s\n" + "fmla z14.s, p3/M, z5.s, z12.s\n" "cmp x4, x20\n" "add x21, x16, #0x1\n" - "movprfx z17, z15\n fmla z17.s, p3/M, z7.s, z9.s\n" - "movprfx z18, z15\n fmla z18.s, p3/M, z6.s, z9.s\n" + "movprfx z18, z19\n fmla z18.s, p3/M, z7.s, z9.s\n" + "movprfx z28, z19\n fmla z28.s, p3/M, z6.s, z9.s\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" "csel x16, x16, x21, LT\n" - "movprfx z20, z15\n fmla z20.s, p3/M, z5.s, z9.s\n" - "movprfx z24, z15\n fmla z24.s, p3/M, z2.s, z9.s\n" + "movprfx z17, z19\n fmla z17.s, p3/M, z5.s, z9.s\n" + "movprfx z26, z19\n fmla z26.s, p3/M, z2.s, z9.s\n" "ld1w { z9.s }, p2/Z, [x11, x7, LSL #2]\n" "mov p0.b, p2.b\n" - "fmla z16.s, p3/M, z0.s, z10.s\n" - "movprfx z19, z15\n fmla z19.s, p3/M, z2.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x24]\n" - "ld1w { z11.s }, p2/Z, [x24, x27, LSL #2]\n" - "fmla z22.s, p3/M, z4.s, z12.s\n" - "fmla z25.s, p3/M, z2.s, z12.s\n" + "fmla z31.s, p3/M, z0.s, z10.s\n" + "movprfx z27, z19\n fmla z27.s, p3/M, z2.s, z11.s\n" + "ld1w { z29.s }, p2/Z, [x24]\n" + "ld1w { z21.s }, p2/Z, [x24, x27, LSL #2]\n" + "fmla z30.s, p3/M, z4.s, z12.s\n" + "fmla z13.s, p3/M, z2.s, z12.s\n" "csel x4, x4, XZR, LT\n" "cmp x16, x20\n" - "fmla z26.s, p3/M, z1.s, z12.s\n" - "movprfx z28, z15\n fmla z28.s, p3/M, z6.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x11, x14, LSL #2]\n" - "fmla z21.s, p3/M, z7.s, z9.s\n" - "fmla z17.s, p3/M, z8.s, z12.s\n" - "fmla z18.s, p3/M, z7.s, z12.s\n" - "fmla z19.s, p3/M, z6.s, z12.s\n" - "movprfx z23, z15\n fmla z23.s, p3/M, z3.s, z12.s\n" - "movprfx z27, z15\n fmla z27.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x8, x5, LSL #2]\n" - "movprfx z31, z15\n fmla z31.s, p3/M, z8.s, z11.s\n" - "fmla z22.s, p3/M, z6.s, z9.s\n" - "ld1w { z11.s }, p2/Z, [x8, x10, LSL #2]\n" - "fmla z25.s, p3/M, z4.s, z9.s\n" - "fmla z26.s, p3/M, z3.s, z9.s\n" - "fmla z20.s, p3/M, z8.s, z9.s\n" - "fmla z24.s, p3/M, z5.s, z9.s\n" - "fmla z28.s, p3/M, z2.s, z9.s\n" - "fmla z21.s, p3/M, z8.s, z10.s\n" - "fmla z16.s, p3/M, z1.s, z12.s\n" - "fmla z17.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x13, x27, LSL #2]\n" - "fmla z18.s, p3/M, z2.s, z11.s\n" - "fmla z19.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x28]\n" - "fmla z22.s, p3/M, z7.s, z10.s\n" - "fmla z23.s, p3/M, z6.s, z10.s\n" - "fmla z25.s, p3/M, z5.s, z10.s\n" - "fmla z26.s, p3/M, z4.s, z10.s\n" - "fmla z27.s, p3/M, z3.s, z10.s\n" - "fmla z31.s, p3/M, z0.s, z10.s\n" - "fmla z24.s, p3/M, z6.s, z11.s\n" - "fmla z28.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x28, x27, LSL #2]\n" - "fmla z19.s, p3/M, z5.s, z12.s\n" - "fmla z23.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x13, x14, LSL #2]\n" - "fmla z27.s, p3/M, z8.s, z11.s\n" - "fmla z31.s, p3/M, z5.s, z11.s\n" - "movprfx z29, z15\n fmla z29.s, p3/M, z1.s, z9.s\n" - "movprfx z30, z15\n fmla z30.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x13]\n" - "fmla z29.s, p3/M, z2.s, z10.s\n" - "fmla z30.s, p3/M, z1.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x13, x7, LSL #2]\n" - "fmla z20.s, p3/M, z0.s, z9.s\n" - "fmla z21.s, p3/M, z1.s, z10.s\n" - "fmla z16.s, p3/M, z3.s, z9.s\n" - "fmla z17.s, p3/M, z4.s, z10.s\n" - "ld1w { z11.s }, p2/Z, [x24, x5, LSL #2]\n" - "fmla z18.s, p3/M, z3.s, z10.s\n" - "fmla z22.s, p3/M, z0.s, z10.s\n" - "fmla z20.s, p3/M, z2.s, z10.s\n" - "fmla z21.s, p3/M, z2.s, z12.s\n" - "fmla z16.s, p3/M, z5.s, z10.s\n" - "fmla z17.s, p3/M, z5.s, z12.s\n" - "ld1w { z10.s }, p2/Z, [x12, x5, LSL #2]\n" - "fmla z18.s, p3/M, z4.s, z12.s\n" - "fmla z19.s, p3/M, z3.s, z12.s\n" - "fmla z22.s, p3/M, z1.s, z12.s\n" - "fmla z23.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x12, x10, LSL #2]\n" - "fmla z28.s, p3/M, z7.s, z11.s\n" - "fmla z29.s, p3/M, z6.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x24, x10, LSL #2]\n" - "fmla z20.s, p3/M, z4.s, z10.s\n" - "fmla z21.s, p3/M, z3.s, z10.s\n" - "fmla z24.s, p3/M, z1.s, z10.s\n" - "fmla z25.s, p3/M, z0.s, z10.s\n" - "fmla z16.s, p3/M, z7.s, z10.s\n" - "fmla z17.s, p3/M, z6.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x8, x7, LSL #2]\n" - "fmla z30.s, p3/M, z8.s, z11.s\n" - "fmla z31.s, p3/M, z7.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x11, x5, LSL #2]\n" + "fmla z20.s, p3/M, z1.s, z12.s\n" + "movprfx z10, z19\n fmla z10.s, p3/M, z6.s, z29.s\n" + "ld1w { z29.s }, p2/Z, [x11, x14, LSL #2]\n" + "fmla z14.s, p3/M, z7.s, z9.s\n" "fmla z18.s, p3/M, z8.s, z12.s\n" - "fmla z19.s, p3/M, z7.s, z12.s\n" - "fmla z22.s, p3/M, z5.s, z12.s\n" - "fmla z23.s, p3/M, z4.s, z12.s\n" - "fmla z26.s, p3/M, z2.s, z12.s\n" - "fmla z27.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x8, x14, LSL #2]\n" - "fmla z20.s, p3/M, z7.s, z11.s\n" - "fmla z21.s, p3/M, z6.s, z11.s\n" - "fmla z24.s, p3/M, z4.s, z11.s\n" - "fmla z25.s, p3/M, z3.s, z11.s\n" - "fmla z28.s, p3/M, z1.s, z11.s\n" - "fmla z29.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x11, x10, LSL #2]\n" - "fmla z16.s, p3/M, z2.s, z10.s\n" - "fmla z17.s, p3/M, z1.s, z10.s\n" - "fmla z18.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x12]\n" - "fmla z30.s, p3/M, z2.s, z11.s\n" - "fmla z19.s, p3/M, z0.s, z12.s\n" - "fmla z20.s, p3/M, z3.s, z10.s\n" - "fmla z24.s, p3/M, z0.s, z10.s\n" - "fmla z22.s, p3/M, z8.s, z11.s\n" - "fmla z23.s, p3/M, z7.s, z11.s\n" - "fmla z26.s, p3/M, z5.s, z11.s\n" - "fmla z27.s, p3/M, z4.s, z11.s\n" - "fmla z31.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x28, x7, LSL #2]\n" - "fmla z17.s, p3/M, z2.s, z12.s\n" - "fmla z18.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x12, x27, LSL #2]\n" - "fmla z16.s, p3/M, z6.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x11]\n" - "fmla z29.s, p3/M, z4.s, z11.s\n" - "fmla z30.s, p3/M, z3.s, z11.s\n" - "fmla z19.s, p3/M, z8.s, z12.s\n" - "fmla z23.s, p3/M, z5.s, z12.s\n" - "fmla z27.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11, x27, LSL #2]\n" - "fmla z20.s, p3/M, z6.s, z10.s\n" - "fmla z24.s, p3/M, z3.s, z10.s\n" - "fmla z28.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x24, x7, LSL #2]\n" - "fmla z31.s, p3/M, z2.s, z12.s\n" - "fmla z29.s, p3/M, z7.s, z10.s\n" - "fmla z30.s, p3/M, z6.s, z10.s\n" - "fmla z24.s, p3/M, z8.s, z11.s\n" - "fmla z25.s, p3/M, z7.s, z11.s\n" - "fmla z26.s, p3/M, z6.s, z11.s\n" - "fmla z28.s, p3/M, z5.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x28, x14, LSL #2]\n" - "fmla z27.s, p3/M, z5.s, z12.s\n" - "fmla z29.s, p3/M, z5.s, z11.s\n" - "fmla z30.s, p3/M, z4.s, z11.s\n" - "fmla z31.s, p3/M, z3.s, z11.s\n" - "fmla z23.s, p3/M, z8.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x24, x14, LSL #2]\n" - "fmla z28.s, p3/M, z8.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x13, x5, LSL #2]\n" - "fmla z25.s, p3/M, z8.s, z11.s\n" - "fmla z26.s, p3/M, z7.s, z11.s\n" - "fmla z27.s, p3/M, z6.s, z11.s\n" - "fmla z29.s, p3/M, z8.s, z12.s\n" - "ld1w { z11.s }, p2/Z, [x13, x10, LSL #2]\n" - "fmla z30.s, p3/M, z7.s, z12.s\n" - "fmla z31.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x28, x5, LSL #2]\n" - "fmla z16.s, p3/M, z4.s, z10.s\n" - "fmla z17.s, p3/M, z3.s, z10.s\n" - "fmax z16.s, p3/M, z16.s, z14.s\n" - "fmax z17.s, p3/M, z17.s, z14.s\n" - "fmla z20.s, p3/M, z1.s, z10.s\n" - "fmla z21.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x28, x10, LSL #2]\n" - "fmax z20.s, p3/M, z20.s, z14.s\n" - "fmla z18.s, p3/M, z5.s, z11.s\n" - "fmla z19.s, p3/M, z4.s, z11.s\n" - "fmax z18.s, p3/M, z18.s, z14.s\n" - "fmax z19.s, p3/M, z19.s, z14.s\n" - "fmla z22.s, p3/M, z2.s, z11.s\n" - "fmla z23.s, p3/M, z1.s, z11.s\n" - "fmax z21.s, p3/M, z21.s, z14.s\n" - "fmax z22.s, p3/M, z22.s, z14.s\n" - "fmla z24.s, p3/M, z7.s, z12.s\n" - "fmla z25.s, p3/M, z6.s, z12.s\n" - "fmax z23.s, p3/M, z23.s, z14.s\n" - "fmax z24.s, p3/M, z24.s, z14.s\n" - "fmla z28.s, p3/M, z4.s, z12.s\n" - "fmla z29.s, p3/M, z3.s, z12.s\n" - "fmax z25.s, p3/M, z25.s, z14.s\n" - "fmax z28.s, p3/M, z28.s, z14.s\n" - "fmla z26.s, p3/M, z8.s, z10.s\n" - "fmla z27.s, p3/M, z7.s, z10.s\n" - "fmax z26.s, p3/M, z26.s, z14.s\n" - "fmax z27.s, p3/M, z27.s, z14.s\n" - "fmla z30.s, p3/M, z5.s, z10.s\n" - "fmla z31.s, p3/M, z4.s, z10.s\n" - "fmax z29.s, p3/M, z29.s, z14.s\n" - "fmax z30.s, p3/M, z30.s, z14.s\n" - "fmax z31.s, p3/M, z31.s, z14.s\n" - "fmin z16.s, p3/M, z16.s, z13.s\n" - "st1w { z16.s }, p0, [x15]\n" - "fmin z17.s, p3/M, z17.s, z13.s\n" - "fmin z18.s, p3/M, z18.s, z13.s\n" - "st1w { z17.s }, p0, [x15, x6, LSL #2]\n" - "fmin z19.s, p3/M, z19.s, z13.s\n" - "fmin z20.s, p3/M, z20.s, z13.s\n" - "st1w { z18.s }, p0, [x15, x25, LSL #2]\n" - "fmin z21.s, p3/M, z21.s, z13.s\n" - "fmin z22.s, p3/M, z22.s, z13.s\n" - "st1w { z19.s }, p0, [x15, x22, LSL #2]\n" - "fmin z23.s, p3/M, z23.s, z13.s\n" - "fmin z24.s, p3/M, z24.s, z13.s\n" - "st1w { z20.s }, p0, [x9]\n" - "fmin z25.s, p3/M, z25.s, z13.s\n" - "fmin z26.s, p3/M, z26.s, z13.s\n" - "st1w { z21.s }, p0, [x9, x6, LSL #2]\n" - "fmin z27.s, p3/M, z27.s, z13.s\n" - "fmin z28.s, p3/M, z28.s, z13.s\n" - "st1w { z22.s }, p0, [x9, x25, LSL #2]\n" - "fmin z29.s, p3/M, z29.s, z13.s\n" - "fmin z30.s, p3/M, z30.s, z13.s\n" - "st1w { z23.s }, p0, [x9, x22, LSL #2]\n" - "fmin z31.s, p3/M, z31.s, z13.s\n" - "st1w { z24.s }, p0, [x26]\n" - "st1w { z25.s }, p0, [x26, x6, LSL #2]\n" - "st1w { z26.s }, p0, [x26, x25, LSL #2]\n" - "st1w { z27.s }, p0, [x26, x22, LSL #2]\n" - "st1w { z28.s }, p0, [x23]\n" - "st1w { z29.s }, p0, [x23, x6, LSL #2]\n" - "st1w { z30.s }, p0, [x23, x25, LSL #2]\n" - "st1w { z31.s }, p0, [x23, x22, LSL #2]\n" + "fmla z28.s, p3/M, z7.s, z12.s\n" + "fmla z27.s, p3/M, z6.s, z12.s\n" + "movprfx z11, z19\n fmla z11.s, p3/M, z3.s, z12.s\n" + "movprfx z25, z19\n fmla z25.s, p3/M, z0.s, z12.s\n" + "ld1w { z22.s }, p2/Z, [x8, x5, LSL #2]\n" + "movprfx z24, z19\n fmla z24.s, p3/M, z8.s, z21.s\n" + "fmla z30.s, p3/M, z6.s, z9.s\n" + "ld1w { z21.s }, p2/Z, [x8, x10, LSL #2]\n" + "fmla z13.s, p3/M, z4.s, z9.s\n" + "fmla z20.s, p3/M, z3.s, z9.s\n" + "movprfx z12, z19\n fmla z12.s, p3/M, z1.s, z9.s\n" + "movprfx z23, z19\n fmla z23.s, p3/M, z0.s, z9.s\n" + "fmla z17.s, p3/M, z8.s, z9.s\n" + "fmla z26.s, p3/M, z5.s, z9.s\n" + "fmla z10.s, p3/M, z2.s, z9.s\n" + "fmla z14.s, p3/M, z8.s, z29.s\n" + "ld1w { z9.s }, p2/Z, [x13]\n" + "fmla z31.s, p3/M, z1.s, z22.s\n" + "fmla z18.s, p3/M, z0.s, z22.s\n" + "ld1w { z22.s }, p2/Z, [x13, x27, LSL #2]\n" + "fmla z28.s, p3/M, z2.s, z21.s\n" + "fmla z27.s, p3/M, z1.s, z21.s\n" + "ld1w { z19.s }, p2/Z, [x28]\n" + "fmla z30.s, p3/M, z7.s, z29.s\n" + "fmla z11.s, p3/M, z6.s, z29.s\n" + "fmla z13.s, p3/M, z5.s, z29.s\n" + "fmla z20.s, p3/M, z4.s, z29.s\n" + "fmla z25.s, p3/M, z3.s, z29.s\n" + "fmla z12.s, p3/M, z2.s, z29.s\n" + "fmla z23.s, p3/M, z1.s, z29.s\n" + "fmla z24.s, p3/M, z0.s, z29.s\n" + "ld1w { z21.s }, p2/Z, [x13, x7, LSL #2]\n" + "fmla z17.s, p3/M, z0.s, z9.s\n" + "fmla z26.s, p3/M, z6.s, z19.s\n" + "fmla z10.s, p3/M, z3.s, z19.s\n" + "fmla z14.s, p3/M, z1.s, z21.s\n" + "ld1w { z19.s }, p2/Z, [x28, x27, LSL #2]\n" + "fmla z31.s, p3/M, z3.s, z9.s\n" + "fmla z27.s, p3/M, z5.s, z22.s\n" + "fmla z11.s, p3/M, z2.s, z22.s\n" + "fmla z18.s, p3/M, z4.s, z21.s\n" + "ld1w { z29.s }, p2/Z, [x13, x14, LSL #2]\n" + "fmla z28.s, p3/M, z3.s, z21.s\n" + "fmla z30.s, p3/M, z0.s, z21.s\n" + "fmla z25.s, p3/M, z8.s, z19.s\n" + "fmla z24.s, p3/M, z5.s, z19.s\n" + "ld1w { z19.s }, p2/Z, [x24, x5, LSL #2]\n" + "fmla z17.s, p3/M, z2.s, z21.s\n" + "fmla z14.s, p3/M, z2.s, z29.s\n" + "fmla z31.s, p3/M, z5.s, z21.s\n" + "fmla z18.s, p3/M, z5.s, z29.s\n" + "ld1w { z22.s }, p2/Z, [x12, x5, LSL #2]\n" + "fmla z28.s, p3/M, z4.s, z29.s\n" + "fmla z27.s, p3/M, z3.s, z29.s\n" + "fmla z30.s, p3/M, z1.s, z29.s\n" + "fmla z11.s, p3/M, z0.s, z29.s\n" + "ld1w { z21.s }, p2/Z, [x12, x10, LSL #2]\n" + "fmla z10.s, p3/M, z7.s, z19.s\n" + "fmla z12.s, p3/M, z6.s, z19.s\n" + "ld1w { z19.s }, p2/Z, [x24, x10, LSL #2]\n" + "fmla z17.s, p3/M, z4.s, z22.s\n" + "fmla z14.s, p3/M, z3.s, z22.s\n" + "fmla z26.s, p3/M, z1.s, z22.s\n" + "fmla z13.s, p3/M, z0.s, z22.s\n" + "fmla z31.s, p3/M, z7.s, z22.s\n" + "fmla z18.s, p3/M, z6.s, z22.s\n" + "ld1w { z29.s }, p2/Z, [x8, x7, LSL #2]\n" + "fmla z23.s, p3/M, z8.s, z19.s\n" + "fmla z24.s, p3/M, z7.s, z19.s\n" + "ld1w { z19.s }, p2/Z, [x11, x5, LSL #2]\n" + "fmla z28.s, p3/M, z8.s, z21.s\n" + "fmla z27.s, p3/M, z7.s, z21.s\n" + "fmla z30.s, p3/M, z5.s, z21.s\n" + "fmla z11.s, p3/M, z4.s, z21.s\n" + "fmla z20.s, p3/M, z2.s, z21.s\n" + "fmla z25.s, p3/M, z1.s, z21.s\n" + "ld1w { z22.s }, p2/Z, [x8, x14, LSL #2]\n" + "fmla z17.s, p3/M, z7.s, z19.s\n" + "fmla z14.s, p3/M, z6.s, z19.s\n" + "fmla z26.s, p3/M, z4.s, z19.s\n" + "fmla z13.s, p3/M, z3.s, z19.s\n" + "fmla z10.s, p3/M, z1.s, z19.s\n" + "fmla z12.s, p3/M, z0.s, z19.s\n" + "ld1w { z21.s }, p2/Z, [x11, x10, LSL #2]\n" + "fmla z31.s, p3/M, z2.s, z29.s\n" + "fmla z18.s, p3/M, z1.s, z29.s\n" + "fmla z28.s, p3/M, z0.s, z29.s\n" + "ld1w { z29.s }, p2/Z, [x12]\n" + "fmla z23.s, p3/M, z2.s, z21.s\n" + "fmla z27.s, p3/M, z0.s, z22.s\n" + "fmla z17.s, p3/M, z3.s, z29.s\n" + "fmla z26.s, p3/M, z0.s, z29.s\n" + "fmla z30.s, p3/M, z8.s, z21.s\n" + "fmla z11.s, p3/M, z7.s, z21.s\n" + "fmla z20.s, p3/M, z5.s, z21.s\n" + "fmla z25.s, p3/M, z4.s, z21.s\n" + "fmla z24.s, p3/M, z1.s, z21.s\n" + "ld1w { z19.s }, p2/Z, [x28, x7, LSL #2]\n" + "fmla z18.s, p3/M, z2.s, z22.s\n" + "fmla z28.s, p3/M, z1.s, z22.s\n" + "ld1w { z21.s }, p2/Z, [x12, x27, LSL #2]\n" + "fmla z31.s, p3/M, z6.s, z29.s\n" + "ld1w { z29.s }, p2/Z, [x11]\n" + "fmla z12.s, p3/M, z4.s, z19.s\n" + "fmla z23.s, p3/M, z3.s, z19.s\n" + "fmla z27.s, p3/M, z8.s, z21.s\n" + "fmla z11.s, p3/M, z5.s, z21.s\n" + "fmla z25.s, p3/M, z2.s, z21.s\n" + "ld1w { z9.s }, p2/Z, [x11, x27, LSL #2]\n" + "fmla z17.s, p3/M, z6.s, z29.s\n" + "fmla z26.s, p3/M, z3.s, z29.s\n" + "fmla z10.s, p3/M, z0.s, z29.s\n" + "ld1w { z22.s }, p2/Z, [x24, x7, LSL #2]\n" + "fmla z24.s, p3/M, z2.s, z9.s\n" + "fmla z12.s, p3/M, z7.s, z22.s\n" + "fmla z23.s, p3/M, z6.s, z22.s\n" + "fmla z26.s, p3/M, z8.s, z19.s\n" + "fmla z13.s, p3/M, z7.s, z19.s\n" + "fmla z20.s, p3/M, z6.s, z19.s\n" + "fmla z10.s, p3/M, z5.s, z19.s\n" + "ld1w { z21.s }, p2/Z, [x28, x14, LSL #2]\n" + "fmla z25.s, p3/M, z5.s, z9.s\n" + "fmla z12.s, p3/M, z5.s, z21.s\n" + "fmla z23.s, p3/M, z4.s, z21.s\n" + "fmla z24.s, p3/M, z3.s, z21.s\n" + "fmla z11.s, p3/M, z8.s, z9.s\n" + "ld1w { z19.s }, p2/Z, [x24, x14, LSL #2]\n" + "fmla z10.s, p3/M, z8.s, z22.s\n" + "ld1w { z22.s }, p2/Z, [x13, x5, LSL #2]\n" + "fmla z13.s, p3/M, z8.s, z21.s\n" + "fmla z20.s, p3/M, z7.s, z21.s\n" + "fmla z25.s, p3/M, z6.s, z21.s\n" + "fmla z12.s, p3/M, z8.s, z19.s\n" + "ld1w { z29.s }, p2/Z, [x13, x10, LSL #2]\n" + "fmla z23.s, p3/M, z7.s, z19.s\n" + "fmla z24.s, p3/M, z6.s, z19.s\n" + "ld1w { z21.s }, p2/Z, [x28, x5, LSL #2]\n" + "fmla z31.s, p3/M, z4.s, z22.s\n" + "fmla z18.s, p3/M, z3.s, z22.s\n" + "fmax z31.s, p3/M, z31.s, z15.s\n" + "fmax z18.s, p3/M, z18.s, z15.s\n" + "fmla z17.s, p3/M, z1.s, z22.s\n" + "fmla z14.s, p3/M, z0.s, z22.s\n" + "ld1w { z9.s }, p2/Z, [x28, x10, LSL #2]\n" + "fmax z17.s, p3/M, z17.s, z15.s\n" + "fmla z28.s, p3/M, z5.s, z29.s\n" + "fmla z27.s, p3/M, z4.s, z29.s\n" + "fmax z28.s, p3/M, z28.s, z15.s\n" + "fmax z27.s, p3/M, z27.s, z15.s\n" + "fmla z30.s, p3/M, z2.s, z29.s\n" + "fmla z11.s, p3/M, z1.s, z29.s\n" + "fmax z14.s, p3/M, z14.s, z15.s\n" + "fmax z30.s, p3/M, z30.s, z15.s\n" + "fmla z26.s, p3/M, z7.s, z21.s\n" + "fmla z13.s, p3/M, z6.s, z21.s\n" + "fmax z11.s, p3/M, z11.s, z15.s\n" + "fmax z26.s, p3/M, z26.s, z15.s\n" + "fmla z10.s, p3/M, z4.s, z21.s\n" + "fmla z12.s, p3/M, z3.s, z21.s\n" + "fmax z13.s, p3/M, z13.s, z15.s\n" + "fmax z10.s, p3/M, z10.s, z15.s\n" + "fmla z20.s, p3/M, z8.s, z9.s\n" + "fmla z25.s, p3/M, z7.s, z9.s\n" + "fmax z20.s, p3/M, z20.s, z15.s\n" + "fmax z25.s, p3/M, z25.s, z15.s\n" + "fmla z23.s, p3/M, z5.s, z9.s\n" + "fmla z24.s, p3/M, z4.s, z9.s\n" + "fmax z12.s, p3/M, z12.s, z15.s\n" + "fmax z23.s, p3/M, z23.s, z15.s\n" + "fmax z24.s, p3/M, z24.s, z15.s\n" + "fmin z31.s, p3/M, z31.s, z16.s\n" + "st1w { z31.s }, p0, [x15]\n" + "fmin z18.s, p3/M, z18.s, z16.s\n" + "fmin z28.s, p3/M, z28.s, z16.s\n" + "st1w { z18.s }, p0, [x15, x6, LSL #2]\n" + "fmin z27.s, p3/M, z27.s, z16.s\n" + "fmin z17.s, p3/M, z17.s, z16.s\n" + "st1w { z28.s }, p0, [x15, x25, LSL #2]\n" + "fmin z14.s, p3/M, z14.s, z16.s\n" + "fmin z30.s, p3/M, z30.s, z16.s\n" + "st1w { z27.s }, p0, [x15, x22, LSL #2]\n" + "fmin z11.s, p3/M, z11.s, z16.s\n" + "fmin z26.s, p3/M, z26.s, z16.s\n" + "st1w { z17.s }, p0, [x9]\n" + "fmin z13.s, p3/M, z13.s, z16.s\n" + "fmin z20.s, p3/M, z20.s, z16.s\n" + "st1w { z14.s }, p0, [x9, x6, LSL #2]\n" + "fmin z25.s, p3/M, z25.s, z16.s\n" + "fmin z10.s, p3/M, z10.s, z16.s\n" + "st1w { z30.s }, p0, [x9, x25, LSL #2]\n" + "fmin z12.s, p3/M, z12.s, z16.s\n" + "fmin z23.s, p3/M, z23.s, z16.s\n" + "st1w { z11.s }, p0, [x9, x22, LSL #2]\n" + "fmin z24.s, p3/M, z24.s, z16.s\n" + "st1w { z26.s }, p0, [x26]\n" + "st1w { z13.s }, p0, [x26, x6, LSL #2]\n" + "st1w { z20.s }, p0, [x26, x25, LSL #2]\n" + "st1w { z25.s }, p0, [x26, x22, LSL #2]\n" + "st1w { z10.s }, p0, [x23]\n" + "st1w { z12.s }, p0, [x23, x6, LSL #2]\n" + "st1w { z23.s }, p0, [x23, x25, LSL #2]\n" + "st1w { z24.s }, p0, [x23, x22, LSL #2]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) @@ -653,4 +653,4 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp index 0b04ae064d957bad785f7aeb4655e317e9557269..3db248924fb2cc39eb528513d1d57de46256403c 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -99,616 +99,616 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( __asm__ __volatile__( "ptrue p3.b\n" - "ldr x17, [%x[params_struct], %[offsetof_args_params]]\n" - "add x16, %x[params_struct], %[offsetof_Args_inptrs]\n" - "ld1w { z15.s }, p3/Z, [x17]\n" - "cntw x15\n" - "mov x14, #0x0\n" - "ld1w { z0.s }, p3/Z, [x17, #1, MUL VL]\n" - "ld1w { z1.s }, p3/Z, [x17, #2, MUL VL]\n" + "ldr x7, [%x[params_struct], %[offsetof_args_params]]\n" + "add x8, %x[params_struct], %[offsetof_Args_inptrs]\n" + "ld1w { z17.s }, p3/Z, [x7]\n" + "cntw x17\n" + "mov x16, #0x0\n" + "ld1w { z0.s }, p3/Z, [x7, #1, MUL VL]\n" + "ld1w { z1.s }, p3/Z, [x7, #2, MUL VL]\n" "whilelt p2.s, XZR, %x[n_channels]\n" - "ld1w { z2.s }, p3/Z, [x17, #3, MUL VL]\n" - "ld1w { z3.s }, p3/Z, [x17, #4, MUL VL]\n" - "cmp x15, %x[n_channels]\n" - "ld1w { z4.s }, p3/Z, [x17, #5, MUL VL]\n" - "ld1w { z5.s }, p3/Z, [x17, #6, MUL VL]\n" - "sub x13, XZR, x15\n" - "ld1w { z6.s }, p3/Z, [x17, #7, MUL VL]\n" - "addvl x17, x17, #16\n" - "ldp x12, x11, [x16, #0x0]\n" - "ldp x10, x9, [x16, #0x10]\n" - "ldr x28, [%x[params_struct], %[offsetof_args_outptrs]]\n" - "ld1rw { z14.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "ld1rw { z13.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "ld1w { z7.s }, p3/Z, [x17, #-8, MUL VL]\n" - "ld1w { z8.s }, p3/Z, [x17, #-7, MUL VL]\n" - "addvl x17, x17, #-6\n" - "ld1w { z9.s }, p2/Z, [x12, x14, LSL #2]\n" - "ld1w { z10.s }, p2/Z, [x11, x14, LSL #2]\n" - "ld1w { z11.s }, p2/Z, [x10, x14, LSL #2]\n" - "ld1w { z12.s }, p2/Z, [x9, x14, LSL #2]\n" + "ld1w { z2.s }, p3/Z, [x7, #3, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x7, #4, MUL VL]\n" + "cmp x17, %x[n_channels]\n" + "ld1w { z4.s }, p3/Z, [x7, #5, MUL VL]\n" + "ld1w { z5.s }, p3/Z, [x7, #6, MUL VL]\n" + "sub x15, XZR, x17\n" + "ld1w { z6.s }, p3/Z, [x7, #7, MUL VL]\n" + "addvl x7, x7, #16\n" + "ldp x23, x22, [x8, #0x0]\n" + "ldp x21, x20, [x8, #0x10]\n" + "ldr x14, [%x[params_struct], %[offsetof_args_outptrs]]\n" + "ld1rw { z16.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rw { z19.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "ld1w { z7.s }, p3/Z, [x7, #-8, MUL VL]\n" + "ld1w { z8.s }, p3/Z, [x7, #-7, MUL VL]\n" + "addvl x7, x7, #-6\n" + "ld1w { z9.s }, p2/Z, [x23, x16, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x22, x16, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x21, x16, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x20, x16, LSL #2]\n" "bge 2f\n" "1:" // Channel loop - "movprfx z21, z15\n fmla z21.s, p3/M, z4.s, z9.s\n" - "movprfx z16, z15\n fmla z16.s, p3/M, z8.s, z9.s\n" - "ldr x27, [x16, #0x20]\n" - "ldr x26, [x16, #0x30]\n" - "movprfx z22, z15\n fmla z22.s, p3/M, z3.s, z9.s\n" - "movprfx z25, z15\n fmla z25.s, p3/M, z1.s, z9.s\n" - "ldr x25, [x16, #0x28]\n" - "ldr x24, [x16, #0x38]\n" - "movprfx z26, z15\n fmla z26.s, p3/M, z0.s, z9.s\n" - "movprfx z17, z15\n fmla z17.s, p3/M, z7.s, z9.s\n" - "ldr x12, [x16, #0x40]\n" - "ldr x11, [x16, #0x48]\n" - "movprfx z18, z15\n fmla z18.s, p3/M, z6.s, z9.s\n" - "fmla z21.s, p3/M, z5.s, z12.s\n" - "ldr x10, [x16, #0x50]\n" - "ldr x9, [x16, #0x58]\n" - "movprfx z20, z15\n fmla z20.s, p3/M, z5.s, z9.s\n" - "movprfx z24, z15\n fmla z24.s, p3/M, z2.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x26, x14, LSL #2]\n" - "ldr x26, [x16, #0x70]\n" - "fmla z16.s, p3/M, z0.s, z10.s\n" - "movprfx z19, z15\n fmla z19.s, p3/M, z2.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x27, x14, LSL #2]\n" - "ld1w { z11.s }, p2/Z, [x25, x14, LSL #2]\n" - "fmla z22.s, p3/M, z4.s, z12.s\n" - "fmla z25.s, p3/M, z2.s, z12.s\n" - "ldr x27, [x16, #0x60]\n" - "ldr x25, [x16, #0x68]\n" - "fmla z26.s, p3/M, z1.s, z12.s\n" - "fmla z17.s, p3/M, z8.s, z12.s\n" - "incw x13\n" + "movprfx z20, z17\n fmla z20.s, p3/M, z4.s, z9.s\n" + "movprfx z26, z17\n fmla z26.s, p3/M, z8.s, z9.s\n" + "ldr x27, [x8, #0x20]\n" + "ldr x24, [x8, #0x30]\n" + "movprfx z24, z17\n fmla z24.s, p3/M, z3.s, z9.s\n" + "movprfx z30, z17\n fmla z30.s, p3/M, z1.s, z9.s\n" + "ldr x23, [x8, #0x28]\n" + "ldr x22, [x8, #0x38]\n" + "movprfx z31, z17\n fmla z31.s, p3/M, z0.s, z9.s\n" + "movprfx z22, z17\n fmla z22.s, p3/M, z7.s, z9.s\n" + "ldr x26, [x8, #0x40]\n" + "ldr x21, [x8, #0x48]\n" + "movprfx z27, z17\n fmla z27.s, p3/M, z6.s, z9.s\n" + "fmla z20.s, p3/M, z5.s, z12.s\n" + "ldr x25, [x8, #0x50]\n" + "ldr x20, [x8, #0x58]\n" + "movprfx z14, z17\n fmla z14.s, p3/M, z5.s, z9.s\n" + "movprfx z23, z17\n fmla z23.s, p3/M, z2.s, z9.s\n" + "ld1w { z25.s }, p2/Z, [x24, x16, LSL #2]\n" + "ldr x13, [x8, #0x70]\n" + "fmla z26.s, p3/M, z0.s, z10.s\n" + "movprfx z9, z17\n fmla z9.s, p3/M, z2.s, z11.s\n" + "ld1w { z28.s }, p2/Z, [x27, x16, LSL #2]\n" + "ld1w { z21.s }, p2/Z, [x23, x16, LSL #2]\n" + "fmla z24.s, p3/M, z4.s, z12.s\n" + "fmla z30.s, p3/M, z2.s, z12.s\n" + "ldr x24, [x8, #0x60]\n" + "ldr x23, [x8, #0x68]\n" + "fmla z31.s, p3/M, z1.s, z12.s\n" + "fmla z22.s, p3/M, z8.s, z12.s\n" + "incw x15\n" "mov p1.b, p2.b\n" - "fmla z18.s, p3/M, z7.s, z12.s\n" - "movprfx z28, z15\n fmla z28.s, p3/M, z6.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x11, x14, LSL #2]\n" - "ldr x11, [x16, #0x88]\n" - "fmla z21.s, p3/M, z7.s, z9.s\n" - "fmla z19.s, p3/M, z6.s, z12.s\n" - "ldr x23, [x28, #0x0]\n" - "ldr x22, [x28, #0x8]\n" - "movprfx z23, z15\n fmla z23.s, p3/M, z3.s, z12.s\n" - "movprfx z27, z15\n fmla z27.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x24, x14, LSL #2]\n" - "ldr x24, [x16, #0x78]\n" - "movprfx z31, z15\n fmla z31.s, p3/M, z8.s, z11.s\n" - "fmla z22.s, p3/M, z6.s, z9.s\n" - "ld1w { z11.s }, p2/Z, [x12, x14, LSL #2]\n" - "ldr x12, [x16, #0x80]\n" - "fmla z25.s, p3/M, z4.s, z9.s\n" - "fmla z26.s, p3/M, z3.s, z9.s\n" - "ldr x21, [x28, #0x10]\n" - "ldr x20, [x28, #0x18]\n" - "fmla z20.s, p3/M, z8.s, z9.s\n" - "fmla z24.s, p3/M, z5.s, z9.s\n" - "whilelt p0.s, x15, %x[n_channels]\n" - "fmla z28.s, p3/M, z2.s, z9.s\n" - "fmla z16.s, p3/M, z1.s, z12.s\n" - "fmla z17.s, p3/M, z0.s, z12.s\n" - "movprfx z29, z15\n fmla z29.s, p3/M, z1.s, z9.s\n" - "movprfx z30, z15\n fmla z30.s, p3/M, z0.s, z9.s\n" - "fmla z18.s, p3/M, z2.s, z11.s\n" - "ld1w { z9.s }, p2/Z, [x10, x14, LSL #2]\n" - "ldr x10, [x16, #0x90]\n" - "fmla z21.s, p3/M, z8.s, z10.s\n" - "fmla z19.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x27, x14, LSL #2]\n" - "ldr x27, [x16, #0xa0]\n" - "fmla z22.s, p3/M, z7.s, z10.s\n" - "fmla z23.s, p3/M, z6.s, z10.s\n" - "fmla z25.s, p3/M, z5.s, z10.s\n" - "fmla z26.s, p3/M, z4.s, z10.s\n" - "fmla z27.s, p3/M, z3.s, z10.s\n" - "fmla z29.s, p3/M, z2.s, z10.s\n" - "fmla z30.s, p3/M, z1.s, z10.s\n" - "fmla z31.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x25, x14, LSL #2]\n" - "ldr x25, [x16, #0xa8]\n" - "fmla z16.s, p3/M, z3.s, z9.s\n" - "fmla z20.s, p3/M, z0.s, z9.s\n" - "ld1w { z12.s }, p2/Z, [x9, x14, LSL #2]\n" - "ldr x9, [x16, #0x98]\n" - "fmla z24.s, p3/M, z6.s, z11.s\n" - "fmla z28.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x14, LSL #2]\n" - "ldr x26, [x16, #0xb0]\n" - "fmla z17.s, p3/M, z4.s, z10.s\n" - "fmla z18.s, p3/M, z3.s, z10.s\n" + "fmla z27.s, p3/M, z7.s, z12.s\n" + "movprfx z15, z17\n fmla z15.s, p3/M, z6.s, z28.s\n" + "ld1w { z10.s }, p2/Z, [x21, x16, LSL #2]\n" + "ldr x28, [x8, #0x88]\n" + "fmla z20.s, p3/M, z7.s, z25.s\n" + "fmla z9.s, p3/M, z6.s, z12.s\n" + "ldr x12, [x14, #0x0]\n" + "ldr x11, [x14, #0x8]\n" + "movprfx z11, z17\n fmla z11.s, p3/M, z3.s, z12.s\n" + "movprfx z13, z17\n fmla z13.s, p3/M, z0.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x22, x16, LSL #2]\n" + "ldr x22, [x8, #0x78]\n" + "movprfx z28, z17\n fmla z28.s, p3/M, z8.s, z21.s\n" + "fmla z24.s, p3/M, z6.s, z25.s\n" + "ld1w { z29.s }, p2/Z, [x26, x16, LSL #2]\n" + "ldr x21, [x8, #0x80]\n" + "fmla z30.s, p3/M, z4.s, z25.s\n" + "fmla z31.s, p3/M, z3.s, z25.s\n" + "ldr x10, [x14, #0x10]\n" + "ldr x9, [x14, #0x18]\n" + "movprfx z18, z17\n fmla z18.s, p3/M, z1.s, z25.s\n" + "movprfx z21, z17\n fmla z21.s, p3/M, z0.s, z25.s\n" + "whilelt p0.s, x17, %x[n_channels]\n" + "ld1w { z17.s }, p3/Z, [x7]\n" + "fmla z14.s, p3/M, z8.s, z25.s\n" + "fmla z23.s, p3/M, z5.s, z25.s\n" + "fmla z15.s, p3/M, z2.s, z25.s\n" + "fmla z26.s, p3/M, z1.s, z12.s\n" + "ld1w { z25.s }, p2/Z, [x25, x16, LSL #2]\n" + "ldr x27, [x8, #0x90]\n" + "fmla z22.s, p3/M, z0.s, z12.s\n" + "fmla z27.s, p3/M, z2.s, z29.s\n" + "ld1w { z12.s }, p2/Z, [x20, x16, LSL #2]\n" + "ldr x20, [x8, #0x98]\n" + "fmla z20.s, p3/M, z8.s, z10.s\n" + "fmla z9.s, p3/M, z1.s, z29.s\n" + "ld1w { z29.s }, p2/Z, [x24, x16, LSL #2]\n" + "ldr x26, [x8, #0xa0]\n" + "fmla z24.s, p3/M, z7.s, z10.s\n" + "fmla z11.s, p3/M, z6.s, z10.s\n" + "fmla z30.s, p3/M, z5.s, z10.s\n" + "fmla z31.s, p3/M, z4.s, z10.s\n" + "fmla z13.s, p3/M, z3.s, z10.s\n" + "fmla z18.s, p3/M, z2.s, z10.s\n" "fmla z21.s, p3/M, z1.s, z10.s\n" - "fmla z19.s, p3/M, z5.s, z12.s\n" - "fmla z23.s, p3/M, z2.s, z12.s\n" - "fmla z22.s, p3/M, z0.s, z10.s\n" - "ld1w { z12.s }, p2/Z, [x24, x14, LSL #2]\n" - "ldr x24, [x16, #0xb8]\n" - "fmla z27.s, p3/M, z8.s, z11.s\n" - "fmla z31.s, p3/M, z5.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x12, x14, LSL #2]\n" - "ldr x12, [x16, #0xc0]\n" - "fmla z16.s, p3/M, z5.s, z10.s\n" - "fmla z20.s, p3/M, z2.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x11, x14, LSL #2]\n" - "ldr x11, [x16, #0xc8]\n" - "fmla z17.s, p3/M, z5.s, z12.s\n" - "fmla z18.s, p3/M, z4.s, z12.s\n" - "fmla z21.s, p3/M, z2.s, z12.s\n" - "fmla z19.s, p3/M, z3.s, z12.s\n" - "fmla z22.s, p3/M, z1.s, z12.s\n" - "fmla z23.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x9, x14, LSL #2]\n" - "ldr x9, [x16, #0xd8]\n" - "fmla z28.s, p3/M, z7.s, z11.s\n" - "fmla z29.s, p3/M, z6.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x10, x14, LSL #2]\n" - "ldr x10, [x16, #0xd0]\n" - "fmla z16.s, p3/M, z7.s, z10.s\n" - "fmla z17.s, p3/M, z6.s, z10.s\n" - "fmla z20.s, p3/M, z4.s, z10.s\n" - "fmla z21.s, p3/M, z3.s, z10.s\n" - "fmla z24.s, p3/M, z1.s, z10.s\n" - "fmla z25.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x27, x14, LSL #2]\n" - "ldr x27, [x16, #0xe0]\n" - "fmla z18.s, p3/M, z8.s, z12.s\n" - "fmla z30.s, p3/M, z8.s, z11.s\n" - "fmla z31.s, p3/M, z7.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x25, x14, LSL #2]\n" - "fmla z27.s, p3/M, z1.s, z12.s\n" - "ldr x25, [x16, #0xe8]\n" - "fmla z19.s, p3/M, z7.s, z12.s\n" - "fmla z22.s, p3/M, z5.s, z12.s\n" - "fmla z23.s, p3/M, z4.s, z12.s\n" - "fmla z26.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x26, x14, LSL #2]\n" - "ldr x26, [x16, #0xf0]\n" - "fmla z16.s, p3/M, z2.s, z10.s\n" - "fmla z17.s, p3/M, z1.s, z10.s\n" - "fmla z18.s, p3/M, z0.s, z10.s\n" - "fmla z20.s, p3/M, z7.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x24, x14, LSL #2]\n" - "ldr x24, [x16, #0xf8]\n" - "fmla z21.s, p3/M, z6.s, z11.s\n" - "fmla z24.s, p3/M, z4.s, z11.s\n" - "fmla z25.s, p3/M, z3.s, z11.s\n" - "fmla z28.s, p3/M, z1.s, z11.s\n" - "fmla z29.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x12, x14, LSL #2]\n" - "fmla z27.s, p3/M, z4.s, z11.s\n" - "ldr x12, [x16, #0x100]\n" - "fmla z30.s, p3/M, z2.s, z11.s\n" - "fmla z17.s, p3/M, z2.s, z12.s\n" - "fmla z18.s, p3/M, z1.s, z12.s\n" - "fmla z19.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11, x14, LSL #2]\n" - "ldr x11, [x16, #0x108]\n" - "fmla z16.s, p3/M, z6.s, z10.s\n" - "fmla z20.s, p3/M, z3.s, z10.s\n" - "fmla z24.s, p3/M, z0.s, z10.s\n" - "fmla z22.s, p3/M, z8.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x10, x14, LSL #2]\n" - "ldr x10, [x16, #0x110]\n" - "fmla z23.s, p3/M, z7.s, z11.s\n" - "fmla z26.s, p3/M, z5.s, z11.s\n" - "fmla z31.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x9, x14, LSL #2]\n" - "fmla z27.s, p3/M, z2.s, z12.s\n" - "ldr x9, [x16, #0x118]\n" "fmla z28.s, p3/M, z0.s, z10.s\n" - "fmla z29.s, p3/M, z4.s, z11.s\n" - "fmla z30.s, p3/M, z3.s, z11.s\n" - "fmla z19.s, p3/M, z8.s, z12.s\n" - "fmla z23.s, p3/M, z5.s, z12.s\n" - "fmla z20.s, p3/M, z6.s, z10.s\n" - "ld1w { z12.s }, p2/Z, [x27, x14, LSL #2]\n" - "fmla z24.s, p3/M, z3.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x25, x14, LSL #2]\n" - "fmla z25.s, p3/M, z7.s, z11.s\n" - "fmla z26.s, p3/M, z6.s, z11.s\n" - "fmla z28.s, p3/M, z5.s, z11.s\n" - "fmla z27.s, p3/M, z5.s, z12.s\n" - "fmla z31.s, p3/M, z2.s, z12.s\n" - "fmla z29.s, p3/M, z7.s, z10.s\n" - "fmla z30.s, p3/M, z6.s, z10.s\n" - "fmla z24.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x14, LSL #2]\n" - "fmla z28.s, p3/M, z8.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x12, x14, LSL #2]\n" - "fmla z25.s, p3/M, z8.s, z11.s\n" - "fmla z26.s, p3/M, z7.s, z11.s\n" - "fmla z27.s, p3/M, z6.s, z11.s\n" - "fmla z29.s, p3/M, z5.s, z11.s\n" - "fmla z30.s, p3/M, z4.s, z11.s\n" - "fmla z31.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x11, x14, LSL #2]\n" - "ldp x12, x11, [x16, #0x0]\n" - "fmla z23.s, p3/M, z8.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x24, x14, LSL #2]\n" - "fmla z16.s, p3/M, z4.s, z10.s\n" - "fmax z16.s, p3/M, z16.s, z14.s\n" - "fmla z17.s, p3/M, z3.s, z10.s\n" - "fmla z18.s, p3/M, z5.s, z11.s\n" - "fmax z17.s, p3/M, z17.s, z14.s\n" - "fmax z18.s, p3/M, z18.s, z14.s\n" - "fmla z19.s, p3/M, z4.s, z11.s\n" - "fmla z29.s, p3/M, z8.s, z12.s\n" - "fmax z19.s, p3/M, z19.s, z14.s\n" - "fmin z16.s, p3/M, z16.s, z13.s\n" - "fmla z30.s, p3/M, z7.s, z12.s\n" - "fmla z31.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x10, x14, LSL #2]\n" - "fmin z17.s, p3/M, z17.s, z13.s\n" + "ld1w { z10.s }, p2/Z, [x23, x16, LSL #2]\n" + "ldr x25, [x8, #0xa8]\n" + "fmla z26.s, p3/M, z3.s, z25.s\n" + "fmla z14.s, p3/M, z0.s, z25.s\n" + "fmla z23.s, p3/M, z6.s, z29.s\n" + "fmla z15.s, p3/M, z3.s, z29.s\n" + "ld1w { z25.s }, p2/Z, [x13, x16, LSL #2]\n" + "ldr x24, [x8, #0xb0]\n" + "fmla z22.s, p3/M, z4.s, z10.s\n" + "fmla z27.s, p3/M, z3.s, z10.s\n" "fmla z20.s, p3/M, z1.s, z10.s\n" - "fmla z21.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x9, x14, LSL #2]\n" - "fmin z18.s, p3/M, z18.s, z13.s\n" - "fmla z22.s, p3/M, z2.s, z11.s\n" - "fmla z23.s, p3/M, z1.s, z11.s\n" - "fmin z19.s, p3/M, z19.s, z13.s\n" - "fmax z20.s, p3/M, z20.s, z14.s\n" - "fmla z24.s, p3/M, z7.s, z12.s\n" - "fmla z25.s, p3/M, z6.s, z12.s\n" - "fmax z21.s, p3/M, z21.s, z14.s\n" - "fmax z22.s, p3/M, z22.s, z14.s\n" - "fmla z26.s, p3/M, z8.s, z10.s\n" - "fmla z27.s, p3/M, z7.s, z10.s\n" - "fmax z23.s, p3/M, z23.s, z14.s\n" - "st1w { z16.s }, p1, [x23, x13, LSL #2]\n" - "st1w { z17.s }, p1, [x22, x13, LSL #2]\n" - "ldr x23, [x28, #0x20]\n" - "ldr x22, [x28, #0x28]\n" + "fmla z9.s, p3/M, z5.s, z12.s\n" + "fmla z11.s, p3/M, z2.s, z12.s\n" + "fmla z24.s, p3/M, z0.s, z10.s\n" + "ld1w { z12.s }, p2/Z, [x22, x16, LSL #2]\n" + "ldr x23, [x8, #0xb8]\n" + "fmla z13.s, p3/M, z8.s, z25.s\n" + "fmla z28.s, p3/M, z5.s, z25.s\n" + "ld1w { z25.s }, p2/Z, [x21, x16, LSL #2]\n" + "ldr x22, [x8, #0xc0]\n" + "fmla z26.s, p3/M, z5.s, z10.s\n" + "fmla z14.s, p3/M, z2.s, z10.s\n" + "ld1w { z29.s }, p2/Z, [x28, x16, LSL #2]\n" + "ldr x21, [x8, #0xc8]\n" + "fmla z22.s, p3/M, z5.s, z12.s\n" + "fmla z27.s, p3/M, z4.s, z12.s\n" + "fmla z20.s, p3/M, z2.s, z12.s\n" + "fmla z9.s, p3/M, z3.s, z12.s\n" + "fmla z24.s, p3/M, z1.s, z12.s\n" + "fmla z11.s, p3/M, z0.s, z12.s\n" + "ld1w { z10.s }, p2/Z, [x20, x16, LSL #2]\n" + "ldr x28, [x8, #0xd8]\n" + "fmla z15.s, p3/M, z7.s, z25.s\n" + "fmla z18.s, p3/M, z6.s, z25.s\n" + "ld1w { z25.s }, p2/Z, [x27, x16, LSL #2]\n" + "ldr x20, [x8, #0xd0]\n" + "fmla z26.s, p3/M, z7.s, z29.s\n" + "fmla z22.s, p3/M, z6.s, z29.s\n" + "fmla z14.s, p3/M, z4.s, z29.s\n" + "fmla z20.s, p3/M, z3.s, z29.s\n" + "fmla z23.s, p3/M, z1.s, z29.s\n" + "fmla z30.s, p3/M, z0.s, z29.s\n" + "ld1w { z29.s }, p2/Z, [x26, x16, LSL #2]\n" + "ldr x27, [x8, #0xe0]\n" + "fmla z27.s, p3/M, z8.s, z10.s\n" + "fmla z21.s, p3/M, z8.s, z25.s\n" + "fmla z28.s, p3/M, z7.s, z25.s\n" + "ld1w { z25.s }, p2/Z, [x25, x16, LSL #2]\n" + "fmla z13.s, p3/M, z1.s, z10.s\n" + "ldr x26, [x8, #0xe8]\n" + "fmla z9.s, p3/M, z7.s, z10.s\n" + "fmla z24.s, p3/M, z5.s, z10.s\n" + "fmla z11.s, p3/M, z4.s, z10.s\n" + "fmla z31.s, p3/M, z2.s, z10.s\n" + "ld1w { z10.s }, p2/Z, [x24, x16, LSL #2]\n" + "ldr x25, [x8, #0xf0]\n" + "fmla z26.s, p3/M, z2.s, z29.s\n" + "fmla z22.s, p3/M, z1.s, z29.s\n" + "fmla z27.s, p3/M, z0.s, z29.s\n" + "fmla z14.s, p3/M, z7.s, z25.s\n" + "ld1w { z29.s }, p2/Z, [x23, x16, LSL #2]\n" + "ldr x24, [x8, #0xf8]\n" + "fmla z20.s, p3/M, z6.s, z25.s\n" + "fmla z23.s, p3/M, z4.s, z25.s\n" + "fmla z30.s, p3/M, z3.s, z25.s\n" + "fmla z15.s, p3/M, z1.s, z25.s\n" + "fmla z18.s, p3/M, z0.s, z25.s\n" + "ld1w { z25.s }, p2/Z, [x22, x16, LSL #2]\n" + "fmla z13.s, p3/M, z4.s, z25.s\n" + "ldr x23, [x8, #0x100]\n" + "fmla z21.s, p3/M, z2.s, z25.s\n" + "fmla z22.s, p3/M, z2.s, z10.s\n" + "fmla z27.s, p3/M, z1.s, z10.s\n" + "fmla z9.s, p3/M, z0.s, z10.s\n" + "ld1w { z12.s }, p2/Z, [x21, x16, LSL #2]\n" + "ldr x22, [x8, #0x108]\n" + "fmla z26.s, p3/M, z6.s, z29.s\n" + "fmla z14.s, p3/M, z3.s, z29.s\n" + "fmla z23.s, p3/M, z0.s, z29.s\n" + "fmla z24.s, p3/M, z8.s, z25.s\n" + "ld1w { z10.s }, p2/Z, [x20, x16, LSL #2]\n" + "ldr x21, [x8, #0x110]\n" + "fmla z11.s, p3/M, z7.s, z25.s\n" + "fmla z31.s, p3/M, z5.s, z25.s\n" + "fmla z28.s, p3/M, z1.s, z25.s\n" + "ld1w { z25.s }, p2/Z, [x28, x16, LSL #2]\n" + "fmla z13.s, p3/M, z2.s, z12.s\n" + "ldr x20, [x8, #0x118]\n" + "fmla z15.s, p3/M, z0.s, z10.s\n" + "fmla z18.s, p3/M, z4.s, z25.s\n" + "fmla z21.s, p3/M, z3.s, z25.s\n" + "fmla z9.s, p3/M, z8.s, z12.s\n" + "fmla z11.s, p3/M, z5.s, z12.s\n" + "fmla z14.s, p3/M, z6.s, z10.s\n" + "ld1w { z12.s }, p2/Z, [x27, x16, LSL #2]\n" + "fmla z23.s, p3/M, z3.s, z10.s\n" + "ld1w { z29.s }, p2/Z, [x26, x16, LSL #2]\n" + "fmla z30.s, p3/M, z7.s, z25.s\n" + "fmla z31.s, p3/M, z6.s, z25.s\n" + "fmla z15.s, p3/M, z5.s, z25.s\n" + "fmla z13.s, p3/M, z5.s, z12.s\n" + "fmla z28.s, p3/M, z2.s, z12.s\n" + "fmla z18.s, p3/M, z7.s, z29.s\n" + "fmla z21.s, p3/M, z6.s, z29.s\n" + "fmla z23.s, p3/M, z8.s, z25.s\n" + "ld1w { z25.s }, p2/Z, [x25, x16, LSL #2]\n" + "fmla z15.s, p3/M, z8.s, z29.s\n" + "ld1w { z29.s }, p2/Z, [x23, x16, LSL #2]\n" + "fmla z30.s, p3/M, z8.s, z25.s\n" + "fmla z31.s, p3/M, z7.s, z25.s\n" + "fmla z13.s, p3/M, z6.s, z25.s\n" + "fmla z18.s, p3/M, z5.s, z25.s\n" + "fmla z21.s, p3/M, z4.s, z25.s\n" + "fmla z28.s, p3/M, z3.s, z25.s\n" + "ld1w { z25.s }, p2/Z, [x22, x16, LSL #2]\n" + "ldp x27, x26, [x8, #0x0]\n" + "fmla z11.s, p3/M, z8.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x24, x16, LSL #2]\n" + "fmla z26.s, p3/M, z4.s, z29.s\n" + "fmax z26.s, p3/M, z26.s, z16.s\n" + "fmla z22.s, p3/M, z3.s, z29.s\n" + "fmla z27.s, p3/M, z5.s, z25.s\n" + "fmax z22.s, p3/M, z22.s, z16.s\n" + "fmax z27.s, p3/M, z27.s, z16.s\n" + "fmla z9.s, p3/M, z4.s, z25.s\n" + "fmla z18.s, p3/M, z8.s, z12.s\n" + "fmax z9.s, p3/M, z9.s, z16.s\n" + "fmin z26.s, p3/M, z26.s, z19.s\n" + "fmla z21.s, p3/M, z7.s, z12.s\n" + "fmla z28.s, p3/M, z6.s, z12.s\n" + "ld1w { z10.s }, p2/Z, [x21, x16, LSL #2]\n" + "fmin z22.s, p3/M, z22.s, z19.s\n" + "fmla z14.s, p3/M, z1.s, z29.s\n" + "fmla z20.s, p3/M, z0.s, z29.s\n" + "ld1w { z12.s }, p2/Z, [x20, x16, LSL #2]\n" + "fmin z27.s, p3/M, z27.s, z19.s\n" + "fmla z24.s, p3/M, z2.s, z25.s\n" + "fmla z11.s, p3/M, z1.s, z25.s\n" + "fmin z9.s, p3/M, z9.s, z19.s\n" + "fmax z14.s, p3/M, z14.s, z16.s\n" + "fmla z23.s, p3/M, z7.s, z10.s\n" + "fmla z30.s, p3/M, z6.s, z10.s\n" + "fmax z20.s, p3/M, z20.s, z16.s\n" + "fmax z24.s, p3/M, z24.s, z16.s\n" + "fmla z31.s, p3/M, z8.s, z12.s\n" + "fmla z13.s, p3/M, z7.s, z12.s\n" + "fmax z11.s, p3/M, z11.s, z16.s\n" + "st1w { z26.s }, p1, [x12, x15, LSL #2]\n" + "st1w { z22.s }, p1, [x11, x15, LSL #2]\n" + "ldr x23, [x14, #0x20]\n" + "ldr x22, [x14, #0x28]\n" + "fmla z15.s, p3/M, z4.s, z10.s\n" + "st1w { z27.s }, p1, [x10, x15, LSL #2]\n" + "ldr x21, [x14, #0x30]\n" + "fmla z18.s, p3/M, z3.s, z10.s\n" + "fmla z21.s, p3/M, z5.s, z12.s\n" + "st1w { z9.s }, p1, [x9, x15, LSL #2]\n" + "ldr x20, [x14, #0x38]\n" "fmla z28.s, p3/M, z4.s, z12.s\n" - "st1w { z18.s }, p1, [x21, x13, LSL #2]\n" - "ldr x21, [x28, #0x30]\n" - "fmla z29.s, p3/M, z3.s, z12.s\n" - "fmla z30.s, p3/M, z5.s, z10.s\n" - "st1w { z19.s }, p1, [x20, x13, LSL #2]\n" - "ldr x20, [x28, #0x38]\n" - "fmla z31.s, p3/M, z4.s, z10.s\n" - "ldp x10, x9, [x16, #0x10]\n" - "fmin z20.s, p3/M, z20.s, z13.s\n" - "fmin z21.s, p3/M, z21.s, z13.s\n" - "st1w { z20.s }, p1, [x23, x13, LSL #2]\n" - "ldr x23, [x28, #0x40]\n" - "fmin z22.s, p3/M, z22.s, z13.s\n" - "fmin z23.s, p3/M, z23.s, z13.s\n" - "st1w { z21.s }, p1, [x22, x13, LSL #2]\n" - "ldr x22, [x28, #0x48]\n" - "fmax z24.s, p3/M, z24.s, z14.s\n" - "fmax z25.s, p3/M, z25.s, z14.s\n" - "st1w { z22.s }, p1, [x21, x13, LSL #2]\n" - "ldr x21, [x28, #0x50]\n" - "fmax z26.s, p3/M, z26.s, z14.s\n" - "fmax z27.s, p3/M, z27.s, z14.s\n" - "st1w { z23.s }, p1, [x20, x13, LSL #2]\n" - "ldr x20, [x28, #0x58]\n" - "incw x14\n" - "ld1w { z9.s }, p0/Z, [x12, x15, LSL #2]\n" - "ld1w { z10.s }, p0/Z, [x11, x15, LSL #2]\n" - "fmin z24.s, p3/M, z24.s, z13.s\n" - "ld1w { z11.s }, p0/Z, [x10, x15, LSL #2]\n" - "ld1w { z12.s }, p0/Z, [x9, x15, LSL #2]\n" - "incw x15\n" - "fmin z25.s, p3/M, z25.s, z13.s\n" - "fmin z26.s, p3/M, z26.s, z13.s\n" - "fmin z27.s, p3/M, z27.s, z13.s\n" - "st1w { z24.s }, p1, [x23, x13, LSL #2]\n" - "ldr x23, [x28, #0x60]\n" - "fmax z28.s, p3/M, z28.s, z14.s\n" - "fmax z29.s, p3/M, z29.s, z14.s\n" - "st1w { z25.s }, p1, [x22, x13, LSL #2]\n" - "ldr x22, [x28, #0x68]\n" - "fmax z30.s, p3/M, z30.s, z14.s\n" - "fmax z31.s, p3/M, z31.s, z14.s\n" - "st1w { z26.s }, p1, [x21, x13, LSL #2]\n" - "ldr x21, [x28, #0x70]\n" - "st1w { z27.s }, p1, [x20, x13, LSL #2]\n" - "ldr x20, [x28, #0x78]\n" - "ld1w { z15.s }, p3/Z, [x17]\n" - "whilelt p2.s, x14, %x[n_channels]\n" - "ld1w { z0.s }, p3/Z, [x17, #1, MUL VL]\n" - "ld1w { z1.s }, p3/Z, [x17, #2, MUL VL]\n" - "cmp x15, %x[n_channels]\n" - "fmin z28.s, p3/M, z28.s, z13.s\n" - "ld1w { z2.s }, p3/Z, [x17, #3, MUL VL]\n" - "ld1w { z3.s }, p3/Z, [x17, #4, MUL VL]\n" - "fmin z29.s, p3/M, z29.s, z13.s\n" - "fmin z30.s, p3/M, z30.s, z13.s\n" - "ld1w { z4.s }, p3/Z, [x17, #5, MUL VL]\n" - "ld1w { z5.s }, p3/Z, [x17, #6, MUL VL]\n" - "fmin z31.s, p3/M, z31.s, z13.s\n" - "st1w { z28.s }, p1, [x23, x13, LSL #2]\n" - "ld1w { z6.s }, p3/Z, [x17, #7, MUL VL]\n" - "addvl x17, x17, #16\n" - "st1w { z29.s }, p1, [x22, x13, LSL #2]\n" - "ld1w { z7.s }, p3/Z, [x17, #-8, MUL VL]\n" - "st1w { z30.s }, p1, [x21, x13, LSL #2]\n" - "ld1w { z8.s }, p3/Z, [x17, #-7, MUL VL]\n" - "addvl x17, x17, #-6\n" - "st1w { z31.s }, p1, [x20, x13, LSL #2]\n" + "ldp x25, x24, [x8, #0x10]\n" + "fmin z14.s, p3/M, z14.s, z19.s\n" + "fmin z20.s, p3/M, z20.s, z19.s\n" + "st1w { z14.s }, p1, [x23, x15, LSL #2]\n" + "ldr x23, [x14, #0x40]\n" + "fmin z24.s, p3/M, z24.s, z19.s\n" + "fmin z11.s, p3/M, z11.s, z19.s\n" + "st1w { z20.s }, p1, [x22, x15, LSL #2]\n" + "ldr x22, [x14, #0x48]\n" + "fmax z23.s, p3/M, z23.s, z16.s\n" + "fmax z30.s, p3/M, z30.s, z16.s\n" + "st1w { z24.s }, p1, [x21, x15, LSL #2]\n" + "ldr x21, [x14, #0x50]\n" + "fmax z31.s, p3/M, z31.s, z16.s\n" + "fmax z13.s, p3/M, z13.s, z16.s\n" + "st1w { z11.s }, p1, [x20, x15, LSL #2]\n" + "ldr x20, [x14, #0x58]\n" + "incw x16\n" + "ld1w { z9.s }, p0/Z, [x27, x17, LSL #2]\n" + "ld1w { z10.s }, p0/Z, [x26, x17, LSL #2]\n" + "fmin z23.s, p3/M, z23.s, z19.s\n" + "ld1w { z11.s }, p0/Z, [x25, x17, LSL #2]\n" + "ld1w { z12.s }, p0/Z, [x24, x17, LSL #2]\n" + "incw x17\n" + "fmin z30.s, p3/M, z30.s, z19.s\n" + "fmin z31.s, p3/M, z31.s, z19.s\n" + "fmin z13.s, p3/M, z13.s, z19.s\n" + "st1w { z23.s }, p1, [x23, x15, LSL #2]\n" + "ldr x23, [x14, #0x60]\n" + "fmax z15.s, p3/M, z15.s, z16.s\n" + "fmax z18.s, p3/M, z18.s, z16.s\n" + "st1w { z30.s }, p1, [x22, x15, LSL #2]\n" + "ldr x22, [x14, #0x68]\n" + "fmax z21.s, p3/M, z21.s, z16.s\n" + "fmax z28.s, p3/M, z28.s, z16.s\n" + "st1w { z31.s }, p1, [x21, x15, LSL #2]\n" + "ldr x21, [x14, #0x70]\n" + "st1w { z13.s }, p1, [x20, x15, LSL #2]\n" + "ldr x20, [x14, #0x78]\n" + "ld1w { z0.s }, p3/Z, [x7, #1, MUL VL]\n" + "whilelt p2.s, x16, %x[n_channels]\n" + "ld1w { z1.s }, p3/Z, [x7, #2, MUL VL]\n" + "ld1w { z2.s }, p3/Z, [x7, #3, MUL VL]\n" + "cmp x17, %x[n_channels]\n" + "fmin z15.s, p3/M, z15.s, z19.s\n" + "ld1w { z3.s }, p3/Z, [x7, #4, MUL VL]\n" + "ld1w { z4.s }, p3/Z, [x7, #5, MUL VL]\n" + "fmin z18.s, p3/M, z18.s, z19.s\n" + "fmin z21.s, p3/M, z21.s, z19.s\n" + "ld1w { z5.s }, p3/Z, [x7, #6, MUL VL]\n" + "ld1w { z6.s }, p3/Z, [x7, #7, MUL VL]\n" + "addvl x7, x7, #16\n" + "fmin z28.s, p3/M, z28.s, z19.s\n" + "st1w { z15.s }, p1, [x23, x15, LSL #2]\n" + "ld1w { z7.s }, p3/Z, [x7, #-8, MUL VL]\n" + "ld1w { z8.s }, p3/Z, [x7, #-7, MUL VL]\n" + "addvl x7, x7, #-6\n" + "st1w { z18.s }, p1, [x22, x15, LSL #2]\n" + "st1w { z21.s }, p1, [x21, x15, LSL #2]\n" + "st1w { z28.s }, p1, [x20, x15, LSL #2]\n" "blt 1b\n" "2:" // Channel tail - "movprfx z21, z15\n fmla z21.s, p3/M, z4.s, z9.s\n" - "movprfx z16, z15\n fmla z16.s, p3/M, z8.s, z9.s\n" - "ldr x27, [x16, #0x20]\n" - "ldr x26, [x16, #0x30]\n" - "movprfx z22, z15\n fmla z22.s, p3/M, z3.s, z9.s\n" - "movprfx z25, z15\n fmla z25.s, p3/M, z1.s, z9.s\n" - "ldr x25, [x16, #0x28]\n" - "ldr x24, [x16, #0x38]\n" - "movprfx z26, z15\n fmla z26.s, p3/M, z0.s, z9.s\n" - "movprfx z17, z15\n fmla z17.s, p3/M, z7.s, z9.s\n" - "ldr x12, [x16, #0x40]\n" - "ldr x11, [x16, #0x48]\n" - "movprfx z18, z15\n fmla z18.s, p3/M, z6.s, z9.s\n" - "fmla z21.s, p3/M, z5.s, z12.s\n" - "ldr x10, [x16, #0x50]\n" - "ldr x9, [x16, #0x58]\n" - "movprfx z20, z15\n fmla z20.s, p3/M, z5.s, z9.s\n" - "movprfx z24, z15\n fmla z24.s, p3/M, z2.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x26, x14, LSL #2]\n" - "ldr x26, [x16, #0x70]\n" - "fmla z16.s, p3/M, z0.s, z10.s\n" - "movprfx z19, z15\n fmla z19.s, p3/M, z2.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x27, x14, LSL #2]\n" - "ld1w { z11.s }, p2/Z, [x25, x14, LSL #2]\n" - "fmla z22.s, p3/M, z4.s, z12.s\n" - "fmla z25.s, p3/M, z2.s, z12.s\n" - "ldr x27, [x16, #0x60]\n" - "ldr x25, [x16, #0x68]\n" - "fmla z26.s, p3/M, z1.s, z12.s\n" - "fmla z17.s, p3/M, z8.s, z12.s\n" - "incw x13\n" - "mov p1.b, p2.b\n" - "fmla z18.s, p3/M, z7.s, z12.s\n" - "movprfx z28, z15\n fmla z28.s, p3/M, z6.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x11, x14, LSL #2]\n" - "ldr x11, [x16, #0x88]\n" - "fmla z21.s, p3/M, z7.s, z9.s\n" - "fmla z19.s, p3/M, z6.s, z12.s\n" - "ldr x23, [x28, #0x0]\n" - "ldr x22, [x28, #0x8]\n" - "movprfx z23, z15\n fmla z23.s, p3/M, z3.s, z12.s\n" - "movprfx z27, z15\n fmla z27.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x24, x14, LSL #2]\n" - "ldr x24, [x16, #0x78]\n" - "movprfx z31, z15\n fmla z31.s, p3/M, z8.s, z11.s\n" - "fmla z22.s, p3/M, z6.s, z9.s\n" - "ld1w { z11.s }, p2/Z, [x12, x14, LSL #2]\n" - "ldr x12, [x16, #0x80]\n" - "fmla z25.s, p3/M, z4.s, z9.s\n" - "fmla z26.s, p3/M, z3.s, z9.s\n" - "ldr x21, [x28, #0x10]\n" - "ldr x20, [x28, #0x18]\n" - "fmla z20.s, p3/M, z8.s, z9.s\n" - "fmla z24.s, p3/M, z5.s, z9.s\n" - "fmla z28.s, p3/M, z2.s, z9.s\n" - "fmla z16.s, p3/M, z1.s, z12.s\n" - "fmla z17.s, p3/M, z0.s, z12.s\n" - "movprfx z29, z15\n fmla z29.s, p3/M, z1.s, z9.s\n" - "movprfx z30, z15\n fmla z30.s, p3/M, z0.s, z9.s\n" - "fmla z18.s, p3/M, z2.s, z11.s\n" - "ld1w { z9.s }, p2/Z, [x10, x14, LSL #2]\n" - "ldr x10, [x16, #0x90]\n" - "fmla z21.s, p3/M, z8.s, z10.s\n" - "fmla z19.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x27, x14, LSL #2]\n" - "ldr x27, [x16, #0xa0]\n" - "fmla z22.s, p3/M, z7.s, z10.s\n" - "fmla z23.s, p3/M, z6.s, z10.s\n" - "fmla z25.s, p3/M, z5.s, z10.s\n" - "fmla z26.s, p3/M, z4.s, z10.s\n" - "fmla z27.s, p3/M, z3.s, z10.s\n" - "fmla z29.s, p3/M, z2.s, z10.s\n" - "fmla z30.s, p3/M, z1.s, z10.s\n" - "fmla z31.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x25, x14, LSL #2]\n" - "ldr x25, [x16, #0xa8]\n" - "fmla z16.s, p3/M, z3.s, z9.s\n" - "fmla z20.s, p3/M, z0.s, z9.s\n" - "ld1w { z12.s }, p2/Z, [x9, x14, LSL #2]\n" - "ldr x9, [x16, #0x98]\n" - "fmla z24.s, p3/M, z6.s, z11.s\n" - "fmla z28.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x14, LSL #2]\n" - "ldr x26, [x16, #0xb0]\n" - "fmla z17.s, p3/M, z4.s, z10.s\n" - "fmla z18.s, p3/M, z3.s, z10.s\n" - "fmla z21.s, p3/M, z1.s, z10.s\n" - "fmla z19.s, p3/M, z5.s, z12.s\n" - "fmla z23.s, p3/M, z2.s, z12.s\n" - "fmla z22.s, p3/M, z0.s, z10.s\n" - "ld1w { z12.s }, p2/Z, [x24, x14, LSL #2]\n" - "ldr x24, [x16, #0xb8]\n" - "fmla z27.s, p3/M, z8.s, z11.s\n" - "fmla z31.s, p3/M, z5.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x12, x14, LSL #2]\n" - "ldr x12, [x16, #0xc0]\n" - "fmla z16.s, p3/M, z5.s, z10.s\n" - "fmla z20.s, p3/M, z2.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x11, x14, LSL #2]\n" - "ldr x11, [x16, #0xc8]\n" - "fmla z17.s, p3/M, z5.s, z12.s\n" - "fmla z18.s, p3/M, z4.s, z12.s\n" - "fmla z21.s, p3/M, z2.s, z12.s\n" - "fmla z19.s, p3/M, z3.s, z12.s\n" - "fmla z22.s, p3/M, z1.s, z12.s\n" - "fmla z23.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x9, x14, LSL #2]\n" - "ldr x9, [x16, #0xd8]\n" - "fmla z28.s, p3/M, z7.s, z11.s\n" - "fmla z29.s, p3/M, z6.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x10, x14, LSL #2]\n" - "ldr x10, [x16, #0xd0]\n" - "fmla z16.s, p3/M, z7.s, z10.s\n" - "fmla z17.s, p3/M, z6.s, z10.s\n" - "fmla z20.s, p3/M, z4.s, z10.s\n" - "fmla z21.s, p3/M, z3.s, z10.s\n" - "fmla z24.s, p3/M, z1.s, z10.s\n" - "fmla z25.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x27, x14, LSL #2]\n" - "ldr x27, [x16, #0xe0]\n" - "fmla z18.s, p3/M, z8.s, z12.s\n" - "fmla z30.s, p3/M, z8.s, z11.s\n" - "fmla z31.s, p3/M, z7.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x25, x14, LSL #2]\n" - "fmla z27.s, p3/M, z1.s, z12.s\n" - "ldr x25, [x16, #0xe8]\n" - "fmla z19.s, p3/M, z7.s, z12.s\n" - "fmla z22.s, p3/M, z5.s, z12.s\n" - "fmla z23.s, p3/M, z4.s, z12.s\n" - "fmla z26.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x26, x14, LSL #2]\n" - "ldr x26, [x16, #0xf0]\n" - "fmla z16.s, p3/M, z2.s, z10.s\n" - "fmla z17.s, p3/M, z1.s, z10.s\n" + "movprfx z14, z17\n fmla z14.s, p3/M, z4.s, z9.s\n" + "movprfx z18, z17\n fmla z18.s, p3/M, z8.s, z9.s\n" + "ldr x27, [x8, #0x20]\n" + "ldr x24, [x8, #0x30]\n" + "movprfx z15, z17\n fmla z15.s, p3/M, z3.s, z9.s\n" + "movprfx z30, z17\n fmla z30.s, p3/M, z1.s, z9.s\n" + "ldr x23, [x8, #0x28]\n" + "ldr x22, [x8, #0x38]\n" + "movprfx z20, z17\n fmla z20.s, p3/M, z0.s, z9.s\n" + "movprfx z13, z17\n fmla z13.s, p3/M, z7.s, z9.s\n" + "ldr x26, [x8, #0x40]\n" + "ldr x21, [x8, #0x48]\n" + "movprfx z22, z17\n fmla z22.s, p3/M, z6.s, z9.s\n" + "fmla z14.s, p3/M, z5.s, z12.s\n" + "ldr x25, [x8, #0x50]\n" + "ldr x20, [x8, #0x58]\n" + "movprfx z27, z17\n fmla z27.s, p3/M, z5.s, z9.s\n" + "movprfx z31, z17\n fmla z31.s, p3/M, z2.s, z9.s\n" + "ld1w { z23.s }, p2/Z, [x24, x16, LSL #2]\n" + "ldr x13, [x8, #0x70]\n" "fmla z18.s, p3/M, z0.s, z10.s\n" - "fmla z20.s, p3/M, z7.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x24, x14, LSL #2]\n" - "ldr x24, [x16, #0xf8]\n" - "fmla z21.s, p3/M, z6.s, z11.s\n" - "fmla z24.s, p3/M, z4.s, z11.s\n" - "fmla z25.s, p3/M, z3.s, z11.s\n" - "fmla z28.s, p3/M, z1.s, z11.s\n" - "fmla z29.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x12, x14, LSL #2]\n" - "fmla z27.s, p3/M, z4.s, z11.s\n" - "ldr x12, [x16, #0x100]\n" - "fmla z30.s, p3/M, z2.s, z11.s\n" - "fmla z17.s, p3/M, z2.s, z12.s\n" + "movprfx z9, z17\n fmla z9.s, p3/M, z2.s, z11.s\n" + "ld1w { z21.s }, p2/Z, [x27, x16, LSL #2]\n" + "ld1w { z25.s }, p2/Z, [x23, x16, LSL #2]\n" + "fmla z15.s, p3/M, z4.s, z12.s\n" + "fmla z30.s, p3/M, z2.s, z12.s\n" + "ldr x24, [x8, #0x60]\n" + "ldr x23, [x8, #0x68]\n" + "fmla z20.s, p3/M, z1.s, z12.s\n" + "fmla z13.s, p3/M, z8.s, z12.s\n" + "incw x15\n" + "mov p0.b, p2.b\n" + "fmla z22.s, p3/M, z7.s, z12.s\n" + "movprfx z28, z17\n fmla z28.s, p3/M, z6.s, z21.s\n" + "ld1w { z29.s }, p2/Z, [x21, x16, LSL #2]\n" + "ldr x28, [x8, #0x88]\n" + "fmla z14.s, p3/M, z7.s, z23.s\n" + "fmla z9.s, p3/M, z6.s, z12.s\n" + "ldr x12, [x14, #0x0]\n" + "ldr x11, [x14, #0x8]\n" + "movprfx z11, z17\n fmla z11.s, p3/M, z3.s, z12.s\n" + "movprfx z10, z17\n fmla z10.s, p3/M, z0.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x22, x16, LSL #2]\n" + "ldr x22, [x8, #0x78]\n" + "movprfx z26, z17\n fmla z26.s, p3/M, z8.s, z25.s\n" + "fmla z15.s, p3/M, z6.s, z23.s\n" + "ld1w { z21.s }, p2/Z, [x26, x16, LSL #2]\n" + "ldr x21, [x8, #0x80]\n" + "fmla z30.s, p3/M, z4.s, z23.s\n" + "fmla z20.s, p3/M, z3.s, z23.s\n" + "ldr x10, [x14, #0x10]\n" + "ldr x9, [x14, #0x18]\n" + "movprfx z25, z17\n fmla z25.s, p3/M, z1.s, z23.s\n" + "movprfx z24, z17\n fmla z24.s, p3/M, z0.s, z23.s\n" + "fmla z27.s, p3/M, z8.s, z23.s\n" + "fmla z31.s, p3/M, z5.s, z23.s\n" + "fmla z28.s, p3/M, z2.s, z23.s\n" "fmla z18.s, p3/M, z1.s, z12.s\n" - "fmla z19.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11, x14, LSL #2]\n" - "ldr x11, [x16, #0x108]\n" - "fmla z16.s, p3/M, z6.s, z10.s\n" - "fmla z20.s, p3/M, z3.s, z10.s\n" - "fmla z24.s, p3/M, z0.s, z10.s\n" - "fmla z22.s, p3/M, z8.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x10, x14, LSL #2]\n" - "ldr x10, [x16, #0x110]\n" - "fmla z23.s, p3/M, z7.s, z11.s\n" - "fmla z26.s, p3/M, z5.s, z11.s\n" - "fmla z31.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x9, x14, LSL #2]\n" - "fmla z27.s, p3/M, z2.s, z12.s\n" - "ldr x9, [x16, #0x118]\n" - "fmla z28.s, p3/M, z0.s, z10.s\n" - "fmla z29.s, p3/M, z4.s, z11.s\n" - "fmla z30.s, p3/M, z3.s, z11.s\n" - "fmla z19.s, p3/M, z8.s, z12.s\n" - "fmla z23.s, p3/M, z5.s, z12.s\n" - "fmla z20.s, p3/M, z6.s, z10.s\n" - "ld1w { z12.s }, p2/Z, [x27, x14, LSL #2]\n" - "fmla z24.s, p3/M, z3.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x25, x14, LSL #2]\n" - "fmla z25.s, p3/M, z7.s, z11.s\n" - "fmla z26.s, p3/M, z6.s, z11.s\n" - "fmla z28.s, p3/M, z5.s, z11.s\n" - "fmla z27.s, p3/M, z5.s, z12.s\n" - "fmla z31.s, p3/M, z2.s, z12.s\n" - "fmla z29.s, p3/M, z7.s, z10.s\n" - "fmla z30.s, p3/M, z6.s, z10.s\n" - "fmla z24.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x14, LSL #2]\n" - "fmla z28.s, p3/M, z8.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x12, x14, LSL #2]\n" - "fmla z25.s, p3/M, z8.s, z11.s\n" - "fmla z26.s, p3/M, z7.s, z11.s\n" - "fmla z27.s, p3/M, z6.s, z11.s\n" - "fmla z29.s, p3/M, z5.s, z11.s\n" - "fmla z30.s, p3/M, z4.s, z11.s\n" - "fmla z31.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x11, x14, LSL #2]\n" - "fmla z23.s, p3/M, z8.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x24, x14, LSL #2]\n" - "fmla z16.s, p3/M, z4.s, z10.s\n" - "fmax z16.s, p3/M, z16.s, z14.s\n" - "fmla z17.s, p3/M, z3.s, z10.s\n" - "fmla z18.s, p3/M, z5.s, z11.s\n" - "fmax z17.s, p3/M, z17.s, z14.s\n" - "fmax z18.s, p3/M, z18.s, z14.s\n" - "fmla z19.s, p3/M, z4.s, z11.s\n" - "fmla z29.s, p3/M, z8.s, z12.s\n" - "fmax z19.s, p3/M, z19.s, z14.s\n" - "fmin z16.s, p3/M, z16.s, z13.s\n" - "fmla z30.s, p3/M, z7.s, z12.s\n" - "fmla z31.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x10, x14, LSL #2]\n" - "fmin z17.s, p3/M, z17.s, z13.s\n" - "fmla z20.s, p3/M, z1.s, z10.s\n" - "fmla z21.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x9, x14, LSL #2]\n" - "fmin z18.s, p3/M, z18.s, z13.s\n" - "fmla z22.s, p3/M, z2.s, z11.s\n" - "fmla z23.s, p3/M, z1.s, z11.s\n" - "fmin z19.s, p3/M, z19.s, z13.s\n" - "fmax z20.s, p3/M, z20.s, z14.s\n" - "fmla z24.s, p3/M, z7.s, z12.s\n" - "fmla z25.s, p3/M, z6.s, z12.s\n" - "fmax z21.s, p3/M, z21.s, z14.s\n" - "fmax z22.s, p3/M, z22.s, z14.s\n" - "fmla z26.s, p3/M, z8.s, z10.s\n" - "fmla z27.s, p3/M, z7.s, z10.s\n" - "fmax z23.s, p3/M, z23.s, z14.s\n" - "st1w { z16.s }, p1, [x23, x13, LSL #2]\n" - "st1w { z17.s }, p1, [x22, x13, LSL #2]\n" - "ldr x23, [x28, #0x20]\n" - "ldr x22, [x28, #0x28]\n" - "fmla z28.s, p3/M, z4.s, z12.s\n" - "st1w { z18.s }, p1, [x21, x13, LSL #2]\n" - "ldr x21, [x28, #0x30]\n" - "fmla z29.s, p3/M, z3.s, z12.s\n" - "fmla z30.s, p3/M, z5.s, z10.s\n" - "st1w { z19.s }, p1, [x20, x13, LSL #2]\n" - "ldr x20, [x28, #0x38]\n" - "fmla z31.s, p3/M, z4.s, z10.s\n" - "fmin z20.s, p3/M, z20.s, z13.s\n" - "fmin z21.s, p3/M, z21.s, z13.s\n" - "fmin z22.s, p3/M, z22.s, z13.s\n" - "st1w { z20.s }, p1, [x23, x13, LSL #2]\n" - "ldr x23, [x28, #0x40]\n" - "fmin z23.s, p3/M, z23.s, z13.s\n" - "fmax z24.s, p3/M, z24.s, z14.s\n" - "st1w { z21.s }, p1, [x22, x13, LSL #2]\n" - "ldr x22, [x28, #0x48]\n" - "fmax z25.s, p3/M, z25.s, z14.s\n" - "fmax z26.s, p3/M, z26.s, z14.s\n" - "st1w { z22.s }, p1, [x21, x13, LSL #2]\n" - "ldr x21, [x28, #0x50]\n" - "fmax z27.s, p3/M, z27.s, z14.s\n" - "st1w { z23.s }, p1, [x20, x13, LSL #2]\n" - "ldr x20, [x28, #0x58]\n" - "fmin z24.s, p3/M, z24.s, z13.s\n" - "fmin z25.s, p3/M, z25.s, z13.s\n" - "fmin z26.s, p3/M, z26.s, z13.s\n" - "st1w { z24.s }, p1, [x23, x13, LSL #2]\n" - "ldr x23, [x28, #0x60]\n" - "fmin z27.s, p3/M, z27.s, z13.s\n" - "fmax z28.s, p3/M, z28.s, z14.s\n" - "st1w { z25.s }, p1, [x22, x13, LSL #2]\n" - "ldr x22, [x28, #0x68]\n" - "fmax z29.s, p3/M, z29.s, z14.s\n" - "fmax z30.s, p3/M, z30.s, z14.s\n" - "st1w { z26.s }, p1, [x21, x13, LSL #2]\n" - "ldr x21, [x28, #0x70]\n" - "fmax z31.s, p3/M, z31.s, z14.s\n" - "st1w { z27.s }, p1, [x20, x13, LSL #2]\n" - "ldr x20, [x28, #0x78]\n" - "fmin z28.s, p3/M, z28.s, z13.s\n" - "fmin z29.s, p3/M, z29.s, z13.s\n" - "fmin z30.s, p3/M, z30.s, z13.s\n" - "st1w { z28.s }, p1, [x23, x13, LSL #2]\n" - "fmin z31.s, p3/M, z31.s, z13.s\n" - "st1w { z29.s }, p1, [x22, x13, LSL #2]\n" - "st1w { z30.s }, p1, [x21, x13, LSL #2]\n" - "st1w { z31.s }, p1, [x20, x13, LSL #2]\n" + "ld1w { z23.s }, p2/Z, [x25, x16, LSL #2]\n" + "ldr x27, [x8, #0x90]\n" + "fmla z13.s, p3/M, z0.s, z12.s\n" + "fmla z22.s, p3/M, z2.s, z21.s\n" + "ld1w { z12.s }, p2/Z, [x20, x16, LSL #2]\n" + "ldr x20, [x8, #0x98]\n" + "fmla z14.s, p3/M, z8.s, z29.s\n" + "fmla z9.s, p3/M, z1.s, z21.s\n" + "ld1w { z21.s }, p2/Z, [x24, x16, LSL #2]\n" + "ldr x26, [x8, #0xa0]\n" + "fmla z15.s, p3/M, z7.s, z29.s\n" + "fmla z11.s, p3/M, z6.s, z29.s\n" + "fmla z30.s, p3/M, z5.s, z29.s\n" + "fmla z20.s, p3/M, z4.s, z29.s\n" + "fmla z10.s, p3/M, z3.s, z29.s\n" + "fmla z25.s, p3/M, z2.s, z29.s\n" + "fmla z24.s, p3/M, z1.s, z29.s\n" + "fmla z26.s, p3/M, z0.s, z29.s\n" + "ld1w { z29.s }, p2/Z, [x23, x16, LSL #2]\n" + "ldr x25, [x8, #0xa8]\n" + "fmla z18.s, p3/M, z3.s, z23.s\n" + "fmla z27.s, p3/M, z0.s, z23.s\n" + "fmla z31.s, p3/M, z6.s, z21.s\n" + "fmla z28.s, p3/M, z3.s, z21.s\n" + "ld1w { z21.s }, p2/Z, [x13, x16, LSL #2]\n" + "ldr x24, [x8, #0xb0]\n" + "fmla z13.s, p3/M, z4.s, z29.s\n" + "fmla z22.s, p3/M, z3.s, z29.s\n" + "fmla z14.s, p3/M, z1.s, z29.s\n" + "fmla z9.s, p3/M, z5.s, z12.s\n" + "fmla z11.s, p3/M, z2.s, z12.s\n" + "fmla z15.s, p3/M, z0.s, z29.s\n" + "ld1w { z17.s }, p2/Z, [x22, x16, LSL #2]\n" + "ldr x23, [x8, #0xb8]\n" + "fmla z10.s, p3/M, z8.s, z21.s\n" + "fmla z26.s, p3/M, z5.s, z21.s\n" + "ld1w { z23.s }, p2/Z, [x21, x16, LSL #2]\n" + "ldr x22, [x8, #0xc0]\n" + "fmla z18.s, p3/M, z5.s, z29.s\n" + "fmla z27.s, p3/M, z2.s, z29.s\n" + "ld1w { z21.s }, p2/Z, [x28, x16, LSL #2]\n" + "ldr x21, [x8, #0xc8]\n" + "fmla z13.s, p3/M, z5.s, z17.s\n" + "fmla z22.s, p3/M, z4.s, z17.s\n" + "fmla z14.s, p3/M, z2.s, z17.s\n" + "fmla z9.s, p3/M, z3.s, z17.s\n" + "fmla z15.s, p3/M, z1.s, z17.s\n" + "fmla z11.s, p3/M, z0.s, z17.s\n" + "ld1w { z29.s }, p2/Z, [x20, x16, LSL #2]\n" + "ldr x28, [x8, #0xd8]\n" + "fmla z28.s, p3/M, z7.s, z23.s\n" + "fmla z25.s, p3/M, z6.s, z23.s\n" + "ld1w { z23.s }, p2/Z, [x27, x16, LSL #2]\n" + "ldr x20, [x8, #0xd0]\n" + "fmla z18.s, p3/M, z7.s, z21.s\n" + "fmla z13.s, p3/M, z6.s, z21.s\n" + "fmla z27.s, p3/M, z4.s, z21.s\n" + "fmla z14.s, p3/M, z3.s, z21.s\n" + "fmla z31.s, p3/M, z1.s, z21.s\n" + "fmla z30.s, p3/M, z0.s, z21.s\n" + "ld1w { z21.s }, p2/Z, [x26, x16, LSL #2]\n" + "ldr x27, [x8, #0xe0]\n" + "fmla z22.s, p3/M, z8.s, z29.s\n" + "fmla z24.s, p3/M, z8.s, z23.s\n" + "fmla z26.s, p3/M, z7.s, z23.s\n" + "ld1w { z23.s }, p2/Z, [x25, x16, LSL #2]\n" + "fmla z10.s, p3/M, z1.s, z29.s\n" + "ldr x26, [x8, #0xe8]\n" + "fmla z9.s, p3/M, z7.s, z29.s\n" + "fmla z15.s, p3/M, z5.s, z29.s\n" + "fmla z11.s, p3/M, z4.s, z29.s\n" + "fmla z20.s, p3/M, z2.s, z29.s\n" + "ld1w { z29.s }, p2/Z, [x24, x16, LSL #2]\n" + "ldr x25, [x8, #0xf0]\n" + "fmla z18.s, p3/M, z2.s, z21.s\n" + "fmla z13.s, p3/M, z1.s, z21.s\n" + "fmla z22.s, p3/M, z0.s, z21.s\n" + "fmla z27.s, p3/M, z7.s, z23.s\n" + "ld1w { z21.s }, p2/Z, [x23, x16, LSL #2]\n" + "ldr x24, [x8, #0xf8]\n" + "fmla z14.s, p3/M, z6.s, z23.s\n" + "fmla z31.s, p3/M, z4.s, z23.s\n" + "fmla z30.s, p3/M, z3.s, z23.s\n" + "fmla z28.s, p3/M, z1.s, z23.s\n" + "fmla z25.s, p3/M, z0.s, z23.s\n" + "ld1w { z17.s }, p2/Z, [x22, x16, LSL #2]\n" + "fmla z10.s, p3/M, z4.s, z17.s\n" + "ldr x23, [x8, #0x100]\n" + "fmla z24.s, p3/M, z2.s, z17.s\n" + "fmla z13.s, p3/M, z2.s, z29.s\n" + "fmla z22.s, p3/M, z1.s, z29.s\n" + "fmla z9.s, p3/M, z0.s, z29.s\n" + "ld1w { z23.s }, p2/Z, [x21, x16, LSL #2]\n" + "ldr x22, [x8, #0x108]\n" + "fmla z18.s, p3/M, z6.s, z21.s\n" + "fmla z27.s, p3/M, z3.s, z21.s\n" + "fmla z31.s, p3/M, z0.s, z21.s\n" + "fmla z15.s, p3/M, z8.s, z17.s\n" + "ld1w { z29.s }, p2/Z, [x20, x16, LSL #2]\n" + "ldr x21, [x8, #0x110]\n" + "fmla z11.s, p3/M, z7.s, z17.s\n" + "fmla z20.s, p3/M, z5.s, z17.s\n" + "fmla z26.s, p3/M, z1.s, z17.s\n" + "ld1w { z21.s }, p2/Z, [x28, x16, LSL #2]\n" + "fmla z10.s, p3/M, z2.s, z23.s\n" + "ldr x20, [x8, #0x118]\n" + "fmla z28.s, p3/M, z0.s, z29.s\n" + "fmla z25.s, p3/M, z4.s, z21.s\n" + "fmla z24.s, p3/M, z3.s, z21.s\n" + "fmla z9.s, p3/M, z8.s, z23.s\n" + "fmla z11.s, p3/M, z5.s, z23.s\n" + "fmla z27.s, p3/M, z6.s, z29.s\n" + "ld1w { z23.s }, p2/Z, [x27, x16, LSL #2]\n" + "fmla z31.s, p3/M, z3.s, z29.s\n" + "ld1w { z17.s }, p2/Z, [x26, x16, LSL #2]\n" + "fmla z30.s, p3/M, z7.s, z21.s\n" + "fmla z20.s, p3/M, z6.s, z21.s\n" + "fmla z28.s, p3/M, z5.s, z21.s\n" + "fmla z10.s, p3/M, z5.s, z23.s\n" + "fmla z26.s, p3/M, z2.s, z23.s\n" + "fmla z25.s, p3/M, z7.s, z17.s\n" + "fmla z24.s, p3/M, z6.s, z17.s\n" + "fmla z31.s, p3/M, z8.s, z21.s\n" + "ld1w { z21.s }, p2/Z, [x25, x16, LSL #2]\n" + "fmla z28.s, p3/M, z8.s, z17.s\n" + "ld1w { z12.s }, p2/Z, [x23, x16, LSL #2]\n" + "fmla z30.s, p3/M, z8.s, z21.s\n" + "fmla z20.s, p3/M, z7.s, z21.s\n" + "fmla z10.s, p3/M, z6.s, z21.s\n" + "fmla z25.s, p3/M, z5.s, z21.s\n" + "fmla z24.s, p3/M, z4.s, z21.s\n" + "fmla z26.s, p3/M, z3.s, z21.s\n" + "ld1w { z21.s }, p2/Z, [x22, x16, LSL #2]\n" + "fmla z11.s, p3/M, z8.s, z23.s\n" + "ld1w { z29.s }, p2/Z, [x24, x16, LSL #2]\n" + "fmla z18.s, p3/M, z4.s, z12.s\n" + "fmax z18.s, p3/M, z18.s, z16.s\n" + "fmla z13.s, p3/M, z3.s, z12.s\n" + "fmla z22.s, p3/M, z5.s, z21.s\n" + "fmax z13.s, p3/M, z13.s, z16.s\n" + "fmax z22.s, p3/M, z22.s, z16.s\n" + "fmla z9.s, p3/M, z4.s, z21.s\n" + "fmla z25.s, p3/M, z8.s, z29.s\n" + "fmax z9.s, p3/M, z9.s, z16.s\n" + "fmin z18.s, p3/M, z18.s, z19.s\n" + "fmla z24.s, p3/M, z7.s, z29.s\n" + "fmla z26.s, p3/M, z6.s, z29.s\n" + "ld1w { z23.s }, p2/Z, [x21, x16, LSL #2]\n" + "fmin z13.s, p3/M, z13.s, z19.s\n" + "fmla z27.s, p3/M, z1.s, z12.s\n" + "fmla z14.s, p3/M, z0.s, z12.s\n" + "ld1w { z29.s }, p2/Z, [x20, x16, LSL #2]\n" + "fmin z22.s, p3/M, z22.s, z19.s\n" + "fmla z15.s, p3/M, z2.s, z21.s\n" + "fmla z11.s, p3/M, z1.s, z21.s\n" + "fmin z9.s, p3/M, z9.s, z19.s\n" + "fmax z27.s, p3/M, z27.s, z16.s\n" + "fmla z31.s, p3/M, z7.s, z23.s\n" + "fmla z30.s, p3/M, z6.s, z23.s\n" + "fmax z14.s, p3/M, z14.s, z16.s\n" + "fmax z15.s, p3/M, z15.s, z16.s\n" + "fmla z20.s, p3/M, z8.s, z29.s\n" + "fmla z10.s, p3/M, z7.s, z29.s\n" + "fmax z11.s, p3/M, z11.s, z16.s\n" + "st1w { z18.s }, p0, [x12, x15, LSL #2]\n" + "st1w { z13.s }, p0, [x11, x15, LSL #2]\n" + "ldr x23, [x14, #0x20]\n" + "ldr x22, [x14, #0x28]\n" + "fmla z28.s, p3/M, z4.s, z23.s\n" + "st1w { z22.s }, p0, [x10, x15, LSL #2]\n" + "ldr x21, [x14, #0x30]\n" + "fmla z25.s, p3/M, z3.s, z23.s\n" + "fmla z24.s, p3/M, z5.s, z29.s\n" + "st1w { z9.s }, p0, [x9, x15, LSL #2]\n" + "ldr x20, [x14, #0x38]\n" + "fmla z26.s, p3/M, z4.s, z29.s\n" + "fmin z27.s, p3/M, z27.s, z19.s\n" + "fmin z14.s, p3/M, z14.s, z19.s\n" + "fmin z15.s, p3/M, z15.s, z19.s\n" + "st1w { z27.s }, p0, [x23, x15, LSL #2]\n" + "ldr x23, [x14, #0x40]\n" + "fmin z11.s, p3/M, z11.s, z19.s\n" + "fmax z31.s, p3/M, z31.s, z16.s\n" + "st1w { z14.s }, p0, [x22, x15, LSL #2]\n" + "ldr x22, [x14, #0x48]\n" + "fmax z30.s, p3/M, z30.s, z16.s\n" + "fmax z20.s, p3/M, z20.s, z16.s\n" + "st1w { z15.s }, p0, [x21, x15, LSL #2]\n" + "ldr x21, [x14, #0x50]\n" + "fmax z10.s, p3/M, z10.s, z16.s\n" + "st1w { z11.s }, p0, [x20, x15, LSL #2]\n" + "ldr x20, [x14, #0x58]\n" + "fmin z31.s, p3/M, z31.s, z19.s\n" + "fmin z30.s, p3/M, z30.s, z19.s\n" + "fmin z20.s, p3/M, z20.s, z19.s\n" + "st1w { z31.s }, p0, [x23, x15, LSL #2]\n" + "ldr x23, [x14, #0x60]\n" + "fmin z10.s, p3/M, z10.s, z19.s\n" + "fmax z28.s, p3/M, z28.s, z16.s\n" + "st1w { z30.s }, p0, [x22, x15, LSL #2]\n" + "ldr x22, [x14, #0x68]\n" + "fmax z25.s, p3/M, z25.s, z16.s\n" + "fmax z24.s, p3/M, z24.s, z16.s\n" + "st1w { z20.s }, p0, [x21, x15, LSL #2]\n" + "ldr x21, [x14, #0x70]\n" + "fmax z26.s, p3/M, z26.s, z16.s\n" + "st1w { z10.s }, p0, [x20, x15, LSL #2]\n" + "ldr x20, [x14, #0x78]\n" + "fmin z28.s, p3/M, z28.s, z19.s\n" + "fmin z25.s, p3/M, z25.s, z19.s\n" + "fmin z24.s, p3/M, z24.s, z19.s\n" + "st1w { z28.s }, p0, [x23, x15, LSL #2]\n" + "fmin z26.s, p3/M, z26.s, z19.s\n" + "st1w { z25.s }, p0, [x22, x15, LSL #2]\n" + "st1w { z24.s }, p0, [x21, x15, LSL #2]\n" + "st1w { z26.s }, p0, [x20, x15, LSL #2]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp index 17ac74e2230d7f1981d5fb69af05c6e4b86babda..75d62007ab9c48ccdf6444c4f1464d2bd5663eb2 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,19 +22,19 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include #pragma once -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { -void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); -void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); +void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(const float *const *const input_ptrs, float *const *const outptrs, const void *params, unsigned int n_channels, const float activation_min, const float activation_max); +void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(const unsigned int n_tile_rows, const unsigned int n_tile_cols, const float *inptr, int64_t ld_input_row, int64_t ld_input_col, float *outptr, int64_t ld_output_row, int64_t ld_output_col, const void *params, unsigned int n_channels, const float activation_min, const float activation_max); class sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst : public DepthwiseDepthfirstStrategy { @@ -57,7 +57,7 @@ class sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst : public DepthwiseDepthfirst constexpr static unsigned int output_cols = 2; sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst(const CPUInfo *) - : DepthwiseDepthfirstStrategy(2, 3, 2) {} + : Parent(output_rows, output_cols, kernel_rows, kernel_cols, stride_rows, stride_cols) {} arm_gemm::VLType get_vl_type(void) const override { return vl_type; } @@ -68,4 +68,4 @@ class sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst : public DepthwiseDepthfirst } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp index 5a1f309b88ba63f10a338212ece195e13fe52783..e6090fda9432ae5e29c31c6e4d22c58a6c67230e 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -112,7 +112,7 @@ void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "add x28, x12, x23, LSL #2\n" "madd x20, x16, x14, x20\n" // offset += tile_j * ld_output_col "whilelt p2.s, XZR, %x[n_channels]\n" - "ld1w { z19.s }, p3/Z, [x11]\n" + "ld1w { z30.s }, p3/Z, [x11]\n" "ld1w { z0.s }, p3/Z, [x11, #1, MUL VL]\n" "mul x20, x20, x24\n" // offset *= output_tile_size "ld1w { z1.s }, p3/Z, [x11, #2, MUL VL]\n" @@ -128,8 +128,8 @@ void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "add x24, x26, x15\n" "add x9, x9, x20, LSL #2\n" // outptrs[0] += offset * sizeof(float) "cmp x13, %x[n_channels]\n" - "ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "ld1rw { z29.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rw { z28.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "add x23, x25, x23, LSL #2\n" "add x22, x9, x21, LSL #2\n" "ld1w { z7.s }, p3/Z, [x11, #-8, MUL VL]\n" @@ -147,191 +147,191 @@ void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "ld1w { z16.s }, p2/Z, [x12, x10, LSL #2]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "movprfx z28, z19\n fmla z28.s, p3/M, z8.s, z9.s\n" - "movprfx z29, z19\n fmla z29.s, p3/M, z6.s, z9.s\n" + "movprfx z27, z30\n fmla z27.s, p3/M, z8.s, z9.s\n" + "movprfx z26, z30\n fmla z26.s, p3/M, z6.s, z9.s\n" "whilelt p1.s, x13, %x[n_channels]\n" "incw x21\n" - "fmla z28.s, p3/M, z0.s, z10.s\n" - "fmla z29.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x28, x24, LSL #2]\n" + "fmla z27.s, p3/M, z0.s, z10.s\n" + "fmla z26.s, p3/M, z1.s, z12.s\n" + "ld1w { z20.s }, p2/Z, [x28, x24, LSL #2]\n" "incw x13\n" - "fmla z28.s, p3/M, z1.s, z11.s\n" - "fmla z29.s, p3/M, z2.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x28, x26, LSL #2]\n" - "ld1w { z13.s }, p2/Z, [x28, x10, LSL #2]\n" - "fmla z28.s, p3/M, z3.s, z14.s\n" - "fmla z29.s, p3/M, z0.s, z16.s\n" - "ld1w { z14.s }, p2/Z, [x25]\n" + "fmla z27.s, p3/M, z1.s, z11.s\n" + "fmla z26.s, p3/M, z2.s, z13.s\n" + "ld1w { z17.s }, p2/Z, [x28, x26, LSL #2]\n" + "ld1w { z19.s }, p2/Z, [x28, x10, LSL #2]\n" + "fmla z27.s, p3/M, z3.s, z14.s\n" + "fmla z26.s, p3/M, z0.s, z16.s\n" + "ld1w { z18.s }, p2/Z, [x25]\n" "mov p0.b, p2.b\n" - "fmla z28.s, p3/M, z4.s, z15.s\n" - "fmla z29.s, p3/M, z4.s, z11.s\n" - "ld1w { z15.s }, p2/Z, [x27]\n" - "ld1w { z11.s }, p2/Z, [x25, x15, LSL #2]\n" - "fmla z28.s, p3/M, z2.s, z16.s\n" - "fmla z29.s, p3/M, z5.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x27, x26, LSL #2]\n" - "ld1w { z16.s }, p2/Z, [x27, x15, LSL #2]\n" - "movprfx z30, z19\n fmla z30.s, p3/M, z2.s, z9.s\n" - "movprfx z31, z19\n fmla z31.s, p3/M, z0.s, z9.s\n" + "fmla z27.s, p3/M, z4.s, z15.s\n" + "fmla z26.s, p3/M, z4.s, z17.s\n" + "ld1w { z25.s }, p2/Z, [x27]\n" + "ld1w { z17.s }, p2/Z, [x25, x15, LSL #2]\n" + "fmla z27.s, p3/M, z2.s, z16.s\n" + "fmla z26.s, p3/M, z5.s, z20.s\n" + "ld1w { z24.s }, p2/Z, [x27, x26, LSL #2]\n" + "ld1w { z23.s }, p2/Z, [x27, x15, LSL #2]\n" + "movprfx z22, z30\n fmla z22.s, p3/M, z2.s, z9.s\n" + "movprfx z21, z30\n fmla z21.s, p3/M, z0.s, z9.s\n" "addvl x12, x12, #1\n" "addvl x28, x28, #1\n" - "fmla z28.s, p3/M, z5.s, z13.s\n" - "fmla z29.s, p3/M, z3.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x25, x26, LSL #2]\n" - "ld1w { z19.s }, p3/Z, [x11]\n" - "fmla z30.s, p3/M, z3.s, z14.s\n" - "fmla z31.s, p3/M, z4.s, z13.s\n" - "ld1w { z14.s }, p2/Z, [x25, x24, LSL #2]\n" - "ld1w { z13.s }, p2/Z, [x23, x15, LSL #2]\n" - "fmla z30.s, p3/M, z0.s, z15.s\n" - "fmla z31.s, p3/M, z1.s, z12.s\n" + "fmla z27.s, p3/M, z5.s, z19.s\n" + "fmla z26.s, p3/M, z3.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x25, x26, LSL #2]\n" + "ld1w { z30.s }, p3/Z, [x11]\n" + "fmla z22.s, p3/M, z3.s, z18.s\n" + "fmla z21.s, p3/M, z4.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x25, x24, LSL #2]\n" + "ld1w { z20.s }, p2/Z, [x23, x15, LSL #2]\n" + "fmla z22.s, p3/M, z0.s, z25.s\n" + "fmla z21.s, p3/M, z1.s, z24.s\n" "ld1w { z0.s }, p3/Z, [x11, #1, MUL VL]\n" "incw x20\n" - "fmla z30.s, p3/M, z4.s, z11.s\n" - "fmla z31.s, p3/M, z5.s, z14.s\n" - "ld1w { z11.s }, p2/Z, [x27, x24, LSL #2]\n" - "ld1w { z14.s }, p2/Z, [x23, x26, LSL #2]\n" - "fmla z28.s, p3/M, z6.s, z15.s\n" - "fmla z30.s, p3/M, z1.s, z16.s\n" - "ld1w { z15.s }, p2/Z, [x23]\n" + "fmla z22.s, p3/M, z4.s, z17.s\n" + "fmla z21.s, p3/M, z5.s, z16.s\n" + "ld1w { z19.s }, p2/Z, [x27, x24, LSL #2]\n" + "ld1w { z18.s }, p2/Z, [x23, x26, LSL #2]\n" + "fmla z27.s, p3/M, z6.s, z25.s\n" + "fmla z22.s, p3/M, z1.s, z23.s\n" + "ld1w { z17.s }, p2/Z, [x23]\n" "addvl x27, x27, #1\n" - "fmla z31.s, p3/M, z2.s, z11.s\n" - "fmla z28.s, p3/M, z7.s, z16.s\n" + "fmla z21.s, p3/M, z2.s, z19.s\n" + "fmla z27.s, p3/M, z7.s, z23.s\n" "ld1w { z16.s }, p2/Z, [x25, x10, LSL #2]\n" - "fmax z28.s, p3/M, z28.s, z18.s\n" - "fmla z30.s, p3/M, z6.s, z15.s\n" - "fmla z31.s, p3/M, z3.s, z16.s\n" - "ld1w { z15.s }, p2/Z, [x23, x10, LSL #2]\n" + "fmax z27.s, p3/M, z27.s, z29.s\n" + "fmla z22.s, p3/M, z6.s, z17.s\n" + "fmla z21.s, p3/M, z3.s, z16.s\n" + "ld1w { z17.s }, p2/Z, [x23, x10, LSL #2]\n" "ld1w { z1.s }, p3/Z, [x11, #2, MUL VL]\n" - "fmla z30.s, p3/M, z7.s, z13.s\n" - "fmla z31.s, p3/M, z7.s, z14.s\n" + "fmla z22.s, p3/M, z7.s, z20.s\n" + "fmla z21.s, p3/M, z7.s, z18.s\n" "ld1w { z2.s }, p3/Z, [x11, #3, MUL VL]\n" "ld1w { z3.s }, p3/Z, [x11, #4, MUL VL]\n" - "fmla z29.s, p3/M, z7.s, z12.s\n" - "fmla z30.s, p3/M, z5.s, z16.s\n" + "fmla z26.s, p3/M, z7.s, z24.s\n" + "fmla z22.s, p3/M, z5.s, z16.s\n" "ld1w { z4.s }, p3/Z, [x11, #5, MUL VL]\n" "ld1w { z5.s }, p3/Z, [x11, #6, MUL VL]\n" - "fmla z31.s, p3/M, z6.s, z15.s\n" - "fmla z29.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x23, x24, LSL #2]\n" - "fmax z29.s, p3/M, z29.s, z18.s\n" - "fmla z30.s, p3/M, z8.s, z15.s\n" - "fmla z31.s, p3/M, z8.s, z11.s\n" - "fmax z30.s, p3/M, z30.s, z18.s\n" - "fmax z31.s, p3/M, z31.s, z18.s\n" + "fmla z21.s, p3/M, z6.s, z17.s\n" + "fmla z26.s, p3/M, z8.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x23, x24, LSL #2]\n" + "fmax z26.s, p3/M, z26.s, z29.s\n" + "fmla z22.s, p3/M, z8.s, z17.s\n" + "fmla z21.s, p3/M, z8.s, z16.s\n" + "fmax z22.s, p3/M, z22.s, z29.s\n" + "fmax z21.s, p3/M, z21.s, z29.s\n" "ld1w { z6.s }, p3/Z, [x11, #7, MUL VL]\n" "addvl x11, x11, #16\n" "whilelt p2.s, x21, %x[n_channels]\n" "ld1w { z9.s }, p1/Z, [x27, x10, LSL #2]\n" "cmp x13, %x[n_channels]\n" - "fmin z28.s, p3/M, z28.s, z17.s\n" + "fmin z27.s, p3/M, z27.s, z28.s\n" "ld1w { z10.s }, p1/Z, [x12]\n" "ld1w { z11.s }, p1/Z, [x12, x15, LSL #2]\n" - "fmin z29.s, p3/M, z29.s, z17.s\n" - "fmin z30.s, p3/M, z30.s, z17.s\n" + "fmin z26.s, p3/M, z26.s, z28.s\n" + "fmin z22.s, p3/M, z22.s, z28.s\n" "ld1w { z12.s }, p1/Z, [x12, x26, LSL #2]\n" "ld1w { z13.s }, p1/Z, [x12, x24, LSL #2]\n" - "fmin z31.s, p3/M, z31.s, z17.s\n" + "fmin z21.s, p3/M, z21.s, z28.s\n" "addvl x25, x25, #1\n" "ld1w { z14.s }, p1/Z, [x28]\n" "ld1w { z15.s }, p1/Z, [x28, x15, LSL #2]\n" "addvl x23, x23, #1\n" "ld1w { z16.s }, p1/Z, [x12, x10, LSL #2]\n" - "st1w { z28.s }, p0, [x9]\n" + "st1w { z27.s }, p0, [x9]\n" "ld1w { z7.s }, p3/Z, [x11, #-8, MUL VL]\n" - "st1w { z29.s }, p0, [x9, x14, LSL #2]\n" + "st1w { z26.s }, p0, [x9, x14, LSL #2]\n" "addvl x9, x9, #1\n" "ld1w { z8.s }, p3/Z, [x11, #-7, MUL VL]\n" "addvl x11, x11, #-6\n" - "st1w { z30.s }, p0, [x22]\n" - "st1w { z31.s }, p0, [x22, x14, LSL #2]\n" + "st1w { z22.s }, p0, [x22]\n" + "st1w { z21.s }, p0, [x22, x14, LSL #2]\n" "addvl x22, x22, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "movprfx z28, z19\n fmla z28.s, p3/M, z8.s, z9.s\n" - "movprfx z29, z19\n fmla z29.s, p3/M, z6.s, z9.s\n" + "movprfx z27, z30\n fmla z27.s, p3/M, z8.s, z9.s\n" + "movprfx z26, z30\n fmla z26.s, p3/M, z6.s, z9.s\n" "ldr x16, [%x[params_struct], %[offsetof_args_tile_j]]\n" "ldr x11, [%x[params_struct], %[offsetof_args_tile_i]]\n" - "fmla z28.s, p3/M, z0.s, z10.s\n" - "fmla z29.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x28, x24, LSL #2]\n" + "fmla z27.s, p3/M, z0.s, z10.s\n" + "fmla z26.s, p3/M, z1.s, z12.s\n" + "ld1w { z20.s }, p2/Z, [x28, x24, LSL #2]\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" - "fmla z28.s, p3/M, z1.s, z11.s\n" - "fmla z29.s, p3/M, z2.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x28, x26, LSL #2]\n" - "ld1w { z13.s }, p2/Z, [x28, x10, LSL #2]\n" - "fmla z28.s, p3/M, z3.s, z14.s\n" - "fmla z29.s, p3/M, z0.s, z16.s\n" - "ld1w { z14.s }, p2/Z, [x25]\n" + "fmla z27.s, p3/M, z1.s, z11.s\n" + "fmla z26.s, p3/M, z2.s, z13.s\n" + "ld1w { z17.s }, p2/Z, [x28, x26, LSL #2]\n" + "ld1w { z19.s }, p2/Z, [x28, x10, LSL #2]\n" + "fmla z27.s, p3/M, z3.s, z14.s\n" + "fmla z26.s, p3/M, z0.s, z16.s\n" + "ld1w { z18.s }, p2/Z, [x25]\n" "add x16, x16, #0x1\n" - "fmla z28.s, p3/M, z4.s, z15.s\n" - "fmla z29.s, p3/M, z4.s, z11.s\n" - "ld1w { z15.s }, p2/Z, [x27]\n" - "ld1w { z11.s }, p2/Z, [x25, x15, LSL #2]\n" - "fmla z28.s, p3/M, z2.s, z16.s\n" - "fmla z29.s, p3/M, z5.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x27, x26, LSL #2]\n" - "ld1w { z16.s }, p2/Z, [x27, x15, LSL #2]\n" - "movprfx z30, z19\n fmla z30.s, p3/M, z2.s, z9.s\n" - "movprfx z31, z19\n fmla z31.s, p3/M, z0.s, z9.s\n" + "fmla z27.s, p3/M, z4.s, z15.s\n" + "fmla z26.s, p3/M, z4.s, z17.s\n" + "ld1w { z25.s }, p2/Z, [x27]\n" + "ld1w { z17.s }, p2/Z, [x25, x15, LSL #2]\n" + "fmla z27.s, p3/M, z2.s, z16.s\n" + "fmla z26.s, p3/M, z5.s, z20.s\n" + "ld1w { z24.s }, p2/Z, [x27, x26, LSL #2]\n" + "ld1w { z23.s }, p2/Z, [x27, x15, LSL #2]\n" + "movprfx z22, z30\n fmla z22.s, p3/M, z2.s, z9.s\n" + "movprfx z21, z30\n fmla z21.s, p3/M, z0.s, z9.s\n" "cmp x16, x20\n" "add x21, x11, #0x1\n" - "fmla z28.s, p3/M, z5.s, z13.s\n" - "fmla z29.s, p3/M, z3.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x25, x26, LSL #2]\n" + "fmla z27.s, p3/M, z5.s, z19.s\n" + "fmla z26.s, p3/M, z3.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x25, x26, LSL #2]\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" - "fmla z30.s, p3/M, z3.s, z14.s\n" - "fmla z31.s, p3/M, z4.s, z13.s\n" - "ld1w { z14.s }, p2/Z, [x25, x24, LSL #2]\n" - "ld1w { z13.s }, p2/Z, [x23, x15, LSL #2]\n" - "fmla z30.s, p3/M, z0.s, z15.s\n" - "fmla z31.s, p3/M, z1.s, z12.s\n" + "fmla z22.s, p3/M, z3.s, z18.s\n" + "fmla z21.s, p3/M, z4.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x25, x24, LSL #2]\n" + "ld1w { z20.s }, p2/Z, [x23, x15, LSL #2]\n" + "fmla z22.s, p3/M, z0.s, z25.s\n" + "fmla z21.s, p3/M, z1.s, z24.s\n" "csel x11, x11, x21, LT\n" "mov p0.b, p2.b\n" - "fmla z30.s, p3/M, z4.s, z11.s\n" - "fmla z31.s, p3/M, z5.s, z14.s\n" - "ld1w { z11.s }, p2/Z, [x27, x24, LSL #2]\n" - "ld1w { z14.s }, p2/Z, [x23, x26, LSL #2]\n" - "fmla z28.s, p3/M, z6.s, z15.s\n" - "fmla z30.s, p3/M, z1.s, z16.s\n" - "ld1w { z15.s }, p2/Z, [x23]\n" + "fmla z22.s, p3/M, z4.s, z17.s\n" + "fmla z21.s, p3/M, z5.s, z16.s\n" + "ld1w { z19.s }, p2/Z, [x27, x24, LSL #2]\n" + "ld1w { z18.s }, p2/Z, [x23, x26, LSL #2]\n" + "fmla z27.s, p3/M, z6.s, z25.s\n" + "fmla z22.s, p3/M, z1.s, z23.s\n" + "ld1w { z17.s }, p2/Z, [x23]\n" "csel x16, x16, XZR, LT\n" - "fmla z31.s, p3/M, z2.s, z11.s\n" - "fmla z28.s, p3/M, z7.s, z16.s\n" + "fmla z21.s, p3/M, z2.s, z19.s\n" + "fmla z27.s, p3/M, z7.s, z23.s\n" "ld1w { z16.s }, p2/Z, [x25, x10, LSL #2]\n" - "fmax z28.s, p3/M, z28.s, z18.s\n" - "fmla z30.s, p3/M, z6.s, z15.s\n" - "fmla z31.s, p3/M, z3.s, z16.s\n" - "ld1w { z15.s }, p2/Z, [x23, x10, LSL #2]\n" + "fmax z27.s, p3/M, z27.s, z29.s\n" + "fmla z22.s, p3/M, z6.s, z17.s\n" + "fmla z21.s, p3/M, z3.s, z16.s\n" + "ld1w { z17.s }, p2/Z, [x23, x10, LSL #2]\n" "cmp x11, x20\n" - "fmla z30.s, p3/M, z7.s, z13.s\n" - "fmla z31.s, p3/M, z7.s, z14.s\n" - "fmin z28.s, p3/M, z28.s, z17.s\n" - "st1w { z28.s }, p0, [x9]\n" - "fmla z29.s, p3/M, z7.s, z12.s\n" - "fmla z30.s, p3/M, z5.s, z16.s\n" - "fmla z31.s, p3/M, z6.s, z15.s\n" - "fmla z29.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x23, x24, LSL #2]\n" - "fmax z29.s, p3/M, z29.s, z18.s\n" - "fmla z30.s, p3/M, z8.s, z15.s\n" - "fmla z31.s, p3/M, z8.s, z11.s\n" - "fmax z30.s, p3/M, z30.s, z18.s\n" - "fmax z31.s, p3/M, z31.s, z18.s\n" - "fmin z29.s, p3/M, z29.s, z17.s\n" - "fmin z30.s, p3/M, z30.s, z17.s\n" - "st1w { z29.s }, p0, [x9, x14, LSL #2]\n" - "fmin z31.s, p3/M, z31.s, z17.s\n" - "st1w { z30.s }, p0, [x22]\n" - "st1w { z31.s }, p0, [x22, x14, LSL #2]\n" + "fmla z22.s, p3/M, z7.s, z20.s\n" + "fmla z21.s, p3/M, z7.s, z18.s\n" + "fmin z27.s, p3/M, z27.s, z28.s\n" + "st1w { z27.s }, p0, [x9]\n" + "fmla z26.s, p3/M, z7.s, z24.s\n" + "fmla z22.s, p3/M, z5.s, z16.s\n" + "fmla z21.s, p3/M, z6.s, z17.s\n" + "fmla z26.s, p3/M, z8.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x23, x24, LSL #2]\n" + "fmax z26.s, p3/M, z26.s, z29.s\n" + "fmla z22.s, p3/M, z8.s, z17.s\n" + "fmla z21.s, p3/M, z8.s, z16.s\n" + "fmax z22.s, p3/M, z22.s, z29.s\n" + "fmax z21.s, p3/M, z21.s, z29.s\n" + "fmin z26.s, p3/M, z26.s, z28.s\n" + "fmin z22.s, p3/M, z22.s, z28.s\n" + "st1w { z26.s }, p0, [x9, x14, LSL #2]\n" + "fmin z21.s, p3/M, z21.s, z28.s\n" + "st1w { z22.s }, p0, [x22]\n" + "st1w { z21.s }, p0, [x22, x14, LSL #2]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp index eb6c2daa97282ea96942c6b1d8fc1ff1dfb03109..98427701fa1605716a99107e5eb20fd7c533f91c 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -96,7 +96,7 @@ void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( "ldp x11, x10, [x20, #0x10]\n" "mov x9, #0x0\n" "whilelt p2.s, XZR, %x[n_channels]\n" - "ld1w { z19.s }, p3/Z, [x16]\n" + "ld1w { z20.s }, p3/Z, [x16]\n" "ld1w { z0.s }, p3/Z, [x16, #1, MUL VL]\n" "cmp x14, %x[n_channels]\n" "ld1w { z1.s }, p3/Z, [x16, #2, MUL VL]\n" @@ -111,8 +111,8 @@ void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( "ldp x25, x24, [x15, #0x10]\n" "ldp x23, x22, [x15, #0x20]\n" "ldp x21, x20, [x15, #0x30]\n" - "ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "ld1rw { z26.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rw { z25.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "ld1w { z7.s }, p3/Z, [x16, #-8, MUL VL]\n" "ld1w { z8.s }, p3/Z, [x16, #-7, MUL VL]\n" "addvl x16, x16, #-6\n" @@ -126,89 +126,89 @@ void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( "ld1w { z16.s }, p2/Z, [x20, x9, LSL #2]\n" "bge 2f\n" "1:" // Channel loop - "movprfx z28, z19\n fmla z28.s, p3/M, z8.s, z9.s\n" - "movprfx z29, z19\n fmla z29.s, p3/M, z6.s, z9.s\n" - "ldr x27, [x15, #0x40]\n" - "ldr x26, [x15, #0x48]\n" - "fmla z28.s, p3/M, z0.s, z10.s\n" - "fmla z29.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x26, x9, LSL #2]\n" - "ldr x25, [x15, #0x50]\n" - "fmla z28.s, p3/M, z1.s, z11.s\n" - "fmla z29.s, p3/M, z2.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x27, x9, LSL #2]\n" - "ld1w { z13.s }, p2/Z, [x25, x9, LSL #2]\n" - "fmla z28.s, p3/M, z3.s, z14.s\n" - "fmla z29.s, p3/M, z0.s, z16.s\n" - "ldr x24, [x15, #0x58]\n" - "ldr x20, [x15, #0x78]\n" - "fmla z28.s, p3/M, z4.s, z15.s\n" - "fmla z29.s, p3/M, z4.s, z11.s\n" - "ld1w { z14.s }, p2/Z, [x24, x9, LSL #2]\n" - "ldr x23, [x15, #0x60]\n" - "fmla z28.s, p3/M, z2.s, z16.s\n" - "fmla z29.s, p3/M, z5.s, z12.s\n" - "ldr x27, [x15, #0x80]\n" - "ld1w { z15.s }, p2/Z, [x23, x9, LSL #2]\n" - "movprfx z30, z19\n fmla z30.s, p3/M, z2.s, z9.s\n" - "movprfx z31, z19\n fmla z31.s, p3/M, z0.s, z9.s\n" - "ld1w { z12.s }, p2/Z, [x27, x9, LSL #2]\n" - "ldr x22, [x15, #0x68]\n" - "fmla z28.s, p3/M, z5.s, z13.s\n" - "fmla z29.s, p3/M, z3.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x20, x9, LSL #2]\n" - "ldr x26, [x15, #0x88]\n" - "fmla z30.s, p3/M, z3.s, z14.s\n" - "fmla z31.s, p3/M, z4.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x22, x9, LSL #2]\n" - "ld1w { z14.s }, p2/Z, [x26, x9, LSL #2]\n" - "fmla z30.s, p3/M, z0.s, z15.s\n" - "fmla z31.s, p3/M, z1.s, z12.s\n" + "movprfx z24, z20\n fmla z24.s, p3/M, z8.s, z9.s\n" + "movprfx z23, z20\n fmla z23.s, p3/M, z6.s, z9.s\n" + "ldr x21, [x15, #0x40]\n" + "ldr x20, [x15, #0x48]\n" + "fmla z24.s, p3/M, z0.s, z10.s\n" + "fmla z23.s, p3/M, z1.s, z12.s\n" + "ld1w { z18.s }, p2/Z, [x20, x9, LSL #2]\n" + "ldr x20, [x15, #0x50]\n" + "fmla z24.s, p3/M, z1.s, z11.s\n" + "fmla z23.s, p3/M, z2.s, z13.s\n" + "ld1w { z17.s }, p2/Z, [x21, x9, LSL #2]\n" + "ld1w { z19.s }, p2/Z, [x20, x9, LSL #2]\n" + "fmla z24.s, p3/M, z3.s, z14.s\n" + "fmla z23.s, p3/M, z0.s, z16.s\n" + "ldr x20, [x15, #0x58]\n" + "ldr x22, [x15, #0x78]\n" + "fmla z24.s, p3/M, z4.s, z15.s\n" + "fmla z23.s, p3/M, z4.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x20, x9, LSL #2]\n" + "ldr x21, [x15, #0x60]\n" + "fmla z24.s, p3/M, z2.s, z16.s\n" + "fmla z23.s, p3/M, z5.s, z18.s\n" + "ldr x20, [x15, #0x80]\n" + "ld1w { z18.s }, p2/Z, [x21, x9, LSL #2]\n" + "movprfx z22, z20\n fmla z22.s, p3/M, z2.s, z9.s\n" + "movprfx z21, z20\n fmla z21.s, p3/M, z0.s, z9.s\n" + "ld1w { z20.s }, p2/Z, [x20, x9, LSL #2]\n" + "ldr x21, [x15, #0x68]\n" + "fmla z24.s, p3/M, z5.s, z19.s\n" + "fmla z23.s, p3/M, z3.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x22, x9, LSL #2]\n" + "ldr x20, [x15, #0x88]\n" + "fmla z22.s, p3/M, z3.s, z17.s\n" + "fmla z21.s, p3/M, z4.s, z16.s\n" + "ld1w { z17.s }, p2/Z, [x21, x9, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x20, x9, LSL #2]\n" + "fmla z22.s, p3/M, z0.s, z18.s\n" + "fmla z21.s, p3/M, z1.s, z20.s\n" "ldr x21, [x15, #0x70]\n" - "ldr x24, [x15, #0x98]\n" - "fmla z30.s, p3/M, z4.s, z11.s\n" - "fmla z31.s, p3/M, z5.s, z14.s\n" + "ldr x20, [x15, #0x98]\n" + "fmla z22.s, p3/M, z4.s, z17.s\n" + "fmla z21.s, p3/M, z5.s, z16.s\n" "ld1w { z16.s }, p2/Z, [x21, x9, LSL #2]\n" - "ld1w { z11.s }, p2/Z, [x24, x9, LSL #2]\n" - "fmla z28.s, p3/M, z6.s, z15.s\n" - "ldr x25, [x15, #0x90]\n" - "ldr x22, [x15, #0xa8]\n" - "fmla z30.s, p3/M, z1.s, z16.s\n" - "fmla z31.s, p3/M, z2.s, z11.s\n" - "fmla z28.s, p3/M, z7.s, z16.s\n" - "ld1w { z15.s }, p2/Z, [x25, x9, LSL #2]\n" - "ld1w { z16.s }, p2/Z, [x22, x9, LSL #2]\n" - "ldr x23, [x15, #0xa0]\n" - "ldr x21, [x15, #0xb0]\n" - "fmla z30.s, p3/M, z6.s, z15.s\n" - "fmla z31.s, p3/M, z3.s, z16.s\n" - "ld1w { z13.s }, p2/Z, [x23, x9, LSL #2]\n" - "ld1w { z14.s }, p2/Z, [x21, x9, LSL #2]\n" - "fmla z30.s, p3/M, z7.s, z13.s\n" - "fmla z31.s, p3/M, z7.s, z14.s\n" + "ld1w { z19.s }, p2/Z, [x20, x9, LSL #2]\n" + "fmla z24.s, p3/M, z6.s, z18.s\n" + "ldr x21, [x15, #0x90]\n" + "ldr x20, [x15, #0xa8]\n" + "fmla z22.s, p3/M, z1.s, z16.s\n" + "fmla z21.s, p3/M, z2.s, z19.s\n" + "fmla z24.s, p3/M, z7.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x21, x9, LSL #2]\n" + "ld1w { z18.s }, p2/Z, [x20, x9, LSL #2]\n" + "ldr x21, [x15, #0xa0]\n" + "ldr x20, [x15, #0xb0]\n" + "fmla z22.s, p3/M, z6.s, z16.s\n" + "fmla z21.s, p3/M, z3.s, z18.s\n" + "ld1w { z17.s }, p2/Z, [x21, x9, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x20, x9, LSL #2]\n" + "fmla z22.s, p3/M, z7.s, z17.s\n" + "fmla z21.s, p3/M, z7.s, z16.s\n" "ldr x20, [x15, #0xb8]\n" - "fmla z29.s, p3/M, z7.s, z12.s\n" - "ld1w { z15.s }, p2/Z, [x20, x9, LSL #2]\n" - "fmla z30.s, p3/M, z5.s, z16.s\n" - "ldr x27, [x15, #0xc0]\n" - "fmla z31.s, p3/M, z6.s, z15.s\n" - "fmla z29.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x27, x9, LSL #2]\n" - "fmla z30.s, p3/M, z8.s, z15.s\n" - "fmla z31.s, p3/M, z8.s, z11.s\n" + "fmla z23.s, p3/M, z7.s, z20.s\n" + "ld1w { z17.s }, p2/Z, [x20, x9, LSL #2]\n" + "fmla z22.s, p3/M, z5.s, z18.s\n" + "ldr x20, [x15, #0xc0]\n" + "fmla z21.s, p3/M, z6.s, z17.s\n" + "fmla z23.s, p3/M, z8.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x20, x9, LSL #2]\n" + "fmla z22.s, p3/M, z8.s, z17.s\n" + "fmla z21.s, p3/M, z8.s, z16.s\n" "whilelt p1.s, x14, %x[n_channels]\n" "ldp x27, x26, [x15, #0x0]\n" "ldp x25, x24, [x15, #0x10]\n" "ldp x23, x22, [x15, #0x20]\n" "incw x9\n" - "fmax z28.s, p3/M, z28.s, z18.s\n" + "fmax z24.s, p3/M, z24.s, z26.s\n" "ldp x21, x20, [x15, #0x30]\n" "ld1w { z9.s }, p1/Z, [x27, x14, LSL #2]\n" - "fmax z29.s, p3/M, z29.s, z18.s\n" - "fmax z30.s, p3/M, z30.s, z18.s\n" + "fmax z23.s, p3/M, z23.s, z26.s\n" + "fmax z22.s, p3/M, z22.s, z26.s\n" "ld1w { z10.s }, p1/Z, [x26, x14, LSL #2]\n" "ld1w { z11.s }, p1/Z, [x25, x14, LSL #2]\n" - "fmax z31.s, p3/M, z31.s, z18.s\n" + "fmax z21.s, p3/M, z21.s, z26.s\n" "incw x28\n" "ld1w { z12.s }, p1/Z, [x24, x14, LSL #2]\n" "ld1w { z13.s }, p1/Z, [x23, x14, LSL #2]\n" @@ -216,122 +216,122 @@ void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( "whilelt p2.s, x9, %x[n_channels]\n" "ld1w { z14.s }, p1/Z, [x22, x14, LSL #2]\n" "ld1w { z15.s }, p1/Z, [x21, x14, LSL #2]\n" - "fmin z28.s, p3/M, z28.s, z17.s\n" - "fmin z29.s, p3/M, z29.s, z17.s\n" + "fmin z24.s, p3/M, z24.s, z25.s\n" + "fmin z23.s, p3/M, z23.s, z25.s\n" "ld1w { z16.s }, p1/Z, [x20, x14, LSL #2]\n" "incw x14\n" - "ld1w { z19.s }, p3/Z, [x16]\n" + "ld1w { z20.s }, p3/Z, [x16]\n" "cmp x14, %x[n_channels]\n" "ld1w { z0.s }, p3/Z, [x16, #1, MUL VL]\n" "ld1w { z1.s }, p3/Z, [x16, #2, MUL VL]\n" - "fmin z30.s, p3/M, z30.s, z17.s\n" - "fmin z31.s, p3/M, z31.s, z17.s\n" + "fmin z22.s, p3/M, z22.s, z25.s\n" + "fmin z21.s, p3/M, z21.s, z25.s\n" "ld1w { z2.s }, p3/Z, [x16, #3, MUL VL]\n" "ld1w { z3.s }, p3/Z, [x16, #4, MUL VL]\n" - "st1w { z28.s }, p0, [x13, x28, LSL #2]\n" + "st1w { z24.s }, p0, [x13, x28, LSL #2]\n" "ld1w { z4.s }, p3/Z, [x16, #5, MUL VL]\n" "ld1w { z5.s }, p3/Z, [x16, #6, MUL VL]\n" - "st1w { z29.s }, p0, [x12, x28, LSL #2]\n" + "st1w { z23.s }, p0, [x12, x28, LSL #2]\n" "ld1w { z6.s }, p3/Z, [x16, #7, MUL VL]\n" "addvl x16, x16, #16\n" - "st1w { z30.s }, p0, [x11, x28, LSL #2]\n" + "st1w { z22.s }, p0, [x11, x28, LSL #2]\n" "ld1w { z7.s }, p3/Z, [x16, #-8, MUL VL]\n" - "st1w { z31.s }, p0, [x10, x28, LSL #2]\n" + "st1w { z21.s }, p0, [x10, x28, LSL #2]\n" "ld1w { z8.s }, p3/Z, [x16, #-7, MUL VL]\n" "addvl x16, x16, #-6\n" "blt 1b\n" "2:" // Channel tail - "movprfx z28, z19\n fmla z28.s, p3/M, z8.s, z9.s\n" - "movprfx z29, z19\n fmla z29.s, p3/M, z6.s, z9.s\n" - "ldr x27, [x15, #0x40]\n" - "ldr x26, [x15, #0x48]\n" - "fmla z28.s, p3/M, z0.s, z10.s\n" - "fmla z29.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x26, x9, LSL #2]\n" - "ldr x25, [x15, #0x50]\n" - "fmla z28.s, p3/M, z1.s, z11.s\n" - "fmla z29.s, p3/M, z2.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x27, x9, LSL #2]\n" - "ld1w { z13.s }, p2/Z, [x25, x9, LSL #2]\n" - "fmla z28.s, p3/M, z3.s, z14.s\n" - "fmla z29.s, p3/M, z0.s, z16.s\n" - "ldr x24, [x15, #0x58]\n" - "ldr x20, [x15, #0x78]\n" - "fmla z28.s, p3/M, z4.s, z15.s\n" - "fmla z29.s, p3/M, z4.s, z11.s\n" - "ld1w { z14.s }, p2/Z, [x24, x9, LSL #2]\n" - "ldr x23, [x15, #0x60]\n" - "fmla z28.s, p3/M, z2.s, z16.s\n" - "fmla z29.s, p3/M, z5.s, z12.s\n" - "ldr x27, [x15, #0x80]\n" - "ld1w { z15.s }, p2/Z, [x23, x9, LSL #2]\n" - "movprfx z30, z19\n fmla z30.s, p3/M, z2.s, z9.s\n" - "movprfx z31, z19\n fmla z31.s, p3/M, z0.s, z9.s\n" - "ld1w { z12.s }, p2/Z, [x27, x9, LSL #2]\n" - "ldr x22, [x15, #0x68]\n" - "fmla z28.s, p3/M, z5.s, z13.s\n" - "fmla z29.s, p3/M, z3.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x20, x9, LSL #2]\n" - "ldr x26, [x15, #0x88]\n" - "fmla z30.s, p3/M, z3.s, z14.s\n" - "fmla z31.s, p3/M, z4.s, z13.s\n" - "ld1w { z11.s }, p2/Z, [x22, x9, LSL #2]\n" - "ld1w { z14.s }, p2/Z, [x26, x9, LSL #2]\n" - "fmla z30.s, p3/M, z0.s, z15.s\n" - "fmla z31.s, p3/M, z1.s, z12.s\n" + "movprfx z24, z20\n fmla z24.s, p3/M, z8.s, z9.s\n" + "movprfx z23, z20\n fmla z23.s, p3/M, z6.s, z9.s\n" + "ldr x21, [x15, #0x40]\n" + "ldr x20, [x15, #0x48]\n" + "fmla z24.s, p3/M, z0.s, z10.s\n" + "fmla z23.s, p3/M, z1.s, z12.s\n" + "ld1w { z18.s }, p2/Z, [x20, x9, LSL #2]\n" + "ldr x20, [x15, #0x50]\n" + "fmla z24.s, p3/M, z1.s, z11.s\n" + "fmla z23.s, p3/M, z2.s, z13.s\n" + "ld1w { z17.s }, p2/Z, [x21, x9, LSL #2]\n" + "ld1w { z19.s }, p2/Z, [x20, x9, LSL #2]\n" + "fmla z24.s, p3/M, z3.s, z14.s\n" + "fmla z23.s, p3/M, z0.s, z16.s\n" + "ldr x20, [x15, #0x58]\n" + "ldr x22, [x15, #0x78]\n" + "fmla z24.s, p3/M, z4.s, z15.s\n" + "fmla z23.s, p3/M, z4.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x20, x9, LSL #2]\n" + "ldr x21, [x15, #0x60]\n" + "fmla z24.s, p3/M, z2.s, z16.s\n" + "fmla z23.s, p3/M, z5.s, z18.s\n" + "ldr x20, [x15, #0x80]\n" + "ld1w { z18.s }, p2/Z, [x21, x9, LSL #2]\n" + "movprfx z22, z20\n fmla z22.s, p3/M, z2.s, z9.s\n" + "movprfx z21, z20\n fmla z21.s, p3/M, z0.s, z9.s\n" + "ld1w { z20.s }, p2/Z, [x20, x9, LSL #2]\n" + "ldr x21, [x15, #0x68]\n" + "fmla z24.s, p3/M, z5.s, z19.s\n" + "fmla z23.s, p3/M, z3.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x22, x9, LSL #2]\n" + "ldr x20, [x15, #0x88]\n" + "fmla z22.s, p3/M, z3.s, z17.s\n" + "fmla z21.s, p3/M, z4.s, z16.s\n" + "ld1w { z17.s }, p2/Z, [x21, x9, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x20, x9, LSL #2]\n" + "fmla z22.s, p3/M, z0.s, z18.s\n" + "fmla z21.s, p3/M, z1.s, z20.s\n" "ldr x21, [x15, #0x70]\n" - "ldr x24, [x15, #0x98]\n" - "fmla z30.s, p3/M, z4.s, z11.s\n" - "fmla z31.s, p3/M, z5.s, z14.s\n" + "ldr x20, [x15, #0x98]\n" + "fmla z22.s, p3/M, z4.s, z17.s\n" + "fmla z21.s, p3/M, z5.s, z16.s\n" "ld1w { z16.s }, p2/Z, [x21, x9, LSL #2]\n" - "ld1w { z11.s }, p2/Z, [x24, x9, LSL #2]\n" - "fmla z28.s, p3/M, z6.s, z15.s\n" - "ldr x25, [x15, #0x90]\n" - "ldr x22, [x15, #0xa8]\n" - "fmla z30.s, p3/M, z1.s, z16.s\n" - "fmla z31.s, p3/M, z2.s, z11.s\n" - "fmla z28.s, p3/M, z7.s, z16.s\n" - "ld1w { z15.s }, p2/Z, [x25, x9, LSL #2]\n" - "ld1w { z16.s }, p2/Z, [x22, x9, LSL #2]\n" - "ldr x23, [x15, #0xa0]\n" - "ldr x21, [x15, #0xb0]\n" - "fmla z30.s, p3/M, z6.s, z15.s\n" - "fmla z31.s, p3/M, z3.s, z16.s\n" - "ld1w { z13.s }, p2/Z, [x23, x9, LSL #2]\n" - "ld1w { z14.s }, p2/Z, [x21, x9, LSL #2]\n" - "fmla z30.s, p3/M, z7.s, z13.s\n" - "fmla z31.s, p3/M, z7.s, z14.s\n" + "ld1w { z19.s }, p2/Z, [x20, x9, LSL #2]\n" + "fmla z24.s, p3/M, z6.s, z18.s\n" + "ldr x21, [x15, #0x90]\n" + "ldr x20, [x15, #0xa8]\n" + "fmla z22.s, p3/M, z1.s, z16.s\n" + "fmla z21.s, p3/M, z2.s, z19.s\n" + "fmla z24.s, p3/M, z7.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x21, x9, LSL #2]\n" + "ld1w { z18.s }, p2/Z, [x20, x9, LSL #2]\n" + "ldr x21, [x15, #0xa0]\n" + "ldr x20, [x15, #0xb0]\n" + "fmla z22.s, p3/M, z6.s, z16.s\n" + "fmla z21.s, p3/M, z3.s, z18.s\n" + "ld1w { z17.s }, p2/Z, [x21, x9, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x20, x9, LSL #2]\n" + "fmla z22.s, p3/M, z7.s, z17.s\n" + "fmla z21.s, p3/M, z7.s, z16.s\n" "ldr x20, [x15, #0xb8]\n" - "fmla z29.s, p3/M, z7.s, z12.s\n" - "ld1w { z15.s }, p2/Z, [x20, x9, LSL #2]\n" - "fmla z30.s, p3/M, z5.s, z16.s\n" - "ldr x27, [x15, #0xc0]\n" - "fmla z31.s, p3/M, z6.s, z15.s\n" - "fmla z29.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x27, x9, LSL #2]\n" - "fmla z30.s, p3/M, z8.s, z15.s\n" - "fmla z31.s, p3/M, z8.s, z11.s\n" + "fmla z23.s, p3/M, z7.s, z20.s\n" + "ld1w { z17.s }, p2/Z, [x20, x9, LSL #2]\n" + "fmla z22.s, p3/M, z5.s, z18.s\n" + "ldr x20, [x15, #0xc0]\n" + "fmla z21.s, p3/M, z6.s, z17.s\n" + "fmla z23.s, p3/M, z8.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x20, x9, LSL #2]\n" + "fmla z22.s, p3/M, z8.s, z17.s\n" + "fmla z21.s, p3/M, z8.s, z16.s\n" "incw x28\n" "mov p0.b, p2.b\n" - "fmax z28.s, p3/M, z28.s, z18.s\n" - "fmax z29.s, p3/M, z29.s, z18.s\n" - "fmax z30.s, p3/M, z30.s, z18.s\n" - "fmax z31.s, p3/M, z31.s, z18.s\n" - "fmin z28.s, p3/M, z28.s, z17.s\n" - "fmin z29.s, p3/M, z29.s, z17.s\n" - "st1w { z28.s }, p0, [x13, x28, LSL #2]\n" - "fmin z30.s, p3/M, z30.s, z17.s\n" - "fmin z31.s, p3/M, z31.s, z17.s\n" - "st1w { z29.s }, p0, [x12, x28, LSL #2]\n" - "st1w { z30.s }, p0, [x11, x28, LSL #2]\n" - "st1w { z31.s }, p0, [x10, x28, LSL #2]\n" + "fmax z24.s, p3/M, z24.s, z26.s\n" + "fmax z23.s, p3/M, z23.s, z26.s\n" + "fmax z22.s, p3/M, z22.s, z26.s\n" + "fmax z21.s, p3/M, z21.s, z26.s\n" + "fmin z24.s, p3/M, z24.s, z25.s\n" + "fmin z23.s, p3/M, z23.s, z25.s\n" + "st1w { z24.s }, p0, [x13, x28, LSL #2]\n" + "fmin z22.s, p3/M, z22.s, z25.s\n" + "fmin z21.s, p3/M, z21.s, z25.s\n" + "st1w { z23.s }, p0, [x12, x28, LSL #2]\n" + "st1w { z22.s }, p0, [x11, x28, LSL #2]\n" + "st1w { z21.s }, p0, [x10, x28, LSL #2]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp index 2449c96637d7c1df8a08340b279b8413950e5c47..ae89a64c6b8fbcbe86ae3e80b5dae090a86d8cc4 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,19 +22,19 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include #pragma once -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { -void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); -void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); +void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(const float *const *const input_ptrs, float *const *const outptrs, const void *params, unsigned int n_channels, const float activation_min, const float activation_max); +void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int n_tile_rows, const unsigned int n_tile_cols, const float *inptr, int64_t ld_input_row, int64_t ld_input_col, float *outptr, int64_t ld_output_row, int64_t ld_output_col, const void *params, unsigned int n_channels, const float activation_min, const float activation_max); class sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirstStrategy { @@ -57,7 +57,7 @@ class sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirst constexpr static unsigned int output_cols = 2; sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst(const CPUInfo *) - : DepthwiseDepthfirstStrategy(2, 5, 1) {} + : Parent(output_rows, output_cols, kernel_rows, kernel_cols, stride_rows, stride_cols) {} arm_gemm::VLType get_vl_type(void) const override { return vl_type; } @@ -68,4 +68,4 @@ class sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirst } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp index b4cf6c858238a8598eb34c8e61a62c0230eea317..075181a488271d8c6669c46640c93f7558a8e848 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -113,14 +113,14 @@ void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "madd x20, x8, x16, x20\n" // offset += tile_j * ld_output_col "add x9, x11, x23, LSL #2\n" "add x28, x15, x17\n" - "ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rw { z15.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" "mul x20, x20, x24\n" // offset *= output_tile_size "whilelt p2.s, XZR, %x[n_channels]\n" "add x27, x9, x23, LSL #2\n" - "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "ld1rw { z28.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "add x26, x28, x17\n" "add x25, x27, x23, LSL #2\n" - "ld1w { z16.s }, p3/Z, [x10]\n" + "ld1w { z29.s }, p3/Z, [x10]\n" "ld1w { z0.s }, p3/Z, [x10, #1, MUL VL]\n" "add x24, x26, x17\n" "add x13, x13, x20, LSL #2\n" // outptrs[0] += offset * sizeof(float) @@ -146,378 +146,378 @@ void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "ld1w { z14.s }, p2/Z, [x9]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z5.s\n" - "movprfx z29, z16\n fmla z29.s, p3/M, z0.s, z6.s\n" - "ld1w { z5.s }, p2/Z, [x11, x28, LSL #2]\n" + "movprfx z27, z29\n fmla z27.s, p3/M, z0.s, z5.s\n" + "movprfx z31, z29\n fmla z31.s, p3/M, z0.s, z6.s\n" + "ld1w { z24.s }, p2/Z, [x11, x28, LSL #2]\n" "whilelt p1.s, x12, %x[n_channels]\n" - "movprfx z30, z16\n fmla z30.s, p3/M, z0.s, z7.s\n" - "movprfx z31, z16\n fmla z31.s, p3/M, z0.s, z8.s\n" - "ld1w { z0.s }, p3/Z, [x10]\n" + "movprfx z26, z29\n fmla z26.s, p3/M, z0.s, z7.s\n" + "movprfx z30, z29\n fmla z30.s, p3/M, z0.s, z8.s\n" + "ld1w { z18.s }, p3/Z, [x10]\n" "incw x21\n" - "fmla z28.s, p3/M, z1.s, z6.s\n" - "fmla z29.s, p3/M, z1.s, z9.s\n" - "ld1w { z6.s }, p2/Z, [x11, x26, LSL #2]\n" + "fmla z27.s, p3/M, z1.s, z6.s\n" + "fmla z31.s, p3/M, z1.s, z9.s\n" + "ld1w { z23.s }, p2/Z, [x11, x26, LSL #2]\n" "incw x12\n" - "fmla z30.s, p3/M, z1.s, z8.s\n" - "fmla z31.s, p3/M, z1.s, z13.s\n" - "ld1w { z1.s }, p3/Z, [x10, #1, MUL VL]\n" + "fmla z26.s, p3/M, z1.s, z8.s\n" + "fmla z30.s, p3/M, z1.s, z13.s\n" + "ld1w { z22.s }, p3/Z, [x10, #1, MUL VL]\n" "mov p0.b, p2.b\n" - "fmla z28.s, p3/M, z2.s, z9.s\n" - "fmla z29.s, p3/M, z2.s, z11.s\n" - "ld1w { z9.s }, p2/Z, [x14, x24, LSL #2]\n" + "fmla z27.s, p3/M, z2.s, z9.s\n" + "fmla z31.s, p3/M, z2.s, z11.s\n" + "ld1w { z16.s }, p2/Z, [x14, x24, LSL #2]\n" "addvl x14, x14, #1\n" - "fmla z30.s, p3/M, z2.s, z13.s\n" - "fmla z31.s, p3/M, z2.s, z5.s\n" - "ld1w { z2.s }, p3/Z, [x10, #2, MUL VL]\n" + "fmla z26.s, p3/M, z2.s, z13.s\n" + "fmla z30.s, p3/M, z2.s, z24.s\n" + "ld1w { z20.s }, p3/Z, [x10, #2, MUL VL]\n" "addvl x11, x11, #1\n" - "fmla z28.s, p3/M, z3.s, z11.s\n" - "fmla z29.s, p3/M, z3.s, z12.s\n" - "ld1w { z11.s }, p2/Z, [x9, x17, LSL #2]\n" + "fmla z27.s, p3/M, z3.s, z11.s\n" + "fmla z31.s, p3/M, z3.s, z12.s\n" + "ld1w { z0.s }, p2/Z, [x9, x17, LSL #2]\n" "incw x20\n" - "fmla z30.s, p3/M, z3.s, z5.s\n" - "fmla z31.s, p3/M, z3.s, z6.s\n" - "ld1w { z3.s }, p3/Z, [x10, #3, MUL VL]\n" - "fmla z28.s, p3/M, z4.s, z12.s\n" - "fmla z29.s, p3/M, z4.s, z9.s\n" - "ld1w { z12.s }, p2/Z, [x9, x15, LSL #2]\n" - "ld1w { z9.s }, p2/Z, [x9, x28, LSL #2]\n" - "fmla z30.s, p3/M, z4.s, z6.s\n" - "fmla z31.s, p3/M, z4.s, z10.s\n" - "ld1w { z4.s }, p3/Z, [x10, #4, MUL VL]\n" - "fmla z28.s, p3/M, z0.s, z7.s\n" - "fmla z29.s, p3/M, z0.s, z8.s\n" + "fmla z26.s, p3/M, z3.s, z24.s\n" + "fmla z30.s, p3/M, z3.s, z23.s\n" + "ld1w { z17.s }, p3/Z, [x10, #3, MUL VL]\n" + "fmla z27.s, p3/M, z4.s, z12.s\n" + "fmla z31.s, p3/M, z4.s, z16.s\n" + "ld1w { z19.s }, p2/Z, [x9, x15, LSL #2]\n" + "ld1w { z5.s }, p2/Z, [x9, x28, LSL #2]\n" + "fmla z26.s, p3/M, z4.s, z23.s\n" + "fmla z30.s, p3/M, z4.s, z10.s\n" + "ld1w { z21.s }, p3/Z, [x10, #4, MUL VL]\n" + "fmla z27.s, p3/M, z18.s, z7.s\n" + "fmla z31.s, p3/M, z18.s, z8.s\n" "ld1w { z7.s }, p1/Z, [x11]\n" - "fmla z30.s, p3/M, z0.s, z14.s\n" - "fmla z31.s, p3/M, z0.s, z11.s\n" - "ld1w { z0.s }, p3/Z, [x10, #5, MUL VL]\n" - "fmla z28.s, p3/M, z1.s, z8.s\n" - "fmla z29.s, p3/M, z1.s, z13.s\n" - "ld1w { z8.s }, p2/Z, [x9, x24, LSL #2]\n" - "fmla z30.s, p3/M, z1.s, z11.s\n" - "fmla z31.s, p3/M, z1.s, z12.s\n" - "ld1w { z1.s }, p3/Z, [x10, #6, MUL VL]\n" - "fmla z28.s, p3/M, z2.s, z13.s\n" - "fmla z29.s, p3/M, z2.s, z5.s\n" - "ld1w { z13.s }, p2/Z, [x9, x26, LSL #2]\n" + "fmla z26.s, p3/M, z18.s, z14.s\n" + "fmla z30.s, p3/M, z18.s, z0.s\n" + "ld1w { z18.s }, p3/Z, [x10, #5, MUL VL]\n" + "fmla z27.s, p3/M, z22.s, z8.s\n" + "fmla z31.s, p3/M, z22.s, z13.s\n" + "ld1w { z3.s }, p2/Z, [x9, x24, LSL #2]\n" + "fmla z26.s, p3/M, z22.s, z0.s\n" + "fmla z30.s, p3/M, z22.s, z19.s\n" + "ld1w { z8.s }, p3/Z, [x10, #6, MUL VL]\n" + "fmla z27.s, p3/M, z20.s, z13.s\n" + "fmla z31.s, p3/M, z20.s, z24.s\n" + "ld1w { z2.s }, p2/Z, [x9, x26, LSL #2]\n" "addvl x9, x9, #1\n" - "fmla z30.s, p3/M, z2.s, z12.s\n" - "fmla z31.s, p3/M, z2.s, z9.s\n" - "ld1w { z2.s }, p3/Z, [x10, #7, MUL VL]\n" + "fmla z26.s, p3/M, z20.s, z19.s\n" + "fmla z30.s, p3/M, z20.s, z5.s\n" + "ld1w { z16.s }, p3/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "fmla z28.s, p3/M, z3.s, z5.s\n" - "fmla z29.s, p3/M, z3.s, z6.s\n" - "ld1w { z5.s }, p2/Z, [x27]\n" - "ld1w { z16.s }, p3/Z, [x10, #4, MUL VL]\n" - "fmla z30.s, p3/M, z3.s, z9.s\n" - "fmla z31.s, p3/M, z3.s, z13.s\n" - "ld1w { z3.s }, p3/Z, [x10, #-8, MUL VL]\n" - "fmla z28.s, p3/M, z4.s, z6.s\n" - "fmla z29.s, p3/M, z4.s, z10.s\n" - "ld1w { z6.s }, p2/Z, [x27, x17, LSL #2]\n" - "ld1w { z10.s }, p2/Z, [x27, x15, LSL #2]\n" - "fmla z30.s, p3/M, z4.s, z13.s\n" - "fmla z31.s, p3/M, z4.s, z8.s\n" - "ld1w { z4.s }, p3/Z, [x10, #-7, MUL VL]\n" - "fmla z28.s, p3/M, z0.s, z14.s\n" - "fmla z29.s, p3/M, z0.s, z11.s\n" - "ld1w { z14.s }, p2/Z, [x27, x24, LSL #2]\n" - "fmla z30.s, p3/M, z0.s, z5.s\n" - "fmla z31.s, p3/M, z0.s, z6.s\n" - "ld1w { z0.s }, p3/Z, [x10, #-6, MUL VL]\n" - "fmla z28.s, p3/M, z1.s, z11.s\n" - "fmla z29.s, p3/M, z1.s, z12.s\n" - "ld1w { z11.s }, p2/Z, [x27, x28, LSL #2]\n" - "fmla z30.s, p3/M, z1.s, z6.s\n" - "fmla z31.s, p3/M, z1.s, z10.s\n" - "ld1w { z1.s }, p3/Z, [x10, #-5, MUL VL]\n" - "fmla z28.s, p3/M, z2.s, z12.s\n" - "fmla z29.s, p3/M, z2.s, z9.s\n" - "ld1w { z12.s }, p2/Z, [x27, x26, LSL #2]\n" + "fmla z27.s, p3/M, z17.s, z24.s\n" + "fmla z31.s, p3/M, z17.s, z23.s\n" + "ld1w { z25.s }, p2/Z, [x27]\n" + "ld1w { z29.s }, p3/Z, [x10, #4, MUL VL]\n" + "fmla z26.s, p3/M, z17.s, z5.s\n" + "fmla z30.s, p3/M, z17.s, z2.s\n" + "ld1w { z17.s }, p3/Z, [x10, #-8, MUL VL]\n" + "fmla z27.s, p3/M, z21.s, z23.s\n" + "fmla z31.s, p3/M, z21.s, z10.s\n" + "ld1w { z24.s }, p2/Z, [x27, x17, LSL #2]\n" + "ld1w { z22.s }, p2/Z, [x27, x15, LSL #2]\n" + "fmla z26.s, p3/M, z21.s, z2.s\n" + "fmla z30.s, p3/M, z21.s, z3.s\n" + "ld1w { z21.s }, p3/Z, [x10, #-7, MUL VL]\n" + "fmla z27.s, p3/M, z18.s, z14.s\n" + "fmla z31.s, p3/M, z18.s, z0.s\n" + "ld1w { z1.s }, p2/Z, [x27, x24, LSL #2]\n" + "fmla z26.s, p3/M, z18.s, z25.s\n" + "fmla z30.s, p3/M, z18.s, z24.s\n" + "ld1w { z23.s }, p3/Z, [x10, #-6, MUL VL]\n" + "fmla z27.s, p3/M, z8.s, z0.s\n" + "fmla z31.s, p3/M, z8.s, z19.s\n" + "ld1w { z0.s }, p2/Z, [x27, x28, LSL #2]\n" + "fmla z26.s, p3/M, z8.s, z24.s\n" + "fmla z30.s, p3/M, z8.s, z22.s\n" + "ld1w { z20.s }, p3/Z, [x10, #-5, MUL VL]\n" + "fmla z27.s, p3/M, z16.s, z19.s\n" + "fmla z31.s, p3/M, z16.s, z5.s\n" + "ld1w { z19.s }, p2/Z, [x27, x26, LSL #2]\n" "addvl x27, x27, #1\n" - "fmla z30.s, p3/M, z2.s, z10.s\n" - "fmla z31.s, p3/M, z2.s, z11.s\n" - "ld1w { z2.s }, p3/Z, [x10, #-4, MUL VL]\n" - "fmla z28.s, p3/M, z3.s, z9.s\n" - "fmla z29.s, p3/M, z3.s, z13.s\n" - "ld1w { z9.s }, p2/Z, [x25]\n" - "fmla z30.s, p3/M, z3.s, z11.s\n" - "fmla z31.s, p3/M, z3.s, z12.s\n" - "ld1w { z3.s }, p3/Z, [x10, #-3, MUL VL]\n" - "fmla z28.s, p3/M, z4.s, z13.s\n" - "fmla z29.s, p3/M, z4.s, z8.s\n" - "ld1w { z13.s }, p2/Z, [x25, x17, LSL #2]\n" + "fmla z26.s, p3/M, z16.s, z22.s\n" + "fmla z30.s, p3/M, z16.s, z0.s\n" + "ld1w { z18.s }, p3/Z, [x10, #-4, MUL VL]\n" + "fmla z27.s, p3/M, z17.s, z5.s\n" + "fmla z31.s, p3/M, z17.s, z2.s\n" + "ld1w { z16.s }, p2/Z, [x25]\n" + "fmla z26.s, p3/M, z17.s, z0.s\n" + "fmla z30.s, p3/M, z17.s, z19.s\n" + "ld1w { z17.s }, p3/Z, [x10, #-3, MUL VL]\n" + "fmla z27.s, p3/M, z21.s, z2.s\n" + "fmla z31.s, p3/M, z21.s, z3.s\n" + "ld1w { z4.s }, p2/Z, [x25, x17, LSL #2]\n" "ld1w { z8.s }, p2/Z, [x25, x26, LSL #2]\n" - "fmla z30.s, p3/M, z4.s, z12.s\n" - "fmla z31.s, p3/M, z4.s, z14.s\n" - "ld1w { z4.s }, p3/Z, [x10, #-2, MUL VL]\n" - "fmla z28.s, p3/M, z0.s, z5.s\n" - "fmla z29.s, p3/M, z0.s, z6.s\n" - "ld1w { z5.s }, p2/Z, [x25, x15, LSL #2]\n" - "fmla z30.s, p3/M, z0.s, z9.s\n" - "fmla z31.s, p3/M, z0.s, z13.s\n" - "ld1w { z0.s }, p3/Z, [x10, #-1, MUL VL]\n" - "fmla z28.s, p3/M, z1.s, z6.s\n" - "fmla z29.s, p3/M, z1.s, z10.s\n" - "ld1w { z6.s }, p2/Z, [x25, x28, LSL #2]\n" - "fmla z30.s, p3/M, z1.s, z13.s\n" - "fmla z31.s, p3/M, z1.s, z5.s\n" - "ld1w { z1.s }, p3/Z, [x10]\n" - "fmla z28.s, p3/M, z2.s, z10.s\n" - "fmla z29.s, p3/M, z2.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x25, x24, LSL #2]\n" + "fmla z26.s, p3/M, z21.s, z19.s\n" + "fmla z30.s, p3/M, z21.s, z1.s\n" + "ld1w { z13.s }, p3/Z, [x10, #-2, MUL VL]\n" + "fmla z27.s, p3/M, z23.s, z25.s\n" + "fmla z31.s, p3/M, z23.s, z24.s\n" + "ld1w { z25.s }, p2/Z, [x25, x15, LSL #2]\n" + "fmla z26.s, p3/M, z23.s, z16.s\n" + "fmla z30.s, p3/M, z23.s, z4.s\n" + "ld1w { z5.s }, p3/Z, [x10, #-1, MUL VL]\n" + "fmla z27.s, p3/M, z20.s, z24.s\n" + "fmla z31.s, p3/M, z20.s, z22.s\n" + "ld1w { z24.s }, p2/Z, [x25, x28, LSL #2]\n" + "fmla z26.s, p3/M, z20.s, z4.s\n" + "fmla z30.s, p3/M, z20.s, z25.s\n" + "ld1w { z23.s }, p3/Z, [x10]\n" + "fmla z27.s, p3/M, z18.s, z22.s\n" + "fmla z31.s, p3/M, z18.s, z0.s\n" + "ld1w { z22.s }, p2/Z, [x25, x24, LSL #2]\n" "addvl x25, x25, #1\n" - "fmla z30.s, p3/M, z2.s, z5.s\n" - "fmla z31.s, p3/M, z2.s, z6.s\n" - "ld1w { z2.s }, p3/Z, [x10, #1, MUL VL]\n" - "fmla z28.s, p3/M, z3.s, z11.s\n" - "fmla z29.s, p3/M, z3.s, z12.s\n" - "ld1w { z11.s }, p2/Z, [x23]\n" - "fmla z30.s, p3/M, z3.s, z6.s\n" - "fmla z31.s, p3/M, z3.s, z8.s\n" - "ld1w { z3.s }, p3/Z, [x10, #2, MUL VL]\n" - "fmla z28.s, p3/M, z4.s, z12.s\n" - "fmla z29.s, p3/M, z4.s, z14.s\n" - "ld1w { z12.s }, p2/Z, [x23, x17, LSL #2]\n" + "fmla z26.s, p3/M, z18.s, z25.s\n" + "fmla z30.s, p3/M, z18.s, z24.s\n" + "ld1w { z21.s }, p3/Z, [x10, #1, MUL VL]\n" + "fmla z27.s, p3/M, z17.s, z0.s\n" + "fmla z31.s, p3/M, z17.s, z19.s\n" + "ld1w { z18.s }, p2/Z, [x23]\n" + "fmla z26.s, p3/M, z17.s, z24.s\n" + "fmla z30.s, p3/M, z17.s, z8.s\n" + "ld1w { z20.s }, p3/Z, [x10, #2, MUL VL]\n" + "fmla z27.s, p3/M, z13.s, z19.s\n" + "fmla z31.s, p3/M, z13.s, z1.s\n" + "ld1w { z17.s }, p2/Z, [x23, x17, LSL #2]\n" "ld1w { z14.s }, p1/Z, [x9]\n" - "fmla z30.s, p3/M, z4.s, z8.s\n" - "fmla z31.s, p3/M, z4.s, z10.s\n" - "ld1w { z4.s }, p3/Z, [x10, #3, MUL VL]\n" - "fmla z28.s, p3/M, z0.s, z9.s\n" - "fmla z29.s, p3/M, z0.s, z13.s\n" - "ld1w { z9.s }, p2/Z, [x23, x15, LSL #2]\n" - "fmla z30.s, p3/M, z0.s, z11.s\n" - "fmla z31.s, p3/M, z0.s, z12.s\n" - "ld1w { z11.s }, p2/Z, [x23, x28, LSL #2]\n" + "fmla z26.s, p3/M, z13.s, z8.s\n" + "fmla z30.s, p3/M, z13.s, z22.s\n" + "ld1w { z19.s }, p3/Z, [x10, #3, MUL VL]\n" + "fmla z27.s, p3/M, z5.s, z16.s\n" + "fmla z31.s, p3/M, z5.s, z4.s\n" + "ld1w { z16.s }, p2/Z, [x23, x15, LSL #2]\n" + "fmla z26.s, p3/M, z5.s, z18.s\n" + "fmla z30.s, p3/M, z5.s, z17.s\n" + "ld1w { z18.s }, p2/Z, [x23, x28, LSL #2]\n" "ld1w { z0.s }, p3/Z, [x10, #5, MUL VL]\n" - "fmla z28.s, p3/M, z1.s, z13.s\n" - "fmla z29.s, p3/M, z1.s, z5.s\n" + "fmla z27.s, p3/M, z23.s, z4.s\n" + "fmla z31.s, p3/M, z23.s, z25.s\n" "ld1w { z13.s }, p1/Z, [x11, x15, LSL #2]\n" - "fmla z30.s, p3/M, z1.s, z12.s\n" - "fmla z31.s, p3/M, z1.s, z9.s\n" - "ld1w { z12.s }, p2/Z, [x23, x26, LSL #2]\n" + "fmla z26.s, p3/M, z23.s, z17.s\n" + "fmla z30.s, p3/M, z23.s, z16.s\n" + "ld1w { z17.s }, p2/Z, [x23, x26, LSL #2]\n" "ld1w { z1.s }, p3/Z, [x10, #6, MUL VL]\n" - "fmla z28.s, p3/M, z2.s, z5.s\n" - "fmla z29.s, p3/M, z2.s, z6.s\n" + "fmla z27.s, p3/M, z21.s, z25.s\n" + "fmla z31.s, p3/M, z21.s, z24.s\n" "ld1w { z5.s }, p1/Z, [x14]\n" - "fmla z30.s, p3/M, z2.s, z9.s\n" - "fmla z31.s, p3/M, z2.s, z11.s\n" - "ld1w { z9.s }, p2/Z, [x23, x24, LSL #2]\n" + "fmla z26.s, p3/M, z21.s, z16.s\n" + "fmla z30.s, p3/M, z21.s, z18.s\n" + "ld1w { z16.s }, p2/Z, [x23, x24, LSL #2]\n" "ld1w { z2.s }, p3/Z, [x10, #7, MUL VL]\n" - "fmla z28.s, p3/M, z3.s, z6.s\n" - "fmla z29.s, p3/M, z3.s, z8.s\n" + "fmla z27.s, p3/M, z20.s, z24.s\n" + "fmla z31.s, p3/M, z20.s, z8.s\n" "addvl x10, x10, #16\n" "whilelt p2.s, x21, %x[n_channels]\n" - "fmla z30.s, p3/M, z3.s, z11.s\n" - "fmla z31.s, p3/M, z3.s, z12.s\n" + "fmla z26.s, p3/M, z20.s, z18.s\n" + "fmla z30.s, p3/M, z20.s, z17.s\n" "cmp x12, %x[n_channels]\n" "addvl x23, x23, #1\n" - "fmla z28.s, p3/M, z4.s, z8.s\n" - "fmla z29.s, p3/M, z4.s, z10.s\n" - "fmax z28.s, p3/M, z28.s, z18.s\n" - "fmax z29.s, p3/M, z29.s, z18.s\n" - "fmla z30.s, p3/M, z4.s, z12.s\n" - "fmla z31.s, p3/M, z4.s, z9.s\n" - "fmax z30.s, p3/M, z30.s, z18.s\n" - "fmax z31.s, p3/M, z31.s, z18.s\n" - "fmin z28.s, p3/M, z28.s, z17.s\n" - "fmin z29.s, p3/M, z29.s, z17.s\n" + "fmla z27.s, p3/M, z19.s, z8.s\n" + "fmla z31.s, p3/M, z19.s, z22.s\n" + "fmax z27.s, p3/M, z27.s, z15.s\n" + "fmax z31.s, p3/M, z31.s, z15.s\n" + "fmla z26.s, p3/M, z19.s, z17.s\n" + "fmla z30.s, p3/M, z19.s, z16.s\n" + "fmax z26.s, p3/M, z26.s, z15.s\n" + "fmax z30.s, p3/M, z30.s, z15.s\n" + "fmin z27.s, p3/M, z27.s, z28.s\n" + "fmin z31.s, p3/M, z31.s, z28.s\n" "ld1w { z6.s }, p1/Z, [x14, x17, LSL #2]\n" "ld1w { z8.s }, p1/Z, [x11, x17, LSL #2]\n" - "fmin z30.s, p3/M, z30.s, z17.s\n" - "fmin z31.s, p3/M, z31.s, z17.s\n" + "fmin z26.s, p3/M, z26.s, z28.s\n" + "fmin z30.s, p3/M, z30.s, z28.s\n" "ld1w { z9.s }, p1/Z, [x14, x15, LSL #2]\n" "ld1w { z11.s }, p1/Z, [x14, x28, LSL #2]\n" "ld1w { z12.s }, p1/Z, [x14, x26, LSL #2]\n" "ld1w { z10.s }, p1/Z, [x11, x24, LSL #2]\n" - "st1w { z28.s }, p0, [x13]\n" - "st1w { z29.s }, p0, [x13, x16, LSL #2]\n" + "st1w { z27.s }, p0, [x13]\n" + "st1w { z31.s }, p0, [x13, x16, LSL #2]\n" "addvl x13, x13, #1\n" "ld1w { z3.s }, p3/Z, [x10, #-8, MUL VL]\n" "ld1w { z4.s }, p3/Z, [x10, #-7, MUL VL]\n" - "st1w { z30.s }, p0, [x22]\n" + "st1w { z26.s }, p0, [x22]\n" "addvl x10, x10, #-6\n" - "st1w { z31.s }, p0, [x22, x16, LSL #2]\n" + "st1w { z30.s }, p0, [x22, x16, LSL #2]\n" "addvl x22, x22, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z5.s\n" - "movprfx z29, z16\n fmla z29.s, p3/M, z0.s, z6.s\n" - "ld1w { z5.s }, p2/Z, [x11, x28, LSL #2]\n" + "movprfx z30, z29\n fmla z30.s, p3/M, z0.s, z5.s\n" + "movprfx z31, z29\n fmla z31.s, p3/M, z0.s, z6.s\n" + "ld1w { z22.s }, p2/Z, [x11, x28, LSL #2]\n" "ldr x8, [%x[params_struct], %[offsetof_args_tile_j]]\n" - "movprfx z30, z16\n fmla z30.s, p3/M, z0.s, z7.s\n" - "movprfx z31, z16\n fmla z31.s, p3/M, z0.s, z8.s\n" - "ld1w { z0.s }, p3/Z, [x10]\n" + "movprfx z5, z29\n fmla z5.s, p3/M, z0.s, z7.s\n" + "fmla z29.s, p3/M, z0.s, z8.s\n" + "ld1w { z20.s }, p3/Z, [x10]\n" "ldr x12, [%x[params_struct], %[offsetof_args_tile_i]]\n" - "fmla z28.s, p3/M, z1.s, z6.s\n" - "fmla z29.s, p3/M, z1.s, z9.s\n" + "fmla z30.s, p3/M, z1.s, z6.s\n" + "fmla z31.s, p3/M, z1.s, z9.s\n" "ld1w { z6.s }, p2/Z, [x11, x26, LSL #2]\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" - "fmla z30.s, p3/M, z1.s, z8.s\n" - "fmla z31.s, p3/M, z1.s, z13.s\n" - "ld1w { z1.s }, p3/Z, [x10, #1, MUL VL]\n" + "fmla z5.s, p3/M, z1.s, z8.s\n" + "fmla z29.s, p3/M, z1.s, z13.s\n" + "ld1w { z19.s }, p3/Z, [x10, #1, MUL VL]\n" "add x8, x8, #0x1\n" - "fmla z28.s, p3/M, z2.s, z9.s\n" - "fmla z29.s, p3/M, z2.s, z11.s\n" - "ld1w { z9.s }, p2/Z, [x14, x24, LSL #2]\n" + "fmla z30.s, p3/M, z2.s, z9.s\n" + "fmla z31.s, p3/M, z2.s, z11.s\n" + "ld1w { z16.s }, p2/Z, [x14, x24, LSL #2]\n" "cmp x8, x20\n" - "fmla z30.s, p3/M, z2.s, z13.s\n" - "fmla z31.s, p3/M, z2.s, z5.s\n" - "ld1w { z2.s }, p3/Z, [x10, #2, MUL VL]\n" + "fmla z5.s, p3/M, z2.s, z13.s\n" + "fmla z29.s, p3/M, z2.s, z22.s\n" + "ld1w { z18.s }, p3/Z, [x10, #2, MUL VL]\n" "add x21, x12, #0x1\n" - "fmla z28.s, p3/M, z3.s, z11.s\n" - "fmla z29.s, p3/M, z3.s, z12.s\n" - "ld1w { z11.s }, p2/Z, [x9, x17, LSL #2]\n" + "fmla z30.s, p3/M, z3.s, z11.s\n" + "fmla z31.s, p3/M, z3.s, z12.s\n" + "ld1w { z1.s }, p2/Z, [x9, x17, LSL #2]\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" - "fmla z30.s, p3/M, z3.s, z5.s\n" - "fmla z31.s, p3/M, z3.s, z6.s\n" - "ld1w { z3.s }, p3/Z, [x10, #3, MUL VL]\n" + "fmla z5.s, p3/M, z3.s, z22.s\n" + "fmla z29.s, p3/M, z3.s, z6.s\n" + "ld1w { z17.s }, p3/Z, [x10, #3, MUL VL]\n" "csel x12, x12, x21, LT\n" - "fmla z28.s, p3/M, z4.s, z12.s\n" - "fmla z29.s, p3/M, z4.s, z9.s\n" - "ld1w { z12.s }, p2/Z, [x9, x15, LSL #2]\n" - "ld1w { z9.s }, p2/Z, [x9, x28, LSL #2]\n" - "fmla z30.s, p3/M, z4.s, z6.s\n" - "fmla z31.s, p3/M, z4.s, z10.s\n" - "ld1w { z4.s }, p3/Z, [x10, #4, MUL VL]\n" + "fmla z30.s, p3/M, z4.s, z12.s\n" + "fmla z31.s, p3/M, z4.s, z16.s\n" + "ld1w { z0.s }, p2/Z, [x9, x15, LSL #2]\n" + "ld1w { z27.s }, p2/Z, [x9, x28, LSL #2]\n" + "fmla z5.s, p3/M, z4.s, z6.s\n" + "fmla z29.s, p3/M, z4.s, z10.s\n" + "ld1w { z16.s }, p3/Z, [x10, #4, MUL VL]\n" "mov p0.b, p2.b\n" - "fmla z28.s, p3/M, z0.s, z7.s\n" - "fmla z29.s, p3/M, z0.s, z8.s\n" + "fmla z30.s, p3/M, z20.s, z7.s\n" + "fmla z31.s, p3/M, z20.s, z8.s\n" "csel x8, x8, XZR, LT\n" "cmp x12, x20\n" - "fmla z30.s, p3/M, z0.s, z14.s\n" - "fmla z31.s, p3/M, z0.s, z11.s\n" - "ld1w { z0.s }, p3/Z, [x10, #5, MUL VL]\n" - "fmla z28.s, p3/M, z1.s, z8.s\n" - "fmla z29.s, p3/M, z1.s, z13.s\n" - "ld1w { z8.s }, p2/Z, [x9, x24, LSL #2]\n" - "fmla z30.s, p3/M, z1.s, z11.s\n" - "fmla z31.s, p3/M, z1.s, z12.s\n" - "ld1w { z1.s }, p3/Z, [x10, #6, MUL VL]\n" - "fmla z28.s, p3/M, z2.s, z13.s\n" - "fmla z29.s, p3/M, z2.s, z5.s\n" - "ld1w { z13.s }, p2/Z, [x9, x26, LSL #2]\n" - "fmla z30.s, p3/M, z2.s, z12.s\n" - "fmla z31.s, p3/M, z2.s, z9.s\n" - "ld1w { z2.s }, p3/Z, [x10, #7, MUL VL]\n" + "fmla z5.s, p3/M, z20.s, z14.s\n" + "fmla z29.s, p3/M, z20.s, z1.s\n" + "ld1w { z21.s }, p3/Z, [x10, #5, MUL VL]\n" + "fmla z30.s, p3/M, z19.s, z8.s\n" + "fmla z31.s, p3/M, z19.s, z13.s\n" + "ld1w { z26.s }, p2/Z, [x9, x24, LSL #2]\n" + "fmla z5.s, p3/M, z19.s, z1.s\n" + "fmla z29.s, p3/M, z19.s, z0.s\n" + "ld1w { z25.s }, p3/Z, [x10, #6, MUL VL]\n" + "fmla z30.s, p3/M, z18.s, z13.s\n" + "fmla z31.s, p3/M, z18.s, z22.s\n" + "ld1w { z24.s }, p2/Z, [x9, x26, LSL #2]\n" + "fmla z5.s, p3/M, z18.s, z0.s\n" + "fmla z29.s, p3/M, z18.s, z27.s\n" + "ld1w { z23.s }, p3/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "fmla z28.s, p3/M, z3.s, z5.s\n" - "fmla z29.s, p3/M, z3.s, z6.s\n" - "ld1w { z5.s }, p2/Z, [x27]\n" - "fmla z30.s, p3/M, z3.s, z9.s\n" - "fmla z31.s, p3/M, z3.s, z13.s\n" - "ld1w { z3.s }, p3/Z, [x10, #-8, MUL VL]\n" - "fmla z28.s, p3/M, z4.s, z6.s\n" - "fmla z29.s, p3/M, z4.s, z10.s\n" - "ld1w { z6.s }, p2/Z, [x27, x17, LSL #2]\n" - "ld1w { z10.s }, p2/Z, [x27, x15, LSL #2]\n" - "fmla z30.s, p3/M, z4.s, z13.s\n" - "fmla z31.s, p3/M, z4.s, z8.s\n" - "ld1w { z4.s }, p3/Z, [x10, #-7, MUL VL]\n" - "fmla z28.s, p3/M, z0.s, z14.s\n" - "fmla z29.s, p3/M, z0.s, z11.s\n" - "ld1w { z14.s }, p2/Z, [x27, x24, LSL #2]\n" - "fmla z30.s, p3/M, z0.s, z5.s\n" - "fmla z31.s, p3/M, z0.s, z6.s\n" - "ld1w { z0.s }, p3/Z, [x10, #-6, MUL VL]\n" - "fmla z28.s, p3/M, z1.s, z11.s\n" - "fmla z29.s, p3/M, z1.s, z12.s\n" - "ld1w { z11.s }, p2/Z, [x27, x28, LSL #2]\n" - "fmla z30.s, p3/M, z1.s, z6.s\n" - "fmla z31.s, p3/M, z1.s, z10.s\n" - "ld1w { z1.s }, p3/Z, [x10, #-5, MUL VL]\n" - "fmla z28.s, p3/M, z2.s, z12.s\n" - "fmla z29.s, p3/M, z2.s, z9.s\n" - "ld1w { z12.s }, p2/Z, [x27, x26, LSL #2]\n" - "fmla z30.s, p3/M, z2.s, z10.s\n" - "fmla z31.s, p3/M, z2.s, z11.s\n" - "ld1w { z2.s }, p3/Z, [x10, #-4, MUL VL]\n" - "fmla z28.s, p3/M, z3.s, z9.s\n" - "fmla z29.s, p3/M, z3.s, z13.s\n" - "ld1w { z9.s }, p2/Z, [x25]\n" - "fmla z30.s, p3/M, z3.s, z11.s\n" - "fmla z31.s, p3/M, z3.s, z12.s\n" - "ld1w { z3.s }, p3/Z, [x10, #-3, MUL VL]\n" - "fmla z28.s, p3/M, z4.s, z13.s\n" - "fmla z29.s, p3/M, z4.s, z8.s\n" - "ld1w { z13.s }, p2/Z, [x25, x17, LSL #2]\n" - "ld1w { z8.s }, p2/Z, [x25, x26, LSL #2]\n" - "fmla z30.s, p3/M, z4.s, z12.s\n" - "fmla z31.s, p3/M, z4.s, z14.s\n" - "ld1w { z4.s }, p3/Z, [x10, #-2, MUL VL]\n" - "fmla z28.s, p3/M, z0.s, z5.s\n" - "fmla z29.s, p3/M, z0.s, z6.s\n" - "ld1w { z5.s }, p2/Z, [x25, x15, LSL #2]\n" - "fmla z30.s, p3/M, z0.s, z9.s\n" - "fmla z31.s, p3/M, z0.s, z13.s\n" - "ld1w { z0.s }, p3/Z, [x10, #-1, MUL VL]\n" - "fmla z28.s, p3/M, z1.s, z6.s\n" - "fmla z29.s, p3/M, z1.s, z10.s\n" - "ld1w { z6.s }, p2/Z, [x25, x28, LSL #2]\n" - "fmla z30.s, p3/M, z1.s, z13.s\n" - "fmla z31.s, p3/M, z1.s, z5.s\n" - "ld1w { z1.s }, p3/Z, [x10]\n" - "fmla z28.s, p3/M, z2.s, z10.s\n" - "fmla z29.s, p3/M, z2.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x25, x24, LSL #2]\n" - "fmla z30.s, p3/M, z2.s, z5.s\n" - "fmla z31.s, p3/M, z2.s, z6.s\n" - "ld1w { z2.s }, p3/Z, [x10, #1, MUL VL]\n" - "fmla z28.s, p3/M, z3.s, z11.s\n" - "fmla z29.s, p3/M, z3.s, z12.s\n" - "ld1w { z11.s }, p2/Z, [x23]\n" - "fmla z30.s, p3/M, z3.s, z6.s\n" - "fmla z31.s, p3/M, z3.s, z8.s\n" - "ld1w { z3.s }, p3/Z, [x10, #2, MUL VL]\n" - "fmla z28.s, p3/M, z4.s, z12.s\n" - "fmla z29.s, p3/M, z4.s, z14.s\n" - "ld1w { z12.s }, p2/Z, [x23, x17, LSL #2]\n" - "fmla z30.s, p3/M, z4.s, z8.s\n" - "fmla z31.s, p3/M, z4.s, z10.s\n" - "ld1w { z4.s }, p3/Z, [x10, #3, MUL VL]\n" - "fmla z28.s, p3/M, z0.s, z9.s\n" - "fmla z29.s, p3/M, z0.s, z13.s\n" - "ld1w { z9.s }, p2/Z, [x23, x15, LSL #2]\n" - "fmla z30.s, p3/M, z0.s, z11.s\n" - "fmla z31.s, p3/M, z0.s, z12.s\n" - "ld1w { z11.s }, p2/Z, [x23, x28, LSL #2]\n" - "fmla z28.s, p3/M, z1.s, z13.s\n" - "fmla z29.s, p3/M, z1.s, z5.s\n" - "fmla z30.s, p3/M, z1.s, z12.s\n" - "fmla z31.s, p3/M, z1.s, z9.s\n" - "ld1w { z12.s }, p2/Z, [x23, x26, LSL #2]\n" - "fmla z28.s, p3/M, z2.s, z5.s\n" - "fmla z29.s, p3/M, z2.s, z6.s\n" - "fmla z30.s, p3/M, z2.s, z9.s\n" - "fmla z31.s, p3/M, z2.s, z11.s\n" - "ld1w { z9.s }, p2/Z, [x23, x24, LSL #2]\n" - "fmla z28.s, p3/M, z3.s, z6.s\n" - "fmla z29.s, p3/M, z3.s, z8.s\n" - "fmla z30.s, p3/M, z3.s, z11.s\n" - "fmla z31.s, p3/M, z3.s, z12.s\n" - "fmla z28.s, p3/M, z4.s, z8.s\n" - "fmla z29.s, p3/M, z4.s, z10.s\n" - "fmax z28.s, p3/M, z28.s, z18.s\n" - "fmax z29.s, p3/M, z29.s, z18.s\n" - "fmla z30.s, p3/M, z4.s, z12.s\n" - "fmla z31.s, p3/M, z4.s, z9.s\n" - "fmax z30.s, p3/M, z30.s, z18.s\n" - "fmax z31.s, p3/M, z31.s, z18.s\n" - "fmin z28.s, p3/M, z28.s, z17.s\n" - "fmin z29.s, p3/M, z29.s, z17.s\n" - "st1w { z28.s }, p0, [x13]\n" - "fmin z30.s, p3/M, z30.s, z17.s\n" - "fmin z31.s, p3/M, z31.s, z17.s\n" - "st1w { z29.s }, p0, [x13, x16, LSL #2]\n" - "st1w { z30.s }, p0, [x22]\n" - "st1w { z31.s }, p0, [x22, x16, LSL #2]\n" + "fmla z30.s, p3/M, z17.s, z22.s\n" + "fmla z31.s, p3/M, z17.s, z6.s\n" + "ld1w { z22.s }, p2/Z, [x27]\n" + "fmla z5.s, p3/M, z17.s, z27.s\n" + "fmla z29.s, p3/M, z17.s, z24.s\n" + "ld1w { z20.s }, p3/Z, [x10, #-8, MUL VL]\n" + "fmla z30.s, p3/M, z16.s, z6.s\n" + "fmla z31.s, p3/M, z16.s, z10.s\n" + "ld1w { z19.s }, p2/Z, [x27, x17, LSL #2]\n" + "ld1w { z18.s }, p2/Z, [x27, x15, LSL #2]\n" + "fmla z5.s, p3/M, z16.s, z24.s\n" + "fmla z29.s, p3/M, z16.s, z26.s\n" + "ld1w { z16.s }, p3/Z, [x10, #-7, MUL VL]\n" + "fmla z30.s, p3/M, z21.s, z14.s\n" + "fmla z31.s, p3/M, z21.s, z1.s\n" + "ld1w { z17.s }, p2/Z, [x27, x24, LSL #2]\n" + "fmla z5.s, p3/M, z21.s, z22.s\n" + "fmla z29.s, p3/M, z21.s, z19.s\n" + "ld1w { z21.s }, p3/Z, [x10, #-6, MUL VL]\n" + "fmla z30.s, p3/M, z25.s, z1.s\n" + "fmla z31.s, p3/M, z25.s, z0.s\n" + "ld1w { z7.s }, p2/Z, [x27, x28, LSL #2]\n" + "fmla z5.s, p3/M, z25.s, z19.s\n" + "fmla z29.s, p3/M, z25.s, z18.s\n" + "ld1w { z10.s }, p3/Z, [x10, #-5, MUL VL]\n" + "fmla z30.s, p3/M, z23.s, z0.s\n" + "fmla z31.s, p3/M, z23.s, z27.s\n" + "ld1w { z11.s }, p2/Z, [x27, x26, LSL #2]\n" + "fmla z5.s, p3/M, z23.s, z18.s\n" + "fmla z29.s, p3/M, z23.s, z7.s\n" + "ld1w { z6.s }, p3/Z, [x10, #-4, MUL VL]\n" + "fmla z30.s, p3/M, z20.s, z27.s\n" + "fmla z31.s, p3/M, z20.s, z24.s\n" + "ld1w { z0.s }, p2/Z, [x25]\n" + "fmla z5.s, p3/M, z20.s, z7.s\n" + "fmla z29.s, p3/M, z20.s, z11.s\n" + "ld1w { z9.s }, p3/Z, [x10, #-3, MUL VL]\n" + "fmla z30.s, p3/M, z16.s, z24.s\n" + "fmla z31.s, p3/M, z16.s, z26.s\n" + "ld1w { z3.s }, p2/Z, [x25, x17, LSL #2]\n" + "ld1w { z27.s }, p2/Z, [x25, x26, LSL #2]\n" + "fmla z5.s, p3/M, z16.s, z11.s\n" + "fmla z29.s, p3/M, z16.s, z17.s\n" + "ld1w { z16.s }, p3/Z, [x10, #-2, MUL VL]\n" + "fmla z30.s, p3/M, z21.s, z22.s\n" + "fmla z31.s, p3/M, z21.s, z19.s\n" + "ld1w { z26.s }, p2/Z, [x25, x15, LSL #2]\n" + "fmla z5.s, p3/M, z21.s, z0.s\n" + "fmla z29.s, p3/M, z21.s, z3.s\n" + "ld1w { z25.s }, p3/Z, [x10, #-1, MUL VL]\n" + "fmla z30.s, p3/M, z10.s, z19.s\n" + "fmla z31.s, p3/M, z10.s, z18.s\n" + "ld1w { z24.s }, p2/Z, [x25, x28, LSL #2]\n" + "fmla z5.s, p3/M, z10.s, z3.s\n" + "fmla z29.s, p3/M, z10.s, z26.s\n" + "ld1w { z23.s }, p3/Z, [x10]\n" + "fmla z30.s, p3/M, z6.s, z18.s\n" + "fmla z31.s, p3/M, z6.s, z7.s\n" + "ld1w { z22.s }, p2/Z, [x25, x24, LSL #2]\n" + "fmla z5.s, p3/M, z6.s, z26.s\n" + "fmla z29.s, p3/M, z6.s, z24.s\n" + "ld1w { z21.s }, p3/Z, [x10, #1, MUL VL]\n" + "fmla z30.s, p3/M, z9.s, z7.s\n" + "fmla z31.s, p3/M, z9.s, z11.s\n" + "ld1w { z18.s }, p2/Z, [x23]\n" + "fmla z5.s, p3/M, z9.s, z24.s\n" + "fmla z29.s, p3/M, z9.s, z27.s\n" + "ld1w { z20.s }, p3/Z, [x10, #2, MUL VL]\n" + "fmla z30.s, p3/M, z16.s, z11.s\n" + "fmla z31.s, p3/M, z16.s, z17.s\n" + "ld1w { z17.s }, p2/Z, [x23, x17, LSL #2]\n" + "fmla z5.s, p3/M, z16.s, z27.s\n" + "fmla z29.s, p3/M, z16.s, z22.s\n" + "ld1w { z19.s }, p3/Z, [x10, #3, MUL VL]\n" + "fmla z30.s, p3/M, z25.s, z0.s\n" + "fmla z31.s, p3/M, z25.s, z3.s\n" + "ld1w { z16.s }, p2/Z, [x23, x15, LSL #2]\n" + "fmla z5.s, p3/M, z25.s, z18.s\n" + "fmla z29.s, p3/M, z25.s, z17.s\n" + "ld1w { z18.s }, p2/Z, [x23, x28, LSL #2]\n" + "fmla z30.s, p3/M, z23.s, z3.s\n" + "fmla z31.s, p3/M, z23.s, z26.s\n" + "fmla z5.s, p3/M, z23.s, z17.s\n" + "fmla z29.s, p3/M, z23.s, z16.s\n" + "ld1w { z17.s }, p2/Z, [x23, x26, LSL #2]\n" + "fmla z30.s, p3/M, z21.s, z26.s\n" + "fmla z31.s, p3/M, z21.s, z24.s\n" + "fmla z5.s, p3/M, z21.s, z16.s\n" + "fmla z29.s, p3/M, z21.s, z18.s\n" + "ld1w { z16.s }, p2/Z, [x23, x24, LSL #2]\n" + "fmla z30.s, p3/M, z20.s, z24.s\n" + "fmla z31.s, p3/M, z20.s, z27.s\n" + "fmla z5.s, p3/M, z20.s, z18.s\n" + "fmla z29.s, p3/M, z20.s, z17.s\n" + "fmla z30.s, p3/M, z19.s, z27.s\n" + "fmla z31.s, p3/M, z19.s, z22.s\n" + "fmax z30.s, p3/M, z30.s, z15.s\n" + "fmax z31.s, p3/M, z31.s, z15.s\n" + "fmla z5.s, p3/M, z19.s, z17.s\n" + "fmla z29.s, p3/M, z19.s, z16.s\n" + "fmax z5.s, p3/M, z5.s, z15.s\n" + "fmax z29.s, p3/M, z29.s, z15.s\n" + "fmin z30.s, p3/M, z30.s, z28.s\n" + "fmin z31.s, p3/M, z31.s, z28.s\n" + "st1w { z30.s }, p0, [x13]\n" + "fmin z5.s, p3/M, z5.s, z28.s\n" + "fmin z29.s, p3/M, z29.s, z28.s\n" + "st1w { z31.s }, p0, [x13, x16, LSL #2]\n" + "st1w { z5.s }, p0, [x22]\n" + "st1w { z29.s }, p0, [x22, x16, LSL #2]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp index cb70bd2b6f4af81de06ec6d5833c0bd0afaa4ff5..bf65e04d32ed2ce85513693b4f6e6144a52185cf 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -104,448 +104,448 @@ void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl( "mov x13, #0x0\n" "ldp x12, x11, [x20, #0x10]\n" "whilelt p3.s, XZR, %x[n_channels]\n" - "ldp x10, x9, [x16, #0x0]\n" - "cntw x28\n" + "ldp x21, x20, [x16, #0x0]\n" + "cntw x10\n" "ptrue p2.b\n" - "ldr x27, [%x[params_struct], %[offsetof_args_params]]\n" - "ld1w { z5.s }, p3/Z, [x10, x13, LSL #2]\n" - "cmp x28, %x[n_channels]\n" - "ld1w { z6.s }, p3/Z, [x9, x13, LSL #2]\n" - "ldp x26, x25, [x16, #0x10]\n" - "sub x24, XZR, x28\n" - "ldp x23, x22, [x16, #0x20]\n" - "ldp x21, x20, [x16, #0x30]\n" - "ldp x10, x9, [x16, #0x40]\n" - "ld1rw { z18.s }, p2/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "ld1rw { z17.s }, p2/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "ld1w { z16.s }, p2/Z, [x27]\n" - "ld1w { z0.s }, p2/Z, [x27, #1, MUL VL]\n" - "ld1w { z1.s }, p2/Z, [x27, #2, MUL VL]\n" - "ld1w { z2.s }, p2/Z, [x27, #3, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x27, #4, MUL VL]\n" - "ld1w { z4.s }, p2/Z, [x27, #5, MUL VL]\n" - "ld1w { z7.s }, p3/Z, [x26, x13, LSL #2]\n" - "addvl x27, x27, #6\n" - "ld1w { z8.s }, p3/Z, [x25, x13, LSL #2]\n" - "ld1w { z9.s }, p3/Z, [x23, x13, LSL #2]\n" - "ld1w { z13.s }, p3/Z, [x22, x13, LSL #2]\n" - "ld1w { z11.s }, p3/Z, [x21, x13, LSL #2]\n" - "ld1w { z12.s }, p3/Z, [x20, x13, LSL #2]\n" - "ld1w { z10.s }, p3/Z, [x10, x13, LSL #2]\n" - "ld1w { z14.s }, p3/Z, [x9, x13, LSL #2]\n" + "ldr x9, [%x[params_struct], %[offsetof_args_params]]\n" + "ld1w { z5.s }, p3/Z, [x21, x13, LSL #2]\n" + "cmp x10, %x[n_channels]\n" + "ld1w { z6.s }, p3/Z, [x20, x13, LSL #2]\n" + "ldp x27, x26, [x16, #0x10]\n" + "sub x28, XZR, x10\n" + "ldp x25, x24, [x16, #0x20]\n" + "ldp x23, x22, [x16, #0x30]\n" + "ldp x21, x20, [x16, #0x40]\n" + "ld1rw { z15.s }, p2/Z, [%x[params_struct], %[offsetof_args_min]]\n" + "ld1rw { z28.s }, p2/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "ld1w { z29.s }, p2/Z, [x9]\n" + "ld1w { z0.s }, p2/Z, [x9, #1, MUL VL]\n" + "ld1w { z1.s }, p2/Z, [x9, #2, MUL VL]\n" + "ld1w { z2.s }, p2/Z, [x9, #3, MUL VL]\n" + "ld1w { z3.s }, p2/Z, [x9, #4, MUL VL]\n" + "ld1w { z4.s }, p2/Z, [x9, #5, MUL VL]\n" + "ld1w { z7.s }, p3/Z, [x27, x13, LSL #2]\n" + "addvl x9, x9, #6\n" + "ld1w { z8.s }, p3/Z, [x26, x13, LSL #2]\n" + "ld1w { z9.s }, p3/Z, [x25, x13, LSL #2]\n" + "ld1w { z13.s }, p3/Z, [x24, x13, LSL #2]\n" + "ld1w { z11.s }, p3/Z, [x23, x13, LSL #2]\n" + "ld1w { z12.s }, p3/Z, [x22, x13, LSL #2]\n" + "ld1w { z10.s }, p3/Z, [x21, x13, LSL #2]\n" + "ld1w { z14.s }, p3/Z, [x20, x13, LSL #2]\n" "bge 2f\n" "1:" // Channel loop - "movprfx z28, z16\n fmla z28.s, p2/M, z0.s, z5.s\n" - "movprfx z29, z16\n fmla z29.s, p2/M, z0.s, z6.s\n" - "ldr x26, [x16, #0x50]\n" - "ld1w { z5.s }, p3/Z, [x26, x13, LSL #2]\n" - "movprfx z30, z16\n fmla z30.s, p2/M, z0.s, z7.s\n" - "movprfx z31, z16\n fmla z31.s, p2/M, z0.s, z8.s\n" - "ldr x25, [x16, #0x58]\n" - "ldr x23, [x16, #0x60]\n" - "fmla z28.s, p2/M, z1.s, z6.s\n" - "fmla z29.s, p2/M, z1.s, z9.s\n" - "ld1w { z6.s }, p3/Z, [x25, x13, LSL #2]\n" - "ldr x22, [x16, #0x68]\n" - "fmla z30.s, p2/M, z1.s, z8.s\n" - "fmla z31.s, p2/M, z1.s, z13.s\n" - "ld1w { z0.s }, p2/Z, [x27]\n" - "ldr x21, [x16, #0x70]\n" - "fmla z28.s, p2/M, z2.s, z9.s\n" - "fmla z29.s, p2/M, z2.s, z11.s\n" - "ld1w { z9.s }, p3/Z, [x23, x13, LSL #2]\n" - "ld1w { z1.s }, p2/Z, [x27, #1, MUL VL]\n" - "fmla z30.s, p2/M, z2.s, z13.s\n" - "fmla z31.s, p2/M, z2.s, z5.s\n" - "ldr x20, [x16, #0x78]\n" - "ld1w { z2.s }, p2/Z, [x27, #2, MUL VL]\n" - "fmla z28.s, p2/M, z3.s, z11.s\n" - "fmla z29.s, p2/M, z3.s, z12.s\n" - "ld1w { z11.s }, p3/Z, [x22, x13, LSL #2]\n" - "ldr x10, [x16, #0x80]\n" - "fmla z30.s, p2/M, z3.s, z5.s\n" - "fmla z31.s, p2/M, z3.s, z6.s\n" - "ld1w { z3.s }, p2/Z, [x27, #3, MUL VL]\n" - "ldr x9, [x16, #0x88]\n" - "fmla z28.s, p2/M, z4.s, z12.s\n" - "fmla z29.s, p2/M, z4.s, z9.s\n" - "ld1w { z12.s }, p3/Z, [x21, x13, LSL #2]\n" - "ld1w { z9.s }, p3/Z, [x20, x13, LSL #2]\n" - "fmla z30.s, p2/M, z4.s, z6.s\n" - "fmla z31.s, p2/M, z4.s, z10.s\n" - "ld1w { z4.s }, p2/Z, [x27, #4, MUL VL]\n" - "ldr x26, [x16, #0x90]\n" - "fmla z28.s, p2/M, z0.s, z7.s\n" - "fmla z29.s, p2/M, z0.s, z8.s\n" - "ldr x25, [x16, #0x98]\n" - "ldr x23, [x16, #0xa0]\n" - "fmla z30.s, p2/M, z0.s, z14.s\n" - "fmla z31.s, p2/M, z0.s, z11.s\n" - "ld1w { z0.s }, p2/Z, [x27, #5, MUL VL]\n" - "ldr x22, [x16, #0xa8]\n" - "fmla z28.s, p2/M, z1.s, z8.s\n" - "fmla z29.s, p2/M, z1.s, z13.s\n" - "ld1w { z8.s }, p3/Z, [x9, x13, LSL #2]\n" - "ldr x21, [x16, #0xb0]\n" - "fmla z30.s, p2/M, z1.s, z11.s\n" - "fmla z31.s, p2/M, z1.s, z12.s\n" - "ld1w { z1.s }, p2/Z, [x27, #6, MUL VL]\n" - "ldr x20, [x16, #0xb8]\n" - "fmla z28.s, p2/M, z2.s, z13.s\n" - "fmla z29.s, p2/M, z2.s, z5.s\n" - "ld1w { z13.s }, p3/Z, [x10, x13, LSL #2]\n" - "ldr x10, [x16, #0xc0]\n" - "fmla z30.s, p2/M, z2.s, z12.s\n" - "fmla z31.s, p2/M, z2.s, z9.s\n" - "ld1w { z2.s }, p2/Z, [x27, #7, MUL VL]\n" - "addvl x27, x27, #16\n" - "fmla z28.s, p2/M, z3.s, z5.s\n" - "fmla z29.s, p2/M, z3.s, z6.s\n" - "ld1w { z5.s }, p3/Z, [x26, x13, LSL #2]\n" - "ldr x9, [x16, #0xc8]\n" - "fmla z30.s, p2/M, z3.s, z9.s\n" - "fmla z31.s, p2/M, z3.s, z13.s\n" - "ld1w { z3.s }, p2/Z, [x27, #-8, MUL VL]\n" - "ldr x26, [x16, #0xd0]\n" - "fmla z28.s, p2/M, z4.s, z6.s\n" - "fmla z29.s, p2/M, z4.s, z10.s\n" - "ld1w { z6.s }, p3/Z, [x25, x13, LSL #2]\n" - "ld1w { z10.s }, p3/Z, [x23, x13, LSL #2]\n" - "fmla z30.s, p2/M, z4.s, z13.s\n" - "fmla z31.s, p2/M, z4.s, z8.s\n" - "ld1w { z4.s }, p2/Z, [x27, #-7, MUL VL]\n" - "ldr x25, [x16, #0xd8]\n" - "fmla z28.s, p2/M, z0.s, z14.s\n" - "fmla z29.s, p2/M, z0.s, z11.s\n" - "ld1w { z14.s }, p3/Z, [x20, x13, LSL #2]\n" - "ldr x23, [x16, #0xe0]\n" - "fmla z30.s, p2/M, z0.s, z5.s\n" - "fmla z31.s, p2/M, z0.s, z6.s\n" - "ld1w { z0.s }, p2/Z, [x27, #-6, MUL VL]\n" - "ldr x20, [x16, #0xf8]\n" - "fmla z28.s, p2/M, z1.s, z11.s\n" - "fmla z29.s, p2/M, z1.s, z12.s\n" - "ld1w { z11.s }, p3/Z, [x22, x13, LSL #2]\n" - "ldr x22, [x16, #0xe8]\n" + "movprfx z30, z29\n fmla z30.s, p2/M, z0.s, z5.s\n" + "movprfx z27, z29\n fmla z27.s, p2/M, z0.s, z6.s\n" + "ldr x20, [x16, #0x50]\n" + "ld1w { z5.s }, p3/Z, [x20, x13, LSL #2]\n" + "movprfx z31, z29\n fmla z31.s, p2/M, z0.s, z7.s\n" + "movprfx z26, z29\n fmla z26.s, p2/M, z0.s, z8.s\n" + "ldr x20, [x16, #0x58]\n" + "ldr x21, [x16, #0x60]\n" "fmla z30.s, p2/M, z1.s, z6.s\n" - "fmla z31.s, p2/M, z1.s, z10.s\n" - "ld1w { z1.s }, p2/Z, [x27, #-5, MUL VL]\n" - "whilelt p1.s, x28, %x[n_channels]\n" - "fmla z28.s, p2/M, z2.s, z12.s\n" - "fmla z29.s, p2/M, z2.s, z9.s\n" - "ld1w { z12.s }, p3/Z, [x21, x13, LSL #2]\n" - "ldr x21, [x16, #0xf0]\n" - "fmla z30.s, p2/M, z2.s, z10.s\n" - "fmla z31.s, p2/M, z2.s, z11.s\n" - "ld1w { z2.s }, p2/Z, [x27, #-4, MUL VL]\n" - "incw x24\n" - "fmla z28.s, p2/M, z3.s, z9.s\n" - "fmla z29.s, p2/M, z3.s, z13.s\n" - "ld1w { z9.s }, p3/Z, [x10, x13, LSL #2]\n" - "ldr x10, [x16, #0x100]\n" - "fmla z30.s, p2/M, z3.s, z11.s\n" - "fmla z31.s, p2/M, z3.s, z12.s\n" - "ld1w { z3.s }, p2/Z, [x27, #-3, MUL VL]\n" - "mov p0.b, p3.b\n" - "fmla z28.s, p2/M, z4.s, z13.s\n" - "fmla z29.s, p2/M, z4.s, z8.s\n" - "ld1w { z13.s }, p3/Z, [x9, x13, LSL #2]\n" - "ld1w { z8.s }, p3/Z, [x23, x13, LSL #2]\n" - "fmla z30.s, p2/M, z4.s, z12.s\n" - "fmla z31.s, p2/M, z4.s, z14.s\n" - "ld1w { z4.s }, p2/Z, [x27, #-2, MUL VL]\n" - "ldr x9, [x16, #0x108]\n" - "fmla z28.s, p2/M, z0.s, z5.s\n" - "fmla z29.s, p2/M, z0.s, z6.s\n" - "ld1w { z5.s }, p3/Z, [x26, x13, LSL #2]\n" - "ldr x26, [x16, #0x110]\n" - "fmla z30.s, p2/M, z0.s, z9.s\n" - "fmla z31.s, p2/M, z0.s, z13.s\n" - "ld1w { z0.s }, p2/Z, [x27, #-1, MUL VL]\n" - "ld1w { z16.s }, p2/Z, [x27, #4, MUL VL]\n" - "fmla z28.s, p2/M, z1.s, z6.s\n" - "fmla z29.s, p2/M, z1.s, z10.s\n" - "ld1w { z6.s }, p3/Z, [x25, x13, LSL #2]\n" - "ldr x25, [x16, #0x118]\n" - "fmla z30.s, p2/M, z1.s, z13.s\n" - "fmla z31.s, p2/M, z1.s, z5.s\n" - "ld1w { z1.s }, p2/Z, [x27]\n" - "fmla z28.s, p2/M, z2.s, z10.s\n" - "fmla z29.s, p2/M, z2.s, z11.s\n" - "ld1w { z10.s }, p3/Z, [x22, x13, LSL #2]\n" - "fmla z30.s, p2/M, z2.s, z5.s\n" - "fmla z31.s, p2/M, z2.s, z6.s\n" - "ld1w { z2.s }, p2/Z, [x27, #1, MUL VL]\n" - "fmla z28.s, p2/M, z3.s, z11.s\n" - "fmla z29.s, p2/M, z3.s, z12.s\n" - "ld1w { z11.s }, p3/Z, [x21, x13, LSL #2]\n" - "fmla z30.s, p2/M, z3.s, z6.s\n" - "fmla z31.s, p2/M, z3.s, z8.s\n" - "ld1w { z3.s }, p2/Z, [x27, #2, MUL VL]\n" - "fmla z28.s, p2/M, z4.s, z12.s\n" - "fmla z29.s, p2/M, z4.s, z14.s\n" - "ld1w { z12.s }, p3/Z, [x20, x13, LSL #2]\n" - "fmla z30.s, p2/M, z4.s, z8.s\n" - "fmla z31.s, p2/M, z4.s, z10.s\n" - "ld1w { z4.s }, p2/Z, [x27, #3, MUL VL]\n" - "fmla z28.s, p2/M, z0.s, z9.s\n" - "fmla z29.s, p2/M, z0.s, z13.s\n" - "ld1w { z9.s }, p3/Z, [x10, x13, LSL #2]\n" - "fmla z30.s, p2/M, z0.s, z11.s\n" - "fmla z31.s, p2/M, z0.s, z12.s\n" - "ld1w { z11.s }, p3/Z, [x9, x13, LSL #2]\n" - "ldp x10, x9, [x16, #0x0]\n" - "fmla z28.s, p2/M, z1.s, z13.s\n" - "fmla z29.s, p2/M, z1.s, z5.s\n" - "ld1w { z0.s }, p2/Z, [x27, #5, MUL VL]\n" - "fmla z30.s, p2/M, z1.s, z12.s\n" - "fmla z31.s, p2/M, z1.s, z9.s\n" - "ld1w { z12.s }, p3/Z, [x26, x13, LSL #2]\n" - "ld1w { z1.s }, p2/Z, [x27, #6, MUL VL]\n" - "fmla z28.s, p2/M, z2.s, z5.s\n" - "fmla z29.s, p2/M, z2.s, z6.s\n" - "ld1w { z5.s }, p1/Z, [x10, x28, LSL #2]\n" + "fmla z27.s, p2/M, z1.s, z9.s\n" + "ld1w { z22.s }, p3/Z, [x20, x13, LSL #2]\n" + "ldr x20, [x16, #0x68]\n" + "fmla z31.s, p2/M, z1.s, z8.s\n" + "fmla z26.s, p2/M, z1.s, z13.s\n" + "ld1w { z21.s }, p2/Z, [x9]\n" + "ldr x23, [x16, #0x70]\n" "fmla z30.s, p2/M, z2.s, z9.s\n" - "fmla z31.s, p2/M, z2.s, z11.s\n" - "ld1w { z9.s }, p3/Z, [x25, x13, LSL #2]\n" - "ldp x26, x25, [x16, #0x10]\n" - "fmla z28.s, p2/M, z3.s, z6.s\n" - "fmla z29.s, p2/M, z3.s, z8.s\n" - "ld1w { z6.s }, p1/Z, [x9, x28, LSL #2]\n" - "ldp x23, x22, [x16, #0x20]\n" + "fmla z27.s, p2/M, z2.s, z11.s\n" + "ld1w { z20.s }, p3/Z, [x21, x13, LSL #2]\n" + "ld1w { z18.s }, p2/Z, [x9, #1, MUL VL]\n" + "fmla z31.s, p2/M, z2.s, z13.s\n" + "fmla z26.s, p2/M, z2.s, z5.s\n" + "ldr x22, [x16, #0x78]\n" + "ld1w { z17.s }, p2/Z, [x9, #2, MUL VL]\n" "fmla z30.s, p2/M, z3.s, z11.s\n" - "fmla z31.s, p2/M, z3.s, z12.s\n" - "ldp x21, x20, [x16, #0x30]\n" - "ldp x10, x9, [x16, #0x40]\n" - "fmla z28.s, p2/M, z4.s, z8.s\n" - "fmla z29.s, p2/M, z4.s, z10.s\n" - "incw x13\n" - "ld1w { z7.s }, p1/Z, [x26, x28, LSL #2]\n" + "fmla z27.s, p2/M, z3.s, z12.s\n" + "ld1w { z11.s }, p3/Z, [x20, x13, LSL #2]\n" + "ldr x21, [x16, #0x80]\n" + "fmla z31.s, p2/M, z3.s, z5.s\n" + "fmla z26.s, p2/M, z3.s, z22.s\n" + "ld1w { z16.s }, p2/Z, [x9, #3, MUL VL]\n" + "ldr x20, [x16, #0x88]\n" "fmla z30.s, p2/M, z4.s, z12.s\n" - "fmla z31.s, p2/M, z4.s, z9.s\n" - "ld1w { z8.s }, p1/Z, [x25, x28, LSL #2]\n" - "ld1w { z9.s }, p1/Z, [x23, x28, LSL #2]\n" - "ld1w { z13.s }, p1/Z, [x22, x28, LSL #2]\n" - "ld1w { z11.s }, p1/Z, [x21, x28, LSL #2]\n" - "fmax z28.s, p2/M, z28.s, z18.s\n" - "fmax z29.s, p2/M, z29.s, z18.s\n" - "ld1w { z12.s }, p1/Z, [x20, x28, LSL #2]\n" - "ld1w { z10.s }, p1/Z, [x10, x28, LSL #2]\n" - "fmax z30.s, p2/M, z30.s, z18.s\n" - "fmax z31.s, p2/M, z31.s, z18.s\n" - "ld1w { z14.s }, p1/Z, [x9, x28, LSL #2]\n" + "fmla z27.s, p2/M, z4.s, z20.s\n" + "ld1w { z0.s }, p3/Z, [x23, x13, LSL #2]\n" + "ld1w { z29.s }, p3/Z, [x22, x13, LSL #2]\n" + "fmla z31.s, p2/M, z4.s, z22.s\n" + "fmla z26.s, p2/M, z4.s, z10.s\n" + "ld1w { z19.s }, p2/Z, [x9, #4, MUL VL]\n" + "ldr x23, [x16, #0x90]\n" + "fmla z30.s, p2/M, z21.s, z7.s\n" + "fmla z27.s, p2/M, z21.s, z8.s\n" + "ldr x26, [x16, #0x98]\n" + "ldr x22, [x16, #0xa0]\n" + "fmla z31.s, p2/M, z21.s, z14.s\n" + "fmla z26.s, p2/M, z21.s, z11.s\n" + "ld1w { z25.s }, p2/Z, [x9, #5, MUL VL]\n" + "ldr x25, [x16, #0xa8]\n" + "fmla z30.s, p2/M, z18.s, z8.s\n" + "fmla z27.s, p2/M, z18.s, z13.s\n" + "ld1w { z24.s }, p3/Z, [x20, x13, LSL #2]\n" + "ldr x24, [x16, #0xb0]\n" + "fmla z31.s, p2/M, z18.s, z11.s\n" + "fmla z26.s, p2/M, z18.s, z0.s\n" + "ld1w { z18.s }, p2/Z, [x9, #6, MUL VL]\n" + "ldr x20, [x16, #0xb8]\n" + "fmla z30.s, p2/M, z17.s, z13.s\n" + "fmla z27.s, p2/M, z17.s, z5.s\n" + "ld1w { z3.s }, p3/Z, [x21, x13, LSL #2]\n" + "ldr x21, [x16, #0xc0]\n" + "fmla z31.s, p2/M, z17.s, z0.s\n" + "fmla z26.s, p2/M, z17.s, z29.s\n" + "ld1w { z17.s }, p2/Z, [x9, #7, MUL VL]\n" + "addvl x9, x9, #16\n" + "fmla z30.s, p2/M, z16.s, z5.s\n" + "fmla z27.s, p2/M, z16.s, z22.s\n" + "ld1w { z6.s }, p3/Z, [x23, x13, LSL #2]\n" + "ldr x27, [x16, #0xc8]\n" + "fmla z31.s, p2/M, z16.s, z29.s\n" + "fmla z26.s, p2/M, z16.s, z3.s\n" + "ld1w { z16.s }, p2/Z, [x9, #-8, MUL VL]\n" + "ldr x23, [x16, #0xd0]\n" + "fmla z30.s, p2/M, z19.s, z22.s\n" + "fmla z27.s, p2/M, z19.s, z10.s\n" + "ld1w { z23.s }, p3/Z, [x26, x13, LSL #2]\n" + "ld1w { z22.s }, p3/Z, [x22, x13, LSL #2]\n" + "fmla z31.s, p2/M, z19.s, z3.s\n" + "fmla z26.s, p2/M, z19.s, z24.s\n" + "ld1w { z21.s }, p2/Z, [x9, #-7, MUL VL]\n" + "ldr x22, [x16, #0xd8]\n" + "fmla z30.s, p2/M, z25.s, z14.s\n" + "fmla z27.s, p2/M, z25.s, z11.s\n" + "ld1w { z1.s }, p3/Z, [x20, x13, LSL #2]\n" + "ldr x20, [x16, #0xe0]\n" + "fmla z31.s, p2/M, z25.s, z6.s\n" + "fmla z26.s, p2/M, z25.s, z23.s\n" + "ld1w { z20.s }, p2/Z, [x9, #-6, MUL VL]\n" + "ldr x26, [x16, #0xf8]\n" + "fmla z30.s, p2/M, z18.s, z11.s\n" + "fmla z27.s, p2/M, z18.s, z0.s\n" + "ld1w { z7.s }, p3/Z, [x25, x13, LSL #2]\n" + "ldr x25, [x16, #0xe8]\n" + "fmla z31.s, p2/M, z18.s, z23.s\n" + "fmla z26.s, p2/M, z18.s, z22.s\n" + "ld1w { z18.s }, p2/Z, [x9, #-5, MUL VL]\n" + "whilelt p1.s, x10, %x[n_channels]\n" + "fmla z30.s, p2/M, z17.s, z0.s\n" + "fmla z27.s, p2/M, z17.s, z29.s\n" + "ld1w { z19.s }, p3/Z, [x24, x13, LSL #2]\n" + "ldr x24, [x16, #0xf0]\n" + "fmla z31.s, p2/M, z17.s, z22.s\n" + "fmla z26.s, p2/M, z17.s, z7.s\n" + "ld1w { z17.s }, p2/Z, [x9, #-4, MUL VL]\n" "incw x28\n" - "ld1w { z2.s }, p2/Z, [x27, #7, MUL VL]\n" - "addvl x27, x27, #16\n" + "fmla z30.s, p2/M, z16.s, z29.s\n" + "fmla z27.s, p2/M, z16.s, z3.s\n" + "ld1w { z0.s }, p3/Z, [x21, x13, LSL #2]\n" + "ldr x21, [x16, #0x100]\n" + "fmla z31.s, p2/M, z16.s, z7.s\n" + "fmla z26.s, p2/M, z16.s, z19.s\n" + "ld1w { z16.s }, p2/Z, [x9, #-3, MUL VL]\n" + "mov p0.b, p3.b\n" + "fmla z30.s, p2/M, z21.s, z3.s\n" + "fmla z27.s, p2/M, z21.s, z24.s\n" + "ld1w { z11.s }, p3/Z, [x27, x13, LSL #2]\n" + "ld1w { z13.s }, p3/Z, [x20, x13, LSL #2]\n" + "fmla z31.s, p2/M, z21.s, z19.s\n" + "fmla z26.s, p2/M, z21.s, z1.s\n" + "ld1w { z10.s }, p2/Z, [x9, #-2, MUL VL]\n" + "ldr x20, [x16, #0x108]\n" + "fmla z30.s, p2/M, z20.s, z6.s\n" + "fmla z27.s, p2/M, z20.s, z23.s\n" + "ld1w { z25.s }, p3/Z, [x23, x13, LSL #2]\n" + "ldr x23, [x16, #0x110]\n" + "fmla z31.s, p2/M, z20.s, z0.s\n" + "fmla z26.s, p2/M, z20.s, z11.s\n" + "ld1w { z8.s }, p2/Z, [x9, #-1, MUL VL]\n" + "ld1w { z29.s }, p2/Z, [x9, #4, MUL VL]\n" + "fmla z30.s, p2/M, z18.s, z23.s\n" + "fmla z27.s, p2/M, z18.s, z22.s\n" + "ld1w { z24.s }, p3/Z, [x22, x13, LSL #2]\n" + "ldr x22, [x16, #0x118]\n" + "fmla z31.s, p2/M, z18.s, z11.s\n" + "fmla z26.s, p2/M, z18.s, z25.s\n" + "ld1w { z23.s }, p2/Z, [x9]\n" + "fmla z30.s, p2/M, z17.s, z22.s\n" + "fmla z27.s, p2/M, z17.s, z7.s\n" + "ld1w { z22.s }, p3/Z, [x25, x13, LSL #2]\n" + "fmla z31.s, p2/M, z17.s, z25.s\n" + "fmla z26.s, p2/M, z17.s, z24.s\n" + "ld1w { z21.s }, p2/Z, [x9, #1, MUL VL]\n" + "fmla z30.s, p2/M, z16.s, z7.s\n" + "fmla z27.s, p2/M, z16.s, z19.s\n" + "ld1w { z18.s }, p3/Z, [x24, x13, LSL #2]\n" + "fmla z31.s, p2/M, z16.s, z24.s\n" + "fmla z26.s, p2/M, z16.s, z13.s\n" + "ld1w { z20.s }, p2/Z, [x9, #2, MUL VL]\n" + "fmla z30.s, p2/M, z10.s, z19.s\n" + "fmla z27.s, p2/M, z10.s, z1.s\n" + "ld1w { z17.s }, p3/Z, [x26, x13, LSL #2]\n" + "fmla z31.s, p2/M, z10.s, z13.s\n" + "fmla z26.s, p2/M, z10.s, z22.s\n" + "ld1w { z19.s }, p2/Z, [x9, #3, MUL VL]\n" + "fmla z30.s, p2/M, z8.s, z0.s\n" + "fmla z27.s, p2/M, z8.s, z11.s\n" + "ld1w { z16.s }, p3/Z, [x21, x13, LSL #2]\n" + "fmla z31.s, p2/M, z8.s, z18.s\n" + "fmla z26.s, p2/M, z8.s, z17.s\n" + "ld1w { z18.s }, p3/Z, [x20, x13, LSL #2]\n" + "ldp x21, x20, [x16, #0x0]\n" + "fmla z30.s, p2/M, z23.s, z11.s\n" + "fmla z27.s, p2/M, z23.s, z25.s\n" + "ld1w { z0.s }, p2/Z, [x9, #5, MUL VL]\n" + "fmla z31.s, p2/M, z23.s, z17.s\n" + "fmla z26.s, p2/M, z23.s, z16.s\n" + "ld1w { z17.s }, p3/Z, [x23, x13, LSL #2]\n" + "ld1w { z1.s }, p2/Z, [x9, #6, MUL VL]\n" + "fmla z30.s, p2/M, z21.s, z25.s\n" + "fmla z27.s, p2/M, z21.s, z24.s\n" + "ld1w { z5.s }, p1/Z, [x21, x10, LSL #2]\n" + "fmla z31.s, p2/M, z21.s, z16.s\n" + "fmla z26.s, p2/M, z21.s, z18.s\n" + "ld1w { z16.s }, p3/Z, [x22, x13, LSL #2]\n" + "ldp x27, x26, [x16, #0x10]\n" + "fmla z30.s, p2/M, z20.s, z24.s\n" + "fmla z27.s, p2/M, z20.s, z13.s\n" + "ld1w { z6.s }, p1/Z, [x20, x10, LSL #2]\n" + "ldp x25, x24, [x16, #0x20]\n" + "fmla z31.s, p2/M, z20.s, z18.s\n" + "fmla z26.s, p2/M, z20.s, z17.s\n" + "ldp x23, x22, [x16, #0x30]\n" + "ldp x21, x20, [x16, #0x40]\n" + "fmla z30.s, p2/M, z19.s, z13.s\n" + "fmla z27.s, p2/M, z19.s, z22.s\n" + "incw x13\n" + "ld1w { z7.s }, p1/Z, [x27, x10, LSL #2]\n" + "fmla z31.s, p2/M, z19.s, z17.s\n" + "fmla z26.s, p2/M, z19.s, z16.s\n" + "ld1w { z8.s }, p1/Z, [x26, x10, LSL #2]\n" + "ld1w { z9.s }, p1/Z, [x25, x10, LSL #2]\n" + "ld1w { z13.s }, p1/Z, [x24, x10, LSL #2]\n" + "ld1w { z11.s }, p1/Z, [x23, x10, LSL #2]\n" + "fmax z30.s, p2/M, z30.s, z15.s\n" + "fmax z27.s, p2/M, z27.s, z15.s\n" + "ld1w { z12.s }, p1/Z, [x22, x10, LSL #2]\n" + "ld1w { z10.s }, p1/Z, [x21, x10, LSL #2]\n" + "fmax z31.s, p2/M, z31.s, z15.s\n" + "fmax z26.s, p2/M, z26.s, z15.s\n" + "ld1w { z14.s }, p1/Z, [x20, x10, LSL #2]\n" + "incw x10\n" + "ld1w { z2.s }, p2/Z, [x9, #7, MUL VL]\n" + "addvl x9, x9, #16\n" "whilelt p3.s, x13, %x[n_channels]\n" - "cmp x28, %x[n_channels]\n" - "ld1w { z3.s }, p2/Z, [x27, #-8, MUL VL]\n" - "ld1w { z4.s }, p2/Z, [x27, #-7, MUL VL]\n" - "fmin z28.s, p2/M, z28.s, z17.s\n" - "fmin z29.s, p2/M, z29.s, z17.s\n" - "st1w { z28.s }, p0, [x15, x24, LSL #2]\n" - "fmin z30.s, p2/M, z30.s, z17.s\n" - "fmin z31.s, p2/M, z31.s, z17.s\n" - "st1w { z29.s }, p0, [x14, x24, LSL #2]\n" - "st1w { z30.s }, p0, [x12, x24, LSL #2]\n" - "addvl x27, x27, #-6\n" - "st1w { z31.s }, p0, [x11, x24, LSL #2]\n" + "cmp x10, %x[n_channels]\n" + "ld1w { z3.s }, p2/Z, [x9, #-8, MUL VL]\n" + "ld1w { z4.s }, p2/Z, [x9, #-7, MUL VL]\n" + "fmin z30.s, p2/M, z30.s, z28.s\n" + "fmin z27.s, p2/M, z27.s, z28.s\n" + "st1w { z30.s }, p0, [x15, x28, LSL #2]\n" + "fmin z31.s, p2/M, z31.s, z28.s\n" + "fmin z26.s, p2/M, z26.s, z28.s\n" + "st1w { z27.s }, p0, [x14, x28, LSL #2]\n" + "st1w { z31.s }, p0, [x12, x28, LSL #2]\n" + "addvl x9, x9, #-6\n" + "st1w { z26.s }, p0, [x11, x28, LSL #2]\n" "blt 1b\n" "2:" // Channel tail - "movprfx z28, z16\n fmla z28.s, p2/M, z0.s, z5.s\n" - "movprfx z29, z16\n fmla z29.s, p2/M, z0.s, z6.s\n" - "ldr x26, [x16, #0x50]\n" - "ld1w { z5.s }, p3/Z, [x26, x13, LSL #2]\n" - "movprfx z30, z16\n fmla z30.s, p2/M, z0.s, z7.s\n" - "movprfx z31, z16\n fmla z31.s, p2/M, z0.s, z8.s\n" - "ldr x25, [x16, #0x58]\n" - "ldr x23, [x16, #0x60]\n" - "fmla z28.s, p2/M, z1.s, z6.s\n" - "fmla z29.s, p2/M, z1.s, z9.s\n" - "ld1w { z6.s }, p3/Z, [x25, x13, LSL #2]\n" - "ldr x22, [x16, #0x68]\n" - "fmla z30.s, p2/M, z1.s, z8.s\n" - "fmla z31.s, p2/M, z1.s, z13.s\n" - "ld1w { z0.s }, p2/Z, [x27]\n" - "ldr x21, [x16, #0x70]\n" - "fmla z28.s, p2/M, z2.s, z9.s\n" - "fmla z29.s, p2/M, z2.s, z11.s\n" - "ld1w { z9.s }, p3/Z, [x23, x13, LSL #2]\n" - "ld1w { z1.s }, p2/Z, [x27, #1, MUL VL]\n" - "fmla z30.s, p2/M, z2.s, z13.s\n" - "fmla z31.s, p2/M, z2.s, z5.s\n" - "ldr x20, [x16, #0x78]\n" - "ld1w { z2.s }, p2/Z, [x27, #2, MUL VL]\n" - "fmla z28.s, p2/M, z3.s, z11.s\n" - "fmla z29.s, p2/M, z3.s, z12.s\n" - "ld1w { z11.s }, p3/Z, [x22, x13, LSL #2]\n" - "ldr x10, [x16, #0x80]\n" - "fmla z30.s, p2/M, z3.s, z5.s\n" - "fmla z31.s, p2/M, z3.s, z6.s\n" - "ld1w { z3.s }, p2/Z, [x27, #3, MUL VL]\n" - "ldr x9, [x16, #0x88]\n" - "fmla z28.s, p2/M, z4.s, z12.s\n" - "fmla z29.s, p2/M, z4.s, z9.s\n" - "ld1w { z12.s }, p3/Z, [x21, x13, LSL #2]\n" - "ld1w { z9.s }, p3/Z, [x20, x13, LSL #2]\n" - "fmla z30.s, p2/M, z4.s, z6.s\n" - "fmla z31.s, p2/M, z4.s, z10.s\n" - "ld1w { z4.s }, p2/Z, [x27, #4, MUL VL]\n" - "ldr x26, [x16, #0x90]\n" - "fmla z28.s, p2/M, z0.s, z7.s\n" + "movprfx z30, z29\n fmla z30.s, p2/M, z0.s, z5.s\n" + "movprfx z31, z29\n fmla z31.s, p2/M, z0.s, z6.s\n" + "ldr x20, [x16, #0x50]\n" + "ld1w { z22.s }, p3/Z, [x20, x13, LSL #2]\n" + "movprfx z5, z29\n fmla z5.s, p2/M, z0.s, z7.s\n" "fmla z29.s, p2/M, z0.s, z8.s\n" - "ldr x25, [x16, #0x98]\n" - "ldr x23, [x16, #0xa0]\n" - "fmla z30.s, p2/M, z0.s, z14.s\n" - "fmla z31.s, p2/M, z0.s, z11.s\n" - "ld1w { z0.s }, p2/Z, [x27, #5, MUL VL]\n" - "ldr x22, [x16, #0xa8]\n" - "fmla z28.s, p2/M, z1.s, z8.s\n" - "fmla z29.s, p2/M, z1.s, z13.s\n" - "ld1w { z8.s }, p3/Z, [x9, x13, LSL #2]\n" - "ldr x21, [x16, #0xb0]\n" - "fmla z30.s, p2/M, z1.s, z11.s\n" - "fmla z31.s, p2/M, z1.s, z12.s\n" - "ld1w { z1.s }, p2/Z, [x27, #6, MUL VL]\n" - "ldr x20, [x16, #0xb8]\n" - "fmla z28.s, p2/M, z2.s, z13.s\n" - "fmla z29.s, p2/M, z2.s, z5.s\n" - "ld1w { z13.s }, p3/Z, [x10, x13, LSL #2]\n" - "ldr x10, [x16, #0xc0]\n" - "fmla z30.s, p2/M, z2.s, z12.s\n" - "fmla z31.s, p2/M, z2.s, z9.s\n" - "ld1w { z2.s }, p2/Z, [x27, #7, MUL VL]\n" - "addvl x27, x27, #16\n" - "fmla z28.s, p2/M, z3.s, z5.s\n" - "fmla z29.s, p2/M, z3.s, z6.s\n" - "ld1w { z5.s }, p3/Z, [x26, x13, LSL #2]\n" - "ldr x9, [x16, #0xc8]\n" - "fmla z30.s, p2/M, z3.s, z9.s\n" - "fmla z31.s, p2/M, z3.s, z13.s\n" - "ld1w { z3.s }, p2/Z, [x27, #-8, MUL VL]\n" - "ldr x26, [x16, #0xd0]\n" - "fmla z28.s, p2/M, z4.s, z6.s\n" - "fmla z29.s, p2/M, z4.s, z10.s\n" - "ld1w { z6.s }, p3/Z, [x25, x13, LSL #2]\n" - "ld1w { z10.s }, p3/Z, [x23, x13, LSL #2]\n" - "fmla z30.s, p2/M, z4.s, z13.s\n" - "fmla z31.s, p2/M, z4.s, z8.s\n" - "ld1w { z4.s }, p2/Z, [x27, #-7, MUL VL]\n" - "ldr x25, [x16, #0xd8]\n" - "fmla z28.s, p2/M, z0.s, z14.s\n" - "fmla z29.s, p2/M, z0.s, z11.s\n" - "ld1w { z14.s }, p3/Z, [x20, x13, LSL #2]\n" - "ldr x23, [x16, #0xe0]\n" - "fmla z30.s, p2/M, z0.s, z5.s\n" - "fmla z31.s, p2/M, z0.s, z6.s\n" - "ld1w { z0.s }, p2/Z, [x27, #-6, MUL VL]\n" - "ldr x20, [x16, #0xf8]\n" - "fmla z28.s, p2/M, z1.s, z11.s\n" - "fmla z29.s, p2/M, z1.s, z12.s\n" - "ld1w { z11.s }, p3/Z, [x22, x13, LSL #2]\n" - "ldr x22, [x16, #0xe8]\n" + "ldr x20, [x16, #0x58]\n" + "ldr x21, [x16, #0x60]\n" "fmla z30.s, p2/M, z1.s, z6.s\n" - "fmla z31.s, p2/M, z1.s, z10.s\n" - "ld1w { z1.s }, p2/Z, [x27, #-5, MUL VL]\n" - "incw x24\n" - "fmla z28.s, p2/M, z2.s, z12.s\n" - "fmla z29.s, p2/M, z2.s, z9.s\n" - "ld1w { z12.s }, p3/Z, [x21, x13, LSL #2]\n" - "ldr x21, [x16, #0xf0]\n" - "fmla z30.s, p2/M, z2.s, z10.s\n" - "fmla z31.s, p2/M, z2.s, z11.s\n" - "ld1w { z2.s }, p2/Z, [x27, #-4, MUL VL]\n" - "mov p0.b, p3.b\n" - "fmla z28.s, p2/M, z3.s, z9.s\n" - "fmla z29.s, p2/M, z3.s, z13.s\n" - "ld1w { z9.s }, p3/Z, [x10, x13, LSL #2]\n" - "ldr x10, [x16, #0x100]\n" - "fmla z30.s, p2/M, z3.s, z11.s\n" - "fmla z31.s, p2/M, z3.s, z12.s\n" - "ld1w { z3.s }, p2/Z, [x27, #-3, MUL VL]\n" - "fmla z28.s, p2/M, z4.s, z13.s\n" - "fmla z29.s, p2/M, z4.s, z8.s\n" - "ld1w { z13.s }, p3/Z, [x9, x13, LSL #2]\n" - "ld1w { z8.s }, p3/Z, [x23, x13, LSL #2]\n" - "fmla z30.s, p2/M, z4.s, z12.s\n" - "fmla z31.s, p2/M, z4.s, z14.s\n" - "ld1w { z4.s }, p2/Z, [x27, #-2, MUL VL]\n" - "ldr x9, [x16, #0x108]\n" - "fmla z28.s, p2/M, z0.s, z5.s\n" - "fmla z29.s, p2/M, z0.s, z6.s\n" - "ld1w { z5.s }, p3/Z, [x26, x13, LSL #2]\n" - "ldr x26, [x16, #0x110]\n" - "fmla z30.s, p2/M, z0.s, z9.s\n" - "fmla z31.s, p2/M, z0.s, z13.s\n" - "ld1w { z0.s }, p2/Z, [x27, #-1, MUL VL]\n" - "fmla z28.s, p2/M, z1.s, z6.s\n" - "fmla z29.s, p2/M, z1.s, z10.s\n" - "ld1w { z6.s }, p3/Z, [x25, x13, LSL #2]\n" - "ldr x25, [x16, #0x118]\n" - "fmla z30.s, p2/M, z1.s, z13.s\n" - "fmla z31.s, p2/M, z1.s, z5.s\n" - "ld1w { z1.s }, p2/Z, [x27]\n" - "fmla z28.s, p2/M, z2.s, z10.s\n" - "fmla z29.s, p2/M, z2.s, z11.s\n" - "ld1w { z10.s }, p3/Z, [x22, x13, LSL #2]\n" - "fmla z30.s, p2/M, z2.s, z5.s\n" - "fmla z31.s, p2/M, z2.s, z6.s\n" - "ld1w { z2.s }, p2/Z, [x27, #1, MUL VL]\n" - "fmla z28.s, p2/M, z3.s, z11.s\n" - "fmla z29.s, p2/M, z3.s, z12.s\n" - "ld1w { z11.s }, p3/Z, [x21, x13, LSL #2]\n" - "fmla z30.s, p2/M, z3.s, z6.s\n" - "fmla z31.s, p2/M, z3.s, z8.s\n" - "ld1w { z3.s }, p2/Z, [x27, #2, MUL VL]\n" - "fmla z28.s, p2/M, z4.s, z12.s\n" - "fmla z29.s, p2/M, z4.s, z14.s\n" - "ld1w { z12.s }, p3/Z, [x20, x13, LSL #2]\n" - "fmla z30.s, p2/M, z4.s, z8.s\n" - "fmla z31.s, p2/M, z4.s, z10.s\n" - "ld1w { z4.s }, p2/Z, [x27, #3, MUL VL]\n" - "fmla z28.s, p2/M, z0.s, z9.s\n" - "fmla z29.s, p2/M, z0.s, z13.s\n" - "ld1w { z9.s }, p3/Z, [x10, x13, LSL #2]\n" - "fmla z30.s, p2/M, z0.s, z11.s\n" - "fmla z31.s, p2/M, z0.s, z12.s\n" - "ld1w { z11.s }, p3/Z, [x9, x13, LSL #2]\n" - "fmla z28.s, p2/M, z1.s, z13.s\n" - "fmla z29.s, p2/M, z1.s, z5.s\n" - "fmla z30.s, p2/M, z1.s, z12.s\n" "fmla z31.s, p2/M, z1.s, z9.s\n" - "ld1w { z12.s }, p3/Z, [x26, x13, LSL #2]\n" - "fmla z28.s, p2/M, z2.s, z5.s\n" - "fmla z29.s, p2/M, z2.s, z6.s\n" + "ld1w { z6.s }, p3/Z, [x20, x13, LSL #2]\n" + "ldr x20, [x16, #0x68]\n" + "fmla z5.s, p2/M, z1.s, z8.s\n" + "fmla z29.s, p2/M, z1.s, z13.s\n" + "ld1w { z20.s }, p2/Z, [x9]\n" + "ldr x23, [x16, #0x70]\n" "fmla z30.s, p2/M, z2.s, z9.s\n" "fmla z31.s, p2/M, z2.s, z11.s\n" - "ld1w { z9.s }, p3/Z, [x25, x13, LSL #2]\n" - "fmla z28.s, p2/M, z3.s, z6.s\n" - "fmla z29.s, p2/M, z3.s, z8.s\n" + "ld1w { z16.s }, p3/Z, [x21, x13, LSL #2]\n" + "ld1w { z19.s }, p2/Z, [x9, #1, MUL VL]\n" + "fmla z5.s, p2/M, z2.s, z13.s\n" + "fmla z29.s, p2/M, z2.s, z22.s\n" + "ldr x21, [x16, #0x78]\n" + "ld1w { z18.s }, p2/Z, [x9, #2, MUL VL]\n" "fmla z30.s, p2/M, z3.s, z11.s\n" "fmla z31.s, p2/M, z3.s, z12.s\n" - "fmla z28.s, p2/M, z4.s, z8.s\n" - "fmla z29.s, p2/M, z4.s, z10.s\n" - "fmax z28.s, p2/M, z28.s, z18.s\n" - "fmax z29.s, p2/M, z29.s, z18.s\n" + "ld1w { z1.s }, p3/Z, [x20, x13, LSL #2]\n" + "ldr x22, [x16, #0x80]\n" + "fmla z5.s, p2/M, z3.s, z22.s\n" + "fmla z29.s, p2/M, z3.s, z6.s\n" + "ld1w { z17.s }, p2/Z, [x9, #3, MUL VL]\n" + "ldr x20, [x16, #0x88]\n" "fmla z30.s, p2/M, z4.s, z12.s\n" - "fmla z31.s, p2/M, z4.s, z9.s\n" - "fmax z30.s, p2/M, z30.s, z18.s\n" - "fmax z31.s, p2/M, z31.s, z18.s\n" - "fmin z28.s, p2/M, z28.s, z17.s\n" - "fmin z29.s, p2/M, z29.s, z17.s\n" - "st1w { z28.s }, p0, [x15, x24, LSL #2]\n" - "fmin z30.s, p2/M, z30.s, z17.s\n" - "fmin z31.s, p2/M, z31.s, z17.s\n" - "st1w { z29.s }, p0, [x14, x24, LSL #2]\n" - "st1w { z30.s }, p0, [x12, x24, LSL #2]\n" - "st1w { z31.s }, p0, [x11, x24, LSL #2]\n" + "fmla z31.s, p2/M, z4.s, z16.s\n" + "ld1w { z0.s }, p3/Z, [x23, x13, LSL #2]\n" + "ld1w { z27.s }, p3/Z, [x21, x13, LSL #2]\n" + "fmla z5.s, p2/M, z4.s, z6.s\n" + "fmla z29.s, p2/M, z4.s, z10.s\n" + "ld1w { z16.s }, p2/Z, [x9, #4, MUL VL]\n" + "ldr x21, [x16, #0x90]\n" + "fmla z30.s, p2/M, z20.s, z7.s\n" + "fmla z31.s, p2/M, z20.s, z8.s\n" + "ldr x27, [x16, #0x98]\n" + "ldr x26, [x16, #0xa0]\n" + "fmla z5.s, p2/M, z20.s, z14.s\n" + "fmla z29.s, p2/M, z20.s, z1.s\n" + "ld1w { z21.s }, p2/Z, [x9, #5, MUL VL]\n" + "ldr x25, [x16, #0xa8]\n" + "fmla z30.s, p2/M, z19.s, z8.s\n" + "fmla z31.s, p2/M, z19.s, z13.s\n" + "ld1w { z26.s }, p3/Z, [x20, x13, LSL #2]\n" + "ldr x24, [x16, #0xb0]\n" + "fmla z5.s, p2/M, z19.s, z1.s\n" + "fmla z29.s, p2/M, z19.s, z0.s\n" + "ld1w { z25.s }, p2/Z, [x9, #6, MUL VL]\n" + "ldr x20, [x16, #0xb8]\n" + "fmla z30.s, p2/M, z18.s, z13.s\n" + "fmla z31.s, p2/M, z18.s, z22.s\n" + "ld1w { z24.s }, p3/Z, [x22, x13, LSL #2]\n" + "ldr x23, [x16, #0xc0]\n" + "fmla z5.s, p2/M, z18.s, z0.s\n" + "fmla z29.s, p2/M, z18.s, z27.s\n" + "ld1w { z23.s }, p2/Z, [x9, #7, MUL VL]\n" + "addvl x9, x9, #16\n" + "fmla z30.s, p2/M, z17.s, z22.s\n" + "fmla z31.s, p2/M, z17.s, z6.s\n" + "ld1w { z22.s }, p3/Z, [x21, x13, LSL #2]\n" + "ldr x22, [x16, #0xc8]\n" + "fmla z5.s, p2/M, z17.s, z27.s\n" + "fmla z29.s, p2/M, z17.s, z24.s\n" + "ld1w { z20.s }, p2/Z, [x9, #-8, MUL VL]\n" + "ldr x21, [x16, #0xd0]\n" + "fmla z30.s, p2/M, z16.s, z6.s\n" + "fmla z31.s, p2/M, z16.s, z10.s\n" + "ld1w { z19.s }, p3/Z, [x27, x13, LSL #2]\n" + "ld1w { z18.s }, p3/Z, [x26, x13, LSL #2]\n" + "fmla z5.s, p2/M, z16.s, z24.s\n" + "fmla z29.s, p2/M, z16.s, z26.s\n" + "ld1w { z16.s }, p2/Z, [x9, #-7, MUL VL]\n" + "ldr x27, [x16, #0xd8]\n" + "fmla z30.s, p2/M, z21.s, z14.s\n" + "fmla z31.s, p2/M, z21.s, z1.s\n" + "ld1w { z17.s }, p3/Z, [x20, x13, LSL #2]\n" + "ldr x20, [x16, #0xe0]\n" + "fmla z5.s, p2/M, z21.s, z22.s\n" + "fmla z29.s, p2/M, z21.s, z19.s\n" + "ld1w { z21.s }, p2/Z, [x9, #-6, MUL VL]\n" + "ldr x26, [x16, #0xf8]\n" + "fmla z30.s, p2/M, z25.s, z1.s\n" + "fmla z31.s, p2/M, z25.s, z0.s\n" + "ld1w { z9.s }, p3/Z, [x25, x13, LSL #2]\n" + "ldr x25, [x16, #0xe8]\n" + "fmla z5.s, p2/M, z25.s, z19.s\n" + "fmla z29.s, p2/M, z25.s, z18.s\n" + "ld1w { z4.s }, p2/Z, [x9, #-5, MUL VL]\n" + "incw x28\n" + "fmla z30.s, p2/M, z23.s, z0.s\n" + "fmla z31.s, p2/M, z23.s, z27.s\n" + "ld1w { z8.s }, p3/Z, [x24, x13, LSL #2]\n" + "ldr x24, [x16, #0xf0]\n" + "fmla z5.s, p2/M, z23.s, z18.s\n" + "fmla z29.s, p2/M, z23.s, z9.s\n" + "ld1w { z6.s }, p2/Z, [x9, #-4, MUL VL]\n" + "mov p0.b, p3.b\n" + "fmla z30.s, p2/M, z20.s, z27.s\n" + "fmla z31.s, p2/M, z20.s, z24.s\n" + "ld1w { z10.s }, p3/Z, [x23, x13, LSL #2]\n" + "ldr x23, [x16, #0x100]\n" + "fmla z5.s, p2/M, z20.s, z9.s\n" + "fmla z29.s, p2/M, z20.s, z8.s\n" + "ld1w { z11.s }, p2/Z, [x9, #-3, MUL VL]\n" + "fmla z30.s, p2/M, z16.s, z24.s\n" + "fmla z31.s, p2/M, z16.s, z26.s\n" + "ld1w { z0.s }, p3/Z, [x22, x13, LSL #2]\n" + "ld1w { z27.s }, p3/Z, [x20, x13, LSL #2]\n" + "fmla z5.s, p2/M, z16.s, z8.s\n" + "fmla z29.s, p2/M, z16.s, z17.s\n" + "ld1w { z16.s }, p2/Z, [x9, #-2, MUL VL]\n" + "ldr x22, [x16, #0x108]\n" + "fmla z30.s, p2/M, z21.s, z22.s\n" + "fmla z31.s, p2/M, z21.s, z19.s\n" + "ld1w { z26.s }, p3/Z, [x21, x13, LSL #2]\n" + "ldr x21, [x16, #0x110]\n" + "fmla z5.s, p2/M, z21.s, z10.s\n" + "fmla z29.s, p2/M, z21.s, z0.s\n" + "ld1w { z25.s }, p2/Z, [x9, #-1, MUL VL]\n" + "fmla z30.s, p2/M, z4.s, z19.s\n" + "fmla z31.s, p2/M, z4.s, z18.s\n" + "ld1w { z24.s }, p3/Z, [x27, x13, LSL #2]\n" + "ldr x20, [x16, #0x118]\n" + "fmla z5.s, p2/M, z4.s, z0.s\n" + "fmla z29.s, p2/M, z4.s, z26.s\n" + "ld1w { z23.s }, p2/Z, [x9]\n" + "fmla z30.s, p2/M, z6.s, z18.s\n" + "fmla z31.s, p2/M, z6.s, z9.s\n" + "ld1w { z22.s }, p3/Z, [x25, x13, LSL #2]\n" + "fmla z5.s, p2/M, z6.s, z26.s\n" + "fmla z29.s, p2/M, z6.s, z24.s\n" + "ld1w { z21.s }, p2/Z, [x9, #1, MUL VL]\n" + "fmla z30.s, p2/M, z11.s, z9.s\n" + "fmla z31.s, p2/M, z11.s, z8.s\n" + "ld1w { z18.s }, p3/Z, [x24, x13, LSL #2]\n" + "fmla z5.s, p2/M, z11.s, z24.s\n" + "fmla z29.s, p2/M, z11.s, z27.s\n" + "ld1w { z20.s }, p2/Z, [x9, #2, MUL VL]\n" + "fmla z30.s, p2/M, z16.s, z8.s\n" + "fmla z31.s, p2/M, z16.s, z17.s\n" + "ld1w { z17.s }, p3/Z, [x26, x13, LSL #2]\n" + "fmla z5.s, p2/M, z16.s, z27.s\n" + "fmla z29.s, p2/M, z16.s, z22.s\n" + "ld1w { z19.s }, p2/Z, [x9, #3, MUL VL]\n" + "fmla z30.s, p2/M, z25.s, z10.s\n" + "fmla z31.s, p2/M, z25.s, z0.s\n" + "ld1w { z16.s }, p3/Z, [x23, x13, LSL #2]\n" + "fmla z5.s, p2/M, z25.s, z18.s\n" + "fmla z29.s, p2/M, z25.s, z17.s\n" + "ld1w { z18.s }, p3/Z, [x22, x13, LSL #2]\n" + "fmla z30.s, p2/M, z23.s, z0.s\n" + "fmla z31.s, p2/M, z23.s, z26.s\n" + "fmla z5.s, p2/M, z23.s, z17.s\n" + "fmla z29.s, p2/M, z23.s, z16.s\n" + "ld1w { z17.s }, p3/Z, [x21, x13, LSL #2]\n" + "fmla z30.s, p2/M, z21.s, z26.s\n" + "fmla z31.s, p2/M, z21.s, z24.s\n" + "fmla z5.s, p2/M, z21.s, z16.s\n" + "fmla z29.s, p2/M, z21.s, z18.s\n" + "ld1w { z16.s }, p3/Z, [x20, x13, LSL #2]\n" + "fmla z30.s, p2/M, z20.s, z24.s\n" + "fmla z31.s, p2/M, z20.s, z27.s\n" + "fmla z5.s, p2/M, z20.s, z18.s\n" + "fmla z29.s, p2/M, z20.s, z17.s\n" + "fmla z30.s, p2/M, z19.s, z27.s\n" + "fmla z31.s, p2/M, z19.s, z22.s\n" + "fmax z30.s, p2/M, z30.s, z15.s\n" + "fmax z31.s, p2/M, z31.s, z15.s\n" + "fmla z5.s, p2/M, z19.s, z17.s\n" + "fmla z29.s, p2/M, z19.s, z16.s\n" + "fmax z5.s, p2/M, z5.s, z15.s\n" + "fmax z29.s, p2/M, z29.s, z15.s\n" + "fmin z30.s, p2/M, z30.s, z28.s\n" + "fmin z31.s, p2/M, z31.s, z28.s\n" + "st1w { z30.s }, p0, [x15, x28, LSL #2]\n" + "fmin z5.s, p2/M, z5.s, z28.s\n" + "fmin z29.s, p2/M, z29.s, z28.s\n" + "st1w { z31.s }, p0, [x14, x28, LSL #2]\n" + "st1w { z5.s }, p0, [x12, x28, LSL #2]\n" + "st1w { z29.s }, p0, [x11, x28, LSL #2]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_generic_output9_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_generic_output9_mla_depthfirst.hpp index 62faca97a9b0a83ff651391c90da9a46a33beda5..6b155fc8554156c0ee4e45589c26df4a527fd57f 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_generic_output9_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_generic_output9_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_generic_output9_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_generic_output9_mla_depthfirst/generic.cpp index 204f36edca09a20552d30efdfd3361cc7e1cebd9..d53daaa8a0509948dc7ba91fe28777a7e2b5d2d9 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_generic_output9_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_generic_output9_mla_depthfirst/generic.cpp @@ -55,9 +55,9 @@ void sve_fp32_nhwc_generic_output9_mla_depthfirst_impl( "ld1w { z23.s }, p0/Z, [%x[bias], x11, LSL #2]\n" "2:" // Channel loop: Load bias: Done "mov x10, %x[inptrs]\n" - "ldp x9, x28, [x10], #0x10\n" - "ldp x27, x26, [x10], #0x10\n" - "subs x25, %x[n_points], #0x1\n" + "ldp x28, x27, [x10], #0x10\n" + "ldp x26, x25, [x10], #0x10\n" + "subs x9, %x[n_points], #0x1\n" "ldp x24, x23, [x10], #0x10\n" "ldp x22, x21, [x10], #0x10\n" "mov z24.d, z23.d\n" @@ -68,12 +68,12 @@ void sve_fp32_nhwc_generic_output9_mla_depthfirst_impl( "ld1w { z0.s }, p1/Z, [%x[params]]\n" "mov z28.d, z23.d\n" "mov z29.d, z23.d\n" - "ld1w { z14.s }, p0/Z, [x9, x11, LSL #2]\n" - "ld1w { z15.s }, p0/Z, [x28, x11, LSL #2]\n" + "ld1w { z14.s }, p0/Z, [x28, x11, LSL #2]\n" + "ld1w { z15.s }, p0/Z, [x27, x11, LSL #2]\n" "mov z30.d, z23.d\n" "mov z31.d, z23.d\n" - "ld1w { z16.s }, p0/Z, [x27, x11, LSL #2]\n" - "ld1w { z17.s }, p0/Z, [x26, x11, LSL #2]\n" + "ld1w { z16.s }, p0/Z, [x26, x11, LSL #2]\n" + "ld1w { z17.s }, p0/Z, [x25, x11, LSL #2]\n" "ld1w { z18.s }, p0/Z, [x24, x11, LSL #2]\n" "ld1w { z19.s }, p0/Z, [x23, x11, LSL #2]\n" "addvl %x[params], %x[params], #1\n" @@ -82,9 +82,9 @@ void sve_fp32_nhwc_generic_output9_mla_depthfirst_impl( "ld1w { z22.s }, p0/Z, [x20, x11, LSL #2]\n" "ble 4f\n" "3:" // Channel loop: Planar loop - "ldp x9, x28, [x10], #0x10\n" - "ldp x27, x26, [x10], #0x10\n" - "subs x25, x25, #0x1\n" + "ldp x28, x27, [x10], #0x10\n" + "ldp x26, x25, [x10], #0x10\n" + "subs x9, x9, #0x1\n" "fmla z23.s, p1/M, z14.s, z0.s\n" "ldp x24, x23, [x10], #0x10\n" "ldp x22, x21, [x10], #0x10\n" @@ -93,15 +93,15 @@ void sve_fp32_nhwc_generic_output9_mla_depthfirst_impl( "ldr x20, [x10], #0x8\n" "fmla z26.s, p1/M, z17.s, z0.s\n" "fmla z27.s, p1/M, z18.s, z0.s\n" - "ld1w { z14.s }, p0/Z, [x9, x11, LSL #2]\n" + "ld1w { z14.s }, p0/Z, [x28, x11, LSL #2]\n" "fmla z28.s, p1/M, z19.s, z0.s\n" "fmla z29.s, p1/M, z20.s, z0.s\n" - "ld1w { z15.s }, p0/Z, [x28, x11, LSL #2]\n" - "ld1w { z16.s }, p0/Z, [x27, x11, LSL #2]\n" + "ld1w { z15.s }, p0/Z, [x27, x11, LSL #2]\n" + "ld1w { z16.s }, p0/Z, [x26, x11, LSL #2]\n" "fmla z30.s, p1/M, z21.s, z0.s\n" "fmla z31.s, p1/M, z22.s, z0.s\n" "ld1w { z0.s }, p1/Z, [%x[params]]\n" - "ld1w { z17.s }, p0/Z, [x26, x11, LSL #2]\n" + "ld1w { z17.s }, p0/Z, [x25, x11, LSL #2]\n" "ld1w { z18.s }, p0/Z, [x24, x11, LSL #2]\n" "ld1w { z19.s }, p0/Z, [x23, x11, LSL #2]\n" "addvl %x[params], %x[params], #1\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst.hpp index 8640343747e09dec8943bbb7f041fbcdea043648..eb1b111c36717243c7c926bbf7f01150ff395a2d 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst/generic.cpp index 7ba0edd99161c2801e457a0ba15155d5f5aad228..3a71baaf614d6b6c52b3fd05467efd0cac59826d 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst/generic.cpp @@ -53,21 +53,21 @@ void sve_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst_imp "ldr x11, [%x[inptrs], #0x20]\n" "ldr x10, [%x[inptrs], #0x28]\n" "ldr x9, [%x[inptrs], #0x30]\n" - "ld1w { z26.s }, p2/Z, [%x[params]]\n" - "mov z25.d, z26.d\n" - "mov z24.d, z26.d\n" + "ld1w { z24.s }, p2/Z, [%x[params]]\n" + "mov z21.d, z24.d\n" + "mov z25.d, z24.d\n" "ldp x28, x27, [%x[outptrs], #0x0]\n" "ldp x26, x25, [%x[outptrs], #0x10]\n" - "mov z23.d, z26.d\n" - "mov z22.d, z26.d\n" + "mov z27.d, z24.d\n" + "mov z26.d, z24.d\n" "ldp x24, x23, [%x[outptrs], #0x20]\n" "ldp x22, x21, [%x[outptrs], #0x30]\n" - "mov z21.d, z26.d\n" - "mov z20.d, z26.d\n" + "mov z28.d, z24.d\n" + "mov z20.d, z24.d\n" "ldr x20, [%x[outptrs], #0x40]\n" "ld1rqw { z2.s }, p1/Z, [x16]\n" - "mov z19.d, z26.d\n" - "mov z18.d, z26.d\n" + "mov z23.d, z24.d\n" + "mov z19.d, z24.d\n" "ld1rqw { z3.s }, p1/Z, [x16, #16]\n" "ld1rqw { z4.s }, p1/Z, [x15]\n" "ld1rqw { z5.s }, p1/Z, [x15, #16]\n" @@ -81,175 +81,175 @@ void sve_fp32_packed_to_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst_imp "ld1rqw { z13.s }, p1/Z, [x10, #16]\n" "ld1rqw { z14.s }, p1/Z, [x9]\n" "ld1rqw { z15.s }, p1/Z, [x9, #16]\n" - "ld1rw { z17.s }, p1/Z, [%x[clamps]]\n" + "ld1rw { z22.s }, p1/Z, [%x[clamps]]\n" "ld1rw { z16.s }, p1/Z, [%x[clamps], #4]\n" "ld1w { z31.s }, p2/Z, [%x[params], #1, MUL VL]\n" "ld1w { z30.s }, p2/Z, [%x[params], #2, MUL VL]\n" "ld1w { z29.s }, p2/Z, [%x[params], #3, MUL VL]\n" "addvl %x[params], %x[params], #4\n" "1:" // Output channel complete vector loop - "fmla z26.s, z31.s, z2.s[0]\n" - "fmla z23.s, z31.s, z6.s[0]\n" - "mov z0.d, z10.d\n" + "fmla z24.s, z31.s, z2.s[0]\n" + "fmla z27.s, z31.s, z6.s[0]\n" + "mov z1.d, z10.d\n" "incw x17\n" - "fmla z22.s, z31.s, z6.s[2]\n" - "fmla z21.s, z31.s, z7.s[0]\n" - "mov z1.d, z11.d\n" + "fmla z26.s, z31.s, z6.s[2]\n" + "fmla z28.s, z31.s, z7.s[0]\n" + "mov z0.d, z11.d\n" "mov p0.b, p2.b\n" - "fmla z25.s, z31.s, z2.s[2]\n" - "fmla z24.s, z31.s, z3.s[0]\n" + "fmla z21.s, z31.s, z2.s[2]\n" + "fmla z25.s, z31.s, z3.s[0]\n" "whilelt p2.s, x17, %x[channel_multiplier]\n" - "fmla z20.s, z31.s, z0.s[0]\n" - "fmla z19.s, z31.s, z0.s[2]\n" - "fmla z18.s, z31.s, z1.s[0]\n" - "fmla z26.s, z30.s, z2.s[1]\n" - "ld1w { z31.s }, p1/Z, [%x[params]]\n" - "fmla z23.s, z30.s, z6.s[1]\n" - "fmla z22.s, z30.s, z6.s[3]\n" - "fmla z21.s, z30.s, z7.s[1]\n" - "fmla z25.s, z30.s, z2.s[3]\n" - "fmla z24.s, z30.s, z3.s[1]\n" - "fmla z20.s, z30.s, z0.s[1]\n" - "fmla z19.s, z30.s, z0.s[3]\n" - "fmla z18.s, z30.s, z1.s[1]\n" - "ld1w { z30.s }, p1/Z, [%x[params], #1, MUL VL]\n" - "fmla z26.s, z29.s, z2.s[2]\n" - "fmla z23.s, z29.s, z6.s[2]\n" - "fmla z22.s, z29.s, z7.s[0]\n" - "fmla z21.s, z29.s, z7.s[2]\n" - "fmla z25.s, z29.s, z3.s[0]\n" - "fmla z24.s, z29.s, z3.s[2]\n" - "fmla z20.s, z29.s, z0.s[2]\n" - "fmla z19.s, z29.s, z1.s[0]\n" - "mov z0.d, z8.d\n" - "fmla z18.s, z29.s, z1.s[2]\n" - "mov z1.d, z9.d\n" - "fmla z26.s, z31.s, z4.s[0]\n" - "ld1w { z29.s }, p1/Z, [%x[params], #2, MUL VL]\n" + "fmla z20.s, z31.s, z1.s[0]\n" + "fmla z23.s, z31.s, z1.s[2]\n" + "fmla z19.s, z31.s, z0.s[0]\n" + "fmla z24.s, z30.s, z2.s[1]\n" + "ld1w { z18.s }, p1/Z, [%x[params]]\n" + "fmla z27.s, z30.s, z6.s[1]\n" + "fmla z26.s, z30.s, z6.s[3]\n" + "fmla z28.s, z30.s, z7.s[1]\n" + "fmla z21.s, z30.s, z2.s[3]\n" + "fmla z25.s, z30.s, z3.s[1]\n" + "fmla z20.s, z30.s, z1.s[1]\n" + "fmla z23.s, z30.s, z1.s[3]\n" + "fmla z19.s, z30.s, z0.s[1]\n" + "ld1w { z17.s }, p1/Z, [%x[params], #1, MUL VL]\n" + "fmla z24.s, z29.s, z2.s[2]\n" + "fmla z27.s, z29.s, z6.s[2]\n" + "fmla z26.s, z29.s, z7.s[0]\n" + "fmla z28.s, z29.s, z7.s[2]\n" + "fmla z21.s, z29.s, z3.s[0]\n" + "fmla z25.s, z29.s, z3.s[2]\n" + "fmla z20.s, z29.s, z1.s[2]\n" + "fmla z23.s, z29.s, z0.s[0]\n" + "mov z1.d, z8.d\n" + "fmla z19.s, z29.s, z0.s[2]\n" + "mov z0.d, z9.d\n" + "fmla z24.s, z18.s, z4.s[0]\n" + "ld1w { z31.s }, p1/Z, [%x[params], #2, MUL VL]\n" + "fmla z27.s, z18.s, z1.s[0]\n" + "fmla z26.s, z18.s, z1.s[2]\n" + "mov z1.d, z12.d\n" + "fmla z28.s, z18.s, z0.s[0]\n" + "mov z0.d, z13.d\n" + "fmla z21.s, z18.s, z4.s[2]\n" + "fmla z25.s, z18.s, z5.s[0]\n" + "fmla z20.s, z18.s, z1.s[0]\n" + "fmla z23.s, z18.s, z1.s[2]\n" + "fmla z19.s, z18.s, z0.s[0]\n" + "mov z1.d, z8.d\n" + "ld1w { z18.s }, p1/Z, [%x[params], #3, MUL VL]\n" + "mov z0.d, z9.d\n" + "fmla z24.s, z17.s, z4.s[1]\n" + "fmla z27.s, z17.s, z1.s[1]\n" + "fmla z26.s, z17.s, z1.s[3]\n" + "fmla z28.s, z17.s, z0.s[1]\n" + "mov z1.d, z12.d\n" + "mov z0.d, z13.d\n" + "fmla z21.s, z17.s, z4.s[3]\n" + "fmla z25.s, z17.s, z5.s[1]\n" + "fmla z20.s, z17.s, z1.s[1]\n" + "fmla z23.s, z17.s, z1.s[3]\n" + "mov z1.d, z8.d\n" + "fmla z19.s, z17.s, z0.s[1]\n" + "mov z0.d, z9.d\n" + "fmla z24.s, z31.s, z4.s[2]\n" + "ld1w { z17.s }, p1/Z, [%x[params], #4, MUL VL]\n" + "fmla z27.s, z31.s, z1.s[2]\n" + "fmla z26.s, z31.s, z0.s[0]\n" + "mov z1.d, z12.d\n" + "fmla z28.s, z31.s, z0.s[2]\n" + "mov z0.d, z13.d\n" + "fmla z21.s, z31.s, z5.s[0]\n" + "fmla z25.s, z31.s, z5.s[2]\n" + "fmla z20.s, z31.s, z1.s[2]\n" + "mov z1.d, z10.d\n" "fmla z23.s, z31.s, z0.s[0]\n" - "fmla z22.s, z31.s, z0.s[2]\n" - "mov z0.d, z12.d\n" - "fmla z21.s, z31.s, z1.s[0]\n" - "mov z1.d, z13.d\n" - "fmla z25.s, z31.s, z4.s[2]\n" - "fmla z24.s, z31.s, z5.s[0]\n" - "fmla z20.s, z31.s, z0.s[0]\n" "fmla z19.s, z31.s, z0.s[2]\n" - "fmla z18.s, z31.s, z1.s[0]\n" - "mov z0.d, z8.d\n" - "ld1w { z31.s }, p1/Z, [%x[params], #3, MUL VL]\n" - "mov z1.d, z9.d\n" - "fmla z26.s, z30.s, z4.s[1]\n" - "fmla z23.s, z30.s, z0.s[1]\n" - "fmla z22.s, z30.s, z0.s[3]\n" - "fmla z21.s, z30.s, z1.s[1]\n" - "mov z0.d, z12.d\n" - "mov z1.d, z13.d\n" - "fmla z25.s, z30.s, z4.s[3]\n" - "fmla z24.s, z30.s, z5.s[1]\n" - "fmla z20.s, z30.s, z0.s[1]\n" - "fmla z19.s, z30.s, z0.s[3]\n" - "mov z0.d, z8.d\n" - "fmla z18.s, z30.s, z1.s[1]\n" - "mov z1.d, z9.d\n" - "fmla z26.s, z29.s, z4.s[2]\n" - "ld1w { z30.s }, p1/Z, [%x[params], #4, MUL VL]\n" - "fmla z23.s, z29.s, z0.s[2]\n" - "fmla z22.s, z29.s, z1.s[0]\n" - "mov z0.d, z12.d\n" - "fmla z21.s, z29.s, z1.s[2]\n" - "mov z1.d, z13.d\n" - "fmla z25.s, z29.s, z5.s[0]\n" - "fmla z24.s, z29.s, z5.s[2]\n" - "fmla z20.s, z29.s, z0.s[2]\n" - "mov z0.d, z10.d\n" - "fmla z19.s, z29.s, z1.s[0]\n" - "fmla z18.s, z29.s, z1.s[2]\n" - "mov z1.d, z11.d\n" + "mov z0.d, z11.d\n" "ld1w { z29.s }, p1/Z, [%x[params], #5, MUL VL]\n" - "fmla z26.s, z31.s, z6.s[0]\n" - "fmla z23.s, z31.s, z0.s[0]\n" - "fmla z22.s, z31.s, z0.s[2]\n" - "fmla z21.s, z31.s, z1.s[0]\n" - "mov z0.d, z14.d\n" - "mov z1.d, z15.d\n" - "fmla z25.s, z31.s, z6.s[2]\n" - "fmla z24.s, z31.s, z7.s[0]\n" - "fmla z20.s, z31.s, z0.s[0]\n" - "fmla z19.s, z31.s, z0.s[2]\n" - "mov z0.d, z10.d\n" - "fmla z18.s, z31.s, z1.s[0]\n" - "mov z1.d, z11.d\n" - "fmla z26.s, z30.s, z6.s[1]\n" + "fmla z24.s, z18.s, z6.s[0]\n" + "fmla z27.s, z18.s, z1.s[0]\n" + "fmla z26.s, z18.s, z1.s[2]\n" + "fmla z28.s, z18.s, z0.s[0]\n" + "mov z1.d, z14.d\n" + "mov z0.d, z15.d\n" + "fmla z21.s, z18.s, z6.s[2]\n" + "fmla z25.s, z18.s, z7.s[0]\n" + "fmla z20.s, z18.s, z1.s[0]\n" + "fmla z23.s, z18.s, z1.s[2]\n" + "mov z1.d, z10.d\n" + "fmla z19.s, z18.s, z0.s[0]\n" + "mov z0.d, z11.d\n" + "fmla z24.s, z17.s, z6.s[1]\n" "ld1w { z31.s }, p2/Z, [%x[params], #7, MUL VL]\n" - "fmla z23.s, z30.s, z0.s[1]\n" - "fmla z22.s, z30.s, z0.s[3]\n" - "mov z0.d, z14.d\n" - "fmla z21.s, z30.s, z1.s[1]\n" - "mov z1.d, z15.d\n" - "fmla z25.s, z30.s, z6.s[3]\n" - "fmla z24.s, z30.s, z7.s[1]\n" - "fmla z20.s, z30.s, z0.s[1]\n" - "fmla z19.s, z30.s, z0.s[3]\n" - "fmla z18.s, z30.s, z1.s[1]\n" - "mov z0.d, z10.d\n" - "mov z1.d, z11.d\n" - "fmla z26.s, z29.s, z6.s[2]\n" - "fmla z23.s, z29.s, z0.s[2]\n" - "fmin z26.s, p1/M, z26.s, z16.s\n" - "fmla z22.s, z29.s, z1.s[0]\n" - "fmla z21.s, z29.s, z1.s[2]\n" - "mov z0.d, z14.d\n" - "fmax z26.s, p1/M, z26.s, z17.s\n" - "mov z1.d, z15.d\n" - "fmla z25.s, z29.s, z7.s[0]\n" - "fmla z24.s, z29.s, z7.s[2]\n" - "fmin z25.s, p1/M, z25.s, z16.s\n" - "fmla z20.s, z29.s, z0.s[2]\n" - "fmla z19.s, z29.s, z1.s[0]\n" + "fmla z27.s, z17.s, z1.s[1]\n" + "fmla z26.s, z17.s, z1.s[3]\n" + "mov z1.d, z14.d\n" + "fmla z28.s, z17.s, z0.s[1]\n" + "mov z0.d, z15.d\n" + "fmla z21.s, z17.s, z6.s[3]\n" + "fmla z25.s, z17.s, z7.s[1]\n" + "fmla z20.s, z17.s, z1.s[1]\n" + "fmla z23.s, z17.s, z1.s[3]\n" + "fmla z19.s, z17.s, z0.s[1]\n" + "mov z1.d, z10.d\n" + "mov z0.d, z11.d\n" + "fmla z24.s, z29.s, z6.s[2]\n" + "fmla z27.s, z29.s, z1.s[2]\n" "fmin z24.s, p1/M, z24.s, z16.s\n" - "fmin z23.s, p1/M, z23.s, z16.s\n" - "fmla z18.s, z29.s, z1.s[2]\n" - "fmin z22.s, p1/M, z22.s, z16.s\n" + "fmla z26.s, z29.s, z0.s[0]\n" + "fmla z28.s, z29.s, z0.s[2]\n" + "mov z1.d, z14.d\n" + "fmax z24.s, p1/M, z24.s, z22.s\n" + "mov z0.d, z15.d\n" + "fmla z21.s, z29.s, z7.s[0]\n" + "fmla z25.s, z29.s, z7.s[2]\n" "fmin z21.s, p1/M, z21.s, z16.s\n" - "st1w { z26.s }, p0, [x28, x12, LSL #2]\n" + "fmla z20.s, z29.s, z1.s[2]\n" + "fmla z23.s, z29.s, z0.s[0]\n" + "fmin z25.s, p1/M, z25.s, z16.s\n" + "fmin z27.s, p1/M, z27.s, z16.s\n" + "fmla z19.s, z29.s, z0.s[2]\n" + "fmin z26.s, p1/M, z26.s, z16.s\n" + "fmin z28.s, p1/M, z28.s, z16.s\n" + "st1w { z24.s }, p0, [x28, x12, LSL #2]\n" "fmin z20.s, p1/M, z20.s, z16.s\n" + "fmin z23.s, p1/M, z23.s, z16.s\n" + "ld1w { z24.s }, p2/Z, [%x[params], #6, MUL VL]\n" "fmin z19.s, p1/M, z19.s, z16.s\n" - "ld1w { z26.s }, p2/Z, [%x[params], #6, MUL VL]\n" - "fmin z18.s, p1/M, z18.s, z16.s\n" "addvl %x[params], %x[params], #16\n" "ld1w { z30.s }, p2/Z, [%x[params], #-8, MUL VL]\n" "ld1w { z29.s }, p2/Z, [%x[params], #-7, MUL VL]\n" - "fmax z25.s, p1/M, z25.s, z17.s\n" - "fmax z24.s, p1/M, z24.s, z17.s\n" - "st1w { z25.s }, p0, [x27, x12, LSL #2]\n" - "mov z25.d, z26.d\n" - "fmax z23.s, p1/M, z23.s, z17.s\n" - "fmax z22.s, p1/M, z22.s, z17.s\n" - "st1w { z24.s }, p0, [x26, x12, LSL #2]\n" - "mov z24.d, z26.d\n" - "fmax z21.s, p1/M, z21.s, z17.s\n" - "fmax z20.s, p1/M, z20.s, z17.s\n" - "st1w { z23.s }, p0, [x25, x12, LSL #2]\n" - "mov z23.d, z26.d\n" - "fmax z19.s, p1/M, z19.s, z17.s\n" - "fmax z18.s, p1/M, z18.s, z17.s\n" - "st1w { z22.s }, p0, [x24, x12, LSL #2]\n" - "mov z22.d, z26.d\n" - "st1w { z21.s }, p0, [x23, x12, LSL #2]\n" - "mov z21.d, z26.d\n" + "fmax z21.s, p1/M, z21.s, z22.s\n" + "fmax z25.s, p1/M, z25.s, z22.s\n" + "st1w { z21.s }, p0, [x27, x12, LSL #2]\n" + "mov z21.d, z24.d\n" + "fmax z27.s, p1/M, z27.s, z22.s\n" + "fmax z26.s, p1/M, z26.s, z22.s\n" + "st1w { z25.s }, p0, [x26, x12, LSL #2]\n" + "mov z25.d, z24.d\n" + "fmax z28.s, p1/M, z28.s, z22.s\n" + "fmax z20.s, p1/M, z20.s, z22.s\n" + "st1w { z27.s }, p0, [x25, x12, LSL #2]\n" + "mov z27.d, z24.d\n" + "fmax z23.s, p1/M, z23.s, z22.s\n" + "fmax z19.s, p1/M, z19.s, z22.s\n" + "st1w { z26.s }, p0, [x24, x12, LSL #2]\n" + "mov z26.d, z24.d\n" + "st1w { z28.s }, p0, [x23, x12, LSL #2]\n" + "mov z28.d, z24.d\n" "addvl %x[params], %x[params], #-6\n" "st1w { z20.s }, p0, [x22, x12, LSL #2]\n" - "mov z20.d, z26.d\n" - "st1w { z19.s }, p0, [x21, x12, LSL #2]\n" - "mov z19.d, z26.d\n" - "st1w { z18.s }, p0, [x20, x12, LSL #2]\n" + "mov z20.d, z24.d\n" + "st1w { z23.s }, p0, [x21, x12, LSL #2]\n" + "mov z23.d, z24.d\n" + "st1w { z19.s }, p0, [x20, x12, LSL #2]\n" "incw x12\n" - "mov z18.d, z26.d\n" + "mov z19.d, z24.d\n" "b.any 1b\n" : [params] "+&r" (params) : [channel_multiplier] "r" (n_output_channels), [clamps] "r" (minmax_vals), [inptrs] "r" (inptrs), [outptrs] "r" (outptrs) - : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst.hpp index a4ee87cce22c506bf69d111833bfbe68c512a982..cc0c4236a8fd1d985f09a904dd30dadd3981c917 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst/generic.cpp index 2ea116fc9ed6d12dd6153e7ed8d507f409ccdf96..84ab4b5035b0916a11d3a5142a1cab9326b61a17 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst/generic.cpp @@ -52,21 +52,21 @@ void sve_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst_imp "mov x10, #0x0\n" "ldr x9, [%x[inptrs], #0x20]\n" "ldr x28, [%x[inptrs], #0x28]\n" - "ld1w { z25.s }, p2/Z, [%x[params]]\n" + "ld1w { z16.s }, p2/Z, [%x[params]]\n" "ldp x27, x26, [%x[outptrs], #0x0]\n" - "mov z24.d, z25.d\n" - "mov z23.d, z25.d\n" + "mov z25.d, z16.d\n" + "mov z15.d, z16.d\n" "ldp x25, x24, [%x[outptrs], #0x10]\n" "ldp x23, x22, [%x[outptrs], #0x20]\n" - "mov z22.d, z25.d\n" - "mov z21.d, z25.d\n" + "mov z24.d, z16.d\n" + "mov z14.d, z16.d\n" "ldp x21, x20, [%x[outptrs], #0x30]\n" "ld1rqw { z2.s }, p1/Z, [x14]\n" - "mov z20.d, z25.d\n" - "mov z19.d, z25.d\n" + "mov z26.d, z16.d\n" + "mov z17.d, z16.d\n" "ld1rqw { z3.s }, p1/Z, [x14, #16]\n" "ld1rqw { z4.s }, p1/Z, [x13]\n" - "mov z18.d, z25.d\n" + "mov z23.d, z16.d\n" "ld1rqw { z5.s }, p1/Z, [x13, #16]\n" "ld1rqw { z6.s }, p1/Z, [x12]\n" "ld1rqw { z7.s }, p1/Z, [x12, #16]\n" @@ -76,8 +76,8 @@ void sve_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst_imp "ld1rqw { z11.s }, p1/Z, [x9, #16]\n" "ld1rqw { z12.s }, p1/Z, [x28]\n" "ld1rqw { z13.s }, p1/Z, [x28, #16]\n" - "ld1rw { z17.s }, p1/Z, [%x[clamps]]\n" - "ld1rw { z16.s }, p1/Z, [%x[clamps], #4]\n" + "ld1rw { z21.s }, p1/Z, [%x[clamps]]\n" + "ld1rw { z22.s }, p1/Z, [%x[clamps], #4]\n" "ld1w { z31.s }, p2/Z, [%x[params], #1, MUL VL]\n" "ld1w { z30.s }, p2/Z, [%x[params], #2, MUL VL]\n" "ld1w { z29.s }, p2/Z, [%x[params], #3, MUL VL]\n" @@ -85,304 +85,304 @@ void sve_fp32_packed_to_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst_imp "ld1w { z27.s }, p2/Z, [%x[params], #5, MUL VL]\n" "addvl %x[params], %x[params], #6\n" "1:" // Output channel complete vector loop - "fmla z25.s, z31.s, z2.s[0]\n" - "fmla z24.s, z31.s, z2.s[1]\n" + "fmla z16.s, z31.s, z2.s[0]\n" + "fmla z25.s, z31.s, z2.s[1]\n" "mov z0.d, z8.d\n" "incw x15\n" - "fmla z23.s, z31.s, z2.s[2]\n" - "fmla z22.s, z31.s, z2.s[3]\n" + "fmla z15.s, z31.s, z2.s[2]\n" + "fmla z24.s, z31.s, z2.s[3]\n" "mov z1.d, z9.d\n" "mov p0.b, p2.b\n" - "fmla z21.s, z31.s, z4.s[0]\n" - "fmla z20.s, z31.s, z4.s[1]\n" + "fmla z14.s, z31.s, z4.s[0]\n" + "fmla z26.s, z31.s, z4.s[1]\n" "whilelt p2.s, x15, %x[channel_multiplier]\n" - "fmla z19.s, z31.s, z4.s[2]\n" - "fmla z18.s, z31.s, z4.s[3]\n" - "ld1w { z31.s }, p1/Z, [%x[params]]\n" - "fmla z25.s, z30.s, z2.s[1]\n" - "fmla z24.s, z30.s, z2.s[2]\n" - "fmla z23.s, z30.s, z2.s[3]\n" - "fmla z22.s, z30.s, z3.s[0]\n" - "fmla z21.s, z30.s, z4.s[1]\n" - "fmla z20.s, z30.s, z4.s[2]\n" - "fmla z19.s, z30.s, z4.s[3]\n" - "fmla z18.s, z30.s, z5.s[0]\n" - "ld1w { z30.s }, p1/Z, [%x[params], #1, MUL VL]\n" - "fmla z25.s, z29.s, z2.s[2]\n" - "fmla z24.s, z29.s, z2.s[3]\n" - "fmla z23.s, z29.s, z3.s[0]\n" - "fmla z22.s, z29.s, z3.s[1]\n" - "fmla z21.s, z29.s, z4.s[2]\n" - "fmla z20.s, z29.s, z4.s[3]\n" - "fmla z19.s, z29.s, z5.s[0]\n" - "fmla z18.s, z29.s, z5.s[1]\n" - "ld1w { z29.s }, p1/Z, [%x[params], #2, MUL VL]\n" - "fmla z25.s, z28.s, z2.s[3]\n" - "fmla z24.s, z28.s, z3.s[0]\n" - "fmla z23.s, z28.s, z3.s[1]\n" - "fmla z22.s, z28.s, z3.s[2]\n" - "fmla z21.s, z28.s, z4.s[3]\n" - "fmla z20.s, z28.s, z5.s[0]\n" - "fmla z19.s, z28.s, z5.s[1]\n" - "fmla z18.s, z28.s, z5.s[2]\n" + "fmla z17.s, z31.s, z4.s[2]\n" + "fmla z23.s, z31.s, z4.s[3]\n" + "ld1w { z20.s }, p1/Z, [%x[params]]\n" + "fmla z16.s, z30.s, z2.s[1]\n" + "fmla z25.s, z30.s, z2.s[2]\n" + "fmla z15.s, z30.s, z2.s[3]\n" + "fmla z24.s, z30.s, z3.s[0]\n" + "fmla z14.s, z30.s, z4.s[1]\n" + "fmla z26.s, z30.s, z4.s[2]\n" + "fmla z17.s, z30.s, z4.s[3]\n" + "fmla z23.s, z30.s, z5.s[0]\n" + "ld1w { z19.s }, p1/Z, [%x[params], #1, MUL VL]\n" + "fmla z16.s, z29.s, z2.s[2]\n" + "fmla z25.s, z29.s, z2.s[3]\n" + "fmla z15.s, z29.s, z3.s[0]\n" + "fmla z24.s, z29.s, z3.s[1]\n" + "fmla z14.s, z29.s, z4.s[2]\n" + "fmla z26.s, z29.s, z4.s[3]\n" + "fmla z17.s, z29.s, z5.s[0]\n" + "fmla z23.s, z29.s, z5.s[1]\n" + "ld1w { z18.s }, p1/Z, [%x[params], #2, MUL VL]\n" + "fmla z16.s, z28.s, z2.s[3]\n" + "fmla z25.s, z28.s, z3.s[0]\n" + "fmla z15.s, z28.s, z3.s[1]\n" + "fmla z24.s, z28.s, z3.s[2]\n" + "fmla z14.s, z28.s, z4.s[3]\n" + "fmla z26.s, z28.s, z5.s[0]\n" + "fmla z17.s, z28.s, z5.s[1]\n" + "fmla z23.s, z28.s, z5.s[2]\n" "ld1w { z28.s }, p1/Z, [%x[params], #3, MUL VL]\n" - "fmla z25.s, z27.s, z3.s[0]\n" - "fmla z24.s, z27.s, z3.s[1]\n" - "fmla z23.s, z27.s, z3.s[2]\n" - "fmla z22.s, z27.s, z3.s[3]\n" - "fmla z21.s, z27.s, z5.s[0]\n" - "fmla z20.s, z27.s, z5.s[1]\n" - "fmla z19.s, z27.s, z5.s[2]\n" - "fmla z18.s, z27.s, z5.s[3]\n" + "fmla z16.s, z27.s, z3.s[0]\n" + "fmla z25.s, z27.s, z3.s[1]\n" + "fmla z15.s, z27.s, z3.s[2]\n" + "fmla z24.s, z27.s, z3.s[3]\n" + "fmla z14.s, z27.s, z5.s[0]\n" + "fmla z26.s, z27.s, z5.s[1]\n" + "fmla z17.s, z27.s, z5.s[2]\n" + "fmla z23.s, z27.s, z5.s[3]\n" "ld1w { z27.s }, p1/Z, [%x[params], #4, MUL VL]\n" - "fmla z25.s, z31.s, z4.s[0]\n" - "fmla z24.s, z31.s, z4.s[1]\n" - "fmla z23.s, z31.s, z4.s[2]\n" - "fmla z22.s, z31.s, z4.s[3]\n" - "fmla z21.s, z31.s, z6.s[0]\n" - "fmla z20.s, z31.s, z6.s[1]\n" - "fmla z19.s, z31.s, z6.s[2]\n" - "fmla z18.s, z31.s, z6.s[3]\n" - "ld1w { z31.s }, p1/Z, [%x[params], #5, MUL VL]\n" - "fmla z25.s, z30.s, z4.s[1]\n" - "fmla z24.s, z30.s, z4.s[2]\n" - "fmla z23.s, z30.s, z4.s[3]\n" - "fmla z22.s, z30.s, z5.s[0]\n" - "fmla z21.s, z30.s, z6.s[1]\n" - "fmla z20.s, z30.s, z6.s[2]\n" - "fmla z19.s, z30.s, z6.s[3]\n" - "fmla z18.s, z30.s, z7.s[0]\n" - "ld1w { z30.s }, p1/Z, [%x[params], #6, MUL VL]\n" - "fmla z25.s, z29.s, z4.s[2]\n" - "fmla z24.s, z29.s, z4.s[3]\n" - "fmla z23.s, z29.s, z5.s[0]\n" - "fmla z22.s, z29.s, z5.s[1]\n" - "fmla z21.s, z29.s, z6.s[2]\n" - "fmla z20.s, z29.s, z6.s[3]\n" - "fmla z19.s, z29.s, z7.s[0]\n" - "fmla z18.s, z29.s, z7.s[1]\n" - "ld1w { z29.s }, p1/Z, [%x[params], #7, MUL VL]\n" + "fmla z16.s, z20.s, z4.s[0]\n" + "fmla z25.s, z20.s, z4.s[1]\n" + "fmla z15.s, z20.s, z4.s[2]\n" + "fmla z24.s, z20.s, z4.s[3]\n" + "fmla z14.s, z20.s, z6.s[0]\n" + "fmla z26.s, z20.s, z6.s[1]\n" + "fmla z17.s, z20.s, z6.s[2]\n" + "fmla z23.s, z20.s, z6.s[3]\n" + "ld1w { z20.s }, p1/Z, [%x[params], #5, MUL VL]\n" + "fmla z16.s, z19.s, z4.s[1]\n" + "fmla z25.s, z19.s, z4.s[2]\n" + "fmla z15.s, z19.s, z4.s[3]\n" + "fmla z24.s, z19.s, z5.s[0]\n" + "fmla z14.s, z19.s, z6.s[1]\n" + "fmla z26.s, z19.s, z6.s[2]\n" + "fmla z17.s, z19.s, z6.s[3]\n" + "fmla z23.s, z19.s, z7.s[0]\n" + "ld1w { z19.s }, p1/Z, [%x[params], #6, MUL VL]\n" + "fmla z16.s, z18.s, z4.s[2]\n" + "fmla z25.s, z18.s, z4.s[3]\n" + "fmla z15.s, z18.s, z5.s[0]\n" + "fmla z24.s, z18.s, z5.s[1]\n" + "fmla z14.s, z18.s, z6.s[2]\n" + "fmla z26.s, z18.s, z6.s[3]\n" + "fmla z17.s, z18.s, z7.s[0]\n" + "fmla z23.s, z18.s, z7.s[1]\n" + "ld1w { z18.s }, p1/Z, [%x[params], #7, MUL VL]\n" "addvl %x[params], %x[params], #16\n" - "fmla z25.s, z28.s, z4.s[3]\n" - "fmla z24.s, z28.s, z5.s[0]\n" - "fmla z23.s, z28.s, z5.s[1]\n" - "fmla z22.s, z28.s, z5.s[2]\n" - "fmla z21.s, z28.s, z6.s[3]\n" - "fmla z20.s, z28.s, z7.s[0]\n" - "fmla z19.s, z28.s, z7.s[1]\n" - "fmla z18.s, z28.s, z7.s[2]\n" - "ld1w { z28.s }, p1/Z, [%x[params], #-8, MUL VL]\n" - "fmla z25.s, z27.s, z5.s[0]\n" - "fmla z24.s, z27.s, z5.s[1]\n" - "fmla z23.s, z27.s, z5.s[2]\n" - "fmla z22.s, z27.s, z5.s[3]\n" - "fmla z21.s, z27.s, z7.s[0]\n" - "fmla z20.s, z27.s, z7.s[1]\n" - "fmla z19.s, z27.s, z7.s[2]\n" - "fmla z18.s, z27.s, z7.s[3]\n" + "fmla z16.s, z28.s, z4.s[3]\n" + "fmla z25.s, z28.s, z5.s[0]\n" + "fmla z15.s, z28.s, z5.s[1]\n" + "fmla z24.s, z28.s, z5.s[2]\n" + "fmla z14.s, z28.s, z6.s[3]\n" + "fmla z26.s, z28.s, z7.s[0]\n" + "fmla z17.s, z28.s, z7.s[1]\n" + "fmla z23.s, z28.s, z7.s[2]\n" + "ld1w { z30.s }, p1/Z, [%x[params], #-8, MUL VL]\n" + "fmla z16.s, z27.s, z5.s[0]\n" + "fmla z25.s, z27.s, z5.s[1]\n" + "fmla z15.s, z27.s, z5.s[2]\n" + "fmla z24.s, z27.s, z5.s[3]\n" + "fmla z14.s, z27.s, z7.s[0]\n" + "fmla z26.s, z27.s, z7.s[1]\n" + "fmla z17.s, z27.s, z7.s[2]\n" + "fmla z23.s, z27.s, z7.s[3]\n" "ld1w { z27.s }, p1/Z, [%x[params], #-7, MUL VL]\n" - "fmla z25.s, z31.s, z6.s[0]\n" - "fmla z24.s, z31.s, z6.s[1]\n" - "fmla z23.s, z31.s, z6.s[2]\n" - "fmla z22.s, z31.s, z6.s[3]\n" - "fmla z21.s, z31.s, z0.s[0]\n" - "fmla z20.s, z31.s, z0.s[1]\n" - "fmla z19.s, z31.s, z0.s[2]\n" - "fmla z18.s, z31.s, z0.s[3]\n" - "ld1w { z31.s }, p1/Z, [%x[params], #-6, MUL VL]\n" - "fmla z25.s, z30.s, z6.s[1]\n" - "fmla z24.s, z30.s, z6.s[2]\n" - "fmla z23.s, z30.s, z6.s[3]\n" - "fmla z22.s, z30.s, z7.s[0]\n" - "fmla z21.s, z30.s, z0.s[1]\n" - "fmla z20.s, z30.s, z0.s[2]\n" - "fmla z19.s, z30.s, z0.s[3]\n" - "fmla z18.s, z30.s, z1.s[0]\n" - "ld1w { z30.s }, p1/Z, [%x[params], #-5, MUL VL]\n" - "fmla z25.s, z29.s, z6.s[2]\n" - "fmla z24.s, z29.s, z6.s[3]\n" - "fmla z23.s, z29.s, z7.s[0]\n" - "fmla z22.s, z29.s, z7.s[1]\n" - "fmla z21.s, z29.s, z0.s[2]\n" - "fmla z20.s, z29.s, z0.s[3]\n" - "fmla z19.s, z29.s, z1.s[0]\n" - "fmla z18.s, z29.s, z1.s[1]\n" - "ld1w { z29.s }, p1/Z, [%x[params], #-4, MUL VL]\n" - "fmla z25.s, z28.s, z6.s[3]\n" - "fmla z24.s, z28.s, z7.s[0]\n" - "fmla z23.s, z28.s, z7.s[1]\n" - "fmla z22.s, z28.s, z7.s[2]\n" - "fmla z21.s, z28.s, z0.s[3]\n" - "fmla z20.s, z28.s, z1.s[0]\n" - "fmla z19.s, z28.s, z1.s[1]\n" - "fmla z18.s, z28.s, z1.s[2]\n" - "ld1w { z28.s }, p1/Z, [%x[params], #-3, MUL VL]\n" - "fmla z25.s, z27.s, z7.s[0]\n" - "fmla z24.s, z27.s, z7.s[1]\n" - "fmla z23.s, z27.s, z7.s[2]\n" - "fmla z22.s, z27.s, z7.s[3]\n" - "fmla z21.s, z27.s, z1.s[0]\n" - "fmla z20.s, z27.s, z1.s[1]\n" - "fmla z19.s, z27.s, z1.s[2]\n" - "fmla z18.s, z27.s, z1.s[3]\n" + "fmla z16.s, z20.s, z6.s[0]\n" + "fmla z25.s, z20.s, z6.s[1]\n" + "fmla z15.s, z20.s, z6.s[2]\n" + "fmla z24.s, z20.s, z6.s[3]\n" + "fmla z14.s, z20.s, z0.s[0]\n" + "fmla z26.s, z20.s, z0.s[1]\n" + "fmla z17.s, z20.s, z0.s[2]\n" + "fmla z23.s, z20.s, z0.s[3]\n" + "ld1w { z20.s }, p1/Z, [%x[params], #-6, MUL VL]\n" + "fmla z16.s, z19.s, z6.s[1]\n" + "fmla z25.s, z19.s, z6.s[2]\n" + "fmla z15.s, z19.s, z6.s[3]\n" + "fmla z24.s, z19.s, z7.s[0]\n" + "fmla z14.s, z19.s, z0.s[1]\n" + "fmla z26.s, z19.s, z0.s[2]\n" + "fmla z17.s, z19.s, z0.s[3]\n" + "fmla z23.s, z19.s, z1.s[0]\n" + "ld1w { z19.s }, p1/Z, [%x[params], #-5, MUL VL]\n" + "fmla z16.s, z18.s, z6.s[2]\n" + "fmla z25.s, z18.s, z6.s[3]\n" + "fmla z15.s, z18.s, z7.s[0]\n" + "fmla z24.s, z18.s, z7.s[1]\n" + "fmla z14.s, z18.s, z0.s[2]\n" + "fmla z26.s, z18.s, z0.s[3]\n" + "fmla z17.s, z18.s, z1.s[0]\n" + "fmla z23.s, z18.s, z1.s[1]\n" + "ld1w { z18.s }, p1/Z, [%x[params], #-4, MUL VL]\n" + "fmla z16.s, z30.s, z6.s[3]\n" + "fmla z25.s, z30.s, z7.s[0]\n" + "fmla z15.s, z30.s, z7.s[1]\n" + "fmla z24.s, z30.s, z7.s[2]\n" + "fmla z14.s, z30.s, z0.s[3]\n" + "fmla z26.s, z30.s, z1.s[0]\n" + "fmla z17.s, z30.s, z1.s[1]\n" + "fmla z23.s, z30.s, z1.s[2]\n" + "ld1w { z31.s }, p1/Z, [%x[params], #-3, MUL VL]\n" + "fmla z16.s, z27.s, z7.s[0]\n" + "fmla z25.s, z27.s, z7.s[1]\n" + "fmla z15.s, z27.s, z7.s[2]\n" + "fmla z24.s, z27.s, z7.s[3]\n" + "fmla z14.s, z27.s, z1.s[0]\n" + "fmla z26.s, z27.s, z1.s[1]\n" + "fmla z17.s, z27.s, z1.s[2]\n" + "fmla z23.s, z27.s, z1.s[3]\n" "ld1w { z27.s }, p1/Z, [%x[params], #-2, MUL VL]\n" - "fmla z25.s, z31.s, z0.s[0]\n" - "fmla z24.s, z31.s, z0.s[1]\n" - "fmla z23.s, z31.s, z0.s[2]\n" - "fmla z22.s, z31.s, z0.s[3]\n" - "mov z0.d, z10.d\n" - "fmla z21.s, z31.s, z0.s[0]\n" - "fmla z20.s, z31.s, z0.s[1]\n" - "fmla z19.s, z31.s, z0.s[2]\n" - "fmla z18.s, z31.s, z0.s[3]\n" - "mov z0.d, z8.d\n" - "ld1w { z31.s }, p1/Z, [%x[params], #-1, MUL VL]\n" - "fmla z25.s, z30.s, z0.s[1]\n" - "fmla z24.s, z30.s, z0.s[2]\n" - "fmla z23.s, z30.s, z0.s[3]\n" - "fmla z22.s, z30.s, z1.s[0]\n" - "mov z0.d, z10.d\n" - "mov z1.d, z11.d\n" - "fmla z21.s, z30.s, z0.s[1]\n" - "fmla z20.s, z30.s, z0.s[2]\n" - "fmla z19.s, z30.s, z0.s[3]\n" - "fmla z18.s, z30.s, z1.s[0]\n" - "mov z0.d, z8.d\n" - "ld1w { z30.s }, p1/Z, [%x[params]]\n" - "mov z1.d, z9.d\n" - "fmla z25.s, z29.s, z0.s[2]\n" - "fmla z24.s, z29.s, z0.s[3]\n" - "fmla z23.s, z29.s, z1.s[0]\n" - "fmla z22.s, z29.s, z1.s[1]\n" + "fmla z16.s, z20.s, z0.s[0]\n" + "fmla z25.s, z20.s, z0.s[1]\n" + "fmla z15.s, z20.s, z0.s[2]\n" + "fmla z24.s, z20.s, z0.s[3]\n" "mov z0.d, z10.d\n" - "mov z1.d, z11.d\n" - "fmla z21.s, z29.s, z0.s[2]\n" - "fmla z20.s, z29.s, z0.s[3]\n" - "fmla z19.s, z29.s, z1.s[0]\n" - "fmla z18.s, z29.s, z1.s[1]\n" + "fmla z14.s, z20.s, z0.s[0]\n" + "fmla z26.s, z20.s, z0.s[1]\n" + "fmla z17.s, z20.s, z0.s[2]\n" + "fmla z23.s, z20.s, z0.s[3]\n" "mov z0.d, z8.d\n" - "ld1w { z29.s }, p1/Z, [%x[params], #1, MUL VL]\n" - "mov z1.d, z9.d\n" - "fmla z25.s, z28.s, z0.s[3]\n" - "fmla z24.s, z28.s, z1.s[0]\n" - "fmla z23.s, z28.s, z1.s[1]\n" - "fmla z22.s, z28.s, z1.s[2]\n" + "ld1w { z20.s }, p1/Z, [%x[params], #-1, MUL VL]\n" + "fmla z16.s, z19.s, z0.s[1]\n" + "fmla z25.s, z19.s, z0.s[2]\n" + "fmla z15.s, z19.s, z0.s[3]\n" + "fmla z24.s, z19.s, z1.s[0]\n" + "mov z1.d, z10.d\n" + "mov z0.d, z11.d\n" + "fmla z14.s, z19.s, z1.s[1]\n" + "fmla z26.s, z19.s, z1.s[2]\n" + "fmla z17.s, z19.s, z1.s[3]\n" + "fmla z23.s, z19.s, z0.s[0]\n" + "mov z1.d, z8.d\n" + "ld1w { z19.s }, p1/Z, [%x[params]]\n" + "mov z0.d, z9.d\n" + "fmla z16.s, z18.s, z1.s[2]\n" + "fmla z25.s, z18.s, z1.s[3]\n" + "fmla z15.s, z18.s, z0.s[0]\n" + "fmla z24.s, z18.s, z0.s[1]\n" + "mov z1.d, z10.d\n" + "mov z0.d, z11.d\n" + "fmla z14.s, z18.s, z1.s[2]\n" + "fmla z26.s, z18.s, z1.s[3]\n" + "fmla z17.s, z18.s, z0.s[0]\n" + "fmla z23.s, z18.s, z0.s[1]\n" + "mov z1.d, z8.d\n" + "ld1w { z18.s }, p1/Z, [%x[params], #1, MUL VL]\n" + "mov z0.d, z9.d\n" + "fmla z16.s, z31.s, z1.s[3]\n" + "fmla z25.s, z31.s, z0.s[0]\n" + "fmla z15.s, z31.s, z0.s[1]\n" + "fmla z24.s, z31.s, z0.s[2]\n" "mov z0.d, z10.d\n" "mov z1.d, z11.d\n" - "fmla z21.s, z28.s, z0.s[3]\n" - "fmla z20.s, z28.s, z1.s[0]\n" - "fmla z19.s, z28.s, z1.s[1]\n" - "fmla z18.s, z28.s, z1.s[2]\n" + "fmla z14.s, z31.s, z0.s[3]\n" + "fmla z26.s, z31.s, z1.s[0]\n" + "fmla z17.s, z31.s, z1.s[1]\n" + "fmla z23.s, z31.s, z1.s[2]\n" "mov z1.d, z9.d\n" "ld1w { z28.s }, p1/Z, [%x[params], #2, MUL VL]\n" - "fmla z25.s, z27.s, z1.s[0]\n" - "fmla z24.s, z27.s, z1.s[1]\n" - "fmla z23.s, z27.s, z1.s[2]\n" - "fmla z22.s, z27.s, z1.s[3]\n" + "fmla z16.s, z27.s, z1.s[0]\n" + "fmla z25.s, z27.s, z1.s[1]\n" + "fmla z15.s, z27.s, z1.s[2]\n" + "fmla z24.s, z27.s, z1.s[3]\n" "mov z1.d, z11.d\n" - "fmla z21.s, z27.s, z1.s[0]\n" - "fmla z20.s, z27.s, z1.s[1]\n" - "fmla z19.s, z27.s, z1.s[2]\n" - "fmla z18.s, z27.s, z1.s[3]\n" + "fmla z14.s, z27.s, z1.s[0]\n" + "fmla z26.s, z27.s, z1.s[1]\n" + "fmla z17.s, z27.s, z1.s[2]\n" + "fmla z23.s, z27.s, z1.s[3]\n" "ld1w { z27.s }, p1/Z, [%x[params], #3, MUL VL]\n" - "fmla z25.s, z31.s, z0.s[0]\n" - "fmla z24.s, z31.s, z0.s[1]\n" - "fmla z23.s, z31.s, z0.s[2]\n" - "fmla z22.s, z31.s, z0.s[3]\n" + "fmla z16.s, z20.s, z0.s[0]\n" + "fmla z25.s, z20.s, z0.s[1]\n" + "fmla z15.s, z20.s, z0.s[2]\n" + "fmla z24.s, z20.s, z0.s[3]\n" "mov z0.d, z12.d\n" - "fmla z21.s, z31.s, z0.s[0]\n" - "fmla z20.s, z31.s, z0.s[1]\n" - "fmla z19.s, z31.s, z0.s[2]\n" - "fmla z18.s, z31.s, z0.s[3]\n" + "fmla z14.s, z20.s, z0.s[0]\n" + "fmla z26.s, z20.s, z0.s[1]\n" + "fmla z17.s, z20.s, z0.s[2]\n" + "fmla z23.s, z20.s, z0.s[3]\n" "mov z0.d, z10.d\n" "ld1w { z31.s }, p2/Z, [%x[params], #5, MUL VL]\n" - "fmla z25.s, z30.s, z0.s[1]\n" - "fmla z24.s, z30.s, z0.s[2]\n" - "fmla z23.s, z30.s, z0.s[3]\n" - "fmla z22.s, z30.s, z1.s[0]\n" - "mov z0.d, z12.d\n" - "mov z1.d, z13.d\n" - "fmla z21.s, z30.s, z0.s[1]\n" - "fmla z20.s, z30.s, z0.s[2]\n" - "fmla z19.s, z30.s, z0.s[3]\n" - "fmla z18.s, z30.s, z1.s[0]\n" - "mov z0.d, z10.d\n" + "fmla z16.s, z19.s, z0.s[1]\n" + "fmla z25.s, z19.s, z0.s[2]\n" + "fmla z15.s, z19.s, z0.s[3]\n" + "fmla z24.s, z19.s, z1.s[0]\n" + "mov z1.d, z12.d\n" + "mov z0.d, z13.d\n" + "fmla z14.s, z19.s, z1.s[1]\n" + "fmla z26.s, z19.s, z1.s[2]\n" + "fmla z17.s, z19.s, z1.s[3]\n" + "fmla z23.s, z19.s, z0.s[0]\n" + "mov z1.d, z10.d\n" "ld1w { z30.s }, p2/Z, [%x[params], #6, MUL VL]\n" - "mov z1.d, z11.d\n" - "fmla z25.s, z29.s, z0.s[2]\n" - "fmla z24.s, z29.s, z0.s[3]\n" - "fmla z23.s, z29.s, z1.s[0]\n" - "fmla z22.s, z29.s, z1.s[1]\n" - "mov z0.d, z12.d\n" - "mov z1.d, z13.d\n" - "fmla z21.s, z29.s, z0.s[2]\n" - "fmla z20.s, z29.s, z0.s[3]\n" - "fmla z19.s, z29.s, z1.s[0]\n" - "fmla z18.s, z29.s, z1.s[1]\n" - "mov z0.d, z10.d\n" + "mov z0.d, z11.d\n" + "fmla z16.s, z18.s, z1.s[2]\n" + "fmla z25.s, z18.s, z1.s[3]\n" + "fmla z15.s, z18.s, z0.s[0]\n" + "fmla z24.s, z18.s, z0.s[1]\n" + "mov z1.d, z12.d\n" + "mov z0.d, z13.d\n" + "fmla z14.s, z18.s, z1.s[2]\n" + "fmla z26.s, z18.s, z1.s[3]\n" + "fmla z17.s, z18.s, z0.s[0]\n" + "fmla z23.s, z18.s, z0.s[1]\n" + "mov z1.d, z10.d\n" "ld1w { z29.s }, p2/Z, [%x[params], #7, MUL VL]\n" - "mov z1.d, z11.d\n" - "fmla z25.s, z28.s, z0.s[3]\n" - "fmla z24.s, z28.s, z1.s[0]\n" - "fmla z23.s, z28.s, z1.s[1]\n" - "fmla z22.s, z28.s, z1.s[2]\n" - "mov z1.d, z13.d\n" - "mov z0.d, z12.d\n" - "fmla z20.s, z28.s, z1.s[0]\n" - "fmla z19.s, z28.s, z1.s[1]\n" - "fmla z18.s, z28.s, z1.s[2]\n" - "mov z1.d, z11.d\n" - "fmla z21.s, z28.s, z0.s[3]\n" - "fmla z25.s, z27.s, z1.s[0]\n" - "fmla z24.s, z27.s, z1.s[1]\n" - "fmin z25.s, p1/M, z25.s, z16.s\n" - "fmax z25.s, p1/M, z25.s, z17.s\n" - "fmla z23.s, z27.s, z1.s[2]\n" - "fmla z22.s, z27.s, z1.s[3]\n" - "mov z1.d, z13.d\n" - "fmin z24.s, p1/M, z24.s, z16.s\n" - "fmla z21.s, z27.s, z1.s[0]\n" - "fmla z20.s, z27.s, z1.s[1]\n" - "fmin z23.s, p1/M, z23.s, z16.s\n" - "fmin z22.s, p1/M, z22.s, z16.s\n" - "fmla z19.s, z27.s, z1.s[2]\n" - "fmla z18.s, z27.s, z1.s[3]\n" - "fmin z21.s, p1/M, z21.s, z16.s\n" - "fmin z20.s, p1/M, z20.s, z16.s\n" - "fmin z19.s, p1/M, z19.s, z16.s\n" - "fmin z18.s, p1/M, z18.s, z16.s\n" - "st1w { z25.s }, p0, [x27, x10, LSL #2]\n" - "ld1w { z25.s }, p2/Z, [%x[params], #4, MUL VL]\n" + "mov z0.d, z11.d\n" + "fmla z16.s, z28.s, z1.s[3]\n" + "fmla z25.s, z28.s, z0.s[0]\n" + "fmla z15.s, z28.s, z0.s[1]\n" + "fmla z24.s, z28.s, z0.s[2]\n" + "mov z0.d, z13.d\n" + "mov z1.d, z12.d\n" + "fmla z26.s, z28.s, z0.s[0]\n" + "fmla z17.s, z28.s, z0.s[1]\n" + "fmla z23.s, z28.s, z0.s[2]\n" + "mov z0.d, z11.d\n" + "fmla z14.s, z28.s, z1.s[3]\n" + "fmla z16.s, z27.s, z0.s[0]\n" + "fmla z25.s, z27.s, z0.s[1]\n" + "fmin z16.s, p1/M, z16.s, z22.s\n" + "fmax z16.s, p1/M, z16.s, z21.s\n" + "fmla z15.s, z27.s, z0.s[2]\n" + "fmla z24.s, z27.s, z0.s[3]\n" + "mov z0.d, z13.d\n" + "fmin z25.s, p1/M, z25.s, z22.s\n" + "fmla z14.s, z27.s, z0.s[0]\n" + "fmla z26.s, z27.s, z0.s[1]\n" + "fmin z15.s, p1/M, z15.s, z22.s\n" + "fmin z24.s, p1/M, z24.s, z22.s\n" + "fmla z17.s, z27.s, z0.s[2]\n" + "fmla z23.s, z27.s, z0.s[3]\n" + "fmin z14.s, p1/M, z14.s, z22.s\n" + "fmin z26.s, p1/M, z26.s, z22.s\n" + "fmin z17.s, p1/M, z17.s, z22.s\n" + "fmin z23.s, p1/M, z23.s, z22.s\n" + "st1w { z16.s }, p0, [x27, x10, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [%x[params], #4, MUL VL]\n" "addvl %x[params], %x[params], #16\n" - "fmax z24.s, p1/M, z24.s, z17.s\n" - "st1w { z24.s }, p0, [x26, x10, LSL #2]\n" - "mov z24.d, z25.d\n" - "fmax z23.s, p1/M, z23.s, z17.s\n" - "fmax z22.s, p1/M, z22.s, z17.s\n" - "st1w { z23.s }, p0, [x25, x10, LSL #2]\n" - "mov z23.d, z25.d\n" - "fmax z21.s, p1/M, z21.s, z17.s\n" - "fmax z20.s, p1/M, z20.s, z17.s\n" - "st1w { z22.s }, p0, [x24, x10, LSL #2]\n" - "mov z22.d, z25.d\n" - "fmax z19.s, p1/M, z19.s, z17.s\n" - "fmax z18.s, p1/M, z18.s, z17.s\n" - "st1w { z21.s }, p0, [x23, x10, LSL #2]\n" - "mov z21.d, z25.d\n" - "st1w { z20.s }, p0, [x22, x10, LSL #2]\n" - "mov z20.d, z25.d\n" + "fmax z25.s, p1/M, z25.s, z21.s\n" + "st1w { z25.s }, p0, [x26, x10, LSL #2]\n" + "mov z25.d, z16.d\n" + "fmax z15.s, p1/M, z15.s, z21.s\n" + "fmax z24.s, p1/M, z24.s, z21.s\n" + "st1w { z15.s }, p0, [x25, x10, LSL #2]\n" + "mov z15.d, z16.d\n" + "fmax z14.s, p1/M, z14.s, z21.s\n" + "fmax z26.s, p1/M, z26.s, z21.s\n" + "st1w { z24.s }, p0, [x24, x10, LSL #2]\n" + "mov z24.d, z16.d\n" + "fmax z17.s, p1/M, z17.s, z21.s\n" + "fmax z23.s, p1/M, z23.s, z21.s\n" + "st1w { z14.s }, p0, [x23, x10, LSL #2]\n" + "mov z14.d, z16.d\n" + "st1w { z26.s }, p0, [x22, x10, LSL #2]\n" + "mov z26.d, z16.d\n" "ld1w { z28.s }, p2/Z, [%x[params], #-8, MUL VL]\n" "ld1w { z27.s }, p2/Z, [%x[params], #-7, MUL VL]\n" - "st1w { z19.s }, p0, [x21, x10, LSL #2]\n" - "mov z19.d, z25.d\n" + "st1w { z17.s }, p0, [x21, x10, LSL #2]\n" + "mov z17.d, z16.d\n" "addvl %x[params], %x[params], #-6\n" - "st1w { z18.s }, p0, [x20, x10, LSL #2]\n" + "st1w { z23.s }, p0, [x20, x10, LSL #2]\n" "incw x10\n" - "mov z18.d, z25.d\n" + "mov z23.d, z16.d\n" "b.any 1b\n" : [params] "+&r" (params) : [channel_multiplier] "r" (n_output_channels), [clamps] "r" (minmax_vals), [inptrs] "r" (inptrs), [outptrs] "r" (outptrs) - : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp index e1f0b50d898c47f24d46a56d1ed212ce1f523707..f83767d8ae132acf5a2a7df42372f0fe0d53c90a 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,13 +22,13 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include #pragma once -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -49,4 +49,4 @@ struct sve_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp index a43b81d7e85c57fc43a7cafb7d0fc6920fb798cb..1770ec182cc8dd0e41cf8f026967145eb315d699 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst/generic.cpp @@ -46,405 +46,405 @@ void sve_fp32_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst_im __asm__ __volatile__( "ptrue p1.b\n" "mov x9, #0x0\n" - "ld1rw { z10.s }, p1/Z, [%x[minmax_vals]]\n" - "ld1rw { z13.s }, p1/Z, [%x[minmax_vals], #4]\n" + "ld1rw { z15.s }, p1/Z, [%x[minmax_vals]]\n" + "ld1rw { z14.s }, p1/Z, [%x[minmax_vals], #4]\n" "whilelt p0.s, x9, %x[n_output_channels]\n" "1:" // Output channel loop - "mov z5.b, #0x0\n" + "mov z31.b, #0x0\n" "cbz %x[bias], 2f\n" - "ld1w { z5.s }, p0/Z, [%x[bias], x9, LSL #2]\n" + "ld1w { z31.s }, p0/Z, [%x[bias], x9, LSL #2]\n" "2:" // Output channel loop: Load bias: Done - "mov x21, %x[inptrs]\n" - "ldp x24, x28, [x21], #0x10\n" - "lsr x20, %x[kernel_points], #0x1\n" - "mov z16.d, z5.d\n" - "mov z17.d, z5.d\n" - "mov z18.d, z5.d\n" - "ld1rqw { z1.s }, p1/Z, [x24]\n" - "ld1rqw { z6.s }, p1/Z, [x24, #16]\n" - "mov z19.d, z5.d\n" - "mov z20.d, z5.d\n" - "ld1rqw { z0.s }, p1/Z, [x28]\n" - "ld1rqw { z4.s }, p1/Z, [x28, #16]\n" - "mov z21.d, z5.d\n" - "mov z22.d, z5.d\n" + "mov x23, %x[inptrs]\n" + "ldp x21, x20, [x23], #0x10\n" + "lsr x22, %x[kernel_points], #0x1\n" + "mov z16.d, z31.d\n" + "mov z17.d, z31.d\n" + "mov z18.d, z31.d\n" + "ld1rqw { z6.s }, p1/Z, [x21]\n" + "ld1rqw { z5.s }, p1/Z, [x21, #16]\n" + "mov z19.d, z31.d\n" + "mov z20.d, z31.d\n" + "ld1rqw { z1.s }, p1/Z, [x20]\n" + "ld1rqw { z2.s }, p1/Z, [x20, #16]\n" + "mov z21.d, z31.d\n" + "mov z22.d, z31.d\n" "ld1w { z8.s }, p1/Z, [%x[weights]]\n" "addvl %x[weights], %x[weights], #1\n" - "mov z23.d, z5.d\n" - "mov z24.d, z5.d\n" - "mov z25.d, z5.d\n" - "mov z26.d, z5.d\n" - "mov z27.d, z5.d\n" - "mov z28.d, z5.d\n" - "mov z29.d, z5.d\n" - "mov z30.d, z5.d\n" - "mov z31.d, z5.d\n" - "cbz x20, 6f\n" - "ldp x24, x28, [x21], #0x10\n" - "subs x20, x20, #0x1\n" - "ld1rqw { z5.s }, p1/Z, [x24]\n" - "ld1rqw { z7.s }, p1/Z, [x24, #16]\n" - "ld1rqw { z3.s }, p1/Z, [x28]\n" - "ld1rqw { z2.s }, p1/Z, [x28, #16]\n" + "mov z23.d, z31.d\n" + "mov z24.d, z31.d\n" + "mov z25.d, z31.d\n" + "mov z26.d, z31.d\n" + "mov z27.d, z31.d\n" + "mov z28.d, z31.d\n" + "mov z29.d, z31.d\n" + "mov z30.d, z31.d\n" + "mov z31.d, z31.d\n" + "cbz x22, 6f\n" + "ldp x21, x20, [x23], #0x10\n" + "subs x22, x22, #0x1\n" + "ld1rqw { z0.s }, p1/Z, [x21]\n" + "ld1rqw { z4.s }, p1/Z, [x21, #16]\n" + "ld1rqw { z7.s }, p1/Z, [x20]\n" + "ld1rqw { z3.s }, p1/Z, [x20, #16]\n" "ld1w { z11.s }, p1/Z, [%x[weights]]\n" "addvl %x[weights], %x[weights], #1\n" "beq 4f\n" "3:" // Output channel loop: Kernel loop - "ldp x24, x28, [x21], #0x10\n" - "fmla z16.s, z8.s, z1.s[0]\n" - "fmla z17.s, z8.s, z1.s[1]\n" - "subs x20, x20, #0x1\n" - "fmla z18.s, z8.s, z1.s[2]\n" - "fmla z19.s, z8.s, z1.s[3]\n" - "ld1rqw { z1.s }, p1/Z, [x24]\n" - "fmla z20.s, z8.s, z6.s[0]\n" - "fmla z21.s, z8.s, z6.s[1]\n" - "fmla z22.s, z8.s, z6.s[2]\n" - "fmla z23.s, z8.s, z6.s[3]\n" - "ld1rqw { z6.s }, p1/Z, [x24, #16]\n" - "fmla z24.s, z8.s, z0.s[0]\n" - "fmla z25.s, z8.s, z0.s[1]\n" - "fmla z26.s, z8.s, z0.s[2]\n" - "fmla z27.s, z8.s, z0.s[3]\n" - "ld1rqw { z0.s }, p1/Z, [x28]\n" - "fmla z28.s, z8.s, z4.s[0]\n" - "fmla z29.s, z8.s, z4.s[1]\n" - "fmla z30.s, z8.s, z4.s[2]\n" - "fmla z31.s, z8.s, z4.s[3]\n" - "ld1rqw { z4.s }, p1/Z, [x28, #16]\n" - "ldp x24, x28, [x21], #0x10\n" + "ldp x21, x20, [x23], #0x10\n" + "fmla z16.s, z8.s, z6.s[0]\n" + "fmla z17.s, z8.s, z6.s[1]\n" + "subs x22, x22, #0x1\n" + "fmla z18.s, z8.s, z6.s[2]\n" + "fmla z19.s, z8.s, z6.s[3]\n" + "ld1rqw { z6.s }, p1/Z, [x21]\n" + "fmla z20.s, z8.s, z5.s[0]\n" + "fmla z21.s, z8.s, z5.s[1]\n" + "fmla z22.s, z8.s, z5.s[2]\n" + "fmla z23.s, z8.s, z5.s[3]\n" + "ld1rqw { z5.s }, p1/Z, [x21, #16]\n" + "fmla z24.s, z8.s, z1.s[0]\n" + "fmla z25.s, z8.s, z1.s[1]\n" + "fmla z26.s, z8.s, z1.s[2]\n" + "fmla z27.s, z8.s, z1.s[3]\n" + "ld1rqw { z1.s }, p1/Z, [x20]\n" + "fmla z28.s, z8.s, z2.s[0]\n" + "fmla z29.s, z8.s, z2.s[1]\n" + "fmla z30.s, z8.s, z2.s[2]\n" + "fmla z31.s, z8.s, z2.s[3]\n" + "ld1rqw { z2.s }, p1/Z, [x20, #16]\n" + "ldp x21, x20, [x23], #0x10\n" "ld1w { z8.s }, p1/Z, [%x[weights]]\n" - "fmla z16.s, z11.s, z5.s[0]\n" - "fmla z17.s, z11.s, z5.s[1]\n" - "fmla z18.s, z11.s, z5.s[2]\n" - "fmla z19.s, z11.s, z5.s[3]\n" - "ld1rqw { z5.s }, p1/Z, [x24]\n" - "fmla z20.s, z11.s, z7.s[0]\n" - "fmla z21.s, z11.s, z7.s[1]\n" - "fmla z22.s, z11.s, z7.s[2]\n" - "fmla z23.s, z11.s, z7.s[3]\n" - "ld1rqw { z7.s }, p1/Z, [x24, #16]\n" - "fmla z24.s, z11.s, z3.s[0]\n" - "fmla z25.s, z11.s, z3.s[1]\n" - "fmla z26.s, z11.s, z3.s[2]\n" - "fmla z27.s, z11.s, z3.s[3]\n" - "ld1rqw { z3.s }, p1/Z, [x28]\n" - "fmla z28.s, z11.s, z2.s[0]\n" - "fmla z29.s, z11.s, z2.s[1]\n" - "fmla z30.s, z11.s, z2.s[2]\n" - "fmla z31.s, z11.s, z2.s[3]\n" - "ld1rqw { z2.s }, p1/Z, [x28, #16]\n" + "fmla z16.s, z11.s, z0.s[0]\n" + "fmla z17.s, z11.s, z0.s[1]\n" + "fmla z18.s, z11.s, z0.s[2]\n" + "fmla z19.s, z11.s, z0.s[3]\n" + "ld1rqw { z0.s }, p1/Z, [x21]\n" + "fmla z20.s, z11.s, z4.s[0]\n" + "fmla z21.s, z11.s, z4.s[1]\n" + "fmla z22.s, z11.s, z4.s[2]\n" + "fmla z23.s, z11.s, z4.s[3]\n" + "ld1rqw { z4.s }, p1/Z, [x21, #16]\n" + "fmla z24.s, z11.s, z7.s[0]\n" + "fmla z25.s, z11.s, z7.s[1]\n" + "fmla z26.s, z11.s, z7.s[2]\n" + "fmla z27.s, z11.s, z7.s[3]\n" + "ld1rqw { z7.s }, p1/Z, [x20]\n" + "fmla z28.s, z11.s, z3.s[0]\n" + "fmla z29.s, z11.s, z3.s[1]\n" + "fmla z30.s, z11.s, z3.s[2]\n" + "fmla z31.s, z11.s, z3.s[3]\n" + "ld1rqw { z3.s }, p1/Z, [x20, #16]\n" "ld1w { z11.s }, p1/Z, [%x[weights], #1, MUL VL]\n" "addvl %x[weights], %x[weights], #2\n" "bgt 3b\n" "4:" // Output channel loop: Kernel loop tail "tbnz %x[kernel_points], #0, 5f\n" - "fmla z16.s, z8.s, z1.s[0]\n" - "fmla z17.s, z8.s, z1.s[1]\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "fmla z18.s, z8.s, z1.s[2]\n" - "fmla z19.s, z8.s, z1.s[3]\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "fmla z20.s, z8.s, z6.s[0]\n" - "fmla z21.s, z8.s, z6.s[1]\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "fmla z22.s, z8.s, z6.s[2]\n" - "fmla z23.s, z8.s, z6.s[3]\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "fmla z24.s, z8.s, z0.s[0]\n" - "fmla z25.s, z8.s, z0.s[1]\n" - "fmla z26.s, z8.s, z0.s[2]\n" - "fmla z27.s, z8.s, z0.s[3]\n" - "fmla z28.s, z8.s, z4.s[0]\n" - "fmla z29.s, z8.s, z4.s[1]\n" - "fmla z30.s, z8.s, z4.s[2]\n" - "fmla z31.s, z8.s, z4.s[3]\n" - "fmla z16.s, z11.s, z5.s[0]\n" - "fmla z17.s, z11.s, z5.s[1]\n" - "fmin z16.s, p1/M, z16.s, z13.s\n" - "fmin z17.s, p1/M, z17.s, z13.s\n" - "fmla z18.s, z11.s, z5.s[2]\n" - "fmla z19.s, z11.s, z5.s[3]\n" - "fmin z18.s, p1/M, z18.s, z13.s\n" - "fmin z19.s, p1/M, z19.s, z13.s\n" - "fmla z20.s, z11.s, z7.s[0]\n" - "fmla z21.s, z11.s, z7.s[1]\n" - "fmin z20.s, p1/M, z20.s, z13.s\n" - "fmin z21.s, p1/M, z21.s, z13.s\n" - "fmla z22.s, z11.s, z7.s[2]\n" - "fmla z23.s, z11.s, z7.s[3]\n" - "fmin z22.s, p1/M, z22.s, z13.s\n" - "fmin z23.s, p1/M, z23.s, z13.s\n" - "fmla z24.s, z11.s, z3.s[0]\n" - "fmla z25.s, z11.s, z3.s[1]\n" - "fmax z16.s, p1/M, z16.s, z10.s\n" - "fmax z17.s, p1/M, z17.s, z10.s\n" - "fmla z26.s, z11.s, z3.s[2]\n" - "fmla z27.s, z11.s, z3.s[3]\n" - "fmax z18.s, p1/M, z18.s, z10.s\n" - "fmax z19.s, p1/M, z19.s, z10.s\n" - "fmla z28.s, z11.s, z2.s[0]\n" - "fmla z29.s, z11.s, z2.s[1]\n" - "fmax z20.s, p1/M, z20.s, z10.s\n" - "fmax z21.s, p1/M, z21.s, z10.s\n" - "fmla z30.s, z11.s, z2.s[2]\n" - "fmla z31.s, z11.s, z2.s[3]\n" - "fmax z22.s, p1/M, z22.s, z10.s\n" - "fmax z23.s, p1/M, z23.s, z10.s\n" - "fmin z24.s, p1/M, z24.s, z13.s\n" - "fmin z25.s, p1/M, z25.s, z13.s\n" - "st1w { z16.s }, p0, [x20, x9, LSL #2]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "fmin z26.s, p1/M, z26.s, z13.s\n" - "fmin z27.s, p1/M, z27.s, z13.s\n" - "st1w { z17.s }, p0, [x21, x9, LSL #2]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "fmin z28.s, p1/M, z28.s, z13.s\n" - "fmin z29.s, p1/M, z29.s, z13.s\n" - "st1w { z18.s }, p0, [x22, x9, LSL #2]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "fmin z30.s, p1/M, z30.s, z13.s\n" - "fmin z31.s, p1/M, z31.s, z13.s\n" - "st1w { z19.s }, p0, [x23, x9, LSL #2]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "st1w { z20.s }, p0, [x24, x9, LSL #2]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "fmax z24.s, p1/M, z24.s, z10.s\n" - "fmax z25.s, p1/M, z25.s, z10.s\n" - "st1w { z21.s }, p0, [x25, x9, LSL #2]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "fmax z26.s, p1/M, z26.s, z10.s\n" - "fmax z27.s, p1/M, z27.s, z10.s\n" - "st1w { z22.s }, p0, [x26, x9, LSL #2]\n" - "ldr x26, [%x[outptrs], #0x70]\n" - "fmax z28.s, p1/M, z28.s, z10.s\n" - "fmax z29.s, p1/M, z29.s, z10.s\n" - "st1w { z23.s }, p0, [x27, x9, LSL #2]\n" - "ldr x27, [%x[outptrs], #0x78]\n" - "fmax z30.s, p1/M, z30.s, z10.s\n" - "fmax z31.s, p1/M, z31.s, z10.s\n" - "st1w { z24.s }, p0, [x20, x9, LSL #2]\n" - "st1w { z25.s }, p0, [x21, x9, LSL #2]\n" - "st1w { z26.s }, p0, [x22, x9, LSL #2]\n" - "st1w { z27.s }, p0, [x23, x9, LSL #2]\n" - "st1w { z28.s }, p0, [x24, x9, LSL #2]\n" - "st1w { z29.s }, p0, [x25, x9, LSL #2]\n" - "st1w { z30.s }, p0, [x26, x9, LSL #2]\n" - "st1w { z31.s }, p0, [x27, x9, LSL #2]\n" + "fmla z16.s, z8.s, z6.s[0]\n" + "fmla z17.s, z8.s, z6.s[1]\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" + "fmla z18.s, z8.s, z6.s[2]\n" + "fmla z19.s, z8.s, z6.s[3]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "fmla z20.s, z8.s, z5.s[0]\n" + "fmla z21.s, z8.s, z5.s[1]\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "fmla z22.s, z8.s, z5.s[2]\n" + "fmla z23.s, z8.s, z5.s[3]\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "fmla z24.s, z8.s, z1.s[0]\n" + "fmla z25.s, z8.s, z1.s[1]\n" + "fmla z26.s, z8.s, z1.s[2]\n" + "fmla z27.s, z8.s, z1.s[3]\n" + "fmla z28.s, z8.s, z2.s[0]\n" + "fmla z29.s, z8.s, z2.s[1]\n" + "fmla z30.s, z8.s, z2.s[2]\n" + "fmla z31.s, z8.s, z2.s[3]\n" + "fmla z16.s, z11.s, z0.s[0]\n" + "fmla z17.s, z11.s, z0.s[1]\n" + "fmin z16.s, p1/M, z16.s, z14.s\n" + "fmin z17.s, p1/M, z17.s, z14.s\n" + "fmla z18.s, z11.s, z0.s[2]\n" + "fmla z19.s, z11.s, z0.s[3]\n" + "fmin z18.s, p1/M, z18.s, z14.s\n" + "fmin z19.s, p1/M, z19.s, z14.s\n" + "fmla z20.s, z11.s, z4.s[0]\n" + "fmla z21.s, z11.s, z4.s[1]\n" + "fmin z20.s, p1/M, z20.s, z14.s\n" + "fmin z21.s, p1/M, z21.s, z14.s\n" + "fmla z22.s, z11.s, z4.s[2]\n" + "fmla z23.s, z11.s, z4.s[3]\n" + "fmin z22.s, p1/M, z22.s, z14.s\n" + "fmin z23.s, p1/M, z23.s, z14.s\n" + "fmla z24.s, z11.s, z7.s[0]\n" + "fmla z25.s, z11.s, z7.s[1]\n" + "fmax z16.s, p1/M, z16.s, z15.s\n" + "fmax z17.s, p1/M, z17.s, z15.s\n" + "fmla z26.s, z11.s, z7.s[2]\n" + "fmla z27.s, z11.s, z7.s[3]\n" + "fmax z18.s, p1/M, z18.s, z15.s\n" + "fmax z19.s, p1/M, z19.s, z15.s\n" + "fmla z28.s, z11.s, z3.s[0]\n" + "fmla z29.s, z11.s, z3.s[1]\n" + "fmax z20.s, p1/M, z20.s, z15.s\n" + "fmax z21.s, p1/M, z21.s, z15.s\n" + "fmla z30.s, z11.s, z3.s[2]\n" + "fmla z31.s, z11.s, z3.s[3]\n" + "fmax z22.s, p1/M, z22.s, z15.s\n" + "fmax z23.s, p1/M, z23.s, z15.s\n" + "fmin z24.s, p1/M, z24.s, z14.s\n" + "fmin z25.s, p1/M, z25.s, z14.s\n" + "st1w { z16.s }, p0, [x27, x9, LSL #2]\n" + "ldr x27, [%x[outptrs], #0x40]\n" + "fmin z26.s, p1/M, z26.s, z14.s\n" + "fmin z27.s, p1/M, z27.s, z14.s\n" + "st1w { z17.s }, p0, [x26, x9, LSL #2]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "fmin z28.s, p1/M, z28.s, z14.s\n" + "fmin z29.s, p1/M, z29.s, z14.s\n" + "st1w { z18.s }, p0, [x25, x9, LSL #2]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "fmin z30.s, p1/M, z30.s, z14.s\n" + "fmin z31.s, p1/M, z31.s, z14.s\n" + "st1w { z19.s }, p0, [x24, x9, LSL #2]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "st1w { z20.s }, p0, [x23, x9, LSL #2]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "fmax z24.s, p1/M, z24.s, z15.s\n" + "fmax z25.s, p1/M, z25.s, z15.s\n" + "st1w { z21.s }, p0, [x22, x9, LSL #2]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "fmax z26.s, p1/M, z26.s, z15.s\n" + "fmax z27.s, p1/M, z27.s, z15.s\n" + "st1w { z22.s }, p0, [x21, x9, LSL #2]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "fmax z28.s, p1/M, z28.s, z15.s\n" + "fmax z29.s, p1/M, z29.s, z15.s\n" + "st1w { z23.s }, p0, [x20, x9, LSL #2]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "fmax z30.s, p1/M, z30.s, z15.s\n" + "fmax z31.s, p1/M, z31.s, z15.s\n" + "st1w { z24.s }, p0, [x27, x9, LSL #2]\n" + "st1w { z25.s }, p0, [x26, x9, LSL #2]\n" + "st1w { z26.s }, p0, [x25, x9, LSL #2]\n" + "st1w { z27.s }, p0, [x24, x9, LSL #2]\n" + "st1w { z28.s }, p0, [x23, x9, LSL #2]\n" + "st1w { z29.s }, p0, [x22, x9, LSL #2]\n" + "st1w { z30.s }, p0, [x21, x9, LSL #2]\n" + "st1w { z31.s }, p0, [x20, x9, LSL #2]\n" "b 7f\n" "5:" // Output channel loop: Odd tail - "fmla z16.s, z8.s, z1.s[0]\n" - "fmla z17.s, z8.s, z1.s[1]\n" - "ldp x24, x28, [x21], #0x10\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "fmla z18.s, z8.s, z1.s[2]\n" - "fmla z19.s, z8.s, z1.s[3]\n" - "ld1rqw { z1.s }, p1/Z, [x24]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "fmla z20.s, z8.s, z6.s[0]\n" - "fmla z21.s, z8.s, z6.s[1]\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "fmla z22.s, z8.s, z6.s[2]\n" - "fmla z23.s, z8.s, z6.s[3]\n" - "ld1rqw { z6.s }, p1/Z, [x24, #16]\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "fmla z24.s, z8.s, z0.s[0]\n" - "fmla z25.s, z8.s, z0.s[1]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "fmla z26.s, z8.s, z0.s[2]\n" - "fmla z27.s, z8.s, z0.s[3]\n" - "ld1rqw { z0.s }, p1/Z, [x28]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "fmla z28.s, z8.s, z4.s[0]\n" - "fmla z29.s, z8.s, z4.s[1]\n" - "fmla z30.s, z8.s, z4.s[2]\n" - "fmla z31.s, z8.s, z4.s[3]\n" - "ld1w { z8.s }, p1/Z, [%x[weights]]\n" - "ld1rqw { z4.s }, p1/Z, [x28, #16]\n" - "fmla z16.s, z11.s, z5.s[0]\n" - "fmla z17.s, z11.s, z5.s[1]\n" + "fmla z16.s, z8.s, z6.s[0]\n" + "fmla z17.s, z8.s, z6.s[1]\n" + "ldp x20, x28, [x23], #0x10\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "fmla z18.s, z8.s, z6.s[2]\n" + "fmla z19.s, z8.s, z6.s[3]\n" + "ld1rqw { z6.s }, p1/Z, [x20]\n" + "ldr x26, [%x[outptrs], #0x8]\n" + "fmla z20.s, z8.s, z5.s[0]\n" + "fmla z21.s, z8.s, z5.s[1]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "fmla z22.s, z8.s, z5.s[2]\n" + "fmla z23.s, z8.s, z5.s[3]\n" + "ld1rqw { z5.s }, p1/Z, [x20, #16]\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "fmla z24.s, z8.s, z1.s[0]\n" + "fmla z25.s, z8.s, z1.s[1]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "fmla z26.s, z8.s, z1.s[2]\n" + "fmla z27.s, z8.s, z1.s[3]\n" + "ld1rqw { z1.s }, p1/Z, [x28]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "fmla z28.s, z8.s, z2.s[0]\n" + "fmla z29.s, z8.s, z2.s[1]\n" + "fmla z30.s, z8.s, z2.s[2]\n" + "fmla z31.s, z8.s, z2.s[3]\n" + "ld1w { z10.s }, p1/Z, [%x[weights]]\n" + "ld1rqw { z2.s }, p1/Z, [x28, #16]\n" + "fmla z16.s, z11.s, z0.s[0]\n" + "fmla z17.s, z11.s, z0.s[1]\n" "addvl %x[weights], %x[weights], #1\n" - "fmla z18.s, z11.s, z5.s[2]\n" - "fmla z19.s, z11.s, z5.s[3]\n" - "fmla z20.s, z11.s, z7.s[0]\n" - "fmla z21.s, z11.s, z7.s[1]\n" - "fmla z22.s, z11.s, z7.s[2]\n" - "fmla z23.s, z11.s, z7.s[3]\n" - "fmla z24.s, z11.s, z3.s[0]\n" - "fmla z25.s, z11.s, z3.s[1]\n" - "fmla z26.s, z11.s, z3.s[2]\n" - "fmla z27.s, z11.s, z3.s[3]\n" - "fmla z28.s, z11.s, z2.s[0]\n" - "fmla z29.s, z11.s, z2.s[1]\n" - "fmla z30.s, z11.s, z2.s[2]\n" - "fmla z31.s, z11.s, z2.s[3]\n" - "fmla z16.s, z8.s, z1.s[0]\n" - "fmla z17.s, z8.s, z1.s[1]\n" - "fmin z16.s, p1/M, z16.s, z13.s\n" - "fmin z17.s, p1/M, z17.s, z13.s\n" - "fmla z18.s, z8.s, z1.s[2]\n" - "fmla z19.s, z8.s, z1.s[3]\n" - "fmin z18.s, p1/M, z18.s, z13.s\n" - "fmin z19.s, p1/M, z19.s, z13.s\n" - "fmla z20.s, z8.s, z6.s[0]\n" - "fmla z21.s, z8.s, z6.s[1]\n" - "fmin z20.s, p1/M, z20.s, z13.s\n" - "fmin z21.s, p1/M, z21.s, z13.s\n" - "fmla z22.s, z8.s, z6.s[2]\n" - "fmla z23.s, z8.s, z6.s[3]\n" - "fmin z22.s, p1/M, z22.s, z13.s\n" - "fmin z23.s, p1/M, z23.s, z13.s\n" - "fmla z24.s, z8.s, z0.s[0]\n" - "fmla z25.s, z8.s, z0.s[1]\n" - "fmax z16.s, p1/M, z16.s, z10.s\n" - "fmax z17.s, p1/M, z17.s, z10.s\n" - "fmla z26.s, z8.s, z0.s[2]\n" - "fmla z27.s, z8.s, z0.s[3]\n" - "fmax z18.s, p1/M, z18.s, z10.s\n" - "fmax z19.s, p1/M, z19.s, z10.s\n" - "fmla z28.s, z8.s, z4.s[0]\n" - "fmla z29.s, z8.s, z4.s[1]\n" - "fmax z20.s, p1/M, z20.s, z10.s\n" - "fmax z21.s, p1/M, z21.s, z10.s\n" - "fmla z30.s, z8.s, z4.s[2]\n" - "fmla z31.s, z8.s, z4.s[3]\n" - "fmax z22.s, p1/M, z22.s, z10.s\n" - "fmax z23.s, p1/M, z23.s, z10.s\n" - "fmin z24.s, p1/M, z24.s, z13.s\n" - "fmin z25.s, p1/M, z25.s, z13.s\n" - "st1w { z16.s }, p0, [x20, x9, LSL #2]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "fmin z26.s, p1/M, z26.s, z13.s\n" - "fmin z27.s, p1/M, z27.s, z13.s\n" - "st1w { z17.s }, p0, [x21, x9, LSL #2]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "fmin z28.s, p1/M, z28.s, z13.s\n" - "fmin z29.s, p1/M, z29.s, z13.s\n" - "st1w { z18.s }, p0, [x22, x9, LSL #2]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "fmin z30.s, p1/M, z30.s, z13.s\n" - "fmin z31.s, p1/M, z31.s, z13.s\n" - "st1w { z19.s }, p0, [x23, x9, LSL #2]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "st1w { z20.s }, p0, [x24, x9, LSL #2]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "fmax z24.s, p1/M, z24.s, z10.s\n" - "fmax z25.s, p1/M, z25.s, z10.s\n" - "st1w { z21.s }, p0, [x25, x9, LSL #2]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "fmax z26.s, p1/M, z26.s, z10.s\n" - "fmax z27.s, p1/M, z27.s, z10.s\n" - "st1w { z22.s }, p0, [x26, x9, LSL #2]\n" - "ldr x26, [%x[outptrs], #0x70]\n" - "fmax z28.s, p1/M, z28.s, z10.s\n" - "fmax z29.s, p1/M, z29.s, z10.s\n" - "st1w { z23.s }, p0, [x27, x9, LSL #2]\n" - "ldr x27, [%x[outptrs], #0x78]\n" - "fmax z30.s, p1/M, z30.s, z10.s\n" - "fmax z31.s, p1/M, z31.s, z10.s\n" - "st1w { z24.s }, p0, [x20, x9, LSL #2]\n" - "st1w { z25.s }, p0, [x21, x9, LSL #2]\n" - "st1w { z26.s }, p0, [x22, x9, LSL #2]\n" - "st1w { z27.s }, p0, [x23, x9, LSL #2]\n" - "st1w { z28.s }, p0, [x24, x9, LSL #2]\n" - "st1w { z29.s }, p0, [x25, x9, LSL #2]\n" - "st1w { z30.s }, p0, [x26, x9, LSL #2]\n" - "st1w { z31.s }, p0, [x27, x9, LSL #2]\n" + "fmla z18.s, z11.s, z0.s[2]\n" + "fmla z19.s, z11.s, z0.s[3]\n" + "fmla z20.s, z11.s, z4.s[0]\n" + "fmla z21.s, z11.s, z4.s[1]\n" + "fmla z22.s, z11.s, z4.s[2]\n" + "fmla z23.s, z11.s, z4.s[3]\n" + "fmla z24.s, z11.s, z7.s[0]\n" + "fmla z25.s, z11.s, z7.s[1]\n" + "fmla z26.s, z11.s, z7.s[2]\n" + "fmla z27.s, z11.s, z7.s[3]\n" + "fmla z28.s, z11.s, z3.s[0]\n" + "fmla z29.s, z11.s, z3.s[1]\n" + "fmla z30.s, z11.s, z3.s[2]\n" + "fmla z31.s, z11.s, z3.s[3]\n" + "fmla z16.s, z10.s, z6.s[0]\n" + "fmla z17.s, z10.s, z6.s[1]\n" + "fmin z16.s, p1/M, z16.s, z14.s\n" + "fmin z17.s, p1/M, z17.s, z14.s\n" + "fmla z18.s, z10.s, z6.s[2]\n" + "fmla z19.s, z10.s, z6.s[3]\n" + "fmin z18.s, p1/M, z18.s, z14.s\n" + "fmin z19.s, p1/M, z19.s, z14.s\n" + "fmla z20.s, z10.s, z5.s[0]\n" + "fmla z21.s, z10.s, z5.s[1]\n" + "fmin z20.s, p1/M, z20.s, z14.s\n" + "fmin z21.s, p1/M, z21.s, z14.s\n" + "fmla z22.s, z10.s, z5.s[2]\n" + "fmla z23.s, z10.s, z5.s[3]\n" + "fmin z22.s, p1/M, z22.s, z14.s\n" + "fmin z23.s, p1/M, z23.s, z14.s\n" + "fmla z24.s, z10.s, z1.s[0]\n" + "fmla z25.s, z10.s, z1.s[1]\n" + "fmax z16.s, p1/M, z16.s, z15.s\n" + "fmax z17.s, p1/M, z17.s, z15.s\n" + "fmla z26.s, z10.s, z1.s[2]\n" + "fmla z27.s, z10.s, z1.s[3]\n" + "fmax z18.s, p1/M, z18.s, z15.s\n" + "fmax z19.s, p1/M, z19.s, z15.s\n" + "fmla z28.s, z10.s, z2.s[0]\n" + "fmla z29.s, z10.s, z2.s[1]\n" + "fmax z20.s, p1/M, z20.s, z15.s\n" + "fmax z21.s, p1/M, z21.s, z15.s\n" + "fmla z30.s, z10.s, z2.s[2]\n" + "fmla z31.s, z10.s, z2.s[3]\n" + "fmax z22.s, p1/M, z22.s, z15.s\n" + "fmax z23.s, p1/M, z23.s, z15.s\n" + "fmin z24.s, p1/M, z24.s, z14.s\n" + "fmin z25.s, p1/M, z25.s, z14.s\n" + "st1w { z16.s }, p0, [x27, x9, LSL #2]\n" + "ldr x27, [%x[outptrs], #0x40]\n" + "fmin z26.s, p1/M, z26.s, z14.s\n" + "fmin z27.s, p1/M, z27.s, z14.s\n" + "st1w { z17.s }, p0, [x26, x9, LSL #2]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "fmin z28.s, p1/M, z28.s, z14.s\n" + "fmin z29.s, p1/M, z29.s, z14.s\n" + "st1w { z18.s }, p0, [x25, x9, LSL #2]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "fmin z30.s, p1/M, z30.s, z14.s\n" + "fmin z31.s, p1/M, z31.s, z14.s\n" + "st1w { z19.s }, p0, [x24, x9, LSL #2]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "st1w { z20.s }, p0, [x23, x9, LSL #2]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "fmax z24.s, p1/M, z24.s, z15.s\n" + "fmax z25.s, p1/M, z25.s, z15.s\n" + "st1w { z21.s }, p0, [x22, x9, LSL #2]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "fmax z26.s, p1/M, z26.s, z15.s\n" + "fmax z27.s, p1/M, z27.s, z15.s\n" + "st1w { z22.s }, p0, [x21, x9, LSL #2]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "fmax z28.s, p1/M, z28.s, z15.s\n" + "fmax z29.s, p1/M, z29.s, z15.s\n" + "st1w { z23.s }, p0, [x20, x9, LSL #2]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "fmax z30.s, p1/M, z30.s, z15.s\n" + "fmax z31.s, p1/M, z31.s, z15.s\n" + "st1w { z24.s }, p0, [x27, x9, LSL #2]\n" + "st1w { z25.s }, p0, [x26, x9, LSL #2]\n" + "st1w { z26.s }, p0, [x25, x9, LSL #2]\n" + "st1w { z27.s }, p0, [x24, x9, LSL #2]\n" + "st1w { z28.s }, p0, [x23, x9, LSL #2]\n" + "st1w { z29.s }, p0, [x22, x9, LSL #2]\n" + "st1w { z30.s }, p0, [x21, x9, LSL #2]\n" + "st1w { z31.s }, p0, [x20, x9, LSL #2]\n" "b 7f\n" "6:" // Output channel loop: Single kernel point - "fmla z16.s, z8.s, z1.s[0]\n" - "fmla z17.s, z8.s, z1.s[1]\n" - "fmin z16.s, p1/M, z16.s, z13.s\n" - "fmin z17.s, p1/M, z17.s, z13.s\n" - "fmla z18.s, z8.s, z1.s[2]\n" - "fmla z19.s, z8.s, z1.s[3]\n" - "fmin z18.s, p1/M, z18.s, z13.s\n" - "fmin z19.s, p1/M, z19.s, z13.s\n" - "fmla z20.s, z8.s, z6.s[0]\n" - "fmla z21.s, z8.s, z6.s[1]\n" - "fmin z20.s, p1/M, z20.s, z13.s\n" - "fmin z21.s, p1/M, z21.s, z13.s\n" - "fmla z22.s, z8.s, z6.s[2]\n" - "fmla z23.s, z8.s, z6.s[3]\n" - "fmin z22.s, p1/M, z22.s, z13.s\n" - "fmin z23.s, p1/M, z23.s, z13.s\n" - "fmla z24.s, z8.s, z0.s[0]\n" - "fmla z25.s, z8.s, z0.s[1]\n" - "ldr x20, [%x[outptrs], #0x0]\n" - "ldr x21, [%x[outptrs], #0x8]\n" - "fmla z26.s, z8.s, z0.s[2]\n" - "fmla z27.s, z8.s, z0.s[3]\n" - "ldr x22, [%x[outptrs], #0x10]\n" - "ldr x23, [%x[outptrs], #0x18]\n" - "fmla z28.s, z8.s, z4.s[0]\n" - "fmla z29.s, z8.s, z4.s[1]\n" - "ldr x24, [%x[outptrs], #0x20]\n" - "ldr x25, [%x[outptrs], #0x28]\n" - "fmla z30.s, z8.s, z4.s[2]\n" - "fmla z31.s, z8.s, z4.s[3]\n" - "ldr x26, [%x[outptrs], #0x30]\n" - "ldr x27, [%x[outptrs], #0x38]\n" - "fmax z16.s, p1/M, z16.s, z10.s\n" - "fmax z17.s, p1/M, z17.s, z10.s\n" - "st1w { z16.s }, p0, [x20, x9, LSL #2]\n" - "ldr x20, [%x[outptrs], #0x40]\n" - "fmax z18.s, p1/M, z18.s, z10.s\n" - "fmax z19.s, p1/M, z19.s, z10.s\n" - "st1w { z17.s }, p0, [x21, x9, LSL #2]\n" - "ldr x21, [%x[outptrs], #0x48]\n" - "fmax z20.s, p1/M, z20.s, z10.s\n" - "fmax z21.s, p1/M, z21.s, z10.s\n" - "st1w { z18.s }, p0, [x22, x9, LSL #2]\n" - "ldr x22, [%x[outptrs], #0x50]\n" - "fmax z22.s, p1/M, z22.s, z10.s\n" - "fmax z23.s, p1/M, z23.s, z10.s\n" - "st1w { z19.s }, p0, [x23, x9, LSL #2]\n" - "ldr x23, [%x[outptrs], #0x58]\n" - "fmin z24.s, p1/M, z24.s, z13.s\n" - "fmin z25.s, p1/M, z25.s, z13.s\n" - "st1w { z20.s }, p0, [x24, x9, LSL #2]\n" - "ldr x24, [%x[outptrs], #0x60]\n" - "fmin z26.s, p1/M, z26.s, z13.s\n" - "fmin z27.s, p1/M, z27.s, z13.s\n" - "st1w { z21.s }, p0, [x25, x9, LSL #2]\n" - "ldr x25, [%x[outptrs], #0x68]\n" - "fmin z28.s, p1/M, z28.s, z13.s\n" - "fmin z29.s, p1/M, z29.s, z13.s\n" - "st1w { z22.s }, p0, [x26, x9, LSL #2]\n" - "ldr x26, [%x[outptrs], #0x70]\n" - "fmin z30.s, p1/M, z30.s, z13.s\n" - "fmin z31.s, p1/M, z31.s, z13.s\n" - "st1w { z23.s }, p0, [x27, x9, LSL #2]\n" - "ldr x27, [%x[outptrs], #0x78]\n" - "fmax z24.s, p1/M, z24.s, z10.s\n" - "fmax z25.s, p1/M, z25.s, z10.s\n" - "st1w { z24.s }, p0, [x20, x9, LSL #2]\n" - "fmax z26.s, p1/M, z26.s, z10.s\n" - "fmax z27.s, p1/M, z27.s, z10.s\n" - "st1w { z25.s }, p0, [x21, x9, LSL #2]\n" - "fmax z28.s, p1/M, z28.s, z10.s\n" - "fmax z29.s, p1/M, z29.s, z10.s\n" - "st1w { z26.s }, p0, [x22, x9, LSL #2]\n" - "fmax z30.s, p1/M, z30.s, z10.s\n" - "fmax z31.s, p1/M, z31.s, z10.s\n" - "st1w { z27.s }, p0, [x23, x9, LSL #2]\n" - "st1w { z28.s }, p0, [x24, x9, LSL #2]\n" - "st1w { z29.s }, p0, [x25, x9, LSL #2]\n" - "st1w { z30.s }, p0, [x26, x9, LSL #2]\n" - "st1w { z31.s }, p0, [x27, x9, LSL #2]\n" + "fmla z16.s, z8.s, z6.s[0]\n" + "fmla z17.s, z8.s, z6.s[1]\n" + "fmin z16.s, p1/M, z16.s, z14.s\n" + "fmin z17.s, p1/M, z17.s, z14.s\n" + "fmla z18.s, z8.s, z6.s[2]\n" + "fmla z19.s, z8.s, z6.s[3]\n" + "fmin z18.s, p1/M, z18.s, z14.s\n" + "fmin z19.s, p1/M, z19.s, z14.s\n" + "fmla z20.s, z8.s, z5.s[0]\n" + "fmla z21.s, z8.s, z5.s[1]\n" + "fmin z20.s, p1/M, z20.s, z14.s\n" + "fmin z21.s, p1/M, z21.s, z14.s\n" + "fmla z22.s, z8.s, z5.s[2]\n" + "fmla z23.s, z8.s, z5.s[3]\n" + "fmin z22.s, p1/M, z22.s, z14.s\n" + "fmin z23.s, p1/M, z23.s, z14.s\n" + "fmla z24.s, z8.s, z1.s[0]\n" + "fmla z25.s, z8.s, z1.s[1]\n" + "ldr x27, [%x[outptrs], #0x0]\n" + "ldr x26, [%x[outptrs], #0x8]\n" + "fmla z26.s, z8.s, z1.s[2]\n" + "fmla z27.s, z8.s, z1.s[3]\n" + "ldr x25, [%x[outptrs], #0x10]\n" + "ldr x24, [%x[outptrs], #0x18]\n" + "fmla z28.s, z8.s, z2.s[0]\n" + "fmla z29.s, z8.s, z2.s[1]\n" + "ldr x23, [%x[outptrs], #0x20]\n" + "ldr x22, [%x[outptrs], #0x28]\n" + "fmla z30.s, z8.s, z2.s[2]\n" + "fmla z31.s, z8.s, z2.s[3]\n" + "ldr x21, [%x[outptrs], #0x30]\n" + "ldr x20, [%x[outptrs], #0x38]\n" + "fmax z16.s, p1/M, z16.s, z15.s\n" + "fmax z17.s, p1/M, z17.s, z15.s\n" + "st1w { z16.s }, p0, [x27, x9, LSL #2]\n" + "ldr x27, [%x[outptrs], #0x40]\n" + "fmax z18.s, p1/M, z18.s, z15.s\n" + "fmax z19.s, p1/M, z19.s, z15.s\n" + "st1w { z17.s }, p0, [x26, x9, LSL #2]\n" + "ldr x26, [%x[outptrs], #0x48]\n" + "fmax z20.s, p1/M, z20.s, z15.s\n" + "fmax z21.s, p1/M, z21.s, z15.s\n" + "st1w { z18.s }, p0, [x25, x9, LSL #2]\n" + "ldr x25, [%x[outptrs], #0x50]\n" + "fmax z22.s, p1/M, z22.s, z15.s\n" + "fmax z23.s, p1/M, z23.s, z15.s\n" + "st1w { z19.s }, p0, [x24, x9, LSL #2]\n" + "ldr x24, [%x[outptrs], #0x58]\n" + "fmin z24.s, p1/M, z24.s, z14.s\n" + "fmin z25.s, p1/M, z25.s, z14.s\n" + "st1w { z20.s }, p0, [x23, x9, LSL #2]\n" + "ldr x23, [%x[outptrs], #0x60]\n" + "fmin z26.s, p1/M, z26.s, z14.s\n" + "fmin z27.s, p1/M, z27.s, z14.s\n" + "st1w { z21.s }, p0, [x22, x9, LSL #2]\n" + "ldr x22, [%x[outptrs], #0x68]\n" + "fmin z28.s, p1/M, z28.s, z14.s\n" + "fmin z29.s, p1/M, z29.s, z14.s\n" + "st1w { z22.s }, p0, [x21, x9, LSL #2]\n" + "ldr x21, [%x[outptrs], #0x70]\n" + "fmin z30.s, p1/M, z30.s, z14.s\n" + "fmin z31.s, p1/M, z31.s, z14.s\n" + "st1w { z23.s }, p0, [x20, x9, LSL #2]\n" + "ldr x20, [%x[outptrs], #0x78]\n" + "fmax z24.s, p1/M, z24.s, z15.s\n" + "fmax z25.s, p1/M, z25.s, z15.s\n" + "st1w { z24.s }, p0, [x27, x9, LSL #2]\n" + "fmax z26.s, p1/M, z26.s, z15.s\n" + "fmax z27.s, p1/M, z27.s, z15.s\n" + "st1w { z25.s }, p0, [x26, x9, LSL #2]\n" + "fmax z28.s, p1/M, z28.s, z15.s\n" + "fmax z29.s, p1/M, z29.s, z15.s\n" + "st1w { z26.s }, p0, [x25, x9, LSL #2]\n" + "fmax z30.s, p1/M, z30.s, z15.s\n" + "fmax z31.s, p1/M, z31.s, z15.s\n" + "st1w { z27.s }, p0, [x24, x9, LSL #2]\n" + "st1w { z28.s }, p0, [x23, x9, LSL #2]\n" + "st1w { z29.s }, p0, [x22, x9, LSL #2]\n" + "st1w { z30.s }, p0, [x21, x9, LSL #2]\n" + "st1w { z31.s }, p0, [x20, x9, LSL #2]\n" "7:" // Output channel loop: Done "incw x9\n" "whilelt p0.s, x9, %x[n_output_channels]\n" "b.any 1b\n" : [weights] "+&r" (weights) : [bias] "r" (bias), [inptrs] "r" (inptrs), [kernel_points] "r" ((uint64_t) kernel_points), [minmax_vals] "r" (minmax_vals), [n_output_channels] "r" ((uint64_t) n_output_channels), [outptrs] "r" (outptrs) - : "cc", "memory", "p0", "p1", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z10", "z11", "z13", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z10", "z11", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp index 4e2ee43374ada735e7b04f31b82d0a1c14ad942c..04cf0d4036c777b1dc33e691c2af08771caa3c19 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include @@ -64,7 +64,7 @@ class sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst : public DepthwiseDepthfirstS ) const override { interleave_sve_s8q_3x3_dot::pack_parameters( - args.input_channels, buffer, reinterpret_cast(biases), + args.input_channels * args.channel_multiplier, buffer, reinterpret_cast(biases), reinterpret_cast(weights), qp, ld_weight_col, ld_weight_row ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp index 4eae5961a007fad5635dfe9ed97fe7e2d2f83bf5..0cee302c56f64a2f1db927155a4ca509039dcf42 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp @@ -30,472 +30,464 @@ namespace arm_conv { namespace depthwise { -void sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst_impl( - const unsigned int n_channels, - const int8_t *const *const inptrs, - const int8_t *params, - const int32_t *, // Bias, should be wrapped into the parameters - const arm_gemm::Requantize32& qp, - const int32_t *, const int32_t *, // Requant parameters, also wrapped - int8_t *const *const outptrs -) +void sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst_impl(const unsigned int n_channels, const int8_t *const *const inptrs, const int8_t *params, const int32_t *, const arm_gemm::Requantize32& qp, const int32_t *, const int32_t *, int8_t *const *const outptrs) { __asm__ __volatile__( - "mov x13, #0x0\n" - "whilelt p2.b, x13, %x[n_channels]\n" - "ldp x12, x11, [%x[inptrs], #0x0]\n" - "ldp x10, x9, [%x[inptrs], #0x10]\n" - "ldp x28, x27, [%x[inptrs], #0x20]\n" - "ldp x26, x25, [%x[inptrs], #0x30]\n" + "mov x14, #0x0\n" + "whilelt p0.b, x14, %x[n_channels]\n" + "ldp x27, x26, [%x[inptrs], #0x0]\n" + "ldp x25, x24, [%x[inptrs], #0x10]\n" + "ldp x23, x22, [%x[inptrs], #0x20]\n" + "ldp x13, x21, [%x[inptrs], #0x30]\n" "mov x20, #0x1\n" - "ptrue p1.b\n" - "ldp x24, x23, [%x[outptrs], #0x0]\n" - "ldp x22, x21, [%x[outptrs], #0x10]\n" + "ptrue p2.b\n" + "ldp x12, x11, [%x[outptrs], #0x0]\n" + "ldp x10, x9, [%x[outptrs], #0x10]\n" "orr x20, x20, #0x100\n" "orr x20, x20, #0x10000\n" - "ld1b { z14.b }, p2/Z, [x12, x13]\n" - "ld1b { z13.b }, p2/Z, [x11, x13]\n" - "dup z12.s, w20\n" - "mov x20, #0x0\n" - "ldp x12, x11, [%x[inptrs], #0x40]\n" - "ld1b { z11.b }, p2/Z, [x10, x13]\n" - "zip2 z10.b, z14.b, z11.b\n" - "zip1 z14.b, z14.b, z11.b\n" - "ld1b { z9.b }, p2/Z, [x9, x13]\n" - "ldp x10, x9, [%x[inptrs], #0x50]\n" - "zip1 z11.b, z13.b, z9.b\n" - "zip2 z9.b, z13.b, z9.b\n" - "ld1b { z8.b }, p2/Z, [x28, x13]\n" - "ld1b { z7.b }, p2/Z, [x27, x13]\n" - "zip2 z13.b, z14.b, z11.b\n" - "zip1 z14.b, z14.b, z11.b\n" - "ldp x28, x27, [%x[inptrs], #0x60]\n" - "ld1b { z6.b }, p2/Z, [x26, x13]\n" - "zip1 z11.b, z10.b, z9.b\n" - "zip2 z9.b, z10.b, z9.b\n" - "ld1b { z5.b }, p2/Z, [x25, x13]\n" - "ldp x26, x25, [%x[inptrs], #0x70]\n" - "zip2 z4.b, z8.b, z6.b\n" - "zip1 z8.b, z8.b, z6.b\n" - "ld1b { z3.b }, p2/Z, [x12, x13]\n" - "ld1b { z2.b }, p2/Z, [x11, x13]\n" - "zip1 z6.b, z7.b, z5.b\n" - "zip2 z5.b, z7.b, z5.b\n" - "ld1b { z1.b }, p2/Z, [x10, x13]\n" - "ld1b { z0.b }, p2/Z, [x9, x13]\n" - "zip2 z31.b, z3.b, z1.b\n" - "zip1 z3.b, z3.b, z1.b\n" - "ld1b { z30.b }, p2/Z, [x28, x13]\n" - "ld1b { z29.b }, p2/Z, [x27, x13]\n" - "zip1 z1.b, z2.b, z0.b\n" - "zip2 z0.b, z2.b, z0.b\n" - "ld1b { z28.b }, p2/Z, [x26, x13]\n" - "ld1b { z27.b }, p2/Z, [x25, x13]\n" - "zip2 z26.b, z30.b, z28.b\n" - "zip1 z30.b, z30.b, z28.b\n" - "zip1 z28.b, z29.b, z27.b\n" - "zip2 z27.b, z29.b, z27.b\n" - "ld1w { z10.s }, p1/Z, [%x[params]]\n" - "ld1rw { z25.s }, p1/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" - "ld1rw { z24.s }, p1/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" - "ld1rw { z23.s }, p1/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" - "zip2 z7.b, z8.b, z6.b\n" - "zip1 z8.b, z8.b, z6.b\n" - "ld1rw { z22.s }, p1/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" - "ldp x12, x11, [%x[inptrs], #0x0]\n" - "zip1 z6.b, z4.b, z5.b\n" - "zip2 z5.b, z4.b, z5.b\n" - "ldp x10, x9, [%x[inptrs], #0x10]\n" - "ldp x28, x27, [%x[inptrs], #0x20]\n" - "zip2 z2.b, z3.b, z1.b\n" - "zip1 z3.b, z3.b, z1.b\n" - "ldp x26, x25, [%x[inptrs], #0x30]\n" - "zip1 z1.b, z31.b, z0.b\n" - "zip2 z0.b, z31.b, z0.b\n" - "ld1b { z21.b }, p1/Z, [%x[params], #1, MUL VL]\n" - "zip2 z29.b, z30.b, z28.b\n" - "zip1 z30.b, z30.b, z28.b\n" - "ld1b { z16.b }, p1/Z, [%x[params], #2, MUL VL]\n" - "ld1b { z20.b }, p1/Z, [%x[params], #3, MUL VL]\n" - "zip1 z28.b, z26.b, z27.b\n" - "zip2 z27.b, z26.b, z27.b\n" + "ld1b { z15.b }, p0/Z, [x27, x14]\n" + "ld1b { z21.b }, p0/Z, [x26, x14]\n" + "dup z25.s, w20\n" + "mov x28, #0x0\n" + "ldp x27, x26, [%x[inptrs], #0x40]\n" + "ld1b { z31.b }, p0/Z, [x25, x14]\n" + "zip2 z16.b, z15.b, z31.b\n" + "zip1 z15.b, z15.b, z31.b\n" + "ld1b { z29.b }, p0/Z, [x24, x14]\n" + "ldp x25, x24, [%x[inptrs], #0x50]\n" + "zip1 z30.b, z21.b, z29.b\n" + "zip2 z29.b, z21.b, z29.b\n" + "ld1b { z9.b }, p0/Z, [x23, x14]\n" + "ld1b { z20.b }, p0/Z, [x22, x14]\n" + "zip2 z13.b, z15.b, z30.b\n" + "zip1 z15.b, z15.b, z30.b\n" + "ldp x23, x22, [%x[inptrs], #0x60]\n" + "ld1b { z5.b }, p0/Z, [x13, x14]\n" + "zip1 z14.b, z16.b, z29.b\n" + "zip2 z29.b, z16.b, z29.b\n" + "ld1b { z17.b }, p0/Z, [x21, x14]\n" + "ldp x21, x20, [%x[inptrs], #0x70]\n" + "zip2 z31.b, z9.b, z5.b\n" + "zip1 z9.b, z9.b, z5.b\n" + "ld1b { z18.b }, p0/Z, [x27, x14]\n" + "ld1b { z28.b }, p0/Z, [x26, x14]\n" + "zip1 z21.b, z20.b, z17.b\n" + "zip2 z17.b, z20.b, z17.b\n" + "ld1b { z6.b }, p0/Z, [x25, x14]\n" + "ld1b { z4.b }, p0/Z, [x24, x14]\n" + "zip2 z23.b, z18.b, z6.b\n" + "zip1 z18.b, z18.b, z6.b\n" + "ld1b { z2.b }, p0/Z, [x23, x14]\n" + "ld1b { z19.b }, p0/Z, [x22, x14]\n" + "zip1 z24.b, z28.b, z4.b\n" + "zip2 z4.b, z28.b, z4.b\n" + "ld1b { z16.b }, p0/Z, [x21, x14]\n" + "ld1b { z5.b }, p0/Z, [x20, x14]\n" + "zip2 z22.b, z2.b, z16.b\n" + "zip1 z2.b, z2.b, z16.b\n" + "zip1 z0.b, z19.b, z5.b\n" + "zip2 z5.b, z19.b, z5.b\n" + "ld1w { z10.s }, p2/Z, [%x[params]]\n" + "ld1rw { z7.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" + "ld1rw { z6.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" + "ld1rw { z8.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" + "zip2 z19.b, z9.b, z21.b\n" + "zip1 z9.b, z9.b, z21.b\n" + "ld1rw { z16.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" + "ldp x27, x26, [%x[inptrs], #0x0]\n" + "zip1 z11.b, z31.b, z17.b\n" + "zip2 z17.b, z31.b, z17.b\n" + "ldp x25, x23, [%x[inptrs], #0x10]\n" + "ldp x24, x22, [%x[inptrs], #0x20]\n" + "zip2 z12.b, z18.b, z24.b\n" + "zip1 z18.b, z18.b, z24.b\n" + "ldp x21, x20, [%x[inptrs], #0x30]\n" + "zip1 z20.b, z23.b, z4.b\n" + "zip2 z4.b, z23.b, z4.b\n" + "ld1b { z26.b }, p2/Z, [%x[params], #1, MUL VL]\n" + "zip2 z24.b, z2.b, z0.b\n" + "zip1 z2.b, z2.b, z0.b\n" + "ld1b { z3.b }, p2/Z, [%x[params], #2, MUL VL]\n" + "ld1b { z1.b }, p2/Z, [%x[params], #3, MUL VL]\n" + "zip1 z0.b, z22.b, z5.b\n" + "zip2 z5.b, z22.b, z5.b\n" "addvl %x[params], %x[params], #4\n" - "mov z4.d, z10.d\n" + "mov z22.d, z10.d\n" "mov z31.d, z10.d\n" - "mov z26.d, z10.d\n" + "mov z21.d, z10.d\n" "1:" // Loop - "mov z19.s, #0x0\n" - "sdot z19.s, z12.b, z8.b\n" - "sdot z10.s, z21.b, z14.b\n" - "whilelt p0.s, x20, %x[n_channels]\n" - "sdot z19.s, z12.b, z3.b\n" - "sdot z31.s, z21.b, z8.b\n" - "incw x13, ALL, MUL #4\n" - "sdot z10.s, z16.b, z8.b\n" - "ext z8.b, z8.b, z8.b, #0x1\n" - "movprfx z18, z19\n sdot z18.s, z12.b, z30.b\n" - "sdot z19.s, z12.b, z14.b\n" - "ext z14.b, z14.b, z14.b, #0x1\n" - "sdot z31.s, z16.b, z3.b\n" - "sdot z10.s, z20.b, z3.b\n" - "ext z3.b, z3.b, z3.b, #0x1\n" - "sdot z4.s, z21.b, z14.b\n" - "sdot z26.s, z21.b, z8.b\n" - "mov z17.s, #0x0\n" - "sdot z17.s, z12.b, z8.b\n" - "sdot z17.s, z12.b, z3.b\n" - "sdot z31.s, z20.b, z30.b\n" - "ext z30.b, z30.b, z30.b, #0x1\n" - "sdot z4.s, z16.b, z8.b\n" - "sdot z26.s, z16.b, z3.b\n" - "ld1w { z8.s }, p1/Z, [%x[params], #1, MUL VL]\n" - "mls z10.s, p1/M, z19.s, z23.s\n" - "movprfx z16, z17\n sdot z16.s, z12.b, z30.b\n" - "mov z19.s, #0x0\n" - "sdot z17.s, z12.b, z14.b\n" - "ld1w { z14.s }, p1/Z, [%x[params]]\n" - "sdot z4.s, z20.b, z3.b\n" - ".inst 0x04ae754a // sqrdmulh z10.s, z10.s, z14.s\n" - "sdot z26.s, z20.b, z30.b\n" - "mls z4.s, p1/M, z17.s, z23.s\n" - "and z21.d, z10.d, z8.d\n" - "mls z31.s, p1/M, z18.s, z23.s\n" - "mls z26.s, p1/M, z16.s, z23.s\n" - "asr z21.s, z21.s, #0x1f\n" - ".inst 0x04ae7484 // sqrdmulh z4.s, z4.s, z14.s\n" - ".inst 0x04ae77ff // sqrdmulh z31.s, z31.s, z14.s\n" - "sdot z19.s, z12.b, z7.b\n" - ".inst 0x04ae775a // sqrdmulh z26.s, z26.s, z14.s\n" - "sqadd z10.s, z10.s, z21.s\n" - ".inst 0x4482850a // srshl z10.s, p1/M, z10.s, z8.s\n" - "sdot z19.s, z12.b, z2.b\n" - "and z16.d, z4.d, z8.d\n" - "and z20.d, z31.d, z8.d\n" - "movprfx z18, z19\n sdot z18.s, z12.b, z29.b\n" - "ld1w { z14.s }, p1/Z, [%x[params], #6, MUL VL]\n" - "and z21.d, z26.d, z8.d\n" - "asr z16.s, z16.s, #0x1f\n" - "sdot z19.s, z12.b, z13.b\n" - "asr z20.s, z20.s, #0x1f\n" - "asr z21.s, z21.s, #0x1f\n" - "sqadd z4.s, z4.s, z16.s\n" - "sqadd z31.s, z31.s, z20.s\n" - ".inst 0x44828504 // srshl z4.s, p1/M, z4.s, z8.s\n" - ".inst 0x4482851f // srshl z31.s, p1/M, z31.s, z8.s\n" - "sqadd z26.s, z26.s, z21.s\n" - "add z10.s, z10.s, z22.s\n" - ".inst 0x4482851a // srshl z26.s, p1/M, z26.s, z8.s\n" - "smax z10.s, p1/M, z10.s, z25.s\n" - "add z4.s, z4.s, z22.s\n" - "add z31.s, z31.s, z22.s\n" - "smin z10.s, p1/M, z10.s, z24.s\n" - "smax z4.s, p1/M, z4.s, z25.s\n" - "add z26.s, z26.s, z22.s\n" - "smax z31.s, p1/M, z31.s, z25.s\n" - "smax z26.s, p1/M, z26.s, z25.s\n" - "st1b { z10.s }, p0, [x24, x20]\n" - "ld1w { z10.s }, p1/Z, [%x[params], #2, MUL VL]\n" - "ld1b { z21.b }, p1/Z, [%x[params], #3, MUL VL]\n" - "smin z4.s, p1/M, z4.s, z24.s\n" - "smin z31.s, p1/M, z31.s, z24.s\n" - "smin z26.s, p1/M, z26.s, z24.s\n" - "st1b { z4.s }, p0, [x23, x20]\n" - "mov z4.d, z10.d\n" - "ld1b { z16.b }, p1/Z, [%x[params], #4, MUL VL]\n" - "st1b { z31.s }, p0, [x22, x20]\n" - "mov z31.d, z10.d\n" - "sdot z31.s, z21.b, z7.b\n" - "ld1b { z20.b }, p1/Z, [%x[params], #5, MUL VL]\n" - "st1b { z26.s }, p0, [x21, x20]\n" - "mov z26.d, z10.d\n" - "sdot z10.s, z21.b, z13.b\n" - "sdot z10.s, z16.b, z7.b\n" - "ext z13.b, z13.b, z13.b, #0x1\n" - "ext z7.b, z7.b, z7.b, #0x1\n" - "sdot z4.s, z21.b, z13.b\n" - "ld1w { z8.s }, p1/Z, [%x[params], #7, MUL VL]\n" - "mov z17.s, #0x0\n" - "sdot z26.s, z21.b, z7.b\n" - "sdot z17.s, z12.b, z7.b\n" - "incw x20\n" - "sdot z31.s, z16.b, z2.b\n" - "sdot z10.s, z20.b, z2.b\n" + "mov z30.s, #0x0\n" + "sdot z30.s, z25.b, z9.b\n" + "sdot z10.s, z26.b, z15.b\n" + "whilelt p0.s, x28, %x[n_channels]\n" + "sdot z30.s, z25.b, z18.b\n" + "sdot z31.s, z26.b, z9.b\n" + "mov z27.s, #0x0\n" + "incw x14, ALL, MUL #4\n" + "sdot z10.s, z3.b, z9.b\n" + "ext z9.b, z9.b, z9.b, #0x1\n" + "movprfx z28, z30\n sdot z28.s, z25.b, z2.b\n" + "sdot z30.s, z25.b, z15.b\n" + "ext z15.b, z15.b, z15.b, #0x1\n" + "sdot z27.s, z25.b, z9.b\n" + "sdot z31.s, z3.b, z18.b\n" + "sdot z10.s, z1.b, z18.b\n" + "ext z18.b, z18.b, z18.b, #0x1\n" + "sdot z22.s, z26.b, z15.b\n" + "sdot z21.s, z26.b, z9.b\n" + "sdot z27.s, z25.b, z18.b\n" + "sdot z31.s, z1.b, z2.b\n" "ext z2.b, z2.b, z2.b, #0x1\n" - "whilelt p0.s, x20, %x[n_channels]\n" - "sdot z4.s, z16.b, z7.b\n" - "sdot z26.s, z16.b, z2.b\n" + "sdot z22.s, z3.b, z9.b\n" + "sdot z21.s, z3.b, z18.b\n" + "ld1w { z3.s }, p2/Z, [%x[params], #1, MUL VL]\n" + "mls z10.s, p2/M, z30.s, z8.s\n" + "movprfx z26, z27\n sdot z26.s, z25.b, z2.b\n" + "mov z9.s, #0x0\n" + "sdot z27.s, z25.b, z15.b\n" + "ld1w { z23.s }, p2/Z, [%x[params]]\n" + "sdot z22.s, z1.b, z18.b\n" + ".inst 0x04b7754a // sqrdmulh z10.s, z10.s, z23.s\n" + "sdot z21.s, z1.b, z2.b\n" + "mls z22.s, p2/M, z27.s, z8.s\n" + "and z18.d, z10.d, z3.d\n" + "mls z31.s, p2/M, z28.s, z8.s\n" + "mls z21.s, p2/M, z26.s, z8.s\n" + "asr z18.s, z18.s, #0x1f\n" + ".inst 0x04b776d6 // sqrdmulh z22.s, z22.s, z23.s\n" + ".inst 0x04b777ff // sqrdmulh z31.s, z31.s, z23.s\n" + "sdot z9.s, z25.b, z19.b\n" + ".inst 0x04b776b5 // sqrdmulh z21.s, z21.s, z23.s\n" + "sqadd z10.s, z10.s, z18.s\n" + ".inst 0x4482886a // srshl z10.s, p2/M, z10.s, z3.s\n" + "sdot z9.s, z25.b, z12.b\n" + "and z28.d, z22.d, z3.d\n" + "and z23.d, z31.d, z3.d\n" + "movprfx z27, z9\n sdot z27.s, z25.b, z24.b\n" + "ld1w { z30.s }, p2/Z, [%x[params], #6, MUL VL]\n" + "and z18.d, z21.d, z3.d\n" + "asr z28.s, z28.s, #0x1f\n" + "sdot z9.s, z25.b, z13.b\n" + "asr z23.s, z23.s, #0x1f\n" + "asr z18.s, z18.s, #0x1f\n" + "sqadd z22.s, z22.s, z28.s\n" + "sqadd z31.s, z31.s, z23.s\n" + ".inst 0x44828876 // srshl z22.s, p2/M, z22.s, z3.s\n" + ".inst 0x4482887f // srshl z31.s, p2/M, z31.s, z3.s\n" + "sqadd z21.s, z21.s, z18.s\n" + "add z10.s, z10.s, z16.s\n" + ".inst 0x44828875 // srshl z21.s, p2/M, z21.s, z3.s\n" + "smax z10.s, p2/M, z10.s, z7.s\n" + "add z22.s, z22.s, z16.s\n" + "add z31.s, z31.s, z16.s\n" + "smin z10.s, p2/M, z10.s, z6.s\n" + "smax z22.s, p2/M, z22.s, z7.s\n" + "add z21.s, z21.s, z16.s\n" + "smax z31.s, p2/M, z31.s, z7.s\n" + "smax z21.s, p2/M, z21.s, z7.s\n" + "st1b { z10.s }, p0, [x12, x28]\n" + "ld1w { z28.s }, p2/Z, [%x[params], #2, MUL VL]\n" + "ld1b { z1.b }, p2/Z, [%x[params], #3, MUL VL]\n" + "smin z22.s, p2/M, z22.s, z6.s\n" + "smin z31.s, p2/M, z31.s, z6.s\n" + "smin z21.s, p2/M, z21.s, z6.s\n" + "st1b { z22.s }, p0, [x11, x28]\n" + "mov z26.d, z28.d\n" + "ld1b { z15.b }, p2/Z, [%x[params], #4, MUL VL]\n" + "st1b { z31.s }, p0, [x10, x28]\n" + "mov z31.d, z28.d\n" + "sdot z31.s, z1.b, z19.b\n" + "ld1b { z23.b }, p2/Z, [%x[params], #5, MUL VL]\n" + "st1b { z21.s }, p0, [x9, x28]\n" + "mov z22.d, z28.d\n" + "sdot z28.s, z1.b, z13.b\n" + "sdot z28.s, z15.b, z19.b\n" + "ext z13.b, z13.b, z13.b, #0x1\n" + "ext z19.b, z19.b, z19.b, #0x1\n" + "sdot z26.s, z1.b, z13.b\n" + "ld1w { z21.s }, p2/Z, [%x[params], #7, MUL VL]\n" + "mov z18.s, #0x0\n" + "sdot z22.s, z1.b, z19.b\n" + "sdot z18.s, z25.b, z19.b\n" + "incw x28\n" + "sdot z31.s, z15.b, z12.b\n" + "sdot z28.s, z23.b, z12.b\n" + "ext z12.b, z12.b, z12.b, #0x1\n" + "whilelt p0.s, x28, %x[n_channels]\n" + "sdot z26.s, z15.b, z19.b\n" + "sdot z22.s, z15.b, z12.b\n" "addvl %x[params], %x[params], #16\n" - "sdot z17.s, z12.b, z2.b\n" - "sdot z31.s, z20.b, z29.b\n" - "ext z29.b, z29.b, z29.b, #0x1\n" - "mls z10.s, p1/M, z19.s, z23.s\n" - "sdot z4.s, z20.b, z2.b\n" - ".inst 0x04ae754a // sqrdmulh z10.s, z10.s, z14.s\n" - "sdot z26.s, z20.b, z29.b\n" - "movprfx z16, z17\n sdot z16.s, z12.b, z29.b\n" - "and z21.d, z10.d, z8.d\n" - "sdot z17.s, z12.b, z13.b\n" - "mls z4.s, p1/M, z17.s, z23.s\n" - "asr z21.s, z21.s, #0x1f\n" - "mls z31.s, p1/M, z18.s, z23.s\n" - "mls z26.s, p1/M, z16.s, z23.s\n" - ".inst 0x04ae7484 // sqrdmulh z4.s, z4.s, z14.s\n" - ".inst 0x04ae77ff // sqrdmulh z31.s, z31.s, z14.s\n" - ".inst 0x04ae775a // sqrdmulh z26.s, z26.s, z14.s\n" - "ld1w { z14.s }, p1/Z, [%x[params], #-4, MUL VL]\n" - "sqadd z10.s, z10.s, z21.s\n" - "and z16.d, z4.d, z8.d\n" - ".inst 0x4482850a // srshl z10.s, p1/M, z10.s, z8.s\n" - "and z20.d, z31.d, z8.d\n" - "and z21.d, z26.d, z8.d\n" - "asr z16.s, z16.s, #0x1f\n" - "asr z20.s, z20.s, #0x1f\n" - "asr z21.s, z21.s, #0x1f\n" - "sqadd z4.s, z4.s, z16.s\n" - ".inst 0x44828504 // srshl z4.s, p1/M, z4.s, z8.s\n" - "ld1b { z16.b }, p1/Z, [%x[params], #-6, MUL VL]\n" - "sqadd z31.s, z31.s, z20.s\n" - "sqadd z26.s, z26.s, z21.s\n" - ".inst 0x4482851f // srshl z31.s, p1/M, z31.s, z8.s\n" - ".inst 0x4482851a // srshl z26.s, p1/M, z26.s, z8.s\n" - "add z10.s, z10.s, z22.s\n" - "smax z10.s, p1/M, z10.s, z25.s\n" - "add z4.s, z4.s, z22.s\n" - "smin z10.s, p1/M, z10.s, z24.s\n" - "add z31.s, z31.s, z22.s\n" - "add z26.s, z26.s, z22.s\n" - "smax z4.s, p1/M, z4.s, z25.s\n" - "smax z31.s, p1/M, z31.s, z25.s\n" - "mov z19.s, #0x0\n" - "sdot z19.s, z12.b, z6.b\n" - "smax z26.s, p1/M, z26.s, z25.s\n" - "st1b { z10.s }, p0, [x24, x20]\n" - "ld1w { z10.s }, p1/Z, [%x[params], #-8, MUL VL]\n" - "ld1b { z21.b }, p1/Z, [%x[params], #-7, MUL VL]\n" - "smin z4.s, p1/M, z4.s, z24.s\n" - "smin z31.s, p1/M, z31.s, z24.s\n" - "smin z26.s, p1/M, z26.s, z24.s\n" - "st1b { z4.s }, p0, [x23, x20]\n" - "mov z4.d, z10.d\n" - "sdot z19.s, z12.b, z1.b\n" - "st1b { z31.s }, p0, [x22, x20]\n" - "mov z31.d, z10.d\n" - "sdot z31.s, z21.b, z6.b\n" - "movprfx z18, z19\n sdot z18.s, z12.b, z28.b\n" - "st1b { z26.s }, p0, [x21, x20]\n" - "mov z26.d, z10.d\n" - "sdot z10.s, z21.b, z11.b\n" - "sdot z10.s, z16.b, z6.b\n" - "sdot z19.s, z12.b, z11.b\n" + "sdot z18.s, z25.b, z12.b\n" + "sdot z31.s, z23.b, z24.b\n" + "ext z24.b, z24.b, z24.b, #0x1\n" + "mls z28.s, p2/M, z9.s, z8.s\n" + "sdot z26.s, z23.b, z12.b\n" + ".inst 0x04be779c // sqrdmulh z28.s, z28.s, z30.s\n" + "sdot z22.s, z23.b, z24.b\n" + "movprfx z12, z18\n sdot z12.s, z25.b, z24.b\n" + "and z2.d, z28.d, z21.d\n" + "sdot z18.s, z25.b, z13.b\n" + "mls z26.s, p2/M, z18.s, z8.s\n" + "asr z2.s, z2.s, #0x1f\n" + "mls z31.s, p2/M, z27.s, z8.s\n" + "mls z22.s, p2/M, z12.s, z8.s\n" + ".inst 0x04be775a // sqrdmulh z26.s, z26.s, z30.s\n" + ".inst 0x04be77ff // sqrdmulh z31.s, z31.s, z30.s\n" + ".inst 0x04be76d6 // sqrdmulh z22.s, z22.s, z30.s\n" + "ld1w { z1.s }, p2/Z, [%x[params], #-4, MUL VL]\n" + "sqadd z28.s, z28.s, z2.s\n" + "and z24.d, z26.d, z21.d\n" + ".inst 0x44828abc // srshl z28.s, p2/M, z28.s, z21.s\n" + "and z23.d, z31.d, z21.d\n" + "and z18.d, z22.d, z21.d\n" + "asr z24.s, z24.s, #0x1f\n" + "asr z23.s, z23.s, #0x1f\n" + "asr z18.s, z18.s, #0x1f\n" + "sqadd z26.s, z26.s, z24.s\n" + ".inst 0x44828aba // srshl z26.s, p2/M, z26.s, z21.s\n" + "ld1b { z30.b }, p2/Z, [%x[params], #-6, MUL VL]\n" + "sqadd z31.s, z31.s, z23.s\n" + "sqadd z22.s, z22.s, z18.s\n" + ".inst 0x44828abf // srshl z31.s, p2/M, z31.s, z21.s\n" + ".inst 0x44828ab6 // srshl z22.s, p2/M, z22.s, z21.s\n" + "add z28.s, z28.s, z16.s\n" + "smax z28.s, p2/M, z28.s, z7.s\n" + "add z26.s, z26.s, z16.s\n" + "smin z28.s, p2/M, z28.s, z6.s\n" + "add z31.s, z31.s, z16.s\n" + "add z22.s, z22.s, z16.s\n" + "smax z26.s, p2/M, z26.s, z7.s\n" + "smax z31.s, p2/M, z31.s, z7.s\n" + "mov z24.s, #0x0\n" + "sdot z24.s, z25.b, z11.b\n" + "smax z22.s, p2/M, z22.s, z7.s\n" + "st1b { z28.s }, p0, [x12, x28]\n" + "ld1w { z23.s }, p2/Z, [%x[params], #-8, MUL VL]\n" + "ld1b { z19.b }, p2/Z, [%x[params], #-7, MUL VL]\n" + "smin z26.s, p2/M, z26.s, z6.s\n" + "smin z31.s, p2/M, z31.s, z6.s\n" + "smin z22.s, p2/M, z22.s, z6.s\n" + "st1b { z26.s }, p0, [x11, x28]\n" + "mov z28.d, z23.d\n" + "sdot z24.s, z25.b, z20.b\n" + "st1b { z31.s }, p0, [x10, x28]\n" + "mov z27.d, z23.d\n" + "sdot z27.s, z19.b, z11.b\n" + "movprfx z13, z24\n sdot z13.s, z25.b, z0.b\n" + "st1b { z22.s }, p0, [x9, x28]\n" + "mov z26.d, z23.d\n" + "sdot z23.s, z19.b, z14.b\n" + "sdot z23.s, z30.b, z11.b\n" + "sdot z24.s, z25.b, z14.b\n" + "ext z14.b, z14.b, z14.b, #0x1\n" + "ld1b { z21.b }, p2/Z, [%x[params], #-5, MUL VL]\n" + "sdot z28.s, z19.b, z14.b\n" "ext z11.b, z11.b, z11.b, #0x1\n" - "ld1b { z20.b }, p1/Z, [%x[params], #-5, MUL VL]\n" - "sdot z4.s, z21.b, z11.b\n" - "ext z6.b, z6.b, z6.b, #0x1\n" - "mov z17.s, #0x0\n" - "sdot z26.s, z21.b, z6.b\n" - "ld1w { z8.s }, p1/Z, [%x[params], #-3, MUL VL]\n" - "sdot z17.s, z12.b, z6.b\n" - "sdot z31.s, z16.b, z1.b\n" - "incw x20\n" - "whilelt p0.s, x20, %x[n_channels]\n" - "sdot z10.s, z20.b, z1.b\n" - "ext z1.b, z1.b, z1.b, #0x1\n" - "sdot z4.s, z16.b, z6.b\n" - "sdot z26.s, z16.b, z1.b\n" - "sdot z17.s, z12.b, z1.b\n" - "sdot z31.s, z20.b, z28.b\n" - "ext z28.b, z28.b, z28.b, #0x1\n" - "mls z10.s, p1/M, z19.s, z23.s\n" - "sdot z4.s, z20.b, z1.b\n" - "sdot z26.s, z20.b, z28.b\n" - ".inst 0x04ae754a // sqrdmulh z10.s, z10.s, z14.s\n" - "movprfx z16, z17\n sdot z16.s, z12.b, z28.b\n" - "sdot z17.s, z12.b, z11.b\n" - "and z21.d, z10.d, z8.d\n" - "mls z4.s, p1/M, z17.s, z23.s\n" - "mls z31.s, p1/M, z18.s, z23.s\n" - "asr z21.s, z21.s, #0x1f\n" - "mls z26.s, p1/M, z16.s, z23.s\n" - ".inst 0x04ae7484 // sqrdmulh z4.s, z4.s, z14.s\n" - ".inst 0x04ae77ff // sqrdmulh z31.s, z31.s, z14.s\n" - ".inst 0x04ae775a // sqrdmulh z26.s, z26.s, z14.s\n" - "ld1w { z14.s }, p1/Z, [%x[params], #2, MUL VL]\n" - "sqadd z10.s, z10.s, z21.s\n" - "and z16.d, z4.d, z8.d\n" - ".inst 0x4482850a // srshl z10.s, p1/M, z10.s, z8.s\n" - "and z20.d, z31.d, z8.d\n" - "and z21.d, z26.d, z8.d\n" - "asr z16.s, z16.s, #0x1f\n" + "mov z12.s, #0x0\n" + "sdot z26.s, z19.b, z11.b\n" + "ld1w { z22.s }, p2/Z, [%x[params], #-3, MUL VL]\n" + "sdot z12.s, z25.b, z11.b\n" + "sdot z27.s, z30.b, z20.b\n" + "incw x28\n" + "whilelt p0.s, x28, %x[n_channels]\n" + "sdot z23.s, z21.b, z20.b\n" + "ext z20.b, z20.b, z20.b, #0x1\n" + "sdot z28.s, z30.b, z11.b\n" + "sdot z26.s, z30.b, z20.b\n" + "sdot z12.s, z25.b, z20.b\n" + "sdot z27.s, z21.b, z0.b\n" + "ext z0.b, z0.b, z0.b, #0x1\n" + "mls z23.s, p2/M, z24.s, z8.s\n" + "sdot z28.s, z21.b, z20.b\n" + "sdot z26.s, z21.b, z0.b\n" + ".inst 0x04a176f7 // sqrdmulh z23.s, z23.s, z1.s\n" + "movprfx z19, z12\n sdot z19.s, z25.b, z0.b\n" + "sdot z12.s, z25.b, z14.b\n" + "and z18.d, z23.d, z22.d\n" + "mls z28.s, p2/M, z12.s, z8.s\n" + "mls z27.s, p2/M, z13.s, z8.s\n" + "asr z18.s, z18.s, #0x1f\n" + "mls z26.s, p2/M, z19.s, z8.s\n" + ".inst 0x04a1779c // sqrdmulh z28.s, z28.s, z1.s\n" + ".inst 0x04a1777b // sqrdmulh z27.s, z27.s, z1.s\n" + ".inst 0x04a1775a // sqrdmulh z26.s, z26.s, z1.s\n" + "ld1w { z2.s }, p2/Z, [%x[params], #2, MUL VL]\n" + "sqadd z23.s, z23.s, z18.s\n" + "and z20.d, z28.d, z22.d\n" + ".inst 0x44828ad7 // srshl z23.s, p2/M, z23.s, z22.s\n" + "and z19.d, z27.d, z22.d\n" + "and z18.d, z26.d, z22.d\n" "asr z20.s, z20.s, #0x1f\n" - "asr z21.s, z21.s, #0x1f\n" - "sqadd z4.s, z4.s, z16.s\n" - ".inst 0x44828504 // srshl z4.s, p1/M, z4.s, z8.s\n" - "ld1b { z16.b }, p1/Z, [%x[params]]\n" - "sqadd z31.s, z31.s, z20.s\n" - "sqadd z26.s, z26.s, z21.s\n" - ".inst 0x4482851f // srshl z31.s, p1/M, z31.s, z8.s\n" - ".inst 0x4482851a // srshl z26.s, p1/M, z26.s, z8.s\n" - "add z10.s, z10.s, z22.s\n" - "smax z10.s, p1/M, z10.s, z25.s\n" - "add z4.s, z4.s, z22.s\n" - "smin z10.s, p1/M, z10.s, z24.s\n" - "add z31.s, z31.s, z22.s\n" - "add z26.s, z26.s, z22.s\n" - "smax z4.s, p1/M, z4.s, z25.s\n" - "smax z31.s, p1/M, z31.s, z25.s\n" + "asr z19.s, z19.s, #0x1f\n" + "asr z18.s, z18.s, #0x1f\n" + "sqadd z28.s, z28.s, z20.s\n" + ".inst 0x44828adc // srshl z28.s, p2/M, z28.s, z22.s\n" + "ld1b { z13.b }, p2/Z, [%x[params]]\n" + "sqadd z27.s, z27.s, z19.s\n" + "sqadd z26.s, z26.s, z18.s\n" + ".inst 0x44828adb // srshl z27.s, p2/M, z27.s, z22.s\n" + ".inst 0x44828ada // srshl z26.s, p2/M, z26.s, z22.s\n" + "add z23.s, z23.s, z16.s\n" + "smax z23.s, p2/M, z23.s, z7.s\n" + "add z28.s, z28.s, z16.s\n" + "smin z23.s, p2/M, z23.s, z6.s\n" + "add z27.s, z27.s, z16.s\n" + "add z26.s, z26.s, z16.s\n" + "smax z28.s, p2/M, z28.s, z7.s\n" + "smax z27.s, p2/M, z27.s, z7.s\n" + "mov z24.s, #0x0\n" + "sdot z24.s, z25.b, z17.b\n" + "smax z26.s, p2/M, z26.s, z7.s\n" + "st1b { z23.s }, p0, [x12, x28]\n" + "ld1w { z1.s }, p2/Z, [%x[params], #-2, MUL VL]\n" + "ld1b { z21.b }, p2/Z, [%x[params], #-1, MUL VL]\n" + "smin z28.s, p2/M, z28.s, z6.s\n" + "smin z27.s, p2/M, z27.s, z6.s\n" + "smin z26.s, p2/M, z26.s, z6.s\n" + "st1b { z28.s }, p0, [x11, x28]\n" + "mov z0.d, z1.d\n" + "sdot z24.s, z25.b, z4.b\n" + "st1b { z27.s }, p0, [x10, x28]\n" + "mov z31.d, z1.d\n" + "sdot z31.s, z21.b, z17.b\n" + "movprfx z23, z24\n sdot z23.s, z25.b, z5.b\n" + "st1b { z26.s }, p0, [x9, x28]\n" + "mov z30.d, z1.d\n" + "sdot z1.s, z21.b, z29.b\n" + "sdot z1.s, z13.b, z17.b\n" + "sdot z24.s, z25.b, z29.b\n" + "ext z29.b, z29.b, z29.b, #0x1\n" + "ld1b { z20.b }, p2/Z, [%x[params], #1, MUL VL]\n" + "sdot z0.s, z21.b, z29.b\n" + "ext z17.b, z17.b, z17.b, #0x1\n" "mov z19.s, #0x0\n" - "sdot z19.s, z12.b, z5.b\n" - "smax z26.s, p1/M, z26.s, z25.s\n" - "st1b { z10.s }, p0, [x24, x20]\n" - "ld1w { z10.s }, p1/Z, [%x[params], #-2, MUL VL]\n" - "ld1b { z21.b }, p1/Z, [%x[params], #-1, MUL VL]\n" - "smin z4.s, p1/M, z4.s, z24.s\n" - "smin z31.s, p1/M, z31.s, z24.s\n" - "smin z26.s, p1/M, z26.s, z24.s\n" - "st1b { z4.s }, p0, [x23, x20]\n" - "mov z4.d, z10.d\n" - "sdot z19.s, z12.b, z0.b\n" - "st1b { z31.s }, p0, [x22, x20]\n" - "mov z31.d, z10.d\n" - "sdot z31.s, z21.b, z5.b\n" - "movprfx z18, z19\n sdot z18.s, z12.b, z27.b\n" - "st1b { z26.s }, p0, [x21, x20]\n" - "mov z26.d, z10.d\n" - "sdot z10.s, z21.b, z9.b\n" - "sdot z10.s, z16.b, z5.b\n" - "sdot z19.s, z12.b, z9.b\n" - "ext z9.b, z9.b, z9.b, #0x1\n" - "ld1b { z20.b }, p1/Z, [%x[params], #1, MUL VL]\n" - "sdot z4.s, z21.b, z9.b\n" + "sdot z30.s, z21.b, z17.b\n" + "ld1w { z22.s }, p2/Z, [%x[params], #3, MUL VL]\n" + "sdot z19.s, z25.b, z17.b\n" + "sdot z31.s, z13.b, z4.b\n" + "incw x28\n" + "whilelt p1.s, x28, %x[n_channels]\n" + "sdot z1.s, z20.b, z4.b\n" + "ext z4.b, z4.b, z4.b, #0x1\n" + "sdot z0.s, z13.b, z17.b\n" + "whilelt p0.b, x14, %x[n_channels]\n" + "sdot z30.s, z13.b, z4.b\n" + "sdot z19.s, z25.b, z4.b\n" + "ld1b { z13.b }, p0/Z, [x26, x14]\n" + "ld1b { z28.b }, p0/Z, [x25, x14]\n" + "sdot z31.s, z20.b, z5.b\n" "ext z5.b, z5.b, z5.b, #0x1\n" - "mov z17.s, #0x0\n" - "sdot z26.s, z21.b, z5.b\n" - "ld1w { z8.s }, p1/Z, [%x[params], #3, MUL VL]\n" - "sdot z17.s, z12.b, z5.b\n" - "sdot z31.s, z16.b, z0.b\n" - "incw x20\n" - "whilelt p0.s, x20, %x[n_channels]\n" - "sdot z10.s, z20.b, z0.b\n" - "ext z0.b, z0.b, z0.b, #0x1\n" - "sdot z4.s, z16.b, z5.b\n" - "whilelt p2.b, x13, %x[n_channels]\n" - "sdot z26.s, z16.b, z0.b\n" - "sdot z17.s, z12.b, z0.b\n" - "ld1b { z13.b }, p2/Z, [x11, x13]\n" - "ld1b { z11.b }, p2/Z, [x10, x13]\n" - "sdot z31.s, z20.b, z27.b\n" - "ext z27.b, z27.b, z27.b, #0x1\n" - "mls z10.s, p1/M, z19.s, z23.s\n" - "ld1b { z7.b }, p2/Z, [x27, x13]\n" - "sdot z4.s, z20.b, z0.b\n" - "sdot z26.s, z20.b, z27.b\n" - ".inst 0x04ae754a // sqrdmulh z10.s, z10.s, z14.s\n" - "ld1b { z6.b }, p2/Z, [x26, x13]\n" - "movprfx z16, z17\n sdot z16.s, z12.b, z27.b\n" - "sdot z17.s, z12.b, z9.b\n" - "and z21.d, z10.d, z8.d\n" - "ld1b { z9.b }, p2/Z, [x9, x13]\n" - "mls z4.s, p1/M, z17.s, z23.s\n" - "mls z31.s, p1/M, z18.s, z23.s\n" + "mls z1.s, p2/M, z24.s, z8.s\n" + "ld1b { z27.b }, p0/Z, [x22, x14]\n" + "sdot z0.s, z20.b, z4.b\n" + "sdot z30.s, z20.b, z5.b\n" + ".inst 0x04a27421 // sqrdmulh z1.s, z1.s, z2.s\n" + "ld1b { z26.b }, p0/Z, [x21, x14]\n" + "movprfx z18, z19\n sdot z18.s, z25.b, z5.b\n" + "sdot z19.s, z25.b, z29.b\n" + "and z11.d, z1.d, z22.d\n" + "ld1b { z29.b }, p0/Z, [x23, x14]\n" + "mls z0.s, p2/M, z19.s, z8.s\n" + "mls z31.s, p2/M, z23.s, z8.s\n" + "asr z11.s, z11.s, #0x1f\n" + "ld1b { z17.b }, p0/Z, [x20, x14]\n" + "mls z30.s, p2/M, z18.s, z8.s\n" + ".inst 0x04a27400 // sqrdmulh z0.s, z0.s, z2.s\n" + ".inst 0x04a277ff // sqrdmulh z31.s, z31.s, z2.s\n" + ".inst 0x04a277de // sqrdmulh z30.s, z30.s, z2.s\n" + "ld1b { z15.b }, p0/Z, [x27, x14]\n" + "ldp x23, x22, [%x[inptrs], #0x40]\n" + "sqadd z1.s, z1.s, z11.s\n" + "and z21.d, z0.d, z22.d\n" + ".inst 0x44828ac1 // srshl z1.s, p2/M, z1.s, z22.s\n" + "ldp x21, x20, [%x[inptrs], #0x50]\n" + "and z20.d, z31.d, z22.d\n" + "and z19.d, z30.d, z22.d\n" + "ld1b { z18.b }, p0/Z, [x23, x14]\n" + "ld1b { z11.b }, p0/Z, [x22, x14]\n" "asr z21.s, z21.s, #0x1f\n" - "ld1b { z5.b }, p2/Z, [x25, x13]\n" - "mls z26.s, p1/M, z16.s, z23.s\n" - ".inst 0x04ae7484 // sqrdmulh z4.s, z4.s, z14.s\n" - ".inst 0x04ae77ff // sqrdmulh z31.s, z31.s, z14.s\n" - ".inst 0x04ae775a // sqrdmulh z26.s, z26.s, z14.s\n" - "ld1b { z14.b }, p2/Z, [x12, x13]\n" - "ldp x12, x11, [%x[inptrs], #0x40]\n" - "sqadd z10.s, z10.s, z21.s\n" - "and z16.d, z4.d, z8.d\n" - ".inst 0x4482850a // srshl z10.s, p1/M, z10.s, z8.s\n" - "ldp x10, x9, [%x[inptrs], #0x50]\n" - "and z20.d, z31.d, z8.d\n" - "and z21.d, z26.d, z8.d\n" - "ld1b { z3.b }, p2/Z, [x12, x13]\n" - "ld1b { z2.b }, p2/Z, [x11, x13]\n" - "asr z16.s, z16.s, #0x1f\n" "asr z20.s, z20.s, #0x1f\n" - "ld1b { z1.b }, p2/Z, [x10, x13]\n" - "ld1b { z0.b }, p2/Z, [x9, x13]\n" - "asr z21.s, z21.s, #0x1f\n" - "sqadd z4.s, z4.s, z16.s\n" - ".inst 0x44828504 // srshl z4.s, p1/M, z4.s, z8.s\n" - "ld1b { z16.b }, p1/Z, [%x[params], #6, MUL VL]\n" + "ld1b { z24.b }, p0/Z, [x21, x14]\n" + "ld1b { z4.b }, p0/Z, [x20, x14]\n" + "asr z19.s, z19.s, #0x1f\n" + "sqadd z0.s, z0.s, z21.s\n" + ".inst 0x44828ac0 // srshl z0.s, p2/M, z0.s, z22.s\n" + "ld1b { z3.b }, p2/Z, [%x[params], #6, MUL VL]\n" "sqadd z31.s, z31.s, z20.s\n" - "sqadd z26.s, z26.s, z21.s\n" - ".inst 0x4482851f // srshl z31.s, p1/M, z31.s, z8.s\n" - ".inst 0x4482851a // srshl z26.s, p1/M, z26.s, z8.s\n" - "add z10.s, z10.s, z22.s\n" - "smax z10.s, p1/M, z10.s, z25.s\n" - "add z4.s, z4.s, z22.s\n" - "ld1b { z8.b }, p2/Z, [x28, x13]\n" - "add z31.s, z31.s, z22.s\n" - "add z26.s, z26.s, z22.s\n" - "ldp x28, x27, [%x[inptrs], #0x60]\n" - "ldp x26, x25, [%x[inptrs], #0x70]\n" - "smin z10.s, p1/M, z10.s, z24.s\n" - "smax z4.s, p1/M, z4.s, z25.s\n" - "st1b { z10.s }, p0, [x24, x20]\n" - "ld1b { z30.b }, p2/Z, [x28, x13]\n" - "smax z31.s, p1/M, z31.s, z25.s\n" - "smax z26.s, p1/M, z26.s, z25.s\n" - "ld1b { z29.b }, p2/Z, [x27, x13]\n" - "ld1b { z28.b }, p2/Z, [x26, x13]\n" - "ld1b { z27.b }, p2/Z, [x25, x13]\n" - "zip2 z10.b, z14.b, z11.b\n" - "zip1 z14.b, z14.b, z11.b\n" - "smin z4.s, p1/M, z4.s, z24.s\n" - "zip1 z11.b, z13.b, z9.b\n" - "zip2 z9.b, z13.b, z9.b\n" - "smin z31.s, p1/M, z31.s, z24.s\n" - "smin z26.s, p1/M, z26.s, z24.s\n" - "st1b { z4.s }, p0, [x23, x20]\n" - "zip2 z13.b, z14.b, z11.b\n" - "zip1 z14.b, z14.b, z11.b\n" - "ldp x12, x11, [%x[inptrs], #0x0]\n" - "st1b { z31.s }, p0, [x22, x20]\n" - "zip1 z11.b, z10.b, z9.b\n" - "zip2 z9.b, z10.b, z9.b\n" - "ld1w { z10.s }, p1/Z, [%x[params], #4, MUL VL]\n" - "st1b { z26.s }, p0, [x21, x20]\n" - "zip2 z4.b, z8.b, z6.b\n" - "zip1 z8.b, z8.b, z6.b\n" - "incw x20\n" - "zip1 z6.b, z7.b, z5.b\n" - "zip2 z5.b, z7.b, z5.b\n" - "ldp x10, x9, [%x[inptrs], #0x10]\n" - "ldp x28, x27, [%x[inptrs], #0x20]\n" - "zip2 z31.b, z3.b, z1.b\n" - "zip1 z3.b, z3.b, z1.b\n" - "ldp x26, x25, [%x[inptrs], #0x30]\n" - "ld1b { z21.b }, p1/Z, [%x[params], #5, MUL VL]\n" - "zip1 z1.b, z2.b, z0.b\n" - "zip2 z0.b, z2.b, z0.b\n" - "ld1b { z20.b }, p1/Z, [%x[params], #7, MUL VL]\n" + "sqadd z30.s, z30.s, z19.s\n" + ".inst 0x44828adf // srshl z31.s, p2/M, z31.s, z22.s\n" + ".inst 0x44828ade // srshl z30.s, p2/M, z30.s, z22.s\n" + "add z1.s, z1.s, z16.s\n" + "smax z1.s, p2/M, z1.s, z7.s\n" + "add z0.s, z0.s, z16.s\n" + "ld1b { z9.b }, p0/Z, [x24, x14]\n" + "add z31.s, z31.s, z16.s\n" + "add z30.s, z30.s, z16.s\n" + "ldp x23, x22, [%x[inptrs], #0x60]\n" + "ldp x21, x20, [%x[inptrs], #0x70]\n" + "smin z1.s, p2/M, z1.s, z6.s\n" + "smax z0.s, p2/M, z0.s, z7.s\n" + "st1b { z1.s }, p1, [x12, x28]\n" + "ld1b { z2.b }, p0/Z, [x23, x14]\n" + "smax z31.s, p2/M, z31.s, z7.s\n" + "smax z30.s, p2/M, z30.s, z7.s\n" + "ld1b { z23.b }, p0/Z, [x22, x14]\n" + "ld1b { z22.b }, p0/Z, [x21, x14]\n" + "ld1b { z5.b }, p0/Z, [x20, x14]\n" + "zip2 z20.b, z15.b, z28.b\n" + "zip1 z15.b, z15.b, z28.b\n" + "smin z0.s, p2/M, z0.s, z6.s\n" + "zip1 z19.b, z13.b, z29.b\n" + "zip2 z29.b, z13.b, z29.b\n" + "smin z31.s, p2/M, z31.s, z6.s\n" + "smin z30.s, p2/M, z30.s, z6.s\n" + "st1b { z0.s }, p1, [x11, x28]\n" + "zip2 z13.b, z15.b, z19.b\n" + "zip1 z15.b, z15.b, z19.b\n" + "ldp x27, x26, [%x[inptrs], #0x0]\n" + "st1b { z31.s }, p1, [x10, x28]\n" + "zip1 z14.b, z20.b, z29.b\n" + "zip2 z29.b, z20.b, z29.b\n" + "ld1w { z10.s }, p2/Z, [%x[params], #4, MUL VL]\n" + "st1b { z30.s }, p1, [x9, x28]\n" + "zip2 z21.b, z9.b, z26.b\n" + "zip1 z9.b, z9.b, z26.b\n" + "incw x28\n" + "zip1 z20.b, z27.b, z17.b\n" + "zip2 z17.b, z27.b, z17.b\n" + "ldp x25, x23, [%x[inptrs], #0x10]\n" + "ldp x24, x22, [%x[inptrs], #0x20]\n" + "zip2 z31.b, z18.b, z24.b\n" + "zip1 z18.b, z18.b, z24.b\n" + "ldp x21, x20, [%x[inptrs], #0x30]\n" + "ld1b { z26.b }, p2/Z, [%x[params], #5, MUL VL]\n" + "zip1 z27.b, z11.b, z4.b\n" + "zip2 z4.b, z11.b, z4.b\n" + "ld1b { z1.b }, p2/Z, [%x[params], #7, MUL VL]\n" "addvl %x[params], %x[params], #8\n" - "zip2 z26.b, z30.b, z28.b\n" - "zip1 z30.b, z30.b, z28.b\n" - "zip1 z28.b, z29.b, z27.b\n" - "zip2 z27.b, z29.b, z27.b\n" - "zip2 z7.b, z8.b, z6.b\n" - "zip1 z8.b, z8.b, z6.b\n" - "zip1 z6.b, z4.b, z5.b\n" - "zip2 z5.b, z4.b, z5.b\n" - "zip2 z2.b, z3.b, z1.b\n" - "zip1 z3.b, z3.b, z1.b\n" - "zip1 z1.b, z31.b, z0.b\n" - "zip2 z0.b, z31.b, z0.b\n" - "zip2 z29.b, z30.b, z28.b\n" - "zip1 z30.b, z30.b, z28.b\n" - "zip1 z28.b, z26.b, z27.b\n" - "zip2 z27.b, z26.b, z27.b\n" - "mov z4.d, z10.d\n" + "zip2 z30.b, z2.b, z22.b\n" + "zip1 z2.b, z2.b, z22.b\n" + "zip1 z28.b, z23.b, z5.b\n" + "zip2 z5.b, z23.b, z5.b\n" + "zip2 z19.b, z9.b, z20.b\n" + "zip1 z9.b, z9.b, z20.b\n" + "zip1 z11.b, z21.b, z17.b\n" + "zip2 z17.b, z21.b, z17.b\n" + "zip2 z12.b, z18.b, z27.b\n" + "zip1 z18.b, z18.b, z27.b\n" + "zip1 z20.b, z31.b, z4.b\n" + "zip2 z4.b, z31.b, z4.b\n" + "zip2 z24.b, z2.b, z28.b\n" + "zip1 z2.b, z2.b, z28.b\n" + "zip1 z0.b, z30.b, z5.b\n" + "zip2 z5.b, z30.b, z5.b\n" + "mov z22.d, z10.d\n" "mov z31.d, z10.d\n" - "mov z26.d, z10.d\n" + "mov z21.d, z10.d\n" "b.any 1b\n" : [params] "+&r" (params) : [inptrs] "r" (inptrs), [n_channels] "r" (n_channels), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [outptrs] "r" (outptrs), [qp] "r" (&qp) - : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp index 3e9765165c9748992cb1f9d460593349fe1ee1c7..c9b4daf334bbd02e9afa47e910abf6b52d9c5f1f 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,14 +22,14 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include #pragma once -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -47,17 +47,16 @@ class sve_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirstS constexpr static unsigned int stride_rows = 1; constexpr static unsigned int stride_cols = 1; - arm_gemm::VLType get_vl_type(void) const override { return arm_gemm::VLType::SVE; } - unsigned int get_accumulator_depth_vl(void) const override { return 2; } - sve_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst(const CPUInfo *) : Parent(2, 2, 3, 3, 1, 1) {} - Parent::KernelType kernel = sve_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl; + arm_gemm::VLType get_vl_type(void) const override { return arm_gemm::VLType::SVE; } + Parent::KernelType kernel = sve_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl; Parent::KernelType get_kernel(void) const override { return kernel; } + unsigned int get_accumulator_depth_vl(void) const override { return 2; } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp index 4ebf5be2859a7655cbb04b0e3fb4c57bcb696373..8ac522dc9a22268afefcd470880e72bf9d3fbffc 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp @@ -27,7 +27,7 @@ #include #include -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -46,7 +46,7 @@ void sve_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( struct Params { long unsigned int n_channels; - const int8_t *weights; + const void *weights; const int32_t *bias; const arm_gemm::Requantize32 *requant; const int32_t *const requant_muls; @@ -57,7 +57,7 @@ void sve_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( Params( long unsigned int n_channels, const int8_t *const *inptrs_raw, - const int8_t *const weights, + const void *const weights, const int32_t *const bias, const arm_gemm::Requantize32 &qp, const int32_t *const requant_muls, @@ -91,320 +91,320 @@ void sve_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( requant_muls, requant_shifts, outptrs); __asm__ __volatile__( - "mov x8, #0x0\n" + "mov x16, #0x0\n" "ldr x25, [%x[params], %[offsetof_Params_requant]]\n" "ptrue p4.b\n" "ldr x24, [%x[params], %[offsetof_Params_outptrs]]\n" - "mov x23, x8\n" + "mov x23, x16\n" "add x21, x25, %[offsetof_Requantize32_a_offset]\n" - "ldr x17, [%x[params], %[offsetof_Params_n_channels]]\n" - "ldr x16, [%x[params], %[offsetof_Params_weights]]\n" + "ldr x15, [%x[params], %[offsetof_Params_n_channels]]\n" + "ldr x14, [%x[params], %[offsetof_Params_weights]]\n" "add x20, x25, %[offsetof_Requantize32_b_offset]\n" "add x22, x25, %[offsetof_Requantize32_c_offset]\n" - "ld1rb { z23.b }, p4/Z, [x21]\n" - "ld1rb { z15.b }, p4/Z, [x20]\n" + "ld1rb { z12.b }, p4/Z, [x21]\n" + "ld1rb { z30.b }, p4/Z, [x20]\n" "add x21, x25, %[offsetof_Requantize32_minval]\n" "add x20, x25, %[offsetof_Requantize32_maxval]\n" - "ld1rh { z14.h }, p4/Z, [x22]\n" - "ld1rh { z12.h }, p4/Z, [x21]\n" - "ld1rh { z11.h }, p4/Z, [x20]\n" - "ldp x15, x14, [x24, #0x0]\n" + "ld1rh { z24.h }, p4/Z, [x22]\n" + "ld1rh { z11.h }, p4/Z, [x21]\n" + "ld1rh { z26.h }, p4/Z, [x20]\n" + "ldp x13, x12, [x24, #0x0]\n" "incw x23\n" - "whilelt p3.h, x8, x17\n" - "ldp x13, x12, [x24, #0x10]\n" - "whilelt p2.s, x8, x17\n" - "whilelt p1.s, x23, x17\n" - "ldr x26, [%x[params], %[offsetof_Params_bias]]\n" - "ld1sb { z0.h }, p4/Z, [x16]\n" - "ld1sb { z1.h }, p4/Z, [x16, #1, MUL VL]\n" - "add x11, %x[params], %[offsetof_Params_inptrs]\n" - "mov x10, #0x0\n" - "ld1sb { z2.h }, p4/Z, [x16, #2, MUL VL]\n" - "ld1sb { z3.h }, p4/Z, [x16, #3, MUL VL]\n" - ".inst 0x454f1000 // ssublb z0.h, z0.b, z15.b\n" - ".inst 0x454f1021 // ssublb z1.h, z1.b, z15.b\n" - "ld1sb { z4.h }, p4/Z, [x16, #4, MUL VL]\n" - "ld1sb { z5.h }, p4/Z, [x16, #5, MUL VL]\n" - ".inst 0x454f1042 // ssublb z2.h, z2.b, z15.b\n" - ".inst 0x454f1063 // ssublb z3.h, z3.b, z15.b\n" - "ld1sb { z6.h }, p4/Z, [x16, #6, MUL VL]\n" - "ld1sb { z7.h }, p4/Z, [x16, #7, MUL VL]\n" - "inch x16, ALL, MUL #8\n" - ".inst 0x454f1084 // ssublb z4.h, z4.b, z15.b\n" - "ld1w { z17.s }, p2/Z, [x26]\n" - "ld1w { z16.s }, p1/Z, [x26, #1, MUL VL]\n" - "uzp1 z13.s, z17.s, z16.s\n" - "uzp2 z17.s, z17.s, z16.s\n" - "ld1sb { z8.h }, p4/Z, [x16]\n" - "ldp x24, x23, [x11, #0x0]\n" - "addvl x26, x26, #2\n" - "mov z26.d, z13.d\n" - "ldp x22, x21, [x11, #0x10]\n" - "ldr x20, [x11, #0x20]\n" - "mov z10.d, z17.d\n" - "mov z24.d, z13.d\n" - "ld1sb { z31.h }, p3/Z, [x24, x8]\n" - "ld1sb { z30.h }, p3/Z, [x23, x8]\n" - "mov z16.d, z17.d\n" - "mov z25.d, z13.d\n" - "ld1sb { z29.h }, p3/Z, [x22, x8]\n" - "ld1sb { z28.h }, p3/Z, [x21, x8]\n" - "mov z9.d, z17.d\n" - ".inst 0x454f10a5 // ssublb z5.h, z5.b, z15.b\n" - "ld1sb { z27.h }, p3/Z, [x20, x8]\n" - "ldr x9, [%x[params], %[offsetof_Params_requant_muls]]\n" - ".inst 0x454f10c6 // ssublb z6.h, z6.b, z15.b\n" - ".inst 0x454f10e7 // ssublb z7.h, z7.b, z15.b\n" - "ldr x28, [%x[params], %[offsetof_Params_requant_shifts]]\n" - "str x26, [%x[params], %[offsetof_Params_bias]]\n" - ".inst 0x454f1108 // ssublb z8.h, z8.b, z15.b\n" - ".inst 0x455713ff // ssublb z31.h, z31.b, z23.b\n" - ".inst 0x455713de // ssublb z30.h, z30.b, z23.b\n" - ".inst 0x455713bd // ssublb z29.h, z29.b, z23.b\n" - ".inst 0x4557139c // ssublb z28.h, z28.b, z23.b\n" - ".inst 0x4557137b // ssublb z27.h, z27.b, z23.b\n" + "whilelt p3.h, x16, x15\n" + "ldp x11, x10, [x24, #0x10]\n" + "whilelt p2.s, x16, x15\n" + "whilelt p1.s, x23, x15\n" + "ldr x9, [%x[params], %[offsetof_Params_bias]]\n" + "ld1sb { z14.h }, p4/Z, [x14]\n" + "ld1sb { z21.h }, p4/Z, [x14, #1, MUL VL]\n" + "add x28, %x[params], %[offsetof_Params_inptrs]\n" + "mov x27, #0x0\n" + "ld1sb { z1.h }, p4/Z, [x14, #2, MUL VL]\n" + "ld1sb { z6.h }, p4/Z, [x14, #3, MUL VL]\n" + ".inst 0x455e11ce // ssublb z14.h, z14.b, z30.b\n" + ".inst 0x455e12b5 // ssublb z21.h, z21.b, z30.b\n" + "ld1sb { z2.h }, p4/Z, [x14, #4, MUL VL]\n" + "ld1sb { z18.h }, p4/Z, [x14, #5, MUL VL]\n" + ".inst 0x455e1021 // ssublb z1.h, z1.b, z30.b\n" + ".inst 0x455e10c6 // ssublb z6.h, z6.b, z30.b\n" + "ld1sb { z7.h }, p4/Z, [x14, #6, MUL VL]\n" + "ld1sb { z10.h }, p4/Z, [x14, #7, MUL VL]\n" + "inch x14, ALL, MUL #8\n" + ".inst 0x455e1042 // ssublb z2.h, z2.b, z30.b\n" + "ld1w { z17.s }, p2/Z, [x9]\n" + "ld1w { z16.s }, p1/Z, [x9, #1, MUL VL]\n" + "uzp1 z5.s, z17.s, z16.s\n" + "uzp2 z9.s, z17.s, z16.s\n" + "ld1sb { z8.h }, p4/Z, [x14]\n" + "ldp x24, x23, [x28, #0x0]\n" + "addvl x9, x9, #2\n" + "mov z17.d, z5.d\n" + "ldp x22, x21, [x28, #0x10]\n" + "ldr x20, [x28, #0x20]\n" + "mov z25.d, z9.d\n" + "mov z16.d, z5.d\n" + "ld1sb { z0.h }, p3/Z, [x24, x16]\n" + "ld1sb { z29.h }, p3/Z, [x23, x16]\n" + "mov z23.d, z9.d\n" + "mov z22.d, z5.d\n" + "ld1sb { z4.h }, p3/Z, [x22, x16]\n" + "ld1sb { z13.h }, p3/Z, [x21, x16]\n" + "mov z27.d, z9.d\n" + ".inst 0x455e1252 // ssublb z18.h, z18.b, z30.b\n" + "ld1sb { z20.h }, p3/Z, [x20, x16]\n" + "ldr x26, [%x[params], %[offsetof_Params_requant_muls]]\n" + ".inst 0x455e10e7 // ssublb z7.h, z7.b, z30.b\n" + ".inst 0x455e114a // ssublb z10.h, z10.b, z30.b\n" + "ldr x25, [%x[params], %[offsetof_Params_requant_shifts]]\n" + "str x9, [%x[params], %[offsetof_Params_bias]]\n" + ".inst 0x455e1108 // ssublb z8.h, z8.b, z30.b\n" + ".inst 0x454c1000 // ssublb z0.h, z0.b, z12.b\n" + ".inst 0x454c13bd // ssublb z29.h, z29.b, z12.b\n" + ".inst 0x454c1084 // ssublb z4.h, z4.b, z12.b\n" + ".inst 0x454c11ad // ssublb z13.h, z13.b, z12.b\n" + ".inst 0x454c1294 // ssublb z20.h, z20.b, z12.b\n" "1:" // Loop - ".inst 0x448443ed // smlalb z13.s, p4/M, z31.h, z4.h\n" - ".inst 0x448447f1 // smlalt z17.s, p4/M, z31.h, z4.h\n" - "ldr x22, [x11, #0x28]\n" - "ldr x27, [x11, #0x38]\n" - ".inst 0x448343fa // smlalb z26.s, p4/M, z31.h, z3.h\n" - ".inst 0x448347ea // smlalt z10.s, p4/M, z31.h, z3.h\n" - "ldr x21, [x11, #0x30]\n" - "ldr x26, [x11, #0x40]\n" - ".inst 0x448043cd // smlalb z13.s, p4/M, z30.h, z0.h\n" - ".inst 0x448047d1 // smlalt z17.s, p4/M, z30.h, z0.h\n" - "ldr x20, [x11, #0x48]\n" - "ld1sb { z30.h }, p3/Z, [x20, x8]\n" - ".inst 0x448243ba // smlalb z26.s, p4/M, z29.h, z2.h\n" - ".inst 0x448247aa // smlalt z10.s, p4/M, z29.h, z2.h\n" - "ld1sb { z29.h }, p3/Z, [x21, x8]\n" - ".inst 0x455713bd // ssublb z29.h, z29.b, z23.b\n" - ".inst 0x448143f8 // smlalb z24.s, p4/M, z31.h, z1.h\n" - ".inst 0x448147f0 // smlalt z16.s, p4/M, z31.h, z1.h\n" - "ldr x25, [x11, #0x50]\n" - "ldr x24, [x11, #0x58]\n" - ".inst 0x448043f9 // smlalb z25.s, p4/M, z31.h, z0.h\n" - ".inst 0x448047e9 // smlalt z9.s, p4/M, z31.h, z0.h\n" - "ld1sb { z31.h }, p3/Z, [x22, x8]\n" - ".inst 0x455713ff // ssublb z31.h, z31.b, z23.b\n" - ".inst 0x4485438d // smlalb z13.s, p4/M, z28.h, z5.h\n" - ".inst 0x44854791 // smlalt z17.s, p4/M, z28.h, z5.h\n" - ".inst 0x455713de // ssublb z30.h, z30.b, z23.b\n" - "ldr x23, [x11, #0x60]\n" - ".inst 0x4484439a // smlalb z26.s, p4/M, z28.h, z4.h\n" - ".inst 0x4484478a // smlalt z10.s, p4/M, z28.h, z4.h\n" - "ldr x22, [x11, #0x68]\n" - "ldr x21, [x11, #0x70]\n" - ".inst 0x44824398 // smlalb z24.s, p4/M, z28.h, z2.h\n" - ".inst 0x44824790 // smlalt z16.s, p4/M, z28.h, z2.h\n" - "ldr x20, [x11, #0x78]\n" - "ld1w { z20.s }, p2/Z, [x9]\n" - ".inst 0x44814399 // smlalb z25.s, p4/M, z28.h, z1.h\n" - ".inst 0x44814789 // smlalt z9.s, p4/M, z28.h, z1.h\n" - "ld1sb { z28.h }, p3/Z, [x27, x8]\n" - ".inst 0x4557139c // ssublb z28.h, z28.b, z23.b\n" - ".inst 0x4487436d // smlalb z13.s, p4/M, z27.h, z7.h\n" - ".inst 0x44874771 // smlalt z17.s, p4/M, z27.h, z7.h\n" - "ld1w { z18.s }, p1/Z, [x9, #1, MUL VL]\n" - "uzp1 z19.s, z20.s, z18.s\n" - ".inst 0x4486437a // smlalb z26.s, p4/M, z27.h, z6.h\n" - ".inst 0x4486476a // smlalt z10.s, p4/M, z27.h, z6.h\n" - "uzp2 z22.s, z20.s, z18.s\n" - "ld1w { z20.s }, p2/Z, [x28]\n" - ".inst 0x448643f8 // smlalb z24.s, p4/M, z31.h, z6.h\n" - ".inst 0x448647f0 // smlalt z16.s, p4/M, z31.h, z6.h\n" - "ld1sb { z31.h }, p3/Z, [x26, x8]\n" - ".inst 0x455713ff // ssublb z31.h, z31.b, z23.b\n" - ".inst 0x44834379 // smlalb z25.s, p4/M, z27.h, z3.h\n" - ".inst 0x44834769 // smlalt z9.s, p4/M, z27.h, z3.h\n" - "whilelt p0.h, x10, x17\n" + ".inst 0x44824005 // smlalb z5.s, p4/M, z0.h, z2.h\n" + ".inst 0x44824409 // smlalt z9.s, p4/M, z0.h, z2.h\n" + "ldr x20, [x28, #0x28]\n" + "ldr x21, [x28, #0x38]\n" + ".inst 0x448e43a5 // smlalb z5.s, p4/M, z29.h, z14.h\n" + ".inst 0x44864011 // smlalb z17.s, p4/M, z0.h, z6.h\n" + "ld1sb { z3.h }, p3/Z, [x20, x16]\n" + "ldr x20, [x28, #0x30]\n" + ".inst 0x44954010 // smlalb z16.s, p4/M, z0.h, z21.h\n" + ".inst 0x448e4016 // smlalb z22.s, p4/M, z0.h, z14.h\n" + "ld1sb { z31.h }, p3/Z, [x21, x16]\n" + ".inst 0x454c1063 // ssublb z3.h, z3.b, z12.b\n" + ".inst 0x448e47a9 // smlalt z9.s, p4/M, z29.h, z14.h\n" + ".inst 0x449241a5 // smlalb z5.s, p4/M, z13.h, z18.h\n" + "ldr x21, [x28, #0x40]\n" + "ld1sb { z15.h }, p3/Z, [x20, x16]\n" + ".inst 0x44864419 // smlalt z25.s, p4/M, z0.h, z6.h\n" + ".inst 0x44954417 // smlalt z23.s, p4/M, z0.h, z21.h\n" + ".inst 0x454c13ff // ssublb z31.h, z31.b, z12.b\n" + "ldr x20, [x28, #0x48]\n" + ".inst 0x448e441b // smlalt z27.s, p4/M, z0.h, z14.h\n" + ".inst 0x44814091 // smlalb z17.s, p4/M, z4.h, z1.h\n" + "ld1sb { z19.h }, p3/Z, [x21, x16]\n" + ".inst 0x454c11ef // ssublb z15.h, z15.b, z12.b\n" + ".inst 0x448141b0 // smlalb z16.s, p4/M, z13.h, z1.h\n" + ".inst 0x449541b6 // smlalb z22.s, p4/M, z13.h, z21.h\n" + "ld1sb { z28.h }, p3/Z, [x20, x16]\n" + ".inst 0x454c1273 // ssublb z19.h, z19.b, z12.b\n" + ".inst 0x449245a9 // smlalt z9.s, p4/M, z13.h, z18.h\n" + ".inst 0x448a4285 // smlalb z5.s, p4/M, z20.h, z10.h\n" + "ldr x21, [x28, #0x50]\n" + "ldr x20, [x28, #0x58]\n" + ".inst 0x44814499 // smlalt z25.s, p4/M, z4.h, z1.h\n" + ".inst 0x448145b7 // smlalt z23.s, p4/M, z13.h, z1.h\n" + ".inst 0x454c139c // ssublb z28.h, z28.b, z12.b\n" + "ld1sb { z4.h }, p3/Z, [x21, x16]\n" + ".inst 0x449545bb // smlalt z27.s, p4/M, z13.h, z21.h\n" + ".inst 0x448241b1 // smlalb z17.s, p4/M, z13.h, z2.h\n" + "ld1sb { z29.h }, p3/Z, [x20, x16]\n" + "ldr x21, [x28, #0x60]\n" + ".inst 0x44874070 // smlalb z16.s, p4/M, z3.h, z7.h\n" + ".inst 0x44864296 // smlalb z22.s, p4/M, z20.h, z6.h\n" + "ldr x20, [x28, #0x68]\n" + ".inst 0x454c1084 // ssublb z4.h, z4.b, z12.b\n" + ".inst 0x448a4689 // smlalt z9.s, p4/M, z20.h, z10.h\n" + ".inst 0x449543e5 // smlalb z5.s, p4/M, z31.h, z21.h\n" + ".inst 0x454c13bd // ssublb z29.h, z29.b, z12.b\n" + "ld1sb { z0.h }, p3/Z, [x21, x16]\n" + ".inst 0x448245b9 // smlalt z25.s, p4/M, z13.h, z2.h\n" + ".inst 0x44874477 // smlalt z23.s, p4/M, z3.h, z7.h\n" + "ld1sb { z3.h }, p3/Z, [x20, x16]\n" + "ldr x20, [x28, #0x70]\n" + ".inst 0x4486469b // smlalt z27.s, p4/M, z20.h, z6.h\n" + ".inst 0x44874291 // smlalb z17.s, p4/M, z20.h, z7.h\n" + ".inst 0x454c1000 // ssublb z0.h, z0.b, z12.b\n" + "ld1sb { z13.h }, p3/Z, [x20, x16]\n" + ".inst 0x44824290 // smlalb z16.s, p4/M, z20.h, z2.h\n" + ".inst 0x448841f6 // smlalb z22.s, p4/M, z15.h, z8.h\n" + ".inst 0x454c1063 // ssublb z3.h, z3.b, z12.b\n" + "ldr x20, [x28, #0x78]\n" + ".inst 0x449547e9 // smlalt z9.s, p4/M, z31.h, z21.h\n" + ".inst 0x44814265 // smlalb z5.s, p4/M, z19.h, z1.h\n" + ".inst 0x454c11ad // ssublb z13.h, z13.b, z12.b\n" + "whilelt p0.h, x27, x15\n" + ".inst 0x44874699 // smlalt z25.s, p4/M, z20.h, z7.h\n" + ".inst 0x44824697 // smlalt z23.s, p4/M, z20.h, z2.h\n" + "ld1w { z20.s }, p2/Z, [x26]\n" + "inch x14\n" + ".inst 0x448845fb // smlalt z27.s, p4/M, z15.h, z8.h\n" + ".inst 0x448e43f1 // smlalb z17.s, p4/M, z31.h, z14.h\n" + "ld1w { z15.s }, p1/Z, [x26, #1, MUL VL]\n" + "ldr x21, [%x[params], %[offsetof_Params_bias]]\n" + ".inst 0x44924390 // smlalb z16.s, p4/M, z28.h, z18.h\n" + ".inst 0x44824396 // smlalb z22.s, p4/M, z28.h, z2.h\n" + "addvl x26, x26, #2\n" + ".inst 0x44814669 // smlalt z9.s, p4/M, z19.h, z1.h\n" + ".inst 0x44884385 // smlalb z5.s, p4/M, z28.h, z8.h\n" + ".inst 0x448e47f9 // smlalt z25.s, p4/M, z31.h, z14.h\n" + ".inst 0x44924797 // smlalt z23.s, p4/M, z28.h, z18.h\n" + "ld1sb { z31.h }, p3/Z, [x20, x16]\n" + ".inst 0x454c13ff // ssublb z31.h, z31.b, z12.b\n" + ".inst 0x4482479b // smlalt z27.s, p4/M, z28.h, z2.h\n" + ".inst 0x44954271 // smlalb z17.s, p4/M, z19.h, z21.h\n" + "uzp1 z2.s, z20.s, z15.s\n" "inch x16\n" - ".inst 0x4481438d // smlalb z13.s, p4/M, z28.h, z1.h\n" - ".inst 0x44814791 // smlalt z17.s, p4/M, z28.h, z1.h\n" - "ldr x26, [%x[params], %[offsetof_Params_bias]]\n" - "addvl x9, x9, #2\n" - ".inst 0x4480439a // smlalb z26.s, p4/M, z28.h, z0.h\n" - ".inst 0x4480478a // smlalt z10.s, p4/M, z28.h, z0.h\n" - "ld1sb { z28.h }, p3/Z, [x24, x8]\n" - ".inst 0x4557139c // ssublb z28.h, z28.b, z23.b\n" - ".inst 0x44844378 // smlalb z24.s, p4/M, z27.h, z4.h\n" - ".inst 0x448843b9 // smlalb z25.s, p4/M, z29.h, z8.h\n" - ".inst 0x44844770 // smlalt z16.s, p4/M, z27.h, z4.h\n" - ".inst 0x448847a9 // smlalt z9.s, p4/M, z29.h, z8.h\n" - "ld1sb { z29.h }, p3/Z, [x25, x8]\n" - ".inst 0x455713bd // ssublb z29.h, z29.b, z23.b\n" - ".inst 0x448243ed // smlalb z13.s, p4/M, z31.h, z2.h\n" - ".inst 0x448247f1 // smlalt z17.s, p4/M, z31.h, z2.h\n" - "ld1w { z18.s }, p1/Z, [x28, #1, MUL VL]\n" - "addvl x28, x28, #2\n" - ".inst 0x448143fa // smlalb z26.s, p4/M, z31.h, z1.h\n" - ".inst 0x448147ea // smlalt z10.s, p4/M, z31.h, z1.h\n" - "ld1sb { z31.h }, p3/Z, [x23, x8]\n" - ".inst 0x455713ff // ssublb z31.h, z31.b, z23.b\n" - ".inst 0x448543d8 // smlalb z24.s, p4/M, z30.h, z5.h\n" - ".inst 0x448443d9 // smlalb z25.s, p4/M, z30.h, z4.h\n" - "uzp1 z1.s, z20.s, z18.s\n" - ".inst 0x448843cd // smlalb z13.s, p4/M, z30.h, z8.h\n" - ".inst 0x448847d1 // smlalt z17.s, p4/M, z30.h, z8.h\n" - "uzp2 z27.s, z20.s, z18.s\n" - ".inst 0x448743da // smlalb z26.s, p4/M, z30.h, z7.h\n" - ".inst 0x448747ca // smlalt z10.s, p4/M, z30.h, z7.h\n" - ".inst 0x448547d0 // smlalt z16.s, p4/M, z30.h, z5.h\n" - ".inst 0x448447c9 // smlalt z9.s, p4/M, z30.h, z4.h\n" - "ld1sb { z30.h }, p3/Z, [x22, x8]\n" - ".inst 0x455713de // ssublb z30.h, z30.b, z23.b\n" - ".inst 0x448043b8 // smlalb z24.s, p4/M, z29.h, z0.h\n" - ".inst 0x44824399 // smlalb z25.s, p4/M, z28.h, z2.h\n" - ".inst 0x448343ad // smlalb z13.s, p4/M, z29.h, z3.h\n" - ".inst 0x448347b1 // smlalt z17.s, p4/M, z29.h, z3.h\n" - ".inst 0x448047b0 // smlalt z16.s, p4/M, z29.h, z0.h\n" - "ld1sb { z29.h }, p3/Z, [x21, x8]\n" - ".inst 0x44824789 // smlalt z9.s, p4/M, z28.h, z2.h\n" - ".inst 0x455713bd // ssublb z29.h, z29.b, z23.b\n" - ".inst 0x448343f8 // smlalb z24.s, p4/M, z31.h, z3.h\n" - ".inst 0x448543d9 // smlalb z25.s, p4/M, z30.h, z5.h\n" - ".inst 0x4485439a // smlalb z26.s, p4/M, z28.h, z5.h\n" - ".inst 0x4485478a // smlalt z10.s, p4/M, z28.h, z5.h\n" - "ld1sb { z28.h }, p3/Z, [x20, x8]\n" - ".inst 0x4557139c // ssublb z28.h, z28.b, z23.b\n" - ".inst 0x448643ed // smlalb z13.s, p4/M, z31.h, z6.h\n" - ".inst 0x448347f0 // smlalt z16.s, p4/M, z31.h, z3.h\n" - ".inst 0x04b375ad // sqrdmulh z13.s, z13.s, z19.s\n" - "inch x8\n" - ".inst 0x448547c9 // smlalt z9.s, p4/M, z30.h, z5.h\n" - ".inst 0x448743b8 // smlalb z24.s, p4/M, z29.h, z7.h\n" - "and z21.d, z13.d, z1.d\n" - "mov x20, x8\n" - ".inst 0x448643b9 // smlalb z25.s, p4/M, z29.h, z6.h\n" - ".inst 0x448647f1 // smlalt z17.s, p4/M, z31.h, z6.h\n" - ".inst 0x04b67631 // sqrdmulh z17.s, z17.s, z22.s\n" + ".inst 0x448e4090 // smlalb z16.s, p4/M, z4.h, z14.h\n" + ".inst 0x448143b6 // smlalb z22.s, p4/M, z29.h, z1.h\n" + "uzp2 z15.s, z20.s, z15.s\n" + "ld1w { z20.s }, p2/Z, [x25]\n" + ".inst 0x44884789 // smlalt z9.s, p4/M, z28.h, z8.h\n" + ".inst 0x44864085 // smlalb z5.s, p4/M, z4.h, z6.h\n" + "mov x20, x16\n" "incw x20\n" - ".inst 0x448747b0 // smlalt z16.s, p4/M, z29.h, z7.h\n" - ".inst 0x448647a9 // smlalt z9.s, p4/M, z29.h, z6.h\n" - "asr z21.s, z21.s, #0x1f\n" - "whilelt p2.s, x8, x17\n" - ".inst 0x448843da // smlalb z26.s, p4/M, z30.h, z8.h\n" - ".inst 0x44884398 // smlalb z24.s, p4/M, z28.h, z8.h\n" - "and z20.d, z17.d, z27.d\n" - "whilelt p1.s, x20, x17\n" - ".inst 0x44874399 // smlalb z25.s, p4/M, z28.h, z7.h\n" - ".inst 0x448847ca // smlalt z10.s, p4/M, z30.h, z8.h\n" - ".inst 0x04b3775a // sqrdmulh z26.s, z26.s, z19.s\n" - "whilelt p3.h, x8, x17\n" - ".inst 0x44884790 // smlalt z16.s, p4/M, z28.h, z8.h\n" - ".inst 0x44874789 // smlalt z9.s, p4/M, z28.h, z7.h\n" - ".inst 0x04b37718 // sqrdmulh z24.s, z24.s, z19.s\n" - ".inst 0x04b37739 // sqrdmulh z25.s, z25.s, z19.s\n" - "sqadd z13.s, z13.s, z21.s\n" - ".inst 0x4482902d // srshl z13.s, p4/M, z13.s, z1.s\n" - "asr z20.s, z20.s, #0x1f\n" - "and z19.d, z26.d, z1.d\n" - ".inst 0x04b6754a // sqrdmulh z10.s, z10.s, z22.s\n" - "and z18.d, z24.d, z1.d\n" - ".inst 0x04b67610 // sqrdmulh z16.s, z16.s, z22.s\n" - "and z21.d, z25.d, z1.d\n" - ".inst 0x04b67529 // sqrdmulh z9.s, z9.s, z22.s\n" - "sqadd z17.s, z17.s, z20.s\n" - ".inst 0x44829371 // srshl z17.s, p4/M, z17.s, z27.s\n" + ".inst 0x44954679 // smlalt z25.s, p4/M, z19.h, z21.h\n" + ".inst 0x448e4497 // smlalt z23.s, p4/M, z4.h, z14.h\n" + "ld1w { z19.s }, p1/Z, [x25, #1, MUL VL]\n" + "uzp1 z21.s, z20.s, z19.s\n" + ".inst 0x448147bb // smlalt z27.s, p4/M, z29.h, z1.h\n" + ".inst 0x448a4391 // smlalb z17.s, p4/M, z28.h, z10.h\n" + "uzp2 z1.s, z20.s, z19.s\n" + "whilelt p2.s, x16, x15\n" + ".inst 0x44864010 // smlalb z16.s, p4/M, z0.h, z6.h\n" + ".inst 0x44924076 // smlalb z22.s, p4/M, z3.h, z18.h\n" + "whilelt p1.s, x20, x15\n" + "whilelt p3.h, x16, x15\n" + ".inst 0x44864489 // smlalt z9.s, p4/M, z4.h, z6.h\n" + ".inst 0x44874005 // smlalb z5.s, p4/M, z0.h, z7.h\n" + ".inst 0x04a274a5 // sqrdmulh z5.s, z5.s, z2.s\n" + "addvl x25, x25, #2\n" + ".inst 0x448a4799 // smlalt z25.s, p4/M, z28.h, z10.h\n" + ".inst 0x44864417 // smlalt z23.s, p4/M, z0.h, z6.h\n" + "and z19.d, z5.d, z21.d\n" + ".inst 0x4492447b // smlalt z27.s, p4/M, z3.h, z18.h\n" + ".inst 0x449243b1 // smlalb z17.s, p4/M, z29.h, z18.h\n" "asr z19.s, z19.s, #0x1f\n" - "and z2.d, z10.d, z27.d\n" + ".inst 0x448a41b0 // smlalb z16.s, p4/M, z13.h, z10.h\n" + ".inst 0x448741b6 // smlalb z22.s, p4/M, z13.h, z7.h\n" + "sqadd z5.s, z5.s, z19.s\n" + ".inst 0x448292a5 // srshl z5.s, p4/M, z5.s, z21.s\n" + ".inst 0x44874409 // smlalt z9.s, p4/M, z0.h, z7.h\n" + ".inst 0x449247b9 // smlalt z25.s, p4/M, z29.h, z18.h\n" + ".inst 0x04af7529 // sqrdmulh z9.s, z9.s, z15.s\n" + ".inst 0x448a45b7 // smlalt z23.s, p4/M, z13.h, z10.h\n" + ".inst 0x448745bb // smlalt z27.s, p4/M, z13.h, z7.h\n" + "and z29.d, z9.d, z1.d\n" + ".inst 0x44884071 // smlalb z17.s, p4/M, z3.h, z8.h\n" + ".inst 0x448843f0 // smlalb z16.s, p4/M, z31.h, z8.h\n" + ".inst 0x04a27631 // sqrdmulh z17.s, z17.s, z2.s\n" + ".inst 0x448a43f6 // smlalb z22.s, p4/M, z31.h, z10.h\n" + ".inst 0x44884479 // smlalt z25.s, p4/M, z3.h, z8.h\n" + ".inst 0x04a27610 // sqrdmulh z16.s, z16.s, z2.s\n" + ".inst 0x448847f7 // smlalt z23.s, p4/M, z31.h, z8.h\n" + ".inst 0x448a47fb // smlalt z27.s, p4/M, z31.h, z10.h\n" + ".inst 0x04a276d6 // sqrdmulh z22.s, z22.s, z2.s\n" + "asr z29.s, z29.s, #0x1f\n" + "and z18.d, z17.d, z21.d\n" + ".inst 0x04af7739 // sqrdmulh z25.s, z25.s, z15.s\n" + "and z20.d, z16.d, z21.d\n" + ".inst 0x04af76f7 // sqrdmulh z23.s, z23.s, z15.s\n" + "and z19.d, z22.d, z21.d\n" + ".inst 0x04af777b // sqrdmulh z27.s, z27.s, z15.s\n" + "sqadd z9.s, z9.s, z29.s\n" + ".inst 0x44829029 // srshl z9.s, p4/M, z9.s, z1.s\n" "asr z18.s, z18.s, #0x1f\n" - "and z22.d, z16.d, z27.d\n" - "asr z21.s, z21.s, #0x1f\n" - "and z20.d, z9.d, z27.d\n" - "sqadd z26.s, z26.s, z19.s\n" - "asr z2.s, z2.s, #0x1f\n" - ".inst 0x4482903a // srshl z26.s, p4/M, z26.s, z1.s\n" - "sqadd z24.s, z24.s, z18.s\n" - "asr z22.s, z22.s, #0x1f\n" - ".inst 0x44829038 // srshl z24.s, p4/M, z24.s, z1.s\n" - "sqadd z25.s, z25.s, z21.s\n" + "and z7.d, z25.d, z1.d\n" "asr z20.s, z20.s, #0x1f\n" + "and z6.d, z23.d, z1.d\n" + "asr z19.s, z19.s, #0x1f\n" + "and z2.d, z27.d, z1.d\n" + "sqadd z17.s, z17.s, z18.s\n" + "asr z7.s, z7.s, #0x1f\n" + ".inst 0x448292b1 // srshl z17.s, p4/M, z17.s, z21.s\n" + "sqadd z16.s, z16.s, z20.s\n" + "asr z6.s, z6.s, #0x1f\n" + ".inst 0x448292b0 // srshl z16.s, p4/M, z16.s, z21.s\n" + "sqadd z22.s, z22.s, z19.s\n" + "asr z2.s, z2.s, #0x1f\n" + ".inst 0x448292b6 // srshl z22.s, p4/M, z22.s, z21.s\n" + "sqadd z25.s, z25.s, z7.s\n" + "sqadd z23.s, z23.s, z6.s\n" ".inst 0x44829039 // srshl z25.s, p4/M, z25.s, z1.s\n" - "sqadd z10.s, z10.s, z2.s\n" - "sqadd z16.s, z16.s, z22.s\n" - ".inst 0x4482936a // srshl z10.s, p4/M, z10.s, z27.s\n" - ".inst 0x44829370 // srshl z16.s, p4/M, z16.s, z27.s\n" - "sqadd z9.s, z9.s, z20.s\n" - ".inst 0x453041ad // sqxtnb z13.h, z13.s\n" - ".inst 0x44829369 // srshl z9.s, p4/M, z9.s, z27.s\n" - ".inst 0x4530435a // sqxtnb z26.h, z26.s\n" - ".inst 0x45304318 // sqxtnb z24.h, z24.s\n" - ".inst 0x45304339 // sqxtnb z25.h, z25.s\n" - ".inst 0x4530462d // sqxtnt z13.h, z17.s\n" - ".inst 0x4530455a // sqxtnt z26.h, z10.s\n" - ".inst 0x45304618 // sqxtnt z24.h, z16.s\n" - ".inst 0x45304539 // sqxtnt z25.h, z9.s\n" - "sqadd z13.h, z13.h, z14.h\n" - "smax z13.h, p4/M, z13.h, z12.h\n" - "smin z13.h, p4/M, z13.h, z11.h\n" - "sqadd z26.h, z26.h, z14.h\n" - "sqadd z24.h, z24.h, z14.h\n" - "smax z26.h, p4/M, z26.h, z12.h\n" - "smax z24.h, p4/M, z24.h, z12.h\n" - "sqadd z25.h, z25.h, z14.h\n" - "smax z25.h, p4/M, z25.h, z12.h\n" - "smin z26.h, p4/M, z26.h, z11.h\n" - "st1b { z13.h }, p0, [x15, x10]\n" - "smin z24.h, p4/M, z24.h, z11.h\n" - "smin z25.h, p4/M, z25.h, z11.h\n" - "st1b { z26.h }, p0, [x14, x10]\n" - "st1b { z24.h }, p0, [x13, x10]\n" - "st1b { z25.h }, p0, [x12, x10]\n" - "ld1sb { z0.h }, p4/Z, [x16]\n" - "ld1sb { z1.h }, p4/Z, [x16, #1, MUL VL]\n" - "inch x10\n" - "ld1sb { z2.h }, p4/Z, [x16, #2, MUL VL]\n" - "ld1sb { z3.h }, p4/Z, [x16, #3, MUL VL]\n" - ".inst 0x454f1000 // ssublb z0.h, z0.b, z15.b\n" - ".inst 0x454f1021 // ssublb z1.h, z1.b, z15.b\n" - "ld1sb { z4.h }, p4/Z, [x16, #4, MUL VL]\n" - "ld1sb { z5.h }, p4/Z, [x16, #5, MUL VL]\n" - ".inst 0x454f1042 // ssublb z2.h, z2.b, z15.b\n" - ".inst 0x454f1063 // ssublb z3.h, z3.b, z15.b\n" - "ld1sb { z6.h }, p4/Z, [x16, #6, MUL VL]\n" - "ld1sb { z7.h }, p4/Z, [x16, #7, MUL VL]\n" - "inch x16, ALL, MUL #8\n" - ".inst 0x454f1084 // ssublb z4.h, z4.b, z15.b\n" - "ld1w { z17.s }, p2/Z, [x26]\n" - "ld1w { z16.s }, p1/Z, [x26, #1, MUL VL]\n" - "uzp1 z13.s, z17.s, z16.s\n" - "uzp2 z17.s, z17.s, z16.s\n" - "ld1sb { z8.h }, p4/Z, [x16]\n" - "ldp x24, x23, [x11, #0x0]\n" - "addvl x26, x26, #2\n" - "str x26, [%x[params], %[offsetof_Params_bias]]\n" - "ldp x22, x21, [x11, #0x10]\n" - "ldr x20, [x11, #0x20]\n" - "mov z26.d, z13.d\n" - "mov z10.d, z17.d\n" - "ld1sb { z31.h }, p3/Z, [x24, x8]\n" - "ld1sb { z30.h }, p3/Z, [x23, x8]\n" - "mov z24.d, z13.d\n" - "mov z16.d, z17.d\n" - "ld1sb { z29.h }, p3/Z, [x22, x8]\n" - "ld1sb { z28.h }, p3/Z, [x21, x8]\n" - "mov z25.d, z13.d\n" - "mov z9.d, z17.d\n" - "ld1sb { z27.h }, p3/Z, [x20, x8]\n" - ".inst 0x454f10a5 // ssublb z5.h, z5.b, z15.b\n" - ".inst 0x454f10c6 // ssublb z6.h, z6.b, z15.b\n" - ".inst 0x454f10e7 // ssublb z7.h, z7.b, z15.b\n" - ".inst 0x454f1108 // ssublb z8.h, z8.b, z15.b\n" - ".inst 0x455713ff // ssublb z31.h, z31.b, z23.b\n" - ".inst 0x455713de // ssublb z30.h, z30.b, z23.b\n" - ".inst 0x455713bd // ssublb z29.h, z29.b, z23.b\n" - ".inst 0x4557139c // ssublb z28.h, z28.b, z23.b\n" - ".inst 0x4557137b // ssublb z27.h, z27.b, z23.b\n" + ".inst 0x44829037 // srshl z23.s, p4/M, z23.s, z1.s\n" + "sqadd z27.s, z27.s, z2.s\n" + ".inst 0x453040a5 // sqxtnb z5.h, z5.s\n" + ".inst 0x4482903b // srshl z27.s, p4/M, z27.s, z1.s\n" + ".inst 0x45304231 // sqxtnb z17.h, z17.s\n" + ".inst 0x45304210 // sqxtnb z16.h, z16.s\n" + ".inst 0x453042d6 // sqxtnb z22.h, z22.s\n" + ".inst 0x45304525 // sqxtnt z5.h, z9.s\n" + ".inst 0x45304731 // sqxtnt z17.h, z25.s\n" + ".inst 0x453046f0 // sqxtnt z16.h, z23.s\n" + ".inst 0x45304776 // sqxtnt z22.h, z27.s\n" + "sqadd z5.h, z5.h, z24.h\n" + "smax z5.h, p4/M, z5.h, z11.h\n" + "smin z5.h, p4/M, z5.h, z26.h\n" + "sqadd z17.h, z17.h, z24.h\n" + "sqadd z16.h, z16.h, z24.h\n" + "smax z17.h, p4/M, z17.h, z11.h\n" + "smax z16.h, p4/M, z16.h, z11.h\n" + "sqadd z22.h, z22.h, z24.h\n" + "smax z22.h, p4/M, z22.h, z11.h\n" + "smin z17.h, p4/M, z17.h, z26.h\n" + "st1b { z5.h }, p0, [x13, x27]\n" + "smin z16.h, p4/M, z16.h, z26.h\n" + "smin z22.h, p4/M, z22.h, z26.h\n" + "st1b { z17.h }, p0, [x12, x27]\n" + "st1b { z16.h }, p0, [x11, x27]\n" + "st1b { z22.h }, p0, [x10, x27]\n" + "ld1sb { z14.h }, p4/Z, [x14]\n" + "ld1sb { z21.h }, p4/Z, [x14, #1, MUL VL]\n" + "inch x27\n" + "ld1sb { z1.h }, p4/Z, [x14, #2, MUL VL]\n" + "ld1sb { z6.h }, p4/Z, [x14, #3, MUL VL]\n" + ".inst 0x455e11ce // ssublb z14.h, z14.b, z30.b\n" + ".inst 0x455e12b5 // ssublb z21.h, z21.b, z30.b\n" + "ld1sb { z2.h }, p4/Z, [x14, #4, MUL VL]\n" + "ld1sb { z18.h }, p4/Z, [x14, #5, MUL VL]\n" + ".inst 0x455e1021 // ssublb z1.h, z1.b, z30.b\n" + ".inst 0x455e10c6 // ssublb z6.h, z6.b, z30.b\n" + "ld1sb { z7.h }, p4/Z, [x14, #6, MUL VL]\n" + "ld1sb { z10.h }, p4/Z, [x14, #7, MUL VL]\n" + "inch x14, ALL, MUL #8\n" + ".inst 0x455e1042 // ssublb z2.h, z2.b, z30.b\n" + "ld1w { z17.s }, p2/Z, [x21]\n" + "ld1w { z16.s }, p1/Z, [x21, #1, MUL VL]\n" + "uzp1 z5.s, z17.s, z16.s\n" + "uzp2 z9.s, z17.s, z16.s\n" + "ld1sb { z8.h }, p4/Z, [x14]\n" + "ldp x24, x23, [x28, #0x0]\n" + "addvl x21, x21, #2\n" + "str x21, [%x[params], %[offsetof_Params_bias]]\n" + "ldp x22, x21, [x28, #0x10]\n" + "ldr x20, [x28, #0x20]\n" + "mov z17.d, z5.d\n" + "mov z25.d, z9.d\n" + "ld1sb { z0.h }, p3/Z, [x24, x16]\n" + "ld1sb { z29.h }, p3/Z, [x23, x16]\n" + "mov z16.d, z5.d\n" + "mov z23.d, z9.d\n" + "ld1sb { z4.h }, p3/Z, [x22, x16]\n" + "ld1sb { z13.h }, p3/Z, [x21, x16]\n" + "mov z22.d, z5.d\n" + "mov z27.d, z9.d\n" + "ld1sb { z20.h }, p3/Z, [x20, x16]\n" + ".inst 0x455e1252 // ssublb z18.h, z18.b, z30.b\n" + ".inst 0x455e10e7 // ssublb z7.h, z7.b, z30.b\n" + ".inst 0x455e114a // ssublb z10.h, z10.b, z30.b\n" + ".inst 0x455e1108 // ssublb z8.h, z8.b, z30.b\n" + ".inst 0x454c1000 // ssublb z0.h, z0.b, z12.b\n" + ".inst 0x454c13bd // ssublb z29.h, z29.b, z12.b\n" + ".inst 0x454c1084 // ssublb z4.h, z4.b, z12.b\n" + ".inst 0x454c11ad // ssublb z13.h, z13.b, z12.b\n" + ".inst 0x454c1294 // ssublb z20.h, z20.b, z12.b\n" "b.any 1b\n" : : [offsetof_Params_bias] "I" (offsetof(Params, bias)), [offsetof_Params_inptrs] "I" (offsetof(Params, inptrs)), [offsetof_Params_n_channels] "I" (offsetof(Params, n_channels)), [offsetof_Params_outptrs] "I" (offsetof(Params, outptrs)), [offsetof_Params_requant] "I" (offsetof(Params, requant)), [offsetof_Params_requant_muls] "I" (offsetof(Params, requant_muls)), [offsetof_Params_requant_shifts] "I" (offsetof(Params, requant_shifts)), [offsetof_Params_weights] "I" (offsetof(Params, weights)), [offsetof_Requantize32_a_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [params] "r" (¶ms) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp index 78bcd1407fa61dc30859ef688e5c710e7bfc69d0..7a9b8a5bde7d82a480925871a1a9fe3e44b3ee8f 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,14 +22,14 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include #pragma once -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -47,17 +47,16 @@ class sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst : public DepthwiseDepthfirstS constexpr static unsigned int stride_rows = 2; constexpr static unsigned int stride_cols = 2; - arm_gemm::VLType get_vl_type(void) const override { return arm_gemm::VLType::SVE; } - unsigned int get_accumulator_depth_vl(void) const override { return 2; } - sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst(const CPUInfo *) : Parent(2, 2, 3, 3, 2, 2) {} - Parent::KernelType kernel = sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl; + arm_gemm::VLType get_vl_type(void) const override { return arm_gemm::VLType::SVE; } + Parent::KernelType kernel = sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl; Parent::KernelType get_kernel(void) const override { return kernel; } + unsigned int get_accumulator_depth_vl(void) const override { return 2; } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp index 357c9f8399717d2dc2e78ec923e3863d28ce04cc..fc9a48bb46f5e29d8825fe5f3e25631919776012 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp @@ -27,7 +27,7 @@ #include #include -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -46,7 +46,7 @@ void sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( struct Params { long unsigned int n_channels; - const int8_t *weights; + const void *weights; const int32_t *bias; const arm_gemm::Requantize32 *requant; const int32_t *const requant_muls; @@ -57,7 +57,7 @@ void sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( Params( long unsigned int n_channels, const int8_t *const *inptrs_raw, - const int8_t *const weights, + const void *const weights, const int32_t *const bias, const arm_gemm::Requantize32 &qp, const int32_t *const requant_muls, @@ -110,13 +110,13 @@ void sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ldr x17, [%x[params], %[offsetof_Params_weights]]\n" "add x20, x25, %[offsetof_Requantize32_b_offset]\n" "add x22, x25, %[offsetof_Requantize32_c_offset]\n" - "ld1rb { z23.b }, p4/Z, [x21]\n" - "ld1rb { z12.b }, p4/Z, [x20]\n" + "ld1rb { z26.b }, p4/Z, [x21]\n" + "ld1rb { z13.b }, p4/Z, [x20]\n" "add x21, x25, %[offsetof_Requantize32_minval]\n" "add x20, x25, %[offsetof_Requantize32_maxval]\n" - "ld1rh { z14.h }, p4/Z, [x22]\n" - "ld1rh { z16.h }, p4/Z, [x21]\n" - "ld1rh { z15.h }, p4/Z, [x20]\n" + "ld1rh { z19.h }, p4/Z, [x22]\n" + "ld1rh { z12.h }, p4/Z, [x21]\n" + "ld1rh { z9.h }, p4/Z, [x20]\n" "ldp x16, x15, [x24, #0x0]\n" "incw x23\n" "whilelt p3.h, x7, x8\n" @@ -124,320 +124,320 @@ void sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "whilelt p2.s, x7, x8\n" "whilelt p1.s, x23, x8\n" "ldr x12, [%x[params], %[offsetof_Params_bias]]\n" - "ld1sb { z0.h }, p4/Z, [x17]\n" - "ld1sb { z1.h }, p4/Z, [x17, #1, MUL VL]\n" + "ld1sb { z25.h }, p4/Z, [x17]\n" + "ld1sb { z30.h }, p4/Z, [x17, #1, MUL VL]\n" "add x11, %x[params], %[offsetof_Params_inptrs]\n" "mov x10, #0x0\n" - "ld1sb { z2.h }, p4/Z, [x17, #2, MUL VL]\n" - "ld1sb { z3.h }, p4/Z, [x17, #3, MUL VL]\n" - ".inst 0x454c1000 // ssublb z0.h, z0.b, z12.b\n" - ".inst 0x454c1021 // ssublb z1.h, z1.b, z12.b\n" - "ld1sb { z4.h }, p4/Z, [x17, #4, MUL VL]\n" - "ld1sb { z5.h }, p4/Z, [x17, #5, MUL VL]\n" - ".inst 0x454c1042 // ssublb z2.h, z2.b, z12.b\n" - ".inst 0x454c1063 // ssublb z3.h, z3.b, z12.b\n" - "ld1sb { z6.h }, p4/Z, [x17, #6, MUL VL]\n" + "ld1sb { z14.h }, p4/Z, [x17, #2, MUL VL]\n" + "ld1sb { z4.h }, p4/Z, [x17, #3, MUL VL]\n" + ".inst 0x454d1339 // ssublb z25.h, z25.b, z13.b\n" + ".inst 0x454d13de // ssublb z30.h, z30.b, z13.b\n" + "ld1sb { z10.h }, p4/Z, [x17, #4, MUL VL]\n" + "ld1sb { z3.h }, p4/Z, [x17, #5, MUL VL]\n" + ".inst 0x454d11ce // ssublb z14.h, z14.b, z13.b\n" + ".inst 0x454d1084 // ssublb z4.h, z4.b, z13.b\n" + "ld1sb { z23.h }, p4/Z, [x17, #6, MUL VL]\n" "ld1sb { z7.h }, p4/Z, [x17, #7, MUL VL]\n" "inch x17, ALL, MUL #8\n" - ".inst 0x454c1084 // ssublb z4.h, z4.b, z12.b\n" - "ld1w { z18.s }, p2/Z, [x12]\n" - "ld1w { z8.s }, p1/Z, [x12, #1, MUL VL]\n" - "uzp1 z13.s, z18.s, z8.s\n" - "uzp2 z17.s, z18.s, z8.s\n" - "ld1sb { z8.h }, p4/Z, [x17]\n" - "ldp x9, x28, [x11, #0x0]\n" + ".inst 0x454d114a // ssublb z10.h, z10.b, z13.b\n" + "ld1w { z17.s }, p2/Z, [x12]\n" + "ld1w { z16.s }, p1/Z, [x12, #1, MUL VL]\n" + "uzp1 z8.s, z17.s, z16.s\n" + "uzp2 z24.s, z17.s, z16.s\n" + "ld1sb { z2.h }, p4/Z, [x17]\n" + "ldp x27, x26, [x11, #0x0]\n" "addvl x12, x12, #2\n" - "mov z9.d, z13.d\n" + "mov z18.d, z8.d\n" "ldp x25, x24, [x11, #0x10]\n" "ldp x23, x22, [x11, #0x20]\n" - "mov z10.d, z17.d\n" - "mov z11.d, z13.d\n" + "mov z0.d, z24.d\n" + "mov z15.d, z8.d\n" "ldp x21, x20, [x11, #0x30]\n" - "ld1sb { z31.h }, p3/Z, [x9, x7]\n" - "mov z22.d, z17.d\n" - "mov z21.d, z13.d\n" - "ld1sb { z30.h }, p3/Z, [x28, x7]\n" - "ld1sb { z29.h }, p3/Z, [x25, x7]\n" - "mov z18.d, z17.d\n" - ".inst 0x454c10a5 // ssublb z5.h, z5.b, z12.b\n" - "ld1sb { z28.h }, p3/Z, [x24, x7]\n" + "ld1sb { z21.h }, p3/Z, [x27, x7]\n" + "mov z1.d, z24.d\n" + "mov z5.d, z8.d\n" + "ld1sb { z22.h }, p3/Z, [x26, x7]\n" + "ld1sb { z11.h }, p3/Z, [x25, x7]\n" + "mov z6.d, z24.d\n" + ".inst 0x454d1063 // ssublb z3.h, z3.b, z13.b\n" + "ld1sb { z20.h }, p3/Z, [x24, x7]\n" "ld1sb { z27.h }, p3/Z, [x23, x7]\n" - ".inst 0x454c10c6 // ssublb z6.h, z6.b, z12.b\n" - ".inst 0x454c10e7 // ssublb z7.h, z7.b, z12.b\n" - "ld1sb { z26.h }, p3/Z, [x22, x7]\n" - "ld1sb { z25.h }, p3/Z, [x21, x7]\n" - ".inst 0x454c1108 // ssublb z8.h, z8.b, z12.b\n" - ".inst 0x455713ff // ssublb z31.h, z31.b, z23.b\n" - "ld1sb { z24.h }, p3/Z, [x20, x7]\n" - "ldr x27, [%x[params], %[offsetof_Params_requant_muls]]\n" - ".inst 0x455713de // ssublb z30.h, z30.b, z23.b\n" - ".inst 0x455713bd // ssublb z29.h, z29.b, z23.b\n" - "ldr x26, [%x[params], %[offsetof_Params_requant_shifts]]\n" + ".inst 0x454d12f7 // ssublb z23.h, z23.b, z13.b\n" + ".inst 0x454d10e7 // ssublb z7.h, z7.b, z13.b\n" + "ld1sb { z28.h }, p3/Z, [x22, x7]\n" + "ld1sb { z16.h }, p3/Z, [x21, x7]\n" + ".inst 0x454d1042 // ssublb z2.h, z2.b, z13.b\n" + ".inst 0x455a12b5 // ssublb z21.h, z21.b, z26.b\n" + "ld1sb { z31.h }, p3/Z, [x20, x7]\n" + "ldr x9, [%x[params], %[offsetof_Params_requant_muls]]\n" + ".inst 0x455a12d6 // ssublb z22.h, z22.b, z26.b\n" + ".inst 0x455a116b // ssublb z11.h, z11.b, z26.b\n" + "ldr x28, [%x[params], %[offsetof_Params_requant_shifts]]\n" "str x12, [%x[params], %[offsetof_Params_bias]]\n" - ".inst 0x4557139c // ssublb z28.h, z28.b, z23.b\n" - ".inst 0x4557137b // ssublb z27.h, z27.b, z23.b\n" - ".inst 0x4557135a // ssublb z26.h, z26.b, z23.b\n" - ".inst 0x45571339 // ssublb z25.h, z25.b, z23.b\n" - ".inst 0x45571318 // ssublb z24.h, z24.b, z23.b\n" + ".inst 0x455a1294 // ssublb z20.h, z20.b, z26.b\n" + ".inst 0x455a137b // ssublb z27.h, z27.b, z26.b\n" + ".inst 0x455a139c // ssublb z28.h, z28.b, z26.b\n" + ".inst 0x455a1210 // ssublb z16.h, z16.b, z26.b\n" + ".inst 0x455a13ff // ssublb z31.h, z31.b, z26.b\n" "1:" // Loop - ".inst 0x448843ed // smlalb z13.s, p4/M, z31.h, z8.h\n" - ".inst 0x448847f1 // smlalt z17.s, p4/M, z31.h, z8.h\n" - "ldr x25, [x11, #0x40]\n" - "ldr x24, [x11, #0x48]\n" - ".inst 0x448643e9 // smlalb z9.s, p4/M, z31.h, z6.h\n" - ".inst 0x448647ea // smlalt z10.s, p4/M, z31.h, z6.h\n" - "ldr x22, [x11, #0x50]\n" - "ldr x20, [x11, #0x58]\n" - ".inst 0x448043cd // smlalb z13.s, p4/M, z30.h, z0.h\n" - ".inst 0x448047d1 // smlalt z17.s, p4/M, z30.h, z0.h\n" - "ldr x23, [x11, #0x78]\n" - "ldr x21, [x11, #0x60]\n" - ".inst 0x44814389 // smlalb z9.s, p4/M, z28.h, z1.h\n" - ".inst 0x4481478a // smlalt z10.s, p4/M, z28.h, z1.h\n" - "ld1sb { z28.h }, p3/Z, [x24, x7]\n" - ".inst 0x4557139c // ssublb z28.h, z28.b, z23.b\n" - ".inst 0x448143ad // smlalb z13.s, p4/M, z29.h, z1.h\n" - ".inst 0x448147b1 // smlalt z17.s, p4/M, z29.h, z1.h\n" - "ld1sb { z29.h }, p3/Z, [x25, x7]\n" - ".inst 0x455713bd // ssublb z29.h, z29.b, z23.b\n" - ".inst 0x44824369 // smlalb z9.s, p4/M, z27.h, z2.h\n" - ".inst 0x4482476a // smlalt z10.s, p4/M, z27.h, z2.h\n" - "ld1sb { z27.h }, p3/Z, [x22, x7]\n" - ".inst 0x4557137b // ssublb z27.h, z27.b, z23.b\n" - ".inst 0x4483434d // smlalb z13.s, p4/M, z26.h, z3.h\n" - ".inst 0x44834751 // smlalt z17.s, p4/M, z26.h, z3.h\n" - "ld1sb { z26.h }, p3/Z, [x20, x7]\n" - ".inst 0x4557135a // ssublb z26.h, z26.b, z23.b\n" - ".inst 0x44804309 // smlalb z9.s, p4/M, z24.h, z0.h\n" - ".inst 0x4480470a // smlalt z10.s, p4/M, z24.h, z0.h\n" - "ldr x22, [x11, #0x80]\n" - "ldr x20, [x11, #0x68]\n" - ".inst 0x4484432d // smlalb z13.s, p4/M, z25.h, z4.h\n" - ".inst 0x44844731 // smlalt z17.s, p4/M, z25.h, z4.h\n" - "ld1sb { z25.h }, p3/Z, [x21, x7]\n" - ".inst 0x45571339 // ssublb z25.h, z25.b, z23.b\n" - ".inst 0x448443a9 // smlalb z9.s, p4/M, z29.h, z4.h\n" - ".inst 0x448447aa // smlalt z10.s, p4/M, z29.h, z4.h\n" - "ldr x21, [x11, #0x88]\n" + ".inst 0x448242a8 // smlalb z8.s, p4/M, z21.h, z2.h\n" + "ldr x21, [x11, #0x58]\n" + "ldr x20, [x11, #0x78]\n" + ".inst 0x448246b8 // smlalt z24.s, p4/M, z21.h, z2.h\n" + ".inst 0x449942c8 // smlalb z8.s, p4/M, z22.h, z25.h\n" + "ld1sb { z17.h }, p3/Z, [x21, x7]\n" "ld1sb { z29.h }, p3/Z, [x20, x7]\n" - ".inst 0x4482430d // smlalb z13.s, p4/M, z24.h, z2.h\n" - ".inst 0x44824711 // smlalt z17.s, p4/M, z24.h, z2.h\n" - "ldr x20, [x11, #0x70]\n" - ".inst 0x455713bd // ssublb z29.h, z29.b, z23.b\n" - ".inst 0x44854389 // smlalb z9.s, p4/M, z28.h, z5.h\n" - ".inst 0x4485478a // smlalt z10.s, p4/M, z28.h, z5.h\n" - "ld1sb { z28.h }, p3/Z, [x22, x7]\n" - ".inst 0x4557139c // ssublb z28.h, z28.b, z23.b\n" - ".inst 0x448243eb // smlalb z11.s, p4/M, z31.h, z2.h\n" - ".inst 0x448247f6 // smlalt z22.s, p4/M, z31.h, z2.h\n" - "ldr x25, [x11, #0x98]\n" - "ld1sb { z24.h }, p3/Z, [x20, x7]\n" - ".inst 0x4485436d // smlalb z13.s, p4/M, z27.h, z5.h\n" - ".inst 0x44854771 // smlalt z17.s, p4/M, z27.h, z5.h\n" - ".inst 0x45571318 // ssublb z24.h, z24.b, z23.b\n" - "ldr x24, [x11, #0x90]\n" - ".inst 0x44834369 // smlalb z9.s, p4/M, z27.h, z3.h\n" - ".inst 0x4483476a // smlalt z10.s, p4/M, z27.h, z3.h\n" + ".inst 0x449742b2 // smlalb z18.s, p4/M, z21.h, z23.h\n" + "ldr x21, [x11, #0x60]\n" + "ldr x20, [x11, #0x80]\n" + ".inst 0x448e42af // smlalb z15.s, p4/M, z21.h, z14.h\n" + ".inst 0x449942a5 // smlalb z5.s, p4/M, z21.h, z25.h\n" + ".inst 0x449946d8 // smlalt z24.s, p4/M, z22.h, z25.h\n" + ".inst 0x455a1231 // ssublb z17.h, z17.b, z26.b\n" + ".inst 0x449e4168 // smlalb z8.s, p4/M, z11.h, z30.h\n" + "ld1sb { z22.h }, p3/Z, [x21, x7]\n" + ".inst 0x455a13bd // ssublb z29.h, z29.b, z26.b\n" + ".inst 0x449746a0 // smlalt z0.s, p4/M, z21.h, z23.h\n" + ".inst 0x448e46a1 // smlalt z1.s, p4/M, z21.h, z14.h\n" + "ldr x21, [x11, #0x68]\n" + ".inst 0x449946a6 // smlalt z6.s, p4/M, z21.h, z25.h\n" + "ld1sb { z21.h }, p3/Z, [x20, x7]\n" + "ldr x20, [x11, #0x88]\n" + ".inst 0x449e4292 // smlalb z18.s, p4/M, z20.h, z30.h\n" + ".inst 0x4484422f // smlalb z15.s, p4/M, z17.h, z4.h\n" + ".inst 0x448a43a5 // smlalb z5.s, p4/M, z29.h, z10.h\n" + ".inst 0x455a12d6 // ssublb z22.h, z22.b, z26.b\n" + "ldr x22, [x11, #0x40]\n" + ".inst 0x449e4578 // smlalt z24.s, p4/M, z11.h, z30.h\n" + ".inst 0x455a12b5 // ssublb z21.h, z21.b, z26.b\n" + ".inst 0x44844388 // smlalb z8.s, p4/M, z28.h, z4.h\n" + "ld1sb { z11.h }, p3/Z, [x21, x7]\n" + ".inst 0x449e4680 // smlalt z0.s, p4/M, z20.h, z30.h\n" + "ld1sb { z20.h }, p3/Z, [x20, x7]\n" + ".inst 0x44844621 // smlalt z1.s, p4/M, z17.h, z4.h\n" + "ldr x21, [x11, #0x70]\n" + ".inst 0x448a47a6 // smlalt z6.s, p4/M, z29.h, z10.h\n" + "ldr x20, [x11, #0x98]\n" + ".inst 0x448e4372 // smlalb z18.s, p4/M, z27.h, z14.h\n" + "ldr x23, [x11, #0x50]\n" + ".inst 0x449942cf // smlalb z15.s, p4/M, z22.h, z25.h\n" + ".inst 0x449e42a5 // smlalb z5.s, p4/M, z21.h, z30.h\n" + ".inst 0x455a116b // ssublb z11.h, z11.b, z26.b\n" + "ld1sb { z17.h }, p3/Z, [x22, x7]\n" + ".inst 0x44844798 // smlalt z24.s, p4/M, z28.h, z4.h\n" + ".inst 0x455a1294 // ssublb z20.h, z20.b, z26.b\n" + ".inst 0x448a4208 // smlalb z8.s, p4/M, z16.h, z10.h\n" + "ld1sb { z29.h }, p3/Z, [x21, x7]\n" + "ld1sb { z28.h }, p3/Z, [x20, x7]\n" + ".inst 0x448e4760 // smlalt z0.s, p4/M, z27.h, z14.h\n" + "ldr x22, [x11, #0x48]\n" + ".inst 0x449946c1 // smlalt z1.s, p4/M, z22.h, z25.h\n" + ".inst 0x449e46a6 // smlalt z6.s, p4/M, z21.h, z30.h\n" + "ldr x21, [x11, #0x90]\n" + "ldr x20, [x11, #0xa8]\n" + ".inst 0x449943f2 // smlalb z18.s, p4/M, z31.h, z25.h\n" "ld1sb { z27.h }, p3/Z, [x23, x7]\n" - ".inst 0x4557137b // ssublb z27.h, z27.b, z23.b\n" - ".inst 0x448043f5 // smlalb z21.s, p4/M, z31.h, z0.h\n" - ".inst 0x4483434b // smlalb z11.s, p4/M, z26.h, z3.h\n" - "ldr x23, [x11, #0xa8]\n" - "ldr x20, [x11, #0xa0]\n" - ".inst 0x44834756 // smlalt z22.s, p4/M, z26.h, z3.h\n" - ".inst 0x448047f2 // smlalt z18.s, p4/M, z31.h, z0.h\n" - "ld1sb { z26.h }, p3/Z, [x21, x7]\n" - ".inst 0x4557135a // ssublb z26.h, z26.b, z23.b\n" - ".inst 0x44844375 // smlalb z21.s, p4/M, z27.h, z4.h\n" - ".inst 0x4480432b // smlalb z11.s, p4/M, z25.h, z0.h\n" - "ldr x22, [x11, #0xb0]\n" - "ldr x21, [x11, #0xb8]\n" - ".inst 0x44804736 // smlalt z22.s, p4/M, z25.h, z0.h\n" - ".inst 0x44844772 // smlalt z18.s, p4/M, z27.h, z4.h\n" - "ld1sb { z27.h }, p3/Z, [x20, x7]\n" - ".inst 0x4557137b // ssublb z27.h, z27.b, z23.b\n" - ".inst 0x44814395 // smlalb z21.s, p4/M, z28.h, z1.h\n" - ".inst 0x4486432d // smlalb z13.s, p4/M, z25.h, z6.h\n" - "ldr x20, [x11, #0xc0]\n" - "ld1w { z31.s }, p2/Z, [x27]\n" - ".inst 0x44864731 // smlalt z17.s, p4/M, z25.h, z6.h\n" - ".inst 0x448443ab // smlalb z11.s, p4/M, z29.h, z4.h\n" - "ld1sb { z25.h }, p3/Z, [x24, x7]\n" - ".inst 0x45571339 // ssublb z25.h, z25.b, z23.b\n" - ".inst 0x448447b6 // smlalt z22.s, p4/M, z29.h, z4.h\n" - "ld1sb { z29.h }, p3/Z, [x25, x7]\n" - ".inst 0x44814792 // smlalt z18.s, p4/M, z28.h, z1.h\n" - ".inst 0x455713bd // ssublb z29.h, z29.b, z23.b\n" - ".inst 0x44854355 // smlalb z21.s, p4/M, z26.h, z5.h\n" - ".inst 0x4487430d // smlalb z13.s, p4/M, z24.h, z7.h\n" - "ld1w { z20.s }, p1/Z, [x27, #1, MUL VL]\n" - "uzp1 z19.s, z31.s, z20.s\n" - ".inst 0x44874711 // smlalt z17.s, p4/M, z24.h, z7.h\n" - ".inst 0x4481430b // smlalb z11.s, p4/M, z24.h, z1.h\n" - "uzp2 z30.s, z31.s, z20.s\n" - "ld1w { z31.s }, p2/Z, [x26]\n" - ".inst 0x44814716 // smlalt z22.s, p4/M, z24.h, z1.h\n" - "ld1sb { z24.h }, p3/Z, [x23, x7]\n" - ".inst 0x44854752 // smlalt z18.s, p4/M, z26.h, z5.h\n" - ".inst 0x45571318 // ssublb z24.h, z24.b, z23.b\n" - ".inst 0x448243b5 // smlalb z21.s, p4/M, z29.h, z2.h\n" - "ld1sb { z26.h }, p3/Z, [x22, x7]\n" - ".inst 0x448247b2 // smlalt z18.s, p4/M, z29.h, z2.h\n" - ".inst 0x4557135a // ssublb z26.h, z26.b, z23.b\n" - ".inst 0x4486432b // smlalb z11.s, p4/M, z25.h, z6.h\n" - ".inst 0x44834315 // smlalb z21.s, p4/M, z24.h, z3.h\n" - "ld1w { z20.s }, p1/Z, [x26, #1, MUL VL]\n" - "uzp1 z1.s, z31.s, z20.s\n" - ".inst 0x44874389 // smlalb z9.s, p4/M, z28.h, z7.h\n" - ".inst 0x4487478a // smlalt z10.s, p4/M, z28.h, z7.h\n" - ".inst 0x04b375ad // sqrdmulh z13.s, z13.s, z19.s\n" - "whilelt p0.h, x10, x8\n" - ".inst 0x44864736 // smlalt z22.s, p4/M, z25.h, z6.h\n" + ".inst 0x448a416f // smlalb z15.s, p4/M, z11.h, z10.h\n" + ".inst 0x44834285 // smlalb z5.s, p4/M, z20.h, z3.h\n" + ".inst 0x455a1231 // ssublb z17.h, z17.b, z26.b\n" + ".inst 0x448a4618 // smlalt z24.s, p4/M, z16.h, z10.h\n" + ".inst 0x455a13bd // ssublb z29.h, z29.b, z26.b\n" + ".inst 0x448e43e8 // smlalb z8.s, p4/M, z31.h, z14.h\n" + "ld1sb { z16.h }, p3/Z, [x22, x7]\n" + ".inst 0x455a139c // ssublb z28.h, z28.b, z26.b\n" + ".inst 0x449947e0 // smlalt z0.s, p4/M, z31.h, z25.h\n" "ld1sb { z25.h }, p3/Z, [x21, x7]\n" - ".inst 0x44834712 // smlalt z18.s, p4/M, z24.h, z3.h\n" - ".inst 0x45571339 // ssublb z25.h, z25.b, z23.b\n" - ".inst 0x4487436b // smlalb z11.s, p4/M, z27.h, z7.h\n" - ".inst 0x44874355 // smlalb z21.s, p4/M, z26.h, z7.h\n" - "uzp2 z31.s, z31.s, z20.s\n" - "inch x17\n" - ".inst 0x448843a9 // smlalb z9.s, p4/M, z29.h, z8.h\n" - ".inst 0x448847aa // smlalt z10.s, p4/M, z29.h, z8.h\n" - "ld1sb { z29.h }, p3/Z, [x20, x7]\n" - ".inst 0x455713bd // ssublb z29.h, z29.b, z23.b\n" - ".inst 0x44874776 // smlalt z22.s, p4/M, z27.h, z7.h\n" - ".inst 0x44874752 // smlalt z18.s, p4/M, z26.h, z7.h\n" - "and z0.d, z13.d, z1.d\n" + ".inst 0x448a4561 // smlalt z1.s, p4/M, z11.h, z10.h\n" + "ld1sb { z11.h }, p3/Z, [x20, x7]\n" + ".inst 0x455a137b // ssublb z27.h, z27.b, z26.b\n" + ".inst 0x44834686 // smlalt z6.s, p4/M, z20.h, z3.h\n" + "ldr x21, [x11, #0xa0]\n" + "ldr x20, [x11, #0xb0]\n" + ".inst 0x448a4232 // smlalb z18.s, p4/M, z17.h, z10.h\n" + ".inst 0x449e43af // smlalb z15.s, p4/M, z29.h, z30.h\n" + ".inst 0x455a1210 // ssublb z16.h, z16.b, z26.b\n" + ".inst 0x448e4385 // smlalb z5.s, p4/M, z28.h, z14.h\n" + ".inst 0x448e47f8 // smlalt z24.s, p4/M, z31.h, z14.h\n" + ".inst 0x455a1339 // ssublb z25.h, z25.b, z26.b\n" + "ld1sb { z20.h }, p3/Z, [x21, x7]\n" + ".inst 0x455a116b // ssublb z11.h, z11.b, z26.b\n" + ".inst 0x44834368 // smlalb z8.s, p4/M, z27.h, z3.h\n" + "ld1sb { z31.h }, p3/Z, [x20, x7]\n" + ".inst 0x448a4620 // smlalt z0.s, p4/M, z17.h, z10.h\n" + ".inst 0x449e47a1 // smlalt z1.s, p4/M, z29.h, z30.h\n" + ".inst 0x448e4786 // smlalt z6.s, p4/M, z28.h, z14.h\n" + "ldr x20, [x11, #0xb8]\n" + ".inst 0x455a1294 // ssublb z20.h, z20.b, z26.b\n" + ".inst 0x44834212 // smlalb z18.s, p4/M, z16.h, z3.h\n" + ".inst 0x4497432f // smlalb z15.s, p4/M, z25.h, z23.h\n" + ".inst 0x455a13ff // ssublb z31.h, z31.b, z26.b\n" + "ld1sb { z30.h }, p3/Z, [x20, x7]\n" + ".inst 0x44844165 // smlalb z5.s, p4/M, z11.h, z4.h\n" + ".inst 0x44834778 // smlalt z24.s, p4/M, z27.h, z3.h\n" + "ldr x20, [x11, #0xc0]\n" + "ld1w { z17.s }, p2/Z, [x9]\n" + ".inst 0x449742c8 // smlalb z8.s, p4/M, z22.h, z23.h\n" + ".inst 0x44834600 // smlalt z0.s, p4/M, z16.h, z3.h\n" + "ld1w { z14.s }, p1/Z, [x9, #1, MUL VL]\n" + ".inst 0x455a13de // ssublb z30.h, z30.b, z26.b\n" + ".inst 0x44974721 // smlalt z1.s, p4/M, z25.h, z23.h\n" + ".inst 0x44844566 // smlalt z6.s, p4/M, z11.h, z4.h\n" + "ld1sb { z25.h }, p3/Z, [x20, x7]\n" + "uzp1 z10.s, z17.s, z14.s\n" + ".inst 0x44844372 // smlalb z18.s, p4/M, z27.h, z4.h\n" + ".inst 0x4487428f // smlalb z15.s, p4/M, z20.h, z7.h\n" + "uzp2 z14.s, z17.s, z14.s\n" + "ld1w { z17.s }, p2/Z, [x28]\n" + ".inst 0x448743e5 // smlalb z5.s, p4/M, z31.h, z7.h\n" + ".inst 0x449746d8 // smlalt z24.s, p4/M, z22.h, z23.h\n" + "ld1w { z16.s }, p1/Z, [x28, #1, MUL VL]\n" + ".inst 0x455a1339 // ssublb z25.h, z25.b, z26.b\n" + ".inst 0x448743a8 // smlalb z8.s, p4/M, z29.h, z7.h\n" + ".inst 0x44844760 // smlalt z0.s, p4/M, z27.h, z4.h\n" + "uzp1 z4.s, z17.s, z16.s\n" "inch x7\n" - ".inst 0x4485430b // smlalb z11.s, p4/M, z24.h, z5.h\n" - ".inst 0x44864335 // smlalb z21.s, p4/M, z25.h, z6.h\n" - ".inst 0x04be7631 // sqrdmulh z17.s, z17.s, z30.s\n" + ".inst 0x44874681 // smlalt z1.s, p4/M, z20.h, z7.h\n" + ".inst 0x448747e6 // smlalt z6.s, p4/M, z31.h, z7.h\n" + ".inst 0x04aa7508 // sqrdmulh z8.s, z8.s, z10.s\n" + "whilelt p0.h, x10, x8\n" + ".inst 0x448742b2 // smlalb z18.s, p4/M, z21.h, z7.h\n" + ".inst 0x4483416f // smlalb z15.s, p4/M, z11.h, z3.h\n" + "uzp2 z22.s, z17.s, z16.s\n" "mov x20, x7\n" - ".inst 0x44854716 // smlalt z22.s, p4/M, z24.h, z5.h\n" - ".inst 0x44864732 // smlalt z18.s, p4/M, z25.h, z6.h\n" - "asr z0.s, z0.s, #0x1f\n" + ".inst 0x449743c5 // smlalb z5.s, p4/M, z30.h, z23.h\n" + ".inst 0x448747b8 // smlalt z24.s, p4/M, z29.h, z7.h\n" + "and z17.d, z8.d, z4.d\n" + "inch x17\n" + ".inst 0x448746a0 // smlalt z0.s, p4/M, z21.h, z7.h\n" + ".inst 0x44834561 // smlalt z1.s, p4/M, z11.h, z3.h\n" + ".inst 0x04ae7718 // sqrdmulh z24.s, z24.s, z14.s\n" "incw x20\n" - ".inst 0x4488432b // smlalb z11.s, p4/M, z25.h, z8.h\n" - ".inst 0x448843b5 // smlalb z21.s, p4/M, z29.h, z8.h\n" - "and z20.d, z17.d, z31.d\n" + ".inst 0x449747c6 // smlalt z6.s, p4/M, z30.h, z23.h\n" + ".inst 0x44824392 // smlalb z18.s, p4/M, z28.h, z2.h\n" + "asr z17.s, z17.s, #0x1f\n" "whilelt p2.s, x7, x8\n" - ".inst 0x44884736 // smlalt z22.s, p4/M, z25.h, z8.h\n" - ".inst 0x448847b2 // smlalt z18.s, p4/M, z29.h, z8.h\n" - ".inst 0x04b37529 // sqrdmulh z9.s, z9.s, z19.s\n" + ".inst 0x448243cf // smlalb z15.s, p4/M, z30.h, z2.h\n" + ".inst 0x44824325 // smlalb z5.s, p4/M, z25.h, z2.h\n" + "and z16.d, z24.d, z22.d\n" "whilelt p1.s, x20, x8\n" - ".inst 0x04b3756b // sqrdmulh z11.s, z11.s, z19.s\n" - ".inst 0x04b376b5 // sqrdmulh z21.s, z21.s, z19.s\n" - "ldr x12, [%x[params], %[offsetof_Params_bias]]\n" + ".inst 0x44824780 // smlalt z0.s, p4/M, z28.h, z2.h\n" + ".inst 0x448247c1 // smlalt z1.s, p4/M, z30.h, z2.h\n" + ".inst 0x04aa7652 // sqrdmulh z18.s, z18.s, z10.s\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + ".inst 0x44824726 // smlalt z6.s, p4/M, z25.h, z2.h\n" + ".inst 0x04aa75ef // sqrdmulh z15.s, z15.s, z10.s\n" "whilelt p3.h, x7, x8\n" - "sqadd z13.s, z13.s, z0.s\n" - "asr z20.s, z20.s, #0x1f\n" - ".inst 0x4482902d // srshl z13.s, p4/M, z13.s, z1.s\n" - "addvl x27, x27, #2\n" - "and z19.d, z9.d, z1.d\n" - ".inst 0x04be754a // sqrdmulh z10.s, z10.s, z30.s\n" - "addvl x26, x26, #2\n" - "and z2.d, z11.d, z1.d\n" - ".inst 0x04be76d6 // sqrdmulh z22.s, z22.s, z30.s\n" - "and z0.d, z21.d, z1.d\n" - ".inst 0x04be7652 // sqrdmulh z18.s, z18.s, z30.s\n" - "sqadd z17.s, z17.s, z20.s\n" - "asr z19.s, z19.s, #0x1f\n" - ".inst 0x448293f1 // srshl z17.s, p4/M, z17.s, z31.s\n" - "and z3.d, z10.d, z31.d\n" - "asr z2.s, z2.s, #0x1f\n" - "and z26.d, z22.d, z31.d\n" - "asr z0.s, z0.s, #0x1f\n" - "and z20.d, z18.d, z31.d\n" - "sqadd z9.s, z9.s, z19.s\n" - ".inst 0x44829029 // srshl z9.s, p4/M, z9.s, z1.s\n" - "asr z3.s, z3.s, #0x1f\n" - "sqadd z11.s, z11.s, z2.s\n" - ".inst 0x4482902b // srshl z11.s, p4/M, z11.s, z1.s\n" - "asr z26.s, z26.s, #0x1f\n" - "sqadd z21.s, z21.s, z0.s\n" - ".inst 0x44829035 // srshl z21.s, p4/M, z21.s, z1.s\n" + "addvl x9, x9, #2\n" + ".inst 0x04aa74a5 // sqrdmulh z5.s, z5.s, z10.s\n" + "sqadd z8.s, z8.s, z17.s\n" + ".inst 0x44829088 // srshl z8.s, p4/M, z8.s, z4.s\n" + "addvl x28, x28, #2\n" + "asr z16.s, z16.s, #0x1f\n" + "and z21.d, z18.d, z4.d\n" + ".inst 0x04ae7400 // sqrdmulh z0.s, z0.s, z14.s\n" + "and z20.d, z15.d, z4.d\n" + ".inst 0x04ae7421 // sqrdmulh z1.s, z1.s, z14.s\n" + "and z28.d, z5.d, z4.d\n" + ".inst 0x04ae74c6 // sqrdmulh z6.s, z6.s, z14.s\n" + "sqadd z24.s, z24.s, z16.s\n" + ".inst 0x448292d8 // srshl z24.s, p4/M, z24.s, z22.s\n" + "asr z21.s, z21.s, #0x1f\n" + "and z25.d, z0.d, z22.d\n" "asr z20.s, z20.s, #0x1f\n" - "sqadd z10.s, z10.s, z3.s\n" - ".inst 0x448293ea // srshl z10.s, p4/M, z10.s, z31.s\n" - "sqadd z22.s, z22.s, z26.s\n" - "sqadd z18.s, z18.s, z20.s\n" - ".inst 0x448293f6 // srshl z22.s, p4/M, z22.s, z31.s\n" - ".inst 0x448293f2 // srshl z18.s, p4/M, z18.s, z31.s\n" - ".inst 0x453041ad // sqxtnb z13.h, z13.s\n" - ".inst 0x45304129 // sqxtnb z9.h, z9.s\n" - ".inst 0x4530416b // sqxtnb z11.h, z11.s\n" - ".inst 0x453042b5 // sqxtnb z21.h, z21.s\n" - ".inst 0x4530462d // sqxtnt z13.h, z17.s\n" - ".inst 0x45304549 // sqxtnt z9.h, z10.s\n" - ".inst 0x453046cb // sqxtnt z11.h, z22.s\n" - ".inst 0x45304655 // sqxtnt z21.h, z18.s\n" - "sqadd z13.h, z13.h, z14.h\n" - "sqadd z9.h, z9.h, z14.h\n" - "smax z13.h, p4/M, z13.h, z16.h\n" - "smax z9.h, p4/M, z9.h, z16.h\n" - "sqadd z11.h, z11.h, z14.h\n" - "sqadd z21.h, z21.h, z14.h\n" - "smax z11.h, p4/M, z11.h, z16.h\n" - "smax z21.h, p4/M, z21.h, z16.h\n" - "smin z13.h, p4/M, z13.h, z15.h\n" - "smin z9.h, p4/M, z9.h, z15.h\n" - "st1b { z13.h }, p0, [x16, x10]\n" - "smin z11.h, p4/M, z11.h, z15.h\n" - "smin z21.h, p4/M, z21.h, z15.h\n" - "st1b { z9.h }, p0, [x15, x10]\n" - "st1b { z11.h }, p0, [x14, x10]\n" - "st1b { z21.h }, p0, [x13, x10]\n" - "ld1sb { z0.h }, p4/Z, [x17]\n" - "ld1sb { z1.h }, p4/Z, [x17, #1, MUL VL]\n" + "and z17.d, z1.d, z22.d\n" + "asr z28.s, z28.s, #0x1f\n" + "and z16.d, z6.d, z22.d\n" + "sqadd z18.s, z18.s, z21.s\n" + "asr z25.s, z25.s, #0x1f\n" + ".inst 0x44829092 // srshl z18.s, p4/M, z18.s, z4.s\n" + "sqadd z15.s, z15.s, z20.s\n" + "asr z17.s, z17.s, #0x1f\n" + ".inst 0x4482908f // srshl z15.s, p4/M, z15.s, z4.s\n" + "sqadd z5.s, z5.s, z28.s\n" + "asr z16.s, z16.s, #0x1f\n" + ".inst 0x44829085 // srshl z5.s, p4/M, z5.s, z4.s\n" + "sqadd z0.s, z0.s, z25.s\n" + "sqadd z1.s, z1.s, z17.s\n" + ".inst 0x448292c0 // srshl z0.s, p4/M, z0.s, z22.s\n" + ".inst 0x448292c1 // srshl z1.s, p4/M, z1.s, z22.s\n" + "sqadd z6.s, z6.s, z16.s\n" + ".inst 0x45304108 // sqxtnb z8.h, z8.s\n" + ".inst 0x448292c6 // srshl z6.s, p4/M, z6.s, z22.s\n" + ".inst 0x45304252 // sqxtnb z18.h, z18.s\n" + ".inst 0x453041ef // sqxtnb z15.h, z15.s\n" + ".inst 0x453040a5 // sqxtnb z5.h, z5.s\n" + ".inst 0x45304708 // sqxtnt z8.h, z24.s\n" + ".inst 0x45304412 // sqxtnt z18.h, z0.s\n" + ".inst 0x4530442f // sqxtnt z15.h, z1.s\n" + ".inst 0x453044c5 // sqxtnt z5.h, z6.s\n" + "sqadd z8.h, z8.h, z19.h\n" + "smax z8.h, p4/M, z8.h, z12.h\n" + "smin z8.h, p4/M, z8.h, z9.h\n" + "sqadd z18.h, z18.h, z19.h\n" + "sqadd z15.h, z15.h, z19.h\n" + "smax z18.h, p4/M, z18.h, z12.h\n" + "smax z15.h, p4/M, z15.h, z12.h\n" + "sqadd z5.h, z5.h, z19.h\n" + "smax z5.h, p4/M, z5.h, z12.h\n" + "smin z18.h, p4/M, z18.h, z9.h\n" + "st1b { z8.h }, p0, [x16, x10]\n" + "smin z15.h, p4/M, z15.h, z9.h\n" + "smin z5.h, p4/M, z5.h, z9.h\n" + "st1b { z18.h }, p0, [x15, x10]\n" + "st1b { z15.h }, p0, [x14, x10]\n" + "st1b { z5.h }, p0, [x13, x10]\n" + "ld1sb { z25.h }, p4/Z, [x17]\n" + "ld1sb { z30.h }, p4/Z, [x17, #1, MUL VL]\n" "inch x10\n" - "ld1sb { z2.h }, p4/Z, [x17, #2, MUL VL]\n" - "ld1sb { z3.h }, p4/Z, [x17, #3, MUL VL]\n" - ".inst 0x454c1000 // ssublb z0.h, z0.b, z12.b\n" - ".inst 0x454c1021 // ssublb z1.h, z1.b, z12.b\n" - "ld1sb { z4.h }, p4/Z, [x17, #4, MUL VL]\n" - "ld1sb { z5.h }, p4/Z, [x17, #5, MUL VL]\n" - ".inst 0x454c1042 // ssublb z2.h, z2.b, z12.b\n" - ".inst 0x454c1063 // ssublb z3.h, z3.b, z12.b\n" - "ld1sb { z6.h }, p4/Z, [x17, #6, MUL VL]\n" + "ld1sb { z14.h }, p4/Z, [x17, #2, MUL VL]\n" + "ld1sb { z4.h }, p4/Z, [x17, #3, MUL VL]\n" + ".inst 0x454d1339 // ssublb z25.h, z25.b, z13.b\n" + ".inst 0x454d13de // ssublb z30.h, z30.b, z13.b\n" + "ld1sb { z10.h }, p4/Z, [x17, #4, MUL VL]\n" + "ld1sb { z3.h }, p4/Z, [x17, #5, MUL VL]\n" + ".inst 0x454d11ce // ssublb z14.h, z14.b, z13.b\n" + ".inst 0x454d1084 // ssublb z4.h, z4.b, z13.b\n" + "ld1sb { z23.h }, p4/Z, [x17, #6, MUL VL]\n" "ld1sb { z7.h }, p4/Z, [x17, #7, MUL VL]\n" "inch x17, ALL, MUL #8\n" - ".inst 0x454c1084 // ssublb z4.h, z4.b, z12.b\n" - "ld1w { z18.s }, p2/Z, [x12]\n" - "ld1w { z8.s }, p1/Z, [x12, #1, MUL VL]\n" - "uzp1 z13.s, z18.s, z8.s\n" - "uzp2 z17.s, z18.s, z8.s\n" - "ld1sb { z8.h }, p4/Z, [x17]\n" - "ldp x9, x28, [x11, #0x0]\n" - "addvl x12, x12, #2\n" - "str x12, [%x[params], %[offsetof_Params_bias]]\n" + ".inst 0x454d114a // ssublb z10.h, z10.b, z13.b\n" + "ld1w { z17.s }, p2/Z, [x20]\n" + "ld1w { z16.s }, p1/Z, [x20, #1, MUL VL]\n" + "uzp1 z8.s, z17.s, z16.s\n" + "uzp2 z24.s, z17.s, z16.s\n" + "ld1sb { z2.h }, p4/Z, [x17]\n" + "ldp x27, x26, [x11, #0x0]\n" + "addvl x20, x20, #2\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" "ldp x25, x24, [x11, #0x10]\n" "ldp x23, x22, [x11, #0x20]\n" - "mov z9.d, z13.d\n" - "mov z10.d, z17.d\n" + "mov z18.d, z8.d\n" + "mov z0.d, z24.d\n" "ldp x21, x20, [x11, #0x30]\n" - "ld1sb { z31.h }, p3/Z, [x9, x7]\n" - "mov z11.d, z13.d\n" - "mov z22.d, z17.d\n" - "ld1sb { z30.h }, p3/Z, [x28, x7]\n" - "ld1sb { z29.h }, p3/Z, [x25, x7]\n" - "mov z21.d, z13.d\n" - "mov z18.d, z17.d\n" - "ld1sb { z28.h }, p3/Z, [x24, x7]\n" + "ld1sb { z21.h }, p3/Z, [x27, x7]\n" + "mov z15.d, z8.d\n" + "mov z1.d, z24.d\n" + "ld1sb { z22.h }, p3/Z, [x26, x7]\n" + "ld1sb { z11.h }, p3/Z, [x25, x7]\n" + "mov z5.d, z8.d\n" + "mov z6.d, z24.d\n" + "ld1sb { z20.h }, p3/Z, [x24, x7]\n" "ld1sb { z27.h }, p3/Z, [x23, x7]\n" - ".inst 0x454c10a5 // ssublb z5.h, z5.b, z12.b\n" - ".inst 0x454c10c6 // ssublb z6.h, z6.b, z12.b\n" - "ld1sb { z26.h }, p3/Z, [x22, x7]\n" - "ld1sb { z25.h }, p3/Z, [x21, x7]\n" - ".inst 0x454c10e7 // ssublb z7.h, z7.b, z12.b\n" - ".inst 0x454c1108 // ssublb z8.h, z8.b, z12.b\n" - "ld1sb { z24.h }, p3/Z, [x20, x7]\n" - ".inst 0x455713ff // ssublb z31.h, z31.b, z23.b\n" - ".inst 0x455713de // ssublb z30.h, z30.b, z23.b\n" - ".inst 0x455713bd // ssublb z29.h, z29.b, z23.b\n" - ".inst 0x4557139c // ssublb z28.h, z28.b, z23.b\n" - ".inst 0x4557137b // ssublb z27.h, z27.b, z23.b\n" - ".inst 0x4557135a // ssublb z26.h, z26.b, z23.b\n" - ".inst 0x45571339 // ssublb z25.h, z25.b, z23.b\n" - ".inst 0x45571318 // ssublb z24.h, z24.b, z23.b\n" + ".inst 0x454d1063 // ssublb z3.h, z3.b, z13.b\n" + ".inst 0x454d12f7 // ssublb z23.h, z23.b, z13.b\n" + "ld1sb { z28.h }, p3/Z, [x22, x7]\n" + "ld1sb { z16.h }, p3/Z, [x21, x7]\n" + ".inst 0x454d10e7 // ssublb z7.h, z7.b, z13.b\n" + ".inst 0x454d1042 // ssublb z2.h, z2.b, z13.b\n" + "ld1sb { z31.h }, p3/Z, [x20, x7]\n" + ".inst 0x455a12b5 // ssublb z21.h, z21.b, z26.b\n" + ".inst 0x455a12d6 // ssublb z22.h, z22.b, z26.b\n" + ".inst 0x455a116b // ssublb z11.h, z11.b, z26.b\n" + ".inst 0x455a1294 // ssublb z20.h, z20.b, z26.b\n" + ".inst 0x455a137b // ssublb z27.h, z27.b, z26.b\n" + ".inst 0x455a139c // ssublb z28.h, z28.b, z26.b\n" + ".inst 0x455a1210 // ssublb z16.h, z16.b, z26.b\n" + ".inst 0x455a13ff // ssublb z31.h, z31.b, z26.b\n" "b.any 1b\n" : : [offsetof_Params_bias] "I" (offsetof(Params, bias)), [offsetof_Params_inptrs] "I" (offsetof(Params, inptrs)), [offsetof_Params_n_channels] "I" (offsetof(Params, n_channels)), [offsetof_Params_outptrs] "I" (offsetof(Params, outptrs)), [offsetof_Params_requant] "I" (offsetof(Params, requant)), [offsetof_Params_requant_muls] "I" (offsetof(Params, requant_muls)), [offsetof_Params_requant_shifts] "I" (offsetof(Params, requant_shifts)), [offsetof_Params_weights] "I" (offsetof(Params, weights)), [offsetof_Requantize32_a_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [params] "r" (¶ms) @@ -448,4 +448,4 @@ void sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp index 41ecd520ae2e25ded59223a84ada3ff5d61f73f5..1f8d6c5213358ce628897fa9e63b8716c7a19839 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,14 +22,14 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include #pragma once -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -47,17 +47,16 @@ class sve_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirstS constexpr static unsigned int stride_rows = 1; constexpr static unsigned int stride_cols = 1; - arm_gemm::VLType get_vl_type(void) const override { return arm_gemm::VLType::SVE; } - unsigned int get_accumulator_depth_vl(void) const override { return 2; } - sve_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst(const CPUInfo *) : Parent(2, 2, 5, 5, 1, 1) {} - Parent::KernelType kernel = sve_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl; + arm_gemm::VLType get_vl_type(void) const override { return arm_gemm::VLType::SVE; } + Parent::KernelType kernel = sve_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl; Parent::KernelType get_kernel(void) const override { return kernel; } + unsigned int get_accumulator_depth_vl(void) const override { return 2; } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp index d8f4d8d199ac4212b6ca4bb2863547202ed8128d..7ff724ddd8a5cfb15bb57b216922b3e42359c01d 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp @@ -27,7 +27,7 @@ #include #include -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -46,7 +46,7 @@ void sve_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( struct Params { long unsigned int n_channels; - const int8_t *weights; + const void *weights; const int32_t *bias; const arm_gemm::Requantize32 *requant; const int32_t *const requant_muls; @@ -57,7 +57,7 @@ void sve_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( Params( long unsigned int n_channels, const int8_t *const *inptrs_raw, - const int8_t *const weights, + const void *const weights, const int32_t *const bias, const arm_gemm::Requantize32 &qp, const int32_t *const requant_muls, @@ -111,542 +111,542 @@ void sve_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( requant_muls, requant_shifts, outptrs); __asm__ __volatile__( - "mov x0, #0x0\n" - "mov x24, x0\n" + "mov x2, #0x0\n" + "mov x24, x2\n" "ldr x23, [%x[params], %[offsetof_Params_requant]]\n" - "ldr x1, [%x[params], %[offsetof_Params_n_channels]]\n" + "ldr x3, [%x[params], %[offsetof_Params_n_channels]]\n" "ptrue p4.b\n" "ldr x22, [%x[params], %[offsetof_Params_outptrs]]\n" "incw x24\n" - "ldr x2, [%x[params], %[offsetof_Params_weights]]\n" + "ldr x4, [%x[params], %[offsetof_Params_weights]]\n" "add x21, x23, %[offsetof_Requantize32_a_offset]\n" "add x20, x23, %[offsetof_Requantize32_b_offset]\n" - "ld1rb { z15.b }, p4/Z, [x21]\n" - "ld1rb { z17.b }, p4/Z, [x20]\n" + "ld1rb { z30.b }, p4/Z, [x21]\n" + "ld1rb { z10.b }, p4/Z, [x20]\n" "add x21, x23, %[offsetof_Requantize32_c_offset]\n" "add x20, x23, %[offsetof_Requantize32_minval]\n" - "ld1rh { z12.h }, p4/Z, [x21]\n" - "ld1rh { z13.h }, p4/Z, [x20]\n" + "ld1rh { z15.h }, p4/Z, [x21]\n" + "ld1rh { z12.h }, p4/Z, [x20]\n" "add x20, x23, %[offsetof_Requantize32_maxval]\n" - "ld1rh { z11.h }, p4/Z, [x20]\n" - "ldp x3, x4, [x22, #0x0]\n" - "whilelt p3.h, x0, x1\n" - "ldp x5, x6, [x22, #0x10]\n" - "whilelt p2.s, x0, x1\n" - "whilelt p1.s, x24, x1\n" - "ldr x14, [%x[params], %[offsetof_Params_bias]]\n" - "add x7, %x[params], %[offsetof_Params_inptrs]\n" - "ld1w { z30.s }, p2/Z, [x14]\n" - "ld1w { z16.s }, p1/Z, [x14, #1, MUL VL]\n" - "uzp1 z14.s, z30.s, z16.s\n" - "ld1sb { z0.h }, p4/Z, [x2]\n" - "ld1sb { z1.h }, p4/Z, [x2, #1, MUL VL]\n" - "uzp2 z10.s, z30.s, z16.s\n" - "addvl x14, x14, #2\n" - "ld1sb { z2.h }, p4/Z, [x2, #2, MUL VL]\n" - "ld1sb { z3.h }, p4/Z, [x2, #3, MUL VL]\n" - "mov x8, #0x0\n" - "mov z20.d, z14.d\n" - "ld1sb { z4.h }, p4/Z, [x2, #4, MUL VL]\n" - "ldp x9, x28, [x7, #0x0]\n" - "mov z7.d, z10.d\n" - "mov z8.d, z14.d\n" - "ldp x27, x26, [x7, #0x10]\n" - "ldp x25, x24, [x7, #0x20]\n" - "mov z16.d, z10.d\n" + "ld1rh { z13.h }, p4/Z, [x20]\n" + "ldp x5, x6, [x22, #0x0]\n" + "whilelt p3.h, x2, x3\n" + "ldp x7, x8, [x22, #0x10]\n" + "whilelt p2.s, x2, x3\n" + "whilelt p1.s, x24, x3\n" + "ldr x10, [%x[params], %[offsetof_Params_bias]]\n" + "add x17, %x[params], %[offsetof_Params_inptrs]\n" + "ld1w { z17.s }, p2/Z, [x10]\n" + "ld1w { z16.s }, p1/Z, [x10, #1, MUL VL]\n" + "uzp1 z14.s, z17.s, z16.s\n" + "ld1sb { z26.h }, p4/Z, [x4]\n" + "ld1sb { z8.h }, p4/Z, [x4, #1, MUL VL]\n" + "uzp2 z23.s, z17.s, z16.s\n" + "addvl x10, x10, #2\n" + "ld1sb { z16.h }, p4/Z, [x4, #2, MUL VL]\n" + "ld1sb { z21.h }, p4/Z, [x4, #3, MUL VL]\n" + "mov x16, #0x0\n" "mov z6.d, z14.d\n" - "ldp x23, x22, [x7, #0x30]\n" - "ldp x21, x20, [x7, #0x40]\n" - "mov z5.d, z10.d\n" - ".inst 0x45511000 // ssublb z0.h, z0.b, z17.b\n" - "ld1sb { z31.h }, p3/Z, [x9, x0]\n" - "ld1sb { z30.h }, p3/Z, [x28, x0]\n" - ".inst 0x45511021 // ssublb z1.h, z1.b, z17.b\n" - ".inst 0x45511042 // ssublb z2.h, z2.b, z17.b\n" - "ld1sb { z29.h }, p3/Z, [x27, x0]\n" - "ld1sb { z28.h }, p3/Z, [x26, x0]\n" - ".inst 0x45511063 // ssublb z3.h, z3.b, z17.b\n" - ".inst 0x45511084 // ssublb z4.h, z4.b, z17.b\n" - "ld1sb { z27.h }, p3/Z, [x25, x0]\n" - "ld1sb { z23.h }, p3/Z, [x24, x0]\n" - ".inst 0x454f13ff // ssublb z31.h, z31.b, z15.b\n" - ".inst 0x454f13de // ssublb z30.h, z30.b, z15.b\n" - "ld1sb { z25.h }, p3/Z, [x23, x0]\n" - "ld1sb { z24.h }, p3/Z, [x22, x0]\n" - ".inst 0x454f13bd // ssublb z29.h, z29.b, z15.b\n" - ".inst 0x454f139c // ssublb z28.h, z28.b, z15.b\n" - "ld1sb { z26.h }, p3/Z, [x21, x0]\n" - "ld1sb { z22.h }, p3/Z, [x20, x0]\n" - ".inst 0x454f137b // ssublb z27.h, z27.b, z15.b\n" - ".inst 0x454f12f7 // ssublb z23.h, z23.b, z15.b\n" - "ldr x17, [%x[params], %[offsetof_Params_requant_muls]]\n" - "ldr x16, [%x[params], %[offsetof_Params_requant_shifts]]\n" - "str x14, [%x[params], %[offsetof_Params_bias]]\n" - ".inst 0x454f1339 // ssublb z25.h, z25.b, z15.b\n" - ".inst 0x454f1318 // ssublb z24.h, z24.b, z15.b\n" - ".inst 0x454f135a // ssublb z26.h, z26.b, z15.b\n" - ".inst 0x454f12d6 // ssublb z22.h, z22.b, z15.b\n" + "ld1sb { z17.h }, p4/Z, [x4, #4, MUL VL]\n" + "ldp x9, x28, [x17, #0x0]\n" + "mov z18.d, z23.d\n" + "mov z9.d, z14.d\n" + "ldp x27, x26, [x17, #0x10]\n" + "ldp x25, x24, [x17, #0x20]\n" + "mov z20.d, z23.d\n" + "mov z7.d, z14.d\n" + "ldp x23, x22, [x17, #0x30]\n" + "ldp x21, x20, [x17, #0x40]\n" + "mov z1.d, z23.d\n" + ".inst 0x454a135a // ssublb z26.h, z26.b, z10.b\n" + "ld1sb { z22.h }, p3/Z, [x9, x2]\n" + "ld1sb { z2.h }, p3/Z, [x28, x2]\n" + ".inst 0x454a1108 // ssublb z8.h, z8.b, z10.b\n" + ".inst 0x454a1210 // ssublb z16.h, z16.b, z10.b\n" + "ld1sb { z11.h }, p3/Z, [x27, x2]\n" + "ld1sb { z3.h }, p3/Z, [x26, x2]\n" + ".inst 0x454a12b5 // ssublb z21.h, z21.b, z10.b\n" + ".inst 0x454a1231 // ssublb z17.h, z17.b, z10.b\n" + "ld1sb { z29.h }, p3/Z, [x25, x2]\n" + "ld1sb { z4.h }, p3/Z, [x24, x2]\n" + ".inst 0x455e12d6 // ssublb z22.h, z22.b, z30.b\n" + ".inst 0x455e1042 // ssublb z2.h, z2.b, z30.b\n" + "ld1sb { z31.h }, p3/Z, [x23, x2]\n" + "ld1sb { z0.h }, p3/Z, [x22, x2]\n" + ".inst 0x455e116b // ssublb z11.h, z11.b, z30.b\n" + ".inst 0x455e1063 // ssublb z3.h, z3.b, z30.b\n" + "ld1sb { z19.h }, p3/Z, [x21, x2]\n" + "ld1sb { z28.h }, p3/Z, [x20, x2]\n" + ".inst 0x455e13bd // ssublb z29.h, z29.b, z30.b\n" + ".inst 0x455e1084 // ssublb z4.h, z4.b, z30.b\n" + "ldr x15, [%x[params], %[offsetof_Params_requant_muls]]\n" + "ldr x14, [%x[params], %[offsetof_Params_requant_shifts]]\n" + "str x10, [%x[params], %[offsetof_Params_bias]]\n" + ".inst 0x455e13ff // ssublb z31.h, z31.b, z30.b\n" + ".inst 0x455e1000 // ssublb z0.h, z0.b, z30.b\n" + ".inst 0x455e1273 // ssublb z19.h, z19.b, z30.b\n" + ".inst 0x455e139c // ssublb z28.h, z28.b, z30.b\n" "1:" // Loop - ".inst 0x448043ee // smlalb z14.s, p4/M, z31.h, z0.h\n" - ".inst 0x448047ea // smlalt z10.s, p4/M, z31.h, z0.h\n" - "ldr x20, [x7, #0x50]\n" - "ld1sb { z31.h }, p3/Z, [x20, x0]\n" - ".inst 0x448143ce // smlalb z14.s, p4/M, z30.h, z1.h\n" - ".inst 0x448043d4 // smlalb z20.s, p4/M, z30.h, z0.h\n" - "ldr x22, [x7, #0x58]\n" - ".inst 0x454f13ff // ssublb z31.h, z31.b, z15.b\n" - ".inst 0x448043a8 // smlalb z8.s, p4/M, z29.h, z0.h\n" - ".inst 0x44804386 // smlalb z6.s, p4/M, z28.h, z0.h\n" - "ldr x21, [x7, #0x60]\n" - "ldr x20, [x7, #0x68]\n" - ".inst 0x448147ca // smlalt z10.s, p4/M, z30.h, z1.h\n" - ".inst 0x448047c7 // smlalt z7.s, p4/M, z30.h, z0.h\n" - "ld1sb { z30.h }, p3/Z, [x22, x0]\n" - ".inst 0x454f13de // ssublb z30.h, z30.b, z15.b\n" - ".inst 0x448047b0 // smlalt z16.s, p4/M, z29.h, z0.h\n" - ".inst 0x4482436e // smlalb z14.s, p4/M, z27.h, z2.h\n" - "ldr x25, [x7, #0x70]\n" - "ldr x24, [x7, #0x78]\n" - ".inst 0x44804785 // smlalt z5.s, p4/M, z28.h, z0.h\n" - ".inst 0x44814374 // smlalb z20.s, p4/M, z27.h, z1.h\n" - "ld1sb { z0.h }, p4/Z, [x2, #5, MUL VL]\n" - ".inst 0x45511000 // ssublb z0.h, z0.b, z17.b\n" - ".inst 0x44814388 // smlalb z8.s, p4/M, z28.h, z1.h\n" - ".inst 0x448142e6 // smlalb z6.s, p4/M, z23.h, z1.h\n" - "ldr x15, [x7, #0x80]\n" - "ldr x23, [x7, #0x88]\n" - ".inst 0x4482476a // smlalt z10.s, p4/M, z27.h, z2.h\n" - ".inst 0x44814767 // smlalt z7.s, p4/M, z27.h, z1.h\n" - "ld1sb { z27.h }, p3/Z, [x21, x0]\n" - ".inst 0x454f137b // ssublb z27.h, z27.b, z15.b\n" - ".inst 0x44814790 // smlalt z16.s, p4/M, z28.h, z1.h\n" - ".inst 0x4483432e // smlalb z14.s, p4/M, z25.h, z3.h\n" - "ldr x22, [x7, #0x90]\n" - "ldr x21, [x7, #0x98]\n" - ".inst 0x448146e5 // smlalt z5.s, p4/M, z23.h, z1.h\n" - ".inst 0x44824334 // smlalb z20.s, p4/M, z25.h, z2.h\n" - "ld1sb { z1.h }, p4/Z, [x2, #6, MUL VL]\n" - ".inst 0x45511021 // ssublb z1.h, z1.b, z17.b\n" - ".inst 0x448242e8 // smlalb z8.s, p4/M, z23.h, z2.h\n" - ".inst 0x448243e6 // smlalb z6.s, p4/M, z31.h, z2.h\n" - "ldr x14, [x7, #0xa0]\n" - "ldr x13, [x7, #0xa8]\n" - ".inst 0x4483472a // smlalt z10.s, p4/M, z25.h, z3.h\n" - ".inst 0x44824727 // smlalt z7.s, p4/M, z25.h, z2.h\n" - "ld1sb { z25.h }, p3/Z, [x20, x0]\n" - ".inst 0x454f1339 // ssublb z25.h, z25.b, z15.b\n" - ".inst 0x448246f0 // smlalt z16.s, p4/M, z23.h, z2.h\n" - ".inst 0x4484430e // smlalb z14.s, p4/M, z24.h, z4.h\n" - "ldr x12, [x7, #0xb0]\n" - "ldr x20, [x7, #0xb8]\n" - ".inst 0x448247e5 // smlalt z5.s, p4/M, z31.h, z2.h\n" - ".inst 0x44834314 // smlalb z20.s, p4/M, z24.h, z3.h\n" - "ld1sb { z2.h }, p4/Z, [x2, #7, MUL VL]\n" - "inch x2, ALL, MUL #8\n" - ".inst 0x448343e8 // smlalb z8.s, p4/M, z31.h, z3.h\n" - ".inst 0x448343c6 // smlalb z6.s, p4/M, z30.h, z3.h\n" - ".inst 0x45511042 // ssublb z2.h, z2.b, z17.b\n" - "ldr x11, [x7, #0xc0]\n" - ".inst 0x4484470a // smlalt z10.s, p4/M, z24.h, z4.h\n" - ".inst 0x44834707 // smlalt z7.s, p4/M, z24.h, z3.h\n" - "ld1sb { z24.h }, p3/Z, [x25, x0]\n" - ".inst 0x454f1318 // ssublb z24.h, z24.b, z15.b\n" - ".inst 0x448347f0 // smlalt z16.s, p4/M, z31.h, z3.h\n" - ".inst 0x448043ae // smlalb z14.s, p4/M, z29.h, z0.h\n" - "ldr x10, [x7, #0xc8]\n" - "ldr x9, [x7, #0xd0]\n" - ".inst 0x448347c5 // smlalt z5.s, p4/M, z30.h, z3.h\n" - ".inst 0x44844374 // smlalb z20.s, p4/M, z27.h, z4.h\n" - "ld1sb { z3.h }, p4/Z, [x2]\n" - ".inst 0x45511063 // ssublb z3.h, z3.b, z17.b\n" - ".inst 0x448443c8 // smlalb z8.s, p4/M, z30.h, z4.h\n" - ".inst 0x44844346 // smlalb z6.s, p4/M, z26.h, z4.h\n" - "ldr x28, [x7, #0xd8]\n" - "ldr x27, [x7, #0xe0]\n" - ".inst 0x448047aa // smlalt z10.s, p4/M, z29.h, z0.h\n" - ".inst 0x44844767 // smlalt z7.s, p4/M, z27.h, z4.h\n" - "ld1sb { z27.h }, p3/Z, [x24, x0]\n" - ".inst 0x454f137b // ssublb z27.h, z27.b, z15.b\n" - ".inst 0x448447d0 // smlalt z16.s, p4/M, z30.h, z4.h\n" - ".inst 0x4481438e // smlalb z14.s, p4/M, z28.h, z1.h\n" - "ldr x26, [x7, #0xe8]\n" - "ldr x25, [x7, #0xf0]\n" - ".inst 0x44844745 // smlalt z5.s, p4/M, z26.h, z4.h\n" - ".inst 0x44804394 // smlalb z20.s, p4/M, z28.h, z0.h\n" - "ld1sb { z4.h }, p4/Z, [x2, #1, MUL VL]\n" - ".inst 0x45511084 // ssublb z4.h, z4.b, z17.b\n" - ".inst 0x448042c8 // smlalb z8.s, p4/M, z22.h, z0.h\n" - ".inst 0x44804326 // smlalb z6.s, p4/M, z25.h, z0.h\n" - "ld1w { z19.s }, p2/Z, [x17]\n" - "ld1w { z18.s }, p1/Z, [x17, #1, MUL VL]\n" - ".inst 0x4481478a // smlalt z10.s, p4/M, z28.h, z1.h\n" - ".inst 0x44804787 // smlalt z7.s, p4/M, z28.h, z0.h\n" - "ld1sb { z28.h }, p3/Z, [x23, x0]\n" - ".inst 0x454f139c // ssublb z28.h, z28.b, z15.b\n" - ".inst 0x448046d0 // smlalt z16.s, p4/M, z22.h, z0.h\n" - ".inst 0x448242ee // smlalb z14.s, p4/M, z23.h, z2.h\n" - "ldr x24, [x7, #0xf8]\n" - "uzp1 z9.s, z19.s, z18.s\n" - ".inst 0x44804725 // smlalt z5.s, p4/M, z25.h, z0.h\n" - ".inst 0x448142f4 // smlalb z20.s, p4/M, z23.h, z1.h\n" - "ld1sb { z0.h }, p4/Z, [x2, #2, MUL VL]\n" - ".inst 0x45511000 // ssublb z0.h, z0.b, z17.b\n" - ".inst 0x44814328 // smlalb z8.s, p4/M, z25.h, z1.h\n" - ".inst 0x44814306 // smlalb z6.s, p4/M, z24.h, z1.h\n" - "uzp2 z29.s, z19.s, z18.s\n" - "ld1w { z19.s }, p2/Z, [x16]\n" - ".inst 0x448246ea // smlalt z10.s, p4/M, z23.h, z2.h\n" - ".inst 0x448146e7 // smlalt z7.s, p4/M, z23.h, z1.h\n" - "ld1sb { z23.h }, p3/Z, [x15, x0]\n" - ".inst 0x454f12f7 // ssublb z23.h, z23.b, z15.b\n" - ".inst 0x44814730 // smlalt z16.s, p4/M, z25.h, z1.h\n" - ".inst 0x448343ee // smlalb z14.s, p4/M, z31.h, z3.h\n" - "ldr x23, [x7, #0x100]\n" - "whilelt p0.h, x8, x1\n" - ".inst 0x44814705 // smlalt z5.s, p4/M, z24.h, z1.h\n" - ".inst 0x448243f4 // smlalb z20.s, p4/M, z31.h, z2.h\n" - "ld1sb { z1.h }, p4/Z, [x2, #3, MUL VL]\n" - ".inst 0x45511021 // ssublb z1.h, z1.b, z17.b\n" - ".inst 0x44824308 // smlalb z8.s, p4/M, z24.h, z2.h\n" - ".inst 0x44824366 // smlalb z6.s, p4/M, z27.h, z2.h\n" - "addvl x17, x17, #2\n" - ".inst 0x448347ea // smlalt z10.s, p4/M, z31.h, z3.h\n" - ".inst 0x448247e7 // smlalt z7.s, p4/M, z31.h, z2.h\n" - "ld1sb { z31.h }, p3/Z, [x22, x0]\n" - ".inst 0x454f13ff // ssublb z31.h, z31.b, z15.b\n" - ".inst 0x44824710 // smlalt z16.s, p4/M, z24.h, z2.h\n" - ".inst 0x448443ce // smlalb z14.s, p4/M, z30.h, z4.h\n" - "ldr x22, [x7, #0x108]\n" - ".inst 0x44824765 // smlalt z5.s, p4/M, z27.h, z2.h\n" - ".inst 0x448343d4 // smlalb z20.s, p4/M, z30.h, z3.h\n" - "ld1sb { z2.h }, p4/Z, [x2, #4, MUL VL]\n" - ".inst 0x45511042 // ssublb z2.h, z2.b, z17.b\n" - ".inst 0x44834368 // smlalb z8.s, p4/M, z27.h, z3.h\n" - ".inst 0x448342e6 // smlalb z6.s, p4/M, z23.h, z3.h\n" - ".inst 0x448447ca // smlalt z10.s, p4/M, z30.h, z4.h\n" - ".inst 0x448347c7 // smlalt z7.s, p4/M, z30.h, z3.h\n" - "ld1sb { z30.h }, p3/Z, [x21, x0]\n" - ".inst 0x454f13de // ssublb z30.h, z30.b, z15.b\n" - ".inst 0x44834770 // smlalt z16.s, p4/M, z27.h, z3.h\n" - ".inst 0x448042ce // smlalb z14.s, p4/M, z22.h, z0.h\n" - "ldr x21, [x7, #0x110]\n" - ".inst 0x448346e5 // smlalt z5.s, p4/M, z23.h, z3.h\n" - ".inst 0x44844354 // smlalb z20.s, p4/M, z26.h, z4.h\n" - "ld1sb { z3.h }, p4/Z, [x2, #5, MUL VL]\n" - ".inst 0x45511063 // ssublb z3.h, z3.b, z17.b\n" - ".inst 0x448442e8 // smlalb z8.s, p4/M, z23.h, z4.h\n" - ".inst 0x44844386 // smlalb z6.s, p4/M, z28.h, z4.h\n" - ".inst 0x448046ca // smlalt z10.s, p4/M, z22.h, z0.h\n" - ".inst 0x44844747 // smlalt z7.s, p4/M, z26.h, z4.h\n" - "ld1sb { z26.h }, p3/Z, [x14, x0]\n" - ".inst 0x454f135a // ssublb z26.h, z26.b, z15.b\n" - ".inst 0x448446f0 // smlalt z16.s, p4/M, z23.h, z4.h\n" - ".inst 0x4481432e // smlalb z14.s, p4/M, z25.h, z1.h\n" - "ld1sb { z22.h }, p3/Z, [x20, x0]\n" - ".inst 0x454f12d6 // ssublb z22.h, z22.b, z15.b\n" - ".inst 0x44844785 // smlalt z5.s, p4/M, z28.h, z4.h\n" - ".inst 0x44804334 // smlalb z20.s, p4/M, z25.h, z0.h\n" - "ld1sb { z4.h }, p4/Z, [x2, #6, MUL VL]\n" - ".inst 0x45511084 // ssublb z4.h, z4.b, z17.b\n" - ".inst 0x448043e8 // smlalb z8.s, p4/M, z31.h, z0.h\n" - ".inst 0x448043c6 // smlalb z6.s, p4/M, z30.h, z0.h\n" - "ldr x20, [x7, #0x118]\n" - "ldr x14, [%x[params], %[offsetof_Params_bias]]\n" - ".inst 0x4481472a // smlalt z10.s, p4/M, z25.h, z1.h\n" - ".inst 0x44804727 // smlalt z7.s, p4/M, z25.h, z0.h\n" - "ld1sb { z25.h }, p3/Z, [x13, x0]\n" - ".inst 0x454f1339 // ssublb z25.h, z25.b, z15.b\n" - ".inst 0x448047f0 // smlalt z16.s, p4/M, z31.h, z0.h\n" - ".inst 0x4482430e // smlalb z14.s, p4/M, z24.h, z2.h\n" - ".inst 0x448047c5 // smlalt z5.s, p4/M, z30.h, z0.h\n" - ".inst 0x44814314 // smlalb z20.s, p4/M, z24.h, z1.h\n" - "ld1sb { z0.h }, p4/Z, [x2, #7, MUL VL]\n" - "inch x2, ALL, MUL #8\n" - ".inst 0x448143c8 // smlalb z8.s, p4/M, z30.h, z1.h\n" - ".inst 0x44814346 // smlalb z6.s, p4/M, z26.h, z1.h\n" - ".inst 0x45511000 // ssublb z0.h, z0.b, z17.b\n" - ".inst 0x4482470a // smlalt z10.s, p4/M, z24.h, z2.h\n" - ".inst 0x44814707 // smlalt z7.s, p4/M, z24.h, z1.h\n" - "ld1sb { z24.h }, p3/Z, [x12, x0]\n" - ".inst 0x454f1318 // ssublb z24.h, z24.b, z15.b\n" - ".inst 0x448147d0 // smlalt z16.s, p4/M, z30.h, z1.h\n" - ".inst 0x4483436e // smlalb z14.s, p4/M, z27.h, z3.h\n" - ".inst 0x44814745 // smlalt z5.s, p4/M, z26.h, z1.h\n" - ".inst 0x44824374 // smlalb z20.s, p4/M, z27.h, z2.h\n" - "ld1sb { z1.h }, p4/Z, [x2]\n" - ".inst 0x45511021 // ssublb z1.h, z1.b, z17.b\n" - ".inst 0x44824348 // smlalb z8.s, p4/M, z26.h, z2.h\n" - ".inst 0x44824326 // smlalb z6.s, p4/M, z25.h, z2.h\n" - ".inst 0x4483476a // smlalt z10.s, p4/M, z27.h, z3.h\n" - ".inst 0x44824767 // smlalt z7.s, p4/M, z27.h, z2.h\n" - "ld1sb { z27.h }, p3/Z, [x11, x0]\n" - ".inst 0x454f137b // ssublb z27.h, z27.b, z15.b\n" - ".inst 0x44824750 // smlalt z16.s, p4/M, z26.h, z2.h\n" - ".inst 0x448442ee // smlalb z14.s, p4/M, z23.h, z4.h\n" - ".inst 0x44824725 // smlalt z5.s, p4/M, z25.h, z2.h\n" - ".inst 0x448342f4 // smlalb z20.s, p4/M, z23.h, z3.h\n" - "ld1sb { z2.h }, p4/Z, [x2, #1, MUL VL]\n" - ".inst 0x45511042 // ssublb z2.h, z2.b, z17.b\n" - ".inst 0x44834328 // smlalb z8.s, p4/M, z25.h, z3.h\n" - ".inst 0x44834306 // smlalb z6.s, p4/M, z24.h, z3.h\n" - ".inst 0x448446ea // smlalt z10.s, p4/M, z23.h, z4.h\n" - ".inst 0x448346e7 // smlalt z7.s, p4/M, z23.h, z3.h\n" - "ld1sb { z23.h }, p3/Z, [x10, x0]\n" - ".inst 0x454f12f7 // ssublb z23.h, z23.b, z15.b\n" - ".inst 0x44834730 // smlalt z16.s, p4/M, z25.h, z3.h\n" - ".inst 0x448043ee // smlalb z14.s, p4/M, z31.h, z0.h\n" - ".inst 0x44834705 // smlalt z5.s, p4/M, z24.h, z3.h\n" - ".inst 0x44844394 // smlalb z20.s, p4/M, z28.h, z4.h\n" - "ld1sb { z3.h }, p4/Z, [x2, #2, MUL VL]\n" - ".inst 0x45511063 // ssublb z3.h, z3.b, z17.b\n" - ".inst 0x44844308 // smlalb z8.s, p4/M, z24.h, z4.h\n" - ".inst 0x448442c6 // smlalb z6.s, p4/M, z22.h, z4.h\n" - ".inst 0x448047ea // smlalt z10.s, p4/M, z31.h, z0.h\n" - ".inst 0x44844787 // smlalt z7.s, p4/M, z28.h, z4.h\n" - "ld1sb { z31.h }, p3/Z, [x9, x0]\n" - ".inst 0x454f13ff // ssublb z31.h, z31.b, z15.b\n" - ".inst 0x44844710 // smlalt z16.s, p4/M, z24.h, z4.h\n" - ".inst 0x448143ce // smlalb z14.s, p4/M, z30.h, z1.h\n" - "ld1sb { z28.h }, p3/Z, [x27, x0]\n" - ".inst 0x454f139c // ssublb z28.h, z28.b, z15.b\n" - ".inst 0x448446c5 // smlalt z5.s, p4/M, z22.h, z4.h\n" - ".inst 0x448043d4 // smlalb z20.s, p4/M, z30.h, z0.h\n" - "ld1sb { z4.h }, p4/Z, [x2, #3, MUL VL]\n" - ".inst 0x45511084 // ssublb z4.h, z4.b, z17.b\n" - ".inst 0x44804368 // smlalb z8.s, p4/M, z27.h, z0.h\n" - ".inst 0x448042e6 // smlalb z6.s, p4/M, z23.h, z0.h\n" - ".inst 0x448147ca // smlalt z10.s, p4/M, z30.h, z1.h\n" - ".inst 0x448047c7 // smlalt z7.s, p4/M, z30.h, z0.h\n" - "ld1sb { z30.h }, p3/Z, [x28, x0]\n" - ".inst 0x454f13de // ssublb z30.h, z30.b, z15.b\n" - ".inst 0x44804770 // smlalt z16.s, p4/M, z27.h, z0.h\n" - ".inst 0x4482434e // smlalb z14.s, p4/M, z26.h, z2.h\n" - ".inst 0x448046e5 // smlalt z5.s, p4/M, z23.h, z0.h\n" - ".inst 0x44814354 // smlalb z20.s, p4/M, z26.h, z1.h\n" - "ld1sb { z0.h }, p4/Z, [x2, #4, MUL VL]\n" - ".inst 0x45511000 // ssublb z0.h, z0.b, z17.b\n" - ".inst 0x448142e8 // smlalb z8.s, p4/M, z23.h, z1.h\n" - ".inst 0x448143e6 // smlalb z6.s, p4/M, z31.h, z1.h\n" - ".inst 0x4482474a // smlalt z10.s, p4/M, z26.h, z2.h\n" - ".inst 0x44814747 // smlalt z7.s, p4/M, z26.h, z1.h\n" - "ld1sb { z26.h }, p3/Z, [x26, x0]\n" - ".inst 0x454f135a // ssublb z26.h, z26.b, z15.b\n" - ".inst 0x448146f0 // smlalt z16.s, p4/M, z23.h, z1.h\n" - ".inst 0x4483432e // smlalb z14.s, p4/M, z25.h, z3.h\n" - ".inst 0x448147e5 // smlalt z5.s, p4/M, z31.h, z1.h\n" - ".inst 0x44824334 // smlalb z20.s, p4/M, z25.h, z2.h\n" - "ld1sb { z1.h }, p4/Z, [x2, #5, MUL VL]\n" - ".inst 0x45511021 // ssublb z1.h, z1.b, z17.b\n" - ".inst 0x448243e8 // smlalb z8.s, p4/M, z31.h, z2.h\n" - ".inst 0x448243c6 // smlalb z6.s, p4/M, z30.h, z2.h\n" - ".inst 0x4483472a // smlalt z10.s, p4/M, z25.h, z3.h\n" - ".inst 0x44824727 // smlalt z7.s, p4/M, z25.h, z2.h\n" - "ld1sb { z25.h }, p3/Z, [x25, x0]\n" - ".inst 0x454f1339 // ssublb z25.h, z25.b, z15.b\n" - ".inst 0x448247f0 // smlalt z16.s, p4/M, z31.h, z2.h\n" - ".inst 0x4484430e // smlalb z14.s, p4/M, z24.h, z4.h\n" - ".inst 0x448247c5 // smlalt z5.s, p4/M, z30.h, z2.h\n" - ".inst 0x44834314 // smlalb z20.s, p4/M, z24.h, z3.h\n" - "ld1sb { z2.h }, p4/Z, [x2, #6, MUL VL]\n" - ".inst 0x45511042 // ssublb z2.h, z2.b, z17.b\n" - ".inst 0x448343c8 // smlalb z8.s, p4/M, z30.h, z3.h\n" - ".inst 0x44834386 // smlalb z6.s, p4/M, z28.h, z3.h\n" - ".inst 0x4484470a // smlalt z10.s, p4/M, z24.h, z4.h\n" - ".inst 0x44834707 // smlalt z7.s, p4/M, z24.h, z3.h\n" - "ld1sb { z24.h }, p3/Z, [x24, x0]\n" - ".inst 0x454f1318 // ssublb z24.h, z24.b, z15.b\n" - ".inst 0x448347d0 // smlalt z16.s, p4/M, z30.h, z3.h\n" - ".inst 0x4480436e // smlalb z14.s, p4/M, z27.h, z0.h\n" - ".inst 0x44834785 // smlalt z5.s, p4/M, z28.h, z3.h\n" - ".inst 0x448442d4 // smlalb z20.s, p4/M, z22.h, z4.h\n" - "ld1sb { z3.h }, p4/Z, [x2, #7, MUL VL]\n" - "inch x2, ALL, MUL #8\n" - ".inst 0x44844388 // smlalb z8.s, p4/M, z28.h, z4.h\n" - ".inst 0x44844346 // smlalb z6.s, p4/M, z26.h, z4.h\n" - ".inst 0x45511063 // ssublb z3.h, z3.b, z17.b\n" - ".inst 0x4480476a // smlalt z10.s, p4/M, z27.h, z0.h\n" - ".inst 0x44844790 // smlalt z16.s, p4/M, z28.h, z4.h\n" - "ld1sb { z27.h }, p3/Z, [x23, x0]\n" - ".inst 0x454f137b // ssublb z27.h, z27.b, z15.b\n" - ".inst 0x448142ee // smlalb z14.s, p4/M, z23.h, z1.h\n" - ".inst 0x448446c7 // smlalt z7.s, p4/M, z22.h, z4.h\n" - "ld1w { z18.s }, p1/Z, [x16, #1, MUL VL]\n" - "addvl x16, x16, #2\n" - ".inst 0x44844745 // smlalt z5.s, p4/M, z26.h, z4.h\n" - ".inst 0x448042f4 // smlalb z20.s, p4/M, z23.h, z0.h\n" - "ld1sb { z4.h }, p4/Z, [x2]\n" - ".inst 0x45511084 // ssublb z4.h, z4.b, z17.b\n" - ".inst 0x44804328 // smlalb z8.s, p4/M, z25.h, z0.h\n" - ".inst 0x44804306 // smlalb z6.s, p4/M, z24.h, z0.h\n" - "inch x2\n" - ".inst 0x448146ea // smlalt z10.s, p4/M, z23.h, z1.h\n" - ".inst 0x44804730 // smlalt z16.s, p4/M, z25.h, z0.h\n" - "ld1sb { z25.h }, p3/Z, [x22, x0]\n" - ".inst 0x454f1339 // ssublb z25.h, z25.b, z15.b\n" + ".inst 0x449a42ce // smlalb z14.s, p4/M, z22.h, z26.h\n" + ".inst 0x449a46d7 // smlalt z23.s, p4/M, z22.h, z26.h\n" + "ldr x20, [x17, #0x50]\n" + "ld1sb { z27.h }, p3/Z, [x20, x2]\n" + ".inst 0x4488404e // smlalb z14.s, p4/M, z2.h, z8.h\n" + ".inst 0x449a4046 // smlalb z6.s, p4/M, z2.h, z26.h\n" + "ldr x20, [x17, #0x58]\n" + ".inst 0x455e137b // ssublb z27.h, z27.b, z30.b\n" + ".inst 0x449a4169 // smlalb z9.s, p4/M, z11.h, z26.h\n" + ".inst 0x449a4067 // smlalb z7.s, p4/M, z3.h, z26.h\n" + "ld1sb { z5.h }, p3/Z, [x20, x2]\n" + "ldr x20, [x17, #0x60]\n" + ".inst 0x44884457 // smlalt z23.s, p4/M, z2.h, z8.h\n" + ".inst 0x449043ae // smlalb z14.s, p4/M, z29.h, z16.h\n" + "ld1sb { z25.h }, p4/Z, [x4, #5, MUL VL]\n" + ".inst 0x455e10a5 // ssublb z5.h, z5.b, z30.b\n" + ".inst 0x449a4452 // smlalt z18.s, p4/M, z2.h, z26.h\n" + ".inst 0x449a4574 // smlalt z20.s, p4/M, z11.h, z26.h\n" + "ld1sb { z22.h }, p3/Z, [x20, x2]\n" + ".inst 0x454a1339 // ssublb z25.h, z25.b, z10.b\n" + ".inst 0x449a4461 // smlalt z1.s, p4/M, z3.h, z26.h\n" + ".inst 0x448843a6 // smlalb z6.s, p4/M, z29.h, z8.h\n" + "ldr x20, [x17, #0x68]\n" + "ld1sb { z2.h }, p4/Z, [x4, #6, MUL VL]\n" + ".inst 0x44884069 // smlalb z9.s, p4/M, z3.h, z8.h\n" + ".inst 0x44884087 // smlalb z7.s, p4/M, z4.h, z8.h\n" + ".inst 0x455e12d6 // ssublb z22.h, z22.b, z30.b\n" + "ld1sb { z26.h }, p3/Z, [x20, x2]\n" + ".inst 0x449047b7 // smlalt z23.s, p4/M, z29.h, z16.h\n" + ".inst 0x449543ee // smlalb z14.s, p4/M, z31.h, z21.h\n" + ".inst 0x454a1042 // ssublb z2.h, z2.b, z10.b\n" + "ldr x20, [x17, #0x70]\n" + ".inst 0x448847b2 // smlalt z18.s, p4/M, z29.h, z8.h\n" + ".inst 0x44884474 // smlalt z20.s, p4/M, z3.h, z8.h\n" + "ld1sb { z29.h }, p4/Z, [x4, #7, MUL VL]\n" + ".inst 0x455e135a // ssublb z26.h, z26.b, z30.b\n" + ".inst 0x44884481 // smlalt z1.s, p4/M, z4.h, z8.h\n" + ".inst 0x449043e6 // smlalb z6.s, p4/M, z31.h, z16.h\n" + "inch x4, ALL, MUL #8\n" + "ld1sb { z8.h }, p3/Z, [x20, x2]\n" + ".inst 0x44904089 // smlalb z9.s, p4/M, z4.h, z16.h\n" + ".inst 0x44904367 // smlalb z7.s, p4/M, z27.h, z16.h\n" + ".inst 0x454a13bd // ssublb z29.h, z29.b, z10.b\n" + "ldr x20, [x17, #0x78]\n" + ".inst 0x449547f7 // smlalt z23.s, p4/M, z31.h, z21.h\n" + ".inst 0x4491400e // smlalb z14.s, p4/M, z0.h, z17.h\n" + "ld1sb { z24.h }, p4/Z, [x4]\n" + ".inst 0x455e1108 // ssublb z8.h, z8.b, z30.b\n" + ".inst 0x449047f2 // smlalt z18.s, p4/M, z31.h, z16.h\n" + ".inst 0x44904494 // smlalt z20.s, p4/M, z4.h, z16.h\n" + "ld1sb { z31.h }, p3/Z, [x20, x2]\n" + ".inst 0x454a1318 // ssublb z24.h, z24.b, z10.b\n" + ".inst 0x44904761 // smlalt z1.s, p4/M, z27.h, z16.h\n" + ".inst 0x44954006 // smlalb z6.s, p4/M, z0.h, z21.h\n" + "ldr x22, [x17, #0x80]\n" + "ld1sb { z16.h }, p4/Z, [x4, #1, MUL VL]\n" + ".inst 0x44954369 // smlalb z9.s, p4/M, z27.h, z21.h\n" + ".inst 0x449540a7 // smlalb z7.s, p4/M, z5.h, z21.h\n" + ".inst 0x455e13ff // ssublb z31.h, z31.b, z30.b\n" + "ldr x21, [x17, #0x88]\n" + ".inst 0x44914417 // smlalt z23.s, p4/M, z0.h, z17.h\n" + ".inst 0x4499416e // smlalb z14.s, p4/M, z11.h, z25.h\n" + ".inst 0x454a1210 // ssublb z16.h, z16.b, z10.b\n" + "ldr x20, [x17, #0x90]\n" + ".inst 0x44954412 // smlalt z18.s, p4/M, z0.h, z21.h\n" + ".inst 0x44954774 // smlalt z20.s, p4/M, z27.h, z21.h\n" + "ld1sb { z0.h }, p3/Z, [x22, x2]\n" + ".inst 0x455e1000 // ssublb z0.h, z0.b, z30.b\n" + ".inst 0x449544a1 // smlalt z1.s, p4/M, z5.h, z21.h\n" + ".inst 0x449142c6 // smlalb z6.s, p4/M, z22.h, z17.h\n" + "ld1sb { z21.h }, p4/Z, [x4, #2, MUL VL]\n" + ".inst 0x454a12b5 // ssublb z21.h, z21.b, z10.b\n" + ".inst 0x449140a9 // smlalb z9.s, p4/M, z5.h, z17.h\n" + ".inst 0x44914267 // smlalb z7.s, p4/M, z19.h, z17.h\n" + "ldr x23, [x17, #0x98]\n" + "ldr x22, [x17, #0xa0]\n" + ".inst 0x44994577 // smlalt z23.s, p4/M, z11.h, z25.h\n" + ".inst 0x4482406e // smlalb z14.s, p4/M, z3.h, z2.h\n" + "ld1sb { z11.h }, p3/Z, [x21, x2]\n" + ".inst 0x455e116b // ssublb z11.h, z11.b, z30.b\n" + ".inst 0x449146d2 // smlalt z18.s, p4/M, z22.h, z17.h\n" + ".inst 0x449144b4 // smlalt z20.s, p4/M, z5.h, z17.h\n" + "ld1sb { z22.h }, p4/Z, [x4, #3, MUL VL]\n" + ".inst 0x454a12d6 // ssublb z22.h, z22.b, z10.b\n" + ".inst 0x44914661 // smlalt z1.s, p4/M, z19.h, z17.h\n" + ".inst 0x44994066 // smlalb z6.s, p4/M, z3.h, z25.h\n" + "ld1sb { z17.h }, p3/Z, [x20, x2]\n" + ".inst 0x455e1231 // ssublb z17.h, z17.b, z30.b\n" + ".inst 0x44994389 // smlalb z9.s, p4/M, z28.h, z25.h\n" + ".inst 0x44994347 // smlalb z7.s, p4/M, z26.h, z25.h\n" + "ldr x20, [x17, #0xa8]\n" + "ldr x21, [x17, #0xb0]\n" + ".inst 0x44824477 // smlalt z23.s, p4/M, z3.h, z2.h\n" + ".inst 0x449d408e // smlalb z14.s, p4/M, z4.h, z29.h\n" + "ldr x13, [x17, #0xb8]\n" + "ldr x12, [x17, #0xc0]\n" + ".inst 0x44994472 // smlalt z18.s, p4/M, z3.h, z25.h\n" + ".inst 0x44994794 // smlalt z20.s, p4/M, z28.h, z25.h\n" + "ld1sb { z3.h }, p3/Z, [x23, x2]\n" + ".inst 0x455e1063 // ssublb z3.h, z3.b, z30.b\n" + ".inst 0x44994741 // smlalt z1.s, p4/M, z26.h, z25.h\n" + ".inst 0x44824086 // smlalb z6.s, p4/M, z4.h, z2.h\n" + "ld1sb { z25.h }, p4/Z, [x4, #4, MUL VL]\n" + ".inst 0x454a1339 // ssublb z25.h, z25.b, z10.b\n" + ".inst 0x44824349 // smlalb z9.s, p4/M, z26.h, z2.h\n" + ".inst 0x44824107 // smlalb z7.s, p4/M, z8.h, z2.h\n" + "ldr x11, [x17, #0xc8]\n" + "ldr x10, [x17, #0xd0]\n" + ".inst 0x449d4497 // smlalt z23.s, p4/M, z4.h, z29.h\n" + ".inst 0x4498436e // smlalb z14.s, p4/M, z27.h, z24.h\n" + "ldr x9, [x17, #0xd8]\n" + "ldr x28, [x17, #0xe0]\n" + ".inst 0x44824492 // smlalt z18.s, p4/M, z4.h, z2.h\n" + ".inst 0x44824754 // smlalt z20.s, p4/M, z26.h, z2.h\n" + "ld1sb { z4.h }, p3/Z, [x22, x2]\n" + ".inst 0x455e1084 // ssublb z4.h, z4.b, z30.b\n" + ".inst 0x44824501 // smlalt z1.s, p4/M, z8.h, z2.h\n" + ".inst 0x449d4366 // smlalb z6.s, p4/M, z27.h, z29.h\n" + "ld1sb { z2.h }, p4/Z, [x4, #5, MUL VL]\n" + ".inst 0x454a1042 // ssublb z2.h, z2.b, z10.b\n" + ".inst 0x449d4109 // smlalb z9.s, p4/M, z8.h, z29.h\n" + ".inst 0x449d43e7 // smlalb z7.s, p4/M, z31.h, z29.h\n" + "ldr x27, [x17, #0xe8]\n" + "ldr x26, [x17, #0xf0]\n" + ".inst 0x44984777 // smlalt z23.s, p4/M, z27.h, z24.h\n" + ".inst 0x449040ae // smlalb z14.s, p4/M, z5.h, z16.h\n" + "ldr x25, [x17, #0xf8]\n" + "ldr x24, [x17, #0x100]\n" + ".inst 0x449d4772 // smlalt z18.s, p4/M, z27.h, z29.h\n" + ".inst 0x449d4514 // smlalt z20.s, p4/M, z8.h, z29.h\n" + "ld1sb { z27.h }, p3/Z, [x20, x2]\n" + ".inst 0x455e137b // ssublb z27.h, z27.b, z30.b\n" + ".inst 0x449d47e1 // smlalt z1.s, p4/M, z31.h, z29.h\n" + ".inst 0x449840a6 // smlalb z6.s, p4/M, z5.h, z24.h\n" + "ld1sb { z29.h }, p4/Z, [x4, #6, MUL VL]\n" + ".inst 0x454a13bd // ssublb z29.h, z29.b, z10.b\n" + ".inst 0x449843e9 // smlalb z9.s, p4/M, z31.h, z24.h\n" + ".inst 0x44984007 // smlalb z7.s, p4/M, z0.h, z24.h\n" + "ldr x23, [x17, #0x108]\n" + "ldr x22, [x17, #0x110]\n" + ".inst 0x449044b7 // smlalt z23.s, p4/M, z5.h, z16.h\n" + ".inst 0x4495438e // smlalb z14.s, p4/M, z28.h, z21.h\n" + "ldr x20, [x17, #0x118]\n" + "whilelt p0.h, x16, x3\n" + ".inst 0x449844b2 // smlalt z18.s, p4/M, z5.h, z24.h\n" + ".inst 0x449847f4 // smlalt z20.s, p4/M, z31.h, z24.h\n" + "ld1sb { z5.h }, p3/Z, [x21, x2]\n" + ".inst 0x455e10a5 // ssublb z5.h, z5.b, z30.b\n" + ".inst 0x44984401 // smlalt z1.s, p4/M, z0.h, z24.h\n" + ".inst 0x44904266 // smlalb z6.s, p4/M, z19.h, z16.h\n" + "ld1sb { z24.h }, p4/Z, [x4, #7, MUL VL]\n" + "inch x4, ALL, MUL #8\n" + ".inst 0x44904009 // smlalb z9.s, p4/M, z0.h, z16.h\n" + ".inst 0x44904167 // smlalb z7.s, p4/M, z11.h, z16.h\n" + ".inst 0x454a1318 // ssublb z24.h, z24.b, z10.b\n" + "ldr x21, [%x[params], %[offsetof_Params_bias]]\n" + ".inst 0x44954797 // smlalt z23.s, p4/M, z28.h, z21.h\n" + ".inst 0x4496434e // smlalb z14.s, p4/M, z26.h, z22.h\n" + "ld1sb { z28.h }, p3/Z, [x13, x2]\n" + ".inst 0x455e139c // ssublb z28.h, z28.b, z30.b\n" + ".inst 0x44904672 // smlalt z18.s, p4/M, z19.h, z16.h\n" + ".inst 0x44904414 // smlalt z20.s, p4/M, z0.h, z16.h\n" + "ld1sb { z19.h }, p4/Z, [x4]\n" + ".inst 0x454a1273 // ssublb z19.h, z19.b, z10.b\n" + ".inst 0x44904561 // smlalt z1.s, p4/M, z11.h, z16.h\n" + ".inst 0x44954346 // smlalb z6.s, p4/M, z26.h, z21.h\n" + "ld1sb { z16.h }, p3/Z, [x12, x2]\n" + ".inst 0x455e1210 // ssublb z16.h, z16.b, z30.b\n" + ".inst 0x44954229 // smlalb z9.s, p4/M, z17.h, z21.h\n" + ".inst 0x44954067 // smlalb z7.s, p4/M, z3.h, z21.h\n" + ".inst 0x44964757 // smlalt z23.s, p4/M, z26.h, z22.h\n" + ".inst 0x4499410e // smlalb z14.s, p4/M, z8.h, z25.h\n" + ".inst 0x44954752 // smlalt z18.s, p4/M, z26.h, z21.h\n" + ".inst 0x44954634 // smlalt z20.s, p4/M, z17.h, z21.h\n" + "ld1sb { z26.h }, p3/Z, [x11, x2]\n" + ".inst 0x455e135a // ssublb z26.h, z26.b, z30.b\n" + ".inst 0x44954461 // smlalt z1.s, p4/M, z3.h, z21.h\n" + ".inst 0x44964106 // smlalb z6.s, p4/M, z8.h, z22.h\n" + "ld1sb { z21.h }, p4/Z, [x4, #1, MUL VL]\n" + ".inst 0x454a12b5 // ssublb z21.h, z21.b, z10.b\n" + ".inst 0x44964069 // smlalb z9.s, p4/M, z3.h, z22.h\n" + ".inst 0x44964087 // smlalb z7.s, p4/M, z4.h, z22.h\n" + ".inst 0x44994517 // smlalt z23.s, p4/M, z8.h, z25.h\n" ".inst 0x448243ee // smlalb z14.s, p4/M, z31.h, z2.h\n" - ".inst 0x448046e7 // smlalt z7.s, p4/M, z23.h, z0.h\n" - "uzp1 z23.s, z19.s, z18.s\n" - ".inst 0x44804705 // smlalt z5.s, p4/M, z24.h, z0.h\n" - ".inst 0x448143f4 // smlalb z20.s, p4/M, z31.h, z1.h\n" - "uzp2 z22.s, z19.s, z18.s\n" - ".inst 0x44814308 // smlalb z8.s, p4/M, z24.h, z1.h\n" - ".inst 0x44814366 // smlalb z6.s, p4/M, z27.h, z1.h\n" - ".inst 0x448247ea // smlalt z10.s, p4/M, z31.h, z2.h\n" - ".inst 0x44814710 // smlalt z16.s, p4/M, z24.h, z1.h\n" - "ld1sb { z24.h }, p3/Z, [x21, x0]\n" - ".inst 0x454f1318 // ssublb z24.h, z24.b, z15.b\n" - ".inst 0x448343ce // smlalb z14.s, p4/M, z30.h, z3.h\n" - ".inst 0x448147e7 // smlalt z7.s, p4/M, z31.h, z1.h\n" - ".inst 0x44814765 // smlalt z5.s, p4/M, z27.h, z1.h\n" - ".inst 0x448243d4 // smlalb z20.s, p4/M, z30.h, z2.h\n" - ".inst 0x44824368 // smlalb z8.s, p4/M, z27.h, z2.h\n" - ".inst 0x44824326 // smlalb z6.s, p4/M, z25.h, z2.h\n" - ".inst 0x448347ca // smlalt z10.s, p4/M, z30.h, z3.h\n" - ".inst 0x44824770 // smlalt z16.s, p4/M, z27.h, z2.h\n" - "ld1sb { z27.h }, p3/Z, [x20, x0]\n" - ".inst 0x454f137b // ssublb z27.h, z27.b, z15.b\n" - ".inst 0x4484438e // smlalb z14.s, p4/M, z28.h, z4.h\n" - ".inst 0x448247c7 // smlalt z7.s, p4/M, z30.h, z2.h\n" - ".inst 0x04a975ce // sqrdmulh z14.s, z14.s, z9.s\n" - "inch x0\n" - ".inst 0x44824725 // smlalt z5.s, p4/M, z25.h, z2.h\n" - ".inst 0x44834394 // smlalb z20.s, p4/M, z28.h, z3.h\n" - "and z21.d, z14.d, z23.d\n" - "mov x20, x0\n" - ".inst 0x44834328 // smlalb z8.s, p4/M, z25.h, z3.h\n" - ".inst 0x44834306 // smlalb z6.s, p4/M, z24.h, z3.h\n" - "asr z21.s, z21.s, #0x1f\n" + ".inst 0x44964512 // smlalt z18.s, p4/M, z8.h, z22.h\n" + ".inst 0x44964474 // smlalt z20.s, p4/M, z3.h, z22.h\n" + "ld1sb { z8.h }, p3/Z, [x10, x2]\n" + ".inst 0x455e1108 // ssublb z8.h, z8.b, z30.b\n" + ".inst 0x44964481 // smlalt z1.s, p4/M, z4.h, z22.h\n" + ".inst 0x449943e6 // smlalb z6.s, p4/M, z31.h, z25.h\n" + "ld1sb { z22.h }, p4/Z, [x4, #2, MUL VL]\n" + ".inst 0x454a12d6 // ssublb z22.h, z22.b, z10.b\n" + ".inst 0x44994089 // smlalb z9.s, p4/M, z4.h, z25.h\n" + ".inst 0x44994367 // smlalb z7.s, p4/M, z27.h, z25.h\n" + ".inst 0x448247f7 // smlalt z23.s, p4/M, z31.h, z2.h\n" + ".inst 0x449d400e // smlalb z14.s, p4/M, z0.h, z29.h\n" + ".inst 0x449947f2 // smlalt z18.s, p4/M, z31.h, z25.h\n" + ".inst 0x44994494 // smlalt z20.s, p4/M, z4.h, z25.h\n" + "ld1sb { z31.h }, p3/Z, [x9, x2]\n" + ".inst 0x455e13ff // ssublb z31.h, z31.b, z30.b\n" + ".inst 0x44994761 // smlalt z1.s, p4/M, z27.h, z25.h\n" + ".inst 0x44824006 // smlalb z6.s, p4/M, z0.h, z2.h\n" + "ld1sb { z25.h }, p4/Z, [x4, #3, MUL VL]\n" + ".inst 0x454a1339 // ssublb z25.h, z25.b, z10.b\n" + ".inst 0x44824369 // smlalb z9.s, p4/M, z27.h, z2.h\n" + ".inst 0x448240a7 // smlalb z7.s, p4/M, z5.h, z2.h\n" + ".inst 0x449d4417 // smlalt z23.s, p4/M, z0.h, z29.h\n" + ".inst 0x4498422e // smlalb z14.s, p4/M, z17.h, z24.h\n" + ".inst 0x44824412 // smlalt z18.s, p4/M, z0.h, z2.h\n" + ".inst 0x44824774 // smlalt z20.s, p4/M, z27.h, z2.h\n" + "ld1sb { z0.h }, p3/Z, [x28, x2]\n" + ".inst 0x455e1000 // ssublb z0.h, z0.b, z30.b\n" + ".inst 0x448244a1 // smlalt z1.s, p4/M, z5.h, z2.h\n" + ".inst 0x449d4166 // smlalb z6.s, p4/M, z11.h, z29.h\n" + "ld1sb { z2.h }, p4/Z, [x4, #4, MUL VL]\n" + ".inst 0x454a1042 // ssublb z2.h, z2.b, z10.b\n" + ".inst 0x449d40a9 // smlalb z9.s, p4/M, z5.h, z29.h\n" + ".inst 0x449d4387 // smlalb z7.s, p4/M, z28.h, z29.h\n" + ".inst 0x44984637 // smlalt z23.s, p4/M, z17.h, z24.h\n" + ".inst 0x4493406e // smlalb z14.s, p4/M, z3.h, z19.h\n" + "ld1sb { z17.h }, p3/Z, [x27, x2]\n" + ".inst 0x455e1231 // ssublb z17.h, z17.b, z30.b\n" + ".inst 0x449d4572 // smlalt z18.s, p4/M, z11.h, z29.h\n" + ".inst 0x449d44b4 // smlalt z20.s, p4/M, z5.h, z29.h\n" + "ld1sb { z11.h }, p4/Z, [x4, #5, MUL VL]\n" + ".inst 0x454a116b // ssublb z11.h, z11.b, z10.b\n" + ".inst 0x449d4781 // smlalt z1.s, p4/M, z28.h, z29.h\n" + ".inst 0x44984066 // smlalb z6.s, p4/M, z3.h, z24.h\n" + "ld1sb { z29.h }, p3/Z, [x26, x2]\n" + ".inst 0x455e13bd // ssublb z29.h, z29.b, z30.b\n" + ".inst 0x44984209 // smlalb z9.s, p4/M, z16.h, z24.h\n" + ".inst 0x44984347 // smlalb z7.s, p4/M, z26.h, z24.h\n" + ".inst 0x44934477 // smlalt z23.s, p4/M, z3.h, z19.h\n" + ".inst 0x4495408e // smlalb z14.s, p4/M, z4.h, z21.h\n" + ".inst 0x44984472 // smlalt z18.s, p4/M, z3.h, z24.h\n" + ".inst 0x44984614 // smlalt z20.s, p4/M, z16.h, z24.h\n" + "ld1sb { z3.h }, p3/Z, [x25, x2]\n" + ".inst 0x455e1063 // ssublb z3.h, z3.b, z30.b\n" + ".inst 0x44984741 // smlalt z1.s, p4/M, z26.h, z24.h\n" + ".inst 0x44934086 // smlalb z6.s, p4/M, z4.h, z19.h\n" + "ld1sb { z24.h }, p4/Z, [x4, #6, MUL VL]\n" + ".inst 0x454a1318 // ssublb z24.h, z24.b, z10.b\n" + ".inst 0x44934349 // smlalb z9.s, p4/M, z26.h, z19.h\n" + ".inst 0x44934107 // smlalb z7.s, p4/M, z8.h, z19.h\n" + ".inst 0x44954497 // smlalt z23.s, p4/M, z4.h, z21.h\n" + ".inst 0x4496436e // smlalb z14.s, p4/M, z27.h, z22.h\n" + ".inst 0x44934492 // smlalt z18.s, p4/M, z4.h, z19.h\n" + ".inst 0x44934754 // smlalt z20.s, p4/M, z26.h, z19.h\n" + "ld1sb { z4.h }, p3/Z, [x24, x2]\n" + ".inst 0x455e1084 // ssublb z4.h, z4.b, z30.b\n" + ".inst 0x44934501 // smlalt z1.s, p4/M, z8.h, z19.h\n" + ".inst 0x44954366 // smlalb z6.s, p4/M, z27.h, z21.h\n" + "ld1sb { z19.h }, p4/Z, [x4, #7, MUL VL]\n" + "inch x4, ALL, MUL #8\n" + ".inst 0x44954109 // smlalb z9.s, p4/M, z8.h, z21.h\n" + ".inst 0x449543e7 // smlalb z7.s, p4/M, z31.h, z21.h\n" + ".inst 0x454a1273 // ssublb z19.h, z19.b, z10.b\n" + ".inst 0x44964777 // smlalt z23.s, p4/M, z27.h, z22.h\n" + ".inst 0x449940ae // smlalb z14.s, p4/M, z5.h, z25.h\n" + ".inst 0x44954772 // smlalt z18.s, p4/M, z27.h, z21.h\n" + ".inst 0x44954514 // smlalt z20.s, p4/M, z8.h, z21.h\n" + "ld1sb { z27.h }, p3/Z, [x23, x2]\n" + ".inst 0x455e137b // ssublb z27.h, z27.b, z30.b\n" + ".inst 0x449547e1 // smlalt z1.s, p4/M, z31.h, z21.h\n" + ".inst 0x449640a6 // smlalb z6.s, p4/M, z5.h, z22.h\n" + "ld1sb { z21.h }, p4/Z, [x4]\n" + ".inst 0x454a12b5 // ssublb z21.h, z21.b, z10.b\n" + ".inst 0x449643e9 // smlalb z9.s, p4/M, z31.h, z22.h\n" + ".inst 0x44964007 // smlalb z7.s, p4/M, z0.h, z22.h\n" + "inch x4\n" + ".inst 0x449944b7 // smlalt z23.s, p4/M, z5.h, z25.h\n" + ".inst 0x4482420e // smlalb z14.s, p4/M, z16.h, z2.h\n" + ".inst 0x449644b2 // smlalt z18.s, p4/M, z5.h, z22.h\n" + ".inst 0x449647f4 // smlalt z20.s, p4/M, z31.h, z22.h\n" + "ld1sb { z5.h }, p3/Z, [x22, x2]\n" + ".inst 0x455e10a5 // ssublb z5.h, z5.b, z30.b\n" + ".inst 0x44964401 // smlalt z1.s, p4/M, z0.h, z22.h\n" + ".inst 0x44994386 // smlalb z6.s, p4/M, z28.h, z25.h\n" + "ld1w { z22.s }, p2/Z, [x15]\n" + ".inst 0x44994009 // smlalb z9.s, p4/M, z0.h, z25.h\n" + ".inst 0x44994227 // smlalb z7.s, p4/M, z17.h, z25.h\n" + ".inst 0x44824617 // smlalt z23.s, p4/M, z16.h, z2.h\n" + ".inst 0x448b434e // smlalb z14.s, p4/M, z26.h, z11.h\n" + "ld1w { z16.s }, p1/Z, [x15, #1, MUL VL]\n" + "addvl x15, x15, #2\n" + ".inst 0x44994792 // smlalt z18.s, p4/M, z28.h, z25.h\n" + ".inst 0x44994414 // smlalt z20.s, p4/M, z0.h, z25.h\n" + "ld1sb { z28.h }, p3/Z, [x20, x2]\n" + ".inst 0x455e139c // ssublb z28.h, z28.b, z30.b\n" + ".inst 0x44994621 // smlalt z1.s, p4/M, z17.h, z25.h\n" + ".inst 0x44824346 // smlalb z6.s, p4/M, z26.h, z2.h\n" + "uzp1 z25.s, z22.s, z16.s\n" + "inch x2\n" + ".inst 0x448243a9 // smlalb z9.s, p4/M, z29.h, z2.h\n" + ".inst 0x44824067 // smlalb z7.s, p4/M, z3.h, z2.h\n" + "uzp2 z16.s, z22.s, z16.s\n" + "ld1w { z22.s }, p2/Z, [x14]\n" + ".inst 0x448b4757 // smlalt z23.s, p4/M, z26.h, z11.h\n" + ".inst 0x4498410e // smlalb z14.s, p4/M, z8.h, z24.h\n" + "mov x20, x2\n" "incw x20\n" - ".inst 0x4484478a // smlalt z10.s, p4/M, z28.h, z4.h\n" - ".inst 0x44834787 // smlalt z7.s, p4/M, z28.h, z3.h\n" - ".inst 0x04bd754a // sqrdmulh z10.s, z10.s, z29.s\n" - "whilelt p2.s, x0, x1\n" - ".inst 0x44834730 // smlalt z16.s, p4/M, z25.h, z3.h\n" - ".inst 0x44834705 // smlalt z5.s, p4/M, z24.h, z3.h\n" - "and z3.d, z10.d, z22.d\n" - "whilelt p1.s, x20, x1\n" - ".inst 0x44844354 // smlalb z20.s, p4/M, z26.h, z4.h\n" - ".inst 0x44844308 // smlalb z8.s, p4/M, z24.h, z4.h\n" - ".inst 0x04a97694 // sqrdmulh z20.s, z20.s, z9.s\n" - "whilelt p3.h, x0, x1\n" - ".inst 0x44844366 // smlalb z6.s, p4/M, z27.h, z4.h\n" - ".inst 0x44844747 // smlalt z7.s, p4/M, z26.h, z4.h\n" - ".inst 0x04a97508 // sqrdmulh z8.s, z8.s, z9.s\n" - ".inst 0x44844710 // smlalt z16.s, p4/M, z24.h, z4.h\n" - ".inst 0x44844765 // smlalt z5.s, p4/M, z27.h, z4.h\n" - ".inst 0x04a974c6 // sqrdmulh z6.s, z6.s, z9.s\n" - "sqadd z14.s, z14.s, z21.s\n" + ".inst 0x44824752 // smlalt z18.s, p4/M, z26.h, z2.h\n" + ".inst 0x448247b4 // smlalt z20.s, p4/M, z29.h, z2.h\n" + "ld1w { z26.s }, p1/Z, [x14, #1, MUL VL]\n" + "uzp1 z29.s, z22.s, z26.s\n" + ".inst 0x44824461 // smlalt z1.s, p4/M, z3.h, z2.h\n" + ".inst 0x448b4106 // smlalb z6.s, p4/M, z8.h, z11.h\n" + "uzp2 z22.s, z22.s, z26.s\n" + "whilelt p2.s, x2, x3\n" + ".inst 0x448b4069 // smlalb z9.s, p4/M, z3.h, z11.h\n" + ".inst 0x448b4087 // smlalb z7.s, p4/M, z4.h, z11.h\n" + "whilelt p1.s, x20, x3\n" + "whilelt p3.h, x2, x3\n" + ".inst 0x44984517 // smlalt z23.s, p4/M, z8.h, z24.h\n" + ".inst 0x449343ee // smlalb z14.s, p4/M, z31.h, z19.h\n" + "addvl x14, x14, #2\n" + ".inst 0x448b4512 // smlalt z18.s, p4/M, z8.h, z11.h\n" + ".inst 0x448b4474 // smlalt z20.s, p4/M, z3.h, z11.h\n" + ".inst 0x448b4481 // smlalt z1.s, p4/M, z4.h, z11.h\n" + ".inst 0x449843e6 // smlalb z6.s, p4/M, z31.h, z24.h\n" + ".inst 0x44984089 // smlalb z9.s, p4/M, z4.h, z24.h\n" + ".inst 0x44984367 // smlalb z7.s, p4/M, z27.h, z24.h\n" + ".inst 0x449347f7 // smlalt z23.s, p4/M, z31.h, z19.h\n" + ".inst 0x4495400e // smlalb z14.s, p4/M, z0.h, z21.h\n" + ".inst 0x04b975ce // sqrdmulh z14.s, z14.s, z25.s\n" + ".inst 0x449847f2 // smlalt z18.s, p4/M, z31.h, z24.h\n" + ".inst 0x44984494 // smlalt z20.s, p4/M, z4.h, z24.h\n" + "and z3.d, z14.d, z29.d\n" + ".inst 0x44984761 // smlalt z1.s, p4/M, z27.h, z24.h\n" + ".inst 0x44934006 // smlalb z6.s, p4/M, z0.h, z19.h\n" + "asr z3.s, z3.s, #0x1f\n" + ".inst 0x44934369 // smlalb z9.s, p4/M, z27.h, z19.h\n" + ".inst 0x449340a7 // smlalb z7.s, p4/M, z5.h, z19.h\n" + "sqadd z14.s, z14.s, z3.s\n" + ".inst 0x448293ae // srshl z14.s, p4/M, z14.s, z29.s\n" + ".inst 0x44954417 // smlalt z23.s, p4/M, z0.h, z21.h\n" + ".inst 0x44934412 // smlalt z18.s, p4/M, z0.h, z19.h\n" + ".inst 0x04b076f7 // sqrdmulh z23.s, z23.s, z16.s\n" + ".inst 0x44934774 // smlalt z20.s, p4/M, z27.h, z19.h\n" + ".inst 0x449344a1 // smlalt z1.s, p4/M, z5.h, z19.h\n" + "and z31.d, z23.d, z22.d\n" + ".inst 0x44954226 // smlalb z6.s, p4/M, z17.h, z21.h\n" + ".inst 0x449540a9 // smlalb z9.s, p4/M, z5.h, z21.h\n" + ".inst 0x04b974c6 // sqrdmulh z6.s, z6.s, z25.s\n" + ".inst 0x44954387 // smlalb z7.s, p4/M, z28.h, z21.h\n" + ".inst 0x44954632 // smlalt z18.s, p4/M, z17.h, z21.h\n" + ".inst 0x04b97529 // sqrdmulh z9.s, z9.s, z25.s\n" + ".inst 0x449544b4 // smlalt z20.s, p4/M, z5.h, z21.h\n" + ".inst 0x44954781 // smlalt z1.s, p4/M, z28.h, z21.h\n" + ".inst 0x04b974e7 // sqrdmulh z7.s, z7.s, z25.s\n" + "asr z31.s, z31.s, #0x1f\n" + "and z3.d, z6.d, z29.d\n" + ".inst 0x04b07652 // sqrdmulh z18.s, z18.s, z16.s\n" + "and z0.d, z9.d, z29.d\n" + ".inst 0x04b07694 // sqrdmulh z20.s, z20.s, z16.s\n" + "and z19.d, z7.d, z29.d\n" + ".inst 0x04b07421 // sqrdmulh z1.s, z1.s, z16.s\n" + "sqadd z23.s, z23.s, z31.s\n" + ".inst 0x448292d7 // srshl z23.s, p4/M, z23.s, z22.s\n" "asr z3.s, z3.s, #0x1f\n" - ".inst 0x448292ee // srshl z14.s, p4/M, z14.s, z23.s\n" - "and z19.d, z20.d, z23.d\n" - ".inst 0x04bd74e7 // sqrdmulh z7.s, z7.s, z29.s\n" - "and z18.d, z8.d, z23.d\n" - ".inst 0x04bd7610 // sqrdmulh z16.s, z16.s, z29.s\n" - "and z21.d, z6.d, z23.d\n" - ".inst 0x04bd74a5 // sqrdmulh z5.s, z5.s, z29.s\n" - "sqadd z10.s, z10.s, z3.s\n" + "and z21.d, z18.d, z22.d\n" + "asr z0.s, z0.s, #0x1f\n" + "and z17.d, z20.d, z22.d\n" "asr z19.s, z19.s, #0x1f\n" - ".inst 0x448292ca // srshl z10.s, p4/M, z10.s, z22.s\n" - "and z1.d, z7.d, z22.d\n" - "asr z18.s, z18.s, #0x1f\n" - "and z2.d, z16.d, z22.d\n" + "and z16.d, z1.d, z22.d\n" + "sqadd z6.s, z6.s, z3.s\n" "asr z21.s, z21.s, #0x1f\n" - "and z3.d, z5.d, z22.d\n" - "sqadd z20.s, z20.s, z19.s\n" - ".inst 0x448292f4 // srshl z20.s, p4/M, z20.s, z23.s\n" - "asr z1.s, z1.s, #0x1f\n" - "sqadd z8.s, z8.s, z18.s\n" - ".inst 0x448292e8 // srshl z8.s, p4/M, z8.s, z23.s\n" - "asr z2.s, z2.s, #0x1f\n" - "sqadd z6.s, z6.s, z21.s\n" - ".inst 0x448292e6 // srshl z6.s, p4/M, z6.s, z23.s\n" - "asr z3.s, z3.s, #0x1f\n" - "sqadd z7.s, z7.s, z1.s\n" - ".inst 0x448292c7 // srshl z7.s, p4/M, z7.s, z22.s\n" - "sqadd z16.s, z16.s, z2.s\n" - "sqadd z5.s, z5.s, z3.s\n" - ".inst 0x448292d0 // srshl z16.s, p4/M, z16.s, z22.s\n" - ".inst 0x448292c5 // srshl z5.s, p4/M, z5.s, z22.s\n" + ".inst 0x448293a6 // srshl z6.s, p4/M, z6.s, z29.s\n" + "sqadd z9.s, z9.s, z0.s\n" + "asr z17.s, z17.s, #0x1f\n" + ".inst 0x448293a9 // srshl z9.s, p4/M, z9.s, z29.s\n" + "sqadd z7.s, z7.s, z19.s\n" + "asr z16.s, z16.s, #0x1f\n" + ".inst 0x448293a7 // srshl z7.s, p4/M, z7.s, z29.s\n" + "sqadd z18.s, z18.s, z21.s\n" + "sqadd z20.s, z20.s, z17.s\n" + ".inst 0x448292d2 // srshl z18.s, p4/M, z18.s, z22.s\n" + ".inst 0x448292d4 // srshl z20.s, p4/M, z20.s, z22.s\n" + "sqadd z1.s, z1.s, z16.s\n" ".inst 0x453041ce // sqxtnb z14.h, z14.s\n" - ".inst 0x45304294 // sqxtnb z20.h, z20.s\n" - ".inst 0x45304108 // sqxtnb z8.h, z8.s\n" + ".inst 0x448292c1 // srshl z1.s, p4/M, z1.s, z22.s\n" ".inst 0x453040c6 // sqxtnb z6.h, z6.s\n" - ".inst 0x4530454e // sqxtnt z14.h, z10.s\n" - ".inst 0x453044f4 // sqxtnt z20.h, z7.s\n" - ".inst 0x45304608 // sqxtnt z8.h, z16.s\n" - ".inst 0x453044a6 // sqxtnt z6.h, z5.s\n" - "sqadd z14.h, z14.h, z12.h\n" - "sqadd z20.h, z20.h, z12.h\n" - "smax z14.h, p4/M, z14.h, z13.h\n" - "smax z20.h, p4/M, z20.h, z13.h\n" - "sqadd z8.h, z8.h, z12.h\n" - "sqadd z6.h, z6.h, z12.h\n" - "smax z8.h, p4/M, z8.h, z13.h\n" - "smax z6.h, p4/M, z6.h, z13.h\n" - "smin z14.h, p4/M, z14.h, z11.h\n" - "smin z20.h, p4/M, z20.h, z11.h\n" - "st1b { z14.h }, p0, [x3, x8]\n" - "smin z8.h, p4/M, z8.h, z11.h\n" - "smin z6.h, p4/M, z6.h, z11.h\n" - "st1b { z20.h }, p0, [x4, x8]\n" - "st1b { z8.h }, p0, [x5, x8]\n" - "st1b { z6.h }, p0, [x6, x8]\n" - "ld1w { z30.s }, p2/Z, [x14]\n" - "ld1w { z16.s }, p1/Z, [x14, #1, MUL VL]\n" - "uzp1 z14.s, z30.s, z16.s\n" - "ld1sb { z0.h }, p4/Z, [x2]\n" - "ld1sb { z1.h }, p4/Z, [x2, #1, MUL VL]\n" - "uzp2 z10.s, z30.s, z16.s\n" - "addvl x14, x14, #2\n" - "ld1sb { z2.h }, p4/Z, [x2, #2, MUL VL]\n" - "ld1sb { z3.h }, p4/Z, [x2, #3, MUL VL]\n" - "inch x8\n" - "str x14, [%x[params], %[offsetof_Params_bias]]\n" - "ld1sb { z4.h }, p4/Z, [x2, #4, MUL VL]\n" - "ldp x9, x28, [x7, #0x0]\n" - "mov z20.d, z14.d\n" - "mov z7.d, z10.d\n" - "ldp x27, x26, [x7, #0x10]\n" - "ldp x25, x24, [x7, #0x20]\n" - "mov z8.d, z14.d\n" - "mov z16.d, z10.d\n" - "ldp x23, x22, [x7, #0x30]\n" - "ldp x21, x20, [x7, #0x40]\n" + ".inst 0x45304129 // sqxtnb z9.h, z9.s\n" + ".inst 0x453040e7 // sqxtnb z7.h, z7.s\n" + ".inst 0x453046ee // sqxtnt z14.h, z23.s\n" + ".inst 0x45304646 // sqxtnt z6.h, z18.s\n" + ".inst 0x45304689 // sqxtnt z9.h, z20.s\n" + ".inst 0x45304427 // sqxtnt z7.h, z1.s\n" + "sqadd z14.h, z14.h, z15.h\n" + "smax z14.h, p4/M, z14.h, z12.h\n" + "smin z14.h, p4/M, z14.h, z13.h\n" + "sqadd z6.h, z6.h, z15.h\n" + "sqadd z9.h, z9.h, z15.h\n" + "smax z6.h, p4/M, z6.h, z12.h\n" + "smax z9.h, p4/M, z9.h, z12.h\n" + "sqadd z7.h, z7.h, z15.h\n" + "smax z7.h, p4/M, z7.h, z12.h\n" + "smin z6.h, p4/M, z6.h, z13.h\n" + "st1b { z14.h }, p0, [x5, x16]\n" + "smin z9.h, p4/M, z9.h, z13.h\n" + "smin z7.h, p4/M, z7.h, z13.h\n" + "st1b { z6.h }, p0, [x6, x16]\n" + "st1b { z9.h }, p0, [x7, x16]\n" + "st1b { z7.h }, p0, [x8, x16]\n" + "ld1w { z17.s }, p2/Z, [x21]\n" + "ld1w { z16.s }, p1/Z, [x21, #1, MUL VL]\n" + "uzp1 z14.s, z17.s, z16.s\n" + "ld1sb { z26.h }, p4/Z, [x4]\n" + "ld1sb { z8.h }, p4/Z, [x4, #1, MUL VL]\n" + "uzp2 z23.s, z17.s, z16.s\n" + "addvl x21, x21, #2\n" + "ld1sb { z16.h }, p4/Z, [x4, #2, MUL VL]\n" + "ld1sb { z21.h }, p4/Z, [x4, #3, MUL VL]\n" + "inch x16\n" + "str x21, [%x[params], %[offsetof_Params_bias]]\n" + "ld1sb { z17.h }, p4/Z, [x4, #4, MUL VL]\n" + "ldp x9, x28, [x17, #0x0]\n" "mov z6.d, z14.d\n" - "mov z5.d, z10.d\n" - "ld1sb { z31.h }, p3/Z, [x9, x0]\n" - "ld1sb { z30.h }, p3/Z, [x28, x0]\n" - ".inst 0x45511000 // ssublb z0.h, z0.b, z17.b\n" - ".inst 0x45511021 // ssublb z1.h, z1.b, z17.b\n" - "ld1sb { z29.h }, p3/Z, [x27, x0]\n" - "ld1sb { z28.h }, p3/Z, [x26, x0]\n" - ".inst 0x45511042 // ssublb z2.h, z2.b, z17.b\n" - ".inst 0x45511063 // ssublb z3.h, z3.b, z17.b\n" - "ld1sb { z27.h }, p3/Z, [x25, x0]\n" - "ld1sb { z23.h }, p3/Z, [x24, x0]\n" - ".inst 0x45511084 // ssublb z4.h, z4.b, z17.b\n" - ".inst 0x454f13ff // ssublb z31.h, z31.b, z15.b\n" - "ld1sb { z25.h }, p3/Z, [x23, x0]\n" - "ld1sb { z24.h }, p3/Z, [x22, x0]\n" - ".inst 0x454f13de // ssublb z30.h, z30.b, z15.b\n" - ".inst 0x454f13bd // ssublb z29.h, z29.b, z15.b\n" - "ld1sb { z26.h }, p3/Z, [x21, x0]\n" - "ld1sb { z22.h }, p3/Z, [x20, x0]\n" - ".inst 0x454f139c // ssublb z28.h, z28.b, z15.b\n" - ".inst 0x454f137b // ssublb z27.h, z27.b, z15.b\n" - ".inst 0x454f12f7 // ssublb z23.h, z23.b, z15.b\n" - ".inst 0x454f1339 // ssublb z25.h, z25.b, z15.b\n" - ".inst 0x454f1318 // ssublb z24.h, z24.b, z15.b\n" - ".inst 0x454f135a // ssublb z26.h, z26.b, z15.b\n" - ".inst 0x454f12d6 // ssublb z22.h, z22.b, z15.b\n" + "mov z18.d, z23.d\n" + "ldp x27, x26, [x17, #0x10]\n" + "ldp x25, x24, [x17, #0x20]\n" + "mov z9.d, z14.d\n" + "mov z20.d, z23.d\n" + "ldp x23, x22, [x17, #0x30]\n" + "ldp x21, x20, [x17, #0x40]\n" + "mov z7.d, z14.d\n" + "mov z1.d, z23.d\n" + "ld1sb { z22.h }, p3/Z, [x9, x2]\n" + "ld1sb { z2.h }, p3/Z, [x28, x2]\n" + ".inst 0x454a135a // ssublb z26.h, z26.b, z10.b\n" + ".inst 0x454a1108 // ssublb z8.h, z8.b, z10.b\n" + "ld1sb { z11.h }, p3/Z, [x27, x2]\n" + "ld1sb { z3.h }, p3/Z, [x26, x2]\n" + ".inst 0x454a1210 // ssublb z16.h, z16.b, z10.b\n" + ".inst 0x454a12b5 // ssublb z21.h, z21.b, z10.b\n" + "ld1sb { z29.h }, p3/Z, [x25, x2]\n" + "ld1sb { z4.h }, p3/Z, [x24, x2]\n" + ".inst 0x454a1231 // ssublb z17.h, z17.b, z10.b\n" + ".inst 0x455e12d6 // ssublb z22.h, z22.b, z30.b\n" + "ld1sb { z31.h }, p3/Z, [x23, x2]\n" + "ld1sb { z0.h }, p3/Z, [x22, x2]\n" + ".inst 0x455e1042 // ssublb z2.h, z2.b, z30.b\n" + ".inst 0x455e116b // ssublb z11.h, z11.b, z30.b\n" + "ld1sb { z19.h }, p3/Z, [x21, x2]\n" + "ld1sb { z28.h }, p3/Z, [x20, x2]\n" + ".inst 0x455e1063 // ssublb z3.h, z3.b, z30.b\n" + ".inst 0x455e13bd // ssublb z29.h, z29.b, z30.b\n" + ".inst 0x455e1084 // ssublb z4.h, z4.b, z30.b\n" + ".inst 0x455e13ff // ssublb z31.h, z31.b, z30.b\n" + ".inst 0x455e1000 // ssublb z0.h, z0.b, z30.b\n" + ".inst 0x455e1273 // ssublb z19.h, z19.b, z30.b\n" + ".inst 0x455e139c // ssublb z28.h, z28.b, z30.b\n" "b.any 1b\n" : : [offsetof_Params_bias] "I" (offsetof(Params, bias)), [offsetof_Params_inptrs] "I" (offsetof(Params, inptrs)), [offsetof_Params_n_channels] "I" (offsetof(Params, n_channels)), [offsetof_Params_outptrs] "I" (offsetof(Params, outptrs)), [offsetof_Params_requant] "I" (offsetof(Params, requant)), [offsetof_Params_requant_muls] "I" (offsetof(Params, requant_muls)), [offsetof_Params_requant_shifts] "I" (offsetof(Params, requant_shifts)), [offsetof_Params_weights] "I" (offsetof(Params, weights)), [offsetof_Requantize32_a_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [params] "r" (¶ms) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst.hpp index 2e8c2019db537d699898557483a4ad7064d90cff..abc09ee5a330a1d5d7466438f60a8211c25709ec 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst/generic.cpp index 6fba4d47d2d088a227dc8a6e2d60713d89180c54..274b29dcfc11ad2f332c547e8c3128b6ade7d6ad 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst/generic.cpp @@ -47,285 +47,285 @@ void sve_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst_impl "ldr x20, [%x[inptrs], #0x10]\n" "ldr x22, [%x[inptrs], #0x20]\n" "ldr x21, [%x[inptrs], #0x0]\n" - "mov z15.b, #0x1\n" - "lsr z15.s, z15.s, #0x8\n" + "mov z13.b, #0x1\n" + "lsr z13.s, z13.s, #0x8\n" "ld1b { z1.b }, p0/Z, [x23]\n" "ld1b { z2.b }, p0/Z, [x20]\n" - "mov z30.d, z1.d\n" - "mov z29.d, z1.d\n" + "mov z8.d, z1.d\n" + "mov z27.d, z1.d\n" "ldr x20, [%x[inptrs], #0x18]\n" "ld1b { z4.b }, p0/Z, [x22]\n" - "mov z28.d, z1.d\n" - "mov z27.d, z2.d\n" + "mov z31.d, z1.d\n" + "mov z28.d, z2.d\n" "ld1b { z0.b }, p0/Z, [x21]\n" + "mov z30.d, z2.d\n" "mov z26.d, z2.d\n" - "mov z25.d, z2.d\n" "ld1b { z3.b }, p0/Z, [x20]\n" - "mov z24.d, z4.d\n" - "mov z23.d, z4.d\n" - "ptrue p2.b\n" - "ld1rw { z14.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" "mov z22.d, z4.d\n" - "ext z30.b, z30.b, z30.b, #0x2\n" + "mov z10.d, z4.d\n" + "ptrue p2.b\n" + "ld1rw { z11.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" + "mov z18.d, z4.d\n" + "ext z8.b, z8.b, z8.b, #0x2\n" "lsl x10, %x[n_channels], #0x2\n" - "neg z14.s, p2/M, z14.s\n" - "ext z29.b, z29.b, z29.b, #0x4\n" - "ext z28.b, z28.b, z28.b, #0x6\n" + "neg z11.s, p2/M, z11.s\n" + "ext z27.b, z27.b, z27.b, #0x4\n" + "ext z31.b, z31.b, z31.b, #0x6\n" "mov x9, #0x0\n" - "whilelt p1.b, x9, x10\n" - "ext z27.b, z27.b, z27.b, #0x2\n" - "ext z26.b, z26.b, z26.b, #0x4\n" - "ld1w { z13.s }, p1/Z, [%x[params]]\n" + "whilelt p0.b, x9, x10\n" + "ext z28.b, z28.b, z28.b, #0x2\n" + "ext z30.b, z30.b, z30.b, #0x4\n" + "ld1w { z14.s }, p0/Z, [%x[params]]\n" "mov x28, #0x0\n" - "ext z25.b, z25.b, z25.b, #0x6\n" - "ext z24.b, z24.b, z24.b, #0x2\n" + "ext z26.b, z26.b, z26.b, #0x6\n" + "ext z22.b, z22.b, z22.b, #0x2\n" "ldp x27, x26, [%x[outptrs], #0x0]\n" "ldp x25, x24, [%x[outptrs], #0x10]\n" - "ext z23.b, z23.b, z23.b, #0x4\n" - "ext z22.b, z22.b, z22.b, #0x6\n" + "ext z10.b, z10.b, z10.b, #0x4\n" + "ext z18.b, z18.b, z18.b, #0x6\n" "ldp x23, x22, [%x[outptrs], #0x20]\n" "ldp x21, x20, [%x[outptrs], #0x30]\n" "mov z21.d, z0.d\n" "mov z20.d, z0.d\n" - "ld1rw { z12.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" - "ld1rw { z11.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" + "ld1rw { z9.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" + "ld1rw { z15.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" "mov z19.d, z0.d\n" - "mov z18.d, z3.d\n" - "ld1rw { z10.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" - "ld1b { z5.b }, p1/Z, [%x[params], #1, MUL VL]\n" + "mov z24.d, z3.d\n" + "ld1rw { z12.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" + "ld1b { z5.b }, p0/Z, [%x[params], #1, MUL VL]\n" "mov z17.d, z3.d\n" "mov z16.d, z3.d\n" - "ld1b { z6.b }, p1/Z, [%x[params], #2, MUL VL]\n" - "ld1b { z7.b }, p1/Z, [%x[params], #3, MUL VL]\n" + "ld1b { z6.b }, p0/Z, [%x[params], #2, MUL VL]\n" + "ld1b { z7.b }, p0/Z, [%x[params], #3, MUL VL]\n" "ext z21.b, z21.b, z21.b, #0x2\n" "ext z20.b, z20.b, z20.b, #0x4\n" "addvl %x[params], %x[params], #4\n" "ext z19.b, z19.b, z19.b, #0x6\n" - "zip1 z1.s, z1.s, z29.s\n" - "zip1 z30.s, z30.s, z28.s\n" - "zip1 z2.s, z2.s, z26.s\n" - "zip1 z27.s, z27.s, z25.s\n" - "ext z18.b, z18.b, z18.b, #0x2\n" + "zip1 z1.s, z1.s, z27.s\n" + "zip1 z8.s, z8.s, z31.s\n" + "zip1 z2.s, z2.s, z30.s\n" + "zip1 z28.s, z28.s, z26.s\n" + "ext z24.b, z24.b, z24.b, #0x2\n" "ext z17.b, z17.b, z17.b, #0x4\n" "ext z16.b, z16.b, z16.b, #0x6\n" - "zip1 z4.s, z4.s, z23.s\n" - "zip1 z24.s, z24.s, z22.s\n" + "zip1 z4.s, z4.s, z10.s\n" + "zip1 z22.s, z22.s, z18.s\n" "zip1 z0.s, z0.s, z20.s\n" "zip1 z21.s, z21.s, z19.s\n" - "zip1 z1.s, z1.s, z30.s\n" - "zip1 z2.s, z2.s, z27.s\n" + "zip1 z1.s, z1.s, z8.s\n" + "zip1 z2.s, z2.s, z28.s\n" "zip1 z3.s, z3.s, z17.s\n" - "zip1 z18.s, z18.s, z16.s\n" - "zip1 z4.s, z4.s, z24.s\n" + "zip1 z24.s, z24.s, z16.s\n" + "zip1 z4.s, z4.s, z22.s\n" "zip1 z0.s, z0.s, z21.s\n" "mov z1.q, z1.q[0]\n" "mov z2.q, z2.q[0]\n" - "zip1 z3.s, z3.s, z18.s\n" + "zip1 z3.s, z3.s, z24.s\n" "mov z4.q, z4.q[0]\n" "mov z24.s, #0x0\n" "mov z25.s, #0x0\n" - "sdot z24.s, z15.b, z1.b[0]\n" + "sdot z24.s, z13.b, z1.b[0]\n" "mov z23.s, #0x0\n" "mov z22.s, #0x0\n" - "sdot z25.s, z15.b, z1.b[1]\n" + "sdot z25.s, z13.b, z1.b[1]\n" "mov z21.s, #0x0\n" - "mov z20.s, #0x0\n" - "sdot z23.s, z15.b, z1.b[2]\n" - "mov z9.s, #0x0\n" - "mov z8.s, #0x0\n" - "sdot z22.s, z15.b, z1.b[3]\n" "mov z19.s, #0x0\n" + "sdot z23.s, z13.b, z1.b[2]\n" + "mov z10.s, #0x0\n" + "mov z8.s, #0x0\n" + "sdot z22.s, z13.b, z1.b[3]\n" + "mov z20.s, #0x0\n" "mov z18.s, #0x0\n" - "sdot z21.s, z15.b, z2.b[0]\n" + "sdot z21.s, z13.b, z2.b[0]\n" "mov z17.s, #0x0\n" "mov z16.s, #0x0\n" - "sdot z20.s, z15.b, z2.b[1]\n" - "sdot z9.s, z15.b, z2.b[2]\n" - "sdot z8.s, z15.b, z2.b[3]\n" + "sdot z19.s, z13.b, z2.b[1]\n" + "sdot z10.s, z13.b, z2.b[2]\n" + "sdot z8.s, z13.b, z2.b[3]\n" "mov z0.q, z0.q[0]\n" - "sdot z19.s, z15.b, z4.b[0]\n" - "sdot z18.s, z15.b, z4.b[1]\n" + "sdot z20.s, z13.b, z4.b[0]\n" + "sdot z18.s, z13.b, z4.b[1]\n" "mov z3.q, z3.q[0]\n" - "sdot z17.s, z15.b, z4.b[2]\n" - "sdot z16.s, z15.b, z4.b[3]\n" + "sdot z17.s, z13.b, z4.b[2]\n" + "sdot z16.s, z13.b, z4.b[3]\n" "mov z31.s, #0x0\n" "mov z30.s, #0x0\n" - "mov z29.s, #0x0\n" - "sdot z31.s, z15.b, z0.b[0]\n" + "mov z26.s, #0x0\n" + "sdot z31.s, z13.b, z0.b[0]\n" + "mov z27.s, #0x0\n" "mov z28.s, #0x0\n" - "sdot z30.s, z15.b, z0.b[1]\n" - "sdot z29.s, z15.b, z0.b[2]\n" - "sdot z28.s, z15.b, z0.b[3]\n" + "sdot z30.s, z13.b, z0.b[1]\n" + "mov z29.s, #0x0\n" + "sdot z26.s, z13.b, z0.b[2]\n" + "sdot z27.s, z13.b, z0.b[3]\n" + "sdot z28.s, z13.b, z3.b[0]\n" + "sdot z29.s, z13.b, z3.b[1]\n" "add z24.s, z24.s, z21.s\n" - "add z25.s, z25.s, z20.s\n" - "add z26.s, z23.s, z9.s\n" - "add z27.s, z22.s, z8.s\n" - "add z23.s, z19.s, z21.s\n" - "mov z22.s, #0x0\n" - "sdot z22.s, z15.b, z3.b[0]\n" - "add z21.s, z18.s, z20.s\n" + "add z25.s, z25.s, z19.s\n" + "add z23.s, z23.s, z10.s\n" + "add z22.s, z22.s, z8.s\n" + "add z21.s, z20.s, z21.s\n" "mov z20.s, #0x0\n" - "sdot z20.s, z15.b, z3.b[1]\n" - "add z19.s, z17.s, z9.s\n" + "sdot z20.s, z13.b, z3.b[2]\n" + "add z19.s, z18.s, z19.s\n" "mov z18.s, #0x0\n" - "sdot z18.s, z15.b, z3.b[2]\n" - "add z17.s, z16.s, z8.s\n" - "mov z16.s, #0x0\n" - "sdot z16.s, z15.b, z3.b[3]\n" + "sdot z18.s, z13.b, z3.b[3]\n" + "add z17.s, z17.s, z10.s\n" + "add z16.s, z16.s, z8.s\n" "add z24.s, z24.s, z31.s\n" "add z25.s, z25.s, z30.s\n" - "mul z24.s, p2/M, z24.s, z14.s\n" - "mul z25.s, p2/M, z25.s, z14.s\n" - "add z26.s, z26.s, z29.s\n" - "add z27.s, z27.s, z28.s\n" - "mul z26.s, p2/M, z26.s, z14.s\n" - "mul z27.s, p2/M, z27.s, z14.s\n" - "add z28.s, z23.s, z22.s\n" - "add z29.s, z21.s, z20.s\n" - "mul z28.s, p2/M, z28.s, z14.s\n" - "mul z29.s, p2/M, z29.s, z14.s\n" - "add z30.s, z19.s, z18.s\n" - "add z31.s, z17.s, z16.s\n" - "mul z30.s, p2/M, z30.s, z14.s\n" - "mul z31.s, p2/M, z31.s, z14.s\n" + "mul z24.s, p2/M, z24.s, z11.s\n" + "mul z25.s, p2/M, z25.s, z11.s\n" + "add z26.s, z23.s, z26.s\n" + "add z27.s, z22.s, z27.s\n" + "mul z26.s, p2/M, z26.s, z11.s\n" + "mul z27.s, p2/M, z27.s, z11.s\n" + "add z28.s, z21.s, z28.s\n" + "add z29.s, z19.s, z29.s\n" + "mul z28.s, p2/M, z28.s, z11.s\n" + "mul z29.s, p2/M, z29.s, z11.s\n" + "add z30.s, z17.s, z20.s\n" + "add z31.s, z16.s, z18.s\n" + "mul z30.s, p2/M, z30.s, z11.s\n" + "mul z31.s, p2/M, z31.s, z11.s\n" "zip1 z19.s, z24.s, z26.s\n" "zip1 z18.s, z25.s, z27.s\n" "zip1 z17.s, z28.s, z30.s\n" "zip1 z16.s, z29.s, z31.s\n" "zip1 z22.s, z19.s, z18.s\n" "zip1 z23.s, z17.s, z16.s\n" - "add z24.s, z24.s, z13.s\n" - "add z25.s, z25.s, z13.s\n" - "add z26.s, z26.s, z13.s\n" - "add z27.s, z27.s, z13.s\n" - "add z28.s, z28.s, z13.s\n" - "add z29.s, z29.s, z13.s\n" - "add z30.s, z30.s, z13.s\n" - "add z31.s, z31.s, z13.s\n" + "add z24.s, z24.s, z14.s\n" + "add z25.s, z25.s, z14.s\n" + "add z26.s, z26.s, z14.s\n" + "add z27.s, z27.s, z14.s\n" + "add z28.s, z28.s, z14.s\n" + "add z29.s, z29.s, z14.s\n" + "add z30.s, z30.s, z14.s\n" + "add z31.s, z31.s, z14.s\n" "1:" // Loop "sdot z24.s, z5.b, z0.b[0]\n" "sdot z25.s, z5.b, z0.b[1]\n" - "ld1w { z21.s }, p2/Z, [%x[params]]\n" - "ld1w { z20.s }, p2/Z, [%x[params], #1, MUL VL]\n" + "ld1w { z8.s }, p2/Z, [%x[params]]\n" + "ld1w { z21.s }, p2/Z, [%x[params], #1, MUL VL]\n" "sdot z26.s, z5.b, z0.b[2]\n" "sdot z27.s, z5.b, z0.b[3]\n" "incb x9\n" - "whilelt p0.s, x28, %x[n_channels]\n" + "whilelt p1.s, x28, %x[n_channels]\n" "sdot z24.s, z6.b, z1.b[0]\n" "sdot z25.s, z6.b, z1.b[1]\n" - "whilelt p1.b, x9, x10\n" - "ld1w { z13.s }, p1/Z, [%x[params], #2, MUL VL]\n" + "whilelt p0.b, x9, x10\n" + "ld1w { z20.s }, p0/Z, [%x[params], #2, MUL VL]\n" "sdot z26.s, z6.b, z1.b[2]\n" "sdot z27.s, z6.b, z1.b[3]\n" "sdot z28.s, z5.b, z2.b[0]\n" "sdot z29.s, z5.b, z2.b[1]\n" "sdot z30.s, z5.b, z2.b[2]\n" "sdot z31.s, z5.b, z2.b[3]\n" - "ld1b { z5.b }, p1/Z, [%x[params], #3, MUL VL]\n" + "ld1b { z5.b }, p0/Z, [%x[params], #3, MUL VL]\n" "sdot z24.s, z7.b, z2.b[0]\n" "sdot z25.s, z7.b, z2.b[1]\n" - ".inst 0x04b57718 // sqrdmulh z24.s, z24.s, z21.s\n" + ".inst 0x04a87718 // sqrdmulh z24.s, z24.s, z8.s\n" "sdot z26.s, z7.b, z2.b[2]\n" "sdot z27.s, z7.b, z2.b[3]\n" - ".inst 0x04b57739 // sqrdmulh z25.s, z25.s, z21.s\n" + ".inst 0x04a87739 // sqrdmulh z25.s, z25.s, z8.s\n" "sdot z28.s, z6.b, z3.b[0]\n" "sdot z29.s, z6.b, z3.b[1]\n" - ".inst 0x04b5775a // sqrdmulh z26.s, z26.s, z21.s\n" + ".inst 0x04a8775a // sqrdmulh z26.s, z26.s, z8.s\n" "sdot z30.s, z6.b, z3.b[2]\n" "sdot z31.s, z6.b, z3.b[3]\n" - ".inst 0x04b5777b // sqrdmulh z27.s, z27.s, z21.s\n" - "ld1b { z6.b }, p1/Z, [%x[params], #4, MUL VL]\n" + ".inst 0x04a8777b // sqrdmulh z27.s, z27.s, z8.s\n" + "ld1b { z6.b }, p0/Z, [%x[params], #4, MUL VL]\n" "sdot z28.s, z7.b, z4.b[0]\n" "sdot z29.s, z7.b, z4.b[1]\n" - "and z19.d, z24.d, z20.d\n" + "and z19.d, z24.d, z21.d\n" "sdot z30.s, z7.b, z4.b[2]\n" "sdot z31.s, z7.b, z4.b[3]\n" - "and z18.d, z25.d, z20.d\n" - "ld1b { z7.b }, p1/Z, [%x[params], #5, MUL VL]\n" - "and z17.d, z26.d, z20.d\n" - "and z16.d, z27.d, z20.d\n" + "and z18.d, z25.d, z21.d\n" + "ld1b { z7.b }, p0/Z, [%x[params], #5, MUL VL]\n" + "and z17.d, z26.d, z21.d\n" + "and z16.d, z27.d, z21.d\n" "addvl %x[params], %x[params], #6\n" "asr z19.s, z19.s, #0x1f\n" "asr z18.s, z18.s, #0x1f\n" "asr z17.s, z17.s, #0x1f\n" "asr z16.s, z16.s, #0x1f\n" - ".inst 0x04b5779c // sqrdmulh z28.s, z28.s, z21.s\n" - ".inst 0x04b577bd // sqrdmulh z29.s, z29.s, z21.s\n" - ".inst 0x04b577de // sqrdmulh z30.s, z30.s, z21.s\n" - ".inst 0x04b577ff // sqrdmulh z31.s, z31.s, z21.s\n" + ".inst 0x04a8779c // sqrdmulh z28.s, z28.s, z8.s\n" + ".inst 0x04a877bd // sqrdmulh z29.s, z29.s, z8.s\n" + ".inst 0x04a877de // sqrdmulh z30.s, z30.s, z8.s\n" + ".inst 0x04a877ff // sqrdmulh z31.s, z31.s, z8.s\n" "sqadd z24.s, z24.s, z19.s\n" "sqadd z25.s, z25.s, z18.s\n" - ".inst 0x44828a98 // srshl z24.s, p2/M, z24.s, z20.s\n" - ".inst 0x44828a99 // srshl z25.s, p2/M, z25.s, z20.s\n" + ".inst 0x44828ab8 // srshl z24.s, p2/M, z24.s, z21.s\n" + ".inst 0x44828ab9 // srshl z25.s, p2/M, z25.s, z21.s\n" "sqadd z26.s, z26.s, z17.s\n" "sqadd z27.s, z27.s, z16.s\n" - ".inst 0x44828a9a // srshl z26.s, p2/M, z26.s, z20.s\n" - ".inst 0x44828a9b // srshl z27.s, p2/M, z27.s, z20.s\n" - "and z19.d, z28.d, z20.d\n" - "and z18.d, z29.d, z20.d\n" - "and z17.d, z30.d, z20.d\n" - "and z16.d, z31.d, z20.d\n" + ".inst 0x44828aba // srshl z26.s, p2/M, z26.s, z21.s\n" + ".inst 0x44828abb // srshl z27.s, p2/M, z27.s, z21.s\n" + "and z19.d, z28.d, z21.d\n" + "and z18.d, z29.d, z21.d\n" + "and z17.d, z30.d, z21.d\n" + "and z16.d, z31.d, z21.d\n" "asr z19.s, z19.s, #0x1f\n" "asr z18.s, z18.s, #0x1f\n" "asr z17.s, z17.s, #0x1f\n" "asr z16.s, z16.s, #0x1f\n" "sqadd z28.s, z28.s, z19.s\n" "sqadd z29.s, z29.s, z18.s\n" - ".inst 0x44828a9c // srshl z28.s, p2/M, z28.s, z20.s\n" - ".inst 0x44828a9d // srshl z29.s, p2/M, z29.s, z20.s\n" + ".inst 0x44828abc // srshl z28.s, p2/M, z28.s, z21.s\n" + ".inst 0x44828abd // srshl z29.s, p2/M, z29.s, z21.s\n" "sqadd z30.s, z30.s, z17.s\n" "sqadd z31.s, z31.s, z16.s\n" - ".inst 0x44828a9e // srshl z30.s, p2/M, z30.s, z20.s\n" - ".inst 0x44828a9f // srshl z31.s, p2/M, z31.s, z20.s\n" - "add z24.s, z24.s, z12.s\n" - "add z25.s, z25.s, z12.s\n" - "smin z24.s, p2/M, z24.s, z10.s\n" - "smin z25.s, p2/M, z25.s, z10.s\n" - "add z26.s, z26.s, z12.s\n" - "add z27.s, z27.s, z12.s\n" - "smin z26.s, p2/M, z26.s, z10.s\n" - "smin z27.s, p2/M, z27.s, z10.s\n" - "add z28.s, z28.s, z12.s\n" - "add z29.s, z29.s, z12.s\n" - "smin z28.s, p2/M, z28.s, z10.s\n" - "smin z29.s, p2/M, z29.s, z10.s\n" - "add z30.s, z30.s, z12.s\n" - "add z31.s, z31.s, z12.s\n" - "smin z30.s, p2/M, z30.s, z10.s\n" - "smin z31.s, p2/M, z31.s, z10.s\n" - "smax z24.s, p2/M, z24.s, z11.s\n" - "smax z25.s, p2/M, z25.s, z11.s\n" - "st1b { z24.s }, p0, [x27, x28]\n" + ".inst 0x44828abe // srshl z30.s, p2/M, z30.s, z21.s\n" + ".inst 0x44828abf // srshl z31.s, p2/M, z31.s, z21.s\n" + "add z24.s, z24.s, z9.s\n" + "add z25.s, z25.s, z9.s\n" + "smin z24.s, p2/M, z24.s, z12.s\n" + "smin z25.s, p2/M, z25.s, z12.s\n" + "add z26.s, z26.s, z9.s\n" + "add z27.s, z27.s, z9.s\n" + "smin z26.s, p2/M, z26.s, z12.s\n" + "smin z27.s, p2/M, z27.s, z12.s\n" + "add z28.s, z28.s, z9.s\n" + "add z29.s, z29.s, z9.s\n" + "smin z28.s, p2/M, z28.s, z12.s\n" + "smin z29.s, p2/M, z29.s, z12.s\n" + "add z30.s, z30.s, z9.s\n" + "add z31.s, z31.s, z9.s\n" + "smin z30.s, p2/M, z30.s, z12.s\n" + "smin z31.s, p2/M, z31.s, z12.s\n" + "smax z24.s, p2/M, z24.s, z15.s\n" + "smax z25.s, p2/M, z25.s, z15.s\n" + "st1b { z24.s }, p1, [x27, x28]\n" "mov z24.s, z22.s[0]\n" - "smax z26.s, p2/M, z26.s, z11.s\n" - "smax z27.s, p2/M, z27.s, z11.s\n" - "st1b { z25.s }, p0, [x26, x28]\n" + "smax z26.s, p2/M, z26.s, z15.s\n" + "smax z27.s, p2/M, z27.s, z15.s\n" + "st1b { z25.s }, p1, [x26, x28]\n" "mov z25.s, z22.s[1]\n" - "smax z28.s, p2/M, z28.s, z11.s\n" - "smax z29.s, p2/M, z29.s, z11.s\n" - "st1b { z26.s }, p0, [x25, x28]\n" + "smax z28.s, p2/M, z28.s, z15.s\n" + "smax z29.s, p2/M, z29.s, z15.s\n" + "st1b { z26.s }, p1, [x25, x28]\n" "mov z26.s, z22.s[2]\n" - "smax z30.s, p2/M, z30.s, z11.s\n" - "smax z31.s, p2/M, z31.s, z11.s\n" - "st1b { z27.s }, p0, [x24, x28]\n" + "smax z30.s, p2/M, z30.s, z15.s\n" + "smax z31.s, p2/M, z31.s, z15.s\n" + "st1b { z27.s }, p1, [x24, x28]\n" "mov z27.s, z22.s[3]\n" - "st1b { z28.s }, p0, [x23, x28]\n" + "st1b { z28.s }, p1, [x23, x28]\n" "mov z28.s, z23.s[0]\n" - "add z24.s, z24.s, z13.s\n" - "st1b { z29.s }, p0, [x22, x28]\n" + "add z24.s, z24.s, z20.s\n" + "st1b { z29.s }, p1, [x22, x28]\n" "mov z29.s, z23.s[1]\n" - "add z25.s, z25.s, z13.s\n" - "st1b { z30.s }, p0, [x21, x28]\n" + "add z25.s, z25.s, z20.s\n" + "st1b { z30.s }, p1, [x21, x28]\n" "mov z30.s, z23.s[2]\n" - "add z26.s, z26.s, z13.s\n" - "st1b { z31.s }, p0, [x20, x28]\n" + "add z26.s, z26.s, z20.s\n" + "st1b { z31.s }, p1, [x20, x28]\n" "mov z31.s, z23.s[3]\n" "incw x28\n" - "add z27.s, z27.s, z13.s\n" - "add z28.s, z28.s, z13.s\n" - "add z29.s, z29.s, z13.s\n" - "add z30.s, z30.s, z13.s\n" - "add z31.s, z31.s, z13.s\n" + "add z27.s, z27.s, z20.s\n" + "add z28.s, z28.s, z20.s\n" + "add z29.s, z29.s, z20.s\n" + "add z30.s, z30.s, z20.s\n" + "add z31.s, z31.s, z20.s\n" "b.any 1b\n" : [params] "+&r" (params) : [inptrs] "r" (inptrs), [n_channels] "r" (n_output_channels), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [outptrs] "r" (outptrs), [qp] "r" (&qp) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst.hpp index 4874fb9a77dab270d2102351853e35eede2e534f..701948f264968e7288a2e85f9b3bde8068c50bd7 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst/generic.cpp index 2ed7cfc8159dd097feb02654d444d6b718ba3208..a3b2b429c0976fe7dc12c47a3a95850588ac3350 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst/generic.cpp @@ -36,7 +36,7 @@ void sve_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl const int8_t *const *const inptrs, int8_t *const *const outptrs, const void *params, - const unsigned int n_output_channels, + unsigned int n_output_channels, const arm_gemm::Requantize32& qp ) { @@ -47,8 +47,8 @@ void sve_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl "ldr x21, [%x[inptrs], #0x20]\n" "ldr x20, [%x[inptrs], #0x10]\n" "ld1b { z3.b }, p0/Z, [x22]\n" - "mov z20.d, z3.d\n" - "ext z20.b, z20.b, z20.b, #0x1\n" + "mov z23.d, z3.d\n" + "ext z23.b, z23.b, z23.b, #0x1\n" "ld1b { z4.b }, p0/Z, [x21]\n" "ldr x24, [%x[inptrs], #0x8]\n" "mov z18.d, z4.d\n" @@ -59,132 +59,132 @@ void sve_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl "ext z15.b, z15.b, z15.b, #0x1\n" "ldr x22, [%x[inptrs], #0x30]\n" "ldr x21, [%x[inptrs], #0x38]\n" - "zip1 z3.d, z3.d, z20.d\n" + "zip1 z3.d, z3.d, z23.d\n" "zip1 z4.d, z4.d, z18.d\n" "ldr x20, [%x[inptrs], #0x0]\n" "ld1b { z1.b }, p0/Z, [x24]\n" - "mov z20.d, z1.d\n" - "ext z20.b, z20.b, z20.b, #0x1\n" + "mov z19.d, z1.d\n" + "ext z19.b, z19.b, z19.b, #0x1\n" "ld1b { z5.b }, p0/Z, [x23]\n" "ld1b { z6.b }, p0/Z, [x22]\n" - "mov z13.d, z5.d\n" - "mov z19.d, z6.d\n" + "mov z18.d, z5.d\n" + "mov z22.d, z6.d\n" "ld1b { z7.b }, p0/Z, [x21]\n" "ld1b { z0.b }, p0/Z, [x20]\n" - "mov z25.d, z7.d\n" + "mov z8.d, z7.d\n" "zip1 z2.d, z2.d, z15.d\n" "mov z3.q, z3.q[0]\n" "mov z4.q, z4.q[0]\n" "ptrue p2.b\n" "ld1rw { z23.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" - "ext z13.b, z13.b, z13.b, #0x1\n" - "ext z19.b, z19.b, z19.b, #0x1\n" + "ext z18.b, z18.b, z18.b, #0x1\n" + "ext z22.b, z22.b, z22.b, #0x1\n" "lsl x10, %x[n_channels], #0x2\n" "neg z23.s, p2/M, z23.s\n" - "ext z25.b, z25.b, z25.b, #0x1\n" - "mov z30.b, #0x1\n" + "ext z8.b, z8.b, z8.b, #0x1\n" + "mov z28.b, #0x1\n" "mov x9, #0x0\n" - "whilelt p1.b, x9, x10\n" + "whilelt p0.b, x9, x10\n" + "mov z25.s, #0x0\n" "mov z24.s, #0x0\n" - "mov z28.s, #0x0\n" - "sdot z24.s, z30.b, z3.b[0]\n" - "ld1w { z12.s }, p1/Z, [%x[params]]\n" - "mov z18.s, #0x0\n" + "sdot z25.s, z28.b, z3.b[0]\n" + "ld1w { z12.s }, p0/Z, [%x[params]]\n" "mov z17.s, #0x0\n" - "sdot z28.s, z30.b, z3.b[2]\n" + "mov z16.s, #0x0\n" + "sdot z24.s, z28.b, z3.b[2]\n" "mov x28, #0x0\n" - "mov z16.d, z0.d\n" - "sdot z18.s, z30.b, z4.b[0]\n" - "sdot z17.s, z30.b, z4.b[2]\n" + "mov z27.d, z0.d\n" + "sdot z17.s, z28.b, z4.b[0]\n" + "sdot z16.s, z28.b, z4.b[2]\n" "ldp x27, x26, [%x[outptrs], #0x0]\n" - "ext z16.b, z16.b, z16.b, #0x1\n" - "zip1 z1.d, z1.d, z20.d\n" + "ext z27.b, z27.b, z27.b, #0x1\n" + "zip1 z1.d, z1.d, z19.d\n" "ldp x25, x24, [%x[outptrs], #0x10]\n" "ldp x23, x22, [%x[outptrs], #0x20]\n" "mov z2.q, z2.q[0]\n" - "zip1 z5.d, z5.d, z13.d\n" + "zip1 z5.d, z5.d, z18.d\n" "ldp x21, x20, [%x[outptrs], #0x30]\n" - "ld1rw { z14.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" - "zip1 z6.d, z6.d, z19.d\n" - "zip1 z7.d, z7.d, z25.d\n" - "ld1rw { z13.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" + "ld1rw { z13.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" + "zip1 z6.d, z6.d, z22.d\n" + "zip1 z7.d, z7.d, z8.d\n" + "ld1rw { z14.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" "ld1rw { z15.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" - "mov z26.s, #0x0\n" - "mov z22.s, #0x0\n" - "sdot z26.s, z30.b, z2.b[0]\n" - "ld1b { z8.b }, p1/Z, [%x[params], #1, MUL VL]\n" + "mov z30.s, #0x0\n" + "mov z31.s, #0x0\n" + "sdot z30.s, z28.b, z2.b[0]\n" + "ld1b { z8.b }, p0/Z, [%x[params], #1, MUL VL]\n" "mov z29.s, #0x1\n" - "sdot z22.s, z30.b, z2.b[2]\n" - "sdot z24.s, z29.b, z3.b[1]\n" - "ld1b { z9.b }, p1/Z, [%x[params], #2, MUL VL]\n" - "zip1 z0.d, z0.d, z16.d\n" + "sdot z31.s, z28.b, z2.b[2]\n" + "sdot z25.s, z29.b, z3.b[1]\n" + "ld1b { z9.b }, p0/Z, [%x[params], #2, MUL VL]\n" + "zip1 z0.d, z0.d, z27.d\n" "mov z1.q, z1.q[0]\n" - "sdot z28.s, z29.b, z3.b[3]\n" - "ld1b { z10.b }, p1/Z, [%x[params], #3, MUL VL]\n" + "sdot z24.s, z29.b, z3.b[3]\n" + "ld1b { z10.b }, p0/Z, [%x[params], #3, MUL VL]\n" "mov z5.q, z5.q[0]\n" "mov z6.q, z6.q[0]\n" - "sdot z18.s, z29.b, z4.b[1]\n" - "ld1b { z11.b }, p1/Z, [%x[params], #4, MUL VL]\n" + "sdot z17.s, z29.b, z4.b[1]\n" + "ld1b { z11.b }, p0/Z, [%x[params], #4, MUL VL]\n" "mov z7.q, z7.q[0]\n" - "mov z21.s, #0x0\n" - "sdot z17.s, z29.b, z4.b[3]\n" + "mov z22.s, #0x0\n" + "sdot z16.s, z29.b, z4.b[3]\n" "addvl %x[params], %x[params], #5\n" - "mov z20.s, #0x0\n" - "mov z25.s, #0x0\n" - "sdot z21.s, z30.b, z1.b[0]\n" + "mov z21.s, #0x0\n" + "mov z26.s, #0x0\n" + "sdot z22.s, z28.b, z1.b[0]\n" "mov z27.s, #0x0\n" + "mov z20.s, #0x0\n" + "sdot z21.s, z28.b, z1.b[2]\n" "mov z19.s, #0x0\n" - "sdot z20.s, z30.b, z1.b[2]\n" - "sdot z25.s, z30.b, z5.b[0]\n" - "sdot z27.s, z30.b, z5.b[2]\n" - "mov z0.q, z0.q[0]\n" - "sdot z19.s, z30.b, z6.b[0]\n" - "sdot z26.s, z29.b, z2.b[1]\n" - "add z24.s, z24.s, z18.s\n" "mov z18.s, #0x0\n" - "sdot z18.s, z30.b, z6.b[2]\n" - "sdot z22.s, z29.b, z2.b[3]\n" - "add z17.s, z28.s, z17.s\n" - "mov z16.s, #0x0\n" - "sdot z16.s, z30.b, z7.b[0]\n" - "sdot z21.s, z29.b, z1.b[1]\n" - "sdot z20.s, z29.b, z1.b[3]\n" - "add z28.s, z26.s, z24.s\n" - "sdot z25.s, z29.b, z5.b[1]\n" + "sdot z26.s, z28.b, z5.b[0]\n" + "sdot z27.s, z28.b, z5.b[2]\n" + "sdot z20.s, z28.b, z6.b[0]\n" + "mov z0.q, z0.q[0]\n" + "sdot z19.s, z28.b, z6.b[2]\n" + "sdot z18.s, z28.b, z7.b[0]\n" + "add z17.s, z25.s, z17.s\n" + "mov z25.s, #0x0\n" + "sdot z25.s, z28.b, z7.b[2]\n" + "sdot z30.s, z29.b, z2.b[1]\n" + "sdot z31.s, z29.b, z2.b[3]\n" + "add z16.s, z24.s, z16.s\n" + "sdot z22.s, z29.b, z1.b[1]\n" + "mov z24.s, #0x0\n" + "sdot z24.s, z28.b, z0.b[0]\n" + "sdot z21.s, z29.b, z1.b[3]\n" + "sdot z26.s, z29.b, z5.b[1]\n" "sdot z27.s, z29.b, z5.b[3]\n" - "add z31.s, z22.s, z17.s\n" - "sdot z19.s, z29.b, z6.b[1]\n" - "sdot z18.s, z29.b, z6.b[3]\n" - "add z22.s, z21.s, z28.s\n" - "sdot z16.s, z29.b, z7.b[1]\n" - "add z21.s, z20.s, z31.s\n" - "add z20.s, z25.s, z19.s\n" - "add z19.s, z27.s, z18.s\n" - "add z18.s, z16.s, z24.s\n" - "mov z16.s, #0x0\n" - "sdot z16.s, z30.b, z7.b[2]\n" - "sdot z16.s, z29.b, z7.b[3]\n" - "add z17.s, z16.s, z17.s\n" - "mov z16.s, #0x0\n" - "sdot z16.s, z30.b, z0.b[0]\n" - "sdot z16.s, z29.b, z0.b[1]\n" - "add z24.s, z22.s, z16.s\n" - "add z26.s, z22.s, z25.s\n" + "add z30.s, z30.s, z17.s\n" + "sdot z20.s, z29.b, z6.b[1]\n" + "sdot z19.s, z29.b, z6.b[3]\n" + "add z31.s, z31.s, z16.s\n" + "sdot z18.s, z29.b, z7.b[1]\n" + "sdot z25.s, z29.b, z7.b[3]\n" + "add z22.s, z22.s, z30.s\n" + "sdot z24.s, z29.b, z0.b[1]\n" + "add z21.s, z21.s, z31.s\n" + "add z20.s, z26.s, z20.s\n" + "add z19.s, z27.s, z19.s\n" + "add z18.s, z18.s, z17.s\n" + "mov z17.s, #0x0\n" + "sdot z17.s, z28.b, z0.b[2]\n" + "sdot z17.s, z29.b, z0.b[3]\n" + "add z16.s, z25.s, z16.s\n" + "add z24.s, z22.s, z24.s\n" + "add z25.s, z21.s, z17.s\n" "mul z24.s, p2/M, z24.s, z23.s\n" - "mul z26.s, p2/M, z26.s, z23.s\n" - "mov z16.s, #0x0\n" - "sdot z16.s, z30.b, z0.b[2]\n" - "sdot z16.s, z29.b, z0.b[3]\n" - "add z25.s, z21.s, z16.s\n" - "add z27.s, z21.s, z27.s\n" "mul z25.s, p2/M, z25.s, z23.s\n" + "add z26.s, z26.s, z22.s\n" + "add z27.s, z27.s, z21.s\n" + "mul z26.s, p2/M, z26.s, z23.s\n" "mul z27.s, p2/M, z27.s, z23.s\n" - "add z28.s, z20.s, z28.s\n" + "add z28.s, z20.s, z30.s\n" "add z29.s, z19.s, z31.s\n" "mul z28.s, p2/M, z28.s, z23.s\n" "mul z29.s, p2/M, z29.s, z23.s\n" - "add z30.s, z18.s, z20.s\n" - "add z31.s, z17.s, z19.s\n" + "add z30.s, z20.s, z18.s\n" + "add z31.s, z19.s, z16.s\n" "mul z30.s, p2/M, z30.s, z23.s\n" "mul z31.s, p2/M, z31.s, z23.s\n" "zip1 z19.s, z24.s, z26.s\n" @@ -204,22 +204,22 @@ void sve_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl "1:" // Loop "sdot z24.s, z8.b, z0.b[0]\n" "sdot z25.s, z8.b, z0.b[2]\n" - "ld1w { z17.s }, p2/Z, [%x[params], #6, MUL VL]\n" - "ld1w { z19.s }, p2/Z, [%x[params], #7, MUL VL]\n" + "ld1w { z12.s }, p2/Z, [%x[params], #6, MUL VL]\n" + "ld1w { z21.s }, p2/Z, [%x[params], #7, MUL VL]\n" "sdot z26.s, z8.b, z1.b[0]\n" "sdot z27.s, z8.b, z1.b[2]\n" "incb x9\n" - "whilelt p0.s, x28, %x[n_channels]\n" + "whilelt p1.s, x28, %x[n_channels]\n" "sdot z24.s, z9.b, z0.b[1]\n" "sdot z25.s, z9.b, z0.b[3]\n" - "whilelt p1.b, x9, x10\n" + "whilelt p0.b, x9, x10\n" "sdot z26.s, z9.b, z1.b[1]\n" "sdot z27.s, z9.b, z1.b[3]\n" "sdot z28.s, z8.b, z2.b[0]\n" "sdot z29.s, z8.b, z2.b[2]\n" "sdot z30.s, z8.b, z3.b[0]\n" "sdot z31.s, z8.b, z3.b[2]\n" - "ld1b { z8.b }, p2/Z, [%x[params]]\n" + "ld1b { z17.b }, p2/Z, [%x[params]]\n" "sdot z24.s, z10.b, z1.b[0]\n" "sdot z25.s, z10.b, z1.b[2]\n" "sdot z26.s, z10.b, z2.b[0]\n" @@ -228,7 +228,7 @@ void sve_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl "sdot z29.s, z9.b, z2.b[3]\n" "sdot z30.s, z9.b, z3.b[1]\n" "sdot z31.s, z9.b, z3.b[3]\n" - "ld1b { z9.b }, p2/Z, [%x[params], #1, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [%x[params], #1, MUL VL]\n" "sdot z24.s, z11.b, z1.b[1]\n" "sdot z25.s, z11.b, z1.b[3]\n" "sdot z26.s, z11.b, z2.b[1]\n" @@ -237,158 +237,158 @@ void sve_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl "sdot z29.s, z10.b, z3.b[2]\n" "sdot z30.s, z10.b, z4.b[0]\n" "sdot z31.s, z10.b, z4.b[2]\n" - "ld1b { z10.b }, p2/Z, [%x[params], #2, MUL VL]\n" - "sdot z24.s, z8.b, z2.b[0]\n" - "sdot z25.s, z8.b, z2.b[2]\n" - "sdot z26.s, z8.b, z3.b[0]\n" - "sdot z27.s, z8.b, z3.b[2]\n" + "ld1b { z19.b }, p2/Z, [%x[params], #2, MUL VL]\n" + "sdot z24.s, z17.b, z2.b[0]\n" + "sdot z25.s, z17.b, z2.b[2]\n" + "sdot z26.s, z17.b, z3.b[0]\n" + "sdot z27.s, z17.b, z3.b[2]\n" "sdot z28.s, z11.b, z3.b[1]\n" "sdot z29.s, z11.b, z3.b[3]\n" "sdot z30.s, z11.b, z4.b[1]\n" "sdot z31.s, z11.b, z4.b[3]\n" - "ld1b { z11.b }, p2/Z, [%x[params], #3, MUL VL]\n" - "sdot z24.s, z9.b, z2.b[1]\n" - "sdot z25.s, z9.b, z2.b[3]\n" - "sdot z26.s, z9.b, z3.b[1]\n" - "sdot z27.s, z9.b, z3.b[3]\n" - "sdot z28.s, z8.b, z4.b[0]\n" - "sdot z29.s, z8.b, z4.b[2]\n" - "sdot z30.s, z8.b, z5.b[0]\n" - "sdot z31.s, z8.b, z5.b[2]\n" - "ld1b { z8.b }, p2/Z, [%x[params], #4, MUL VL]\n" - "sdot z24.s, z10.b, z3.b[0]\n" - "sdot z25.s, z10.b, z3.b[2]\n" - "sdot z26.s, z10.b, z4.b[0]\n" - "sdot z27.s, z10.b, z4.b[2]\n" - "sdot z28.s, z9.b, z4.b[1]\n" - "sdot z29.s, z9.b, z4.b[3]\n" - "sdot z30.s, z9.b, z5.b[1]\n" - "sdot z31.s, z9.b, z5.b[3]\n" - "ld1b { z9.b }, p2/Z, [%x[params], #5, MUL VL]\n" + "ld1b { z18.b }, p2/Z, [%x[params], #3, MUL VL]\n" + "sdot z24.s, z16.b, z2.b[1]\n" + "sdot z25.s, z16.b, z2.b[3]\n" + "sdot z26.s, z16.b, z3.b[1]\n" + "sdot z27.s, z16.b, z3.b[3]\n" + "sdot z28.s, z17.b, z4.b[0]\n" + "sdot z29.s, z17.b, z4.b[2]\n" + "sdot z30.s, z17.b, z5.b[0]\n" + "sdot z31.s, z17.b, z5.b[2]\n" + "ld1b { z17.b }, p2/Z, [%x[params], #4, MUL VL]\n" + "sdot z24.s, z19.b, z3.b[0]\n" + "sdot z25.s, z19.b, z3.b[2]\n" + "sdot z26.s, z19.b, z4.b[0]\n" + "sdot z27.s, z19.b, z4.b[2]\n" + "sdot z28.s, z16.b, z4.b[1]\n" + "sdot z29.s, z16.b, z4.b[3]\n" + "sdot z30.s, z16.b, z5.b[1]\n" + "sdot z31.s, z16.b, z5.b[3]\n" + "ld1b { z16.b }, p2/Z, [%x[params], #5, MUL VL]\n" "addvl %x[params], %x[params], #16\n" - "sdot z24.s, z11.b, z3.b[1]\n" - "sdot z25.s, z11.b, z3.b[3]\n" - "ld1w { z12.s }, p1/Z, [%x[params], #-8, MUL VL]\n" - "sdot z26.s, z11.b, z4.b[1]\n" - "sdot z27.s, z11.b, z4.b[3]\n" - "sdot z28.s, z10.b, z5.b[0]\n" - "sdot z29.s, z10.b, z5.b[2]\n" - "sdot z30.s, z10.b, z6.b[0]\n" - "sdot z31.s, z10.b, z6.b[2]\n" - "ld1b { z10.b }, p1/Z, [%x[params], #-5, MUL VL]\n" - "sdot z24.s, z8.b, z4.b[0]\n" - "sdot z25.s, z8.b, z4.b[2]\n" - "sdot z26.s, z8.b, z5.b[0]\n" - "sdot z27.s, z8.b, z5.b[2]\n" - "sdot z28.s, z11.b, z5.b[1]\n" - "sdot z29.s, z11.b, z5.b[3]\n" - "sdot z30.s, z11.b, z6.b[1]\n" - "sdot z31.s, z11.b, z6.b[3]\n" - "ld1b { z11.b }, p1/Z, [%x[params], #-4, MUL VL]\n" - "sdot z24.s, z9.b, z4.b[1]\n" - "sdot z25.s, z9.b, z4.b[3]\n" - ".inst 0x04b17718 // sqrdmulh z24.s, z24.s, z17.s\n" - "sdot z26.s, z9.b, z5.b[1]\n" - "sdot z27.s, z9.b, z5.b[3]\n" - ".inst 0x04b17739 // sqrdmulh z25.s, z25.s, z17.s\n" - "sdot z28.s, z8.b, z6.b[0]\n" - "sdot z29.s, z8.b, z6.b[2]\n" - ".inst 0x04b1775a // sqrdmulh z26.s, z26.s, z17.s\n" - "sdot z30.s, z8.b, z7.b[0]\n" - "sdot z31.s, z8.b, z7.b[2]\n" - ".inst 0x04b1777b // sqrdmulh z27.s, z27.s, z17.s\n" - "ld1b { z8.b }, p1/Z, [%x[params], #-7, MUL VL]\n" - "sdot z28.s, z9.b, z6.b[1]\n" - "sdot z29.s, z9.b, z6.b[3]\n" - "and z16.d, z24.d, z19.d\n" - "sdot z30.s, z9.b, z7.b[1]\n" - "sdot z31.s, z9.b, z7.b[3]\n" - "and z18.d, z25.d, z19.d\n" - "ld1b { z9.b }, p1/Z, [%x[params], #-6, MUL VL]\n" - "asr z16.s, z16.s, #0x1f\n" - "asr z18.s, z18.s, #0x1f\n" + "sdot z24.s, z18.b, z3.b[1]\n" + "sdot z25.s, z18.b, z3.b[3]\n" + "ld1w { z20.s }, p0/Z, [%x[params], #-8, MUL VL]\n" + "sdot z26.s, z18.b, z4.b[1]\n" + "sdot z27.s, z18.b, z4.b[3]\n" + "sdot z28.s, z19.b, z5.b[0]\n" + "sdot z29.s, z19.b, z5.b[2]\n" + "sdot z30.s, z19.b, z6.b[0]\n" + "sdot z31.s, z19.b, z6.b[2]\n" + "ld1b { z10.b }, p0/Z, [%x[params], #-5, MUL VL]\n" + "sdot z24.s, z17.b, z4.b[0]\n" + "sdot z25.s, z17.b, z4.b[2]\n" + "sdot z26.s, z17.b, z5.b[0]\n" + "sdot z27.s, z17.b, z5.b[2]\n" + "sdot z28.s, z18.b, z5.b[1]\n" + "sdot z29.s, z18.b, z5.b[3]\n" + "sdot z30.s, z18.b, z6.b[1]\n" + "sdot z31.s, z18.b, z6.b[3]\n" + "ld1b { z11.b }, p0/Z, [%x[params], #-4, MUL VL]\n" + "sdot z24.s, z16.b, z4.b[1]\n" + "sdot z25.s, z16.b, z4.b[3]\n" + ".inst 0x04ac7718 // sqrdmulh z24.s, z24.s, z12.s\n" + "sdot z26.s, z16.b, z5.b[1]\n" + "sdot z27.s, z16.b, z5.b[3]\n" + ".inst 0x04ac7739 // sqrdmulh z25.s, z25.s, z12.s\n" + "sdot z28.s, z17.b, z6.b[0]\n" + "sdot z29.s, z17.b, z6.b[2]\n" + ".inst 0x04ac775a // sqrdmulh z26.s, z26.s, z12.s\n" + "sdot z30.s, z17.b, z7.b[0]\n" + "sdot z31.s, z17.b, z7.b[2]\n" + ".inst 0x04ac777b // sqrdmulh z27.s, z27.s, z12.s\n" + "ld1b { z8.b }, p0/Z, [%x[params], #-7, MUL VL]\n" + "sdot z28.s, z16.b, z6.b[1]\n" + "sdot z29.s, z16.b, z6.b[3]\n" + "and z19.d, z24.d, z21.d\n" + "sdot z30.s, z16.b, z7.b[1]\n" + "sdot z31.s, z16.b, z7.b[3]\n" + "and z18.d, z25.d, z21.d\n" + "ld1b { z9.b }, p0/Z, [%x[params], #-6, MUL VL]\n" + "and z17.d, z26.d, z21.d\n" + "and z16.d, z27.d, z21.d\n" "addvl %x[params], %x[params], #-3\n" - ".inst 0x04b1779c // sqrdmulh z28.s, z28.s, z17.s\n" - ".inst 0x04b177bd // sqrdmulh z29.s, z29.s, z17.s\n" - ".inst 0x04b177de // sqrdmulh z30.s, z30.s, z17.s\n" - ".inst 0x04b177ff // sqrdmulh z31.s, z31.s, z17.s\n" - "and z17.d, z26.d, z19.d\n" + "asr z19.s, z19.s, #0x1f\n" + "asr z18.s, z18.s, #0x1f\n" "asr z17.s, z17.s, #0x1f\n" - "sqadd z24.s, z24.s, z16.s\n" - "and z16.d, z27.d, z19.d\n" - ".inst 0x44828a78 // srshl z24.s, p2/M, z24.s, z19.s\n" "asr z16.s, z16.s, #0x1f\n" + ".inst 0x04ac779c // sqrdmulh z28.s, z28.s, z12.s\n" + ".inst 0x04ac77bd // sqrdmulh z29.s, z29.s, z12.s\n" + ".inst 0x04ac77de // sqrdmulh z30.s, z30.s, z12.s\n" + ".inst 0x04ac77ff // sqrdmulh z31.s, z31.s, z12.s\n" + "sqadd z24.s, z24.s, z19.s\n" "sqadd z25.s, z25.s, z18.s\n" - ".inst 0x44828a79 // srshl z25.s, p2/M, z25.s, z19.s\n" + ".inst 0x44828ab8 // srshl z24.s, p2/M, z24.s, z21.s\n" + ".inst 0x44828ab9 // srshl z25.s, p2/M, z25.s, z21.s\n" "sqadd z26.s, z26.s, z17.s\n" "sqadd z27.s, z27.s, z16.s\n" - ".inst 0x44828a7a // srshl z26.s, p2/M, z26.s, z19.s\n" - ".inst 0x44828a7b // srshl z27.s, p2/M, z27.s, z19.s\n" - "and z16.d, z28.d, z19.d\n" - "and z18.d, z29.d, z19.d\n" - "and z17.d, z30.d, z19.d\n" - "asr z16.s, z16.s, #0x1f\n" + ".inst 0x44828aba // srshl z26.s, p2/M, z26.s, z21.s\n" + ".inst 0x44828abb // srshl z27.s, p2/M, z27.s, z21.s\n" + "and z19.d, z28.d, z21.d\n" + "and z18.d, z29.d, z21.d\n" + "and z17.d, z30.d, z21.d\n" + "and z16.d, z31.d, z21.d\n" + "asr z19.s, z19.s, #0x1f\n" "asr z18.s, z18.s, #0x1f\n" "asr z17.s, z17.s, #0x1f\n" - "sqadd z28.s, z28.s, z16.s\n" - "and z16.d, z31.d, z19.d\n" - ".inst 0x44828a7c // srshl z28.s, p2/M, z28.s, z19.s\n" "asr z16.s, z16.s, #0x1f\n" + "sqadd z28.s, z28.s, z19.s\n" "sqadd z29.s, z29.s, z18.s\n" - ".inst 0x44828a7d // srshl z29.s, p2/M, z29.s, z19.s\n" + ".inst 0x44828abc // srshl z28.s, p2/M, z28.s, z21.s\n" + ".inst 0x44828abd // srshl z29.s, p2/M, z29.s, z21.s\n" "sqadd z30.s, z30.s, z17.s\n" "sqadd z31.s, z31.s, z16.s\n" - ".inst 0x44828a7e // srshl z30.s, p2/M, z30.s, z19.s\n" - ".inst 0x44828a7f // srshl z31.s, p2/M, z31.s, z19.s\n" - "add z24.s, z24.s, z14.s\n" - "add z25.s, z25.s, z14.s\n" + ".inst 0x44828abe // srshl z30.s, p2/M, z30.s, z21.s\n" + ".inst 0x44828abf // srshl z31.s, p2/M, z31.s, z21.s\n" + "add z24.s, z24.s, z13.s\n" + "add z25.s, z25.s, z13.s\n" "smin z24.s, p2/M, z24.s, z15.s\n" "smin z25.s, p2/M, z25.s, z15.s\n" - "add z26.s, z26.s, z14.s\n" - "add z27.s, z27.s, z14.s\n" + "add z26.s, z26.s, z13.s\n" + "add z27.s, z27.s, z13.s\n" "smin z26.s, p2/M, z26.s, z15.s\n" "smin z27.s, p2/M, z27.s, z15.s\n" - "add z28.s, z28.s, z14.s\n" - "add z29.s, z29.s, z14.s\n" + "add z28.s, z28.s, z13.s\n" + "add z29.s, z29.s, z13.s\n" "smin z28.s, p2/M, z28.s, z15.s\n" "smin z29.s, p2/M, z29.s, z15.s\n" - "add z30.s, z30.s, z14.s\n" - "add z31.s, z31.s, z14.s\n" + "add z30.s, z30.s, z13.s\n" + "add z31.s, z31.s, z13.s\n" "smin z30.s, p2/M, z30.s, z15.s\n" "smin z31.s, p2/M, z31.s, z15.s\n" - "smax z24.s, p2/M, z24.s, z13.s\n" - "smax z25.s, p2/M, z25.s, z13.s\n" - "st1b { z24.s }, p0, [x27, x28]\n" + "smax z24.s, p2/M, z24.s, z14.s\n" + "smax z25.s, p2/M, z25.s, z14.s\n" + "st1b { z24.s }, p1, [x27, x28]\n" "mov z24.s, z22.s[0]\n" - "smax z26.s, p2/M, z26.s, z13.s\n" - "smax z27.s, p2/M, z27.s, z13.s\n" - "st1b { z25.s }, p0, [x26, x28]\n" + "smax z26.s, p2/M, z26.s, z14.s\n" + "smax z27.s, p2/M, z27.s, z14.s\n" + "st1b { z25.s }, p1, [x26, x28]\n" "mov z25.s, z22.s[1]\n" - "smax z28.s, p2/M, z28.s, z13.s\n" - "smax z29.s, p2/M, z29.s, z13.s\n" - "st1b { z26.s }, p0, [x25, x28]\n" + "smax z28.s, p2/M, z28.s, z14.s\n" + "smax z29.s, p2/M, z29.s, z14.s\n" + "st1b { z26.s }, p1, [x25, x28]\n" "mov z26.s, z22.s[2]\n" - "smax z30.s, p2/M, z30.s, z13.s\n" - "smax z31.s, p2/M, z31.s, z13.s\n" - "st1b { z27.s }, p0, [x24, x28]\n" + "smax z30.s, p2/M, z30.s, z14.s\n" + "smax z31.s, p2/M, z31.s, z14.s\n" + "st1b { z27.s }, p1, [x24, x28]\n" "mov z27.s, z22.s[3]\n" - "st1b { z28.s }, p0, [x23, x28]\n" + "st1b { z28.s }, p1, [x23, x28]\n" "mov z28.s, z23.s[0]\n" - "add z24.s, z24.s, z12.s\n" - "st1b { z29.s }, p0, [x22, x28]\n" + "add z24.s, z24.s, z20.s\n" + "st1b { z29.s }, p1, [x22, x28]\n" "mov z29.s, z23.s[1]\n" - "add z25.s, z25.s, z12.s\n" - "st1b { z30.s }, p0, [x21, x28]\n" + "add z25.s, z25.s, z20.s\n" + "st1b { z30.s }, p1, [x21, x28]\n" "mov z30.s, z23.s[2]\n" - "add z26.s, z26.s, z12.s\n" - "st1b { z31.s }, p0, [x20, x28]\n" + "add z26.s, z26.s, z20.s\n" + "st1b { z31.s }, p1, [x20, x28]\n" "mov z31.s, z23.s[3]\n" "incw x28\n" - "add z27.s, z27.s, z12.s\n" - "add z28.s, z28.s, z12.s\n" - "add z29.s, z29.s, z12.s\n" - "add z30.s, z30.s, z12.s\n" - "add z31.s, z31.s, z12.s\n" + "add z27.s, z27.s, z20.s\n" + "add z28.s, z28.s, z20.s\n" + "add z29.s, z29.s, z20.s\n" + "add z30.s, z30.s, z20.s\n" + "add z31.s, z31.s, z20.s\n" "b.any 1b\n" : [params] "+&r" (params) : [inptrs] "r" (inptrs), [n_channels] "r" (n_output_channels), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [outptrs] "r" (outptrs), [qp] "r" (&qp) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp index 0d185fcafcd02cff0a8856a7d920f5f9489fcbc0..6799b10ed95dd484316b4f91d99c246aff394a3b 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include @@ -64,7 +64,7 @@ class sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst : public DepthwiseDepthfirst ) const override { interleave_sve_s8q_3x3_dot::pack_parameters( - args.input_channels, buffer, reinterpret_cast(biases), + args.input_channels * args.channel_multiplier, buffer, reinterpret_cast(biases), reinterpret_cast(weights), qp, ld_weight_col, ld_weight_row ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp index 6a432e19613c2121534892cf693d961e3f3389f9..d9c8644fc4bc084ea413265c530e9fb5f6153839 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp @@ -30,411 +30,403 @@ namespace arm_conv { namespace depthwise { -void sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst_impl( - const unsigned int n_channels, - const int8_t *const *const inptrs, - const int8_t *params, - const int32_t *, // Bias, should be wrapped into the parameters - const arm_gemm::Requantize32& qp, - const int32_t *, const int32_t *, // Requant parameters, also wrapped - int8_t *const *const outptrs -) +void sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst_impl(const unsigned int n_channels, const int8_t *const *const inptrs, const int8_t *params, const int32_t *, const arm_gemm::Requantize32& qp, const int32_t *, const int32_t *, int8_t *const *const outptrs) { __asm__ __volatile__( "mov x13, #0x0\n" - "whilelt p2.b, x13, %x[n_channels]\n" - "ldp x12, x11, [%x[inptrs], #0x0]\n" - "ldp x10, x9, [%x[inptrs], #0x10]\n" - "ldp x28, x27, [%x[inptrs], #0x20]\n" - "ldp x26, x25, [%x[inptrs], #0x30]\n" - "ptrue p1.b\n" - "mov x24, #0x0\n" - "ldp x23, x22, [%x[outptrs], #0x0]\n" - "ldp x21, x20, [%x[outptrs], #0x10]\n" - "ld1b { z9.b }, p2/Z, [x12, x13]\n" - "ld1b { z8.b }, p2/Z, [x11, x13]\n" - "ldp x12, x11, [%x[inptrs], #0x40]\n" - "ld1b { z7.b }, p2/Z, [x10, x13]\n" - "zip2 z6.b, z9.b, z7.b\n" - "zip1 z9.b, z9.b, z7.b\n" - "ld1b { z5.b }, p2/Z, [x9, x13]\n" - "ldp x10, x9, [%x[inptrs], #0x50]\n" - "zip1 z7.b, z8.b, z5.b\n" - "zip2 z5.b, z8.b, z5.b\n" - "ld1b { z4.b }, p2/Z, [x28, x13]\n" - "ld1b { z3.b }, p2/Z, [x27, x13]\n" - "zip2 z8.b, z9.b, z7.b\n" - "zip1 z9.b, z9.b, z7.b\n" - "ldp x28, x27, [%x[inptrs], #0x60]\n" - "ld1b { z2.b }, p2/Z, [x26, x13]\n" - "zip1 z7.b, z6.b, z5.b\n" - "zip2 z5.b, z6.b, z5.b\n" - "ld1b { z1.b }, p2/Z, [x25, x13]\n" - "ldp x26, x25, [%x[inptrs], #0x70]\n" - "zip2 z0.b, z4.b, z2.b\n" - "zip1 z4.b, z4.b, z2.b\n" - "ld1b { z31.b }, p2/Z, [x12, x13]\n" - "ld1b { z30.b }, p2/Z, [x11, x13]\n" - "zip1 z2.b, z3.b, z1.b\n" - "zip2 z1.b, z3.b, z1.b\n" - "ld1b { z29.b }, p2/Z, [x10, x13]\n" - "ld1b { z28.b }, p2/Z, [x9, x13]\n" - "zip2 z27.b, z31.b, z29.b\n" - "zip1 z31.b, z31.b, z29.b\n" - "ld1b { z26.b }, p2/Z, [x28, x13]\n" - "ld1b { z25.b }, p2/Z, [x27, x13]\n" - "zip1 z29.b, z30.b, z28.b\n" - "zip2 z28.b, z30.b, z28.b\n" - "ld1b { z24.b }, p2/Z, [x26, x13]\n" - "ld1b { z23.b }, p2/Z, [x25, x13]\n" - "zip2 z22.b, z26.b, z24.b\n" - "zip1 z26.b, z26.b, z24.b\n" - "zip1 z24.b, z25.b, z23.b\n" - "zip2 z23.b, z25.b, z23.b\n" - "ld1w { z6.s }, p1/Z, [%x[params]]\n" - "ld1rw { z21.s }, p1/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" - "ld1rw { z20.s }, p1/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" - "ld1rw { z19.s }, p1/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" - "zip2 z3.b, z4.b, z2.b\n" - "zip1 z4.b, z4.b, z2.b\n" - "ldp x12, x11, [%x[inptrs], #0x0]\n" - "ldp x10, x9, [%x[inptrs], #0x10]\n" - "zip1 z2.b, z0.b, z1.b\n" - "zip2 z1.b, z0.b, z1.b\n" - "ldp x28, x27, [%x[inptrs], #0x20]\n" - "ldp x26, x25, [%x[inptrs], #0x30]\n" - "zip2 z30.b, z31.b, z29.b\n" - "zip1 z31.b, z31.b, z29.b\n" - "zip1 z29.b, z27.b, z28.b\n" - "zip2 z28.b, z27.b, z28.b\n" - "ld1b { z18.b }, p1/Z, [%x[params], #1, MUL VL]\n" - "ld1b { z17.b }, p1/Z, [%x[params], #2, MUL VL]\n" - "zip2 z25.b, z26.b, z24.b\n" - "zip1 z26.b, z26.b, z24.b\n" - "ld1b { z16.b }, p1/Z, [%x[params], #3, MUL VL]\n" + "whilelt p0.b, x13, %x[n_channels]\n" + "ldp x27, x26, [%x[inptrs], #0x0]\n" + "ldp x25, x24, [%x[inptrs], #0x10]\n" + "ldp x23, x22, [%x[inptrs], #0x20]\n" + "ldp x21, x20, [%x[inptrs], #0x30]\n" + "ptrue p2.b\n" + "mov x12, #0x0\n" + "ldp x11, x10, [%x[outptrs], #0x0]\n" + "ldp x9, x28, [%x[outptrs], #0x10]\n" + "ld1b { z15.b }, p0/Z, [x27, x13]\n" + "ld1b { z18.b }, p0/Z, [x26, x13]\n" + "ldp x27, x26, [%x[inptrs], #0x40]\n" + "ld1b { z16.b }, p0/Z, [x25, x13]\n" + "zip2 z17.b, z15.b, z16.b\n" + "zip1 z15.b, z15.b, z16.b\n" + "ld1b { z14.b }, p0/Z, [x24, x13]\n" + "ldp x25, x24, [%x[inptrs], #0x50]\n" + "zip1 z16.b, z18.b, z14.b\n" + "zip2 z14.b, z18.b, z14.b\n" + "ld1b { z13.b }, p0/Z, [x23, x13]\n" + "ld1b { z18.b }, p0/Z, [x22, x13]\n" + "zip2 z12.b, z15.b, z16.b\n" + "zip1 z15.b, z15.b, z16.b\n" + "ldp x23, x22, [%x[inptrs], #0x60]\n" + "ld1b { z16.b }, p0/Z, [x21, x13]\n" + "zip1 z11.b, z17.b, z14.b\n" + "zip2 z14.b, z17.b, z14.b\n" + "ld1b { z10.b }, p0/Z, [x20, x13]\n" + "ldp x21, x20, [%x[inptrs], #0x70]\n" + "zip2 z22.b, z13.b, z16.b\n" + "zip1 z13.b, z13.b, z16.b\n" + "ld1b { z9.b }, p0/Z, [x27, x13]\n" + "ld1b { z17.b }, p0/Z, [x26, x13]\n" + "zip1 z21.b, z18.b, z10.b\n" + "zip2 z10.b, z18.b, z10.b\n" + "ld1b { z16.b }, p0/Z, [x25, x13]\n" + "ld1b { z8.b }, p0/Z, [x24, x13]\n" + "zip2 z20.b, z9.b, z16.b\n" + "zip1 z9.b, z9.b, z16.b\n" + "ld1b { z7.b }, p0/Z, [x23, x13]\n" + "ld1b { z19.b }, p0/Z, [x22, x13]\n" + "zip1 z18.b, z17.b, z8.b\n" + "zip2 z8.b, z17.b, z8.b\n" + "ld1b { z16.b }, p0/Z, [x21, x13]\n" + "ld1b { z6.b }, p0/Z, [x20, x13]\n" + "zip2 z17.b, z7.b, z16.b\n" + "zip1 z7.b, z7.b, z16.b\n" + "zip1 z16.b, z19.b, z6.b\n" + "zip2 z6.b, z19.b, z6.b\n" + "ld1w { z5.s }, p2/Z, [%x[params]]\n" + "ld1rw { z4.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" + "ld1rw { z3.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" + "ld1rw { z2.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" + "zip2 z1.b, z13.b, z21.b\n" + "zip1 z13.b, z13.b, z21.b\n" + "ldp x27, x26, [%x[inptrs], #0x0]\n" + "ldp x25, x23, [%x[inptrs], #0x10]\n" + "zip1 z0.b, z22.b, z10.b\n" + "zip2 z10.b, z22.b, z10.b\n" + "ldp x24, x22, [%x[inptrs], #0x20]\n" + "ldp x21, x20, [%x[inptrs], #0x30]\n" + "zip2 z31.b, z9.b, z18.b\n" + "zip1 z9.b, z9.b, z18.b\n" + "zip1 z30.b, z20.b, z8.b\n" + "zip2 z8.b, z20.b, z8.b\n" + "ld1b { z29.b }, p2/Z, [%x[params], #1, MUL VL]\n" + "ld1b { z28.b }, p2/Z, [%x[params], #2, MUL VL]\n" + "zip2 z27.b, z7.b, z16.b\n" + "zip1 z7.b, z7.b, z16.b\n" + "ld1b { z26.b }, p2/Z, [%x[params], #3, MUL VL]\n" "addvl %x[params], %x[params], #4\n" - "zip1 z24.b, z22.b, z23.b\n" - "zip2 z23.b, z22.b, z23.b\n" - "mov z0.d, z6.d\n" - "mov z27.d, z6.d\n" - "mov z22.d, z6.d\n" + "zip1 z25.b, z17.b, z6.b\n" + "zip2 z6.b, z17.b, z6.b\n" + "mov z24.d, z5.d\n" + "mov z22.d, z5.d\n" + "mov z21.d, z5.d\n" "1:" // Loop - "sdot z6.s, z18.b, z9.b\n" - "sdot z27.s, z18.b, z4.b\n" - "ext z9.b, z9.b, z9.b, #0x1\n" - "whilelt p0.s, x24, %x[n_channels]\n" - "sdot z6.s, z17.b, z4.b\n" - "ext z4.b, z4.b, z4.b, #0x1\n" - "sdot z0.s, z18.b, z9.b\n" - "ld1w { z9.s }, p1/Z, [%x[params]]\n" - "sdot z22.s, z18.b, z4.b\n" - "sdot z27.s, z17.b, z31.b\n" + "sdot z5.s, z29.b, z15.b\n" + "sdot z22.s, z29.b, z13.b\n" + "ext z15.b, z15.b, z15.b, #0x1\n" + "whilelt p0.s, x12, %x[n_channels]\n" + "sdot z5.s, z28.b, z13.b\n" + "ext z13.b, z13.b, z13.b, #0x1\n" + "sdot z24.s, z29.b, z15.b\n" + "ld1w { z17.s }, p2/Z, [%x[params]]\n" + "sdot z21.s, z29.b, z13.b\n" + "sdot z22.s, z28.b, z9.b\n" "incw x13, ALL, MUL #4\n" - "sdot z6.s, z16.b, z31.b\n" - "ext z31.b, z31.b, z31.b, #0x1\n" - "sdot z0.s, z17.b, z4.b\n" - "ld1w { z4.s }, p1/Z, [%x[params], #1, MUL VL]\n" - "sdot z22.s, z17.b, z31.b\n" - "sdot z27.s, z16.b, z26.b\n" - "ext z26.b, z26.b, z26.b, #0x1\n" - ".inst 0x04a974c6 // sqrdmulh z6.s, z6.s, z9.s\n" - "sdot z0.s, z16.b, z31.b\n" - "sdot z22.s, z16.b, z26.b\n" - "and z18.d, z6.d, z4.d\n" + "sdot z5.s, z26.b, z9.b\n" + "ext z9.b, z9.b, z9.b, #0x1\n" + "sdot z24.s, z28.b, z13.b\n" + "ld1w { z20.s }, p2/Z, [%x[params], #1, MUL VL]\n" + "sdot z21.s, z28.b, z9.b\n" + "sdot z22.s, z26.b, z7.b\n" + "ext z7.b, z7.b, z7.b, #0x1\n" + ".inst 0x04b174a5 // sqrdmulh z5.s, z5.s, z17.s\n" + "sdot z24.s, z26.b, z9.b\n" + "sdot z21.s, z26.b, z7.b\n" + "and z16.d, z5.d, z20.d\n" + "asr z16.s, z16.s, #0x1f\n" + ".inst 0x04b17718 // sqrdmulh z24.s, z24.s, z17.s\n" + ".inst 0x04b176d6 // sqrdmulh z22.s, z22.s, z17.s\n" + ".inst 0x04b176b5 // sqrdmulh z21.s, z21.s, z17.s\n" + "sqadd z5.s, z5.s, z16.s\n" + ".inst 0x44828a85 // srshl z5.s, p2/M, z5.s, z20.s\n" + "ld1w { z19.s }, p2/Z, [%x[params], #6, MUL VL]\n" + "and z18.d, z24.d, z20.d\n" + "and z17.d, z22.d, z20.d\n" + "and z16.d, z21.d, z20.d\n" "asr z18.s, z18.s, #0x1f\n" - ".inst 0x04a97400 // sqrdmulh z0.s, z0.s, z9.s\n" - ".inst 0x04a9777b // sqrdmulh z27.s, z27.s, z9.s\n" - ".inst 0x04a976d6 // sqrdmulh z22.s, z22.s, z9.s\n" - "sqadd z6.s, z6.s, z18.s\n" - ".inst 0x44828486 // srshl z6.s, p1/M, z6.s, z4.s\n" - "ld1w { z9.s }, p1/Z, [%x[params], #6, MUL VL]\n" - "and z17.d, z0.d, z4.d\n" - "and z16.d, z27.d, z4.d\n" - "and z18.d, z22.d, z4.d\n" "asr z17.s, z17.s, #0x1f\n" "asr z16.s, z16.s, #0x1f\n" - "asr z18.s, z18.s, #0x1f\n" - "sqadd z0.s, z0.s, z17.s\n" - "sqadd z27.s, z27.s, z16.s\n" - ".inst 0x44828480 // srshl z0.s, p1/M, z0.s, z4.s\n" - ".inst 0x4482849b // srshl z27.s, p1/M, z27.s, z4.s\n" - "sqadd z22.s, z22.s, z18.s\n" - "add z6.s, z6.s, z19.s\n" - ".inst 0x44828496 // srshl z22.s, p1/M, z22.s, z4.s\n" - "smax z6.s, p1/M, z6.s, z21.s\n" - "add z0.s, z0.s, z19.s\n" - "add z27.s, z27.s, z19.s\n" - "smin z6.s, p1/M, z6.s, z20.s\n" - "smax z0.s, p1/M, z0.s, z21.s\n" - "add z22.s, z22.s, z19.s\n" - "smax z27.s, p1/M, z27.s, z21.s\n" - "smax z22.s, p1/M, z22.s, z21.s\n" - "st1b { z6.s }, p0, [x23, x24]\n" - "ld1w { z6.s }, p1/Z, [%x[params], #2, MUL VL]\n" - "ld1b { z18.b }, p1/Z, [%x[params], #3, MUL VL]\n" - "smin z0.s, p1/M, z0.s, z20.s\n" - "smin z27.s, p1/M, z27.s, z20.s\n" - "smin z22.s, p1/M, z22.s, z20.s\n" - "st1b { z0.s }, p0, [x22, x24]\n" - "mov z0.d, z6.d\n" - "ld1b { z17.b }, p1/Z, [%x[params], #4, MUL VL]\n" - "st1b { z27.s }, p0, [x21, x24]\n" - "mov z27.d, z6.d\n" - "sdot z27.s, z18.b, z3.b\n" - "ld1b { z16.b }, p1/Z, [%x[params], #5, MUL VL]\n" - "st1b { z22.s }, p0, [x20, x24]\n" - "mov z22.d, z6.d\n" - "sdot z6.s, z18.b, z8.b\n" - "sdot z6.s, z17.b, z3.b\n" - "ext z8.b, z8.b, z8.b, #0x1\n" - "ext z3.b, z3.b, z3.b, #0x1\n" - "sdot z0.s, z18.b, z8.b\n" - "ld1w { z4.s }, p1/Z, [%x[params], #7, MUL VL]\n" - "sdot z22.s, z18.b, z3.b\n" - "sdot z27.s, z17.b, z30.b\n" - "incw x24\n" - "whilelt p0.s, x24, %x[n_channels]\n" - "sdot z6.s, z16.b, z30.b\n" - "ext z30.b, z30.b, z30.b, #0x1\n" - "sdot z0.s, z17.b, z3.b\n" + "sqadd z24.s, z24.s, z18.s\n" + "sqadd z22.s, z22.s, z17.s\n" + ".inst 0x44828a98 // srshl z24.s, p2/M, z24.s, z20.s\n" + ".inst 0x44828a96 // srshl z22.s, p2/M, z22.s, z20.s\n" + "sqadd z21.s, z21.s, z16.s\n" + "add z5.s, z5.s, z2.s\n" + ".inst 0x44828a95 // srshl z21.s, p2/M, z21.s, z20.s\n" + "smax z5.s, p2/M, z5.s, z4.s\n" + "add z24.s, z24.s, z2.s\n" + "add z22.s, z22.s, z2.s\n" + "smin z5.s, p2/M, z5.s, z3.s\n" + "smax z24.s, p2/M, z24.s, z4.s\n" + "add z21.s, z21.s, z2.s\n" + "smax z22.s, p2/M, z22.s, z4.s\n" + "smax z21.s, p2/M, z21.s, z4.s\n" + "st1b { z5.s }, p0, [x11, x12]\n" + "ld1w { z23.s }, p2/Z, [%x[params], #2, MUL VL]\n" + "ld1b { z18.b }, p2/Z, [%x[params], #3, MUL VL]\n" + "smin z24.s, p2/M, z24.s, z3.s\n" + "smin z22.s, p2/M, z22.s, z3.s\n" + "smin z21.s, p2/M, z21.s, z3.s\n" + "st1b { z24.s }, p0, [x10, x12]\n" + "mov z24.d, z23.d\n" + "ld1b { z17.b }, p2/Z, [%x[params], #4, MUL VL]\n" + "st1b { z22.s }, p0, [x9, x12]\n" + "mov z22.d, z23.d\n" + "sdot z22.s, z18.b, z1.b\n" + "ld1b { z16.b }, p2/Z, [%x[params], #5, MUL VL]\n" + "st1b { z21.s }, p0, [x28, x12]\n" + "mov z21.d, z23.d\n" + "sdot z23.s, z18.b, z12.b\n" + "sdot z23.s, z17.b, z1.b\n" + "ext z12.b, z12.b, z12.b, #0x1\n" + "ext z1.b, z1.b, z1.b, #0x1\n" + "sdot z24.s, z18.b, z12.b\n" + "ld1w { z20.s }, p2/Z, [%x[params], #7, MUL VL]\n" + "sdot z21.s, z18.b, z1.b\n" + "sdot z22.s, z17.b, z31.b\n" + "incw x12\n" + "whilelt p0.s, x12, %x[n_channels]\n" + "sdot z23.s, z16.b, z31.b\n" + "ext z31.b, z31.b, z31.b, #0x1\n" + "sdot z24.s, z17.b, z1.b\n" "addvl %x[params], %x[params], #16\n" - "sdot z22.s, z17.b, z30.b\n" - "sdot z27.s, z16.b, z25.b\n" - "ext z25.b, z25.b, z25.b, #0x1\n" - ".inst 0x04a974c6 // sqrdmulh z6.s, z6.s, z9.s\n" - "sdot z0.s, z16.b, z30.b\n" - "sdot z22.s, z16.b, z25.b\n" - "and z18.d, z6.d, z4.d\n" + "sdot z21.s, z17.b, z31.b\n" + "sdot z22.s, z16.b, z27.b\n" + "ext z27.b, z27.b, z27.b, #0x1\n" + ".inst 0x04b376f7 // sqrdmulh z23.s, z23.s, z19.s\n" + "sdot z24.s, z16.b, z31.b\n" + "sdot z21.s, z16.b, z27.b\n" + "and z16.d, z23.d, z20.d\n" + "asr z16.s, z16.s, #0x1f\n" + ".inst 0x04b37718 // sqrdmulh z24.s, z24.s, z19.s\n" + ".inst 0x04b376d6 // sqrdmulh z22.s, z22.s, z19.s\n" + ".inst 0x04b376b5 // sqrdmulh z21.s, z21.s, z19.s\n" + "sqadd z23.s, z23.s, z16.s\n" + ".inst 0x44828a97 // srshl z23.s, p2/M, z23.s, z20.s\n" + "ld1w { z19.s }, p2/Z, [%x[params], #-4, MUL VL]\n" + "and z18.d, z24.d, z20.d\n" + "and z17.d, z22.d, z20.d\n" + "and z16.d, z21.d, z20.d\n" "asr z18.s, z18.s, #0x1f\n" - ".inst 0x04a97400 // sqrdmulh z0.s, z0.s, z9.s\n" - ".inst 0x04a9777b // sqrdmulh z27.s, z27.s, z9.s\n" - ".inst 0x04a976d6 // sqrdmulh z22.s, z22.s, z9.s\n" - "sqadd z6.s, z6.s, z18.s\n" - ".inst 0x44828486 // srshl z6.s, p1/M, z6.s, z4.s\n" - "ld1w { z9.s }, p1/Z, [%x[params], #-4, MUL VL]\n" - "and z17.d, z0.d, z4.d\n" - "and z16.d, z27.d, z4.d\n" - "and z18.d, z22.d, z4.d\n" "asr z17.s, z17.s, #0x1f\n" "asr z16.s, z16.s, #0x1f\n" + "sqadd z24.s, z24.s, z18.s\n" + "sqadd z22.s, z22.s, z17.s\n" + ".inst 0x44828a98 // srshl z24.s, p2/M, z24.s, z20.s\n" + ".inst 0x44828a96 // srshl z22.s, p2/M, z22.s, z20.s\n" + "sqadd z21.s, z21.s, z16.s\n" + "add z23.s, z23.s, z2.s\n" + ".inst 0x44828a95 // srshl z21.s, p2/M, z21.s, z20.s\n" + "smax z23.s, p2/M, z23.s, z4.s\n" + "add z24.s, z24.s, z2.s\n" + "add z22.s, z22.s, z2.s\n" + "smin z23.s, p2/M, z23.s, z3.s\n" + "smax z24.s, p2/M, z24.s, z4.s\n" + "add z21.s, z21.s, z2.s\n" + "smax z22.s, p2/M, z22.s, z4.s\n" + "smax z21.s, p2/M, z21.s, z4.s\n" + "st1b { z23.s }, p0, [x11, x12]\n" + "ld1w { z23.s }, p2/Z, [%x[params], #-8, MUL VL]\n" + "ld1b { z18.b }, p2/Z, [%x[params], #-7, MUL VL]\n" + "smin z24.s, p2/M, z24.s, z3.s\n" + "smin z22.s, p2/M, z22.s, z3.s\n" + "smin z21.s, p2/M, z21.s, z3.s\n" + "st1b { z24.s }, p0, [x10, x12]\n" + "mov z24.d, z23.d\n" + "ld1b { z17.b }, p2/Z, [%x[params], #-6, MUL VL]\n" + "st1b { z22.s }, p0, [x9, x12]\n" + "mov z22.d, z23.d\n" + "sdot z22.s, z18.b, z0.b\n" + "ld1b { z16.b }, p2/Z, [%x[params], #-5, MUL VL]\n" + "st1b { z21.s }, p0, [x28, x12]\n" + "mov z21.d, z23.d\n" + "sdot z23.s, z18.b, z11.b\n" + "sdot z23.s, z17.b, z0.b\n" + "ext z11.b, z11.b, z11.b, #0x1\n" + "ext z0.b, z0.b, z0.b, #0x1\n" + "sdot z24.s, z18.b, z11.b\n" + "ld1w { z20.s }, p2/Z, [%x[params], #-3, MUL VL]\n" + "sdot z21.s, z18.b, z0.b\n" + "sdot z22.s, z17.b, z30.b\n" + "incw x12\n" + "whilelt p0.s, x12, %x[n_channels]\n" + "sdot z23.s, z16.b, z30.b\n" + "ext z30.b, z30.b, z30.b, #0x1\n" + "sdot z24.s, z17.b, z0.b\n" + "sdot z21.s, z17.b, z30.b\n" + "sdot z22.s, z16.b, z25.b\n" + "ext z25.b, z25.b, z25.b, #0x1\n" + ".inst 0x04b376f7 // sqrdmulh z23.s, z23.s, z19.s\n" + "sdot z24.s, z16.b, z30.b\n" + "sdot z21.s, z16.b, z25.b\n" + "and z16.d, z23.d, z20.d\n" + "asr z16.s, z16.s, #0x1f\n" + ".inst 0x04b37718 // sqrdmulh z24.s, z24.s, z19.s\n" + ".inst 0x04b376d6 // sqrdmulh z22.s, z22.s, z19.s\n" + ".inst 0x04b376b5 // sqrdmulh z21.s, z21.s, z19.s\n" + "sqadd z23.s, z23.s, z16.s\n" + ".inst 0x44828a97 // srshl z23.s, p2/M, z23.s, z20.s\n" + "ld1w { z19.s }, p2/Z, [%x[params], #2, MUL VL]\n" + "and z18.d, z24.d, z20.d\n" + "and z17.d, z22.d, z20.d\n" + "and z16.d, z21.d, z20.d\n" "asr z18.s, z18.s, #0x1f\n" - "sqadd z0.s, z0.s, z17.s\n" - "sqadd z27.s, z27.s, z16.s\n" - ".inst 0x44828480 // srshl z0.s, p1/M, z0.s, z4.s\n" - ".inst 0x4482849b // srshl z27.s, p1/M, z27.s, z4.s\n" - "sqadd z22.s, z22.s, z18.s\n" - "add z6.s, z6.s, z19.s\n" - ".inst 0x44828496 // srshl z22.s, p1/M, z22.s, z4.s\n" - "smax z6.s, p1/M, z6.s, z21.s\n" - "add z0.s, z0.s, z19.s\n" - "add z27.s, z27.s, z19.s\n" - "smin z6.s, p1/M, z6.s, z20.s\n" - "smax z0.s, p1/M, z0.s, z21.s\n" - "add z22.s, z22.s, z19.s\n" - "smax z27.s, p1/M, z27.s, z21.s\n" - "smax z22.s, p1/M, z22.s, z21.s\n" - "st1b { z6.s }, p0, [x23, x24]\n" - "ld1w { z6.s }, p1/Z, [%x[params], #-8, MUL VL]\n" - "ld1b { z18.b }, p1/Z, [%x[params], #-7, MUL VL]\n" - "smin z0.s, p1/M, z0.s, z20.s\n" - "smin z27.s, p1/M, z27.s, z20.s\n" - "smin z22.s, p1/M, z22.s, z20.s\n" - "st1b { z0.s }, p0, [x22, x24]\n" - "mov z0.d, z6.d\n" - "ld1b { z17.b }, p1/Z, [%x[params], #-6, MUL VL]\n" - "st1b { z27.s }, p0, [x21, x24]\n" - "mov z27.d, z6.d\n" - "sdot z27.s, z18.b, z2.b\n" - "ld1b { z16.b }, p1/Z, [%x[params], #-5, MUL VL]\n" - "st1b { z22.s }, p0, [x20, x24]\n" - "mov z22.d, z6.d\n" - "sdot z6.s, z18.b, z7.b\n" - "sdot z6.s, z17.b, z2.b\n" - "ext z7.b, z7.b, z7.b, #0x1\n" - "ext z2.b, z2.b, z2.b, #0x1\n" - "sdot z0.s, z18.b, z7.b\n" - "ld1w { z4.s }, p1/Z, [%x[params], #-3, MUL VL]\n" - "sdot z22.s, z18.b, z2.b\n" - "sdot z27.s, z17.b, z29.b\n" - "incw x24\n" - "whilelt p0.s, x24, %x[n_channels]\n" - "sdot z6.s, z16.b, z29.b\n" - "ext z29.b, z29.b, z29.b, #0x1\n" - "sdot z0.s, z17.b, z2.b\n" - "sdot z22.s, z17.b, z29.b\n" - "sdot z27.s, z16.b, z24.b\n" - "ext z24.b, z24.b, z24.b, #0x1\n" - ".inst 0x04a974c6 // sqrdmulh z6.s, z6.s, z9.s\n" - "sdot z0.s, z16.b, z29.b\n" - "sdot z22.s, z16.b, z24.b\n" - "and z18.d, z6.d, z4.d\n" - "asr z18.s, z18.s, #0x1f\n" - ".inst 0x04a97400 // sqrdmulh z0.s, z0.s, z9.s\n" - ".inst 0x04a9777b // sqrdmulh z27.s, z27.s, z9.s\n" - ".inst 0x04a976d6 // sqrdmulh z22.s, z22.s, z9.s\n" - "sqadd z6.s, z6.s, z18.s\n" - ".inst 0x44828486 // srshl z6.s, p1/M, z6.s, z4.s\n" - "ld1w { z9.s }, p1/Z, [%x[params], #2, MUL VL]\n" - "and z17.d, z0.d, z4.d\n" - "and z16.d, z27.d, z4.d\n" - "and z18.d, z22.d, z4.d\n" "asr z17.s, z17.s, #0x1f\n" "asr z16.s, z16.s, #0x1f\n" - "asr z18.s, z18.s, #0x1f\n" - "sqadd z0.s, z0.s, z17.s\n" - "sqadd z27.s, z27.s, z16.s\n" - ".inst 0x44828480 // srshl z0.s, p1/M, z0.s, z4.s\n" - ".inst 0x4482849b // srshl z27.s, p1/M, z27.s, z4.s\n" - "sqadd z22.s, z22.s, z18.s\n" - "add z6.s, z6.s, z19.s\n" - ".inst 0x44828496 // srshl z22.s, p1/M, z22.s, z4.s\n" - "smax z6.s, p1/M, z6.s, z21.s\n" - "add z0.s, z0.s, z19.s\n" - "add z27.s, z27.s, z19.s\n" - "smin z6.s, p1/M, z6.s, z20.s\n" - "smax z0.s, p1/M, z0.s, z21.s\n" - "add z22.s, z22.s, z19.s\n" - "smax z27.s, p1/M, z27.s, z21.s\n" - "smax z22.s, p1/M, z22.s, z21.s\n" - "st1b { z6.s }, p0, [x23, x24]\n" - "ld1w { z6.s }, p1/Z, [%x[params], #-2, MUL VL]\n" - "ld1b { z18.b }, p1/Z, [%x[params], #-1, MUL VL]\n" - "smin z0.s, p1/M, z0.s, z20.s\n" - "smin z27.s, p1/M, z27.s, z20.s\n" - "smin z22.s, p1/M, z22.s, z20.s\n" - "st1b { z0.s }, p0, [x22, x24]\n" - "mov z0.d, z6.d\n" - "ld1b { z17.b }, p1/Z, [%x[params]]\n" - "st1b { z27.s }, p0, [x21, x24]\n" - "mov z27.d, z6.d\n" - "sdot z27.s, z18.b, z1.b\n" - "ld1b { z16.b }, p1/Z, [%x[params], #1, MUL VL]\n" - "st1b { z22.s }, p0, [x20, x24]\n" - "mov z22.d, z6.d\n" - "sdot z6.s, z18.b, z5.b\n" - "sdot z6.s, z17.b, z1.b\n" - "ext z5.b, z5.b, z5.b, #0x1\n" - "ext z1.b, z1.b, z1.b, #0x1\n" - "sdot z0.s, z18.b, z5.b\n" - "ld1w { z4.s }, p1/Z, [%x[params], #3, MUL VL]\n" - "sdot z22.s, z18.b, z1.b\n" - "sdot z27.s, z17.b, z28.b\n" - "incw x24\n" - "whilelt p0.s, x24, %x[n_channels]\n" - "sdot z6.s, z16.b, z28.b\n" - "ext z28.b, z28.b, z28.b, #0x1\n" - "sdot z0.s, z17.b, z1.b\n" - "whilelt p2.b, x13, %x[n_channels]\n" - "sdot z22.s, z17.b, z28.b\n" - "sdot z27.s, z16.b, z23.b\n" - "ext z23.b, z23.b, z23.b, #0x1\n" - "ld1b { z8.b }, p2/Z, [x11, x13]\n" - ".inst 0x04a974c6 // sqrdmulh z6.s, z6.s, z9.s\n" - "sdot z0.s, z16.b, z28.b\n" - "sdot z22.s, z16.b, z23.b\n" - "ld1b { z7.b }, p2/Z, [x10, x13]\n" - "and z18.d, z6.d, z4.d\n" - "asr z18.s, z18.s, #0x1f\n" - "ld1b { z5.b }, p2/Z, [x9, x13]\n" - "ld1b { z3.b }, p2/Z, [x27, x13]\n" - ".inst 0x04a97400 // sqrdmulh z0.s, z0.s, z9.s\n" - ".inst 0x04a9777b // sqrdmulh z27.s, z27.s, z9.s\n" - "ld1b { z2.b }, p2/Z, [x26, x13]\n" - "ld1b { z1.b }, p2/Z, [x25, x13]\n" - ".inst 0x04a976d6 // sqrdmulh z22.s, z22.s, z9.s\n" - "sqadd z6.s, z6.s, z18.s\n" - ".inst 0x44828486 // srshl z6.s, p1/M, z6.s, z4.s\n" - "ld1b { z9.b }, p2/Z, [x12, x13]\n" - "and z17.d, z0.d, z4.d\n" - "and z16.d, z27.d, z4.d\n" - "ldp x12, x11, [%x[inptrs], #0x40]\n" - "ldp x10, x9, [%x[inptrs], #0x50]\n" - "and z18.d, z22.d, z4.d\n" + "sqadd z24.s, z24.s, z18.s\n" + "sqadd z22.s, z22.s, z17.s\n" + ".inst 0x44828a98 // srshl z24.s, p2/M, z24.s, z20.s\n" + ".inst 0x44828a96 // srshl z22.s, p2/M, z22.s, z20.s\n" + "sqadd z21.s, z21.s, z16.s\n" + "add z23.s, z23.s, z2.s\n" + ".inst 0x44828a95 // srshl z21.s, p2/M, z21.s, z20.s\n" + "smax z23.s, p2/M, z23.s, z4.s\n" + "add z24.s, z24.s, z2.s\n" + "add z22.s, z22.s, z2.s\n" + "smin z23.s, p2/M, z23.s, z3.s\n" + "smax z24.s, p2/M, z24.s, z4.s\n" + "add z21.s, z21.s, z2.s\n" + "smax z22.s, p2/M, z22.s, z4.s\n" + "smax z21.s, p2/M, z21.s, z4.s\n" + "st1b { z23.s }, p0, [x11, x12]\n" + "ld1w { z23.s }, p2/Z, [%x[params], #-2, MUL VL]\n" + "ld1b { z18.b }, p2/Z, [%x[params], #-1, MUL VL]\n" + "smin z24.s, p2/M, z24.s, z3.s\n" + "smin z22.s, p2/M, z22.s, z3.s\n" + "smin z21.s, p2/M, z21.s, z3.s\n" + "st1b { z24.s }, p0, [x10, x12]\n" + "mov z29.d, z23.d\n" + "ld1b { z17.b }, p2/Z, [%x[params]]\n" + "st1b { z22.s }, p0, [x9, x12]\n" + "mov z28.d, z23.d\n" + "sdot z28.s, z18.b, z10.b\n" + "ld1b { z16.b }, p2/Z, [%x[params], #1, MUL VL]\n" + "st1b { z21.s }, p0, [x28, x12]\n" + "mov z27.d, z23.d\n" + "sdot z23.s, z18.b, z14.b\n" + "sdot z23.s, z17.b, z10.b\n" + "ext z14.b, z14.b, z14.b, #0x1\n" + "ext z10.b, z10.b, z10.b, #0x1\n" + "sdot z29.s, z18.b, z14.b\n" + "ld1w { z22.s }, p2/Z, [%x[params], #3, MUL VL]\n" + "sdot z27.s, z18.b, z10.b\n" + "sdot z28.s, z17.b, z8.b\n" + "incw x12\n" + "whilelt p1.s, x12, %x[n_channels]\n" + "sdot z23.s, z16.b, z8.b\n" + "ext z8.b, z8.b, z8.b, #0x1\n" + "sdot z29.s, z17.b, z10.b\n" + "whilelt p0.b, x13, %x[n_channels]\n" + "sdot z27.s, z17.b, z8.b\n" + "sdot z28.s, z16.b, z6.b\n" + "ext z6.b, z6.b, z6.b, #0x1\n" + "ld1b { z26.b }, p0/Z, [x26, x13]\n" + ".inst 0x04b376f7 // sqrdmulh z23.s, z23.s, z19.s\n" + "sdot z29.s, z16.b, z8.b\n" + "sdot z27.s, z16.b, z6.b\n" + "ld1b { z21.b }, p0/Z, [x25, x13]\n" + "and z16.d, z23.d, z22.d\n" + "asr z16.s, z16.s, #0x1f\n" + "ld1b { z14.b }, p0/Z, [x23, x13]\n" + "ld1b { z25.b }, p0/Z, [x22, x13]\n" + ".inst 0x04b377bd // sqrdmulh z29.s, z29.s, z19.s\n" + ".inst 0x04b3779c // sqrdmulh z28.s, z28.s, z19.s\n" + "ld1b { z20.b }, p0/Z, [x21, x13]\n" + "ld1b { z10.b }, p0/Z, [x20, x13]\n" + ".inst 0x04b3777b // sqrdmulh z27.s, z27.s, z19.s\n" + "sqadd z23.s, z23.s, z16.s\n" + ".inst 0x44828ad7 // srshl z23.s, p2/M, z23.s, z22.s\n" + "ld1b { z15.b }, p0/Z, [x27, x13]\n" + "and z19.d, z29.d, z22.d\n" + "and z17.d, z28.d, z22.d\n" + "ldp x23, x22, [%x[inptrs], #0x40]\n" + "ldp x21, x20, [%x[inptrs], #0x50]\n" + "and z16.d, z27.d, z22.d\n" + "asr z19.s, z19.s, #0x1f\n" + "ld1b { z9.b }, p0/Z, [x23, x13]\n" + "ld1b { z24.b }, p0/Z, [x22, x13]\n" "asr z17.s, z17.s, #0x1f\n" - "ld1b { z31.b }, p2/Z, [x12, x13]\n" - "ld1b { z30.b }, p2/Z, [x11, x13]\n" "asr z16.s, z16.s, #0x1f\n" - "asr z18.s, z18.s, #0x1f\n" - "ld1b { z29.b }, p2/Z, [x10, x13]\n" - "ld1b { z28.b }, p2/Z, [x9, x13]\n" - "sqadd z0.s, z0.s, z17.s\n" + "ld1b { z18.b }, p0/Z, [x21, x13]\n" + "ld1b { z8.b }, p0/Z, [x20, x13]\n" + "sqadd z29.s, z29.s, z19.s\n" + "sqadd z28.s, z28.s, z17.s\n" + ".inst 0x44828add // srshl z29.s, p2/M, z29.s, z22.s\n" + ".inst 0x44828adc // srshl z28.s, p2/M, z28.s, z22.s\n" "sqadd z27.s, z27.s, z16.s\n" - ".inst 0x44828480 // srshl z0.s, p1/M, z0.s, z4.s\n" - ".inst 0x4482849b // srshl z27.s, p1/M, z27.s, z4.s\n" - "sqadd z22.s, z22.s, z18.s\n" - "add z6.s, z6.s, z19.s\n" - ".inst 0x44828496 // srshl z22.s, p1/M, z22.s, z4.s\n" - "smax z6.s, p1/M, z6.s, z21.s\n" - "add z0.s, z0.s, z19.s\n" - "add z27.s, z27.s, z19.s\n" - "ld1b { z4.b }, p2/Z, [x28, x13]\n" - "ldp x28, x27, [%x[inptrs], #0x60]\n" - "add z22.s, z22.s, z19.s\n" - "ldp x26, x25, [%x[inptrs], #0x70]\n" - "smin z6.s, p1/M, z6.s, z20.s\n" - "smax z0.s, p1/M, z0.s, z21.s\n" - "smax z27.s, p1/M, z27.s, z21.s\n" - "smax z22.s, p1/M, z22.s, z21.s\n" - "st1b { z6.s }, p0, [x23, x24]\n" - "ld1b { z26.b }, p2/Z, [x28, x13]\n" - "ld1b { z25.b }, p2/Z, [x27, x13]\n" - "ld1b { z24.b }, p2/Z, [x26, x13]\n" - "zip2 z6.b, z9.b, z7.b\n" - "zip1 z9.b, z9.b, z7.b\n" - "ld1b { z23.b }, p2/Z, [x25, x13]\n" - "zip1 z7.b, z8.b, z5.b\n" - "zip2 z5.b, z8.b, z5.b\n" - "smin z0.s, p1/M, z0.s, z20.s\n" - "smin z27.s, p1/M, z27.s, z20.s\n" - "smin z22.s, p1/M, z22.s, z20.s\n" - "st1b { z0.s }, p0, [x22, x24]\n" - "zip2 z8.b, z9.b, z7.b\n" - "st1b { z27.s }, p0, [x21, x24]\n" - "zip1 z9.b, z9.b, z7.b\n" - "zip1 z7.b, z6.b, z5.b\n" - "ldp x12, x11, [%x[inptrs], #0x0]\n" - "st1b { z22.s }, p0, [x20, x24]\n" - "zip2 z5.b, z6.b, z5.b\n" - "zip2 z0.b, z4.b, z2.b\n" - "ld1w { z6.s }, p1/Z, [%x[params], #4, MUL VL]\n" - "zip1 z4.b, z4.b, z2.b\n" - "zip1 z2.b, z3.b, z1.b\n" - "incw x24\n" - "ldp x10, x9, [%x[inptrs], #0x10]\n" - "zip2 z1.b, z3.b, z1.b\n" - "zip2 z27.b, z31.b, z29.b\n" - "ldp x28, x27, [%x[inptrs], #0x20]\n" - "ldp x26, x25, [%x[inptrs], #0x30]\n" - "zip1 z31.b, z31.b, z29.b\n" - "zip1 z29.b, z30.b, z28.b\n" - "ld1b { z18.b }, p1/Z, [%x[params], #5, MUL VL]\n" - "ld1b { z17.b }, p1/Z, [%x[params], #6, MUL VL]\n" - "zip2 z28.b, z30.b, z28.b\n" - "zip2 z22.b, z26.b, z24.b\n" - "ld1b { z16.b }, p1/Z, [%x[params], #7, MUL VL]\n" + "add z23.s, z23.s, z2.s\n" + ".inst 0x44828adb // srshl z27.s, p2/M, z27.s, z22.s\n" + "smax z23.s, p2/M, z23.s, z4.s\n" + "add z29.s, z29.s, z2.s\n" + "add z28.s, z28.s, z2.s\n" + "ld1b { z13.b }, p0/Z, [x24, x13]\n" + "ldp x23, x22, [%x[inptrs], #0x60]\n" + "add z27.s, z27.s, z2.s\n" + "ldp x21, x20, [%x[inptrs], #0x70]\n" + "smin z23.s, p2/M, z23.s, z3.s\n" + "smax z29.s, p2/M, z29.s, z4.s\n" + "smax z28.s, p2/M, z28.s, z4.s\n" + "smax z27.s, p2/M, z27.s, z4.s\n" + "st1b { z23.s }, p1, [x11, x12]\n" + "ld1b { z7.b }, p0/Z, [x23, x13]\n" + "ld1b { z23.b }, p0/Z, [x22, x13]\n" + "ld1b { z22.b }, p0/Z, [x21, x13]\n" + "zip2 z17.b, z15.b, z21.b\n" + "zip1 z15.b, z15.b, z21.b\n" + "ld1b { z6.b }, p0/Z, [x20, x13]\n" + "zip1 z16.b, z26.b, z14.b\n" + "zip2 z14.b, z26.b, z14.b\n" + "smin z29.s, p2/M, z29.s, z3.s\n" + "smin z28.s, p2/M, z28.s, z3.s\n" + "smin z27.s, p2/M, z27.s, z3.s\n" + "st1b { z29.s }, p1, [x10, x12]\n" + "zip2 z12.b, z15.b, z16.b\n" + "st1b { z28.s }, p1, [x9, x12]\n" + "zip1 z15.b, z15.b, z16.b\n" + "zip1 z11.b, z17.b, z14.b\n" + "ldp x27, x26, [%x[inptrs], #0x0]\n" + "st1b { z27.s }, p1, [x28, x12]\n" + "zip2 z14.b, z17.b, z14.b\n" + "zip2 z21.b, z13.b, z20.b\n" + "ld1w { z5.s }, p2/Z, [%x[params], #4, MUL VL]\n" + "zip1 z13.b, z13.b, z20.b\n" + "zip1 z20.b, z25.b, z10.b\n" + "incw x12\n" + "ldp x25, x23, [%x[inptrs], #0x10]\n" + "zip2 z10.b, z25.b, z10.b\n" + "zip2 z19.b, z9.b, z18.b\n" + "ldp x24, x22, [%x[inptrs], #0x20]\n" + "ldp x21, x20, [%x[inptrs], #0x30]\n" + "zip1 z9.b, z9.b, z18.b\n" + "zip1 z18.b, z24.b, z8.b\n" + "ld1b { z29.b }, p2/Z, [%x[params], #5, MUL VL]\n" + "ld1b { z28.b }, p2/Z, [%x[params], #6, MUL VL]\n" + "zip2 z8.b, z24.b, z8.b\n" + "zip2 z17.b, z7.b, z22.b\n" + "ld1b { z26.b }, p2/Z, [%x[params], #7, MUL VL]\n" "addvl %x[params], %x[params], #8\n" - "zip1 z26.b, z26.b, z24.b\n" - "zip1 z24.b, z25.b, z23.b\n" - "zip2 z23.b, z25.b, z23.b\n" - "zip2 z3.b, z4.b, z2.b\n" - "zip1 z4.b, z4.b, z2.b\n" - "zip1 z2.b, z0.b, z1.b\n" - "zip2 z1.b, z0.b, z1.b\n" - "zip2 z30.b, z31.b, z29.b\n" - "zip1 z31.b, z31.b, z29.b\n" - "zip1 z29.b, z27.b, z28.b\n" - "zip2 z28.b, z27.b, z28.b\n" - "zip2 z25.b, z26.b, z24.b\n" - "zip1 z26.b, z26.b, z24.b\n" - "zip1 z24.b, z22.b, z23.b\n" - "zip2 z23.b, z22.b, z23.b\n" - "mov z0.d, z6.d\n" - "mov z27.d, z6.d\n" - "mov z22.d, z6.d\n" + "zip1 z7.b, z7.b, z22.b\n" + "zip1 z16.b, z23.b, z6.b\n" + "zip2 z6.b, z23.b, z6.b\n" + "zip2 z1.b, z13.b, z20.b\n" + "zip1 z13.b, z13.b, z20.b\n" + "zip1 z0.b, z21.b, z10.b\n" + "zip2 z10.b, z21.b, z10.b\n" + "zip2 z31.b, z9.b, z18.b\n" + "zip1 z9.b, z9.b, z18.b\n" + "zip1 z30.b, z19.b, z8.b\n" + "zip2 z8.b, z19.b, z8.b\n" + "zip2 z27.b, z7.b, z16.b\n" + "zip1 z7.b, z7.b, z16.b\n" + "zip1 z25.b, z17.b, z6.b\n" + "zip2 z6.b, z17.b, z6.b\n" + "mov z24.d, z5.d\n" + "mov z22.d, z5.d\n" + "mov z21.d, z5.d\n" "b.any 1b\n" : [params] "+&r" (params) : [inptrs] "r" (inptrs), [n_channels] "r" (n_channels), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [outptrs] "r" (outptrs), [qp] "r" (&qp) - : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp index 648b2da1634c77117ac2df585e1cc43ceea8bc2f..6b006e8d51c3ddcddae8151ee0721a9d734a5457 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include @@ -64,7 +64,7 @@ class sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst : public DepthwiseDepthfirstS ) const override { interleave_sve_u8q_3x3_dot::pack_parameters( - args.input_channels, buffer, reinterpret_cast(biases), + args.input_channels * args.channel_multiplier, buffer, reinterpret_cast(biases), reinterpret_cast(weights), qp, ld_weight_col, ld_weight_row ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp index 257c4d44dcf06d04bacd55b52ea5e00b8e4b5cbc..f0860c98b97bf82828d2877c0e882b17b051d7a0 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp @@ -30,472 +30,464 @@ namespace arm_conv { namespace depthwise { -void sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst_impl( - const unsigned int n_channels, - const uint8_t *const *const inptrs, - const uint8_t *params, - const int32_t *, // Bias, should be wrapped into the parameters - const arm_gemm::Requantize32& qp, - const int32_t *, const int32_t *, // Requant parameters, also wrapped - uint8_t *const *const outptrs -) +void sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst_impl(const unsigned int n_channels, const uint8_t *const *const inptrs, const uint8_t *params, const int32_t *, const arm_gemm::Requantize32& qp, const int32_t *, const int32_t *, uint8_t *const *const outptrs) { __asm__ __volatile__( - "mov x13, #0x0\n" - "whilelt p2.b, x13, %x[n_channels]\n" - "ldp x12, x11, [%x[inptrs], #0x0]\n" - "ldp x10, x9, [%x[inptrs], #0x10]\n" - "ldp x28, x27, [%x[inptrs], #0x20]\n" - "ldp x26, x25, [%x[inptrs], #0x30]\n" + "mov x14, #0x0\n" + "whilelt p0.b, x14, %x[n_channels]\n" + "ldp x27, x26, [%x[inptrs], #0x0]\n" + "ldp x25, x24, [%x[inptrs], #0x10]\n" + "ldp x23, x22, [%x[inptrs], #0x20]\n" + "ldp x13, x21, [%x[inptrs], #0x30]\n" "mov x20, #0x1\n" - "ptrue p1.b\n" - "ldp x24, x23, [%x[outptrs], #0x0]\n" - "ldp x22, x21, [%x[outptrs], #0x10]\n" + "ptrue p2.b\n" + "ldp x12, x11, [%x[outptrs], #0x0]\n" + "ldp x10, x9, [%x[outptrs], #0x10]\n" "orr x20, x20, #0x100\n" "orr x20, x20, #0x10000\n" - "ld1b { z14.b }, p2/Z, [x12, x13]\n" - "ld1b { z13.b }, p2/Z, [x11, x13]\n" - "dup z12.s, w20\n" - "mov x20, #0x0\n" - "ldp x12, x11, [%x[inptrs], #0x40]\n" - "ld1b { z11.b }, p2/Z, [x10, x13]\n" - "zip2 z10.b, z14.b, z11.b\n" - "zip1 z14.b, z14.b, z11.b\n" - "ld1b { z9.b }, p2/Z, [x9, x13]\n" - "ldp x10, x9, [%x[inptrs], #0x50]\n" - "zip1 z11.b, z13.b, z9.b\n" - "zip2 z9.b, z13.b, z9.b\n" - "ld1b { z8.b }, p2/Z, [x28, x13]\n" - "ld1b { z7.b }, p2/Z, [x27, x13]\n" - "zip2 z13.b, z14.b, z11.b\n" - "zip1 z14.b, z14.b, z11.b\n" - "ldp x28, x27, [%x[inptrs], #0x60]\n" - "ld1b { z6.b }, p2/Z, [x26, x13]\n" - "zip1 z11.b, z10.b, z9.b\n" - "zip2 z9.b, z10.b, z9.b\n" - "ld1b { z5.b }, p2/Z, [x25, x13]\n" - "ldp x26, x25, [%x[inptrs], #0x70]\n" - "zip2 z4.b, z8.b, z6.b\n" - "zip1 z8.b, z8.b, z6.b\n" - "ld1b { z3.b }, p2/Z, [x12, x13]\n" - "ld1b { z2.b }, p2/Z, [x11, x13]\n" - "zip1 z6.b, z7.b, z5.b\n" - "zip2 z5.b, z7.b, z5.b\n" - "ld1b { z1.b }, p2/Z, [x10, x13]\n" - "ld1b { z0.b }, p2/Z, [x9, x13]\n" - "zip2 z31.b, z3.b, z1.b\n" - "zip1 z3.b, z3.b, z1.b\n" - "ld1b { z30.b }, p2/Z, [x28, x13]\n" - "ld1b { z29.b }, p2/Z, [x27, x13]\n" - "zip1 z1.b, z2.b, z0.b\n" - "zip2 z0.b, z2.b, z0.b\n" - "ld1b { z28.b }, p2/Z, [x26, x13]\n" - "ld1b { z27.b }, p2/Z, [x25, x13]\n" - "zip2 z26.b, z30.b, z28.b\n" - "zip1 z30.b, z30.b, z28.b\n" - "zip1 z28.b, z29.b, z27.b\n" - "zip2 z27.b, z29.b, z27.b\n" - "ld1w { z10.s }, p1/Z, [%x[params]]\n" - "ld1rw { z25.s }, p1/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" - "ld1rw { z24.s }, p1/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" - "ld1rw { z23.s }, p1/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" - "zip2 z7.b, z8.b, z6.b\n" - "zip1 z8.b, z8.b, z6.b\n" - "ld1rw { z22.s }, p1/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" - "ldp x12, x11, [%x[inptrs], #0x0]\n" - "zip1 z6.b, z4.b, z5.b\n" - "zip2 z5.b, z4.b, z5.b\n" - "ldp x10, x9, [%x[inptrs], #0x10]\n" - "ldp x28, x27, [%x[inptrs], #0x20]\n" - "zip2 z2.b, z3.b, z1.b\n" - "zip1 z3.b, z3.b, z1.b\n" - "ldp x26, x25, [%x[inptrs], #0x30]\n" - "zip1 z1.b, z31.b, z0.b\n" - "zip2 z0.b, z31.b, z0.b\n" - "ld1b { z21.b }, p1/Z, [%x[params], #1, MUL VL]\n" - "zip2 z29.b, z30.b, z28.b\n" - "zip1 z30.b, z30.b, z28.b\n" - "ld1b { z16.b }, p1/Z, [%x[params], #2, MUL VL]\n" - "ld1b { z20.b }, p1/Z, [%x[params], #3, MUL VL]\n" - "zip1 z28.b, z26.b, z27.b\n" - "zip2 z27.b, z26.b, z27.b\n" + "ld1b { z15.b }, p0/Z, [x27, x14]\n" + "ld1b { z21.b }, p0/Z, [x26, x14]\n" + "dup z25.s, w20\n" + "mov x28, #0x0\n" + "ldp x27, x26, [%x[inptrs], #0x40]\n" + "ld1b { z31.b }, p0/Z, [x25, x14]\n" + "zip2 z16.b, z15.b, z31.b\n" + "zip1 z15.b, z15.b, z31.b\n" + "ld1b { z29.b }, p0/Z, [x24, x14]\n" + "ldp x25, x24, [%x[inptrs], #0x50]\n" + "zip1 z30.b, z21.b, z29.b\n" + "zip2 z29.b, z21.b, z29.b\n" + "ld1b { z9.b }, p0/Z, [x23, x14]\n" + "ld1b { z20.b }, p0/Z, [x22, x14]\n" + "zip2 z13.b, z15.b, z30.b\n" + "zip1 z15.b, z15.b, z30.b\n" + "ldp x23, x22, [%x[inptrs], #0x60]\n" + "ld1b { z5.b }, p0/Z, [x13, x14]\n" + "zip1 z14.b, z16.b, z29.b\n" + "zip2 z29.b, z16.b, z29.b\n" + "ld1b { z17.b }, p0/Z, [x21, x14]\n" + "ldp x21, x20, [%x[inptrs], #0x70]\n" + "zip2 z31.b, z9.b, z5.b\n" + "zip1 z9.b, z9.b, z5.b\n" + "ld1b { z18.b }, p0/Z, [x27, x14]\n" + "ld1b { z28.b }, p0/Z, [x26, x14]\n" + "zip1 z21.b, z20.b, z17.b\n" + "zip2 z17.b, z20.b, z17.b\n" + "ld1b { z6.b }, p0/Z, [x25, x14]\n" + "ld1b { z4.b }, p0/Z, [x24, x14]\n" + "zip2 z23.b, z18.b, z6.b\n" + "zip1 z18.b, z18.b, z6.b\n" + "ld1b { z2.b }, p0/Z, [x23, x14]\n" + "ld1b { z19.b }, p0/Z, [x22, x14]\n" + "zip1 z24.b, z28.b, z4.b\n" + "zip2 z4.b, z28.b, z4.b\n" + "ld1b { z16.b }, p0/Z, [x21, x14]\n" + "ld1b { z5.b }, p0/Z, [x20, x14]\n" + "zip2 z22.b, z2.b, z16.b\n" + "zip1 z2.b, z2.b, z16.b\n" + "zip1 z0.b, z19.b, z5.b\n" + "zip2 z5.b, z19.b, z5.b\n" + "ld1w { z10.s }, p2/Z, [%x[params]]\n" + "ld1rw { z7.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" + "ld1rw { z6.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" + "ld1rw { z8.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" + "zip2 z19.b, z9.b, z21.b\n" + "zip1 z9.b, z9.b, z21.b\n" + "ld1rw { z16.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" + "ldp x27, x26, [%x[inptrs], #0x0]\n" + "zip1 z11.b, z31.b, z17.b\n" + "zip2 z17.b, z31.b, z17.b\n" + "ldp x25, x23, [%x[inptrs], #0x10]\n" + "ldp x24, x22, [%x[inptrs], #0x20]\n" + "zip2 z12.b, z18.b, z24.b\n" + "zip1 z18.b, z18.b, z24.b\n" + "ldp x21, x20, [%x[inptrs], #0x30]\n" + "zip1 z20.b, z23.b, z4.b\n" + "zip2 z4.b, z23.b, z4.b\n" + "ld1b { z26.b }, p2/Z, [%x[params], #1, MUL VL]\n" + "zip2 z24.b, z2.b, z0.b\n" + "zip1 z2.b, z2.b, z0.b\n" + "ld1b { z3.b }, p2/Z, [%x[params], #2, MUL VL]\n" + "ld1b { z1.b }, p2/Z, [%x[params], #3, MUL VL]\n" + "zip1 z0.b, z22.b, z5.b\n" + "zip2 z5.b, z22.b, z5.b\n" "addvl %x[params], %x[params], #4\n" - "mov z4.d, z10.d\n" + "mov z22.d, z10.d\n" "mov z31.d, z10.d\n" - "mov z26.d, z10.d\n" + "mov z21.d, z10.d\n" "1:" // Loop - "mov z19.s, #0x0\n" - "udot z19.s, z12.b, z8.b\n" - "udot z10.s, z21.b, z14.b\n" - "whilelt p0.s, x20, %x[n_channels]\n" - "udot z19.s, z12.b, z3.b\n" - "udot z31.s, z21.b, z8.b\n" - "incw x13, ALL, MUL #4\n" - "udot z10.s, z16.b, z8.b\n" - "ext z8.b, z8.b, z8.b, #0x1\n" - "movprfx z18, z19\n udot z18.s, z12.b, z30.b\n" - "udot z19.s, z12.b, z14.b\n" - "ext z14.b, z14.b, z14.b, #0x1\n" - "udot z31.s, z16.b, z3.b\n" - "udot z10.s, z20.b, z3.b\n" - "ext z3.b, z3.b, z3.b, #0x1\n" - "udot z4.s, z21.b, z14.b\n" - "udot z26.s, z21.b, z8.b\n" - "mov z17.s, #0x0\n" - "udot z17.s, z12.b, z8.b\n" - "udot z17.s, z12.b, z3.b\n" - "udot z31.s, z20.b, z30.b\n" - "ext z30.b, z30.b, z30.b, #0x1\n" - "udot z4.s, z16.b, z8.b\n" - "udot z26.s, z16.b, z3.b\n" - "ld1w { z8.s }, p1/Z, [%x[params], #1, MUL VL]\n" - "mls z10.s, p1/M, z19.s, z23.s\n" - "movprfx z16, z17\n udot z16.s, z12.b, z30.b\n" - "mov z19.s, #0x0\n" - "udot z17.s, z12.b, z14.b\n" - "ld1w { z14.s }, p1/Z, [%x[params]]\n" - "udot z4.s, z20.b, z3.b\n" - ".inst 0x04ae754a // sqrdmulh z10.s, z10.s, z14.s\n" - "udot z26.s, z20.b, z30.b\n" - "mls z4.s, p1/M, z17.s, z23.s\n" - "and z21.d, z10.d, z8.d\n" - "mls z31.s, p1/M, z18.s, z23.s\n" - "mls z26.s, p1/M, z16.s, z23.s\n" - "asr z21.s, z21.s, #0x1f\n" - ".inst 0x04ae7484 // sqrdmulh z4.s, z4.s, z14.s\n" - ".inst 0x04ae77ff // sqrdmulh z31.s, z31.s, z14.s\n" - "udot z19.s, z12.b, z7.b\n" - ".inst 0x04ae775a // sqrdmulh z26.s, z26.s, z14.s\n" - "sqadd z10.s, z10.s, z21.s\n" - ".inst 0x4482850a // srshl z10.s, p1/M, z10.s, z8.s\n" - "udot z19.s, z12.b, z2.b\n" - "and z16.d, z4.d, z8.d\n" - "and z20.d, z31.d, z8.d\n" - "movprfx z18, z19\n udot z18.s, z12.b, z29.b\n" - "ld1w { z14.s }, p1/Z, [%x[params], #6, MUL VL]\n" - "and z21.d, z26.d, z8.d\n" - "asr z16.s, z16.s, #0x1f\n" - "udot z19.s, z12.b, z13.b\n" - "asr z20.s, z20.s, #0x1f\n" - "asr z21.s, z21.s, #0x1f\n" - "sqadd z4.s, z4.s, z16.s\n" - "sqadd z31.s, z31.s, z20.s\n" - ".inst 0x44828504 // srshl z4.s, p1/M, z4.s, z8.s\n" - ".inst 0x4482851f // srshl z31.s, p1/M, z31.s, z8.s\n" - "sqadd z26.s, z26.s, z21.s\n" - "add z10.s, z10.s, z22.s\n" - ".inst 0x4482851a // srshl z26.s, p1/M, z26.s, z8.s\n" - "smax z10.s, p1/M, z10.s, z25.s\n" - "add z4.s, z4.s, z22.s\n" - "add z31.s, z31.s, z22.s\n" - "smin z10.s, p1/M, z10.s, z24.s\n" - "smax z4.s, p1/M, z4.s, z25.s\n" - "add z26.s, z26.s, z22.s\n" - "smax z31.s, p1/M, z31.s, z25.s\n" - "smax z26.s, p1/M, z26.s, z25.s\n" - "st1b { z10.s }, p0, [x24, x20]\n" - "ld1w { z10.s }, p1/Z, [%x[params], #2, MUL VL]\n" - "ld1b { z21.b }, p1/Z, [%x[params], #3, MUL VL]\n" - "smin z4.s, p1/M, z4.s, z24.s\n" - "smin z31.s, p1/M, z31.s, z24.s\n" - "smin z26.s, p1/M, z26.s, z24.s\n" - "st1b { z4.s }, p0, [x23, x20]\n" - "mov z4.d, z10.d\n" - "ld1b { z16.b }, p1/Z, [%x[params], #4, MUL VL]\n" - "st1b { z31.s }, p0, [x22, x20]\n" - "mov z31.d, z10.d\n" - "udot z31.s, z21.b, z7.b\n" - "ld1b { z20.b }, p1/Z, [%x[params], #5, MUL VL]\n" - "st1b { z26.s }, p0, [x21, x20]\n" - "mov z26.d, z10.d\n" - "udot z10.s, z21.b, z13.b\n" - "udot z10.s, z16.b, z7.b\n" - "ext z13.b, z13.b, z13.b, #0x1\n" - "ext z7.b, z7.b, z7.b, #0x1\n" - "udot z4.s, z21.b, z13.b\n" - "ld1w { z8.s }, p1/Z, [%x[params], #7, MUL VL]\n" - "mov z17.s, #0x0\n" - "udot z26.s, z21.b, z7.b\n" - "udot z17.s, z12.b, z7.b\n" - "incw x20\n" - "udot z31.s, z16.b, z2.b\n" - "udot z10.s, z20.b, z2.b\n" + "mov z30.s, #0x0\n" + "udot z30.s, z25.b, z9.b\n" + "udot z10.s, z26.b, z15.b\n" + "whilelt p0.s, x28, %x[n_channels]\n" + "udot z30.s, z25.b, z18.b\n" + "udot z31.s, z26.b, z9.b\n" + "mov z27.s, #0x0\n" + "incw x14, ALL, MUL #4\n" + "udot z10.s, z3.b, z9.b\n" + "ext z9.b, z9.b, z9.b, #0x1\n" + "movprfx z28, z30\n udot z28.s, z25.b, z2.b\n" + "udot z30.s, z25.b, z15.b\n" + "ext z15.b, z15.b, z15.b, #0x1\n" + "udot z27.s, z25.b, z9.b\n" + "udot z31.s, z3.b, z18.b\n" + "udot z10.s, z1.b, z18.b\n" + "ext z18.b, z18.b, z18.b, #0x1\n" + "udot z22.s, z26.b, z15.b\n" + "udot z21.s, z26.b, z9.b\n" + "udot z27.s, z25.b, z18.b\n" + "udot z31.s, z1.b, z2.b\n" "ext z2.b, z2.b, z2.b, #0x1\n" - "whilelt p0.s, x20, %x[n_channels]\n" - "udot z4.s, z16.b, z7.b\n" - "udot z26.s, z16.b, z2.b\n" + "udot z22.s, z3.b, z9.b\n" + "udot z21.s, z3.b, z18.b\n" + "ld1w { z3.s }, p2/Z, [%x[params], #1, MUL VL]\n" + "mls z10.s, p2/M, z30.s, z8.s\n" + "movprfx z26, z27\n udot z26.s, z25.b, z2.b\n" + "mov z9.s, #0x0\n" + "udot z27.s, z25.b, z15.b\n" + "ld1w { z23.s }, p2/Z, [%x[params]]\n" + "udot z22.s, z1.b, z18.b\n" + ".inst 0x04b7754a // sqrdmulh z10.s, z10.s, z23.s\n" + "udot z21.s, z1.b, z2.b\n" + "mls z22.s, p2/M, z27.s, z8.s\n" + "and z18.d, z10.d, z3.d\n" + "mls z31.s, p2/M, z28.s, z8.s\n" + "mls z21.s, p2/M, z26.s, z8.s\n" + "asr z18.s, z18.s, #0x1f\n" + ".inst 0x04b776d6 // sqrdmulh z22.s, z22.s, z23.s\n" + ".inst 0x04b777ff // sqrdmulh z31.s, z31.s, z23.s\n" + "udot z9.s, z25.b, z19.b\n" + ".inst 0x04b776b5 // sqrdmulh z21.s, z21.s, z23.s\n" + "sqadd z10.s, z10.s, z18.s\n" + ".inst 0x4482886a // srshl z10.s, p2/M, z10.s, z3.s\n" + "udot z9.s, z25.b, z12.b\n" + "and z28.d, z22.d, z3.d\n" + "and z23.d, z31.d, z3.d\n" + "movprfx z27, z9\n udot z27.s, z25.b, z24.b\n" + "ld1w { z30.s }, p2/Z, [%x[params], #6, MUL VL]\n" + "and z18.d, z21.d, z3.d\n" + "asr z28.s, z28.s, #0x1f\n" + "udot z9.s, z25.b, z13.b\n" + "asr z23.s, z23.s, #0x1f\n" + "asr z18.s, z18.s, #0x1f\n" + "sqadd z22.s, z22.s, z28.s\n" + "sqadd z31.s, z31.s, z23.s\n" + ".inst 0x44828876 // srshl z22.s, p2/M, z22.s, z3.s\n" + ".inst 0x4482887f // srshl z31.s, p2/M, z31.s, z3.s\n" + "sqadd z21.s, z21.s, z18.s\n" + "add z10.s, z10.s, z16.s\n" + ".inst 0x44828875 // srshl z21.s, p2/M, z21.s, z3.s\n" + "smax z10.s, p2/M, z10.s, z7.s\n" + "add z22.s, z22.s, z16.s\n" + "add z31.s, z31.s, z16.s\n" + "smin z10.s, p2/M, z10.s, z6.s\n" + "smax z22.s, p2/M, z22.s, z7.s\n" + "add z21.s, z21.s, z16.s\n" + "smax z31.s, p2/M, z31.s, z7.s\n" + "smax z21.s, p2/M, z21.s, z7.s\n" + "st1b { z10.s }, p0, [x12, x28]\n" + "ld1w { z28.s }, p2/Z, [%x[params], #2, MUL VL]\n" + "ld1b { z1.b }, p2/Z, [%x[params], #3, MUL VL]\n" + "smin z22.s, p2/M, z22.s, z6.s\n" + "smin z31.s, p2/M, z31.s, z6.s\n" + "smin z21.s, p2/M, z21.s, z6.s\n" + "st1b { z22.s }, p0, [x11, x28]\n" + "mov z26.d, z28.d\n" + "ld1b { z15.b }, p2/Z, [%x[params], #4, MUL VL]\n" + "st1b { z31.s }, p0, [x10, x28]\n" + "mov z31.d, z28.d\n" + "udot z31.s, z1.b, z19.b\n" + "ld1b { z23.b }, p2/Z, [%x[params], #5, MUL VL]\n" + "st1b { z21.s }, p0, [x9, x28]\n" + "mov z22.d, z28.d\n" + "udot z28.s, z1.b, z13.b\n" + "udot z28.s, z15.b, z19.b\n" + "ext z13.b, z13.b, z13.b, #0x1\n" + "ext z19.b, z19.b, z19.b, #0x1\n" + "udot z26.s, z1.b, z13.b\n" + "ld1w { z21.s }, p2/Z, [%x[params], #7, MUL VL]\n" + "mov z18.s, #0x0\n" + "udot z22.s, z1.b, z19.b\n" + "udot z18.s, z25.b, z19.b\n" + "incw x28\n" + "udot z31.s, z15.b, z12.b\n" + "udot z28.s, z23.b, z12.b\n" + "ext z12.b, z12.b, z12.b, #0x1\n" + "whilelt p0.s, x28, %x[n_channels]\n" + "udot z26.s, z15.b, z19.b\n" + "udot z22.s, z15.b, z12.b\n" "addvl %x[params], %x[params], #16\n" - "udot z17.s, z12.b, z2.b\n" - "udot z31.s, z20.b, z29.b\n" - "ext z29.b, z29.b, z29.b, #0x1\n" - "mls z10.s, p1/M, z19.s, z23.s\n" - "udot z4.s, z20.b, z2.b\n" - ".inst 0x04ae754a // sqrdmulh z10.s, z10.s, z14.s\n" - "udot z26.s, z20.b, z29.b\n" - "movprfx z16, z17\n udot z16.s, z12.b, z29.b\n" - "and z21.d, z10.d, z8.d\n" - "udot z17.s, z12.b, z13.b\n" - "mls z4.s, p1/M, z17.s, z23.s\n" - "asr z21.s, z21.s, #0x1f\n" - "mls z31.s, p1/M, z18.s, z23.s\n" - "mls z26.s, p1/M, z16.s, z23.s\n" - ".inst 0x04ae7484 // sqrdmulh z4.s, z4.s, z14.s\n" - ".inst 0x04ae77ff // sqrdmulh z31.s, z31.s, z14.s\n" - ".inst 0x04ae775a // sqrdmulh z26.s, z26.s, z14.s\n" - "ld1w { z14.s }, p1/Z, [%x[params], #-4, MUL VL]\n" - "sqadd z10.s, z10.s, z21.s\n" - "and z16.d, z4.d, z8.d\n" - ".inst 0x4482850a // srshl z10.s, p1/M, z10.s, z8.s\n" - "and z20.d, z31.d, z8.d\n" - "and z21.d, z26.d, z8.d\n" - "asr z16.s, z16.s, #0x1f\n" - "asr z20.s, z20.s, #0x1f\n" - "asr z21.s, z21.s, #0x1f\n" - "sqadd z4.s, z4.s, z16.s\n" - ".inst 0x44828504 // srshl z4.s, p1/M, z4.s, z8.s\n" - "ld1b { z16.b }, p1/Z, [%x[params], #-6, MUL VL]\n" - "sqadd z31.s, z31.s, z20.s\n" - "sqadd z26.s, z26.s, z21.s\n" - ".inst 0x4482851f // srshl z31.s, p1/M, z31.s, z8.s\n" - ".inst 0x4482851a // srshl z26.s, p1/M, z26.s, z8.s\n" - "add z10.s, z10.s, z22.s\n" - "smax z10.s, p1/M, z10.s, z25.s\n" - "add z4.s, z4.s, z22.s\n" - "smin z10.s, p1/M, z10.s, z24.s\n" - "add z31.s, z31.s, z22.s\n" - "add z26.s, z26.s, z22.s\n" - "smax z4.s, p1/M, z4.s, z25.s\n" - "smax z31.s, p1/M, z31.s, z25.s\n" - "mov z19.s, #0x0\n" - "udot z19.s, z12.b, z6.b\n" - "smax z26.s, p1/M, z26.s, z25.s\n" - "st1b { z10.s }, p0, [x24, x20]\n" - "ld1w { z10.s }, p1/Z, [%x[params], #-8, MUL VL]\n" - "ld1b { z21.b }, p1/Z, [%x[params], #-7, MUL VL]\n" - "smin z4.s, p1/M, z4.s, z24.s\n" - "smin z31.s, p1/M, z31.s, z24.s\n" - "smin z26.s, p1/M, z26.s, z24.s\n" - "st1b { z4.s }, p0, [x23, x20]\n" - "mov z4.d, z10.d\n" - "udot z19.s, z12.b, z1.b\n" - "st1b { z31.s }, p0, [x22, x20]\n" - "mov z31.d, z10.d\n" - "udot z31.s, z21.b, z6.b\n" - "movprfx z18, z19\n udot z18.s, z12.b, z28.b\n" - "st1b { z26.s }, p0, [x21, x20]\n" - "mov z26.d, z10.d\n" - "udot z10.s, z21.b, z11.b\n" - "udot z10.s, z16.b, z6.b\n" - "udot z19.s, z12.b, z11.b\n" + "udot z18.s, z25.b, z12.b\n" + "udot z31.s, z23.b, z24.b\n" + "ext z24.b, z24.b, z24.b, #0x1\n" + "mls z28.s, p2/M, z9.s, z8.s\n" + "udot z26.s, z23.b, z12.b\n" + ".inst 0x04be779c // sqrdmulh z28.s, z28.s, z30.s\n" + "udot z22.s, z23.b, z24.b\n" + "movprfx z12, z18\n udot z12.s, z25.b, z24.b\n" + "and z2.d, z28.d, z21.d\n" + "udot z18.s, z25.b, z13.b\n" + "mls z26.s, p2/M, z18.s, z8.s\n" + "asr z2.s, z2.s, #0x1f\n" + "mls z31.s, p2/M, z27.s, z8.s\n" + "mls z22.s, p2/M, z12.s, z8.s\n" + ".inst 0x04be775a // sqrdmulh z26.s, z26.s, z30.s\n" + ".inst 0x04be77ff // sqrdmulh z31.s, z31.s, z30.s\n" + ".inst 0x04be76d6 // sqrdmulh z22.s, z22.s, z30.s\n" + "ld1w { z1.s }, p2/Z, [%x[params], #-4, MUL VL]\n" + "sqadd z28.s, z28.s, z2.s\n" + "and z24.d, z26.d, z21.d\n" + ".inst 0x44828abc // srshl z28.s, p2/M, z28.s, z21.s\n" + "and z23.d, z31.d, z21.d\n" + "and z18.d, z22.d, z21.d\n" + "asr z24.s, z24.s, #0x1f\n" + "asr z23.s, z23.s, #0x1f\n" + "asr z18.s, z18.s, #0x1f\n" + "sqadd z26.s, z26.s, z24.s\n" + ".inst 0x44828aba // srshl z26.s, p2/M, z26.s, z21.s\n" + "ld1b { z30.b }, p2/Z, [%x[params], #-6, MUL VL]\n" + "sqadd z31.s, z31.s, z23.s\n" + "sqadd z22.s, z22.s, z18.s\n" + ".inst 0x44828abf // srshl z31.s, p2/M, z31.s, z21.s\n" + ".inst 0x44828ab6 // srshl z22.s, p2/M, z22.s, z21.s\n" + "add z28.s, z28.s, z16.s\n" + "smax z28.s, p2/M, z28.s, z7.s\n" + "add z26.s, z26.s, z16.s\n" + "smin z28.s, p2/M, z28.s, z6.s\n" + "add z31.s, z31.s, z16.s\n" + "add z22.s, z22.s, z16.s\n" + "smax z26.s, p2/M, z26.s, z7.s\n" + "smax z31.s, p2/M, z31.s, z7.s\n" + "mov z24.s, #0x0\n" + "udot z24.s, z25.b, z11.b\n" + "smax z22.s, p2/M, z22.s, z7.s\n" + "st1b { z28.s }, p0, [x12, x28]\n" + "ld1w { z23.s }, p2/Z, [%x[params], #-8, MUL VL]\n" + "ld1b { z19.b }, p2/Z, [%x[params], #-7, MUL VL]\n" + "smin z26.s, p2/M, z26.s, z6.s\n" + "smin z31.s, p2/M, z31.s, z6.s\n" + "smin z22.s, p2/M, z22.s, z6.s\n" + "st1b { z26.s }, p0, [x11, x28]\n" + "mov z28.d, z23.d\n" + "udot z24.s, z25.b, z20.b\n" + "st1b { z31.s }, p0, [x10, x28]\n" + "mov z27.d, z23.d\n" + "udot z27.s, z19.b, z11.b\n" + "movprfx z13, z24\n udot z13.s, z25.b, z0.b\n" + "st1b { z22.s }, p0, [x9, x28]\n" + "mov z26.d, z23.d\n" + "udot z23.s, z19.b, z14.b\n" + "udot z23.s, z30.b, z11.b\n" + "udot z24.s, z25.b, z14.b\n" + "ext z14.b, z14.b, z14.b, #0x1\n" + "ld1b { z21.b }, p2/Z, [%x[params], #-5, MUL VL]\n" + "udot z28.s, z19.b, z14.b\n" "ext z11.b, z11.b, z11.b, #0x1\n" - "ld1b { z20.b }, p1/Z, [%x[params], #-5, MUL VL]\n" - "udot z4.s, z21.b, z11.b\n" - "ext z6.b, z6.b, z6.b, #0x1\n" - "mov z17.s, #0x0\n" - "udot z26.s, z21.b, z6.b\n" - "ld1w { z8.s }, p1/Z, [%x[params], #-3, MUL VL]\n" - "udot z17.s, z12.b, z6.b\n" - "udot z31.s, z16.b, z1.b\n" - "incw x20\n" - "whilelt p0.s, x20, %x[n_channels]\n" - "udot z10.s, z20.b, z1.b\n" - "ext z1.b, z1.b, z1.b, #0x1\n" - "udot z4.s, z16.b, z6.b\n" - "udot z26.s, z16.b, z1.b\n" - "udot z17.s, z12.b, z1.b\n" - "udot z31.s, z20.b, z28.b\n" - "ext z28.b, z28.b, z28.b, #0x1\n" - "mls z10.s, p1/M, z19.s, z23.s\n" - "udot z4.s, z20.b, z1.b\n" - "udot z26.s, z20.b, z28.b\n" - ".inst 0x04ae754a // sqrdmulh z10.s, z10.s, z14.s\n" - "movprfx z16, z17\n udot z16.s, z12.b, z28.b\n" - "udot z17.s, z12.b, z11.b\n" - "and z21.d, z10.d, z8.d\n" - "mls z4.s, p1/M, z17.s, z23.s\n" - "mls z31.s, p1/M, z18.s, z23.s\n" - "asr z21.s, z21.s, #0x1f\n" - "mls z26.s, p1/M, z16.s, z23.s\n" - ".inst 0x04ae7484 // sqrdmulh z4.s, z4.s, z14.s\n" - ".inst 0x04ae77ff // sqrdmulh z31.s, z31.s, z14.s\n" - ".inst 0x04ae775a // sqrdmulh z26.s, z26.s, z14.s\n" - "ld1w { z14.s }, p1/Z, [%x[params], #2, MUL VL]\n" - "sqadd z10.s, z10.s, z21.s\n" - "and z16.d, z4.d, z8.d\n" - ".inst 0x4482850a // srshl z10.s, p1/M, z10.s, z8.s\n" - "and z20.d, z31.d, z8.d\n" - "and z21.d, z26.d, z8.d\n" - "asr z16.s, z16.s, #0x1f\n" + "mov z12.s, #0x0\n" + "udot z26.s, z19.b, z11.b\n" + "ld1w { z22.s }, p2/Z, [%x[params], #-3, MUL VL]\n" + "udot z12.s, z25.b, z11.b\n" + "udot z27.s, z30.b, z20.b\n" + "incw x28\n" + "whilelt p0.s, x28, %x[n_channels]\n" + "udot z23.s, z21.b, z20.b\n" + "ext z20.b, z20.b, z20.b, #0x1\n" + "udot z28.s, z30.b, z11.b\n" + "udot z26.s, z30.b, z20.b\n" + "udot z12.s, z25.b, z20.b\n" + "udot z27.s, z21.b, z0.b\n" + "ext z0.b, z0.b, z0.b, #0x1\n" + "mls z23.s, p2/M, z24.s, z8.s\n" + "udot z28.s, z21.b, z20.b\n" + "udot z26.s, z21.b, z0.b\n" + ".inst 0x04a176f7 // sqrdmulh z23.s, z23.s, z1.s\n" + "movprfx z19, z12\n udot z19.s, z25.b, z0.b\n" + "udot z12.s, z25.b, z14.b\n" + "and z18.d, z23.d, z22.d\n" + "mls z28.s, p2/M, z12.s, z8.s\n" + "mls z27.s, p2/M, z13.s, z8.s\n" + "asr z18.s, z18.s, #0x1f\n" + "mls z26.s, p2/M, z19.s, z8.s\n" + ".inst 0x04a1779c // sqrdmulh z28.s, z28.s, z1.s\n" + ".inst 0x04a1777b // sqrdmulh z27.s, z27.s, z1.s\n" + ".inst 0x04a1775a // sqrdmulh z26.s, z26.s, z1.s\n" + "ld1w { z2.s }, p2/Z, [%x[params], #2, MUL VL]\n" + "sqadd z23.s, z23.s, z18.s\n" + "and z20.d, z28.d, z22.d\n" + ".inst 0x44828ad7 // srshl z23.s, p2/M, z23.s, z22.s\n" + "and z19.d, z27.d, z22.d\n" + "and z18.d, z26.d, z22.d\n" "asr z20.s, z20.s, #0x1f\n" - "asr z21.s, z21.s, #0x1f\n" - "sqadd z4.s, z4.s, z16.s\n" - ".inst 0x44828504 // srshl z4.s, p1/M, z4.s, z8.s\n" - "ld1b { z16.b }, p1/Z, [%x[params]]\n" - "sqadd z31.s, z31.s, z20.s\n" - "sqadd z26.s, z26.s, z21.s\n" - ".inst 0x4482851f // srshl z31.s, p1/M, z31.s, z8.s\n" - ".inst 0x4482851a // srshl z26.s, p1/M, z26.s, z8.s\n" - "add z10.s, z10.s, z22.s\n" - "smax z10.s, p1/M, z10.s, z25.s\n" - "add z4.s, z4.s, z22.s\n" - "smin z10.s, p1/M, z10.s, z24.s\n" - "add z31.s, z31.s, z22.s\n" - "add z26.s, z26.s, z22.s\n" - "smax z4.s, p1/M, z4.s, z25.s\n" - "smax z31.s, p1/M, z31.s, z25.s\n" + "asr z19.s, z19.s, #0x1f\n" + "asr z18.s, z18.s, #0x1f\n" + "sqadd z28.s, z28.s, z20.s\n" + ".inst 0x44828adc // srshl z28.s, p2/M, z28.s, z22.s\n" + "ld1b { z13.b }, p2/Z, [%x[params]]\n" + "sqadd z27.s, z27.s, z19.s\n" + "sqadd z26.s, z26.s, z18.s\n" + ".inst 0x44828adb // srshl z27.s, p2/M, z27.s, z22.s\n" + ".inst 0x44828ada // srshl z26.s, p2/M, z26.s, z22.s\n" + "add z23.s, z23.s, z16.s\n" + "smax z23.s, p2/M, z23.s, z7.s\n" + "add z28.s, z28.s, z16.s\n" + "smin z23.s, p2/M, z23.s, z6.s\n" + "add z27.s, z27.s, z16.s\n" + "add z26.s, z26.s, z16.s\n" + "smax z28.s, p2/M, z28.s, z7.s\n" + "smax z27.s, p2/M, z27.s, z7.s\n" + "mov z24.s, #0x0\n" + "udot z24.s, z25.b, z17.b\n" + "smax z26.s, p2/M, z26.s, z7.s\n" + "st1b { z23.s }, p0, [x12, x28]\n" + "ld1w { z1.s }, p2/Z, [%x[params], #-2, MUL VL]\n" + "ld1b { z21.b }, p2/Z, [%x[params], #-1, MUL VL]\n" + "smin z28.s, p2/M, z28.s, z6.s\n" + "smin z27.s, p2/M, z27.s, z6.s\n" + "smin z26.s, p2/M, z26.s, z6.s\n" + "st1b { z28.s }, p0, [x11, x28]\n" + "mov z0.d, z1.d\n" + "udot z24.s, z25.b, z4.b\n" + "st1b { z27.s }, p0, [x10, x28]\n" + "mov z31.d, z1.d\n" + "udot z31.s, z21.b, z17.b\n" + "movprfx z23, z24\n udot z23.s, z25.b, z5.b\n" + "st1b { z26.s }, p0, [x9, x28]\n" + "mov z30.d, z1.d\n" + "udot z1.s, z21.b, z29.b\n" + "udot z1.s, z13.b, z17.b\n" + "udot z24.s, z25.b, z29.b\n" + "ext z29.b, z29.b, z29.b, #0x1\n" + "ld1b { z20.b }, p2/Z, [%x[params], #1, MUL VL]\n" + "udot z0.s, z21.b, z29.b\n" + "ext z17.b, z17.b, z17.b, #0x1\n" "mov z19.s, #0x0\n" - "udot z19.s, z12.b, z5.b\n" - "smax z26.s, p1/M, z26.s, z25.s\n" - "st1b { z10.s }, p0, [x24, x20]\n" - "ld1w { z10.s }, p1/Z, [%x[params], #-2, MUL VL]\n" - "ld1b { z21.b }, p1/Z, [%x[params], #-1, MUL VL]\n" - "smin z4.s, p1/M, z4.s, z24.s\n" - "smin z31.s, p1/M, z31.s, z24.s\n" - "smin z26.s, p1/M, z26.s, z24.s\n" - "st1b { z4.s }, p0, [x23, x20]\n" - "mov z4.d, z10.d\n" - "udot z19.s, z12.b, z0.b\n" - "st1b { z31.s }, p0, [x22, x20]\n" - "mov z31.d, z10.d\n" - "udot z31.s, z21.b, z5.b\n" - "movprfx z18, z19\n udot z18.s, z12.b, z27.b\n" - "st1b { z26.s }, p0, [x21, x20]\n" - "mov z26.d, z10.d\n" - "udot z10.s, z21.b, z9.b\n" - "udot z10.s, z16.b, z5.b\n" - "udot z19.s, z12.b, z9.b\n" - "ext z9.b, z9.b, z9.b, #0x1\n" - "ld1b { z20.b }, p1/Z, [%x[params], #1, MUL VL]\n" - "udot z4.s, z21.b, z9.b\n" + "udot z30.s, z21.b, z17.b\n" + "ld1w { z22.s }, p2/Z, [%x[params], #3, MUL VL]\n" + "udot z19.s, z25.b, z17.b\n" + "udot z31.s, z13.b, z4.b\n" + "incw x28\n" + "whilelt p1.s, x28, %x[n_channels]\n" + "udot z1.s, z20.b, z4.b\n" + "ext z4.b, z4.b, z4.b, #0x1\n" + "udot z0.s, z13.b, z17.b\n" + "whilelt p0.b, x14, %x[n_channels]\n" + "udot z30.s, z13.b, z4.b\n" + "udot z19.s, z25.b, z4.b\n" + "ld1b { z13.b }, p0/Z, [x26, x14]\n" + "ld1b { z28.b }, p0/Z, [x25, x14]\n" + "udot z31.s, z20.b, z5.b\n" "ext z5.b, z5.b, z5.b, #0x1\n" - "mov z17.s, #0x0\n" - "udot z26.s, z21.b, z5.b\n" - "ld1w { z8.s }, p1/Z, [%x[params], #3, MUL VL]\n" - "udot z17.s, z12.b, z5.b\n" - "udot z31.s, z16.b, z0.b\n" - "incw x20\n" - "whilelt p0.s, x20, %x[n_channels]\n" - "udot z10.s, z20.b, z0.b\n" - "ext z0.b, z0.b, z0.b, #0x1\n" - "udot z4.s, z16.b, z5.b\n" - "whilelt p2.b, x13, %x[n_channels]\n" - "udot z26.s, z16.b, z0.b\n" - "udot z17.s, z12.b, z0.b\n" - "ld1b { z13.b }, p2/Z, [x11, x13]\n" - "ld1b { z11.b }, p2/Z, [x10, x13]\n" - "udot z31.s, z20.b, z27.b\n" - "ext z27.b, z27.b, z27.b, #0x1\n" - "mls z10.s, p1/M, z19.s, z23.s\n" - "ld1b { z7.b }, p2/Z, [x27, x13]\n" - "udot z4.s, z20.b, z0.b\n" - "udot z26.s, z20.b, z27.b\n" - ".inst 0x04ae754a // sqrdmulh z10.s, z10.s, z14.s\n" - "ld1b { z6.b }, p2/Z, [x26, x13]\n" - "movprfx z16, z17\n udot z16.s, z12.b, z27.b\n" - "udot z17.s, z12.b, z9.b\n" - "and z21.d, z10.d, z8.d\n" - "ld1b { z9.b }, p2/Z, [x9, x13]\n" - "mls z4.s, p1/M, z17.s, z23.s\n" - "mls z31.s, p1/M, z18.s, z23.s\n" + "mls z1.s, p2/M, z24.s, z8.s\n" + "ld1b { z27.b }, p0/Z, [x22, x14]\n" + "udot z0.s, z20.b, z4.b\n" + "udot z30.s, z20.b, z5.b\n" + ".inst 0x04a27421 // sqrdmulh z1.s, z1.s, z2.s\n" + "ld1b { z26.b }, p0/Z, [x21, x14]\n" + "movprfx z18, z19\n udot z18.s, z25.b, z5.b\n" + "udot z19.s, z25.b, z29.b\n" + "and z11.d, z1.d, z22.d\n" + "ld1b { z29.b }, p0/Z, [x23, x14]\n" + "mls z0.s, p2/M, z19.s, z8.s\n" + "mls z31.s, p2/M, z23.s, z8.s\n" + "asr z11.s, z11.s, #0x1f\n" + "ld1b { z17.b }, p0/Z, [x20, x14]\n" + "mls z30.s, p2/M, z18.s, z8.s\n" + ".inst 0x04a27400 // sqrdmulh z0.s, z0.s, z2.s\n" + ".inst 0x04a277ff // sqrdmulh z31.s, z31.s, z2.s\n" + ".inst 0x04a277de // sqrdmulh z30.s, z30.s, z2.s\n" + "ld1b { z15.b }, p0/Z, [x27, x14]\n" + "ldp x23, x22, [%x[inptrs], #0x40]\n" + "sqadd z1.s, z1.s, z11.s\n" + "and z21.d, z0.d, z22.d\n" + ".inst 0x44828ac1 // srshl z1.s, p2/M, z1.s, z22.s\n" + "ldp x21, x20, [%x[inptrs], #0x50]\n" + "and z20.d, z31.d, z22.d\n" + "and z19.d, z30.d, z22.d\n" + "ld1b { z18.b }, p0/Z, [x23, x14]\n" + "ld1b { z11.b }, p0/Z, [x22, x14]\n" "asr z21.s, z21.s, #0x1f\n" - "ld1b { z5.b }, p2/Z, [x25, x13]\n" - "mls z26.s, p1/M, z16.s, z23.s\n" - ".inst 0x04ae7484 // sqrdmulh z4.s, z4.s, z14.s\n" - ".inst 0x04ae77ff // sqrdmulh z31.s, z31.s, z14.s\n" - ".inst 0x04ae775a // sqrdmulh z26.s, z26.s, z14.s\n" - "ld1b { z14.b }, p2/Z, [x12, x13]\n" - "ldp x12, x11, [%x[inptrs], #0x40]\n" - "sqadd z10.s, z10.s, z21.s\n" - "and z16.d, z4.d, z8.d\n" - ".inst 0x4482850a // srshl z10.s, p1/M, z10.s, z8.s\n" - "ldp x10, x9, [%x[inptrs], #0x50]\n" - "and z20.d, z31.d, z8.d\n" - "and z21.d, z26.d, z8.d\n" - "ld1b { z3.b }, p2/Z, [x12, x13]\n" - "ld1b { z2.b }, p2/Z, [x11, x13]\n" - "asr z16.s, z16.s, #0x1f\n" "asr z20.s, z20.s, #0x1f\n" - "ld1b { z1.b }, p2/Z, [x10, x13]\n" - "ld1b { z0.b }, p2/Z, [x9, x13]\n" - "asr z21.s, z21.s, #0x1f\n" - "sqadd z4.s, z4.s, z16.s\n" - ".inst 0x44828504 // srshl z4.s, p1/M, z4.s, z8.s\n" - "ld1b { z16.b }, p1/Z, [%x[params], #6, MUL VL]\n" + "ld1b { z24.b }, p0/Z, [x21, x14]\n" + "ld1b { z4.b }, p0/Z, [x20, x14]\n" + "asr z19.s, z19.s, #0x1f\n" + "sqadd z0.s, z0.s, z21.s\n" + ".inst 0x44828ac0 // srshl z0.s, p2/M, z0.s, z22.s\n" + "ld1b { z3.b }, p2/Z, [%x[params], #6, MUL VL]\n" "sqadd z31.s, z31.s, z20.s\n" - "sqadd z26.s, z26.s, z21.s\n" - ".inst 0x4482851f // srshl z31.s, p1/M, z31.s, z8.s\n" - ".inst 0x4482851a // srshl z26.s, p1/M, z26.s, z8.s\n" - "add z10.s, z10.s, z22.s\n" - "smax z10.s, p1/M, z10.s, z25.s\n" - "add z4.s, z4.s, z22.s\n" - "ld1b { z8.b }, p2/Z, [x28, x13]\n" - "add z31.s, z31.s, z22.s\n" - "add z26.s, z26.s, z22.s\n" - "ldp x28, x27, [%x[inptrs], #0x60]\n" - "ldp x26, x25, [%x[inptrs], #0x70]\n" - "smin z10.s, p1/M, z10.s, z24.s\n" - "smax z4.s, p1/M, z4.s, z25.s\n" - "st1b { z10.s }, p0, [x24, x20]\n" - "ld1b { z30.b }, p2/Z, [x28, x13]\n" - "smax z31.s, p1/M, z31.s, z25.s\n" - "smax z26.s, p1/M, z26.s, z25.s\n" - "ld1b { z29.b }, p2/Z, [x27, x13]\n" - "ld1b { z28.b }, p2/Z, [x26, x13]\n" - "ld1b { z27.b }, p2/Z, [x25, x13]\n" - "zip2 z10.b, z14.b, z11.b\n" - "zip1 z14.b, z14.b, z11.b\n" - "smin z4.s, p1/M, z4.s, z24.s\n" - "zip1 z11.b, z13.b, z9.b\n" - "zip2 z9.b, z13.b, z9.b\n" - "smin z31.s, p1/M, z31.s, z24.s\n" - "smin z26.s, p1/M, z26.s, z24.s\n" - "st1b { z4.s }, p0, [x23, x20]\n" - "zip2 z13.b, z14.b, z11.b\n" - "zip1 z14.b, z14.b, z11.b\n" - "ldp x12, x11, [%x[inptrs], #0x0]\n" - "st1b { z31.s }, p0, [x22, x20]\n" - "zip1 z11.b, z10.b, z9.b\n" - "zip2 z9.b, z10.b, z9.b\n" - "ld1w { z10.s }, p1/Z, [%x[params], #4, MUL VL]\n" - "st1b { z26.s }, p0, [x21, x20]\n" - "zip2 z4.b, z8.b, z6.b\n" - "zip1 z8.b, z8.b, z6.b\n" - "incw x20\n" - "zip1 z6.b, z7.b, z5.b\n" - "zip2 z5.b, z7.b, z5.b\n" - "ldp x10, x9, [%x[inptrs], #0x10]\n" - "ldp x28, x27, [%x[inptrs], #0x20]\n" - "zip2 z31.b, z3.b, z1.b\n" - "zip1 z3.b, z3.b, z1.b\n" - "ldp x26, x25, [%x[inptrs], #0x30]\n" - "ld1b { z21.b }, p1/Z, [%x[params], #5, MUL VL]\n" - "zip1 z1.b, z2.b, z0.b\n" - "zip2 z0.b, z2.b, z0.b\n" - "ld1b { z20.b }, p1/Z, [%x[params], #7, MUL VL]\n" + "sqadd z30.s, z30.s, z19.s\n" + ".inst 0x44828adf // srshl z31.s, p2/M, z31.s, z22.s\n" + ".inst 0x44828ade // srshl z30.s, p2/M, z30.s, z22.s\n" + "add z1.s, z1.s, z16.s\n" + "smax z1.s, p2/M, z1.s, z7.s\n" + "add z0.s, z0.s, z16.s\n" + "ld1b { z9.b }, p0/Z, [x24, x14]\n" + "add z31.s, z31.s, z16.s\n" + "add z30.s, z30.s, z16.s\n" + "ldp x23, x22, [%x[inptrs], #0x60]\n" + "ldp x21, x20, [%x[inptrs], #0x70]\n" + "smin z1.s, p2/M, z1.s, z6.s\n" + "smax z0.s, p2/M, z0.s, z7.s\n" + "st1b { z1.s }, p1, [x12, x28]\n" + "ld1b { z2.b }, p0/Z, [x23, x14]\n" + "smax z31.s, p2/M, z31.s, z7.s\n" + "smax z30.s, p2/M, z30.s, z7.s\n" + "ld1b { z23.b }, p0/Z, [x22, x14]\n" + "ld1b { z22.b }, p0/Z, [x21, x14]\n" + "ld1b { z5.b }, p0/Z, [x20, x14]\n" + "zip2 z20.b, z15.b, z28.b\n" + "zip1 z15.b, z15.b, z28.b\n" + "smin z0.s, p2/M, z0.s, z6.s\n" + "zip1 z19.b, z13.b, z29.b\n" + "zip2 z29.b, z13.b, z29.b\n" + "smin z31.s, p2/M, z31.s, z6.s\n" + "smin z30.s, p2/M, z30.s, z6.s\n" + "st1b { z0.s }, p1, [x11, x28]\n" + "zip2 z13.b, z15.b, z19.b\n" + "zip1 z15.b, z15.b, z19.b\n" + "ldp x27, x26, [%x[inptrs], #0x0]\n" + "st1b { z31.s }, p1, [x10, x28]\n" + "zip1 z14.b, z20.b, z29.b\n" + "zip2 z29.b, z20.b, z29.b\n" + "ld1w { z10.s }, p2/Z, [%x[params], #4, MUL VL]\n" + "st1b { z30.s }, p1, [x9, x28]\n" + "zip2 z21.b, z9.b, z26.b\n" + "zip1 z9.b, z9.b, z26.b\n" + "incw x28\n" + "zip1 z20.b, z27.b, z17.b\n" + "zip2 z17.b, z27.b, z17.b\n" + "ldp x25, x23, [%x[inptrs], #0x10]\n" + "ldp x24, x22, [%x[inptrs], #0x20]\n" + "zip2 z31.b, z18.b, z24.b\n" + "zip1 z18.b, z18.b, z24.b\n" + "ldp x21, x20, [%x[inptrs], #0x30]\n" + "ld1b { z26.b }, p2/Z, [%x[params], #5, MUL VL]\n" + "zip1 z27.b, z11.b, z4.b\n" + "zip2 z4.b, z11.b, z4.b\n" + "ld1b { z1.b }, p2/Z, [%x[params], #7, MUL VL]\n" "addvl %x[params], %x[params], #8\n" - "zip2 z26.b, z30.b, z28.b\n" - "zip1 z30.b, z30.b, z28.b\n" - "zip1 z28.b, z29.b, z27.b\n" - "zip2 z27.b, z29.b, z27.b\n" - "zip2 z7.b, z8.b, z6.b\n" - "zip1 z8.b, z8.b, z6.b\n" - "zip1 z6.b, z4.b, z5.b\n" - "zip2 z5.b, z4.b, z5.b\n" - "zip2 z2.b, z3.b, z1.b\n" - "zip1 z3.b, z3.b, z1.b\n" - "zip1 z1.b, z31.b, z0.b\n" - "zip2 z0.b, z31.b, z0.b\n" - "zip2 z29.b, z30.b, z28.b\n" - "zip1 z30.b, z30.b, z28.b\n" - "zip1 z28.b, z26.b, z27.b\n" - "zip2 z27.b, z26.b, z27.b\n" - "mov z4.d, z10.d\n" + "zip2 z30.b, z2.b, z22.b\n" + "zip1 z2.b, z2.b, z22.b\n" + "zip1 z28.b, z23.b, z5.b\n" + "zip2 z5.b, z23.b, z5.b\n" + "zip2 z19.b, z9.b, z20.b\n" + "zip1 z9.b, z9.b, z20.b\n" + "zip1 z11.b, z21.b, z17.b\n" + "zip2 z17.b, z21.b, z17.b\n" + "zip2 z12.b, z18.b, z27.b\n" + "zip1 z18.b, z18.b, z27.b\n" + "zip1 z20.b, z31.b, z4.b\n" + "zip2 z4.b, z31.b, z4.b\n" + "zip2 z24.b, z2.b, z28.b\n" + "zip1 z2.b, z2.b, z28.b\n" + "zip1 z0.b, z30.b, z5.b\n" + "zip2 z5.b, z30.b, z5.b\n" + "mov z22.d, z10.d\n" "mov z31.d, z10.d\n" - "mov z26.d, z10.d\n" + "mov z21.d, z10.d\n" "b.any 1b\n" : [params] "+&r" (params) : [inptrs] "r" (inptrs), [n_channels] "r" (n_channels), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [outptrs] "r" (outptrs), [qp] "r" (&qp) - : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp index 1cf20ef721870e6bf5648fc3a3ce93cfab2833e9..0300b71d7cb3481a1a40908eda6c733e496ff086 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,14 +22,14 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include #pragma once -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -47,17 +47,16 @@ class sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirstS constexpr static unsigned int stride_rows = 1; constexpr static unsigned int stride_cols = 1; - arm_gemm::VLType get_vl_type(void) const override { return arm_gemm::VLType::SVE; } - unsigned int get_accumulator_depth_vl(void) const override { return 2; } - sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst(const CPUInfo *) : Parent(2, 2, 3, 3, 1, 1) {} - Parent::KernelType kernel = sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl; + arm_gemm::VLType get_vl_type(void) const override { return arm_gemm::VLType::SVE; } + Parent::KernelType kernel = sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl; Parent::KernelType get_kernel(void) const override { return kernel; } + unsigned int get_accumulator_depth_vl(void) const override { return 2; } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp index 386eb96cff6d982a1f075758fe5e56c12a35d4e7..5c26010c0df8bd72e3e04e11703315a6decc0cb8 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp @@ -27,7 +27,7 @@ #include #include -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -46,7 +46,7 @@ void sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( struct Params { long unsigned int n_channels; - const uint8_t *weights; + const void *weights; const int32_t *bias; const arm_gemm::Requantize32 *requant; const int32_t *const requant_muls; @@ -57,7 +57,7 @@ void sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( Params( long unsigned int n_channels, const uint8_t *const *inptrs_raw, - const uint8_t *const weights, + const void *const weights, const int32_t *const bias, const arm_gemm::Requantize32 &qp, const int32_t *const requant_muls, @@ -91,320 +91,320 @@ void sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( requant_muls, requant_shifts, outptrs); __asm__ __volatile__( - "mov x8, #0x0\n" + "mov x16, #0x0\n" "ldr x25, [%x[params], %[offsetof_Params_requant]]\n" "ptrue p4.b\n" "ldr x24, [%x[params], %[offsetof_Params_outptrs]]\n" - "mov x23, x8\n" + "mov x23, x16\n" "add x21, x25, %[offsetof_Requantize32_a_offset]\n" - "ldr x17, [%x[params], %[offsetof_Params_n_channels]]\n" - "ldr x16, [%x[params], %[offsetof_Params_weights]]\n" + "ldr x15, [%x[params], %[offsetof_Params_n_channels]]\n" + "ldr x14, [%x[params], %[offsetof_Params_weights]]\n" "add x20, x25, %[offsetof_Requantize32_b_offset]\n" "add x22, x25, %[offsetof_Requantize32_c_offset]\n" - "ld1rb { z23.b }, p4/Z, [x21]\n" - "ld1rb { z15.b }, p4/Z, [x20]\n" + "ld1rb { z12.b }, p4/Z, [x21]\n" + "ld1rb { z30.b }, p4/Z, [x20]\n" "add x21, x25, %[offsetof_Requantize32_minval]\n" "add x20, x25, %[offsetof_Requantize32_maxval]\n" - "ld1rh { z14.h }, p4/Z, [x22]\n" - "ld1rh { z12.h }, p4/Z, [x21]\n" - "ld1rh { z11.h }, p4/Z, [x20]\n" - "ldp x15, x14, [x24, #0x0]\n" + "ld1rh { z24.h }, p4/Z, [x22]\n" + "ld1rh { z11.h }, p4/Z, [x21]\n" + "ld1rh { z26.h }, p4/Z, [x20]\n" + "ldp x13, x12, [x24, #0x0]\n" "incw x23\n" - "whilelt p3.h, x8, x17\n" - "ldp x13, x12, [x24, #0x10]\n" - "whilelt p2.s, x8, x17\n" - "whilelt p1.s, x23, x17\n" - "ldr x26, [%x[params], %[offsetof_Params_bias]]\n" - "ld1b { z0.h }, p4/Z, [x16]\n" - "ld1b { z1.h }, p4/Z, [x16, #1, MUL VL]\n" - "add x11, %x[params], %[offsetof_Params_inptrs]\n" - "mov x10, #0x0\n" - "ld1b { z2.h }, p4/Z, [x16, #2, MUL VL]\n" - "ld1b { z3.h }, p4/Z, [x16, #3, MUL VL]\n" - ".inst 0x454f1800 // usublb z0.h, z0.b, z15.b\n" - ".inst 0x454f1821 // usublb z1.h, z1.b, z15.b\n" - "ld1b { z4.h }, p4/Z, [x16, #4, MUL VL]\n" - "ld1b { z5.h }, p4/Z, [x16, #5, MUL VL]\n" - ".inst 0x454f1842 // usublb z2.h, z2.b, z15.b\n" - ".inst 0x454f1863 // usublb z3.h, z3.b, z15.b\n" - "ld1b { z6.h }, p4/Z, [x16, #6, MUL VL]\n" - "ld1b { z7.h }, p4/Z, [x16, #7, MUL VL]\n" - "inch x16, ALL, MUL #8\n" - ".inst 0x454f1884 // usublb z4.h, z4.b, z15.b\n" - "ld1w { z17.s }, p2/Z, [x26]\n" - "ld1w { z16.s }, p1/Z, [x26, #1, MUL VL]\n" - "uzp1 z13.s, z17.s, z16.s\n" - "uzp2 z17.s, z17.s, z16.s\n" - "ld1b { z8.h }, p4/Z, [x16]\n" - "ldp x24, x23, [x11, #0x0]\n" - "addvl x26, x26, #2\n" - "mov z26.d, z13.d\n" - "ldp x22, x21, [x11, #0x10]\n" - "ldr x20, [x11, #0x20]\n" - "mov z10.d, z17.d\n" - "mov z24.d, z13.d\n" - "ld1b { z31.h }, p3/Z, [x24, x8]\n" - "ld1b { z30.h }, p3/Z, [x23, x8]\n" - "mov z16.d, z17.d\n" - "mov z25.d, z13.d\n" - "ld1b { z29.h }, p3/Z, [x22, x8]\n" - "ld1b { z28.h }, p3/Z, [x21, x8]\n" - "mov z9.d, z17.d\n" - ".inst 0x454f18a5 // usublb z5.h, z5.b, z15.b\n" - "ld1b { z27.h }, p3/Z, [x20, x8]\n" - "ldr x9, [%x[params], %[offsetof_Params_requant_muls]]\n" - ".inst 0x454f18c6 // usublb z6.h, z6.b, z15.b\n" - ".inst 0x454f18e7 // usublb z7.h, z7.b, z15.b\n" - "ldr x28, [%x[params], %[offsetof_Params_requant_shifts]]\n" - "str x26, [%x[params], %[offsetof_Params_bias]]\n" - ".inst 0x454f1908 // usublb z8.h, z8.b, z15.b\n" - ".inst 0x45571bff // usublb z31.h, z31.b, z23.b\n" - ".inst 0x45571bde // usublb z30.h, z30.b, z23.b\n" - ".inst 0x45571bbd // usublb z29.h, z29.b, z23.b\n" - ".inst 0x45571b9c // usublb z28.h, z28.b, z23.b\n" - ".inst 0x45571b7b // usublb z27.h, z27.b, z23.b\n" + "whilelt p3.h, x16, x15\n" + "ldp x11, x10, [x24, #0x10]\n" + "whilelt p2.s, x16, x15\n" + "whilelt p1.s, x23, x15\n" + "ldr x9, [%x[params], %[offsetof_Params_bias]]\n" + "ld1b { z14.h }, p4/Z, [x14]\n" + "ld1b { z21.h }, p4/Z, [x14, #1, MUL VL]\n" + "add x28, %x[params], %[offsetof_Params_inptrs]\n" + "mov x27, #0x0\n" + "ld1b { z1.h }, p4/Z, [x14, #2, MUL VL]\n" + "ld1b { z6.h }, p4/Z, [x14, #3, MUL VL]\n" + ".inst 0x455e19ce // usublb z14.h, z14.b, z30.b\n" + ".inst 0x455e1ab5 // usublb z21.h, z21.b, z30.b\n" + "ld1b { z2.h }, p4/Z, [x14, #4, MUL VL]\n" + "ld1b { z18.h }, p4/Z, [x14, #5, MUL VL]\n" + ".inst 0x455e1821 // usublb z1.h, z1.b, z30.b\n" + ".inst 0x455e18c6 // usublb z6.h, z6.b, z30.b\n" + "ld1b { z7.h }, p4/Z, [x14, #6, MUL VL]\n" + "ld1b { z10.h }, p4/Z, [x14, #7, MUL VL]\n" + "inch x14, ALL, MUL #8\n" + ".inst 0x455e1842 // usublb z2.h, z2.b, z30.b\n" + "ld1w { z17.s }, p2/Z, [x9]\n" + "ld1w { z16.s }, p1/Z, [x9, #1, MUL VL]\n" + "uzp1 z5.s, z17.s, z16.s\n" + "uzp2 z9.s, z17.s, z16.s\n" + "ld1b { z8.h }, p4/Z, [x14]\n" + "ldp x24, x23, [x28, #0x0]\n" + "addvl x9, x9, #2\n" + "mov z17.d, z5.d\n" + "ldp x22, x21, [x28, #0x10]\n" + "ldr x20, [x28, #0x20]\n" + "mov z25.d, z9.d\n" + "mov z16.d, z5.d\n" + "ld1b { z0.h }, p3/Z, [x24, x16]\n" + "ld1b { z29.h }, p3/Z, [x23, x16]\n" + "mov z23.d, z9.d\n" + "mov z22.d, z5.d\n" + "ld1b { z4.h }, p3/Z, [x22, x16]\n" + "ld1b { z13.h }, p3/Z, [x21, x16]\n" + "mov z27.d, z9.d\n" + ".inst 0x455e1a52 // usublb z18.h, z18.b, z30.b\n" + "ld1b { z20.h }, p3/Z, [x20, x16]\n" + "ldr x26, [%x[params], %[offsetof_Params_requant_muls]]\n" + ".inst 0x455e18e7 // usublb z7.h, z7.b, z30.b\n" + ".inst 0x455e194a // usublb z10.h, z10.b, z30.b\n" + "ldr x25, [%x[params], %[offsetof_Params_requant_shifts]]\n" + "str x9, [%x[params], %[offsetof_Params_bias]]\n" + ".inst 0x455e1908 // usublb z8.h, z8.b, z30.b\n" + ".inst 0x454c1800 // usublb z0.h, z0.b, z12.b\n" + ".inst 0x454c1bbd // usublb z29.h, z29.b, z12.b\n" + ".inst 0x454c1884 // usublb z4.h, z4.b, z12.b\n" + ".inst 0x454c19ad // usublb z13.h, z13.b, z12.b\n" + ".inst 0x454c1a94 // usublb z20.h, z20.b, z12.b\n" "1:" // Loop - ".inst 0x448443ed // smlalb z13.s, p4/M, z31.h, z4.h\n" - ".inst 0x448447f1 // smlalt z17.s, p4/M, z31.h, z4.h\n" - "ldr x22, [x11, #0x28]\n" - "ldr x27, [x11, #0x38]\n" - ".inst 0x448343fa // smlalb z26.s, p4/M, z31.h, z3.h\n" - ".inst 0x448347ea // smlalt z10.s, p4/M, z31.h, z3.h\n" - "ldr x21, [x11, #0x30]\n" - "ldr x26, [x11, #0x40]\n" - ".inst 0x448043cd // smlalb z13.s, p4/M, z30.h, z0.h\n" - ".inst 0x448047d1 // smlalt z17.s, p4/M, z30.h, z0.h\n" - "ldr x20, [x11, #0x48]\n" - "ld1b { z30.h }, p3/Z, [x20, x8]\n" - ".inst 0x448243ba // smlalb z26.s, p4/M, z29.h, z2.h\n" - ".inst 0x448247aa // smlalt z10.s, p4/M, z29.h, z2.h\n" - "ld1b { z29.h }, p3/Z, [x21, x8]\n" - ".inst 0x45571bbd // usublb z29.h, z29.b, z23.b\n" - ".inst 0x448143f8 // smlalb z24.s, p4/M, z31.h, z1.h\n" - ".inst 0x448147f0 // smlalt z16.s, p4/M, z31.h, z1.h\n" - "ldr x25, [x11, #0x50]\n" - "ldr x24, [x11, #0x58]\n" - ".inst 0x448043f9 // smlalb z25.s, p4/M, z31.h, z0.h\n" - ".inst 0x448047e9 // smlalt z9.s, p4/M, z31.h, z0.h\n" - "ld1b { z31.h }, p3/Z, [x22, x8]\n" - ".inst 0x45571bff // usublb z31.h, z31.b, z23.b\n" - ".inst 0x4485438d // smlalb z13.s, p4/M, z28.h, z5.h\n" - ".inst 0x44854791 // smlalt z17.s, p4/M, z28.h, z5.h\n" - ".inst 0x45571bde // usublb z30.h, z30.b, z23.b\n" - "ldr x23, [x11, #0x60]\n" - ".inst 0x4484439a // smlalb z26.s, p4/M, z28.h, z4.h\n" - ".inst 0x4484478a // smlalt z10.s, p4/M, z28.h, z4.h\n" - "ldr x22, [x11, #0x68]\n" - "ldr x21, [x11, #0x70]\n" - ".inst 0x44824398 // smlalb z24.s, p4/M, z28.h, z2.h\n" - ".inst 0x44824790 // smlalt z16.s, p4/M, z28.h, z2.h\n" - "ldr x20, [x11, #0x78]\n" - "ld1w { z20.s }, p2/Z, [x9]\n" - ".inst 0x44814399 // smlalb z25.s, p4/M, z28.h, z1.h\n" - ".inst 0x44814789 // smlalt z9.s, p4/M, z28.h, z1.h\n" - "ld1b { z28.h }, p3/Z, [x27, x8]\n" - ".inst 0x45571b9c // usublb z28.h, z28.b, z23.b\n" - ".inst 0x4487436d // smlalb z13.s, p4/M, z27.h, z7.h\n" - ".inst 0x44874771 // smlalt z17.s, p4/M, z27.h, z7.h\n" - "ld1w { z18.s }, p1/Z, [x9, #1, MUL VL]\n" - "uzp1 z19.s, z20.s, z18.s\n" - ".inst 0x4486437a // smlalb z26.s, p4/M, z27.h, z6.h\n" - ".inst 0x4486476a // smlalt z10.s, p4/M, z27.h, z6.h\n" - "uzp2 z22.s, z20.s, z18.s\n" - "ld1w { z20.s }, p2/Z, [x28]\n" - ".inst 0x448643f8 // smlalb z24.s, p4/M, z31.h, z6.h\n" - ".inst 0x448647f0 // smlalt z16.s, p4/M, z31.h, z6.h\n" - "ld1b { z31.h }, p3/Z, [x26, x8]\n" - ".inst 0x45571bff // usublb z31.h, z31.b, z23.b\n" - ".inst 0x44834379 // smlalb z25.s, p4/M, z27.h, z3.h\n" - ".inst 0x44834769 // smlalt z9.s, p4/M, z27.h, z3.h\n" - "whilelt p0.h, x10, x17\n" + ".inst 0x44824005 // smlalb z5.s, p4/M, z0.h, z2.h\n" + ".inst 0x44824409 // smlalt z9.s, p4/M, z0.h, z2.h\n" + "ldr x20, [x28, #0x28]\n" + "ldr x21, [x28, #0x38]\n" + ".inst 0x448e43a5 // smlalb z5.s, p4/M, z29.h, z14.h\n" + ".inst 0x44864011 // smlalb z17.s, p4/M, z0.h, z6.h\n" + "ld1b { z3.h }, p3/Z, [x20, x16]\n" + "ldr x20, [x28, #0x30]\n" + ".inst 0x44954010 // smlalb z16.s, p4/M, z0.h, z21.h\n" + ".inst 0x448e4016 // smlalb z22.s, p4/M, z0.h, z14.h\n" + "ld1b { z31.h }, p3/Z, [x21, x16]\n" + ".inst 0x454c1863 // usublb z3.h, z3.b, z12.b\n" + ".inst 0x448e47a9 // smlalt z9.s, p4/M, z29.h, z14.h\n" + ".inst 0x449241a5 // smlalb z5.s, p4/M, z13.h, z18.h\n" + "ldr x21, [x28, #0x40]\n" + "ld1b { z15.h }, p3/Z, [x20, x16]\n" + ".inst 0x44864419 // smlalt z25.s, p4/M, z0.h, z6.h\n" + ".inst 0x44954417 // smlalt z23.s, p4/M, z0.h, z21.h\n" + ".inst 0x454c1bff // usublb z31.h, z31.b, z12.b\n" + "ldr x20, [x28, #0x48]\n" + ".inst 0x448e441b // smlalt z27.s, p4/M, z0.h, z14.h\n" + ".inst 0x44814091 // smlalb z17.s, p4/M, z4.h, z1.h\n" + "ld1b { z19.h }, p3/Z, [x21, x16]\n" + ".inst 0x454c19ef // usublb z15.h, z15.b, z12.b\n" + ".inst 0x448141b0 // smlalb z16.s, p4/M, z13.h, z1.h\n" + ".inst 0x449541b6 // smlalb z22.s, p4/M, z13.h, z21.h\n" + "ld1b { z28.h }, p3/Z, [x20, x16]\n" + ".inst 0x454c1a73 // usublb z19.h, z19.b, z12.b\n" + ".inst 0x449245a9 // smlalt z9.s, p4/M, z13.h, z18.h\n" + ".inst 0x448a4285 // smlalb z5.s, p4/M, z20.h, z10.h\n" + "ldr x21, [x28, #0x50]\n" + "ldr x20, [x28, #0x58]\n" + ".inst 0x44814499 // smlalt z25.s, p4/M, z4.h, z1.h\n" + ".inst 0x448145b7 // smlalt z23.s, p4/M, z13.h, z1.h\n" + ".inst 0x454c1b9c // usublb z28.h, z28.b, z12.b\n" + "ld1b { z4.h }, p3/Z, [x21, x16]\n" + ".inst 0x449545bb // smlalt z27.s, p4/M, z13.h, z21.h\n" + ".inst 0x448241b1 // smlalb z17.s, p4/M, z13.h, z2.h\n" + "ld1b { z29.h }, p3/Z, [x20, x16]\n" + "ldr x21, [x28, #0x60]\n" + ".inst 0x44874070 // smlalb z16.s, p4/M, z3.h, z7.h\n" + ".inst 0x44864296 // smlalb z22.s, p4/M, z20.h, z6.h\n" + "ldr x20, [x28, #0x68]\n" + ".inst 0x454c1884 // usublb z4.h, z4.b, z12.b\n" + ".inst 0x448a4689 // smlalt z9.s, p4/M, z20.h, z10.h\n" + ".inst 0x449543e5 // smlalb z5.s, p4/M, z31.h, z21.h\n" + ".inst 0x454c1bbd // usublb z29.h, z29.b, z12.b\n" + "ld1b { z0.h }, p3/Z, [x21, x16]\n" + ".inst 0x448245b9 // smlalt z25.s, p4/M, z13.h, z2.h\n" + ".inst 0x44874477 // smlalt z23.s, p4/M, z3.h, z7.h\n" + "ld1b { z3.h }, p3/Z, [x20, x16]\n" + "ldr x20, [x28, #0x70]\n" + ".inst 0x4486469b // smlalt z27.s, p4/M, z20.h, z6.h\n" + ".inst 0x44874291 // smlalb z17.s, p4/M, z20.h, z7.h\n" + ".inst 0x454c1800 // usublb z0.h, z0.b, z12.b\n" + "ld1b { z13.h }, p3/Z, [x20, x16]\n" + ".inst 0x44824290 // smlalb z16.s, p4/M, z20.h, z2.h\n" + ".inst 0x448841f6 // smlalb z22.s, p4/M, z15.h, z8.h\n" + ".inst 0x454c1863 // usublb z3.h, z3.b, z12.b\n" + "ldr x20, [x28, #0x78]\n" + ".inst 0x449547e9 // smlalt z9.s, p4/M, z31.h, z21.h\n" + ".inst 0x44814265 // smlalb z5.s, p4/M, z19.h, z1.h\n" + ".inst 0x454c19ad // usublb z13.h, z13.b, z12.b\n" + "whilelt p0.h, x27, x15\n" + ".inst 0x44874699 // smlalt z25.s, p4/M, z20.h, z7.h\n" + ".inst 0x44824697 // smlalt z23.s, p4/M, z20.h, z2.h\n" + "ld1w { z20.s }, p2/Z, [x26]\n" + "inch x14\n" + ".inst 0x448845fb // smlalt z27.s, p4/M, z15.h, z8.h\n" + ".inst 0x448e43f1 // smlalb z17.s, p4/M, z31.h, z14.h\n" + "ld1w { z15.s }, p1/Z, [x26, #1, MUL VL]\n" + "ldr x21, [%x[params], %[offsetof_Params_bias]]\n" + ".inst 0x44924390 // smlalb z16.s, p4/M, z28.h, z18.h\n" + ".inst 0x44824396 // smlalb z22.s, p4/M, z28.h, z2.h\n" + "addvl x26, x26, #2\n" + ".inst 0x44814669 // smlalt z9.s, p4/M, z19.h, z1.h\n" + ".inst 0x44884385 // smlalb z5.s, p4/M, z28.h, z8.h\n" + ".inst 0x448e47f9 // smlalt z25.s, p4/M, z31.h, z14.h\n" + ".inst 0x44924797 // smlalt z23.s, p4/M, z28.h, z18.h\n" + "ld1b { z31.h }, p3/Z, [x20, x16]\n" + ".inst 0x454c1bff // usublb z31.h, z31.b, z12.b\n" + ".inst 0x4482479b // smlalt z27.s, p4/M, z28.h, z2.h\n" + ".inst 0x44954271 // smlalb z17.s, p4/M, z19.h, z21.h\n" + "uzp1 z2.s, z20.s, z15.s\n" "inch x16\n" - ".inst 0x4481438d // smlalb z13.s, p4/M, z28.h, z1.h\n" - ".inst 0x44814791 // smlalt z17.s, p4/M, z28.h, z1.h\n" - "ldr x26, [%x[params], %[offsetof_Params_bias]]\n" - "addvl x9, x9, #2\n" - ".inst 0x4480439a // smlalb z26.s, p4/M, z28.h, z0.h\n" - ".inst 0x4480478a // smlalt z10.s, p4/M, z28.h, z0.h\n" - "ld1b { z28.h }, p3/Z, [x24, x8]\n" - ".inst 0x45571b9c // usublb z28.h, z28.b, z23.b\n" - ".inst 0x44844378 // smlalb z24.s, p4/M, z27.h, z4.h\n" - ".inst 0x448843b9 // smlalb z25.s, p4/M, z29.h, z8.h\n" - ".inst 0x44844770 // smlalt z16.s, p4/M, z27.h, z4.h\n" - ".inst 0x448847a9 // smlalt z9.s, p4/M, z29.h, z8.h\n" - "ld1b { z29.h }, p3/Z, [x25, x8]\n" - ".inst 0x45571bbd // usublb z29.h, z29.b, z23.b\n" - ".inst 0x448243ed // smlalb z13.s, p4/M, z31.h, z2.h\n" - ".inst 0x448247f1 // smlalt z17.s, p4/M, z31.h, z2.h\n" - "ld1w { z18.s }, p1/Z, [x28, #1, MUL VL]\n" - "addvl x28, x28, #2\n" - ".inst 0x448143fa // smlalb z26.s, p4/M, z31.h, z1.h\n" - ".inst 0x448147ea // smlalt z10.s, p4/M, z31.h, z1.h\n" - "ld1b { z31.h }, p3/Z, [x23, x8]\n" - ".inst 0x45571bff // usublb z31.h, z31.b, z23.b\n" - ".inst 0x448543d8 // smlalb z24.s, p4/M, z30.h, z5.h\n" - ".inst 0x448443d9 // smlalb z25.s, p4/M, z30.h, z4.h\n" - "uzp1 z1.s, z20.s, z18.s\n" - ".inst 0x448843cd // smlalb z13.s, p4/M, z30.h, z8.h\n" - ".inst 0x448847d1 // smlalt z17.s, p4/M, z30.h, z8.h\n" - "uzp2 z27.s, z20.s, z18.s\n" - ".inst 0x448743da // smlalb z26.s, p4/M, z30.h, z7.h\n" - ".inst 0x448747ca // smlalt z10.s, p4/M, z30.h, z7.h\n" - ".inst 0x448547d0 // smlalt z16.s, p4/M, z30.h, z5.h\n" - ".inst 0x448447c9 // smlalt z9.s, p4/M, z30.h, z4.h\n" - "ld1b { z30.h }, p3/Z, [x22, x8]\n" - ".inst 0x45571bde // usublb z30.h, z30.b, z23.b\n" - ".inst 0x448043b8 // smlalb z24.s, p4/M, z29.h, z0.h\n" - ".inst 0x44824399 // smlalb z25.s, p4/M, z28.h, z2.h\n" - ".inst 0x448343ad // smlalb z13.s, p4/M, z29.h, z3.h\n" - ".inst 0x448347b1 // smlalt z17.s, p4/M, z29.h, z3.h\n" - ".inst 0x448047b0 // smlalt z16.s, p4/M, z29.h, z0.h\n" - "ld1b { z29.h }, p3/Z, [x21, x8]\n" - ".inst 0x44824789 // smlalt z9.s, p4/M, z28.h, z2.h\n" - ".inst 0x45571bbd // usublb z29.h, z29.b, z23.b\n" - ".inst 0x448343f8 // smlalb z24.s, p4/M, z31.h, z3.h\n" - ".inst 0x448543d9 // smlalb z25.s, p4/M, z30.h, z5.h\n" - ".inst 0x4485439a // smlalb z26.s, p4/M, z28.h, z5.h\n" - ".inst 0x4485478a // smlalt z10.s, p4/M, z28.h, z5.h\n" - "ld1b { z28.h }, p3/Z, [x20, x8]\n" - ".inst 0x45571b9c // usublb z28.h, z28.b, z23.b\n" - ".inst 0x448643ed // smlalb z13.s, p4/M, z31.h, z6.h\n" - ".inst 0x448347f0 // smlalt z16.s, p4/M, z31.h, z3.h\n" - ".inst 0x04b375ad // sqrdmulh z13.s, z13.s, z19.s\n" - "inch x8\n" - ".inst 0x448547c9 // smlalt z9.s, p4/M, z30.h, z5.h\n" - ".inst 0x448743b8 // smlalb z24.s, p4/M, z29.h, z7.h\n" - "and z21.d, z13.d, z1.d\n" - "mov x20, x8\n" - ".inst 0x448643b9 // smlalb z25.s, p4/M, z29.h, z6.h\n" - ".inst 0x448647f1 // smlalt z17.s, p4/M, z31.h, z6.h\n" - ".inst 0x04b67631 // sqrdmulh z17.s, z17.s, z22.s\n" + ".inst 0x448e4090 // smlalb z16.s, p4/M, z4.h, z14.h\n" + ".inst 0x448143b6 // smlalb z22.s, p4/M, z29.h, z1.h\n" + "uzp2 z15.s, z20.s, z15.s\n" + "ld1w { z20.s }, p2/Z, [x25]\n" + ".inst 0x44884789 // smlalt z9.s, p4/M, z28.h, z8.h\n" + ".inst 0x44864085 // smlalb z5.s, p4/M, z4.h, z6.h\n" + "mov x20, x16\n" "incw x20\n" - ".inst 0x448747b0 // smlalt z16.s, p4/M, z29.h, z7.h\n" - ".inst 0x448647a9 // smlalt z9.s, p4/M, z29.h, z6.h\n" - "asr z21.s, z21.s, #0x1f\n" - "whilelt p2.s, x8, x17\n" - ".inst 0x448843da // smlalb z26.s, p4/M, z30.h, z8.h\n" - ".inst 0x44884398 // smlalb z24.s, p4/M, z28.h, z8.h\n" - "and z20.d, z17.d, z27.d\n" - "whilelt p1.s, x20, x17\n" - ".inst 0x44874399 // smlalb z25.s, p4/M, z28.h, z7.h\n" - ".inst 0x448847ca // smlalt z10.s, p4/M, z30.h, z8.h\n" - ".inst 0x04b3775a // sqrdmulh z26.s, z26.s, z19.s\n" - "whilelt p3.h, x8, x17\n" - ".inst 0x44884790 // smlalt z16.s, p4/M, z28.h, z8.h\n" - ".inst 0x44874789 // smlalt z9.s, p4/M, z28.h, z7.h\n" - ".inst 0x04b37718 // sqrdmulh z24.s, z24.s, z19.s\n" - ".inst 0x04b37739 // sqrdmulh z25.s, z25.s, z19.s\n" - "sqadd z13.s, z13.s, z21.s\n" - ".inst 0x4482902d // srshl z13.s, p4/M, z13.s, z1.s\n" - "asr z20.s, z20.s, #0x1f\n" - "and z19.d, z26.d, z1.d\n" - ".inst 0x04b6754a // sqrdmulh z10.s, z10.s, z22.s\n" - "and z18.d, z24.d, z1.d\n" - ".inst 0x04b67610 // sqrdmulh z16.s, z16.s, z22.s\n" - "and z21.d, z25.d, z1.d\n" - ".inst 0x04b67529 // sqrdmulh z9.s, z9.s, z22.s\n" - "sqadd z17.s, z17.s, z20.s\n" - ".inst 0x44829371 // srshl z17.s, p4/M, z17.s, z27.s\n" + ".inst 0x44954679 // smlalt z25.s, p4/M, z19.h, z21.h\n" + ".inst 0x448e4497 // smlalt z23.s, p4/M, z4.h, z14.h\n" + "ld1w { z19.s }, p1/Z, [x25, #1, MUL VL]\n" + "uzp1 z21.s, z20.s, z19.s\n" + ".inst 0x448147bb // smlalt z27.s, p4/M, z29.h, z1.h\n" + ".inst 0x448a4391 // smlalb z17.s, p4/M, z28.h, z10.h\n" + "uzp2 z1.s, z20.s, z19.s\n" + "whilelt p2.s, x16, x15\n" + ".inst 0x44864010 // smlalb z16.s, p4/M, z0.h, z6.h\n" + ".inst 0x44924076 // smlalb z22.s, p4/M, z3.h, z18.h\n" + "whilelt p1.s, x20, x15\n" + "whilelt p3.h, x16, x15\n" + ".inst 0x44864489 // smlalt z9.s, p4/M, z4.h, z6.h\n" + ".inst 0x44874005 // smlalb z5.s, p4/M, z0.h, z7.h\n" + ".inst 0x04a274a5 // sqrdmulh z5.s, z5.s, z2.s\n" + "addvl x25, x25, #2\n" + ".inst 0x448a4799 // smlalt z25.s, p4/M, z28.h, z10.h\n" + ".inst 0x44864417 // smlalt z23.s, p4/M, z0.h, z6.h\n" + "and z19.d, z5.d, z21.d\n" + ".inst 0x4492447b // smlalt z27.s, p4/M, z3.h, z18.h\n" + ".inst 0x449243b1 // smlalb z17.s, p4/M, z29.h, z18.h\n" "asr z19.s, z19.s, #0x1f\n" - "and z2.d, z10.d, z27.d\n" + ".inst 0x448a41b0 // smlalb z16.s, p4/M, z13.h, z10.h\n" + ".inst 0x448741b6 // smlalb z22.s, p4/M, z13.h, z7.h\n" + "sqadd z5.s, z5.s, z19.s\n" + ".inst 0x448292a5 // srshl z5.s, p4/M, z5.s, z21.s\n" + ".inst 0x44874409 // smlalt z9.s, p4/M, z0.h, z7.h\n" + ".inst 0x449247b9 // smlalt z25.s, p4/M, z29.h, z18.h\n" + ".inst 0x04af7529 // sqrdmulh z9.s, z9.s, z15.s\n" + ".inst 0x448a45b7 // smlalt z23.s, p4/M, z13.h, z10.h\n" + ".inst 0x448745bb // smlalt z27.s, p4/M, z13.h, z7.h\n" + "and z29.d, z9.d, z1.d\n" + ".inst 0x44884071 // smlalb z17.s, p4/M, z3.h, z8.h\n" + ".inst 0x448843f0 // smlalb z16.s, p4/M, z31.h, z8.h\n" + ".inst 0x04a27631 // sqrdmulh z17.s, z17.s, z2.s\n" + ".inst 0x448a43f6 // smlalb z22.s, p4/M, z31.h, z10.h\n" + ".inst 0x44884479 // smlalt z25.s, p4/M, z3.h, z8.h\n" + ".inst 0x04a27610 // sqrdmulh z16.s, z16.s, z2.s\n" + ".inst 0x448847f7 // smlalt z23.s, p4/M, z31.h, z8.h\n" + ".inst 0x448a47fb // smlalt z27.s, p4/M, z31.h, z10.h\n" + ".inst 0x04a276d6 // sqrdmulh z22.s, z22.s, z2.s\n" + "asr z29.s, z29.s, #0x1f\n" + "and z18.d, z17.d, z21.d\n" + ".inst 0x04af7739 // sqrdmulh z25.s, z25.s, z15.s\n" + "and z20.d, z16.d, z21.d\n" + ".inst 0x04af76f7 // sqrdmulh z23.s, z23.s, z15.s\n" + "and z19.d, z22.d, z21.d\n" + ".inst 0x04af777b // sqrdmulh z27.s, z27.s, z15.s\n" + "sqadd z9.s, z9.s, z29.s\n" + ".inst 0x44829029 // srshl z9.s, p4/M, z9.s, z1.s\n" "asr z18.s, z18.s, #0x1f\n" - "and z22.d, z16.d, z27.d\n" - "asr z21.s, z21.s, #0x1f\n" - "and z20.d, z9.d, z27.d\n" - "sqadd z26.s, z26.s, z19.s\n" - "asr z2.s, z2.s, #0x1f\n" - ".inst 0x4482903a // srshl z26.s, p4/M, z26.s, z1.s\n" - "sqadd z24.s, z24.s, z18.s\n" - "asr z22.s, z22.s, #0x1f\n" - ".inst 0x44829038 // srshl z24.s, p4/M, z24.s, z1.s\n" - "sqadd z25.s, z25.s, z21.s\n" + "and z7.d, z25.d, z1.d\n" "asr z20.s, z20.s, #0x1f\n" + "and z6.d, z23.d, z1.d\n" + "asr z19.s, z19.s, #0x1f\n" + "and z2.d, z27.d, z1.d\n" + "sqadd z17.s, z17.s, z18.s\n" + "asr z7.s, z7.s, #0x1f\n" + ".inst 0x448292b1 // srshl z17.s, p4/M, z17.s, z21.s\n" + "sqadd z16.s, z16.s, z20.s\n" + "asr z6.s, z6.s, #0x1f\n" + ".inst 0x448292b0 // srshl z16.s, p4/M, z16.s, z21.s\n" + "sqadd z22.s, z22.s, z19.s\n" + "asr z2.s, z2.s, #0x1f\n" + ".inst 0x448292b6 // srshl z22.s, p4/M, z22.s, z21.s\n" + "sqadd z25.s, z25.s, z7.s\n" + "sqadd z23.s, z23.s, z6.s\n" ".inst 0x44829039 // srshl z25.s, p4/M, z25.s, z1.s\n" - "sqadd z10.s, z10.s, z2.s\n" - "sqadd z16.s, z16.s, z22.s\n" - ".inst 0x4482936a // srshl z10.s, p4/M, z10.s, z27.s\n" - ".inst 0x44829370 // srshl z16.s, p4/M, z16.s, z27.s\n" - "sqadd z9.s, z9.s, z20.s\n" - ".inst 0x453041ad // sqxtnb z13.h, z13.s\n" - ".inst 0x44829369 // srshl z9.s, p4/M, z9.s, z27.s\n" - ".inst 0x4530435a // sqxtnb z26.h, z26.s\n" - ".inst 0x45304318 // sqxtnb z24.h, z24.s\n" - ".inst 0x45304339 // sqxtnb z25.h, z25.s\n" - ".inst 0x4530462d // sqxtnt z13.h, z17.s\n" - ".inst 0x4530455a // sqxtnt z26.h, z10.s\n" - ".inst 0x45304618 // sqxtnt z24.h, z16.s\n" - ".inst 0x45304539 // sqxtnt z25.h, z9.s\n" - "sqadd z13.h, z13.h, z14.h\n" - "smax z13.h, p4/M, z13.h, z12.h\n" - "smin z13.h, p4/M, z13.h, z11.h\n" - "sqadd z26.h, z26.h, z14.h\n" - "sqadd z24.h, z24.h, z14.h\n" - "smax z26.h, p4/M, z26.h, z12.h\n" - "smax z24.h, p4/M, z24.h, z12.h\n" - "sqadd z25.h, z25.h, z14.h\n" - "smax z25.h, p4/M, z25.h, z12.h\n" - "smin z26.h, p4/M, z26.h, z11.h\n" - "st1b { z13.h }, p0, [x15, x10]\n" - "smin z24.h, p4/M, z24.h, z11.h\n" - "smin z25.h, p4/M, z25.h, z11.h\n" - "st1b { z26.h }, p0, [x14, x10]\n" - "st1b { z24.h }, p0, [x13, x10]\n" - "st1b { z25.h }, p0, [x12, x10]\n" - "ld1b { z0.h }, p4/Z, [x16]\n" - "ld1b { z1.h }, p4/Z, [x16, #1, MUL VL]\n" - "inch x10\n" - "ld1b { z2.h }, p4/Z, [x16, #2, MUL VL]\n" - "ld1b { z3.h }, p4/Z, [x16, #3, MUL VL]\n" - ".inst 0x454f1800 // usublb z0.h, z0.b, z15.b\n" - ".inst 0x454f1821 // usublb z1.h, z1.b, z15.b\n" - "ld1b { z4.h }, p4/Z, [x16, #4, MUL VL]\n" - "ld1b { z5.h }, p4/Z, [x16, #5, MUL VL]\n" - ".inst 0x454f1842 // usublb z2.h, z2.b, z15.b\n" - ".inst 0x454f1863 // usublb z3.h, z3.b, z15.b\n" - "ld1b { z6.h }, p4/Z, [x16, #6, MUL VL]\n" - "ld1b { z7.h }, p4/Z, [x16, #7, MUL VL]\n" - "inch x16, ALL, MUL #8\n" - ".inst 0x454f1884 // usublb z4.h, z4.b, z15.b\n" - "ld1w { z17.s }, p2/Z, [x26]\n" - "ld1w { z16.s }, p1/Z, [x26, #1, MUL VL]\n" - "uzp1 z13.s, z17.s, z16.s\n" - "uzp2 z17.s, z17.s, z16.s\n" - "ld1b { z8.h }, p4/Z, [x16]\n" - "ldp x24, x23, [x11, #0x0]\n" - "addvl x26, x26, #2\n" - "str x26, [%x[params], %[offsetof_Params_bias]]\n" - "ldp x22, x21, [x11, #0x10]\n" - "ldr x20, [x11, #0x20]\n" - "mov z26.d, z13.d\n" - "mov z10.d, z17.d\n" - "ld1b { z31.h }, p3/Z, [x24, x8]\n" - "ld1b { z30.h }, p3/Z, [x23, x8]\n" - "mov z24.d, z13.d\n" - "mov z16.d, z17.d\n" - "ld1b { z29.h }, p3/Z, [x22, x8]\n" - "ld1b { z28.h }, p3/Z, [x21, x8]\n" - "mov z25.d, z13.d\n" - "mov z9.d, z17.d\n" - "ld1b { z27.h }, p3/Z, [x20, x8]\n" - ".inst 0x454f18a5 // usublb z5.h, z5.b, z15.b\n" - ".inst 0x454f18c6 // usublb z6.h, z6.b, z15.b\n" - ".inst 0x454f18e7 // usublb z7.h, z7.b, z15.b\n" - ".inst 0x454f1908 // usublb z8.h, z8.b, z15.b\n" - ".inst 0x45571bff // usublb z31.h, z31.b, z23.b\n" - ".inst 0x45571bde // usublb z30.h, z30.b, z23.b\n" - ".inst 0x45571bbd // usublb z29.h, z29.b, z23.b\n" - ".inst 0x45571b9c // usublb z28.h, z28.b, z23.b\n" - ".inst 0x45571b7b // usublb z27.h, z27.b, z23.b\n" + ".inst 0x44829037 // srshl z23.s, p4/M, z23.s, z1.s\n" + "sqadd z27.s, z27.s, z2.s\n" + ".inst 0x453040a5 // sqxtnb z5.h, z5.s\n" + ".inst 0x4482903b // srshl z27.s, p4/M, z27.s, z1.s\n" + ".inst 0x45304231 // sqxtnb z17.h, z17.s\n" + ".inst 0x45304210 // sqxtnb z16.h, z16.s\n" + ".inst 0x453042d6 // sqxtnb z22.h, z22.s\n" + ".inst 0x45304525 // sqxtnt z5.h, z9.s\n" + ".inst 0x45304731 // sqxtnt z17.h, z25.s\n" + ".inst 0x453046f0 // sqxtnt z16.h, z23.s\n" + ".inst 0x45304776 // sqxtnt z22.h, z27.s\n" + "sqadd z5.h, z5.h, z24.h\n" + "smax z5.h, p4/M, z5.h, z11.h\n" + "smin z5.h, p4/M, z5.h, z26.h\n" + "sqadd z17.h, z17.h, z24.h\n" + "sqadd z16.h, z16.h, z24.h\n" + "smax z17.h, p4/M, z17.h, z11.h\n" + "smax z16.h, p4/M, z16.h, z11.h\n" + "sqadd z22.h, z22.h, z24.h\n" + "smax z22.h, p4/M, z22.h, z11.h\n" + "smin z17.h, p4/M, z17.h, z26.h\n" + "st1b { z5.h }, p0, [x13, x27]\n" + "smin z16.h, p4/M, z16.h, z26.h\n" + "smin z22.h, p4/M, z22.h, z26.h\n" + "st1b { z17.h }, p0, [x12, x27]\n" + "st1b { z16.h }, p0, [x11, x27]\n" + "st1b { z22.h }, p0, [x10, x27]\n" + "ld1b { z14.h }, p4/Z, [x14]\n" + "ld1b { z21.h }, p4/Z, [x14, #1, MUL VL]\n" + "inch x27\n" + "ld1b { z1.h }, p4/Z, [x14, #2, MUL VL]\n" + "ld1b { z6.h }, p4/Z, [x14, #3, MUL VL]\n" + ".inst 0x455e19ce // usublb z14.h, z14.b, z30.b\n" + ".inst 0x455e1ab5 // usublb z21.h, z21.b, z30.b\n" + "ld1b { z2.h }, p4/Z, [x14, #4, MUL VL]\n" + "ld1b { z18.h }, p4/Z, [x14, #5, MUL VL]\n" + ".inst 0x455e1821 // usublb z1.h, z1.b, z30.b\n" + ".inst 0x455e18c6 // usublb z6.h, z6.b, z30.b\n" + "ld1b { z7.h }, p4/Z, [x14, #6, MUL VL]\n" + "ld1b { z10.h }, p4/Z, [x14, #7, MUL VL]\n" + "inch x14, ALL, MUL #8\n" + ".inst 0x455e1842 // usublb z2.h, z2.b, z30.b\n" + "ld1w { z17.s }, p2/Z, [x21]\n" + "ld1w { z16.s }, p1/Z, [x21, #1, MUL VL]\n" + "uzp1 z5.s, z17.s, z16.s\n" + "uzp2 z9.s, z17.s, z16.s\n" + "ld1b { z8.h }, p4/Z, [x14]\n" + "ldp x24, x23, [x28, #0x0]\n" + "addvl x21, x21, #2\n" + "str x21, [%x[params], %[offsetof_Params_bias]]\n" + "ldp x22, x21, [x28, #0x10]\n" + "ldr x20, [x28, #0x20]\n" + "mov z17.d, z5.d\n" + "mov z25.d, z9.d\n" + "ld1b { z0.h }, p3/Z, [x24, x16]\n" + "ld1b { z29.h }, p3/Z, [x23, x16]\n" + "mov z16.d, z5.d\n" + "mov z23.d, z9.d\n" + "ld1b { z4.h }, p3/Z, [x22, x16]\n" + "ld1b { z13.h }, p3/Z, [x21, x16]\n" + "mov z22.d, z5.d\n" + "mov z27.d, z9.d\n" + "ld1b { z20.h }, p3/Z, [x20, x16]\n" + ".inst 0x455e1a52 // usublb z18.h, z18.b, z30.b\n" + ".inst 0x455e18e7 // usublb z7.h, z7.b, z30.b\n" + ".inst 0x455e194a // usublb z10.h, z10.b, z30.b\n" + ".inst 0x455e1908 // usublb z8.h, z8.b, z30.b\n" + ".inst 0x454c1800 // usublb z0.h, z0.b, z12.b\n" + ".inst 0x454c1bbd // usublb z29.h, z29.b, z12.b\n" + ".inst 0x454c1884 // usublb z4.h, z4.b, z12.b\n" + ".inst 0x454c19ad // usublb z13.h, z13.b, z12.b\n" + ".inst 0x454c1a94 // usublb z20.h, z20.b, z12.b\n" "b.any 1b\n" : : [offsetof_Params_bias] "I" (offsetof(Params, bias)), [offsetof_Params_inptrs] "I" (offsetof(Params, inptrs)), [offsetof_Params_n_channels] "I" (offsetof(Params, n_channels)), [offsetof_Params_outptrs] "I" (offsetof(Params, outptrs)), [offsetof_Params_requant] "I" (offsetof(Params, requant)), [offsetof_Params_requant_muls] "I" (offsetof(Params, requant_muls)), [offsetof_Params_requant_shifts] "I" (offsetof(Params, requant_shifts)), [offsetof_Params_weights] "I" (offsetof(Params, weights)), [offsetof_Requantize32_a_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [params] "r" (¶ms) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp index a794095c6fc9c43c9fdd5477b57a93fe0b3cecbb..bcd0d60d3ceca3bbffc72597b9dd0081e23c4491 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,14 +22,14 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include #pragma once -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -47,17 +47,16 @@ class sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst : public DepthwiseDepthfirstS constexpr static unsigned int stride_rows = 2; constexpr static unsigned int stride_cols = 2; - arm_gemm::VLType get_vl_type(void) const override { return arm_gemm::VLType::SVE; } - unsigned int get_accumulator_depth_vl(void) const override { return 2; } - sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst(const CPUInfo *) : Parent(2, 2, 3, 3, 2, 2) {} - Parent::KernelType kernel = sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl; + arm_gemm::VLType get_vl_type(void) const override { return arm_gemm::VLType::SVE; } + Parent::KernelType kernel = sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl; Parent::KernelType get_kernel(void) const override { return kernel; } + unsigned int get_accumulator_depth_vl(void) const override { return 2; } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp index 9f214018409b2f138ce55aaeba78924efeb615a0..1ea2fcbfbdece3453da75c8d3d73317c52c100b0 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp @@ -27,7 +27,7 @@ #include #include -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -46,7 +46,7 @@ void sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( struct Params { long unsigned int n_channels; - const uint8_t *weights; + const void *weights; const int32_t *bias; const arm_gemm::Requantize32 *requant; const int32_t *const requant_muls; @@ -57,7 +57,7 @@ void sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( Params( long unsigned int n_channels, const uint8_t *const *inptrs_raw, - const uint8_t *const weights, + const void *const weights, const int32_t *const bias, const arm_gemm::Requantize32 &qp, const int32_t *const requant_muls, @@ -110,13 +110,13 @@ void sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ldr x17, [%x[params], %[offsetof_Params_weights]]\n" "add x20, x25, %[offsetof_Requantize32_b_offset]\n" "add x22, x25, %[offsetof_Requantize32_c_offset]\n" - "ld1rb { z23.b }, p4/Z, [x21]\n" - "ld1rb { z12.b }, p4/Z, [x20]\n" + "ld1rb { z26.b }, p4/Z, [x21]\n" + "ld1rb { z13.b }, p4/Z, [x20]\n" "add x21, x25, %[offsetof_Requantize32_minval]\n" "add x20, x25, %[offsetof_Requantize32_maxval]\n" - "ld1rh { z14.h }, p4/Z, [x22]\n" - "ld1rh { z16.h }, p4/Z, [x21]\n" - "ld1rh { z15.h }, p4/Z, [x20]\n" + "ld1rh { z19.h }, p4/Z, [x22]\n" + "ld1rh { z12.h }, p4/Z, [x21]\n" + "ld1rh { z9.h }, p4/Z, [x20]\n" "ldp x16, x15, [x24, #0x0]\n" "incw x23\n" "whilelt p3.h, x7, x8\n" @@ -124,320 +124,320 @@ void sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "whilelt p2.s, x7, x8\n" "whilelt p1.s, x23, x8\n" "ldr x12, [%x[params], %[offsetof_Params_bias]]\n" - "ld1b { z0.h }, p4/Z, [x17]\n" - "ld1b { z1.h }, p4/Z, [x17, #1, MUL VL]\n" + "ld1b { z25.h }, p4/Z, [x17]\n" + "ld1b { z30.h }, p4/Z, [x17, #1, MUL VL]\n" "add x11, %x[params], %[offsetof_Params_inptrs]\n" "mov x10, #0x0\n" - "ld1b { z2.h }, p4/Z, [x17, #2, MUL VL]\n" - "ld1b { z3.h }, p4/Z, [x17, #3, MUL VL]\n" - ".inst 0x454c1800 // usublb z0.h, z0.b, z12.b\n" - ".inst 0x454c1821 // usublb z1.h, z1.b, z12.b\n" - "ld1b { z4.h }, p4/Z, [x17, #4, MUL VL]\n" - "ld1b { z5.h }, p4/Z, [x17, #5, MUL VL]\n" - ".inst 0x454c1842 // usublb z2.h, z2.b, z12.b\n" - ".inst 0x454c1863 // usublb z3.h, z3.b, z12.b\n" - "ld1b { z6.h }, p4/Z, [x17, #6, MUL VL]\n" + "ld1b { z14.h }, p4/Z, [x17, #2, MUL VL]\n" + "ld1b { z4.h }, p4/Z, [x17, #3, MUL VL]\n" + ".inst 0x454d1b39 // usublb z25.h, z25.b, z13.b\n" + ".inst 0x454d1bde // usublb z30.h, z30.b, z13.b\n" + "ld1b { z10.h }, p4/Z, [x17, #4, MUL VL]\n" + "ld1b { z3.h }, p4/Z, [x17, #5, MUL VL]\n" + ".inst 0x454d19ce // usublb z14.h, z14.b, z13.b\n" + ".inst 0x454d1884 // usublb z4.h, z4.b, z13.b\n" + "ld1b { z23.h }, p4/Z, [x17, #6, MUL VL]\n" "ld1b { z7.h }, p4/Z, [x17, #7, MUL VL]\n" "inch x17, ALL, MUL #8\n" - ".inst 0x454c1884 // usublb z4.h, z4.b, z12.b\n" - "ld1w { z18.s }, p2/Z, [x12]\n" - "ld1w { z8.s }, p1/Z, [x12, #1, MUL VL]\n" - "uzp1 z13.s, z18.s, z8.s\n" - "uzp2 z17.s, z18.s, z8.s\n" - "ld1b { z8.h }, p4/Z, [x17]\n" - "ldp x9, x28, [x11, #0x0]\n" + ".inst 0x454d194a // usublb z10.h, z10.b, z13.b\n" + "ld1w { z17.s }, p2/Z, [x12]\n" + "ld1w { z16.s }, p1/Z, [x12, #1, MUL VL]\n" + "uzp1 z8.s, z17.s, z16.s\n" + "uzp2 z24.s, z17.s, z16.s\n" + "ld1b { z2.h }, p4/Z, [x17]\n" + "ldp x27, x26, [x11, #0x0]\n" "addvl x12, x12, #2\n" - "mov z9.d, z13.d\n" + "mov z18.d, z8.d\n" "ldp x25, x24, [x11, #0x10]\n" "ldp x23, x22, [x11, #0x20]\n" - "mov z10.d, z17.d\n" - "mov z11.d, z13.d\n" + "mov z0.d, z24.d\n" + "mov z15.d, z8.d\n" "ldp x21, x20, [x11, #0x30]\n" - "ld1b { z31.h }, p3/Z, [x9, x7]\n" - "mov z22.d, z17.d\n" - "mov z21.d, z13.d\n" - "ld1b { z30.h }, p3/Z, [x28, x7]\n" - "ld1b { z29.h }, p3/Z, [x25, x7]\n" - "mov z18.d, z17.d\n" - ".inst 0x454c18a5 // usublb z5.h, z5.b, z12.b\n" - "ld1b { z28.h }, p3/Z, [x24, x7]\n" + "ld1b { z21.h }, p3/Z, [x27, x7]\n" + "mov z1.d, z24.d\n" + "mov z5.d, z8.d\n" + "ld1b { z22.h }, p3/Z, [x26, x7]\n" + "ld1b { z11.h }, p3/Z, [x25, x7]\n" + "mov z6.d, z24.d\n" + ".inst 0x454d1863 // usublb z3.h, z3.b, z13.b\n" + "ld1b { z20.h }, p3/Z, [x24, x7]\n" "ld1b { z27.h }, p3/Z, [x23, x7]\n" - ".inst 0x454c18c6 // usublb z6.h, z6.b, z12.b\n" - ".inst 0x454c18e7 // usublb z7.h, z7.b, z12.b\n" - "ld1b { z26.h }, p3/Z, [x22, x7]\n" - "ld1b { z25.h }, p3/Z, [x21, x7]\n" - ".inst 0x454c1908 // usublb z8.h, z8.b, z12.b\n" - ".inst 0x45571bff // usublb z31.h, z31.b, z23.b\n" - "ld1b { z24.h }, p3/Z, [x20, x7]\n" - "ldr x27, [%x[params], %[offsetof_Params_requant_muls]]\n" - ".inst 0x45571bde // usublb z30.h, z30.b, z23.b\n" - ".inst 0x45571bbd // usublb z29.h, z29.b, z23.b\n" - "ldr x26, [%x[params], %[offsetof_Params_requant_shifts]]\n" + ".inst 0x454d1af7 // usublb z23.h, z23.b, z13.b\n" + ".inst 0x454d18e7 // usublb z7.h, z7.b, z13.b\n" + "ld1b { z28.h }, p3/Z, [x22, x7]\n" + "ld1b { z16.h }, p3/Z, [x21, x7]\n" + ".inst 0x454d1842 // usublb z2.h, z2.b, z13.b\n" + ".inst 0x455a1ab5 // usublb z21.h, z21.b, z26.b\n" + "ld1b { z31.h }, p3/Z, [x20, x7]\n" + "ldr x9, [%x[params], %[offsetof_Params_requant_muls]]\n" + ".inst 0x455a1ad6 // usublb z22.h, z22.b, z26.b\n" + ".inst 0x455a196b // usublb z11.h, z11.b, z26.b\n" + "ldr x28, [%x[params], %[offsetof_Params_requant_shifts]]\n" "str x12, [%x[params], %[offsetof_Params_bias]]\n" - ".inst 0x45571b9c // usublb z28.h, z28.b, z23.b\n" - ".inst 0x45571b7b // usublb z27.h, z27.b, z23.b\n" - ".inst 0x45571b5a // usublb z26.h, z26.b, z23.b\n" - ".inst 0x45571b39 // usublb z25.h, z25.b, z23.b\n" - ".inst 0x45571b18 // usublb z24.h, z24.b, z23.b\n" + ".inst 0x455a1a94 // usublb z20.h, z20.b, z26.b\n" + ".inst 0x455a1b7b // usublb z27.h, z27.b, z26.b\n" + ".inst 0x455a1b9c // usublb z28.h, z28.b, z26.b\n" + ".inst 0x455a1a10 // usublb z16.h, z16.b, z26.b\n" + ".inst 0x455a1bff // usublb z31.h, z31.b, z26.b\n" "1:" // Loop - ".inst 0x448843ed // smlalb z13.s, p4/M, z31.h, z8.h\n" - ".inst 0x448847f1 // smlalt z17.s, p4/M, z31.h, z8.h\n" - "ldr x25, [x11, #0x40]\n" - "ldr x24, [x11, #0x48]\n" - ".inst 0x448643e9 // smlalb z9.s, p4/M, z31.h, z6.h\n" - ".inst 0x448647ea // smlalt z10.s, p4/M, z31.h, z6.h\n" - "ldr x22, [x11, #0x50]\n" - "ldr x20, [x11, #0x58]\n" - ".inst 0x448043cd // smlalb z13.s, p4/M, z30.h, z0.h\n" - ".inst 0x448047d1 // smlalt z17.s, p4/M, z30.h, z0.h\n" - "ldr x23, [x11, #0x78]\n" - "ldr x21, [x11, #0x60]\n" - ".inst 0x44814389 // smlalb z9.s, p4/M, z28.h, z1.h\n" - ".inst 0x4481478a // smlalt z10.s, p4/M, z28.h, z1.h\n" - "ld1b { z28.h }, p3/Z, [x24, x7]\n" - ".inst 0x45571b9c // usublb z28.h, z28.b, z23.b\n" - ".inst 0x448143ad // smlalb z13.s, p4/M, z29.h, z1.h\n" - ".inst 0x448147b1 // smlalt z17.s, p4/M, z29.h, z1.h\n" - "ld1b { z29.h }, p3/Z, [x25, x7]\n" - ".inst 0x45571bbd // usublb z29.h, z29.b, z23.b\n" - ".inst 0x44824369 // smlalb z9.s, p4/M, z27.h, z2.h\n" - ".inst 0x4482476a // smlalt z10.s, p4/M, z27.h, z2.h\n" - "ld1b { z27.h }, p3/Z, [x22, x7]\n" - ".inst 0x45571b7b // usublb z27.h, z27.b, z23.b\n" - ".inst 0x4483434d // smlalb z13.s, p4/M, z26.h, z3.h\n" - ".inst 0x44834751 // smlalt z17.s, p4/M, z26.h, z3.h\n" - "ld1b { z26.h }, p3/Z, [x20, x7]\n" - ".inst 0x45571b5a // usublb z26.h, z26.b, z23.b\n" - ".inst 0x44804309 // smlalb z9.s, p4/M, z24.h, z0.h\n" - ".inst 0x4480470a // smlalt z10.s, p4/M, z24.h, z0.h\n" - "ldr x22, [x11, #0x80]\n" - "ldr x20, [x11, #0x68]\n" - ".inst 0x4484432d // smlalb z13.s, p4/M, z25.h, z4.h\n" - ".inst 0x44844731 // smlalt z17.s, p4/M, z25.h, z4.h\n" - "ld1b { z25.h }, p3/Z, [x21, x7]\n" - ".inst 0x45571b39 // usublb z25.h, z25.b, z23.b\n" - ".inst 0x448443a9 // smlalb z9.s, p4/M, z29.h, z4.h\n" - ".inst 0x448447aa // smlalt z10.s, p4/M, z29.h, z4.h\n" - "ldr x21, [x11, #0x88]\n" + ".inst 0x448242a8 // smlalb z8.s, p4/M, z21.h, z2.h\n" + "ldr x21, [x11, #0x58]\n" + "ldr x20, [x11, #0x78]\n" + ".inst 0x448246b8 // smlalt z24.s, p4/M, z21.h, z2.h\n" + ".inst 0x449942c8 // smlalb z8.s, p4/M, z22.h, z25.h\n" + "ld1b { z17.h }, p3/Z, [x21, x7]\n" "ld1b { z29.h }, p3/Z, [x20, x7]\n" - ".inst 0x4482430d // smlalb z13.s, p4/M, z24.h, z2.h\n" - ".inst 0x44824711 // smlalt z17.s, p4/M, z24.h, z2.h\n" - "ldr x20, [x11, #0x70]\n" - ".inst 0x45571bbd // usublb z29.h, z29.b, z23.b\n" - ".inst 0x44854389 // smlalb z9.s, p4/M, z28.h, z5.h\n" - ".inst 0x4485478a // smlalt z10.s, p4/M, z28.h, z5.h\n" - "ld1b { z28.h }, p3/Z, [x22, x7]\n" - ".inst 0x45571b9c // usublb z28.h, z28.b, z23.b\n" - ".inst 0x448243eb // smlalb z11.s, p4/M, z31.h, z2.h\n" - ".inst 0x448247f6 // smlalt z22.s, p4/M, z31.h, z2.h\n" - "ldr x25, [x11, #0x98]\n" - "ld1b { z24.h }, p3/Z, [x20, x7]\n" - ".inst 0x4485436d // smlalb z13.s, p4/M, z27.h, z5.h\n" - ".inst 0x44854771 // smlalt z17.s, p4/M, z27.h, z5.h\n" - ".inst 0x45571b18 // usublb z24.h, z24.b, z23.b\n" - "ldr x24, [x11, #0x90]\n" - ".inst 0x44834369 // smlalb z9.s, p4/M, z27.h, z3.h\n" - ".inst 0x4483476a // smlalt z10.s, p4/M, z27.h, z3.h\n" + ".inst 0x449742b2 // smlalb z18.s, p4/M, z21.h, z23.h\n" + "ldr x21, [x11, #0x60]\n" + "ldr x20, [x11, #0x80]\n" + ".inst 0x448e42af // smlalb z15.s, p4/M, z21.h, z14.h\n" + ".inst 0x449942a5 // smlalb z5.s, p4/M, z21.h, z25.h\n" + ".inst 0x449946d8 // smlalt z24.s, p4/M, z22.h, z25.h\n" + ".inst 0x455a1a31 // usublb z17.h, z17.b, z26.b\n" + ".inst 0x449e4168 // smlalb z8.s, p4/M, z11.h, z30.h\n" + "ld1b { z22.h }, p3/Z, [x21, x7]\n" + ".inst 0x455a1bbd // usublb z29.h, z29.b, z26.b\n" + ".inst 0x449746a0 // smlalt z0.s, p4/M, z21.h, z23.h\n" + ".inst 0x448e46a1 // smlalt z1.s, p4/M, z21.h, z14.h\n" + "ldr x21, [x11, #0x68]\n" + ".inst 0x449946a6 // smlalt z6.s, p4/M, z21.h, z25.h\n" + "ld1b { z21.h }, p3/Z, [x20, x7]\n" + "ldr x20, [x11, #0x88]\n" + ".inst 0x449e4292 // smlalb z18.s, p4/M, z20.h, z30.h\n" + ".inst 0x4484422f // smlalb z15.s, p4/M, z17.h, z4.h\n" + ".inst 0x448a43a5 // smlalb z5.s, p4/M, z29.h, z10.h\n" + ".inst 0x455a1ad6 // usublb z22.h, z22.b, z26.b\n" + "ldr x22, [x11, #0x40]\n" + ".inst 0x449e4578 // smlalt z24.s, p4/M, z11.h, z30.h\n" + ".inst 0x455a1ab5 // usublb z21.h, z21.b, z26.b\n" + ".inst 0x44844388 // smlalb z8.s, p4/M, z28.h, z4.h\n" + "ld1b { z11.h }, p3/Z, [x21, x7]\n" + ".inst 0x449e4680 // smlalt z0.s, p4/M, z20.h, z30.h\n" + "ld1b { z20.h }, p3/Z, [x20, x7]\n" + ".inst 0x44844621 // smlalt z1.s, p4/M, z17.h, z4.h\n" + "ldr x21, [x11, #0x70]\n" + ".inst 0x448a47a6 // smlalt z6.s, p4/M, z29.h, z10.h\n" + "ldr x20, [x11, #0x98]\n" + ".inst 0x448e4372 // smlalb z18.s, p4/M, z27.h, z14.h\n" + "ldr x23, [x11, #0x50]\n" + ".inst 0x449942cf // smlalb z15.s, p4/M, z22.h, z25.h\n" + ".inst 0x449e42a5 // smlalb z5.s, p4/M, z21.h, z30.h\n" + ".inst 0x455a196b // usublb z11.h, z11.b, z26.b\n" + "ld1b { z17.h }, p3/Z, [x22, x7]\n" + ".inst 0x44844798 // smlalt z24.s, p4/M, z28.h, z4.h\n" + ".inst 0x455a1a94 // usublb z20.h, z20.b, z26.b\n" + ".inst 0x448a4208 // smlalb z8.s, p4/M, z16.h, z10.h\n" + "ld1b { z29.h }, p3/Z, [x21, x7]\n" + "ld1b { z28.h }, p3/Z, [x20, x7]\n" + ".inst 0x448e4760 // smlalt z0.s, p4/M, z27.h, z14.h\n" + "ldr x22, [x11, #0x48]\n" + ".inst 0x449946c1 // smlalt z1.s, p4/M, z22.h, z25.h\n" + ".inst 0x449e46a6 // smlalt z6.s, p4/M, z21.h, z30.h\n" + "ldr x21, [x11, #0x90]\n" + "ldr x20, [x11, #0xa8]\n" + ".inst 0x449943f2 // smlalb z18.s, p4/M, z31.h, z25.h\n" "ld1b { z27.h }, p3/Z, [x23, x7]\n" - ".inst 0x45571b7b // usublb z27.h, z27.b, z23.b\n" - ".inst 0x448043f5 // smlalb z21.s, p4/M, z31.h, z0.h\n" - ".inst 0x4483434b // smlalb z11.s, p4/M, z26.h, z3.h\n" - "ldr x23, [x11, #0xa8]\n" - "ldr x20, [x11, #0xa0]\n" - ".inst 0x44834756 // smlalt z22.s, p4/M, z26.h, z3.h\n" - ".inst 0x448047f2 // smlalt z18.s, p4/M, z31.h, z0.h\n" - "ld1b { z26.h }, p3/Z, [x21, x7]\n" - ".inst 0x45571b5a // usublb z26.h, z26.b, z23.b\n" - ".inst 0x44844375 // smlalb z21.s, p4/M, z27.h, z4.h\n" - ".inst 0x4480432b // smlalb z11.s, p4/M, z25.h, z0.h\n" - "ldr x22, [x11, #0xb0]\n" - "ldr x21, [x11, #0xb8]\n" - ".inst 0x44804736 // smlalt z22.s, p4/M, z25.h, z0.h\n" - ".inst 0x44844772 // smlalt z18.s, p4/M, z27.h, z4.h\n" - "ld1b { z27.h }, p3/Z, [x20, x7]\n" - ".inst 0x45571b7b // usublb z27.h, z27.b, z23.b\n" - ".inst 0x44814395 // smlalb z21.s, p4/M, z28.h, z1.h\n" - ".inst 0x4486432d // smlalb z13.s, p4/M, z25.h, z6.h\n" - "ldr x20, [x11, #0xc0]\n" - "ld1w { z31.s }, p2/Z, [x27]\n" - ".inst 0x44864731 // smlalt z17.s, p4/M, z25.h, z6.h\n" - ".inst 0x448443ab // smlalb z11.s, p4/M, z29.h, z4.h\n" - "ld1b { z25.h }, p3/Z, [x24, x7]\n" - ".inst 0x45571b39 // usublb z25.h, z25.b, z23.b\n" - ".inst 0x448447b6 // smlalt z22.s, p4/M, z29.h, z4.h\n" - "ld1b { z29.h }, p3/Z, [x25, x7]\n" - ".inst 0x44814792 // smlalt z18.s, p4/M, z28.h, z1.h\n" - ".inst 0x45571bbd // usublb z29.h, z29.b, z23.b\n" - ".inst 0x44854355 // smlalb z21.s, p4/M, z26.h, z5.h\n" - ".inst 0x4487430d // smlalb z13.s, p4/M, z24.h, z7.h\n" - "ld1w { z20.s }, p1/Z, [x27, #1, MUL VL]\n" - "uzp1 z19.s, z31.s, z20.s\n" - ".inst 0x44874711 // smlalt z17.s, p4/M, z24.h, z7.h\n" - ".inst 0x4481430b // smlalb z11.s, p4/M, z24.h, z1.h\n" - "uzp2 z30.s, z31.s, z20.s\n" - "ld1w { z31.s }, p2/Z, [x26]\n" - ".inst 0x44814716 // smlalt z22.s, p4/M, z24.h, z1.h\n" - "ld1b { z24.h }, p3/Z, [x23, x7]\n" - ".inst 0x44854752 // smlalt z18.s, p4/M, z26.h, z5.h\n" - ".inst 0x45571b18 // usublb z24.h, z24.b, z23.b\n" - ".inst 0x448243b5 // smlalb z21.s, p4/M, z29.h, z2.h\n" - "ld1b { z26.h }, p3/Z, [x22, x7]\n" - ".inst 0x448247b2 // smlalt z18.s, p4/M, z29.h, z2.h\n" - ".inst 0x45571b5a // usublb z26.h, z26.b, z23.b\n" - ".inst 0x4486432b // smlalb z11.s, p4/M, z25.h, z6.h\n" - ".inst 0x44834315 // smlalb z21.s, p4/M, z24.h, z3.h\n" - "ld1w { z20.s }, p1/Z, [x26, #1, MUL VL]\n" - "uzp1 z1.s, z31.s, z20.s\n" - ".inst 0x44874389 // smlalb z9.s, p4/M, z28.h, z7.h\n" - ".inst 0x4487478a // smlalt z10.s, p4/M, z28.h, z7.h\n" - ".inst 0x04b375ad // sqrdmulh z13.s, z13.s, z19.s\n" - "whilelt p0.h, x10, x8\n" - ".inst 0x44864736 // smlalt z22.s, p4/M, z25.h, z6.h\n" + ".inst 0x448a416f // smlalb z15.s, p4/M, z11.h, z10.h\n" + ".inst 0x44834285 // smlalb z5.s, p4/M, z20.h, z3.h\n" + ".inst 0x455a1a31 // usublb z17.h, z17.b, z26.b\n" + ".inst 0x448a4618 // smlalt z24.s, p4/M, z16.h, z10.h\n" + ".inst 0x455a1bbd // usublb z29.h, z29.b, z26.b\n" + ".inst 0x448e43e8 // smlalb z8.s, p4/M, z31.h, z14.h\n" + "ld1b { z16.h }, p3/Z, [x22, x7]\n" + ".inst 0x455a1b9c // usublb z28.h, z28.b, z26.b\n" + ".inst 0x449947e0 // smlalt z0.s, p4/M, z31.h, z25.h\n" "ld1b { z25.h }, p3/Z, [x21, x7]\n" - ".inst 0x44834712 // smlalt z18.s, p4/M, z24.h, z3.h\n" - ".inst 0x45571b39 // usublb z25.h, z25.b, z23.b\n" - ".inst 0x4487436b // smlalb z11.s, p4/M, z27.h, z7.h\n" - ".inst 0x44874355 // smlalb z21.s, p4/M, z26.h, z7.h\n" - "uzp2 z31.s, z31.s, z20.s\n" - "inch x17\n" - ".inst 0x448843a9 // smlalb z9.s, p4/M, z29.h, z8.h\n" - ".inst 0x448847aa // smlalt z10.s, p4/M, z29.h, z8.h\n" - "ld1b { z29.h }, p3/Z, [x20, x7]\n" - ".inst 0x45571bbd // usublb z29.h, z29.b, z23.b\n" - ".inst 0x44874776 // smlalt z22.s, p4/M, z27.h, z7.h\n" - ".inst 0x44874752 // smlalt z18.s, p4/M, z26.h, z7.h\n" - "and z0.d, z13.d, z1.d\n" + ".inst 0x448a4561 // smlalt z1.s, p4/M, z11.h, z10.h\n" + "ld1b { z11.h }, p3/Z, [x20, x7]\n" + ".inst 0x455a1b7b // usublb z27.h, z27.b, z26.b\n" + ".inst 0x44834686 // smlalt z6.s, p4/M, z20.h, z3.h\n" + "ldr x21, [x11, #0xa0]\n" + "ldr x20, [x11, #0xb0]\n" + ".inst 0x448a4232 // smlalb z18.s, p4/M, z17.h, z10.h\n" + ".inst 0x449e43af // smlalb z15.s, p4/M, z29.h, z30.h\n" + ".inst 0x455a1a10 // usublb z16.h, z16.b, z26.b\n" + ".inst 0x448e4385 // smlalb z5.s, p4/M, z28.h, z14.h\n" + ".inst 0x448e47f8 // smlalt z24.s, p4/M, z31.h, z14.h\n" + ".inst 0x455a1b39 // usublb z25.h, z25.b, z26.b\n" + "ld1b { z20.h }, p3/Z, [x21, x7]\n" + ".inst 0x455a196b // usublb z11.h, z11.b, z26.b\n" + ".inst 0x44834368 // smlalb z8.s, p4/M, z27.h, z3.h\n" + "ld1b { z31.h }, p3/Z, [x20, x7]\n" + ".inst 0x448a4620 // smlalt z0.s, p4/M, z17.h, z10.h\n" + ".inst 0x449e47a1 // smlalt z1.s, p4/M, z29.h, z30.h\n" + ".inst 0x448e4786 // smlalt z6.s, p4/M, z28.h, z14.h\n" + "ldr x20, [x11, #0xb8]\n" + ".inst 0x455a1a94 // usublb z20.h, z20.b, z26.b\n" + ".inst 0x44834212 // smlalb z18.s, p4/M, z16.h, z3.h\n" + ".inst 0x4497432f // smlalb z15.s, p4/M, z25.h, z23.h\n" + ".inst 0x455a1bff // usublb z31.h, z31.b, z26.b\n" + "ld1b { z30.h }, p3/Z, [x20, x7]\n" + ".inst 0x44844165 // smlalb z5.s, p4/M, z11.h, z4.h\n" + ".inst 0x44834778 // smlalt z24.s, p4/M, z27.h, z3.h\n" + "ldr x20, [x11, #0xc0]\n" + "ld1w { z17.s }, p2/Z, [x9]\n" + ".inst 0x449742c8 // smlalb z8.s, p4/M, z22.h, z23.h\n" + ".inst 0x44834600 // smlalt z0.s, p4/M, z16.h, z3.h\n" + "ld1w { z14.s }, p1/Z, [x9, #1, MUL VL]\n" + ".inst 0x455a1bde // usublb z30.h, z30.b, z26.b\n" + ".inst 0x44974721 // smlalt z1.s, p4/M, z25.h, z23.h\n" + ".inst 0x44844566 // smlalt z6.s, p4/M, z11.h, z4.h\n" + "ld1b { z25.h }, p3/Z, [x20, x7]\n" + "uzp1 z10.s, z17.s, z14.s\n" + ".inst 0x44844372 // smlalb z18.s, p4/M, z27.h, z4.h\n" + ".inst 0x4487428f // smlalb z15.s, p4/M, z20.h, z7.h\n" + "uzp2 z14.s, z17.s, z14.s\n" + "ld1w { z17.s }, p2/Z, [x28]\n" + ".inst 0x448743e5 // smlalb z5.s, p4/M, z31.h, z7.h\n" + ".inst 0x449746d8 // smlalt z24.s, p4/M, z22.h, z23.h\n" + "ld1w { z16.s }, p1/Z, [x28, #1, MUL VL]\n" + ".inst 0x455a1b39 // usublb z25.h, z25.b, z26.b\n" + ".inst 0x448743a8 // smlalb z8.s, p4/M, z29.h, z7.h\n" + ".inst 0x44844760 // smlalt z0.s, p4/M, z27.h, z4.h\n" + "uzp1 z4.s, z17.s, z16.s\n" "inch x7\n" - ".inst 0x4485430b // smlalb z11.s, p4/M, z24.h, z5.h\n" - ".inst 0x44864335 // smlalb z21.s, p4/M, z25.h, z6.h\n" - ".inst 0x04be7631 // sqrdmulh z17.s, z17.s, z30.s\n" + ".inst 0x44874681 // smlalt z1.s, p4/M, z20.h, z7.h\n" + ".inst 0x448747e6 // smlalt z6.s, p4/M, z31.h, z7.h\n" + ".inst 0x04aa7508 // sqrdmulh z8.s, z8.s, z10.s\n" + "whilelt p0.h, x10, x8\n" + ".inst 0x448742b2 // smlalb z18.s, p4/M, z21.h, z7.h\n" + ".inst 0x4483416f // smlalb z15.s, p4/M, z11.h, z3.h\n" + "uzp2 z22.s, z17.s, z16.s\n" "mov x20, x7\n" - ".inst 0x44854716 // smlalt z22.s, p4/M, z24.h, z5.h\n" - ".inst 0x44864732 // smlalt z18.s, p4/M, z25.h, z6.h\n" - "asr z0.s, z0.s, #0x1f\n" + ".inst 0x449743c5 // smlalb z5.s, p4/M, z30.h, z23.h\n" + ".inst 0x448747b8 // smlalt z24.s, p4/M, z29.h, z7.h\n" + "and z17.d, z8.d, z4.d\n" + "inch x17\n" + ".inst 0x448746a0 // smlalt z0.s, p4/M, z21.h, z7.h\n" + ".inst 0x44834561 // smlalt z1.s, p4/M, z11.h, z3.h\n" + ".inst 0x04ae7718 // sqrdmulh z24.s, z24.s, z14.s\n" "incw x20\n" - ".inst 0x4488432b // smlalb z11.s, p4/M, z25.h, z8.h\n" - ".inst 0x448843b5 // smlalb z21.s, p4/M, z29.h, z8.h\n" - "and z20.d, z17.d, z31.d\n" + ".inst 0x449747c6 // smlalt z6.s, p4/M, z30.h, z23.h\n" + ".inst 0x44824392 // smlalb z18.s, p4/M, z28.h, z2.h\n" + "asr z17.s, z17.s, #0x1f\n" "whilelt p2.s, x7, x8\n" - ".inst 0x44884736 // smlalt z22.s, p4/M, z25.h, z8.h\n" - ".inst 0x448847b2 // smlalt z18.s, p4/M, z29.h, z8.h\n" - ".inst 0x04b37529 // sqrdmulh z9.s, z9.s, z19.s\n" + ".inst 0x448243cf // smlalb z15.s, p4/M, z30.h, z2.h\n" + ".inst 0x44824325 // smlalb z5.s, p4/M, z25.h, z2.h\n" + "and z16.d, z24.d, z22.d\n" "whilelt p1.s, x20, x8\n" - ".inst 0x04b3756b // sqrdmulh z11.s, z11.s, z19.s\n" - ".inst 0x04b376b5 // sqrdmulh z21.s, z21.s, z19.s\n" - "ldr x12, [%x[params], %[offsetof_Params_bias]]\n" + ".inst 0x44824780 // smlalt z0.s, p4/M, z28.h, z2.h\n" + ".inst 0x448247c1 // smlalt z1.s, p4/M, z30.h, z2.h\n" + ".inst 0x04aa7652 // sqrdmulh z18.s, z18.s, z10.s\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + ".inst 0x44824726 // smlalt z6.s, p4/M, z25.h, z2.h\n" + ".inst 0x04aa75ef // sqrdmulh z15.s, z15.s, z10.s\n" "whilelt p3.h, x7, x8\n" - "sqadd z13.s, z13.s, z0.s\n" - "asr z20.s, z20.s, #0x1f\n" - ".inst 0x4482902d // srshl z13.s, p4/M, z13.s, z1.s\n" - "addvl x27, x27, #2\n" - "and z19.d, z9.d, z1.d\n" - ".inst 0x04be754a // sqrdmulh z10.s, z10.s, z30.s\n" - "addvl x26, x26, #2\n" - "and z2.d, z11.d, z1.d\n" - ".inst 0x04be76d6 // sqrdmulh z22.s, z22.s, z30.s\n" - "and z0.d, z21.d, z1.d\n" - ".inst 0x04be7652 // sqrdmulh z18.s, z18.s, z30.s\n" - "sqadd z17.s, z17.s, z20.s\n" - "asr z19.s, z19.s, #0x1f\n" - ".inst 0x448293f1 // srshl z17.s, p4/M, z17.s, z31.s\n" - "and z3.d, z10.d, z31.d\n" - "asr z2.s, z2.s, #0x1f\n" - "and z26.d, z22.d, z31.d\n" - "asr z0.s, z0.s, #0x1f\n" - "and z20.d, z18.d, z31.d\n" - "sqadd z9.s, z9.s, z19.s\n" - ".inst 0x44829029 // srshl z9.s, p4/M, z9.s, z1.s\n" - "asr z3.s, z3.s, #0x1f\n" - "sqadd z11.s, z11.s, z2.s\n" - ".inst 0x4482902b // srshl z11.s, p4/M, z11.s, z1.s\n" - "asr z26.s, z26.s, #0x1f\n" - "sqadd z21.s, z21.s, z0.s\n" - ".inst 0x44829035 // srshl z21.s, p4/M, z21.s, z1.s\n" + "addvl x9, x9, #2\n" + ".inst 0x04aa74a5 // sqrdmulh z5.s, z5.s, z10.s\n" + "sqadd z8.s, z8.s, z17.s\n" + ".inst 0x44829088 // srshl z8.s, p4/M, z8.s, z4.s\n" + "addvl x28, x28, #2\n" + "asr z16.s, z16.s, #0x1f\n" + "and z21.d, z18.d, z4.d\n" + ".inst 0x04ae7400 // sqrdmulh z0.s, z0.s, z14.s\n" + "and z20.d, z15.d, z4.d\n" + ".inst 0x04ae7421 // sqrdmulh z1.s, z1.s, z14.s\n" + "and z28.d, z5.d, z4.d\n" + ".inst 0x04ae74c6 // sqrdmulh z6.s, z6.s, z14.s\n" + "sqadd z24.s, z24.s, z16.s\n" + ".inst 0x448292d8 // srshl z24.s, p4/M, z24.s, z22.s\n" + "asr z21.s, z21.s, #0x1f\n" + "and z25.d, z0.d, z22.d\n" "asr z20.s, z20.s, #0x1f\n" - "sqadd z10.s, z10.s, z3.s\n" - ".inst 0x448293ea // srshl z10.s, p4/M, z10.s, z31.s\n" - "sqadd z22.s, z22.s, z26.s\n" - "sqadd z18.s, z18.s, z20.s\n" - ".inst 0x448293f6 // srshl z22.s, p4/M, z22.s, z31.s\n" - ".inst 0x448293f2 // srshl z18.s, p4/M, z18.s, z31.s\n" - ".inst 0x453041ad // sqxtnb z13.h, z13.s\n" - ".inst 0x45304129 // sqxtnb z9.h, z9.s\n" - ".inst 0x4530416b // sqxtnb z11.h, z11.s\n" - ".inst 0x453042b5 // sqxtnb z21.h, z21.s\n" - ".inst 0x4530462d // sqxtnt z13.h, z17.s\n" - ".inst 0x45304549 // sqxtnt z9.h, z10.s\n" - ".inst 0x453046cb // sqxtnt z11.h, z22.s\n" - ".inst 0x45304655 // sqxtnt z21.h, z18.s\n" - "sqadd z13.h, z13.h, z14.h\n" - "sqadd z9.h, z9.h, z14.h\n" - "smax z13.h, p4/M, z13.h, z16.h\n" - "smax z9.h, p4/M, z9.h, z16.h\n" - "sqadd z11.h, z11.h, z14.h\n" - "sqadd z21.h, z21.h, z14.h\n" - "smax z11.h, p4/M, z11.h, z16.h\n" - "smax z21.h, p4/M, z21.h, z16.h\n" - "smin z13.h, p4/M, z13.h, z15.h\n" - "smin z9.h, p4/M, z9.h, z15.h\n" - "st1b { z13.h }, p0, [x16, x10]\n" - "smin z11.h, p4/M, z11.h, z15.h\n" - "smin z21.h, p4/M, z21.h, z15.h\n" - "st1b { z9.h }, p0, [x15, x10]\n" - "st1b { z11.h }, p0, [x14, x10]\n" - "st1b { z21.h }, p0, [x13, x10]\n" - "ld1b { z0.h }, p4/Z, [x17]\n" - "ld1b { z1.h }, p4/Z, [x17, #1, MUL VL]\n" + "and z17.d, z1.d, z22.d\n" + "asr z28.s, z28.s, #0x1f\n" + "and z16.d, z6.d, z22.d\n" + "sqadd z18.s, z18.s, z21.s\n" + "asr z25.s, z25.s, #0x1f\n" + ".inst 0x44829092 // srshl z18.s, p4/M, z18.s, z4.s\n" + "sqadd z15.s, z15.s, z20.s\n" + "asr z17.s, z17.s, #0x1f\n" + ".inst 0x4482908f // srshl z15.s, p4/M, z15.s, z4.s\n" + "sqadd z5.s, z5.s, z28.s\n" + "asr z16.s, z16.s, #0x1f\n" + ".inst 0x44829085 // srshl z5.s, p4/M, z5.s, z4.s\n" + "sqadd z0.s, z0.s, z25.s\n" + "sqadd z1.s, z1.s, z17.s\n" + ".inst 0x448292c0 // srshl z0.s, p4/M, z0.s, z22.s\n" + ".inst 0x448292c1 // srshl z1.s, p4/M, z1.s, z22.s\n" + "sqadd z6.s, z6.s, z16.s\n" + ".inst 0x45304108 // sqxtnb z8.h, z8.s\n" + ".inst 0x448292c6 // srshl z6.s, p4/M, z6.s, z22.s\n" + ".inst 0x45304252 // sqxtnb z18.h, z18.s\n" + ".inst 0x453041ef // sqxtnb z15.h, z15.s\n" + ".inst 0x453040a5 // sqxtnb z5.h, z5.s\n" + ".inst 0x45304708 // sqxtnt z8.h, z24.s\n" + ".inst 0x45304412 // sqxtnt z18.h, z0.s\n" + ".inst 0x4530442f // sqxtnt z15.h, z1.s\n" + ".inst 0x453044c5 // sqxtnt z5.h, z6.s\n" + "sqadd z8.h, z8.h, z19.h\n" + "smax z8.h, p4/M, z8.h, z12.h\n" + "smin z8.h, p4/M, z8.h, z9.h\n" + "sqadd z18.h, z18.h, z19.h\n" + "sqadd z15.h, z15.h, z19.h\n" + "smax z18.h, p4/M, z18.h, z12.h\n" + "smax z15.h, p4/M, z15.h, z12.h\n" + "sqadd z5.h, z5.h, z19.h\n" + "smax z5.h, p4/M, z5.h, z12.h\n" + "smin z18.h, p4/M, z18.h, z9.h\n" + "st1b { z8.h }, p0, [x16, x10]\n" + "smin z15.h, p4/M, z15.h, z9.h\n" + "smin z5.h, p4/M, z5.h, z9.h\n" + "st1b { z18.h }, p0, [x15, x10]\n" + "st1b { z15.h }, p0, [x14, x10]\n" + "st1b { z5.h }, p0, [x13, x10]\n" + "ld1b { z25.h }, p4/Z, [x17]\n" + "ld1b { z30.h }, p4/Z, [x17, #1, MUL VL]\n" "inch x10\n" - "ld1b { z2.h }, p4/Z, [x17, #2, MUL VL]\n" - "ld1b { z3.h }, p4/Z, [x17, #3, MUL VL]\n" - ".inst 0x454c1800 // usublb z0.h, z0.b, z12.b\n" - ".inst 0x454c1821 // usublb z1.h, z1.b, z12.b\n" - "ld1b { z4.h }, p4/Z, [x17, #4, MUL VL]\n" - "ld1b { z5.h }, p4/Z, [x17, #5, MUL VL]\n" - ".inst 0x454c1842 // usublb z2.h, z2.b, z12.b\n" - ".inst 0x454c1863 // usublb z3.h, z3.b, z12.b\n" - "ld1b { z6.h }, p4/Z, [x17, #6, MUL VL]\n" + "ld1b { z14.h }, p4/Z, [x17, #2, MUL VL]\n" + "ld1b { z4.h }, p4/Z, [x17, #3, MUL VL]\n" + ".inst 0x454d1b39 // usublb z25.h, z25.b, z13.b\n" + ".inst 0x454d1bde // usublb z30.h, z30.b, z13.b\n" + "ld1b { z10.h }, p4/Z, [x17, #4, MUL VL]\n" + "ld1b { z3.h }, p4/Z, [x17, #5, MUL VL]\n" + ".inst 0x454d19ce // usublb z14.h, z14.b, z13.b\n" + ".inst 0x454d1884 // usublb z4.h, z4.b, z13.b\n" + "ld1b { z23.h }, p4/Z, [x17, #6, MUL VL]\n" "ld1b { z7.h }, p4/Z, [x17, #7, MUL VL]\n" "inch x17, ALL, MUL #8\n" - ".inst 0x454c1884 // usublb z4.h, z4.b, z12.b\n" - "ld1w { z18.s }, p2/Z, [x12]\n" - "ld1w { z8.s }, p1/Z, [x12, #1, MUL VL]\n" - "uzp1 z13.s, z18.s, z8.s\n" - "uzp2 z17.s, z18.s, z8.s\n" - "ld1b { z8.h }, p4/Z, [x17]\n" - "ldp x9, x28, [x11, #0x0]\n" - "addvl x12, x12, #2\n" - "str x12, [%x[params], %[offsetof_Params_bias]]\n" + ".inst 0x454d194a // usublb z10.h, z10.b, z13.b\n" + "ld1w { z17.s }, p2/Z, [x20]\n" + "ld1w { z16.s }, p1/Z, [x20, #1, MUL VL]\n" + "uzp1 z8.s, z17.s, z16.s\n" + "uzp2 z24.s, z17.s, z16.s\n" + "ld1b { z2.h }, p4/Z, [x17]\n" + "ldp x27, x26, [x11, #0x0]\n" + "addvl x20, x20, #2\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" "ldp x25, x24, [x11, #0x10]\n" "ldp x23, x22, [x11, #0x20]\n" - "mov z9.d, z13.d\n" - "mov z10.d, z17.d\n" + "mov z18.d, z8.d\n" + "mov z0.d, z24.d\n" "ldp x21, x20, [x11, #0x30]\n" - "ld1b { z31.h }, p3/Z, [x9, x7]\n" - "mov z11.d, z13.d\n" - "mov z22.d, z17.d\n" - "ld1b { z30.h }, p3/Z, [x28, x7]\n" - "ld1b { z29.h }, p3/Z, [x25, x7]\n" - "mov z21.d, z13.d\n" - "mov z18.d, z17.d\n" - "ld1b { z28.h }, p3/Z, [x24, x7]\n" + "ld1b { z21.h }, p3/Z, [x27, x7]\n" + "mov z15.d, z8.d\n" + "mov z1.d, z24.d\n" + "ld1b { z22.h }, p3/Z, [x26, x7]\n" + "ld1b { z11.h }, p3/Z, [x25, x7]\n" + "mov z5.d, z8.d\n" + "mov z6.d, z24.d\n" + "ld1b { z20.h }, p3/Z, [x24, x7]\n" "ld1b { z27.h }, p3/Z, [x23, x7]\n" - ".inst 0x454c18a5 // usublb z5.h, z5.b, z12.b\n" - ".inst 0x454c18c6 // usublb z6.h, z6.b, z12.b\n" - "ld1b { z26.h }, p3/Z, [x22, x7]\n" - "ld1b { z25.h }, p3/Z, [x21, x7]\n" - ".inst 0x454c18e7 // usublb z7.h, z7.b, z12.b\n" - ".inst 0x454c1908 // usublb z8.h, z8.b, z12.b\n" - "ld1b { z24.h }, p3/Z, [x20, x7]\n" - ".inst 0x45571bff // usublb z31.h, z31.b, z23.b\n" - ".inst 0x45571bde // usublb z30.h, z30.b, z23.b\n" - ".inst 0x45571bbd // usublb z29.h, z29.b, z23.b\n" - ".inst 0x45571b9c // usublb z28.h, z28.b, z23.b\n" - ".inst 0x45571b7b // usublb z27.h, z27.b, z23.b\n" - ".inst 0x45571b5a // usublb z26.h, z26.b, z23.b\n" - ".inst 0x45571b39 // usublb z25.h, z25.b, z23.b\n" - ".inst 0x45571b18 // usublb z24.h, z24.b, z23.b\n" + ".inst 0x454d1863 // usublb z3.h, z3.b, z13.b\n" + ".inst 0x454d1af7 // usublb z23.h, z23.b, z13.b\n" + "ld1b { z28.h }, p3/Z, [x22, x7]\n" + "ld1b { z16.h }, p3/Z, [x21, x7]\n" + ".inst 0x454d18e7 // usublb z7.h, z7.b, z13.b\n" + ".inst 0x454d1842 // usublb z2.h, z2.b, z13.b\n" + "ld1b { z31.h }, p3/Z, [x20, x7]\n" + ".inst 0x455a1ab5 // usublb z21.h, z21.b, z26.b\n" + ".inst 0x455a1ad6 // usublb z22.h, z22.b, z26.b\n" + ".inst 0x455a196b // usublb z11.h, z11.b, z26.b\n" + ".inst 0x455a1a94 // usublb z20.h, z20.b, z26.b\n" + ".inst 0x455a1b7b // usublb z27.h, z27.b, z26.b\n" + ".inst 0x455a1b9c // usublb z28.h, z28.b, z26.b\n" + ".inst 0x455a1a10 // usublb z16.h, z16.b, z26.b\n" + ".inst 0x455a1bff // usublb z31.h, z31.b, z26.b\n" "b.any 1b\n" : : [offsetof_Params_bias] "I" (offsetof(Params, bias)), [offsetof_Params_inptrs] "I" (offsetof(Params, inptrs)), [offsetof_Params_n_channels] "I" (offsetof(Params, n_channels)), [offsetof_Params_outptrs] "I" (offsetof(Params, outptrs)), [offsetof_Params_requant] "I" (offsetof(Params, requant)), [offsetof_Params_requant_muls] "I" (offsetof(Params, requant_muls)), [offsetof_Params_requant_shifts] "I" (offsetof(Params, requant_shifts)), [offsetof_Params_weights] "I" (offsetof(Params, weights)), [offsetof_Requantize32_a_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [params] "r" (¶ms) @@ -448,4 +448,4 @@ void sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp index ac0a00b245234614927faac066962d5c333c7b41..dfaa059e9fbb6cdc40240765424c749d3a1947d5 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,14 +22,14 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include #pragma once -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -47,17 +47,16 @@ class sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst : public DepthwiseDepthfirstS constexpr static unsigned int stride_rows = 1; constexpr static unsigned int stride_cols = 1; - arm_gemm::VLType get_vl_type(void) const override { return arm_gemm::VLType::SVE; } - unsigned int get_accumulator_depth_vl(void) const override { return 2; } - sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst(const CPUInfo *) : Parent(2, 2, 5, 5, 1, 1) {} - Parent::KernelType kernel = sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl; + arm_gemm::VLType get_vl_type(void) const override { return arm_gemm::VLType::SVE; } + Parent::KernelType kernel = sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl; Parent::KernelType get_kernel(void) const override { return kernel; } + unsigned int get_accumulator_depth_vl(void) const override { return 2; } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp index 40e2f5df254fb814dd730b743b9afb53cc364bf5..b8adbb8262e22e6ac8c6047c57c43a032eb6ffde 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp @@ -27,7 +27,7 @@ #include #include -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -46,7 +46,7 @@ void sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( struct Params { long unsigned int n_channels; - const uint8_t *weights; + const void *weights; const int32_t *bias; const arm_gemm::Requantize32 *requant; const int32_t *const requant_muls; @@ -57,7 +57,7 @@ void sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( Params( long unsigned int n_channels, const uint8_t *const *inptrs_raw, - const uint8_t *const weights, + const void *const weights, const int32_t *const bias, const arm_gemm::Requantize32 &qp, const int32_t *const requant_muls, @@ -111,542 +111,542 @@ void sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( requant_muls, requant_shifts, outptrs); __asm__ __volatile__( - "mov x0, #0x0\n" - "mov x24, x0\n" + "mov x2, #0x0\n" + "mov x24, x2\n" "ldr x23, [%x[params], %[offsetof_Params_requant]]\n" - "ldr x1, [%x[params], %[offsetof_Params_n_channels]]\n" + "ldr x3, [%x[params], %[offsetof_Params_n_channels]]\n" "ptrue p4.b\n" "ldr x22, [%x[params], %[offsetof_Params_outptrs]]\n" "incw x24\n" - "ldr x2, [%x[params], %[offsetof_Params_weights]]\n" + "ldr x4, [%x[params], %[offsetof_Params_weights]]\n" "add x21, x23, %[offsetof_Requantize32_a_offset]\n" "add x20, x23, %[offsetof_Requantize32_b_offset]\n" - "ld1rb { z15.b }, p4/Z, [x21]\n" - "ld1rb { z17.b }, p4/Z, [x20]\n" + "ld1rb { z30.b }, p4/Z, [x21]\n" + "ld1rb { z10.b }, p4/Z, [x20]\n" "add x21, x23, %[offsetof_Requantize32_c_offset]\n" "add x20, x23, %[offsetof_Requantize32_minval]\n" - "ld1rh { z12.h }, p4/Z, [x21]\n" - "ld1rh { z13.h }, p4/Z, [x20]\n" + "ld1rh { z15.h }, p4/Z, [x21]\n" + "ld1rh { z12.h }, p4/Z, [x20]\n" "add x20, x23, %[offsetof_Requantize32_maxval]\n" - "ld1rh { z11.h }, p4/Z, [x20]\n" - "ldp x3, x4, [x22, #0x0]\n" - "whilelt p3.h, x0, x1\n" - "ldp x5, x6, [x22, #0x10]\n" - "whilelt p2.s, x0, x1\n" - "whilelt p1.s, x24, x1\n" - "ldr x14, [%x[params], %[offsetof_Params_bias]]\n" - "add x7, %x[params], %[offsetof_Params_inptrs]\n" - "ld1w { z30.s }, p2/Z, [x14]\n" - "ld1w { z16.s }, p1/Z, [x14, #1, MUL VL]\n" - "uzp1 z14.s, z30.s, z16.s\n" - "ld1b { z0.h }, p4/Z, [x2]\n" - "ld1b { z1.h }, p4/Z, [x2, #1, MUL VL]\n" - "uzp2 z10.s, z30.s, z16.s\n" - "addvl x14, x14, #2\n" - "ld1b { z2.h }, p4/Z, [x2, #2, MUL VL]\n" - "ld1b { z3.h }, p4/Z, [x2, #3, MUL VL]\n" - "mov x8, #0x0\n" - "mov z20.d, z14.d\n" - "ld1b { z4.h }, p4/Z, [x2, #4, MUL VL]\n" - "ldp x9, x28, [x7, #0x0]\n" - "mov z7.d, z10.d\n" - "mov z8.d, z14.d\n" - "ldp x27, x26, [x7, #0x10]\n" - "ldp x25, x24, [x7, #0x20]\n" - "mov z16.d, z10.d\n" + "ld1rh { z13.h }, p4/Z, [x20]\n" + "ldp x5, x6, [x22, #0x0]\n" + "whilelt p3.h, x2, x3\n" + "ldp x7, x8, [x22, #0x10]\n" + "whilelt p2.s, x2, x3\n" + "whilelt p1.s, x24, x3\n" + "ldr x10, [%x[params], %[offsetof_Params_bias]]\n" + "add x17, %x[params], %[offsetof_Params_inptrs]\n" + "ld1w { z17.s }, p2/Z, [x10]\n" + "ld1w { z16.s }, p1/Z, [x10, #1, MUL VL]\n" + "uzp1 z14.s, z17.s, z16.s\n" + "ld1b { z26.h }, p4/Z, [x4]\n" + "ld1b { z8.h }, p4/Z, [x4, #1, MUL VL]\n" + "uzp2 z23.s, z17.s, z16.s\n" + "addvl x10, x10, #2\n" + "ld1b { z16.h }, p4/Z, [x4, #2, MUL VL]\n" + "ld1b { z21.h }, p4/Z, [x4, #3, MUL VL]\n" + "mov x16, #0x0\n" "mov z6.d, z14.d\n" - "ldp x23, x22, [x7, #0x30]\n" - "ldp x21, x20, [x7, #0x40]\n" - "mov z5.d, z10.d\n" - ".inst 0x45511800 // usublb z0.h, z0.b, z17.b\n" - "ld1b { z31.h }, p3/Z, [x9, x0]\n" - "ld1b { z30.h }, p3/Z, [x28, x0]\n" - ".inst 0x45511821 // usublb z1.h, z1.b, z17.b\n" - ".inst 0x45511842 // usublb z2.h, z2.b, z17.b\n" - "ld1b { z29.h }, p3/Z, [x27, x0]\n" - "ld1b { z28.h }, p3/Z, [x26, x0]\n" - ".inst 0x45511863 // usublb z3.h, z3.b, z17.b\n" - ".inst 0x45511884 // usublb z4.h, z4.b, z17.b\n" - "ld1b { z27.h }, p3/Z, [x25, x0]\n" - "ld1b { z23.h }, p3/Z, [x24, x0]\n" - ".inst 0x454f1bff // usublb z31.h, z31.b, z15.b\n" - ".inst 0x454f1bde // usublb z30.h, z30.b, z15.b\n" - "ld1b { z25.h }, p3/Z, [x23, x0]\n" - "ld1b { z24.h }, p3/Z, [x22, x0]\n" - ".inst 0x454f1bbd // usublb z29.h, z29.b, z15.b\n" - ".inst 0x454f1b9c // usublb z28.h, z28.b, z15.b\n" - "ld1b { z26.h }, p3/Z, [x21, x0]\n" - "ld1b { z22.h }, p3/Z, [x20, x0]\n" - ".inst 0x454f1b7b // usublb z27.h, z27.b, z15.b\n" - ".inst 0x454f1af7 // usublb z23.h, z23.b, z15.b\n" - "ldr x17, [%x[params], %[offsetof_Params_requant_muls]]\n" - "ldr x16, [%x[params], %[offsetof_Params_requant_shifts]]\n" - "str x14, [%x[params], %[offsetof_Params_bias]]\n" - ".inst 0x454f1b39 // usublb z25.h, z25.b, z15.b\n" - ".inst 0x454f1b18 // usublb z24.h, z24.b, z15.b\n" - ".inst 0x454f1b5a // usublb z26.h, z26.b, z15.b\n" - ".inst 0x454f1ad6 // usublb z22.h, z22.b, z15.b\n" + "ld1b { z17.h }, p4/Z, [x4, #4, MUL VL]\n" + "ldp x9, x28, [x17, #0x0]\n" + "mov z18.d, z23.d\n" + "mov z9.d, z14.d\n" + "ldp x27, x26, [x17, #0x10]\n" + "ldp x25, x24, [x17, #0x20]\n" + "mov z20.d, z23.d\n" + "mov z7.d, z14.d\n" + "ldp x23, x22, [x17, #0x30]\n" + "ldp x21, x20, [x17, #0x40]\n" + "mov z1.d, z23.d\n" + ".inst 0x454a1b5a // usublb z26.h, z26.b, z10.b\n" + "ld1b { z22.h }, p3/Z, [x9, x2]\n" + "ld1b { z2.h }, p3/Z, [x28, x2]\n" + ".inst 0x454a1908 // usublb z8.h, z8.b, z10.b\n" + ".inst 0x454a1a10 // usublb z16.h, z16.b, z10.b\n" + "ld1b { z11.h }, p3/Z, [x27, x2]\n" + "ld1b { z3.h }, p3/Z, [x26, x2]\n" + ".inst 0x454a1ab5 // usublb z21.h, z21.b, z10.b\n" + ".inst 0x454a1a31 // usublb z17.h, z17.b, z10.b\n" + "ld1b { z29.h }, p3/Z, [x25, x2]\n" + "ld1b { z4.h }, p3/Z, [x24, x2]\n" + ".inst 0x455e1ad6 // usublb z22.h, z22.b, z30.b\n" + ".inst 0x455e1842 // usublb z2.h, z2.b, z30.b\n" + "ld1b { z31.h }, p3/Z, [x23, x2]\n" + "ld1b { z0.h }, p3/Z, [x22, x2]\n" + ".inst 0x455e196b // usublb z11.h, z11.b, z30.b\n" + ".inst 0x455e1863 // usublb z3.h, z3.b, z30.b\n" + "ld1b { z19.h }, p3/Z, [x21, x2]\n" + "ld1b { z28.h }, p3/Z, [x20, x2]\n" + ".inst 0x455e1bbd // usublb z29.h, z29.b, z30.b\n" + ".inst 0x455e1884 // usublb z4.h, z4.b, z30.b\n" + "ldr x15, [%x[params], %[offsetof_Params_requant_muls]]\n" + "ldr x14, [%x[params], %[offsetof_Params_requant_shifts]]\n" + "str x10, [%x[params], %[offsetof_Params_bias]]\n" + ".inst 0x455e1bff // usublb z31.h, z31.b, z30.b\n" + ".inst 0x455e1800 // usublb z0.h, z0.b, z30.b\n" + ".inst 0x455e1a73 // usublb z19.h, z19.b, z30.b\n" + ".inst 0x455e1b9c // usublb z28.h, z28.b, z30.b\n" "1:" // Loop - ".inst 0x448043ee // smlalb z14.s, p4/M, z31.h, z0.h\n" - ".inst 0x448047ea // smlalt z10.s, p4/M, z31.h, z0.h\n" - "ldr x20, [x7, #0x50]\n" - "ld1b { z31.h }, p3/Z, [x20, x0]\n" - ".inst 0x448143ce // smlalb z14.s, p4/M, z30.h, z1.h\n" - ".inst 0x448043d4 // smlalb z20.s, p4/M, z30.h, z0.h\n" - "ldr x22, [x7, #0x58]\n" - ".inst 0x454f1bff // usublb z31.h, z31.b, z15.b\n" - ".inst 0x448043a8 // smlalb z8.s, p4/M, z29.h, z0.h\n" - ".inst 0x44804386 // smlalb z6.s, p4/M, z28.h, z0.h\n" - "ldr x21, [x7, #0x60]\n" - "ldr x20, [x7, #0x68]\n" - ".inst 0x448147ca // smlalt z10.s, p4/M, z30.h, z1.h\n" - ".inst 0x448047c7 // smlalt z7.s, p4/M, z30.h, z0.h\n" - "ld1b { z30.h }, p3/Z, [x22, x0]\n" - ".inst 0x454f1bde // usublb z30.h, z30.b, z15.b\n" - ".inst 0x448047b0 // smlalt z16.s, p4/M, z29.h, z0.h\n" - ".inst 0x4482436e // smlalb z14.s, p4/M, z27.h, z2.h\n" - "ldr x25, [x7, #0x70]\n" - "ldr x24, [x7, #0x78]\n" - ".inst 0x44804785 // smlalt z5.s, p4/M, z28.h, z0.h\n" - ".inst 0x44814374 // smlalb z20.s, p4/M, z27.h, z1.h\n" - "ld1b { z0.h }, p4/Z, [x2, #5, MUL VL]\n" - ".inst 0x45511800 // usublb z0.h, z0.b, z17.b\n" - ".inst 0x44814388 // smlalb z8.s, p4/M, z28.h, z1.h\n" - ".inst 0x448142e6 // smlalb z6.s, p4/M, z23.h, z1.h\n" - "ldr x15, [x7, #0x80]\n" - "ldr x23, [x7, #0x88]\n" - ".inst 0x4482476a // smlalt z10.s, p4/M, z27.h, z2.h\n" - ".inst 0x44814767 // smlalt z7.s, p4/M, z27.h, z1.h\n" - "ld1b { z27.h }, p3/Z, [x21, x0]\n" - ".inst 0x454f1b7b // usublb z27.h, z27.b, z15.b\n" - ".inst 0x44814790 // smlalt z16.s, p4/M, z28.h, z1.h\n" - ".inst 0x4483432e // smlalb z14.s, p4/M, z25.h, z3.h\n" - "ldr x22, [x7, #0x90]\n" - "ldr x21, [x7, #0x98]\n" - ".inst 0x448146e5 // smlalt z5.s, p4/M, z23.h, z1.h\n" - ".inst 0x44824334 // smlalb z20.s, p4/M, z25.h, z2.h\n" - "ld1b { z1.h }, p4/Z, [x2, #6, MUL VL]\n" - ".inst 0x45511821 // usublb z1.h, z1.b, z17.b\n" - ".inst 0x448242e8 // smlalb z8.s, p4/M, z23.h, z2.h\n" - ".inst 0x448243e6 // smlalb z6.s, p4/M, z31.h, z2.h\n" - "ldr x14, [x7, #0xa0]\n" - "ldr x13, [x7, #0xa8]\n" - ".inst 0x4483472a // smlalt z10.s, p4/M, z25.h, z3.h\n" - ".inst 0x44824727 // smlalt z7.s, p4/M, z25.h, z2.h\n" - "ld1b { z25.h }, p3/Z, [x20, x0]\n" - ".inst 0x454f1b39 // usublb z25.h, z25.b, z15.b\n" - ".inst 0x448246f0 // smlalt z16.s, p4/M, z23.h, z2.h\n" - ".inst 0x4484430e // smlalb z14.s, p4/M, z24.h, z4.h\n" - "ldr x12, [x7, #0xb0]\n" - "ldr x20, [x7, #0xb8]\n" - ".inst 0x448247e5 // smlalt z5.s, p4/M, z31.h, z2.h\n" - ".inst 0x44834314 // smlalb z20.s, p4/M, z24.h, z3.h\n" - "ld1b { z2.h }, p4/Z, [x2, #7, MUL VL]\n" - "inch x2, ALL, MUL #8\n" - ".inst 0x448343e8 // smlalb z8.s, p4/M, z31.h, z3.h\n" - ".inst 0x448343c6 // smlalb z6.s, p4/M, z30.h, z3.h\n" - ".inst 0x45511842 // usublb z2.h, z2.b, z17.b\n" - "ldr x11, [x7, #0xc0]\n" - ".inst 0x4484470a // smlalt z10.s, p4/M, z24.h, z4.h\n" - ".inst 0x44834707 // smlalt z7.s, p4/M, z24.h, z3.h\n" - "ld1b { z24.h }, p3/Z, [x25, x0]\n" - ".inst 0x454f1b18 // usublb z24.h, z24.b, z15.b\n" - ".inst 0x448347f0 // smlalt z16.s, p4/M, z31.h, z3.h\n" - ".inst 0x448043ae // smlalb z14.s, p4/M, z29.h, z0.h\n" - "ldr x10, [x7, #0xc8]\n" - "ldr x9, [x7, #0xd0]\n" - ".inst 0x448347c5 // smlalt z5.s, p4/M, z30.h, z3.h\n" - ".inst 0x44844374 // smlalb z20.s, p4/M, z27.h, z4.h\n" - "ld1b { z3.h }, p4/Z, [x2]\n" - ".inst 0x45511863 // usublb z3.h, z3.b, z17.b\n" - ".inst 0x448443c8 // smlalb z8.s, p4/M, z30.h, z4.h\n" - ".inst 0x44844346 // smlalb z6.s, p4/M, z26.h, z4.h\n" - "ldr x28, [x7, #0xd8]\n" - "ldr x27, [x7, #0xe0]\n" - ".inst 0x448047aa // smlalt z10.s, p4/M, z29.h, z0.h\n" - ".inst 0x44844767 // smlalt z7.s, p4/M, z27.h, z4.h\n" - "ld1b { z27.h }, p3/Z, [x24, x0]\n" - ".inst 0x454f1b7b // usublb z27.h, z27.b, z15.b\n" - ".inst 0x448447d0 // smlalt z16.s, p4/M, z30.h, z4.h\n" - ".inst 0x4481438e // smlalb z14.s, p4/M, z28.h, z1.h\n" - "ldr x26, [x7, #0xe8]\n" - "ldr x25, [x7, #0xf0]\n" - ".inst 0x44844745 // smlalt z5.s, p4/M, z26.h, z4.h\n" - ".inst 0x44804394 // smlalb z20.s, p4/M, z28.h, z0.h\n" - "ld1b { z4.h }, p4/Z, [x2, #1, MUL VL]\n" - ".inst 0x45511884 // usublb z4.h, z4.b, z17.b\n" - ".inst 0x448042c8 // smlalb z8.s, p4/M, z22.h, z0.h\n" - ".inst 0x44804326 // smlalb z6.s, p4/M, z25.h, z0.h\n" - "ld1w { z19.s }, p2/Z, [x17]\n" - "ld1w { z18.s }, p1/Z, [x17, #1, MUL VL]\n" - ".inst 0x4481478a // smlalt z10.s, p4/M, z28.h, z1.h\n" - ".inst 0x44804787 // smlalt z7.s, p4/M, z28.h, z0.h\n" - "ld1b { z28.h }, p3/Z, [x23, x0]\n" - ".inst 0x454f1b9c // usublb z28.h, z28.b, z15.b\n" - ".inst 0x448046d0 // smlalt z16.s, p4/M, z22.h, z0.h\n" - ".inst 0x448242ee // smlalb z14.s, p4/M, z23.h, z2.h\n" - "ldr x24, [x7, #0xf8]\n" - "uzp1 z9.s, z19.s, z18.s\n" - ".inst 0x44804725 // smlalt z5.s, p4/M, z25.h, z0.h\n" - ".inst 0x448142f4 // smlalb z20.s, p4/M, z23.h, z1.h\n" - "ld1b { z0.h }, p4/Z, [x2, #2, MUL VL]\n" - ".inst 0x45511800 // usublb z0.h, z0.b, z17.b\n" - ".inst 0x44814328 // smlalb z8.s, p4/M, z25.h, z1.h\n" - ".inst 0x44814306 // smlalb z6.s, p4/M, z24.h, z1.h\n" - "uzp2 z29.s, z19.s, z18.s\n" - "ld1w { z19.s }, p2/Z, [x16]\n" - ".inst 0x448246ea // smlalt z10.s, p4/M, z23.h, z2.h\n" - ".inst 0x448146e7 // smlalt z7.s, p4/M, z23.h, z1.h\n" - "ld1b { z23.h }, p3/Z, [x15, x0]\n" - ".inst 0x454f1af7 // usublb z23.h, z23.b, z15.b\n" - ".inst 0x44814730 // smlalt z16.s, p4/M, z25.h, z1.h\n" - ".inst 0x448343ee // smlalb z14.s, p4/M, z31.h, z3.h\n" - "ldr x23, [x7, #0x100]\n" - "whilelt p0.h, x8, x1\n" - ".inst 0x44814705 // smlalt z5.s, p4/M, z24.h, z1.h\n" - ".inst 0x448243f4 // smlalb z20.s, p4/M, z31.h, z2.h\n" - "ld1b { z1.h }, p4/Z, [x2, #3, MUL VL]\n" - ".inst 0x45511821 // usublb z1.h, z1.b, z17.b\n" - ".inst 0x44824308 // smlalb z8.s, p4/M, z24.h, z2.h\n" - ".inst 0x44824366 // smlalb z6.s, p4/M, z27.h, z2.h\n" - "addvl x17, x17, #2\n" - ".inst 0x448347ea // smlalt z10.s, p4/M, z31.h, z3.h\n" - ".inst 0x448247e7 // smlalt z7.s, p4/M, z31.h, z2.h\n" - "ld1b { z31.h }, p3/Z, [x22, x0]\n" - ".inst 0x454f1bff // usublb z31.h, z31.b, z15.b\n" - ".inst 0x44824710 // smlalt z16.s, p4/M, z24.h, z2.h\n" - ".inst 0x448443ce // smlalb z14.s, p4/M, z30.h, z4.h\n" - "ldr x22, [x7, #0x108]\n" - ".inst 0x44824765 // smlalt z5.s, p4/M, z27.h, z2.h\n" - ".inst 0x448343d4 // smlalb z20.s, p4/M, z30.h, z3.h\n" - "ld1b { z2.h }, p4/Z, [x2, #4, MUL VL]\n" - ".inst 0x45511842 // usublb z2.h, z2.b, z17.b\n" - ".inst 0x44834368 // smlalb z8.s, p4/M, z27.h, z3.h\n" - ".inst 0x448342e6 // smlalb z6.s, p4/M, z23.h, z3.h\n" - ".inst 0x448447ca // smlalt z10.s, p4/M, z30.h, z4.h\n" - ".inst 0x448347c7 // smlalt z7.s, p4/M, z30.h, z3.h\n" - "ld1b { z30.h }, p3/Z, [x21, x0]\n" - ".inst 0x454f1bde // usublb z30.h, z30.b, z15.b\n" - ".inst 0x44834770 // smlalt z16.s, p4/M, z27.h, z3.h\n" - ".inst 0x448042ce // smlalb z14.s, p4/M, z22.h, z0.h\n" - "ldr x21, [x7, #0x110]\n" - ".inst 0x448346e5 // smlalt z5.s, p4/M, z23.h, z3.h\n" - ".inst 0x44844354 // smlalb z20.s, p4/M, z26.h, z4.h\n" - "ld1b { z3.h }, p4/Z, [x2, #5, MUL VL]\n" - ".inst 0x45511863 // usublb z3.h, z3.b, z17.b\n" - ".inst 0x448442e8 // smlalb z8.s, p4/M, z23.h, z4.h\n" - ".inst 0x44844386 // smlalb z6.s, p4/M, z28.h, z4.h\n" - ".inst 0x448046ca // smlalt z10.s, p4/M, z22.h, z0.h\n" - ".inst 0x44844747 // smlalt z7.s, p4/M, z26.h, z4.h\n" - "ld1b { z26.h }, p3/Z, [x14, x0]\n" - ".inst 0x454f1b5a // usublb z26.h, z26.b, z15.b\n" - ".inst 0x448446f0 // smlalt z16.s, p4/M, z23.h, z4.h\n" - ".inst 0x4481432e // smlalb z14.s, p4/M, z25.h, z1.h\n" - "ld1b { z22.h }, p3/Z, [x20, x0]\n" - ".inst 0x454f1ad6 // usublb z22.h, z22.b, z15.b\n" - ".inst 0x44844785 // smlalt z5.s, p4/M, z28.h, z4.h\n" - ".inst 0x44804334 // smlalb z20.s, p4/M, z25.h, z0.h\n" - "ld1b { z4.h }, p4/Z, [x2, #6, MUL VL]\n" - ".inst 0x45511884 // usublb z4.h, z4.b, z17.b\n" - ".inst 0x448043e8 // smlalb z8.s, p4/M, z31.h, z0.h\n" - ".inst 0x448043c6 // smlalb z6.s, p4/M, z30.h, z0.h\n" - "ldr x20, [x7, #0x118]\n" - "ldr x14, [%x[params], %[offsetof_Params_bias]]\n" - ".inst 0x4481472a // smlalt z10.s, p4/M, z25.h, z1.h\n" - ".inst 0x44804727 // smlalt z7.s, p4/M, z25.h, z0.h\n" - "ld1b { z25.h }, p3/Z, [x13, x0]\n" - ".inst 0x454f1b39 // usublb z25.h, z25.b, z15.b\n" - ".inst 0x448047f0 // smlalt z16.s, p4/M, z31.h, z0.h\n" - ".inst 0x4482430e // smlalb z14.s, p4/M, z24.h, z2.h\n" - ".inst 0x448047c5 // smlalt z5.s, p4/M, z30.h, z0.h\n" - ".inst 0x44814314 // smlalb z20.s, p4/M, z24.h, z1.h\n" - "ld1b { z0.h }, p4/Z, [x2, #7, MUL VL]\n" - "inch x2, ALL, MUL #8\n" - ".inst 0x448143c8 // smlalb z8.s, p4/M, z30.h, z1.h\n" - ".inst 0x44814346 // smlalb z6.s, p4/M, z26.h, z1.h\n" - ".inst 0x45511800 // usublb z0.h, z0.b, z17.b\n" - ".inst 0x4482470a // smlalt z10.s, p4/M, z24.h, z2.h\n" - ".inst 0x44814707 // smlalt z7.s, p4/M, z24.h, z1.h\n" - "ld1b { z24.h }, p3/Z, [x12, x0]\n" - ".inst 0x454f1b18 // usublb z24.h, z24.b, z15.b\n" - ".inst 0x448147d0 // smlalt z16.s, p4/M, z30.h, z1.h\n" - ".inst 0x4483436e // smlalb z14.s, p4/M, z27.h, z3.h\n" - ".inst 0x44814745 // smlalt z5.s, p4/M, z26.h, z1.h\n" - ".inst 0x44824374 // smlalb z20.s, p4/M, z27.h, z2.h\n" - "ld1b { z1.h }, p4/Z, [x2]\n" - ".inst 0x45511821 // usublb z1.h, z1.b, z17.b\n" - ".inst 0x44824348 // smlalb z8.s, p4/M, z26.h, z2.h\n" - ".inst 0x44824326 // smlalb z6.s, p4/M, z25.h, z2.h\n" - ".inst 0x4483476a // smlalt z10.s, p4/M, z27.h, z3.h\n" - ".inst 0x44824767 // smlalt z7.s, p4/M, z27.h, z2.h\n" - "ld1b { z27.h }, p3/Z, [x11, x0]\n" - ".inst 0x454f1b7b // usublb z27.h, z27.b, z15.b\n" - ".inst 0x44824750 // smlalt z16.s, p4/M, z26.h, z2.h\n" - ".inst 0x448442ee // smlalb z14.s, p4/M, z23.h, z4.h\n" - ".inst 0x44824725 // smlalt z5.s, p4/M, z25.h, z2.h\n" - ".inst 0x448342f4 // smlalb z20.s, p4/M, z23.h, z3.h\n" - "ld1b { z2.h }, p4/Z, [x2, #1, MUL VL]\n" - ".inst 0x45511842 // usublb z2.h, z2.b, z17.b\n" - ".inst 0x44834328 // smlalb z8.s, p4/M, z25.h, z3.h\n" - ".inst 0x44834306 // smlalb z6.s, p4/M, z24.h, z3.h\n" - ".inst 0x448446ea // smlalt z10.s, p4/M, z23.h, z4.h\n" - ".inst 0x448346e7 // smlalt z7.s, p4/M, z23.h, z3.h\n" - "ld1b { z23.h }, p3/Z, [x10, x0]\n" - ".inst 0x454f1af7 // usublb z23.h, z23.b, z15.b\n" - ".inst 0x44834730 // smlalt z16.s, p4/M, z25.h, z3.h\n" - ".inst 0x448043ee // smlalb z14.s, p4/M, z31.h, z0.h\n" - ".inst 0x44834705 // smlalt z5.s, p4/M, z24.h, z3.h\n" - ".inst 0x44844394 // smlalb z20.s, p4/M, z28.h, z4.h\n" - "ld1b { z3.h }, p4/Z, [x2, #2, MUL VL]\n" - ".inst 0x45511863 // usublb z3.h, z3.b, z17.b\n" - ".inst 0x44844308 // smlalb z8.s, p4/M, z24.h, z4.h\n" - ".inst 0x448442c6 // smlalb z6.s, p4/M, z22.h, z4.h\n" - ".inst 0x448047ea // smlalt z10.s, p4/M, z31.h, z0.h\n" - ".inst 0x44844787 // smlalt z7.s, p4/M, z28.h, z4.h\n" - "ld1b { z31.h }, p3/Z, [x9, x0]\n" - ".inst 0x454f1bff // usublb z31.h, z31.b, z15.b\n" - ".inst 0x44844710 // smlalt z16.s, p4/M, z24.h, z4.h\n" - ".inst 0x448143ce // smlalb z14.s, p4/M, z30.h, z1.h\n" - "ld1b { z28.h }, p3/Z, [x27, x0]\n" - ".inst 0x454f1b9c // usublb z28.h, z28.b, z15.b\n" - ".inst 0x448446c5 // smlalt z5.s, p4/M, z22.h, z4.h\n" - ".inst 0x448043d4 // smlalb z20.s, p4/M, z30.h, z0.h\n" - "ld1b { z4.h }, p4/Z, [x2, #3, MUL VL]\n" - ".inst 0x45511884 // usublb z4.h, z4.b, z17.b\n" - ".inst 0x44804368 // smlalb z8.s, p4/M, z27.h, z0.h\n" - ".inst 0x448042e6 // smlalb z6.s, p4/M, z23.h, z0.h\n" - ".inst 0x448147ca // smlalt z10.s, p4/M, z30.h, z1.h\n" - ".inst 0x448047c7 // smlalt z7.s, p4/M, z30.h, z0.h\n" - "ld1b { z30.h }, p3/Z, [x28, x0]\n" - ".inst 0x454f1bde // usublb z30.h, z30.b, z15.b\n" - ".inst 0x44804770 // smlalt z16.s, p4/M, z27.h, z0.h\n" - ".inst 0x4482434e // smlalb z14.s, p4/M, z26.h, z2.h\n" - ".inst 0x448046e5 // smlalt z5.s, p4/M, z23.h, z0.h\n" - ".inst 0x44814354 // smlalb z20.s, p4/M, z26.h, z1.h\n" - "ld1b { z0.h }, p4/Z, [x2, #4, MUL VL]\n" - ".inst 0x45511800 // usublb z0.h, z0.b, z17.b\n" - ".inst 0x448142e8 // smlalb z8.s, p4/M, z23.h, z1.h\n" - ".inst 0x448143e6 // smlalb z6.s, p4/M, z31.h, z1.h\n" - ".inst 0x4482474a // smlalt z10.s, p4/M, z26.h, z2.h\n" - ".inst 0x44814747 // smlalt z7.s, p4/M, z26.h, z1.h\n" - "ld1b { z26.h }, p3/Z, [x26, x0]\n" - ".inst 0x454f1b5a // usublb z26.h, z26.b, z15.b\n" - ".inst 0x448146f0 // smlalt z16.s, p4/M, z23.h, z1.h\n" - ".inst 0x4483432e // smlalb z14.s, p4/M, z25.h, z3.h\n" - ".inst 0x448147e5 // smlalt z5.s, p4/M, z31.h, z1.h\n" - ".inst 0x44824334 // smlalb z20.s, p4/M, z25.h, z2.h\n" - "ld1b { z1.h }, p4/Z, [x2, #5, MUL VL]\n" - ".inst 0x45511821 // usublb z1.h, z1.b, z17.b\n" - ".inst 0x448243e8 // smlalb z8.s, p4/M, z31.h, z2.h\n" - ".inst 0x448243c6 // smlalb z6.s, p4/M, z30.h, z2.h\n" - ".inst 0x4483472a // smlalt z10.s, p4/M, z25.h, z3.h\n" - ".inst 0x44824727 // smlalt z7.s, p4/M, z25.h, z2.h\n" - "ld1b { z25.h }, p3/Z, [x25, x0]\n" - ".inst 0x454f1b39 // usublb z25.h, z25.b, z15.b\n" - ".inst 0x448247f0 // smlalt z16.s, p4/M, z31.h, z2.h\n" - ".inst 0x4484430e // smlalb z14.s, p4/M, z24.h, z4.h\n" - ".inst 0x448247c5 // smlalt z5.s, p4/M, z30.h, z2.h\n" - ".inst 0x44834314 // smlalb z20.s, p4/M, z24.h, z3.h\n" - "ld1b { z2.h }, p4/Z, [x2, #6, MUL VL]\n" - ".inst 0x45511842 // usublb z2.h, z2.b, z17.b\n" - ".inst 0x448343c8 // smlalb z8.s, p4/M, z30.h, z3.h\n" - ".inst 0x44834386 // smlalb z6.s, p4/M, z28.h, z3.h\n" - ".inst 0x4484470a // smlalt z10.s, p4/M, z24.h, z4.h\n" - ".inst 0x44834707 // smlalt z7.s, p4/M, z24.h, z3.h\n" - "ld1b { z24.h }, p3/Z, [x24, x0]\n" - ".inst 0x454f1b18 // usublb z24.h, z24.b, z15.b\n" - ".inst 0x448347d0 // smlalt z16.s, p4/M, z30.h, z3.h\n" - ".inst 0x4480436e // smlalb z14.s, p4/M, z27.h, z0.h\n" - ".inst 0x44834785 // smlalt z5.s, p4/M, z28.h, z3.h\n" - ".inst 0x448442d4 // smlalb z20.s, p4/M, z22.h, z4.h\n" - "ld1b { z3.h }, p4/Z, [x2, #7, MUL VL]\n" - "inch x2, ALL, MUL #8\n" - ".inst 0x44844388 // smlalb z8.s, p4/M, z28.h, z4.h\n" - ".inst 0x44844346 // smlalb z6.s, p4/M, z26.h, z4.h\n" - ".inst 0x45511863 // usublb z3.h, z3.b, z17.b\n" - ".inst 0x4480476a // smlalt z10.s, p4/M, z27.h, z0.h\n" - ".inst 0x44844790 // smlalt z16.s, p4/M, z28.h, z4.h\n" - "ld1b { z27.h }, p3/Z, [x23, x0]\n" - ".inst 0x454f1b7b // usublb z27.h, z27.b, z15.b\n" - ".inst 0x448142ee // smlalb z14.s, p4/M, z23.h, z1.h\n" - ".inst 0x448446c7 // smlalt z7.s, p4/M, z22.h, z4.h\n" - "ld1w { z18.s }, p1/Z, [x16, #1, MUL VL]\n" - "addvl x16, x16, #2\n" - ".inst 0x44844745 // smlalt z5.s, p4/M, z26.h, z4.h\n" - ".inst 0x448042f4 // smlalb z20.s, p4/M, z23.h, z0.h\n" - "ld1b { z4.h }, p4/Z, [x2]\n" - ".inst 0x45511884 // usublb z4.h, z4.b, z17.b\n" - ".inst 0x44804328 // smlalb z8.s, p4/M, z25.h, z0.h\n" - ".inst 0x44804306 // smlalb z6.s, p4/M, z24.h, z0.h\n" - "inch x2\n" - ".inst 0x448146ea // smlalt z10.s, p4/M, z23.h, z1.h\n" - ".inst 0x44804730 // smlalt z16.s, p4/M, z25.h, z0.h\n" - "ld1b { z25.h }, p3/Z, [x22, x0]\n" - ".inst 0x454f1b39 // usublb z25.h, z25.b, z15.b\n" + ".inst 0x449a42ce // smlalb z14.s, p4/M, z22.h, z26.h\n" + ".inst 0x449a46d7 // smlalt z23.s, p4/M, z22.h, z26.h\n" + "ldr x20, [x17, #0x50]\n" + "ld1b { z27.h }, p3/Z, [x20, x2]\n" + ".inst 0x4488404e // smlalb z14.s, p4/M, z2.h, z8.h\n" + ".inst 0x449a4046 // smlalb z6.s, p4/M, z2.h, z26.h\n" + "ldr x20, [x17, #0x58]\n" + ".inst 0x455e1b7b // usublb z27.h, z27.b, z30.b\n" + ".inst 0x449a4169 // smlalb z9.s, p4/M, z11.h, z26.h\n" + ".inst 0x449a4067 // smlalb z7.s, p4/M, z3.h, z26.h\n" + "ld1b { z5.h }, p3/Z, [x20, x2]\n" + "ldr x20, [x17, #0x60]\n" + ".inst 0x44884457 // smlalt z23.s, p4/M, z2.h, z8.h\n" + ".inst 0x449043ae // smlalb z14.s, p4/M, z29.h, z16.h\n" + "ld1b { z25.h }, p4/Z, [x4, #5, MUL VL]\n" + ".inst 0x455e18a5 // usublb z5.h, z5.b, z30.b\n" + ".inst 0x449a4452 // smlalt z18.s, p4/M, z2.h, z26.h\n" + ".inst 0x449a4574 // smlalt z20.s, p4/M, z11.h, z26.h\n" + "ld1b { z22.h }, p3/Z, [x20, x2]\n" + ".inst 0x454a1b39 // usublb z25.h, z25.b, z10.b\n" + ".inst 0x449a4461 // smlalt z1.s, p4/M, z3.h, z26.h\n" + ".inst 0x448843a6 // smlalb z6.s, p4/M, z29.h, z8.h\n" + "ldr x20, [x17, #0x68]\n" + "ld1b { z2.h }, p4/Z, [x4, #6, MUL VL]\n" + ".inst 0x44884069 // smlalb z9.s, p4/M, z3.h, z8.h\n" + ".inst 0x44884087 // smlalb z7.s, p4/M, z4.h, z8.h\n" + ".inst 0x455e1ad6 // usublb z22.h, z22.b, z30.b\n" + "ld1b { z26.h }, p3/Z, [x20, x2]\n" + ".inst 0x449047b7 // smlalt z23.s, p4/M, z29.h, z16.h\n" + ".inst 0x449543ee // smlalb z14.s, p4/M, z31.h, z21.h\n" + ".inst 0x454a1842 // usublb z2.h, z2.b, z10.b\n" + "ldr x20, [x17, #0x70]\n" + ".inst 0x448847b2 // smlalt z18.s, p4/M, z29.h, z8.h\n" + ".inst 0x44884474 // smlalt z20.s, p4/M, z3.h, z8.h\n" + "ld1b { z29.h }, p4/Z, [x4, #7, MUL VL]\n" + ".inst 0x455e1b5a // usublb z26.h, z26.b, z30.b\n" + ".inst 0x44884481 // smlalt z1.s, p4/M, z4.h, z8.h\n" + ".inst 0x449043e6 // smlalb z6.s, p4/M, z31.h, z16.h\n" + "inch x4, ALL, MUL #8\n" + "ld1b { z8.h }, p3/Z, [x20, x2]\n" + ".inst 0x44904089 // smlalb z9.s, p4/M, z4.h, z16.h\n" + ".inst 0x44904367 // smlalb z7.s, p4/M, z27.h, z16.h\n" + ".inst 0x454a1bbd // usublb z29.h, z29.b, z10.b\n" + "ldr x20, [x17, #0x78]\n" + ".inst 0x449547f7 // smlalt z23.s, p4/M, z31.h, z21.h\n" + ".inst 0x4491400e // smlalb z14.s, p4/M, z0.h, z17.h\n" + "ld1b { z24.h }, p4/Z, [x4]\n" + ".inst 0x455e1908 // usublb z8.h, z8.b, z30.b\n" + ".inst 0x449047f2 // smlalt z18.s, p4/M, z31.h, z16.h\n" + ".inst 0x44904494 // smlalt z20.s, p4/M, z4.h, z16.h\n" + "ld1b { z31.h }, p3/Z, [x20, x2]\n" + ".inst 0x454a1b18 // usublb z24.h, z24.b, z10.b\n" + ".inst 0x44904761 // smlalt z1.s, p4/M, z27.h, z16.h\n" + ".inst 0x44954006 // smlalb z6.s, p4/M, z0.h, z21.h\n" + "ldr x22, [x17, #0x80]\n" + "ld1b { z16.h }, p4/Z, [x4, #1, MUL VL]\n" + ".inst 0x44954369 // smlalb z9.s, p4/M, z27.h, z21.h\n" + ".inst 0x449540a7 // smlalb z7.s, p4/M, z5.h, z21.h\n" + ".inst 0x455e1bff // usublb z31.h, z31.b, z30.b\n" + "ldr x21, [x17, #0x88]\n" + ".inst 0x44914417 // smlalt z23.s, p4/M, z0.h, z17.h\n" + ".inst 0x4499416e // smlalb z14.s, p4/M, z11.h, z25.h\n" + ".inst 0x454a1a10 // usublb z16.h, z16.b, z10.b\n" + "ldr x20, [x17, #0x90]\n" + ".inst 0x44954412 // smlalt z18.s, p4/M, z0.h, z21.h\n" + ".inst 0x44954774 // smlalt z20.s, p4/M, z27.h, z21.h\n" + "ld1b { z0.h }, p3/Z, [x22, x2]\n" + ".inst 0x455e1800 // usublb z0.h, z0.b, z30.b\n" + ".inst 0x449544a1 // smlalt z1.s, p4/M, z5.h, z21.h\n" + ".inst 0x449142c6 // smlalb z6.s, p4/M, z22.h, z17.h\n" + "ld1b { z21.h }, p4/Z, [x4, #2, MUL VL]\n" + ".inst 0x454a1ab5 // usublb z21.h, z21.b, z10.b\n" + ".inst 0x449140a9 // smlalb z9.s, p4/M, z5.h, z17.h\n" + ".inst 0x44914267 // smlalb z7.s, p4/M, z19.h, z17.h\n" + "ldr x23, [x17, #0x98]\n" + "ldr x22, [x17, #0xa0]\n" + ".inst 0x44994577 // smlalt z23.s, p4/M, z11.h, z25.h\n" + ".inst 0x4482406e // smlalb z14.s, p4/M, z3.h, z2.h\n" + "ld1b { z11.h }, p3/Z, [x21, x2]\n" + ".inst 0x455e196b // usublb z11.h, z11.b, z30.b\n" + ".inst 0x449146d2 // smlalt z18.s, p4/M, z22.h, z17.h\n" + ".inst 0x449144b4 // smlalt z20.s, p4/M, z5.h, z17.h\n" + "ld1b { z22.h }, p4/Z, [x4, #3, MUL VL]\n" + ".inst 0x454a1ad6 // usublb z22.h, z22.b, z10.b\n" + ".inst 0x44914661 // smlalt z1.s, p4/M, z19.h, z17.h\n" + ".inst 0x44994066 // smlalb z6.s, p4/M, z3.h, z25.h\n" + "ld1b { z17.h }, p3/Z, [x20, x2]\n" + ".inst 0x455e1a31 // usublb z17.h, z17.b, z30.b\n" + ".inst 0x44994389 // smlalb z9.s, p4/M, z28.h, z25.h\n" + ".inst 0x44994347 // smlalb z7.s, p4/M, z26.h, z25.h\n" + "ldr x20, [x17, #0xa8]\n" + "ldr x21, [x17, #0xb0]\n" + ".inst 0x44824477 // smlalt z23.s, p4/M, z3.h, z2.h\n" + ".inst 0x449d408e // smlalb z14.s, p4/M, z4.h, z29.h\n" + "ldr x13, [x17, #0xb8]\n" + "ldr x12, [x17, #0xc0]\n" + ".inst 0x44994472 // smlalt z18.s, p4/M, z3.h, z25.h\n" + ".inst 0x44994794 // smlalt z20.s, p4/M, z28.h, z25.h\n" + "ld1b { z3.h }, p3/Z, [x23, x2]\n" + ".inst 0x455e1863 // usublb z3.h, z3.b, z30.b\n" + ".inst 0x44994741 // smlalt z1.s, p4/M, z26.h, z25.h\n" + ".inst 0x44824086 // smlalb z6.s, p4/M, z4.h, z2.h\n" + "ld1b { z25.h }, p4/Z, [x4, #4, MUL VL]\n" + ".inst 0x454a1b39 // usublb z25.h, z25.b, z10.b\n" + ".inst 0x44824349 // smlalb z9.s, p4/M, z26.h, z2.h\n" + ".inst 0x44824107 // smlalb z7.s, p4/M, z8.h, z2.h\n" + "ldr x11, [x17, #0xc8]\n" + "ldr x10, [x17, #0xd0]\n" + ".inst 0x449d4497 // smlalt z23.s, p4/M, z4.h, z29.h\n" + ".inst 0x4498436e // smlalb z14.s, p4/M, z27.h, z24.h\n" + "ldr x9, [x17, #0xd8]\n" + "ldr x28, [x17, #0xe0]\n" + ".inst 0x44824492 // smlalt z18.s, p4/M, z4.h, z2.h\n" + ".inst 0x44824754 // smlalt z20.s, p4/M, z26.h, z2.h\n" + "ld1b { z4.h }, p3/Z, [x22, x2]\n" + ".inst 0x455e1884 // usublb z4.h, z4.b, z30.b\n" + ".inst 0x44824501 // smlalt z1.s, p4/M, z8.h, z2.h\n" + ".inst 0x449d4366 // smlalb z6.s, p4/M, z27.h, z29.h\n" + "ld1b { z2.h }, p4/Z, [x4, #5, MUL VL]\n" + ".inst 0x454a1842 // usublb z2.h, z2.b, z10.b\n" + ".inst 0x449d4109 // smlalb z9.s, p4/M, z8.h, z29.h\n" + ".inst 0x449d43e7 // smlalb z7.s, p4/M, z31.h, z29.h\n" + "ldr x27, [x17, #0xe8]\n" + "ldr x26, [x17, #0xf0]\n" + ".inst 0x44984777 // smlalt z23.s, p4/M, z27.h, z24.h\n" + ".inst 0x449040ae // smlalb z14.s, p4/M, z5.h, z16.h\n" + "ldr x25, [x17, #0xf8]\n" + "ldr x24, [x17, #0x100]\n" + ".inst 0x449d4772 // smlalt z18.s, p4/M, z27.h, z29.h\n" + ".inst 0x449d4514 // smlalt z20.s, p4/M, z8.h, z29.h\n" + "ld1b { z27.h }, p3/Z, [x20, x2]\n" + ".inst 0x455e1b7b // usublb z27.h, z27.b, z30.b\n" + ".inst 0x449d47e1 // smlalt z1.s, p4/M, z31.h, z29.h\n" + ".inst 0x449840a6 // smlalb z6.s, p4/M, z5.h, z24.h\n" + "ld1b { z29.h }, p4/Z, [x4, #6, MUL VL]\n" + ".inst 0x454a1bbd // usublb z29.h, z29.b, z10.b\n" + ".inst 0x449843e9 // smlalb z9.s, p4/M, z31.h, z24.h\n" + ".inst 0x44984007 // smlalb z7.s, p4/M, z0.h, z24.h\n" + "ldr x23, [x17, #0x108]\n" + "ldr x22, [x17, #0x110]\n" + ".inst 0x449044b7 // smlalt z23.s, p4/M, z5.h, z16.h\n" + ".inst 0x4495438e // smlalb z14.s, p4/M, z28.h, z21.h\n" + "ldr x20, [x17, #0x118]\n" + "whilelt p0.h, x16, x3\n" + ".inst 0x449844b2 // smlalt z18.s, p4/M, z5.h, z24.h\n" + ".inst 0x449847f4 // smlalt z20.s, p4/M, z31.h, z24.h\n" + "ld1b { z5.h }, p3/Z, [x21, x2]\n" + ".inst 0x455e18a5 // usublb z5.h, z5.b, z30.b\n" + ".inst 0x44984401 // smlalt z1.s, p4/M, z0.h, z24.h\n" + ".inst 0x44904266 // smlalb z6.s, p4/M, z19.h, z16.h\n" + "ld1b { z24.h }, p4/Z, [x4, #7, MUL VL]\n" + "inch x4, ALL, MUL #8\n" + ".inst 0x44904009 // smlalb z9.s, p4/M, z0.h, z16.h\n" + ".inst 0x44904167 // smlalb z7.s, p4/M, z11.h, z16.h\n" + ".inst 0x454a1b18 // usublb z24.h, z24.b, z10.b\n" + "ldr x21, [%x[params], %[offsetof_Params_bias]]\n" + ".inst 0x44954797 // smlalt z23.s, p4/M, z28.h, z21.h\n" + ".inst 0x4496434e // smlalb z14.s, p4/M, z26.h, z22.h\n" + "ld1b { z28.h }, p3/Z, [x13, x2]\n" + ".inst 0x455e1b9c // usublb z28.h, z28.b, z30.b\n" + ".inst 0x44904672 // smlalt z18.s, p4/M, z19.h, z16.h\n" + ".inst 0x44904414 // smlalt z20.s, p4/M, z0.h, z16.h\n" + "ld1b { z19.h }, p4/Z, [x4]\n" + ".inst 0x454a1a73 // usublb z19.h, z19.b, z10.b\n" + ".inst 0x44904561 // smlalt z1.s, p4/M, z11.h, z16.h\n" + ".inst 0x44954346 // smlalb z6.s, p4/M, z26.h, z21.h\n" + "ld1b { z16.h }, p3/Z, [x12, x2]\n" + ".inst 0x455e1a10 // usublb z16.h, z16.b, z30.b\n" + ".inst 0x44954229 // smlalb z9.s, p4/M, z17.h, z21.h\n" + ".inst 0x44954067 // smlalb z7.s, p4/M, z3.h, z21.h\n" + ".inst 0x44964757 // smlalt z23.s, p4/M, z26.h, z22.h\n" + ".inst 0x4499410e // smlalb z14.s, p4/M, z8.h, z25.h\n" + ".inst 0x44954752 // smlalt z18.s, p4/M, z26.h, z21.h\n" + ".inst 0x44954634 // smlalt z20.s, p4/M, z17.h, z21.h\n" + "ld1b { z26.h }, p3/Z, [x11, x2]\n" + ".inst 0x455e1b5a // usublb z26.h, z26.b, z30.b\n" + ".inst 0x44954461 // smlalt z1.s, p4/M, z3.h, z21.h\n" + ".inst 0x44964106 // smlalb z6.s, p4/M, z8.h, z22.h\n" + "ld1b { z21.h }, p4/Z, [x4, #1, MUL VL]\n" + ".inst 0x454a1ab5 // usublb z21.h, z21.b, z10.b\n" + ".inst 0x44964069 // smlalb z9.s, p4/M, z3.h, z22.h\n" + ".inst 0x44964087 // smlalb z7.s, p4/M, z4.h, z22.h\n" + ".inst 0x44994517 // smlalt z23.s, p4/M, z8.h, z25.h\n" ".inst 0x448243ee // smlalb z14.s, p4/M, z31.h, z2.h\n" - ".inst 0x448046e7 // smlalt z7.s, p4/M, z23.h, z0.h\n" - "uzp1 z23.s, z19.s, z18.s\n" - ".inst 0x44804705 // smlalt z5.s, p4/M, z24.h, z0.h\n" - ".inst 0x448143f4 // smlalb z20.s, p4/M, z31.h, z1.h\n" - "uzp2 z22.s, z19.s, z18.s\n" - ".inst 0x44814308 // smlalb z8.s, p4/M, z24.h, z1.h\n" - ".inst 0x44814366 // smlalb z6.s, p4/M, z27.h, z1.h\n" - ".inst 0x448247ea // smlalt z10.s, p4/M, z31.h, z2.h\n" - ".inst 0x44814710 // smlalt z16.s, p4/M, z24.h, z1.h\n" - "ld1b { z24.h }, p3/Z, [x21, x0]\n" - ".inst 0x454f1b18 // usublb z24.h, z24.b, z15.b\n" - ".inst 0x448343ce // smlalb z14.s, p4/M, z30.h, z3.h\n" - ".inst 0x448147e7 // smlalt z7.s, p4/M, z31.h, z1.h\n" - ".inst 0x44814765 // smlalt z5.s, p4/M, z27.h, z1.h\n" - ".inst 0x448243d4 // smlalb z20.s, p4/M, z30.h, z2.h\n" - ".inst 0x44824368 // smlalb z8.s, p4/M, z27.h, z2.h\n" - ".inst 0x44824326 // smlalb z6.s, p4/M, z25.h, z2.h\n" - ".inst 0x448347ca // smlalt z10.s, p4/M, z30.h, z3.h\n" - ".inst 0x44824770 // smlalt z16.s, p4/M, z27.h, z2.h\n" - "ld1b { z27.h }, p3/Z, [x20, x0]\n" - ".inst 0x454f1b7b // usublb z27.h, z27.b, z15.b\n" - ".inst 0x4484438e // smlalb z14.s, p4/M, z28.h, z4.h\n" - ".inst 0x448247c7 // smlalt z7.s, p4/M, z30.h, z2.h\n" - ".inst 0x04a975ce // sqrdmulh z14.s, z14.s, z9.s\n" - "inch x0\n" - ".inst 0x44824725 // smlalt z5.s, p4/M, z25.h, z2.h\n" - ".inst 0x44834394 // smlalb z20.s, p4/M, z28.h, z3.h\n" - "and z21.d, z14.d, z23.d\n" - "mov x20, x0\n" - ".inst 0x44834328 // smlalb z8.s, p4/M, z25.h, z3.h\n" - ".inst 0x44834306 // smlalb z6.s, p4/M, z24.h, z3.h\n" - "asr z21.s, z21.s, #0x1f\n" + ".inst 0x44964512 // smlalt z18.s, p4/M, z8.h, z22.h\n" + ".inst 0x44964474 // smlalt z20.s, p4/M, z3.h, z22.h\n" + "ld1b { z8.h }, p3/Z, [x10, x2]\n" + ".inst 0x455e1908 // usublb z8.h, z8.b, z30.b\n" + ".inst 0x44964481 // smlalt z1.s, p4/M, z4.h, z22.h\n" + ".inst 0x449943e6 // smlalb z6.s, p4/M, z31.h, z25.h\n" + "ld1b { z22.h }, p4/Z, [x4, #2, MUL VL]\n" + ".inst 0x454a1ad6 // usublb z22.h, z22.b, z10.b\n" + ".inst 0x44994089 // smlalb z9.s, p4/M, z4.h, z25.h\n" + ".inst 0x44994367 // smlalb z7.s, p4/M, z27.h, z25.h\n" + ".inst 0x448247f7 // smlalt z23.s, p4/M, z31.h, z2.h\n" + ".inst 0x449d400e // smlalb z14.s, p4/M, z0.h, z29.h\n" + ".inst 0x449947f2 // smlalt z18.s, p4/M, z31.h, z25.h\n" + ".inst 0x44994494 // smlalt z20.s, p4/M, z4.h, z25.h\n" + "ld1b { z31.h }, p3/Z, [x9, x2]\n" + ".inst 0x455e1bff // usublb z31.h, z31.b, z30.b\n" + ".inst 0x44994761 // smlalt z1.s, p4/M, z27.h, z25.h\n" + ".inst 0x44824006 // smlalb z6.s, p4/M, z0.h, z2.h\n" + "ld1b { z25.h }, p4/Z, [x4, #3, MUL VL]\n" + ".inst 0x454a1b39 // usublb z25.h, z25.b, z10.b\n" + ".inst 0x44824369 // smlalb z9.s, p4/M, z27.h, z2.h\n" + ".inst 0x448240a7 // smlalb z7.s, p4/M, z5.h, z2.h\n" + ".inst 0x449d4417 // smlalt z23.s, p4/M, z0.h, z29.h\n" + ".inst 0x4498422e // smlalb z14.s, p4/M, z17.h, z24.h\n" + ".inst 0x44824412 // smlalt z18.s, p4/M, z0.h, z2.h\n" + ".inst 0x44824774 // smlalt z20.s, p4/M, z27.h, z2.h\n" + "ld1b { z0.h }, p3/Z, [x28, x2]\n" + ".inst 0x455e1800 // usublb z0.h, z0.b, z30.b\n" + ".inst 0x448244a1 // smlalt z1.s, p4/M, z5.h, z2.h\n" + ".inst 0x449d4166 // smlalb z6.s, p4/M, z11.h, z29.h\n" + "ld1b { z2.h }, p4/Z, [x4, #4, MUL VL]\n" + ".inst 0x454a1842 // usublb z2.h, z2.b, z10.b\n" + ".inst 0x449d40a9 // smlalb z9.s, p4/M, z5.h, z29.h\n" + ".inst 0x449d4387 // smlalb z7.s, p4/M, z28.h, z29.h\n" + ".inst 0x44984637 // smlalt z23.s, p4/M, z17.h, z24.h\n" + ".inst 0x4493406e // smlalb z14.s, p4/M, z3.h, z19.h\n" + "ld1b { z17.h }, p3/Z, [x27, x2]\n" + ".inst 0x455e1a31 // usublb z17.h, z17.b, z30.b\n" + ".inst 0x449d4572 // smlalt z18.s, p4/M, z11.h, z29.h\n" + ".inst 0x449d44b4 // smlalt z20.s, p4/M, z5.h, z29.h\n" + "ld1b { z11.h }, p4/Z, [x4, #5, MUL VL]\n" + ".inst 0x454a196b // usublb z11.h, z11.b, z10.b\n" + ".inst 0x449d4781 // smlalt z1.s, p4/M, z28.h, z29.h\n" + ".inst 0x44984066 // smlalb z6.s, p4/M, z3.h, z24.h\n" + "ld1b { z29.h }, p3/Z, [x26, x2]\n" + ".inst 0x455e1bbd // usublb z29.h, z29.b, z30.b\n" + ".inst 0x44984209 // smlalb z9.s, p4/M, z16.h, z24.h\n" + ".inst 0x44984347 // smlalb z7.s, p4/M, z26.h, z24.h\n" + ".inst 0x44934477 // smlalt z23.s, p4/M, z3.h, z19.h\n" + ".inst 0x4495408e // smlalb z14.s, p4/M, z4.h, z21.h\n" + ".inst 0x44984472 // smlalt z18.s, p4/M, z3.h, z24.h\n" + ".inst 0x44984614 // smlalt z20.s, p4/M, z16.h, z24.h\n" + "ld1b { z3.h }, p3/Z, [x25, x2]\n" + ".inst 0x455e1863 // usublb z3.h, z3.b, z30.b\n" + ".inst 0x44984741 // smlalt z1.s, p4/M, z26.h, z24.h\n" + ".inst 0x44934086 // smlalb z6.s, p4/M, z4.h, z19.h\n" + "ld1b { z24.h }, p4/Z, [x4, #6, MUL VL]\n" + ".inst 0x454a1b18 // usublb z24.h, z24.b, z10.b\n" + ".inst 0x44934349 // smlalb z9.s, p4/M, z26.h, z19.h\n" + ".inst 0x44934107 // smlalb z7.s, p4/M, z8.h, z19.h\n" + ".inst 0x44954497 // smlalt z23.s, p4/M, z4.h, z21.h\n" + ".inst 0x4496436e // smlalb z14.s, p4/M, z27.h, z22.h\n" + ".inst 0x44934492 // smlalt z18.s, p4/M, z4.h, z19.h\n" + ".inst 0x44934754 // smlalt z20.s, p4/M, z26.h, z19.h\n" + "ld1b { z4.h }, p3/Z, [x24, x2]\n" + ".inst 0x455e1884 // usublb z4.h, z4.b, z30.b\n" + ".inst 0x44934501 // smlalt z1.s, p4/M, z8.h, z19.h\n" + ".inst 0x44954366 // smlalb z6.s, p4/M, z27.h, z21.h\n" + "ld1b { z19.h }, p4/Z, [x4, #7, MUL VL]\n" + "inch x4, ALL, MUL #8\n" + ".inst 0x44954109 // smlalb z9.s, p4/M, z8.h, z21.h\n" + ".inst 0x449543e7 // smlalb z7.s, p4/M, z31.h, z21.h\n" + ".inst 0x454a1a73 // usublb z19.h, z19.b, z10.b\n" + ".inst 0x44964777 // smlalt z23.s, p4/M, z27.h, z22.h\n" + ".inst 0x449940ae // smlalb z14.s, p4/M, z5.h, z25.h\n" + ".inst 0x44954772 // smlalt z18.s, p4/M, z27.h, z21.h\n" + ".inst 0x44954514 // smlalt z20.s, p4/M, z8.h, z21.h\n" + "ld1b { z27.h }, p3/Z, [x23, x2]\n" + ".inst 0x455e1b7b // usublb z27.h, z27.b, z30.b\n" + ".inst 0x449547e1 // smlalt z1.s, p4/M, z31.h, z21.h\n" + ".inst 0x449640a6 // smlalb z6.s, p4/M, z5.h, z22.h\n" + "ld1b { z21.h }, p4/Z, [x4]\n" + ".inst 0x454a1ab5 // usublb z21.h, z21.b, z10.b\n" + ".inst 0x449643e9 // smlalb z9.s, p4/M, z31.h, z22.h\n" + ".inst 0x44964007 // smlalb z7.s, p4/M, z0.h, z22.h\n" + "inch x4\n" + ".inst 0x449944b7 // smlalt z23.s, p4/M, z5.h, z25.h\n" + ".inst 0x4482420e // smlalb z14.s, p4/M, z16.h, z2.h\n" + ".inst 0x449644b2 // smlalt z18.s, p4/M, z5.h, z22.h\n" + ".inst 0x449647f4 // smlalt z20.s, p4/M, z31.h, z22.h\n" + "ld1b { z5.h }, p3/Z, [x22, x2]\n" + ".inst 0x455e18a5 // usublb z5.h, z5.b, z30.b\n" + ".inst 0x44964401 // smlalt z1.s, p4/M, z0.h, z22.h\n" + ".inst 0x44994386 // smlalb z6.s, p4/M, z28.h, z25.h\n" + "ld1w { z22.s }, p2/Z, [x15]\n" + ".inst 0x44994009 // smlalb z9.s, p4/M, z0.h, z25.h\n" + ".inst 0x44994227 // smlalb z7.s, p4/M, z17.h, z25.h\n" + ".inst 0x44824617 // smlalt z23.s, p4/M, z16.h, z2.h\n" + ".inst 0x448b434e // smlalb z14.s, p4/M, z26.h, z11.h\n" + "ld1w { z16.s }, p1/Z, [x15, #1, MUL VL]\n" + "addvl x15, x15, #2\n" + ".inst 0x44994792 // smlalt z18.s, p4/M, z28.h, z25.h\n" + ".inst 0x44994414 // smlalt z20.s, p4/M, z0.h, z25.h\n" + "ld1b { z28.h }, p3/Z, [x20, x2]\n" + ".inst 0x455e1b9c // usublb z28.h, z28.b, z30.b\n" + ".inst 0x44994621 // smlalt z1.s, p4/M, z17.h, z25.h\n" + ".inst 0x44824346 // smlalb z6.s, p4/M, z26.h, z2.h\n" + "uzp1 z25.s, z22.s, z16.s\n" + "inch x2\n" + ".inst 0x448243a9 // smlalb z9.s, p4/M, z29.h, z2.h\n" + ".inst 0x44824067 // smlalb z7.s, p4/M, z3.h, z2.h\n" + "uzp2 z16.s, z22.s, z16.s\n" + "ld1w { z22.s }, p2/Z, [x14]\n" + ".inst 0x448b4757 // smlalt z23.s, p4/M, z26.h, z11.h\n" + ".inst 0x4498410e // smlalb z14.s, p4/M, z8.h, z24.h\n" + "mov x20, x2\n" "incw x20\n" - ".inst 0x4484478a // smlalt z10.s, p4/M, z28.h, z4.h\n" - ".inst 0x44834787 // smlalt z7.s, p4/M, z28.h, z3.h\n" - ".inst 0x04bd754a // sqrdmulh z10.s, z10.s, z29.s\n" - "whilelt p2.s, x0, x1\n" - ".inst 0x44834730 // smlalt z16.s, p4/M, z25.h, z3.h\n" - ".inst 0x44834705 // smlalt z5.s, p4/M, z24.h, z3.h\n" - "and z3.d, z10.d, z22.d\n" - "whilelt p1.s, x20, x1\n" - ".inst 0x44844354 // smlalb z20.s, p4/M, z26.h, z4.h\n" - ".inst 0x44844308 // smlalb z8.s, p4/M, z24.h, z4.h\n" - ".inst 0x04a97694 // sqrdmulh z20.s, z20.s, z9.s\n" - "whilelt p3.h, x0, x1\n" - ".inst 0x44844366 // smlalb z6.s, p4/M, z27.h, z4.h\n" - ".inst 0x44844747 // smlalt z7.s, p4/M, z26.h, z4.h\n" - ".inst 0x04a97508 // sqrdmulh z8.s, z8.s, z9.s\n" - ".inst 0x44844710 // smlalt z16.s, p4/M, z24.h, z4.h\n" - ".inst 0x44844765 // smlalt z5.s, p4/M, z27.h, z4.h\n" - ".inst 0x04a974c6 // sqrdmulh z6.s, z6.s, z9.s\n" - "sqadd z14.s, z14.s, z21.s\n" + ".inst 0x44824752 // smlalt z18.s, p4/M, z26.h, z2.h\n" + ".inst 0x448247b4 // smlalt z20.s, p4/M, z29.h, z2.h\n" + "ld1w { z26.s }, p1/Z, [x14, #1, MUL VL]\n" + "uzp1 z29.s, z22.s, z26.s\n" + ".inst 0x44824461 // smlalt z1.s, p4/M, z3.h, z2.h\n" + ".inst 0x448b4106 // smlalb z6.s, p4/M, z8.h, z11.h\n" + "uzp2 z22.s, z22.s, z26.s\n" + "whilelt p2.s, x2, x3\n" + ".inst 0x448b4069 // smlalb z9.s, p4/M, z3.h, z11.h\n" + ".inst 0x448b4087 // smlalb z7.s, p4/M, z4.h, z11.h\n" + "whilelt p1.s, x20, x3\n" + "whilelt p3.h, x2, x3\n" + ".inst 0x44984517 // smlalt z23.s, p4/M, z8.h, z24.h\n" + ".inst 0x449343ee // smlalb z14.s, p4/M, z31.h, z19.h\n" + "addvl x14, x14, #2\n" + ".inst 0x448b4512 // smlalt z18.s, p4/M, z8.h, z11.h\n" + ".inst 0x448b4474 // smlalt z20.s, p4/M, z3.h, z11.h\n" + ".inst 0x448b4481 // smlalt z1.s, p4/M, z4.h, z11.h\n" + ".inst 0x449843e6 // smlalb z6.s, p4/M, z31.h, z24.h\n" + ".inst 0x44984089 // smlalb z9.s, p4/M, z4.h, z24.h\n" + ".inst 0x44984367 // smlalb z7.s, p4/M, z27.h, z24.h\n" + ".inst 0x449347f7 // smlalt z23.s, p4/M, z31.h, z19.h\n" + ".inst 0x4495400e // smlalb z14.s, p4/M, z0.h, z21.h\n" + ".inst 0x04b975ce // sqrdmulh z14.s, z14.s, z25.s\n" + ".inst 0x449847f2 // smlalt z18.s, p4/M, z31.h, z24.h\n" + ".inst 0x44984494 // smlalt z20.s, p4/M, z4.h, z24.h\n" + "and z3.d, z14.d, z29.d\n" + ".inst 0x44984761 // smlalt z1.s, p4/M, z27.h, z24.h\n" + ".inst 0x44934006 // smlalb z6.s, p4/M, z0.h, z19.h\n" + "asr z3.s, z3.s, #0x1f\n" + ".inst 0x44934369 // smlalb z9.s, p4/M, z27.h, z19.h\n" + ".inst 0x449340a7 // smlalb z7.s, p4/M, z5.h, z19.h\n" + "sqadd z14.s, z14.s, z3.s\n" + ".inst 0x448293ae // srshl z14.s, p4/M, z14.s, z29.s\n" + ".inst 0x44954417 // smlalt z23.s, p4/M, z0.h, z21.h\n" + ".inst 0x44934412 // smlalt z18.s, p4/M, z0.h, z19.h\n" + ".inst 0x04b076f7 // sqrdmulh z23.s, z23.s, z16.s\n" + ".inst 0x44934774 // smlalt z20.s, p4/M, z27.h, z19.h\n" + ".inst 0x449344a1 // smlalt z1.s, p4/M, z5.h, z19.h\n" + "and z31.d, z23.d, z22.d\n" + ".inst 0x44954226 // smlalb z6.s, p4/M, z17.h, z21.h\n" + ".inst 0x449540a9 // smlalb z9.s, p4/M, z5.h, z21.h\n" + ".inst 0x04b974c6 // sqrdmulh z6.s, z6.s, z25.s\n" + ".inst 0x44954387 // smlalb z7.s, p4/M, z28.h, z21.h\n" + ".inst 0x44954632 // smlalt z18.s, p4/M, z17.h, z21.h\n" + ".inst 0x04b97529 // sqrdmulh z9.s, z9.s, z25.s\n" + ".inst 0x449544b4 // smlalt z20.s, p4/M, z5.h, z21.h\n" + ".inst 0x44954781 // smlalt z1.s, p4/M, z28.h, z21.h\n" + ".inst 0x04b974e7 // sqrdmulh z7.s, z7.s, z25.s\n" + "asr z31.s, z31.s, #0x1f\n" + "and z3.d, z6.d, z29.d\n" + ".inst 0x04b07652 // sqrdmulh z18.s, z18.s, z16.s\n" + "and z0.d, z9.d, z29.d\n" + ".inst 0x04b07694 // sqrdmulh z20.s, z20.s, z16.s\n" + "and z19.d, z7.d, z29.d\n" + ".inst 0x04b07421 // sqrdmulh z1.s, z1.s, z16.s\n" + "sqadd z23.s, z23.s, z31.s\n" + ".inst 0x448292d7 // srshl z23.s, p4/M, z23.s, z22.s\n" "asr z3.s, z3.s, #0x1f\n" - ".inst 0x448292ee // srshl z14.s, p4/M, z14.s, z23.s\n" - "and z19.d, z20.d, z23.d\n" - ".inst 0x04bd74e7 // sqrdmulh z7.s, z7.s, z29.s\n" - "and z18.d, z8.d, z23.d\n" - ".inst 0x04bd7610 // sqrdmulh z16.s, z16.s, z29.s\n" - "and z21.d, z6.d, z23.d\n" - ".inst 0x04bd74a5 // sqrdmulh z5.s, z5.s, z29.s\n" - "sqadd z10.s, z10.s, z3.s\n" + "and z21.d, z18.d, z22.d\n" + "asr z0.s, z0.s, #0x1f\n" + "and z17.d, z20.d, z22.d\n" "asr z19.s, z19.s, #0x1f\n" - ".inst 0x448292ca // srshl z10.s, p4/M, z10.s, z22.s\n" - "and z1.d, z7.d, z22.d\n" - "asr z18.s, z18.s, #0x1f\n" - "and z2.d, z16.d, z22.d\n" + "and z16.d, z1.d, z22.d\n" + "sqadd z6.s, z6.s, z3.s\n" "asr z21.s, z21.s, #0x1f\n" - "and z3.d, z5.d, z22.d\n" - "sqadd z20.s, z20.s, z19.s\n" - ".inst 0x448292f4 // srshl z20.s, p4/M, z20.s, z23.s\n" - "asr z1.s, z1.s, #0x1f\n" - "sqadd z8.s, z8.s, z18.s\n" - ".inst 0x448292e8 // srshl z8.s, p4/M, z8.s, z23.s\n" - "asr z2.s, z2.s, #0x1f\n" - "sqadd z6.s, z6.s, z21.s\n" - ".inst 0x448292e6 // srshl z6.s, p4/M, z6.s, z23.s\n" - "asr z3.s, z3.s, #0x1f\n" - "sqadd z7.s, z7.s, z1.s\n" - ".inst 0x448292c7 // srshl z7.s, p4/M, z7.s, z22.s\n" - "sqadd z16.s, z16.s, z2.s\n" - "sqadd z5.s, z5.s, z3.s\n" - ".inst 0x448292d0 // srshl z16.s, p4/M, z16.s, z22.s\n" - ".inst 0x448292c5 // srshl z5.s, p4/M, z5.s, z22.s\n" + ".inst 0x448293a6 // srshl z6.s, p4/M, z6.s, z29.s\n" + "sqadd z9.s, z9.s, z0.s\n" + "asr z17.s, z17.s, #0x1f\n" + ".inst 0x448293a9 // srshl z9.s, p4/M, z9.s, z29.s\n" + "sqadd z7.s, z7.s, z19.s\n" + "asr z16.s, z16.s, #0x1f\n" + ".inst 0x448293a7 // srshl z7.s, p4/M, z7.s, z29.s\n" + "sqadd z18.s, z18.s, z21.s\n" + "sqadd z20.s, z20.s, z17.s\n" + ".inst 0x448292d2 // srshl z18.s, p4/M, z18.s, z22.s\n" + ".inst 0x448292d4 // srshl z20.s, p4/M, z20.s, z22.s\n" + "sqadd z1.s, z1.s, z16.s\n" ".inst 0x453041ce // sqxtnb z14.h, z14.s\n" - ".inst 0x45304294 // sqxtnb z20.h, z20.s\n" - ".inst 0x45304108 // sqxtnb z8.h, z8.s\n" + ".inst 0x448292c1 // srshl z1.s, p4/M, z1.s, z22.s\n" ".inst 0x453040c6 // sqxtnb z6.h, z6.s\n" - ".inst 0x4530454e // sqxtnt z14.h, z10.s\n" - ".inst 0x453044f4 // sqxtnt z20.h, z7.s\n" - ".inst 0x45304608 // sqxtnt z8.h, z16.s\n" - ".inst 0x453044a6 // sqxtnt z6.h, z5.s\n" - "sqadd z14.h, z14.h, z12.h\n" - "sqadd z20.h, z20.h, z12.h\n" - "smax z14.h, p4/M, z14.h, z13.h\n" - "smax z20.h, p4/M, z20.h, z13.h\n" - "sqadd z8.h, z8.h, z12.h\n" - "sqadd z6.h, z6.h, z12.h\n" - "smax z8.h, p4/M, z8.h, z13.h\n" - "smax z6.h, p4/M, z6.h, z13.h\n" - "smin z14.h, p4/M, z14.h, z11.h\n" - "smin z20.h, p4/M, z20.h, z11.h\n" - "st1b { z14.h }, p0, [x3, x8]\n" - "smin z8.h, p4/M, z8.h, z11.h\n" - "smin z6.h, p4/M, z6.h, z11.h\n" - "st1b { z20.h }, p0, [x4, x8]\n" - "st1b { z8.h }, p0, [x5, x8]\n" - "st1b { z6.h }, p0, [x6, x8]\n" - "ld1w { z30.s }, p2/Z, [x14]\n" - "ld1w { z16.s }, p1/Z, [x14, #1, MUL VL]\n" - "uzp1 z14.s, z30.s, z16.s\n" - "ld1b { z0.h }, p4/Z, [x2]\n" - "ld1b { z1.h }, p4/Z, [x2, #1, MUL VL]\n" - "uzp2 z10.s, z30.s, z16.s\n" - "addvl x14, x14, #2\n" - "ld1b { z2.h }, p4/Z, [x2, #2, MUL VL]\n" - "ld1b { z3.h }, p4/Z, [x2, #3, MUL VL]\n" - "inch x8\n" - "str x14, [%x[params], %[offsetof_Params_bias]]\n" - "ld1b { z4.h }, p4/Z, [x2, #4, MUL VL]\n" - "ldp x9, x28, [x7, #0x0]\n" - "mov z20.d, z14.d\n" - "mov z7.d, z10.d\n" - "ldp x27, x26, [x7, #0x10]\n" - "ldp x25, x24, [x7, #0x20]\n" - "mov z8.d, z14.d\n" - "mov z16.d, z10.d\n" - "ldp x23, x22, [x7, #0x30]\n" - "ldp x21, x20, [x7, #0x40]\n" + ".inst 0x45304129 // sqxtnb z9.h, z9.s\n" + ".inst 0x453040e7 // sqxtnb z7.h, z7.s\n" + ".inst 0x453046ee // sqxtnt z14.h, z23.s\n" + ".inst 0x45304646 // sqxtnt z6.h, z18.s\n" + ".inst 0x45304689 // sqxtnt z9.h, z20.s\n" + ".inst 0x45304427 // sqxtnt z7.h, z1.s\n" + "sqadd z14.h, z14.h, z15.h\n" + "smax z14.h, p4/M, z14.h, z12.h\n" + "smin z14.h, p4/M, z14.h, z13.h\n" + "sqadd z6.h, z6.h, z15.h\n" + "sqadd z9.h, z9.h, z15.h\n" + "smax z6.h, p4/M, z6.h, z12.h\n" + "smax z9.h, p4/M, z9.h, z12.h\n" + "sqadd z7.h, z7.h, z15.h\n" + "smax z7.h, p4/M, z7.h, z12.h\n" + "smin z6.h, p4/M, z6.h, z13.h\n" + "st1b { z14.h }, p0, [x5, x16]\n" + "smin z9.h, p4/M, z9.h, z13.h\n" + "smin z7.h, p4/M, z7.h, z13.h\n" + "st1b { z6.h }, p0, [x6, x16]\n" + "st1b { z9.h }, p0, [x7, x16]\n" + "st1b { z7.h }, p0, [x8, x16]\n" + "ld1w { z17.s }, p2/Z, [x21]\n" + "ld1w { z16.s }, p1/Z, [x21, #1, MUL VL]\n" + "uzp1 z14.s, z17.s, z16.s\n" + "ld1b { z26.h }, p4/Z, [x4]\n" + "ld1b { z8.h }, p4/Z, [x4, #1, MUL VL]\n" + "uzp2 z23.s, z17.s, z16.s\n" + "addvl x21, x21, #2\n" + "ld1b { z16.h }, p4/Z, [x4, #2, MUL VL]\n" + "ld1b { z21.h }, p4/Z, [x4, #3, MUL VL]\n" + "inch x16\n" + "str x21, [%x[params], %[offsetof_Params_bias]]\n" + "ld1b { z17.h }, p4/Z, [x4, #4, MUL VL]\n" + "ldp x9, x28, [x17, #0x0]\n" "mov z6.d, z14.d\n" - "mov z5.d, z10.d\n" - "ld1b { z31.h }, p3/Z, [x9, x0]\n" - "ld1b { z30.h }, p3/Z, [x28, x0]\n" - ".inst 0x45511800 // usublb z0.h, z0.b, z17.b\n" - ".inst 0x45511821 // usublb z1.h, z1.b, z17.b\n" - "ld1b { z29.h }, p3/Z, [x27, x0]\n" - "ld1b { z28.h }, p3/Z, [x26, x0]\n" - ".inst 0x45511842 // usublb z2.h, z2.b, z17.b\n" - ".inst 0x45511863 // usublb z3.h, z3.b, z17.b\n" - "ld1b { z27.h }, p3/Z, [x25, x0]\n" - "ld1b { z23.h }, p3/Z, [x24, x0]\n" - ".inst 0x45511884 // usublb z4.h, z4.b, z17.b\n" - ".inst 0x454f1bff // usublb z31.h, z31.b, z15.b\n" - "ld1b { z25.h }, p3/Z, [x23, x0]\n" - "ld1b { z24.h }, p3/Z, [x22, x0]\n" - ".inst 0x454f1bde // usublb z30.h, z30.b, z15.b\n" - ".inst 0x454f1bbd // usublb z29.h, z29.b, z15.b\n" - "ld1b { z26.h }, p3/Z, [x21, x0]\n" - "ld1b { z22.h }, p3/Z, [x20, x0]\n" - ".inst 0x454f1b9c // usublb z28.h, z28.b, z15.b\n" - ".inst 0x454f1b7b // usublb z27.h, z27.b, z15.b\n" - ".inst 0x454f1af7 // usublb z23.h, z23.b, z15.b\n" - ".inst 0x454f1b39 // usublb z25.h, z25.b, z15.b\n" - ".inst 0x454f1b18 // usublb z24.h, z24.b, z15.b\n" - ".inst 0x454f1b5a // usublb z26.h, z26.b, z15.b\n" - ".inst 0x454f1ad6 // usublb z22.h, z22.b, z15.b\n" + "mov z18.d, z23.d\n" + "ldp x27, x26, [x17, #0x10]\n" + "ldp x25, x24, [x17, #0x20]\n" + "mov z9.d, z14.d\n" + "mov z20.d, z23.d\n" + "ldp x23, x22, [x17, #0x30]\n" + "ldp x21, x20, [x17, #0x40]\n" + "mov z7.d, z14.d\n" + "mov z1.d, z23.d\n" + "ld1b { z22.h }, p3/Z, [x9, x2]\n" + "ld1b { z2.h }, p3/Z, [x28, x2]\n" + ".inst 0x454a1b5a // usublb z26.h, z26.b, z10.b\n" + ".inst 0x454a1908 // usublb z8.h, z8.b, z10.b\n" + "ld1b { z11.h }, p3/Z, [x27, x2]\n" + "ld1b { z3.h }, p3/Z, [x26, x2]\n" + ".inst 0x454a1a10 // usublb z16.h, z16.b, z10.b\n" + ".inst 0x454a1ab5 // usublb z21.h, z21.b, z10.b\n" + "ld1b { z29.h }, p3/Z, [x25, x2]\n" + "ld1b { z4.h }, p3/Z, [x24, x2]\n" + ".inst 0x454a1a31 // usublb z17.h, z17.b, z10.b\n" + ".inst 0x455e1ad6 // usublb z22.h, z22.b, z30.b\n" + "ld1b { z31.h }, p3/Z, [x23, x2]\n" + "ld1b { z0.h }, p3/Z, [x22, x2]\n" + ".inst 0x455e1842 // usublb z2.h, z2.b, z30.b\n" + ".inst 0x455e196b // usublb z11.h, z11.b, z30.b\n" + "ld1b { z19.h }, p3/Z, [x21, x2]\n" + "ld1b { z28.h }, p3/Z, [x20, x2]\n" + ".inst 0x455e1863 // usublb z3.h, z3.b, z30.b\n" + ".inst 0x455e1bbd // usublb z29.h, z29.b, z30.b\n" + ".inst 0x455e1884 // usublb z4.h, z4.b, z30.b\n" + ".inst 0x455e1bff // usublb z31.h, z31.b, z30.b\n" + ".inst 0x455e1800 // usublb z0.h, z0.b, z30.b\n" + ".inst 0x455e1a73 // usublb z19.h, z19.b, z30.b\n" + ".inst 0x455e1b9c // usublb z28.h, z28.b, z30.b\n" "b.any 1b\n" : : [offsetof_Params_bias] "I" (offsetof(Params, bias)), [offsetof_Params_inptrs] "I" (offsetof(Params, inptrs)), [offsetof_Params_n_channels] "I" (offsetof(Params, n_channels)), [offsetof_Params_outptrs] "I" (offsetof(Params, outptrs)), [offsetof_Params_requant] "I" (offsetof(Params, requant)), [offsetof_Params_requant_muls] "I" (offsetof(Params, requant_muls)), [offsetof_Params_requant_shifts] "I" (offsetof(Params, requant_shifts)), [offsetof_Params_weights] "I" (offsetof(Params, weights)), [offsetof_Requantize32_a_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [params] "r" (¶ms) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst.hpp index 81c954a11bde35377781b475357d67a7f9318004..d5382533a848cda45e46846ff822a819d62ded45 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst/generic.cpp index 66c24c34b51b9ce3797c9ecb782fd48ccad96dad..a9cd8a7fa984c64ea4a65983f0b0235fe321bed0 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst/generic.cpp @@ -47,285 +47,285 @@ void sve_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst_impl "ldr x20, [%x[inptrs], #0x10]\n" "ldr x22, [%x[inptrs], #0x20]\n" "ldr x21, [%x[inptrs], #0x0]\n" - "mov z15.b, #0x1\n" - "lsr z15.s, z15.s, #0x8\n" + "mov z13.b, #0x1\n" + "lsr z13.s, z13.s, #0x8\n" "ld1b { z1.b }, p0/Z, [x23]\n" "ld1b { z2.b }, p0/Z, [x20]\n" - "mov z30.d, z1.d\n" - "mov z29.d, z1.d\n" + "mov z8.d, z1.d\n" + "mov z27.d, z1.d\n" "ldr x20, [%x[inptrs], #0x18]\n" "ld1b { z4.b }, p0/Z, [x22]\n" - "mov z28.d, z1.d\n" - "mov z27.d, z2.d\n" + "mov z31.d, z1.d\n" + "mov z28.d, z2.d\n" "ld1b { z0.b }, p0/Z, [x21]\n" + "mov z30.d, z2.d\n" "mov z26.d, z2.d\n" - "mov z25.d, z2.d\n" "ld1b { z3.b }, p0/Z, [x20]\n" - "mov z24.d, z4.d\n" - "mov z23.d, z4.d\n" - "ptrue p2.b\n" - "ld1rw { z14.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" "mov z22.d, z4.d\n" - "ext z30.b, z30.b, z30.b, #0x2\n" + "mov z10.d, z4.d\n" + "ptrue p2.b\n" + "ld1rw { z11.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" + "mov z18.d, z4.d\n" + "ext z8.b, z8.b, z8.b, #0x2\n" "lsl x10, %x[n_channels], #0x2\n" - "neg z14.s, p2/M, z14.s\n" - "ext z29.b, z29.b, z29.b, #0x4\n" - "ext z28.b, z28.b, z28.b, #0x6\n" + "neg z11.s, p2/M, z11.s\n" + "ext z27.b, z27.b, z27.b, #0x4\n" + "ext z31.b, z31.b, z31.b, #0x6\n" "mov x9, #0x0\n" - "whilelt p1.b, x9, x10\n" - "ext z27.b, z27.b, z27.b, #0x2\n" - "ext z26.b, z26.b, z26.b, #0x4\n" - "ld1w { z13.s }, p1/Z, [%x[params]]\n" + "whilelt p0.b, x9, x10\n" + "ext z28.b, z28.b, z28.b, #0x2\n" + "ext z30.b, z30.b, z30.b, #0x4\n" + "ld1w { z14.s }, p0/Z, [%x[params]]\n" "mov x28, #0x0\n" - "ext z25.b, z25.b, z25.b, #0x6\n" - "ext z24.b, z24.b, z24.b, #0x2\n" + "ext z26.b, z26.b, z26.b, #0x6\n" + "ext z22.b, z22.b, z22.b, #0x2\n" "ldp x27, x26, [%x[outptrs], #0x0]\n" "ldp x25, x24, [%x[outptrs], #0x10]\n" - "ext z23.b, z23.b, z23.b, #0x4\n" - "ext z22.b, z22.b, z22.b, #0x6\n" + "ext z10.b, z10.b, z10.b, #0x4\n" + "ext z18.b, z18.b, z18.b, #0x6\n" "ldp x23, x22, [%x[outptrs], #0x20]\n" "ldp x21, x20, [%x[outptrs], #0x30]\n" "mov z21.d, z0.d\n" "mov z20.d, z0.d\n" - "ld1rw { z12.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" - "ld1rw { z11.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" + "ld1rw { z9.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" + "ld1rw { z15.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" "mov z19.d, z0.d\n" - "mov z18.d, z3.d\n" - "ld1rw { z10.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" - "ld1b { z5.b }, p1/Z, [%x[params], #1, MUL VL]\n" + "mov z24.d, z3.d\n" + "ld1rw { z12.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" + "ld1b { z5.b }, p0/Z, [%x[params], #1, MUL VL]\n" "mov z17.d, z3.d\n" "mov z16.d, z3.d\n" - "ld1b { z6.b }, p1/Z, [%x[params], #2, MUL VL]\n" - "ld1b { z7.b }, p1/Z, [%x[params], #3, MUL VL]\n" + "ld1b { z6.b }, p0/Z, [%x[params], #2, MUL VL]\n" + "ld1b { z7.b }, p0/Z, [%x[params], #3, MUL VL]\n" "ext z21.b, z21.b, z21.b, #0x2\n" "ext z20.b, z20.b, z20.b, #0x4\n" "addvl %x[params], %x[params], #4\n" "ext z19.b, z19.b, z19.b, #0x6\n" - "zip1 z1.s, z1.s, z29.s\n" - "zip1 z30.s, z30.s, z28.s\n" - "zip1 z2.s, z2.s, z26.s\n" - "zip1 z27.s, z27.s, z25.s\n" - "ext z18.b, z18.b, z18.b, #0x2\n" + "zip1 z1.s, z1.s, z27.s\n" + "zip1 z8.s, z8.s, z31.s\n" + "zip1 z2.s, z2.s, z30.s\n" + "zip1 z28.s, z28.s, z26.s\n" + "ext z24.b, z24.b, z24.b, #0x2\n" "ext z17.b, z17.b, z17.b, #0x4\n" "ext z16.b, z16.b, z16.b, #0x6\n" - "zip1 z4.s, z4.s, z23.s\n" - "zip1 z24.s, z24.s, z22.s\n" + "zip1 z4.s, z4.s, z10.s\n" + "zip1 z22.s, z22.s, z18.s\n" "zip1 z0.s, z0.s, z20.s\n" "zip1 z21.s, z21.s, z19.s\n" - "zip1 z1.s, z1.s, z30.s\n" - "zip1 z2.s, z2.s, z27.s\n" + "zip1 z1.s, z1.s, z8.s\n" + "zip1 z2.s, z2.s, z28.s\n" "zip1 z3.s, z3.s, z17.s\n" - "zip1 z18.s, z18.s, z16.s\n" - "zip1 z4.s, z4.s, z24.s\n" + "zip1 z24.s, z24.s, z16.s\n" + "zip1 z4.s, z4.s, z22.s\n" "zip1 z0.s, z0.s, z21.s\n" "mov z1.q, z1.q[0]\n" "mov z2.q, z2.q[0]\n" - "zip1 z3.s, z3.s, z18.s\n" + "zip1 z3.s, z3.s, z24.s\n" "mov z4.q, z4.q[0]\n" "mov z24.s, #0x0\n" "mov z25.s, #0x0\n" - "udot z24.s, z15.b, z1.b[0]\n" + "udot z24.s, z13.b, z1.b[0]\n" "mov z23.s, #0x0\n" "mov z22.s, #0x0\n" - "udot z25.s, z15.b, z1.b[1]\n" + "udot z25.s, z13.b, z1.b[1]\n" "mov z21.s, #0x0\n" - "mov z20.s, #0x0\n" - "udot z23.s, z15.b, z1.b[2]\n" - "mov z9.s, #0x0\n" - "mov z8.s, #0x0\n" - "udot z22.s, z15.b, z1.b[3]\n" "mov z19.s, #0x0\n" + "udot z23.s, z13.b, z1.b[2]\n" + "mov z10.s, #0x0\n" + "mov z8.s, #0x0\n" + "udot z22.s, z13.b, z1.b[3]\n" + "mov z20.s, #0x0\n" "mov z18.s, #0x0\n" - "udot z21.s, z15.b, z2.b[0]\n" + "udot z21.s, z13.b, z2.b[0]\n" "mov z17.s, #0x0\n" "mov z16.s, #0x0\n" - "udot z20.s, z15.b, z2.b[1]\n" - "udot z9.s, z15.b, z2.b[2]\n" - "udot z8.s, z15.b, z2.b[3]\n" + "udot z19.s, z13.b, z2.b[1]\n" + "udot z10.s, z13.b, z2.b[2]\n" + "udot z8.s, z13.b, z2.b[3]\n" "mov z0.q, z0.q[0]\n" - "udot z19.s, z15.b, z4.b[0]\n" - "udot z18.s, z15.b, z4.b[1]\n" + "udot z20.s, z13.b, z4.b[0]\n" + "udot z18.s, z13.b, z4.b[1]\n" "mov z3.q, z3.q[0]\n" - "udot z17.s, z15.b, z4.b[2]\n" - "udot z16.s, z15.b, z4.b[3]\n" + "udot z17.s, z13.b, z4.b[2]\n" + "udot z16.s, z13.b, z4.b[3]\n" "mov z31.s, #0x0\n" "mov z30.s, #0x0\n" - "mov z29.s, #0x0\n" - "udot z31.s, z15.b, z0.b[0]\n" + "mov z26.s, #0x0\n" + "udot z31.s, z13.b, z0.b[0]\n" + "mov z27.s, #0x0\n" "mov z28.s, #0x0\n" - "udot z30.s, z15.b, z0.b[1]\n" - "udot z29.s, z15.b, z0.b[2]\n" - "udot z28.s, z15.b, z0.b[3]\n" + "udot z30.s, z13.b, z0.b[1]\n" + "mov z29.s, #0x0\n" + "udot z26.s, z13.b, z0.b[2]\n" + "udot z27.s, z13.b, z0.b[3]\n" + "udot z28.s, z13.b, z3.b[0]\n" + "udot z29.s, z13.b, z3.b[1]\n" "add z24.s, z24.s, z21.s\n" - "add z25.s, z25.s, z20.s\n" - "add z26.s, z23.s, z9.s\n" - "add z27.s, z22.s, z8.s\n" - "add z23.s, z19.s, z21.s\n" - "mov z22.s, #0x0\n" - "udot z22.s, z15.b, z3.b[0]\n" - "add z21.s, z18.s, z20.s\n" + "add z25.s, z25.s, z19.s\n" + "add z23.s, z23.s, z10.s\n" + "add z22.s, z22.s, z8.s\n" + "add z21.s, z20.s, z21.s\n" "mov z20.s, #0x0\n" - "udot z20.s, z15.b, z3.b[1]\n" - "add z19.s, z17.s, z9.s\n" + "udot z20.s, z13.b, z3.b[2]\n" + "add z19.s, z18.s, z19.s\n" "mov z18.s, #0x0\n" - "udot z18.s, z15.b, z3.b[2]\n" - "add z17.s, z16.s, z8.s\n" - "mov z16.s, #0x0\n" - "udot z16.s, z15.b, z3.b[3]\n" + "udot z18.s, z13.b, z3.b[3]\n" + "add z17.s, z17.s, z10.s\n" + "add z16.s, z16.s, z8.s\n" "add z24.s, z24.s, z31.s\n" "add z25.s, z25.s, z30.s\n" - "mul z24.s, p2/M, z24.s, z14.s\n" - "mul z25.s, p2/M, z25.s, z14.s\n" - "add z26.s, z26.s, z29.s\n" - "add z27.s, z27.s, z28.s\n" - "mul z26.s, p2/M, z26.s, z14.s\n" - "mul z27.s, p2/M, z27.s, z14.s\n" - "add z28.s, z23.s, z22.s\n" - "add z29.s, z21.s, z20.s\n" - "mul z28.s, p2/M, z28.s, z14.s\n" - "mul z29.s, p2/M, z29.s, z14.s\n" - "add z30.s, z19.s, z18.s\n" - "add z31.s, z17.s, z16.s\n" - "mul z30.s, p2/M, z30.s, z14.s\n" - "mul z31.s, p2/M, z31.s, z14.s\n" + "mul z24.s, p2/M, z24.s, z11.s\n" + "mul z25.s, p2/M, z25.s, z11.s\n" + "add z26.s, z23.s, z26.s\n" + "add z27.s, z22.s, z27.s\n" + "mul z26.s, p2/M, z26.s, z11.s\n" + "mul z27.s, p2/M, z27.s, z11.s\n" + "add z28.s, z21.s, z28.s\n" + "add z29.s, z19.s, z29.s\n" + "mul z28.s, p2/M, z28.s, z11.s\n" + "mul z29.s, p2/M, z29.s, z11.s\n" + "add z30.s, z17.s, z20.s\n" + "add z31.s, z16.s, z18.s\n" + "mul z30.s, p2/M, z30.s, z11.s\n" + "mul z31.s, p2/M, z31.s, z11.s\n" "zip1 z19.s, z24.s, z26.s\n" "zip1 z18.s, z25.s, z27.s\n" "zip1 z17.s, z28.s, z30.s\n" "zip1 z16.s, z29.s, z31.s\n" "zip1 z22.s, z19.s, z18.s\n" "zip1 z23.s, z17.s, z16.s\n" - "add z24.s, z24.s, z13.s\n" - "add z25.s, z25.s, z13.s\n" - "add z26.s, z26.s, z13.s\n" - "add z27.s, z27.s, z13.s\n" - "add z28.s, z28.s, z13.s\n" - "add z29.s, z29.s, z13.s\n" - "add z30.s, z30.s, z13.s\n" - "add z31.s, z31.s, z13.s\n" + "add z24.s, z24.s, z14.s\n" + "add z25.s, z25.s, z14.s\n" + "add z26.s, z26.s, z14.s\n" + "add z27.s, z27.s, z14.s\n" + "add z28.s, z28.s, z14.s\n" + "add z29.s, z29.s, z14.s\n" + "add z30.s, z30.s, z14.s\n" + "add z31.s, z31.s, z14.s\n" "1:" // Loop "udot z24.s, z5.b, z0.b[0]\n" "udot z25.s, z5.b, z0.b[1]\n" - "ld1w { z21.s }, p2/Z, [%x[params]]\n" - "ld1w { z20.s }, p2/Z, [%x[params], #1, MUL VL]\n" + "ld1w { z8.s }, p2/Z, [%x[params]]\n" + "ld1w { z21.s }, p2/Z, [%x[params], #1, MUL VL]\n" "udot z26.s, z5.b, z0.b[2]\n" "udot z27.s, z5.b, z0.b[3]\n" "incb x9\n" - "whilelt p0.s, x28, %x[n_channels]\n" + "whilelt p1.s, x28, %x[n_channels]\n" "udot z24.s, z6.b, z1.b[0]\n" "udot z25.s, z6.b, z1.b[1]\n" - "whilelt p1.b, x9, x10\n" - "ld1w { z13.s }, p1/Z, [%x[params], #2, MUL VL]\n" + "whilelt p0.b, x9, x10\n" + "ld1w { z20.s }, p0/Z, [%x[params], #2, MUL VL]\n" "udot z26.s, z6.b, z1.b[2]\n" "udot z27.s, z6.b, z1.b[3]\n" "udot z28.s, z5.b, z2.b[0]\n" "udot z29.s, z5.b, z2.b[1]\n" "udot z30.s, z5.b, z2.b[2]\n" "udot z31.s, z5.b, z2.b[3]\n" - "ld1b { z5.b }, p1/Z, [%x[params], #3, MUL VL]\n" + "ld1b { z5.b }, p0/Z, [%x[params], #3, MUL VL]\n" "udot z24.s, z7.b, z2.b[0]\n" "udot z25.s, z7.b, z2.b[1]\n" - ".inst 0x04b57718 // sqrdmulh z24.s, z24.s, z21.s\n" + ".inst 0x04a87718 // sqrdmulh z24.s, z24.s, z8.s\n" "udot z26.s, z7.b, z2.b[2]\n" "udot z27.s, z7.b, z2.b[3]\n" - ".inst 0x04b57739 // sqrdmulh z25.s, z25.s, z21.s\n" + ".inst 0x04a87739 // sqrdmulh z25.s, z25.s, z8.s\n" "udot z28.s, z6.b, z3.b[0]\n" "udot z29.s, z6.b, z3.b[1]\n" - ".inst 0x04b5775a // sqrdmulh z26.s, z26.s, z21.s\n" + ".inst 0x04a8775a // sqrdmulh z26.s, z26.s, z8.s\n" "udot z30.s, z6.b, z3.b[2]\n" "udot z31.s, z6.b, z3.b[3]\n" - ".inst 0x04b5777b // sqrdmulh z27.s, z27.s, z21.s\n" - "ld1b { z6.b }, p1/Z, [%x[params], #4, MUL VL]\n" + ".inst 0x04a8777b // sqrdmulh z27.s, z27.s, z8.s\n" + "ld1b { z6.b }, p0/Z, [%x[params], #4, MUL VL]\n" "udot z28.s, z7.b, z4.b[0]\n" "udot z29.s, z7.b, z4.b[1]\n" - "and z19.d, z24.d, z20.d\n" + "and z19.d, z24.d, z21.d\n" "udot z30.s, z7.b, z4.b[2]\n" "udot z31.s, z7.b, z4.b[3]\n" - "and z18.d, z25.d, z20.d\n" - "ld1b { z7.b }, p1/Z, [%x[params], #5, MUL VL]\n" - "and z17.d, z26.d, z20.d\n" - "and z16.d, z27.d, z20.d\n" + "and z18.d, z25.d, z21.d\n" + "ld1b { z7.b }, p0/Z, [%x[params], #5, MUL VL]\n" + "and z17.d, z26.d, z21.d\n" + "and z16.d, z27.d, z21.d\n" "addvl %x[params], %x[params], #6\n" "asr z19.s, z19.s, #0x1f\n" "asr z18.s, z18.s, #0x1f\n" "asr z17.s, z17.s, #0x1f\n" "asr z16.s, z16.s, #0x1f\n" - ".inst 0x04b5779c // sqrdmulh z28.s, z28.s, z21.s\n" - ".inst 0x04b577bd // sqrdmulh z29.s, z29.s, z21.s\n" - ".inst 0x04b577de // sqrdmulh z30.s, z30.s, z21.s\n" - ".inst 0x04b577ff // sqrdmulh z31.s, z31.s, z21.s\n" + ".inst 0x04a8779c // sqrdmulh z28.s, z28.s, z8.s\n" + ".inst 0x04a877bd // sqrdmulh z29.s, z29.s, z8.s\n" + ".inst 0x04a877de // sqrdmulh z30.s, z30.s, z8.s\n" + ".inst 0x04a877ff // sqrdmulh z31.s, z31.s, z8.s\n" "sqadd z24.s, z24.s, z19.s\n" "sqadd z25.s, z25.s, z18.s\n" - ".inst 0x44828a98 // srshl z24.s, p2/M, z24.s, z20.s\n" - ".inst 0x44828a99 // srshl z25.s, p2/M, z25.s, z20.s\n" + ".inst 0x44828ab8 // srshl z24.s, p2/M, z24.s, z21.s\n" + ".inst 0x44828ab9 // srshl z25.s, p2/M, z25.s, z21.s\n" "sqadd z26.s, z26.s, z17.s\n" "sqadd z27.s, z27.s, z16.s\n" - ".inst 0x44828a9a // srshl z26.s, p2/M, z26.s, z20.s\n" - ".inst 0x44828a9b // srshl z27.s, p2/M, z27.s, z20.s\n" - "and z19.d, z28.d, z20.d\n" - "and z18.d, z29.d, z20.d\n" - "and z17.d, z30.d, z20.d\n" - "and z16.d, z31.d, z20.d\n" + ".inst 0x44828aba // srshl z26.s, p2/M, z26.s, z21.s\n" + ".inst 0x44828abb // srshl z27.s, p2/M, z27.s, z21.s\n" + "and z19.d, z28.d, z21.d\n" + "and z18.d, z29.d, z21.d\n" + "and z17.d, z30.d, z21.d\n" + "and z16.d, z31.d, z21.d\n" "asr z19.s, z19.s, #0x1f\n" "asr z18.s, z18.s, #0x1f\n" "asr z17.s, z17.s, #0x1f\n" "asr z16.s, z16.s, #0x1f\n" "sqadd z28.s, z28.s, z19.s\n" "sqadd z29.s, z29.s, z18.s\n" - ".inst 0x44828a9c // srshl z28.s, p2/M, z28.s, z20.s\n" - ".inst 0x44828a9d // srshl z29.s, p2/M, z29.s, z20.s\n" + ".inst 0x44828abc // srshl z28.s, p2/M, z28.s, z21.s\n" + ".inst 0x44828abd // srshl z29.s, p2/M, z29.s, z21.s\n" "sqadd z30.s, z30.s, z17.s\n" "sqadd z31.s, z31.s, z16.s\n" - ".inst 0x44828a9e // srshl z30.s, p2/M, z30.s, z20.s\n" - ".inst 0x44828a9f // srshl z31.s, p2/M, z31.s, z20.s\n" - "add z24.s, z24.s, z12.s\n" - "add z25.s, z25.s, z12.s\n" - "smin z24.s, p2/M, z24.s, z10.s\n" - "smin z25.s, p2/M, z25.s, z10.s\n" - "add z26.s, z26.s, z12.s\n" - "add z27.s, z27.s, z12.s\n" - "smin z26.s, p2/M, z26.s, z10.s\n" - "smin z27.s, p2/M, z27.s, z10.s\n" - "add z28.s, z28.s, z12.s\n" - "add z29.s, z29.s, z12.s\n" - "smin z28.s, p2/M, z28.s, z10.s\n" - "smin z29.s, p2/M, z29.s, z10.s\n" - "add z30.s, z30.s, z12.s\n" - "add z31.s, z31.s, z12.s\n" - "smin z30.s, p2/M, z30.s, z10.s\n" - "smin z31.s, p2/M, z31.s, z10.s\n" - "smax z24.s, p2/M, z24.s, z11.s\n" - "smax z25.s, p2/M, z25.s, z11.s\n" - "st1b { z24.s }, p0, [x27, x28]\n" + ".inst 0x44828abe // srshl z30.s, p2/M, z30.s, z21.s\n" + ".inst 0x44828abf // srshl z31.s, p2/M, z31.s, z21.s\n" + "add z24.s, z24.s, z9.s\n" + "add z25.s, z25.s, z9.s\n" + "smin z24.s, p2/M, z24.s, z12.s\n" + "smin z25.s, p2/M, z25.s, z12.s\n" + "add z26.s, z26.s, z9.s\n" + "add z27.s, z27.s, z9.s\n" + "smin z26.s, p2/M, z26.s, z12.s\n" + "smin z27.s, p2/M, z27.s, z12.s\n" + "add z28.s, z28.s, z9.s\n" + "add z29.s, z29.s, z9.s\n" + "smin z28.s, p2/M, z28.s, z12.s\n" + "smin z29.s, p2/M, z29.s, z12.s\n" + "add z30.s, z30.s, z9.s\n" + "add z31.s, z31.s, z9.s\n" + "smin z30.s, p2/M, z30.s, z12.s\n" + "smin z31.s, p2/M, z31.s, z12.s\n" + "smax z24.s, p2/M, z24.s, z15.s\n" + "smax z25.s, p2/M, z25.s, z15.s\n" + "st1b { z24.s }, p1, [x27, x28]\n" "mov z24.s, z22.s[0]\n" - "smax z26.s, p2/M, z26.s, z11.s\n" - "smax z27.s, p2/M, z27.s, z11.s\n" - "st1b { z25.s }, p0, [x26, x28]\n" + "smax z26.s, p2/M, z26.s, z15.s\n" + "smax z27.s, p2/M, z27.s, z15.s\n" + "st1b { z25.s }, p1, [x26, x28]\n" "mov z25.s, z22.s[1]\n" - "smax z28.s, p2/M, z28.s, z11.s\n" - "smax z29.s, p2/M, z29.s, z11.s\n" - "st1b { z26.s }, p0, [x25, x28]\n" + "smax z28.s, p2/M, z28.s, z15.s\n" + "smax z29.s, p2/M, z29.s, z15.s\n" + "st1b { z26.s }, p1, [x25, x28]\n" "mov z26.s, z22.s[2]\n" - "smax z30.s, p2/M, z30.s, z11.s\n" - "smax z31.s, p2/M, z31.s, z11.s\n" - "st1b { z27.s }, p0, [x24, x28]\n" + "smax z30.s, p2/M, z30.s, z15.s\n" + "smax z31.s, p2/M, z31.s, z15.s\n" + "st1b { z27.s }, p1, [x24, x28]\n" "mov z27.s, z22.s[3]\n" - "st1b { z28.s }, p0, [x23, x28]\n" + "st1b { z28.s }, p1, [x23, x28]\n" "mov z28.s, z23.s[0]\n" - "add z24.s, z24.s, z13.s\n" - "st1b { z29.s }, p0, [x22, x28]\n" + "add z24.s, z24.s, z20.s\n" + "st1b { z29.s }, p1, [x22, x28]\n" "mov z29.s, z23.s[1]\n" - "add z25.s, z25.s, z13.s\n" - "st1b { z30.s }, p0, [x21, x28]\n" + "add z25.s, z25.s, z20.s\n" + "st1b { z30.s }, p1, [x21, x28]\n" "mov z30.s, z23.s[2]\n" - "add z26.s, z26.s, z13.s\n" - "st1b { z31.s }, p0, [x20, x28]\n" + "add z26.s, z26.s, z20.s\n" + "st1b { z31.s }, p1, [x20, x28]\n" "mov z31.s, z23.s[3]\n" "incw x28\n" - "add z27.s, z27.s, z13.s\n" - "add z28.s, z28.s, z13.s\n" - "add z29.s, z29.s, z13.s\n" - "add z30.s, z30.s, z13.s\n" - "add z31.s, z31.s, z13.s\n" + "add z27.s, z27.s, z20.s\n" + "add z28.s, z28.s, z20.s\n" + "add z29.s, z29.s, z20.s\n" + "add z30.s, z30.s, z20.s\n" + "add z31.s, z31.s, z20.s\n" "b.any 1b\n" : [params] "+&r" (params) : [inptrs] "r" (inptrs), [n_channels] "r" (n_output_channels), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [outptrs] "r" (outptrs), [qp] "r" (&qp) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst.hpp index e7173de65a17aa51d9081837ee633fd1c09fa086..55b6edea2cf95ba74f0f5a9a0a3c5b7faa9cff47 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst/generic.cpp index debaa8c2966d55d4ff9f61d93d0c8bd577c23674..4b65a67309d20a7e608aad249ea6c63677645344 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst/generic.cpp @@ -47,8 +47,8 @@ void sve_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl "ldr x21, [%x[inptrs], #0x20]\n" "ldr x20, [%x[inptrs], #0x10]\n" "ld1b { z3.b }, p0/Z, [x22]\n" - "mov z20.d, z3.d\n" - "ext z20.b, z20.b, z20.b, #0x1\n" + "mov z23.d, z3.d\n" + "ext z23.b, z23.b, z23.b, #0x1\n" "ld1b { z4.b }, p0/Z, [x21]\n" "ldr x24, [%x[inptrs], #0x8]\n" "mov z18.d, z4.d\n" @@ -59,132 +59,132 @@ void sve_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl "ext z15.b, z15.b, z15.b, #0x1\n" "ldr x22, [%x[inptrs], #0x30]\n" "ldr x21, [%x[inptrs], #0x38]\n" - "zip1 z3.d, z3.d, z20.d\n" + "zip1 z3.d, z3.d, z23.d\n" "zip1 z4.d, z4.d, z18.d\n" "ldr x20, [%x[inptrs], #0x0]\n" "ld1b { z1.b }, p0/Z, [x24]\n" - "mov z20.d, z1.d\n" - "ext z20.b, z20.b, z20.b, #0x1\n" + "mov z19.d, z1.d\n" + "ext z19.b, z19.b, z19.b, #0x1\n" "ld1b { z5.b }, p0/Z, [x23]\n" "ld1b { z6.b }, p0/Z, [x22]\n" - "mov z13.d, z5.d\n" - "mov z19.d, z6.d\n" + "mov z18.d, z5.d\n" + "mov z22.d, z6.d\n" "ld1b { z7.b }, p0/Z, [x21]\n" "ld1b { z0.b }, p0/Z, [x20]\n" - "mov z25.d, z7.d\n" + "mov z8.d, z7.d\n" "zip1 z2.d, z2.d, z15.d\n" "mov z3.q, z3.q[0]\n" "mov z4.q, z4.q[0]\n" "ptrue p2.b\n" "ld1rw { z23.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_b_offset]]\n" - "ext z13.b, z13.b, z13.b, #0x1\n" - "ext z19.b, z19.b, z19.b, #0x1\n" + "ext z18.b, z18.b, z18.b, #0x1\n" + "ext z22.b, z22.b, z22.b, #0x1\n" "lsl x10, %x[n_channels], #0x2\n" "neg z23.s, p2/M, z23.s\n" - "ext z25.b, z25.b, z25.b, #0x1\n" - "mov z30.b, #0x1\n" + "ext z8.b, z8.b, z8.b, #0x1\n" + "mov z28.b, #0x1\n" "mov x9, #0x0\n" - "whilelt p1.b, x9, x10\n" + "whilelt p0.b, x9, x10\n" + "mov z25.s, #0x0\n" "mov z24.s, #0x0\n" - "mov z28.s, #0x0\n" - "udot z24.s, z30.b, z3.b[0]\n" - "ld1w { z12.s }, p1/Z, [%x[params]]\n" - "mov z18.s, #0x0\n" + "udot z25.s, z28.b, z3.b[0]\n" + "ld1w { z12.s }, p0/Z, [%x[params]]\n" "mov z17.s, #0x0\n" - "udot z28.s, z30.b, z3.b[2]\n" + "mov z16.s, #0x0\n" + "udot z24.s, z28.b, z3.b[2]\n" "mov x28, #0x0\n" - "mov z16.d, z0.d\n" - "udot z18.s, z30.b, z4.b[0]\n" - "udot z17.s, z30.b, z4.b[2]\n" + "mov z27.d, z0.d\n" + "udot z17.s, z28.b, z4.b[0]\n" + "udot z16.s, z28.b, z4.b[2]\n" "ldp x27, x26, [%x[outptrs], #0x0]\n" - "ext z16.b, z16.b, z16.b, #0x1\n" - "zip1 z1.d, z1.d, z20.d\n" + "ext z27.b, z27.b, z27.b, #0x1\n" + "zip1 z1.d, z1.d, z19.d\n" "ldp x25, x24, [%x[outptrs], #0x10]\n" "ldp x23, x22, [%x[outptrs], #0x20]\n" "mov z2.q, z2.q[0]\n" - "zip1 z5.d, z5.d, z13.d\n" + "zip1 z5.d, z5.d, z18.d\n" "ldp x21, x20, [%x[outptrs], #0x30]\n" - "ld1rw { z14.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" - "zip1 z6.d, z6.d, z19.d\n" - "zip1 z7.d, z7.d, z25.d\n" - "ld1rw { z13.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" + "ld1rw { z13.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_c_offset]]\n" + "zip1 z6.d, z6.d, z22.d\n" + "zip1 z7.d, z7.d, z8.d\n" + "ld1rw { z14.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_minval]]\n" "ld1rw { z15.s }, p2/Z, [%x[qp], %[offsetof_Requantize32_maxval]]\n" - "mov z26.s, #0x0\n" - "mov z22.s, #0x0\n" - "udot z26.s, z30.b, z2.b[0]\n" - "ld1b { z8.b }, p1/Z, [%x[params], #1, MUL VL]\n" + "mov z30.s, #0x0\n" + "mov z31.s, #0x0\n" + "udot z30.s, z28.b, z2.b[0]\n" + "ld1b { z8.b }, p0/Z, [%x[params], #1, MUL VL]\n" "mov z29.s, #0x1\n" - "udot z22.s, z30.b, z2.b[2]\n" - "udot z24.s, z29.b, z3.b[1]\n" - "ld1b { z9.b }, p1/Z, [%x[params], #2, MUL VL]\n" - "zip1 z0.d, z0.d, z16.d\n" + "udot z31.s, z28.b, z2.b[2]\n" + "udot z25.s, z29.b, z3.b[1]\n" + "ld1b { z9.b }, p0/Z, [%x[params], #2, MUL VL]\n" + "zip1 z0.d, z0.d, z27.d\n" "mov z1.q, z1.q[0]\n" - "udot z28.s, z29.b, z3.b[3]\n" - "ld1b { z10.b }, p1/Z, [%x[params], #3, MUL VL]\n" + "udot z24.s, z29.b, z3.b[3]\n" + "ld1b { z10.b }, p0/Z, [%x[params], #3, MUL VL]\n" "mov z5.q, z5.q[0]\n" "mov z6.q, z6.q[0]\n" - "udot z18.s, z29.b, z4.b[1]\n" - "ld1b { z11.b }, p1/Z, [%x[params], #4, MUL VL]\n" + "udot z17.s, z29.b, z4.b[1]\n" + "ld1b { z11.b }, p0/Z, [%x[params], #4, MUL VL]\n" "mov z7.q, z7.q[0]\n" - "mov z21.s, #0x0\n" - "udot z17.s, z29.b, z4.b[3]\n" + "mov z22.s, #0x0\n" + "udot z16.s, z29.b, z4.b[3]\n" "addvl %x[params], %x[params], #5\n" - "mov z20.s, #0x0\n" - "mov z25.s, #0x0\n" - "udot z21.s, z30.b, z1.b[0]\n" + "mov z21.s, #0x0\n" + "mov z26.s, #0x0\n" + "udot z22.s, z28.b, z1.b[0]\n" "mov z27.s, #0x0\n" + "mov z20.s, #0x0\n" + "udot z21.s, z28.b, z1.b[2]\n" "mov z19.s, #0x0\n" - "udot z20.s, z30.b, z1.b[2]\n" - "udot z25.s, z30.b, z5.b[0]\n" - "udot z27.s, z30.b, z5.b[2]\n" - "mov z0.q, z0.q[0]\n" - "udot z19.s, z30.b, z6.b[0]\n" - "udot z26.s, z29.b, z2.b[1]\n" - "add z24.s, z24.s, z18.s\n" "mov z18.s, #0x0\n" - "udot z18.s, z30.b, z6.b[2]\n" - "udot z22.s, z29.b, z2.b[3]\n" - "add z17.s, z28.s, z17.s\n" - "mov z16.s, #0x0\n" - "udot z16.s, z30.b, z7.b[0]\n" - "udot z21.s, z29.b, z1.b[1]\n" - "udot z20.s, z29.b, z1.b[3]\n" - "add z28.s, z26.s, z24.s\n" - "udot z25.s, z29.b, z5.b[1]\n" + "udot z26.s, z28.b, z5.b[0]\n" + "udot z27.s, z28.b, z5.b[2]\n" + "udot z20.s, z28.b, z6.b[0]\n" + "mov z0.q, z0.q[0]\n" + "udot z19.s, z28.b, z6.b[2]\n" + "udot z18.s, z28.b, z7.b[0]\n" + "add z17.s, z25.s, z17.s\n" + "mov z25.s, #0x0\n" + "udot z25.s, z28.b, z7.b[2]\n" + "udot z30.s, z29.b, z2.b[1]\n" + "udot z31.s, z29.b, z2.b[3]\n" + "add z16.s, z24.s, z16.s\n" + "udot z22.s, z29.b, z1.b[1]\n" + "mov z24.s, #0x0\n" + "udot z24.s, z28.b, z0.b[0]\n" + "udot z21.s, z29.b, z1.b[3]\n" + "udot z26.s, z29.b, z5.b[1]\n" "udot z27.s, z29.b, z5.b[3]\n" - "add z31.s, z22.s, z17.s\n" - "udot z19.s, z29.b, z6.b[1]\n" - "udot z18.s, z29.b, z6.b[3]\n" - "add z22.s, z21.s, z28.s\n" - "udot z16.s, z29.b, z7.b[1]\n" - "add z21.s, z20.s, z31.s\n" - "add z20.s, z25.s, z19.s\n" - "add z19.s, z27.s, z18.s\n" - "add z18.s, z16.s, z24.s\n" - "mov z16.s, #0x0\n" - "udot z16.s, z30.b, z7.b[2]\n" - "udot z16.s, z29.b, z7.b[3]\n" - "add z17.s, z16.s, z17.s\n" - "mov z16.s, #0x0\n" - "udot z16.s, z30.b, z0.b[0]\n" - "udot z16.s, z29.b, z0.b[1]\n" - "add z24.s, z22.s, z16.s\n" - "add z26.s, z22.s, z25.s\n" + "add z30.s, z30.s, z17.s\n" + "udot z20.s, z29.b, z6.b[1]\n" + "udot z19.s, z29.b, z6.b[3]\n" + "add z31.s, z31.s, z16.s\n" + "udot z18.s, z29.b, z7.b[1]\n" + "udot z25.s, z29.b, z7.b[3]\n" + "add z22.s, z22.s, z30.s\n" + "udot z24.s, z29.b, z0.b[1]\n" + "add z21.s, z21.s, z31.s\n" + "add z20.s, z26.s, z20.s\n" + "add z19.s, z27.s, z19.s\n" + "add z18.s, z18.s, z17.s\n" + "mov z17.s, #0x0\n" + "udot z17.s, z28.b, z0.b[2]\n" + "udot z17.s, z29.b, z0.b[3]\n" + "add z16.s, z25.s, z16.s\n" + "add z24.s, z22.s, z24.s\n" + "add z25.s, z21.s, z17.s\n" "mul z24.s, p2/M, z24.s, z23.s\n" - "mul z26.s, p2/M, z26.s, z23.s\n" - "mov z16.s, #0x0\n" - "udot z16.s, z30.b, z0.b[2]\n" - "udot z16.s, z29.b, z0.b[3]\n" - "add z25.s, z21.s, z16.s\n" - "add z27.s, z21.s, z27.s\n" "mul z25.s, p2/M, z25.s, z23.s\n" + "add z26.s, z26.s, z22.s\n" + "add z27.s, z27.s, z21.s\n" + "mul z26.s, p2/M, z26.s, z23.s\n" "mul z27.s, p2/M, z27.s, z23.s\n" - "add z28.s, z20.s, z28.s\n" + "add z28.s, z20.s, z30.s\n" "add z29.s, z19.s, z31.s\n" "mul z28.s, p2/M, z28.s, z23.s\n" "mul z29.s, p2/M, z29.s, z23.s\n" - "add z30.s, z18.s, z20.s\n" - "add z31.s, z17.s, z19.s\n" + "add z30.s, z20.s, z18.s\n" + "add z31.s, z19.s, z16.s\n" "mul z30.s, p2/M, z30.s, z23.s\n" "mul z31.s, p2/M, z31.s, z23.s\n" "zip1 z19.s, z24.s, z26.s\n" @@ -204,22 +204,22 @@ void sve_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl "1:" // Loop "udot z24.s, z8.b, z0.b[0]\n" "udot z25.s, z8.b, z0.b[2]\n" - "ld1w { z17.s }, p2/Z, [%x[params], #6, MUL VL]\n" - "ld1w { z19.s }, p2/Z, [%x[params], #7, MUL VL]\n" + "ld1w { z12.s }, p2/Z, [%x[params], #6, MUL VL]\n" + "ld1w { z21.s }, p2/Z, [%x[params], #7, MUL VL]\n" "udot z26.s, z8.b, z1.b[0]\n" "udot z27.s, z8.b, z1.b[2]\n" "incb x9\n" - "whilelt p0.s, x28, %x[n_channels]\n" + "whilelt p1.s, x28, %x[n_channels]\n" "udot z24.s, z9.b, z0.b[1]\n" "udot z25.s, z9.b, z0.b[3]\n" - "whilelt p1.b, x9, x10\n" + "whilelt p0.b, x9, x10\n" "udot z26.s, z9.b, z1.b[1]\n" "udot z27.s, z9.b, z1.b[3]\n" "udot z28.s, z8.b, z2.b[0]\n" "udot z29.s, z8.b, z2.b[2]\n" "udot z30.s, z8.b, z3.b[0]\n" "udot z31.s, z8.b, z3.b[2]\n" - "ld1b { z8.b }, p2/Z, [%x[params]]\n" + "ld1b { z17.b }, p2/Z, [%x[params]]\n" "udot z24.s, z10.b, z1.b[0]\n" "udot z25.s, z10.b, z1.b[2]\n" "udot z26.s, z10.b, z2.b[0]\n" @@ -228,7 +228,7 @@ void sve_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl "udot z29.s, z9.b, z2.b[3]\n" "udot z30.s, z9.b, z3.b[1]\n" "udot z31.s, z9.b, z3.b[3]\n" - "ld1b { z9.b }, p2/Z, [%x[params], #1, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [%x[params], #1, MUL VL]\n" "udot z24.s, z11.b, z1.b[1]\n" "udot z25.s, z11.b, z1.b[3]\n" "udot z26.s, z11.b, z2.b[1]\n" @@ -237,158 +237,158 @@ void sve_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst_impl "udot z29.s, z10.b, z3.b[2]\n" "udot z30.s, z10.b, z4.b[0]\n" "udot z31.s, z10.b, z4.b[2]\n" - "ld1b { z10.b }, p2/Z, [%x[params], #2, MUL VL]\n" - "udot z24.s, z8.b, z2.b[0]\n" - "udot z25.s, z8.b, z2.b[2]\n" - "udot z26.s, z8.b, z3.b[0]\n" - "udot z27.s, z8.b, z3.b[2]\n" + "ld1b { z19.b }, p2/Z, [%x[params], #2, MUL VL]\n" + "udot z24.s, z17.b, z2.b[0]\n" + "udot z25.s, z17.b, z2.b[2]\n" + "udot z26.s, z17.b, z3.b[0]\n" + "udot z27.s, z17.b, z3.b[2]\n" "udot z28.s, z11.b, z3.b[1]\n" "udot z29.s, z11.b, z3.b[3]\n" "udot z30.s, z11.b, z4.b[1]\n" "udot z31.s, z11.b, z4.b[3]\n" - "ld1b { z11.b }, p2/Z, [%x[params], #3, MUL VL]\n" - "udot z24.s, z9.b, z2.b[1]\n" - "udot z25.s, z9.b, z2.b[3]\n" - "udot z26.s, z9.b, z3.b[1]\n" - "udot z27.s, z9.b, z3.b[3]\n" - "udot z28.s, z8.b, z4.b[0]\n" - "udot z29.s, z8.b, z4.b[2]\n" - "udot z30.s, z8.b, z5.b[0]\n" - "udot z31.s, z8.b, z5.b[2]\n" - "ld1b { z8.b }, p2/Z, [%x[params], #4, MUL VL]\n" - "udot z24.s, z10.b, z3.b[0]\n" - "udot z25.s, z10.b, z3.b[2]\n" - "udot z26.s, z10.b, z4.b[0]\n" - "udot z27.s, z10.b, z4.b[2]\n" - "udot z28.s, z9.b, z4.b[1]\n" - "udot z29.s, z9.b, z4.b[3]\n" - "udot z30.s, z9.b, z5.b[1]\n" - "udot z31.s, z9.b, z5.b[3]\n" - "ld1b { z9.b }, p2/Z, [%x[params], #5, MUL VL]\n" + "ld1b { z18.b }, p2/Z, [%x[params], #3, MUL VL]\n" + "udot z24.s, z16.b, z2.b[1]\n" + "udot z25.s, z16.b, z2.b[3]\n" + "udot z26.s, z16.b, z3.b[1]\n" + "udot z27.s, z16.b, z3.b[3]\n" + "udot z28.s, z17.b, z4.b[0]\n" + "udot z29.s, z17.b, z4.b[2]\n" + "udot z30.s, z17.b, z5.b[0]\n" + "udot z31.s, z17.b, z5.b[2]\n" + "ld1b { z17.b }, p2/Z, [%x[params], #4, MUL VL]\n" + "udot z24.s, z19.b, z3.b[0]\n" + "udot z25.s, z19.b, z3.b[2]\n" + "udot z26.s, z19.b, z4.b[0]\n" + "udot z27.s, z19.b, z4.b[2]\n" + "udot z28.s, z16.b, z4.b[1]\n" + "udot z29.s, z16.b, z4.b[3]\n" + "udot z30.s, z16.b, z5.b[1]\n" + "udot z31.s, z16.b, z5.b[3]\n" + "ld1b { z16.b }, p2/Z, [%x[params], #5, MUL VL]\n" "addvl %x[params], %x[params], #16\n" - "udot z24.s, z11.b, z3.b[1]\n" - "udot z25.s, z11.b, z3.b[3]\n" - "ld1w { z12.s }, p1/Z, [%x[params], #-8, MUL VL]\n" - "udot z26.s, z11.b, z4.b[1]\n" - "udot z27.s, z11.b, z4.b[3]\n" - "udot z28.s, z10.b, z5.b[0]\n" - "udot z29.s, z10.b, z5.b[2]\n" - "udot z30.s, z10.b, z6.b[0]\n" - "udot z31.s, z10.b, z6.b[2]\n" - "ld1b { z10.b }, p1/Z, [%x[params], #-5, MUL VL]\n" - "udot z24.s, z8.b, z4.b[0]\n" - "udot z25.s, z8.b, z4.b[2]\n" - "udot z26.s, z8.b, z5.b[0]\n" - "udot z27.s, z8.b, z5.b[2]\n" - "udot z28.s, z11.b, z5.b[1]\n" - "udot z29.s, z11.b, z5.b[3]\n" - "udot z30.s, z11.b, z6.b[1]\n" - "udot z31.s, z11.b, z6.b[3]\n" - "ld1b { z11.b }, p1/Z, [%x[params], #-4, MUL VL]\n" - "udot z24.s, z9.b, z4.b[1]\n" - "udot z25.s, z9.b, z4.b[3]\n" - ".inst 0x04b17718 // sqrdmulh z24.s, z24.s, z17.s\n" - "udot z26.s, z9.b, z5.b[1]\n" - "udot z27.s, z9.b, z5.b[3]\n" - ".inst 0x04b17739 // sqrdmulh z25.s, z25.s, z17.s\n" - "udot z28.s, z8.b, z6.b[0]\n" - "udot z29.s, z8.b, z6.b[2]\n" - ".inst 0x04b1775a // sqrdmulh z26.s, z26.s, z17.s\n" - "udot z30.s, z8.b, z7.b[0]\n" - "udot z31.s, z8.b, z7.b[2]\n" - ".inst 0x04b1777b // sqrdmulh z27.s, z27.s, z17.s\n" - "ld1b { z8.b }, p1/Z, [%x[params], #-7, MUL VL]\n" - "udot z28.s, z9.b, z6.b[1]\n" - "udot z29.s, z9.b, z6.b[3]\n" - "and z16.d, z24.d, z19.d\n" - "udot z30.s, z9.b, z7.b[1]\n" - "udot z31.s, z9.b, z7.b[3]\n" - "and z18.d, z25.d, z19.d\n" - "ld1b { z9.b }, p1/Z, [%x[params], #-6, MUL VL]\n" - "asr z16.s, z16.s, #0x1f\n" - "asr z18.s, z18.s, #0x1f\n" + "udot z24.s, z18.b, z3.b[1]\n" + "udot z25.s, z18.b, z3.b[3]\n" + "ld1w { z20.s }, p0/Z, [%x[params], #-8, MUL VL]\n" + "udot z26.s, z18.b, z4.b[1]\n" + "udot z27.s, z18.b, z4.b[3]\n" + "udot z28.s, z19.b, z5.b[0]\n" + "udot z29.s, z19.b, z5.b[2]\n" + "udot z30.s, z19.b, z6.b[0]\n" + "udot z31.s, z19.b, z6.b[2]\n" + "ld1b { z10.b }, p0/Z, [%x[params], #-5, MUL VL]\n" + "udot z24.s, z17.b, z4.b[0]\n" + "udot z25.s, z17.b, z4.b[2]\n" + "udot z26.s, z17.b, z5.b[0]\n" + "udot z27.s, z17.b, z5.b[2]\n" + "udot z28.s, z18.b, z5.b[1]\n" + "udot z29.s, z18.b, z5.b[3]\n" + "udot z30.s, z18.b, z6.b[1]\n" + "udot z31.s, z18.b, z6.b[3]\n" + "ld1b { z11.b }, p0/Z, [%x[params], #-4, MUL VL]\n" + "udot z24.s, z16.b, z4.b[1]\n" + "udot z25.s, z16.b, z4.b[3]\n" + ".inst 0x04ac7718 // sqrdmulh z24.s, z24.s, z12.s\n" + "udot z26.s, z16.b, z5.b[1]\n" + "udot z27.s, z16.b, z5.b[3]\n" + ".inst 0x04ac7739 // sqrdmulh z25.s, z25.s, z12.s\n" + "udot z28.s, z17.b, z6.b[0]\n" + "udot z29.s, z17.b, z6.b[2]\n" + ".inst 0x04ac775a // sqrdmulh z26.s, z26.s, z12.s\n" + "udot z30.s, z17.b, z7.b[0]\n" + "udot z31.s, z17.b, z7.b[2]\n" + ".inst 0x04ac777b // sqrdmulh z27.s, z27.s, z12.s\n" + "ld1b { z8.b }, p0/Z, [%x[params], #-7, MUL VL]\n" + "udot z28.s, z16.b, z6.b[1]\n" + "udot z29.s, z16.b, z6.b[3]\n" + "and z19.d, z24.d, z21.d\n" + "udot z30.s, z16.b, z7.b[1]\n" + "udot z31.s, z16.b, z7.b[3]\n" + "and z18.d, z25.d, z21.d\n" + "ld1b { z9.b }, p0/Z, [%x[params], #-6, MUL VL]\n" + "and z17.d, z26.d, z21.d\n" + "and z16.d, z27.d, z21.d\n" "addvl %x[params], %x[params], #-3\n" - ".inst 0x04b1779c // sqrdmulh z28.s, z28.s, z17.s\n" - ".inst 0x04b177bd // sqrdmulh z29.s, z29.s, z17.s\n" - ".inst 0x04b177de // sqrdmulh z30.s, z30.s, z17.s\n" - ".inst 0x04b177ff // sqrdmulh z31.s, z31.s, z17.s\n" - "and z17.d, z26.d, z19.d\n" + "asr z19.s, z19.s, #0x1f\n" + "asr z18.s, z18.s, #0x1f\n" "asr z17.s, z17.s, #0x1f\n" - "sqadd z24.s, z24.s, z16.s\n" - "and z16.d, z27.d, z19.d\n" - ".inst 0x44828a78 // srshl z24.s, p2/M, z24.s, z19.s\n" "asr z16.s, z16.s, #0x1f\n" + ".inst 0x04ac779c // sqrdmulh z28.s, z28.s, z12.s\n" + ".inst 0x04ac77bd // sqrdmulh z29.s, z29.s, z12.s\n" + ".inst 0x04ac77de // sqrdmulh z30.s, z30.s, z12.s\n" + ".inst 0x04ac77ff // sqrdmulh z31.s, z31.s, z12.s\n" + "sqadd z24.s, z24.s, z19.s\n" "sqadd z25.s, z25.s, z18.s\n" - ".inst 0x44828a79 // srshl z25.s, p2/M, z25.s, z19.s\n" + ".inst 0x44828ab8 // srshl z24.s, p2/M, z24.s, z21.s\n" + ".inst 0x44828ab9 // srshl z25.s, p2/M, z25.s, z21.s\n" "sqadd z26.s, z26.s, z17.s\n" "sqadd z27.s, z27.s, z16.s\n" - ".inst 0x44828a7a // srshl z26.s, p2/M, z26.s, z19.s\n" - ".inst 0x44828a7b // srshl z27.s, p2/M, z27.s, z19.s\n" - "and z16.d, z28.d, z19.d\n" - "and z18.d, z29.d, z19.d\n" - "and z17.d, z30.d, z19.d\n" - "asr z16.s, z16.s, #0x1f\n" + ".inst 0x44828aba // srshl z26.s, p2/M, z26.s, z21.s\n" + ".inst 0x44828abb // srshl z27.s, p2/M, z27.s, z21.s\n" + "and z19.d, z28.d, z21.d\n" + "and z18.d, z29.d, z21.d\n" + "and z17.d, z30.d, z21.d\n" + "and z16.d, z31.d, z21.d\n" + "asr z19.s, z19.s, #0x1f\n" "asr z18.s, z18.s, #0x1f\n" "asr z17.s, z17.s, #0x1f\n" - "sqadd z28.s, z28.s, z16.s\n" - "and z16.d, z31.d, z19.d\n" - ".inst 0x44828a7c // srshl z28.s, p2/M, z28.s, z19.s\n" "asr z16.s, z16.s, #0x1f\n" + "sqadd z28.s, z28.s, z19.s\n" "sqadd z29.s, z29.s, z18.s\n" - ".inst 0x44828a7d // srshl z29.s, p2/M, z29.s, z19.s\n" + ".inst 0x44828abc // srshl z28.s, p2/M, z28.s, z21.s\n" + ".inst 0x44828abd // srshl z29.s, p2/M, z29.s, z21.s\n" "sqadd z30.s, z30.s, z17.s\n" "sqadd z31.s, z31.s, z16.s\n" - ".inst 0x44828a7e // srshl z30.s, p2/M, z30.s, z19.s\n" - ".inst 0x44828a7f // srshl z31.s, p2/M, z31.s, z19.s\n" - "add z24.s, z24.s, z14.s\n" - "add z25.s, z25.s, z14.s\n" + ".inst 0x44828abe // srshl z30.s, p2/M, z30.s, z21.s\n" + ".inst 0x44828abf // srshl z31.s, p2/M, z31.s, z21.s\n" + "add z24.s, z24.s, z13.s\n" + "add z25.s, z25.s, z13.s\n" "smin z24.s, p2/M, z24.s, z15.s\n" "smin z25.s, p2/M, z25.s, z15.s\n" - "add z26.s, z26.s, z14.s\n" - "add z27.s, z27.s, z14.s\n" + "add z26.s, z26.s, z13.s\n" + "add z27.s, z27.s, z13.s\n" "smin z26.s, p2/M, z26.s, z15.s\n" "smin z27.s, p2/M, z27.s, z15.s\n" - "add z28.s, z28.s, z14.s\n" - "add z29.s, z29.s, z14.s\n" + "add z28.s, z28.s, z13.s\n" + "add z29.s, z29.s, z13.s\n" "smin z28.s, p2/M, z28.s, z15.s\n" "smin z29.s, p2/M, z29.s, z15.s\n" - "add z30.s, z30.s, z14.s\n" - "add z31.s, z31.s, z14.s\n" + "add z30.s, z30.s, z13.s\n" + "add z31.s, z31.s, z13.s\n" "smin z30.s, p2/M, z30.s, z15.s\n" "smin z31.s, p2/M, z31.s, z15.s\n" - "smax z24.s, p2/M, z24.s, z13.s\n" - "smax z25.s, p2/M, z25.s, z13.s\n" - "st1b { z24.s }, p0, [x27, x28]\n" + "smax z24.s, p2/M, z24.s, z14.s\n" + "smax z25.s, p2/M, z25.s, z14.s\n" + "st1b { z24.s }, p1, [x27, x28]\n" "mov z24.s, z22.s[0]\n" - "smax z26.s, p2/M, z26.s, z13.s\n" - "smax z27.s, p2/M, z27.s, z13.s\n" - "st1b { z25.s }, p0, [x26, x28]\n" + "smax z26.s, p2/M, z26.s, z14.s\n" + "smax z27.s, p2/M, z27.s, z14.s\n" + "st1b { z25.s }, p1, [x26, x28]\n" "mov z25.s, z22.s[1]\n" - "smax z28.s, p2/M, z28.s, z13.s\n" - "smax z29.s, p2/M, z29.s, z13.s\n" - "st1b { z26.s }, p0, [x25, x28]\n" + "smax z28.s, p2/M, z28.s, z14.s\n" + "smax z29.s, p2/M, z29.s, z14.s\n" + "st1b { z26.s }, p1, [x25, x28]\n" "mov z26.s, z22.s[2]\n" - "smax z30.s, p2/M, z30.s, z13.s\n" - "smax z31.s, p2/M, z31.s, z13.s\n" - "st1b { z27.s }, p0, [x24, x28]\n" + "smax z30.s, p2/M, z30.s, z14.s\n" + "smax z31.s, p2/M, z31.s, z14.s\n" + "st1b { z27.s }, p1, [x24, x28]\n" "mov z27.s, z22.s[3]\n" - "st1b { z28.s }, p0, [x23, x28]\n" + "st1b { z28.s }, p1, [x23, x28]\n" "mov z28.s, z23.s[0]\n" - "add z24.s, z24.s, z12.s\n" - "st1b { z29.s }, p0, [x22, x28]\n" + "add z24.s, z24.s, z20.s\n" + "st1b { z29.s }, p1, [x22, x28]\n" "mov z29.s, z23.s[1]\n" - "add z25.s, z25.s, z12.s\n" - "st1b { z30.s }, p0, [x21, x28]\n" + "add z25.s, z25.s, z20.s\n" + "st1b { z30.s }, p1, [x21, x28]\n" "mov z30.s, z23.s[2]\n" - "add z26.s, z26.s, z12.s\n" - "st1b { z31.s }, p0, [x20, x28]\n" + "add z26.s, z26.s, z20.s\n" + "st1b { z31.s }, p1, [x20, x28]\n" "mov z31.s, z23.s[3]\n" "incw x28\n" - "add z27.s, z27.s, z12.s\n" - "add z28.s, z28.s, z12.s\n" - "add z29.s, z29.s, z12.s\n" - "add z30.s, z30.s, z12.s\n" - "add z31.s, z31.s, z12.s\n" + "add z27.s, z27.s, z20.s\n" + "add z28.s, z28.s, z20.s\n" + "add z29.s, z29.s, z20.s\n" + "add z30.s, z30.s, z20.s\n" + "add z31.s, z31.s, z20.s\n" "b.any 1b\n" : [params] "+&r" (params) : [inptrs] "r" (inptrs), [n_channels] "r" (n_output_channels), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [outptrs] "r" (outptrs), [qp] "r" (&qp) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp index 3d475daf722a30d2c838c7e6c89fe901468ebcd9..0f1030c0d756187f329c6085b11ee2307b4f1c76 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,14 +22,14 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include #pragma once -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -47,17 +47,16 @@ class sve_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst : public DepthwiseDepthfi constexpr static unsigned int stride_rows = 1; constexpr static unsigned int stride_cols = 1; - arm_gemm::VLType get_vl_type(void) const override { return arm_gemm::VLType::SVE; } - unsigned int get_accumulator_depth_vl(void) const override { return 2; } - sve_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst(const CPUInfo *) : Parent(2, 2, 3, 3, 1, 1) {} - Parent::KernelType kernel = sve_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl; + arm_gemm::VLType get_vl_type(void) const override { return arm_gemm::VLType::SVE; } + Parent::KernelType kernel = sve_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl; Parent::KernelType get_kernel(void) const override { return kernel; } + unsigned int get_accumulator_depth_vl(void) const override { return 2; } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp index ff3ec0ba48bbe2b2ac9c77a8d84d0ef1ae72ad58..887eccf1e925745d10589179bc3a37fddf99728d 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp @@ -27,7 +27,7 @@ #include #include -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -46,7 +46,7 @@ void sve_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( struct Params { long unsigned int n_channels; - const int8_t *weights; + const void *weights; const int32_t *bias; const arm_gemm::Requantize32 *requant; const int32_t *const requant_muls; @@ -57,7 +57,7 @@ void sve_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( Params( long unsigned int n_channels, const uint8_t *const *inptrs_raw, - const int8_t *const weights, + const void *const weights, const int32_t *const bias, const arm_gemm::Requantize32 &qp, const int32_t *const requant_muls, @@ -91,320 +91,320 @@ void sve_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( requant_muls, requant_shifts, outptrs); __asm__ __volatile__( - "mov x8, #0x0\n" + "mov x16, #0x0\n" "ldr x25, [%x[params], %[offsetof_Params_requant]]\n" "ptrue p4.b\n" "ldr x24, [%x[params], %[offsetof_Params_outptrs]]\n" - "mov x23, x8\n" + "mov x23, x16\n" "add x21, x25, %[offsetof_Requantize32_a_offset]\n" - "ldr x17, [%x[params], %[offsetof_Params_n_channels]]\n" - "ldr x16, [%x[params], %[offsetof_Params_weights]]\n" + "ldr x15, [%x[params], %[offsetof_Params_n_channels]]\n" + "ldr x14, [%x[params], %[offsetof_Params_weights]]\n" "add x20, x25, %[offsetof_Requantize32_b_offset]\n" "add x22, x25, %[offsetof_Requantize32_c_offset]\n" - "ld1rb { z23.b }, p4/Z, [x21]\n" - "ld1rb { z15.b }, p4/Z, [x20]\n" + "ld1rb { z12.b }, p4/Z, [x21]\n" + "ld1rb { z30.b }, p4/Z, [x20]\n" "add x21, x25, %[offsetof_Requantize32_minval]\n" "add x20, x25, %[offsetof_Requantize32_maxval]\n" - "ld1rh { z14.h }, p4/Z, [x22]\n" - "ld1rh { z12.h }, p4/Z, [x21]\n" - "ld1rh { z11.h }, p4/Z, [x20]\n" - "ldp x15, x14, [x24, #0x0]\n" + "ld1rh { z24.h }, p4/Z, [x22]\n" + "ld1rh { z11.h }, p4/Z, [x21]\n" + "ld1rh { z26.h }, p4/Z, [x20]\n" + "ldp x13, x12, [x24, #0x0]\n" "incw x23\n" - "whilelt p3.h, x8, x17\n" - "ldp x13, x12, [x24, #0x10]\n" - "whilelt p2.s, x8, x17\n" - "whilelt p1.s, x23, x17\n" - "ldr x26, [%x[params], %[offsetof_Params_bias]]\n" - "ld1sb { z0.h }, p4/Z, [x16]\n" - "ld1sb { z1.h }, p4/Z, [x16, #1, MUL VL]\n" - "add x11, %x[params], %[offsetof_Params_inptrs]\n" - "mov x10, #0x0\n" - "ld1sb { z2.h }, p4/Z, [x16, #2, MUL VL]\n" - "ld1sb { z3.h }, p4/Z, [x16, #3, MUL VL]\n" - ".inst 0x454f1000 // ssublb z0.h, z0.b, z15.b\n" - ".inst 0x454f1021 // ssublb z1.h, z1.b, z15.b\n" - "ld1sb { z4.h }, p4/Z, [x16, #4, MUL VL]\n" - "ld1sb { z5.h }, p4/Z, [x16, #5, MUL VL]\n" - ".inst 0x454f1042 // ssublb z2.h, z2.b, z15.b\n" - ".inst 0x454f1063 // ssublb z3.h, z3.b, z15.b\n" - "ld1sb { z6.h }, p4/Z, [x16, #6, MUL VL]\n" - "ld1sb { z7.h }, p4/Z, [x16, #7, MUL VL]\n" - "inch x16, ALL, MUL #8\n" - ".inst 0x454f1084 // ssublb z4.h, z4.b, z15.b\n" - "ld1w { z17.s }, p2/Z, [x26]\n" - "ld1w { z16.s }, p1/Z, [x26, #1, MUL VL]\n" - "uzp1 z13.s, z17.s, z16.s\n" - "uzp2 z17.s, z17.s, z16.s\n" - "ld1sb { z8.h }, p4/Z, [x16]\n" - "ldp x24, x23, [x11, #0x0]\n" - "addvl x26, x26, #2\n" - "mov z26.d, z13.d\n" - "ldp x22, x21, [x11, #0x10]\n" - "ldr x20, [x11, #0x20]\n" - "mov z10.d, z17.d\n" - "mov z24.d, z13.d\n" - "ld1b { z31.h }, p3/Z, [x24, x8]\n" - "ld1b { z30.h }, p3/Z, [x23, x8]\n" - "mov z16.d, z17.d\n" - "mov z25.d, z13.d\n" - "ld1b { z29.h }, p3/Z, [x22, x8]\n" - "ld1b { z28.h }, p3/Z, [x21, x8]\n" - "mov z9.d, z17.d\n" - ".inst 0x454f10a5 // ssublb z5.h, z5.b, z15.b\n" - "ld1b { z27.h }, p3/Z, [x20, x8]\n" - "ldr x9, [%x[params], %[offsetof_Params_requant_muls]]\n" - ".inst 0x454f10c6 // ssublb z6.h, z6.b, z15.b\n" - ".inst 0x454f10e7 // ssublb z7.h, z7.b, z15.b\n" - "ldr x28, [%x[params], %[offsetof_Params_requant_shifts]]\n" - "str x26, [%x[params], %[offsetof_Params_bias]]\n" - ".inst 0x454f1108 // ssublb z8.h, z8.b, z15.b\n" - ".inst 0x45571bff // usublb z31.h, z31.b, z23.b\n" - ".inst 0x45571bde // usublb z30.h, z30.b, z23.b\n" - ".inst 0x45571bbd // usublb z29.h, z29.b, z23.b\n" - ".inst 0x45571b9c // usublb z28.h, z28.b, z23.b\n" - ".inst 0x45571b7b // usublb z27.h, z27.b, z23.b\n" + "whilelt p3.h, x16, x15\n" + "ldp x11, x10, [x24, #0x10]\n" + "whilelt p2.s, x16, x15\n" + "whilelt p1.s, x23, x15\n" + "ldr x9, [%x[params], %[offsetof_Params_bias]]\n" + "ld1sb { z14.h }, p4/Z, [x14]\n" + "ld1sb { z21.h }, p4/Z, [x14, #1, MUL VL]\n" + "add x28, %x[params], %[offsetof_Params_inptrs]\n" + "mov x27, #0x0\n" + "ld1sb { z1.h }, p4/Z, [x14, #2, MUL VL]\n" + "ld1sb { z6.h }, p4/Z, [x14, #3, MUL VL]\n" + ".inst 0x455e11ce // ssublb z14.h, z14.b, z30.b\n" + ".inst 0x455e12b5 // ssublb z21.h, z21.b, z30.b\n" + "ld1sb { z2.h }, p4/Z, [x14, #4, MUL VL]\n" + "ld1sb { z18.h }, p4/Z, [x14, #5, MUL VL]\n" + ".inst 0x455e1021 // ssublb z1.h, z1.b, z30.b\n" + ".inst 0x455e10c6 // ssublb z6.h, z6.b, z30.b\n" + "ld1sb { z7.h }, p4/Z, [x14, #6, MUL VL]\n" + "ld1sb { z10.h }, p4/Z, [x14, #7, MUL VL]\n" + "inch x14, ALL, MUL #8\n" + ".inst 0x455e1042 // ssublb z2.h, z2.b, z30.b\n" + "ld1w { z17.s }, p2/Z, [x9]\n" + "ld1w { z16.s }, p1/Z, [x9, #1, MUL VL]\n" + "uzp1 z5.s, z17.s, z16.s\n" + "uzp2 z9.s, z17.s, z16.s\n" + "ld1sb { z8.h }, p4/Z, [x14]\n" + "ldp x24, x23, [x28, #0x0]\n" + "addvl x9, x9, #2\n" + "mov z17.d, z5.d\n" + "ldp x22, x21, [x28, #0x10]\n" + "ldr x20, [x28, #0x20]\n" + "mov z25.d, z9.d\n" + "mov z16.d, z5.d\n" + "ld1b { z0.h }, p3/Z, [x24, x16]\n" + "ld1b { z29.h }, p3/Z, [x23, x16]\n" + "mov z23.d, z9.d\n" + "mov z22.d, z5.d\n" + "ld1b { z4.h }, p3/Z, [x22, x16]\n" + "ld1b { z13.h }, p3/Z, [x21, x16]\n" + "mov z27.d, z9.d\n" + ".inst 0x455e1252 // ssublb z18.h, z18.b, z30.b\n" + "ld1b { z20.h }, p3/Z, [x20, x16]\n" + "ldr x26, [%x[params], %[offsetof_Params_requant_muls]]\n" + ".inst 0x455e10e7 // ssublb z7.h, z7.b, z30.b\n" + ".inst 0x455e114a // ssublb z10.h, z10.b, z30.b\n" + "ldr x25, [%x[params], %[offsetof_Params_requant_shifts]]\n" + "str x9, [%x[params], %[offsetof_Params_bias]]\n" + ".inst 0x455e1108 // ssublb z8.h, z8.b, z30.b\n" + ".inst 0x454c1800 // usublb z0.h, z0.b, z12.b\n" + ".inst 0x454c1bbd // usublb z29.h, z29.b, z12.b\n" + ".inst 0x454c1884 // usublb z4.h, z4.b, z12.b\n" + ".inst 0x454c19ad // usublb z13.h, z13.b, z12.b\n" + ".inst 0x454c1a94 // usublb z20.h, z20.b, z12.b\n" "1:" // Loop - ".inst 0x448443ed // smlalb z13.s, p4/M, z31.h, z4.h\n" - ".inst 0x448447f1 // smlalt z17.s, p4/M, z31.h, z4.h\n" - "ldr x22, [x11, #0x28]\n" - "ldr x27, [x11, #0x38]\n" - ".inst 0x448343fa // smlalb z26.s, p4/M, z31.h, z3.h\n" - ".inst 0x448347ea // smlalt z10.s, p4/M, z31.h, z3.h\n" - "ldr x21, [x11, #0x30]\n" - "ldr x26, [x11, #0x40]\n" - ".inst 0x448043cd // smlalb z13.s, p4/M, z30.h, z0.h\n" - ".inst 0x448047d1 // smlalt z17.s, p4/M, z30.h, z0.h\n" - "ldr x20, [x11, #0x48]\n" - "ld1b { z30.h }, p3/Z, [x20, x8]\n" - ".inst 0x448243ba // smlalb z26.s, p4/M, z29.h, z2.h\n" - ".inst 0x448247aa // smlalt z10.s, p4/M, z29.h, z2.h\n" - "ld1b { z29.h }, p3/Z, [x21, x8]\n" - ".inst 0x45571bbd // usublb z29.h, z29.b, z23.b\n" - ".inst 0x448143f8 // smlalb z24.s, p4/M, z31.h, z1.h\n" - ".inst 0x448147f0 // smlalt z16.s, p4/M, z31.h, z1.h\n" - "ldr x25, [x11, #0x50]\n" - "ldr x24, [x11, #0x58]\n" - ".inst 0x448043f9 // smlalb z25.s, p4/M, z31.h, z0.h\n" - ".inst 0x448047e9 // smlalt z9.s, p4/M, z31.h, z0.h\n" - "ld1b { z31.h }, p3/Z, [x22, x8]\n" - ".inst 0x45571bff // usublb z31.h, z31.b, z23.b\n" - ".inst 0x4485438d // smlalb z13.s, p4/M, z28.h, z5.h\n" - ".inst 0x44854791 // smlalt z17.s, p4/M, z28.h, z5.h\n" - ".inst 0x45571bde // usublb z30.h, z30.b, z23.b\n" - "ldr x23, [x11, #0x60]\n" - ".inst 0x4484439a // smlalb z26.s, p4/M, z28.h, z4.h\n" - ".inst 0x4484478a // smlalt z10.s, p4/M, z28.h, z4.h\n" - "ldr x22, [x11, #0x68]\n" - "ldr x21, [x11, #0x70]\n" - ".inst 0x44824398 // smlalb z24.s, p4/M, z28.h, z2.h\n" - ".inst 0x44824790 // smlalt z16.s, p4/M, z28.h, z2.h\n" - "ldr x20, [x11, #0x78]\n" - "ld1w { z20.s }, p2/Z, [x9]\n" - ".inst 0x44814399 // smlalb z25.s, p4/M, z28.h, z1.h\n" - ".inst 0x44814789 // smlalt z9.s, p4/M, z28.h, z1.h\n" - "ld1b { z28.h }, p3/Z, [x27, x8]\n" - ".inst 0x45571b9c // usublb z28.h, z28.b, z23.b\n" - ".inst 0x4487436d // smlalb z13.s, p4/M, z27.h, z7.h\n" - ".inst 0x44874771 // smlalt z17.s, p4/M, z27.h, z7.h\n" - "ld1w { z18.s }, p1/Z, [x9, #1, MUL VL]\n" - "uzp1 z19.s, z20.s, z18.s\n" - ".inst 0x4486437a // smlalb z26.s, p4/M, z27.h, z6.h\n" - ".inst 0x4486476a // smlalt z10.s, p4/M, z27.h, z6.h\n" - "uzp2 z22.s, z20.s, z18.s\n" - "ld1w { z20.s }, p2/Z, [x28]\n" - ".inst 0x448643f8 // smlalb z24.s, p4/M, z31.h, z6.h\n" - ".inst 0x448647f0 // smlalt z16.s, p4/M, z31.h, z6.h\n" - "ld1b { z31.h }, p3/Z, [x26, x8]\n" - ".inst 0x45571bff // usublb z31.h, z31.b, z23.b\n" - ".inst 0x44834379 // smlalb z25.s, p4/M, z27.h, z3.h\n" - ".inst 0x44834769 // smlalt z9.s, p4/M, z27.h, z3.h\n" - "whilelt p0.h, x10, x17\n" + ".inst 0x44824005 // smlalb z5.s, p4/M, z0.h, z2.h\n" + ".inst 0x44824409 // smlalt z9.s, p4/M, z0.h, z2.h\n" + "ldr x20, [x28, #0x28]\n" + "ldr x21, [x28, #0x38]\n" + ".inst 0x448e43a5 // smlalb z5.s, p4/M, z29.h, z14.h\n" + ".inst 0x44864011 // smlalb z17.s, p4/M, z0.h, z6.h\n" + "ld1b { z3.h }, p3/Z, [x20, x16]\n" + "ldr x20, [x28, #0x30]\n" + ".inst 0x44954010 // smlalb z16.s, p4/M, z0.h, z21.h\n" + ".inst 0x448e4016 // smlalb z22.s, p4/M, z0.h, z14.h\n" + "ld1b { z31.h }, p3/Z, [x21, x16]\n" + ".inst 0x454c1863 // usublb z3.h, z3.b, z12.b\n" + ".inst 0x448e47a9 // smlalt z9.s, p4/M, z29.h, z14.h\n" + ".inst 0x449241a5 // smlalb z5.s, p4/M, z13.h, z18.h\n" + "ldr x21, [x28, #0x40]\n" + "ld1b { z15.h }, p3/Z, [x20, x16]\n" + ".inst 0x44864419 // smlalt z25.s, p4/M, z0.h, z6.h\n" + ".inst 0x44954417 // smlalt z23.s, p4/M, z0.h, z21.h\n" + ".inst 0x454c1bff // usublb z31.h, z31.b, z12.b\n" + "ldr x20, [x28, #0x48]\n" + ".inst 0x448e441b // smlalt z27.s, p4/M, z0.h, z14.h\n" + ".inst 0x44814091 // smlalb z17.s, p4/M, z4.h, z1.h\n" + "ld1b { z19.h }, p3/Z, [x21, x16]\n" + ".inst 0x454c19ef // usublb z15.h, z15.b, z12.b\n" + ".inst 0x448141b0 // smlalb z16.s, p4/M, z13.h, z1.h\n" + ".inst 0x449541b6 // smlalb z22.s, p4/M, z13.h, z21.h\n" + "ld1b { z28.h }, p3/Z, [x20, x16]\n" + ".inst 0x454c1a73 // usublb z19.h, z19.b, z12.b\n" + ".inst 0x449245a9 // smlalt z9.s, p4/M, z13.h, z18.h\n" + ".inst 0x448a4285 // smlalb z5.s, p4/M, z20.h, z10.h\n" + "ldr x21, [x28, #0x50]\n" + "ldr x20, [x28, #0x58]\n" + ".inst 0x44814499 // smlalt z25.s, p4/M, z4.h, z1.h\n" + ".inst 0x448145b7 // smlalt z23.s, p4/M, z13.h, z1.h\n" + ".inst 0x454c1b9c // usublb z28.h, z28.b, z12.b\n" + "ld1b { z4.h }, p3/Z, [x21, x16]\n" + ".inst 0x449545bb // smlalt z27.s, p4/M, z13.h, z21.h\n" + ".inst 0x448241b1 // smlalb z17.s, p4/M, z13.h, z2.h\n" + "ld1b { z29.h }, p3/Z, [x20, x16]\n" + "ldr x21, [x28, #0x60]\n" + ".inst 0x44874070 // smlalb z16.s, p4/M, z3.h, z7.h\n" + ".inst 0x44864296 // smlalb z22.s, p4/M, z20.h, z6.h\n" + "ldr x20, [x28, #0x68]\n" + ".inst 0x454c1884 // usublb z4.h, z4.b, z12.b\n" + ".inst 0x448a4689 // smlalt z9.s, p4/M, z20.h, z10.h\n" + ".inst 0x449543e5 // smlalb z5.s, p4/M, z31.h, z21.h\n" + ".inst 0x454c1bbd // usublb z29.h, z29.b, z12.b\n" + "ld1b { z0.h }, p3/Z, [x21, x16]\n" + ".inst 0x448245b9 // smlalt z25.s, p4/M, z13.h, z2.h\n" + ".inst 0x44874477 // smlalt z23.s, p4/M, z3.h, z7.h\n" + "ld1b { z3.h }, p3/Z, [x20, x16]\n" + "ldr x20, [x28, #0x70]\n" + ".inst 0x4486469b // smlalt z27.s, p4/M, z20.h, z6.h\n" + ".inst 0x44874291 // smlalb z17.s, p4/M, z20.h, z7.h\n" + ".inst 0x454c1800 // usublb z0.h, z0.b, z12.b\n" + "ld1b { z13.h }, p3/Z, [x20, x16]\n" + ".inst 0x44824290 // smlalb z16.s, p4/M, z20.h, z2.h\n" + ".inst 0x448841f6 // smlalb z22.s, p4/M, z15.h, z8.h\n" + ".inst 0x454c1863 // usublb z3.h, z3.b, z12.b\n" + "ldr x20, [x28, #0x78]\n" + ".inst 0x449547e9 // smlalt z9.s, p4/M, z31.h, z21.h\n" + ".inst 0x44814265 // smlalb z5.s, p4/M, z19.h, z1.h\n" + ".inst 0x454c19ad // usublb z13.h, z13.b, z12.b\n" + "whilelt p0.h, x27, x15\n" + ".inst 0x44874699 // smlalt z25.s, p4/M, z20.h, z7.h\n" + ".inst 0x44824697 // smlalt z23.s, p4/M, z20.h, z2.h\n" + "ld1w { z20.s }, p2/Z, [x26]\n" + "inch x14\n" + ".inst 0x448845fb // smlalt z27.s, p4/M, z15.h, z8.h\n" + ".inst 0x448e43f1 // smlalb z17.s, p4/M, z31.h, z14.h\n" + "ld1w { z15.s }, p1/Z, [x26, #1, MUL VL]\n" + "ldr x21, [%x[params], %[offsetof_Params_bias]]\n" + ".inst 0x44924390 // smlalb z16.s, p4/M, z28.h, z18.h\n" + ".inst 0x44824396 // smlalb z22.s, p4/M, z28.h, z2.h\n" + "addvl x26, x26, #2\n" + ".inst 0x44814669 // smlalt z9.s, p4/M, z19.h, z1.h\n" + ".inst 0x44884385 // smlalb z5.s, p4/M, z28.h, z8.h\n" + ".inst 0x448e47f9 // smlalt z25.s, p4/M, z31.h, z14.h\n" + ".inst 0x44924797 // smlalt z23.s, p4/M, z28.h, z18.h\n" + "ld1b { z31.h }, p3/Z, [x20, x16]\n" + ".inst 0x454c1bff // usublb z31.h, z31.b, z12.b\n" + ".inst 0x4482479b // smlalt z27.s, p4/M, z28.h, z2.h\n" + ".inst 0x44954271 // smlalb z17.s, p4/M, z19.h, z21.h\n" + "uzp1 z2.s, z20.s, z15.s\n" "inch x16\n" - ".inst 0x4481438d // smlalb z13.s, p4/M, z28.h, z1.h\n" - ".inst 0x44814791 // smlalt z17.s, p4/M, z28.h, z1.h\n" - "ldr x26, [%x[params], %[offsetof_Params_bias]]\n" - "addvl x9, x9, #2\n" - ".inst 0x4480439a // smlalb z26.s, p4/M, z28.h, z0.h\n" - ".inst 0x4480478a // smlalt z10.s, p4/M, z28.h, z0.h\n" - "ld1b { z28.h }, p3/Z, [x24, x8]\n" - ".inst 0x45571b9c // usublb z28.h, z28.b, z23.b\n" - ".inst 0x44844378 // smlalb z24.s, p4/M, z27.h, z4.h\n" - ".inst 0x448843b9 // smlalb z25.s, p4/M, z29.h, z8.h\n" - ".inst 0x44844770 // smlalt z16.s, p4/M, z27.h, z4.h\n" - ".inst 0x448847a9 // smlalt z9.s, p4/M, z29.h, z8.h\n" - "ld1b { z29.h }, p3/Z, [x25, x8]\n" - ".inst 0x45571bbd // usublb z29.h, z29.b, z23.b\n" - ".inst 0x448243ed // smlalb z13.s, p4/M, z31.h, z2.h\n" - ".inst 0x448247f1 // smlalt z17.s, p4/M, z31.h, z2.h\n" - "ld1w { z18.s }, p1/Z, [x28, #1, MUL VL]\n" - "addvl x28, x28, #2\n" - ".inst 0x448143fa // smlalb z26.s, p4/M, z31.h, z1.h\n" - ".inst 0x448147ea // smlalt z10.s, p4/M, z31.h, z1.h\n" - "ld1b { z31.h }, p3/Z, [x23, x8]\n" - ".inst 0x45571bff // usublb z31.h, z31.b, z23.b\n" - ".inst 0x448543d8 // smlalb z24.s, p4/M, z30.h, z5.h\n" - ".inst 0x448443d9 // smlalb z25.s, p4/M, z30.h, z4.h\n" - "uzp1 z1.s, z20.s, z18.s\n" - ".inst 0x448843cd // smlalb z13.s, p4/M, z30.h, z8.h\n" - ".inst 0x448847d1 // smlalt z17.s, p4/M, z30.h, z8.h\n" - "uzp2 z27.s, z20.s, z18.s\n" - ".inst 0x448743da // smlalb z26.s, p4/M, z30.h, z7.h\n" - ".inst 0x448747ca // smlalt z10.s, p4/M, z30.h, z7.h\n" - ".inst 0x448547d0 // smlalt z16.s, p4/M, z30.h, z5.h\n" - ".inst 0x448447c9 // smlalt z9.s, p4/M, z30.h, z4.h\n" - "ld1b { z30.h }, p3/Z, [x22, x8]\n" - ".inst 0x45571bde // usublb z30.h, z30.b, z23.b\n" - ".inst 0x448043b8 // smlalb z24.s, p4/M, z29.h, z0.h\n" - ".inst 0x44824399 // smlalb z25.s, p4/M, z28.h, z2.h\n" - ".inst 0x448343ad // smlalb z13.s, p4/M, z29.h, z3.h\n" - ".inst 0x448347b1 // smlalt z17.s, p4/M, z29.h, z3.h\n" - ".inst 0x448047b0 // smlalt z16.s, p4/M, z29.h, z0.h\n" - "ld1b { z29.h }, p3/Z, [x21, x8]\n" - ".inst 0x44824789 // smlalt z9.s, p4/M, z28.h, z2.h\n" - ".inst 0x45571bbd // usublb z29.h, z29.b, z23.b\n" - ".inst 0x448343f8 // smlalb z24.s, p4/M, z31.h, z3.h\n" - ".inst 0x448543d9 // smlalb z25.s, p4/M, z30.h, z5.h\n" - ".inst 0x4485439a // smlalb z26.s, p4/M, z28.h, z5.h\n" - ".inst 0x4485478a // smlalt z10.s, p4/M, z28.h, z5.h\n" - "ld1b { z28.h }, p3/Z, [x20, x8]\n" - ".inst 0x45571b9c // usublb z28.h, z28.b, z23.b\n" - ".inst 0x448643ed // smlalb z13.s, p4/M, z31.h, z6.h\n" - ".inst 0x448347f0 // smlalt z16.s, p4/M, z31.h, z3.h\n" - ".inst 0x04b375ad // sqrdmulh z13.s, z13.s, z19.s\n" - "inch x8\n" - ".inst 0x448547c9 // smlalt z9.s, p4/M, z30.h, z5.h\n" - ".inst 0x448743b8 // smlalb z24.s, p4/M, z29.h, z7.h\n" - "and z21.d, z13.d, z1.d\n" - "mov x20, x8\n" - ".inst 0x448643b9 // smlalb z25.s, p4/M, z29.h, z6.h\n" - ".inst 0x448647f1 // smlalt z17.s, p4/M, z31.h, z6.h\n" - ".inst 0x04b67631 // sqrdmulh z17.s, z17.s, z22.s\n" + ".inst 0x448e4090 // smlalb z16.s, p4/M, z4.h, z14.h\n" + ".inst 0x448143b6 // smlalb z22.s, p4/M, z29.h, z1.h\n" + "uzp2 z15.s, z20.s, z15.s\n" + "ld1w { z20.s }, p2/Z, [x25]\n" + ".inst 0x44884789 // smlalt z9.s, p4/M, z28.h, z8.h\n" + ".inst 0x44864085 // smlalb z5.s, p4/M, z4.h, z6.h\n" + "mov x20, x16\n" "incw x20\n" - ".inst 0x448747b0 // smlalt z16.s, p4/M, z29.h, z7.h\n" - ".inst 0x448647a9 // smlalt z9.s, p4/M, z29.h, z6.h\n" - "asr z21.s, z21.s, #0x1f\n" - "whilelt p2.s, x8, x17\n" - ".inst 0x448843da // smlalb z26.s, p4/M, z30.h, z8.h\n" - ".inst 0x44884398 // smlalb z24.s, p4/M, z28.h, z8.h\n" - "and z20.d, z17.d, z27.d\n" - "whilelt p1.s, x20, x17\n" - ".inst 0x44874399 // smlalb z25.s, p4/M, z28.h, z7.h\n" - ".inst 0x448847ca // smlalt z10.s, p4/M, z30.h, z8.h\n" - ".inst 0x04b3775a // sqrdmulh z26.s, z26.s, z19.s\n" - "whilelt p3.h, x8, x17\n" - ".inst 0x44884790 // smlalt z16.s, p4/M, z28.h, z8.h\n" - ".inst 0x44874789 // smlalt z9.s, p4/M, z28.h, z7.h\n" - ".inst 0x04b37718 // sqrdmulh z24.s, z24.s, z19.s\n" - ".inst 0x04b37739 // sqrdmulh z25.s, z25.s, z19.s\n" - "sqadd z13.s, z13.s, z21.s\n" - ".inst 0x4482902d // srshl z13.s, p4/M, z13.s, z1.s\n" - "asr z20.s, z20.s, #0x1f\n" - "and z19.d, z26.d, z1.d\n" - ".inst 0x04b6754a // sqrdmulh z10.s, z10.s, z22.s\n" - "and z18.d, z24.d, z1.d\n" - ".inst 0x04b67610 // sqrdmulh z16.s, z16.s, z22.s\n" - "and z21.d, z25.d, z1.d\n" - ".inst 0x04b67529 // sqrdmulh z9.s, z9.s, z22.s\n" - "sqadd z17.s, z17.s, z20.s\n" - ".inst 0x44829371 // srshl z17.s, p4/M, z17.s, z27.s\n" + ".inst 0x44954679 // smlalt z25.s, p4/M, z19.h, z21.h\n" + ".inst 0x448e4497 // smlalt z23.s, p4/M, z4.h, z14.h\n" + "ld1w { z19.s }, p1/Z, [x25, #1, MUL VL]\n" + "uzp1 z21.s, z20.s, z19.s\n" + ".inst 0x448147bb // smlalt z27.s, p4/M, z29.h, z1.h\n" + ".inst 0x448a4391 // smlalb z17.s, p4/M, z28.h, z10.h\n" + "uzp2 z1.s, z20.s, z19.s\n" + "whilelt p2.s, x16, x15\n" + ".inst 0x44864010 // smlalb z16.s, p4/M, z0.h, z6.h\n" + ".inst 0x44924076 // smlalb z22.s, p4/M, z3.h, z18.h\n" + "whilelt p1.s, x20, x15\n" + "whilelt p3.h, x16, x15\n" + ".inst 0x44864489 // smlalt z9.s, p4/M, z4.h, z6.h\n" + ".inst 0x44874005 // smlalb z5.s, p4/M, z0.h, z7.h\n" + ".inst 0x04a274a5 // sqrdmulh z5.s, z5.s, z2.s\n" + "addvl x25, x25, #2\n" + ".inst 0x448a4799 // smlalt z25.s, p4/M, z28.h, z10.h\n" + ".inst 0x44864417 // smlalt z23.s, p4/M, z0.h, z6.h\n" + "and z19.d, z5.d, z21.d\n" + ".inst 0x4492447b // smlalt z27.s, p4/M, z3.h, z18.h\n" + ".inst 0x449243b1 // smlalb z17.s, p4/M, z29.h, z18.h\n" "asr z19.s, z19.s, #0x1f\n" - "and z2.d, z10.d, z27.d\n" + ".inst 0x448a41b0 // smlalb z16.s, p4/M, z13.h, z10.h\n" + ".inst 0x448741b6 // smlalb z22.s, p4/M, z13.h, z7.h\n" + "sqadd z5.s, z5.s, z19.s\n" + ".inst 0x448292a5 // srshl z5.s, p4/M, z5.s, z21.s\n" + ".inst 0x44874409 // smlalt z9.s, p4/M, z0.h, z7.h\n" + ".inst 0x449247b9 // smlalt z25.s, p4/M, z29.h, z18.h\n" + ".inst 0x04af7529 // sqrdmulh z9.s, z9.s, z15.s\n" + ".inst 0x448a45b7 // smlalt z23.s, p4/M, z13.h, z10.h\n" + ".inst 0x448745bb // smlalt z27.s, p4/M, z13.h, z7.h\n" + "and z29.d, z9.d, z1.d\n" + ".inst 0x44884071 // smlalb z17.s, p4/M, z3.h, z8.h\n" + ".inst 0x448843f0 // smlalb z16.s, p4/M, z31.h, z8.h\n" + ".inst 0x04a27631 // sqrdmulh z17.s, z17.s, z2.s\n" + ".inst 0x448a43f6 // smlalb z22.s, p4/M, z31.h, z10.h\n" + ".inst 0x44884479 // smlalt z25.s, p4/M, z3.h, z8.h\n" + ".inst 0x04a27610 // sqrdmulh z16.s, z16.s, z2.s\n" + ".inst 0x448847f7 // smlalt z23.s, p4/M, z31.h, z8.h\n" + ".inst 0x448a47fb // smlalt z27.s, p4/M, z31.h, z10.h\n" + ".inst 0x04a276d6 // sqrdmulh z22.s, z22.s, z2.s\n" + "asr z29.s, z29.s, #0x1f\n" + "and z18.d, z17.d, z21.d\n" + ".inst 0x04af7739 // sqrdmulh z25.s, z25.s, z15.s\n" + "and z20.d, z16.d, z21.d\n" + ".inst 0x04af76f7 // sqrdmulh z23.s, z23.s, z15.s\n" + "and z19.d, z22.d, z21.d\n" + ".inst 0x04af777b // sqrdmulh z27.s, z27.s, z15.s\n" + "sqadd z9.s, z9.s, z29.s\n" + ".inst 0x44829029 // srshl z9.s, p4/M, z9.s, z1.s\n" "asr z18.s, z18.s, #0x1f\n" - "and z22.d, z16.d, z27.d\n" - "asr z21.s, z21.s, #0x1f\n" - "and z20.d, z9.d, z27.d\n" - "sqadd z26.s, z26.s, z19.s\n" - "asr z2.s, z2.s, #0x1f\n" - ".inst 0x4482903a // srshl z26.s, p4/M, z26.s, z1.s\n" - "sqadd z24.s, z24.s, z18.s\n" - "asr z22.s, z22.s, #0x1f\n" - ".inst 0x44829038 // srshl z24.s, p4/M, z24.s, z1.s\n" - "sqadd z25.s, z25.s, z21.s\n" + "and z7.d, z25.d, z1.d\n" "asr z20.s, z20.s, #0x1f\n" + "and z6.d, z23.d, z1.d\n" + "asr z19.s, z19.s, #0x1f\n" + "and z2.d, z27.d, z1.d\n" + "sqadd z17.s, z17.s, z18.s\n" + "asr z7.s, z7.s, #0x1f\n" + ".inst 0x448292b1 // srshl z17.s, p4/M, z17.s, z21.s\n" + "sqadd z16.s, z16.s, z20.s\n" + "asr z6.s, z6.s, #0x1f\n" + ".inst 0x448292b0 // srshl z16.s, p4/M, z16.s, z21.s\n" + "sqadd z22.s, z22.s, z19.s\n" + "asr z2.s, z2.s, #0x1f\n" + ".inst 0x448292b6 // srshl z22.s, p4/M, z22.s, z21.s\n" + "sqadd z25.s, z25.s, z7.s\n" + "sqadd z23.s, z23.s, z6.s\n" ".inst 0x44829039 // srshl z25.s, p4/M, z25.s, z1.s\n" - "sqadd z10.s, z10.s, z2.s\n" - "sqadd z16.s, z16.s, z22.s\n" - ".inst 0x4482936a // srshl z10.s, p4/M, z10.s, z27.s\n" - ".inst 0x44829370 // srshl z16.s, p4/M, z16.s, z27.s\n" - "sqadd z9.s, z9.s, z20.s\n" - ".inst 0x453041ad // sqxtnb z13.h, z13.s\n" - ".inst 0x44829369 // srshl z9.s, p4/M, z9.s, z27.s\n" - ".inst 0x4530435a // sqxtnb z26.h, z26.s\n" - ".inst 0x45304318 // sqxtnb z24.h, z24.s\n" - ".inst 0x45304339 // sqxtnb z25.h, z25.s\n" - ".inst 0x4530462d // sqxtnt z13.h, z17.s\n" - ".inst 0x4530455a // sqxtnt z26.h, z10.s\n" - ".inst 0x45304618 // sqxtnt z24.h, z16.s\n" - ".inst 0x45304539 // sqxtnt z25.h, z9.s\n" - "sqadd z13.h, z13.h, z14.h\n" - "smax z13.h, p4/M, z13.h, z12.h\n" - "smin z13.h, p4/M, z13.h, z11.h\n" - "sqadd z26.h, z26.h, z14.h\n" - "sqadd z24.h, z24.h, z14.h\n" - "smax z26.h, p4/M, z26.h, z12.h\n" - "smax z24.h, p4/M, z24.h, z12.h\n" - "sqadd z25.h, z25.h, z14.h\n" - "smax z25.h, p4/M, z25.h, z12.h\n" - "smin z26.h, p4/M, z26.h, z11.h\n" - "st1b { z13.h }, p0, [x15, x10]\n" - "smin z24.h, p4/M, z24.h, z11.h\n" - "smin z25.h, p4/M, z25.h, z11.h\n" - "st1b { z26.h }, p0, [x14, x10]\n" - "st1b { z24.h }, p0, [x13, x10]\n" - "st1b { z25.h }, p0, [x12, x10]\n" - "ld1sb { z0.h }, p4/Z, [x16]\n" - "ld1sb { z1.h }, p4/Z, [x16, #1, MUL VL]\n" - "inch x10\n" - "ld1sb { z2.h }, p4/Z, [x16, #2, MUL VL]\n" - "ld1sb { z3.h }, p4/Z, [x16, #3, MUL VL]\n" - ".inst 0x454f1000 // ssublb z0.h, z0.b, z15.b\n" - ".inst 0x454f1021 // ssublb z1.h, z1.b, z15.b\n" - "ld1sb { z4.h }, p4/Z, [x16, #4, MUL VL]\n" - "ld1sb { z5.h }, p4/Z, [x16, #5, MUL VL]\n" - ".inst 0x454f1042 // ssublb z2.h, z2.b, z15.b\n" - ".inst 0x454f1063 // ssublb z3.h, z3.b, z15.b\n" - "ld1sb { z6.h }, p4/Z, [x16, #6, MUL VL]\n" - "ld1sb { z7.h }, p4/Z, [x16, #7, MUL VL]\n" - "inch x16, ALL, MUL #8\n" - ".inst 0x454f1084 // ssublb z4.h, z4.b, z15.b\n" - "ld1w { z17.s }, p2/Z, [x26]\n" - "ld1w { z16.s }, p1/Z, [x26, #1, MUL VL]\n" - "uzp1 z13.s, z17.s, z16.s\n" - "uzp2 z17.s, z17.s, z16.s\n" - "ld1sb { z8.h }, p4/Z, [x16]\n" - "ldp x24, x23, [x11, #0x0]\n" - "addvl x26, x26, #2\n" - "str x26, [%x[params], %[offsetof_Params_bias]]\n" - "ldp x22, x21, [x11, #0x10]\n" - "ldr x20, [x11, #0x20]\n" - "mov z26.d, z13.d\n" - "mov z10.d, z17.d\n" - "ld1b { z31.h }, p3/Z, [x24, x8]\n" - "ld1b { z30.h }, p3/Z, [x23, x8]\n" - "mov z24.d, z13.d\n" - "mov z16.d, z17.d\n" - "ld1b { z29.h }, p3/Z, [x22, x8]\n" - "ld1b { z28.h }, p3/Z, [x21, x8]\n" - "mov z25.d, z13.d\n" - "mov z9.d, z17.d\n" - "ld1b { z27.h }, p3/Z, [x20, x8]\n" - ".inst 0x454f10a5 // ssublb z5.h, z5.b, z15.b\n" - ".inst 0x454f10c6 // ssublb z6.h, z6.b, z15.b\n" - ".inst 0x454f10e7 // ssublb z7.h, z7.b, z15.b\n" - ".inst 0x454f1108 // ssublb z8.h, z8.b, z15.b\n" - ".inst 0x45571bff // usublb z31.h, z31.b, z23.b\n" - ".inst 0x45571bde // usublb z30.h, z30.b, z23.b\n" - ".inst 0x45571bbd // usublb z29.h, z29.b, z23.b\n" - ".inst 0x45571b9c // usublb z28.h, z28.b, z23.b\n" - ".inst 0x45571b7b // usublb z27.h, z27.b, z23.b\n" + ".inst 0x44829037 // srshl z23.s, p4/M, z23.s, z1.s\n" + "sqadd z27.s, z27.s, z2.s\n" + ".inst 0x453040a5 // sqxtnb z5.h, z5.s\n" + ".inst 0x4482903b // srshl z27.s, p4/M, z27.s, z1.s\n" + ".inst 0x45304231 // sqxtnb z17.h, z17.s\n" + ".inst 0x45304210 // sqxtnb z16.h, z16.s\n" + ".inst 0x453042d6 // sqxtnb z22.h, z22.s\n" + ".inst 0x45304525 // sqxtnt z5.h, z9.s\n" + ".inst 0x45304731 // sqxtnt z17.h, z25.s\n" + ".inst 0x453046f0 // sqxtnt z16.h, z23.s\n" + ".inst 0x45304776 // sqxtnt z22.h, z27.s\n" + "sqadd z5.h, z5.h, z24.h\n" + "smax z5.h, p4/M, z5.h, z11.h\n" + "smin z5.h, p4/M, z5.h, z26.h\n" + "sqadd z17.h, z17.h, z24.h\n" + "sqadd z16.h, z16.h, z24.h\n" + "smax z17.h, p4/M, z17.h, z11.h\n" + "smax z16.h, p4/M, z16.h, z11.h\n" + "sqadd z22.h, z22.h, z24.h\n" + "smax z22.h, p4/M, z22.h, z11.h\n" + "smin z17.h, p4/M, z17.h, z26.h\n" + "st1b { z5.h }, p0, [x13, x27]\n" + "smin z16.h, p4/M, z16.h, z26.h\n" + "smin z22.h, p4/M, z22.h, z26.h\n" + "st1b { z17.h }, p0, [x12, x27]\n" + "st1b { z16.h }, p0, [x11, x27]\n" + "st1b { z22.h }, p0, [x10, x27]\n" + "ld1sb { z14.h }, p4/Z, [x14]\n" + "ld1sb { z21.h }, p4/Z, [x14, #1, MUL VL]\n" + "inch x27\n" + "ld1sb { z1.h }, p4/Z, [x14, #2, MUL VL]\n" + "ld1sb { z6.h }, p4/Z, [x14, #3, MUL VL]\n" + ".inst 0x455e11ce // ssublb z14.h, z14.b, z30.b\n" + ".inst 0x455e12b5 // ssublb z21.h, z21.b, z30.b\n" + "ld1sb { z2.h }, p4/Z, [x14, #4, MUL VL]\n" + "ld1sb { z18.h }, p4/Z, [x14, #5, MUL VL]\n" + ".inst 0x455e1021 // ssublb z1.h, z1.b, z30.b\n" + ".inst 0x455e10c6 // ssublb z6.h, z6.b, z30.b\n" + "ld1sb { z7.h }, p4/Z, [x14, #6, MUL VL]\n" + "ld1sb { z10.h }, p4/Z, [x14, #7, MUL VL]\n" + "inch x14, ALL, MUL #8\n" + ".inst 0x455e1042 // ssublb z2.h, z2.b, z30.b\n" + "ld1w { z17.s }, p2/Z, [x21]\n" + "ld1w { z16.s }, p1/Z, [x21, #1, MUL VL]\n" + "uzp1 z5.s, z17.s, z16.s\n" + "uzp2 z9.s, z17.s, z16.s\n" + "ld1sb { z8.h }, p4/Z, [x14]\n" + "ldp x24, x23, [x28, #0x0]\n" + "addvl x21, x21, #2\n" + "str x21, [%x[params], %[offsetof_Params_bias]]\n" + "ldp x22, x21, [x28, #0x10]\n" + "ldr x20, [x28, #0x20]\n" + "mov z17.d, z5.d\n" + "mov z25.d, z9.d\n" + "ld1b { z0.h }, p3/Z, [x24, x16]\n" + "ld1b { z29.h }, p3/Z, [x23, x16]\n" + "mov z16.d, z5.d\n" + "mov z23.d, z9.d\n" + "ld1b { z4.h }, p3/Z, [x22, x16]\n" + "ld1b { z13.h }, p3/Z, [x21, x16]\n" + "mov z22.d, z5.d\n" + "mov z27.d, z9.d\n" + "ld1b { z20.h }, p3/Z, [x20, x16]\n" + ".inst 0x455e1252 // ssublb z18.h, z18.b, z30.b\n" + ".inst 0x455e10e7 // ssublb z7.h, z7.b, z30.b\n" + ".inst 0x455e114a // ssublb z10.h, z10.b, z30.b\n" + ".inst 0x455e1108 // ssublb z8.h, z8.b, z30.b\n" + ".inst 0x454c1800 // usublb z0.h, z0.b, z12.b\n" + ".inst 0x454c1bbd // usublb z29.h, z29.b, z12.b\n" + ".inst 0x454c1884 // usublb z4.h, z4.b, z12.b\n" + ".inst 0x454c19ad // usublb z13.h, z13.b, z12.b\n" + ".inst 0x454c1a94 // usublb z20.h, z20.b, z12.b\n" "b.any 1b\n" : : [offsetof_Params_bias] "I" (offsetof(Params, bias)), [offsetof_Params_inptrs] "I" (offsetof(Params, inptrs)), [offsetof_Params_n_channels] "I" (offsetof(Params, n_channels)), [offsetof_Params_outptrs] "I" (offsetof(Params, outptrs)), [offsetof_Params_requant] "I" (offsetof(Params, requant)), [offsetof_Params_requant_muls] "I" (offsetof(Params, requant_muls)), [offsetof_Params_requant_shifts] "I" (offsetof(Params, requant_shifts)), [offsetof_Params_weights] "I" (offsetof(Params, weights)), [offsetof_Requantize32_a_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [params] "r" (¶ms) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp index 9a3db20f73adb3e68646879e925b91cc50a9ebf3..79e3fd5f548f8fab1bd8d6628bc0a7fcd29db103 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,14 +22,14 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include #pragma once -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -47,17 +47,16 @@ class sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst : public DepthwiseDepthfi constexpr static unsigned int stride_rows = 2; constexpr static unsigned int stride_cols = 2; - arm_gemm::VLType get_vl_type(void) const override { return arm_gemm::VLType::SVE; } - unsigned int get_accumulator_depth_vl(void) const override { return 2; } - sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst(const CPUInfo *) : Parent(2, 2, 3, 3, 2, 2) {} - Parent::KernelType kernel = sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl; + arm_gemm::VLType get_vl_type(void) const override { return arm_gemm::VLType::SVE; } + Parent::KernelType kernel = sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl; Parent::KernelType get_kernel(void) const override { return kernel; } + unsigned int get_accumulator_depth_vl(void) const override { return 2; } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp index 24c4bf713daf225176488657baacf55863294748..754d06d443a8dc331c174e1c5f6ac9f5c3f9f46c 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp @@ -27,7 +27,7 @@ #include #include -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -46,7 +46,7 @@ void sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( struct Params { long unsigned int n_channels; - const int8_t *weights; + const void *weights; const int32_t *bias; const arm_gemm::Requantize32 *requant; const int32_t *const requant_muls; @@ -57,7 +57,7 @@ void sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( Params( long unsigned int n_channels, const uint8_t *const *inptrs_raw, - const int8_t *const weights, + const void *const weights, const int32_t *const bias, const arm_gemm::Requantize32 &qp, const int32_t *const requant_muls, @@ -110,13 +110,13 @@ void sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ldr x17, [%x[params], %[offsetof_Params_weights]]\n" "add x20, x25, %[offsetof_Requantize32_b_offset]\n" "add x22, x25, %[offsetof_Requantize32_c_offset]\n" - "ld1rb { z23.b }, p4/Z, [x21]\n" - "ld1rb { z12.b }, p4/Z, [x20]\n" + "ld1rb { z26.b }, p4/Z, [x21]\n" + "ld1rb { z13.b }, p4/Z, [x20]\n" "add x21, x25, %[offsetof_Requantize32_minval]\n" "add x20, x25, %[offsetof_Requantize32_maxval]\n" - "ld1rh { z14.h }, p4/Z, [x22]\n" - "ld1rh { z16.h }, p4/Z, [x21]\n" - "ld1rh { z15.h }, p4/Z, [x20]\n" + "ld1rh { z19.h }, p4/Z, [x22]\n" + "ld1rh { z12.h }, p4/Z, [x21]\n" + "ld1rh { z9.h }, p4/Z, [x20]\n" "ldp x16, x15, [x24, #0x0]\n" "incw x23\n" "whilelt p3.h, x7, x8\n" @@ -124,320 +124,320 @@ void sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "whilelt p2.s, x7, x8\n" "whilelt p1.s, x23, x8\n" "ldr x12, [%x[params], %[offsetof_Params_bias]]\n" - "ld1sb { z0.h }, p4/Z, [x17]\n" - "ld1sb { z1.h }, p4/Z, [x17, #1, MUL VL]\n" + "ld1sb { z25.h }, p4/Z, [x17]\n" + "ld1sb { z30.h }, p4/Z, [x17, #1, MUL VL]\n" "add x11, %x[params], %[offsetof_Params_inptrs]\n" "mov x10, #0x0\n" - "ld1sb { z2.h }, p4/Z, [x17, #2, MUL VL]\n" - "ld1sb { z3.h }, p4/Z, [x17, #3, MUL VL]\n" - ".inst 0x454c1000 // ssublb z0.h, z0.b, z12.b\n" - ".inst 0x454c1021 // ssublb z1.h, z1.b, z12.b\n" - "ld1sb { z4.h }, p4/Z, [x17, #4, MUL VL]\n" - "ld1sb { z5.h }, p4/Z, [x17, #5, MUL VL]\n" - ".inst 0x454c1042 // ssublb z2.h, z2.b, z12.b\n" - ".inst 0x454c1063 // ssublb z3.h, z3.b, z12.b\n" - "ld1sb { z6.h }, p4/Z, [x17, #6, MUL VL]\n" + "ld1sb { z14.h }, p4/Z, [x17, #2, MUL VL]\n" + "ld1sb { z4.h }, p4/Z, [x17, #3, MUL VL]\n" + ".inst 0x454d1339 // ssublb z25.h, z25.b, z13.b\n" + ".inst 0x454d13de // ssublb z30.h, z30.b, z13.b\n" + "ld1sb { z10.h }, p4/Z, [x17, #4, MUL VL]\n" + "ld1sb { z3.h }, p4/Z, [x17, #5, MUL VL]\n" + ".inst 0x454d11ce // ssublb z14.h, z14.b, z13.b\n" + ".inst 0x454d1084 // ssublb z4.h, z4.b, z13.b\n" + "ld1sb { z23.h }, p4/Z, [x17, #6, MUL VL]\n" "ld1sb { z7.h }, p4/Z, [x17, #7, MUL VL]\n" "inch x17, ALL, MUL #8\n" - ".inst 0x454c1084 // ssublb z4.h, z4.b, z12.b\n" - "ld1w { z18.s }, p2/Z, [x12]\n" - "ld1w { z8.s }, p1/Z, [x12, #1, MUL VL]\n" - "uzp1 z13.s, z18.s, z8.s\n" - "uzp2 z17.s, z18.s, z8.s\n" - "ld1sb { z8.h }, p4/Z, [x17]\n" - "ldp x9, x28, [x11, #0x0]\n" + ".inst 0x454d114a // ssublb z10.h, z10.b, z13.b\n" + "ld1w { z17.s }, p2/Z, [x12]\n" + "ld1w { z16.s }, p1/Z, [x12, #1, MUL VL]\n" + "uzp1 z8.s, z17.s, z16.s\n" + "uzp2 z24.s, z17.s, z16.s\n" + "ld1sb { z2.h }, p4/Z, [x17]\n" + "ldp x27, x26, [x11, #0x0]\n" "addvl x12, x12, #2\n" - "mov z9.d, z13.d\n" + "mov z18.d, z8.d\n" "ldp x25, x24, [x11, #0x10]\n" "ldp x23, x22, [x11, #0x20]\n" - "mov z10.d, z17.d\n" - "mov z11.d, z13.d\n" + "mov z0.d, z24.d\n" + "mov z15.d, z8.d\n" "ldp x21, x20, [x11, #0x30]\n" - "ld1b { z31.h }, p3/Z, [x9, x7]\n" - "mov z22.d, z17.d\n" - "mov z21.d, z13.d\n" - "ld1b { z30.h }, p3/Z, [x28, x7]\n" - "ld1b { z29.h }, p3/Z, [x25, x7]\n" - "mov z18.d, z17.d\n" - ".inst 0x454c10a5 // ssublb z5.h, z5.b, z12.b\n" - "ld1b { z28.h }, p3/Z, [x24, x7]\n" + "ld1b { z21.h }, p3/Z, [x27, x7]\n" + "mov z1.d, z24.d\n" + "mov z5.d, z8.d\n" + "ld1b { z22.h }, p3/Z, [x26, x7]\n" + "ld1b { z11.h }, p3/Z, [x25, x7]\n" + "mov z6.d, z24.d\n" + ".inst 0x454d1063 // ssublb z3.h, z3.b, z13.b\n" + "ld1b { z20.h }, p3/Z, [x24, x7]\n" "ld1b { z27.h }, p3/Z, [x23, x7]\n" - ".inst 0x454c10c6 // ssublb z6.h, z6.b, z12.b\n" - ".inst 0x454c10e7 // ssublb z7.h, z7.b, z12.b\n" - "ld1b { z26.h }, p3/Z, [x22, x7]\n" - "ld1b { z25.h }, p3/Z, [x21, x7]\n" - ".inst 0x454c1108 // ssublb z8.h, z8.b, z12.b\n" - ".inst 0x45571bff // usublb z31.h, z31.b, z23.b\n" - "ld1b { z24.h }, p3/Z, [x20, x7]\n" - "ldr x27, [%x[params], %[offsetof_Params_requant_muls]]\n" - ".inst 0x45571bde // usublb z30.h, z30.b, z23.b\n" - ".inst 0x45571bbd // usublb z29.h, z29.b, z23.b\n" - "ldr x26, [%x[params], %[offsetof_Params_requant_shifts]]\n" + ".inst 0x454d12f7 // ssublb z23.h, z23.b, z13.b\n" + ".inst 0x454d10e7 // ssublb z7.h, z7.b, z13.b\n" + "ld1b { z28.h }, p3/Z, [x22, x7]\n" + "ld1b { z16.h }, p3/Z, [x21, x7]\n" + ".inst 0x454d1042 // ssublb z2.h, z2.b, z13.b\n" + ".inst 0x455a1ab5 // usublb z21.h, z21.b, z26.b\n" + "ld1b { z31.h }, p3/Z, [x20, x7]\n" + "ldr x9, [%x[params], %[offsetof_Params_requant_muls]]\n" + ".inst 0x455a1ad6 // usublb z22.h, z22.b, z26.b\n" + ".inst 0x455a196b // usublb z11.h, z11.b, z26.b\n" + "ldr x28, [%x[params], %[offsetof_Params_requant_shifts]]\n" "str x12, [%x[params], %[offsetof_Params_bias]]\n" - ".inst 0x45571b9c // usublb z28.h, z28.b, z23.b\n" - ".inst 0x45571b7b // usublb z27.h, z27.b, z23.b\n" - ".inst 0x45571b5a // usublb z26.h, z26.b, z23.b\n" - ".inst 0x45571b39 // usublb z25.h, z25.b, z23.b\n" - ".inst 0x45571b18 // usublb z24.h, z24.b, z23.b\n" + ".inst 0x455a1a94 // usublb z20.h, z20.b, z26.b\n" + ".inst 0x455a1b7b // usublb z27.h, z27.b, z26.b\n" + ".inst 0x455a1b9c // usublb z28.h, z28.b, z26.b\n" + ".inst 0x455a1a10 // usublb z16.h, z16.b, z26.b\n" + ".inst 0x455a1bff // usublb z31.h, z31.b, z26.b\n" "1:" // Loop - ".inst 0x448843ed // smlalb z13.s, p4/M, z31.h, z8.h\n" - ".inst 0x448847f1 // smlalt z17.s, p4/M, z31.h, z8.h\n" - "ldr x25, [x11, #0x40]\n" - "ldr x24, [x11, #0x48]\n" - ".inst 0x448643e9 // smlalb z9.s, p4/M, z31.h, z6.h\n" - ".inst 0x448647ea // smlalt z10.s, p4/M, z31.h, z6.h\n" - "ldr x22, [x11, #0x50]\n" - "ldr x20, [x11, #0x58]\n" - ".inst 0x448043cd // smlalb z13.s, p4/M, z30.h, z0.h\n" - ".inst 0x448047d1 // smlalt z17.s, p4/M, z30.h, z0.h\n" - "ldr x23, [x11, #0x78]\n" - "ldr x21, [x11, #0x60]\n" - ".inst 0x44814389 // smlalb z9.s, p4/M, z28.h, z1.h\n" - ".inst 0x4481478a // smlalt z10.s, p4/M, z28.h, z1.h\n" - "ld1b { z28.h }, p3/Z, [x24, x7]\n" - ".inst 0x45571b9c // usublb z28.h, z28.b, z23.b\n" - ".inst 0x448143ad // smlalb z13.s, p4/M, z29.h, z1.h\n" - ".inst 0x448147b1 // smlalt z17.s, p4/M, z29.h, z1.h\n" - "ld1b { z29.h }, p3/Z, [x25, x7]\n" - ".inst 0x45571bbd // usublb z29.h, z29.b, z23.b\n" - ".inst 0x44824369 // smlalb z9.s, p4/M, z27.h, z2.h\n" - ".inst 0x4482476a // smlalt z10.s, p4/M, z27.h, z2.h\n" - "ld1b { z27.h }, p3/Z, [x22, x7]\n" - ".inst 0x45571b7b // usublb z27.h, z27.b, z23.b\n" - ".inst 0x4483434d // smlalb z13.s, p4/M, z26.h, z3.h\n" - ".inst 0x44834751 // smlalt z17.s, p4/M, z26.h, z3.h\n" - "ld1b { z26.h }, p3/Z, [x20, x7]\n" - ".inst 0x45571b5a // usublb z26.h, z26.b, z23.b\n" - ".inst 0x44804309 // smlalb z9.s, p4/M, z24.h, z0.h\n" - ".inst 0x4480470a // smlalt z10.s, p4/M, z24.h, z0.h\n" - "ldr x22, [x11, #0x80]\n" - "ldr x20, [x11, #0x68]\n" - ".inst 0x4484432d // smlalb z13.s, p4/M, z25.h, z4.h\n" - ".inst 0x44844731 // smlalt z17.s, p4/M, z25.h, z4.h\n" - "ld1b { z25.h }, p3/Z, [x21, x7]\n" - ".inst 0x45571b39 // usublb z25.h, z25.b, z23.b\n" - ".inst 0x448443a9 // smlalb z9.s, p4/M, z29.h, z4.h\n" - ".inst 0x448447aa // smlalt z10.s, p4/M, z29.h, z4.h\n" - "ldr x21, [x11, #0x88]\n" + ".inst 0x448242a8 // smlalb z8.s, p4/M, z21.h, z2.h\n" + "ldr x21, [x11, #0x58]\n" + "ldr x20, [x11, #0x78]\n" + ".inst 0x448246b8 // smlalt z24.s, p4/M, z21.h, z2.h\n" + ".inst 0x449942c8 // smlalb z8.s, p4/M, z22.h, z25.h\n" + "ld1b { z17.h }, p3/Z, [x21, x7]\n" "ld1b { z29.h }, p3/Z, [x20, x7]\n" - ".inst 0x4482430d // smlalb z13.s, p4/M, z24.h, z2.h\n" - ".inst 0x44824711 // smlalt z17.s, p4/M, z24.h, z2.h\n" - "ldr x20, [x11, #0x70]\n" - ".inst 0x45571bbd // usublb z29.h, z29.b, z23.b\n" - ".inst 0x44854389 // smlalb z9.s, p4/M, z28.h, z5.h\n" - ".inst 0x4485478a // smlalt z10.s, p4/M, z28.h, z5.h\n" - "ld1b { z28.h }, p3/Z, [x22, x7]\n" - ".inst 0x45571b9c // usublb z28.h, z28.b, z23.b\n" - ".inst 0x448243eb // smlalb z11.s, p4/M, z31.h, z2.h\n" - ".inst 0x448247f6 // smlalt z22.s, p4/M, z31.h, z2.h\n" - "ldr x25, [x11, #0x98]\n" - "ld1b { z24.h }, p3/Z, [x20, x7]\n" - ".inst 0x4485436d // smlalb z13.s, p4/M, z27.h, z5.h\n" - ".inst 0x44854771 // smlalt z17.s, p4/M, z27.h, z5.h\n" - ".inst 0x45571b18 // usublb z24.h, z24.b, z23.b\n" - "ldr x24, [x11, #0x90]\n" - ".inst 0x44834369 // smlalb z9.s, p4/M, z27.h, z3.h\n" - ".inst 0x4483476a // smlalt z10.s, p4/M, z27.h, z3.h\n" + ".inst 0x449742b2 // smlalb z18.s, p4/M, z21.h, z23.h\n" + "ldr x21, [x11, #0x60]\n" + "ldr x20, [x11, #0x80]\n" + ".inst 0x448e42af // smlalb z15.s, p4/M, z21.h, z14.h\n" + ".inst 0x449942a5 // smlalb z5.s, p4/M, z21.h, z25.h\n" + ".inst 0x449946d8 // smlalt z24.s, p4/M, z22.h, z25.h\n" + ".inst 0x455a1a31 // usublb z17.h, z17.b, z26.b\n" + ".inst 0x449e4168 // smlalb z8.s, p4/M, z11.h, z30.h\n" + "ld1b { z22.h }, p3/Z, [x21, x7]\n" + ".inst 0x455a1bbd // usublb z29.h, z29.b, z26.b\n" + ".inst 0x449746a0 // smlalt z0.s, p4/M, z21.h, z23.h\n" + ".inst 0x448e46a1 // smlalt z1.s, p4/M, z21.h, z14.h\n" + "ldr x21, [x11, #0x68]\n" + ".inst 0x449946a6 // smlalt z6.s, p4/M, z21.h, z25.h\n" + "ld1b { z21.h }, p3/Z, [x20, x7]\n" + "ldr x20, [x11, #0x88]\n" + ".inst 0x449e4292 // smlalb z18.s, p4/M, z20.h, z30.h\n" + ".inst 0x4484422f // smlalb z15.s, p4/M, z17.h, z4.h\n" + ".inst 0x448a43a5 // smlalb z5.s, p4/M, z29.h, z10.h\n" + ".inst 0x455a1ad6 // usublb z22.h, z22.b, z26.b\n" + "ldr x22, [x11, #0x40]\n" + ".inst 0x449e4578 // smlalt z24.s, p4/M, z11.h, z30.h\n" + ".inst 0x455a1ab5 // usublb z21.h, z21.b, z26.b\n" + ".inst 0x44844388 // smlalb z8.s, p4/M, z28.h, z4.h\n" + "ld1b { z11.h }, p3/Z, [x21, x7]\n" + ".inst 0x449e4680 // smlalt z0.s, p4/M, z20.h, z30.h\n" + "ld1b { z20.h }, p3/Z, [x20, x7]\n" + ".inst 0x44844621 // smlalt z1.s, p4/M, z17.h, z4.h\n" + "ldr x21, [x11, #0x70]\n" + ".inst 0x448a47a6 // smlalt z6.s, p4/M, z29.h, z10.h\n" + "ldr x20, [x11, #0x98]\n" + ".inst 0x448e4372 // smlalb z18.s, p4/M, z27.h, z14.h\n" + "ldr x23, [x11, #0x50]\n" + ".inst 0x449942cf // smlalb z15.s, p4/M, z22.h, z25.h\n" + ".inst 0x449e42a5 // smlalb z5.s, p4/M, z21.h, z30.h\n" + ".inst 0x455a196b // usublb z11.h, z11.b, z26.b\n" + "ld1b { z17.h }, p3/Z, [x22, x7]\n" + ".inst 0x44844798 // smlalt z24.s, p4/M, z28.h, z4.h\n" + ".inst 0x455a1a94 // usublb z20.h, z20.b, z26.b\n" + ".inst 0x448a4208 // smlalb z8.s, p4/M, z16.h, z10.h\n" + "ld1b { z29.h }, p3/Z, [x21, x7]\n" + "ld1b { z28.h }, p3/Z, [x20, x7]\n" + ".inst 0x448e4760 // smlalt z0.s, p4/M, z27.h, z14.h\n" + "ldr x22, [x11, #0x48]\n" + ".inst 0x449946c1 // smlalt z1.s, p4/M, z22.h, z25.h\n" + ".inst 0x449e46a6 // smlalt z6.s, p4/M, z21.h, z30.h\n" + "ldr x21, [x11, #0x90]\n" + "ldr x20, [x11, #0xa8]\n" + ".inst 0x449943f2 // smlalb z18.s, p4/M, z31.h, z25.h\n" "ld1b { z27.h }, p3/Z, [x23, x7]\n" - ".inst 0x45571b7b // usublb z27.h, z27.b, z23.b\n" - ".inst 0x448043f5 // smlalb z21.s, p4/M, z31.h, z0.h\n" - ".inst 0x4483434b // smlalb z11.s, p4/M, z26.h, z3.h\n" - "ldr x23, [x11, #0xa8]\n" - "ldr x20, [x11, #0xa0]\n" - ".inst 0x44834756 // smlalt z22.s, p4/M, z26.h, z3.h\n" - ".inst 0x448047f2 // smlalt z18.s, p4/M, z31.h, z0.h\n" - "ld1b { z26.h }, p3/Z, [x21, x7]\n" - ".inst 0x45571b5a // usublb z26.h, z26.b, z23.b\n" - ".inst 0x44844375 // smlalb z21.s, p4/M, z27.h, z4.h\n" - ".inst 0x4480432b // smlalb z11.s, p4/M, z25.h, z0.h\n" - "ldr x22, [x11, #0xb0]\n" - "ldr x21, [x11, #0xb8]\n" - ".inst 0x44804736 // smlalt z22.s, p4/M, z25.h, z0.h\n" - ".inst 0x44844772 // smlalt z18.s, p4/M, z27.h, z4.h\n" - "ld1b { z27.h }, p3/Z, [x20, x7]\n" - ".inst 0x45571b7b // usublb z27.h, z27.b, z23.b\n" - ".inst 0x44814395 // smlalb z21.s, p4/M, z28.h, z1.h\n" - ".inst 0x4486432d // smlalb z13.s, p4/M, z25.h, z6.h\n" - "ldr x20, [x11, #0xc0]\n" - "ld1w { z31.s }, p2/Z, [x27]\n" - ".inst 0x44864731 // smlalt z17.s, p4/M, z25.h, z6.h\n" - ".inst 0x448443ab // smlalb z11.s, p4/M, z29.h, z4.h\n" - "ld1b { z25.h }, p3/Z, [x24, x7]\n" - ".inst 0x45571b39 // usublb z25.h, z25.b, z23.b\n" - ".inst 0x448447b6 // smlalt z22.s, p4/M, z29.h, z4.h\n" - "ld1b { z29.h }, p3/Z, [x25, x7]\n" - ".inst 0x44814792 // smlalt z18.s, p4/M, z28.h, z1.h\n" - ".inst 0x45571bbd // usublb z29.h, z29.b, z23.b\n" - ".inst 0x44854355 // smlalb z21.s, p4/M, z26.h, z5.h\n" - ".inst 0x4487430d // smlalb z13.s, p4/M, z24.h, z7.h\n" - "ld1w { z20.s }, p1/Z, [x27, #1, MUL VL]\n" - "uzp1 z19.s, z31.s, z20.s\n" - ".inst 0x44874711 // smlalt z17.s, p4/M, z24.h, z7.h\n" - ".inst 0x4481430b // smlalb z11.s, p4/M, z24.h, z1.h\n" - "uzp2 z30.s, z31.s, z20.s\n" - "ld1w { z31.s }, p2/Z, [x26]\n" - ".inst 0x44814716 // smlalt z22.s, p4/M, z24.h, z1.h\n" - "ld1b { z24.h }, p3/Z, [x23, x7]\n" - ".inst 0x44854752 // smlalt z18.s, p4/M, z26.h, z5.h\n" - ".inst 0x45571b18 // usublb z24.h, z24.b, z23.b\n" - ".inst 0x448243b5 // smlalb z21.s, p4/M, z29.h, z2.h\n" - "ld1b { z26.h }, p3/Z, [x22, x7]\n" - ".inst 0x448247b2 // smlalt z18.s, p4/M, z29.h, z2.h\n" - ".inst 0x45571b5a // usublb z26.h, z26.b, z23.b\n" - ".inst 0x4486432b // smlalb z11.s, p4/M, z25.h, z6.h\n" - ".inst 0x44834315 // smlalb z21.s, p4/M, z24.h, z3.h\n" - "ld1w { z20.s }, p1/Z, [x26, #1, MUL VL]\n" - "uzp1 z1.s, z31.s, z20.s\n" - ".inst 0x44874389 // smlalb z9.s, p4/M, z28.h, z7.h\n" - ".inst 0x4487478a // smlalt z10.s, p4/M, z28.h, z7.h\n" - ".inst 0x04b375ad // sqrdmulh z13.s, z13.s, z19.s\n" - "whilelt p0.h, x10, x8\n" - ".inst 0x44864736 // smlalt z22.s, p4/M, z25.h, z6.h\n" + ".inst 0x448a416f // smlalb z15.s, p4/M, z11.h, z10.h\n" + ".inst 0x44834285 // smlalb z5.s, p4/M, z20.h, z3.h\n" + ".inst 0x455a1a31 // usublb z17.h, z17.b, z26.b\n" + ".inst 0x448a4618 // smlalt z24.s, p4/M, z16.h, z10.h\n" + ".inst 0x455a1bbd // usublb z29.h, z29.b, z26.b\n" + ".inst 0x448e43e8 // smlalb z8.s, p4/M, z31.h, z14.h\n" + "ld1b { z16.h }, p3/Z, [x22, x7]\n" + ".inst 0x455a1b9c // usublb z28.h, z28.b, z26.b\n" + ".inst 0x449947e0 // smlalt z0.s, p4/M, z31.h, z25.h\n" "ld1b { z25.h }, p3/Z, [x21, x7]\n" - ".inst 0x44834712 // smlalt z18.s, p4/M, z24.h, z3.h\n" - ".inst 0x45571b39 // usublb z25.h, z25.b, z23.b\n" - ".inst 0x4487436b // smlalb z11.s, p4/M, z27.h, z7.h\n" - ".inst 0x44874355 // smlalb z21.s, p4/M, z26.h, z7.h\n" - "uzp2 z31.s, z31.s, z20.s\n" - "inch x17\n" - ".inst 0x448843a9 // smlalb z9.s, p4/M, z29.h, z8.h\n" - ".inst 0x448847aa // smlalt z10.s, p4/M, z29.h, z8.h\n" - "ld1b { z29.h }, p3/Z, [x20, x7]\n" - ".inst 0x45571bbd // usublb z29.h, z29.b, z23.b\n" - ".inst 0x44874776 // smlalt z22.s, p4/M, z27.h, z7.h\n" - ".inst 0x44874752 // smlalt z18.s, p4/M, z26.h, z7.h\n" - "and z0.d, z13.d, z1.d\n" + ".inst 0x448a4561 // smlalt z1.s, p4/M, z11.h, z10.h\n" + "ld1b { z11.h }, p3/Z, [x20, x7]\n" + ".inst 0x455a1b7b // usublb z27.h, z27.b, z26.b\n" + ".inst 0x44834686 // smlalt z6.s, p4/M, z20.h, z3.h\n" + "ldr x21, [x11, #0xa0]\n" + "ldr x20, [x11, #0xb0]\n" + ".inst 0x448a4232 // smlalb z18.s, p4/M, z17.h, z10.h\n" + ".inst 0x449e43af // smlalb z15.s, p4/M, z29.h, z30.h\n" + ".inst 0x455a1a10 // usublb z16.h, z16.b, z26.b\n" + ".inst 0x448e4385 // smlalb z5.s, p4/M, z28.h, z14.h\n" + ".inst 0x448e47f8 // smlalt z24.s, p4/M, z31.h, z14.h\n" + ".inst 0x455a1b39 // usublb z25.h, z25.b, z26.b\n" + "ld1b { z20.h }, p3/Z, [x21, x7]\n" + ".inst 0x455a196b // usublb z11.h, z11.b, z26.b\n" + ".inst 0x44834368 // smlalb z8.s, p4/M, z27.h, z3.h\n" + "ld1b { z31.h }, p3/Z, [x20, x7]\n" + ".inst 0x448a4620 // smlalt z0.s, p4/M, z17.h, z10.h\n" + ".inst 0x449e47a1 // smlalt z1.s, p4/M, z29.h, z30.h\n" + ".inst 0x448e4786 // smlalt z6.s, p4/M, z28.h, z14.h\n" + "ldr x20, [x11, #0xb8]\n" + ".inst 0x455a1a94 // usublb z20.h, z20.b, z26.b\n" + ".inst 0x44834212 // smlalb z18.s, p4/M, z16.h, z3.h\n" + ".inst 0x4497432f // smlalb z15.s, p4/M, z25.h, z23.h\n" + ".inst 0x455a1bff // usublb z31.h, z31.b, z26.b\n" + "ld1b { z30.h }, p3/Z, [x20, x7]\n" + ".inst 0x44844165 // smlalb z5.s, p4/M, z11.h, z4.h\n" + ".inst 0x44834778 // smlalt z24.s, p4/M, z27.h, z3.h\n" + "ldr x20, [x11, #0xc0]\n" + "ld1w { z17.s }, p2/Z, [x9]\n" + ".inst 0x449742c8 // smlalb z8.s, p4/M, z22.h, z23.h\n" + ".inst 0x44834600 // smlalt z0.s, p4/M, z16.h, z3.h\n" + "ld1w { z14.s }, p1/Z, [x9, #1, MUL VL]\n" + ".inst 0x455a1bde // usublb z30.h, z30.b, z26.b\n" + ".inst 0x44974721 // smlalt z1.s, p4/M, z25.h, z23.h\n" + ".inst 0x44844566 // smlalt z6.s, p4/M, z11.h, z4.h\n" + "ld1b { z25.h }, p3/Z, [x20, x7]\n" + "uzp1 z10.s, z17.s, z14.s\n" + ".inst 0x44844372 // smlalb z18.s, p4/M, z27.h, z4.h\n" + ".inst 0x4487428f // smlalb z15.s, p4/M, z20.h, z7.h\n" + "uzp2 z14.s, z17.s, z14.s\n" + "ld1w { z17.s }, p2/Z, [x28]\n" + ".inst 0x448743e5 // smlalb z5.s, p4/M, z31.h, z7.h\n" + ".inst 0x449746d8 // smlalt z24.s, p4/M, z22.h, z23.h\n" + "ld1w { z16.s }, p1/Z, [x28, #1, MUL VL]\n" + ".inst 0x455a1b39 // usublb z25.h, z25.b, z26.b\n" + ".inst 0x448743a8 // smlalb z8.s, p4/M, z29.h, z7.h\n" + ".inst 0x44844760 // smlalt z0.s, p4/M, z27.h, z4.h\n" + "uzp1 z4.s, z17.s, z16.s\n" "inch x7\n" - ".inst 0x4485430b // smlalb z11.s, p4/M, z24.h, z5.h\n" - ".inst 0x44864335 // smlalb z21.s, p4/M, z25.h, z6.h\n" - ".inst 0x04be7631 // sqrdmulh z17.s, z17.s, z30.s\n" + ".inst 0x44874681 // smlalt z1.s, p4/M, z20.h, z7.h\n" + ".inst 0x448747e6 // smlalt z6.s, p4/M, z31.h, z7.h\n" + ".inst 0x04aa7508 // sqrdmulh z8.s, z8.s, z10.s\n" + "whilelt p0.h, x10, x8\n" + ".inst 0x448742b2 // smlalb z18.s, p4/M, z21.h, z7.h\n" + ".inst 0x4483416f // smlalb z15.s, p4/M, z11.h, z3.h\n" + "uzp2 z22.s, z17.s, z16.s\n" "mov x20, x7\n" - ".inst 0x44854716 // smlalt z22.s, p4/M, z24.h, z5.h\n" - ".inst 0x44864732 // smlalt z18.s, p4/M, z25.h, z6.h\n" - "asr z0.s, z0.s, #0x1f\n" + ".inst 0x449743c5 // smlalb z5.s, p4/M, z30.h, z23.h\n" + ".inst 0x448747b8 // smlalt z24.s, p4/M, z29.h, z7.h\n" + "and z17.d, z8.d, z4.d\n" + "inch x17\n" + ".inst 0x448746a0 // smlalt z0.s, p4/M, z21.h, z7.h\n" + ".inst 0x44834561 // smlalt z1.s, p4/M, z11.h, z3.h\n" + ".inst 0x04ae7718 // sqrdmulh z24.s, z24.s, z14.s\n" "incw x20\n" - ".inst 0x4488432b // smlalb z11.s, p4/M, z25.h, z8.h\n" - ".inst 0x448843b5 // smlalb z21.s, p4/M, z29.h, z8.h\n" - "and z20.d, z17.d, z31.d\n" + ".inst 0x449747c6 // smlalt z6.s, p4/M, z30.h, z23.h\n" + ".inst 0x44824392 // smlalb z18.s, p4/M, z28.h, z2.h\n" + "asr z17.s, z17.s, #0x1f\n" "whilelt p2.s, x7, x8\n" - ".inst 0x44884736 // smlalt z22.s, p4/M, z25.h, z8.h\n" - ".inst 0x448847b2 // smlalt z18.s, p4/M, z29.h, z8.h\n" - ".inst 0x04b37529 // sqrdmulh z9.s, z9.s, z19.s\n" + ".inst 0x448243cf // smlalb z15.s, p4/M, z30.h, z2.h\n" + ".inst 0x44824325 // smlalb z5.s, p4/M, z25.h, z2.h\n" + "and z16.d, z24.d, z22.d\n" "whilelt p1.s, x20, x8\n" - ".inst 0x04b3756b // sqrdmulh z11.s, z11.s, z19.s\n" - ".inst 0x04b376b5 // sqrdmulh z21.s, z21.s, z19.s\n" - "ldr x12, [%x[params], %[offsetof_Params_bias]]\n" + ".inst 0x44824780 // smlalt z0.s, p4/M, z28.h, z2.h\n" + ".inst 0x448247c1 // smlalt z1.s, p4/M, z30.h, z2.h\n" + ".inst 0x04aa7652 // sqrdmulh z18.s, z18.s, z10.s\n" + "ldr x20, [%x[params], %[offsetof_Params_bias]]\n" + ".inst 0x44824726 // smlalt z6.s, p4/M, z25.h, z2.h\n" + ".inst 0x04aa75ef // sqrdmulh z15.s, z15.s, z10.s\n" "whilelt p3.h, x7, x8\n" - "sqadd z13.s, z13.s, z0.s\n" - "asr z20.s, z20.s, #0x1f\n" - ".inst 0x4482902d // srshl z13.s, p4/M, z13.s, z1.s\n" - "addvl x27, x27, #2\n" - "and z19.d, z9.d, z1.d\n" - ".inst 0x04be754a // sqrdmulh z10.s, z10.s, z30.s\n" - "addvl x26, x26, #2\n" - "and z2.d, z11.d, z1.d\n" - ".inst 0x04be76d6 // sqrdmulh z22.s, z22.s, z30.s\n" - "and z0.d, z21.d, z1.d\n" - ".inst 0x04be7652 // sqrdmulh z18.s, z18.s, z30.s\n" - "sqadd z17.s, z17.s, z20.s\n" - "asr z19.s, z19.s, #0x1f\n" - ".inst 0x448293f1 // srshl z17.s, p4/M, z17.s, z31.s\n" - "and z3.d, z10.d, z31.d\n" - "asr z2.s, z2.s, #0x1f\n" - "and z26.d, z22.d, z31.d\n" - "asr z0.s, z0.s, #0x1f\n" - "and z20.d, z18.d, z31.d\n" - "sqadd z9.s, z9.s, z19.s\n" - ".inst 0x44829029 // srshl z9.s, p4/M, z9.s, z1.s\n" - "asr z3.s, z3.s, #0x1f\n" - "sqadd z11.s, z11.s, z2.s\n" - ".inst 0x4482902b // srshl z11.s, p4/M, z11.s, z1.s\n" - "asr z26.s, z26.s, #0x1f\n" - "sqadd z21.s, z21.s, z0.s\n" - ".inst 0x44829035 // srshl z21.s, p4/M, z21.s, z1.s\n" + "addvl x9, x9, #2\n" + ".inst 0x04aa74a5 // sqrdmulh z5.s, z5.s, z10.s\n" + "sqadd z8.s, z8.s, z17.s\n" + ".inst 0x44829088 // srshl z8.s, p4/M, z8.s, z4.s\n" + "addvl x28, x28, #2\n" + "asr z16.s, z16.s, #0x1f\n" + "and z21.d, z18.d, z4.d\n" + ".inst 0x04ae7400 // sqrdmulh z0.s, z0.s, z14.s\n" + "and z20.d, z15.d, z4.d\n" + ".inst 0x04ae7421 // sqrdmulh z1.s, z1.s, z14.s\n" + "and z28.d, z5.d, z4.d\n" + ".inst 0x04ae74c6 // sqrdmulh z6.s, z6.s, z14.s\n" + "sqadd z24.s, z24.s, z16.s\n" + ".inst 0x448292d8 // srshl z24.s, p4/M, z24.s, z22.s\n" + "asr z21.s, z21.s, #0x1f\n" + "and z25.d, z0.d, z22.d\n" "asr z20.s, z20.s, #0x1f\n" - "sqadd z10.s, z10.s, z3.s\n" - ".inst 0x448293ea // srshl z10.s, p4/M, z10.s, z31.s\n" - "sqadd z22.s, z22.s, z26.s\n" - "sqadd z18.s, z18.s, z20.s\n" - ".inst 0x448293f6 // srshl z22.s, p4/M, z22.s, z31.s\n" - ".inst 0x448293f2 // srshl z18.s, p4/M, z18.s, z31.s\n" - ".inst 0x453041ad // sqxtnb z13.h, z13.s\n" - ".inst 0x45304129 // sqxtnb z9.h, z9.s\n" - ".inst 0x4530416b // sqxtnb z11.h, z11.s\n" - ".inst 0x453042b5 // sqxtnb z21.h, z21.s\n" - ".inst 0x4530462d // sqxtnt z13.h, z17.s\n" - ".inst 0x45304549 // sqxtnt z9.h, z10.s\n" - ".inst 0x453046cb // sqxtnt z11.h, z22.s\n" - ".inst 0x45304655 // sqxtnt z21.h, z18.s\n" - "sqadd z13.h, z13.h, z14.h\n" - "sqadd z9.h, z9.h, z14.h\n" - "smax z13.h, p4/M, z13.h, z16.h\n" - "smax z9.h, p4/M, z9.h, z16.h\n" - "sqadd z11.h, z11.h, z14.h\n" - "sqadd z21.h, z21.h, z14.h\n" - "smax z11.h, p4/M, z11.h, z16.h\n" - "smax z21.h, p4/M, z21.h, z16.h\n" - "smin z13.h, p4/M, z13.h, z15.h\n" - "smin z9.h, p4/M, z9.h, z15.h\n" - "st1b { z13.h }, p0, [x16, x10]\n" - "smin z11.h, p4/M, z11.h, z15.h\n" - "smin z21.h, p4/M, z21.h, z15.h\n" - "st1b { z9.h }, p0, [x15, x10]\n" - "st1b { z11.h }, p0, [x14, x10]\n" - "st1b { z21.h }, p0, [x13, x10]\n" - "ld1sb { z0.h }, p4/Z, [x17]\n" - "ld1sb { z1.h }, p4/Z, [x17, #1, MUL VL]\n" + "and z17.d, z1.d, z22.d\n" + "asr z28.s, z28.s, #0x1f\n" + "and z16.d, z6.d, z22.d\n" + "sqadd z18.s, z18.s, z21.s\n" + "asr z25.s, z25.s, #0x1f\n" + ".inst 0x44829092 // srshl z18.s, p4/M, z18.s, z4.s\n" + "sqadd z15.s, z15.s, z20.s\n" + "asr z17.s, z17.s, #0x1f\n" + ".inst 0x4482908f // srshl z15.s, p4/M, z15.s, z4.s\n" + "sqadd z5.s, z5.s, z28.s\n" + "asr z16.s, z16.s, #0x1f\n" + ".inst 0x44829085 // srshl z5.s, p4/M, z5.s, z4.s\n" + "sqadd z0.s, z0.s, z25.s\n" + "sqadd z1.s, z1.s, z17.s\n" + ".inst 0x448292c0 // srshl z0.s, p4/M, z0.s, z22.s\n" + ".inst 0x448292c1 // srshl z1.s, p4/M, z1.s, z22.s\n" + "sqadd z6.s, z6.s, z16.s\n" + ".inst 0x45304108 // sqxtnb z8.h, z8.s\n" + ".inst 0x448292c6 // srshl z6.s, p4/M, z6.s, z22.s\n" + ".inst 0x45304252 // sqxtnb z18.h, z18.s\n" + ".inst 0x453041ef // sqxtnb z15.h, z15.s\n" + ".inst 0x453040a5 // sqxtnb z5.h, z5.s\n" + ".inst 0x45304708 // sqxtnt z8.h, z24.s\n" + ".inst 0x45304412 // sqxtnt z18.h, z0.s\n" + ".inst 0x4530442f // sqxtnt z15.h, z1.s\n" + ".inst 0x453044c5 // sqxtnt z5.h, z6.s\n" + "sqadd z8.h, z8.h, z19.h\n" + "smax z8.h, p4/M, z8.h, z12.h\n" + "smin z8.h, p4/M, z8.h, z9.h\n" + "sqadd z18.h, z18.h, z19.h\n" + "sqadd z15.h, z15.h, z19.h\n" + "smax z18.h, p4/M, z18.h, z12.h\n" + "smax z15.h, p4/M, z15.h, z12.h\n" + "sqadd z5.h, z5.h, z19.h\n" + "smax z5.h, p4/M, z5.h, z12.h\n" + "smin z18.h, p4/M, z18.h, z9.h\n" + "st1b { z8.h }, p0, [x16, x10]\n" + "smin z15.h, p4/M, z15.h, z9.h\n" + "smin z5.h, p4/M, z5.h, z9.h\n" + "st1b { z18.h }, p0, [x15, x10]\n" + "st1b { z15.h }, p0, [x14, x10]\n" + "st1b { z5.h }, p0, [x13, x10]\n" + "ld1sb { z25.h }, p4/Z, [x17]\n" + "ld1sb { z30.h }, p4/Z, [x17, #1, MUL VL]\n" "inch x10\n" - "ld1sb { z2.h }, p4/Z, [x17, #2, MUL VL]\n" - "ld1sb { z3.h }, p4/Z, [x17, #3, MUL VL]\n" - ".inst 0x454c1000 // ssublb z0.h, z0.b, z12.b\n" - ".inst 0x454c1021 // ssublb z1.h, z1.b, z12.b\n" - "ld1sb { z4.h }, p4/Z, [x17, #4, MUL VL]\n" - "ld1sb { z5.h }, p4/Z, [x17, #5, MUL VL]\n" - ".inst 0x454c1042 // ssublb z2.h, z2.b, z12.b\n" - ".inst 0x454c1063 // ssublb z3.h, z3.b, z12.b\n" - "ld1sb { z6.h }, p4/Z, [x17, #6, MUL VL]\n" + "ld1sb { z14.h }, p4/Z, [x17, #2, MUL VL]\n" + "ld1sb { z4.h }, p4/Z, [x17, #3, MUL VL]\n" + ".inst 0x454d1339 // ssublb z25.h, z25.b, z13.b\n" + ".inst 0x454d13de // ssublb z30.h, z30.b, z13.b\n" + "ld1sb { z10.h }, p4/Z, [x17, #4, MUL VL]\n" + "ld1sb { z3.h }, p4/Z, [x17, #5, MUL VL]\n" + ".inst 0x454d11ce // ssublb z14.h, z14.b, z13.b\n" + ".inst 0x454d1084 // ssublb z4.h, z4.b, z13.b\n" + "ld1sb { z23.h }, p4/Z, [x17, #6, MUL VL]\n" "ld1sb { z7.h }, p4/Z, [x17, #7, MUL VL]\n" "inch x17, ALL, MUL #8\n" - ".inst 0x454c1084 // ssublb z4.h, z4.b, z12.b\n" - "ld1w { z18.s }, p2/Z, [x12]\n" - "ld1w { z8.s }, p1/Z, [x12, #1, MUL VL]\n" - "uzp1 z13.s, z18.s, z8.s\n" - "uzp2 z17.s, z18.s, z8.s\n" - "ld1sb { z8.h }, p4/Z, [x17]\n" - "ldp x9, x28, [x11, #0x0]\n" - "addvl x12, x12, #2\n" - "str x12, [%x[params], %[offsetof_Params_bias]]\n" + ".inst 0x454d114a // ssublb z10.h, z10.b, z13.b\n" + "ld1w { z17.s }, p2/Z, [x20]\n" + "ld1w { z16.s }, p1/Z, [x20, #1, MUL VL]\n" + "uzp1 z8.s, z17.s, z16.s\n" + "uzp2 z24.s, z17.s, z16.s\n" + "ld1sb { z2.h }, p4/Z, [x17]\n" + "ldp x27, x26, [x11, #0x0]\n" + "addvl x20, x20, #2\n" + "str x20, [%x[params], %[offsetof_Params_bias]]\n" "ldp x25, x24, [x11, #0x10]\n" "ldp x23, x22, [x11, #0x20]\n" - "mov z9.d, z13.d\n" - "mov z10.d, z17.d\n" + "mov z18.d, z8.d\n" + "mov z0.d, z24.d\n" "ldp x21, x20, [x11, #0x30]\n" - "ld1b { z31.h }, p3/Z, [x9, x7]\n" - "mov z11.d, z13.d\n" - "mov z22.d, z17.d\n" - "ld1b { z30.h }, p3/Z, [x28, x7]\n" - "ld1b { z29.h }, p3/Z, [x25, x7]\n" - "mov z21.d, z13.d\n" - "mov z18.d, z17.d\n" - "ld1b { z28.h }, p3/Z, [x24, x7]\n" + "ld1b { z21.h }, p3/Z, [x27, x7]\n" + "mov z15.d, z8.d\n" + "mov z1.d, z24.d\n" + "ld1b { z22.h }, p3/Z, [x26, x7]\n" + "ld1b { z11.h }, p3/Z, [x25, x7]\n" + "mov z5.d, z8.d\n" + "mov z6.d, z24.d\n" + "ld1b { z20.h }, p3/Z, [x24, x7]\n" "ld1b { z27.h }, p3/Z, [x23, x7]\n" - ".inst 0x454c10a5 // ssublb z5.h, z5.b, z12.b\n" - ".inst 0x454c10c6 // ssublb z6.h, z6.b, z12.b\n" - "ld1b { z26.h }, p3/Z, [x22, x7]\n" - "ld1b { z25.h }, p3/Z, [x21, x7]\n" - ".inst 0x454c10e7 // ssublb z7.h, z7.b, z12.b\n" - ".inst 0x454c1108 // ssublb z8.h, z8.b, z12.b\n" - "ld1b { z24.h }, p3/Z, [x20, x7]\n" - ".inst 0x45571bff // usublb z31.h, z31.b, z23.b\n" - ".inst 0x45571bde // usublb z30.h, z30.b, z23.b\n" - ".inst 0x45571bbd // usublb z29.h, z29.b, z23.b\n" - ".inst 0x45571b9c // usublb z28.h, z28.b, z23.b\n" - ".inst 0x45571b7b // usublb z27.h, z27.b, z23.b\n" - ".inst 0x45571b5a // usublb z26.h, z26.b, z23.b\n" - ".inst 0x45571b39 // usublb z25.h, z25.b, z23.b\n" - ".inst 0x45571b18 // usublb z24.h, z24.b, z23.b\n" + ".inst 0x454d1063 // ssublb z3.h, z3.b, z13.b\n" + ".inst 0x454d12f7 // ssublb z23.h, z23.b, z13.b\n" + "ld1b { z28.h }, p3/Z, [x22, x7]\n" + "ld1b { z16.h }, p3/Z, [x21, x7]\n" + ".inst 0x454d10e7 // ssublb z7.h, z7.b, z13.b\n" + ".inst 0x454d1042 // ssublb z2.h, z2.b, z13.b\n" + "ld1b { z31.h }, p3/Z, [x20, x7]\n" + ".inst 0x455a1ab5 // usublb z21.h, z21.b, z26.b\n" + ".inst 0x455a1ad6 // usublb z22.h, z22.b, z26.b\n" + ".inst 0x455a196b // usublb z11.h, z11.b, z26.b\n" + ".inst 0x455a1a94 // usublb z20.h, z20.b, z26.b\n" + ".inst 0x455a1b7b // usublb z27.h, z27.b, z26.b\n" + ".inst 0x455a1b9c // usublb z28.h, z28.b, z26.b\n" + ".inst 0x455a1a10 // usublb z16.h, z16.b, z26.b\n" + ".inst 0x455a1bff // usublb z31.h, z31.b, z26.b\n" "b.any 1b\n" : : [offsetof_Params_bias] "I" (offsetof(Params, bias)), [offsetof_Params_inptrs] "I" (offsetof(Params, inptrs)), [offsetof_Params_n_channels] "I" (offsetof(Params, n_channels)), [offsetof_Params_outptrs] "I" (offsetof(Params, outptrs)), [offsetof_Params_requant] "I" (offsetof(Params, requant)), [offsetof_Params_requant_muls] "I" (offsetof(Params, requant_muls)), [offsetof_Params_requant_shifts] "I" (offsetof(Params, requant_shifts)), [offsetof_Params_weights] "I" (offsetof(Params, weights)), [offsetof_Requantize32_a_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [params] "r" (¶ms) @@ -448,4 +448,4 @@ void sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp index 06ca42eed95874a379f7118c9ac5a3ecec406d28..0ff853ec2dc7b107d11d3ab76162f29f48783e9f 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,14 +22,14 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" #include "src/core/NEON/kernels/arm_conv/depthwise/interleaves/list.hpp" #include #pragma once -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -47,17 +47,16 @@ class sve_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst : public DepthwiseDepthfi constexpr static unsigned int stride_rows = 1; constexpr static unsigned int stride_cols = 1; - arm_gemm::VLType get_vl_type(void) const override { return arm_gemm::VLType::SVE; } - unsigned int get_accumulator_depth_vl(void) const override { return 2; } - sve_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst(const CPUInfo *) : Parent(2, 2, 5, 5, 1, 1) {} - Parent::KernelType kernel = sve_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl; + arm_gemm::VLType get_vl_type(void) const override { return arm_gemm::VLType::SVE; } + Parent::KernelType kernel = sve_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl; Parent::KernelType get_kernel(void) const override { return kernel; } + unsigned int get_accumulator_depth_vl(void) const override { return 2; } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp index 9c291ae18649cee77dafb340b516618f37a30aec..f24a258484ac358d4f9e11e71d5be5270db4e5b6 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp @@ -27,7 +27,7 @@ #include #include -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -46,7 +46,7 @@ void sve_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( struct Params { long unsigned int n_channels; - const int8_t *weights; + const void *weights; const int32_t *bias; const arm_gemm::Requantize32 *requant; const int32_t *const requant_muls; @@ -57,7 +57,7 @@ void sve_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( Params( long unsigned int n_channels, const uint8_t *const *inptrs_raw, - const int8_t *const weights, + const void *const weights, const int32_t *const bias, const arm_gemm::Requantize32 &qp, const int32_t *const requant_muls, @@ -111,542 +111,542 @@ void sve_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( requant_muls, requant_shifts, outptrs); __asm__ __volatile__( - "mov x0, #0x0\n" - "mov x24, x0\n" + "mov x2, #0x0\n" + "mov x24, x2\n" "ldr x23, [%x[params], %[offsetof_Params_requant]]\n" - "ldr x1, [%x[params], %[offsetof_Params_n_channels]]\n" + "ldr x3, [%x[params], %[offsetof_Params_n_channels]]\n" "ptrue p4.b\n" "ldr x22, [%x[params], %[offsetof_Params_outptrs]]\n" "incw x24\n" - "ldr x2, [%x[params], %[offsetof_Params_weights]]\n" + "ldr x4, [%x[params], %[offsetof_Params_weights]]\n" "add x21, x23, %[offsetof_Requantize32_a_offset]\n" "add x20, x23, %[offsetof_Requantize32_b_offset]\n" - "ld1rb { z15.b }, p4/Z, [x21]\n" - "ld1rb { z17.b }, p4/Z, [x20]\n" + "ld1rb { z30.b }, p4/Z, [x21]\n" + "ld1rb { z10.b }, p4/Z, [x20]\n" "add x21, x23, %[offsetof_Requantize32_c_offset]\n" "add x20, x23, %[offsetof_Requantize32_minval]\n" - "ld1rh { z12.h }, p4/Z, [x21]\n" - "ld1rh { z13.h }, p4/Z, [x20]\n" + "ld1rh { z15.h }, p4/Z, [x21]\n" + "ld1rh { z12.h }, p4/Z, [x20]\n" "add x20, x23, %[offsetof_Requantize32_maxval]\n" - "ld1rh { z11.h }, p4/Z, [x20]\n" - "ldp x3, x4, [x22, #0x0]\n" - "whilelt p3.h, x0, x1\n" - "ldp x5, x6, [x22, #0x10]\n" - "whilelt p2.s, x0, x1\n" - "whilelt p1.s, x24, x1\n" - "ldr x14, [%x[params], %[offsetof_Params_bias]]\n" - "add x7, %x[params], %[offsetof_Params_inptrs]\n" - "ld1w { z30.s }, p2/Z, [x14]\n" - "ld1w { z16.s }, p1/Z, [x14, #1, MUL VL]\n" - "uzp1 z14.s, z30.s, z16.s\n" - "ld1sb { z0.h }, p4/Z, [x2]\n" - "ld1sb { z1.h }, p4/Z, [x2, #1, MUL VL]\n" - "uzp2 z10.s, z30.s, z16.s\n" - "addvl x14, x14, #2\n" - "ld1sb { z2.h }, p4/Z, [x2, #2, MUL VL]\n" - "ld1sb { z3.h }, p4/Z, [x2, #3, MUL VL]\n" - "mov x8, #0x0\n" - "mov z20.d, z14.d\n" - "ld1sb { z4.h }, p4/Z, [x2, #4, MUL VL]\n" - "ldp x9, x28, [x7, #0x0]\n" - "mov z7.d, z10.d\n" - "mov z8.d, z14.d\n" - "ldp x27, x26, [x7, #0x10]\n" - "ldp x25, x24, [x7, #0x20]\n" - "mov z16.d, z10.d\n" + "ld1rh { z13.h }, p4/Z, [x20]\n" + "ldp x5, x6, [x22, #0x0]\n" + "whilelt p3.h, x2, x3\n" + "ldp x7, x8, [x22, #0x10]\n" + "whilelt p2.s, x2, x3\n" + "whilelt p1.s, x24, x3\n" + "ldr x10, [%x[params], %[offsetof_Params_bias]]\n" + "add x17, %x[params], %[offsetof_Params_inptrs]\n" + "ld1w { z17.s }, p2/Z, [x10]\n" + "ld1w { z16.s }, p1/Z, [x10, #1, MUL VL]\n" + "uzp1 z14.s, z17.s, z16.s\n" + "ld1sb { z26.h }, p4/Z, [x4]\n" + "ld1sb { z8.h }, p4/Z, [x4, #1, MUL VL]\n" + "uzp2 z23.s, z17.s, z16.s\n" + "addvl x10, x10, #2\n" + "ld1sb { z16.h }, p4/Z, [x4, #2, MUL VL]\n" + "ld1sb { z21.h }, p4/Z, [x4, #3, MUL VL]\n" + "mov x16, #0x0\n" "mov z6.d, z14.d\n" - "ldp x23, x22, [x7, #0x30]\n" - "ldp x21, x20, [x7, #0x40]\n" - "mov z5.d, z10.d\n" - ".inst 0x45511000 // ssublb z0.h, z0.b, z17.b\n" - "ld1b { z31.h }, p3/Z, [x9, x0]\n" - "ld1b { z30.h }, p3/Z, [x28, x0]\n" - ".inst 0x45511021 // ssublb z1.h, z1.b, z17.b\n" - ".inst 0x45511042 // ssublb z2.h, z2.b, z17.b\n" - "ld1b { z29.h }, p3/Z, [x27, x0]\n" - "ld1b { z28.h }, p3/Z, [x26, x0]\n" - ".inst 0x45511063 // ssublb z3.h, z3.b, z17.b\n" - ".inst 0x45511084 // ssublb z4.h, z4.b, z17.b\n" - "ld1b { z27.h }, p3/Z, [x25, x0]\n" - "ld1b { z23.h }, p3/Z, [x24, x0]\n" - ".inst 0x454f1bff // usublb z31.h, z31.b, z15.b\n" - ".inst 0x454f1bde // usublb z30.h, z30.b, z15.b\n" - "ld1b { z25.h }, p3/Z, [x23, x0]\n" - "ld1b { z24.h }, p3/Z, [x22, x0]\n" - ".inst 0x454f1bbd // usublb z29.h, z29.b, z15.b\n" - ".inst 0x454f1b9c // usublb z28.h, z28.b, z15.b\n" - "ld1b { z26.h }, p3/Z, [x21, x0]\n" - "ld1b { z22.h }, p3/Z, [x20, x0]\n" - ".inst 0x454f1b7b // usublb z27.h, z27.b, z15.b\n" - ".inst 0x454f1af7 // usublb z23.h, z23.b, z15.b\n" - "ldr x17, [%x[params], %[offsetof_Params_requant_muls]]\n" - "ldr x16, [%x[params], %[offsetof_Params_requant_shifts]]\n" - "str x14, [%x[params], %[offsetof_Params_bias]]\n" - ".inst 0x454f1b39 // usublb z25.h, z25.b, z15.b\n" - ".inst 0x454f1b18 // usublb z24.h, z24.b, z15.b\n" - ".inst 0x454f1b5a // usublb z26.h, z26.b, z15.b\n" - ".inst 0x454f1ad6 // usublb z22.h, z22.b, z15.b\n" + "ld1sb { z17.h }, p4/Z, [x4, #4, MUL VL]\n" + "ldp x9, x28, [x17, #0x0]\n" + "mov z18.d, z23.d\n" + "mov z9.d, z14.d\n" + "ldp x27, x26, [x17, #0x10]\n" + "ldp x25, x24, [x17, #0x20]\n" + "mov z20.d, z23.d\n" + "mov z7.d, z14.d\n" + "ldp x23, x22, [x17, #0x30]\n" + "ldp x21, x20, [x17, #0x40]\n" + "mov z1.d, z23.d\n" + ".inst 0x454a135a // ssublb z26.h, z26.b, z10.b\n" + "ld1b { z22.h }, p3/Z, [x9, x2]\n" + "ld1b { z2.h }, p3/Z, [x28, x2]\n" + ".inst 0x454a1108 // ssublb z8.h, z8.b, z10.b\n" + ".inst 0x454a1210 // ssublb z16.h, z16.b, z10.b\n" + "ld1b { z11.h }, p3/Z, [x27, x2]\n" + "ld1b { z3.h }, p3/Z, [x26, x2]\n" + ".inst 0x454a12b5 // ssublb z21.h, z21.b, z10.b\n" + ".inst 0x454a1231 // ssublb z17.h, z17.b, z10.b\n" + "ld1b { z29.h }, p3/Z, [x25, x2]\n" + "ld1b { z4.h }, p3/Z, [x24, x2]\n" + ".inst 0x455e1ad6 // usublb z22.h, z22.b, z30.b\n" + ".inst 0x455e1842 // usublb z2.h, z2.b, z30.b\n" + "ld1b { z31.h }, p3/Z, [x23, x2]\n" + "ld1b { z0.h }, p3/Z, [x22, x2]\n" + ".inst 0x455e196b // usublb z11.h, z11.b, z30.b\n" + ".inst 0x455e1863 // usublb z3.h, z3.b, z30.b\n" + "ld1b { z19.h }, p3/Z, [x21, x2]\n" + "ld1b { z28.h }, p3/Z, [x20, x2]\n" + ".inst 0x455e1bbd // usublb z29.h, z29.b, z30.b\n" + ".inst 0x455e1884 // usublb z4.h, z4.b, z30.b\n" + "ldr x15, [%x[params], %[offsetof_Params_requant_muls]]\n" + "ldr x14, [%x[params], %[offsetof_Params_requant_shifts]]\n" + "str x10, [%x[params], %[offsetof_Params_bias]]\n" + ".inst 0x455e1bff // usublb z31.h, z31.b, z30.b\n" + ".inst 0x455e1800 // usublb z0.h, z0.b, z30.b\n" + ".inst 0x455e1a73 // usublb z19.h, z19.b, z30.b\n" + ".inst 0x455e1b9c // usublb z28.h, z28.b, z30.b\n" "1:" // Loop - ".inst 0x448043ee // smlalb z14.s, p4/M, z31.h, z0.h\n" - ".inst 0x448047ea // smlalt z10.s, p4/M, z31.h, z0.h\n" - "ldr x20, [x7, #0x50]\n" - "ld1b { z31.h }, p3/Z, [x20, x0]\n" - ".inst 0x448143ce // smlalb z14.s, p4/M, z30.h, z1.h\n" - ".inst 0x448043d4 // smlalb z20.s, p4/M, z30.h, z0.h\n" - "ldr x22, [x7, #0x58]\n" - ".inst 0x454f1bff // usublb z31.h, z31.b, z15.b\n" - ".inst 0x448043a8 // smlalb z8.s, p4/M, z29.h, z0.h\n" - ".inst 0x44804386 // smlalb z6.s, p4/M, z28.h, z0.h\n" - "ldr x21, [x7, #0x60]\n" - "ldr x20, [x7, #0x68]\n" - ".inst 0x448147ca // smlalt z10.s, p4/M, z30.h, z1.h\n" - ".inst 0x448047c7 // smlalt z7.s, p4/M, z30.h, z0.h\n" - "ld1b { z30.h }, p3/Z, [x22, x0]\n" - ".inst 0x454f1bde // usublb z30.h, z30.b, z15.b\n" - ".inst 0x448047b0 // smlalt z16.s, p4/M, z29.h, z0.h\n" - ".inst 0x4482436e // smlalb z14.s, p4/M, z27.h, z2.h\n" - "ldr x25, [x7, #0x70]\n" - "ldr x24, [x7, #0x78]\n" - ".inst 0x44804785 // smlalt z5.s, p4/M, z28.h, z0.h\n" - ".inst 0x44814374 // smlalb z20.s, p4/M, z27.h, z1.h\n" - "ld1sb { z0.h }, p4/Z, [x2, #5, MUL VL]\n" - ".inst 0x45511000 // ssublb z0.h, z0.b, z17.b\n" - ".inst 0x44814388 // smlalb z8.s, p4/M, z28.h, z1.h\n" - ".inst 0x448142e6 // smlalb z6.s, p4/M, z23.h, z1.h\n" - "ldr x15, [x7, #0x80]\n" - "ldr x23, [x7, #0x88]\n" - ".inst 0x4482476a // smlalt z10.s, p4/M, z27.h, z2.h\n" - ".inst 0x44814767 // smlalt z7.s, p4/M, z27.h, z1.h\n" - "ld1b { z27.h }, p3/Z, [x21, x0]\n" - ".inst 0x454f1b7b // usublb z27.h, z27.b, z15.b\n" - ".inst 0x44814790 // smlalt z16.s, p4/M, z28.h, z1.h\n" - ".inst 0x4483432e // smlalb z14.s, p4/M, z25.h, z3.h\n" - "ldr x22, [x7, #0x90]\n" - "ldr x21, [x7, #0x98]\n" - ".inst 0x448146e5 // smlalt z5.s, p4/M, z23.h, z1.h\n" - ".inst 0x44824334 // smlalb z20.s, p4/M, z25.h, z2.h\n" - "ld1sb { z1.h }, p4/Z, [x2, #6, MUL VL]\n" - ".inst 0x45511021 // ssublb z1.h, z1.b, z17.b\n" - ".inst 0x448242e8 // smlalb z8.s, p4/M, z23.h, z2.h\n" - ".inst 0x448243e6 // smlalb z6.s, p4/M, z31.h, z2.h\n" - "ldr x14, [x7, #0xa0]\n" - "ldr x13, [x7, #0xa8]\n" - ".inst 0x4483472a // smlalt z10.s, p4/M, z25.h, z3.h\n" - ".inst 0x44824727 // smlalt z7.s, p4/M, z25.h, z2.h\n" - "ld1b { z25.h }, p3/Z, [x20, x0]\n" - ".inst 0x454f1b39 // usublb z25.h, z25.b, z15.b\n" - ".inst 0x448246f0 // smlalt z16.s, p4/M, z23.h, z2.h\n" - ".inst 0x4484430e // smlalb z14.s, p4/M, z24.h, z4.h\n" - "ldr x12, [x7, #0xb0]\n" - "ldr x20, [x7, #0xb8]\n" - ".inst 0x448247e5 // smlalt z5.s, p4/M, z31.h, z2.h\n" - ".inst 0x44834314 // smlalb z20.s, p4/M, z24.h, z3.h\n" - "ld1sb { z2.h }, p4/Z, [x2, #7, MUL VL]\n" - "inch x2, ALL, MUL #8\n" - ".inst 0x448343e8 // smlalb z8.s, p4/M, z31.h, z3.h\n" - ".inst 0x448343c6 // smlalb z6.s, p4/M, z30.h, z3.h\n" - ".inst 0x45511042 // ssublb z2.h, z2.b, z17.b\n" - "ldr x11, [x7, #0xc0]\n" - ".inst 0x4484470a // smlalt z10.s, p4/M, z24.h, z4.h\n" - ".inst 0x44834707 // smlalt z7.s, p4/M, z24.h, z3.h\n" - "ld1b { z24.h }, p3/Z, [x25, x0]\n" - ".inst 0x454f1b18 // usublb z24.h, z24.b, z15.b\n" - ".inst 0x448347f0 // smlalt z16.s, p4/M, z31.h, z3.h\n" - ".inst 0x448043ae // smlalb z14.s, p4/M, z29.h, z0.h\n" - "ldr x10, [x7, #0xc8]\n" - "ldr x9, [x7, #0xd0]\n" - ".inst 0x448347c5 // smlalt z5.s, p4/M, z30.h, z3.h\n" - ".inst 0x44844374 // smlalb z20.s, p4/M, z27.h, z4.h\n" - "ld1sb { z3.h }, p4/Z, [x2]\n" - ".inst 0x45511063 // ssublb z3.h, z3.b, z17.b\n" - ".inst 0x448443c8 // smlalb z8.s, p4/M, z30.h, z4.h\n" - ".inst 0x44844346 // smlalb z6.s, p4/M, z26.h, z4.h\n" - "ldr x28, [x7, #0xd8]\n" - "ldr x27, [x7, #0xe0]\n" - ".inst 0x448047aa // smlalt z10.s, p4/M, z29.h, z0.h\n" - ".inst 0x44844767 // smlalt z7.s, p4/M, z27.h, z4.h\n" - "ld1b { z27.h }, p3/Z, [x24, x0]\n" - ".inst 0x454f1b7b // usublb z27.h, z27.b, z15.b\n" - ".inst 0x448447d0 // smlalt z16.s, p4/M, z30.h, z4.h\n" - ".inst 0x4481438e // smlalb z14.s, p4/M, z28.h, z1.h\n" - "ldr x26, [x7, #0xe8]\n" - "ldr x25, [x7, #0xf0]\n" - ".inst 0x44844745 // smlalt z5.s, p4/M, z26.h, z4.h\n" - ".inst 0x44804394 // smlalb z20.s, p4/M, z28.h, z0.h\n" - "ld1sb { z4.h }, p4/Z, [x2, #1, MUL VL]\n" - ".inst 0x45511084 // ssublb z4.h, z4.b, z17.b\n" - ".inst 0x448042c8 // smlalb z8.s, p4/M, z22.h, z0.h\n" - ".inst 0x44804326 // smlalb z6.s, p4/M, z25.h, z0.h\n" - "ld1w { z19.s }, p2/Z, [x17]\n" - "ld1w { z18.s }, p1/Z, [x17, #1, MUL VL]\n" - ".inst 0x4481478a // smlalt z10.s, p4/M, z28.h, z1.h\n" - ".inst 0x44804787 // smlalt z7.s, p4/M, z28.h, z0.h\n" - "ld1b { z28.h }, p3/Z, [x23, x0]\n" - ".inst 0x454f1b9c // usublb z28.h, z28.b, z15.b\n" - ".inst 0x448046d0 // smlalt z16.s, p4/M, z22.h, z0.h\n" - ".inst 0x448242ee // smlalb z14.s, p4/M, z23.h, z2.h\n" - "ldr x24, [x7, #0xf8]\n" - "uzp1 z9.s, z19.s, z18.s\n" - ".inst 0x44804725 // smlalt z5.s, p4/M, z25.h, z0.h\n" - ".inst 0x448142f4 // smlalb z20.s, p4/M, z23.h, z1.h\n" - "ld1sb { z0.h }, p4/Z, [x2, #2, MUL VL]\n" - ".inst 0x45511000 // ssublb z0.h, z0.b, z17.b\n" - ".inst 0x44814328 // smlalb z8.s, p4/M, z25.h, z1.h\n" - ".inst 0x44814306 // smlalb z6.s, p4/M, z24.h, z1.h\n" - "uzp2 z29.s, z19.s, z18.s\n" - "ld1w { z19.s }, p2/Z, [x16]\n" - ".inst 0x448246ea // smlalt z10.s, p4/M, z23.h, z2.h\n" - ".inst 0x448146e7 // smlalt z7.s, p4/M, z23.h, z1.h\n" - "ld1b { z23.h }, p3/Z, [x15, x0]\n" - ".inst 0x454f1af7 // usublb z23.h, z23.b, z15.b\n" - ".inst 0x44814730 // smlalt z16.s, p4/M, z25.h, z1.h\n" - ".inst 0x448343ee // smlalb z14.s, p4/M, z31.h, z3.h\n" - "ldr x23, [x7, #0x100]\n" - "whilelt p0.h, x8, x1\n" - ".inst 0x44814705 // smlalt z5.s, p4/M, z24.h, z1.h\n" - ".inst 0x448243f4 // smlalb z20.s, p4/M, z31.h, z2.h\n" - "ld1sb { z1.h }, p4/Z, [x2, #3, MUL VL]\n" - ".inst 0x45511021 // ssublb z1.h, z1.b, z17.b\n" - ".inst 0x44824308 // smlalb z8.s, p4/M, z24.h, z2.h\n" - ".inst 0x44824366 // smlalb z6.s, p4/M, z27.h, z2.h\n" - "addvl x17, x17, #2\n" - ".inst 0x448347ea // smlalt z10.s, p4/M, z31.h, z3.h\n" - ".inst 0x448247e7 // smlalt z7.s, p4/M, z31.h, z2.h\n" - "ld1b { z31.h }, p3/Z, [x22, x0]\n" - ".inst 0x454f1bff // usublb z31.h, z31.b, z15.b\n" - ".inst 0x44824710 // smlalt z16.s, p4/M, z24.h, z2.h\n" - ".inst 0x448443ce // smlalb z14.s, p4/M, z30.h, z4.h\n" - "ldr x22, [x7, #0x108]\n" - ".inst 0x44824765 // smlalt z5.s, p4/M, z27.h, z2.h\n" - ".inst 0x448343d4 // smlalb z20.s, p4/M, z30.h, z3.h\n" - "ld1sb { z2.h }, p4/Z, [x2, #4, MUL VL]\n" - ".inst 0x45511042 // ssublb z2.h, z2.b, z17.b\n" - ".inst 0x44834368 // smlalb z8.s, p4/M, z27.h, z3.h\n" - ".inst 0x448342e6 // smlalb z6.s, p4/M, z23.h, z3.h\n" - ".inst 0x448447ca // smlalt z10.s, p4/M, z30.h, z4.h\n" - ".inst 0x448347c7 // smlalt z7.s, p4/M, z30.h, z3.h\n" - "ld1b { z30.h }, p3/Z, [x21, x0]\n" - ".inst 0x454f1bde // usublb z30.h, z30.b, z15.b\n" - ".inst 0x44834770 // smlalt z16.s, p4/M, z27.h, z3.h\n" - ".inst 0x448042ce // smlalb z14.s, p4/M, z22.h, z0.h\n" - "ldr x21, [x7, #0x110]\n" - ".inst 0x448346e5 // smlalt z5.s, p4/M, z23.h, z3.h\n" - ".inst 0x44844354 // smlalb z20.s, p4/M, z26.h, z4.h\n" - "ld1sb { z3.h }, p4/Z, [x2, #5, MUL VL]\n" - ".inst 0x45511063 // ssublb z3.h, z3.b, z17.b\n" - ".inst 0x448442e8 // smlalb z8.s, p4/M, z23.h, z4.h\n" - ".inst 0x44844386 // smlalb z6.s, p4/M, z28.h, z4.h\n" - ".inst 0x448046ca // smlalt z10.s, p4/M, z22.h, z0.h\n" - ".inst 0x44844747 // smlalt z7.s, p4/M, z26.h, z4.h\n" - "ld1b { z26.h }, p3/Z, [x14, x0]\n" - ".inst 0x454f1b5a // usublb z26.h, z26.b, z15.b\n" - ".inst 0x448446f0 // smlalt z16.s, p4/M, z23.h, z4.h\n" - ".inst 0x4481432e // smlalb z14.s, p4/M, z25.h, z1.h\n" - "ld1b { z22.h }, p3/Z, [x20, x0]\n" - ".inst 0x454f1ad6 // usublb z22.h, z22.b, z15.b\n" - ".inst 0x44844785 // smlalt z5.s, p4/M, z28.h, z4.h\n" - ".inst 0x44804334 // smlalb z20.s, p4/M, z25.h, z0.h\n" - "ld1sb { z4.h }, p4/Z, [x2, #6, MUL VL]\n" - ".inst 0x45511084 // ssublb z4.h, z4.b, z17.b\n" - ".inst 0x448043e8 // smlalb z8.s, p4/M, z31.h, z0.h\n" - ".inst 0x448043c6 // smlalb z6.s, p4/M, z30.h, z0.h\n" - "ldr x20, [x7, #0x118]\n" - "ldr x14, [%x[params], %[offsetof_Params_bias]]\n" - ".inst 0x4481472a // smlalt z10.s, p4/M, z25.h, z1.h\n" - ".inst 0x44804727 // smlalt z7.s, p4/M, z25.h, z0.h\n" - "ld1b { z25.h }, p3/Z, [x13, x0]\n" - ".inst 0x454f1b39 // usublb z25.h, z25.b, z15.b\n" - ".inst 0x448047f0 // smlalt z16.s, p4/M, z31.h, z0.h\n" - ".inst 0x4482430e // smlalb z14.s, p4/M, z24.h, z2.h\n" - ".inst 0x448047c5 // smlalt z5.s, p4/M, z30.h, z0.h\n" - ".inst 0x44814314 // smlalb z20.s, p4/M, z24.h, z1.h\n" - "ld1sb { z0.h }, p4/Z, [x2, #7, MUL VL]\n" - "inch x2, ALL, MUL #8\n" - ".inst 0x448143c8 // smlalb z8.s, p4/M, z30.h, z1.h\n" - ".inst 0x44814346 // smlalb z6.s, p4/M, z26.h, z1.h\n" - ".inst 0x45511000 // ssublb z0.h, z0.b, z17.b\n" - ".inst 0x4482470a // smlalt z10.s, p4/M, z24.h, z2.h\n" - ".inst 0x44814707 // smlalt z7.s, p4/M, z24.h, z1.h\n" - "ld1b { z24.h }, p3/Z, [x12, x0]\n" - ".inst 0x454f1b18 // usublb z24.h, z24.b, z15.b\n" - ".inst 0x448147d0 // smlalt z16.s, p4/M, z30.h, z1.h\n" - ".inst 0x4483436e // smlalb z14.s, p4/M, z27.h, z3.h\n" - ".inst 0x44814745 // smlalt z5.s, p4/M, z26.h, z1.h\n" - ".inst 0x44824374 // smlalb z20.s, p4/M, z27.h, z2.h\n" - "ld1sb { z1.h }, p4/Z, [x2]\n" - ".inst 0x45511021 // ssublb z1.h, z1.b, z17.b\n" - ".inst 0x44824348 // smlalb z8.s, p4/M, z26.h, z2.h\n" - ".inst 0x44824326 // smlalb z6.s, p4/M, z25.h, z2.h\n" - ".inst 0x4483476a // smlalt z10.s, p4/M, z27.h, z3.h\n" - ".inst 0x44824767 // smlalt z7.s, p4/M, z27.h, z2.h\n" - "ld1b { z27.h }, p3/Z, [x11, x0]\n" - ".inst 0x454f1b7b // usublb z27.h, z27.b, z15.b\n" - ".inst 0x44824750 // smlalt z16.s, p4/M, z26.h, z2.h\n" - ".inst 0x448442ee // smlalb z14.s, p4/M, z23.h, z4.h\n" - ".inst 0x44824725 // smlalt z5.s, p4/M, z25.h, z2.h\n" - ".inst 0x448342f4 // smlalb z20.s, p4/M, z23.h, z3.h\n" - "ld1sb { z2.h }, p4/Z, [x2, #1, MUL VL]\n" - ".inst 0x45511042 // ssublb z2.h, z2.b, z17.b\n" - ".inst 0x44834328 // smlalb z8.s, p4/M, z25.h, z3.h\n" - ".inst 0x44834306 // smlalb z6.s, p4/M, z24.h, z3.h\n" - ".inst 0x448446ea // smlalt z10.s, p4/M, z23.h, z4.h\n" - ".inst 0x448346e7 // smlalt z7.s, p4/M, z23.h, z3.h\n" - "ld1b { z23.h }, p3/Z, [x10, x0]\n" - ".inst 0x454f1af7 // usublb z23.h, z23.b, z15.b\n" - ".inst 0x44834730 // smlalt z16.s, p4/M, z25.h, z3.h\n" - ".inst 0x448043ee // smlalb z14.s, p4/M, z31.h, z0.h\n" - ".inst 0x44834705 // smlalt z5.s, p4/M, z24.h, z3.h\n" - ".inst 0x44844394 // smlalb z20.s, p4/M, z28.h, z4.h\n" - "ld1sb { z3.h }, p4/Z, [x2, #2, MUL VL]\n" - ".inst 0x45511063 // ssublb z3.h, z3.b, z17.b\n" - ".inst 0x44844308 // smlalb z8.s, p4/M, z24.h, z4.h\n" - ".inst 0x448442c6 // smlalb z6.s, p4/M, z22.h, z4.h\n" - ".inst 0x448047ea // smlalt z10.s, p4/M, z31.h, z0.h\n" - ".inst 0x44844787 // smlalt z7.s, p4/M, z28.h, z4.h\n" - "ld1b { z31.h }, p3/Z, [x9, x0]\n" - ".inst 0x454f1bff // usublb z31.h, z31.b, z15.b\n" - ".inst 0x44844710 // smlalt z16.s, p4/M, z24.h, z4.h\n" - ".inst 0x448143ce // smlalb z14.s, p4/M, z30.h, z1.h\n" - "ld1b { z28.h }, p3/Z, [x27, x0]\n" - ".inst 0x454f1b9c // usublb z28.h, z28.b, z15.b\n" - ".inst 0x448446c5 // smlalt z5.s, p4/M, z22.h, z4.h\n" - ".inst 0x448043d4 // smlalb z20.s, p4/M, z30.h, z0.h\n" - "ld1sb { z4.h }, p4/Z, [x2, #3, MUL VL]\n" - ".inst 0x45511084 // ssublb z4.h, z4.b, z17.b\n" - ".inst 0x44804368 // smlalb z8.s, p4/M, z27.h, z0.h\n" - ".inst 0x448042e6 // smlalb z6.s, p4/M, z23.h, z0.h\n" - ".inst 0x448147ca // smlalt z10.s, p4/M, z30.h, z1.h\n" - ".inst 0x448047c7 // smlalt z7.s, p4/M, z30.h, z0.h\n" - "ld1b { z30.h }, p3/Z, [x28, x0]\n" - ".inst 0x454f1bde // usublb z30.h, z30.b, z15.b\n" - ".inst 0x44804770 // smlalt z16.s, p4/M, z27.h, z0.h\n" - ".inst 0x4482434e // smlalb z14.s, p4/M, z26.h, z2.h\n" - ".inst 0x448046e5 // smlalt z5.s, p4/M, z23.h, z0.h\n" - ".inst 0x44814354 // smlalb z20.s, p4/M, z26.h, z1.h\n" - "ld1sb { z0.h }, p4/Z, [x2, #4, MUL VL]\n" - ".inst 0x45511000 // ssublb z0.h, z0.b, z17.b\n" - ".inst 0x448142e8 // smlalb z8.s, p4/M, z23.h, z1.h\n" - ".inst 0x448143e6 // smlalb z6.s, p4/M, z31.h, z1.h\n" - ".inst 0x4482474a // smlalt z10.s, p4/M, z26.h, z2.h\n" - ".inst 0x44814747 // smlalt z7.s, p4/M, z26.h, z1.h\n" - "ld1b { z26.h }, p3/Z, [x26, x0]\n" - ".inst 0x454f1b5a // usublb z26.h, z26.b, z15.b\n" - ".inst 0x448146f0 // smlalt z16.s, p4/M, z23.h, z1.h\n" - ".inst 0x4483432e // smlalb z14.s, p4/M, z25.h, z3.h\n" - ".inst 0x448147e5 // smlalt z5.s, p4/M, z31.h, z1.h\n" - ".inst 0x44824334 // smlalb z20.s, p4/M, z25.h, z2.h\n" - "ld1sb { z1.h }, p4/Z, [x2, #5, MUL VL]\n" - ".inst 0x45511021 // ssublb z1.h, z1.b, z17.b\n" - ".inst 0x448243e8 // smlalb z8.s, p4/M, z31.h, z2.h\n" - ".inst 0x448243c6 // smlalb z6.s, p4/M, z30.h, z2.h\n" - ".inst 0x4483472a // smlalt z10.s, p4/M, z25.h, z3.h\n" - ".inst 0x44824727 // smlalt z7.s, p4/M, z25.h, z2.h\n" - "ld1b { z25.h }, p3/Z, [x25, x0]\n" - ".inst 0x454f1b39 // usublb z25.h, z25.b, z15.b\n" - ".inst 0x448247f0 // smlalt z16.s, p4/M, z31.h, z2.h\n" - ".inst 0x4484430e // smlalb z14.s, p4/M, z24.h, z4.h\n" - ".inst 0x448247c5 // smlalt z5.s, p4/M, z30.h, z2.h\n" - ".inst 0x44834314 // smlalb z20.s, p4/M, z24.h, z3.h\n" - "ld1sb { z2.h }, p4/Z, [x2, #6, MUL VL]\n" - ".inst 0x45511042 // ssublb z2.h, z2.b, z17.b\n" - ".inst 0x448343c8 // smlalb z8.s, p4/M, z30.h, z3.h\n" - ".inst 0x44834386 // smlalb z6.s, p4/M, z28.h, z3.h\n" - ".inst 0x4484470a // smlalt z10.s, p4/M, z24.h, z4.h\n" - ".inst 0x44834707 // smlalt z7.s, p4/M, z24.h, z3.h\n" - "ld1b { z24.h }, p3/Z, [x24, x0]\n" - ".inst 0x454f1b18 // usublb z24.h, z24.b, z15.b\n" - ".inst 0x448347d0 // smlalt z16.s, p4/M, z30.h, z3.h\n" - ".inst 0x4480436e // smlalb z14.s, p4/M, z27.h, z0.h\n" - ".inst 0x44834785 // smlalt z5.s, p4/M, z28.h, z3.h\n" - ".inst 0x448442d4 // smlalb z20.s, p4/M, z22.h, z4.h\n" - "ld1sb { z3.h }, p4/Z, [x2, #7, MUL VL]\n" - "inch x2, ALL, MUL #8\n" - ".inst 0x44844388 // smlalb z8.s, p4/M, z28.h, z4.h\n" - ".inst 0x44844346 // smlalb z6.s, p4/M, z26.h, z4.h\n" - ".inst 0x45511063 // ssublb z3.h, z3.b, z17.b\n" - ".inst 0x4480476a // smlalt z10.s, p4/M, z27.h, z0.h\n" - ".inst 0x44844790 // smlalt z16.s, p4/M, z28.h, z4.h\n" - "ld1b { z27.h }, p3/Z, [x23, x0]\n" - ".inst 0x454f1b7b // usublb z27.h, z27.b, z15.b\n" - ".inst 0x448142ee // smlalb z14.s, p4/M, z23.h, z1.h\n" - ".inst 0x448446c7 // smlalt z7.s, p4/M, z22.h, z4.h\n" - "ld1w { z18.s }, p1/Z, [x16, #1, MUL VL]\n" - "addvl x16, x16, #2\n" - ".inst 0x44844745 // smlalt z5.s, p4/M, z26.h, z4.h\n" - ".inst 0x448042f4 // smlalb z20.s, p4/M, z23.h, z0.h\n" - "ld1sb { z4.h }, p4/Z, [x2]\n" - ".inst 0x45511084 // ssublb z4.h, z4.b, z17.b\n" - ".inst 0x44804328 // smlalb z8.s, p4/M, z25.h, z0.h\n" - ".inst 0x44804306 // smlalb z6.s, p4/M, z24.h, z0.h\n" - "inch x2\n" - ".inst 0x448146ea // smlalt z10.s, p4/M, z23.h, z1.h\n" - ".inst 0x44804730 // smlalt z16.s, p4/M, z25.h, z0.h\n" - "ld1b { z25.h }, p3/Z, [x22, x0]\n" - ".inst 0x454f1b39 // usublb z25.h, z25.b, z15.b\n" + ".inst 0x449a42ce // smlalb z14.s, p4/M, z22.h, z26.h\n" + ".inst 0x449a46d7 // smlalt z23.s, p4/M, z22.h, z26.h\n" + "ldr x20, [x17, #0x50]\n" + "ld1b { z27.h }, p3/Z, [x20, x2]\n" + ".inst 0x4488404e // smlalb z14.s, p4/M, z2.h, z8.h\n" + ".inst 0x449a4046 // smlalb z6.s, p4/M, z2.h, z26.h\n" + "ldr x20, [x17, #0x58]\n" + ".inst 0x455e1b7b // usublb z27.h, z27.b, z30.b\n" + ".inst 0x449a4169 // smlalb z9.s, p4/M, z11.h, z26.h\n" + ".inst 0x449a4067 // smlalb z7.s, p4/M, z3.h, z26.h\n" + "ld1b { z5.h }, p3/Z, [x20, x2]\n" + "ldr x20, [x17, #0x60]\n" + ".inst 0x44884457 // smlalt z23.s, p4/M, z2.h, z8.h\n" + ".inst 0x449043ae // smlalb z14.s, p4/M, z29.h, z16.h\n" + "ld1sb { z25.h }, p4/Z, [x4, #5, MUL VL]\n" + ".inst 0x455e18a5 // usublb z5.h, z5.b, z30.b\n" + ".inst 0x449a4452 // smlalt z18.s, p4/M, z2.h, z26.h\n" + ".inst 0x449a4574 // smlalt z20.s, p4/M, z11.h, z26.h\n" + "ld1b { z22.h }, p3/Z, [x20, x2]\n" + ".inst 0x454a1339 // ssublb z25.h, z25.b, z10.b\n" + ".inst 0x449a4461 // smlalt z1.s, p4/M, z3.h, z26.h\n" + ".inst 0x448843a6 // smlalb z6.s, p4/M, z29.h, z8.h\n" + "ldr x20, [x17, #0x68]\n" + "ld1sb { z2.h }, p4/Z, [x4, #6, MUL VL]\n" + ".inst 0x44884069 // smlalb z9.s, p4/M, z3.h, z8.h\n" + ".inst 0x44884087 // smlalb z7.s, p4/M, z4.h, z8.h\n" + ".inst 0x455e1ad6 // usublb z22.h, z22.b, z30.b\n" + "ld1b { z26.h }, p3/Z, [x20, x2]\n" + ".inst 0x449047b7 // smlalt z23.s, p4/M, z29.h, z16.h\n" + ".inst 0x449543ee // smlalb z14.s, p4/M, z31.h, z21.h\n" + ".inst 0x454a1042 // ssublb z2.h, z2.b, z10.b\n" + "ldr x20, [x17, #0x70]\n" + ".inst 0x448847b2 // smlalt z18.s, p4/M, z29.h, z8.h\n" + ".inst 0x44884474 // smlalt z20.s, p4/M, z3.h, z8.h\n" + "ld1sb { z29.h }, p4/Z, [x4, #7, MUL VL]\n" + ".inst 0x455e1b5a // usublb z26.h, z26.b, z30.b\n" + ".inst 0x44884481 // smlalt z1.s, p4/M, z4.h, z8.h\n" + ".inst 0x449043e6 // smlalb z6.s, p4/M, z31.h, z16.h\n" + "inch x4, ALL, MUL #8\n" + "ld1b { z8.h }, p3/Z, [x20, x2]\n" + ".inst 0x44904089 // smlalb z9.s, p4/M, z4.h, z16.h\n" + ".inst 0x44904367 // smlalb z7.s, p4/M, z27.h, z16.h\n" + ".inst 0x454a13bd // ssublb z29.h, z29.b, z10.b\n" + "ldr x20, [x17, #0x78]\n" + ".inst 0x449547f7 // smlalt z23.s, p4/M, z31.h, z21.h\n" + ".inst 0x4491400e // smlalb z14.s, p4/M, z0.h, z17.h\n" + "ld1sb { z24.h }, p4/Z, [x4]\n" + ".inst 0x455e1908 // usublb z8.h, z8.b, z30.b\n" + ".inst 0x449047f2 // smlalt z18.s, p4/M, z31.h, z16.h\n" + ".inst 0x44904494 // smlalt z20.s, p4/M, z4.h, z16.h\n" + "ld1b { z31.h }, p3/Z, [x20, x2]\n" + ".inst 0x454a1318 // ssublb z24.h, z24.b, z10.b\n" + ".inst 0x44904761 // smlalt z1.s, p4/M, z27.h, z16.h\n" + ".inst 0x44954006 // smlalb z6.s, p4/M, z0.h, z21.h\n" + "ldr x22, [x17, #0x80]\n" + "ld1sb { z16.h }, p4/Z, [x4, #1, MUL VL]\n" + ".inst 0x44954369 // smlalb z9.s, p4/M, z27.h, z21.h\n" + ".inst 0x449540a7 // smlalb z7.s, p4/M, z5.h, z21.h\n" + ".inst 0x455e1bff // usublb z31.h, z31.b, z30.b\n" + "ldr x21, [x17, #0x88]\n" + ".inst 0x44914417 // smlalt z23.s, p4/M, z0.h, z17.h\n" + ".inst 0x4499416e // smlalb z14.s, p4/M, z11.h, z25.h\n" + ".inst 0x454a1210 // ssublb z16.h, z16.b, z10.b\n" + "ldr x20, [x17, #0x90]\n" + ".inst 0x44954412 // smlalt z18.s, p4/M, z0.h, z21.h\n" + ".inst 0x44954774 // smlalt z20.s, p4/M, z27.h, z21.h\n" + "ld1b { z0.h }, p3/Z, [x22, x2]\n" + ".inst 0x455e1800 // usublb z0.h, z0.b, z30.b\n" + ".inst 0x449544a1 // smlalt z1.s, p4/M, z5.h, z21.h\n" + ".inst 0x449142c6 // smlalb z6.s, p4/M, z22.h, z17.h\n" + "ld1sb { z21.h }, p4/Z, [x4, #2, MUL VL]\n" + ".inst 0x454a12b5 // ssublb z21.h, z21.b, z10.b\n" + ".inst 0x449140a9 // smlalb z9.s, p4/M, z5.h, z17.h\n" + ".inst 0x44914267 // smlalb z7.s, p4/M, z19.h, z17.h\n" + "ldr x23, [x17, #0x98]\n" + "ldr x22, [x17, #0xa0]\n" + ".inst 0x44994577 // smlalt z23.s, p4/M, z11.h, z25.h\n" + ".inst 0x4482406e // smlalb z14.s, p4/M, z3.h, z2.h\n" + "ld1b { z11.h }, p3/Z, [x21, x2]\n" + ".inst 0x455e196b // usublb z11.h, z11.b, z30.b\n" + ".inst 0x449146d2 // smlalt z18.s, p4/M, z22.h, z17.h\n" + ".inst 0x449144b4 // smlalt z20.s, p4/M, z5.h, z17.h\n" + "ld1sb { z22.h }, p4/Z, [x4, #3, MUL VL]\n" + ".inst 0x454a12d6 // ssublb z22.h, z22.b, z10.b\n" + ".inst 0x44914661 // smlalt z1.s, p4/M, z19.h, z17.h\n" + ".inst 0x44994066 // smlalb z6.s, p4/M, z3.h, z25.h\n" + "ld1b { z17.h }, p3/Z, [x20, x2]\n" + ".inst 0x455e1a31 // usublb z17.h, z17.b, z30.b\n" + ".inst 0x44994389 // smlalb z9.s, p4/M, z28.h, z25.h\n" + ".inst 0x44994347 // smlalb z7.s, p4/M, z26.h, z25.h\n" + "ldr x20, [x17, #0xa8]\n" + "ldr x21, [x17, #0xb0]\n" + ".inst 0x44824477 // smlalt z23.s, p4/M, z3.h, z2.h\n" + ".inst 0x449d408e // smlalb z14.s, p4/M, z4.h, z29.h\n" + "ldr x13, [x17, #0xb8]\n" + "ldr x12, [x17, #0xc0]\n" + ".inst 0x44994472 // smlalt z18.s, p4/M, z3.h, z25.h\n" + ".inst 0x44994794 // smlalt z20.s, p4/M, z28.h, z25.h\n" + "ld1b { z3.h }, p3/Z, [x23, x2]\n" + ".inst 0x455e1863 // usublb z3.h, z3.b, z30.b\n" + ".inst 0x44994741 // smlalt z1.s, p4/M, z26.h, z25.h\n" + ".inst 0x44824086 // smlalb z6.s, p4/M, z4.h, z2.h\n" + "ld1sb { z25.h }, p4/Z, [x4, #4, MUL VL]\n" + ".inst 0x454a1339 // ssublb z25.h, z25.b, z10.b\n" + ".inst 0x44824349 // smlalb z9.s, p4/M, z26.h, z2.h\n" + ".inst 0x44824107 // smlalb z7.s, p4/M, z8.h, z2.h\n" + "ldr x11, [x17, #0xc8]\n" + "ldr x10, [x17, #0xd0]\n" + ".inst 0x449d4497 // smlalt z23.s, p4/M, z4.h, z29.h\n" + ".inst 0x4498436e // smlalb z14.s, p4/M, z27.h, z24.h\n" + "ldr x9, [x17, #0xd8]\n" + "ldr x28, [x17, #0xe0]\n" + ".inst 0x44824492 // smlalt z18.s, p4/M, z4.h, z2.h\n" + ".inst 0x44824754 // smlalt z20.s, p4/M, z26.h, z2.h\n" + "ld1b { z4.h }, p3/Z, [x22, x2]\n" + ".inst 0x455e1884 // usublb z4.h, z4.b, z30.b\n" + ".inst 0x44824501 // smlalt z1.s, p4/M, z8.h, z2.h\n" + ".inst 0x449d4366 // smlalb z6.s, p4/M, z27.h, z29.h\n" + "ld1sb { z2.h }, p4/Z, [x4, #5, MUL VL]\n" + ".inst 0x454a1042 // ssublb z2.h, z2.b, z10.b\n" + ".inst 0x449d4109 // smlalb z9.s, p4/M, z8.h, z29.h\n" + ".inst 0x449d43e7 // smlalb z7.s, p4/M, z31.h, z29.h\n" + "ldr x27, [x17, #0xe8]\n" + "ldr x26, [x17, #0xf0]\n" + ".inst 0x44984777 // smlalt z23.s, p4/M, z27.h, z24.h\n" + ".inst 0x449040ae // smlalb z14.s, p4/M, z5.h, z16.h\n" + "ldr x25, [x17, #0xf8]\n" + "ldr x24, [x17, #0x100]\n" + ".inst 0x449d4772 // smlalt z18.s, p4/M, z27.h, z29.h\n" + ".inst 0x449d4514 // smlalt z20.s, p4/M, z8.h, z29.h\n" + "ld1b { z27.h }, p3/Z, [x20, x2]\n" + ".inst 0x455e1b7b // usublb z27.h, z27.b, z30.b\n" + ".inst 0x449d47e1 // smlalt z1.s, p4/M, z31.h, z29.h\n" + ".inst 0x449840a6 // smlalb z6.s, p4/M, z5.h, z24.h\n" + "ld1sb { z29.h }, p4/Z, [x4, #6, MUL VL]\n" + ".inst 0x454a13bd // ssublb z29.h, z29.b, z10.b\n" + ".inst 0x449843e9 // smlalb z9.s, p4/M, z31.h, z24.h\n" + ".inst 0x44984007 // smlalb z7.s, p4/M, z0.h, z24.h\n" + "ldr x23, [x17, #0x108]\n" + "ldr x22, [x17, #0x110]\n" + ".inst 0x449044b7 // smlalt z23.s, p4/M, z5.h, z16.h\n" + ".inst 0x4495438e // smlalb z14.s, p4/M, z28.h, z21.h\n" + "ldr x20, [x17, #0x118]\n" + "whilelt p0.h, x16, x3\n" + ".inst 0x449844b2 // smlalt z18.s, p4/M, z5.h, z24.h\n" + ".inst 0x449847f4 // smlalt z20.s, p4/M, z31.h, z24.h\n" + "ld1b { z5.h }, p3/Z, [x21, x2]\n" + ".inst 0x455e18a5 // usublb z5.h, z5.b, z30.b\n" + ".inst 0x44984401 // smlalt z1.s, p4/M, z0.h, z24.h\n" + ".inst 0x44904266 // smlalb z6.s, p4/M, z19.h, z16.h\n" + "ld1sb { z24.h }, p4/Z, [x4, #7, MUL VL]\n" + "inch x4, ALL, MUL #8\n" + ".inst 0x44904009 // smlalb z9.s, p4/M, z0.h, z16.h\n" + ".inst 0x44904167 // smlalb z7.s, p4/M, z11.h, z16.h\n" + ".inst 0x454a1318 // ssublb z24.h, z24.b, z10.b\n" + "ldr x21, [%x[params], %[offsetof_Params_bias]]\n" + ".inst 0x44954797 // smlalt z23.s, p4/M, z28.h, z21.h\n" + ".inst 0x4496434e // smlalb z14.s, p4/M, z26.h, z22.h\n" + "ld1b { z28.h }, p3/Z, [x13, x2]\n" + ".inst 0x455e1b9c // usublb z28.h, z28.b, z30.b\n" + ".inst 0x44904672 // smlalt z18.s, p4/M, z19.h, z16.h\n" + ".inst 0x44904414 // smlalt z20.s, p4/M, z0.h, z16.h\n" + "ld1sb { z19.h }, p4/Z, [x4]\n" + ".inst 0x454a1273 // ssublb z19.h, z19.b, z10.b\n" + ".inst 0x44904561 // smlalt z1.s, p4/M, z11.h, z16.h\n" + ".inst 0x44954346 // smlalb z6.s, p4/M, z26.h, z21.h\n" + "ld1b { z16.h }, p3/Z, [x12, x2]\n" + ".inst 0x455e1a10 // usublb z16.h, z16.b, z30.b\n" + ".inst 0x44954229 // smlalb z9.s, p4/M, z17.h, z21.h\n" + ".inst 0x44954067 // smlalb z7.s, p4/M, z3.h, z21.h\n" + ".inst 0x44964757 // smlalt z23.s, p4/M, z26.h, z22.h\n" + ".inst 0x4499410e // smlalb z14.s, p4/M, z8.h, z25.h\n" + ".inst 0x44954752 // smlalt z18.s, p4/M, z26.h, z21.h\n" + ".inst 0x44954634 // smlalt z20.s, p4/M, z17.h, z21.h\n" + "ld1b { z26.h }, p3/Z, [x11, x2]\n" + ".inst 0x455e1b5a // usublb z26.h, z26.b, z30.b\n" + ".inst 0x44954461 // smlalt z1.s, p4/M, z3.h, z21.h\n" + ".inst 0x44964106 // smlalb z6.s, p4/M, z8.h, z22.h\n" + "ld1sb { z21.h }, p4/Z, [x4, #1, MUL VL]\n" + ".inst 0x454a12b5 // ssublb z21.h, z21.b, z10.b\n" + ".inst 0x44964069 // smlalb z9.s, p4/M, z3.h, z22.h\n" + ".inst 0x44964087 // smlalb z7.s, p4/M, z4.h, z22.h\n" + ".inst 0x44994517 // smlalt z23.s, p4/M, z8.h, z25.h\n" ".inst 0x448243ee // smlalb z14.s, p4/M, z31.h, z2.h\n" - ".inst 0x448046e7 // smlalt z7.s, p4/M, z23.h, z0.h\n" - "uzp1 z23.s, z19.s, z18.s\n" - ".inst 0x44804705 // smlalt z5.s, p4/M, z24.h, z0.h\n" - ".inst 0x448143f4 // smlalb z20.s, p4/M, z31.h, z1.h\n" - "uzp2 z22.s, z19.s, z18.s\n" - ".inst 0x44814308 // smlalb z8.s, p4/M, z24.h, z1.h\n" - ".inst 0x44814366 // smlalb z6.s, p4/M, z27.h, z1.h\n" - ".inst 0x448247ea // smlalt z10.s, p4/M, z31.h, z2.h\n" - ".inst 0x44814710 // smlalt z16.s, p4/M, z24.h, z1.h\n" - "ld1b { z24.h }, p3/Z, [x21, x0]\n" - ".inst 0x454f1b18 // usublb z24.h, z24.b, z15.b\n" - ".inst 0x448343ce // smlalb z14.s, p4/M, z30.h, z3.h\n" - ".inst 0x448147e7 // smlalt z7.s, p4/M, z31.h, z1.h\n" - ".inst 0x44814765 // smlalt z5.s, p4/M, z27.h, z1.h\n" - ".inst 0x448243d4 // smlalb z20.s, p4/M, z30.h, z2.h\n" - ".inst 0x44824368 // smlalb z8.s, p4/M, z27.h, z2.h\n" - ".inst 0x44824326 // smlalb z6.s, p4/M, z25.h, z2.h\n" - ".inst 0x448347ca // smlalt z10.s, p4/M, z30.h, z3.h\n" - ".inst 0x44824770 // smlalt z16.s, p4/M, z27.h, z2.h\n" - "ld1b { z27.h }, p3/Z, [x20, x0]\n" - ".inst 0x454f1b7b // usublb z27.h, z27.b, z15.b\n" - ".inst 0x4484438e // smlalb z14.s, p4/M, z28.h, z4.h\n" - ".inst 0x448247c7 // smlalt z7.s, p4/M, z30.h, z2.h\n" - ".inst 0x04a975ce // sqrdmulh z14.s, z14.s, z9.s\n" - "inch x0\n" - ".inst 0x44824725 // smlalt z5.s, p4/M, z25.h, z2.h\n" - ".inst 0x44834394 // smlalb z20.s, p4/M, z28.h, z3.h\n" - "and z21.d, z14.d, z23.d\n" - "mov x20, x0\n" - ".inst 0x44834328 // smlalb z8.s, p4/M, z25.h, z3.h\n" - ".inst 0x44834306 // smlalb z6.s, p4/M, z24.h, z3.h\n" - "asr z21.s, z21.s, #0x1f\n" + ".inst 0x44964512 // smlalt z18.s, p4/M, z8.h, z22.h\n" + ".inst 0x44964474 // smlalt z20.s, p4/M, z3.h, z22.h\n" + "ld1b { z8.h }, p3/Z, [x10, x2]\n" + ".inst 0x455e1908 // usublb z8.h, z8.b, z30.b\n" + ".inst 0x44964481 // smlalt z1.s, p4/M, z4.h, z22.h\n" + ".inst 0x449943e6 // smlalb z6.s, p4/M, z31.h, z25.h\n" + "ld1sb { z22.h }, p4/Z, [x4, #2, MUL VL]\n" + ".inst 0x454a12d6 // ssublb z22.h, z22.b, z10.b\n" + ".inst 0x44994089 // smlalb z9.s, p4/M, z4.h, z25.h\n" + ".inst 0x44994367 // smlalb z7.s, p4/M, z27.h, z25.h\n" + ".inst 0x448247f7 // smlalt z23.s, p4/M, z31.h, z2.h\n" + ".inst 0x449d400e // smlalb z14.s, p4/M, z0.h, z29.h\n" + ".inst 0x449947f2 // smlalt z18.s, p4/M, z31.h, z25.h\n" + ".inst 0x44994494 // smlalt z20.s, p4/M, z4.h, z25.h\n" + "ld1b { z31.h }, p3/Z, [x9, x2]\n" + ".inst 0x455e1bff // usublb z31.h, z31.b, z30.b\n" + ".inst 0x44994761 // smlalt z1.s, p4/M, z27.h, z25.h\n" + ".inst 0x44824006 // smlalb z6.s, p4/M, z0.h, z2.h\n" + "ld1sb { z25.h }, p4/Z, [x4, #3, MUL VL]\n" + ".inst 0x454a1339 // ssublb z25.h, z25.b, z10.b\n" + ".inst 0x44824369 // smlalb z9.s, p4/M, z27.h, z2.h\n" + ".inst 0x448240a7 // smlalb z7.s, p4/M, z5.h, z2.h\n" + ".inst 0x449d4417 // smlalt z23.s, p4/M, z0.h, z29.h\n" + ".inst 0x4498422e // smlalb z14.s, p4/M, z17.h, z24.h\n" + ".inst 0x44824412 // smlalt z18.s, p4/M, z0.h, z2.h\n" + ".inst 0x44824774 // smlalt z20.s, p4/M, z27.h, z2.h\n" + "ld1b { z0.h }, p3/Z, [x28, x2]\n" + ".inst 0x455e1800 // usublb z0.h, z0.b, z30.b\n" + ".inst 0x448244a1 // smlalt z1.s, p4/M, z5.h, z2.h\n" + ".inst 0x449d4166 // smlalb z6.s, p4/M, z11.h, z29.h\n" + "ld1sb { z2.h }, p4/Z, [x4, #4, MUL VL]\n" + ".inst 0x454a1042 // ssublb z2.h, z2.b, z10.b\n" + ".inst 0x449d40a9 // smlalb z9.s, p4/M, z5.h, z29.h\n" + ".inst 0x449d4387 // smlalb z7.s, p4/M, z28.h, z29.h\n" + ".inst 0x44984637 // smlalt z23.s, p4/M, z17.h, z24.h\n" + ".inst 0x4493406e // smlalb z14.s, p4/M, z3.h, z19.h\n" + "ld1b { z17.h }, p3/Z, [x27, x2]\n" + ".inst 0x455e1a31 // usublb z17.h, z17.b, z30.b\n" + ".inst 0x449d4572 // smlalt z18.s, p4/M, z11.h, z29.h\n" + ".inst 0x449d44b4 // smlalt z20.s, p4/M, z5.h, z29.h\n" + "ld1sb { z11.h }, p4/Z, [x4, #5, MUL VL]\n" + ".inst 0x454a116b // ssublb z11.h, z11.b, z10.b\n" + ".inst 0x449d4781 // smlalt z1.s, p4/M, z28.h, z29.h\n" + ".inst 0x44984066 // smlalb z6.s, p4/M, z3.h, z24.h\n" + "ld1b { z29.h }, p3/Z, [x26, x2]\n" + ".inst 0x455e1bbd // usublb z29.h, z29.b, z30.b\n" + ".inst 0x44984209 // smlalb z9.s, p4/M, z16.h, z24.h\n" + ".inst 0x44984347 // smlalb z7.s, p4/M, z26.h, z24.h\n" + ".inst 0x44934477 // smlalt z23.s, p4/M, z3.h, z19.h\n" + ".inst 0x4495408e // smlalb z14.s, p4/M, z4.h, z21.h\n" + ".inst 0x44984472 // smlalt z18.s, p4/M, z3.h, z24.h\n" + ".inst 0x44984614 // smlalt z20.s, p4/M, z16.h, z24.h\n" + "ld1b { z3.h }, p3/Z, [x25, x2]\n" + ".inst 0x455e1863 // usublb z3.h, z3.b, z30.b\n" + ".inst 0x44984741 // smlalt z1.s, p4/M, z26.h, z24.h\n" + ".inst 0x44934086 // smlalb z6.s, p4/M, z4.h, z19.h\n" + "ld1sb { z24.h }, p4/Z, [x4, #6, MUL VL]\n" + ".inst 0x454a1318 // ssublb z24.h, z24.b, z10.b\n" + ".inst 0x44934349 // smlalb z9.s, p4/M, z26.h, z19.h\n" + ".inst 0x44934107 // smlalb z7.s, p4/M, z8.h, z19.h\n" + ".inst 0x44954497 // smlalt z23.s, p4/M, z4.h, z21.h\n" + ".inst 0x4496436e // smlalb z14.s, p4/M, z27.h, z22.h\n" + ".inst 0x44934492 // smlalt z18.s, p4/M, z4.h, z19.h\n" + ".inst 0x44934754 // smlalt z20.s, p4/M, z26.h, z19.h\n" + "ld1b { z4.h }, p3/Z, [x24, x2]\n" + ".inst 0x455e1884 // usublb z4.h, z4.b, z30.b\n" + ".inst 0x44934501 // smlalt z1.s, p4/M, z8.h, z19.h\n" + ".inst 0x44954366 // smlalb z6.s, p4/M, z27.h, z21.h\n" + "ld1sb { z19.h }, p4/Z, [x4, #7, MUL VL]\n" + "inch x4, ALL, MUL #8\n" + ".inst 0x44954109 // smlalb z9.s, p4/M, z8.h, z21.h\n" + ".inst 0x449543e7 // smlalb z7.s, p4/M, z31.h, z21.h\n" + ".inst 0x454a1273 // ssublb z19.h, z19.b, z10.b\n" + ".inst 0x44964777 // smlalt z23.s, p4/M, z27.h, z22.h\n" + ".inst 0x449940ae // smlalb z14.s, p4/M, z5.h, z25.h\n" + ".inst 0x44954772 // smlalt z18.s, p4/M, z27.h, z21.h\n" + ".inst 0x44954514 // smlalt z20.s, p4/M, z8.h, z21.h\n" + "ld1b { z27.h }, p3/Z, [x23, x2]\n" + ".inst 0x455e1b7b // usublb z27.h, z27.b, z30.b\n" + ".inst 0x449547e1 // smlalt z1.s, p4/M, z31.h, z21.h\n" + ".inst 0x449640a6 // smlalb z6.s, p4/M, z5.h, z22.h\n" + "ld1sb { z21.h }, p4/Z, [x4]\n" + ".inst 0x454a12b5 // ssublb z21.h, z21.b, z10.b\n" + ".inst 0x449643e9 // smlalb z9.s, p4/M, z31.h, z22.h\n" + ".inst 0x44964007 // smlalb z7.s, p4/M, z0.h, z22.h\n" + "inch x4\n" + ".inst 0x449944b7 // smlalt z23.s, p4/M, z5.h, z25.h\n" + ".inst 0x4482420e // smlalb z14.s, p4/M, z16.h, z2.h\n" + ".inst 0x449644b2 // smlalt z18.s, p4/M, z5.h, z22.h\n" + ".inst 0x449647f4 // smlalt z20.s, p4/M, z31.h, z22.h\n" + "ld1b { z5.h }, p3/Z, [x22, x2]\n" + ".inst 0x455e18a5 // usublb z5.h, z5.b, z30.b\n" + ".inst 0x44964401 // smlalt z1.s, p4/M, z0.h, z22.h\n" + ".inst 0x44994386 // smlalb z6.s, p4/M, z28.h, z25.h\n" + "ld1w { z22.s }, p2/Z, [x15]\n" + ".inst 0x44994009 // smlalb z9.s, p4/M, z0.h, z25.h\n" + ".inst 0x44994227 // smlalb z7.s, p4/M, z17.h, z25.h\n" + ".inst 0x44824617 // smlalt z23.s, p4/M, z16.h, z2.h\n" + ".inst 0x448b434e // smlalb z14.s, p4/M, z26.h, z11.h\n" + "ld1w { z16.s }, p1/Z, [x15, #1, MUL VL]\n" + "addvl x15, x15, #2\n" + ".inst 0x44994792 // smlalt z18.s, p4/M, z28.h, z25.h\n" + ".inst 0x44994414 // smlalt z20.s, p4/M, z0.h, z25.h\n" + "ld1b { z28.h }, p3/Z, [x20, x2]\n" + ".inst 0x455e1b9c // usublb z28.h, z28.b, z30.b\n" + ".inst 0x44994621 // smlalt z1.s, p4/M, z17.h, z25.h\n" + ".inst 0x44824346 // smlalb z6.s, p4/M, z26.h, z2.h\n" + "uzp1 z25.s, z22.s, z16.s\n" + "inch x2\n" + ".inst 0x448243a9 // smlalb z9.s, p4/M, z29.h, z2.h\n" + ".inst 0x44824067 // smlalb z7.s, p4/M, z3.h, z2.h\n" + "uzp2 z16.s, z22.s, z16.s\n" + "ld1w { z22.s }, p2/Z, [x14]\n" + ".inst 0x448b4757 // smlalt z23.s, p4/M, z26.h, z11.h\n" + ".inst 0x4498410e // smlalb z14.s, p4/M, z8.h, z24.h\n" + "mov x20, x2\n" "incw x20\n" - ".inst 0x4484478a // smlalt z10.s, p4/M, z28.h, z4.h\n" - ".inst 0x44834787 // smlalt z7.s, p4/M, z28.h, z3.h\n" - ".inst 0x04bd754a // sqrdmulh z10.s, z10.s, z29.s\n" - "whilelt p2.s, x0, x1\n" - ".inst 0x44834730 // smlalt z16.s, p4/M, z25.h, z3.h\n" - ".inst 0x44834705 // smlalt z5.s, p4/M, z24.h, z3.h\n" - "and z3.d, z10.d, z22.d\n" - "whilelt p1.s, x20, x1\n" - ".inst 0x44844354 // smlalb z20.s, p4/M, z26.h, z4.h\n" - ".inst 0x44844308 // smlalb z8.s, p4/M, z24.h, z4.h\n" - ".inst 0x04a97694 // sqrdmulh z20.s, z20.s, z9.s\n" - "whilelt p3.h, x0, x1\n" - ".inst 0x44844366 // smlalb z6.s, p4/M, z27.h, z4.h\n" - ".inst 0x44844747 // smlalt z7.s, p4/M, z26.h, z4.h\n" - ".inst 0x04a97508 // sqrdmulh z8.s, z8.s, z9.s\n" - ".inst 0x44844710 // smlalt z16.s, p4/M, z24.h, z4.h\n" - ".inst 0x44844765 // smlalt z5.s, p4/M, z27.h, z4.h\n" - ".inst 0x04a974c6 // sqrdmulh z6.s, z6.s, z9.s\n" - "sqadd z14.s, z14.s, z21.s\n" + ".inst 0x44824752 // smlalt z18.s, p4/M, z26.h, z2.h\n" + ".inst 0x448247b4 // smlalt z20.s, p4/M, z29.h, z2.h\n" + "ld1w { z26.s }, p1/Z, [x14, #1, MUL VL]\n" + "uzp1 z29.s, z22.s, z26.s\n" + ".inst 0x44824461 // smlalt z1.s, p4/M, z3.h, z2.h\n" + ".inst 0x448b4106 // smlalb z6.s, p4/M, z8.h, z11.h\n" + "uzp2 z22.s, z22.s, z26.s\n" + "whilelt p2.s, x2, x3\n" + ".inst 0x448b4069 // smlalb z9.s, p4/M, z3.h, z11.h\n" + ".inst 0x448b4087 // smlalb z7.s, p4/M, z4.h, z11.h\n" + "whilelt p1.s, x20, x3\n" + "whilelt p3.h, x2, x3\n" + ".inst 0x44984517 // smlalt z23.s, p4/M, z8.h, z24.h\n" + ".inst 0x449343ee // smlalb z14.s, p4/M, z31.h, z19.h\n" + "addvl x14, x14, #2\n" + ".inst 0x448b4512 // smlalt z18.s, p4/M, z8.h, z11.h\n" + ".inst 0x448b4474 // smlalt z20.s, p4/M, z3.h, z11.h\n" + ".inst 0x448b4481 // smlalt z1.s, p4/M, z4.h, z11.h\n" + ".inst 0x449843e6 // smlalb z6.s, p4/M, z31.h, z24.h\n" + ".inst 0x44984089 // smlalb z9.s, p4/M, z4.h, z24.h\n" + ".inst 0x44984367 // smlalb z7.s, p4/M, z27.h, z24.h\n" + ".inst 0x449347f7 // smlalt z23.s, p4/M, z31.h, z19.h\n" + ".inst 0x4495400e // smlalb z14.s, p4/M, z0.h, z21.h\n" + ".inst 0x04b975ce // sqrdmulh z14.s, z14.s, z25.s\n" + ".inst 0x449847f2 // smlalt z18.s, p4/M, z31.h, z24.h\n" + ".inst 0x44984494 // smlalt z20.s, p4/M, z4.h, z24.h\n" + "and z3.d, z14.d, z29.d\n" + ".inst 0x44984761 // smlalt z1.s, p4/M, z27.h, z24.h\n" + ".inst 0x44934006 // smlalb z6.s, p4/M, z0.h, z19.h\n" + "asr z3.s, z3.s, #0x1f\n" + ".inst 0x44934369 // smlalb z9.s, p4/M, z27.h, z19.h\n" + ".inst 0x449340a7 // smlalb z7.s, p4/M, z5.h, z19.h\n" + "sqadd z14.s, z14.s, z3.s\n" + ".inst 0x448293ae // srshl z14.s, p4/M, z14.s, z29.s\n" + ".inst 0x44954417 // smlalt z23.s, p4/M, z0.h, z21.h\n" + ".inst 0x44934412 // smlalt z18.s, p4/M, z0.h, z19.h\n" + ".inst 0x04b076f7 // sqrdmulh z23.s, z23.s, z16.s\n" + ".inst 0x44934774 // smlalt z20.s, p4/M, z27.h, z19.h\n" + ".inst 0x449344a1 // smlalt z1.s, p4/M, z5.h, z19.h\n" + "and z31.d, z23.d, z22.d\n" + ".inst 0x44954226 // smlalb z6.s, p4/M, z17.h, z21.h\n" + ".inst 0x449540a9 // smlalb z9.s, p4/M, z5.h, z21.h\n" + ".inst 0x04b974c6 // sqrdmulh z6.s, z6.s, z25.s\n" + ".inst 0x44954387 // smlalb z7.s, p4/M, z28.h, z21.h\n" + ".inst 0x44954632 // smlalt z18.s, p4/M, z17.h, z21.h\n" + ".inst 0x04b97529 // sqrdmulh z9.s, z9.s, z25.s\n" + ".inst 0x449544b4 // smlalt z20.s, p4/M, z5.h, z21.h\n" + ".inst 0x44954781 // smlalt z1.s, p4/M, z28.h, z21.h\n" + ".inst 0x04b974e7 // sqrdmulh z7.s, z7.s, z25.s\n" + "asr z31.s, z31.s, #0x1f\n" + "and z3.d, z6.d, z29.d\n" + ".inst 0x04b07652 // sqrdmulh z18.s, z18.s, z16.s\n" + "and z0.d, z9.d, z29.d\n" + ".inst 0x04b07694 // sqrdmulh z20.s, z20.s, z16.s\n" + "and z19.d, z7.d, z29.d\n" + ".inst 0x04b07421 // sqrdmulh z1.s, z1.s, z16.s\n" + "sqadd z23.s, z23.s, z31.s\n" + ".inst 0x448292d7 // srshl z23.s, p4/M, z23.s, z22.s\n" "asr z3.s, z3.s, #0x1f\n" - ".inst 0x448292ee // srshl z14.s, p4/M, z14.s, z23.s\n" - "and z19.d, z20.d, z23.d\n" - ".inst 0x04bd74e7 // sqrdmulh z7.s, z7.s, z29.s\n" - "and z18.d, z8.d, z23.d\n" - ".inst 0x04bd7610 // sqrdmulh z16.s, z16.s, z29.s\n" - "and z21.d, z6.d, z23.d\n" - ".inst 0x04bd74a5 // sqrdmulh z5.s, z5.s, z29.s\n" - "sqadd z10.s, z10.s, z3.s\n" + "and z21.d, z18.d, z22.d\n" + "asr z0.s, z0.s, #0x1f\n" + "and z17.d, z20.d, z22.d\n" "asr z19.s, z19.s, #0x1f\n" - ".inst 0x448292ca // srshl z10.s, p4/M, z10.s, z22.s\n" - "and z1.d, z7.d, z22.d\n" - "asr z18.s, z18.s, #0x1f\n" - "and z2.d, z16.d, z22.d\n" + "and z16.d, z1.d, z22.d\n" + "sqadd z6.s, z6.s, z3.s\n" "asr z21.s, z21.s, #0x1f\n" - "and z3.d, z5.d, z22.d\n" - "sqadd z20.s, z20.s, z19.s\n" - ".inst 0x448292f4 // srshl z20.s, p4/M, z20.s, z23.s\n" - "asr z1.s, z1.s, #0x1f\n" - "sqadd z8.s, z8.s, z18.s\n" - ".inst 0x448292e8 // srshl z8.s, p4/M, z8.s, z23.s\n" - "asr z2.s, z2.s, #0x1f\n" - "sqadd z6.s, z6.s, z21.s\n" - ".inst 0x448292e6 // srshl z6.s, p4/M, z6.s, z23.s\n" - "asr z3.s, z3.s, #0x1f\n" - "sqadd z7.s, z7.s, z1.s\n" - ".inst 0x448292c7 // srshl z7.s, p4/M, z7.s, z22.s\n" - "sqadd z16.s, z16.s, z2.s\n" - "sqadd z5.s, z5.s, z3.s\n" - ".inst 0x448292d0 // srshl z16.s, p4/M, z16.s, z22.s\n" - ".inst 0x448292c5 // srshl z5.s, p4/M, z5.s, z22.s\n" + ".inst 0x448293a6 // srshl z6.s, p4/M, z6.s, z29.s\n" + "sqadd z9.s, z9.s, z0.s\n" + "asr z17.s, z17.s, #0x1f\n" + ".inst 0x448293a9 // srshl z9.s, p4/M, z9.s, z29.s\n" + "sqadd z7.s, z7.s, z19.s\n" + "asr z16.s, z16.s, #0x1f\n" + ".inst 0x448293a7 // srshl z7.s, p4/M, z7.s, z29.s\n" + "sqadd z18.s, z18.s, z21.s\n" + "sqadd z20.s, z20.s, z17.s\n" + ".inst 0x448292d2 // srshl z18.s, p4/M, z18.s, z22.s\n" + ".inst 0x448292d4 // srshl z20.s, p4/M, z20.s, z22.s\n" + "sqadd z1.s, z1.s, z16.s\n" ".inst 0x453041ce // sqxtnb z14.h, z14.s\n" - ".inst 0x45304294 // sqxtnb z20.h, z20.s\n" - ".inst 0x45304108 // sqxtnb z8.h, z8.s\n" + ".inst 0x448292c1 // srshl z1.s, p4/M, z1.s, z22.s\n" ".inst 0x453040c6 // sqxtnb z6.h, z6.s\n" - ".inst 0x4530454e // sqxtnt z14.h, z10.s\n" - ".inst 0x453044f4 // sqxtnt z20.h, z7.s\n" - ".inst 0x45304608 // sqxtnt z8.h, z16.s\n" - ".inst 0x453044a6 // sqxtnt z6.h, z5.s\n" - "sqadd z14.h, z14.h, z12.h\n" - "sqadd z20.h, z20.h, z12.h\n" - "smax z14.h, p4/M, z14.h, z13.h\n" - "smax z20.h, p4/M, z20.h, z13.h\n" - "sqadd z8.h, z8.h, z12.h\n" - "sqadd z6.h, z6.h, z12.h\n" - "smax z8.h, p4/M, z8.h, z13.h\n" - "smax z6.h, p4/M, z6.h, z13.h\n" - "smin z14.h, p4/M, z14.h, z11.h\n" - "smin z20.h, p4/M, z20.h, z11.h\n" - "st1b { z14.h }, p0, [x3, x8]\n" - "smin z8.h, p4/M, z8.h, z11.h\n" - "smin z6.h, p4/M, z6.h, z11.h\n" - "st1b { z20.h }, p0, [x4, x8]\n" - "st1b { z8.h }, p0, [x5, x8]\n" - "st1b { z6.h }, p0, [x6, x8]\n" - "ld1w { z30.s }, p2/Z, [x14]\n" - "ld1w { z16.s }, p1/Z, [x14, #1, MUL VL]\n" - "uzp1 z14.s, z30.s, z16.s\n" - "ld1sb { z0.h }, p4/Z, [x2]\n" - "ld1sb { z1.h }, p4/Z, [x2, #1, MUL VL]\n" - "uzp2 z10.s, z30.s, z16.s\n" - "addvl x14, x14, #2\n" - "ld1sb { z2.h }, p4/Z, [x2, #2, MUL VL]\n" - "ld1sb { z3.h }, p4/Z, [x2, #3, MUL VL]\n" - "inch x8\n" - "str x14, [%x[params], %[offsetof_Params_bias]]\n" - "ld1sb { z4.h }, p4/Z, [x2, #4, MUL VL]\n" - "ldp x9, x28, [x7, #0x0]\n" - "mov z20.d, z14.d\n" - "mov z7.d, z10.d\n" - "ldp x27, x26, [x7, #0x10]\n" - "ldp x25, x24, [x7, #0x20]\n" - "mov z8.d, z14.d\n" - "mov z16.d, z10.d\n" - "ldp x23, x22, [x7, #0x30]\n" - "ldp x21, x20, [x7, #0x40]\n" + ".inst 0x45304129 // sqxtnb z9.h, z9.s\n" + ".inst 0x453040e7 // sqxtnb z7.h, z7.s\n" + ".inst 0x453046ee // sqxtnt z14.h, z23.s\n" + ".inst 0x45304646 // sqxtnt z6.h, z18.s\n" + ".inst 0x45304689 // sqxtnt z9.h, z20.s\n" + ".inst 0x45304427 // sqxtnt z7.h, z1.s\n" + "sqadd z14.h, z14.h, z15.h\n" + "smax z14.h, p4/M, z14.h, z12.h\n" + "smin z14.h, p4/M, z14.h, z13.h\n" + "sqadd z6.h, z6.h, z15.h\n" + "sqadd z9.h, z9.h, z15.h\n" + "smax z6.h, p4/M, z6.h, z12.h\n" + "smax z9.h, p4/M, z9.h, z12.h\n" + "sqadd z7.h, z7.h, z15.h\n" + "smax z7.h, p4/M, z7.h, z12.h\n" + "smin z6.h, p4/M, z6.h, z13.h\n" + "st1b { z14.h }, p0, [x5, x16]\n" + "smin z9.h, p4/M, z9.h, z13.h\n" + "smin z7.h, p4/M, z7.h, z13.h\n" + "st1b { z6.h }, p0, [x6, x16]\n" + "st1b { z9.h }, p0, [x7, x16]\n" + "st1b { z7.h }, p0, [x8, x16]\n" + "ld1w { z17.s }, p2/Z, [x21]\n" + "ld1w { z16.s }, p1/Z, [x21, #1, MUL VL]\n" + "uzp1 z14.s, z17.s, z16.s\n" + "ld1sb { z26.h }, p4/Z, [x4]\n" + "ld1sb { z8.h }, p4/Z, [x4, #1, MUL VL]\n" + "uzp2 z23.s, z17.s, z16.s\n" + "addvl x21, x21, #2\n" + "ld1sb { z16.h }, p4/Z, [x4, #2, MUL VL]\n" + "ld1sb { z21.h }, p4/Z, [x4, #3, MUL VL]\n" + "inch x16\n" + "str x21, [%x[params], %[offsetof_Params_bias]]\n" + "ld1sb { z17.h }, p4/Z, [x4, #4, MUL VL]\n" + "ldp x9, x28, [x17, #0x0]\n" "mov z6.d, z14.d\n" - "mov z5.d, z10.d\n" - "ld1b { z31.h }, p3/Z, [x9, x0]\n" - "ld1b { z30.h }, p3/Z, [x28, x0]\n" - ".inst 0x45511000 // ssublb z0.h, z0.b, z17.b\n" - ".inst 0x45511021 // ssublb z1.h, z1.b, z17.b\n" - "ld1b { z29.h }, p3/Z, [x27, x0]\n" - "ld1b { z28.h }, p3/Z, [x26, x0]\n" - ".inst 0x45511042 // ssublb z2.h, z2.b, z17.b\n" - ".inst 0x45511063 // ssublb z3.h, z3.b, z17.b\n" - "ld1b { z27.h }, p3/Z, [x25, x0]\n" - "ld1b { z23.h }, p3/Z, [x24, x0]\n" - ".inst 0x45511084 // ssublb z4.h, z4.b, z17.b\n" - ".inst 0x454f1bff // usublb z31.h, z31.b, z15.b\n" - "ld1b { z25.h }, p3/Z, [x23, x0]\n" - "ld1b { z24.h }, p3/Z, [x22, x0]\n" - ".inst 0x454f1bde // usublb z30.h, z30.b, z15.b\n" - ".inst 0x454f1bbd // usublb z29.h, z29.b, z15.b\n" - "ld1b { z26.h }, p3/Z, [x21, x0]\n" - "ld1b { z22.h }, p3/Z, [x20, x0]\n" - ".inst 0x454f1b9c // usublb z28.h, z28.b, z15.b\n" - ".inst 0x454f1b7b // usublb z27.h, z27.b, z15.b\n" - ".inst 0x454f1af7 // usublb z23.h, z23.b, z15.b\n" - ".inst 0x454f1b39 // usublb z25.h, z25.b, z15.b\n" - ".inst 0x454f1b18 // usublb z24.h, z24.b, z15.b\n" - ".inst 0x454f1b5a // usublb z26.h, z26.b, z15.b\n" - ".inst 0x454f1ad6 // usublb z22.h, z22.b, z15.b\n" + "mov z18.d, z23.d\n" + "ldp x27, x26, [x17, #0x10]\n" + "ldp x25, x24, [x17, #0x20]\n" + "mov z9.d, z14.d\n" + "mov z20.d, z23.d\n" + "ldp x23, x22, [x17, #0x30]\n" + "ldp x21, x20, [x17, #0x40]\n" + "mov z7.d, z14.d\n" + "mov z1.d, z23.d\n" + "ld1b { z22.h }, p3/Z, [x9, x2]\n" + "ld1b { z2.h }, p3/Z, [x28, x2]\n" + ".inst 0x454a135a // ssublb z26.h, z26.b, z10.b\n" + ".inst 0x454a1108 // ssublb z8.h, z8.b, z10.b\n" + "ld1b { z11.h }, p3/Z, [x27, x2]\n" + "ld1b { z3.h }, p3/Z, [x26, x2]\n" + ".inst 0x454a1210 // ssublb z16.h, z16.b, z10.b\n" + ".inst 0x454a12b5 // ssublb z21.h, z21.b, z10.b\n" + "ld1b { z29.h }, p3/Z, [x25, x2]\n" + "ld1b { z4.h }, p3/Z, [x24, x2]\n" + ".inst 0x454a1231 // ssublb z17.h, z17.b, z10.b\n" + ".inst 0x455e1ad6 // usublb z22.h, z22.b, z30.b\n" + "ld1b { z31.h }, p3/Z, [x23, x2]\n" + "ld1b { z0.h }, p3/Z, [x22, x2]\n" + ".inst 0x455e1842 // usublb z2.h, z2.b, z30.b\n" + ".inst 0x455e196b // usublb z11.h, z11.b, z30.b\n" + "ld1b { z19.h }, p3/Z, [x21, x2]\n" + "ld1b { z28.h }, p3/Z, [x20, x2]\n" + ".inst 0x455e1863 // usublb z3.h, z3.b, z30.b\n" + ".inst 0x455e1bbd // usublb z29.h, z29.b, z30.b\n" + ".inst 0x455e1884 // usublb z4.h, z4.b, z30.b\n" + ".inst 0x455e1bff // usublb z31.h, z31.b, z30.b\n" + ".inst 0x455e1800 // usublb z0.h, z0.b, z30.b\n" + ".inst 0x455e1a73 // usublb z19.h, z19.b, z30.b\n" + ".inst 0x455e1b9c // usublb z28.h, z28.b, z30.b\n" "b.any 1b\n" : : [offsetof_Params_bias] "I" (offsetof(Params, bias)), [offsetof_Params_inptrs] "I" (offsetof(Params, inptrs)), [offsetof_Params_n_channels] "I" (offsetof(Params, n_channels)), [offsetof_Params_outptrs] "I" (offsetof(Params, outptrs)), [offsetof_Params_requant] "I" (offsetof(Params, requant)), [offsetof_Params_requant_muls] "I" (offsetof(Params, requant_muls)), [offsetof_Params_requant_shifts] "I" (offsetof(Params, requant_shifts)), [offsetof_Params_weights] "I" (offsetof(Params, weights)), [offsetof_Requantize32_a_offset] "I" (offsetof(arm_gemm::Requantize32, a_offset)), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [params] "r" (¶ms) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/premultiply.cpp b/src/core/NEON/kernels/arm_conv/depthwise/premultiply.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8a49c775d321ab110c6d387bc011344ccb1c30df --- /dev/null +++ b/src/core/NEON/kernels/arm_conv/depthwise/premultiply.cpp @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include + +#define CHANNEL_MULTIPLIER 6 +#define BLOCK_SIZE 4 + +void do_premultiply_float_6(const float *in_ptr, + const unsigned int ld_row, + const unsigned int ld_col, + float *out_ptr, + const unsigned int out_ld_row, + const unsigned int out_ld_col, + const unsigned int tile_rows, + const unsigned int tile_cols, + const unsigned input_channels) +{ + for(unsigned int i = 0; i < tile_rows; i++) + { + const float *ip2 = in_ptr + i * ld_row; + float *op2 = out_ptr + i * out_ld_row; + for(unsigned int j = 0; j < tile_cols; j++) + { + const float *ip = ip2; + float *op = op2; + + unsigned int num_blocks = input_channels / BLOCK_SIZE; + for(unsigned int c = 0; c < num_blocks; c++) + { + float vals[BLOCK_SIZE]; + for(unsigned int v = 0; v < BLOCK_SIZE; v++) + { + vals[v] = ip[v]; + } + ip += BLOCK_SIZE; + + for(unsigned int v = 0; v < BLOCK_SIZE; v++) + { + for(unsigned int r = 0; r < CHANNEL_MULTIPLIER; r++) + { + op[r] = vals[v]; + } + op += CHANNEL_MULTIPLIER; + } + } + + unsigned int rem = input_channels - num_blocks * BLOCK_SIZE; + for(unsigned int c = 0; c < rem; c++) + { + float val = ip[c]; + for(unsigned int r = 0; r < CHANNEL_MULTIPLIER; r++) + { + op[r] = val; + } + op += CHANNEL_MULTIPLIER; + } + + ip2 += ld_col; + op2 += out_ld_col; + } + } +} diff --git a/src/core/NEON/kernels/arm_conv/depthwise/working_space.hpp b/src/core/NEON/kernels/arm_conv/depthwise/working_space.hpp index e9b29ca87764b7706837a3e6909a73d4fab4f4be..9805fd354f5565eb30c9596496ddc795e1ed56e6 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/working_space.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/working_space.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -91,7 +91,7 @@ #include "depthwise.hpp" #include "depthfirst_driver.hpp" -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" namespace arm_conv { namespace depthwise { @@ -217,7 +217,7 @@ class InputBufferElement template static size_t get_element_size(const WorkspaceArgs &args) { - return sizeof(T) * args.depthwise_args.input_channels; + return sizeof(T) * args.depthwise_args.input_channels * args.depthwise_args.channel_multiplier; } template @@ -278,6 +278,36 @@ class OutputArrayElement }; +/* Intermediate array to store results of premultiplication. + * Used as input to the kernel instead of the original input array. + */ +template +class IntermediateBufferElement +{ +public: + struct Workspace + { + T *intermediate_buffer; + }; + + template + static size_t get_element_size(const WorkspaceArgs &args) + { + auto cols = args.depthwise_args.input_cols + args.depthwise_args.kernel_cols; + auto rows = args.strategy->get_input_rows() + args.depthwise_args.kernel_rows; + auto channels = args.depthwise_args.input_channels * args.depthwise_args.channel_multiplier; + return sizeof(T) * cols * rows * channels; + } + + template + static void *initialise(WorkspaceType *ws, void *buffer, const WorkspaceArgs &args) + { + ws->intermediate_buffer = reinterpret_cast(buffer); + return reinterpret_cast(buffer) + get_element_size(args); + } +}; + + /* Container for requantization parameters. * * This removes the distinction between per-layer and per-channel diff --git a/src/core/NEON/kernels/arm_conv/pooling/depthfirst_driver.hpp b/src/core/NEON/kernels/arm_conv/pooling/depthfirst_driver.hpp index 8473fc0838a9cf67e90cb44e870c4d39f4775d89..d0e8639229a219a77788eac4d6d013e7f16060ad 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/depthfirst_driver.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/depthfirst_driver.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,7 +25,7 @@ #pragma once #include "pooling.hpp" -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "utils.hpp" namespace arm_conv { namespace pooling { @@ -64,10 +64,10 @@ class DepthfirstDriver : public PoolingCommon std::unique_ptr m_strat; /* Compute the amount of working space required for a single thread. */ - virtual size_t get_working_size_per_thread(unsigned int n_input_channels) const = 0; + virtual size_t get_working_size_per_thread() const = 0; /* Initialise the working space for a thread. */ - virtual void initialise_working_space(void *, unsigned int n_input_channels) const = 0; + virtual void initialise_working_space(void *) const = 0; /* Compute a portion of the output tensor with padding. */ virtual void compute_tile_padded( @@ -148,8 +148,8 @@ class DepthfirstDriver : public PoolingCommon { // Get and initialise the working space for this thread. void *thread_working_space = - static_cast(working_space) + thread_id * this->get_working_size_per_thread(n_channels); - this->initialise_working_space(thread_working_space, n_channels); + static_cast(working_space) + thread_id * this->get_working_size_per_thread(); + this->initialise_working_space(thread_working_space); // Construct convenient representations of the input/output tensors. TensorSpec input_tensor(reinterpret_cast(input), ld_input_row, ld_input_col); @@ -289,14 +289,9 @@ class DepthfirstDriver : public PoolingCommon { } - size_t get_working_size(unsigned int n_threads) const override + size_t get_working_size(unsigned int n_threads) const override final { - return this->get_working_size(n_threads, this->m_args.n_channels); - } - - size_t get_working_size(unsigned int n_threads, unsigned int n_channels) const override final - { - return n_threads * this->get_working_size_per_thread(n_channels); + return n_threads * this->get_working_size_per_thread(); } }; diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst.hpp index a670bb81bb34c6e89ce42b2247126072589afbe0..6b3ebe66644ee594dd278002d7f0c83ee17fbc18 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,7 +24,7 @@ #pragma once -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace pooling { @@ -48,4 +48,4 @@ struct a64_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst : public DepthfirstStrategy } // namespace pooling } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp index 647103d3a46d4950e5408ac5ba2cd0de01b3f142..5df848d1ddf42ea2f527c0bd2e489a688a6d668f 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp @@ -196,38 +196,38 @@ void a64_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst_impl( "add x5, x5, #0x10\n" "cbz x3, 4f\n" "3:" // Oddments - "ldr h6, [x11, x4]\n" - "ldr h5, [x10, x4]\n" - "fadd v17.8h, v6.8h, v5.8h\n" + "ldr h17, [x11, x4]\n" + "ldr h16, [x10, x4]\n" + "fadd v18.8h, v17.8h, v16.8h\n" "subs x3, x3, #0x1\n" - "ldr h4, [x27, x4]\n" - "ldr h3, [x26, x4]\n" - "fadd v16.8h, v4.8h, v3.8h\n" - "fadd v19.8h, v17.8h, v16.8h\n" - "ldr h2, [x15, x4]\n" - "ldr h1, [x14, x4]\n" - "fadd v18.8h, v2.8h, v1.8h\n" - "fadd v21.8h, v18.8h, v19.8h\n" - "ldr h0, [x12, x4]\n" - "ldr h31, [x28, x4]\n" - "fadd v17.8h, v0.8h, v31.8h\n" - "ldr h30, [x9, x4]\n" - "ldr h29, [x25, x4]\n" - "fadd v22.8h, v30.8h, v29.8h\n" - "ldr h28, [x23, x4]\n" - "ldr h27, [x22, x4]\n" - "fadd v16.8h, v28.8h, v27.8h\n" - "fadd v20.8h, v16.8h, v19.8h\n" - "ldr h26, [x16, x4]\n" - "ldr h25, [x13, x4]\n" - "fadd v19.8h, v26.8h, v17.8h\n" - "fadd v18.8h, v25.8h, v22.8h\n" - "ldr h24, [x24, x4]\n" - "ldr h23, [x21, x4]\n" - "fadd v17.8h, v24.8h, v17.8h\n" - "fadd v16.8h, v23.8h, v22.8h\n" - "fadd v19.8h, v21.8h, v19.8h\n" - "fadd v18.8h, v21.8h, v18.8h\n" + "ldr h17, [x27, x4]\n" + "ldr h16, [x26, x4]\n" + "fadd v16.8h, v17.8h, v16.8h\n" + "fadd v18.8h, v18.8h, v16.8h\n" + "ldr h17, [x15, x4]\n" + "ldr h16, [x14, x4]\n" + "fadd v16.8h, v17.8h, v16.8h\n" + "fadd v23.8h, v16.8h, v18.8h\n" + "ldr h17, [x12, x4]\n" + "ldr h16, [x28, x4]\n" + "fadd v22.8h, v17.8h, v16.8h\n" + "ldr h17, [x9, x4]\n" + "ldr h16, [x25, x4]\n" + "fadd v21.8h, v17.8h, v16.8h\n" + "ldr h17, [x23, x4]\n" + "ldr h16, [x22, x4]\n" + "fadd v16.8h, v17.8h, v16.8h\n" + "fadd v20.8h, v16.8h, v18.8h\n" + "ldr h17, [x16, x4]\n" + "ldr h16, [x13, x4]\n" + "fadd v19.8h, v17.8h, v22.8h\n" + "fadd v18.8h, v16.8h, v21.8h\n" + "ldr h17, [x24, x4]\n" + "ldr h16, [x21, x4]\n" + "fadd v17.8h, v17.8h, v22.8h\n" + "fadd v16.8h, v16.8h, v21.8h\n" + "fadd v19.8h, v23.8h, v19.8h\n" + "fadd v18.8h, v23.8h, v18.8h\n" "add x4, x4, #0x2\n" "fadd v17.8h, v17.8h, v20.8h\n" "fadd v16.8h, v16.8h, v20.8h\n" diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_generic_depthfirst/generic.cpp index 44adb4ffcfec962932a51274e7da321c08ef46c2..f7be92e53f04328cd9e477f72dbee6e57d2deab1 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_generic_depthfirst/generic.cpp @@ -22,7 +22,6 @@ * SOFTWARE. */ - #include #include @@ -45,77 +44,77 @@ void a64_fp16_nhwc_avg_generic_depthfirst_impl( __asm__ __volatile__( "ld1r { v9.8h }, [%x[rescale_ptr]]\n" "cmp %x[n_channels], #0x20\n" - "mov x9, #0x0\n" - "mov x28, #0x10\n" // cntb _, ALL, #1 - "mov x27, #0x20\n" // cntb _, ALL, #2 - "mov x26, #0x30\n" // cntb _, ALL, #3 + "mov x27, #0x0\n" + "mov x26, #0x10\n" // cntb _, ALL, #1 + "mov x24, #0x20\n" // cntb _, ALL, #2 + "mov x23, #0x30\n" // cntb _, ALL, #3 "blt 7f\n" "1:" // 4-vectors of channels "lsr x25, %x[n_valid_cells], #0x2\n" "movi v8.16b, #0x0\n" "movi v7.16b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "movi v6.16b, #0x0\n" "movi v5.16b, #0x0\n" "cbz x25, 4f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" "subs x25, x25, #0x1\n" - "ldr q3, [x23, x9]\n" - "ldr q2, [x24, x28]\n" - "ldr q1, [x23, x28]\n" - "ldr q0, [x24, x27]\n" - "ldr q31, [x23, x27]\n" - "ldr q30, [x24, x26]\n" - "ldr q29, [x23, x26]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" - "ldr q27, [x22, x28]\n" - "ldr q21, [x21, x28]\n" - "ldr q26, [x22, x27]\n" - "ldr q20, [x21, x27]\n" - "ldr q25, [x22, x26]\n" - "ldr q24, [x21, x26]\n" + "ldr q3, [x20, x27]\n" + "ldr q2, [x21, x26]\n" + "ldr q1, [x20, x26]\n" + "ldr q0, [x21, x24]\n" + "ldr q31, [x20, x24]\n" + "ldr q30, [x21, x23]\n" + "ldr q29, [x20, x23]\n" + "ldp x21, x20, [x22, #0x10]\n" + "add x22, x22, #0x20\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" + "ldr q27, [x21, x26]\n" + "ldr q21, [x20, x26]\n" + "ldr q26, [x21, x24]\n" + "ldr q20, [x20, x24]\n" + "ldr q25, [x21, x23]\n" + "ldr q24, [x20, x23]\n" "beq 3f\n" "2:" // 4-vectors of channels: 4 inputs loop "fadd v23.8h, v4.8h, v3.8h\n" "fadd v19.8h, v28.8h, v22.8h\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" - "ldr q3, [x23, x9]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" + "ldr q3, [x20, x27]\n" "fadd v22.8h, v2.8h, v1.8h\n" - "ldr q2, [x24, x28]\n" + "ldr q2, [x21, x26]\n" "fadd v18.8h, v27.8h, v21.8h\n" - "ldr q1, [x23, x28]\n" + "ldr q1, [x20, x26]\n" "fadd v21.8h, v0.8h, v31.8h\n" - "ldr q0, [x24, x27]\n" + "ldr q0, [x21, x24]\n" "fadd v17.8h, v26.8h, v20.8h\n" - "ldr q31, [x23, x27]\n" + "ldr q31, [x20, x24]\n" "fadd v20.8h, v30.8h, v29.8h\n" - "ldr q30, [x24, x26]\n" + "ldr q30, [x21, x23]\n" "fadd v16.8h, v25.8h, v24.8h\n" - "ldr q29, [x23, x26]\n" + "ldr q29, [x20, x23]\n" "fadd v19.8h, v23.8h, v19.8h\n" "fadd v18.8h, v22.8h, v18.8h\n" - "ldp x22, x21, [x20, #0x10]\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" + "ldp x21, x20, [x22, #0x10]\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" "fadd v17.8h, v21.8h, v17.8h\n" "fadd v16.8h, v20.8h, v16.8h\n" - "ldr q27, [x22, x28]\n" - "ldr q21, [x21, x28]\n" + "ldr q27, [x21, x26]\n" + "ldr q21, [x20, x26]\n" "subs x25, x25, #0x1\n" "fadd v8.8h, v8.8h, v19.8h\n" - "ldr q26, [x22, x27]\n" - "ldr q20, [x21, x27]\n" + "ldr q26, [x21, x24]\n" + "ldr q20, [x20, x24]\n" "fadd v7.8h, v7.8h, v18.8h\n" "fadd v6.8h, v6.8h, v17.8h\n" - "ldr q25, [x22, x26]\n" - "ldr q24, [x21, x26]\n" + "ldr q25, [x21, x23]\n" + "ldr q24, [x20, x23]\n" "fadd v5.8h, v5.8h, v16.8h\n" - "add x20, x20, #0x20\n" + "add x22, x22, #0x20\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "fadd v23.8h, v4.8h, v3.8h\n" @@ -138,16 +137,16 @@ void a64_fp16_nhwc_avg_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x24, [x20], #0x8\n" - "ldr q4, [x24, x9]\n" + "ldr x20, [x22], #0x8\n" + "ldr q16, [x20, x27]\n" "subs x21, x21, #0x1\n" - "fadd v8.8h, v8.8h, v4.8h\n" - "ldr q2, [x24, x28]\n" - "ldr q0, [x24, x27]\n" - "fadd v7.8h, v7.8h, v2.8h\n" - "fadd v6.8h, v6.8h, v0.8h\n" - "ldr q30, [x24, x26]\n" - "fadd v5.8h, v5.8h, v30.8h\n" + "fadd v8.8h, v8.8h, v16.8h\n" + "ldr q17, [x20, x26]\n" + "ldr q16, [x20, x24]\n" + "fadd v7.8h, v7.8h, v17.8h\n" + "fadd v6.8h, v6.8h, v16.8h\n" + "ldr q16, [x20, x23]\n" + "fadd v5.8h, v5.8h, v16.8h\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End "sub %x[n_channels], %x[n_channels], #0x20\n" @@ -156,14 +155,14 @@ void a64_fp16_nhwc_avg_generic_depthfirst_impl( "fmul v7.8h, v7.8h, v9.8h\n" "fmul v6.8h, v6.8h, v9.8h\n" "fmul v5.8h, v5.8h, v9.8h\n" - "str q8, [%x[outptr], x9]\n" - "add x9, x9, #0x40\n" - "str q7, [%x[outptr], x28]\n" - "add x28, x28, #0x40\n" - "str q6, [%x[outptr], x27]\n" + "str q8, [%x[outptr], x27]\n" "add x27, x27, #0x40\n" - "str q5, [%x[outptr], x26]\n" + "str q7, [%x[outptr], x26]\n" "add x26, x26, #0x40\n" + "str q6, [%x[outptr], x24]\n" + "add x24, x24, #0x40\n" + "str q5, [%x[outptr], x23]\n" + "add x23, x23, #0x40\n" "bge 1b\n" "cbz %x[n_channels], 31f\n" "7:" // Single vector of channels @@ -172,146 +171,146 @@ void a64_fp16_nhwc_avg_generic_depthfirst_impl( "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" "movi v8.16b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" "subs x25, x25, #0x1\n" - "ldr q3, [x23, x9]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" + "ldr q3, [x20, x27]\n" + "ldp x21, x20, [x22, #0x10]\n" + "add x22, x22, #0x20\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 4 inputs loop - "fadd v23.8h, v4.8h, v3.8h\n" - "fadd v19.8h, v28.8h, v22.8h\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" - "ldr q3, [x23, x9]\n" - "fadd v19.8h, v23.8h, v19.8h\n" - "ldp x22, x21, [x20, #0x10]\n" + "fadd v17.8h, v4.8h, v3.8h\n" + "fadd v16.8h, v28.8h, v22.8h\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" + "ldr q3, [x20, x27]\n" + "fadd v16.8h, v17.8h, v16.8h\n" + "ldp x21, x20, [x22, #0x10]\n" "subs x25, x25, #0x1\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" - "fadd v8.8h, v8.8h, v19.8h\n" - "add x20, x20, #0x20\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" + "fadd v8.8h, v8.8h, v16.8h\n" + "add x22, x22, #0x20\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail - "fadd v23.8h, v4.8h, v3.8h\n" - "fadd v19.8h, v28.8h, v22.8h\n" - "fadd v19.8h, v23.8h, v19.8h\n" - "fadd v8.8h, v8.8h, v19.8h\n" + "fadd v17.8h, v4.8h, v3.8h\n" + "fadd v16.8h, v28.8h, v22.8h\n" + "fadd v16.8h, v17.8h, v16.8h\n" + "fadd v8.8h, v8.8h, v16.8h\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x24, [x20], #0x8\n" - "ldr q4, [x24, x9]\n" + "ldr x20, [x22], #0x8\n" + "ldr q16, [x20, x27]\n" "subs x21, x21, #0x1\n" - "fadd v8.8h, v8.8h, v4.8h\n" + "fadd v8.8h, v8.8h, v16.8h\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "sub %x[n_channels], %x[n_channels], #0x8\n" "cmp %x[n_channels], #0x8\n" "fmul v8.8h, v8.8h, v9.8h\n" - "str q8, [%x[outptr], x9]\n" - "add x9, x9, #0x10\n" + "str q8, [%x[outptr], x27]\n" + "add x27, x27, #0x10\n" "bge 8b\n" "cbz %x[n_channels], 31f\n" "14:" // Oddments "lsr x25, %x[n_valid_cells], #0x2\n" - "add %x[outptr], %x[outptr], x9\n" + "add %x[outptr], %x[outptr], x27\n" "movi v8.16b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "cbz x25, 20f\n" "15:" // Oddments: 4 inputs loop - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "add x24, x24, x9\n" - "add x23, x23, x9\n" - "add x22, x22, x9\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" + "add x23, x23, x27\n" + "add x22, x22, x27\n" + "add x21, x21, x27\n" "movi v4.16b, #0x0\n" "movi v3.16b, #0x0\n" - "add x21, x21, x9\n" + "add x20, x20, x27\n" "movi v28.16b, #0x0\n" "movi v22.16b, #0x0\n" "tbz %x[n_channels], #2, 17f\n" - "ldr d4, [x24], #0x8\n" - "ldr d3, [x23], #0x8\n" - "ldr d28, [x22], #0x8\n" - "ldr d22, [x21], #0x8\n" + "ldr d4, [x23], #0x8\n" + "ldr d3, [x22], #0x8\n" + "ldr d28, [x21], #0x8\n" + "ldr d22, [x20], #0x8\n" "tbz %x[n_channels], #1, 16f\n" - "ld1 { v4.s }[2], [x24], #0x4\n" - "ld1 { v3.s }[2], [x23], #0x4\n" - "ld1 { v28.s }[2], [x22], #0x4\n" - "ld1 { v22.s }[2], [x21], #0x4\n" + "ld1 { v4.s }[2], [x23], #0x4\n" + "ld1 { v3.s }[2], [x22], #0x4\n" + "ld1 { v28.s }[2], [x21], #0x4\n" + "ld1 { v22.s }[2], [x20], #0x4\n" "tbz %x[n_channels], #0, 19f\n" - "ld1 { v4.h }[6], [x24], #0x2\n" - "ld1 { v3.h }[6], [x23], #0x2\n" - "ld1 { v28.h }[6], [x22], #0x2\n" - "ld1 { v22.h }[6], [x21], #0x2\n" + "ld1 { v4.h }[6], [x23], #0x2\n" + "ld1 { v3.h }[6], [x22], #0x2\n" + "ld1 { v28.h }[6], [x21], #0x2\n" + "ld1 { v22.h }[6], [x20], #0x2\n" "b 19f\n" "16:" // Oddments: 4 inputs loop: Load: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 19f\n" - "ld1 { v4.h }[4], [x24], #0x2\n" - "ld1 { v3.h }[4], [x23], #0x2\n" - "ld1 { v28.h }[4], [x22], #0x2\n" - "ld1 { v22.h }[4], [x21], #0x2\n" + "ld1 { v4.h }[4], [x23], #0x2\n" + "ld1 { v3.h }[4], [x22], #0x2\n" + "ld1 { v28.h }[4], [x21], #0x2\n" + "ld1 { v22.h }[4], [x20], #0x2\n" "b 19f\n" "17:" // Oddments: 4 inputs loop: Load: Bit 2: Unset "tbz %x[n_channels], #1, 18f\n" - "ldr s4, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr s28, [x22], #0x4\n" - "ldr s22, [x21], #0x4\n" + "ldr s4, [x23], #0x4\n" + "ldr s3, [x22], #0x4\n" + "ldr s28, [x21], #0x4\n" + "ldr s22, [x20], #0x4\n" "tbz %x[n_channels], #0, 19f\n" - "ld1 { v4.h }[2], [x24], #0x2\n" - "ld1 { v3.h }[2], [x23], #0x2\n" - "ld1 { v28.h }[2], [x22], #0x2\n" - "ld1 { v22.h }[2], [x21], #0x2\n" + "ld1 { v4.h }[2], [x23], #0x2\n" + "ld1 { v3.h }[2], [x22], #0x2\n" + "ld1 { v28.h }[2], [x21], #0x2\n" + "ld1 { v22.h }[2], [x20], #0x2\n" "b 19f\n" "18:" // Oddments: 4 inputs loop: Load: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 19f\n" - "ldr h4, [x24], #0x2\n" - "ldr h3, [x23], #0x2\n" - "ldr h28, [x22], #0x2\n" - "ldr h22, [x21], #0x2\n" + "ldr h4, [x23], #0x2\n" + "ldr h3, [x22], #0x2\n" + "ldr h28, [x21], #0x2\n" + "ldr h22, [x20], #0x2\n" "19:" // Oddments: 4 inputs loop: Load: Bit 2: End - "fadd v23.8h, v4.8h, v3.8h\n" - "fadd v19.8h, v28.8h, v22.8h\n" + "fadd v17.8h, v4.8h, v3.8h\n" + "fadd v16.8h, v28.8h, v22.8h\n" "subs x25, x25, #0x1\n" - "fadd v19.8h, v23.8h, v19.8h\n" - "fadd v8.8h, v8.8h, v19.8h\n" + "fadd v16.8h, v17.8h, v16.8h\n" + "fadd v8.8h, v8.8h, v16.8h\n" "bgt 15b\n" "20:" // Oddments: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 26f\n" "21:" // Oddments: Single input loop - "ldr x24, [x20], #0x8\n" - "add x24, x24, x9\n" + "ldr x23, [x24], #0x8\n" + "add x23, x23, x27\n" "movi v4.16b, #0x0\n" "tbz %x[n_channels], #2, 23f\n" - "ldr d4, [x24], #0x8\n" + "ldr d4, [x23], #0x8\n" "tbz %x[n_channels], #1, 22f\n" - "ld1 { v4.s }[2], [x24], #0x4\n" + "ld1 { v4.s }[2], [x23], #0x4\n" "tbz %x[n_channels], #0, 25f\n" - "ld1 { v4.h }[6], [x24], #0x2\n" + "ld1 { v4.h }[6], [x23], #0x2\n" "b 25f\n" "22:" // Oddments: Single input loop: Load: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 25f\n" - "ld1 { v4.h }[4], [x24], #0x2\n" + "ld1 { v4.h }[4], [x23], #0x2\n" "b 25f\n" "23:" // Oddments: Single input loop: Load: Bit 2: Unset "tbz %x[n_channels], #1, 24f\n" - "ldr s4, [x24], #0x4\n" + "ldr s4, [x23], #0x4\n" "tbz %x[n_channels], #0, 25f\n" - "ld1 { v4.h }[2], [x24], #0x2\n" + "ld1 { v4.h }[2], [x23], #0x2\n" "b 25f\n" "24:" // Oddments: Single input loop: Load: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 25f\n" - "ldr h4, [x24], #0x2\n" + "ldr h4, [x23], #0x2\n" "25:" // Oddments: Single input loop: Load: Bit 2: End "subs x21, x21, #0x1\n" "fadd v8.8h, v8.8h, v4.8h\n" @@ -342,7 +341,7 @@ void a64_fp16_nhwc_avg_generic_depthfirst_impl( "31:" // End : [n_channels] "+&r" (n_channels), [outptr] "+&r" (outptr) : [inptrs] "r" (inptrs), [n_valid_cells] "r" (n_valid_cells), [rescale_ptr] "r" (&rescale_value) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27" ); } diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_2x2_s1_output2x2_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_2x2_s1_output2x2_depthfirst.hpp index 23a9164b76f31715dee7abef1e3cb99e3413481c..b65ac7e9fa52c88da66ec904ca8a54d7f9b8d66f 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_2x2_s1_output2x2_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_2x2_s1_output2x2_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,7 +24,7 @@ #pragma once -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace pooling { @@ -48,4 +48,4 @@ struct a64_fp16_nhwc_max_2x2_s1_output2x2_depthfirst : public DepthfirstStrategy } // namespace pooling } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp index 8041453cb16e0073d64ef7ba248fb598158a0273..4b073b90766b2cccf8f3ed010e981295d3b8fa50 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp @@ -22,6 +22,7 @@ * SOFTWARE. */ + #include #include @@ -111,7 +112,7 @@ void a64_fp16_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "fmax v18.8h, v18.8h, v21.8h\n" "fmax v17.8h, v17.8h, v20.8h\n" "add x15, x15, #0x10\n" - "fmax v16.8h, v16.8h, v20.8h\n" + "fmax v16.8h, v20.8h, v16.8h\n" "str q19, [x14, x12]\n" "str q18, [x13, x12]\n" "str q17, [x11, x12]\n" @@ -121,43 +122,43 @@ void a64_fp16_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "2:" // Vector: Tail "fmax v21.8h, v30.8h, v29.8h\n" "fmax v20.8h, v29.8h, v28.8h\n" - "fmax v19.8h, v27.8h, v26.8h\n" + "fmax v16.8h, v27.8h, v26.8h\n" "fmax v18.8h, v25.8h, v24.8h\n" "fmax v17.8h, v27.8h, v23.8h\n" - "fmax v16.8h, v24.8h, v22.8h\n" - "fmax v19.8h, v21.8h, v19.8h\n" + "fmax v19.8h, v24.8h, v22.8h\n" + "fmax v16.8h, v21.8h, v16.8h\n" "fmax v18.8h, v18.8h, v21.8h\n" - "str q19, [x14, x12]\n" + "str q16, [x14, x12]\n" "fmax v17.8h, v17.8h, v20.8h\n" - "fmax v16.8h, v16.8h, v20.8h\n" + "fmax v16.8h, v20.8h, v19.8h\n" "str q18, [x13, x12]\n" "str q17, [x11, x12]\n" "str q16, [x10, x12]\n" "add x12, x12, #0x10\n" "cbz x16, 4f\n" "3:" // Oddments - "ldr h30, [x28, x15]\n" - "ldr h29, [x25, x15]\n" - "fmax v21.8h, v30.8h, v29.8h\n" + "ldr h16, [x28, x15]\n" + "ldr h17, [x25, x15]\n" + "fmax v23.8h, v16.8h, v17.8h\n" "subs x16, x16, #0x1\n" - "ldr h28, [x22, x15]\n" - "ldr h27, [x26, x15]\n" - "fmax v20.8h, v29.8h, v28.8h\n" - "ldr h26, [x9, x15]\n" - "ldr h25, [x27, x15]\n" - "fmax v19.8h, v27.8h, v26.8h\n" - "fmax v19.8h, v21.8h, v19.8h\n" - "ldr h24, [x24, x15]\n" - "ldr h23, [x23, x15]\n" - "fmax v18.8h, v25.8h, v24.8h\n" - "fmax v17.8h, v27.8h, v23.8h\n" - "ldr h22, [x21, x15]\n" - "fmax v16.8h, v24.8h, v22.8h\n" + "ldr h16, [x22, x15]\n" + "ldr h22, [x26, x15]\n" + "fmax v21.8h, v17.8h, v16.8h\n" + "ldr h16, [x9, x15]\n" + "ldr h17, [x27, x15]\n" + "fmax v16.8h, v22.8h, v16.8h\n" + "fmax v20.8h, v23.8h, v16.8h\n" + "ldr h19, [x24, x15]\n" + "ldr h16, [x23, x15]\n" + "fmax v18.8h, v17.8h, v19.8h\n" + "fmax v17.8h, v22.8h, v16.8h\n" + "ldr h16, [x21, x15]\n" + "fmax v16.8h, v19.8h, v16.8h\n" "add x15, x15, #0x2\n" - "fmax v18.8h, v18.8h, v21.8h\n" - "fmax v17.8h, v17.8h, v20.8h\n" - "fmax v16.8h, v16.8h, v20.8h\n" - "str h19, [x14, x12]\n" + "fmax v18.8h, v18.8h, v23.8h\n" + "fmax v17.8h, v17.8h, v21.8h\n" + "fmax v16.8h, v21.8h, v16.8h\n" + "str h20, [x14, x12]\n" "str h18, [x13, x12]\n" "str h17, [x11, x12]\n" "str h16, [x10, x12]\n" diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_generic_depthfirst/generic.cpp index e4de9fb79c77771b5900014e60770a70b445171e..c92e2cdebd819d6cc362abdf90fd70a08776df3b 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_generic_depthfirst/generic.cpp @@ -41,10 +41,10 @@ void a64_fp16_nhwc_max_generic_depthfirst_impl( { __asm__ __volatile__( "cmp %x[n_channels], #0x20\n" - "mov x9, #0x0\n" - "mov x28, #0x10\n" // cntb _, ALL, #1 - "mov x27, #0x20\n" // cntb _, ALL, #2 - "mov x26, #0x30\n" // cntb _, ALL, #3 + "mov x27, #0x0\n" + "mov x26, #0x10\n" // cntb _, ALL, #1 + "mov x24, #0x20\n" // cntb _, ALL, #2 + "mov x23, #0x30\n" // cntb _, ALL, #3 "blt 7f\n" "1:" // 4-vectors of channels "mov w20, #0xfc00\n" @@ -53,66 +53,66 @@ void a64_fp16_nhwc_max_generic_depthfirst_impl( "dup v7.8h, w20\n" "dup v6.8h, w20\n" "dup v5.8h, w20\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "cbz x25, 4f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" "subs x25, x25, #0x1\n" - "ldr q3, [x23, x9]\n" - "ldr q2, [x24, x28]\n" - "ldr q1, [x23, x28]\n" - "ldr q0, [x24, x27]\n" - "ldr q31, [x23, x27]\n" - "ldr q30, [x24, x26]\n" - "ldr q29, [x23, x26]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" - "ldr q27, [x22, x28]\n" - "ldr q21, [x21, x28]\n" - "ldr q26, [x22, x27]\n" - "ldr q20, [x21, x27]\n" - "ldr q25, [x22, x26]\n" - "ldr q24, [x21, x26]\n" + "ldr q3, [x20, x27]\n" + "ldr q2, [x21, x26]\n" + "ldr q1, [x20, x26]\n" + "ldr q0, [x21, x24]\n" + "ldr q31, [x20, x24]\n" + "ldr q30, [x21, x23]\n" + "ldr q29, [x20, x23]\n" + "ldp x21, x20, [x22, #0x10]\n" + "add x22, x22, #0x20\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" + "ldr q27, [x21, x26]\n" + "ldr q21, [x20, x26]\n" + "ldr q26, [x21, x24]\n" + "ldr q20, [x20, x24]\n" + "ldr q25, [x21, x23]\n" + "ldr q24, [x20, x23]\n" "beq 3f\n" "2:" // 4-vectors of channels: 4 inputs loop "fmax v23.8h, v4.8h, v3.8h\n" "fmax v19.8h, v28.8h, v22.8h\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" - "ldr q3, [x23, x9]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" + "ldr q3, [x20, x27]\n" "fmax v22.8h, v2.8h, v1.8h\n" - "ldr q2, [x24, x28]\n" + "ldr q2, [x21, x26]\n" "fmax v18.8h, v27.8h, v21.8h\n" - "ldr q1, [x23, x28]\n" + "ldr q1, [x20, x26]\n" "fmax v21.8h, v0.8h, v31.8h\n" - "ldr q0, [x24, x27]\n" + "ldr q0, [x21, x24]\n" "fmax v17.8h, v26.8h, v20.8h\n" - "ldr q31, [x23, x27]\n" + "ldr q31, [x20, x24]\n" "fmax v20.8h, v30.8h, v29.8h\n" - "ldr q30, [x24, x26]\n" + "ldr q30, [x21, x23]\n" "fmax v16.8h, v25.8h, v24.8h\n" - "ldr q29, [x23, x26]\n" + "ldr q29, [x20, x23]\n" "fmax v19.8h, v23.8h, v19.8h\n" "fmax v18.8h, v22.8h, v18.8h\n" - "ldp x22, x21, [x20, #0x10]\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" + "ldp x21, x20, [x22, #0x10]\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" "fmax v17.8h, v21.8h, v17.8h\n" "fmax v16.8h, v20.8h, v16.8h\n" - "ldr q27, [x22, x28]\n" - "ldr q21, [x21, x28]\n" + "ldr q27, [x21, x26]\n" + "ldr q21, [x20, x26]\n" "subs x25, x25, #0x1\n" "fmax v8.8h, v8.8h, v19.8h\n" - "ldr q26, [x22, x27]\n" - "ldr q20, [x21, x27]\n" + "ldr q26, [x21, x24]\n" + "ldr q20, [x20, x24]\n" "fmax v7.8h, v7.8h, v18.8h\n" "fmax v6.8h, v6.8h, v17.8h\n" - "ldr q25, [x22, x26]\n" - "ldr q24, [x21, x26]\n" + "ldr q25, [x21, x23]\n" + "ldr q24, [x20, x23]\n" "fmax v5.8h, v5.8h, v16.8h\n" - "add x20, x20, #0x20\n" + "add x22, x22, #0x20\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "fmax v23.8h, v4.8h, v3.8h\n" @@ -135,28 +135,28 @@ void a64_fp16_nhwc_max_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x24, [x20], #0x8\n" - "ldr q4, [x24, x9]\n" + "ldr x20, [x22], #0x8\n" + "ldr q16, [x20, x27]\n" "subs x21, x21, #0x1\n" - "fmax v8.8h, v8.8h, v4.8h\n" - "ldr q2, [x24, x28]\n" - "ldr q0, [x24, x27]\n" - "fmax v7.8h, v7.8h, v2.8h\n" - "fmax v6.8h, v6.8h, v0.8h\n" - "ldr q30, [x24, x26]\n" - "fmax v5.8h, v5.8h, v30.8h\n" + "fmax v8.8h, v8.8h, v16.8h\n" + "ldr q17, [x20, x26]\n" + "ldr q16, [x20, x24]\n" + "fmax v7.8h, v7.8h, v17.8h\n" + "fmax v6.8h, v6.8h, v16.8h\n" + "ldr q16, [x20, x23]\n" + "fmax v5.8h, v5.8h, v16.8h\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End "sub %x[n_channels], %x[n_channels], #0x20\n" "cmp %x[n_channels], #0x20\n" - "str q8, [%x[outptr], x9]\n" - "str q7, [%x[outptr], x28]\n" - "add x9, x9, #0x40\n" - "add x28, x28, #0x40\n" - "str q6, [%x[outptr], x27]\n" + "str q8, [%x[outptr], x27]\n" + "str q7, [%x[outptr], x26]\n" "add x27, x27, #0x40\n" - "str q5, [%x[outptr], x26]\n" "add x26, x26, #0x40\n" + "str q6, [%x[outptr], x24]\n" + "add x24, x24, #0x40\n" + "str q5, [%x[outptr], x23]\n" + "add x23, x23, #0x40\n" "bge 1b\n" "cbz %x[n_channels], 31f\n" "7:" // Single vector of channels @@ -166,146 +166,146 @@ void a64_fp16_nhwc_max_generic_depthfirst_impl( "mov w20, #0xfc00\n" "lsr x25, %x[n_valid_cells], #0x2\n" "dup v8.8h, w20\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" "subs x25, x25, #0x1\n" - "ldr q3, [x23, x9]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" + "ldr q3, [x20, x27]\n" + "ldp x21, x20, [x22, #0x10]\n" + "add x22, x22, #0x20\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 4 inputs loop - "fmax v23.8h, v4.8h, v3.8h\n" - "fmax v19.8h, v28.8h, v22.8h\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" - "ldr q3, [x23, x9]\n" - "fmax v19.8h, v23.8h, v19.8h\n" - "ldp x22, x21, [x20, #0x10]\n" + "fmax v17.8h, v4.8h, v3.8h\n" + "fmax v16.8h, v28.8h, v22.8h\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" + "ldr q3, [x20, x27]\n" + "fmax v16.8h, v17.8h, v16.8h\n" + "ldp x21, x20, [x22, #0x10]\n" "subs x25, x25, #0x1\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" - "fmax v8.8h, v8.8h, v19.8h\n" - "add x20, x20, #0x20\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" + "fmax v8.8h, v8.8h, v16.8h\n" + "add x22, x22, #0x20\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail - "fmax v23.8h, v4.8h, v3.8h\n" - "fmax v19.8h, v28.8h, v22.8h\n" - "fmax v19.8h, v23.8h, v19.8h\n" - "fmax v8.8h, v8.8h, v19.8h\n" + "fmax v17.8h, v4.8h, v3.8h\n" + "fmax v16.8h, v28.8h, v22.8h\n" + "fmax v16.8h, v17.8h, v16.8h\n" + "fmax v8.8h, v8.8h, v16.8h\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x24, [x20], #0x8\n" - "ldr q4, [x24, x9]\n" + "ldr x20, [x22], #0x8\n" + "ldr q16, [x20, x27]\n" "subs x21, x21, #0x1\n" - "fmax v8.8h, v8.8h, v4.8h\n" + "fmax v8.8h, v8.8h, v16.8h\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "sub %x[n_channels], %x[n_channels], #0x8\n" "cmp %x[n_channels], #0x8\n" - "str q8, [%x[outptr], x9]\n" - "add x9, x9, #0x10\n" + "str q8, [%x[outptr], x27]\n" + "add x27, x27, #0x10\n" "bge 8b\n" "cbz %x[n_channels], 31f\n" "14:" // Oddments "mov w20, #0xfc00\n" "lsr x25, %x[n_valid_cells], #0x2\n" "dup v8.8h, w20\n" - "add %x[outptr], %x[outptr], x9\n" - "mov x20, %x[inptrs]\n" + "add %x[outptr], %x[outptr], x27\n" + "mov x24, %x[inptrs]\n" "cbz x25, 20f\n" "15:" // Oddments: 4 inputs loop - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "add x24, x24, x9\n" - "add x23, x23, x9\n" - "add x22, x22, x9\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" + "add x23, x23, x27\n" + "add x22, x22, x27\n" + "add x21, x21, x27\n" "movi v4.16b, #0x0\n" "movi v3.16b, #0x0\n" - "add x21, x21, x9\n" + "add x20, x20, x27\n" "movi v28.16b, #0x0\n" "movi v22.16b, #0x0\n" "tbz %x[n_channels], #2, 17f\n" - "ldr d4, [x24], #0x8\n" - "ldr d3, [x23], #0x8\n" - "ldr d28, [x22], #0x8\n" - "ldr d22, [x21], #0x8\n" + "ldr d4, [x23], #0x8\n" + "ldr d3, [x22], #0x8\n" + "ldr d28, [x21], #0x8\n" + "ldr d22, [x20], #0x8\n" "tbz %x[n_channels], #1, 16f\n" - "ld1 { v4.s }[2], [x24], #0x4\n" - "ld1 { v3.s }[2], [x23], #0x4\n" - "ld1 { v28.s }[2], [x22], #0x4\n" - "ld1 { v22.s }[2], [x21], #0x4\n" + "ld1 { v4.s }[2], [x23], #0x4\n" + "ld1 { v3.s }[2], [x22], #0x4\n" + "ld1 { v28.s }[2], [x21], #0x4\n" + "ld1 { v22.s }[2], [x20], #0x4\n" "tbz %x[n_channels], #0, 19f\n" - "ld1 { v4.h }[6], [x24], #0x2\n" - "ld1 { v3.h }[6], [x23], #0x2\n" - "ld1 { v28.h }[6], [x22], #0x2\n" - "ld1 { v22.h }[6], [x21], #0x2\n" + "ld1 { v4.h }[6], [x23], #0x2\n" + "ld1 { v3.h }[6], [x22], #0x2\n" + "ld1 { v28.h }[6], [x21], #0x2\n" + "ld1 { v22.h }[6], [x20], #0x2\n" "b 19f\n" "16:" // Oddments: 4 inputs loop: Load: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 19f\n" - "ld1 { v4.h }[4], [x24], #0x2\n" - "ld1 { v3.h }[4], [x23], #0x2\n" - "ld1 { v28.h }[4], [x22], #0x2\n" - "ld1 { v22.h }[4], [x21], #0x2\n" + "ld1 { v4.h }[4], [x23], #0x2\n" + "ld1 { v3.h }[4], [x22], #0x2\n" + "ld1 { v28.h }[4], [x21], #0x2\n" + "ld1 { v22.h }[4], [x20], #0x2\n" "b 19f\n" "17:" // Oddments: 4 inputs loop: Load: Bit 2: Unset "tbz %x[n_channels], #1, 18f\n" - "ldr s4, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr s28, [x22], #0x4\n" - "ldr s22, [x21], #0x4\n" + "ldr s4, [x23], #0x4\n" + "ldr s3, [x22], #0x4\n" + "ldr s28, [x21], #0x4\n" + "ldr s22, [x20], #0x4\n" "tbz %x[n_channels], #0, 19f\n" - "ld1 { v4.h }[2], [x24], #0x2\n" - "ld1 { v3.h }[2], [x23], #0x2\n" - "ld1 { v28.h }[2], [x22], #0x2\n" - "ld1 { v22.h }[2], [x21], #0x2\n" + "ld1 { v4.h }[2], [x23], #0x2\n" + "ld1 { v3.h }[2], [x22], #0x2\n" + "ld1 { v28.h }[2], [x21], #0x2\n" + "ld1 { v22.h }[2], [x20], #0x2\n" "b 19f\n" "18:" // Oddments: 4 inputs loop: Load: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 19f\n" - "ldr h4, [x24], #0x2\n" - "ldr h3, [x23], #0x2\n" - "ldr h28, [x22], #0x2\n" - "ldr h22, [x21], #0x2\n" + "ldr h4, [x23], #0x2\n" + "ldr h3, [x22], #0x2\n" + "ldr h28, [x21], #0x2\n" + "ldr h22, [x20], #0x2\n" "19:" // Oddments: 4 inputs loop: Load: Bit 2: End - "fmax v23.8h, v4.8h, v3.8h\n" - "fmax v19.8h, v28.8h, v22.8h\n" + "fmax v17.8h, v4.8h, v3.8h\n" + "fmax v16.8h, v28.8h, v22.8h\n" "subs x25, x25, #0x1\n" - "fmax v19.8h, v23.8h, v19.8h\n" - "fmax v8.8h, v8.8h, v19.8h\n" + "fmax v16.8h, v17.8h, v16.8h\n" + "fmax v8.8h, v8.8h, v16.8h\n" "bgt 15b\n" "20:" // Oddments: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 26f\n" "21:" // Oddments: Single input loop - "ldr x24, [x20], #0x8\n" - "add x24, x24, x9\n" + "ldr x23, [x24], #0x8\n" + "add x23, x23, x27\n" "movi v4.16b, #0x0\n" "tbz %x[n_channels], #2, 23f\n" - "ldr d4, [x24], #0x8\n" + "ldr d4, [x23], #0x8\n" "tbz %x[n_channels], #1, 22f\n" - "ld1 { v4.s }[2], [x24], #0x4\n" + "ld1 { v4.s }[2], [x23], #0x4\n" "tbz %x[n_channels], #0, 25f\n" - "ld1 { v4.h }[6], [x24], #0x2\n" + "ld1 { v4.h }[6], [x23], #0x2\n" "b 25f\n" "22:" // Oddments: Single input loop: Load: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 25f\n" - "ld1 { v4.h }[4], [x24], #0x2\n" + "ld1 { v4.h }[4], [x23], #0x2\n" "b 25f\n" "23:" // Oddments: Single input loop: Load: Bit 2: Unset "tbz %x[n_channels], #1, 24f\n" - "ldr s4, [x24], #0x4\n" + "ldr s4, [x23], #0x4\n" "tbz %x[n_channels], #0, 25f\n" - "ld1 { v4.h }[2], [x24], #0x2\n" + "ld1 { v4.h }[2], [x23], #0x2\n" "b 25f\n" "24:" // Oddments: Single input loop: Load: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 25f\n" - "ldr h4, [x24], #0x2\n" + "ldr h4, [x23], #0x2\n" "25:" // Oddments: Single input loop: Load: Bit 2: End "subs x21, x21, #0x1\n" "fmax v8.8h, v8.8h, v4.8h\n" @@ -335,7 +335,7 @@ void a64_fp16_nhwc_max_generic_depthfirst_impl( "31:" // End : [n_channels] "+&r" (n_channels), [outptr] "+&r" (outptr) : [inptrs] "r" (inptrs), [n_valid_cells] "r" (n_valid_cells) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27" ); } diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst.hpp index 813e685606efede90b626013eb5419612a21b976..7add5feb1d17f7250400cfb8d46bc8761d570602 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,8 @@ #pragma once +#if defined(__aarch64__) + namespace arm_conv { namespace pooling { @@ -45,3 +47,5 @@ struct a64_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst : public DepthfirstStrategy } // namespace pooling } // namespace arm_conv + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp index 9db65d62b0ba8a5d35cd0ac4cb060a0564755fa1..cf0047638eeeae27472b341e7531b511de39a912 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp @@ -22,12 +22,12 @@ * SOFTWARE. */ -#if defined(__aarch64__) - #include #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace pooling { @@ -196,38 +196,38 @@ void a64_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst_impl( "add x5, x5, #0x10\n" "cbz x3, 4f\n" "3:" // Oddments - "ldr s6, [x11, x4]\n" - "ldr s5, [x10, x4]\n" - "fadd v17.4s, v6.4s, v5.4s\n" + "ldr s17, [x11, x4]\n" + "ldr s16, [x10, x4]\n" + "fadd v18.4s, v17.4s, v16.4s\n" "subs x3, x3, #0x1\n" - "ldr s4, [x27, x4]\n" - "ldr s3, [x26, x4]\n" - "fadd v16.4s, v4.4s, v3.4s\n" - "fadd v19.4s, v17.4s, v16.4s\n" - "ldr s2, [x15, x4]\n" - "ldr s1, [x14, x4]\n" - "fadd v18.4s, v2.4s, v1.4s\n" - "fadd v21.4s, v18.4s, v19.4s\n" - "ldr s0, [x12, x4]\n" - "ldr s31, [x28, x4]\n" - "fadd v17.4s, v0.4s, v31.4s\n" - "ldr s30, [x9, x4]\n" - "ldr s29, [x25, x4]\n" - "fadd v22.4s, v30.4s, v29.4s\n" - "ldr s28, [x23, x4]\n" - "ldr s27, [x22, x4]\n" - "fadd v16.4s, v28.4s, v27.4s\n" - "fadd v20.4s, v16.4s, v19.4s\n" - "ldr s26, [x16, x4]\n" - "ldr s25, [x13, x4]\n" - "fadd v19.4s, v26.4s, v17.4s\n" - "fadd v18.4s, v25.4s, v22.4s\n" - "ldr s24, [x24, x4]\n" - "ldr s23, [x21, x4]\n" - "fadd v17.4s, v24.4s, v17.4s\n" - "fadd v16.4s, v23.4s, v22.4s\n" - "fadd v19.4s, v21.4s, v19.4s\n" - "fadd v18.4s, v21.4s, v18.4s\n" + "ldr s17, [x27, x4]\n" + "ldr s16, [x26, x4]\n" + "fadd v16.4s, v17.4s, v16.4s\n" + "fadd v18.4s, v18.4s, v16.4s\n" + "ldr s17, [x15, x4]\n" + "ldr s16, [x14, x4]\n" + "fadd v16.4s, v17.4s, v16.4s\n" + "fadd v23.4s, v16.4s, v18.4s\n" + "ldr s17, [x12, x4]\n" + "ldr s16, [x28, x4]\n" + "fadd v22.4s, v17.4s, v16.4s\n" + "ldr s17, [x9, x4]\n" + "ldr s16, [x25, x4]\n" + "fadd v21.4s, v17.4s, v16.4s\n" + "ldr s17, [x23, x4]\n" + "ldr s16, [x22, x4]\n" + "fadd v16.4s, v17.4s, v16.4s\n" + "fadd v20.4s, v16.4s, v18.4s\n" + "ldr s17, [x16, x4]\n" + "ldr s16, [x13, x4]\n" + "fadd v19.4s, v17.4s, v22.4s\n" + "fadd v18.4s, v16.4s, v21.4s\n" + "ldr s17, [x24, x4]\n" + "ldr s16, [x21, x4]\n" + "fadd v17.4s, v17.4s, v22.4s\n" + "fadd v16.4s, v16.4s, v21.4s\n" + "fadd v19.4s, v23.4s, v19.4s\n" + "fadd v18.4s, v23.4s, v18.4s\n" "add x4, x4, #0x4\n" "fadd v17.4s, v17.4s, v20.4s\n" "fadd v16.4s, v16.4s, v20.4s\n" @@ -250,4 +250,5 @@ void a64_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst_impl( } // namespace pooling } // namespace arm_conv + #endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_avg_generic_depthfirst/generic.cpp index 3f90610591142cf3b70404431cda268030c44d3d..d236f07b1c2112ed32cb93a03a9b8124adb8ec3e 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_avg_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_avg_generic_depthfirst/generic.cpp @@ -22,11 +22,11 @@ * SOFTWARE. */ -#if defined(__aarch64__) - #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace pooling { @@ -44,77 +44,77 @@ void a64_fp32_nhwc_avg_generic_depthfirst_impl( __asm__ __volatile__( "ld1r { v9.4s }, [%x[rescale_ptr]]\n" "cmp %x[n_channels], #0x10\n" - "mov x9, #0x0\n" - "mov x28, #0x10\n" // cntb _, ALL, #1 - "mov x27, #0x20\n" // cntb _, ALL, #2 - "mov x26, #0x30\n" // cntb _, ALL, #3 + "mov x27, #0x0\n" + "mov x26, #0x10\n" // cntb _, ALL, #1 + "mov x24, #0x20\n" // cntb _, ALL, #2 + "mov x23, #0x30\n" // cntb _, ALL, #3 "blt 7f\n" "1:" // 4-vectors of channels "lsr x25, %x[n_valid_cells], #0x2\n" "movi v8.16b, #0x0\n" "movi v7.16b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "movi v6.16b, #0x0\n" "movi v5.16b, #0x0\n" "cbz x25, 4f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" "subs x25, x25, #0x1\n" - "ldr q3, [x23, x9]\n" - "ldr q2, [x24, x28]\n" - "ldr q1, [x23, x28]\n" - "ldr q0, [x24, x27]\n" - "ldr q31, [x23, x27]\n" - "ldr q30, [x24, x26]\n" - "ldr q29, [x23, x26]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" - "ldr q27, [x22, x28]\n" - "ldr q21, [x21, x28]\n" - "ldr q26, [x22, x27]\n" - "ldr q20, [x21, x27]\n" - "ldr q25, [x22, x26]\n" - "ldr q24, [x21, x26]\n" + "ldr q3, [x20, x27]\n" + "ldr q2, [x21, x26]\n" + "ldr q1, [x20, x26]\n" + "ldr q0, [x21, x24]\n" + "ldr q31, [x20, x24]\n" + "ldr q30, [x21, x23]\n" + "ldr q29, [x20, x23]\n" + "ldp x21, x20, [x22, #0x10]\n" + "add x22, x22, #0x20\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" + "ldr q27, [x21, x26]\n" + "ldr q21, [x20, x26]\n" + "ldr q26, [x21, x24]\n" + "ldr q20, [x20, x24]\n" + "ldr q25, [x21, x23]\n" + "ldr q24, [x20, x23]\n" "beq 3f\n" "2:" // 4-vectors of channels: 4 inputs loop "fadd v23.4s, v4.4s, v3.4s\n" "fadd v19.4s, v28.4s, v22.4s\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" - "ldr q3, [x23, x9]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" + "ldr q3, [x20, x27]\n" "fadd v22.4s, v2.4s, v1.4s\n" - "ldr q2, [x24, x28]\n" + "ldr q2, [x21, x26]\n" "fadd v18.4s, v27.4s, v21.4s\n" - "ldr q1, [x23, x28]\n" + "ldr q1, [x20, x26]\n" "fadd v21.4s, v0.4s, v31.4s\n" - "ldr q0, [x24, x27]\n" + "ldr q0, [x21, x24]\n" "fadd v17.4s, v26.4s, v20.4s\n" - "ldr q31, [x23, x27]\n" + "ldr q31, [x20, x24]\n" "fadd v20.4s, v30.4s, v29.4s\n" - "ldr q30, [x24, x26]\n" + "ldr q30, [x21, x23]\n" "fadd v16.4s, v25.4s, v24.4s\n" - "ldr q29, [x23, x26]\n" + "ldr q29, [x20, x23]\n" "fadd v19.4s, v23.4s, v19.4s\n" "fadd v18.4s, v22.4s, v18.4s\n" - "ldp x22, x21, [x20, #0x10]\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" + "ldp x21, x20, [x22, #0x10]\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" "fadd v17.4s, v21.4s, v17.4s\n" "fadd v16.4s, v20.4s, v16.4s\n" - "ldr q27, [x22, x28]\n" - "ldr q21, [x21, x28]\n" + "ldr q27, [x21, x26]\n" + "ldr q21, [x20, x26]\n" "subs x25, x25, #0x1\n" "fadd v8.4s, v8.4s, v19.4s\n" - "ldr q26, [x22, x27]\n" - "ldr q20, [x21, x27]\n" + "ldr q26, [x21, x24]\n" + "ldr q20, [x20, x24]\n" "fadd v7.4s, v7.4s, v18.4s\n" "fadd v6.4s, v6.4s, v17.4s\n" - "ldr q25, [x22, x26]\n" - "ldr q24, [x21, x26]\n" + "ldr q25, [x21, x23]\n" + "ldr q24, [x20, x23]\n" "fadd v5.4s, v5.4s, v16.4s\n" - "add x20, x20, #0x20\n" + "add x22, x22, #0x20\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "fadd v23.4s, v4.4s, v3.4s\n" @@ -137,16 +137,16 @@ void a64_fp32_nhwc_avg_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x24, [x20], #0x8\n" - "ldr q4, [x24, x9]\n" + "ldr x20, [x22], #0x8\n" + "ldr q16, [x20, x27]\n" "subs x21, x21, #0x1\n" - "fadd v8.4s, v8.4s, v4.4s\n" - "ldr q2, [x24, x28]\n" - "ldr q0, [x24, x27]\n" - "fadd v7.4s, v7.4s, v2.4s\n" - "fadd v6.4s, v6.4s, v0.4s\n" - "ldr q30, [x24, x26]\n" - "fadd v5.4s, v5.4s, v30.4s\n" + "fadd v8.4s, v8.4s, v16.4s\n" + "ldr q17, [x20, x26]\n" + "ldr q16, [x20, x24]\n" + "fadd v7.4s, v7.4s, v17.4s\n" + "fadd v6.4s, v6.4s, v16.4s\n" + "ldr q16, [x20, x23]\n" + "fadd v5.4s, v5.4s, v16.4s\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End "sub %x[n_channels], %x[n_channels], #0x10\n" @@ -155,14 +155,14 @@ void a64_fp32_nhwc_avg_generic_depthfirst_impl( "fmul v7.4s, v7.4s, v9.4s\n" "fmul v6.4s, v6.4s, v9.4s\n" "fmul v5.4s, v5.4s, v9.4s\n" - "str q8, [%x[outptr], x9]\n" - "add x9, x9, #0x40\n" - "str q7, [%x[outptr], x28]\n" - "add x28, x28, #0x40\n" - "str q6, [%x[outptr], x27]\n" + "str q8, [%x[outptr], x27]\n" "add x27, x27, #0x40\n" - "str q5, [%x[outptr], x26]\n" + "str q7, [%x[outptr], x26]\n" "add x26, x26, #0x40\n" + "str q6, [%x[outptr], x24]\n" + "add x24, x24, #0x40\n" + "str q5, [%x[outptr], x23]\n" + "add x23, x23, #0x40\n" "bge 1b\n" "cbz %x[n_channels], 25f\n" "7:" // Single vector of channels @@ -171,110 +171,110 @@ void a64_fp32_nhwc_avg_generic_depthfirst_impl( "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" "movi v8.16b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" "subs x25, x25, #0x1\n" - "ldr q3, [x23, x9]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" + "ldr q3, [x20, x27]\n" + "ldp x21, x20, [x22, #0x10]\n" + "add x22, x22, #0x20\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 4 inputs loop - "fadd v23.4s, v4.4s, v3.4s\n" - "fadd v19.4s, v28.4s, v22.4s\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" - "ldr q3, [x23, x9]\n" - "fadd v19.4s, v23.4s, v19.4s\n" - "ldp x22, x21, [x20, #0x10]\n" + "fadd v17.4s, v4.4s, v3.4s\n" + "fadd v16.4s, v28.4s, v22.4s\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" + "ldr q3, [x20, x27]\n" + "fadd v16.4s, v17.4s, v16.4s\n" + "ldp x21, x20, [x22, #0x10]\n" "subs x25, x25, #0x1\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" - "fadd v8.4s, v8.4s, v19.4s\n" - "add x20, x20, #0x20\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" + "fadd v8.4s, v8.4s, v16.4s\n" + "add x22, x22, #0x20\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail - "fadd v23.4s, v4.4s, v3.4s\n" - "fadd v19.4s, v28.4s, v22.4s\n" - "fadd v19.4s, v23.4s, v19.4s\n" - "fadd v8.4s, v8.4s, v19.4s\n" + "fadd v17.4s, v4.4s, v3.4s\n" + "fadd v16.4s, v28.4s, v22.4s\n" + "fadd v16.4s, v17.4s, v16.4s\n" + "fadd v8.4s, v8.4s, v16.4s\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x24, [x20], #0x8\n" - "ldr q4, [x24, x9]\n" + "ldr x20, [x22], #0x8\n" + "ldr q16, [x20, x27]\n" "subs x21, x21, #0x1\n" - "fadd v8.4s, v8.4s, v4.4s\n" + "fadd v8.4s, v8.4s, v16.4s\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "sub %x[n_channels], %x[n_channels], #0x4\n" "cmp %x[n_channels], #0x4\n" "fmul v8.4s, v8.4s, v9.4s\n" - "str q8, [%x[outptr], x9]\n" - "add x9, x9, #0x10\n" + "str q8, [%x[outptr], x27]\n" + "add x27, x27, #0x10\n" "bge 8b\n" "cbz %x[n_channels], 25f\n" "14:" // Oddments "lsr x25, %x[n_valid_cells], #0x2\n" - "add %x[outptr], %x[outptr], x9\n" + "add %x[outptr], %x[outptr], x27\n" "movi v8.16b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "cbz x25, 18f\n" "15:" // Oddments: 4 inputs loop - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "add x24, x24, x9\n" - "add x23, x23, x9\n" - "add x22, x22, x9\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" + "add x23, x23, x27\n" + "add x22, x22, x27\n" + "add x21, x21, x27\n" "movi v4.16b, #0x0\n" "movi v3.16b, #0x0\n" - "add x21, x21, x9\n" + "add x20, x20, x27\n" "movi v28.16b, #0x0\n" "movi v22.16b, #0x0\n" "tbz %x[n_channels], #1, 16f\n" - "ldr d4, [x24], #0x8\n" - "ldr d3, [x23], #0x8\n" - "ldr d28, [x22], #0x8\n" - "ldr d22, [x21], #0x8\n" + "ldr d4, [x23], #0x8\n" + "ldr d3, [x22], #0x8\n" + "ldr d28, [x21], #0x8\n" + "ldr d22, [x20], #0x8\n" "tbz %x[n_channels], #0, 17f\n" - "ld1 { v4.s }[2], [x24], #0x4\n" - "ld1 { v3.s }[2], [x23], #0x4\n" - "ld1 { v28.s }[2], [x22], #0x4\n" - "ld1 { v22.s }[2], [x21], #0x4\n" + "ld1 { v4.s }[2], [x23], #0x4\n" + "ld1 { v3.s }[2], [x22], #0x4\n" + "ld1 { v28.s }[2], [x21], #0x4\n" + "ld1 { v22.s }[2], [x20], #0x4\n" "b 17f\n" "16:" // Oddments: 4 inputs loop: Load: Bit 1: Unset "tbz %x[n_channels], #0, 17f\n" - "ldr s4, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr s28, [x22], #0x4\n" - "ldr s22, [x21], #0x4\n" + "ldr s4, [x23], #0x4\n" + "ldr s3, [x22], #0x4\n" + "ldr s28, [x21], #0x4\n" + "ldr s22, [x20], #0x4\n" "17:" // Oddments: 4 inputs loop: Load: Bit 1: End - "fadd v23.4s, v4.4s, v3.4s\n" - "fadd v19.4s, v28.4s, v22.4s\n" + "fadd v17.4s, v4.4s, v3.4s\n" + "fadd v16.4s, v28.4s, v22.4s\n" "subs x25, x25, #0x1\n" - "fadd v19.4s, v23.4s, v19.4s\n" - "fadd v8.4s, v8.4s, v19.4s\n" + "fadd v16.4s, v17.4s, v16.4s\n" + "fadd v8.4s, v8.4s, v16.4s\n" "bgt 15b\n" "18:" // Oddments: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 22f\n" "19:" // Oddments: Single input loop - "ldr x24, [x20], #0x8\n" - "add x24, x24, x9\n" + "ldr x23, [x24], #0x8\n" + "add x23, x23, x27\n" "movi v4.16b, #0x0\n" "tbz %x[n_channels], #1, 20f\n" - "ldr d4, [x24], #0x8\n" + "ldr d4, [x23], #0x8\n" "tbz %x[n_channels], #0, 21f\n" - "ld1 { v4.s }[2], [x24], #0x4\n" + "ld1 { v4.s }[2], [x23], #0x4\n" "b 21f\n" "20:" // Oddments: Single input loop: Load: Bit 1: Unset "tbz %x[n_channels], #0, 21f\n" - "ldr s4, [x24], #0x4\n" + "ldr s4, [x23], #0x4\n" "21:" // Oddments: Single input loop: Load: Bit 1: End "subs x21, x21, #0x1\n" "fadd v8.4s, v8.4s, v4.4s\n" @@ -293,10 +293,11 @@ void a64_fp32_nhwc_avg_generic_depthfirst_impl( "25:" // End : [n_channels] "+&r" (n_channels), [outptr] "+&r" (outptr) : [inptrs] "r" (inptrs), [n_valid_cells] "r" (n_valid_cells), [rescale_ptr] "r" (&rescale_value) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27" ); } } // namespace pooling } // namespace arm_conv + #endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_max_2x2_s1_output2x2_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_max_2x2_s1_output2x2_depthfirst.hpp index 4bf5770857015909ee593082e8afb6dfdfeac34b..2f72b59d709c89ca641d9f3636bd160a24db57ce 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_max_2x2_s1_output2x2_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_max_2x2_s1_output2x2_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,8 @@ #pragma once +#if defined(__aarch64__) + namespace arm_conv { namespace pooling { @@ -45,3 +47,5 @@ struct a64_fp32_nhwc_max_2x2_s1_output2x2_depthfirst : public DepthfirstStrategy } // namespace pooling } // namespace arm_conv + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp index 2e7fb3c5b1cefe68fcecc98306950cbc3598d6c2..f4202de1ed147baaa25b809480584e61136c114f 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp @@ -22,11 +22,12 @@ * SOFTWARE. */ -#if defined(__aarch64__) #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace pooling { @@ -111,7 +112,7 @@ void a64_fp32_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "fmax v18.4s, v18.4s, v21.4s\n" "fmax v17.4s, v17.4s, v20.4s\n" "add x15, x15, #0x10\n" - "fmax v16.4s, v16.4s, v20.4s\n" + "fmax v16.4s, v20.4s, v16.4s\n" "str q19, [x14, x12]\n" "str q18, [x13, x12]\n" "str q17, [x11, x12]\n" @@ -121,43 +122,43 @@ void a64_fp32_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "2:" // Vector: Tail "fmax v21.4s, v30.4s, v29.4s\n" "fmax v20.4s, v29.4s, v28.4s\n" - "fmax v19.4s, v27.4s, v26.4s\n" + "fmax v16.4s, v27.4s, v26.4s\n" "fmax v18.4s, v25.4s, v24.4s\n" "fmax v17.4s, v27.4s, v23.4s\n" - "fmax v16.4s, v24.4s, v22.4s\n" - "fmax v19.4s, v21.4s, v19.4s\n" + "fmax v19.4s, v24.4s, v22.4s\n" + "fmax v16.4s, v21.4s, v16.4s\n" "fmax v18.4s, v18.4s, v21.4s\n" - "str q19, [x14, x12]\n" + "str q16, [x14, x12]\n" "fmax v17.4s, v17.4s, v20.4s\n" - "fmax v16.4s, v16.4s, v20.4s\n" + "fmax v16.4s, v20.4s, v19.4s\n" "str q18, [x13, x12]\n" "str q17, [x11, x12]\n" "str q16, [x10, x12]\n" "add x12, x12, #0x10\n" "cbz x16, 4f\n" "3:" // Oddments - "ldr s30, [x28, x15]\n" - "ldr s29, [x25, x15]\n" - "fmax v21.4s, v30.4s, v29.4s\n" + "ldr s16, [x28, x15]\n" + "ldr s17, [x25, x15]\n" + "fmax v23.4s, v16.4s, v17.4s\n" "subs x16, x16, #0x1\n" - "ldr s28, [x22, x15]\n" - "ldr s27, [x26, x15]\n" - "fmax v20.4s, v29.4s, v28.4s\n" - "ldr s26, [x9, x15]\n" - "ldr s25, [x27, x15]\n" - "fmax v19.4s, v27.4s, v26.4s\n" - "fmax v19.4s, v21.4s, v19.4s\n" - "ldr s24, [x24, x15]\n" - "ldr s23, [x23, x15]\n" - "fmax v18.4s, v25.4s, v24.4s\n" - "fmax v17.4s, v27.4s, v23.4s\n" - "ldr s22, [x21, x15]\n" - "fmax v16.4s, v24.4s, v22.4s\n" + "ldr s16, [x22, x15]\n" + "ldr s22, [x26, x15]\n" + "fmax v21.4s, v17.4s, v16.4s\n" + "ldr s16, [x9, x15]\n" + "ldr s17, [x27, x15]\n" + "fmax v16.4s, v22.4s, v16.4s\n" + "fmax v20.4s, v23.4s, v16.4s\n" + "ldr s19, [x24, x15]\n" + "ldr s16, [x23, x15]\n" + "fmax v18.4s, v17.4s, v19.4s\n" + "fmax v17.4s, v22.4s, v16.4s\n" + "ldr s16, [x21, x15]\n" + "fmax v16.4s, v19.4s, v16.4s\n" "add x15, x15, #0x4\n" - "fmax v18.4s, v18.4s, v21.4s\n" - "fmax v17.4s, v17.4s, v20.4s\n" - "fmax v16.4s, v16.4s, v20.4s\n" - "str s19, [x14, x12]\n" + "fmax v18.4s, v18.4s, v23.4s\n" + "fmax v17.4s, v17.4s, v21.4s\n" + "fmax v16.4s, v21.4s, v16.4s\n" + "str s20, [x14, x12]\n" "str s18, [x13, x12]\n" "str s17, [x11, x12]\n" "str s16, [x10, x12]\n" @@ -172,4 +173,5 @@ void a64_fp32_nhwc_max_2x2_s1_output2x2_depthfirst_impl( } // namespace pooling } // namespace arm_conv + #endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_max_generic_depthfirst/generic.cpp index 4f1af09e085a80aaebe008b4d98026e9fc1e365f..f4706635dccd92bcde62ff3047ebcc2f62c9cc33 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_max_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_max_generic_depthfirst/generic.cpp @@ -22,11 +22,11 @@ * SOFTWARE. */ -#if defined(__aarch64__) - #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace pooling { @@ -41,10 +41,10 @@ void a64_fp32_nhwc_max_generic_depthfirst_impl( { __asm__ __volatile__( "cmp %x[n_channels], #0x10\n" - "mov x9, #0x0\n" - "mov x28, #0x10\n" // cntb _, ALL, #1 - "mov x27, #0x20\n" // cntb _, ALL, #2 - "mov x26, #0x30\n" // cntb _, ALL, #3 + "mov x27, #0x0\n" + "mov x26, #0x10\n" // cntb _, ALL, #1 + "mov x24, #0x20\n" // cntb _, ALL, #2 + "mov x23, #0x30\n" // cntb _, ALL, #3 "blt 7f\n" "1:" // 4-vectors of channels "mov w20, #0xff800000\n" @@ -53,66 +53,66 @@ void a64_fp32_nhwc_max_generic_depthfirst_impl( "dup v7.4s, w20\n" "dup v6.4s, w20\n" "dup v5.4s, w20\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "cbz x25, 4f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" "subs x25, x25, #0x1\n" - "ldr q3, [x23, x9]\n" - "ldr q2, [x24, x28]\n" - "ldr q1, [x23, x28]\n" - "ldr q0, [x24, x27]\n" - "ldr q31, [x23, x27]\n" - "ldr q30, [x24, x26]\n" - "ldr q29, [x23, x26]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" - "ldr q27, [x22, x28]\n" - "ldr q21, [x21, x28]\n" - "ldr q26, [x22, x27]\n" - "ldr q20, [x21, x27]\n" - "ldr q25, [x22, x26]\n" - "ldr q24, [x21, x26]\n" + "ldr q3, [x20, x27]\n" + "ldr q2, [x21, x26]\n" + "ldr q1, [x20, x26]\n" + "ldr q0, [x21, x24]\n" + "ldr q31, [x20, x24]\n" + "ldr q30, [x21, x23]\n" + "ldr q29, [x20, x23]\n" + "ldp x21, x20, [x22, #0x10]\n" + "add x22, x22, #0x20\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" + "ldr q27, [x21, x26]\n" + "ldr q21, [x20, x26]\n" + "ldr q26, [x21, x24]\n" + "ldr q20, [x20, x24]\n" + "ldr q25, [x21, x23]\n" + "ldr q24, [x20, x23]\n" "beq 3f\n" "2:" // 4-vectors of channels: 4 inputs loop "fmax v23.4s, v4.4s, v3.4s\n" "fmax v19.4s, v28.4s, v22.4s\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" - "ldr q3, [x23, x9]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" + "ldr q3, [x20, x27]\n" "fmax v22.4s, v2.4s, v1.4s\n" - "ldr q2, [x24, x28]\n" + "ldr q2, [x21, x26]\n" "fmax v18.4s, v27.4s, v21.4s\n" - "ldr q1, [x23, x28]\n" + "ldr q1, [x20, x26]\n" "fmax v21.4s, v0.4s, v31.4s\n" - "ldr q0, [x24, x27]\n" + "ldr q0, [x21, x24]\n" "fmax v17.4s, v26.4s, v20.4s\n" - "ldr q31, [x23, x27]\n" + "ldr q31, [x20, x24]\n" "fmax v20.4s, v30.4s, v29.4s\n" - "ldr q30, [x24, x26]\n" + "ldr q30, [x21, x23]\n" "fmax v16.4s, v25.4s, v24.4s\n" - "ldr q29, [x23, x26]\n" + "ldr q29, [x20, x23]\n" "fmax v19.4s, v23.4s, v19.4s\n" "fmax v18.4s, v22.4s, v18.4s\n" - "ldp x22, x21, [x20, #0x10]\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" + "ldp x21, x20, [x22, #0x10]\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" "fmax v17.4s, v21.4s, v17.4s\n" "fmax v16.4s, v20.4s, v16.4s\n" - "ldr q27, [x22, x28]\n" - "ldr q21, [x21, x28]\n" + "ldr q27, [x21, x26]\n" + "ldr q21, [x20, x26]\n" "subs x25, x25, #0x1\n" "fmax v8.4s, v8.4s, v19.4s\n" - "ldr q26, [x22, x27]\n" - "ldr q20, [x21, x27]\n" + "ldr q26, [x21, x24]\n" + "ldr q20, [x20, x24]\n" "fmax v7.4s, v7.4s, v18.4s\n" "fmax v6.4s, v6.4s, v17.4s\n" - "ldr q25, [x22, x26]\n" - "ldr q24, [x21, x26]\n" + "ldr q25, [x21, x23]\n" + "ldr q24, [x20, x23]\n" "fmax v5.4s, v5.4s, v16.4s\n" - "add x20, x20, #0x20\n" + "add x22, x22, #0x20\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "fmax v23.4s, v4.4s, v3.4s\n" @@ -135,28 +135,28 @@ void a64_fp32_nhwc_max_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x24, [x20], #0x8\n" - "ldr q4, [x24, x9]\n" + "ldr x20, [x22], #0x8\n" + "ldr q16, [x20, x27]\n" "subs x21, x21, #0x1\n" - "fmax v8.4s, v8.4s, v4.4s\n" - "ldr q2, [x24, x28]\n" - "ldr q0, [x24, x27]\n" - "fmax v7.4s, v7.4s, v2.4s\n" - "fmax v6.4s, v6.4s, v0.4s\n" - "ldr q30, [x24, x26]\n" - "fmax v5.4s, v5.4s, v30.4s\n" + "fmax v8.4s, v8.4s, v16.4s\n" + "ldr q17, [x20, x26]\n" + "ldr q16, [x20, x24]\n" + "fmax v7.4s, v7.4s, v17.4s\n" + "fmax v6.4s, v6.4s, v16.4s\n" + "ldr q16, [x20, x23]\n" + "fmax v5.4s, v5.4s, v16.4s\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End "sub %x[n_channels], %x[n_channels], #0x10\n" "cmp %x[n_channels], #0x10\n" - "str q8, [%x[outptr], x9]\n" - "str q7, [%x[outptr], x28]\n" - "add x9, x9, #0x40\n" - "add x28, x28, #0x40\n" - "str q6, [%x[outptr], x27]\n" + "str q8, [%x[outptr], x27]\n" + "str q7, [%x[outptr], x26]\n" "add x27, x27, #0x40\n" - "str q5, [%x[outptr], x26]\n" "add x26, x26, #0x40\n" + "str q6, [%x[outptr], x24]\n" + "add x24, x24, #0x40\n" + "str q5, [%x[outptr], x23]\n" + "add x23, x23, #0x40\n" "bge 1b\n" "cbz %x[n_channels], 25f\n" "7:" // Single vector of channels @@ -166,110 +166,110 @@ void a64_fp32_nhwc_max_generic_depthfirst_impl( "mov w20, #0xff800000\n" "lsr x25, %x[n_valid_cells], #0x2\n" "dup v8.4s, w20\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" "subs x25, x25, #0x1\n" - "ldr q3, [x23, x9]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" + "ldr q3, [x20, x27]\n" + "ldp x21, x20, [x22, #0x10]\n" + "add x22, x22, #0x20\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 4 inputs loop - "fmax v23.4s, v4.4s, v3.4s\n" - "fmax v19.4s, v28.4s, v22.4s\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" - "ldr q3, [x23, x9]\n" - "fmax v19.4s, v23.4s, v19.4s\n" - "ldp x22, x21, [x20, #0x10]\n" + "fmax v17.4s, v4.4s, v3.4s\n" + "fmax v16.4s, v28.4s, v22.4s\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" + "ldr q3, [x20, x27]\n" + "fmax v16.4s, v17.4s, v16.4s\n" + "ldp x21, x20, [x22, #0x10]\n" "subs x25, x25, #0x1\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" - "fmax v8.4s, v8.4s, v19.4s\n" - "add x20, x20, #0x20\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" + "fmax v8.4s, v8.4s, v16.4s\n" + "add x22, x22, #0x20\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail - "fmax v23.4s, v4.4s, v3.4s\n" - "fmax v19.4s, v28.4s, v22.4s\n" - "fmax v19.4s, v23.4s, v19.4s\n" - "fmax v8.4s, v8.4s, v19.4s\n" + "fmax v17.4s, v4.4s, v3.4s\n" + "fmax v16.4s, v28.4s, v22.4s\n" + "fmax v16.4s, v17.4s, v16.4s\n" + "fmax v8.4s, v8.4s, v16.4s\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x24, [x20], #0x8\n" - "ldr q4, [x24, x9]\n" + "ldr x20, [x22], #0x8\n" + "ldr q16, [x20, x27]\n" "subs x21, x21, #0x1\n" - "fmax v8.4s, v8.4s, v4.4s\n" + "fmax v8.4s, v8.4s, v16.4s\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "sub %x[n_channels], %x[n_channels], #0x4\n" "cmp %x[n_channels], #0x4\n" - "str q8, [%x[outptr], x9]\n" - "add x9, x9, #0x10\n" + "str q8, [%x[outptr], x27]\n" + "add x27, x27, #0x10\n" "bge 8b\n" "cbz %x[n_channels], 25f\n" "14:" // Oddments "mov w20, #0xff800000\n" "lsr x25, %x[n_valid_cells], #0x2\n" "dup v8.4s, w20\n" - "add %x[outptr], %x[outptr], x9\n" - "mov x20, %x[inptrs]\n" + "add %x[outptr], %x[outptr], x27\n" + "mov x24, %x[inptrs]\n" "cbz x25, 18f\n" "15:" // Oddments: 4 inputs loop - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "add x24, x24, x9\n" - "add x23, x23, x9\n" - "add x22, x22, x9\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" + "add x23, x23, x27\n" + "add x22, x22, x27\n" + "add x21, x21, x27\n" "movi v4.16b, #0x0\n" "movi v3.16b, #0x0\n" - "add x21, x21, x9\n" + "add x20, x20, x27\n" "movi v28.16b, #0x0\n" "movi v22.16b, #0x0\n" "tbz %x[n_channels], #1, 16f\n" - "ldr d4, [x24], #0x8\n" - "ldr d3, [x23], #0x8\n" - "ldr d28, [x22], #0x8\n" - "ldr d22, [x21], #0x8\n" + "ldr d4, [x23], #0x8\n" + "ldr d3, [x22], #0x8\n" + "ldr d28, [x21], #0x8\n" + "ldr d22, [x20], #0x8\n" "tbz %x[n_channels], #0, 17f\n" - "ld1 { v4.s }[2], [x24], #0x4\n" - "ld1 { v3.s }[2], [x23], #0x4\n" - "ld1 { v28.s }[2], [x22], #0x4\n" - "ld1 { v22.s }[2], [x21], #0x4\n" + "ld1 { v4.s }[2], [x23], #0x4\n" + "ld1 { v3.s }[2], [x22], #0x4\n" + "ld1 { v28.s }[2], [x21], #0x4\n" + "ld1 { v22.s }[2], [x20], #0x4\n" "b 17f\n" "16:" // Oddments: 4 inputs loop: Load: Bit 1: Unset "tbz %x[n_channels], #0, 17f\n" - "ldr s4, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr s28, [x22], #0x4\n" - "ldr s22, [x21], #0x4\n" + "ldr s4, [x23], #0x4\n" + "ldr s3, [x22], #0x4\n" + "ldr s28, [x21], #0x4\n" + "ldr s22, [x20], #0x4\n" "17:" // Oddments: 4 inputs loop: Load: Bit 1: End - "fmax v23.4s, v4.4s, v3.4s\n" - "fmax v19.4s, v28.4s, v22.4s\n" + "fmax v17.4s, v4.4s, v3.4s\n" + "fmax v16.4s, v28.4s, v22.4s\n" "subs x25, x25, #0x1\n" - "fmax v19.4s, v23.4s, v19.4s\n" - "fmax v8.4s, v8.4s, v19.4s\n" + "fmax v16.4s, v17.4s, v16.4s\n" + "fmax v8.4s, v8.4s, v16.4s\n" "bgt 15b\n" "18:" // Oddments: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 22f\n" "19:" // Oddments: Single input loop - "ldr x24, [x20], #0x8\n" - "add x24, x24, x9\n" + "ldr x23, [x24], #0x8\n" + "add x23, x23, x27\n" "movi v4.16b, #0x0\n" "tbz %x[n_channels], #1, 20f\n" - "ldr d4, [x24], #0x8\n" + "ldr d4, [x23], #0x8\n" "tbz %x[n_channels], #0, 21f\n" - "ld1 { v4.s }[2], [x24], #0x4\n" + "ld1 { v4.s }[2], [x23], #0x4\n" "b 21f\n" "20:" // Oddments: Single input loop: Load: Bit 1: Unset "tbz %x[n_channels], #0, 21f\n" - "ldr s4, [x24], #0x4\n" + "ldr s4, [x23], #0x4\n" "21:" // Oddments: Single input loop: Load: Bit 1: End "subs x21, x21, #0x1\n" "fmax v8.4s, v8.4s, v4.4s\n" @@ -287,10 +287,11 @@ void a64_fp32_nhwc_max_generic_depthfirst_impl( "25:" // End : [n_channels] "+&r" (n_channels), [outptr] "+&r" (outptr) : [inptrs] "r" (inptrs), [n_valid_cells] "r" (n_valid_cells) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27" ); } } // namespace pooling } // namespace arm_conv + #endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_avg_generic_depthfirst/generic.cpp index 5a7e5f981b25033ff744cd62fd3c81d0b56f4db9..5d082102b34e0b2b697f62d519e5b9228f924aa7 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_avg_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_avg_generic_depthfirst/generic.cpp @@ -22,14 +22,14 @@ * SOFTWARE. */ -#if defined(__aarch64__) - #include #include #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace pooling { @@ -105,7 +105,7 @@ void a64_s8_nhwc_avg_generic_depthfirst_impl( "lsr x23, %x[n_valid_cells], #0x1\n" "movi v15.4s, #0x0\n" "movi v14.4s, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "movi v13.4s, #0x0\n" "movi v12.4s, #0x0\n" "movi v11.4s, #0x0\n" @@ -121,42 +121,42 @@ void a64_s8_nhwc_avg_generic_depthfirst_impl( "movi v1.4s, #0x0\n" "movi v0.4s, #0x0\n" "cbz x23, 4f\n" - "ldp x22, x21, [x20, #0x0]\n" - "ldr q31, [x22, x27]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q31, [x21, x27]\n" "subs x23, x23, #0x1\n" - "add x20, x20, #0x10\n" - "ldr q30, [x21, x27]\n" - "ldr q29, [x22, x26]\n" - "ldr q28, [x21, x26]\n" - "ldr q27, [x22, x25]\n" - "ldr q26, [x21, x25]\n" - "ldr q25, [x22, x24]\n" - "ldr q24, [x21, x24]\n" + "add x22, x22, #0x10\n" + "ldr q30, [x20, x27]\n" + "ldr q29, [x21, x26]\n" + "ldr q28, [x20, x26]\n" + "ldr q27, [x21, x25]\n" + "ldr q26, [x20, x25]\n" + "ldr q25, [x21, x24]\n" + "ldr q24, [x20, x24]\n" "beq 3f\n" "2:" // 4-vectors of channels: 2 inputs loop "saddl v23.8h, v31.8b, v30.8b\n" "saddl2 v22.8h, v31.16b, v30.16b\n" - "ldp x22, x21, [x20, #0x0]\n" - "ldr q31, [x22, x27]\n" - "ldr q30, [x21, x27]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q31, [x21, x27]\n" + "ldr q30, [x20, x27]\n" "saddl v21.8h, v29.8b, v28.8b\n" "saddl2 v20.8h, v29.16b, v28.16b\n" - "ldr q29, [x22, x26]\n" - "ldr q28, [x21, x26]\n" + "ldr q29, [x21, x26]\n" + "ldr q28, [x20, x26]\n" "saddl v19.8h, v27.8b, v26.8b\n" "saddl2 v18.8h, v27.16b, v26.16b\n" - "ldr q27, [x22, x25]\n" - "ldr q26, [x21, x25]\n" + "ldr q27, [x21, x25]\n" + "ldr q26, [x20, x25]\n" + "saddl v17.8h, v25.8b, v24.8b\n" + "saddl2 v16.8h, v25.16b, v24.16b\n" + "ldr q25, [x21, x24]\n" + "ldr q24, [x20, x24]\n" "subs x23, x23, #0x1\n" "saddw v15.4s, v15.4s, v23.4h\n" "saddw2 v14.4s, v14.4s, v23.8h\n" - "saddl v17.8h, v25.8b, v24.8b\n" - "saddl2 v16.8h, v25.16b, v24.16b\n" - "ldr q25, [x22, x24]\n" - "add x20, x20, #0x10\n" "saddw v13.4s, v13.4s, v22.4h\n" "saddw2 v12.4s, v12.4s, v22.8h\n" - "ldr q24, [x21, x24]\n" + "add x22, x22, #0x10\n" "saddw v11.4s, v11.4s, v21.4h\n" "saddw2 v10.4s, v10.4s, v21.8h\n" "saddw v9.4s, v9.4s, v20.4h\n" @@ -196,23 +196,23 @@ void a64_s8_nhwc_avg_generic_depthfirst_impl( "saddw v1.4s, v1.4s, v16.4h\n" "saddw2 v0.4s, v0.4s, v16.8h\n" "4:" // 4-vectors of channels: After loop - "ands x21, %x[n_valid_cells], #0x1\n" + "ands x23, %x[n_valid_cells], #0x1\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x22, [x20], #0x8\n" - "ldr q31, [x22, x27]\n" - "sxtl v23.8h, v31.8b\n" - "sxtl2 v22.8h, v31.16b\n" - "ldr q29, [x22, x26]\n" - "ldr q27, [x22, x25]\n" - "sxtl v21.8h, v29.8b\n" - "sxtl2 v20.8h, v29.16b\n" - "ldr q25, [x22, x24]\n" - "sxtl v19.8h, v27.8b\n" - "sxtl2 v18.8h, v27.16b\n" - "subs x21, x21, #0x1\n" - "sxtl v17.8h, v25.8b\n" - "sxtl2 v16.8h, v25.16b\n" + "ldr x20, [x22], #0x8\n" + "ldr q16, [x20, x27]\n" + "sxtl v23.8h, v16.8b\n" + "sxtl2 v22.8h, v16.16b\n" + "ldr q16, [x20, x26]\n" + "ldr q17, [x20, x25]\n" + "sxtl v21.8h, v16.8b\n" + "sxtl2 v20.8h, v16.16b\n" + "ldr q16, [x20, x24]\n" + "sxtl v19.8h, v17.8b\n" + "sxtl2 v18.8h, v17.16b\n" + "subs x23, x23, #0x1\n" + "sxtl v17.8h, v16.8b\n" + "sxtl2 v16.8h, v16.16b\n" "saddw v15.4s, v15.4s, v23.4h\n" "saddw2 v14.4s, v14.4s, v23.8h\n" "saddw v13.4s, v13.4s, v22.4h\n" @@ -330,49 +330,49 @@ void a64_s8_nhwc_avg_generic_depthfirst_impl( "lsr x23, %x[n_valid_cells], #0x1\n" "movi v15.4s, #0x0\n" "movi v14.4s, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "movi v13.4s, #0x0\n" "movi v12.4s, #0x0\n" "cbz x23, 11f\n" - "ldp x22, x21, [x20, #0x0]\n" - "ldr q31, [x22, x27]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q31, [x21, x27]\n" "subs x23, x23, #0x1\n" - "add x20, x20, #0x10\n" - "ldr q30, [x21, x27]\n" + "add x22, x22, #0x10\n" + "ldr q30, [x20, x27]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 2 inputs loop - "saddl v23.8h, v31.8b, v30.8b\n" - "saddl2 v22.8h, v31.16b, v30.16b\n" - "ldp x22, x21, [x20, #0x0]\n" - "ldr q31, [x22, x27]\n" - "ldr q30, [x21, x27]\n" + "saddl v17.8h, v31.8b, v30.8b\n" + "saddl2 v16.8h, v31.16b, v30.16b\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q31, [x21, x27]\n" + "ldr q30, [x20, x27]\n" "subs x23, x23, #0x1\n" - "saddw v15.4s, v15.4s, v23.4h\n" - "saddw2 v14.4s, v14.4s, v23.8h\n" - "saddw v13.4s, v13.4s, v22.4h\n" - "saddw2 v12.4s, v12.4s, v22.8h\n" - "add x20, x20, #0x10\n" + "saddw v15.4s, v15.4s, v17.4h\n" + "saddw2 v14.4s, v14.4s, v17.8h\n" + "saddw v13.4s, v13.4s, v16.4h\n" + "saddw2 v12.4s, v12.4s, v16.8h\n" + "add x22, x22, #0x10\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 2 inputs tail - "saddl v23.8h, v31.8b, v30.8b\n" - "saddl2 v22.8h, v31.16b, v30.16b\n" - "saddw v15.4s, v15.4s, v23.4h\n" - "saddw2 v14.4s, v14.4s, v23.8h\n" - "saddw v13.4s, v13.4s, v22.4h\n" - "saddw2 v12.4s, v12.4s, v22.8h\n" + "saddl v17.8h, v31.8b, v30.8b\n" + "saddl2 v16.8h, v31.16b, v30.16b\n" + "saddw v15.4s, v15.4s, v17.4h\n" + "saddw2 v14.4s, v14.4s, v17.8h\n" + "saddw v13.4s, v13.4s, v16.4h\n" + "saddw2 v12.4s, v12.4s, v16.8h\n" "11:" // Single vector of channels: Loop: After loop - "ands x21, %x[n_valid_cells], #0x1\n" + "ands x23, %x[n_valid_cells], #0x1\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x22, [x20], #0x8\n" - "ldr q31, [x22, x27]\n" - "sxtl v23.8h, v31.8b\n" - "sxtl2 v22.8h, v31.16b\n" - "subs x21, x21, #0x1\n" - "saddw v15.4s, v15.4s, v23.4h\n" - "saddw2 v14.4s, v14.4s, v23.8h\n" - "saddw v13.4s, v13.4s, v22.4h\n" - "saddw2 v12.4s, v12.4s, v22.8h\n" + "ldr x20, [x22], #0x8\n" + "ldr q16, [x20, x27]\n" + "sxtl v17.8h, v16.8b\n" + "sxtl2 v16.8h, v16.16b\n" + "subs x23, x23, #0x1\n" + "saddw v15.4s, v15.4s, v17.4h\n" + "saddw2 v14.4s, v14.4s, v17.8h\n" + "saddw v13.4s, v13.4s, v16.4h\n" + "saddw2 v12.4s, v12.4s, v16.8h\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "ld1r { v17.4s }, [%x[rescale_ptr]]\n" @@ -397,9 +397,9 @@ void a64_s8_nhwc_avg_generic_depthfirst_impl( "smin v14.4s, v14.4s, v17.4s\n" "smin v13.4s, v13.4s, v17.4s\n" "smin v12.4s, v12.4s, v17.4s\n" - "uzp1 v23.16b, v15.16b, v14.16b\n" + "uzp1 v17.16b, v15.16b, v14.16b\n" "uzp1 v16.16b, v13.16b, v12.16b\n" - "uzp1 v16.16b, v23.16b, v16.16b\n" + "uzp1 v16.16b, v17.16b, v16.16b\n" "str q16, [%x[outptr], x27]\n" "add x27, x27, #0x10\n" "bge 8b\n" @@ -411,142 +411,142 @@ void a64_s8_nhwc_avg_generic_depthfirst_impl( "movi v14.4s, #0x0\n" "movi v13.4s, #0x0\n" "movi v12.4s, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "cbz x23, 24f\n" "15:" // Oddments: 2 inputs loop - "ldp x22, x21, [x20, #0x0]\n" - "add x20, x20, #0x10\n" - "add x22, x22, x27\n" - "movi v31.16b, #0x0\n" + "ldp x21, x20, [x22, #0x0]\n" + "add x22, x22, #0x10\n" "add x21, x21, x27\n" + "movi v31.16b, #0x0\n" + "add x20, x20, x27\n" "movi v30.16b, #0x0\n" "tbz %x[n_channels], #3, 19f\n" - "ldr d31, [x22], #0x8\n" - "ldr d30, [x21], #0x8\n" + "ldr d31, [x21], #0x8\n" + "ldr d30, [x20], #0x8\n" "tbz %x[n_channels], #2, 17f\n" - "ld1 { v31.s }[2], [x22], #0x4\n" - "ld1 { v30.s }[2], [x21], #0x4\n" + "ld1 { v31.s }[2], [x21], #0x4\n" + "ld1 { v30.s }[2], [x20], #0x4\n" "tbz %x[n_channels], #1, 16f\n" - "ld1 { v31.h }[6], [x22], #0x2\n" - "ld1 { v30.h }[6], [x21], #0x2\n" + "ld1 { v31.h }[6], [x21], #0x2\n" + "ld1 { v30.h }[6], [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[14], [x22], #0x1\n" - "ld1 { v30.b }[14], [x21], #0x1\n" + "ld1 { v31.b }[14], [x21], #0x1\n" + "ld1 { v30.b }[14], [x20], #0x1\n" "b 23f\n" "16:" // Oddments: 2 inputs loop: Load: Bit 3: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[12], [x22], #0x1\n" - "ld1 { v30.b }[12], [x21], #0x1\n" + "ld1 { v31.b }[12], [x21], #0x1\n" + "ld1 { v30.b }[12], [x20], #0x1\n" "b 23f\n" "17:" // Oddments: 2 inputs loop: Load: Bit 3: Bit 2: Unset "tbz %x[n_channels], #1, 18f\n" - "ld1 { v31.h }[4], [x22], #0x2\n" - "ld1 { v30.h }[4], [x21], #0x2\n" + "ld1 { v31.h }[4], [x21], #0x2\n" + "ld1 { v30.h }[4], [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[10], [x22], #0x1\n" - "ld1 { v30.b }[10], [x21], #0x1\n" + "ld1 { v31.b }[10], [x21], #0x1\n" + "ld1 { v30.b }[10], [x20], #0x1\n" "b 23f\n" "18:" // Oddments: 2 inputs loop: Load: Bit 3: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[8], [x22], #0x1\n" - "ld1 { v30.b }[8], [x21], #0x1\n" + "ld1 { v31.b }[8], [x21], #0x1\n" + "ld1 { v30.b }[8], [x20], #0x1\n" "b 23f\n" "19:" // Oddments: 2 inputs loop: Load: Bit 3: Unset "tbz %x[n_channels], #2, 21f\n" - "ldr s31, [x22], #0x4\n" - "ldr s30, [x21], #0x4\n" + "ldr s31, [x21], #0x4\n" + "ldr s30, [x20], #0x4\n" "tbz %x[n_channels], #1, 20f\n" - "ld1 { v31.h }[2], [x22], #0x2\n" - "ld1 { v30.h }[2], [x21], #0x2\n" + "ld1 { v31.h }[2], [x21], #0x2\n" + "ld1 { v30.h }[2], [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[6], [x22], #0x1\n" - "ld1 { v30.b }[6], [x21], #0x1\n" + "ld1 { v31.b }[6], [x21], #0x1\n" + "ld1 { v30.b }[6], [x20], #0x1\n" "b 23f\n" "20:" // Oddments: 2 inputs loop: Load: Bit 3: Unset: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[4], [x22], #0x1\n" - "ld1 { v30.b }[4], [x21], #0x1\n" + "ld1 { v31.b }[4], [x21], #0x1\n" + "ld1 { v30.b }[4], [x20], #0x1\n" "b 23f\n" "21:" // Oddments: 2 inputs loop: Load: Bit 3: Unset: Bit 2: Unset "tbz %x[n_channels], #1, 22f\n" - "ldr h31, [x22], #0x2\n" - "ldr h30, [x21], #0x2\n" + "ldr h31, [x21], #0x2\n" + "ldr h30, [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[2], [x22], #0x1\n" - "ld1 { v30.b }[2], [x21], #0x1\n" + "ld1 { v31.b }[2], [x21], #0x1\n" + "ld1 { v30.b }[2], [x20], #0x1\n" "b 23f\n" "22:" // Oddments: 2 inputs loop: Load: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ldr b31, [x22], #0x1\n" - "ldr b30, [x21], #0x1\n" + "ldr b31, [x21], #0x1\n" + "ldr b30, [x20], #0x1\n" "23:" // Oddments: 2 inputs loop: Load: Bit 3: End - "saddl v23.8h, v31.8b, v30.8b\n" - "saddl2 v22.8h, v31.16b, v30.16b\n" + "saddl v17.8h, v31.8b, v30.8b\n" + "saddl2 v16.8h, v31.16b, v30.16b\n" "subs x23, x23, #0x1\n" - "saddw v15.4s, v15.4s, v23.4h\n" - "saddw2 v14.4s, v14.4s, v23.8h\n" - "saddw v13.4s, v13.4s, v22.4h\n" - "saddw2 v12.4s, v12.4s, v22.8h\n" + "saddw v15.4s, v15.4s, v17.4h\n" + "saddw2 v14.4s, v14.4s, v17.8h\n" + "saddw v13.4s, v13.4s, v16.4h\n" + "saddw2 v12.4s, v12.4s, v16.8h\n" "bgt 15b\n" "24:" // Oddments: After loop - "ands x21, %x[n_valid_cells], #0x1\n" + "ands x23, %x[n_valid_cells], #0x1\n" "beq 34f\n" "25:" // Oddments: Single input loop - "ldr x22, [x20], #0x8\n" - "add x22, x22, x27\n" + "ldr x21, [x22], #0x8\n" + "add x21, x21, x27\n" "movi v31.16b, #0x0\n" "tbz %x[n_channels], #3, 29f\n" - "ldr d31, [x22], #0x8\n" + "ldr d31, [x21], #0x8\n" "tbz %x[n_channels], #2, 27f\n" - "ld1 { v31.s }[2], [x22], #0x4\n" + "ld1 { v31.s }[2], [x21], #0x4\n" "tbz %x[n_channels], #1, 26f\n" - "ld1 { v31.h }[6], [x22], #0x2\n" + "ld1 { v31.h }[6], [x21], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[14], [x22], #0x1\n" + "ld1 { v31.b }[14], [x21], #0x1\n" "b 33f\n" "26:" // Oddments: Single input loop: Load: Bit 3: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[12], [x22], #0x1\n" + "ld1 { v31.b }[12], [x21], #0x1\n" "b 33f\n" "27:" // Oddments: Single input loop: Load: Bit 3: Bit 2: Unset "tbz %x[n_channels], #1, 28f\n" - "ld1 { v31.h }[4], [x22], #0x2\n" + "ld1 { v31.h }[4], [x21], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[10], [x22], #0x1\n" + "ld1 { v31.b }[10], [x21], #0x1\n" "b 33f\n" "28:" // Oddments: Single input loop: Load: Bit 3: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[8], [x22], #0x1\n" + "ld1 { v31.b }[8], [x21], #0x1\n" "b 33f\n" "29:" // Oddments: Single input loop: Load: Bit 3: Unset "tbz %x[n_channels], #2, 31f\n" - "ldr s31, [x22], #0x4\n" + "ldr s31, [x21], #0x4\n" "tbz %x[n_channels], #1, 30f\n" - "ld1 { v31.h }[2], [x22], #0x2\n" + "ld1 { v31.h }[2], [x21], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[6], [x22], #0x1\n" + "ld1 { v31.b }[6], [x21], #0x1\n" "b 33f\n" "30:" // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[4], [x22], #0x1\n" + "ld1 { v31.b }[4], [x21], #0x1\n" "b 33f\n" "31:" // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Unset "tbz %x[n_channels], #1, 32f\n" - "ldr h31, [x22], #0x2\n" + "ldr h31, [x21], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[2], [x22], #0x1\n" + "ld1 { v31.b }[2], [x21], #0x1\n" "b 33f\n" "32:" // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ldr b31, [x22], #0x1\n" + "ldr b31, [x21], #0x1\n" "33:" // Oddments: Single input loop: Load: Bit 3: End - "sxtl v23.8h, v31.8b\n" - "sxtl2 v22.8h, v31.16b\n" - "subs x21, x21, #0x1\n" - "saddw v15.4s, v15.4s, v23.4h\n" - "saddw2 v14.4s, v14.4s, v23.8h\n" - "saddw v13.4s, v13.4s, v22.4h\n" - "saddw2 v12.4s, v12.4s, v22.8h\n" + "sxtl v17.8h, v31.8b\n" + "sxtl2 v16.8h, v31.16b\n" + "subs x23, x23, #0x1\n" + "saddw v15.4s, v15.4s, v17.4h\n" + "saddw2 v14.4s, v14.4s, v17.8h\n" + "saddw v13.4s, v13.4s, v16.4h\n" + "saddw2 v12.4s, v12.4s, v16.8h\n" "bgt 25b\n" "34:" // Oddments: Single input loop: End "ld1r { v17.4s }, [%x[rescale_ptr]]\n" @@ -569,9 +569,9 @@ void a64_s8_nhwc_avg_generic_depthfirst_impl( "smin v14.4s, v14.4s, v17.4s\n" "smin v13.4s, v13.4s, v17.4s\n" "smin v12.4s, v12.4s, v17.4s\n" - "uzp1 v23.16b, v15.16b, v14.16b\n" + "uzp1 v17.16b, v15.16b, v14.16b\n" "uzp1 v16.16b, v13.16b, v12.16b\n" - "uzp1 v16.16b, v23.16b, v16.16b\n" + "uzp1 v16.16b, v17.16b, v16.16b\n" "tbz %x[n_channels], #3, 38f\n" "st1 { v16.d }[0], [%x[outptr]], #0x8\n" "tbz %x[n_channels], #2, 36f\n" @@ -626,4 +626,5 @@ void a64_s8_nhwc_avg_generic_depthfirst_impl( } // namespace pooling } // namespace arm_conv + #endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_max_2x2_s1_output2x2_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_max_2x2_s1_output2x2_depthfirst.hpp index 234b4442c8a447be6c4d552a6ac7fe6816fc0d10..f8f11348662d807b06486818d925381fc6045586 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_max_2x2_s1_output2x2_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_max_2x2_s1_output2x2_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,8 @@ #pragma once +#if defined(__aarch64__) + namespace arm_conv { namespace pooling { @@ -45,3 +47,5 @@ struct a64_s8_nhwc_max_2x2_s1_output2x2_depthfirst : public DepthfirstStrategy #include +#if defined(__aarch64__) + namespace arm_conv { namespace pooling { @@ -111,7 +112,7 @@ void a64_s8_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "smax v18.16b, v18.16b, v21.16b\n" "smax v17.16b, v17.16b, v20.16b\n" "add x15, x15, #0x10\n" - "smax v16.16b, v16.16b, v20.16b\n" + "smax v16.16b, v20.16b, v16.16b\n" "str q19, [x14, x12]\n" "str q18, [x13, x12]\n" "str q17, [x11, x12]\n" @@ -121,43 +122,43 @@ void a64_s8_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "2:" // Vector: Tail "smax v21.16b, v30.16b, v29.16b\n" "smax v20.16b, v29.16b, v28.16b\n" - "smax v19.16b, v27.16b, v26.16b\n" + "smax v16.16b, v27.16b, v26.16b\n" "smax v18.16b, v25.16b, v24.16b\n" "smax v17.16b, v27.16b, v23.16b\n" - "smax v16.16b, v24.16b, v22.16b\n" - "smax v19.16b, v21.16b, v19.16b\n" + "smax v19.16b, v24.16b, v22.16b\n" + "smax v16.16b, v21.16b, v16.16b\n" "smax v18.16b, v18.16b, v21.16b\n" - "str q19, [x14, x12]\n" + "str q16, [x14, x12]\n" "smax v17.16b, v17.16b, v20.16b\n" - "smax v16.16b, v16.16b, v20.16b\n" + "smax v16.16b, v20.16b, v19.16b\n" "str q18, [x13, x12]\n" "str q17, [x11, x12]\n" "str q16, [x10, x12]\n" "add x12, x12, #0x10\n" "cbz x16, 4f\n" "3:" // Oddments - "ldr b30, [x28, x15]\n" - "ldr b29, [x25, x15]\n" - "smax v21.16b, v30.16b, v29.16b\n" + "ldr b16, [x28, x15]\n" + "ldr b17, [x25, x15]\n" + "smax v23.16b, v16.16b, v17.16b\n" "subs x16, x16, #0x1\n" - "ldr b28, [x22, x15]\n" - "ldr b27, [x26, x15]\n" - "smax v20.16b, v29.16b, v28.16b\n" - "ldr b26, [x9, x15]\n" - "ldr b25, [x27, x15]\n" - "smax v19.16b, v27.16b, v26.16b\n" - "smax v19.16b, v21.16b, v19.16b\n" - "ldr b24, [x24, x15]\n" - "ldr b23, [x23, x15]\n" - "smax v18.16b, v25.16b, v24.16b\n" - "smax v17.16b, v27.16b, v23.16b\n" - "ldr b22, [x21, x15]\n" - "smax v16.16b, v24.16b, v22.16b\n" + "ldr b16, [x22, x15]\n" + "ldr b22, [x26, x15]\n" + "smax v21.16b, v17.16b, v16.16b\n" + "ldr b16, [x9, x15]\n" + "ldr b17, [x27, x15]\n" + "smax v16.16b, v22.16b, v16.16b\n" + "smax v20.16b, v23.16b, v16.16b\n" + "ldr b19, [x24, x15]\n" + "ldr b16, [x23, x15]\n" + "smax v18.16b, v17.16b, v19.16b\n" + "smax v17.16b, v22.16b, v16.16b\n" + "ldr b16, [x21, x15]\n" + "smax v16.16b, v19.16b, v16.16b\n" "add x15, x15, #0x1\n" - "smax v18.16b, v18.16b, v21.16b\n" - "smax v17.16b, v17.16b, v20.16b\n" - "smax v16.16b, v16.16b, v20.16b\n" - "str b19, [x14, x12]\n" + "smax v18.16b, v18.16b, v23.16b\n" + "smax v17.16b, v17.16b, v21.16b\n" + "smax v16.16b, v21.16b, v16.16b\n" + "str b20, [x14, x12]\n" "str b18, [x13, x12]\n" "str b17, [x11, x12]\n" "str b16, [x10, x12]\n" @@ -172,4 +173,5 @@ void a64_s8_nhwc_max_2x2_s1_output2x2_depthfirst_impl( } // namespace pooling } // namespace arm_conv + #endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_max_generic_depthfirst/generic.cpp index 6168a57ca4ebf9e3cd7c40ce35754e5a8d62736f..411fd11460e66d476990c4a535107a20a613a708 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_max_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_max_generic_depthfirst/generic.cpp @@ -22,11 +22,11 @@ * SOFTWARE. */ -#if defined(__aarch64__) - #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace pooling { @@ -41,77 +41,77 @@ void a64_s8_nhwc_max_generic_depthfirst_impl( { __asm__ __volatile__( "cmp %x[n_channels], #0x40\n" - "mov x9, #0x0\n" - "mov x28, #0x10\n" // cntb _, ALL, #1 - "mov x27, #0x20\n" // cntb _, ALL, #2 - "mov x26, #0x30\n" // cntb _, ALL, #3 + "mov x27, #0x0\n" + "mov x26, #0x10\n" // cntb _, ALL, #1 + "mov x24, #0x20\n" // cntb _, ALL, #2 + "mov x23, #0x30\n" // cntb _, ALL, #3 "blt 7f\n" "1:" // 4-vectors of channels "lsr x25, %x[n_valid_cells], #0x2\n" "movi v8.16b, #0x80\n" "movi v7.16b, #0x80\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "movi v6.16b, #0x80\n" "movi v5.16b, #0x80\n" "cbz x25, 4f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" "subs x25, x25, #0x1\n" - "ldr q3, [x23, x9]\n" - "ldr q2, [x24, x28]\n" - "ldr q1, [x23, x28]\n" - "ldr q0, [x24, x27]\n" - "ldr q31, [x23, x27]\n" - "ldr q30, [x24, x26]\n" - "ldr q29, [x23, x26]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" - "ldr q27, [x22, x28]\n" - "ldr q21, [x21, x28]\n" - "ldr q26, [x22, x27]\n" - "ldr q20, [x21, x27]\n" - "ldr q25, [x22, x26]\n" - "ldr q24, [x21, x26]\n" + "ldr q3, [x20, x27]\n" + "ldr q2, [x21, x26]\n" + "ldr q1, [x20, x26]\n" + "ldr q0, [x21, x24]\n" + "ldr q31, [x20, x24]\n" + "ldr q30, [x21, x23]\n" + "ldr q29, [x20, x23]\n" + "ldp x21, x20, [x22, #0x10]\n" + "add x22, x22, #0x20\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" + "ldr q27, [x21, x26]\n" + "ldr q21, [x20, x26]\n" + "ldr q26, [x21, x24]\n" + "ldr q20, [x20, x24]\n" + "ldr q25, [x21, x23]\n" + "ldr q24, [x20, x23]\n" "beq 3f\n" "2:" // 4-vectors of channels: 4 inputs loop "smax v23.16b, v4.16b, v3.16b\n" "smax v19.16b, v28.16b, v22.16b\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" - "ldr q3, [x23, x9]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" + "ldr q3, [x20, x27]\n" "smax v22.16b, v2.16b, v1.16b\n" - "ldr q2, [x24, x28]\n" + "ldr q2, [x21, x26]\n" "smax v18.16b, v27.16b, v21.16b\n" - "ldr q1, [x23, x28]\n" + "ldr q1, [x20, x26]\n" "smax v21.16b, v0.16b, v31.16b\n" - "ldr q0, [x24, x27]\n" + "ldr q0, [x21, x24]\n" "smax v17.16b, v26.16b, v20.16b\n" - "ldr q31, [x23, x27]\n" + "ldr q31, [x20, x24]\n" "smax v20.16b, v30.16b, v29.16b\n" - "ldr q30, [x24, x26]\n" + "ldr q30, [x21, x23]\n" "smax v16.16b, v25.16b, v24.16b\n" - "ldr q29, [x23, x26]\n" + "ldr q29, [x20, x23]\n" "smax v19.16b, v23.16b, v19.16b\n" "smax v18.16b, v22.16b, v18.16b\n" - "ldp x22, x21, [x20, #0x10]\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" + "ldp x21, x20, [x22, #0x10]\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" "smax v17.16b, v21.16b, v17.16b\n" "smax v16.16b, v20.16b, v16.16b\n" - "ldr q27, [x22, x28]\n" - "ldr q21, [x21, x28]\n" + "ldr q27, [x21, x26]\n" + "ldr q21, [x20, x26]\n" "subs x25, x25, #0x1\n" "smax v8.16b, v8.16b, v19.16b\n" - "ldr q26, [x22, x27]\n" - "ldr q20, [x21, x27]\n" + "ldr q26, [x21, x24]\n" + "ldr q20, [x20, x24]\n" "smax v7.16b, v7.16b, v18.16b\n" "smax v6.16b, v6.16b, v17.16b\n" - "ldr q25, [x22, x26]\n" - "ldr q24, [x21, x26]\n" + "ldr q25, [x21, x23]\n" + "ldr q24, [x20, x23]\n" "smax v5.16b, v5.16b, v16.16b\n" - "add x20, x20, #0x20\n" + "add x22, x22, #0x20\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "smax v23.16b, v4.16b, v3.16b\n" @@ -134,28 +134,28 @@ void a64_s8_nhwc_max_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x24, [x20], #0x8\n" - "ldr q4, [x24, x9]\n" + "ldr x20, [x22], #0x8\n" + "ldr q16, [x20, x27]\n" "subs x21, x21, #0x1\n" - "smax v8.16b, v8.16b, v4.16b\n" - "ldr q2, [x24, x28]\n" - "ldr q0, [x24, x27]\n" - "smax v7.16b, v7.16b, v2.16b\n" - "smax v6.16b, v6.16b, v0.16b\n" - "ldr q30, [x24, x26]\n" - "smax v5.16b, v5.16b, v30.16b\n" + "smax v8.16b, v8.16b, v16.16b\n" + "ldr q17, [x20, x26]\n" + "ldr q16, [x20, x24]\n" + "smax v7.16b, v7.16b, v17.16b\n" + "smax v6.16b, v6.16b, v16.16b\n" + "ldr q16, [x20, x23]\n" + "smax v5.16b, v5.16b, v16.16b\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End "sub %x[n_channels], %x[n_channels], #0x40\n" "cmp %x[n_channels], #0x40\n" - "str q8, [%x[outptr], x9]\n" - "str q7, [%x[outptr], x28]\n" - "add x9, x9, #0x40\n" - "add x28, x28, #0x40\n" - "str q6, [%x[outptr], x27]\n" + "str q8, [%x[outptr], x27]\n" + "str q7, [%x[outptr], x26]\n" "add x27, x27, #0x40\n" - "str q5, [%x[outptr], x26]\n" "add x26, x26, #0x40\n" + "str q6, [%x[outptr], x24]\n" + "add x24, x24, #0x40\n" + "str q5, [%x[outptr], x23]\n" + "add x23, x23, #0x40\n" "bge 1b\n" "cbz %x[n_channels], 43f\n" "7:" // Single vector of channels @@ -164,217 +164,217 @@ void a64_s8_nhwc_max_generic_depthfirst_impl( "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" "movi v8.16b, #0x80\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" "subs x25, x25, #0x1\n" - "ldr q3, [x23, x9]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" + "ldr q3, [x20, x27]\n" + "ldp x21, x20, [x22, #0x10]\n" + "add x22, x22, #0x20\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 4 inputs loop - "smax v23.16b, v4.16b, v3.16b\n" - "smax v19.16b, v28.16b, v22.16b\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" - "ldr q3, [x23, x9]\n" - "smax v19.16b, v23.16b, v19.16b\n" - "ldp x22, x21, [x20, #0x10]\n" + "smax v17.16b, v4.16b, v3.16b\n" + "smax v16.16b, v28.16b, v22.16b\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" + "ldr q3, [x20, x27]\n" + "smax v16.16b, v17.16b, v16.16b\n" + "ldp x21, x20, [x22, #0x10]\n" "subs x25, x25, #0x1\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" - "smax v8.16b, v8.16b, v19.16b\n" - "add x20, x20, #0x20\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" + "smax v8.16b, v8.16b, v16.16b\n" + "add x22, x22, #0x20\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail - "smax v23.16b, v4.16b, v3.16b\n" - "smax v19.16b, v28.16b, v22.16b\n" - "smax v19.16b, v23.16b, v19.16b\n" - "smax v8.16b, v8.16b, v19.16b\n" + "smax v17.16b, v4.16b, v3.16b\n" + "smax v16.16b, v28.16b, v22.16b\n" + "smax v16.16b, v17.16b, v16.16b\n" + "smax v8.16b, v8.16b, v16.16b\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x24, [x20], #0x8\n" - "ldr q4, [x24, x9]\n" + "ldr x20, [x22], #0x8\n" + "ldr q16, [x20, x27]\n" "subs x21, x21, #0x1\n" - "smax v8.16b, v8.16b, v4.16b\n" + "smax v8.16b, v8.16b, v16.16b\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "sub %x[n_channels], %x[n_channels], #0x10\n" "cmp %x[n_channels], #0x10\n" - "str q8, [%x[outptr], x9]\n" - "add x9, x9, #0x10\n" + "str q8, [%x[outptr], x27]\n" + "add x27, x27, #0x10\n" "bge 8b\n" "cbz %x[n_channels], 43f\n" "14:" // Oddments "lsr x25, %x[n_valid_cells], #0x2\n" - "add %x[outptr], %x[outptr], x9\n" + "add %x[outptr], %x[outptr], x27\n" "movi v8.16b, #0x80\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "cbz x25, 24f\n" "15:" // Oddments: 4 inputs loop - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "add x24, x24, x9\n" - "add x23, x23, x9\n" - "add x22, x22, x9\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" + "add x23, x23, x27\n" + "add x22, x22, x27\n" + "add x21, x21, x27\n" "movi v4.16b, #0x0\n" "movi v3.16b, #0x0\n" - "add x21, x21, x9\n" + "add x20, x20, x27\n" "movi v28.16b, #0x0\n" "movi v22.16b, #0x0\n" "tbz %x[n_channels], #3, 19f\n" - "ldr d4, [x24], #0x8\n" - "ldr d3, [x23], #0x8\n" - "ldr d28, [x22], #0x8\n" - "ldr d22, [x21], #0x8\n" + "ldr d4, [x23], #0x8\n" + "ldr d3, [x22], #0x8\n" + "ldr d28, [x21], #0x8\n" + "ldr d22, [x20], #0x8\n" "tbz %x[n_channels], #2, 17f\n" - "ld1 { v4.s }[2], [x24], #0x4\n" - "ld1 { v3.s }[2], [x23], #0x4\n" - "ld1 { v28.s }[2], [x22], #0x4\n" - "ld1 { v22.s }[2], [x21], #0x4\n" + "ld1 { v4.s }[2], [x23], #0x4\n" + "ld1 { v3.s }[2], [x22], #0x4\n" + "ld1 { v28.s }[2], [x21], #0x4\n" + "ld1 { v22.s }[2], [x20], #0x4\n" "tbz %x[n_channels], #1, 16f\n" - "ld1 { v4.h }[6], [x24], #0x2\n" - "ld1 { v3.h }[6], [x23], #0x2\n" - "ld1 { v28.h }[6], [x22], #0x2\n" - "ld1 { v22.h }[6], [x21], #0x2\n" + "ld1 { v4.h }[6], [x23], #0x2\n" + "ld1 { v3.h }[6], [x22], #0x2\n" + "ld1 { v28.h }[6], [x21], #0x2\n" + "ld1 { v22.h }[6], [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[14], [x24], #0x1\n" - "ld1 { v3.b }[14], [x23], #0x1\n" - "ld1 { v28.b }[14], [x22], #0x1\n" - "ld1 { v22.b }[14], [x21], #0x1\n" + "ld1 { v4.b }[14], [x23], #0x1\n" + "ld1 { v3.b }[14], [x22], #0x1\n" + "ld1 { v28.b }[14], [x21], #0x1\n" + "ld1 { v22.b }[14], [x20], #0x1\n" "b 23f\n" "16:" // Oddments: 4 inputs loop: Load: Bit 3: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[12], [x24], #0x1\n" - "ld1 { v3.b }[12], [x23], #0x1\n" - "ld1 { v28.b }[12], [x22], #0x1\n" - "ld1 { v22.b }[12], [x21], #0x1\n" + "ld1 { v4.b }[12], [x23], #0x1\n" + "ld1 { v3.b }[12], [x22], #0x1\n" + "ld1 { v28.b }[12], [x21], #0x1\n" + "ld1 { v22.b }[12], [x20], #0x1\n" "b 23f\n" "17:" // Oddments: 4 inputs loop: Load: Bit 3: Bit 2: Unset "tbz %x[n_channels], #1, 18f\n" - "ld1 { v4.h }[4], [x24], #0x2\n" - "ld1 { v3.h }[4], [x23], #0x2\n" - "ld1 { v28.h }[4], [x22], #0x2\n" - "ld1 { v22.h }[4], [x21], #0x2\n" + "ld1 { v4.h }[4], [x23], #0x2\n" + "ld1 { v3.h }[4], [x22], #0x2\n" + "ld1 { v28.h }[4], [x21], #0x2\n" + "ld1 { v22.h }[4], [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[10], [x24], #0x1\n" - "ld1 { v3.b }[10], [x23], #0x1\n" - "ld1 { v28.b }[10], [x22], #0x1\n" - "ld1 { v22.b }[10], [x21], #0x1\n" + "ld1 { v4.b }[10], [x23], #0x1\n" + "ld1 { v3.b }[10], [x22], #0x1\n" + "ld1 { v28.b }[10], [x21], #0x1\n" + "ld1 { v22.b }[10], [x20], #0x1\n" "b 23f\n" "18:" // Oddments: 4 inputs loop: Load: Bit 3: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[8], [x24], #0x1\n" - "ld1 { v3.b }[8], [x23], #0x1\n" - "ld1 { v28.b }[8], [x22], #0x1\n" - "ld1 { v22.b }[8], [x21], #0x1\n" + "ld1 { v4.b }[8], [x23], #0x1\n" + "ld1 { v3.b }[8], [x22], #0x1\n" + "ld1 { v28.b }[8], [x21], #0x1\n" + "ld1 { v22.b }[8], [x20], #0x1\n" "b 23f\n" "19:" // Oddments: 4 inputs loop: Load: Bit 3: Unset "tbz %x[n_channels], #2, 21f\n" - "ldr s4, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr s28, [x22], #0x4\n" - "ldr s22, [x21], #0x4\n" + "ldr s4, [x23], #0x4\n" + "ldr s3, [x22], #0x4\n" + "ldr s28, [x21], #0x4\n" + "ldr s22, [x20], #0x4\n" "tbz %x[n_channels], #1, 20f\n" - "ld1 { v4.h }[2], [x24], #0x2\n" - "ld1 { v3.h }[2], [x23], #0x2\n" - "ld1 { v28.h }[2], [x22], #0x2\n" - "ld1 { v22.h }[2], [x21], #0x2\n" + "ld1 { v4.h }[2], [x23], #0x2\n" + "ld1 { v3.h }[2], [x22], #0x2\n" + "ld1 { v28.h }[2], [x21], #0x2\n" + "ld1 { v22.h }[2], [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[6], [x24], #0x1\n" - "ld1 { v3.b }[6], [x23], #0x1\n" - "ld1 { v28.b }[6], [x22], #0x1\n" - "ld1 { v22.b }[6], [x21], #0x1\n" + "ld1 { v4.b }[6], [x23], #0x1\n" + "ld1 { v3.b }[6], [x22], #0x1\n" + "ld1 { v28.b }[6], [x21], #0x1\n" + "ld1 { v22.b }[6], [x20], #0x1\n" "b 23f\n" "20:" // Oddments: 4 inputs loop: Load: Bit 3: Unset: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[4], [x24], #0x1\n" - "ld1 { v3.b }[4], [x23], #0x1\n" - "ld1 { v28.b }[4], [x22], #0x1\n" - "ld1 { v22.b }[4], [x21], #0x1\n" + "ld1 { v4.b }[4], [x23], #0x1\n" + "ld1 { v3.b }[4], [x22], #0x1\n" + "ld1 { v28.b }[4], [x21], #0x1\n" + "ld1 { v22.b }[4], [x20], #0x1\n" "b 23f\n" "21:" // Oddments: 4 inputs loop: Load: Bit 3: Unset: Bit 2: Unset "tbz %x[n_channels], #1, 22f\n" - "ldr h4, [x24], #0x2\n" - "ldr h3, [x23], #0x2\n" - "ldr h28, [x22], #0x2\n" - "ldr h22, [x21], #0x2\n" + "ldr h4, [x23], #0x2\n" + "ldr h3, [x22], #0x2\n" + "ldr h28, [x21], #0x2\n" + "ldr h22, [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[2], [x24], #0x1\n" - "ld1 { v3.b }[2], [x23], #0x1\n" - "ld1 { v28.b }[2], [x22], #0x1\n" - "ld1 { v22.b }[2], [x21], #0x1\n" + "ld1 { v4.b }[2], [x23], #0x1\n" + "ld1 { v3.b }[2], [x22], #0x1\n" + "ld1 { v28.b }[2], [x21], #0x1\n" + "ld1 { v22.b }[2], [x20], #0x1\n" "b 23f\n" "22:" // Oddments: 4 inputs loop: Load: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ldr b4, [x24], #0x1\n" - "ldr b3, [x23], #0x1\n" - "ldr b28, [x22], #0x1\n" - "ldr b22, [x21], #0x1\n" + "ldr b4, [x23], #0x1\n" + "ldr b3, [x22], #0x1\n" + "ldr b28, [x21], #0x1\n" + "ldr b22, [x20], #0x1\n" "23:" // Oddments: 4 inputs loop: Load: Bit 3: End - "smax v23.16b, v4.16b, v3.16b\n" - "smax v19.16b, v28.16b, v22.16b\n" + "smax v17.16b, v4.16b, v3.16b\n" + "smax v16.16b, v28.16b, v22.16b\n" "subs x25, x25, #0x1\n" - "smax v19.16b, v23.16b, v19.16b\n" - "smax v8.16b, v8.16b, v19.16b\n" + "smax v16.16b, v17.16b, v16.16b\n" + "smax v8.16b, v8.16b, v16.16b\n" "bgt 15b\n" "24:" // Oddments: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 34f\n" "25:" // Oddments: Single input loop - "ldr x24, [x20], #0x8\n" - "add x24, x24, x9\n" + "ldr x23, [x24], #0x8\n" + "add x23, x23, x27\n" "movi v4.16b, #0x0\n" "tbz %x[n_channels], #3, 29f\n" - "ldr d4, [x24], #0x8\n" + "ldr d4, [x23], #0x8\n" "tbz %x[n_channels], #2, 27f\n" - "ld1 { v4.s }[2], [x24], #0x4\n" + "ld1 { v4.s }[2], [x23], #0x4\n" "tbz %x[n_channels], #1, 26f\n" - "ld1 { v4.h }[6], [x24], #0x2\n" + "ld1 { v4.h }[6], [x23], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[14], [x24], #0x1\n" + "ld1 { v4.b }[14], [x23], #0x1\n" "b 33f\n" "26:" // Oddments: Single input loop: Load: Bit 3: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[12], [x24], #0x1\n" + "ld1 { v4.b }[12], [x23], #0x1\n" "b 33f\n" "27:" // Oddments: Single input loop: Load: Bit 3: Bit 2: Unset "tbz %x[n_channels], #1, 28f\n" - "ld1 { v4.h }[4], [x24], #0x2\n" + "ld1 { v4.h }[4], [x23], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[10], [x24], #0x1\n" + "ld1 { v4.b }[10], [x23], #0x1\n" "b 33f\n" "28:" // Oddments: Single input loop: Load: Bit 3: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[8], [x24], #0x1\n" + "ld1 { v4.b }[8], [x23], #0x1\n" "b 33f\n" "29:" // Oddments: Single input loop: Load: Bit 3: Unset "tbz %x[n_channels], #2, 31f\n" - "ldr s4, [x24], #0x4\n" + "ldr s4, [x23], #0x4\n" "tbz %x[n_channels], #1, 30f\n" - "ld1 { v4.h }[2], [x24], #0x2\n" + "ld1 { v4.h }[2], [x23], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[6], [x24], #0x1\n" + "ld1 { v4.b }[6], [x23], #0x1\n" "b 33f\n" "30:" // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[4], [x24], #0x1\n" + "ld1 { v4.b }[4], [x23], #0x1\n" "b 33f\n" "31:" // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Unset "tbz %x[n_channels], #1, 32f\n" - "ldr h4, [x24], #0x2\n" + "ldr h4, [x23], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[2], [x24], #0x1\n" + "ld1 { v4.b }[2], [x23], #0x1\n" "b 33f\n" "32:" // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ldr b4, [x24], #0x1\n" + "ldr b4, [x23], #0x1\n" "33:" // Oddments: Single input loop: Load: Bit 3: End "subs x21, x21, #0x1\n" "smax v8.16b, v8.16b, v4.16b\n" @@ -428,10 +428,11 @@ void a64_s8_nhwc_max_generic_depthfirst_impl( "43:" // End : [n_channels] "+&r" (n_channels), [outptr] "+&r" (outptr) : [inptrs] "r" (inptrs), [n_valid_cells] "r" (n_valid_cells) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27" ); } } // namespace pooling } // namespace arm_conv + #endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_avg_generic_depthfirst/generic.cpp index e889782fa30d60861a327ae7ca7e5621bc971f60..019f402911cf9a3f24296609a253134ea34e122c 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_avg_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_avg_generic_depthfirst/generic.cpp @@ -22,8 +22,6 @@ * SOFTWARE. */ -#if defined(__aarch64__) - #include "pooling.hpp" #include #include @@ -31,6 +29,8 @@ #include +#if defined(__aarch64__) + namespace arm_conv { namespace pooling { @@ -124,7 +124,7 @@ void a64_s8q_nhwc_avg_generic_depthfirst_impl( "lsr x23, %x[n_valid_cells], #0x1\n" "movi v15.4s, #0x0\n" "movi v14.4s, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "movi v13.4s, #0x0\n" "movi v12.4s, #0x0\n" "movi v11.4s, #0x0\n" @@ -140,42 +140,42 @@ void a64_s8q_nhwc_avg_generic_depthfirst_impl( "movi v1.4s, #0x0\n" "movi v0.4s, #0x0\n" "cbz x23, 4f\n" - "ldp x22, x21, [x20, #0x0]\n" - "ldr q31, [x22, x27]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q31, [x21, x27]\n" "subs x23, x23, #0x1\n" - "add x20, x20, #0x10\n" - "ldr q30, [x21, x27]\n" - "ldr q29, [x22, x26]\n" - "ldr q28, [x21, x26]\n" - "ldr q27, [x22, x25]\n" - "ldr q26, [x21, x25]\n" - "ldr q25, [x22, x24]\n" - "ldr q24, [x21, x24]\n" + "add x22, x22, #0x10\n" + "ldr q30, [x20, x27]\n" + "ldr q29, [x21, x26]\n" + "ldr q28, [x20, x26]\n" + "ldr q27, [x21, x25]\n" + "ldr q26, [x20, x25]\n" + "ldr q25, [x21, x24]\n" + "ldr q24, [x20, x24]\n" "beq 3f\n" "2:" // 4-vectors of channels: 2 inputs loop "saddl v23.8h, v31.8b, v30.8b\n" "saddl2 v22.8h, v31.16b, v30.16b\n" - "ldp x22, x21, [x20, #0x0]\n" - "ldr q31, [x22, x27]\n" - "ldr q30, [x21, x27]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q31, [x21, x27]\n" + "ldr q30, [x20, x27]\n" "saddl v21.8h, v29.8b, v28.8b\n" "saddl2 v20.8h, v29.16b, v28.16b\n" - "ldr q29, [x22, x26]\n" - "ldr q28, [x21, x26]\n" + "ldr q29, [x21, x26]\n" + "ldr q28, [x20, x26]\n" "saddl v19.8h, v27.8b, v26.8b\n" "saddl2 v18.8h, v27.16b, v26.16b\n" - "ldr q27, [x22, x25]\n" - "ldr q26, [x21, x25]\n" + "ldr q27, [x21, x25]\n" + "ldr q26, [x20, x25]\n" + "saddl v17.8h, v25.8b, v24.8b\n" + "saddl2 v16.8h, v25.16b, v24.16b\n" + "ldr q25, [x21, x24]\n" + "ldr q24, [x20, x24]\n" "subs x23, x23, #0x1\n" "saddw v15.4s, v15.4s, v23.4h\n" "saddw2 v14.4s, v14.4s, v23.8h\n" - "saddl v17.8h, v25.8b, v24.8b\n" - "saddl2 v16.8h, v25.16b, v24.16b\n" - "ldr q25, [x22, x24]\n" - "add x20, x20, #0x10\n" "saddw v13.4s, v13.4s, v22.4h\n" "saddw2 v12.4s, v12.4s, v22.8h\n" - "ldr q24, [x21, x24]\n" + "add x22, x22, #0x10\n" "saddw v11.4s, v11.4s, v21.4h\n" "saddw2 v10.4s, v10.4s, v21.8h\n" "saddw v9.4s, v9.4s, v20.4h\n" @@ -215,23 +215,23 @@ void a64_s8q_nhwc_avg_generic_depthfirst_impl( "saddw v1.4s, v1.4s, v16.4h\n" "saddw2 v0.4s, v0.4s, v16.8h\n" "4:" // 4-vectors of channels: After loop - "ands x21, %x[n_valid_cells], #0x1\n" + "ands x23, %x[n_valid_cells], #0x1\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x22, [x20], #0x8\n" - "ldr q31, [x22, x27]\n" - "sxtl v23.8h, v31.8b\n" - "sxtl2 v22.8h, v31.16b\n" - "ldr q29, [x22, x26]\n" - "ldr q27, [x22, x25]\n" - "sxtl v21.8h, v29.8b\n" - "sxtl2 v20.8h, v29.16b\n" - "ldr q25, [x22, x24]\n" - "sxtl v19.8h, v27.8b\n" - "sxtl2 v18.8h, v27.16b\n" - "subs x21, x21, #0x1\n" - "sxtl v17.8h, v25.8b\n" - "sxtl2 v16.8h, v25.16b\n" + "ldr x20, [x22], #0x8\n" + "ldr q16, [x20, x27]\n" + "sxtl v23.8h, v16.8b\n" + "sxtl2 v22.8h, v16.16b\n" + "ldr q16, [x20, x26]\n" + "ldr q17, [x20, x25]\n" + "sxtl v21.8h, v16.8b\n" + "sxtl2 v20.8h, v16.16b\n" + "ldr q16, [x20, x24]\n" + "sxtl v19.8h, v17.8b\n" + "sxtl2 v18.8h, v17.16b\n" + "subs x23, x23, #0x1\n" + "sxtl v17.8h, v16.8b\n" + "sxtl2 v16.8h, v16.16b\n" "saddw v15.4s, v15.4s, v23.4h\n" "saddw2 v14.4s, v14.4s, v23.8h\n" "saddw v13.4s, v13.4s, v22.4h\n" @@ -366,49 +366,49 @@ void a64_s8q_nhwc_avg_generic_depthfirst_impl( "lsr x23, %x[n_valid_cells], #0x1\n" "movi v15.4s, #0x0\n" "movi v14.4s, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "movi v13.4s, #0x0\n" "movi v12.4s, #0x0\n" "cbz x23, 11f\n" - "ldp x22, x21, [x20, #0x0]\n" - "ldr q31, [x22, x27]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q31, [x21, x27]\n" "subs x23, x23, #0x1\n" - "add x20, x20, #0x10\n" - "ldr q30, [x21, x27]\n" + "add x22, x22, #0x10\n" + "ldr q30, [x20, x27]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 2 inputs loop - "saddl v23.8h, v31.8b, v30.8b\n" - "saddl2 v22.8h, v31.16b, v30.16b\n" - "ldp x22, x21, [x20, #0x0]\n" - "ldr q31, [x22, x27]\n" - "ldr q30, [x21, x27]\n" + "saddl v17.8h, v31.8b, v30.8b\n" + "saddl2 v16.8h, v31.16b, v30.16b\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q31, [x21, x27]\n" + "ldr q30, [x20, x27]\n" "subs x23, x23, #0x1\n" - "saddw v15.4s, v15.4s, v23.4h\n" - "saddw2 v14.4s, v14.4s, v23.8h\n" - "saddw v13.4s, v13.4s, v22.4h\n" - "saddw2 v12.4s, v12.4s, v22.8h\n" - "add x20, x20, #0x10\n" + "saddw v15.4s, v15.4s, v17.4h\n" + "saddw2 v14.4s, v14.4s, v17.8h\n" + "saddw v13.4s, v13.4s, v16.4h\n" + "saddw2 v12.4s, v12.4s, v16.8h\n" + "add x22, x22, #0x10\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 2 inputs tail - "saddl v23.8h, v31.8b, v30.8b\n" - "saddl2 v22.8h, v31.16b, v30.16b\n" - "saddw v15.4s, v15.4s, v23.4h\n" - "saddw2 v14.4s, v14.4s, v23.8h\n" - "saddw v13.4s, v13.4s, v22.4h\n" - "saddw2 v12.4s, v12.4s, v22.8h\n" + "saddl v17.8h, v31.8b, v30.8b\n" + "saddl2 v16.8h, v31.16b, v30.16b\n" + "saddw v15.4s, v15.4s, v17.4h\n" + "saddw2 v14.4s, v14.4s, v17.8h\n" + "saddw v13.4s, v13.4s, v16.4h\n" + "saddw2 v12.4s, v12.4s, v16.8h\n" "11:" // Single vector of channels: Loop: After loop - "ands x21, %x[n_valid_cells], #0x1\n" + "ands x23, %x[n_valid_cells], #0x1\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x22, [x20], #0x8\n" - "ldr q31, [x22, x27]\n" - "sxtl v23.8h, v31.8b\n" - "sxtl2 v22.8h, v31.16b\n" - "subs x21, x21, #0x1\n" - "saddw v15.4s, v15.4s, v23.4h\n" - "saddw2 v14.4s, v14.4s, v23.8h\n" - "saddw v13.4s, v13.4s, v22.4h\n" - "saddw2 v12.4s, v12.4s, v22.8h\n" + "ldr x20, [x22], #0x8\n" + "ldr q16, [x20, x27]\n" + "sxtl v17.8h, v16.8b\n" + "sxtl2 v16.8h, v16.16b\n" + "subs x23, x23, #0x1\n" + "saddw v15.4s, v15.4s, v17.4h\n" + "saddw2 v14.4s, v14.4s, v17.8h\n" + "saddw v13.4s, v13.4s, v16.4h\n" + "saddw2 v12.4s, v12.4s, v16.8h\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "ld1r { v18.4s }, [%x[left_shift]]\n" @@ -438,9 +438,9 @@ void a64_s8q_nhwc_avg_generic_depthfirst_impl( "smin v14.4s, v14.4s, v17.4s\n" "smin v13.4s, v13.4s, v17.4s\n" "smin v12.4s, v12.4s, v17.4s\n" - "uzp1 v23.16b, v15.16b, v14.16b\n" + "uzp1 v17.16b, v15.16b, v14.16b\n" "uzp1 v16.16b, v13.16b, v12.16b\n" - "uzp1 v16.16b, v23.16b, v16.16b\n" + "uzp1 v16.16b, v17.16b, v16.16b\n" "str q16, [%x[outptr], x27]\n" "add x27, x27, #0x10\n" "bge 8b\n" @@ -452,142 +452,142 @@ void a64_s8q_nhwc_avg_generic_depthfirst_impl( "movi v14.4s, #0x0\n" "movi v13.4s, #0x0\n" "movi v12.4s, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "cbz x23, 24f\n" "15:" // Oddments: 2 inputs loop - "ldp x22, x21, [x20, #0x0]\n" - "add x20, x20, #0x10\n" - "add x22, x22, x27\n" - "movi v31.16b, #0x0\n" + "ldp x21, x20, [x22, #0x0]\n" + "add x22, x22, #0x10\n" "add x21, x21, x27\n" + "movi v31.16b, #0x0\n" + "add x20, x20, x27\n" "movi v30.16b, #0x0\n" "tbz %x[n_channels], #3, 19f\n" - "ldr d31, [x22], #0x8\n" - "ldr d30, [x21], #0x8\n" + "ldr d31, [x21], #0x8\n" + "ldr d30, [x20], #0x8\n" "tbz %x[n_channels], #2, 17f\n" - "ld1 { v31.s }[2], [x22], #0x4\n" - "ld1 { v30.s }[2], [x21], #0x4\n" + "ld1 { v31.s }[2], [x21], #0x4\n" + "ld1 { v30.s }[2], [x20], #0x4\n" "tbz %x[n_channels], #1, 16f\n" - "ld1 { v31.h }[6], [x22], #0x2\n" - "ld1 { v30.h }[6], [x21], #0x2\n" + "ld1 { v31.h }[6], [x21], #0x2\n" + "ld1 { v30.h }[6], [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[14], [x22], #0x1\n" - "ld1 { v30.b }[14], [x21], #0x1\n" + "ld1 { v31.b }[14], [x21], #0x1\n" + "ld1 { v30.b }[14], [x20], #0x1\n" "b 23f\n" "16:" // Oddments: 2 inputs loop: Load: Bit 3: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[12], [x22], #0x1\n" - "ld1 { v30.b }[12], [x21], #0x1\n" + "ld1 { v31.b }[12], [x21], #0x1\n" + "ld1 { v30.b }[12], [x20], #0x1\n" "b 23f\n" "17:" // Oddments: 2 inputs loop: Load: Bit 3: Bit 2: Unset "tbz %x[n_channels], #1, 18f\n" - "ld1 { v31.h }[4], [x22], #0x2\n" - "ld1 { v30.h }[4], [x21], #0x2\n" + "ld1 { v31.h }[4], [x21], #0x2\n" + "ld1 { v30.h }[4], [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[10], [x22], #0x1\n" - "ld1 { v30.b }[10], [x21], #0x1\n" + "ld1 { v31.b }[10], [x21], #0x1\n" + "ld1 { v30.b }[10], [x20], #0x1\n" "b 23f\n" "18:" // Oddments: 2 inputs loop: Load: Bit 3: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[8], [x22], #0x1\n" - "ld1 { v30.b }[8], [x21], #0x1\n" + "ld1 { v31.b }[8], [x21], #0x1\n" + "ld1 { v30.b }[8], [x20], #0x1\n" "b 23f\n" "19:" // Oddments: 2 inputs loop: Load: Bit 3: Unset "tbz %x[n_channels], #2, 21f\n" - "ldr s31, [x22], #0x4\n" - "ldr s30, [x21], #0x4\n" + "ldr s31, [x21], #0x4\n" + "ldr s30, [x20], #0x4\n" "tbz %x[n_channels], #1, 20f\n" - "ld1 { v31.h }[2], [x22], #0x2\n" - "ld1 { v30.h }[2], [x21], #0x2\n" + "ld1 { v31.h }[2], [x21], #0x2\n" + "ld1 { v30.h }[2], [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[6], [x22], #0x1\n" - "ld1 { v30.b }[6], [x21], #0x1\n" + "ld1 { v31.b }[6], [x21], #0x1\n" + "ld1 { v30.b }[6], [x20], #0x1\n" "b 23f\n" "20:" // Oddments: 2 inputs loop: Load: Bit 3: Unset: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[4], [x22], #0x1\n" - "ld1 { v30.b }[4], [x21], #0x1\n" + "ld1 { v31.b }[4], [x21], #0x1\n" + "ld1 { v30.b }[4], [x20], #0x1\n" "b 23f\n" "21:" // Oddments: 2 inputs loop: Load: Bit 3: Unset: Bit 2: Unset "tbz %x[n_channels], #1, 22f\n" - "ldr h31, [x22], #0x2\n" - "ldr h30, [x21], #0x2\n" + "ldr h31, [x21], #0x2\n" + "ldr h30, [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[2], [x22], #0x1\n" - "ld1 { v30.b }[2], [x21], #0x1\n" + "ld1 { v31.b }[2], [x21], #0x1\n" + "ld1 { v30.b }[2], [x20], #0x1\n" "b 23f\n" "22:" // Oddments: 2 inputs loop: Load: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ldr b31, [x22], #0x1\n" - "ldr b30, [x21], #0x1\n" + "ldr b31, [x21], #0x1\n" + "ldr b30, [x20], #0x1\n" "23:" // Oddments: 2 inputs loop: Load: Bit 3: End - "saddl v23.8h, v31.8b, v30.8b\n" - "saddl2 v22.8h, v31.16b, v30.16b\n" + "saddl v17.8h, v31.8b, v30.8b\n" + "saddl2 v16.8h, v31.16b, v30.16b\n" "subs x23, x23, #0x1\n" - "saddw v15.4s, v15.4s, v23.4h\n" - "saddw2 v14.4s, v14.4s, v23.8h\n" - "saddw v13.4s, v13.4s, v22.4h\n" - "saddw2 v12.4s, v12.4s, v22.8h\n" + "saddw v15.4s, v15.4s, v17.4h\n" + "saddw2 v14.4s, v14.4s, v17.8h\n" + "saddw v13.4s, v13.4s, v16.4h\n" + "saddw2 v12.4s, v12.4s, v16.8h\n" "bgt 15b\n" "24:" // Oddments: After loop - "ands x21, %x[n_valid_cells], #0x1\n" + "ands x23, %x[n_valid_cells], #0x1\n" "beq 34f\n" "25:" // Oddments: Single input loop - "ldr x22, [x20], #0x8\n" - "add x22, x22, x27\n" + "ldr x21, [x22], #0x8\n" + "add x21, x21, x27\n" "movi v31.16b, #0x0\n" "tbz %x[n_channels], #3, 29f\n" - "ldr d31, [x22], #0x8\n" + "ldr d31, [x21], #0x8\n" "tbz %x[n_channels], #2, 27f\n" - "ld1 { v31.s }[2], [x22], #0x4\n" + "ld1 { v31.s }[2], [x21], #0x4\n" "tbz %x[n_channels], #1, 26f\n" - "ld1 { v31.h }[6], [x22], #0x2\n" + "ld1 { v31.h }[6], [x21], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[14], [x22], #0x1\n" + "ld1 { v31.b }[14], [x21], #0x1\n" "b 33f\n" "26:" // Oddments: Single input loop: Load: Bit 3: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[12], [x22], #0x1\n" + "ld1 { v31.b }[12], [x21], #0x1\n" "b 33f\n" "27:" // Oddments: Single input loop: Load: Bit 3: Bit 2: Unset "tbz %x[n_channels], #1, 28f\n" - "ld1 { v31.h }[4], [x22], #0x2\n" + "ld1 { v31.h }[4], [x21], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[10], [x22], #0x1\n" + "ld1 { v31.b }[10], [x21], #0x1\n" "b 33f\n" "28:" // Oddments: Single input loop: Load: Bit 3: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[8], [x22], #0x1\n" + "ld1 { v31.b }[8], [x21], #0x1\n" "b 33f\n" "29:" // Oddments: Single input loop: Load: Bit 3: Unset "tbz %x[n_channels], #2, 31f\n" - "ldr s31, [x22], #0x4\n" + "ldr s31, [x21], #0x4\n" "tbz %x[n_channels], #1, 30f\n" - "ld1 { v31.h }[2], [x22], #0x2\n" + "ld1 { v31.h }[2], [x21], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[6], [x22], #0x1\n" + "ld1 { v31.b }[6], [x21], #0x1\n" "b 33f\n" "30:" // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[4], [x22], #0x1\n" + "ld1 { v31.b }[4], [x21], #0x1\n" "b 33f\n" "31:" // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Unset "tbz %x[n_channels], #1, 32f\n" - "ldr h31, [x22], #0x2\n" + "ldr h31, [x21], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[2], [x22], #0x1\n" + "ld1 { v31.b }[2], [x21], #0x1\n" "b 33f\n" "32:" // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ldr b31, [x22], #0x1\n" + "ldr b31, [x21], #0x1\n" "33:" // Oddments: Single input loop: Load: Bit 3: End - "sxtl v23.8h, v31.8b\n" - "sxtl2 v22.8h, v31.16b\n" - "subs x21, x21, #0x1\n" - "saddw v15.4s, v15.4s, v23.4h\n" - "saddw2 v14.4s, v14.4s, v23.8h\n" - "saddw v13.4s, v13.4s, v22.4h\n" - "saddw2 v12.4s, v12.4s, v22.8h\n" + "sxtl v17.8h, v31.8b\n" + "sxtl2 v16.8h, v31.16b\n" + "subs x23, x23, #0x1\n" + "saddw v15.4s, v15.4s, v17.4h\n" + "saddw2 v14.4s, v14.4s, v17.8h\n" + "saddw v13.4s, v13.4s, v16.4h\n" + "saddw2 v12.4s, v12.4s, v16.8h\n" "bgt 25b\n" "34:" // Oddments: Single input loop: End "ld1r { v18.4s }, [%x[left_shift]]\n" @@ -615,9 +615,9 @@ void a64_s8q_nhwc_avg_generic_depthfirst_impl( "smin v14.4s, v14.4s, v17.4s\n" "smin v13.4s, v13.4s, v17.4s\n" "smin v12.4s, v12.4s, v17.4s\n" - "uzp1 v23.16b, v15.16b, v14.16b\n" + "uzp1 v17.16b, v15.16b, v14.16b\n" "uzp1 v16.16b, v13.16b, v12.16b\n" - "uzp1 v16.16b, v23.16b, v16.16b\n" + "uzp1 v16.16b, v17.16b, v16.16b\n" "tbz %x[n_channels], #3, 38f\n" "st1 { v16.d }[0], [%x[outptr]], #0x8\n" "tbz %x[n_channels], #2, 36f\n" @@ -672,4 +672,5 @@ void a64_s8q_nhwc_avg_generic_depthfirst_impl( } // namespace pooling } // namespace arm_conv + #endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_max_generic_depthfirst/generic.cpp index 90a31ec6773cb8637e52d4f87f021e100b65821f..f7b8dc761cf0b2b5524282d46cf42b5b89660997 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_max_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_max_generic_depthfirst/generic.cpp @@ -21,12 +21,13 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#if defined(__aarch64__) #include "pooling.hpp" #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace pooling { @@ -42,77 +43,77 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl( { __asm__ __volatile__( "cmp %x[n_channels], #0x40\n" - "mov x9, #0x0\n" - "mov x28, #0x10\n" // cntb _, ALL, #1 - "mov x27, #0x20\n" // cntb _, ALL, #2 - "mov x26, #0x30\n" // cntb _, ALL, #3 + "mov x27, #0x0\n" + "mov x26, #0x10\n" // cntb _, ALL, #1 + "mov x24, #0x20\n" // cntb _, ALL, #2 + "mov x23, #0x30\n" // cntb _, ALL, #3 "blt 7f\n" "1:" // 4-vectors of channels "lsr x25, %x[n_valid_cells], #0x2\n" "movi v8.16b, #0x80\n" "movi v7.16b, #0x80\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "movi v6.16b, #0x80\n" "movi v5.16b, #0x80\n" "cbz x25, 4f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" "subs x25, x25, #0x1\n" - "ldr q3, [x23, x9]\n" - "ldr q2, [x24, x28]\n" - "ldr q1, [x23, x28]\n" - "ldr q0, [x24, x27]\n" - "ldr q31, [x23, x27]\n" - "ldr q30, [x24, x26]\n" - "ldr q29, [x23, x26]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" - "ldr q27, [x22, x28]\n" - "ldr q21, [x21, x28]\n" - "ldr q26, [x22, x27]\n" - "ldr q20, [x21, x27]\n" - "ldr q25, [x22, x26]\n" - "ldr q24, [x21, x26]\n" + "ldr q3, [x20, x27]\n" + "ldr q2, [x21, x26]\n" + "ldr q1, [x20, x26]\n" + "ldr q0, [x21, x24]\n" + "ldr q31, [x20, x24]\n" + "ldr q30, [x21, x23]\n" + "ldr q29, [x20, x23]\n" + "ldp x21, x20, [x22, #0x10]\n" + "add x22, x22, #0x20\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" + "ldr q27, [x21, x26]\n" + "ldr q21, [x20, x26]\n" + "ldr q26, [x21, x24]\n" + "ldr q20, [x20, x24]\n" + "ldr q25, [x21, x23]\n" + "ldr q24, [x20, x23]\n" "beq 3f\n" "2:" // 4-vectors of channels: 4 inputs loop "smax v23.16b, v4.16b, v3.16b\n" "smax v19.16b, v28.16b, v22.16b\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" - "ldr q3, [x23, x9]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" + "ldr q3, [x20, x27]\n" "smax v22.16b, v2.16b, v1.16b\n" - "ldr q2, [x24, x28]\n" + "ldr q2, [x21, x26]\n" "smax v18.16b, v27.16b, v21.16b\n" - "ldr q1, [x23, x28]\n" + "ldr q1, [x20, x26]\n" "smax v21.16b, v0.16b, v31.16b\n" - "ldr q0, [x24, x27]\n" + "ldr q0, [x21, x24]\n" "smax v17.16b, v26.16b, v20.16b\n" - "ldr q31, [x23, x27]\n" + "ldr q31, [x20, x24]\n" "smax v20.16b, v30.16b, v29.16b\n" - "ldr q30, [x24, x26]\n" + "ldr q30, [x21, x23]\n" "smax v16.16b, v25.16b, v24.16b\n" - "ldr q29, [x23, x26]\n" + "ldr q29, [x20, x23]\n" "smax v19.16b, v23.16b, v19.16b\n" "smax v18.16b, v22.16b, v18.16b\n" - "ldp x22, x21, [x20, #0x10]\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" + "ldp x21, x20, [x22, #0x10]\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" "smax v17.16b, v21.16b, v17.16b\n" "smax v16.16b, v20.16b, v16.16b\n" - "ldr q27, [x22, x28]\n" - "ldr q21, [x21, x28]\n" + "ldr q27, [x21, x26]\n" + "ldr q21, [x20, x26]\n" "subs x25, x25, #0x1\n" "smax v8.16b, v8.16b, v19.16b\n" - "ldr q26, [x22, x27]\n" - "ldr q20, [x21, x27]\n" + "ldr q26, [x21, x24]\n" + "ldr q20, [x20, x24]\n" "smax v7.16b, v7.16b, v18.16b\n" "smax v6.16b, v6.16b, v17.16b\n" - "ldr q25, [x22, x26]\n" - "ldr q24, [x21, x26]\n" + "ldr q25, [x21, x23]\n" + "ldr q24, [x20, x23]\n" "smax v5.16b, v5.16b, v16.16b\n" - "add x20, x20, #0x20\n" + "add x22, x22, #0x20\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "smax v23.16b, v4.16b, v3.16b\n" @@ -135,16 +136,16 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x24, [x20], #0x8\n" - "ldr q4, [x24, x9]\n" + "ldr x20, [x22], #0x8\n" + "ldr q16, [x20, x27]\n" "subs x21, x21, #0x1\n" - "smax v8.16b, v8.16b, v4.16b\n" - "ldr q2, [x24, x28]\n" - "ldr q0, [x24, x27]\n" - "smax v7.16b, v7.16b, v2.16b\n" - "smax v6.16b, v6.16b, v0.16b\n" - "ldr q30, [x24, x26]\n" - "smax v5.16b, v5.16b, v30.16b\n" + "smax v8.16b, v8.16b, v16.16b\n" + "ldr q17, [x20, x26]\n" + "ldr q16, [x20, x24]\n" + "smax v7.16b, v7.16b, v17.16b\n" + "smax v6.16b, v6.16b, v16.16b\n" + "ldr q16, [x20, x23]\n" + "smax v5.16b, v5.16b, v16.16b\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End "sxtl v23.8h, v8.8b\n" @@ -271,16 +272,16 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl( "uzp1 v19.16b, v24.16b, v19.16b\n" "uzp1 v16.16b, v23.16b, v16.16b\n" "uzp1 v18.16b, v22.16b, v18.16b\n" - "str q16, [%x[outptr], x9]\n" - "add x9, x9, #0x40\n" + "str q16, [%x[outptr], x27]\n" + "add x27, x27, #0x40\n" "uzp1 v17.16b, v21.16b, v17.16b\n" "uzp1 v16.16b, v20.16b, v19.16b\n" - "str q18, [%x[outptr], x28]\n" - "add x28, x28, #0x40\n" - "str q17, [%x[outptr], x27]\n" - "add x27, x27, #0x40\n" - "str q16, [%x[outptr], x26]\n" + "str q18, [%x[outptr], x26]\n" "add x26, x26, #0x40\n" + "str q17, [%x[outptr], x24]\n" + "add x24, x24, #0x40\n" + "str q16, [%x[outptr], x23]\n" + "add x23, x23, #0x40\n" "bge 1b\n" "cbz %x[n_channels], 43f\n" "7:" // Single vector of channels @@ -289,296 +290,296 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl( "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" "movi v8.16b, #0x80\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" "subs x25, x25, #0x1\n" - "ldr q3, [x23, x9]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" + "ldr q3, [x20, x27]\n" + "ldp x21, x20, [x22, #0x10]\n" + "add x22, x22, #0x20\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 4 inputs loop - "smax v23.16b, v4.16b, v3.16b\n" - "smax v19.16b, v28.16b, v22.16b\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" - "ldr q3, [x23, x9]\n" - "smax v19.16b, v23.16b, v19.16b\n" - "ldp x22, x21, [x20, #0x10]\n" + "smax v17.16b, v4.16b, v3.16b\n" + "smax v16.16b, v28.16b, v22.16b\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" + "ldr q3, [x20, x27]\n" + "smax v16.16b, v17.16b, v16.16b\n" + "ldp x21, x20, [x22, #0x10]\n" "subs x25, x25, #0x1\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" - "smax v8.16b, v8.16b, v19.16b\n" - "add x20, x20, #0x20\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" + "smax v8.16b, v8.16b, v16.16b\n" + "add x22, x22, #0x20\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail - "smax v23.16b, v4.16b, v3.16b\n" - "smax v19.16b, v28.16b, v22.16b\n" - "smax v19.16b, v23.16b, v19.16b\n" - "smax v8.16b, v8.16b, v19.16b\n" + "smax v17.16b, v4.16b, v3.16b\n" + "smax v16.16b, v28.16b, v22.16b\n" + "smax v16.16b, v17.16b, v16.16b\n" + "smax v8.16b, v8.16b, v16.16b\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x24, [x20], #0x8\n" - "ldr q4, [x24, x9]\n" + "ldr x20, [x22], #0x8\n" + "ldr q16, [x20, x27]\n" "subs x21, x21, #0x1\n" - "smax v8.16b, v8.16b, v4.16b\n" + "smax v8.16b, v8.16b, v16.16b\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End - "sxtl v23.8h, v8.8b\n" - "sxtl2 v22.8h, v8.16b\n" + "sxtl v17.8h, v8.8b\n" + "sxtl2 v16.8h, v8.16b\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n" - "ld1r { v4.4s }, [x20]\n" - "sxtl v1.4s, v23.4h\n" - "sxtl2 v23.4s, v23.8h\n" + "ld1r { v22.4s }, [x20]\n" + "sxtl v21.4s, v17.4h\n" + "sxtl2 v20.4s, v17.8h\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_mul]\n" - "ld1r { v3.4s }, [x20]\n" - "sxtl v0.4s, v22.4h\n" - "sxtl2 v31.4s, v22.8h\n" + "ld1r { v17.4s }, [x20]\n" + "sxtl v19.4s, v16.4h\n" + "sxtl2 v18.4s, v16.8h\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n" - "ld1r { v2.4s }, [x20]\n" - "srshl v1.4s, v1.4s, v4.4s\n" - "srshl v23.4s, v23.4s, v4.4s\n" + "ld1r { v16.4s }, [x20]\n" + "srshl v21.4s, v21.4s, v22.4s\n" + "srshl v20.4s, v20.4s, v22.4s\n" "sub %x[n_channels], %x[n_channels], #0x10\n" "cmp %x[n_channels], #0x10\n" - "srshl v0.4s, v0.4s, v4.4s\n" - "srshl v31.4s, v31.4s, v4.4s\n" - "sqrdmulh v1.4s, v1.4s, v3.4s\n" - "sqrdmulh v23.4s, v23.4s, v3.4s\n" - "sqrdmulh v0.4s, v0.4s, v3.4s\n" - "sqrdmulh v31.4s, v31.4s, v3.4s\n" + "srshl v19.4s, v19.4s, v22.4s\n" + "srshl v18.4s, v18.4s, v22.4s\n" + "sqrdmulh v21.4s, v21.4s, v17.4s\n" + "sqrdmulh v20.4s, v20.4s, v17.4s\n" + "sqrdmulh v19.4s, v19.4s, v17.4s\n" + "sqrdmulh v18.4s, v18.4s, v17.4s\n" "movi v17.4s, #0x7f\n" - "srshl v1.4s, v1.4s, v2.4s\n" - "srshl v23.4s, v23.4s, v2.4s\n" - "srshl v0.4s, v0.4s, v2.4s\n" - "srshl v31.4s, v31.4s, v2.4s\n" + "srshl v21.4s, v21.4s, v16.4s\n" + "srshl v20.4s, v20.4s, v16.4s\n" + "srshl v19.4s, v19.4s, v16.4s\n" + "srshl v18.4s, v18.4s, v16.4s\n" "not v16.16b, v17.16b\n" - "smax v1.4s, v1.4s, v16.4s\n" - "smax v23.4s, v23.4s, v16.4s\n" - "smax v0.4s, v0.4s, v16.4s\n" - "smax v31.4s, v31.4s, v16.4s\n" - "smin v1.4s, v1.4s, v17.4s\n" - "smin v23.4s, v23.4s, v17.4s\n" - "smin v0.4s, v0.4s, v17.4s\n" - "smin v31.4s, v31.4s, v17.4s\n" - "uzp1 v23.16b, v1.16b, v23.16b\n" - "uzp1 v16.16b, v0.16b, v31.16b\n" - "uzp1 v16.16b, v23.16b, v16.16b\n" - "str q16, [%x[outptr], x9]\n" - "add x9, x9, #0x10\n" + "smax v21.4s, v21.4s, v16.4s\n" + "smax v20.4s, v20.4s, v16.4s\n" + "smax v19.4s, v19.4s, v16.4s\n" + "smax v18.4s, v18.4s, v16.4s\n" + "smin v21.4s, v21.4s, v17.4s\n" + "smin v20.4s, v20.4s, v17.4s\n" + "smin v19.4s, v19.4s, v17.4s\n" + "smin v18.4s, v18.4s, v17.4s\n" + "uzp1 v17.16b, v21.16b, v20.16b\n" + "uzp1 v16.16b, v19.16b, v18.16b\n" + "uzp1 v16.16b, v17.16b, v16.16b\n" + "str q16, [%x[outptr], x27]\n" + "add x27, x27, #0x10\n" "bge 8b\n" "cbz %x[n_channels], 43f\n" "14:" // Oddments "lsr x25, %x[n_valid_cells], #0x2\n" - "add %x[outptr], %x[outptr], x9\n" + "add %x[outptr], %x[outptr], x27\n" "movi v8.16b, #0x80\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "cbz x25, 24f\n" "15:" // Oddments: 4 inputs loop - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "add x24, x24, x9\n" - "add x23, x23, x9\n" - "add x22, x22, x9\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" + "add x23, x23, x27\n" + "add x22, x22, x27\n" + "add x21, x21, x27\n" "movi v4.16b, #0x0\n" "movi v3.16b, #0x0\n" - "add x21, x21, x9\n" + "add x20, x20, x27\n" "movi v28.16b, #0x0\n" "movi v22.16b, #0x0\n" "tbz %x[n_channels], #3, 19f\n" - "ldr d4, [x24], #0x8\n" - "ldr d3, [x23], #0x8\n" - "ldr d28, [x22], #0x8\n" - "ldr d22, [x21], #0x8\n" + "ldr d4, [x23], #0x8\n" + "ldr d3, [x22], #0x8\n" + "ldr d28, [x21], #0x8\n" + "ldr d22, [x20], #0x8\n" "tbz %x[n_channels], #2, 17f\n" - "ld1 { v4.s }[2], [x24], #0x4\n" - "ld1 { v3.s }[2], [x23], #0x4\n" - "ld1 { v28.s }[2], [x22], #0x4\n" - "ld1 { v22.s }[2], [x21], #0x4\n" + "ld1 { v4.s }[2], [x23], #0x4\n" + "ld1 { v3.s }[2], [x22], #0x4\n" + "ld1 { v28.s }[2], [x21], #0x4\n" + "ld1 { v22.s }[2], [x20], #0x4\n" "tbz %x[n_channels], #1, 16f\n" - "ld1 { v4.h }[6], [x24], #0x2\n" - "ld1 { v3.h }[6], [x23], #0x2\n" - "ld1 { v28.h }[6], [x22], #0x2\n" - "ld1 { v22.h }[6], [x21], #0x2\n" + "ld1 { v4.h }[6], [x23], #0x2\n" + "ld1 { v3.h }[6], [x22], #0x2\n" + "ld1 { v28.h }[6], [x21], #0x2\n" + "ld1 { v22.h }[6], [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[14], [x24], #0x1\n" - "ld1 { v3.b }[14], [x23], #0x1\n" - "ld1 { v28.b }[14], [x22], #0x1\n" - "ld1 { v22.b }[14], [x21], #0x1\n" + "ld1 { v4.b }[14], [x23], #0x1\n" + "ld1 { v3.b }[14], [x22], #0x1\n" + "ld1 { v28.b }[14], [x21], #0x1\n" + "ld1 { v22.b }[14], [x20], #0x1\n" "b 23f\n" "16:" // Oddments: 4 inputs loop: Load: Bit 3: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[12], [x24], #0x1\n" - "ld1 { v3.b }[12], [x23], #0x1\n" - "ld1 { v28.b }[12], [x22], #0x1\n" - "ld1 { v22.b }[12], [x21], #0x1\n" + "ld1 { v4.b }[12], [x23], #0x1\n" + "ld1 { v3.b }[12], [x22], #0x1\n" + "ld1 { v28.b }[12], [x21], #0x1\n" + "ld1 { v22.b }[12], [x20], #0x1\n" "b 23f\n" "17:" // Oddments: 4 inputs loop: Load: Bit 3: Bit 2: Unset "tbz %x[n_channels], #1, 18f\n" - "ld1 { v4.h }[4], [x24], #0x2\n" - "ld1 { v3.h }[4], [x23], #0x2\n" - "ld1 { v28.h }[4], [x22], #0x2\n" - "ld1 { v22.h }[4], [x21], #0x2\n" + "ld1 { v4.h }[4], [x23], #0x2\n" + "ld1 { v3.h }[4], [x22], #0x2\n" + "ld1 { v28.h }[4], [x21], #0x2\n" + "ld1 { v22.h }[4], [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[10], [x24], #0x1\n" - "ld1 { v3.b }[10], [x23], #0x1\n" - "ld1 { v28.b }[10], [x22], #0x1\n" - "ld1 { v22.b }[10], [x21], #0x1\n" + "ld1 { v4.b }[10], [x23], #0x1\n" + "ld1 { v3.b }[10], [x22], #0x1\n" + "ld1 { v28.b }[10], [x21], #0x1\n" + "ld1 { v22.b }[10], [x20], #0x1\n" "b 23f\n" "18:" // Oddments: 4 inputs loop: Load: Bit 3: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[8], [x24], #0x1\n" - "ld1 { v3.b }[8], [x23], #0x1\n" - "ld1 { v28.b }[8], [x22], #0x1\n" - "ld1 { v22.b }[8], [x21], #0x1\n" + "ld1 { v4.b }[8], [x23], #0x1\n" + "ld1 { v3.b }[8], [x22], #0x1\n" + "ld1 { v28.b }[8], [x21], #0x1\n" + "ld1 { v22.b }[8], [x20], #0x1\n" "b 23f\n" "19:" // Oddments: 4 inputs loop: Load: Bit 3: Unset "tbz %x[n_channels], #2, 21f\n" - "ldr s4, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr s28, [x22], #0x4\n" - "ldr s22, [x21], #0x4\n" + "ldr s4, [x23], #0x4\n" + "ldr s3, [x22], #0x4\n" + "ldr s28, [x21], #0x4\n" + "ldr s22, [x20], #0x4\n" "tbz %x[n_channels], #1, 20f\n" - "ld1 { v4.h }[2], [x24], #0x2\n" - "ld1 { v3.h }[2], [x23], #0x2\n" - "ld1 { v28.h }[2], [x22], #0x2\n" - "ld1 { v22.h }[2], [x21], #0x2\n" + "ld1 { v4.h }[2], [x23], #0x2\n" + "ld1 { v3.h }[2], [x22], #0x2\n" + "ld1 { v28.h }[2], [x21], #0x2\n" + "ld1 { v22.h }[2], [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[6], [x24], #0x1\n" - "ld1 { v3.b }[6], [x23], #0x1\n" - "ld1 { v28.b }[6], [x22], #0x1\n" - "ld1 { v22.b }[6], [x21], #0x1\n" + "ld1 { v4.b }[6], [x23], #0x1\n" + "ld1 { v3.b }[6], [x22], #0x1\n" + "ld1 { v28.b }[6], [x21], #0x1\n" + "ld1 { v22.b }[6], [x20], #0x1\n" "b 23f\n" "20:" // Oddments: 4 inputs loop: Load: Bit 3: Unset: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[4], [x24], #0x1\n" - "ld1 { v3.b }[4], [x23], #0x1\n" - "ld1 { v28.b }[4], [x22], #0x1\n" - "ld1 { v22.b }[4], [x21], #0x1\n" + "ld1 { v4.b }[4], [x23], #0x1\n" + "ld1 { v3.b }[4], [x22], #0x1\n" + "ld1 { v28.b }[4], [x21], #0x1\n" + "ld1 { v22.b }[4], [x20], #0x1\n" "b 23f\n" "21:" // Oddments: 4 inputs loop: Load: Bit 3: Unset: Bit 2: Unset "tbz %x[n_channels], #1, 22f\n" - "ldr h4, [x24], #0x2\n" - "ldr h3, [x23], #0x2\n" - "ldr h28, [x22], #0x2\n" - "ldr h22, [x21], #0x2\n" + "ldr h4, [x23], #0x2\n" + "ldr h3, [x22], #0x2\n" + "ldr h28, [x21], #0x2\n" + "ldr h22, [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[2], [x24], #0x1\n" - "ld1 { v3.b }[2], [x23], #0x1\n" - "ld1 { v28.b }[2], [x22], #0x1\n" - "ld1 { v22.b }[2], [x21], #0x1\n" + "ld1 { v4.b }[2], [x23], #0x1\n" + "ld1 { v3.b }[2], [x22], #0x1\n" + "ld1 { v28.b }[2], [x21], #0x1\n" + "ld1 { v22.b }[2], [x20], #0x1\n" "b 23f\n" "22:" // Oddments: 4 inputs loop: Load: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ldr b4, [x24], #0x1\n" - "ldr b3, [x23], #0x1\n" - "ldr b28, [x22], #0x1\n" - "ldr b22, [x21], #0x1\n" + "ldr b4, [x23], #0x1\n" + "ldr b3, [x22], #0x1\n" + "ldr b28, [x21], #0x1\n" + "ldr b22, [x20], #0x1\n" "23:" // Oddments: 4 inputs loop: Load: Bit 3: End - "smax v23.16b, v4.16b, v3.16b\n" - "smax v19.16b, v28.16b, v22.16b\n" + "smax v17.16b, v4.16b, v3.16b\n" + "smax v16.16b, v28.16b, v22.16b\n" "subs x25, x25, #0x1\n" - "smax v19.16b, v23.16b, v19.16b\n" - "smax v8.16b, v8.16b, v19.16b\n" + "smax v16.16b, v17.16b, v16.16b\n" + "smax v8.16b, v8.16b, v16.16b\n" "bgt 15b\n" "24:" // Oddments: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 34f\n" "25:" // Oddments: Single input loop - "ldr x24, [x20], #0x8\n" - "add x24, x24, x9\n" + "ldr x23, [x24], #0x8\n" + "add x23, x23, x27\n" "movi v4.16b, #0x0\n" "tbz %x[n_channels], #3, 29f\n" - "ldr d4, [x24], #0x8\n" + "ldr d4, [x23], #0x8\n" "tbz %x[n_channels], #2, 27f\n" - "ld1 { v4.s }[2], [x24], #0x4\n" + "ld1 { v4.s }[2], [x23], #0x4\n" "tbz %x[n_channels], #1, 26f\n" - "ld1 { v4.h }[6], [x24], #0x2\n" + "ld1 { v4.h }[6], [x23], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[14], [x24], #0x1\n" + "ld1 { v4.b }[14], [x23], #0x1\n" "b 33f\n" "26:" // Oddments: Single input loop: Load: Bit 3: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[12], [x24], #0x1\n" + "ld1 { v4.b }[12], [x23], #0x1\n" "b 33f\n" "27:" // Oddments: Single input loop: Load: Bit 3: Bit 2: Unset "tbz %x[n_channels], #1, 28f\n" - "ld1 { v4.h }[4], [x24], #0x2\n" + "ld1 { v4.h }[4], [x23], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[10], [x24], #0x1\n" + "ld1 { v4.b }[10], [x23], #0x1\n" "b 33f\n" "28:" // Oddments: Single input loop: Load: Bit 3: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[8], [x24], #0x1\n" + "ld1 { v4.b }[8], [x23], #0x1\n" "b 33f\n" "29:" // Oddments: Single input loop: Load: Bit 3: Unset "tbz %x[n_channels], #2, 31f\n" - "ldr s4, [x24], #0x4\n" + "ldr s4, [x23], #0x4\n" "tbz %x[n_channels], #1, 30f\n" - "ld1 { v4.h }[2], [x24], #0x2\n" + "ld1 { v4.h }[2], [x23], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[6], [x24], #0x1\n" + "ld1 { v4.b }[6], [x23], #0x1\n" "b 33f\n" "30:" // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[4], [x24], #0x1\n" + "ld1 { v4.b }[4], [x23], #0x1\n" "b 33f\n" "31:" // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Unset "tbz %x[n_channels], #1, 32f\n" - "ldr h4, [x24], #0x2\n" + "ldr h4, [x23], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[2], [x24], #0x1\n" + "ld1 { v4.b }[2], [x23], #0x1\n" "b 33f\n" "32:" // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ldr b4, [x24], #0x1\n" + "ldr b4, [x23], #0x1\n" "33:" // Oddments: Single input loop: Load: Bit 3: End "subs x21, x21, #0x1\n" "smax v8.16b, v8.16b, v4.16b\n" "bgt 25b\n" "34:" // Oddments: Single input loop: End - "sxtl v23.8h, v8.8b\n" - "sxtl2 v22.8h, v8.16b\n" + "sxtl v17.8h, v8.8b\n" + "sxtl2 v16.8h, v8.16b\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n" - "ld1r { v4.4s }, [x20]\n" - "sxtl v1.4s, v23.4h\n" - "sxtl2 v23.4s, v23.8h\n" + "ld1r { v22.4s }, [x20]\n" + "sxtl v21.4s, v17.4h\n" + "sxtl2 v20.4s, v17.8h\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_mul]\n" - "ld1r { v3.4s }, [x20]\n" - "sxtl v0.4s, v22.4h\n" - "sxtl2 v31.4s, v22.8h\n" + "ld1r { v17.4s }, [x20]\n" + "sxtl v19.4s, v16.4h\n" + "sxtl2 v18.4s, v16.8h\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n" - "ld1r { v2.4s }, [x20]\n" - "srshl v1.4s, v1.4s, v4.4s\n" - "srshl v23.4s, v23.4s, v4.4s\n" - "srshl v0.4s, v0.4s, v4.4s\n" - "srshl v31.4s, v31.4s, v4.4s\n" - "sqrdmulh v1.4s, v1.4s, v3.4s\n" - "sqrdmulh v23.4s, v23.4s, v3.4s\n" - "sqrdmulh v0.4s, v0.4s, v3.4s\n" - "sqrdmulh v31.4s, v31.4s, v3.4s\n" + "ld1r { v16.4s }, [x20]\n" + "srshl v21.4s, v21.4s, v22.4s\n" + "srshl v20.4s, v20.4s, v22.4s\n" + "srshl v19.4s, v19.4s, v22.4s\n" + "srshl v18.4s, v18.4s, v22.4s\n" + "sqrdmulh v21.4s, v21.4s, v17.4s\n" + "sqrdmulh v20.4s, v20.4s, v17.4s\n" + "sqrdmulh v19.4s, v19.4s, v17.4s\n" + "sqrdmulh v18.4s, v18.4s, v17.4s\n" "movi v17.4s, #0x7f\n" - "srshl v1.4s, v1.4s, v2.4s\n" - "srshl v23.4s, v23.4s, v2.4s\n" - "srshl v0.4s, v0.4s, v2.4s\n" - "srshl v31.4s, v31.4s, v2.4s\n" + "srshl v21.4s, v21.4s, v16.4s\n" + "srshl v20.4s, v20.4s, v16.4s\n" + "srshl v19.4s, v19.4s, v16.4s\n" + "srshl v18.4s, v18.4s, v16.4s\n" "not v16.16b, v17.16b\n" - "smax v1.4s, v1.4s, v16.4s\n" - "smax v23.4s, v23.4s, v16.4s\n" - "smax v0.4s, v0.4s, v16.4s\n" - "smax v31.4s, v31.4s, v16.4s\n" - "smin v1.4s, v1.4s, v17.4s\n" - "smin v23.4s, v23.4s, v17.4s\n" - "smin v0.4s, v0.4s, v17.4s\n" - "smin v31.4s, v31.4s, v17.4s\n" - "uzp1 v23.16b, v1.16b, v23.16b\n" - "uzp1 v16.16b, v0.16b, v31.16b\n" - "uzp1 v16.16b, v23.16b, v16.16b\n" + "smax v21.4s, v21.4s, v16.4s\n" + "smax v20.4s, v20.4s, v16.4s\n" + "smax v19.4s, v19.4s, v16.4s\n" + "smax v18.4s, v18.4s, v16.4s\n" + "smin v21.4s, v21.4s, v17.4s\n" + "smin v20.4s, v20.4s, v17.4s\n" + "smin v19.4s, v19.4s, v17.4s\n" + "smin v18.4s, v18.4s, v17.4s\n" + "uzp1 v17.16b, v21.16b, v20.16b\n" + "uzp1 v16.16b, v19.16b, v18.16b\n" + "uzp1 v16.16b, v17.16b, v16.16b\n" "tbz %x[n_channels], #3, 38f\n" "st1 { v16.d }[0], [%x[outptr]], #0x8\n" "tbz %x[n_channels], #2, 36f\n" @@ -627,10 +628,11 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl( "43:" // End : [n_channels] "+&r" (n_channels), [outptr] "+&r" (outptr) : [inptrs] "r" (inptrs), [n_valid_cells] "r" (n_valid_cells), [offsetof_qp_per_layer_left_shift] "I" (offsetof(Requantize32, per_layer_left_shift)), [offsetof_qp_per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [offsetof_qp_per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [quant_params] "r" (&qp) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27" ); } } // namespace pooling } // namespace arm_conv + #endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_avg_generic_depthfirst/generic.cpp index 76828a911ed9256cb347b5b56e32e6f5b97ec568..f8984c451c7771e44600293370907b61140d9c80 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_avg_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_avg_generic_depthfirst/generic.cpp @@ -22,14 +22,14 @@ * SOFTWARE. */ -#if defined(__aarch64__) - #include #include #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace pooling { @@ -105,7 +105,7 @@ void a64_u8_nhwc_avg_generic_depthfirst_impl( "lsr x23, %x[n_valid_cells], #0x1\n" "movi v15.4s, #0x0\n" "movi v14.4s, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "movi v13.4s, #0x0\n" "movi v12.4s, #0x0\n" "movi v11.4s, #0x0\n" @@ -121,42 +121,42 @@ void a64_u8_nhwc_avg_generic_depthfirst_impl( "movi v1.4s, #0x0\n" "movi v0.4s, #0x0\n" "cbz x23, 4f\n" - "ldp x22, x21, [x20, #0x0]\n" - "ldr q31, [x22, x27]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q31, [x21, x27]\n" "subs x23, x23, #0x1\n" - "add x20, x20, #0x10\n" - "ldr q30, [x21, x27]\n" - "ldr q29, [x22, x26]\n" - "ldr q28, [x21, x26]\n" - "ldr q27, [x22, x25]\n" - "ldr q26, [x21, x25]\n" - "ldr q25, [x22, x24]\n" - "ldr q24, [x21, x24]\n" + "add x22, x22, #0x10\n" + "ldr q30, [x20, x27]\n" + "ldr q29, [x21, x26]\n" + "ldr q28, [x20, x26]\n" + "ldr q27, [x21, x25]\n" + "ldr q26, [x20, x25]\n" + "ldr q25, [x21, x24]\n" + "ldr q24, [x20, x24]\n" "beq 3f\n" "2:" // 4-vectors of channels: 2 inputs loop "uaddl v23.8h, v31.8b, v30.8b\n" "uaddl2 v22.8h, v31.16b, v30.16b\n" - "ldp x22, x21, [x20, #0x0]\n" - "ldr q31, [x22, x27]\n" - "ldr q30, [x21, x27]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q31, [x21, x27]\n" + "ldr q30, [x20, x27]\n" "uaddl v21.8h, v29.8b, v28.8b\n" "uaddl2 v20.8h, v29.16b, v28.16b\n" - "ldr q29, [x22, x26]\n" - "ldr q28, [x21, x26]\n" + "ldr q29, [x21, x26]\n" + "ldr q28, [x20, x26]\n" "uaddl v19.8h, v27.8b, v26.8b\n" "uaddl2 v18.8h, v27.16b, v26.16b\n" - "ldr q27, [x22, x25]\n" - "ldr q26, [x21, x25]\n" + "ldr q27, [x21, x25]\n" + "ldr q26, [x20, x25]\n" + "uaddl v17.8h, v25.8b, v24.8b\n" + "uaddl2 v16.8h, v25.16b, v24.16b\n" + "ldr q25, [x21, x24]\n" + "ldr q24, [x20, x24]\n" "subs x23, x23, #0x1\n" "uaddw v15.4s, v15.4s, v23.4h\n" "uaddw2 v14.4s, v14.4s, v23.8h\n" - "uaddl v17.8h, v25.8b, v24.8b\n" - "uaddl2 v16.8h, v25.16b, v24.16b\n" - "ldr q25, [x22, x24]\n" - "add x20, x20, #0x10\n" "uaddw v13.4s, v13.4s, v22.4h\n" "uaddw2 v12.4s, v12.4s, v22.8h\n" - "ldr q24, [x21, x24]\n" + "add x22, x22, #0x10\n" "uaddw v11.4s, v11.4s, v21.4h\n" "uaddw2 v10.4s, v10.4s, v21.8h\n" "uaddw v9.4s, v9.4s, v20.4h\n" @@ -196,23 +196,23 @@ void a64_u8_nhwc_avg_generic_depthfirst_impl( "uaddw v1.4s, v1.4s, v16.4h\n" "uaddw2 v0.4s, v0.4s, v16.8h\n" "4:" // 4-vectors of channels: After loop - "ands x21, %x[n_valid_cells], #0x1\n" + "ands x23, %x[n_valid_cells], #0x1\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x22, [x20], #0x8\n" - "ldr q31, [x22, x27]\n" - "uxtl v23.8h, v31.8b\n" - "uxtl2 v22.8h, v31.16b\n" - "ldr q29, [x22, x26]\n" - "ldr q27, [x22, x25]\n" - "uxtl v21.8h, v29.8b\n" - "uxtl2 v20.8h, v29.16b\n" - "ldr q25, [x22, x24]\n" - "uxtl v19.8h, v27.8b\n" - "uxtl2 v18.8h, v27.16b\n" - "subs x21, x21, #0x1\n" - "uxtl v17.8h, v25.8b\n" - "uxtl2 v16.8h, v25.16b\n" + "ldr x20, [x22], #0x8\n" + "ldr q16, [x20, x27]\n" + "uxtl v23.8h, v16.8b\n" + "uxtl2 v22.8h, v16.16b\n" + "ldr q16, [x20, x26]\n" + "ldr q17, [x20, x25]\n" + "uxtl v21.8h, v16.8b\n" + "uxtl2 v20.8h, v16.16b\n" + "ldr q16, [x20, x24]\n" + "uxtl v19.8h, v17.8b\n" + "uxtl2 v18.8h, v17.16b\n" + "subs x23, x23, #0x1\n" + "uxtl v17.8h, v16.8b\n" + "uxtl2 v16.8h, v16.16b\n" "uaddw v15.4s, v15.4s, v23.4h\n" "uaddw2 v14.4s, v14.4s, v23.8h\n" "uaddw v13.4s, v13.4s, v22.4h\n" @@ -330,49 +330,49 @@ void a64_u8_nhwc_avg_generic_depthfirst_impl( "lsr x23, %x[n_valid_cells], #0x1\n" "movi v15.4s, #0x0\n" "movi v14.4s, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "movi v13.4s, #0x0\n" "movi v12.4s, #0x0\n" "cbz x23, 11f\n" - "ldp x22, x21, [x20, #0x0]\n" - "ldr q31, [x22, x27]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q31, [x21, x27]\n" "subs x23, x23, #0x1\n" - "add x20, x20, #0x10\n" - "ldr q30, [x21, x27]\n" + "add x22, x22, #0x10\n" + "ldr q30, [x20, x27]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 2 inputs loop - "uaddl v23.8h, v31.8b, v30.8b\n" - "uaddl2 v22.8h, v31.16b, v30.16b\n" - "ldp x22, x21, [x20, #0x0]\n" - "ldr q31, [x22, x27]\n" - "ldr q30, [x21, x27]\n" + "uaddl v17.8h, v31.8b, v30.8b\n" + "uaddl2 v16.8h, v31.16b, v30.16b\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q31, [x21, x27]\n" + "ldr q30, [x20, x27]\n" "subs x23, x23, #0x1\n" - "uaddw v15.4s, v15.4s, v23.4h\n" - "uaddw2 v14.4s, v14.4s, v23.8h\n" - "uaddw v13.4s, v13.4s, v22.4h\n" - "uaddw2 v12.4s, v12.4s, v22.8h\n" - "add x20, x20, #0x10\n" + "uaddw v15.4s, v15.4s, v17.4h\n" + "uaddw2 v14.4s, v14.4s, v17.8h\n" + "uaddw v13.4s, v13.4s, v16.4h\n" + "uaddw2 v12.4s, v12.4s, v16.8h\n" + "add x22, x22, #0x10\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 2 inputs tail - "uaddl v23.8h, v31.8b, v30.8b\n" - "uaddl2 v22.8h, v31.16b, v30.16b\n" - "uaddw v15.4s, v15.4s, v23.4h\n" - "uaddw2 v14.4s, v14.4s, v23.8h\n" - "uaddw v13.4s, v13.4s, v22.4h\n" - "uaddw2 v12.4s, v12.4s, v22.8h\n" + "uaddl v17.8h, v31.8b, v30.8b\n" + "uaddl2 v16.8h, v31.16b, v30.16b\n" + "uaddw v15.4s, v15.4s, v17.4h\n" + "uaddw2 v14.4s, v14.4s, v17.8h\n" + "uaddw v13.4s, v13.4s, v16.4h\n" + "uaddw2 v12.4s, v12.4s, v16.8h\n" "11:" // Single vector of channels: Loop: After loop - "ands x21, %x[n_valid_cells], #0x1\n" + "ands x23, %x[n_valid_cells], #0x1\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x22, [x20], #0x8\n" - "ldr q31, [x22, x27]\n" - "uxtl v23.8h, v31.8b\n" - "uxtl2 v22.8h, v31.16b\n" - "subs x21, x21, #0x1\n" - "uaddw v15.4s, v15.4s, v23.4h\n" - "uaddw2 v14.4s, v14.4s, v23.8h\n" - "uaddw v13.4s, v13.4s, v22.4h\n" - "uaddw2 v12.4s, v12.4s, v22.8h\n" + "ldr x20, [x22], #0x8\n" + "ldr q16, [x20, x27]\n" + "uxtl v17.8h, v16.8b\n" + "uxtl2 v16.8h, v16.16b\n" + "subs x23, x23, #0x1\n" + "uaddw v15.4s, v15.4s, v17.4h\n" + "uaddw2 v14.4s, v14.4s, v17.8h\n" + "uaddw v13.4s, v13.4s, v16.4h\n" + "uaddw2 v12.4s, v12.4s, v16.8h\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "ld1r { v17.4s }, [%x[rescale_ptr]]\n" @@ -397,9 +397,9 @@ void a64_u8_nhwc_avg_generic_depthfirst_impl( "smin v14.4s, v14.4s, v16.4s\n" "smin v13.4s, v13.4s, v16.4s\n" "smin v12.4s, v12.4s, v16.4s\n" - "uzp1 v23.16b, v15.16b, v14.16b\n" + "uzp1 v17.16b, v15.16b, v14.16b\n" "uzp1 v16.16b, v13.16b, v12.16b\n" - "uzp1 v16.16b, v23.16b, v16.16b\n" + "uzp1 v16.16b, v17.16b, v16.16b\n" "str q16, [%x[outptr], x27]\n" "add x27, x27, #0x10\n" "bge 8b\n" @@ -411,142 +411,142 @@ void a64_u8_nhwc_avg_generic_depthfirst_impl( "movi v14.4s, #0x0\n" "movi v13.4s, #0x0\n" "movi v12.4s, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "cbz x23, 24f\n" "15:" // Oddments: 2 inputs loop - "ldp x22, x21, [x20, #0x0]\n" - "add x20, x20, #0x10\n" - "add x22, x22, x27\n" - "movi v31.16b, #0x0\n" + "ldp x21, x20, [x22, #0x0]\n" + "add x22, x22, #0x10\n" "add x21, x21, x27\n" + "movi v31.16b, #0x0\n" + "add x20, x20, x27\n" "movi v30.16b, #0x0\n" "tbz %x[n_channels], #3, 19f\n" - "ldr d31, [x22], #0x8\n" - "ldr d30, [x21], #0x8\n" + "ldr d31, [x21], #0x8\n" + "ldr d30, [x20], #0x8\n" "tbz %x[n_channels], #2, 17f\n" - "ld1 { v31.s }[2], [x22], #0x4\n" - "ld1 { v30.s }[2], [x21], #0x4\n" + "ld1 { v31.s }[2], [x21], #0x4\n" + "ld1 { v30.s }[2], [x20], #0x4\n" "tbz %x[n_channels], #1, 16f\n" - "ld1 { v31.h }[6], [x22], #0x2\n" - "ld1 { v30.h }[6], [x21], #0x2\n" + "ld1 { v31.h }[6], [x21], #0x2\n" + "ld1 { v30.h }[6], [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[14], [x22], #0x1\n" - "ld1 { v30.b }[14], [x21], #0x1\n" + "ld1 { v31.b }[14], [x21], #0x1\n" + "ld1 { v30.b }[14], [x20], #0x1\n" "b 23f\n" "16:" // Oddments: 2 inputs loop: Load: Bit 3: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[12], [x22], #0x1\n" - "ld1 { v30.b }[12], [x21], #0x1\n" + "ld1 { v31.b }[12], [x21], #0x1\n" + "ld1 { v30.b }[12], [x20], #0x1\n" "b 23f\n" "17:" // Oddments: 2 inputs loop: Load: Bit 3: Bit 2: Unset "tbz %x[n_channels], #1, 18f\n" - "ld1 { v31.h }[4], [x22], #0x2\n" - "ld1 { v30.h }[4], [x21], #0x2\n" + "ld1 { v31.h }[4], [x21], #0x2\n" + "ld1 { v30.h }[4], [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[10], [x22], #0x1\n" - "ld1 { v30.b }[10], [x21], #0x1\n" + "ld1 { v31.b }[10], [x21], #0x1\n" + "ld1 { v30.b }[10], [x20], #0x1\n" "b 23f\n" "18:" // Oddments: 2 inputs loop: Load: Bit 3: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[8], [x22], #0x1\n" - "ld1 { v30.b }[8], [x21], #0x1\n" + "ld1 { v31.b }[8], [x21], #0x1\n" + "ld1 { v30.b }[8], [x20], #0x1\n" "b 23f\n" "19:" // Oddments: 2 inputs loop: Load: Bit 3: Unset "tbz %x[n_channels], #2, 21f\n" - "ldr s31, [x22], #0x4\n" - "ldr s30, [x21], #0x4\n" + "ldr s31, [x21], #0x4\n" + "ldr s30, [x20], #0x4\n" "tbz %x[n_channels], #1, 20f\n" - "ld1 { v31.h }[2], [x22], #0x2\n" - "ld1 { v30.h }[2], [x21], #0x2\n" + "ld1 { v31.h }[2], [x21], #0x2\n" + "ld1 { v30.h }[2], [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[6], [x22], #0x1\n" - "ld1 { v30.b }[6], [x21], #0x1\n" + "ld1 { v31.b }[6], [x21], #0x1\n" + "ld1 { v30.b }[6], [x20], #0x1\n" "b 23f\n" "20:" // Oddments: 2 inputs loop: Load: Bit 3: Unset: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[4], [x22], #0x1\n" - "ld1 { v30.b }[4], [x21], #0x1\n" + "ld1 { v31.b }[4], [x21], #0x1\n" + "ld1 { v30.b }[4], [x20], #0x1\n" "b 23f\n" "21:" // Oddments: 2 inputs loop: Load: Bit 3: Unset: Bit 2: Unset "tbz %x[n_channels], #1, 22f\n" - "ldr h31, [x22], #0x2\n" - "ldr h30, [x21], #0x2\n" + "ldr h31, [x21], #0x2\n" + "ldr h30, [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[2], [x22], #0x1\n" - "ld1 { v30.b }[2], [x21], #0x1\n" + "ld1 { v31.b }[2], [x21], #0x1\n" + "ld1 { v30.b }[2], [x20], #0x1\n" "b 23f\n" "22:" // Oddments: 2 inputs loop: Load: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ldr b31, [x22], #0x1\n" - "ldr b30, [x21], #0x1\n" + "ldr b31, [x21], #0x1\n" + "ldr b30, [x20], #0x1\n" "23:" // Oddments: 2 inputs loop: Load: Bit 3: End - "uaddl v23.8h, v31.8b, v30.8b\n" - "uaddl2 v22.8h, v31.16b, v30.16b\n" + "uaddl v17.8h, v31.8b, v30.8b\n" + "uaddl2 v16.8h, v31.16b, v30.16b\n" "subs x23, x23, #0x1\n" - "uaddw v15.4s, v15.4s, v23.4h\n" - "uaddw2 v14.4s, v14.4s, v23.8h\n" - "uaddw v13.4s, v13.4s, v22.4h\n" - "uaddw2 v12.4s, v12.4s, v22.8h\n" + "uaddw v15.4s, v15.4s, v17.4h\n" + "uaddw2 v14.4s, v14.4s, v17.8h\n" + "uaddw v13.4s, v13.4s, v16.4h\n" + "uaddw2 v12.4s, v12.4s, v16.8h\n" "bgt 15b\n" "24:" // Oddments: After loop - "ands x21, %x[n_valid_cells], #0x1\n" + "ands x23, %x[n_valid_cells], #0x1\n" "beq 34f\n" "25:" // Oddments: Single input loop - "ldr x22, [x20], #0x8\n" - "add x22, x22, x27\n" + "ldr x21, [x22], #0x8\n" + "add x21, x21, x27\n" "movi v31.16b, #0x0\n" "tbz %x[n_channels], #3, 29f\n" - "ldr d31, [x22], #0x8\n" + "ldr d31, [x21], #0x8\n" "tbz %x[n_channels], #2, 27f\n" - "ld1 { v31.s }[2], [x22], #0x4\n" + "ld1 { v31.s }[2], [x21], #0x4\n" "tbz %x[n_channels], #1, 26f\n" - "ld1 { v31.h }[6], [x22], #0x2\n" + "ld1 { v31.h }[6], [x21], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[14], [x22], #0x1\n" + "ld1 { v31.b }[14], [x21], #0x1\n" "b 33f\n" "26:" // Oddments: Single input loop: Load: Bit 3: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[12], [x22], #0x1\n" + "ld1 { v31.b }[12], [x21], #0x1\n" "b 33f\n" "27:" // Oddments: Single input loop: Load: Bit 3: Bit 2: Unset "tbz %x[n_channels], #1, 28f\n" - "ld1 { v31.h }[4], [x22], #0x2\n" + "ld1 { v31.h }[4], [x21], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[10], [x22], #0x1\n" + "ld1 { v31.b }[10], [x21], #0x1\n" "b 33f\n" "28:" // Oddments: Single input loop: Load: Bit 3: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[8], [x22], #0x1\n" + "ld1 { v31.b }[8], [x21], #0x1\n" "b 33f\n" "29:" // Oddments: Single input loop: Load: Bit 3: Unset "tbz %x[n_channels], #2, 31f\n" - "ldr s31, [x22], #0x4\n" + "ldr s31, [x21], #0x4\n" "tbz %x[n_channels], #1, 30f\n" - "ld1 { v31.h }[2], [x22], #0x2\n" + "ld1 { v31.h }[2], [x21], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[6], [x22], #0x1\n" + "ld1 { v31.b }[6], [x21], #0x1\n" "b 33f\n" "30:" // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[4], [x22], #0x1\n" + "ld1 { v31.b }[4], [x21], #0x1\n" "b 33f\n" "31:" // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Unset "tbz %x[n_channels], #1, 32f\n" - "ldr h31, [x22], #0x2\n" + "ldr h31, [x21], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[2], [x22], #0x1\n" + "ld1 { v31.b }[2], [x21], #0x1\n" "b 33f\n" "32:" // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ldr b31, [x22], #0x1\n" + "ldr b31, [x21], #0x1\n" "33:" // Oddments: Single input loop: Load: Bit 3: End - "uxtl v23.8h, v31.8b\n" - "uxtl2 v22.8h, v31.16b\n" - "subs x21, x21, #0x1\n" - "uaddw v15.4s, v15.4s, v23.4h\n" - "uaddw2 v14.4s, v14.4s, v23.8h\n" - "uaddw v13.4s, v13.4s, v22.4h\n" - "uaddw2 v12.4s, v12.4s, v22.8h\n" + "uxtl v17.8h, v31.8b\n" + "uxtl2 v16.8h, v31.16b\n" + "subs x23, x23, #0x1\n" + "uaddw v15.4s, v15.4s, v17.4h\n" + "uaddw2 v14.4s, v14.4s, v17.8h\n" + "uaddw v13.4s, v13.4s, v16.4h\n" + "uaddw2 v12.4s, v12.4s, v16.8h\n" "bgt 25b\n" "34:" // Oddments: Single input loop: End "ld1r { v17.4s }, [%x[rescale_ptr]]\n" @@ -569,9 +569,9 @@ void a64_u8_nhwc_avg_generic_depthfirst_impl( "smin v14.4s, v14.4s, v16.4s\n" "smin v13.4s, v13.4s, v16.4s\n" "smin v12.4s, v12.4s, v16.4s\n" - "uzp1 v23.16b, v15.16b, v14.16b\n" + "uzp1 v17.16b, v15.16b, v14.16b\n" "uzp1 v16.16b, v13.16b, v12.16b\n" - "uzp1 v16.16b, v23.16b, v16.16b\n" + "uzp1 v16.16b, v17.16b, v16.16b\n" "tbz %x[n_channels], #3, 38f\n" "st1 { v16.d }[0], [%x[outptr]], #0x8\n" "tbz %x[n_channels], #2, 36f\n" @@ -626,4 +626,5 @@ void a64_u8_nhwc_avg_generic_depthfirst_impl( } // namespace pooling } // namespace arm_conv + #endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_max_2x2_s1_output2x2_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_max_2x2_s1_output2x2_depthfirst.hpp index 556d833681fc57f35c34d804223a2e1ac5cb6964..9d160bf8f8abc7f7becfc1f0d9220095c4f425d9 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_max_2x2_s1_output2x2_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_max_2x2_s1_output2x2_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,8 @@ #pragma once +#if defined(__aarch64__) + namespace arm_conv { namespace pooling { @@ -45,3 +47,5 @@ struct a64_u8_nhwc_max_2x2_s1_output2x2_depthfirst : public DepthfirstStrategy #include +#if defined(__aarch64__) + namespace arm_conv { namespace pooling { @@ -111,7 +112,7 @@ void a64_u8_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "umax v18.16b, v18.16b, v21.16b\n" "umax v17.16b, v17.16b, v20.16b\n" "add x15, x15, #0x10\n" - "umax v16.16b, v16.16b, v20.16b\n" + "umax v16.16b, v20.16b, v16.16b\n" "str q19, [x14, x12]\n" "str q18, [x13, x12]\n" "str q17, [x11, x12]\n" @@ -121,43 +122,43 @@ void a64_u8_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "2:" // Vector: Tail "umax v21.16b, v30.16b, v29.16b\n" "umax v20.16b, v29.16b, v28.16b\n" - "umax v19.16b, v27.16b, v26.16b\n" + "umax v16.16b, v27.16b, v26.16b\n" "umax v18.16b, v25.16b, v24.16b\n" "umax v17.16b, v27.16b, v23.16b\n" - "umax v16.16b, v24.16b, v22.16b\n" - "umax v19.16b, v21.16b, v19.16b\n" + "umax v19.16b, v24.16b, v22.16b\n" + "umax v16.16b, v21.16b, v16.16b\n" "umax v18.16b, v18.16b, v21.16b\n" - "str q19, [x14, x12]\n" + "str q16, [x14, x12]\n" "umax v17.16b, v17.16b, v20.16b\n" - "umax v16.16b, v16.16b, v20.16b\n" + "umax v16.16b, v20.16b, v19.16b\n" "str q18, [x13, x12]\n" "str q17, [x11, x12]\n" "str q16, [x10, x12]\n" "add x12, x12, #0x10\n" "cbz x16, 4f\n" "3:" // Oddments - "ldr b30, [x28, x15]\n" - "ldr b29, [x25, x15]\n" - "umax v21.16b, v30.16b, v29.16b\n" + "ldr b16, [x28, x15]\n" + "ldr b17, [x25, x15]\n" + "umax v23.16b, v16.16b, v17.16b\n" "subs x16, x16, #0x1\n" - "ldr b28, [x22, x15]\n" - "ldr b27, [x26, x15]\n" - "umax v20.16b, v29.16b, v28.16b\n" - "ldr b26, [x9, x15]\n" - "ldr b25, [x27, x15]\n" - "umax v19.16b, v27.16b, v26.16b\n" - "umax v19.16b, v21.16b, v19.16b\n" - "ldr b24, [x24, x15]\n" - "ldr b23, [x23, x15]\n" - "umax v18.16b, v25.16b, v24.16b\n" - "umax v17.16b, v27.16b, v23.16b\n" - "ldr b22, [x21, x15]\n" - "umax v16.16b, v24.16b, v22.16b\n" + "ldr b16, [x22, x15]\n" + "ldr b22, [x26, x15]\n" + "umax v21.16b, v17.16b, v16.16b\n" + "ldr b16, [x9, x15]\n" + "ldr b17, [x27, x15]\n" + "umax v16.16b, v22.16b, v16.16b\n" + "umax v20.16b, v23.16b, v16.16b\n" + "ldr b19, [x24, x15]\n" + "ldr b16, [x23, x15]\n" + "umax v18.16b, v17.16b, v19.16b\n" + "umax v17.16b, v22.16b, v16.16b\n" + "ldr b16, [x21, x15]\n" + "umax v16.16b, v19.16b, v16.16b\n" "add x15, x15, #0x1\n" - "umax v18.16b, v18.16b, v21.16b\n" - "umax v17.16b, v17.16b, v20.16b\n" - "umax v16.16b, v16.16b, v20.16b\n" - "str b19, [x14, x12]\n" + "umax v18.16b, v18.16b, v23.16b\n" + "umax v17.16b, v17.16b, v21.16b\n" + "umax v16.16b, v21.16b, v16.16b\n" + "str b20, [x14, x12]\n" "str b18, [x13, x12]\n" "str b17, [x11, x12]\n" "str b16, [x10, x12]\n" @@ -172,4 +173,5 @@ void a64_u8_nhwc_max_2x2_s1_output2x2_depthfirst_impl( } // namespace pooling } // namespace arm_conv + #endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_max_generic_depthfirst/generic.cpp index 98f5b8351ca232b250676b8a3d117e6dd7b10f02..2ceef125cabe57ddf0f385da33a6e184ce803196 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_max_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_max_generic_depthfirst/generic.cpp @@ -22,11 +22,11 @@ * SOFTWARE. */ -#if defined(__aarch64__) - #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace pooling { @@ -41,77 +41,77 @@ void a64_u8_nhwc_max_generic_depthfirst_impl( { __asm__ __volatile__( "cmp %x[n_channels], #0x40\n" - "mov x9, #0x0\n" - "mov x28, #0x10\n" // cntb _, ALL, #1 - "mov x27, #0x20\n" // cntb _, ALL, #2 - "mov x26, #0x30\n" // cntb _, ALL, #3 + "mov x27, #0x0\n" + "mov x26, #0x10\n" // cntb _, ALL, #1 + "mov x24, #0x20\n" // cntb _, ALL, #2 + "mov x23, #0x30\n" // cntb _, ALL, #3 "blt 7f\n" "1:" // 4-vectors of channels "lsr x25, %x[n_valid_cells], #0x2\n" "movi v8.16b, #0x0\n" "movi v7.16b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "movi v6.16b, #0x0\n" "movi v5.16b, #0x0\n" "cbz x25, 4f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" "subs x25, x25, #0x1\n" - "ldr q3, [x23, x9]\n" - "ldr q2, [x24, x28]\n" - "ldr q1, [x23, x28]\n" - "ldr q0, [x24, x27]\n" - "ldr q31, [x23, x27]\n" - "ldr q30, [x24, x26]\n" - "ldr q29, [x23, x26]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" - "ldr q27, [x22, x28]\n" - "ldr q21, [x21, x28]\n" - "ldr q26, [x22, x27]\n" - "ldr q20, [x21, x27]\n" - "ldr q25, [x22, x26]\n" - "ldr q24, [x21, x26]\n" + "ldr q3, [x20, x27]\n" + "ldr q2, [x21, x26]\n" + "ldr q1, [x20, x26]\n" + "ldr q0, [x21, x24]\n" + "ldr q31, [x20, x24]\n" + "ldr q30, [x21, x23]\n" + "ldr q29, [x20, x23]\n" + "ldp x21, x20, [x22, #0x10]\n" + "add x22, x22, #0x20\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" + "ldr q27, [x21, x26]\n" + "ldr q21, [x20, x26]\n" + "ldr q26, [x21, x24]\n" + "ldr q20, [x20, x24]\n" + "ldr q25, [x21, x23]\n" + "ldr q24, [x20, x23]\n" "beq 3f\n" "2:" // 4-vectors of channels: 4 inputs loop "umax v23.16b, v4.16b, v3.16b\n" "umax v19.16b, v28.16b, v22.16b\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" - "ldr q3, [x23, x9]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" + "ldr q3, [x20, x27]\n" "umax v22.16b, v2.16b, v1.16b\n" - "ldr q2, [x24, x28]\n" + "ldr q2, [x21, x26]\n" "umax v18.16b, v27.16b, v21.16b\n" - "ldr q1, [x23, x28]\n" + "ldr q1, [x20, x26]\n" "umax v21.16b, v0.16b, v31.16b\n" - "ldr q0, [x24, x27]\n" + "ldr q0, [x21, x24]\n" "umax v17.16b, v26.16b, v20.16b\n" - "ldr q31, [x23, x27]\n" + "ldr q31, [x20, x24]\n" "umax v20.16b, v30.16b, v29.16b\n" - "ldr q30, [x24, x26]\n" + "ldr q30, [x21, x23]\n" "umax v16.16b, v25.16b, v24.16b\n" - "ldr q29, [x23, x26]\n" + "ldr q29, [x20, x23]\n" "umax v19.16b, v23.16b, v19.16b\n" "umax v18.16b, v22.16b, v18.16b\n" - "ldp x22, x21, [x20, #0x10]\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" + "ldp x21, x20, [x22, #0x10]\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" "umax v17.16b, v21.16b, v17.16b\n" "umax v16.16b, v20.16b, v16.16b\n" - "ldr q27, [x22, x28]\n" - "ldr q21, [x21, x28]\n" + "ldr q27, [x21, x26]\n" + "ldr q21, [x20, x26]\n" "subs x25, x25, #0x1\n" "umax v8.16b, v8.16b, v19.16b\n" - "ldr q26, [x22, x27]\n" - "ldr q20, [x21, x27]\n" + "ldr q26, [x21, x24]\n" + "ldr q20, [x20, x24]\n" "umax v7.16b, v7.16b, v18.16b\n" "umax v6.16b, v6.16b, v17.16b\n" - "ldr q25, [x22, x26]\n" - "ldr q24, [x21, x26]\n" + "ldr q25, [x21, x23]\n" + "ldr q24, [x20, x23]\n" "umax v5.16b, v5.16b, v16.16b\n" - "add x20, x20, #0x20\n" + "add x22, x22, #0x20\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "umax v23.16b, v4.16b, v3.16b\n" @@ -134,28 +134,28 @@ void a64_u8_nhwc_max_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x24, [x20], #0x8\n" - "ldr q4, [x24, x9]\n" + "ldr x20, [x22], #0x8\n" + "ldr q16, [x20, x27]\n" "subs x21, x21, #0x1\n" - "umax v8.16b, v8.16b, v4.16b\n" - "ldr q2, [x24, x28]\n" - "ldr q0, [x24, x27]\n" - "umax v7.16b, v7.16b, v2.16b\n" - "umax v6.16b, v6.16b, v0.16b\n" - "ldr q30, [x24, x26]\n" - "umax v5.16b, v5.16b, v30.16b\n" + "umax v8.16b, v8.16b, v16.16b\n" + "ldr q17, [x20, x26]\n" + "ldr q16, [x20, x24]\n" + "umax v7.16b, v7.16b, v17.16b\n" + "umax v6.16b, v6.16b, v16.16b\n" + "ldr q16, [x20, x23]\n" + "umax v5.16b, v5.16b, v16.16b\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End "sub %x[n_channels], %x[n_channels], #0x40\n" "cmp %x[n_channels], #0x40\n" - "str q8, [%x[outptr], x9]\n" - "str q7, [%x[outptr], x28]\n" - "add x9, x9, #0x40\n" - "add x28, x28, #0x40\n" - "str q6, [%x[outptr], x27]\n" + "str q8, [%x[outptr], x27]\n" + "str q7, [%x[outptr], x26]\n" "add x27, x27, #0x40\n" - "str q5, [%x[outptr], x26]\n" "add x26, x26, #0x40\n" + "str q6, [%x[outptr], x24]\n" + "add x24, x24, #0x40\n" + "str q5, [%x[outptr], x23]\n" + "add x23, x23, #0x40\n" "bge 1b\n" "cbz %x[n_channels], 43f\n" "7:" // Single vector of channels @@ -164,217 +164,217 @@ void a64_u8_nhwc_max_generic_depthfirst_impl( "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" "movi v8.16b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" "subs x25, x25, #0x1\n" - "ldr q3, [x23, x9]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" + "ldr q3, [x20, x27]\n" + "ldp x21, x20, [x22, #0x10]\n" + "add x22, x22, #0x20\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 4 inputs loop - "umax v23.16b, v4.16b, v3.16b\n" - "umax v19.16b, v28.16b, v22.16b\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" - "ldr q3, [x23, x9]\n" - "umax v19.16b, v23.16b, v19.16b\n" - "ldp x22, x21, [x20, #0x10]\n" + "umax v17.16b, v4.16b, v3.16b\n" + "umax v16.16b, v28.16b, v22.16b\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" + "ldr q3, [x20, x27]\n" + "umax v16.16b, v17.16b, v16.16b\n" + "ldp x21, x20, [x22, #0x10]\n" "subs x25, x25, #0x1\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" - "umax v8.16b, v8.16b, v19.16b\n" - "add x20, x20, #0x20\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" + "umax v8.16b, v8.16b, v16.16b\n" + "add x22, x22, #0x20\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail - "umax v23.16b, v4.16b, v3.16b\n" - "umax v19.16b, v28.16b, v22.16b\n" - "umax v19.16b, v23.16b, v19.16b\n" - "umax v8.16b, v8.16b, v19.16b\n" + "umax v17.16b, v4.16b, v3.16b\n" + "umax v16.16b, v28.16b, v22.16b\n" + "umax v16.16b, v17.16b, v16.16b\n" + "umax v8.16b, v8.16b, v16.16b\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x24, [x20], #0x8\n" - "ldr q4, [x24, x9]\n" + "ldr x20, [x22], #0x8\n" + "ldr q16, [x20, x27]\n" "subs x21, x21, #0x1\n" - "umax v8.16b, v8.16b, v4.16b\n" + "umax v8.16b, v8.16b, v16.16b\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "sub %x[n_channels], %x[n_channels], #0x10\n" "cmp %x[n_channels], #0x10\n" - "str q8, [%x[outptr], x9]\n" - "add x9, x9, #0x10\n" + "str q8, [%x[outptr], x27]\n" + "add x27, x27, #0x10\n" "bge 8b\n" "cbz %x[n_channels], 43f\n" "14:" // Oddments "lsr x25, %x[n_valid_cells], #0x2\n" - "add %x[outptr], %x[outptr], x9\n" + "add %x[outptr], %x[outptr], x27\n" "movi v8.16b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "cbz x25, 24f\n" "15:" // Oddments: 4 inputs loop - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "add x24, x24, x9\n" - "add x23, x23, x9\n" - "add x22, x22, x9\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" + "add x23, x23, x27\n" + "add x22, x22, x27\n" + "add x21, x21, x27\n" "movi v4.16b, #0x0\n" "movi v3.16b, #0x0\n" - "add x21, x21, x9\n" + "add x20, x20, x27\n" "movi v28.16b, #0x0\n" "movi v22.16b, #0x0\n" "tbz %x[n_channels], #3, 19f\n" - "ldr d4, [x24], #0x8\n" - "ldr d3, [x23], #0x8\n" - "ldr d28, [x22], #0x8\n" - "ldr d22, [x21], #0x8\n" + "ldr d4, [x23], #0x8\n" + "ldr d3, [x22], #0x8\n" + "ldr d28, [x21], #0x8\n" + "ldr d22, [x20], #0x8\n" "tbz %x[n_channels], #2, 17f\n" - "ld1 { v4.s }[2], [x24], #0x4\n" - "ld1 { v3.s }[2], [x23], #0x4\n" - "ld1 { v28.s }[2], [x22], #0x4\n" - "ld1 { v22.s }[2], [x21], #0x4\n" + "ld1 { v4.s }[2], [x23], #0x4\n" + "ld1 { v3.s }[2], [x22], #0x4\n" + "ld1 { v28.s }[2], [x21], #0x4\n" + "ld1 { v22.s }[2], [x20], #0x4\n" "tbz %x[n_channels], #1, 16f\n" - "ld1 { v4.h }[6], [x24], #0x2\n" - "ld1 { v3.h }[6], [x23], #0x2\n" - "ld1 { v28.h }[6], [x22], #0x2\n" - "ld1 { v22.h }[6], [x21], #0x2\n" + "ld1 { v4.h }[6], [x23], #0x2\n" + "ld1 { v3.h }[6], [x22], #0x2\n" + "ld1 { v28.h }[6], [x21], #0x2\n" + "ld1 { v22.h }[6], [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[14], [x24], #0x1\n" - "ld1 { v3.b }[14], [x23], #0x1\n" - "ld1 { v28.b }[14], [x22], #0x1\n" - "ld1 { v22.b }[14], [x21], #0x1\n" + "ld1 { v4.b }[14], [x23], #0x1\n" + "ld1 { v3.b }[14], [x22], #0x1\n" + "ld1 { v28.b }[14], [x21], #0x1\n" + "ld1 { v22.b }[14], [x20], #0x1\n" "b 23f\n" "16:" // Oddments: 4 inputs loop: Load: Bit 3: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[12], [x24], #0x1\n" - "ld1 { v3.b }[12], [x23], #0x1\n" - "ld1 { v28.b }[12], [x22], #0x1\n" - "ld1 { v22.b }[12], [x21], #0x1\n" + "ld1 { v4.b }[12], [x23], #0x1\n" + "ld1 { v3.b }[12], [x22], #0x1\n" + "ld1 { v28.b }[12], [x21], #0x1\n" + "ld1 { v22.b }[12], [x20], #0x1\n" "b 23f\n" "17:" // Oddments: 4 inputs loop: Load: Bit 3: Bit 2: Unset "tbz %x[n_channels], #1, 18f\n" - "ld1 { v4.h }[4], [x24], #0x2\n" - "ld1 { v3.h }[4], [x23], #0x2\n" - "ld1 { v28.h }[4], [x22], #0x2\n" - "ld1 { v22.h }[4], [x21], #0x2\n" + "ld1 { v4.h }[4], [x23], #0x2\n" + "ld1 { v3.h }[4], [x22], #0x2\n" + "ld1 { v28.h }[4], [x21], #0x2\n" + "ld1 { v22.h }[4], [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[10], [x24], #0x1\n" - "ld1 { v3.b }[10], [x23], #0x1\n" - "ld1 { v28.b }[10], [x22], #0x1\n" - "ld1 { v22.b }[10], [x21], #0x1\n" + "ld1 { v4.b }[10], [x23], #0x1\n" + "ld1 { v3.b }[10], [x22], #0x1\n" + "ld1 { v28.b }[10], [x21], #0x1\n" + "ld1 { v22.b }[10], [x20], #0x1\n" "b 23f\n" "18:" // Oddments: 4 inputs loop: Load: Bit 3: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[8], [x24], #0x1\n" - "ld1 { v3.b }[8], [x23], #0x1\n" - "ld1 { v28.b }[8], [x22], #0x1\n" - "ld1 { v22.b }[8], [x21], #0x1\n" + "ld1 { v4.b }[8], [x23], #0x1\n" + "ld1 { v3.b }[8], [x22], #0x1\n" + "ld1 { v28.b }[8], [x21], #0x1\n" + "ld1 { v22.b }[8], [x20], #0x1\n" "b 23f\n" "19:" // Oddments: 4 inputs loop: Load: Bit 3: Unset "tbz %x[n_channels], #2, 21f\n" - "ldr s4, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr s28, [x22], #0x4\n" - "ldr s22, [x21], #0x4\n" + "ldr s4, [x23], #0x4\n" + "ldr s3, [x22], #0x4\n" + "ldr s28, [x21], #0x4\n" + "ldr s22, [x20], #0x4\n" "tbz %x[n_channels], #1, 20f\n" - "ld1 { v4.h }[2], [x24], #0x2\n" - "ld1 { v3.h }[2], [x23], #0x2\n" - "ld1 { v28.h }[2], [x22], #0x2\n" - "ld1 { v22.h }[2], [x21], #0x2\n" + "ld1 { v4.h }[2], [x23], #0x2\n" + "ld1 { v3.h }[2], [x22], #0x2\n" + "ld1 { v28.h }[2], [x21], #0x2\n" + "ld1 { v22.h }[2], [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[6], [x24], #0x1\n" - "ld1 { v3.b }[6], [x23], #0x1\n" - "ld1 { v28.b }[6], [x22], #0x1\n" - "ld1 { v22.b }[6], [x21], #0x1\n" + "ld1 { v4.b }[6], [x23], #0x1\n" + "ld1 { v3.b }[6], [x22], #0x1\n" + "ld1 { v28.b }[6], [x21], #0x1\n" + "ld1 { v22.b }[6], [x20], #0x1\n" "b 23f\n" "20:" // Oddments: 4 inputs loop: Load: Bit 3: Unset: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[4], [x24], #0x1\n" - "ld1 { v3.b }[4], [x23], #0x1\n" - "ld1 { v28.b }[4], [x22], #0x1\n" - "ld1 { v22.b }[4], [x21], #0x1\n" + "ld1 { v4.b }[4], [x23], #0x1\n" + "ld1 { v3.b }[4], [x22], #0x1\n" + "ld1 { v28.b }[4], [x21], #0x1\n" + "ld1 { v22.b }[4], [x20], #0x1\n" "b 23f\n" "21:" // Oddments: 4 inputs loop: Load: Bit 3: Unset: Bit 2: Unset "tbz %x[n_channels], #1, 22f\n" - "ldr h4, [x24], #0x2\n" - "ldr h3, [x23], #0x2\n" - "ldr h28, [x22], #0x2\n" - "ldr h22, [x21], #0x2\n" + "ldr h4, [x23], #0x2\n" + "ldr h3, [x22], #0x2\n" + "ldr h28, [x21], #0x2\n" + "ldr h22, [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[2], [x24], #0x1\n" - "ld1 { v3.b }[2], [x23], #0x1\n" - "ld1 { v28.b }[2], [x22], #0x1\n" - "ld1 { v22.b }[2], [x21], #0x1\n" + "ld1 { v4.b }[2], [x23], #0x1\n" + "ld1 { v3.b }[2], [x22], #0x1\n" + "ld1 { v28.b }[2], [x21], #0x1\n" + "ld1 { v22.b }[2], [x20], #0x1\n" "b 23f\n" "22:" // Oddments: 4 inputs loop: Load: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ldr b4, [x24], #0x1\n" - "ldr b3, [x23], #0x1\n" - "ldr b28, [x22], #0x1\n" - "ldr b22, [x21], #0x1\n" + "ldr b4, [x23], #0x1\n" + "ldr b3, [x22], #0x1\n" + "ldr b28, [x21], #0x1\n" + "ldr b22, [x20], #0x1\n" "23:" // Oddments: 4 inputs loop: Load: Bit 3: End - "umax v23.16b, v4.16b, v3.16b\n" - "umax v19.16b, v28.16b, v22.16b\n" + "umax v17.16b, v4.16b, v3.16b\n" + "umax v16.16b, v28.16b, v22.16b\n" "subs x25, x25, #0x1\n" - "umax v19.16b, v23.16b, v19.16b\n" - "umax v8.16b, v8.16b, v19.16b\n" + "umax v16.16b, v17.16b, v16.16b\n" + "umax v8.16b, v8.16b, v16.16b\n" "bgt 15b\n" "24:" // Oddments: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 34f\n" "25:" // Oddments: Single input loop - "ldr x24, [x20], #0x8\n" - "add x24, x24, x9\n" + "ldr x23, [x24], #0x8\n" + "add x23, x23, x27\n" "movi v4.16b, #0x0\n" "tbz %x[n_channels], #3, 29f\n" - "ldr d4, [x24], #0x8\n" + "ldr d4, [x23], #0x8\n" "tbz %x[n_channels], #2, 27f\n" - "ld1 { v4.s }[2], [x24], #0x4\n" + "ld1 { v4.s }[2], [x23], #0x4\n" "tbz %x[n_channels], #1, 26f\n" - "ld1 { v4.h }[6], [x24], #0x2\n" + "ld1 { v4.h }[6], [x23], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[14], [x24], #0x1\n" + "ld1 { v4.b }[14], [x23], #0x1\n" "b 33f\n" "26:" // Oddments: Single input loop: Load: Bit 3: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[12], [x24], #0x1\n" + "ld1 { v4.b }[12], [x23], #0x1\n" "b 33f\n" "27:" // Oddments: Single input loop: Load: Bit 3: Bit 2: Unset "tbz %x[n_channels], #1, 28f\n" - "ld1 { v4.h }[4], [x24], #0x2\n" + "ld1 { v4.h }[4], [x23], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[10], [x24], #0x1\n" + "ld1 { v4.b }[10], [x23], #0x1\n" "b 33f\n" "28:" // Oddments: Single input loop: Load: Bit 3: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[8], [x24], #0x1\n" + "ld1 { v4.b }[8], [x23], #0x1\n" "b 33f\n" "29:" // Oddments: Single input loop: Load: Bit 3: Unset "tbz %x[n_channels], #2, 31f\n" - "ldr s4, [x24], #0x4\n" + "ldr s4, [x23], #0x4\n" "tbz %x[n_channels], #1, 30f\n" - "ld1 { v4.h }[2], [x24], #0x2\n" + "ld1 { v4.h }[2], [x23], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[6], [x24], #0x1\n" + "ld1 { v4.b }[6], [x23], #0x1\n" "b 33f\n" "30:" // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[4], [x24], #0x1\n" + "ld1 { v4.b }[4], [x23], #0x1\n" "b 33f\n" "31:" // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Unset "tbz %x[n_channels], #1, 32f\n" - "ldr h4, [x24], #0x2\n" + "ldr h4, [x23], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[2], [x24], #0x1\n" + "ld1 { v4.b }[2], [x23], #0x1\n" "b 33f\n" "32:" // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ldr b4, [x24], #0x1\n" + "ldr b4, [x23], #0x1\n" "33:" // Oddments: Single input loop: Load: Bit 3: End "subs x21, x21, #0x1\n" "umax v8.16b, v8.16b, v4.16b\n" @@ -428,10 +428,11 @@ void a64_u8_nhwc_max_generic_depthfirst_impl( "43:" // End : [n_channels] "+&r" (n_channels), [outptr] "+&r" (outptr) : [inptrs] "r" (inptrs), [n_valid_cells] "r" (n_valid_cells) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27" ); } } // namespace pooling } // namespace arm_conv + #endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8q_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8q_nhwc_avg_generic_depthfirst/generic.cpp index 19227d8aaa196abb89c7e9fcff1fd6193c7a44ef..31a3489e5cdc2dbbbf58866847c06fbd7cfe7552 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8q_nhwc_avg_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8q_nhwc_avg_generic_depthfirst/generic.cpp @@ -22,8 +22,6 @@ * SOFTWARE. */ -#if defined(__aarch64__) - #include "pooling.hpp" #include #include @@ -31,6 +29,8 @@ #include +#if defined(__aarch64__) + namespace arm_conv { namespace pooling { @@ -132,7 +132,7 @@ void a64_u8q_nhwc_avg_generic_depthfirst_impl( "mov v13.16b, v15.16b\n" "mov v12.16b, v15.16b\n" "mov v11.16b, v15.16b\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "mov v10.16b, v15.16b\n" "mov v9.16b, v15.16b\n" "mov v8.16b, v15.16b\n" @@ -145,42 +145,42 @@ void a64_u8q_nhwc_avg_generic_depthfirst_impl( "mov v1.16b, v15.16b\n" "mov v0.16b, v15.16b\n" "cbz x23, 4f\n" - "ldp x22, x21, [x20, #0x0]\n" - "ldr q31, [x22, x27]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q31, [x21, x27]\n" "subs x23, x23, #0x1\n" - "add x20, x20, #0x10\n" - "ldr q30, [x21, x27]\n" - "ldr q29, [x22, x26]\n" - "ldr q28, [x21, x26]\n" - "ldr q27, [x22, x25]\n" - "ldr q26, [x21, x25]\n" - "ldr q25, [x22, x24]\n" - "ldr q24, [x21, x24]\n" + "add x22, x22, #0x10\n" + "ldr q30, [x20, x27]\n" + "ldr q29, [x21, x26]\n" + "ldr q28, [x20, x26]\n" + "ldr q27, [x21, x25]\n" + "ldr q26, [x20, x25]\n" + "ldr q25, [x21, x24]\n" + "ldr q24, [x20, x24]\n" "beq 3f\n" "2:" // 4-vectors of channels: 2 inputs loop "uaddl v23.8h, v31.8b, v30.8b\n" "uaddl2 v22.8h, v31.16b, v30.16b\n" - "ldp x22, x21, [x20, #0x0]\n" - "ldr q31, [x22, x27]\n" - "ldr q30, [x21, x27]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q31, [x21, x27]\n" + "ldr q30, [x20, x27]\n" "uaddl v21.8h, v29.8b, v28.8b\n" "uaddl2 v20.8h, v29.16b, v28.16b\n" - "ldr q29, [x22, x26]\n" - "ldr q28, [x21, x26]\n" + "ldr q29, [x21, x26]\n" + "ldr q28, [x20, x26]\n" "uaddl v19.8h, v27.8b, v26.8b\n" "uaddl2 v18.8h, v27.16b, v26.16b\n" - "ldr q27, [x22, x25]\n" - "ldr q26, [x21, x25]\n" + "ldr q27, [x21, x25]\n" + "ldr q26, [x20, x25]\n" + "uaddl v17.8h, v25.8b, v24.8b\n" + "uaddl2 v16.8h, v25.16b, v24.16b\n" + "ldr q25, [x21, x24]\n" + "ldr q24, [x20, x24]\n" "subs x23, x23, #0x1\n" "uaddw v15.4s, v15.4s, v23.4h\n" "uaddw2 v14.4s, v14.4s, v23.8h\n" - "uaddl v17.8h, v25.8b, v24.8b\n" - "uaddl2 v16.8h, v25.16b, v24.16b\n" - "ldr q25, [x22, x24]\n" - "add x20, x20, #0x10\n" "uaddw v13.4s, v13.4s, v22.4h\n" "uaddw2 v12.4s, v12.4s, v22.8h\n" - "ldr q24, [x21, x24]\n" + "add x22, x22, #0x10\n" "uaddw v11.4s, v11.4s, v21.4h\n" "uaddw2 v10.4s, v10.4s, v21.8h\n" "uaddw v9.4s, v9.4s, v20.4h\n" @@ -220,23 +220,23 @@ void a64_u8q_nhwc_avg_generic_depthfirst_impl( "uaddw v1.4s, v1.4s, v16.4h\n" "uaddw2 v0.4s, v0.4s, v16.8h\n" "4:" // 4-vectors of channels: After loop - "ands x21, %x[n_valid_cells], #0x1\n" + "ands x23, %x[n_valid_cells], #0x1\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x22, [x20], #0x8\n" - "ldr q31, [x22, x27]\n" - "uxtl v23.8h, v31.8b\n" - "uxtl2 v22.8h, v31.16b\n" - "ldr q29, [x22, x26]\n" - "ldr q27, [x22, x25]\n" - "uxtl v21.8h, v29.8b\n" - "uxtl2 v20.8h, v29.16b\n" - "ldr q25, [x22, x24]\n" - "uxtl v19.8h, v27.8b\n" - "uxtl2 v18.8h, v27.16b\n" - "subs x21, x21, #0x1\n" - "uxtl v17.8h, v25.8b\n" - "uxtl2 v16.8h, v25.16b\n" + "ldr x20, [x22], #0x8\n" + "ldr q16, [x20, x27]\n" + "uxtl v23.8h, v16.8b\n" + "uxtl2 v22.8h, v16.16b\n" + "ldr q16, [x20, x26]\n" + "ldr q17, [x20, x25]\n" + "uxtl v21.8h, v16.8b\n" + "uxtl2 v20.8h, v16.16b\n" + "ldr q16, [x20, x24]\n" + "uxtl v19.8h, v17.8b\n" + "uxtl2 v18.8h, v17.16b\n" + "subs x23, x23, #0x1\n" + "uxtl v17.8h, v16.8b\n" + "uxtl2 v16.8h, v16.16b\n" "uaddw v15.4s, v15.4s, v23.4h\n" "uaddw2 v14.4s, v14.4s, v23.8h\n" "uaddw v13.4s, v13.4s, v22.4h\n" @@ -391,56 +391,56 @@ void a64_u8q_nhwc_avg_generic_depthfirst_impl( "mov v14.16b, v15.16b\n" "mov v13.16b, v15.16b\n" "mov v12.16b, v15.16b\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "cbz x23, 11f\n" - "ldp x22, x21, [x20, #0x0]\n" - "ldr q31, [x22, x27]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q31, [x21, x27]\n" "subs x23, x23, #0x1\n" - "add x20, x20, #0x10\n" - "ldr q30, [x21, x27]\n" + "add x22, x22, #0x10\n" + "ldr q30, [x20, x27]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 2 inputs loop - "uaddl v23.8h, v31.8b, v30.8b\n" - "uaddl2 v22.8h, v31.16b, v30.16b\n" - "ldp x22, x21, [x20, #0x0]\n" - "ldr q31, [x22, x27]\n" - "ldr q30, [x21, x27]\n" + "uaddl v17.8h, v31.8b, v30.8b\n" + "uaddl2 v16.8h, v31.16b, v30.16b\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q31, [x21, x27]\n" + "ldr q30, [x20, x27]\n" "subs x23, x23, #0x1\n" - "uaddw v15.4s, v15.4s, v23.4h\n" - "uaddw2 v14.4s, v14.4s, v23.8h\n" - "uaddw v13.4s, v13.4s, v22.4h\n" - "uaddw2 v12.4s, v12.4s, v22.8h\n" - "add x20, x20, #0x10\n" + "uaddw v15.4s, v15.4s, v17.4h\n" + "uaddw2 v14.4s, v14.4s, v17.8h\n" + "uaddw v13.4s, v13.4s, v16.4h\n" + "uaddw2 v12.4s, v12.4s, v16.8h\n" + "add x22, x22, #0x10\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 2 inputs tail - "uaddl v23.8h, v31.8b, v30.8b\n" - "uaddl2 v22.8h, v31.16b, v30.16b\n" - "uaddw v15.4s, v15.4s, v23.4h\n" - "uaddw2 v14.4s, v14.4s, v23.8h\n" - "uaddw v13.4s, v13.4s, v22.4h\n" - "uaddw2 v12.4s, v12.4s, v22.8h\n" + "uaddl v17.8h, v31.8b, v30.8b\n" + "uaddl2 v16.8h, v31.16b, v30.16b\n" + "uaddw v15.4s, v15.4s, v17.4h\n" + "uaddw2 v14.4s, v14.4s, v17.8h\n" + "uaddw v13.4s, v13.4s, v16.4h\n" + "uaddw2 v12.4s, v12.4s, v16.8h\n" "11:" // Single vector of channels: Loop: After loop - "ands x21, %x[n_valid_cells], #0x1\n" + "ands x23, %x[n_valid_cells], #0x1\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x22, [x20], #0x8\n" - "ldr q31, [x22, x27]\n" - "uxtl v23.8h, v31.8b\n" - "uxtl2 v22.8h, v31.16b\n" - "subs x21, x21, #0x1\n" - "uaddw v15.4s, v15.4s, v23.4h\n" - "uaddw2 v14.4s, v14.4s, v23.8h\n" - "uaddw v13.4s, v13.4s, v22.4h\n" - "uaddw2 v12.4s, v12.4s, v22.8h\n" + "ldr x20, [x22], #0x8\n" + "ldr q16, [x20, x27]\n" + "uxtl v17.8h, v16.8b\n" + "uxtl2 v16.8h, v16.16b\n" + "subs x23, x23, #0x1\n" + "uaddw v15.4s, v15.4s, v17.4h\n" + "uaddw2 v14.4s, v14.4s, v17.8h\n" + "uaddw v13.4s, v13.4s, v16.4h\n" + "uaddw2 v12.4s, v12.4s, v16.8h\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End - "ld1r { v19.4s }, [%x[left_shift]]\n" + "ld1r { v16.4s }, [%x[left_shift]]\n" "ld1r { v18.4s }, [%x[combined_rescale_value]]\n" - "srshl v15.4s, v15.4s, v19.4s\n" - "srshl v14.4s, v14.4s, v19.4s\n" + "srshl v15.4s, v15.4s, v16.4s\n" + "srshl v14.4s, v14.4s, v16.4s\n" "ld1r { v17.4s }, [%x[right_shift]]\n" - "srshl v13.4s, v13.4s, v19.4s\n" - "srshl v12.4s, v12.4s, v19.4s\n" + "srshl v13.4s, v13.4s, v16.4s\n" + "srshl v12.4s, v12.4s, v16.4s\n" "add x20, %x[quant_params], %[offsetof_qp_output_offset]\n" "ld1r { v16.4s }, [x20]\n" "sqrdmulh v15.4s, v15.4s, v18.4s\n" @@ -467,9 +467,9 @@ void a64_u8q_nhwc_avg_generic_depthfirst_impl( "smin v14.4s, v14.4s, v16.4s\n" "smin v13.4s, v13.4s, v16.4s\n" "smin v12.4s, v12.4s, v16.4s\n" - "uzp1 v23.16b, v15.16b, v14.16b\n" + "uzp1 v17.16b, v15.16b, v14.16b\n" "uzp1 v16.16b, v13.16b, v12.16b\n" - "uzp1 v16.16b, v23.16b, v16.16b\n" + "uzp1 v16.16b, v17.16b, v16.16b\n" "str q16, [%x[outptr], x27]\n" "add x27, x27, #0x10\n" "bge 8b\n" @@ -481,151 +481,151 @@ void a64_u8q_nhwc_avg_generic_depthfirst_impl( "mov v14.16b, v15.16b\n" "mov v13.16b, v15.16b\n" "mov v12.16b, v15.16b\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "cbz x23, 24f\n" "15:" // Oddments: 2 inputs loop - "ldp x22, x21, [x20, #0x0]\n" - "add x20, x20, #0x10\n" - "add x22, x22, x27\n" - "movi v31.16b, #0x0\n" + "ldp x21, x20, [x22, #0x0]\n" + "add x22, x22, #0x10\n" "add x21, x21, x27\n" + "movi v31.16b, #0x0\n" + "add x20, x20, x27\n" "movi v30.16b, #0x0\n" "tbz %x[n_channels], #3, 19f\n" - "ldr d31, [x22], #0x8\n" - "ldr d30, [x21], #0x8\n" + "ldr d31, [x21], #0x8\n" + "ldr d30, [x20], #0x8\n" "tbz %x[n_channels], #2, 17f\n" - "ld1 { v31.s }[2], [x22], #0x4\n" - "ld1 { v30.s }[2], [x21], #0x4\n" + "ld1 { v31.s }[2], [x21], #0x4\n" + "ld1 { v30.s }[2], [x20], #0x4\n" "tbz %x[n_channels], #1, 16f\n" - "ld1 { v31.h }[6], [x22], #0x2\n" - "ld1 { v30.h }[6], [x21], #0x2\n" + "ld1 { v31.h }[6], [x21], #0x2\n" + "ld1 { v30.h }[6], [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[14], [x22], #0x1\n" - "ld1 { v30.b }[14], [x21], #0x1\n" + "ld1 { v31.b }[14], [x21], #0x1\n" + "ld1 { v30.b }[14], [x20], #0x1\n" "b 23f\n" "16:" // Oddments: 2 inputs loop: Load: Bit 3: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[12], [x22], #0x1\n" - "ld1 { v30.b }[12], [x21], #0x1\n" + "ld1 { v31.b }[12], [x21], #0x1\n" + "ld1 { v30.b }[12], [x20], #0x1\n" "b 23f\n" "17:" // Oddments: 2 inputs loop: Load: Bit 3: Bit 2: Unset "tbz %x[n_channels], #1, 18f\n" - "ld1 { v31.h }[4], [x22], #0x2\n" - "ld1 { v30.h }[4], [x21], #0x2\n" + "ld1 { v31.h }[4], [x21], #0x2\n" + "ld1 { v30.h }[4], [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[10], [x22], #0x1\n" - "ld1 { v30.b }[10], [x21], #0x1\n" + "ld1 { v31.b }[10], [x21], #0x1\n" + "ld1 { v30.b }[10], [x20], #0x1\n" "b 23f\n" "18:" // Oddments: 2 inputs loop: Load: Bit 3: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[8], [x22], #0x1\n" - "ld1 { v30.b }[8], [x21], #0x1\n" + "ld1 { v31.b }[8], [x21], #0x1\n" + "ld1 { v30.b }[8], [x20], #0x1\n" "b 23f\n" "19:" // Oddments: 2 inputs loop: Load: Bit 3: Unset "tbz %x[n_channels], #2, 21f\n" - "ldr s31, [x22], #0x4\n" - "ldr s30, [x21], #0x4\n" + "ldr s31, [x21], #0x4\n" + "ldr s30, [x20], #0x4\n" "tbz %x[n_channels], #1, 20f\n" - "ld1 { v31.h }[2], [x22], #0x2\n" - "ld1 { v30.h }[2], [x21], #0x2\n" + "ld1 { v31.h }[2], [x21], #0x2\n" + "ld1 { v30.h }[2], [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[6], [x22], #0x1\n" - "ld1 { v30.b }[6], [x21], #0x1\n" + "ld1 { v31.b }[6], [x21], #0x1\n" + "ld1 { v30.b }[6], [x20], #0x1\n" "b 23f\n" "20:" // Oddments: 2 inputs loop: Load: Bit 3: Unset: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[4], [x22], #0x1\n" - "ld1 { v30.b }[4], [x21], #0x1\n" + "ld1 { v31.b }[4], [x21], #0x1\n" + "ld1 { v30.b }[4], [x20], #0x1\n" "b 23f\n" "21:" // Oddments: 2 inputs loop: Load: Bit 3: Unset: Bit 2: Unset "tbz %x[n_channels], #1, 22f\n" - "ldr h31, [x22], #0x2\n" - "ldr h30, [x21], #0x2\n" + "ldr h31, [x21], #0x2\n" + "ldr h30, [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v31.b }[2], [x22], #0x1\n" - "ld1 { v30.b }[2], [x21], #0x1\n" + "ld1 { v31.b }[2], [x21], #0x1\n" + "ld1 { v30.b }[2], [x20], #0x1\n" "b 23f\n" "22:" // Oddments: 2 inputs loop: Load: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ldr b31, [x22], #0x1\n" - "ldr b30, [x21], #0x1\n" + "ldr b31, [x21], #0x1\n" + "ldr b30, [x20], #0x1\n" "23:" // Oddments: 2 inputs loop: Load: Bit 3: End - "uaddl v23.8h, v31.8b, v30.8b\n" - "uaddl2 v22.8h, v31.16b, v30.16b\n" + "uaddl v17.8h, v31.8b, v30.8b\n" + "uaddl2 v16.8h, v31.16b, v30.16b\n" "subs x23, x23, #0x1\n" - "uaddw v15.4s, v15.4s, v23.4h\n" - "uaddw2 v14.4s, v14.4s, v23.8h\n" - "uaddw v13.4s, v13.4s, v22.4h\n" - "uaddw2 v12.4s, v12.4s, v22.8h\n" + "uaddw v15.4s, v15.4s, v17.4h\n" + "uaddw2 v14.4s, v14.4s, v17.8h\n" + "uaddw v13.4s, v13.4s, v16.4h\n" + "uaddw2 v12.4s, v12.4s, v16.8h\n" "bgt 15b\n" "24:" // Oddments: After loop - "ands x21, %x[n_valid_cells], #0x1\n" + "ands x23, %x[n_valid_cells], #0x1\n" "beq 34f\n" "25:" // Oddments: Single input loop - "ldr x22, [x20], #0x8\n" - "add x22, x22, x27\n" + "ldr x21, [x22], #0x8\n" + "add x21, x21, x27\n" "movi v31.16b, #0x0\n" "tbz %x[n_channels], #3, 29f\n" - "ldr d31, [x22], #0x8\n" + "ldr d31, [x21], #0x8\n" "tbz %x[n_channels], #2, 27f\n" - "ld1 { v31.s }[2], [x22], #0x4\n" + "ld1 { v31.s }[2], [x21], #0x4\n" "tbz %x[n_channels], #1, 26f\n" - "ld1 { v31.h }[6], [x22], #0x2\n" + "ld1 { v31.h }[6], [x21], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[14], [x22], #0x1\n" + "ld1 { v31.b }[14], [x21], #0x1\n" "b 33f\n" "26:" // Oddments: Single input loop: Load: Bit 3: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[12], [x22], #0x1\n" + "ld1 { v31.b }[12], [x21], #0x1\n" "b 33f\n" "27:" // Oddments: Single input loop: Load: Bit 3: Bit 2: Unset "tbz %x[n_channels], #1, 28f\n" - "ld1 { v31.h }[4], [x22], #0x2\n" + "ld1 { v31.h }[4], [x21], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[10], [x22], #0x1\n" + "ld1 { v31.b }[10], [x21], #0x1\n" "b 33f\n" "28:" // Oddments: Single input loop: Load: Bit 3: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[8], [x22], #0x1\n" + "ld1 { v31.b }[8], [x21], #0x1\n" "b 33f\n" "29:" // Oddments: Single input loop: Load: Bit 3: Unset "tbz %x[n_channels], #2, 31f\n" - "ldr s31, [x22], #0x4\n" + "ldr s31, [x21], #0x4\n" "tbz %x[n_channels], #1, 30f\n" - "ld1 { v31.h }[2], [x22], #0x2\n" + "ld1 { v31.h }[2], [x21], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[6], [x22], #0x1\n" + "ld1 { v31.b }[6], [x21], #0x1\n" "b 33f\n" "30:" // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[4], [x22], #0x1\n" + "ld1 { v31.b }[4], [x21], #0x1\n" "b 33f\n" "31:" // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Unset "tbz %x[n_channels], #1, 32f\n" - "ldr h31, [x22], #0x2\n" + "ldr h31, [x21], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v31.b }[2], [x22], #0x1\n" + "ld1 { v31.b }[2], [x21], #0x1\n" "b 33f\n" "32:" // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ldr b31, [x22], #0x1\n" + "ldr b31, [x21], #0x1\n" "33:" // Oddments: Single input loop: Load: Bit 3: End - "uxtl v23.8h, v31.8b\n" - "uxtl2 v22.8h, v31.16b\n" - "subs x21, x21, #0x1\n" - "uaddw v15.4s, v15.4s, v23.4h\n" - "uaddw2 v14.4s, v14.4s, v23.8h\n" - "uaddw v13.4s, v13.4s, v22.4h\n" - "uaddw2 v12.4s, v12.4s, v22.8h\n" + "uxtl v17.8h, v31.8b\n" + "uxtl2 v16.8h, v31.16b\n" + "subs x23, x23, #0x1\n" + "uaddw v15.4s, v15.4s, v17.4h\n" + "uaddw2 v14.4s, v14.4s, v17.8h\n" + "uaddw v13.4s, v13.4s, v16.4h\n" + "uaddw2 v12.4s, v12.4s, v16.8h\n" "bgt 25b\n" "34:" // Oddments: Single input loop: End - "ld1r { v19.4s }, [%x[left_shift]]\n" + "ld1r { v16.4s }, [%x[left_shift]]\n" "ld1r { v18.4s }, [%x[combined_rescale_value]]\n" - "srshl v15.4s, v15.4s, v19.4s\n" - "srshl v14.4s, v14.4s, v19.4s\n" + "srshl v15.4s, v15.4s, v16.4s\n" + "srshl v14.4s, v14.4s, v16.4s\n" "ld1r { v17.4s }, [%x[right_shift]]\n" - "srshl v13.4s, v13.4s, v19.4s\n" - "srshl v12.4s, v12.4s, v19.4s\n" + "srshl v13.4s, v13.4s, v16.4s\n" + "srshl v12.4s, v12.4s, v16.4s\n" "add x20, %x[quant_params], %[offsetof_qp_output_offset]\n" "ld1r { v16.4s }, [x20]\n" "sqrdmulh v15.4s, v15.4s, v18.4s\n" @@ -650,9 +650,9 @@ void a64_u8q_nhwc_avg_generic_depthfirst_impl( "smin v14.4s, v14.4s, v16.4s\n" "smin v13.4s, v13.4s, v16.4s\n" "smin v12.4s, v12.4s, v16.4s\n" - "uzp1 v23.16b, v15.16b, v14.16b\n" + "uzp1 v17.16b, v15.16b, v14.16b\n" "uzp1 v16.16b, v13.16b, v12.16b\n" - "uzp1 v16.16b, v23.16b, v16.16b\n" + "uzp1 v16.16b, v17.16b, v16.16b\n" "tbz %x[n_channels], #3, 38f\n" "st1 { v16.d }[0], [%x[outptr]], #0x8\n" "tbz %x[n_channels], #2, 36f\n" @@ -707,4 +707,5 @@ void a64_u8q_nhwc_avg_generic_depthfirst_impl( } // namespace pooling } // namespace arm_conv + #endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8q_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8q_nhwc_max_generic_depthfirst/generic.cpp index 7eea14f70fb28a52451636f1bf4051199859826a..f4927c55365e51d58de61588337457217b488e92 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8q_nhwc_max_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8q_nhwc_max_generic_depthfirst/generic.cpp @@ -22,12 +22,12 @@ * SOFTWARE. */ -#if defined(__aarch64__) - #include "pooling.hpp" #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace pooling { @@ -43,77 +43,77 @@ void a64_u8q_nhwc_max_generic_depthfirst_impl( { __asm__ __volatile__( "cmp %x[n_channels], #0x40\n" - "mov x9, #0x0\n" - "mov x28, #0x10\n" // cntb _, ALL, #1 - "mov x27, #0x20\n" // cntb _, ALL, #2 - "mov x26, #0x30\n" // cntb _, ALL, #3 + "mov x27, #0x0\n" + "mov x26, #0x10\n" // cntb _, ALL, #1 + "mov x24, #0x20\n" // cntb _, ALL, #2 + "mov x23, #0x30\n" // cntb _, ALL, #3 "blt 7f\n" "1:" // 4-vectors of channels "lsr x25, %x[n_valid_cells], #0x2\n" "movi v8.16b, #0x0\n" "movi v7.16b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "movi v6.16b, #0x0\n" "movi v5.16b, #0x0\n" "cbz x25, 4f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" "subs x25, x25, #0x1\n" - "ldr q3, [x23, x9]\n" - "ldr q2, [x24, x28]\n" - "ldr q1, [x23, x28]\n" - "ldr q0, [x24, x27]\n" - "ldr q31, [x23, x27]\n" - "ldr q30, [x24, x26]\n" - "ldr q29, [x23, x26]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" - "ldr q27, [x22, x28]\n" - "ldr q21, [x21, x28]\n" - "ldr q26, [x22, x27]\n" - "ldr q20, [x21, x27]\n" - "ldr q25, [x22, x26]\n" - "ldr q24, [x21, x26]\n" + "ldr q3, [x20, x27]\n" + "ldr q2, [x21, x26]\n" + "ldr q1, [x20, x26]\n" + "ldr q0, [x21, x24]\n" + "ldr q31, [x20, x24]\n" + "ldr q30, [x21, x23]\n" + "ldr q29, [x20, x23]\n" + "ldp x21, x20, [x22, #0x10]\n" + "add x22, x22, #0x20\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" + "ldr q27, [x21, x26]\n" + "ldr q21, [x20, x26]\n" + "ldr q26, [x21, x24]\n" + "ldr q20, [x20, x24]\n" + "ldr q25, [x21, x23]\n" + "ldr q24, [x20, x23]\n" "beq 3f\n" "2:" // 4-vectors of channels: 4 inputs loop "umax v23.16b, v4.16b, v3.16b\n" "umax v19.16b, v28.16b, v22.16b\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" - "ldr q3, [x23, x9]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" + "ldr q3, [x20, x27]\n" "umax v22.16b, v2.16b, v1.16b\n" - "ldr q2, [x24, x28]\n" + "ldr q2, [x21, x26]\n" "umax v18.16b, v27.16b, v21.16b\n" - "ldr q1, [x23, x28]\n" + "ldr q1, [x20, x26]\n" "umax v21.16b, v0.16b, v31.16b\n" - "ldr q0, [x24, x27]\n" + "ldr q0, [x21, x24]\n" "umax v17.16b, v26.16b, v20.16b\n" - "ldr q31, [x23, x27]\n" + "ldr q31, [x20, x24]\n" "umax v20.16b, v30.16b, v29.16b\n" - "ldr q30, [x24, x26]\n" + "ldr q30, [x21, x23]\n" "umax v16.16b, v25.16b, v24.16b\n" - "ldr q29, [x23, x26]\n" + "ldr q29, [x20, x23]\n" "umax v19.16b, v23.16b, v19.16b\n" "umax v18.16b, v22.16b, v18.16b\n" - "ldp x22, x21, [x20, #0x10]\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" + "ldp x21, x20, [x22, #0x10]\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" "umax v17.16b, v21.16b, v17.16b\n" "umax v16.16b, v20.16b, v16.16b\n" - "ldr q27, [x22, x28]\n" - "ldr q21, [x21, x28]\n" + "ldr q27, [x21, x26]\n" + "ldr q21, [x20, x26]\n" "subs x25, x25, #0x1\n" "umax v8.16b, v8.16b, v19.16b\n" - "ldr q26, [x22, x27]\n" - "ldr q20, [x21, x27]\n" + "ldr q26, [x21, x24]\n" + "ldr q20, [x20, x24]\n" "umax v7.16b, v7.16b, v18.16b\n" "umax v6.16b, v6.16b, v17.16b\n" - "ldr q25, [x22, x26]\n" - "ldr q24, [x21, x26]\n" + "ldr q25, [x21, x23]\n" + "ldr q24, [x20, x23]\n" "umax v5.16b, v5.16b, v16.16b\n" - "add x20, x20, #0x20\n" + "add x22, x22, #0x20\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "umax v23.16b, v4.16b, v3.16b\n" @@ -136,16 +136,16 @@ void a64_u8q_nhwc_max_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x24, [x20], #0x8\n" - "ldr q4, [x24, x9]\n" + "ldr x20, [x22], #0x8\n" + "ldr q16, [x20, x27]\n" "subs x21, x21, #0x1\n" - "umax v8.16b, v8.16b, v4.16b\n" - "ldr q2, [x24, x28]\n" - "ldr q0, [x24, x27]\n" - "umax v7.16b, v7.16b, v2.16b\n" - "umax v6.16b, v6.16b, v0.16b\n" - "ldr q30, [x24, x26]\n" - "umax v5.16b, v5.16b, v30.16b\n" + "umax v8.16b, v8.16b, v16.16b\n" + "ldr q17, [x20, x26]\n" + "ldr q16, [x20, x24]\n" + "umax v7.16b, v7.16b, v17.16b\n" + "umax v6.16b, v6.16b, v16.16b\n" + "ldr q16, [x20, x23]\n" + "umax v5.16b, v5.16b, v16.16b\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End "add x20, %x[quant_params], %[offsetof_qp_input_offset]\n" @@ -292,17 +292,17 @@ void a64_u8q_nhwc_max_generic_depthfirst_impl( "uzp1 v19.16b, v25.16b, v19.16b\n" "uzp1 v18.16b, v24.16b, v18.16b\n" "uzp1 v16.16b, v23.16b, v16.16b\n" - "str q16, [%x[outptr], x9]\n" - "add x9, x9, #0x40\n" + "str q16, [%x[outptr], x27]\n" + "add x27, x27, #0x40\n" "uzp1 v16.16b, v22.16b, v21.16b\n" "uzp1 v17.16b, v20.16b, v17.16b\n" - "str q16, [%x[outptr], x28]\n" - "add x28, x28, #0x40\n" - "uzp1 v16.16b, v19.16b, v18.16b\n" - "str q17, [%x[outptr], x27]\n" - "add x27, x27, #0x40\n" "str q16, [%x[outptr], x26]\n" "add x26, x26, #0x40\n" + "uzp1 v16.16b, v19.16b, v18.16b\n" + "str q17, [%x[outptr], x24]\n" + "add x24, x24, #0x40\n" + "str q16, [%x[outptr], x23]\n" + "add x23, x23, #0x40\n" "bge 1b\n" "cbz %x[n_channels], 43f\n" "7:" // Single vector of channels @@ -311,314 +311,314 @@ void a64_u8q_nhwc_max_generic_depthfirst_impl( "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" "movi v8.16b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" "subs x25, x25, #0x1\n" - "ldr q3, [x23, x9]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" + "ldr q3, [x20, x27]\n" + "ldp x21, x20, [x22, #0x10]\n" + "add x22, x22, #0x20\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 4 inputs loop - "umax v23.16b, v4.16b, v3.16b\n" - "umax v19.16b, v28.16b, v22.16b\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldr q4, [x24, x9]\n" - "ldr q3, [x23, x9]\n" - "umax v19.16b, v23.16b, v19.16b\n" - "ldp x22, x21, [x20, #0x10]\n" + "umax v17.16b, v4.16b, v3.16b\n" + "umax v16.16b, v28.16b, v22.16b\n" + "ldp x21, x20, [x22, #0x0]\n" + "ldr q4, [x21, x27]\n" + "ldr q3, [x20, x27]\n" + "umax v16.16b, v17.16b, v16.16b\n" + "ldp x21, x20, [x22, #0x10]\n" "subs x25, x25, #0x1\n" - "ldr q28, [x22, x9]\n" - "ldr q22, [x21, x9]\n" - "umax v8.16b, v8.16b, v19.16b\n" - "add x20, x20, #0x20\n" + "ldr q28, [x21, x27]\n" + "ldr q22, [x20, x27]\n" + "umax v8.16b, v8.16b, v16.16b\n" + "add x22, x22, #0x20\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail - "umax v23.16b, v4.16b, v3.16b\n" - "umax v19.16b, v28.16b, v22.16b\n" - "umax v19.16b, v23.16b, v19.16b\n" - "umax v8.16b, v8.16b, v19.16b\n" + "umax v17.16b, v4.16b, v3.16b\n" + "umax v16.16b, v28.16b, v22.16b\n" + "umax v16.16b, v17.16b, v16.16b\n" + "umax v8.16b, v8.16b, v16.16b\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x24, [x20], #0x8\n" - "ldr q4, [x24, x9]\n" + "ldr x20, [x22], #0x8\n" + "ldr q16, [x20, x27]\n" "subs x21, x21, #0x1\n" - "umax v8.16b, v8.16b, v4.16b\n" + "umax v8.16b, v8.16b, v16.16b\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "add x20, %x[quant_params], %[offsetof_qp_input_offset]\n" - "ld1r { v4.4s }, [x20]\n" - "uxtl v23.8h, v8.8b\n" - "uxtl2 v24.8h, v8.16b\n" - "neg v4.4s, v4.4s\n" + "ld1r { v18.4s }, [x20]\n" + "uxtl v17.8h, v8.8b\n" + "uxtl2 v16.8h, v8.16b\n" + "neg v18.4s, v18.4s\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n" - "ld1r { v3.4s }, [x20]\n" - "saddw v0.4s, v4.4s, v23.4h\n" - "saddw2 v23.4s, v4.4s, v23.8h\n" - "saddw v31.4s, v4.4s, v24.4h\n" + "ld1r { v23.4s }, [x20]\n" + "saddw v22.4s, v18.4s, v17.4h\n" + "saddw2 v21.4s, v18.4s, v17.8h\n" + "saddw v20.4s, v18.4s, v16.4h\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_mul]\n" - "ld1r { v2.4s }, [x20]\n" - "saddw2 v30.4s, v4.4s, v24.8h\n" - "srshl v0.4s, v0.4s, v3.4s\n" + "ld1r { v19.4s }, [x20]\n" + "saddw2 v18.4s, v18.4s, v16.8h\n" + "srshl v22.4s, v22.4s, v23.4s\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n" - "ld1r { v1.4s }, [x20]\n" - "srshl v23.4s, v23.4s, v3.4s\n" - "srshl v31.4s, v31.4s, v3.4s\n" + "ld1r { v17.4s }, [x20]\n" + "srshl v21.4s, v21.4s, v23.4s\n" + "srshl v20.4s, v20.4s, v23.4s\n" "add x20, %x[quant_params], %[offsetof_qp_output_offset]\n" "ld1r { v16.4s }, [x20]\n" - "srshl v30.4s, v30.4s, v3.4s\n" - "sqrdmulh v0.4s, v0.4s, v2.4s\n" + "srshl v18.4s, v18.4s, v23.4s\n" + "sqrdmulh v22.4s, v22.4s, v19.4s\n" "sub %x[n_channels], %x[n_channels], #0x10\n" "cmp %x[n_channels], #0x10\n" - "sqrdmulh v23.4s, v23.4s, v2.4s\n" - "sqrdmulh v31.4s, v31.4s, v2.4s\n" - "sqrdmulh v30.4s, v30.4s, v2.4s\n" - "srshl v0.4s, v0.4s, v1.4s\n" - "srshl v23.4s, v23.4s, v1.4s\n" - "srshl v31.4s, v31.4s, v1.4s\n" - "srshl v30.4s, v30.4s, v1.4s\n" - "add v0.4s, v0.4s, v16.4s\n" - "add v23.4s, v23.4s, v16.4s\n" - "add v31.4s, v31.4s, v16.4s\n" - "add v30.4s, v30.4s, v16.4s\n" + "sqrdmulh v21.4s, v21.4s, v19.4s\n" + "sqrdmulh v20.4s, v20.4s, v19.4s\n" + "sqrdmulh v18.4s, v18.4s, v19.4s\n" + "srshl v22.4s, v22.4s, v17.4s\n" + "srshl v21.4s, v21.4s, v17.4s\n" + "srshl v20.4s, v20.4s, v17.4s\n" + "srshl v18.4s, v18.4s, v17.4s\n" + "add v22.4s, v22.4s, v16.4s\n" + "add v21.4s, v21.4s, v16.4s\n" + "add v20.4s, v20.4s, v16.4s\n" + "add v18.4s, v18.4s, v16.4s\n" "movi v16.4s, #0x0\n" - "smax v0.4s, v0.4s, v16.4s\n" - "smax v23.4s, v23.4s, v16.4s\n" - "smax v31.4s, v31.4s, v16.4s\n" - "smax v30.4s, v30.4s, v16.4s\n" + "smax v22.4s, v22.4s, v16.4s\n" + "smax v21.4s, v21.4s, v16.4s\n" + "smax v20.4s, v20.4s, v16.4s\n" + "smax v18.4s, v18.4s, v16.4s\n" "movi v16.4s, #0xff\n" - "smin v0.4s, v0.4s, v16.4s\n" - "smin v23.4s, v23.4s, v16.4s\n" - "smin v31.4s, v31.4s, v16.4s\n" - "smin v30.4s, v30.4s, v16.4s\n" - "uzp1 v23.16b, v0.16b, v23.16b\n" - "uzp1 v16.16b, v31.16b, v30.16b\n" - "uzp1 v16.16b, v23.16b, v16.16b\n" - "str q16, [%x[outptr], x9]\n" - "add x9, x9, #0x10\n" + "smin v22.4s, v22.4s, v16.4s\n" + "smin v21.4s, v21.4s, v16.4s\n" + "smin v20.4s, v20.4s, v16.4s\n" + "smin v18.4s, v18.4s, v16.4s\n" + "uzp1 v17.16b, v22.16b, v21.16b\n" + "uzp1 v16.16b, v20.16b, v18.16b\n" + "uzp1 v16.16b, v17.16b, v16.16b\n" + "str q16, [%x[outptr], x27]\n" + "add x27, x27, #0x10\n" "bge 8b\n" "cbz %x[n_channels], 43f\n" "14:" // Oddments "lsr x25, %x[n_valid_cells], #0x2\n" - "add %x[outptr], %x[outptr], x9\n" + "add %x[outptr], %x[outptr], x27\n" "movi v8.16b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "cbz x25, 24f\n" "15:" // Oddments: 4 inputs loop - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "add x24, x24, x9\n" - "add x23, x23, x9\n" - "add x22, x22, x9\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" + "add x23, x23, x27\n" + "add x22, x22, x27\n" + "add x21, x21, x27\n" "movi v4.16b, #0x0\n" "movi v3.16b, #0x0\n" - "add x21, x21, x9\n" + "add x20, x20, x27\n" "movi v28.16b, #0x0\n" "movi v22.16b, #0x0\n" "tbz %x[n_channels], #3, 19f\n" - "ldr d4, [x24], #0x8\n" - "ldr d3, [x23], #0x8\n" - "ldr d28, [x22], #0x8\n" - "ldr d22, [x21], #0x8\n" + "ldr d4, [x23], #0x8\n" + "ldr d3, [x22], #0x8\n" + "ldr d28, [x21], #0x8\n" + "ldr d22, [x20], #0x8\n" "tbz %x[n_channels], #2, 17f\n" - "ld1 { v4.s }[2], [x24], #0x4\n" - "ld1 { v3.s }[2], [x23], #0x4\n" - "ld1 { v28.s }[2], [x22], #0x4\n" - "ld1 { v22.s }[2], [x21], #0x4\n" + "ld1 { v4.s }[2], [x23], #0x4\n" + "ld1 { v3.s }[2], [x22], #0x4\n" + "ld1 { v28.s }[2], [x21], #0x4\n" + "ld1 { v22.s }[2], [x20], #0x4\n" "tbz %x[n_channels], #1, 16f\n" - "ld1 { v4.h }[6], [x24], #0x2\n" - "ld1 { v3.h }[6], [x23], #0x2\n" - "ld1 { v28.h }[6], [x22], #0x2\n" - "ld1 { v22.h }[6], [x21], #0x2\n" + "ld1 { v4.h }[6], [x23], #0x2\n" + "ld1 { v3.h }[6], [x22], #0x2\n" + "ld1 { v28.h }[6], [x21], #0x2\n" + "ld1 { v22.h }[6], [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[14], [x24], #0x1\n" - "ld1 { v3.b }[14], [x23], #0x1\n" - "ld1 { v28.b }[14], [x22], #0x1\n" - "ld1 { v22.b }[14], [x21], #0x1\n" + "ld1 { v4.b }[14], [x23], #0x1\n" + "ld1 { v3.b }[14], [x22], #0x1\n" + "ld1 { v28.b }[14], [x21], #0x1\n" + "ld1 { v22.b }[14], [x20], #0x1\n" "b 23f\n" "16:" // Oddments: 4 inputs loop: Load: Bit 3: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[12], [x24], #0x1\n" - "ld1 { v3.b }[12], [x23], #0x1\n" - "ld1 { v28.b }[12], [x22], #0x1\n" - "ld1 { v22.b }[12], [x21], #0x1\n" + "ld1 { v4.b }[12], [x23], #0x1\n" + "ld1 { v3.b }[12], [x22], #0x1\n" + "ld1 { v28.b }[12], [x21], #0x1\n" + "ld1 { v22.b }[12], [x20], #0x1\n" "b 23f\n" "17:" // Oddments: 4 inputs loop: Load: Bit 3: Bit 2: Unset "tbz %x[n_channels], #1, 18f\n" - "ld1 { v4.h }[4], [x24], #0x2\n" - "ld1 { v3.h }[4], [x23], #0x2\n" - "ld1 { v28.h }[4], [x22], #0x2\n" - "ld1 { v22.h }[4], [x21], #0x2\n" + "ld1 { v4.h }[4], [x23], #0x2\n" + "ld1 { v3.h }[4], [x22], #0x2\n" + "ld1 { v28.h }[4], [x21], #0x2\n" + "ld1 { v22.h }[4], [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[10], [x24], #0x1\n" - "ld1 { v3.b }[10], [x23], #0x1\n" - "ld1 { v28.b }[10], [x22], #0x1\n" - "ld1 { v22.b }[10], [x21], #0x1\n" + "ld1 { v4.b }[10], [x23], #0x1\n" + "ld1 { v3.b }[10], [x22], #0x1\n" + "ld1 { v28.b }[10], [x21], #0x1\n" + "ld1 { v22.b }[10], [x20], #0x1\n" "b 23f\n" "18:" // Oddments: 4 inputs loop: Load: Bit 3: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[8], [x24], #0x1\n" - "ld1 { v3.b }[8], [x23], #0x1\n" - "ld1 { v28.b }[8], [x22], #0x1\n" - "ld1 { v22.b }[8], [x21], #0x1\n" + "ld1 { v4.b }[8], [x23], #0x1\n" + "ld1 { v3.b }[8], [x22], #0x1\n" + "ld1 { v28.b }[8], [x21], #0x1\n" + "ld1 { v22.b }[8], [x20], #0x1\n" "b 23f\n" "19:" // Oddments: 4 inputs loop: Load: Bit 3: Unset "tbz %x[n_channels], #2, 21f\n" - "ldr s4, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr s28, [x22], #0x4\n" - "ldr s22, [x21], #0x4\n" + "ldr s4, [x23], #0x4\n" + "ldr s3, [x22], #0x4\n" + "ldr s28, [x21], #0x4\n" + "ldr s22, [x20], #0x4\n" "tbz %x[n_channels], #1, 20f\n" - "ld1 { v4.h }[2], [x24], #0x2\n" - "ld1 { v3.h }[2], [x23], #0x2\n" - "ld1 { v28.h }[2], [x22], #0x2\n" - "ld1 { v22.h }[2], [x21], #0x2\n" + "ld1 { v4.h }[2], [x23], #0x2\n" + "ld1 { v3.h }[2], [x22], #0x2\n" + "ld1 { v28.h }[2], [x21], #0x2\n" + "ld1 { v22.h }[2], [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[6], [x24], #0x1\n" - "ld1 { v3.b }[6], [x23], #0x1\n" - "ld1 { v28.b }[6], [x22], #0x1\n" - "ld1 { v22.b }[6], [x21], #0x1\n" + "ld1 { v4.b }[6], [x23], #0x1\n" + "ld1 { v3.b }[6], [x22], #0x1\n" + "ld1 { v28.b }[6], [x21], #0x1\n" + "ld1 { v22.b }[6], [x20], #0x1\n" "b 23f\n" "20:" // Oddments: 4 inputs loop: Load: Bit 3: Unset: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[4], [x24], #0x1\n" - "ld1 { v3.b }[4], [x23], #0x1\n" - "ld1 { v28.b }[4], [x22], #0x1\n" - "ld1 { v22.b }[4], [x21], #0x1\n" + "ld1 { v4.b }[4], [x23], #0x1\n" + "ld1 { v3.b }[4], [x22], #0x1\n" + "ld1 { v28.b }[4], [x21], #0x1\n" + "ld1 { v22.b }[4], [x20], #0x1\n" "b 23f\n" "21:" // Oddments: 4 inputs loop: Load: Bit 3: Unset: Bit 2: Unset "tbz %x[n_channels], #1, 22f\n" - "ldr h4, [x24], #0x2\n" - "ldr h3, [x23], #0x2\n" - "ldr h28, [x22], #0x2\n" - "ldr h22, [x21], #0x2\n" + "ldr h4, [x23], #0x2\n" + "ldr h3, [x22], #0x2\n" + "ldr h28, [x21], #0x2\n" + "ldr h22, [x20], #0x2\n" "tbz %x[n_channels], #0, 23f\n" - "ld1 { v4.b }[2], [x24], #0x1\n" - "ld1 { v3.b }[2], [x23], #0x1\n" - "ld1 { v28.b }[2], [x22], #0x1\n" - "ld1 { v22.b }[2], [x21], #0x1\n" + "ld1 { v4.b }[2], [x23], #0x1\n" + "ld1 { v3.b }[2], [x22], #0x1\n" + "ld1 { v28.b }[2], [x21], #0x1\n" + "ld1 { v22.b }[2], [x20], #0x1\n" "b 23f\n" "22:" // Oddments: 4 inputs loop: Load: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 23f\n" - "ldr b4, [x24], #0x1\n" - "ldr b3, [x23], #0x1\n" - "ldr b28, [x22], #0x1\n" - "ldr b22, [x21], #0x1\n" + "ldr b4, [x23], #0x1\n" + "ldr b3, [x22], #0x1\n" + "ldr b28, [x21], #0x1\n" + "ldr b22, [x20], #0x1\n" "23:" // Oddments: 4 inputs loop: Load: Bit 3: End - "umax v23.16b, v4.16b, v3.16b\n" - "umax v19.16b, v28.16b, v22.16b\n" + "umax v17.16b, v4.16b, v3.16b\n" + "umax v16.16b, v28.16b, v22.16b\n" "subs x25, x25, #0x1\n" - "umax v19.16b, v23.16b, v19.16b\n" - "umax v8.16b, v8.16b, v19.16b\n" + "umax v16.16b, v17.16b, v16.16b\n" + "umax v8.16b, v8.16b, v16.16b\n" "bgt 15b\n" "24:" // Oddments: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 34f\n" "25:" // Oddments: Single input loop - "ldr x24, [x20], #0x8\n" - "add x24, x24, x9\n" + "ldr x23, [x24], #0x8\n" + "add x23, x23, x27\n" "movi v4.16b, #0x0\n" "tbz %x[n_channels], #3, 29f\n" - "ldr d4, [x24], #0x8\n" + "ldr d4, [x23], #0x8\n" "tbz %x[n_channels], #2, 27f\n" - "ld1 { v4.s }[2], [x24], #0x4\n" + "ld1 { v4.s }[2], [x23], #0x4\n" "tbz %x[n_channels], #1, 26f\n" - "ld1 { v4.h }[6], [x24], #0x2\n" + "ld1 { v4.h }[6], [x23], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[14], [x24], #0x1\n" + "ld1 { v4.b }[14], [x23], #0x1\n" "b 33f\n" "26:" // Oddments: Single input loop: Load: Bit 3: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[12], [x24], #0x1\n" + "ld1 { v4.b }[12], [x23], #0x1\n" "b 33f\n" "27:" // Oddments: Single input loop: Load: Bit 3: Bit 2: Unset "tbz %x[n_channels], #1, 28f\n" - "ld1 { v4.h }[4], [x24], #0x2\n" + "ld1 { v4.h }[4], [x23], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[10], [x24], #0x1\n" + "ld1 { v4.b }[10], [x23], #0x1\n" "b 33f\n" "28:" // Oddments: Single input loop: Load: Bit 3: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[8], [x24], #0x1\n" + "ld1 { v4.b }[8], [x23], #0x1\n" "b 33f\n" "29:" // Oddments: Single input loop: Load: Bit 3: Unset "tbz %x[n_channels], #2, 31f\n" - "ldr s4, [x24], #0x4\n" + "ldr s4, [x23], #0x4\n" "tbz %x[n_channels], #1, 30f\n" - "ld1 { v4.h }[2], [x24], #0x2\n" + "ld1 { v4.h }[2], [x23], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[6], [x24], #0x1\n" + "ld1 { v4.b }[6], [x23], #0x1\n" "b 33f\n" "30:" // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[4], [x24], #0x1\n" + "ld1 { v4.b }[4], [x23], #0x1\n" "b 33f\n" "31:" // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Unset "tbz %x[n_channels], #1, 32f\n" - "ldr h4, [x24], #0x2\n" + "ldr h4, [x23], #0x2\n" "tbz %x[n_channels], #0, 33f\n" - "ld1 { v4.b }[2], [x24], #0x1\n" + "ld1 { v4.b }[2], [x23], #0x1\n" "b 33f\n" "32:" // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset "tbz %x[n_channels], #0, 33f\n" - "ldr b4, [x24], #0x1\n" + "ldr b4, [x23], #0x1\n" "33:" // Oddments: Single input loop: Load: Bit 3: End "subs x21, x21, #0x1\n" "umax v8.16b, v8.16b, v4.16b\n" "bgt 25b\n" "34:" // Oddments: Single input loop: End "add x20, %x[quant_params], %[offsetof_qp_input_offset]\n" - "ld1r { v4.4s }, [x20]\n" - "uxtl v23.8h, v8.8b\n" - "uxtl2 v24.8h, v8.16b\n" - "neg v4.4s, v4.4s\n" + "ld1r { v18.4s }, [x20]\n" + "uxtl v17.8h, v8.8b\n" + "uxtl2 v16.8h, v8.16b\n" + "neg v18.4s, v18.4s\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n" - "ld1r { v3.4s }, [x20]\n" - "saddw v0.4s, v4.4s, v23.4h\n" - "saddw2 v23.4s, v4.4s, v23.8h\n" - "saddw v31.4s, v4.4s, v24.4h\n" + "ld1r { v23.4s }, [x20]\n" + "saddw v22.4s, v18.4s, v17.4h\n" + "saddw2 v21.4s, v18.4s, v17.8h\n" + "saddw v20.4s, v18.4s, v16.4h\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_mul]\n" - "ld1r { v2.4s }, [x20]\n" - "saddw2 v30.4s, v4.4s, v24.8h\n" - "srshl v0.4s, v0.4s, v3.4s\n" + "ld1r { v19.4s }, [x20]\n" + "saddw2 v18.4s, v18.4s, v16.8h\n" + "srshl v22.4s, v22.4s, v23.4s\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n" - "ld1r { v1.4s }, [x20]\n" - "srshl v23.4s, v23.4s, v3.4s\n" - "srshl v31.4s, v31.4s, v3.4s\n" + "ld1r { v17.4s }, [x20]\n" + "srshl v21.4s, v21.4s, v23.4s\n" + "srshl v20.4s, v20.4s, v23.4s\n" "add x20, %x[quant_params], %[offsetof_qp_output_offset]\n" "ld1r { v16.4s }, [x20]\n" - "srshl v30.4s, v30.4s, v3.4s\n" - "sqrdmulh v0.4s, v0.4s, v2.4s\n" - "sqrdmulh v23.4s, v23.4s, v2.4s\n" - "sqrdmulh v31.4s, v31.4s, v2.4s\n" - "sqrdmulh v30.4s, v30.4s, v2.4s\n" - "srshl v0.4s, v0.4s, v1.4s\n" - "srshl v23.4s, v23.4s, v1.4s\n" - "srshl v31.4s, v31.4s, v1.4s\n" - "srshl v30.4s, v30.4s, v1.4s\n" - "add v0.4s, v0.4s, v16.4s\n" - "add v23.4s, v23.4s, v16.4s\n" - "add v31.4s, v31.4s, v16.4s\n" - "add v30.4s, v30.4s, v16.4s\n" + "srshl v18.4s, v18.4s, v23.4s\n" + "sqrdmulh v22.4s, v22.4s, v19.4s\n" + "sqrdmulh v21.4s, v21.4s, v19.4s\n" + "sqrdmulh v20.4s, v20.4s, v19.4s\n" + "sqrdmulh v18.4s, v18.4s, v19.4s\n" + "srshl v22.4s, v22.4s, v17.4s\n" + "srshl v21.4s, v21.4s, v17.4s\n" + "srshl v20.4s, v20.4s, v17.4s\n" + "srshl v18.4s, v18.4s, v17.4s\n" + "add v22.4s, v22.4s, v16.4s\n" + "add v21.4s, v21.4s, v16.4s\n" + "add v20.4s, v20.4s, v16.4s\n" + "add v18.4s, v18.4s, v16.4s\n" "movi v16.4s, #0x0\n" - "smax v0.4s, v0.4s, v16.4s\n" - "smax v23.4s, v23.4s, v16.4s\n" - "smax v31.4s, v31.4s, v16.4s\n" - "smax v30.4s, v30.4s, v16.4s\n" + "smax v22.4s, v22.4s, v16.4s\n" + "smax v21.4s, v21.4s, v16.4s\n" + "smax v20.4s, v20.4s, v16.4s\n" + "smax v18.4s, v18.4s, v16.4s\n" "movi v16.4s, #0xff\n" - "smin v0.4s, v0.4s, v16.4s\n" - "smin v23.4s, v23.4s, v16.4s\n" - "smin v31.4s, v31.4s, v16.4s\n" - "smin v30.4s, v30.4s, v16.4s\n" - "uzp1 v23.16b, v0.16b, v23.16b\n" - "uzp1 v16.16b, v31.16b, v30.16b\n" - "uzp1 v16.16b, v23.16b, v16.16b\n" + "smin v22.4s, v22.4s, v16.4s\n" + "smin v21.4s, v21.4s, v16.4s\n" + "smin v20.4s, v20.4s, v16.4s\n" + "smin v18.4s, v18.4s, v16.4s\n" + "uzp1 v17.16b, v22.16b, v21.16b\n" + "uzp1 v16.16b, v20.16b, v18.16b\n" + "uzp1 v16.16b, v17.16b, v16.16b\n" "tbz %x[n_channels], #3, 38f\n" "st1 { v16.d }[0], [%x[outptr]], #0x8\n" "tbz %x[n_channels], #2, 36f\n" @@ -667,10 +667,11 @@ void a64_u8q_nhwc_max_generic_depthfirst_impl( "43:" // End : [n_channels] "+&r" (n_channels), [outptr] "+&r" (outptr) : [inptrs] "r" (inptrs), [n_valid_cells] "r" (n_valid_cells), [offsetof_qp_input_offset] "I" (offsetof(Requantize32, input_offset)), [offsetof_qp_output_offset] "I" (offsetof(Requantize32, output_offset)), [offsetof_qp_per_layer_left_shift] "I" (offsetof(Requantize32, per_layer_left_shift)), [offsetof_qp_per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [offsetof_qp_per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [quant_params] "r" (&qp) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27" ); } } // namespace pooling } // namespace arm_conv -#endif // defined(__aarch64__) + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/cpp_nhwc_1x1_stride_any_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/cpp_nhwc_1x1_stride_any_depthfirst/generic.cpp index 2bb22131f74edc3b7d85068d9d4f0238e8d83443..1f8f863de2151ead73f4b85624c6acc19f12e1d9 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/cpp_nhwc_1x1_stride_any_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/cpp_nhwc_1x1_stride_any_depthfirst/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 Arm Limited. + * Copyright (c) 2020, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,10 @@ #include #include +#ifdef ARM_COMPUTE_ENABLE_BF16 +#include "bfloat.hpp" +using arm_gemm::bfloat16; +#endif namespace arm_conv { namespace pooling { @@ -41,9 +45,15 @@ void cpp_nhwc_1x1_stride_any_depthfirst_impl( } template void cpp_nhwc_1x1_stride_any_depthfirst_impl(uint64_t, uint64_t, uint64_t, const float *const *, float *); -#if defined(__ARM_FP16_ARGS) + +#ifdef __ARM_FP16_ARGS template void cpp_nhwc_1x1_stride_any_depthfirst_impl(uint64_t, uint64_t, uint64_t, const __fp16 *const *, __fp16 *); -#endif // defined(__ARM_FP16_ARGS) +#endif + +#ifdef ARM_COMPUTE_ENABLE_BF16 +template void cpp_nhwc_1x1_stride_any_depthfirst_impl(uint64_t, uint64_t, uint64_t, const bfloat16 *const *, bfloat16 *); +#endif + template void cpp_nhwc_1x1_stride_any_depthfirst_impl(uint64_t, uint64_t, uint64_t, const int8_t *const *, int8_t *); template void cpp_nhwc_1x1_stride_any_depthfirst_impl(uint64_t, uint64_t, uint64_t, const uint8_t *const *, uint8_t *); diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst.hpp index 250d92c051a15bb69156a7488c71950fe1bf0df0..f6682e75e2fe754c7dc2fd8c567c14319c1801c1 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,8 @@ #pragma once +#if defined(ARM_COMPUTE_ENABLE_SME) && defined(__ARM_FP16_ARGS) + namespace arm_conv { namespace pooling { @@ -45,3 +47,5 @@ struct sme_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst : public DepthfirstStrategy } // namespace pooling } // namespace arm_conv + +#endif // defined(ARM_COMPUTE_ENABLE_SME) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp index bce623acd11b9d0fac50e3f25b590d23ce034025..67b07205cdd5a5d0a575e15e3d3545cfe0bf8ba5 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp @@ -26,7 +26,7 @@ #include #include -#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FP16_ARGS) && defined(ARM_COMPUTE_ENABLE_SME) +#if defined(ARM_COMPUTE_ENABLE_SME) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace pooling { @@ -91,34 +91,34 @@ void sme_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst_impl( "add x20, %x[args], %[offsetof_rescale]\n" "ld1rqh { z4.h }, p0/Z, [x20]\n" "ldr x5, [%x[args], %[offsetof_n_channels]]\n" - "whilelt p1.h, x3, x5\n" + "whilelt p0.h, x3, x5\n" "mov x6, #0x0\n" "ldp x7, x8, [x21, #0x0]\n" "ldp x17, x16, [x21, #0x10]\n" "ldp x15, x14, [x4, #0x0]\n" - "ld1h { z3.h }, p1/Z, [x14, x3, LSL #1]\n" + "ld1h { z3.h }, p0/Z, [x14, x3, LSL #1]\n" "ldp x13, x12, [x4, #0x10]\n" - "ld1h { z2.h }, p1/Z, [x13, x3, LSL #1]\n" + "ld1h { z2.h }, p0/Z, [x13, x3, LSL #1]\n" "ldp x11, x10, [x4, #0x20]\n" - "ld1h { z1.h }, p1/Z, [x10, x3, LSL #1]\n" + "ld1h { z1.h }, p0/Z, [x10, x3, LSL #1]\n" "ldp x9, x28, [x4, #0x30]\n" - "ld1h { z0.h }, p1/Z, [x9, x3, LSL #1]\n" + "ld1h { z0.h }, p0/Z, [x9, x3, LSL #1]\n" "ldp x27, x26, [x4, #0x40]\n" - "ld1h { z31.h }, p1/Z, [x26, x3, LSL #1]\n" + "ld1h { z31.h }, p0/Z, [x26, x3, LSL #1]\n" "ldp x25, x24, [x4, #0x50]\n" - "ld1h { z30.h }, p1/Z, [x25, x3, LSL #1]\n" + "ld1h { z30.h }, p0/Z, [x25, x3, LSL #1]\n" "ldp x23, x22, [x4, #0x60]\n" - "ld1h { z29.h }, p1/Z, [x11, x3, LSL #1]\n" + "ld1h { z29.h }, p0/Z, [x11, x3, LSL #1]\n" "ldp x21, x20, [x4, #0x70]\n" - "ld1h { z28.h }, p1/Z, [x27, x3, LSL #1]\n" - "ld1h { z27.h }, p1/Z, [x28, x3, LSL #1]\n" - "ld1h { z22.h }, p1/Z, [x24, x3, LSL #1]\n" - "ld1h { z21.h }, p1/Z, [x22, x3, LSL #1]\n" - "ld1h { z20.h }, p1/Z, [x21, x3, LSL #1]\n" - "ld1h { z26.h }, p1/Z, [x15, x3, LSL #1]\n" - "ld1h { z25.h }, p1/Z, [x12, x3, LSL #1]\n" - "ld1h { z24.h }, p1/Z, [x23, x3, LSL #1]\n" - "ld1h { z23.h }, p1/Z, [x20, x3, LSL #1]\n" + "ld1h { z28.h }, p0/Z, [x27, x3, LSL #1]\n" + "ld1h { z27.h }, p0/Z, [x28, x3, LSL #1]\n" + "ld1h { z22.h }, p0/Z, [x24, x3, LSL #1]\n" + "ld1h { z21.h }, p0/Z, [x22, x3, LSL #1]\n" + "ld1h { z20.h }, p0/Z, [x21, x3, LSL #1]\n" + "ld1h { z26.h }, p0/Z, [x15, x3, LSL #1]\n" + "ld1h { z25.h }, p0/Z, [x12, x3, LSL #1]\n" + "ld1h { z24.h }, p0/Z, [x23, x3, LSL #1]\n" + "ld1h { z23.h }, p0/Z, [x20, x3, LSL #1]\n" "incw x3\n" "whilelt p1.h, x3, x5\n" "b.none 2f\n" @@ -206,4 +206,4 @@ void sme_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst_impl( } // namespace pooling } // namespace arm_conv -#endif // defined(__ARM_FEATURE_SVE) && defined(__ARM_FP16_ARGS) && defined(ARM_COMPUTE_ENABLE_SME) +#endif // defined(ARM_COMPUTE_ENABLE_SME) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_avg_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_avg_generic_depthfirst.hpp index 117eb3600713136f3fd72d41785f7ed4435eff8c..cf09f421c403301bbf93d1320476603994a9c90a 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_avg_generic_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_avg_generic_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,8 @@ #pragma once +#if defined(ARM_COMPUTE_ENABLE_SME) && defined(__ARM_FP16_ARGS) + namespace arm_conv { namespace pooling { @@ -40,3 +42,5 @@ struct sme_fp16_nhwc_avg_generic_depthfirst : IGenericDepthfirstStrategy<__fp16, } // namespace pooling } // namespace arm_conv + +#endif // defined(ARM_COMPUTE_ENABLE_SME) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_avg_generic_depthfirst/generic.cpp index c43da42d9e4aed67c11544b09f753bcd680d630a..60f17b7bc274894a03ed2ed96cceb7e3666464be 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_avg_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_avg_generic_depthfirst/generic.cpp @@ -22,9 +22,10 @@ * SOFTWARE. */ -#if defined(ARM_COMPUTE_ENABLE_SME) - #include +#include + +#if defined(ARM_COMPUTE_ENABLE_SME) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace pooling { @@ -57,68 +58,68 @@ void sme_fp16_nhwc_avg_generic_depthfirst_impl( "lsr x25, %x[n_valid_cells], #0x2\n" "mov z5.b, #0x0\n" "mov z4.b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "mov z3.b, #0x0\n" "mov z2.b, #0x0\n" "cbz x25, 4f\n" - "ldp x24, x23, [x20, #0x0]\n" + "ldp x23, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" - "ld1h { z1.h }, p3/Z, [x24, x9, LSL #1]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ld1h { z0.h }, p3/Z, [x23, x9, LSL #1]\n" - "ld1h { z31.h }, p3/Z, [x22, x9, LSL #1]\n" - "ld1h { z30.h }, p3/Z, [x21, x9, LSL #1]\n" - "ld1h { z29.h }, p2/Z, [x24, x28, LSL #1]\n" - "ld1h { z22.h }, p2/Z, [x23, x28, LSL #1]\n" - "ld1h { z28.h }, p2/Z, [x22, x28, LSL #1]\n" - "ld1h { z18.h }, p2/Z, [x21, x28, LSL #1]\n" - "ld1h { z27.h }, p1/Z, [x24, x27, LSL #1]\n" - "ld1h { z21.h }, p1/Z, [x23, x27, LSL #1]\n" - "ld1h { z26.h }, p1/Z, [x22, x27, LSL #1]\n" - "ld1h { z17.h }, p1/Z, [x21, x27, LSL #1]\n" - "ld1h { z25.h }, p0/Z, [x24, x26, LSL #1]\n" - "ld1h { z20.h }, p0/Z, [x23, x26, LSL #1]\n" - "ld1h { z24.h }, p0/Z, [x22, x26, LSL #1]\n" - "ld1h { z16.h }, p0/Z, [x21, x26, LSL #1]\n" + "ld1h { z1.h }, p3/Z, [x23, x9, LSL #1]\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" + "ld1h { z0.h }, p3/Z, [x22, x9, LSL #1]\n" + "ld1h { z31.h }, p3/Z, [x21, x9, LSL #1]\n" + "ld1h { z30.h }, p3/Z, [x20, x9, LSL #1]\n" + "ld1h { z29.h }, p2/Z, [x23, x28, LSL #1]\n" + "ld1h { z22.h }, p2/Z, [x22, x28, LSL #1]\n" + "ld1h { z28.h }, p2/Z, [x21, x28, LSL #1]\n" + "ld1h { z18.h }, p2/Z, [x20, x28, LSL #1]\n" + "ld1h { z27.h }, p1/Z, [x23, x27, LSL #1]\n" + "ld1h { z21.h }, p1/Z, [x22, x27, LSL #1]\n" + "ld1h { z26.h }, p1/Z, [x21, x27, LSL #1]\n" + "ld1h { z17.h }, p1/Z, [x20, x27, LSL #1]\n" + "ld1h { z25.h }, p0/Z, [x23, x26, LSL #1]\n" + "ld1h { z20.h }, p0/Z, [x22, x26, LSL #1]\n" + "ld1h { z24.h }, p0/Z, [x21, x26, LSL #1]\n" + "ld1h { z16.h }, p0/Z, [x20, x26, LSL #1]\n" "beq 3f\n" "2:" // 4-vectors of channels: 4 inputs loop "fadd z23.h, z1.h, z0.h\n" "fadd z19.h, z31.h, z30.h\n" - "ldp x24, x23, [x20, #0x0]\n" + "ldp x23, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" "fadd z22.h, z29.h, z22.h\n" "fadd z18.h, z28.h, z18.h\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" "fadd z21.h, z27.h, z21.h\n" "fadd z17.h, z26.h, z17.h\n" - "ld1h { z1.h }, p3/Z, [x24, x9, LSL #1]\n" + "ld1h { z1.h }, p3/Z, [x23, x9, LSL #1]\n" "fadd z20.h, z25.h, z20.h\n" "fadd z16.h, z24.h, z16.h\n" - "ld1h { z0.h }, p3/Z, [x23, x9, LSL #1]\n" + "ld1h { z0.h }, p3/Z, [x22, x9, LSL #1]\n" "fadd z19.h, z23.h, z19.h\n" "fadd z18.h, z22.h, z18.h\n" - "ld1h { z31.h }, p3/Z, [x22, x9, LSL #1]\n" + "ld1h { z31.h }, p3/Z, [x21, x9, LSL #1]\n" "fadd z17.h, z21.h, z17.h\n" "fadd z16.h, z20.h, z16.h\n" - "ld1h { z30.h }, p3/Z, [x21, x9, LSL #1]\n" + "ld1h { z30.h }, p3/Z, [x20, x9, LSL #1]\n" "fadd z5.h, z5.h, z19.h\n" "fadd z4.h, z4.h, z18.h\n" - "ld1h { z29.h }, p2/Z, [x24, x28, LSL #1]\n" + "ld1h { z29.h }, p2/Z, [x23, x28, LSL #1]\n" "fadd z3.h, z3.h, z17.h\n" "fadd z2.h, z2.h, z16.h\n" - "ld1h { z22.h }, p2/Z, [x23, x28, LSL #1]\n" - "ld1h { z28.h }, p2/Z, [x22, x28, LSL #1]\n" - "ld1h { z18.h }, p2/Z, [x21, x28, LSL #1]\n" - "ld1h { z27.h }, p1/Z, [x24, x27, LSL #1]\n" - "ld1h { z21.h }, p1/Z, [x23, x27, LSL #1]\n" - "ld1h { z26.h }, p1/Z, [x22, x27, LSL #1]\n" - "ld1h { z17.h }, p1/Z, [x21, x27, LSL #1]\n" - "ld1h { z25.h }, p0/Z, [x24, x26, LSL #1]\n" - "ld1h { z20.h }, p0/Z, [x23, x26, LSL #1]\n" - "ld1h { z24.h }, p0/Z, [x22, x26, LSL #1]\n" - "ld1h { z16.h }, p0/Z, [x21, x26, LSL #1]\n" + "ld1h { z22.h }, p2/Z, [x22, x28, LSL #1]\n" + "ld1h { z28.h }, p2/Z, [x21, x28, LSL #1]\n" + "ld1h { z18.h }, p2/Z, [x20, x28, LSL #1]\n" + "ld1h { z27.h }, p1/Z, [x23, x27, LSL #1]\n" + "ld1h { z21.h }, p1/Z, [x22, x27, LSL #1]\n" + "ld1h { z26.h }, p1/Z, [x21, x27, LSL #1]\n" + "ld1h { z17.h }, p1/Z, [x20, x27, LSL #1]\n" + "ld1h { z25.h }, p0/Z, [x23, x26, LSL #1]\n" + "ld1h { z20.h }, p0/Z, [x22, x26, LSL #1]\n" + "ld1h { z24.h }, p0/Z, [x21, x26, LSL #1]\n" + "ld1h { z16.h }, p0/Z, [x20, x26, LSL #1]\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "fadd z23.h, z1.h, z0.h\n" @@ -141,16 +142,16 @@ void sme_fp16_nhwc_avg_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1h { z1.h }, p3/Z, [x24, x9, LSL #1]\n" + "ldr x20, [x24], #0x8\n" + "ld1h { z16.h }, p3/Z, [x20, x9, LSL #1]\n" "subs x21, x21, #0x1\n" - "fadd z5.h, z5.h, z1.h\n" - "ld1h { z29.h }, p2/Z, [x24, x28, LSL #1]\n" - "fadd z4.h, z4.h, z29.h\n" - "ld1h { z27.h }, p1/Z, [x24, x27, LSL #1]\n" - "fadd z3.h, z3.h, z27.h\n" - "ld1h { z25.h }, p0/Z, [x24, x26, LSL #1]\n" - "fadd z2.h, z2.h, z25.h\n" + "fadd z5.h, z5.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x20, x28, LSL #1]\n" + "fadd z4.h, z4.h, z16.h\n" + "ld1h { z16.h }, p1/Z, [x20, x27, LSL #1]\n" + "fadd z3.h, z3.h, z16.h\n" + "ld1h { z16.h }, p0/Z, [x20, x26, LSL #1]\n" + "fadd z2.h, z2.h, z16.h\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End "fmul z5.h, z5.h, z6.h\n" @@ -173,44 +174,44 @@ void sme_fp16_nhwc_avg_generic_depthfirst_impl( "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" "mov z5.b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x24, x23, [x20, #0x0]\n" + "ldp x20, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" - "ld1h { z1.h }, p3/Z, [x24, x9, LSL #1]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ld1h { z0.h }, p3/Z, [x23, x9, LSL #1]\n" - "ld1h { z31.h }, p3/Z, [x22, x9, LSL #1]\n" - "ld1h { z30.h }, p3/Z, [x21, x9, LSL #1]\n" + "ld1h { z1.h }, p3/Z, [x20, x9, LSL #1]\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" + "ld1h { z0.h }, p3/Z, [x22, x9, LSL #1]\n" + "ld1h { z31.h }, p3/Z, [x21, x9, LSL #1]\n" + "ld1h { z30.h }, p3/Z, [x20, x9, LSL #1]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 4 inputs loop - "fadd z23.h, z1.h, z0.h\n" - "fadd z19.h, z31.h, z30.h\n" - "ldp x24, x23, [x20, #0x0]\n" + "fadd z17.h, z1.h, z0.h\n" + "fadd z16.h, z31.h, z30.h\n" + "ldp x23, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" - "fadd z19.h, z23.h, z19.h\n" - "ldp x22, x21, [x20, #0x10]\n" - "fadd z5.h, z5.h, z19.h\n" - "add x20, x20, #0x20\n" - "ld1h { z1.h }, p3/Z, [x24, x9, LSL #1]\n" - "ld1h { z0.h }, p3/Z, [x23, x9, LSL #1]\n" - "ld1h { z31.h }, p3/Z, [x22, x9, LSL #1]\n" - "ld1h { z30.h }, p3/Z, [x21, x9, LSL #1]\n" + "fadd z16.h, z17.h, z16.h\n" + "ldp x21, x20, [x24, #0x10]\n" + "fadd z5.h, z5.h, z16.h\n" + "add x24, x24, #0x20\n" + "ld1h { z1.h }, p3/Z, [x23, x9, LSL #1]\n" + "ld1h { z0.h }, p3/Z, [x22, x9, LSL #1]\n" + "ld1h { z31.h }, p3/Z, [x21, x9, LSL #1]\n" + "ld1h { z30.h }, p3/Z, [x20, x9, LSL #1]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail - "fadd z23.h, z1.h, z0.h\n" - "fadd z19.h, z31.h, z30.h\n" - "fadd z19.h, z23.h, z19.h\n" - "fadd z5.h, z5.h, z19.h\n" + "fadd z17.h, z1.h, z0.h\n" + "fadd z16.h, z31.h, z30.h\n" + "fadd z16.h, z17.h, z16.h\n" + "fadd z5.h, z5.h, z16.h\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1h { z1.h }, p3/Z, [x24, x9, LSL #1]\n" + "ldr x20, [x24], #0x8\n" + "ld1h { z16.h }, p3/Z, [x20, x9, LSL #1]\n" "subs x21, x21, #0x1\n" - "fadd z5.h, z5.h, z1.h\n" + "fadd z5.h, z5.h, z16.h\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "fmul z5.h, z5.h, z6.h\n" @@ -229,4 +230,4 @@ void sme_fp16_nhwc_avg_generic_depthfirst_impl( } // namespace pooling } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SME) +#endif // defined(ARM_COMPUTE_ENABLE_SME) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_max_2x2_s1_output2x2_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_max_2x2_s1_output2x2_depthfirst.hpp index 9489c1f8da1db5d5a673d3ae32e70a0ed08dd10f..cd6c7449a8811b399d3f01bfa5a44e3b2c9f39f2 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_max_2x2_s1_output2x2_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_max_2x2_s1_output2x2_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,8 @@ #pragma once +#if defined(ARM_COMPUTE_ENABLE_SME) && defined(__ARM_FP16_ARGS) + namespace arm_conv { namespace pooling { @@ -45,3 +47,5 @@ struct sme_fp16_nhwc_max_2x2_s1_output2x2_depthfirst : public DepthfirstStrategy } // namespace pooling } // namespace arm_conv + +#endif // defined(ARM_COMPUTE_ENABLE_SME) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp index f71f2625b688a755de65f703cf23c89961b19965..7fc776ed4e0da3236e6bb33ee9078d6eefa401bb 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp @@ -26,7 +26,7 @@ #include #include -#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FP16_ARGS) && defined(ARM_COMPUTE_ENABLE_SME) +#if defined(ARM_COMPUTE_ENABLE_SME) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace pooling { @@ -70,23 +70,23 @@ void sme_fp16_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "ldr x20, [%x[args], %[offsetof_inptrs]]\n" "mov x14, #0x0\n" "ldr x13, [%x[args], %[offsetof_n_channels]]\n" - "whilelt p1.h, x15, x13\n" + "whilelt p0.h, x15, x13\n" "ldp x12, x11, [x21, #0x0]\n" "ldp x10, x9, [x21, #0x10]\n" "ldp x28, x27, [x20, #0x0]\n" - "ld1h { z30.h }, p1/Z, [x27, x15, LSL #1]\n" + "ld1h { z30.h }, p0/Z, [x27, x15, LSL #1]\n" "ldp x26, x25, [x20, #0x10]\n" - "ld1h { z29.h }, p1/Z, [x25, x15, LSL #1]\n" + "ld1h { z29.h }, p0/Z, [x25, x15, LSL #1]\n" "ldp x24, x23, [x20, #0x20]\n" - "ld1h { z28.h }, p1/Z, [x24, x15, LSL #1]\n" + "ld1h { z28.h }, p0/Z, [x24, x15, LSL #1]\n" "ldp x22, x21, [x20, #0x30]\n" - "ld1h { z27.h }, p1/Z, [x21, x15, LSL #1]\n" + "ld1h { z27.h }, p0/Z, [x21, x15, LSL #1]\n" "ldr x20, [x20, #0x40]\n" - "ld1h { z26.h }, p1/Z, [x28, x15, LSL #1]\n" - "ld1h { z25.h }, p1/Z, [x26, x15, LSL #1]\n" - "ld1h { z24.h }, p1/Z, [x23, x15, LSL #1]\n" - "ld1h { z23.h }, p1/Z, [x22, x15, LSL #1]\n" - "ld1h { z19.h }, p1/Z, [x20, x15, LSL #1]\n" + "ld1h { z26.h }, p0/Z, [x28, x15, LSL #1]\n" + "ld1h { z25.h }, p0/Z, [x26, x15, LSL #1]\n" + "ld1h { z24.h }, p0/Z, [x23, x15, LSL #1]\n" + "ld1h { z19.h }, p0/Z, [x22, x15, LSL #1]\n" + "ld1h { z23.h }, p0/Z, [x20, x15, LSL #1]\n" "incw x15\n" "whilelt p1.h, x15, x13\n" "b.none 2f\n" @@ -95,25 +95,25 @@ void sme_fp16_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "movprfx z21, z28\n fmax z21.h, p2/M, z21.h, z27.h\n" "ld1h { z30.h }, p1/Z, [x27, x15, LSL #1]\n" "whilelt p0.h, x14, x13\n" - "movprfx z20, z29\n fmax z20.h, p2/M, z20.h, z26.h\n" - "movprfx z18, z25\n fmax z18.h, p2/M, z18.h, z24.h\n" + "movprfx z18, z29\n fmax z18.h, p2/M, z18.h, z26.h\n" + "movprfx z17, z25\n fmax z17.h, p2/M, z17.h, z24.h\n" "ld1h { z28.h }, p1/Z, [x24, x15, LSL #1]\n" - "movprfx z17, z29\n fmax z17.h, p2/M, z17.h, z23.h\n" - "movprfx z16, z24\n fmax z16.h, p2/M, z16.h, z19.h\n" + "movprfx z16, z29\n fmax z16.h, p2/M, z16.h, z19.h\n" + "movprfx z20, z24\n fmax z20.h, p2/M, z20.h, z23.h\n" "ld1h { z27.h }, p1/Z, [x21, x15, LSL #1]\n" "ld1h { z29.h }, p1/Z, [x25, x15, LSL #1]\n" - "movprfx z19, z22\n fmax z19.h, p2/M, z19.h, z20.h\n" - "fmax z18.h, p2/M, z18.h, z22.h\n" + "movprfx z19, z22\n fmax z19.h, p2/M, z19.h, z18.h\n" + "movprfx z18, z17\n fmax z18.h, p2/M, z18.h, z22.h\n" "ld1h { z26.h }, p1/Z, [x28, x15, LSL #1]\n" - "fmax z17.h, p2/M, z17.h, z21.h\n" - "fmax z16.h, p2/M, z16.h, z21.h\n" + "movprfx z17, z16\n fmax z17.h, p2/M, z17.h, z21.h\n" + "movprfx z16, z21\n fmax z16.h, p2/M, z16.h, z20.h\n" "ld1h { z25.h }, p1/Z, [x26, x15, LSL #1]\n" "st1h { z19.h }, p0, [x12, x14, LSL #1]\n" "ld1h { z24.h }, p1/Z, [x23, x15, LSL #1]\n" "st1h { z18.h }, p0, [x11, x14, LSL #1]\n" - "ld1h { z23.h }, p1/Z, [x22, x15, LSL #1]\n" + "ld1h { z19.h }, p1/Z, [x22, x15, LSL #1]\n" "st1h { z17.h }, p0, [x10, x14, LSL #1]\n" - "ld1h { z19.h }, p1/Z, [x20, x15, LSL #1]\n" + "ld1h { z23.h }, p1/Z, [x20, x15, LSL #1]\n" "incw x15\n" "whilelt p1.h, x15, x13\n" "st1h { z16.h }, p0, [x9, x14, LSL #1]\n" @@ -125,13 +125,13 @@ void sme_fp16_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "whilelt p0.h, x14, x13\n" "movprfx z20, z29\n fmax z20.h, p2/M, z20.h, z26.h\n" "movprfx z18, z25\n fmax z18.h, p2/M, z18.h, z24.h\n" - "movprfx z17, z29\n fmax z17.h, p2/M, z17.h, z23.h\n" - "movprfx z16, z24\n fmax z16.h, p2/M, z16.h, z19.h\n" - "movprfx z19, z22\n fmax z19.h, p2/M, z19.h, z20.h\n" + "movprfx z17, z29\n fmax z17.h, p2/M, z17.h, z19.h\n" + "movprfx z19, z24\n fmax z19.h, p2/M, z19.h, z23.h\n" + "movprfx z16, z22\n fmax z16.h, p2/M, z16.h, z20.h\n" "fmax z18.h, p2/M, z18.h, z22.h\n" - "st1h { z19.h }, p0, [x12, x14, LSL #1]\n" + "st1h { z16.h }, p0, [x12, x14, LSL #1]\n" "fmax z17.h, p2/M, z17.h, z21.h\n" - "fmax z16.h, p2/M, z16.h, z21.h\n" + "movprfx z16, z21\n fmax z16.h, p2/M, z16.h, z19.h\n" "st1h { z18.h }, p0, [x11, x14, LSL #1]\n" "st1h { z17.h }, p0, [x10, x14, LSL #1]\n" "st1h { z16.h }, p0, [x9, x14, LSL #1]\n" @@ -145,4 +145,4 @@ void sme_fp16_nhwc_max_2x2_s1_output2x2_depthfirst_impl( } // namespace pooling } // namespace arm_conv -#endif // defined(__ARM_FEATURE_SVE) && defined(__ARM_FP16_ARGS) && defined(ARM_COMPUTE_ENABLE_SME) +#endif // defined(ARM_COMPUTE_ENABLE_SME) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_max_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_max_generic_depthfirst.hpp index 33ff1f21540a658bcef1d0972a880cfb5875a0ce..bfb3bf5b1a8832091b0d33d167dcc5bcffb25447 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_max_generic_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_max_generic_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,8 @@ #pragma once +#if defined(ARM_COMPUTE_ENABLE_SME) && defined(__ARM_FP16_ARGS) + namespace arm_conv { namespace pooling { @@ -40,3 +42,5 @@ struct sme_fp16_nhwc_max_generic_depthfirst : IGenericDepthfirstStrategy<__fp16, } // namespace pooling } // namespace arm_conv + +#endif // defined(ARM_COMPUTE_ENABLE_SME) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_max_generic_depthfirst/generic.cpp index c07ce972312d02e0ea39f94edc88a906501ebe37..afa2ccbd719fd432d6c3dee077422dd9c00146cc 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_max_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp16_nhwc_max_generic_depthfirst/generic.cpp @@ -22,9 +22,10 @@ * SOFTWARE. */ -#if defined(ARM_COMPUTE_ENABLE_SME) - #include +#include + +#if defined(ARM_COMPUTE_ENABLE_SME) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace pooling { @@ -54,68 +55,68 @@ void sme_fp16_nhwc_max_generic_depthfirst_impl( "lsr x25, %x[n_valid_cells], #0x2\n" "mov z4.h, #0xfc00\n" "mov z3.h, #0xfc00\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "mov z2.h, #0xfc00\n" "mov z1.h, #0xfc00\n" "cbz x25, 4f\n" - "ldp x24, x23, [x20, #0x0]\n" + "ldp x23, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" - "ld1h { z0.h }, p4/Z, [x24, x9, LSL #1]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ld1h { z31.h }, p4/Z, [x23, x9, LSL #1]\n" - "ld1h { z23.h }, p4/Z, [x22, x9, LSL #1]\n" - "ld1h { z30.h }, p4/Z, [x21, x9, LSL #1]\n" - "ld1h { z18.h }, p3/Z, [x24, x28, LSL #1]\n" - "ld1h { z29.h }, p3/Z, [x23, x28, LSL #1]\n" - "ld1h { z22.h }, p3/Z, [x22, x28, LSL #1]\n" - "ld1h { z28.h }, p3/Z, [x21, x28, LSL #1]\n" - "ld1h { z17.h }, p2/Z, [x24, x27, LSL #1]\n" - "ld1h { z27.h }, p2/Z, [x23, x27, LSL #1]\n" - "ld1h { z21.h }, p2/Z, [x22, x27, LSL #1]\n" - "ld1h { z26.h }, p2/Z, [x21, x27, LSL #1]\n" - "ld1h { z16.h }, p1/Z, [x24, x26, LSL #1]\n" - "ld1h { z25.h }, p1/Z, [x23, x26, LSL #1]\n" - "ld1h { z20.h }, p1/Z, [x22, x26, LSL #1]\n" - "ld1h { z24.h }, p1/Z, [x21, x26, LSL #1]\n" + "ld1h { z0.h }, p4/Z, [x23, x9, LSL #1]\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" + "ld1h { z31.h }, p4/Z, [x22, x9, LSL #1]\n" + "ld1h { z23.h }, p4/Z, [x21, x9, LSL #1]\n" + "ld1h { z30.h }, p4/Z, [x20, x9, LSL #1]\n" + "ld1h { z18.h }, p3/Z, [x23, x28, LSL #1]\n" + "ld1h { z29.h }, p3/Z, [x22, x28, LSL #1]\n" + "ld1h { z22.h }, p3/Z, [x21, x28, LSL #1]\n" + "ld1h { z28.h }, p3/Z, [x20, x28, LSL #1]\n" + "ld1h { z17.h }, p2/Z, [x23, x27, LSL #1]\n" + "ld1h { z27.h }, p2/Z, [x22, x27, LSL #1]\n" + "ld1h { z21.h }, p2/Z, [x21, x27, LSL #1]\n" + "ld1h { z26.h }, p2/Z, [x20, x27, LSL #1]\n" + "ld1h { z16.h }, p1/Z, [x23, x26, LSL #1]\n" + "ld1h { z25.h }, p1/Z, [x22, x26, LSL #1]\n" + "ld1h { z20.h }, p1/Z, [x21, x26, LSL #1]\n" + "ld1h { z24.h }, p1/Z, [x20, x26, LSL #1]\n" "beq 3f\n" "2:" // 4-vectors of channels: 4 inputs loop "movprfx z19, z0\n fmax z19.h, p0/M, z19.h, z31.h\n" "fmax z23.h, p0/M, z23.h, z30.h\n" - "ldp x24, x23, [x20, #0x0]\n" + "ldp x23, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" "fmax z18.h, p0/M, z18.h, z29.h\n" "fmax z22.h, p0/M, z22.h, z28.h\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" "fmax z17.h, p0/M, z17.h, z27.h\n" "fmax z21.h, p0/M, z21.h, z26.h\n" - "ld1h { z0.h }, p4/Z, [x24, x9, LSL #1]\n" + "ld1h { z0.h }, p4/Z, [x23, x9, LSL #1]\n" "fmax z16.h, p0/M, z16.h, z25.h\n" "fmax z20.h, p0/M, z20.h, z24.h\n" - "ld1h { z31.h }, p4/Z, [x23, x9, LSL #1]\n" + "ld1h { z31.h }, p4/Z, [x22, x9, LSL #1]\n" "fmax z19.h, p0/M, z19.h, z23.h\n" "fmax z18.h, p0/M, z18.h, z22.h\n" - "ld1h { z23.h }, p4/Z, [x22, x9, LSL #1]\n" + "ld1h { z23.h }, p4/Z, [x21, x9, LSL #1]\n" "fmax z17.h, p0/M, z17.h, z21.h\n" "fmax z16.h, p0/M, z16.h, z20.h\n" - "ld1h { z30.h }, p4/Z, [x21, x9, LSL #1]\n" + "ld1h { z30.h }, p4/Z, [x20, x9, LSL #1]\n" "fmax z4.h, p0/M, z4.h, z19.h\n" "fmax z3.h, p0/M, z3.h, z18.h\n" - "ld1h { z18.h }, p3/Z, [x24, x28, LSL #1]\n" + "ld1h { z18.h }, p3/Z, [x23, x28, LSL #1]\n" "fmax z2.h, p0/M, z2.h, z17.h\n" "fmax z1.h, p0/M, z1.h, z16.h\n" - "ld1h { z29.h }, p3/Z, [x23, x28, LSL #1]\n" - "ld1h { z22.h }, p3/Z, [x22, x28, LSL #1]\n" - "ld1h { z28.h }, p3/Z, [x21, x28, LSL #1]\n" - "ld1h { z17.h }, p2/Z, [x24, x27, LSL #1]\n" - "ld1h { z27.h }, p2/Z, [x23, x27, LSL #1]\n" - "ld1h { z21.h }, p2/Z, [x22, x27, LSL #1]\n" - "ld1h { z26.h }, p2/Z, [x21, x27, LSL #1]\n" - "ld1h { z16.h }, p1/Z, [x24, x26, LSL #1]\n" - "ld1h { z25.h }, p1/Z, [x23, x26, LSL #1]\n" - "ld1h { z20.h }, p1/Z, [x22, x26, LSL #1]\n" - "ld1h { z24.h }, p1/Z, [x21, x26, LSL #1]\n" + "ld1h { z29.h }, p3/Z, [x22, x28, LSL #1]\n" + "ld1h { z22.h }, p3/Z, [x21, x28, LSL #1]\n" + "ld1h { z28.h }, p3/Z, [x20, x28, LSL #1]\n" + "ld1h { z17.h }, p2/Z, [x23, x27, LSL #1]\n" + "ld1h { z27.h }, p2/Z, [x22, x27, LSL #1]\n" + "ld1h { z21.h }, p2/Z, [x21, x27, LSL #1]\n" + "ld1h { z26.h }, p2/Z, [x20, x27, LSL #1]\n" + "ld1h { z16.h }, p1/Z, [x23, x26, LSL #1]\n" + "ld1h { z25.h }, p1/Z, [x22, x26, LSL #1]\n" + "ld1h { z20.h }, p1/Z, [x21, x26, LSL #1]\n" + "ld1h { z24.h }, p1/Z, [x20, x26, LSL #1]\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "movprfx z19, z0\n fmax z19.h, p0/M, z19.h, z31.h\n" @@ -138,15 +139,15 @@ void sme_fp16_nhwc_max_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1h { z0.h }, p4/Z, [x24, x9, LSL #1]\n" + "ldr x20, [x24], #0x8\n" + "ld1h { z16.h }, p4/Z, [x20, x9, LSL #1]\n" "subs x21, x21, #0x1\n" - "fmax z4.h, p0/M, z4.h, z0.h\n" - "ld1h { z18.h }, p3/Z, [x24, x28, LSL #1]\n" - "fmax z3.h, p0/M, z3.h, z18.h\n" - "ld1h { z17.h }, p2/Z, [x24, x27, LSL #1]\n" - "fmax z2.h, p0/M, z2.h, z17.h\n" - "ld1h { z16.h }, p1/Z, [x24, x26, LSL #1]\n" + "fmax z4.h, p0/M, z4.h, z16.h\n" + "ld1h { z16.h }, p3/Z, [x20, x28, LSL #1]\n" + "fmax z3.h, p0/M, z3.h, z16.h\n" + "ld1h { z16.h }, p2/Z, [x20, x27, LSL #1]\n" + "fmax z2.h, p0/M, z2.h, z16.h\n" + "ld1h { z16.h }, p1/Z, [x20, x26, LSL #1]\n" "fmax z1.h, p0/M, z1.h, z16.h\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End @@ -166,44 +167,44 @@ void sme_fp16_nhwc_max_generic_depthfirst_impl( "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" "mov z4.h, #0xfc00\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x24, x23, [x20, #0x0]\n" + "ldp x20, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" - "ld1h { z0.h }, p4/Z, [x24, x9, LSL #1]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ld1h { z31.h }, p4/Z, [x23, x9, LSL #1]\n" - "ld1h { z23.h }, p4/Z, [x22, x9, LSL #1]\n" - "ld1h { z30.h }, p4/Z, [x21, x9, LSL #1]\n" + "ld1h { z0.h }, p4/Z, [x20, x9, LSL #1]\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" + "ld1h { z31.h }, p4/Z, [x22, x9, LSL #1]\n" + "ld1h { z23.h }, p4/Z, [x21, x9, LSL #1]\n" + "ld1h { z30.h }, p4/Z, [x20, x9, LSL #1]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 4 inputs loop - "movprfx z19, z0\n fmax z19.h, p0/M, z19.h, z31.h\n" - "fmax z23.h, p0/M, z23.h, z30.h\n" - "ldp x24, x23, [x20, #0x0]\n" + "movprfx z16, z0\n fmax z16.h, p0/M, z16.h, z31.h\n" + "movprfx z17, z23\n fmax z17.h, p0/M, z17.h, z30.h\n" + "ldp x23, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" - "fmax z19.h, p0/M, z19.h, z23.h\n" - "ldp x22, x21, [x20, #0x10]\n" - "fmax z4.h, p0/M, z4.h, z19.h\n" - "add x20, x20, #0x20\n" - "ld1h { z0.h }, p4/Z, [x24, x9, LSL #1]\n" - "ld1h { z31.h }, p4/Z, [x23, x9, LSL #1]\n" - "ld1h { z23.h }, p4/Z, [x22, x9, LSL #1]\n" - "ld1h { z30.h }, p4/Z, [x21, x9, LSL #1]\n" + "fmax z16.h, p0/M, z16.h, z17.h\n" + "ldp x21, x20, [x24, #0x10]\n" + "fmax z4.h, p0/M, z4.h, z16.h\n" + "add x24, x24, #0x20\n" + "ld1h { z0.h }, p4/Z, [x23, x9, LSL #1]\n" + "ld1h { z31.h }, p4/Z, [x22, x9, LSL #1]\n" + "ld1h { z23.h }, p4/Z, [x21, x9, LSL #1]\n" + "ld1h { z30.h }, p4/Z, [x20, x9, LSL #1]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail - "movprfx z19, z0\n fmax z19.h, p0/M, z19.h, z31.h\n" - "fmax z23.h, p0/M, z23.h, z30.h\n" - "fmax z19.h, p0/M, z19.h, z23.h\n" - "fmax z4.h, p0/M, z4.h, z19.h\n" + "movprfx z16, z0\n fmax z16.h, p0/M, z16.h, z31.h\n" + "movprfx z17, z23\n fmax z17.h, p0/M, z17.h, z30.h\n" + "fmax z16.h, p0/M, z16.h, z17.h\n" + "fmax z4.h, p0/M, z4.h, z16.h\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1h { z0.h }, p4/Z, [x24, x9, LSL #1]\n" + "ldr x20, [x24], #0x8\n" + "ld1h { z16.h }, p4/Z, [x20, x9, LSL #1]\n" "subs x21, x21, #0x1\n" - "fmax z4.h, p0/M, z4.h, z0.h\n" + "fmax z4.h, p0/M, z4.h, z16.h\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "st1h { z4.h }, p4, [%x[outptr], x9, LSL #1]\n" @@ -221,4 +222,4 @@ void sme_fp16_nhwc_max_generic_depthfirst_impl( } // namespace pooling } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SME) +#endif // defined(ARM_COMPUTE_ENABLE_SME) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst.hpp index fa1b4413713ce6b117d87b4b3aef437da73ec85f..23a0eee04eae81e4ffcc4835971ee7ce82b527fe 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,8 @@ #pragma once +#if defined(ARM_COMPUTE_ENABLE_SME) + namespace arm_conv { namespace pooling { @@ -45,3 +47,5 @@ struct sme_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst : public DepthfirstStrategy } // namespace pooling } // namespace arm_conv + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp index cf69800522e2cc4666fe4e7f59aa212882bd6f41..8c8532827aab02db9357825f75c9c1d813865040 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp @@ -26,7 +26,7 @@ #include #include -#if defined(__ARM_FEATURE_SVE) && defined(ARM_COMPUTE_ENABLE_SME) +#if defined(ARM_COMPUTE_ENABLE_SME) namespace arm_conv { namespace pooling { @@ -91,34 +91,34 @@ void sme_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst_impl( "add x20, %x[args], %[offsetof_rescale]\n" "ld1rqw { z4.s }, p0/Z, [x20]\n" "ldr x5, [%x[args], %[offsetof_n_channels]]\n" - "whilelt p1.s, x3, x5\n" + "whilelt p0.s, x3, x5\n" "mov x6, #0x0\n" "ldp x7, x8, [x21, #0x0]\n" "ldp x17, x16, [x21, #0x10]\n" "ldp x15, x14, [x4, #0x0]\n" - "ld1w { z3.s }, p1/Z, [x14, x3, LSL #2]\n" + "ld1w { z3.s }, p0/Z, [x14, x3, LSL #2]\n" "ldp x13, x12, [x4, #0x10]\n" - "ld1w { z2.s }, p1/Z, [x13, x3, LSL #2]\n" + "ld1w { z2.s }, p0/Z, [x13, x3, LSL #2]\n" "ldp x11, x10, [x4, #0x20]\n" - "ld1w { z1.s }, p1/Z, [x10, x3, LSL #2]\n" + "ld1w { z1.s }, p0/Z, [x10, x3, LSL #2]\n" "ldp x9, x28, [x4, #0x30]\n" - "ld1w { z0.s }, p1/Z, [x9, x3, LSL #2]\n" + "ld1w { z0.s }, p0/Z, [x9, x3, LSL #2]\n" "ldp x27, x26, [x4, #0x40]\n" - "ld1w { z31.s }, p1/Z, [x26, x3, LSL #2]\n" + "ld1w { z31.s }, p0/Z, [x26, x3, LSL #2]\n" "ldp x25, x24, [x4, #0x50]\n" - "ld1w { z30.s }, p1/Z, [x25, x3, LSL #2]\n" + "ld1w { z30.s }, p0/Z, [x25, x3, LSL #2]\n" "ldp x23, x22, [x4, #0x60]\n" - "ld1w { z29.s }, p1/Z, [x11, x3, LSL #2]\n" + "ld1w { z29.s }, p0/Z, [x11, x3, LSL #2]\n" "ldp x21, x20, [x4, #0x70]\n" - "ld1w { z28.s }, p1/Z, [x27, x3, LSL #2]\n" - "ld1w { z27.s }, p1/Z, [x28, x3, LSL #2]\n" - "ld1w { z22.s }, p1/Z, [x24, x3, LSL #2]\n" - "ld1w { z21.s }, p1/Z, [x22, x3, LSL #2]\n" - "ld1w { z20.s }, p1/Z, [x21, x3, LSL #2]\n" - "ld1w { z26.s }, p1/Z, [x15, x3, LSL #2]\n" - "ld1w { z25.s }, p1/Z, [x12, x3, LSL #2]\n" - "ld1w { z24.s }, p1/Z, [x23, x3, LSL #2]\n" - "ld1w { z23.s }, p1/Z, [x20, x3, LSL #2]\n" + "ld1w { z28.s }, p0/Z, [x27, x3, LSL #2]\n" + "ld1w { z27.s }, p0/Z, [x28, x3, LSL #2]\n" + "ld1w { z22.s }, p0/Z, [x24, x3, LSL #2]\n" + "ld1w { z21.s }, p0/Z, [x22, x3, LSL #2]\n" + "ld1w { z20.s }, p0/Z, [x21, x3, LSL #2]\n" + "ld1w { z26.s }, p0/Z, [x15, x3, LSL #2]\n" + "ld1w { z25.s }, p0/Z, [x12, x3, LSL #2]\n" + "ld1w { z24.s }, p0/Z, [x23, x3, LSL #2]\n" + "ld1w { z23.s }, p0/Z, [x20, x3, LSL #2]\n" "incw x3\n" "whilelt p1.s, x3, x5\n" "b.none 2f\n" @@ -206,4 +206,4 @@ void sme_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst_impl( } // namespace pooling } // namespace arm_conv -#endif // defined(__ARM_FEATURE_SVE) && defined(ARM_COMPUTE_ENABLE_SME) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp32_nhwc_avg_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp32_nhwc_avg_generic_depthfirst.hpp index 814c89ca23992f44053b9dd984b1a757938b3555..29bcfc5a3bea7b7f5d85f7ae0a295d1ccf6bf061 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp32_nhwc_avg_generic_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp32_nhwc_avg_generic_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,8 @@ #pragma once +#if defined(ARM_COMPUTE_ENABLE_SME) + namespace arm_conv { namespace pooling { @@ -40,3 +42,5 @@ struct sme_fp32_nhwc_avg_generic_depthfirst : IGenericDepthfirstStrategy +#include + +#if defined(ARM_COMPUTE_ENABLE_SME) namespace arm_conv { namespace pooling { @@ -57,68 +58,68 @@ void sme_fp32_nhwc_avg_generic_depthfirst_impl( "lsr x25, %x[n_valid_cells], #0x2\n" "mov z5.b, #0x0\n" "mov z4.b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "mov z3.b, #0x0\n" "mov z2.b, #0x0\n" "cbz x25, 4f\n" - "ldp x24, x23, [x20, #0x0]\n" + "ldp x23, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" - "ld1w { z1.s }, p3/Z, [x24, x9, LSL #2]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ld1w { z0.s }, p3/Z, [x23, x9, LSL #2]\n" - "ld1w { z31.s }, p3/Z, [x22, x9, LSL #2]\n" - "ld1w { z30.s }, p3/Z, [x21, x9, LSL #2]\n" - "ld1w { z29.s }, p2/Z, [x24, x28, LSL #2]\n" - "ld1w { z22.s }, p2/Z, [x23, x28, LSL #2]\n" - "ld1w { z28.s }, p2/Z, [x22, x28, LSL #2]\n" - "ld1w { z18.s }, p2/Z, [x21, x28, LSL #2]\n" - "ld1w { z27.s }, p1/Z, [x24, x27, LSL #2]\n" - "ld1w { z21.s }, p1/Z, [x23, x27, LSL #2]\n" - "ld1w { z26.s }, p1/Z, [x22, x27, LSL #2]\n" - "ld1w { z17.s }, p1/Z, [x21, x27, LSL #2]\n" - "ld1w { z25.s }, p0/Z, [x24, x26, LSL #2]\n" - "ld1w { z20.s }, p0/Z, [x23, x26, LSL #2]\n" - "ld1w { z24.s }, p0/Z, [x22, x26, LSL #2]\n" - "ld1w { z16.s }, p0/Z, [x21, x26, LSL #2]\n" + "ld1w { z1.s }, p3/Z, [x23, x9, LSL #2]\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" + "ld1w { z0.s }, p3/Z, [x22, x9, LSL #2]\n" + "ld1w { z31.s }, p3/Z, [x21, x9, LSL #2]\n" + "ld1w { z30.s }, p3/Z, [x20, x9, LSL #2]\n" + "ld1w { z29.s }, p2/Z, [x23, x28, LSL #2]\n" + "ld1w { z22.s }, p2/Z, [x22, x28, LSL #2]\n" + "ld1w { z28.s }, p2/Z, [x21, x28, LSL #2]\n" + "ld1w { z18.s }, p2/Z, [x20, x28, LSL #2]\n" + "ld1w { z27.s }, p1/Z, [x23, x27, LSL #2]\n" + "ld1w { z21.s }, p1/Z, [x22, x27, LSL #2]\n" + "ld1w { z26.s }, p1/Z, [x21, x27, LSL #2]\n" + "ld1w { z17.s }, p1/Z, [x20, x27, LSL #2]\n" + "ld1w { z25.s }, p0/Z, [x23, x26, LSL #2]\n" + "ld1w { z20.s }, p0/Z, [x22, x26, LSL #2]\n" + "ld1w { z24.s }, p0/Z, [x21, x26, LSL #2]\n" + "ld1w { z16.s }, p0/Z, [x20, x26, LSL #2]\n" "beq 3f\n" "2:" // 4-vectors of channels: 4 inputs loop "fadd z23.s, z1.s, z0.s\n" "fadd z19.s, z31.s, z30.s\n" - "ldp x24, x23, [x20, #0x0]\n" + "ldp x23, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" "fadd z22.s, z29.s, z22.s\n" "fadd z18.s, z28.s, z18.s\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" "fadd z21.s, z27.s, z21.s\n" "fadd z17.s, z26.s, z17.s\n" - "ld1w { z1.s }, p3/Z, [x24, x9, LSL #2]\n" + "ld1w { z1.s }, p3/Z, [x23, x9, LSL #2]\n" "fadd z20.s, z25.s, z20.s\n" "fadd z16.s, z24.s, z16.s\n" - "ld1w { z0.s }, p3/Z, [x23, x9, LSL #2]\n" + "ld1w { z0.s }, p3/Z, [x22, x9, LSL #2]\n" "fadd z19.s, z23.s, z19.s\n" "fadd z18.s, z22.s, z18.s\n" - "ld1w { z31.s }, p3/Z, [x22, x9, LSL #2]\n" + "ld1w { z31.s }, p3/Z, [x21, x9, LSL #2]\n" "fadd z17.s, z21.s, z17.s\n" "fadd z16.s, z20.s, z16.s\n" - "ld1w { z30.s }, p3/Z, [x21, x9, LSL #2]\n" + "ld1w { z30.s }, p3/Z, [x20, x9, LSL #2]\n" "fadd z5.s, z5.s, z19.s\n" "fadd z4.s, z4.s, z18.s\n" - "ld1w { z29.s }, p2/Z, [x24, x28, LSL #2]\n" + "ld1w { z29.s }, p2/Z, [x23, x28, LSL #2]\n" "fadd z3.s, z3.s, z17.s\n" "fadd z2.s, z2.s, z16.s\n" - "ld1w { z22.s }, p2/Z, [x23, x28, LSL #2]\n" - "ld1w { z28.s }, p2/Z, [x22, x28, LSL #2]\n" - "ld1w { z18.s }, p2/Z, [x21, x28, LSL #2]\n" - "ld1w { z27.s }, p1/Z, [x24, x27, LSL #2]\n" - "ld1w { z21.s }, p1/Z, [x23, x27, LSL #2]\n" - "ld1w { z26.s }, p1/Z, [x22, x27, LSL #2]\n" - "ld1w { z17.s }, p1/Z, [x21, x27, LSL #2]\n" - "ld1w { z25.s }, p0/Z, [x24, x26, LSL #2]\n" - "ld1w { z20.s }, p0/Z, [x23, x26, LSL #2]\n" - "ld1w { z24.s }, p0/Z, [x22, x26, LSL #2]\n" - "ld1w { z16.s }, p0/Z, [x21, x26, LSL #2]\n" + "ld1w { z22.s }, p2/Z, [x22, x28, LSL #2]\n" + "ld1w { z28.s }, p2/Z, [x21, x28, LSL #2]\n" + "ld1w { z18.s }, p2/Z, [x20, x28, LSL #2]\n" + "ld1w { z27.s }, p1/Z, [x23, x27, LSL #2]\n" + "ld1w { z21.s }, p1/Z, [x22, x27, LSL #2]\n" + "ld1w { z26.s }, p1/Z, [x21, x27, LSL #2]\n" + "ld1w { z17.s }, p1/Z, [x20, x27, LSL #2]\n" + "ld1w { z25.s }, p0/Z, [x23, x26, LSL #2]\n" + "ld1w { z20.s }, p0/Z, [x22, x26, LSL #2]\n" + "ld1w { z24.s }, p0/Z, [x21, x26, LSL #2]\n" + "ld1w { z16.s }, p0/Z, [x20, x26, LSL #2]\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "fadd z23.s, z1.s, z0.s\n" @@ -141,16 +142,16 @@ void sme_fp32_nhwc_avg_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1w { z1.s }, p3/Z, [x24, x9, LSL #2]\n" + "ldr x20, [x24], #0x8\n" + "ld1w { z16.s }, p3/Z, [x20, x9, LSL #2]\n" "subs x21, x21, #0x1\n" - "fadd z5.s, z5.s, z1.s\n" - "ld1w { z29.s }, p2/Z, [x24, x28, LSL #2]\n" - "fadd z4.s, z4.s, z29.s\n" - "ld1w { z27.s }, p1/Z, [x24, x27, LSL #2]\n" - "fadd z3.s, z3.s, z27.s\n" - "ld1w { z25.s }, p0/Z, [x24, x26, LSL #2]\n" - "fadd z2.s, z2.s, z25.s\n" + "fadd z5.s, z5.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x20, x28, LSL #2]\n" + "fadd z4.s, z4.s, z16.s\n" + "ld1w { z16.s }, p1/Z, [x20, x27, LSL #2]\n" + "fadd z3.s, z3.s, z16.s\n" + "ld1w { z16.s }, p0/Z, [x20, x26, LSL #2]\n" + "fadd z2.s, z2.s, z16.s\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End "fmul z5.s, z5.s, z6.s\n" @@ -173,44 +174,44 @@ void sme_fp32_nhwc_avg_generic_depthfirst_impl( "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" "mov z5.b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x24, x23, [x20, #0x0]\n" + "ldp x20, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" - "ld1w { z1.s }, p3/Z, [x24, x9, LSL #2]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ld1w { z0.s }, p3/Z, [x23, x9, LSL #2]\n" - "ld1w { z31.s }, p3/Z, [x22, x9, LSL #2]\n" - "ld1w { z30.s }, p3/Z, [x21, x9, LSL #2]\n" + "ld1w { z1.s }, p3/Z, [x20, x9, LSL #2]\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" + "ld1w { z0.s }, p3/Z, [x22, x9, LSL #2]\n" + "ld1w { z31.s }, p3/Z, [x21, x9, LSL #2]\n" + "ld1w { z30.s }, p3/Z, [x20, x9, LSL #2]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 4 inputs loop - "fadd z23.s, z1.s, z0.s\n" - "fadd z19.s, z31.s, z30.s\n" - "ldp x24, x23, [x20, #0x0]\n" + "fadd z17.s, z1.s, z0.s\n" + "fadd z16.s, z31.s, z30.s\n" + "ldp x23, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" - "fadd z19.s, z23.s, z19.s\n" - "ldp x22, x21, [x20, #0x10]\n" - "fadd z5.s, z5.s, z19.s\n" - "add x20, x20, #0x20\n" - "ld1w { z1.s }, p3/Z, [x24, x9, LSL #2]\n" - "ld1w { z0.s }, p3/Z, [x23, x9, LSL #2]\n" - "ld1w { z31.s }, p3/Z, [x22, x9, LSL #2]\n" - "ld1w { z30.s }, p3/Z, [x21, x9, LSL #2]\n" + "fadd z16.s, z17.s, z16.s\n" + "ldp x21, x20, [x24, #0x10]\n" + "fadd z5.s, z5.s, z16.s\n" + "add x24, x24, #0x20\n" + "ld1w { z1.s }, p3/Z, [x23, x9, LSL #2]\n" + "ld1w { z0.s }, p3/Z, [x22, x9, LSL #2]\n" + "ld1w { z31.s }, p3/Z, [x21, x9, LSL #2]\n" + "ld1w { z30.s }, p3/Z, [x20, x9, LSL #2]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail - "fadd z23.s, z1.s, z0.s\n" - "fadd z19.s, z31.s, z30.s\n" - "fadd z19.s, z23.s, z19.s\n" - "fadd z5.s, z5.s, z19.s\n" + "fadd z17.s, z1.s, z0.s\n" + "fadd z16.s, z31.s, z30.s\n" + "fadd z16.s, z17.s, z16.s\n" + "fadd z5.s, z5.s, z16.s\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1w { z1.s }, p3/Z, [x24, x9, LSL #2]\n" + "ldr x20, [x24], #0x8\n" + "ld1w { z16.s }, p3/Z, [x20, x9, LSL #2]\n" "subs x21, x21, #0x1\n" - "fadd z5.s, z5.s, z1.s\n" + "fadd z5.s, z5.s, z16.s\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "fmul z5.s, z5.s, z6.s\n" diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp32_nhwc_max_2x2_s1_output2x2_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp32_nhwc_max_2x2_s1_output2x2_depthfirst.hpp index 4e3cd6e22837e1a3e53303794d54e0c486ec9dac..338348231ff2f9f985df45bdb02a62accc6052db 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp32_nhwc_max_2x2_s1_output2x2_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp32_nhwc_max_2x2_s1_output2x2_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,8 @@ #pragma once +#if defined(ARM_COMPUTE_ENABLE_SME) + namespace arm_conv { namespace pooling { @@ -45,3 +47,5 @@ struct sme_fp32_nhwc_max_2x2_s1_output2x2_depthfirst : public DepthfirstStrategy } // namespace pooling } // namespace arm_conv + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp32_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp32_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp index 05edac662349082800279e40efcc50837ff3ed7e..3c7213a4989af6929f8ec38ed5456a6def98477b 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp32_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp32_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp @@ -26,7 +26,7 @@ #include #include -#if defined(__ARM_FEATURE_SVE) && defined(ARM_COMPUTE_ENABLE_SME) +#if defined(ARM_COMPUTE_ENABLE_SME) namespace arm_conv { namespace pooling { @@ -70,23 +70,23 @@ void sme_fp32_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "ldr x20, [%x[args], %[offsetof_inptrs]]\n" "mov x14, #0x0\n" "ldr x13, [%x[args], %[offsetof_n_channels]]\n" - "whilelt p1.s, x15, x13\n" + "whilelt p0.s, x15, x13\n" "ldp x12, x11, [x21, #0x0]\n" "ldp x10, x9, [x21, #0x10]\n" "ldp x28, x27, [x20, #0x0]\n" - "ld1w { z30.s }, p1/Z, [x27, x15, LSL #2]\n" + "ld1w { z30.s }, p0/Z, [x27, x15, LSL #2]\n" "ldp x26, x25, [x20, #0x10]\n" - "ld1w { z29.s }, p1/Z, [x25, x15, LSL #2]\n" + "ld1w { z29.s }, p0/Z, [x25, x15, LSL #2]\n" "ldp x24, x23, [x20, #0x20]\n" - "ld1w { z28.s }, p1/Z, [x24, x15, LSL #2]\n" + "ld1w { z28.s }, p0/Z, [x24, x15, LSL #2]\n" "ldp x22, x21, [x20, #0x30]\n" - "ld1w { z27.s }, p1/Z, [x21, x15, LSL #2]\n" + "ld1w { z27.s }, p0/Z, [x21, x15, LSL #2]\n" "ldr x20, [x20, #0x40]\n" - "ld1w { z26.s }, p1/Z, [x28, x15, LSL #2]\n" - "ld1w { z25.s }, p1/Z, [x26, x15, LSL #2]\n" - "ld1w { z24.s }, p1/Z, [x23, x15, LSL #2]\n" - "ld1w { z23.s }, p1/Z, [x22, x15, LSL #2]\n" - "ld1w { z19.s }, p1/Z, [x20, x15, LSL #2]\n" + "ld1w { z26.s }, p0/Z, [x28, x15, LSL #2]\n" + "ld1w { z25.s }, p0/Z, [x26, x15, LSL #2]\n" + "ld1w { z24.s }, p0/Z, [x23, x15, LSL #2]\n" + "ld1w { z19.s }, p0/Z, [x22, x15, LSL #2]\n" + "ld1w { z23.s }, p0/Z, [x20, x15, LSL #2]\n" "incw x15\n" "whilelt p1.s, x15, x13\n" "b.none 2f\n" @@ -95,25 +95,25 @@ void sme_fp32_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "movprfx z21, z28\n fmax z21.s, p2/M, z21.s, z27.s\n" "ld1w { z30.s }, p1/Z, [x27, x15, LSL #2]\n" "whilelt p0.s, x14, x13\n" - "movprfx z20, z29\n fmax z20.s, p2/M, z20.s, z26.s\n" - "movprfx z18, z25\n fmax z18.s, p2/M, z18.s, z24.s\n" + "movprfx z18, z29\n fmax z18.s, p2/M, z18.s, z26.s\n" + "movprfx z17, z25\n fmax z17.s, p2/M, z17.s, z24.s\n" "ld1w { z28.s }, p1/Z, [x24, x15, LSL #2]\n" - "movprfx z17, z29\n fmax z17.s, p2/M, z17.s, z23.s\n" - "movprfx z16, z24\n fmax z16.s, p2/M, z16.s, z19.s\n" + "movprfx z16, z29\n fmax z16.s, p2/M, z16.s, z19.s\n" + "movprfx z20, z24\n fmax z20.s, p2/M, z20.s, z23.s\n" "ld1w { z27.s }, p1/Z, [x21, x15, LSL #2]\n" "ld1w { z29.s }, p1/Z, [x25, x15, LSL #2]\n" - "movprfx z19, z22\n fmax z19.s, p2/M, z19.s, z20.s\n" - "fmax z18.s, p2/M, z18.s, z22.s\n" + "movprfx z19, z22\n fmax z19.s, p2/M, z19.s, z18.s\n" + "movprfx z18, z17\n fmax z18.s, p2/M, z18.s, z22.s\n" "ld1w { z26.s }, p1/Z, [x28, x15, LSL #2]\n" - "fmax z17.s, p2/M, z17.s, z21.s\n" - "fmax z16.s, p2/M, z16.s, z21.s\n" + "movprfx z17, z16\n fmax z17.s, p2/M, z17.s, z21.s\n" + "movprfx z16, z21\n fmax z16.s, p2/M, z16.s, z20.s\n" "ld1w { z25.s }, p1/Z, [x26, x15, LSL #2]\n" "st1w { z19.s }, p0, [x12, x14, LSL #2]\n" "ld1w { z24.s }, p1/Z, [x23, x15, LSL #2]\n" "st1w { z18.s }, p0, [x11, x14, LSL #2]\n" - "ld1w { z23.s }, p1/Z, [x22, x15, LSL #2]\n" + "ld1w { z19.s }, p1/Z, [x22, x15, LSL #2]\n" "st1w { z17.s }, p0, [x10, x14, LSL #2]\n" - "ld1w { z19.s }, p1/Z, [x20, x15, LSL #2]\n" + "ld1w { z23.s }, p1/Z, [x20, x15, LSL #2]\n" "incw x15\n" "whilelt p1.s, x15, x13\n" "st1w { z16.s }, p0, [x9, x14, LSL #2]\n" @@ -125,13 +125,13 @@ void sme_fp32_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "whilelt p0.s, x14, x13\n" "movprfx z20, z29\n fmax z20.s, p2/M, z20.s, z26.s\n" "movprfx z18, z25\n fmax z18.s, p2/M, z18.s, z24.s\n" - "movprfx z17, z29\n fmax z17.s, p2/M, z17.s, z23.s\n" - "movprfx z16, z24\n fmax z16.s, p2/M, z16.s, z19.s\n" - "movprfx z19, z22\n fmax z19.s, p2/M, z19.s, z20.s\n" + "movprfx z17, z29\n fmax z17.s, p2/M, z17.s, z19.s\n" + "movprfx z19, z24\n fmax z19.s, p2/M, z19.s, z23.s\n" + "movprfx z16, z22\n fmax z16.s, p2/M, z16.s, z20.s\n" "fmax z18.s, p2/M, z18.s, z22.s\n" - "st1w { z19.s }, p0, [x12, x14, LSL #2]\n" + "st1w { z16.s }, p0, [x12, x14, LSL #2]\n" "fmax z17.s, p2/M, z17.s, z21.s\n" - "fmax z16.s, p2/M, z16.s, z21.s\n" + "movprfx z16, z21\n fmax z16.s, p2/M, z16.s, z19.s\n" "st1w { z18.s }, p0, [x11, x14, LSL #2]\n" "st1w { z17.s }, p0, [x10, x14, LSL #2]\n" "st1w { z16.s }, p0, [x9, x14, LSL #2]\n" @@ -145,4 +145,4 @@ void sme_fp32_nhwc_max_2x2_s1_output2x2_depthfirst_impl( } // namespace pooling } // namespace arm_conv -#endif // defined(__ARM_FEATURE_SVE) && defined(ARM_COMPUTE_ENABLE_SME) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp32_nhwc_max_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp32_nhwc_max_generic_depthfirst.hpp index 0c0e445c7a4aca1625afd1162ae4b56b03cd9d9a..9bc1f116017d5d1e344c8b68aa851100c81bd880 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp32_nhwc_max_generic_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_fp32_nhwc_max_generic_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,8 @@ #pragma once +#if defined(ARM_COMPUTE_ENABLE_SME) + namespace arm_conv { namespace pooling { @@ -40,3 +42,5 @@ struct sme_fp32_nhwc_max_generic_depthfirst : IGenericDepthfirstStrategy +#include + +#if defined(ARM_COMPUTE_ENABLE_SME) namespace arm_conv { namespace pooling { @@ -54,68 +55,68 @@ void sme_fp32_nhwc_max_generic_depthfirst_impl( "lsr x25, %x[n_valid_cells], #0x2\n" "mov z4.s, #0xff800000\n" "mov z3.s, #0xff800000\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "mov z2.s, #0xff800000\n" "mov z1.s, #0xff800000\n" "cbz x25, 4f\n" - "ldp x24, x23, [x20, #0x0]\n" + "ldp x23, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" - "ld1w { z0.s }, p4/Z, [x24, x9, LSL #2]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ld1w { z31.s }, p4/Z, [x23, x9, LSL #2]\n" - "ld1w { z23.s }, p4/Z, [x22, x9, LSL #2]\n" - "ld1w { z30.s }, p4/Z, [x21, x9, LSL #2]\n" - "ld1w { z18.s }, p3/Z, [x24, x28, LSL #2]\n" - "ld1w { z29.s }, p3/Z, [x23, x28, LSL #2]\n" - "ld1w { z22.s }, p3/Z, [x22, x28, LSL #2]\n" - "ld1w { z28.s }, p3/Z, [x21, x28, LSL #2]\n" - "ld1w { z17.s }, p2/Z, [x24, x27, LSL #2]\n" - "ld1w { z27.s }, p2/Z, [x23, x27, LSL #2]\n" - "ld1w { z21.s }, p2/Z, [x22, x27, LSL #2]\n" - "ld1w { z26.s }, p2/Z, [x21, x27, LSL #2]\n" - "ld1w { z16.s }, p1/Z, [x24, x26, LSL #2]\n" - "ld1w { z25.s }, p1/Z, [x23, x26, LSL #2]\n" - "ld1w { z20.s }, p1/Z, [x22, x26, LSL #2]\n" - "ld1w { z24.s }, p1/Z, [x21, x26, LSL #2]\n" + "ld1w { z0.s }, p4/Z, [x23, x9, LSL #2]\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" + "ld1w { z31.s }, p4/Z, [x22, x9, LSL #2]\n" + "ld1w { z23.s }, p4/Z, [x21, x9, LSL #2]\n" + "ld1w { z30.s }, p4/Z, [x20, x9, LSL #2]\n" + "ld1w { z18.s }, p3/Z, [x23, x28, LSL #2]\n" + "ld1w { z29.s }, p3/Z, [x22, x28, LSL #2]\n" + "ld1w { z22.s }, p3/Z, [x21, x28, LSL #2]\n" + "ld1w { z28.s }, p3/Z, [x20, x28, LSL #2]\n" + "ld1w { z17.s }, p2/Z, [x23, x27, LSL #2]\n" + "ld1w { z27.s }, p2/Z, [x22, x27, LSL #2]\n" + "ld1w { z21.s }, p2/Z, [x21, x27, LSL #2]\n" + "ld1w { z26.s }, p2/Z, [x20, x27, LSL #2]\n" + "ld1w { z16.s }, p1/Z, [x23, x26, LSL #2]\n" + "ld1w { z25.s }, p1/Z, [x22, x26, LSL #2]\n" + "ld1w { z20.s }, p1/Z, [x21, x26, LSL #2]\n" + "ld1w { z24.s }, p1/Z, [x20, x26, LSL #2]\n" "beq 3f\n" "2:" // 4-vectors of channels: 4 inputs loop "movprfx z19, z0\n fmax z19.s, p0/M, z19.s, z31.s\n" "fmax z23.s, p0/M, z23.s, z30.s\n" - "ldp x24, x23, [x20, #0x0]\n" + "ldp x23, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" "fmax z18.s, p0/M, z18.s, z29.s\n" "fmax z22.s, p0/M, z22.s, z28.s\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" "fmax z17.s, p0/M, z17.s, z27.s\n" "fmax z21.s, p0/M, z21.s, z26.s\n" - "ld1w { z0.s }, p4/Z, [x24, x9, LSL #2]\n" + "ld1w { z0.s }, p4/Z, [x23, x9, LSL #2]\n" "fmax z16.s, p0/M, z16.s, z25.s\n" "fmax z20.s, p0/M, z20.s, z24.s\n" - "ld1w { z31.s }, p4/Z, [x23, x9, LSL #2]\n" + "ld1w { z31.s }, p4/Z, [x22, x9, LSL #2]\n" "fmax z19.s, p0/M, z19.s, z23.s\n" "fmax z18.s, p0/M, z18.s, z22.s\n" - "ld1w { z23.s }, p4/Z, [x22, x9, LSL #2]\n" + "ld1w { z23.s }, p4/Z, [x21, x9, LSL #2]\n" "fmax z17.s, p0/M, z17.s, z21.s\n" "fmax z16.s, p0/M, z16.s, z20.s\n" - "ld1w { z30.s }, p4/Z, [x21, x9, LSL #2]\n" + "ld1w { z30.s }, p4/Z, [x20, x9, LSL #2]\n" "fmax z4.s, p0/M, z4.s, z19.s\n" "fmax z3.s, p0/M, z3.s, z18.s\n" - "ld1w { z18.s }, p3/Z, [x24, x28, LSL #2]\n" + "ld1w { z18.s }, p3/Z, [x23, x28, LSL #2]\n" "fmax z2.s, p0/M, z2.s, z17.s\n" "fmax z1.s, p0/M, z1.s, z16.s\n" - "ld1w { z29.s }, p3/Z, [x23, x28, LSL #2]\n" - "ld1w { z22.s }, p3/Z, [x22, x28, LSL #2]\n" - "ld1w { z28.s }, p3/Z, [x21, x28, LSL #2]\n" - "ld1w { z17.s }, p2/Z, [x24, x27, LSL #2]\n" - "ld1w { z27.s }, p2/Z, [x23, x27, LSL #2]\n" - "ld1w { z21.s }, p2/Z, [x22, x27, LSL #2]\n" - "ld1w { z26.s }, p2/Z, [x21, x27, LSL #2]\n" - "ld1w { z16.s }, p1/Z, [x24, x26, LSL #2]\n" - "ld1w { z25.s }, p1/Z, [x23, x26, LSL #2]\n" - "ld1w { z20.s }, p1/Z, [x22, x26, LSL #2]\n" - "ld1w { z24.s }, p1/Z, [x21, x26, LSL #2]\n" + "ld1w { z29.s }, p3/Z, [x22, x28, LSL #2]\n" + "ld1w { z22.s }, p3/Z, [x21, x28, LSL #2]\n" + "ld1w { z28.s }, p3/Z, [x20, x28, LSL #2]\n" + "ld1w { z17.s }, p2/Z, [x23, x27, LSL #2]\n" + "ld1w { z27.s }, p2/Z, [x22, x27, LSL #2]\n" + "ld1w { z21.s }, p2/Z, [x21, x27, LSL #2]\n" + "ld1w { z26.s }, p2/Z, [x20, x27, LSL #2]\n" + "ld1w { z16.s }, p1/Z, [x23, x26, LSL #2]\n" + "ld1w { z25.s }, p1/Z, [x22, x26, LSL #2]\n" + "ld1w { z20.s }, p1/Z, [x21, x26, LSL #2]\n" + "ld1w { z24.s }, p1/Z, [x20, x26, LSL #2]\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "movprfx z19, z0\n fmax z19.s, p0/M, z19.s, z31.s\n" @@ -138,15 +139,15 @@ void sme_fp32_nhwc_max_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1w { z0.s }, p4/Z, [x24, x9, LSL #2]\n" + "ldr x20, [x24], #0x8\n" + "ld1w { z16.s }, p4/Z, [x20, x9, LSL #2]\n" "subs x21, x21, #0x1\n" - "fmax z4.s, p0/M, z4.s, z0.s\n" - "ld1w { z18.s }, p3/Z, [x24, x28, LSL #2]\n" - "fmax z3.s, p0/M, z3.s, z18.s\n" - "ld1w { z17.s }, p2/Z, [x24, x27, LSL #2]\n" - "fmax z2.s, p0/M, z2.s, z17.s\n" - "ld1w { z16.s }, p1/Z, [x24, x26, LSL #2]\n" + "fmax z4.s, p0/M, z4.s, z16.s\n" + "ld1w { z16.s }, p3/Z, [x20, x28, LSL #2]\n" + "fmax z3.s, p0/M, z3.s, z16.s\n" + "ld1w { z16.s }, p2/Z, [x20, x27, LSL #2]\n" + "fmax z2.s, p0/M, z2.s, z16.s\n" + "ld1w { z16.s }, p1/Z, [x20, x26, LSL #2]\n" "fmax z1.s, p0/M, z1.s, z16.s\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End @@ -166,44 +167,44 @@ void sme_fp32_nhwc_max_generic_depthfirst_impl( "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" "mov z4.s, #0xff800000\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x24, x23, [x20, #0x0]\n" + "ldp x20, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" - "ld1w { z0.s }, p4/Z, [x24, x9, LSL #2]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ld1w { z31.s }, p4/Z, [x23, x9, LSL #2]\n" - "ld1w { z23.s }, p4/Z, [x22, x9, LSL #2]\n" - "ld1w { z30.s }, p4/Z, [x21, x9, LSL #2]\n" + "ld1w { z0.s }, p4/Z, [x20, x9, LSL #2]\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" + "ld1w { z31.s }, p4/Z, [x22, x9, LSL #2]\n" + "ld1w { z23.s }, p4/Z, [x21, x9, LSL #2]\n" + "ld1w { z30.s }, p4/Z, [x20, x9, LSL #2]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 4 inputs loop - "movprfx z19, z0\n fmax z19.s, p0/M, z19.s, z31.s\n" - "fmax z23.s, p0/M, z23.s, z30.s\n" - "ldp x24, x23, [x20, #0x0]\n" + "movprfx z16, z0\n fmax z16.s, p0/M, z16.s, z31.s\n" + "movprfx z17, z23\n fmax z17.s, p0/M, z17.s, z30.s\n" + "ldp x23, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" - "fmax z19.s, p0/M, z19.s, z23.s\n" - "ldp x22, x21, [x20, #0x10]\n" - "fmax z4.s, p0/M, z4.s, z19.s\n" - "add x20, x20, #0x20\n" - "ld1w { z0.s }, p4/Z, [x24, x9, LSL #2]\n" - "ld1w { z31.s }, p4/Z, [x23, x9, LSL #2]\n" - "ld1w { z23.s }, p4/Z, [x22, x9, LSL #2]\n" - "ld1w { z30.s }, p4/Z, [x21, x9, LSL #2]\n" + "fmax z16.s, p0/M, z16.s, z17.s\n" + "ldp x21, x20, [x24, #0x10]\n" + "fmax z4.s, p0/M, z4.s, z16.s\n" + "add x24, x24, #0x20\n" + "ld1w { z0.s }, p4/Z, [x23, x9, LSL #2]\n" + "ld1w { z31.s }, p4/Z, [x22, x9, LSL #2]\n" + "ld1w { z23.s }, p4/Z, [x21, x9, LSL #2]\n" + "ld1w { z30.s }, p4/Z, [x20, x9, LSL #2]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail - "movprfx z19, z0\n fmax z19.s, p0/M, z19.s, z31.s\n" - "fmax z23.s, p0/M, z23.s, z30.s\n" - "fmax z19.s, p0/M, z19.s, z23.s\n" - "fmax z4.s, p0/M, z4.s, z19.s\n" + "movprfx z16, z0\n fmax z16.s, p0/M, z16.s, z31.s\n" + "movprfx z17, z23\n fmax z17.s, p0/M, z17.s, z30.s\n" + "fmax z16.s, p0/M, z16.s, z17.s\n" + "fmax z4.s, p0/M, z4.s, z16.s\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1w { z0.s }, p4/Z, [x24, x9, LSL #2]\n" + "ldr x20, [x24], #0x8\n" + "ld1w { z16.s }, p4/Z, [x20, x9, LSL #2]\n" "subs x21, x21, #0x1\n" - "fmax z4.s, p0/M, z4.s, z0.s\n" + "fmax z4.s, p0/M, z4.s, z16.s\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "st1w { z4.s }, p4, [%x[outptr], x9, LSL #2]\n" diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_s8_nhwc_avg_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_s8_nhwc_avg_generic_depthfirst.hpp index e383a4c3bdf9324585d61f525ac1bb77f15c99a9..318510e697b3b89fb46f4b7f73debe0069f765a6 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_s8_nhwc_avg_generic_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_s8_nhwc_avg_generic_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,8 @@ #pragma once +#if defined(ARM_COMPUTE_ENABLE_SME) + namespace arm_conv { namespace pooling { @@ -40,3 +42,5 @@ struct sme_s8_nhwc_avg_generic_depthfirst : IGenericDepthfirstStrategy +#include #include #include +#if defined(ARM_COMPUTE_ENABLE_SME) + namespace arm_conv { namespace pooling { @@ -109,7 +110,7 @@ void sme_s8_nhwc_avg_generic_depthfirst_impl( "lsr x23, %x[n_valid_cells], #0x1\n" "mov z15.s, #0x0\n" "mov z14.s, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "mov z13.s, #0x0\n" "mov z12.s, #0x0\n" "mov z11.s, #0x0\n" @@ -125,48 +126,48 @@ void sme_s8_nhwc_avg_generic_depthfirst_impl( "mov z1.s, #0x0\n" "mov z0.s, #0x0\n" "cbz x23, 4f\n" - "ldp x22, x21, [x20, #0x0]\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" - "ld1b { z29.b }, p3/Z, [x22, x26]\n" - "ld1b { z28.b }, p3/Z, [x21, x26]\n" - "ld1b { z27.b }, p2/Z, [x22, x25]\n" - "ld1b { z26.b }, p2/Z, [x21, x25]\n" - "ld1b { z25.b }, p1/Z, [x22, x24]\n" - "ld1b { z24.b }, p1/Z, [x21, x24]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" + "ld1b { z29.b }, p3/Z, [x21, x26]\n" + "ld1b { z28.b }, p3/Z, [x20, x26]\n" + "ld1b { z27.b }, p2/Z, [x21, x25]\n" + "ld1b { z26.b }, p2/Z, [x20, x25]\n" + "ld1b { z25.b }, p1/Z, [x21, x24]\n" + "ld1b { z24.b }, p1/Z, [x20, x24]\n" "beq 3f\n" "2:" // 4-vectors of channels: 2 inputs loop ".inst 0x455e03f7 // saddlb z23.h, z31.b, z30.b\n" ".inst 0x455e07f6 // saddlt z22.h, z31.b, z30.b\n" - "ldp x22, x21, [x20, #0x0]\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" ".inst 0x455c03b5 // saddlb z21.h, z29.b, z28.b\n" ".inst 0x455c07b4 // saddlt z20.h, z29.b, z28.b\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" ".inst 0x455a0373 // saddlb z19.h, z27.b, z26.b\n" ".inst 0x455a0772 // saddlt z18.h, z27.b, z26.b\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" ".inst 0x45580331 // saddlb z17.h, z25.b, z24.b\n" ".inst 0x45580730 // saddlt z16.h, z25.b, z24.b\n" - "ld1b { z29.b }, p3/Z, [x22, x26]\n" + "ld1b { z29.b }, p3/Z, [x21, x26]\n" ".inst 0x459741ef // saddwb z15.s, z15.s, z23.h\n" ".inst 0x459745ce // saddwt z14.s, z14.s, z23.h\n" - "ld1b { z28.b }, p3/Z, [x21, x26]\n" + "ld1b { z28.b }, p3/Z, [x20, x26]\n" ".inst 0x459641ad // saddwb z13.s, z13.s, z22.h\n" ".inst 0x4596458c // saddwt z12.s, z12.s, z22.h\n" - "ld1b { z27.b }, p2/Z, [x22, x25]\n" + "ld1b { z27.b }, p2/Z, [x21, x25]\n" ".inst 0x4595416b // saddwb z11.s, z11.s, z21.h\n" ".inst 0x4595454a // saddwt z10.s, z10.s, z21.h\n" - "ld1b { z26.b }, p2/Z, [x21, x25]\n" + "ld1b { z26.b }, p2/Z, [x20, x25]\n" ".inst 0x45944129 // saddwb z9.s, z9.s, z20.h\n" ".inst 0x45944508 // saddwt z8.s, z8.s, z20.h\n" - "ld1b { z25.b }, p1/Z, [x22, x24]\n" + "ld1b { z25.b }, p1/Z, [x21, x24]\n" ".inst 0x459340e7 // saddwb z7.s, z7.s, z19.h\n" ".inst 0x459344c6 // saddwt z6.s, z6.s, z19.h\n" - "ld1b { z24.b }, p1/Z, [x21, x24]\n" + "ld1b { z24.b }, p1/Z, [x20, x24]\n" ".inst 0x459240a5 // saddwb z5.s, z5.s, z18.h\n" ".inst 0x45924484 // saddwt z4.s, z4.s, z18.h\n" ".inst 0x45914063 // saddwb z3.s, z3.s, z17.h\n" @@ -203,20 +204,20 @@ void sme_s8_nhwc_avg_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x1\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x22, [x20], #0x8\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - ".inst 0x4508a3f7 // sshllb z23.h, z31.b, #0x0\n" - ".inst 0x4508a7f6 // sshllt z22.h, z31.b, #0x0\n" - "ld1b { z29.b }, p3/Z, [x22, x26]\n" - ".inst 0x4508a3b5 // sshllb z21.h, z29.b, #0x0\n" - ".inst 0x4508a7b4 // sshllt z20.h, z29.b, #0x0\n" + "ldr x20, [x22], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x27]\n" + ".inst 0x4508a217 // sshllb z23.h, z16.b, #0x0\n" + ".inst 0x4508a616 // sshllt z22.h, z16.b, #0x0\n" + "ld1b { z16.b }, p3/Z, [x20, x26]\n" + ".inst 0x4508a215 // sshllb z21.h, z16.b, #0x0\n" + ".inst 0x4508a614 // sshllt z20.h, z16.b, #0x0\n" "subs x21, x21, #0x1\n" - "ld1b { z27.b }, p2/Z, [x22, x25]\n" - ".inst 0x4508a373 // sshllb z19.h, z27.b, #0x0\n" - ".inst 0x4508a772 // sshllt z18.h, z27.b, #0x0\n" - "ld1b { z25.b }, p1/Z, [x22, x24]\n" - ".inst 0x4508a331 // sshllb z17.h, z25.b, #0x0\n" - ".inst 0x4508a730 // sshllt z16.h, z25.b, #0x0\n" + "ld1b { z16.b }, p2/Z, [x20, x25]\n" + ".inst 0x4508a213 // sshllb z19.h, z16.b, #0x0\n" + ".inst 0x4508a612 // sshllt z18.h, z16.b, #0x0\n" + "ld1b { z16.b }, p1/Z, [x20, x24]\n" + ".inst 0x4508a211 // sshllb z17.h, z16.b, #0x0\n" + ".inst 0x4508a610 // sshllt z16.h, z16.b, #0x0\n" ".inst 0x459741ef // saddwb z15.s, z15.s, z23.h\n" ".inst 0x459745ce // saddwt z14.s, z14.s, z23.h\n" ".inst 0x459641ad // saddwb z13.s, z13.s, z22.h\n" @@ -332,74 +333,74 @@ void sme_s8_nhwc_avg_generic_depthfirst_impl( "lsr x23, %x[n_valid_cells], #0x1\n" "mov z15.s, #0x0\n" "mov z14.s, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "mov z13.s, #0x0\n" "mov z12.s, #0x0\n" "cbz x23, 11f\n" - "ldp x22, x21, [x20, #0x0]\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 2 inputs loop - ".inst 0x455e03f7 // saddlb z23.h, z31.b, z30.b\n" - ".inst 0x455e07f6 // saddlt z22.h, z31.b, z30.b\n" - "ldp x22, x21, [x20, #0x0]\n" + ".inst 0x455e03f1 // saddlb z17.h, z31.b, z30.b\n" + ".inst 0x455e07f0 // saddlt z16.h, z31.b, z30.b\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - ".inst 0x459741ef // saddwb z15.s, z15.s, z23.h\n" - ".inst 0x459745ce // saddwt z14.s, z14.s, z23.h\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - ".inst 0x459641ad // saddwb z13.s, z13.s, z22.h\n" - ".inst 0x4596458c // saddwt z12.s, z12.s, z22.h\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" + ".inst 0x459141ef // saddwb z15.s, z15.s, z17.h\n" + ".inst 0x459145ce // saddwt z14.s, z14.s, z17.h\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" + ".inst 0x459041ad // saddwb z13.s, z13.s, z16.h\n" + ".inst 0x4590458c // saddwt z12.s, z12.s, z16.h\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 2 inputs tail - ".inst 0x455e03f7 // saddlb z23.h, z31.b, z30.b\n" - ".inst 0x455e07f6 // saddlt z22.h, z31.b, z30.b\n" - ".inst 0x459741ef // saddwb z15.s, z15.s, z23.h\n" - ".inst 0x459745ce // saddwt z14.s, z14.s, z23.h\n" - ".inst 0x459641ad // saddwb z13.s, z13.s, z22.h\n" - ".inst 0x4596458c // saddwt z12.s, z12.s, z22.h\n" + ".inst 0x455e03f1 // saddlb z17.h, z31.b, z30.b\n" + ".inst 0x455e07f0 // saddlt z16.h, z31.b, z30.b\n" + ".inst 0x459141ef // saddwb z15.s, z15.s, z17.h\n" + ".inst 0x459145ce // saddwt z14.s, z14.s, z17.h\n" + ".inst 0x459041ad // saddwb z13.s, z13.s, z16.h\n" + ".inst 0x4590458c // saddwt z12.s, z12.s, z16.h\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x1\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x22, [x20], #0x8\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - ".inst 0x4508a3f7 // sshllb z23.h, z31.b, #0x0\n" - ".inst 0x4508a7f6 // sshllt z22.h, z31.b, #0x0\n" + "ldr x20, [x22], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x27]\n" + ".inst 0x4508a211 // sshllb z17.h, z16.b, #0x0\n" + ".inst 0x4508a610 // sshllt z16.h, z16.b, #0x0\n" "subs x21, x21, #0x1\n" - ".inst 0x459741ef // saddwb z15.s, z15.s, z23.h\n" - ".inst 0x459745ce // saddwt z14.s, z14.s, z23.h\n" - ".inst 0x459641ad // saddwb z13.s, z13.s, z22.h\n" - ".inst 0x4596458c // saddwt z12.s, z12.s, z22.h\n" + ".inst 0x459141ef // saddwb z15.s, z15.s, z17.h\n" + ".inst 0x459145ce // saddwt z14.s, z14.s, z17.h\n" + ".inst 0x459041ad // saddwb z13.s, z13.s, z16.h\n" + ".inst 0x4590458c // saddwt z12.s, z12.s, z16.h\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End - "ld1rw { z17.s }, p0/Z, [%x[rescale_ptr]]\n" - ".inst 0x04b175ef // sqdmulh z15.s, z15.s, z17.s\n" - ".inst 0x04b175ce // sqdmulh z14.s, z14.s, z17.s\n" - ".inst 0x04b175ad // sqdmulh z13.s, z13.s, z17.s\n" - ".inst 0x04b1758c // sqdmulh z12.s, z12.s, z17.s\n" + "ld1rw { z16.s }, p0/Z, [%x[rescale_ptr]]\n" + ".inst 0x04b075ef // sqdmulh z15.s, z15.s, z16.s\n" + ".inst 0x04b075ce // sqdmulh z14.s, z14.s, z16.s\n" + ".inst 0x04b075ad // sqdmulh z13.s, z13.s, z16.s\n" + ".inst 0x04b0758c // sqdmulh z12.s, z12.s, z16.s\n" "ld1rw { z16.s }, p0/Z, [%x[shift_ptr]]\n" - "mov z19.s, #0x7f\n" + "mov z18.s, #0x7f\n" ".inst 0x4482820f // srshl z15.s, p0/M, z15.s, z16.s\n" ".inst 0x4482820e // srshl z14.s, p0/M, z14.s, z16.s\n" ".inst 0x4482820d // srshl z13.s, p0/M, z13.s, z16.s\n" ".inst 0x4482820c // srshl z12.s, p0/M, z12.s, z16.s\n" - "not z16.s, p0/M, z19.s\n" + "not z16.s, p0/M, z18.s\n" "smax z15.s, p0/M, z15.s, z16.s\n" "smax z14.s, p0/M, z14.s, z16.s\n" "smax z13.s, p0/M, z13.s, z16.s\n" "smax z12.s, p0/M, z12.s, z16.s\n" - "smin z15.s, p0/M, z15.s, z19.s\n" - "smin z14.s, p0/M, z14.s, z19.s\n" - "trn1 z23.h, z15.h, z14.h\n" - "smin z13.s, p0/M, z13.s, z19.s\n" - "smin z12.s, p0/M, z12.s, z19.s\n" + "smin z15.s, p0/M, z15.s, z18.s\n" + "smin z14.s, p0/M, z14.s, z18.s\n" + "trn1 z17.h, z15.h, z14.h\n" + "smin z13.s, p0/M, z13.s, z18.s\n" + "smin z12.s, p0/M, z12.s, z18.s\n" "trn1 z16.h, z13.h, z12.h\n" - "trn1 z16.b, z23.b, z16.b\n" + "trn1 z16.b, z17.b, z16.b\n" "st1b { z16.b }, p4, [%x[outptr], x27]\n" "incb x27\n" "whilelt p4.b, x27, %x[n_channels]\n" diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_s8_nhwc_max_2x2_s1_output2x2_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_s8_nhwc_max_2x2_s1_output2x2_depthfirst.hpp index 1613970618bc5c7d40b2a29df711f0ed82a5eea0..c9a80e6a5b63d4fb51928b24747893e52ed35d55 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_s8_nhwc_max_2x2_s1_output2x2_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_s8_nhwc_max_2x2_s1_output2x2_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,8 @@ #pragma once +#if defined(ARM_COMPUTE_ENABLE_SME) + namespace arm_conv { namespace pooling { @@ -45,3 +47,5 @@ struct sme_s8_nhwc_max_2x2_s1_output2x2_depthfirst : public DepthfirstStrategy #include -#if defined(__ARM_FEATURE_SVE) && defined(ARM_COMPUTE_ENABLE_SME) +#if defined(ARM_COMPUTE_ENABLE_SME) namespace arm_conv { namespace pooling { @@ -70,23 +70,23 @@ void sme_s8_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "ldr x20, [%x[args], %[offsetof_inptrs]]\n" "mov x14, #0x0\n" "ldr x13, [%x[args], %[offsetof_n_channels]]\n" - "whilelt p1.b, x15, x13\n" + "whilelt p0.b, x15, x13\n" "ldp x12, x11, [x21, #0x0]\n" "ldp x10, x9, [x21, #0x10]\n" "ldp x28, x27, [x20, #0x0]\n" - "ld1b { z30.b }, p1/Z, [x27, x15]\n" + "ld1b { z30.b }, p0/Z, [x27, x15]\n" "ldp x26, x25, [x20, #0x10]\n" - "ld1b { z29.b }, p1/Z, [x25, x15]\n" + "ld1b { z29.b }, p0/Z, [x25, x15]\n" "ldp x24, x23, [x20, #0x20]\n" - "ld1b { z28.b }, p1/Z, [x24, x15]\n" + "ld1b { z28.b }, p0/Z, [x24, x15]\n" "ldp x22, x21, [x20, #0x30]\n" - "ld1b { z27.b }, p1/Z, [x21, x15]\n" + "ld1b { z27.b }, p0/Z, [x21, x15]\n" "ldr x20, [x20, #0x40]\n" - "ld1b { z26.b }, p1/Z, [x28, x15]\n" - "ld1b { z25.b }, p1/Z, [x26, x15]\n" - "ld1b { z24.b }, p1/Z, [x23, x15]\n" - "ld1b { z23.b }, p1/Z, [x22, x15]\n" - "ld1b { z19.b }, p1/Z, [x20, x15]\n" + "ld1b { z26.b }, p0/Z, [x28, x15]\n" + "ld1b { z25.b }, p0/Z, [x26, x15]\n" + "ld1b { z24.b }, p0/Z, [x23, x15]\n" + "ld1b { z19.b }, p0/Z, [x22, x15]\n" + "ld1b { z23.b }, p0/Z, [x20, x15]\n" "incw x15\n" "whilelt p1.b, x15, x13\n" "b.none 2f\n" @@ -95,25 +95,25 @@ void sme_s8_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "movprfx z21, z28\n smax z21.b, p2/M, z21.b, z27.b\n" "ld1b { z30.b }, p1/Z, [x27, x15]\n" "whilelt p0.b, x14, x13\n" - "movprfx z20, z29\n smax z20.b, p2/M, z20.b, z26.b\n" - "movprfx z18, z25\n smax z18.b, p2/M, z18.b, z24.b\n" + "movprfx z18, z29\n smax z18.b, p2/M, z18.b, z26.b\n" + "movprfx z17, z25\n smax z17.b, p2/M, z17.b, z24.b\n" "ld1b { z28.b }, p1/Z, [x24, x15]\n" - "movprfx z17, z29\n smax z17.b, p2/M, z17.b, z23.b\n" - "movprfx z16, z24\n smax z16.b, p2/M, z16.b, z19.b\n" + "movprfx z16, z29\n smax z16.b, p2/M, z16.b, z19.b\n" + "movprfx z20, z24\n smax z20.b, p2/M, z20.b, z23.b\n" "ld1b { z27.b }, p1/Z, [x21, x15]\n" "ld1b { z29.b }, p1/Z, [x25, x15]\n" - "movprfx z19, z22\n smax z19.b, p2/M, z19.b, z20.b\n" - "smax z18.b, p2/M, z18.b, z22.b\n" + "movprfx z19, z22\n smax z19.b, p2/M, z19.b, z18.b\n" + "movprfx z18, z17\n smax z18.b, p2/M, z18.b, z22.b\n" "ld1b { z26.b }, p1/Z, [x28, x15]\n" - "smax z17.b, p2/M, z17.b, z21.b\n" - "smax z16.b, p2/M, z16.b, z21.b\n" + "movprfx z17, z16\n smax z17.b, p2/M, z17.b, z21.b\n" + "movprfx z16, z21\n smax z16.b, p2/M, z16.b, z20.b\n" "ld1b { z25.b }, p1/Z, [x26, x15]\n" "st1b { z19.b }, p0, [x12, x14]\n" "ld1b { z24.b }, p1/Z, [x23, x15]\n" "st1b { z18.b }, p0, [x11, x14]\n" - "ld1b { z23.b }, p1/Z, [x22, x15]\n" + "ld1b { z19.b }, p1/Z, [x22, x15]\n" "st1b { z17.b }, p0, [x10, x14]\n" - "ld1b { z19.b }, p1/Z, [x20, x15]\n" + "ld1b { z23.b }, p1/Z, [x20, x15]\n" "incw x15\n" "whilelt p1.b, x15, x13\n" "st1b { z16.b }, p0, [x9, x14]\n" @@ -125,13 +125,13 @@ void sme_s8_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "whilelt p0.b, x14, x13\n" "movprfx z20, z29\n smax z20.b, p2/M, z20.b, z26.b\n" "movprfx z18, z25\n smax z18.b, p2/M, z18.b, z24.b\n" - "movprfx z17, z29\n smax z17.b, p2/M, z17.b, z23.b\n" - "movprfx z16, z24\n smax z16.b, p2/M, z16.b, z19.b\n" - "movprfx z19, z22\n smax z19.b, p2/M, z19.b, z20.b\n" + "movprfx z17, z29\n smax z17.b, p2/M, z17.b, z19.b\n" + "movprfx z19, z24\n smax z19.b, p2/M, z19.b, z23.b\n" + "movprfx z16, z22\n smax z16.b, p2/M, z16.b, z20.b\n" "smax z18.b, p2/M, z18.b, z22.b\n" - "st1b { z19.b }, p0, [x12, x14]\n" + "st1b { z16.b }, p0, [x12, x14]\n" "smax z17.b, p2/M, z17.b, z21.b\n" - "smax z16.b, p2/M, z16.b, z21.b\n" + "movprfx z16, z21\n smax z16.b, p2/M, z16.b, z19.b\n" "st1b { z18.b }, p0, [x11, x14]\n" "st1b { z17.b }, p0, [x10, x14]\n" "st1b { z16.b }, p0, [x9, x14]\n" @@ -145,4 +145,4 @@ void sme_s8_nhwc_max_2x2_s1_output2x2_depthfirst_impl( } // namespace pooling } // namespace arm_conv -#endif // defined(__ARM_FEATURE_SVE) && defined(ARM_COMPUTE_ENABLE_SME) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_s8_nhwc_max_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_s8_nhwc_max_generic_depthfirst.hpp index 56aa120cfe9b51d7bcab3238bb41ca99f3cb542d..3e0d76c27797f9dd56b87703b2b022ae90c82d71 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_s8_nhwc_max_generic_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_s8_nhwc_max_generic_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,8 @@ #pragma once +#if defined(ARM_COMPUTE_ENABLE_SME) + namespace arm_conv { namespace pooling { @@ -40,3 +42,5 @@ struct sme_s8_nhwc_max_generic_depthfirst : IGenericDepthfirstStrategy +#include + +#if defined(ARM_COMPUTE_ENABLE_SME) namespace arm_conv { namespace pooling { @@ -54,68 +55,68 @@ void sme_s8_nhwc_max_generic_depthfirst_impl( "lsr x25, %x[n_valid_cells], #0x2\n" "mov z4.b, #0x80\n" "mov z3.b, #0x80\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "mov z2.b, #0x80\n" "mov z1.b, #0x80\n" "cbz x25, 4f\n" - "ldp x24, x23, [x20, #0x0]\n" + "ldp x23, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" - "ld1b { z0.b }, p4/Z, [x24, x9]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ld1b { z31.b }, p4/Z, [x23, x9]\n" - "ld1b { z23.b }, p4/Z, [x22, x9]\n" - "ld1b { z30.b }, p4/Z, [x21, x9]\n" - "ld1b { z18.b }, p3/Z, [x24, x28]\n" - "ld1b { z29.b }, p3/Z, [x23, x28]\n" - "ld1b { z22.b }, p3/Z, [x22, x28]\n" - "ld1b { z28.b }, p3/Z, [x21, x28]\n" - "ld1b { z17.b }, p2/Z, [x24, x27]\n" - "ld1b { z27.b }, p2/Z, [x23, x27]\n" - "ld1b { z21.b }, p2/Z, [x22, x27]\n" - "ld1b { z26.b }, p2/Z, [x21, x27]\n" - "ld1b { z16.b }, p1/Z, [x24, x26]\n" - "ld1b { z25.b }, p1/Z, [x23, x26]\n" - "ld1b { z20.b }, p1/Z, [x22, x26]\n" - "ld1b { z24.b }, p1/Z, [x21, x26]\n" + "ld1b { z0.b }, p4/Z, [x23, x9]\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" + "ld1b { z31.b }, p4/Z, [x22, x9]\n" + "ld1b { z23.b }, p4/Z, [x21, x9]\n" + "ld1b { z30.b }, p4/Z, [x20, x9]\n" + "ld1b { z18.b }, p3/Z, [x23, x28]\n" + "ld1b { z29.b }, p3/Z, [x22, x28]\n" + "ld1b { z22.b }, p3/Z, [x21, x28]\n" + "ld1b { z28.b }, p3/Z, [x20, x28]\n" + "ld1b { z17.b }, p2/Z, [x23, x27]\n" + "ld1b { z27.b }, p2/Z, [x22, x27]\n" + "ld1b { z21.b }, p2/Z, [x21, x27]\n" + "ld1b { z26.b }, p2/Z, [x20, x27]\n" + "ld1b { z16.b }, p1/Z, [x23, x26]\n" + "ld1b { z25.b }, p1/Z, [x22, x26]\n" + "ld1b { z20.b }, p1/Z, [x21, x26]\n" + "ld1b { z24.b }, p1/Z, [x20, x26]\n" "beq 3f\n" "2:" // 4-vectors of channels: 4 inputs loop "movprfx z19, z0\n smax z19.b, p0/M, z19.b, z31.b\n" "smax z23.b, p0/M, z23.b, z30.b\n" - "ldp x24, x23, [x20, #0x0]\n" + "ldp x23, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" "smax z18.b, p0/M, z18.b, z29.b\n" "smax z22.b, p0/M, z22.b, z28.b\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" "smax z17.b, p0/M, z17.b, z27.b\n" "smax z21.b, p0/M, z21.b, z26.b\n" - "ld1b { z0.b }, p4/Z, [x24, x9]\n" + "ld1b { z0.b }, p4/Z, [x23, x9]\n" "smax z16.b, p0/M, z16.b, z25.b\n" "smax z20.b, p0/M, z20.b, z24.b\n" - "ld1b { z31.b }, p4/Z, [x23, x9]\n" + "ld1b { z31.b }, p4/Z, [x22, x9]\n" "smax z19.b, p0/M, z19.b, z23.b\n" "smax z18.b, p0/M, z18.b, z22.b\n" - "ld1b { z23.b }, p4/Z, [x22, x9]\n" + "ld1b { z23.b }, p4/Z, [x21, x9]\n" "smax z17.b, p0/M, z17.b, z21.b\n" "smax z16.b, p0/M, z16.b, z20.b\n" - "ld1b { z30.b }, p4/Z, [x21, x9]\n" + "ld1b { z30.b }, p4/Z, [x20, x9]\n" "smax z4.b, p0/M, z4.b, z19.b\n" "smax z3.b, p0/M, z3.b, z18.b\n" - "ld1b { z18.b }, p3/Z, [x24, x28]\n" + "ld1b { z18.b }, p3/Z, [x23, x28]\n" "smax z2.b, p0/M, z2.b, z17.b\n" "smax z1.b, p0/M, z1.b, z16.b\n" - "ld1b { z29.b }, p3/Z, [x23, x28]\n" - "ld1b { z22.b }, p3/Z, [x22, x28]\n" - "ld1b { z28.b }, p3/Z, [x21, x28]\n" - "ld1b { z17.b }, p2/Z, [x24, x27]\n" - "ld1b { z27.b }, p2/Z, [x23, x27]\n" - "ld1b { z21.b }, p2/Z, [x22, x27]\n" - "ld1b { z26.b }, p2/Z, [x21, x27]\n" - "ld1b { z16.b }, p1/Z, [x24, x26]\n" - "ld1b { z25.b }, p1/Z, [x23, x26]\n" - "ld1b { z20.b }, p1/Z, [x22, x26]\n" - "ld1b { z24.b }, p1/Z, [x21, x26]\n" + "ld1b { z29.b }, p3/Z, [x22, x28]\n" + "ld1b { z22.b }, p3/Z, [x21, x28]\n" + "ld1b { z28.b }, p3/Z, [x20, x28]\n" + "ld1b { z17.b }, p2/Z, [x23, x27]\n" + "ld1b { z27.b }, p2/Z, [x22, x27]\n" + "ld1b { z21.b }, p2/Z, [x21, x27]\n" + "ld1b { z26.b }, p2/Z, [x20, x27]\n" + "ld1b { z16.b }, p1/Z, [x23, x26]\n" + "ld1b { z25.b }, p1/Z, [x22, x26]\n" + "ld1b { z20.b }, p1/Z, [x21, x26]\n" + "ld1b { z24.b }, p1/Z, [x20, x26]\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "movprfx z19, z0\n smax z19.b, p0/M, z19.b, z31.b\n" @@ -138,15 +139,15 @@ void sme_s8_nhwc_max_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1b { z0.b }, p4/Z, [x24, x9]\n" + "ldr x20, [x24], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x9]\n" "subs x21, x21, #0x1\n" - "smax z4.b, p0/M, z4.b, z0.b\n" - "ld1b { z18.b }, p3/Z, [x24, x28]\n" - "smax z3.b, p0/M, z3.b, z18.b\n" - "ld1b { z17.b }, p2/Z, [x24, x27]\n" - "smax z2.b, p0/M, z2.b, z17.b\n" - "ld1b { z16.b }, p1/Z, [x24, x26]\n" + "smax z4.b, p0/M, z4.b, z16.b\n" + "ld1b { z16.b }, p3/Z, [x20, x28]\n" + "smax z3.b, p0/M, z3.b, z16.b\n" + "ld1b { z16.b }, p2/Z, [x20, x27]\n" + "smax z2.b, p0/M, z2.b, z16.b\n" + "ld1b { z16.b }, p1/Z, [x20, x26]\n" "smax z1.b, p0/M, z1.b, z16.b\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End @@ -166,44 +167,44 @@ void sme_s8_nhwc_max_generic_depthfirst_impl( "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" "mov z4.b, #0x80\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x24, x23, [x20, #0x0]\n" + "ldp x20, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" - "ld1b { z0.b }, p4/Z, [x24, x9]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ld1b { z31.b }, p4/Z, [x23, x9]\n" - "ld1b { z23.b }, p4/Z, [x22, x9]\n" - "ld1b { z30.b }, p4/Z, [x21, x9]\n" + "ld1b { z0.b }, p4/Z, [x20, x9]\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" + "ld1b { z31.b }, p4/Z, [x22, x9]\n" + "ld1b { z23.b }, p4/Z, [x21, x9]\n" + "ld1b { z30.b }, p4/Z, [x20, x9]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 4 inputs loop - "movprfx z19, z0\n smax z19.b, p0/M, z19.b, z31.b\n" - "smax z23.b, p0/M, z23.b, z30.b\n" - "ldp x24, x23, [x20, #0x0]\n" + "movprfx z16, z0\n smax z16.b, p0/M, z16.b, z31.b\n" + "movprfx z17, z23\n smax z17.b, p0/M, z17.b, z30.b\n" + "ldp x23, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" - "smax z19.b, p0/M, z19.b, z23.b\n" - "ldp x22, x21, [x20, #0x10]\n" - "smax z4.b, p0/M, z4.b, z19.b\n" - "add x20, x20, #0x20\n" - "ld1b { z0.b }, p4/Z, [x24, x9]\n" - "ld1b { z31.b }, p4/Z, [x23, x9]\n" - "ld1b { z23.b }, p4/Z, [x22, x9]\n" - "ld1b { z30.b }, p4/Z, [x21, x9]\n" + "smax z16.b, p0/M, z16.b, z17.b\n" + "ldp x21, x20, [x24, #0x10]\n" + "smax z4.b, p0/M, z4.b, z16.b\n" + "add x24, x24, #0x20\n" + "ld1b { z0.b }, p4/Z, [x23, x9]\n" + "ld1b { z31.b }, p4/Z, [x22, x9]\n" + "ld1b { z23.b }, p4/Z, [x21, x9]\n" + "ld1b { z30.b }, p4/Z, [x20, x9]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail - "movprfx z19, z0\n smax z19.b, p0/M, z19.b, z31.b\n" - "smax z23.b, p0/M, z23.b, z30.b\n" - "smax z19.b, p0/M, z19.b, z23.b\n" - "smax z4.b, p0/M, z4.b, z19.b\n" + "movprfx z16, z0\n smax z16.b, p0/M, z16.b, z31.b\n" + "movprfx z17, z23\n smax z17.b, p0/M, z17.b, z30.b\n" + "smax z16.b, p0/M, z16.b, z17.b\n" + "smax z4.b, p0/M, z4.b, z16.b\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1b { z0.b }, p4/Z, [x24, x9]\n" + "ldr x20, [x24], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x9]\n" "subs x21, x21, #0x1\n" - "smax z4.b, p0/M, z4.b, z0.b\n" + "smax z4.b, p0/M, z4.b, z16.b\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "st1b { z4.b }, p4, [%x[outptr], x9]\n" diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_s8q_nhwc_avg_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_s8q_nhwc_avg_generic_depthfirst.hpp index ee02c60bc1881bd0221c31759e1615e5540df871..c6263f5dbc51ddd27e8f2ed2e83ea615b9162707 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_s8q_nhwc_avg_generic_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_s8q_nhwc_avg_generic_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,8 @@ #pragma once +#if defined(ARM_COMPUTE_ENABLE_SME) + namespace arm_conv { namespace pooling { @@ -40,3 +42,5 @@ struct sme_s8q_nhwc_avg_generic_depthfirst : IGenericDepthfirstStrategy +#include #include #include +#if defined(ARM_COMPUTE_ENABLE_SME) + namespace arm_conv { namespace pooling { @@ -128,7 +129,7 @@ void sme_s8q_nhwc_avg_generic_depthfirst_impl( "lsr x23, %x[n_valid_cells], #0x1\n" "mov z15.s, #0x0\n" "mov z14.s, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "mov z13.s, #0x0\n" "mov z12.s, #0x0\n" "mov z11.s, #0x0\n" @@ -144,48 +145,48 @@ void sme_s8q_nhwc_avg_generic_depthfirst_impl( "mov z1.s, #0x0\n" "mov z0.s, #0x0\n" "cbz x23, 4f\n" - "ldp x22, x21, [x20, #0x0]\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" - "ld1b { z29.b }, p3/Z, [x22, x26]\n" - "ld1b { z28.b }, p3/Z, [x21, x26]\n" - "ld1b { z27.b }, p2/Z, [x22, x25]\n" - "ld1b { z26.b }, p2/Z, [x21, x25]\n" - "ld1b { z25.b }, p1/Z, [x22, x24]\n" - "ld1b { z24.b }, p1/Z, [x21, x24]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" + "ld1b { z29.b }, p3/Z, [x21, x26]\n" + "ld1b { z28.b }, p3/Z, [x20, x26]\n" + "ld1b { z27.b }, p2/Z, [x21, x25]\n" + "ld1b { z26.b }, p2/Z, [x20, x25]\n" + "ld1b { z25.b }, p1/Z, [x21, x24]\n" + "ld1b { z24.b }, p1/Z, [x20, x24]\n" "beq 3f\n" "2:" // 4-vectors of channels: 2 inputs loop ".inst 0x455e03f7 // saddlb z23.h, z31.b, z30.b\n" ".inst 0x455e07f6 // saddlt z22.h, z31.b, z30.b\n" - "ldp x22, x21, [x20, #0x0]\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" ".inst 0x455c03b5 // saddlb z21.h, z29.b, z28.b\n" ".inst 0x455c07b4 // saddlt z20.h, z29.b, z28.b\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" ".inst 0x455a0373 // saddlb z19.h, z27.b, z26.b\n" ".inst 0x455a0772 // saddlt z18.h, z27.b, z26.b\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" ".inst 0x45580331 // saddlb z17.h, z25.b, z24.b\n" ".inst 0x45580730 // saddlt z16.h, z25.b, z24.b\n" - "ld1b { z29.b }, p3/Z, [x22, x26]\n" + "ld1b { z29.b }, p3/Z, [x21, x26]\n" ".inst 0x459741ef // saddwb z15.s, z15.s, z23.h\n" ".inst 0x459745ce // saddwt z14.s, z14.s, z23.h\n" - "ld1b { z28.b }, p3/Z, [x21, x26]\n" + "ld1b { z28.b }, p3/Z, [x20, x26]\n" ".inst 0x459641ad // saddwb z13.s, z13.s, z22.h\n" ".inst 0x4596458c // saddwt z12.s, z12.s, z22.h\n" - "ld1b { z27.b }, p2/Z, [x22, x25]\n" + "ld1b { z27.b }, p2/Z, [x21, x25]\n" ".inst 0x4595416b // saddwb z11.s, z11.s, z21.h\n" ".inst 0x4595454a // saddwt z10.s, z10.s, z21.h\n" - "ld1b { z26.b }, p2/Z, [x21, x25]\n" + "ld1b { z26.b }, p2/Z, [x20, x25]\n" ".inst 0x45944129 // saddwb z9.s, z9.s, z20.h\n" ".inst 0x45944508 // saddwt z8.s, z8.s, z20.h\n" - "ld1b { z25.b }, p1/Z, [x22, x24]\n" + "ld1b { z25.b }, p1/Z, [x21, x24]\n" ".inst 0x459340e7 // saddwb z7.s, z7.s, z19.h\n" ".inst 0x459344c6 // saddwt z6.s, z6.s, z19.h\n" - "ld1b { z24.b }, p1/Z, [x21, x24]\n" + "ld1b { z24.b }, p1/Z, [x20, x24]\n" ".inst 0x459240a5 // saddwb z5.s, z5.s, z18.h\n" ".inst 0x45924484 // saddwt z4.s, z4.s, z18.h\n" ".inst 0x45914063 // saddwb z3.s, z3.s, z17.h\n" @@ -222,20 +223,20 @@ void sme_s8q_nhwc_avg_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x1\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x22, [x20], #0x8\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - ".inst 0x4508a3f7 // sshllb z23.h, z31.b, #0x0\n" - ".inst 0x4508a7f6 // sshllt z22.h, z31.b, #0x0\n" - "ld1b { z29.b }, p3/Z, [x22, x26]\n" - ".inst 0x4508a3b5 // sshllb z21.h, z29.b, #0x0\n" - ".inst 0x4508a7b4 // sshllt z20.h, z29.b, #0x0\n" + "ldr x20, [x22], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x27]\n" + ".inst 0x4508a217 // sshllb z23.h, z16.b, #0x0\n" + ".inst 0x4508a616 // sshllt z22.h, z16.b, #0x0\n" + "ld1b { z16.b }, p3/Z, [x20, x26]\n" + ".inst 0x4508a215 // sshllb z21.h, z16.b, #0x0\n" + ".inst 0x4508a614 // sshllt z20.h, z16.b, #0x0\n" "subs x21, x21, #0x1\n" - "ld1b { z27.b }, p2/Z, [x22, x25]\n" - ".inst 0x4508a373 // sshllb z19.h, z27.b, #0x0\n" - ".inst 0x4508a772 // sshllt z18.h, z27.b, #0x0\n" - "ld1b { z25.b }, p1/Z, [x22, x24]\n" - ".inst 0x4508a331 // sshllb z17.h, z25.b, #0x0\n" - ".inst 0x4508a730 // sshllt z16.h, z25.b, #0x0\n" + "ld1b { z16.b }, p2/Z, [x20, x25]\n" + ".inst 0x4508a213 // sshllb z19.h, z16.b, #0x0\n" + ".inst 0x4508a612 // sshllt z18.h, z16.b, #0x0\n" + "ld1b { z16.b }, p1/Z, [x20, x24]\n" + ".inst 0x4508a211 // sshllb z17.h, z16.b, #0x0\n" + ".inst 0x4508a610 // sshllt z16.h, z16.b, #0x0\n" ".inst 0x459741ef // saddwb z15.s, z15.s, z23.h\n" ".inst 0x459745ce // saddwt z14.s, z14.s, z23.h\n" ".inst 0x459641ad // saddwb z13.s, z13.s, z22.h\n" @@ -368,79 +369,79 @@ void sme_s8q_nhwc_avg_generic_depthfirst_impl( "lsr x23, %x[n_valid_cells], #0x1\n" "mov z15.s, #0x0\n" "mov z14.s, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "mov z13.s, #0x0\n" "mov z12.s, #0x0\n" "cbz x23, 11f\n" - "ldp x22, x21, [x20, #0x0]\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 2 inputs loop - ".inst 0x455e03f7 // saddlb z23.h, z31.b, z30.b\n" - ".inst 0x455e07f6 // saddlt z22.h, z31.b, z30.b\n" - "ldp x22, x21, [x20, #0x0]\n" + ".inst 0x455e03f1 // saddlb z17.h, z31.b, z30.b\n" + ".inst 0x455e07f0 // saddlt z16.h, z31.b, z30.b\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - ".inst 0x459741ef // saddwb z15.s, z15.s, z23.h\n" - ".inst 0x459745ce // saddwt z14.s, z14.s, z23.h\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - ".inst 0x459641ad // saddwb z13.s, z13.s, z22.h\n" - ".inst 0x4596458c // saddwt z12.s, z12.s, z22.h\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" + ".inst 0x459141ef // saddwb z15.s, z15.s, z17.h\n" + ".inst 0x459145ce // saddwt z14.s, z14.s, z17.h\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" + ".inst 0x459041ad // saddwb z13.s, z13.s, z16.h\n" + ".inst 0x4590458c // saddwt z12.s, z12.s, z16.h\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 2 inputs tail - ".inst 0x455e03f7 // saddlb z23.h, z31.b, z30.b\n" - ".inst 0x455e07f6 // saddlt z22.h, z31.b, z30.b\n" - ".inst 0x459741ef // saddwb z15.s, z15.s, z23.h\n" - ".inst 0x459745ce // saddwt z14.s, z14.s, z23.h\n" - ".inst 0x459641ad // saddwb z13.s, z13.s, z22.h\n" - ".inst 0x4596458c // saddwt z12.s, z12.s, z22.h\n" + ".inst 0x455e03f1 // saddlb z17.h, z31.b, z30.b\n" + ".inst 0x455e07f0 // saddlt z16.h, z31.b, z30.b\n" + ".inst 0x459141ef // saddwb z15.s, z15.s, z17.h\n" + ".inst 0x459145ce // saddwt z14.s, z14.s, z17.h\n" + ".inst 0x459041ad // saddwb z13.s, z13.s, z16.h\n" + ".inst 0x4590458c // saddwt z12.s, z12.s, z16.h\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x1\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x22, [x20], #0x8\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - ".inst 0x4508a3f7 // sshllb z23.h, z31.b, #0x0\n" - ".inst 0x4508a7f6 // sshllt z22.h, z31.b, #0x0\n" + "ldr x20, [x22], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x27]\n" + ".inst 0x4508a211 // sshllb z17.h, z16.b, #0x0\n" + ".inst 0x4508a610 // sshllt z16.h, z16.b, #0x0\n" "subs x21, x21, #0x1\n" - ".inst 0x459741ef // saddwb z15.s, z15.s, z23.h\n" - ".inst 0x459745ce // saddwt z14.s, z14.s, z23.h\n" - ".inst 0x459641ad // saddwb z13.s, z13.s, z22.h\n" - ".inst 0x4596458c // saddwt z12.s, z12.s, z22.h\n" + ".inst 0x459141ef // saddwb z15.s, z15.s, z17.h\n" + ".inst 0x459145ce // saddwt z14.s, z14.s, z17.h\n" + ".inst 0x459041ad // saddwb z13.s, z13.s, z16.h\n" + ".inst 0x4590458c // saddwt z12.s, z12.s, z16.h\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End - "ld1rw { z18.s }, p0/Z, [%x[left_shift]]\n" - ".inst 0x4482824f // srshl z15.s, p0/M, z15.s, z18.s\n" - ".inst 0x4482824e // srshl z14.s, p0/M, z14.s, z18.s\n" - ".inst 0x4482824d // srshl z13.s, p0/M, z13.s, z18.s\n" - ".inst 0x4482824c // srshl z12.s, p0/M, z12.s, z18.s\n" + "ld1rw { z16.s }, p0/Z, [%x[left_shift]]\n" + ".inst 0x4482820f // srshl z15.s, p0/M, z15.s, z16.s\n" + ".inst 0x4482820e // srshl z14.s, p0/M, z14.s, z16.s\n" + ".inst 0x4482820d // srshl z13.s, p0/M, z13.s, z16.s\n" + ".inst 0x4482820c // srshl z12.s, p0/M, z12.s, z16.s\n" "ld1rw { z17.s }, p0/Z, [%x[combined_rescale_value]]\n" ".inst 0x04b175ef // sqrdmulh z15.s, z15.s, z17.s\n" ".inst 0x04b175ce // sqrdmulh z14.s, z14.s, z17.s\n" "ld1rw { z16.s }, p0/Z, [%x[right_shift]]\n" ".inst 0x04b175ad // sqrdmulh z13.s, z13.s, z17.s\n" ".inst 0x04b1758c // sqrdmulh z12.s, z12.s, z17.s\n" - "mov z19.s, #0x7f\n" + "mov z18.s, #0x7f\n" ".inst 0x4482820f // srshl z15.s, p0/M, z15.s, z16.s\n" ".inst 0x4482820e // srshl z14.s, p0/M, z14.s, z16.s\n" ".inst 0x4482820d // srshl z13.s, p0/M, z13.s, z16.s\n" ".inst 0x4482820c // srshl z12.s, p0/M, z12.s, z16.s\n" - "not z16.s, p0/M, z19.s\n" + "not z16.s, p0/M, z18.s\n" "smax z15.s, p0/M, z15.s, z16.s\n" "smax z14.s, p0/M, z14.s, z16.s\n" "smax z13.s, p0/M, z13.s, z16.s\n" "smax z12.s, p0/M, z12.s, z16.s\n" - "smin z15.s, p0/M, z15.s, z19.s\n" - "smin z14.s, p0/M, z14.s, z19.s\n" - "trn1 z23.h, z15.h, z14.h\n" - "smin z13.s, p0/M, z13.s, z19.s\n" - "smin z12.s, p0/M, z12.s, z19.s\n" + "smin z15.s, p0/M, z15.s, z18.s\n" + "smin z14.s, p0/M, z14.s, z18.s\n" + "trn1 z17.h, z15.h, z14.h\n" + "smin z13.s, p0/M, z13.s, z18.s\n" + "smin z12.s, p0/M, z12.s, z18.s\n" "trn1 z16.h, z13.h, z12.h\n" - "trn1 z16.b, z23.b, z16.b\n" + "trn1 z16.b, z17.b, z16.b\n" "st1b { z16.b }, p4, [%x[outptr], x27]\n" "incb x27\n" "whilelt p4.b, x27, %x[n_channels]\n" @@ -456,4 +457,4 @@ void sme_s8q_nhwc_avg_generic_depthfirst_impl( } // namespace pooling } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_s8q_nhwc_max_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_s8q_nhwc_max_generic_depthfirst.hpp index 050aff397e99b3e059a46c96d81735f49da9ddae..9667d3795499507b68057cd85375a50251712ca0 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_s8q_nhwc_max_generic_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_s8q_nhwc_max_generic_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,8 @@ #pragma once +#if defined(ARM_COMPUTE_ENABLE_SME) + namespace arm_conv { namespace pooling { @@ -40,3 +42,5 @@ struct sme_s8q_nhwc_max_generic_depthfirst : IGenericDepthfirstStrategy +#include + +#if defined(ARM_COMPUTE_ENABLE_SME) namespace arm_conv { namespace pooling { @@ -56,68 +57,68 @@ void sme_s8q_nhwc_max_generic_depthfirst_impl( "lsr x25, %x[n_valid_cells], #0x2\n" "mov z4.b, #0x80\n" "mov z3.b, #0x80\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "mov z2.b, #0x80\n" "mov z1.b, #0x80\n" "cbz x25, 4f\n" - "ldp x24, x23, [x20, #0x0]\n" + "ldp x23, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" - "ld1b { z0.b }, p4/Z, [x24, x9]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ld1b { z31.b }, p4/Z, [x23, x9]\n" - "ld1b { z23.b }, p4/Z, [x22, x9]\n" - "ld1b { z30.b }, p4/Z, [x21, x9]\n" - "ld1b { z18.b }, p3/Z, [x24, x28]\n" - "ld1b { z29.b }, p3/Z, [x23, x28]\n" - "ld1b { z22.b }, p3/Z, [x22, x28]\n" - "ld1b { z28.b }, p3/Z, [x21, x28]\n" - "ld1b { z17.b }, p2/Z, [x24, x27]\n" - "ld1b { z27.b }, p2/Z, [x23, x27]\n" - "ld1b { z21.b }, p2/Z, [x22, x27]\n" - "ld1b { z26.b }, p2/Z, [x21, x27]\n" - "ld1b { z16.b }, p1/Z, [x24, x26]\n" - "ld1b { z25.b }, p1/Z, [x23, x26]\n" - "ld1b { z20.b }, p1/Z, [x22, x26]\n" - "ld1b { z24.b }, p1/Z, [x21, x26]\n" + "ld1b { z0.b }, p4/Z, [x23, x9]\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" + "ld1b { z31.b }, p4/Z, [x22, x9]\n" + "ld1b { z23.b }, p4/Z, [x21, x9]\n" + "ld1b { z30.b }, p4/Z, [x20, x9]\n" + "ld1b { z18.b }, p3/Z, [x23, x28]\n" + "ld1b { z29.b }, p3/Z, [x22, x28]\n" + "ld1b { z22.b }, p3/Z, [x21, x28]\n" + "ld1b { z28.b }, p3/Z, [x20, x28]\n" + "ld1b { z17.b }, p2/Z, [x23, x27]\n" + "ld1b { z27.b }, p2/Z, [x22, x27]\n" + "ld1b { z21.b }, p2/Z, [x21, x27]\n" + "ld1b { z26.b }, p2/Z, [x20, x27]\n" + "ld1b { z16.b }, p1/Z, [x23, x26]\n" + "ld1b { z25.b }, p1/Z, [x22, x26]\n" + "ld1b { z20.b }, p1/Z, [x21, x26]\n" + "ld1b { z24.b }, p1/Z, [x20, x26]\n" "beq 3f\n" "2:" // 4-vectors of channels: 4 inputs loop "movprfx z19, z0\n smax z19.b, p0/M, z19.b, z31.b\n" "smax z23.b, p0/M, z23.b, z30.b\n" - "ldp x24, x23, [x20, #0x0]\n" + "ldp x23, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" "smax z18.b, p0/M, z18.b, z29.b\n" "smax z22.b, p0/M, z22.b, z28.b\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" "smax z17.b, p0/M, z17.b, z27.b\n" "smax z21.b, p0/M, z21.b, z26.b\n" - "ld1b { z0.b }, p4/Z, [x24, x9]\n" + "ld1b { z0.b }, p4/Z, [x23, x9]\n" "smax z16.b, p0/M, z16.b, z25.b\n" "smax z20.b, p0/M, z20.b, z24.b\n" - "ld1b { z31.b }, p4/Z, [x23, x9]\n" + "ld1b { z31.b }, p4/Z, [x22, x9]\n" "smax z19.b, p0/M, z19.b, z23.b\n" "smax z18.b, p0/M, z18.b, z22.b\n" - "ld1b { z23.b }, p4/Z, [x22, x9]\n" + "ld1b { z23.b }, p4/Z, [x21, x9]\n" "smax z17.b, p0/M, z17.b, z21.b\n" "smax z16.b, p0/M, z16.b, z20.b\n" - "ld1b { z30.b }, p4/Z, [x21, x9]\n" + "ld1b { z30.b }, p4/Z, [x20, x9]\n" "smax z4.b, p0/M, z4.b, z19.b\n" "smax z3.b, p0/M, z3.b, z18.b\n" - "ld1b { z18.b }, p3/Z, [x24, x28]\n" + "ld1b { z18.b }, p3/Z, [x23, x28]\n" "smax z2.b, p0/M, z2.b, z17.b\n" "smax z1.b, p0/M, z1.b, z16.b\n" - "ld1b { z29.b }, p3/Z, [x23, x28]\n" - "ld1b { z22.b }, p3/Z, [x22, x28]\n" - "ld1b { z28.b }, p3/Z, [x21, x28]\n" - "ld1b { z17.b }, p2/Z, [x24, x27]\n" - "ld1b { z27.b }, p2/Z, [x23, x27]\n" - "ld1b { z21.b }, p2/Z, [x22, x27]\n" - "ld1b { z26.b }, p2/Z, [x21, x27]\n" - "ld1b { z16.b }, p1/Z, [x24, x26]\n" - "ld1b { z25.b }, p1/Z, [x23, x26]\n" - "ld1b { z20.b }, p1/Z, [x22, x26]\n" - "ld1b { z24.b }, p1/Z, [x21, x26]\n" + "ld1b { z29.b }, p3/Z, [x22, x28]\n" + "ld1b { z22.b }, p3/Z, [x21, x28]\n" + "ld1b { z28.b }, p3/Z, [x20, x28]\n" + "ld1b { z17.b }, p2/Z, [x23, x27]\n" + "ld1b { z27.b }, p2/Z, [x22, x27]\n" + "ld1b { z21.b }, p2/Z, [x21, x27]\n" + "ld1b { z26.b }, p2/Z, [x20, x27]\n" + "ld1b { z16.b }, p1/Z, [x23, x26]\n" + "ld1b { z25.b }, p1/Z, [x22, x26]\n" + "ld1b { z20.b }, p1/Z, [x21, x26]\n" + "ld1b { z24.b }, p1/Z, [x20, x26]\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "movprfx z19, z0\n smax z19.b, p0/M, z19.b, z31.b\n" @@ -140,15 +141,15 @@ void sme_s8q_nhwc_max_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1b { z0.b }, p4/Z, [x24, x9]\n" + "ldr x20, [x24], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x9]\n" "subs x21, x21, #0x1\n" - "smax z4.b, p0/M, z4.b, z0.b\n" - "ld1b { z18.b }, p3/Z, [x24, x28]\n" - "smax z3.b, p0/M, z3.b, z18.b\n" - "ld1b { z17.b }, p2/Z, [x24, x27]\n" - "smax z2.b, p0/M, z2.b, z17.b\n" - "ld1b { z16.b }, p1/Z, [x24, x26]\n" + "smax z4.b, p0/M, z4.b, z16.b\n" + "ld1b { z16.b }, p3/Z, [x20, x28]\n" + "smax z3.b, p0/M, z3.b, z16.b\n" + "ld1b { z16.b }, p2/Z, [x20, x27]\n" + "smax z2.b, p0/M, z2.b, z16.b\n" + "ld1b { z16.b }, p1/Z, [x20, x26]\n" "smax z1.b, p0/M, z1.b, z16.b\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End @@ -292,83 +293,83 @@ void sme_s8q_nhwc_max_generic_depthfirst_impl( "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" "mov z4.b, #0x80\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x24, x23, [x20, #0x0]\n" + "ldp x20, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" - "ld1b { z0.b }, p4/Z, [x24, x9]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ld1b { z31.b }, p4/Z, [x23, x9]\n" - "ld1b { z23.b }, p4/Z, [x22, x9]\n" - "ld1b { z30.b }, p4/Z, [x21, x9]\n" + "ld1b { z0.b }, p4/Z, [x20, x9]\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" + "ld1b { z31.b }, p4/Z, [x22, x9]\n" + "ld1b { z23.b }, p4/Z, [x21, x9]\n" + "ld1b { z30.b }, p4/Z, [x20, x9]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 4 inputs loop - "movprfx z19, z0\n smax z19.b, p0/M, z19.b, z31.b\n" - "smax z23.b, p0/M, z23.b, z30.b\n" - "ldp x24, x23, [x20, #0x0]\n" + "movprfx z16, z0\n smax z16.b, p0/M, z16.b, z31.b\n" + "movprfx z17, z23\n smax z17.b, p0/M, z17.b, z30.b\n" + "ldp x23, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" - "smax z19.b, p0/M, z19.b, z23.b\n" - "ldp x22, x21, [x20, #0x10]\n" - "smax z4.b, p0/M, z4.b, z19.b\n" - "add x20, x20, #0x20\n" - "ld1b { z0.b }, p4/Z, [x24, x9]\n" - "ld1b { z31.b }, p4/Z, [x23, x9]\n" - "ld1b { z23.b }, p4/Z, [x22, x9]\n" - "ld1b { z30.b }, p4/Z, [x21, x9]\n" + "smax z16.b, p0/M, z16.b, z17.b\n" + "ldp x21, x20, [x24, #0x10]\n" + "smax z4.b, p0/M, z4.b, z16.b\n" + "add x24, x24, #0x20\n" + "ld1b { z0.b }, p4/Z, [x23, x9]\n" + "ld1b { z31.b }, p4/Z, [x22, x9]\n" + "ld1b { z23.b }, p4/Z, [x21, x9]\n" + "ld1b { z30.b }, p4/Z, [x20, x9]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail - "movprfx z19, z0\n smax z19.b, p0/M, z19.b, z31.b\n" - "smax z23.b, p0/M, z23.b, z30.b\n" - "smax z19.b, p0/M, z19.b, z23.b\n" - "smax z4.b, p0/M, z4.b, z19.b\n" + "movprfx z16, z0\n smax z16.b, p0/M, z16.b, z31.b\n" + "movprfx z17, z23\n smax z17.b, p0/M, z17.b, z30.b\n" + "smax z16.b, p0/M, z16.b, z17.b\n" + "smax z4.b, p0/M, z4.b, z16.b\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1b { z0.b }, p4/Z, [x24, x9]\n" + "ldr x20, [x24], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x9]\n" "subs x21, x21, #0x1\n" - "smax z4.b, p0/M, z4.b, z0.b\n" + "smax z4.b, p0/M, z4.b, z16.b\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End - ".inst 0x4508a097 // sshllb z23.h, z4.b, #0x0\n" - ".inst 0x4508a496 // sshllt z22.h, z4.b, #0x0\n" + ".inst 0x4508a091 // sshllb z17.h, z4.b, #0x0\n" + ".inst 0x4508a490 // sshllt z16.h, z4.b, #0x0\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n" - "ld1rw { z4.s }, p0/Z, [x20]\n" - ".inst 0x4510a2e1 // sshllb z1.s, z23.h, #0x0\n" - ".inst 0x4510a6f7 // sshllt z23.s, z23.h, #0x0\n" + "ld1rw { z18.s }, p0/Z, [x20]\n" + ".inst 0x4510a236 // sshllb z22.s, z17.h, #0x0\n" + ".inst 0x4510a635 // sshllt z21.s, z17.h, #0x0\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_mul]\n" - "ld1rw { z3.s }, p0/Z, [x20]\n" - ".inst 0x4510a2c0 // sshllb z0.s, z22.h, #0x0\n" - ".inst 0x4510a6df // sshllt z31.s, z22.h, #0x0\n" + "ld1rw { z17.s }, p0/Z, [x20]\n" + ".inst 0x4510a214 // sshllb z20.s, z16.h, #0x0\n" + ".inst 0x4510a613 // sshllt z19.s, z16.h, #0x0\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n" - "ld1rw { z2.s }, p0/Z, [x20]\n" - ".inst 0x44828081 // srshl z1.s, p0/M, z1.s, z4.s\n" - ".inst 0x44828097 // srshl z23.s, p0/M, z23.s, z4.s\n" - ".inst 0x44828080 // srshl z0.s, p0/M, z0.s, z4.s\n" - ".inst 0x4482809f // srshl z31.s, p0/M, z31.s, z4.s\n" - ".inst 0x04a37421 // sqrdmulh z1.s, z1.s, z3.s\n" - ".inst 0x04a376f7 // sqrdmulh z23.s, z23.s, z3.s\n" - ".inst 0x04a37400 // sqrdmulh z0.s, z0.s, z3.s\n" - ".inst 0x04a377ff // sqrdmulh z31.s, z31.s, z3.s\n" - "mov z19.s, #0x7f\n" - ".inst 0x44828041 // srshl z1.s, p0/M, z1.s, z2.s\n" - ".inst 0x44828057 // srshl z23.s, p0/M, z23.s, z2.s\n" - ".inst 0x44828040 // srshl z0.s, p0/M, z0.s, z2.s\n" - ".inst 0x4482805f // srshl z31.s, p0/M, z31.s, z2.s\n" - "not z16.s, p0/M, z19.s\n" - "smax z1.s, p0/M, z1.s, z16.s\n" - "smax z23.s, p0/M, z23.s, z16.s\n" - "smax z0.s, p0/M, z0.s, z16.s\n" - "smax z31.s, p0/M, z31.s, z16.s\n" - "smin z1.s, p0/M, z1.s, z19.s\n" - "smin z23.s, p0/M, z23.s, z19.s\n" - "trn1 z23.h, z1.h, z23.h\n" - "smin z0.s, p0/M, z0.s, z19.s\n" - "smin z31.s, p0/M, z31.s, z19.s\n" - "trn1 z16.h, z0.h, z31.h\n" - "trn1 z16.b, z23.b, z16.b\n" + "ld1rw { z16.s }, p0/Z, [x20]\n" + ".inst 0x44828256 // srshl z22.s, p0/M, z22.s, z18.s\n" + ".inst 0x44828255 // srshl z21.s, p0/M, z21.s, z18.s\n" + ".inst 0x44828254 // srshl z20.s, p0/M, z20.s, z18.s\n" + ".inst 0x44828253 // srshl z19.s, p0/M, z19.s, z18.s\n" + ".inst 0x04b176d6 // sqrdmulh z22.s, z22.s, z17.s\n" + ".inst 0x04b176b5 // sqrdmulh z21.s, z21.s, z17.s\n" + ".inst 0x04b17694 // sqrdmulh z20.s, z20.s, z17.s\n" + ".inst 0x04b17673 // sqrdmulh z19.s, z19.s, z17.s\n" + "mov z18.s, #0x7f\n" + ".inst 0x44828216 // srshl z22.s, p0/M, z22.s, z16.s\n" + ".inst 0x44828215 // srshl z21.s, p0/M, z21.s, z16.s\n" + ".inst 0x44828214 // srshl z20.s, p0/M, z20.s, z16.s\n" + ".inst 0x44828213 // srshl z19.s, p0/M, z19.s, z16.s\n" + "not z16.s, p0/M, z18.s\n" + "smax z22.s, p0/M, z22.s, z16.s\n" + "smax z21.s, p0/M, z21.s, z16.s\n" + "smax z20.s, p0/M, z20.s, z16.s\n" + "smax z19.s, p0/M, z19.s, z16.s\n" + "smin z22.s, p0/M, z22.s, z18.s\n" + "smin z21.s, p0/M, z21.s, z18.s\n" + "trn1 z17.h, z22.h, z21.h\n" + "smin z20.s, p0/M, z20.s, z18.s\n" + "smin z19.s, p0/M, z19.s, z18.s\n" + "trn1 z16.h, z20.h, z19.h\n" + "trn1 z16.b, z17.b, z16.b\n" "st1b { z16.b }, p4, [%x[outptr], x9]\n" "incb x9\n" "whilelt p4.b, x9, %x[n_channels]\n" diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8_nhwc_avg_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8_nhwc_avg_generic_depthfirst.hpp index 2cdb2883c2aa075fead22a7909b4d164ab8f11fa..29a03ec509fe380d9863d5c551640e08981b4b60 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8_nhwc_avg_generic_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8_nhwc_avg_generic_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,8 @@ #pragma once +#if defined(ARM_COMPUTE_ENABLE_SME) + namespace arm_conv { namespace pooling { @@ -40,3 +42,5 @@ struct sme_u8_nhwc_avg_generic_depthfirst : IGenericDepthfirstStrategy +#include #include #include +#if defined(ARM_COMPUTE_ENABLE_SME) + namespace arm_conv { namespace pooling { @@ -109,7 +110,7 @@ void sme_u8_nhwc_avg_generic_depthfirst_impl( "lsr x23, %x[n_valid_cells], #0x1\n" "mov z15.s, #0x0\n" "mov z14.s, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "mov z13.s, #0x0\n" "mov z12.s, #0x0\n" "mov z11.s, #0x0\n" @@ -125,48 +126,48 @@ void sme_u8_nhwc_avg_generic_depthfirst_impl( "mov z1.s, #0x0\n" "mov z0.s, #0x0\n" "cbz x23, 4f\n" - "ldp x22, x21, [x20, #0x0]\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" - "ld1b { z29.b }, p3/Z, [x22, x26]\n" - "ld1b { z28.b }, p3/Z, [x21, x26]\n" - "ld1b { z27.b }, p2/Z, [x22, x25]\n" - "ld1b { z26.b }, p2/Z, [x21, x25]\n" - "ld1b { z25.b }, p1/Z, [x22, x24]\n" - "ld1b { z24.b }, p1/Z, [x21, x24]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" + "ld1b { z29.b }, p3/Z, [x21, x26]\n" + "ld1b { z28.b }, p3/Z, [x20, x26]\n" + "ld1b { z27.b }, p2/Z, [x21, x25]\n" + "ld1b { z26.b }, p2/Z, [x20, x25]\n" + "ld1b { z25.b }, p1/Z, [x21, x24]\n" + "ld1b { z24.b }, p1/Z, [x20, x24]\n" "beq 3f\n" "2:" // 4-vectors of channels: 2 inputs loop ".inst 0x455e0bf7 // uaddlb z23.h, z31.b, z30.b\n" ".inst 0x455e0ff6 // uaddlt z22.h, z31.b, z30.b\n" - "ldp x22, x21, [x20, #0x0]\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" ".inst 0x455c0bb5 // uaddlb z21.h, z29.b, z28.b\n" ".inst 0x455c0fb4 // uaddlt z20.h, z29.b, z28.b\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" ".inst 0x455a0b73 // uaddlb z19.h, z27.b, z26.b\n" ".inst 0x455a0f72 // uaddlt z18.h, z27.b, z26.b\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" ".inst 0x45580b31 // uaddlb z17.h, z25.b, z24.b\n" ".inst 0x45580f30 // uaddlt z16.h, z25.b, z24.b\n" - "ld1b { z29.b }, p3/Z, [x22, x26]\n" + "ld1b { z29.b }, p3/Z, [x21, x26]\n" ".inst 0x459749ef // uaddwb z15.s, z15.s, z23.h\n" ".inst 0x45974dce // uaddwt z14.s, z14.s, z23.h\n" - "ld1b { z28.b }, p3/Z, [x21, x26]\n" + "ld1b { z28.b }, p3/Z, [x20, x26]\n" ".inst 0x459649ad // uaddwb z13.s, z13.s, z22.h\n" ".inst 0x45964d8c // uaddwt z12.s, z12.s, z22.h\n" - "ld1b { z27.b }, p2/Z, [x22, x25]\n" + "ld1b { z27.b }, p2/Z, [x21, x25]\n" ".inst 0x4595496b // uaddwb z11.s, z11.s, z21.h\n" ".inst 0x45954d4a // uaddwt z10.s, z10.s, z21.h\n" - "ld1b { z26.b }, p2/Z, [x21, x25]\n" + "ld1b { z26.b }, p2/Z, [x20, x25]\n" ".inst 0x45944929 // uaddwb z9.s, z9.s, z20.h\n" ".inst 0x45944d08 // uaddwt z8.s, z8.s, z20.h\n" - "ld1b { z25.b }, p1/Z, [x22, x24]\n" + "ld1b { z25.b }, p1/Z, [x21, x24]\n" ".inst 0x459348e7 // uaddwb z7.s, z7.s, z19.h\n" ".inst 0x45934cc6 // uaddwt z6.s, z6.s, z19.h\n" - "ld1b { z24.b }, p1/Z, [x21, x24]\n" + "ld1b { z24.b }, p1/Z, [x20, x24]\n" ".inst 0x459248a5 // uaddwb z5.s, z5.s, z18.h\n" ".inst 0x45924c84 // uaddwt z4.s, z4.s, z18.h\n" ".inst 0x45914863 // uaddwb z3.s, z3.s, z17.h\n" @@ -203,20 +204,20 @@ void sme_u8_nhwc_avg_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x1\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x22, [x20], #0x8\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - ".inst 0x4508abf7 // ushllb z23.h, z31.b, #0x0\n" - ".inst 0x4508aff6 // ushllt z22.h, z31.b, #0x0\n" - "ld1b { z29.b }, p3/Z, [x22, x26]\n" - ".inst 0x4508abb5 // ushllb z21.h, z29.b, #0x0\n" - ".inst 0x4508afb4 // ushllt z20.h, z29.b, #0x0\n" + "ldr x20, [x22], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x27]\n" + ".inst 0x4508aa17 // ushllb z23.h, z16.b, #0x0\n" + ".inst 0x4508ae16 // ushllt z22.h, z16.b, #0x0\n" + "ld1b { z16.b }, p3/Z, [x20, x26]\n" + ".inst 0x4508aa15 // ushllb z21.h, z16.b, #0x0\n" + ".inst 0x4508ae14 // ushllt z20.h, z16.b, #0x0\n" "subs x21, x21, #0x1\n" - "ld1b { z27.b }, p2/Z, [x22, x25]\n" - ".inst 0x4508ab73 // ushllb z19.h, z27.b, #0x0\n" - ".inst 0x4508af72 // ushllt z18.h, z27.b, #0x0\n" - "ld1b { z25.b }, p1/Z, [x22, x24]\n" - ".inst 0x4508ab31 // ushllb z17.h, z25.b, #0x0\n" - ".inst 0x4508af30 // ushllt z16.h, z25.b, #0x0\n" + "ld1b { z16.b }, p2/Z, [x20, x25]\n" + ".inst 0x4508aa13 // ushllb z19.h, z16.b, #0x0\n" + ".inst 0x4508ae12 // ushllt z18.h, z16.b, #0x0\n" + "ld1b { z16.b }, p1/Z, [x20, x24]\n" + ".inst 0x4508aa11 // ushllb z17.h, z16.b, #0x0\n" + ".inst 0x4508ae10 // ushllt z16.h, z16.b, #0x0\n" ".inst 0x459749ef // uaddwb z15.s, z15.s, z23.h\n" ".inst 0x45974dce // uaddwt z14.s, z14.s, z23.h\n" ".inst 0x459649ad // uaddwb z13.s, z13.s, z22.h\n" @@ -332,74 +333,74 @@ void sme_u8_nhwc_avg_generic_depthfirst_impl( "lsr x23, %x[n_valid_cells], #0x1\n" "mov z15.s, #0x0\n" "mov z14.s, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "mov z13.s, #0x0\n" "mov z12.s, #0x0\n" "cbz x23, 11f\n" - "ldp x22, x21, [x20, #0x0]\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 2 inputs loop - ".inst 0x455e0bf7 // uaddlb z23.h, z31.b, z30.b\n" - ".inst 0x455e0ff6 // uaddlt z22.h, z31.b, z30.b\n" - "ldp x22, x21, [x20, #0x0]\n" + ".inst 0x455e0bf1 // uaddlb z17.h, z31.b, z30.b\n" + ".inst 0x455e0ff0 // uaddlt z16.h, z31.b, z30.b\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - ".inst 0x459749ef // uaddwb z15.s, z15.s, z23.h\n" - ".inst 0x45974dce // uaddwt z14.s, z14.s, z23.h\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - ".inst 0x459649ad // uaddwb z13.s, z13.s, z22.h\n" - ".inst 0x45964d8c // uaddwt z12.s, z12.s, z22.h\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" + ".inst 0x459149ef // uaddwb z15.s, z15.s, z17.h\n" + ".inst 0x45914dce // uaddwt z14.s, z14.s, z17.h\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" + ".inst 0x459049ad // uaddwb z13.s, z13.s, z16.h\n" + ".inst 0x45904d8c // uaddwt z12.s, z12.s, z16.h\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 2 inputs tail - ".inst 0x455e0bf7 // uaddlb z23.h, z31.b, z30.b\n" - ".inst 0x455e0ff6 // uaddlt z22.h, z31.b, z30.b\n" - ".inst 0x459749ef // uaddwb z15.s, z15.s, z23.h\n" - ".inst 0x45974dce // uaddwt z14.s, z14.s, z23.h\n" - ".inst 0x459649ad // uaddwb z13.s, z13.s, z22.h\n" - ".inst 0x45964d8c // uaddwt z12.s, z12.s, z22.h\n" + ".inst 0x455e0bf1 // uaddlb z17.h, z31.b, z30.b\n" + ".inst 0x455e0ff0 // uaddlt z16.h, z31.b, z30.b\n" + ".inst 0x459149ef // uaddwb z15.s, z15.s, z17.h\n" + ".inst 0x45914dce // uaddwt z14.s, z14.s, z17.h\n" + ".inst 0x459049ad // uaddwb z13.s, z13.s, z16.h\n" + ".inst 0x45904d8c // uaddwt z12.s, z12.s, z16.h\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x1\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x22, [x20], #0x8\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - ".inst 0x4508abf7 // ushllb z23.h, z31.b, #0x0\n" - ".inst 0x4508aff6 // ushllt z22.h, z31.b, #0x0\n" + "ldr x20, [x22], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x27]\n" + ".inst 0x4508aa11 // ushllb z17.h, z16.b, #0x0\n" + ".inst 0x4508ae10 // ushllt z16.h, z16.b, #0x0\n" "subs x21, x21, #0x1\n" - ".inst 0x459749ef // uaddwb z15.s, z15.s, z23.h\n" - ".inst 0x45974dce // uaddwt z14.s, z14.s, z23.h\n" - ".inst 0x459649ad // uaddwb z13.s, z13.s, z22.h\n" - ".inst 0x45964d8c // uaddwt z12.s, z12.s, z22.h\n" + ".inst 0x459149ef // uaddwb z15.s, z15.s, z17.h\n" + ".inst 0x45914dce // uaddwt z14.s, z14.s, z17.h\n" + ".inst 0x459049ad // uaddwb z13.s, z13.s, z16.h\n" + ".inst 0x45904d8c // uaddwt z12.s, z12.s, z16.h\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End - "ld1rw { z17.s }, p0/Z, [%x[rescale_ptr]]\n" - ".inst 0x04b175ef // sqdmulh z15.s, z15.s, z17.s\n" - ".inst 0x04b175ce // sqdmulh z14.s, z14.s, z17.s\n" - ".inst 0x04b175ad // sqdmulh z13.s, z13.s, z17.s\n" - ".inst 0x04b1758c // sqdmulh z12.s, z12.s, z17.s\n" + "ld1rw { z16.s }, p0/Z, [%x[rescale_ptr]]\n" + ".inst 0x04b075ef // sqdmulh z15.s, z15.s, z16.s\n" + ".inst 0x04b075ce // sqdmulh z14.s, z14.s, z16.s\n" + ".inst 0x04b075ad // sqdmulh z13.s, z13.s, z16.s\n" + ".inst 0x04b0758c // sqdmulh z12.s, z12.s, z16.s\n" "ld1rw { z16.s }, p0/Z, [%x[shift_ptr]]\n" ".inst 0x4482820f // srshl z15.s, p0/M, z15.s, z16.s\n" ".inst 0x4482820e // srshl z14.s, p0/M, z14.s, z16.s\n" ".inst 0x4482820d // srshl z13.s, p0/M, z13.s, z16.s\n" ".inst 0x4482820c // srshl z12.s, p0/M, z12.s, z16.s\n" - "mov z16.s, #0x0\n" - "mov z19.s, #0xff\n" - "smax z15.s, p0/M, z15.s, z16.s\n" - "smax z14.s, p0/M, z14.s, z16.s\n" - "smax z13.s, p0/M, z13.s, z16.s\n" - "smax z12.s, p0/M, z12.s, z16.s\n" - "smin z15.s, p0/M, z15.s, z19.s\n" - "smin z14.s, p0/M, z14.s, z19.s\n" - "trn1 z23.h, z15.h, z14.h\n" - "smin z13.s, p0/M, z13.s, z19.s\n" - "smin z12.s, p0/M, z12.s, z19.s\n" + "mov z17.s, #0x0\n" + "mov z16.s, #0xff\n" + "smax z15.s, p0/M, z15.s, z17.s\n" + "smax z14.s, p0/M, z14.s, z17.s\n" + "smax z13.s, p0/M, z13.s, z17.s\n" + "smax z12.s, p0/M, z12.s, z17.s\n" + "smin z15.s, p0/M, z15.s, z16.s\n" + "smin z14.s, p0/M, z14.s, z16.s\n" + "trn1 z17.h, z15.h, z14.h\n" + "smin z13.s, p0/M, z13.s, z16.s\n" + "smin z12.s, p0/M, z12.s, z16.s\n" "trn1 z16.h, z13.h, z12.h\n" - "trn1 z16.b, z23.b, z16.b\n" + "trn1 z16.b, z17.b, z16.b\n" "st1b { z16.b }, p4, [%x[outptr], x27]\n" "incb x27\n" "whilelt p4.b, x27, %x[n_channels]\n" diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8_nhwc_max_2x2_s1_output2x2_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8_nhwc_max_2x2_s1_output2x2_depthfirst.hpp index 6d5f53d7a53e4b3118d8f4a0cfecc90962a645c2..3df4e4efb810ed3acb649527ae6f209c088c05a9 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8_nhwc_max_2x2_s1_output2x2_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8_nhwc_max_2x2_s1_output2x2_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,8 @@ #pragma once +#if defined(ARM_COMPUTE_ENABLE_SME) + namespace arm_conv { namespace pooling { @@ -45,3 +47,5 @@ struct sme_u8_nhwc_max_2x2_s1_output2x2_depthfirst : public DepthfirstStrategy #include -#if defined(__ARM_FEATURE_SVE) && defined(ARM_COMPUTE_ENABLE_SME) +#if defined(ARM_COMPUTE_ENABLE_SME) namespace arm_conv { namespace pooling { @@ -70,23 +70,23 @@ void sme_u8_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "ldr x20, [%x[args], %[offsetof_inptrs]]\n" "mov x14, #0x0\n" "ldr x13, [%x[args], %[offsetof_n_channels]]\n" - "whilelt p1.b, x15, x13\n" + "whilelt p0.b, x15, x13\n" "ldp x12, x11, [x21, #0x0]\n" "ldp x10, x9, [x21, #0x10]\n" "ldp x28, x27, [x20, #0x0]\n" - "ld1b { z30.b }, p1/Z, [x27, x15]\n" + "ld1b { z30.b }, p0/Z, [x27, x15]\n" "ldp x26, x25, [x20, #0x10]\n" - "ld1b { z29.b }, p1/Z, [x25, x15]\n" + "ld1b { z29.b }, p0/Z, [x25, x15]\n" "ldp x24, x23, [x20, #0x20]\n" - "ld1b { z28.b }, p1/Z, [x24, x15]\n" + "ld1b { z28.b }, p0/Z, [x24, x15]\n" "ldp x22, x21, [x20, #0x30]\n" - "ld1b { z27.b }, p1/Z, [x21, x15]\n" + "ld1b { z27.b }, p0/Z, [x21, x15]\n" "ldr x20, [x20, #0x40]\n" - "ld1b { z26.b }, p1/Z, [x28, x15]\n" - "ld1b { z25.b }, p1/Z, [x26, x15]\n" - "ld1b { z24.b }, p1/Z, [x23, x15]\n" - "ld1b { z23.b }, p1/Z, [x22, x15]\n" - "ld1b { z19.b }, p1/Z, [x20, x15]\n" + "ld1b { z26.b }, p0/Z, [x28, x15]\n" + "ld1b { z25.b }, p0/Z, [x26, x15]\n" + "ld1b { z24.b }, p0/Z, [x23, x15]\n" + "ld1b { z19.b }, p0/Z, [x22, x15]\n" + "ld1b { z23.b }, p0/Z, [x20, x15]\n" "incw x15\n" "whilelt p1.b, x15, x13\n" "b.none 2f\n" @@ -95,25 +95,25 @@ void sme_u8_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "movprfx z21, z28\n umax z21.b, p2/M, z21.b, z27.b\n" "ld1b { z30.b }, p1/Z, [x27, x15]\n" "whilelt p0.b, x14, x13\n" - "movprfx z20, z29\n umax z20.b, p2/M, z20.b, z26.b\n" - "movprfx z18, z25\n umax z18.b, p2/M, z18.b, z24.b\n" + "movprfx z18, z29\n umax z18.b, p2/M, z18.b, z26.b\n" + "movprfx z17, z25\n umax z17.b, p2/M, z17.b, z24.b\n" "ld1b { z28.b }, p1/Z, [x24, x15]\n" - "movprfx z17, z29\n umax z17.b, p2/M, z17.b, z23.b\n" - "movprfx z16, z24\n umax z16.b, p2/M, z16.b, z19.b\n" + "movprfx z16, z29\n umax z16.b, p2/M, z16.b, z19.b\n" + "movprfx z20, z24\n umax z20.b, p2/M, z20.b, z23.b\n" "ld1b { z27.b }, p1/Z, [x21, x15]\n" "ld1b { z29.b }, p1/Z, [x25, x15]\n" - "movprfx z19, z22\n umax z19.b, p2/M, z19.b, z20.b\n" - "umax z18.b, p2/M, z18.b, z22.b\n" + "movprfx z19, z22\n umax z19.b, p2/M, z19.b, z18.b\n" + "movprfx z18, z17\n umax z18.b, p2/M, z18.b, z22.b\n" "ld1b { z26.b }, p1/Z, [x28, x15]\n" - "umax z17.b, p2/M, z17.b, z21.b\n" - "umax z16.b, p2/M, z16.b, z21.b\n" + "movprfx z17, z16\n umax z17.b, p2/M, z17.b, z21.b\n" + "movprfx z16, z21\n umax z16.b, p2/M, z16.b, z20.b\n" "ld1b { z25.b }, p1/Z, [x26, x15]\n" "st1b { z19.b }, p0, [x12, x14]\n" "ld1b { z24.b }, p1/Z, [x23, x15]\n" "st1b { z18.b }, p0, [x11, x14]\n" - "ld1b { z23.b }, p1/Z, [x22, x15]\n" + "ld1b { z19.b }, p1/Z, [x22, x15]\n" "st1b { z17.b }, p0, [x10, x14]\n" - "ld1b { z19.b }, p1/Z, [x20, x15]\n" + "ld1b { z23.b }, p1/Z, [x20, x15]\n" "incw x15\n" "whilelt p1.b, x15, x13\n" "st1b { z16.b }, p0, [x9, x14]\n" @@ -125,13 +125,13 @@ void sme_u8_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "whilelt p0.b, x14, x13\n" "movprfx z20, z29\n umax z20.b, p2/M, z20.b, z26.b\n" "movprfx z18, z25\n umax z18.b, p2/M, z18.b, z24.b\n" - "movprfx z17, z29\n umax z17.b, p2/M, z17.b, z23.b\n" - "movprfx z16, z24\n umax z16.b, p2/M, z16.b, z19.b\n" - "movprfx z19, z22\n umax z19.b, p2/M, z19.b, z20.b\n" + "movprfx z17, z29\n umax z17.b, p2/M, z17.b, z19.b\n" + "movprfx z19, z24\n umax z19.b, p2/M, z19.b, z23.b\n" + "movprfx z16, z22\n umax z16.b, p2/M, z16.b, z20.b\n" "umax z18.b, p2/M, z18.b, z22.b\n" - "st1b { z19.b }, p0, [x12, x14]\n" + "st1b { z16.b }, p0, [x12, x14]\n" "umax z17.b, p2/M, z17.b, z21.b\n" - "umax z16.b, p2/M, z16.b, z21.b\n" + "movprfx z16, z21\n umax z16.b, p2/M, z16.b, z19.b\n" "st1b { z18.b }, p0, [x11, x14]\n" "st1b { z17.b }, p0, [x10, x14]\n" "st1b { z16.b }, p0, [x9, x14]\n" @@ -145,4 +145,4 @@ void sme_u8_nhwc_max_2x2_s1_output2x2_depthfirst_impl( } // namespace pooling } // namespace arm_conv -#endif // defined(__ARM_FEATURE_SVE) && defined(ARM_COMPUTE_ENABLE_SME) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8_nhwc_max_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8_nhwc_max_generic_depthfirst.hpp index 5c637ec3c373230118efeaca88a055b06e9aca22..077c8ed2f78ade5ec1521edcbf4adf1b83c8ac61 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8_nhwc_max_generic_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8_nhwc_max_generic_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,8 @@ #pragma once +#if defined(ARM_COMPUTE_ENABLE_SME) + namespace arm_conv { namespace pooling { @@ -40,3 +42,5 @@ struct sme_u8_nhwc_max_generic_depthfirst : IGenericDepthfirstStrategy +#include + +#if defined(ARM_COMPUTE_ENABLE_SME) namespace arm_conv { namespace pooling { @@ -54,68 +55,68 @@ void sme_u8_nhwc_max_generic_depthfirst_impl( "lsr x25, %x[n_valid_cells], #0x2\n" "mov z4.b, #0x0\n" "mov z3.b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "mov z2.b, #0x0\n" "mov z1.b, #0x0\n" "cbz x25, 4f\n" - "ldp x24, x23, [x20, #0x0]\n" + "ldp x23, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" - "ld1b { z0.b }, p4/Z, [x24, x9]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ld1b { z31.b }, p4/Z, [x23, x9]\n" - "ld1b { z23.b }, p4/Z, [x22, x9]\n" - "ld1b { z30.b }, p4/Z, [x21, x9]\n" - "ld1b { z18.b }, p3/Z, [x24, x28]\n" - "ld1b { z29.b }, p3/Z, [x23, x28]\n" - "ld1b { z22.b }, p3/Z, [x22, x28]\n" - "ld1b { z28.b }, p3/Z, [x21, x28]\n" - "ld1b { z17.b }, p2/Z, [x24, x27]\n" - "ld1b { z27.b }, p2/Z, [x23, x27]\n" - "ld1b { z21.b }, p2/Z, [x22, x27]\n" - "ld1b { z26.b }, p2/Z, [x21, x27]\n" - "ld1b { z16.b }, p1/Z, [x24, x26]\n" - "ld1b { z25.b }, p1/Z, [x23, x26]\n" - "ld1b { z20.b }, p1/Z, [x22, x26]\n" - "ld1b { z24.b }, p1/Z, [x21, x26]\n" + "ld1b { z0.b }, p4/Z, [x23, x9]\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" + "ld1b { z31.b }, p4/Z, [x22, x9]\n" + "ld1b { z23.b }, p4/Z, [x21, x9]\n" + "ld1b { z30.b }, p4/Z, [x20, x9]\n" + "ld1b { z18.b }, p3/Z, [x23, x28]\n" + "ld1b { z29.b }, p3/Z, [x22, x28]\n" + "ld1b { z22.b }, p3/Z, [x21, x28]\n" + "ld1b { z28.b }, p3/Z, [x20, x28]\n" + "ld1b { z17.b }, p2/Z, [x23, x27]\n" + "ld1b { z27.b }, p2/Z, [x22, x27]\n" + "ld1b { z21.b }, p2/Z, [x21, x27]\n" + "ld1b { z26.b }, p2/Z, [x20, x27]\n" + "ld1b { z16.b }, p1/Z, [x23, x26]\n" + "ld1b { z25.b }, p1/Z, [x22, x26]\n" + "ld1b { z20.b }, p1/Z, [x21, x26]\n" + "ld1b { z24.b }, p1/Z, [x20, x26]\n" "beq 3f\n" "2:" // 4-vectors of channels: 4 inputs loop "movprfx z19, z0\n umax z19.b, p0/M, z19.b, z31.b\n" "umax z23.b, p0/M, z23.b, z30.b\n" - "ldp x24, x23, [x20, #0x0]\n" + "ldp x23, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" "umax z18.b, p0/M, z18.b, z29.b\n" "umax z22.b, p0/M, z22.b, z28.b\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" "umax z17.b, p0/M, z17.b, z27.b\n" "umax z21.b, p0/M, z21.b, z26.b\n" - "ld1b { z0.b }, p4/Z, [x24, x9]\n" + "ld1b { z0.b }, p4/Z, [x23, x9]\n" "umax z16.b, p0/M, z16.b, z25.b\n" "umax z20.b, p0/M, z20.b, z24.b\n" - "ld1b { z31.b }, p4/Z, [x23, x9]\n" + "ld1b { z31.b }, p4/Z, [x22, x9]\n" "umax z19.b, p0/M, z19.b, z23.b\n" "umax z18.b, p0/M, z18.b, z22.b\n" - "ld1b { z23.b }, p4/Z, [x22, x9]\n" + "ld1b { z23.b }, p4/Z, [x21, x9]\n" "umax z17.b, p0/M, z17.b, z21.b\n" "umax z16.b, p0/M, z16.b, z20.b\n" - "ld1b { z30.b }, p4/Z, [x21, x9]\n" + "ld1b { z30.b }, p4/Z, [x20, x9]\n" "umax z4.b, p0/M, z4.b, z19.b\n" "umax z3.b, p0/M, z3.b, z18.b\n" - "ld1b { z18.b }, p3/Z, [x24, x28]\n" + "ld1b { z18.b }, p3/Z, [x23, x28]\n" "umax z2.b, p0/M, z2.b, z17.b\n" "umax z1.b, p0/M, z1.b, z16.b\n" - "ld1b { z29.b }, p3/Z, [x23, x28]\n" - "ld1b { z22.b }, p3/Z, [x22, x28]\n" - "ld1b { z28.b }, p3/Z, [x21, x28]\n" - "ld1b { z17.b }, p2/Z, [x24, x27]\n" - "ld1b { z27.b }, p2/Z, [x23, x27]\n" - "ld1b { z21.b }, p2/Z, [x22, x27]\n" - "ld1b { z26.b }, p2/Z, [x21, x27]\n" - "ld1b { z16.b }, p1/Z, [x24, x26]\n" - "ld1b { z25.b }, p1/Z, [x23, x26]\n" - "ld1b { z20.b }, p1/Z, [x22, x26]\n" - "ld1b { z24.b }, p1/Z, [x21, x26]\n" + "ld1b { z29.b }, p3/Z, [x22, x28]\n" + "ld1b { z22.b }, p3/Z, [x21, x28]\n" + "ld1b { z28.b }, p3/Z, [x20, x28]\n" + "ld1b { z17.b }, p2/Z, [x23, x27]\n" + "ld1b { z27.b }, p2/Z, [x22, x27]\n" + "ld1b { z21.b }, p2/Z, [x21, x27]\n" + "ld1b { z26.b }, p2/Z, [x20, x27]\n" + "ld1b { z16.b }, p1/Z, [x23, x26]\n" + "ld1b { z25.b }, p1/Z, [x22, x26]\n" + "ld1b { z20.b }, p1/Z, [x21, x26]\n" + "ld1b { z24.b }, p1/Z, [x20, x26]\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "movprfx z19, z0\n umax z19.b, p0/M, z19.b, z31.b\n" @@ -138,15 +139,15 @@ void sme_u8_nhwc_max_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1b { z0.b }, p4/Z, [x24, x9]\n" + "ldr x20, [x24], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x9]\n" "subs x21, x21, #0x1\n" - "umax z4.b, p0/M, z4.b, z0.b\n" - "ld1b { z18.b }, p3/Z, [x24, x28]\n" - "umax z3.b, p0/M, z3.b, z18.b\n" - "ld1b { z17.b }, p2/Z, [x24, x27]\n" - "umax z2.b, p0/M, z2.b, z17.b\n" - "ld1b { z16.b }, p1/Z, [x24, x26]\n" + "umax z4.b, p0/M, z4.b, z16.b\n" + "ld1b { z16.b }, p3/Z, [x20, x28]\n" + "umax z3.b, p0/M, z3.b, z16.b\n" + "ld1b { z16.b }, p2/Z, [x20, x27]\n" + "umax z2.b, p0/M, z2.b, z16.b\n" + "ld1b { z16.b }, p1/Z, [x20, x26]\n" "umax z1.b, p0/M, z1.b, z16.b\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End @@ -166,44 +167,44 @@ void sme_u8_nhwc_max_generic_depthfirst_impl( "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" "mov z4.b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x24, x23, [x20, #0x0]\n" + "ldp x20, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" - "ld1b { z0.b }, p4/Z, [x24, x9]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ld1b { z31.b }, p4/Z, [x23, x9]\n" - "ld1b { z23.b }, p4/Z, [x22, x9]\n" - "ld1b { z30.b }, p4/Z, [x21, x9]\n" + "ld1b { z0.b }, p4/Z, [x20, x9]\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" + "ld1b { z31.b }, p4/Z, [x22, x9]\n" + "ld1b { z23.b }, p4/Z, [x21, x9]\n" + "ld1b { z30.b }, p4/Z, [x20, x9]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 4 inputs loop - "movprfx z19, z0\n umax z19.b, p0/M, z19.b, z31.b\n" - "umax z23.b, p0/M, z23.b, z30.b\n" - "ldp x24, x23, [x20, #0x0]\n" + "movprfx z16, z0\n umax z16.b, p0/M, z16.b, z31.b\n" + "movprfx z17, z23\n umax z17.b, p0/M, z17.b, z30.b\n" + "ldp x23, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" - "umax z19.b, p0/M, z19.b, z23.b\n" - "ldp x22, x21, [x20, #0x10]\n" - "umax z4.b, p0/M, z4.b, z19.b\n" - "add x20, x20, #0x20\n" - "ld1b { z0.b }, p4/Z, [x24, x9]\n" - "ld1b { z31.b }, p4/Z, [x23, x9]\n" - "ld1b { z23.b }, p4/Z, [x22, x9]\n" - "ld1b { z30.b }, p4/Z, [x21, x9]\n" + "umax z16.b, p0/M, z16.b, z17.b\n" + "ldp x21, x20, [x24, #0x10]\n" + "umax z4.b, p0/M, z4.b, z16.b\n" + "add x24, x24, #0x20\n" + "ld1b { z0.b }, p4/Z, [x23, x9]\n" + "ld1b { z31.b }, p4/Z, [x22, x9]\n" + "ld1b { z23.b }, p4/Z, [x21, x9]\n" + "ld1b { z30.b }, p4/Z, [x20, x9]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail - "movprfx z19, z0\n umax z19.b, p0/M, z19.b, z31.b\n" - "umax z23.b, p0/M, z23.b, z30.b\n" - "umax z19.b, p0/M, z19.b, z23.b\n" - "umax z4.b, p0/M, z4.b, z19.b\n" + "movprfx z16, z0\n umax z16.b, p0/M, z16.b, z31.b\n" + "movprfx z17, z23\n umax z17.b, p0/M, z17.b, z30.b\n" + "umax z16.b, p0/M, z16.b, z17.b\n" + "umax z4.b, p0/M, z4.b, z16.b\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1b { z0.b }, p4/Z, [x24, x9]\n" + "ldr x20, [x24], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x9]\n" "subs x21, x21, #0x1\n" - "umax z4.b, p0/M, z4.b, z0.b\n" + "umax z4.b, p0/M, z4.b, z16.b\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "st1b { z4.b }, p4, [%x[outptr], x9]\n" diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8q_nhwc_avg_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8q_nhwc_avg_generic_depthfirst.hpp index 2930993800c0f12c97d5948fa1c1aaaf6c56d1b9..bd30a328282d28bb72c43603db499c437780293f 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8q_nhwc_avg_generic_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8q_nhwc_avg_generic_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,8 @@ #pragma once +#if defined(ARM_COMPUTE_ENABLE_SME) + namespace arm_conv { namespace pooling { @@ -40,3 +42,5 @@ struct sme_u8q_nhwc_avg_generic_depthfirst : IGenericDepthfirstStrategy +#include #include #include +#if defined(ARM_COMPUTE_ENABLE_SME) + namespace arm_conv { namespace pooling { @@ -136,7 +137,7 @@ void sme_u8q_nhwc_avg_generic_depthfirst_impl( "mov z13.d, z15.d\n" "mov z12.d, z15.d\n" "mov z11.d, z15.d\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "mov z10.d, z15.d\n" "mov z9.d, z15.d\n" "mov z8.d, z15.d\n" @@ -149,48 +150,48 @@ void sme_u8q_nhwc_avg_generic_depthfirst_impl( "mov z1.d, z15.d\n" "mov z0.d, z15.d\n" "cbz x23, 4f\n" - "ldp x22, x21, [x20, #0x0]\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" - "ld1b { z29.b }, p3/Z, [x22, x26]\n" - "ld1b { z28.b }, p3/Z, [x21, x26]\n" - "ld1b { z27.b }, p2/Z, [x22, x25]\n" - "ld1b { z26.b }, p2/Z, [x21, x25]\n" - "ld1b { z25.b }, p1/Z, [x22, x24]\n" - "ld1b { z24.b }, p1/Z, [x21, x24]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" + "ld1b { z29.b }, p3/Z, [x21, x26]\n" + "ld1b { z28.b }, p3/Z, [x20, x26]\n" + "ld1b { z27.b }, p2/Z, [x21, x25]\n" + "ld1b { z26.b }, p2/Z, [x20, x25]\n" + "ld1b { z25.b }, p1/Z, [x21, x24]\n" + "ld1b { z24.b }, p1/Z, [x20, x24]\n" "beq 3f\n" "2:" // 4-vectors of channels: 2 inputs loop ".inst 0x455e0bf7 // uaddlb z23.h, z31.b, z30.b\n" ".inst 0x455e0ff6 // uaddlt z22.h, z31.b, z30.b\n" - "ldp x22, x21, [x20, #0x0]\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" ".inst 0x455c0bb5 // uaddlb z21.h, z29.b, z28.b\n" ".inst 0x455c0fb4 // uaddlt z20.h, z29.b, z28.b\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" ".inst 0x455a0b73 // uaddlb z19.h, z27.b, z26.b\n" ".inst 0x455a0f72 // uaddlt z18.h, z27.b, z26.b\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" ".inst 0x45580b31 // uaddlb z17.h, z25.b, z24.b\n" ".inst 0x45580f30 // uaddlt z16.h, z25.b, z24.b\n" - "ld1b { z29.b }, p3/Z, [x22, x26]\n" + "ld1b { z29.b }, p3/Z, [x21, x26]\n" ".inst 0x459749ef // uaddwb z15.s, z15.s, z23.h\n" ".inst 0x45974dce // uaddwt z14.s, z14.s, z23.h\n" - "ld1b { z28.b }, p3/Z, [x21, x26]\n" + "ld1b { z28.b }, p3/Z, [x20, x26]\n" ".inst 0x459649ad // uaddwb z13.s, z13.s, z22.h\n" ".inst 0x45964d8c // uaddwt z12.s, z12.s, z22.h\n" - "ld1b { z27.b }, p2/Z, [x22, x25]\n" + "ld1b { z27.b }, p2/Z, [x21, x25]\n" ".inst 0x4595496b // uaddwb z11.s, z11.s, z21.h\n" ".inst 0x45954d4a // uaddwt z10.s, z10.s, z21.h\n" - "ld1b { z26.b }, p2/Z, [x21, x25]\n" + "ld1b { z26.b }, p2/Z, [x20, x25]\n" ".inst 0x45944929 // uaddwb z9.s, z9.s, z20.h\n" ".inst 0x45944d08 // uaddwt z8.s, z8.s, z20.h\n" - "ld1b { z25.b }, p1/Z, [x22, x24]\n" + "ld1b { z25.b }, p1/Z, [x21, x24]\n" ".inst 0x459348e7 // uaddwb z7.s, z7.s, z19.h\n" ".inst 0x45934cc6 // uaddwt z6.s, z6.s, z19.h\n" - "ld1b { z24.b }, p1/Z, [x21, x24]\n" + "ld1b { z24.b }, p1/Z, [x20, x24]\n" ".inst 0x459248a5 // uaddwb z5.s, z5.s, z18.h\n" ".inst 0x45924c84 // uaddwt z4.s, z4.s, z18.h\n" ".inst 0x45914863 // uaddwb z3.s, z3.s, z17.h\n" @@ -227,20 +228,20 @@ void sme_u8q_nhwc_avg_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x1\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x22, [x20], #0x8\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - ".inst 0x4508abf7 // ushllb z23.h, z31.b, #0x0\n" - ".inst 0x4508aff6 // ushllt z22.h, z31.b, #0x0\n" - "ld1b { z29.b }, p3/Z, [x22, x26]\n" - ".inst 0x4508abb5 // ushllb z21.h, z29.b, #0x0\n" - ".inst 0x4508afb4 // ushllt z20.h, z29.b, #0x0\n" + "ldr x20, [x22], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x27]\n" + ".inst 0x4508aa17 // ushllb z23.h, z16.b, #0x0\n" + ".inst 0x4508ae16 // ushllt z22.h, z16.b, #0x0\n" + "ld1b { z16.b }, p3/Z, [x20, x26]\n" + ".inst 0x4508aa15 // ushllb z21.h, z16.b, #0x0\n" + ".inst 0x4508ae14 // ushllt z20.h, z16.b, #0x0\n" "subs x21, x21, #0x1\n" - "ld1b { z27.b }, p2/Z, [x22, x25]\n" - ".inst 0x4508ab73 // ushllb z19.h, z27.b, #0x0\n" - ".inst 0x4508af72 // ushllt z18.h, z27.b, #0x0\n" - "ld1b { z25.b }, p1/Z, [x22, x24]\n" - ".inst 0x4508ab31 // ushllb z17.h, z25.b, #0x0\n" - ".inst 0x4508af30 // ushllt z16.h, z25.b, #0x0\n" + "ld1b { z16.b }, p2/Z, [x20, x25]\n" + ".inst 0x4508aa13 // ushllb z19.h, z16.b, #0x0\n" + ".inst 0x4508ae12 // ushllt z18.h, z16.b, #0x0\n" + "ld1b { z16.b }, p1/Z, [x20, x24]\n" + ".inst 0x4508aa11 // ushllb z17.h, z16.b, #0x0\n" + ".inst 0x4508ae10 // ushllt z16.h, z16.b, #0x0\n" ".inst 0x459749ef // uaddwb z15.s, z15.s, z23.h\n" ".inst 0x45974dce // uaddwt z14.s, z14.s, z23.h\n" ".inst 0x459649ad // uaddwb z13.s, z13.s, z22.h\n" @@ -393,61 +394,61 @@ void sme_u8q_nhwc_avg_generic_depthfirst_impl( "mov z14.d, z15.d\n" "mov z13.d, z15.d\n" "mov z12.d, z15.d\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "cbz x23, 11f\n" - "ldp x22, x21, [x20, #0x0]\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 2 inputs loop - ".inst 0x455e0bf7 // uaddlb z23.h, z31.b, z30.b\n" - ".inst 0x455e0ff6 // uaddlt z22.h, z31.b, z30.b\n" - "ldp x22, x21, [x20, #0x0]\n" + ".inst 0x455e0bf1 // uaddlb z17.h, z31.b, z30.b\n" + ".inst 0x455e0ff0 // uaddlt z16.h, z31.b, z30.b\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - ".inst 0x459749ef // uaddwb z15.s, z15.s, z23.h\n" - ".inst 0x45974dce // uaddwt z14.s, z14.s, z23.h\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - ".inst 0x459649ad // uaddwb z13.s, z13.s, z22.h\n" - ".inst 0x45964d8c // uaddwt z12.s, z12.s, z22.h\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" + ".inst 0x459149ef // uaddwb z15.s, z15.s, z17.h\n" + ".inst 0x45914dce // uaddwt z14.s, z14.s, z17.h\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" + ".inst 0x459049ad // uaddwb z13.s, z13.s, z16.h\n" + ".inst 0x45904d8c // uaddwt z12.s, z12.s, z16.h\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 2 inputs tail - ".inst 0x455e0bf7 // uaddlb z23.h, z31.b, z30.b\n" - ".inst 0x455e0ff6 // uaddlt z22.h, z31.b, z30.b\n" - ".inst 0x459749ef // uaddwb z15.s, z15.s, z23.h\n" - ".inst 0x45974dce // uaddwt z14.s, z14.s, z23.h\n" - ".inst 0x459649ad // uaddwb z13.s, z13.s, z22.h\n" - ".inst 0x45964d8c // uaddwt z12.s, z12.s, z22.h\n" + ".inst 0x455e0bf1 // uaddlb z17.h, z31.b, z30.b\n" + ".inst 0x455e0ff0 // uaddlt z16.h, z31.b, z30.b\n" + ".inst 0x459149ef // uaddwb z15.s, z15.s, z17.h\n" + ".inst 0x45914dce // uaddwt z14.s, z14.s, z17.h\n" + ".inst 0x459049ad // uaddwb z13.s, z13.s, z16.h\n" + ".inst 0x45904d8c // uaddwt z12.s, z12.s, z16.h\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x1\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x22, [x20], #0x8\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - ".inst 0x4508abf7 // ushllb z23.h, z31.b, #0x0\n" - ".inst 0x4508aff6 // ushllt z22.h, z31.b, #0x0\n" + "ldr x20, [x22], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x27]\n" + ".inst 0x4508aa11 // ushllb z17.h, z16.b, #0x0\n" + ".inst 0x4508ae10 // ushllt z16.h, z16.b, #0x0\n" "subs x21, x21, #0x1\n" - ".inst 0x459749ef // uaddwb z15.s, z15.s, z23.h\n" - ".inst 0x45974dce // uaddwt z14.s, z14.s, z23.h\n" - ".inst 0x459649ad // uaddwb z13.s, z13.s, z22.h\n" - ".inst 0x45964d8c // uaddwt z12.s, z12.s, z22.h\n" + ".inst 0x459149ef // uaddwb z15.s, z15.s, z17.h\n" + ".inst 0x45914dce // uaddwt z14.s, z14.s, z17.h\n" + ".inst 0x459049ad // uaddwb z13.s, z13.s, z16.h\n" + ".inst 0x45904d8c // uaddwt z12.s, z12.s, z16.h\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End - "ld1rw { z19.s }, p0/Z, [%x[left_shift]]\n" - ".inst 0x4482826f // srshl z15.s, p0/M, z15.s, z19.s\n" - ".inst 0x4482826e // srshl z14.s, p0/M, z14.s, z19.s\n" + "ld1rw { z16.s }, p0/Z, [%x[left_shift]]\n" + ".inst 0x4482820f // srshl z15.s, p0/M, z15.s, z16.s\n" + ".inst 0x4482820e // srshl z14.s, p0/M, z14.s, z16.s\n" "add x20, %x[quant_params], %[offsetof_qp_output_offset]\n" - ".inst 0x4482826d // srshl z13.s, p0/M, z13.s, z19.s\n" - ".inst 0x4482826c // srshl z12.s, p0/M, z12.s, z19.s\n" - "ld1rw { z18.s }, p0/Z, [%x[combined_rescale_value]]\n" - ".inst 0x04b275ef // sqrdmulh z15.s, z15.s, z18.s\n" - ".inst 0x04b275ce // sqrdmulh z14.s, z14.s, z18.s\n" + ".inst 0x4482820d // srshl z13.s, p0/M, z13.s, z16.s\n" + ".inst 0x4482820c // srshl z12.s, p0/M, z12.s, z16.s\n" + "ld1rw { z16.s }, p0/Z, [%x[combined_rescale_value]]\n" + ".inst 0x04b075ef // sqrdmulh z15.s, z15.s, z16.s\n" + ".inst 0x04b075ce // sqrdmulh z14.s, z14.s, z16.s\n" "ld1rw { z17.s }, p0/Z, [%x[right_shift]]\n" - ".inst 0x04b275ad // sqrdmulh z13.s, z13.s, z18.s\n" - ".inst 0x04b2758c // sqrdmulh z12.s, z12.s, z18.s\n" + ".inst 0x04b075ad // sqrdmulh z13.s, z13.s, z16.s\n" + ".inst 0x04b0758c // sqrdmulh z12.s, z12.s, z16.s\n" "ld1rw { z16.s }, p0/Z, [x20]\n" ".inst 0x4482822f // srshl z15.s, p0/M, z15.s, z17.s\n" ".inst 0x4482822e // srshl z14.s, p0/M, z14.s, z17.s\n" @@ -457,19 +458,19 @@ void sme_u8q_nhwc_avg_generic_depthfirst_impl( "add z14.s, z14.s, z16.s\n" "add z13.s, z13.s, z16.s\n" "add z12.s, z12.s, z16.s\n" - "mov z16.s, #0x0\n" - "mov z19.s, #0xff\n" - "smax z15.s, p0/M, z15.s, z16.s\n" - "smax z14.s, p0/M, z14.s, z16.s\n" - "smax z13.s, p0/M, z13.s, z16.s\n" - "smax z12.s, p0/M, z12.s, z16.s\n" - "smin z15.s, p0/M, z15.s, z19.s\n" - "smin z14.s, p0/M, z14.s, z19.s\n" - "trn1 z23.h, z15.h, z14.h\n" - "smin z13.s, p0/M, z13.s, z19.s\n" - "smin z12.s, p0/M, z12.s, z19.s\n" + "mov z17.s, #0x0\n" + "mov z16.s, #0xff\n" + "smax z15.s, p0/M, z15.s, z17.s\n" + "smax z14.s, p0/M, z14.s, z17.s\n" + "smax z13.s, p0/M, z13.s, z17.s\n" + "smax z12.s, p0/M, z12.s, z17.s\n" + "smin z15.s, p0/M, z15.s, z16.s\n" + "smin z14.s, p0/M, z14.s, z16.s\n" + "trn1 z17.h, z15.h, z14.h\n" + "smin z13.s, p0/M, z13.s, z16.s\n" + "smin z12.s, p0/M, z12.s, z16.s\n" "trn1 z16.h, z13.h, z12.h\n" - "trn1 z16.b, z23.b, z16.b\n" + "trn1 z16.b, z17.b, z16.b\n" "st1b { z16.b }, p4, [%x[outptr], x27]\n" "incb x27\n" "whilelt p4.b, x27, %x[n_channels]\n" diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8q_nhwc_max_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8q_nhwc_max_generic_depthfirst.hpp index d7bf6cbd083d67335a6c6a8a42818d0933e8fef1..69d627c0473f796df5bd9ee65943486c2692e212 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8q_nhwc_max_generic_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8q_nhwc_max_generic_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,8 @@ #pragma once +#if defined(ARM_COMPUTE_ENABLE_SME) + namespace arm_conv { namespace pooling { @@ -40,3 +42,5 @@ struct sme_u8q_nhwc_max_generic_depthfirst : IGenericDepthfirstStrategy +#include + +#if defined(ARM_COMPUTE_ENABLE_SME) namespace arm_conv { namespace pooling { @@ -56,68 +57,68 @@ void sme_u8q_nhwc_max_generic_depthfirst_impl( "lsr x25, %x[n_valid_cells], #0x2\n" "mov z5.b, #0x0\n" "mov z3.b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "mov z2.b, #0x0\n" "mov z1.b, #0x0\n" "cbz x25, 4f\n" - "ldp x24, x23, [x20, #0x0]\n" + "ldp x23, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" - "ld1b { z0.b }, p4/Z, [x24, x9]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ld1b { z31.b }, p4/Z, [x23, x9]\n" - "ld1b { z23.b }, p4/Z, [x22, x9]\n" - "ld1b { z30.b }, p4/Z, [x21, x9]\n" - "ld1b { z18.b }, p3/Z, [x24, x28]\n" - "ld1b { z29.b }, p3/Z, [x23, x28]\n" - "ld1b { z22.b }, p3/Z, [x22, x28]\n" - "ld1b { z28.b }, p3/Z, [x21, x28]\n" - "ld1b { z17.b }, p2/Z, [x24, x27]\n" - "ld1b { z27.b }, p2/Z, [x23, x27]\n" - "ld1b { z21.b }, p2/Z, [x22, x27]\n" - "ld1b { z26.b }, p2/Z, [x21, x27]\n" - "ld1b { z16.b }, p1/Z, [x24, x26]\n" - "ld1b { z25.b }, p1/Z, [x23, x26]\n" - "ld1b { z20.b }, p1/Z, [x22, x26]\n" - "ld1b { z24.b }, p1/Z, [x21, x26]\n" + "ld1b { z0.b }, p4/Z, [x23, x9]\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" + "ld1b { z31.b }, p4/Z, [x22, x9]\n" + "ld1b { z23.b }, p4/Z, [x21, x9]\n" + "ld1b { z30.b }, p4/Z, [x20, x9]\n" + "ld1b { z18.b }, p3/Z, [x23, x28]\n" + "ld1b { z29.b }, p3/Z, [x22, x28]\n" + "ld1b { z22.b }, p3/Z, [x21, x28]\n" + "ld1b { z28.b }, p3/Z, [x20, x28]\n" + "ld1b { z17.b }, p2/Z, [x23, x27]\n" + "ld1b { z27.b }, p2/Z, [x22, x27]\n" + "ld1b { z21.b }, p2/Z, [x21, x27]\n" + "ld1b { z26.b }, p2/Z, [x20, x27]\n" + "ld1b { z16.b }, p1/Z, [x23, x26]\n" + "ld1b { z25.b }, p1/Z, [x22, x26]\n" + "ld1b { z20.b }, p1/Z, [x21, x26]\n" + "ld1b { z24.b }, p1/Z, [x20, x26]\n" "beq 3f\n" "2:" // 4-vectors of channels: 4 inputs loop "movprfx z19, z0\n umax z19.b, p0/M, z19.b, z31.b\n" "umax z23.b, p0/M, z23.b, z30.b\n" - "ldp x24, x23, [x20, #0x0]\n" + "ldp x23, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" "umax z18.b, p0/M, z18.b, z29.b\n" "umax z22.b, p0/M, z22.b, z28.b\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" "umax z17.b, p0/M, z17.b, z27.b\n" "umax z21.b, p0/M, z21.b, z26.b\n" - "ld1b { z0.b }, p4/Z, [x24, x9]\n" + "ld1b { z0.b }, p4/Z, [x23, x9]\n" "umax z16.b, p0/M, z16.b, z25.b\n" "umax z20.b, p0/M, z20.b, z24.b\n" - "ld1b { z31.b }, p4/Z, [x23, x9]\n" + "ld1b { z31.b }, p4/Z, [x22, x9]\n" "umax z19.b, p0/M, z19.b, z23.b\n" "umax z18.b, p0/M, z18.b, z22.b\n" - "ld1b { z23.b }, p4/Z, [x22, x9]\n" + "ld1b { z23.b }, p4/Z, [x21, x9]\n" "umax z17.b, p0/M, z17.b, z21.b\n" "umax z16.b, p0/M, z16.b, z20.b\n" - "ld1b { z30.b }, p4/Z, [x21, x9]\n" + "ld1b { z30.b }, p4/Z, [x20, x9]\n" "umax z5.b, p0/M, z5.b, z19.b\n" "umax z3.b, p0/M, z3.b, z18.b\n" - "ld1b { z18.b }, p3/Z, [x24, x28]\n" + "ld1b { z18.b }, p3/Z, [x23, x28]\n" "umax z2.b, p0/M, z2.b, z17.b\n" "umax z1.b, p0/M, z1.b, z16.b\n" - "ld1b { z29.b }, p3/Z, [x23, x28]\n" - "ld1b { z22.b }, p3/Z, [x22, x28]\n" - "ld1b { z28.b }, p3/Z, [x21, x28]\n" - "ld1b { z17.b }, p2/Z, [x24, x27]\n" - "ld1b { z27.b }, p2/Z, [x23, x27]\n" - "ld1b { z21.b }, p2/Z, [x22, x27]\n" - "ld1b { z26.b }, p2/Z, [x21, x27]\n" - "ld1b { z16.b }, p1/Z, [x24, x26]\n" - "ld1b { z25.b }, p1/Z, [x23, x26]\n" - "ld1b { z20.b }, p1/Z, [x22, x26]\n" - "ld1b { z24.b }, p1/Z, [x21, x26]\n" + "ld1b { z29.b }, p3/Z, [x22, x28]\n" + "ld1b { z22.b }, p3/Z, [x21, x28]\n" + "ld1b { z28.b }, p3/Z, [x20, x28]\n" + "ld1b { z17.b }, p2/Z, [x23, x27]\n" + "ld1b { z27.b }, p2/Z, [x22, x27]\n" + "ld1b { z21.b }, p2/Z, [x21, x27]\n" + "ld1b { z26.b }, p2/Z, [x20, x27]\n" + "ld1b { z16.b }, p1/Z, [x23, x26]\n" + "ld1b { z25.b }, p1/Z, [x22, x26]\n" + "ld1b { z20.b }, p1/Z, [x21, x26]\n" + "ld1b { z24.b }, p1/Z, [x20, x26]\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "movprfx z19, z0\n umax z19.b, p0/M, z19.b, z31.b\n" @@ -140,15 +141,15 @@ void sme_u8q_nhwc_max_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1b { z0.b }, p4/Z, [x24, x9]\n" + "ldr x20, [x24], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x9]\n" "subs x21, x21, #0x1\n" - "umax z5.b, p0/M, z5.b, z0.b\n" - "ld1b { z18.b }, p3/Z, [x24, x28]\n" - "umax z3.b, p0/M, z3.b, z18.b\n" - "ld1b { z17.b }, p2/Z, [x24, x27]\n" - "umax z2.b, p0/M, z2.b, z17.b\n" - "ld1b { z16.b }, p1/Z, [x24, x26]\n" + "umax z5.b, p0/M, z5.b, z16.b\n" + "ld1b { z16.b }, p3/Z, [x20, x28]\n" + "umax z3.b, p0/M, z3.b, z16.b\n" + "ld1b { z16.b }, p2/Z, [x20, x27]\n" + "umax z2.b, p0/M, z2.b, z16.b\n" + "ld1b { z16.b }, p1/Z, [x20, x26]\n" "umax z1.b, p0/M, z1.b, z16.b\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End @@ -313,92 +314,92 @@ void sme_u8q_nhwc_max_generic_depthfirst_impl( "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" "mov z5.b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x24, x23, [x20, #0x0]\n" + "ldp x20, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" - "ld1b { z0.b }, p4/Z, [x24, x9]\n" - "ldp x22, x21, [x20, #0x10]\n" - "add x20, x20, #0x20\n" - "ld1b { z31.b }, p4/Z, [x23, x9]\n" - "ld1b { z23.b }, p4/Z, [x22, x9]\n" - "ld1b { z30.b }, p4/Z, [x21, x9]\n" + "ld1b { z0.b }, p4/Z, [x20, x9]\n" + "ldp x21, x20, [x24, #0x10]\n" + "add x24, x24, #0x20\n" + "ld1b { z31.b }, p4/Z, [x22, x9]\n" + "ld1b { z23.b }, p4/Z, [x21, x9]\n" + "ld1b { z30.b }, p4/Z, [x20, x9]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 4 inputs loop - "movprfx z19, z0\n umax z19.b, p0/M, z19.b, z31.b\n" - "umax z23.b, p0/M, z23.b, z30.b\n" - "ldp x24, x23, [x20, #0x0]\n" + "movprfx z16, z0\n umax z16.b, p0/M, z16.b, z31.b\n" + "movprfx z17, z23\n umax z17.b, p0/M, z17.b, z30.b\n" + "ldp x23, x22, [x24, #0x0]\n" "subs x25, x25, #0x1\n" - "umax z19.b, p0/M, z19.b, z23.b\n" - "ldp x22, x21, [x20, #0x10]\n" - "umax z5.b, p0/M, z5.b, z19.b\n" - "add x20, x20, #0x20\n" - "ld1b { z0.b }, p4/Z, [x24, x9]\n" - "ld1b { z31.b }, p4/Z, [x23, x9]\n" - "ld1b { z23.b }, p4/Z, [x22, x9]\n" - "ld1b { z30.b }, p4/Z, [x21, x9]\n" + "umax z16.b, p0/M, z16.b, z17.b\n" + "ldp x21, x20, [x24, #0x10]\n" + "umax z5.b, p0/M, z5.b, z16.b\n" + "add x24, x24, #0x20\n" + "ld1b { z0.b }, p4/Z, [x23, x9]\n" + "ld1b { z31.b }, p4/Z, [x22, x9]\n" + "ld1b { z23.b }, p4/Z, [x21, x9]\n" + "ld1b { z30.b }, p4/Z, [x20, x9]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail - "movprfx z19, z0\n umax z19.b, p0/M, z19.b, z31.b\n" - "umax z23.b, p0/M, z23.b, z30.b\n" - "umax z19.b, p0/M, z19.b, z23.b\n" - "umax z5.b, p0/M, z5.b, z19.b\n" + "movprfx z16, z0\n umax z16.b, p0/M, z16.b, z31.b\n" + "movprfx z17, z23\n umax z17.b, p0/M, z17.b, z30.b\n" + "umax z16.b, p0/M, z16.b, z17.b\n" + "umax z5.b, p0/M, z5.b, z16.b\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1b { z0.b }, p4/Z, [x24, x9]\n" + "ldr x20, [x24], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x9]\n" "subs x21, x21, #0x1\n" - "umax z5.b, p0/M, z5.b, z0.b\n" + "umax z5.b, p0/M, z5.b, z16.b\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "add x20, %x[quant_params], %[offsetof_qp_input_offset]\n" - "ld1rw { z4.s }, p0/Z, [x20]\n" - ".inst 0x4508a8b7 // ushllb z23.h, z5.b, #0x0\n" - ".inst 0x4508acb9 // ushllt z25.h, z5.b, #0x0\n" - "neg z4.s, p0/M, z4.s\n" - ".inst 0x45974081 // saddwb z1.s, z4.s, z23.h\n" + "ld1rw { z18.s }, p0/Z, [x20]\n" + ".inst 0x4508a8b1 // ushllb z17.h, z5.b, #0x0\n" + ".inst 0x4508acb0 // ushllt z16.h, z5.b, #0x0\n" + "neg z18.s, p0/M, z18.s\n" + ".inst 0x45914257 // saddwb z23.s, z18.s, z17.h\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n" - "ld1rw { z3.s }, p0/Z, [x20]\n" - ".inst 0x45974497 // saddwt z23.s, z4.s, z23.h\n" - ".inst 0x45994080 // saddwb z0.s, z4.s, z25.h\n" + "ld1rw { z22.s }, p0/Z, [x20]\n" + ".inst 0x45914655 // saddwt z21.s, z18.s, z17.h\n" + ".inst 0x45904254 // saddwb z20.s, z18.s, z16.h\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_mul]\n" - "ld1rw { z2.s }, p0/Z, [x20]\n" - ".inst 0x4599449f // saddwt z31.s, z4.s, z25.h\n" - ".inst 0x44828061 // srshl z1.s, p0/M, z1.s, z3.s\n" - "add x20, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n" "ld1rw { z19.s }, p0/Z, [x20]\n" - ".inst 0x44828077 // srshl z23.s, p0/M, z23.s, z3.s\n" - ".inst 0x44828060 // srshl z0.s, p0/M, z0.s, z3.s\n" + ".inst 0x45904652 // saddwt z18.s, z18.s, z16.h\n" + ".inst 0x448282d7 // srshl z23.s, p0/M, z23.s, z22.s\n" + "add x20, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n" + "ld1rw { z17.s }, p0/Z, [x20]\n" + ".inst 0x448282d5 // srshl z21.s, p0/M, z21.s, z22.s\n" + ".inst 0x448282d4 // srshl z20.s, p0/M, z20.s, z22.s\n" "add x20, %x[quant_params], %[offsetof_qp_output_offset]\n" "ld1rw { z16.s }, p0/Z, [x20]\n" - ".inst 0x4482807f // srshl z31.s, p0/M, z31.s, z3.s\n" - ".inst 0x04a27421 // sqrdmulh z1.s, z1.s, z2.s\n" - ".inst 0x04a276f7 // sqrdmulh z23.s, z23.s, z2.s\n" - ".inst 0x04a27400 // sqrdmulh z0.s, z0.s, z2.s\n" - ".inst 0x04a277ff // sqrdmulh z31.s, z31.s, z2.s\n" - ".inst 0x44828261 // srshl z1.s, p0/M, z1.s, z19.s\n" - ".inst 0x44828277 // srshl z23.s, p0/M, z23.s, z19.s\n" - ".inst 0x44828260 // srshl z0.s, p0/M, z0.s, z19.s\n" - ".inst 0x4482827f // srshl z31.s, p0/M, z31.s, z19.s\n" - "add z1.s, z1.s, z16.s\n" + ".inst 0x448282d2 // srshl z18.s, p0/M, z18.s, z22.s\n" + ".inst 0x04b376f7 // sqrdmulh z23.s, z23.s, z19.s\n" + ".inst 0x04b376b5 // sqrdmulh z21.s, z21.s, z19.s\n" + ".inst 0x04b37694 // sqrdmulh z20.s, z20.s, z19.s\n" + ".inst 0x04b37652 // sqrdmulh z18.s, z18.s, z19.s\n" + ".inst 0x44828237 // srshl z23.s, p0/M, z23.s, z17.s\n" + ".inst 0x44828235 // srshl z21.s, p0/M, z21.s, z17.s\n" + ".inst 0x44828234 // srshl z20.s, p0/M, z20.s, z17.s\n" + ".inst 0x44828232 // srshl z18.s, p0/M, z18.s, z17.s\n" "add z23.s, z23.s, z16.s\n" - "add z0.s, z0.s, z16.s\n" - "add z31.s, z31.s, z16.s\n" - "mov z16.s, #0x0\n" - "mov z19.s, #0xff\n" - "smax z1.s, p0/M, z1.s, z16.s\n" - "smax z23.s, p0/M, z23.s, z16.s\n" - "smax z0.s, p0/M, z0.s, z16.s\n" - "smax z31.s, p0/M, z31.s, z16.s\n" - "smin z1.s, p0/M, z1.s, z19.s\n" - "smin z23.s, p0/M, z23.s, z19.s\n" - "smin z0.s, p0/M, z0.s, z19.s\n" - "trn1 z23.h, z1.h, z23.h\n" - "smin z31.s, p0/M, z31.s, z19.s\n" - "trn1 z16.h, z0.h, z31.h\n" - "trn1 z16.b, z23.b, z16.b\n" + "add z21.s, z21.s, z16.s\n" + "add z20.s, z20.s, z16.s\n" + "add z18.s, z18.s, z16.s\n" + "mov z17.s, #0x0\n" + "mov z16.s, #0xff\n" + "smax z23.s, p0/M, z23.s, z17.s\n" + "smax z21.s, p0/M, z21.s, z17.s\n" + "smax z20.s, p0/M, z20.s, z17.s\n" + "smax z18.s, p0/M, z18.s, z17.s\n" + "smin z23.s, p0/M, z23.s, z16.s\n" + "smin z21.s, p0/M, z21.s, z16.s\n" + "smin z20.s, p0/M, z20.s, z16.s\n" + "trn1 z17.h, z23.h, z21.h\n" + "smin z18.s, p0/M, z18.s, z16.s\n" + "trn1 z16.h, z20.h, z18.h\n" + "trn1 z16.b, z17.b, z16.b\n" "st1b { z16.b }, p4, [%x[outptr], x9]\n" "incb x9\n" "whilelt p4.b, x9, %x[n_channels]\n" diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp index 593fb584453775190addad60b574030fa5cd52a8..1ba78f3fbaa48881c722698b1b0cf7f1cc514755 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp @@ -88,8 +88,8 @@ void sve_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst_impl( "mov x20, #0x4\n" "ldr x4, [%x[args], %[offsetof_inptrs]]\n" "ldp x5, x6, [x21, #0x0]\n" - "whilelt p0.h, XZR, x20\n" - "whilelt p1.h, x3, x2\n" + "whilelt p2.h, XZR, x20\n" + "whilelt p0.h, x3, x2\n" "ldp x7, x8, [x21, #0x10]\n" "ldp x17, x16, [x4, #0x0]\n" "add x15, %x[args], %[offsetof_rescale]\n" @@ -101,25 +101,25 @@ void sve_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst_impl( "ldp x25, x24, [x4, #0x50]\n" "ldp x23, x22, [x4, #0x60]\n" "ldp x21, x20, [x4, #0x70]\n" - "ld1h { z7.h }, p1/Z, [x10, x3, LSL #1]\n" - "ld1h { z6.h }, p1/Z, [x9, x3, LSL #1]\n" - "ld1h { z5.h }, p1/Z, [x26, x3, LSL #1]\n" - "ld1h { z4.h }, p1/Z, [x25, x3, LSL #1]\n" - "ld1h { z3.h }, p1/Z, [x16, x3, LSL #1]\n" - "ld1h { z2.h }, p1/Z, [x13, x3, LSL #1]\n" - "ld1h { z1.h }, p1/Z, [x11, x3, LSL #1]\n" - "ld1h { z31.h }, p1/Z, [x27, x3, LSL #1]\n" - "ld1h { z30.h }, p1/Z, [x28, x3, LSL #1]\n" - "ld1h { z29.h }, p1/Z, [x24, x3, LSL #1]\n" - "ld1h { z28.h }, p1/Z, [x22, x3, LSL #1]\n" - "ld1h { z27.h }, p1/Z, [x21, x3, LSL #1]\n" - "ld1h { z26.h }, p1/Z, [x17, x3, LSL #1]\n" - "ld1h { z25.h }, p1/Z, [x12, x3, LSL #1]\n" - "ld1h { z24.h }, p1/Z, [x23, x3, LSL #1]\n" - "ld1h { z23.h }, p1/Z, [x20, x3, LSL #1]\n" + "ld1h { z7.h }, p0/Z, [x10, x3, LSL #1]\n" + "ld1h { z6.h }, p0/Z, [x9, x3, LSL #1]\n" + "ld1h { z5.h }, p0/Z, [x26, x3, LSL #1]\n" + "ld1h { z4.h }, p0/Z, [x25, x3, LSL #1]\n" + "ld1h { z3.h }, p0/Z, [x16, x3, LSL #1]\n" + "ld1h { z2.h }, p0/Z, [x13, x3, LSL #1]\n" + "ld1h { z1.h }, p0/Z, [x11, x3, LSL #1]\n" + "ld1h { z31.h }, p0/Z, [x27, x3, LSL #1]\n" + "ld1h { z30.h }, p0/Z, [x28, x3, LSL #1]\n" + "ld1h { z29.h }, p0/Z, [x24, x3, LSL #1]\n" + "ld1h { z28.h }, p0/Z, [x22, x3, LSL #1]\n" + "ld1h { z27.h }, p0/Z, [x21, x3, LSL #1]\n" + "ld1h { z26.h }, p0/Z, [x17, x3, LSL #1]\n" + "ld1h { z25.h }, p0/Z, [x12, x3, LSL #1]\n" + "ld1h { z24.h }, p0/Z, [x23, x3, LSL #1]\n" + "ld1h { z23.h }, p0/Z, [x20, x3, LSL #1]\n" "incw x3\n" "whilelt p1.h, x3, x2\n" - "ld1rqh { z0.h }, p0/Z, [x15]\n" + "ld1rqh { z0.h }, p2/Z, [x15]\n" "b.none 2f\n" "1:" // Vector: Loop "fadd z17.h, z7.h, z6.h\n" @@ -172,32 +172,32 @@ void sve_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst_impl( "fadd z17.h, z7.h, z6.h\n" "fadd z16.h, z5.h, z4.h\n" "whilelt p0.h, x14, x2\n" - "fadd z19.h, z17.h, z16.h\n" + "fadd z20.h, z17.h, z16.h\n" "fadd z18.h, z3.h, z2.h\n" "fadd z17.h, z1.h, z31.h\n" - "fadd z22.h, z30.h, z29.h\n" + "fadd z19.h, z30.h, z29.h\n" "fadd z16.h, z28.h, z27.h\n" - "fadd z21.h, z18.h, z19.h\n" - "fadd z20.h, z16.h, z19.h\n" - "fadd z19.h, z26.h, z17.h\n" - "fadd z18.h, z25.h, z22.h\n" + "fadd z21.h, z18.h, z20.h\n" + "fadd z20.h, z16.h, z20.h\n" + "fadd z16.h, z26.h, z17.h\n" + "fadd z18.h, z25.h, z19.h\n" "fadd z17.h, z24.h, z17.h\n" - "fadd z16.h, z23.h, z22.h\n" - "fadd z19.h, z21.h, z19.h\n" - "fmul z19.h, z19.h, z0.h[0]\n" - "st1h { z19.h }, p0, [x5, x14, LSL #1]\n" + "fadd z19.h, z23.h, z19.h\n" + "fadd z16.h, z21.h, z16.h\n" + "fmul z16.h, z16.h, z0.h[0]\n" + "st1h { z16.h }, p0, [x5, x14, LSL #1]\n" "fadd z18.h, z21.h, z18.h\n" "fadd z17.h, z17.h, z20.h\n" "fmul z18.h, z18.h, z0.h[1]\n" "fmul z17.h, z17.h, z0.h[2]\n" - "fadd z16.h, z16.h, z20.h\n" + "fadd z16.h, z19.h, z20.h\n" "fmul z16.h, z16.h, z0.h[3]\n" "st1h { z18.h }, p0, [x6, x14, LSL #1]\n" "st1h { z17.h }, p0, [x7, x14, LSL #1]\n" "st1h { z16.h }, p0, [x8, x14, LSL #1]\n" : : [args] "r" (&args), [offsetof_inptrs] "I" (offsetof(KernelArgs, inptrs)), [offsetof_n_channels] "I" (offsetof(KernelArgs, n_channels)), [offsetof_outptrs] "I" (offsetof(KernelArgs, outptrs)), [offsetof_rescale] "I" (offsetof(KernelArgs, rescale_vals)) - : "cc", "memory", "p0", "p1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_avg_generic_depthfirst/generic.cpp index 594c65e18db426db14b09d61bc1437e04996dee8..2bef44ea5c23e78f03982e135ce83040cde3fa8c 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_avg_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_avg_generic_depthfirst/generic.cpp @@ -57,68 +57,68 @@ void sve_fp16_nhwc_avg_generic_depthfirst_impl( "lsr x25, %x[n_valid_cells], #0x2\n" "mov z6.b, #0x0\n" "mov z5.b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "mov z4.b, #0x0\n" "mov z3.b, #0x0\n" "cbz x25, 4f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" "subs x25, x25, #0x1\n" - "add x20, x20, #0x20\n" - "ld1h { z2.h }, p3/Z, [x24, x9, LSL #1]\n" - "ld1h { z1.h }, p3/Z, [x23, x9, LSL #1]\n" - "ld1h { z0.h }, p3/Z, [x22, x9, LSL #1]\n" - "ld1h { z31.h }, p3/Z, [x21, x9, LSL #1]\n" - "ld1h { z30.h }, p2/Z, [x24, x28, LSL #1]\n" - "ld1h { z22.h }, p2/Z, [x23, x28, LSL #1]\n" - "ld1h { z29.h }, p2/Z, [x22, x28, LSL #1]\n" - "ld1h { z28.h }, p2/Z, [x21, x28, LSL #1]\n" - "ld1h { z27.h }, p1/Z, [x24, x27, LSL #1]\n" - "ld1h { z21.h }, p1/Z, [x23, x27, LSL #1]\n" - "ld1h { z26.h }, p1/Z, [x22, x27, LSL #1]\n" - "ld1h { z17.h }, p1/Z, [x21, x27, LSL #1]\n" - "ld1h { z25.h }, p0/Z, [x24, x26, LSL #1]\n" - "ld1h { z20.h }, p0/Z, [x23, x26, LSL #1]\n" - "ld1h { z24.h }, p0/Z, [x22, x26, LSL #1]\n" - "ld1h { z16.h }, p0/Z, [x21, x26, LSL #1]\n" + "add x24, x24, #0x20\n" + "ld1h { z2.h }, p3/Z, [x23, x9, LSL #1]\n" + "ld1h { z1.h }, p3/Z, [x22, x9, LSL #1]\n" + "ld1h { z0.h }, p3/Z, [x21, x9, LSL #1]\n" + "ld1h { z31.h }, p3/Z, [x20, x9, LSL #1]\n" + "ld1h { z30.h }, p2/Z, [x23, x28, LSL #1]\n" + "ld1h { z22.h }, p2/Z, [x22, x28, LSL #1]\n" + "ld1h { z29.h }, p2/Z, [x21, x28, LSL #1]\n" + "ld1h { z28.h }, p2/Z, [x20, x28, LSL #1]\n" + "ld1h { z27.h }, p1/Z, [x23, x27, LSL #1]\n" + "ld1h { z21.h }, p1/Z, [x22, x27, LSL #1]\n" + "ld1h { z26.h }, p1/Z, [x21, x27, LSL #1]\n" + "ld1h { z17.h }, p1/Z, [x20, x27, LSL #1]\n" + "ld1h { z25.h }, p0/Z, [x23, x26, LSL #1]\n" + "ld1h { z20.h }, p0/Z, [x22, x26, LSL #1]\n" + "ld1h { z24.h }, p0/Z, [x21, x26, LSL #1]\n" + "ld1h { z16.h }, p0/Z, [x20, x26, LSL #1]\n" "beq 3f\n" "2:" // 4-vectors of channels: 4 inputs loop "fadd z23.h, z2.h, z1.h\n" "fadd z19.h, z0.h, z31.h\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" "fadd z22.h, z30.h, z22.h\n" "fadd z18.h, z29.h, z28.h\n" "subs x25, x25, #0x1\n" - "add x20, x20, #0x20\n" + "add x24, x24, #0x20\n" "fadd z21.h, z27.h, z21.h\n" "fadd z17.h, z26.h, z17.h\n" - "ld1h { z2.h }, p3/Z, [x24, x9, LSL #1]\n" - "ld1h { z1.h }, p3/Z, [x23, x9, LSL #1]\n" + "ld1h { z2.h }, p3/Z, [x23, x9, LSL #1]\n" + "ld1h { z1.h }, p3/Z, [x22, x9, LSL #1]\n" "fadd z20.h, z25.h, z20.h\n" "fadd z16.h, z24.h, z16.h\n" - "ld1h { z0.h }, p3/Z, [x22, x9, LSL #1]\n" - "ld1h { z31.h }, p3/Z, [x21, x9, LSL #1]\n" + "ld1h { z0.h }, p3/Z, [x21, x9, LSL #1]\n" + "ld1h { z31.h }, p3/Z, [x20, x9, LSL #1]\n" "fadd z19.h, z23.h, z19.h\n" "fadd z18.h, z22.h, z18.h\n" - "ld1h { z30.h }, p2/Z, [x24, x28, LSL #1]\n" - "ld1h { z22.h }, p2/Z, [x23, x28, LSL #1]\n" + "ld1h { z30.h }, p2/Z, [x23, x28, LSL #1]\n" + "ld1h { z22.h }, p2/Z, [x22, x28, LSL #1]\n" "fadd z17.h, z21.h, z17.h\n" "fadd z16.h, z20.h, z16.h\n" - "ld1h { z29.h }, p2/Z, [x22, x28, LSL #1]\n" - "ld1h { z28.h }, p2/Z, [x21, x28, LSL #1]\n" + "ld1h { z29.h }, p2/Z, [x21, x28, LSL #1]\n" + "ld1h { z28.h }, p2/Z, [x20, x28, LSL #1]\n" "fadd z6.h, z6.h, z19.h\n" "fadd z5.h, z5.h, z18.h\n" - "ld1h { z27.h }, p1/Z, [x24, x27, LSL #1]\n" - "ld1h { z21.h }, p1/Z, [x23, x27, LSL #1]\n" + "ld1h { z27.h }, p1/Z, [x23, x27, LSL #1]\n" + "ld1h { z21.h }, p1/Z, [x22, x27, LSL #1]\n" "fadd z4.h, z4.h, z17.h\n" "fadd z3.h, z3.h, z16.h\n" - "ld1h { z26.h }, p1/Z, [x22, x27, LSL #1]\n" - "ld1h { z17.h }, p1/Z, [x21, x27, LSL #1]\n" - "ld1h { z25.h }, p0/Z, [x24, x26, LSL #1]\n" - "ld1h { z20.h }, p0/Z, [x23, x26, LSL #1]\n" - "ld1h { z24.h }, p0/Z, [x22, x26, LSL #1]\n" - "ld1h { z16.h }, p0/Z, [x21, x26, LSL #1]\n" + "ld1h { z26.h }, p1/Z, [x21, x27, LSL #1]\n" + "ld1h { z17.h }, p1/Z, [x20, x27, LSL #1]\n" + "ld1h { z25.h }, p0/Z, [x23, x26, LSL #1]\n" + "ld1h { z20.h }, p0/Z, [x22, x26, LSL #1]\n" + "ld1h { z24.h }, p0/Z, [x21, x26, LSL #1]\n" + "ld1h { z16.h }, p0/Z, [x20, x26, LSL #1]\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "fadd z23.h, z2.h, z1.h\n" @@ -141,16 +141,16 @@ void sve_fp16_nhwc_avg_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1h { z2.h }, p3/Z, [x24, x9, LSL #1]\n" + "ldr x20, [x24], #0x8\n" + "ld1h { z16.h }, p3/Z, [x20, x9, LSL #1]\n" "subs x21, x21, #0x1\n" - "fadd z6.h, z6.h, z2.h\n" - "ld1h { z30.h }, p2/Z, [x24, x28, LSL #1]\n" - "ld1h { z27.h }, p1/Z, [x24, x27, LSL #1]\n" - "fadd z5.h, z5.h, z30.h\n" - "fadd z4.h, z4.h, z27.h\n" - "ld1h { z25.h }, p0/Z, [x24, x26, LSL #1]\n" - "fadd z3.h, z3.h, z25.h\n" + "fadd z6.h, z6.h, z16.h\n" + "ld1h { z17.h }, p2/Z, [x20, x28, LSL #1]\n" + "ld1h { z16.h }, p1/Z, [x20, x27, LSL #1]\n" + "fadd z5.h, z5.h, z17.h\n" + "fadd z4.h, z4.h, z16.h\n" + "ld1h { z16.h }, p0/Z, [x20, x26, LSL #1]\n" + "fadd z3.h, z3.h, z16.h\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End "fmul z6.h, z6.h, z7.h\n" @@ -173,44 +173,44 @@ void sve_fp16_nhwc_avg_generic_depthfirst_impl( "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" "mov z6.b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" "subs x25, x25, #0x1\n" - "add x20, x20, #0x20\n" - "ld1h { z2.h }, p3/Z, [x24, x9, LSL #1]\n" - "ld1h { z1.h }, p3/Z, [x23, x9, LSL #1]\n" - "ld1h { z0.h }, p3/Z, [x22, x9, LSL #1]\n" - "ld1h { z31.h }, p3/Z, [x21, x9, LSL #1]\n" + "add x24, x24, #0x20\n" + "ld1h { z2.h }, p3/Z, [x23, x9, LSL #1]\n" + "ld1h { z1.h }, p3/Z, [x22, x9, LSL #1]\n" + "ld1h { z0.h }, p3/Z, [x21, x9, LSL #1]\n" + "ld1h { z31.h }, p3/Z, [x20, x9, LSL #1]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 4 inputs loop - "fadd z23.h, z2.h, z1.h\n" - "fadd z19.h, z0.h, z31.h\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" - "fadd z19.h, z23.h, z19.h\n" + "fadd z17.h, z2.h, z1.h\n" + "fadd z16.h, z0.h, z31.h\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" + "fadd z16.h, z17.h, z16.h\n" "subs x25, x25, #0x1\n" - "fadd z6.h, z6.h, z19.h\n" - "add x20, x20, #0x20\n" - "ld1h { z2.h }, p3/Z, [x24, x9, LSL #1]\n" - "ld1h { z1.h }, p3/Z, [x23, x9, LSL #1]\n" - "ld1h { z0.h }, p3/Z, [x22, x9, LSL #1]\n" - "ld1h { z31.h }, p3/Z, [x21, x9, LSL #1]\n" + "fadd z6.h, z6.h, z16.h\n" + "add x24, x24, #0x20\n" + "ld1h { z2.h }, p3/Z, [x23, x9, LSL #1]\n" + "ld1h { z1.h }, p3/Z, [x22, x9, LSL #1]\n" + "ld1h { z0.h }, p3/Z, [x21, x9, LSL #1]\n" + "ld1h { z31.h }, p3/Z, [x20, x9, LSL #1]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail - "fadd z23.h, z2.h, z1.h\n" - "fadd z19.h, z0.h, z31.h\n" - "fadd z19.h, z23.h, z19.h\n" - "fadd z6.h, z6.h, z19.h\n" + "fadd z17.h, z2.h, z1.h\n" + "fadd z16.h, z0.h, z31.h\n" + "fadd z16.h, z17.h, z16.h\n" + "fadd z6.h, z6.h, z16.h\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1h { z2.h }, p3/Z, [x24, x9, LSL #1]\n" + "ldr x20, [x24], #0x8\n" + "ld1h { z16.h }, p3/Z, [x20, x9, LSL #1]\n" "subs x21, x21, #0x1\n" - "fadd z6.h, z6.h, z2.h\n" + "fadd z6.h, z6.h, z16.h\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "fmul z6.h, z6.h, z7.h\n" diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp index 838cd3406c5189e578d10cb6b5e3cfe07c7acb1c..31bbfd085ecb8aef473dc68a4e02c66ee11c0306 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp @@ -66,10 +66,10 @@ void sve_fp16_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "ldr x15, [%x[args], %[offsetof_n_channels]]\n" "ldr x21, [%x[args], %[offsetof_outptrs]]\n" "mov x14, #0x0\n" - "whilelt p2.h, x14, x15\n" + "whilelt p0.h, x14, x15\n" "ldr x20, [%x[args], %[offsetof_inptrs]]\n" "ldp x13, x12, [x21, #0x0]\n" - "ptrue p1.b\n" + "ptrue p2.b\n" "mov x11, #0x0\n" "ldp x10, x9, [x21, #0x10]\n" "ldp x28, x27, [x20, #0x0]\n" @@ -77,61 +77,61 @@ void sve_fp16_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "ldp x24, x23, [x20, #0x20]\n" "ldp x22, x21, [x20, #0x30]\n" "ldr x20, [x20, #0x40]\n" - "ld1h { z31.h }, p2/Z, [x27, x14, LSL #1]\n" - "ld1h { z30.h }, p2/Z, [x24, x14, LSL #1]\n" - "ld1h { z29.h }, p2/Z, [x21, x14, LSL #1]\n" - "ld1h { z28.h }, p2/Z, [x25, x14, LSL #1]\n" - "ld1h { z27.h }, p2/Z, [x28, x14, LSL #1]\n" - "ld1h { z26.h }, p2/Z, [x26, x14, LSL #1]\n" - "ld1h { z25.h }, p2/Z, [x23, x14, LSL #1]\n" - "ld1h { z24.h }, p2/Z, [x22, x14, LSL #1]\n" - "ld1h { z23.h }, p2/Z, [x20, x14, LSL #1]\n" + "ld1h { z31.h }, p0/Z, [x27, x14, LSL #1]\n" + "ld1h { z30.h }, p0/Z, [x24, x14, LSL #1]\n" + "ld1h { z29.h }, p0/Z, [x21, x14, LSL #1]\n" + "ld1h { z28.h }, p0/Z, [x25, x14, LSL #1]\n" + "ld1h { z27.h }, p0/Z, [x28, x14, LSL #1]\n" + "ld1h { z26.h }, p0/Z, [x26, x14, LSL #1]\n" + "ld1h { z25.h }, p0/Z, [x23, x14, LSL #1]\n" + "ld1h { z24.h }, p0/Z, [x22, x14, LSL #1]\n" + "ld1h { z23.h }, p0/Z, [x20, x14, LSL #1]\n" "incw x14\n" - "whilelt p2.h, x14, x15\n" + "whilelt p1.h, x14, x15\n" "b.none 2f\n" "1:" // Vector: Loop - "movprfx z22, z31\n fmax z22.h, p1/M, z22.h, z30.h\n" - "movprfx z21, z30\n fmax z21.h, p1/M, z21.h, z29.h\n" - "ld1h { z31.h }, p2/Z, [x27, x14, LSL #1]\n" - "ld1h { z30.h }, p2/Z, [x24, x14, LSL #1]\n" - "movprfx z20, z28\n fmax z20.h, p1/M, z20.h, z27.h\n" - "movprfx z19, z26\n fmax z19.h, p1/M, z19.h, z25.h\n" - "ld1h { z29.h }, p2/Z, [x21, x14, LSL #1]\n" - "ld1h { z27.h }, p2/Z, [x28, x14, LSL #1]\n" - "movprfx z17, z28\n fmax z17.h, p1/M, z17.h, z24.h\n" - "movprfx z18, z25\n fmax z18.h, p1/M, z18.h, z23.h\n" - "ld1h { z28.h }, p2/Z, [x25, x14, LSL #1]\n" - "ld1h { z26.h }, p2/Z, [x26, x14, LSL #1]\n" - "ld1h { z25.h }, p2/Z, [x23, x14, LSL #1]\n" - "ld1h { z24.h }, p2/Z, [x22, x14, LSL #1]\n" + "movprfx z22, z31\n fmax z22.h, p2/M, z22.h, z30.h\n" + "movprfx z21, z30\n fmax z21.h, p2/M, z21.h, z29.h\n" + "ld1h { z31.h }, p1/Z, [x27, x14, LSL #1]\n" + "ld1h { z30.h }, p1/Z, [x24, x14, LSL #1]\n" + "movprfx z20, z28\n fmax z20.h, p2/M, z20.h, z27.h\n" + "movprfx z19, z26\n fmax z19.h, p2/M, z19.h, z25.h\n" + "ld1h { z29.h }, p1/Z, [x21, x14, LSL #1]\n" + "ld1h { z27.h }, p1/Z, [x28, x14, LSL #1]\n" + "movprfx z17, z28\n fmax z17.h, p2/M, z17.h, z24.h\n" + "movprfx z18, z25\n fmax z18.h, p2/M, z18.h, z23.h\n" + "ld1h { z28.h }, p1/Z, [x25, x14, LSL #1]\n" + "ld1h { z26.h }, p1/Z, [x26, x14, LSL #1]\n" + "ld1h { z25.h }, p1/Z, [x23, x14, LSL #1]\n" + "ld1h { z24.h }, p1/Z, [x22, x14, LSL #1]\n" "whilelt p0.h, x11, x15\n" - "movprfx z16, z22\n fmax z16.h, p1/M, z16.h, z20.h\n" - "ld1h { z23.h }, p2/Z, [x20, x14, LSL #1]\n" + "movprfx z16, z22\n fmax z16.h, p2/M, z16.h, z20.h\n" + "ld1h { z23.h }, p1/Z, [x20, x14, LSL #1]\n" "incw x14\n" - "whilelt p2.h, x14, x15\n" + "whilelt p1.h, x14, x15\n" "st1h { z16.h }, p0, [x13, x11, LSL #1]\n" - "movprfx z16, z19\n fmax z16.h, p1/M, z16.h, z22.h\n" - "fmax z17.h, p1/M, z17.h, z21.h\n" + "movprfx z16, z19\n fmax z16.h, p2/M, z16.h, z22.h\n" + "fmax z17.h, p2/M, z17.h, z21.h\n" "st1h { z16.h }, p0, [x12, x11, LSL #1]\n" - "movprfx z16, z18\n fmax z16.h, p1/M, z16.h, z21.h\n" + "movprfx z16, z21\n fmax z16.h, p2/M, z16.h, z18.h\n" "st1h { z17.h }, p0, [x10, x11, LSL #1]\n" "st1h { z16.h }, p0, [x9, x11, LSL #1]\n" "incw x11\n" "b.any 1b\n" "2:" // Vector: Tail - "movprfx z22, z31\n fmax z22.h, p1/M, z22.h, z30.h\n" - "movprfx z21, z30\n fmax z21.h, p1/M, z21.h, z29.h\n" - "movprfx z20, z28\n fmax z20.h, p1/M, z20.h, z27.h\n" - "movprfx z19, z26\n fmax z19.h, p1/M, z19.h, z25.h\n" - "movprfx z17, z28\n fmax z17.h, p1/M, z17.h, z24.h\n" - "movprfx z18, z25\n fmax z18.h, p1/M, z18.h, z23.h\n" + "movprfx z22, z31\n fmax z22.h, p2/M, z22.h, z30.h\n" + "movprfx z21, z30\n fmax z21.h, p2/M, z21.h, z29.h\n" + "movprfx z20, z28\n fmax z20.h, p2/M, z20.h, z27.h\n" + "movprfx z19, z26\n fmax z19.h, p2/M, z19.h, z25.h\n" + "movprfx z17, z28\n fmax z17.h, p2/M, z17.h, z24.h\n" + "movprfx z18, z25\n fmax z18.h, p2/M, z18.h, z23.h\n" "whilelt p0.h, x11, x15\n" - "movprfx z16, z22\n fmax z16.h, p1/M, z16.h, z20.h\n" + "movprfx z16, z22\n fmax z16.h, p2/M, z16.h, z20.h\n" "st1h { z16.h }, p0, [x13, x11, LSL #1]\n" - "movprfx z16, z19\n fmax z16.h, p1/M, z16.h, z22.h\n" - "fmax z17.h, p1/M, z17.h, z21.h\n" + "movprfx z16, z19\n fmax z16.h, p2/M, z16.h, z22.h\n" + "fmax z17.h, p2/M, z17.h, z21.h\n" "st1h { z16.h }, p0, [x12, x11, LSL #1]\n" - "movprfx z16, z18\n fmax z16.h, p1/M, z16.h, z21.h\n" + "movprfx z16, z21\n fmax z16.h, p2/M, z16.h, z18.h\n" "st1h { z17.h }, p0, [x10, x11, LSL #1]\n" "st1h { z16.h }, p0, [x9, x11, LSL #1]\n" : diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_max_generic_depthfirst/generic.cpp index 9f1f9e73776adcaf351a56eb5a7bad3b4e277c7c..1a01412836abcefc36d8adfa290ccc26617426c0 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_max_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_max_generic_depthfirst/generic.cpp @@ -54,68 +54,68 @@ void sve_fp16_nhwc_max_generic_depthfirst_impl( "lsr x25, %x[n_valid_cells], #0x2\n" "mov z8.h, #0xfc00\n" "mov z7.h, #0xfc00\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "mov z6.h, #0xfc00\n" "mov z5.h, #0xfc00\n" "cbz x25, 4f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" "subs x25, x25, #0x1\n" - "add x20, x20, #0x20\n" - "ld1h { z4.h }, p4/Z, [x24, x9, LSL #1]\n" - "ld1h { z3.h }, p4/Z, [x23, x9, LSL #1]\n" - "ld1h { z2.h }, p4/Z, [x22, x9, LSL #1]\n" - "ld1h { z1.h }, p4/Z, [x21, x9, LSL #1]\n" - "ld1h { z0.h }, p3/Z, [x24, x28, LSL #1]\n" - "ld1h { z31.h }, p3/Z, [x23, x28, LSL #1]\n" - "ld1h { z22.h }, p3/Z, [x22, x28, LSL #1]\n" - "ld1h { z30.h }, p3/Z, [x21, x28, LSL #1]\n" - "ld1h { z29.h }, p2/Z, [x24, x27, LSL #1]\n" - "ld1h { z28.h }, p2/Z, [x23, x27, LSL #1]\n" - "ld1h { z21.h }, p2/Z, [x22, x27, LSL #1]\n" - "ld1h { z27.h }, p2/Z, [x21, x27, LSL #1]\n" - "ld1h { z26.h }, p1/Z, [x24, x26, LSL #1]\n" - "ld1h { z25.h }, p1/Z, [x23, x26, LSL #1]\n" - "ld1h { z20.h }, p1/Z, [x22, x26, LSL #1]\n" - "ld1h { z24.h }, p1/Z, [x21, x26, LSL #1]\n" + "add x24, x24, #0x20\n" + "ld1h { z4.h }, p4/Z, [x23, x9, LSL #1]\n" + "ld1h { z3.h }, p4/Z, [x22, x9, LSL #1]\n" + "ld1h { z2.h }, p4/Z, [x21, x9, LSL #1]\n" + "ld1h { z1.h }, p4/Z, [x20, x9, LSL #1]\n" + "ld1h { z0.h }, p3/Z, [x23, x28, LSL #1]\n" + "ld1h { z31.h }, p3/Z, [x22, x28, LSL #1]\n" + "ld1h { z22.h }, p3/Z, [x21, x28, LSL #1]\n" + "ld1h { z30.h }, p3/Z, [x20, x28, LSL #1]\n" + "ld1h { z29.h }, p2/Z, [x23, x27, LSL #1]\n" + "ld1h { z28.h }, p2/Z, [x22, x27, LSL #1]\n" + "ld1h { z21.h }, p2/Z, [x21, x27, LSL #1]\n" + "ld1h { z27.h }, p2/Z, [x20, x27, LSL #1]\n" + "ld1h { z26.h }, p1/Z, [x23, x26, LSL #1]\n" + "ld1h { z25.h }, p1/Z, [x22, x26, LSL #1]\n" + "ld1h { z20.h }, p1/Z, [x21, x26, LSL #1]\n" + "ld1h { z24.h }, p1/Z, [x20, x26, LSL #1]\n" "beq 3f\n" "2:" // 4-vectors of channels: 4 inputs loop "movprfx z19, z4\n fmax z19.h, p0/M, z19.h, z3.h\n" "movprfx z23, z2\n fmax z23.h, p0/M, z23.h, z1.h\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" "movprfx z18, z0\n fmax z18.h, p0/M, z18.h, z31.h\n" "fmax z22.h, p0/M, z22.h, z30.h\n" - "ld1h { z4.h }, p4/Z, [x24, x9, LSL #1]\n" - "ld1h { z3.h }, p4/Z, [x23, x9, LSL #1]\n" + "ld1h { z4.h }, p4/Z, [x23, x9, LSL #1]\n" + "ld1h { z3.h }, p4/Z, [x22, x9, LSL #1]\n" "movprfx z17, z29\n fmax z17.h, p0/M, z17.h, z28.h\n" "fmax z21.h, p0/M, z21.h, z27.h\n" - "ld1h { z2.h }, p4/Z, [x22, x9, LSL #1]\n" - "ld1h { z1.h }, p4/Z, [x21, x9, LSL #1]\n" + "ld1h { z2.h }, p4/Z, [x21, x9, LSL #1]\n" + "ld1h { z1.h }, p4/Z, [x20, x9, LSL #1]\n" "movprfx z16, z26\n fmax z16.h, p0/M, z16.h, z25.h\n" "fmax z20.h, p0/M, z20.h, z24.h\n" - "ld1h { z0.h }, p3/Z, [x24, x28, LSL #1]\n" - "ld1h { z31.h }, p3/Z, [x23, x28, LSL #1]\n" + "ld1h { z0.h }, p3/Z, [x23, x28, LSL #1]\n" + "ld1h { z31.h }, p3/Z, [x22, x28, LSL #1]\n" "fmax z19.h, p0/M, z19.h, z23.h\n" "fmax z18.h, p0/M, z18.h, z22.h\n" - "ld1h { z22.h }, p3/Z, [x22, x28, LSL #1]\n" - "ld1h { z30.h }, p3/Z, [x21, x28, LSL #1]\n" + "ld1h { z22.h }, p3/Z, [x21, x28, LSL #1]\n" + "ld1h { z30.h }, p3/Z, [x20, x28, LSL #1]\n" "fmax z17.h, p0/M, z17.h, z21.h\n" "fmax z16.h, p0/M, z16.h, z20.h\n" - "ld1h { z29.h }, p2/Z, [x24, x27, LSL #1]\n" - "ld1h { z28.h }, p2/Z, [x23, x27, LSL #1]\n" + "ld1h { z29.h }, p2/Z, [x23, x27, LSL #1]\n" + "ld1h { z28.h }, p2/Z, [x22, x27, LSL #1]\n" "subs x25, x25, #0x1\n" "fmax z8.h, p0/M, z8.h, z19.h\n" - "ld1h { z21.h }, p2/Z, [x22, x27, LSL #1]\n" - "ld1h { z27.h }, p2/Z, [x21, x27, LSL #1]\n" + "ld1h { z21.h }, p2/Z, [x21, x27, LSL #1]\n" + "ld1h { z27.h }, p2/Z, [x20, x27, LSL #1]\n" "fmax z7.h, p0/M, z7.h, z18.h\n" "fmax z6.h, p0/M, z6.h, z17.h\n" - "ld1h { z26.h }, p1/Z, [x24, x26, LSL #1]\n" - "ld1h { z25.h }, p1/Z, [x23, x26, LSL #1]\n" + "ld1h { z26.h }, p1/Z, [x23, x26, LSL #1]\n" + "ld1h { z25.h }, p1/Z, [x22, x26, LSL #1]\n" "fmax z5.h, p0/M, z5.h, z16.h\n" - "add x20, x20, #0x20\n" - "ld1h { z20.h }, p1/Z, [x22, x26, LSL #1]\n" - "ld1h { z24.h }, p1/Z, [x21, x26, LSL #1]\n" + "add x24, x24, #0x20\n" + "ld1h { z20.h }, p1/Z, [x21, x26, LSL #1]\n" + "ld1h { z24.h }, p1/Z, [x20, x26, LSL #1]\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "movprfx z19, z4\n fmax z19.h, p0/M, z19.h, z3.h\n" @@ -138,16 +138,16 @@ void sve_fp16_nhwc_max_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1h { z4.h }, p4/Z, [x24, x9, LSL #1]\n" + "ldr x20, [x24], #0x8\n" + "ld1h { z16.h }, p4/Z, [x20, x9, LSL #1]\n" "subs x21, x21, #0x1\n" - "fmax z8.h, p0/M, z8.h, z4.h\n" - "ld1h { z0.h }, p3/Z, [x24, x28, LSL #1]\n" - "ld1h { z29.h }, p2/Z, [x24, x27, LSL #1]\n" - "fmax z7.h, p0/M, z7.h, z0.h\n" - "fmax z6.h, p0/M, z6.h, z29.h\n" - "ld1h { z26.h }, p1/Z, [x24, x26, LSL #1]\n" - "fmax z5.h, p0/M, z5.h, z26.h\n" + "fmax z8.h, p0/M, z8.h, z16.h\n" + "ld1h { z17.h }, p3/Z, [x20, x28, LSL #1]\n" + "ld1h { z16.h }, p2/Z, [x20, x27, LSL #1]\n" + "fmax z7.h, p0/M, z7.h, z17.h\n" + "fmax z6.h, p0/M, z6.h, z16.h\n" + "ld1h { z16.h }, p1/Z, [x20, x26, LSL #1]\n" + "fmax z5.h, p0/M, z5.h, z16.h\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End "st1h { z8.h }, p4, [%x[outptr], x9, LSL #1]\n" @@ -166,44 +166,44 @@ void sve_fp16_nhwc_max_generic_depthfirst_impl( "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" "mov z8.h, #0xfc00\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" "subs x25, x25, #0x1\n" - "add x20, x20, #0x20\n" - "ld1h { z4.h }, p4/Z, [x24, x9, LSL #1]\n" - "ld1h { z3.h }, p4/Z, [x23, x9, LSL #1]\n" - "ld1h { z2.h }, p4/Z, [x22, x9, LSL #1]\n" - "ld1h { z1.h }, p4/Z, [x21, x9, LSL #1]\n" + "add x24, x24, #0x20\n" + "ld1h { z4.h }, p4/Z, [x23, x9, LSL #1]\n" + "ld1h { z3.h }, p4/Z, [x22, x9, LSL #1]\n" + "ld1h { z2.h }, p4/Z, [x21, x9, LSL #1]\n" + "ld1h { z1.h }, p4/Z, [x20, x9, LSL #1]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 4 inputs loop - "movprfx z19, z4\n fmax z19.h, p0/M, z19.h, z3.h\n" - "movprfx z23, z2\n fmax z23.h, p0/M, z23.h, z1.h\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" - "fmax z19.h, p0/M, z19.h, z23.h\n" + "movprfx z16, z4\n fmax z16.h, p0/M, z16.h, z3.h\n" + "movprfx z17, z2\n fmax z17.h, p0/M, z17.h, z1.h\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" + "fmax z16.h, p0/M, z16.h, z17.h\n" "subs x25, x25, #0x1\n" - "ld1h { z4.h }, p4/Z, [x24, x9, LSL #1]\n" - "ld1h { z3.h }, p4/Z, [x23, x9, LSL #1]\n" - "fmax z8.h, p0/M, z8.h, z19.h\n" - "add x20, x20, #0x20\n" - "ld1h { z2.h }, p4/Z, [x22, x9, LSL #1]\n" - "ld1h { z1.h }, p4/Z, [x21, x9, LSL #1]\n" + "ld1h { z4.h }, p4/Z, [x23, x9, LSL #1]\n" + "ld1h { z3.h }, p4/Z, [x22, x9, LSL #1]\n" + "fmax z8.h, p0/M, z8.h, z16.h\n" + "add x24, x24, #0x20\n" + "ld1h { z2.h }, p4/Z, [x21, x9, LSL #1]\n" + "ld1h { z1.h }, p4/Z, [x20, x9, LSL #1]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail - "movprfx z19, z4\n fmax z19.h, p0/M, z19.h, z3.h\n" - "movprfx z23, z2\n fmax z23.h, p0/M, z23.h, z1.h\n" - "fmax z19.h, p0/M, z19.h, z23.h\n" - "fmax z8.h, p0/M, z8.h, z19.h\n" + "movprfx z16, z4\n fmax z16.h, p0/M, z16.h, z3.h\n" + "movprfx z17, z2\n fmax z17.h, p0/M, z17.h, z1.h\n" + "fmax z16.h, p0/M, z16.h, z17.h\n" + "fmax z8.h, p0/M, z8.h, z16.h\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1h { z4.h }, p4/Z, [x24, x9, LSL #1]\n" + "ldr x20, [x24], #0x8\n" + "ld1h { z16.h }, p4/Z, [x20, x9, LSL #1]\n" "subs x21, x21, #0x1\n" - "fmax z8.h, p0/M, z8.h, z4.h\n" + "fmax z8.h, p0/M, z8.h, z16.h\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "st1h { z8.h }, p4, [%x[outptr], x9, LSL #1]\n" diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp index 39197aa04de5fd503f9579f3fa17d1702709d50a..c5ea5adea01e80bf82a3fae8c3a6680982e01543 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp @@ -88,8 +88,8 @@ void sve_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst_impl( "mov x20, #0x4\n" "ldr x4, [%x[args], %[offsetof_inptrs]]\n" "ldp x5, x6, [x21, #0x0]\n" - "whilelt p0.s, XZR, x20\n" - "whilelt p1.s, x3, x2\n" + "whilelt p2.s, XZR, x20\n" + "whilelt p0.s, x3, x2\n" "ldp x7, x8, [x21, #0x10]\n" "ldp x17, x16, [x4, #0x0]\n" "add x15, %x[args], %[offsetof_rescale]\n" @@ -101,25 +101,25 @@ void sve_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst_impl( "ldp x25, x24, [x4, #0x50]\n" "ldp x23, x22, [x4, #0x60]\n" "ldp x21, x20, [x4, #0x70]\n" - "ld1w { z7.s }, p1/Z, [x10, x3, LSL #2]\n" - "ld1w { z6.s }, p1/Z, [x9, x3, LSL #2]\n" - "ld1w { z5.s }, p1/Z, [x26, x3, LSL #2]\n" - "ld1w { z4.s }, p1/Z, [x25, x3, LSL #2]\n" - "ld1w { z3.s }, p1/Z, [x16, x3, LSL #2]\n" - "ld1w { z2.s }, p1/Z, [x13, x3, LSL #2]\n" - "ld1w { z1.s }, p1/Z, [x11, x3, LSL #2]\n" - "ld1w { z31.s }, p1/Z, [x27, x3, LSL #2]\n" - "ld1w { z30.s }, p1/Z, [x28, x3, LSL #2]\n" - "ld1w { z29.s }, p1/Z, [x24, x3, LSL #2]\n" - "ld1w { z28.s }, p1/Z, [x22, x3, LSL #2]\n" - "ld1w { z27.s }, p1/Z, [x21, x3, LSL #2]\n" - "ld1w { z26.s }, p1/Z, [x17, x3, LSL #2]\n" - "ld1w { z25.s }, p1/Z, [x12, x3, LSL #2]\n" - "ld1w { z24.s }, p1/Z, [x23, x3, LSL #2]\n" - "ld1w { z23.s }, p1/Z, [x20, x3, LSL #2]\n" + "ld1w { z7.s }, p0/Z, [x10, x3, LSL #2]\n" + "ld1w { z6.s }, p0/Z, [x9, x3, LSL #2]\n" + "ld1w { z5.s }, p0/Z, [x26, x3, LSL #2]\n" + "ld1w { z4.s }, p0/Z, [x25, x3, LSL #2]\n" + "ld1w { z3.s }, p0/Z, [x16, x3, LSL #2]\n" + "ld1w { z2.s }, p0/Z, [x13, x3, LSL #2]\n" + "ld1w { z1.s }, p0/Z, [x11, x3, LSL #2]\n" + "ld1w { z31.s }, p0/Z, [x27, x3, LSL #2]\n" + "ld1w { z30.s }, p0/Z, [x28, x3, LSL #2]\n" + "ld1w { z29.s }, p0/Z, [x24, x3, LSL #2]\n" + "ld1w { z28.s }, p0/Z, [x22, x3, LSL #2]\n" + "ld1w { z27.s }, p0/Z, [x21, x3, LSL #2]\n" + "ld1w { z26.s }, p0/Z, [x17, x3, LSL #2]\n" + "ld1w { z25.s }, p0/Z, [x12, x3, LSL #2]\n" + "ld1w { z24.s }, p0/Z, [x23, x3, LSL #2]\n" + "ld1w { z23.s }, p0/Z, [x20, x3, LSL #2]\n" "incw x3\n" "whilelt p1.s, x3, x2\n" - "ld1rqw { z0.s }, p0/Z, [x15]\n" + "ld1rqw { z0.s }, p2/Z, [x15]\n" "b.none 2f\n" "1:" // Vector: Loop "fadd z17.s, z7.s, z6.s\n" @@ -172,32 +172,32 @@ void sve_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst_impl( "fadd z17.s, z7.s, z6.s\n" "fadd z16.s, z5.s, z4.s\n" "whilelt p0.s, x14, x2\n" - "fadd z19.s, z17.s, z16.s\n" + "fadd z20.s, z17.s, z16.s\n" "fadd z18.s, z3.s, z2.s\n" "fadd z17.s, z1.s, z31.s\n" - "fadd z22.s, z30.s, z29.s\n" + "fadd z19.s, z30.s, z29.s\n" "fadd z16.s, z28.s, z27.s\n" - "fadd z21.s, z18.s, z19.s\n" - "fadd z20.s, z16.s, z19.s\n" - "fadd z19.s, z26.s, z17.s\n" - "fadd z18.s, z25.s, z22.s\n" + "fadd z21.s, z18.s, z20.s\n" + "fadd z20.s, z16.s, z20.s\n" + "fadd z16.s, z26.s, z17.s\n" + "fadd z18.s, z25.s, z19.s\n" "fadd z17.s, z24.s, z17.s\n" - "fadd z16.s, z23.s, z22.s\n" - "fadd z19.s, z21.s, z19.s\n" - "fmul z19.s, z19.s, z0.s[0]\n" - "st1w { z19.s }, p0, [x5, x14, LSL #2]\n" + "fadd z19.s, z23.s, z19.s\n" + "fadd z16.s, z21.s, z16.s\n" + "fmul z16.s, z16.s, z0.s[0]\n" + "st1w { z16.s }, p0, [x5, x14, LSL #2]\n" "fadd z18.s, z21.s, z18.s\n" "fadd z17.s, z17.s, z20.s\n" "fmul z18.s, z18.s, z0.s[1]\n" "fmul z17.s, z17.s, z0.s[2]\n" - "fadd z16.s, z16.s, z20.s\n" + "fadd z16.s, z19.s, z20.s\n" "fmul z16.s, z16.s, z0.s[3]\n" "st1w { z18.s }, p0, [x6, x14, LSL #2]\n" "st1w { z17.s }, p0, [x7, x14, LSL #2]\n" "st1w { z16.s }, p0, [x8, x14, LSL #2]\n" : : [args] "r" (&args), [offsetof_inptrs] "I" (offsetof(KernelArgs, inptrs)), [offsetof_n_channels] "I" (offsetof(KernelArgs, n_channels)), [offsetof_outptrs] "I" (offsetof(KernelArgs, outptrs)), [offsetof_rescale] "I" (offsetof(KernelArgs, rescale_vals)) - : "cc", "memory", "p0", "p1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_avg_generic_depthfirst/generic.cpp index c1a3e5de84d216c0d5c02c547079fa2c8e8cd5e6..7c9489489222481ea40af8438c7da19cf80036eb 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_avg_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_avg_generic_depthfirst/generic.cpp @@ -57,68 +57,68 @@ void sve_fp32_nhwc_avg_generic_depthfirst_impl( "lsr x25, %x[n_valid_cells], #0x2\n" "mov z6.b, #0x0\n" "mov z5.b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "mov z4.b, #0x0\n" "mov z3.b, #0x0\n" "cbz x25, 4f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" "subs x25, x25, #0x1\n" - "add x20, x20, #0x20\n" - "ld1w { z2.s }, p3/Z, [x24, x9, LSL #2]\n" - "ld1w { z1.s }, p3/Z, [x23, x9, LSL #2]\n" - "ld1w { z0.s }, p3/Z, [x22, x9, LSL #2]\n" - "ld1w { z31.s }, p3/Z, [x21, x9, LSL #2]\n" - "ld1w { z30.s }, p2/Z, [x24, x28, LSL #2]\n" - "ld1w { z22.s }, p2/Z, [x23, x28, LSL #2]\n" - "ld1w { z29.s }, p2/Z, [x22, x28, LSL #2]\n" - "ld1w { z28.s }, p2/Z, [x21, x28, LSL #2]\n" - "ld1w { z27.s }, p1/Z, [x24, x27, LSL #2]\n" - "ld1w { z21.s }, p1/Z, [x23, x27, LSL #2]\n" - "ld1w { z26.s }, p1/Z, [x22, x27, LSL #2]\n" - "ld1w { z17.s }, p1/Z, [x21, x27, LSL #2]\n" - "ld1w { z25.s }, p0/Z, [x24, x26, LSL #2]\n" - "ld1w { z20.s }, p0/Z, [x23, x26, LSL #2]\n" - "ld1w { z24.s }, p0/Z, [x22, x26, LSL #2]\n" - "ld1w { z16.s }, p0/Z, [x21, x26, LSL #2]\n" + "add x24, x24, #0x20\n" + "ld1w { z2.s }, p3/Z, [x23, x9, LSL #2]\n" + "ld1w { z1.s }, p3/Z, [x22, x9, LSL #2]\n" + "ld1w { z0.s }, p3/Z, [x21, x9, LSL #2]\n" + "ld1w { z31.s }, p3/Z, [x20, x9, LSL #2]\n" + "ld1w { z30.s }, p2/Z, [x23, x28, LSL #2]\n" + "ld1w { z22.s }, p2/Z, [x22, x28, LSL #2]\n" + "ld1w { z29.s }, p2/Z, [x21, x28, LSL #2]\n" + "ld1w { z28.s }, p2/Z, [x20, x28, LSL #2]\n" + "ld1w { z27.s }, p1/Z, [x23, x27, LSL #2]\n" + "ld1w { z21.s }, p1/Z, [x22, x27, LSL #2]\n" + "ld1w { z26.s }, p1/Z, [x21, x27, LSL #2]\n" + "ld1w { z17.s }, p1/Z, [x20, x27, LSL #2]\n" + "ld1w { z25.s }, p0/Z, [x23, x26, LSL #2]\n" + "ld1w { z20.s }, p0/Z, [x22, x26, LSL #2]\n" + "ld1w { z24.s }, p0/Z, [x21, x26, LSL #2]\n" + "ld1w { z16.s }, p0/Z, [x20, x26, LSL #2]\n" "beq 3f\n" "2:" // 4-vectors of channels: 4 inputs loop "fadd z23.s, z2.s, z1.s\n" "fadd z19.s, z0.s, z31.s\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" "fadd z22.s, z30.s, z22.s\n" "fadd z18.s, z29.s, z28.s\n" "subs x25, x25, #0x1\n" - "add x20, x20, #0x20\n" + "add x24, x24, #0x20\n" "fadd z21.s, z27.s, z21.s\n" "fadd z17.s, z26.s, z17.s\n" - "ld1w { z2.s }, p3/Z, [x24, x9, LSL #2]\n" - "ld1w { z1.s }, p3/Z, [x23, x9, LSL #2]\n" + "ld1w { z2.s }, p3/Z, [x23, x9, LSL #2]\n" + "ld1w { z1.s }, p3/Z, [x22, x9, LSL #2]\n" "fadd z20.s, z25.s, z20.s\n" "fadd z16.s, z24.s, z16.s\n" - "ld1w { z0.s }, p3/Z, [x22, x9, LSL #2]\n" - "ld1w { z31.s }, p3/Z, [x21, x9, LSL #2]\n" + "ld1w { z0.s }, p3/Z, [x21, x9, LSL #2]\n" + "ld1w { z31.s }, p3/Z, [x20, x9, LSL #2]\n" "fadd z19.s, z23.s, z19.s\n" "fadd z18.s, z22.s, z18.s\n" - "ld1w { z30.s }, p2/Z, [x24, x28, LSL #2]\n" - "ld1w { z22.s }, p2/Z, [x23, x28, LSL #2]\n" + "ld1w { z30.s }, p2/Z, [x23, x28, LSL #2]\n" + "ld1w { z22.s }, p2/Z, [x22, x28, LSL #2]\n" "fadd z17.s, z21.s, z17.s\n" "fadd z16.s, z20.s, z16.s\n" - "ld1w { z29.s }, p2/Z, [x22, x28, LSL #2]\n" - "ld1w { z28.s }, p2/Z, [x21, x28, LSL #2]\n" + "ld1w { z29.s }, p2/Z, [x21, x28, LSL #2]\n" + "ld1w { z28.s }, p2/Z, [x20, x28, LSL #2]\n" "fadd z6.s, z6.s, z19.s\n" "fadd z5.s, z5.s, z18.s\n" - "ld1w { z27.s }, p1/Z, [x24, x27, LSL #2]\n" - "ld1w { z21.s }, p1/Z, [x23, x27, LSL #2]\n" + "ld1w { z27.s }, p1/Z, [x23, x27, LSL #2]\n" + "ld1w { z21.s }, p1/Z, [x22, x27, LSL #2]\n" "fadd z4.s, z4.s, z17.s\n" "fadd z3.s, z3.s, z16.s\n" - "ld1w { z26.s }, p1/Z, [x22, x27, LSL #2]\n" - "ld1w { z17.s }, p1/Z, [x21, x27, LSL #2]\n" - "ld1w { z25.s }, p0/Z, [x24, x26, LSL #2]\n" - "ld1w { z20.s }, p0/Z, [x23, x26, LSL #2]\n" - "ld1w { z24.s }, p0/Z, [x22, x26, LSL #2]\n" - "ld1w { z16.s }, p0/Z, [x21, x26, LSL #2]\n" + "ld1w { z26.s }, p1/Z, [x21, x27, LSL #2]\n" + "ld1w { z17.s }, p1/Z, [x20, x27, LSL #2]\n" + "ld1w { z25.s }, p0/Z, [x23, x26, LSL #2]\n" + "ld1w { z20.s }, p0/Z, [x22, x26, LSL #2]\n" + "ld1w { z24.s }, p0/Z, [x21, x26, LSL #2]\n" + "ld1w { z16.s }, p0/Z, [x20, x26, LSL #2]\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "fadd z23.s, z2.s, z1.s\n" @@ -141,16 +141,16 @@ void sve_fp32_nhwc_avg_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1w { z2.s }, p3/Z, [x24, x9, LSL #2]\n" + "ldr x20, [x24], #0x8\n" + "ld1w { z16.s }, p3/Z, [x20, x9, LSL #2]\n" "subs x21, x21, #0x1\n" - "fadd z6.s, z6.s, z2.s\n" - "ld1w { z30.s }, p2/Z, [x24, x28, LSL #2]\n" - "ld1w { z27.s }, p1/Z, [x24, x27, LSL #2]\n" - "fadd z5.s, z5.s, z30.s\n" - "fadd z4.s, z4.s, z27.s\n" - "ld1w { z25.s }, p0/Z, [x24, x26, LSL #2]\n" - "fadd z3.s, z3.s, z25.s\n" + "fadd z6.s, z6.s, z16.s\n" + "ld1w { z17.s }, p2/Z, [x20, x28, LSL #2]\n" + "ld1w { z16.s }, p1/Z, [x20, x27, LSL #2]\n" + "fadd z5.s, z5.s, z17.s\n" + "fadd z4.s, z4.s, z16.s\n" + "ld1w { z16.s }, p0/Z, [x20, x26, LSL #2]\n" + "fadd z3.s, z3.s, z16.s\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End "fmul z6.s, z6.s, z7.s\n" @@ -173,44 +173,44 @@ void sve_fp32_nhwc_avg_generic_depthfirst_impl( "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" "mov z6.b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" "subs x25, x25, #0x1\n" - "add x20, x20, #0x20\n" - "ld1w { z2.s }, p3/Z, [x24, x9, LSL #2]\n" - "ld1w { z1.s }, p3/Z, [x23, x9, LSL #2]\n" - "ld1w { z0.s }, p3/Z, [x22, x9, LSL #2]\n" - "ld1w { z31.s }, p3/Z, [x21, x9, LSL #2]\n" + "add x24, x24, #0x20\n" + "ld1w { z2.s }, p3/Z, [x23, x9, LSL #2]\n" + "ld1w { z1.s }, p3/Z, [x22, x9, LSL #2]\n" + "ld1w { z0.s }, p3/Z, [x21, x9, LSL #2]\n" + "ld1w { z31.s }, p3/Z, [x20, x9, LSL #2]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 4 inputs loop - "fadd z23.s, z2.s, z1.s\n" - "fadd z19.s, z0.s, z31.s\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" - "fadd z19.s, z23.s, z19.s\n" + "fadd z17.s, z2.s, z1.s\n" + "fadd z16.s, z0.s, z31.s\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" + "fadd z16.s, z17.s, z16.s\n" "subs x25, x25, #0x1\n" - "fadd z6.s, z6.s, z19.s\n" - "add x20, x20, #0x20\n" - "ld1w { z2.s }, p3/Z, [x24, x9, LSL #2]\n" - "ld1w { z1.s }, p3/Z, [x23, x9, LSL #2]\n" - "ld1w { z0.s }, p3/Z, [x22, x9, LSL #2]\n" - "ld1w { z31.s }, p3/Z, [x21, x9, LSL #2]\n" + "fadd z6.s, z6.s, z16.s\n" + "add x24, x24, #0x20\n" + "ld1w { z2.s }, p3/Z, [x23, x9, LSL #2]\n" + "ld1w { z1.s }, p3/Z, [x22, x9, LSL #2]\n" + "ld1w { z0.s }, p3/Z, [x21, x9, LSL #2]\n" + "ld1w { z31.s }, p3/Z, [x20, x9, LSL #2]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail - "fadd z23.s, z2.s, z1.s\n" - "fadd z19.s, z0.s, z31.s\n" - "fadd z19.s, z23.s, z19.s\n" - "fadd z6.s, z6.s, z19.s\n" + "fadd z17.s, z2.s, z1.s\n" + "fadd z16.s, z0.s, z31.s\n" + "fadd z16.s, z17.s, z16.s\n" + "fadd z6.s, z6.s, z16.s\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1w { z2.s }, p3/Z, [x24, x9, LSL #2]\n" + "ldr x20, [x24], #0x8\n" + "ld1w { z16.s }, p3/Z, [x20, x9, LSL #2]\n" "subs x21, x21, #0x1\n" - "fadd z6.s, z6.s, z2.s\n" + "fadd z6.s, z6.s, z16.s\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "fmul z6.s, z6.s, z7.s\n" diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp index da0239cea8928095ccabca38332c7ee1adac95a5..d9cebd136355023a17ef742a519455f544d0e68a 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp @@ -66,10 +66,10 @@ void sve_fp32_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "ldr x15, [%x[args], %[offsetof_n_channels]]\n" "ldr x21, [%x[args], %[offsetof_outptrs]]\n" "mov x14, #0x0\n" - "whilelt p2.s, x14, x15\n" + "whilelt p0.s, x14, x15\n" "ldr x20, [%x[args], %[offsetof_inptrs]]\n" "ldp x13, x12, [x21, #0x0]\n" - "ptrue p1.b\n" + "ptrue p2.b\n" "mov x11, #0x0\n" "ldp x10, x9, [x21, #0x10]\n" "ldp x28, x27, [x20, #0x0]\n" @@ -77,61 +77,61 @@ void sve_fp32_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "ldp x24, x23, [x20, #0x20]\n" "ldp x22, x21, [x20, #0x30]\n" "ldr x20, [x20, #0x40]\n" - "ld1w { z31.s }, p2/Z, [x27, x14, LSL #2]\n" - "ld1w { z30.s }, p2/Z, [x24, x14, LSL #2]\n" - "ld1w { z29.s }, p2/Z, [x21, x14, LSL #2]\n" - "ld1w { z28.s }, p2/Z, [x25, x14, LSL #2]\n" - "ld1w { z27.s }, p2/Z, [x28, x14, LSL #2]\n" - "ld1w { z26.s }, p2/Z, [x26, x14, LSL #2]\n" - "ld1w { z25.s }, p2/Z, [x23, x14, LSL #2]\n" - "ld1w { z24.s }, p2/Z, [x22, x14, LSL #2]\n" - "ld1w { z23.s }, p2/Z, [x20, x14, LSL #2]\n" + "ld1w { z31.s }, p0/Z, [x27, x14, LSL #2]\n" + "ld1w { z30.s }, p0/Z, [x24, x14, LSL #2]\n" + "ld1w { z29.s }, p0/Z, [x21, x14, LSL #2]\n" + "ld1w { z28.s }, p0/Z, [x25, x14, LSL #2]\n" + "ld1w { z27.s }, p0/Z, [x28, x14, LSL #2]\n" + "ld1w { z26.s }, p0/Z, [x26, x14, LSL #2]\n" + "ld1w { z25.s }, p0/Z, [x23, x14, LSL #2]\n" + "ld1w { z24.s }, p0/Z, [x22, x14, LSL #2]\n" + "ld1w { z23.s }, p0/Z, [x20, x14, LSL #2]\n" "incw x14\n" - "whilelt p2.s, x14, x15\n" + "whilelt p1.s, x14, x15\n" "b.none 2f\n" "1:" // Vector: Loop - "movprfx z22, z31\n fmax z22.s, p1/M, z22.s, z30.s\n" - "movprfx z21, z30\n fmax z21.s, p1/M, z21.s, z29.s\n" - "ld1w { z31.s }, p2/Z, [x27, x14, LSL #2]\n" - "ld1w { z30.s }, p2/Z, [x24, x14, LSL #2]\n" - "movprfx z20, z28\n fmax z20.s, p1/M, z20.s, z27.s\n" - "movprfx z19, z26\n fmax z19.s, p1/M, z19.s, z25.s\n" - "ld1w { z29.s }, p2/Z, [x21, x14, LSL #2]\n" - "ld1w { z27.s }, p2/Z, [x28, x14, LSL #2]\n" - "movprfx z17, z28\n fmax z17.s, p1/M, z17.s, z24.s\n" - "movprfx z18, z25\n fmax z18.s, p1/M, z18.s, z23.s\n" - "ld1w { z28.s }, p2/Z, [x25, x14, LSL #2]\n" - "ld1w { z26.s }, p2/Z, [x26, x14, LSL #2]\n" - "ld1w { z25.s }, p2/Z, [x23, x14, LSL #2]\n" - "ld1w { z24.s }, p2/Z, [x22, x14, LSL #2]\n" + "movprfx z22, z31\n fmax z22.s, p2/M, z22.s, z30.s\n" + "movprfx z21, z30\n fmax z21.s, p2/M, z21.s, z29.s\n" + "ld1w { z31.s }, p1/Z, [x27, x14, LSL #2]\n" + "ld1w { z30.s }, p1/Z, [x24, x14, LSL #2]\n" + "movprfx z20, z28\n fmax z20.s, p2/M, z20.s, z27.s\n" + "movprfx z19, z26\n fmax z19.s, p2/M, z19.s, z25.s\n" + "ld1w { z29.s }, p1/Z, [x21, x14, LSL #2]\n" + "ld1w { z27.s }, p1/Z, [x28, x14, LSL #2]\n" + "movprfx z17, z28\n fmax z17.s, p2/M, z17.s, z24.s\n" + "movprfx z18, z25\n fmax z18.s, p2/M, z18.s, z23.s\n" + "ld1w { z28.s }, p1/Z, [x25, x14, LSL #2]\n" + "ld1w { z26.s }, p1/Z, [x26, x14, LSL #2]\n" + "ld1w { z25.s }, p1/Z, [x23, x14, LSL #2]\n" + "ld1w { z24.s }, p1/Z, [x22, x14, LSL #2]\n" "whilelt p0.s, x11, x15\n" - "movprfx z16, z22\n fmax z16.s, p1/M, z16.s, z20.s\n" - "ld1w { z23.s }, p2/Z, [x20, x14, LSL #2]\n" + "movprfx z16, z22\n fmax z16.s, p2/M, z16.s, z20.s\n" + "ld1w { z23.s }, p1/Z, [x20, x14, LSL #2]\n" "incw x14\n" - "whilelt p2.s, x14, x15\n" + "whilelt p1.s, x14, x15\n" "st1w { z16.s }, p0, [x13, x11, LSL #2]\n" - "movprfx z16, z19\n fmax z16.s, p1/M, z16.s, z22.s\n" - "fmax z17.s, p1/M, z17.s, z21.s\n" + "movprfx z16, z19\n fmax z16.s, p2/M, z16.s, z22.s\n" + "fmax z17.s, p2/M, z17.s, z21.s\n" "st1w { z16.s }, p0, [x12, x11, LSL #2]\n" - "movprfx z16, z18\n fmax z16.s, p1/M, z16.s, z21.s\n" + "movprfx z16, z21\n fmax z16.s, p2/M, z16.s, z18.s\n" "st1w { z17.s }, p0, [x10, x11, LSL #2]\n" "st1w { z16.s }, p0, [x9, x11, LSL #2]\n" "incw x11\n" "b.any 1b\n" "2:" // Vector: Tail - "movprfx z22, z31\n fmax z22.s, p1/M, z22.s, z30.s\n" - "movprfx z21, z30\n fmax z21.s, p1/M, z21.s, z29.s\n" - "movprfx z20, z28\n fmax z20.s, p1/M, z20.s, z27.s\n" - "movprfx z19, z26\n fmax z19.s, p1/M, z19.s, z25.s\n" - "movprfx z17, z28\n fmax z17.s, p1/M, z17.s, z24.s\n" - "movprfx z18, z25\n fmax z18.s, p1/M, z18.s, z23.s\n" + "movprfx z22, z31\n fmax z22.s, p2/M, z22.s, z30.s\n" + "movprfx z21, z30\n fmax z21.s, p2/M, z21.s, z29.s\n" + "movprfx z20, z28\n fmax z20.s, p2/M, z20.s, z27.s\n" + "movprfx z19, z26\n fmax z19.s, p2/M, z19.s, z25.s\n" + "movprfx z17, z28\n fmax z17.s, p2/M, z17.s, z24.s\n" + "movprfx z18, z25\n fmax z18.s, p2/M, z18.s, z23.s\n" "whilelt p0.s, x11, x15\n" - "movprfx z16, z22\n fmax z16.s, p1/M, z16.s, z20.s\n" + "movprfx z16, z22\n fmax z16.s, p2/M, z16.s, z20.s\n" "st1w { z16.s }, p0, [x13, x11, LSL #2]\n" - "movprfx z16, z19\n fmax z16.s, p1/M, z16.s, z22.s\n" - "fmax z17.s, p1/M, z17.s, z21.s\n" + "movprfx z16, z19\n fmax z16.s, p2/M, z16.s, z22.s\n" + "fmax z17.s, p2/M, z17.s, z21.s\n" "st1w { z16.s }, p0, [x12, x11, LSL #2]\n" - "movprfx z16, z18\n fmax z16.s, p1/M, z16.s, z21.s\n" + "movprfx z16, z21\n fmax z16.s, p2/M, z16.s, z18.s\n" "st1w { z17.s }, p0, [x10, x11, LSL #2]\n" "st1w { z16.s }, p0, [x9, x11, LSL #2]\n" : diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_max_generic_depthfirst/generic.cpp index ddce2be62c28f7491fadf4a47dfcf083e2597683..87fc75addadb400c51001b4468535d0f1f735658 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_max_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_max_generic_depthfirst/generic.cpp @@ -54,68 +54,68 @@ void sve_fp32_nhwc_max_generic_depthfirst_impl( "lsr x25, %x[n_valid_cells], #0x2\n" "mov z8.s, #0xff800000\n" "mov z7.s, #0xff800000\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "mov z6.s, #0xff800000\n" "mov z5.s, #0xff800000\n" "cbz x25, 4f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" "subs x25, x25, #0x1\n" - "add x20, x20, #0x20\n" - "ld1w { z4.s }, p4/Z, [x24, x9, LSL #2]\n" - "ld1w { z3.s }, p4/Z, [x23, x9, LSL #2]\n" - "ld1w { z2.s }, p4/Z, [x22, x9, LSL #2]\n" - "ld1w { z1.s }, p4/Z, [x21, x9, LSL #2]\n" - "ld1w { z0.s }, p3/Z, [x24, x28, LSL #2]\n" - "ld1w { z31.s }, p3/Z, [x23, x28, LSL #2]\n" - "ld1w { z22.s }, p3/Z, [x22, x28, LSL #2]\n" - "ld1w { z30.s }, p3/Z, [x21, x28, LSL #2]\n" - "ld1w { z29.s }, p2/Z, [x24, x27, LSL #2]\n" - "ld1w { z28.s }, p2/Z, [x23, x27, LSL #2]\n" - "ld1w { z21.s }, p2/Z, [x22, x27, LSL #2]\n" - "ld1w { z27.s }, p2/Z, [x21, x27, LSL #2]\n" - "ld1w { z26.s }, p1/Z, [x24, x26, LSL #2]\n" - "ld1w { z25.s }, p1/Z, [x23, x26, LSL #2]\n" - "ld1w { z20.s }, p1/Z, [x22, x26, LSL #2]\n" - "ld1w { z24.s }, p1/Z, [x21, x26, LSL #2]\n" + "add x24, x24, #0x20\n" + "ld1w { z4.s }, p4/Z, [x23, x9, LSL #2]\n" + "ld1w { z3.s }, p4/Z, [x22, x9, LSL #2]\n" + "ld1w { z2.s }, p4/Z, [x21, x9, LSL #2]\n" + "ld1w { z1.s }, p4/Z, [x20, x9, LSL #2]\n" + "ld1w { z0.s }, p3/Z, [x23, x28, LSL #2]\n" + "ld1w { z31.s }, p3/Z, [x22, x28, LSL #2]\n" + "ld1w { z22.s }, p3/Z, [x21, x28, LSL #2]\n" + "ld1w { z30.s }, p3/Z, [x20, x28, LSL #2]\n" + "ld1w { z29.s }, p2/Z, [x23, x27, LSL #2]\n" + "ld1w { z28.s }, p2/Z, [x22, x27, LSL #2]\n" + "ld1w { z21.s }, p2/Z, [x21, x27, LSL #2]\n" + "ld1w { z27.s }, p2/Z, [x20, x27, LSL #2]\n" + "ld1w { z26.s }, p1/Z, [x23, x26, LSL #2]\n" + "ld1w { z25.s }, p1/Z, [x22, x26, LSL #2]\n" + "ld1w { z20.s }, p1/Z, [x21, x26, LSL #2]\n" + "ld1w { z24.s }, p1/Z, [x20, x26, LSL #2]\n" "beq 3f\n" "2:" // 4-vectors of channels: 4 inputs loop "movprfx z19, z4\n fmax z19.s, p0/M, z19.s, z3.s\n" "movprfx z23, z2\n fmax z23.s, p0/M, z23.s, z1.s\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" "movprfx z18, z0\n fmax z18.s, p0/M, z18.s, z31.s\n" "fmax z22.s, p0/M, z22.s, z30.s\n" - "ld1w { z4.s }, p4/Z, [x24, x9, LSL #2]\n" - "ld1w { z3.s }, p4/Z, [x23, x9, LSL #2]\n" + "ld1w { z4.s }, p4/Z, [x23, x9, LSL #2]\n" + "ld1w { z3.s }, p4/Z, [x22, x9, LSL #2]\n" "movprfx z17, z29\n fmax z17.s, p0/M, z17.s, z28.s\n" "fmax z21.s, p0/M, z21.s, z27.s\n" - "ld1w { z2.s }, p4/Z, [x22, x9, LSL #2]\n" - "ld1w { z1.s }, p4/Z, [x21, x9, LSL #2]\n" + "ld1w { z2.s }, p4/Z, [x21, x9, LSL #2]\n" + "ld1w { z1.s }, p4/Z, [x20, x9, LSL #2]\n" "movprfx z16, z26\n fmax z16.s, p0/M, z16.s, z25.s\n" "fmax z20.s, p0/M, z20.s, z24.s\n" - "ld1w { z0.s }, p3/Z, [x24, x28, LSL #2]\n" - "ld1w { z31.s }, p3/Z, [x23, x28, LSL #2]\n" + "ld1w { z0.s }, p3/Z, [x23, x28, LSL #2]\n" + "ld1w { z31.s }, p3/Z, [x22, x28, LSL #2]\n" "fmax z19.s, p0/M, z19.s, z23.s\n" "fmax z18.s, p0/M, z18.s, z22.s\n" - "ld1w { z22.s }, p3/Z, [x22, x28, LSL #2]\n" - "ld1w { z30.s }, p3/Z, [x21, x28, LSL #2]\n" + "ld1w { z22.s }, p3/Z, [x21, x28, LSL #2]\n" + "ld1w { z30.s }, p3/Z, [x20, x28, LSL #2]\n" "fmax z17.s, p0/M, z17.s, z21.s\n" "fmax z16.s, p0/M, z16.s, z20.s\n" - "ld1w { z29.s }, p2/Z, [x24, x27, LSL #2]\n" - "ld1w { z28.s }, p2/Z, [x23, x27, LSL #2]\n" + "ld1w { z29.s }, p2/Z, [x23, x27, LSL #2]\n" + "ld1w { z28.s }, p2/Z, [x22, x27, LSL #2]\n" "subs x25, x25, #0x1\n" "fmax z8.s, p0/M, z8.s, z19.s\n" - "ld1w { z21.s }, p2/Z, [x22, x27, LSL #2]\n" - "ld1w { z27.s }, p2/Z, [x21, x27, LSL #2]\n" + "ld1w { z21.s }, p2/Z, [x21, x27, LSL #2]\n" + "ld1w { z27.s }, p2/Z, [x20, x27, LSL #2]\n" "fmax z7.s, p0/M, z7.s, z18.s\n" "fmax z6.s, p0/M, z6.s, z17.s\n" - "ld1w { z26.s }, p1/Z, [x24, x26, LSL #2]\n" - "ld1w { z25.s }, p1/Z, [x23, x26, LSL #2]\n" + "ld1w { z26.s }, p1/Z, [x23, x26, LSL #2]\n" + "ld1w { z25.s }, p1/Z, [x22, x26, LSL #2]\n" "fmax z5.s, p0/M, z5.s, z16.s\n" - "add x20, x20, #0x20\n" - "ld1w { z20.s }, p1/Z, [x22, x26, LSL #2]\n" - "ld1w { z24.s }, p1/Z, [x21, x26, LSL #2]\n" + "add x24, x24, #0x20\n" + "ld1w { z20.s }, p1/Z, [x21, x26, LSL #2]\n" + "ld1w { z24.s }, p1/Z, [x20, x26, LSL #2]\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "movprfx z19, z4\n fmax z19.s, p0/M, z19.s, z3.s\n" @@ -138,16 +138,16 @@ void sve_fp32_nhwc_max_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1w { z4.s }, p4/Z, [x24, x9, LSL #2]\n" + "ldr x20, [x24], #0x8\n" + "ld1w { z16.s }, p4/Z, [x20, x9, LSL #2]\n" "subs x21, x21, #0x1\n" - "fmax z8.s, p0/M, z8.s, z4.s\n" - "ld1w { z0.s }, p3/Z, [x24, x28, LSL #2]\n" - "ld1w { z29.s }, p2/Z, [x24, x27, LSL #2]\n" - "fmax z7.s, p0/M, z7.s, z0.s\n" - "fmax z6.s, p0/M, z6.s, z29.s\n" - "ld1w { z26.s }, p1/Z, [x24, x26, LSL #2]\n" - "fmax z5.s, p0/M, z5.s, z26.s\n" + "fmax z8.s, p0/M, z8.s, z16.s\n" + "ld1w { z17.s }, p3/Z, [x20, x28, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x20, x27, LSL #2]\n" + "fmax z7.s, p0/M, z7.s, z17.s\n" + "fmax z6.s, p0/M, z6.s, z16.s\n" + "ld1w { z16.s }, p1/Z, [x20, x26, LSL #2]\n" + "fmax z5.s, p0/M, z5.s, z16.s\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End "st1w { z8.s }, p4, [%x[outptr], x9, LSL #2]\n" @@ -166,44 +166,44 @@ void sve_fp32_nhwc_max_generic_depthfirst_impl( "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" "mov z8.s, #0xff800000\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" "subs x25, x25, #0x1\n" - "add x20, x20, #0x20\n" - "ld1w { z4.s }, p4/Z, [x24, x9, LSL #2]\n" - "ld1w { z3.s }, p4/Z, [x23, x9, LSL #2]\n" - "ld1w { z2.s }, p4/Z, [x22, x9, LSL #2]\n" - "ld1w { z1.s }, p4/Z, [x21, x9, LSL #2]\n" + "add x24, x24, #0x20\n" + "ld1w { z4.s }, p4/Z, [x23, x9, LSL #2]\n" + "ld1w { z3.s }, p4/Z, [x22, x9, LSL #2]\n" + "ld1w { z2.s }, p4/Z, [x21, x9, LSL #2]\n" + "ld1w { z1.s }, p4/Z, [x20, x9, LSL #2]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 4 inputs loop - "movprfx z19, z4\n fmax z19.s, p0/M, z19.s, z3.s\n" - "movprfx z23, z2\n fmax z23.s, p0/M, z23.s, z1.s\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" - "fmax z19.s, p0/M, z19.s, z23.s\n" + "movprfx z16, z4\n fmax z16.s, p0/M, z16.s, z3.s\n" + "movprfx z17, z2\n fmax z17.s, p0/M, z17.s, z1.s\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" + "fmax z16.s, p0/M, z16.s, z17.s\n" "subs x25, x25, #0x1\n" - "ld1w { z4.s }, p4/Z, [x24, x9, LSL #2]\n" - "ld1w { z3.s }, p4/Z, [x23, x9, LSL #2]\n" - "fmax z8.s, p0/M, z8.s, z19.s\n" - "add x20, x20, #0x20\n" - "ld1w { z2.s }, p4/Z, [x22, x9, LSL #2]\n" - "ld1w { z1.s }, p4/Z, [x21, x9, LSL #2]\n" + "ld1w { z4.s }, p4/Z, [x23, x9, LSL #2]\n" + "ld1w { z3.s }, p4/Z, [x22, x9, LSL #2]\n" + "fmax z8.s, p0/M, z8.s, z16.s\n" + "add x24, x24, #0x20\n" + "ld1w { z2.s }, p4/Z, [x21, x9, LSL #2]\n" + "ld1w { z1.s }, p4/Z, [x20, x9, LSL #2]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail - "movprfx z19, z4\n fmax z19.s, p0/M, z19.s, z3.s\n" - "movprfx z23, z2\n fmax z23.s, p0/M, z23.s, z1.s\n" - "fmax z19.s, p0/M, z19.s, z23.s\n" - "fmax z8.s, p0/M, z8.s, z19.s\n" + "movprfx z16, z4\n fmax z16.s, p0/M, z16.s, z3.s\n" + "movprfx z17, z2\n fmax z17.s, p0/M, z17.s, z1.s\n" + "fmax z16.s, p0/M, z16.s, z17.s\n" + "fmax z8.s, p0/M, z8.s, z16.s\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1w { z4.s }, p4/Z, [x24, x9, LSL #2]\n" + "ldr x20, [x24], #0x8\n" + "ld1w { z16.s }, p4/Z, [x20, x9, LSL #2]\n" "subs x21, x21, #0x1\n" - "fmax z8.s, p0/M, z8.s, z4.s\n" + "fmax z8.s, p0/M, z8.s, z16.s\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "st1w { z8.s }, p4, [%x[outptr], x9, LSL #2]\n" diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_avg_generic_depthfirst/generic.cpp index 68bd831d63de46cc9859019b0b41b5e1305d99ac..7925905e643a14b2a304b1029163cac6c5e5255b 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_avg_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_avg_generic_depthfirst/generic.cpp @@ -109,7 +109,7 @@ void sve_s8_nhwc_avg_generic_depthfirst_impl( "lsr x23, %x[n_valid_cells], #0x1\n" "mov z15.s, #0x0\n" "mov z14.s, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "mov z13.s, #0x0\n" "mov z12.s, #0x0\n" "mov z11.s, #0x0\n" @@ -125,42 +125,42 @@ void sve_s8_nhwc_avg_generic_depthfirst_impl( "mov z1.s, #0x0\n" "mov z0.s, #0x0\n" "cbz x23, 4f\n" - "ldp x22, x21, [x20, #0x0]\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" - "ld1b { z29.b }, p3/Z, [x22, x26]\n" - "ld1b { z28.b }, p3/Z, [x21, x26]\n" - "ld1b { z27.b }, p2/Z, [x22, x25]\n" - "ld1b { z26.b }, p2/Z, [x21, x25]\n" - "ld1b { z25.b }, p1/Z, [x22, x24]\n" - "ld1b { z24.b }, p1/Z, [x21, x24]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" + "ld1b { z29.b }, p3/Z, [x21, x26]\n" + "ld1b { z28.b }, p3/Z, [x20, x26]\n" + "ld1b { z27.b }, p2/Z, [x21, x25]\n" + "ld1b { z26.b }, p2/Z, [x20, x25]\n" + "ld1b { z25.b }, p1/Z, [x21, x24]\n" + "ld1b { z24.b }, p1/Z, [x20, x24]\n" "beq 3f\n" "2:" // 4-vectors of channels: 2 inputs loop ".inst 0x455e03f7 // saddlb z23.h, z31.b, z30.b\n" ".inst 0x455e07f6 // saddlt z22.h, z31.b, z30.b\n" - "ldp x22, x21, [x20, #0x0]\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" ".inst 0x455c03b5 // saddlb z21.h, z29.b, z28.b\n" ".inst 0x455c07b4 // saddlt z20.h, z29.b, z28.b\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" ".inst 0x455a0373 // saddlb z19.h, z27.b, z26.b\n" ".inst 0x455a0772 // saddlt z18.h, z27.b, z26.b\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" - "ld1b { z29.b }, p3/Z, [x22, x26]\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" + "ld1b { z29.b }, p3/Z, [x21, x26]\n" ".inst 0x45580331 // saddlb z17.h, z25.b, z24.b\n" ".inst 0x45580730 // saddlt z16.h, z25.b, z24.b\n" - "ld1b { z28.b }, p3/Z, [x21, x26]\n" - "ld1b { z27.b }, p2/Z, [x22, x25]\n" + "ld1b { z28.b }, p3/Z, [x20, x26]\n" + "ld1b { z27.b }, p2/Z, [x21, x25]\n" ".inst 0x459741ef // saddwb z15.s, z15.s, z23.h\n" ".inst 0x459745ce // saddwt z14.s, z14.s, z23.h\n" - "ld1b { z26.b }, p2/Z, [x21, x25]\n" - "ld1b { z25.b }, p1/Z, [x22, x24]\n" + "ld1b { z26.b }, p2/Z, [x20, x25]\n" + "ld1b { z25.b }, p1/Z, [x21, x24]\n" ".inst 0x459641ad // saddwb z13.s, z13.s, z22.h\n" ".inst 0x4596458c // saddwt z12.s, z12.s, z22.h\n" - "ld1b { z24.b }, p1/Z, [x21, x24]\n" + "ld1b { z24.b }, p1/Z, [x20, x24]\n" ".inst 0x4595416b // saddwb z11.s, z11.s, z21.h\n" ".inst 0x4595454a // saddwt z10.s, z10.s, z21.h\n" ".inst 0x45944129 // saddwb z9.s, z9.s, z20.h\n" @@ -203,20 +203,20 @@ void sve_s8_nhwc_avg_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x1\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x22, [x20], #0x8\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - ".inst 0x4508a3f7 // sshllb z23.h, z31.b, #0x0\n" - ".inst 0x4508a7f6 // sshllt z22.h, z31.b, #0x0\n" - "ld1b { z29.b }, p3/Z, [x22, x26]\n" - "ld1b { z27.b }, p2/Z, [x22, x25]\n" - ".inst 0x4508a3b5 // sshllb z21.h, z29.b, #0x0\n" - ".inst 0x4508a7b4 // sshllt z20.h, z29.b, #0x0\n" - "ld1b { z25.b }, p1/Z, [x22, x24]\n" - ".inst 0x4508a373 // sshllb z19.h, z27.b, #0x0\n" - ".inst 0x4508a772 // sshllt z18.h, z27.b, #0x0\n" + "ldr x20, [x22], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x27]\n" + ".inst 0x4508a217 // sshllb z23.h, z16.b, #0x0\n" + ".inst 0x4508a616 // sshllt z22.h, z16.b, #0x0\n" + "ld1b { z16.b }, p3/Z, [x20, x26]\n" + "ld1b { z17.b }, p2/Z, [x20, x25]\n" + ".inst 0x4508a215 // sshllb z21.h, z16.b, #0x0\n" + ".inst 0x4508a614 // sshllt z20.h, z16.b, #0x0\n" + "ld1b { z16.b }, p1/Z, [x20, x24]\n" + ".inst 0x4508a233 // sshllb z19.h, z17.b, #0x0\n" + ".inst 0x4508a632 // sshllt z18.h, z17.b, #0x0\n" "subs x21, x21, #0x1\n" - ".inst 0x4508a331 // sshllb z17.h, z25.b, #0x0\n" - ".inst 0x4508a730 // sshllt z16.h, z25.b, #0x0\n" + ".inst 0x4508a211 // sshllb z17.h, z16.b, #0x0\n" + ".inst 0x4508a610 // sshllt z16.h, z16.b, #0x0\n" ".inst 0x459741ef // saddwb z15.s, z15.s, z23.h\n" ".inst 0x459745ce // saddwt z14.s, z14.s, z23.h\n" ".inst 0x459641ad // saddwb z13.s, z13.s, z22.h\n" @@ -332,49 +332,49 @@ void sve_s8_nhwc_avg_generic_depthfirst_impl( "lsr x23, %x[n_valid_cells], #0x1\n" "mov z15.s, #0x0\n" "mov z14.s, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "mov z13.s, #0x0\n" "mov z12.s, #0x0\n" "cbz x23, 11f\n" - "ldp x22, x21, [x20, #0x0]\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 2 inputs loop - ".inst 0x455e03f7 // saddlb z23.h, z31.b, z30.b\n" - ".inst 0x455e07f6 // saddlt z22.h, z31.b, z30.b\n" - "ldp x22, x21, [x20, #0x0]\n" + ".inst 0x455e03f1 // saddlb z17.h, z31.b, z30.b\n" + ".inst 0x455e07f0 // saddlt z16.h, z31.b, z30.b\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - ".inst 0x459741ef // saddwb z15.s, z15.s, z23.h\n" - ".inst 0x459745ce // saddwt z14.s, z14.s, z23.h\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - ".inst 0x459641ad // saddwb z13.s, z13.s, z22.h\n" - ".inst 0x4596458c // saddwt z12.s, z12.s, z22.h\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" + ".inst 0x459141ef // saddwb z15.s, z15.s, z17.h\n" + ".inst 0x459145ce // saddwt z14.s, z14.s, z17.h\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" + ".inst 0x459041ad // saddwb z13.s, z13.s, z16.h\n" + ".inst 0x4590458c // saddwt z12.s, z12.s, z16.h\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 2 inputs tail - ".inst 0x455e03f7 // saddlb z23.h, z31.b, z30.b\n" - ".inst 0x455e07f6 // saddlt z22.h, z31.b, z30.b\n" - ".inst 0x459741ef // saddwb z15.s, z15.s, z23.h\n" - ".inst 0x459745ce // saddwt z14.s, z14.s, z23.h\n" - ".inst 0x459641ad // saddwb z13.s, z13.s, z22.h\n" - ".inst 0x4596458c // saddwt z12.s, z12.s, z22.h\n" + ".inst 0x455e03f1 // saddlb z17.h, z31.b, z30.b\n" + ".inst 0x455e07f0 // saddlt z16.h, z31.b, z30.b\n" + ".inst 0x459141ef // saddwb z15.s, z15.s, z17.h\n" + ".inst 0x459145ce // saddwt z14.s, z14.s, z17.h\n" + ".inst 0x459041ad // saddwb z13.s, z13.s, z16.h\n" + ".inst 0x4590458c // saddwt z12.s, z12.s, z16.h\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x1\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x22, [x20], #0x8\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - ".inst 0x4508a3f7 // sshllb z23.h, z31.b, #0x0\n" - ".inst 0x4508a7f6 // sshllt z22.h, z31.b, #0x0\n" + "ldr x20, [x22], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x27]\n" + ".inst 0x4508a211 // sshllb z17.h, z16.b, #0x0\n" + ".inst 0x4508a610 // sshllt z16.h, z16.b, #0x0\n" "subs x21, x21, #0x1\n" - ".inst 0x459741ef // saddwb z15.s, z15.s, z23.h\n" - ".inst 0x459745ce // saddwt z14.s, z14.s, z23.h\n" - ".inst 0x459641ad // saddwb z13.s, z13.s, z22.h\n" - ".inst 0x4596458c // saddwt z12.s, z12.s, z22.h\n" + ".inst 0x459141ef // saddwb z15.s, z15.s, z17.h\n" + ".inst 0x459145ce // saddwt z14.s, z14.s, z17.h\n" + ".inst 0x459041ad // saddwb z13.s, z13.s, z16.h\n" + ".inst 0x4590458c // saddwt z12.s, z12.s, z16.h\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "ld1rw { z17.s }, p0/Z, [%x[rescale_ptr]]\n" diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp index 96e20c752ecef9960c1494efca4b2f5c03a5f3a0..5681cc1f3d421e4d6fd40732491bad9bdb000c8a 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp @@ -66,10 +66,10 @@ void sve_s8_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "ldr x15, [%x[args], %[offsetof_n_channels]]\n" "ldr x21, [%x[args], %[offsetof_outptrs]]\n" "mov x14, #0x0\n" - "whilelt p2.b, x14, x15\n" + "whilelt p0.b, x14, x15\n" "ldr x20, [%x[args], %[offsetof_inptrs]]\n" "ldp x13, x12, [x21, #0x0]\n" - "ptrue p1.b\n" + "ptrue p2.b\n" "mov x11, #0x0\n" "ldp x10, x9, [x21, #0x10]\n" "ldp x28, x27, [x20, #0x0]\n" @@ -77,61 +77,61 @@ void sve_s8_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "ldp x24, x23, [x20, #0x20]\n" "ldp x22, x21, [x20, #0x30]\n" "ldr x20, [x20, #0x40]\n" - "ld1b { z31.b }, p2/Z, [x27, x14]\n" - "ld1b { z30.b }, p2/Z, [x24, x14]\n" - "ld1b { z29.b }, p2/Z, [x21, x14]\n" - "ld1b { z28.b }, p2/Z, [x25, x14]\n" - "ld1b { z27.b }, p2/Z, [x28, x14]\n" - "ld1b { z26.b }, p2/Z, [x26, x14]\n" - "ld1b { z25.b }, p2/Z, [x23, x14]\n" - "ld1b { z24.b }, p2/Z, [x22, x14]\n" - "ld1b { z23.b }, p2/Z, [x20, x14]\n" + "ld1b { z31.b }, p0/Z, [x27, x14]\n" + "ld1b { z30.b }, p0/Z, [x24, x14]\n" + "ld1b { z29.b }, p0/Z, [x21, x14]\n" + "ld1b { z28.b }, p0/Z, [x25, x14]\n" + "ld1b { z27.b }, p0/Z, [x28, x14]\n" + "ld1b { z26.b }, p0/Z, [x26, x14]\n" + "ld1b { z25.b }, p0/Z, [x23, x14]\n" + "ld1b { z24.b }, p0/Z, [x22, x14]\n" + "ld1b { z23.b }, p0/Z, [x20, x14]\n" "incw x14\n" - "whilelt p2.b, x14, x15\n" + "whilelt p1.b, x14, x15\n" "b.none 2f\n" "1:" // Vector: Loop - "movprfx z22, z31\n smax z22.b, p1/M, z22.b, z30.b\n" - "movprfx z21, z30\n smax z21.b, p1/M, z21.b, z29.b\n" - "ld1b { z31.b }, p2/Z, [x27, x14]\n" - "ld1b { z30.b }, p2/Z, [x24, x14]\n" - "movprfx z20, z28\n smax z20.b, p1/M, z20.b, z27.b\n" - "movprfx z19, z26\n smax z19.b, p1/M, z19.b, z25.b\n" - "ld1b { z29.b }, p2/Z, [x21, x14]\n" - "ld1b { z27.b }, p2/Z, [x28, x14]\n" - "movprfx z17, z28\n smax z17.b, p1/M, z17.b, z24.b\n" - "movprfx z18, z25\n smax z18.b, p1/M, z18.b, z23.b\n" - "ld1b { z28.b }, p2/Z, [x25, x14]\n" - "ld1b { z26.b }, p2/Z, [x26, x14]\n" - "ld1b { z25.b }, p2/Z, [x23, x14]\n" - "ld1b { z24.b }, p2/Z, [x22, x14]\n" + "movprfx z22, z31\n smax z22.b, p2/M, z22.b, z30.b\n" + "movprfx z21, z30\n smax z21.b, p2/M, z21.b, z29.b\n" + "ld1b { z31.b }, p1/Z, [x27, x14]\n" + "ld1b { z30.b }, p1/Z, [x24, x14]\n" + "movprfx z20, z28\n smax z20.b, p2/M, z20.b, z27.b\n" + "movprfx z19, z26\n smax z19.b, p2/M, z19.b, z25.b\n" + "ld1b { z29.b }, p1/Z, [x21, x14]\n" + "ld1b { z27.b }, p1/Z, [x28, x14]\n" + "movprfx z17, z28\n smax z17.b, p2/M, z17.b, z24.b\n" + "movprfx z18, z25\n smax z18.b, p2/M, z18.b, z23.b\n" + "ld1b { z28.b }, p1/Z, [x25, x14]\n" + "ld1b { z26.b }, p1/Z, [x26, x14]\n" + "ld1b { z25.b }, p1/Z, [x23, x14]\n" + "ld1b { z24.b }, p1/Z, [x22, x14]\n" "whilelt p0.b, x11, x15\n" - "movprfx z16, z22\n smax z16.b, p1/M, z16.b, z20.b\n" - "ld1b { z23.b }, p2/Z, [x20, x14]\n" + "movprfx z16, z22\n smax z16.b, p2/M, z16.b, z20.b\n" + "ld1b { z23.b }, p1/Z, [x20, x14]\n" "incw x14\n" - "whilelt p2.b, x14, x15\n" + "whilelt p1.b, x14, x15\n" "st1b { z16.b }, p0, [x13, x11]\n" - "movprfx z16, z19\n smax z16.b, p1/M, z16.b, z22.b\n" - "smax z17.b, p1/M, z17.b, z21.b\n" + "movprfx z16, z19\n smax z16.b, p2/M, z16.b, z22.b\n" + "smax z17.b, p2/M, z17.b, z21.b\n" "st1b { z16.b }, p0, [x12, x11]\n" - "movprfx z16, z18\n smax z16.b, p1/M, z16.b, z21.b\n" + "movprfx z16, z21\n smax z16.b, p2/M, z16.b, z18.b\n" "st1b { z17.b }, p0, [x10, x11]\n" "st1b { z16.b }, p0, [x9, x11]\n" "incw x11\n" "b.any 1b\n" "2:" // Vector: Tail - "movprfx z22, z31\n smax z22.b, p1/M, z22.b, z30.b\n" - "movprfx z21, z30\n smax z21.b, p1/M, z21.b, z29.b\n" - "movprfx z20, z28\n smax z20.b, p1/M, z20.b, z27.b\n" - "movprfx z19, z26\n smax z19.b, p1/M, z19.b, z25.b\n" - "movprfx z17, z28\n smax z17.b, p1/M, z17.b, z24.b\n" - "movprfx z18, z25\n smax z18.b, p1/M, z18.b, z23.b\n" + "movprfx z22, z31\n smax z22.b, p2/M, z22.b, z30.b\n" + "movprfx z21, z30\n smax z21.b, p2/M, z21.b, z29.b\n" + "movprfx z20, z28\n smax z20.b, p2/M, z20.b, z27.b\n" + "movprfx z19, z26\n smax z19.b, p2/M, z19.b, z25.b\n" + "movprfx z17, z28\n smax z17.b, p2/M, z17.b, z24.b\n" + "movprfx z18, z25\n smax z18.b, p2/M, z18.b, z23.b\n" "whilelt p0.b, x11, x15\n" - "movprfx z16, z22\n smax z16.b, p1/M, z16.b, z20.b\n" + "movprfx z16, z22\n smax z16.b, p2/M, z16.b, z20.b\n" "st1b { z16.b }, p0, [x13, x11]\n" - "movprfx z16, z19\n smax z16.b, p1/M, z16.b, z22.b\n" - "smax z17.b, p1/M, z17.b, z21.b\n" + "movprfx z16, z19\n smax z16.b, p2/M, z16.b, z22.b\n" + "smax z17.b, p2/M, z17.b, z21.b\n" "st1b { z16.b }, p0, [x12, x11]\n" - "movprfx z16, z18\n smax z16.b, p1/M, z16.b, z21.b\n" + "movprfx z16, z21\n smax z16.b, p2/M, z16.b, z18.b\n" "st1b { z17.b }, p0, [x10, x11]\n" "st1b { z16.b }, p0, [x9, x11]\n" : diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_max_generic_depthfirst/generic.cpp index 7d14edddeb06475ac56a63f9435195663d0cfa3a..da9e1408f9175e285bb7f64f3a31d8ab3b520e2d 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_max_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_max_generic_depthfirst/generic.cpp @@ -54,68 +54,68 @@ void sve_s8_nhwc_max_generic_depthfirst_impl( "lsr x25, %x[n_valid_cells], #0x2\n" "mov z8.b, #0x80\n" "mov z7.b, #0x80\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "mov z6.b, #0x80\n" "mov z5.b, #0x80\n" "cbz x25, 4f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" "subs x25, x25, #0x1\n" - "add x20, x20, #0x20\n" - "ld1b { z4.b }, p4/Z, [x24, x9]\n" - "ld1b { z3.b }, p4/Z, [x23, x9]\n" - "ld1b { z2.b }, p4/Z, [x22, x9]\n" - "ld1b { z1.b }, p4/Z, [x21, x9]\n" - "ld1b { z0.b }, p3/Z, [x24, x28]\n" - "ld1b { z31.b }, p3/Z, [x23, x28]\n" - "ld1b { z22.b }, p3/Z, [x22, x28]\n" - "ld1b { z30.b }, p3/Z, [x21, x28]\n" - "ld1b { z29.b }, p2/Z, [x24, x27]\n" - "ld1b { z28.b }, p2/Z, [x23, x27]\n" - "ld1b { z21.b }, p2/Z, [x22, x27]\n" - "ld1b { z27.b }, p2/Z, [x21, x27]\n" - "ld1b { z26.b }, p1/Z, [x24, x26]\n" - "ld1b { z25.b }, p1/Z, [x23, x26]\n" - "ld1b { z20.b }, p1/Z, [x22, x26]\n" - "ld1b { z24.b }, p1/Z, [x21, x26]\n" + "add x24, x24, #0x20\n" + "ld1b { z4.b }, p4/Z, [x23, x9]\n" + "ld1b { z3.b }, p4/Z, [x22, x9]\n" + "ld1b { z2.b }, p4/Z, [x21, x9]\n" + "ld1b { z1.b }, p4/Z, [x20, x9]\n" + "ld1b { z0.b }, p3/Z, [x23, x28]\n" + "ld1b { z31.b }, p3/Z, [x22, x28]\n" + "ld1b { z22.b }, p3/Z, [x21, x28]\n" + "ld1b { z30.b }, p3/Z, [x20, x28]\n" + "ld1b { z29.b }, p2/Z, [x23, x27]\n" + "ld1b { z28.b }, p2/Z, [x22, x27]\n" + "ld1b { z21.b }, p2/Z, [x21, x27]\n" + "ld1b { z27.b }, p2/Z, [x20, x27]\n" + "ld1b { z26.b }, p1/Z, [x23, x26]\n" + "ld1b { z25.b }, p1/Z, [x22, x26]\n" + "ld1b { z20.b }, p1/Z, [x21, x26]\n" + "ld1b { z24.b }, p1/Z, [x20, x26]\n" "beq 3f\n" "2:" // 4-vectors of channels: 4 inputs loop "movprfx z19, z4\n smax z19.b, p0/M, z19.b, z3.b\n" "movprfx z23, z2\n smax z23.b, p0/M, z23.b, z1.b\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" "movprfx z18, z0\n smax z18.b, p0/M, z18.b, z31.b\n" "smax z22.b, p0/M, z22.b, z30.b\n" - "ld1b { z4.b }, p4/Z, [x24, x9]\n" - "ld1b { z3.b }, p4/Z, [x23, x9]\n" + "ld1b { z4.b }, p4/Z, [x23, x9]\n" + "ld1b { z3.b }, p4/Z, [x22, x9]\n" "movprfx z17, z29\n smax z17.b, p0/M, z17.b, z28.b\n" "smax z21.b, p0/M, z21.b, z27.b\n" - "ld1b { z2.b }, p4/Z, [x22, x9]\n" - "ld1b { z1.b }, p4/Z, [x21, x9]\n" + "ld1b { z2.b }, p4/Z, [x21, x9]\n" + "ld1b { z1.b }, p4/Z, [x20, x9]\n" "movprfx z16, z26\n smax z16.b, p0/M, z16.b, z25.b\n" "smax z20.b, p0/M, z20.b, z24.b\n" - "ld1b { z0.b }, p3/Z, [x24, x28]\n" - "ld1b { z31.b }, p3/Z, [x23, x28]\n" + "ld1b { z0.b }, p3/Z, [x23, x28]\n" + "ld1b { z31.b }, p3/Z, [x22, x28]\n" "smax z19.b, p0/M, z19.b, z23.b\n" "smax z18.b, p0/M, z18.b, z22.b\n" - "ld1b { z22.b }, p3/Z, [x22, x28]\n" - "ld1b { z30.b }, p3/Z, [x21, x28]\n" + "ld1b { z22.b }, p3/Z, [x21, x28]\n" + "ld1b { z30.b }, p3/Z, [x20, x28]\n" "smax z17.b, p0/M, z17.b, z21.b\n" "smax z16.b, p0/M, z16.b, z20.b\n" - "ld1b { z29.b }, p2/Z, [x24, x27]\n" - "ld1b { z28.b }, p2/Z, [x23, x27]\n" + "ld1b { z29.b }, p2/Z, [x23, x27]\n" + "ld1b { z28.b }, p2/Z, [x22, x27]\n" "subs x25, x25, #0x1\n" "smax z8.b, p0/M, z8.b, z19.b\n" - "ld1b { z21.b }, p2/Z, [x22, x27]\n" - "ld1b { z27.b }, p2/Z, [x21, x27]\n" + "ld1b { z21.b }, p2/Z, [x21, x27]\n" + "ld1b { z27.b }, p2/Z, [x20, x27]\n" "smax z7.b, p0/M, z7.b, z18.b\n" "smax z6.b, p0/M, z6.b, z17.b\n" - "ld1b { z26.b }, p1/Z, [x24, x26]\n" - "ld1b { z25.b }, p1/Z, [x23, x26]\n" + "ld1b { z26.b }, p1/Z, [x23, x26]\n" + "ld1b { z25.b }, p1/Z, [x22, x26]\n" "smax z5.b, p0/M, z5.b, z16.b\n" - "add x20, x20, #0x20\n" - "ld1b { z20.b }, p1/Z, [x22, x26]\n" - "ld1b { z24.b }, p1/Z, [x21, x26]\n" + "add x24, x24, #0x20\n" + "ld1b { z20.b }, p1/Z, [x21, x26]\n" + "ld1b { z24.b }, p1/Z, [x20, x26]\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "movprfx z19, z4\n smax z19.b, p0/M, z19.b, z3.b\n" @@ -138,16 +138,16 @@ void sve_s8_nhwc_max_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1b { z4.b }, p4/Z, [x24, x9]\n" + "ldr x20, [x24], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x9]\n" "subs x21, x21, #0x1\n" - "smax z8.b, p0/M, z8.b, z4.b\n" - "ld1b { z0.b }, p3/Z, [x24, x28]\n" - "ld1b { z29.b }, p2/Z, [x24, x27]\n" - "smax z7.b, p0/M, z7.b, z0.b\n" - "smax z6.b, p0/M, z6.b, z29.b\n" - "ld1b { z26.b }, p1/Z, [x24, x26]\n" - "smax z5.b, p0/M, z5.b, z26.b\n" + "smax z8.b, p0/M, z8.b, z16.b\n" + "ld1b { z17.b }, p3/Z, [x20, x28]\n" + "ld1b { z16.b }, p2/Z, [x20, x27]\n" + "smax z7.b, p0/M, z7.b, z17.b\n" + "smax z6.b, p0/M, z6.b, z16.b\n" + "ld1b { z16.b }, p1/Z, [x20, x26]\n" + "smax z5.b, p0/M, z5.b, z16.b\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End "st1b { z8.b }, p4, [%x[outptr], x9]\n" @@ -166,44 +166,44 @@ void sve_s8_nhwc_max_generic_depthfirst_impl( "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" "mov z8.b, #0x80\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" "subs x25, x25, #0x1\n" - "add x20, x20, #0x20\n" - "ld1b { z4.b }, p4/Z, [x24, x9]\n" - "ld1b { z3.b }, p4/Z, [x23, x9]\n" - "ld1b { z2.b }, p4/Z, [x22, x9]\n" - "ld1b { z1.b }, p4/Z, [x21, x9]\n" + "add x24, x24, #0x20\n" + "ld1b { z4.b }, p4/Z, [x23, x9]\n" + "ld1b { z3.b }, p4/Z, [x22, x9]\n" + "ld1b { z2.b }, p4/Z, [x21, x9]\n" + "ld1b { z1.b }, p4/Z, [x20, x9]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 4 inputs loop - "movprfx z19, z4\n smax z19.b, p0/M, z19.b, z3.b\n" - "movprfx z23, z2\n smax z23.b, p0/M, z23.b, z1.b\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" - "smax z19.b, p0/M, z19.b, z23.b\n" + "movprfx z16, z4\n smax z16.b, p0/M, z16.b, z3.b\n" + "movprfx z17, z2\n smax z17.b, p0/M, z17.b, z1.b\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" + "smax z16.b, p0/M, z16.b, z17.b\n" "subs x25, x25, #0x1\n" - "ld1b { z4.b }, p4/Z, [x24, x9]\n" - "ld1b { z3.b }, p4/Z, [x23, x9]\n" - "smax z8.b, p0/M, z8.b, z19.b\n" - "add x20, x20, #0x20\n" - "ld1b { z2.b }, p4/Z, [x22, x9]\n" - "ld1b { z1.b }, p4/Z, [x21, x9]\n" + "ld1b { z4.b }, p4/Z, [x23, x9]\n" + "ld1b { z3.b }, p4/Z, [x22, x9]\n" + "smax z8.b, p0/M, z8.b, z16.b\n" + "add x24, x24, #0x20\n" + "ld1b { z2.b }, p4/Z, [x21, x9]\n" + "ld1b { z1.b }, p4/Z, [x20, x9]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail - "movprfx z19, z4\n smax z19.b, p0/M, z19.b, z3.b\n" - "movprfx z23, z2\n smax z23.b, p0/M, z23.b, z1.b\n" - "smax z19.b, p0/M, z19.b, z23.b\n" - "smax z8.b, p0/M, z8.b, z19.b\n" + "movprfx z16, z4\n smax z16.b, p0/M, z16.b, z3.b\n" + "movprfx z17, z2\n smax z17.b, p0/M, z17.b, z1.b\n" + "smax z16.b, p0/M, z16.b, z17.b\n" + "smax z8.b, p0/M, z8.b, z16.b\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1b { z4.b }, p4/Z, [x24, x9]\n" + "ldr x20, [x24], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x9]\n" "subs x21, x21, #0x1\n" - "smax z8.b, p0/M, z8.b, z4.b\n" + "smax z8.b, p0/M, z8.b, z16.b\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "st1b { z8.b }, p4, [%x[outptr], x9]\n" diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_avg_generic_depthfirst/generic.cpp index 7161c4f389e8538bc0f2aad6637e9ee57599a681..19a3b112ad24f5795f923734941cd9e8724f24d3 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_avg_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_avg_generic_depthfirst/generic.cpp @@ -128,7 +128,7 @@ void sve_s8q_nhwc_avg_generic_depthfirst_impl( "lsr x23, %x[n_valid_cells], #0x1\n" "mov z15.s, #0x0\n" "mov z14.s, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "mov z13.s, #0x0\n" "mov z12.s, #0x0\n" "mov z11.s, #0x0\n" @@ -144,42 +144,42 @@ void sve_s8q_nhwc_avg_generic_depthfirst_impl( "mov z1.s, #0x0\n" "mov z0.s, #0x0\n" "cbz x23, 4f\n" - "ldp x22, x21, [x20, #0x0]\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" - "ld1b { z29.b }, p3/Z, [x22, x26]\n" - "ld1b { z28.b }, p3/Z, [x21, x26]\n" - "ld1b { z27.b }, p2/Z, [x22, x25]\n" - "ld1b { z26.b }, p2/Z, [x21, x25]\n" - "ld1b { z25.b }, p1/Z, [x22, x24]\n" - "ld1b { z24.b }, p1/Z, [x21, x24]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" + "ld1b { z29.b }, p3/Z, [x21, x26]\n" + "ld1b { z28.b }, p3/Z, [x20, x26]\n" + "ld1b { z27.b }, p2/Z, [x21, x25]\n" + "ld1b { z26.b }, p2/Z, [x20, x25]\n" + "ld1b { z25.b }, p1/Z, [x21, x24]\n" + "ld1b { z24.b }, p1/Z, [x20, x24]\n" "beq 3f\n" "2:" // 4-vectors of channels: 2 inputs loop ".inst 0x455e03f7 // saddlb z23.h, z31.b, z30.b\n" ".inst 0x455e07f6 // saddlt z22.h, z31.b, z30.b\n" - "ldp x22, x21, [x20, #0x0]\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" ".inst 0x455c03b5 // saddlb z21.h, z29.b, z28.b\n" ".inst 0x455c07b4 // saddlt z20.h, z29.b, z28.b\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" ".inst 0x455a0373 // saddlb z19.h, z27.b, z26.b\n" ".inst 0x455a0772 // saddlt z18.h, z27.b, z26.b\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" - "ld1b { z29.b }, p3/Z, [x22, x26]\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" + "ld1b { z29.b }, p3/Z, [x21, x26]\n" ".inst 0x45580331 // saddlb z17.h, z25.b, z24.b\n" ".inst 0x45580730 // saddlt z16.h, z25.b, z24.b\n" - "ld1b { z28.b }, p3/Z, [x21, x26]\n" - "ld1b { z27.b }, p2/Z, [x22, x25]\n" + "ld1b { z28.b }, p3/Z, [x20, x26]\n" + "ld1b { z27.b }, p2/Z, [x21, x25]\n" ".inst 0x459741ef // saddwb z15.s, z15.s, z23.h\n" ".inst 0x459745ce // saddwt z14.s, z14.s, z23.h\n" - "ld1b { z26.b }, p2/Z, [x21, x25]\n" - "ld1b { z25.b }, p1/Z, [x22, x24]\n" + "ld1b { z26.b }, p2/Z, [x20, x25]\n" + "ld1b { z25.b }, p1/Z, [x21, x24]\n" ".inst 0x459641ad // saddwb z13.s, z13.s, z22.h\n" ".inst 0x4596458c // saddwt z12.s, z12.s, z22.h\n" - "ld1b { z24.b }, p1/Z, [x21, x24]\n" + "ld1b { z24.b }, p1/Z, [x20, x24]\n" ".inst 0x4595416b // saddwb z11.s, z11.s, z21.h\n" ".inst 0x4595454a // saddwt z10.s, z10.s, z21.h\n" ".inst 0x45944129 // saddwb z9.s, z9.s, z20.h\n" @@ -222,20 +222,20 @@ void sve_s8q_nhwc_avg_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x1\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x22, [x20], #0x8\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - ".inst 0x4508a3f7 // sshllb z23.h, z31.b, #0x0\n" - ".inst 0x4508a7f6 // sshllt z22.h, z31.b, #0x0\n" - "ld1b { z29.b }, p3/Z, [x22, x26]\n" - "ld1b { z27.b }, p2/Z, [x22, x25]\n" - ".inst 0x4508a3b5 // sshllb z21.h, z29.b, #0x0\n" - ".inst 0x4508a7b4 // sshllt z20.h, z29.b, #0x0\n" - "ld1b { z25.b }, p1/Z, [x22, x24]\n" - ".inst 0x4508a373 // sshllb z19.h, z27.b, #0x0\n" - ".inst 0x4508a772 // sshllt z18.h, z27.b, #0x0\n" + "ldr x20, [x22], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x27]\n" + ".inst 0x4508a217 // sshllb z23.h, z16.b, #0x0\n" + ".inst 0x4508a616 // sshllt z22.h, z16.b, #0x0\n" + "ld1b { z16.b }, p3/Z, [x20, x26]\n" + "ld1b { z17.b }, p2/Z, [x20, x25]\n" + ".inst 0x4508a215 // sshllb z21.h, z16.b, #0x0\n" + ".inst 0x4508a614 // sshllt z20.h, z16.b, #0x0\n" + "ld1b { z16.b }, p1/Z, [x20, x24]\n" + ".inst 0x4508a233 // sshllb z19.h, z17.b, #0x0\n" + ".inst 0x4508a632 // sshllt z18.h, z17.b, #0x0\n" "subs x21, x21, #0x1\n" - ".inst 0x4508a331 // sshllb z17.h, z25.b, #0x0\n" - ".inst 0x4508a730 // sshllt z16.h, z25.b, #0x0\n" + ".inst 0x4508a211 // sshllb z17.h, z16.b, #0x0\n" + ".inst 0x4508a610 // sshllt z16.h, z16.b, #0x0\n" ".inst 0x459741ef // saddwb z15.s, z15.s, z23.h\n" ".inst 0x459745ce // saddwt z14.s, z14.s, z23.h\n" ".inst 0x459641ad // saddwb z13.s, z13.s, z22.h\n" @@ -368,57 +368,57 @@ void sve_s8q_nhwc_avg_generic_depthfirst_impl( "lsr x23, %x[n_valid_cells], #0x1\n" "mov z15.s, #0x0\n" "mov z14.s, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "mov z13.s, #0x0\n" "mov z12.s, #0x0\n" "cbz x23, 11f\n" - "ldp x22, x21, [x20, #0x0]\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 2 inputs loop - ".inst 0x455e03f7 // saddlb z23.h, z31.b, z30.b\n" - ".inst 0x455e07f6 // saddlt z22.h, z31.b, z30.b\n" - "ldp x22, x21, [x20, #0x0]\n" + ".inst 0x455e03f1 // saddlb z17.h, z31.b, z30.b\n" + ".inst 0x455e07f0 // saddlt z16.h, z31.b, z30.b\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - ".inst 0x459741ef // saddwb z15.s, z15.s, z23.h\n" - ".inst 0x459745ce // saddwt z14.s, z14.s, z23.h\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - ".inst 0x459641ad // saddwb z13.s, z13.s, z22.h\n" - ".inst 0x4596458c // saddwt z12.s, z12.s, z22.h\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" + ".inst 0x459141ef // saddwb z15.s, z15.s, z17.h\n" + ".inst 0x459145ce // saddwt z14.s, z14.s, z17.h\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" + ".inst 0x459041ad // saddwb z13.s, z13.s, z16.h\n" + ".inst 0x4590458c // saddwt z12.s, z12.s, z16.h\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 2 inputs tail - ".inst 0x455e03f7 // saddlb z23.h, z31.b, z30.b\n" - ".inst 0x455e07f6 // saddlt z22.h, z31.b, z30.b\n" - ".inst 0x459741ef // saddwb z15.s, z15.s, z23.h\n" - ".inst 0x459745ce // saddwt z14.s, z14.s, z23.h\n" - ".inst 0x459641ad // saddwb z13.s, z13.s, z22.h\n" - ".inst 0x4596458c // saddwt z12.s, z12.s, z22.h\n" + ".inst 0x455e03f1 // saddlb z17.h, z31.b, z30.b\n" + ".inst 0x455e07f0 // saddlt z16.h, z31.b, z30.b\n" + ".inst 0x459141ef // saddwb z15.s, z15.s, z17.h\n" + ".inst 0x459145ce // saddwt z14.s, z14.s, z17.h\n" + ".inst 0x459041ad // saddwb z13.s, z13.s, z16.h\n" + ".inst 0x4590458c // saddwt z12.s, z12.s, z16.h\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x1\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x22, [x20], #0x8\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - ".inst 0x4508a3f7 // sshllb z23.h, z31.b, #0x0\n" - ".inst 0x4508a7f6 // sshllt z22.h, z31.b, #0x0\n" + "ldr x20, [x22], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x27]\n" + ".inst 0x4508a211 // sshllb z17.h, z16.b, #0x0\n" + ".inst 0x4508a610 // sshllt z16.h, z16.b, #0x0\n" "subs x21, x21, #0x1\n" - ".inst 0x459741ef // saddwb z15.s, z15.s, z23.h\n" - ".inst 0x459745ce // saddwt z14.s, z14.s, z23.h\n" - ".inst 0x459641ad // saddwb z13.s, z13.s, z22.h\n" - ".inst 0x4596458c // saddwt z12.s, z12.s, z22.h\n" + ".inst 0x459141ef // saddwb z15.s, z15.s, z17.h\n" + ".inst 0x459145ce // saddwt z14.s, z14.s, z17.h\n" + ".inst 0x459041ad // saddwb z13.s, z13.s, z16.h\n" + ".inst 0x4590458c // saddwt z12.s, z12.s, z16.h\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End - "ld1rw { z18.s }, p0/Z, [%x[left_shift]]\n" + "ld1rw { z16.s }, p0/Z, [%x[left_shift]]\n" "ld1rw { z17.s }, p0/Z, [%x[combined_rescale_value]]\n" - ".inst 0x4482824f // srshl z15.s, p0/M, z15.s, z18.s\n" - ".inst 0x4482824e // srshl z14.s, p0/M, z14.s, z18.s\n" - ".inst 0x4482824d // srshl z13.s, p0/M, z13.s, z18.s\n" - ".inst 0x4482824c // srshl z12.s, p0/M, z12.s, z18.s\n" + ".inst 0x4482820f // srshl z15.s, p0/M, z15.s, z16.s\n" + ".inst 0x4482820e // srshl z14.s, p0/M, z14.s, z16.s\n" + ".inst 0x4482820d // srshl z13.s, p0/M, z13.s, z16.s\n" + ".inst 0x4482820c // srshl z12.s, p0/M, z12.s, z16.s\n" "ld1rw { z16.s }, p0/Z, [%x[right_shift]]\n" ".inst 0x04b175ef // sqrdmulh z15.s, z15.s, z17.s\n" ".inst 0x04b175ce // sqrdmulh z14.s, z14.s, z17.s\n" diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_max_generic_depthfirst/generic.cpp index 19209811d8c4ae68468a409e02d7b5cd634be841..4fc1532d5a8895ff2ccc55d1325743a5dd4d3d37 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_max_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_max_generic_depthfirst/generic.cpp @@ -56,68 +56,68 @@ void sve_s8q_nhwc_max_generic_depthfirst_impl( "lsr x25, %x[n_valid_cells], #0x2\n" "mov z8.b, #0x80\n" "mov z7.b, #0x80\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "mov z6.b, #0x80\n" "mov z5.b, #0x80\n" "cbz x25, 4f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" "subs x25, x25, #0x1\n" - "add x20, x20, #0x20\n" - "ld1b { z4.b }, p4/Z, [x24, x9]\n" - "ld1b { z3.b }, p4/Z, [x23, x9]\n" - "ld1b { z2.b }, p4/Z, [x22, x9]\n" - "ld1b { z1.b }, p4/Z, [x21, x9]\n" - "ld1b { z0.b }, p3/Z, [x24, x28]\n" - "ld1b { z31.b }, p3/Z, [x23, x28]\n" - "ld1b { z22.b }, p3/Z, [x22, x28]\n" - "ld1b { z30.b }, p3/Z, [x21, x28]\n" - "ld1b { z29.b }, p2/Z, [x24, x27]\n" - "ld1b { z28.b }, p2/Z, [x23, x27]\n" - "ld1b { z21.b }, p2/Z, [x22, x27]\n" - "ld1b { z27.b }, p2/Z, [x21, x27]\n" - "ld1b { z26.b }, p1/Z, [x24, x26]\n" - "ld1b { z25.b }, p1/Z, [x23, x26]\n" - "ld1b { z20.b }, p1/Z, [x22, x26]\n" - "ld1b { z24.b }, p1/Z, [x21, x26]\n" + "add x24, x24, #0x20\n" + "ld1b { z4.b }, p4/Z, [x23, x9]\n" + "ld1b { z3.b }, p4/Z, [x22, x9]\n" + "ld1b { z2.b }, p4/Z, [x21, x9]\n" + "ld1b { z1.b }, p4/Z, [x20, x9]\n" + "ld1b { z0.b }, p3/Z, [x23, x28]\n" + "ld1b { z31.b }, p3/Z, [x22, x28]\n" + "ld1b { z22.b }, p3/Z, [x21, x28]\n" + "ld1b { z30.b }, p3/Z, [x20, x28]\n" + "ld1b { z29.b }, p2/Z, [x23, x27]\n" + "ld1b { z28.b }, p2/Z, [x22, x27]\n" + "ld1b { z21.b }, p2/Z, [x21, x27]\n" + "ld1b { z27.b }, p2/Z, [x20, x27]\n" + "ld1b { z26.b }, p1/Z, [x23, x26]\n" + "ld1b { z25.b }, p1/Z, [x22, x26]\n" + "ld1b { z20.b }, p1/Z, [x21, x26]\n" + "ld1b { z24.b }, p1/Z, [x20, x26]\n" "beq 3f\n" "2:" // 4-vectors of channels: 4 inputs loop "movprfx z19, z4\n smax z19.b, p0/M, z19.b, z3.b\n" "movprfx z23, z2\n smax z23.b, p0/M, z23.b, z1.b\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" "movprfx z18, z0\n smax z18.b, p0/M, z18.b, z31.b\n" "smax z22.b, p0/M, z22.b, z30.b\n" - "ld1b { z4.b }, p4/Z, [x24, x9]\n" - "ld1b { z3.b }, p4/Z, [x23, x9]\n" + "ld1b { z4.b }, p4/Z, [x23, x9]\n" + "ld1b { z3.b }, p4/Z, [x22, x9]\n" "movprfx z17, z29\n smax z17.b, p0/M, z17.b, z28.b\n" "smax z21.b, p0/M, z21.b, z27.b\n" - "ld1b { z2.b }, p4/Z, [x22, x9]\n" - "ld1b { z1.b }, p4/Z, [x21, x9]\n" + "ld1b { z2.b }, p4/Z, [x21, x9]\n" + "ld1b { z1.b }, p4/Z, [x20, x9]\n" "movprfx z16, z26\n smax z16.b, p0/M, z16.b, z25.b\n" "smax z20.b, p0/M, z20.b, z24.b\n" - "ld1b { z0.b }, p3/Z, [x24, x28]\n" - "ld1b { z31.b }, p3/Z, [x23, x28]\n" + "ld1b { z0.b }, p3/Z, [x23, x28]\n" + "ld1b { z31.b }, p3/Z, [x22, x28]\n" "smax z19.b, p0/M, z19.b, z23.b\n" "smax z18.b, p0/M, z18.b, z22.b\n" - "ld1b { z22.b }, p3/Z, [x22, x28]\n" - "ld1b { z30.b }, p3/Z, [x21, x28]\n" + "ld1b { z22.b }, p3/Z, [x21, x28]\n" + "ld1b { z30.b }, p3/Z, [x20, x28]\n" "smax z17.b, p0/M, z17.b, z21.b\n" "smax z16.b, p0/M, z16.b, z20.b\n" - "ld1b { z29.b }, p2/Z, [x24, x27]\n" - "ld1b { z28.b }, p2/Z, [x23, x27]\n" + "ld1b { z29.b }, p2/Z, [x23, x27]\n" + "ld1b { z28.b }, p2/Z, [x22, x27]\n" "subs x25, x25, #0x1\n" "smax z8.b, p0/M, z8.b, z19.b\n" - "ld1b { z21.b }, p2/Z, [x22, x27]\n" - "ld1b { z27.b }, p2/Z, [x21, x27]\n" + "ld1b { z21.b }, p2/Z, [x21, x27]\n" + "ld1b { z27.b }, p2/Z, [x20, x27]\n" "smax z7.b, p0/M, z7.b, z18.b\n" "smax z6.b, p0/M, z6.b, z17.b\n" - "ld1b { z26.b }, p1/Z, [x24, x26]\n" - "ld1b { z25.b }, p1/Z, [x23, x26]\n" + "ld1b { z26.b }, p1/Z, [x23, x26]\n" + "ld1b { z25.b }, p1/Z, [x22, x26]\n" "smax z5.b, p0/M, z5.b, z16.b\n" - "add x20, x20, #0x20\n" - "ld1b { z20.b }, p1/Z, [x22, x26]\n" - "ld1b { z24.b }, p1/Z, [x21, x26]\n" + "add x24, x24, #0x20\n" + "ld1b { z20.b }, p1/Z, [x21, x26]\n" + "ld1b { z24.b }, p1/Z, [x20, x26]\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "movprfx z19, z4\n smax z19.b, p0/M, z19.b, z3.b\n" @@ -140,16 +140,16 @@ void sve_s8q_nhwc_max_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1b { z4.b }, p4/Z, [x24, x9]\n" + "ldr x20, [x24], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x9]\n" "subs x21, x21, #0x1\n" - "smax z8.b, p0/M, z8.b, z4.b\n" - "ld1b { z0.b }, p3/Z, [x24, x28]\n" - "ld1b { z29.b }, p2/Z, [x24, x27]\n" - "smax z7.b, p0/M, z7.b, z0.b\n" - "smax z6.b, p0/M, z6.b, z29.b\n" - "ld1b { z26.b }, p1/Z, [x24, x26]\n" - "smax z5.b, p0/M, z5.b, z26.b\n" + "smax z8.b, p0/M, z8.b, z16.b\n" + "ld1b { z17.b }, p3/Z, [x20, x28]\n" + "ld1b { z16.b }, p2/Z, [x20, x27]\n" + "smax z7.b, p0/M, z7.b, z17.b\n" + "smax z6.b, p0/M, z6.b, z16.b\n" + "ld1b { z16.b }, p1/Z, [x20, x26]\n" + "smax z5.b, p0/M, z5.b, z16.b\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End ".inst 0x4508a111 // sshllb z17.h, z8.b, #0x0\n" @@ -292,82 +292,82 @@ void sve_s8q_nhwc_max_generic_depthfirst_impl( "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" "mov z8.b, #0x80\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" "subs x25, x25, #0x1\n" - "add x20, x20, #0x20\n" - "ld1b { z4.b }, p4/Z, [x24, x9]\n" - "ld1b { z3.b }, p4/Z, [x23, x9]\n" - "ld1b { z2.b }, p4/Z, [x22, x9]\n" - "ld1b { z1.b }, p4/Z, [x21, x9]\n" + "add x24, x24, #0x20\n" + "ld1b { z4.b }, p4/Z, [x23, x9]\n" + "ld1b { z3.b }, p4/Z, [x22, x9]\n" + "ld1b { z2.b }, p4/Z, [x21, x9]\n" + "ld1b { z1.b }, p4/Z, [x20, x9]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 4 inputs loop - "movprfx z19, z4\n smax z19.b, p0/M, z19.b, z3.b\n" - "movprfx z23, z2\n smax z23.b, p0/M, z23.b, z1.b\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" - "smax z19.b, p0/M, z19.b, z23.b\n" + "movprfx z16, z4\n smax z16.b, p0/M, z16.b, z3.b\n" + "movprfx z17, z2\n smax z17.b, p0/M, z17.b, z1.b\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" + "smax z16.b, p0/M, z16.b, z17.b\n" "subs x25, x25, #0x1\n" - "ld1b { z4.b }, p4/Z, [x24, x9]\n" - "ld1b { z3.b }, p4/Z, [x23, x9]\n" - "smax z8.b, p0/M, z8.b, z19.b\n" - "add x20, x20, #0x20\n" - "ld1b { z2.b }, p4/Z, [x22, x9]\n" - "ld1b { z1.b }, p4/Z, [x21, x9]\n" + "ld1b { z4.b }, p4/Z, [x23, x9]\n" + "ld1b { z3.b }, p4/Z, [x22, x9]\n" + "smax z8.b, p0/M, z8.b, z16.b\n" + "add x24, x24, #0x20\n" + "ld1b { z2.b }, p4/Z, [x21, x9]\n" + "ld1b { z1.b }, p4/Z, [x20, x9]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail - "movprfx z19, z4\n smax z19.b, p0/M, z19.b, z3.b\n" - "movprfx z23, z2\n smax z23.b, p0/M, z23.b, z1.b\n" - "smax z19.b, p0/M, z19.b, z23.b\n" - "smax z8.b, p0/M, z8.b, z19.b\n" + "movprfx z16, z4\n smax z16.b, p0/M, z16.b, z3.b\n" + "movprfx z17, z2\n smax z17.b, p0/M, z17.b, z1.b\n" + "smax z16.b, p0/M, z16.b, z17.b\n" + "smax z8.b, p0/M, z8.b, z16.b\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1b { z4.b }, p4/Z, [x24, x9]\n" + "ldr x20, [x24], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x9]\n" "subs x21, x21, #0x1\n" - "smax z8.b, p0/M, z8.b, z4.b\n" + "smax z8.b, p0/M, z8.b, z16.b\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End ".inst 0x4508a111 // sshllb z17.h, z8.b, #0x0\n" - ".inst 0x4508a517 // sshllt z23.h, z8.b, #0x0\n" + ".inst 0x4508a512 // sshllt z18.h, z8.b, #0x0\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n" - "ld1rw { z4.s }, p0/Z, [x20]\n" - ".inst 0x4510a221 // sshllb z1.s, z17.h, #0x0\n" - ".inst 0x4510a631 // sshllt z17.s, z17.h, #0x0\n" + "ld1rw { z16.s }, p0/Z, [x20]\n" + ".inst 0x4510a236 // sshllb z22.s, z17.h, #0x0\n" + ".inst 0x4510a635 // sshllt z21.s, z17.h, #0x0\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_mul]\n" - "ld1rw { z3.s }, p0/Z, [x20]\n" - ".inst 0x4510a2e0 // sshllb z0.s, z23.h, #0x0\n" - ".inst 0x4510a6ff // sshllt z31.s, z23.h, #0x0\n" - ".inst 0x44828081 // srshl z1.s, p0/M, z1.s, z4.s\n" - ".inst 0x44828091 // srshl z17.s, p0/M, z17.s, z4.s\n" - ".inst 0x44828080 // srshl z0.s, p0/M, z0.s, z4.s\n" - ".inst 0x4482809f // srshl z31.s, p0/M, z31.s, z4.s\n" - ".inst 0x04a37421 // sqrdmulh z1.s, z1.s, z3.s\n" - ".inst 0x04a37631 // sqrdmulh z17.s, z17.s, z3.s\n" + "ld1rw { z17.s }, p0/Z, [x20]\n" + ".inst 0x4510a254 // sshllb z20.s, z18.h, #0x0\n" + ".inst 0x4510a653 // sshllt z19.s, z18.h, #0x0\n" + ".inst 0x44828216 // srshl z22.s, p0/M, z22.s, z16.s\n" + ".inst 0x44828215 // srshl z21.s, p0/M, z21.s, z16.s\n" + ".inst 0x44828214 // srshl z20.s, p0/M, z20.s, z16.s\n" + ".inst 0x44828213 // srshl z19.s, p0/M, z19.s, z16.s\n" + ".inst 0x04b176d6 // sqrdmulh z22.s, z22.s, z17.s\n" + ".inst 0x04b176b5 // sqrdmulh z21.s, z21.s, z17.s\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n" - "ld1rw { z2.s }, p0/Z, [x20]\n" - ".inst 0x04a37400 // sqrdmulh z0.s, z0.s, z3.s\n" - ".inst 0x04a377ff // sqrdmulh z31.s, z31.s, z3.s\n" + "ld1rw { z16.s }, p0/Z, [x20]\n" + ".inst 0x04b17694 // sqrdmulh z20.s, z20.s, z17.s\n" + ".inst 0x04b17673 // sqrdmulh z19.s, z19.s, z17.s\n" "mov z18.s, #0x7f\n" - ".inst 0x44828041 // srshl z1.s, p0/M, z1.s, z2.s\n" - ".inst 0x44828051 // srshl z17.s, p0/M, z17.s, z2.s\n" - ".inst 0x44828040 // srshl z0.s, p0/M, z0.s, z2.s\n" - ".inst 0x4482805f // srshl z31.s, p0/M, z31.s, z2.s\n" + ".inst 0x44828216 // srshl z22.s, p0/M, z22.s, z16.s\n" + ".inst 0x44828215 // srshl z21.s, p0/M, z21.s, z16.s\n" + ".inst 0x44828214 // srshl z20.s, p0/M, z20.s, z16.s\n" + ".inst 0x44828213 // srshl z19.s, p0/M, z19.s, z16.s\n" "not z16.s, p0/M, z18.s\n" - "smax z1.s, p0/M, z1.s, z16.s\n" - "smax z17.s, p0/M, z17.s, z16.s\n" - "smax z0.s, p0/M, z0.s, z16.s\n" - "smax z31.s, p0/M, z31.s, z16.s\n" - "smin z1.s, p0/M, z1.s, z18.s\n" - "smin z17.s, p0/M, z17.s, z18.s\n" - "smin z0.s, p0/M, z0.s, z18.s\n" - "trn1 z17.h, z1.h, z17.h\n" - "smin z31.s, p0/M, z31.s, z18.s\n" - "trn1 z16.h, z0.h, z31.h\n" + "smax z22.s, p0/M, z22.s, z16.s\n" + "smax z21.s, p0/M, z21.s, z16.s\n" + "smax z20.s, p0/M, z20.s, z16.s\n" + "smax z19.s, p0/M, z19.s, z16.s\n" + "smin z22.s, p0/M, z22.s, z18.s\n" + "smin z21.s, p0/M, z21.s, z18.s\n" + "smin z20.s, p0/M, z20.s, z18.s\n" + "trn1 z17.h, z22.h, z21.h\n" + "smin z19.s, p0/M, z19.s, z18.s\n" + "trn1 z16.h, z20.h, z19.h\n" "trn1 z16.b, z17.b, z16.b\n" "st1b { z16.b }, p4, [%x[outptr], x9]\n" "incb x9\n" diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_avg_generic_depthfirst/generic.cpp index f888038a2a4496b6e7573981b5c760d2d91450fe..f3f4950a1f98c9222557b2f3fe1e44ebdfa200bc 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_avg_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_avg_generic_depthfirst/generic.cpp @@ -109,7 +109,7 @@ void sve_u8_nhwc_avg_generic_depthfirst_impl( "lsr x23, %x[n_valid_cells], #0x1\n" "mov z15.s, #0x0\n" "mov z14.s, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "mov z13.s, #0x0\n" "mov z12.s, #0x0\n" "mov z11.s, #0x0\n" @@ -125,42 +125,42 @@ void sve_u8_nhwc_avg_generic_depthfirst_impl( "mov z1.s, #0x0\n" "mov z0.s, #0x0\n" "cbz x23, 4f\n" - "ldp x22, x21, [x20, #0x0]\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" - "ld1b { z29.b }, p3/Z, [x22, x26]\n" - "ld1b { z28.b }, p3/Z, [x21, x26]\n" - "ld1b { z27.b }, p2/Z, [x22, x25]\n" - "ld1b { z26.b }, p2/Z, [x21, x25]\n" - "ld1b { z25.b }, p1/Z, [x22, x24]\n" - "ld1b { z24.b }, p1/Z, [x21, x24]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" + "ld1b { z29.b }, p3/Z, [x21, x26]\n" + "ld1b { z28.b }, p3/Z, [x20, x26]\n" + "ld1b { z27.b }, p2/Z, [x21, x25]\n" + "ld1b { z26.b }, p2/Z, [x20, x25]\n" + "ld1b { z25.b }, p1/Z, [x21, x24]\n" + "ld1b { z24.b }, p1/Z, [x20, x24]\n" "beq 3f\n" "2:" // 4-vectors of channels: 2 inputs loop ".inst 0x455e0bf7 // uaddlb z23.h, z31.b, z30.b\n" ".inst 0x455e0ff6 // uaddlt z22.h, z31.b, z30.b\n" - "ldp x22, x21, [x20, #0x0]\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" ".inst 0x455c0bb5 // uaddlb z21.h, z29.b, z28.b\n" ".inst 0x455c0fb4 // uaddlt z20.h, z29.b, z28.b\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" ".inst 0x455a0b73 // uaddlb z19.h, z27.b, z26.b\n" ".inst 0x455a0f72 // uaddlt z18.h, z27.b, z26.b\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" - "ld1b { z29.b }, p3/Z, [x22, x26]\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" + "ld1b { z29.b }, p3/Z, [x21, x26]\n" ".inst 0x45580b31 // uaddlb z17.h, z25.b, z24.b\n" ".inst 0x45580f30 // uaddlt z16.h, z25.b, z24.b\n" - "ld1b { z28.b }, p3/Z, [x21, x26]\n" - "ld1b { z27.b }, p2/Z, [x22, x25]\n" + "ld1b { z28.b }, p3/Z, [x20, x26]\n" + "ld1b { z27.b }, p2/Z, [x21, x25]\n" ".inst 0x459749ef // uaddwb z15.s, z15.s, z23.h\n" ".inst 0x45974dce // uaddwt z14.s, z14.s, z23.h\n" - "ld1b { z26.b }, p2/Z, [x21, x25]\n" - "ld1b { z25.b }, p1/Z, [x22, x24]\n" + "ld1b { z26.b }, p2/Z, [x20, x25]\n" + "ld1b { z25.b }, p1/Z, [x21, x24]\n" ".inst 0x459649ad // uaddwb z13.s, z13.s, z22.h\n" ".inst 0x45964d8c // uaddwt z12.s, z12.s, z22.h\n" - "ld1b { z24.b }, p1/Z, [x21, x24]\n" + "ld1b { z24.b }, p1/Z, [x20, x24]\n" ".inst 0x4595496b // uaddwb z11.s, z11.s, z21.h\n" ".inst 0x45954d4a // uaddwt z10.s, z10.s, z21.h\n" ".inst 0x45944929 // uaddwb z9.s, z9.s, z20.h\n" @@ -203,20 +203,20 @@ void sve_u8_nhwc_avg_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x1\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x22, [x20], #0x8\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - ".inst 0x4508abf7 // ushllb z23.h, z31.b, #0x0\n" - ".inst 0x4508aff6 // ushllt z22.h, z31.b, #0x0\n" - "ld1b { z29.b }, p3/Z, [x22, x26]\n" - "ld1b { z27.b }, p2/Z, [x22, x25]\n" - ".inst 0x4508abb5 // ushllb z21.h, z29.b, #0x0\n" - ".inst 0x4508afb4 // ushllt z20.h, z29.b, #0x0\n" - "ld1b { z25.b }, p1/Z, [x22, x24]\n" - ".inst 0x4508ab73 // ushllb z19.h, z27.b, #0x0\n" - ".inst 0x4508af72 // ushllt z18.h, z27.b, #0x0\n" + "ldr x20, [x22], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x27]\n" + ".inst 0x4508aa17 // ushllb z23.h, z16.b, #0x0\n" + ".inst 0x4508ae16 // ushllt z22.h, z16.b, #0x0\n" + "ld1b { z16.b }, p3/Z, [x20, x26]\n" + "ld1b { z17.b }, p2/Z, [x20, x25]\n" + ".inst 0x4508aa15 // ushllb z21.h, z16.b, #0x0\n" + ".inst 0x4508ae14 // ushllt z20.h, z16.b, #0x0\n" + "ld1b { z16.b }, p1/Z, [x20, x24]\n" + ".inst 0x4508aa33 // ushllb z19.h, z17.b, #0x0\n" + ".inst 0x4508ae32 // ushllt z18.h, z17.b, #0x0\n" "subs x21, x21, #0x1\n" - ".inst 0x4508ab31 // ushllb z17.h, z25.b, #0x0\n" - ".inst 0x4508af30 // ushllt z16.h, z25.b, #0x0\n" + ".inst 0x4508aa11 // ushllb z17.h, z16.b, #0x0\n" + ".inst 0x4508ae10 // ushllt z16.h, z16.b, #0x0\n" ".inst 0x459749ef // uaddwb z15.s, z15.s, z23.h\n" ".inst 0x45974dce // uaddwt z14.s, z14.s, z23.h\n" ".inst 0x459649ad // uaddwb z13.s, z13.s, z22.h\n" @@ -332,49 +332,49 @@ void sve_u8_nhwc_avg_generic_depthfirst_impl( "lsr x23, %x[n_valid_cells], #0x1\n" "mov z15.s, #0x0\n" "mov z14.s, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "mov z13.s, #0x0\n" "mov z12.s, #0x0\n" "cbz x23, 11f\n" - "ldp x22, x21, [x20, #0x0]\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 2 inputs loop - ".inst 0x455e0bf7 // uaddlb z23.h, z31.b, z30.b\n" - ".inst 0x455e0ff6 // uaddlt z22.h, z31.b, z30.b\n" - "ldp x22, x21, [x20, #0x0]\n" + ".inst 0x455e0bf1 // uaddlb z17.h, z31.b, z30.b\n" + ".inst 0x455e0ff0 // uaddlt z16.h, z31.b, z30.b\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - ".inst 0x459749ef // uaddwb z15.s, z15.s, z23.h\n" - ".inst 0x45974dce // uaddwt z14.s, z14.s, z23.h\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - ".inst 0x459649ad // uaddwb z13.s, z13.s, z22.h\n" - ".inst 0x45964d8c // uaddwt z12.s, z12.s, z22.h\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" + ".inst 0x459149ef // uaddwb z15.s, z15.s, z17.h\n" + ".inst 0x45914dce // uaddwt z14.s, z14.s, z17.h\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" + ".inst 0x459049ad // uaddwb z13.s, z13.s, z16.h\n" + ".inst 0x45904d8c // uaddwt z12.s, z12.s, z16.h\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 2 inputs tail - ".inst 0x455e0bf7 // uaddlb z23.h, z31.b, z30.b\n" - ".inst 0x455e0ff6 // uaddlt z22.h, z31.b, z30.b\n" - ".inst 0x459749ef // uaddwb z15.s, z15.s, z23.h\n" - ".inst 0x45974dce // uaddwt z14.s, z14.s, z23.h\n" - ".inst 0x459649ad // uaddwb z13.s, z13.s, z22.h\n" - ".inst 0x45964d8c // uaddwt z12.s, z12.s, z22.h\n" + ".inst 0x455e0bf1 // uaddlb z17.h, z31.b, z30.b\n" + ".inst 0x455e0ff0 // uaddlt z16.h, z31.b, z30.b\n" + ".inst 0x459149ef // uaddwb z15.s, z15.s, z17.h\n" + ".inst 0x45914dce // uaddwt z14.s, z14.s, z17.h\n" + ".inst 0x459049ad // uaddwb z13.s, z13.s, z16.h\n" + ".inst 0x45904d8c // uaddwt z12.s, z12.s, z16.h\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x1\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x22, [x20], #0x8\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - ".inst 0x4508abf7 // ushllb z23.h, z31.b, #0x0\n" - ".inst 0x4508aff6 // ushllt z22.h, z31.b, #0x0\n" + "ldr x20, [x22], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x27]\n" + ".inst 0x4508aa11 // ushllb z17.h, z16.b, #0x0\n" + ".inst 0x4508ae10 // ushllt z16.h, z16.b, #0x0\n" "subs x21, x21, #0x1\n" - ".inst 0x459749ef // uaddwb z15.s, z15.s, z23.h\n" - ".inst 0x45974dce // uaddwt z14.s, z14.s, z23.h\n" - ".inst 0x459649ad // uaddwb z13.s, z13.s, z22.h\n" - ".inst 0x45964d8c // uaddwt z12.s, z12.s, z22.h\n" + ".inst 0x459149ef // uaddwb z15.s, z15.s, z17.h\n" + ".inst 0x45914dce // uaddwt z14.s, z14.s, z17.h\n" + ".inst 0x459049ad // uaddwb z13.s, z13.s, z16.h\n" + ".inst 0x45904d8c // uaddwt z12.s, z12.s, z16.h\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "ld1rw { z17.s }, p0/Z, [%x[rescale_ptr]]\n" @@ -387,17 +387,17 @@ void sve_u8_nhwc_avg_generic_depthfirst_impl( ".inst 0x4482820e // srshl z14.s, p0/M, z14.s, z16.s\n" ".inst 0x4482820d // srshl z13.s, p0/M, z13.s, z16.s\n" ".inst 0x4482820c // srshl z12.s, p0/M, z12.s, z16.s\n" - "mov z16.s, #0x0\n" - "mov z18.s, #0xff\n" - "smax z15.s, p0/M, z15.s, z16.s\n" - "smax z14.s, p0/M, z14.s, z16.s\n" - "smax z13.s, p0/M, z13.s, z16.s\n" - "smax z12.s, p0/M, z12.s, z16.s\n" - "smin z15.s, p0/M, z15.s, z18.s\n" - "smin z14.s, p0/M, z14.s, z18.s\n" + "mov z17.s, #0x0\n" + "mov z16.s, #0xff\n" + "smax z15.s, p0/M, z15.s, z17.s\n" + "smax z14.s, p0/M, z14.s, z17.s\n" + "smax z13.s, p0/M, z13.s, z17.s\n" + "smax z12.s, p0/M, z12.s, z17.s\n" + "smin z15.s, p0/M, z15.s, z16.s\n" + "smin z14.s, p0/M, z14.s, z16.s\n" "trn1 z17.h, z15.h, z14.h\n" - "smin z13.s, p0/M, z13.s, z18.s\n" - "smin z12.s, p0/M, z12.s, z18.s\n" + "smin z13.s, p0/M, z13.s, z16.s\n" + "smin z12.s, p0/M, z12.s, z16.s\n" "trn1 z16.h, z13.h, z12.h\n" "trn1 z16.b, z17.b, z16.b\n" "st1b { z16.b }, p4, [%x[outptr], x27]\n" diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp index 70d308a5854e9d7754568a6b3c17d9445375a12d..8612555bfbd05e2bf199f8748a18db9242f28e5d 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp @@ -66,10 +66,10 @@ void sve_u8_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "ldr x15, [%x[args], %[offsetof_n_channels]]\n" "ldr x21, [%x[args], %[offsetof_outptrs]]\n" "mov x14, #0x0\n" - "whilelt p2.b, x14, x15\n" + "whilelt p0.b, x14, x15\n" "ldr x20, [%x[args], %[offsetof_inptrs]]\n" "ldp x13, x12, [x21, #0x0]\n" - "ptrue p1.b\n" + "ptrue p2.b\n" "mov x11, #0x0\n" "ldp x10, x9, [x21, #0x10]\n" "ldp x28, x27, [x20, #0x0]\n" @@ -77,61 +77,61 @@ void sve_u8_nhwc_max_2x2_s1_output2x2_depthfirst_impl( "ldp x24, x23, [x20, #0x20]\n" "ldp x22, x21, [x20, #0x30]\n" "ldr x20, [x20, #0x40]\n" - "ld1b { z31.b }, p2/Z, [x27, x14]\n" - "ld1b { z30.b }, p2/Z, [x24, x14]\n" - "ld1b { z29.b }, p2/Z, [x21, x14]\n" - "ld1b { z28.b }, p2/Z, [x25, x14]\n" - "ld1b { z27.b }, p2/Z, [x28, x14]\n" - "ld1b { z26.b }, p2/Z, [x26, x14]\n" - "ld1b { z25.b }, p2/Z, [x23, x14]\n" - "ld1b { z24.b }, p2/Z, [x22, x14]\n" - "ld1b { z23.b }, p2/Z, [x20, x14]\n" + "ld1b { z31.b }, p0/Z, [x27, x14]\n" + "ld1b { z30.b }, p0/Z, [x24, x14]\n" + "ld1b { z29.b }, p0/Z, [x21, x14]\n" + "ld1b { z28.b }, p0/Z, [x25, x14]\n" + "ld1b { z27.b }, p0/Z, [x28, x14]\n" + "ld1b { z26.b }, p0/Z, [x26, x14]\n" + "ld1b { z25.b }, p0/Z, [x23, x14]\n" + "ld1b { z24.b }, p0/Z, [x22, x14]\n" + "ld1b { z23.b }, p0/Z, [x20, x14]\n" "incw x14\n" - "whilelt p2.b, x14, x15\n" + "whilelt p1.b, x14, x15\n" "b.none 2f\n" "1:" // Vector: Loop - "movprfx z22, z31\n umax z22.b, p1/M, z22.b, z30.b\n" - "movprfx z21, z30\n umax z21.b, p1/M, z21.b, z29.b\n" - "ld1b { z31.b }, p2/Z, [x27, x14]\n" - "ld1b { z30.b }, p2/Z, [x24, x14]\n" - "movprfx z20, z28\n umax z20.b, p1/M, z20.b, z27.b\n" - "movprfx z19, z26\n umax z19.b, p1/M, z19.b, z25.b\n" - "ld1b { z29.b }, p2/Z, [x21, x14]\n" - "ld1b { z27.b }, p2/Z, [x28, x14]\n" - "movprfx z17, z28\n umax z17.b, p1/M, z17.b, z24.b\n" - "movprfx z18, z25\n umax z18.b, p1/M, z18.b, z23.b\n" - "ld1b { z28.b }, p2/Z, [x25, x14]\n" - "ld1b { z26.b }, p2/Z, [x26, x14]\n" - "ld1b { z25.b }, p2/Z, [x23, x14]\n" - "ld1b { z24.b }, p2/Z, [x22, x14]\n" + "movprfx z22, z31\n umax z22.b, p2/M, z22.b, z30.b\n" + "movprfx z21, z30\n umax z21.b, p2/M, z21.b, z29.b\n" + "ld1b { z31.b }, p1/Z, [x27, x14]\n" + "ld1b { z30.b }, p1/Z, [x24, x14]\n" + "movprfx z20, z28\n umax z20.b, p2/M, z20.b, z27.b\n" + "movprfx z19, z26\n umax z19.b, p2/M, z19.b, z25.b\n" + "ld1b { z29.b }, p1/Z, [x21, x14]\n" + "ld1b { z27.b }, p1/Z, [x28, x14]\n" + "movprfx z17, z28\n umax z17.b, p2/M, z17.b, z24.b\n" + "movprfx z18, z25\n umax z18.b, p2/M, z18.b, z23.b\n" + "ld1b { z28.b }, p1/Z, [x25, x14]\n" + "ld1b { z26.b }, p1/Z, [x26, x14]\n" + "ld1b { z25.b }, p1/Z, [x23, x14]\n" + "ld1b { z24.b }, p1/Z, [x22, x14]\n" "whilelt p0.b, x11, x15\n" - "movprfx z16, z22\n umax z16.b, p1/M, z16.b, z20.b\n" - "ld1b { z23.b }, p2/Z, [x20, x14]\n" + "movprfx z16, z22\n umax z16.b, p2/M, z16.b, z20.b\n" + "ld1b { z23.b }, p1/Z, [x20, x14]\n" "incw x14\n" - "whilelt p2.b, x14, x15\n" + "whilelt p1.b, x14, x15\n" "st1b { z16.b }, p0, [x13, x11]\n" - "movprfx z16, z19\n umax z16.b, p1/M, z16.b, z22.b\n" - "umax z17.b, p1/M, z17.b, z21.b\n" + "movprfx z16, z19\n umax z16.b, p2/M, z16.b, z22.b\n" + "umax z17.b, p2/M, z17.b, z21.b\n" "st1b { z16.b }, p0, [x12, x11]\n" - "movprfx z16, z18\n umax z16.b, p1/M, z16.b, z21.b\n" + "movprfx z16, z21\n umax z16.b, p2/M, z16.b, z18.b\n" "st1b { z17.b }, p0, [x10, x11]\n" "st1b { z16.b }, p0, [x9, x11]\n" "incw x11\n" "b.any 1b\n" "2:" // Vector: Tail - "movprfx z22, z31\n umax z22.b, p1/M, z22.b, z30.b\n" - "movprfx z21, z30\n umax z21.b, p1/M, z21.b, z29.b\n" - "movprfx z20, z28\n umax z20.b, p1/M, z20.b, z27.b\n" - "movprfx z19, z26\n umax z19.b, p1/M, z19.b, z25.b\n" - "movprfx z17, z28\n umax z17.b, p1/M, z17.b, z24.b\n" - "movprfx z18, z25\n umax z18.b, p1/M, z18.b, z23.b\n" + "movprfx z22, z31\n umax z22.b, p2/M, z22.b, z30.b\n" + "movprfx z21, z30\n umax z21.b, p2/M, z21.b, z29.b\n" + "movprfx z20, z28\n umax z20.b, p2/M, z20.b, z27.b\n" + "movprfx z19, z26\n umax z19.b, p2/M, z19.b, z25.b\n" + "movprfx z17, z28\n umax z17.b, p2/M, z17.b, z24.b\n" + "movprfx z18, z25\n umax z18.b, p2/M, z18.b, z23.b\n" "whilelt p0.b, x11, x15\n" - "movprfx z16, z22\n umax z16.b, p1/M, z16.b, z20.b\n" + "movprfx z16, z22\n umax z16.b, p2/M, z16.b, z20.b\n" "st1b { z16.b }, p0, [x13, x11]\n" - "movprfx z16, z19\n umax z16.b, p1/M, z16.b, z22.b\n" - "umax z17.b, p1/M, z17.b, z21.b\n" + "movprfx z16, z19\n umax z16.b, p2/M, z16.b, z22.b\n" + "umax z17.b, p2/M, z17.b, z21.b\n" "st1b { z16.b }, p0, [x12, x11]\n" - "movprfx z16, z18\n umax z16.b, p1/M, z16.b, z21.b\n" + "movprfx z16, z21\n umax z16.b, p2/M, z16.b, z18.b\n" "st1b { z17.b }, p0, [x10, x11]\n" "st1b { z16.b }, p0, [x9, x11]\n" : diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_max_generic_depthfirst/generic.cpp index 34aa5a3dd6e7c5776619ac7b071888a98f899f2b..be0eb398ae79966c949aefb88a76c6a1e6f5ddfa 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_max_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_max_generic_depthfirst/generic.cpp @@ -54,68 +54,68 @@ void sve_u8_nhwc_max_generic_depthfirst_impl( "lsr x25, %x[n_valid_cells], #0x2\n" "mov z8.b, #0x0\n" "mov z7.b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "mov z6.b, #0x0\n" "mov z5.b, #0x0\n" "cbz x25, 4f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" "subs x25, x25, #0x1\n" - "add x20, x20, #0x20\n" - "ld1b { z4.b }, p4/Z, [x24, x9]\n" - "ld1b { z3.b }, p4/Z, [x23, x9]\n" - "ld1b { z2.b }, p4/Z, [x22, x9]\n" - "ld1b { z1.b }, p4/Z, [x21, x9]\n" - "ld1b { z0.b }, p3/Z, [x24, x28]\n" - "ld1b { z31.b }, p3/Z, [x23, x28]\n" - "ld1b { z22.b }, p3/Z, [x22, x28]\n" - "ld1b { z30.b }, p3/Z, [x21, x28]\n" - "ld1b { z29.b }, p2/Z, [x24, x27]\n" - "ld1b { z28.b }, p2/Z, [x23, x27]\n" - "ld1b { z21.b }, p2/Z, [x22, x27]\n" - "ld1b { z27.b }, p2/Z, [x21, x27]\n" - "ld1b { z26.b }, p1/Z, [x24, x26]\n" - "ld1b { z25.b }, p1/Z, [x23, x26]\n" - "ld1b { z20.b }, p1/Z, [x22, x26]\n" - "ld1b { z24.b }, p1/Z, [x21, x26]\n" + "add x24, x24, #0x20\n" + "ld1b { z4.b }, p4/Z, [x23, x9]\n" + "ld1b { z3.b }, p4/Z, [x22, x9]\n" + "ld1b { z2.b }, p4/Z, [x21, x9]\n" + "ld1b { z1.b }, p4/Z, [x20, x9]\n" + "ld1b { z0.b }, p3/Z, [x23, x28]\n" + "ld1b { z31.b }, p3/Z, [x22, x28]\n" + "ld1b { z22.b }, p3/Z, [x21, x28]\n" + "ld1b { z30.b }, p3/Z, [x20, x28]\n" + "ld1b { z29.b }, p2/Z, [x23, x27]\n" + "ld1b { z28.b }, p2/Z, [x22, x27]\n" + "ld1b { z21.b }, p2/Z, [x21, x27]\n" + "ld1b { z27.b }, p2/Z, [x20, x27]\n" + "ld1b { z26.b }, p1/Z, [x23, x26]\n" + "ld1b { z25.b }, p1/Z, [x22, x26]\n" + "ld1b { z20.b }, p1/Z, [x21, x26]\n" + "ld1b { z24.b }, p1/Z, [x20, x26]\n" "beq 3f\n" "2:" // 4-vectors of channels: 4 inputs loop "movprfx z19, z4\n umax z19.b, p0/M, z19.b, z3.b\n" "movprfx z23, z2\n umax z23.b, p0/M, z23.b, z1.b\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" "movprfx z18, z0\n umax z18.b, p0/M, z18.b, z31.b\n" "umax z22.b, p0/M, z22.b, z30.b\n" - "ld1b { z4.b }, p4/Z, [x24, x9]\n" - "ld1b { z3.b }, p4/Z, [x23, x9]\n" + "ld1b { z4.b }, p4/Z, [x23, x9]\n" + "ld1b { z3.b }, p4/Z, [x22, x9]\n" "movprfx z17, z29\n umax z17.b, p0/M, z17.b, z28.b\n" "umax z21.b, p0/M, z21.b, z27.b\n" - "ld1b { z2.b }, p4/Z, [x22, x9]\n" - "ld1b { z1.b }, p4/Z, [x21, x9]\n" + "ld1b { z2.b }, p4/Z, [x21, x9]\n" + "ld1b { z1.b }, p4/Z, [x20, x9]\n" "movprfx z16, z26\n umax z16.b, p0/M, z16.b, z25.b\n" "umax z20.b, p0/M, z20.b, z24.b\n" - "ld1b { z0.b }, p3/Z, [x24, x28]\n" - "ld1b { z31.b }, p3/Z, [x23, x28]\n" + "ld1b { z0.b }, p3/Z, [x23, x28]\n" + "ld1b { z31.b }, p3/Z, [x22, x28]\n" "umax z19.b, p0/M, z19.b, z23.b\n" "umax z18.b, p0/M, z18.b, z22.b\n" - "ld1b { z22.b }, p3/Z, [x22, x28]\n" - "ld1b { z30.b }, p3/Z, [x21, x28]\n" + "ld1b { z22.b }, p3/Z, [x21, x28]\n" + "ld1b { z30.b }, p3/Z, [x20, x28]\n" "umax z17.b, p0/M, z17.b, z21.b\n" "umax z16.b, p0/M, z16.b, z20.b\n" - "ld1b { z29.b }, p2/Z, [x24, x27]\n" - "ld1b { z28.b }, p2/Z, [x23, x27]\n" + "ld1b { z29.b }, p2/Z, [x23, x27]\n" + "ld1b { z28.b }, p2/Z, [x22, x27]\n" "subs x25, x25, #0x1\n" "umax z8.b, p0/M, z8.b, z19.b\n" - "ld1b { z21.b }, p2/Z, [x22, x27]\n" - "ld1b { z27.b }, p2/Z, [x21, x27]\n" + "ld1b { z21.b }, p2/Z, [x21, x27]\n" + "ld1b { z27.b }, p2/Z, [x20, x27]\n" "umax z7.b, p0/M, z7.b, z18.b\n" "umax z6.b, p0/M, z6.b, z17.b\n" - "ld1b { z26.b }, p1/Z, [x24, x26]\n" - "ld1b { z25.b }, p1/Z, [x23, x26]\n" + "ld1b { z26.b }, p1/Z, [x23, x26]\n" + "ld1b { z25.b }, p1/Z, [x22, x26]\n" "umax z5.b, p0/M, z5.b, z16.b\n" - "add x20, x20, #0x20\n" - "ld1b { z20.b }, p1/Z, [x22, x26]\n" - "ld1b { z24.b }, p1/Z, [x21, x26]\n" + "add x24, x24, #0x20\n" + "ld1b { z20.b }, p1/Z, [x21, x26]\n" + "ld1b { z24.b }, p1/Z, [x20, x26]\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "movprfx z19, z4\n umax z19.b, p0/M, z19.b, z3.b\n" @@ -138,16 +138,16 @@ void sve_u8_nhwc_max_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1b { z4.b }, p4/Z, [x24, x9]\n" + "ldr x20, [x24], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x9]\n" "subs x21, x21, #0x1\n" - "umax z8.b, p0/M, z8.b, z4.b\n" - "ld1b { z0.b }, p3/Z, [x24, x28]\n" - "ld1b { z29.b }, p2/Z, [x24, x27]\n" - "umax z7.b, p0/M, z7.b, z0.b\n" - "umax z6.b, p0/M, z6.b, z29.b\n" - "ld1b { z26.b }, p1/Z, [x24, x26]\n" - "umax z5.b, p0/M, z5.b, z26.b\n" + "umax z8.b, p0/M, z8.b, z16.b\n" + "ld1b { z17.b }, p3/Z, [x20, x28]\n" + "ld1b { z16.b }, p2/Z, [x20, x27]\n" + "umax z7.b, p0/M, z7.b, z17.b\n" + "umax z6.b, p0/M, z6.b, z16.b\n" + "ld1b { z16.b }, p1/Z, [x20, x26]\n" + "umax z5.b, p0/M, z5.b, z16.b\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End "st1b { z8.b }, p4, [%x[outptr], x9]\n" @@ -166,44 +166,44 @@ void sve_u8_nhwc_max_generic_depthfirst_impl( "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" "mov z8.b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" "subs x25, x25, #0x1\n" - "add x20, x20, #0x20\n" - "ld1b { z4.b }, p4/Z, [x24, x9]\n" - "ld1b { z3.b }, p4/Z, [x23, x9]\n" - "ld1b { z2.b }, p4/Z, [x22, x9]\n" - "ld1b { z1.b }, p4/Z, [x21, x9]\n" + "add x24, x24, #0x20\n" + "ld1b { z4.b }, p4/Z, [x23, x9]\n" + "ld1b { z3.b }, p4/Z, [x22, x9]\n" + "ld1b { z2.b }, p4/Z, [x21, x9]\n" + "ld1b { z1.b }, p4/Z, [x20, x9]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 4 inputs loop - "movprfx z19, z4\n umax z19.b, p0/M, z19.b, z3.b\n" - "movprfx z23, z2\n umax z23.b, p0/M, z23.b, z1.b\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" - "umax z19.b, p0/M, z19.b, z23.b\n" + "movprfx z16, z4\n umax z16.b, p0/M, z16.b, z3.b\n" + "movprfx z17, z2\n umax z17.b, p0/M, z17.b, z1.b\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" + "umax z16.b, p0/M, z16.b, z17.b\n" "subs x25, x25, #0x1\n" - "ld1b { z4.b }, p4/Z, [x24, x9]\n" - "ld1b { z3.b }, p4/Z, [x23, x9]\n" - "umax z8.b, p0/M, z8.b, z19.b\n" - "add x20, x20, #0x20\n" - "ld1b { z2.b }, p4/Z, [x22, x9]\n" - "ld1b { z1.b }, p4/Z, [x21, x9]\n" + "ld1b { z4.b }, p4/Z, [x23, x9]\n" + "ld1b { z3.b }, p4/Z, [x22, x9]\n" + "umax z8.b, p0/M, z8.b, z16.b\n" + "add x24, x24, #0x20\n" + "ld1b { z2.b }, p4/Z, [x21, x9]\n" + "ld1b { z1.b }, p4/Z, [x20, x9]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail - "movprfx z19, z4\n umax z19.b, p0/M, z19.b, z3.b\n" - "movprfx z23, z2\n umax z23.b, p0/M, z23.b, z1.b\n" - "umax z19.b, p0/M, z19.b, z23.b\n" - "umax z8.b, p0/M, z8.b, z19.b\n" + "movprfx z16, z4\n umax z16.b, p0/M, z16.b, z3.b\n" + "movprfx z17, z2\n umax z17.b, p0/M, z17.b, z1.b\n" + "umax z16.b, p0/M, z16.b, z17.b\n" + "umax z8.b, p0/M, z8.b, z16.b\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1b { z4.b }, p4/Z, [x24, x9]\n" + "ldr x20, [x24], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x9]\n" "subs x21, x21, #0x1\n" - "umax z8.b, p0/M, z8.b, z4.b\n" + "umax z8.b, p0/M, z8.b, z16.b\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "st1b { z8.b }, p4, [%x[outptr], x9]\n" diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_avg_generic_depthfirst/generic.cpp index 36ac3810049ae6a1dce4e861e785cccfe1286792..e8339a2cd99dce9d35f27c2afffd76153b5a7290 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_avg_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_avg_generic_depthfirst/generic.cpp @@ -136,7 +136,7 @@ void sve_u8q_nhwc_avg_generic_depthfirst_impl( "mov z13.d, z15.d\n" "mov z12.d, z15.d\n" "mov z11.d, z15.d\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "mov z10.d, z15.d\n" "mov z9.d, z15.d\n" "mov z8.d, z15.d\n" @@ -149,42 +149,42 @@ void sve_u8q_nhwc_avg_generic_depthfirst_impl( "mov z1.d, z15.d\n" "mov z0.d, z15.d\n" "cbz x23, 4f\n" - "ldp x22, x21, [x20, #0x0]\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" - "ld1b { z29.b }, p3/Z, [x22, x26]\n" - "ld1b { z28.b }, p3/Z, [x21, x26]\n" - "ld1b { z27.b }, p2/Z, [x22, x25]\n" - "ld1b { z26.b }, p2/Z, [x21, x25]\n" - "ld1b { z25.b }, p1/Z, [x22, x24]\n" - "ld1b { z24.b }, p1/Z, [x21, x24]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" + "ld1b { z29.b }, p3/Z, [x21, x26]\n" + "ld1b { z28.b }, p3/Z, [x20, x26]\n" + "ld1b { z27.b }, p2/Z, [x21, x25]\n" + "ld1b { z26.b }, p2/Z, [x20, x25]\n" + "ld1b { z25.b }, p1/Z, [x21, x24]\n" + "ld1b { z24.b }, p1/Z, [x20, x24]\n" "beq 3f\n" "2:" // 4-vectors of channels: 2 inputs loop ".inst 0x455e0bf7 // uaddlb z23.h, z31.b, z30.b\n" ".inst 0x455e0ff6 // uaddlt z22.h, z31.b, z30.b\n" - "ldp x22, x21, [x20, #0x0]\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" ".inst 0x455c0bb5 // uaddlb z21.h, z29.b, z28.b\n" ".inst 0x455c0fb4 // uaddlt z20.h, z29.b, z28.b\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" ".inst 0x455a0b73 // uaddlb z19.h, z27.b, z26.b\n" ".inst 0x455a0f72 // uaddlt z18.h, z27.b, z26.b\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" - "ld1b { z29.b }, p3/Z, [x22, x26]\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" + "ld1b { z29.b }, p3/Z, [x21, x26]\n" ".inst 0x45580b31 // uaddlb z17.h, z25.b, z24.b\n" ".inst 0x45580f30 // uaddlt z16.h, z25.b, z24.b\n" - "ld1b { z28.b }, p3/Z, [x21, x26]\n" - "ld1b { z27.b }, p2/Z, [x22, x25]\n" + "ld1b { z28.b }, p3/Z, [x20, x26]\n" + "ld1b { z27.b }, p2/Z, [x21, x25]\n" ".inst 0x459749ef // uaddwb z15.s, z15.s, z23.h\n" ".inst 0x45974dce // uaddwt z14.s, z14.s, z23.h\n" - "ld1b { z26.b }, p2/Z, [x21, x25]\n" - "ld1b { z25.b }, p1/Z, [x22, x24]\n" + "ld1b { z26.b }, p2/Z, [x20, x25]\n" + "ld1b { z25.b }, p1/Z, [x21, x24]\n" ".inst 0x459649ad // uaddwb z13.s, z13.s, z22.h\n" ".inst 0x45964d8c // uaddwt z12.s, z12.s, z22.h\n" - "ld1b { z24.b }, p1/Z, [x21, x24]\n" + "ld1b { z24.b }, p1/Z, [x20, x24]\n" ".inst 0x4595496b // uaddwb z11.s, z11.s, z21.h\n" ".inst 0x45954d4a // uaddwt z10.s, z10.s, z21.h\n" ".inst 0x45944929 // uaddwb z9.s, z9.s, z20.h\n" @@ -227,20 +227,20 @@ void sve_u8q_nhwc_avg_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x1\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x22, [x20], #0x8\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - ".inst 0x4508abf7 // ushllb z23.h, z31.b, #0x0\n" - ".inst 0x4508aff6 // ushllt z22.h, z31.b, #0x0\n" - "ld1b { z29.b }, p3/Z, [x22, x26]\n" - "ld1b { z27.b }, p2/Z, [x22, x25]\n" - ".inst 0x4508abb5 // ushllb z21.h, z29.b, #0x0\n" - ".inst 0x4508afb4 // ushllt z20.h, z29.b, #0x0\n" - "ld1b { z25.b }, p1/Z, [x22, x24]\n" - ".inst 0x4508ab73 // ushllb z19.h, z27.b, #0x0\n" - ".inst 0x4508af72 // ushllt z18.h, z27.b, #0x0\n" + "ldr x20, [x22], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x27]\n" + ".inst 0x4508aa17 // ushllb z23.h, z16.b, #0x0\n" + ".inst 0x4508ae16 // ushllt z22.h, z16.b, #0x0\n" + "ld1b { z16.b }, p3/Z, [x20, x26]\n" + "ld1b { z17.b }, p2/Z, [x20, x25]\n" + ".inst 0x4508aa15 // ushllb z21.h, z16.b, #0x0\n" + ".inst 0x4508ae14 // ushllt z20.h, z16.b, #0x0\n" + "ld1b { z16.b }, p1/Z, [x20, x24]\n" + ".inst 0x4508aa33 // ushllb z19.h, z17.b, #0x0\n" + ".inst 0x4508ae32 // ushllt z18.h, z17.b, #0x0\n" "subs x21, x21, #0x1\n" - ".inst 0x4508ab31 // ushllb z17.h, z25.b, #0x0\n" - ".inst 0x4508af30 // ushllt z16.h, z25.b, #0x0\n" + ".inst 0x4508aa11 // ushllb z17.h, z16.b, #0x0\n" + ".inst 0x4508ae10 // ushllt z16.h, z16.b, #0x0\n" ".inst 0x459749ef // uaddwb z15.s, z15.s, z23.h\n" ".inst 0x45974dce // uaddwt z14.s, z14.s, z23.h\n" ".inst 0x459649ad // uaddwb z13.s, z13.s, z22.h\n" @@ -393,55 +393,55 @@ void sve_u8q_nhwc_avg_generic_depthfirst_impl( "mov z14.d, z15.d\n" "mov z13.d, z15.d\n" "mov z12.d, z15.d\n" - "mov x20, %x[inptrs]\n" + "mov x22, %x[inptrs]\n" "cbz x23, 11f\n" - "ldp x22, x21, [x20, #0x0]\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 2 inputs loop - ".inst 0x455e0bf7 // uaddlb z23.h, z31.b, z30.b\n" - ".inst 0x455e0ff6 // uaddlt z22.h, z31.b, z30.b\n" - "ldp x22, x21, [x20, #0x0]\n" + ".inst 0x455e0bf1 // uaddlb z17.h, z31.b, z30.b\n" + ".inst 0x455e0ff0 // uaddlt z16.h, z31.b, z30.b\n" + "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - ".inst 0x459749ef // uaddwb z15.s, z15.s, z23.h\n" - ".inst 0x45974dce // uaddwt z14.s, z14.s, z23.h\n" - "add x20, x20, #0x10\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - ".inst 0x459649ad // uaddwb z13.s, z13.s, z22.h\n" - ".inst 0x45964d8c // uaddwt z12.s, z12.s, z22.h\n" - "ld1b { z30.b }, p4/Z, [x21, x27]\n" + ".inst 0x459149ef // uaddwb z15.s, z15.s, z17.h\n" + ".inst 0x45914dce // uaddwt z14.s, z14.s, z17.h\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" + ".inst 0x459049ad // uaddwb z13.s, z13.s, z16.h\n" + ".inst 0x45904d8c // uaddwt z12.s, z12.s, z16.h\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 2 inputs tail - ".inst 0x455e0bf7 // uaddlb z23.h, z31.b, z30.b\n" - ".inst 0x455e0ff6 // uaddlt z22.h, z31.b, z30.b\n" - ".inst 0x459749ef // uaddwb z15.s, z15.s, z23.h\n" - ".inst 0x45974dce // uaddwt z14.s, z14.s, z23.h\n" - ".inst 0x459649ad // uaddwb z13.s, z13.s, z22.h\n" - ".inst 0x45964d8c // uaddwt z12.s, z12.s, z22.h\n" + ".inst 0x455e0bf1 // uaddlb z17.h, z31.b, z30.b\n" + ".inst 0x455e0ff0 // uaddlt z16.h, z31.b, z30.b\n" + ".inst 0x459149ef // uaddwb z15.s, z15.s, z17.h\n" + ".inst 0x45914dce // uaddwt z14.s, z14.s, z17.h\n" + ".inst 0x459049ad // uaddwb z13.s, z13.s, z16.h\n" + ".inst 0x45904d8c // uaddwt z12.s, z12.s, z16.h\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x1\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x22, [x20], #0x8\n" - "ld1b { z31.b }, p4/Z, [x22, x27]\n" - ".inst 0x4508abf7 // ushllb z23.h, z31.b, #0x0\n" - ".inst 0x4508aff6 // ushllt z22.h, z31.b, #0x0\n" + "ldr x20, [x22], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x27]\n" + ".inst 0x4508aa11 // ushllb z17.h, z16.b, #0x0\n" + ".inst 0x4508ae10 // ushllt z16.h, z16.b, #0x0\n" "subs x21, x21, #0x1\n" - ".inst 0x459749ef // uaddwb z15.s, z15.s, z23.h\n" - ".inst 0x45974dce // uaddwt z14.s, z14.s, z23.h\n" - ".inst 0x459649ad // uaddwb z13.s, z13.s, z22.h\n" - ".inst 0x45964d8c // uaddwt z12.s, z12.s, z22.h\n" + ".inst 0x459149ef // uaddwb z15.s, z15.s, z17.h\n" + ".inst 0x45914dce // uaddwt z14.s, z14.s, z17.h\n" + ".inst 0x459049ad // uaddwb z13.s, z13.s, z16.h\n" + ".inst 0x45904d8c // uaddwt z12.s, z12.s, z16.h\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End - "ld1rw { z18.s }, p0/Z, [%x[left_shift]]\n" + "ld1rw { z17.s }, p0/Z, [%x[left_shift]]\n" "ld1rw { z16.s }, p0/Z, [%x[combined_rescale_value]]\n" - ".inst 0x4482824f // srshl z15.s, p0/M, z15.s, z18.s\n" - ".inst 0x4482824e // srshl z14.s, p0/M, z14.s, z18.s\n" - ".inst 0x4482824d // srshl z13.s, p0/M, z13.s, z18.s\n" - ".inst 0x4482824c // srshl z12.s, p0/M, z12.s, z18.s\n" + ".inst 0x4482822f // srshl z15.s, p0/M, z15.s, z17.s\n" + ".inst 0x4482822e // srshl z14.s, p0/M, z14.s, z17.s\n" + ".inst 0x4482822d // srshl z13.s, p0/M, z13.s, z17.s\n" + ".inst 0x4482822c // srshl z12.s, p0/M, z12.s, z17.s\n" "ld1rw { z17.s }, p0/Z, [%x[right_shift]]\n" ".inst 0x04b075ef // sqrdmulh z15.s, z15.s, z16.s\n" ".inst 0x04b075ce // sqrdmulh z14.s, z14.s, z16.s\n" @@ -457,17 +457,17 @@ void sve_u8q_nhwc_avg_generic_depthfirst_impl( "add z14.s, z14.s, z16.s\n" "add z13.s, z13.s, z16.s\n" "add z12.s, z12.s, z16.s\n" - "mov z16.s, #0x0\n" - "smax z15.s, p0/M, z15.s, z16.s\n" - "smax z14.s, p0/M, z14.s, z16.s\n" - "mov z18.s, #0xff\n" - "smax z13.s, p0/M, z13.s, z16.s\n" - "smax z12.s, p0/M, z12.s, z16.s\n" - "smin z15.s, p0/M, z15.s, z18.s\n" - "smin z14.s, p0/M, z14.s, z18.s\n" + "mov z17.s, #0x0\n" + "smax z15.s, p0/M, z15.s, z17.s\n" + "smax z14.s, p0/M, z14.s, z17.s\n" + "mov z16.s, #0xff\n" + "smax z13.s, p0/M, z13.s, z17.s\n" + "smax z12.s, p0/M, z12.s, z17.s\n" + "smin z15.s, p0/M, z15.s, z16.s\n" + "smin z14.s, p0/M, z14.s, z16.s\n" "trn1 z17.h, z15.h, z14.h\n" - "smin z13.s, p0/M, z13.s, z18.s\n" - "smin z12.s, p0/M, z12.s, z18.s\n" + "smin z13.s, p0/M, z13.s, z16.s\n" + "smin z12.s, p0/M, z12.s, z16.s\n" "trn1 z16.h, z13.h, z12.h\n" "trn1 z16.b, z17.b, z16.b\n" "st1b { z16.b }, p4, [%x[outptr], x27]\n" diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_max_generic_depthfirst/generic.cpp index a00cbc59d8a149c64f194c066141a10615d6496c..94522cdaaa2be3ae303f4e7de703c430a2637706 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_max_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_max_generic_depthfirst/generic.cpp @@ -56,68 +56,68 @@ void sve_u8q_nhwc_max_generic_depthfirst_impl( "lsr x25, %x[n_valid_cells], #0x2\n" "mov z8.b, #0x0\n" "mov z7.b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "mov z6.b, #0x0\n" "mov z5.b, #0x0\n" "cbz x25, 4f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" "subs x25, x25, #0x1\n" - "add x20, x20, #0x20\n" - "ld1b { z4.b }, p4/Z, [x24, x9]\n" - "ld1b { z3.b }, p4/Z, [x23, x9]\n" - "ld1b { z2.b }, p4/Z, [x22, x9]\n" - "ld1b { z1.b }, p4/Z, [x21, x9]\n" - "ld1b { z0.b }, p3/Z, [x24, x28]\n" - "ld1b { z31.b }, p3/Z, [x23, x28]\n" - "ld1b { z22.b }, p3/Z, [x22, x28]\n" - "ld1b { z30.b }, p3/Z, [x21, x28]\n" - "ld1b { z29.b }, p2/Z, [x24, x27]\n" - "ld1b { z28.b }, p2/Z, [x23, x27]\n" - "ld1b { z21.b }, p2/Z, [x22, x27]\n" - "ld1b { z27.b }, p2/Z, [x21, x27]\n" - "ld1b { z26.b }, p1/Z, [x24, x26]\n" - "ld1b { z25.b }, p1/Z, [x23, x26]\n" - "ld1b { z20.b }, p1/Z, [x22, x26]\n" - "ld1b { z24.b }, p1/Z, [x21, x26]\n" + "add x24, x24, #0x20\n" + "ld1b { z4.b }, p4/Z, [x23, x9]\n" + "ld1b { z3.b }, p4/Z, [x22, x9]\n" + "ld1b { z2.b }, p4/Z, [x21, x9]\n" + "ld1b { z1.b }, p4/Z, [x20, x9]\n" + "ld1b { z0.b }, p3/Z, [x23, x28]\n" + "ld1b { z31.b }, p3/Z, [x22, x28]\n" + "ld1b { z22.b }, p3/Z, [x21, x28]\n" + "ld1b { z30.b }, p3/Z, [x20, x28]\n" + "ld1b { z29.b }, p2/Z, [x23, x27]\n" + "ld1b { z28.b }, p2/Z, [x22, x27]\n" + "ld1b { z21.b }, p2/Z, [x21, x27]\n" + "ld1b { z27.b }, p2/Z, [x20, x27]\n" + "ld1b { z26.b }, p1/Z, [x23, x26]\n" + "ld1b { z25.b }, p1/Z, [x22, x26]\n" + "ld1b { z20.b }, p1/Z, [x21, x26]\n" + "ld1b { z24.b }, p1/Z, [x20, x26]\n" "beq 3f\n" "2:" // 4-vectors of channels: 4 inputs loop "movprfx z19, z4\n umax z19.b, p0/M, z19.b, z3.b\n" "movprfx z23, z2\n umax z23.b, p0/M, z23.b, z1.b\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" "movprfx z18, z0\n umax z18.b, p0/M, z18.b, z31.b\n" "umax z22.b, p0/M, z22.b, z30.b\n" - "ld1b { z4.b }, p4/Z, [x24, x9]\n" - "ld1b { z3.b }, p4/Z, [x23, x9]\n" + "ld1b { z4.b }, p4/Z, [x23, x9]\n" + "ld1b { z3.b }, p4/Z, [x22, x9]\n" "movprfx z17, z29\n umax z17.b, p0/M, z17.b, z28.b\n" "umax z21.b, p0/M, z21.b, z27.b\n" - "ld1b { z2.b }, p4/Z, [x22, x9]\n" - "ld1b { z1.b }, p4/Z, [x21, x9]\n" + "ld1b { z2.b }, p4/Z, [x21, x9]\n" + "ld1b { z1.b }, p4/Z, [x20, x9]\n" "movprfx z16, z26\n umax z16.b, p0/M, z16.b, z25.b\n" "umax z20.b, p0/M, z20.b, z24.b\n" - "ld1b { z0.b }, p3/Z, [x24, x28]\n" - "ld1b { z31.b }, p3/Z, [x23, x28]\n" + "ld1b { z0.b }, p3/Z, [x23, x28]\n" + "ld1b { z31.b }, p3/Z, [x22, x28]\n" "umax z19.b, p0/M, z19.b, z23.b\n" "umax z18.b, p0/M, z18.b, z22.b\n" - "ld1b { z22.b }, p3/Z, [x22, x28]\n" - "ld1b { z30.b }, p3/Z, [x21, x28]\n" + "ld1b { z22.b }, p3/Z, [x21, x28]\n" + "ld1b { z30.b }, p3/Z, [x20, x28]\n" "umax z17.b, p0/M, z17.b, z21.b\n" "umax z16.b, p0/M, z16.b, z20.b\n" - "ld1b { z29.b }, p2/Z, [x24, x27]\n" - "ld1b { z28.b }, p2/Z, [x23, x27]\n" + "ld1b { z29.b }, p2/Z, [x23, x27]\n" + "ld1b { z28.b }, p2/Z, [x22, x27]\n" "subs x25, x25, #0x1\n" "umax z8.b, p0/M, z8.b, z19.b\n" - "ld1b { z21.b }, p2/Z, [x22, x27]\n" - "ld1b { z27.b }, p2/Z, [x21, x27]\n" + "ld1b { z21.b }, p2/Z, [x21, x27]\n" + "ld1b { z27.b }, p2/Z, [x20, x27]\n" "umax z7.b, p0/M, z7.b, z18.b\n" "umax z6.b, p0/M, z6.b, z17.b\n" - "ld1b { z26.b }, p1/Z, [x24, x26]\n" - "ld1b { z25.b }, p1/Z, [x23, x26]\n" + "ld1b { z26.b }, p1/Z, [x23, x26]\n" + "ld1b { z25.b }, p1/Z, [x22, x26]\n" "umax z5.b, p0/M, z5.b, z16.b\n" - "add x20, x20, #0x20\n" - "ld1b { z20.b }, p1/Z, [x22, x26]\n" - "ld1b { z24.b }, p1/Z, [x21, x26]\n" + "add x24, x24, #0x20\n" + "ld1b { z20.b }, p1/Z, [x21, x26]\n" + "ld1b { z24.b }, p1/Z, [x20, x26]\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "movprfx z19, z4\n umax z19.b, p0/M, z19.b, z3.b\n" @@ -140,103 +140,103 @@ void sve_u8q_nhwc_max_generic_depthfirst_impl( "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" "5:" // 4-vectors of channels: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1b { z4.b }, p4/Z, [x24, x9]\n" + "ldr x20, [x24], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x9]\n" "subs x21, x21, #0x1\n" - "umax z8.b, p0/M, z8.b, z4.b\n" - "ld1b { z0.b }, p3/Z, [x24, x28]\n" - "ld1b { z29.b }, p2/Z, [x24, x27]\n" - "umax z7.b, p0/M, z7.b, z0.b\n" - "umax z6.b, p0/M, z6.b, z29.b\n" - "ld1b { z26.b }, p1/Z, [x24, x26]\n" - "umax z5.b, p0/M, z5.b, z26.b\n" + "umax z8.b, p0/M, z8.b, z16.b\n" + "ld1b { z17.b }, p3/Z, [x20, x28]\n" + "ld1b { z16.b }, p2/Z, [x20, x27]\n" + "umax z7.b, p0/M, z7.b, z17.b\n" + "umax z6.b, p0/M, z6.b, z16.b\n" + "ld1b { z16.b }, p1/Z, [x20, x26]\n" + "umax z5.b, p0/M, z5.b, z16.b\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End "add x20, %x[quant_params], %[offsetof_qp_input_offset]\n" - "ld1rw { z4.s }, p0/Z, [x20]\n" - ".inst 0x4508a918 // ushllb z24.h, z8.b, #0x0\n" - ".inst 0x4508ad17 // ushllt z23.h, z8.b, #0x0\n" - ".inst 0x4508a8f6 // ushllb z22.h, z7.b, #0x0\n" - ".inst 0x4508acf5 // ushllt z21.h, z7.b, #0x0\n" - "neg z4.s, p0/M, z4.s\n" - "add x20, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n" - ".inst 0x4508a8d4 // ushllb z20.h, z6.b, #0x0\n" - ".inst 0x4508acd3 // ushllt z19.h, z6.b, #0x0\n" "ld1rw { z3.s }, p0/Z, [x20]\n" + ".inst 0x4508a911 // ushllb z17.h, z8.b, #0x0\n" + ".inst 0x4508ad18 // ushllt z24.h, z8.b, #0x0\n" + ".inst 0x4508a8f7 // ushllb z23.h, z7.b, #0x0\n" + ".inst 0x4508acf6 // ushllt z22.h, z7.b, #0x0\n" + "neg z3.s, p0/M, z3.s\n" + "add x20, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n" + ".inst 0x4508a8d5 // ushllb z21.h, z6.b, #0x0\n" + ".inst 0x4508acd4 // ushllt z20.h, z6.b, #0x0\n" + "ld1rw { z2.s }, p0/Z, [x20]\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_mul]\n" - ".inst 0x4508a8b2 // ushllb z18.h, z5.b, #0x0\n" - ".inst 0x4508acb1 // ushllt z17.h, z5.b, #0x0\n" - "ld1rw { z16.s }, p0/Z, [x20]\n" + ".inst 0x4508a8b3 // ushllb z19.h, z5.b, #0x0\n" + ".inst 0x4508acb0 // ushllt z16.h, z5.b, #0x0\n" + "ld1rw { z18.s }, p0/Z, [x20]\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n" - ".inst 0x45984082 // saddwb z2.s, z4.s, z24.h\n" - ".inst 0x45984481 // saddwt z1.s, z4.s, z24.h\n" - ".inst 0x44828062 // srshl z2.s, p0/M, z2.s, z3.s\n" - ".inst 0x44828061 // srshl z1.s, p0/M, z1.s, z3.s\n" - ".inst 0x45974080 // saddwb z0.s, z4.s, z23.h\n" - ".inst 0x4597449f // saddwt z31.s, z4.s, z23.h\n" - ".inst 0x44828060 // srshl z0.s, p0/M, z0.s, z3.s\n" - ".inst 0x4482807f // srshl z31.s, p0/M, z31.s, z3.s\n" - ".inst 0x4596409e // saddwb z30.s, z4.s, z22.h\n" - ".inst 0x4596449d // saddwt z29.s, z4.s, z22.h\n" - ".inst 0x4482807e // srshl z30.s, p0/M, z30.s, z3.s\n" - ".inst 0x4482807d // srshl z29.s, p0/M, z29.s, z3.s\n" - ".inst 0x4595409c // saddwb z28.s, z4.s, z21.h\n" - ".inst 0x4595449b // saddwt z27.s, z4.s, z21.h\n" - ".inst 0x4482807c // srshl z28.s, p0/M, z28.s, z3.s\n" - ".inst 0x4482807b // srshl z27.s, p0/M, z27.s, z3.s\n" - ".inst 0x4594409a // saddwb z26.s, z4.s, z20.h\n" - ".inst 0x45944499 // saddwt z25.s, z4.s, z20.h\n" - ".inst 0x4482807a // srshl z26.s, p0/M, z26.s, z3.s\n" - ".inst 0x44828079 // srshl z25.s, p0/M, z25.s, z3.s\n" - ".inst 0x45934098 // saddwb z24.s, z4.s, z19.h\n" - ".inst 0x45934497 // saddwt z23.s, z4.s, z19.h\n" - ".inst 0x44828078 // srshl z24.s, p0/M, z24.s, z3.s\n" - ".inst 0x44828077 // srshl z23.s, p0/M, z23.s, z3.s\n" - ".inst 0x45924096 // saddwb z22.s, z4.s, z18.h\n" - ".inst 0x45924495 // saddwt z21.s, z4.s, z18.h\n" - ".inst 0x44828076 // srshl z22.s, p0/M, z22.s, z3.s\n" - ".inst 0x44828075 // srshl z21.s, p0/M, z21.s, z3.s\n" - ".inst 0x45914094 // saddwb z20.s, z4.s, z17.h\n" - ".inst 0x45914493 // saddwt z19.s, z4.s, z17.h\n" - ".inst 0x44828074 // srshl z20.s, p0/M, z20.s, z3.s\n" - ".inst 0x44828073 // srshl z19.s, p0/M, z19.s, z3.s\n" - "ld1rw { z17.s }, p0/Z, [x20]\n" - ".inst 0x04b07442 // sqrdmulh z2.s, z2.s, z16.s\n" - ".inst 0x04b07421 // sqrdmulh z1.s, z1.s, z16.s\n" + ".inst 0x45914061 // saddwb z1.s, z3.s, z17.h\n" + ".inst 0x45914471 // saddwt z17.s, z3.s, z17.h\n" + ".inst 0x44828041 // srshl z1.s, p0/M, z1.s, z2.s\n" + ".inst 0x44828051 // srshl z17.s, p0/M, z17.s, z2.s\n" + ".inst 0x45984060 // saddwb z0.s, z3.s, z24.h\n" + ".inst 0x4598447f // saddwt z31.s, z3.s, z24.h\n" + ".inst 0x44828040 // srshl z0.s, p0/M, z0.s, z2.s\n" + ".inst 0x4482805f // srshl z31.s, p0/M, z31.s, z2.s\n" + ".inst 0x4597407e // saddwb z30.s, z3.s, z23.h\n" + ".inst 0x4597447d // saddwt z29.s, z3.s, z23.h\n" + ".inst 0x4482805e // srshl z30.s, p0/M, z30.s, z2.s\n" + ".inst 0x4482805d // srshl z29.s, p0/M, z29.s, z2.s\n" + ".inst 0x4596407c // saddwb z28.s, z3.s, z22.h\n" + ".inst 0x4596447b // saddwt z27.s, z3.s, z22.h\n" + ".inst 0x4482805c // srshl z28.s, p0/M, z28.s, z2.s\n" + ".inst 0x4482805b // srshl z27.s, p0/M, z27.s, z2.s\n" + ".inst 0x4595407a // saddwb z26.s, z3.s, z21.h\n" + ".inst 0x45954479 // saddwt z25.s, z3.s, z21.h\n" + ".inst 0x4482805a // srshl z26.s, p0/M, z26.s, z2.s\n" + ".inst 0x44828059 // srshl z25.s, p0/M, z25.s, z2.s\n" + ".inst 0x45944078 // saddwb z24.s, z3.s, z20.h\n" + ".inst 0x45944477 // saddwt z23.s, z3.s, z20.h\n" + ".inst 0x44828058 // srshl z24.s, p0/M, z24.s, z2.s\n" + ".inst 0x44828057 // srshl z23.s, p0/M, z23.s, z2.s\n" + ".inst 0x45934076 // saddwb z22.s, z3.s, z19.h\n" + ".inst 0x45934475 // saddwt z21.s, z3.s, z19.h\n" + ".inst 0x44828056 // srshl z22.s, p0/M, z22.s, z2.s\n" + ".inst 0x44828055 // srshl z21.s, p0/M, z21.s, z2.s\n" + ".inst 0x45904074 // saddwb z20.s, z3.s, z16.h\n" + ".inst 0x45904473 // saddwt z19.s, z3.s, z16.h\n" + ".inst 0x44828054 // srshl z20.s, p0/M, z20.s, z2.s\n" + ".inst 0x44828053 // srshl z19.s, p0/M, z19.s, z2.s\n" + "ld1rw { z16.s }, p0/Z, [x20]\n" + ".inst 0x04b27421 // sqrdmulh z1.s, z1.s, z18.s\n" + ".inst 0x04b27631 // sqrdmulh z17.s, z17.s, z18.s\n" "add x20, %x[quant_params], %[offsetof_qp_output_offset]\n" - ".inst 0x04b07400 // sqrdmulh z0.s, z0.s, z16.s\n" - ".inst 0x04b077ff // sqrdmulh z31.s, z31.s, z16.s\n" - ".inst 0x44828222 // srshl z2.s, p0/M, z2.s, z17.s\n" - ".inst 0x44828221 // srshl z1.s, p0/M, z1.s, z17.s\n" - ".inst 0x04b077de // sqrdmulh z30.s, z30.s, z16.s\n" - ".inst 0x04b077bd // sqrdmulh z29.s, z29.s, z16.s\n" - ".inst 0x44828220 // srshl z0.s, p0/M, z0.s, z17.s\n" - ".inst 0x4482823f // srshl z31.s, p0/M, z31.s, z17.s\n" - ".inst 0x04b0779c // sqrdmulh z28.s, z28.s, z16.s\n" - ".inst 0x04b0777b // sqrdmulh z27.s, z27.s, z16.s\n" - ".inst 0x4482823e // srshl z30.s, p0/M, z30.s, z17.s\n" - ".inst 0x4482823d // srshl z29.s, p0/M, z29.s, z17.s\n" - ".inst 0x04b0775a // sqrdmulh z26.s, z26.s, z16.s\n" - ".inst 0x04b07739 // sqrdmulh z25.s, z25.s, z16.s\n" - ".inst 0x4482823c // srshl z28.s, p0/M, z28.s, z17.s\n" - ".inst 0x4482823b // srshl z27.s, p0/M, z27.s, z17.s\n" - ".inst 0x04b07718 // sqrdmulh z24.s, z24.s, z16.s\n" - ".inst 0x04b076f7 // sqrdmulh z23.s, z23.s, z16.s\n" - ".inst 0x4482823a // srshl z26.s, p0/M, z26.s, z17.s\n" - ".inst 0x44828239 // srshl z25.s, p0/M, z25.s, z17.s\n" - ".inst 0x04b076d6 // sqrdmulh z22.s, z22.s, z16.s\n" - ".inst 0x04b076b5 // sqrdmulh z21.s, z21.s, z16.s\n" - ".inst 0x44828238 // srshl z24.s, p0/M, z24.s, z17.s\n" - ".inst 0x44828237 // srshl z23.s, p0/M, z23.s, z17.s\n" - ".inst 0x04b07694 // sqrdmulh z20.s, z20.s, z16.s\n" - ".inst 0x04b07673 // sqrdmulh z19.s, z19.s, z16.s\n" - ".inst 0x44828236 // srshl z22.s, p0/M, z22.s, z17.s\n" - ".inst 0x44828235 // srshl z21.s, p0/M, z21.s, z17.s\n" - ".inst 0x44828234 // srshl z20.s, p0/M, z20.s, z17.s\n" - ".inst 0x44828233 // srshl z19.s, p0/M, z19.s, z17.s\n" + ".inst 0x04b27400 // sqrdmulh z0.s, z0.s, z18.s\n" + ".inst 0x04b277ff // sqrdmulh z31.s, z31.s, z18.s\n" + ".inst 0x44828201 // srshl z1.s, p0/M, z1.s, z16.s\n" + ".inst 0x44828211 // srshl z17.s, p0/M, z17.s, z16.s\n" + ".inst 0x04b277de // sqrdmulh z30.s, z30.s, z18.s\n" + ".inst 0x04b277bd // sqrdmulh z29.s, z29.s, z18.s\n" + ".inst 0x44828200 // srshl z0.s, p0/M, z0.s, z16.s\n" + ".inst 0x4482821f // srshl z31.s, p0/M, z31.s, z16.s\n" + ".inst 0x04b2779c // sqrdmulh z28.s, z28.s, z18.s\n" + ".inst 0x04b2777b // sqrdmulh z27.s, z27.s, z18.s\n" + ".inst 0x4482821e // srshl z30.s, p0/M, z30.s, z16.s\n" + ".inst 0x4482821d // srshl z29.s, p0/M, z29.s, z16.s\n" + ".inst 0x04b2775a // sqrdmulh z26.s, z26.s, z18.s\n" + ".inst 0x04b27739 // sqrdmulh z25.s, z25.s, z18.s\n" + ".inst 0x4482821c // srshl z28.s, p0/M, z28.s, z16.s\n" + ".inst 0x4482821b // srshl z27.s, p0/M, z27.s, z16.s\n" + ".inst 0x04b27718 // sqrdmulh z24.s, z24.s, z18.s\n" + ".inst 0x04b276f7 // sqrdmulh z23.s, z23.s, z18.s\n" + ".inst 0x4482821a // srshl z26.s, p0/M, z26.s, z16.s\n" + ".inst 0x44828219 // srshl z25.s, p0/M, z25.s, z16.s\n" + ".inst 0x04b276d6 // sqrdmulh z22.s, z22.s, z18.s\n" + ".inst 0x04b276b5 // sqrdmulh z21.s, z21.s, z18.s\n" + ".inst 0x44828218 // srshl z24.s, p0/M, z24.s, z16.s\n" + ".inst 0x44828217 // srshl z23.s, p0/M, z23.s, z16.s\n" + ".inst 0x04b27694 // sqrdmulh z20.s, z20.s, z18.s\n" + ".inst 0x04b27673 // sqrdmulh z19.s, z19.s, z18.s\n" + ".inst 0x44828216 // srshl z22.s, p0/M, z22.s, z16.s\n" + ".inst 0x44828215 // srshl z21.s, p0/M, z21.s, z16.s\n" + ".inst 0x44828214 // srshl z20.s, p0/M, z20.s, z16.s\n" + ".inst 0x44828213 // srshl z19.s, p0/M, z19.s, z16.s\n" "ld1rw { z16.s }, p0/Z, [x20]\n" - "add z2.s, z2.s, z16.s\n" "add z1.s, z1.s, z16.s\n" + "add z17.s, z17.s, z16.s\n" "add z0.s, z0.s, z16.s\n" "add z31.s, z31.s, z16.s\n" "add z30.s, z30.s, z16.s\n" @@ -252,8 +252,8 @@ void sve_u8q_nhwc_max_generic_depthfirst_impl( "add z20.s, z20.s, z16.s\n" "add z19.s, z19.s, z16.s\n" "mov z16.s, #0x0\n" - "smax z2.s, p0/M, z2.s, z16.s\n" "smax z1.s, p0/M, z1.s, z16.s\n" + "smax z17.s, p0/M, z17.s, z16.s\n" "smax z0.s, p0/M, z0.s, z16.s\n" "smax z31.s, p0/M, z31.s, z16.s\n" "mov z18.s, #0xff\n" @@ -269,9 +269,9 @@ void sve_u8q_nhwc_max_generic_depthfirst_impl( "smax z21.s, p0/M, z21.s, z16.s\n" "smax z20.s, p0/M, z20.s, z16.s\n" "smax z19.s, p0/M, z19.s, z16.s\n" - "smin z2.s, p0/M, z2.s, z18.s\n" "smin z1.s, p0/M, z1.s, z18.s\n" - "trn1 z17.h, z2.h, z1.h\n" + "smin z17.s, p0/M, z17.s, z18.s\n" + "trn1 z17.h, z1.h, z17.h\n" "smin z0.s, p0/M, z0.s, z18.s\n" "smin z31.s, p0/M, z31.s, z18.s\n" "trn1 z16.h, z0.h, z31.h\n" @@ -313,91 +313,91 @@ void sve_u8q_nhwc_max_generic_depthfirst_impl( "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" "mov z8.b, #0x0\n" - "mov x20, %x[inptrs]\n" + "mov x24, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" "subs x25, x25, #0x1\n" - "add x20, x20, #0x20\n" - "ld1b { z4.b }, p4/Z, [x24, x9]\n" - "ld1b { z3.b }, p4/Z, [x23, x9]\n" - "ld1b { z2.b }, p4/Z, [x22, x9]\n" - "ld1b { z1.b }, p4/Z, [x21, x9]\n" + "add x24, x24, #0x20\n" + "ld1b { z4.b }, p4/Z, [x23, x9]\n" + "ld1b { z3.b }, p4/Z, [x22, x9]\n" + "ld1b { z2.b }, p4/Z, [x21, x9]\n" + "ld1b { z1.b }, p4/Z, [x20, x9]\n" "beq 10f\n" "9:" // Single vector of channels: Loop: 4 inputs loop - "movprfx z19, z4\n umax z19.b, p0/M, z19.b, z3.b\n" - "movprfx z23, z2\n umax z23.b, p0/M, z23.b, z1.b\n" - "ldp x24, x23, [x20, #0x0]\n" - "ldp x22, x21, [x20, #0x10]\n" - "umax z19.b, p0/M, z19.b, z23.b\n" + "movprfx z16, z4\n umax z16.b, p0/M, z16.b, z3.b\n" + "movprfx z17, z2\n umax z17.b, p0/M, z17.b, z1.b\n" + "ldp x23, x22, [x24, #0x0]\n" + "ldp x21, x20, [x24, #0x10]\n" + "umax z16.b, p0/M, z16.b, z17.b\n" "subs x25, x25, #0x1\n" - "ld1b { z4.b }, p4/Z, [x24, x9]\n" - "ld1b { z3.b }, p4/Z, [x23, x9]\n" - "umax z8.b, p0/M, z8.b, z19.b\n" - "add x20, x20, #0x20\n" - "ld1b { z2.b }, p4/Z, [x22, x9]\n" - "ld1b { z1.b }, p4/Z, [x21, x9]\n" + "ld1b { z4.b }, p4/Z, [x23, x9]\n" + "ld1b { z3.b }, p4/Z, [x22, x9]\n" + "umax z8.b, p0/M, z8.b, z16.b\n" + "add x24, x24, #0x20\n" + "ld1b { z2.b }, p4/Z, [x21, x9]\n" + "ld1b { z1.b }, p4/Z, [x20, x9]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail - "movprfx z19, z4\n umax z19.b, p0/M, z19.b, z3.b\n" - "movprfx z23, z2\n umax z23.b, p0/M, z23.b, z1.b\n" - "umax z19.b, p0/M, z19.b, z23.b\n" - "umax z8.b, p0/M, z8.b, z19.b\n" + "movprfx z16, z4\n umax z16.b, p0/M, z16.b, z3.b\n" + "movprfx z17, z2\n umax z17.b, p0/M, z17.b, z1.b\n" + "umax z16.b, p0/M, z16.b, z17.b\n" + "umax z8.b, p0/M, z8.b, z16.b\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop - "ldr x24, [x20], #0x8\n" - "ld1b { z4.b }, p4/Z, [x24, x9]\n" + "ldr x20, [x24], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x9]\n" "subs x21, x21, #0x1\n" - "umax z8.b, p0/M, z8.b, z4.b\n" + "umax z8.b, p0/M, z8.b, z16.b\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End "add x20, %x[quant_params], %[offsetof_qp_input_offset]\n" - "ld1rw { z4.s }, p0/Z, [x20]\n" - ".inst 0x4508a918 // ushllb z24.h, z8.b, #0x0\n" - ".inst 0x4508ad17 // ushllt z23.h, z8.b, #0x0\n" - "neg z4.s, p0/M, z4.s\n" + "ld1rw { z18.s }, p0/Z, [x20]\n" + ".inst 0x4508a911 // ushllb z17.h, z8.b, #0x0\n" + ".inst 0x4508ad10 // ushllt z16.h, z8.b, #0x0\n" + "neg z18.s, p0/M, z18.s\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n" - ".inst 0x45984082 // saddwb z2.s, z4.s, z24.h\n" - ".inst 0x45984481 // saddwt z1.s, z4.s, z24.h\n" - ".inst 0x45974080 // saddwb z0.s, z4.s, z23.h\n" - ".inst 0x4597449f // saddwt z31.s, z4.s, z23.h\n" - "ld1rw { z3.s }, p0/Z, [x20]\n" + ".inst 0x45914255 // saddwb z21.s, z18.s, z17.h\n" + ".inst 0x45914654 // saddwt z20.s, z18.s, z17.h\n" + ".inst 0x45904253 // saddwb z19.s, z18.s, z16.h\n" + ".inst 0x45904652 // saddwt z18.s, z18.s, z16.h\n" + "ld1rw { z17.s }, p0/Z, [x20]\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_mul]\n" "ld1rw { z16.s }, p0/Z, [x20]\n" - ".inst 0x44828062 // srshl z2.s, p0/M, z2.s, z3.s\n" - ".inst 0x44828061 // srshl z1.s, p0/M, z1.s, z3.s\n" - ".inst 0x04b07442 // sqrdmulh z2.s, z2.s, z16.s\n" - ".inst 0x44828060 // srshl z0.s, p0/M, z0.s, z3.s\n" - ".inst 0x4482807f // srshl z31.s, p0/M, z31.s, z3.s\n" - ".inst 0x04b07421 // sqrdmulh z1.s, z1.s, z16.s\n" - ".inst 0x04b07400 // sqrdmulh z0.s, z0.s, z16.s\n" + ".inst 0x44828235 // srshl z21.s, p0/M, z21.s, z17.s\n" + ".inst 0x44828234 // srshl z20.s, p0/M, z20.s, z17.s\n" + ".inst 0x04b076b5 // sqrdmulh z21.s, z21.s, z16.s\n" + ".inst 0x44828233 // srshl z19.s, p0/M, z19.s, z17.s\n" + ".inst 0x44828232 // srshl z18.s, p0/M, z18.s, z17.s\n" + ".inst 0x04b07694 // sqrdmulh z20.s, z20.s, z16.s\n" + ".inst 0x04b07673 // sqrdmulh z19.s, z19.s, z16.s\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n" "ld1rw { z17.s }, p0/Z, [x20]\n" - ".inst 0x04b077ff // sqrdmulh z31.s, z31.s, z16.s\n" + ".inst 0x04b07652 // sqrdmulh z18.s, z18.s, z16.s\n" "add x20, %x[quant_params], %[offsetof_qp_output_offset]\n" - ".inst 0x44828222 // srshl z2.s, p0/M, z2.s, z17.s\n" - ".inst 0x44828221 // srshl z1.s, p0/M, z1.s, z17.s\n" + ".inst 0x44828235 // srshl z21.s, p0/M, z21.s, z17.s\n" + ".inst 0x44828234 // srshl z20.s, p0/M, z20.s, z17.s\n" "ld1rw { z16.s }, p0/Z, [x20]\n" - "add z2.s, z2.s, z16.s\n" - ".inst 0x44828220 // srshl z0.s, p0/M, z0.s, z17.s\n" - ".inst 0x4482823f // srshl z31.s, p0/M, z31.s, z17.s\n" - "add z1.s, z1.s, z16.s\n" - "add z0.s, z0.s, z16.s\n" - "add z31.s, z31.s, z16.s\n" + "add z21.s, z21.s, z16.s\n" + ".inst 0x44828233 // srshl z19.s, p0/M, z19.s, z17.s\n" + ".inst 0x44828232 // srshl z18.s, p0/M, z18.s, z17.s\n" + "add z20.s, z20.s, z16.s\n" + "add z19.s, z19.s, z16.s\n" + "add z18.s, z18.s, z16.s\n" "mov z16.s, #0x0\n" - "smax z2.s, p0/M, z2.s, z16.s\n" - "smax z1.s, p0/M, z1.s, z16.s\n" - "smax z0.s, p0/M, z0.s, z16.s\n" - "smax z31.s, p0/M, z31.s, z16.s\n" - "mov z18.s, #0xff\n" - "smin z2.s, p0/M, z2.s, z18.s\n" - "smin z1.s, p0/M, z1.s, z18.s\n" - "trn1 z17.h, z2.h, z1.h\n" - "smin z0.s, p0/M, z0.s, z18.s\n" - "smin z31.s, p0/M, z31.s, z18.s\n" - "trn1 z16.h, z0.h, z31.h\n" + "smax z21.s, p0/M, z21.s, z16.s\n" + "smax z20.s, p0/M, z20.s, z16.s\n" + "smax z19.s, p0/M, z19.s, z16.s\n" + "smax z18.s, p0/M, z18.s, z16.s\n" + "mov z16.s, #0xff\n" + "smin z21.s, p0/M, z21.s, z16.s\n" + "smin z20.s, p0/M, z20.s, z16.s\n" + "trn1 z17.h, z21.h, z20.h\n" + "smin z19.s, p0/M, z19.s, z16.s\n" + "smin z18.s, p0/M, z18.s, z16.s\n" + "trn1 z16.h, z19.h, z18.h\n" "trn1 z16.b, z17.b, z16.b\n" "st1b { z16.b }, p4, [%x[outptr], x9]\n" "incb x9\n" diff --git a/src/core/NEON/kernels/arm_conv/pooling/pooling_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/pooling_depthfirst.hpp index 63333c8fb49c474ec458ad18cf346ad86d15fb05..1ca478513c8f691ef903311f56f6d819cfb48a63 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/pooling_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/pooling_depthfirst.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -91,17 +91,17 @@ class PoolingDepthfirst : public DepthfirstDriver protected: /* Compute the amount of working space required for a single thread. */ - size_t get_working_size_per_thread(unsigned int n_channels) const override + size_t get_working_size_per_thread() const override { - return sizeof(WorkingSpace) + n_channels * (sizeof(TInput) + sizeof(TOutput)); + return sizeof(WorkingSpace) + this->m_args.n_channels * (sizeof(TInput) + sizeof(TOutput)); } /* Initialise the working space for a thread. */ - void initialise_working_space(void *raw_ws, unsigned int n_channels) const override + void initialise_working_space(void *raw_ws) const override { auto ws = reinterpret_cast(raw_ws); ws->input_buffer = ws + 1; - ws->output_buffer = reinterpret_cast(ws + 1) + n_channels; + ws->output_buffer = reinterpret_cast(ws + 1) + sizeof(TInput) * this->m_args.n_channels; // Fill the input buffer with an appropriate value TInput fill_val = 0; @@ -119,6 +119,7 @@ class PoolingDepthfirst : public DepthfirstDriver } auto ptr = reinterpret_cast(ws->input_buffer); + auto n_channels = this->m_args.n_channels; for (; n_channels; n_channels--) { *(ptr++) = fill_val; diff --git a/src/core/NEON/kernels/arm_conv/pooling/pooling_depthfirst_cache_oblivious.hpp b/src/core/NEON/kernels/arm_conv/pooling/pooling_depthfirst_cache_oblivious.hpp deleted file mode 100644 index 4aabd957cd35c977ab7c4743d65ff2f4563b562e..0000000000000000000000000000000000000000 --- a/src/core/NEON/kernels/arm_conv/pooling/pooling_depthfirst_cache_oblivious.hpp +++ /dev/null @@ -1,312 +0,0 @@ -/* - * Copyright (c) 2021 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#include "pool_common.hpp" - -#include -#include - -namespace arm_conv { -namespace pooling { - -template -class PoolingDepthfirstCacheOblivious : public PoolingCommon -{ - using TInput = typename strategy::operand_type; - using TOutput = typename strategy::return_type; - - const PoolingArgs m_args; // Copy of arguments - - constexpr static unsigned int input_rows(void) - { - return (strategy::out_rows() - 1)*strategy::stride_rows() + strategy::pool_rows(); - } - - constexpr static unsigned int input_cols(void) - { - return (strategy::out_cols() - 1)*strategy::stride_cols() + strategy::pool_cols(); - } - - size_t sizeof_input_buffer(void) const - { - return sizeof(TInput) * m_args.n_channels; - } - - size_t sizeof_output_buffer(void) const - { - return sizeof(TOutput) * m_args.n_channels; - } - - public: - PoolingDepthfirstCacheOblivious(const PoolingArgs &args) : m_args(args) - { - } - - PoolingDepthfirstCacheOblivious(PoolingDepthfirstCacheOblivious &) = delete; - PoolingDepthfirstCacheOblivious &operator=(PoolingDepthfirstCacheOblivious &) = delete; - - size_t get_working_size(void) const override - { - // We require an array of pointers for the inputs and outputs, a - // channel-length vector in which to dump surplus output, and a - // channel-length vector of padding values. - return sizeof_input_buffer() + sizeof_output_buffer(); - } - - void execute( - const void *const input, - void *const output, - void *const working_space - ) const override - { - const size_t ld_input_col = m_args.n_channels; - const size_t ld_input_row = ld_input_col * m_args.input_cols; - const size_t ld_input_batch = ld_input_row * m_args.input_rows; - const size_t ld_output_col = ld_input_col; - const size_t ld_output_row = ld_output_col * m_args.output_cols; - const size_t ld_output_batch = ld_output_row * m_args.output_rows; - - execute( - input, ld_input_col, ld_input_row, ld_input_batch, - output, ld_output_col, ld_output_row, ld_output_batch, - working_space - ); - } - - void execute( - const void *const input, - size_t ld_input_col, - size_t ld_input_row, - size_t ld_input_batch, - void *const output, - size_t ld_output_col, - size_t ld_output_row, - size_t ld_output_batch, - void *const working_space - ) const override - { - execute( - m_args.n_batches, m_args.input_rows, m_args.input_cols, - m_args.n_channels, - input, ld_input_col, ld_input_row, ld_input_batch, - m_args.padding, - m_args.output_rows, m_args.output_cols, - output, ld_output_col, ld_output_row, ld_output_batch, - working_space - ); - } - - void execute( - unsigned int batches, - unsigned int input_height, - unsigned int input_width, - unsigned int channels, - const void *const _input, - size_t ld_input_col, - size_t ld_input_row, - size_t ld_input_batch, - const PaddingValues &padding, - unsigned int output_height, - unsigned int output_width, - void *const _output, - size_t ld_output_col, - size_t ld_output_row, - size_t ld_output_batch, - void *const _working_space - ) const override - { - strategy strat(m_args.cpu_info); -#ifdef CYCLE_PROFILING - arm_gemm::profiler prof; -#endif // CYCLE_PROFILING - - // Cast input and output pointers into the right types - const TInput *const inptr = static_cast(_input); - TOutput *const outptr = static_cast(_output); - - // Allocate portions of the working space - uint8_t *const working_space = static_cast(_working_space); - TOutput *const output_buffer = reinterpret_cast(working_space); - TInput *const input_buffer = reinterpret_cast(working_space + sizeof_output_buffer()); - - // Fill the input buffer - const TInput pad_value = (m_args.pool_type == PoolingType::AVERAGE) - ? static_cast(0) - : (std::numeric_limits::has_infinity - ? -std::numeric_limits::infinity() - : std::numeric_limits::lowest()); - for (unsigned int i = 0; i < channels; i++) - { - input_buffer[i] = pad_value; - } - - // Keep subdividing the output plane across the longest dimension until we - // reach the size of the tile. Queue items for later processing. Note - we - // can determine the largest size of the queue a priori from the input - // tensor size, this would allow us to allocate memory within the working - // space and improve performance. - struct WorkItem - { - unsigned int output_i, output_j; - unsigned int output_height, output_width; - - WorkItem(unsigned int i, unsigned int j, unsigned int height, unsigned int width) - : output_i(i), output_j(j), output_height(height), output_width(width) {} - }; - - auto execute = [&] (const WorkItem &item) { - // Create an array for the output pointers - TOutput * _outptr_array[strategy::out_rows() * strategy::out_cols()]; - TOutput **const outptr_array = _outptr_array; - - // Construct the output pointer array - { - const auto output_pad_right = strategy::out_rows() - item.output_width; - auto outptr_element = outptr_array; - auto outptr_row = outptr + item.output_i * ld_output_row + item.output_j * ld_output_col; - - // Fill the array with pointers to the output buffer - for (unsigned int i = 0; i < strategy::out_rows() * strategy::out_cols(); i++) - { - outptr_array[i] = output_buffer; - } - - // Fill in the valid portion of the array - for (unsigned int i = 0; i < item.output_height; i++) - { - auto outptr_col = outptr_row; - for (unsigned int j = 0; j < item.output_width; j++) - { - *(outptr_element++) = outptr_col; - outptr_col += ld_output_col; - } - outptr_element += output_pad_right; - outptr_row += ld_output_row; - } - } - - const int start_i = item.output_i * strategy::stride_rows() - padding.top; - const int end_i = start_i + input_rows(); - const unsigned int pad_top = std::max(0, 0 - start_i); - const unsigned int pad_bottom = std::max(0, end_i - static_cast(input_height)); - - const int start_j = item.output_j * strategy::stride_cols() - padding.left; - const int end_j = start_j + input_cols(); - const unsigned int pad_left = std::max(0, 0 - start_j); - const unsigned int pad_right = std::max(0, end_j - static_cast(input_width)); - - // Create an array for the input pointers - const TInput * _inptr_array[input_rows() * input_cols()]; - const TInput **const inptr_array = _inptr_array; - { - const unsigned int row_padding = pad_top + pad_bottom; - const unsigned int valid_rows = input_rows() - row_padding; - - const unsigned int col_padding = pad_left + pad_right; - const unsigned int valid_cols = input_cols() - col_padding; - - // Fill the array with pointers to the input buffer - for (unsigned int i = 0; i < input_rows() * input_cols(); i++) - { - inptr_array[i] = input_buffer; - } - - // Compute valid initial pointer - auto inptr_row = inptr + std::max(start_i, 0) * ld_input_row + std::max(start_j, 0) * ld_input_col; - - // Fill in the valid portion of the input array - auto inptr_element = inptr_array + pad_top * input_cols() + pad_left; - for (unsigned int i = 0; i < valid_rows; i++) - { - auto inptr_col = inptr_row; - for (unsigned int j = 0; j < valid_cols; j++) - { - *(inptr_element++) = inptr_col; - inptr_col += ld_input_col; - } - - inptr_row += ld_input_row; - inptr_element += col_padding; // Skip the padding elements - } - } - - // Call the kernel -#ifdef CYCLE_PROFILING - // TODO Work number - auto p = prof.ScopedProfiler(PROFILE_KERNEL, (unsigned long)(item.output_height * item.output_width * strategy::pool_rows() * strategy::pool_cols())); -#endif // CYCLE_PROFILING - strat.kernel(channels, inptr_array, outptr_array, - pad_left, pad_top, pad_right, pad_bottom); - }; - - // Add the initial work item to the stack of work. - std::stack> stack; - stack.push(WorkItem(0, 0, output_height, output_width)); - while (!stack.empty()) - { - // Pop an item from the stack, bisect the largest dimension and either - // execute the resulting tiles or add them to the stack if they are too - // large. - const WorkItem item(stack.top()); - stack.pop(); - - if (item.output_height <= strategy::out_rows() && - item.output_width <= strategy::out_cols()) - { - execute(item); - } - else - { - // Split the largest dimension, such that we get an exact number of - // tiles in the first partition. - if (item.output_height >= item.output_width) - { - const unsigned int height_in_tiles = (item.output_height + strategy::out_rows() - 1) / strategy::out_rows(); - const unsigned int tiles_first = height_in_tiles - height_in_tiles / 2; - - const unsigned int height_first = tiles_first * strategy::out_rows(); - const unsigned int height_second = item.output_height - height_first; - - stack.push(WorkItem(item.output_i + height_first, item.output_j, height_second, item.output_width)); - stack.push(WorkItem(item.output_i, item.output_j, height_first, item.output_width)); - } - else - { - const unsigned int width_in_tiles = item.output_width / strategy::out_cols(); - const unsigned int tiles_first = width_in_tiles - width_in_tiles / 2; - - const unsigned int width_first = tiles_first * strategy::out_cols(); - const unsigned int width_second = item.output_width - width_first; - - stack.push(WorkItem(item.output_i, item.output_j + width_first, item.output_height, width_second)); - stack.push(WorkItem(item.output_i, item.output_j, item.output_height, width_first)); - } - } - } - } -}; - -} // namespace pooling -} // namespace arm_conv diff --git a/src/core/NEON/kernels/arm_conv/pooling/pooling_depthfirst_generic.hpp b/src/core/NEON/kernels/arm_conv/pooling/pooling_depthfirst_generic.hpp index 65d9a91977517775845a038a66ce9baa9ad0873d..ded2c751272b5716a1dac5730e129657253bfff8 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/pooling_depthfirst_generic.hpp +++ b/src/core/NEON/kernels/arm_conv/pooling/pooling_depthfirst_generic.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,7 +24,6 @@ #pragma once -#include "arm_compute/core/Error.h" #include "depthfirst_driver.hpp" #include "utils.hpp" #if !defined(_WIN64) && !defined(__OpenBSD__) @@ -137,8 +136,8 @@ class PoolingDepthfirstGeneric : public DepthfirstDriver const OutputStage m_os; protected: - size_t get_working_size_per_thread(unsigned int) const override { return 0; } - void initialise_working_space(void *, unsigned int) const override { /* Nothing */ } + size_t get_working_size_per_thread() const override { return 0; } + void initialise_working_space(void *) const override { /* Nothing */ } /* Compute a portion of the output tensor with padding. */ void compute_tile_padded( @@ -208,10 +207,9 @@ class PoolingDepthfirstGeneric : public DepthfirstDriver const unsigned int channel_start, const unsigned int channel_end, const TensorSpec &input, const TensorSpec &output, - void *working_space + void * ) const override { - ARM_COMPUTE_UNUSED(working_space); // Determine start position and padding const int start_i = static_cast(output_i * this->m_args.pool_stride.rows) - this->m_args.padding.top; const auto input_i = static_cast(start_i < 0 ? 0 : start_i); diff --git a/src/core/NEON/kernels/arm_conv/pooling/pooling_depthfirst_generic_quantized.hpp b/src/core/NEON/kernels/arm_conv/pooling/pooling_depthfirst_generic_quantized.hpp deleted file mode 100644 index f3cb9a1d1f2318260e03e97354ab0663dce7c75f..0000000000000000000000000000000000000000 --- a/src/core/NEON/kernels/arm_conv/pooling/pooling_depthfirst_generic_quantized.hpp +++ /dev/null @@ -1,256 +0,0 @@ -/* - * Copyright (c) 2021 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#pragma once - -#include "pool_common.hpp" -#include "utils.hpp" - -namespace arm_conv { -namespace pooling { - -template -class PoolingDepthfirstGenericQuantized : public PoolingCommon -{ - using TInput = typename strategy::operand_type; - using TOutput = typename strategy::return_type; - - const PoolingArgs m_args; // Copy of arguments - const Requantize32 m_requant; // Quantization parameters - - unsigned int input_rows(void) const - { - return m_args.pool_window.rows; - } - - unsigned int input_cols(void) const - { - return m_args.pool_window.cols; - } - - public: - PoolingDepthfirstGenericQuantized(const PoolingArgs &args, const Requantize32 &rq) : m_args(args), m_requant(rq) - { - } - - PoolingDepthfirstGenericQuantized(PoolingDepthfirstGenericQuantized &) = delete; - PoolingDepthfirstGenericQuantized &operator=(PoolingDepthfirstGenericQuantized &) = delete; - - size_t sizeof_input_pointer_array(void) const - { - return sizeof(TInput *) * input_rows() * input_cols(); - } - - size_t get_working_size(unsigned int num_threads) const override - { - return num_threads * sizeof_input_pointer_array(); - } - - void execute( - const void *const input, - void *const output, - void *const working_space, - unsigned int thread_id, - unsigned int num_threads - ) const override - { - const size_t ld_input_col = m_args.n_channels; - const size_t ld_input_row = ld_input_col * m_args.input_cols; - const size_t ld_input_batch = ld_input_row * m_args.input_rows; - const size_t ld_output_col = ld_input_col; - const size_t ld_output_row = ld_output_col * m_args.output_cols; - const size_t ld_output_batch = ld_output_row * m_args.output_rows; - - execute( - input, ld_input_col, ld_input_row, ld_input_batch, - output, ld_output_col, ld_output_row, ld_output_batch, - working_space, - thread_id, num_threads - ); - } - - void execute( - const void *const input, - size_t ld_input_col, - size_t ld_input_row, - size_t ld_input_batch, - void *const output, - size_t ld_output_col, - size_t ld_output_row, - size_t ld_output_batch, - void *const working_space, - unsigned int thread_id, - unsigned int num_threads - ) const override - { - execute( - m_args.n_batches, m_args.input_rows, m_args.input_cols, - m_args.n_channels, - input, ld_input_col, ld_input_row, ld_input_batch, - m_args.padding, - m_args.output_rows, m_args.output_cols, - output, ld_output_col, ld_output_row, ld_output_batch, - working_space, - thread_id, num_threads - ); - } - - void execute( - unsigned int batches, - unsigned int height, - unsigned int width, - unsigned int channels, - const void *const _input, - size_t ld_input_col, - size_t ld_input_row, - size_t ld_input_batch, - const PaddingValues &padding, - unsigned int output_height, - unsigned int output_width, - void *const _output, - size_t ld_output_col, - size_t ld_output_row, - size_t ld_output_batch, - void *const _working_space, - unsigned int thread_id, - unsigned int num_threads - ) const override - { - strategy strat(m_args.cpu_info); -#ifdef CYCLE_PROFILING - arm_gemm::profiler prof; -#endif // CYCLE_PROFILING - - const unsigned int roundup_output_rows = roundup(output_height, num_threads); - const unsigned int rows_per_thread = roundup_output_rows / num_threads; - int start_out_height = static_cast(thread_id * rows_per_thread); - int end_out_height = std::min(output_height, static_cast((thread_id + 1) * rows_per_thread)); - - unsigned int start_channel = 0; - unsigned int end_channel = channels; - if(output_height == 1) - { - const unsigned int channels_per_thread = roundup(channels, num_threads) / num_threads; - start_channel = thread_id * channels_per_thread; - end_channel = std::min(start_channel + channels_per_thread, channels); - - // Reset start and end rows - start_out_height = 0; - end_out_height = output_height; - } - - if(start_channel >= end_channel) - { - // Early exit in case of multiple threads parallelising on channels - return; - } - - // Cast input and output pointers into the right types - const TInput *const inptr = static_cast(_input) + start_channel; - TOutput *const outptr = static_cast(_output) + start_channel; - - // Grab the input pointer array - uint8_t *const working_space = static_cast(_working_space); - const TInput **const inptr_array = reinterpret_cast(working_space + thread_id * sizeof_input_pointer_array()); - - // For each output tile, construct the requisite set of pointers and call - // into the kernel. - for (unsigned int batch = 0; batch < batches; batch++) - { - // Get batch pointers - const auto inptr_batch = inptr + batch * ld_input_batch; - const auto outptr_batch = outptr + batch * ld_output_batch; - - for (int out_i = start_out_height; out_i < end_out_height; out_i++) - { - const int start_in_i = out_i * m_args.pool_stride.rows - padding.top; - const int end_in_i = start_in_i + m_args.pool_window.rows; - - // Compute top/bottom padding - const auto pad_top = static_cast(-std::min(start_in_i, 0)); - const auto pad_bottom = static_cast(-std::min(static_cast(height) - end_in_i, 0)); - - // Compute the number of pooling window rows which are contained in - // either the valid region of the input tensor, or the padding. - const auto padded_bottom = std::min( - start_in_i + m_args.pool_window.rows, height + padding.bottom - ); - const auto n_total_rows = padded_bottom - start_in_i; - - for (int out_j = 0, start_in_j = -padding.left; - out_j < static_cast(output_width); - out_j++, start_in_j += m_args.pool_stride.cols) - { - const int end_in_j = start_in_j + m_args.pool_window.cols; - - // Compute left/right padding - const auto pad_left = static_cast(-std::min(start_in_j, 0)); - const auto pad_right = static_cast(-std::min(static_cast(width) - end_in_j, 0)); - - // Compute the number of pooling window columns which are contained - // in either the valid region of the input tensor, or the padding. - const auto padded_right = std::min( - start_in_j + m_args.pool_window.cols, width + padding.right - ); - const auto n_total_cols = padded_right - start_in_j; - - // Construct the input pointer array - fill in all valid points - // contiguously. - const TInput **ptrs = inptr_array; - for (auto i = pad_top; i < input_rows() - pad_bottom; i++) - { - // Can skip over the left padding because we will have either the - // same or less than the previous tile. - unsigned int j = pad_left; - const TInput *colptr = inptr_batch + (start_in_i + i) * ld_input_row + (start_in_j + j) * ld_input_col; - for (; j < input_cols() - pad_right; j++) - { - *(ptrs++) = colptr; - colptr += ld_input_col; - } - } - - // Compute the number of valid cells - const auto valid_rows = input_rows() - pad_top - pad_bottom; - const auto valid_cols = input_cols() - pad_left - pad_right; - const auto valid_cells = valid_rows * valid_cols; - const auto cells_in_range = n_total_rows * n_total_cols; - const auto window_cells = m_args.exclude_padding ? valid_cells : cells_in_range; - - // Get the output pointer for this call - TOutput *outptr = outptr_batch + out_i * ld_output_row + out_j * ld_output_col; - -#ifdef CYCLE_PROFILING - // TODO Work number - auto p = prof.ScopedProfiler(PROFILE_KERNEL, (unsigned long) 0); -#endif - strat.kernel(window_cells, valid_cells, end_channel - start_channel, inptr_array, outptr, m_requant); - } - } - } - } -}; - -} // namespace pooling -} // namespace arm_conv diff --git a/src/core/NEON/kernels/arm_gemm/gemm_bf16.cpp b/src/core/NEON/kernels/arm_gemm/gemm_bf16.cpp index 515d55c73bf1ef0deba537dbab1a15151da438ed..72b0fac96a26b130ca97009d8fb14a7b66060b65 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_bf16.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_bf16.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2020, 2022 Arm Limited. + * Copyright (c) 2017-2020, 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -57,6 +57,8 @@ #include "kernels/sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL.hpp" #endif // ARM_COMPUTE_ENABLE_SME2 +#include "kernels/sve_ffhybrid_bf16fp32_mmla_6x4VL.hpp" +#include "kernels/sve_ffinterleaved_bf16fp32_mmla_8x3VL.hpp" #include "kernels/sve_hybrid_bf16fp32_dot_6x4VL.hpp" #include "kernels/sve_hybrid_bf16fp32_mmla_6x4VL.hpp" #include "kernels/sve_interleaved_bf16fp32_dot_8x3VL.hpp" @@ -204,6 +206,30 @@ GemmImplementation::with_estimate( [](const GemmArgs &args) { return GemmInterleavedFixedFormat::estimate_cycles(args); }, [](const GemmArgs &args) { return new GemmInterleavedFixedFormat(args); } ), +GemmImplementation::with_estimate( + GemmMethod::GEMM_INTERLEAVED, + "a64_ffinterleaved_bf16fp32_mmla_8x12", + KernelWeightFormat::VL256_BL64, + [](const GemmArgs &args) { return args._ci->has_bf16(); }, + [](const GemmArgs &args) { return GemmInterleavedFixedFormat::estimate_cycles(args); }, + [](const GemmArgs &args) { return new GemmInterleavedFixedFormat(args); } +), +GemmImplementation::with_estimate( + GemmMethod::GEMM_INTERLEAVED, + "a64_ffhybrid_bf16fp32_mmla_6x16", + KernelWeightFormat::VL256_BL64, + [](const GemmArgs &args) { return args._ci->has_bf16(); }, + [](const GemmArgs &args) { return GemmHybridIndirectFixedFormat::estimate_cycles(args); }, + [](const GemmArgs &args) { return new GemmHybridIndirectFixedFormat(args); } +), +GemmImplementation::with_estimate( + GemmMethod::GEMM_INTERLEAVED, + "a64_ffinterleaved_bf16fp32_dot_8x12", + KernelWeightFormat::VL128_BL32, + [](const GemmArgs &args) { return args._ci->has_bf16(); }, + [](const GemmArgs &args) { return GemmInterleavedFixedFormat::estimate_cycles(args); }, + [](const GemmArgs &args) { return new GemmInterleavedFixedFormat(args); } +), #endif // ARM_COMPUTE_ENABLE_FIXED_FORMAT_KERNELS GemmImplementation::with_estimate( GemmMethod::GEMM_INTERLEAVED, @@ -213,17 +239,7 @@ GemmImplementation::with_estimate( [](const GemmArgs &args) { return new GemmInterleaved(args); } ), #endif // ARM_COMPUTE_ENABLE_BF16 -#elif defined(__arm__) -{ - GemmMethod::GEMM_INTERLEAVED, - "sgemm_8x6", - nullptr, - nullptr, - [](const GemmArgs &args) { return new GemmInterleaved(args); } -}, -#else -# error "Unknown Architecture" -#endif +#endif // __aarch64__ { GemmMethod::DEFAULT, "", diff --git a/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp b/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp index ee567a24989334fbea6b46e28b1dad2ac264bab0..44a7bb894a7dcac2e8cf9ce644fea6420557b04a 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp @@ -66,6 +66,10 @@ #include "kernels/sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL.hpp" #endif // ARM_COMPUTE_ENABLE_SME2 +#include "kernels/sve_ffhybrid_fp32_mla_6x4VL.hpp" +#include "kernels/sve_ffhybrid_fp32bf16fp32_mmla_4x6VL.hpp" +#include "kernels/sve_ffinterleaved_fp32_mla_8x3VL.hpp" +#include "kernels/sve_ffinterleaved_bf16fp32_mmla_8x3VL.hpp" #include "kernels/sve_hybrid_fp32bf16fp32_mmla_4x6VL.hpp" #include "kernels/sve_hybrid_fp32bf16fp32_mmla_6x4VL.hpp" #include "kernels/sve_hybrid_fp32_mla_6x4VL.hpp" diff --git a/src/core/NEON/kernels/arm_gemm/gemm_hybrid_indirect.hpp b/src/core/NEON/kernels/arm_gemm/gemm_hybrid_indirect.hpp index 0bbcd10b66e74c0ed51bfedab1bcc30ed0f63d19..1780375c44eb5d91b92c724b3632344c165f915e 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_hybrid_indirect.hpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_hybrid_indirect.hpp @@ -440,7 +440,7 @@ public: in_row_strings = std::vector(_args._Ksections, nullptr); for (unsigned int i=0; i<_args._Ksections; i++) { - in_row_strings[i] = &(in_row_ptrs[i * strategy::out_height()]); + in_row_strings[i] = &(in_row_ptrs.data()[i * strategy::out_height()]); } } diff --git a/src/core/NEON/kernels/arm_gemm/gemm_implementation.hpp b/src/core/NEON/kernels/arm_gemm/gemm_implementation.hpp index 19c8fcadd3841a14d21b189906d15f695320be47..5e77df7d4add91984653f21a1abfe4cdf3ac14ed 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_implementation.hpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_implementation.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2020, 2022 Arm Limited. + * Copyright (c) 2018-2020, 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -212,9 +212,11 @@ struct GemmImplementation { instantiate(instantiate) { } }; -/* "Main" function implemented for each valid combination of types. - * Returns a list of GEMM implementation descriptors for processing by the - * other functions, ended by an implementation with +/* Provides the list of implementation descriptors which is processed by the + * other functions. + * + * A specialised version is provided for each supported combination of types. + * The end of the list is indicated by a sentinel descriptor with * method==GemmMethod::DEFAULT. */ template const GemmImplementation *gemm_implementation_list(); diff --git a/src/core/NEON/kernels/arm_gemm/gemm_int16.cpp b/src/core/NEON/kernels/arm_gemm/gemm_int16.cpp index 18d8fc93129f1f8e3692cfcfe9df49304769cf85..aa6ecc29199ff36e09003fcd244209eabf4ca15b 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_int16.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_int16.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2020, 2022 Arm Limited. + * Copyright (c) 2017-2020, 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -57,6 +57,7 @@ const GemmImplementation *gemm_implementation_list gemm(const GemmArgs &args, const Nothing &); template bool has_opt_gemm(WeightFormat &weight_format, const GemmArgs &args, const Nothing &); +template KernelDescription get_gemm_method(const GemmArgs &args, const Nothing &); template std::vector get_compatible_kernels (const GemmArgs &args, const Nothing &); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp index b0a01886d2a998b35dd1324d2136b6f35488bbc6..fd20e53f60e67d545a6e2b04e002cf507c6b65a3 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp @@ -186,6 +186,7 @@ const GemmImplementation *gemm_implementation_list gemm(const GemmArgs &args, const Nothing &); template bool has_opt_gemm(WeightFormat &weight_format, const GemmArgs &args, const Nothing &); +template KernelDescription get_gemm_method(const GemmArgs &args, const Nothing &); template std::vector get_compatible_kernels (const GemmArgs &args, const Nothing &); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/gemm_interleaved_pretransposed_2d.hpp b/src/core/NEON/kernels/arm_gemm/gemm_interleaved_pretransposed_2d.hpp deleted file mode 100644 index b71f390ab9dbf0ed4de87966332d7fe3055f74fe..0000000000000000000000000000000000000000 --- a/src/core/NEON/kernels/arm_gemm/gemm_interleaved_pretransposed_2d.hpp +++ /dev/null @@ -1,566 +0,0 @@ -/* - * Copyright (c) 2020 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#include "arm_gemm.hpp" -#include "utils.hpp" - -#include "mergeresults.hpp" -#include "transform.hpp" - -#ifdef CYCLE_PROFILING -#include "profiler.hpp" -#endif - -#include -#include -#include - -// Some macros used to decide how much working space to allocate. -// Round allocations up to the next cache line. -#define ALLOC_ROUND 64 -#define ROUND_UP(x) ((((x) + ALLOC_ROUND-1) / ALLOC_ROUND) * ALLOC_ROUND) - -// Implementation of the GemmCommon abstract class. -// -// This implementation interleaves the source matrices in blocks - good for -// larger matrices. -namespace arm_gemm { - -template -class GemmInterleavedPretransposed2d : public GemmCommon { - typedef typename strategy::operand_type Toi; - typedef typename strategy::result_type Tri; - - /* const properties set by constructor */ - const CPUInfo * const _ci; - - const unsigned int _Msize; - const unsigned int _Nsize; - const unsigned int _Ksize; - - const unsigned int _nbatches; - const unsigned int _nmulti; - - const Activation _act; - - const int _maxthreads; - int _nthreads; - - /* Blocking info */ - unsigned int _k_block=0; - unsigned int _x_block=0; - - unsigned int _Mround_div=0; - unsigned int _Mround=0; - unsigned int _Nround_div=0; - unsigned int _Nround=0; - - /* Working space, pretransposed buffer */ - const Toi *_B_transposed=nullptr; - void *_working_space=nullptr; - - /* We will need to walk through the blocks of B in a few contexts, so - * factor that out. */ - class blockwalker { - private: - /* Size loops, etc. based on our parent's configuration */ - const GemmInterleavedPretransposed2d &_parent; - - /* K, X and multi parameters for current iteration. */ - unsigned int _k0=0, _x0=0, _xmin=0, _xmax=0, _multi=0; - - unsigned int _index=0; - bool _done=false; - bool _newkblock=true; - bool _newmulti=true; - - public: - blockwalker(const GemmInterleavedPretransposed2d &parent) - : _parent(parent) - , _xmax { parent._Nsize } - { } - - blockwalker(const GemmInterleavedPretransposed2d &parent, unsigned int x0, unsigned int xmax) - : _parent(parent) - , _x0 { x0 } - , _xmin { x0 } - , _xmax { xmax } - { - assert(_x0 <= _xmax); - } - - unsigned int xmax() { - return std::min(_x0 + _parent._x_block, _xmax); - } - - unsigned int kmax() { - return std::min(_k0 + _parent._k_block, _parent._Ksize); - } - - /* Advance to the next block, return false at the end. */ - bool advance(void) { - if (_done) { - return false; - } - - _newkblock=false; - _x0 += _parent._x_block; - if (_x0 >= _xmax) { - _x0=_xmin; - _k0 += _parent._k_block; - if (_k0 >= _parent._Ksize) { - _k0=0; - _multi++; - if (_multi >= _parent._nmulti) { - _done=true; - return false; - } - _newmulti=true; - } - _newkblock=true; - } - _index++; - - return true; - } - - unsigned int k0(void) { return _k0; } - unsigned int x0(void) { return _x0; } - unsigned int multi(void) { return _multi; } - unsigned int index(void) { return _index; } - bool done(void) { return _done; } - bool newkblock(void) { return _newkblock; } - }; - - // A working size: One of these needed, regardless of thread count. Divided according to window. - size_t get_a_working_size() const { - return ROUND_UP(sizeof(Toi) * _k_block * _Mround * _nbatches) * 2; - } - - // As B will be pretranspose we do not need to alloc any space for it - size_t get_b_working_size() const { - return 0; - } - - // C working size: One needed per thread. - size_t get_c_working_size() const { - return ROUND_UP(sizeof(Tri) * _x_block * strategy::out_height()); - } - - // Internal execute function. - // This supports both the "pretransposed" and "standard" interfaces via the template parameter. - void execute_pretranspose(unsigned int m_start, unsigned int m_end, unsigned int n_start, unsigned int n_end, int threadid, int, int) { - /* Make sure we've been set up correctly. */ - assert(_B_transposed); - assert(_working_space); - assert(this->_Aptr); - assert(this->_Cptr); - -#ifdef CYCLE_PROFILING - profiler prof; -#endif - strategy strat(_ci); - - /* Translate 'start' and 'end' into a position within the batches and rows. */ - const unsigned int window_per_batch = _Mround / strategy::out_height(); - unsigned int batch_0 = m_start / window_per_batch; - unsigned int batch_end = m_end / window_per_batch; - - /* Compute the M values to operate on */ - unsigned int m_0 = (m_start - (batch_0 * window_per_batch)) * strategy::out_height(); - unsigned int m_max = (m_end - (batch_end * window_per_batch)) * strategy::out_height(); - - unsigned int n_0 = std::min(this->_Nsize, strategy::out_width() * n_start); - unsigned int n_max = std::min(this->_Nsize, strategy::out_width() * n_end); - - blockwalker current(*this, n_0, n_max); - - int8_t *working_space_bytes = reinterpret_cast(_working_space); - - auto c_panel_start = working_space_bytes; - auto a_panel_start = c_panel_start + get_c_working_size() * _maxthreads; - - auto c_panel = reinterpret_cast(c_panel_start + get_c_working_size() * threadid); - auto a_panel = reinterpret_cast(a_panel_start + get_a_working_size() * threadid); - - /* B^t is stored in interleaved panels separated by their K-block component - * we want to store a pointer to the start of the current k-page - * then when we come to the next k-block we just add the size of the previous to - * this base pointer - */ - const Toi *b_panel_start = _B_transposed; - // b_panels stores a pointer to the start of our current block inside of the k-block - const Toi *b_panel = b_panel_start; - - // newkblock() is always true on the first iteration, so this will be set properly on the first loop. - unsigned b_page_size = 0; - int kern_k = 0; - for (;!current.done();current.advance()) { - int bblocks = iceildiv(current.xmax() - current.x0(), strategy::out_width()); - - if (current.newkblock()) { - kern_k = iceildiv(current.kmax() - current.k0(), strategy::k_unroll()); - kern_k *= strat.k_unroll(); - - unsigned b_thread_start_offset = iceildiv(current.x0(), strategy::out_width()); - - b_panel_start += b_page_size; - b_panel = b_panel_start + (b_thread_start_offset * strat.out_width() * kern_k); - b_page_size = _Nround * kern_k; - - for (unsigned int batch = batch_0; batch <= batch_end; batch++) { - unsigned int first_m = (batch == batch_0) ? m_0 : 0; - unsigned int last_m = (batch == batch_end) ? m_max : _Msize; - - if (first_m >= last_m) - continue; - - auto a_thread_panel_in = this->_Aptr - + (batch * this->_A_batch_stride) - + (current.multi() * this->_A_multi_stride); - - auto a_thread_panel_out = a_panel + ((batch * _Mround + first_m) * _k_block); - - strat.transforms.PrepareA( - a_thread_panel_out, - a_thread_panel_in, - this->_lda, - first_m, - last_m, - current.k0(), - current.kmax(), - 0); - } - } - - /* Do the actual work. */ - for (unsigned int batch = batch_0; batch <= batch_end; batch++) { - unsigned int first_m = (batch == batch_0) ? m_0 : 0; - unsigned int last_m = (batch == batch_end) ? m_max : _Msize; - - const Toi *a_ptr = a_panel + (batch * _Mround + first_m) * _k_block; - - if (first_m >= last_m) - continue; - - for (unsigned int y=first_m; y_Cptr - + this->_C_batch_stride * batch - + this->_C_multi_stride * current.multi(); - - auto bias = (first_pass && this->_bias) - ? this->_bias + (current.multi() * this->_bias_multi_stride) - : nullptr; - - auto act = last_pass ? _act : Activation(); - - strat.transforms.Merge( - c_panel_out, - c_panel, - this->_ldc, - y, - ymax, - current.x0(), - current.xmax(), - bias, - act, - !first_pass); //Append - } - } - - b_panel += (bblocks * strat.out_width() * kern_k); - } - } - - static unsigned int get_k_block_size(const GemmArgs &args) { - // Work out blocking parameters, or override from provided GemmConfig - if (args._cfg && args._cfg->inner_block_size) { - return args._cfg->inner_block_size; - } - - const unsigned int L1_size = args._ci->get_L1_cache_size(); - unsigned int k_block; - - // k_block: Find out how much of the larger array can be loaded into half the cache. - // This should account for associative caches. - k_block = (L1_size / 2) / (sizeof(Toi) * (std::max(strategy::out_width(), strategy::out_height()))); - - // Needs to be (at least a single) multiple of the K unroll level. - k_block /= strategy::k_unroll(); - k_block = std::max(k_block, 1U) * strategy::k_unroll(); - - // Now tune to presented problem size; this is how many blocks we need. - unsigned int numk_blocks = iceildiv(args._Ksize, k_block); - - // So divide the space equally into that many blocks. - k_block = iceildiv(args._Ksize, numk_blocks); - - // And round UP to the K unroll level required. - k_block = iceildiv(k_block, strategy::k_unroll()); - k_block *= strategy::k_unroll(); - - return k_block; - } - -public: - GemmInterleavedPretransposed2d(GemmInterleavedPretransposed2d &) = delete; - GemmInterleavedPretransposed2d & operator= (GemmInterleavedPretransposed2d &) = delete; - - /* Constructor */ - GemmInterleavedPretransposed2d(const GemmArgs &args) - : _ci(args._ci) - , _Msize(args._Msize) - , _Nsize(args._Nsize) - , _Ksize(args._Ksize) - , _nbatches(args._nbatches) - , _nmulti(args._nmulti) - , _act(args._act) - , _maxthreads(args._maxthreads) - , _nthreads(args._maxthreads) - , _k_block(get_k_block_size(args)) - // Work out the rounded size of M - needed for some buffers. - , _Mround_div ( iceildiv(_Msize, strategy::out_height()) ) - , _Mround ( _Mround_div * strategy::out_height() ) - - , _Nround_div ( iceildiv(_Nsize, strategy::out_width()) ) - , _Nround ( _Nround_div * strategy::out_width() ) - { - assert(_maxthreads > 0); - - const unsigned int L2_size = _ci->get_L2_cache_size(); - - if (args._cfg && args._cfg->outer_block_size) { - _x_block = args._cfg->outer_block_size; - } else { - // x_block: Work out how many rows (of length k_block) will fit in the L2 - // Don't allocate more than 90% of the L2 to allow for overheads, and subtract off the L1 contents. - _x_block = (((L2_size * 9) / 10) - (_k_block * sizeof(Toi) * (strategy::out_width() + strategy::out_height()))) / - (sizeof(Toi) * _k_block); - - // Needs to be (at least a single) multiple of the kernel output width. - _x_block /= strategy::out_width(); - _x_block = std::max(_x_block, 1U) * strategy::out_width(); - - // And tune to the presented problem size. - unsigned int num_x_blocks = iceildiv(_Nsize, _x_block); - _x_block = iceildiv(_Nsize, num_x_blocks); - - _x_block = iceildiv(_x_block, strategy::out_width()); - _x_block *= strategy::out_width(); - } - } - - // Interface implementation - Compulsory functions - ndrange_t get_window_size() const override { - unsigned m = (_Mround / strategy::out_height()) * _nbatches; - unsigned n = _Nround_div; - - return { m, n }; - } - - bool supports_dynamic_scheduling() const override { - return true; - } - - // set_nthreads: pass on to buffer manager to avoid it waiting for non-existant threads. - void set_nthreads(int nthreads) override { - _nthreads = std::min(nthreads, _maxthreads); - } - - void execute(const ndcoord_t& work_range, const ndcoord_t& thread_locator, int threadid) override { - /* This particular GEMM implementation can only be broken up over the M & N - * dimensions, we inform the frame work of this limitation via the get_window_size function - */ - const auto m_start = work_range.get_position(0); - const auto n_start = work_range.get_position(1); - const auto m_size = work_range.get_size(0); - const auto n_size = work_range.get_size(1); - const auto m_end = m_start + m_size; - const auto n_end = n_start + n_size; - - const auto m_threadid = thread_locator.get_position(0); - const auto n_threadid = thread_locator.get_position(1); - - execute_pretranspose(m_start, m_end, n_start, n_end, threadid, m_threadid, n_threadid); - } - - std::size_t get_working_size() const override { - /* Because we do not know how schedular will break up - * the task, we need to ensure that alloc enough - * space to be able to handle the case where every thread - * is parallelised across B AND also every thrread is parallelised across A - * - * If we parallelise across A, then we only need one buffer of A and 64 buffers of B - * If we parallelise across B, then we only need 64 buffer of B and - */ - return get_c_working_size() * _maxthreads - + get_a_working_size() * _maxthreads - + 64; //to account for cacheline alignment - } - - - void set_working_space(void *working_space) override { - // Make sure everything ends up cache line aligned - int8_t *working_space_bytes = reinterpret_cast(working_space); - intptr_t working_space_int = reinterpret_cast(working_space); - - size_t diff=0; - - if (working_space_int & 0x3F) { - diff = 0x40 - (working_space_int & 0x3F); - } - - working_space_bytes += diff; - - _working_space = reinterpret_cast(working_space_bytes); - } - - // Interface implementation - pretransposed - bool B_is_pretransposed() const override { - return true; - } - - bool B_pretranspose_required() const override { - return _B_transposed==nullptr; - } - - // TODO: this could almost certainly be considerably simpler. - size_t get_B_pretransposed_array_size() const override { - size_t total=0; - blockwalker current(*this); - - do { - /* Figure out the size of each block. */ - unsigned int x_size = (current.xmax() - current.x0()); - unsigned int k_size = (current.kmax() - current.k0()); - - /* Round sizes up as needed. */ - x_size = iceildiv(x_size, strategy::out_width()); - x_size *= strategy::out_width(); - - k_size = iceildiv(k_size, strategy::k_unroll()); - k_size *= strategy::k_unroll(); - - total += x_size * k_size * sizeof(Toi); - } while (current.advance()); - - return total; - } - - void pretranspose_B_array(void *in_buffer, const To *B, const int ldb, const int B_multi_stride) override { - blockwalker current(*this); - Toi *buffer = reinterpret_cast(in_buffer); - _B_transposed = buffer; - strategy strat(_ci); - - do { - /* Figure out the size of each block. */ - unsigned int x_size = (current.xmax() - current.x0()); - unsigned int k_size = (current.kmax() - current.k0()); - - /* Round sizes up as needed. */ - x_size = iceildiv(x_size, strategy::out_width()); - x_size *= strategy::out_width(); - - k_size = iceildiv(k_size, strategy::k_unroll()); - k_size *= strategy::k_unroll(); - - strat.transforms.PrepareB(buffer, B + (current.multi() * B_multi_stride), ldb, - current.x0(), current.xmax(), current.k0(), current.kmax()); - - buffer += (x_size * k_size); - } while (current.advance()); - } - - void set_pretransposed_B_data(void *in_buffer) override { - _B_transposed = reinterpret_cast(in_buffer); - } - - // Estimate cycles for given problem given provided parameters - static uint64_t estimate_cycles(const GemmArgs &args, const PerformanceParameters ¶ms) { - unsigned int k_blocks = iceildiv(args._Ksize, get_k_block_size(args)); - unsigned int m_blocks = iceildiv(args._Msize, strategy::out_height()) * args._nbatches; - unsigned int n_blocks = iceildiv(args._Nsize, strategy::out_width()); - - uint64_t total_macs = static_cast(args._nbatches) * args._nmulti * roundup(args._Msize, strategy::out_height()) * roundup(args._Nsize, strategy::out_width()) * roundup(args._Ksize, strategy::k_unroll()); - uint64_t prepare_bytes = static_cast(args._nbatches) * args._nmulti * roundup(args._Msize, strategy::out_height()) * roundup(args._Ksize, strategy::k_unroll()) * sizeof(Toi); - uint64_t merge_bytes = static_cast(args._nbatches) * args._nmulti * k_blocks * roundup(args._Msize, strategy::out_height()) * roundup(args._Nsize, strategy::out_width()) * sizeof(Tr); - - // Wide problems incur extra preparation cost, as it is done per thread. - // Duplicate the logic the scheduler will later use to figure out how much that will affect us - float ratio = m_blocks / static_cast(n_blocks); - - unsigned int ideal_height = static_cast(std::sqrt(args._maxthreads * ratio) + 0.5); - unsigned int height = 1; - - if (ideal_height == 0) { - height = 1; - } else { - for (unsigned int adj=0; adj(total_macs) / params.kernel_macs_cycle; - float prepare_cycles = static_cast(prepare_bytes) / params.prepare_bytes_cycle; - float merge_cycles = static_cast(merge_bytes) / params.merge_bytes_cycle; - - float total_cycles = mac_cycles + prepare_cycles + merge_cycles; - - // We can't thread over multis, which might be a problem in some - // threaded cases. Penalize that here. - float parallelism_available = static_cast(iceildiv(args._Msize, strategy::out_height()) * args._nbatches * iceildiv(args._Nsize, strategy::out_width())) * 0.9; - - if (parallelism_available < args._maxthreads) { - total_cycles *= (static_cast(args._maxthreads) / parallelism_available); - } - - return static_cast(total_cycles); - } -}; - -} // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/gemm_qint8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_qint8.cpp index 9e8907d60f7f38bcaaf7c867ef1c8478a8d79e54..c725815859fd56f59a1209648d29e12a14029b7a 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_qint8.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_qint8.cpp @@ -80,7 +80,7 @@ static const GemmImplementation gemm_qint8_methods { GemmMethod::GEMM_INTERLEAVED, "sme2_interleaved_nomerge_s8q_mopa_1VLx4VL", - [](const GemmArgs &args, const Requantize32 &qp) { return args._ci->has_sme2() && ((qp.per_channel_requant && (qp.per_channel_left_shifts == nullptr)) || (!qp.per_channel_requant && (qp.per_layer_left_shift == 0)));}, + [](const GemmArgs &args, const Requantize32 &qp) { return args._ci->has_sme2() && args._maxthreads == 1 && ((qp.per_channel_requant && (qp.per_channel_left_shifts == nullptr)) || (!qp.per_channel_requant && (qp.per_layer_left_shift == 0)));}, [](const GemmArgs &args, const Requantize32 &) { const auto VL = sme::get_vector_length(); return args._Msize <= VL || (2*VL < args._Msize && args._Msize <= 3*VL); }, [](const GemmArgs &args, const Requantize32 &qp) { return new GemmInterleavedPretransposedNoMergeQuantizedInline(args, qp); } @@ -88,7 +88,7 @@ static const GemmImplementation gemm_qint8_methods { GemmMethod::GEMM_INTERLEAVED, "sme2_interleaved_nomerge_s8q_mopa_4VLx1VL", - [](const GemmArgs &args, const Requantize32 &qp) { return args._ci->has_sme2() && ((qp.per_channel_requant && (qp.per_channel_left_shifts == nullptr)) || (!qp.per_channel_requant && (qp.per_layer_left_shift == 0)));}, + [](const GemmArgs &args, const Requantize32 &qp) { return args._ci->has_sme2() && args._maxthreads == 1 && ((qp.per_channel_requant && (qp.per_channel_left_shifts == nullptr)) || (!qp.per_channel_requant && (qp.per_layer_left_shift == 0)));}, [](const GemmArgs &args, const Requantize32 &) { const auto VL = sme::get_vector_length(); return args._Nsize <= VL || (2*VL < args._Nsize && args._Nsize <= 3*VL); }, [](const GemmArgs &args, const Requantize32 &qp) { return new GemmInterleavedPretransposedNoMergeQuantizedInline(args, qp); } @@ -96,7 +96,7 @@ static const GemmImplementation gemm_qint8_methods { GemmMethod::GEMM_INTERLEAVED, "sme2_interleaved_nomerge_s8q_mopa_2VLx2VL", - [](const GemmArgs &args, const Requantize32 &qp) { return args._ci->has_sme2() && ((qp.per_channel_requant && (qp.per_channel_left_shifts == nullptr)) || (!qp.per_channel_requant && (qp.per_layer_left_shift == 0)));}, + [](const GemmArgs &args, const Requantize32 &qp) { return args._ci->has_sme2() && args._maxthreads == 1 && ((qp.per_channel_requant && (qp.per_channel_left_shifts == nullptr)) || (!qp.per_channel_requant && (qp.per_layer_left_shift == 0)));}, nullptr, [](const GemmArgs &args, const Requantize32 &qp) { return new GemmInterleavedPretransposedNoMergeQuantizedInline(args, qp); } }, @@ -265,6 +265,7 @@ const GemmImplementation *gemm_implementation_list template UniqueGemmCommon gemm(const GemmArgs &args, const Requantize32 &os); template bool has_opt_gemm(WeightFormat &weight_format, const GemmArgs &args, const Requantize32 &os); +template KernelDescription get_gemm_method(const GemmArgs &args, const Requantize32 &os); template std::vector get_compatible_kernels(const GemmArgs &args, const Requantize32 &os); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/gemm_quint8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_quint8.cpp index f93f56b57da40c47c82ca38b20a3729716416568..6254ec668d159b6e4b02f37ee5a672aa524be557 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_quint8.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_quint8.cpp @@ -76,7 +76,7 @@ static const GemmImplementation gemm_quint8_meth { GemmMethod::GEMM_INTERLEAVED, "sme2_interleaved_nomerge_u8q_mopa_1VLx4VL", - [](const GemmArgs &args, const Requantize32 &qp) { return args._ci->has_sme2() && ((qp.per_channel_requant && (qp.per_channel_left_shifts == nullptr)) || (!qp.per_channel_requant && (qp.per_layer_left_shift == 0)));}, + [](const GemmArgs &args, const Requantize32 &qp) { return args._ci->has_sme2() && args._maxthreads == 1 && ((qp.per_channel_requant && (qp.per_channel_left_shifts == nullptr)) || (!qp.per_channel_requant && (qp.per_layer_left_shift == 0)));}, [](const GemmArgs &args, const Requantize32 &) { const auto VL = sme::get_vector_length(); return args._Msize <= VL || (2*VL < args._Msize && args._Msize <= 3*VL); }, [](const GemmArgs &args, const Requantize32 &qp) { return new GemmInterleavedPretransposedNoMergeQuantizedInline(args, qp); } @@ -84,7 +84,7 @@ static const GemmImplementation gemm_quint8_meth { GemmMethod::GEMM_INTERLEAVED, "sme2_interleaved_nomerge_u8q_mopa_4VLx1VL", - [](const GemmArgs &args, const Requantize32 &qp) { return args._ci->has_sme2() && ((qp.per_channel_requant && (qp.per_channel_left_shifts == nullptr)) || (!qp.per_channel_requant && (qp.per_layer_left_shift == 0)));}, + [](const GemmArgs &args, const Requantize32 &qp) { return args._ci->has_sme2() && args._maxthreads == 1 && ((qp.per_channel_requant && (qp.per_channel_left_shifts == nullptr)) || (!qp.per_channel_requant && (qp.per_layer_left_shift == 0)));}, [](const GemmArgs &args, const Requantize32 &) { const auto VL = sme::get_vector_length(); return args._Nsize <= VL || (2*VL < args._Nsize && args._Nsize <= 3*VL); }, [](const GemmArgs &args, const Requantize32 &qp) { return new GemmInterleavedPretransposedNoMergeQuantizedInline(args, qp); } @@ -92,7 +92,7 @@ static const GemmImplementation gemm_quint8_meth { GemmMethod::GEMM_INTERLEAVED, "sme2_interleaved_nomerge_u8q_mopa_2VLx2VL", - [](const GemmArgs &args, const Requantize32 &qp) { return args._ci->has_sme2() && ((qp.per_channel_requant && (qp.per_channel_left_shifts == nullptr)) || (!qp.per_channel_requant && (qp.per_layer_left_shift == 0)));}, + [](const GemmArgs &args, const Requantize32 &qp) { return args._ci->has_sme2() && args._maxthreads == 1 && ((qp.per_channel_requant && (qp.per_channel_left_shifts == nullptr)) || (!qp.per_channel_requant && (qp.per_layer_left_shift == 0)));}, nullptr, [](const GemmArgs &args, const Requantize32 &qp) { return new GemmInterleavedPretransposedNoMergeQuantizedInline(args, qp); } }, @@ -233,6 +233,7 @@ const GemmImplementation *gemm_implementation_li template UniqueGemmCommon gemm(const GemmArgs &args, const Requantize32 &os); template bool has_opt_gemm(WeightFormat &weight_format, const GemmArgs &args, const Requantize32 &os); +template KernelDescription get_gemm_method(const GemmArgs &args, const Requantize32 &os); template std::vector get_compatible_kernels(const GemmArgs &args, const Requantize32 &os); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp b/src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp index fc836f9790990b545f9ced1f42cb30c1c1ed40b9..25b6cf0cf219471894ddd39e8574eb8909b5f851 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_uint16.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2020, 2022 Arm Limited. + * Copyright (c) 2017-2020, 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -57,6 +57,7 @@ const GemmImplementation *gemm_implementation_list gemm(const GemmArgs &args, const Nothing &); template bool has_opt_gemm(WeightFormat &weight_format, const GemmArgs &args, const Nothing &); +template KernelDescription get_gemm_method(const GemmArgs &args, const Nothing &); template std::vector get_compatible_kernels(const GemmArgs &args, const Nothing &); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp index fcc95eb503fb973203815b92c2de82a846a93d34..af5cfbbf2bf27aa32152bf9fe2c9c28ed998f51d 100644 --- a/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp +++ b/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp @@ -150,6 +150,7 @@ const GemmImplementation *gemm_implementation_list gemm(const GemmArgs &args, const Nothing &); template bool has_opt_gemm(WeightFormat &weight_format, const GemmArgs &args, const Nothing &); +template KernelDescription get_gemm_method(const GemmArgs &args, const Nothing &); template std::vector get_compatible_kernels (const GemmArgs &args, const Nothing &); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_s8_s8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_s8_s8.hpp index 4dfe46446ec6af7a13c7c80c5dfdde0da6c4a72c..e4bfc0f6e49a31fe200fb66c120024bc33108ca9 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_s8_s8.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_s8_s8.hpp @@ -170,7 +170,6 @@ void interleave_block<4, 16, VLType::None, false>( "str q16, [%x[out_ptr], #0x30]\n" "add %x[out_ptr], %x[out_ptr], #0x40\n" "12:" // Odds skip - : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) : "cc", "memory", "v16", "v17", "v18", "v19", "x20", "x21", "x22", "x23" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_s8_s8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_s8_s8_summing.hpp index 56ca49a36e145ed03808b95b196a902cd5b0d70c..23800edf20970dcc85e78dd1a349c37215f84323 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_s8_s8_summing.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_s8_s8_summing.hpp @@ -210,8 +210,8 @@ void interleave_block<4, 16, VLType::None, true>( "sadalp v22.4s, v26.8h\n" "sadalp v21.4s, v25.8h\n" "addp v24.4s, v24.4s, v23.4s\n" - "addp v23.4s, v22.4s, v21.4s\n" - "addp v24.4s, v24.4s, v23.4s\n" + "addp v16.4s, v22.4s, v21.4s\n" + "addp v24.4s, v24.4s, v16.4s\n" "add v24.4s, v24.4s, v20.4s\n" "str q24, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_u8_u8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_u8_u8_summing.hpp index 4c7bb71fb223db0dc3971d2f984246b404e0eb81..15545c24dbc49a54edc61bb21ceba65fb8714562 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_u8_u8_summing.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave4_block16_u8_u8_summing.hpp @@ -210,8 +210,8 @@ void interleave_block<4, 16, VLType::None, true>( "uadalp v22.4s, v26.8h\n" "uadalp v21.4s, v25.8h\n" "addp v24.4s, v24.4s, v23.4s\n" - "addp v23.4s, v22.4s, v21.4s\n" - "addp v24.4s, v24.4s, v23.4s\n" + "addp v16.4s, v22.4s, v21.4s\n" + "addp v24.4s, v24.4s, v16.4s\n" "add v24.4s, v24.4s, v20.4s\n" "str q24, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_bf16_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_bf16_fp32.hpp index 2ba2aa854a5fda8ebf915c12c231528349d18a4e..b900c330b771df388dfacc4f8be774202acee8e1 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_bf16_fp32.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_bf16_fp32.hpp @@ -80,36 +80,36 @@ void interleave_block<8, 1, VLType::None, false>( "prfm pldl1keep, [x21, #0x40]\n" "blt 3f\n" "2:" // Main loop head - "ldr d28, [x28], #0x8\n" - "ldr d27, [x27], #0x8\n" - "shll v28.4s, v28.4h, #0x10\n" + "ldr d27, [x28], #0x8\n" + "ldr d26, [x27], #0x8\n" "shll v27.4s, v27.4h, #0x10\n" + "shll v26.4s, v26.4h, #0x10\n" "ldr d22, [x26], #0x8\n" "ldr d21, [x25], #0x8\n" "shll v22.4s, v22.4h, #0x10\n" "shll v21.4s, v21.4h, #0x10\n" - "ldr d26, [x24], #0x8\n" + "ldr d20, [x24], #0x8\n" "ldr d25, [x23], #0x8\n" - "shll v26.4s, v26.4h, #0x10\n" - "shll v25.4s, v25.4h, #0x10\n" - "ldr d20, [x22], #0x8\n" - "ldr d19, [x21], #0x8\n" "shll v20.4s, v20.4h, #0x10\n" + "shll v25.4s, v25.4h, #0x10\n" + "ldr d19, [x22], #0x8\n" + "ldr d16, [x21], #0x8\n" "shll v19.4s, v19.4h, #0x10\n" - "zip1 v24.4s, v28.4s, v22.4s\n" - "zip1 v23.4s, v27.4s, v21.4s\n" + "shll v16.4s, v16.4h, #0x10\n" + "zip1 v24.4s, v27.4s, v22.4s\n" + "zip1 v23.4s, v26.4s, v21.4s\n" "subs %x[width], %x[width], #0x4\n" "cmp %x[width], #0x4\n" - "zip1 v18.4s, v26.4s, v20.4s\n" - "zip1 v17.4s, v25.4s, v19.4s\n" + "zip1 v18.4s, v20.4s, v19.4s\n" + "zip1 v17.4s, v25.4s, v16.4s\n" "prfm pldl1keep, [x28, #0x70]\n" "prfm pldl1keep, [x27, #0x70]\n" - "zip2 v22.4s, v28.4s, v22.4s\n" - "zip2 v21.4s, v27.4s, v21.4s\n" + "zip2 v22.4s, v27.4s, v22.4s\n" + "zip2 v21.4s, v26.4s, v21.4s\n" "prfm pldl1keep, [x26, #0x70]\n" "prfm pldl1keep, [x25, #0x70]\n" - "zip2 v20.4s, v26.4s, v20.4s\n" - "zip2 v19.4s, v25.4s, v19.4s\n" + "zip2 v20.4s, v20.4s, v19.4s\n" + "zip2 v19.4s, v25.4s, v16.4s\n" "prfm pldl1keep, [x24, #0x70]\n" "prfm pldl1keep, [x23, #0x70]\n" "prfm pldl1keep, [x22, #0x70]\n" @@ -138,71 +138,70 @@ void interleave_block<8, 1, VLType::None, false>( "ldr s28, [x28], #0x4\n" "ldr s27, [x27], #0x4\n" "mov x20, #0x2\n" - "ldr s22, [x26], #0x4\n" - "ldr s21, [x25], #0x4\n" - "ldr s26, [x24], #0x4\n" - "ldr s25, [x23], #0x4\n" - "ldr s20, [x22], #0x4\n" - "ldr s19, [x21], #0x4\n" + "ldr s26, [x26], #0x4\n" + "ldr s25, [x25], #0x4\n" + "ldr s24, [x24], #0x4\n" + "ldr s23, [x23], #0x4\n" + "ldr s22, [x22], #0x4\n" + "ldr s21, [x21], #0x4\n" "tbz %x[width], #0, 5f\n" "ld1 { v28.h }[2], [x28]\n" "ld1 { v27.h }[2], [x27]\n" "mov x20, #0x3\n" - "ld1 { v22.h }[2], [x26]\n" - "ld1 { v21.h }[2], [x25]\n" - "ld1 { v26.h }[2], [x24]\n" - "ld1 { v25.h }[2], [x23]\n" - "ld1 { v20.h }[2], [x22]\n" - "ld1 { v19.h }[2], [x21]\n" + "ld1 { v26.h }[2], [x26]\n" + "ld1 { v25.h }[2], [x25]\n" + "ld1 { v24.h }[2], [x24]\n" + "ld1 { v23.h }[2], [x23]\n" + "ld1 { v22.h }[2], [x22]\n" + "ld1 { v21.h }[2], [x21]\n" "b 5f\n" "4:" // odd_loads_1_0 "ldr h28, [x28, #0x0]\n" "ldr h27, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr h22, [x26, #0x0]\n" - "ldr h21, [x25, #0x0]\n" - "ldr h26, [x24, #0x0]\n" - "ldr h25, [x23, #0x0]\n" - "ldr h20, [x22, #0x0]\n" - "ldr h19, [x21, #0x0]\n" + "ldr h26, [x26, #0x0]\n" + "ldr h25, [x25, #0x0]\n" + "ldr h24, [x24, #0x0]\n" + "ldr h23, [x23, #0x0]\n" + "ldr h22, [x22, #0x0]\n" + "ldr h21, [x21, #0x0]\n" "5:" // Odd load end "shll v28.4s, v28.4h, #0x10\n" "shll v27.4s, v27.4h, #0x10\n" "subs x20, x20, #0x1\n" - "shll v22.4s, v22.4h, #0x10\n" - "shll v21.4s, v21.4h, #0x10\n" "shll v26.4s, v26.4h, #0x10\n" "shll v25.4s, v25.4h, #0x10\n" - "shll v20.4s, v20.4h, #0x10\n" - "shll v19.4s, v19.4h, #0x10\n" - "zip1 v24.4s, v28.4s, v22.4s\n" - "zip1 v23.4s, v27.4s, v21.4s\n" - "zip1 v18.4s, v26.4s, v20.4s\n" - "zip1 v17.4s, v25.4s, v19.4s\n" - "zip1 v16.4s, v24.4s, v23.4s\n" + "shll v24.4s, v24.4h, #0x10\n" + "shll v23.4s, v23.4h, #0x10\n" + "shll v22.4s, v22.4h, #0x10\n" + "shll v21.4s, v21.4h, #0x10\n" + "zip1 v20.4s, v28.4s, v26.4s\n" + "zip1 v19.4s, v27.4s, v25.4s\n" + "zip1 v18.4s, v24.4s, v22.4s\n" + "zip1 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v20.4s, v19.4s\n" "str q16, [%x[out_ptr], #0x0]\n" "zip1 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 6f\n" "subs x20, x20, #0x1\n" - "zip2 v16.4s, v24.4s, v23.4s\n" + "zip2 v16.4s, v20.4s, v19.4s\n" "str q16, [%x[out_ptr], #0x0]\n" - "zip2 v17.4s, v18.4s, v17.4s\n" - "str q17, [%x[out_ptr], #0x10]\n" + "zip2 v16.4s, v18.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 6f\n" - "zip2 v22.4s, v28.4s, v22.4s\n" - "zip2 v21.4s, v27.4s, v21.4s\n" - "zip2 v20.4s, v26.4s, v20.4s\n" - "zip2 v19.4s, v25.4s, v19.4s\n" - "zip1 v16.4s, v22.4s, v21.4s\n" + "zip2 v19.4s, v28.4s, v26.4s\n" + "zip2 v16.4s, v27.4s, v25.4s\n" + "zip2 v18.4s, v24.4s, v22.4s\n" + "zip2 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v19.4s, v16.4s\n" "str q16, [%x[out_ptr], #0x0]\n" - "zip1 v18.4s, v20.4s, v19.4s\n" - "str q18, [%x[out_ptr], #0x10]\n" + "zip1 v16.4s, v18.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "6:" // Odds skip - : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp16.hpp index f55c2be4a4fff419ec4b0e5c4228331ac3616d0b..e54b3b9f41f64acc1781f18403780b741d6a311a 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp16.hpp @@ -80,33 +80,33 @@ void interleave_block<8, 1, VLType::None, false>( "blt 3f\n" "2:" // Main loop head "ldr q25, [x28], #0x10\n" - "ldr q30, [x27], #0x10\n" + "ldr q27, [x27], #0x10\n" "subs %x[width], %x[width], #0x8\n" "cmp %x[width], #0x8\n" - "ldr q29, [x26], #0x10\n" - "ldr q28, [x25], #0x10\n" + "ldr q26, [x26], #0x10\n" + "ldr q24, [x25], #0x10\n" "ldr q21, [x24], #0x10\n" - "ldr q27, [x23], #0x10\n" + "ldr q20, [x23], #0x10\n" "zip1 v23.8h, v25.8h, v21.8h\n" - "zip1 v26.8h, v30.8h, v27.8h\n" - "ldr q20, [x22], #0x10\n" - "ldr q22, [x21], #0x10\n" - "zip1 v19.8h, v29.8h, v20.8h\n" - "zip1 v18.8h, v28.8h, v22.8h\n" + "zip1 v22.8h, v27.8h, v20.8h\n" + "ldr q17, [x22], #0x10\n" + "ldr q16, [x21], #0x10\n" + "zip1 v19.8h, v26.8h, v17.8h\n" + "zip1 v18.8h, v24.8h, v16.8h\n" "zip2 v25.8h, v25.8h, v21.8h\n" - "zip2 v21.8h, v29.8h, v20.8h\n" + "zip2 v21.8h, v26.8h, v17.8h\n" "prfm pldl1keep, [x28, #0x70]\n" "prfm pldl1keep, [x27, #0x70]\n" - "zip2 v20.8h, v30.8h, v27.8h\n" - "zip2 v16.8h, v28.8h, v22.8h\n" + "zip2 v20.8h, v27.8h, v20.8h\n" + "zip2 v16.8h, v24.8h, v16.8h\n" "prfm pldl1keep, [x26, #0x70]\n" "prfm pldl1keep, [x25, #0x70]\n" "zip1 v24.8h, v23.8h, v19.8h\n" - "zip1 v17.8h, v26.8h, v18.8h\n" + "zip1 v17.8h, v22.8h, v18.8h\n" "prfm pldl1keep, [x24, #0x70]\n" "prfm pldl1keep, [x23, #0x70]\n" "zip2 v23.8h, v23.8h, v19.8h\n" - "zip2 v19.8h, v26.8h, v18.8h\n" + "zip2 v19.8h, v22.8h, v18.8h\n" "prfm pldl1keep, [x22, #0x70]\n" "prfm pldl1keep, [x21, #0x70]\n" "zip1 v22.8h, v25.8h, v21.8h\n" @@ -134,132 +134,131 @@ void interleave_block<8, 1, VLType::None, false>( "3:" // Main loop skip "cbz %x[width], 8f\n" "tbz %x[width], #2, 5f\n" - "ldr d25, [x28], #0x8\n" - "ldr d30, [x27], #0x8\n" - "ldr d29, [x26], #0x8\n" - "ldr d28, [x25], #0x8\n" - "ldr d21, [x24], #0x8\n" - "ldr d27, [x23], #0x8\n" - "ldr d20, [x22], #0x8\n" - "ldr d22, [x21], #0x8\n" + "ldr d30, [x28], #0x8\n" + "ldr d29, [x27], #0x8\n" + "ldr d28, [x26], #0x8\n" + "ldr d27, [x25], #0x8\n" + "ldr d26, [x24], #0x8\n" + "ldr d25, [x23], #0x8\n" + "ldr d24, [x22], #0x8\n" + "ldr d23, [x21], #0x8\n" "tbz %x[width], #1, 4f\n" - "ld1 { v25.s }[2], [x28], #0x4\n" - "ld1 { v30.s }[2], [x27], #0x4\n" + "ld1 { v30.s }[2], [x28], #0x4\n" + "ld1 { v29.s }[2], [x27], #0x4\n" "mov x20, #0x6\n" - "ld1 { v29.s }[2], [x26], #0x4\n" - "ld1 { v28.s }[2], [x25], #0x4\n" - "ld1 { v21.s }[2], [x24], #0x4\n" - "ld1 { v27.s }[2], [x23], #0x4\n" - "ld1 { v20.s }[2], [x22], #0x4\n" - "ld1 { v22.s }[2], [x21], #0x4\n" + "ld1 { v28.s }[2], [x26], #0x4\n" + "ld1 { v27.s }[2], [x25], #0x4\n" + "ld1 { v26.s }[2], [x24], #0x4\n" + "ld1 { v25.s }[2], [x23], #0x4\n" + "ld1 { v24.s }[2], [x22], #0x4\n" + "ld1 { v23.s }[2], [x21], #0x4\n" "tbz %x[width], #0, 7f\n" - "ld1 { v25.h }[6], [x28]\n" - "ld1 { v30.h }[6], [x27]\n" + "ld1 { v30.h }[6], [x28]\n" + "ld1 { v29.h }[6], [x27]\n" "mov x20, #0x7\n" - "ld1 { v29.h }[6], [x26]\n" - "ld1 { v28.h }[6], [x25]\n" - "ld1 { v21.h }[6], [x24]\n" - "ld1 { v27.h }[6], [x23]\n" - "ld1 { v20.h }[6], [x22]\n" - "ld1 { v22.h }[6], [x21]\n" + "ld1 { v28.h }[6], [x26]\n" + "ld1 { v27.h }[6], [x25]\n" + "ld1 { v26.h }[6], [x24]\n" + "ld1 { v25.h }[6], [x23]\n" + "ld1 { v24.h }[6], [x22]\n" + "ld1 { v23.h }[6], [x21]\n" "b 7f\n" "4:" // odd_loads_1_4 "mov x20, #0x4\n" "tbz %x[width], #0, 7f\n" - "ld1 { v25.h }[4], [x28]\n" - "ld1 { v30.h }[4], [x27]\n" + "ld1 { v30.h }[4], [x28]\n" + "ld1 { v29.h }[4], [x27]\n" "mov x20, #0x5\n" - "ld1 { v29.h }[4], [x26]\n" - "ld1 { v28.h }[4], [x25]\n" - "ld1 { v21.h }[4], [x24]\n" - "ld1 { v27.h }[4], [x23]\n" - "ld1 { v20.h }[4], [x22]\n" - "ld1 { v22.h }[4], [x21]\n" + "ld1 { v28.h }[4], [x26]\n" + "ld1 { v27.h }[4], [x25]\n" + "ld1 { v26.h }[4], [x24]\n" + "ld1 { v25.h }[4], [x23]\n" + "ld1 { v24.h }[4], [x22]\n" + "ld1 { v23.h }[4], [x21]\n" "b 7f\n" "5:" // odd_loads_2_0 "tbz %x[width], #1, 6f\n" - "ldr s25, [x28], #0x4\n" - "ldr s30, [x27], #0x4\n" + "ldr s30, [x28], #0x4\n" + "ldr s29, [x27], #0x4\n" "mov x20, #0x2\n" - "ldr s29, [x26], #0x4\n" - "ldr s28, [x25], #0x4\n" - "ldr s21, [x24], #0x4\n" - "ldr s27, [x23], #0x4\n" - "ldr s20, [x22], #0x4\n" - "ldr s22, [x21], #0x4\n" + "ldr s28, [x26], #0x4\n" + "ldr s27, [x25], #0x4\n" + "ldr s26, [x24], #0x4\n" + "ldr s25, [x23], #0x4\n" + "ldr s24, [x22], #0x4\n" + "ldr s23, [x21], #0x4\n" "tbz %x[width], #0, 7f\n" - "ld1 { v25.h }[2], [x28]\n" - "ld1 { v30.h }[2], [x27]\n" + "ld1 { v30.h }[2], [x28]\n" + "ld1 { v29.h }[2], [x27]\n" "mov x20, #0x3\n" - "ld1 { v29.h }[2], [x26]\n" - "ld1 { v28.h }[2], [x25]\n" - "ld1 { v21.h }[2], [x24]\n" - "ld1 { v27.h }[2], [x23]\n" - "ld1 { v20.h }[2], [x22]\n" - "ld1 { v22.h }[2], [x21]\n" + "ld1 { v28.h }[2], [x26]\n" + "ld1 { v27.h }[2], [x25]\n" + "ld1 { v26.h }[2], [x24]\n" + "ld1 { v25.h }[2], [x23]\n" + "ld1 { v24.h }[2], [x22]\n" + "ld1 { v23.h }[2], [x21]\n" "b 7f\n" "6:" // odd_loads_1_0 - "ldr h25, [x28, #0x0]\n" - "ldr h30, [x27, #0x0]\n" + "ldr h30, [x28, #0x0]\n" + "ldr h29, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr h29, [x26, #0x0]\n" - "ldr h28, [x25, #0x0]\n" - "ldr h21, [x24, #0x0]\n" - "ldr h27, [x23, #0x0]\n" - "ldr h20, [x22, #0x0]\n" - "ldr h22, [x21, #0x0]\n" + "ldr h28, [x26, #0x0]\n" + "ldr h27, [x25, #0x0]\n" + "ldr h26, [x24, #0x0]\n" + "ldr h25, [x23, #0x0]\n" + "ldr h24, [x22, #0x0]\n" + "ldr h23, [x21, #0x0]\n" "7:" // Odd load end - "zip1 v23.8h, v25.8h, v21.8h\n" - "zip1 v19.8h, v29.8h, v20.8h\n" + "zip1 v22.8h, v30.8h, v26.8h\n" + "zip1 v21.8h, v28.8h, v24.8h\n" "subs x20, x20, #0x1\n" - "zip1 v26.8h, v30.8h, v27.8h\n" - "zip1 v18.8h, v28.8h, v22.8h\n" - "zip1 v24.8h, v23.8h, v19.8h\n" - "zip1 v17.8h, v26.8h, v18.8h\n" - "zip1 v16.8h, v24.8h, v17.8h\n" + "zip1 v20.8h, v29.8h, v25.8h\n" + "zip1 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" "subs x20, x20, #0x1\n" - "zip2 v16.8h, v24.8h, v17.8h\n" + "zip2 v16.8h, v18.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" - "zip2 v23.8h, v23.8h, v19.8h\n" - "zip2 v19.8h, v26.8h, v18.8h\n" + "zip2 v18.8h, v22.8h, v21.8h\n" + "zip2 v17.8h, v20.8h, v19.8h\n" "subs x20, x20, #0x1\n" - "zip1 v17.8h, v23.8h, v19.8h\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" "subs x20, x20, #0x1\n" - "zip2 v16.8h, v23.8h, v19.8h\n" + "zip2 v16.8h, v18.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" - "zip2 v25.8h, v25.8h, v21.8h\n" - "zip2 v21.8h, v29.8h, v20.8h\n" + "zip2 v22.8h, v30.8h, v26.8h\n" + "zip2 v21.8h, v28.8h, v24.8h\n" "subs x20, x20, #0x1\n" - "zip2 v20.8h, v30.8h, v27.8h\n" - "zip2 v16.8h, v28.8h, v22.8h\n" - "zip1 v22.8h, v25.8h, v21.8h\n" - "zip1 v18.8h, v20.8h, v16.8h\n" - "zip1 v19.8h, v22.8h, v18.8h\n" - "str q19, [%x[out_ptr], #0x0]\n" + "zip2 v20.8h, v29.8h, v25.8h\n" + "zip2 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" "subs x20, x20, #0x1\n" - "zip2 v18.8h, v22.8h, v18.8h\n" - "str q18, [%x[out_ptr], #0x0]\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" - "zip2 v21.8h, v25.8h, v21.8h\n" - "zip2 v20.8h, v20.8h, v16.8h\n" - "zip1 v17.8h, v21.8h, v20.8h\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip2 v17.8h, v22.8h, v21.8h\n" + "zip2 v16.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v17.8h, v16.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "8:" // Odds skip - : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp32.hpp index f64db0b4768ed6a241e0a9c6800621d2447a7439..3a5dcf4a6b26412dd4117d6ce9f2eaf2f4f3269b 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp32.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp16_fp32.hpp @@ -79,36 +79,36 @@ void interleave_block<8, 1, VLType::None, false>( "prfm pldl1keep, [x21, #0x40]\n" "blt 3f\n" "2:" // Main loop head - "ldr d28, [x28], #0x8\n" - "ldr d27, [x27], #0x8\n" - "fcvtl v28.4s, v28.4h\n" + "ldr d27, [x28], #0x8\n" + "ldr d26, [x27], #0x8\n" "fcvtl v27.4s, v27.4h\n" + "fcvtl v26.4s, v26.4h\n" "ldr d22, [x26], #0x8\n" "ldr d21, [x25], #0x8\n" "fcvtl v22.4s, v22.4h\n" "fcvtl v21.4s, v21.4h\n" - "ldr d26, [x24], #0x8\n" + "ldr d20, [x24], #0x8\n" "ldr d25, [x23], #0x8\n" - "fcvtl v26.4s, v26.4h\n" - "fcvtl v25.4s, v25.4h\n" - "ldr d20, [x22], #0x8\n" - "ldr d19, [x21], #0x8\n" "fcvtl v20.4s, v20.4h\n" + "fcvtl v25.4s, v25.4h\n" + "ldr d19, [x22], #0x8\n" + "ldr d16, [x21], #0x8\n" "fcvtl v19.4s, v19.4h\n" - "zip1 v24.4s, v28.4s, v22.4s\n" - "zip1 v23.4s, v27.4s, v21.4s\n" + "fcvtl v16.4s, v16.4h\n" + "zip1 v24.4s, v27.4s, v22.4s\n" + "zip1 v23.4s, v26.4s, v21.4s\n" "subs %x[width], %x[width], #0x4\n" "cmp %x[width], #0x4\n" - "zip1 v18.4s, v26.4s, v20.4s\n" - "zip1 v17.4s, v25.4s, v19.4s\n" + "zip1 v18.4s, v20.4s, v19.4s\n" + "zip1 v17.4s, v25.4s, v16.4s\n" "prfm pldl1keep, [x28, #0x70]\n" "prfm pldl1keep, [x27, #0x70]\n" - "zip2 v22.4s, v28.4s, v22.4s\n" - "zip2 v21.4s, v27.4s, v21.4s\n" + "zip2 v22.4s, v27.4s, v22.4s\n" + "zip2 v21.4s, v26.4s, v21.4s\n" "prfm pldl1keep, [x26, #0x70]\n" "prfm pldl1keep, [x25, #0x70]\n" - "zip2 v20.4s, v26.4s, v20.4s\n" - "zip2 v19.4s, v25.4s, v19.4s\n" + "zip2 v20.4s, v20.4s, v19.4s\n" + "zip2 v19.4s, v25.4s, v16.4s\n" "prfm pldl1keep, [x24, #0x70]\n" "prfm pldl1keep, [x23, #0x70]\n" "prfm pldl1keep, [x22, #0x70]\n" @@ -137,71 +137,70 @@ void interleave_block<8, 1, VLType::None, false>( "ldr s28, [x28], #0x4\n" "ldr s27, [x27], #0x4\n" "mov x20, #0x2\n" - "ldr s22, [x26], #0x4\n" - "ldr s21, [x25], #0x4\n" - "ldr s26, [x24], #0x4\n" - "ldr s25, [x23], #0x4\n" - "ldr s20, [x22], #0x4\n" - "ldr s19, [x21], #0x4\n" + "ldr s26, [x26], #0x4\n" + "ldr s25, [x25], #0x4\n" + "ldr s24, [x24], #0x4\n" + "ldr s23, [x23], #0x4\n" + "ldr s22, [x22], #0x4\n" + "ldr s21, [x21], #0x4\n" "tbz %x[width], #0, 5f\n" "ld1 { v28.h }[2], [x28]\n" "ld1 { v27.h }[2], [x27]\n" "mov x20, #0x3\n" - "ld1 { v22.h }[2], [x26]\n" - "ld1 { v21.h }[2], [x25]\n" - "ld1 { v26.h }[2], [x24]\n" - "ld1 { v25.h }[2], [x23]\n" - "ld1 { v20.h }[2], [x22]\n" - "ld1 { v19.h }[2], [x21]\n" + "ld1 { v26.h }[2], [x26]\n" + "ld1 { v25.h }[2], [x25]\n" + "ld1 { v24.h }[2], [x24]\n" + "ld1 { v23.h }[2], [x23]\n" + "ld1 { v22.h }[2], [x22]\n" + "ld1 { v21.h }[2], [x21]\n" "b 5f\n" "4:" // odd_loads_1_0 "ldr h28, [x28, #0x0]\n" "ldr h27, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr h22, [x26, #0x0]\n" - "ldr h21, [x25, #0x0]\n" - "ldr h26, [x24, #0x0]\n" - "ldr h25, [x23, #0x0]\n" - "ldr h20, [x22, #0x0]\n" - "ldr h19, [x21, #0x0]\n" + "ldr h26, [x26, #0x0]\n" + "ldr h25, [x25, #0x0]\n" + "ldr h24, [x24, #0x0]\n" + "ldr h23, [x23, #0x0]\n" + "ldr h22, [x22, #0x0]\n" + "ldr h21, [x21, #0x0]\n" "5:" // Odd load end "fcvtl v28.4s, v28.4h\n" "fcvtl v27.4s, v27.4h\n" "subs x20, x20, #0x1\n" - "fcvtl v22.4s, v22.4h\n" - "fcvtl v21.4s, v21.4h\n" "fcvtl v26.4s, v26.4h\n" "fcvtl v25.4s, v25.4h\n" - "fcvtl v20.4s, v20.4h\n" - "fcvtl v19.4s, v19.4h\n" - "zip1 v24.4s, v28.4s, v22.4s\n" - "zip1 v23.4s, v27.4s, v21.4s\n" - "zip1 v18.4s, v26.4s, v20.4s\n" - "zip1 v17.4s, v25.4s, v19.4s\n" - "zip1 v16.4s, v24.4s, v23.4s\n" + "fcvtl v24.4s, v24.4h\n" + "fcvtl v23.4s, v23.4h\n" + "fcvtl v22.4s, v22.4h\n" + "fcvtl v21.4s, v21.4h\n" + "zip1 v20.4s, v28.4s, v26.4s\n" + "zip1 v19.4s, v27.4s, v25.4s\n" + "zip1 v18.4s, v24.4s, v22.4s\n" + "zip1 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v20.4s, v19.4s\n" "str q16, [%x[out_ptr], #0x0]\n" "zip1 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 6f\n" "subs x20, x20, #0x1\n" - "zip2 v16.4s, v24.4s, v23.4s\n" + "zip2 v16.4s, v20.4s, v19.4s\n" "str q16, [%x[out_ptr], #0x0]\n" - "zip2 v17.4s, v18.4s, v17.4s\n" - "str q17, [%x[out_ptr], #0x10]\n" + "zip2 v16.4s, v18.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 6f\n" - "zip2 v22.4s, v28.4s, v22.4s\n" - "zip2 v21.4s, v27.4s, v21.4s\n" - "zip2 v20.4s, v26.4s, v20.4s\n" - "zip2 v19.4s, v25.4s, v19.4s\n" - "zip1 v16.4s, v22.4s, v21.4s\n" + "zip2 v19.4s, v28.4s, v26.4s\n" + "zip2 v16.4s, v27.4s, v25.4s\n" + "zip2 v18.4s, v24.4s, v22.4s\n" + "zip2 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v19.4s, v16.4s\n" "str q16, [%x[out_ptr], #0x0]\n" - "zip1 v18.4s, v20.4s, v19.4s\n" - "str q18, [%x[out_ptr], #0x10]\n" + "zip1 v16.4s, v18.4s, v17.4s\n" + "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "6:" // Odds skip - : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp32_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp32_fp32.hpp index 6c009b34b8ef76608da68eb5dc8949f73c7b903d..80c387db475f1f0003d87e1024f26f8299bba0df 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp32_fp32.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_fp32_fp32.hpp @@ -79,29 +79,29 @@ void interleave_block<8, 1, VLType::None, false>( "prfm pldl1keep, [x21, #0x40]\n" "blt 3f\n" "2:" // Main loop head - "ldr q28, [x28], #0x10\n" - "ldr q27, [x27], #0x10\n" + "ldr q20, [x28], #0x10\n" + "ldr q18, [x27], #0x10\n" "subs %x[width], %x[width], #0x4\n" "cmp %x[width], #0x4\n" - "ldr q22, [x26], #0x10\n" - "ldr q21, [x25], #0x10\n" - "zip1 v26.4s, v28.4s, v22.4s\n" - "zip1 v25.4s, v27.4s, v21.4s\n" - "ldr q24, [x24], #0x10\n" + "ldr q17, [x26], #0x10\n" + "ldr q16, [x25], #0x10\n" + "zip1 v25.4s, v20.4s, v17.4s\n" + "zip1 v24.4s, v18.4s, v16.4s\n" + "ldr q19, [x24], #0x10\n" "ldr q23, [x23], #0x10\n" - "zip2 v22.4s, v28.4s, v22.4s\n" - "zip2 v21.4s, v27.4s, v21.4s\n" - "ldr q19, [x22], #0x10\n" - "ldr q18, [x21], #0x10\n" - "zip1 v20.4s, v24.4s, v19.4s\n" - "zip1 v17.4s, v23.4s, v18.4s\n" - "zip2 v19.4s, v24.4s, v19.4s\n" - "zip2 v18.4s, v23.4s, v18.4s\n" + "zip2 v22.4s, v20.4s, v17.4s\n" + "zip2 v21.4s, v18.4s, v16.4s\n" + "ldr q18, [x22], #0x10\n" + "ldr q16, [x21], #0x10\n" + "zip1 v20.4s, v19.4s, v18.4s\n" + "zip1 v17.4s, v23.4s, v16.4s\n" + "zip2 v19.4s, v19.4s, v18.4s\n" + "zip2 v18.4s, v23.4s, v16.4s\n" "prfm pldl1keep, [x28, #0x70]\n" "prfm pldl1keep, [x27, #0x70]\n" "prfm pldl1keep, [x26, #0x70]\n" "prfm pldl1keep, [x25, #0x70]\n" - "zip1 v16.4s, v26.4s, v25.4s\n" + "zip1 v16.4s, v25.4s, v24.4s\n" "str q16, [%x[out_ptr], #0x0]\n" "prfm pldl1keep, [x24, #0x70]\n" "prfm pldl1keep, [x23, #0x70]\n" @@ -109,7 +109,7 @@ void interleave_block<8, 1, VLType::None, false>( "str q16, [%x[out_ptr], #0x10]\n" "prfm pldl1keep, [x22, #0x70]\n" "prfm pldl1keep, [x21, #0x70]\n" - "zip2 v16.4s, v26.4s, v25.4s\n" + "zip2 v16.4s, v25.4s, v24.4s\n" "str q16, [%x[out_ptr], #0x20]\n" "zip2 v16.4s, v20.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x30]\n" @@ -129,63 +129,62 @@ void interleave_block<8, 1, VLType::None, false>( "ldr d28, [x28], #0x8\n" "ldr d27, [x27], #0x8\n" "mov x20, #0x2\n" - "ldr d22, [x26], #0x8\n" - "ldr d21, [x25], #0x8\n" + "ldr d26, [x26], #0x8\n" + "ldr d25, [x25], #0x8\n" "ldr d24, [x24], #0x8\n" "ldr d23, [x23], #0x8\n" - "ldr d19, [x22], #0x8\n" - "ldr d18, [x21], #0x8\n" + "ldr d22, [x22], #0x8\n" + "ldr d21, [x21], #0x8\n" "tbz %x[width], #0, 5f\n" "ld1 { v28.s }[2], [x28]\n" "ld1 { v27.s }[2], [x27]\n" "mov x20, #0x3\n" - "ld1 { v22.s }[2], [x26]\n" - "ld1 { v21.s }[2], [x25]\n" + "ld1 { v26.s }[2], [x26]\n" + "ld1 { v25.s }[2], [x25]\n" "ld1 { v24.s }[2], [x24]\n" "ld1 { v23.s }[2], [x23]\n" - "ld1 { v19.s }[2], [x22]\n" - "ld1 { v18.s }[2], [x21]\n" + "ld1 { v22.s }[2], [x22]\n" + "ld1 { v21.s }[2], [x21]\n" "b 5f\n" "4:" // odd_loads_1_0 "ldr s28, [x28, #0x0]\n" "ldr s27, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr s22, [x26, #0x0]\n" - "ldr s21, [x25, #0x0]\n" + "ldr s26, [x26, #0x0]\n" + "ldr s25, [x25, #0x0]\n" "ldr s24, [x24, #0x0]\n" "ldr s23, [x23, #0x0]\n" - "ldr s19, [x22, #0x0]\n" - "ldr s18, [x21, #0x0]\n" + "ldr s22, [x22, #0x0]\n" + "ldr s21, [x21, #0x0]\n" "5:" // Odd load end - "zip1 v26.4s, v28.4s, v22.4s\n" - "zip1 v25.4s, v27.4s, v21.4s\n" + "zip1 v20.4s, v28.4s, v26.4s\n" + "zip1 v19.4s, v27.4s, v25.4s\n" "subs x20, x20, #0x1\n" - "zip1 v20.4s, v24.4s, v19.4s\n" - "zip1 v17.4s, v23.4s, v18.4s\n" - "zip1 v16.4s, v26.4s, v25.4s\n" + "zip1 v18.4s, v24.4s, v22.4s\n" + "zip1 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v20.4s, v19.4s\n" "str q16, [%x[out_ptr], #0x0]\n" - "zip1 v16.4s, v20.4s, v17.4s\n" + "zip1 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 6f\n" "subs x20, x20, #0x1\n" - "zip2 v16.4s, v26.4s, v25.4s\n" + "zip2 v16.4s, v20.4s, v19.4s\n" "str q16, [%x[out_ptr], #0x0]\n" - "zip2 v16.4s, v20.4s, v17.4s\n" + "zip2 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 6f\n" - "zip2 v22.4s, v28.4s, v22.4s\n" - "zip2 v21.4s, v27.4s, v21.4s\n" - "zip2 v19.4s, v24.4s, v19.4s\n" - "zip2 v18.4s, v23.4s, v18.4s\n" - "zip1 v16.4s, v22.4s, v21.4s\n" + "zip2 v19.4s, v28.4s, v26.4s\n" + "zip2 v16.4s, v27.4s, v25.4s\n" + "zip2 v18.4s, v24.4s, v22.4s\n" + "zip2 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v19.4s, v16.4s\n" "str q16, [%x[out_ptr], #0x0]\n" - "zip1 v16.4s, v19.4s, v18.4s\n" + "zip1 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "6:" // Odds skip - : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s16_s16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s16_s16.hpp index 767d468ad1767fd7f919272ed855ed130c6d85f7..8e06b7ecab5c813f7eb6e3350ff41992a47d7b78 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s16_s16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s16_s16.hpp @@ -80,33 +80,33 @@ void interleave_block<8, 1, VLType::None, false>( "blt 3f\n" "2:" // Main loop head "ldr q25, [x28], #0x10\n" - "ldr q30, [x27], #0x10\n" + "ldr q27, [x27], #0x10\n" "subs %x[width], %x[width], #0x8\n" "cmp %x[width], #0x8\n" - "ldr q29, [x26], #0x10\n" - "ldr q28, [x25], #0x10\n" + "ldr q26, [x26], #0x10\n" + "ldr q24, [x25], #0x10\n" "ldr q21, [x24], #0x10\n" - "ldr q27, [x23], #0x10\n" + "ldr q20, [x23], #0x10\n" "zip1 v23.8h, v25.8h, v21.8h\n" - "zip1 v26.8h, v30.8h, v27.8h\n" - "ldr q20, [x22], #0x10\n" - "ldr q22, [x21], #0x10\n" - "zip1 v19.8h, v29.8h, v20.8h\n" - "zip1 v18.8h, v28.8h, v22.8h\n" + "zip1 v22.8h, v27.8h, v20.8h\n" + "ldr q17, [x22], #0x10\n" + "ldr q16, [x21], #0x10\n" + "zip1 v19.8h, v26.8h, v17.8h\n" + "zip1 v18.8h, v24.8h, v16.8h\n" "zip2 v25.8h, v25.8h, v21.8h\n" - "zip2 v21.8h, v29.8h, v20.8h\n" + "zip2 v21.8h, v26.8h, v17.8h\n" "prfm pldl1keep, [x28, #0x70]\n" "prfm pldl1keep, [x27, #0x70]\n" - "zip2 v20.8h, v30.8h, v27.8h\n" - "zip2 v16.8h, v28.8h, v22.8h\n" + "zip2 v20.8h, v27.8h, v20.8h\n" + "zip2 v16.8h, v24.8h, v16.8h\n" "prfm pldl1keep, [x26, #0x70]\n" "prfm pldl1keep, [x25, #0x70]\n" "zip1 v24.8h, v23.8h, v19.8h\n" - "zip1 v17.8h, v26.8h, v18.8h\n" + "zip1 v17.8h, v22.8h, v18.8h\n" "prfm pldl1keep, [x24, #0x70]\n" "prfm pldl1keep, [x23, #0x70]\n" "zip2 v23.8h, v23.8h, v19.8h\n" - "zip2 v19.8h, v26.8h, v18.8h\n" + "zip2 v19.8h, v22.8h, v18.8h\n" "prfm pldl1keep, [x22, #0x70]\n" "prfm pldl1keep, [x21, #0x70]\n" "zip1 v22.8h, v25.8h, v21.8h\n" @@ -134,132 +134,131 @@ void interleave_block<8, 1, VLType::None, false>( "3:" // Main loop skip "cbz %x[width], 8f\n" "tbz %x[width], #2, 5f\n" - "ldr d25, [x28], #0x8\n" - "ldr d30, [x27], #0x8\n" - "ldr d29, [x26], #0x8\n" - "ldr d28, [x25], #0x8\n" - "ldr d21, [x24], #0x8\n" - "ldr d27, [x23], #0x8\n" - "ldr d20, [x22], #0x8\n" - "ldr d22, [x21], #0x8\n" + "ldr d30, [x28], #0x8\n" + "ldr d29, [x27], #0x8\n" + "ldr d28, [x26], #0x8\n" + "ldr d27, [x25], #0x8\n" + "ldr d26, [x24], #0x8\n" + "ldr d25, [x23], #0x8\n" + "ldr d24, [x22], #0x8\n" + "ldr d23, [x21], #0x8\n" "tbz %x[width], #1, 4f\n" - "ld1 { v25.s }[2], [x28], #0x4\n" - "ld1 { v30.s }[2], [x27], #0x4\n" + "ld1 { v30.s }[2], [x28], #0x4\n" + "ld1 { v29.s }[2], [x27], #0x4\n" "mov x20, #0x6\n" - "ld1 { v29.s }[2], [x26], #0x4\n" - "ld1 { v28.s }[2], [x25], #0x4\n" - "ld1 { v21.s }[2], [x24], #0x4\n" - "ld1 { v27.s }[2], [x23], #0x4\n" - "ld1 { v20.s }[2], [x22], #0x4\n" - "ld1 { v22.s }[2], [x21], #0x4\n" + "ld1 { v28.s }[2], [x26], #0x4\n" + "ld1 { v27.s }[2], [x25], #0x4\n" + "ld1 { v26.s }[2], [x24], #0x4\n" + "ld1 { v25.s }[2], [x23], #0x4\n" + "ld1 { v24.s }[2], [x22], #0x4\n" + "ld1 { v23.s }[2], [x21], #0x4\n" "tbz %x[width], #0, 7f\n" - "ld1 { v25.h }[6], [x28]\n" - "ld1 { v30.h }[6], [x27]\n" + "ld1 { v30.h }[6], [x28]\n" + "ld1 { v29.h }[6], [x27]\n" "mov x20, #0x7\n" - "ld1 { v29.h }[6], [x26]\n" - "ld1 { v28.h }[6], [x25]\n" - "ld1 { v21.h }[6], [x24]\n" - "ld1 { v27.h }[6], [x23]\n" - "ld1 { v20.h }[6], [x22]\n" - "ld1 { v22.h }[6], [x21]\n" + "ld1 { v28.h }[6], [x26]\n" + "ld1 { v27.h }[6], [x25]\n" + "ld1 { v26.h }[6], [x24]\n" + "ld1 { v25.h }[6], [x23]\n" + "ld1 { v24.h }[6], [x22]\n" + "ld1 { v23.h }[6], [x21]\n" "b 7f\n" "4:" // odd_loads_1_4 "mov x20, #0x4\n" "tbz %x[width], #0, 7f\n" - "ld1 { v25.h }[4], [x28]\n" - "ld1 { v30.h }[4], [x27]\n" + "ld1 { v30.h }[4], [x28]\n" + "ld1 { v29.h }[4], [x27]\n" "mov x20, #0x5\n" - "ld1 { v29.h }[4], [x26]\n" - "ld1 { v28.h }[4], [x25]\n" - "ld1 { v21.h }[4], [x24]\n" - "ld1 { v27.h }[4], [x23]\n" - "ld1 { v20.h }[4], [x22]\n" - "ld1 { v22.h }[4], [x21]\n" + "ld1 { v28.h }[4], [x26]\n" + "ld1 { v27.h }[4], [x25]\n" + "ld1 { v26.h }[4], [x24]\n" + "ld1 { v25.h }[4], [x23]\n" + "ld1 { v24.h }[4], [x22]\n" + "ld1 { v23.h }[4], [x21]\n" "b 7f\n" "5:" // odd_loads_2_0 "tbz %x[width], #1, 6f\n" - "ldr s25, [x28], #0x4\n" - "ldr s30, [x27], #0x4\n" + "ldr s30, [x28], #0x4\n" + "ldr s29, [x27], #0x4\n" "mov x20, #0x2\n" - "ldr s29, [x26], #0x4\n" - "ldr s28, [x25], #0x4\n" - "ldr s21, [x24], #0x4\n" - "ldr s27, [x23], #0x4\n" - "ldr s20, [x22], #0x4\n" - "ldr s22, [x21], #0x4\n" + "ldr s28, [x26], #0x4\n" + "ldr s27, [x25], #0x4\n" + "ldr s26, [x24], #0x4\n" + "ldr s25, [x23], #0x4\n" + "ldr s24, [x22], #0x4\n" + "ldr s23, [x21], #0x4\n" "tbz %x[width], #0, 7f\n" - "ld1 { v25.h }[2], [x28]\n" - "ld1 { v30.h }[2], [x27]\n" + "ld1 { v30.h }[2], [x28]\n" + "ld1 { v29.h }[2], [x27]\n" "mov x20, #0x3\n" - "ld1 { v29.h }[2], [x26]\n" - "ld1 { v28.h }[2], [x25]\n" - "ld1 { v21.h }[2], [x24]\n" - "ld1 { v27.h }[2], [x23]\n" - "ld1 { v20.h }[2], [x22]\n" - "ld1 { v22.h }[2], [x21]\n" + "ld1 { v28.h }[2], [x26]\n" + "ld1 { v27.h }[2], [x25]\n" + "ld1 { v26.h }[2], [x24]\n" + "ld1 { v25.h }[2], [x23]\n" + "ld1 { v24.h }[2], [x22]\n" + "ld1 { v23.h }[2], [x21]\n" "b 7f\n" "6:" // odd_loads_1_0 - "ldr h25, [x28, #0x0]\n" - "ldr h30, [x27, #0x0]\n" + "ldr h30, [x28, #0x0]\n" + "ldr h29, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr h29, [x26, #0x0]\n" - "ldr h28, [x25, #0x0]\n" - "ldr h21, [x24, #0x0]\n" - "ldr h27, [x23, #0x0]\n" - "ldr h20, [x22, #0x0]\n" - "ldr h22, [x21, #0x0]\n" + "ldr h28, [x26, #0x0]\n" + "ldr h27, [x25, #0x0]\n" + "ldr h26, [x24, #0x0]\n" + "ldr h25, [x23, #0x0]\n" + "ldr h24, [x22, #0x0]\n" + "ldr h23, [x21, #0x0]\n" "7:" // Odd load end - "zip1 v23.8h, v25.8h, v21.8h\n" - "zip1 v19.8h, v29.8h, v20.8h\n" + "zip1 v22.8h, v30.8h, v26.8h\n" + "zip1 v21.8h, v28.8h, v24.8h\n" "subs x20, x20, #0x1\n" - "zip1 v26.8h, v30.8h, v27.8h\n" - "zip1 v18.8h, v28.8h, v22.8h\n" - "zip1 v24.8h, v23.8h, v19.8h\n" - "zip1 v17.8h, v26.8h, v18.8h\n" - "zip1 v16.8h, v24.8h, v17.8h\n" + "zip1 v20.8h, v29.8h, v25.8h\n" + "zip1 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" "subs x20, x20, #0x1\n" - "zip2 v16.8h, v24.8h, v17.8h\n" + "zip2 v16.8h, v18.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" - "zip2 v23.8h, v23.8h, v19.8h\n" - "zip2 v19.8h, v26.8h, v18.8h\n" + "zip2 v18.8h, v22.8h, v21.8h\n" + "zip2 v17.8h, v20.8h, v19.8h\n" "subs x20, x20, #0x1\n" - "zip1 v17.8h, v23.8h, v19.8h\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" "subs x20, x20, #0x1\n" - "zip2 v16.8h, v23.8h, v19.8h\n" + "zip2 v16.8h, v18.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" - "zip2 v25.8h, v25.8h, v21.8h\n" - "zip2 v21.8h, v29.8h, v20.8h\n" + "zip2 v22.8h, v30.8h, v26.8h\n" + "zip2 v21.8h, v28.8h, v24.8h\n" "subs x20, x20, #0x1\n" - "zip2 v20.8h, v30.8h, v27.8h\n" - "zip2 v16.8h, v28.8h, v22.8h\n" - "zip1 v22.8h, v25.8h, v21.8h\n" - "zip1 v18.8h, v20.8h, v16.8h\n" - "zip1 v19.8h, v22.8h, v18.8h\n" - "str q19, [%x[out_ptr], #0x0]\n" + "zip2 v20.8h, v29.8h, v25.8h\n" + "zip2 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" "subs x20, x20, #0x1\n" - "zip2 v18.8h, v22.8h, v18.8h\n" - "str q18, [%x[out_ptr], #0x0]\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" - "zip2 v21.8h, v25.8h, v21.8h\n" - "zip2 v20.8h, v20.8h, v16.8h\n" - "zip1 v17.8h, v21.8h, v20.8h\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip2 v17.8h, v22.8h, v21.8h\n" + "zip2 v16.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v17.8h, v16.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "8:" // Odds skip - : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s16_s16_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s16_s16_summing.hpp index a73792036ac7d620b526265ad091e421dcd3c754..b91ae8a948cb4b2591139916509b69b002f6e025 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s16_s16_summing.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s16_s16_summing.hpp @@ -159,101 +159,101 @@ void interleave_block<8, 1, VLType::None, true>( "5:" // Main loop skip "cbz %x[width], 10f\n" "tbz %x[width], #2, 7f\n" - "ldr d31, [x28], #0x8\n" - "ldr d30, [x27], #0x8\n" - "ldr d29, [x26], #0x8\n" - "ldr d28, [x25], #0x8\n" - "ldr d27, [x24], #0x8\n" - "ldr d26, [x23], #0x8\n" + "ldr d30, [x28], #0x8\n" + "ldr d29, [x27], #0x8\n" + "ldr d28, [x26], #0x8\n" + "ldr d27, [x25], #0x8\n" + "ldr d26, [x24], #0x8\n" + "ldr d25, [x23], #0x8\n" "ldr d24, [x22], #0x8\n" "ldr d23, [x21], #0x8\n" "tbz %x[width], #1, 6f\n" - "ld1 { v31.s }[2], [x28], #0x4\n" - "ld1 { v30.s }[2], [x27], #0x4\n" + "ld1 { v30.s }[2], [x28], #0x4\n" + "ld1 { v29.s }[2], [x27], #0x4\n" "mov x20, #0x6\n" - "ld1 { v29.s }[2], [x26], #0x4\n" - "ld1 { v28.s }[2], [x25], #0x4\n" - "ld1 { v27.s }[2], [x24], #0x4\n" - "ld1 { v26.s }[2], [x23], #0x4\n" + "ld1 { v28.s }[2], [x26], #0x4\n" + "ld1 { v27.s }[2], [x25], #0x4\n" + "ld1 { v26.s }[2], [x24], #0x4\n" + "ld1 { v25.s }[2], [x23], #0x4\n" "ld1 { v24.s }[2], [x22], #0x4\n" "ld1 { v23.s }[2], [x21], #0x4\n" "tbz %x[width], #0, 9f\n" - "ld1 { v31.h }[6], [x28]\n" - "ld1 { v30.h }[6], [x27]\n" + "ld1 { v30.h }[6], [x28]\n" + "ld1 { v29.h }[6], [x27]\n" "mov x20, #0x7\n" - "ld1 { v29.h }[6], [x26]\n" - "ld1 { v28.h }[6], [x25]\n" - "ld1 { v27.h }[6], [x24]\n" - "ld1 { v26.h }[6], [x23]\n" + "ld1 { v28.h }[6], [x26]\n" + "ld1 { v27.h }[6], [x25]\n" + "ld1 { v26.h }[6], [x24]\n" + "ld1 { v25.h }[6], [x23]\n" "ld1 { v24.h }[6], [x22]\n" "ld1 { v23.h }[6], [x21]\n" "b 9f\n" "6:" // odd_loads_1_4 "mov x20, #0x4\n" "tbz %x[width], #0, 9f\n" - "ld1 { v31.h }[4], [x28]\n" - "ld1 { v30.h }[4], [x27]\n" + "ld1 { v30.h }[4], [x28]\n" + "ld1 { v29.h }[4], [x27]\n" "mov x20, #0x5\n" - "ld1 { v29.h }[4], [x26]\n" - "ld1 { v28.h }[4], [x25]\n" - "ld1 { v27.h }[4], [x24]\n" - "ld1 { v26.h }[4], [x23]\n" + "ld1 { v28.h }[4], [x26]\n" + "ld1 { v27.h }[4], [x25]\n" + "ld1 { v26.h }[4], [x24]\n" + "ld1 { v25.h }[4], [x23]\n" "ld1 { v24.h }[4], [x22]\n" "ld1 { v23.h }[4], [x21]\n" "b 9f\n" "7:" // odd_loads_2_0 "tbz %x[width], #1, 8f\n" - "ldr s31, [x28], #0x4\n" - "ldr s30, [x27], #0x4\n" + "ldr s30, [x28], #0x4\n" + "ldr s29, [x27], #0x4\n" "mov x20, #0x2\n" - "ldr s29, [x26], #0x4\n" - "ldr s28, [x25], #0x4\n" - "ldr s27, [x24], #0x4\n" - "ldr s26, [x23], #0x4\n" + "ldr s28, [x26], #0x4\n" + "ldr s27, [x25], #0x4\n" + "ldr s26, [x24], #0x4\n" + "ldr s25, [x23], #0x4\n" "ldr s24, [x22], #0x4\n" "ldr s23, [x21], #0x4\n" "tbz %x[width], #0, 9f\n" - "ld1 { v31.h }[2], [x28]\n" - "ld1 { v30.h }[2], [x27]\n" + "ld1 { v30.h }[2], [x28]\n" + "ld1 { v29.h }[2], [x27]\n" "mov x20, #0x3\n" - "ld1 { v29.h }[2], [x26]\n" - "ld1 { v28.h }[2], [x25]\n" - "ld1 { v27.h }[2], [x24]\n" - "ld1 { v26.h }[2], [x23]\n" + "ld1 { v28.h }[2], [x26]\n" + "ld1 { v27.h }[2], [x25]\n" + "ld1 { v26.h }[2], [x24]\n" + "ld1 { v25.h }[2], [x23]\n" "ld1 { v24.h }[2], [x22]\n" "ld1 { v23.h }[2], [x21]\n" "b 9f\n" "8:" // odd_loads_1_0 - "ldr h31, [x28, #0x0]\n" - "ldr h30, [x27, #0x0]\n" + "ldr h30, [x28, #0x0]\n" + "ldr h29, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr h29, [x26, #0x0]\n" - "ldr h28, [x25, #0x0]\n" - "ldr h27, [x24, #0x0]\n" - "ldr h26, [x23, #0x0]\n" + "ldr h28, [x26, #0x0]\n" + "ldr h27, [x25, #0x0]\n" + "ldr h26, [x24, #0x0]\n" + "ldr h25, [x23, #0x0]\n" "ldr h24, [x22, #0x0]\n" "ldr h23, [x21, #0x0]\n" "9:" // Odd load end - "zip1 v25.8h, v31.8h, v27.8h\n" - "zip1 v18.8h, v29.8h, v24.8h\n" - "subs x20, x20, #0x1\n" "zip1 v22.8h, v30.8h, v26.8h\n" - "zip1 v21.8h, v28.8h, v23.8h\n" - "zip1 v17.8h, v25.8h, v18.8h\n" - "zip1 v16.8h, v22.8h, v21.8h\n" - "zip1 v20.8h, v17.8h, v16.8h\n" - "str q20, [%x[out_ptr], #0x0]\n" - "add v2.8h, v2.8h, v20.8h\n" + "zip1 v21.8h, v28.8h, v24.8h\n" + "subs x20, x20, #0x1\n" + "zip1 v20.8h, v29.8h, v25.8h\n" + "zip1 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 10f\n" - "zip2 v19.8h, v17.8h, v16.8h\n" + "zip2 v16.8h, v18.8h, v17.8h\n" "subs x20, x20, #0x1\n" - "str q19, [%x[out_ptr], #0x0]\n" - "add v2.8h, v2.8h, v19.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 10f\n" - "zip2 v18.8h, v25.8h, v18.8h\n" - "zip2 v17.8h, v22.8h, v21.8h\n" + "zip2 v18.8h, v22.8h, v21.8h\n" + "zip2 v17.8h, v20.8h, v19.8h\n" "subs x20, x20, #0x1\n" "zip1 v16.8h, v18.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x0]\n" @@ -266,11 +266,11 @@ void interleave_block<8, 1, VLType::None, true>( "add v2.8h, v2.8h, v16.8h\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 10f\n" - "zip2 v22.8h, v31.8h, v27.8h\n" - "zip2 v21.8h, v29.8h, v24.8h\n" + "zip2 v22.8h, v30.8h, v26.8h\n" + "zip2 v21.8h, v28.8h, v24.8h\n" "subs x20, x20, #0x1\n" - "zip2 v20.8h, v30.8h, v26.8h\n" - "zip2 v19.8h, v28.8h, v23.8h\n" + "zip2 v20.8h, v29.8h, v25.8h\n" + "zip2 v19.8h, v27.8h, v23.8h\n" "zip1 v18.8h, v22.8h, v21.8h\n" "zip1 v17.8h, v20.8h, v19.8h\n" "zip1 v16.8h, v18.8h, v17.8h\n" @@ -284,9 +284,9 @@ void interleave_block<8, 1, VLType::None, true>( "add v2.8h, v2.8h, v16.8h\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 10f\n" - "zip2 v18.8h, v22.8h, v21.8h\n" - "zip2 v17.8h, v20.8h, v19.8h\n" - "zip1 v16.8h, v18.8h, v17.8h\n" + "zip2 v17.8h, v22.8h, v21.8h\n" + "zip2 v16.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v17.8h, v16.8h\n" "str q16, [%x[out_ptr], #0x0]\n" "add v2.8h, v2.8h, v16.8h\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s8_s16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s8_s16.hpp index 4a38187638d19be20d9ee2e9470c0662f5625b58..c41120c698ef1bb3b9cc7794561b714b56b7d35d 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s8_s16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s8_s16.hpp @@ -80,35 +80,35 @@ void interleave_block<8, 1, VLType::None, false>( "blt 3f\n" "2:" // Main loop head "ldr d25, [x28], #0x8\n" - "ldr d30, [x27], #0x8\n" + "ldr d27, [x27], #0x8\n" "sshll v25.8h, v25.8b, #0x0\n" - "sshll v30.8h, v30.8b, #0x0\n" - "ldr d29, [x26], #0x8\n" - "ldr d28, [x25], #0x8\n" - "sshll v29.8h, v29.8b, #0x0\n" - "sshll v28.8h, v28.8b, #0x0\n" + "sshll v27.8h, v27.8b, #0x0\n" + "ldr d26, [x26], #0x8\n" + "ldr d24, [x25], #0x8\n" + "sshll v26.8h, v26.8b, #0x0\n" + "sshll v24.8h, v24.8b, #0x0\n" "ldr d21, [x24], #0x8\n" - "ldr d27, [x23], #0x8\n" + "ldr d20, [x23], #0x8\n" "sshll v21.8h, v21.8b, #0x0\n" - "sshll v27.8h, v27.8b, #0x0\n" - "ldr d20, [x22], #0x8\n" - "ldr d26, [x21], #0x8\n" "sshll v20.8h, v20.8b, #0x0\n" - "sshll v26.8h, v26.8b, #0x0\n" + "ldr d17, [x22], #0x8\n" + "ldr d16, [x21], #0x8\n" + "sshll v17.8h, v17.8b, #0x0\n" + "sshll v16.8h, v16.8b, #0x0\n" "zip1 v23.8h, v25.8h, v21.8h\n" - "zip1 v22.8h, v29.8h, v20.8h\n" + "zip1 v22.8h, v26.8h, v17.8h\n" "subs %x[width], %x[width], #0x8\n" "cmp %x[width], #0x8\n" - "zip1 v19.8h, v30.8h, v27.8h\n" - "zip1 v18.8h, v28.8h, v26.8h\n" + "zip1 v19.8h, v27.8h, v20.8h\n" + "zip1 v18.8h, v24.8h, v16.8h\n" "prfm pldl1keep, [x28, #0x70]\n" "prfm pldl1keep, [x27, #0x70]\n" "zip2 v25.8h, v25.8h, v21.8h\n" - "zip2 v21.8h, v29.8h, v20.8h\n" + "zip2 v21.8h, v26.8h, v17.8h\n" "prfm pldl1keep, [x26, #0x70]\n" "prfm pldl1keep, [x25, #0x70]\n" - "zip2 v20.8h, v30.8h, v27.8h\n" - "zip2 v16.8h, v28.8h, v26.8h\n" + "zip2 v20.8h, v27.8h, v20.8h\n" + "zip2 v16.8h, v24.8h, v16.8h\n" "prfm pldl1keep, [x24, #0x70]\n" "prfm pldl1keep, [x23, #0x70]\n" "zip1 v24.8h, v23.8h, v22.8h\n" @@ -142,140 +142,139 @@ void interleave_block<8, 1, VLType::None, false>( "3:" // Main loop skip "cbz %x[width], 8f\n" "tbz %x[width], #2, 5f\n" - "ldr s25, [x28], #0x4\n" - "ldr s30, [x27], #0x4\n" - "ldr s29, [x26], #0x4\n" - "ldr s28, [x25], #0x4\n" - "ldr s21, [x24], #0x4\n" - "ldr s27, [x23], #0x4\n" - "ldr s20, [x22], #0x4\n" - "ldr s26, [x21], #0x4\n" + "ldr s30, [x28], #0x4\n" + "ldr s29, [x27], #0x4\n" + "ldr s28, [x26], #0x4\n" + "ldr s27, [x25], #0x4\n" + "ldr s26, [x24], #0x4\n" + "ldr s25, [x23], #0x4\n" + "ldr s24, [x22], #0x4\n" + "ldr s23, [x21], #0x4\n" "tbz %x[width], #1, 4f\n" - "ld1 { v25.h }[2], [x28], #0x2\n" - "ld1 { v30.h }[2], [x27], #0x2\n" + "ld1 { v30.h }[2], [x28], #0x2\n" + "ld1 { v29.h }[2], [x27], #0x2\n" "mov x20, #0x6\n" - "ld1 { v29.h }[2], [x26], #0x2\n" - "ld1 { v28.h }[2], [x25], #0x2\n" - "ld1 { v21.h }[2], [x24], #0x2\n" - "ld1 { v27.h }[2], [x23], #0x2\n" - "ld1 { v20.h }[2], [x22], #0x2\n" - "ld1 { v26.h }[2], [x21], #0x2\n" + "ld1 { v28.h }[2], [x26], #0x2\n" + "ld1 { v27.h }[2], [x25], #0x2\n" + "ld1 { v26.h }[2], [x24], #0x2\n" + "ld1 { v25.h }[2], [x23], #0x2\n" + "ld1 { v24.h }[2], [x22], #0x2\n" + "ld1 { v23.h }[2], [x21], #0x2\n" "tbz %x[width], #0, 7f\n" - "ld1 { v25.b }[6], [x28]\n" - "ld1 { v30.b }[6], [x27]\n" + "ld1 { v30.b }[6], [x28]\n" + "ld1 { v29.b }[6], [x27]\n" "mov x20, #0x7\n" - "ld1 { v29.b }[6], [x26]\n" - "ld1 { v28.b }[6], [x25]\n" - "ld1 { v21.b }[6], [x24]\n" - "ld1 { v27.b }[6], [x23]\n" - "ld1 { v20.b }[6], [x22]\n" - "ld1 { v26.b }[6], [x21]\n" + "ld1 { v28.b }[6], [x26]\n" + "ld1 { v27.b }[6], [x25]\n" + "ld1 { v26.b }[6], [x24]\n" + "ld1 { v25.b }[6], [x23]\n" + "ld1 { v24.b }[6], [x22]\n" + "ld1 { v23.b }[6], [x21]\n" "b 7f\n" "4:" // odd_loads_1_4 "mov x20, #0x4\n" "tbz %x[width], #0, 7f\n" - "ld1 { v25.b }[4], [x28]\n" - "ld1 { v30.b }[4], [x27]\n" + "ld1 { v30.b }[4], [x28]\n" + "ld1 { v29.b }[4], [x27]\n" "mov x20, #0x5\n" - "ld1 { v29.b }[4], [x26]\n" - "ld1 { v28.b }[4], [x25]\n" - "ld1 { v21.b }[4], [x24]\n" - "ld1 { v27.b }[4], [x23]\n" - "ld1 { v20.b }[4], [x22]\n" - "ld1 { v26.b }[4], [x21]\n" + "ld1 { v28.b }[4], [x26]\n" + "ld1 { v27.b }[4], [x25]\n" + "ld1 { v26.b }[4], [x24]\n" + "ld1 { v25.b }[4], [x23]\n" + "ld1 { v24.b }[4], [x22]\n" + "ld1 { v23.b }[4], [x21]\n" "b 7f\n" "5:" // odd_loads_2_0 "tbz %x[width], #1, 6f\n" - "ldr h25, [x28], #0x2\n" - "ldr h30, [x27], #0x2\n" + "ldr h30, [x28], #0x2\n" + "ldr h29, [x27], #0x2\n" "mov x20, #0x2\n" - "ldr h29, [x26], #0x2\n" - "ldr h28, [x25], #0x2\n" - "ldr h21, [x24], #0x2\n" - "ldr h27, [x23], #0x2\n" - "ldr h20, [x22], #0x2\n" - "ldr h26, [x21], #0x2\n" + "ldr h28, [x26], #0x2\n" + "ldr h27, [x25], #0x2\n" + "ldr h26, [x24], #0x2\n" + "ldr h25, [x23], #0x2\n" + "ldr h24, [x22], #0x2\n" + "ldr h23, [x21], #0x2\n" "tbz %x[width], #0, 7f\n" - "ld1 { v25.b }[2], [x28]\n" - "ld1 { v30.b }[2], [x27]\n" + "ld1 { v30.b }[2], [x28]\n" + "ld1 { v29.b }[2], [x27]\n" "mov x20, #0x3\n" - "ld1 { v29.b }[2], [x26]\n" - "ld1 { v28.b }[2], [x25]\n" - "ld1 { v21.b }[2], [x24]\n" - "ld1 { v27.b }[2], [x23]\n" - "ld1 { v20.b }[2], [x22]\n" - "ld1 { v26.b }[2], [x21]\n" + "ld1 { v28.b }[2], [x26]\n" + "ld1 { v27.b }[2], [x25]\n" + "ld1 { v26.b }[2], [x24]\n" + "ld1 { v25.b }[2], [x23]\n" + "ld1 { v24.b }[2], [x22]\n" + "ld1 { v23.b }[2], [x21]\n" "b 7f\n" "6:" // odd_loads_1_0 - "ldr b25, [x28, #0x0]\n" - "ldr b30, [x27, #0x0]\n" + "ldr b30, [x28, #0x0]\n" + "ldr b29, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr b29, [x26, #0x0]\n" - "ldr b28, [x25, #0x0]\n" - "ldr b21, [x24, #0x0]\n" - "ldr b27, [x23, #0x0]\n" - "ldr b20, [x22, #0x0]\n" - "ldr b26, [x21, #0x0]\n" + "ldr b28, [x26, #0x0]\n" + "ldr b27, [x25, #0x0]\n" + "ldr b26, [x24, #0x0]\n" + "ldr b25, [x23, #0x0]\n" + "ldr b24, [x22, #0x0]\n" + "ldr b23, [x21, #0x0]\n" "7:" // Odd load end - "sshll v25.8h, v25.8b, #0x0\n" "sshll v30.8h, v30.8b, #0x0\n" - "subs x20, x20, #0x1\n" "sshll v29.8h, v29.8b, #0x0\n" + "subs x20, x20, #0x1\n" "sshll v28.8h, v28.8b, #0x0\n" - "sshll v21.8h, v21.8b, #0x0\n" "sshll v27.8h, v27.8b, #0x0\n" - "sshll v20.8h, v20.8b, #0x0\n" "sshll v26.8h, v26.8b, #0x0\n" - "zip1 v23.8h, v25.8h, v21.8h\n" - "zip1 v22.8h, v29.8h, v20.8h\n" - "zip1 v19.8h, v30.8h, v27.8h\n" - "zip1 v18.8h, v28.8h, v26.8h\n" - "zip1 v24.8h, v23.8h, v22.8h\n" - "zip1 v17.8h, v19.8h, v18.8h\n" - "zip1 v16.8h, v24.8h, v17.8h\n" + "sshll v25.8h, v25.8b, #0x0\n" + "sshll v24.8h, v24.8b, #0x0\n" + "sshll v23.8h, v23.8b, #0x0\n" + "zip1 v22.8h, v30.8h, v26.8h\n" + "zip1 v21.8h, v28.8h, v24.8h\n" + "zip1 v20.8h, v29.8h, v25.8h\n" + "zip1 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" "subs x20, x20, #0x1\n" - "zip2 v16.8h, v24.8h, v17.8h\n" + "zip2 v16.8h, v18.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" - "zip2 v23.8h, v23.8h, v22.8h\n" - "zip2 v19.8h, v19.8h, v18.8h\n" + "zip2 v18.8h, v22.8h, v21.8h\n" + "zip2 v17.8h, v20.8h, v19.8h\n" "subs x20, x20, #0x1\n" - "zip1 v17.8h, v23.8h, v19.8h\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" "subs x20, x20, #0x1\n" - "zip2 v16.8h, v23.8h, v19.8h\n" + "zip2 v16.8h, v18.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" - "zip2 v25.8h, v25.8h, v21.8h\n" - "zip2 v21.8h, v29.8h, v20.8h\n" + "zip2 v22.8h, v30.8h, v26.8h\n" + "zip2 v21.8h, v28.8h, v24.8h\n" "subs x20, x20, #0x1\n" - "zip2 v20.8h, v30.8h, v27.8h\n" - "zip2 v16.8h, v28.8h, v26.8h\n" - "zip1 v22.8h, v25.8h, v21.8h\n" - "zip1 v18.8h, v20.8h, v16.8h\n" - "zip1 v19.8h, v22.8h, v18.8h\n" - "str q19, [%x[out_ptr], #0x0]\n" + "zip2 v20.8h, v29.8h, v25.8h\n" + "zip2 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" "subs x20, x20, #0x1\n" - "zip2 v18.8h, v22.8h, v18.8h\n" - "str q18, [%x[out_ptr], #0x0]\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" - "zip2 v21.8h, v25.8h, v21.8h\n" - "zip2 v20.8h, v20.8h, v16.8h\n" - "zip1 v17.8h, v21.8h, v20.8h\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip2 v17.8h, v22.8h, v21.8h\n" + "zip2 v16.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v17.8h, v16.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "8:" // Odds skip - : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s8_s16_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s8_s16_summing.hpp index 3ad103c8d4014e74460a9a8c9c4c397b4fe055b5..9ac7053ad8b2a03d6542df702209e5ac1336499d 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s8_s16_summing.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_s8_s16_summing.hpp @@ -167,109 +167,109 @@ void interleave_block<8, 1, VLType::None, true>( "5:" // Main loop skip "cbz %x[width], 10f\n" "tbz %x[width], #2, 7f\n" - "ldr s31, [x28], #0x4\n" - "ldr s30, [x27], #0x4\n" - "ldr s29, [x26], #0x4\n" - "ldr s28, [x25], #0x4\n" - "ldr s27, [x24], #0x4\n" - "ldr s26, [x23], #0x4\n" - "ldr s25, [x22], #0x4\n" - "ldr s24, [x21], #0x4\n" + "ldr s30, [x28], #0x4\n" + "ldr s29, [x27], #0x4\n" + "ldr s28, [x26], #0x4\n" + "ldr s27, [x25], #0x4\n" + "ldr s26, [x24], #0x4\n" + "ldr s25, [x23], #0x4\n" + "ldr s24, [x22], #0x4\n" + "ldr s23, [x21], #0x4\n" "tbz %x[width], #1, 6f\n" - "ld1 { v31.h }[2], [x28], #0x2\n" - "ld1 { v30.h }[2], [x27], #0x2\n" + "ld1 { v30.h }[2], [x28], #0x2\n" + "ld1 { v29.h }[2], [x27], #0x2\n" "mov x20, #0x6\n" - "ld1 { v29.h }[2], [x26], #0x2\n" - "ld1 { v28.h }[2], [x25], #0x2\n" - "ld1 { v27.h }[2], [x24], #0x2\n" - "ld1 { v26.h }[2], [x23], #0x2\n" - "ld1 { v25.h }[2], [x22], #0x2\n" - "ld1 { v24.h }[2], [x21], #0x2\n" + "ld1 { v28.h }[2], [x26], #0x2\n" + "ld1 { v27.h }[2], [x25], #0x2\n" + "ld1 { v26.h }[2], [x24], #0x2\n" + "ld1 { v25.h }[2], [x23], #0x2\n" + "ld1 { v24.h }[2], [x22], #0x2\n" + "ld1 { v23.h }[2], [x21], #0x2\n" "tbz %x[width], #0, 9f\n" - "ld1 { v31.b }[6], [x28]\n" - "ld1 { v30.b }[6], [x27]\n" + "ld1 { v30.b }[6], [x28]\n" + "ld1 { v29.b }[6], [x27]\n" "mov x20, #0x7\n" - "ld1 { v29.b }[6], [x26]\n" - "ld1 { v28.b }[6], [x25]\n" - "ld1 { v27.b }[6], [x24]\n" - "ld1 { v26.b }[6], [x23]\n" - "ld1 { v25.b }[6], [x22]\n" - "ld1 { v24.b }[6], [x21]\n" + "ld1 { v28.b }[6], [x26]\n" + "ld1 { v27.b }[6], [x25]\n" + "ld1 { v26.b }[6], [x24]\n" + "ld1 { v25.b }[6], [x23]\n" + "ld1 { v24.b }[6], [x22]\n" + "ld1 { v23.b }[6], [x21]\n" "b 9f\n" "6:" // odd_loads_1_4 "mov x20, #0x4\n" "tbz %x[width], #0, 9f\n" - "ld1 { v31.b }[4], [x28]\n" - "ld1 { v30.b }[4], [x27]\n" + "ld1 { v30.b }[4], [x28]\n" + "ld1 { v29.b }[4], [x27]\n" "mov x20, #0x5\n" - "ld1 { v29.b }[4], [x26]\n" - "ld1 { v28.b }[4], [x25]\n" - "ld1 { v27.b }[4], [x24]\n" - "ld1 { v26.b }[4], [x23]\n" - "ld1 { v25.b }[4], [x22]\n" - "ld1 { v24.b }[4], [x21]\n" + "ld1 { v28.b }[4], [x26]\n" + "ld1 { v27.b }[4], [x25]\n" + "ld1 { v26.b }[4], [x24]\n" + "ld1 { v25.b }[4], [x23]\n" + "ld1 { v24.b }[4], [x22]\n" + "ld1 { v23.b }[4], [x21]\n" "b 9f\n" "7:" // odd_loads_2_0 "tbz %x[width], #1, 8f\n" - "ldr h31, [x28], #0x2\n" - "ldr h30, [x27], #0x2\n" + "ldr h30, [x28], #0x2\n" + "ldr h29, [x27], #0x2\n" "mov x20, #0x2\n" - "ldr h29, [x26], #0x2\n" - "ldr h28, [x25], #0x2\n" - "ldr h27, [x24], #0x2\n" - "ldr h26, [x23], #0x2\n" - "ldr h25, [x22], #0x2\n" - "ldr h24, [x21], #0x2\n" + "ldr h28, [x26], #0x2\n" + "ldr h27, [x25], #0x2\n" + "ldr h26, [x24], #0x2\n" + "ldr h25, [x23], #0x2\n" + "ldr h24, [x22], #0x2\n" + "ldr h23, [x21], #0x2\n" "tbz %x[width], #0, 9f\n" - "ld1 { v31.b }[2], [x28]\n" - "ld1 { v30.b }[2], [x27]\n" + "ld1 { v30.b }[2], [x28]\n" + "ld1 { v29.b }[2], [x27]\n" "mov x20, #0x3\n" - "ld1 { v29.b }[2], [x26]\n" - "ld1 { v28.b }[2], [x25]\n" - "ld1 { v27.b }[2], [x24]\n" - "ld1 { v26.b }[2], [x23]\n" - "ld1 { v25.b }[2], [x22]\n" - "ld1 { v24.b }[2], [x21]\n" + "ld1 { v28.b }[2], [x26]\n" + "ld1 { v27.b }[2], [x25]\n" + "ld1 { v26.b }[2], [x24]\n" + "ld1 { v25.b }[2], [x23]\n" + "ld1 { v24.b }[2], [x22]\n" + "ld1 { v23.b }[2], [x21]\n" "b 9f\n" "8:" // odd_loads_1_0 - "ldr b31, [x28, #0x0]\n" - "ldr b30, [x27, #0x0]\n" + "ldr b30, [x28, #0x0]\n" + "ldr b29, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr b29, [x26, #0x0]\n" - "ldr b28, [x25, #0x0]\n" - "ldr b27, [x24, #0x0]\n" - "ldr b26, [x23, #0x0]\n" - "ldr b25, [x22, #0x0]\n" - "ldr b24, [x21, #0x0]\n" + "ldr b28, [x26, #0x0]\n" + "ldr b27, [x25, #0x0]\n" + "ldr b26, [x24, #0x0]\n" + "ldr b25, [x23, #0x0]\n" + "ldr b24, [x22, #0x0]\n" + "ldr b23, [x21, #0x0]\n" "9:" // Odd load end - "sshll v31.8h, v31.8b, #0x0\n" "sshll v30.8h, v30.8b, #0x0\n" - "subs x20, x20, #0x1\n" "sshll v29.8h, v29.8b, #0x0\n" + "subs x20, x20, #0x1\n" "sshll v28.8h, v28.8b, #0x0\n" "sshll v27.8h, v27.8b, #0x0\n" "sshll v26.8h, v26.8b, #0x0\n" "sshll v25.8h, v25.8b, #0x0\n" "sshll v24.8h, v24.8b, #0x0\n" - "zip1 v23.8h, v31.8h, v27.8h\n" - "zip1 v22.8h, v29.8h, v25.8h\n" - "zip1 v21.8h, v30.8h, v26.8h\n" - "zip1 v20.8h, v28.8h, v24.8h\n" - "zip1 v18.8h, v23.8h, v22.8h\n" - "zip1 v17.8h, v21.8h, v20.8h\n" + "sshll v23.8h, v23.8b, #0x0\n" + "zip1 v22.8h, v30.8h, v26.8h\n" + "zip1 v21.8h, v28.8h, v24.8h\n" + "zip1 v20.8h, v29.8h, v25.8h\n" + "zip1 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" "zip1 v16.8h, v18.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x0]\n" "add v2.8h, v2.8h, v16.8h\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 10f\n" - "zip2 v19.8h, v18.8h, v17.8h\n" + "zip2 v16.8h, v18.8h, v17.8h\n" "subs x20, x20, #0x1\n" - "str q19, [%x[out_ptr], #0x0]\n" - "add v2.8h, v2.8h, v19.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 10f\n" - "zip2 v18.8h, v23.8h, v22.8h\n" - "zip2 v17.8h, v21.8h, v20.8h\n" + "zip2 v18.8h, v22.8h, v21.8h\n" + "zip2 v17.8h, v20.8h, v19.8h\n" "subs x20, x20, #0x1\n" "zip1 v16.8h, v18.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x0]\n" @@ -282,11 +282,11 @@ void interleave_block<8, 1, VLType::None, true>( "add v2.8h, v2.8h, v16.8h\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 10f\n" - "zip2 v22.8h, v31.8h, v27.8h\n" - "zip2 v21.8h, v29.8h, v25.8h\n" + "zip2 v22.8h, v30.8h, v26.8h\n" + "zip2 v21.8h, v28.8h, v24.8h\n" "subs x20, x20, #0x1\n" - "zip2 v20.8h, v30.8h, v26.8h\n" - "zip2 v19.8h, v28.8h, v24.8h\n" + "zip2 v20.8h, v29.8h, v25.8h\n" + "zip2 v19.8h, v27.8h, v23.8h\n" "zip1 v18.8h, v22.8h, v21.8h\n" "zip1 v17.8h, v20.8h, v19.8h\n" "zip1 v16.8h, v18.8h, v17.8h\n" @@ -300,9 +300,9 @@ void interleave_block<8, 1, VLType::None, true>( "add v2.8h, v2.8h, v16.8h\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 10f\n" - "zip2 v18.8h, v22.8h, v21.8h\n" - "zip2 v17.8h, v20.8h, v19.8h\n" - "zip1 v16.8h, v18.8h, v17.8h\n" + "zip2 v17.8h, v22.8h, v21.8h\n" + "zip2 v16.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v17.8h, v16.8h\n" "str q16, [%x[out_ptr], #0x0]\n" "add v2.8h, v2.8h, v16.8h\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u16_u16_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u16_u16_summing.hpp index de29d77a22c1e011b20d3c35c5bc1aa4ac83c46a..c01d980f49c8c9e61cff010ffb32f81d69775e4f 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u16_u16_summing.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u16_u16_summing.hpp @@ -159,101 +159,101 @@ void interleave_block<8, 1, VLType::None, true>( "5:" // Main loop skip "cbz %x[width], 10f\n" "tbz %x[width], #2, 7f\n" - "ldr d31, [x28], #0x8\n" - "ldr d30, [x27], #0x8\n" - "ldr d29, [x26], #0x8\n" - "ldr d28, [x25], #0x8\n" - "ldr d27, [x24], #0x8\n" - "ldr d26, [x23], #0x8\n" + "ldr d30, [x28], #0x8\n" + "ldr d29, [x27], #0x8\n" + "ldr d28, [x26], #0x8\n" + "ldr d27, [x25], #0x8\n" + "ldr d26, [x24], #0x8\n" + "ldr d25, [x23], #0x8\n" "ldr d24, [x22], #0x8\n" "ldr d23, [x21], #0x8\n" "tbz %x[width], #1, 6f\n" - "ld1 { v31.s }[2], [x28], #0x4\n" - "ld1 { v30.s }[2], [x27], #0x4\n" + "ld1 { v30.s }[2], [x28], #0x4\n" + "ld1 { v29.s }[2], [x27], #0x4\n" "mov x20, #0x6\n" - "ld1 { v29.s }[2], [x26], #0x4\n" - "ld1 { v28.s }[2], [x25], #0x4\n" - "ld1 { v27.s }[2], [x24], #0x4\n" - "ld1 { v26.s }[2], [x23], #0x4\n" + "ld1 { v28.s }[2], [x26], #0x4\n" + "ld1 { v27.s }[2], [x25], #0x4\n" + "ld1 { v26.s }[2], [x24], #0x4\n" + "ld1 { v25.s }[2], [x23], #0x4\n" "ld1 { v24.s }[2], [x22], #0x4\n" "ld1 { v23.s }[2], [x21], #0x4\n" "tbz %x[width], #0, 9f\n" - "ld1 { v31.h }[6], [x28]\n" - "ld1 { v30.h }[6], [x27]\n" + "ld1 { v30.h }[6], [x28]\n" + "ld1 { v29.h }[6], [x27]\n" "mov x20, #0x7\n" - "ld1 { v29.h }[6], [x26]\n" - "ld1 { v28.h }[6], [x25]\n" - "ld1 { v27.h }[6], [x24]\n" - "ld1 { v26.h }[6], [x23]\n" + "ld1 { v28.h }[6], [x26]\n" + "ld1 { v27.h }[6], [x25]\n" + "ld1 { v26.h }[6], [x24]\n" + "ld1 { v25.h }[6], [x23]\n" "ld1 { v24.h }[6], [x22]\n" "ld1 { v23.h }[6], [x21]\n" "b 9f\n" "6:" // odd_loads_1_4 "mov x20, #0x4\n" "tbz %x[width], #0, 9f\n" - "ld1 { v31.h }[4], [x28]\n" - "ld1 { v30.h }[4], [x27]\n" + "ld1 { v30.h }[4], [x28]\n" + "ld1 { v29.h }[4], [x27]\n" "mov x20, #0x5\n" - "ld1 { v29.h }[4], [x26]\n" - "ld1 { v28.h }[4], [x25]\n" - "ld1 { v27.h }[4], [x24]\n" - "ld1 { v26.h }[4], [x23]\n" + "ld1 { v28.h }[4], [x26]\n" + "ld1 { v27.h }[4], [x25]\n" + "ld1 { v26.h }[4], [x24]\n" + "ld1 { v25.h }[4], [x23]\n" "ld1 { v24.h }[4], [x22]\n" "ld1 { v23.h }[4], [x21]\n" "b 9f\n" "7:" // odd_loads_2_0 "tbz %x[width], #1, 8f\n" - "ldr s31, [x28], #0x4\n" - "ldr s30, [x27], #0x4\n" + "ldr s30, [x28], #0x4\n" + "ldr s29, [x27], #0x4\n" "mov x20, #0x2\n" - "ldr s29, [x26], #0x4\n" - "ldr s28, [x25], #0x4\n" - "ldr s27, [x24], #0x4\n" - "ldr s26, [x23], #0x4\n" + "ldr s28, [x26], #0x4\n" + "ldr s27, [x25], #0x4\n" + "ldr s26, [x24], #0x4\n" + "ldr s25, [x23], #0x4\n" "ldr s24, [x22], #0x4\n" "ldr s23, [x21], #0x4\n" "tbz %x[width], #0, 9f\n" - "ld1 { v31.h }[2], [x28]\n" - "ld1 { v30.h }[2], [x27]\n" + "ld1 { v30.h }[2], [x28]\n" + "ld1 { v29.h }[2], [x27]\n" "mov x20, #0x3\n" - "ld1 { v29.h }[2], [x26]\n" - "ld1 { v28.h }[2], [x25]\n" - "ld1 { v27.h }[2], [x24]\n" - "ld1 { v26.h }[2], [x23]\n" + "ld1 { v28.h }[2], [x26]\n" + "ld1 { v27.h }[2], [x25]\n" + "ld1 { v26.h }[2], [x24]\n" + "ld1 { v25.h }[2], [x23]\n" "ld1 { v24.h }[2], [x22]\n" "ld1 { v23.h }[2], [x21]\n" "b 9f\n" "8:" // odd_loads_1_0 - "ldr h31, [x28, #0x0]\n" - "ldr h30, [x27, #0x0]\n" + "ldr h30, [x28, #0x0]\n" + "ldr h29, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr h29, [x26, #0x0]\n" - "ldr h28, [x25, #0x0]\n" - "ldr h27, [x24, #0x0]\n" - "ldr h26, [x23, #0x0]\n" + "ldr h28, [x26, #0x0]\n" + "ldr h27, [x25, #0x0]\n" + "ldr h26, [x24, #0x0]\n" + "ldr h25, [x23, #0x0]\n" "ldr h24, [x22, #0x0]\n" "ldr h23, [x21, #0x0]\n" "9:" // Odd load end - "zip1 v25.8h, v31.8h, v27.8h\n" - "zip1 v18.8h, v29.8h, v24.8h\n" - "subs x20, x20, #0x1\n" "zip1 v22.8h, v30.8h, v26.8h\n" - "zip1 v21.8h, v28.8h, v23.8h\n" - "zip1 v17.8h, v25.8h, v18.8h\n" - "zip1 v16.8h, v22.8h, v21.8h\n" - "zip1 v20.8h, v17.8h, v16.8h\n" - "str q20, [%x[out_ptr], #0x0]\n" - "add v2.8h, v2.8h, v20.8h\n" + "zip1 v21.8h, v28.8h, v24.8h\n" + "subs x20, x20, #0x1\n" + "zip1 v20.8h, v29.8h, v25.8h\n" + "zip1 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 10f\n" - "zip2 v19.8h, v17.8h, v16.8h\n" + "zip2 v16.8h, v18.8h, v17.8h\n" "subs x20, x20, #0x1\n" - "str q19, [%x[out_ptr], #0x0]\n" - "add v2.8h, v2.8h, v19.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 10f\n" - "zip2 v18.8h, v25.8h, v18.8h\n" - "zip2 v17.8h, v22.8h, v21.8h\n" + "zip2 v18.8h, v22.8h, v21.8h\n" + "zip2 v17.8h, v20.8h, v19.8h\n" "subs x20, x20, #0x1\n" "zip1 v16.8h, v18.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x0]\n" @@ -266,11 +266,11 @@ void interleave_block<8, 1, VLType::None, true>( "add v2.8h, v2.8h, v16.8h\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 10f\n" - "zip2 v22.8h, v31.8h, v27.8h\n" - "zip2 v21.8h, v29.8h, v24.8h\n" + "zip2 v22.8h, v30.8h, v26.8h\n" + "zip2 v21.8h, v28.8h, v24.8h\n" "subs x20, x20, #0x1\n" - "zip2 v20.8h, v30.8h, v26.8h\n" - "zip2 v19.8h, v28.8h, v23.8h\n" + "zip2 v20.8h, v29.8h, v25.8h\n" + "zip2 v19.8h, v27.8h, v23.8h\n" "zip1 v18.8h, v22.8h, v21.8h\n" "zip1 v17.8h, v20.8h, v19.8h\n" "zip1 v16.8h, v18.8h, v17.8h\n" @@ -284,9 +284,9 @@ void interleave_block<8, 1, VLType::None, true>( "add v2.8h, v2.8h, v16.8h\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 10f\n" - "zip2 v18.8h, v22.8h, v21.8h\n" - "zip2 v17.8h, v20.8h, v19.8h\n" - "zip1 v16.8h, v18.8h, v17.8h\n" + "zip2 v17.8h, v22.8h, v21.8h\n" + "zip2 v16.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v17.8h, v16.8h\n" "str q16, [%x[out_ptr], #0x0]\n" "add v2.8h, v2.8h, v16.8h\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u8_u16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u8_u16.hpp index 43a3a4680105e198a30b1a99a856da31e4d2ac08..d29a995b4664d3ac6c0b014a8160b03fd63a396c 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u8_u16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u8_u16.hpp @@ -80,35 +80,35 @@ void interleave_block<8, 1, VLType::None, false>( "blt 3f\n" "2:" // Main loop head "ldr d25, [x28], #0x8\n" - "ldr d30, [x27], #0x8\n" + "ldr d27, [x27], #0x8\n" "ushll v25.8h, v25.8b, #0x0\n" - "ushll v30.8h, v30.8b, #0x0\n" - "ldr d29, [x26], #0x8\n" - "ldr d28, [x25], #0x8\n" - "ushll v29.8h, v29.8b, #0x0\n" - "ushll v28.8h, v28.8b, #0x0\n" + "ushll v27.8h, v27.8b, #0x0\n" + "ldr d26, [x26], #0x8\n" + "ldr d24, [x25], #0x8\n" + "ushll v26.8h, v26.8b, #0x0\n" + "ushll v24.8h, v24.8b, #0x0\n" "ldr d21, [x24], #0x8\n" - "ldr d27, [x23], #0x8\n" + "ldr d20, [x23], #0x8\n" "ushll v21.8h, v21.8b, #0x0\n" - "ushll v27.8h, v27.8b, #0x0\n" - "ldr d20, [x22], #0x8\n" - "ldr d26, [x21], #0x8\n" "ushll v20.8h, v20.8b, #0x0\n" - "ushll v26.8h, v26.8b, #0x0\n" + "ldr d17, [x22], #0x8\n" + "ldr d16, [x21], #0x8\n" + "ushll v17.8h, v17.8b, #0x0\n" + "ushll v16.8h, v16.8b, #0x0\n" "zip1 v23.8h, v25.8h, v21.8h\n" - "zip1 v22.8h, v29.8h, v20.8h\n" + "zip1 v22.8h, v26.8h, v17.8h\n" "subs %x[width], %x[width], #0x8\n" "cmp %x[width], #0x8\n" - "zip1 v19.8h, v30.8h, v27.8h\n" - "zip1 v18.8h, v28.8h, v26.8h\n" + "zip1 v19.8h, v27.8h, v20.8h\n" + "zip1 v18.8h, v24.8h, v16.8h\n" "prfm pldl1keep, [x28, #0x70]\n" "prfm pldl1keep, [x27, #0x70]\n" "zip2 v25.8h, v25.8h, v21.8h\n" - "zip2 v21.8h, v29.8h, v20.8h\n" + "zip2 v21.8h, v26.8h, v17.8h\n" "prfm pldl1keep, [x26, #0x70]\n" "prfm pldl1keep, [x25, #0x70]\n" - "zip2 v20.8h, v30.8h, v27.8h\n" - "zip2 v16.8h, v28.8h, v26.8h\n" + "zip2 v20.8h, v27.8h, v20.8h\n" + "zip2 v16.8h, v24.8h, v16.8h\n" "prfm pldl1keep, [x24, #0x70]\n" "prfm pldl1keep, [x23, #0x70]\n" "zip1 v24.8h, v23.8h, v22.8h\n" @@ -142,140 +142,139 @@ void interleave_block<8, 1, VLType::None, false>( "3:" // Main loop skip "cbz %x[width], 8f\n" "tbz %x[width], #2, 5f\n" - "ldr s25, [x28], #0x4\n" - "ldr s30, [x27], #0x4\n" - "ldr s29, [x26], #0x4\n" - "ldr s28, [x25], #0x4\n" - "ldr s21, [x24], #0x4\n" - "ldr s27, [x23], #0x4\n" - "ldr s20, [x22], #0x4\n" - "ldr s26, [x21], #0x4\n" + "ldr s30, [x28], #0x4\n" + "ldr s29, [x27], #0x4\n" + "ldr s28, [x26], #0x4\n" + "ldr s27, [x25], #0x4\n" + "ldr s26, [x24], #0x4\n" + "ldr s25, [x23], #0x4\n" + "ldr s24, [x22], #0x4\n" + "ldr s23, [x21], #0x4\n" "tbz %x[width], #1, 4f\n" - "ld1 { v25.h }[2], [x28], #0x2\n" - "ld1 { v30.h }[2], [x27], #0x2\n" + "ld1 { v30.h }[2], [x28], #0x2\n" + "ld1 { v29.h }[2], [x27], #0x2\n" "mov x20, #0x6\n" - "ld1 { v29.h }[2], [x26], #0x2\n" - "ld1 { v28.h }[2], [x25], #0x2\n" - "ld1 { v21.h }[2], [x24], #0x2\n" - "ld1 { v27.h }[2], [x23], #0x2\n" - "ld1 { v20.h }[2], [x22], #0x2\n" - "ld1 { v26.h }[2], [x21], #0x2\n" + "ld1 { v28.h }[2], [x26], #0x2\n" + "ld1 { v27.h }[2], [x25], #0x2\n" + "ld1 { v26.h }[2], [x24], #0x2\n" + "ld1 { v25.h }[2], [x23], #0x2\n" + "ld1 { v24.h }[2], [x22], #0x2\n" + "ld1 { v23.h }[2], [x21], #0x2\n" "tbz %x[width], #0, 7f\n" - "ld1 { v25.b }[6], [x28]\n" - "ld1 { v30.b }[6], [x27]\n" + "ld1 { v30.b }[6], [x28]\n" + "ld1 { v29.b }[6], [x27]\n" "mov x20, #0x7\n" - "ld1 { v29.b }[6], [x26]\n" - "ld1 { v28.b }[6], [x25]\n" - "ld1 { v21.b }[6], [x24]\n" - "ld1 { v27.b }[6], [x23]\n" - "ld1 { v20.b }[6], [x22]\n" - "ld1 { v26.b }[6], [x21]\n" + "ld1 { v28.b }[6], [x26]\n" + "ld1 { v27.b }[6], [x25]\n" + "ld1 { v26.b }[6], [x24]\n" + "ld1 { v25.b }[6], [x23]\n" + "ld1 { v24.b }[6], [x22]\n" + "ld1 { v23.b }[6], [x21]\n" "b 7f\n" "4:" // odd_loads_1_4 "mov x20, #0x4\n" "tbz %x[width], #0, 7f\n" - "ld1 { v25.b }[4], [x28]\n" - "ld1 { v30.b }[4], [x27]\n" + "ld1 { v30.b }[4], [x28]\n" + "ld1 { v29.b }[4], [x27]\n" "mov x20, #0x5\n" - "ld1 { v29.b }[4], [x26]\n" - "ld1 { v28.b }[4], [x25]\n" - "ld1 { v21.b }[4], [x24]\n" - "ld1 { v27.b }[4], [x23]\n" - "ld1 { v20.b }[4], [x22]\n" - "ld1 { v26.b }[4], [x21]\n" + "ld1 { v28.b }[4], [x26]\n" + "ld1 { v27.b }[4], [x25]\n" + "ld1 { v26.b }[4], [x24]\n" + "ld1 { v25.b }[4], [x23]\n" + "ld1 { v24.b }[4], [x22]\n" + "ld1 { v23.b }[4], [x21]\n" "b 7f\n" "5:" // odd_loads_2_0 "tbz %x[width], #1, 6f\n" - "ldr h25, [x28], #0x2\n" - "ldr h30, [x27], #0x2\n" + "ldr h30, [x28], #0x2\n" + "ldr h29, [x27], #0x2\n" "mov x20, #0x2\n" - "ldr h29, [x26], #0x2\n" - "ldr h28, [x25], #0x2\n" - "ldr h21, [x24], #0x2\n" - "ldr h27, [x23], #0x2\n" - "ldr h20, [x22], #0x2\n" - "ldr h26, [x21], #0x2\n" + "ldr h28, [x26], #0x2\n" + "ldr h27, [x25], #0x2\n" + "ldr h26, [x24], #0x2\n" + "ldr h25, [x23], #0x2\n" + "ldr h24, [x22], #0x2\n" + "ldr h23, [x21], #0x2\n" "tbz %x[width], #0, 7f\n" - "ld1 { v25.b }[2], [x28]\n" - "ld1 { v30.b }[2], [x27]\n" + "ld1 { v30.b }[2], [x28]\n" + "ld1 { v29.b }[2], [x27]\n" "mov x20, #0x3\n" - "ld1 { v29.b }[2], [x26]\n" - "ld1 { v28.b }[2], [x25]\n" - "ld1 { v21.b }[2], [x24]\n" - "ld1 { v27.b }[2], [x23]\n" - "ld1 { v20.b }[2], [x22]\n" - "ld1 { v26.b }[2], [x21]\n" + "ld1 { v28.b }[2], [x26]\n" + "ld1 { v27.b }[2], [x25]\n" + "ld1 { v26.b }[2], [x24]\n" + "ld1 { v25.b }[2], [x23]\n" + "ld1 { v24.b }[2], [x22]\n" + "ld1 { v23.b }[2], [x21]\n" "b 7f\n" "6:" // odd_loads_1_0 - "ldr b25, [x28, #0x0]\n" - "ldr b30, [x27, #0x0]\n" + "ldr b30, [x28, #0x0]\n" + "ldr b29, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr b29, [x26, #0x0]\n" - "ldr b28, [x25, #0x0]\n" - "ldr b21, [x24, #0x0]\n" - "ldr b27, [x23, #0x0]\n" - "ldr b20, [x22, #0x0]\n" - "ldr b26, [x21, #0x0]\n" + "ldr b28, [x26, #0x0]\n" + "ldr b27, [x25, #0x0]\n" + "ldr b26, [x24, #0x0]\n" + "ldr b25, [x23, #0x0]\n" + "ldr b24, [x22, #0x0]\n" + "ldr b23, [x21, #0x0]\n" "7:" // Odd load end - "ushll v25.8h, v25.8b, #0x0\n" "ushll v30.8h, v30.8b, #0x0\n" - "subs x20, x20, #0x1\n" "ushll v29.8h, v29.8b, #0x0\n" + "subs x20, x20, #0x1\n" "ushll v28.8h, v28.8b, #0x0\n" - "ushll v21.8h, v21.8b, #0x0\n" "ushll v27.8h, v27.8b, #0x0\n" - "ushll v20.8h, v20.8b, #0x0\n" "ushll v26.8h, v26.8b, #0x0\n" - "zip1 v23.8h, v25.8h, v21.8h\n" - "zip1 v22.8h, v29.8h, v20.8h\n" - "zip1 v19.8h, v30.8h, v27.8h\n" - "zip1 v18.8h, v28.8h, v26.8h\n" - "zip1 v24.8h, v23.8h, v22.8h\n" - "zip1 v17.8h, v19.8h, v18.8h\n" - "zip1 v16.8h, v24.8h, v17.8h\n" + "ushll v25.8h, v25.8b, #0x0\n" + "ushll v24.8h, v24.8b, #0x0\n" + "ushll v23.8h, v23.8b, #0x0\n" + "zip1 v22.8h, v30.8h, v26.8h\n" + "zip1 v21.8h, v28.8h, v24.8h\n" + "zip1 v20.8h, v29.8h, v25.8h\n" + "zip1 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" "subs x20, x20, #0x1\n" - "zip2 v16.8h, v24.8h, v17.8h\n" + "zip2 v16.8h, v18.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" - "zip2 v23.8h, v23.8h, v22.8h\n" - "zip2 v19.8h, v19.8h, v18.8h\n" + "zip2 v18.8h, v22.8h, v21.8h\n" + "zip2 v17.8h, v20.8h, v19.8h\n" "subs x20, x20, #0x1\n" - "zip1 v17.8h, v23.8h, v19.8h\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" "subs x20, x20, #0x1\n" - "zip2 v16.8h, v23.8h, v19.8h\n" + "zip2 v16.8h, v18.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" - "zip2 v25.8h, v25.8h, v21.8h\n" - "zip2 v21.8h, v29.8h, v20.8h\n" + "zip2 v22.8h, v30.8h, v26.8h\n" + "zip2 v21.8h, v28.8h, v24.8h\n" "subs x20, x20, #0x1\n" - "zip2 v20.8h, v30.8h, v27.8h\n" - "zip2 v16.8h, v28.8h, v26.8h\n" - "zip1 v22.8h, v25.8h, v21.8h\n" - "zip1 v18.8h, v20.8h, v16.8h\n" - "zip1 v19.8h, v22.8h, v18.8h\n" - "str q19, [%x[out_ptr], #0x0]\n" + "zip2 v20.8h, v29.8h, v25.8h\n" + "zip2 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" "subs x20, x20, #0x1\n" - "zip2 v18.8h, v22.8h, v18.8h\n" - "str q18, [%x[out_ptr], #0x0]\n" + "zip2 v16.8h, v18.8h, v17.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 8f\n" - "zip2 v21.8h, v25.8h, v21.8h\n" - "zip2 v20.8h, v20.8h, v16.8h\n" - "zip1 v17.8h, v21.8h, v20.8h\n" - "str q17, [%x[out_ptr], #0x0]\n" + "zip2 v17.8h, v22.8h, v21.8h\n" + "zip2 v16.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v17.8h, v16.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "8:" // Odds skip - : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u8_u16_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u8_u16_summing.hpp index 3ab24365afa989d9a9391771d64028361132ef70..ae4bf9bf3b86a3408736c77f9e27e538c2fe3de0 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u8_u16_summing.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block1_u8_u16_summing.hpp @@ -167,109 +167,109 @@ void interleave_block<8, 1, VLType::None, true>( "5:" // Main loop skip "cbz %x[width], 10f\n" "tbz %x[width], #2, 7f\n" - "ldr s31, [x28], #0x4\n" - "ldr s30, [x27], #0x4\n" - "ldr s29, [x26], #0x4\n" - "ldr s28, [x25], #0x4\n" - "ldr s27, [x24], #0x4\n" - "ldr s26, [x23], #0x4\n" - "ldr s25, [x22], #0x4\n" - "ldr s24, [x21], #0x4\n" + "ldr s30, [x28], #0x4\n" + "ldr s29, [x27], #0x4\n" + "ldr s28, [x26], #0x4\n" + "ldr s27, [x25], #0x4\n" + "ldr s26, [x24], #0x4\n" + "ldr s25, [x23], #0x4\n" + "ldr s24, [x22], #0x4\n" + "ldr s23, [x21], #0x4\n" "tbz %x[width], #1, 6f\n" - "ld1 { v31.h }[2], [x28], #0x2\n" - "ld1 { v30.h }[2], [x27], #0x2\n" + "ld1 { v30.h }[2], [x28], #0x2\n" + "ld1 { v29.h }[2], [x27], #0x2\n" "mov x20, #0x6\n" - "ld1 { v29.h }[2], [x26], #0x2\n" - "ld1 { v28.h }[2], [x25], #0x2\n" - "ld1 { v27.h }[2], [x24], #0x2\n" - "ld1 { v26.h }[2], [x23], #0x2\n" - "ld1 { v25.h }[2], [x22], #0x2\n" - "ld1 { v24.h }[2], [x21], #0x2\n" + "ld1 { v28.h }[2], [x26], #0x2\n" + "ld1 { v27.h }[2], [x25], #0x2\n" + "ld1 { v26.h }[2], [x24], #0x2\n" + "ld1 { v25.h }[2], [x23], #0x2\n" + "ld1 { v24.h }[2], [x22], #0x2\n" + "ld1 { v23.h }[2], [x21], #0x2\n" "tbz %x[width], #0, 9f\n" - "ld1 { v31.b }[6], [x28]\n" - "ld1 { v30.b }[6], [x27]\n" + "ld1 { v30.b }[6], [x28]\n" + "ld1 { v29.b }[6], [x27]\n" "mov x20, #0x7\n" - "ld1 { v29.b }[6], [x26]\n" - "ld1 { v28.b }[6], [x25]\n" - "ld1 { v27.b }[6], [x24]\n" - "ld1 { v26.b }[6], [x23]\n" - "ld1 { v25.b }[6], [x22]\n" - "ld1 { v24.b }[6], [x21]\n" + "ld1 { v28.b }[6], [x26]\n" + "ld1 { v27.b }[6], [x25]\n" + "ld1 { v26.b }[6], [x24]\n" + "ld1 { v25.b }[6], [x23]\n" + "ld1 { v24.b }[6], [x22]\n" + "ld1 { v23.b }[6], [x21]\n" "b 9f\n" "6:" // odd_loads_1_4 "mov x20, #0x4\n" "tbz %x[width], #0, 9f\n" - "ld1 { v31.b }[4], [x28]\n" - "ld1 { v30.b }[4], [x27]\n" + "ld1 { v30.b }[4], [x28]\n" + "ld1 { v29.b }[4], [x27]\n" "mov x20, #0x5\n" - "ld1 { v29.b }[4], [x26]\n" - "ld1 { v28.b }[4], [x25]\n" - "ld1 { v27.b }[4], [x24]\n" - "ld1 { v26.b }[4], [x23]\n" - "ld1 { v25.b }[4], [x22]\n" - "ld1 { v24.b }[4], [x21]\n" + "ld1 { v28.b }[4], [x26]\n" + "ld1 { v27.b }[4], [x25]\n" + "ld1 { v26.b }[4], [x24]\n" + "ld1 { v25.b }[4], [x23]\n" + "ld1 { v24.b }[4], [x22]\n" + "ld1 { v23.b }[4], [x21]\n" "b 9f\n" "7:" // odd_loads_2_0 "tbz %x[width], #1, 8f\n" - "ldr h31, [x28], #0x2\n" - "ldr h30, [x27], #0x2\n" + "ldr h30, [x28], #0x2\n" + "ldr h29, [x27], #0x2\n" "mov x20, #0x2\n" - "ldr h29, [x26], #0x2\n" - "ldr h28, [x25], #0x2\n" - "ldr h27, [x24], #0x2\n" - "ldr h26, [x23], #0x2\n" - "ldr h25, [x22], #0x2\n" - "ldr h24, [x21], #0x2\n" + "ldr h28, [x26], #0x2\n" + "ldr h27, [x25], #0x2\n" + "ldr h26, [x24], #0x2\n" + "ldr h25, [x23], #0x2\n" + "ldr h24, [x22], #0x2\n" + "ldr h23, [x21], #0x2\n" "tbz %x[width], #0, 9f\n" - "ld1 { v31.b }[2], [x28]\n" - "ld1 { v30.b }[2], [x27]\n" + "ld1 { v30.b }[2], [x28]\n" + "ld1 { v29.b }[2], [x27]\n" "mov x20, #0x3\n" - "ld1 { v29.b }[2], [x26]\n" - "ld1 { v28.b }[2], [x25]\n" - "ld1 { v27.b }[2], [x24]\n" - "ld1 { v26.b }[2], [x23]\n" - "ld1 { v25.b }[2], [x22]\n" - "ld1 { v24.b }[2], [x21]\n" + "ld1 { v28.b }[2], [x26]\n" + "ld1 { v27.b }[2], [x25]\n" + "ld1 { v26.b }[2], [x24]\n" + "ld1 { v25.b }[2], [x23]\n" + "ld1 { v24.b }[2], [x22]\n" + "ld1 { v23.b }[2], [x21]\n" "b 9f\n" "8:" // odd_loads_1_0 - "ldr b31, [x28, #0x0]\n" - "ldr b30, [x27, #0x0]\n" + "ldr b30, [x28, #0x0]\n" + "ldr b29, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr b29, [x26, #0x0]\n" - "ldr b28, [x25, #0x0]\n" - "ldr b27, [x24, #0x0]\n" - "ldr b26, [x23, #0x0]\n" - "ldr b25, [x22, #0x0]\n" - "ldr b24, [x21, #0x0]\n" + "ldr b28, [x26, #0x0]\n" + "ldr b27, [x25, #0x0]\n" + "ldr b26, [x24, #0x0]\n" + "ldr b25, [x23, #0x0]\n" + "ldr b24, [x22, #0x0]\n" + "ldr b23, [x21, #0x0]\n" "9:" // Odd load end - "ushll v31.8h, v31.8b, #0x0\n" "ushll v30.8h, v30.8b, #0x0\n" - "subs x20, x20, #0x1\n" "ushll v29.8h, v29.8b, #0x0\n" + "subs x20, x20, #0x1\n" "ushll v28.8h, v28.8b, #0x0\n" "ushll v27.8h, v27.8b, #0x0\n" "ushll v26.8h, v26.8b, #0x0\n" "ushll v25.8h, v25.8b, #0x0\n" "ushll v24.8h, v24.8b, #0x0\n" - "zip1 v23.8h, v31.8h, v27.8h\n" - "zip1 v22.8h, v29.8h, v25.8h\n" - "zip1 v21.8h, v30.8h, v26.8h\n" - "zip1 v20.8h, v28.8h, v24.8h\n" - "zip1 v18.8h, v23.8h, v22.8h\n" - "zip1 v17.8h, v21.8h, v20.8h\n" + "ushll v23.8h, v23.8b, #0x0\n" + "zip1 v22.8h, v30.8h, v26.8h\n" + "zip1 v21.8h, v28.8h, v24.8h\n" + "zip1 v20.8h, v29.8h, v25.8h\n" + "zip1 v19.8h, v27.8h, v23.8h\n" + "zip1 v18.8h, v22.8h, v21.8h\n" + "zip1 v17.8h, v20.8h, v19.8h\n" "zip1 v16.8h, v18.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x0]\n" "add v2.8h, v2.8h, v16.8h\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 10f\n" - "zip2 v19.8h, v18.8h, v17.8h\n" + "zip2 v16.8h, v18.8h, v17.8h\n" "subs x20, x20, #0x1\n" - "str q19, [%x[out_ptr], #0x0]\n" - "add v2.8h, v2.8h, v19.8h\n" + "str q16, [%x[out_ptr], #0x0]\n" + "add v2.8h, v2.8h, v16.8h\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 10f\n" - "zip2 v18.8h, v23.8h, v22.8h\n" - "zip2 v17.8h, v21.8h, v20.8h\n" + "zip2 v18.8h, v22.8h, v21.8h\n" + "zip2 v17.8h, v20.8h, v19.8h\n" "subs x20, x20, #0x1\n" "zip1 v16.8h, v18.8h, v17.8h\n" "str q16, [%x[out_ptr], #0x0]\n" @@ -282,11 +282,11 @@ void interleave_block<8, 1, VLType::None, true>( "add v2.8h, v2.8h, v16.8h\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 10f\n" - "zip2 v22.8h, v31.8h, v27.8h\n" - "zip2 v21.8h, v29.8h, v25.8h\n" + "zip2 v22.8h, v30.8h, v26.8h\n" + "zip2 v21.8h, v28.8h, v24.8h\n" "subs x20, x20, #0x1\n" - "zip2 v20.8h, v30.8h, v26.8h\n" - "zip2 v19.8h, v28.8h, v24.8h\n" + "zip2 v20.8h, v29.8h, v25.8h\n" + "zip2 v19.8h, v27.8h, v23.8h\n" "zip1 v18.8h, v22.8h, v21.8h\n" "zip1 v17.8h, v20.8h, v19.8h\n" "zip1 v16.8h, v18.8h, v17.8h\n" @@ -300,9 +300,9 @@ void interleave_block<8, 1, VLType::None, true>( "add v2.8h, v2.8h, v16.8h\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" "beq 10f\n" - "zip2 v18.8h, v22.8h, v21.8h\n" - "zip2 v17.8h, v20.8h, v19.8h\n" - "zip1 v16.8h, v18.8h, v17.8h\n" + "zip2 v17.8h, v22.8h, v21.8h\n" + "zip2 v16.8h, v20.8h, v19.8h\n" + "zip1 v16.8h, v17.8h, v16.8h\n" "str q16, [%x[out_ptr], #0x0]\n" "add v2.8h, v2.8h, v16.8h\n" "add %x[out_ptr], %x[out_ptr], #0x10\n" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_bf16_bf16.hpp index d4d150456f6cb346b15c5b24e2e9fc8f42c47cfe..43d9d20c10ab37a353b661f9e9f7baf26d1be02d 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_bf16_bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_bf16_bf16.hpp @@ -79,29 +79,29 @@ void interleave_block<8, 2, VLType::None, false>( "prfm pldl1keep, [x21, #0x40]\n" "blt 3f\n" "2:" // Main loop head - "ldr q28, [x28], #0x10\n" - "ldr q27, [x27], #0x10\n" + "ldr q20, [x28], #0x10\n" + "ldr q18, [x27], #0x10\n" "subs %x[width], %x[width], #0x8\n" "cmp %x[width], #0x8\n" - "ldr q22, [x26], #0x10\n" - "ldr q21, [x25], #0x10\n" - "zip1 v26.4s, v28.4s, v22.4s\n" - "zip1 v25.4s, v27.4s, v21.4s\n" - "ldr q24, [x24], #0x10\n" + "ldr q17, [x26], #0x10\n" + "ldr q16, [x25], #0x10\n" + "zip1 v25.4s, v20.4s, v17.4s\n" + "zip1 v24.4s, v18.4s, v16.4s\n" + "ldr q19, [x24], #0x10\n" "ldr q23, [x23], #0x10\n" - "zip2 v22.4s, v28.4s, v22.4s\n" - "zip2 v21.4s, v27.4s, v21.4s\n" - "ldr q19, [x22], #0x10\n" - "ldr q18, [x21], #0x10\n" - "zip1 v20.4s, v24.4s, v19.4s\n" - "zip1 v17.4s, v23.4s, v18.4s\n" - "zip2 v19.4s, v24.4s, v19.4s\n" - "zip2 v18.4s, v23.4s, v18.4s\n" + "zip2 v22.4s, v20.4s, v17.4s\n" + "zip2 v21.4s, v18.4s, v16.4s\n" + "ldr q18, [x22], #0x10\n" + "ldr q16, [x21], #0x10\n" + "zip1 v20.4s, v19.4s, v18.4s\n" + "zip1 v17.4s, v23.4s, v16.4s\n" + "zip2 v19.4s, v19.4s, v18.4s\n" + "zip2 v18.4s, v23.4s, v16.4s\n" "prfm pldl1keep, [x28, #0x70]\n" "prfm pldl1keep, [x27, #0x70]\n" "prfm pldl1keep, [x26, #0x70]\n" "prfm pldl1keep, [x25, #0x70]\n" - "zip1 v16.4s, v26.4s, v25.4s\n" + "zip1 v16.4s, v25.4s, v24.4s\n" "str q16, [%x[out_ptr], #0x0]\n" "prfm pldl1keep, [x24, #0x70]\n" "prfm pldl1keep, [x23, #0x70]\n" @@ -109,7 +109,7 @@ void interleave_block<8, 2, VLType::None, false>( "str q16, [%x[out_ptr], #0x10]\n" "prfm pldl1keep, [x22, #0x70]\n" "prfm pldl1keep, [x21, #0x70]\n" - "zip2 v16.4s, v26.4s, v25.4s\n" + "zip2 v16.4s, v25.4s, v24.4s\n" "str q16, [%x[out_ptr], #0x20]\n" "zip2 v16.4s, v20.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x30]\n" @@ -128,32 +128,32 @@ void interleave_block<8, 2, VLType::None, false>( "tbz %x[width], #2, 5f\n" "ldr d28, [x28], #0x8\n" "ldr d27, [x27], #0x8\n" - "ldr d22, [x26], #0x8\n" - "ldr d21, [x25], #0x8\n" + "ldr d26, [x26], #0x8\n" + "ldr d25, [x25], #0x8\n" "ldr d24, [x24], #0x8\n" "ldr d23, [x23], #0x8\n" - "ldr d19, [x22], #0x8\n" - "ldr d18, [x21], #0x8\n" + "ldr d22, [x22], #0x8\n" + "ldr d21, [x21], #0x8\n" "tbz %x[width], #1, 4f\n" "ld1 { v28.s }[2], [x28], #0x4\n" "ld1 { v27.s }[2], [x27], #0x4\n" "mov x20, #0x3\n" - "ld1 { v22.s }[2], [x26], #0x4\n" - "ld1 { v21.s }[2], [x25], #0x4\n" + "ld1 { v26.s }[2], [x26], #0x4\n" + "ld1 { v25.s }[2], [x25], #0x4\n" "ld1 { v24.s }[2], [x24], #0x4\n" "ld1 { v23.s }[2], [x23], #0x4\n" - "ld1 { v19.s }[2], [x22], #0x4\n" - "ld1 { v18.s }[2], [x21], #0x4\n" + "ld1 { v22.s }[2], [x22], #0x4\n" + "ld1 { v21.s }[2], [x21], #0x4\n" "tbz %x[width], #0, 7f\n" "ld1 { v28.h }[6], [x28]\n" "ld1 { v27.h }[6], [x27]\n" "mov x20, #0x4\n" - "ld1 { v22.h }[6], [x26]\n" - "ld1 { v21.h }[6], [x25]\n" + "ld1 { v26.h }[6], [x26]\n" + "ld1 { v25.h }[6], [x25]\n" "ld1 { v24.h }[6], [x24]\n" "ld1 { v23.h }[6], [x23]\n" - "ld1 { v19.h }[6], [x22]\n" - "ld1 { v18.h }[6], [x21]\n" + "ld1 { v22.h }[6], [x22]\n" + "ld1 { v21.h }[6], [x21]\n" "b 7f\n" "4:" // odd_loads_1_4 "mov x20, #0x2\n" @@ -161,82 +161,81 @@ void interleave_block<8, 2, VLType::None, false>( "ld1 { v28.h }[4], [x28]\n" "ld1 { v27.h }[4], [x27]\n" "mov x20, #0x3\n" - "ld1 { v22.h }[4], [x26]\n" - "ld1 { v21.h }[4], [x25]\n" + "ld1 { v26.h }[4], [x26]\n" + "ld1 { v25.h }[4], [x25]\n" "ld1 { v24.h }[4], [x24]\n" "ld1 { v23.h }[4], [x23]\n" - "ld1 { v19.h }[4], [x22]\n" - "ld1 { v18.h }[4], [x21]\n" + "ld1 { v22.h }[4], [x22]\n" + "ld1 { v21.h }[4], [x21]\n" "b 7f\n" "5:" // odd_loads_2_0 "tbz %x[width], #1, 6f\n" "ldr s28, [x28], #0x4\n" "ldr s27, [x27], #0x4\n" "mov x20, #0x1\n" - "ldr s22, [x26], #0x4\n" - "ldr s21, [x25], #0x4\n" + "ldr s26, [x26], #0x4\n" + "ldr s25, [x25], #0x4\n" "ldr s24, [x24], #0x4\n" "ldr s23, [x23], #0x4\n" - "ldr s19, [x22], #0x4\n" - "ldr s18, [x21], #0x4\n" + "ldr s22, [x22], #0x4\n" + "ldr s21, [x21], #0x4\n" "tbz %x[width], #0, 7f\n" "ld1 { v28.h }[2], [x28]\n" "ld1 { v27.h }[2], [x27]\n" "mov x20, #0x2\n" - "ld1 { v22.h }[2], [x26]\n" - "ld1 { v21.h }[2], [x25]\n" + "ld1 { v26.h }[2], [x26]\n" + "ld1 { v25.h }[2], [x25]\n" "ld1 { v24.h }[2], [x24]\n" "ld1 { v23.h }[2], [x23]\n" - "ld1 { v19.h }[2], [x22]\n" - "ld1 { v18.h }[2], [x21]\n" + "ld1 { v22.h }[2], [x22]\n" + "ld1 { v21.h }[2], [x21]\n" "b 7f\n" "6:" // odd_loads_1_0 "ldr h28, [x28, #0x0]\n" "ldr h27, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr h22, [x26, #0x0]\n" - "ldr h21, [x25, #0x0]\n" + "ldr h26, [x26, #0x0]\n" + "ldr h25, [x25, #0x0]\n" "ldr h24, [x24, #0x0]\n" "ldr h23, [x23, #0x0]\n" - "ldr h19, [x22, #0x0]\n" - "ldr h18, [x21, #0x0]\n" + "ldr h22, [x22, #0x0]\n" + "ldr h21, [x21, #0x0]\n" "7:" // Odd load end - "zip1 v26.4s, v28.4s, v22.4s\n" - "zip1 v25.4s, v27.4s, v21.4s\n" + "zip1 v20.4s, v28.4s, v26.4s\n" + "zip1 v19.4s, v27.4s, v25.4s\n" "subs x20, x20, #0x1\n" - "zip1 v20.4s, v24.4s, v19.4s\n" - "zip1 v17.4s, v23.4s, v18.4s\n" - "zip1 v16.4s, v26.4s, v25.4s\n" + "zip1 v18.4s, v24.4s, v22.4s\n" + "zip1 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v20.4s, v19.4s\n" "str q16, [%x[out_ptr], #0x0]\n" - "zip1 v16.4s, v20.4s, v17.4s\n" + "zip1 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 8f\n" "subs x20, x20, #0x1\n" - "zip2 v16.4s, v26.4s, v25.4s\n" + "zip2 v16.4s, v20.4s, v19.4s\n" "str q16, [%x[out_ptr], #0x0]\n" - "zip2 v16.4s, v20.4s, v17.4s\n" + "zip2 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 8f\n" - "zip2 v22.4s, v28.4s, v22.4s\n" - "zip2 v21.4s, v27.4s, v21.4s\n" + "zip2 v20.4s, v28.4s, v26.4s\n" + "zip2 v19.4s, v27.4s, v25.4s\n" "subs x20, x20, #0x1\n" - "zip2 v19.4s, v24.4s, v19.4s\n" - "zip2 v18.4s, v23.4s, v18.4s\n" - "zip1 v16.4s, v22.4s, v21.4s\n" + "zip2 v18.4s, v24.4s, v22.4s\n" + "zip2 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v20.4s, v19.4s\n" "str q16, [%x[out_ptr], #0x0]\n" - "zip1 v16.4s, v19.4s, v18.4s\n" + "zip1 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 8f\n" - "zip2 v17.4s, v22.4s, v21.4s\n" - "str q17, [%x[out_ptr], #0x0]\n" - "zip2 v16.4s, v19.4s, v18.4s\n" + "zip2 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "8:" // Odds skip - : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_fp32_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_fp32_fp32.hpp index 358b83ad1bab0311f76d7bfc088fbf6e540a7ed0..3ec03370a019ef6ded8d07f8cbc6f34481b0b5f9 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_fp32_fp32.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block2_fp32_fp32.hpp @@ -79,18 +79,18 @@ void interleave_block<8, 2, VLType::None, false>( "prfm pldl1keep, [x21, #0x40]\n" "blt 3f\n" "2:" // Main loop head - "ldr q26, [x28], #0x10\n" - "ldr q21, [x27], #0x10\n" + "ldr q20, [x28], #0x10\n" + "ldr q19, [x27], #0x10\n" "subs %x[width], %x[width], #0x4\n" "cmp %x[width], #0x4\n" "ldr q25, [x26], #0x10\n" "ldr q24, [x25], #0x10\n" - "zip1 v16.2d, v26.2d, v21.2d\n" + "zip1 v16.2d, v20.2d, v19.2d\n" "zip1 v18.2d, v25.2d, v24.2d\n" "ldr q23, [x24], #0x10\n" "ldr q22, [x23], #0x10\n" "zip1 v17.2d, v23.2d, v22.2d\n" - "zip2 v21.2d, v26.2d, v21.2d\n" + "zip2 v21.2d, v20.2d, v19.2d\n" "ldr q20, [x22], #0x10\n" "ldr q19, [x21], #0x10\n" "str q16, [%x[out_ptr], #0x0]\n" @@ -118,62 +118,61 @@ void interleave_block<8, 2, VLType::None, false>( "3:" // Main loop skip "cbz %x[width], 6f\n" "tbz %x[width], #1, 4f\n" - "ldr d26, [x28], #0x8\n" - "ldr d21, [x27], #0x8\n" + "ldr d25, [x28], #0x8\n" + "ldr d24, [x27], #0x8\n" "mov x20, #0x1\n" - "ldr d25, [x26], #0x8\n" - "ldr d24, [x25], #0x8\n" - "ldr d23, [x24], #0x8\n" - "ldr d22, [x23], #0x8\n" - "ldr d20, [x22], #0x8\n" - "ldr d19, [x21], #0x8\n" + "ldr d23, [x26], #0x8\n" + "ldr d22, [x25], #0x8\n" + "ldr d21, [x24], #0x8\n" + "ldr d20, [x23], #0x8\n" + "ldr d19, [x22], #0x8\n" + "ldr d18, [x21], #0x8\n" "tbz %x[width], #0, 5f\n" - "ld1 { v26.s }[2], [x28]\n" - "ld1 { v21.s }[2], [x27]\n" + "ld1 { v25.s }[2], [x28]\n" + "ld1 { v24.s }[2], [x27]\n" "mov x20, #0x2\n" - "ld1 { v25.s }[2], [x26]\n" - "ld1 { v24.s }[2], [x25]\n" - "ld1 { v23.s }[2], [x24]\n" - "ld1 { v22.s }[2], [x23]\n" - "ld1 { v20.s }[2], [x22]\n" - "ld1 { v19.s }[2], [x21]\n" + "ld1 { v23.s }[2], [x26]\n" + "ld1 { v22.s }[2], [x25]\n" + "ld1 { v21.s }[2], [x24]\n" + "ld1 { v20.s }[2], [x23]\n" + "ld1 { v19.s }[2], [x22]\n" + "ld1 { v18.s }[2], [x21]\n" "b 5f\n" "4:" // odd_loads_1_0 - "ldr s26, [x28, #0x0]\n" - "ldr s21, [x27, #0x0]\n" + "ldr s25, [x28, #0x0]\n" + "ldr s24, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr s25, [x26, #0x0]\n" - "ldr s24, [x25, #0x0]\n" - "ldr s23, [x24, #0x0]\n" - "ldr s22, [x23, #0x0]\n" - "ldr s20, [x22, #0x0]\n" - "ldr s19, [x21, #0x0]\n" + "ldr s23, [x26, #0x0]\n" + "ldr s22, [x25, #0x0]\n" + "ldr s21, [x24, #0x0]\n" + "ldr s20, [x23, #0x0]\n" + "ldr s19, [x22, #0x0]\n" + "ldr s18, [x21, #0x0]\n" "5:" // Odd load end "subs x20, x20, #0x1\n" - "zip1 v16.2d, v26.2d, v21.2d\n" + "zip1 v16.2d, v25.2d, v24.2d\n" "str q16, [%x[out_ptr], #0x0]\n" - "zip1 v18.2d, v25.2d, v24.2d\n" - "str q18, [%x[out_ptr], #0x10]\n" - "zip1 v17.2d, v23.2d, v22.2d\n" - "zip1 v16.2d, v20.2d, v19.2d\n" + "zip1 v16.2d, v23.2d, v22.2d\n" + "str q16, [%x[out_ptr], #0x10]\n" + "zip1 v17.2d, v21.2d, v20.2d\n" + "zip1 v16.2d, v19.2d, v18.2d\n" "str q17, [%x[out_ptr], #0x20]\n" "str q16, [%x[out_ptr], #0x30]\n" "add %x[out_ptr], %x[out_ptr], #0x40\n" "beq 6f\n" - "zip2 v21.2d, v26.2d, v21.2d\n" - "str q21, [%x[out_ptr], #0x0]\n" - "zip2 v18.2d, v25.2d, v24.2d\n" - "str q18, [%x[out_ptr], #0x10]\n" - "zip2 v17.2d, v23.2d, v22.2d\n" - "zip2 v16.2d, v20.2d, v19.2d\n" + "zip2 v16.2d, v25.2d, v24.2d\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.2d, v23.2d, v22.2d\n" + "str q16, [%x[out_ptr], #0x10]\n" + "zip2 v17.2d, v21.2d, v20.2d\n" + "zip2 v16.2d, v19.2d, v18.2d\n" "str q17, [%x[out_ptr], #0x20]\n" "str q16, [%x[out_ptr], #0x30]\n" "add %x[out_ptr], %x[out_ptr], #0x40\n" "6:" // Odds skip - : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) - : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_bf16_bf16.hpp index d606d5a5b66b046997ef533abce80e1f059583cc..e9799f87a96c1bc4601e994e18a463fb7619c22e 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_bf16_bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_bf16_bf16.hpp @@ -79,18 +79,18 @@ void interleave_block<8, 4, VLType::None, false>( "prfm pldl1keep, [x21, #0x40]\n" "blt 3f\n" "2:" // Main loop head - "ldr q26, [x28], #0x10\n" - "ldr q21, [x27], #0x10\n" + "ldr q20, [x28], #0x10\n" + "ldr q19, [x27], #0x10\n" "subs %x[width], %x[width], #0x8\n" "cmp %x[width], #0x8\n" "ldr q25, [x26], #0x10\n" "ldr q24, [x25], #0x10\n" - "zip1 v16.2d, v26.2d, v21.2d\n" + "zip1 v16.2d, v20.2d, v19.2d\n" "zip1 v18.2d, v25.2d, v24.2d\n" "ldr q23, [x24], #0x10\n" "ldr q22, [x23], #0x10\n" "zip1 v17.2d, v23.2d, v22.2d\n" - "zip2 v21.2d, v26.2d, v21.2d\n" + "zip2 v21.2d, v20.2d, v19.2d\n" "ldr q20, [x22], #0x10\n" "ldr q19, [x21], #0x10\n" "str q16, [%x[out_ptr], #0x0]\n" @@ -118,104 +118,103 @@ void interleave_block<8, 4, VLType::None, false>( "3:" // Main loop skip "cbz %x[width], 8f\n" "tbz %x[width], #2, 5f\n" - "ldr d26, [x28], #0x8\n" - "ldr d21, [x27], #0x8\n" - "ldr d25, [x26], #0x8\n" - "ldr d24, [x25], #0x8\n" - "ldr d23, [x24], #0x8\n" - "ldr d22, [x23], #0x8\n" - "ldr d20, [x22], #0x8\n" - "ldr d19, [x21], #0x8\n" + "ldr d25, [x28], #0x8\n" + "ldr d24, [x27], #0x8\n" + "ldr d23, [x26], #0x8\n" + "ldr d22, [x25], #0x8\n" + "ldr d21, [x24], #0x8\n" + "ldr d20, [x23], #0x8\n" + "ldr d19, [x22], #0x8\n" + "ldr d18, [x21], #0x8\n" "tbz %x[width], #1, 4f\n" - "ld1 { v26.s }[2], [x28], #0x4\n" - "ld1 { v21.s }[2], [x27], #0x4\n" + "ld1 { v25.s }[2], [x28], #0x4\n" + "ld1 { v24.s }[2], [x27], #0x4\n" "mov x20, #0x2\n" - "ld1 { v25.s }[2], [x26], #0x4\n" - "ld1 { v24.s }[2], [x25], #0x4\n" - "ld1 { v23.s }[2], [x24], #0x4\n" - "ld1 { v22.s }[2], [x23], #0x4\n" - "ld1 { v20.s }[2], [x22], #0x4\n" - "ld1 { v19.s }[2], [x21], #0x4\n" + "ld1 { v23.s }[2], [x26], #0x4\n" + "ld1 { v22.s }[2], [x25], #0x4\n" + "ld1 { v21.s }[2], [x24], #0x4\n" + "ld1 { v20.s }[2], [x23], #0x4\n" + "ld1 { v19.s }[2], [x22], #0x4\n" + "ld1 { v18.s }[2], [x21], #0x4\n" "tbz %x[width], #0, 7f\n" - "ld1 { v26.h }[6], [x28]\n" - "ld1 { v21.h }[6], [x27]\n" - "ld1 { v25.h }[6], [x26]\n" - "ld1 { v24.h }[6], [x25]\n" - "ld1 { v23.h }[6], [x24]\n" - "ld1 { v22.h }[6], [x23]\n" - "ld1 { v20.h }[6], [x22]\n" - "ld1 { v19.h }[6], [x21]\n" + "ld1 { v25.h }[6], [x28]\n" + "ld1 { v24.h }[6], [x27]\n" + "ld1 { v23.h }[6], [x26]\n" + "ld1 { v22.h }[6], [x25]\n" + "ld1 { v21.h }[6], [x24]\n" + "ld1 { v20.h }[6], [x23]\n" + "ld1 { v19.h }[6], [x22]\n" + "ld1 { v18.h }[6], [x21]\n" "b 7f\n" "4:" // odd_loads_1_4 "mov x20, #0x1\n" "tbz %x[width], #0, 7f\n" - "ld1 { v26.h }[4], [x28]\n" - "ld1 { v21.h }[4], [x27]\n" + "ld1 { v25.h }[4], [x28]\n" + "ld1 { v24.h }[4], [x27]\n" "mov x20, #0x2\n" - "ld1 { v25.h }[4], [x26]\n" - "ld1 { v24.h }[4], [x25]\n" - "ld1 { v23.h }[4], [x24]\n" - "ld1 { v22.h }[4], [x23]\n" - "ld1 { v20.h }[4], [x22]\n" - "ld1 { v19.h }[4], [x21]\n" + "ld1 { v23.h }[4], [x26]\n" + "ld1 { v22.h }[4], [x25]\n" + "ld1 { v21.h }[4], [x24]\n" + "ld1 { v20.h }[4], [x23]\n" + "ld1 { v19.h }[4], [x22]\n" + "ld1 { v18.h }[4], [x21]\n" "b 7f\n" "5:" // odd_loads_2_0 "tbz %x[width], #1, 6f\n" - "ldr s26, [x28], #0x4\n" - "ldr s21, [x27], #0x4\n" + "ldr s25, [x28], #0x4\n" + "ldr s24, [x27], #0x4\n" "mov x20, #0x1\n" - "ldr s25, [x26], #0x4\n" - "ldr s24, [x25], #0x4\n" - "ldr s23, [x24], #0x4\n" - "ldr s22, [x23], #0x4\n" - "ldr s20, [x22], #0x4\n" - "ldr s19, [x21], #0x4\n" + "ldr s23, [x26], #0x4\n" + "ldr s22, [x25], #0x4\n" + "ldr s21, [x24], #0x4\n" + "ldr s20, [x23], #0x4\n" + "ldr s19, [x22], #0x4\n" + "ldr s18, [x21], #0x4\n" "tbz %x[width], #0, 7f\n" - "ld1 { v26.h }[2], [x28]\n" - "ld1 { v21.h }[2], [x27]\n" - "ld1 { v25.h }[2], [x26]\n" - "ld1 { v24.h }[2], [x25]\n" - "ld1 { v23.h }[2], [x24]\n" - "ld1 { v22.h }[2], [x23]\n" - "ld1 { v20.h }[2], [x22]\n" - "ld1 { v19.h }[2], [x21]\n" + "ld1 { v25.h }[2], [x28]\n" + "ld1 { v24.h }[2], [x27]\n" + "ld1 { v23.h }[2], [x26]\n" + "ld1 { v22.h }[2], [x25]\n" + "ld1 { v21.h }[2], [x24]\n" + "ld1 { v20.h }[2], [x23]\n" + "ld1 { v19.h }[2], [x22]\n" + "ld1 { v18.h }[2], [x21]\n" "b 7f\n" "6:" // odd_loads_1_0 - "ldr h26, [x28, #0x0]\n" - "ldr h21, [x27, #0x0]\n" + "ldr h25, [x28, #0x0]\n" + "ldr h24, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr h25, [x26, #0x0]\n" - "ldr h24, [x25, #0x0]\n" - "ldr h23, [x24, #0x0]\n" - "ldr h22, [x23, #0x0]\n" - "ldr h20, [x22, #0x0]\n" - "ldr h19, [x21, #0x0]\n" + "ldr h23, [x26, #0x0]\n" + "ldr h22, [x25, #0x0]\n" + "ldr h21, [x24, #0x0]\n" + "ldr h20, [x23, #0x0]\n" + "ldr h19, [x22, #0x0]\n" + "ldr h18, [x21, #0x0]\n" "7:" // Odd load end "subs x20, x20, #0x1\n" - "zip1 v16.2d, v26.2d, v21.2d\n" + "zip1 v16.2d, v25.2d, v24.2d\n" "str q16, [%x[out_ptr], #0x0]\n" - "zip1 v18.2d, v25.2d, v24.2d\n" - "str q18, [%x[out_ptr], #0x10]\n" - "zip1 v17.2d, v23.2d, v22.2d\n" - "zip1 v16.2d, v20.2d, v19.2d\n" + "zip1 v16.2d, v23.2d, v22.2d\n" + "str q16, [%x[out_ptr], #0x10]\n" + "zip1 v17.2d, v21.2d, v20.2d\n" + "zip1 v16.2d, v19.2d, v18.2d\n" "str q17, [%x[out_ptr], #0x20]\n" "str q16, [%x[out_ptr], #0x30]\n" "add %x[out_ptr], %x[out_ptr], #0x40\n" "beq 8f\n" - "zip2 v21.2d, v26.2d, v21.2d\n" - "str q21, [%x[out_ptr], #0x0]\n" - "zip2 v18.2d, v25.2d, v24.2d\n" - "str q18, [%x[out_ptr], #0x10]\n" - "zip2 v17.2d, v23.2d, v22.2d\n" - "zip2 v16.2d, v20.2d, v19.2d\n" + "zip2 v16.2d, v25.2d, v24.2d\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.2d, v23.2d, v22.2d\n" + "str q16, [%x[out_ptr], #0x10]\n" + "zip2 v17.2d, v21.2d, v20.2d\n" + "zip2 v16.2d, v19.2d, v18.2d\n" "str q17, [%x[out_ptr], #0x20]\n" "str q16, [%x[out_ptr], #0x30]\n" "add %x[out_ptr], %x[out_ptr], #0x40\n" "8:" // Odds skip - : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) - : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_fp32_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_fp32_bf16.hpp index dfec14358b3935b478eeed017a33a67672e17c9a..730bfd6342855e3bf82b6dfa78fc1582201e0345 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_fp32_bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_fp32_bf16.hpp @@ -79,14 +79,14 @@ void interleave_block<8, 4, VLType::None, false>( "prfm pldl1keep, [x21, #0x40]\n" "blt 3f\n" "2:" // Main loop head - "ldr q23, [x28], #0x10\n" - "ldr q22, [x26], #0x10\n" - ".inst 0x0ea16af7 // bfcvtn v23.4h, v23.4s\n" - ".inst 0x0ea16ad6 // bfcvtn v22.4h, v22.4s\n" - "ldr q21, [x24], #0x10\n" - "ldr q20, [x22], #0x10\n" - ".inst 0x0ea16ab5 // bfcvtn v21.4h, v21.4s\n" - ".inst 0x0ea16a94 // bfcvtn v20.4h, v20.4s\n" + "ldr q17, [x28], #0x10\n" + "ldr q16, [x26], #0x10\n" + ".inst 0x0ea16a37 // bfcvtn v23.4h, v17.4s\n" + ".inst 0x0ea16a16 // bfcvtn v22.4h, v16.4s\n" + "ldr q17, [x24], #0x10\n" + "ldr q16, [x22], #0x10\n" + ".inst 0x0ea16a35 // bfcvtn v21.4h, v17.4s\n" + ".inst 0x0ea16a14 // bfcvtn v20.4h, v16.4s\n" "ldr q19, [x27], #0x10\n" "ldr q18, [x25], #0x10\n" "subs %x[width], %x[width], #0x4\n" @@ -114,51 +114,50 @@ void interleave_block<8, 4, VLType::None, false>( "3:" // Main loop skip "cbz %x[width], 6f\n" "tbz %x[width], #1, 4f\n" - "ldr d23, [x28], #0x8\n" - "ldr d19, [x27], #0x8\n" + "ldr d19, [x28], #0x8\n" + "ldr d23, [x27], #0x8\n" "mov x20, #0x1\n" - "ldr d22, [x26], #0x8\n" - "ldr d18, [x25], #0x8\n" - "ldr d21, [x24], #0x8\n" - "ldr d17, [x23], #0x8\n" - "ldr d20, [x22], #0x8\n" - "ldr d16, [x21], #0x8\n" + "ldr d18, [x26], #0x8\n" + "ldr d22, [x25], #0x8\n" + "ldr d17, [x24], #0x8\n" + "ldr d21, [x23], #0x8\n" + "ldr d16, [x22], #0x8\n" + "ldr d20, [x21], #0x8\n" "tbz %x[width], #0, 5f\n" - "ld1 { v23.s }[2], [x28]\n" - "ld1 { v19.s }[2], [x27]\n" - "ld1 { v22.s }[2], [x26]\n" - "ld1 { v18.s }[2], [x25]\n" - "ld1 { v21.s }[2], [x24]\n" - "ld1 { v17.s }[2], [x23]\n" - "ld1 { v20.s }[2], [x22]\n" - "ld1 { v16.s }[2], [x21]\n" + "ld1 { v19.s }[2], [x28]\n" + "ld1 { v23.s }[2], [x27]\n" + "ld1 { v18.s }[2], [x26]\n" + "ld1 { v22.s }[2], [x25]\n" + "ld1 { v17.s }[2], [x24]\n" + "ld1 { v21.s }[2], [x23]\n" + "ld1 { v16.s }[2], [x22]\n" + "ld1 { v20.s }[2], [x21]\n" "b 5f\n" "4:" // odd_loads_1_0 - "ldr s23, [x28, #0x0]\n" - "ldr s19, [x27, #0x0]\n" + "ldr s19, [x28, #0x0]\n" + "ldr s23, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr s22, [x26, #0x0]\n" - "ldr s18, [x25, #0x0]\n" - "ldr s21, [x24, #0x0]\n" - "ldr s17, [x23, #0x0]\n" - "ldr s20, [x22, #0x0]\n" - "ldr s16, [x21, #0x0]\n" + "ldr s18, [x26, #0x0]\n" + "ldr s22, [x25, #0x0]\n" + "ldr s17, [x24, #0x0]\n" + "ldr s21, [x23, #0x0]\n" + "ldr s16, [x22, #0x0]\n" + "ldr s20, [x21, #0x0]\n" "5:" // Odd load end - ".inst 0x0ea16af7 // bfcvtn v23.4h, v23.4s\n" - ".inst 0x0ea16ad6 // bfcvtn v22.4h, v22.4s\n" - ".inst 0x0ea16ab5 // bfcvtn v21.4h, v21.4s\n" - ".inst 0x0ea16a94 // bfcvtn v20.4h, v20.4s\n" - ".inst 0x4ea16a77 // bfcvtn2 v23.8h, v19.4s\n" - ".inst 0x4ea16a56 // bfcvtn2 v22.8h, v18.4s\n" - "str q23, [%x[out_ptr], #0x0]\n" - ".inst 0x4ea16a35 // bfcvtn2 v21.8h, v17.4s\n" - ".inst 0x4ea16a14 // bfcvtn2 v20.8h, v16.4s\n" - "str q22, [%x[out_ptr], #0x10]\n" - "str q21, [%x[out_ptr], #0x20]\n" - "str q20, [%x[out_ptr], #0x30]\n" + ".inst 0x0ea16a73 // bfcvtn v19.4h, v19.4s\n" + ".inst 0x0ea16a52 // bfcvtn v18.4h, v18.4s\n" + ".inst 0x0ea16a31 // bfcvtn v17.4h, v17.4s\n" + ".inst 0x0ea16a10 // bfcvtn v16.4h, v16.4s\n" + ".inst 0x4ea16af3 // bfcvtn2 v19.8h, v23.4s\n" + ".inst 0x4ea16ad2 // bfcvtn2 v18.8h, v22.4s\n" + "str q19, [%x[out_ptr], #0x0]\n" + ".inst 0x4ea16ab1 // bfcvtn2 v17.8h, v21.4s\n" + ".inst 0x4ea16a90 // bfcvtn2 v16.8h, v20.4s\n" + "str q18, [%x[out_ptr], #0x10]\n" + "str q17, [%x[out_ptr], #0x20]\n" + "str q16, [%x[out_ptr], #0x30]\n" "add %x[out_ptr], %x[out_ptr], #0x40\n" "6:" // Odds skip - : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_s8_s8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_s8_s8.hpp index 54f15f8a5c791d68d29728cc4373160981d721c5..15d8ddbe5341b981e0c6d302c83037a7463ff389 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_s8_s8.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_s8_s8.hpp @@ -79,29 +79,29 @@ void interleave_block<8, 4, VLType::None, false>( "prfm pldl1keep, [x21, #0x40]\n" "blt 3f\n" "2:" // Main loop head - "ldr q28, [x28], #0x10\n" - "ldr q27, [x27], #0x10\n" + "ldr q20, [x28], #0x10\n" + "ldr q18, [x27], #0x10\n" "subs %x[width], %x[width], #0x10\n" "cmp %x[width], #0x10\n" - "ldr q22, [x26], #0x10\n" - "ldr q21, [x25], #0x10\n" - "zip1 v26.4s, v28.4s, v22.4s\n" - "zip1 v25.4s, v27.4s, v21.4s\n" - "ldr q24, [x24], #0x10\n" + "ldr q17, [x26], #0x10\n" + "ldr q16, [x25], #0x10\n" + "zip1 v25.4s, v20.4s, v17.4s\n" + "zip1 v24.4s, v18.4s, v16.4s\n" + "ldr q19, [x24], #0x10\n" "ldr q23, [x23], #0x10\n" - "zip2 v22.4s, v28.4s, v22.4s\n" - "zip2 v21.4s, v27.4s, v21.4s\n" - "ldr q19, [x22], #0x10\n" - "ldr q18, [x21], #0x10\n" - "zip1 v20.4s, v24.4s, v19.4s\n" - "zip1 v17.4s, v23.4s, v18.4s\n" - "zip2 v19.4s, v24.4s, v19.4s\n" - "zip2 v18.4s, v23.4s, v18.4s\n" + "zip2 v22.4s, v20.4s, v17.4s\n" + "zip2 v21.4s, v18.4s, v16.4s\n" + "ldr q18, [x22], #0x10\n" + "ldr q16, [x21], #0x10\n" + "zip1 v20.4s, v19.4s, v18.4s\n" + "zip1 v17.4s, v23.4s, v16.4s\n" + "zip2 v19.4s, v19.4s, v18.4s\n" + "zip2 v18.4s, v23.4s, v16.4s\n" "prfm pldl1keep, [x28, #0x70]\n" "prfm pldl1keep, [x27, #0x70]\n" "prfm pldl1keep, [x26, #0x70]\n" "prfm pldl1keep, [x25, #0x70]\n" - "zip1 v16.4s, v26.4s, v25.4s\n" + "zip1 v16.4s, v25.4s, v24.4s\n" "str q16, [%x[out_ptr], #0x0]\n" "prfm pldl1keep, [x24, #0x70]\n" "prfm pldl1keep, [x23, #0x70]\n" @@ -109,7 +109,7 @@ void interleave_block<8, 4, VLType::None, false>( "str q16, [%x[out_ptr], #0x10]\n" "prfm pldl1keep, [x22, #0x70]\n" "prfm pldl1keep, [x21, #0x70]\n" - "zip2 v16.4s, v26.4s, v25.4s\n" + "zip2 v16.4s, v25.4s, v24.4s\n" "str q16, [%x[out_ptr], #0x20]\n" "zip2 v16.4s, v20.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x30]\n" @@ -128,40 +128,40 @@ void interleave_block<8, 4, VLType::None, false>( "tbz %x[width], #3, 7f\n" "ldr d28, [x28], #0x8\n" "ldr d27, [x27], #0x8\n" - "ldr d22, [x26], #0x8\n" - "ldr d21, [x25], #0x8\n" + "ldr d26, [x26], #0x8\n" + "ldr d25, [x25], #0x8\n" "ldr d24, [x24], #0x8\n" "ldr d23, [x23], #0x8\n" - "ldr d19, [x22], #0x8\n" - "ldr d18, [x21], #0x8\n" + "ldr d22, [x22], #0x8\n" + "ldr d21, [x21], #0x8\n" "tbz %x[width], #2, 5f\n" "ld1 { v28.s }[2], [x28], #0x4\n" "ld1 { v27.s }[2], [x27], #0x4\n" - "ld1 { v22.s }[2], [x26], #0x4\n" - "ld1 { v21.s }[2], [x25], #0x4\n" + "ld1 { v26.s }[2], [x26], #0x4\n" + "ld1 { v25.s }[2], [x25], #0x4\n" "ld1 { v24.s }[2], [x24], #0x4\n" "ld1 { v23.s }[2], [x23], #0x4\n" - "ld1 { v19.s }[2], [x22], #0x4\n" - "ld1 { v18.s }[2], [x21], #0x4\n" + "ld1 { v22.s }[2], [x22], #0x4\n" + "ld1 { v21.s }[2], [x21], #0x4\n" "tbz %x[width], #1, 4f\n" "ld1 { v28.h }[6], [x28], #0x2\n" "ld1 { v27.h }[6], [x27], #0x2\n" "mov x20, #0x4\n" - "ld1 { v22.h }[6], [x26], #0x2\n" - "ld1 { v21.h }[6], [x25], #0x2\n" + "ld1 { v26.h }[6], [x26], #0x2\n" + "ld1 { v25.h }[6], [x25], #0x2\n" "ld1 { v24.h }[6], [x24], #0x2\n" "ld1 { v23.h }[6], [x23], #0x2\n" - "ld1 { v19.h }[6], [x22], #0x2\n" - "ld1 { v18.h }[6], [x21], #0x2\n" + "ld1 { v22.h }[6], [x22], #0x2\n" + "ld1 { v21.h }[6], [x21], #0x2\n" "tbz %x[width], #0, 11f\n" "ld1 { v28.b }[14], [x28]\n" "ld1 { v27.b }[14], [x27]\n" - "ld1 { v22.b }[14], [x26]\n" - "ld1 { v21.b }[14], [x25]\n" + "ld1 { v26.b }[14], [x26]\n" + "ld1 { v25.b }[14], [x25]\n" "ld1 { v24.b }[14], [x24]\n" "ld1 { v23.b }[14], [x23]\n" - "ld1 { v19.b }[14], [x22]\n" - "ld1 { v18.b }[14], [x21]\n" + "ld1 { v22.b }[14], [x22]\n" + "ld1 { v21.b }[14], [x21]\n" "b 11f\n" "4:" // odd_loads_1_12 "mov x20, #0x3\n" @@ -169,33 +169,33 @@ void interleave_block<8, 4, VLType::None, false>( "ld1 { v28.b }[12], [x28]\n" "ld1 { v27.b }[12], [x27]\n" "mov x20, #0x4\n" - "ld1 { v22.b }[12], [x26]\n" - "ld1 { v21.b }[12], [x25]\n" + "ld1 { v26.b }[12], [x26]\n" + "ld1 { v25.b }[12], [x25]\n" "ld1 { v24.b }[12], [x24]\n" "ld1 { v23.b }[12], [x23]\n" - "ld1 { v19.b }[12], [x22]\n" - "ld1 { v18.b }[12], [x21]\n" + "ld1 { v22.b }[12], [x22]\n" + "ld1 { v21.b }[12], [x21]\n" "b 11f\n" "5:" // odd_loads_2_8 "tbz %x[width], #1, 6f\n" "ld1 { v28.h }[4], [x28], #0x2\n" "ld1 { v27.h }[4], [x27], #0x2\n" "mov x20, #0x3\n" - "ld1 { v22.h }[4], [x26], #0x2\n" - "ld1 { v21.h }[4], [x25], #0x2\n" + "ld1 { v26.h }[4], [x26], #0x2\n" + "ld1 { v25.h }[4], [x25], #0x2\n" "ld1 { v24.h }[4], [x24], #0x2\n" "ld1 { v23.h }[4], [x23], #0x2\n" - "ld1 { v19.h }[4], [x22], #0x2\n" - "ld1 { v18.h }[4], [x21], #0x2\n" + "ld1 { v22.h }[4], [x22], #0x2\n" + "ld1 { v21.h }[4], [x21], #0x2\n" "tbz %x[width], #0, 11f\n" "ld1 { v28.b }[10], [x28]\n" "ld1 { v27.b }[10], [x27]\n" - "ld1 { v22.b }[10], [x26]\n" - "ld1 { v21.b }[10], [x25]\n" + "ld1 { v26.b }[10], [x26]\n" + "ld1 { v25.b }[10], [x25]\n" "ld1 { v24.b }[10], [x24]\n" "ld1 { v23.b }[10], [x23]\n" - "ld1 { v19.b }[10], [x22]\n" - "ld1 { v18.b }[10], [x21]\n" + "ld1 { v22.b }[10], [x22]\n" + "ld1 { v21.b }[10], [x21]\n" "b 11f\n" "6:" // odd_loads_1_8 "mov x20, #0x2\n" @@ -203,42 +203,42 @@ void interleave_block<8, 4, VLType::None, false>( "ld1 { v28.b }[8], [x28]\n" "ld1 { v27.b }[8], [x27]\n" "mov x20, #0x3\n" - "ld1 { v22.b }[8], [x26]\n" - "ld1 { v21.b }[8], [x25]\n" + "ld1 { v26.b }[8], [x26]\n" + "ld1 { v25.b }[8], [x25]\n" "ld1 { v24.b }[8], [x24]\n" "ld1 { v23.b }[8], [x23]\n" - "ld1 { v19.b }[8], [x22]\n" - "ld1 { v18.b }[8], [x21]\n" + "ld1 { v22.b }[8], [x22]\n" + "ld1 { v21.b }[8], [x21]\n" "b 11f\n" "7:" // odd_loads_4_0 "tbz %x[width], #2, 9f\n" "ldr s28, [x28], #0x4\n" "ldr s27, [x27], #0x4\n" - "ldr s22, [x26], #0x4\n" - "ldr s21, [x25], #0x4\n" + "ldr s26, [x26], #0x4\n" + "ldr s25, [x25], #0x4\n" "ldr s24, [x24], #0x4\n" "ldr s23, [x23], #0x4\n" - "ldr s19, [x22], #0x4\n" - "ldr s18, [x21], #0x4\n" + "ldr s22, [x22], #0x4\n" + "ldr s21, [x21], #0x4\n" "tbz %x[width], #1, 8f\n" "ld1 { v28.h }[2], [x28], #0x2\n" "ld1 { v27.h }[2], [x27], #0x2\n" "mov x20, #0x2\n" - "ld1 { v22.h }[2], [x26], #0x2\n" - "ld1 { v21.h }[2], [x25], #0x2\n" + "ld1 { v26.h }[2], [x26], #0x2\n" + "ld1 { v25.h }[2], [x25], #0x2\n" "ld1 { v24.h }[2], [x24], #0x2\n" "ld1 { v23.h }[2], [x23], #0x2\n" - "ld1 { v19.h }[2], [x22], #0x2\n" - "ld1 { v18.h }[2], [x21], #0x2\n" + "ld1 { v22.h }[2], [x22], #0x2\n" + "ld1 { v21.h }[2], [x21], #0x2\n" "tbz %x[width], #0, 11f\n" "ld1 { v28.b }[6], [x28]\n" "ld1 { v27.b }[6], [x27]\n" - "ld1 { v22.b }[6], [x26]\n" - "ld1 { v21.b }[6], [x25]\n" + "ld1 { v26.b }[6], [x26]\n" + "ld1 { v25.b }[6], [x25]\n" "ld1 { v24.b }[6], [x24]\n" "ld1 { v23.b }[6], [x23]\n" - "ld1 { v19.b }[6], [x22]\n" - "ld1 { v18.b }[6], [x21]\n" + "ld1 { v22.b }[6], [x22]\n" + "ld1 { v21.b }[6], [x21]\n" "b 11f\n" "8:" // odd_loads_1_4 "mov x20, #0x1\n" @@ -246,81 +246,80 @@ void interleave_block<8, 4, VLType::None, false>( "ld1 { v28.b }[4], [x28]\n" "ld1 { v27.b }[4], [x27]\n" "mov x20, #0x2\n" - "ld1 { v22.b }[4], [x26]\n" - "ld1 { v21.b }[4], [x25]\n" + "ld1 { v26.b }[4], [x26]\n" + "ld1 { v25.b }[4], [x25]\n" "ld1 { v24.b }[4], [x24]\n" "ld1 { v23.b }[4], [x23]\n" - "ld1 { v19.b }[4], [x22]\n" - "ld1 { v18.b }[4], [x21]\n" + "ld1 { v22.b }[4], [x22]\n" + "ld1 { v21.b }[4], [x21]\n" "b 11f\n" "9:" // odd_loads_2_0 "tbz %x[width], #1, 10f\n" "ldr h28, [x28], #0x2\n" "ldr h27, [x27], #0x2\n" "mov x20, #0x1\n" - "ldr h22, [x26], #0x2\n" - "ldr h21, [x25], #0x2\n" + "ldr h26, [x26], #0x2\n" + "ldr h25, [x25], #0x2\n" "ldr h24, [x24], #0x2\n" "ldr h23, [x23], #0x2\n" - "ldr h19, [x22], #0x2\n" - "ldr h18, [x21], #0x2\n" + "ldr h22, [x22], #0x2\n" + "ldr h21, [x21], #0x2\n" "tbz %x[width], #0, 11f\n" "ld1 { v28.b }[2], [x28]\n" "ld1 { v27.b }[2], [x27]\n" - "ld1 { v22.b }[2], [x26]\n" - "ld1 { v21.b }[2], [x25]\n" + "ld1 { v26.b }[2], [x26]\n" + "ld1 { v25.b }[2], [x25]\n" "ld1 { v24.b }[2], [x24]\n" "ld1 { v23.b }[2], [x23]\n" - "ld1 { v19.b }[2], [x22]\n" - "ld1 { v18.b }[2], [x21]\n" + "ld1 { v22.b }[2], [x22]\n" + "ld1 { v21.b }[2], [x21]\n" "b 11f\n" "10:" // odd_loads_1_0 "ldr b28, [x28, #0x0]\n" "ldr b27, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr b22, [x26, #0x0]\n" - "ldr b21, [x25, #0x0]\n" + "ldr b26, [x26, #0x0]\n" + "ldr b25, [x25, #0x0]\n" "ldr b24, [x24, #0x0]\n" "ldr b23, [x23, #0x0]\n" - "ldr b19, [x22, #0x0]\n" - "ldr b18, [x21, #0x0]\n" + "ldr b22, [x22, #0x0]\n" + "ldr b21, [x21, #0x0]\n" "11:" // Odd load end - "zip1 v26.4s, v28.4s, v22.4s\n" - "zip1 v25.4s, v27.4s, v21.4s\n" + "zip1 v20.4s, v28.4s, v26.4s\n" + "zip1 v19.4s, v27.4s, v25.4s\n" "subs x20, x20, #0x1\n" - "zip1 v20.4s, v24.4s, v19.4s\n" - "zip1 v17.4s, v23.4s, v18.4s\n" - "zip1 v16.4s, v26.4s, v25.4s\n" + "zip1 v18.4s, v24.4s, v22.4s\n" + "zip1 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v20.4s, v19.4s\n" "str q16, [%x[out_ptr], #0x0]\n" - "zip1 v16.4s, v20.4s, v17.4s\n" + "zip1 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 12f\n" "subs x20, x20, #0x1\n" - "zip2 v16.4s, v26.4s, v25.4s\n" + "zip2 v16.4s, v20.4s, v19.4s\n" "str q16, [%x[out_ptr], #0x0]\n" - "zip2 v16.4s, v20.4s, v17.4s\n" + "zip2 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 12f\n" - "zip2 v22.4s, v28.4s, v22.4s\n" - "zip2 v21.4s, v27.4s, v21.4s\n" + "zip2 v20.4s, v28.4s, v26.4s\n" + "zip2 v19.4s, v27.4s, v25.4s\n" "subs x20, x20, #0x1\n" - "zip2 v19.4s, v24.4s, v19.4s\n" - "zip2 v18.4s, v23.4s, v18.4s\n" - "zip1 v16.4s, v22.4s, v21.4s\n" + "zip2 v18.4s, v24.4s, v22.4s\n" + "zip2 v17.4s, v23.4s, v21.4s\n" + "zip1 v16.4s, v20.4s, v19.4s\n" "str q16, [%x[out_ptr], #0x0]\n" - "zip1 v16.4s, v19.4s, v18.4s\n" + "zip1 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 12f\n" - "zip2 v17.4s, v22.4s, v21.4s\n" - "str q17, [%x[out_ptr], #0x0]\n" - "zip2 v16.4s, v19.4s, v18.4s\n" + "zip2 v16.4s, v20.4s, v19.4s\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.4s, v18.4s, v17.4s\n" "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "12:" // Odds skip - : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_s8_s8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_s8_s8_summing.hpp index 2db54126c040607b633ab6bd06bd25644c48a6d4..6c41b5fdfbe8a4e8fd789e627d31e12d5edae5d4 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_s8_s8_summing.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_s8_s8_summing.hpp @@ -153,202 +153,202 @@ void interleave_block<8, 4, VLType::None, true>( "5:" // Main loop skip "cbz %x[width], 14f\n" "tbz %x[width], #3, 9f\n" - "ldr d30, [x28], #0x8\n" - "ldr d29, [x27], #0x8\n" - "ldr d28, [x26], #0x8\n" - "ldr d27, [x25], #0x8\n" - "ldr d20, [x24], #0x8\n" - "ldr d26, [x23], #0x8\n" - "ldr d19, [x22], #0x8\n" - "ldr d18, [x21], #0x8\n" + "ldr d29, [x28], #0x8\n" + "ldr d28, [x27], #0x8\n" + "ldr d27, [x26], #0x8\n" + "ldr d26, [x25], #0x8\n" + "ldr d25, [x24], #0x8\n" + "ldr d24, [x23], #0x8\n" + "ldr d23, [x22], #0x8\n" + "ldr d22, [x21], #0x8\n" "tbz %x[width], #2, 7f\n" - "ld1 { v30.s }[2], [x28], #0x4\n" - "ld1 { v29.s }[2], [x27], #0x4\n" - "ld1 { v28.s }[2], [x26], #0x4\n" - "ld1 { v27.s }[2], [x25], #0x4\n" - "ld1 { v20.s }[2], [x24], #0x4\n" - "ld1 { v26.s }[2], [x23], #0x4\n" - "ld1 { v19.s }[2], [x22], #0x4\n" - "ld1 { v18.s }[2], [x21], #0x4\n" + "ld1 { v29.s }[2], [x28], #0x4\n" + "ld1 { v28.s }[2], [x27], #0x4\n" + "ld1 { v27.s }[2], [x26], #0x4\n" + "ld1 { v26.s }[2], [x25], #0x4\n" + "ld1 { v25.s }[2], [x24], #0x4\n" + "ld1 { v24.s }[2], [x23], #0x4\n" + "ld1 { v23.s }[2], [x22], #0x4\n" + "ld1 { v22.s }[2], [x21], #0x4\n" "tbz %x[width], #1, 6f\n" - "ld1 { v30.h }[6], [x28], #0x2\n" - "ld1 { v29.h }[6], [x27], #0x2\n" + "ld1 { v29.h }[6], [x28], #0x2\n" + "ld1 { v28.h }[6], [x27], #0x2\n" "mov x20, #0x4\n" - "ld1 { v28.h }[6], [x26], #0x2\n" - "ld1 { v27.h }[6], [x25], #0x2\n" - "ld1 { v20.h }[6], [x24], #0x2\n" - "ld1 { v26.h }[6], [x23], #0x2\n" - "ld1 { v19.h }[6], [x22], #0x2\n" - "ld1 { v18.h }[6], [x21], #0x2\n" + "ld1 { v27.h }[6], [x26], #0x2\n" + "ld1 { v26.h }[6], [x25], #0x2\n" + "ld1 { v25.h }[6], [x24], #0x2\n" + "ld1 { v24.h }[6], [x23], #0x2\n" + "ld1 { v23.h }[6], [x22], #0x2\n" + "ld1 { v22.h }[6], [x21], #0x2\n" "tbz %x[width], #0, 13f\n" - "ld1 { v30.b }[14], [x28]\n" - "ld1 { v29.b }[14], [x27]\n" - "ld1 { v28.b }[14], [x26]\n" - "ld1 { v27.b }[14], [x25]\n" - "ld1 { v20.b }[14], [x24]\n" - "ld1 { v26.b }[14], [x23]\n" - "ld1 { v19.b }[14], [x22]\n" - "ld1 { v18.b }[14], [x21]\n" + "ld1 { v29.b }[14], [x28]\n" + "ld1 { v28.b }[14], [x27]\n" + "ld1 { v27.b }[14], [x26]\n" + "ld1 { v26.b }[14], [x25]\n" + "ld1 { v25.b }[14], [x24]\n" + "ld1 { v24.b }[14], [x23]\n" + "ld1 { v23.b }[14], [x22]\n" + "ld1 { v22.b }[14], [x21]\n" "b 13f\n" "6:" // odd_loads_1_12 "mov x20, #0x3\n" "tbz %x[width], #0, 13f\n" - "ld1 { v30.b }[12], [x28]\n" - "ld1 { v29.b }[12], [x27]\n" + "ld1 { v29.b }[12], [x28]\n" + "ld1 { v28.b }[12], [x27]\n" "mov x20, #0x4\n" - "ld1 { v28.b }[12], [x26]\n" - "ld1 { v27.b }[12], [x25]\n" - "ld1 { v20.b }[12], [x24]\n" - "ld1 { v26.b }[12], [x23]\n" - "ld1 { v19.b }[12], [x22]\n" - "ld1 { v18.b }[12], [x21]\n" + "ld1 { v27.b }[12], [x26]\n" + "ld1 { v26.b }[12], [x25]\n" + "ld1 { v25.b }[12], [x24]\n" + "ld1 { v24.b }[12], [x23]\n" + "ld1 { v23.b }[12], [x22]\n" + "ld1 { v22.b }[12], [x21]\n" "b 13f\n" "7:" // odd_loads_2_8 "tbz %x[width], #1, 8f\n" - "ld1 { v30.h }[4], [x28], #0x2\n" - "ld1 { v29.h }[4], [x27], #0x2\n" + "ld1 { v29.h }[4], [x28], #0x2\n" + "ld1 { v28.h }[4], [x27], #0x2\n" "mov x20, #0x3\n" - "ld1 { v28.h }[4], [x26], #0x2\n" - "ld1 { v27.h }[4], [x25], #0x2\n" - "ld1 { v20.h }[4], [x24], #0x2\n" - "ld1 { v26.h }[4], [x23], #0x2\n" - "ld1 { v19.h }[4], [x22], #0x2\n" - "ld1 { v18.h }[4], [x21], #0x2\n" + "ld1 { v27.h }[4], [x26], #0x2\n" + "ld1 { v26.h }[4], [x25], #0x2\n" + "ld1 { v25.h }[4], [x24], #0x2\n" + "ld1 { v24.h }[4], [x23], #0x2\n" + "ld1 { v23.h }[4], [x22], #0x2\n" + "ld1 { v22.h }[4], [x21], #0x2\n" "tbz %x[width], #0, 13f\n" - "ld1 { v30.b }[10], [x28]\n" - "ld1 { v29.b }[10], [x27]\n" - "ld1 { v28.b }[10], [x26]\n" - "ld1 { v27.b }[10], [x25]\n" - "ld1 { v20.b }[10], [x24]\n" - "ld1 { v26.b }[10], [x23]\n" - "ld1 { v19.b }[10], [x22]\n" - "ld1 { v18.b }[10], [x21]\n" + "ld1 { v29.b }[10], [x28]\n" + "ld1 { v28.b }[10], [x27]\n" + "ld1 { v27.b }[10], [x26]\n" + "ld1 { v26.b }[10], [x25]\n" + "ld1 { v25.b }[10], [x24]\n" + "ld1 { v24.b }[10], [x23]\n" + "ld1 { v23.b }[10], [x22]\n" + "ld1 { v22.b }[10], [x21]\n" "b 13f\n" "8:" // odd_loads_1_8 "mov x20, #0x2\n" "tbz %x[width], #0, 13f\n" - "ld1 { v30.b }[8], [x28]\n" - "ld1 { v29.b }[8], [x27]\n" + "ld1 { v29.b }[8], [x28]\n" + "ld1 { v28.b }[8], [x27]\n" "mov x20, #0x3\n" - "ld1 { v28.b }[8], [x26]\n" - "ld1 { v27.b }[8], [x25]\n" - "ld1 { v20.b }[8], [x24]\n" - "ld1 { v26.b }[8], [x23]\n" - "ld1 { v19.b }[8], [x22]\n" - "ld1 { v18.b }[8], [x21]\n" + "ld1 { v27.b }[8], [x26]\n" + "ld1 { v26.b }[8], [x25]\n" + "ld1 { v25.b }[8], [x24]\n" + "ld1 { v24.b }[8], [x23]\n" + "ld1 { v23.b }[8], [x22]\n" + "ld1 { v22.b }[8], [x21]\n" "b 13f\n" "9:" // odd_loads_4_0 "tbz %x[width], #2, 11f\n" - "ldr s30, [x28], #0x4\n" - "ldr s29, [x27], #0x4\n" - "ldr s28, [x26], #0x4\n" - "ldr s27, [x25], #0x4\n" - "ldr s20, [x24], #0x4\n" - "ldr s26, [x23], #0x4\n" - "ldr s19, [x22], #0x4\n" - "ldr s18, [x21], #0x4\n" + "ldr s29, [x28], #0x4\n" + "ldr s28, [x27], #0x4\n" + "ldr s27, [x26], #0x4\n" + "ldr s26, [x25], #0x4\n" + "ldr s25, [x24], #0x4\n" + "ldr s24, [x23], #0x4\n" + "ldr s23, [x22], #0x4\n" + "ldr s22, [x21], #0x4\n" "tbz %x[width], #1, 10f\n" - "ld1 { v30.h }[2], [x28], #0x2\n" - "ld1 { v29.h }[2], [x27], #0x2\n" + "ld1 { v29.h }[2], [x28], #0x2\n" + "ld1 { v28.h }[2], [x27], #0x2\n" "mov x20, #0x2\n" - "ld1 { v28.h }[2], [x26], #0x2\n" - "ld1 { v27.h }[2], [x25], #0x2\n" - "ld1 { v20.h }[2], [x24], #0x2\n" - "ld1 { v26.h }[2], [x23], #0x2\n" - "ld1 { v19.h }[2], [x22], #0x2\n" - "ld1 { v18.h }[2], [x21], #0x2\n" + "ld1 { v27.h }[2], [x26], #0x2\n" + "ld1 { v26.h }[2], [x25], #0x2\n" + "ld1 { v25.h }[2], [x24], #0x2\n" + "ld1 { v24.h }[2], [x23], #0x2\n" + "ld1 { v23.h }[2], [x22], #0x2\n" + "ld1 { v22.h }[2], [x21], #0x2\n" "tbz %x[width], #0, 13f\n" - "ld1 { v30.b }[6], [x28]\n" - "ld1 { v29.b }[6], [x27]\n" - "ld1 { v28.b }[6], [x26]\n" - "ld1 { v27.b }[6], [x25]\n" - "ld1 { v20.b }[6], [x24]\n" - "ld1 { v26.b }[6], [x23]\n" - "ld1 { v19.b }[6], [x22]\n" - "ld1 { v18.b }[6], [x21]\n" + "ld1 { v29.b }[6], [x28]\n" + "ld1 { v28.b }[6], [x27]\n" + "ld1 { v27.b }[6], [x26]\n" + "ld1 { v26.b }[6], [x25]\n" + "ld1 { v25.b }[6], [x24]\n" + "ld1 { v24.b }[6], [x23]\n" + "ld1 { v23.b }[6], [x22]\n" + "ld1 { v22.b }[6], [x21]\n" "b 13f\n" "10:" // odd_loads_1_4 "mov x20, #0x1\n" "tbz %x[width], #0, 13f\n" - "ld1 { v30.b }[4], [x28]\n" - "ld1 { v29.b }[4], [x27]\n" + "ld1 { v29.b }[4], [x28]\n" + "ld1 { v28.b }[4], [x27]\n" "mov x20, #0x2\n" - "ld1 { v28.b }[4], [x26]\n" - "ld1 { v27.b }[4], [x25]\n" - "ld1 { v20.b }[4], [x24]\n" - "ld1 { v26.b }[4], [x23]\n" - "ld1 { v19.b }[4], [x22]\n" - "ld1 { v18.b }[4], [x21]\n" + "ld1 { v27.b }[4], [x26]\n" + "ld1 { v26.b }[4], [x25]\n" + "ld1 { v25.b }[4], [x24]\n" + "ld1 { v24.b }[4], [x23]\n" + "ld1 { v23.b }[4], [x22]\n" + "ld1 { v22.b }[4], [x21]\n" "b 13f\n" "11:" // odd_loads_2_0 "tbz %x[width], #1, 12f\n" - "ldr h30, [x28], #0x2\n" - "ldr h29, [x27], #0x2\n" + "ldr h29, [x28], #0x2\n" + "ldr h28, [x27], #0x2\n" "mov x20, #0x1\n" - "ldr h28, [x26], #0x2\n" - "ldr h27, [x25], #0x2\n" - "ldr h20, [x24], #0x2\n" - "ldr h26, [x23], #0x2\n" - "ldr h19, [x22], #0x2\n" - "ldr h18, [x21], #0x2\n" + "ldr h27, [x26], #0x2\n" + "ldr h26, [x25], #0x2\n" + "ldr h25, [x24], #0x2\n" + "ldr h24, [x23], #0x2\n" + "ldr h23, [x22], #0x2\n" + "ldr h22, [x21], #0x2\n" "tbz %x[width], #0, 13f\n" - "ld1 { v30.b }[2], [x28]\n" - "ld1 { v29.b }[2], [x27]\n" - "ld1 { v28.b }[2], [x26]\n" - "ld1 { v27.b }[2], [x25]\n" - "ld1 { v20.b }[2], [x24]\n" - "ld1 { v26.b }[2], [x23]\n" - "ld1 { v19.b }[2], [x22]\n" - "ld1 { v18.b }[2], [x21]\n" + "ld1 { v29.b }[2], [x28]\n" + "ld1 { v28.b }[2], [x27]\n" + "ld1 { v27.b }[2], [x26]\n" + "ld1 { v26.b }[2], [x25]\n" + "ld1 { v25.b }[2], [x24]\n" + "ld1 { v24.b }[2], [x23]\n" + "ld1 { v23.b }[2], [x22]\n" + "ld1 { v22.b }[2], [x21]\n" "b 13f\n" "12:" // odd_loads_1_0 - "ldr b30, [x28, #0x0]\n" - "ldr b29, [x27, #0x0]\n" + "ldr b29, [x28, #0x0]\n" + "ldr b28, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr b28, [x26, #0x0]\n" - "ldr b27, [x25, #0x0]\n" - "ldr b20, [x24, #0x0]\n" - "ldr b26, [x23, #0x0]\n" - "ldr b19, [x22, #0x0]\n" - "ldr b18, [x21, #0x0]\n" + "ldr b27, [x26, #0x0]\n" + "ldr b26, [x25, #0x0]\n" + "ldr b25, [x24, #0x0]\n" + "ldr b24, [x23, #0x0]\n" + "ldr b23, [x22, #0x0]\n" + "ldr b22, [x21, #0x0]\n" "13:" // Odd load end - "zip1 v22.4s, v30.4s, v28.4s\n" "zip1 v21.4s, v29.4s, v27.4s\n" + "zip1 v20.4s, v28.4s, v26.4s\n" "subs x20, x20, #0x1\n" - "zip1 v17.4s, v20.4s, v19.4s\n" - "zip1 v16.4s, v26.4s, v18.4s\n" - "zip1 v25.4s, v22.4s, v21.4s\n" - "zip1 v24.4s, v17.4s, v16.4s\n" - "str q25, [%x[out_ptr], #0x0]\n" - "sadalp v2.8h, v25.16b\n" - "str q24, [%x[out_ptr], #0x10]\n" - "sadalp v1.8h, v24.16b\n" + "zip1 v19.4s, v25.4s, v23.4s\n" + "zip1 v18.4s, v24.4s, v22.4s\n" + "zip1 v17.4s, v21.4s, v20.4s\n" + "zip1 v16.4s, v19.4s, v18.4s\n" + "str q17, [%x[out_ptr], #0x0]\n" + "sadalp v2.8h, v17.16b\n" + "str q16, [%x[out_ptr], #0x10]\n" + "sadalp v1.8h, v16.16b\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 14f\n" - "zip2 v23.4s, v22.4s, v21.4s\n" - "zip2 v22.4s, v17.4s, v16.4s\n" + "zip2 v17.4s, v21.4s, v20.4s\n" + "zip2 v16.4s, v19.4s, v18.4s\n" "subs x20, x20, #0x1\n" - "str q23, [%x[out_ptr], #0x0]\n" - "sadalp v2.8h, v23.16b\n" - "str q22, [%x[out_ptr], #0x10]\n" - "sadalp v1.8h, v22.16b\n" + "str q17, [%x[out_ptr], #0x0]\n" + "sadalp v2.8h, v17.16b\n" + "str q16, [%x[out_ptr], #0x10]\n" + "sadalp v1.8h, v16.16b\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 14f\n" - "zip2 v21.4s, v30.4s, v28.4s\n" - "zip2 v17.4s, v29.4s, v27.4s\n" + "zip2 v21.4s, v29.4s, v27.4s\n" + "zip2 v20.4s, v28.4s, v26.4s\n" "subs x20, x20, #0x1\n" - "zip2 v20.4s, v20.4s, v19.4s\n" - "zip2 v16.4s, v26.4s, v18.4s\n" - "zip1 v19.4s, v21.4s, v17.4s\n" - "zip1 v18.4s, v20.4s, v16.4s\n" - "str q19, [%x[out_ptr], #0x0]\n" - "sadalp v2.8h, v19.16b\n" - "str q18, [%x[out_ptr], #0x10]\n" - "sadalp v1.8h, v18.16b\n" + "zip2 v19.4s, v25.4s, v23.4s\n" + "zip2 v18.4s, v24.4s, v22.4s\n" + "zip1 v17.4s, v21.4s, v20.4s\n" + "zip1 v16.4s, v19.4s, v18.4s\n" + "str q17, [%x[out_ptr], #0x0]\n" + "sadalp v2.8h, v17.16b\n" + "str q16, [%x[out_ptr], #0x10]\n" + "sadalp v1.8h, v16.16b\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 14f\n" - "zip2 v17.4s, v21.4s, v17.4s\n" - "zip2 v16.4s, v20.4s, v16.4s\n" + "zip2 v17.4s, v21.4s, v20.4s\n" + "zip2 v16.4s, v19.4s, v18.4s\n" "str q17, [%x[out_ptr], #0x0]\n" "sadalp v2.8h, v17.16b\n" "str q16, [%x[out_ptr], #0x10]\n" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_u8_u8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_u8_u8_summing.hpp index 44a79c0f0a3271d81749eba69a35132ff7693a35..17eb7d55568598fa40d8f81862d4f76a2befd531 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_u8_u8_summing.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block4_u8_u8_summing.hpp @@ -153,202 +153,202 @@ void interleave_block<8, 4, VLType::None, true>( "5:" // Main loop skip "cbz %x[width], 14f\n" "tbz %x[width], #3, 9f\n" - "ldr d30, [x28], #0x8\n" - "ldr d29, [x27], #0x8\n" - "ldr d28, [x26], #0x8\n" - "ldr d27, [x25], #0x8\n" - "ldr d20, [x24], #0x8\n" - "ldr d26, [x23], #0x8\n" - "ldr d19, [x22], #0x8\n" - "ldr d18, [x21], #0x8\n" + "ldr d29, [x28], #0x8\n" + "ldr d28, [x27], #0x8\n" + "ldr d27, [x26], #0x8\n" + "ldr d26, [x25], #0x8\n" + "ldr d25, [x24], #0x8\n" + "ldr d24, [x23], #0x8\n" + "ldr d23, [x22], #0x8\n" + "ldr d22, [x21], #0x8\n" "tbz %x[width], #2, 7f\n" - "ld1 { v30.s }[2], [x28], #0x4\n" - "ld1 { v29.s }[2], [x27], #0x4\n" - "ld1 { v28.s }[2], [x26], #0x4\n" - "ld1 { v27.s }[2], [x25], #0x4\n" - "ld1 { v20.s }[2], [x24], #0x4\n" - "ld1 { v26.s }[2], [x23], #0x4\n" - "ld1 { v19.s }[2], [x22], #0x4\n" - "ld1 { v18.s }[2], [x21], #0x4\n" + "ld1 { v29.s }[2], [x28], #0x4\n" + "ld1 { v28.s }[2], [x27], #0x4\n" + "ld1 { v27.s }[2], [x26], #0x4\n" + "ld1 { v26.s }[2], [x25], #0x4\n" + "ld1 { v25.s }[2], [x24], #0x4\n" + "ld1 { v24.s }[2], [x23], #0x4\n" + "ld1 { v23.s }[2], [x22], #0x4\n" + "ld1 { v22.s }[2], [x21], #0x4\n" "tbz %x[width], #1, 6f\n" - "ld1 { v30.h }[6], [x28], #0x2\n" - "ld1 { v29.h }[6], [x27], #0x2\n" + "ld1 { v29.h }[6], [x28], #0x2\n" + "ld1 { v28.h }[6], [x27], #0x2\n" "mov x20, #0x4\n" - "ld1 { v28.h }[6], [x26], #0x2\n" - "ld1 { v27.h }[6], [x25], #0x2\n" - "ld1 { v20.h }[6], [x24], #0x2\n" - "ld1 { v26.h }[6], [x23], #0x2\n" - "ld1 { v19.h }[6], [x22], #0x2\n" - "ld1 { v18.h }[6], [x21], #0x2\n" + "ld1 { v27.h }[6], [x26], #0x2\n" + "ld1 { v26.h }[6], [x25], #0x2\n" + "ld1 { v25.h }[6], [x24], #0x2\n" + "ld1 { v24.h }[6], [x23], #0x2\n" + "ld1 { v23.h }[6], [x22], #0x2\n" + "ld1 { v22.h }[6], [x21], #0x2\n" "tbz %x[width], #0, 13f\n" - "ld1 { v30.b }[14], [x28]\n" - "ld1 { v29.b }[14], [x27]\n" - "ld1 { v28.b }[14], [x26]\n" - "ld1 { v27.b }[14], [x25]\n" - "ld1 { v20.b }[14], [x24]\n" - "ld1 { v26.b }[14], [x23]\n" - "ld1 { v19.b }[14], [x22]\n" - "ld1 { v18.b }[14], [x21]\n" + "ld1 { v29.b }[14], [x28]\n" + "ld1 { v28.b }[14], [x27]\n" + "ld1 { v27.b }[14], [x26]\n" + "ld1 { v26.b }[14], [x25]\n" + "ld1 { v25.b }[14], [x24]\n" + "ld1 { v24.b }[14], [x23]\n" + "ld1 { v23.b }[14], [x22]\n" + "ld1 { v22.b }[14], [x21]\n" "b 13f\n" "6:" // odd_loads_1_12 "mov x20, #0x3\n" "tbz %x[width], #0, 13f\n" - "ld1 { v30.b }[12], [x28]\n" - "ld1 { v29.b }[12], [x27]\n" + "ld1 { v29.b }[12], [x28]\n" + "ld1 { v28.b }[12], [x27]\n" "mov x20, #0x4\n" - "ld1 { v28.b }[12], [x26]\n" - "ld1 { v27.b }[12], [x25]\n" - "ld1 { v20.b }[12], [x24]\n" - "ld1 { v26.b }[12], [x23]\n" - "ld1 { v19.b }[12], [x22]\n" - "ld1 { v18.b }[12], [x21]\n" + "ld1 { v27.b }[12], [x26]\n" + "ld1 { v26.b }[12], [x25]\n" + "ld1 { v25.b }[12], [x24]\n" + "ld1 { v24.b }[12], [x23]\n" + "ld1 { v23.b }[12], [x22]\n" + "ld1 { v22.b }[12], [x21]\n" "b 13f\n" "7:" // odd_loads_2_8 "tbz %x[width], #1, 8f\n" - "ld1 { v30.h }[4], [x28], #0x2\n" - "ld1 { v29.h }[4], [x27], #0x2\n" + "ld1 { v29.h }[4], [x28], #0x2\n" + "ld1 { v28.h }[4], [x27], #0x2\n" "mov x20, #0x3\n" - "ld1 { v28.h }[4], [x26], #0x2\n" - "ld1 { v27.h }[4], [x25], #0x2\n" - "ld1 { v20.h }[4], [x24], #0x2\n" - "ld1 { v26.h }[4], [x23], #0x2\n" - "ld1 { v19.h }[4], [x22], #0x2\n" - "ld1 { v18.h }[4], [x21], #0x2\n" + "ld1 { v27.h }[4], [x26], #0x2\n" + "ld1 { v26.h }[4], [x25], #0x2\n" + "ld1 { v25.h }[4], [x24], #0x2\n" + "ld1 { v24.h }[4], [x23], #0x2\n" + "ld1 { v23.h }[4], [x22], #0x2\n" + "ld1 { v22.h }[4], [x21], #0x2\n" "tbz %x[width], #0, 13f\n" - "ld1 { v30.b }[10], [x28]\n" - "ld1 { v29.b }[10], [x27]\n" - "ld1 { v28.b }[10], [x26]\n" - "ld1 { v27.b }[10], [x25]\n" - "ld1 { v20.b }[10], [x24]\n" - "ld1 { v26.b }[10], [x23]\n" - "ld1 { v19.b }[10], [x22]\n" - "ld1 { v18.b }[10], [x21]\n" + "ld1 { v29.b }[10], [x28]\n" + "ld1 { v28.b }[10], [x27]\n" + "ld1 { v27.b }[10], [x26]\n" + "ld1 { v26.b }[10], [x25]\n" + "ld1 { v25.b }[10], [x24]\n" + "ld1 { v24.b }[10], [x23]\n" + "ld1 { v23.b }[10], [x22]\n" + "ld1 { v22.b }[10], [x21]\n" "b 13f\n" "8:" // odd_loads_1_8 "mov x20, #0x2\n" "tbz %x[width], #0, 13f\n" - "ld1 { v30.b }[8], [x28]\n" - "ld1 { v29.b }[8], [x27]\n" + "ld1 { v29.b }[8], [x28]\n" + "ld1 { v28.b }[8], [x27]\n" "mov x20, #0x3\n" - "ld1 { v28.b }[8], [x26]\n" - "ld1 { v27.b }[8], [x25]\n" - "ld1 { v20.b }[8], [x24]\n" - "ld1 { v26.b }[8], [x23]\n" - "ld1 { v19.b }[8], [x22]\n" - "ld1 { v18.b }[8], [x21]\n" + "ld1 { v27.b }[8], [x26]\n" + "ld1 { v26.b }[8], [x25]\n" + "ld1 { v25.b }[8], [x24]\n" + "ld1 { v24.b }[8], [x23]\n" + "ld1 { v23.b }[8], [x22]\n" + "ld1 { v22.b }[8], [x21]\n" "b 13f\n" "9:" // odd_loads_4_0 "tbz %x[width], #2, 11f\n" - "ldr s30, [x28], #0x4\n" - "ldr s29, [x27], #0x4\n" - "ldr s28, [x26], #0x4\n" - "ldr s27, [x25], #0x4\n" - "ldr s20, [x24], #0x4\n" - "ldr s26, [x23], #0x4\n" - "ldr s19, [x22], #0x4\n" - "ldr s18, [x21], #0x4\n" + "ldr s29, [x28], #0x4\n" + "ldr s28, [x27], #0x4\n" + "ldr s27, [x26], #0x4\n" + "ldr s26, [x25], #0x4\n" + "ldr s25, [x24], #0x4\n" + "ldr s24, [x23], #0x4\n" + "ldr s23, [x22], #0x4\n" + "ldr s22, [x21], #0x4\n" "tbz %x[width], #1, 10f\n" - "ld1 { v30.h }[2], [x28], #0x2\n" - "ld1 { v29.h }[2], [x27], #0x2\n" + "ld1 { v29.h }[2], [x28], #0x2\n" + "ld1 { v28.h }[2], [x27], #0x2\n" "mov x20, #0x2\n" - "ld1 { v28.h }[2], [x26], #0x2\n" - "ld1 { v27.h }[2], [x25], #0x2\n" - "ld1 { v20.h }[2], [x24], #0x2\n" - "ld1 { v26.h }[2], [x23], #0x2\n" - "ld1 { v19.h }[2], [x22], #0x2\n" - "ld1 { v18.h }[2], [x21], #0x2\n" + "ld1 { v27.h }[2], [x26], #0x2\n" + "ld1 { v26.h }[2], [x25], #0x2\n" + "ld1 { v25.h }[2], [x24], #0x2\n" + "ld1 { v24.h }[2], [x23], #0x2\n" + "ld1 { v23.h }[2], [x22], #0x2\n" + "ld1 { v22.h }[2], [x21], #0x2\n" "tbz %x[width], #0, 13f\n" - "ld1 { v30.b }[6], [x28]\n" - "ld1 { v29.b }[6], [x27]\n" - "ld1 { v28.b }[6], [x26]\n" - "ld1 { v27.b }[6], [x25]\n" - "ld1 { v20.b }[6], [x24]\n" - "ld1 { v26.b }[6], [x23]\n" - "ld1 { v19.b }[6], [x22]\n" - "ld1 { v18.b }[6], [x21]\n" + "ld1 { v29.b }[6], [x28]\n" + "ld1 { v28.b }[6], [x27]\n" + "ld1 { v27.b }[6], [x26]\n" + "ld1 { v26.b }[6], [x25]\n" + "ld1 { v25.b }[6], [x24]\n" + "ld1 { v24.b }[6], [x23]\n" + "ld1 { v23.b }[6], [x22]\n" + "ld1 { v22.b }[6], [x21]\n" "b 13f\n" "10:" // odd_loads_1_4 "mov x20, #0x1\n" "tbz %x[width], #0, 13f\n" - "ld1 { v30.b }[4], [x28]\n" - "ld1 { v29.b }[4], [x27]\n" + "ld1 { v29.b }[4], [x28]\n" + "ld1 { v28.b }[4], [x27]\n" "mov x20, #0x2\n" - "ld1 { v28.b }[4], [x26]\n" - "ld1 { v27.b }[4], [x25]\n" - "ld1 { v20.b }[4], [x24]\n" - "ld1 { v26.b }[4], [x23]\n" - "ld1 { v19.b }[4], [x22]\n" - "ld1 { v18.b }[4], [x21]\n" + "ld1 { v27.b }[4], [x26]\n" + "ld1 { v26.b }[4], [x25]\n" + "ld1 { v25.b }[4], [x24]\n" + "ld1 { v24.b }[4], [x23]\n" + "ld1 { v23.b }[4], [x22]\n" + "ld1 { v22.b }[4], [x21]\n" "b 13f\n" "11:" // odd_loads_2_0 "tbz %x[width], #1, 12f\n" - "ldr h30, [x28], #0x2\n" - "ldr h29, [x27], #0x2\n" + "ldr h29, [x28], #0x2\n" + "ldr h28, [x27], #0x2\n" "mov x20, #0x1\n" - "ldr h28, [x26], #0x2\n" - "ldr h27, [x25], #0x2\n" - "ldr h20, [x24], #0x2\n" - "ldr h26, [x23], #0x2\n" - "ldr h19, [x22], #0x2\n" - "ldr h18, [x21], #0x2\n" + "ldr h27, [x26], #0x2\n" + "ldr h26, [x25], #0x2\n" + "ldr h25, [x24], #0x2\n" + "ldr h24, [x23], #0x2\n" + "ldr h23, [x22], #0x2\n" + "ldr h22, [x21], #0x2\n" "tbz %x[width], #0, 13f\n" - "ld1 { v30.b }[2], [x28]\n" - "ld1 { v29.b }[2], [x27]\n" - "ld1 { v28.b }[2], [x26]\n" - "ld1 { v27.b }[2], [x25]\n" - "ld1 { v20.b }[2], [x24]\n" - "ld1 { v26.b }[2], [x23]\n" - "ld1 { v19.b }[2], [x22]\n" - "ld1 { v18.b }[2], [x21]\n" + "ld1 { v29.b }[2], [x28]\n" + "ld1 { v28.b }[2], [x27]\n" + "ld1 { v27.b }[2], [x26]\n" + "ld1 { v26.b }[2], [x25]\n" + "ld1 { v25.b }[2], [x24]\n" + "ld1 { v24.b }[2], [x23]\n" + "ld1 { v23.b }[2], [x22]\n" + "ld1 { v22.b }[2], [x21]\n" "b 13f\n" "12:" // odd_loads_1_0 - "ldr b30, [x28, #0x0]\n" - "ldr b29, [x27, #0x0]\n" + "ldr b29, [x28, #0x0]\n" + "ldr b28, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr b28, [x26, #0x0]\n" - "ldr b27, [x25, #0x0]\n" - "ldr b20, [x24, #0x0]\n" - "ldr b26, [x23, #0x0]\n" - "ldr b19, [x22, #0x0]\n" - "ldr b18, [x21, #0x0]\n" + "ldr b27, [x26, #0x0]\n" + "ldr b26, [x25, #0x0]\n" + "ldr b25, [x24, #0x0]\n" + "ldr b24, [x23, #0x0]\n" + "ldr b23, [x22, #0x0]\n" + "ldr b22, [x21, #0x0]\n" "13:" // Odd load end - "zip1 v22.4s, v30.4s, v28.4s\n" "zip1 v21.4s, v29.4s, v27.4s\n" + "zip1 v20.4s, v28.4s, v26.4s\n" "subs x20, x20, #0x1\n" - "zip1 v17.4s, v20.4s, v19.4s\n" - "zip1 v16.4s, v26.4s, v18.4s\n" - "zip1 v25.4s, v22.4s, v21.4s\n" - "zip1 v24.4s, v17.4s, v16.4s\n" - "str q25, [%x[out_ptr], #0x0]\n" - "uadalp v2.8h, v25.16b\n" - "str q24, [%x[out_ptr], #0x10]\n" - "uadalp v1.8h, v24.16b\n" + "zip1 v19.4s, v25.4s, v23.4s\n" + "zip1 v18.4s, v24.4s, v22.4s\n" + "zip1 v17.4s, v21.4s, v20.4s\n" + "zip1 v16.4s, v19.4s, v18.4s\n" + "str q17, [%x[out_ptr], #0x0]\n" + "uadalp v2.8h, v17.16b\n" + "str q16, [%x[out_ptr], #0x10]\n" + "uadalp v1.8h, v16.16b\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 14f\n" - "zip2 v23.4s, v22.4s, v21.4s\n" - "zip2 v22.4s, v17.4s, v16.4s\n" + "zip2 v17.4s, v21.4s, v20.4s\n" + "zip2 v16.4s, v19.4s, v18.4s\n" "subs x20, x20, #0x1\n" - "str q23, [%x[out_ptr], #0x0]\n" - "uadalp v2.8h, v23.16b\n" - "str q22, [%x[out_ptr], #0x10]\n" - "uadalp v1.8h, v22.16b\n" + "str q17, [%x[out_ptr], #0x0]\n" + "uadalp v2.8h, v17.16b\n" + "str q16, [%x[out_ptr], #0x10]\n" + "uadalp v1.8h, v16.16b\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 14f\n" - "zip2 v21.4s, v30.4s, v28.4s\n" - "zip2 v17.4s, v29.4s, v27.4s\n" + "zip2 v21.4s, v29.4s, v27.4s\n" + "zip2 v20.4s, v28.4s, v26.4s\n" "subs x20, x20, #0x1\n" - "zip2 v20.4s, v20.4s, v19.4s\n" - "zip2 v16.4s, v26.4s, v18.4s\n" - "zip1 v19.4s, v21.4s, v17.4s\n" - "zip1 v18.4s, v20.4s, v16.4s\n" - "str q19, [%x[out_ptr], #0x0]\n" - "uadalp v2.8h, v19.16b\n" - "str q18, [%x[out_ptr], #0x10]\n" - "uadalp v1.8h, v18.16b\n" + "zip2 v19.4s, v25.4s, v23.4s\n" + "zip2 v18.4s, v24.4s, v22.4s\n" + "zip1 v17.4s, v21.4s, v20.4s\n" + "zip1 v16.4s, v19.4s, v18.4s\n" + "str q17, [%x[out_ptr], #0x0]\n" + "uadalp v2.8h, v17.16b\n" + "str q16, [%x[out_ptr], #0x10]\n" + "uadalp v1.8h, v16.16b\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" "beq 14f\n" - "zip2 v17.4s, v21.4s, v17.4s\n" - "zip2 v16.4s, v20.4s, v16.4s\n" + "zip2 v17.4s, v21.4s, v20.4s\n" + "zip2 v16.4s, v19.4s, v18.4s\n" "str q17, [%x[out_ptr], #0x0]\n" "uadalp v2.8h, v17.16b\n" "str q16, [%x[out_ptr], #0x10]\n" diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_s8_s8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_s8_s8.hpp index 4bfb36082e5201f9a826f128eadd66cc1244cad6..7b445ef3d4fea1a1806e512383909802463d8335 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_s8_s8.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_s8_s8.hpp @@ -79,18 +79,18 @@ void interleave_block<8, 8, VLType::None, false>( "prfm pldl1keep, [x21, #0x40]\n" "blt 3f\n" "2:" // Main loop head - "ldr q26, [x28], #0x10\n" - "ldr q21, [x27], #0x10\n" + "ldr q20, [x28], #0x10\n" + "ldr q19, [x27], #0x10\n" "subs %x[width], %x[width], #0x10\n" "cmp %x[width], #0x10\n" "ldr q25, [x26], #0x10\n" "ldr q24, [x25], #0x10\n" - "zip1 v16.2d, v26.2d, v21.2d\n" + "zip1 v16.2d, v20.2d, v19.2d\n" "zip1 v18.2d, v25.2d, v24.2d\n" "ldr q23, [x24], #0x10\n" "ldr q22, [x23], #0x10\n" "zip1 v17.2d, v23.2d, v22.2d\n" - "zip2 v21.2d, v26.2d, v21.2d\n" + "zip2 v21.2d, v20.2d, v19.2d\n" "ldr q20, [x22], #0x10\n" "ldr q19, [x21], #0x10\n" "str q16, [%x[out_ptr], #0x0]\n" @@ -118,188 +118,187 @@ void interleave_block<8, 8, VLType::None, false>( "3:" // Main loop skip "cbz %x[width], 12f\n" "tbz %x[width], #3, 7f\n" - "ldr d26, [x28], #0x8\n" - "ldr d21, [x27], #0x8\n" - "ldr d25, [x26], #0x8\n" - "ldr d24, [x25], #0x8\n" - "ldr d23, [x24], #0x8\n" - "ldr d22, [x23], #0x8\n" - "ldr d20, [x22], #0x8\n" - "ldr d19, [x21], #0x8\n" + "ldr d25, [x28], #0x8\n" + "ldr d24, [x27], #0x8\n" + "ldr d23, [x26], #0x8\n" + "ldr d22, [x25], #0x8\n" + "ldr d21, [x24], #0x8\n" + "ldr d20, [x23], #0x8\n" + "ldr d19, [x22], #0x8\n" + "ldr d18, [x21], #0x8\n" "tbz %x[width], #2, 5f\n" - "ld1 { v26.s }[2], [x28], #0x4\n" - "ld1 { v21.s }[2], [x27], #0x4\n" - "ld1 { v25.s }[2], [x26], #0x4\n" - "ld1 { v24.s }[2], [x25], #0x4\n" - "ld1 { v23.s }[2], [x24], #0x4\n" - "ld1 { v22.s }[2], [x23], #0x4\n" - "ld1 { v20.s }[2], [x22], #0x4\n" - "ld1 { v19.s }[2], [x21], #0x4\n" + "ld1 { v25.s }[2], [x28], #0x4\n" + "ld1 { v24.s }[2], [x27], #0x4\n" + "ld1 { v23.s }[2], [x26], #0x4\n" + "ld1 { v22.s }[2], [x25], #0x4\n" + "ld1 { v21.s }[2], [x24], #0x4\n" + "ld1 { v20.s }[2], [x23], #0x4\n" + "ld1 { v19.s }[2], [x22], #0x4\n" + "ld1 { v18.s }[2], [x21], #0x4\n" "tbz %x[width], #1, 4f\n" - "ld1 { v26.h }[6], [x28], #0x2\n" - "ld1 { v21.h }[6], [x27], #0x2\n" + "ld1 { v25.h }[6], [x28], #0x2\n" + "ld1 { v24.h }[6], [x27], #0x2\n" "mov x20, #0x2\n" - "ld1 { v25.h }[6], [x26], #0x2\n" - "ld1 { v24.h }[6], [x25], #0x2\n" - "ld1 { v23.h }[6], [x24], #0x2\n" - "ld1 { v22.h }[6], [x23], #0x2\n" - "ld1 { v20.h }[6], [x22], #0x2\n" - "ld1 { v19.h }[6], [x21], #0x2\n" + "ld1 { v23.h }[6], [x26], #0x2\n" + "ld1 { v22.h }[6], [x25], #0x2\n" + "ld1 { v21.h }[6], [x24], #0x2\n" + "ld1 { v20.h }[6], [x23], #0x2\n" + "ld1 { v19.h }[6], [x22], #0x2\n" + "ld1 { v18.h }[6], [x21], #0x2\n" "tbz %x[width], #0, 11f\n" - "ld1 { v26.b }[14], [x28]\n" - "ld1 { v21.b }[14], [x27]\n" - "ld1 { v25.b }[14], [x26]\n" - "ld1 { v24.b }[14], [x25]\n" - "ld1 { v23.b }[14], [x24]\n" - "ld1 { v22.b }[14], [x23]\n" - "ld1 { v20.b }[14], [x22]\n" - "ld1 { v19.b }[14], [x21]\n" + "ld1 { v25.b }[14], [x28]\n" + "ld1 { v24.b }[14], [x27]\n" + "ld1 { v23.b }[14], [x26]\n" + "ld1 { v22.b }[14], [x25]\n" + "ld1 { v21.b }[14], [x24]\n" + "ld1 { v20.b }[14], [x23]\n" + "ld1 { v19.b }[14], [x22]\n" + "ld1 { v18.b }[14], [x21]\n" "b 11f\n" "4:" // odd_loads_1_12 "mov x20, #0x2\n" "tbz %x[width], #0, 11f\n" - "ld1 { v26.b }[12], [x28]\n" - "ld1 { v21.b }[12], [x27]\n" - "ld1 { v25.b }[12], [x26]\n" - "ld1 { v24.b }[12], [x25]\n" - "ld1 { v23.b }[12], [x24]\n" - "ld1 { v22.b }[12], [x23]\n" - "ld1 { v20.b }[12], [x22]\n" - "ld1 { v19.b }[12], [x21]\n" + "ld1 { v25.b }[12], [x28]\n" + "ld1 { v24.b }[12], [x27]\n" + "ld1 { v23.b }[12], [x26]\n" + "ld1 { v22.b }[12], [x25]\n" + "ld1 { v21.b }[12], [x24]\n" + "ld1 { v20.b }[12], [x23]\n" + "ld1 { v19.b }[12], [x22]\n" + "ld1 { v18.b }[12], [x21]\n" "b 11f\n" "5:" // odd_loads_2_8 "tbz %x[width], #1, 6f\n" - "ld1 { v26.h }[4], [x28], #0x2\n" - "ld1 { v21.h }[4], [x27], #0x2\n" + "ld1 { v25.h }[4], [x28], #0x2\n" + "ld1 { v24.h }[4], [x27], #0x2\n" "mov x20, #0x2\n" - "ld1 { v25.h }[4], [x26], #0x2\n" - "ld1 { v24.h }[4], [x25], #0x2\n" - "ld1 { v23.h }[4], [x24], #0x2\n" - "ld1 { v22.h }[4], [x23], #0x2\n" - "ld1 { v20.h }[4], [x22], #0x2\n" - "ld1 { v19.h }[4], [x21], #0x2\n" + "ld1 { v23.h }[4], [x26], #0x2\n" + "ld1 { v22.h }[4], [x25], #0x2\n" + "ld1 { v21.h }[4], [x24], #0x2\n" + "ld1 { v20.h }[4], [x23], #0x2\n" + "ld1 { v19.h }[4], [x22], #0x2\n" + "ld1 { v18.h }[4], [x21], #0x2\n" "tbz %x[width], #0, 11f\n" - "ld1 { v26.b }[10], [x28]\n" - "ld1 { v21.b }[10], [x27]\n" - "ld1 { v25.b }[10], [x26]\n" - "ld1 { v24.b }[10], [x25]\n" - "ld1 { v23.b }[10], [x24]\n" - "ld1 { v22.b }[10], [x23]\n" - "ld1 { v20.b }[10], [x22]\n" - "ld1 { v19.b }[10], [x21]\n" + "ld1 { v25.b }[10], [x28]\n" + "ld1 { v24.b }[10], [x27]\n" + "ld1 { v23.b }[10], [x26]\n" + "ld1 { v22.b }[10], [x25]\n" + "ld1 { v21.b }[10], [x24]\n" + "ld1 { v20.b }[10], [x23]\n" + "ld1 { v19.b }[10], [x22]\n" + "ld1 { v18.b }[10], [x21]\n" "b 11f\n" "6:" // odd_loads_1_8 "mov x20, #0x1\n" "tbz %x[width], #0, 11f\n" - "ld1 { v26.b }[8], [x28]\n" - "ld1 { v21.b }[8], [x27]\n" + "ld1 { v25.b }[8], [x28]\n" + "ld1 { v24.b }[8], [x27]\n" "mov x20, #0x2\n" - "ld1 { v25.b }[8], [x26]\n" - "ld1 { v24.b }[8], [x25]\n" - "ld1 { v23.b }[8], [x24]\n" - "ld1 { v22.b }[8], [x23]\n" - "ld1 { v20.b }[8], [x22]\n" - "ld1 { v19.b }[8], [x21]\n" + "ld1 { v23.b }[8], [x26]\n" + "ld1 { v22.b }[8], [x25]\n" + "ld1 { v21.b }[8], [x24]\n" + "ld1 { v20.b }[8], [x23]\n" + "ld1 { v19.b }[8], [x22]\n" + "ld1 { v18.b }[8], [x21]\n" "b 11f\n" "7:" // odd_loads_4_0 "tbz %x[width], #2, 9f\n" - "ldr s26, [x28], #0x4\n" - "ldr s21, [x27], #0x4\n" - "ldr s25, [x26], #0x4\n" - "ldr s24, [x25], #0x4\n" - "ldr s23, [x24], #0x4\n" - "ldr s22, [x23], #0x4\n" - "ldr s20, [x22], #0x4\n" - "ldr s19, [x21], #0x4\n" + "ldr s25, [x28], #0x4\n" + "ldr s24, [x27], #0x4\n" + "ldr s23, [x26], #0x4\n" + "ldr s22, [x25], #0x4\n" + "ldr s21, [x24], #0x4\n" + "ldr s20, [x23], #0x4\n" + "ldr s19, [x22], #0x4\n" + "ldr s18, [x21], #0x4\n" "tbz %x[width], #1, 8f\n" - "ld1 { v26.h }[2], [x28], #0x2\n" - "ld1 { v21.h }[2], [x27], #0x2\n" + "ld1 { v25.h }[2], [x28], #0x2\n" + "ld1 { v24.h }[2], [x27], #0x2\n" "mov x20, #0x1\n" - "ld1 { v25.h }[2], [x26], #0x2\n" - "ld1 { v24.h }[2], [x25], #0x2\n" - "ld1 { v23.h }[2], [x24], #0x2\n" - "ld1 { v22.h }[2], [x23], #0x2\n" - "ld1 { v20.h }[2], [x22], #0x2\n" - "ld1 { v19.h }[2], [x21], #0x2\n" + "ld1 { v23.h }[2], [x26], #0x2\n" + "ld1 { v22.h }[2], [x25], #0x2\n" + "ld1 { v21.h }[2], [x24], #0x2\n" + "ld1 { v20.h }[2], [x23], #0x2\n" + "ld1 { v19.h }[2], [x22], #0x2\n" + "ld1 { v18.h }[2], [x21], #0x2\n" "tbz %x[width], #0, 11f\n" - "ld1 { v26.b }[6], [x28]\n" - "ld1 { v21.b }[6], [x27]\n" - "ld1 { v25.b }[6], [x26]\n" - "ld1 { v24.b }[6], [x25]\n" - "ld1 { v23.b }[6], [x24]\n" - "ld1 { v22.b }[6], [x23]\n" - "ld1 { v20.b }[6], [x22]\n" - "ld1 { v19.b }[6], [x21]\n" + "ld1 { v25.b }[6], [x28]\n" + "ld1 { v24.b }[6], [x27]\n" + "ld1 { v23.b }[6], [x26]\n" + "ld1 { v22.b }[6], [x25]\n" + "ld1 { v21.b }[6], [x24]\n" + "ld1 { v20.b }[6], [x23]\n" + "ld1 { v19.b }[6], [x22]\n" + "ld1 { v18.b }[6], [x21]\n" "b 11f\n" "8:" // odd_loads_1_4 "mov x20, #0x1\n" "tbz %x[width], #0, 11f\n" - "ld1 { v26.b }[4], [x28]\n" - "ld1 { v21.b }[4], [x27]\n" - "ld1 { v25.b }[4], [x26]\n" - "ld1 { v24.b }[4], [x25]\n" - "ld1 { v23.b }[4], [x24]\n" - "ld1 { v22.b }[4], [x23]\n" - "ld1 { v20.b }[4], [x22]\n" - "ld1 { v19.b }[4], [x21]\n" + "ld1 { v25.b }[4], [x28]\n" + "ld1 { v24.b }[4], [x27]\n" + "ld1 { v23.b }[4], [x26]\n" + "ld1 { v22.b }[4], [x25]\n" + "ld1 { v21.b }[4], [x24]\n" + "ld1 { v20.b }[4], [x23]\n" + "ld1 { v19.b }[4], [x22]\n" + "ld1 { v18.b }[4], [x21]\n" "b 11f\n" "9:" // odd_loads_2_0 "tbz %x[width], #1, 10f\n" - "ldr h26, [x28], #0x2\n" - "ldr h21, [x27], #0x2\n" + "ldr h25, [x28], #0x2\n" + "ldr h24, [x27], #0x2\n" "mov x20, #0x1\n" - "ldr h25, [x26], #0x2\n" - "ldr h24, [x25], #0x2\n" - "ldr h23, [x24], #0x2\n" - "ldr h22, [x23], #0x2\n" - "ldr h20, [x22], #0x2\n" - "ldr h19, [x21], #0x2\n" + "ldr h23, [x26], #0x2\n" + "ldr h22, [x25], #0x2\n" + "ldr h21, [x24], #0x2\n" + "ldr h20, [x23], #0x2\n" + "ldr h19, [x22], #0x2\n" + "ldr h18, [x21], #0x2\n" "tbz %x[width], #0, 11f\n" - "ld1 { v26.b }[2], [x28]\n" - "ld1 { v21.b }[2], [x27]\n" - "ld1 { v25.b }[2], [x26]\n" - "ld1 { v24.b }[2], [x25]\n" - "ld1 { v23.b }[2], [x24]\n" - "ld1 { v22.b }[2], [x23]\n" - "ld1 { v20.b }[2], [x22]\n" - "ld1 { v19.b }[2], [x21]\n" + "ld1 { v25.b }[2], [x28]\n" + "ld1 { v24.b }[2], [x27]\n" + "ld1 { v23.b }[2], [x26]\n" + "ld1 { v22.b }[2], [x25]\n" + "ld1 { v21.b }[2], [x24]\n" + "ld1 { v20.b }[2], [x23]\n" + "ld1 { v19.b }[2], [x22]\n" + "ld1 { v18.b }[2], [x21]\n" "b 11f\n" "10:" // odd_loads_1_0 - "ldr b26, [x28, #0x0]\n" - "ldr b21, [x27, #0x0]\n" + "ldr b25, [x28, #0x0]\n" + "ldr b24, [x27, #0x0]\n" "mov x20, #0x1\n" - "ldr b25, [x26, #0x0]\n" - "ldr b24, [x25, #0x0]\n" - "ldr b23, [x24, #0x0]\n" - "ldr b22, [x23, #0x0]\n" - "ldr b20, [x22, #0x0]\n" - "ldr b19, [x21, #0x0]\n" + "ldr b23, [x26, #0x0]\n" + "ldr b22, [x25, #0x0]\n" + "ldr b21, [x24, #0x0]\n" + "ldr b20, [x23, #0x0]\n" + "ldr b19, [x22, #0x0]\n" + "ldr b18, [x21, #0x0]\n" "11:" // Odd load end "subs x20, x20, #0x1\n" - "zip1 v16.2d, v26.2d, v21.2d\n" + "zip1 v16.2d, v25.2d, v24.2d\n" "str q16, [%x[out_ptr], #0x0]\n" - "zip1 v18.2d, v25.2d, v24.2d\n" - "str q18, [%x[out_ptr], #0x10]\n" - "zip1 v17.2d, v23.2d, v22.2d\n" - "zip1 v16.2d, v20.2d, v19.2d\n" + "zip1 v16.2d, v23.2d, v22.2d\n" + "str q16, [%x[out_ptr], #0x10]\n" + "zip1 v17.2d, v21.2d, v20.2d\n" + "zip1 v16.2d, v19.2d, v18.2d\n" "str q17, [%x[out_ptr], #0x20]\n" "str q16, [%x[out_ptr], #0x30]\n" "add %x[out_ptr], %x[out_ptr], #0x40\n" "beq 12f\n" - "zip2 v21.2d, v26.2d, v21.2d\n" - "str q21, [%x[out_ptr], #0x0]\n" - "zip2 v18.2d, v25.2d, v24.2d\n" - "str q18, [%x[out_ptr], #0x10]\n" - "zip2 v17.2d, v23.2d, v22.2d\n" - "zip2 v16.2d, v20.2d, v19.2d\n" + "zip2 v16.2d, v25.2d, v24.2d\n" + "str q16, [%x[out_ptr], #0x0]\n" + "zip2 v16.2d, v23.2d, v22.2d\n" + "str q16, [%x[out_ptr], #0x10]\n" + "zip2 v17.2d, v21.2d, v20.2d\n" + "zip2 v16.2d, v19.2d, v18.2d\n" "str q17, [%x[out_ptr], #0x20]\n" "str q16, [%x[out_ptr], #0x30]\n" "add %x[out_ptr], %x[out_ptr], #0x40\n" "12:" // Odds skip - : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) - : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_s8_s8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_s8_s8_summing.hpp index c6ad2949f5f36c53888a76552c0b3dc327b2890b..a2288e8299909100b4f50bbd09ac5d17b325fda1 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_s8_s8_summing.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_s8_s8_summing.hpp @@ -156,182 +156,182 @@ void interleave_block<8, 8, VLType::None, true>( "cbz %x[width], 14f\n" "tbz %x[width], #3, 9f\n" "ldr d27, [x28], #0x8\n" - "ldr d19, [x27], #0x8\n" + "ldr d26, [x27], #0x8\n" "ldr d25, [x26], #0x8\n" - "ldr d18, [x25], #0x8\n" + "ldr d24, [x25], #0x8\n" "ldr d23, [x24], #0x8\n" - "ldr d17, [x23], #0x8\n" + "ldr d22, [x23], #0x8\n" "ldr d21, [x22], #0x8\n" - "ldr d16, [x21], #0x8\n" + "ldr d20, [x21], #0x8\n" "tbz %x[width], #2, 7f\n" "ld1 { v27.s }[2], [x28], #0x4\n" - "ld1 { v19.s }[2], [x27], #0x4\n" + "ld1 { v26.s }[2], [x27], #0x4\n" "ld1 { v25.s }[2], [x26], #0x4\n" - "ld1 { v18.s }[2], [x25], #0x4\n" + "ld1 { v24.s }[2], [x25], #0x4\n" "ld1 { v23.s }[2], [x24], #0x4\n" - "ld1 { v17.s }[2], [x23], #0x4\n" + "ld1 { v22.s }[2], [x23], #0x4\n" "ld1 { v21.s }[2], [x22], #0x4\n" - "ld1 { v16.s }[2], [x21], #0x4\n" + "ld1 { v20.s }[2], [x21], #0x4\n" "tbz %x[width], #1, 6f\n" "ld1 { v27.h }[6], [x28], #0x2\n" - "ld1 { v19.h }[6], [x27], #0x2\n" + "ld1 { v26.h }[6], [x27], #0x2\n" "mov x20, #0x2\n" "ld1 { v25.h }[6], [x26], #0x2\n" - "ld1 { v18.h }[6], [x25], #0x2\n" + "ld1 { v24.h }[6], [x25], #0x2\n" "ld1 { v23.h }[6], [x24], #0x2\n" - "ld1 { v17.h }[6], [x23], #0x2\n" + "ld1 { v22.h }[6], [x23], #0x2\n" "ld1 { v21.h }[6], [x22], #0x2\n" - "ld1 { v16.h }[6], [x21], #0x2\n" + "ld1 { v20.h }[6], [x21], #0x2\n" "tbz %x[width], #0, 13f\n" "ld1 { v27.b }[14], [x28]\n" - "ld1 { v19.b }[14], [x27]\n" + "ld1 { v26.b }[14], [x27]\n" "ld1 { v25.b }[14], [x26]\n" - "ld1 { v18.b }[14], [x25]\n" + "ld1 { v24.b }[14], [x25]\n" "ld1 { v23.b }[14], [x24]\n" - "ld1 { v17.b }[14], [x23]\n" + "ld1 { v22.b }[14], [x23]\n" "ld1 { v21.b }[14], [x22]\n" - "ld1 { v16.b }[14], [x21]\n" + "ld1 { v20.b }[14], [x21]\n" "b 13f\n" "6:" // odd_loads_1_12 "mov x20, #0x2\n" "tbz %x[width], #0, 13f\n" "ld1 { v27.b }[12], [x28]\n" - "ld1 { v19.b }[12], [x27]\n" + "ld1 { v26.b }[12], [x27]\n" "ld1 { v25.b }[12], [x26]\n" - "ld1 { v18.b }[12], [x25]\n" + "ld1 { v24.b }[12], [x25]\n" "ld1 { v23.b }[12], [x24]\n" - "ld1 { v17.b }[12], [x23]\n" + "ld1 { v22.b }[12], [x23]\n" "ld1 { v21.b }[12], [x22]\n" - "ld1 { v16.b }[12], [x21]\n" + "ld1 { v20.b }[12], [x21]\n" "b 13f\n" "7:" // odd_loads_2_8 "tbz %x[width], #1, 8f\n" "ld1 { v27.h }[4], [x28], #0x2\n" - "ld1 { v19.h }[4], [x27], #0x2\n" + "ld1 { v26.h }[4], [x27], #0x2\n" "mov x20, #0x2\n" "ld1 { v25.h }[4], [x26], #0x2\n" - "ld1 { v18.h }[4], [x25], #0x2\n" + "ld1 { v24.h }[4], [x25], #0x2\n" "ld1 { v23.h }[4], [x24], #0x2\n" - "ld1 { v17.h }[4], [x23], #0x2\n" + "ld1 { v22.h }[4], [x23], #0x2\n" "ld1 { v21.h }[4], [x22], #0x2\n" - "ld1 { v16.h }[4], [x21], #0x2\n" + "ld1 { v20.h }[4], [x21], #0x2\n" "tbz %x[width], #0, 13f\n" "ld1 { v27.b }[10], [x28]\n" - "ld1 { v19.b }[10], [x27]\n" + "ld1 { v26.b }[10], [x27]\n" "ld1 { v25.b }[10], [x26]\n" - "ld1 { v18.b }[10], [x25]\n" + "ld1 { v24.b }[10], [x25]\n" "ld1 { v23.b }[10], [x24]\n" - "ld1 { v17.b }[10], [x23]\n" + "ld1 { v22.b }[10], [x23]\n" "ld1 { v21.b }[10], [x22]\n" - "ld1 { v16.b }[10], [x21]\n" + "ld1 { v20.b }[10], [x21]\n" "b 13f\n" "8:" // odd_loads_1_8 "mov x20, #0x1\n" "tbz %x[width], #0, 13f\n" "ld1 { v27.b }[8], [x28]\n" - "ld1 { v19.b }[8], [x27]\n" + "ld1 { v26.b }[8], [x27]\n" "mov x20, #0x2\n" "ld1 { v25.b }[8], [x26]\n" - "ld1 { v18.b }[8], [x25]\n" + "ld1 { v24.b }[8], [x25]\n" "ld1 { v23.b }[8], [x24]\n" - "ld1 { v17.b }[8], [x23]\n" + "ld1 { v22.b }[8], [x23]\n" "ld1 { v21.b }[8], [x22]\n" - "ld1 { v16.b }[8], [x21]\n" + "ld1 { v20.b }[8], [x21]\n" "b 13f\n" "9:" // odd_loads_4_0 "tbz %x[width], #2, 11f\n" "ldr s27, [x28], #0x4\n" - "ldr s19, [x27], #0x4\n" + "ldr s26, [x27], #0x4\n" "ldr s25, [x26], #0x4\n" - "ldr s18, [x25], #0x4\n" + "ldr s24, [x25], #0x4\n" "ldr s23, [x24], #0x4\n" - "ldr s17, [x23], #0x4\n" + "ldr s22, [x23], #0x4\n" "ldr s21, [x22], #0x4\n" - "ldr s16, [x21], #0x4\n" + "ldr s20, [x21], #0x4\n" "tbz %x[width], #1, 10f\n" "ld1 { v27.h }[2], [x28], #0x2\n" - "ld1 { v19.h }[2], [x27], #0x2\n" + "ld1 { v26.h }[2], [x27], #0x2\n" "mov x20, #0x1\n" "ld1 { v25.h }[2], [x26], #0x2\n" - "ld1 { v18.h }[2], [x25], #0x2\n" + "ld1 { v24.h }[2], [x25], #0x2\n" "ld1 { v23.h }[2], [x24], #0x2\n" - "ld1 { v17.h }[2], [x23], #0x2\n" + "ld1 { v22.h }[2], [x23], #0x2\n" "ld1 { v21.h }[2], [x22], #0x2\n" - "ld1 { v16.h }[2], [x21], #0x2\n" + "ld1 { v20.h }[2], [x21], #0x2\n" "tbz %x[width], #0, 13f\n" "ld1 { v27.b }[6], [x28]\n" - "ld1 { v19.b }[6], [x27]\n" + "ld1 { v26.b }[6], [x27]\n" "ld1 { v25.b }[6], [x26]\n" - "ld1 { v18.b }[6], [x25]\n" + "ld1 { v24.b }[6], [x25]\n" "ld1 { v23.b }[6], [x24]\n" - "ld1 { v17.b }[6], [x23]\n" + "ld1 { v22.b }[6], [x23]\n" "ld1 { v21.b }[6], [x22]\n" - "ld1 { v16.b }[6], [x21]\n" + "ld1 { v20.b }[6], [x21]\n" "b 13f\n" "10:" // odd_loads_1_4 "mov x20, #0x1\n" "tbz %x[width], #0, 13f\n" "ld1 { v27.b }[4], [x28]\n" - "ld1 { v19.b }[4], [x27]\n" + "ld1 { v26.b }[4], [x27]\n" "ld1 { v25.b }[4], [x26]\n" - "ld1 { v18.b }[4], [x25]\n" + "ld1 { v24.b }[4], [x25]\n" "ld1 { v23.b }[4], [x24]\n" - "ld1 { v17.b }[4], [x23]\n" + "ld1 { v22.b }[4], [x23]\n" "ld1 { v21.b }[4], [x22]\n" - "ld1 { v16.b }[4], [x21]\n" + "ld1 { v20.b }[4], [x21]\n" "b 13f\n" "11:" // odd_loads_2_0 "tbz %x[width], #1, 12f\n" "ldr h27, [x28], #0x2\n" - "ldr h19, [x27], #0x2\n" + "ldr h26, [x27], #0x2\n" "mov x20, #0x1\n" "ldr h25, [x26], #0x2\n" - "ldr h18, [x25], #0x2\n" + "ldr h24, [x25], #0x2\n" "ldr h23, [x24], #0x2\n" - "ldr h17, [x23], #0x2\n" + "ldr h22, [x23], #0x2\n" "ldr h21, [x22], #0x2\n" - "ldr h16, [x21], #0x2\n" + "ldr h20, [x21], #0x2\n" "tbz %x[width], #0, 13f\n" "ld1 { v27.b }[2], [x28]\n" - "ld1 { v19.b }[2], [x27]\n" + "ld1 { v26.b }[2], [x27]\n" "ld1 { v25.b }[2], [x26]\n" - "ld1 { v18.b }[2], [x25]\n" + "ld1 { v24.b }[2], [x25]\n" "ld1 { v23.b }[2], [x24]\n" - "ld1 { v17.b }[2], [x23]\n" + "ld1 { v22.b }[2], [x23]\n" "ld1 { v21.b }[2], [x22]\n" - "ld1 { v16.b }[2], [x21]\n" + "ld1 { v20.b }[2], [x21]\n" "b 13f\n" "12:" // odd_loads_1_0 "ldr b27, [x28, #0x0]\n" - "ldr b19, [x27, #0x0]\n" + "ldr b26, [x27, #0x0]\n" "mov x20, #0x1\n" "ldr b25, [x26, #0x0]\n" - "ldr b18, [x25, #0x0]\n" + "ldr b24, [x25, #0x0]\n" "ldr b23, [x24, #0x0]\n" - "ldr b17, [x23, #0x0]\n" + "ldr b22, [x23, #0x0]\n" "ldr b21, [x22, #0x0]\n" - "ldr b16, [x21, #0x0]\n" + "ldr b20, [x21, #0x0]\n" "13:" // Odd load end - "zip1 v26.2d, v27.2d, v19.2d\n" - "zip1 v24.2d, v25.2d, v18.2d\n" + "zip1 v19.2d, v27.2d, v26.2d\n" + "zip1 v18.2d, v25.2d, v24.2d\n" "subs x20, x20, #0x1\n" - "str q26, [%x[out_ptr], #0x0]\n" - "zip1 v22.2d, v23.2d, v17.2d\n" - "zip1 v20.2d, v21.2d, v16.2d\n" - "str q24, [%x[out_ptr], #0x10]\n" - "sadalp v5.8h, v26.16b\n" - "sadalp v4.8h, v24.16b\n" - "str q22, [%x[out_ptr], #0x20]\n" - "sadalp v3.8h, v22.16b\n" - "str q20, [%x[out_ptr], #0x30]\n" - "sadalp v2.8h, v20.16b\n" + "str q19, [%x[out_ptr], #0x0]\n" + "zip1 v17.2d, v23.2d, v22.2d\n" + "zip1 v16.2d, v21.2d, v20.2d\n" + "str q18, [%x[out_ptr], #0x10]\n" + "sadalp v5.8h, v19.16b\n" + "sadalp v4.8h, v18.16b\n" + "str q17, [%x[out_ptr], #0x20]\n" + "sadalp v3.8h, v17.16b\n" + "str q16, [%x[out_ptr], #0x30]\n" + "sadalp v2.8h, v16.16b\n" "add %x[out_ptr], %x[out_ptr], #0x40\n" "beq 14f\n" - "zip2 v19.2d, v27.2d, v19.2d\n" - "zip2 v18.2d, v25.2d, v18.2d\n" + "zip2 v19.2d, v27.2d, v26.2d\n" + "zip2 v18.2d, v25.2d, v24.2d\n" "str q19, [%x[out_ptr], #0x0]\n" - "zip2 v17.2d, v23.2d, v17.2d\n" - "zip2 v16.2d, v21.2d, v16.2d\n" + "zip2 v17.2d, v23.2d, v22.2d\n" + "zip2 v16.2d, v21.2d, v20.2d\n" "str q18, [%x[out_ptr], #0x10]\n" "sadalp v5.8h, v19.16b\n" "sadalp v4.8h, v18.16b\n" @@ -346,11 +346,11 @@ void interleave_block<8, 8, VLType::None, true>( "sadalp v31.4s, v3.8h\n" "sadalp v30.4s, v2.8h\n" "addp v1.4s, v1.4s, v0.4s\n" - "addp v0.4s, v31.4s, v30.4s\n" + "addp v16.4s, v31.4s, v30.4s\n" "add v1.4s, v1.4s, v29.4s\n" - "add v0.4s, v0.4s, v28.4s\n" + "add v16.4s, v16.4s, v28.4s\n" "str q1, [%x[out_ptr], #0x0]\n" - "str q0, [%x[out_ptr], #0x10]\n" + "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_u8_u8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_u8_u8_summing.hpp index 6c4a5fa62b83dab26d3175fc0189d197fee40b67..56d34a8a64f9d9e8cb1006c12903153bf3f276e6 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_u8_u8_summing.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/a64_interleave8_block8_u8_u8_summing.hpp @@ -156,182 +156,182 @@ void interleave_block<8, 8, VLType::None, true>( "cbz %x[width], 14f\n" "tbz %x[width], #3, 9f\n" "ldr d27, [x28], #0x8\n" - "ldr d19, [x27], #0x8\n" + "ldr d26, [x27], #0x8\n" "ldr d25, [x26], #0x8\n" - "ldr d18, [x25], #0x8\n" + "ldr d24, [x25], #0x8\n" "ldr d23, [x24], #0x8\n" - "ldr d17, [x23], #0x8\n" + "ldr d22, [x23], #0x8\n" "ldr d21, [x22], #0x8\n" - "ldr d16, [x21], #0x8\n" + "ldr d20, [x21], #0x8\n" "tbz %x[width], #2, 7f\n" "ld1 { v27.s }[2], [x28], #0x4\n" - "ld1 { v19.s }[2], [x27], #0x4\n" + "ld1 { v26.s }[2], [x27], #0x4\n" "ld1 { v25.s }[2], [x26], #0x4\n" - "ld1 { v18.s }[2], [x25], #0x4\n" + "ld1 { v24.s }[2], [x25], #0x4\n" "ld1 { v23.s }[2], [x24], #0x4\n" - "ld1 { v17.s }[2], [x23], #0x4\n" + "ld1 { v22.s }[2], [x23], #0x4\n" "ld1 { v21.s }[2], [x22], #0x4\n" - "ld1 { v16.s }[2], [x21], #0x4\n" + "ld1 { v20.s }[2], [x21], #0x4\n" "tbz %x[width], #1, 6f\n" "ld1 { v27.h }[6], [x28], #0x2\n" - "ld1 { v19.h }[6], [x27], #0x2\n" + "ld1 { v26.h }[6], [x27], #0x2\n" "mov x20, #0x2\n" "ld1 { v25.h }[6], [x26], #0x2\n" - "ld1 { v18.h }[6], [x25], #0x2\n" + "ld1 { v24.h }[6], [x25], #0x2\n" "ld1 { v23.h }[6], [x24], #0x2\n" - "ld1 { v17.h }[6], [x23], #0x2\n" + "ld1 { v22.h }[6], [x23], #0x2\n" "ld1 { v21.h }[6], [x22], #0x2\n" - "ld1 { v16.h }[6], [x21], #0x2\n" + "ld1 { v20.h }[6], [x21], #0x2\n" "tbz %x[width], #0, 13f\n" "ld1 { v27.b }[14], [x28]\n" - "ld1 { v19.b }[14], [x27]\n" + "ld1 { v26.b }[14], [x27]\n" "ld1 { v25.b }[14], [x26]\n" - "ld1 { v18.b }[14], [x25]\n" + "ld1 { v24.b }[14], [x25]\n" "ld1 { v23.b }[14], [x24]\n" - "ld1 { v17.b }[14], [x23]\n" + "ld1 { v22.b }[14], [x23]\n" "ld1 { v21.b }[14], [x22]\n" - "ld1 { v16.b }[14], [x21]\n" + "ld1 { v20.b }[14], [x21]\n" "b 13f\n" "6:" // odd_loads_1_12 "mov x20, #0x2\n" "tbz %x[width], #0, 13f\n" "ld1 { v27.b }[12], [x28]\n" - "ld1 { v19.b }[12], [x27]\n" + "ld1 { v26.b }[12], [x27]\n" "ld1 { v25.b }[12], [x26]\n" - "ld1 { v18.b }[12], [x25]\n" + "ld1 { v24.b }[12], [x25]\n" "ld1 { v23.b }[12], [x24]\n" - "ld1 { v17.b }[12], [x23]\n" + "ld1 { v22.b }[12], [x23]\n" "ld1 { v21.b }[12], [x22]\n" - "ld1 { v16.b }[12], [x21]\n" + "ld1 { v20.b }[12], [x21]\n" "b 13f\n" "7:" // odd_loads_2_8 "tbz %x[width], #1, 8f\n" "ld1 { v27.h }[4], [x28], #0x2\n" - "ld1 { v19.h }[4], [x27], #0x2\n" + "ld1 { v26.h }[4], [x27], #0x2\n" "mov x20, #0x2\n" "ld1 { v25.h }[4], [x26], #0x2\n" - "ld1 { v18.h }[4], [x25], #0x2\n" + "ld1 { v24.h }[4], [x25], #0x2\n" "ld1 { v23.h }[4], [x24], #0x2\n" - "ld1 { v17.h }[4], [x23], #0x2\n" + "ld1 { v22.h }[4], [x23], #0x2\n" "ld1 { v21.h }[4], [x22], #0x2\n" - "ld1 { v16.h }[4], [x21], #0x2\n" + "ld1 { v20.h }[4], [x21], #0x2\n" "tbz %x[width], #0, 13f\n" "ld1 { v27.b }[10], [x28]\n" - "ld1 { v19.b }[10], [x27]\n" + "ld1 { v26.b }[10], [x27]\n" "ld1 { v25.b }[10], [x26]\n" - "ld1 { v18.b }[10], [x25]\n" + "ld1 { v24.b }[10], [x25]\n" "ld1 { v23.b }[10], [x24]\n" - "ld1 { v17.b }[10], [x23]\n" + "ld1 { v22.b }[10], [x23]\n" "ld1 { v21.b }[10], [x22]\n" - "ld1 { v16.b }[10], [x21]\n" + "ld1 { v20.b }[10], [x21]\n" "b 13f\n" "8:" // odd_loads_1_8 "mov x20, #0x1\n" "tbz %x[width], #0, 13f\n" "ld1 { v27.b }[8], [x28]\n" - "ld1 { v19.b }[8], [x27]\n" + "ld1 { v26.b }[8], [x27]\n" "mov x20, #0x2\n" "ld1 { v25.b }[8], [x26]\n" - "ld1 { v18.b }[8], [x25]\n" + "ld1 { v24.b }[8], [x25]\n" "ld1 { v23.b }[8], [x24]\n" - "ld1 { v17.b }[8], [x23]\n" + "ld1 { v22.b }[8], [x23]\n" "ld1 { v21.b }[8], [x22]\n" - "ld1 { v16.b }[8], [x21]\n" + "ld1 { v20.b }[8], [x21]\n" "b 13f\n" "9:" // odd_loads_4_0 "tbz %x[width], #2, 11f\n" "ldr s27, [x28], #0x4\n" - "ldr s19, [x27], #0x4\n" + "ldr s26, [x27], #0x4\n" "ldr s25, [x26], #0x4\n" - "ldr s18, [x25], #0x4\n" + "ldr s24, [x25], #0x4\n" "ldr s23, [x24], #0x4\n" - "ldr s17, [x23], #0x4\n" + "ldr s22, [x23], #0x4\n" "ldr s21, [x22], #0x4\n" - "ldr s16, [x21], #0x4\n" + "ldr s20, [x21], #0x4\n" "tbz %x[width], #1, 10f\n" "ld1 { v27.h }[2], [x28], #0x2\n" - "ld1 { v19.h }[2], [x27], #0x2\n" + "ld1 { v26.h }[2], [x27], #0x2\n" "mov x20, #0x1\n" "ld1 { v25.h }[2], [x26], #0x2\n" - "ld1 { v18.h }[2], [x25], #0x2\n" + "ld1 { v24.h }[2], [x25], #0x2\n" "ld1 { v23.h }[2], [x24], #0x2\n" - "ld1 { v17.h }[2], [x23], #0x2\n" + "ld1 { v22.h }[2], [x23], #0x2\n" "ld1 { v21.h }[2], [x22], #0x2\n" - "ld1 { v16.h }[2], [x21], #0x2\n" + "ld1 { v20.h }[2], [x21], #0x2\n" "tbz %x[width], #0, 13f\n" "ld1 { v27.b }[6], [x28]\n" - "ld1 { v19.b }[6], [x27]\n" + "ld1 { v26.b }[6], [x27]\n" "ld1 { v25.b }[6], [x26]\n" - "ld1 { v18.b }[6], [x25]\n" + "ld1 { v24.b }[6], [x25]\n" "ld1 { v23.b }[6], [x24]\n" - "ld1 { v17.b }[6], [x23]\n" + "ld1 { v22.b }[6], [x23]\n" "ld1 { v21.b }[6], [x22]\n" - "ld1 { v16.b }[6], [x21]\n" + "ld1 { v20.b }[6], [x21]\n" "b 13f\n" "10:" // odd_loads_1_4 "mov x20, #0x1\n" "tbz %x[width], #0, 13f\n" "ld1 { v27.b }[4], [x28]\n" - "ld1 { v19.b }[4], [x27]\n" + "ld1 { v26.b }[4], [x27]\n" "ld1 { v25.b }[4], [x26]\n" - "ld1 { v18.b }[4], [x25]\n" + "ld1 { v24.b }[4], [x25]\n" "ld1 { v23.b }[4], [x24]\n" - "ld1 { v17.b }[4], [x23]\n" + "ld1 { v22.b }[4], [x23]\n" "ld1 { v21.b }[4], [x22]\n" - "ld1 { v16.b }[4], [x21]\n" + "ld1 { v20.b }[4], [x21]\n" "b 13f\n" "11:" // odd_loads_2_0 "tbz %x[width], #1, 12f\n" "ldr h27, [x28], #0x2\n" - "ldr h19, [x27], #0x2\n" + "ldr h26, [x27], #0x2\n" "mov x20, #0x1\n" "ldr h25, [x26], #0x2\n" - "ldr h18, [x25], #0x2\n" + "ldr h24, [x25], #0x2\n" "ldr h23, [x24], #0x2\n" - "ldr h17, [x23], #0x2\n" + "ldr h22, [x23], #0x2\n" "ldr h21, [x22], #0x2\n" - "ldr h16, [x21], #0x2\n" + "ldr h20, [x21], #0x2\n" "tbz %x[width], #0, 13f\n" "ld1 { v27.b }[2], [x28]\n" - "ld1 { v19.b }[2], [x27]\n" + "ld1 { v26.b }[2], [x27]\n" "ld1 { v25.b }[2], [x26]\n" - "ld1 { v18.b }[2], [x25]\n" + "ld1 { v24.b }[2], [x25]\n" "ld1 { v23.b }[2], [x24]\n" - "ld1 { v17.b }[2], [x23]\n" + "ld1 { v22.b }[2], [x23]\n" "ld1 { v21.b }[2], [x22]\n" - "ld1 { v16.b }[2], [x21]\n" + "ld1 { v20.b }[2], [x21]\n" "b 13f\n" "12:" // odd_loads_1_0 "ldr b27, [x28, #0x0]\n" - "ldr b19, [x27, #0x0]\n" + "ldr b26, [x27, #0x0]\n" "mov x20, #0x1\n" "ldr b25, [x26, #0x0]\n" - "ldr b18, [x25, #0x0]\n" + "ldr b24, [x25, #0x0]\n" "ldr b23, [x24, #0x0]\n" - "ldr b17, [x23, #0x0]\n" + "ldr b22, [x23, #0x0]\n" "ldr b21, [x22, #0x0]\n" - "ldr b16, [x21, #0x0]\n" + "ldr b20, [x21, #0x0]\n" "13:" // Odd load end - "zip1 v26.2d, v27.2d, v19.2d\n" - "zip1 v24.2d, v25.2d, v18.2d\n" + "zip1 v19.2d, v27.2d, v26.2d\n" + "zip1 v18.2d, v25.2d, v24.2d\n" "subs x20, x20, #0x1\n" - "str q26, [%x[out_ptr], #0x0]\n" - "zip1 v22.2d, v23.2d, v17.2d\n" - "zip1 v20.2d, v21.2d, v16.2d\n" - "str q24, [%x[out_ptr], #0x10]\n" - "uadalp v5.8h, v26.16b\n" - "uadalp v4.8h, v24.16b\n" - "str q22, [%x[out_ptr], #0x20]\n" - "uadalp v3.8h, v22.16b\n" - "str q20, [%x[out_ptr], #0x30]\n" - "uadalp v2.8h, v20.16b\n" + "str q19, [%x[out_ptr], #0x0]\n" + "zip1 v17.2d, v23.2d, v22.2d\n" + "zip1 v16.2d, v21.2d, v20.2d\n" + "str q18, [%x[out_ptr], #0x10]\n" + "uadalp v5.8h, v19.16b\n" + "uadalp v4.8h, v18.16b\n" + "str q17, [%x[out_ptr], #0x20]\n" + "uadalp v3.8h, v17.16b\n" + "str q16, [%x[out_ptr], #0x30]\n" + "uadalp v2.8h, v16.16b\n" "add %x[out_ptr], %x[out_ptr], #0x40\n" "beq 14f\n" - "zip2 v19.2d, v27.2d, v19.2d\n" - "zip2 v18.2d, v25.2d, v18.2d\n" + "zip2 v19.2d, v27.2d, v26.2d\n" + "zip2 v18.2d, v25.2d, v24.2d\n" "str q19, [%x[out_ptr], #0x0]\n" - "zip2 v17.2d, v23.2d, v17.2d\n" - "zip2 v16.2d, v21.2d, v16.2d\n" + "zip2 v17.2d, v23.2d, v22.2d\n" + "zip2 v16.2d, v21.2d, v20.2d\n" "str q18, [%x[out_ptr], #0x10]\n" "uadalp v5.8h, v19.16b\n" "uadalp v4.8h, v18.16b\n" @@ -346,11 +346,11 @@ void interleave_block<8, 8, VLType::None, true>( "uadalp v31.4s, v3.8h\n" "uadalp v30.4s, v2.8h\n" "addp v1.4s, v1.4s, v0.4s\n" - "addp v0.4s, v31.4s, v30.4s\n" + "addp v16.4s, v31.4s, v30.4s\n" "add v1.4s, v1.4s, v29.4s\n" - "add v0.4s, v0.4s, v28.4s\n" + "add v16.4s, v16.4s, v28.4s\n" "str q1, [%x[out_ptr], #0x0]\n" - "str q0, [%x[out_ptr], #0x10]\n" + "str q16, [%x[out_ptr], #0x10]\n" "add %x[out_ptr], %x[out_ptr], #0x20\n" : [out_ptr] "+&r" (out_ptr), [width] "+&r" (width) : [first] "r" (first), [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave1VL_block2_fp32_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave1VL_block2_fp32_bf16.hpp index 51b91d16e153e1d7c721618ffb6fdff4aa258269..a5f4754d3daf1200e70f7c4f90d66a8d53922ae0 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave1VL_block2_fp32_bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave1VL_block2_fp32_bf16.hpp @@ -22,16 +22,14 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME2) template <> void interleave_block<1, 2, VLType::SME, false>( bfloat16 * &out, const float * const *in, - size_t width, size_t height, size_t row_offset, bool first + size_t width, size_t height, size_t row_offset, bool ) { - ARM_COMPUTE_UNUSED(first); - __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" "cntw x22, ALL, MUL #2\n" @@ -153,4 +151,4 @@ void interleave_block<1, 2, VLType::SME, false>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME2) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave2VL_block2_fp32_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave2VL_block2_fp32_bf16.hpp index 25bfad18b11c7ad3d0912b48f291987151e46979..c1d0ac5bc76852febd5809c8777275788a555c7f 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave2VL_block2_fp32_bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave2VL_block2_fp32_bf16.hpp @@ -22,16 +22,14 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME2) template <> void interleave_block<2, 2, VLType::SME, false>( bfloat16 * &out, const float * const *in, - size_t width, size_t height, size_t row_offset, bool first + size_t width, size_t height, size_t row_offset, bool ) { - ARM_COMPUTE_UNUSED(first); - __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" "cntw x22, ALL, MUL #2\n" @@ -184,4 +182,4 @@ void interleave_block<2, 2, VLType::SME, false>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME2) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave4VL_block2_fp32_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave4VL_block2_fp32_bf16.hpp index 9255831e868e22780435b9836ff9f54c7d748ebd..03575d7ff27e5f77b4f727f276b716c1fb42600e 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave4VL_block2_fp32_bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme2_interleave4VL_block2_fp32_bf16.hpp @@ -22,16 +22,14 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME2) template <> void interleave_block<4, 2, VLType::SME, false>( bfloat16 * &out, const float * const *in, - size_t width, size_t height, size_t row_offset, bool first + size_t width, size_t height, size_t row_offset, bool ) { - ARM_COMPUTE_UNUSED(first); - __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" "cntw x23, ALL, MUL #2\n" @@ -159,4 +157,4 @@ void interleave_block<4, 2, VLType::SME, false>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME2) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_bf16_bf16.hpp index 9b66a6fb100a0f5e751a3a92aa783d4057b966ad..453778ae3f48cee85d42c66746fcb060b9593377 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_bf16_bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_bf16_bf16.hpp @@ -22,16 +22,14 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME) template <> void interleave_block<1, 1, VLType::SME, false>( bfloat16 * &out, const bfloat16 * const *in, - size_t width, size_t height, size_t row_offset, bool first + size_t width, size_t height, size_t row_offset, bool ) { - ARM_COMPUTE_UNUSED(first); - __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" "mov x21, %x[width]\n" @@ -168,9 +166,9 @@ void interleave_block<1, 1, VLType::SME, false>( "9:" // K loop: Tails: Even: First ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" ".inst 0xe07f82a0 // st1h { za0v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" - "ldr x25, [x26, #0x0]\n" + "ldr x20, [x26, #0x0]\n" ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" - ".inst 0xe0560328 // ld1h { za1h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" + ".inst 0xe0560288 // ld1h { za1h.h[x12] }, p0/Z, [x20, x22, LSL #1]\n" "add x12, x12, #0x1\n" "cmp x12, x11\n" "add x26, x26, #0x8\n" @@ -186,7 +184,7 @@ void interleave_block<1, 1, VLType::SME, false>( "cmp x12, x10\n" "addvl x21, x21, #1\n" "blt 10b\n" - "whilelt p9.h, x27, %x[width]\n" + "whilelt p8.h, x27, %x[width]\n" "b 13f\n" "11:" // K loop: Tails: Odd "mov x12, #0x0\n" @@ -206,4 +204,4 @@ void interleave_block<1, 1, VLType::SME, false>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block2_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block2_bf16_bf16.hpp index d0375de76f5c2ec565ff91cbbafc29b8d7f08c0b..98bdcd2fa24c706231517ed08fe4873ad0dd7f59 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block2_bf16_bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block2_bf16_bf16.hpp @@ -22,16 +22,14 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME) template <> void interleave_block<1, 2, VLType::SME, false>( bfloat16 * &out, const bfloat16 * const *in, - size_t width, size_t height, size_t row_offset, bool first + size_t width, size_t height, size_t row_offset, bool ) { - ARM_COMPUTE_UNUSED(first); - __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" "cnth x22\n" @@ -176,11 +174,11 @@ void interleave_block<1, 2, VLType::SME, false>( "9:" // K loop: Tails: Even: First ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" - "ldr x25, [x26, #0x0]\n" + "ldr x20, [x26, #0x0]\n" "add x12, x12, #0x1\n" ".inst 0x25396140 // psel p0.h, p8.h/Z, p10.h[w13, #1]\n" "cmp x12, x10\n" - ".inst 0xe0562321 // ld1h { za0h.h[x13, #1] }, p0/Z, [x25, x22, LSL #1]\n" + ".inst 0xe0562281 // ld1h { za0h.h[x13, #1] }, p0/Z, [x20, x22, LSL #1]\n" "add x26, x26, #0x8\n" "addvl x21, x21, #1\n" "add x13, x13, #0x2\n" @@ -197,7 +195,7 @@ void interleave_block<1, 2, VLType::SME, false>( "addvl x21, x21, #1\n" "add x20, x20, #0x2\n" "blt 10b\n" - "whilelt p9.h, x27, %x[width]\n" + "whilelt p8.h, x27, %x[width]\n" "b 13f\n" "11:" // K loop: Tails: Odd "mov x12, #0x0\n" @@ -217,4 +215,4 @@ void interleave_block<1, 2, VLType::SME, false>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8.hpp index 622d9aa4fc4976599e53f239ec5a93a746adf59e..4390bb7c7fa92d8eaa4800cdd19f8415a452ab8a 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8.hpp @@ -22,16 +22,14 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME) template <> void interleave_block<1, 4, VLType::SME, false>( int8_t * &out, const int8_t * const *in, - size_t width, size_t height, size_t row_offset, bool first + size_t width, size_t height, size_t row_offset, bool ) { - ARM_COMPUTE_UNUSED(first); - __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" "cntb x21\n" @@ -179,11 +177,11 @@ void interleave_block<1, 4, VLType::SME, false>( "9:" // K loop: Tails: Even: First ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" - "ldr x25, [x26, #0x0]\n" + "ldr x20, [x26, #0x0]\n" "add x12, x12, #0x1\n" ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" "cmp x12, x9\n" - ".inst 0xe0162322 // ld1b { za0h.b[x13, #2] }, p0/Z, [x25, x22]\n" + ".inst 0xe0162282 // ld1b { za0h.b[x13, #2] }, p0/Z, [x20, x22]\n" "add x26, x26, #0x8\n" "addvl x21, x21, #1\n" "add x13, x13, #0x4\n" @@ -200,7 +198,7 @@ void interleave_block<1, 4, VLType::SME, false>( "addvl x21, x21, #1\n" "add x20, x20, #0x4\n" "blt 10b\n" - "whilelt p9.b, x27, %x[width]\n" + "whilelt p8.b, x27, %x[width]\n" "b 13f\n" "11:" // K loop: Tails: Odd "mov x12, #0x0\n" @@ -220,4 +218,4 @@ void interleave_block<1, 4, VLType::SME, false>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8_summing.hpp index 07f03702d9444ac82ea18876a462ae77835f3239..f5ee26196425452e4138f16b34131485442bb5c6 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8_summing.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_s8_s8_summing.hpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME) template <> void interleave_block<1, 4, VLType::SME, true>( @@ -200,12 +200,12 @@ void interleave_block<1, 4, VLType::SME, true>( "10:" // K loop: Tails: Even: First ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" - "ldr x22, [x23, #0x0]\n" + "ldr x20, [x23, #0x0]\n" ".inst 0xc0828810 // mova z16.s, p2/M, za0v.s[x12]\n" "add x12, x12, #0x1\n" ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" "sdot z17.s, z16.b, z18.b\n" - ".inst 0xe01922c2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x22, x25]\n" + ".inst 0xe0192282 // ld1b { za0h.b[x13, #2] }, p0/Z, [x20, x25]\n" "cmp x12, x9\n" "add x23, x23, #0x8\n" "addvl x24, x24, #1\n" @@ -225,7 +225,7 @@ void interleave_block<1, 4, VLType::SME, true>( "addvl x24, x24, #1\n" "add x20, x20, #0x4\n" "blt 11b\n" - "whilelt p9.b, x28, %x[width]\n" + "whilelt p8.b, x28, %x[width]\n" "b 14f\n" "12:" // K loop: Tails: Odd "mov x12, #0x0\n" @@ -249,4 +249,4 @@ void interleave_block<1, 4, VLType::SME, true>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8.hpp index 618570de08309751b6fe44d3920494723477fcf7..76c1d053cd42738db2cf8926a16210df80f896f7 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8.hpp @@ -22,16 +22,14 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME) template <> void interleave_block<1, 4, VLType::SME, false>( uint8_t * &out, const uint8_t * const *in, - size_t width, size_t height, size_t row_offset, bool first + size_t width, size_t height, size_t row_offset, bool ) { - ARM_COMPUTE_UNUSED(first); - __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" "cntb x21\n" @@ -179,11 +177,11 @@ void interleave_block<1, 4, VLType::SME, false>( "9:" // K loop: Tails: Even: First ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" - "ldr x25, [x26, #0x0]\n" + "ldr x20, [x26, #0x0]\n" "add x12, x12, #0x1\n" ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" "cmp x12, x9\n" - ".inst 0xe0162322 // ld1b { za0h.b[x13, #2] }, p0/Z, [x25, x22]\n" + ".inst 0xe0162282 // ld1b { za0h.b[x13, #2] }, p0/Z, [x20, x22]\n" "add x26, x26, #0x8\n" "addvl x21, x21, #1\n" "add x13, x13, #0x4\n" @@ -200,7 +198,7 @@ void interleave_block<1, 4, VLType::SME, false>( "addvl x21, x21, #1\n" "add x20, x20, #0x4\n" "blt 10b\n" - "whilelt p9.b, x27, %x[width]\n" + "whilelt p8.b, x27, %x[width]\n" "b 13f\n" "11:" // K loop: Tails: Odd "mov x12, #0x0\n" @@ -220,4 +218,4 @@ void interleave_block<1, 4, VLType::SME, false>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8_summing.hpp index 646db0caa8feb9d51d1b4bba87ff3e088b8287dc..daf2d3a1002b8a7f8aff12d506534becf066ecd9 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8_summing.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_block4_u8_u8_summing.hpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME) template <> void interleave_block<1, 4, VLType::SME, true>( @@ -200,12 +200,12 @@ void interleave_block<1, 4, VLType::SME, true>( "10:" // K loop: Tails: Even: First ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xe0bf8300 // st1w { za0v.s[x12] }, p0/Z, [x24, XZR, LSL #2]\n" - "ldr x22, [x23, #0x0]\n" + "ldr x20, [x23, #0x0]\n" ".inst 0xc0828810 // mova z16.s, p2/M, za0v.s[x12]\n" "add x12, x12, #0x1\n" ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" "udot z17.s, z16.b, z18.b\n" - ".inst 0xe01922c2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x22, x25]\n" + ".inst 0xe0192282 // ld1b { za0h.b[x13, #2] }, p0/Z, [x20, x25]\n" "cmp x12, x9\n" "add x23, x23, #0x8\n" "addvl x24, x24, #1\n" @@ -225,7 +225,7 @@ void interleave_block<1, 4, VLType::SME, true>( "addvl x24, x24, #1\n" "add x20, x20, #0x4\n" "blt 11b\n" - "whilelt p9.b, x28, %x[width]\n" + "whilelt p8.b, x28, %x[width]\n" "b 14f\n" "12:" // K loop: Tails: Odd "mov x12, #0x0\n" @@ -249,4 +249,4 @@ void interleave_block<1, 4, VLType::SME, true>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp16_fp16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp16_fp16.hpp index 788c1a2ecae3c861a20aa98fe4d730c45397b3e6..274f69f37083723f15c389d3d1d4cbf1e7488a29 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp16_fp16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp16_fp16.hpp @@ -22,16 +22,14 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME) template <> void interleave_block<1, 1, VLType::SME, false>( __fp16 * &out, const __fp16 * const *in, - size_t width, size_t height, size_t row_offset, bool first + size_t width, size_t height, size_t row_offset, bool ) { - ARM_COMPUTE_UNUSED(first); - __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" "mov x21, %x[width]\n" @@ -168,9 +166,9 @@ void interleave_block<1, 1, VLType::SME, false>( "9:" // K loop: Tails: Even: First ".inst 0x25286d20 // psel p0.h, p11.h/Z, p9.h[w12]\n" ".inst 0xe07f82a0 // st1h { za0v.h[x12] }, p0/Z, [x21, XZR, LSL #1]\n" - "ldr x25, [x26, #0x0]\n" + "ldr x20, [x26, #0x0]\n" ".inst 0x25286140 // psel p0.h, p8.h/Z, p10.h[w12]\n" - ".inst 0xe0560328 // ld1h { za1h.h[x12] }, p0/Z, [x25, x22, LSL #1]\n" + ".inst 0xe0560288 // ld1h { za1h.h[x12] }, p0/Z, [x20, x22, LSL #1]\n" "add x12, x12, #0x1\n" "cmp x12, x11\n" "add x26, x26, #0x8\n" @@ -186,7 +184,7 @@ void interleave_block<1, 1, VLType::SME, false>( "cmp x12, x10\n" "addvl x21, x21, #1\n" "blt 10b\n" - "whilelt p9.h, x27, %x[width]\n" + "whilelt p8.h, x27, %x[width]\n" "b 13f\n" "11:" // K loop: Tails: Odd "mov x12, #0x0\n" @@ -206,4 +204,4 @@ void interleave_block<1, 1, VLType::SME, false>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp32_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp32_fp32.hpp index 7de88543d7449cefa54abe9885b728960053243e..ab290649fd6fe6c6b581e851d34dafa48c641791 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp32_fp32.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave1VL_fp32_fp32.hpp @@ -22,16 +22,14 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME) template <> void interleave_block<1, 1, VLType::SME, false>( float * &out, const float * const *in, - size_t width, size_t height, size_t row_offset, bool first + size_t width, size_t height, size_t row_offset, bool ) { - ARM_COMPUTE_UNUSED(first); - __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" "mov x22, %x[width]\n" @@ -167,9 +165,9 @@ void interleave_block<1, 1, VLType::SME, false>( "9:" // K loop: Tails: Even: First ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" - "ldr x25, [x26, #0x0]\n" + "ldr x20, [x26, #0x0]\n" ".inst 0x25306140 // psel p0.s, p8.s/Z, p10.s[w12]\n" - ".inst 0xe0960328 // ld1w { za2h.s[x12] }, p0/Z, [x25, x22, LSL #2]\n" + ".inst 0xe0960288 // ld1w { za2h.s[x12] }, p0/Z, [x20, x22, LSL #2]\n" "add x12, x12, #0x1\n" "cmp x12, x10\n" "add x26, x26, #0x8\n" @@ -185,7 +183,7 @@ void interleave_block<1, 1, VLType::SME, false>( "cmp x12, x9\n" "addvl x21, x21, #1\n" "blt 10b\n" - "whilelt p9.s, x27, %x[width]\n" + "whilelt p8.s, x27, %x[width]\n" "b 13f\n" "11:" // K loop: Tails: Odd "mov x12, #0x0\n" @@ -205,4 +203,4 @@ void interleave_block<1, 1, VLType::SME, false>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_bf16_bf16.hpp index 14ee5d630469e71b1da97eda55cbcd0d2cc94b42..dc6d12b61ee780b1785e6b71aeba4b8f0269da13 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_bf16_bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_bf16_bf16.hpp @@ -22,16 +22,14 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME) template <> void interleave_block<2, 1, VLType::SME, false>( bfloat16 * &out, const bfloat16 * const *in, - size_t width, size_t height, size_t row_offset, bool first + size_t width, size_t height, size_t row_offset, bool ) { - ARM_COMPUTE_UNUSED(first); - __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" "cnth x28\n" @@ -97,4 +95,4 @@ void interleave_block<2, 1, VLType::SME, false>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_bf16_bf16.hpp index f648ccf771375f4f412c210917750ff94f50e551..d9189258c1f2d1a87eb5632025c99ac469e05dfc 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_bf16_bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_bf16_bf16.hpp @@ -22,32 +22,30 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME) template <> void interleave_block<2, 2, VLType::SME, false>( bfloat16 * &out, const bfloat16 * const *in, - size_t width, size_t height, size_t row_offset, bool first + size_t width, size_t height, size_t row_offset, bool ) { - ARM_COMPUTE_UNUSED(first); - __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" - "cnth x21\n" - "mov x22, %x[width]\n" - "inch x22\n" + "cnth x22\n" + "mov x21, %x[width]\n" + "inch x21\n" "mov x20, %x[width]\n" - "sub x17, x21, #0x1\n" - "sub x22, x22, #0x1\n" + "sub x17, x22, #0x1\n" + "sub x21, x21, #0x1\n" "ands x17, x20, x17\n" "cntw x16\n" - "udiv x22, x22, x21\n" // n_passes = ceildiv(width, VL) - "csel x17, x17, x21, NE\n" - "sub x13, x22, #0x1\n" + "udiv x21, x21, x22\n" // n_passes = ceildiv(width, VL) + "csel x17, x17, x22, NE\n" + "sub x13, x21, #0x1\n" "add x17, x17, #0x1\n" "sub x15, x16, #0x2\n" - "lsl x21, %x[height], #0x1\n" // height * 2 + "lsl x22, %x[height], #0x1\n" // height * 2 "lsl x20, x16, #0x1\n" "mov x14, #0x0\n" "mov x11, %x[in]\n" @@ -57,15 +55,15 @@ void interleave_block<2, 2, VLType::SME, false>( "cntw x27, ALL, MUL #3\n" "ldr x26, [x10, #0x0]\n" "lsr x13, x13, #0x1\n" // n_loops = (n_passes - 1) / 2 - "and x25, x22, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "and x25, x21, #0x1\n" // odd_tail = bool(n_passes & 0x1) "ldr x24, [x11, #0x8]\n" "lsr x17, x17, #0x1\n" "ptrue p13.s\n" - "ldr x23, [x10, #0x8]\n" - "whilelt p12.h, XZR, x21\n" - "whilelt p11.h, x20, x21\n" - "mov x22, %x[row_offset]\n" - "mov x21, %x[out]\n" + "ldr x21, [x10, #0x8]\n" + "whilelt p12.h, XZR, x22\n" + "whilelt p11.h, x20, x22\n" + "mov x23, %x[row_offset]\n" + "mov x22, %x[out]\n" "whilelt p10.h, x14, %x[width]\n" "whilelt p9.h, x14, %x[width]\n" "whilelt p8.h, x14, %x[width]\n" @@ -76,39 +74,39 @@ void interleave_block<2, 2, VLType::SME, false>( "1:" // K loop: Charge: Loop ".inst 0x25286581 // psel p1.h, p9.h/Z, p12.h[w12]\n" ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" - ".inst 0xe0560520 // ld1h { za0h.h[x12] }, p1/Z, [x9, x22, LSL #1]\n" + ".inst 0xe0570520 // ld1h { za0h.h[x12] }, p1/Z, [x9, x23, LSL #1]\n" "ldr x9, [x11, #0x0]\n" - ".inst 0xe0560348 // ld1h { za1h.h[x12] }, p0/Z, [x26, x22, LSL #1]\n" + ".inst 0xe0570348 // ld1h { za1h.h[x12] }, p0/Z, [x26, x23, LSL #1]\n" ".inst 0x25686581 // psel p1.h, p9.h/Z, p12.h[w12, #2]\n" ".inst 0x25686160 // psel p0.h, p8.h/Z, p11.h[w12, #2]\n" "ldr x26, [x10, #0x0]\n" - ".inst 0xe0560702 // ld1h { za0h.h[x12, #2] }, p1/Z, [x24, x22, LSL #1]\n" + ".inst 0xe0570702 // ld1h { za0h.h[x12, #2] }, p1/Z, [x24, x23, LSL #1]\n" "ldr x24, [x11, #0x8]\n" "add x11, x11, #0x10\n" - ".inst 0xe05602ea // ld1h { za1h.h[x12, #2] }, p0/Z, [x23, x22, LSL #1]\n" + ".inst 0xe05702aa // ld1h { za1h.h[x12, #2] }, p0/Z, [x21, x23, LSL #1]\n" "add x12, x12, #0x4\n" "cmp x12, x15, LSL #1\n" - "ldr x23, [x10, #0x8]\n" + "ldr x21, [x10, #0x8]\n" "add x10, x10, #0x10\n" "blt 1b\n" "2:" // K loop: Charge: End ".inst 0x25286581 // psel p1.h, p9.h/Z, p12.h[w12]\n" ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" - ".inst 0xe0560520 // ld1h { za0h.h[x12] }, p1/Z, [x9, x22, LSL #1]\n" - ".inst 0xe0560348 // ld1h { za1h.h[x12] }, p0/Z, [x26, x22, LSL #1]\n" + ".inst 0xe0570520 // ld1h { za0h.h[x12] }, p1/Z, [x9, x23, LSL #1]\n" + ".inst 0xe0570348 // ld1h { za1h.h[x12] }, p0/Z, [x26, x23, LSL #1]\n" ".inst 0x25686581 // psel p1.h, p9.h/Z, p12.h[w12, #2]\n" ".inst 0x25686160 // psel p0.h, p8.h/Z, p11.h[w12, #2]\n" "mov x11, %x[in]\n" "add x10, %x[in], x16, LSL #3\n" - ".inst 0xe0560702 // ld1h { za0h.h[x12, #2] }, p1/Z, [x24, x22, LSL #1]\n" + ".inst 0xe0570702 // ld1h { za0h.h[x12, #2] }, p1/Z, [x24, x23, LSL #1]\n" "ldr x9, [x11, #0x0]\n" - ".inst 0xe05602ea // ld1h { za1h.h[x12, #2] }, p0/Z, [x23, x22, LSL #1]\n" + ".inst 0xe05702aa // ld1h { za1h.h[x12, #2] }, p0/Z, [x21, x23, LSL #1]\n" "ldr x26, [x10, #0x0]\n" - "inch x22\n" + "inch x23\n" "inch x14\n" "ldr x24, [x11, #0x8]\n" "add x11, x11, #0x10\n" - "ldr x23, [x10, #0x8]\n" + "ldr x21, [x10, #0x8]\n" "add x10, x10, #0x10\n" "cbz x13, 8f\n" "mov x20, x13\n" @@ -121,60 +119,60 @@ void interleave_block<2, 2, VLType::SME, false>( "4:" // K loop: Main loop: First: Loop ".inst 0x25396581 // psel p1.h, p9.h/Z, p12.h[w13, #1]\n" ".inst 0x25396160 // psel p0.h, p8.h/Z, p11.h[w13, #1]\n" - ".inst 0xe0562521 // ld1h { za0h.h[x13, #1] }, p1/Z, [x9, x22, LSL #1]\n" + ".inst 0xe0572521 // ld1h { za0h.h[x13, #1] }, p1/Z, [x9, x23, LSL #1]\n" "ldr x9, [x11, #0x0]\n" - ".inst 0xe0562349 // ld1h { za1h.h[x13, #1] }, p0/Z, [x26, x22, LSL #1]\n" + ".inst 0xe0572349 // ld1h { za1h.h[x13, #1] }, p0/Z, [x26, x23, LSL #1]\n" ".inst 0x25796580 // psel p0.h, p9.h/Z, p12.h[w13, #3]\n" ".inst 0x25796162 // psel p2.h, p8.h/Z, p11.h[w13, #3]\n" "ldr x26, [x10, #0x0]\n" ".inst 0x25307541 // psel p1.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0562303 // ld1h { za0h.h[x13, #3] }, p0/Z, [x24, x22, LSL #1]\n" + ".inst 0xe0572303 // ld1h { za0h.h[x13, #3] }, p0/Z, [x24, x23, LSL #1]\n" "ldr x24, [x11, #0x8]\n" ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0562aeb // ld1h { za1h.h[x13, #3] }, p2/Z, [x23, x22, LSL #1]\n" - "ldr x23, [x10, #0x8]\n" - ".inst 0xe0bf86a0 // st1w { za0v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" + ".inst 0xe0572aab // ld1h { za1h.h[x13, #3] }, p2/Z, [x21, x23, LSL #1]\n" + "ldr x21, [x10, #0x8]\n" + ".inst 0xe0bf86c0 // st1w { za0v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" - ".inst 0xe0b082a4 // st1w { za1v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" "add x11, x11, #0x10\n" - ".inst 0xe0bc86a1 // st1w { za0v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" + ".inst 0xe0bc86c1 // st1w { za0v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" "add x10, x10, #0x10\n" "add x13, x13, #0x4\n" - ".inst 0xe0bb82a5 // st1w { za1v.s[x12, #1] }, p0/Z, [x21, x27, LSL #2]\n" + ".inst 0xe0bb82c5 // st1w { za1v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" "add x12, x12, #0x2\n" "cmp x12, x15\n" - "addvl x21, x21, #4\n" + "addvl x22, x22, #4\n" "blt 4b\n" "5:" // K loop: Main loop: First: Tail ".inst 0x25396581 // psel p1.h, p9.h/Z, p12.h[w13, #1]\n" ".inst 0x25396160 // psel p0.h, p8.h/Z, p11.h[w13, #1]\n" - ".inst 0xe0562521 // ld1h { za0h.h[x13, #1] }, p1/Z, [x9, x22, LSL #1]\n" - ".inst 0xe0562349 // ld1h { za1h.h[x13, #1] }, p0/Z, [x26, x22, LSL #1]\n" + ".inst 0xe0572521 // ld1h { za0h.h[x13, #1] }, p1/Z, [x9, x23, LSL #1]\n" + ".inst 0xe0572349 // ld1h { za1h.h[x13, #1] }, p0/Z, [x26, x23, LSL #1]\n" "mov x11, %x[in]\n" "add x10, %x[in], x16, LSL #3\n" "ldr x9, [x11, #0x0]\n" ".inst 0x25796580 // psel p0.h, p9.h/Z, p12.h[w13, #3]\n" ".inst 0x25796161 // psel p1.h, p8.h/Z, p11.h[w13, #3]\n" - ".inst 0xe0562303 // ld1h { za0h.h[x13, #3] }, p0/Z, [x24, x22, LSL #1]\n" + ".inst 0xe0572303 // ld1h { za0h.h[x13, #3] }, p0/Z, [x24, x23, LSL #1]\n" "ldr x26, [x10, #0x0]\n" ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe05626eb // ld1h { za1h.h[x13, #3] }, p1/Z, [x23, x22, LSL #1]\n" + ".inst 0xe05726ab // ld1h { za1h.h[x13, #3] }, p1/Z, [x21, x23, LSL #1]\n" "ldr x24, [x11, #0x8]\n" ".inst 0x25307542 // psel p2.s, p13.s/Z, p10.s[w12]\n" - "ldr x23, [x10, #0x8]\n" - ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + "ldr x21, [x10, #0x8]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" - ".inst 0xe0b08aa4 // st1w { za1v.s[x12] }, p2/Z, [x21, x16, LSL #2]\n" + ".inst 0xe0b08ac4 // st1w { za1v.s[x12] }, p2/Z, [x22, x16, LSL #2]\n" "whilelt p10.h, x14, %x[width]\n" "inch x14\n" - ".inst 0xe0bc86a1 // st1w { za0v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" + ".inst 0xe0bc86c1 // st1w { za0v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" "add x11, x11, #0x10\n" "add x10, x10, #0x10\n" - ".inst 0xe0bb82a5 // st1w { za1v.s[x12, #1] }, p0/Z, [x21, x27, LSL #2]\n" - "addvl x21, x21, #4\n" - "inch x22\n" + ".inst 0xe0bb82c5 // st1w { za1v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" + "addvl x22, x22, #4\n" + "inch x23\n" "whilelt p9.h, x14, %x[width]\n" "whilelt p8.h, x14, %x[width]\n" "mov x13, #0x0\n" @@ -183,61 +181,61 @@ void interleave_block<2, 2, VLType::SME, false>( "6:" // K loop: Main loop: Second: Loop ".inst 0x25296581 // psel p1.h, p9.h/Z, p12.h[w13]\n" ".inst 0x25296160 // psel p0.h, p8.h/Z, p11.h[w13]\n" - ".inst 0xe0562520 // ld1h { za0h.h[x13] }, p1/Z, [x9, x22, LSL #1]\n" + ".inst 0xe0572520 // ld1h { za0h.h[x13] }, p1/Z, [x9, x23, LSL #1]\n" "ldr x9, [x11, #0x0]\n" - ".inst 0xe0562348 // ld1h { za1h.h[x13] }, p0/Z, [x26, x22, LSL #1]\n" + ".inst 0xe0572348 // ld1h { za1h.h[x13] }, p0/Z, [x26, x23, LSL #1]\n" ".inst 0x25696580 // psel p0.h, p9.h/Z, p12.h[w13, #2]\n" ".inst 0x25696162 // psel p2.h, p8.h/Z, p11.h[w13, #2]\n" "ldr x26, [x10, #0x0]\n" ".inst 0x25307541 // psel p1.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0562302 // ld1h { za0h.h[x13, #2] }, p0/Z, [x24, x22, LSL #1]\n" + ".inst 0xe0572302 // ld1h { za0h.h[x13, #2] }, p0/Z, [x24, x23, LSL #1]\n" "ldr x24, [x11, #0x8]\n" ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0562aea // ld1h { za1h.h[x13, #2] }, p2/Z, [x23, x22, LSL #1]\n" - "ldr x23, [x10, #0x8]\n" - ".inst 0xe0bf86a8 // st1w { za2v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" + ".inst 0xe0572aaa // ld1h { za1h.h[x13, #2] }, p2/Z, [x21, x23, LSL #1]\n" + "ldr x21, [x10, #0x8]\n" + ".inst 0xe0bf86c8 // st1w { za2v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" - ".inst 0xe0b082ac // st1w { za3v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + ".inst 0xe0b082cc // st1w { za3v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" "add x11, x11, #0x10\n" - ".inst 0xe0bc86a9 // st1w { za2v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" + ".inst 0xe0bc86c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" "add x10, x10, #0x10\n" "add x13, x13, #0x4\n" - ".inst 0xe0bb82ad // st1w { za3v.s[x12, #1] }, p0/Z, [x21, x27, LSL #2]\n" + ".inst 0xe0bb82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" "add x12, x12, #0x2\n" "cmp x12, x15\n" - "addvl x21, x21, #4\n" + "addvl x22, x22, #4\n" "blt 6b\n" "7:" // K loop: Main loop: Second: Tail ".inst 0x25296581 // psel p1.h, p9.h/Z, p12.h[w13]\n" ".inst 0x25296160 // psel p0.h, p8.h/Z, p11.h[w13]\n" - ".inst 0xe0562520 // ld1h { za0h.h[x13] }, p1/Z, [x9, x22, LSL #1]\n" - ".inst 0xe0562348 // ld1h { za1h.h[x13] }, p0/Z, [x26, x22, LSL #1]\n" + ".inst 0xe0572520 // ld1h { za0h.h[x13] }, p1/Z, [x9, x23, LSL #1]\n" + ".inst 0xe0572348 // ld1h { za1h.h[x13] }, p0/Z, [x26, x23, LSL #1]\n" "mov x11, %x[in]\n" "add x10, %x[in], x16, LSL #3\n" "ldr x9, [x11, #0x0]\n" ".inst 0x25696580 // psel p0.h, p9.h/Z, p12.h[w13, #2]\n" ".inst 0x25696161 // psel p1.h, p8.h/Z, p11.h[w13, #2]\n" - ".inst 0xe0562302 // ld1h { za0h.h[x13, #2] }, p0/Z, [x24, x22, LSL #1]\n" + ".inst 0xe0572302 // ld1h { za0h.h[x13, #2] }, p0/Z, [x24, x23, LSL #1]\n" "ldr x26, [x10, #0x0]\n" ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe05626ea // ld1h { za1h.h[x13, #2] }, p1/Z, [x23, x22, LSL #1]\n" + ".inst 0xe05726aa // ld1h { za1h.h[x13, #2] }, p1/Z, [x21, x23, LSL #1]\n" "ldr x24, [x11, #0x8]\n" ".inst 0x25307542 // psel p2.s, p13.s/Z, p10.s[w12]\n" - "ldr x23, [x10, #0x8]\n" - ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + "ldr x21, [x10, #0x8]\n" + ".inst 0xe0bf82c8 // st1w { za2v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" - ".inst 0xe0b08aac // st1w { za3v.s[x12] }, p2/Z, [x21, x16, LSL #2]\n" + ".inst 0xe0b08acc // st1w { za3v.s[x12] }, p2/Z, [x22, x16, LSL #2]\n" "whilelt p10.h, x14, %x[width]\n" "subs x20, x20, #0x1\n" - ".inst 0xe0bc86a9 // st1w { za2v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" + ".inst 0xe0bc86c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" "add x11, x11, #0x10\n" "add x10, x10, #0x10\n" - ".inst 0xe0bb82ad // st1w { za3v.s[x12, #1] }, p0/Z, [x21, x27, LSL #2]\n" - "addvl x21, x21, #4\n" + ".inst 0xe0bb82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" + "addvl x22, x22, #4\n" "inch x14\n" - "inch x22\n" + "inch x23\n" "bgt 3b\n" "8:" // K loop: Tails "cbnz x25, 11f\n" @@ -248,51 +246,51 @@ void interleave_block<2, 2, VLType::SME, false>( "mov x12, #0x0\n" "9:" // K loop: Tails: Even: First ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0b082a4 // st1w { za1v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" - "ldr x9, [x11, #0x0]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" + "ldr x21, [x11, #0x0]\n" "add x12, x12, #0x1\n" ".inst 0x25396581 // psel p1.h, p9.h/Z, p12.h[w13, #1]\n" - "ldr x26, [x11, x16, LSL #0x3]\n" + "ldr x20, [x11, x16, LSL #0x3]\n" ".inst 0x25396160 // psel p0.h, p8.h/Z, p11.h[w13, #1]\n" "cmp x12, x16\n" - ".inst 0xe0562521 // ld1h { za0h.h[x13, #1] }, p1/Z, [x9, x22, LSL #1]\n" - ".inst 0xe0562349 // ld1h { za1h.h[x13, #1] }, p0/Z, [x26, x22, LSL #1]\n" + ".inst 0xe05726a1 // ld1h { za0h.h[x13, #1] }, p1/Z, [x21, x23, LSL #1]\n" + ".inst 0xe0572289 // ld1h { za1h.h[x13, #1] }, p0/Z, [x20, x23, LSL #1]\n" "add x11, x11, #0x8\n" - "addvl x21, x21, #2\n" + "addvl x22, x22, #2\n" "add x13, x13, #0x2\n" "blt 9b\n" "whilelt p10.h, x14, %x[width]\n" - "whilelt p9.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" "whilelt p8.h, x14, %x[width]\n" "mov x20, #0x0\n" "mov x12, #0x0\n" "10:" // K loop: Tails: Even: Second ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0xe0bf82c8 // st1w { za2v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0b082ac // st1w { za3v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + ".inst 0xe0b082cc // st1w { za3v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" "add x12, x12, #0x1\n" "cmp x12, x17\n" - "addvl x21, x21, #2\n" + "addvl x22, x22, #2\n" "add x20, x20, #0x2\n" "blt 10b\n" - "whilelt p10.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" "b 13f\n" "11:" // K loop: Tails: Odd "mov x12, #0x0\n" "12:" // K loop: Tails: Odd: Loop ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0b082a4 // st1w { za1v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" "add x12, x12, #0x1\n" "cmp x12, x17\n" - "addvl x21, x21, #2\n" + "addvl x22, x22, #2\n" "blt 12b\n" "13:" // K loop: End - "mov %x[out], x21\n" + "mov %x[out], x22\n" ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) @@ -300,4 +298,4 @@ void interleave_block<2, 2, VLType::SME, false>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_fp16_fp16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_fp16_fp16.hpp index 61536d38a58c952dd4912625dc147b3566d161b9..ef787c89b9d9e28589b63e247dc07972004a1242 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_fp16_fp16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block2_fp16_fp16.hpp @@ -22,32 +22,30 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME) template <> void interleave_block<2, 2, VLType::SME, false>( __fp16 * &out, const __fp16 * const *in, - size_t width, size_t height, size_t row_offset, bool first + size_t width, size_t height, size_t row_offset, bool ) { - ARM_COMPUTE_UNUSED(first); - __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" - "cnth x21\n" - "mov x22, %x[width]\n" - "inch x22\n" + "cnth x22\n" + "mov x21, %x[width]\n" + "inch x21\n" "mov x20, %x[width]\n" - "sub x17, x21, #0x1\n" - "sub x22, x22, #0x1\n" + "sub x17, x22, #0x1\n" + "sub x21, x21, #0x1\n" "ands x17, x20, x17\n" "cntw x16\n" - "udiv x22, x22, x21\n" // n_passes = ceildiv(width, VL) - "csel x17, x17, x21, NE\n" - "sub x13, x22, #0x1\n" + "udiv x21, x21, x22\n" // n_passes = ceildiv(width, VL) + "csel x17, x17, x22, NE\n" + "sub x13, x21, #0x1\n" "add x17, x17, #0x1\n" "sub x15, x16, #0x2\n" - "lsl x21, %x[height], #0x1\n" // height * 2 + "lsl x22, %x[height], #0x1\n" // height * 2 "lsl x20, x16, #0x1\n" "mov x14, #0x0\n" "mov x11, %x[in]\n" @@ -57,15 +55,15 @@ void interleave_block<2, 2, VLType::SME, false>( "cntw x27, ALL, MUL #3\n" "ldr x26, [x10, #0x0]\n" "lsr x13, x13, #0x1\n" // n_loops = (n_passes - 1) / 2 - "and x25, x22, #0x1\n" // odd_tail = bool(n_passes & 0x1) + "and x25, x21, #0x1\n" // odd_tail = bool(n_passes & 0x1) "ldr x24, [x11, #0x8]\n" "lsr x17, x17, #0x1\n" "ptrue p13.s\n" - "ldr x23, [x10, #0x8]\n" - "whilelt p12.h, XZR, x21\n" - "whilelt p11.h, x20, x21\n" - "mov x22, %x[row_offset]\n" - "mov x21, %x[out]\n" + "ldr x21, [x10, #0x8]\n" + "whilelt p12.h, XZR, x22\n" + "whilelt p11.h, x20, x22\n" + "mov x23, %x[row_offset]\n" + "mov x22, %x[out]\n" "whilelt p10.h, x14, %x[width]\n" "whilelt p9.h, x14, %x[width]\n" "whilelt p8.h, x14, %x[width]\n" @@ -76,39 +74,39 @@ void interleave_block<2, 2, VLType::SME, false>( "1:" // K loop: Charge: Loop ".inst 0x25286581 // psel p1.h, p9.h/Z, p12.h[w12]\n" ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" - ".inst 0xe0560520 // ld1h { za0h.h[x12] }, p1/Z, [x9, x22, LSL #1]\n" + ".inst 0xe0570520 // ld1h { za0h.h[x12] }, p1/Z, [x9, x23, LSL #1]\n" "ldr x9, [x11, #0x0]\n" - ".inst 0xe0560348 // ld1h { za1h.h[x12] }, p0/Z, [x26, x22, LSL #1]\n" + ".inst 0xe0570348 // ld1h { za1h.h[x12] }, p0/Z, [x26, x23, LSL #1]\n" ".inst 0x25686581 // psel p1.h, p9.h/Z, p12.h[w12, #2]\n" ".inst 0x25686160 // psel p0.h, p8.h/Z, p11.h[w12, #2]\n" "ldr x26, [x10, #0x0]\n" - ".inst 0xe0560702 // ld1h { za0h.h[x12, #2] }, p1/Z, [x24, x22, LSL #1]\n" + ".inst 0xe0570702 // ld1h { za0h.h[x12, #2] }, p1/Z, [x24, x23, LSL #1]\n" "ldr x24, [x11, #0x8]\n" "add x11, x11, #0x10\n" - ".inst 0xe05602ea // ld1h { za1h.h[x12, #2] }, p0/Z, [x23, x22, LSL #1]\n" + ".inst 0xe05702aa // ld1h { za1h.h[x12, #2] }, p0/Z, [x21, x23, LSL #1]\n" "add x12, x12, #0x4\n" "cmp x12, x15, LSL #1\n" - "ldr x23, [x10, #0x8]\n" + "ldr x21, [x10, #0x8]\n" "add x10, x10, #0x10\n" "blt 1b\n" "2:" // K loop: Charge: End ".inst 0x25286581 // psel p1.h, p9.h/Z, p12.h[w12]\n" ".inst 0x25286160 // psel p0.h, p8.h/Z, p11.h[w12]\n" - ".inst 0xe0560520 // ld1h { za0h.h[x12] }, p1/Z, [x9, x22, LSL #1]\n" - ".inst 0xe0560348 // ld1h { za1h.h[x12] }, p0/Z, [x26, x22, LSL #1]\n" + ".inst 0xe0570520 // ld1h { za0h.h[x12] }, p1/Z, [x9, x23, LSL #1]\n" + ".inst 0xe0570348 // ld1h { za1h.h[x12] }, p0/Z, [x26, x23, LSL #1]\n" ".inst 0x25686581 // psel p1.h, p9.h/Z, p12.h[w12, #2]\n" ".inst 0x25686160 // psel p0.h, p8.h/Z, p11.h[w12, #2]\n" "mov x11, %x[in]\n" "add x10, %x[in], x16, LSL #3\n" - ".inst 0xe0560702 // ld1h { za0h.h[x12, #2] }, p1/Z, [x24, x22, LSL #1]\n" + ".inst 0xe0570702 // ld1h { za0h.h[x12, #2] }, p1/Z, [x24, x23, LSL #1]\n" "ldr x9, [x11, #0x0]\n" - ".inst 0xe05602ea // ld1h { za1h.h[x12, #2] }, p0/Z, [x23, x22, LSL #1]\n" + ".inst 0xe05702aa // ld1h { za1h.h[x12, #2] }, p0/Z, [x21, x23, LSL #1]\n" "ldr x26, [x10, #0x0]\n" - "inch x22\n" + "inch x23\n" "inch x14\n" "ldr x24, [x11, #0x8]\n" "add x11, x11, #0x10\n" - "ldr x23, [x10, #0x8]\n" + "ldr x21, [x10, #0x8]\n" "add x10, x10, #0x10\n" "cbz x13, 8f\n" "mov x20, x13\n" @@ -121,60 +119,60 @@ void interleave_block<2, 2, VLType::SME, false>( "4:" // K loop: Main loop: First: Loop ".inst 0x25396581 // psel p1.h, p9.h/Z, p12.h[w13, #1]\n" ".inst 0x25396160 // psel p0.h, p8.h/Z, p11.h[w13, #1]\n" - ".inst 0xe0562521 // ld1h { za0h.h[x13, #1] }, p1/Z, [x9, x22, LSL #1]\n" + ".inst 0xe0572521 // ld1h { za0h.h[x13, #1] }, p1/Z, [x9, x23, LSL #1]\n" "ldr x9, [x11, #0x0]\n" - ".inst 0xe0562349 // ld1h { za1h.h[x13, #1] }, p0/Z, [x26, x22, LSL #1]\n" + ".inst 0xe0572349 // ld1h { za1h.h[x13, #1] }, p0/Z, [x26, x23, LSL #1]\n" ".inst 0x25796580 // psel p0.h, p9.h/Z, p12.h[w13, #3]\n" ".inst 0x25796162 // psel p2.h, p8.h/Z, p11.h[w13, #3]\n" "ldr x26, [x10, #0x0]\n" ".inst 0x25307541 // psel p1.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0562303 // ld1h { za0h.h[x13, #3] }, p0/Z, [x24, x22, LSL #1]\n" + ".inst 0xe0572303 // ld1h { za0h.h[x13, #3] }, p0/Z, [x24, x23, LSL #1]\n" "ldr x24, [x11, #0x8]\n" ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0562aeb // ld1h { za1h.h[x13, #3] }, p2/Z, [x23, x22, LSL #1]\n" - "ldr x23, [x10, #0x8]\n" - ".inst 0xe0bf86a0 // st1w { za0v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" + ".inst 0xe0572aab // ld1h { za1h.h[x13, #3] }, p2/Z, [x21, x23, LSL #1]\n" + "ldr x21, [x10, #0x8]\n" + ".inst 0xe0bf86c0 // st1w { za0v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" - ".inst 0xe0b082a4 // st1w { za1v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" "add x11, x11, #0x10\n" - ".inst 0xe0bc86a1 // st1w { za0v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" + ".inst 0xe0bc86c1 // st1w { za0v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" "add x10, x10, #0x10\n" "add x13, x13, #0x4\n" - ".inst 0xe0bb82a5 // st1w { za1v.s[x12, #1] }, p0/Z, [x21, x27, LSL #2]\n" + ".inst 0xe0bb82c5 // st1w { za1v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" "add x12, x12, #0x2\n" "cmp x12, x15\n" - "addvl x21, x21, #4\n" + "addvl x22, x22, #4\n" "blt 4b\n" "5:" // K loop: Main loop: First: Tail ".inst 0x25396581 // psel p1.h, p9.h/Z, p12.h[w13, #1]\n" ".inst 0x25396160 // psel p0.h, p8.h/Z, p11.h[w13, #1]\n" - ".inst 0xe0562521 // ld1h { za0h.h[x13, #1] }, p1/Z, [x9, x22, LSL #1]\n" - ".inst 0xe0562349 // ld1h { za1h.h[x13, #1] }, p0/Z, [x26, x22, LSL #1]\n" + ".inst 0xe0572521 // ld1h { za0h.h[x13, #1] }, p1/Z, [x9, x23, LSL #1]\n" + ".inst 0xe0572349 // ld1h { za1h.h[x13, #1] }, p0/Z, [x26, x23, LSL #1]\n" "mov x11, %x[in]\n" "add x10, %x[in], x16, LSL #3\n" "ldr x9, [x11, #0x0]\n" ".inst 0x25796580 // psel p0.h, p9.h/Z, p12.h[w13, #3]\n" ".inst 0x25796161 // psel p1.h, p8.h/Z, p11.h[w13, #3]\n" - ".inst 0xe0562303 // ld1h { za0h.h[x13, #3] }, p0/Z, [x24, x22, LSL #1]\n" + ".inst 0xe0572303 // ld1h { za0h.h[x13, #3] }, p0/Z, [x24, x23, LSL #1]\n" "ldr x26, [x10, #0x0]\n" ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe05626eb // ld1h { za1h.h[x13, #3] }, p1/Z, [x23, x22, LSL #1]\n" + ".inst 0xe05726ab // ld1h { za1h.h[x13, #3] }, p1/Z, [x21, x23, LSL #1]\n" "ldr x24, [x11, #0x8]\n" ".inst 0x25307542 // psel p2.s, p13.s/Z, p10.s[w12]\n" - "ldr x23, [x10, #0x8]\n" - ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + "ldr x21, [x10, #0x8]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" - ".inst 0xe0b08aa4 // st1w { za1v.s[x12] }, p2/Z, [x21, x16, LSL #2]\n" + ".inst 0xe0b08ac4 // st1w { za1v.s[x12] }, p2/Z, [x22, x16, LSL #2]\n" "whilelt p10.h, x14, %x[width]\n" "inch x14\n" - ".inst 0xe0bc86a1 // st1w { za0v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" + ".inst 0xe0bc86c1 // st1w { za0v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" "add x11, x11, #0x10\n" "add x10, x10, #0x10\n" - ".inst 0xe0bb82a5 // st1w { za1v.s[x12, #1] }, p0/Z, [x21, x27, LSL #2]\n" - "addvl x21, x21, #4\n" - "inch x22\n" + ".inst 0xe0bb82c5 // st1w { za1v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" + "addvl x22, x22, #4\n" + "inch x23\n" "whilelt p9.h, x14, %x[width]\n" "whilelt p8.h, x14, %x[width]\n" "mov x13, #0x0\n" @@ -183,61 +181,61 @@ void interleave_block<2, 2, VLType::SME, false>( "6:" // K loop: Main loop: Second: Loop ".inst 0x25296581 // psel p1.h, p9.h/Z, p12.h[w13]\n" ".inst 0x25296160 // psel p0.h, p8.h/Z, p11.h[w13]\n" - ".inst 0xe0562520 // ld1h { za0h.h[x13] }, p1/Z, [x9, x22, LSL #1]\n" + ".inst 0xe0572520 // ld1h { za0h.h[x13] }, p1/Z, [x9, x23, LSL #1]\n" "ldr x9, [x11, #0x0]\n" - ".inst 0xe0562348 // ld1h { za1h.h[x13] }, p0/Z, [x26, x22, LSL #1]\n" + ".inst 0xe0572348 // ld1h { za1h.h[x13] }, p0/Z, [x26, x23, LSL #1]\n" ".inst 0x25696580 // psel p0.h, p9.h/Z, p12.h[w13, #2]\n" ".inst 0x25696162 // psel p2.h, p8.h/Z, p11.h[w13, #2]\n" "ldr x26, [x10, #0x0]\n" ".inst 0x25307541 // psel p1.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0562302 // ld1h { za0h.h[x13, #2] }, p0/Z, [x24, x22, LSL #1]\n" + ".inst 0xe0572302 // ld1h { za0h.h[x13, #2] }, p0/Z, [x24, x23, LSL #1]\n" "ldr x24, [x11, #0x8]\n" ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0562aea // ld1h { za1h.h[x13, #2] }, p2/Z, [x23, x22, LSL #1]\n" - "ldr x23, [x10, #0x8]\n" - ".inst 0xe0bf86a8 // st1w { za2v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" + ".inst 0xe0572aaa // ld1h { za1h.h[x13, #2] }, p2/Z, [x21, x23, LSL #1]\n" + "ldr x21, [x10, #0x8]\n" + ".inst 0xe0bf86c8 // st1w { za2v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" - ".inst 0xe0b082ac // st1w { za3v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + ".inst 0xe0b082cc // st1w { za3v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" "add x11, x11, #0x10\n" - ".inst 0xe0bc86a9 // st1w { za2v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" + ".inst 0xe0bc86c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" "add x10, x10, #0x10\n" "add x13, x13, #0x4\n" - ".inst 0xe0bb82ad // st1w { za3v.s[x12, #1] }, p0/Z, [x21, x27, LSL #2]\n" + ".inst 0xe0bb82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" "add x12, x12, #0x2\n" "cmp x12, x15\n" - "addvl x21, x21, #4\n" + "addvl x22, x22, #4\n" "blt 6b\n" "7:" // K loop: Main loop: Second: Tail ".inst 0x25296581 // psel p1.h, p9.h/Z, p12.h[w13]\n" ".inst 0x25296160 // psel p0.h, p8.h/Z, p11.h[w13]\n" - ".inst 0xe0562520 // ld1h { za0h.h[x13] }, p1/Z, [x9, x22, LSL #1]\n" - ".inst 0xe0562348 // ld1h { za1h.h[x13] }, p0/Z, [x26, x22, LSL #1]\n" + ".inst 0xe0572520 // ld1h { za0h.h[x13] }, p1/Z, [x9, x23, LSL #1]\n" + ".inst 0xe0572348 // ld1h { za1h.h[x13] }, p0/Z, [x26, x23, LSL #1]\n" "mov x11, %x[in]\n" "add x10, %x[in], x16, LSL #3\n" "ldr x9, [x11, #0x0]\n" ".inst 0x25696580 // psel p0.h, p9.h/Z, p12.h[w13, #2]\n" ".inst 0x25696161 // psel p1.h, p8.h/Z, p11.h[w13, #2]\n" - ".inst 0xe0562302 // ld1h { za0h.h[x13, #2] }, p0/Z, [x24, x22, LSL #1]\n" + ".inst 0xe0572302 // ld1h { za0h.h[x13, #2] }, p0/Z, [x24, x23, LSL #1]\n" "ldr x26, [x10, #0x0]\n" ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe05626ea // ld1h { za1h.h[x13, #2] }, p1/Z, [x23, x22, LSL #1]\n" + ".inst 0xe05726aa // ld1h { za1h.h[x13, #2] }, p1/Z, [x21, x23, LSL #1]\n" "ldr x24, [x11, #0x8]\n" ".inst 0x25307542 // psel p2.s, p13.s/Z, p10.s[w12]\n" - "ldr x23, [x10, #0x8]\n" - ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + "ldr x21, [x10, #0x8]\n" + ".inst 0xe0bf82c8 // st1w { za2v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" - ".inst 0xe0b08aac // st1w { za3v.s[x12] }, p2/Z, [x21, x16, LSL #2]\n" + ".inst 0xe0b08acc // st1w { za3v.s[x12] }, p2/Z, [x22, x16, LSL #2]\n" "whilelt p10.h, x14, %x[width]\n" "subs x20, x20, #0x1\n" - ".inst 0xe0bc86a9 // st1w { za2v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" + ".inst 0xe0bc86c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" "add x11, x11, #0x10\n" "add x10, x10, #0x10\n" - ".inst 0xe0bb82ad // st1w { za3v.s[x12, #1] }, p0/Z, [x21, x27, LSL #2]\n" - "addvl x21, x21, #4\n" + ".inst 0xe0bb82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x27, LSL #2]\n" + "addvl x22, x22, #4\n" "inch x14\n" - "inch x22\n" + "inch x23\n" "bgt 3b\n" "8:" // K loop: Tails "cbnz x25, 11f\n" @@ -248,51 +246,51 @@ void interleave_block<2, 2, VLType::SME, false>( "mov x12, #0x0\n" "9:" // K loop: Tails: Even: First ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0b082a4 // st1w { za1v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" - "ldr x9, [x11, #0x0]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" + "ldr x21, [x11, #0x0]\n" "add x12, x12, #0x1\n" ".inst 0x25396581 // psel p1.h, p9.h/Z, p12.h[w13, #1]\n" - "ldr x26, [x11, x16, LSL #0x3]\n" + "ldr x20, [x11, x16, LSL #0x3]\n" ".inst 0x25396160 // psel p0.h, p8.h/Z, p11.h[w13, #1]\n" "cmp x12, x16\n" - ".inst 0xe0562521 // ld1h { za0h.h[x13, #1] }, p1/Z, [x9, x22, LSL #1]\n" - ".inst 0xe0562349 // ld1h { za1h.h[x13, #1] }, p0/Z, [x26, x22, LSL #1]\n" + ".inst 0xe05726a1 // ld1h { za0h.h[x13, #1] }, p1/Z, [x21, x23, LSL #1]\n" + ".inst 0xe0572289 // ld1h { za1h.h[x13, #1] }, p0/Z, [x20, x23, LSL #1]\n" "add x11, x11, #0x8\n" - "addvl x21, x21, #2\n" + "addvl x22, x22, #2\n" "add x13, x13, #0x2\n" "blt 9b\n" "whilelt p10.h, x14, %x[width]\n" - "whilelt p9.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" "whilelt p8.h, x14, %x[width]\n" "mov x20, #0x0\n" "mov x12, #0x0\n" "10:" // K loop: Tails: Even: Second ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0xe0bf82c8 // st1w { za2v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0b082ac // st1w { za3v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + ".inst 0xe0b082cc // st1w { za3v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" "add x12, x12, #0x1\n" "cmp x12, x17\n" - "addvl x21, x21, #2\n" + "addvl x22, x22, #2\n" "add x20, x20, #0x2\n" "blt 10b\n" - "whilelt p10.h, x14, %x[width]\n" + "whilelt p8.h, x14, %x[width]\n" "b 13f\n" "11:" // K loop: Tails: Odd "mov x12, #0x0\n" "12:" // K loop: Tails: Odd: Loop ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0b082a4 // st1w { za1v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" "add x12, x12, #0x1\n" "cmp x12, x17\n" - "addvl x21, x21, #2\n" + "addvl x22, x22, #2\n" "blt 12b\n" "13:" // K loop: End - "mov %x[out], x21\n" + "mov %x[out], x22\n" ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) @@ -300,4 +298,4 @@ void interleave_block<2, 2, VLType::SME, false>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8.hpp index 4c701cff1969e747bc8fd02c05ce6d0d8150bf99..905c6b41eb13b9537c5de7d5abbbb67648e81a90 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8.hpp @@ -22,16 +22,14 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME) template <> void interleave_block<2, 4, VLType::SME, false>( int8_t * &out, const int8_t * const *in, - size_t width, size_t height, size_t row_offset, bool first + size_t width, size_t height, size_t row_offset, bool ) { - ARM_COMPUTE_UNUSED(first); - __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" "cntb x21\n" @@ -248,13 +246,13 @@ void interleave_block<2, 4, VLType::SME, false>( ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xe0b082a4 // st1w { za1v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" - "ldr x9, [x11, #0x0]\n" + "ldr x20, [x11, #0x0]\n" ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" - ".inst 0xe0162122 // ld1b { za0h.b[x13, #2] }, p0/Z, [x9, x22]\n" - "ldr x26, [x11, x16, LSL #0x3]\n" + ".inst 0xe0162282 // ld1b { za0h.b[x13, #2] }, p0/Z, [x20, x22]\n" + "ldr x20, [x11, x16, LSL #0x3]\n" "add x12, x12, #0x1\n" ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" - ".inst 0xe0162343 // ld1b { za0h.b[x13, #3] }, p0/Z, [x26, x22]\n" + ".inst 0xe0162283 // ld1b { za0h.b[x13, #3] }, p0/Z, [x20, x22]\n" "cmp x12, x16\n" "add x11, x11, #0x8\n" "addvl x21, x21, #2\n" @@ -274,7 +272,7 @@ void interleave_block<2, 4, VLType::SME, false>( "addvl x21, x21, #2\n" "add x20, x20, #0x4\n" "blt 10b\n" - "whilelt p9.b, x14, %x[width]\n" + "whilelt p8.b, x14, %x[width]\n" "b 13f\n" "11:" // K loop: Tails: Odd "mov x12, #0x0\n" @@ -296,4 +294,4 @@ void interleave_block<2, 4, VLType::SME, false>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8_summing.hpp index 25262d3db99d0df98070114148944f5ec60b1136..c5c5af20e2fa61c23ba2c422df04eea0fa5ca991 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8_summing.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_s8_s8_summing.hpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME) template <> void interleave_block<2, 4, VLType::SME, true>( @@ -140,23 +140,23 @@ void interleave_block<2, 4, VLType::SME, true>( ".inst 0xe01c2aa7 // ld1b { za0h.b[x13, #7] }, p2/Z, [x21, x28]\n" "ldr x21, [x25, #0x8]\n" ".inst 0xe0bf8760 // st1w { za0v.s[x12] }, p1/Z, [x27, XZR, LSL #2]\n" - ".inst 0xc0829010 // mova z16.s, p4/M, za0v.s[x12]\n" - ".inst 0xc0829091 // mova z17.s, p4/M, za1v.s[x12]\n" - "sdot z19.s, z16.b, z20.b\n" + ".inst 0xc0829011 // mova z17.s, p4/M, za0v.s[x12]\n" + ".inst 0xc0829090 // mova z16.s, p4/M, za1v.s[x12]\n" + "sdot z19.s, z17.b, z20.b\n" ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" - "sdot z18.s, z17.b, z20.b\n" + "sdot z18.s, z16.b, z20.b\n" ".inst 0xe0ae8361 // st1w { za0v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" - ".inst 0xc0829030 // mova z16.s, p4/M, za0v.s[x12, #1]\n" + ".inst 0xc0829031 // mova z17.s, p4/M, za0v.s[x12, #1]\n" ".inst 0xe0ab8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x11, LSL #2]\n" - ".inst 0xc08290b1 // mova z17.s, p4/M, za1v.s[x12, #1]\n" + ".inst 0xc08290b0 // mova z16.s, p4/M, za1v.s[x12, #1]\n" "add x12, x12, #0x2\n" "cmp x12, x9\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" - "sdot z19.s, z16.b, z20.b\n" - "sdot z18.s, z17.b, z20.b\n" + "sdot z19.s, z17.b, z20.b\n" + "sdot z18.s, z16.b, z20.b\n" "addvl x27, x27, #4\n" "add x13, x13, #0x8\n" "blt 5b\n" @@ -172,28 +172,28 @@ void interleave_block<2, 4, VLType::SME, true>( "add x25, %x[in], x16, LSL #3\n" "ldr x24, [x26, #0x0]\n" ".inst 0xe01c22a7 // ld1b { za0h.b[x13, #7] }, p0/Z, [x21, x28]\n" - ".inst 0xc0829010 // mova z16.s, p4/M, za0v.s[x12]\n" + ".inst 0xc0829011 // mova z17.s, p4/M, za0v.s[x12]\n" ".inst 0x25306d23 // psel p3.s, p11.s/Z, p9.s[w12]\n" - "sdot z19.s, z16.b, z20.b\n" - ".inst 0xc0829091 // mova z17.s, p4/M, za1v.s[x12]\n" - "sdot z18.s, z17.b, z20.b\n" + "sdot z19.s, z17.b, z20.b\n" + ".inst 0xc0829090 // mova z16.s, p4/M, za1v.s[x12]\n" + "sdot z18.s, z16.b, z20.b\n" "ldr x23, [x25, #0x0]\n" ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" "ldr x22, [x26, #0x8]\n" ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" - ".inst 0xc0829030 // mova z16.s, p4/M, za0v.s[x12, #1]\n" + ".inst 0xc0829031 // mova z17.s, p4/M, za0v.s[x12, #1]\n" ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" "ldr x21, [x25, #0x8]\n" ".inst 0xe0bf8f60 // st1w { za0v.s[x12] }, p3/Z, [x27, XZR, LSL #2]\n" - ".inst 0xc08290b1 // mova z17.s, p4/M, za1v.s[x12, #1]\n" + ".inst 0xc08290b0 // mova z16.s, p4/M, za1v.s[x12, #1]\n" "whilelt p9.b, x15, %x[width]\n" ".inst 0xe0b08b64 // st1w { za1v.s[x12] }, p2/Z, [x27, x16, LSL #2]\n" "incb x15\n" "add x26, x26, #0x10\n" - "sdot z19.s, z16.b, z20.b\n" + "sdot z19.s, z17.b, z20.b\n" ".inst 0xe0ae8761 // st1w { za0v.s[x12, #1] }, p1/Z, [x27, x14, LSL #2]\n" "add x25, x25, #0x10\n" - "sdot z18.s, z17.b, z20.b\n" + "sdot z18.s, z16.b, z20.b\n" "incb x28\n" ".inst 0xe0ab8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x11, LSL #2]\n" "addvl x27, x27, #4\n" @@ -217,23 +217,23 @@ void interleave_block<2, 4, VLType::SME, true>( ".inst 0xe01c2aa5 // ld1b { za0h.b[x13, #5] }, p2/Z, [x21, x28]\n" "ldr x21, [x25, #0x8]\n" ".inst 0xe0bf8768 // st1w { za2v.s[x12] }, p1/Z, [x27, XZR, LSL #2]\n" - ".inst 0xc0829110 // mova z16.s, p4/M, za2v.s[x12]\n" - ".inst 0xc0829191 // mova z17.s, p4/M, za3v.s[x12]\n" - "sdot z19.s, z16.b, z20.b\n" + ".inst 0xc0829111 // mova z17.s, p4/M, za2v.s[x12]\n" + ".inst 0xc0829190 // mova z16.s, p4/M, za3v.s[x12]\n" + "sdot z19.s, z17.b, z20.b\n" ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" - "sdot z18.s, z17.b, z20.b\n" + "sdot z18.s, z16.b, z20.b\n" ".inst 0xe0ae8369 // st1w { za2v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" - ".inst 0xc0829130 // mova z16.s, p4/M, za2v.s[x12, #1]\n" + ".inst 0xc0829131 // mova z17.s, p4/M, za2v.s[x12, #1]\n" ".inst 0xe0ab836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x11, LSL #2]\n" - ".inst 0xc08291b1 // mova z17.s, p4/M, za3v.s[x12, #1]\n" + ".inst 0xc08291b0 // mova z16.s, p4/M, za3v.s[x12, #1]\n" "add x12, x12, #0x2\n" "cmp x12, x9\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" - "sdot z19.s, z16.b, z20.b\n" - "sdot z18.s, z17.b, z20.b\n" + "sdot z19.s, z17.b, z20.b\n" + "sdot z18.s, z16.b, z20.b\n" "addvl x27, x27, #4\n" "add x13, x13, #0x8\n" "blt 7b\n" @@ -249,28 +249,28 @@ void interleave_block<2, 4, VLType::SME, true>( "add x25, %x[in], x16, LSL #3\n" "ldr x24, [x26, #0x0]\n" ".inst 0xe01c22a5 // ld1b { za0h.b[x13, #5] }, p0/Z, [x21, x28]\n" - ".inst 0xc0829110 // mova z16.s, p4/M, za2v.s[x12]\n" + ".inst 0xc0829111 // mova z17.s, p4/M, za2v.s[x12]\n" ".inst 0x25306d23 // psel p3.s, p11.s/Z, p9.s[w12]\n" - "sdot z19.s, z16.b, z20.b\n" - ".inst 0xc0829191 // mova z17.s, p4/M, za3v.s[x12]\n" - "sdot z18.s, z17.b, z20.b\n" + "sdot z19.s, z17.b, z20.b\n" + ".inst 0xc0829190 // mova z16.s, p4/M, za3v.s[x12]\n" + "sdot z18.s, z16.b, z20.b\n" "ldr x23, [x25, #0x0]\n" ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" "ldr x22, [x26, #0x8]\n" ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" - ".inst 0xc0829130 // mova z16.s, p4/M, za2v.s[x12, #1]\n" + ".inst 0xc0829131 // mova z17.s, p4/M, za2v.s[x12, #1]\n" ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" "ldr x21, [x25, #0x8]\n" ".inst 0xe0bf8f68 // st1w { za2v.s[x12] }, p3/Z, [x27, XZR, LSL #2]\n" - ".inst 0xc08291b1 // mova z17.s, p4/M, za3v.s[x12, #1]\n" + ".inst 0xc08291b0 // mova z16.s, p4/M, za3v.s[x12, #1]\n" "whilelt p9.b, x15, %x[width]\n" ".inst 0xe0b08b6c // st1w { za3v.s[x12] }, p2/Z, [x27, x16, LSL #2]\n" "subs x20, x20, #0x1\n" "add x26, x26, #0x10\n" - "sdot z19.s, z16.b, z20.b\n" + "sdot z19.s, z17.b, z20.b\n" ".inst 0xe0ae8769 // st1w { za2v.s[x12, #1] }, p1/Z, [x27, x14, LSL #2]\n" "add x25, x25, #0x10\n" - "sdot z18.s, z17.b, z20.b\n" + "sdot z18.s, z16.b, z20.b\n" "incb x15\n" ".inst 0xe0ab836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x11, LSL #2]\n" "addvl x27, x27, #4\n" @@ -286,19 +286,19 @@ void interleave_block<2, 4, VLType::SME, true>( ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - ".inst 0xc0829010 // mova z16.s, p4/M, za0v.s[x12]\n" + ".inst 0xc0829011 // mova z17.s, p4/M, za0v.s[x12]\n" ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" - "ldr x24, [x26, #0x0]\n" + "ldr x21, [x26, #0x0]\n" ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" - ".inst 0xc0829091 // mova z17.s, p4/M, za1v.s[x12]\n" - "ldr x23, [x26, x16, LSL #0x3]\n" - ".inst 0xe01c2302 // ld1b { za0h.b[x13, #2] }, p0/Z, [x24, x28]\n" + ".inst 0xc0829090 // mova z16.s, p4/M, za1v.s[x12]\n" + "ldr x20, [x26, x16, LSL #0x3]\n" + ".inst 0xe01c22a2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x21, x28]\n" "add x12, x12, #0x1\n" ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" "cmp x12, x16\n" - "sdot z19.s, z16.b, z20.b\n" - "sdot z18.s, z17.b, z20.b\n" - ".inst 0xe01c22e3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x23, x28]\n" + "sdot z19.s, z17.b, z20.b\n" + "sdot z18.s, z16.b, z20.b\n" + ".inst 0xe01c2283 // ld1b { za0h.b[x13, #3] }, p0/Z, [x20, x28]\n" "add x26, x26, #0x8\n" "addvl x27, x27, #2\n" "add x13, x13, #0x4\n" @@ -311,17 +311,17 @@ void interleave_block<2, 4, VLType::SME, true>( ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - ".inst 0xc0829110 // mova z16.s, p4/M, za2v.s[x12]\n" + ".inst 0xc0829111 // mova z17.s, p4/M, za2v.s[x12]\n" ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" - ".inst 0xc0829191 // mova z17.s, p4/M, za3v.s[x12]\n" + ".inst 0xc0829190 // mova z16.s, p4/M, za3v.s[x12]\n" "add x12, x12, #0x1\n" "cmp x12, x17\n" - "sdot z19.s, z16.b, z20.b\n" - "sdot z18.s, z17.b, z20.b\n" + "sdot z19.s, z17.b, z20.b\n" + "sdot z18.s, z16.b, z20.b\n" "addvl x27, x27, #2\n" "add x20, x20, #0x4\n" "blt 11b\n" - "whilelt p9.b, x15, %x[width]\n" + "whilelt p8.b, x15, %x[width]\n" "b 14f\n" "12:" // K loop: Tails: Odd "mov x12, #0x0\n" @@ -329,13 +329,13 @@ void interleave_block<2, 4, VLType::SME, true>( ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - ".inst 0xc0829010 // mova z16.s, p4/M, za0v.s[x12]\n" + ".inst 0xc0829011 // mova z17.s, p4/M, za0v.s[x12]\n" ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" - ".inst 0xc0829091 // mova z17.s, p4/M, za1v.s[x12]\n" + ".inst 0xc0829090 // mova z16.s, p4/M, za1v.s[x12]\n" "add x12, x12, #0x1\n" "cmp x12, x17\n" - "sdot z19.s, z16.b, z20.b\n" - "sdot z18.s, z17.b, z20.b\n" + "sdot z19.s, z17.b, z20.b\n" + "sdot z18.s, z16.b, z20.b\n" "addvl x27, x27, #2\n" "blt 13b\n" "14:" // K loop: End @@ -350,4 +350,4 @@ void interleave_block<2, 4, VLType::SME, true>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8.hpp index 683a315a968922328906fbfcbdd4a8848ecb6ac6..ce9a0065c7e8ef0c13b7b82a568c30bfa4cf56b1 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8.hpp @@ -22,16 +22,14 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME) template <> void interleave_block<2, 4, VLType::SME, false>( uint8_t * &out, const uint8_t * const *in, - size_t width, size_t height, size_t row_offset, bool first + size_t width, size_t height, size_t row_offset, bool ) { - ARM_COMPUTE_UNUSED(first); - __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" "cntb x21\n" @@ -248,13 +246,13 @@ void interleave_block<2, 4, VLType::SME, false>( ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xe0b082a4 // st1w { za1v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" - "ldr x9, [x11, #0x0]\n" + "ldr x20, [x11, #0x0]\n" ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" - ".inst 0xe0162122 // ld1b { za0h.b[x13, #2] }, p0/Z, [x9, x22]\n" - "ldr x26, [x11, x16, LSL #0x3]\n" + ".inst 0xe0162282 // ld1b { za0h.b[x13, #2] }, p0/Z, [x20, x22]\n" + "ldr x20, [x11, x16, LSL #0x3]\n" "add x12, x12, #0x1\n" ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" - ".inst 0xe0162343 // ld1b { za0h.b[x13, #3] }, p0/Z, [x26, x22]\n" + ".inst 0xe0162283 // ld1b { za0h.b[x13, #3] }, p0/Z, [x20, x22]\n" "cmp x12, x16\n" "add x11, x11, #0x8\n" "addvl x21, x21, #2\n" @@ -274,7 +272,7 @@ void interleave_block<2, 4, VLType::SME, false>( "addvl x21, x21, #2\n" "add x20, x20, #0x4\n" "blt 10b\n" - "whilelt p9.b, x14, %x[width]\n" + "whilelt p8.b, x14, %x[width]\n" "b 13f\n" "11:" // K loop: Tails: Odd "mov x12, #0x0\n" @@ -296,4 +294,4 @@ void interleave_block<2, 4, VLType::SME, false>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8_summing.hpp index e7571f7da766600f6034f7772e01df64e7967d5d..7805152656fc22080c0c9e0da68f618e82632579 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8_summing.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_block4_u8_u8_summing.hpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME) template <> void interleave_block<2, 4, VLType::SME, true>( @@ -140,23 +140,23 @@ void interleave_block<2, 4, VLType::SME, true>( ".inst 0xe01c2aa7 // ld1b { za0h.b[x13, #7] }, p2/Z, [x21, x28]\n" "ldr x21, [x25, #0x8]\n" ".inst 0xe0bf8760 // st1w { za0v.s[x12] }, p1/Z, [x27, XZR, LSL #2]\n" - ".inst 0xc0829011 // mova z17.s, p4/M, za0v.s[x12]\n" - ".inst 0xc0829090 // mova z16.s, p4/M, za1v.s[x12]\n" - "udot z19.s, z17.b, z20.b\n" + ".inst 0xc0829010 // mova z16.s, p4/M, za0v.s[x12]\n" + ".inst 0xc0829091 // mova z17.s, p4/M, za1v.s[x12]\n" + "udot z19.s, z16.b, z20.b\n" ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" - "udot z18.s, z16.b, z20.b\n" + "udot z18.s, z17.b, z20.b\n" ".inst 0xe0ae8361 // st1w { za0v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" - ".inst 0xc0829031 // mova z17.s, p4/M, za0v.s[x12, #1]\n" + ".inst 0xc0829030 // mova z16.s, p4/M, za0v.s[x12, #1]\n" ".inst 0xe0ab8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x11, LSL #2]\n" - ".inst 0xc08290b0 // mova z16.s, p4/M, za1v.s[x12, #1]\n" + ".inst 0xc08290b1 // mova z17.s, p4/M, za1v.s[x12, #1]\n" "add x12, x12, #0x2\n" "cmp x12, x9\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" - "udot z19.s, z17.b, z20.b\n" - "udot z18.s, z16.b, z20.b\n" + "udot z19.s, z16.b, z20.b\n" + "udot z18.s, z17.b, z20.b\n" "addvl x27, x27, #4\n" "add x13, x13, #0x8\n" "blt 5b\n" @@ -172,28 +172,28 @@ void interleave_block<2, 4, VLType::SME, true>( "add x25, %x[in], x16, LSL #3\n" "ldr x24, [x26, #0x0]\n" ".inst 0xe01c22a7 // ld1b { za0h.b[x13, #7] }, p0/Z, [x21, x28]\n" - ".inst 0xc0829011 // mova z17.s, p4/M, za0v.s[x12]\n" + ".inst 0xc0829010 // mova z16.s, p4/M, za0v.s[x12]\n" ".inst 0x25306d23 // psel p3.s, p11.s/Z, p9.s[w12]\n" - "udot z19.s, z17.b, z20.b\n" - ".inst 0xc0829090 // mova z16.s, p4/M, za1v.s[x12]\n" - "udot z18.s, z16.b, z20.b\n" + "udot z19.s, z16.b, z20.b\n" + ".inst 0xc0829091 // mova z17.s, p4/M, za1v.s[x12]\n" + "udot z18.s, z17.b, z20.b\n" "ldr x23, [x25, #0x0]\n" ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" "ldr x22, [x26, #0x8]\n" ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" - ".inst 0xc0829031 // mova z17.s, p4/M, za0v.s[x12, #1]\n" + ".inst 0xc0829030 // mova z16.s, p4/M, za0v.s[x12, #1]\n" ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" "ldr x21, [x25, #0x8]\n" ".inst 0xe0bf8f60 // st1w { za0v.s[x12] }, p3/Z, [x27, XZR, LSL #2]\n" - ".inst 0xc08290b0 // mova z16.s, p4/M, za1v.s[x12, #1]\n" + ".inst 0xc08290b1 // mova z17.s, p4/M, za1v.s[x12, #1]\n" "whilelt p9.b, x15, %x[width]\n" ".inst 0xe0b08b64 // st1w { za1v.s[x12] }, p2/Z, [x27, x16, LSL #2]\n" "incb x15\n" "add x26, x26, #0x10\n" - "udot z19.s, z17.b, z20.b\n" + "udot z19.s, z16.b, z20.b\n" ".inst 0xe0ae8761 // st1w { za0v.s[x12, #1] }, p1/Z, [x27, x14, LSL #2]\n" "add x25, x25, #0x10\n" - "udot z18.s, z16.b, z20.b\n" + "udot z18.s, z17.b, z20.b\n" "incb x28\n" ".inst 0xe0ab8365 // st1w { za1v.s[x12, #1] }, p0/Z, [x27, x11, LSL #2]\n" "addvl x27, x27, #4\n" @@ -217,23 +217,23 @@ void interleave_block<2, 4, VLType::SME, true>( ".inst 0xe01c2aa5 // ld1b { za0h.b[x13, #5] }, p2/Z, [x21, x28]\n" "ldr x21, [x25, #0x8]\n" ".inst 0xe0bf8768 // st1w { za2v.s[x12] }, p1/Z, [x27, XZR, LSL #2]\n" - ".inst 0xc0829111 // mova z17.s, p4/M, za2v.s[x12]\n" - ".inst 0xc0829190 // mova z16.s, p4/M, za3v.s[x12]\n" - "udot z19.s, z17.b, z20.b\n" + ".inst 0xc0829110 // mova z16.s, p4/M, za2v.s[x12]\n" + ".inst 0xc0829191 // mova z17.s, p4/M, za3v.s[x12]\n" + "udot z19.s, z16.b, z20.b\n" ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" - "udot z18.s, z16.b, z20.b\n" + "udot z18.s, z17.b, z20.b\n" ".inst 0xe0ae8369 // st1w { za2v.s[x12, #1] }, p0/Z, [x27, x14, LSL #2]\n" ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" - ".inst 0xc0829131 // mova z17.s, p4/M, za2v.s[x12, #1]\n" + ".inst 0xc0829130 // mova z16.s, p4/M, za2v.s[x12, #1]\n" ".inst 0xe0ab836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x11, LSL #2]\n" - ".inst 0xc08291b0 // mova z16.s, p4/M, za3v.s[x12, #1]\n" + ".inst 0xc08291b1 // mova z17.s, p4/M, za3v.s[x12, #1]\n" "add x12, x12, #0x2\n" "cmp x12, x9\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" - "udot z19.s, z17.b, z20.b\n" - "udot z18.s, z16.b, z20.b\n" + "udot z19.s, z16.b, z20.b\n" + "udot z18.s, z17.b, z20.b\n" "addvl x27, x27, #4\n" "add x13, x13, #0x8\n" "blt 7b\n" @@ -249,28 +249,28 @@ void interleave_block<2, 4, VLType::SME, true>( "add x25, %x[in], x16, LSL #3\n" "ldr x24, [x26, #0x0]\n" ".inst 0xe01c22a5 // ld1b { za0h.b[x13, #5] }, p0/Z, [x21, x28]\n" - ".inst 0xc0829111 // mova z17.s, p4/M, za2v.s[x12]\n" + ".inst 0xc0829110 // mova z16.s, p4/M, za2v.s[x12]\n" ".inst 0x25306d23 // psel p3.s, p11.s/Z, p9.s[w12]\n" - "udot z19.s, z17.b, z20.b\n" - ".inst 0xc0829190 // mova z16.s, p4/M, za3v.s[x12]\n" - "udot z18.s, z16.b, z20.b\n" + "udot z19.s, z16.b, z20.b\n" + ".inst 0xc0829191 // mova z17.s, p4/M, za3v.s[x12]\n" + "udot z18.s, z17.b, z20.b\n" "ldr x23, [x25, #0x0]\n" ".inst 0x25306d22 // psel p2.s, p11.s/Z, p9.s[w12]\n" "ldr x22, [x26, #0x8]\n" ".inst 0x25706d21 // psel p1.s, p11.s/Z, p9.s[w12, #1]\n" - ".inst 0xc0829131 // mova z17.s, p4/M, za2v.s[x12, #1]\n" + ".inst 0xc0829130 // mova z16.s, p4/M, za2v.s[x12, #1]\n" ".inst 0x25706d20 // psel p0.s, p11.s/Z, p9.s[w12, #1]\n" "ldr x21, [x25, #0x8]\n" ".inst 0xe0bf8f68 // st1w { za2v.s[x12] }, p3/Z, [x27, XZR, LSL #2]\n" - ".inst 0xc08291b0 // mova z16.s, p4/M, za3v.s[x12, #1]\n" + ".inst 0xc08291b1 // mova z17.s, p4/M, za3v.s[x12, #1]\n" "whilelt p9.b, x15, %x[width]\n" ".inst 0xe0b08b6c // st1w { za3v.s[x12] }, p2/Z, [x27, x16, LSL #2]\n" "subs x20, x20, #0x1\n" "add x26, x26, #0x10\n" - "udot z19.s, z17.b, z20.b\n" + "udot z19.s, z16.b, z20.b\n" ".inst 0xe0ae8769 // st1w { za2v.s[x12, #1] }, p1/Z, [x27, x14, LSL #2]\n" "add x25, x25, #0x10\n" - "udot z18.s, z16.b, z20.b\n" + "udot z18.s, z17.b, z20.b\n" "incb x15\n" ".inst 0xe0ab836d // st1w { za3v.s[x12, #1] }, p0/Z, [x27, x11, LSL #2]\n" "addvl x27, x27, #4\n" @@ -286,19 +286,19 @@ void interleave_block<2, 4, VLType::SME, true>( ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - ".inst 0xc0829011 // mova z17.s, p4/M, za0v.s[x12]\n" + ".inst 0xc0829010 // mova z16.s, p4/M, za0v.s[x12]\n" ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" - "ldr x24, [x26, #0x0]\n" + "ldr x21, [x26, #0x0]\n" ".inst 0x25356140 // psel p0.b, p8.b/Z, p10.b[w13, #2]\n" - ".inst 0xc0829090 // mova z16.s, p4/M, za1v.s[x12]\n" - "ldr x23, [x26, x16, LSL #0x3]\n" - ".inst 0xe01c2302 // ld1b { za0h.b[x13, #2] }, p0/Z, [x24, x28]\n" + ".inst 0xc0829091 // mova z17.s, p4/M, za1v.s[x12]\n" + "ldr x20, [x26, x16, LSL #0x3]\n" + ".inst 0xe01c22a2 // ld1b { za0h.b[x13, #2] }, p0/Z, [x21, x28]\n" "add x12, x12, #0x1\n" ".inst 0x253d6140 // psel p0.b, p8.b/Z, p10.b[w13, #3]\n" "cmp x12, x16\n" - "udot z19.s, z17.b, z20.b\n" - "udot z18.s, z16.b, z20.b\n" - ".inst 0xe01c22e3 // ld1b { za0h.b[x13, #3] }, p0/Z, [x23, x28]\n" + "udot z19.s, z16.b, z20.b\n" + "udot z18.s, z17.b, z20.b\n" + ".inst 0xe01c2283 // ld1b { za0h.b[x13, #3] }, p0/Z, [x20, x28]\n" "add x26, x26, #0x8\n" "addvl x27, x27, #2\n" "add x13, x13, #0x4\n" @@ -311,17 +311,17 @@ void interleave_block<2, 4, VLType::SME, true>( ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xe0bf8368 // st1w { za2v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - ".inst 0xc0829111 // mova z17.s, p4/M, za2v.s[x12]\n" + ".inst 0xc0829110 // mova z16.s, p4/M, za2v.s[x12]\n" ".inst 0xe0b0836c // st1w { za3v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" - ".inst 0xc0829190 // mova z16.s, p4/M, za3v.s[x12]\n" + ".inst 0xc0829191 // mova z17.s, p4/M, za3v.s[x12]\n" "add x12, x12, #0x1\n" "cmp x12, x17\n" - "udot z19.s, z17.b, z20.b\n" - "udot z18.s, z16.b, z20.b\n" + "udot z19.s, z16.b, z20.b\n" + "udot z18.s, z17.b, z20.b\n" "addvl x27, x27, #2\n" "add x20, x20, #0x4\n" "blt 11b\n" - "whilelt p9.b, x15, %x[width]\n" + "whilelt p8.b, x15, %x[width]\n" "b 14f\n" "12:" // K loop: Tails: Odd "mov x12, #0x0\n" @@ -329,13 +329,13 @@ void interleave_block<2, 4, VLType::SME, true>( ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xe0bf8360 // st1w { za0v.s[x12] }, p0/Z, [x27, XZR, LSL #2]\n" ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - ".inst 0xc0829011 // mova z17.s, p4/M, za0v.s[x12]\n" + ".inst 0xc0829010 // mova z16.s, p4/M, za0v.s[x12]\n" ".inst 0xe0b08364 // st1w { za1v.s[x12] }, p0/Z, [x27, x16, LSL #2]\n" - ".inst 0xc0829090 // mova z16.s, p4/M, za1v.s[x12]\n" + ".inst 0xc0829091 // mova z17.s, p4/M, za1v.s[x12]\n" "add x12, x12, #0x1\n" "cmp x12, x17\n" - "udot z19.s, z17.b, z20.b\n" - "udot z18.s, z16.b, z20.b\n" + "udot z19.s, z16.b, z20.b\n" + "udot z18.s, z17.b, z20.b\n" "addvl x27, x27, #2\n" "blt 13b\n" "14:" // K loop: End @@ -350,4 +350,4 @@ void interleave_block<2, 4, VLType::SME, true>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp16_fp16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp16_fp16.hpp index 522f310cc066b4c30cfda8cbcfcf8e89ea95641a..96ab55ee060eb3492e0fb6f44d3685a48d80697c 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp16_fp16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp16_fp16.hpp @@ -22,16 +22,14 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME) template <> void interleave_block<2, 1, VLType::SME, false>( __fp16 * &out, const __fp16 * const *in, - size_t width, size_t height, size_t row_offset, bool first + size_t width, size_t height, size_t row_offset, bool ) { - ARM_COMPUTE_UNUSED(first); - __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" "cnth x28\n" @@ -97,4 +95,4 @@ void interleave_block<2, 1, VLType::SME, false>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp32_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp32_fp32.hpp index 949e003598959ca6836275ec4164e09fe6788d37..ac4b1b5086afd9159ba9766aa55dddc05c3a040e 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp32_fp32.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave2VL_fp32_fp32.hpp @@ -22,16 +22,14 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME) template <> void interleave_block<2, 1, VLType::SME, false>( float * &out, const float * const *in, - size_t width, size_t height, size_t row_offset, bool first + size_t width, size_t height, size_t row_offset, bool ) { - ARM_COMPUTE_UNUSED(first); - __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" "mov x22, %x[width]\n" @@ -55,12 +53,12 @@ void interleave_block<2, 1, VLType::SME, false>( "ldr x25, [x11, #0x8]\n" "and x24, x22, #0x1\n" // odd_tail = bool(n_passes & 0x1) "csel x15, x15, x16, NE\n" - "ldr x23, [x9, #0x8]\n" + "ldr x21, [x9, #0x8]\n" "ptrue p13.s\n" "whilelt p12.s, XZR, %x[height]\n" "whilelt p11.s, x16, %x[height]\n" - "mov x22, %x[row_offset]\n" - "mov x21, %x[out]\n" + "mov x23, %x[row_offset]\n" + "mov x22, %x[out]\n" "whilelt p10.s, x13, %x[width]\n" "whilelt p9.s, x13, %x[width]\n" "whilelt p8.s, x13, %x[width]\n" @@ -71,39 +69,39 @@ void interleave_block<2, 1, VLType::SME, false>( "1:" // K loop: Charge: Loop ".inst 0x25306581 // psel p1.s, p9.s/Z, p12.s[w12]\n" ".inst 0x25306160 // psel p0.s, p8.s/Z, p11.s[w12]\n" - ".inst 0xe0960540 // ld1w { za0h.s[x12] }, p1/Z, [x10, x22, LSL #2]\n" + ".inst 0xe0970540 // ld1w { za0h.s[x12] }, p1/Z, [x10, x23, LSL #2]\n" "ldr x10, [x11, #0x0]\n" - ".inst 0xe0960364 // ld1w { za1h.s[x12] }, p0/Z, [x27, x22, LSL #2]\n" + ".inst 0xe0970364 // ld1w { za1h.s[x12] }, p0/Z, [x27, x23, LSL #2]\n" ".inst 0x25706581 // psel p1.s, p9.s/Z, p12.s[w12, #1]\n" ".inst 0x25706160 // psel p0.s, p8.s/Z, p11.s[w12, #1]\n" "ldr x27, [x9, #0x0]\n" - ".inst 0xe0960721 // ld1w { za0h.s[x12, #1] }, p1/Z, [x25, x22, LSL #2]\n" + ".inst 0xe0970721 // ld1w { za0h.s[x12, #1] }, p1/Z, [x25, x23, LSL #2]\n" "ldr x25, [x11, #0x8]\n" "add x11, x11, #0x10\n" - ".inst 0xe09602e5 // ld1w { za1h.s[x12, #1] }, p0/Z, [x23, x22, LSL #2]\n" + ".inst 0xe09702a5 // ld1w { za1h.s[x12, #1] }, p0/Z, [x21, x23, LSL #2]\n" "add x12, x12, #0x2\n" "cmp x12, x14\n" - "ldr x23, [x9, #0x8]\n" + "ldr x21, [x9, #0x8]\n" "add x9, x9, #0x10\n" "blt 1b\n" "2:" // K loop: Charge: End ".inst 0x25306581 // psel p1.s, p9.s/Z, p12.s[w12]\n" ".inst 0x25306160 // psel p0.s, p8.s/Z, p11.s[w12]\n" - ".inst 0xe0960540 // ld1w { za0h.s[x12] }, p1/Z, [x10, x22, LSL #2]\n" - ".inst 0xe0960364 // ld1w { za1h.s[x12] }, p0/Z, [x27, x22, LSL #2]\n" + ".inst 0xe0970540 // ld1w { za0h.s[x12] }, p1/Z, [x10, x23, LSL #2]\n" + ".inst 0xe0970364 // ld1w { za1h.s[x12] }, p0/Z, [x27, x23, LSL #2]\n" ".inst 0x25706581 // psel p1.s, p9.s/Z, p12.s[w12, #1]\n" ".inst 0x25706160 // psel p0.s, p8.s/Z, p11.s[w12, #1]\n" "mov x11, %x[in]\n" "add x9, %x[in], x16, LSL #3\n" - ".inst 0xe0960721 // ld1w { za0h.s[x12, #1] }, p1/Z, [x25, x22, LSL #2]\n" + ".inst 0xe0970721 // ld1w { za0h.s[x12, #1] }, p1/Z, [x25, x23, LSL #2]\n" "ldr x10, [x11, #0x0]\n" - ".inst 0xe09602e5 // ld1w { za1h.s[x12, #1] }, p0/Z, [x23, x22, LSL #2]\n" + ".inst 0xe09702a5 // ld1w { za1h.s[x12, #1] }, p0/Z, [x21, x23, LSL #2]\n" "ldr x27, [x9, #0x0]\n" - "incw x22\n" + "incw x23\n" "incw x13\n" "ldr x25, [x11, #0x8]\n" "add x11, x11, #0x10\n" - "ldr x23, [x9, #0x8]\n" + "ldr x21, [x9, #0x8]\n" "add x9, x9, #0x10\n" "cbz x20, 8f\n" "mov x20, x20\n" @@ -115,59 +113,59 @@ void interleave_block<2, 1, VLType::SME, false>( "4:" // K loop: Main loop: First: Loop ".inst 0x25306581 // psel p1.s, p9.s/Z, p12.s[w12]\n" ".inst 0x25306160 // psel p0.s, p8.s/Z, p11.s[w12]\n" - ".inst 0xe0960548 // ld1w { za2h.s[x12] }, p1/Z, [x10, x22, LSL #2]\n" + ".inst 0xe0970548 // ld1w { za2h.s[x12] }, p1/Z, [x10, x23, LSL #2]\n" "ldr x10, [x11, #0x0]\n" - ".inst 0xe096036c // ld1w { za3h.s[x12] }, p0/Z, [x27, x22, LSL #2]\n" + ".inst 0xe097036c // ld1w { za3h.s[x12] }, p0/Z, [x27, x23, LSL #2]\n" ".inst 0x25706580 // psel p0.s, p9.s/Z, p12.s[w12, #1]\n" ".inst 0x25706162 // psel p2.s, p8.s/Z, p11.s[w12, #1]\n" "ldr x27, [x9, #0x0]\n" ".inst 0x25307541 // psel p1.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0960329 // ld1w { za2h.s[x12, #1] }, p0/Z, [x25, x22, LSL #2]\n" + ".inst 0xe0970329 // ld1w { za2h.s[x12, #1] }, p0/Z, [x25, x23, LSL #2]\n" "ldr x25, [x11, #0x8]\n" ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0960aed // ld1w { za3h.s[x12, #1] }, p2/Z, [x23, x22, LSL #2]\n" - "ldr x23, [x9, #0x8]\n" - ".inst 0xe0bf86a0 // st1w { za0v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" + ".inst 0xe0970aad // ld1w { za3h.s[x12, #1] }, p2/Z, [x21, x23, LSL #2]\n" + "ldr x21, [x9, #0x8]\n" + ".inst 0xe0bf86c0 // st1w { za0v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" - ".inst 0xe0b082a4 // st1w { za1v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" "add x11, x11, #0x10\n" - ".inst 0xe0bc86a1 // st1w { za0v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" + ".inst 0xe0bc86c1 // st1w { za0v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" "add x9, x9, #0x10\n" - ".inst 0xe0ba82a5 // st1w { za1v.s[x12, #1] }, p0/Z, [x21, x26, LSL #2]\n" + ".inst 0xe0ba82c5 // st1w { za1v.s[x12, #1] }, p0/Z, [x22, x26, LSL #2]\n" "add x12, x12, #0x2\n" "cmp x12, x14\n" - "addvl x21, x21, #4\n" + "addvl x22, x22, #4\n" "blt 4b\n" "5:" // K loop: Main loop: First: Tail ".inst 0x25306581 // psel p1.s, p9.s/Z, p12.s[w12]\n" ".inst 0x25306160 // psel p0.s, p8.s/Z, p11.s[w12]\n" - ".inst 0xe0960548 // ld1w { za2h.s[x12] }, p1/Z, [x10, x22, LSL #2]\n" - ".inst 0xe096036c // ld1w { za3h.s[x12] }, p0/Z, [x27, x22, LSL #2]\n" + ".inst 0xe0970548 // ld1w { za2h.s[x12] }, p1/Z, [x10, x23, LSL #2]\n" + ".inst 0xe097036c // ld1w { za3h.s[x12] }, p0/Z, [x27, x23, LSL #2]\n" "mov x11, %x[in]\n" "add x9, %x[in], x16, LSL #3\n" "ldr x10, [x11, #0x0]\n" ".inst 0x25706580 // psel p0.s, p9.s/Z, p12.s[w12, #1]\n" ".inst 0x25706161 // psel p1.s, p8.s/Z, p11.s[w12, #1]\n" - ".inst 0xe0960329 // ld1w { za2h.s[x12, #1] }, p0/Z, [x25, x22, LSL #2]\n" + ".inst 0xe0970329 // ld1w { za2h.s[x12, #1] }, p0/Z, [x25, x23, LSL #2]\n" "ldr x27, [x9, #0x0]\n" ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe09606ed // ld1w { za3h.s[x12, #1] }, p1/Z, [x23, x22, LSL #2]\n" + ".inst 0xe09706ad // ld1w { za3h.s[x12, #1] }, p1/Z, [x21, x23, LSL #2]\n" "ldr x25, [x11, #0x8]\n" ".inst 0x25307542 // psel p2.s, p13.s/Z, p10.s[w12]\n" - "ldr x23, [x9, #0x8]\n" - ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + "ldr x21, [x9, #0x8]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" - ".inst 0xe0b08aa4 // st1w { za1v.s[x12] }, p2/Z, [x21, x16, LSL #2]\n" + ".inst 0xe0b08ac4 // st1w { za1v.s[x12] }, p2/Z, [x22, x16, LSL #2]\n" "whilelt p10.s, x13, %x[width]\n" "incw x13\n" - ".inst 0xe0bc86a1 // st1w { za0v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" + ".inst 0xe0bc86c1 // st1w { za0v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" "add x11, x11, #0x10\n" "add x9, x9, #0x10\n" - ".inst 0xe0ba82a5 // st1w { za1v.s[x12, #1] }, p0/Z, [x21, x26, LSL #2]\n" - "addvl x21, x21, #4\n" - "incw x22\n" + ".inst 0xe0ba82c5 // st1w { za1v.s[x12, #1] }, p0/Z, [x22, x26, LSL #2]\n" + "addvl x22, x22, #4\n" + "incw x23\n" "whilelt p9.s, x13, %x[width]\n" "whilelt p8.s, x13, %x[width]\n" "mov x12, #0x0\n" @@ -175,60 +173,60 @@ void interleave_block<2, 1, VLType::SME, false>( "6:" // K loop: Main loop: Second: Loop ".inst 0x25306581 // psel p1.s, p9.s/Z, p12.s[w12]\n" ".inst 0x25306160 // psel p0.s, p8.s/Z, p11.s[w12]\n" - ".inst 0xe0960540 // ld1w { za0h.s[x12] }, p1/Z, [x10, x22, LSL #2]\n" + ".inst 0xe0970540 // ld1w { za0h.s[x12] }, p1/Z, [x10, x23, LSL #2]\n" "ldr x10, [x11, #0x0]\n" - ".inst 0xe0960364 // ld1w { za1h.s[x12] }, p0/Z, [x27, x22, LSL #2]\n" + ".inst 0xe0970364 // ld1w { za1h.s[x12] }, p0/Z, [x27, x23, LSL #2]\n" ".inst 0x25706580 // psel p0.s, p9.s/Z, p12.s[w12, #1]\n" ".inst 0x25706162 // psel p2.s, p8.s/Z, p11.s[w12, #1]\n" "ldr x27, [x9, #0x0]\n" ".inst 0x25307541 // psel p1.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0960321 // ld1w { za0h.s[x12, #1] }, p0/Z, [x25, x22, LSL #2]\n" + ".inst 0xe0970321 // ld1w { za0h.s[x12, #1] }, p0/Z, [x25, x23, LSL #2]\n" "ldr x25, [x11, #0x8]\n" ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0960ae5 // ld1w { za1h.s[x12, #1] }, p2/Z, [x23, x22, LSL #2]\n" - "ldr x23, [x9, #0x8]\n" - ".inst 0xe0bf86a8 // st1w { za2v.s[x12] }, p1/Z, [x21, XZR, LSL #2]\n" + ".inst 0xe0970aa5 // ld1w { za1h.s[x12, #1] }, p2/Z, [x21, x23, LSL #2]\n" + "ldr x21, [x9, #0x8]\n" + ".inst 0xe0bf86c8 // st1w { za2v.s[x12] }, p1/Z, [x22, XZR, LSL #2]\n" ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" - ".inst 0xe0b082ac // st1w { za3v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + ".inst 0xe0b082cc // st1w { za3v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" "add x11, x11, #0x10\n" - ".inst 0xe0bc86a9 // st1w { za2v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" + ".inst 0xe0bc86c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" "add x9, x9, #0x10\n" - ".inst 0xe0ba82ad // st1w { za3v.s[x12, #1] }, p0/Z, [x21, x26, LSL #2]\n" + ".inst 0xe0ba82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x26, LSL #2]\n" "add x12, x12, #0x2\n" "cmp x12, x14\n" - "addvl x21, x21, #4\n" + "addvl x22, x22, #4\n" "blt 6b\n" "7:" // K loop: Main loop: Second: Tail ".inst 0x25306581 // psel p1.s, p9.s/Z, p12.s[w12]\n" ".inst 0x25306160 // psel p0.s, p8.s/Z, p11.s[w12]\n" - ".inst 0xe0960540 // ld1w { za0h.s[x12] }, p1/Z, [x10, x22, LSL #2]\n" - ".inst 0xe0960364 // ld1w { za1h.s[x12] }, p0/Z, [x27, x22, LSL #2]\n" + ".inst 0xe0970540 // ld1w { za0h.s[x12] }, p1/Z, [x10, x23, LSL #2]\n" + ".inst 0xe0970364 // ld1w { za1h.s[x12] }, p0/Z, [x27, x23, LSL #2]\n" "mov x11, %x[in]\n" "add x9, %x[in], x16, LSL #3\n" "ldr x10, [x11, #0x0]\n" ".inst 0x25706580 // psel p0.s, p9.s/Z, p12.s[w12, #1]\n" ".inst 0x25706161 // psel p1.s, p8.s/Z, p11.s[w12, #1]\n" - ".inst 0xe0960321 // ld1w { za0h.s[x12, #1] }, p0/Z, [x25, x22, LSL #2]\n" + ".inst 0xe0970321 // ld1w { za0h.s[x12, #1] }, p0/Z, [x25, x23, LSL #2]\n" "ldr x27, [x9, #0x0]\n" ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe09606e5 // ld1w { za1h.s[x12, #1] }, p1/Z, [x23, x22, LSL #2]\n" + ".inst 0xe09706a5 // ld1w { za1h.s[x12, #1] }, p1/Z, [x21, x23, LSL #2]\n" "ldr x25, [x11, #0x8]\n" ".inst 0x25307542 // psel p2.s, p13.s/Z, p10.s[w12]\n" - "ldr x23, [x9, #0x8]\n" - ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + "ldr x21, [x9, #0x8]\n" + ".inst 0xe0bf82c8 // st1w { za2v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" ".inst 0x25707541 // psel p1.s, p13.s/Z, p10.s[w12, #1]\n" ".inst 0x25707540 // psel p0.s, p13.s/Z, p10.s[w12, #1]\n" - ".inst 0xe0b08aac // st1w { za3v.s[x12] }, p2/Z, [x21, x16, LSL #2]\n" + ".inst 0xe0b08acc // st1w { za3v.s[x12] }, p2/Z, [x22, x16, LSL #2]\n" "whilelt p10.s, x13, %x[width]\n" "subs x20, x20, #0x1\n" - ".inst 0xe0bc86a9 // st1w { za2v.s[x12, #1] }, p1/Z, [x21, x28, LSL #2]\n" + ".inst 0xe0bc86c9 // st1w { za2v.s[x12, #1] }, p1/Z, [x22, x28, LSL #2]\n" "add x11, x11, #0x10\n" "add x9, x9, #0x10\n" - ".inst 0xe0ba82ad // st1w { za3v.s[x12, #1] }, p0/Z, [x21, x26, LSL #2]\n" - "addvl x21, x21, #4\n" + ".inst 0xe0ba82cd // st1w { za3v.s[x12, #1] }, p0/Z, [x22, x26, LSL #2]\n" + "addvl x22, x22, #4\n" "incw x13\n" - "incw x22\n" + "incw x23\n" "bgt 3b\n" "8:" // K loop: Tails "cbnz x24, 11f\n" @@ -238,48 +236,48 @@ void interleave_block<2, 1, VLType::SME, false>( "mov x12, #0x0\n" "9:" // K loop: Tails: Even: First ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0b082a4 // st1w { za1v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" - "ldr x10, [x11, #0x0]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" + "ldr x21, [x11, #0x0]\n" ".inst 0x25306581 // psel p1.s, p9.s/Z, p12.s[w12]\n" ".inst 0x25306160 // psel p0.s, p8.s/Z, p11.s[w12]\n" - "ldr x27, [x11, x16, LSL #0x3]\n" - ".inst 0xe0960548 // ld1w { za2h.s[x12] }, p1/Z, [x10, x22, LSL #2]\n" + "ldr x20, [x11, x16, LSL #0x3]\n" + ".inst 0xe09706a8 // ld1w { za2h.s[x12] }, p1/Z, [x21, x23, LSL #2]\n" "add x11, x11, #0x8\n" - "addvl x21, x21, #2\n" - ".inst 0xe096036c // ld1w { za3h.s[x12] }, p0/Z, [x27, x22, LSL #2]\n" + "addvl x22, x22, #2\n" + ".inst 0xe097028c // ld1w { za3h.s[x12] }, p0/Z, [x20, x23, LSL #2]\n" "add x12, x12, #0x1\n" "cmp x12, x16\n" "blt 9b\n" "whilelt p10.s, x13, %x[width]\n" - "whilelt p9.s, x13, %x[width]\n" + "whilelt p8.s, x13, %x[width]\n" "whilelt p8.s, x13, %x[width]\n" "mov x12, #0x0\n" "10:" // K loop: Tails: Even: Second ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0bf82a8 // st1w { za2v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0xe0bf82c8 // st1w { za2v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0b082ac // st1w { za3v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + ".inst 0xe0b082cc // st1w { za3v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" "add x12, x12, #0x1\n" "cmp x12, x15\n" - "addvl x21, x21, #2\n" + "addvl x22, x22, #2\n" "blt 10b\n" - "whilelt p10.s, x13, %x[width]\n" + "whilelt p8.s, x13, %x[width]\n" "b 13f\n" "11:" // K loop: Tails: Odd "mov x12, #0x0\n" "12:" // K loop: Tails: Odd: Loop ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0bf82a0 // st1w { za0v.s[x12] }, p0/Z, [x21, XZR, LSL #2]\n" + ".inst 0xe0bf82c0 // st1w { za0v.s[x12] }, p0/Z, [x22, XZR, LSL #2]\n" ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" - ".inst 0xe0b082a4 // st1w { za1v.s[x12] }, p0/Z, [x21, x16, LSL #2]\n" + ".inst 0xe0b082c4 // st1w { za1v.s[x12] }, p0/Z, [x22, x16, LSL #2]\n" "add x12, x12, #0x1\n" "cmp x12, x15\n" - "addvl x21, x21, #2\n" + "addvl x22, x22, #2\n" "blt 12b\n" "13:" // K loop: End - "mov %x[out], x21\n" + "mov %x[out], x22\n" ".inst 0xd503467f // SMSTOP\n" : [out] "+&r" (out) : [height] "r" (height), [in] "r" (in), [row_offset] "r" (row_offset), [width] "r" (width) @@ -287,4 +285,4 @@ void interleave_block<2, 1, VLType::SME, false>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_bf16_bf16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_bf16_bf16.hpp index 4cc84d344a6b044e88aa087cee7fc440989586a0..2e53475b5c12ee8432b1613c4acded2ca5d63b1e 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_bf16_bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_bf16_bf16.hpp @@ -22,16 +22,14 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME) template <> void interleave_block<4, 2, VLType::SME, false>( bfloat16 * &out, const bfloat16 * const *in, - size_t width, size_t height, size_t row_offset, bool first + size_t width, size_t height, size_t row_offset, bool ) { - ARM_COMPUTE_UNUSED(first); - __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" "cntw x16\n" @@ -124,4 +122,4 @@ void interleave_block<4, 2, VLType::SME, false>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8.hpp index 465939c30d0d004d91ce61dbe7ba741d295dcdee..67dd5a9bb7af905908579169e88bddefbb9a91f2 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8.hpp @@ -22,16 +22,14 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME) template <> void interleave_block<4, 4, VLType::SME, false>( int8_t * &out, const int8_t * const *in, - size_t width, size_t height, size_t row_offset, bool first + size_t width, size_t height, size_t row_offset, bool ) { - ARM_COMPUTE_UNUSED(first); - __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" "cntw x16\n" @@ -123,4 +121,4 @@ void interleave_block<4, 4, VLType::SME, false>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8_summing.hpp index ffd9384a13fe4423d0fd4b16186471abc7e9bc34..21d9378368e89a90de16356e6020b27e17c390fd 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8_summing.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_s8_s8_summing.hpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME) template <> void interleave_block<4, 4, VLType::SME, true>( @@ -112,22 +112,22 @@ void interleave_block<4, 4, VLType::SME, true>( ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xe0bf8120 // st1w { za0v.s[x12] }, p0/Z, [x9, XZR, LSL #2]\n" ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - ".inst 0xc0828812 // mova z18.s, p2/M, za0v.s[x12]\n" + ".inst 0xc0828811 // mova z17.s, p2/M, za0v.s[x12]\n" ".inst 0xe0af8124 // st1w { za1v.s[x12] }, p0/Z, [x9, x15, LSL #2]\n" ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - ".inst 0xc0828891 // mova z17.s, p2/M, za1v.s[x12]\n" + ".inst 0xc0828893 // mova z19.s, p2/M, za1v.s[x12]\n" ".inst 0xe0ae8528 // st1w { za2v.s[x12] }, p1/Z, [x9, x14, LSL #2]\n" ".inst 0xc0828910 // mova z16.s, p2/M, za2v.s[x12]\n" - "sdot z23.s, z18.b, z24.b\n" + "sdot z23.s, z17.b, z24.b\n" ".inst 0xe0ad812c // st1w { za3v.s[x12] }, p0/Z, [x9, x13, LSL #2]\n" - ".inst 0xc0828993 // mova z19.s, p2/M, za3v.s[x12]\n" + ".inst 0xc0828992 // mova z18.s, p2/M, za3v.s[x12]\n" "add x12, x12, #0x1\n" "cmp x12, x20\n" - "sdot z22.s, z17.b, z24.b\n" + "sdot z22.s, z19.b, z24.b\n" "sdot z21.s, z16.b, z24.b\n" "addvl x9, x9, #4\n" - "sdot z20.s, z19.b, z24.b\n" + "sdot z20.s, z18.b, z24.b\n" "blt 5b\n" "incb x28\n" "whilelt p9.b, x28, %x[width]\n" @@ -147,4 +147,4 @@ void interleave_block<4, 4, VLType::SME, true>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8.hpp index 9f5db6ba3db6640767a5de8c7c483caedee51c4d..f149c932935a0de5b666707adfcf9dcf525ea861 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8.hpp @@ -22,16 +22,14 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME) template <> void interleave_block<4, 4, VLType::SME, false>( uint8_t * &out, const uint8_t * const *in, - size_t width, size_t height, size_t row_offset, bool first + size_t width, size_t height, size_t row_offset, bool ) { - ARM_COMPUTE_UNUSED(first); - __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" "cntw x16\n" @@ -123,4 +121,4 @@ void interleave_block<4, 4, VLType::SME, false>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8_summing.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8_summing.hpp index 49d2acf1cd5f131c82a34a5f3d30d6858786a60f..252152e3da6f585a503cf23d4d5e822de5015caf 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8_summing.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block4_u8_u8_summing.hpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME) template <> void interleave_block<4, 4, VLType::SME, true>( @@ -112,22 +112,22 @@ void interleave_block<4, 4, VLType::SME, true>( ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xe0bf8120 // st1w { za0v.s[x12] }, p0/Z, [x9, XZR, LSL #2]\n" ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" - ".inst 0xc0828813 // mova z19.s, p2/M, za0v.s[x12]\n" + ".inst 0xc0828810 // mova z16.s, p2/M, za0v.s[x12]\n" ".inst 0xe0af8124 // st1w { za1v.s[x12] }, p0/Z, [x9, x15, LSL #2]\n" ".inst 0x25306d21 // psel p1.s, p11.s/Z, p9.s[w12]\n" ".inst 0x25306d20 // psel p0.s, p11.s/Z, p9.s[w12]\n" ".inst 0xc0828891 // mova z17.s, p2/M, za1v.s[x12]\n" ".inst 0xe0ae8528 // st1w { za2v.s[x12] }, p1/Z, [x9, x14, LSL #2]\n" - ".inst 0xc0828912 // mova z18.s, p2/M, za2v.s[x12]\n" - "udot z23.s, z19.b, z24.b\n" + ".inst 0xc0828913 // mova z19.s, p2/M, za2v.s[x12]\n" + "udot z23.s, z16.b, z24.b\n" ".inst 0xe0ad812c // st1w { za3v.s[x12] }, p0/Z, [x9, x13, LSL #2]\n" - ".inst 0xc0828990 // mova z16.s, p2/M, za3v.s[x12]\n" + ".inst 0xc0828992 // mova z18.s, p2/M, za3v.s[x12]\n" "add x12, x12, #0x1\n" "cmp x12, x20\n" "udot z22.s, z17.b, z24.b\n" - "udot z21.s, z18.b, z24.b\n" + "udot z21.s, z19.b, z24.b\n" "addvl x9, x9, #4\n" - "udot z20.s, z16.b, z24.b\n" + "udot z20.s, z18.b, z24.b\n" "blt 5b\n" "incb x28\n" "whilelt p9.b, x28, %x[width]\n" @@ -147,4 +147,4 @@ void interleave_block<4, 4, VLType::SME, true>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_fp32_fp32.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_fp32_fp32.hpp index 9579263204a06e9e91b3f962302a9bfc98f92f41..b11bb93c4253cd94a1e0d01a19087362934f4c44 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_fp32_fp32.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_fp32_fp32.hpp @@ -22,16 +22,14 @@ * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME) template <> void interleave_block<4, 1, VLType::SME, false>( float * &out, const float * const *in, - size_t width, size_t height, size_t row_offset, bool first + size_t width, size_t height, size_t row_offset, bool ) { - ARM_COMPUTE_UNUSED(first); - __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" "cntw x15\n" @@ -123,4 +121,4 @@ void interleave_block<4, 1, VLType::SME, false>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/interleave_indirect_impl.hpp b/src/core/NEON/kernels/arm_gemm/interleave_indirect_impl.hpp index 4f25da2877a834f36314bbb30657c77360d92a32..b921fd16d2420dacb8c42c7faf53b873d42b3f60 100644 --- a/src/core/NEON/kernels/arm_gemm/interleave_indirect_impl.hpp +++ b/src/core/NEON/kernels/arm_gemm/interleave_indirect_impl.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -39,8 +39,12 @@ */ template void interleave_block( TOut * &out, const TIn * const *in, size_t width, size_t height, size_t row_offset, bool first) { +#ifdef ARM_COMPUTE_ENABLE_SVE const unsigned int int_by = height_vectors * (vlt == VLType::SVE ? get_vector_length() / block : (vlt == VLType::SME ? sme::get_vector_length() / block : 1 )); +#else + const unsigned int int_by = height_vectors; +#endif std::vector the_sums; @@ -104,8 +108,12 @@ void interleave_block( TOut * &out, const TIn * const *in, size_t width, size_t template inline void FixupRowSums(TOut * &out, const int32_t row_sum_multiplier) { +#ifdef ARM_COMPUTE_ENABLE_SVE const unsigned int height = height_vectors * (vlt == VLType::SVE ? get_vector_length() / block : (vlt == VLType::SME ? sme::get_vector_length() / block : 1 )); +#else + const unsigned int height = height_vectors; +#endif // If we are integrating row sums, we need to do some fix up, depending on whether the multiplier is non-zero or not. if (row_sum_multiplier) { @@ -138,8 +146,12 @@ void IndirectInterleave(TOut *out, const TIn * const * const *ptr, unsigned int unsigned int rounded_stringlen, const unsigned int y0, const unsigned int ymax, const unsigned int k0, const unsigned int kmax, bool integrate_sums, const int32_t row_sum_multiplier) { +#ifdef ARM_COMPUTE_ENABLE_SVE const unsigned int height = height_vectors * (vlt == VLType::SVE ? get_vector_length() / block : (vlt == VLType::SME ? sme::get_vector_length() / block : 1 )); +#else + const unsigned int height = height_vectors; +#endif // 'interleave_block' implementations are entitled to read a pointer for each row they handle from the input // pointer array, even for out of range rows (although they must not subsequently dereference those pointers for @@ -208,8 +220,12 @@ void IndirectInterleave(TOut *out, const TIn * const * const *ptr, unsigned int template void ConvolutionInterleave(TOut *out, const TIn *in, size_t in_stride, const convolver &conv, const unsigned int rounded_stringlen, const unsigned int y0, const unsigned int ymax, const unsigned int k0, const unsigned int kmax, bool integrate_sums, const int32_t row_sum_multiplier) { +#ifdef ARM_COMPUTE_ENABLE_SVE const unsigned int height = height_vectors * (vlt == VLType::SVE ? get_vector_length() / block : (vlt == VLType::SME ? sme::get_vector_length() / block : 1 )); +#else + const unsigned int height = height_vectors; +#endif auto conv_cols = conv.process_columns(in, in_stride, k0, kmax, rounded_stringlen); // Use alloca here as a std::vector can be expensive in highly threaded scenarios. @@ -246,8 +262,12 @@ void ConvolutionInterleave(TOut *out, const TIn *in, size_t in_stride, const con template void Interleave(TOut *out, const TIn *in, size_t in_stride, const unsigned int y0, const unsigned int ymax, const unsigned int k0, const unsigned int kmax, bool integrate_sums, const int32_t row_sum_multiplier) { +#ifdef ARM_COMPUTE_ENABLE_SVE const unsigned int height = height_vectors * (vlt == VLType::SVE ? get_vector_length() / block : (vlt == VLType::SME ? sme::get_vector_length() / block : 1 )); +#else + const unsigned int height = height_vectors; +#endif // Use alloca here as a std::vector can be expensive in highly threaded scenarios. const TIn **row_ptrs = reinterpret_cast(alloca(height * sizeof(const TIn *))); diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_bf16fp32_mmla_6x16.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_bf16fp32_mmla_6x16.hpp index 9a871d4b88793ab03a4b7559f5c9b44ae0bddead..72e414969ee3a43d53085a521804b425aef42ee5 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_bf16fp32_mmla_6x16.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_bf16fp32_mmla_6x16.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,16 +10,16 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once #ifdef __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_bf16fp32_mmla_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_bf16fp32_mmla_6x16/generic.cpp index 74791f8d307a709d33a10d0180b91e3c01ffe106..377daddae9d91cf9a3f6f3f100e461167b9d97e9 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_bf16fp32_mmla_6x16/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_bf16fp32_mmla_6x16/generic.cpp @@ -231,11 +231,11 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "17:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 18f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 19f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -251,41 +251,41 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "ldr q6, [x12, #0x10]\n" "blt 21f\n" "20:" // Height 1: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - "ldr q7, [x11, #0x0]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - "ldr q6, [x11, #0x10]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x0]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x10]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" - "ldr q7, [x9, #0x0]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" - "ldr q6, [x9, #0x10]\n" - "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" - "ldr q7, [x12, #0x20]\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" - "ldr q6, [x12, #0x30]\n" - ".inst 0x6e47ec28 // bfmmla v8.4s, v1.8h, v7.8h\n" - "ldr q7, [x11, #0x20]\n" - ".inst 0x6e46ec2c // bfmmla v12.4s, v1.8h, v6.8h\n" - "ldr q6, [x11, #0x30]\n" - ".inst 0x6e47ec29 // bfmmla v9.4s, v1.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec2d // bfmmla v13.4s, v1.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec2a // bfmmla v10.4s, v1.8h, v7.8h\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - "ldr q6, [x9, #0x30]\n" + "trn1 v20.2d, v1.2d, v21.2d\n" + ".inst 0x6e47ee88 // bfmmla v8.4s, v20.8h, v7.8h\n" + "ldr q17, [x11, #0x0]\n" + ".inst 0x6e46ee8c // bfmmla v12.4s, v20.8h, v6.8h\n" + "ldr q19, [x11, #0x10]\n" + ".inst 0x6e51ee89 // bfmmla v9.4s, v20.8h, v17.8h\n" + "ldr q18, [x10, #0x0]\n" + ".inst 0x6e53ee8d // bfmmla v13.4s, v20.8h, v19.8h\n" + "ldr q17, [x10, #0x10]\n" + ".inst 0x6e52ee8a // bfmmla v10.4s, v20.8h, v18.8h\n" + "ldr q18, [x9, #0x0]\n" + ".inst 0x6e51ee8e // bfmmla v14.4s, v20.8h, v17.8h\n" + "ldr q17, [x9, #0x10]\n" + "trn2 v1.2d, v1.2d, v21.2d\n" + ".inst 0x6e52ee8b // bfmmla v11.4s, v20.8h, v18.8h\n" + "ldr q18, [x12, #0x20]\n" + ".inst 0x6e51ee8f // bfmmla v15.4s, v20.8h, v17.8h\n" + "ldr q17, [x12, #0x30]\n" + ".inst 0x6e52ec28 // bfmmla v8.4s, v1.8h, v18.8h\n" + "ldr q18, [x11, #0x20]\n" + ".inst 0x6e51ec2c // bfmmla v12.4s, v1.8h, v17.8h\n" + "ldr q17, [x11, #0x30]\n" + ".inst 0x6e52ec29 // bfmmla v9.4s, v1.8h, v18.8h\n" + "ldr q18, [x10, #0x20]\n" + ".inst 0x6e51ec2d // bfmmla v13.4s, v1.8h, v17.8h\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x6e52ec2a // bfmmla v10.4s, v1.8h, v18.8h\n" + "ldr q18, [x9, #0x20]\n" + ".inst 0x6e51ec2e // bfmmla v14.4s, v1.8h, v17.8h\n" + "ldr q17, [x9, #0x30]\n" "sub x27, x27, #0x8\n" "cmp x27, #0x10\n" "add x26, x26, #0x10\n" - ".inst 0x6e47ec2b // bfmmla v11.4s, v1.8h, v7.8h\n" - ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" + ".inst 0x6e52ec2b // bfmmla v11.4s, v1.8h, v18.8h\n" + ".inst 0x6e51ec2f // bfmmla v15.4s, v1.8h, v17.8h\n" "ldr q1, [x26, #0x0]\n" "add x12, x12, #0x40\n" "ldr q7, [x12, #0x0]\n" @@ -295,39 +295,39 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "add x9, x9, #0x40\n" "bge 20b\n" "21:" // Height 1: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - "ldr q7, [x11, #0x0]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - "ldr q6, [x11, #0x10]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x0]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x10]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" - "ldr q7, [x9, #0x0]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" - "ldr q6, [x9, #0x10]\n" - "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" - "ldr q7, [x12, #0x20]\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" - "ldr q6, [x12, #0x30]\n" - ".inst 0x6e47ec28 // bfmmla v8.4s, v1.8h, v7.8h\n" - "ldr q7, [x11, #0x20]\n" - ".inst 0x6e46ec2c // bfmmla v12.4s, v1.8h, v6.8h\n" - "ldr q6, [x11, #0x30]\n" - ".inst 0x6e47ec29 // bfmmla v9.4s, v1.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec2d // bfmmla v13.4s, v1.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec2a // bfmmla v10.4s, v1.8h, v7.8h\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - "ldr q6, [x9, #0x30]\n" + "trn1 v19.2d, v1.2d, v20.2d\n" + ".inst 0x6e47ee68 // bfmmla v8.4s, v19.8h, v7.8h\n" + "ldr q17, [x11, #0x0]\n" + ".inst 0x6e46ee6c // bfmmla v12.4s, v19.8h, v6.8h\n" + "ldr q18, [x11, #0x10]\n" + ".inst 0x6e51ee69 // bfmmla v9.4s, v19.8h, v17.8h\n" + "ldr q17, [x10, #0x0]\n" + ".inst 0x6e52ee6d // bfmmla v13.4s, v19.8h, v18.8h\n" + "ldr q18, [x10, #0x10]\n" + ".inst 0x6e51ee6a // bfmmla v10.4s, v19.8h, v17.8h\n" + "ldr q17, [x9, #0x0]\n" + ".inst 0x6e52ee6e // bfmmla v14.4s, v19.8h, v18.8h\n" + "ldr q24, [x9, #0x10]\n" + "trn2 v1.2d, v1.2d, v20.2d\n" + ".inst 0x6e51ee6b // bfmmla v11.4s, v19.8h, v17.8h\n" + "ldr q18, [x12, #0x20]\n" + ".inst 0x6e58ee6f // bfmmla v15.4s, v19.8h, v24.8h\n" + "ldr q17, [x12, #0x30]\n" + ".inst 0x6e52ec28 // bfmmla v8.4s, v1.8h, v18.8h\n" + "ldr q19, [x11, #0x20]\n" + ".inst 0x6e51ec2c // bfmmla v12.4s, v1.8h, v17.8h\n" + "ldr q17, [x11, #0x30]\n" + ".inst 0x6e53ec29 // bfmmla v9.4s, v1.8h, v19.8h\n" + "ldr q18, [x10, #0x20]\n" + ".inst 0x6e51ec2d // bfmmla v13.4s, v1.8h, v17.8h\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x6e52ec2a // bfmmla v10.4s, v1.8h, v18.8h\n" + "ldr q18, [x9, #0x20]\n" + ".inst 0x6e51ec2e // bfmmla v14.4s, v1.8h, v17.8h\n" + "ldr q17, [x9, #0x30]\n" "sub x27, x27, #0x8\n" - ".inst 0x6e47ec2b // bfmmla v11.4s, v1.8h, v7.8h\n" - ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" + ".inst 0x6e52ec2b // bfmmla v11.4s, v1.8h, v18.8h\n" + ".inst 0x6e51ec2f // bfmmla v15.4s, v1.8h, v17.8h\n" "add x26, x26, #0x10\n" "add x12, x12, #0x40\n" "add x11, x11, #0x40\n" @@ -338,26 +338,26 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "cmp x27, #0x4\n" "blt 24f\n" "23:" // Height 1: Multiply loop: Odd block loop - "ldr d1, [x26], #0x8\n" - "ldr q6, [x12, #0x0]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "ldr q7, [x12, #0x10]\n" - ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - "ldr q6, [x11, #0x0]\n" - ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" - "ldr q7, [x11, #0x10]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x0]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - "ldr q6, [x9, #0x0]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - "ldr q7, [x9, #0x10]\n" + "ldr d19, [x26], #0x8\n" + "ldr q18, [x12, #0x0]\n" + "trn1 v19.2d, v19.2d, v17.2d\n" + "ldr q17, [x12, #0x10]\n" + ".inst 0x6e52ee68 // bfmmla v8.4s, v19.8h, v18.8h\n" + "ldr q18, [x11, #0x0]\n" + ".inst 0x6e51ee6c // bfmmla v12.4s, v19.8h, v17.8h\n" + "ldr q17, [x11, #0x10]\n" + ".inst 0x6e52ee69 // bfmmla v9.4s, v19.8h, v18.8h\n" + "ldr q18, [x10, #0x0]\n" + ".inst 0x6e51ee6d // bfmmla v13.4s, v19.8h, v17.8h\n" + "ldr q17, [x10, #0x10]\n" + ".inst 0x6e52ee6a // bfmmla v10.4s, v19.8h, v18.8h\n" + "ldr q18, [x9, #0x0]\n" + ".inst 0x6e51ee6e // bfmmla v14.4s, v19.8h, v17.8h\n" + "ldr q17, [x9, #0x10]\n" "sub x27, x27, #0x4\n" "cmp x27, #0x4\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" + ".inst 0x6e52ee6b // bfmmla v11.4s, v19.8h, v18.8h\n" + ".inst 0x6e51ee6f // bfmmla v15.4s, v19.8h, v17.8h\n" "add x12, x12, #0x20\n" "add x11, x11, #0x20\n" "add x10, x10, #0x20\n" @@ -373,23 +373,23 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "25:" // Height 1: Multiply loop: Ragged operand read: partial_1_0 "ldr h1, [x26, #0x0]\n" "26:" // Height 1: Multiply loop: Ragged operand read: Done - "ldr q7, [x12, #0x0]\n" - "ldr q6, [x12, #0x10]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - "ldr q7, [x11, #0x0]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - "ldr q6, [x11, #0x10]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x0]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" + "ldr q20, [x12, #0x0]\n" + "ldr q18, [x12, #0x10]\n" + "trn1 v19.2d, v1.2d, v17.2d\n" + ".inst 0x6e54ee68 // bfmmla v8.4s, v19.8h, v20.8h\n" + "ldr q17, [x11, #0x0]\n" + ".inst 0x6e52ee6c // bfmmla v12.4s, v19.8h, v18.8h\n" + "ldr q18, [x11, #0x10]\n" + ".inst 0x6e51ee69 // bfmmla v9.4s, v19.8h, v17.8h\n" + "ldr q17, [x10, #0x0]\n" + ".inst 0x6e52ee6d // bfmmla v13.4s, v19.8h, v18.8h\n" "ldr q6, [x10, #0x10]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" - "ldr q7, [x9, #0x0]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" - "ldr q6, [x9, #0x10]\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" + ".inst 0x6e51ee6a // bfmmla v10.4s, v19.8h, v17.8h\n" + "ldr q18, [x9, #0x0]\n" + ".inst 0x6e46ee6e // bfmmla v14.4s, v19.8h, v6.8h\n" + "ldr q17, [x9, #0x10]\n" + ".inst 0x6e52ee6b // bfmmla v11.4s, v19.8h, v18.8h\n" + ".inst 0x6e51ee6f // bfmmla v15.4s, v19.8h, v17.8h\n" "add x12, x12, #0x20\n" "add x11, x11, #0x20\n" "add x10, x10, #0x20\n" @@ -405,17 +405,17 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "uzp1 v11.2d, v11.2d, v15.2d\n" "tbz %x[flags], #1, 28f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v18.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" + "ld1r { v17.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v18.4s\n" + "fmin v9.4s, v9.4s, v18.4s\n" + "fmin v10.4s, v10.4s, v18.4s\n" + "fmin v11.4s, v11.4s, v18.4s\n" + "fmax v8.4s, v8.4s, v17.4s\n" + "fmax v9.4s, v9.4s, v17.4s\n" + "fmax v10.4s, v10.4s, v17.4s\n" + "fmax v11.4s, v11.4s, v17.4s\n" "28:" // Height 1: No activation "cmp x14, #0x10\n" "bge 37f\n" @@ -624,12 +624,12 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "55:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 56f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 57f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -637,7 +637,7 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "b 57f\n" "56:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" "57:" // Height 2: input setup done "cmp x27, #0x8\n" "blt 60f\n" @@ -648,45 +648,45 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "ldr q6, [x12, #0x10]\n" "blt 59f\n" "58:" // Height 2: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - "ldr q7, [x11, #0x0]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - "ldr q6, [x11, #0x10]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x0]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x10]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" - "ldr q7, [x9, #0x0]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" - "ldr q6, [x9, #0x10]\n" + "trn1 v19.2d, v1.2d, v2.2d\n" + ".inst 0x6e47ee68 // bfmmla v8.4s, v19.8h, v7.8h\n" + "ldr q18, [x11, #0x0]\n" + ".inst 0x6e46ee6c // bfmmla v12.4s, v19.8h, v6.8h\n" + "ldr q17, [x11, #0x10]\n" + ".inst 0x6e52ee69 // bfmmla v9.4s, v19.8h, v18.8h\n" + "ldr q18, [x10, #0x0]\n" + ".inst 0x6e51ee6d // bfmmla v13.4s, v19.8h, v17.8h\n" + "ldr q17, [x10, #0x10]\n" + ".inst 0x6e52ee6a // bfmmla v10.4s, v19.8h, v18.8h\n" + "ldr q18, [x9, #0x0]\n" + ".inst 0x6e51ee6e // bfmmla v14.4s, v19.8h, v17.8h\n" + "ldr q17, [x9, #0x10]\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" - "ldr q7, [x12, #0x20]\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" - "ldr q6, [x12, #0x30]\n" - ".inst 0x6e47ec28 // bfmmla v8.4s, v1.8h, v7.8h\n" - "ldr q7, [x11, #0x20]\n" - ".inst 0x6e46ec2c // bfmmla v12.4s, v1.8h, v6.8h\n" - "ldr q6, [x11, #0x30]\n" - ".inst 0x6e47ec29 // bfmmla v9.4s, v1.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec2d // bfmmla v13.4s, v1.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec2a // bfmmla v10.4s, v1.8h, v7.8h\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - "ldr q6, [x9, #0x30]\n" + ".inst 0x6e52ee6b // bfmmla v11.4s, v19.8h, v18.8h\n" + "ldr q18, [x12, #0x20]\n" + ".inst 0x6e51ee6f // bfmmla v15.4s, v19.8h, v17.8h\n" + "ldr q17, [x12, #0x30]\n" + ".inst 0x6e52ec28 // bfmmla v8.4s, v1.8h, v18.8h\n" + "ldr q18, [x11, #0x20]\n" + ".inst 0x6e51ec2c // bfmmla v12.4s, v1.8h, v17.8h\n" + "ldr q17, [x11, #0x30]\n" + ".inst 0x6e52ec29 // bfmmla v9.4s, v1.8h, v18.8h\n" + "ldr q18, [x10, #0x20]\n" + ".inst 0x6e51ec2d // bfmmla v13.4s, v1.8h, v17.8h\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x6e52ec2a // bfmmla v10.4s, v1.8h, v18.8h\n" + "ldr q18, [x9, #0x20]\n" + ".inst 0x6e51ec2e // bfmmla v14.4s, v1.8h, v17.8h\n" + "ldr q17, [x9, #0x30]\n" "sub x27, x27, #0x8\n" "cmp x27, #0x10\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" "ldr q2, [x25, #0x0]\n" - ".inst 0x6e47ec2b // bfmmla v11.4s, v1.8h, v7.8h\n" + ".inst 0x6e52ec2b // bfmmla v11.4s, v1.8h, v18.8h\n" "add x12, x12, #0x40\n" "ldr q7, [x12, #0x0]\n" - ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" + ".inst 0x6e51ec2f // bfmmla v15.4s, v1.8h, v17.8h\n" "ldr q1, [x26, #0x0]\n" "ldr q6, [x12, #0x10]\n" "add x11, x11, #0x40\n" @@ -694,39 +694,39 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "add x9, x9, #0x40\n" "bge 58b\n" "59:" // Height 2: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - "ldr q7, [x11, #0x0]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - "ldr q6, [x11, #0x10]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x0]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x10]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" - "ldr q7, [x9, #0x0]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" - "ldr q6, [x9, #0x10]\n" + "trn1 v19.2d, v1.2d, v2.2d\n" + ".inst 0x6e47ee68 // bfmmla v8.4s, v19.8h, v7.8h\n" + "ldr q18, [x11, #0x0]\n" + ".inst 0x6e46ee6c // bfmmla v12.4s, v19.8h, v6.8h\n" + "ldr q17, [x11, #0x10]\n" + ".inst 0x6e52ee69 // bfmmla v9.4s, v19.8h, v18.8h\n" + "ldr q18, [x10, #0x0]\n" + ".inst 0x6e51ee6d // bfmmla v13.4s, v19.8h, v17.8h\n" + "ldr q17, [x10, #0x10]\n" + ".inst 0x6e52ee6a // bfmmla v10.4s, v19.8h, v18.8h\n" + "ldr q18, [x9, #0x0]\n" + ".inst 0x6e51ee6e // bfmmla v14.4s, v19.8h, v17.8h\n" + "ldr q17, [x9, #0x10]\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" - "ldr q7, [x12, #0x20]\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" - "ldr q6, [x12, #0x30]\n" - ".inst 0x6e47ec28 // bfmmla v8.4s, v1.8h, v7.8h\n" - "ldr q7, [x11, #0x20]\n" - ".inst 0x6e46ec2c // bfmmla v12.4s, v1.8h, v6.8h\n" - "ldr q6, [x11, #0x30]\n" - ".inst 0x6e47ec29 // bfmmla v9.4s, v1.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec2d // bfmmla v13.4s, v1.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec2a // bfmmla v10.4s, v1.8h, v7.8h\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - "ldr q6, [x9, #0x30]\n" + ".inst 0x6e52ee6b // bfmmla v11.4s, v19.8h, v18.8h\n" + "ldr q18, [x12, #0x20]\n" + ".inst 0x6e51ee6f // bfmmla v15.4s, v19.8h, v17.8h\n" + "ldr q17, [x12, #0x30]\n" + ".inst 0x6e52ec28 // bfmmla v8.4s, v1.8h, v18.8h\n" + "ldr q18, [x11, #0x20]\n" + ".inst 0x6e51ec2c // bfmmla v12.4s, v1.8h, v17.8h\n" + "ldr q17, [x11, #0x30]\n" + ".inst 0x6e52ec29 // bfmmla v9.4s, v1.8h, v18.8h\n" + "ldr q18, [x10, #0x20]\n" + ".inst 0x6e51ec2d // bfmmla v13.4s, v1.8h, v17.8h\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x6e52ec2a // bfmmla v10.4s, v1.8h, v18.8h\n" + "ldr q18, [x9, #0x20]\n" + ".inst 0x6e51ec2e // bfmmla v14.4s, v1.8h, v17.8h\n" + "ldr q17, [x9, #0x30]\n" "sub x27, x27, #0x8\n" - ".inst 0x6e47ec2b // bfmmla v11.4s, v1.8h, v7.8h\n" - ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" + ".inst 0x6e52ec2b // bfmmla v11.4s, v1.8h, v18.8h\n" + ".inst 0x6e51ec2f // bfmmla v15.4s, v1.8h, v17.8h\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" "add x12, x12, #0x40\n" @@ -738,27 +738,27 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "cmp x27, #0x4\n" "blt 62f\n" "61:" // Height 2: Multiply loop: Odd block loop - "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" + "ldr d18, [x26], #0x8\n" + "ldr d17, [x25], #0x8\n" + "trn1 v19.2d, v18.2d, v17.2d\n" "sub x27, x27, #0x4\n" - "ldr q6, [x12, #0x0]\n" - "ldr q7, [x12, #0x10]\n" - ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" - "ldr q6, [x11, #0x0]\n" - "ldr q7, [x11, #0x10]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - "ldr q6, [x9, #0x0]\n" - "ldr q7, [x9, #0x10]\n" + "ldr q18, [x12, #0x0]\n" + "ldr q17, [x12, #0x10]\n" + ".inst 0x6e52ee68 // bfmmla v8.4s, v19.8h, v18.8h\n" + ".inst 0x6e51ee6c // bfmmla v12.4s, v19.8h, v17.8h\n" + "ldr q26, [x11, #0x0]\n" + "ldr q6, [x11, #0x10]\n" + ".inst 0x6e5aee69 // bfmmla v9.4s, v19.8h, v26.8h\n" + ".inst 0x6e46ee6d // bfmmla v13.4s, v19.8h, v6.8h\n" + "ldr q18, [x10, #0x0]\n" + "ldr q17, [x10, #0x10]\n" + ".inst 0x6e52ee6a // bfmmla v10.4s, v19.8h, v18.8h\n" + ".inst 0x6e51ee6e // bfmmla v14.4s, v19.8h, v17.8h\n" + "ldr q18, [x9, #0x0]\n" + "ldr q17, [x9, #0x10]\n" "cmp x27, #0x4\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" + ".inst 0x6e52ee6b // bfmmla v11.4s, v19.8h, v18.8h\n" + ".inst 0x6e51ee6f // bfmmla v15.4s, v19.8h, v17.8h\n" "add x12, x12, #0x20\n" "add x11, x11, #0x20\n" "add x10, x10, #0x20\n" @@ -777,23 +777,23 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "ldr h1, [x26, #0x0]\n" "ldr h2, [x25, #0x0]\n" "64:" // Height 2: Multiply loop: Ragged operand read: Done - "ldr q7, [x12, #0x0]\n" - "ldr q6, [x12, #0x10]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - "ldr q7, [x11, #0x0]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - "ldr q6, [x11, #0x10]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x0]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x10]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" - "ldr q7, [x9, #0x0]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" - "ldr q6, [x9, #0x10]\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" + "ldr q18, [x12, #0x0]\n" + "ldr q17, [x12, #0x10]\n" + "trn1 v19.2d, v1.2d, v2.2d\n" + ".inst 0x6e52ee68 // bfmmla v8.4s, v19.8h, v18.8h\n" + "ldr q18, [x11, #0x0]\n" + ".inst 0x6e51ee6c // bfmmla v12.4s, v19.8h, v17.8h\n" + "ldr q17, [x11, #0x10]\n" + ".inst 0x6e52ee69 // bfmmla v9.4s, v19.8h, v18.8h\n" + "ldr q3, [x10, #0x0]\n" + ".inst 0x6e51ee6d // bfmmla v13.4s, v19.8h, v17.8h\n" + "ldr q27, [x10, #0x10]\n" + ".inst 0x6e43ee6a // bfmmla v10.4s, v19.8h, v3.8h\n" + "ldr q18, [x9, #0x0]\n" + ".inst 0x6e5bee6e // bfmmla v14.4s, v19.8h, v27.8h\n" + "ldr q17, [x9, #0x10]\n" + ".inst 0x6e52ee6b // bfmmla v11.4s, v19.8h, v18.8h\n" + ".inst 0x6e51ee6f // bfmmla v15.4s, v19.8h, v17.8h\n" "add x12, x12, #0x20\n" "add x11, x11, #0x20\n" "add x10, x10, #0x20\n" @@ -815,25 +815,25 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "uzp2 v11.2d, v11.2d, v15.2d\n" "tbz %x[flags], #1, 66f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v18.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v7.4s, v7.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmax v7.4s, v7.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" + "ld1r { v17.4s }, [x20]\n" + "fmin v7.4s, v7.4s, v18.4s\n" + "fmin v12.4s, v12.4s, v18.4s\n" + "fmin v13.4s, v13.4s, v18.4s\n" + "fmin v14.4s, v14.4s, v18.4s\n" + "fmin v8.4s, v8.4s, v18.4s\n" + "fmin v9.4s, v9.4s, v18.4s\n" + "fmin v10.4s, v10.4s, v18.4s\n" + "fmin v11.4s, v11.4s, v18.4s\n" + "fmax v7.4s, v7.4s, v17.4s\n" + "fmax v12.4s, v12.4s, v17.4s\n" + "fmax v13.4s, v13.4s, v17.4s\n" + "fmax v14.4s, v14.4s, v17.4s\n" + "fmax v8.4s, v8.4s, v17.4s\n" + "fmax v9.4s, v9.4s, v17.4s\n" + "fmax v10.4s, v10.4s, v17.4s\n" + "fmax v11.4s, v11.4s, v17.4s\n" "66:" // Height 2: No activation "cmp x14, #0x10\n" "bge 75f\n" @@ -1107,13 +1107,13 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "93:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 94f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 95f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -1122,8 +1122,8 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "b 95f\n" "94:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" "95:" // Height 3: input setup done "cmp x27, #0x8\n" "blt 98f\n" @@ -1135,170 +1135,170 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "ldr q6, [x12, #0x10]\n" "blt 97f\n" "96:" // Height 3: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v28.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x6e47ec50 // bfmmla v16.4s, v2.8h, v7.8h\n" - "ldr q7, [x11, #0x0]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" - "ldr q6, [x11, #0x10]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" - "trn2 v3.2d, v3.2d, v4.2d\n" - ".inst 0x6e47ec51 // bfmmla v17.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x0]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" + ".inst 0x6e47ef88 // bfmmla v8.4s, v28.8h, v7.8h\n" + "trn1 v27.2d, v3.2d, v29.2d\n" + ".inst 0x6e47ef70 // bfmmla v16.4s, v27.8h, v7.8h\n" + "ldr q26, [x11, #0x0]\n" + ".inst 0x6e46ef8c // bfmmla v12.4s, v28.8h, v6.8h\n" + ".inst 0x6e46ef74 // bfmmla v20.4s, v27.8h, v6.8h\n" + "ldr q25, [x11, #0x10]\n" + ".inst 0x6e5aef89 // bfmmla v9.4s, v28.8h, v26.8h\n" + "trn2 v3.2d, v3.2d, v29.2d\n" + ".inst 0x6e5aef71 // bfmmla v17.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x0]\n" + ".inst 0x6e59ef8d // bfmmla v13.4s, v28.8h, v25.8h\n" "sub x27, x27, #0x8\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x10]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" + ".inst 0x6e59ef75 // bfmmla v21.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x10]\n" + ".inst 0x6e5aef8a // bfmmla v10.4s, v28.8h, v26.8h\n" "cmp x27, #0x10\n" - ".inst 0x6e47ec52 // bfmmla v18.4s, v2.8h, v7.8h\n" - "ldr q7, [x9, #0x0]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" + ".inst 0x6e5aef72 // bfmmla v18.4s, v27.8h, v26.8h\n" + "ldr q26, [x9, #0x0]\n" + ".inst 0x6e59ef8e // bfmmla v14.4s, v28.8h, v25.8h\n" "add x26, x26, #0x10\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" - "ldr q6, [x9, #0x10]\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" + ".inst 0x6e59ef76 // bfmmla v22.4s, v27.8h, v25.8h\n" + "ldr q25, [x9, #0x10]\n" + ".inst 0x6e5aef8b // bfmmla v11.4s, v28.8h, v26.8h\n" "add x25, x25, #0x10\n" - ".inst 0x6e47ec53 // bfmmla v19.4s, v2.8h, v7.8h\n" - "ldr q7, [x12, #0x20]\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" + ".inst 0x6e5aef73 // bfmmla v19.4s, v27.8h, v26.8h\n" + "ldr q26, [x12, #0x20]\n" + ".inst 0x6e59ef8f // bfmmla v15.4s, v28.8h, v25.8h\n" "add x24, x24, #0x10\n" - ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" - "ldr q6, [x12, #0x30]\n" + ".inst 0x6e59ef77 // bfmmla v23.4s, v27.8h, v25.8h\n" + "ldr q25, [x12, #0x30]\n" "ldr q2, [x25, #0x0]\n" - ".inst 0x6e47ec28 // bfmmla v8.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec70 // bfmmla v16.4s, v3.8h, v7.8h\n" - "ldr q7, [x11, #0x20]\n" - ".inst 0x6e46ec2c // bfmmla v12.4s, v1.8h, v6.8h\n" + ".inst 0x6e5aec28 // bfmmla v8.4s, v1.8h, v26.8h\n" + ".inst 0x6e5aec70 // bfmmla v16.4s, v3.8h, v26.8h\n" + "ldr q26, [x11, #0x20]\n" + ".inst 0x6e59ec2c // bfmmla v12.4s, v1.8h, v25.8h\n" "add x12, x12, #0x40\n" - ".inst 0x6e46ec74 // bfmmla v20.4s, v3.8h, v6.8h\n" - "ldr q6, [x11, #0x30]\n" - ".inst 0x6e47ec29 // bfmmla v9.4s, v1.8h, v7.8h\n" + ".inst 0x6e59ec74 // bfmmla v20.4s, v3.8h, v25.8h\n" + "ldr q25, [x11, #0x30]\n" + ".inst 0x6e5aec29 // bfmmla v9.4s, v1.8h, v26.8h\n" "add x11, x11, #0x40\n" - ".inst 0x6e47ec71 // bfmmla v17.4s, v3.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec2d // bfmmla v13.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec75 // bfmmla v21.4s, v3.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec2a // bfmmla v10.4s, v1.8h, v7.8h\n" + ".inst 0x6e5aec71 // bfmmla v17.4s, v3.8h, v26.8h\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x6e59ec2d // bfmmla v13.4s, v1.8h, v25.8h\n" + ".inst 0x6e59ec75 // bfmmla v21.4s, v3.8h, v25.8h\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e5aec2a // bfmmla v10.4s, v1.8h, v26.8h\n" "add x10, x10, #0x40\n" - ".inst 0x6e47ec72 // bfmmla v18.4s, v3.8h, v7.8h\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec76 // bfmmla v22.4s, v3.8h, v6.8h\n" - "ldr q6, [x9, #0x30]\n" + ".inst 0x6e5aec72 // bfmmla v18.4s, v3.8h, v26.8h\n" + "ldr q26, [x9, #0x20]\n" + ".inst 0x6e59ec2e // bfmmla v14.4s, v1.8h, v25.8h\n" + ".inst 0x6e59ec76 // bfmmla v22.4s, v3.8h, v25.8h\n" + "ldr q25, [x9, #0x30]\n" "add x9, x9, #0x40\n" - ".inst 0x6e47ec2b // bfmmla v11.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec73 // bfmmla v19.4s, v3.8h, v7.8h\n" + ".inst 0x6e5aec2b // bfmmla v11.4s, v1.8h, v26.8h\n" + ".inst 0x6e5aec73 // bfmmla v19.4s, v3.8h, v26.8h\n" "ldr q7, [x12, #0x0]\n" - ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" + ".inst 0x6e59ec2f // bfmmla v15.4s, v1.8h, v25.8h\n" "ldr q1, [x26, #0x0]\n" - ".inst 0x6e46ec77 // bfmmla v23.4s, v3.8h, v6.8h\n" + ".inst 0x6e59ec77 // bfmmla v23.4s, v3.8h, v25.8h\n" "ldr q3, [x24, #0x0]\n" "ldr q6, [x12, #0x10]\n" "bge 96b\n" "97:" // Height 3: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v28.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x6e47ec50 // bfmmla v16.4s, v2.8h, v7.8h\n" - "ldr q7, [x11, #0x0]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" - "ldr q6, [x11, #0x10]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" - "trn2 v3.2d, v3.2d, v4.2d\n" - ".inst 0x6e47ec51 // bfmmla v17.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x0]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" + ".inst 0x6e47ef88 // bfmmla v8.4s, v28.8h, v7.8h\n" + "trn1 v27.2d, v3.2d, v29.2d\n" + ".inst 0x6e47ef70 // bfmmla v16.4s, v27.8h, v7.8h\n" + "ldr q26, [x11, #0x0]\n" + ".inst 0x6e46ef8c // bfmmla v12.4s, v28.8h, v6.8h\n" + ".inst 0x6e46ef74 // bfmmla v20.4s, v27.8h, v6.8h\n" + "ldr q25, [x11, #0x10]\n" + ".inst 0x6e5aef89 // bfmmla v9.4s, v28.8h, v26.8h\n" + "trn2 v3.2d, v3.2d, v29.2d\n" + ".inst 0x6e5aef71 // bfmmla v17.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x0]\n" + ".inst 0x6e59ef8d // bfmmla v13.4s, v28.8h, v25.8h\n" "sub x27, x27, #0x8\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x10]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" + ".inst 0x6e59ef75 // bfmmla v21.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x10]\n" + ".inst 0x6e5aef8a // bfmmla v10.4s, v28.8h, v26.8h\n" "add x26, x26, #0x10\n" - ".inst 0x6e47ec52 // bfmmla v18.4s, v2.8h, v7.8h\n" - "ldr q7, [x9, #0x0]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" + ".inst 0x6e5aef72 // bfmmla v18.4s, v27.8h, v26.8h\n" + "ldr q26, [x9, #0x0]\n" + ".inst 0x6e59ef8e // bfmmla v14.4s, v28.8h, v25.8h\n" "add x25, x25, #0x10\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" - "ldr q6, [x9, #0x10]\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" + ".inst 0x6e59ef76 // bfmmla v22.4s, v27.8h, v25.8h\n" + "ldr q25, [x9, #0x10]\n" + ".inst 0x6e5aef8b // bfmmla v11.4s, v28.8h, v26.8h\n" "add x24, x24, #0x10\n" - ".inst 0x6e47ec53 // bfmmla v19.4s, v2.8h, v7.8h\n" - "ldr q7, [x12, #0x20]\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" - "ldr q6, [x12, #0x30]\n" - ".inst 0x6e47ec28 // bfmmla v8.4s, v1.8h, v7.8h\n" + ".inst 0x6e5aef73 // bfmmla v19.4s, v27.8h, v26.8h\n" + "ldr q26, [x12, #0x20]\n" + ".inst 0x6e59ef8f // bfmmla v15.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef77 // bfmmla v23.4s, v27.8h, v25.8h\n" + "ldr q25, [x12, #0x30]\n" + ".inst 0x6e5aec28 // bfmmla v8.4s, v1.8h, v26.8h\n" "add x12, x12, #0x40\n" - ".inst 0x6e47ec70 // bfmmla v16.4s, v3.8h, v7.8h\n" - "ldr q7, [x11, #0x20]\n" - ".inst 0x6e46ec2c // bfmmla v12.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec74 // bfmmla v20.4s, v3.8h, v6.8h\n" - "ldr q6, [x11, #0x30]\n" - ".inst 0x6e47ec29 // bfmmla v9.4s, v1.8h, v7.8h\n" + ".inst 0x6e5aec70 // bfmmla v16.4s, v3.8h, v26.8h\n" + "ldr q26, [x11, #0x20]\n" + ".inst 0x6e59ec2c // bfmmla v12.4s, v1.8h, v25.8h\n" + ".inst 0x6e59ec74 // bfmmla v20.4s, v3.8h, v25.8h\n" + "ldr q25, [x11, #0x30]\n" + ".inst 0x6e5aec29 // bfmmla v9.4s, v1.8h, v26.8h\n" "add x11, x11, #0x40\n" - ".inst 0x6e47ec71 // bfmmla v17.4s, v3.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec2d // bfmmla v13.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec75 // bfmmla v21.4s, v3.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec2a // bfmmla v10.4s, v1.8h, v7.8h\n" + ".inst 0x6e5aec71 // bfmmla v17.4s, v3.8h, v26.8h\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x6e59ec2d // bfmmla v13.4s, v1.8h, v25.8h\n" + ".inst 0x6e59ec75 // bfmmla v21.4s, v3.8h, v25.8h\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e5aec2a // bfmmla v10.4s, v1.8h, v26.8h\n" "add x10, x10, #0x40\n" - ".inst 0x6e47ec72 // bfmmla v18.4s, v3.8h, v7.8h\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec76 // bfmmla v22.4s, v3.8h, v6.8h\n" - "ldr q6, [x9, #0x30]\n" + ".inst 0x6e5aec72 // bfmmla v18.4s, v3.8h, v26.8h\n" + "ldr q26, [x9, #0x20]\n" + ".inst 0x6e59ec2e // bfmmla v14.4s, v1.8h, v25.8h\n" + ".inst 0x6e59ec76 // bfmmla v22.4s, v3.8h, v25.8h\n" + "ldr q25, [x9, #0x30]\n" "add x9, x9, #0x40\n" - ".inst 0x6e47ec2b // bfmmla v11.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec73 // bfmmla v19.4s, v3.8h, v7.8h\n" - ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec77 // bfmmla v23.4s, v3.8h, v6.8h\n" + ".inst 0x6e5aec2b // bfmmla v11.4s, v1.8h, v26.8h\n" + ".inst 0x6e5aec73 // bfmmla v19.4s, v3.8h, v26.8h\n" + ".inst 0x6e59ec2f // bfmmla v15.4s, v1.8h, v25.8h\n" + ".inst 0x6e59ec77 // bfmmla v23.4s, v3.8h, v25.8h\n" "98:" // Height 3: Multiply loop: Main loop skip "cbz x27, 103f\n" "cmp x27, #0x4\n" "blt 100f\n" "99:" // Height 3: Multiply loop: Odd block loop - "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "ldr d3, [x24], #0x8\n" - "ldr q6, [x12, #0x0]\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - "ldr q7, [x12, #0x10]\n" - ".inst 0x6e46ec50 // bfmmla v16.4s, v2.8h, v6.8h\n" - "ldr q6, [x11, #0x0]\n" - ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec54 // bfmmla v20.4s, v2.8h, v7.8h\n" - "ldr q7, [x11, #0x10]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" + "ldr d26, [x26], #0x8\n" + "ldr d25, [x25], #0x8\n" + "trn1 v28.2d, v26.2d, v25.2d\n" + "ldr d25, [x24], #0x8\n" + "ldr q26, [x12, #0x0]\n" + "trn1 v27.2d, v25.2d, v27.2d\n" + ".inst 0x6e5aef88 // bfmmla v8.4s, v28.8h, v26.8h\n" + "ldr q25, [x12, #0x10]\n" + ".inst 0x6e5aef70 // bfmmla v16.4s, v27.8h, v26.8h\n" + "ldr q26, [x11, #0x0]\n" + ".inst 0x6e59ef8c // bfmmla v12.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef74 // bfmmla v20.4s, v27.8h, v25.8h\n" + "ldr q25, [x11, #0x10]\n" + ".inst 0x6e5aef89 // bfmmla v9.4s, v28.8h, v26.8h\n" "sub x27, x27, #0x4\n" - ".inst 0x6e46ec51 // bfmmla v17.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x0]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" + ".inst 0x6e5aef71 // bfmmla v17.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x0]\n" + ".inst 0x6e59ef8d // bfmmla v13.4s, v28.8h, v25.8h\n" "cmp x27, #0x4\n" - ".inst 0x6e47ec55 // bfmmla v21.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" + ".inst 0x6e59ef75 // bfmmla v21.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x10]\n" + ".inst 0x6e5aef8a // bfmmla v10.4s, v28.8h, v26.8h\n" "add x12, x12, #0x20\n" - ".inst 0x6e46ec52 // bfmmla v18.4s, v2.8h, v6.8h\n" - "ldr q6, [x9, #0x0]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" + ".inst 0x6e5aef72 // bfmmla v18.4s, v27.8h, v26.8h\n" + "ldr q26, [x9, #0x0]\n" + ".inst 0x6e59ef8e // bfmmla v14.4s, v28.8h, v25.8h\n" "add x11, x11, #0x20\n" - ".inst 0x6e47ec56 // bfmmla v22.4s, v2.8h, v7.8h\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" + ".inst 0x6e59ef76 // bfmmla v22.4s, v27.8h, v25.8h\n" + "ldr q25, [x9, #0x10]\n" + ".inst 0x6e5aef8b // bfmmla v11.4s, v28.8h, v26.8h\n" "add x10, x10, #0x20\n" - ".inst 0x6e46ec53 // bfmmla v19.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" + ".inst 0x6e5aef73 // bfmmla v19.4s, v27.8h, v26.8h\n" + ".inst 0x6e59ef8f // bfmmla v15.4s, v28.8h, v25.8h\n" "add x9, x9, #0x20\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" + ".inst 0x6e59ef77 // bfmmla v23.4s, v27.8h, v25.8h\n" "bge 99b\n" "100:" // Height 3: Multiply loop: Skip odd blocks "cbz x27, 103f\n" @@ -1316,36 +1316,36 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "ldr h2, [x25, #0x0]\n" "ldr h3, [x24, #0x0]\n" "102:" // Height 3: Multiply loop: Ragged operand read: Done - "ldr q7, [x12, #0x0]\n" - "ldr q6, [x12, #0x10]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec50 // bfmmla v16.4s, v2.8h, v7.8h\n" - "ldr q7, [x11, #0x0]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" - "ldr q6, [x11, #0x10]\n" + "ldr q26, [x12, #0x0]\n" + "ldr q29, [x12, #0x10]\n" + "trn1 v28.2d, v1.2d, v2.2d\n" + "trn1 v27.2d, v3.2d, v25.2d\n" + ".inst 0x6e5aef88 // bfmmla v8.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef70 // bfmmla v16.4s, v27.8h, v26.8h\n" + "ldr q26, [x11, #0x0]\n" + ".inst 0x6e5def8c // bfmmla v12.4s, v28.8h, v29.8h\n" + ".inst 0x6e5def74 // bfmmla v20.4s, v27.8h, v29.8h\n" + "ldr q25, [x11, #0x10]\n" "add x12, x12, #0x20\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec51 // bfmmla v17.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x0]\n" + ".inst 0x6e5aef89 // bfmmla v9.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef71 // bfmmla v17.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x0]\n" "add x11, x11, #0x20\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x10]\n" + ".inst 0x6e59ef8d // bfmmla v13.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef75 // bfmmla v21.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x10]\n" "add x10, x10, #0x20\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec52 // bfmmla v18.4s, v2.8h, v7.8h\n" - "ldr q7, [x9, #0x0]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" - "ldr q6, [x9, #0x10]\n" + ".inst 0x6e5aef8a // bfmmla v10.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef72 // bfmmla v18.4s, v27.8h, v26.8h\n" + "ldr q26, [x9, #0x0]\n" + ".inst 0x6e59ef8e // bfmmla v14.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef76 // bfmmla v22.4s, v27.8h, v25.8h\n" + "ldr q25, [x9, #0x10]\n" "add x9, x9, #0x20\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec53 // bfmmla v19.4s, v2.8h, v7.8h\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" + ".inst 0x6e5aef8b // bfmmla v11.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef73 // bfmmla v19.4s, v27.8h, v26.8h\n" + ".inst 0x6e59ef8f // bfmmla v15.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef77 // bfmmla v23.4s, v27.8h, v25.8h\n" "103:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1368,33 +1368,33 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "uzp1 v19.2d, v19.2d, v23.2d\n" "tbz %x[flags], #1, 104f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v26.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v7.4s, v7.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v16.4s, v16.4s, v1.4s\n" - "fmin v17.4s, v17.4s, v1.4s\n" - "fmin v18.4s, v18.4s, v1.4s\n" - "fmin v19.4s, v19.4s, v1.4s\n" - "fmax v7.4s, v7.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v16.4s, v16.4s, v0.4s\n" - "fmax v17.4s, v17.4s, v0.4s\n" - "fmax v18.4s, v18.4s, v0.4s\n" - "fmax v19.4s, v19.4s, v0.4s\n" + "ld1r { v25.4s }, [x20]\n" + "fmin v7.4s, v7.4s, v26.4s\n" + "fmin v12.4s, v12.4s, v26.4s\n" + "fmin v13.4s, v13.4s, v26.4s\n" + "fmin v14.4s, v14.4s, v26.4s\n" + "fmin v8.4s, v8.4s, v26.4s\n" + "fmin v9.4s, v9.4s, v26.4s\n" + "fmin v10.4s, v10.4s, v26.4s\n" + "fmin v11.4s, v11.4s, v26.4s\n" + "fmin v16.4s, v16.4s, v26.4s\n" + "fmin v17.4s, v17.4s, v26.4s\n" + "fmin v18.4s, v18.4s, v26.4s\n" + "fmin v19.4s, v19.4s, v26.4s\n" + "fmax v7.4s, v7.4s, v25.4s\n" + "fmax v12.4s, v12.4s, v25.4s\n" + "fmax v13.4s, v13.4s, v25.4s\n" + "fmax v14.4s, v14.4s, v25.4s\n" + "fmax v8.4s, v8.4s, v25.4s\n" + "fmax v9.4s, v9.4s, v25.4s\n" + "fmax v10.4s, v10.4s, v25.4s\n" + "fmax v11.4s, v11.4s, v25.4s\n" + "fmax v16.4s, v16.4s, v25.4s\n" + "fmax v17.4s, v17.4s, v25.4s\n" + "fmax v18.4s, v18.4s, v25.4s\n" + "fmax v19.4s, v19.4s, v25.4s\n" "104:" // Height 3: No activation "cmp x14, #0x10\n" "bge 113f\n" @@ -1709,14 +1709,14 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "131:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 132f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 133f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -1726,9 +1726,9 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "b 133f\n" "132:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" "133:" // Height 4: input setup done "cmp x27, #0x8\n" "blt 136f\n" @@ -1741,174 +1741,174 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "ldr q6, [x12, #0x10]\n" "blt 135f\n" "134:" // Height 4: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v28.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" + ".inst 0x6e47ef88 // bfmmla v8.4s, v28.8h, v7.8h\n" "sub x27, x27, #0x8\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x6e47ec50 // bfmmla v16.4s, v2.8h, v7.8h\n" - "ldr q7, [x11, #0x0]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" - "ldr q6, [x11, #0x10]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" + "trn1 v27.2d, v3.2d, v4.2d\n" + ".inst 0x6e47ef70 // bfmmla v16.4s, v27.8h, v7.8h\n" + "ldr q26, [x11, #0x0]\n" + ".inst 0x6e46ef8c // bfmmla v12.4s, v28.8h, v6.8h\n" + ".inst 0x6e46ef74 // bfmmla v20.4s, v27.8h, v6.8h\n" + "ldr q25, [x11, #0x10]\n" + ".inst 0x6e5aef89 // bfmmla v9.4s, v28.8h, v26.8h\n" "trn2 v3.2d, v3.2d, v4.2d\n" - ".inst 0x6e47ec51 // bfmmla v17.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x0]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" + ".inst 0x6e5aef71 // bfmmla v17.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x0]\n" + ".inst 0x6e59ef8d // bfmmla v13.4s, v28.8h, v25.8h\n" "cmp x27, #0x10\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x10]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" + ".inst 0x6e59ef75 // bfmmla v21.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x10]\n" + ".inst 0x6e5aef8a // bfmmla v10.4s, v28.8h, v26.8h\n" "add x26, x26, #0x10\n" - ".inst 0x6e47ec52 // bfmmla v18.4s, v2.8h, v7.8h\n" - "ldr q7, [x9, #0x0]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" + ".inst 0x6e5aef72 // bfmmla v18.4s, v27.8h, v26.8h\n" + "ldr q26, [x9, #0x0]\n" + ".inst 0x6e59ef8e // bfmmla v14.4s, v28.8h, v25.8h\n" "add x25, x25, #0x10\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" - "ldr q6, [x9, #0x10]\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" + ".inst 0x6e59ef76 // bfmmla v22.4s, v27.8h, v25.8h\n" + "ldr q25, [x9, #0x10]\n" + ".inst 0x6e5aef8b // bfmmla v11.4s, v28.8h, v26.8h\n" "add x24, x24, #0x10\n" - ".inst 0x6e47ec53 // bfmmla v19.4s, v2.8h, v7.8h\n" - "ldr q7, [x12, #0x20]\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" + ".inst 0x6e5aef73 // bfmmla v19.4s, v27.8h, v26.8h\n" + "ldr q26, [x12, #0x20]\n" + ".inst 0x6e59ef8f // bfmmla v15.4s, v28.8h, v25.8h\n" "add x23, x23, #0x10\n" "ldr q4, [x23, #0x0]\n" - ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" - "ldr q6, [x12, #0x30]\n" - ".inst 0x6e47ec28 // bfmmla v8.4s, v1.8h, v7.8h\n" + ".inst 0x6e59ef77 // bfmmla v23.4s, v27.8h, v25.8h\n" + "ldr q25, [x12, #0x30]\n" + ".inst 0x6e5aec28 // bfmmla v8.4s, v1.8h, v26.8h\n" "ldr q2, [x25, #0x0]\n" - ".inst 0x6e47ec70 // bfmmla v16.4s, v3.8h, v7.8h\n" - "ldr q7, [x11, #0x20]\n" - ".inst 0x6e46ec2c // bfmmla v12.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec74 // bfmmla v20.4s, v3.8h, v6.8h\n" - "ldr q6, [x11, #0x30]\n" - ".inst 0x6e47ec29 // bfmmla v9.4s, v1.8h, v7.8h\n" + ".inst 0x6e5aec70 // bfmmla v16.4s, v3.8h, v26.8h\n" + "ldr q26, [x11, #0x20]\n" + ".inst 0x6e59ec2c // bfmmla v12.4s, v1.8h, v25.8h\n" + ".inst 0x6e59ec74 // bfmmla v20.4s, v3.8h, v25.8h\n" + "ldr q25, [x11, #0x30]\n" + ".inst 0x6e5aec29 // bfmmla v9.4s, v1.8h, v26.8h\n" "add x12, x12, #0x40\n" - ".inst 0x6e47ec71 // bfmmla v17.4s, v3.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec2d // bfmmla v13.4s, v1.8h, v6.8h\n" + ".inst 0x6e5aec71 // bfmmla v17.4s, v3.8h, v26.8h\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x6e59ec2d // bfmmla v13.4s, v1.8h, v25.8h\n" "add x11, x11, #0x40\n" - ".inst 0x6e46ec75 // bfmmla v21.4s, v3.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec2a // bfmmla v10.4s, v1.8h, v7.8h\n" + ".inst 0x6e59ec75 // bfmmla v21.4s, v3.8h, v25.8h\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e5aec2a // bfmmla v10.4s, v1.8h, v26.8h\n" "add x10, x10, #0x40\n" - ".inst 0x6e47ec72 // bfmmla v18.4s, v3.8h, v7.8h\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec76 // bfmmla v22.4s, v3.8h, v6.8h\n" - "ldr q6, [x9, #0x30]\n" + ".inst 0x6e5aec72 // bfmmla v18.4s, v3.8h, v26.8h\n" + "ldr q26, [x9, #0x20]\n" + ".inst 0x6e59ec2e // bfmmla v14.4s, v1.8h, v25.8h\n" + ".inst 0x6e59ec76 // bfmmla v22.4s, v3.8h, v25.8h\n" + "ldr q25, [x9, #0x30]\n" "add x9, x9, #0x40\n" - ".inst 0x6e47ec2b // bfmmla v11.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec73 // bfmmla v19.4s, v3.8h, v7.8h\n" + ".inst 0x6e5aec2b // bfmmla v11.4s, v1.8h, v26.8h\n" + ".inst 0x6e5aec73 // bfmmla v19.4s, v3.8h, v26.8h\n" "ldr q7, [x12, #0x0]\n" - ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" + ".inst 0x6e59ec2f // bfmmla v15.4s, v1.8h, v25.8h\n" "ldr q1, [x26, #0x0]\n" - ".inst 0x6e46ec77 // bfmmla v23.4s, v3.8h, v6.8h\n" + ".inst 0x6e59ec77 // bfmmla v23.4s, v3.8h, v25.8h\n" "ldr q3, [x24, #0x0]\n" "ldr q6, [x12, #0x10]\n" "bge 134b\n" "135:" // Height 4: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v28.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" + ".inst 0x6e47ef88 // bfmmla v8.4s, v28.8h, v7.8h\n" "sub x27, x27, #0x8\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x6e47ec50 // bfmmla v16.4s, v2.8h, v7.8h\n" - "ldr q7, [x11, #0x0]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" - "ldr q6, [x11, #0x10]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" + "trn1 v27.2d, v3.2d, v4.2d\n" + ".inst 0x6e47ef70 // bfmmla v16.4s, v27.8h, v7.8h\n" + "ldr q26, [x11, #0x0]\n" + ".inst 0x6e46ef8c // bfmmla v12.4s, v28.8h, v6.8h\n" + ".inst 0x6e46ef74 // bfmmla v20.4s, v27.8h, v6.8h\n" + "ldr q25, [x11, #0x10]\n" + ".inst 0x6e5aef89 // bfmmla v9.4s, v28.8h, v26.8h\n" "trn2 v3.2d, v3.2d, v4.2d\n" - ".inst 0x6e47ec51 // bfmmla v17.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x0]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" + ".inst 0x6e5aef71 // bfmmla v17.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x0]\n" + ".inst 0x6e59ef8d // bfmmla v13.4s, v28.8h, v25.8h\n" "add x26, x26, #0x10\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x10]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" + ".inst 0x6e59ef75 // bfmmla v21.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x10]\n" + ".inst 0x6e5aef8a // bfmmla v10.4s, v28.8h, v26.8h\n" "add x25, x25, #0x10\n" - ".inst 0x6e47ec52 // bfmmla v18.4s, v2.8h, v7.8h\n" - "ldr q7, [x9, #0x0]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" + ".inst 0x6e5aef72 // bfmmla v18.4s, v27.8h, v26.8h\n" + "ldr q26, [x9, #0x0]\n" + ".inst 0x6e59ef8e // bfmmla v14.4s, v28.8h, v25.8h\n" "add x24, x24, #0x10\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" - "ldr q6, [x9, #0x10]\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" + ".inst 0x6e59ef76 // bfmmla v22.4s, v27.8h, v25.8h\n" + "ldr q25, [x9, #0x10]\n" + ".inst 0x6e5aef8b // bfmmla v11.4s, v28.8h, v26.8h\n" "add x23, x23, #0x10\n" - ".inst 0x6e47ec53 // bfmmla v19.4s, v2.8h, v7.8h\n" - "ldr q7, [x12, #0x20]\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" - "ldr q6, [x12, #0x30]\n" - ".inst 0x6e47ec28 // bfmmla v8.4s, v1.8h, v7.8h\n" + ".inst 0x6e5aef73 // bfmmla v19.4s, v27.8h, v26.8h\n" + "ldr q26, [x12, #0x20]\n" + ".inst 0x6e59ef8f // bfmmla v15.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef77 // bfmmla v23.4s, v27.8h, v25.8h\n" + "ldr q25, [x12, #0x30]\n" + ".inst 0x6e5aec28 // bfmmla v8.4s, v1.8h, v26.8h\n" "add x12, x12, #0x40\n" - ".inst 0x6e47ec70 // bfmmla v16.4s, v3.8h, v7.8h\n" - "ldr q7, [x11, #0x20]\n" - ".inst 0x6e46ec2c // bfmmla v12.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec74 // bfmmla v20.4s, v3.8h, v6.8h\n" - "ldr q6, [x11, #0x30]\n" - ".inst 0x6e47ec29 // bfmmla v9.4s, v1.8h, v7.8h\n" + ".inst 0x6e5aec70 // bfmmla v16.4s, v3.8h, v26.8h\n" + "ldr q26, [x11, #0x20]\n" + ".inst 0x6e59ec2c // bfmmla v12.4s, v1.8h, v25.8h\n" + ".inst 0x6e59ec74 // bfmmla v20.4s, v3.8h, v25.8h\n" + "ldr q25, [x11, #0x30]\n" + ".inst 0x6e5aec29 // bfmmla v9.4s, v1.8h, v26.8h\n" "add x11, x11, #0x40\n" - ".inst 0x6e47ec71 // bfmmla v17.4s, v3.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec2d // bfmmla v13.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec75 // bfmmla v21.4s, v3.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec2a // bfmmla v10.4s, v1.8h, v7.8h\n" + ".inst 0x6e5aec71 // bfmmla v17.4s, v3.8h, v26.8h\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x6e59ec2d // bfmmla v13.4s, v1.8h, v25.8h\n" + ".inst 0x6e59ec75 // bfmmla v21.4s, v3.8h, v25.8h\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e5aec2a // bfmmla v10.4s, v1.8h, v26.8h\n" "add x10, x10, #0x40\n" - ".inst 0x6e47ec72 // bfmmla v18.4s, v3.8h, v7.8h\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec76 // bfmmla v22.4s, v3.8h, v6.8h\n" - "ldr q6, [x9, #0x30]\n" + ".inst 0x6e5aec72 // bfmmla v18.4s, v3.8h, v26.8h\n" + "ldr q26, [x9, #0x20]\n" + ".inst 0x6e59ec2e // bfmmla v14.4s, v1.8h, v25.8h\n" + ".inst 0x6e59ec76 // bfmmla v22.4s, v3.8h, v25.8h\n" + "ldr q25, [x9, #0x30]\n" "add x9, x9, #0x40\n" - ".inst 0x6e47ec2b // bfmmla v11.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec73 // bfmmla v19.4s, v3.8h, v7.8h\n" - ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec77 // bfmmla v23.4s, v3.8h, v6.8h\n" + ".inst 0x6e5aec2b // bfmmla v11.4s, v1.8h, v26.8h\n" + ".inst 0x6e5aec73 // bfmmla v19.4s, v3.8h, v26.8h\n" + ".inst 0x6e59ec2f // bfmmla v15.4s, v1.8h, v25.8h\n" + ".inst 0x6e59ec77 // bfmmla v23.4s, v3.8h, v25.8h\n" "136:" // Height 4: Multiply loop: Main loop skip "cbz x27, 141f\n" "cmp x27, #0x4\n" "blt 138f\n" "137:" // Height 4: Multiply loop: Odd block loop - "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" + "ldr d26, [x26], #0x8\n" + "ldr d25, [x25], #0x8\n" + "trn1 v28.2d, v26.2d, v25.2d\n" "sub x27, x27, #0x4\n" - "ldr d3, [x24], #0x8\n" - "ldr d4, [x23], #0x8\n" - "trn1 v2.2d, v3.2d, v4.2d\n" + "ldr d26, [x24], #0x8\n" + "ldr d25, [x23], #0x8\n" + "trn1 v27.2d, v26.2d, v25.2d\n" "cmp x27, #0x4\n" - "ldr q6, [x12, #0x0]\n" - "ldr q7, [x12, #0x10]\n" - ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec50 // bfmmla v16.4s, v2.8h, v6.8h\n" - "ldr q6, [x11, #0x0]\n" - ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec54 // bfmmla v20.4s, v2.8h, v7.8h\n" - "ldr q7, [x11, #0x10]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec51 // bfmmla v17.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x0]\n" + "ldr q26, [x12, #0x0]\n" + "ldr q25, [x12, #0x10]\n" + ".inst 0x6e5aef88 // bfmmla v8.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef70 // bfmmla v16.4s, v27.8h, v26.8h\n" + "ldr q26, [x11, #0x0]\n" + ".inst 0x6e59ef8c // bfmmla v12.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef74 // bfmmla v20.4s, v27.8h, v25.8h\n" + "ldr q25, [x11, #0x10]\n" + ".inst 0x6e5aef89 // bfmmla v9.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef71 // bfmmla v17.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x0]\n" "add x12, x12, #0x20\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec55 // bfmmla v21.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x10]\n" + ".inst 0x6e59ef8d // bfmmla v13.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef75 // bfmmla v21.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x10]\n" "add x11, x11, #0x20\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec52 // bfmmla v18.4s, v2.8h, v6.8h\n" - "ldr q6, [x9, #0x0]\n" + ".inst 0x6e5aef8a // bfmmla v10.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef72 // bfmmla v18.4s, v27.8h, v26.8h\n" + "ldr q26, [x9, #0x0]\n" "add x10, x10, #0x20\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec56 // bfmmla v22.4s, v2.8h, v7.8h\n" - "ldr q7, [x9, #0x10]\n" + ".inst 0x6e59ef8e // bfmmla v14.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef76 // bfmmla v22.4s, v27.8h, v25.8h\n" + "ldr q25, [x9, #0x10]\n" "add x9, x9, #0x20\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec53 // bfmmla v19.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" + ".inst 0x6e5aef8b // bfmmla v11.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef73 // bfmmla v19.4s, v27.8h, v26.8h\n" + ".inst 0x6e59ef8f // bfmmla v15.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef77 // bfmmla v23.4s, v27.8h, v25.8h\n" "bge 137b\n" "138:" // Height 4: Multiply loop: Skip odd blocks "cbz x27, 141f\n" @@ -1929,36 +1929,36 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "ldr h3, [x24, #0x0]\n" "ldr h4, [x23, #0x0]\n" "140:" // Height 4: Multiply loop: Ragged operand read: Done - "ldr q7, [x12, #0x0]\n" - "ldr q6, [x12, #0x10]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec50 // bfmmla v16.4s, v2.8h, v7.8h\n" - "ldr q7, [x11, #0x0]\n" + "ldr q26, [x12, #0x0]\n" + "ldr q25, [x12, #0x10]\n" + "trn1 v28.2d, v1.2d, v2.2d\n" + "trn1 v27.2d, v3.2d, v4.2d\n" + ".inst 0x6e5aef88 // bfmmla v8.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef70 // bfmmla v16.4s, v27.8h, v26.8h\n" + "ldr q26, [x11, #0x0]\n" "add x12, x12, #0x20\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" - "ldr q6, [x11, #0x10]\n" + ".inst 0x6e59ef8c // bfmmla v12.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef74 // bfmmla v20.4s, v27.8h, v25.8h\n" + "ldr q25, [x11, #0x10]\n" "add x11, x11, #0x20\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec51 // bfmmla v17.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x0]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x10]\n" + ".inst 0x6e5aef89 // bfmmla v9.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef71 // bfmmla v17.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x0]\n" + ".inst 0x6e59ef8d // bfmmla v13.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef75 // bfmmla v21.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x10]\n" "add x10, x10, #0x20\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec52 // bfmmla v18.4s, v2.8h, v7.8h\n" - "ldr q7, [x9, #0x0]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" - "ldr q6, [x9, #0x10]\n" + ".inst 0x6e5aef8a // bfmmla v10.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef72 // bfmmla v18.4s, v27.8h, v26.8h\n" + "ldr q26, [x9, #0x0]\n" + ".inst 0x6e59ef8e // bfmmla v14.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef76 // bfmmla v22.4s, v27.8h, v25.8h\n" + "ldr q25, [x9, #0x10]\n" "add x9, x9, #0x20\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec53 // bfmmla v19.4s, v2.8h, v7.8h\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" + ".inst 0x6e5aef8b // bfmmla v11.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef73 // bfmmla v19.4s, v27.8h, v26.8h\n" + ".inst 0x6e59ef8f // bfmmla v15.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef77 // bfmmla v23.4s, v27.8h, v25.8h\n" "141:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1986,41 +1986,41 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "uzp2 v19.2d, v19.2d, v23.2d\n" "tbz %x[flags], #1, 142f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v26.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v7.4s, v7.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v15.4s, v15.4s, v1.4s\n" - "fmin v20.4s, v20.4s, v1.4s\n" - "fmin v21.4s, v21.4s, v1.4s\n" - "fmin v22.4s, v22.4s, v1.4s\n" - "fmin v16.4s, v16.4s, v1.4s\n" - "fmin v17.4s, v17.4s, v1.4s\n" - "fmin v18.4s, v18.4s, v1.4s\n" - "fmin v19.4s, v19.4s, v1.4s\n" - "fmax v7.4s, v7.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" - "fmax v20.4s, v20.4s, v0.4s\n" - "fmax v21.4s, v21.4s, v0.4s\n" - "fmax v22.4s, v22.4s, v0.4s\n" - "fmax v16.4s, v16.4s, v0.4s\n" - "fmax v17.4s, v17.4s, v0.4s\n" - "fmax v18.4s, v18.4s, v0.4s\n" - "fmax v19.4s, v19.4s, v0.4s\n" + "ld1r { v25.4s }, [x20]\n" + "fmin v7.4s, v7.4s, v26.4s\n" + "fmin v12.4s, v12.4s, v26.4s\n" + "fmin v13.4s, v13.4s, v26.4s\n" + "fmin v14.4s, v14.4s, v26.4s\n" + "fmin v8.4s, v8.4s, v26.4s\n" + "fmin v9.4s, v9.4s, v26.4s\n" + "fmin v10.4s, v10.4s, v26.4s\n" + "fmin v11.4s, v11.4s, v26.4s\n" + "fmin v15.4s, v15.4s, v26.4s\n" + "fmin v20.4s, v20.4s, v26.4s\n" + "fmin v21.4s, v21.4s, v26.4s\n" + "fmin v22.4s, v22.4s, v26.4s\n" + "fmin v16.4s, v16.4s, v26.4s\n" + "fmin v17.4s, v17.4s, v26.4s\n" + "fmin v18.4s, v18.4s, v26.4s\n" + "fmin v19.4s, v19.4s, v26.4s\n" + "fmax v7.4s, v7.4s, v25.4s\n" + "fmax v12.4s, v12.4s, v25.4s\n" + "fmax v13.4s, v13.4s, v25.4s\n" + "fmax v14.4s, v14.4s, v25.4s\n" + "fmax v8.4s, v8.4s, v25.4s\n" + "fmax v9.4s, v9.4s, v25.4s\n" + "fmax v10.4s, v10.4s, v25.4s\n" + "fmax v11.4s, v11.4s, v25.4s\n" + "fmax v15.4s, v15.4s, v25.4s\n" + "fmax v20.4s, v20.4s, v25.4s\n" + "fmax v21.4s, v21.4s, v25.4s\n" + "fmax v22.4s, v22.4s, v25.4s\n" + "fmax v16.4s, v16.4s, v25.4s\n" + "fmax v17.4s, v17.4s, v25.4s\n" + "fmax v18.4s, v18.4s, v25.4s\n" + "fmax v19.4s, v19.4s, v25.4s\n" "142:" // Height 4: No activation "cmp x14, #0x10\n" "bge 151f\n" @@ -2400,15 +2400,15 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "169:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 170f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 171f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -2419,10 +2419,10 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "b 171f\n" "170:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" - "add x22, x23, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" + "add x22, x23, x21, LSL #1\n" "171:" // Height 5: input setup done "cmp x27, #0x8\n" "blt 174f\n" @@ -2435,170 +2435,170 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "ldr q7, [x12, #0x0]\n" "blt 173f\n" "172:" // Height 5: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v6.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" + ".inst 0x6e47ecc8 // bfmmla v8.4s, v6.8h, v7.8h\n" "trn1 v2.2d, v3.2d, v4.2d\n" "trn2 v3.2d, v3.2d, v4.2d\n" ".inst 0x6e47ec50 // bfmmla v16.4s, v2.8h, v7.8h\n" "sub x27, x27, #0x8\n" - "trn1 v4.2d, v5.2d, v6.2d\n" - "trn2 v5.2d, v5.2d, v6.2d\n" - "ldr q6, [x12, #0x10]\n" + "trn1 v4.2d, v5.2d, v0.2d\n" + "trn2 v5.2d, v5.2d, v0.2d\n" + "ldr q0, [x12, #0x10]\n" ".inst 0x6e47ec98 // bfmmla v24.4s, v4.8h, v7.8h\n" "ldr q7, [x11, #0x0]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" + ".inst 0x6e40eccc // bfmmla v12.4s, v6.8h, v0.8h\n" + ".inst 0x6e40ec54 // bfmmla v20.4s, v2.8h, v0.8h\n" "cmp x27, #0x10\n" - ".inst 0x6e46ec9c // bfmmla v28.4s, v4.8h, v6.8h\n" - "ldr q6, [x11, #0x10]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" + ".inst 0x6e40ec9c // bfmmla v28.4s, v4.8h, v0.8h\n" + "ldr q0, [x11, #0x10]\n" + ".inst 0x6e47ecc9 // bfmmla v9.4s, v6.8h, v7.8h\n" "add x26, x26, #0x10\n" ".inst 0x6e47ec51 // bfmmla v17.4s, v2.8h, v7.8h\n" ".inst 0x6e47ec99 // bfmmla v25.4s, v4.8h, v7.8h\n" "ldr q7, [x10, #0x0]\n" "add x25, x25, #0x10\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" + ".inst 0x6e40eccd // bfmmla v13.4s, v6.8h, v0.8h\n" + ".inst 0x6e40ec55 // bfmmla v21.4s, v2.8h, v0.8h\n" "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" - ".inst 0x6e46ec9d // bfmmla v29.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x10]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" + ".inst 0x6e40ec9d // bfmmla v29.4s, v4.8h, v0.8h\n" + "ldr q0, [x10, #0x10]\n" + ".inst 0x6e47ecca // bfmmla v10.4s, v6.8h, v7.8h\n" "add x22, x22, #0x10\n" ".inst 0x6e47ec52 // bfmmla v18.4s, v2.8h, v7.8h\n" ".inst 0x6e47ec9a // bfmmla v26.4s, v4.8h, v7.8h\n" "ldr q7, [x9, #0x0]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9e // bfmmla v30.4s, v4.8h, v6.8h\n" - "ldr q6, [x9, #0x10]\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" + ".inst 0x6e40ecce // bfmmla v14.4s, v6.8h, v0.8h\n" + ".inst 0x6e40ec56 // bfmmla v22.4s, v2.8h, v0.8h\n" + ".inst 0x6e40ec9e // bfmmla v30.4s, v4.8h, v0.8h\n" + "ldr q0, [x9, #0x10]\n" + ".inst 0x6e47eccb // bfmmla v11.4s, v6.8h, v7.8h\n" ".inst 0x6e47ec53 // bfmmla v19.4s, v2.8h, v7.8h\n" ".inst 0x6e47ec9b // bfmmla v27.4s, v4.8h, v7.8h\n" "ldr q7, [x12, #0x20]\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" + ".inst 0x6e40eccf // bfmmla v15.4s, v6.8h, v0.8h\n" + ".inst 0x6e40ec57 // bfmmla v23.4s, v2.8h, v0.8h\n" "ldr q2, [x25, #0x0]\n" - ".inst 0x6e46ec9f // bfmmla v31.4s, v4.8h, v6.8h\n" - "ldr q6, [x12, #0x30]\n" + ".inst 0x6e40ec9f // bfmmla v31.4s, v4.8h, v0.8h\n" + "ldr q0, [x12, #0x30]\n" "ldr q4, [x23, #0x0]\n" ".inst 0x6e47ec28 // bfmmla v8.4s, v1.8h, v7.8h\n" ".inst 0x6e47ec70 // bfmmla v16.4s, v3.8h, v7.8h\n" ".inst 0x6e47ecb8 // bfmmla v24.4s, v5.8h, v7.8h\n" - "ldr q7, [x11, #0x20]\n" + "ldr q6, [x11, #0x20]\n" "add x12, x12, #0x40\n" - ".inst 0x6e46ec2c // bfmmla v12.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec74 // bfmmla v20.4s, v3.8h, v6.8h\n" - ".inst 0x6e46ecbc // bfmmla v28.4s, v5.8h, v6.8h\n" - "ldr q6, [x11, #0x30]\n" - ".inst 0x6e47ec29 // bfmmla v9.4s, v1.8h, v7.8h\n" + ".inst 0x6e40ec2c // bfmmla v12.4s, v1.8h, v0.8h\n" + ".inst 0x6e40ec74 // bfmmla v20.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ecbc // bfmmla v28.4s, v5.8h, v0.8h\n" + "ldr q0, [x11, #0x30]\n" + ".inst 0x6e46ec29 // bfmmla v9.4s, v1.8h, v6.8h\n" "add x11, x11, #0x40\n" - ".inst 0x6e47ec71 // bfmmla v17.4s, v3.8h, v7.8h\n" - ".inst 0x6e47ecb9 // bfmmla v25.4s, v5.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec2d // bfmmla v13.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec75 // bfmmla v21.4s, v3.8h, v6.8h\n" - ".inst 0x6e46ecbd // bfmmla v29.4s, v5.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec2a // bfmmla v10.4s, v1.8h, v7.8h\n" + ".inst 0x6e46ec71 // bfmmla v17.4s, v3.8h, v6.8h\n" + ".inst 0x6e46ecb9 // bfmmla v25.4s, v5.8h, v6.8h\n" + "ldr q6, [x10, #0x20]\n" + ".inst 0x6e40ec2d // bfmmla v13.4s, v1.8h, v0.8h\n" + ".inst 0x6e40ec75 // bfmmla v21.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ecbd // bfmmla v29.4s, v5.8h, v0.8h\n" + "ldr q0, [x10, #0x30]\n" + ".inst 0x6e46ec2a // bfmmla v10.4s, v1.8h, v6.8h\n" "add x10, x10, #0x40\n" - ".inst 0x6e47ec72 // bfmmla v18.4s, v3.8h, v7.8h\n" - ".inst 0x6e47ecba // bfmmla v26.4s, v5.8h, v7.8h\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec76 // bfmmla v22.4s, v3.8h, v6.8h\n" - ".inst 0x6e46ecbe // bfmmla v30.4s, v5.8h, v6.8h\n" - "ldr q6, [x9, #0x30]\n" + ".inst 0x6e46ec72 // bfmmla v18.4s, v3.8h, v6.8h\n" + ".inst 0x6e46ecba // bfmmla v26.4s, v5.8h, v6.8h\n" + "ldr q6, [x9, #0x20]\n" + ".inst 0x6e40ec2e // bfmmla v14.4s, v1.8h, v0.8h\n" + ".inst 0x6e40ec76 // bfmmla v22.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ecbe // bfmmla v30.4s, v5.8h, v0.8h\n" + "ldr q0, [x9, #0x30]\n" "add x9, x9, #0x40\n" - ".inst 0x6e47ec2b // bfmmla v11.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec73 // bfmmla v19.4s, v3.8h, v7.8h\n" - ".inst 0x6e47ecbb // bfmmla v27.4s, v5.8h, v7.8h\n" + ".inst 0x6e46ec2b // bfmmla v11.4s, v1.8h, v6.8h\n" + ".inst 0x6e46ec73 // bfmmla v19.4s, v3.8h, v6.8h\n" + ".inst 0x6e46ecbb // bfmmla v27.4s, v5.8h, v6.8h\n" "ldr q7, [x12, #0x0]\n" - ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" + ".inst 0x6e40ec2f // bfmmla v15.4s, v1.8h, v0.8h\n" "ldr q1, [x26, #0x0]\n" - ".inst 0x6e46ec77 // bfmmla v23.4s, v3.8h, v6.8h\n" + ".inst 0x6e40ec77 // bfmmla v23.4s, v3.8h, v0.8h\n" "ldr q3, [x24, #0x0]\n" - ".inst 0x6e46ecbf // bfmmla v31.4s, v5.8h, v6.8h\n" + ".inst 0x6e40ecbf // bfmmla v31.4s, v5.8h, v0.8h\n" "ldr q5, [x22, #0x0]\n" "bge 172b\n" "173:" // Height 5: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v6.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" + ".inst 0x6e47ecc8 // bfmmla v8.4s, v6.8h, v7.8h\n" "trn1 v2.2d, v3.2d, v4.2d\n" "trn2 v3.2d, v3.2d, v4.2d\n" ".inst 0x6e47ec50 // bfmmla v16.4s, v2.8h, v7.8h\n" "sub x27, x27, #0x8\n" - "trn1 v4.2d, v5.2d, v6.2d\n" - "trn2 v5.2d, v5.2d, v6.2d\n" - "ldr q6, [x12, #0x10]\n" + "trn1 v4.2d, v5.2d, v0.2d\n" + "trn2 v5.2d, v5.2d, v0.2d\n" + "ldr q0, [x12, #0x10]\n" ".inst 0x6e47ec98 // bfmmla v24.4s, v4.8h, v7.8h\n" "ldr q7, [x11, #0x0]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" + ".inst 0x6e40eccc // bfmmla v12.4s, v6.8h, v0.8h\n" + ".inst 0x6e40ec54 // bfmmla v20.4s, v2.8h, v0.8h\n" "add x26, x26, #0x10\n" - ".inst 0x6e46ec9c // bfmmla v28.4s, v4.8h, v6.8h\n" - "ldr q6, [x11, #0x10]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" + ".inst 0x6e40ec9c // bfmmla v28.4s, v4.8h, v0.8h\n" + "ldr q0, [x11, #0x10]\n" + ".inst 0x6e47ecc9 // bfmmla v9.4s, v6.8h, v7.8h\n" "add x25, x25, #0x10\n" ".inst 0x6e47ec51 // bfmmla v17.4s, v2.8h, v7.8h\n" ".inst 0x6e47ec99 // bfmmla v25.4s, v4.8h, v7.8h\n" "ldr q7, [x10, #0x0]\n" "add x24, x24, #0x10\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" + ".inst 0x6e40eccd // bfmmla v13.4s, v6.8h, v0.8h\n" + ".inst 0x6e40ec55 // bfmmla v21.4s, v2.8h, v0.8h\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - ".inst 0x6e46ec9d // bfmmla v29.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x10]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" + ".inst 0x6e40ec9d // bfmmla v29.4s, v4.8h, v0.8h\n" + "ldr q0, [x10, #0x10]\n" + ".inst 0x6e47ecca // bfmmla v10.4s, v6.8h, v7.8h\n" ".inst 0x6e47ec52 // bfmmla v18.4s, v2.8h, v7.8h\n" ".inst 0x6e47ec9a // bfmmla v26.4s, v4.8h, v7.8h\n" "ldr q7, [x9, #0x0]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9e // bfmmla v30.4s, v4.8h, v6.8h\n" - "ldr q6, [x9, #0x10]\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" + ".inst 0x6e40ecce // bfmmla v14.4s, v6.8h, v0.8h\n" + ".inst 0x6e40ec56 // bfmmla v22.4s, v2.8h, v0.8h\n" + ".inst 0x6e40ec9e // bfmmla v30.4s, v4.8h, v0.8h\n" + "ldr q0, [x9, #0x10]\n" + ".inst 0x6e47eccb // bfmmla v11.4s, v6.8h, v7.8h\n" ".inst 0x6e47ec53 // bfmmla v19.4s, v2.8h, v7.8h\n" ".inst 0x6e47ec9b // bfmmla v27.4s, v4.8h, v7.8h\n" "ldr q7, [x12, #0x20]\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9f // bfmmla v31.4s, v4.8h, v6.8h\n" - "ldr q6, [x12, #0x30]\n" + ".inst 0x6e40eccf // bfmmla v15.4s, v6.8h, v0.8h\n" + ".inst 0x6e40ec57 // bfmmla v23.4s, v2.8h, v0.8h\n" + ".inst 0x6e40ec9f // bfmmla v31.4s, v4.8h, v0.8h\n" + "ldr q2, [x12, #0x30]\n" ".inst 0x6e47ec28 // bfmmla v8.4s, v1.8h, v7.8h\n" "add x12, x12, #0x40\n" ".inst 0x6e47ec70 // bfmmla v16.4s, v3.8h, v7.8h\n" ".inst 0x6e47ecb8 // bfmmla v24.4s, v5.8h, v7.8h\n" - "ldr q7, [x11, #0x20]\n" - ".inst 0x6e46ec2c // bfmmla v12.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec74 // bfmmla v20.4s, v3.8h, v6.8h\n" - ".inst 0x6e46ecbc // bfmmla v28.4s, v5.8h, v6.8h\n" - "ldr q6, [x11, #0x30]\n" - ".inst 0x6e47ec29 // bfmmla v9.4s, v1.8h, v7.8h\n" + "ldr q0, [x11, #0x20]\n" + ".inst 0x6e42ec2c // bfmmla v12.4s, v1.8h, v2.8h\n" + ".inst 0x6e42ec74 // bfmmla v20.4s, v3.8h, v2.8h\n" + ".inst 0x6e42ecbc // bfmmla v28.4s, v5.8h, v2.8h\n" + "ldr q2, [x11, #0x30]\n" + ".inst 0x6e40ec29 // bfmmla v9.4s, v1.8h, v0.8h\n" "add x11, x11, #0x40\n" - ".inst 0x6e47ec71 // bfmmla v17.4s, v3.8h, v7.8h\n" - ".inst 0x6e47ecb9 // bfmmla v25.4s, v5.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec2d // bfmmla v13.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec75 // bfmmla v21.4s, v3.8h, v6.8h\n" - ".inst 0x6e46ecbd // bfmmla v29.4s, v5.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec2a // bfmmla v10.4s, v1.8h, v7.8h\n" + ".inst 0x6e40ec71 // bfmmla v17.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ecb9 // bfmmla v25.4s, v5.8h, v0.8h\n" + "ldr q0, [x10, #0x20]\n" + ".inst 0x6e42ec2d // bfmmla v13.4s, v1.8h, v2.8h\n" + ".inst 0x6e42ec75 // bfmmla v21.4s, v3.8h, v2.8h\n" + ".inst 0x6e42ecbd // bfmmla v29.4s, v5.8h, v2.8h\n" + "ldr q2, [x10, #0x30]\n" + ".inst 0x6e40ec2a // bfmmla v10.4s, v1.8h, v0.8h\n" "add x10, x10, #0x40\n" - ".inst 0x6e47ec72 // bfmmla v18.4s, v3.8h, v7.8h\n" - ".inst 0x6e47ecba // bfmmla v26.4s, v5.8h, v7.8h\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec76 // bfmmla v22.4s, v3.8h, v6.8h\n" - ".inst 0x6e46ecbe // bfmmla v30.4s, v5.8h, v6.8h\n" + ".inst 0x6e40ec72 // bfmmla v18.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ecba // bfmmla v26.4s, v5.8h, v0.8h\n" + "ldr q0, [x9, #0x20]\n" + ".inst 0x6e42ec2e // bfmmla v14.4s, v1.8h, v2.8h\n" + ".inst 0x6e42ec76 // bfmmla v22.4s, v3.8h, v2.8h\n" + ".inst 0x6e42ecbe // bfmmla v30.4s, v5.8h, v2.8h\n" "ldr q6, [x9, #0x30]\n" "add x9, x9, #0x40\n" - ".inst 0x6e47ec2b // bfmmla v11.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec73 // bfmmla v19.4s, v3.8h, v7.8h\n" - ".inst 0x6e47ecbb // bfmmla v27.4s, v5.8h, v7.8h\n" + ".inst 0x6e40ec2b // bfmmla v11.4s, v1.8h, v0.8h\n" + ".inst 0x6e40ec73 // bfmmla v19.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ecbb // bfmmla v27.4s, v5.8h, v0.8h\n" ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" ".inst 0x6e46ec77 // bfmmla v23.4s, v3.8h, v6.8h\n" ".inst 0x6e46ecbf // bfmmla v31.4s, v5.8h, v6.8h\n" @@ -2608,51 +2608,51 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "blt 176f\n" "175:" // Height 5: Multiply loop: Odd block loop "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "ldr d3, [x24], #0x8\n" - "ldr d4, [x23], #0x8\n" - "trn1 v2.2d, v3.2d, v4.2d\n" + "ldr d0, [x25], #0x8\n" + "trn1 v4.2d, v1.2d, v0.2d\n" + "ldr d1, [x24], #0x8\n" + "ldr d0, [x23], #0x8\n" + "trn1 v3.2d, v1.2d, v0.2d\n" "sub x27, x27, #0x4\n" - "ldr d5, [x22], #0x8\n" - "ldr q6, [x12, #0x0]\n" - "trn1 v4.2d, v5.2d, v7.2d\n" - ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - "ldr q7, [x12, #0x10]\n" - ".inst 0x6e46ec50 // bfmmla v16.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec98 // bfmmla v24.4s, v4.8h, v6.8h\n" - "ldr q6, [x11, #0x0]\n" - ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec54 // bfmmla v20.4s, v2.8h, v7.8h\n" + "ldr d0, [x22], #0x8\n" + "ldr q1, [x12, #0x0]\n" + "trn1 v2.2d, v0.2d, v2.2d\n" + ".inst 0x6e41ec88 // bfmmla v8.4s, v4.8h, v1.8h\n" + "ldr q0, [x12, #0x10]\n" + ".inst 0x6e41ec70 // bfmmla v16.4s, v3.8h, v1.8h\n" + ".inst 0x6e41ec58 // bfmmla v24.4s, v2.8h, v1.8h\n" + "ldr q1, [x11, #0x0]\n" + ".inst 0x6e40ec8c // bfmmla v12.4s, v4.8h, v0.8h\n" + ".inst 0x6e40ec74 // bfmmla v20.4s, v3.8h, v0.8h\n" "cmp x27, #0x4\n" "add x12, x12, #0x20\n" - ".inst 0x6e47ec9c // bfmmla v28.4s, v4.8h, v7.8h\n" - "ldr q7, [x11, #0x10]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" + ".inst 0x6e40ec5c // bfmmla v28.4s, v2.8h, v0.8h\n" + "ldr q0, [x11, #0x10]\n" + ".inst 0x6e41ec89 // bfmmla v9.4s, v4.8h, v1.8h\n" "add x11, x11, #0x20\n" - ".inst 0x6e46ec51 // bfmmla v17.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec99 // bfmmla v25.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x0]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec55 // bfmmla v21.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9d // bfmmla v29.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" + ".inst 0x6e41ec71 // bfmmla v17.4s, v3.8h, v1.8h\n" + ".inst 0x6e41ec59 // bfmmla v25.4s, v2.8h, v1.8h\n" + "ldr q1, [x10, #0x0]\n" + ".inst 0x6e40ec8d // bfmmla v13.4s, v4.8h, v0.8h\n" + ".inst 0x6e40ec75 // bfmmla v21.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec5d // bfmmla v29.4s, v2.8h, v0.8h\n" + "ldr q0, [x10, #0x10]\n" + ".inst 0x6e41ec8a // bfmmla v10.4s, v4.8h, v1.8h\n" "add x10, x10, #0x20\n" - ".inst 0x6e46ec52 // bfmmla v18.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9a // bfmmla v26.4s, v4.8h, v6.8h\n" + ".inst 0x6e41ec72 // bfmmla v18.4s, v3.8h, v1.8h\n" + ".inst 0x6e41ec5a // bfmmla v26.4s, v2.8h, v1.8h\n" "ldr q6, [x9, #0x0]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec56 // bfmmla v22.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9e // bfmmla v30.4s, v4.8h, v7.8h\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" + ".inst 0x6e40ec8e // bfmmla v14.4s, v4.8h, v0.8h\n" + ".inst 0x6e40ec76 // bfmmla v22.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec5e // bfmmla v30.4s, v2.8h, v0.8h\n" + "ldr q0, [x9, #0x10]\n" + ".inst 0x6e46ec8b // bfmmla v11.4s, v4.8h, v6.8h\n" "add x9, x9, #0x20\n" - ".inst 0x6e46ec53 // bfmmla v19.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9b // bfmmla v27.4s, v4.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9f // bfmmla v31.4s, v4.8h, v7.8h\n" + ".inst 0x6e46ec73 // bfmmla v19.4s, v3.8h, v6.8h\n" + ".inst 0x6e46ec5b // bfmmla v27.4s, v2.8h, v6.8h\n" + ".inst 0x6e40ec8f // bfmmla v15.4s, v4.8h, v0.8h\n" + ".inst 0x6e40ec77 // bfmmla v23.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec5f // bfmmla v31.4s, v2.8h, v0.8h\n" "bge 175b\n" "176:" // Height 5: Multiply loop: Skip odd blocks "cbz x27, 179f\n" @@ -2676,45 +2676,45 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "ldr h4, [x23, #0x0]\n" "ldr h5, [x22, #0x0]\n" "178:" // Height 5: Multiply loop: Ragged operand read: Done - "ldr q7, [x12, #0x0]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - "trn1 v4.2d, v5.2d, v6.2d\n" - "ldr q6, [x12, #0x10]\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec50 // bfmmla v16.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec98 // bfmmla v24.4s, v4.8h, v7.8h\n" - "ldr q7, [x11, #0x0]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" + "ldr q6, [x12, #0x0]\n" + "trn1 v7.2d, v1.2d, v2.2d\n" + "trn1 v3.2d, v3.2d, v4.2d\n" + "trn1 v2.2d, v5.2d, v0.2d\n" + "ldr q1, [x12, #0x10]\n" + ".inst 0x6e46ece8 // bfmmla v8.4s, v7.8h, v6.8h\n" + ".inst 0x6e46ec70 // bfmmla v16.4s, v3.8h, v6.8h\n" + ".inst 0x6e46ec58 // bfmmla v24.4s, v2.8h, v6.8h\n" + "ldr q0, [x11, #0x0]\n" + ".inst 0x6e41ecec // bfmmla v12.4s, v7.8h, v1.8h\n" "add x12, x12, #0x20\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9c // bfmmla v28.4s, v4.8h, v6.8h\n" - "ldr q6, [x11, #0x10]\n" + ".inst 0x6e41ec74 // bfmmla v20.4s, v3.8h, v1.8h\n" + ".inst 0x6e41ec5c // bfmmla v28.4s, v2.8h, v1.8h\n" + "ldr q1, [x11, #0x10]\n" "add x11, x11, #0x20\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec51 // bfmmla v17.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec99 // bfmmla v25.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x0]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9d // bfmmla v29.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x10]\n" + ".inst 0x6e40ece9 // bfmmla v9.4s, v7.8h, v0.8h\n" + ".inst 0x6e40ec71 // bfmmla v17.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec59 // bfmmla v25.4s, v2.8h, v0.8h\n" + "ldr q0, [x10, #0x0]\n" + ".inst 0x6e41eced // bfmmla v13.4s, v7.8h, v1.8h\n" + ".inst 0x6e41ec75 // bfmmla v21.4s, v3.8h, v1.8h\n" + ".inst 0x6e41ec5d // bfmmla v29.4s, v2.8h, v1.8h\n" + "ldr q1, [x10, #0x10]\n" "add x10, x10, #0x20\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec52 // bfmmla v18.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9a // bfmmla v26.4s, v4.8h, v7.8h\n" - "ldr q7, [x9, #0x0]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9e // bfmmla v30.4s, v4.8h, v6.8h\n" + ".inst 0x6e40ecea // bfmmla v10.4s, v7.8h, v0.8h\n" + ".inst 0x6e40ec72 // bfmmla v18.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec5a // bfmmla v26.4s, v2.8h, v0.8h\n" + "ldr q0, [x9, #0x0]\n" + ".inst 0x6e41ecee // bfmmla v14.4s, v7.8h, v1.8h\n" + ".inst 0x6e41ec76 // bfmmla v22.4s, v3.8h, v1.8h\n" + ".inst 0x6e41ec5e // bfmmla v30.4s, v2.8h, v1.8h\n" "ldr q6, [x9, #0x10]\n" "add x9, x9, #0x20\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec53 // bfmmla v19.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9b // bfmmla v27.4s, v4.8h, v7.8h\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9f // bfmmla v31.4s, v4.8h, v6.8h\n" + ".inst 0x6e40eceb // bfmmla v11.4s, v7.8h, v0.8h\n" + ".inst 0x6e40ec73 // bfmmla v19.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec5b // bfmmla v27.4s, v2.8h, v0.8h\n" + ".inst 0x6e46ecef // bfmmla v15.4s, v7.8h, v6.8h\n" + ".inst 0x6e46ec77 // bfmmla v23.4s, v3.8h, v6.8h\n" + ".inst 0x6e46ec5f // bfmmla v31.4s, v2.8h, v6.8h\n" "179:" // Height 5: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -3213,16 +3213,16 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "207:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 208f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 209f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -3234,11 +3234,11 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "b 209f\n" "208:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" - "add x22, x23, x20, LSL #1\n" - "add x21, x22, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" + "add x22, x23, x21, LSL #1\n" + "add x21, x22, x21, LSL #1\n" "209:" // Height 6: input setup done "cmp x27, #0x8\n" "blt 212f\n" @@ -3299,45 +3299,45 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" "ldr q2, [x25, #0x0]\n" ".inst 0x6e46ec9f // bfmmla v31.4s, v4.8h, v6.8h\n" - "ldr q6, [x12, #0x30]\n" + "ldr q0, [x12, #0x30]\n" "ldr q4, [x23, #0x0]\n" ".inst 0x6e47ec28 // bfmmla v8.4s, v1.8h, v7.8h\n" ".inst 0x6e47ec70 // bfmmla v16.4s, v3.8h, v7.8h\n" ".inst 0x6e47ecb8 // bfmmla v24.4s, v5.8h, v7.8h\n" - "ldr q7, [x11, #0x20]\n" + "ldr q6, [x11, #0x20]\n" "add x12, x12, #0x40\n" - ".inst 0x6e46ec2c // bfmmla v12.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec74 // bfmmla v20.4s, v3.8h, v6.8h\n" - ".inst 0x6e46ecbc // bfmmla v28.4s, v5.8h, v6.8h\n" - "ldr q6, [x11, #0x30]\n" - ".inst 0x6e47ec29 // bfmmla v9.4s, v1.8h, v7.8h\n" + ".inst 0x6e40ec2c // bfmmla v12.4s, v1.8h, v0.8h\n" + ".inst 0x6e40ec74 // bfmmla v20.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ecbc // bfmmla v28.4s, v5.8h, v0.8h\n" + "ldr q0, [x11, #0x30]\n" + ".inst 0x6e46ec29 // bfmmla v9.4s, v1.8h, v6.8h\n" "add x11, x11, #0x40\n" - ".inst 0x6e47ec71 // bfmmla v17.4s, v3.8h, v7.8h\n" - ".inst 0x6e47ecb9 // bfmmla v25.4s, v5.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec2d // bfmmla v13.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec75 // bfmmla v21.4s, v3.8h, v6.8h\n" - ".inst 0x6e46ecbd // bfmmla v29.4s, v5.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec2a // bfmmla v10.4s, v1.8h, v7.8h\n" + ".inst 0x6e46ec71 // bfmmla v17.4s, v3.8h, v6.8h\n" + ".inst 0x6e46ecb9 // bfmmla v25.4s, v5.8h, v6.8h\n" + "ldr q6, [x10, #0x20]\n" + ".inst 0x6e40ec2d // bfmmla v13.4s, v1.8h, v0.8h\n" + ".inst 0x6e40ec75 // bfmmla v21.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ecbd // bfmmla v29.4s, v5.8h, v0.8h\n" + "ldr q0, [x10, #0x30]\n" + ".inst 0x6e46ec2a // bfmmla v10.4s, v1.8h, v6.8h\n" "add x10, x10, #0x40\n" - ".inst 0x6e47ec72 // bfmmla v18.4s, v3.8h, v7.8h\n" - ".inst 0x6e47ecba // bfmmla v26.4s, v5.8h, v7.8h\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec76 // bfmmla v22.4s, v3.8h, v6.8h\n" - ".inst 0x6e46ecbe // bfmmla v30.4s, v5.8h, v6.8h\n" - "ldr q6, [x9, #0x30]\n" + ".inst 0x6e46ec72 // bfmmla v18.4s, v3.8h, v6.8h\n" + ".inst 0x6e46ecba // bfmmla v26.4s, v5.8h, v6.8h\n" + "ldr q6, [x9, #0x20]\n" + ".inst 0x6e40ec2e // bfmmla v14.4s, v1.8h, v0.8h\n" + ".inst 0x6e40ec76 // bfmmla v22.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ecbe // bfmmla v30.4s, v5.8h, v0.8h\n" + "ldr q0, [x9, #0x30]\n" "add x9, x9, #0x40\n" - ".inst 0x6e47ec2b // bfmmla v11.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec73 // bfmmla v19.4s, v3.8h, v7.8h\n" - ".inst 0x6e47ecbb // bfmmla v27.4s, v5.8h, v7.8h\n" + ".inst 0x6e46ec2b // bfmmla v11.4s, v1.8h, v6.8h\n" + ".inst 0x6e46ec73 // bfmmla v19.4s, v3.8h, v6.8h\n" + ".inst 0x6e46ecbb // bfmmla v27.4s, v5.8h, v6.8h\n" "ldr q7, [x12, #0x0]\n" - ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" + ".inst 0x6e40ec2f // bfmmla v15.4s, v1.8h, v0.8h\n" "ldr q1, [x26, #0x0]\n" - ".inst 0x6e46ec77 // bfmmla v23.4s, v3.8h, v6.8h\n" + ".inst 0x6e40ec77 // bfmmla v23.4s, v3.8h, v0.8h\n" "ldr q3, [x24, #0x0]\n" - ".inst 0x6e46ecbf // bfmmla v31.4s, v5.8h, v6.8h\n" + ".inst 0x6e40ecbf // bfmmla v31.4s, v5.8h, v0.8h\n" "ldr q5, [x22, #0x0]\n" "ldr q6, [x21, #0x0]\n" "bge 210b\n" @@ -3387,38 +3387,38 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" ".inst 0x6e46ec9f // bfmmla v31.4s, v4.8h, v6.8h\n" - "ldr q6, [x12, #0x30]\n" + "ldr q2, [x12, #0x30]\n" ".inst 0x6e47ec28 // bfmmla v8.4s, v1.8h, v7.8h\n" "add x12, x12, #0x40\n" ".inst 0x6e47ec70 // bfmmla v16.4s, v3.8h, v7.8h\n" ".inst 0x6e47ecb8 // bfmmla v24.4s, v5.8h, v7.8h\n" - "ldr q7, [x11, #0x20]\n" - ".inst 0x6e46ec2c // bfmmla v12.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec74 // bfmmla v20.4s, v3.8h, v6.8h\n" - ".inst 0x6e46ecbc // bfmmla v28.4s, v5.8h, v6.8h\n" - "ldr q6, [x11, #0x30]\n" - ".inst 0x6e47ec29 // bfmmla v9.4s, v1.8h, v7.8h\n" + "ldr q0, [x11, #0x20]\n" + ".inst 0x6e42ec2c // bfmmla v12.4s, v1.8h, v2.8h\n" + ".inst 0x6e42ec74 // bfmmla v20.4s, v3.8h, v2.8h\n" + ".inst 0x6e42ecbc // bfmmla v28.4s, v5.8h, v2.8h\n" + "ldr q2, [x11, #0x30]\n" + ".inst 0x6e40ec29 // bfmmla v9.4s, v1.8h, v0.8h\n" "add x11, x11, #0x40\n" - ".inst 0x6e47ec71 // bfmmla v17.4s, v3.8h, v7.8h\n" - ".inst 0x6e47ecb9 // bfmmla v25.4s, v5.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec2d // bfmmla v13.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec75 // bfmmla v21.4s, v3.8h, v6.8h\n" - ".inst 0x6e46ecbd // bfmmla v29.4s, v5.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec2a // bfmmla v10.4s, v1.8h, v7.8h\n" + ".inst 0x6e40ec71 // bfmmla v17.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ecb9 // bfmmla v25.4s, v5.8h, v0.8h\n" + "ldr q0, [x10, #0x20]\n" + ".inst 0x6e42ec2d // bfmmla v13.4s, v1.8h, v2.8h\n" + ".inst 0x6e42ec75 // bfmmla v21.4s, v3.8h, v2.8h\n" + ".inst 0x6e42ecbd // bfmmla v29.4s, v5.8h, v2.8h\n" + "ldr q2, [x10, #0x30]\n" + ".inst 0x6e40ec2a // bfmmla v10.4s, v1.8h, v0.8h\n" "add x10, x10, #0x40\n" - ".inst 0x6e47ec72 // bfmmla v18.4s, v3.8h, v7.8h\n" - ".inst 0x6e47ecba // bfmmla v26.4s, v5.8h, v7.8h\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec76 // bfmmla v22.4s, v3.8h, v6.8h\n" - ".inst 0x6e46ecbe // bfmmla v30.4s, v5.8h, v6.8h\n" + ".inst 0x6e40ec72 // bfmmla v18.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ecba // bfmmla v26.4s, v5.8h, v0.8h\n" + "ldr q0, [x9, #0x20]\n" + ".inst 0x6e42ec2e // bfmmla v14.4s, v1.8h, v2.8h\n" + ".inst 0x6e42ec76 // bfmmla v22.4s, v3.8h, v2.8h\n" + ".inst 0x6e42ecbe // bfmmla v30.4s, v5.8h, v2.8h\n" "ldr q6, [x9, #0x30]\n" "add x9, x9, #0x40\n" - ".inst 0x6e47ec2b // bfmmla v11.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec73 // bfmmla v19.4s, v3.8h, v7.8h\n" - ".inst 0x6e47ecbb // bfmmla v27.4s, v5.8h, v7.8h\n" + ".inst 0x6e40ec2b // bfmmla v11.4s, v1.8h, v0.8h\n" + ".inst 0x6e40ec73 // bfmmla v19.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ecbb // bfmmla v27.4s, v5.8h, v0.8h\n" ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" ".inst 0x6e46ec77 // bfmmla v23.4s, v3.8h, v6.8h\n" ".inst 0x6e46ecbf // bfmmla v31.4s, v5.8h, v6.8h\n" @@ -3428,52 +3428,52 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "blt 214f\n" "213:" // Height 6: Multiply loop: Odd block loop "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" + "ldr d0, [x25], #0x8\n" + "trn1 v4.2d, v1.2d, v0.2d\n" "sub x27, x27, #0x4\n" - "ldr d3, [x24], #0x8\n" - "ldr d4, [x23], #0x8\n" - "trn1 v2.2d, v3.2d, v4.2d\n" + "ldr d1, [x24], #0x8\n" + "ldr d0, [x23], #0x8\n" + "trn1 v3.2d, v1.2d, v0.2d\n" "cmp x27, #0x4\n" - "ldr d5, [x22], #0x8\n" - "ldr d7, [x21], #0x8\n" - "trn1 v4.2d, v5.2d, v7.2d\n" - "ldr q6, [x12, #0x0]\n" - "ldr q7, [x12, #0x10]\n" - ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec50 // bfmmla v16.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec98 // bfmmla v24.4s, v4.8h, v6.8h\n" - "ldr q6, [x11, #0x0]\n" - ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" + "ldr d1, [x22], #0x8\n" + "ldr d0, [x21], #0x8\n" + "trn1 v2.2d, v1.2d, v0.2d\n" + "ldr q1, [x12, #0x0]\n" + "ldr q0, [x12, #0x10]\n" + ".inst 0x6e41ec88 // bfmmla v8.4s, v4.8h, v1.8h\n" + ".inst 0x6e41ec70 // bfmmla v16.4s, v3.8h, v1.8h\n" + ".inst 0x6e41ec58 // bfmmla v24.4s, v2.8h, v1.8h\n" + "ldr q1, [x11, #0x0]\n" + ".inst 0x6e40ec8c // bfmmla v12.4s, v4.8h, v0.8h\n" "add x12, x12, #0x20\n" - ".inst 0x6e47ec54 // bfmmla v20.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9c // bfmmla v28.4s, v4.8h, v7.8h\n" - "ldr q7, [x11, #0x10]\n" + ".inst 0x6e40ec74 // bfmmla v20.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec5c // bfmmla v28.4s, v2.8h, v0.8h\n" + "ldr q0, [x11, #0x10]\n" "add x11, x11, #0x20\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec51 // bfmmla v17.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec99 // bfmmla v25.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x0]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec55 // bfmmla v21.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9d // bfmmla v29.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x10]\n" + ".inst 0x6e41ec89 // bfmmla v9.4s, v4.8h, v1.8h\n" + ".inst 0x6e41ec71 // bfmmla v17.4s, v3.8h, v1.8h\n" + ".inst 0x6e41ec59 // bfmmla v25.4s, v2.8h, v1.8h\n" + "ldr q1, [x10, #0x0]\n" + ".inst 0x6e40ec8d // bfmmla v13.4s, v4.8h, v0.8h\n" + ".inst 0x6e40ec75 // bfmmla v21.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec5d // bfmmla v29.4s, v2.8h, v0.8h\n" + "ldr q0, [x10, #0x10]\n" "add x10, x10, #0x20\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec52 // bfmmla v18.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9a // bfmmla v26.4s, v4.8h, v6.8h\n" + ".inst 0x6e41ec8a // bfmmla v10.4s, v4.8h, v1.8h\n" + ".inst 0x6e41ec72 // bfmmla v18.4s, v3.8h, v1.8h\n" + ".inst 0x6e41ec5a // bfmmla v26.4s, v2.8h, v1.8h\n" "ldr q6, [x9, #0x0]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec56 // bfmmla v22.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9e // bfmmla v30.4s, v4.8h, v7.8h\n" - "ldr q7, [x9, #0x10]\n" + ".inst 0x6e40ec8e // bfmmla v14.4s, v4.8h, v0.8h\n" + ".inst 0x6e40ec76 // bfmmla v22.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec5e // bfmmla v30.4s, v2.8h, v0.8h\n" + "ldr q0, [x9, #0x10]\n" "add x9, x9, #0x20\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec53 // bfmmla v19.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9b // bfmmla v27.4s, v4.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9f // bfmmla v31.4s, v4.8h, v7.8h\n" + ".inst 0x6e46ec8b // bfmmla v11.4s, v4.8h, v6.8h\n" + ".inst 0x6e46ec73 // bfmmla v19.4s, v3.8h, v6.8h\n" + ".inst 0x6e46ec5b // bfmmla v27.4s, v2.8h, v6.8h\n" + ".inst 0x6e40ec8f // bfmmla v15.4s, v4.8h, v0.8h\n" + ".inst 0x6e40ec77 // bfmmla v23.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec5f // bfmmla v31.4s, v2.8h, v0.8h\n" "bge 213b\n" "214:" // Height 6: Multiply loop: Skip odd blocks "cbz x27, 217f\n" @@ -3500,45 +3500,45 @@ void a64_ffhybrid_bf16fp32_mmla_6x16 ( "ldr h5, [x22, #0x0]\n" "ldr h6, [x21, #0x0]\n" "216:" // Height 6: Multiply loop: Ragged operand read: Done - "ldr q7, [x12, #0x0]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - "trn1 v4.2d, v5.2d, v6.2d\n" - "ldr q6, [x12, #0x10]\n" - ".inst 0x6e47ec50 // bfmmla v16.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec98 // bfmmla v24.4s, v4.8h, v7.8h\n" - "ldr q7, [x11, #0x0]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" + "ldr q0, [x12, #0x0]\n" + "trn1 v7.2d, v1.2d, v2.2d\n" + "trn1 v3.2d, v3.2d, v4.2d\n" + ".inst 0x6e40ece8 // bfmmla v8.4s, v7.8h, v0.8h\n" + "trn1 v2.2d, v5.2d, v6.2d\n" + "ldr q1, [x12, #0x10]\n" + ".inst 0x6e40ec70 // bfmmla v16.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec58 // bfmmla v24.4s, v2.8h, v0.8h\n" + "ldr q0, [x11, #0x0]\n" + ".inst 0x6e41ecec // bfmmla v12.4s, v7.8h, v1.8h\n" + ".inst 0x6e41ec74 // bfmmla v20.4s, v3.8h, v1.8h\n" "add x12, x12, #0x20\n" - ".inst 0x6e46ec9c // bfmmla v28.4s, v4.8h, v6.8h\n" - "ldr q6, [x11, #0x10]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" + ".inst 0x6e41ec5c // bfmmla v28.4s, v2.8h, v1.8h\n" + "ldr q1, [x11, #0x10]\n" + ".inst 0x6e40ece9 // bfmmla v9.4s, v7.8h, v0.8h\n" "add x11, x11, #0x20\n" - ".inst 0x6e47ec51 // bfmmla v17.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec99 // bfmmla v25.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x0]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9d // bfmmla v29.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x10]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" + ".inst 0x6e40ec71 // bfmmla v17.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec59 // bfmmla v25.4s, v2.8h, v0.8h\n" + "ldr q0, [x10, #0x0]\n" + ".inst 0x6e41eced // bfmmla v13.4s, v7.8h, v1.8h\n" + ".inst 0x6e41ec75 // bfmmla v21.4s, v3.8h, v1.8h\n" + ".inst 0x6e41ec5d // bfmmla v29.4s, v2.8h, v1.8h\n" + "ldr q1, [x10, #0x10]\n" + ".inst 0x6e40ecea // bfmmla v10.4s, v7.8h, v0.8h\n" "add x10, x10, #0x20\n" - ".inst 0x6e47ec52 // bfmmla v18.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9a // bfmmla v26.4s, v4.8h, v7.8h\n" - "ldr q7, [x9, #0x0]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9e // bfmmla v30.4s, v4.8h, v6.8h\n" + ".inst 0x6e40ec72 // bfmmla v18.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec5a // bfmmla v26.4s, v2.8h, v0.8h\n" + "ldr q0, [x9, #0x0]\n" + ".inst 0x6e41ecee // bfmmla v14.4s, v7.8h, v1.8h\n" + ".inst 0x6e41ec76 // bfmmla v22.4s, v3.8h, v1.8h\n" + ".inst 0x6e41ec5e // bfmmla v30.4s, v2.8h, v1.8h\n" "ldr q6, [x9, #0x10]\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" + ".inst 0x6e40eceb // bfmmla v11.4s, v7.8h, v0.8h\n" "add x9, x9, #0x20\n" - ".inst 0x6e47ec53 // bfmmla v19.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9b // bfmmla v27.4s, v4.8h, v7.8h\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9f // bfmmla v31.4s, v4.8h, v6.8h\n" + ".inst 0x6e40ec73 // bfmmla v19.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec5b // bfmmla v27.4s, v2.8h, v0.8h\n" + ".inst 0x6e46ecef // bfmmla v15.4s, v7.8h, v6.8h\n" + ".inst 0x6e46ec77 // bfmmla v23.4s, v3.8h, v6.8h\n" + ".inst 0x6e46ec5f // bfmmla v31.4s, v2.8h, v6.8h\n" "217:" // Height 6: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp16_mla_6x32.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp16_mla_6x32.hpp index f7506e5123b2d0ae2e4298908290f69f5b0250bf..4924b3a54907dcca74ede2ed705a4192204871e9 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp16_mla_6x32.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp16_mla_6x32.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,16 +10,16 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once #ifdef __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp16_mla_6x32/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp16_mla_6x32/generic.cpp index 18a2db506915f8d8b1271d49b9ee72b9bcb24965..8038612200110b8523a047dae22b1b23210d86a6 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp16_mla_6x32/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp16_mla_6x32/generic.cpp @@ -265,11 +265,11 @@ void a64_ffhybrid_fp16_mla_6x32 ( "24:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 25f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 26f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -286,69 +286,69 @@ void a64_ffhybrid_fp16_mla_6x32 ( "blt 28f\n" "27:" // Height 1: Multiply loop: Main loop head "fmla v8.8h, v6.8h, v0.h[0]\n" - "ldr q6, [x10, #0x0]\n" + "ldr q17, [x10, #0x0]\n" "fmla v9.8h, v7.8h, v0.h[0]\n" - "ldr q7, [x9, #0x0]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "ldr q6, [x12, #0x10]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "ldr q7, [x11, #0x10]\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "ldr q6, [x10, #0x10]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "ldr q7, [x9, #0x10]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "ldr q6, [x12, #0x20]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "ldr q7, [x11, #0x20]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "ldr q7, [x9, #0x20]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "ldr q6, [x12, #0x30]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "ldr q7, [x11, #0x30]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "ldr q6, [x10, #0x30]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "ldr q7, [x9, #0x30]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "ldr q6, [x12, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "ldr q7, [x11, #0x40]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "ldr q6, [x10, #0x40]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "ldr q7, [x9, #0x40]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "ldr q6, [x12, #0x50]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "ldr q7, [x11, #0x50]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "ldr q6, [x10, #0x50]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "ldr q7, [x9, #0x50]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "ldr q6, [x12, #0x60]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "ldr q7, [x11, #0x60]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "ldr q7, [x9, #0x60]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "ldr q6, [x12, #0x70]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "ldr q7, [x11, #0x70]\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "ldr q6, [x10, #0x70]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "ldr q7, [x9, #0x70]\n" + "ldr q16, [x9, #0x0]\n" + "fmla v10.8h, v17.8h, v0.h[0]\n" + "ldr q17, [x12, #0x10]\n" + "fmla v11.8h, v16.8h, v0.h[0]\n" + "ldr q16, [x11, #0x10]\n" + "fmla v8.8h, v17.8h, v0.h[1]\n" + "ldr q17, [x10, #0x10]\n" + "fmla v9.8h, v16.8h, v0.h[1]\n" + "ldr q16, [x9, #0x10]\n" + "fmla v10.8h, v17.8h, v0.h[1]\n" + "ldr q17, [x12, #0x20]\n" + "fmla v11.8h, v16.8h, v0.h[1]\n" + "ldr q16, [x11, #0x20]\n" + "fmla v8.8h, v17.8h, v0.h[2]\n" + "ldr q17, [x10, #0x20]\n" + "fmla v9.8h, v16.8h, v0.h[2]\n" + "ldr q16, [x9, #0x20]\n" + "fmla v10.8h, v17.8h, v0.h[2]\n" + "ldr q17, [x12, #0x30]\n" + "fmla v11.8h, v16.8h, v0.h[2]\n" + "ldr q16, [x11, #0x30]\n" + "fmla v8.8h, v17.8h, v0.h[3]\n" + "ldr q17, [x10, #0x30]\n" + "fmla v9.8h, v16.8h, v0.h[3]\n" + "ldr q16, [x9, #0x30]\n" + "fmla v10.8h, v17.8h, v0.h[3]\n" + "ldr q17, [x12, #0x40]\n" + "fmla v11.8h, v16.8h, v0.h[3]\n" + "ldr q16, [x11, #0x40]\n" + "fmla v8.8h, v17.8h, v0.h[4]\n" + "ldr q17, [x10, #0x40]\n" + "fmla v9.8h, v16.8h, v0.h[4]\n" + "ldr q16, [x9, #0x40]\n" + "fmla v10.8h, v17.8h, v0.h[4]\n" + "ldr q17, [x12, #0x50]\n" + "fmla v11.8h, v16.8h, v0.h[4]\n" + "ldr q16, [x11, #0x50]\n" + "fmla v8.8h, v17.8h, v0.h[5]\n" + "ldr q17, [x10, #0x50]\n" + "fmla v9.8h, v16.8h, v0.h[5]\n" + "ldr q16, [x9, #0x50]\n" + "fmla v10.8h, v17.8h, v0.h[5]\n" + "ldr q17, [x12, #0x60]\n" + "fmla v11.8h, v16.8h, v0.h[5]\n" + "ldr q16, [x11, #0x60]\n" + "fmla v8.8h, v17.8h, v0.h[6]\n" + "ldr q17, [x10, #0x60]\n" + "fmla v9.8h, v16.8h, v0.h[6]\n" + "ldr q16, [x9, #0x60]\n" + "fmla v10.8h, v17.8h, v0.h[6]\n" + "ldr q17, [x12, #0x70]\n" + "fmla v11.8h, v16.8h, v0.h[6]\n" + "ldr q16, [x11, #0x70]\n" + "fmla v8.8h, v17.8h, v0.h[7]\n" + "ldr q17, [x10, #0x70]\n" + "fmla v9.8h, v16.8h, v0.h[7]\n" + "ldr q16, [x9, #0x70]\n" "sub x27, x27, #0x8\n" "cmp x27, #0x10\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" + "fmla v10.8h, v17.8h, v0.h[7]\n" + "fmla v11.8h, v16.8h, v0.h[7]\n" "add x26, x26, #0x10\n" "ldr q0, [x26, #0x0]\n" "add x12, x12, #0x80\n" @@ -360,68 +360,68 @@ void a64_ffhybrid_fp16_mla_6x32 ( "bge 27b\n" "28:" // Height 1: Multiply loop: Single iteration only "fmla v8.8h, v6.8h, v0.h[0]\n" - "ldr q6, [x10, #0x0]\n" + "ldr q17, [x10, #0x0]\n" "fmla v9.8h, v7.8h, v0.h[0]\n" - "ldr q7, [x9, #0x0]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "ldr q6, [x12, #0x10]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "ldr q7, [x11, #0x10]\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "ldr q6, [x10, #0x10]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "ldr q7, [x9, #0x10]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "ldr q6, [x12, #0x20]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "ldr q7, [x11, #0x20]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "ldr q7, [x9, #0x20]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "ldr q6, [x12, #0x30]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "ldr q7, [x11, #0x30]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "ldr q6, [x10, #0x30]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "ldr q7, [x9, #0x30]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "ldr q6, [x12, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "ldr q7, [x11, #0x40]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "ldr q6, [x10, #0x40]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "ldr q7, [x9, #0x40]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "ldr q6, [x12, #0x50]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "ldr q7, [x11, #0x50]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "ldr q6, [x10, #0x50]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "ldr q7, [x9, #0x50]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "ldr q6, [x12, #0x60]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "ldr q7, [x11, #0x60]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "ldr q7, [x9, #0x60]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "ldr q6, [x12, #0x70]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "ldr q7, [x11, #0x70]\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "ldr q6, [x10, #0x70]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "ldr q7, [x9, #0x70]\n" + "ldr q16, [x9, #0x0]\n" + "fmla v10.8h, v17.8h, v0.h[0]\n" + "ldr q17, [x12, #0x10]\n" + "fmla v11.8h, v16.8h, v0.h[0]\n" + "ldr q16, [x11, #0x10]\n" + "fmla v8.8h, v17.8h, v0.h[1]\n" + "ldr q17, [x10, #0x10]\n" + "fmla v9.8h, v16.8h, v0.h[1]\n" + "ldr q16, [x9, #0x10]\n" + "fmla v10.8h, v17.8h, v0.h[1]\n" + "ldr q17, [x12, #0x20]\n" + "fmla v11.8h, v16.8h, v0.h[1]\n" + "ldr q16, [x11, #0x20]\n" + "fmla v8.8h, v17.8h, v0.h[2]\n" + "ldr q17, [x10, #0x20]\n" + "fmla v9.8h, v16.8h, v0.h[2]\n" + "ldr q16, [x9, #0x20]\n" + "fmla v10.8h, v17.8h, v0.h[2]\n" + "ldr q17, [x12, #0x30]\n" + "fmla v11.8h, v16.8h, v0.h[2]\n" + "ldr q16, [x11, #0x30]\n" + "fmla v8.8h, v17.8h, v0.h[3]\n" + "ldr q17, [x10, #0x30]\n" + "fmla v9.8h, v16.8h, v0.h[3]\n" + "ldr q16, [x9, #0x30]\n" + "fmla v10.8h, v17.8h, v0.h[3]\n" + "ldr q17, [x12, #0x40]\n" + "fmla v11.8h, v16.8h, v0.h[3]\n" + "ldr q16, [x11, #0x40]\n" + "fmla v8.8h, v17.8h, v0.h[4]\n" + "ldr q17, [x10, #0x40]\n" + "fmla v9.8h, v16.8h, v0.h[4]\n" + "ldr q16, [x9, #0x40]\n" + "fmla v10.8h, v17.8h, v0.h[4]\n" + "ldr q17, [x12, #0x50]\n" + "fmla v11.8h, v16.8h, v0.h[4]\n" + "ldr q16, [x11, #0x50]\n" + "fmla v8.8h, v17.8h, v0.h[5]\n" + "ldr q17, [x10, #0x50]\n" + "fmla v9.8h, v16.8h, v0.h[5]\n" + "ldr q16, [x9, #0x50]\n" + "fmla v10.8h, v17.8h, v0.h[5]\n" + "ldr q17, [x12, #0x60]\n" + "fmla v11.8h, v16.8h, v0.h[5]\n" + "ldr q16, [x11, #0x60]\n" + "fmla v8.8h, v17.8h, v0.h[6]\n" + "ldr q17, [x10, #0x60]\n" + "fmla v9.8h, v16.8h, v0.h[6]\n" + "ldr q16, [x9, #0x60]\n" + "fmla v10.8h, v17.8h, v0.h[6]\n" + "ldr q17, [x12, #0x70]\n" + "fmla v11.8h, v16.8h, v0.h[6]\n" + "ldr q16, [x11, #0x70]\n" + "fmla v8.8h, v17.8h, v0.h[7]\n" + "ldr q17, [x10, #0x70]\n" + "fmla v9.8h, v16.8h, v0.h[7]\n" + "ldr q16, [x9, #0x70]\n" "sub x27, x27, #0x8\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" + "fmla v10.8h, v17.8h, v0.h[7]\n" + "fmla v11.8h, v16.8h, v0.h[7]\n" "add x26, x26, #0x10\n" "add x12, x12, #0x80\n" "add x11, x11, #0x80\n" @@ -431,15 +431,15 @@ void a64_ffhybrid_fp16_mla_6x32 ( "cbz x27, 31f\n" "30:" // Height 1: Multiply loop: Odd block loop "ldr h0, [x26], #0x2\n" - "ldr q6, [x12, #0x0]\n" - "fmla v8.8h, v6.8h, v0.h[0]\n" + "ldr q16, [x12, #0x0]\n" + "fmla v8.8h, v16.8h, v0.h[0]\n" "sub x27, x27, #0x1\n" - "ldr q7, [x11, #0x0]\n" - "ldr q6, [x10, #0x0]\n" - "fmla v9.8h, v7.8h, v0.h[0]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "ldr q7, [x9, #0x0]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" + "ldr q17, [x11, #0x0]\n" + "ldr q16, [x10, #0x0]\n" + "fmla v9.8h, v17.8h, v0.h[0]\n" + "fmla v10.8h, v16.8h, v0.h[0]\n" + "ldr q16, [x9, #0x0]\n" + "fmla v11.8h, v16.8h, v0.h[0]\n" "add x12, x12, #0x10\n" "add x11, x11, #0x10\n" "add x10, x10, #0x10\n" @@ -452,17 +452,17 @@ void a64_ffhybrid_fp16_mla_6x32 ( "bne 24b\n" "tbz %x[flags], #1, 32f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.8h }, [x20]\n" + "ld1r { v17.8h }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.8h }, [x20]\n" - "fmin v8.8h, v8.8h, v1.8h\n" - "fmin v9.8h, v9.8h, v1.8h\n" - "fmin v10.8h, v10.8h, v1.8h\n" - "fmin v11.8h, v11.8h, v1.8h\n" - "fmax v8.8h, v8.8h, v0.8h\n" - "fmax v9.8h, v9.8h, v0.8h\n" - "fmax v10.8h, v10.8h, v0.8h\n" - "fmax v11.8h, v11.8h, v0.8h\n" + "ld1r { v16.8h }, [x20]\n" + "fmin v8.8h, v8.8h, v17.8h\n" + "fmin v9.8h, v9.8h, v17.8h\n" + "fmin v10.8h, v10.8h, v17.8h\n" + "fmin v11.8h, v11.8h, v17.8h\n" + "fmax v8.8h, v8.8h, v16.8h\n" + "fmax v9.8h, v9.8h, v16.8h\n" + "fmax v10.8h, v10.8h, v16.8h\n" + "fmax v11.8h, v11.8h, v16.8h\n" "32:" // Height 1: No activation "cmp x14, #0x20\n" "bge 49f\n" @@ -778,12 +778,12 @@ void a64_ffhybrid_fp16_mla_6x32 ( "74:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 75f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 76f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -791,7 +791,7 @@ void a64_ffhybrid_fp16_mla_6x32 ( "b 76f\n" "75:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" "76:" // Height 2: input setup done "cmp x27, #0x8\n" "blt 79f\n" @@ -804,233 +804,233 @@ void a64_ffhybrid_fp16_mla_6x32 ( "77:" // Height 2: Multiply loop: Main loop head "fmla v8.8h, v6.8h, v0.h[0]\n" "fmla v12.8h, v6.8h, v1.h[0]\n" - "ldr q6, [x10, #0x0]\n" + "ldr q17, [x10, #0x0]\n" "sub x27, x27, #0x8\n" "fmla v9.8h, v7.8h, v0.h[0]\n" "fmla v13.8h, v7.8h, v1.h[0]\n" - "ldr q7, [x9, #0x0]\n" + "ldr q16, [x9, #0x0]\n" "cmp x27, #0x10\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "ldr q6, [x12, #0x10]\n" + "fmla v10.8h, v17.8h, v0.h[0]\n" + "fmla v14.8h, v17.8h, v1.h[0]\n" + "ldr q17, [x12, #0x10]\n" "add x26, x26, #0x10\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "ldr q7, [x11, #0x10]\n" + "fmla v11.8h, v16.8h, v0.h[0]\n" + "fmla v15.8h, v16.8h, v1.h[0]\n" + "ldr q16, [x11, #0x10]\n" "add x25, x25, #0x10\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "ldr q6, [x10, #0x10]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "ldr q7, [x9, #0x10]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "fmla v14.8h, v6.8h, v1.h[1]\n" - "ldr q6, [x12, #0x20]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "fmla v15.8h, v7.8h, v1.h[1]\n" - "ldr q7, [x11, #0x20]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "fmla v12.8h, v6.8h, v1.h[2]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "fmla v13.8h, v7.8h, v1.h[2]\n" - "ldr q7, [x9, #0x20]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "fmla v14.8h, v6.8h, v1.h[2]\n" - "ldr q6, [x12, #0x30]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "fmla v15.8h, v7.8h, v1.h[2]\n" - "ldr q7, [x11, #0x30]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "fmla v12.8h, v6.8h, v1.h[3]\n" - "ldr q6, [x10, #0x30]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "fmla v13.8h, v7.8h, v1.h[3]\n" - "ldr q7, [x9, #0x30]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "fmla v14.8h, v6.8h, v1.h[3]\n" - "ldr q6, [x12, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "fmla v15.8h, v7.8h, v1.h[3]\n" - "ldr q7, [x11, #0x40]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "fmla v12.8h, v6.8h, v1.h[4]\n" - "ldr q6, [x10, #0x40]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "fmla v13.8h, v7.8h, v1.h[4]\n" - "ldr q7, [x9, #0x40]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "fmla v14.8h, v6.8h, v1.h[4]\n" - "ldr q6, [x12, #0x50]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "fmla v15.8h, v7.8h, v1.h[4]\n" - "ldr q7, [x11, #0x50]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "fmla v12.8h, v6.8h, v1.h[5]\n" - "ldr q6, [x10, #0x50]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "fmla v13.8h, v7.8h, v1.h[5]\n" - "ldr q7, [x9, #0x50]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "fmla v14.8h, v6.8h, v1.h[5]\n" - "ldr q6, [x12, #0x60]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "fmla v15.8h, v7.8h, v1.h[5]\n" - "ldr q7, [x11, #0x60]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "fmla v12.8h, v6.8h, v1.h[6]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "fmla v13.8h, v7.8h, v1.h[6]\n" - "ldr q7, [x9, #0x60]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "fmla v14.8h, v6.8h, v1.h[6]\n" - "ldr q6, [x12, #0x70]\n" + "fmla v8.8h, v17.8h, v0.h[1]\n" + "fmla v12.8h, v17.8h, v1.h[1]\n" + "ldr q17, [x10, #0x10]\n" + "fmla v9.8h, v16.8h, v0.h[1]\n" + "fmla v13.8h, v16.8h, v1.h[1]\n" + "ldr q16, [x9, #0x10]\n" + "fmla v10.8h, v17.8h, v0.h[1]\n" + "fmla v14.8h, v17.8h, v1.h[1]\n" + "ldr q17, [x12, #0x20]\n" + "fmla v11.8h, v16.8h, v0.h[1]\n" + "fmla v15.8h, v16.8h, v1.h[1]\n" + "ldr q16, [x11, #0x20]\n" + "fmla v8.8h, v17.8h, v0.h[2]\n" + "fmla v12.8h, v17.8h, v1.h[2]\n" + "ldr q17, [x10, #0x20]\n" + "fmla v9.8h, v16.8h, v0.h[2]\n" + "fmla v13.8h, v16.8h, v1.h[2]\n" + "ldr q16, [x9, #0x20]\n" + "fmla v10.8h, v17.8h, v0.h[2]\n" + "fmla v14.8h, v17.8h, v1.h[2]\n" + "ldr q17, [x12, #0x30]\n" + "fmla v11.8h, v16.8h, v0.h[2]\n" + "fmla v15.8h, v16.8h, v1.h[2]\n" + "ldr q16, [x11, #0x30]\n" + "fmla v8.8h, v17.8h, v0.h[3]\n" + "fmla v12.8h, v17.8h, v1.h[3]\n" + "ldr q17, [x10, #0x30]\n" + "fmla v9.8h, v16.8h, v0.h[3]\n" + "fmla v13.8h, v16.8h, v1.h[3]\n" + "ldr q16, [x9, #0x30]\n" + "fmla v10.8h, v17.8h, v0.h[3]\n" + "fmla v14.8h, v17.8h, v1.h[3]\n" + "ldr q17, [x12, #0x40]\n" + "fmla v11.8h, v16.8h, v0.h[3]\n" + "fmla v15.8h, v16.8h, v1.h[3]\n" + "ldr q16, [x11, #0x40]\n" + "fmla v8.8h, v17.8h, v0.h[4]\n" + "fmla v12.8h, v17.8h, v1.h[4]\n" + "ldr q17, [x10, #0x40]\n" + "fmla v9.8h, v16.8h, v0.h[4]\n" + "fmla v13.8h, v16.8h, v1.h[4]\n" + "ldr q16, [x9, #0x40]\n" + "fmla v10.8h, v17.8h, v0.h[4]\n" + "fmla v14.8h, v17.8h, v1.h[4]\n" + "ldr q17, [x12, #0x50]\n" + "fmla v11.8h, v16.8h, v0.h[4]\n" + "fmla v15.8h, v16.8h, v1.h[4]\n" + "ldr q16, [x11, #0x50]\n" + "fmla v8.8h, v17.8h, v0.h[5]\n" + "fmla v12.8h, v17.8h, v1.h[5]\n" + "ldr q17, [x10, #0x50]\n" + "fmla v9.8h, v16.8h, v0.h[5]\n" + "fmla v13.8h, v16.8h, v1.h[5]\n" + "ldr q16, [x9, #0x50]\n" + "fmla v10.8h, v17.8h, v0.h[5]\n" + "fmla v14.8h, v17.8h, v1.h[5]\n" + "ldr q17, [x12, #0x60]\n" + "fmla v11.8h, v16.8h, v0.h[5]\n" + "fmla v15.8h, v16.8h, v1.h[5]\n" + "ldr q16, [x11, #0x60]\n" + "fmla v8.8h, v17.8h, v0.h[6]\n" + "fmla v12.8h, v17.8h, v1.h[6]\n" + "ldr q17, [x10, #0x60]\n" + "fmla v9.8h, v16.8h, v0.h[6]\n" + "fmla v13.8h, v16.8h, v1.h[6]\n" + "ldr q16, [x9, #0x60]\n" + "fmla v10.8h, v17.8h, v0.h[6]\n" + "fmla v14.8h, v17.8h, v1.h[6]\n" + "ldr q17, [x12, #0x70]\n" "add x12, x12, #0x80\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "fmla v15.8h, v7.8h, v1.h[6]\n" - "ldr q7, [x11, #0x70]\n" + "fmla v11.8h, v16.8h, v0.h[6]\n" + "fmla v15.8h, v16.8h, v1.h[6]\n" + "ldr q16, [x11, #0x70]\n" "add x11, x11, #0x80\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "fmla v12.8h, v6.8h, v1.h[7]\n" - "ldr q6, [x10, #0x70]\n" + "fmla v8.8h, v17.8h, v0.h[7]\n" + "fmla v12.8h, v17.8h, v1.h[7]\n" + "ldr q17, [x10, #0x70]\n" "add x10, x10, #0x80\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "fmla v13.8h, v7.8h, v1.h[7]\n" - "ldr q7, [x9, #0x70]\n" + "fmla v9.8h, v16.8h, v0.h[7]\n" + "fmla v13.8h, v16.8h, v1.h[7]\n" + "ldr q16, [x9, #0x70]\n" "add x9, x9, #0x80\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "fmla v14.8h, v6.8h, v1.h[7]\n" + "fmla v10.8h, v17.8h, v0.h[7]\n" + "fmla v14.8h, v17.8h, v1.h[7]\n" "ldr q6, [x12, #0x0]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" + "fmla v11.8h, v16.8h, v0.h[7]\n" "ldr q0, [x26, #0x0]\n" - "fmla v15.8h, v7.8h, v1.h[7]\n" + "fmla v15.8h, v16.8h, v1.h[7]\n" "ldr q1, [x25, #0x0]\n" "ldr q7, [x11, #0x0]\n" "bge 77b\n" "78:" // Height 2: Multiply loop: Single iteration only "fmla v8.8h, v6.8h, v0.h[0]\n" "fmla v12.8h, v6.8h, v1.h[0]\n" - "ldr q6, [x10, #0x0]\n" + "ldr q17, [x10, #0x0]\n" "sub x27, x27, #0x8\n" "fmla v9.8h, v7.8h, v0.h[0]\n" "fmla v13.8h, v7.8h, v1.h[0]\n" - "ldr q7, [x9, #0x0]\n" + "ldr q16, [x9, #0x0]\n" "add x26, x26, #0x10\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "ldr q6, [x12, #0x10]\n" + "fmla v10.8h, v17.8h, v0.h[0]\n" + "fmla v14.8h, v17.8h, v1.h[0]\n" + "ldr q17, [x12, #0x10]\n" "add x25, x25, #0x10\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "ldr q7, [x11, #0x10]\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "ldr q6, [x10, #0x10]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "ldr q7, [x9, #0x10]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "fmla v14.8h, v6.8h, v1.h[1]\n" - "ldr q6, [x12, #0x20]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "fmla v15.8h, v7.8h, v1.h[1]\n" - "ldr q7, [x11, #0x20]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "fmla v12.8h, v6.8h, v1.h[2]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "fmla v13.8h, v7.8h, v1.h[2]\n" - "ldr q7, [x9, #0x20]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "fmla v14.8h, v6.8h, v1.h[2]\n" - "ldr q6, [x12, #0x30]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "fmla v15.8h, v7.8h, v1.h[2]\n" - "ldr q7, [x11, #0x30]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "fmla v12.8h, v6.8h, v1.h[3]\n" - "ldr q6, [x10, #0x30]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "fmla v13.8h, v7.8h, v1.h[3]\n" - "ldr q7, [x9, #0x30]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "fmla v14.8h, v6.8h, v1.h[3]\n" - "ldr q6, [x12, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "fmla v15.8h, v7.8h, v1.h[3]\n" - "ldr q7, [x11, #0x40]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "fmla v12.8h, v6.8h, v1.h[4]\n" - "ldr q6, [x10, #0x40]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "fmla v13.8h, v7.8h, v1.h[4]\n" - "ldr q7, [x9, #0x40]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "fmla v14.8h, v6.8h, v1.h[4]\n" - "ldr q6, [x12, #0x50]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "fmla v15.8h, v7.8h, v1.h[4]\n" - "ldr q7, [x11, #0x50]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "fmla v12.8h, v6.8h, v1.h[5]\n" - "ldr q6, [x10, #0x50]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "fmla v13.8h, v7.8h, v1.h[5]\n" - "ldr q7, [x9, #0x50]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "fmla v14.8h, v6.8h, v1.h[5]\n" - "ldr q6, [x12, #0x60]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "fmla v15.8h, v7.8h, v1.h[5]\n" - "ldr q7, [x11, #0x60]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "fmla v12.8h, v6.8h, v1.h[6]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "fmla v13.8h, v7.8h, v1.h[6]\n" - "ldr q7, [x9, #0x60]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "fmla v14.8h, v6.8h, v1.h[6]\n" - "ldr q6, [x12, #0x70]\n" + "fmla v11.8h, v16.8h, v0.h[0]\n" + "fmla v15.8h, v16.8h, v1.h[0]\n" + "ldr q16, [x11, #0x10]\n" + "fmla v8.8h, v17.8h, v0.h[1]\n" + "fmla v12.8h, v17.8h, v1.h[1]\n" + "ldr q17, [x10, #0x10]\n" + "fmla v9.8h, v16.8h, v0.h[1]\n" + "fmla v13.8h, v16.8h, v1.h[1]\n" + "ldr q16, [x9, #0x10]\n" + "fmla v10.8h, v17.8h, v0.h[1]\n" + "fmla v14.8h, v17.8h, v1.h[1]\n" + "ldr q17, [x12, #0x20]\n" + "fmla v11.8h, v16.8h, v0.h[1]\n" + "fmla v15.8h, v16.8h, v1.h[1]\n" + "ldr q16, [x11, #0x20]\n" + "fmla v8.8h, v17.8h, v0.h[2]\n" + "fmla v12.8h, v17.8h, v1.h[2]\n" + "ldr q17, [x10, #0x20]\n" + "fmla v9.8h, v16.8h, v0.h[2]\n" + "fmla v13.8h, v16.8h, v1.h[2]\n" + "ldr q16, [x9, #0x20]\n" + "fmla v10.8h, v17.8h, v0.h[2]\n" + "fmla v14.8h, v17.8h, v1.h[2]\n" + "ldr q17, [x12, #0x30]\n" + "fmla v11.8h, v16.8h, v0.h[2]\n" + "fmla v15.8h, v16.8h, v1.h[2]\n" + "ldr q16, [x11, #0x30]\n" + "fmla v8.8h, v17.8h, v0.h[3]\n" + "fmla v12.8h, v17.8h, v1.h[3]\n" + "ldr q17, [x10, #0x30]\n" + "fmla v9.8h, v16.8h, v0.h[3]\n" + "fmla v13.8h, v16.8h, v1.h[3]\n" + "ldr q16, [x9, #0x30]\n" + "fmla v10.8h, v17.8h, v0.h[3]\n" + "fmla v14.8h, v17.8h, v1.h[3]\n" + "ldr q17, [x12, #0x40]\n" + "fmla v11.8h, v16.8h, v0.h[3]\n" + "fmla v15.8h, v16.8h, v1.h[3]\n" + "ldr q16, [x11, #0x40]\n" + "fmla v8.8h, v17.8h, v0.h[4]\n" + "fmla v12.8h, v17.8h, v1.h[4]\n" + "ldr q17, [x10, #0x40]\n" + "fmla v9.8h, v16.8h, v0.h[4]\n" + "fmla v13.8h, v16.8h, v1.h[4]\n" + "ldr q16, [x9, #0x40]\n" + "fmla v10.8h, v17.8h, v0.h[4]\n" + "fmla v14.8h, v17.8h, v1.h[4]\n" + "ldr q17, [x12, #0x50]\n" + "fmla v11.8h, v16.8h, v0.h[4]\n" + "fmla v15.8h, v16.8h, v1.h[4]\n" + "ldr q16, [x11, #0x50]\n" + "fmla v8.8h, v17.8h, v0.h[5]\n" + "fmla v12.8h, v17.8h, v1.h[5]\n" + "ldr q17, [x10, #0x50]\n" + "fmla v9.8h, v16.8h, v0.h[5]\n" + "fmla v13.8h, v16.8h, v1.h[5]\n" + "ldr q16, [x9, #0x50]\n" + "fmla v10.8h, v17.8h, v0.h[5]\n" + "fmla v14.8h, v17.8h, v1.h[5]\n" + "ldr q17, [x12, #0x60]\n" + "fmla v11.8h, v16.8h, v0.h[5]\n" + "fmla v15.8h, v16.8h, v1.h[5]\n" + "ldr q16, [x11, #0x60]\n" + "fmla v8.8h, v17.8h, v0.h[6]\n" + "fmla v12.8h, v17.8h, v1.h[6]\n" + "ldr q17, [x10, #0x60]\n" + "fmla v9.8h, v16.8h, v0.h[6]\n" + "fmla v13.8h, v16.8h, v1.h[6]\n" + "ldr q16, [x9, #0x60]\n" + "fmla v10.8h, v17.8h, v0.h[6]\n" + "fmla v14.8h, v17.8h, v1.h[6]\n" + "ldr q17, [x12, #0x70]\n" "add x12, x12, #0x80\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "fmla v15.8h, v7.8h, v1.h[6]\n" - "ldr q7, [x11, #0x70]\n" + "fmla v11.8h, v16.8h, v0.h[6]\n" + "fmla v15.8h, v16.8h, v1.h[6]\n" + "ldr q16, [x11, #0x70]\n" "add x11, x11, #0x80\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "fmla v12.8h, v6.8h, v1.h[7]\n" - "ldr q6, [x10, #0x70]\n" + "fmla v8.8h, v17.8h, v0.h[7]\n" + "fmla v12.8h, v17.8h, v1.h[7]\n" + "ldr q17, [x10, #0x70]\n" "add x10, x10, #0x80\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "fmla v13.8h, v7.8h, v1.h[7]\n" - "ldr q7, [x9, #0x70]\n" + "fmla v9.8h, v16.8h, v0.h[7]\n" + "fmla v13.8h, v16.8h, v1.h[7]\n" + "ldr q16, [x9, #0x70]\n" "add x9, x9, #0x80\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "fmla v14.8h, v6.8h, v1.h[7]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" - "fmla v15.8h, v7.8h, v1.h[7]\n" + "fmla v10.8h, v17.8h, v0.h[7]\n" + "fmla v14.8h, v17.8h, v1.h[7]\n" + "fmla v11.8h, v16.8h, v0.h[7]\n" + "fmla v15.8h, v16.8h, v1.h[7]\n" "79:" // Height 2: Multiply loop: Main loop skip "cbz x27, 81f\n" "80:" // Height 2: Multiply loop: Odd block loop - "ldr h0, [x26], #0x2\n" - "ldr h1, [x25], #0x2\n" + "ldr h1, [x26], #0x2\n" + "ldr h0, [x25], #0x2\n" "sub x27, x27, #0x1\n" - "ldr q6, [x12, #0x0]\n" - "ldr q7, [x11, #0x0]\n" - "fmla v8.8h, v6.8h, v0.h[0]\n" - "fmla v12.8h, v6.8h, v1.h[0]\n" - "ldr q6, [x10, #0x0]\n" - "fmla v9.8h, v7.8h, v0.h[0]\n" - "fmla v13.8h, v7.8h, v1.h[0]\n" - "ldr q7, [x9, #0x0]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" + "ldr q17, [x12, #0x0]\n" + "ldr q16, [x11, #0x0]\n" + "fmla v8.8h, v17.8h, v1.h[0]\n" + "fmla v12.8h, v17.8h, v0.h[0]\n" + "ldr q17, [x10, #0x0]\n" + "fmla v9.8h, v16.8h, v1.h[0]\n" + "fmla v13.8h, v16.8h, v0.h[0]\n" + "ldr q16, [x9, #0x0]\n" + "fmla v10.8h, v17.8h, v1.h[0]\n" + "fmla v14.8h, v17.8h, v0.h[0]\n" "add x12, x12, #0x10\n" "add x11, x11, #0x10\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" + "fmla v11.8h, v16.8h, v1.h[0]\n" + "fmla v15.8h, v16.8h, v0.h[0]\n" "add x10, x10, #0x10\n" "add x9, x9, #0x10\n" "cbnz x27, 80b\n" @@ -1043,25 +1043,25 @@ void a64_ffhybrid_fp16_mla_6x32 ( "add x25, x13, x20, LSL #1\n" "tbz %x[flags], #1, 82f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.8h }, [x20]\n" + "ld1r { v17.8h }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.8h }, [x20]\n" - "fmin v8.8h, v8.8h, v1.8h\n" - "fmin v9.8h, v9.8h, v1.8h\n" - "fmin v10.8h, v10.8h, v1.8h\n" - "fmin v11.8h, v11.8h, v1.8h\n" - "fmin v12.8h, v12.8h, v1.8h\n" - "fmin v13.8h, v13.8h, v1.8h\n" - "fmin v14.8h, v14.8h, v1.8h\n" - "fmin v15.8h, v15.8h, v1.8h\n" - "fmax v8.8h, v8.8h, v0.8h\n" - "fmax v9.8h, v9.8h, v0.8h\n" - "fmax v10.8h, v10.8h, v0.8h\n" - "fmax v11.8h, v11.8h, v0.8h\n" - "fmax v12.8h, v12.8h, v0.8h\n" - "fmax v13.8h, v13.8h, v0.8h\n" - "fmax v14.8h, v14.8h, v0.8h\n" - "fmax v15.8h, v15.8h, v0.8h\n" + "ld1r { v16.8h }, [x20]\n" + "fmin v8.8h, v8.8h, v17.8h\n" + "fmin v9.8h, v9.8h, v17.8h\n" + "fmin v10.8h, v10.8h, v17.8h\n" + "fmin v11.8h, v11.8h, v17.8h\n" + "fmin v12.8h, v12.8h, v17.8h\n" + "fmin v13.8h, v13.8h, v17.8h\n" + "fmin v14.8h, v14.8h, v17.8h\n" + "fmin v15.8h, v15.8h, v17.8h\n" + "fmax v8.8h, v8.8h, v16.8h\n" + "fmax v9.8h, v9.8h, v16.8h\n" + "fmax v10.8h, v10.8h, v16.8h\n" + "fmax v11.8h, v11.8h, v16.8h\n" + "fmax v12.8h, v12.8h, v16.8h\n" + "fmax v13.8h, v13.8h, v16.8h\n" + "fmax v14.8h, v14.8h, v16.8h\n" + "fmax v15.8h, v15.8h, v16.8h\n" "82:" // Height 2: No activation "cmp x14, #0x20\n" "bge 99f\n" @@ -1458,13 +1458,13 @@ void a64_ffhybrid_fp16_mla_6x32 ( "124:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 125f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 126f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -1473,8 +1473,8 @@ void a64_ffhybrid_fp16_mla_6x32 ( "b 126f\n" "125:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" "126:" // Height 3: input setup done "cmp x27, #0x8\n" "blt 129f\n" @@ -1491,139 +1491,139 @@ void a64_ffhybrid_fp16_mla_6x32 ( "sub x27, x27, #0x8\n" "cmp x27, #0x10\n" "fmla v16.8h, v6.8h, v2.h[0]\n" - "ldr q6, [x10, #0x0]\n" + "ldr q21, [x10, #0x0]\n" "fmla v9.8h, v7.8h, v0.h[0]\n" "add x26, x26, #0x10\n" "fmla v13.8h, v7.8h, v1.h[0]\n" "fmla v17.8h, v7.8h, v2.h[0]\n" - "ldr q7, [x9, #0x0]\n" + "ldr q20, [x9, #0x0]\n" "add x25, x25, #0x10\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" + "fmla v10.8h, v21.8h, v0.h[0]\n" + "fmla v14.8h, v21.8h, v1.h[0]\n" "add x24, x24, #0x10\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "ldr q6, [x12, #0x10]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "ldr q7, [x11, #0x10]\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "fmla v16.8h, v6.8h, v2.h[1]\n" - "ldr q6, [x10, #0x10]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "fmla v17.8h, v7.8h, v2.h[1]\n" - "ldr q7, [x9, #0x10]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "fmla v14.8h, v6.8h, v1.h[1]\n" - "fmla v18.8h, v6.8h, v2.h[1]\n" - "ldr q6, [x12, #0x20]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "fmla v15.8h, v7.8h, v1.h[1]\n" - "fmla v19.8h, v7.8h, v2.h[1]\n" - "ldr q7, [x11, #0x20]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "fmla v12.8h, v6.8h, v1.h[2]\n" - "fmla v16.8h, v6.8h, v2.h[2]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "fmla v13.8h, v7.8h, v1.h[2]\n" - "fmla v17.8h, v7.8h, v2.h[2]\n" - "ldr q7, [x9, #0x20]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "fmla v14.8h, v6.8h, v1.h[2]\n" - "fmla v18.8h, v6.8h, v2.h[2]\n" - "ldr q6, [x12, #0x30]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "fmla v15.8h, v7.8h, v1.h[2]\n" - "fmla v19.8h, v7.8h, v2.h[2]\n" - "ldr q7, [x11, #0x30]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "fmla v12.8h, v6.8h, v1.h[3]\n" - "fmla v16.8h, v6.8h, v2.h[3]\n" - "ldr q6, [x10, #0x30]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "fmla v13.8h, v7.8h, v1.h[3]\n" - "fmla v17.8h, v7.8h, v2.h[3]\n" - "ldr q7, [x9, #0x30]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "fmla v14.8h, v6.8h, v1.h[3]\n" - "fmla v18.8h, v6.8h, v2.h[3]\n" - "ldr q6, [x12, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "fmla v15.8h, v7.8h, v1.h[3]\n" - "fmla v19.8h, v7.8h, v2.h[3]\n" - "ldr q7, [x11, #0x40]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "fmla v12.8h, v6.8h, v1.h[4]\n" - "fmla v16.8h, v6.8h, v2.h[4]\n" - "ldr q6, [x10, #0x40]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "fmla v13.8h, v7.8h, v1.h[4]\n" - "fmla v17.8h, v7.8h, v2.h[4]\n" - "ldr q7, [x9, #0x40]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "fmla v14.8h, v6.8h, v1.h[4]\n" - "fmla v18.8h, v6.8h, v2.h[4]\n" - "ldr q6, [x12, #0x50]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "fmla v15.8h, v7.8h, v1.h[4]\n" - "fmla v19.8h, v7.8h, v2.h[4]\n" - "ldr q7, [x11, #0x50]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "fmla v12.8h, v6.8h, v1.h[5]\n" - "fmla v16.8h, v6.8h, v2.h[5]\n" - "ldr q6, [x10, #0x50]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "fmla v13.8h, v7.8h, v1.h[5]\n" - "fmla v17.8h, v7.8h, v2.h[5]\n" - "ldr q7, [x9, #0x50]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "fmla v14.8h, v6.8h, v1.h[5]\n" - "fmla v18.8h, v6.8h, v2.h[5]\n" - "ldr q6, [x12, #0x60]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "fmla v15.8h, v7.8h, v1.h[5]\n" - "fmla v19.8h, v7.8h, v2.h[5]\n" - "ldr q7, [x11, #0x60]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "fmla v12.8h, v6.8h, v1.h[6]\n" - "fmla v16.8h, v6.8h, v2.h[6]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "fmla v13.8h, v7.8h, v1.h[6]\n" - "fmla v17.8h, v7.8h, v2.h[6]\n" - "ldr q7, [x9, #0x60]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "fmla v14.8h, v6.8h, v1.h[6]\n" - "fmla v18.8h, v6.8h, v2.h[6]\n" - "ldr q6, [x12, #0x70]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" + "fmla v18.8h, v21.8h, v2.h[0]\n" + "ldr q21, [x12, #0x10]\n" + "fmla v11.8h, v20.8h, v0.h[0]\n" + "fmla v15.8h, v20.8h, v1.h[0]\n" + "fmla v19.8h, v20.8h, v2.h[0]\n" + "ldr q20, [x11, #0x10]\n" + "fmla v8.8h, v21.8h, v0.h[1]\n" + "fmla v12.8h, v21.8h, v1.h[1]\n" + "fmla v16.8h, v21.8h, v2.h[1]\n" + "ldr q21, [x10, #0x10]\n" + "fmla v9.8h, v20.8h, v0.h[1]\n" + "fmla v13.8h, v20.8h, v1.h[1]\n" + "fmla v17.8h, v20.8h, v2.h[1]\n" + "ldr q20, [x9, #0x10]\n" + "fmla v10.8h, v21.8h, v0.h[1]\n" + "fmla v14.8h, v21.8h, v1.h[1]\n" + "fmla v18.8h, v21.8h, v2.h[1]\n" + "ldr q21, [x12, #0x20]\n" + "fmla v11.8h, v20.8h, v0.h[1]\n" + "fmla v15.8h, v20.8h, v1.h[1]\n" + "fmla v19.8h, v20.8h, v2.h[1]\n" + "ldr q20, [x11, #0x20]\n" + "fmla v8.8h, v21.8h, v0.h[2]\n" + "fmla v12.8h, v21.8h, v1.h[2]\n" + "fmla v16.8h, v21.8h, v2.h[2]\n" + "ldr q21, [x10, #0x20]\n" + "fmla v9.8h, v20.8h, v0.h[2]\n" + "fmla v13.8h, v20.8h, v1.h[2]\n" + "fmla v17.8h, v20.8h, v2.h[2]\n" + "ldr q20, [x9, #0x20]\n" + "fmla v10.8h, v21.8h, v0.h[2]\n" + "fmla v14.8h, v21.8h, v1.h[2]\n" + "fmla v18.8h, v21.8h, v2.h[2]\n" + "ldr q21, [x12, #0x30]\n" + "fmla v11.8h, v20.8h, v0.h[2]\n" + "fmla v15.8h, v20.8h, v1.h[2]\n" + "fmla v19.8h, v20.8h, v2.h[2]\n" + "ldr q20, [x11, #0x30]\n" + "fmla v8.8h, v21.8h, v0.h[3]\n" + "fmla v12.8h, v21.8h, v1.h[3]\n" + "fmla v16.8h, v21.8h, v2.h[3]\n" + "ldr q21, [x10, #0x30]\n" + "fmla v9.8h, v20.8h, v0.h[3]\n" + "fmla v13.8h, v20.8h, v1.h[3]\n" + "fmla v17.8h, v20.8h, v2.h[3]\n" + "ldr q20, [x9, #0x30]\n" + "fmla v10.8h, v21.8h, v0.h[3]\n" + "fmla v14.8h, v21.8h, v1.h[3]\n" + "fmla v18.8h, v21.8h, v2.h[3]\n" + "ldr q21, [x12, #0x40]\n" + "fmla v11.8h, v20.8h, v0.h[3]\n" + "fmla v15.8h, v20.8h, v1.h[3]\n" + "fmla v19.8h, v20.8h, v2.h[3]\n" + "ldr q20, [x11, #0x40]\n" + "fmla v8.8h, v21.8h, v0.h[4]\n" + "fmla v12.8h, v21.8h, v1.h[4]\n" + "fmla v16.8h, v21.8h, v2.h[4]\n" + "ldr q21, [x10, #0x40]\n" + "fmla v9.8h, v20.8h, v0.h[4]\n" + "fmla v13.8h, v20.8h, v1.h[4]\n" + "fmla v17.8h, v20.8h, v2.h[4]\n" + "ldr q20, [x9, #0x40]\n" + "fmla v10.8h, v21.8h, v0.h[4]\n" + "fmla v14.8h, v21.8h, v1.h[4]\n" + "fmla v18.8h, v21.8h, v2.h[4]\n" + "ldr q21, [x12, #0x50]\n" + "fmla v11.8h, v20.8h, v0.h[4]\n" + "fmla v15.8h, v20.8h, v1.h[4]\n" + "fmla v19.8h, v20.8h, v2.h[4]\n" + "ldr q20, [x11, #0x50]\n" + "fmla v8.8h, v21.8h, v0.h[5]\n" + "fmla v12.8h, v21.8h, v1.h[5]\n" + "fmla v16.8h, v21.8h, v2.h[5]\n" + "ldr q21, [x10, #0x50]\n" + "fmla v9.8h, v20.8h, v0.h[5]\n" + "fmla v13.8h, v20.8h, v1.h[5]\n" + "fmla v17.8h, v20.8h, v2.h[5]\n" + "ldr q20, [x9, #0x50]\n" + "fmla v10.8h, v21.8h, v0.h[5]\n" + "fmla v14.8h, v21.8h, v1.h[5]\n" + "fmla v18.8h, v21.8h, v2.h[5]\n" + "ldr q21, [x12, #0x60]\n" + "fmla v11.8h, v20.8h, v0.h[5]\n" + "fmla v15.8h, v20.8h, v1.h[5]\n" + "fmla v19.8h, v20.8h, v2.h[5]\n" + "ldr q20, [x11, #0x60]\n" + "fmla v8.8h, v21.8h, v0.h[6]\n" + "fmla v12.8h, v21.8h, v1.h[6]\n" + "fmla v16.8h, v21.8h, v2.h[6]\n" + "ldr q21, [x10, #0x60]\n" + "fmla v9.8h, v20.8h, v0.h[6]\n" + "fmla v13.8h, v20.8h, v1.h[6]\n" + "fmla v17.8h, v20.8h, v2.h[6]\n" + "ldr q20, [x9, #0x60]\n" + "fmla v10.8h, v21.8h, v0.h[6]\n" + "fmla v14.8h, v21.8h, v1.h[6]\n" + "fmla v18.8h, v21.8h, v2.h[6]\n" + "ldr q21, [x12, #0x70]\n" + "fmla v11.8h, v20.8h, v0.h[6]\n" "add x12, x12, #0x80\n" - "fmla v15.8h, v7.8h, v1.h[6]\n" - "fmla v19.8h, v7.8h, v2.h[6]\n" - "ldr q7, [x11, #0x70]\n" + "fmla v15.8h, v20.8h, v1.h[6]\n" + "fmla v19.8h, v20.8h, v2.h[6]\n" + "ldr q20, [x11, #0x70]\n" "add x11, x11, #0x80\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "fmla v12.8h, v6.8h, v1.h[7]\n" - "fmla v16.8h, v6.8h, v2.h[7]\n" - "ldr q6, [x10, #0x70]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" + "fmla v8.8h, v21.8h, v0.h[7]\n" + "fmla v12.8h, v21.8h, v1.h[7]\n" + "fmla v16.8h, v21.8h, v2.h[7]\n" + "ldr q21, [x10, #0x70]\n" + "fmla v9.8h, v20.8h, v0.h[7]\n" "add x10, x10, #0x80\n" - "fmla v13.8h, v7.8h, v1.h[7]\n" - "fmla v17.8h, v7.8h, v2.h[7]\n" - "ldr q7, [x9, #0x70]\n" + "fmla v13.8h, v20.8h, v1.h[7]\n" + "fmla v17.8h, v20.8h, v2.h[7]\n" + "ldr q20, [x9, #0x70]\n" "add x9, x9, #0x80\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "fmla v14.8h, v6.8h, v1.h[7]\n" - "fmla v18.8h, v6.8h, v2.h[7]\n" + "fmla v10.8h, v21.8h, v0.h[7]\n" + "fmla v14.8h, v21.8h, v1.h[7]\n" + "fmla v18.8h, v21.8h, v2.h[7]\n" "ldr q6, [x12, #0x0]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" + "fmla v11.8h, v20.8h, v0.h[7]\n" "ldr q0, [x26, #0x0]\n" - "fmla v15.8h, v7.8h, v1.h[7]\n" + "fmla v15.8h, v20.8h, v1.h[7]\n" "ldr q1, [x25, #0x0]\n" - "fmla v19.8h, v7.8h, v2.h[7]\n" + "fmla v19.8h, v20.8h, v2.h[7]\n" "ldr q2, [x24, #0x0]\n" "ldr q7, [x11, #0x0]\n" "bge 127b\n" @@ -1633,162 +1633,162 @@ void a64_ffhybrid_fp16_mla_6x32 ( "sub x27, x27, #0x8\n" "add x26, x26, #0x10\n" "fmla v16.8h, v6.8h, v2.h[0]\n" - "ldr q6, [x10, #0x0]\n" + "ldr q21, [x10, #0x0]\n" "fmla v9.8h, v7.8h, v0.h[0]\n" "add x25, x25, #0x10\n" "fmla v13.8h, v7.8h, v1.h[0]\n" "fmla v17.8h, v7.8h, v2.h[0]\n" - "ldr q7, [x9, #0x0]\n" + "ldr q20, [x9, #0x0]\n" "add x24, x24, #0x10\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "ldr q6, [x12, #0x10]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "ldr q7, [x11, #0x10]\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "fmla v16.8h, v6.8h, v2.h[1]\n" - "ldr q6, [x10, #0x10]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "fmla v17.8h, v7.8h, v2.h[1]\n" - "ldr q7, [x9, #0x10]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "fmla v14.8h, v6.8h, v1.h[1]\n" - "fmla v18.8h, v6.8h, v2.h[1]\n" - "ldr q6, [x12, #0x20]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "fmla v15.8h, v7.8h, v1.h[1]\n" - "fmla v19.8h, v7.8h, v2.h[1]\n" - "ldr q7, [x11, #0x20]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "fmla v12.8h, v6.8h, v1.h[2]\n" - "fmla v16.8h, v6.8h, v2.h[2]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "fmla v13.8h, v7.8h, v1.h[2]\n" - "fmla v17.8h, v7.8h, v2.h[2]\n" - "ldr q7, [x9, #0x20]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "fmla v14.8h, v6.8h, v1.h[2]\n" - "fmla v18.8h, v6.8h, v2.h[2]\n" - "ldr q6, [x12, #0x30]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "fmla v15.8h, v7.8h, v1.h[2]\n" - "fmla v19.8h, v7.8h, v2.h[2]\n" - "ldr q7, [x11, #0x30]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "fmla v12.8h, v6.8h, v1.h[3]\n" - "fmla v16.8h, v6.8h, v2.h[3]\n" - "ldr q6, [x10, #0x30]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "fmla v13.8h, v7.8h, v1.h[3]\n" - "fmla v17.8h, v7.8h, v2.h[3]\n" - "ldr q7, [x9, #0x30]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "fmla v14.8h, v6.8h, v1.h[3]\n" - "fmla v18.8h, v6.8h, v2.h[3]\n" - "ldr q6, [x12, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "fmla v15.8h, v7.8h, v1.h[3]\n" - "fmla v19.8h, v7.8h, v2.h[3]\n" - "ldr q7, [x11, #0x40]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "fmla v12.8h, v6.8h, v1.h[4]\n" - "fmla v16.8h, v6.8h, v2.h[4]\n" - "ldr q6, [x10, #0x40]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "fmla v13.8h, v7.8h, v1.h[4]\n" - "fmla v17.8h, v7.8h, v2.h[4]\n" - "ldr q7, [x9, #0x40]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "fmla v14.8h, v6.8h, v1.h[4]\n" - "fmla v18.8h, v6.8h, v2.h[4]\n" - "ldr q6, [x12, #0x50]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "fmla v15.8h, v7.8h, v1.h[4]\n" - "fmla v19.8h, v7.8h, v2.h[4]\n" - "ldr q7, [x11, #0x50]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "fmla v12.8h, v6.8h, v1.h[5]\n" - "fmla v16.8h, v6.8h, v2.h[5]\n" - "ldr q6, [x10, #0x50]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "fmla v13.8h, v7.8h, v1.h[5]\n" - "fmla v17.8h, v7.8h, v2.h[5]\n" - "ldr q7, [x9, #0x50]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "fmla v14.8h, v6.8h, v1.h[5]\n" - "fmla v18.8h, v6.8h, v2.h[5]\n" - "ldr q6, [x12, #0x60]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "fmla v15.8h, v7.8h, v1.h[5]\n" - "fmla v19.8h, v7.8h, v2.h[5]\n" - "ldr q7, [x11, #0x60]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "fmla v12.8h, v6.8h, v1.h[6]\n" - "fmla v16.8h, v6.8h, v2.h[6]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "fmla v13.8h, v7.8h, v1.h[6]\n" - "fmla v17.8h, v7.8h, v2.h[6]\n" - "ldr q7, [x9, #0x60]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "fmla v14.8h, v6.8h, v1.h[6]\n" - "fmla v18.8h, v6.8h, v2.h[6]\n" - "ldr q6, [x12, #0x70]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" + "fmla v10.8h, v21.8h, v0.h[0]\n" + "fmla v14.8h, v21.8h, v1.h[0]\n" + "fmla v18.8h, v21.8h, v2.h[0]\n" + "ldr q21, [x12, #0x10]\n" + "fmla v11.8h, v20.8h, v0.h[0]\n" + "fmla v15.8h, v20.8h, v1.h[0]\n" + "fmla v19.8h, v20.8h, v2.h[0]\n" + "ldr q20, [x11, #0x10]\n" + "fmla v8.8h, v21.8h, v0.h[1]\n" + "fmla v12.8h, v21.8h, v1.h[1]\n" + "fmla v16.8h, v21.8h, v2.h[1]\n" + "ldr q21, [x10, #0x10]\n" + "fmla v9.8h, v20.8h, v0.h[1]\n" + "fmla v13.8h, v20.8h, v1.h[1]\n" + "fmla v17.8h, v20.8h, v2.h[1]\n" + "ldr q20, [x9, #0x10]\n" + "fmla v10.8h, v21.8h, v0.h[1]\n" + "fmla v14.8h, v21.8h, v1.h[1]\n" + "fmla v18.8h, v21.8h, v2.h[1]\n" + "ldr q21, [x12, #0x20]\n" + "fmla v11.8h, v20.8h, v0.h[1]\n" + "fmla v15.8h, v20.8h, v1.h[1]\n" + "fmla v19.8h, v20.8h, v2.h[1]\n" + "ldr q20, [x11, #0x20]\n" + "fmla v8.8h, v21.8h, v0.h[2]\n" + "fmla v12.8h, v21.8h, v1.h[2]\n" + "fmla v16.8h, v21.8h, v2.h[2]\n" + "ldr q21, [x10, #0x20]\n" + "fmla v9.8h, v20.8h, v0.h[2]\n" + "fmla v13.8h, v20.8h, v1.h[2]\n" + "fmla v17.8h, v20.8h, v2.h[2]\n" + "ldr q20, [x9, #0x20]\n" + "fmla v10.8h, v21.8h, v0.h[2]\n" + "fmla v14.8h, v21.8h, v1.h[2]\n" + "fmla v18.8h, v21.8h, v2.h[2]\n" + "ldr q21, [x12, #0x30]\n" + "fmla v11.8h, v20.8h, v0.h[2]\n" + "fmla v15.8h, v20.8h, v1.h[2]\n" + "fmla v19.8h, v20.8h, v2.h[2]\n" + "ldr q20, [x11, #0x30]\n" + "fmla v8.8h, v21.8h, v0.h[3]\n" + "fmla v12.8h, v21.8h, v1.h[3]\n" + "fmla v16.8h, v21.8h, v2.h[3]\n" + "ldr q21, [x10, #0x30]\n" + "fmla v9.8h, v20.8h, v0.h[3]\n" + "fmla v13.8h, v20.8h, v1.h[3]\n" + "fmla v17.8h, v20.8h, v2.h[3]\n" + "ldr q20, [x9, #0x30]\n" + "fmla v10.8h, v21.8h, v0.h[3]\n" + "fmla v14.8h, v21.8h, v1.h[3]\n" + "fmla v18.8h, v21.8h, v2.h[3]\n" + "ldr q21, [x12, #0x40]\n" + "fmla v11.8h, v20.8h, v0.h[3]\n" + "fmla v15.8h, v20.8h, v1.h[3]\n" + "fmla v19.8h, v20.8h, v2.h[3]\n" + "ldr q20, [x11, #0x40]\n" + "fmla v8.8h, v21.8h, v0.h[4]\n" + "fmla v12.8h, v21.8h, v1.h[4]\n" + "fmla v16.8h, v21.8h, v2.h[4]\n" + "ldr q21, [x10, #0x40]\n" + "fmla v9.8h, v20.8h, v0.h[4]\n" + "fmla v13.8h, v20.8h, v1.h[4]\n" + "fmla v17.8h, v20.8h, v2.h[4]\n" + "ldr q20, [x9, #0x40]\n" + "fmla v10.8h, v21.8h, v0.h[4]\n" + "fmla v14.8h, v21.8h, v1.h[4]\n" + "fmla v18.8h, v21.8h, v2.h[4]\n" + "ldr q21, [x12, #0x50]\n" + "fmla v11.8h, v20.8h, v0.h[4]\n" + "fmla v15.8h, v20.8h, v1.h[4]\n" + "fmla v19.8h, v20.8h, v2.h[4]\n" + "ldr q20, [x11, #0x50]\n" + "fmla v8.8h, v21.8h, v0.h[5]\n" + "fmla v12.8h, v21.8h, v1.h[5]\n" + "fmla v16.8h, v21.8h, v2.h[5]\n" + "ldr q21, [x10, #0x50]\n" + "fmla v9.8h, v20.8h, v0.h[5]\n" + "fmla v13.8h, v20.8h, v1.h[5]\n" + "fmla v17.8h, v20.8h, v2.h[5]\n" + "ldr q20, [x9, #0x50]\n" + "fmla v10.8h, v21.8h, v0.h[5]\n" + "fmla v14.8h, v21.8h, v1.h[5]\n" + "fmla v18.8h, v21.8h, v2.h[5]\n" + "ldr q21, [x12, #0x60]\n" + "fmla v11.8h, v20.8h, v0.h[5]\n" + "fmla v15.8h, v20.8h, v1.h[5]\n" + "fmla v19.8h, v20.8h, v2.h[5]\n" + "ldr q20, [x11, #0x60]\n" + "fmla v8.8h, v21.8h, v0.h[6]\n" + "fmla v12.8h, v21.8h, v1.h[6]\n" + "fmla v16.8h, v21.8h, v2.h[6]\n" + "ldr q21, [x10, #0x60]\n" + "fmla v9.8h, v20.8h, v0.h[6]\n" + "fmla v13.8h, v20.8h, v1.h[6]\n" + "fmla v17.8h, v20.8h, v2.h[6]\n" + "ldr q20, [x9, #0x60]\n" + "fmla v10.8h, v21.8h, v0.h[6]\n" + "fmla v14.8h, v21.8h, v1.h[6]\n" + "fmla v18.8h, v21.8h, v2.h[6]\n" + "ldr q21, [x12, #0x70]\n" + "fmla v11.8h, v20.8h, v0.h[6]\n" "add x12, x12, #0x80\n" - "fmla v15.8h, v7.8h, v1.h[6]\n" - "fmla v19.8h, v7.8h, v2.h[6]\n" - "ldr q7, [x11, #0x70]\n" + "fmla v15.8h, v20.8h, v1.h[6]\n" + "fmla v19.8h, v20.8h, v2.h[6]\n" + "ldr q20, [x11, #0x70]\n" "add x11, x11, #0x80\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "fmla v12.8h, v6.8h, v1.h[7]\n" - "fmla v16.8h, v6.8h, v2.h[7]\n" - "ldr q6, [x10, #0x70]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" + "fmla v8.8h, v21.8h, v0.h[7]\n" + "fmla v12.8h, v21.8h, v1.h[7]\n" + "fmla v16.8h, v21.8h, v2.h[7]\n" + "ldr q21, [x10, #0x70]\n" + "fmla v9.8h, v20.8h, v0.h[7]\n" "add x10, x10, #0x80\n" - "fmla v13.8h, v7.8h, v1.h[7]\n" - "fmla v17.8h, v7.8h, v2.h[7]\n" - "ldr q7, [x9, #0x70]\n" + "fmla v13.8h, v20.8h, v1.h[7]\n" + "fmla v17.8h, v20.8h, v2.h[7]\n" + "ldr q20, [x9, #0x70]\n" "add x9, x9, #0x80\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "fmla v14.8h, v6.8h, v1.h[7]\n" - "fmla v18.8h, v6.8h, v2.h[7]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" - "fmla v15.8h, v7.8h, v1.h[7]\n" - "fmla v19.8h, v7.8h, v2.h[7]\n" + "fmla v10.8h, v21.8h, v0.h[7]\n" + "fmla v14.8h, v21.8h, v1.h[7]\n" + "fmla v18.8h, v21.8h, v2.h[7]\n" + "fmla v11.8h, v20.8h, v0.h[7]\n" + "fmla v15.8h, v20.8h, v1.h[7]\n" + "fmla v19.8h, v20.8h, v2.h[7]\n" "129:" // Height 3: Multiply loop: Main loop skip "cbz x27, 131f\n" "130:" // Height 3: Multiply loop: Odd block loop - "ldr h0, [x26], #0x2\n" + "ldr h2, [x26], #0x2\n" "ldr h1, [x25], #0x2\n" "sub x27, x27, #0x1\n" - "ldr h2, [x24], #0x2\n" - "ldr q6, [x12, #0x0]\n" - "fmla v8.8h, v6.8h, v0.h[0]\n" - "fmla v12.8h, v6.8h, v1.h[0]\n" - "ldr q7, [x11, #0x0]\n" - "fmla v16.8h, v6.8h, v2.h[0]\n" - "ldr q6, [x10, #0x0]\n" - "fmla v9.8h, v7.8h, v0.h[0]\n" - "fmla v13.8h, v7.8h, v1.h[0]\n" - "fmla v17.8h, v7.8h, v2.h[0]\n" - "ldr q7, [x9, #0x0]\n" + "ldr h0, [x24], #0x2\n" + "ldr q21, [x12, #0x0]\n" + "fmla v8.8h, v21.8h, v2.h[0]\n" + "fmla v12.8h, v21.8h, v1.h[0]\n" + "ldr q20, [x11, #0x0]\n" + "fmla v16.8h, v21.8h, v0.h[0]\n" + "ldr q21, [x10, #0x0]\n" + "fmla v9.8h, v20.8h, v2.h[0]\n" + "fmla v13.8h, v20.8h, v1.h[0]\n" + "fmla v17.8h, v20.8h, v0.h[0]\n" + "ldr q20, [x9, #0x0]\n" "add x12, x12, #0x10\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" + "fmla v10.8h, v21.8h, v2.h[0]\n" + "fmla v14.8h, v21.8h, v1.h[0]\n" "add x11, x11, #0x10\n" "add x10, x10, #0x10\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" + "fmla v18.8h, v21.8h, v0.h[0]\n" + "fmla v11.8h, v20.8h, v2.h[0]\n" "add x9, x9, #0x10\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" + "fmla v15.8h, v20.8h, v1.h[0]\n" + "fmla v19.8h, v20.8h, v0.h[0]\n" "cbnz x27, 130b\n" "131:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -1800,33 +1800,33 @@ void a64_ffhybrid_fp16_mla_6x32 ( "add x24, x25, x20, LSL #1\n" "tbz %x[flags], #1, 132f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.8h }, [x20]\n" + "ld1r { v21.8h }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.8h }, [x20]\n" - "fmin v8.8h, v8.8h, v1.8h\n" - "fmin v9.8h, v9.8h, v1.8h\n" - "fmin v10.8h, v10.8h, v1.8h\n" - "fmin v11.8h, v11.8h, v1.8h\n" - "fmin v12.8h, v12.8h, v1.8h\n" - "fmin v13.8h, v13.8h, v1.8h\n" - "fmin v14.8h, v14.8h, v1.8h\n" - "fmin v15.8h, v15.8h, v1.8h\n" - "fmin v16.8h, v16.8h, v1.8h\n" - "fmin v17.8h, v17.8h, v1.8h\n" - "fmin v18.8h, v18.8h, v1.8h\n" - "fmin v19.8h, v19.8h, v1.8h\n" - "fmax v8.8h, v8.8h, v0.8h\n" - "fmax v9.8h, v9.8h, v0.8h\n" - "fmax v10.8h, v10.8h, v0.8h\n" - "fmax v11.8h, v11.8h, v0.8h\n" - "fmax v12.8h, v12.8h, v0.8h\n" - "fmax v13.8h, v13.8h, v0.8h\n" - "fmax v14.8h, v14.8h, v0.8h\n" - "fmax v15.8h, v15.8h, v0.8h\n" - "fmax v16.8h, v16.8h, v0.8h\n" - "fmax v17.8h, v17.8h, v0.8h\n" - "fmax v18.8h, v18.8h, v0.8h\n" - "fmax v19.8h, v19.8h, v0.8h\n" + "ld1r { v20.8h }, [x20]\n" + "fmin v8.8h, v8.8h, v21.8h\n" + "fmin v9.8h, v9.8h, v21.8h\n" + "fmin v10.8h, v10.8h, v21.8h\n" + "fmin v11.8h, v11.8h, v21.8h\n" + "fmin v12.8h, v12.8h, v21.8h\n" + "fmin v13.8h, v13.8h, v21.8h\n" + "fmin v14.8h, v14.8h, v21.8h\n" + "fmin v15.8h, v15.8h, v21.8h\n" + "fmin v16.8h, v16.8h, v21.8h\n" + "fmin v17.8h, v17.8h, v21.8h\n" + "fmin v18.8h, v18.8h, v21.8h\n" + "fmin v19.8h, v19.8h, v21.8h\n" + "fmax v8.8h, v8.8h, v20.8h\n" + "fmax v9.8h, v9.8h, v20.8h\n" + "fmax v10.8h, v10.8h, v20.8h\n" + "fmax v11.8h, v11.8h, v20.8h\n" + "fmax v12.8h, v12.8h, v20.8h\n" + "fmax v13.8h, v13.8h, v20.8h\n" + "fmax v14.8h, v14.8h, v20.8h\n" + "fmax v15.8h, v15.8h, v20.8h\n" + "fmax v16.8h, v16.8h, v20.8h\n" + "fmax v17.8h, v17.8h, v20.8h\n" + "fmax v18.8h, v18.8h, v20.8h\n" + "fmax v19.8h, v19.8h, v20.8h\n" "132:" // Height 3: No activation "cmp x14, #0x20\n" "bge 149f\n" @@ -2304,14 +2304,14 @@ void a64_ffhybrid_fp16_mla_6x32 ( "174:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 175f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 176f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -2321,9 +2321,9 @@ void a64_ffhybrid_fp16_mla_6x32 ( "b 176f\n" "175:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" "176:" // Height 4: input setup done "cmp x27, #0x8\n" "blt 179f\n" @@ -2342,7 +2342,7 @@ void a64_ffhybrid_fp16_mla_6x32 ( "cmp x27, #0x10\n" "fmla v16.8h, v6.8h, v2.h[0]\n" "fmla v20.8h, v6.8h, v3.h[0]\n" - "ldr q6, [x10, #0x0]\n" + "ldr q25, [x10, #0x0]\n" "add x26, x26, #0x10\n" "fmla v9.8h, v7.8h, v0.h[0]\n" "fmla v13.8h, v7.8h, v1.h[0]\n" @@ -2350,164 +2350,164 @@ void a64_ffhybrid_fp16_mla_6x32 ( "add x24, x24, #0x10\n" "fmla v17.8h, v7.8h, v2.h[0]\n" "fmla v21.8h, v7.8h, v3.h[0]\n" - "ldr q7, [x9, #0x0]\n" + "ldr q24, [x9, #0x0]\n" "add x23, x23, #0x10\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "fmla v22.8h, v6.8h, v3.h[0]\n" - "ldr q6, [x12, #0x10]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "fmla v23.8h, v7.8h, v3.h[0]\n" - "ldr q7, [x11, #0x10]\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "fmla v16.8h, v6.8h, v2.h[1]\n" - "fmla v20.8h, v6.8h, v3.h[1]\n" - "ldr q6, [x10, #0x10]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "fmla v17.8h, v7.8h, v2.h[1]\n" - "fmla v21.8h, v7.8h, v3.h[1]\n" - "ldr q7, [x9, #0x10]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "fmla v14.8h, v6.8h, v1.h[1]\n" - "fmla v18.8h, v6.8h, v2.h[1]\n" - "fmla v22.8h, v6.8h, v3.h[1]\n" - "ldr q6, [x12, #0x20]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "fmla v15.8h, v7.8h, v1.h[1]\n" - "fmla v19.8h, v7.8h, v2.h[1]\n" - "fmla v23.8h, v7.8h, v3.h[1]\n" - "ldr q7, [x11, #0x20]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "fmla v12.8h, v6.8h, v1.h[2]\n" - "fmla v16.8h, v6.8h, v2.h[2]\n" - "fmla v20.8h, v6.8h, v3.h[2]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "fmla v13.8h, v7.8h, v1.h[2]\n" - "fmla v17.8h, v7.8h, v2.h[2]\n" - "fmla v21.8h, v7.8h, v3.h[2]\n" - "ldr q7, [x9, #0x20]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "fmla v14.8h, v6.8h, v1.h[2]\n" - "fmla v18.8h, v6.8h, v2.h[2]\n" - "fmla v22.8h, v6.8h, v3.h[2]\n" - "ldr q6, [x12, #0x30]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "fmla v15.8h, v7.8h, v1.h[2]\n" - "fmla v19.8h, v7.8h, v2.h[2]\n" - "fmla v23.8h, v7.8h, v3.h[2]\n" - "ldr q7, [x11, #0x30]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "fmla v12.8h, v6.8h, v1.h[3]\n" - "fmla v16.8h, v6.8h, v2.h[3]\n" - "fmla v20.8h, v6.8h, v3.h[3]\n" - "ldr q6, [x10, #0x30]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "fmla v13.8h, v7.8h, v1.h[3]\n" - "fmla v17.8h, v7.8h, v2.h[3]\n" - "fmla v21.8h, v7.8h, v3.h[3]\n" - "ldr q7, [x9, #0x30]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "fmla v14.8h, v6.8h, v1.h[3]\n" - "fmla v18.8h, v6.8h, v2.h[3]\n" - "fmla v22.8h, v6.8h, v3.h[3]\n" - "ldr q6, [x12, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "fmla v15.8h, v7.8h, v1.h[3]\n" - "fmla v19.8h, v7.8h, v2.h[3]\n" - "fmla v23.8h, v7.8h, v3.h[3]\n" - "ldr q7, [x11, #0x40]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "fmla v12.8h, v6.8h, v1.h[4]\n" - "fmla v16.8h, v6.8h, v2.h[4]\n" - "fmla v20.8h, v6.8h, v3.h[4]\n" - "ldr q6, [x10, #0x40]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "fmla v13.8h, v7.8h, v1.h[4]\n" - "fmla v17.8h, v7.8h, v2.h[4]\n" - "fmla v21.8h, v7.8h, v3.h[4]\n" - "ldr q7, [x9, #0x40]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "fmla v14.8h, v6.8h, v1.h[4]\n" - "fmla v18.8h, v6.8h, v2.h[4]\n" - "fmla v22.8h, v6.8h, v3.h[4]\n" - "ldr q6, [x12, #0x50]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "fmla v15.8h, v7.8h, v1.h[4]\n" - "fmla v19.8h, v7.8h, v2.h[4]\n" - "fmla v23.8h, v7.8h, v3.h[4]\n" - "ldr q7, [x11, #0x50]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "fmla v12.8h, v6.8h, v1.h[5]\n" - "fmla v16.8h, v6.8h, v2.h[5]\n" - "fmla v20.8h, v6.8h, v3.h[5]\n" - "ldr q6, [x10, #0x50]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "fmla v13.8h, v7.8h, v1.h[5]\n" - "fmla v17.8h, v7.8h, v2.h[5]\n" - "fmla v21.8h, v7.8h, v3.h[5]\n" - "ldr q7, [x9, #0x50]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "fmla v14.8h, v6.8h, v1.h[5]\n" - "fmla v18.8h, v6.8h, v2.h[5]\n" - "fmla v22.8h, v6.8h, v3.h[5]\n" - "ldr q6, [x12, #0x60]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "fmla v15.8h, v7.8h, v1.h[5]\n" - "fmla v19.8h, v7.8h, v2.h[5]\n" - "fmla v23.8h, v7.8h, v3.h[5]\n" - "ldr q7, [x11, #0x60]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "fmla v12.8h, v6.8h, v1.h[6]\n" - "fmla v16.8h, v6.8h, v2.h[6]\n" - "fmla v20.8h, v6.8h, v3.h[6]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "fmla v13.8h, v7.8h, v1.h[6]\n" - "fmla v17.8h, v7.8h, v2.h[6]\n" - "fmla v21.8h, v7.8h, v3.h[6]\n" - "ldr q7, [x9, #0x60]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "fmla v14.8h, v6.8h, v1.h[6]\n" - "fmla v18.8h, v6.8h, v2.h[6]\n" - "fmla v22.8h, v6.8h, v3.h[6]\n" - "ldr q6, [x12, #0x70]\n" + "fmla v10.8h, v25.8h, v0.h[0]\n" + "fmla v14.8h, v25.8h, v1.h[0]\n" + "fmla v18.8h, v25.8h, v2.h[0]\n" + "fmla v22.8h, v25.8h, v3.h[0]\n" + "ldr q25, [x12, #0x10]\n" + "fmla v11.8h, v24.8h, v0.h[0]\n" + "fmla v15.8h, v24.8h, v1.h[0]\n" + "fmla v19.8h, v24.8h, v2.h[0]\n" + "fmla v23.8h, v24.8h, v3.h[0]\n" + "ldr q24, [x11, #0x10]\n" + "fmla v8.8h, v25.8h, v0.h[1]\n" + "fmla v12.8h, v25.8h, v1.h[1]\n" + "fmla v16.8h, v25.8h, v2.h[1]\n" + "fmla v20.8h, v25.8h, v3.h[1]\n" + "ldr q25, [x10, #0x10]\n" + "fmla v9.8h, v24.8h, v0.h[1]\n" + "fmla v13.8h, v24.8h, v1.h[1]\n" + "fmla v17.8h, v24.8h, v2.h[1]\n" + "fmla v21.8h, v24.8h, v3.h[1]\n" + "ldr q24, [x9, #0x10]\n" + "fmla v10.8h, v25.8h, v0.h[1]\n" + "fmla v14.8h, v25.8h, v1.h[1]\n" + "fmla v18.8h, v25.8h, v2.h[1]\n" + "fmla v22.8h, v25.8h, v3.h[1]\n" + "ldr q25, [x12, #0x20]\n" + "fmla v11.8h, v24.8h, v0.h[1]\n" + "fmla v15.8h, v24.8h, v1.h[1]\n" + "fmla v19.8h, v24.8h, v2.h[1]\n" + "fmla v23.8h, v24.8h, v3.h[1]\n" + "ldr q24, [x11, #0x20]\n" + "fmla v8.8h, v25.8h, v0.h[2]\n" + "fmla v12.8h, v25.8h, v1.h[2]\n" + "fmla v16.8h, v25.8h, v2.h[2]\n" + "fmla v20.8h, v25.8h, v3.h[2]\n" + "ldr q25, [x10, #0x20]\n" + "fmla v9.8h, v24.8h, v0.h[2]\n" + "fmla v13.8h, v24.8h, v1.h[2]\n" + "fmla v17.8h, v24.8h, v2.h[2]\n" + "fmla v21.8h, v24.8h, v3.h[2]\n" + "ldr q24, [x9, #0x20]\n" + "fmla v10.8h, v25.8h, v0.h[2]\n" + "fmla v14.8h, v25.8h, v1.h[2]\n" + "fmla v18.8h, v25.8h, v2.h[2]\n" + "fmla v22.8h, v25.8h, v3.h[2]\n" + "ldr q25, [x12, #0x30]\n" + "fmla v11.8h, v24.8h, v0.h[2]\n" + "fmla v15.8h, v24.8h, v1.h[2]\n" + "fmla v19.8h, v24.8h, v2.h[2]\n" + "fmla v23.8h, v24.8h, v3.h[2]\n" + "ldr q24, [x11, #0x30]\n" + "fmla v8.8h, v25.8h, v0.h[3]\n" + "fmla v12.8h, v25.8h, v1.h[3]\n" + "fmla v16.8h, v25.8h, v2.h[3]\n" + "fmla v20.8h, v25.8h, v3.h[3]\n" + "ldr q25, [x10, #0x30]\n" + "fmla v9.8h, v24.8h, v0.h[3]\n" + "fmla v13.8h, v24.8h, v1.h[3]\n" + "fmla v17.8h, v24.8h, v2.h[3]\n" + "fmla v21.8h, v24.8h, v3.h[3]\n" + "ldr q24, [x9, #0x30]\n" + "fmla v10.8h, v25.8h, v0.h[3]\n" + "fmla v14.8h, v25.8h, v1.h[3]\n" + "fmla v18.8h, v25.8h, v2.h[3]\n" + "fmla v22.8h, v25.8h, v3.h[3]\n" + "ldr q25, [x12, #0x40]\n" + "fmla v11.8h, v24.8h, v0.h[3]\n" + "fmla v15.8h, v24.8h, v1.h[3]\n" + "fmla v19.8h, v24.8h, v2.h[3]\n" + "fmla v23.8h, v24.8h, v3.h[3]\n" + "ldr q24, [x11, #0x40]\n" + "fmla v8.8h, v25.8h, v0.h[4]\n" + "fmla v12.8h, v25.8h, v1.h[4]\n" + "fmla v16.8h, v25.8h, v2.h[4]\n" + "fmla v20.8h, v25.8h, v3.h[4]\n" + "ldr q25, [x10, #0x40]\n" + "fmla v9.8h, v24.8h, v0.h[4]\n" + "fmla v13.8h, v24.8h, v1.h[4]\n" + "fmla v17.8h, v24.8h, v2.h[4]\n" + "fmla v21.8h, v24.8h, v3.h[4]\n" + "ldr q24, [x9, #0x40]\n" + "fmla v10.8h, v25.8h, v0.h[4]\n" + "fmla v14.8h, v25.8h, v1.h[4]\n" + "fmla v18.8h, v25.8h, v2.h[4]\n" + "fmla v22.8h, v25.8h, v3.h[4]\n" + "ldr q25, [x12, #0x50]\n" + "fmla v11.8h, v24.8h, v0.h[4]\n" + "fmla v15.8h, v24.8h, v1.h[4]\n" + "fmla v19.8h, v24.8h, v2.h[4]\n" + "fmla v23.8h, v24.8h, v3.h[4]\n" + "ldr q24, [x11, #0x50]\n" + "fmla v8.8h, v25.8h, v0.h[5]\n" + "fmla v12.8h, v25.8h, v1.h[5]\n" + "fmla v16.8h, v25.8h, v2.h[5]\n" + "fmla v20.8h, v25.8h, v3.h[5]\n" + "ldr q25, [x10, #0x50]\n" + "fmla v9.8h, v24.8h, v0.h[5]\n" + "fmla v13.8h, v24.8h, v1.h[5]\n" + "fmla v17.8h, v24.8h, v2.h[5]\n" + "fmla v21.8h, v24.8h, v3.h[5]\n" + "ldr q24, [x9, #0x50]\n" + "fmla v10.8h, v25.8h, v0.h[5]\n" + "fmla v14.8h, v25.8h, v1.h[5]\n" + "fmla v18.8h, v25.8h, v2.h[5]\n" + "fmla v22.8h, v25.8h, v3.h[5]\n" + "ldr q25, [x12, #0x60]\n" + "fmla v11.8h, v24.8h, v0.h[5]\n" + "fmla v15.8h, v24.8h, v1.h[5]\n" + "fmla v19.8h, v24.8h, v2.h[5]\n" + "fmla v23.8h, v24.8h, v3.h[5]\n" + "ldr q24, [x11, #0x60]\n" + "fmla v8.8h, v25.8h, v0.h[6]\n" + "fmla v12.8h, v25.8h, v1.h[6]\n" + "fmla v16.8h, v25.8h, v2.h[6]\n" + "fmla v20.8h, v25.8h, v3.h[6]\n" + "ldr q25, [x10, #0x60]\n" + "fmla v9.8h, v24.8h, v0.h[6]\n" + "fmla v13.8h, v24.8h, v1.h[6]\n" + "fmla v17.8h, v24.8h, v2.h[6]\n" + "fmla v21.8h, v24.8h, v3.h[6]\n" + "ldr q24, [x9, #0x60]\n" + "fmla v10.8h, v25.8h, v0.h[6]\n" + "fmla v14.8h, v25.8h, v1.h[6]\n" + "fmla v18.8h, v25.8h, v2.h[6]\n" + "fmla v22.8h, v25.8h, v3.h[6]\n" + "ldr q25, [x12, #0x70]\n" "add x12, x12, #0x80\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "fmla v15.8h, v7.8h, v1.h[6]\n" - "fmla v19.8h, v7.8h, v2.h[6]\n" - "fmla v23.8h, v7.8h, v3.h[6]\n" - "ldr q7, [x11, #0x70]\n" + "fmla v11.8h, v24.8h, v0.h[6]\n" + "fmla v15.8h, v24.8h, v1.h[6]\n" + "fmla v19.8h, v24.8h, v2.h[6]\n" + "fmla v23.8h, v24.8h, v3.h[6]\n" + "ldr q24, [x11, #0x70]\n" "add x11, x11, #0x80\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "fmla v12.8h, v6.8h, v1.h[7]\n" - "fmla v16.8h, v6.8h, v2.h[7]\n" - "fmla v20.8h, v6.8h, v3.h[7]\n" - "ldr q6, [x10, #0x70]\n" + "fmla v8.8h, v25.8h, v0.h[7]\n" + "fmla v12.8h, v25.8h, v1.h[7]\n" + "fmla v16.8h, v25.8h, v2.h[7]\n" + "fmla v20.8h, v25.8h, v3.h[7]\n" + "ldr q25, [x10, #0x70]\n" "add x10, x10, #0x80\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "fmla v13.8h, v7.8h, v1.h[7]\n" - "fmla v17.8h, v7.8h, v2.h[7]\n" - "fmla v21.8h, v7.8h, v3.h[7]\n" - "ldr q7, [x9, #0x70]\n" + "fmla v9.8h, v24.8h, v0.h[7]\n" + "fmla v13.8h, v24.8h, v1.h[7]\n" + "fmla v17.8h, v24.8h, v2.h[7]\n" + "fmla v21.8h, v24.8h, v3.h[7]\n" + "ldr q24, [x9, #0x70]\n" "add x9, x9, #0x80\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "fmla v14.8h, v6.8h, v1.h[7]\n" - "fmla v18.8h, v6.8h, v2.h[7]\n" - "fmla v22.8h, v6.8h, v3.h[7]\n" + "fmla v10.8h, v25.8h, v0.h[7]\n" + "fmla v14.8h, v25.8h, v1.h[7]\n" + "fmla v18.8h, v25.8h, v2.h[7]\n" + "fmla v22.8h, v25.8h, v3.h[7]\n" "ldr q6, [x12, #0x0]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" + "fmla v11.8h, v24.8h, v0.h[7]\n" "ldr q0, [x26, #0x0]\n" - "fmla v15.8h, v7.8h, v1.h[7]\n" + "fmla v15.8h, v24.8h, v1.h[7]\n" "ldr q1, [x25, #0x0]\n" - "fmla v19.8h, v7.8h, v2.h[7]\n" + "fmla v19.8h, v24.8h, v2.h[7]\n" "ldr q2, [x24, #0x0]\n" - "fmla v23.8h, v7.8h, v3.h[7]\n" + "fmla v23.8h, v24.8h, v3.h[7]\n" "ldr q3, [x23, #0x0]\n" "ldr q7, [x11, #0x0]\n" "bge 177b\n" @@ -2518,7 +2518,7 @@ void a64_ffhybrid_fp16_mla_6x32 ( "add x26, x26, #0x10\n" "fmla v16.8h, v6.8h, v2.h[0]\n" "fmla v20.8h, v6.8h, v3.h[0]\n" - "ldr q6, [x10, #0x0]\n" + "ldr q25, [x10, #0x0]\n" "add x25, x25, #0x10\n" "fmla v9.8h, v7.8h, v0.h[0]\n" "fmla v13.8h, v7.8h, v1.h[0]\n" @@ -2526,191 +2526,191 @@ void a64_ffhybrid_fp16_mla_6x32 ( "add x23, x23, #0x10\n" "fmla v17.8h, v7.8h, v2.h[0]\n" "fmla v21.8h, v7.8h, v3.h[0]\n" - "ldr q7, [x9, #0x0]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "fmla v22.8h, v6.8h, v3.h[0]\n" - "ldr q6, [x12, #0x10]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "fmla v23.8h, v7.8h, v3.h[0]\n" - "ldr q7, [x11, #0x10]\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "fmla v16.8h, v6.8h, v2.h[1]\n" - "fmla v20.8h, v6.8h, v3.h[1]\n" - "ldr q6, [x10, #0x10]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "fmla v17.8h, v7.8h, v2.h[1]\n" - "fmla v21.8h, v7.8h, v3.h[1]\n" - "ldr q7, [x9, #0x10]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "fmla v14.8h, v6.8h, v1.h[1]\n" - "fmla v18.8h, v6.8h, v2.h[1]\n" - "fmla v22.8h, v6.8h, v3.h[1]\n" - "ldr q6, [x12, #0x20]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "fmla v15.8h, v7.8h, v1.h[1]\n" - "fmla v19.8h, v7.8h, v2.h[1]\n" - "fmla v23.8h, v7.8h, v3.h[1]\n" - "ldr q7, [x11, #0x20]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "fmla v12.8h, v6.8h, v1.h[2]\n" - "fmla v16.8h, v6.8h, v2.h[2]\n" - "fmla v20.8h, v6.8h, v3.h[2]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "fmla v13.8h, v7.8h, v1.h[2]\n" - "fmla v17.8h, v7.8h, v2.h[2]\n" - "fmla v21.8h, v7.8h, v3.h[2]\n" - "ldr q7, [x9, #0x20]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "fmla v14.8h, v6.8h, v1.h[2]\n" - "fmla v18.8h, v6.8h, v2.h[2]\n" - "fmla v22.8h, v6.8h, v3.h[2]\n" - "ldr q6, [x12, #0x30]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "fmla v15.8h, v7.8h, v1.h[2]\n" - "fmla v19.8h, v7.8h, v2.h[2]\n" - "fmla v23.8h, v7.8h, v3.h[2]\n" - "ldr q7, [x11, #0x30]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "fmla v12.8h, v6.8h, v1.h[3]\n" - "fmla v16.8h, v6.8h, v2.h[3]\n" - "fmla v20.8h, v6.8h, v3.h[3]\n" - "ldr q6, [x10, #0x30]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "fmla v13.8h, v7.8h, v1.h[3]\n" - "fmla v17.8h, v7.8h, v2.h[3]\n" - "fmla v21.8h, v7.8h, v3.h[3]\n" - "ldr q7, [x9, #0x30]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "fmla v14.8h, v6.8h, v1.h[3]\n" - "fmla v18.8h, v6.8h, v2.h[3]\n" - "fmla v22.8h, v6.8h, v3.h[3]\n" - "ldr q6, [x12, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "fmla v15.8h, v7.8h, v1.h[3]\n" - "fmla v19.8h, v7.8h, v2.h[3]\n" - "fmla v23.8h, v7.8h, v3.h[3]\n" - "ldr q7, [x11, #0x40]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "fmla v12.8h, v6.8h, v1.h[4]\n" - "fmla v16.8h, v6.8h, v2.h[4]\n" - "fmla v20.8h, v6.8h, v3.h[4]\n" - "ldr q6, [x10, #0x40]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "fmla v13.8h, v7.8h, v1.h[4]\n" - "fmla v17.8h, v7.8h, v2.h[4]\n" - "fmla v21.8h, v7.8h, v3.h[4]\n" - "ldr q7, [x9, #0x40]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "fmla v14.8h, v6.8h, v1.h[4]\n" - "fmla v18.8h, v6.8h, v2.h[4]\n" - "fmla v22.8h, v6.8h, v3.h[4]\n" - "ldr q6, [x12, #0x50]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "fmla v15.8h, v7.8h, v1.h[4]\n" - "fmla v19.8h, v7.8h, v2.h[4]\n" - "fmla v23.8h, v7.8h, v3.h[4]\n" - "ldr q7, [x11, #0x50]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "fmla v12.8h, v6.8h, v1.h[5]\n" - "fmla v16.8h, v6.8h, v2.h[5]\n" - "fmla v20.8h, v6.8h, v3.h[5]\n" - "ldr q6, [x10, #0x50]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "fmla v13.8h, v7.8h, v1.h[5]\n" - "fmla v17.8h, v7.8h, v2.h[5]\n" - "fmla v21.8h, v7.8h, v3.h[5]\n" - "ldr q7, [x9, #0x50]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "fmla v14.8h, v6.8h, v1.h[5]\n" - "fmla v18.8h, v6.8h, v2.h[5]\n" - "fmla v22.8h, v6.8h, v3.h[5]\n" - "ldr q6, [x12, #0x60]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "fmla v15.8h, v7.8h, v1.h[5]\n" - "fmla v19.8h, v7.8h, v2.h[5]\n" - "fmla v23.8h, v7.8h, v3.h[5]\n" - "ldr q7, [x11, #0x60]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "fmla v12.8h, v6.8h, v1.h[6]\n" - "fmla v16.8h, v6.8h, v2.h[6]\n" - "fmla v20.8h, v6.8h, v3.h[6]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "fmla v13.8h, v7.8h, v1.h[6]\n" - "fmla v17.8h, v7.8h, v2.h[6]\n" - "fmla v21.8h, v7.8h, v3.h[6]\n" - "ldr q7, [x9, #0x60]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "fmla v14.8h, v6.8h, v1.h[6]\n" - "fmla v18.8h, v6.8h, v2.h[6]\n" - "fmla v22.8h, v6.8h, v3.h[6]\n" - "ldr q6, [x12, #0x70]\n" + "ldr q24, [x9, #0x0]\n" + "fmla v10.8h, v25.8h, v0.h[0]\n" + "fmla v14.8h, v25.8h, v1.h[0]\n" + "fmla v18.8h, v25.8h, v2.h[0]\n" + "fmla v22.8h, v25.8h, v3.h[0]\n" + "ldr q25, [x12, #0x10]\n" + "fmla v11.8h, v24.8h, v0.h[0]\n" + "fmla v15.8h, v24.8h, v1.h[0]\n" + "fmla v19.8h, v24.8h, v2.h[0]\n" + "fmla v23.8h, v24.8h, v3.h[0]\n" + "ldr q24, [x11, #0x10]\n" + "fmla v8.8h, v25.8h, v0.h[1]\n" + "fmla v12.8h, v25.8h, v1.h[1]\n" + "fmla v16.8h, v25.8h, v2.h[1]\n" + "fmla v20.8h, v25.8h, v3.h[1]\n" + "ldr q25, [x10, #0x10]\n" + "fmla v9.8h, v24.8h, v0.h[1]\n" + "fmla v13.8h, v24.8h, v1.h[1]\n" + "fmla v17.8h, v24.8h, v2.h[1]\n" + "fmla v21.8h, v24.8h, v3.h[1]\n" + "ldr q24, [x9, #0x10]\n" + "fmla v10.8h, v25.8h, v0.h[1]\n" + "fmla v14.8h, v25.8h, v1.h[1]\n" + "fmla v18.8h, v25.8h, v2.h[1]\n" + "fmla v22.8h, v25.8h, v3.h[1]\n" + "ldr q25, [x12, #0x20]\n" + "fmla v11.8h, v24.8h, v0.h[1]\n" + "fmla v15.8h, v24.8h, v1.h[1]\n" + "fmla v19.8h, v24.8h, v2.h[1]\n" + "fmla v23.8h, v24.8h, v3.h[1]\n" + "ldr q24, [x11, #0x20]\n" + "fmla v8.8h, v25.8h, v0.h[2]\n" + "fmla v12.8h, v25.8h, v1.h[2]\n" + "fmla v16.8h, v25.8h, v2.h[2]\n" + "fmla v20.8h, v25.8h, v3.h[2]\n" + "ldr q25, [x10, #0x20]\n" + "fmla v9.8h, v24.8h, v0.h[2]\n" + "fmla v13.8h, v24.8h, v1.h[2]\n" + "fmla v17.8h, v24.8h, v2.h[2]\n" + "fmla v21.8h, v24.8h, v3.h[2]\n" + "ldr q24, [x9, #0x20]\n" + "fmla v10.8h, v25.8h, v0.h[2]\n" + "fmla v14.8h, v25.8h, v1.h[2]\n" + "fmla v18.8h, v25.8h, v2.h[2]\n" + "fmla v22.8h, v25.8h, v3.h[2]\n" + "ldr q25, [x12, #0x30]\n" + "fmla v11.8h, v24.8h, v0.h[2]\n" + "fmla v15.8h, v24.8h, v1.h[2]\n" + "fmla v19.8h, v24.8h, v2.h[2]\n" + "fmla v23.8h, v24.8h, v3.h[2]\n" + "ldr q24, [x11, #0x30]\n" + "fmla v8.8h, v25.8h, v0.h[3]\n" + "fmla v12.8h, v25.8h, v1.h[3]\n" + "fmla v16.8h, v25.8h, v2.h[3]\n" + "fmla v20.8h, v25.8h, v3.h[3]\n" + "ldr q25, [x10, #0x30]\n" + "fmla v9.8h, v24.8h, v0.h[3]\n" + "fmla v13.8h, v24.8h, v1.h[3]\n" + "fmla v17.8h, v24.8h, v2.h[3]\n" + "fmla v21.8h, v24.8h, v3.h[3]\n" + "ldr q24, [x9, #0x30]\n" + "fmla v10.8h, v25.8h, v0.h[3]\n" + "fmla v14.8h, v25.8h, v1.h[3]\n" + "fmla v18.8h, v25.8h, v2.h[3]\n" + "fmla v22.8h, v25.8h, v3.h[3]\n" + "ldr q25, [x12, #0x40]\n" + "fmla v11.8h, v24.8h, v0.h[3]\n" + "fmla v15.8h, v24.8h, v1.h[3]\n" + "fmla v19.8h, v24.8h, v2.h[3]\n" + "fmla v23.8h, v24.8h, v3.h[3]\n" + "ldr q24, [x11, #0x40]\n" + "fmla v8.8h, v25.8h, v0.h[4]\n" + "fmla v12.8h, v25.8h, v1.h[4]\n" + "fmla v16.8h, v25.8h, v2.h[4]\n" + "fmla v20.8h, v25.8h, v3.h[4]\n" + "ldr q25, [x10, #0x40]\n" + "fmla v9.8h, v24.8h, v0.h[4]\n" + "fmla v13.8h, v24.8h, v1.h[4]\n" + "fmla v17.8h, v24.8h, v2.h[4]\n" + "fmla v21.8h, v24.8h, v3.h[4]\n" + "ldr q24, [x9, #0x40]\n" + "fmla v10.8h, v25.8h, v0.h[4]\n" + "fmla v14.8h, v25.8h, v1.h[4]\n" + "fmla v18.8h, v25.8h, v2.h[4]\n" + "fmla v22.8h, v25.8h, v3.h[4]\n" + "ldr q25, [x12, #0x50]\n" + "fmla v11.8h, v24.8h, v0.h[4]\n" + "fmla v15.8h, v24.8h, v1.h[4]\n" + "fmla v19.8h, v24.8h, v2.h[4]\n" + "fmla v23.8h, v24.8h, v3.h[4]\n" + "ldr q24, [x11, #0x50]\n" + "fmla v8.8h, v25.8h, v0.h[5]\n" + "fmla v12.8h, v25.8h, v1.h[5]\n" + "fmla v16.8h, v25.8h, v2.h[5]\n" + "fmla v20.8h, v25.8h, v3.h[5]\n" + "ldr q25, [x10, #0x50]\n" + "fmla v9.8h, v24.8h, v0.h[5]\n" + "fmla v13.8h, v24.8h, v1.h[5]\n" + "fmla v17.8h, v24.8h, v2.h[5]\n" + "fmla v21.8h, v24.8h, v3.h[5]\n" + "ldr q24, [x9, #0x50]\n" + "fmla v10.8h, v25.8h, v0.h[5]\n" + "fmla v14.8h, v25.8h, v1.h[5]\n" + "fmla v18.8h, v25.8h, v2.h[5]\n" + "fmla v22.8h, v25.8h, v3.h[5]\n" + "ldr q25, [x12, #0x60]\n" + "fmla v11.8h, v24.8h, v0.h[5]\n" + "fmla v15.8h, v24.8h, v1.h[5]\n" + "fmla v19.8h, v24.8h, v2.h[5]\n" + "fmla v23.8h, v24.8h, v3.h[5]\n" + "ldr q24, [x11, #0x60]\n" + "fmla v8.8h, v25.8h, v0.h[6]\n" + "fmla v12.8h, v25.8h, v1.h[6]\n" + "fmla v16.8h, v25.8h, v2.h[6]\n" + "fmla v20.8h, v25.8h, v3.h[6]\n" + "ldr q25, [x10, #0x60]\n" + "fmla v9.8h, v24.8h, v0.h[6]\n" + "fmla v13.8h, v24.8h, v1.h[6]\n" + "fmla v17.8h, v24.8h, v2.h[6]\n" + "fmla v21.8h, v24.8h, v3.h[6]\n" + "ldr q24, [x9, #0x60]\n" + "fmla v10.8h, v25.8h, v0.h[6]\n" + "fmla v14.8h, v25.8h, v1.h[6]\n" + "fmla v18.8h, v25.8h, v2.h[6]\n" + "fmla v22.8h, v25.8h, v3.h[6]\n" + "ldr q25, [x12, #0x70]\n" "add x12, x12, #0x80\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "fmla v15.8h, v7.8h, v1.h[6]\n" - "fmla v19.8h, v7.8h, v2.h[6]\n" - "fmla v23.8h, v7.8h, v3.h[6]\n" - "ldr q7, [x11, #0x70]\n" + "fmla v11.8h, v24.8h, v0.h[6]\n" + "fmla v15.8h, v24.8h, v1.h[6]\n" + "fmla v19.8h, v24.8h, v2.h[6]\n" + "fmla v23.8h, v24.8h, v3.h[6]\n" + "ldr q24, [x11, #0x70]\n" "add x11, x11, #0x80\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "fmla v12.8h, v6.8h, v1.h[7]\n" - "fmla v16.8h, v6.8h, v2.h[7]\n" - "fmla v20.8h, v6.8h, v3.h[7]\n" - "ldr q6, [x10, #0x70]\n" + "fmla v8.8h, v25.8h, v0.h[7]\n" + "fmla v12.8h, v25.8h, v1.h[7]\n" + "fmla v16.8h, v25.8h, v2.h[7]\n" + "fmla v20.8h, v25.8h, v3.h[7]\n" + "ldr q25, [x10, #0x70]\n" "add x10, x10, #0x80\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "fmla v13.8h, v7.8h, v1.h[7]\n" - "fmla v17.8h, v7.8h, v2.h[7]\n" - "fmla v21.8h, v7.8h, v3.h[7]\n" - "ldr q7, [x9, #0x70]\n" + "fmla v9.8h, v24.8h, v0.h[7]\n" + "fmla v13.8h, v24.8h, v1.h[7]\n" + "fmla v17.8h, v24.8h, v2.h[7]\n" + "fmla v21.8h, v24.8h, v3.h[7]\n" + "ldr q24, [x9, #0x70]\n" "add x9, x9, #0x80\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "fmla v14.8h, v6.8h, v1.h[7]\n" - "fmla v18.8h, v6.8h, v2.h[7]\n" - "fmla v22.8h, v6.8h, v3.h[7]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" - "fmla v15.8h, v7.8h, v1.h[7]\n" - "fmla v19.8h, v7.8h, v2.h[7]\n" - "fmla v23.8h, v7.8h, v3.h[7]\n" + "fmla v10.8h, v25.8h, v0.h[7]\n" + "fmla v14.8h, v25.8h, v1.h[7]\n" + "fmla v18.8h, v25.8h, v2.h[7]\n" + "fmla v22.8h, v25.8h, v3.h[7]\n" + "fmla v11.8h, v24.8h, v0.h[7]\n" + "fmla v15.8h, v24.8h, v1.h[7]\n" + "fmla v19.8h, v24.8h, v2.h[7]\n" + "fmla v23.8h, v24.8h, v3.h[7]\n" "179:" // Height 4: Multiply loop: Main loop skip "cbz x27, 181f\n" "180:" // Height 4: Multiply loop: Odd block loop - "ldr h0, [x26], #0x2\n" - "ldr h1, [x25], #0x2\n" + "ldr h3, [x26], #0x2\n" + "ldr h2, [x25], #0x2\n" "sub x27, x27, #0x1\n" - "ldr h2, [x24], #0x2\n" - "ldr h3, [x23], #0x2\n" - "ldr q6, [x12, #0x0]\n" - "ldr q7, [x11, #0x0]\n" - "fmla v8.8h, v6.8h, v0.h[0]\n" - "fmla v12.8h, v6.8h, v1.h[0]\n" - "fmla v16.8h, v6.8h, v2.h[0]\n" - "fmla v20.8h, v6.8h, v3.h[0]\n" - "ldr q6, [x10, #0x0]\n" + "ldr h1, [x24], #0x2\n" + "ldr h0, [x23], #0x2\n" + "ldr q25, [x12, #0x0]\n" + "ldr q24, [x11, #0x0]\n" + "fmla v8.8h, v25.8h, v3.h[0]\n" + "fmla v12.8h, v25.8h, v2.h[0]\n" + "fmla v16.8h, v25.8h, v1.h[0]\n" + "fmla v20.8h, v25.8h, v0.h[0]\n" + "ldr q25, [x10, #0x0]\n" "add x12, x12, #0x10\n" - "fmla v9.8h, v7.8h, v0.h[0]\n" - "fmla v13.8h, v7.8h, v1.h[0]\n" + "fmla v9.8h, v24.8h, v3.h[0]\n" + "fmla v13.8h, v24.8h, v2.h[0]\n" "add x11, x11, #0x10\n" "add x10, x10, #0x10\n" - "fmla v17.8h, v7.8h, v2.h[0]\n" - "fmla v21.8h, v7.8h, v3.h[0]\n" - "ldr q7, [x9, #0x0]\n" + "fmla v17.8h, v24.8h, v1.h[0]\n" + "fmla v21.8h, v24.8h, v0.h[0]\n" + "ldr q24, [x9, #0x0]\n" "add x9, x9, #0x10\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "fmla v22.8h, v6.8h, v3.h[0]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "fmla v23.8h, v7.8h, v3.h[0]\n" + "fmla v10.8h, v25.8h, v3.h[0]\n" + "fmla v14.8h, v25.8h, v2.h[0]\n" + "fmla v18.8h, v25.8h, v1.h[0]\n" + "fmla v22.8h, v25.8h, v0.h[0]\n" + "fmla v11.8h, v24.8h, v3.h[0]\n" + "fmla v15.8h, v24.8h, v2.h[0]\n" + "fmla v19.8h, v24.8h, v1.h[0]\n" + "fmla v23.8h, v24.8h, v0.h[0]\n" "cbnz x27, 180b\n" "181:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -2723,41 +2723,41 @@ void a64_ffhybrid_fp16_mla_6x32 ( "add x23, x24, x20, LSL #1\n" "tbz %x[flags], #1, 182f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.8h }, [x20]\n" + "ld1r { v25.8h }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.8h }, [x20]\n" - "fmin v8.8h, v8.8h, v1.8h\n" - "fmin v9.8h, v9.8h, v1.8h\n" - "fmin v10.8h, v10.8h, v1.8h\n" - "fmin v11.8h, v11.8h, v1.8h\n" - "fmin v12.8h, v12.8h, v1.8h\n" - "fmin v13.8h, v13.8h, v1.8h\n" - "fmin v14.8h, v14.8h, v1.8h\n" - "fmin v15.8h, v15.8h, v1.8h\n" - "fmin v16.8h, v16.8h, v1.8h\n" - "fmin v17.8h, v17.8h, v1.8h\n" - "fmin v18.8h, v18.8h, v1.8h\n" - "fmin v19.8h, v19.8h, v1.8h\n" - "fmin v20.8h, v20.8h, v1.8h\n" - "fmin v21.8h, v21.8h, v1.8h\n" - "fmin v22.8h, v22.8h, v1.8h\n" - "fmin v23.8h, v23.8h, v1.8h\n" - "fmax v8.8h, v8.8h, v0.8h\n" - "fmax v9.8h, v9.8h, v0.8h\n" - "fmax v10.8h, v10.8h, v0.8h\n" - "fmax v11.8h, v11.8h, v0.8h\n" - "fmax v12.8h, v12.8h, v0.8h\n" - "fmax v13.8h, v13.8h, v0.8h\n" - "fmax v14.8h, v14.8h, v0.8h\n" - "fmax v15.8h, v15.8h, v0.8h\n" - "fmax v16.8h, v16.8h, v0.8h\n" - "fmax v17.8h, v17.8h, v0.8h\n" - "fmax v18.8h, v18.8h, v0.8h\n" - "fmax v19.8h, v19.8h, v0.8h\n" - "fmax v20.8h, v20.8h, v0.8h\n" - "fmax v21.8h, v21.8h, v0.8h\n" - "fmax v22.8h, v22.8h, v0.8h\n" - "fmax v23.8h, v23.8h, v0.8h\n" + "ld1r { v24.8h }, [x20]\n" + "fmin v8.8h, v8.8h, v25.8h\n" + "fmin v9.8h, v9.8h, v25.8h\n" + "fmin v10.8h, v10.8h, v25.8h\n" + "fmin v11.8h, v11.8h, v25.8h\n" + "fmin v12.8h, v12.8h, v25.8h\n" + "fmin v13.8h, v13.8h, v25.8h\n" + "fmin v14.8h, v14.8h, v25.8h\n" + "fmin v15.8h, v15.8h, v25.8h\n" + "fmin v16.8h, v16.8h, v25.8h\n" + "fmin v17.8h, v17.8h, v25.8h\n" + "fmin v18.8h, v18.8h, v25.8h\n" + "fmin v19.8h, v19.8h, v25.8h\n" + "fmin v20.8h, v20.8h, v25.8h\n" + "fmin v21.8h, v21.8h, v25.8h\n" + "fmin v22.8h, v22.8h, v25.8h\n" + "fmin v23.8h, v23.8h, v25.8h\n" + "fmax v8.8h, v8.8h, v24.8h\n" + "fmax v9.8h, v9.8h, v24.8h\n" + "fmax v10.8h, v10.8h, v24.8h\n" + "fmax v11.8h, v11.8h, v24.8h\n" + "fmax v12.8h, v12.8h, v24.8h\n" + "fmax v13.8h, v13.8h, v24.8h\n" + "fmax v14.8h, v14.8h, v24.8h\n" + "fmax v15.8h, v15.8h, v24.8h\n" + "fmax v16.8h, v16.8h, v24.8h\n" + "fmax v17.8h, v17.8h, v24.8h\n" + "fmax v18.8h, v18.8h, v24.8h\n" + "fmax v19.8h, v19.8h, v24.8h\n" + "fmax v20.8h, v20.8h, v24.8h\n" + "fmax v21.8h, v21.8h, v24.8h\n" + "fmax v22.8h, v22.8h, v24.8h\n" + "fmax v23.8h, v23.8h, v24.8h\n" "182:" // Height 4: No activation "cmp x14, #0x20\n" "bge 199f\n" @@ -3256,549 +3256,549 @@ void a64_ffhybrid_fp16_mla_6x32 ( "ld1 { v12.h }[2], [x25]\n" "ld1 { v16.h }[2], [x24]\n" "ld1 { v20.h }[2], [x23]\n" - "ld1 { v24.h }[2], [x22]\n" - "b 220f\n" - "219:" // Height 5: Partial accumulate: partial_1_0 - "ldr h8, [x13, #0x0]\n" - "ldr h12, [x25, #0x0]\n" - "mov x20, #0x0\n" - "ldr h16, [x24, #0x0]\n" - "ldr h20, [x23, #0x0]\n" - "ldr h24, [x22, #0x0]\n" - "220:" // Height 5: Partial accumulate: Done - "sub x13, x13, x20\n" - "b 223f\n" - "221:" // Height 5: full accumulate - "ldr q8, [x13, #0x0]\n" - "ldr q9, [x13, #0x10]\n" - "ldr q10, [x13, #0x20]\n" - "ldr q11, [x13, #0x30]\n" - "ldr q12, [x25, #0x0]\n" - "ldr q13, [x25, #0x10]\n" - "ldr q14, [x25, #0x20]\n" - "ldr q15, [x25, #0x30]\n" - "ldr q16, [x24, #0x0]\n" - "ldr q17, [x24, #0x10]\n" - "ldr q18, [x24, #0x20]\n" - "ldr q19, [x24, #0x30]\n" - "ldr q20, [x23, #0x0]\n" - "ldr q21, [x23, #0x10]\n" - "ldr q22, [x23, #0x20]\n" - "ldr q23, [x23, #0x30]\n" - "ldr q24, [x22, #0x0]\n" - "ldr q25, [x22, #0x10]\n" - "ldr q26, [x22, #0x20]\n" - "ldr q27, [x22, #0x30]\n" - "b 223f\n" - "222:" // Height 5: no accumulate - "movi v8.16b, #0x0\n" - "movi v9.16b, #0x0\n" - "movi v10.16b, #0x0\n" - "movi v11.16b, #0x0\n" - "movi v12.16b, #0x0\n" - "movi v13.16b, #0x0\n" - "movi v14.16b, #0x0\n" - "movi v15.16b, #0x0\n" - "movi v16.16b, #0x0\n" - "movi v17.16b, #0x0\n" - "movi v18.16b, #0x0\n" - "movi v19.16b, #0x0\n" - "movi v20.16b, #0x0\n" - "movi v21.16b, #0x0\n" - "movi v22.16b, #0x0\n" - "movi v23.16b, #0x0\n" - "movi v24.16b, #0x0\n" - "movi v25.16b, #0x0\n" - "movi v26.16b, #0x0\n" - "movi v27.16b, #0x0\n" - "223:" // Height 5: setup done - "mov x28, #0x0\n" - "224:" // Height 5: String loop - "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" - "tbz %x[flags], #3, 225f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "cbnz x28, 226f\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x26, x26, x20, LSL #1\n" - "add x25, x25, x20, LSL #1\n" - "add x24, x24, x20, LSL #1\n" - "add x23, x23, x20, LSL #1\n" - "add x22, x22, x20, LSL #1\n" - "b 226f\n" - "225:" // Height 5: setup direct input - "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" - "add x22, x23, x20, LSL #1\n" - "226:" // Height 5: input setup done - "cmp x27, #0x8\n" - "blt 229f\n" - "ldr q0, [x26, #0x0]\n" - "ldr q1, [x25, #0x0]\n" - "cmp x27, #0x10\n" - "ldr q2, [x24, #0x0]\n" - "ldr q3, [x23, #0x0]\n" - "ldr q4, [x22, #0x0]\n" - "ldr q6, [x12, #0x0]\n" - "ldr q7, [x11, #0x0]\n" - "blt 228f\n" - "227:" // Height 5: Multiply loop: Main loop head - "fmla v8.8h, v6.8h, v0.h[0]\n" - "fmla v12.8h, v6.8h, v1.h[0]\n" - "sub x27, x27, #0x8\n" - "cmp x27, #0x10\n" - "fmla v16.8h, v6.8h, v2.h[0]\n" - "fmla v20.8h, v6.8h, v3.h[0]\n" - "add x26, x26, #0x10\n" - "add x25, x25, #0x10\n" - "fmla v24.8h, v6.8h, v4.h[0]\n" - "ldr q6, [x10, #0x0]\n" - "fmla v9.8h, v7.8h, v0.h[0]\n" - "add x24, x24, #0x10\n" - "fmla v13.8h, v7.8h, v1.h[0]\n" - "fmla v17.8h, v7.8h, v2.h[0]\n" - "add x23, x23, #0x10\n" - "add x22, x22, #0x10\n" - "fmla v21.8h, v7.8h, v3.h[0]\n" - "fmla v25.8h, v7.8h, v4.h[0]\n" - "ldr q7, [x9, #0x0]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "fmla v22.8h, v6.8h, v3.h[0]\n" - "fmla v26.8h, v6.8h, v4.h[0]\n" - "ldr q6, [x12, #0x10]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "fmla v23.8h, v7.8h, v3.h[0]\n" - "fmla v27.8h, v7.8h, v4.h[0]\n" - "ldr q7, [x11, #0x10]\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "fmla v16.8h, v6.8h, v2.h[1]\n" - "fmla v20.8h, v6.8h, v3.h[1]\n" - "fmla v24.8h, v6.8h, v4.h[1]\n" - "ldr q6, [x10, #0x10]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "fmla v17.8h, v7.8h, v2.h[1]\n" - "fmla v21.8h, v7.8h, v3.h[1]\n" - "fmla v25.8h, v7.8h, v4.h[1]\n" - "ldr q7, [x9, #0x10]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "fmla v14.8h, v6.8h, v1.h[1]\n" - "fmla v18.8h, v6.8h, v2.h[1]\n" - "fmla v22.8h, v6.8h, v3.h[1]\n" - "fmla v26.8h, v6.8h, v4.h[1]\n" - "ldr q6, [x12, #0x20]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "fmla v15.8h, v7.8h, v1.h[1]\n" - "fmla v19.8h, v7.8h, v2.h[1]\n" - "fmla v23.8h, v7.8h, v3.h[1]\n" - "fmla v27.8h, v7.8h, v4.h[1]\n" - "ldr q7, [x11, #0x20]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "fmla v12.8h, v6.8h, v1.h[2]\n" - "fmla v16.8h, v6.8h, v2.h[2]\n" - "fmla v20.8h, v6.8h, v3.h[2]\n" - "fmla v24.8h, v6.8h, v4.h[2]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "fmla v13.8h, v7.8h, v1.h[2]\n" - "fmla v17.8h, v7.8h, v2.h[2]\n" - "fmla v21.8h, v7.8h, v3.h[2]\n" - "fmla v25.8h, v7.8h, v4.h[2]\n" - "ldr q7, [x9, #0x20]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "fmla v14.8h, v6.8h, v1.h[2]\n" - "fmla v18.8h, v6.8h, v2.h[2]\n" - "fmla v22.8h, v6.8h, v3.h[2]\n" - "fmla v26.8h, v6.8h, v4.h[2]\n" - "ldr q6, [x12, #0x30]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "fmla v15.8h, v7.8h, v1.h[2]\n" - "fmla v19.8h, v7.8h, v2.h[2]\n" - "fmla v23.8h, v7.8h, v3.h[2]\n" - "fmla v27.8h, v7.8h, v4.h[2]\n" - "ldr q7, [x11, #0x30]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "fmla v12.8h, v6.8h, v1.h[3]\n" - "fmla v16.8h, v6.8h, v2.h[3]\n" - "fmla v20.8h, v6.8h, v3.h[3]\n" - "fmla v24.8h, v6.8h, v4.h[3]\n" - "ldr q6, [x10, #0x30]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "fmla v13.8h, v7.8h, v1.h[3]\n" - "fmla v17.8h, v7.8h, v2.h[3]\n" - "fmla v21.8h, v7.8h, v3.h[3]\n" - "fmla v25.8h, v7.8h, v4.h[3]\n" - "ldr q7, [x9, #0x30]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "fmla v14.8h, v6.8h, v1.h[3]\n" - "fmla v18.8h, v6.8h, v2.h[3]\n" - "fmla v22.8h, v6.8h, v3.h[3]\n" - "fmla v26.8h, v6.8h, v4.h[3]\n" - "ldr q6, [x12, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "fmla v15.8h, v7.8h, v1.h[3]\n" - "fmla v19.8h, v7.8h, v2.h[3]\n" - "fmla v23.8h, v7.8h, v3.h[3]\n" - "fmla v27.8h, v7.8h, v4.h[3]\n" - "ldr q7, [x11, #0x40]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "fmla v12.8h, v6.8h, v1.h[4]\n" - "fmla v16.8h, v6.8h, v2.h[4]\n" - "fmla v20.8h, v6.8h, v3.h[4]\n" - "fmla v24.8h, v6.8h, v4.h[4]\n" - "ldr q6, [x10, #0x40]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "fmla v13.8h, v7.8h, v1.h[4]\n" - "fmla v17.8h, v7.8h, v2.h[4]\n" - "fmla v21.8h, v7.8h, v3.h[4]\n" - "fmla v25.8h, v7.8h, v4.h[4]\n" - "ldr q7, [x9, #0x40]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "fmla v14.8h, v6.8h, v1.h[4]\n" - "fmla v18.8h, v6.8h, v2.h[4]\n" - "fmla v22.8h, v6.8h, v3.h[4]\n" - "fmla v26.8h, v6.8h, v4.h[4]\n" - "ldr q6, [x12, #0x50]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "fmla v15.8h, v7.8h, v1.h[4]\n" - "fmla v19.8h, v7.8h, v2.h[4]\n" - "fmla v23.8h, v7.8h, v3.h[4]\n" - "fmla v27.8h, v7.8h, v4.h[4]\n" - "ldr q7, [x11, #0x50]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "fmla v12.8h, v6.8h, v1.h[5]\n" - "fmla v16.8h, v6.8h, v2.h[5]\n" - "fmla v20.8h, v6.8h, v3.h[5]\n" - "fmla v24.8h, v6.8h, v4.h[5]\n" - "ldr q6, [x10, #0x50]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "fmla v13.8h, v7.8h, v1.h[5]\n" - "fmla v17.8h, v7.8h, v2.h[5]\n" - "fmla v21.8h, v7.8h, v3.h[5]\n" - "fmla v25.8h, v7.8h, v4.h[5]\n" - "ldr q7, [x9, #0x50]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "fmla v14.8h, v6.8h, v1.h[5]\n" - "fmla v18.8h, v6.8h, v2.h[5]\n" - "fmla v22.8h, v6.8h, v3.h[5]\n" - "fmla v26.8h, v6.8h, v4.h[5]\n" - "ldr q6, [x12, #0x60]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "fmla v15.8h, v7.8h, v1.h[5]\n" - "fmla v19.8h, v7.8h, v2.h[5]\n" - "fmla v23.8h, v7.8h, v3.h[5]\n" - "fmla v27.8h, v7.8h, v4.h[5]\n" - "ldr q7, [x11, #0x60]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "fmla v12.8h, v6.8h, v1.h[6]\n" - "fmla v16.8h, v6.8h, v2.h[6]\n" - "fmla v20.8h, v6.8h, v3.h[6]\n" - "fmla v24.8h, v6.8h, v4.h[6]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "fmla v13.8h, v7.8h, v1.h[6]\n" - "fmla v17.8h, v7.8h, v2.h[6]\n" - "fmla v21.8h, v7.8h, v3.h[6]\n" - "fmla v25.8h, v7.8h, v4.h[6]\n" - "ldr q7, [x9, #0x60]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "fmla v14.8h, v6.8h, v1.h[6]\n" - "fmla v18.8h, v6.8h, v2.h[6]\n" - "fmla v22.8h, v6.8h, v3.h[6]\n" - "fmla v26.8h, v6.8h, v4.h[6]\n" - "ldr q6, [x12, #0x70]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "add x12, x12, #0x80\n" - "fmla v15.8h, v7.8h, v1.h[6]\n" - "fmla v19.8h, v7.8h, v2.h[6]\n" - "fmla v23.8h, v7.8h, v3.h[6]\n" - "fmla v27.8h, v7.8h, v4.h[6]\n" - "ldr q7, [x11, #0x70]\n" - "add x11, x11, #0x80\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "fmla v12.8h, v6.8h, v1.h[7]\n" - "fmla v16.8h, v6.8h, v2.h[7]\n" - "fmla v20.8h, v6.8h, v3.h[7]\n" - "fmla v24.8h, v6.8h, v4.h[7]\n" - "ldr q6, [x10, #0x70]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "add x10, x10, #0x80\n" - "fmla v13.8h, v7.8h, v1.h[7]\n" - "fmla v17.8h, v7.8h, v2.h[7]\n" - "fmla v21.8h, v7.8h, v3.h[7]\n" - "fmla v25.8h, v7.8h, v4.h[7]\n" - "ldr q7, [x9, #0x70]\n" - "add x9, x9, #0x80\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "fmla v14.8h, v6.8h, v1.h[7]\n" - "fmla v18.8h, v6.8h, v2.h[7]\n" - "fmla v22.8h, v6.8h, v3.h[7]\n" - "fmla v26.8h, v6.8h, v4.h[7]\n" - "ldr q6, [x12, #0x0]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" - "ldr q0, [x26, #0x0]\n" - "fmla v15.8h, v7.8h, v1.h[7]\n" - "ldr q1, [x25, #0x0]\n" - "fmla v19.8h, v7.8h, v2.h[7]\n" - "ldr q2, [x24, #0x0]\n" - "fmla v23.8h, v7.8h, v3.h[7]\n" - "ldr q3, [x23, #0x0]\n" - "fmla v27.8h, v7.8h, v4.h[7]\n" - "ldr q4, [x22, #0x0]\n" - "ldr q7, [x11, #0x0]\n" - "bge 227b\n" - "228:" // Height 5: Multiply loop: Single iteration only - "fmla v8.8h, v6.8h, v0.h[0]\n" - "fmla v12.8h, v6.8h, v1.h[0]\n" - "sub x27, x27, #0x8\n" - "add x26, x26, #0x10\n" - "fmla v16.8h, v6.8h, v2.h[0]\n" - "fmla v20.8h, v6.8h, v3.h[0]\n" - "add x25, x25, #0x10\n" - "add x24, x24, #0x10\n" - "fmla v24.8h, v6.8h, v4.h[0]\n" - "ldr q6, [x10, #0x0]\n" - "fmla v9.8h, v7.8h, v0.h[0]\n" - "add x23, x23, #0x10\n" - "fmla v13.8h, v7.8h, v1.h[0]\n" - "fmla v17.8h, v7.8h, v2.h[0]\n" - "add x22, x22, #0x10\n" - "fmla v21.8h, v7.8h, v3.h[0]\n" - "fmla v25.8h, v7.8h, v4.h[0]\n" - "ldr q7, [x9, #0x0]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "fmla v22.8h, v6.8h, v3.h[0]\n" - "fmla v26.8h, v6.8h, v4.h[0]\n" - "ldr q6, [x12, #0x10]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "fmla v23.8h, v7.8h, v3.h[0]\n" - "fmla v27.8h, v7.8h, v4.h[0]\n" - "ldr q7, [x11, #0x10]\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "fmla v16.8h, v6.8h, v2.h[1]\n" - "fmla v20.8h, v6.8h, v3.h[1]\n" - "fmla v24.8h, v6.8h, v4.h[1]\n" - "ldr q6, [x10, #0x10]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "fmla v17.8h, v7.8h, v2.h[1]\n" - "fmla v21.8h, v7.8h, v3.h[1]\n" - "fmla v25.8h, v7.8h, v4.h[1]\n" - "ldr q7, [x9, #0x10]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "fmla v14.8h, v6.8h, v1.h[1]\n" - "fmla v18.8h, v6.8h, v2.h[1]\n" - "fmla v22.8h, v6.8h, v3.h[1]\n" - "fmla v26.8h, v6.8h, v4.h[1]\n" - "ldr q6, [x12, #0x20]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "fmla v15.8h, v7.8h, v1.h[1]\n" - "fmla v19.8h, v7.8h, v2.h[1]\n" - "fmla v23.8h, v7.8h, v3.h[1]\n" - "fmla v27.8h, v7.8h, v4.h[1]\n" - "ldr q7, [x11, #0x20]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "fmla v12.8h, v6.8h, v1.h[2]\n" - "fmla v16.8h, v6.8h, v2.h[2]\n" - "fmla v20.8h, v6.8h, v3.h[2]\n" - "fmla v24.8h, v6.8h, v4.h[2]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "fmla v13.8h, v7.8h, v1.h[2]\n" - "fmla v17.8h, v7.8h, v2.h[2]\n" - "fmla v21.8h, v7.8h, v3.h[2]\n" - "fmla v25.8h, v7.8h, v4.h[2]\n" - "ldr q7, [x9, #0x20]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "fmla v14.8h, v6.8h, v1.h[2]\n" - "fmla v18.8h, v6.8h, v2.h[2]\n" - "fmla v22.8h, v6.8h, v3.h[2]\n" - "fmla v26.8h, v6.8h, v4.h[2]\n" - "ldr q6, [x12, #0x30]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "fmla v15.8h, v7.8h, v1.h[2]\n" - "fmla v19.8h, v7.8h, v2.h[2]\n" - "fmla v23.8h, v7.8h, v3.h[2]\n" - "fmla v27.8h, v7.8h, v4.h[2]\n" - "ldr q7, [x11, #0x30]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "fmla v12.8h, v6.8h, v1.h[3]\n" - "fmla v16.8h, v6.8h, v2.h[3]\n" - "fmla v20.8h, v6.8h, v3.h[3]\n" - "fmla v24.8h, v6.8h, v4.h[3]\n" - "ldr q6, [x10, #0x30]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "fmla v13.8h, v7.8h, v1.h[3]\n" - "fmla v17.8h, v7.8h, v2.h[3]\n" - "fmla v21.8h, v7.8h, v3.h[3]\n" - "fmla v25.8h, v7.8h, v4.h[3]\n" - "ldr q7, [x9, #0x30]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "fmla v14.8h, v6.8h, v1.h[3]\n" - "fmla v18.8h, v6.8h, v2.h[3]\n" - "fmla v22.8h, v6.8h, v3.h[3]\n" - "fmla v26.8h, v6.8h, v4.h[3]\n" - "ldr q6, [x12, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "fmla v15.8h, v7.8h, v1.h[3]\n" - "fmla v19.8h, v7.8h, v2.h[3]\n" - "fmla v23.8h, v7.8h, v3.h[3]\n" - "fmla v27.8h, v7.8h, v4.h[3]\n" - "ldr q7, [x11, #0x40]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "fmla v12.8h, v6.8h, v1.h[4]\n" - "fmla v16.8h, v6.8h, v2.h[4]\n" - "fmla v20.8h, v6.8h, v3.h[4]\n" - "fmla v24.8h, v6.8h, v4.h[4]\n" - "ldr q6, [x10, #0x40]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "fmla v13.8h, v7.8h, v1.h[4]\n" - "fmla v17.8h, v7.8h, v2.h[4]\n" - "fmla v21.8h, v7.8h, v3.h[4]\n" - "fmla v25.8h, v7.8h, v4.h[4]\n" - "ldr q7, [x9, #0x40]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "fmla v14.8h, v6.8h, v1.h[4]\n" - "fmla v18.8h, v6.8h, v2.h[4]\n" - "fmla v22.8h, v6.8h, v3.h[4]\n" - "fmla v26.8h, v6.8h, v4.h[4]\n" - "ldr q6, [x12, #0x50]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "fmla v15.8h, v7.8h, v1.h[4]\n" - "fmla v19.8h, v7.8h, v2.h[4]\n" - "fmla v23.8h, v7.8h, v3.h[4]\n" - "fmla v27.8h, v7.8h, v4.h[4]\n" - "ldr q7, [x11, #0x50]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "fmla v12.8h, v6.8h, v1.h[5]\n" - "fmla v16.8h, v6.8h, v2.h[5]\n" - "fmla v20.8h, v6.8h, v3.h[5]\n" - "fmla v24.8h, v6.8h, v4.h[5]\n" - "ldr q6, [x10, #0x50]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "fmla v13.8h, v7.8h, v1.h[5]\n" - "fmla v17.8h, v7.8h, v2.h[5]\n" - "fmla v21.8h, v7.8h, v3.h[5]\n" - "fmla v25.8h, v7.8h, v4.h[5]\n" - "ldr q7, [x9, #0x50]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "fmla v14.8h, v6.8h, v1.h[5]\n" - "fmla v18.8h, v6.8h, v2.h[5]\n" - "fmla v22.8h, v6.8h, v3.h[5]\n" - "fmla v26.8h, v6.8h, v4.h[5]\n" - "ldr q6, [x12, #0x60]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "fmla v15.8h, v7.8h, v1.h[5]\n" - "fmla v19.8h, v7.8h, v2.h[5]\n" - "fmla v23.8h, v7.8h, v3.h[5]\n" - "fmla v27.8h, v7.8h, v4.h[5]\n" - "ldr q7, [x11, #0x60]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "fmla v12.8h, v6.8h, v1.h[6]\n" - "fmla v16.8h, v6.8h, v2.h[6]\n" - "fmla v20.8h, v6.8h, v3.h[6]\n" - "fmla v24.8h, v6.8h, v4.h[6]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "fmla v13.8h, v7.8h, v1.h[6]\n" - "fmla v17.8h, v7.8h, v2.h[6]\n" - "fmla v21.8h, v7.8h, v3.h[6]\n" - "fmla v25.8h, v7.8h, v4.h[6]\n" - "ldr q7, [x9, #0x60]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "fmla v14.8h, v6.8h, v1.h[6]\n" - "fmla v18.8h, v6.8h, v2.h[6]\n" - "fmla v22.8h, v6.8h, v3.h[6]\n" - "fmla v26.8h, v6.8h, v4.h[6]\n" - "ldr q6, [x12, #0x70]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" + "ld1 { v24.h }[2], [x22]\n" + "b 220f\n" + "219:" // Height 5: Partial accumulate: partial_1_0 + "ldr h8, [x13, #0x0]\n" + "ldr h12, [x25, #0x0]\n" + "mov x20, #0x0\n" + "ldr h16, [x24, #0x0]\n" + "ldr h20, [x23, #0x0]\n" + "ldr h24, [x22, #0x0]\n" + "220:" // Height 5: Partial accumulate: Done + "sub x13, x13, x20\n" + "b 223f\n" + "221:" // Height 5: full accumulate + "ldr q8, [x13, #0x0]\n" + "ldr q9, [x13, #0x10]\n" + "ldr q10, [x13, #0x20]\n" + "ldr q11, [x13, #0x30]\n" + "ldr q12, [x25, #0x0]\n" + "ldr q13, [x25, #0x10]\n" + "ldr q14, [x25, #0x20]\n" + "ldr q15, [x25, #0x30]\n" + "ldr q16, [x24, #0x0]\n" + "ldr q17, [x24, #0x10]\n" + "ldr q18, [x24, #0x20]\n" + "ldr q19, [x24, #0x30]\n" + "ldr q20, [x23, #0x0]\n" + "ldr q21, [x23, #0x10]\n" + "ldr q22, [x23, #0x20]\n" + "ldr q23, [x23, #0x30]\n" + "ldr q24, [x22, #0x0]\n" + "ldr q25, [x22, #0x10]\n" + "ldr q26, [x22, #0x20]\n" + "ldr q27, [x22, #0x30]\n" + "b 223f\n" + "222:" // Height 5: no accumulate + "movi v8.16b, #0x0\n" + "movi v9.16b, #0x0\n" + "movi v10.16b, #0x0\n" + "movi v11.16b, #0x0\n" + "movi v12.16b, #0x0\n" + "movi v13.16b, #0x0\n" + "movi v14.16b, #0x0\n" + "movi v15.16b, #0x0\n" + "movi v16.16b, #0x0\n" + "movi v17.16b, #0x0\n" + "movi v18.16b, #0x0\n" + "movi v19.16b, #0x0\n" + "movi v20.16b, #0x0\n" + "movi v21.16b, #0x0\n" + "movi v22.16b, #0x0\n" + "movi v23.16b, #0x0\n" + "movi v24.16b, #0x0\n" + "movi v25.16b, #0x0\n" + "movi v26.16b, #0x0\n" + "movi v27.16b, #0x0\n" + "223:" // Height 5: setup done + "mov x28, #0x0\n" + "224:" // Height 5: String loop + "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" + "ldr w27, [x20, x28, LSL #0x2]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" + "tbz %x[flags], #3, 225f\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "cbnz x28, 226f\n" + "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" + "add x26, x26, x20, LSL #1\n" + "add x25, x25, x20, LSL #1\n" + "add x24, x24, x20, LSL #1\n" + "add x23, x23, x20, LSL #1\n" + "add x22, x22, x20, LSL #1\n" + "b 226f\n" + "225:" // Height 5: setup direct input + "mov x26, %x[input_ptr]\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" + "add x22, x23, x21, LSL #1\n" + "226:" // Height 5: input setup done + "cmp x27, #0x8\n" + "blt 229f\n" + "ldr q0, [x26, #0x0]\n" + "ldr q1, [x25, #0x0]\n" + "cmp x27, #0x10\n" + "ldr q2, [x24, #0x0]\n" + "ldr q3, [x23, #0x0]\n" + "ldr q4, [x22, #0x0]\n" + "ldr q6, [x12, #0x0]\n" + "ldr q7, [x11, #0x0]\n" + "blt 228f\n" + "227:" // Height 5: Multiply loop: Main loop head + "fmla v8.8h, v6.8h, v0.h[0]\n" + "fmla v12.8h, v6.8h, v1.h[0]\n" + "sub x27, x27, #0x8\n" + "cmp x27, #0x10\n" + "fmla v16.8h, v6.8h, v2.h[0]\n" + "fmla v20.8h, v6.8h, v3.h[0]\n" + "add x26, x26, #0x10\n" + "add x25, x25, #0x10\n" + "fmla v24.8h, v6.8h, v4.h[0]\n" + "ldr q29, [x10, #0x0]\n" + "fmla v9.8h, v7.8h, v0.h[0]\n" + "add x24, x24, #0x10\n" + "fmla v13.8h, v7.8h, v1.h[0]\n" + "fmla v17.8h, v7.8h, v2.h[0]\n" + "add x23, x23, #0x10\n" + "add x22, x22, #0x10\n" + "fmla v21.8h, v7.8h, v3.h[0]\n" + "fmla v25.8h, v7.8h, v4.h[0]\n" + "ldr q28, [x9, #0x0]\n" + "fmla v10.8h, v29.8h, v0.h[0]\n" + "fmla v14.8h, v29.8h, v1.h[0]\n" + "fmla v18.8h, v29.8h, v2.h[0]\n" + "fmla v22.8h, v29.8h, v3.h[0]\n" + "fmla v26.8h, v29.8h, v4.h[0]\n" + "ldr q29, [x12, #0x10]\n" + "fmla v11.8h, v28.8h, v0.h[0]\n" + "fmla v15.8h, v28.8h, v1.h[0]\n" + "fmla v19.8h, v28.8h, v2.h[0]\n" + "fmla v23.8h, v28.8h, v3.h[0]\n" + "fmla v27.8h, v28.8h, v4.h[0]\n" + "ldr q28, [x11, #0x10]\n" + "fmla v8.8h, v29.8h, v0.h[1]\n" + "fmla v12.8h, v29.8h, v1.h[1]\n" + "fmla v16.8h, v29.8h, v2.h[1]\n" + "fmla v20.8h, v29.8h, v3.h[1]\n" + "fmla v24.8h, v29.8h, v4.h[1]\n" + "ldr q29, [x10, #0x10]\n" + "fmla v9.8h, v28.8h, v0.h[1]\n" + "fmla v13.8h, v28.8h, v1.h[1]\n" + "fmla v17.8h, v28.8h, v2.h[1]\n" + "fmla v21.8h, v28.8h, v3.h[1]\n" + "fmla v25.8h, v28.8h, v4.h[1]\n" + "ldr q28, [x9, #0x10]\n" + "fmla v10.8h, v29.8h, v0.h[1]\n" + "fmla v14.8h, v29.8h, v1.h[1]\n" + "fmla v18.8h, v29.8h, v2.h[1]\n" + "fmla v22.8h, v29.8h, v3.h[1]\n" + "fmla v26.8h, v29.8h, v4.h[1]\n" + "ldr q29, [x12, #0x20]\n" + "fmla v11.8h, v28.8h, v0.h[1]\n" + "fmla v15.8h, v28.8h, v1.h[1]\n" + "fmla v19.8h, v28.8h, v2.h[1]\n" + "fmla v23.8h, v28.8h, v3.h[1]\n" + "fmla v27.8h, v28.8h, v4.h[1]\n" + "ldr q28, [x11, #0x20]\n" + "fmla v8.8h, v29.8h, v0.h[2]\n" + "fmla v12.8h, v29.8h, v1.h[2]\n" + "fmla v16.8h, v29.8h, v2.h[2]\n" + "fmla v20.8h, v29.8h, v3.h[2]\n" + "fmla v24.8h, v29.8h, v4.h[2]\n" + "ldr q29, [x10, #0x20]\n" + "fmla v9.8h, v28.8h, v0.h[2]\n" + "fmla v13.8h, v28.8h, v1.h[2]\n" + "fmla v17.8h, v28.8h, v2.h[2]\n" + "fmla v21.8h, v28.8h, v3.h[2]\n" + "fmla v25.8h, v28.8h, v4.h[2]\n" + "ldr q28, [x9, #0x20]\n" + "fmla v10.8h, v29.8h, v0.h[2]\n" + "fmla v14.8h, v29.8h, v1.h[2]\n" + "fmla v18.8h, v29.8h, v2.h[2]\n" + "fmla v22.8h, v29.8h, v3.h[2]\n" + "fmla v26.8h, v29.8h, v4.h[2]\n" + "ldr q29, [x12, #0x30]\n" + "fmla v11.8h, v28.8h, v0.h[2]\n" + "fmla v15.8h, v28.8h, v1.h[2]\n" + "fmla v19.8h, v28.8h, v2.h[2]\n" + "fmla v23.8h, v28.8h, v3.h[2]\n" + "fmla v27.8h, v28.8h, v4.h[2]\n" + "ldr q28, [x11, #0x30]\n" + "fmla v8.8h, v29.8h, v0.h[3]\n" + "fmla v12.8h, v29.8h, v1.h[3]\n" + "fmla v16.8h, v29.8h, v2.h[3]\n" + "fmla v20.8h, v29.8h, v3.h[3]\n" + "fmla v24.8h, v29.8h, v4.h[3]\n" + "ldr q29, [x10, #0x30]\n" + "fmla v9.8h, v28.8h, v0.h[3]\n" + "fmla v13.8h, v28.8h, v1.h[3]\n" + "fmla v17.8h, v28.8h, v2.h[3]\n" + "fmla v21.8h, v28.8h, v3.h[3]\n" + "fmla v25.8h, v28.8h, v4.h[3]\n" + "ldr q28, [x9, #0x30]\n" + "fmla v10.8h, v29.8h, v0.h[3]\n" + "fmla v14.8h, v29.8h, v1.h[3]\n" + "fmla v18.8h, v29.8h, v2.h[3]\n" + "fmla v22.8h, v29.8h, v3.h[3]\n" + "fmla v26.8h, v29.8h, v4.h[3]\n" + "ldr q29, [x12, #0x40]\n" + "fmla v11.8h, v28.8h, v0.h[3]\n" + "fmla v15.8h, v28.8h, v1.h[3]\n" + "fmla v19.8h, v28.8h, v2.h[3]\n" + "fmla v23.8h, v28.8h, v3.h[3]\n" + "fmla v27.8h, v28.8h, v4.h[3]\n" + "ldr q28, [x11, #0x40]\n" + "fmla v8.8h, v29.8h, v0.h[4]\n" + "fmla v12.8h, v29.8h, v1.h[4]\n" + "fmla v16.8h, v29.8h, v2.h[4]\n" + "fmla v20.8h, v29.8h, v3.h[4]\n" + "fmla v24.8h, v29.8h, v4.h[4]\n" + "ldr q29, [x10, #0x40]\n" + "fmla v9.8h, v28.8h, v0.h[4]\n" + "fmla v13.8h, v28.8h, v1.h[4]\n" + "fmla v17.8h, v28.8h, v2.h[4]\n" + "fmla v21.8h, v28.8h, v3.h[4]\n" + "fmla v25.8h, v28.8h, v4.h[4]\n" + "ldr q28, [x9, #0x40]\n" + "fmla v10.8h, v29.8h, v0.h[4]\n" + "fmla v14.8h, v29.8h, v1.h[4]\n" + "fmla v18.8h, v29.8h, v2.h[4]\n" + "fmla v22.8h, v29.8h, v3.h[4]\n" + "fmla v26.8h, v29.8h, v4.h[4]\n" + "ldr q29, [x12, #0x50]\n" + "fmla v11.8h, v28.8h, v0.h[4]\n" + "fmla v15.8h, v28.8h, v1.h[4]\n" + "fmla v19.8h, v28.8h, v2.h[4]\n" + "fmla v23.8h, v28.8h, v3.h[4]\n" + "fmla v27.8h, v28.8h, v4.h[4]\n" + "ldr q28, [x11, #0x50]\n" + "fmla v8.8h, v29.8h, v0.h[5]\n" + "fmla v12.8h, v29.8h, v1.h[5]\n" + "fmla v16.8h, v29.8h, v2.h[5]\n" + "fmla v20.8h, v29.8h, v3.h[5]\n" + "fmla v24.8h, v29.8h, v4.h[5]\n" + "ldr q29, [x10, #0x50]\n" + "fmla v9.8h, v28.8h, v0.h[5]\n" + "fmla v13.8h, v28.8h, v1.h[5]\n" + "fmla v17.8h, v28.8h, v2.h[5]\n" + "fmla v21.8h, v28.8h, v3.h[5]\n" + "fmla v25.8h, v28.8h, v4.h[5]\n" + "ldr q28, [x9, #0x50]\n" + "fmla v10.8h, v29.8h, v0.h[5]\n" + "fmla v14.8h, v29.8h, v1.h[5]\n" + "fmla v18.8h, v29.8h, v2.h[5]\n" + "fmla v22.8h, v29.8h, v3.h[5]\n" + "fmla v26.8h, v29.8h, v4.h[5]\n" + "ldr q29, [x12, #0x60]\n" + "fmla v11.8h, v28.8h, v0.h[5]\n" + "fmla v15.8h, v28.8h, v1.h[5]\n" + "fmla v19.8h, v28.8h, v2.h[5]\n" + "fmla v23.8h, v28.8h, v3.h[5]\n" + "fmla v27.8h, v28.8h, v4.h[5]\n" + "ldr q28, [x11, #0x60]\n" + "fmla v8.8h, v29.8h, v0.h[6]\n" + "fmla v12.8h, v29.8h, v1.h[6]\n" + "fmla v16.8h, v29.8h, v2.h[6]\n" + "fmla v20.8h, v29.8h, v3.h[6]\n" + "fmla v24.8h, v29.8h, v4.h[6]\n" + "ldr q29, [x10, #0x60]\n" + "fmla v9.8h, v28.8h, v0.h[6]\n" + "fmla v13.8h, v28.8h, v1.h[6]\n" + "fmla v17.8h, v28.8h, v2.h[6]\n" + "fmla v21.8h, v28.8h, v3.h[6]\n" + "fmla v25.8h, v28.8h, v4.h[6]\n" + "ldr q28, [x9, #0x60]\n" + "fmla v10.8h, v29.8h, v0.h[6]\n" + "fmla v14.8h, v29.8h, v1.h[6]\n" + "fmla v18.8h, v29.8h, v2.h[6]\n" + "fmla v22.8h, v29.8h, v3.h[6]\n" + "fmla v26.8h, v29.8h, v4.h[6]\n" + "ldr q29, [x12, #0x70]\n" + "fmla v11.8h, v28.8h, v0.h[6]\n" "add x12, x12, #0x80\n" - "fmla v15.8h, v7.8h, v1.h[6]\n" - "fmla v19.8h, v7.8h, v2.h[6]\n" - "fmla v23.8h, v7.8h, v3.h[6]\n" - "fmla v27.8h, v7.8h, v4.h[6]\n" - "ldr q7, [x11, #0x70]\n" + "fmla v15.8h, v28.8h, v1.h[6]\n" + "fmla v19.8h, v28.8h, v2.h[6]\n" + "fmla v23.8h, v28.8h, v3.h[6]\n" + "fmla v27.8h, v28.8h, v4.h[6]\n" + "ldr q28, [x11, #0x70]\n" "add x11, x11, #0x80\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "fmla v12.8h, v6.8h, v1.h[7]\n" - "fmla v16.8h, v6.8h, v2.h[7]\n" - "fmla v20.8h, v6.8h, v3.h[7]\n" - "fmla v24.8h, v6.8h, v4.h[7]\n" - "ldr q6, [x10, #0x70]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" + "fmla v8.8h, v29.8h, v0.h[7]\n" + "fmla v12.8h, v29.8h, v1.h[7]\n" + "fmla v16.8h, v29.8h, v2.h[7]\n" + "fmla v20.8h, v29.8h, v3.h[7]\n" + "fmla v24.8h, v29.8h, v4.h[7]\n" + "ldr q29, [x10, #0x70]\n" + "fmla v9.8h, v28.8h, v0.h[7]\n" "add x10, x10, #0x80\n" - "fmla v13.8h, v7.8h, v1.h[7]\n" - "fmla v17.8h, v7.8h, v2.h[7]\n" - "fmla v21.8h, v7.8h, v3.h[7]\n" - "fmla v25.8h, v7.8h, v4.h[7]\n" - "ldr q7, [x9, #0x70]\n" + "fmla v13.8h, v28.8h, v1.h[7]\n" + "fmla v17.8h, v28.8h, v2.h[7]\n" + "fmla v21.8h, v28.8h, v3.h[7]\n" + "fmla v25.8h, v28.8h, v4.h[7]\n" + "ldr q28, [x9, #0x70]\n" "add x9, x9, #0x80\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "fmla v14.8h, v6.8h, v1.h[7]\n" - "fmla v18.8h, v6.8h, v2.h[7]\n" - "fmla v22.8h, v6.8h, v3.h[7]\n" - "fmla v26.8h, v6.8h, v4.h[7]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" - "fmla v15.8h, v7.8h, v1.h[7]\n" - "fmla v19.8h, v7.8h, v2.h[7]\n" - "fmla v23.8h, v7.8h, v3.h[7]\n" - "fmla v27.8h, v7.8h, v4.h[7]\n" - "229:" // Height 5: Multiply loop: Main loop skip - "cbz x27, 231f\n" - "230:" // Height 5: Multiply loop: Odd block loop - "ldr h0, [x26], #0x2\n" - "ldr h1, [x25], #0x2\n" - "sub x27, x27, #0x1\n" - "ldr h2, [x24], #0x2\n" - "ldr h3, [x23], #0x2\n" - "ldr h4, [x22], #0x2\n" + "fmla v10.8h, v29.8h, v0.h[7]\n" + "fmla v14.8h, v29.8h, v1.h[7]\n" + "fmla v18.8h, v29.8h, v2.h[7]\n" + "fmla v22.8h, v29.8h, v3.h[7]\n" + "fmla v26.8h, v29.8h, v4.h[7]\n" "ldr q6, [x12, #0x0]\n" + "fmla v11.8h, v28.8h, v0.h[7]\n" + "ldr q0, [x26, #0x0]\n" + "fmla v15.8h, v28.8h, v1.h[7]\n" + "ldr q1, [x25, #0x0]\n" + "fmla v19.8h, v28.8h, v2.h[7]\n" + "ldr q2, [x24, #0x0]\n" + "fmla v23.8h, v28.8h, v3.h[7]\n" + "ldr q3, [x23, #0x0]\n" + "fmla v27.8h, v28.8h, v4.h[7]\n" + "ldr q4, [x22, #0x0]\n" + "ldr q7, [x11, #0x0]\n" + "bge 227b\n" + "228:" // Height 5: Multiply loop: Single iteration only "fmla v8.8h, v6.8h, v0.h[0]\n" "fmla v12.8h, v6.8h, v1.h[0]\n" - "ldr q7, [x11, #0x0]\n" + "sub x27, x27, #0x8\n" + "add x26, x26, #0x10\n" "fmla v16.8h, v6.8h, v2.h[0]\n" "fmla v20.8h, v6.8h, v3.h[0]\n" - "add x12, x12, #0x10\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" "fmla v24.8h, v6.8h, v4.h[0]\n" - "ldr q6, [x10, #0x0]\n" + "ldr q29, [x10, #0x0]\n" "fmla v9.8h, v7.8h, v0.h[0]\n" - "add x11, x11, #0x10\n" + "add x23, x23, #0x10\n" "fmla v13.8h, v7.8h, v1.h[0]\n" "fmla v17.8h, v7.8h, v2.h[0]\n" - "add x10, x10, #0x10\n" + "add x22, x22, #0x10\n" "fmla v21.8h, v7.8h, v3.h[0]\n" "fmla v25.8h, v7.8h, v4.h[0]\n" - "ldr q7, [x9, #0x0]\n" + "ldr q28, [x9, #0x0]\n" + "fmla v10.8h, v29.8h, v0.h[0]\n" + "fmla v14.8h, v29.8h, v1.h[0]\n" + "fmla v18.8h, v29.8h, v2.h[0]\n" + "fmla v22.8h, v29.8h, v3.h[0]\n" + "fmla v26.8h, v29.8h, v4.h[0]\n" + "ldr q29, [x12, #0x10]\n" + "fmla v11.8h, v28.8h, v0.h[0]\n" + "fmla v15.8h, v28.8h, v1.h[0]\n" + "fmla v19.8h, v28.8h, v2.h[0]\n" + "fmla v23.8h, v28.8h, v3.h[0]\n" + "fmla v27.8h, v28.8h, v4.h[0]\n" + "ldr q28, [x11, #0x10]\n" + "fmla v8.8h, v29.8h, v0.h[1]\n" + "fmla v12.8h, v29.8h, v1.h[1]\n" + "fmla v16.8h, v29.8h, v2.h[1]\n" + "fmla v20.8h, v29.8h, v3.h[1]\n" + "fmla v24.8h, v29.8h, v4.h[1]\n" + "ldr q29, [x10, #0x10]\n" + "fmla v9.8h, v28.8h, v0.h[1]\n" + "fmla v13.8h, v28.8h, v1.h[1]\n" + "fmla v17.8h, v28.8h, v2.h[1]\n" + "fmla v21.8h, v28.8h, v3.h[1]\n" + "fmla v25.8h, v28.8h, v4.h[1]\n" + "ldr q28, [x9, #0x10]\n" + "fmla v10.8h, v29.8h, v0.h[1]\n" + "fmla v14.8h, v29.8h, v1.h[1]\n" + "fmla v18.8h, v29.8h, v2.h[1]\n" + "fmla v22.8h, v29.8h, v3.h[1]\n" + "fmla v26.8h, v29.8h, v4.h[1]\n" + "ldr q29, [x12, #0x20]\n" + "fmla v11.8h, v28.8h, v0.h[1]\n" + "fmla v15.8h, v28.8h, v1.h[1]\n" + "fmla v19.8h, v28.8h, v2.h[1]\n" + "fmla v23.8h, v28.8h, v3.h[1]\n" + "fmla v27.8h, v28.8h, v4.h[1]\n" + "ldr q28, [x11, #0x20]\n" + "fmla v8.8h, v29.8h, v0.h[2]\n" + "fmla v12.8h, v29.8h, v1.h[2]\n" + "fmla v16.8h, v29.8h, v2.h[2]\n" + "fmla v20.8h, v29.8h, v3.h[2]\n" + "fmla v24.8h, v29.8h, v4.h[2]\n" + "ldr q29, [x10, #0x20]\n" + "fmla v9.8h, v28.8h, v0.h[2]\n" + "fmla v13.8h, v28.8h, v1.h[2]\n" + "fmla v17.8h, v28.8h, v2.h[2]\n" + "fmla v21.8h, v28.8h, v3.h[2]\n" + "fmla v25.8h, v28.8h, v4.h[2]\n" + "ldr q28, [x9, #0x20]\n" + "fmla v10.8h, v29.8h, v0.h[2]\n" + "fmla v14.8h, v29.8h, v1.h[2]\n" + "fmla v18.8h, v29.8h, v2.h[2]\n" + "fmla v22.8h, v29.8h, v3.h[2]\n" + "fmla v26.8h, v29.8h, v4.h[2]\n" + "ldr q29, [x12, #0x30]\n" + "fmla v11.8h, v28.8h, v0.h[2]\n" + "fmla v15.8h, v28.8h, v1.h[2]\n" + "fmla v19.8h, v28.8h, v2.h[2]\n" + "fmla v23.8h, v28.8h, v3.h[2]\n" + "fmla v27.8h, v28.8h, v4.h[2]\n" + "ldr q28, [x11, #0x30]\n" + "fmla v8.8h, v29.8h, v0.h[3]\n" + "fmla v12.8h, v29.8h, v1.h[3]\n" + "fmla v16.8h, v29.8h, v2.h[3]\n" + "fmla v20.8h, v29.8h, v3.h[3]\n" + "fmla v24.8h, v29.8h, v4.h[3]\n" + "ldr q29, [x10, #0x30]\n" + "fmla v9.8h, v28.8h, v0.h[3]\n" + "fmla v13.8h, v28.8h, v1.h[3]\n" + "fmla v17.8h, v28.8h, v2.h[3]\n" + "fmla v21.8h, v28.8h, v3.h[3]\n" + "fmla v25.8h, v28.8h, v4.h[3]\n" + "ldr q28, [x9, #0x30]\n" + "fmla v10.8h, v29.8h, v0.h[3]\n" + "fmla v14.8h, v29.8h, v1.h[3]\n" + "fmla v18.8h, v29.8h, v2.h[3]\n" + "fmla v22.8h, v29.8h, v3.h[3]\n" + "fmla v26.8h, v29.8h, v4.h[3]\n" + "ldr q29, [x12, #0x40]\n" + "fmla v11.8h, v28.8h, v0.h[3]\n" + "fmla v15.8h, v28.8h, v1.h[3]\n" + "fmla v19.8h, v28.8h, v2.h[3]\n" + "fmla v23.8h, v28.8h, v3.h[3]\n" + "fmla v27.8h, v28.8h, v4.h[3]\n" + "ldr q28, [x11, #0x40]\n" + "fmla v8.8h, v29.8h, v0.h[4]\n" + "fmla v12.8h, v29.8h, v1.h[4]\n" + "fmla v16.8h, v29.8h, v2.h[4]\n" + "fmla v20.8h, v29.8h, v3.h[4]\n" + "fmla v24.8h, v29.8h, v4.h[4]\n" + "ldr q29, [x10, #0x40]\n" + "fmla v9.8h, v28.8h, v0.h[4]\n" + "fmla v13.8h, v28.8h, v1.h[4]\n" + "fmla v17.8h, v28.8h, v2.h[4]\n" + "fmla v21.8h, v28.8h, v3.h[4]\n" + "fmla v25.8h, v28.8h, v4.h[4]\n" + "ldr q28, [x9, #0x40]\n" + "fmla v10.8h, v29.8h, v0.h[4]\n" + "fmla v14.8h, v29.8h, v1.h[4]\n" + "fmla v18.8h, v29.8h, v2.h[4]\n" + "fmla v22.8h, v29.8h, v3.h[4]\n" + "fmla v26.8h, v29.8h, v4.h[4]\n" + "ldr q29, [x12, #0x50]\n" + "fmla v11.8h, v28.8h, v0.h[4]\n" + "fmla v15.8h, v28.8h, v1.h[4]\n" + "fmla v19.8h, v28.8h, v2.h[4]\n" + "fmla v23.8h, v28.8h, v3.h[4]\n" + "fmla v27.8h, v28.8h, v4.h[4]\n" + "ldr q28, [x11, #0x50]\n" + "fmla v8.8h, v29.8h, v0.h[5]\n" + "fmla v12.8h, v29.8h, v1.h[5]\n" + "fmla v16.8h, v29.8h, v2.h[5]\n" + "fmla v20.8h, v29.8h, v3.h[5]\n" + "fmla v24.8h, v29.8h, v4.h[5]\n" + "ldr q29, [x10, #0x50]\n" + "fmla v9.8h, v28.8h, v0.h[5]\n" + "fmla v13.8h, v28.8h, v1.h[5]\n" + "fmla v17.8h, v28.8h, v2.h[5]\n" + "fmla v21.8h, v28.8h, v3.h[5]\n" + "fmla v25.8h, v28.8h, v4.h[5]\n" + "ldr q28, [x9, #0x50]\n" + "fmla v10.8h, v29.8h, v0.h[5]\n" + "fmla v14.8h, v29.8h, v1.h[5]\n" + "fmla v18.8h, v29.8h, v2.h[5]\n" + "fmla v22.8h, v29.8h, v3.h[5]\n" + "fmla v26.8h, v29.8h, v4.h[5]\n" + "ldr q29, [x12, #0x60]\n" + "fmla v11.8h, v28.8h, v0.h[5]\n" + "fmla v15.8h, v28.8h, v1.h[5]\n" + "fmla v19.8h, v28.8h, v2.h[5]\n" + "fmla v23.8h, v28.8h, v3.h[5]\n" + "fmla v27.8h, v28.8h, v4.h[5]\n" + "ldr q28, [x11, #0x60]\n" + "fmla v8.8h, v29.8h, v0.h[6]\n" + "fmla v12.8h, v29.8h, v1.h[6]\n" + "fmla v16.8h, v29.8h, v2.h[6]\n" + "fmla v20.8h, v29.8h, v3.h[6]\n" + "fmla v24.8h, v29.8h, v4.h[6]\n" + "ldr q29, [x10, #0x60]\n" + "fmla v9.8h, v28.8h, v0.h[6]\n" + "fmla v13.8h, v28.8h, v1.h[6]\n" + "fmla v17.8h, v28.8h, v2.h[6]\n" + "fmla v21.8h, v28.8h, v3.h[6]\n" + "fmla v25.8h, v28.8h, v4.h[6]\n" + "ldr q28, [x9, #0x60]\n" + "fmla v10.8h, v29.8h, v0.h[6]\n" + "fmla v14.8h, v29.8h, v1.h[6]\n" + "fmla v18.8h, v29.8h, v2.h[6]\n" + "fmla v22.8h, v29.8h, v3.h[6]\n" + "fmla v26.8h, v29.8h, v4.h[6]\n" + "ldr q29, [x12, #0x70]\n" + "fmla v11.8h, v28.8h, v0.h[6]\n" + "add x12, x12, #0x80\n" + "fmla v15.8h, v28.8h, v1.h[6]\n" + "fmla v19.8h, v28.8h, v2.h[6]\n" + "fmla v23.8h, v28.8h, v3.h[6]\n" + "fmla v27.8h, v28.8h, v4.h[6]\n" + "ldr q28, [x11, #0x70]\n" + "add x11, x11, #0x80\n" + "fmla v8.8h, v29.8h, v0.h[7]\n" + "fmla v12.8h, v29.8h, v1.h[7]\n" + "fmla v16.8h, v29.8h, v2.h[7]\n" + "fmla v20.8h, v29.8h, v3.h[7]\n" + "fmla v24.8h, v29.8h, v4.h[7]\n" + "ldr q29, [x10, #0x70]\n" + "fmla v9.8h, v28.8h, v0.h[7]\n" + "add x10, x10, #0x80\n" + "fmla v13.8h, v28.8h, v1.h[7]\n" + "fmla v17.8h, v28.8h, v2.h[7]\n" + "fmla v21.8h, v28.8h, v3.h[7]\n" + "fmla v25.8h, v28.8h, v4.h[7]\n" + "ldr q28, [x9, #0x70]\n" + "add x9, x9, #0x80\n" + "fmla v10.8h, v29.8h, v0.h[7]\n" + "fmla v14.8h, v29.8h, v1.h[7]\n" + "fmla v18.8h, v29.8h, v2.h[7]\n" + "fmla v22.8h, v29.8h, v3.h[7]\n" + "fmla v26.8h, v29.8h, v4.h[7]\n" + "fmla v11.8h, v28.8h, v0.h[7]\n" + "fmla v15.8h, v28.8h, v1.h[7]\n" + "fmla v19.8h, v28.8h, v2.h[7]\n" + "fmla v23.8h, v28.8h, v3.h[7]\n" + "fmla v27.8h, v28.8h, v4.h[7]\n" + "229:" // Height 5: Multiply loop: Main loop skip + "cbz x27, 231f\n" + "230:" // Height 5: Multiply loop: Odd block loop + "ldr h4, [x26], #0x2\n" + "ldr h3, [x25], #0x2\n" + "sub x27, x27, #0x1\n" + "ldr h2, [x24], #0x2\n" + "ldr h1, [x23], #0x2\n" + "ldr h0, [x22], #0x2\n" + "ldr q29, [x12, #0x0]\n" + "fmla v8.8h, v29.8h, v4.h[0]\n" + "fmla v12.8h, v29.8h, v3.h[0]\n" + "ldr q28, [x11, #0x0]\n" + "fmla v16.8h, v29.8h, v2.h[0]\n" + "fmla v20.8h, v29.8h, v1.h[0]\n" + "add x12, x12, #0x10\n" + "fmla v24.8h, v29.8h, v0.h[0]\n" + "ldr q29, [x10, #0x0]\n" + "fmla v9.8h, v28.8h, v4.h[0]\n" + "add x11, x11, #0x10\n" + "fmla v13.8h, v28.8h, v3.h[0]\n" + "fmla v17.8h, v28.8h, v2.h[0]\n" + "add x10, x10, #0x10\n" + "fmla v21.8h, v28.8h, v1.h[0]\n" + "fmla v25.8h, v28.8h, v0.h[0]\n" + "ldr q28, [x9, #0x0]\n" "add x9, x9, #0x10\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "fmla v22.8h, v6.8h, v3.h[0]\n" - "fmla v26.8h, v6.8h, v4.h[0]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "fmla v23.8h, v7.8h, v3.h[0]\n" - "fmla v27.8h, v7.8h, v4.h[0]\n" + "fmla v10.8h, v29.8h, v4.h[0]\n" + "fmla v14.8h, v29.8h, v3.h[0]\n" + "fmla v18.8h, v29.8h, v2.h[0]\n" + "fmla v22.8h, v29.8h, v1.h[0]\n" + "fmla v26.8h, v29.8h, v0.h[0]\n" + "fmla v11.8h, v28.8h, v4.h[0]\n" + "fmla v15.8h, v28.8h, v3.h[0]\n" + "fmla v19.8h, v28.8h, v2.h[0]\n" + "fmla v23.8h, v28.8h, v1.h[0]\n" + "fmla v27.8h, v28.8h, v0.h[0]\n" "cbnz x27, 230b\n" "231:" // Height 5: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -3812,49 +3812,49 @@ void a64_ffhybrid_fp16_mla_6x32 ( "add x22, x23, x20, LSL #1\n" "tbz %x[flags], #1, 232f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.8h }, [x20]\n" + "ld1r { v29.8h }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.8h }, [x20]\n" - "fmin v8.8h, v8.8h, v1.8h\n" - "fmin v9.8h, v9.8h, v1.8h\n" - "fmin v10.8h, v10.8h, v1.8h\n" - "fmin v11.8h, v11.8h, v1.8h\n" - "fmin v12.8h, v12.8h, v1.8h\n" - "fmin v13.8h, v13.8h, v1.8h\n" - "fmin v14.8h, v14.8h, v1.8h\n" - "fmin v15.8h, v15.8h, v1.8h\n" - "fmin v16.8h, v16.8h, v1.8h\n" - "fmin v17.8h, v17.8h, v1.8h\n" - "fmin v18.8h, v18.8h, v1.8h\n" - "fmin v19.8h, v19.8h, v1.8h\n" - "fmin v20.8h, v20.8h, v1.8h\n" - "fmin v21.8h, v21.8h, v1.8h\n" - "fmin v22.8h, v22.8h, v1.8h\n" - "fmin v23.8h, v23.8h, v1.8h\n" - "fmin v24.8h, v24.8h, v1.8h\n" - "fmin v25.8h, v25.8h, v1.8h\n" - "fmin v26.8h, v26.8h, v1.8h\n" - "fmin v27.8h, v27.8h, v1.8h\n" - "fmax v8.8h, v8.8h, v0.8h\n" - "fmax v9.8h, v9.8h, v0.8h\n" - "fmax v10.8h, v10.8h, v0.8h\n" - "fmax v11.8h, v11.8h, v0.8h\n" - "fmax v12.8h, v12.8h, v0.8h\n" - "fmax v13.8h, v13.8h, v0.8h\n" - "fmax v14.8h, v14.8h, v0.8h\n" - "fmax v15.8h, v15.8h, v0.8h\n" - "fmax v16.8h, v16.8h, v0.8h\n" - "fmax v17.8h, v17.8h, v0.8h\n" - "fmax v18.8h, v18.8h, v0.8h\n" - "fmax v19.8h, v19.8h, v0.8h\n" - "fmax v20.8h, v20.8h, v0.8h\n" - "fmax v21.8h, v21.8h, v0.8h\n" - "fmax v22.8h, v22.8h, v0.8h\n" - "fmax v23.8h, v23.8h, v0.8h\n" - "fmax v24.8h, v24.8h, v0.8h\n" - "fmax v25.8h, v25.8h, v0.8h\n" - "fmax v26.8h, v26.8h, v0.8h\n" - "fmax v27.8h, v27.8h, v0.8h\n" + "ld1r { v28.8h }, [x20]\n" + "fmin v8.8h, v8.8h, v29.8h\n" + "fmin v9.8h, v9.8h, v29.8h\n" + "fmin v10.8h, v10.8h, v29.8h\n" + "fmin v11.8h, v11.8h, v29.8h\n" + "fmin v12.8h, v12.8h, v29.8h\n" + "fmin v13.8h, v13.8h, v29.8h\n" + "fmin v14.8h, v14.8h, v29.8h\n" + "fmin v15.8h, v15.8h, v29.8h\n" + "fmin v16.8h, v16.8h, v29.8h\n" + "fmin v17.8h, v17.8h, v29.8h\n" + "fmin v18.8h, v18.8h, v29.8h\n" + "fmin v19.8h, v19.8h, v29.8h\n" + "fmin v20.8h, v20.8h, v29.8h\n" + "fmin v21.8h, v21.8h, v29.8h\n" + "fmin v22.8h, v22.8h, v29.8h\n" + "fmin v23.8h, v23.8h, v29.8h\n" + "fmin v24.8h, v24.8h, v29.8h\n" + "fmin v25.8h, v25.8h, v29.8h\n" + "fmin v26.8h, v26.8h, v29.8h\n" + "fmin v27.8h, v27.8h, v29.8h\n" + "fmax v8.8h, v8.8h, v28.8h\n" + "fmax v9.8h, v9.8h, v28.8h\n" + "fmax v10.8h, v10.8h, v28.8h\n" + "fmax v11.8h, v11.8h, v28.8h\n" + "fmax v12.8h, v12.8h, v28.8h\n" + "fmax v13.8h, v13.8h, v28.8h\n" + "fmax v14.8h, v14.8h, v28.8h\n" + "fmax v15.8h, v15.8h, v28.8h\n" + "fmax v16.8h, v16.8h, v28.8h\n" + "fmax v17.8h, v17.8h, v28.8h\n" + "fmax v18.8h, v18.8h, v28.8h\n" + "fmax v19.8h, v19.8h, v28.8h\n" + "fmax v20.8h, v20.8h, v28.8h\n" + "fmax v21.8h, v21.8h, v28.8h\n" + "fmax v22.8h, v22.8h, v28.8h\n" + "fmax v23.8h, v23.8h, v28.8h\n" + "fmax v24.8h, v24.8h, v28.8h\n" + "fmax v25.8h, v25.8h, v28.8h\n" + "fmax v26.8h, v26.8h, v28.8h\n" + "fmax v27.8h, v27.8h, v28.8h\n" "232:" // Height 5: No activation "cmp x14, #0x20\n" "bge 249f\n" @@ -4497,16 +4497,16 @@ void a64_ffhybrid_fp16_mla_6x32 ( "274:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 275f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 276f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -4518,11 +4518,11 @@ void a64_ffhybrid_fp16_mla_6x32 ( "b 276f\n" "275:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" - "add x22, x23, x20, LSL #1\n" - "add x21, x22, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" + "add x22, x23, x21, LSL #1\n" + "add x21, x22, x21, LSL #1\n" "276:" // Height 6: input setup done "cmp x27, #0x8\n" "blt 279f\n" @@ -5017,45 +5017,45 @@ void a64_ffhybrid_fp16_mla_6x32 ( "279:" // Height 6: Multiply loop: Main loop skip "cbz x27, 281f\n" "280:" // Height 6: Multiply loop: Odd block loop - "ldr h0, [x26], #0x2\n" - "ldr h1, [x25], #0x2\n" + "ldr h7, [x26], #0x2\n" + "ldr h6, [x25], #0x2\n" "sub x27, x27, #0x1\n" - "ldr h2, [x24], #0x2\n" - "ldr h3, [x23], #0x2\n" - "ldr h4, [x22], #0x2\n" - "ldr h5, [x21], #0x2\n" - "ldr q6, [x12, #0x0]\n" - "ldr q7, [x11, #0x0]\n" - "fmla v8.8h, v6.8h, v0.h[0]\n" - "fmla v12.8h, v6.8h, v1.h[0]\n" - "fmla v16.8h, v6.8h, v2.h[0]\n" - "fmla v20.8h, v6.8h, v3.h[0]\n" + "ldr h5, [x24], #0x2\n" + "ldr h4, [x23], #0x2\n" + "ldr h3, [x22], #0x2\n" + "ldr h2, [x21], #0x2\n" + "ldr q1, [x12, #0x0]\n" + "ldr q0, [x11, #0x0]\n" + "fmla v8.8h, v1.8h, v7.h[0]\n" + "fmla v12.8h, v1.8h, v6.h[0]\n" + "fmla v16.8h, v1.8h, v5.h[0]\n" + "fmla v20.8h, v1.8h, v4.h[0]\n" "add x12, x12, #0x10\n" "add x11, x11, #0x10\n" - "fmla v24.8h, v6.8h, v4.h[0]\n" - "fmla v28.8h, v6.8h, v5.h[0]\n" - "ldr q6, [x10, #0x0]\n" + "fmla v24.8h, v1.8h, v3.h[0]\n" + "fmla v28.8h, v1.8h, v2.h[0]\n" + "ldr q1, [x10, #0x0]\n" "add x10, x10, #0x10\n" - "fmla v9.8h, v7.8h, v0.h[0]\n" - "fmla v13.8h, v7.8h, v1.h[0]\n" - "fmla v17.8h, v7.8h, v2.h[0]\n" - "fmla v21.8h, v7.8h, v3.h[0]\n" - "fmla v25.8h, v7.8h, v4.h[0]\n" - "fmla v29.8h, v7.8h, v5.h[0]\n" - "ldr q7, [x9, #0x0]\n" + "fmla v9.8h, v0.8h, v7.h[0]\n" + "fmla v13.8h, v0.8h, v6.h[0]\n" + "fmla v17.8h, v0.8h, v5.h[0]\n" + "fmla v21.8h, v0.8h, v4.h[0]\n" + "fmla v25.8h, v0.8h, v3.h[0]\n" + "fmla v29.8h, v0.8h, v2.h[0]\n" + "ldr q0, [x9, #0x0]\n" "add x9, x9, #0x10\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "fmla v22.8h, v6.8h, v3.h[0]\n" - "fmla v26.8h, v6.8h, v4.h[0]\n" - "fmla v30.8h, v6.8h, v5.h[0]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "fmla v23.8h, v7.8h, v3.h[0]\n" - "fmla v27.8h, v7.8h, v4.h[0]\n" - "fmla v31.8h, v7.8h, v5.h[0]\n" + "fmla v10.8h, v1.8h, v7.h[0]\n" + "fmla v14.8h, v1.8h, v6.h[0]\n" + "fmla v18.8h, v1.8h, v5.h[0]\n" + "fmla v22.8h, v1.8h, v4.h[0]\n" + "fmla v26.8h, v1.8h, v3.h[0]\n" + "fmla v30.8h, v1.8h, v2.h[0]\n" + "fmla v11.8h, v0.8h, v7.h[0]\n" + "fmla v15.8h, v0.8h, v6.h[0]\n" + "fmla v19.8h, v0.8h, v5.h[0]\n" + "fmla v23.8h, v0.8h, v4.h[0]\n" + "fmla v27.8h, v0.8h, v3.h[0]\n" + "fmla v31.8h, v0.8h, v2.h[0]\n" "cbnz x27, 280b\n" "281:" // Height 6: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp32_mla_6x16.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp32_mla_6x16.hpp index 08f5aeb2d8e113c967f61c562d4d399df3d2aaf1..94fb84e40912a15b6e0c40019eed42cd65eb1a7c 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp32_mla_6x16.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp32_mla_6x16.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,16 +10,16 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once #ifdef __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp32_mla_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp32_mla_6x16/generic.cpp index e0fbe17bad19363c4a8b61607854ac0aae2b3e20..b1cd6dc970ae3bee990da4e2d8701deac1ff1f29 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp32_mla_6x16/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp32_mla_6x16/generic.cpp @@ -209,11 +209,11 @@ void a64_ffhybrid_fp32_mla_6x16 ( "16:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 17f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 18f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -230,37 +230,37 @@ void a64_ffhybrid_fp32_mla_6x16 ( "blt 20f\n" "19:" // Height 1: Multiply loop: Main loop head "fmla v8.4s, v6.4s, v0.s[0]\n" - "ldr q6, [x10, #0x0]\n" + "ldr q17, [x10, #0x0]\n" "fmla v9.4s, v7.4s, v0.s[0]\n" - "ldr q7, [x9, #0x0]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "ldr q6, [x12, #0x10]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "ldr q7, [x11, #0x10]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "ldr q6, [x10, #0x10]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "ldr q7, [x9, #0x10]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "ldr q6, [x12, #0x20]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "ldr q7, [x11, #0x20]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "ldr q7, [x9, #0x20]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "ldr q6, [x12, #0x30]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "ldr q7, [x11, #0x30]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "ldr q6, [x10, #0x30]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "ldr q7, [x9, #0x30]\n" + "ldr q16, [x9, #0x0]\n" + "fmla v10.4s, v17.4s, v0.s[0]\n" + "ldr q17, [x12, #0x10]\n" + "fmla v11.4s, v16.4s, v0.s[0]\n" + "ldr q16, [x11, #0x10]\n" + "fmla v8.4s, v17.4s, v0.s[1]\n" + "ldr q17, [x10, #0x10]\n" + "fmla v9.4s, v16.4s, v0.s[1]\n" + "ldr q16, [x9, #0x10]\n" + "fmla v10.4s, v17.4s, v0.s[1]\n" + "ldr q17, [x12, #0x20]\n" + "fmla v11.4s, v16.4s, v0.s[1]\n" + "ldr q16, [x11, #0x20]\n" + "fmla v8.4s, v17.4s, v0.s[2]\n" + "ldr q17, [x10, #0x20]\n" + "fmla v9.4s, v16.4s, v0.s[2]\n" + "ldr q16, [x9, #0x20]\n" + "fmla v10.4s, v17.4s, v0.s[2]\n" + "ldr q17, [x12, #0x30]\n" + "fmla v11.4s, v16.4s, v0.s[2]\n" + "ldr q16, [x11, #0x30]\n" + "fmla v8.4s, v17.4s, v0.s[3]\n" + "ldr q17, [x10, #0x30]\n" + "fmla v9.4s, v16.4s, v0.s[3]\n" + "ldr q16, [x9, #0x30]\n" "sub x27, x27, #0x4\n" "cmp x27, #0x8\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" + "fmla v10.4s, v17.4s, v0.s[3]\n" + "fmla v11.4s, v16.4s, v0.s[3]\n" "add x26, x26, #0x10\n" "ldr q0, [x26, #0x0]\n" "add x12, x12, #0x40\n" @@ -272,36 +272,36 @@ void a64_ffhybrid_fp32_mla_6x16 ( "bge 19b\n" "20:" // Height 1: Multiply loop: Single iteration only "fmla v8.4s, v6.4s, v0.s[0]\n" - "ldr q6, [x10, #0x0]\n" + "ldr q17, [x10, #0x0]\n" "fmla v9.4s, v7.4s, v0.s[0]\n" - "ldr q7, [x9, #0x0]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "ldr q6, [x12, #0x10]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "ldr q7, [x11, #0x10]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "ldr q6, [x10, #0x10]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "ldr q7, [x9, #0x10]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "ldr q6, [x12, #0x20]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "ldr q7, [x11, #0x20]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "ldr q7, [x9, #0x20]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "ldr q6, [x12, #0x30]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "ldr q7, [x11, #0x30]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "ldr q6, [x10, #0x30]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "ldr q7, [x9, #0x30]\n" + "ldr q16, [x9, #0x0]\n" + "fmla v10.4s, v17.4s, v0.s[0]\n" + "ldr q17, [x12, #0x10]\n" + "fmla v11.4s, v16.4s, v0.s[0]\n" + "ldr q16, [x11, #0x10]\n" + "fmla v8.4s, v17.4s, v0.s[1]\n" + "ldr q17, [x10, #0x10]\n" + "fmla v9.4s, v16.4s, v0.s[1]\n" + "ldr q16, [x9, #0x10]\n" + "fmla v10.4s, v17.4s, v0.s[1]\n" + "ldr q17, [x12, #0x20]\n" + "fmla v11.4s, v16.4s, v0.s[1]\n" + "ldr q16, [x11, #0x20]\n" + "fmla v8.4s, v17.4s, v0.s[2]\n" + "ldr q17, [x10, #0x20]\n" + "fmla v9.4s, v16.4s, v0.s[2]\n" + "ldr q16, [x9, #0x20]\n" + "fmla v10.4s, v17.4s, v0.s[2]\n" + "ldr q17, [x12, #0x30]\n" + "fmla v11.4s, v16.4s, v0.s[2]\n" + "ldr q16, [x11, #0x30]\n" + "fmla v8.4s, v17.4s, v0.s[3]\n" + "ldr q17, [x10, #0x30]\n" + "fmla v9.4s, v16.4s, v0.s[3]\n" + "ldr q16, [x9, #0x30]\n" "sub x27, x27, #0x4\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" + "fmla v10.4s, v17.4s, v0.s[3]\n" + "fmla v11.4s, v16.4s, v0.s[3]\n" "add x26, x26, #0x10\n" "add x12, x12, #0x40\n" "add x11, x11, #0x40\n" @@ -310,16 +310,16 @@ void a64_ffhybrid_fp32_mla_6x16 ( "21:" // Height 1: Multiply loop: Main loop skip "cbz x27, 23f\n" "22:" // Height 1: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr q6, [x12, #0x0]\n" - "fmla v8.4s, v6.4s, v0.s[0]\n" + "ldr s18, [x26], #0x4\n" + "ldr q16, [x12, #0x0]\n" + "fmla v8.4s, v16.4s, v18.s[0]\n" "sub x27, x27, #0x1\n" - "ldr q7, [x11, #0x0]\n" - "ldr q6, [x10, #0x0]\n" - "fmla v9.4s, v7.4s, v0.s[0]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "ldr q7, [x9, #0x0]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" + "ldr q17, [x11, #0x0]\n" + "ldr q16, [x10, #0x0]\n" + "fmla v9.4s, v17.4s, v18.s[0]\n" + "fmla v10.4s, v16.4s, v18.s[0]\n" + "ldr q16, [x9, #0x0]\n" + "fmla v11.4s, v16.4s, v18.s[0]\n" "add x12, x12, #0x10\n" "add x11, x11, #0x10\n" "add x10, x10, #0x10\n" @@ -332,17 +332,17 @@ void a64_ffhybrid_fp32_mla_6x16 ( "bne 16b\n" "tbz %x[flags], #1, 24f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v17.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" + "ld1r { v16.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v17.4s\n" + "fmin v9.4s, v9.4s, v17.4s\n" + "fmin v10.4s, v10.4s, v17.4s\n" + "fmin v11.4s, v11.4s, v17.4s\n" + "fmax v8.4s, v8.4s, v16.4s\n" + "fmax v9.4s, v9.4s, v16.4s\n" + "fmax v10.4s, v10.4s, v16.4s\n" + "fmax v11.4s, v11.4s, v16.4s\n" "24:" // Height 1: No activation "cmp x14, #0x10\n" "bge 33f\n" @@ -538,12 +538,12 @@ void a64_ffhybrid_fp32_mla_6x16 ( "50:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 51f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 52f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -551,7 +551,7 @@ void a64_ffhybrid_fp32_mla_6x16 ( "b 52f\n" "51:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" "52:" // Height 2: input setup done "cmp x27, #0x4\n" "blt 55f\n" @@ -564,137 +564,137 @@ void a64_ffhybrid_fp32_mla_6x16 ( "53:" // Height 2: Multiply loop: Main loop head "fmla v8.4s, v6.4s, v0.s[0]\n" "fmla v12.4s, v6.4s, v1.s[0]\n" - "ldr q6, [x10, #0x0]\n" + "ldr q17, [x10, #0x0]\n" "sub x27, x27, #0x4\n" "fmla v9.4s, v7.4s, v0.s[0]\n" "fmla v13.4s, v7.4s, v1.s[0]\n" - "ldr q7, [x9, #0x0]\n" + "ldr q16, [x9, #0x0]\n" "cmp x27, #0x8\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "ldr q6, [x12, #0x10]\n" + "fmla v10.4s, v17.4s, v0.s[0]\n" + "fmla v14.4s, v17.4s, v1.s[0]\n" + "ldr q17, [x12, #0x10]\n" "add x26, x26, #0x10\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "ldr q7, [x11, #0x10]\n" + "fmla v11.4s, v16.4s, v0.s[0]\n" + "fmla v15.4s, v16.4s, v1.s[0]\n" + "ldr q16, [x11, #0x10]\n" "add x25, x25, #0x10\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v12.4s, v6.4s, v1.s[1]\n" - "ldr q6, [x10, #0x10]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v13.4s, v7.4s, v1.s[1]\n" - "ldr q7, [x9, #0x10]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "ldr q6, [x12, #0x20]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "ldr q7, [x11, #0x20]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "fmla v12.4s, v6.4s, v1.s[2]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "fmla v13.4s, v7.4s, v1.s[2]\n" - "ldr q7, [x9, #0x20]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v14.4s, v6.4s, v1.s[2]\n" - "ldr q6, [x12, #0x30]\n" + "fmla v8.4s, v17.4s, v0.s[1]\n" + "fmla v12.4s, v17.4s, v1.s[1]\n" + "ldr q17, [x10, #0x10]\n" + "fmla v9.4s, v16.4s, v0.s[1]\n" + "fmla v13.4s, v16.4s, v1.s[1]\n" + "ldr q16, [x9, #0x10]\n" + "fmla v10.4s, v17.4s, v0.s[1]\n" + "fmla v14.4s, v17.4s, v1.s[1]\n" + "ldr q17, [x12, #0x20]\n" + "fmla v11.4s, v16.4s, v0.s[1]\n" + "fmla v15.4s, v16.4s, v1.s[1]\n" + "ldr q16, [x11, #0x20]\n" + "fmla v8.4s, v17.4s, v0.s[2]\n" + "fmla v12.4s, v17.4s, v1.s[2]\n" + "ldr q17, [x10, #0x20]\n" + "fmla v9.4s, v16.4s, v0.s[2]\n" + "fmla v13.4s, v16.4s, v1.s[2]\n" + "ldr q16, [x9, #0x20]\n" + "fmla v10.4s, v17.4s, v0.s[2]\n" + "fmla v14.4s, v17.4s, v1.s[2]\n" + "ldr q17, [x12, #0x30]\n" "add x12, x12, #0x40\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "fmla v15.4s, v7.4s, v1.s[2]\n" - "ldr q7, [x11, #0x30]\n" + "fmla v11.4s, v16.4s, v0.s[2]\n" + "fmla v15.4s, v16.4s, v1.s[2]\n" + "ldr q16, [x11, #0x30]\n" "add x11, x11, #0x40\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v12.4s, v6.4s, v1.s[3]\n" - "ldr q6, [x10, #0x30]\n" + "fmla v8.4s, v17.4s, v0.s[3]\n" + "fmla v12.4s, v17.4s, v1.s[3]\n" + "ldr q17, [x10, #0x30]\n" "add x10, x10, #0x40\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "fmla v13.4s, v7.4s, v1.s[3]\n" - "ldr q7, [x9, #0x30]\n" + "fmla v9.4s, v16.4s, v0.s[3]\n" + "fmla v13.4s, v16.4s, v1.s[3]\n" + "ldr q16, [x9, #0x30]\n" "add x9, x9, #0x40\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" + "fmla v10.4s, v17.4s, v0.s[3]\n" + "fmla v14.4s, v17.4s, v1.s[3]\n" "ldr q6, [x12, #0x0]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" + "fmla v11.4s, v16.4s, v0.s[3]\n" "ldr q0, [x26, #0x0]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" + "fmla v15.4s, v16.4s, v1.s[3]\n" "ldr q1, [x25, #0x0]\n" "ldr q7, [x11, #0x0]\n" "bge 53b\n" "54:" // Height 2: Multiply loop: Single iteration only "fmla v8.4s, v6.4s, v0.s[0]\n" "fmla v12.4s, v6.4s, v1.s[0]\n" - "ldr q6, [x10, #0x0]\n" + "ldr q17, [x10, #0x0]\n" "sub x27, x27, #0x4\n" "fmla v9.4s, v7.4s, v0.s[0]\n" "fmla v13.4s, v7.4s, v1.s[0]\n" - "ldr q7, [x9, #0x0]\n" + "ldr q16, [x9, #0x0]\n" "add x26, x26, #0x10\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "ldr q6, [x12, #0x10]\n" + "fmla v10.4s, v17.4s, v0.s[0]\n" + "fmla v14.4s, v17.4s, v1.s[0]\n" + "ldr q17, [x12, #0x10]\n" "add x25, x25, #0x10\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "ldr q7, [x11, #0x10]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v12.4s, v6.4s, v1.s[1]\n" - "ldr q6, [x10, #0x10]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v13.4s, v7.4s, v1.s[1]\n" - "ldr q7, [x9, #0x10]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "ldr q6, [x12, #0x20]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "ldr q7, [x11, #0x20]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "fmla v12.4s, v6.4s, v1.s[2]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "fmla v13.4s, v7.4s, v1.s[2]\n" - "ldr q7, [x9, #0x20]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v14.4s, v6.4s, v1.s[2]\n" - "ldr q6, [x12, #0x30]\n" + "fmla v11.4s, v16.4s, v0.s[0]\n" + "fmla v15.4s, v16.4s, v1.s[0]\n" + "ldr q16, [x11, #0x10]\n" + "fmla v8.4s, v17.4s, v0.s[1]\n" + "fmla v12.4s, v17.4s, v1.s[1]\n" + "ldr q17, [x10, #0x10]\n" + "fmla v9.4s, v16.4s, v0.s[1]\n" + "fmla v13.4s, v16.4s, v1.s[1]\n" + "ldr q16, [x9, #0x10]\n" + "fmla v10.4s, v17.4s, v0.s[1]\n" + "fmla v14.4s, v17.4s, v1.s[1]\n" + "ldr q17, [x12, #0x20]\n" + "fmla v11.4s, v16.4s, v0.s[1]\n" + "fmla v15.4s, v16.4s, v1.s[1]\n" + "ldr q16, [x11, #0x20]\n" + "fmla v8.4s, v17.4s, v0.s[2]\n" + "fmla v12.4s, v17.4s, v1.s[2]\n" + "ldr q17, [x10, #0x20]\n" + "fmla v9.4s, v16.4s, v0.s[2]\n" + "fmla v13.4s, v16.4s, v1.s[2]\n" + "ldr q16, [x9, #0x20]\n" + "fmla v10.4s, v17.4s, v0.s[2]\n" + "fmla v14.4s, v17.4s, v1.s[2]\n" + "ldr q17, [x12, #0x30]\n" "add x12, x12, #0x40\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "fmla v15.4s, v7.4s, v1.s[2]\n" - "ldr q7, [x11, #0x30]\n" + "fmla v11.4s, v16.4s, v0.s[2]\n" + "fmla v15.4s, v16.4s, v1.s[2]\n" + "ldr q16, [x11, #0x30]\n" "add x11, x11, #0x40\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v12.4s, v6.4s, v1.s[3]\n" - "ldr q6, [x10, #0x30]\n" + "fmla v8.4s, v17.4s, v0.s[3]\n" + "fmla v12.4s, v17.4s, v1.s[3]\n" + "ldr q17, [x10, #0x30]\n" "add x10, x10, #0x40\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "fmla v13.4s, v7.4s, v1.s[3]\n" - "ldr q7, [x9, #0x30]\n" + "fmla v9.4s, v16.4s, v0.s[3]\n" + "fmla v13.4s, v16.4s, v1.s[3]\n" + "ldr q16, [x9, #0x30]\n" "add x9, x9, #0x40\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" + "fmla v10.4s, v17.4s, v0.s[3]\n" + "fmla v14.4s, v17.4s, v1.s[3]\n" + "fmla v11.4s, v16.4s, v0.s[3]\n" + "fmla v15.4s, v16.4s, v1.s[3]\n" "55:" // Height 2: Multiply loop: Main loop skip "cbz x27, 57f\n" "56:" // Height 2: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s19, [x26], #0x4\n" + "ldr s18, [x25], #0x4\n" "sub x27, x27, #0x1\n" - "ldr q6, [x12, #0x0]\n" - "ldr q7, [x11, #0x0]\n" - "fmla v8.4s, v6.4s, v0.s[0]\n" - "fmla v12.4s, v6.4s, v1.s[0]\n" - "ldr q6, [x10, #0x0]\n" - "fmla v9.4s, v7.4s, v0.s[0]\n" - "fmla v13.4s, v7.4s, v1.s[0]\n" - "ldr q7, [x9, #0x0]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" + "ldr q17, [x12, #0x0]\n" + "ldr q16, [x11, #0x0]\n" + "fmla v8.4s, v17.4s, v19.s[0]\n" + "fmla v12.4s, v17.4s, v18.s[0]\n" + "ldr q17, [x10, #0x0]\n" + "fmla v9.4s, v16.4s, v19.s[0]\n" + "fmla v13.4s, v16.4s, v18.s[0]\n" + "ldr q16, [x9, #0x0]\n" + "fmla v10.4s, v17.4s, v19.s[0]\n" + "fmla v14.4s, v17.4s, v18.s[0]\n" "add x12, x12, #0x10\n" "add x11, x11, #0x10\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" + "fmla v11.4s, v16.4s, v19.s[0]\n" + "fmla v15.4s, v16.4s, v18.s[0]\n" "add x10, x10, #0x10\n" "add x9, x9, #0x10\n" "cbnz x27, 56b\n" @@ -707,25 +707,25 @@ void a64_ffhybrid_fp32_mla_6x16 ( "add x25, x13, x20, LSL #2\n" "tbz %x[flags], #1, 58f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v17.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v15.4s, v15.4s, v1.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" + "ld1r { v16.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v17.4s\n" + "fmin v9.4s, v9.4s, v17.4s\n" + "fmin v10.4s, v10.4s, v17.4s\n" + "fmin v11.4s, v11.4s, v17.4s\n" + "fmin v12.4s, v12.4s, v17.4s\n" + "fmin v13.4s, v13.4s, v17.4s\n" + "fmin v14.4s, v14.4s, v17.4s\n" + "fmin v15.4s, v15.4s, v17.4s\n" + "fmax v8.4s, v8.4s, v16.4s\n" + "fmax v9.4s, v9.4s, v16.4s\n" + "fmax v10.4s, v10.4s, v16.4s\n" + "fmax v11.4s, v11.4s, v16.4s\n" + "fmax v12.4s, v12.4s, v16.4s\n" + "fmax v13.4s, v13.4s, v16.4s\n" + "fmax v14.4s, v14.4s, v16.4s\n" + "fmax v15.4s, v15.4s, v16.4s\n" "58:" // Height 2: No activation "cmp x14, #0x10\n" "bge 67f\n" @@ -970,13 +970,13 @@ void a64_ffhybrid_fp32_mla_6x16 ( "84:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 85f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 86f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -985,8 +985,8 @@ void a64_ffhybrid_fp32_mla_6x16 ( "b 86f\n" "85:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" "86:" // Height 3: input setup done "cmp x27, #0x4\n" "blt 89f\n" @@ -1003,75 +1003,75 @@ void a64_ffhybrid_fp32_mla_6x16 ( "sub x27, x27, #0x4\n" "cmp x27, #0x8\n" "fmla v16.4s, v6.4s, v2.s[0]\n" - "ldr q6, [x10, #0x0]\n" + "ldr q21, [x10, #0x0]\n" "fmla v9.4s, v7.4s, v0.s[0]\n" "add x26, x26, #0x10\n" "fmla v13.4s, v7.4s, v1.s[0]\n" "fmla v17.4s, v7.4s, v2.s[0]\n" - "ldr q7, [x9, #0x0]\n" + "ldr q20, [x9, #0x0]\n" "add x25, x25, #0x10\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" + "fmla v10.4s, v21.4s, v0.s[0]\n" + "fmla v14.4s, v21.4s, v1.s[0]\n" "add x24, x24, #0x10\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "ldr q6, [x12, #0x10]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "ldr q7, [x11, #0x10]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v12.4s, v6.4s, v1.s[1]\n" - "fmla v16.4s, v6.4s, v2.s[1]\n" - "ldr q6, [x10, #0x10]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v13.4s, v7.4s, v1.s[1]\n" - "fmla v17.4s, v7.4s, v2.s[1]\n" - "ldr q7, [x9, #0x10]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "fmla v18.4s, v6.4s, v2.s[1]\n" - "ldr q6, [x12, #0x20]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "fmla v19.4s, v7.4s, v2.s[1]\n" - "ldr q7, [x11, #0x20]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "fmla v12.4s, v6.4s, v1.s[2]\n" - "fmla v16.4s, v6.4s, v2.s[2]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "fmla v13.4s, v7.4s, v1.s[2]\n" - "fmla v17.4s, v7.4s, v2.s[2]\n" - "ldr q7, [x9, #0x20]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v14.4s, v6.4s, v1.s[2]\n" - "fmla v18.4s, v6.4s, v2.s[2]\n" - "ldr q6, [x12, #0x30]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" + "fmla v18.4s, v21.4s, v2.s[0]\n" + "ldr q21, [x12, #0x10]\n" + "fmla v11.4s, v20.4s, v0.s[0]\n" + "fmla v15.4s, v20.4s, v1.s[0]\n" + "fmla v19.4s, v20.4s, v2.s[0]\n" + "ldr q20, [x11, #0x10]\n" + "fmla v8.4s, v21.4s, v0.s[1]\n" + "fmla v12.4s, v21.4s, v1.s[1]\n" + "fmla v16.4s, v21.4s, v2.s[1]\n" + "ldr q21, [x10, #0x10]\n" + "fmla v9.4s, v20.4s, v0.s[1]\n" + "fmla v13.4s, v20.4s, v1.s[1]\n" + "fmla v17.4s, v20.4s, v2.s[1]\n" + "ldr q20, [x9, #0x10]\n" + "fmla v10.4s, v21.4s, v0.s[1]\n" + "fmla v14.4s, v21.4s, v1.s[1]\n" + "fmla v18.4s, v21.4s, v2.s[1]\n" + "ldr q21, [x12, #0x20]\n" + "fmla v11.4s, v20.4s, v0.s[1]\n" + "fmla v15.4s, v20.4s, v1.s[1]\n" + "fmla v19.4s, v20.4s, v2.s[1]\n" + "ldr q20, [x11, #0x20]\n" + "fmla v8.4s, v21.4s, v0.s[2]\n" + "fmla v12.4s, v21.4s, v1.s[2]\n" + "fmla v16.4s, v21.4s, v2.s[2]\n" + "ldr q21, [x10, #0x20]\n" + "fmla v9.4s, v20.4s, v0.s[2]\n" + "fmla v13.4s, v20.4s, v1.s[2]\n" + "fmla v17.4s, v20.4s, v2.s[2]\n" + "ldr q20, [x9, #0x20]\n" + "fmla v10.4s, v21.4s, v0.s[2]\n" + "fmla v14.4s, v21.4s, v1.s[2]\n" + "fmla v18.4s, v21.4s, v2.s[2]\n" + "ldr q21, [x12, #0x30]\n" + "fmla v11.4s, v20.4s, v0.s[2]\n" "add x12, x12, #0x40\n" - "fmla v15.4s, v7.4s, v1.s[2]\n" - "fmla v19.4s, v7.4s, v2.s[2]\n" - "ldr q7, [x11, #0x30]\n" + "fmla v15.4s, v20.4s, v1.s[2]\n" + "fmla v19.4s, v20.4s, v2.s[2]\n" + "ldr q20, [x11, #0x30]\n" "add x11, x11, #0x40\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v12.4s, v6.4s, v1.s[3]\n" - "fmla v16.4s, v6.4s, v2.s[3]\n" - "ldr q6, [x10, #0x30]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" + "fmla v8.4s, v21.4s, v0.s[3]\n" + "fmla v12.4s, v21.4s, v1.s[3]\n" + "fmla v16.4s, v21.4s, v2.s[3]\n" + "ldr q21, [x10, #0x30]\n" + "fmla v9.4s, v20.4s, v0.s[3]\n" "add x10, x10, #0x40\n" - "fmla v13.4s, v7.4s, v1.s[3]\n" - "fmla v17.4s, v7.4s, v2.s[3]\n" - "ldr q7, [x9, #0x30]\n" + "fmla v13.4s, v20.4s, v1.s[3]\n" + "fmla v17.4s, v20.4s, v2.s[3]\n" + "ldr q20, [x9, #0x30]\n" "add x9, x9, #0x40\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "fmla v18.4s, v6.4s, v2.s[3]\n" + "fmla v10.4s, v21.4s, v0.s[3]\n" + "fmla v14.4s, v21.4s, v1.s[3]\n" + "fmla v18.4s, v21.4s, v2.s[3]\n" "ldr q6, [x12, #0x0]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" + "fmla v11.4s, v20.4s, v0.s[3]\n" "ldr q0, [x26, #0x0]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" + "fmla v15.4s, v20.4s, v1.s[3]\n" "ldr q1, [x25, #0x0]\n" - "fmla v19.4s, v7.4s, v2.s[3]\n" + "fmla v19.4s, v20.4s, v2.s[3]\n" "ldr q2, [x24, #0x0]\n" "ldr q7, [x11, #0x0]\n" "bge 87b\n" @@ -1081,98 +1081,98 @@ void a64_ffhybrid_fp32_mla_6x16 ( "sub x27, x27, #0x4\n" "add x26, x26, #0x10\n" "fmla v16.4s, v6.4s, v2.s[0]\n" - "ldr q6, [x10, #0x0]\n" + "ldr q21, [x10, #0x0]\n" "fmla v9.4s, v7.4s, v0.s[0]\n" "add x25, x25, #0x10\n" "fmla v13.4s, v7.4s, v1.s[0]\n" "fmla v17.4s, v7.4s, v2.s[0]\n" - "ldr q7, [x9, #0x0]\n" + "ldr q20, [x9, #0x0]\n" "add x24, x24, #0x10\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "ldr q6, [x12, #0x10]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "ldr q7, [x11, #0x10]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v12.4s, v6.4s, v1.s[1]\n" - "fmla v16.4s, v6.4s, v2.s[1]\n" - "ldr q6, [x10, #0x10]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v13.4s, v7.4s, v1.s[1]\n" - "fmla v17.4s, v7.4s, v2.s[1]\n" - "ldr q7, [x9, #0x10]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "fmla v18.4s, v6.4s, v2.s[1]\n" - "ldr q6, [x12, #0x20]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "fmla v19.4s, v7.4s, v2.s[1]\n" - "ldr q7, [x11, #0x20]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "fmla v12.4s, v6.4s, v1.s[2]\n" - "fmla v16.4s, v6.4s, v2.s[2]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "fmla v13.4s, v7.4s, v1.s[2]\n" - "fmla v17.4s, v7.4s, v2.s[2]\n" - "ldr q7, [x9, #0x20]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v14.4s, v6.4s, v1.s[2]\n" - "fmla v18.4s, v6.4s, v2.s[2]\n" - "ldr q6, [x12, #0x30]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" + "fmla v10.4s, v21.4s, v0.s[0]\n" + "fmla v14.4s, v21.4s, v1.s[0]\n" + "fmla v18.4s, v21.4s, v2.s[0]\n" + "ldr q21, [x12, #0x10]\n" + "fmla v11.4s, v20.4s, v0.s[0]\n" + "fmla v15.4s, v20.4s, v1.s[0]\n" + "fmla v19.4s, v20.4s, v2.s[0]\n" + "ldr q20, [x11, #0x10]\n" + "fmla v8.4s, v21.4s, v0.s[1]\n" + "fmla v12.4s, v21.4s, v1.s[1]\n" + "fmla v16.4s, v21.4s, v2.s[1]\n" + "ldr q21, [x10, #0x10]\n" + "fmla v9.4s, v20.4s, v0.s[1]\n" + "fmla v13.4s, v20.4s, v1.s[1]\n" + "fmla v17.4s, v20.4s, v2.s[1]\n" + "ldr q20, [x9, #0x10]\n" + "fmla v10.4s, v21.4s, v0.s[1]\n" + "fmla v14.4s, v21.4s, v1.s[1]\n" + "fmla v18.4s, v21.4s, v2.s[1]\n" + "ldr q21, [x12, #0x20]\n" + "fmla v11.4s, v20.4s, v0.s[1]\n" + "fmla v15.4s, v20.4s, v1.s[1]\n" + "fmla v19.4s, v20.4s, v2.s[1]\n" + "ldr q20, [x11, #0x20]\n" + "fmla v8.4s, v21.4s, v0.s[2]\n" + "fmla v12.4s, v21.4s, v1.s[2]\n" + "fmla v16.4s, v21.4s, v2.s[2]\n" + "ldr q21, [x10, #0x20]\n" + "fmla v9.4s, v20.4s, v0.s[2]\n" + "fmla v13.4s, v20.4s, v1.s[2]\n" + "fmla v17.4s, v20.4s, v2.s[2]\n" + "ldr q20, [x9, #0x20]\n" + "fmla v10.4s, v21.4s, v0.s[2]\n" + "fmla v14.4s, v21.4s, v1.s[2]\n" + "fmla v18.4s, v21.4s, v2.s[2]\n" + "ldr q21, [x12, #0x30]\n" + "fmla v11.4s, v20.4s, v0.s[2]\n" "add x12, x12, #0x40\n" - "fmla v15.4s, v7.4s, v1.s[2]\n" - "fmla v19.4s, v7.4s, v2.s[2]\n" - "ldr q7, [x11, #0x30]\n" + "fmla v15.4s, v20.4s, v1.s[2]\n" + "fmla v19.4s, v20.4s, v2.s[2]\n" + "ldr q20, [x11, #0x30]\n" "add x11, x11, #0x40\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v12.4s, v6.4s, v1.s[3]\n" - "fmla v16.4s, v6.4s, v2.s[3]\n" - "ldr q6, [x10, #0x30]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" + "fmla v8.4s, v21.4s, v0.s[3]\n" + "fmla v12.4s, v21.4s, v1.s[3]\n" + "fmla v16.4s, v21.4s, v2.s[3]\n" + "ldr q21, [x10, #0x30]\n" + "fmla v9.4s, v20.4s, v0.s[3]\n" "add x10, x10, #0x40\n" - "fmla v13.4s, v7.4s, v1.s[3]\n" - "fmla v17.4s, v7.4s, v2.s[3]\n" - "ldr q7, [x9, #0x30]\n" + "fmla v13.4s, v20.4s, v1.s[3]\n" + "fmla v17.4s, v20.4s, v2.s[3]\n" + "ldr q20, [x9, #0x30]\n" "add x9, x9, #0x40\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "fmla v18.4s, v6.4s, v2.s[3]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" - "fmla v19.4s, v7.4s, v2.s[3]\n" + "fmla v10.4s, v21.4s, v0.s[3]\n" + "fmla v14.4s, v21.4s, v1.s[3]\n" + "fmla v18.4s, v21.4s, v2.s[3]\n" + "fmla v11.4s, v20.4s, v0.s[3]\n" + "fmla v15.4s, v20.4s, v1.s[3]\n" + "fmla v19.4s, v20.4s, v2.s[3]\n" "89:" // Height 3: Multiply loop: Main loop skip "cbz x27, 91f\n" "90:" // Height 3: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s24, [x26], #0x4\n" + "ldr s23, [x25], #0x4\n" "sub x27, x27, #0x1\n" - "ldr s2, [x24], #0x4\n" - "ldr q6, [x12, #0x0]\n" - "fmla v8.4s, v6.4s, v0.s[0]\n" - "fmla v12.4s, v6.4s, v1.s[0]\n" - "ldr q7, [x11, #0x0]\n" - "fmla v16.4s, v6.4s, v2.s[0]\n" - "ldr q6, [x10, #0x0]\n" - "fmla v9.4s, v7.4s, v0.s[0]\n" - "fmla v13.4s, v7.4s, v1.s[0]\n" - "fmla v17.4s, v7.4s, v2.s[0]\n" - "ldr q7, [x9, #0x0]\n" + "ldr s22, [x24], #0x4\n" + "ldr q21, [x12, #0x0]\n" + "fmla v8.4s, v21.4s, v24.s[0]\n" + "fmla v12.4s, v21.4s, v23.s[0]\n" + "ldr q20, [x11, #0x0]\n" + "fmla v16.4s, v21.4s, v22.s[0]\n" + "ldr q21, [x10, #0x0]\n" + "fmla v9.4s, v20.4s, v24.s[0]\n" + "fmla v13.4s, v20.4s, v23.s[0]\n" + "fmla v17.4s, v20.4s, v22.s[0]\n" + "ldr q20, [x9, #0x0]\n" "add x12, x12, #0x10\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" + "fmla v10.4s, v21.4s, v24.s[0]\n" + "fmla v14.4s, v21.4s, v23.s[0]\n" "add x11, x11, #0x10\n" "add x10, x10, #0x10\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" + "fmla v18.4s, v21.4s, v22.s[0]\n" + "fmla v11.4s, v20.4s, v24.s[0]\n" "add x9, x9, #0x10\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" + "fmla v15.4s, v20.4s, v23.s[0]\n" + "fmla v19.4s, v20.4s, v22.s[0]\n" "cbnz x27, 90b\n" "91:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -1184,33 +1184,33 @@ void a64_ffhybrid_fp32_mla_6x16 ( "add x24, x25, x20, LSL #2\n" "tbz %x[flags], #1, 92f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v21.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v15.4s, v15.4s, v1.4s\n" - "fmin v16.4s, v16.4s, v1.4s\n" - "fmin v17.4s, v17.4s, v1.4s\n" - "fmin v18.4s, v18.4s, v1.4s\n" - "fmin v19.4s, v19.4s, v1.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" - "fmax v16.4s, v16.4s, v0.4s\n" - "fmax v17.4s, v17.4s, v0.4s\n" - "fmax v18.4s, v18.4s, v0.4s\n" - "fmax v19.4s, v19.4s, v0.4s\n" + "ld1r { v20.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v21.4s\n" + "fmin v9.4s, v9.4s, v21.4s\n" + "fmin v10.4s, v10.4s, v21.4s\n" + "fmin v11.4s, v11.4s, v21.4s\n" + "fmin v12.4s, v12.4s, v21.4s\n" + "fmin v13.4s, v13.4s, v21.4s\n" + "fmin v14.4s, v14.4s, v21.4s\n" + "fmin v15.4s, v15.4s, v21.4s\n" + "fmin v16.4s, v16.4s, v21.4s\n" + "fmin v17.4s, v17.4s, v21.4s\n" + "fmin v18.4s, v18.4s, v21.4s\n" + "fmin v19.4s, v19.4s, v21.4s\n" + "fmax v8.4s, v8.4s, v20.4s\n" + "fmax v9.4s, v9.4s, v20.4s\n" + "fmax v10.4s, v10.4s, v20.4s\n" + "fmax v11.4s, v11.4s, v20.4s\n" + "fmax v12.4s, v12.4s, v20.4s\n" + "fmax v13.4s, v13.4s, v20.4s\n" + "fmax v14.4s, v14.4s, v20.4s\n" + "fmax v15.4s, v15.4s, v20.4s\n" + "fmax v16.4s, v16.4s, v20.4s\n" + "fmax v17.4s, v17.4s, v20.4s\n" + "fmax v18.4s, v18.4s, v20.4s\n" + "fmax v19.4s, v19.4s, v20.4s\n" "92:" // Height 3: No activation "cmp x14, #0x10\n" "bge 101f\n" @@ -1504,14 +1504,14 @@ void a64_ffhybrid_fp32_mla_6x16 ( "118:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 119f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 120f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -1521,9 +1521,9 @@ void a64_ffhybrid_fp32_mla_6x16 ( "b 120f\n" "119:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" "120:" // Height 4: input setup done "cmp x27, #0x4\n" "blt 123f\n" @@ -1542,7 +1542,7 @@ void a64_ffhybrid_fp32_mla_6x16 ( "cmp x27, #0x8\n" "fmla v16.4s, v6.4s, v2.s[0]\n" "fmla v20.4s, v6.4s, v3.s[0]\n" - "ldr q6, [x10, #0x0]\n" + "ldr q25, [x10, #0x0]\n" "add x26, x26, #0x10\n" "fmla v9.4s, v7.4s, v0.s[0]\n" "fmla v13.4s, v7.4s, v1.s[0]\n" @@ -1550,84 +1550,84 @@ void a64_ffhybrid_fp32_mla_6x16 ( "add x24, x24, #0x10\n" "fmla v17.4s, v7.4s, v2.s[0]\n" "fmla v21.4s, v7.4s, v3.s[0]\n" - "ldr q7, [x9, #0x0]\n" + "ldr q24, [x9, #0x0]\n" "add x23, x23, #0x10\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "fmla v22.4s, v6.4s, v3.s[0]\n" - "ldr q6, [x12, #0x10]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "fmla v23.4s, v7.4s, v3.s[0]\n" - "ldr q7, [x11, #0x10]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v12.4s, v6.4s, v1.s[1]\n" - "fmla v16.4s, v6.4s, v2.s[1]\n" - "fmla v20.4s, v6.4s, v3.s[1]\n" - "ldr q6, [x10, #0x10]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v13.4s, v7.4s, v1.s[1]\n" - "fmla v17.4s, v7.4s, v2.s[1]\n" - "fmla v21.4s, v7.4s, v3.s[1]\n" - "ldr q7, [x9, #0x10]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "fmla v18.4s, v6.4s, v2.s[1]\n" - "fmla v22.4s, v6.4s, v3.s[1]\n" - "ldr q6, [x12, #0x20]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "fmla v19.4s, v7.4s, v2.s[1]\n" - "fmla v23.4s, v7.4s, v3.s[1]\n" - "ldr q7, [x11, #0x20]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "fmla v12.4s, v6.4s, v1.s[2]\n" - "fmla v16.4s, v6.4s, v2.s[2]\n" - "fmla v20.4s, v6.4s, v3.s[2]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "fmla v13.4s, v7.4s, v1.s[2]\n" - "fmla v17.4s, v7.4s, v2.s[2]\n" - "fmla v21.4s, v7.4s, v3.s[2]\n" - "ldr q7, [x9, #0x20]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v14.4s, v6.4s, v1.s[2]\n" - "fmla v18.4s, v6.4s, v2.s[2]\n" - "fmla v22.4s, v6.4s, v3.s[2]\n" - "ldr q6, [x12, #0x30]\n" + "fmla v10.4s, v25.4s, v0.s[0]\n" + "fmla v14.4s, v25.4s, v1.s[0]\n" + "fmla v18.4s, v25.4s, v2.s[0]\n" + "fmla v22.4s, v25.4s, v3.s[0]\n" + "ldr q25, [x12, #0x10]\n" + "fmla v11.4s, v24.4s, v0.s[0]\n" + "fmla v15.4s, v24.4s, v1.s[0]\n" + "fmla v19.4s, v24.4s, v2.s[0]\n" + "fmla v23.4s, v24.4s, v3.s[0]\n" + "ldr q24, [x11, #0x10]\n" + "fmla v8.4s, v25.4s, v0.s[1]\n" + "fmla v12.4s, v25.4s, v1.s[1]\n" + "fmla v16.4s, v25.4s, v2.s[1]\n" + "fmla v20.4s, v25.4s, v3.s[1]\n" + "ldr q25, [x10, #0x10]\n" + "fmla v9.4s, v24.4s, v0.s[1]\n" + "fmla v13.4s, v24.4s, v1.s[1]\n" + "fmla v17.4s, v24.4s, v2.s[1]\n" + "fmla v21.4s, v24.4s, v3.s[1]\n" + "ldr q24, [x9, #0x10]\n" + "fmla v10.4s, v25.4s, v0.s[1]\n" + "fmla v14.4s, v25.4s, v1.s[1]\n" + "fmla v18.4s, v25.4s, v2.s[1]\n" + "fmla v22.4s, v25.4s, v3.s[1]\n" + "ldr q25, [x12, #0x20]\n" + "fmla v11.4s, v24.4s, v0.s[1]\n" + "fmla v15.4s, v24.4s, v1.s[1]\n" + "fmla v19.4s, v24.4s, v2.s[1]\n" + "fmla v23.4s, v24.4s, v3.s[1]\n" + "ldr q24, [x11, #0x20]\n" + "fmla v8.4s, v25.4s, v0.s[2]\n" + "fmla v12.4s, v25.4s, v1.s[2]\n" + "fmla v16.4s, v25.4s, v2.s[2]\n" + "fmla v20.4s, v25.4s, v3.s[2]\n" + "ldr q25, [x10, #0x20]\n" + "fmla v9.4s, v24.4s, v0.s[2]\n" + "fmla v13.4s, v24.4s, v1.s[2]\n" + "fmla v17.4s, v24.4s, v2.s[2]\n" + "fmla v21.4s, v24.4s, v3.s[2]\n" + "ldr q24, [x9, #0x20]\n" + "fmla v10.4s, v25.4s, v0.s[2]\n" + "fmla v14.4s, v25.4s, v1.s[2]\n" + "fmla v18.4s, v25.4s, v2.s[2]\n" + "fmla v22.4s, v25.4s, v3.s[2]\n" + "ldr q25, [x12, #0x30]\n" "add x12, x12, #0x40\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "fmla v15.4s, v7.4s, v1.s[2]\n" - "fmla v19.4s, v7.4s, v2.s[2]\n" - "fmla v23.4s, v7.4s, v3.s[2]\n" - "ldr q7, [x11, #0x30]\n" + "fmla v11.4s, v24.4s, v0.s[2]\n" + "fmla v15.4s, v24.4s, v1.s[2]\n" + "fmla v19.4s, v24.4s, v2.s[2]\n" + "fmla v23.4s, v24.4s, v3.s[2]\n" + "ldr q24, [x11, #0x30]\n" "add x11, x11, #0x40\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v12.4s, v6.4s, v1.s[3]\n" - "fmla v16.4s, v6.4s, v2.s[3]\n" - "fmla v20.4s, v6.4s, v3.s[3]\n" - "ldr q6, [x10, #0x30]\n" + "fmla v8.4s, v25.4s, v0.s[3]\n" + "fmla v12.4s, v25.4s, v1.s[3]\n" + "fmla v16.4s, v25.4s, v2.s[3]\n" + "fmla v20.4s, v25.4s, v3.s[3]\n" + "ldr q25, [x10, #0x30]\n" "add x10, x10, #0x40\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "fmla v13.4s, v7.4s, v1.s[3]\n" - "fmla v17.4s, v7.4s, v2.s[3]\n" - "fmla v21.4s, v7.4s, v3.s[3]\n" - "ldr q7, [x9, #0x30]\n" + "fmla v9.4s, v24.4s, v0.s[3]\n" + "fmla v13.4s, v24.4s, v1.s[3]\n" + "fmla v17.4s, v24.4s, v2.s[3]\n" + "fmla v21.4s, v24.4s, v3.s[3]\n" + "ldr q24, [x9, #0x30]\n" "add x9, x9, #0x40\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "fmla v18.4s, v6.4s, v2.s[3]\n" - "fmla v22.4s, v6.4s, v3.s[3]\n" + "fmla v10.4s, v25.4s, v0.s[3]\n" + "fmla v14.4s, v25.4s, v1.s[3]\n" + "fmla v18.4s, v25.4s, v2.s[3]\n" + "fmla v22.4s, v25.4s, v3.s[3]\n" "ldr q6, [x12, #0x0]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" + "fmla v11.4s, v24.4s, v0.s[3]\n" "ldr q0, [x26, #0x0]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" + "fmla v15.4s, v24.4s, v1.s[3]\n" "ldr q1, [x25, #0x0]\n" - "fmla v19.4s, v7.4s, v2.s[3]\n" + "fmla v19.4s, v24.4s, v2.s[3]\n" "ldr q2, [x24, #0x0]\n" - "fmla v23.4s, v7.4s, v3.s[3]\n" + "fmla v23.4s, v24.4s, v3.s[3]\n" "ldr q3, [x23, #0x0]\n" "ldr q7, [x11, #0x0]\n" "bge 121b\n" @@ -1638,7 +1638,7 @@ void a64_ffhybrid_fp32_mla_6x16 ( "add x26, x26, #0x10\n" "fmla v16.4s, v6.4s, v2.s[0]\n" "fmla v20.4s, v6.4s, v3.s[0]\n" - "ldr q6, [x10, #0x0]\n" + "ldr q25, [x10, #0x0]\n" "add x25, x25, #0x10\n" "fmla v9.4s, v7.4s, v0.s[0]\n" "fmla v13.4s, v7.4s, v1.s[0]\n" @@ -1646,111 +1646,111 @@ void a64_ffhybrid_fp32_mla_6x16 ( "add x23, x23, #0x10\n" "fmla v17.4s, v7.4s, v2.s[0]\n" "fmla v21.4s, v7.4s, v3.s[0]\n" - "ldr q7, [x9, #0x0]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "fmla v22.4s, v6.4s, v3.s[0]\n" - "ldr q6, [x12, #0x10]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "fmla v23.4s, v7.4s, v3.s[0]\n" - "ldr q7, [x11, #0x10]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v12.4s, v6.4s, v1.s[1]\n" - "fmla v16.4s, v6.4s, v2.s[1]\n" - "fmla v20.4s, v6.4s, v3.s[1]\n" - "ldr q6, [x10, #0x10]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v13.4s, v7.4s, v1.s[1]\n" - "fmla v17.4s, v7.4s, v2.s[1]\n" - "fmla v21.4s, v7.4s, v3.s[1]\n" - "ldr q7, [x9, #0x10]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "fmla v18.4s, v6.4s, v2.s[1]\n" - "fmla v22.4s, v6.4s, v3.s[1]\n" - "ldr q6, [x12, #0x20]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "fmla v19.4s, v7.4s, v2.s[1]\n" - "fmla v23.4s, v7.4s, v3.s[1]\n" - "ldr q7, [x11, #0x20]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "fmla v12.4s, v6.4s, v1.s[2]\n" - "fmla v16.4s, v6.4s, v2.s[2]\n" - "fmla v20.4s, v6.4s, v3.s[2]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "fmla v13.4s, v7.4s, v1.s[2]\n" - "fmla v17.4s, v7.4s, v2.s[2]\n" - "fmla v21.4s, v7.4s, v3.s[2]\n" - "ldr q7, [x9, #0x20]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v14.4s, v6.4s, v1.s[2]\n" - "fmla v18.4s, v6.4s, v2.s[2]\n" - "fmla v22.4s, v6.4s, v3.s[2]\n" - "ldr q6, [x12, #0x30]\n" + "ldr q24, [x9, #0x0]\n" + "fmla v10.4s, v25.4s, v0.s[0]\n" + "fmla v14.4s, v25.4s, v1.s[0]\n" + "fmla v18.4s, v25.4s, v2.s[0]\n" + "fmla v22.4s, v25.4s, v3.s[0]\n" + "ldr q25, [x12, #0x10]\n" + "fmla v11.4s, v24.4s, v0.s[0]\n" + "fmla v15.4s, v24.4s, v1.s[0]\n" + "fmla v19.4s, v24.4s, v2.s[0]\n" + "fmla v23.4s, v24.4s, v3.s[0]\n" + "ldr q24, [x11, #0x10]\n" + "fmla v8.4s, v25.4s, v0.s[1]\n" + "fmla v12.4s, v25.4s, v1.s[1]\n" + "fmla v16.4s, v25.4s, v2.s[1]\n" + "fmla v20.4s, v25.4s, v3.s[1]\n" + "ldr q25, [x10, #0x10]\n" + "fmla v9.4s, v24.4s, v0.s[1]\n" + "fmla v13.4s, v24.4s, v1.s[1]\n" + "fmla v17.4s, v24.4s, v2.s[1]\n" + "fmla v21.4s, v24.4s, v3.s[1]\n" + "ldr q24, [x9, #0x10]\n" + "fmla v10.4s, v25.4s, v0.s[1]\n" + "fmla v14.4s, v25.4s, v1.s[1]\n" + "fmla v18.4s, v25.4s, v2.s[1]\n" + "fmla v22.4s, v25.4s, v3.s[1]\n" + "ldr q25, [x12, #0x20]\n" + "fmla v11.4s, v24.4s, v0.s[1]\n" + "fmla v15.4s, v24.4s, v1.s[1]\n" + "fmla v19.4s, v24.4s, v2.s[1]\n" + "fmla v23.4s, v24.4s, v3.s[1]\n" + "ldr q24, [x11, #0x20]\n" + "fmla v8.4s, v25.4s, v0.s[2]\n" + "fmla v12.4s, v25.4s, v1.s[2]\n" + "fmla v16.4s, v25.4s, v2.s[2]\n" + "fmla v20.4s, v25.4s, v3.s[2]\n" + "ldr q25, [x10, #0x20]\n" + "fmla v9.4s, v24.4s, v0.s[2]\n" + "fmla v13.4s, v24.4s, v1.s[2]\n" + "fmla v17.4s, v24.4s, v2.s[2]\n" + "fmla v21.4s, v24.4s, v3.s[2]\n" + "ldr q24, [x9, #0x20]\n" + "fmla v10.4s, v25.4s, v0.s[2]\n" + "fmla v14.4s, v25.4s, v1.s[2]\n" + "fmla v18.4s, v25.4s, v2.s[2]\n" + "fmla v22.4s, v25.4s, v3.s[2]\n" + "ldr q25, [x12, #0x30]\n" "add x12, x12, #0x40\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "fmla v15.4s, v7.4s, v1.s[2]\n" - "fmla v19.4s, v7.4s, v2.s[2]\n" - "fmla v23.4s, v7.4s, v3.s[2]\n" - "ldr q7, [x11, #0x30]\n" + "fmla v11.4s, v24.4s, v0.s[2]\n" + "fmla v15.4s, v24.4s, v1.s[2]\n" + "fmla v19.4s, v24.4s, v2.s[2]\n" + "fmla v23.4s, v24.4s, v3.s[2]\n" + "ldr q24, [x11, #0x30]\n" "add x11, x11, #0x40\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v12.4s, v6.4s, v1.s[3]\n" - "fmla v16.4s, v6.4s, v2.s[3]\n" - "fmla v20.4s, v6.4s, v3.s[3]\n" - "ldr q6, [x10, #0x30]\n" + "fmla v8.4s, v25.4s, v0.s[3]\n" + "fmla v12.4s, v25.4s, v1.s[3]\n" + "fmla v16.4s, v25.4s, v2.s[3]\n" + "fmla v20.4s, v25.4s, v3.s[3]\n" + "ldr q25, [x10, #0x30]\n" "add x10, x10, #0x40\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "fmla v13.4s, v7.4s, v1.s[3]\n" - "fmla v17.4s, v7.4s, v2.s[3]\n" - "fmla v21.4s, v7.4s, v3.s[3]\n" - "ldr q7, [x9, #0x30]\n" + "fmla v9.4s, v24.4s, v0.s[3]\n" + "fmla v13.4s, v24.4s, v1.s[3]\n" + "fmla v17.4s, v24.4s, v2.s[3]\n" + "fmla v21.4s, v24.4s, v3.s[3]\n" + "ldr q24, [x9, #0x30]\n" "add x9, x9, #0x40\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "fmla v18.4s, v6.4s, v2.s[3]\n" - "fmla v22.4s, v6.4s, v3.s[3]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" - "fmla v19.4s, v7.4s, v2.s[3]\n" - "fmla v23.4s, v7.4s, v3.s[3]\n" + "fmla v10.4s, v25.4s, v0.s[3]\n" + "fmla v14.4s, v25.4s, v1.s[3]\n" + "fmla v18.4s, v25.4s, v2.s[3]\n" + "fmla v22.4s, v25.4s, v3.s[3]\n" + "fmla v11.4s, v24.4s, v0.s[3]\n" + "fmla v15.4s, v24.4s, v1.s[3]\n" + "fmla v19.4s, v24.4s, v2.s[3]\n" + "fmla v23.4s, v24.4s, v3.s[3]\n" "123:" // Height 4: Multiply loop: Main loop skip "cbz x27, 125f\n" "124:" // Height 4: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s29, [x26], #0x4\n" + "ldr s28, [x25], #0x4\n" "sub x27, x27, #0x1\n" - "ldr s2, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr q6, [x12, #0x0]\n" - "ldr q7, [x11, #0x0]\n" - "fmla v8.4s, v6.4s, v0.s[0]\n" - "fmla v12.4s, v6.4s, v1.s[0]\n" - "fmla v16.4s, v6.4s, v2.s[0]\n" - "fmla v20.4s, v6.4s, v3.s[0]\n" - "ldr q6, [x10, #0x0]\n" + "ldr s27, [x24], #0x4\n" + "ldr s26, [x23], #0x4\n" + "ldr q25, [x12, #0x0]\n" + "ldr q24, [x11, #0x0]\n" + "fmla v8.4s, v25.4s, v29.s[0]\n" + "fmla v12.4s, v25.4s, v28.s[0]\n" + "fmla v16.4s, v25.4s, v27.s[0]\n" + "fmla v20.4s, v25.4s, v26.s[0]\n" + "ldr q25, [x10, #0x0]\n" "add x12, x12, #0x10\n" - "fmla v9.4s, v7.4s, v0.s[0]\n" - "fmla v13.4s, v7.4s, v1.s[0]\n" + "fmla v9.4s, v24.4s, v29.s[0]\n" + "fmla v13.4s, v24.4s, v28.s[0]\n" "add x11, x11, #0x10\n" "add x10, x10, #0x10\n" - "fmla v17.4s, v7.4s, v2.s[0]\n" - "fmla v21.4s, v7.4s, v3.s[0]\n" - "ldr q7, [x9, #0x0]\n" + "fmla v17.4s, v24.4s, v27.s[0]\n" + "fmla v21.4s, v24.4s, v26.s[0]\n" + "ldr q24, [x9, #0x0]\n" "add x9, x9, #0x10\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "fmla v22.4s, v6.4s, v3.s[0]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "fmla v23.4s, v7.4s, v3.s[0]\n" + "fmla v10.4s, v25.4s, v29.s[0]\n" + "fmla v14.4s, v25.4s, v28.s[0]\n" + "fmla v18.4s, v25.4s, v27.s[0]\n" + "fmla v22.4s, v25.4s, v26.s[0]\n" + "fmla v11.4s, v24.4s, v29.s[0]\n" + "fmla v15.4s, v24.4s, v28.s[0]\n" + "fmla v19.4s, v24.4s, v27.s[0]\n" + "fmla v23.4s, v24.4s, v26.s[0]\n" "cbnz x27, 124b\n" "125:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -1763,41 +1763,41 @@ void a64_ffhybrid_fp32_mla_6x16 ( "add x23, x24, x20, LSL #2\n" "tbz %x[flags], #1, 126f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v25.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v15.4s, v15.4s, v1.4s\n" - "fmin v16.4s, v16.4s, v1.4s\n" - "fmin v17.4s, v17.4s, v1.4s\n" - "fmin v18.4s, v18.4s, v1.4s\n" - "fmin v19.4s, v19.4s, v1.4s\n" - "fmin v20.4s, v20.4s, v1.4s\n" - "fmin v21.4s, v21.4s, v1.4s\n" - "fmin v22.4s, v22.4s, v1.4s\n" - "fmin v23.4s, v23.4s, v1.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" - "fmax v16.4s, v16.4s, v0.4s\n" - "fmax v17.4s, v17.4s, v0.4s\n" - "fmax v18.4s, v18.4s, v0.4s\n" - "fmax v19.4s, v19.4s, v0.4s\n" - "fmax v20.4s, v20.4s, v0.4s\n" - "fmax v21.4s, v21.4s, v0.4s\n" - "fmax v22.4s, v22.4s, v0.4s\n" - "fmax v23.4s, v23.4s, v0.4s\n" + "ld1r { v24.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v25.4s\n" + "fmin v9.4s, v9.4s, v25.4s\n" + "fmin v10.4s, v10.4s, v25.4s\n" + "fmin v11.4s, v11.4s, v25.4s\n" + "fmin v12.4s, v12.4s, v25.4s\n" + "fmin v13.4s, v13.4s, v25.4s\n" + "fmin v14.4s, v14.4s, v25.4s\n" + "fmin v15.4s, v15.4s, v25.4s\n" + "fmin v16.4s, v16.4s, v25.4s\n" + "fmin v17.4s, v17.4s, v25.4s\n" + "fmin v18.4s, v18.4s, v25.4s\n" + "fmin v19.4s, v19.4s, v25.4s\n" + "fmin v20.4s, v20.4s, v25.4s\n" + "fmin v21.4s, v21.4s, v25.4s\n" + "fmin v22.4s, v22.4s, v25.4s\n" + "fmin v23.4s, v23.4s, v25.4s\n" + "fmax v8.4s, v8.4s, v24.4s\n" + "fmax v9.4s, v9.4s, v24.4s\n" + "fmax v10.4s, v10.4s, v24.4s\n" + "fmax v11.4s, v11.4s, v24.4s\n" + "fmax v12.4s, v12.4s, v24.4s\n" + "fmax v13.4s, v13.4s, v24.4s\n" + "fmax v14.4s, v14.4s, v24.4s\n" + "fmax v15.4s, v15.4s, v24.4s\n" + "fmax v16.4s, v16.4s, v24.4s\n" + "fmax v17.4s, v17.4s, v24.4s\n" + "fmax v18.4s, v18.4s, v24.4s\n" + "fmax v19.4s, v19.4s, v24.4s\n" + "fmax v20.4s, v20.4s, v24.4s\n" + "fmax v21.4s, v21.4s, v24.4s\n" + "fmax v22.4s, v22.4s, v24.4s\n" + "fmax v23.4s, v23.4s, v24.4s\n" "126:" // Height 4: No activation "cmp x14, #0x10\n" "bge 135f\n" @@ -2137,155 +2137,155 @@ void a64_ffhybrid_fp32_mla_6x16 ( "movi v27.16b, #0x0\n" "151:" // Height 5: setup done "mov x28, #0x0\n" - "152:" // Height 5: String loop - "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" - "tbz %x[flags], #3, 153f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "cbnz x28, 154f\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x26, x26, x20, LSL #2\n" - "add x25, x25, x20, LSL #2\n" - "add x24, x24, x20, LSL #2\n" - "add x23, x23, x20, LSL #2\n" - "add x22, x22, x20, LSL #2\n" - "b 154f\n" - "153:" // Height 5: setup direct input - "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "154:" // Height 5: input setup done - "cmp x27, #0x4\n" - "blt 157f\n" - "ldr q0, [x26, #0x0]\n" - "ldr q1, [x25, #0x0]\n" - "cmp x27, #0x8\n" - "ldr q2, [x24, #0x0]\n" - "ldr q3, [x23, #0x0]\n" - "ldr q4, [x22, #0x0]\n" - "ldr q6, [x12, #0x0]\n" - "ldr q7, [x11, #0x0]\n" - "blt 156f\n" - "155:" // Height 5: Multiply loop: Main loop head - "fmla v8.4s, v6.4s, v0.s[0]\n" - "fmla v12.4s, v6.4s, v1.s[0]\n" - "sub x27, x27, #0x4\n" - "cmp x27, #0x8\n" - "fmla v16.4s, v6.4s, v2.s[0]\n" - "fmla v20.4s, v6.4s, v3.s[0]\n" - "add x26, x26, #0x10\n" - "add x25, x25, #0x10\n" - "fmla v24.4s, v6.4s, v4.s[0]\n" - "ldr q6, [x10, #0x0]\n" - "fmla v9.4s, v7.4s, v0.s[0]\n" - "add x24, x24, #0x10\n" - "fmla v13.4s, v7.4s, v1.s[0]\n" - "fmla v17.4s, v7.4s, v2.s[0]\n" - "add x23, x23, #0x10\n" - "add x22, x22, #0x10\n" - "fmla v21.4s, v7.4s, v3.s[0]\n" - "fmla v25.4s, v7.4s, v4.s[0]\n" - "ldr q7, [x9, #0x0]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "fmla v22.4s, v6.4s, v3.s[0]\n" - "fmla v26.4s, v6.4s, v4.s[0]\n" - "ldr q6, [x12, #0x10]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "fmla v23.4s, v7.4s, v3.s[0]\n" - "fmla v27.4s, v7.4s, v4.s[0]\n" - "ldr q7, [x11, #0x10]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v12.4s, v6.4s, v1.s[1]\n" - "fmla v16.4s, v6.4s, v2.s[1]\n" - "fmla v20.4s, v6.4s, v3.s[1]\n" - "fmla v24.4s, v6.4s, v4.s[1]\n" - "ldr q6, [x10, #0x10]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v13.4s, v7.4s, v1.s[1]\n" - "fmla v17.4s, v7.4s, v2.s[1]\n" - "fmla v21.4s, v7.4s, v3.s[1]\n" - "fmla v25.4s, v7.4s, v4.s[1]\n" - "ldr q7, [x9, #0x10]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "fmla v18.4s, v6.4s, v2.s[1]\n" - "fmla v22.4s, v6.4s, v3.s[1]\n" - "fmla v26.4s, v6.4s, v4.s[1]\n" - "ldr q6, [x12, #0x20]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "fmla v19.4s, v7.4s, v2.s[1]\n" - "fmla v23.4s, v7.4s, v3.s[1]\n" - "fmla v27.4s, v7.4s, v4.s[1]\n" - "ldr q7, [x11, #0x20]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "fmla v12.4s, v6.4s, v1.s[2]\n" - "fmla v16.4s, v6.4s, v2.s[2]\n" - "fmla v20.4s, v6.4s, v3.s[2]\n" - "fmla v24.4s, v6.4s, v4.s[2]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "fmla v13.4s, v7.4s, v1.s[2]\n" - "fmla v17.4s, v7.4s, v2.s[2]\n" - "fmla v21.4s, v7.4s, v3.s[2]\n" - "fmla v25.4s, v7.4s, v4.s[2]\n" - "ldr q7, [x9, #0x20]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v14.4s, v6.4s, v1.s[2]\n" - "fmla v18.4s, v6.4s, v2.s[2]\n" - "fmla v22.4s, v6.4s, v3.s[2]\n" - "fmla v26.4s, v6.4s, v4.s[2]\n" - "ldr q6, [x12, #0x30]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" + "152:" // Height 5: String loop + "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" + "ldr w27, [x20, x28, LSL #0x2]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" + "tbz %x[flags], #3, 153f\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "cbnz x28, 154f\n" + "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" + "add x26, x26, x20, LSL #2\n" + "add x25, x25, x20, LSL #2\n" + "add x24, x24, x20, LSL #2\n" + "add x23, x23, x20, LSL #2\n" + "add x22, x22, x20, LSL #2\n" + "b 154f\n" + "153:" // Height 5: setup direct input + "mov x26, %x[input_ptr]\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" + "154:" // Height 5: input setup done + "cmp x27, #0x4\n" + "blt 157f\n" + "ldr q0, [x26, #0x0]\n" + "ldr q1, [x25, #0x0]\n" + "cmp x27, #0x8\n" + "ldr q2, [x24, #0x0]\n" + "ldr q3, [x23, #0x0]\n" + "ldr q4, [x22, #0x0]\n" + "ldr q6, [x12, #0x0]\n" + "ldr q7, [x11, #0x0]\n" + "blt 156f\n" + "155:" // Height 5: Multiply loop: Main loop head + "fmla v8.4s, v6.4s, v0.s[0]\n" + "fmla v12.4s, v6.4s, v1.s[0]\n" + "sub x27, x27, #0x4\n" + "cmp x27, #0x8\n" + "fmla v16.4s, v6.4s, v2.s[0]\n" + "fmla v20.4s, v6.4s, v3.s[0]\n" + "add x26, x26, #0x10\n" + "add x25, x25, #0x10\n" + "fmla v24.4s, v6.4s, v4.s[0]\n" + "ldr q29, [x10, #0x0]\n" + "fmla v9.4s, v7.4s, v0.s[0]\n" + "add x24, x24, #0x10\n" + "fmla v13.4s, v7.4s, v1.s[0]\n" + "fmla v17.4s, v7.4s, v2.s[0]\n" + "add x23, x23, #0x10\n" + "add x22, x22, #0x10\n" + "fmla v21.4s, v7.4s, v3.s[0]\n" + "fmla v25.4s, v7.4s, v4.s[0]\n" + "ldr q28, [x9, #0x0]\n" + "fmla v10.4s, v29.4s, v0.s[0]\n" + "fmla v14.4s, v29.4s, v1.s[0]\n" + "fmla v18.4s, v29.4s, v2.s[0]\n" + "fmla v22.4s, v29.4s, v3.s[0]\n" + "fmla v26.4s, v29.4s, v4.s[0]\n" + "ldr q29, [x12, #0x10]\n" + "fmla v11.4s, v28.4s, v0.s[0]\n" + "fmla v15.4s, v28.4s, v1.s[0]\n" + "fmla v19.4s, v28.4s, v2.s[0]\n" + "fmla v23.4s, v28.4s, v3.s[0]\n" + "fmla v27.4s, v28.4s, v4.s[0]\n" + "ldr q28, [x11, #0x10]\n" + "fmla v8.4s, v29.4s, v0.s[1]\n" + "fmla v12.4s, v29.4s, v1.s[1]\n" + "fmla v16.4s, v29.4s, v2.s[1]\n" + "fmla v20.4s, v29.4s, v3.s[1]\n" + "fmla v24.4s, v29.4s, v4.s[1]\n" + "ldr q29, [x10, #0x10]\n" + "fmla v9.4s, v28.4s, v0.s[1]\n" + "fmla v13.4s, v28.4s, v1.s[1]\n" + "fmla v17.4s, v28.4s, v2.s[1]\n" + "fmla v21.4s, v28.4s, v3.s[1]\n" + "fmla v25.4s, v28.4s, v4.s[1]\n" + "ldr q28, [x9, #0x10]\n" + "fmla v10.4s, v29.4s, v0.s[1]\n" + "fmla v14.4s, v29.4s, v1.s[1]\n" + "fmla v18.4s, v29.4s, v2.s[1]\n" + "fmla v22.4s, v29.4s, v3.s[1]\n" + "fmla v26.4s, v29.4s, v4.s[1]\n" + "ldr q29, [x12, #0x20]\n" + "fmla v11.4s, v28.4s, v0.s[1]\n" + "fmla v15.4s, v28.4s, v1.s[1]\n" + "fmla v19.4s, v28.4s, v2.s[1]\n" + "fmla v23.4s, v28.4s, v3.s[1]\n" + "fmla v27.4s, v28.4s, v4.s[1]\n" + "ldr q28, [x11, #0x20]\n" + "fmla v8.4s, v29.4s, v0.s[2]\n" + "fmla v12.4s, v29.4s, v1.s[2]\n" + "fmla v16.4s, v29.4s, v2.s[2]\n" + "fmla v20.4s, v29.4s, v3.s[2]\n" + "fmla v24.4s, v29.4s, v4.s[2]\n" + "ldr q29, [x10, #0x20]\n" + "fmla v9.4s, v28.4s, v0.s[2]\n" + "fmla v13.4s, v28.4s, v1.s[2]\n" + "fmla v17.4s, v28.4s, v2.s[2]\n" + "fmla v21.4s, v28.4s, v3.s[2]\n" + "fmla v25.4s, v28.4s, v4.s[2]\n" + "ldr q28, [x9, #0x20]\n" + "fmla v10.4s, v29.4s, v0.s[2]\n" + "fmla v14.4s, v29.4s, v1.s[2]\n" + "fmla v18.4s, v29.4s, v2.s[2]\n" + "fmla v22.4s, v29.4s, v3.s[2]\n" + "fmla v26.4s, v29.4s, v4.s[2]\n" + "ldr q29, [x12, #0x30]\n" + "fmla v11.4s, v28.4s, v0.s[2]\n" "add x12, x12, #0x40\n" - "fmla v15.4s, v7.4s, v1.s[2]\n" - "fmla v19.4s, v7.4s, v2.s[2]\n" - "fmla v23.4s, v7.4s, v3.s[2]\n" - "fmla v27.4s, v7.4s, v4.s[2]\n" - "ldr q7, [x11, #0x30]\n" + "fmla v15.4s, v28.4s, v1.s[2]\n" + "fmla v19.4s, v28.4s, v2.s[2]\n" + "fmla v23.4s, v28.4s, v3.s[2]\n" + "fmla v27.4s, v28.4s, v4.s[2]\n" + "ldr q28, [x11, #0x30]\n" "add x11, x11, #0x40\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v12.4s, v6.4s, v1.s[3]\n" - "fmla v16.4s, v6.4s, v2.s[3]\n" - "fmla v20.4s, v6.4s, v3.s[3]\n" - "fmla v24.4s, v6.4s, v4.s[3]\n" - "ldr q6, [x10, #0x30]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" + "fmla v8.4s, v29.4s, v0.s[3]\n" + "fmla v12.4s, v29.4s, v1.s[3]\n" + "fmla v16.4s, v29.4s, v2.s[3]\n" + "fmla v20.4s, v29.4s, v3.s[3]\n" + "fmla v24.4s, v29.4s, v4.s[3]\n" + "ldr q29, [x10, #0x30]\n" + "fmla v9.4s, v28.4s, v0.s[3]\n" "add x10, x10, #0x40\n" - "fmla v13.4s, v7.4s, v1.s[3]\n" - "fmla v17.4s, v7.4s, v2.s[3]\n" - "fmla v21.4s, v7.4s, v3.s[3]\n" - "fmla v25.4s, v7.4s, v4.s[3]\n" - "ldr q7, [x9, #0x30]\n" + "fmla v13.4s, v28.4s, v1.s[3]\n" + "fmla v17.4s, v28.4s, v2.s[3]\n" + "fmla v21.4s, v28.4s, v3.s[3]\n" + "fmla v25.4s, v28.4s, v4.s[3]\n" + "ldr q28, [x9, #0x30]\n" "add x9, x9, #0x40\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "fmla v18.4s, v6.4s, v2.s[3]\n" - "fmla v22.4s, v6.4s, v3.s[3]\n" - "fmla v26.4s, v6.4s, v4.s[3]\n" + "fmla v10.4s, v29.4s, v0.s[3]\n" + "fmla v14.4s, v29.4s, v1.s[3]\n" + "fmla v18.4s, v29.4s, v2.s[3]\n" + "fmla v22.4s, v29.4s, v3.s[3]\n" + "fmla v26.4s, v29.4s, v4.s[3]\n" "ldr q6, [x12, #0x0]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" + "fmla v11.4s, v28.4s, v0.s[3]\n" "ldr q0, [x26, #0x0]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" + "fmla v15.4s, v28.4s, v1.s[3]\n" "ldr q1, [x25, #0x0]\n" - "fmla v19.4s, v7.4s, v2.s[3]\n" + "fmla v19.4s, v28.4s, v2.s[3]\n" "ldr q2, [x24, #0x0]\n" - "fmla v23.4s, v7.4s, v3.s[3]\n" + "fmla v23.4s, v28.4s, v3.s[3]\n" "ldr q3, [x23, #0x0]\n" - "fmla v27.4s, v7.4s, v4.s[3]\n" + "fmla v27.4s, v28.4s, v4.s[3]\n" "ldr q4, [x22, #0x0]\n" "ldr q7, [x11, #0x0]\n" "bge 155b\n" @@ -2299,7 +2299,7 @@ void a64_ffhybrid_fp32_mla_6x16 ( "add x25, x25, #0x10\n" "add x24, x24, #0x10\n" "fmla v24.4s, v6.4s, v4.s[0]\n" - "ldr q6, [x10, #0x0]\n" + "ldr q29, [x10, #0x0]\n" "fmla v9.4s, v7.4s, v0.s[0]\n" "add x23, x23, #0x10\n" "fmla v13.4s, v7.4s, v1.s[0]\n" @@ -2307,130 +2307,130 @@ void a64_ffhybrid_fp32_mla_6x16 ( "add x22, x22, #0x10\n" "fmla v21.4s, v7.4s, v3.s[0]\n" "fmla v25.4s, v7.4s, v4.s[0]\n" - "ldr q7, [x9, #0x0]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "fmla v22.4s, v6.4s, v3.s[0]\n" - "fmla v26.4s, v6.4s, v4.s[0]\n" - "ldr q6, [x12, #0x10]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "fmla v23.4s, v7.4s, v3.s[0]\n" - "fmla v27.4s, v7.4s, v4.s[0]\n" - "ldr q7, [x11, #0x10]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v12.4s, v6.4s, v1.s[1]\n" - "fmla v16.4s, v6.4s, v2.s[1]\n" - "fmla v20.4s, v6.4s, v3.s[1]\n" - "fmla v24.4s, v6.4s, v4.s[1]\n" - "ldr q6, [x10, #0x10]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v13.4s, v7.4s, v1.s[1]\n" - "fmla v17.4s, v7.4s, v2.s[1]\n" - "fmla v21.4s, v7.4s, v3.s[1]\n" - "fmla v25.4s, v7.4s, v4.s[1]\n" - "ldr q7, [x9, #0x10]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "fmla v18.4s, v6.4s, v2.s[1]\n" - "fmla v22.4s, v6.4s, v3.s[1]\n" - "fmla v26.4s, v6.4s, v4.s[1]\n" - "ldr q6, [x12, #0x20]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "fmla v19.4s, v7.4s, v2.s[1]\n" - "fmla v23.4s, v7.4s, v3.s[1]\n" - "fmla v27.4s, v7.4s, v4.s[1]\n" - "ldr q7, [x11, #0x20]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "fmla v12.4s, v6.4s, v1.s[2]\n" - "fmla v16.4s, v6.4s, v2.s[2]\n" - "fmla v20.4s, v6.4s, v3.s[2]\n" - "fmla v24.4s, v6.4s, v4.s[2]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "fmla v13.4s, v7.4s, v1.s[2]\n" - "fmla v17.4s, v7.4s, v2.s[2]\n" - "fmla v21.4s, v7.4s, v3.s[2]\n" - "fmla v25.4s, v7.4s, v4.s[2]\n" - "ldr q7, [x9, #0x20]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v14.4s, v6.4s, v1.s[2]\n" - "fmla v18.4s, v6.4s, v2.s[2]\n" - "fmla v22.4s, v6.4s, v3.s[2]\n" - "fmla v26.4s, v6.4s, v4.s[2]\n" - "ldr q6, [x12, #0x30]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" + "ldr q28, [x9, #0x0]\n" + "fmla v10.4s, v29.4s, v0.s[0]\n" + "fmla v14.4s, v29.4s, v1.s[0]\n" + "fmla v18.4s, v29.4s, v2.s[0]\n" + "fmla v22.4s, v29.4s, v3.s[0]\n" + "fmla v26.4s, v29.4s, v4.s[0]\n" + "ldr q29, [x12, #0x10]\n" + "fmla v11.4s, v28.4s, v0.s[0]\n" + "fmla v15.4s, v28.4s, v1.s[0]\n" + "fmla v19.4s, v28.4s, v2.s[0]\n" + "fmla v23.4s, v28.4s, v3.s[0]\n" + "fmla v27.4s, v28.4s, v4.s[0]\n" + "ldr q28, [x11, #0x10]\n" + "fmla v8.4s, v29.4s, v0.s[1]\n" + "fmla v12.4s, v29.4s, v1.s[1]\n" + "fmla v16.4s, v29.4s, v2.s[1]\n" + "fmla v20.4s, v29.4s, v3.s[1]\n" + "fmla v24.4s, v29.4s, v4.s[1]\n" + "ldr q29, [x10, #0x10]\n" + "fmla v9.4s, v28.4s, v0.s[1]\n" + "fmla v13.4s, v28.4s, v1.s[1]\n" + "fmla v17.4s, v28.4s, v2.s[1]\n" + "fmla v21.4s, v28.4s, v3.s[1]\n" + "fmla v25.4s, v28.4s, v4.s[1]\n" + "ldr q28, [x9, #0x10]\n" + "fmla v10.4s, v29.4s, v0.s[1]\n" + "fmla v14.4s, v29.4s, v1.s[1]\n" + "fmla v18.4s, v29.4s, v2.s[1]\n" + "fmla v22.4s, v29.4s, v3.s[1]\n" + "fmla v26.4s, v29.4s, v4.s[1]\n" + "ldr q29, [x12, #0x20]\n" + "fmla v11.4s, v28.4s, v0.s[1]\n" + "fmla v15.4s, v28.4s, v1.s[1]\n" + "fmla v19.4s, v28.4s, v2.s[1]\n" + "fmla v23.4s, v28.4s, v3.s[1]\n" + "fmla v27.4s, v28.4s, v4.s[1]\n" + "ldr q28, [x11, #0x20]\n" + "fmla v8.4s, v29.4s, v0.s[2]\n" + "fmla v12.4s, v29.4s, v1.s[2]\n" + "fmla v16.4s, v29.4s, v2.s[2]\n" + "fmla v20.4s, v29.4s, v3.s[2]\n" + "fmla v24.4s, v29.4s, v4.s[2]\n" + "ldr q29, [x10, #0x20]\n" + "fmla v9.4s, v28.4s, v0.s[2]\n" + "fmla v13.4s, v28.4s, v1.s[2]\n" + "fmla v17.4s, v28.4s, v2.s[2]\n" + "fmla v21.4s, v28.4s, v3.s[2]\n" + "fmla v25.4s, v28.4s, v4.s[2]\n" + "ldr q28, [x9, #0x20]\n" + "fmla v10.4s, v29.4s, v0.s[2]\n" + "fmla v14.4s, v29.4s, v1.s[2]\n" + "fmla v18.4s, v29.4s, v2.s[2]\n" + "fmla v22.4s, v29.4s, v3.s[2]\n" + "fmla v26.4s, v29.4s, v4.s[2]\n" + "ldr q29, [x12, #0x30]\n" + "fmla v11.4s, v28.4s, v0.s[2]\n" "add x12, x12, #0x40\n" - "fmla v15.4s, v7.4s, v1.s[2]\n" - "fmla v19.4s, v7.4s, v2.s[2]\n" - "fmla v23.4s, v7.4s, v3.s[2]\n" - "fmla v27.4s, v7.4s, v4.s[2]\n" - "ldr q7, [x11, #0x30]\n" + "fmla v15.4s, v28.4s, v1.s[2]\n" + "fmla v19.4s, v28.4s, v2.s[2]\n" + "fmla v23.4s, v28.4s, v3.s[2]\n" + "fmla v27.4s, v28.4s, v4.s[2]\n" + "ldr q28, [x11, #0x30]\n" "add x11, x11, #0x40\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v12.4s, v6.4s, v1.s[3]\n" - "fmla v16.4s, v6.4s, v2.s[3]\n" - "fmla v20.4s, v6.4s, v3.s[3]\n" - "fmla v24.4s, v6.4s, v4.s[3]\n" - "ldr q6, [x10, #0x30]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" + "fmla v8.4s, v29.4s, v0.s[3]\n" + "fmla v12.4s, v29.4s, v1.s[3]\n" + "fmla v16.4s, v29.4s, v2.s[3]\n" + "fmla v20.4s, v29.4s, v3.s[3]\n" + "fmla v24.4s, v29.4s, v4.s[3]\n" + "ldr q29, [x10, #0x30]\n" + "fmla v9.4s, v28.4s, v0.s[3]\n" "add x10, x10, #0x40\n" - "fmla v13.4s, v7.4s, v1.s[3]\n" - "fmla v17.4s, v7.4s, v2.s[3]\n" - "fmla v21.4s, v7.4s, v3.s[3]\n" - "fmla v25.4s, v7.4s, v4.s[3]\n" - "ldr q7, [x9, #0x30]\n" + "fmla v13.4s, v28.4s, v1.s[3]\n" + "fmla v17.4s, v28.4s, v2.s[3]\n" + "fmla v21.4s, v28.4s, v3.s[3]\n" + "fmla v25.4s, v28.4s, v4.s[3]\n" + "ldr q28, [x9, #0x30]\n" "add x9, x9, #0x40\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "fmla v18.4s, v6.4s, v2.s[3]\n" - "fmla v22.4s, v6.4s, v3.s[3]\n" - "fmla v26.4s, v6.4s, v4.s[3]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" - "fmla v19.4s, v7.4s, v2.s[3]\n" - "fmla v23.4s, v7.4s, v3.s[3]\n" - "fmla v27.4s, v7.4s, v4.s[3]\n" + "fmla v10.4s, v29.4s, v0.s[3]\n" + "fmla v14.4s, v29.4s, v1.s[3]\n" + "fmla v18.4s, v29.4s, v2.s[3]\n" + "fmla v22.4s, v29.4s, v3.s[3]\n" + "fmla v26.4s, v29.4s, v4.s[3]\n" + "fmla v11.4s, v28.4s, v0.s[3]\n" + "fmla v15.4s, v28.4s, v1.s[3]\n" + "fmla v19.4s, v28.4s, v2.s[3]\n" + "fmla v23.4s, v28.4s, v3.s[3]\n" + "fmla v27.4s, v28.4s, v4.s[3]\n" "157:" // Height 5: Multiply loop: Main loop skip "cbz x27, 159f\n" "158:" // Height 5: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" + "ldr s2, [x26], #0x4\n" "ldr s1, [x25], #0x4\n" "sub x27, x27, #0x1\n" - "ldr s2, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr s4, [x22], #0x4\n" - "ldr q6, [x12, #0x0]\n" - "fmla v8.4s, v6.4s, v0.s[0]\n" - "fmla v12.4s, v6.4s, v1.s[0]\n" - "ldr q7, [x11, #0x0]\n" - "fmla v16.4s, v6.4s, v2.s[0]\n" - "fmla v20.4s, v6.4s, v3.s[0]\n" + "ldr s0, [x24], #0x4\n" + "ldr s31, [x23], #0x4\n" + "ldr s30, [x22], #0x4\n" + "ldr q29, [x12, #0x0]\n" + "fmla v8.4s, v29.4s, v2.s[0]\n" + "fmla v12.4s, v29.4s, v1.s[0]\n" + "ldr q28, [x11, #0x0]\n" + "fmla v16.4s, v29.4s, v0.s[0]\n" + "fmla v20.4s, v29.4s, v31.s[0]\n" "add x12, x12, #0x10\n" - "fmla v24.4s, v6.4s, v4.s[0]\n" - "ldr q6, [x10, #0x0]\n" - "fmla v9.4s, v7.4s, v0.s[0]\n" + "fmla v24.4s, v29.4s, v30.s[0]\n" + "ldr q29, [x10, #0x0]\n" + "fmla v9.4s, v28.4s, v2.s[0]\n" "add x11, x11, #0x10\n" - "fmla v13.4s, v7.4s, v1.s[0]\n" - "fmla v17.4s, v7.4s, v2.s[0]\n" + "fmla v13.4s, v28.4s, v1.s[0]\n" + "fmla v17.4s, v28.4s, v0.s[0]\n" "add x10, x10, #0x10\n" - "fmla v21.4s, v7.4s, v3.s[0]\n" - "fmla v25.4s, v7.4s, v4.s[0]\n" - "ldr q7, [x9, #0x0]\n" + "fmla v21.4s, v28.4s, v31.s[0]\n" + "fmla v25.4s, v28.4s, v30.s[0]\n" + "ldr q28, [x9, #0x0]\n" "add x9, x9, #0x10\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "fmla v22.4s, v6.4s, v3.s[0]\n" - "fmla v26.4s, v6.4s, v4.s[0]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "fmla v23.4s, v7.4s, v3.s[0]\n" - "fmla v27.4s, v7.4s, v4.s[0]\n" + "fmla v10.4s, v29.4s, v2.s[0]\n" + "fmla v14.4s, v29.4s, v1.s[0]\n" + "fmla v18.4s, v29.4s, v0.s[0]\n" + "fmla v22.4s, v29.4s, v31.s[0]\n" + "fmla v26.4s, v29.4s, v30.s[0]\n" + "fmla v11.4s, v28.4s, v2.s[0]\n" + "fmla v15.4s, v28.4s, v1.s[0]\n" + "fmla v19.4s, v28.4s, v0.s[0]\n" + "fmla v23.4s, v28.4s, v31.s[0]\n" + "fmla v27.4s, v28.4s, v30.s[0]\n" "cbnz x27, 158b\n" "159:" // Height 5: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -2444,49 +2444,49 @@ void a64_ffhybrid_fp32_mla_6x16 ( "add x22, x23, x20, LSL #2\n" "tbz %x[flags], #1, 160f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v29.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v15.4s, v15.4s, v1.4s\n" - "fmin v16.4s, v16.4s, v1.4s\n" - "fmin v17.4s, v17.4s, v1.4s\n" - "fmin v18.4s, v18.4s, v1.4s\n" - "fmin v19.4s, v19.4s, v1.4s\n" - "fmin v20.4s, v20.4s, v1.4s\n" - "fmin v21.4s, v21.4s, v1.4s\n" - "fmin v22.4s, v22.4s, v1.4s\n" - "fmin v23.4s, v23.4s, v1.4s\n" - "fmin v24.4s, v24.4s, v1.4s\n" - "fmin v25.4s, v25.4s, v1.4s\n" - "fmin v26.4s, v26.4s, v1.4s\n" - "fmin v27.4s, v27.4s, v1.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" - "fmax v16.4s, v16.4s, v0.4s\n" - "fmax v17.4s, v17.4s, v0.4s\n" - "fmax v18.4s, v18.4s, v0.4s\n" - "fmax v19.4s, v19.4s, v0.4s\n" - "fmax v20.4s, v20.4s, v0.4s\n" - "fmax v21.4s, v21.4s, v0.4s\n" - "fmax v22.4s, v22.4s, v0.4s\n" - "fmax v23.4s, v23.4s, v0.4s\n" - "fmax v24.4s, v24.4s, v0.4s\n" - "fmax v25.4s, v25.4s, v0.4s\n" - "fmax v26.4s, v26.4s, v0.4s\n" - "fmax v27.4s, v27.4s, v0.4s\n" + "ld1r { v28.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v29.4s\n" + "fmin v9.4s, v9.4s, v29.4s\n" + "fmin v10.4s, v10.4s, v29.4s\n" + "fmin v11.4s, v11.4s, v29.4s\n" + "fmin v12.4s, v12.4s, v29.4s\n" + "fmin v13.4s, v13.4s, v29.4s\n" + "fmin v14.4s, v14.4s, v29.4s\n" + "fmin v15.4s, v15.4s, v29.4s\n" + "fmin v16.4s, v16.4s, v29.4s\n" + "fmin v17.4s, v17.4s, v29.4s\n" + "fmin v18.4s, v18.4s, v29.4s\n" + "fmin v19.4s, v19.4s, v29.4s\n" + "fmin v20.4s, v20.4s, v29.4s\n" + "fmin v21.4s, v21.4s, v29.4s\n" + "fmin v22.4s, v22.4s, v29.4s\n" + "fmin v23.4s, v23.4s, v29.4s\n" + "fmin v24.4s, v24.4s, v29.4s\n" + "fmin v25.4s, v25.4s, v29.4s\n" + "fmin v26.4s, v26.4s, v29.4s\n" + "fmin v27.4s, v27.4s, v29.4s\n" + "fmax v8.4s, v8.4s, v28.4s\n" + "fmax v9.4s, v9.4s, v28.4s\n" + "fmax v10.4s, v10.4s, v28.4s\n" + "fmax v11.4s, v11.4s, v28.4s\n" + "fmax v12.4s, v12.4s, v28.4s\n" + "fmax v13.4s, v13.4s, v28.4s\n" + "fmax v14.4s, v14.4s, v28.4s\n" + "fmax v15.4s, v15.4s, v28.4s\n" + "fmax v16.4s, v16.4s, v28.4s\n" + "fmax v17.4s, v17.4s, v28.4s\n" + "fmax v18.4s, v18.4s, v28.4s\n" + "fmax v19.4s, v19.4s, v28.4s\n" + "fmax v20.4s, v20.4s, v28.4s\n" + "fmax v21.4s, v21.4s, v28.4s\n" + "fmax v22.4s, v22.4s, v28.4s\n" + "fmax v23.4s, v23.4s, v28.4s\n" + "fmax v24.4s, v24.4s, v28.4s\n" + "fmax v25.4s, v25.4s, v28.4s\n" + "fmax v26.4s, v26.4s, v28.4s\n" + "fmax v27.4s, v27.4s, v28.4s\n" "160:" // Height 5: No activation "cmp x14, #0x10\n" "bge 169f\n" @@ -2881,16 +2881,16 @@ void a64_ffhybrid_fp32_mla_6x16 ( "186:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 187f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 188f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -2902,11 +2902,11 @@ void a64_ffhybrid_fp32_mla_6x16 ( "b 188f\n" "187:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "add x21, x22, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" + "add x21, x22, x21, LSL #2\n" "188:" // Height 6: input setup done "cmp x27, #0x4\n" "blt 191f\n" @@ -3177,45 +3177,45 @@ void a64_ffhybrid_fp32_mla_6x16 ( "191:" // Height 6: Multiply loop: Main loop skip "cbz x27, 193f\n" "192:" // Height 6: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s7, [x26], #0x4\n" + "ldr s6, [x25], #0x4\n" "sub x27, x27, #0x1\n" - "ldr s2, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr s4, [x22], #0x4\n" - "ldr s5, [x21], #0x4\n" - "ldr q6, [x12, #0x0]\n" - "ldr q7, [x11, #0x0]\n" - "fmla v8.4s, v6.4s, v0.s[0]\n" - "fmla v12.4s, v6.4s, v1.s[0]\n" - "fmla v16.4s, v6.4s, v2.s[0]\n" - "fmla v20.4s, v6.4s, v3.s[0]\n" + "ldr s5, [x24], #0x4\n" + "ldr s4, [x23], #0x4\n" + "ldr s3, [x22], #0x4\n" + "ldr s2, [x21], #0x4\n" + "ldr q1, [x12, #0x0]\n" + "ldr q0, [x11, #0x0]\n" + "fmla v8.4s, v1.4s, v7.s[0]\n" + "fmla v12.4s, v1.4s, v6.s[0]\n" + "fmla v16.4s, v1.4s, v5.s[0]\n" + "fmla v20.4s, v1.4s, v4.s[0]\n" "add x12, x12, #0x10\n" "add x11, x11, #0x10\n" - "fmla v24.4s, v6.4s, v4.s[0]\n" - "fmla v28.4s, v6.4s, v5.s[0]\n" - "ldr q6, [x10, #0x0]\n" + "fmla v24.4s, v1.4s, v3.s[0]\n" + "fmla v28.4s, v1.4s, v2.s[0]\n" + "ldr q1, [x10, #0x0]\n" "add x10, x10, #0x10\n" - "fmla v9.4s, v7.4s, v0.s[0]\n" - "fmla v13.4s, v7.4s, v1.s[0]\n" - "fmla v17.4s, v7.4s, v2.s[0]\n" - "fmla v21.4s, v7.4s, v3.s[0]\n" - "fmla v25.4s, v7.4s, v4.s[0]\n" - "fmla v29.4s, v7.4s, v5.s[0]\n" - "ldr q7, [x9, #0x0]\n" + "fmla v9.4s, v0.4s, v7.s[0]\n" + "fmla v13.4s, v0.4s, v6.s[0]\n" + "fmla v17.4s, v0.4s, v5.s[0]\n" + "fmla v21.4s, v0.4s, v4.s[0]\n" + "fmla v25.4s, v0.4s, v3.s[0]\n" + "fmla v29.4s, v0.4s, v2.s[0]\n" + "ldr q0, [x9, #0x0]\n" "add x9, x9, #0x10\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "fmla v22.4s, v6.4s, v3.s[0]\n" - "fmla v26.4s, v6.4s, v4.s[0]\n" - "fmla v30.4s, v6.4s, v5.s[0]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "fmla v23.4s, v7.4s, v3.s[0]\n" - "fmla v27.4s, v7.4s, v4.s[0]\n" - "fmla v31.4s, v7.4s, v5.s[0]\n" + "fmla v10.4s, v1.4s, v7.s[0]\n" + "fmla v14.4s, v1.4s, v6.s[0]\n" + "fmla v18.4s, v1.4s, v5.s[0]\n" + "fmla v22.4s, v1.4s, v4.s[0]\n" + "fmla v26.4s, v1.4s, v3.s[0]\n" + "fmla v30.4s, v1.4s, v2.s[0]\n" + "fmla v11.4s, v0.4s, v7.s[0]\n" + "fmla v15.4s, v0.4s, v6.s[0]\n" + "fmla v19.4s, v0.4s, v5.s[0]\n" + "fmla v23.4s, v0.4s, v4.s[0]\n" + "fmla v27.4s, v0.4s, v3.s[0]\n" + "fmla v31.4s, v0.4s, v2.s[0]\n" "cbnz x27, 192b\n" "193:" // Height 6: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp32bf16fp32_mmla_4x24.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp32bf16fp32_mmla_4x24.hpp index af2c1e5ae00c00b839e2df7ccd752ba19c03fadf..923d008bb10ca28959664be661a45b07d2cc64b9 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp32bf16fp32_mmla_4x24.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp32bf16fp32_mmla_4x24.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,16 +10,16 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once #ifdef __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp32bf16fp32_mmla_4x24/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp32bf16fp32_mmla_4x24/generic.cpp index 1f707fa962e238e63218e6419aa1a7f9d2e91ae2..8961e615d75afa606a2208d2ad5842bdabbd281a 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp32bf16fp32_mmla_4x24/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffhybrid_fp32bf16fp32_mmla_4x24/generic.cpp @@ -283,11 +283,11 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 ( "21:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 22f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" "cbnz x26, 23f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20, LSL #2\n" @@ -307,32 +307,32 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 ( "24:" // Height 1: Multiply loop: Main loop head ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" ".inst 0x6e44ec08 // bfmmla v8.4s, v0.8h, v4.8h\n" - "ldr q4, [x10, #0x0]\n" + "ldr q24, [x10, #0x0]\n" ".inst 0x6e45ec0e // bfmmla v14.4s, v0.8h, v5.8h\n" - "ldr q5, [x10, #0x10]\n" + "ldr q23, [x10, #0x10]\n" ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - "ldr q6, [x9, #0x0]\n" + "ldr q22, [x9, #0x0]\n" ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" - "ldr q4, [x28, #0x0]\n" - ".inst 0x6e45ec10 // bfmmla v16.4s, v0.8h, v5.8h\n" - "ldr q5, [x28, #0x10]\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - "ldr q6, [x27, #0x0]\n" - ".inst 0x6e47ec11 // bfmmla v17.4s, v0.8h, v7.8h\n" - "ldr q7, [x27, #0x10]\n" + "ldr q21, [x9, #0x10]\n" + ".inst 0x6e58ec0a // bfmmla v10.4s, v0.8h, v24.8h\n" + "ldr q24, [x28, #0x0]\n" + ".inst 0x6e57ec10 // bfmmla v16.4s, v0.8h, v23.8h\n" + "ldr q23, [x28, #0x10]\n" + ".inst 0x6e56ec0b // bfmmla v11.4s, v0.8h, v22.8h\n" + "ldr q22, [x27, #0x0]\n" + ".inst 0x6e55ec11 // bfmmla v17.4s, v0.8h, v21.8h\n" + "ldr q21, [x27, #0x10]\n" "sub x25, x25, #0x4\n" "cmp x25, #0x8\n" - ".inst 0x6e44ec0c // bfmmla v12.4s, v0.8h, v4.8h\n" + ".inst 0x6e58ec0c // bfmmla v12.4s, v0.8h, v24.8h\n" "add x12, x12, #0x20\n" "ldr q4, [x12, #0x0]\n" "add x11, x11, #0x20\n" - ".inst 0x6e45ec12 // bfmmla v18.4s, v0.8h, v5.8h\n" + ".inst 0x6e57ec12 // bfmmla v18.4s, v0.8h, v23.8h\n" "ldr q5, [x12, #0x10]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" + ".inst 0x6e56ec0d // bfmmla v13.4s, v0.8h, v22.8h\n" "ldr q6, [x11, #0x0]\n" - ".inst 0x6e47ec13 // bfmmla v19.4s, v0.8h, v7.8h\n" + ".inst 0x6e55ec13 // bfmmla v19.4s, v0.8h, v21.8h\n" "ld1 { v0.4s }, [x24], #0x10\n" "ldr q7, [x11, #0x10]\n" "add x10, x10, #0x20\n" @@ -343,28 +343,28 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 ( "25:" // Height 1: Multiply loop: Single iteration only ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" ".inst 0x6e44ec08 // bfmmla v8.4s, v0.8h, v4.8h\n" - "ldr q4, [x10, #0x0]\n" + "ldr q22, [x10, #0x0]\n" ".inst 0x6e45ec0e // bfmmla v14.4s, v0.8h, v5.8h\n" - "ldr q5, [x10, #0x10]\n" + "ldr q25, [x10, #0x10]\n" ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - "ldr q6, [x9, #0x0]\n" + "ldr q21, [x9, #0x0]\n" ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" - "ldr q4, [x28, #0x0]\n" - ".inst 0x6e45ec10 // bfmmla v16.4s, v0.8h, v5.8h\n" - "ldr q5, [x28, #0x10]\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - "ldr q6, [x27, #0x0]\n" - ".inst 0x6e47ec11 // bfmmla v17.4s, v0.8h, v7.8h\n" - "ldr q7, [x27, #0x10]\n" + "ldr q24, [x9, #0x10]\n" + ".inst 0x6e56ec0a // bfmmla v10.4s, v0.8h, v22.8h\n" + "ldr q23, [x28, #0x0]\n" + ".inst 0x6e59ec10 // bfmmla v16.4s, v0.8h, v25.8h\n" + "ldr q22, [x28, #0x10]\n" + ".inst 0x6e55ec0b // bfmmla v11.4s, v0.8h, v21.8h\n" + "ldr q21, [x27, #0x0]\n" + ".inst 0x6e58ec11 // bfmmla v17.4s, v0.8h, v24.8h\n" + "ldr q3, [x27, #0x10]\n" "sub x25, x25, #0x4\n" - ".inst 0x6e44ec0c // bfmmla v12.4s, v0.8h, v4.8h\n" - ".inst 0x6e45ec12 // bfmmla v18.4s, v0.8h, v5.8h\n" + ".inst 0x6e57ec0c // bfmmla v12.4s, v0.8h, v23.8h\n" + ".inst 0x6e56ec12 // bfmmla v18.4s, v0.8h, v22.8h\n" "add x12, x12, #0x20\n" "add x11, x11, #0x20\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec13 // bfmmla v19.4s, v0.8h, v7.8h\n" + ".inst 0x6e55ec0d // bfmmla v13.4s, v0.8h, v21.8h\n" + ".inst 0x6e43ec13 // bfmmla v19.4s, v0.8h, v3.8h\n" "add x10, x10, #0x20\n" "add x9, x9, #0x20\n" "add x28, x28, #0x20\n" @@ -380,31 +380,31 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 ( "27:" // Height 1: Multiply loop: Ragged operand read: partial_1_0 "ldr s0, [x24, #0x0]\n" "28:" // Height 1: Multiply loop: Ragged operand read: Done - "ldr q4, [x12, #0x0]\n" - "ldr q5, [x12, #0x10]\n" + "ldr q21, [x12, #0x0]\n" + "ldr q30, [x12, #0x10]\n" ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" - ".inst 0x6e44ec08 // bfmmla v8.4s, v0.8h, v4.8h\n" - "ldr q6, [x11, #0x0]\n" - "ldr q7, [x11, #0x10]\n" - ".inst 0x6e45ec0e // bfmmla v14.4s, v0.8h, v5.8h\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - "ldr q4, [x10, #0x0]\n" - "ldr q5, [x10, #0x10]\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" - "ldr q6, [x9, #0x0]\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x6e45ec10 // bfmmla v16.4s, v0.8h, v5.8h\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - "ldr q4, [x28, #0x0]\n" - "ldr q5, [x28, #0x10]\n" - ".inst 0x6e47ec11 // bfmmla v17.4s, v0.8h, v7.8h\n" - ".inst 0x6e44ec0c // bfmmla v12.4s, v0.8h, v4.8h\n" - "ldr q6, [x27, #0x0]\n" - "ldr q7, [x27, #0x10]\n" - ".inst 0x6e45ec12 // bfmmla v18.4s, v0.8h, v5.8h\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec13 // bfmmla v19.4s, v0.8h, v7.8h\n" + ".inst 0x6e55ec08 // bfmmla v8.4s, v0.8h, v21.8h\n" + "ldr q21, [x11, #0x0]\n" + "ldr q22, [x11, #0x10]\n" + ".inst 0x6e5eec0e // bfmmla v14.4s, v0.8h, v30.8h\n" + ".inst 0x6e55ec09 // bfmmla v9.4s, v0.8h, v21.8h\n" + "ldr q21, [x10, #0x0]\n" + "ldr q23, [x10, #0x10]\n" + ".inst 0x6e56ec0f // bfmmla v15.4s, v0.8h, v22.8h\n" + ".inst 0x6e55ec0a // bfmmla v10.4s, v0.8h, v21.8h\n" + "ldr q21, [x9, #0x0]\n" + "ldr q22, [x9, #0x10]\n" + ".inst 0x6e57ec10 // bfmmla v16.4s, v0.8h, v23.8h\n" + ".inst 0x6e55ec0b // bfmmla v11.4s, v0.8h, v21.8h\n" + "ldr q21, [x28, #0x0]\n" + "ldr q23, [x28, #0x10]\n" + ".inst 0x6e56ec11 // bfmmla v17.4s, v0.8h, v22.8h\n" + ".inst 0x6e55ec0c // bfmmla v12.4s, v0.8h, v21.8h\n" + "ldr q22, [x27, #0x0]\n" + "ldr q21, [x27, #0x10]\n" + ".inst 0x6e57ec12 // bfmmla v18.4s, v0.8h, v23.8h\n" + ".inst 0x6e56ec0d // bfmmla v13.4s, v0.8h, v22.8h\n" + ".inst 0x6e55ec13 // bfmmla v19.4s, v0.8h, v21.8h\n" "add x12, x12, #0x20\n" "add x11, x11, #0x20\n" "add x10, x10, #0x20\n" @@ -424,21 +424,21 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 ( "uzp1 v13.2d, v13.2d, v19.2d\n" "tbz %x[flags], #1, 30f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v22.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" + "ld1r { v21.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v22.4s\n" + "fmin v9.4s, v9.4s, v22.4s\n" + "fmin v10.4s, v10.4s, v22.4s\n" + "fmin v11.4s, v11.4s, v22.4s\n" + "fmin v12.4s, v12.4s, v22.4s\n" + "fmin v13.4s, v13.4s, v22.4s\n" + "fmax v8.4s, v8.4s, v21.4s\n" + "fmax v9.4s, v9.4s, v21.4s\n" + "fmax v10.4s, v10.4s, v21.4s\n" + "fmax v11.4s, v11.4s, v21.4s\n" + "fmax v12.4s, v12.4s, v21.4s\n" + "fmax v13.4s, v13.4s, v21.4s\n" "30:" // Height 1: No activation "cmp x14, #0x18\n" "bge 43f\n" @@ -744,12 +744,12 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 ( "65:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 66f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" "cbnz x26, 67f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20, LSL #2\n" @@ -757,7 +757,7 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 ( "b 67f\n" "66:" // Height 2: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20, LSL #2\n" + "add x23, x24, x21, LSL #2\n" "67:" // Height 2: input setup done "cmp x25, #0x4\n" "blt 70f\n" @@ -774,32 +774,32 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 ( ".inst 0x4ea16820 // bfcvtn2 v0.8h, v1.4s\n" "ld1 { v1.4s }, [x23], #0x10\n" ".inst 0x6e44ec08 // bfmmla v8.4s, v0.8h, v4.8h\n" - "ldr q4, [x10, #0x0]\n" + "ldr q30, [x10, #0x0]\n" ".inst 0x6e45ec0e // bfmmla v14.4s, v0.8h, v5.8h\n" - "ldr q5, [x10, #0x10]\n" + "ldr q23, [x10, #0x10]\n" ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - "ldr q6, [x9, #0x0]\n" + "ldr q22, [x9, #0x0]\n" ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" - "ldr q4, [x28, #0x0]\n" - ".inst 0x6e45ec10 // bfmmla v16.4s, v0.8h, v5.8h\n" - "ldr q5, [x28, #0x10]\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - "ldr q6, [x27, #0x0]\n" - ".inst 0x6e47ec11 // bfmmla v17.4s, v0.8h, v7.8h\n" - "ldr q7, [x27, #0x10]\n" + "ldr q21, [x9, #0x10]\n" + ".inst 0x6e5eec0a // bfmmla v10.4s, v0.8h, v30.8h\n" + "ldr q2, [x28, #0x0]\n" + ".inst 0x6e57ec10 // bfmmla v16.4s, v0.8h, v23.8h\n" + "ldr q23, [x28, #0x10]\n" + ".inst 0x6e56ec0b // bfmmla v11.4s, v0.8h, v22.8h\n" + "ldr q22, [x27, #0x0]\n" + ".inst 0x6e55ec11 // bfmmla v17.4s, v0.8h, v21.8h\n" + "ldr q21, [x27, #0x10]\n" "sub x25, x25, #0x4\n" "cmp x25, #0x8\n" "add x12, x12, #0x20\n" - ".inst 0x6e44ec0c // bfmmla v12.4s, v0.8h, v4.8h\n" + ".inst 0x6e42ec0c // bfmmla v12.4s, v0.8h, v2.8h\n" "ldr q4, [x12, #0x0]\n" "add x11, x11, #0x20\n" - ".inst 0x6e45ec12 // bfmmla v18.4s, v0.8h, v5.8h\n" + ".inst 0x6e57ec12 // bfmmla v18.4s, v0.8h, v23.8h\n" "ldr q5, [x12, #0x10]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" + ".inst 0x6e56ec0d // bfmmla v13.4s, v0.8h, v22.8h\n" "ldr q6, [x11, #0x0]\n" - ".inst 0x6e47ec13 // bfmmla v19.4s, v0.8h, v7.8h\n" + ".inst 0x6e55ec13 // bfmmla v19.4s, v0.8h, v21.8h\n" "ld1 { v0.4s }, [x24], #0x10\n" "add x10, x10, #0x20\n" "ldr q7, [x11, #0x10]\n" @@ -811,28 +811,28 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 ( ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" ".inst 0x4ea16820 // bfcvtn2 v0.8h, v1.4s\n" ".inst 0x6e44ec08 // bfmmla v8.4s, v0.8h, v4.8h\n" - "ldr q4, [x10, #0x0]\n" + "ldr q24, [x10, #0x0]\n" ".inst 0x6e45ec0e // bfmmla v14.4s, v0.8h, v5.8h\n" - "ldr q5, [x10, #0x10]\n" + "ldr q23, [x10, #0x10]\n" ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - "ldr q6, [x9, #0x0]\n" + "ldr q22, [x9, #0x0]\n" ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" - "ldr q4, [x28, #0x0]\n" - ".inst 0x6e45ec10 // bfmmla v16.4s, v0.8h, v5.8h\n" - "ldr q5, [x28, #0x10]\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - "ldr q6, [x27, #0x0]\n" - ".inst 0x6e47ec11 // bfmmla v17.4s, v0.8h, v7.8h\n" - "ldr q7, [x27, #0x10]\n" + "ldr q21, [x9, #0x10]\n" + ".inst 0x6e58ec0a // bfmmla v10.4s, v0.8h, v24.8h\n" + "ldr q24, [x28, #0x0]\n" + ".inst 0x6e57ec10 // bfmmla v16.4s, v0.8h, v23.8h\n" + "ldr q23, [x28, #0x10]\n" + ".inst 0x6e56ec0b // bfmmla v11.4s, v0.8h, v22.8h\n" + "ldr q22, [x27, #0x0]\n" + ".inst 0x6e55ec11 // bfmmla v17.4s, v0.8h, v21.8h\n" + "ldr q21, [x27, #0x10]\n" "sub x25, x25, #0x4\n" - ".inst 0x6e44ec0c // bfmmla v12.4s, v0.8h, v4.8h\n" - ".inst 0x6e45ec12 // bfmmla v18.4s, v0.8h, v5.8h\n" + ".inst 0x6e58ec0c // bfmmla v12.4s, v0.8h, v24.8h\n" + ".inst 0x6e57ec12 // bfmmla v18.4s, v0.8h, v23.8h\n" "add x12, x12, #0x20\n" "add x11, x11, #0x20\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec13 // bfmmla v19.4s, v0.8h, v7.8h\n" + ".inst 0x6e56ec0d // bfmmla v13.4s, v0.8h, v22.8h\n" + ".inst 0x6e55ec13 // bfmmla v19.4s, v0.8h, v21.8h\n" "add x10, x10, #0x20\n" "add x9, x9, #0x20\n" "add x28, x28, #0x20\n" @@ -851,32 +851,32 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 ( "ldr s0, [x24, #0x0]\n" "ldr s1, [x23, #0x0]\n" "72:" // Height 2: Multiply loop: Ragged operand read: Done - "ldr q4, [x12, #0x0]\n" - "ldr q5, [x12, #0x10]\n" + "ldr q24, [x12, #0x0]\n" + "ldr q23, [x12, #0x10]\n" ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" ".inst 0x4ea16820 // bfcvtn2 v0.8h, v1.4s\n" - "ldr q6, [x11, #0x0]\n" - "ldr q7, [x11, #0x10]\n" - ".inst 0x6e44ec08 // bfmmla v8.4s, v0.8h, v4.8h\n" - ".inst 0x6e45ec0e // bfmmla v14.4s, v0.8h, v5.8h\n" - "ldr q4, [x10, #0x0]\n" - "ldr q5, [x10, #0x10]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - "ldr q6, [x9, #0x0]\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" - ".inst 0x6e45ec10 // bfmmla v16.4s, v0.8h, v5.8h\n" - "ldr q4, [x28, #0x0]\n" - "ldr q5, [x28, #0x10]\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec11 // bfmmla v17.4s, v0.8h, v7.8h\n" - "ldr q6, [x27, #0x0]\n" - "ldr q7, [x27, #0x10]\n" - ".inst 0x6e44ec0c // bfmmla v12.4s, v0.8h, v4.8h\n" - ".inst 0x6e45ec12 // bfmmla v18.4s, v0.8h, v5.8h\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec13 // bfmmla v19.4s, v0.8h, v7.8h\n" + "ldr q22, [x11, #0x0]\n" + "ldr q21, [x11, #0x10]\n" + ".inst 0x6e58ec08 // bfmmla v8.4s, v0.8h, v24.8h\n" + ".inst 0x6e57ec0e // bfmmla v14.4s, v0.8h, v23.8h\n" + "ldr q24, [x10, #0x0]\n" + "ldr q23, [x10, #0x10]\n" + ".inst 0x6e56ec09 // bfmmla v9.4s, v0.8h, v22.8h\n" + ".inst 0x6e55ec0f // bfmmla v15.4s, v0.8h, v21.8h\n" + "ldr q22, [x9, #0x0]\n" + "ldr q21, [x9, #0x10]\n" + ".inst 0x6e58ec0a // bfmmla v10.4s, v0.8h, v24.8h\n" + ".inst 0x6e57ec10 // bfmmla v16.4s, v0.8h, v23.8h\n" + "ldr q24, [x28, #0x0]\n" + "ldr q23, [x28, #0x10]\n" + ".inst 0x6e56ec0b // bfmmla v11.4s, v0.8h, v22.8h\n" + ".inst 0x6e55ec11 // bfmmla v17.4s, v0.8h, v21.8h\n" + "ldr q22, [x27, #0x0]\n" + "ldr q21, [x27, #0x10]\n" + ".inst 0x6e58ec0c // bfmmla v12.4s, v0.8h, v24.8h\n" + ".inst 0x6e57ec12 // bfmmla v18.4s, v0.8h, v23.8h\n" + ".inst 0x6e56ec0d // bfmmla v13.4s, v0.8h, v22.8h\n" + ".inst 0x6e55ec13 // bfmmla v19.4s, v0.8h, v21.8h\n" "add x12, x12, #0x20\n" "add x11, x11, #0x20\n" "add x10, x10, #0x20\n" @@ -904,33 +904,33 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 ( "uzp2 v13.2d, v13.2d, v19.2d\n" "tbz %x[flags], #1, 74f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v22.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v4.4s, v4.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v15.4s, v15.4s, v1.4s\n" - "fmin v16.4s, v16.4s, v1.4s\n" - "fmin v17.4s, v17.4s, v1.4s\n" - "fmin v18.4s, v18.4s, v1.4s\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmax v4.4s, v4.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" - "fmax v16.4s, v16.4s, v0.4s\n" - "fmax v17.4s, v17.4s, v0.4s\n" - "fmax v18.4s, v18.4s, v0.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" + "ld1r { v21.4s }, [x20]\n" + "fmin v4.4s, v4.4s, v22.4s\n" + "fmin v14.4s, v14.4s, v22.4s\n" + "fmin v15.4s, v15.4s, v22.4s\n" + "fmin v16.4s, v16.4s, v22.4s\n" + "fmin v17.4s, v17.4s, v22.4s\n" + "fmin v18.4s, v18.4s, v22.4s\n" + "fmin v8.4s, v8.4s, v22.4s\n" + "fmin v9.4s, v9.4s, v22.4s\n" + "fmin v10.4s, v10.4s, v22.4s\n" + "fmin v11.4s, v11.4s, v22.4s\n" + "fmin v12.4s, v12.4s, v22.4s\n" + "fmin v13.4s, v13.4s, v22.4s\n" + "fmax v4.4s, v4.4s, v21.4s\n" + "fmax v14.4s, v14.4s, v21.4s\n" + "fmax v15.4s, v15.4s, v21.4s\n" + "fmax v16.4s, v16.4s, v21.4s\n" + "fmax v17.4s, v17.4s, v21.4s\n" + "fmax v18.4s, v18.4s, v21.4s\n" + "fmax v8.4s, v8.4s, v21.4s\n" + "fmax v9.4s, v9.4s, v21.4s\n" + "fmax v10.4s, v10.4s, v21.4s\n" + "fmax v11.4s, v11.4s, v21.4s\n" + "fmax v12.4s, v12.4s, v21.4s\n" + "fmax v13.4s, v13.4s, v21.4s\n" "74:" // Height 2: No activation "cmp x14, #0x18\n" "bge 87f\n" @@ -1339,13 +1339,13 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 ( "109:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 110f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" "cbnz x26, 111f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20, LSL #2\n" @@ -1354,8 +1354,8 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 ( "b 111f\n" "110:" // Height 3: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" "111:" // Height 3: input setup done "cmp x25, #0x4\n" "blt 114f\n" @@ -1386,7 +1386,7 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 ( ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" "cmp x25, #0x8\n" ".inst 0x6e47ec5b // bfmmla v27.4s, v2.8h, v7.8h\n" - "ldr q7, [x9, #0x10]\n" + "ldr q3, [x9, #0x10]\n" ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" "add x12, x12, #0x20\n" ".inst 0x6e44ec56 // bfmmla v22.4s, v2.8h, v4.8h\n" @@ -1399,10 +1399,10 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 ( "add x10, x10, #0x20\n" ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" "ldr q6, [x27, #0x0]\n" - ".inst 0x6e47ec11 // bfmmla v17.4s, v0.8h, v7.8h\n" + ".inst 0x6e43ec11 // bfmmla v17.4s, v0.8h, v3.8h\n" "add x9, x9, #0x20\n" - ".inst 0x6e47ec5d // bfmmla v29.4s, v2.8h, v7.8h\n" - "ldr q7, [x27, #0x10]\n" + ".inst 0x6e43ec5d // bfmmla v29.4s, v2.8h, v3.8h\n" + "ldr q3, [x27, #0x10]\n" "add x28, x28, #0x20\n" ".inst 0x6e44ec0c // bfmmla v12.4s, v0.8h, v4.8h\n" "add x27, x27, #0x20\n" @@ -1414,9 +1414,9 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 ( ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" ".inst 0x6e46ec59 // bfmmla v25.4s, v2.8h, v6.8h\n" "ldr q6, [x11, #0x0]\n" - ".inst 0x6e47ec13 // bfmmla v19.4s, v0.8h, v7.8h\n" + ".inst 0x6e43ec13 // bfmmla v19.4s, v0.8h, v3.8h\n" "ld1 { v0.4s }, [x24], #0x10\n" - ".inst 0x6e47ec5f // bfmmla v31.4s, v2.8h, v7.8h\n" + ".inst 0x6e43ec5f // bfmmla v31.4s, v2.8h, v3.8h\n" "ld1 { v2.4s }, [x22], #0x10\n" "ldr q7, [x11, #0x10]\n" "bge 112b\n" @@ -1427,10 +1427,10 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 ( "sub x25, x25, #0x4\n" ".inst 0x0ea16842 // bfcvtn v2.4h, v2.4s\n" ".inst 0x6e44ec54 // bfmmla v20.4s, v2.8h, v4.8h\n" - "ldr q4, [x10, #0x0]\n" + "ldr q3, [x10, #0x0]\n" ".inst 0x6e45ec0e // bfmmla v14.4s, v0.8h, v5.8h\n" ".inst 0x6e45ec5a // bfmmla v26.4s, v2.8h, v5.8h\n" - "ldr q5, [x10, #0x10]\n" + "ldr q4, [x10, #0x10]\n" ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" "add x12, x12, #0x20\n" ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" @@ -1438,31 +1438,31 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 ( ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" "add x11, x11, #0x20\n" ".inst 0x6e47ec5b // bfmmla v27.4s, v2.8h, v7.8h\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" + "ldr q1, [x9, #0x10]\n" + ".inst 0x6e43ec0a // bfmmla v10.4s, v0.8h, v3.8h\n" "add x10, x10, #0x20\n" - ".inst 0x6e44ec56 // bfmmla v22.4s, v2.8h, v4.8h\n" - "ldr q4, [x28, #0x0]\n" - ".inst 0x6e45ec10 // bfmmla v16.4s, v0.8h, v5.8h\n" + ".inst 0x6e43ec56 // bfmmla v22.4s, v2.8h, v3.8h\n" + "ldr q5, [x28, #0x0]\n" + ".inst 0x6e44ec10 // bfmmla v16.4s, v0.8h, v4.8h\n" "add x9, x9, #0x20\n" - ".inst 0x6e45ec5c // bfmmla v28.4s, v2.8h, v5.8h\n" - "ldr q5, [x28, #0x10]\n" + ".inst 0x6e44ec5c // bfmmla v28.4s, v2.8h, v4.8h\n" + "ldr q4, [x28, #0x10]\n" ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" "add x28, x28, #0x20\n" ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" - "ldr q6, [x27, #0x0]\n" - ".inst 0x6e47ec11 // bfmmla v17.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec5d // bfmmla v29.4s, v2.8h, v7.8h\n" - "ldr q7, [x27, #0x10]\n" + "ldr q3, [x27, #0x0]\n" + ".inst 0x6e41ec11 // bfmmla v17.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec5d // bfmmla v29.4s, v2.8h, v1.8h\n" + "ldr q1, [x27, #0x10]\n" "add x27, x27, #0x20\n" - ".inst 0x6e44ec0c // bfmmla v12.4s, v0.8h, v4.8h\n" - ".inst 0x6e44ec58 // bfmmla v24.4s, v2.8h, v4.8h\n" - ".inst 0x6e45ec12 // bfmmla v18.4s, v0.8h, v5.8h\n" - ".inst 0x6e45ec5e // bfmmla v30.4s, v2.8h, v5.8h\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec59 // bfmmla v25.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec13 // bfmmla v19.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec5f // bfmmla v31.4s, v2.8h, v7.8h\n" + ".inst 0x6e45ec0c // bfmmla v12.4s, v0.8h, v5.8h\n" + ".inst 0x6e45ec58 // bfmmla v24.4s, v2.8h, v5.8h\n" + ".inst 0x6e44ec12 // bfmmla v18.4s, v0.8h, v4.8h\n" + ".inst 0x6e44ec5e // bfmmla v30.4s, v2.8h, v4.8h\n" + ".inst 0x6e43ec0d // bfmmla v13.4s, v0.8h, v3.8h\n" + ".inst 0x6e43ec59 // bfmmla v25.4s, v2.8h, v3.8h\n" + ".inst 0x6e41ec13 // bfmmla v19.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec5f // bfmmla v31.4s, v2.8h, v1.8h\n" "114:" // Height 3: Multiply loop: Main loop skip "cbz x25, 117f\n" "cbz x25, 117f\n" @@ -1480,51 +1480,51 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 ( "ldr s1, [x23, #0x0]\n" "ldr s2, [x22, #0x0]\n" "116:" // Height 3: Multiply loop: Ragged operand read: Done - "ldr q4, [x12, #0x0]\n" - "ldr q5, [x12, #0x10]\n" + "ldr q5, [x12, #0x0]\n" + "ldr q4, [x12, #0x10]\n" ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" ".inst 0x4ea16820 // bfcvtn2 v0.8h, v1.4s\n" - "ldr q6, [x11, #0x0]\n" - "ldr q7, [x11, #0x10]\n" + "ldr q3, [x11, #0x0]\n" + "ldr q1, [x11, #0x10]\n" ".inst 0x0ea16842 // bfcvtn v2.4h, v2.4s\n" - ".inst 0x6e44ec08 // bfmmla v8.4s, v0.8h, v4.8h\n" - ".inst 0x6e44ec54 // bfmmla v20.4s, v2.8h, v4.8h\n" - "ldr q4, [x10, #0x0]\n" - ".inst 0x6e45ec0e // bfmmla v14.4s, v0.8h, v5.8h\n" + ".inst 0x6e45ec08 // bfmmla v8.4s, v0.8h, v5.8h\n" + ".inst 0x6e45ec54 // bfmmla v20.4s, v2.8h, v5.8h\n" + "ldr q5, [x10, #0x0]\n" + ".inst 0x6e44ec0e // bfmmla v14.4s, v0.8h, v4.8h\n" "add x12, x12, #0x20\n" - ".inst 0x6e45ec5a // bfmmla v26.4s, v2.8h, v5.8h\n" - "ldr q5, [x10, #0x10]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" + ".inst 0x6e44ec5a // bfmmla v26.4s, v2.8h, v4.8h\n" + "ldr q4, [x10, #0x10]\n" + ".inst 0x6e43ec09 // bfmmla v9.4s, v0.8h, v3.8h\n" "add x11, x11, #0x20\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" - "ldr q6, [x9, #0x0]\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" + ".inst 0x6e43ec55 // bfmmla v21.4s, v2.8h, v3.8h\n" + "ldr q3, [x9, #0x0]\n" + ".inst 0x6e41ec0f // bfmmla v15.4s, v0.8h, v1.8h\n" "add x10, x10, #0x20\n" - ".inst 0x6e47ec5b // bfmmla v27.4s, v2.8h, v7.8h\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" + ".inst 0x6e41ec5b // bfmmla v27.4s, v2.8h, v1.8h\n" + "ldr q1, [x9, #0x10]\n" + ".inst 0x6e45ec0a // bfmmla v10.4s, v0.8h, v5.8h\n" "add x9, x9, #0x20\n" - ".inst 0x6e44ec56 // bfmmla v22.4s, v2.8h, v4.8h\n" - "ldr q4, [x28, #0x0]\n" - ".inst 0x6e45ec10 // bfmmla v16.4s, v0.8h, v5.8h\n" - ".inst 0x6e45ec5c // bfmmla v28.4s, v2.8h, v5.8h\n" - "ldr q5, [x28, #0x10]\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" + ".inst 0x6e45ec56 // bfmmla v22.4s, v2.8h, v5.8h\n" + "ldr q5, [x28, #0x0]\n" + ".inst 0x6e44ec10 // bfmmla v16.4s, v0.8h, v4.8h\n" + ".inst 0x6e44ec5c // bfmmla v28.4s, v2.8h, v4.8h\n" + "ldr q4, [x28, #0x10]\n" + ".inst 0x6e43ec0b // bfmmla v11.4s, v0.8h, v3.8h\n" "add x28, x28, #0x20\n" - ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" - "ldr q6, [x27, #0x0]\n" - ".inst 0x6e47ec11 // bfmmla v17.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec5d // bfmmla v29.4s, v2.8h, v7.8h\n" - "ldr q7, [x27, #0x10]\n" - ".inst 0x6e44ec0c // bfmmla v12.4s, v0.8h, v4.8h\n" + ".inst 0x6e43ec57 // bfmmla v23.4s, v2.8h, v3.8h\n" + "ldr q3, [x27, #0x0]\n" + ".inst 0x6e41ec11 // bfmmla v17.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec5d // bfmmla v29.4s, v2.8h, v1.8h\n" + "ldr q1, [x27, #0x10]\n" + ".inst 0x6e45ec0c // bfmmla v12.4s, v0.8h, v5.8h\n" "add x27, x27, #0x20\n" - ".inst 0x6e44ec58 // bfmmla v24.4s, v2.8h, v4.8h\n" - ".inst 0x6e45ec12 // bfmmla v18.4s, v0.8h, v5.8h\n" - ".inst 0x6e45ec5e // bfmmla v30.4s, v2.8h, v5.8h\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec59 // bfmmla v25.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec13 // bfmmla v19.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec5f // bfmmla v31.4s, v2.8h, v7.8h\n" + ".inst 0x6e45ec58 // bfmmla v24.4s, v2.8h, v5.8h\n" + ".inst 0x6e44ec12 // bfmmla v18.4s, v0.8h, v4.8h\n" + ".inst 0x6e44ec5e // bfmmla v30.4s, v2.8h, v4.8h\n" + ".inst 0x6e43ec0d // bfmmla v13.4s, v0.8h, v3.8h\n" + ".inst 0x6e43ec59 // bfmmla v25.4s, v2.8h, v3.8h\n" + ".inst 0x6e41ec13 // bfmmla v19.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec5f // bfmmla v31.4s, v2.8h, v1.8h\n" "117:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x26, x26, #0x1\n" @@ -2070,14 +2070,14 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 ( "153:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 154f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" - "ldr x21, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" + "ldr x21, [x20, #0x18]\n" "cbnz x26, 155f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20, LSL #2\n" @@ -2087,9 +2087,9 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 ( "b 155f\n" "154:" // Height 4: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "add x21, x22, x20, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" + "add x21, x22, x21, LSL #2\n" "155:" // Height 4: input setup done "cmp x25, #0x4\n" "blt 158f\n" @@ -2167,40 +2167,40 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 ( ".inst 0x6e44ec08 // bfmmla v8.4s, v0.8h, v4.8h\n" "add x11, x11, #0x20\n" ".inst 0x6e44ec54 // bfmmla v20.4s, v2.8h, v4.8h\n" - "ldr q4, [x10, #0x0]\n" + "ldr q3, [x10, #0x0]\n" ".inst 0x6e45ec0e // bfmmla v14.4s, v0.8h, v5.8h\n" ".inst 0x6e45ec5a // bfmmla v26.4s, v2.8h, v5.8h\n" - "ldr q5, [x10, #0x10]\n" + "ldr q4, [x10, #0x10]\n" ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" "add x10, x10, #0x20\n" ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" "ldr q6, [x9, #0x0]\n" ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" ".inst 0x6e47ec5b // bfmmla v27.4s, v2.8h, v7.8h\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" + "ldr q1, [x9, #0x10]\n" + ".inst 0x6e43ec0a // bfmmla v10.4s, v0.8h, v3.8h\n" "add x9, x9, #0x20\n" - ".inst 0x6e44ec56 // bfmmla v22.4s, v2.8h, v4.8h\n" - "ldr q4, [x28, #0x0]\n" - ".inst 0x6e45ec10 // bfmmla v16.4s, v0.8h, v5.8h\n" - ".inst 0x6e45ec5c // bfmmla v28.4s, v2.8h, v5.8h\n" - "ldr q5, [x28, #0x10]\n" + ".inst 0x6e43ec56 // bfmmla v22.4s, v2.8h, v3.8h\n" + "ldr q5, [x28, #0x0]\n" + ".inst 0x6e44ec10 // bfmmla v16.4s, v0.8h, v4.8h\n" + ".inst 0x6e44ec5c // bfmmla v28.4s, v2.8h, v4.8h\n" + "ldr q4, [x28, #0x10]\n" ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" "add x28, x28, #0x20\n" ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" - "ldr q6, [x27, #0x0]\n" - ".inst 0x6e47ec11 // bfmmla v17.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec5d // bfmmla v29.4s, v2.8h, v7.8h\n" - "ldr q7, [x27, #0x10]\n" + "ldr q3, [x27, #0x0]\n" + ".inst 0x6e41ec11 // bfmmla v17.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec5d // bfmmla v29.4s, v2.8h, v1.8h\n" + "ldr q1, [x27, #0x10]\n" "add x27, x27, #0x20\n" - ".inst 0x6e44ec0c // bfmmla v12.4s, v0.8h, v4.8h\n" - ".inst 0x6e44ec58 // bfmmla v24.4s, v2.8h, v4.8h\n" - ".inst 0x6e45ec12 // bfmmla v18.4s, v0.8h, v5.8h\n" - ".inst 0x6e45ec5e // bfmmla v30.4s, v2.8h, v5.8h\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec59 // bfmmla v25.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec13 // bfmmla v19.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec5f // bfmmla v31.4s, v2.8h, v7.8h\n" + ".inst 0x6e45ec0c // bfmmla v12.4s, v0.8h, v5.8h\n" + ".inst 0x6e45ec58 // bfmmla v24.4s, v2.8h, v5.8h\n" + ".inst 0x6e44ec12 // bfmmla v18.4s, v0.8h, v4.8h\n" + ".inst 0x6e44ec5e // bfmmla v30.4s, v2.8h, v4.8h\n" + ".inst 0x6e43ec0d // bfmmla v13.4s, v0.8h, v3.8h\n" + ".inst 0x6e43ec59 // bfmmla v25.4s, v2.8h, v3.8h\n" + ".inst 0x6e41ec13 // bfmmla v19.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec5f // bfmmla v31.4s, v2.8h, v1.8h\n" "158:" // Height 4: Multiply loop: Main loop skip "cbz x25, 161f\n" "cbz x25, 161f\n" @@ -2221,52 +2221,52 @@ void a64_ffhybrid_fp32bf16fp32_mmla_4x24 ( "ldr s2, [x22, #0x0]\n" "ldr s3, [x21, #0x0]\n" "160:" // Height 4: Multiply loop: Ragged operand read: Done - "ldr q4, [x12, #0x0]\n" - "ldr q5, [x12, #0x10]\n" + "ldr q5, [x12, #0x0]\n" + "ldr q4, [x12, #0x10]\n" ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" ".inst 0x0ea16842 // bfcvtn v2.4h, v2.4s\n" - "ldr q6, [x11, #0x0]\n" - "ldr q7, [x11, #0x10]\n" + "ldr q7, [x11, #0x0]\n" + "ldr q6, [x11, #0x10]\n" ".inst 0x4ea16820 // bfcvtn2 v0.8h, v1.4s\n" ".inst 0x4ea16862 // bfcvtn2 v2.8h, v3.4s\n" - ".inst 0x6e44ec08 // bfmmla v8.4s, v0.8h, v4.8h\n" - ".inst 0x6e44ec54 // bfmmla v20.4s, v2.8h, v4.8h\n" - "ldr q4, [x10, #0x0]\n" + ".inst 0x6e45ec08 // bfmmla v8.4s, v0.8h, v5.8h\n" + ".inst 0x6e45ec54 // bfmmla v20.4s, v2.8h, v5.8h\n" + "ldr q5, [x10, #0x0]\n" "add x12, x12, #0x20\n" - ".inst 0x6e45ec0e // bfmmla v14.4s, v0.8h, v5.8h\n" - ".inst 0x6e45ec5a // bfmmla v26.4s, v2.8h, v5.8h\n" - "ldr q5, [x10, #0x10]\n" + ".inst 0x6e44ec0e // bfmmla v14.4s, v0.8h, v4.8h\n" + ".inst 0x6e44ec5a // bfmmla v26.4s, v2.8h, v4.8h\n" + "ldr q4, [x10, #0x10]\n" "add x11, x11, #0x20\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" - "ldr q6, [x9, #0x0]\n" + ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" + ".inst 0x6e47ec55 // bfmmla v21.4s, v2.8h, v7.8h\n" + "ldr q3, [x9, #0x0]\n" "add x10, x10, #0x20\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec5b // bfmmla v27.4s, v2.8h, v7.8h\n" - "ldr q7, [x9, #0x10]\n" + ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" + ".inst 0x6e46ec5b // bfmmla v27.4s, v2.8h, v6.8h\n" + "ldr q1, [x9, #0x10]\n" "add x9, x9, #0x20\n" - ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" - ".inst 0x6e44ec56 // bfmmla v22.4s, v2.8h, v4.8h\n" - "ldr q4, [x28, #0x0]\n" - ".inst 0x6e45ec10 // bfmmla v16.4s, v0.8h, v5.8h\n" - ".inst 0x6e45ec5c // bfmmla v28.4s, v2.8h, v5.8h\n" - "ldr q5, [x28, #0x10]\n" + ".inst 0x6e45ec0a // bfmmla v10.4s, v0.8h, v5.8h\n" + ".inst 0x6e45ec56 // bfmmla v22.4s, v2.8h, v5.8h\n" + "ldr q5, [x28, #0x0]\n" + ".inst 0x6e44ec10 // bfmmla v16.4s, v0.8h, v4.8h\n" + ".inst 0x6e44ec5c // bfmmla v28.4s, v2.8h, v4.8h\n" + "ldr q4, [x28, #0x10]\n" "add x28, x28, #0x20\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" - "ldr q6, [x27, #0x0]\n" - ".inst 0x6e47ec11 // bfmmla v17.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec5d // bfmmla v29.4s, v2.8h, v7.8h\n" - "ldr q7, [x27, #0x10]\n" + ".inst 0x6e43ec0b // bfmmla v11.4s, v0.8h, v3.8h\n" + ".inst 0x6e43ec57 // bfmmla v23.4s, v2.8h, v3.8h\n" + "ldr q3, [x27, #0x0]\n" + ".inst 0x6e41ec11 // bfmmla v17.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec5d // bfmmla v29.4s, v2.8h, v1.8h\n" + "ldr q1, [x27, #0x10]\n" "add x27, x27, #0x20\n" - ".inst 0x6e44ec0c // bfmmla v12.4s, v0.8h, v4.8h\n" - ".inst 0x6e44ec58 // bfmmla v24.4s, v2.8h, v4.8h\n" - ".inst 0x6e45ec12 // bfmmla v18.4s, v0.8h, v5.8h\n" - ".inst 0x6e45ec5e // bfmmla v30.4s, v2.8h, v5.8h\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec59 // bfmmla v25.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec13 // bfmmla v19.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec5f // bfmmla v31.4s, v2.8h, v7.8h\n" + ".inst 0x6e45ec0c // bfmmla v12.4s, v0.8h, v5.8h\n" + ".inst 0x6e45ec58 // bfmmla v24.4s, v2.8h, v5.8h\n" + ".inst 0x6e44ec12 // bfmmla v18.4s, v0.8h, v4.8h\n" + ".inst 0x6e44ec5e // bfmmla v30.4s, v2.8h, v4.8h\n" + ".inst 0x6e43ec0d // bfmmla v13.4s, v0.8h, v3.8h\n" + ".inst 0x6e43ec59 // bfmmla v25.4s, v2.8h, v3.8h\n" + ".inst 0x6e41ec13 // bfmmla v19.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec5f // bfmmla v31.4s, v2.8h, v1.8h\n" "161:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x26, x26, #0x1\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_bf16fp32_dot_8x12.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_bf16fp32_dot_8x12.hpp index e24dab68e81573ab6643a33d1eceb90d49bc18ea..745f89eff610cd4468bb6bb513ee859709a609bf 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_bf16fp32_dot_8x12.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_bf16fp32_dot_8x12.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,16 +10,16 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once #ifdef __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_bf16fp32_dot_8x12/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_bf16fp32_dot_8x12/generic.cpp index 2458d6a035bb897eba41c19aa4b96dac645d91d9..5f4fcac690fcdffbcca8045ea82f8c6e8de4d8ad 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_bf16fp32_dot_8x12/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_bf16fp32_dot_8x12/generic.cpp @@ -52,29 +52,29 @@ void a64_ffinterleaved_bf16fp32_dot_8x12( __asm__ __volatile__( "1:" // Height loop - "ldr x25, [%x[args_ptr], %[offsetof_Bpanel]]\n" - "ldr x24, [%x[args_ptr], %[offsetof_N]]\n" - "str x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" - "mov x23, %x[Apanel]\n" + "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n" + "ldr x25, [%x[args_ptr], %[offsetof_N]]\n" + "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" + "mov x24, %x[Apanel]\n" "2:" // Width loop - "ldr x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" + "ldr x23, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n" - "add x22, x25, x20, LSL #1\n" + "add x22, x23, x20, LSL #1\n" "add x21, x22, x20, LSL #1\n" "add x20, x21, x20, LSL #1\n" "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" - "cmp x24, #0x8\n" - "mov %x[Apanel], x23\n" + "cmp x25, #0x8\n" + "mov %x[Apanel], x24\n" "bgt 3f\n" - "cmp x24, #0x4\n" - "mov x21, x25\n" + "cmp x25, #0x4\n" + "mov x21, x23\n" "bgt 3f\n" - "mov x22, x25\n" + "mov x22, x23\n" "3:" // B setup done "ldr q0, [%x[Apanel], #0x0]\n" "ldr q1, [%x[Apanel], #0x10]\n" "movi v8.16b, #0x0\n" - "ldr q4, [x25, #0x0]\n" + "ldr q4, [x23, #0x0]\n" "ldr q5, [x22, #0x0]\n" "movi v9.16b, #0x0\n" "ldr q6, [x21, #0x0]\n" @@ -104,8 +104,8 @@ void a64_ffinterleaved_bf16fp32_dot_8x12( "movi v31.16b, #0x0\n" "blt 5f\n" "4:" // main loop head - "ldr q2, [%x[Apanel], #0x20]\n" - "ldr q3, [%x[Apanel], #0x30]\n" + "ldr q3, [%x[Apanel], #0x20]\n" + "ldr q7, [%x[Apanel], #0x30]\n" ".inst 0x4f40f088 // bfdot v8.4s, v4.8h, v0.h[0]\n" ".inst 0x4f60f08b // bfdot v11.4s, v4.8h, v0.h[1]\n" ".inst 0x4f40f88e // bfdot v14.4s, v4.8h, v0.h[2]\n" @@ -117,11 +117,11 @@ void a64_ffinterleaved_bf16fp32_dot_8x12( ".inst 0x4f41f89a // bfdot v26.4s, v4.8h, v1.h[2]\n" "add %x[Apanel], %x[Apanel], #0x40\n" ".inst 0x4f61f89d // bfdot v29.4s, v4.8h, v1.h[3]\n" - "ldr q4, [x25, #0x10]\n" + "ldr q4, [x23, #0x10]\n" ".inst 0x4f40f0a9 // bfdot v9.4s, v5.8h, v0.h[0]\n" ".inst 0x4f60f0ac // bfdot v12.4s, v5.8h, v0.h[1]\n" ".inst 0x4f40f8af // bfdot v15.4s, v5.8h, v0.h[2]\n" - "add x25, x25, #0x20\n" + "add x23, x23, #0x20\n" ".inst 0x4f60f8b2 // bfdot v18.4s, v5.8h, v0.h[3]\n" ".inst 0x4f41f0b5 // bfdot v21.4s, v5.8h, v1.h[0]\n" ".inst 0x4f61f0b8 // bfdot v24.4s, v5.8h, v1.h[1]\n" @@ -138,35 +138,35 @@ void a64_ffinterleaved_bf16fp32_dot_8x12( ".inst 0x4f61f0d9 // bfdot v25.4s, v6.8h, v1.h[1]\n" ".inst 0x4f41f8dc // bfdot v28.4s, v6.8h, v1.h[2]\n" ".inst 0x4f61f8df // bfdot v31.4s, v6.8h, v1.h[3]\n" - "ldr q6, [x21, #0x10]\n" + "ldr q2, [x21, #0x10]\n" "ldr q1, [%x[Apanel], #0x10]\n" "add x21, x21, #0x20\n" - ".inst 0x4f42f088 // bfdot v8.4s, v4.8h, v2.h[0]\n" - ".inst 0x4f62f08b // bfdot v11.4s, v4.8h, v2.h[1]\n" - ".inst 0x4f42f88e // bfdot v14.4s, v4.8h, v2.h[2]\n" - ".inst 0x4f62f891 // bfdot v17.4s, v4.8h, v2.h[3]\n" - ".inst 0x4f43f094 // bfdot v20.4s, v4.8h, v3.h[0]\n" - ".inst 0x4f63f097 // bfdot v23.4s, v4.8h, v3.h[1]\n" - ".inst 0x4f43f89a // bfdot v26.4s, v4.8h, v3.h[2]\n" - ".inst 0x4f63f89d // bfdot v29.4s, v4.8h, v3.h[3]\n" - "ldr q4, [x25, #0x0]\n" - ".inst 0x4f42f0a9 // bfdot v9.4s, v5.8h, v2.h[0]\n" - ".inst 0x4f62f0ac // bfdot v12.4s, v5.8h, v2.h[1]\n" - ".inst 0x4f42f8af // bfdot v15.4s, v5.8h, v2.h[2]\n" - ".inst 0x4f62f8b2 // bfdot v18.4s, v5.8h, v2.h[3]\n" - ".inst 0x4f43f0b5 // bfdot v21.4s, v5.8h, v3.h[0]\n" - ".inst 0x4f63f0b8 // bfdot v24.4s, v5.8h, v3.h[1]\n" - ".inst 0x4f43f8bb // bfdot v27.4s, v5.8h, v3.h[2]\n" - ".inst 0x4f63f8be // bfdot v30.4s, v5.8h, v3.h[3]\n" + ".inst 0x4f43f088 // bfdot v8.4s, v4.8h, v3.h[0]\n" + ".inst 0x4f63f08b // bfdot v11.4s, v4.8h, v3.h[1]\n" + ".inst 0x4f43f88e // bfdot v14.4s, v4.8h, v3.h[2]\n" + ".inst 0x4f63f891 // bfdot v17.4s, v4.8h, v3.h[3]\n" + ".inst 0x4f47f094 // bfdot v20.4s, v4.8h, v7.h[0]\n" + ".inst 0x4f67f097 // bfdot v23.4s, v4.8h, v7.h[1]\n" + ".inst 0x4f47f89a // bfdot v26.4s, v4.8h, v7.h[2]\n" + ".inst 0x4f67f89d // bfdot v29.4s, v4.8h, v7.h[3]\n" + "ldr q4, [x23, #0x0]\n" + ".inst 0x4f43f0a9 // bfdot v9.4s, v5.8h, v3.h[0]\n" + ".inst 0x4f63f0ac // bfdot v12.4s, v5.8h, v3.h[1]\n" + ".inst 0x4f43f8af // bfdot v15.4s, v5.8h, v3.h[2]\n" + ".inst 0x4f63f8b2 // bfdot v18.4s, v5.8h, v3.h[3]\n" + ".inst 0x4f47f0b5 // bfdot v21.4s, v5.8h, v7.h[0]\n" + ".inst 0x4f67f0b8 // bfdot v24.4s, v5.8h, v7.h[1]\n" + ".inst 0x4f47f8bb // bfdot v27.4s, v5.8h, v7.h[2]\n" + ".inst 0x4f67f8be // bfdot v30.4s, v5.8h, v7.h[3]\n" "ldr q5, [x22, #0x0]\n" - ".inst 0x4f42f0ca // bfdot v10.4s, v6.8h, v2.h[0]\n" - ".inst 0x4f62f0cd // bfdot v13.4s, v6.8h, v2.h[1]\n" - ".inst 0x4f42f8d0 // bfdot v16.4s, v6.8h, v2.h[2]\n" - ".inst 0x4f62f8d3 // bfdot v19.4s, v6.8h, v2.h[3]\n" - ".inst 0x4f43f0d6 // bfdot v22.4s, v6.8h, v3.h[0]\n" - ".inst 0x4f63f0d9 // bfdot v25.4s, v6.8h, v3.h[1]\n" - ".inst 0x4f43f8dc // bfdot v28.4s, v6.8h, v3.h[2]\n" - ".inst 0x4f63f8df // bfdot v31.4s, v6.8h, v3.h[3]\n" + ".inst 0x4f43f04a // bfdot v10.4s, v2.8h, v3.h[0]\n" + ".inst 0x4f63f04d // bfdot v13.4s, v2.8h, v3.h[1]\n" + ".inst 0x4f43f850 // bfdot v16.4s, v2.8h, v3.h[2]\n" + ".inst 0x4f63f853 // bfdot v19.4s, v2.8h, v3.h[3]\n" + ".inst 0x4f47f056 // bfdot v22.4s, v2.8h, v7.h[0]\n" + ".inst 0x4f67f059 // bfdot v25.4s, v2.8h, v7.h[1]\n" + ".inst 0x4f47f85c // bfdot v28.4s, v2.8h, v7.h[2]\n" + ".inst 0x4f67f85f // bfdot v31.4s, v2.8h, v7.h[3]\n" "ldr q6, [x21, #0x0]\n" "bge 4b\n" "5:" // main loop skip @@ -175,7 +175,7 @@ void a64_ffinterleaved_bf16fp32_dot_8x12( "add %x[Apanel], %x[Apanel], #0x20\n" ".inst 0x4f40f88e // bfdot v14.4s, v4.8h, v0.h[2]\n" ".inst 0x4f60f891 // bfdot v17.4s, v4.8h, v0.h[3]\n" - "add x25, x25, #0x10\n" + "add x23, x23, #0x10\n" ".inst 0x4f41f094 // bfdot v20.4s, v4.8h, v1.h[0]\n" ".inst 0x4f61f097 // bfdot v23.4s, v4.8h, v1.h[1]\n" "add x22, x22, #0x10\n" @@ -199,38 +199,38 @@ void a64_ffinterleaved_bf16fp32_dot_8x12( ".inst 0x4f41f8dc // bfdot v28.4s, v6.8h, v1.h[2]\n" ".inst 0x4f61f8df // bfdot v31.4s, v6.8h, v1.h[3]\n" "cbz x20, 6f\n" - "ldr q0, [%x[Apanel], #0x0]\n" - "ldr q1, [%x[Apanel], #0x10]\n" + "ldr q4, [%x[Apanel], #0x0]\n" + "ldr q3, [%x[Apanel], #0x10]\n" "add %x[Apanel], %x[Apanel], #0x20\n" - "ldr q7, [x25, #0x0]\n" - "ldr q4, [x22, #0x0]\n" - ".inst 0x4f40f0e8 // bfdot v8.4s, v7.8h, v0.h[0]\n" - "ldr q5, [x21, #0x0]\n" - ".inst 0x4f60f0eb // bfdot v11.4s, v7.8h, v0.h[1]\n" - ".inst 0x4f40f8ee // bfdot v14.4s, v7.8h, v0.h[2]\n" - ".inst 0x4f60f8f1 // bfdot v17.4s, v7.8h, v0.h[3]\n" - ".inst 0x4f41f0f4 // bfdot v20.4s, v7.8h, v1.h[0]\n" - ".inst 0x4f61f0f7 // bfdot v23.4s, v7.8h, v1.h[1]\n" - ".inst 0x4f41f8fa // bfdot v26.4s, v7.8h, v1.h[2]\n" - ".inst 0x4f61f8fd // bfdot v29.4s, v7.8h, v1.h[3]\n" - ".inst 0x4f40f089 // bfdot v9.4s, v4.8h, v0.h[0]\n" - ".inst 0x4f60f08c // bfdot v12.4s, v4.8h, v0.h[1]\n" - ".inst 0x4f40f88f // bfdot v15.4s, v4.8h, v0.h[2]\n" - ".inst 0x4f60f892 // bfdot v18.4s, v4.8h, v0.h[3]\n" - ".inst 0x4f41f095 // bfdot v21.4s, v4.8h, v1.h[0]\n" - ".inst 0x4f61f098 // bfdot v24.4s, v4.8h, v1.h[1]\n" - ".inst 0x4f41f89b // bfdot v27.4s, v4.8h, v1.h[2]\n" - ".inst 0x4f61f89e // bfdot v30.4s, v4.8h, v1.h[3]\n" - ".inst 0x4f40f0aa // bfdot v10.4s, v5.8h, v0.h[0]\n" - ".inst 0x4f60f0ad // bfdot v13.4s, v5.8h, v0.h[1]\n" - ".inst 0x4f40f8b0 // bfdot v16.4s, v5.8h, v0.h[2]\n" - ".inst 0x4f60f8b3 // bfdot v19.4s, v5.8h, v0.h[3]\n" - ".inst 0x4f41f0b6 // bfdot v22.4s, v5.8h, v1.h[0]\n" - ".inst 0x4f61f0b9 // bfdot v25.4s, v5.8h, v1.h[1]\n" - ".inst 0x4f41f8bc // bfdot v28.4s, v5.8h, v1.h[2]\n" - ".inst 0x4f61f8bf // bfdot v31.4s, v5.8h, v1.h[3]\n" + "ldr q2, [x23, #0x0]\n" + "ldr q1, [x22, #0x0]\n" + ".inst 0x4f44f048 // bfdot v8.4s, v2.8h, v4.h[0]\n" + "ldr q0, [x21, #0x0]\n" + ".inst 0x4f64f04b // bfdot v11.4s, v2.8h, v4.h[1]\n" + ".inst 0x4f44f84e // bfdot v14.4s, v2.8h, v4.h[2]\n" + ".inst 0x4f64f851 // bfdot v17.4s, v2.8h, v4.h[3]\n" + ".inst 0x4f43f054 // bfdot v20.4s, v2.8h, v3.h[0]\n" + ".inst 0x4f63f057 // bfdot v23.4s, v2.8h, v3.h[1]\n" + ".inst 0x4f43f85a // bfdot v26.4s, v2.8h, v3.h[2]\n" + ".inst 0x4f63f85d // bfdot v29.4s, v2.8h, v3.h[3]\n" + ".inst 0x4f44f029 // bfdot v9.4s, v1.8h, v4.h[0]\n" + ".inst 0x4f64f02c // bfdot v12.4s, v1.8h, v4.h[1]\n" + ".inst 0x4f44f82f // bfdot v15.4s, v1.8h, v4.h[2]\n" + ".inst 0x4f64f832 // bfdot v18.4s, v1.8h, v4.h[3]\n" + ".inst 0x4f43f035 // bfdot v21.4s, v1.8h, v3.h[0]\n" + ".inst 0x4f63f038 // bfdot v24.4s, v1.8h, v3.h[1]\n" + ".inst 0x4f43f83b // bfdot v27.4s, v1.8h, v3.h[2]\n" + ".inst 0x4f63f83e // bfdot v30.4s, v1.8h, v3.h[3]\n" + ".inst 0x4f44f00a // bfdot v10.4s, v0.8h, v4.h[0]\n" + ".inst 0x4f64f00d // bfdot v13.4s, v0.8h, v4.h[1]\n" + ".inst 0x4f44f810 // bfdot v16.4s, v0.8h, v4.h[2]\n" + ".inst 0x4f64f813 // bfdot v19.4s, v0.8h, v4.h[3]\n" + ".inst 0x4f43f016 // bfdot v22.4s, v0.8h, v3.h[0]\n" + ".inst 0x4f63f019 // bfdot v25.4s, v0.8h, v3.h[1]\n" + ".inst 0x4f43f81c // bfdot v28.4s, v0.8h, v3.h[2]\n" + ".inst 0x4f63f81f // bfdot v31.4s, v0.8h, v3.h[3]\n" "6:" // multiply loop done - "subs x24, x24, #0xc\n" + "subs x25, x25, #0xc\n" "str q8, [%x[Cpanel], #0x0]\n" "str q9, [%x[Cpanel], #0x10]\n" "str q10, [%x[Cpanel], #0x20]\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_bf16fp32_mmla_8x12.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_bf16fp32_mmla_8x12.hpp index c61315b80aa8ad51380be0320706f16657b076eb..cf4d74266af7bd98c69292e1e7167179ac85553e 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_bf16fp32_mmla_8x12.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_bf16fp32_mmla_8x12.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,16 +10,16 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once #ifdef __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_bf16fp32_mmla_8x12/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_bf16fp32_mmla_8x12/generic.cpp index 47991114af9d96e09a421ad3490c8a83432f4300..4a1c1b56383fa111770cefb1a22dbadeb0b72b39 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_bf16fp32_mmla_8x12/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_bf16fp32_mmla_8x12/generic.cpp @@ -52,37 +52,37 @@ void a64_ffinterleaved_bf16fp32_mmla_8x12( __asm__ __volatile__( "1:" // Height loop - "ldr x25, [%x[args_ptr], %[offsetof_Bpanel]]\n" - "ldr x24, [%x[args_ptr], %[offsetof_N]]\n" - "str x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" - "mov x23, %x[Apanel]\n" + "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n" + "ldr x25, [%x[args_ptr], %[offsetof_N]]\n" + "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" + "mov x24, %x[Apanel]\n" "2:" // Width loop - "ldr x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" + "ldr x23, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n" - "add x22, x25, x20, LSL #1\n" + "add x22, x23, x20, LSL #1\n" "add x21, x22, x20, LSL #1\n" "add x20, x21, x20, LSL #1\n" "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" - "cmp x24, #0x8\n" - "mov %x[Apanel], x23\n" + "cmp x25, #0x8\n" + "mov %x[Apanel], x24\n" "bgt 3f\n" - "cmp x24, #0x4\n" - "mov x21, x25\n" + "cmp x25, #0x4\n" + "mov x21, x23\n" "bgt 3f\n" - "mov x22, x25\n" + "mov x22, x23\n" "3:" // B setup done - "ldr q4, [x25, #0x0]\n" + "ldr q4, [x23, #0x0]\n" "ldr q0, [%x[Apanel], #0x0]\n" "movi v8.16b, #0x0\n" "ldr q1, [%x[Apanel], #0x10]\n" - "ldr q5, [x25, #0x10]\n" + "ldr q5, [x23, #0x10]\n" "movi v9.16b, #0x0\n" "ldr q2, [%x[Apanel], #0x20]\n" "ldr x20, [%x[args_ptr], %[offsetof_K]]\n" "cmp x20, #0x2\n" "movi v10.16b, #0x0\n" "movi v11.16b, #0x0\n" - "add x25, x25, #0x20\n" + "add x23, x23, #0x20\n" "movi v12.16b, #0x0\n" "movi v13.16b, #0x0\n" "add %x[Apanel], %x[Apanel], #0x30\n" @@ -106,31 +106,31 @@ void a64_ffinterleaved_bf16fp32_mmla_8x12( "movi v31.16b, #0x0\n" "blt 5f\n" "4:" // main loop head - "ldr q3, [%x[Apanel], #0x0]\n" - "ldr q6, [x22, #0x0]\n" + "ldr q6, [%x[Apanel], #0x0]\n" + "ldr q7, [x22, #0x0]\n" ".inst 0x6e44ec08 // bfmmla v8.4s, v0.8h, v4.8h\n" - "ldr q7, [x22, #0x10]\n" + "ldr q3, [x22, #0x10]\n" ".inst 0x6e45ec0b // bfmmla v11.4s, v0.8h, v5.8h\n" ".inst 0x6e44ec2e // bfmmla v14.4s, v1.8h, v4.8h\n" ".inst 0x6e45ec31 // bfmmla v17.4s, v1.8h, v5.8h\n" ".inst 0x6e44ec54 // bfmmla v20.4s, v2.8h, v4.8h\n" "sub x20, x20, #0x2\n" ".inst 0x6e45ec57 // bfmmla v23.4s, v2.8h, v5.8h\n" - ".inst 0x6e44ec7a // bfmmla v26.4s, v3.8h, v4.8h\n" + ".inst 0x6e44ecda // bfmmla v26.4s, v6.8h, v4.8h\n" "ldr q4, [x21, #0x0]\n" - ".inst 0x6e45ec7d // bfmmla v29.4s, v3.8h, v5.8h\n" + ".inst 0x6e45ecdd // bfmmla v29.4s, v6.8h, v5.8h\n" "ldr q5, [x21, #0x10]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" - ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" + ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" + ".inst 0x6e43ec0c // bfmmla v12.4s, v0.8h, v3.8h\n" + ".inst 0x6e47ec2f // bfmmla v15.4s, v1.8h, v7.8h\n" "cmp x20, #0x2\n" - ".inst 0x6e47ec32 // bfmmla v18.4s, v1.8h, v7.8h\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec58 // bfmmla v24.4s, v2.8h, v7.8h\n" - ".inst 0x6e46ec7b // bfmmla v27.4s, v3.8h, v6.8h\n" - "ldr q6, [x25, #0x0]\n" - ".inst 0x6e47ec7e // bfmmla v30.4s, v3.8h, v7.8h\n" - "ldr q7, [x25, #0x10]\n" + ".inst 0x6e43ec32 // bfmmla v18.4s, v1.8h, v3.8h\n" + ".inst 0x6e47ec55 // bfmmla v21.4s, v2.8h, v7.8h\n" + ".inst 0x6e43ec58 // bfmmla v24.4s, v2.8h, v3.8h\n" + ".inst 0x6e47ecdb // bfmmla v27.4s, v6.8h, v7.8h\n" + "ldr q7, [x23, #0x0]\n" + ".inst 0x6e43ecde // bfmmla v30.4s, v6.8h, v3.8h\n" + "ldr q3, [x23, #0x10]\n" ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" ".inst 0x6e45ec0d // bfmmla v13.4s, v0.8h, v5.8h\n" "ldr q0, [%x[Apanel], #0x10]\n" @@ -140,22 +140,22 @@ void a64_ffinterleaved_bf16fp32_mmla_8x12( ".inst 0x6e44ec56 // bfmmla v22.4s, v2.8h, v4.8h\n" ".inst 0x6e45ec59 // bfmmla v25.4s, v2.8h, v5.8h\n" "ldr q2, [%x[Apanel], #0x30]\n" - ".inst 0x6e44ec7c // bfmmla v28.4s, v3.8h, v4.8h\n" + ".inst 0x6e44ecdc // bfmmla v28.4s, v6.8h, v4.8h\n" "ldr q4, [x22, #0x20]\n" - ".inst 0x6e45ec7f // bfmmla v31.4s, v3.8h, v5.8h\n" - "ldr q3, [%x[Apanel], #0x40]\n" + ".inst 0x6e45ecdf // bfmmla v31.4s, v6.8h, v5.8h\n" + "ldr q6, [%x[Apanel], #0x40]\n" "ldr q5, [x22, #0x30]\n" - ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - ".inst 0x6e47ec31 // bfmmla v17.4s, v1.8h, v7.8h\n" + ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" + ".inst 0x6e43ec0b // bfmmla v11.4s, v0.8h, v3.8h\n" + ".inst 0x6e47ec2e // bfmmla v14.4s, v1.8h, v7.8h\n" + ".inst 0x6e43ec31 // bfmmla v17.4s, v1.8h, v3.8h\n" "add x22, x22, #0x40\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" - ".inst 0x6e46ec7a // bfmmla v26.4s, v3.8h, v6.8h\n" - "ldr q6, [x21, #0x20]\n" - ".inst 0x6e47ec7d // bfmmla v29.4s, v3.8h, v7.8h\n" - "ldr q7, [x21, #0x30]\n" + ".inst 0x6e47ec54 // bfmmla v20.4s, v2.8h, v7.8h\n" + ".inst 0x6e43ec57 // bfmmla v23.4s, v2.8h, v3.8h\n" + ".inst 0x6e47ecda // bfmmla v26.4s, v6.8h, v7.8h\n" + "ldr q7, [x21, #0x20]\n" + ".inst 0x6e43ecdd // bfmmla v29.4s, v6.8h, v3.8h\n" + "ldr q3, [x21, #0x30]\n" ".inst 0x6e44ec09 // bfmmla v9.4s, v0.8h, v4.8h\n" ".inst 0x6e45ec0c // bfmmla v12.4s, v0.8h, v5.8h\n" ".inst 0x6e44ec2f // bfmmla v15.4s, v1.8h, v4.8h\n" @@ -163,23 +163,23 @@ void a64_ffinterleaved_bf16fp32_mmla_8x12( "add x21, x21, #0x40\n" ".inst 0x6e44ec55 // bfmmla v21.4s, v2.8h, v4.8h\n" ".inst 0x6e45ec58 // bfmmla v24.4s, v2.8h, v5.8h\n" - ".inst 0x6e44ec7b // bfmmla v27.4s, v3.8h, v4.8h\n" - "ldr q4, [x25, #0x20]\n" - ".inst 0x6e45ec7e // bfmmla v30.4s, v3.8h, v5.8h\n" - "ldr q5, [x25, #0x30]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" + ".inst 0x6e44ecdb // bfmmla v27.4s, v6.8h, v4.8h\n" + "ldr q4, [x23, #0x20]\n" + ".inst 0x6e45ecde // bfmmla v30.4s, v6.8h, v5.8h\n" + "ldr q5, [x23, #0x30]\n" + ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" + ".inst 0x6e43ec0d // bfmmla v13.4s, v0.8h, v3.8h\n" "ldr q0, [%x[Apanel], #0x50]\n" - ".inst 0x6e46ec30 // bfmmla v16.4s, v1.8h, v6.8h\n" - ".inst 0x6e47ec33 // bfmmla v19.4s, v1.8h, v7.8h\n" + ".inst 0x6e47ec30 // bfmmla v16.4s, v1.8h, v7.8h\n" + ".inst 0x6e43ec33 // bfmmla v19.4s, v1.8h, v3.8h\n" "ldr q1, [%x[Apanel], #0x60]\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec59 // bfmmla v25.4s, v2.8h, v7.8h\n" + ".inst 0x6e47ec56 // bfmmla v22.4s, v2.8h, v7.8h\n" + ".inst 0x6e43ec59 // bfmmla v25.4s, v2.8h, v3.8h\n" "ldr q2, [%x[Apanel], #0x70]\n" - ".inst 0x6e46ec7c // bfmmla v28.4s, v3.8h, v6.8h\n" - ".inst 0x6e47ec7f // bfmmla v31.4s, v3.8h, v7.8h\n" + ".inst 0x6e47ecdc // bfmmla v28.4s, v6.8h, v7.8h\n" + ".inst 0x6e43ecdf // bfmmla v31.4s, v6.8h, v3.8h\n" "add %x[Apanel], %x[Apanel], #0x80\n" - "add x25, x25, #0x40\n" + "add x23, x23, #0x40\n" "bge 4b\n" "5:" // main loop skip "ldr q3, [%x[Apanel], #0x0]\n" @@ -215,88 +215,88 @@ void a64_ffinterleaved_bf16fp32_mmla_8x12( ".inst 0x6e44ec7c // bfmmla v28.4s, v3.8h, v4.8h\n" ".inst 0x6e45ec7f // bfmmla v31.4s, v3.8h, v5.8h\n" "cbz x20, 6f\n" - "ldr q6, [x25, #0x0]\n" - "ldr q0, [%x[Apanel], #0x0]\n" - ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - "ldr q1, [%x[Apanel], #0x10]\n" - "ldr q7, [x25, #0x10]\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" - "ldr q2, [%x[Apanel], #0x20]\n" - "ldr q3, [%x[Apanel], #0x30]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - "ldr q4, [x22, #0x0]\n" - "ldr q5, [x22, #0x10]\n" - ".inst 0x6e47ec31 // bfmmla v17.4s, v1.8h, v7.8h\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" + "ldr q1, [x23, #0x0]\n" + "ldr q7, [%x[Apanel], #0x0]\n" + ".inst 0x6e41ece8 // bfmmla v8.4s, v7.8h, v1.8h\n" + "ldr q6, [%x[Apanel], #0x10]\n" + "ldr q0, [x23, #0x10]\n" + ".inst 0x6e40eceb // bfmmla v11.4s, v7.8h, v0.8h\n" + "ldr q5, [%x[Apanel], #0x20]\n" + "ldr q4, [%x[Apanel], #0x30]\n" + ".inst 0x6e41ecce // bfmmla v14.4s, v6.8h, v1.8h\n" + "ldr q3, [x22, #0x0]\n" + "ldr q2, [x22, #0x10]\n" + ".inst 0x6e40ecd1 // bfmmla v17.4s, v6.8h, v0.8h\n" + ".inst 0x6e41ecb4 // bfmmla v20.4s, v5.8h, v1.8h\n" + ".inst 0x6e40ecb7 // bfmmla v23.4s, v5.8h, v0.8h\n" "add %x[Apanel], %x[Apanel], #0x40\n" - ".inst 0x6e46ec7a // bfmmla v26.4s, v3.8h, v6.8h\n" - "ldr q6, [x21, #0x0]\n" - ".inst 0x6e47ec7d // bfmmla v29.4s, v3.8h, v7.8h\n" - "ldr q7, [x21, #0x10]\n" - ".inst 0x6e44ec09 // bfmmla v9.4s, v0.8h, v4.8h\n" - ".inst 0x6e45ec0c // bfmmla v12.4s, v0.8h, v5.8h\n" - ".inst 0x6e44ec2f // bfmmla v15.4s, v1.8h, v4.8h\n" - ".inst 0x6e45ec32 // bfmmla v18.4s, v1.8h, v5.8h\n" - ".inst 0x6e44ec55 // bfmmla v21.4s, v2.8h, v4.8h\n" - ".inst 0x6e45ec58 // bfmmla v24.4s, v2.8h, v5.8h\n" - ".inst 0x6e44ec7b // bfmmla v27.4s, v3.8h, v4.8h\n" - ".inst 0x6e45ec7e // bfmmla v30.4s, v3.8h, v5.8h\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - ".inst 0x6e46ec30 // bfmmla v16.4s, v1.8h, v6.8h\n" - ".inst 0x6e47ec33 // bfmmla v19.4s, v1.8h, v7.8h\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec59 // bfmmla v25.4s, v2.8h, v7.8h\n" - ".inst 0x6e46ec7c // bfmmla v28.4s, v3.8h, v6.8h\n" - ".inst 0x6e47ec7f // bfmmla v31.4s, v3.8h, v7.8h\n" + ".inst 0x6e41ec9a // bfmmla v26.4s, v4.8h, v1.8h\n" + "ldr q1, [x21, #0x0]\n" + ".inst 0x6e40ec9d // bfmmla v29.4s, v4.8h, v0.8h\n" + "ldr q0, [x21, #0x10]\n" + ".inst 0x6e43ece9 // bfmmla v9.4s, v7.8h, v3.8h\n" + ".inst 0x6e42ecec // bfmmla v12.4s, v7.8h, v2.8h\n" + ".inst 0x6e43eccf // bfmmla v15.4s, v6.8h, v3.8h\n" + ".inst 0x6e42ecd2 // bfmmla v18.4s, v6.8h, v2.8h\n" + ".inst 0x6e43ecb5 // bfmmla v21.4s, v5.8h, v3.8h\n" + ".inst 0x6e42ecb8 // bfmmla v24.4s, v5.8h, v2.8h\n" + ".inst 0x6e43ec9b // bfmmla v27.4s, v4.8h, v3.8h\n" + ".inst 0x6e42ec9e // bfmmla v30.4s, v4.8h, v2.8h\n" + ".inst 0x6e41ecea // bfmmla v10.4s, v7.8h, v1.8h\n" + ".inst 0x6e40eced // bfmmla v13.4s, v7.8h, v0.8h\n" + ".inst 0x6e41ecd0 // bfmmla v16.4s, v6.8h, v1.8h\n" + ".inst 0x6e40ecd3 // bfmmla v19.4s, v6.8h, v0.8h\n" + ".inst 0x6e41ecb6 // bfmmla v22.4s, v5.8h, v1.8h\n" + ".inst 0x6e40ecb9 // bfmmla v25.4s, v5.8h, v0.8h\n" + ".inst 0x6e41ec9c // bfmmla v28.4s, v4.8h, v1.8h\n" + ".inst 0x6e40ec9f // bfmmla v31.4s, v4.8h, v0.8h\n" "6:" // multiply loop done - "subs x24, x24, #0xc\n" - "uzp1 v4.2d, v8.2d, v11.2d\n" + "subs x25, x25, #0xc\n" + "uzp1 v0.2d, v8.2d, v11.2d\n" "uzp2 v8.2d, v8.2d, v11.2d\n" - "uzp1 v11.2d, v9.2d, v12.2d\n" + "uzp1 v1.2d, v9.2d, v12.2d\n" "uzp2 v9.2d, v9.2d, v12.2d\n" - "str q4, [%x[Cpanel], #0x0]\n" - "uzp1 v12.2d, v10.2d, v13.2d\n" + "str q0, [%x[Cpanel], #0x0]\n" + "uzp1 v0.2d, v10.2d, v13.2d\n" "uzp2 v10.2d, v10.2d, v13.2d\n" - "str q11, [%x[Cpanel], #0x10]\n" - "str q12, [%x[Cpanel], #0x20]\n" - "uzp1 v13.2d, v14.2d, v17.2d\n" + "str q1, [%x[Cpanel], #0x10]\n" + "str q0, [%x[Cpanel], #0x20]\n" + "uzp1 v0.2d, v14.2d, v17.2d\n" "uzp2 v14.2d, v14.2d, v17.2d\n" "str q8, [%x[Cpanel], #0x30]\n" - "uzp1 v17.2d, v15.2d, v18.2d\n" + "uzp1 v2.2d, v15.2d, v18.2d\n" "uzp2 v15.2d, v15.2d, v18.2d\n" "str q9, [%x[Cpanel], #0x40]\n" - "uzp1 v18.2d, v16.2d, v19.2d\n" + "uzp1 v17.2d, v16.2d, v19.2d\n" "uzp2 v16.2d, v16.2d, v19.2d\n" "str q10, [%x[Cpanel], #0x50]\n" - "uzp1 v19.2d, v20.2d, v23.2d\n" + "uzp1 v1.2d, v20.2d, v23.2d\n" "uzp2 v20.2d, v20.2d, v23.2d\n" - "str q13, [%x[Cpanel], #0x60]\n" - "uzp1 v23.2d, v21.2d, v24.2d\n" + "str q0, [%x[Cpanel], #0x60]\n" + "uzp1 v0.2d, v21.2d, v24.2d\n" "uzp2 v21.2d, v21.2d, v24.2d\n" - "str q17, [%x[Cpanel], #0x70]\n" - "uzp1 v24.2d, v22.2d, v25.2d\n" + "str q2, [%x[Cpanel], #0x70]\n" + "uzp1 v23.2d, v22.2d, v25.2d\n" "uzp2 v22.2d, v22.2d, v25.2d\n" - "str q18, [%x[Cpanel], #0x80]\n" - "uzp1 v25.2d, v26.2d, v29.2d\n" + "str q17, [%x[Cpanel], #0x80]\n" + "uzp1 v19.2d, v26.2d, v29.2d\n" "uzp2 v26.2d, v26.2d, v29.2d\n" "str q14, [%x[Cpanel], #0x90]\n" - "uzp1 v29.2d, v27.2d, v30.2d\n" + "uzp1 v18.2d, v27.2d, v30.2d\n" "uzp2 v27.2d, v27.2d, v30.2d\n" "str q15, [%x[Cpanel], #0xa0]\n" - "uzp1 v30.2d, v28.2d, v31.2d\n" + "uzp1 v17.2d, v28.2d, v31.2d\n" "uzp2 v28.2d, v28.2d, v31.2d\n" "str q16, [%x[Cpanel], #0xb0]\n" - "str q19, [%x[Cpanel], #0xc0]\n" - "str q23, [%x[Cpanel], #0xd0]\n" - "str q24, [%x[Cpanel], #0xe0]\n" + "str q1, [%x[Cpanel], #0xc0]\n" + "str q0, [%x[Cpanel], #0xd0]\n" + "str q23, [%x[Cpanel], #0xe0]\n" "str q20, [%x[Cpanel], #0xf0]\n" "str q21, [%x[Cpanel], #0x100]\n" "str q22, [%x[Cpanel], #0x110]\n" - "str q25, [%x[Cpanel], #0x120]\n" - "str q29, [%x[Cpanel], #0x130]\n" - "str q30, [%x[Cpanel], #0x140]\n" + "str q19, [%x[Cpanel], #0x120]\n" + "str q18, [%x[Cpanel], #0x130]\n" + "str q17, [%x[Cpanel], #0x140]\n" "str q26, [%x[Cpanel], #0x150]\n" "str q27, [%x[Cpanel], #0x160]\n" "str q28, [%x[Cpanel], #0x170]\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_fp16_mla_8x24.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_fp16_mla_8x24.hpp index 149530687925da33203c5fd3905b3c3e91e2f01b..b9b4ad54df82759f065f2b4f412b4166de97f0a8 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_fp16_mla_8x24.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_fp16_mla_8x24.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,16 +10,16 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once #ifdef __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_fp16_mla_8x24/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_fp16_mla_8x24/generic.cpp index 36bfccf52fd315b0f891bf7c41f7e0151b937b4f..1e3f2f300b21eea5c2adcb6ce84511826c4b015f 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_fp16_mla_8x24/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_fp16_mla_8x24/generic.cpp @@ -51,27 +51,27 @@ void a64_ffinterleaved_fp16_mla_8x24( __asm__ __volatile__( "1:" // Height loop - "ldr x25, [%x[args_ptr], %[offsetof_Bpanel]]\n" - "ldr x24, [%x[args_ptr], %[offsetof_N]]\n" - "str x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" - "mov x23, %x[Apanel]\n" + "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n" + "ldr x25, [%x[args_ptr], %[offsetof_N]]\n" + "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" + "mov x24, %x[Apanel]\n" "2:" // Width loop - "ldr x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" + "ldr x23, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n" - "add x22, x25, x20, LSL #1\n" + "add x22, x23, x20, LSL #1\n" "add x21, x22, x20, LSL #1\n" "add x20, x21, x20, LSL #1\n" "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" - "cmp x24, #0x10\n" - "mov %x[Apanel], x23\n" + "cmp x25, #0x10\n" + "mov %x[Apanel], x24\n" "bgt 3f\n" - "cmp x24, #0x8\n" - "mov x21, x25\n" + "cmp x25, #0x8\n" + "mov x21, x23\n" "bgt 3f\n" - "mov x22, x25\n" + "mov x22, x23\n" "3:" // B setup done "ldr q0, [%x[Apanel], #0x0]\n" - "ldr q2, [x25, #0x0]\n" + "ldr q2, [x23, #0x0]\n" "movi v8.16b, #0x0\n" "ldr q3, [x22, #0x0]\n" "ldr q4, [x21, #0x0]\n" @@ -102,11 +102,11 @@ void a64_ffinterleaved_fp16_mla_8x24( "movi v31.16b, #0x0\n" "blt 5f\n" "4:" // main loop head - "ldr q1, [%x[Apanel], #0x10]\n" - "ldr q5, [x25, #0x10]\n" + "ldr q7, [%x[Apanel], #0x10]\n" + "ldr q6, [x23, #0x10]\n" "fmla v8.8h, v2.8h, v0.h[0]\n" - "ldr q6, [x22, #0x10]\n" - "ldr q7, [x21, #0x10]\n" + "ldr q5, [x22, #0x10]\n" + "ldr q1, [x21, #0x10]\n" "fmla v11.8h, v2.8h, v0.h[1]\n" "fmla v14.8h, v2.8h, v0.h[2]\n" "fmla v17.8h, v2.8h, v0.h[3]\n" @@ -119,8 +119,8 @@ void a64_ffinterleaved_fp16_mla_8x24( "add %x[Apanel], %x[Apanel], #0x20\n" "fmla v9.8h, v3.8h, v0.h[0]\n" "fmla v12.8h, v3.8h, v0.h[1]\n" - "add x25, x25, #0x20\n" - "ldr q2, [x25, #0x0]\n" + "add x23, x23, #0x20\n" + "ldr q2, [x23, #0x0]\n" "fmla v15.8h, v3.8h, v0.h[2]\n" "fmla v18.8h, v3.8h, v0.h[3]\n" "fmla v21.8h, v3.8h, v0.h[4]\n" @@ -140,30 +140,30 @@ void a64_ffinterleaved_fp16_mla_8x24( "fmla v31.8h, v4.8h, v0.h[7]\n" "ldr q0, [%x[Apanel], #0x0]\n" "ldr q4, [x21, #0x0]\n" - "fmla v8.8h, v5.8h, v1.h[0]\n" - "fmla v11.8h, v5.8h, v1.h[1]\n" - "fmla v14.8h, v5.8h, v1.h[2]\n" - "fmla v17.8h, v5.8h, v1.h[3]\n" - "fmla v20.8h, v5.8h, v1.h[4]\n" - "fmla v23.8h, v5.8h, v1.h[5]\n" - "fmla v26.8h, v5.8h, v1.h[6]\n" - "fmla v29.8h, v5.8h, v1.h[7]\n" - "fmla v9.8h, v6.8h, v1.h[0]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "fmla v15.8h, v6.8h, v1.h[2]\n" - "fmla v18.8h, v6.8h, v1.h[3]\n" - "fmla v21.8h, v6.8h, v1.h[4]\n" - "fmla v24.8h, v6.8h, v1.h[5]\n" - "fmla v27.8h, v6.8h, v1.h[6]\n" - "fmla v30.8h, v6.8h, v1.h[7]\n" - "fmla v10.8h, v7.8h, v1.h[0]\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "fmla v16.8h, v7.8h, v1.h[2]\n" - "fmla v19.8h, v7.8h, v1.h[3]\n" - "fmla v22.8h, v7.8h, v1.h[4]\n" - "fmla v25.8h, v7.8h, v1.h[5]\n" - "fmla v28.8h, v7.8h, v1.h[6]\n" - "fmla v31.8h, v7.8h, v1.h[7]\n" + "fmla v8.8h, v6.8h, v7.h[0]\n" + "fmla v11.8h, v6.8h, v7.h[1]\n" + "fmla v14.8h, v6.8h, v7.h[2]\n" + "fmla v17.8h, v6.8h, v7.h[3]\n" + "fmla v20.8h, v6.8h, v7.h[4]\n" + "fmla v23.8h, v6.8h, v7.h[5]\n" + "fmla v26.8h, v6.8h, v7.h[6]\n" + "fmla v29.8h, v6.8h, v7.h[7]\n" + "fmla v9.8h, v5.8h, v7.h[0]\n" + "fmla v12.8h, v5.8h, v7.h[1]\n" + "fmla v15.8h, v5.8h, v7.h[2]\n" + "fmla v18.8h, v5.8h, v7.h[3]\n" + "fmla v21.8h, v5.8h, v7.h[4]\n" + "fmla v24.8h, v5.8h, v7.h[5]\n" + "fmla v27.8h, v5.8h, v7.h[6]\n" + "fmla v30.8h, v5.8h, v7.h[7]\n" + "fmla v10.8h, v1.8h, v7.h[0]\n" + "fmla v13.8h, v1.8h, v7.h[1]\n" + "fmla v16.8h, v1.8h, v7.h[2]\n" + "fmla v19.8h, v1.8h, v7.h[3]\n" + "fmla v22.8h, v1.8h, v7.h[4]\n" + "fmla v25.8h, v1.8h, v7.h[5]\n" + "fmla v28.8h, v1.8h, v7.h[6]\n" + "fmla v31.8h, v1.8h, v7.h[7]\n" "bge 4b\n" "5:" // main loop skip "fmla v8.8h, v2.8h, v0.h[0]\n" @@ -171,7 +171,7 @@ void a64_ffinterleaved_fp16_mla_8x24( "add %x[Apanel], %x[Apanel], #0x10\n" "fmla v14.8h, v2.8h, v0.h[2]\n" "fmla v17.8h, v2.8h, v0.h[3]\n" - "add x25, x25, #0x10\n" + "add x23, x23, #0x10\n" "fmla v20.8h, v2.8h, v0.h[4]\n" "fmla v23.8h, v2.8h, v0.h[5]\n" "add x22, x22, #0x10\n" @@ -195,37 +195,37 @@ void a64_ffinterleaved_fp16_mla_8x24( "fmla v28.8h, v4.8h, v0.h[6]\n" "fmla v31.8h, v4.8h, v0.h[7]\n" "cbz x20, 6f\n" - "ldr q0, [%x[Apanel], #0x0]\n" - "ldr q5, [x25, #0x0]\n" - "fmla v8.8h, v5.8h, v0.h[0]\n" - "ldr q6, [x22, #0x0]\n" - "ldr q7, [x21, #0x0]\n" - "fmla v11.8h, v5.8h, v0.h[1]\n" - "fmla v14.8h, v5.8h, v0.h[2]\n" - "fmla v17.8h, v5.8h, v0.h[3]\n" + "ldr q3, [%x[Apanel], #0x0]\n" + "ldr q2, [x23, #0x0]\n" + "fmla v8.8h, v2.8h, v3.h[0]\n" + "ldr q1, [x22, #0x0]\n" + "ldr q0, [x21, #0x0]\n" + "fmla v11.8h, v2.8h, v3.h[1]\n" + "fmla v14.8h, v2.8h, v3.h[2]\n" + "fmla v17.8h, v2.8h, v3.h[3]\n" "add %x[Apanel], %x[Apanel], #0x10\n" - "fmla v20.8h, v5.8h, v0.h[4]\n" - "fmla v23.8h, v5.8h, v0.h[5]\n" - "fmla v26.8h, v5.8h, v0.h[6]\n" - "fmla v29.8h, v5.8h, v0.h[7]\n" - "fmla v9.8h, v6.8h, v0.h[0]\n" - "fmla v12.8h, v6.8h, v0.h[1]\n" - "fmla v15.8h, v6.8h, v0.h[2]\n" - "fmla v18.8h, v6.8h, v0.h[3]\n" - "fmla v21.8h, v6.8h, v0.h[4]\n" - "fmla v24.8h, v6.8h, v0.h[5]\n" - "fmla v27.8h, v6.8h, v0.h[6]\n" - "fmla v30.8h, v6.8h, v0.h[7]\n" - "fmla v10.8h, v7.8h, v0.h[0]\n" - "fmla v13.8h, v7.8h, v0.h[1]\n" - "fmla v16.8h, v7.8h, v0.h[2]\n" - "fmla v19.8h, v7.8h, v0.h[3]\n" - "fmla v22.8h, v7.8h, v0.h[4]\n" - "fmla v25.8h, v7.8h, v0.h[5]\n" - "fmla v28.8h, v7.8h, v0.h[6]\n" - "fmla v31.8h, v7.8h, v0.h[7]\n" + "fmla v20.8h, v2.8h, v3.h[4]\n" + "fmla v23.8h, v2.8h, v3.h[5]\n" + "fmla v26.8h, v2.8h, v3.h[6]\n" + "fmla v29.8h, v2.8h, v3.h[7]\n" + "fmla v9.8h, v1.8h, v3.h[0]\n" + "fmla v12.8h, v1.8h, v3.h[1]\n" + "fmla v15.8h, v1.8h, v3.h[2]\n" + "fmla v18.8h, v1.8h, v3.h[3]\n" + "fmla v21.8h, v1.8h, v3.h[4]\n" + "fmla v24.8h, v1.8h, v3.h[5]\n" + "fmla v27.8h, v1.8h, v3.h[6]\n" + "fmla v30.8h, v1.8h, v3.h[7]\n" + "fmla v10.8h, v0.8h, v3.h[0]\n" + "fmla v13.8h, v0.8h, v3.h[1]\n" + "fmla v16.8h, v0.8h, v3.h[2]\n" + "fmla v19.8h, v0.8h, v3.h[3]\n" + "fmla v22.8h, v0.8h, v3.h[4]\n" + "fmla v25.8h, v0.8h, v3.h[5]\n" + "fmla v28.8h, v0.8h, v3.h[6]\n" + "fmla v31.8h, v0.8h, v3.h[7]\n" "6:" // multiply loop done - "subs x24, x24, #0x18\n" + "subs x25, x25, #0x18\n" "str q8, [%x[Cpanel], #0x0]\n" "str q9, [%x[Cpanel], #0x10]\n" "str q10, [%x[Cpanel], #0x20]\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_fp32_mla_8x12.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_fp32_mla_8x12.hpp index f2a836c9b4c75c1442f10ec6ab66c4dc7e4baf97..c4445ba14a671eff902376a6722de45154b46224 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_fp32_mla_8x12.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_fp32_mla_8x12.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,16 +10,16 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once #ifdef __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_fp32_mla_8x12/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_fp32_mla_8x12/generic.cpp index ec99d64f4a043c44d63fd49d54bc67cbe6bfd97a..6de0a380ebfcff85c5a8b4193e02156a9b005941 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_fp32_mla_8x12/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_ffinterleaved_fp32_mla_8x12/generic.cpp @@ -51,29 +51,29 @@ void a64_ffinterleaved_fp32_mla_8x12( __asm__ __volatile__( "1:" // Height loop - "ldr x25, [%x[args_ptr], %[offsetof_Bpanel]]\n" - "ldr x24, [%x[args_ptr], %[offsetof_N]]\n" - "str x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" - "mov x23, %x[Apanel]\n" + "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n" + "ldr x25, [%x[args_ptr], %[offsetof_N]]\n" + "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" + "mov x24, %x[Apanel]\n" "2:" // Width loop - "ldr x25, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" + "ldr x23, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n" - "add x22, x25, x20, LSL #2\n" + "add x22, x23, x20, LSL #2\n" "add x21, x22, x20, LSL #2\n" "add x20, x21, x20, LSL #2\n" "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" - "cmp x24, #0x8\n" - "mov %x[Apanel], x23\n" + "cmp x25, #0x8\n" + "mov %x[Apanel], x24\n" "bgt 3f\n" - "cmp x24, #0x4\n" - "mov x21, x25\n" + "cmp x25, #0x4\n" + "mov x21, x23\n" "bgt 3f\n" - "mov x22, x25\n" + "mov x22, x23\n" "3:" // B setup done "ldr q0, [%x[Apanel], #0x0]\n" "ldr q1, [%x[Apanel], #0x10]\n" "movi v8.16b, #0x0\n" - "ldr q4, [x25, #0x0]\n" + "ldr q4, [x23, #0x0]\n" "ldr q5, [x22, #0x0]\n" "movi v9.16b, #0x0\n" "ldr q6, [x21, #0x0]\n" @@ -103,10 +103,10 @@ void a64_ffinterleaved_fp32_mla_8x12( "movi v31.16b, #0x0\n" "blt 5f\n" "4:" // main loop head - "ldr q2, [%x[Apanel], #0x20]\n" - "ldr q3, [%x[Apanel], #0x30]\n" + "ldr q3, [%x[Apanel], #0x20]\n" + "ldr q7, [%x[Apanel], #0x30]\n" "fmla v8.4s, v4.4s, v0.s[0]\n" - "ldr q7, [x25, #0x10]\n" + "ldr q2, [x23, #0x10]\n" "fmla v11.4s, v4.4s, v0.s[1]\n" "fmla v14.4s, v4.4s, v0.s[2]\n" "fmla v17.4s, v4.4s, v0.s[3]\n" @@ -136,36 +136,36 @@ void a64_ffinterleaved_fp32_mla_8x12( "fmla v28.4s, v6.4s, v1.s[2]\n" "fmla v31.4s, v6.4s, v1.s[3]\n" "ldr q1, [%x[Apanel], #0x50]\n" - "ldr q6, [x25, #0x20]\n" - "fmla v8.4s, v7.4s, v2.s[0]\n" - "fmla v11.4s, v7.4s, v2.s[1]\n" - "fmla v14.4s, v7.4s, v2.s[2]\n" - "fmla v17.4s, v7.4s, v2.s[3]\n" - "fmla v20.4s, v7.4s, v3.s[0]\n" - "fmla v23.4s, v7.4s, v3.s[1]\n" - "fmla v26.4s, v7.4s, v3.s[2]\n" - "fmla v29.4s, v7.4s, v3.s[3]\n" - "ldr q7, [x22, #0x20]\n" - "fmla v9.4s, v4.4s, v2.s[0]\n" - "fmla v12.4s, v4.4s, v2.s[1]\n" - "fmla v15.4s, v4.4s, v2.s[2]\n" - "fmla v18.4s, v4.4s, v2.s[3]\n" - "fmla v21.4s, v4.4s, v3.s[0]\n" - "fmla v24.4s, v4.4s, v3.s[1]\n" - "fmla v27.4s, v4.4s, v3.s[2]\n" - "fmla v30.4s, v4.4s, v3.s[3]\n" + "ldr q6, [x23, #0x20]\n" + "fmla v8.4s, v2.4s, v3.s[0]\n" + "fmla v11.4s, v2.4s, v3.s[1]\n" + "fmla v14.4s, v2.4s, v3.s[2]\n" + "fmla v17.4s, v2.4s, v3.s[3]\n" + "fmla v20.4s, v2.4s, v7.s[0]\n" + "fmla v23.4s, v2.4s, v7.s[1]\n" + "fmla v26.4s, v2.4s, v7.s[2]\n" + "fmla v29.4s, v2.4s, v7.s[3]\n" + "ldr q2, [x22, #0x20]\n" + "fmla v9.4s, v4.4s, v3.s[0]\n" + "fmla v12.4s, v4.4s, v3.s[1]\n" + "fmla v15.4s, v4.4s, v3.s[2]\n" + "fmla v18.4s, v4.4s, v3.s[3]\n" + "fmla v21.4s, v4.4s, v7.s[0]\n" + "fmla v24.4s, v4.4s, v7.s[1]\n" + "fmla v27.4s, v4.4s, v7.s[2]\n" + "fmla v30.4s, v4.4s, v7.s[3]\n" "ldr q4, [x21, #0x20]\n" - "fmla v10.4s, v5.4s, v2.s[0]\n" - "fmla v13.4s, v5.4s, v2.s[1]\n" - "fmla v16.4s, v5.4s, v2.s[2]\n" - "fmla v19.4s, v5.4s, v2.s[3]\n" - "ldr q2, [%x[Apanel], #0x60]\n" - "fmla v22.4s, v5.4s, v3.s[0]\n" - "fmla v25.4s, v5.4s, v3.s[1]\n" - "fmla v28.4s, v5.4s, v3.s[2]\n" - "fmla v31.4s, v5.4s, v3.s[3]\n" - "ldr q3, [%x[Apanel], #0x70]\n" - "ldr q5, [x25, #0x30]\n" + "fmla v10.4s, v5.4s, v3.s[0]\n" + "fmla v13.4s, v5.4s, v3.s[1]\n" + "fmla v16.4s, v5.4s, v3.s[2]\n" + "fmla v19.4s, v5.4s, v3.s[3]\n" + "ldr q3, [%x[Apanel], #0x60]\n" + "fmla v22.4s, v5.4s, v7.s[0]\n" + "fmla v25.4s, v5.4s, v7.s[1]\n" + "fmla v28.4s, v5.4s, v7.s[2]\n" + "fmla v31.4s, v5.4s, v7.s[3]\n" + "ldr q7, [%x[Apanel], #0x70]\n" + "ldr q5, [x23, #0x30]\n" "fmla v8.4s, v6.4s, v0.s[0]\n" "fmla v11.4s, v6.4s, v0.s[1]\n" "fmla v14.4s, v6.4s, v0.s[2]\n" @@ -173,20 +173,20 @@ void a64_ffinterleaved_fp32_mla_8x12( "add %x[Apanel], %x[Apanel], #0x80\n" "fmla v20.4s, v6.4s, v1.s[0]\n" "fmla v23.4s, v6.4s, v1.s[1]\n" - "add x25, x25, #0x40\n" + "add x23, x23, #0x40\n" "fmla v26.4s, v6.4s, v1.s[2]\n" "fmla v29.4s, v6.4s, v1.s[3]\n" "ldr q6, [x22, #0x30]\n" - "fmla v9.4s, v7.4s, v0.s[0]\n" - "fmla v12.4s, v7.4s, v0.s[1]\n" + "fmla v9.4s, v2.4s, v0.s[0]\n" + "fmla v12.4s, v2.4s, v0.s[1]\n" "add x22, x22, #0x40\n" - "fmla v15.4s, v7.4s, v0.s[2]\n" - "fmla v18.4s, v7.4s, v0.s[3]\n" - "fmla v21.4s, v7.4s, v1.s[0]\n" - "fmla v24.4s, v7.4s, v1.s[1]\n" - "fmla v27.4s, v7.4s, v1.s[2]\n" - "fmla v30.4s, v7.4s, v1.s[3]\n" - "ldr q7, [x21, #0x30]\n" + "fmla v15.4s, v2.4s, v0.s[2]\n" + "fmla v18.4s, v2.4s, v0.s[3]\n" + "fmla v21.4s, v2.4s, v1.s[0]\n" + "fmla v24.4s, v2.4s, v1.s[1]\n" + "fmla v27.4s, v2.4s, v1.s[2]\n" + "fmla v30.4s, v2.4s, v1.s[3]\n" + "ldr q2, [x21, #0x30]\n" "fmla v10.4s, v4.4s, v0.s[0]\n" "fmla v13.4s, v4.4s, v0.s[1]\n" "add x21, x21, #0x40\n" @@ -198,33 +198,33 @@ void a64_ffinterleaved_fp32_mla_8x12( "fmla v28.4s, v4.4s, v1.s[2]\n" "fmla v31.4s, v4.4s, v1.s[3]\n" "ldr q1, [%x[Apanel], #0x10]\n" - "ldr q4, [x25, #0x0]\n" - "fmla v8.4s, v5.4s, v2.s[0]\n" - "fmla v11.4s, v5.4s, v2.s[1]\n" - "fmla v14.4s, v5.4s, v2.s[2]\n" - "fmla v17.4s, v5.4s, v2.s[3]\n" - "fmla v20.4s, v5.4s, v3.s[0]\n" - "fmla v23.4s, v5.4s, v3.s[1]\n" - "fmla v26.4s, v5.4s, v3.s[2]\n" - "fmla v29.4s, v5.4s, v3.s[3]\n" + "ldr q4, [x23, #0x0]\n" + "fmla v8.4s, v5.4s, v3.s[0]\n" + "fmla v11.4s, v5.4s, v3.s[1]\n" + "fmla v14.4s, v5.4s, v3.s[2]\n" + "fmla v17.4s, v5.4s, v3.s[3]\n" + "fmla v20.4s, v5.4s, v7.s[0]\n" + "fmla v23.4s, v5.4s, v7.s[1]\n" + "fmla v26.4s, v5.4s, v7.s[2]\n" + "fmla v29.4s, v5.4s, v7.s[3]\n" "ldr q5, [x22, #0x0]\n" - "fmla v9.4s, v6.4s, v2.s[0]\n" - "fmla v12.4s, v6.4s, v2.s[1]\n" - "fmla v15.4s, v6.4s, v2.s[2]\n" - "fmla v18.4s, v6.4s, v2.s[3]\n" - "fmla v21.4s, v6.4s, v3.s[0]\n" - "fmla v24.4s, v6.4s, v3.s[1]\n" - "fmla v27.4s, v6.4s, v3.s[2]\n" - "fmla v30.4s, v6.4s, v3.s[3]\n" + "fmla v9.4s, v6.4s, v3.s[0]\n" + "fmla v12.4s, v6.4s, v3.s[1]\n" + "fmla v15.4s, v6.4s, v3.s[2]\n" + "fmla v18.4s, v6.4s, v3.s[3]\n" + "fmla v21.4s, v6.4s, v7.s[0]\n" + "fmla v24.4s, v6.4s, v7.s[1]\n" + "fmla v27.4s, v6.4s, v7.s[2]\n" + "fmla v30.4s, v6.4s, v7.s[3]\n" "ldr q6, [x21, #0x0]\n" - "fmla v10.4s, v7.4s, v2.s[0]\n" - "fmla v13.4s, v7.4s, v2.s[1]\n" - "fmla v16.4s, v7.4s, v2.s[2]\n" - "fmla v19.4s, v7.4s, v2.s[3]\n" - "fmla v22.4s, v7.4s, v3.s[0]\n" - "fmla v25.4s, v7.4s, v3.s[1]\n" - "fmla v28.4s, v7.4s, v3.s[2]\n" - "fmla v31.4s, v7.4s, v3.s[3]\n" + "fmla v10.4s, v2.4s, v3.s[0]\n" + "fmla v13.4s, v2.4s, v3.s[1]\n" + "fmla v16.4s, v2.4s, v3.s[2]\n" + "fmla v19.4s, v2.4s, v3.s[3]\n" + "fmla v22.4s, v2.4s, v7.s[0]\n" + "fmla v25.4s, v2.4s, v7.s[1]\n" + "fmla v28.4s, v2.4s, v7.s[2]\n" + "fmla v31.4s, v2.4s, v7.s[3]\n" "bge 4b\n" "5:" // main loop skip "fmla v8.4s, v4.4s, v0.s[0]\n" @@ -232,7 +232,7 @@ void a64_ffinterleaved_fp32_mla_8x12( "add %x[Apanel], %x[Apanel], #0x20\n" "fmla v14.4s, v4.4s, v0.s[2]\n" "fmla v17.4s, v4.4s, v0.s[3]\n" - "add x25, x25, #0x10\n" + "add x23, x23, #0x10\n" "fmla v20.4s, v4.4s, v1.s[0]\n" "fmla v23.4s, v4.4s, v1.s[1]\n" "add x22, x22, #0x10\n" @@ -257,43 +257,43 @@ void a64_ffinterleaved_fp32_mla_8x12( "fmla v31.4s, v6.4s, v1.s[3]\n" "cbz x20, 7f\n" "6:" // odd loop - "ldr q0, [%x[Apanel], #0x0]\n" - "ldr q1, [%x[Apanel], #0x10]\n" + "ldr q4, [%x[Apanel], #0x0]\n" + "ldr q3, [%x[Apanel], #0x10]\n" "subs x20, x20, #0x1\n" - "ldr q7, [x25, #0x0]\n" - "ldr q4, [x22, #0x0]\n" - "fmla v8.4s, v7.4s, v0.s[0]\n" - "ldr q5, [x21, #0x0]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "fmla v14.4s, v7.4s, v0.s[2]\n" - "fmla v17.4s, v7.4s, v0.s[3]\n" - "fmla v20.4s, v7.4s, v1.s[0]\n" + "ldr q2, [x23, #0x0]\n" + "ldr q1, [x22, #0x0]\n" + "fmla v8.4s, v2.4s, v4.s[0]\n" + "ldr q0, [x21, #0x0]\n" + "fmla v11.4s, v2.4s, v4.s[1]\n" + "fmla v14.4s, v2.4s, v4.s[2]\n" + "fmla v17.4s, v2.4s, v4.s[3]\n" + "fmla v20.4s, v2.4s, v3.s[0]\n" "add %x[Apanel], %x[Apanel], #0x20\n" - "fmla v23.4s, v7.4s, v1.s[1]\n" - "fmla v26.4s, v7.4s, v1.s[2]\n" - "add x25, x25, #0x10\n" - "fmla v29.4s, v7.4s, v1.s[3]\n" - "fmla v9.4s, v4.4s, v0.s[0]\n" + "fmla v23.4s, v2.4s, v3.s[1]\n" + "fmla v26.4s, v2.4s, v3.s[2]\n" + "add x23, x23, #0x10\n" + "fmla v29.4s, v2.4s, v3.s[3]\n" + "fmla v9.4s, v1.4s, v4.s[0]\n" "add x22, x22, #0x10\n" - "fmla v12.4s, v4.4s, v0.s[1]\n" - "fmla v15.4s, v4.4s, v0.s[2]\n" + "fmla v12.4s, v1.4s, v4.s[1]\n" + "fmla v15.4s, v1.4s, v4.s[2]\n" "add x21, x21, #0x10\n" - "fmla v18.4s, v4.4s, v0.s[3]\n" - "fmla v21.4s, v4.4s, v1.s[0]\n" - "fmla v24.4s, v4.4s, v1.s[1]\n" - "fmla v27.4s, v4.4s, v1.s[2]\n" - "fmla v30.4s, v4.4s, v1.s[3]\n" - "fmla v10.4s, v5.4s, v0.s[0]\n" - "fmla v13.4s, v5.4s, v0.s[1]\n" - "fmla v16.4s, v5.4s, v0.s[2]\n" - "fmla v19.4s, v5.4s, v0.s[3]\n" - "fmla v22.4s, v5.4s, v1.s[0]\n" - "fmla v25.4s, v5.4s, v1.s[1]\n" - "fmla v28.4s, v5.4s, v1.s[2]\n" - "fmla v31.4s, v5.4s, v1.s[3]\n" + "fmla v18.4s, v1.4s, v4.s[3]\n" + "fmla v21.4s, v1.4s, v3.s[0]\n" + "fmla v24.4s, v1.4s, v3.s[1]\n" + "fmla v27.4s, v1.4s, v3.s[2]\n" + "fmla v30.4s, v1.4s, v3.s[3]\n" + "fmla v10.4s, v0.4s, v4.s[0]\n" + "fmla v13.4s, v0.4s, v4.s[1]\n" + "fmla v16.4s, v0.4s, v4.s[2]\n" + "fmla v19.4s, v0.4s, v4.s[3]\n" + "fmla v22.4s, v0.4s, v3.s[0]\n" + "fmla v25.4s, v0.4s, v3.s[1]\n" + "fmla v28.4s, v0.4s, v3.s[2]\n" + "fmla v31.4s, v0.4s, v3.s[3]\n" "bne 6b\n" "7:" // multiply loop done - "subs x24, x24, #0xc\n" + "subs x25, x25, #0xc\n" "str q8, [%x[Cpanel], #0x0]\n" "str q9, [%x[Cpanel], #0x10]\n" "str q10, [%x[Cpanel], #0x20]\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_dot_6x16.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_dot_6x16.hpp index 3b8770e1530683a84de3604c1111d756f9106df2..f1427669ea40c01127cc88eabdff37191f53666f 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_dot_6x16.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_dot_6x16.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef __aarch64__ + #include "../std_transforms_fixed.hpp" #include "../bfloat.hpp" #include "../performance_parameters.hpp" @@ -99,5 +99,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_dot_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_dot_6x16/generic.cpp index 02d2434356e0d5995ba0528171908fb8594bfc6b..fc323ea4fc9821f9fc64c02dd6b5ab170bcd9e98 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_dot_6x16/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_dot_6x16/generic.cpp @@ -93,7 +93,6 @@ void a64_hybrid_bf16fp32_dot_6x16 ( break; } __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x6\n" "bge 176f\n" @@ -190,11 +189,11 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "15:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 16f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 17f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -211,37 +210,37 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "blt 19f\n" "18:" // Height 1: Multiply loop: Main loop head ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q17, [x10, #0x20]\n" ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x4f60f0c8 // bfdot v8.4s, v6.8h, v0.h[1]\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x4f60f0e9 // bfdot v9.4s, v7.8h, v0.h[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x4f60f0ca // bfdot v10.4s, v6.8h, v0.h[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x4f60f0eb // bfdot v11.4s, v7.8h, v0.h[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x4f40f8c8 // bfdot v8.4s, v6.8h, v0.h[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x4f40f8e9 // bfdot v9.4s, v7.8h, v0.h[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x4f40f8ca // bfdot v10.4s, v6.8h, v0.h[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x4f40f8eb // bfdot v11.4s, v7.8h, v0.h[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x4f60f8c8 // bfdot v8.4s, v6.8h, v0.h[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x4f60f8e9 // bfdot v9.4s, v7.8h, v0.h[3]\n" - "ldr q7, [x10, #0xf0]\n" + "ldr q16, [x10, #0x30]\n" + ".inst 0x4f40f22a // bfdot v10.4s, v17.8h, v0.h[0]\n" + "ldr q17, [x10, #0x40]\n" + ".inst 0x4f40f20b // bfdot v11.4s, v16.8h, v0.h[0]\n" + "ldr q16, [x10, #0x50]\n" + ".inst 0x4f60f228 // bfdot v8.4s, v17.8h, v0.h[1]\n" + "ldr q17, [x10, #0x60]\n" + ".inst 0x4f60f209 // bfdot v9.4s, v16.8h, v0.h[1]\n" + "ldr q16, [x10, #0x70]\n" + ".inst 0x4f60f22a // bfdot v10.4s, v17.8h, v0.h[1]\n" + "ldr q17, [x10, #0x80]\n" + ".inst 0x4f60f20b // bfdot v11.4s, v16.8h, v0.h[1]\n" + "ldr q16, [x10, #0x90]\n" + ".inst 0x4f40fa28 // bfdot v8.4s, v17.8h, v0.h[2]\n" + "ldr q17, [x10, #0xa0]\n" + ".inst 0x4f40fa09 // bfdot v9.4s, v16.8h, v0.h[2]\n" + "ldr q16, [x10, #0xb0]\n" + ".inst 0x4f40fa2a // bfdot v10.4s, v17.8h, v0.h[2]\n" + "ldr q17, [x10, #0xc0]\n" + ".inst 0x4f40fa0b // bfdot v11.4s, v16.8h, v0.h[2]\n" + "ldr q16, [x10, #0xd0]\n" + ".inst 0x4f60fa28 // bfdot v8.4s, v17.8h, v0.h[3]\n" + "ldr q17, [x10, #0xe0]\n" + ".inst 0x4f60fa09 // bfdot v9.4s, v16.8h, v0.h[3]\n" + "ldr q16, [x10, #0xf0]\n" "sub x27, x27, #0x8\n" "add x26, x26, #0x10\n" - ".inst 0x4f60f8ca // bfdot v10.4s, v6.8h, v0.h[3]\n" - ".inst 0x4f60f8eb // bfdot v11.4s, v7.8h, v0.h[3]\n" + ".inst 0x4f60fa2a // bfdot v10.4s, v17.8h, v0.h[3]\n" + ".inst 0x4f60fa0b // bfdot v11.4s, v16.8h, v0.h[3]\n" "ldr q0, [x26, #0x0]\n" "cmp x27, #0x10\n" "add x10, x10, #0x100\n" @@ -251,37 +250,37 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "bge 18b\n" "19:" // Height 1: Multiply loop: Single iteration only ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q17, [x10, #0x20]\n" ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x4f60f0c8 // bfdot v8.4s, v6.8h, v0.h[1]\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x4f60f0e9 // bfdot v9.4s, v7.8h, v0.h[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x4f60f0ca // bfdot v10.4s, v6.8h, v0.h[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x4f60f0eb // bfdot v11.4s, v7.8h, v0.h[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x4f40f8c8 // bfdot v8.4s, v6.8h, v0.h[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x4f40f8e9 // bfdot v9.4s, v7.8h, v0.h[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x4f40f8ca // bfdot v10.4s, v6.8h, v0.h[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x4f40f8eb // bfdot v11.4s, v7.8h, v0.h[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x4f60f8c8 // bfdot v8.4s, v6.8h, v0.h[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x4f60f8e9 // bfdot v9.4s, v7.8h, v0.h[3]\n" - "ldr q7, [x10, #0xf0]\n" + "ldr q16, [x10, #0x30]\n" + ".inst 0x4f40f22a // bfdot v10.4s, v17.8h, v0.h[0]\n" + "ldr q17, [x10, #0x40]\n" + ".inst 0x4f40f20b // bfdot v11.4s, v16.8h, v0.h[0]\n" + "ldr q16, [x10, #0x50]\n" + ".inst 0x4f60f228 // bfdot v8.4s, v17.8h, v0.h[1]\n" + "ldr q17, [x10, #0x60]\n" + ".inst 0x4f60f209 // bfdot v9.4s, v16.8h, v0.h[1]\n" + "ldr q16, [x10, #0x70]\n" + ".inst 0x4f60f22a // bfdot v10.4s, v17.8h, v0.h[1]\n" + "ldr q17, [x10, #0x80]\n" + ".inst 0x4f60f20b // bfdot v11.4s, v16.8h, v0.h[1]\n" + "ldr q16, [x10, #0x90]\n" + ".inst 0x4f40fa28 // bfdot v8.4s, v17.8h, v0.h[2]\n" + "ldr q17, [x10, #0xa0]\n" + ".inst 0x4f40fa09 // bfdot v9.4s, v16.8h, v0.h[2]\n" + "ldr q16, [x10, #0xb0]\n" + ".inst 0x4f40fa2a // bfdot v10.4s, v17.8h, v0.h[2]\n" + "ldr q17, [x10, #0xc0]\n" + ".inst 0x4f40fa0b // bfdot v11.4s, v16.8h, v0.h[2]\n" + "ldr q16, [x10, #0xd0]\n" + ".inst 0x4f60fa28 // bfdot v8.4s, v17.8h, v0.h[3]\n" + "ldr q17, [x10, #0xe0]\n" + ".inst 0x4f60fa09 // bfdot v9.4s, v16.8h, v0.h[3]\n" + "ldr q16, [x10, #0xf0]\n" "add x26, x26, #0x10\n" "sub x27, x27, #0x8\n" - ".inst 0x4f60f8ca // bfdot v10.4s, v6.8h, v0.h[3]\n" - ".inst 0x4f60f8eb // bfdot v11.4s, v7.8h, v0.h[3]\n" + ".inst 0x4f60fa2a // bfdot v10.4s, v17.8h, v0.h[3]\n" + ".inst 0x4f60fa0b // bfdot v11.4s, v16.8h, v0.h[3]\n" "prfm pldl1keep, [x26, #0x80]\n" "add x10, x10, #0x100\n" "20:" // Height 1: Multiply loop: Main loop skip @@ -289,31 +288,31 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "cmp x27, #0x2\n" "blt 22f\n" "21:" // Height 1: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr q6, [x10, #0x0]\n" - ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n" + "ldr s18, [x26], #0x4\n" + "ldr q16, [x10, #0x0]\n" + ".inst 0x4f52f208 // bfdot v8.4s, v16.8h, v18.h[0]\n" "sub x27, x27, #0x2\n" - "ldr q7, [x10, #0x10]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n" + "ldr q16, [x10, #0x10]\n" + "ldr q17, [x10, #0x20]\n" + ".inst 0x4f52f209 // bfdot v9.4s, v16.8h, v18.h[0]\n" "cmp x27, #0x2\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n" + "ldr q16, [x10, #0x30]\n" + ".inst 0x4f52f22a // bfdot v10.4s, v17.8h, v18.h[0]\n" + ".inst 0x4f52f20b // bfdot v11.4s, v16.8h, v18.h[0]\n" "add x10, x10, #0x40\n" "bge 21b\n" "22:" // Height 1: Multiply loop: Skip odd blocks "cbz x27, 24f\n" "ldr h0, [x26, #0x0]\n" "23:" // Height 1: Multiply loop: Ragged operand read: Done - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n" - "ldr q6, [x10, #0x20]\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n" + "ldr q17, [x10, #0x0]\n" + "ldr q16, [x10, #0x10]\n" + ".inst 0x4f40f228 // bfdot v8.4s, v17.8h, v0.h[0]\n" + ".inst 0x4f40f209 // bfdot v9.4s, v16.8h, v0.h[0]\n" + "ldr q17, [x10, #0x20]\n" + "ldr q16, [x10, #0x30]\n" + ".inst 0x4f40f22a // bfdot v10.4s, v17.8h, v0.h[0]\n" + ".inst 0x4f40f20b // bfdot v11.4s, v16.8h, v0.h[0]\n" "add x10, x10, #0x40\n" "24:" // Height 1: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -323,17 +322,17 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "prfm pstl1keep, [x9, #0x0]\n" "tbz %x[flags], #1, 25f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v17.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" + "ld1r { v16.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v17.4s\n" + "fmin v9.4s, v9.4s, v17.4s\n" + "fmin v10.4s, v10.4s, v17.4s\n" + "fmin v11.4s, v11.4s, v17.4s\n" + "fmax v8.4s, v8.4s, v16.4s\n" + "fmax v9.4s, v9.4s, v16.4s\n" + "fmax v10.4s, v10.4s, v16.4s\n" + "fmax v11.4s, v11.4s, v16.4s\n" "25:" // Height 1: No activation "cmp x11, #0x10\n" "bge 34f\n" @@ -511,12 +510,12 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "50:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 51f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 52f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -524,7 +523,7 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "b 52f\n" "51:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" "52:" // Height 2: input setup done "cmp x27, #0x8\n" "blt 55f\n" @@ -537,156 +536,156 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "53:" // Height 2: Multiply loop: Main loop head ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n" ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q17, [x10, #0x20]\n" "sub x27, x27, #0x8\n" ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n" ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q16, [x10, #0x30]\n" "add x26, x26, #0x10\n" - ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n" - "ldr q6, [x10, #0x40]\n" + ".inst 0x4f40f22a // bfdot v10.4s, v17.8h, v0.h[0]\n" + ".inst 0x4f41f22e // bfdot v14.4s, v17.8h, v1.h[0]\n" + "ldr q17, [x10, #0x40]\n" "add x25, x25, #0x10\n" - ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n" - "ldr q7, [x10, #0x50]\n" + ".inst 0x4f40f20b // bfdot v11.4s, v16.8h, v0.h[0]\n" + ".inst 0x4f41f20f // bfdot v15.4s, v16.8h, v1.h[0]\n" + "ldr q16, [x10, #0x50]\n" "cmp x27, #0x10\n" - ".inst 0x4f60f0c8 // bfdot v8.4s, v6.8h, v0.h[1]\n" - ".inst 0x4f61f0cc // bfdot v12.4s, v6.8h, v1.h[1]\n" - "ldr q6, [x10, #0x60]\n" + ".inst 0x4f60f228 // bfdot v8.4s, v17.8h, v0.h[1]\n" + ".inst 0x4f61f22c // bfdot v12.4s, v17.8h, v1.h[1]\n" + "ldr q17, [x10, #0x60]\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x4f60f0e9 // bfdot v9.4s, v7.8h, v0.h[1]\n" - ".inst 0x4f61f0ed // bfdot v13.4s, v7.8h, v1.h[1]\n" - "ldr q7, [x10, #0x70]\n" + ".inst 0x4f60f209 // bfdot v9.4s, v16.8h, v0.h[1]\n" + ".inst 0x4f61f20d // bfdot v13.4s, v16.8h, v1.h[1]\n" + "ldr q16, [x10, #0x70]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4f60f0ca // bfdot v10.4s, v6.8h, v0.h[1]\n" - ".inst 0x4f61f0ce // bfdot v14.4s, v6.8h, v1.h[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x4f60f0eb // bfdot v11.4s, v7.8h, v0.h[1]\n" - ".inst 0x4f61f0ef // bfdot v15.4s, v7.8h, v1.h[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x4f40f8c8 // bfdot v8.4s, v6.8h, v0.h[2]\n" - ".inst 0x4f41f8cc // bfdot v12.4s, v6.8h, v1.h[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x4f40f8e9 // bfdot v9.4s, v7.8h, v0.h[2]\n" - ".inst 0x4f41f8ed // bfdot v13.4s, v7.8h, v1.h[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x4f40f8ca // bfdot v10.4s, v6.8h, v0.h[2]\n" - ".inst 0x4f41f8ce // bfdot v14.4s, v6.8h, v1.h[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x4f40f8eb // bfdot v11.4s, v7.8h, v0.h[2]\n" - ".inst 0x4f41f8ef // bfdot v15.4s, v7.8h, v1.h[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x4f60f8c8 // bfdot v8.4s, v6.8h, v0.h[3]\n" - ".inst 0x4f61f8cc // bfdot v12.4s, v6.8h, v1.h[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x4f60f8e9 // bfdot v9.4s, v7.8h, v0.h[3]\n" - ".inst 0x4f61f8ed // bfdot v13.4s, v7.8h, v1.h[3]\n" - "ldr q7, [x10, #0xf0]\n" + ".inst 0x4f60f22a // bfdot v10.4s, v17.8h, v0.h[1]\n" + ".inst 0x4f61f22e // bfdot v14.4s, v17.8h, v1.h[1]\n" + "ldr q17, [x10, #0x80]\n" + ".inst 0x4f60f20b // bfdot v11.4s, v16.8h, v0.h[1]\n" + ".inst 0x4f61f20f // bfdot v15.4s, v16.8h, v1.h[1]\n" + "ldr q16, [x10, #0x90]\n" + ".inst 0x4f40fa28 // bfdot v8.4s, v17.8h, v0.h[2]\n" + ".inst 0x4f41fa2c // bfdot v12.4s, v17.8h, v1.h[2]\n" + "ldr q17, [x10, #0xa0]\n" + ".inst 0x4f40fa09 // bfdot v9.4s, v16.8h, v0.h[2]\n" + ".inst 0x4f41fa0d // bfdot v13.4s, v16.8h, v1.h[2]\n" + "ldr q16, [x10, #0xb0]\n" + ".inst 0x4f40fa2a // bfdot v10.4s, v17.8h, v0.h[2]\n" + ".inst 0x4f41fa2e // bfdot v14.4s, v17.8h, v1.h[2]\n" + "ldr q17, [x10, #0xc0]\n" + ".inst 0x4f40fa0b // bfdot v11.4s, v16.8h, v0.h[2]\n" + ".inst 0x4f41fa0f // bfdot v15.4s, v16.8h, v1.h[2]\n" + "ldr q16, [x10, #0xd0]\n" + ".inst 0x4f60fa28 // bfdot v8.4s, v17.8h, v0.h[3]\n" + ".inst 0x4f61fa2c // bfdot v12.4s, v17.8h, v1.h[3]\n" + "ldr q17, [x10, #0xe0]\n" + ".inst 0x4f60fa09 // bfdot v9.4s, v16.8h, v0.h[3]\n" + ".inst 0x4f61fa0d // bfdot v13.4s, v16.8h, v1.h[3]\n" + "ldr q16, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x4f60f8ca // bfdot v10.4s, v6.8h, v0.h[3]\n" - ".inst 0x4f61f8ce // bfdot v14.4s, v6.8h, v1.h[3]\n" + ".inst 0x4f60fa2a // bfdot v10.4s, v17.8h, v0.h[3]\n" + ".inst 0x4f61fa2e // bfdot v14.4s, v17.8h, v1.h[3]\n" "ldr q6, [x10, #0x0]\n" - ".inst 0x4f60f8eb // bfdot v11.4s, v7.8h, v0.h[3]\n" + ".inst 0x4f60fa0b // bfdot v11.4s, v16.8h, v0.h[3]\n" "ldr q0, [x26, #0x0]\n" - ".inst 0x4f61f8ef // bfdot v15.4s, v7.8h, v1.h[3]\n" + ".inst 0x4f61fa0f // bfdot v15.4s, v16.8h, v1.h[3]\n" "ldr q1, [x25, #0x0]\n" "ldr q7, [x10, #0x10]\n" "bge 53b\n" "54:" // Height 2: Multiply loop: Single iteration only ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n" ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q17, [x10, #0x20]\n" "add x26, x26, #0x10\n" ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n" ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q16, [x10, #0x30]\n" "add x25, x25, #0x10\n" - ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n" - "ldr q6, [x10, #0x40]\n" + ".inst 0x4f40f22a // bfdot v10.4s, v17.8h, v0.h[0]\n" + ".inst 0x4f41f22e // bfdot v14.4s, v17.8h, v1.h[0]\n" + "ldr q17, [x10, #0x40]\n" "sub x27, x27, #0x8\n" - ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n" - "ldr q7, [x10, #0x50]\n" + ".inst 0x4f40f20b // bfdot v11.4s, v16.8h, v0.h[0]\n" + ".inst 0x4f41f20f // bfdot v15.4s, v16.8h, v1.h[0]\n" + "ldr q16, [x10, #0x50]\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x4f60f0c8 // bfdot v8.4s, v6.8h, v0.h[1]\n" - ".inst 0x4f61f0cc // bfdot v12.4s, v6.8h, v1.h[1]\n" - "ldr q6, [x10, #0x60]\n" + ".inst 0x4f60f228 // bfdot v8.4s, v17.8h, v0.h[1]\n" + ".inst 0x4f61f22c // bfdot v12.4s, v17.8h, v1.h[1]\n" + "ldr q17, [x10, #0x60]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4f60f0e9 // bfdot v9.4s, v7.8h, v0.h[1]\n" - ".inst 0x4f61f0ed // bfdot v13.4s, v7.8h, v1.h[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x4f60f0ca // bfdot v10.4s, v6.8h, v0.h[1]\n" - ".inst 0x4f61f0ce // bfdot v14.4s, v6.8h, v1.h[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x4f60f0eb // bfdot v11.4s, v7.8h, v0.h[1]\n" - ".inst 0x4f61f0ef // bfdot v15.4s, v7.8h, v1.h[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x4f40f8c8 // bfdot v8.4s, v6.8h, v0.h[2]\n" - ".inst 0x4f41f8cc // bfdot v12.4s, v6.8h, v1.h[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x4f40f8e9 // bfdot v9.4s, v7.8h, v0.h[2]\n" - ".inst 0x4f41f8ed // bfdot v13.4s, v7.8h, v1.h[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x4f40f8ca // bfdot v10.4s, v6.8h, v0.h[2]\n" - ".inst 0x4f41f8ce // bfdot v14.4s, v6.8h, v1.h[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x4f40f8eb // bfdot v11.4s, v7.8h, v0.h[2]\n" - ".inst 0x4f41f8ef // bfdot v15.4s, v7.8h, v1.h[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x4f60f8c8 // bfdot v8.4s, v6.8h, v0.h[3]\n" - ".inst 0x4f61f8cc // bfdot v12.4s, v6.8h, v1.h[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x4f60f8e9 // bfdot v9.4s, v7.8h, v0.h[3]\n" - ".inst 0x4f61f8ed // bfdot v13.4s, v7.8h, v1.h[3]\n" - "ldr q7, [x10, #0xf0]\n" + ".inst 0x4f60f209 // bfdot v9.4s, v16.8h, v0.h[1]\n" + ".inst 0x4f61f20d // bfdot v13.4s, v16.8h, v1.h[1]\n" + "ldr q16, [x10, #0x70]\n" + ".inst 0x4f60f22a // bfdot v10.4s, v17.8h, v0.h[1]\n" + ".inst 0x4f61f22e // bfdot v14.4s, v17.8h, v1.h[1]\n" + "ldr q17, [x10, #0x80]\n" + ".inst 0x4f60f20b // bfdot v11.4s, v16.8h, v0.h[1]\n" + ".inst 0x4f61f20f // bfdot v15.4s, v16.8h, v1.h[1]\n" + "ldr q16, [x10, #0x90]\n" + ".inst 0x4f40fa28 // bfdot v8.4s, v17.8h, v0.h[2]\n" + ".inst 0x4f41fa2c // bfdot v12.4s, v17.8h, v1.h[2]\n" + "ldr q17, [x10, #0xa0]\n" + ".inst 0x4f40fa09 // bfdot v9.4s, v16.8h, v0.h[2]\n" + ".inst 0x4f41fa0d // bfdot v13.4s, v16.8h, v1.h[2]\n" + "ldr q16, [x10, #0xb0]\n" + ".inst 0x4f40fa2a // bfdot v10.4s, v17.8h, v0.h[2]\n" + ".inst 0x4f41fa2e // bfdot v14.4s, v17.8h, v1.h[2]\n" + "ldr q17, [x10, #0xc0]\n" + ".inst 0x4f40fa0b // bfdot v11.4s, v16.8h, v0.h[2]\n" + ".inst 0x4f41fa0f // bfdot v15.4s, v16.8h, v1.h[2]\n" + "ldr q16, [x10, #0xd0]\n" + ".inst 0x4f60fa28 // bfdot v8.4s, v17.8h, v0.h[3]\n" + ".inst 0x4f61fa2c // bfdot v12.4s, v17.8h, v1.h[3]\n" + "ldr q17, [x10, #0xe0]\n" + ".inst 0x4f60fa09 // bfdot v9.4s, v16.8h, v0.h[3]\n" + ".inst 0x4f61fa0d // bfdot v13.4s, v16.8h, v1.h[3]\n" + "ldr q16, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x4f60f8ca // bfdot v10.4s, v6.8h, v0.h[3]\n" - ".inst 0x4f61f8ce // bfdot v14.4s, v6.8h, v1.h[3]\n" - ".inst 0x4f60f8eb // bfdot v11.4s, v7.8h, v0.h[3]\n" - ".inst 0x4f61f8ef // bfdot v15.4s, v7.8h, v1.h[3]\n" + ".inst 0x4f60fa2a // bfdot v10.4s, v17.8h, v0.h[3]\n" + ".inst 0x4f61fa2e // bfdot v14.4s, v17.8h, v1.h[3]\n" + ".inst 0x4f60fa0b // bfdot v11.4s, v16.8h, v0.h[3]\n" + ".inst 0x4f61fa0f // bfdot v15.4s, v16.8h, v1.h[3]\n" "55:" // Height 2: Multiply loop: Main loop skip "cbz x27, 59f\n" "cmp x27, #0x2\n" "blt 57f\n" "56:" // Height 2: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s19, [x26], #0x4\n" + "ldr s18, [x25], #0x4\n" "sub x27, x27, #0x2\n" "cmp x27, #0x2\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n" + "ldr q17, [x10, #0x0]\n" + "ldr q16, [x10, #0x10]\n" + ".inst 0x4f53f228 // bfdot v8.4s, v17.8h, v19.h[0]\n" + ".inst 0x4f52f22c // bfdot v12.4s, v17.8h, v18.h[0]\n" + "ldr q17, [x10, #0x20]\n" + ".inst 0x4f53f209 // bfdot v9.4s, v16.8h, v19.h[0]\n" + ".inst 0x4f52f20d // bfdot v13.4s, v16.8h, v18.h[0]\n" + "ldr q16, [x10, #0x30]\n" + ".inst 0x4f53f22a // bfdot v10.4s, v17.8h, v19.h[0]\n" + ".inst 0x4f52f22e // bfdot v14.4s, v17.8h, v18.h[0]\n" "add x10, x10, #0x40\n" - ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n" + ".inst 0x4f53f20b // bfdot v11.4s, v16.8h, v19.h[0]\n" + ".inst 0x4f52f20f // bfdot v15.4s, v16.8h, v18.h[0]\n" "bge 56b\n" "57:" // Height 2: Multiply loop: Skip odd blocks "cbz x27, 59f\n" "ldr h0, [x26, #0x0]\n" "ldr h1, [x25, #0x0]\n" "58:" // Height 2: Multiply loop: Ragged operand read: Done - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n" + "ldr q17, [x10, #0x0]\n" + "ldr q16, [x10, #0x10]\n" + ".inst 0x4f40f228 // bfdot v8.4s, v17.8h, v0.h[0]\n" + ".inst 0x4f41f22c // bfdot v12.4s, v17.8h, v1.h[0]\n" + "ldr q17, [x10, #0x20]\n" + ".inst 0x4f40f209 // bfdot v9.4s, v16.8h, v0.h[0]\n" + ".inst 0x4f41f20d // bfdot v13.4s, v16.8h, v1.h[0]\n" + "ldr q16, [x10, #0x30]\n" + ".inst 0x4f40f22a // bfdot v10.4s, v17.8h, v0.h[0]\n" + ".inst 0x4f41f22e // bfdot v14.4s, v17.8h, v1.h[0]\n" "add x10, x10, #0x40\n" - ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n" + ".inst 0x4f40f20b // bfdot v11.4s, v16.8h, v0.h[0]\n" + ".inst 0x4f41f20f // bfdot v15.4s, v16.8h, v1.h[0]\n" "59:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -698,25 +697,25 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "prfm pstl1keep, [x25, #0x0]\n" "tbz %x[flags], #1, 60f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v17.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v15.4s, v15.4s, v1.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" + "ld1r { v16.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v17.4s\n" + "fmin v9.4s, v9.4s, v17.4s\n" + "fmin v10.4s, v10.4s, v17.4s\n" + "fmin v11.4s, v11.4s, v17.4s\n" + "fmin v12.4s, v12.4s, v17.4s\n" + "fmin v13.4s, v13.4s, v17.4s\n" + "fmin v14.4s, v14.4s, v17.4s\n" + "fmin v15.4s, v15.4s, v17.4s\n" + "fmax v8.4s, v8.4s, v16.4s\n" + "fmax v9.4s, v9.4s, v16.4s\n" + "fmax v10.4s, v10.4s, v16.4s\n" + "fmax v11.4s, v11.4s, v16.4s\n" + "fmax v12.4s, v12.4s, v16.4s\n" + "fmax v13.4s, v13.4s, v16.4s\n" + "fmax v14.4s, v14.4s, v16.4s\n" + "fmax v15.4s, v15.4s, v16.4s\n" "60:" // Height 2: No activation "cmp x11, #0x10\n" "bge 69f\n" @@ -943,13 +942,13 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "85:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 86f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 87f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -958,8 +957,8 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "b 87f\n" "86:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" "87:" // Height 3: input setup done "cmp x27, #0x8\n" "blt 90f\n" @@ -976,75 +975,75 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "sub x27, x27, #0x8\n" "add x26, x26, #0x10\n" ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q21, [x10, #0x20]\n" ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n" "add x25, x25, #0x10\n" ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n" ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q20, [x10, #0x30]\n" "add x24, x24, #0x10\n" - ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n" + ".inst 0x4f40f2aa // bfdot v10.4s, v21.8h, v0.h[0]\n" + ".inst 0x4f41f2ae // bfdot v14.4s, v21.8h, v1.h[0]\n" "cmp x27, #0x10\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n" + ".inst 0x4f42f2b2 // bfdot v18.4s, v21.8h, v2.h[0]\n" + "ldr q21, [x10, #0x40]\n" + ".inst 0x4f40f28b // bfdot v11.4s, v20.8h, v0.h[0]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n" - ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n" - "ldr q7, [x10, #0x50]\n" + ".inst 0x4f41f28f // bfdot v15.4s, v20.8h, v1.h[0]\n" + ".inst 0x4f42f293 // bfdot v19.4s, v20.8h, v2.h[0]\n" + "ldr q20, [x10, #0x50]\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x4f60f0c8 // bfdot v8.4s, v6.8h, v0.h[1]\n" - ".inst 0x4f61f0cc // bfdot v12.4s, v6.8h, v1.h[1]\n" - ".inst 0x4f62f0d0 // bfdot v16.4s, v6.8h, v2.h[1]\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x4f60f0e9 // bfdot v9.4s, v7.8h, v0.h[1]\n" - ".inst 0x4f61f0ed // bfdot v13.4s, v7.8h, v1.h[1]\n" - ".inst 0x4f62f0f1 // bfdot v17.4s, v7.8h, v2.h[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x4f60f0ca // bfdot v10.4s, v6.8h, v0.h[1]\n" - ".inst 0x4f61f0ce // bfdot v14.4s, v6.8h, v1.h[1]\n" - ".inst 0x4f62f0d2 // bfdot v18.4s, v6.8h, v2.h[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x4f60f0eb // bfdot v11.4s, v7.8h, v0.h[1]\n" - ".inst 0x4f61f0ef // bfdot v15.4s, v7.8h, v1.h[1]\n" - ".inst 0x4f62f0f3 // bfdot v19.4s, v7.8h, v2.h[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x4f40f8c8 // bfdot v8.4s, v6.8h, v0.h[2]\n" - ".inst 0x4f41f8cc // bfdot v12.4s, v6.8h, v1.h[2]\n" - ".inst 0x4f42f8d0 // bfdot v16.4s, v6.8h, v2.h[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x4f40f8e9 // bfdot v9.4s, v7.8h, v0.h[2]\n" - ".inst 0x4f41f8ed // bfdot v13.4s, v7.8h, v1.h[2]\n" - ".inst 0x4f42f8f1 // bfdot v17.4s, v7.8h, v2.h[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x4f40f8ca // bfdot v10.4s, v6.8h, v0.h[2]\n" - ".inst 0x4f41f8ce // bfdot v14.4s, v6.8h, v1.h[2]\n" - ".inst 0x4f42f8d2 // bfdot v18.4s, v6.8h, v2.h[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x4f40f8eb // bfdot v11.4s, v7.8h, v0.h[2]\n" - ".inst 0x4f41f8ef // bfdot v15.4s, v7.8h, v1.h[2]\n" - ".inst 0x4f42f8f3 // bfdot v19.4s, v7.8h, v2.h[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x4f60f8c8 // bfdot v8.4s, v6.8h, v0.h[3]\n" - ".inst 0x4f61f8cc // bfdot v12.4s, v6.8h, v1.h[3]\n" - ".inst 0x4f62f8d0 // bfdot v16.4s, v6.8h, v2.h[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x4f60f8e9 // bfdot v9.4s, v7.8h, v0.h[3]\n" - ".inst 0x4f61f8ed // bfdot v13.4s, v7.8h, v1.h[3]\n" - ".inst 0x4f62f8f1 // bfdot v17.4s, v7.8h, v2.h[3]\n" - "ldr q7, [x10, #0xf0]\n" + ".inst 0x4f60f2a8 // bfdot v8.4s, v21.8h, v0.h[1]\n" + ".inst 0x4f61f2ac // bfdot v12.4s, v21.8h, v1.h[1]\n" + ".inst 0x4f62f2b0 // bfdot v16.4s, v21.8h, v2.h[1]\n" + "ldr q21, [x10, #0x60]\n" + ".inst 0x4f60f289 // bfdot v9.4s, v20.8h, v0.h[1]\n" + ".inst 0x4f61f28d // bfdot v13.4s, v20.8h, v1.h[1]\n" + ".inst 0x4f62f291 // bfdot v17.4s, v20.8h, v2.h[1]\n" + "ldr q20, [x10, #0x70]\n" + ".inst 0x4f60f2aa // bfdot v10.4s, v21.8h, v0.h[1]\n" + ".inst 0x4f61f2ae // bfdot v14.4s, v21.8h, v1.h[1]\n" + ".inst 0x4f62f2b2 // bfdot v18.4s, v21.8h, v2.h[1]\n" + "ldr q21, [x10, #0x80]\n" + ".inst 0x4f60f28b // bfdot v11.4s, v20.8h, v0.h[1]\n" + ".inst 0x4f61f28f // bfdot v15.4s, v20.8h, v1.h[1]\n" + ".inst 0x4f62f293 // bfdot v19.4s, v20.8h, v2.h[1]\n" + "ldr q20, [x10, #0x90]\n" + ".inst 0x4f40faa8 // bfdot v8.4s, v21.8h, v0.h[2]\n" + ".inst 0x4f41faac // bfdot v12.4s, v21.8h, v1.h[2]\n" + ".inst 0x4f42fab0 // bfdot v16.4s, v21.8h, v2.h[2]\n" + "ldr q21, [x10, #0xa0]\n" + ".inst 0x4f40fa89 // bfdot v9.4s, v20.8h, v0.h[2]\n" + ".inst 0x4f41fa8d // bfdot v13.4s, v20.8h, v1.h[2]\n" + ".inst 0x4f42fa91 // bfdot v17.4s, v20.8h, v2.h[2]\n" + "ldr q20, [x10, #0xb0]\n" + ".inst 0x4f40faaa // bfdot v10.4s, v21.8h, v0.h[2]\n" + ".inst 0x4f41faae // bfdot v14.4s, v21.8h, v1.h[2]\n" + ".inst 0x4f42fab2 // bfdot v18.4s, v21.8h, v2.h[2]\n" + "ldr q21, [x10, #0xc0]\n" + ".inst 0x4f40fa8b // bfdot v11.4s, v20.8h, v0.h[2]\n" + ".inst 0x4f41fa8f // bfdot v15.4s, v20.8h, v1.h[2]\n" + ".inst 0x4f42fa93 // bfdot v19.4s, v20.8h, v2.h[2]\n" + "ldr q20, [x10, #0xd0]\n" + ".inst 0x4f60faa8 // bfdot v8.4s, v21.8h, v0.h[3]\n" + ".inst 0x4f61faac // bfdot v12.4s, v21.8h, v1.h[3]\n" + ".inst 0x4f62fab0 // bfdot v16.4s, v21.8h, v2.h[3]\n" + "ldr q21, [x10, #0xe0]\n" + ".inst 0x4f60fa89 // bfdot v9.4s, v20.8h, v0.h[3]\n" + ".inst 0x4f61fa8d // bfdot v13.4s, v20.8h, v1.h[3]\n" + ".inst 0x4f62fa91 // bfdot v17.4s, v20.8h, v2.h[3]\n" + "ldr q20, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x4f60f8ca // bfdot v10.4s, v6.8h, v0.h[3]\n" - ".inst 0x4f61f8ce // bfdot v14.4s, v6.8h, v1.h[3]\n" - ".inst 0x4f62f8d2 // bfdot v18.4s, v6.8h, v2.h[3]\n" + ".inst 0x4f60faaa // bfdot v10.4s, v21.8h, v0.h[3]\n" + ".inst 0x4f61faae // bfdot v14.4s, v21.8h, v1.h[3]\n" + ".inst 0x4f62fab2 // bfdot v18.4s, v21.8h, v2.h[3]\n" "ldr q6, [x10, #0x0]\n" - ".inst 0x4f60f8eb // bfdot v11.4s, v7.8h, v0.h[3]\n" + ".inst 0x4f60fa8b // bfdot v11.4s, v20.8h, v0.h[3]\n" "ldr q0, [x26, #0x0]\n" - ".inst 0x4f61f8ef // bfdot v15.4s, v7.8h, v1.h[3]\n" + ".inst 0x4f61fa8f // bfdot v15.4s, v20.8h, v1.h[3]\n" "ldr q1, [x25, #0x0]\n" - ".inst 0x4f62f8f3 // bfdot v19.4s, v7.8h, v2.h[3]\n" + ".inst 0x4f62fa93 // bfdot v19.4s, v20.8h, v2.h[3]\n" "ldr q2, [x24, #0x0]\n" "ldr q7, [x10, #0x10]\n" "bge 88b\n" @@ -1054,98 +1053,98 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q21, [x10, #0x20]\n" ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n" "add x24, x24, #0x10\n" ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n" ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q20, [x10, #0x30]\n" "sub x27, x27, #0x8\n" - ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n" + ".inst 0x4f40f2aa // bfdot v10.4s, v21.8h, v0.h[0]\n" + ".inst 0x4f41f2ae // bfdot v14.4s, v21.8h, v1.h[0]\n" "prfm pldl1keep, [x26, #0x80]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n" + ".inst 0x4f42f2b2 // bfdot v18.4s, v21.8h, v2.h[0]\n" + "ldr q21, [x10, #0x40]\n" + ".inst 0x4f40f28b // bfdot v11.4s, v20.8h, v0.h[0]\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n" - ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x4f60f0c8 // bfdot v8.4s, v6.8h, v0.h[1]\n" - ".inst 0x4f61f0cc // bfdot v12.4s, v6.8h, v1.h[1]\n" - ".inst 0x4f62f0d0 // bfdot v16.4s, v6.8h, v2.h[1]\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x4f60f0e9 // bfdot v9.4s, v7.8h, v0.h[1]\n" - ".inst 0x4f61f0ed // bfdot v13.4s, v7.8h, v1.h[1]\n" - ".inst 0x4f62f0f1 // bfdot v17.4s, v7.8h, v2.h[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x4f60f0ca // bfdot v10.4s, v6.8h, v0.h[1]\n" - ".inst 0x4f61f0ce // bfdot v14.4s, v6.8h, v1.h[1]\n" - ".inst 0x4f62f0d2 // bfdot v18.4s, v6.8h, v2.h[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x4f60f0eb // bfdot v11.4s, v7.8h, v0.h[1]\n" - ".inst 0x4f61f0ef // bfdot v15.4s, v7.8h, v1.h[1]\n" - ".inst 0x4f62f0f3 // bfdot v19.4s, v7.8h, v2.h[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x4f40f8c8 // bfdot v8.4s, v6.8h, v0.h[2]\n" - ".inst 0x4f41f8cc // bfdot v12.4s, v6.8h, v1.h[2]\n" - ".inst 0x4f42f8d0 // bfdot v16.4s, v6.8h, v2.h[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x4f40f8e9 // bfdot v9.4s, v7.8h, v0.h[2]\n" - ".inst 0x4f41f8ed // bfdot v13.4s, v7.8h, v1.h[2]\n" - ".inst 0x4f42f8f1 // bfdot v17.4s, v7.8h, v2.h[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x4f40f8ca // bfdot v10.4s, v6.8h, v0.h[2]\n" - ".inst 0x4f41f8ce // bfdot v14.4s, v6.8h, v1.h[2]\n" - ".inst 0x4f42f8d2 // bfdot v18.4s, v6.8h, v2.h[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x4f40f8eb // bfdot v11.4s, v7.8h, v0.h[2]\n" - ".inst 0x4f41f8ef // bfdot v15.4s, v7.8h, v1.h[2]\n" - ".inst 0x4f42f8f3 // bfdot v19.4s, v7.8h, v2.h[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x4f60f8c8 // bfdot v8.4s, v6.8h, v0.h[3]\n" - ".inst 0x4f61f8cc // bfdot v12.4s, v6.8h, v1.h[3]\n" - ".inst 0x4f62f8d0 // bfdot v16.4s, v6.8h, v2.h[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x4f60f8e9 // bfdot v9.4s, v7.8h, v0.h[3]\n" - ".inst 0x4f61f8ed // bfdot v13.4s, v7.8h, v1.h[3]\n" - ".inst 0x4f62f8f1 // bfdot v17.4s, v7.8h, v2.h[3]\n" - "ldr q7, [x10, #0xf0]\n" + ".inst 0x4f41f28f // bfdot v15.4s, v20.8h, v1.h[0]\n" + ".inst 0x4f42f293 // bfdot v19.4s, v20.8h, v2.h[0]\n" + "ldr q20, [x10, #0x50]\n" + ".inst 0x4f60f2a8 // bfdot v8.4s, v21.8h, v0.h[1]\n" + ".inst 0x4f61f2ac // bfdot v12.4s, v21.8h, v1.h[1]\n" + ".inst 0x4f62f2b0 // bfdot v16.4s, v21.8h, v2.h[1]\n" + "ldr q21, [x10, #0x60]\n" + ".inst 0x4f60f289 // bfdot v9.4s, v20.8h, v0.h[1]\n" + ".inst 0x4f61f28d // bfdot v13.4s, v20.8h, v1.h[1]\n" + ".inst 0x4f62f291 // bfdot v17.4s, v20.8h, v2.h[1]\n" + "ldr q20, [x10, #0x70]\n" + ".inst 0x4f60f2aa // bfdot v10.4s, v21.8h, v0.h[1]\n" + ".inst 0x4f61f2ae // bfdot v14.4s, v21.8h, v1.h[1]\n" + ".inst 0x4f62f2b2 // bfdot v18.4s, v21.8h, v2.h[1]\n" + "ldr q21, [x10, #0x80]\n" + ".inst 0x4f60f28b // bfdot v11.4s, v20.8h, v0.h[1]\n" + ".inst 0x4f61f28f // bfdot v15.4s, v20.8h, v1.h[1]\n" + ".inst 0x4f62f293 // bfdot v19.4s, v20.8h, v2.h[1]\n" + "ldr q20, [x10, #0x90]\n" + ".inst 0x4f40faa8 // bfdot v8.4s, v21.8h, v0.h[2]\n" + ".inst 0x4f41faac // bfdot v12.4s, v21.8h, v1.h[2]\n" + ".inst 0x4f42fab0 // bfdot v16.4s, v21.8h, v2.h[2]\n" + "ldr q21, [x10, #0xa0]\n" + ".inst 0x4f40fa89 // bfdot v9.4s, v20.8h, v0.h[2]\n" + ".inst 0x4f41fa8d // bfdot v13.4s, v20.8h, v1.h[2]\n" + ".inst 0x4f42fa91 // bfdot v17.4s, v20.8h, v2.h[2]\n" + "ldr q20, [x10, #0xb0]\n" + ".inst 0x4f40faaa // bfdot v10.4s, v21.8h, v0.h[2]\n" + ".inst 0x4f41faae // bfdot v14.4s, v21.8h, v1.h[2]\n" + ".inst 0x4f42fab2 // bfdot v18.4s, v21.8h, v2.h[2]\n" + "ldr q21, [x10, #0xc0]\n" + ".inst 0x4f40fa8b // bfdot v11.4s, v20.8h, v0.h[2]\n" + ".inst 0x4f41fa8f // bfdot v15.4s, v20.8h, v1.h[2]\n" + ".inst 0x4f42fa93 // bfdot v19.4s, v20.8h, v2.h[2]\n" + "ldr q20, [x10, #0xd0]\n" + ".inst 0x4f60faa8 // bfdot v8.4s, v21.8h, v0.h[3]\n" + ".inst 0x4f61faac // bfdot v12.4s, v21.8h, v1.h[3]\n" + ".inst 0x4f62fab0 // bfdot v16.4s, v21.8h, v2.h[3]\n" + "ldr q21, [x10, #0xe0]\n" + ".inst 0x4f60fa89 // bfdot v9.4s, v20.8h, v0.h[3]\n" + ".inst 0x4f61fa8d // bfdot v13.4s, v20.8h, v1.h[3]\n" + ".inst 0x4f62fa91 // bfdot v17.4s, v20.8h, v2.h[3]\n" + "ldr q20, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x4f60f8ca // bfdot v10.4s, v6.8h, v0.h[3]\n" - ".inst 0x4f61f8ce // bfdot v14.4s, v6.8h, v1.h[3]\n" - ".inst 0x4f62f8d2 // bfdot v18.4s, v6.8h, v2.h[3]\n" - ".inst 0x4f60f8eb // bfdot v11.4s, v7.8h, v0.h[3]\n" - ".inst 0x4f61f8ef // bfdot v15.4s, v7.8h, v1.h[3]\n" - ".inst 0x4f62f8f3 // bfdot v19.4s, v7.8h, v2.h[3]\n" + ".inst 0x4f60faaa // bfdot v10.4s, v21.8h, v0.h[3]\n" + ".inst 0x4f61faae // bfdot v14.4s, v21.8h, v1.h[3]\n" + ".inst 0x4f62fab2 // bfdot v18.4s, v21.8h, v2.h[3]\n" + ".inst 0x4f60fa8b // bfdot v11.4s, v20.8h, v0.h[3]\n" + ".inst 0x4f61fa8f // bfdot v15.4s, v20.8h, v1.h[3]\n" + ".inst 0x4f62fa93 // bfdot v19.4s, v20.8h, v2.h[3]\n" "90:" // Height 3: Multiply loop: Main loop skip "cbz x27, 94f\n" "cmp x27, #0x2\n" "blt 92f\n" "91:" // Height 3: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s24, [x26], #0x4\n" + "ldr s23, [x25], #0x4\n" "sub x27, x27, #0x2\n" "cmp x27, #0x2\n" - "ldr s2, [x24], #0x4\n" - "ldr q6, [x10, #0x0]\n" - ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n" - ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr s22, [x24], #0x4\n" + "ldr q21, [x10, #0x0]\n" + ".inst 0x4f58f2a8 // bfdot v8.4s, v21.8h, v24.h[0]\n" + ".inst 0x4f57f2ac // bfdot v12.4s, v21.8h, v23.h[0]\n" + "ldr q20, [x10, #0x10]\n" + ".inst 0x4f56f2b0 // bfdot v16.4s, v21.8h, v22.h[0]\n" + "ldr q21, [x10, #0x20]\n" + ".inst 0x4f58f289 // bfdot v9.4s, v20.8h, v24.h[0]\n" + ".inst 0x4f57f28d // bfdot v13.4s, v20.8h, v23.h[0]\n" + ".inst 0x4f56f291 // bfdot v17.4s, v20.8h, v22.h[0]\n" + "ldr q20, [x10, #0x30]\n" "add x10, x10, #0x40\n" - ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n" - ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n" - ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n" - ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n" + ".inst 0x4f58f2aa // bfdot v10.4s, v21.8h, v24.h[0]\n" + ".inst 0x4f57f2ae // bfdot v14.4s, v21.8h, v23.h[0]\n" + ".inst 0x4f56f2b2 // bfdot v18.4s, v21.8h, v22.h[0]\n" + ".inst 0x4f58f28b // bfdot v11.4s, v20.8h, v24.h[0]\n" + ".inst 0x4f57f28f // bfdot v15.4s, v20.8h, v23.h[0]\n" + ".inst 0x4f56f293 // bfdot v19.4s, v20.8h, v22.h[0]\n" "bge 91b\n" "92:" // Height 3: Multiply loop: Skip odd blocks "cbz x27, 94f\n" @@ -1153,23 +1152,23 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "ldr h1, [x25, #0x0]\n" "ldr h2, [x24, #0x0]\n" "93:" // Height 3: Multiply loop: Ragged operand read: Done - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n" - ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n" - ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q21, [x10, #0x0]\n" + "ldr q20, [x10, #0x10]\n" + ".inst 0x4f40f2a8 // bfdot v8.4s, v21.8h, v0.h[0]\n" + ".inst 0x4f41f2ac // bfdot v12.4s, v21.8h, v1.h[0]\n" + ".inst 0x4f42f2b0 // bfdot v16.4s, v21.8h, v2.h[0]\n" + "ldr q21, [x10, #0x20]\n" + ".inst 0x4f40f289 // bfdot v9.4s, v20.8h, v0.h[0]\n" + ".inst 0x4f41f28d // bfdot v13.4s, v20.8h, v1.h[0]\n" + ".inst 0x4f42f291 // bfdot v17.4s, v20.8h, v2.h[0]\n" + "ldr q20, [x10, #0x30]\n" "add x10, x10, #0x40\n" - ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n" - ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n" - ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n" - ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n" + ".inst 0x4f40f2aa // bfdot v10.4s, v21.8h, v0.h[0]\n" + ".inst 0x4f41f2ae // bfdot v14.4s, v21.8h, v1.h[0]\n" + ".inst 0x4f42f2b2 // bfdot v18.4s, v21.8h, v2.h[0]\n" + ".inst 0x4f40f28b // bfdot v11.4s, v20.8h, v0.h[0]\n" + ".inst 0x4f41f28f // bfdot v15.4s, v20.8h, v1.h[0]\n" + ".inst 0x4f42f293 // bfdot v19.4s, v20.8h, v2.h[0]\n" "94:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1183,33 +1182,33 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "prfm pstl1keep, [x24, #0x0]\n" "tbz %x[flags], #1, 95f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v21.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v15.4s, v15.4s, v1.4s\n" - "fmin v16.4s, v16.4s, v1.4s\n" - "fmin v17.4s, v17.4s, v1.4s\n" - "fmin v18.4s, v18.4s, v1.4s\n" - "fmin v19.4s, v19.4s, v1.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" - "fmax v16.4s, v16.4s, v0.4s\n" - "fmax v17.4s, v17.4s, v0.4s\n" - "fmax v18.4s, v18.4s, v0.4s\n" - "fmax v19.4s, v19.4s, v0.4s\n" + "ld1r { v20.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v21.4s\n" + "fmin v9.4s, v9.4s, v21.4s\n" + "fmin v10.4s, v10.4s, v21.4s\n" + "fmin v11.4s, v11.4s, v21.4s\n" + "fmin v12.4s, v12.4s, v21.4s\n" + "fmin v13.4s, v13.4s, v21.4s\n" + "fmin v14.4s, v14.4s, v21.4s\n" + "fmin v15.4s, v15.4s, v21.4s\n" + "fmin v16.4s, v16.4s, v21.4s\n" + "fmin v17.4s, v17.4s, v21.4s\n" + "fmin v18.4s, v18.4s, v21.4s\n" + "fmin v19.4s, v19.4s, v21.4s\n" + "fmax v8.4s, v8.4s, v20.4s\n" + "fmax v9.4s, v9.4s, v20.4s\n" + "fmax v10.4s, v10.4s, v20.4s\n" + "fmax v11.4s, v11.4s, v20.4s\n" + "fmax v12.4s, v12.4s, v20.4s\n" + "fmax v13.4s, v13.4s, v20.4s\n" + "fmax v14.4s, v14.4s, v20.4s\n" + "fmax v15.4s, v15.4s, v20.4s\n" + "fmax v16.4s, v16.4s, v20.4s\n" + "fmax v17.4s, v17.4s, v20.4s\n" + "fmax v18.4s, v18.4s, v20.4s\n" + "fmax v19.4s, v19.4s, v20.4s\n" "95:" // Height 3: No activation "cmp x11, #0x10\n" "bge 104f\n" @@ -1485,14 +1484,14 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "120:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 121f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 122f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -1502,9 +1501,9 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "b 122f\n" "121:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" "122:" // Height 4: input setup done "cmp x27, #0x8\n" "blt 125f\n" @@ -1523,7 +1522,7 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "add x26, x26, #0x10\n" ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n" ".inst 0x4f43f0d4 // bfdot v20.4s, v6.8h, v3.h[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q25, [x10, #0x20]\n" "add x25, x25, #0x10\n" ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n" ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n" @@ -1531,85 +1530,85 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "add x23, x23, #0x10\n" ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n" ".inst 0x4f43f0f5 // bfdot v21.4s, v7.8h, v3.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q24, [x10, #0x30]\n" "cmp x27, #0x10\n" - ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n" + ".inst 0x4f40f32a // bfdot v10.4s, v25.8h, v0.h[0]\n" + ".inst 0x4f41f32e // bfdot v14.4s, v25.8h, v1.h[0]\n" "prfm pldl1keep, [x26, #0x80]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n" - ".inst 0x4f43f0d6 // bfdot v22.4s, v6.8h, v3.h[0]\n" - "ldr q6, [x10, #0x40]\n" + ".inst 0x4f42f332 // bfdot v18.4s, v25.8h, v2.h[0]\n" + ".inst 0x4f43f336 // bfdot v22.4s, v25.8h, v3.h[0]\n" + "ldr q25, [x10, #0x40]\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n" + ".inst 0x4f40f30b // bfdot v11.4s, v24.8h, v0.h[0]\n" + ".inst 0x4f41f30f // bfdot v15.4s, v24.8h, v1.h[0]\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n" - ".inst 0x4f43f0f7 // bfdot v23.4s, v7.8h, v3.h[0]\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x4f60f0c8 // bfdot v8.4s, v6.8h, v0.h[1]\n" - ".inst 0x4f61f0cc // bfdot v12.4s, v6.8h, v1.h[1]\n" - ".inst 0x4f62f0d0 // bfdot v16.4s, v6.8h, v2.h[1]\n" - ".inst 0x4f63f0d4 // bfdot v20.4s, v6.8h, v3.h[1]\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x4f60f0e9 // bfdot v9.4s, v7.8h, v0.h[1]\n" - ".inst 0x4f61f0ed // bfdot v13.4s, v7.8h, v1.h[1]\n" - ".inst 0x4f62f0f1 // bfdot v17.4s, v7.8h, v2.h[1]\n" - ".inst 0x4f63f0f5 // bfdot v21.4s, v7.8h, v3.h[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x4f60f0ca // bfdot v10.4s, v6.8h, v0.h[1]\n" - ".inst 0x4f61f0ce // bfdot v14.4s, v6.8h, v1.h[1]\n" - ".inst 0x4f62f0d2 // bfdot v18.4s, v6.8h, v2.h[1]\n" - ".inst 0x4f63f0d6 // bfdot v22.4s, v6.8h, v3.h[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x4f60f0eb // bfdot v11.4s, v7.8h, v0.h[1]\n" - ".inst 0x4f61f0ef // bfdot v15.4s, v7.8h, v1.h[1]\n" - ".inst 0x4f62f0f3 // bfdot v19.4s, v7.8h, v2.h[1]\n" - ".inst 0x4f63f0f7 // bfdot v23.4s, v7.8h, v3.h[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x4f40f8c8 // bfdot v8.4s, v6.8h, v0.h[2]\n" - ".inst 0x4f41f8cc // bfdot v12.4s, v6.8h, v1.h[2]\n" - ".inst 0x4f42f8d0 // bfdot v16.4s, v6.8h, v2.h[2]\n" - ".inst 0x4f43f8d4 // bfdot v20.4s, v6.8h, v3.h[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x4f40f8e9 // bfdot v9.4s, v7.8h, v0.h[2]\n" - ".inst 0x4f41f8ed // bfdot v13.4s, v7.8h, v1.h[2]\n" - ".inst 0x4f42f8f1 // bfdot v17.4s, v7.8h, v2.h[2]\n" - ".inst 0x4f43f8f5 // bfdot v21.4s, v7.8h, v3.h[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x4f40f8ca // bfdot v10.4s, v6.8h, v0.h[2]\n" - ".inst 0x4f41f8ce // bfdot v14.4s, v6.8h, v1.h[2]\n" - ".inst 0x4f42f8d2 // bfdot v18.4s, v6.8h, v2.h[2]\n" - ".inst 0x4f43f8d6 // bfdot v22.4s, v6.8h, v3.h[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x4f40f8eb // bfdot v11.4s, v7.8h, v0.h[2]\n" - ".inst 0x4f41f8ef // bfdot v15.4s, v7.8h, v1.h[2]\n" - ".inst 0x4f42f8f3 // bfdot v19.4s, v7.8h, v2.h[2]\n" - ".inst 0x4f43f8f7 // bfdot v23.4s, v7.8h, v3.h[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x4f60f8c8 // bfdot v8.4s, v6.8h, v0.h[3]\n" - ".inst 0x4f61f8cc // bfdot v12.4s, v6.8h, v1.h[3]\n" - ".inst 0x4f62f8d0 // bfdot v16.4s, v6.8h, v2.h[3]\n" - ".inst 0x4f63f8d4 // bfdot v20.4s, v6.8h, v3.h[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x4f60f8e9 // bfdot v9.4s, v7.8h, v0.h[3]\n" - ".inst 0x4f61f8ed // bfdot v13.4s, v7.8h, v1.h[3]\n" - ".inst 0x4f62f8f1 // bfdot v17.4s, v7.8h, v2.h[3]\n" - ".inst 0x4f63f8f5 // bfdot v21.4s, v7.8h, v3.h[3]\n" - "ldr q7, [x10, #0xf0]\n" + ".inst 0x4f42f313 // bfdot v19.4s, v24.8h, v2.h[0]\n" + ".inst 0x4f43f317 // bfdot v23.4s, v24.8h, v3.h[0]\n" + "ldr q24, [x10, #0x50]\n" + ".inst 0x4f60f328 // bfdot v8.4s, v25.8h, v0.h[1]\n" + ".inst 0x4f61f32c // bfdot v12.4s, v25.8h, v1.h[1]\n" + ".inst 0x4f62f330 // bfdot v16.4s, v25.8h, v2.h[1]\n" + ".inst 0x4f63f334 // bfdot v20.4s, v25.8h, v3.h[1]\n" + "ldr q25, [x10, #0x60]\n" + ".inst 0x4f60f309 // bfdot v9.4s, v24.8h, v0.h[1]\n" + ".inst 0x4f61f30d // bfdot v13.4s, v24.8h, v1.h[1]\n" + ".inst 0x4f62f311 // bfdot v17.4s, v24.8h, v2.h[1]\n" + ".inst 0x4f63f315 // bfdot v21.4s, v24.8h, v3.h[1]\n" + "ldr q24, [x10, #0x70]\n" + ".inst 0x4f60f32a // bfdot v10.4s, v25.8h, v0.h[1]\n" + ".inst 0x4f61f32e // bfdot v14.4s, v25.8h, v1.h[1]\n" + ".inst 0x4f62f332 // bfdot v18.4s, v25.8h, v2.h[1]\n" + ".inst 0x4f63f336 // bfdot v22.4s, v25.8h, v3.h[1]\n" + "ldr q25, [x10, #0x80]\n" + ".inst 0x4f60f30b // bfdot v11.4s, v24.8h, v0.h[1]\n" + ".inst 0x4f61f30f // bfdot v15.4s, v24.8h, v1.h[1]\n" + ".inst 0x4f62f313 // bfdot v19.4s, v24.8h, v2.h[1]\n" + ".inst 0x4f63f317 // bfdot v23.4s, v24.8h, v3.h[1]\n" + "ldr q24, [x10, #0x90]\n" + ".inst 0x4f40fb28 // bfdot v8.4s, v25.8h, v0.h[2]\n" + ".inst 0x4f41fb2c // bfdot v12.4s, v25.8h, v1.h[2]\n" + ".inst 0x4f42fb30 // bfdot v16.4s, v25.8h, v2.h[2]\n" + ".inst 0x4f43fb34 // bfdot v20.4s, v25.8h, v3.h[2]\n" + "ldr q25, [x10, #0xa0]\n" + ".inst 0x4f40fb09 // bfdot v9.4s, v24.8h, v0.h[2]\n" + ".inst 0x4f41fb0d // bfdot v13.4s, v24.8h, v1.h[2]\n" + ".inst 0x4f42fb11 // bfdot v17.4s, v24.8h, v2.h[2]\n" + ".inst 0x4f43fb15 // bfdot v21.4s, v24.8h, v3.h[2]\n" + "ldr q24, [x10, #0xb0]\n" + ".inst 0x4f40fb2a // bfdot v10.4s, v25.8h, v0.h[2]\n" + ".inst 0x4f41fb2e // bfdot v14.4s, v25.8h, v1.h[2]\n" + ".inst 0x4f42fb32 // bfdot v18.4s, v25.8h, v2.h[2]\n" + ".inst 0x4f43fb36 // bfdot v22.4s, v25.8h, v3.h[2]\n" + "ldr q25, [x10, #0xc0]\n" + ".inst 0x4f40fb0b // bfdot v11.4s, v24.8h, v0.h[2]\n" + ".inst 0x4f41fb0f // bfdot v15.4s, v24.8h, v1.h[2]\n" + ".inst 0x4f42fb13 // bfdot v19.4s, v24.8h, v2.h[2]\n" + ".inst 0x4f43fb17 // bfdot v23.4s, v24.8h, v3.h[2]\n" + "ldr q24, [x10, #0xd0]\n" + ".inst 0x4f60fb28 // bfdot v8.4s, v25.8h, v0.h[3]\n" + ".inst 0x4f61fb2c // bfdot v12.4s, v25.8h, v1.h[3]\n" + ".inst 0x4f62fb30 // bfdot v16.4s, v25.8h, v2.h[3]\n" + ".inst 0x4f63fb34 // bfdot v20.4s, v25.8h, v3.h[3]\n" + "ldr q25, [x10, #0xe0]\n" + ".inst 0x4f60fb09 // bfdot v9.4s, v24.8h, v0.h[3]\n" + ".inst 0x4f61fb0d // bfdot v13.4s, v24.8h, v1.h[3]\n" + ".inst 0x4f62fb11 // bfdot v17.4s, v24.8h, v2.h[3]\n" + ".inst 0x4f63fb15 // bfdot v21.4s, v24.8h, v3.h[3]\n" + "ldr q24, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x4f60f8ca // bfdot v10.4s, v6.8h, v0.h[3]\n" - ".inst 0x4f61f8ce // bfdot v14.4s, v6.8h, v1.h[3]\n" - ".inst 0x4f62f8d2 // bfdot v18.4s, v6.8h, v2.h[3]\n" - ".inst 0x4f63f8d6 // bfdot v22.4s, v6.8h, v3.h[3]\n" + ".inst 0x4f60fb2a // bfdot v10.4s, v25.8h, v0.h[3]\n" + ".inst 0x4f61fb2e // bfdot v14.4s, v25.8h, v1.h[3]\n" + ".inst 0x4f62fb32 // bfdot v18.4s, v25.8h, v2.h[3]\n" + ".inst 0x4f63fb36 // bfdot v22.4s, v25.8h, v3.h[3]\n" "ldr q6, [x10, #0x0]\n" - ".inst 0x4f60f8eb // bfdot v11.4s, v7.8h, v0.h[3]\n" + ".inst 0x4f60fb0b // bfdot v11.4s, v24.8h, v0.h[3]\n" "ldr q0, [x26, #0x0]\n" - ".inst 0x4f61f8ef // bfdot v15.4s, v7.8h, v1.h[3]\n" + ".inst 0x4f61fb0f // bfdot v15.4s, v24.8h, v1.h[3]\n" "ldr q1, [x25, #0x0]\n" - ".inst 0x4f62f8f3 // bfdot v19.4s, v7.8h, v2.h[3]\n" + ".inst 0x4f62fb13 // bfdot v19.4s, v24.8h, v2.h[3]\n" "ldr q2, [x24, #0x0]\n" - ".inst 0x4f63f8f7 // bfdot v23.4s, v7.8h, v3.h[3]\n" + ".inst 0x4f63fb17 // bfdot v23.4s, v24.8h, v3.h[3]\n" "ldr q3, [x23, #0x0]\n" "ldr q7, [x10, #0x10]\n" "bge 123b\n" @@ -1620,7 +1619,7 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "add x25, x25, #0x10\n" ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n" ".inst 0x4f43f0d4 // bfdot v20.4s, v6.8h, v3.h[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q25, [x10, #0x20]\n" "add x24, x24, #0x10\n" ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n" ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n" @@ -1628,112 +1627,112 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "sub x27, x27, #0x8\n" ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n" ".inst 0x4f43f0f5 // bfdot v21.4s, v7.8h, v3.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q24, [x10, #0x30]\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n" + ".inst 0x4f40f32a // bfdot v10.4s, v25.8h, v0.h[0]\n" + ".inst 0x4f41f32e // bfdot v14.4s, v25.8h, v1.h[0]\n" "prfm pldl1keep, [x25, #0x80]\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n" - ".inst 0x4f43f0d6 // bfdot v22.4s, v6.8h, v3.h[0]\n" - "ldr q6, [x10, #0x40]\n" + ".inst 0x4f42f332 // bfdot v18.4s, v25.8h, v2.h[0]\n" + ".inst 0x4f43f336 // bfdot v22.4s, v25.8h, v3.h[0]\n" + "ldr q25, [x10, #0x40]\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n" - ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n" - ".inst 0x4f43f0f7 // bfdot v23.4s, v7.8h, v3.h[0]\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x4f60f0c8 // bfdot v8.4s, v6.8h, v0.h[1]\n" - ".inst 0x4f61f0cc // bfdot v12.4s, v6.8h, v1.h[1]\n" - ".inst 0x4f62f0d0 // bfdot v16.4s, v6.8h, v2.h[1]\n" - ".inst 0x4f63f0d4 // bfdot v20.4s, v6.8h, v3.h[1]\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x4f60f0e9 // bfdot v9.4s, v7.8h, v0.h[1]\n" - ".inst 0x4f61f0ed // bfdot v13.4s, v7.8h, v1.h[1]\n" - ".inst 0x4f62f0f1 // bfdot v17.4s, v7.8h, v2.h[1]\n" - ".inst 0x4f63f0f5 // bfdot v21.4s, v7.8h, v3.h[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x4f60f0ca // bfdot v10.4s, v6.8h, v0.h[1]\n" - ".inst 0x4f61f0ce // bfdot v14.4s, v6.8h, v1.h[1]\n" - ".inst 0x4f62f0d2 // bfdot v18.4s, v6.8h, v2.h[1]\n" - ".inst 0x4f63f0d6 // bfdot v22.4s, v6.8h, v3.h[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x4f60f0eb // bfdot v11.4s, v7.8h, v0.h[1]\n" - ".inst 0x4f61f0ef // bfdot v15.4s, v7.8h, v1.h[1]\n" - ".inst 0x4f62f0f3 // bfdot v19.4s, v7.8h, v2.h[1]\n" - ".inst 0x4f63f0f7 // bfdot v23.4s, v7.8h, v3.h[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x4f40f8c8 // bfdot v8.4s, v6.8h, v0.h[2]\n" - ".inst 0x4f41f8cc // bfdot v12.4s, v6.8h, v1.h[2]\n" - ".inst 0x4f42f8d0 // bfdot v16.4s, v6.8h, v2.h[2]\n" - ".inst 0x4f43f8d4 // bfdot v20.4s, v6.8h, v3.h[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x4f40f8e9 // bfdot v9.4s, v7.8h, v0.h[2]\n" - ".inst 0x4f41f8ed // bfdot v13.4s, v7.8h, v1.h[2]\n" - ".inst 0x4f42f8f1 // bfdot v17.4s, v7.8h, v2.h[2]\n" - ".inst 0x4f43f8f5 // bfdot v21.4s, v7.8h, v3.h[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x4f40f8ca // bfdot v10.4s, v6.8h, v0.h[2]\n" - ".inst 0x4f41f8ce // bfdot v14.4s, v6.8h, v1.h[2]\n" - ".inst 0x4f42f8d2 // bfdot v18.4s, v6.8h, v2.h[2]\n" - ".inst 0x4f43f8d6 // bfdot v22.4s, v6.8h, v3.h[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x4f40f8eb // bfdot v11.4s, v7.8h, v0.h[2]\n" - ".inst 0x4f41f8ef // bfdot v15.4s, v7.8h, v1.h[2]\n" - ".inst 0x4f42f8f3 // bfdot v19.4s, v7.8h, v2.h[2]\n" - ".inst 0x4f43f8f7 // bfdot v23.4s, v7.8h, v3.h[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x4f60f8c8 // bfdot v8.4s, v6.8h, v0.h[3]\n" - ".inst 0x4f61f8cc // bfdot v12.4s, v6.8h, v1.h[3]\n" - ".inst 0x4f62f8d0 // bfdot v16.4s, v6.8h, v2.h[3]\n" - ".inst 0x4f63f8d4 // bfdot v20.4s, v6.8h, v3.h[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x4f60f8e9 // bfdot v9.4s, v7.8h, v0.h[3]\n" - ".inst 0x4f61f8ed // bfdot v13.4s, v7.8h, v1.h[3]\n" - ".inst 0x4f62f8f1 // bfdot v17.4s, v7.8h, v2.h[3]\n" - ".inst 0x4f63f8f5 // bfdot v21.4s, v7.8h, v3.h[3]\n" - "ldr q7, [x10, #0xf0]\n" + ".inst 0x4f40f30b // bfdot v11.4s, v24.8h, v0.h[0]\n" + ".inst 0x4f41f30f // bfdot v15.4s, v24.8h, v1.h[0]\n" + ".inst 0x4f42f313 // bfdot v19.4s, v24.8h, v2.h[0]\n" + ".inst 0x4f43f317 // bfdot v23.4s, v24.8h, v3.h[0]\n" + "ldr q24, [x10, #0x50]\n" + ".inst 0x4f60f328 // bfdot v8.4s, v25.8h, v0.h[1]\n" + ".inst 0x4f61f32c // bfdot v12.4s, v25.8h, v1.h[1]\n" + ".inst 0x4f62f330 // bfdot v16.4s, v25.8h, v2.h[1]\n" + ".inst 0x4f63f334 // bfdot v20.4s, v25.8h, v3.h[1]\n" + "ldr q25, [x10, #0x60]\n" + ".inst 0x4f60f309 // bfdot v9.4s, v24.8h, v0.h[1]\n" + ".inst 0x4f61f30d // bfdot v13.4s, v24.8h, v1.h[1]\n" + ".inst 0x4f62f311 // bfdot v17.4s, v24.8h, v2.h[1]\n" + ".inst 0x4f63f315 // bfdot v21.4s, v24.8h, v3.h[1]\n" + "ldr q24, [x10, #0x70]\n" + ".inst 0x4f60f32a // bfdot v10.4s, v25.8h, v0.h[1]\n" + ".inst 0x4f61f32e // bfdot v14.4s, v25.8h, v1.h[1]\n" + ".inst 0x4f62f332 // bfdot v18.4s, v25.8h, v2.h[1]\n" + ".inst 0x4f63f336 // bfdot v22.4s, v25.8h, v3.h[1]\n" + "ldr q25, [x10, #0x80]\n" + ".inst 0x4f60f30b // bfdot v11.4s, v24.8h, v0.h[1]\n" + ".inst 0x4f61f30f // bfdot v15.4s, v24.8h, v1.h[1]\n" + ".inst 0x4f62f313 // bfdot v19.4s, v24.8h, v2.h[1]\n" + ".inst 0x4f63f317 // bfdot v23.4s, v24.8h, v3.h[1]\n" + "ldr q24, [x10, #0x90]\n" + ".inst 0x4f40fb28 // bfdot v8.4s, v25.8h, v0.h[2]\n" + ".inst 0x4f41fb2c // bfdot v12.4s, v25.8h, v1.h[2]\n" + ".inst 0x4f42fb30 // bfdot v16.4s, v25.8h, v2.h[2]\n" + ".inst 0x4f43fb34 // bfdot v20.4s, v25.8h, v3.h[2]\n" + "ldr q25, [x10, #0xa0]\n" + ".inst 0x4f40fb09 // bfdot v9.4s, v24.8h, v0.h[2]\n" + ".inst 0x4f41fb0d // bfdot v13.4s, v24.8h, v1.h[2]\n" + ".inst 0x4f42fb11 // bfdot v17.4s, v24.8h, v2.h[2]\n" + ".inst 0x4f43fb15 // bfdot v21.4s, v24.8h, v3.h[2]\n" + "ldr q24, [x10, #0xb0]\n" + ".inst 0x4f40fb2a // bfdot v10.4s, v25.8h, v0.h[2]\n" + ".inst 0x4f41fb2e // bfdot v14.4s, v25.8h, v1.h[2]\n" + ".inst 0x4f42fb32 // bfdot v18.4s, v25.8h, v2.h[2]\n" + ".inst 0x4f43fb36 // bfdot v22.4s, v25.8h, v3.h[2]\n" + "ldr q25, [x10, #0xc0]\n" + ".inst 0x4f40fb0b // bfdot v11.4s, v24.8h, v0.h[2]\n" + ".inst 0x4f41fb0f // bfdot v15.4s, v24.8h, v1.h[2]\n" + ".inst 0x4f42fb13 // bfdot v19.4s, v24.8h, v2.h[2]\n" + ".inst 0x4f43fb17 // bfdot v23.4s, v24.8h, v3.h[2]\n" + "ldr q24, [x10, #0xd0]\n" + ".inst 0x4f60fb28 // bfdot v8.4s, v25.8h, v0.h[3]\n" + ".inst 0x4f61fb2c // bfdot v12.4s, v25.8h, v1.h[3]\n" + ".inst 0x4f62fb30 // bfdot v16.4s, v25.8h, v2.h[3]\n" + ".inst 0x4f63fb34 // bfdot v20.4s, v25.8h, v3.h[3]\n" + "ldr q25, [x10, #0xe0]\n" + ".inst 0x4f60fb09 // bfdot v9.4s, v24.8h, v0.h[3]\n" + ".inst 0x4f61fb0d // bfdot v13.4s, v24.8h, v1.h[3]\n" + ".inst 0x4f62fb11 // bfdot v17.4s, v24.8h, v2.h[3]\n" + ".inst 0x4f63fb15 // bfdot v21.4s, v24.8h, v3.h[3]\n" + "ldr q24, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x4f60f8ca // bfdot v10.4s, v6.8h, v0.h[3]\n" - ".inst 0x4f61f8ce // bfdot v14.4s, v6.8h, v1.h[3]\n" - ".inst 0x4f62f8d2 // bfdot v18.4s, v6.8h, v2.h[3]\n" - ".inst 0x4f63f8d6 // bfdot v22.4s, v6.8h, v3.h[3]\n" - ".inst 0x4f60f8eb // bfdot v11.4s, v7.8h, v0.h[3]\n" - ".inst 0x4f61f8ef // bfdot v15.4s, v7.8h, v1.h[3]\n" - ".inst 0x4f62f8f3 // bfdot v19.4s, v7.8h, v2.h[3]\n" - ".inst 0x4f63f8f7 // bfdot v23.4s, v7.8h, v3.h[3]\n" + ".inst 0x4f60fb2a // bfdot v10.4s, v25.8h, v0.h[3]\n" + ".inst 0x4f61fb2e // bfdot v14.4s, v25.8h, v1.h[3]\n" + ".inst 0x4f62fb32 // bfdot v18.4s, v25.8h, v2.h[3]\n" + ".inst 0x4f63fb36 // bfdot v22.4s, v25.8h, v3.h[3]\n" + ".inst 0x4f60fb0b // bfdot v11.4s, v24.8h, v0.h[3]\n" + ".inst 0x4f61fb0f // bfdot v15.4s, v24.8h, v1.h[3]\n" + ".inst 0x4f62fb13 // bfdot v19.4s, v24.8h, v2.h[3]\n" + ".inst 0x4f63fb17 // bfdot v23.4s, v24.8h, v3.h[3]\n" "125:" // Height 4: Multiply loop: Main loop skip "cbz x27, 129f\n" "cmp x27, #0x2\n" "blt 127f\n" "126:" // Height 4: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s29, [x26], #0x4\n" + "ldr s28, [x25], #0x4\n" "sub x27, x27, #0x2\n" "cmp x27, #0x2\n" - "ldr s2, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n" - ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n" - ".inst 0x4f43f0d4 // bfdot v20.4s, v6.8h, v3.h[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n" - ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n" - ".inst 0x4f43f0f5 // bfdot v21.4s, v7.8h, v3.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr s27, [x24], #0x4\n" + "ldr s26, [x23], #0x4\n" + "ldr q25, [x10, #0x0]\n" + "ldr q24, [x10, #0x10]\n" + ".inst 0x4f5df328 // bfdot v8.4s, v25.8h, v29.h[0]\n" + ".inst 0x4f5cf32c // bfdot v12.4s, v25.8h, v28.h[0]\n" + ".inst 0x4f5bf330 // bfdot v16.4s, v25.8h, v27.h[0]\n" + ".inst 0x4f5af334 // bfdot v20.4s, v25.8h, v26.h[0]\n" + "ldr q25, [x10, #0x20]\n" + ".inst 0x4f5df309 // bfdot v9.4s, v24.8h, v29.h[0]\n" + ".inst 0x4f5cf30d // bfdot v13.4s, v24.8h, v28.h[0]\n" + ".inst 0x4f5bf311 // bfdot v17.4s, v24.8h, v27.h[0]\n" + ".inst 0x4f5af315 // bfdot v21.4s, v24.8h, v26.h[0]\n" + "ldr q24, [x10, #0x30]\n" "add x10, x10, #0x40\n" - ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n" - ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n" - ".inst 0x4f43f0d6 // bfdot v22.4s, v6.8h, v3.h[0]\n" - ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n" - ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n" - ".inst 0x4f43f0f7 // bfdot v23.4s, v7.8h, v3.h[0]\n" + ".inst 0x4f5df32a // bfdot v10.4s, v25.8h, v29.h[0]\n" + ".inst 0x4f5cf32e // bfdot v14.4s, v25.8h, v28.h[0]\n" + ".inst 0x4f5bf332 // bfdot v18.4s, v25.8h, v27.h[0]\n" + ".inst 0x4f5af336 // bfdot v22.4s, v25.8h, v26.h[0]\n" + ".inst 0x4f5df30b // bfdot v11.4s, v24.8h, v29.h[0]\n" + ".inst 0x4f5cf30f // bfdot v15.4s, v24.8h, v28.h[0]\n" + ".inst 0x4f5bf313 // bfdot v19.4s, v24.8h, v27.h[0]\n" + ".inst 0x4f5af317 // bfdot v23.4s, v24.8h, v26.h[0]\n" "bge 126b\n" "127:" // Height 4: Multiply loop: Skip odd blocks "cbz x27, 129f\n" @@ -1742,27 +1741,27 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "ldr h2, [x24, #0x0]\n" "ldr h3, [x23, #0x0]\n" "128:" // Height 4: Multiply loop: Ragged operand read: Done - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n" - ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n" - ".inst 0x4f43f0d4 // bfdot v20.4s, v6.8h, v3.h[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n" - ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n" - ".inst 0x4f43f0f5 // bfdot v21.4s, v7.8h, v3.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q25, [x10, #0x0]\n" + "ldr q24, [x10, #0x10]\n" + ".inst 0x4f40f328 // bfdot v8.4s, v25.8h, v0.h[0]\n" + ".inst 0x4f41f32c // bfdot v12.4s, v25.8h, v1.h[0]\n" + ".inst 0x4f42f330 // bfdot v16.4s, v25.8h, v2.h[0]\n" + ".inst 0x4f43f334 // bfdot v20.4s, v25.8h, v3.h[0]\n" + "ldr q25, [x10, #0x20]\n" + ".inst 0x4f40f309 // bfdot v9.4s, v24.8h, v0.h[0]\n" + ".inst 0x4f41f30d // bfdot v13.4s, v24.8h, v1.h[0]\n" + ".inst 0x4f42f311 // bfdot v17.4s, v24.8h, v2.h[0]\n" + ".inst 0x4f43f315 // bfdot v21.4s, v24.8h, v3.h[0]\n" + "ldr q24, [x10, #0x30]\n" "add x10, x10, #0x40\n" - ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n" - ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n" - ".inst 0x4f43f0d6 // bfdot v22.4s, v6.8h, v3.h[0]\n" - ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n" - ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n" - ".inst 0x4f43f0f7 // bfdot v23.4s, v7.8h, v3.h[0]\n" + ".inst 0x4f40f32a // bfdot v10.4s, v25.8h, v0.h[0]\n" + ".inst 0x4f41f32e // bfdot v14.4s, v25.8h, v1.h[0]\n" + ".inst 0x4f42f332 // bfdot v18.4s, v25.8h, v2.h[0]\n" + ".inst 0x4f43f336 // bfdot v22.4s, v25.8h, v3.h[0]\n" + ".inst 0x4f40f30b // bfdot v11.4s, v24.8h, v0.h[0]\n" + ".inst 0x4f41f30f // bfdot v15.4s, v24.8h, v1.h[0]\n" + ".inst 0x4f42f313 // bfdot v19.4s, v24.8h, v2.h[0]\n" + ".inst 0x4f43f317 // bfdot v23.4s, v24.8h, v3.h[0]\n" "129:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1778,41 +1777,41 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "prfm pstl1keep, [x23, #0x0]\n" "tbz %x[flags], #1, 130f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v25.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v15.4s, v15.4s, v1.4s\n" - "fmin v16.4s, v16.4s, v1.4s\n" - "fmin v17.4s, v17.4s, v1.4s\n" - "fmin v18.4s, v18.4s, v1.4s\n" - "fmin v19.4s, v19.4s, v1.4s\n" - "fmin v20.4s, v20.4s, v1.4s\n" - "fmin v21.4s, v21.4s, v1.4s\n" - "fmin v22.4s, v22.4s, v1.4s\n" - "fmin v23.4s, v23.4s, v1.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" - "fmax v16.4s, v16.4s, v0.4s\n" - "fmax v17.4s, v17.4s, v0.4s\n" - "fmax v18.4s, v18.4s, v0.4s\n" - "fmax v19.4s, v19.4s, v0.4s\n" - "fmax v20.4s, v20.4s, v0.4s\n" - "fmax v21.4s, v21.4s, v0.4s\n" - "fmax v22.4s, v22.4s, v0.4s\n" - "fmax v23.4s, v23.4s, v0.4s\n" + "ld1r { v24.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v25.4s\n" + "fmin v9.4s, v9.4s, v25.4s\n" + "fmin v10.4s, v10.4s, v25.4s\n" + "fmin v11.4s, v11.4s, v25.4s\n" + "fmin v12.4s, v12.4s, v25.4s\n" + "fmin v13.4s, v13.4s, v25.4s\n" + "fmin v14.4s, v14.4s, v25.4s\n" + "fmin v15.4s, v15.4s, v25.4s\n" + "fmin v16.4s, v16.4s, v25.4s\n" + "fmin v17.4s, v17.4s, v25.4s\n" + "fmin v18.4s, v18.4s, v25.4s\n" + "fmin v19.4s, v19.4s, v25.4s\n" + "fmin v20.4s, v20.4s, v25.4s\n" + "fmin v21.4s, v21.4s, v25.4s\n" + "fmin v22.4s, v22.4s, v25.4s\n" + "fmin v23.4s, v23.4s, v25.4s\n" + "fmax v8.4s, v8.4s, v24.4s\n" + "fmax v9.4s, v9.4s, v24.4s\n" + "fmax v10.4s, v10.4s, v24.4s\n" + "fmax v11.4s, v11.4s, v24.4s\n" + "fmax v12.4s, v12.4s, v24.4s\n" + "fmax v13.4s, v13.4s, v24.4s\n" + "fmax v14.4s, v14.4s, v24.4s\n" + "fmax v15.4s, v15.4s, v24.4s\n" + "fmax v16.4s, v16.4s, v24.4s\n" + "fmax v17.4s, v17.4s, v24.4s\n" + "fmax v18.4s, v18.4s, v24.4s\n" + "fmax v19.4s, v19.4s, v24.4s\n" + "fmax v20.4s, v20.4s, v24.4s\n" + "fmax v21.4s, v21.4s, v24.4s\n" + "fmax v22.4s, v22.4s, v24.4s\n" + "fmax v23.4s, v23.4s, v24.4s\n" "130:" // Height 4: No activation "cmp x11, #0x10\n" "bge 139f\n" @@ -2137,15 +2136,15 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "155:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 156f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 157f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -2156,10 +2155,10 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "b 157f\n" "156:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" - "add x22, x23, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" + "add x22, x23, x21, LSL #1\n" "157:" // Height 5: input setup done "cmp x27, #0x8\n" "blt 160f\n" @@ -2182,109 +2181,109 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "add x25, x25, #0x10\n" "add x24, x24, #0x10\n" ".inst 0x4f44f0d8 // bfdot v24.4s, v6.8h, v4.h[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n" - "add x23, x23, #0x10\n" - ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n" - ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n" - "add x22, x22, #0x10\n" - "cmp x27, #0x10\n" - ".inst 0x4f43f0f5 // bfdot v21.4s, v7.8h, v3.h[0]\n" - ".inst 0x4f44f0f9 // bfdot v25.4s, v7.8h, v4.h[0]\n" - "ldr q7, [x10, #0x30]\n" - "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n" - "prfm pldl1keep, [x25, #0x80]\n" - "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n" - ".inst 0x4f43f0d6 // bfdot v22.4s, v6.8h, v3.h[0]\n" - "prfm pldl1keep, [x23, #0x80]\n" - "prfm pldl1keep, [x22, #0x80]\n" - ".inst 0x4f44f0da // bfdot v26.4s, v6.8h, v4.h[0]\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n" - ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n" - ".inst 0x4f43f0f7 // bfdot v23.4s, v7.8h, v3.h[0]\n" - ".inst 0x4f44f0fb // bfdot v27.4s, v7.8h, v4.h[0]\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x4f60f0c8 // bfdot v8.4s, v6.8h, v0.h[1]\n" - ".inst 0x4f61f0cc // bfdot v12.4s, v6.8h, v1.h[1]\n" - ".inst 0x4f62f0d0 // bfdot v16.4s, v6.8h, v2.h[1]\n" - ".inst 0x4f63f0d4 // bfdot v20.4s, v6.8h, v3.h[1]\n" - ".inst 0x4f64f0d8 // bfdot v24.4s, v6.8h, v4.h[1]\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x4f60f0e9 // bfdot v9.4s, v7.8h, v0.h[1]\n" - ".inst 0x4f61f0ed // bfdot v13.4s, v7.8h, v1.h[1]\n" - ".inst 0x4f62f0f1 // bfdot v17.4s, v7.8h, v2.h[1]\n" - ".inst 0x4f63f0f5 // bfdot v21.4s, v7.8h, v3.h[1]\n" - ".inst 0x4f64f0f9 // bfdot v25.4s, v7.8h, v4.h[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x4f60f0ca // bfdot v10.4s, v6.8h, v0.h[1]\n" - ".inst 0x4f61f0ce // bfdot v14.4s, v6.8h, v1.h[1]\n" - ".inst 0x4f62f0d2 // bfdot v18.4s, v6.8h, v2.h[1]\n" - ".inst 0x4f63f0d6 // bfdot v22.4s, v6.8h, v3.h[1]\n" - ".inst 0x4f64f0da // bfdot v26.4s, v6.8h, v4.h[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x4f60f0eb // bfdot v11.4s, v7.8h, v0.h[1]\n" - ".inst 0x4f61f0ef // bfdot v15.4s, v7.8h, v1.h[1]\n" - ".inst 0x4f62f0f3 // bfdot v19.4s, v7.8h, v2.h[1]\n" - ".inst 0x4f63f0f7 // bfdot v23.4s, v7.8h, v3.h[1]\n" - ".inst 0x4f64f0fb // bfdot v27.4s, v7.8h, v4.h[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x4f40f8c8 // bfdot v8.4s, v6.8h, v0.h[2]\n" - ".inst 0x4f41f8cc // bfdot v12.4s, v6.8h, v1.h[2]\n" - ".inst 0x4f42f8d0 // bfdot v16.4s, v6.8h, v2.h[2]\n" - ".inst 0x4f43f8d4 // bfdot v20.4s, v6.8h, v3.h[2]\n" - ".inst 0x4f44f8d8 // bfdot v24.4s, v6.8h, v4.h[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x4f40f8e9 // bfdot v9.4s, v7.8h, v0.h[2]\n" - ".inst 0x4f41f8ed // bfdot v13.4s, v7.8h, v1.h[2]\n" - ".inst 0x4f42f8f1 // bfdot v17.4s, v7.8h, v2.h[2]\n" - ".inst 0x4f43f8f5 // bfdot v21.4s, v7.8h, v3.h[2]\n" - ".inst 0x4f44f8f9 // bfdot v25.4s, v7.8h, v4.h[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x4f40f8ca // bfdot v10.4s, v6.8h, v0.h[2]\n" - ".inst 0x4f41f8ce // bfdot v14.4s, v6.8h, v1.h[2]\n" - ".inst 0x4f42f8d2 // bfdot v18.4s, v6.8h, v2.h[2]\n" - ".inst 0x4f43f8d6 // bfdot v22.4s, v6.8h, v3.h[2]\n" - ".inst 0x4f44f8da // bfdot v26.4s, v6.8h, v4.h[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x4f40f8eb // bfdot v11.4s, v7.8h, v0.h[2]\n" - ".inst 0x4f41f8ef // bfdot v15.4s, v7.8h, v1.h[2]\n" - ".inst 0x4f42f8f3 // bfdot v19.4s, v7.8h, v2.h[2]\n" - ".inst 0x4f43f8f7 // bfdot v23.4s, v7.8h, v3.h[2]\n" - ".inst 0x4f44f8fb // bfdot v27.4s, v7.8h, v4.h[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x4f60f8c8 // bfdot v8.4s, v6.8h, v0.h[3]\n" - ".inst 0x4f61f8cc // bfdot v12.4s, v6.8h, v1.h[3]\n" - ".inst 0x4f62f8d0 // bfdot v16.4s, v6.8h, v2.h[3]\n" - ".inst 0x4f63f8d4 // bfdot v20.4s, v6.8h, v3.h[3]\n" - ".inst 0x4f64f8d8 // bfdot v24.4s, v6.8h, v4.h[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x4f60f8e9 // bfdot v9.4s, v7.8h, v0.h[3]\n" - ".inst 0x4f61f8ed // bfdot v13.4s, v7.8h, v1.h[3]\n" - ".inst 0x4f62f8f1 // bfdot v17.4s, v7.8h, v2.h[3]\n" - ".inst 0x4f63f8f5 // bfdot v21.4s, v7.8h, v3.h[3]\n" - ".inst 0x4f64f8f9 // bfdot v25.4s, v7.8h, v4.h[3]\n" - "ldr q7, [x10, #0xf0]\n" + "ldr q29, [x10, #0x20]\n" + ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n" + "add x23, x23, #0x10\n" + ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n" + ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n" + "add x22, x22, #0x10\n" + "cmp x27, #0x10\n" + ".inst 0x4f43f0f5 // bfdot v21.4s, v7.8h, v3.h[0]\n" + ".inst 0x4f44f0f9 // bfdot v25.4s, v7.8h, v4.h[0]\n" + "ldr q28, [x10, #0x30]\n" + "prfm pldl1keep, [x26, #0x80]\n" + ".inst 0x4f40f3aa // bfdot v10.4s, v29.8h, v0.h[0]\n" + ".inst 0x4f41f3ae // bfdot v14.4s, v29.8h, v1.h[0]\n" + "prfm pldl1keep, [x25, #0x80]\n" + "prfm pldl1keep, [x24, #0x80]\n" + ".inst 0x4f42f3b2 // bfdot v18.4s, v29.8h, v2.h[0]\n" + ".inst 0x4f43f3b6 // bfdot v22.4s, v29.8h, v3.h[0]\n" + "prfm pldl1keep, [x23, #0x80]\n" + "prfm pldl1keep, [x22, #0x80]\n" + ".inst 0x4f44f3ba // bfdot v26.4s, v29.8h, v4.h[0]\n" + "ldr q29, [x10, #0x40]\n" + ".inst 0x4f40f38b // bfdot v11.4s, v28.8h, v0.h[0]\n" + ".inst 0x4f41f38f // bfdot v15.4s, v28.8h, v1.h[0]\n" + ".inst 0x4f42f393 // bfdot v19.4s, v28.8h, v2.h[0]\n" + ".inst 0x4f43f397 // bfdot v23.4s, v28.8h, v3.h[0]\n" + ".inst 0x4f44f39b // bfdot v27.4s, v28.8h, v4.h[0]\n" + "ldr q28, [x10, #0x50]\n" + ".inst 0x4f60f3a8 // bfdot v8.4s, v29.8h, v0.h[1]\n" + ".inst 0x4f61f3ac // bfdot v12.4s, v29.8h, v1.h[1]\n" + ".inst 0x4f62f3b0 // bfdot v16.4s, v29.8h, v2.h[1]\n" + ".inst 0x4f63f3b4 // bfdot v20.4s, v29.8h, v3.h[1]\n" + ".inst 0x4f64f3b8 // bfdot v24.4s, v29.8h, v4.h[1]\n" + "ldr q29, [x10, #0x60]\n" + ".inst 0x4f60f389 // bfdot v9.4s, v28.8h, v0.h[1]\n" + ".inst 0x4f61f38d // bfdot v13.4s, v28.8h, v1.h[1]\n" + ".inst 0x4f62f391 // bfdot v17.4s, v28.8h, v2.h[1]\n" + ".inst 0x4f63f395 // bfdot v21.4s, v28.8h, v3.h[1]\n" + ".inst 0x4f64f399 // bfdot v25.4s, v28.8h, v4.h[1]\n" + "ldr q28, [x10, #0x70]\n" + ".inst 0x4f60f3aa // bfdot v10.4s, v29.8h, v0.h[1]\n" + ".inst 0x4f61f3ae // bfdot v14.4s, v29.8h, v1.h[1]\n" + ".inst 0x4f62f3b2 // bfdot v18.4s, v29.8h, v2.h[1]\n" + ".inst 0x4f63f3b6 // bfdot v22.4s, v29.8h, v3.h[1]\n" + ".inst 0x4f64f3ba // bfdot v26.4s, v29.8h, v4.h[1]\n" + "ldr q29, [x10, #0x80]\n" + ".inst 0x4f60f38b // bfdot v11.4s, v28.8h, v0.h[1]\n" + ".inst 0x4f61f38f // bfdot v15.4s, v28.8h, v1.h[1]\n" + ".inst 0x4f62f393 // bfdot v19.4s, v28.8h, v2.h[1]\n" + ".inst 0x4f63f397 // bfdot v23.4s, v28.8h, v3.h[1]\n" + ".inst 0x4f64f39b // bfdot v27.4s, v28.8h, v4.h[1]\n" + "ldr q28, [x10, #0x90]\n" + ".inst 0x4f40fba8 // bfdot v8.4s, v29.8h, v0.h[2]\n" + ".inst 0x4f41fbac // bfdot v12.4s, v29.8h, v1.h[2]\n" + ".inst 0x4f42fbb0 // bfdot v16.4s, v29.8h, v2.h[2]\n" + ".inst 0x4f43fbb4 // bfdot v20.4s, v29.8h, v3.h[2]\n" + ".inst 0x4f44fbb8 // bfdot v24.4s, v29.8h, v4.h[2]\n" + "ldr q29, [x10, #0xa0]\n" + ".inst 0x4f40fb89 // bfdot v9.4s, v28.8h, v0.h[2]\n" + ".inst 0x4f41fb8d // bfdot v13.4s, v28.8h, v1.h[2]\n" + ".inst 0x4f42fb91 // bfdot v17.4s, v28.8h, v2.h[2]\n" + ".inst 0x4f43fb95 // bfdot v21.4s, v28.8h, v3.h[2]\n" + ".inst 0x4f44fb99 // bfdot v25.4s, v28.8h, v4.h[2]\n" + "ldr q28, [x10, #0xb0]\n" + ".inst 0x4f40fbaa // bfdot v10.4s, v29.8h, v0.h[2]\n" + ".inst 0x4f41fbae // bfdot v14.4s, v29.8h, v1.h[2]\n" + ".inst 0x4f42fbb2 // bfdot v18.4s, v29.8h, v2.h[2]\n" + ".inst 0x4f43fbb6 // bfdot v22.4s, v29.8h, v3.h[2]\n" + ".inst 0x4f44fbba // bfdot v26.4s, v29.8h, v4.h[2]\n" + "ldr q29, [x10, #0xc0]\n" + ".inst 0x4f40fb8b // bfdot v11.4s, v28.8h, v0.h[2]\n" + ".inst 0x4f41fb8f // bfdot v15.4s, v28.8h, v1.h[2]\n" + ".inst 0x4f42fb93 // bfdot v19.4s, v28.8h, v2.h[2]\n" + ".inst 0x4f43fb97 // bfdot v23.4s, v28.8h, v3.h[2]\n" + ".inst 0x4f44fb9b // bfdot v27.4s, v28.8h, v4.h[2]\n" + "ldr q28, [x10, #0xd0]\n" + ".inst 0x4f60fba8 // bfdot v8.4s, v29.8h, v0.h[3]\n" + ".inst 0x4f61fbac // bfdot v12.4s, v29.8h, v1.h[3]\n" + ".inst 0x4f62fbb0 // bfdot v16.4s, v29.8h, v2.h[3]\n" + ".inst 0x4f63fbb4 // bfdot v20.4s, v29.8h, v3.h[3]\n" + ".inst 0x4f64fbb8 // bfdot v24.4s, v29.8h, v4.h[3]\n" + "ldr q29, [x10, #0xe0]\n" + ".inst 0x4f60fb89 // bfdot v9.4s, v28.8h, v0.h[3]\n" + ".inst 0x4f61fb8d // bfdot v13.4s, v28.8h, v1.h[3]\n" + ".inst 0x4f62fb91 // bfdot v17.4s, v28.8h, v2.h[3]\n" + ".inst 0x4f63fb95 // bfdot v21.4s, v28.8h, v3.h[3]\n" + ".inst 0x4f64fb99 // bfdot v25.4s, v28.8h, v4.h[3]\n" + "ldr q28, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x4f60f8ca // bfdot v10.4s, v6.8h, v0.h[3]\n" - ".inst 0x4f61f8ce // bfdot v14.4s, v6.8h, v1.h[3]\n" - ".inst 0x4f62f8d2 // bfdot v18.4s, v6.8h, v2.h[3]\n" - ".inst 0x4f63f8d6 // bfdot v22.4s, v6.8h, v3.h[3]\n" - ".inst 0x4f64f8da // bfdot v26.4s, v6.8h, v4.h[3]\n" + ".inst 0x4f60fbaa // bfdot v10.4s, v29.8h, v0.h[3]\n" + ".inst 0x4f61fbae // bfdot v14.4s, v29.8h, v1.h[3]\n" + ".inst 0x4f62fbb2 // bfdot v18.4s, v29.8h, v2.h[3]\n" + ".inst 0x4f63fbb6 // bfdot v22.4s, v29.8h, v3.h[3]\n" + ".inst 0x4f64fbba // bfdot v26.4s, v29.8h, v4.h[3]\n" "ldr q6, [x10, #0x0]\n" - ".inst 0x4f60f8eb // bfdot v11.4s, v7.8h, v0.h[3]\n" + ".inst 0x4f60fb8b // bfdot v11.4s, v28.8h, v0.h[3]\n" "ldr q0, [x26, #0x0]\n" - ".inst 0x4f61f8ef // bfdot v15.4s, v7.8h, v1.h[3]\n" + ".inst 0x4f61fb8f // bfdot v15.4s, v28.8h, v1.h[3]\n" "ldr q1, [x25, #0x0]\n" - ".inst 0x4f62f8f3 // bfdot v19.4s, v7.8h, v2.h[3]\n" + ".inst 0x4f62fb93 // bfdot v19.4s, v28.8h, v2.h[3]\n" "ldr q2, [x24, #0x0]\n" - ".inst 0x4f63f8f7 // bfdot v23.4s, v7.8h, v3.h[3]\n" + ".inst 0x4f63fb97 // bfdot v23.4s, v28.8h, v3.h[3]\n" "ldr q3, [x23, #0x0]\n" - ".inst 0x4f64f8fb // bfdot v27.4s, v7.8h, v4.h[3]\n" + ".inst 0x4f64fb9b // bfdot v27.4s, v28.8h, v4.h[3]\n" "ldr q4, [x22, #0x0]\n" "ldr q7, [x10, #0x10]\n" "bge 158b\n" @@ -2298,7 +2297,7 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" ".inst 0x4f44f0d8 // bfdot v24.4s, v6.8h, v4.h[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q29, [x10, #0x20]\n" ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n" "add x22, x22, #0x10\n" ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n" @@ -2307,131 +2306,131 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "prfm pldl1keep, [x26, #0x80]\n" ".inst 0x4f43f0f5 // bfdot v21.4s, v7.8h, v3.h[0]\n" ".inst 0x4f44f0f9 // bfdot v25.4s, v7.8h, v4.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q28, [x10, #0x30]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n" + ".inst 0x4f40f3aa // bfdot v10.4s, v29.8h, v0.h[0]\n" + ".inst 0x4f41f3ae // bfdot v14.4s, v29.8h, v1.h[0]\n" "prfm pldl1keep, [x24, #0x80]\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n" - ".inst 0x4f43f0d6 // bfdot v22.4s, v6.8h, v3.h[0]\n" + ".inst 0x4f42f3b2 // bfdot v18.4s, v29.8h, v2.h[0]\n" + ".inst 0x4f43f3b6 // bfdot v22.4s, v29.8h, v3.h[0]\n" "prfm pldl1keep, [x22, #0x80]\n" - ".inst 0x4f44f0da // bfdot v26.4s, v6.8h, v4.h[0]\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n" - ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n" - ".inst 0x4f43f0f7 // bfdot v23.4s, v7.8h, v3.h[0]\n" - ".inst 0x4f44f0fb // bfdot v27.4s, v7.8h, v4.h[0]\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x4f60f0c8 // bfdot v8.4s, v6.8h, v0.h[1]\n" - ".inst 0x4f61f0cc // bfdot v12.4s, v6.8h, v1.h[1]\n" - ".inst 0x4f62f0d0 // bfdot v16.4s, v6.8h, v2.h[1]\n" - ".inst 0x4f63f0d4 // bfdot v20.4s, v6.8h, v3.h[1]\n" - ".inst 0x4f64f0d8 // bfdot v24.4s, v6.8h, v4.h[1]\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x4f60f0e9 // bfdot v9.4s, v7.8h, v0.h[1]\n" - ".inst 0x4f61f0ed // bfdot v13.4s, v7.8h, v1.h[1]\n" - ".inst 0x4f62f0f1 // bfdot v17.4s, v7.8h, v2.h[1]\n" - ".inst 0x4f63f0f5 // bfdot v21.4s, v7.8h, v3.h[1]\n" - ".inst 0x4f64f0f9 // bfdot v25.4s, v7.8h, v4.h[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x4f60f0ca // bfdot v10.4s, v6.8h, v0.h[1]\n" - ".inst 0x4f61f0ce // bfdot v14.4s, v6.8h, v1.h[1]\n" - ".inst 0x4f62f0d2 // bfdot v18.4s, v6.8h, v2.h[1]\n" - ".inst 0x4f63f0d6 // bfdot v22.4s, v6.8h, v3.h[1]\n" - ".inst 0x4f64f0da // bfdot v26.4s, v6.8h, v4.h[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x4f60f0eb // bfdot v11.4s, v7.8h, v0.h[1]\n" - ".inst 0x4f61f0ef // bfdot v15.4s, v7.8h, v1.h[1]\n" - ".inst 0x4f62f0f3 // bfdot v19.4s, v7.8h, v2.h[1]\n" - ".inst 0x4f63f0f7 // bfdot v23.4s, v7.8h, v3.h[1]\n" - ".inst 0x4f64f0fb // bfdot v27.4s, v7.8h, v4.h[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x4f40f8c8 // bfdot v8.4s, v6.8h, v0.h[2]\n" - ".inst 0x4f41f8cc // bfdot v12.4s, v6.8h, v1.h[2]\n" - ".inst 0x4f42f8d0 // bfdot v16.4s, v6.8h, v2.h[2]\n" - ".inst 0x4f43f8d4 // bfdot v20.4s, v6.8h, v3.h[2]\n" - ".inst 0x4f44f8d8 // bfdot v24.4s, v6.8h, v4.h[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x4f40f8e9 // bfdot v9.4s, v7.8h, v0.h[2]\n" - ".inst 0x4f41f8ed // bfdot v13.4s, v7.8h, v1.h[2]\n" - ".inst 0x4f42f8f1 // bfdot v17.4s, v7.8h, v2.h[2]\n" - ".inst 0x4f43f8f5 // bfdot v21.4s, v7.8h, v3.h[2]\n" - ".inst 0x4f44f8f9 // bfdot v25.4s, v7.8h, v4.h[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x4f40f8ca // bfdot v10.4s, v6.8h, v0.h[2]\n" - ".inst 0x4f41f8ce // bfdot v14.4s, v6.8h, v1.h[2]\n" - ".inst 0x4f42f8d2 // bfdot v18.4s, v6.8h, v2.h[2]\n" - ".inst 0x4f43f8d6 // bfdot v22.4s, v6.8h, v3.h[2]\n" - ".inst 0x4f44f8da // bfdot v26.4s, v6.8h, v4.h[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x4f40f8eb // bfdot v11.4s, v7.8h, v0.h[2]\n" - ".inst 0x4f41f8ef // bfdot v15.4s, v7.8h, v1.h[2]\n" - ".inst 0x4f42f8f3 // bfdot v19.4s, v7.8h, v2.h[2]\n" - ".inst 0x4f43f8f7 // bfdot v23.4s, v7.8h, v3.h[2]\n" - ".inst 0x4f44f8fb // bfdot v27.4s, v7.8h, v4.h[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x4f60f8c8 // bfdot v8.4s, v6.8h, v0.h[3]\n" - ".inst 0x4f61f8cc // bfdot v12.4s, v6.8h, v1.h[3]\n" - ".inst 0x4f62f8d0 // bfdot v16.4s, v6.8h, v2.h[3]\n" - ".inst 0x4f63f8d4 // bfdot v20.4s, v6.8h, v3.h[3]\n" - ".inst 0x4f64f8d8 // bfdot v24.4s, v6.8h, v4.h[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x4f60f8e9 // bfdot v9.4s, v7.8h, v0.h[3]\n" - ".inst 0x4f61f8ed // bfdot v13.4s, v7.8h, v1.h[3]\n" - ".inst 0x4f62f8f1 // bfdot v17.4s, v7.8h, v2.h[3]\n" - ".inst 0x4f63f8f5 // bfdot v21.4s, v7.8h, v3.h[3]\n" - ".inst 0x4f64f8f9 // bfdot v25.4s, v7.8h, v4.h[3]\n" - "ldr q7, [x10, #0xf0]\n" + ".inst 0x4f44f3ba // bfdot v26.4s, v29.8h, v4.h[0]\n" + "ldr q29, [x10, #0x40]\n" + ".inst 0x4f40f38b // bfdot v11.4s, v28.8h, v0.h[0]\n" + ".inst 0x4f41f38f // bfdot v15.4s, v28.8h, v1.h[0]\n" + ".inst 0x4f42f393 // bfdot v19.4s, v28.8h, v2.h[0]\n" + ".inst 0x4f43f397 // bfdot v23.4s, v28.8h, v3.h[0]\n" + ".inst 0x4f44f39b // bfdot v27.4s, v28.8h, v4.h[0]\n" + "ldr q28, [x10, #0x50]\n" + ".inst 0x4f60f3a8 // bfdot v8.4s, v29.8h, v0.h[1]\n" + ".inst 0x4f61f3ac // bfdot v12.4s, v29.8h, v1.h[1]\n" + ".inst 0x4f62f3b0 // bfdot v16.4s, v29.8h, v2.h[1]\n" + ".inst 0x4f63f3b4 // bfdot v20.4s, v29.8h, v3.h[1]\n" + ".inst 0x4f64f3b8 // bfdot v24.4s, v29.8h, v4.h[1]\n" + "ldr q29, [x10, #0x60]\n" + ".inst 0x4f60f389 // bfdot v9.4s, v28.8h, v0.h[1]\n" + ".inst 0x4f61f38d // bfdot v13.4s, v28.8h, v1.h[1]\n" + ".inst 0x4f62f391 // bfdot v17.4s, v28.8h, v2.h[1]\n" + ".inst 0x4f63f395 // bfdot v21.4s, v28.8h, v3.h[1]\n" + ".inst 0x4f64f399 // bfdot v25.4s, v28.8h, v4.h[1]\n" + "ldr q28, [x10, #0x70]\n" + ".inst 0x4f60f3aa // bfdot v10.4s, v29.8h, v0.h[1]\n" + ".inst 0x4f61f3ae // bfdot v14.4s, v29.8h, v1.h[1]\n" + ".inst 0x4f62f3b2 // bfdot v18.4s, v29.8h, v2.h[1]\n" + ".inst 0x4f63f3b6 // bfdot v22.4s, v29.8h, v3.h[1]\n" + ".inst 0x4f64f3ba // bfdot v26.4s, v29.8h, v4.h[1]\n" + "ldr q29, [x10, #0x80]\n" + ".inst 0x4f60f38b // bfdot v11.4s, v28.8h, v0.h[1]\n" + ".inst 0x4f61f38f // bfdot v15.4s, v28.8h, v1.h[1]\n" + ".inst 0x4f62f393 // bfdot v19.4s, v28.8h, v2.h[1]\n" + ".inst 0x4f63f397 // bfdot v23.4s, v28.8h, v3.h[1]\n" + ".inst 0x4f64f39b // bfdot v27.4s, v28.8h, v4.h[1]\n" + "ldr q28, [x10, #0x90]\n" + ".inst 0x4f40fba8 // bfdot v8.4s, v29.8h, v0.h[2]\n" + ".inst 0x4f41fbac // bfdot v12.4s, v29.8h, v1.h[2]\n" + ".inst 0x4f42fbb0 // bfdot v16.4s, v29.8h, v2.h[2]\n" + ".inst 0x4f43fbb4 // bfdot v20.4s, v29.8h, v3.h[2]\n" + ".inst 0x4f44fbb8 // bfdot v24.4s, v29.8h, v4.h[2]\n" + "ldr q29, [x10, #0xa0]\n" + ".inst 0x4f40fb89 // bfdot v9.4s, v28.8h, v0.h[2]\n" + ".inst 0x4f41fb8d // bfdot v13.4s, v28.8h, v1.h[2]\n" + ".inst 0x4f42fb91 // bfdot v17.4s, v28.8h, v2.h[2]\n" + ".inst 0x4f43fb95 // bfdot v21.4s, v28.8h, v3.h[2]\n" + ".inst 0x4f44fb99 // bfdot v25.4s, v28.8h, v4.h[2]\n" + "ldr q28, [x10, #0xb0]\n" + ".inst 0x4f40fbaa // bfdot v10.4s, v29.8h, v0.h[2]\n" + ".inst 0x4f41fbae // bfdot v14.4s, v29.8h, v1.h[2]\n" + ".inst 0x4f42fbb2 // bfdot v18.4s, v29.8h, v2.h[2]\n" + ".inst 0x4f43fbb6 // bfdot v22.4s, v29.8h, v3.h[2]\n" + ".inst 0x4f44fbba // bfdot v26.4s, v29.8h, v4.h[2]\n" + "ldr q29, [x10, #0xc0]\n" + ".inst 0x4f40fb8b // bfdot v11.4s, v28.8h, v0.h[2]\n" + ".inst 0x4f41fb8f // bfdot v15.4s, v28.8h, v1.h[2]\n" + ".inst 0x4f42fb93 // bfdot v19.4s, v28.8h, v2.h[2]\n" + ".inst 0x4f43fb97 // bfdot v23.4s, v28.8h, v3.h[2]\n" + ".inst 0x4f44fb9b // bfdot v27.4s, v28.8h, v4.h[2]\n" + "ldr q28, [x10, #0xd0]\n" + ".inst 0x4f60fba8 // bfdot v8.4s, v29.8h, v0.h[3]\n" + ".inst 0x4f61fbac // bfdot v12.4s, v29.8h, v1.h[3]\n" + ".inst 0x4f62fbb0 // bfdot v16.4s, v29.8h, v2.h[3]\n" + ".inst 0x4f63fbb4 // bfdot v20.4s, v29.8h, v3.h[3]\n" + ".inst 0x4f64fbb8 // bfdot v24.4s, v29.8h, v4.h[3]\n" + "ldr q29, [x10, #0xe0]\n" + ".inst 0x4f60fb89 // bfdot v9.4s, v28.8h, v0.h[3]\n" + ".inst 0x4f61fb8d // bfdot v13.4s, v28.8h, v1.h[3]\n" + ".inst 0x4f62fb91 // bfdot v17.4s, v28.8h, v2.h[3]\n" + ".inst 0x4f63fb95 // bfdot v21.4s, v28.8h, v3.h[3]\n" + ".inst 0x4f64fb99 // bfdot v25.4s, v28.8h, v4.h[3]\n" + "ldr q28, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x4f60f8ca // bfdot v10.4s, v6.8h, v0.h[3]\n" - ".inst 0x4f61f8ce // bfdot v14.4s, v6.8h, v1.h[3]\n" - ".inst 0x4f62f8d2 // bfdot v18.4s, v6.8h, v2.h[3]\n" - ".inst 0x4f63f8d6 // bfdot v22.4s, v6.8h, v3.h[3]\n" - ".inst 0x4f64f8da // bfdot v26.4s, v6.8h, v4.h[3]\n" - ".inst 0x4f60f8eb // bfdot v11.4s, v7.8h, v0.h[3]\n" - ".inst 0x4f61f8ef // bfdot v15.4s, v7.8h, v1.h[3]\n" - ".inst 0x4f62f8f3 // bfdot v19.4s, v7.8h, v2.h[3]\n" - ".inst 0x4f63f8f7 // bfdot v23.4s, v7.8h, v3.h[3]\n" - ".inst 0x4f64f8fb // bfdot v27.4s, v7.8h, v4.h[3]\n" + ".inst 0x4f60fbaa // bfdot v10.4s, v29.8h, v0.h[3]\n" + ".inst 0x4f61fbae // bfdot v14.4s, v29.8h, v1.h[3]\n" + ".inst 0x4f62fbb2 // bfdot v18.4s, v29.8h, v2.h[3]\n" + ".inst 0x4f63fbb6 // bfdot v22.4s, v29.8h, v3.h[3]\n" + ".inst 0x4f64fbba // bfdot v26.4s, v29.8h, v4.h[3]\n" + ".inst 0x4f60fb8b // bfdot v11.4s, v28.8h, v0.h[3]\n" + ".inst 0x4f61fb8f // bfdot v15.4s, v28.8h, v1.h[3]\n" + ".inst 0x4f62fb93 // bfdot v19.4s, v28.8h, v2.h[3]\n" + ".inst 0x4f63fb97 // bfdot v23.4s, v28.8h, v3.h[3]\n" + ".inst 0x4f64fb9b // bfdot v27.4s, v28.8h, v4.h[3]\n" "160:" // Height 5: Multiply loop: Main loop skip "cbz x27, 164f\n" "cmp x27, #0x2\n" "blt 162f\n" "161:" // Height 5: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" + "ldr s2, [x26], #0x4\n" "ldr s1, [x25], #0x4\n" "sub x27, x27, #0x2\n" "cmp x27, #0x2\n" - "ldr s2, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr s4, [x22], #0x4\n" - "ldr q6, [x10, #0x0]\n" - ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n" - ".inst 0x4f43f0d4 // bfdot v20.4s, v6.8h, v3.h[0]\n" - ".inst 0x4f44f0d8 // bfdot v24.4s, v6.8h, v4.h[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n" - ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n" - ".inst 0x4f43f0f5 // bfdot v21.4s, v7.8h, v3.h[0]\n" - ".inst 0x4f44f0f9 // bfdot v25.4s, v7.8h, v4.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr s0, [x24], #0x4\n" + "ldr s31, [x23], #0x4\n" + "ldr s30, [x22], #0x4\n" + "ldr q29, [x10, #0x0]\n" + ".inst 0x4f42f3a8 // bfdot v8.4s, v29.8h, v2.h[0]\n" + ".inst 0x4f41f3ac // bfdot v12.4s, v29.8h, v1.h[0]\n" + "ldr q28, [x10, #0x10]\n" + ".inst 0x4f40f3b0 // bfdot v16.4s, v29.8h, v0.h[0]\n" + ".inst 0x4f5ff3b4 // bfdot v20.4s, v29.8h, v31.h[0]\n" + ".inst 0x4f5ef3b8 // bfdot v24.4s, v29.8h, v30.h[0]\n" + "ldr q29, [x10, #0x20]\n" + ".inst 0x4f42f389 // bfdot v9.4s, v28.8h, v2.h[0]\n" + ".inst 0x4f41f38d // bfdot v13.4s, v28.8h, v1.h[0]\n" + ".inst 0x4f40f391 // bfdot v17.4s, v28.8h, v0.h[0]\n" + ".inst 0x4f5ff395 // bfdot v21.4s, v28.8h, v31.h[0]\n" + ".inst 0x4f5ef399 // bfdot v25.4s, v28.8h, v30.h[0]\n" + "ldr q28, [x10, #0x30]\n" "add x10, x10, #0x40\n" - ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n" - ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n" - ".inst 0x4f43f0d6 // bfdot v22.4s, v6.8h, v3.h[0]\n" - ".inst 0x4f44f0da // bfdot v26.4s, v6.8h, v4.h[0]\n" - ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n" - ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n" - ".inst 0x4f43f0f7 // bfdot v23.4s, v7.8h, v3.h[0]\n" - ".inst 0x4f44f0fb // bfdot v27.4s, v7.8h, v4.h[0]\n" + ".inst 0x4f42f3aa // bfdot v10.4s, v29.8h, v2.h[0]\n" + ".inst 0x4f41f3ae // bfdot v14.4s, v29.8h, v1.h[0]\n" + ".inst 0x4f40f3b2 // bfdot v18.4s, v29.8h, v0.h[0]\n" + ".inst 0x4f5ff3b6 // bfdot v22.4s, v29.8h, v31.h[0]\n" + ".inst 0x4f5ef3ba // bfdot v26.4s, v29.8h, v30.h[0]\n" + ".inst 0x4f42f38b // bfdot v11.4s, v28.8h, v2.h[0]\n" + ".inst 0x4f41f38f // bfdot v15.4s, v28.8h, v1.h[0]\n" + ".inst 0x4f40f393 // bfdot v19.4s, v28.8h, v0.h[0]\n" + ".inst 0x4f5ff397 // bfdot v23.4s, v28.8h, v31.h[0]\n" + ".inst 0x4f5ef39b // bfdot v27.4s, v28.8h, v30.h[0]\n" "bge 161b\n" "162:" // Height 5: Multiply loop: Skip odd blocks "cbz x27, 164f\n" @@ -2441,31 +2440,31 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "ldr h3, [x23, #0x0]\n" "ldr h4, [x22, #0x0]\n" "163:" // Height 5: Multiply loop: Ragged operand read: Done - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n" - ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n" - ".inst 0x4f43f0d4 // bfdot v20.4s, v6.8h, v3.h[0]\n" - ".inst 0x4f44f0d8 // bfdot v24.4s, v6.8h, v4.h[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n" - ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n" - ".inst 0x4f43f0f5 // bfdot v21.4s, v7.8h, v3.h[0]\n" - ".inst 0x4f44f0f9 // bfdot v25.4s, v7.8h, v4.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q29, [x10, #0x0]\n" + "ldr q28, [x10, #0x10]\n" + ".inst 0x4f40f3a8 // bfdot v8.4s, v29.8h, v0.h[0]\n" + ".inst 0x4f41f3ac // bfdot v12.4s, v29.8h, v1.h[0]\n" + ".inst 0x4f42f3b0 // bfdot v16.4s, v29.8h, v2.h[0]\n" + ".inst 0x4f43f3b4 // bfdot v20.4s, v29.8h, v3.h[0]\n" + ".inst 0x4f44f3b8 // bfdot v24.4s, v29.8h, v4.h[0]\n" + "ldr q29, [x10, #0x20]\n" + ".inst 0x4f40f389 // bfdot v9.4s, v28.8h, v0.h[0]\n" + ".inst 0x4f41f38d // bfdot v13.4s, v28.8h, v1.h[0]\n" + ".inst 0x4f42f391 // bfdot v17.4s, v28.8h, v2.h[0]\n" + ".inst 0x4f43f395 // bfdot v21.4s, v28.8h, v3.h[0]\n" + ".inst 0x4f44f399 // bfdot v25.4s, v28.8h, v4.h[0]\n" + "ldr q28, [x10, #0x30]\n" "add x10, x10, #0x40\n" - ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n" - ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n" - ".inst 0x4f43f0d6 // bfdot v22.4s, v6.8h, v3.h[0]\n" - ".inst 0x4f44f0da // bfdot v26.4s, v6.8h, v4.h[0]\n" - ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n" - ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n" - ".inst 0x4f43f0f7 // bfdot v23.4s, v7.8h, v3.h[0]\n" - ".inst 0x4f44f0fb // bfdot v27.4s, v7.8h, v4.h[0]\n" + ".inst 0x4f40f3aa // bfdot v10.4s, v29.8h, v0.h[0]\n" + ".inst 0x4f41f3ae // bfdot v14.4s, v29.8h, v1.h[0]\n" + ".inst 0x4f42f3b2 // bfdot v18.4s, v29.8h, v2.h[0]\n" + ".inst 0x4f43f3b6 // bfdot v22.4s, v29.8h, v3.h[0]\n" + ".inst 0x4f44f3ba // bfdot v26.4s, v29.8h, v4.h[0]\n" + ".inst 0x4f40f38b // bfdot v11.4s, v28.8h, v0.h[0]\n" + ".inst 0x4f41f38f // bfdot v15.4s, v28.8h, v1.h[0]\n" + ".inst 0x4f42f393 // bfdot v19.4s, v28.8h, v2.h[0]\n" + ".inst 0x4f43f397 // bfdot v23.4s, v28.8h, v3.h[0]\n" + ".inst 0x4f44f39b // bfdot v27.4s, v28.8h, v4.h[0]\n" "164:" // Height 5: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -2483,49 +2482,49 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "prfm pstl1keep, [x22, #0x0]\n" "tbz %x[flags], #1, 165f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v29.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v15.4s, v15.4s, v1.4s\n" - "fmin v16.4s, v16.4s, v1.4s\n" - "fmin v17.4s, v17.4s, v1.4s\n" - "fmin v18.4s, v18.4s, v1.4s\n" - "fmin v19.4s, v19.4s, v1.4s\n" - "fmin v20.4s, v20.4s, v1.4s\n" - "fmin v21.4s, v21.4s, v1.4s\n" - "fmin v22.4s, v22.4s, v1.4s\n" - "fmin v23.4s, v23.4s, v1.4s\n" - "fmin v24.4s, v24.4s, v1.4s\n" - "fmin v25.4s, v25.4s, v1.4s\n" - "fmin v26.4s, v26.4s, v1.4s\n" - "fmin v27.4s, v27.4s, v1.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" - "fmax v16.4s, v16.4s, v0.4s\n" - "fmax v17.4s, v17.4s, v0.4s\n" - "fmax v18.4s, v18.4s, v0.4s\n" - "fmax v19.4s, v19.4s, v0.4s\n" - "fmax v20.4s, v20.4s, v0.4s\n" - "fmax v21.4s, v21.4s, v0.4s\n" - "fmax v22.4s, v22.4s, v0.4s\n" - "fmax v23.4s, v23.4s, v0.4s\n" - "fmax v24.4s, v24.4s, v0.4s\n" - "fmax v25.4s, v25.4s, v0.4s\n" - "fmax v26.4s, v26.4s, v0.4s\n" - "fmax v27.4s, v27.4s, v0.4s\n" + "ld1r { v28.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v29.4s\n" + "fmin v9.4s, v9.4s, v29.4s\n" + "fmin v10.4s, v10.4s, v29.4s\n" + "fmin v11.4s, v11.4s, v29.4s\n" + "fmin v12.4s, v12.4s, v29.4s\n" + "fmin v13.4s, v13.4s, v29.4s\n" + "fmin v14.4s, v14.4s, v29.4s\n" + "fmin v15.4s, v15.4s, v29.4s\n" + "fmin v16.4s, v16.4s, v29.4s\n" + "fmin v17.4s, v17.4s, v29.4s\n" + "fmin v18.4s, v18.4s, v29.4s\n" + "fmin v19.4s, v19.4s, v29.4s\n" + "fmin v20.4s, v20.4s, v29.4s\n" + "fmin v21.4s, v21.4s, v29.4s\n" + "fmin v22.4s, v22.4s, v29.4s\n" + "fmin v23.4s, v23.4s, v29.4s\n" + "fmin v24.4s, v24.4s, v29.4s\n" + "fmin v25.4s, v25.4s, v29.4s\n" + "fmin v26.4s, v26.4s, v29.4s\n" + "fmin v27.4s, v27.4s, v29.4s\n" + "fmax v8.4s, v8.4s, v28.4s\n" + "fmax v9.4s, v9.4s, v28.4s\n" + "fmax v10.4s, v10.4s, v28.4s\n" + "fmax v11.4s, v11.4s, v28.4s\n" + "fmax v12.4s, v12.4s, v28.4s\n" + "fmax v13.4s, v13.4s, v28.4s\n" + "fmax v14.4s, v14.4s, v28.4s\n" + "fmax v15.4s, v15.4s, v28.4s\n" + "fmax v16.4s, v16.4s, v28.4s\n" + "fmax v17.4s, v17.4s, v28.4s\n" + "fmax v18.4s, v18.4s, v28.4s\n" + "fmax v19.4s, v19.4s, v28.4s\n" + "fmax v20.4s, v20.4s, v28.4s\n" + "fmax v21.4s, v21.4s, v28.4s\n" + "fmax v22.4s, v22.4s, v28.4s\n" + "fmax v23.4s, v23.4s, v28.4s\n" + "fmax v24.4s, v24.4s, v28.4s\n" + "fmax v25.4s, v25.4s, v28.4s\n" + "fmax v26.4s, v26.4s, v28.4s\n" + "fmax v27.4s, v27.4s, v28.4s\n" "165:" // Height 5: No activation "cmp x11, #0x10\n" "bge 174f\n" @@ -2902,16 +2901,16 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "190:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 191f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 192f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -2923,11 +2922,11 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "b 192f\n" "191:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" - "add x22, x23, x20, LSL #1\n" - "add x21, x22, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" + "add x22, x23, x21, LSL #1\n" + "add x21, x22, x21, LSL #1\n" "192:" // Height 6: input setup done "cmp x27, #0x8\n" "blt 195f\n" @@ -3206,43 +3205,43 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "cmp x27, #0x2\n" "blt 197f\n" "196:" // Height 6: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s7, [x26], #0x4\n" + "ldr s6, [x25], #0x4\n" "sub x27, x27, #0x2\n" "cmp x27, #0x2\n" - "ldr s2, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr s4, [x22], #0x4\n" - "ldr s5, [x21], #0x4\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n" - ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n" - ".inst 0x4f43f0d4 // bfdot v20.4s, v6.8h, v3.h[0]\n" - ".inst 0x4f44f0d8 // bfdot v24.4s, v6.8h, v4.h[0]\n" - ".inst 0x4f45f0dc // bfdot v28.4s, v6.8h, v5.h[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n" - ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n" - ".inst 0x4f43f0f5 // bfdot v21.4s, v7.8h, v3.h[0]\n" - ".inst 0x4f44f0f9 // bfdot v25.4s, v7.8h, v4.h[0]\n" - ".inst 0x4f45f0fd // bfdot v29.4s, v7.8h, v5.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr s5, [x24], #0x4\n" + "ldr s4, [x23], #0x4\n" + "ldr s3, [x22], #0x4\n" + "ldr s2, [x21], #0x4\n" + "ldr q1, [x10, #0x0]\n" + "ldr q0, [x10, #0x10]\n" + ".inst 0x4f47f028 // bfdot v8.4s, v1.8h, v7.h[0]\n" + ".inst 0x4f46f02c // bfdot v12.4s, v1.8h, v6.h[0]\n" + ".inst 0x4f45f030 // bfdot v16.4s, v1.8h, v5.h[0]\n" + ".inst 0x4f44f034 // bfdot v20.4s, v1.8h, v4.h[0]\n" + ".inst 0x4f43f038 // bfdot v24.4s, v1.8h, v3.h[0]\n" + ".inst 0x4f42f03c // bfdot v28.4s, v1.8h, v2.h[0]\n" + "ldr q1, [x10, #0x20]\n" + ".inst 0x4f47f009 // bfdot v9.4s, v0.8h, v7.h[0]\n" + ".inst 0x4f46f00d // bfdot v13.4s, v0.8h, v6.h[0]\n" + ".inst 0x4f45f011 // bfdot v17.4s, v0.8h, v5.h[0]\n" + ".inst 0x4f44f015 // bfdot v21.4s, v0.8h, v4.h[0]\n" + ".inst 0x4f43f019 // bfdot v25.4s, v0.8h, v3.h[0]\n" + ".inst 0x4f42f01d // bfdot v29.4s, v0.8h, v2.h[0]\n" + "ldr q0, [x10, #0x30]\n" "add x10, x10, #0x40\n" - ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n" - ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n" - ".inst 0x4f43f0d6 // bfdot v22.4s, v6.8h, v3.h[0]\n" - ".inst 0x4f44f0da // bfdot v26.4s, v6.8h, v4.h[0]\n" - ".inst 0x4f45f0de // bfdot v30.4s, v6.8h, v5.h[0]\n" - ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n" - ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n" - ".inst 0x4f43f0f7 // bfdot v23.4s, v7.8h, v3.h[0]\n" - ".inst 0x4f44f0fb // bfdot v27.4s, v7.8h, v4.h[0]\n" - ".inst 0x4f45f0ff // bfdot v31.4s, v7.8h, v5.h[0]\n" + ".inst 0x4f47f02a // bfdot v10.4s, v1.8h, v7.h[0]\n" + ".inst 0x4f46f02e // bfdot v14.4s, v1.8h, v6.h[0]\n" + ".inst 0x4f45f032 // bfdot v18.4s, v1.8h, v5.h[0]\n" + ".inst 0x4f44f036 // bfdot v22.4s, v1.8h, v4.h[0]\n" + ".inst 0x4f43f03a // bfdot v26.4s, v1.8h, v3.h[0]\n" + ".inst 0x4f42f03e // bfdot v30.4s, v1.8h, v2.h[0]\n" + ".inst 0x4f47f00b // bfdot v11.4s, v0.8h, v7.h[0]\n" + ".inst 0x4f46f00f // bfdot v15.4s, v0.8h, v6.h[0]\n" + ".inst 0x4f45f013 // bfdot v19.4s, v0.8h, v5.h[0]\n" + ".inst 0x4f44f017 // bfdot v23.4s, v0.8h, v4.h[0]\n" + ".inst 0x4f43f01b // bfdot v27.4s, v0.8h, v3.h[0]\n" + ".inst 0x4f42f01f // bfdot v31.4s, v0.8h, v2.h[0]\n" "bge 196b\n" "197:" // Height 6: Multiply loop: Skip odd blocks "cbz x27, 199f\n" @@ -3253,35 +3252,35 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "ldr h4, [x22, #0x0]\n" "ldr h5, [x21, #0x0]\n" "198:" // Height 6: Multiply loop: Ragged operand read: Done - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4f40f0c8 // bfdot v8.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0cc // bfdot v12.4s, v6.8h, v1.h[0]\n" - ".inst 0x4f42f0d0 // bfdot v16.4s, v6.8h, v2.h[0]\n" - ".inst 0x4f43f0d4 // bfdot v20.4s, v6.8h, v3.h[0]\n" - ".inst 0x4f44f0d8 // bfdot v24.4s, v6.8h, v4.h[0]\n" - ".inst 0x4f45f0dc // bfdot v28.4s, v6.8h, v5.h[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4f40f0e9 // bfdot v9.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ed // bfdot v13.4s, v7.8h, v1.h[0]\n" - ".inst 0x4f42f0f1 // bfdot v17.4s, v7.8h, v2.h[0]\n" - ".inst 0x4f43f0f5 // bfdot v21.4s, v7.8h, v3.h[0]\n" - ".inst 0x4f44f0f9 // bfdot v25.4s, v7.8h, v4.h[0]\n" - ".inst 0x4f45f0fd // bfdot v29.4s, v7.8h, v5.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q7, [x10, #0x0]\n" + "ldr q6, [x10, #0x10]\n" + ".inst 0x4f40f0e8 // bfdot v8.4s, v7.8h, v0.h[0]\n" + ".inst 0x4f41f0ec // bfdot v12.4s, v7.8h, v1.h[0]\n" + ".inst 0x4f42f0f0 // bfdot v16.4s, v7.8h, v2.h[0]\n" + ".inst 0x4f43f0f4 // bfdot v20.4s, v7.8h, v3.h[0]\n" + ".inst 0x4f44f0f8 // bfdot v24.4s, v7.8h, v4.h[0]\n" + ".inst 0x4f45f0fc // bfdot v28.4s, v7.8h, v5.h[0]\n" + "ldr q7, [x10, #0x20]\n" + ".inst 0x4f40f0c9 // bfdot v9.4s, v6.8h, v0.h[0]\n" + ".inst 0x4f41f0cd // bfdot v13.4s, v6.8h, v1.h[0]\n" + ".inst 0x4f42f0d1 // bfdot v17.4s, v6.8h, v2.h[0]\n" + ".inst 0x4f43f0d5 // bfdot v21.4s, v6.8h, v3.h[0]\n" + ".inst 0x4f44f0d9 // bfdot v25.4s, v6.8h, v4.h[0]\n" + ".inst 0x4f45f0dd // bfdot v29.4s, v6.8h, v5.h[0]\n" + "ldr q6, [x10, #0x30]\n" "add x10, x10, #0x40\n" - ".inst 0x4f40f0ca // bfdot v10.4s, v6.8h, v0.h[0]\n" - ".inst 0x4f41f0ce // bfdot v14.4s, v6.8h, v1.h[0]\n" - ".inst 0x4f42f0d2 // bfdot v18.4s, v6.8h, v2.h[0]\n" - ".inst 0x4f43f0d6 // bfdot v22.4s, v6.8h, v3.h[0]\n" - ".inst 0x4f44f0da // bfdot v26.4s, v6.8h, v4.h[0]\n" - ".inst 0x4f45f0de // bfdot v30.4s, v6.8h, v5.h[0]\n" - ".inst 0x4f40f0eb // bfdot v11.4s, v7.8h, v0.h[0]\n" - ".inst 0x4f41f0ef // bfdot v15.4s, v7.8h, v1.h[0]\n" - ".inst 0x4f42f0f3 // bfdot v19.4s, v7.8h, v2.h[0]\n" - ".inst 0x4f43f0f7 // bfdot v23.4s, v7.8h, v3.h[0]\n" - ".inst 0x4f44f0fb // bfdot v27.4s, v7.8h, v4.h[0]\n" - ".inst 0x4f45f0ff // bfdot v31.4s, v7.8h, v5.h[0]\n" + ".inst 0x4f40f0ea // bfdot v10.4s, v7.8h, v0.h[0]\n" + ".inst 0x4f41f0ee // bfdot v14.4s, v7.8h, v1.h[0]\n" + ".inst 0x4f42f0f2 // bfdot v18.4s, v7.8h, v2.h[0]\n" + ".inst 0x4f43f0f6 // bfdot v22.4s, v7.8h, v3.h[0]\n" + ".inst 0x4f44f0fa // bfdot v26.4s, v7.8h, v4.h[0]\n" + ".inst 0x4f45f0fe // bfdot v30.4s, v7.8h, v5.h[0]\n" + ".inst 0x4f40f0cb // bfdot v11.4s, v6.8h, v0.h[0]\n" + ".inst 0x4f41f0cf // bfdot v15.4s, v6.8h, v1.h[0]\n" + ".inst 0x4f42f0d3 // bfdot v19.4s, v6.8h, v2.h[0]\n" + ".inst 0x4f43f0d7 // bfdot v23.4s, v6.8h, v3.h[0]\n" + ".inst 0x4f44f0db // bfdot v27.4s, v6.8h, v4.h[0]\n" + ".inst 0x4f45f0df // bfdot v31.4s, v6.8h, v5.h[0]\n" "199:" // Height 6: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -3522,7 +3521,6 @@ void a64_hybrid_bf16fp32_dot_6x16 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "212:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_mmla_6x16.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_mmla_6x16.hpp index 8cb743b777eb38f0c4ab2052462ca5f11830e754..d9e7259fa2294912bc313d8c709c327cdabefaa4 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_mmla_6x16.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_mmla_6x16.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef __aarch64__ + #include "../std_transforms_fixed.hpp" #include "../bfloat.hpp" #include "../performance_parameters.hpp" @@ -99,5 +99,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_mmla_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_mmla_6x16/generic.cpp index 5a000c69af1e1ff202f4eec63e2fb51be3f7d016..f6389e27d13e870642b5bb383ac86de59d209fc6 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_mmla_6x16/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_bf16fp32_mmla_6x16/generic.cpp @@ -93,7 +93,6 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( break; } __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x6\n" "bge 186f\n" @@ -211,11 +210,11 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "16:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 17f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 18f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -231,41 +230,41 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "ldr q6, [x10, #0x10]\n" "blt 20f\n" "19:" // Height 1: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x70]\n" - "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x80]\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x90]\n" - ".inst 0x6e47ec28 // bfmmla v8.4s, v1.8h, v7.8h\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x6e46ec2c // bfmmla v12.4s, v1.8h, v6.8h\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x6e47ec29 // bfmmla v9.4s, v1.8h, v7.8h\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x6e46ec2d // bfmmla v13.4s, v1.8h, v6.8h\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x6e47ec2a // bfmmla v10.4s, v1.8h, v7.8h\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - "ldr q6, [x10, #0xf0]\n" + "trn1 v20.2d, v1.2d, v21.2d\n" + ".inst 0x6e47ee88 // bfmmla v8.4s, v20.8h, v7.8h\n" + "ldr q17, [x10, #0x20]\n" + ".inst 0x6e46ee8c // bfmmla v12.4s, v20.8h, v6.8h\n" + "ldr q19, [x10, #0x30]\n" + ".inst 0x6e51ee89 // bfmmla v9.4s, v20.8h, v17.8h\n" + "ldr q18, [x10, #0x40]\n" + ".inst 0x6e53ee8d // bfmmla v13.4s, v20.8h, v19.8h\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x6e52ee8a // bfmmla v10.4s, v20.8h, v18.8h\n" + "ldr q18, [x10, #0x60]\n" + ".inst 0x6e51ee8e // bfmmla v14.4s, v20.8h, v17.8h\n" + "ldr q17, [x10, #0x70]\n" + "trn2 v1.2d, v1.2d, v21.2d\n" + ".inst 0x6e52ee8b // bfmmla v11.4s, v20.8h, v18.8h\n" + "ldr q18, [x10, #0x80]\n" + ".inst 0x6e51ee8f // bfmmla v15.4s, v20.8h, v17.8h\n" + "ldr q17, [x10, #0x90]\n" + ".inst 0x6e52ec28 // bfmmla v8.4s, v1.8h, v18.8h\n" + "ldr q18, [x10, #0xa0]\n" + ".inst 0x6e51ec2c // bfmmla v12.4s, v1.8h, v17.8h\n" + "ldr q17, [x10, #0xb0]\n" + ".inst 0x6e52ec29 // bfmmla v9.4s, v1.8h, v18.8h\n" + "ldr q18, [x10, #0xc0]\n" + ".inst 0x6e51ec2d // bfmmla v13.4s, v1.8h, v17.8h\n" + "ldr q17, [x10, #0xd0]\n" + ".inst 0x6e52ec2a // bfmmla v10.4s, v1.8h, v18.8h\n" + "ldr q18, [x10, #0xe0]\n" + ".inst 0x6e51ec2e // bfmmla v14.4s, v1.8h, v17.8h\n" + "ldr q17, [x10, #0xf0]\n" "sub x27, x27, #0x8\n" "add x26, x26, #0x10\n" "cmp x27, #0x10\n" - ".inst 0x6e47ec2b // bfmmla v11.4s, v1.8h, v7.8h\n" - ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" + ".inst 0x6e52ec2b // bfmmla v11.4s, v1.8h, v18.8h\n" + ".inst 0x6e51ec2f // bfmmla v15.4s, v1.8h, v17.8h\n" "ldr q1, [x26, #0x0]\n" "add x10, x10, #0x100\n" "ldr q7, [x10, #0x0]\n" @@ -273,40 +272,40 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "prfm pldl1keep, [x26, #0x80]\n" "bge 19b\n" "20:" // Height 1: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x70]\n" - "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x80]\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x90]\n" - ".inst 0x6e47ec28 // bfmmla v8.4s, v1.8h, v7.8h\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x6e46ec2c // bfmmla v12.4s, v1.8h, v6.8h\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x6e47ec29 // bfmmla v9.4s, v1.8h, v7.8h\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x6e46ec2d // bfmmla v13.4s, v1.8h, v6.8h\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x6e47ec2a // bfmmla v10.4s, v1.8h, v7.8h\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - "ldr q6, [x10, #0xf0]\n" + "trn1 v19.2d, v1.2d, v20.2d\n" + ".inst 0x6e47ee68 // bfmmla v8.4s, v19.8h, v7.8h\n" + "ldr q17, [x10, #0x20]\n" + ".inst 0x6e46ee6c // bfmmla v12.4s, v19.8h, v6.8h\n" + "ldr q18, [x10, #0x30]\n" + ".inst 0x6e51ee69 // bfmmla v9.4s, v19.8h, v17.8h\n" + "ldr q17, [x10, #0x40]\n" + ".inst 0x6e52ee6d // bfmmla v13.4s, v19.8h, v18.8h\n" + "ldr q18, [x10, #0x50]\n" + ".inst 0x6e51ee6a // bfmmla v10.4s, v19.8h, v17.8h\n" + "ldr q17, [x10, #0x60]\n" + ".inst 0x6e52ee6e // bfmmla v14.4s, v19.8h, v18.8h\n" + "ldr q24, [x10, #0x70]\n" + "trn2 v1.2d, v1.2d, v20.2d\n" + ".inst 0x6e51ee6b // bfmmla v11.4s, v19.8h, v17.8h\n" + "ldr q17, [x10, #0x80]\n" + ".inst 0x6e58ee6f // bfmmla v15.4s, v19.8h, v24.8h\n" + "ldr q2, [x10, #0x90]\n" + ".inst 0x6e51ec28 // bfmmla v8.4s, v1.8h, v17.8h\n" + "ldr q18, [x10, #0xa0]\n" + ".inst 0x6e42ec2c // bfmmla v12.4s, v1.8h, v2.8h\n" + "ldr q17, [x10, #0xb0]\n" + ".inst 0x6e52ec29 // bfmmla v9.4s, v1.8h, v18.8h\n" + "ldr q18, [x10, #0xc0]\n" + ".inst 0x6e51ec2d // bfmmla v13.4s, v1.8h, v17.8h\n" + "ldr q17, [x10, #0xd0]\n" + ".inst 0x6e52ec2a // bfmmla v10.4s, v1.8h, v18.8h\n" + "ldr q18, [x10, #0xe0]\n" + ".inst 0x6e51ec2e // bfmmla v14.4s, v1.8h, v17.8h\n" + "ldr q17, [x10, #0xf0]\n" "add x26, x26, #0x10\n" "sub x27, x27, #0x8\n" - ".inst 0x6e47ec2b // bfmmla v11.4s, v1.8h, v7.8h\n" - ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" + ".inst 0x6e52ec2b // bfmmla v11.4s, v1.8h, v18.8h\n" + ".inst 0x6e51ec2f // bfmmla v15.4s, v1.8h, v17.8h\n" "prfm pldl1keep, [x26, #0x80]\n" "add x10, x10, #0x100\n" "21:" // Height 1: Multiply loop: Main loop skip @@ -314,26 +313,26 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "cmp x27, #0x4\n" "blt 23f\n" "22:" // Height 1: Multiply loop: Odd block loop - "ldr d1, [x26], #0x8\n" - "ldr q6, [x10, #0x0]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x70]\n" + "ldr d19, [x26], #0x8\n" + "ldr q18, [x10, #0x0]\n" + "trn1 v19.2d, v19.2d, v17.2d\n" + "ldr q17, [x10, #0x10]\n" + ".inst 0x6e52ee68 // bfmmla v8.4s, v19.8h, v18.8h\n" + "ldr q18, [x10, #0x20]\n" + ".inst 0x6e51ee6c // bfmmla v12.4s, v19.8h, v17.8h\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x6e52ee69 // bfmmla v9.4s, v19.8h, v18.8h\n" + "ldr q18, [x10, #0x40]\n" + ".inst 0x6e51ee6d // bfmmla v13.4s, v19.8h, v17.8h\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x6e52ee6a // bfmmla v10.4s, v19.8h, v18.8h\n" + "ldr q18, [x10, #0x60]\n" + ".inst 0x6e51ee6e // bfmmla v14.4s, v19.8h, v17.8h\n" + "ldr q17, [x10, #0x70]\n" "sub x27, x27, #0x4\n" "cmp x27, #0x4\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" + ".inst 0x6e52ee6b // bfmmla v11.4s, v19.8h, v18.8h\n" + ".inst 0x6e51ee6f // bfmmla v15.4s, v19.8h, v17.8h\n" "add x10, x10, #0x80\n" "bge 22b\n" "23:" // Height 1: Multiply loop: Skip odd blocks @@ -346,23 +345,23 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "24:" // Height 1: Multiply loop: Ragged operand read: partial_1_0 "ldr h1, [x26, #0x0]\n" "25:" // Height 1: Multiply loop: Ragged operand read: Done - "ldr q7, [x10, #0x0]\n" - "ldr q6, [x10, #0x10]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x70]\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" + "ldr q20, [x10, #0x0]\n" + "ldr q18, [x10, #0x10]\n" + "trn1 v19.2d, v1.2d, v17.2d\n" + ".inst 0x6e54ee68 // bfmmla v8.4s, v19.8h, v20.8h\n" + "ldr q17, [x10, #0x20]\n" + ".inst 0x6e52ee6c // bfmmla v12.4s, v19.8h, v18.8h\n" + "ldr q18, [x10, #0x30]\n" + ".inst 0x6e51ee69 // bfmmla v9.4s, v19.8h, v17.8h\n" + "ldr q17, [x10, #0x40]\n" + ".inst 0x6e52ee6d // bfmmla v13.4s, v19.8h, v18.8h\n" + "ldr q2, [x10, #0x50]\n" + ".inst 0x6e51ee6a // bfmmla v10.4s, v19.8h, v17.8h\n" + "ldr q18, [x10, #0x60]\n" + ".inst 0x6e42ee6e // bfmmla v14.4s, v19.8h, v2.8h\n" + "ldr q17, [x10, #0x70]\n" + ".inst 0x6e52ee6b // bfmmla v11.4s, v19.8h, v18.8h\n" + ".inst 0x6e51ee6f // bfmmla v15.4s, v19.8h, v17.8h\n" "add x10, x10, #0x80\n" "26:" // Height 1: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -376,17 +375,17 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "uzp1 v11.2d, v11.2d, v15.2d\n" "tbz %x[flags], #1, 27f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v18.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" + "ld1r { v17.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v18.4s\n" + "fmin v9.4s, v9.4s, v18.4s\n" + "fmin v10.4s, v10.4s, v18.4s\n" + "fmin v11.4s, v11.4s, v18.4s\n" + "fmax v8.4s, v8.4s, v17.4s\n" + "fmax v9.4s, v9.4s, v17.4s\n" + "fmax v10.4s, v10.4s, v17.4s\n" + "fmax v11.4s, v11.4s, v17.4s\n" "27:" // Height 1: No activation "cmp x11, #0x10\n" "bge 36f\n" @@ -577,12 +576,12 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "53:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 54f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 55f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -590,7 +589,7 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "b 55f\n" "54:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" "55:" // Height 2: input setup done "cmp x27, #0x8\n" "blt 58f\n" @@ -601,85 +600,85 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "ldr q6, [x10, #0x10]\n" "blt 57f\n" "56:" // Height 2: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x70]\n" + "trn1 v19.2d, v1.2d, v2.2d\n" + ".inst 0x6e47ee68 // bfmmla v8.4s, v19.8h, v7.8h\n" + "ldr q18, [x10, #0x20]\n" + ".inst 0x6e46ee6c // bfmmla v12.4s, v19.8h, v6.8h\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x6e52ee69 // bfmmla v9.4s, v19.8h, v18.8h\n" + "ldr q18, [x10, #0x40]\n" + ".inst 0x6e51ee6d // bfmmla v13.4s, v19.8h, v17.8h\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x6e52ee6a // bfmmla v10.4s, v19.8h, v18.8h\n" + "ldr q18, [x10, #0x60]\n" + ".inst 0x6e51ee6e // bfmmla v14.4s, v19.8h, v17.8h\n" + "ldr q17, [x10, #0x70]\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x80]\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x90]\n" - ".inst 0x6e47ec28 // bfmmla v8.4s, v1.8h, v7.8h\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x6e46ec2c // bfmmla v12.4s, v1.8h, v6.8h\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x6e47ec29 // bfmmla v9.4s, v1.8h, v7.8h\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x6e46ec2d // bfmmla v13.4s, v1.8h, v6.8h\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x6e47ec2a // bfmmla v10.4s, v1.8h, v7.8h\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - "ldr q6, [x10, #0xf0]\n" + ".inst 0x6e52ee6b // bfmmla v11.4s, v19.8h, v18.8h\n" + "ldr q18, [x10, #0x80]\n" + ".inst 0x6e51ee6f // bfmmla v15.4s, v19.8h, v17.8h\n" + "ldr q17, [x10, #0x90]\n" + ".inst 0x6e52ec28 // bfmmla v8.4s, v1.8h, v18.8h\n" + "ldr q18, [x10, #0xa0]\n" + ".inst 0x6e51ec2c // bfmmla v12.4s, v1.8h, v17.8h\n" + "ldr q17, [x10, #0xb0]\n" + ".inst 0x6e52ec29 // bfmmla v9.4s, v1.8h, v18.8h\n" + "ldr q18, [x10, #0xc0]\n" + ".inst 0x6e51ec2d // bfmmla v13.4s, v1.8h, v17.8h\n" + "ldr q17, [x10, #0xd0]\n" + ".inst 0x6e52ec2a // bfmmla v10.4s, v1.8h, v18.8h\n" + "ldr q18, [x10, #0xe0]\n" + ".inst 0x6e51ec2e // bfmmla v14.4s, v1.8h, v17.8h\n" + "ldr q17, [x10, #0xf0]\n" "sub x27, x27, #0x8\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" "ldr q2, [x25, #0x0]\n" "cmp x27, #0x10\n" - ".inst 0x6e47ec2b // bfmmla v11.4s, v1.8h, v7.8h\n" + ".inst 0x6e52ec2b // bfmmla v11.4s, v1.8h, v18.8h\n" "add x10, x10, #0x100\n" "ldr q7, [x10, #0x0]\n" - ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" + ".inst 0x6e51ec2f // bfmmla v15.4s, v1.8h, v17.8h\n" "ldr q1, [x26, #0x0]\n" "ldr q6, [x10, #0x10]\n" "prfm pldl1keep, [x26, #0x80]\n" "prfm pldl1keep, [x25, #0x80]\n" "bge 56b\n" "57:" // Height 2: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x70]\n" + "trn1 v19.2d, v1.2d, v2.2d\n" + ".inst 0x6e47ee68 // bfmmla v8.4s, v19.8h, v7.8h\n" + "ldr q18, [x10, #0x20]\n" + ".inst 0x6e46ee6c // bfmmla v12.4s, v19.8h, v6.8h\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x6e52ee69 // bfmmla v9.4s, v19.8h, v18.8h\n" + "ldr q18, [x10, #0x40]\n" + ".inst 0x6e51ee6d // bfmmla v13.4s, v19.8h, v17.8h\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x6e52ee6a // bfmmla v10.4s, v19.8h, v18.8h\n" + "ldr q18, [x10, #0x60]\n" + ".inst 0x6e51ee6e // bfmmla v14.4s, v19.8h, v17.8h\n" + "ldr q17, [x10, #0x70]\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x80]\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x90]\n" - ".inst 0x6e47ec28 // bfmmla v8.4s, v1.8h, v7.8h\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x6e46ec2c // bfmmla v12.4s, v1.8h, v6.8h\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x6e47ec29 // bfmmla v9.4s, v1.8h, v7.8h\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x6e46ec2d // bfmmla v13.4s, v1.8h, v6.8h\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x6e47ec2a // bfmmla v10.4s, v1.8h, v7.8h\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - "ldr q6, [x10, #0xf0]\n" + ".inst 0x6e52ee6b // bfmmla v11.4s, v19.8h, v18.8h\n" + "ldr q18, [x10, #0x80]\n" + ".inst 0x6e51ee6f // bfmmla v15.4s, v19.8h, v17.8h\n" + "ldr q17, [x10, #0x90]\n" + ".inst 0x6e52ec28 // bfmmla v8.4s, v1.8h, v18.8h\n" + "ldr q18, [x10, #0xa0]\n" + ".inst 0x6e51ec2c // bfmmla v12.4s, v1.8h, v17.8h\n" + "ldr q17, [x10, #0xb0]\n" + ".inst 0x6e52ec29 // bfmmla v9.4s, v1.8h, v18.8h\n" + "ldr q18, [x10, #0xc0]\n" + ".inst 0x6e51ec2d // bfmmla v13.4s, v1.8h, v17.8h\n" + "ldr q17, [x10, #0xd0]\n" + ".inst 0x6e52ec2a // bfmmla v10.4s, v1.8h, v18.8h\n" + "ldr q18, [x10, #0xe0]\n" + ".inst 0x6e51ec2e // bfmmla v14.4s, v1.8h, v17.8h\n" + "ldr q17, [x10, #0xf0]\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" - ".inst 0x6e47ec2b // bfmmla v11.4s, v1.8h, v7.8h\n" - ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" + ".inst 0x6e52ec2b // bfmmla v11.4s, v1.8h, v18.8h\n" + ".inst 0x6e51ec2f // bfmmla v15.4s, v1.8h, v17.8h\n" "sub x27, x27, #0x8\n" "prfm pldl1keep, [x26, #0x80]\n" "prfm pldl1keep, [x25, #0x80]\n" @@ -689,27 +688,27 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "cmp x27, #0x4\n" "blt 60f\n" "59:" // Height 2: Multiply loop: Odd block loop - "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" + "ldr d18, [x26], #0x8\n" + "ldr d17, [x25], #0x8\n" + "trn1 v19.2d, v18.2d, v17.2d\n" "sub x27, x27, #0x4\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" - "ldr q6, [x10, #0x20]\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - "ldr q6, [x10, #0x40]\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - "ldr q6, [x10, #0x60]\n" - "ldr q7, [x10, #0x70]\n" + "ldr q18, [x10, #0x0]\n" + "ldr q17, [x10, #0x10]\n" + ".inst 0x6e52ee68 // bfmmla v8.4s, v19.8h, v18.8h\n" + ".inst 0x6e51ee6c // bfmmla v12.4s, v19.8h, v17.8h\n" + "ldr q26, [x10, #0x20]\n" + "ldr q5, [x10, #0x30]\n" + ".inst 0x6e5aee69 // bfmmla v9.4s, v19.8h, v26.8h\n" + ".inst 0x6e45ee6d // bfmmla v13.4s, v19.8h, v5.8h\n" + "ldr q18, [x10, #0x40]\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x6e52ee6a // bfmmla v10.4s, v19.8h, v18.8h\n" + ".inst 0x6e51ee6e // bfmmla v14.4s, v19.8h, v17.8h\n" + "ldr q18, [x10, #0x60]\n" + "ldr q17, [x10, #0x70]\n" "cmp x27, #0x4\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" + ".inst 0x6e52ee6b // bfmmla v11.4s, v19.8h, v18.8h\n" + ".inst 0x6e51ee6f // bfmmla v15.4s, v19.8h, v17.8h\n" "add x10, x10, #0x80\n" "bge 59b\n" "60:" // Height 2: Multiply loop: Skip odd blocks @@ -725,23 +724,23 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "ldr h1, [x26, #0x0]\n" "ldr h2, [x25, #0x0]\n" "62:" // Height 2: Multiply loop: Ragged operand read: Done - "ldr q7, [x10, #0x0]\n" - "ldr q6, [x10, #0x10]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x70]\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" + "ldr q18, [x10, #0x0]\n" + "ldr q17, [x10, #0x10]\n" + "trn1 v19.2d, v1.2d, v2.2d\n" + ".inst 0x6e52ee68 // bfmmla v8.4s, v19.8h, v18.8h\n" + "ldr q18, [x10, #0x20]\n" + ".inst 0x6e51ee6c // bfmmla v12.4s, v19.8h, v17.8h\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x6e52ee69 // bfmmla v9.4s, v19.8h, v18.8h\n" + "ldr q30, [x10, #0x40]\n" + ".inst 0x6e51ee6d // bfmmla v13.4s, v19.8h, v17.8h\n" + "ldr q26, [x10, #0x50]\n" + ".inst 0x6e5eee6a // bfmmla v10.4s, v19.8h, v30.8h\n" + "ldr q18, [x10, #0x60]\n" + ".inst 0x6e5aee6e // bfmmla v14.4s, v19.8h, v26.8h\n" + "ldr q17, [x10, #0x70]\n" + ".inst 0x6e52ee6b // bfmmla v11.4s, v19.8h, v18.8h\n" + ".inst 0x6e51ee6f // bfmmla v15.4s, v19.8h, v17.8h\n" "add x10, x10, #0x80\n" "63:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -762,25 +761,25 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "uzp2 v11.2d, v11.2d, v15.2d\n" "tbz %x[flags], #1, 64f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v18.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v7.4s, v7.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmax v7.4s, v7.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" + "ld1r { v17.4s }, [x20]\n" + "fmin v7.4s, v7.4s, v18.4s\n" + "fmin v12.4s, v12.4s, v18.4s\n" + "fmin v13.4s, v13.4s, v18.4s\n" + "fmin v14.4s, v14.4s, v18.4s\n" + "fmin v8.4s, v8.4s, v18.4s\n" + "fmin v9.4s, v9.4s, v18.4s\n" + "fmin v10.4s, v10.4s, v18.4s\n" + "fmin v11.4s, v11.4s, v18.4s\n" + "fmax v7.4s, v7.4s, v17.4s\n" + "fmax v12.4s, v12.4s, v17.4s\n" + "fmax v13.4s, v13.4s, v17.4s\n" + "fmax v14.4s, v14.4s, v17.4s\n" + "fmax v8.4s, v8.4s, v17.4s\n" + "fmax v9.4s, v9.4s, v17.4s\n" + "fmax v10.4s, v10.4s, v17.4s\n" + "fmax v11.4s, v11.4s, v17.4s\n" "64:" // Height 2: No activation "cmp x11, #0x10\n" "bge 73f\n" @@ -1036,13 +1035,13 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "90:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 91f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 92f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -1051,8 +1050,8 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "b 92f\n" "91:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" "92:" // Height 3: input setup done "cmp x27, #0x8\n" "blt 95f\n" @@ -1064,167 +1063,167 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "ldr q6, [x10, #0x10]\n" "blt 94f\n" "93:" // Height 3: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v28.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x6e47ec50 // bfmmla v16.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" - "trn2 v3.2d, v3.2d, v4.2d\n" - ".inst 0x6e47ec51 // bfmmla v17.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" + ".inst 0x6e47ef88 // bfmmla v8.4s, v28.8h, v7.8h\n" + "trn1 v27.2d, v3.2d, v29.2d\n" + ".inst 0x6e47ef70 // bfmmla v16.4s, v27.8h, v7.8h\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x6e46ef8c // bfmmla v12.4s, v28.8h, v6.8h\n" + ".inst 0x6e46ef74 // bfmmla v20.4s, v27.8h, v6.8h\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e5aef89 // bfmmla v9.4s, v28.8h, v26.8h\n" + "trn2 v3.2d, v3.2d, v29.2d\n" + ".inst 0x6e5aef71 // bfmmla v17.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x6e59ef8d // bfmmla v13.4s, v28.8h, v25.8h\n" "sub x27, x27, #0x8\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" + ".inst 0x6e59ef75 // bfmmla v21.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x6e5aef8a // bfmmla v10.4s, v28.8h, v26.8h\n" "add x26, x26, #0x10\n" - ".inst 0x6e47ec52 // bfmmla v18.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" + ".inst 0x6e5aef72 // bfmmla v18.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x6e59ef8e // bfmmla v14.4s, v28.8h, v25.8h\n" "add x25, x25, #0x10\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x70]\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" + ".inst 0x6e59ef76 // bfmmla v22.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x70]\n" + ".inst 0x6e5aef8b // bfmmla v11.4s, v28.8h, v26.8h\n" "add x24, x24, #0x10\n" - ".inst 0x6e47ec53 // bfmmla v19.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x80]\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" + ".inst 0x6e5aef73 // bfmmla v19.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x80]\n" + ".inst 0x6e59ef8f // bfmmla v15.4s, v28.8h, v25.8h\n" "cmp x27, #0x10\n" - ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x90]\n" + ".inst 0x6e59ef77 // bfmmla v23.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x90]\n" "ldr q2, [x25, #0x0]\n" - ".inst 0x6e47ec28 // bfmmla v8.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec70 // bfmmla v16.4s, v3.8h, v7.8h\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x6e46ec2c // bfmmla v12.4s, v1.8h, v6.8h\n" + ".inst 0x6e5aec28 // bfmmla v8.4s, v1.8h, v26.8h\n" + ".inst 0x6e5aec70 // bfmmla v16.4s, v3.8h, v26.8h\n" + "ldr q26, [x10, #0xa0]\n" + ".inst 0x6e59ec2c // bfmmla v12.4s, v1.8h, v25.8h\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x6e46ec74 // bfmmla v20.4s, v3.8h, v6.8h\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x6e47ec29 // bfmmla v9.4s, v1.8h, v7.8h\n" + ".inst 0x6e59ec74 // bfmmla v20.4s, v3.8h, v25.8h\n" + "ldr q25, [x10, #0xb0]\n" + ".inst 0x6e5aec29 // bfmmla v9.4s, v1.8h, v26.8h\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x6e47ec71 // bfmmla v17.4s, v3.8h, v7.8h\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x6e46ec2d // bfmmla v13.4s, v1.8h, v6.8h\n" + ".inst 0x6e5aec71 // bfmmla v17.4s, v3.8h, v26.8h\n" + "ldr q26, [x10, #0xc0]\n" + ".inst 0x6e59ec2d // bfmmla v13.4s, v1.8h, v25.8h\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x6e46ec75 // bfmmla v21.4s, v3.8h, v6.8h\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x6e47ec2a // bfmmla v10.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec72 // bfmmla v18.4s, v3.8h, v7.8h\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec76 // bfmmla v22.4s, v3.8h, v6.8h\n" - "ldr q6, [x10, #0xf0]\n" + ".inst 0x6e59ec75 // bfmmla v21.4s, v3.8h, v25.8h\n" + "ldr q25, [x10, #0xd0]\n" + ".inst 0x6e5aec2a // bfmmla v10.4s, v1.8h, v26.8h\n" + ".inst 0x6e5aec72 // bfmmla v18.4s, v3.8h, v26.8h\n" + "ldr q26, [x10, #0xe0]\n" + ".inst 0x6e59ec2e // bfmmla v14.4s, v1.8h, v25.8h\n" + ".inst 0x6e59ec76 // bfmmla v22.4s, v3.8h, v25.8h\n" + "ldr q25, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x6e47ec2b // bfmmla v11.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec73 // bfmmla v19.4s, v3.8h, v7.8h\n" + ".inst 0x6e5aec2b // bfmmla v11.4s, v1.8h, v26.8h\n" + ".inst 0x6e5aec73 // bfmmla v19.4s, v3.8h, v26.8h\n" "ldr q7, [x10, #0x0]\n" - ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" + ".inst 0x6e59ec2f // bfmmla v15.4s, v1.8h, v25.8h\n" "ldr q1, [x26, #0x0]\n" - ".inst 0x6e46ec77 // bfmmla v23.4s, v3.8h, v6.8h\n" + ".inst 0x6e59ec77 // bfmmla v23.4s, v3.8h, v25.8h\n" "ldr q3, [x24, #0x0]\n" "ldr q6, [x10, #0x10]\n" "bge 93b\n" "94:" // Height 3: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v28.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x6e47ec50 // bfmmla v16.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" - "trn2 v3.2d, v3.2d, v4.2d\n" - ".inst 0x6e47ec51 // bfmmla v17.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" + ".inst 0x6e47ef88 // bfmmla v8.4s, v28.8h, v7.8h\n" + "trn1 v27.2d, v3.2d, v29.2d\n" + ".inst 0x6e47ef70 // bfmmla v16.4s, v27.8h, v7.8h\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x6e46ef8c // bfmmla v12.4s, v28.8h, v6.8h\n" + ".inst 0x6e46ef74 // bfmmla v20.4s, v27.8h, v6.8h\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e5aef89 // bfmmla v9.4s, v28.8h, v26.8h\n" + "trn2 v3.2d, v3.2d, v29.2d\n" + ".inst 0x6e5aef71 // bfmmla v17.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x6e59ef8d // bfmmla v13.4s, v28.8h, v25.8h\n" "add x26, x26, #0x10\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" + ".inst 0x6e59ef75 // bfmmla v21.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x6e5aef8a // bfmmla v10.4s, v28.8h, v26.8h\n" "add x25, x25, #0x10\n" - ".inst 0x6e47ec52 // bfmmla v18.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" + ".inst 0x6e5aef72 // bfmmla v18.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x6e59ef8e // bfmmla v14.4s, v28.8h, v25.8h\n" "add x24, x24, #0x10\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x70]\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" + ".inst 0x6e59ef76 // bfmmla v22.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x70]\n" + ".inst 0x6e5aef8b // bfmmla v11.4s, v28.8h, v26.8h\n" "sub x27, x27, #0x8\n" - ".inst 0x6e47ec53 // bfmmla v19.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x80]\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" + ".inst 0x6e5aef73 // bfmmla v19.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x80]\n" + ".inst 0x6e59ef8f // bfmmla v15.4s, v28.8h, v25.8h\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x90]\n" - ".inst 0x6e47ec28 // bfmmla v8.4s, v1.8h, v7.8h\n" + ".inst 0x6e59ef77 // bfmmla v23.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x90]\n" + ".inst 0x6e5aec28 // bfmmla v8.4s, v1.8h, v26.8h\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x6e47ec70 // bfmmla v16.4s, v3.8h, v7.8h\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x6e46ec2c // bfmmla v12.4s, v1.8h, v6.8h\n" + ".inst 0x6e5aec70 // bfmmla v16.4s, v3.8h, v26.8h\n" + "ldr q26, [x10, #0xa0]\n" + ".inst 0x6e59ec2c // bfmmla v12.4s, v1.8h, v25.8h\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x6e46ec74 // bfmmla v20.4s, v3.8h, v6.8h\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x6e47ec29 // bfmmla v9.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec71 // bfmmla v17.4s, v3.8h, v7.8h\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x6e46ec2d // bfmmla v13.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec75 // bfmmla v21.4s, v3.8h, v6.8h\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x6e47ec2a // bfmmla v10.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec72 // bfmmla v18.4s, v3.8h, v7.8h\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec76 // bfmmla v22.4s, v3.8h, v6.8h\n" - "ldr q6, [x10, #0xf0]\n" + ".inst 0x6e59ec74 // bfmmla v20.4s, v3.8h, v25.8h\n" + "ldr q25, [x10, #0xb0]\n" + ".inst 0x6e5aec29 // bfmmla v9.4s, v1.8h, v26.8h\n" + ".inst 0x6e5aec71 // bfmmla v17.4s, v3.8h, v26.8h\n" + "ldr q26, [x10, #0xc0]\n" + ".inst 0x6e59ec2d // bfmmla v13.4s, v1.8h, v25.8h\n" + ".inst 0x6e59ec75 // bfmmla v21.4s, v3.8h, v25.8h\n" + "ldr q25, [x10, #0xd0]\n" + ".inst 0x6e5aec2a // bfmmla v10.4s, v1.8h, v26.8h\n" + ".inst 0x6e5aec72 // bfmmla v18.4s, v3.8h, v26.8h\n" + "ldr q26, [x10, #0xe0]\n" + ".inst 0x6e59ec2e // bfmmla v14.4s, v1.8h, v25.8h\n" + ".inst 0x6e59ec76 // bfmmla v22.4s, v3.8h, v25.8h\n" + "ldr q25, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x6e47ec2b // bfmmla v11.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec73 // bfmmla v19.4s, v3.8h, v7.8h\n" - ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec77 // bfmmla v23.4s, v3.8h, v6.8h\n" + ".inst 0x6e5aec2b // bfmmla v11.4s, v1.8h, v26.8h\n" + ".inst 0x6e5aec73 // bfmmla v19.4s, v3.8h, v26.8h\n" + ".inst 0x6e59ec2f // bfmmla v15.4s, v1.8h, v25.8h\n" + ".inst 0x6e59ec77 // bfmmla v23.4s, v3.8h, v25.8h\n" "95:" // Height 3: Multiply loop: Main loop skip "cbz x27, 100f\n" "cmp x27, #0x4\n" "blt 97f\n" "96:" // Height 3: Multiply loop: Odd block loop - "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "ldr d3, [x24], #0x8\n" - "ldr q6, [x10, #0x0]\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6e46ec50 // bfmmla v16.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec54 // bfmmla v20.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" + "ldr d26, [x26], #0x8\n" + "ldr d25, [x25], #0x8\n" + "trn1 v28.2d, v26.2d, v25.2d\n" + "ldr d25, [x24], #0x8\n" + "ldr q26, [x10, #0x0]\n" + "trn1 v27.2d, v25.2d, v27.2d\n" + ".inst 0x6e5aef88 // bfmmla v8.4s, v28.8h, v26.8h\n" + "ldr q25, [x10, #0x10]\n" + ".inst 0x6e5aef70 // bfmmla v16.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x6e59ef8c // bfmmla v12.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef74 // bfmmla v20.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e5aef89 // bfmmla v9.4s, v28.8h, v26.8h\n" "sub x27, x27, #0x4\n" - ".inst 0x6e46ec51 // bfmmla v17.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" + ".inst 0x6e5aef71 // bfmmla v17.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x6e59ef8d // bfmmla v13.4s, v28.8h, v25.8h\n" "cmp x27, #0x4\n" - ".inst 0x6e47ec55 // bfmmla v21.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec52 // bfmmla v18.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec56 // bfmmla v22.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" + ".inst 0x6e59ef75 // bfmmla v21.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x6e5aef8a // bfmmla v10.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef72 // bfmmla v18.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x6e59ef8e // bfmmla v14.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef76 // bfmmla v22.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x70]\n" + ".inst 0x6e5aef8b // bfmmla v11.4s, v28.8h, v26.8h\n" "add x10, x10, #0x80\n" - ".inst 0x6e46ec53 // bfmmla v19.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" + ".inst 0x6e5aef73 // bfmmla v19.4s, v27.8h, v26.8h\n" + ".inst 0x6e59ef8f // bfmmla v15.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef77 // bfmmla v23.4s, v27.8h, v25.8h\n" "bge 96b\n" "97:" // Height 3: Multiply loop: Skip odd blocks "cbz x27, 100f\n" @@ -1242,33 +1241,33 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "ldr h2, [x25, #0x0]\n" "ldr h3, [x24, #0x0]\n" "99:" // Height 3: Multiply loop: Ragged operand read: Done - "ldr q7, [x10, #0x0]\n" - "ldr q6, [x10, #0x10]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec50 // bfmmla v16.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec51 // bfmmla v17.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec52 // bfmmla v18.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x70]\n" + "ldr q26, [x10, #0x0]\n" + "ldr q29, [x10, #0x10]\n" + "trn1 v28.2d, v1.2d, v2.2d\n" + "trn1 v27.2d, v3.2d, v25.2d\n" + ".inst 0x6e5aef88 // bfmmla v8.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef70 // bfmmla v16.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x6e5def8c // bfmmla v12.4s, v28.8h, v29.8h\n" + ".inst 0x6e5def74 // bfmmla v20.4s, v27.8h, v29.8h\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e5aef89 // bfmmla v9.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef71 // bfmmla v17.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x6e59ef8d // bfmmla v13.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef75 // bfmmla v21.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x6e5aef8a // bfmmla v10.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef72 // bfmmla v18.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x6e59ef8e // bfmmla v14.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef76 // bfmmla v22.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x70]\n" "add x10, x10, #0x80\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec53 // bfmmla v19.4s, v2.8h, v7.8h\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" + ".inst 0x6e5aef8b // bfmmla v11.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef73 // bfmmla v19.4s, v27.8h, v26.8h\n" + ".inst 0x6e59ef8f // bfmmla v15.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef77 // bfmmla v23.4s, v27.8h, v25.8h\n" "100:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1294,33 +1293,33 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "uzp1 v19.2d, v19.2d, v23.2d\n" "tbz %x[flags], #1, 101f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v26.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v7.4s, v7.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v16.4s, v16.4s, v1.4s\n" - "fmin v17.4s, v17.4s, v1.4s\n" - "fmin v18.4s, v18.4s, v1.4s\n" - "fmin v19.4s, v19.4s, v1.4s\n" - "fmax v7.4s, v7.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v16.4s, v16.4s, v0.4s\n" - "fmax v17.4s, v17.4s, v0.4s\n" - "fmax v18.4s, v18.4s, v0.4s\n" - "fmax v19.4s, v19.4s, v0.4s\n" + "ld1r { v25.4s }, [x20]\n" + "fmin v7.4s, v7.4s, v26.4s\n" + "fmin v12.4s, v12.4s, v26.4s\n" + "fmin v13.4s, v13.4s, v26.4s\n" + "fmin v14.4s, v14.4s, v26.4s\n" + "fmin v8.4s, v8.4s, v26.4s\n" + "fmin v9.4s, v9.4s, v26.4s\n" + "fmin v10.4s, v10.4s, v26.4s\n" + "fmin v11.4s, v11.4s, v26.4s\n" + "fmin v16.4s, v16.4s, v26.4s\n" + "fmin v17.4s, v17.4s, v26.4s\n" + "fmin v18.4s, v18.4s, v26.4s\n" + "fmin v19.4s, v19.4s, v26.4s\n" + "fmax v7.4s, v7.4s, v25.4s\n" + "fmax v12.4s, v12.4s, v25.4s\n" + "fmax v13.4s, v13.4s, v25.4s\n" + "fmax v14.4s, v14.4s, v25.4s\n" + "fmax v8.4s, v8.4s, v25.4s\n" + "fmax v9.4s, v9.4s, v25.4s\n" + "fmax v10.4s, v10.4s, v25.4s\n" + "fmax v11.4s, v11.4s, v25.4s\n" + "fmax v16.4s, v16.4s, v25.4s\n" + "fmax v17.4s, v17.4s, v25.4s\n" + "fmax v18.4s, v18.4s, v25.4s\n" + "fmax v19.4s, v19.4s, v25.4s\n" "101:" // Height 3: No activation "cmp x11, #0x10\n" "bge 110f\n" @@ -1617,14 +1616,14 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "127:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 128f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 129f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -1634,9 +1633,9 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "b 129f\n" "128:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" "129:" // Height 4: input setup done "cmp x27, #0x8\n" "blt 132f\n" @@ -1645,177 +1644,177 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "cmp x27, #0x10\n" "ldr q3, [x24, #0x0]\n" "ldr q4, [x23, #0x0]\n" - "ldr q7, [x10, #0x0]\n" - "ldr q6, [x10, #0x10]\n" - "blt 131f\n" - "130:" // Height 4: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" - "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - "sub x27, x27, #0x8\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x6e47ec50 // bfmmla v16.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" + "ldr q7, [x10, #0x0]\n" + "ldr q6, [x10, #0x10]\n" + "blt 131f\n" + "130:" // Height 4: Multiply loop: Main loop head + "trn1 v28.2d, v1.2d, v2.2d\n" + "trn2 v1.2d, v1.2d, v2.2d\n" + ".inst 0x6e47ef88 // bfmmla v8.4s, v28.8h, v7.8h\n" + "sub x27, x27, #0x8\n" + "trn1 v27.2d, v3.2d, v4.2d\n" + ".inst 0x6e47ef70 // bfmmla v16.4s, v27.8h, v7.8h\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x6e46ef8c // bfmmla v12.4s, v28.8h, v6.8h\n" + ".inst 0x6e46ef74 // bfmmla v20.4s, v27.8h, v6.8h\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e5aef89 // bfmmla v9.4s, v28.8h, v26.8h\n" "trn2 v3.2d, v3.2d, v4.2d\n" - ".inst 0x6e47ec51 // bfmmla v17.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" + ".inst 0x6e5aef71 // bfmmla v17.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x6e59ef8d // bfmmla v13.4s, v28.8h, v25.8h\n" "add x26, x26, #0x10\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" + ".inst 0x6e59ef75 // bfmmla v21.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x6e5aef8a // bfmmla v10.4s, v28.8h, v26.8h\n" "add x25, x25, #0x10\n" - ".inst 0x6e47ec52 // bfmmla v18.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" + ".inst 0x6e5aef72 // bfmmla v18.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x6e59ef8e // bfmmla v14.4s, v28.8h, v25.8h\n" "add x24, x24, #0x10\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x70]\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" + ".inst 0x6e59ef76 // bfmmla v22.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x70]\n" + ".inst 0x6e5aef8b // bfmmla v11.4s, v28.8h, v26.8h\n" "add x23, x23, #0x10\n" "ldr q4, [x23, #0x0]\n" - ".inst 0x6e47ec53 // bfmmla v19.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x80]\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x90]\n" + ".inst 0x6e5aef73 // bfmmla v19.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x80]\n" + ".inst 0x6e59ef8f // bfmmla v15.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef77 // bfmmla v23.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x90]\n" "ldr q2, [x25, #0x0]\n" - ".inst 0x6e47ec28 // bfmmla v8.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec70 // bfmmla v16.4s, v3.8h, v7.8h\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x6e46ec2c // bfmmla v12.4s, v1.8h, v6.8h\n" + ".inst 0x6e5aec28 // bfmmla v8.4s, v1.8h, v26.8h\n" + ".inst 0x6e5aec70 // bfmmla v16.4s, v3.8h, v26.8h\n" + "ldr q26, [x10, #0xa0]\n" + ".inst 0x6e59ec2c // bfmmla v12.4s, v1.8h, v25.8h\n" "cmp x27, #0x10\n" - ".inst 0x6e46ec74 // bfmmla v20.4s, v3.8h, v6.8h\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x6e47ec29 // bfmmla v9.4s, v1.8h, v7.8h\n" + ".inst 0x6e59ec74 // bfmmla v20.4s, v3.8h, v25.8h\n" + "ldr q25, [x10, #0xb0]\n" + ".inst 0x6e5aec29 // bfmmla v9.4s, v1.8h, v26.8h\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x6e47ec71 // bfmmla v17.4s, v3.8h, v7.8h\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x6e46ec2d // bfmmla v13.4s, v1.8h, v6.8h\n" + ".inst 0x6e5aec71 // bfmmla v17.4s, v3.8h, v26.8h\n" + "ldr q26, [x10, #0xc0]\n" + ".inst 0x6e59ec2d // bfmmla v13.4s, v1.8h, v25.8h\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x6e46ec75 // bfmmla v21.4s, v3.8h, v6.8h\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x6e47ec2a // bfmmla v10.4s, v1.8h, v7.8h\n" + ".inst 0x6e59ec75 // bfmmla v21.4s, v3.8h, v25.8h\n" + "ldr q25, [x10, #0xd0]\n" + ".inst 0x6e5aec2a // bfmmla v10.4s, v1.8h, v26.8h\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x6e47ec72 // bfmmla v18.4s, v3.8h, v7.8h\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" + ".inst 0x6e5aec72 // bfmmla v18.4s, v3.8h, v26.8h\n" + "ldr q26, [x10, #0xe0]\n" + ".inst 0x6e59ec2e // bfmmla v14.4s, v1.8h, v25.8h\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x6e46ec76 // bfmmla v22.4s, v3.8h, v6.8h\n" - "ldr q6, [x10, #0xf0]\n" + ".inst 0x6e59ec76 // bfmmla v22.4s, v3.8h, v25.8h\n" + "ldr q25, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x6e47ec2b // bfmmla v11.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec73 // bfmmla v19.4s, v3.8h, v7.8h\n" + ".inst 0x6e5aec2b // bfmmla v11.4s, v1.8h, v26.8h\n" + ".inst 0x6e5aec73 // bfmmla v19.4s, v3.8h, v26.8h\n" "ldr q7, [x10, #0x0]\n" - ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" + ".inst 0x6e59ec2f // bfmmla v15.4s, v1.8h, v25.8h\n" "ldr q1, [x26, #0x0]\n" - ".inst 0x6e46ec77 // bfmmla v23.4s, v3.8h, v6.8h\n" + ".inst 0x6e59ec77 // bfmmla v23.4s, v3.8h, v25.8h\n" "ldr q3, [x24, #0x0]\n" "ldr q6, [x10, #0x10]\n" "bge 130b\n" "131:" // Height 4: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v28.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" + ".inst 0x6e47ef88 // bfmmla v8.4s, v28.8h, v7.8h\n" "add x26, x26, #0x10\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x6e47ec50 // bfmmla v16.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" + "trn1 v27.2d, v3.2d, v4.2d\n" + ".inst 0x6e47ef70 // bfmmla v16.4s, v27.8h, v7.8h\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x6e46ef8c // bfmmla v12.4s, v28.8h, v6.8h\n" + ".inst 0x6e46ef74 // bfmmla v20.4s, v27.8h, v6.8h\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e5aef89 // bfmmla v9.4s, v28.8h, v26.8h\n" "trn2 v3.2d, v3.2d, v4.2d\n" - ".inst 0x6e47ec51 // bfmmla v17.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" + ".inst 0x6e5aef71 // bfmmla v17.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x6e59ef8d // bfmmla v13.4s, v28.8h, v25.8h\n" "add x25, x25, #0x10\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" + ".inst 0x6e59ef75 // bfmmla v21.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x6e5aef8a // bfmmla v10.4s, v28.8h, v26.8h\n" "add x24, x24, #0x10\n" - ".inst 0x6e47ec52 // bfmmla v18.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" + ".inst 0x6e5aef72 // bfmmla v18.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x6e59ef8e // bfmmla v14.4s, v28.8h, v25.8h\n" "add x23, x23, #0x10\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x70]\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" + ".inst 0x6e59ef76 // bfmmla v22.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x70]\n" + ".inst 0x6e5aef8b // bfmmla v11.4s, v28.8h, v26.8h\n" "sub x27, x27, #0x8\n" - ".inst 0x6e47ec53 // bfmmla v19.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x80]\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" + ".inst 0x6e5aef73 // bfmmla v19.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x80]\n" + ".inst 0x6e59ef8f // bfmmla v15.4s, v28.8h, v25.8h\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x90]\n" - ".inst 0x6e47ec28 // bfmmla v8.4s, v1.8h, v7.8h\n" + ".inst 0x6e59ef77 // bfmmla v23.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x90]\n" + ".inst 0x6e5aec28 // bfmmla v8.4s, v1.8h, v26.8h\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x6e47ec70 // bfmmla v16.4s, v3.8h, v7.8h\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x6e46ec2c // bfmmla v12.4s, v1.8h, v6.8h\n" + ".inst 0x6e5aec70 // bfmmla v16.4s, v3.8h, v26.8h\n" + "ldr q26, [x10, #0xa0]\n" + ".inst 0x6e59ec2c // bfmmla v12.4s, v1.8h, v25.8h\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x6e46ec74 // bfmmla v20.4s, v3.8h, v6.8h\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x6e47ec29 // bfmmla v9.4s, v1.8h, v7.8h\n" + ".inst 0x6e59ec74 // bfmmla v20.4s, v3.8h, v25.8h\n" + "ldr q25, [x10, #0xb0]\n" + ".inst 0x6e5aec29 // bfmmla v9.4s, v1.8h, v26.8h\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x6e47ec71 // bfmmla v17.4s, v3.8h, v7.8h\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x6e46ec2d // bfmmla v13.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec75 // bfmmla v21.4s, v3.8h, v6.8h\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x6e47ec2a // bfmmla v10.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec72 // bfmmla v18.4s, v3.8h, v7.8h\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec76 // bfmmla v22.4s, v3.8h, v6.8h\n" - "ldr q6, [x10, #0xf0]\n" + ".inst 0x6e5aec71 // bfmmla v17.4s, v3.8h, v26.8h\n" + "ldr q26, [x10, #0xc0]\n" + ".inst 0x6e59ec2d // bfmmla v13.4s, v1.8h, v25.8h\n" + ".inst 0x6e59ec75 // bfmmla v21.4s, v3.8h, v25.8h\n" + "ldr q25, [x10, #0xd0]\n" + ".inst 0x6e5aec2a // bfmmla v10.4s, v1.8h, v26.8h\n" + ".inst 0x6e5aec72 // bfmmla v18.4s, v3.8h, v26.8h\n" + "ldr q26, [x10, #0xe0]\n" + ".inst 0x6e59ec2e // bfmmla v14.4s, v1.8h, v25.8h\n" + ".inst 0x6e59ec76 // bfmmla v22.4s, v3.8h, v25.8h\n" + "ldr q25, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x6e47ec2b // bfmmla v11.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec73 // bfmmla v19.4s, v3.8h, v7.8h\n" - ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec77 // bfmmla v23.4s, v3.8h, v6.8h\n" + ".inst 0x6e5aec2b // bfmmla v11.4s, v1.8h, v26.8h\n" + ".inst 0x6e5aec73 // bfmmla v19.4s, v3.8h, v26.8h\n" + ".inst 0x6e59ec2f // bfmmla v15.4s, v1.8h, v25.8h\n" + ".inst 0x6e59ec77 // bfmmla v23.4s, v3.8h, v25.8h\n" "132:" // Height 4: Multiply loop: Main loop skip "cbz x27, 137f\n" "cmp x27, #0x4\n" "blt 134f\n" "133:" // Height 4: Multiply loop: Odd block loop - "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" + "ldr d26, [x26], #0x8\n" + "ldr d25, [x25], #0x8\n" + "trn1 v28.2d, v26.2d, v25.2d\n" "sub x27, x27, #0x4\n" - "ldr d3, [x24], #0x8\n" - "ldr d4, [x23], #0x8\n" - "trn1 v2.2d, v3.2d, v4.2d\n" + "ldr d26, [x24], #0x8\n" + "ldr d25, [x23], #0x8\n" + "trn1 v27.2d, v26.2d, v25.2d\n" "cmp x27, #0x4\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec50 // bfmmla v16.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec54 // bfmmla v20.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec51 // bfmmla v17.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec55 // bfmmla v21.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec52 // bfmmla v18.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec56 // bfmmla v22.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x70]\n" + "ldr q26, [x10, #0x0]\n" + "ldr q25, [x10, #0x10]\n" + ".inst 0x6e5aef88 // bfmmla v8.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef70 // bfmmla v16.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x6e59ef8c // bfmmla v12.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef74 // bfmmla v20.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e5aef89 // bfmmla v9.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef71 // bfmmla v17.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x6e59ef8d // bfmmla v13.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef75 // bfmmla v21.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x6e5aef8a // bfmmla v10.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef72 // bfmmla v18.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x6e59ef8e // bfmmla v14.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef76 // bfmmla v22.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x70]\n" "add x10, x10, #0x80\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec53 // bfmmla v19.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" + ".inst 0x6e5aef8b // bfmmla v11.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef73 // bfmmla v19.4s, v27.8h, v26.8h\n" + ".inst 0x6e59ef8f // bfmmla v15.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef77 // bfmmla v23.4s, v27.8h, v25.8h\n" "bge 133b\n" "134:" // Height 4: Multiply loop: Skip odd blocks "cbz x27, 137f\n" @@ -1836,33 +1835,33 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "ldr h3, [x24, #0x0]\n" "ldr h4, [x23, #0x0]\n" "136:" // Height 4: Multiply loop: Ragged operand read: Done - "ldr q7, [x10, #0x0]\n" - "ldr q6, [x10, #0x10]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec50 // bfmmla v16.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec51 // bfmmla v17.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec52 // bfmmla v18.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x70]\n" + "ldr q26, [x10, #0x0]\n" + "ldr q25, [x10, #0x10]\n" + "trn1 v28.2d, v1.2d, v2.2d\n" + "trn1 v27.2d, v3.2d, v4.2d\n" + ".inst 0x6e5aef88 // bfmmla v8.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef70 // bfmmla v16.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x6e59ef8c // bfmmla v12.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef74 // bfmmla v20.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e5aef89 // bfmmla v9.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef71 // bfmmla v17.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x6e59ef8d // bfmmla v13.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef75 // bfmmla v21.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x6e5aef8a // bfmmla v10.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef72 // bfmmla v18.4s, v27.8h, v26.8h\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x6e59ef8e // bfmmla v14.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef76 // bfmmla v22.4s, v27.8h, v25.8h\n" + "ldr q25, [x10, #0x70]\n" "add x10, x10, #0x80\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec53 // bfmmla v19.4s, v2.8h, v7.8h\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" + ".inst 0x6e5aef8b // bfmmla v11.4s, v28.8h, v26.8h\n" + ".inst 0x6e5aef73 // bfmmla v19.4s, v27.8h, v26.8h\n" + ".inst 0x6e59ef8f // bfmmla v15.4s, v28.8h, v25.8h\n" + ".inst 0x6e59ef77 // bfmmla v23.4s, v27.8h, v25.8h\n" "137:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1894,41 +1893,41 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "uzp2 v19.2d, v19.2d, v23.2d\n" "tbz %x[flags], #1, 138f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v26.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v7.4s, v7.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v15.4s, v15.4s, v1.4s\n" - "fmin v20.4s, v20.4s, v1.4s\n" - "fmin v21.4s, v21.4s, v1.4s\n" - "fmin v22.4s, v22.4s, v1.4s\n" - "fmin v16.4s, v16.4s, v1.4s\n" - "fmin v17.4s, v17.4s, v1.4s\n" - "fmin v18.4s, v18.4s, v1.4s\n" - "fmin v19.4s, v19.4s, v1.4s\n" - "fmax v7.4s, v7.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" - "fmax v20.4s, v20.4s, v0.4s\n" - "fmax v21.4s, v21.4s, v0.4s\n" - "fmax v22.4s, v22.4s, v0.4s\n" - "fmax v16.4s, v16.4s, v0.4s\n" - "fmax v17.4s, v17.4s, v0.4s\n" - "fmax v18.4s, v18.4s, v0.4s\n" - "fmax v19.4s, v19.4s, v0.4s\n" + "ld1r { v25.4s }, [x20]\n" + "fmin v7.4s, v7.4s, v26.4s\n" + "fmin v12.4s, v12.4s, v26.4s\n" + "fmin v13.4s, v13.4s, v26.4s\n" + "fmin v14.4s, v14.4s, v26.4s\n" + "fmin v8.4s, v8.4s, v26.4s\n" + "fmin v9.4s, v9.4s, v26.4s\n" + "fmin v10.4s, v10.4s, v26.4s\n" + "fmin v11.4s, v11.4s, v26.4s\n" + "fmin v15.4s, v15.4s, v26.4s\n" + "fmin v20.4s, v20.4s, v26.4s\n" + "fmin v21.4s, v21.4s, v26.4s\n" + "fmin v22.4s, v22.4s, v26.4s\n" + "fmin v16.4s, v16.4s, v26.4s\n" + "fmin v17.4s, v17.4s, v26.4s\n" + "fmin v18.4s, v18.4s, v26.4s\n" + "fmin v19.4s, v19.4s, v26.4s\n" + "fmax v7.4s, v7.4s, v25.4s\n" + "fmax v12.4s, v12.4s, v25.4s\n" + "fmax v13.4s, v13.4s, v25.4s\n" + "fmax v14.4s, v14.4s, v25.4s\n" + "fmax v8.4s, v8.4s, v25.4s\n" + "fmax v9.4s, v9.4s, v25.4s\n" + "fmax v10.4s, v10.4s, v25.4s\n" + "fmax v11.4s, v11.4s, v25.4s\n" + "fmax v15.4s, v15.4s, v25.4s\n" + "fmax v20.4s, v20.4s, v25.4s\n" + "fmax v21.4s, v21.4s, v25.4s\n" + "fmax v22.4s, v22.4s, v25.4s\n" + "fmax v16.4s, v16.4s, v25.4s\n" + "fmax v17.4s, v17.4s, v25.4s\n" + "fmax v18.4s, v18.4s, v25.4s\n" + "fmax v19.4s, v19.4s, v25.4s\n" "138:" // Height 4: No activation "cmp x11, #0x10\n" "bge 147f\n" @@ -2290,15 +2289,15 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "164:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 165f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 166f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -2309,10 +2308,10 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "b 166f\n" "165:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" - "add x22, x23, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" + "add x22, x23, x21, LSL #1\n" "166:" // Height 5: input setup done "cmp x27, #0x8\n" "blt 169f\n" @@ -2325,174 +2324,174 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "ldr q7, [x10, #0x0]\n" "blt 168f\n" "167:" // Height 5: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v6.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" + ".inst 0x6e47ecc8 // bfmmla v8.4s, v6.8h, v7.8h\n" "trn1 v2.2d, v3.2d, v4.2d\n" "trn2 v3.2d, v3.2d, v4.2d\n" ".inst 0x6e47ec50 // bfmmla v16.4s, v2.8h, v7.8h\n" "sub x27, x27, #0x8\n" - "trn1 v4.2d, v5.2d, v6.2d\n" - "trn2 v5.2d, v5.2d, v6.2d\n" - "ldr q6, [x10, #0x10]\n" + "trn1 v4.2d, v5.2d, v0.2d\n" + "trn2 v5.2d, v5.2d, v0.2d\n" + "ldr q0, [x10, #0x10]\n" ".inst 0x6e47ec98 // bfmmla v24.4s, v4.8h, v7.8h\n" "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" + ".inst 0x6e40eccc // bfmmla v12.4s, v6.8h, v0.8h\n" + ".inst 0x6e40ec54 // bfmmla v20.4s, v2.8h, v0.8h\n" "add x26, x26, #0x10\n" - ".inst 0x6e46ec9c // bfmmla v28.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" + ".inst 0x6e40ec9c // bfmmla v28.4s, v4.8h, v0.8h\n" + "ldr q0, [x10, #0x30]\n" + ".inst 0x6e47ecc9 // bfmmla v9.4s, v6.8h, v7.8h\n" "add x25, x25, #0x10\n" ".inst 0x6e47ec51 // bfmmla v17.4s, v2.8h, v7.8h\n" ".inst 0x6e47ec99 // bfmmla v25.4s, v4.8h, v7.8h\n" "ldr q7, [x10, #0x40]\n" "add x24, x24, #0x10\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" + ".inst 0x6e40eccd // bfmmla v13.4s, v6.8h, v0.8h\n" + ".inst 0x6e40ec55 // bfmmla v21.4s, v2.8h, v0.8h\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - ".inst 0x6e46ec9d // bfmmla v29.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" + ".inst 0x6e40ec9d // bfmmla v29.4s, v4.8h, v0.8h\n" + "ldr q0, [x10, #0x50]\n" + ".inst 0x6e47ecca // bfmmla v10.4s, v6.8h, v7.8h\n" "cmp x27, #0x10\n" ".inst 0x6e47ec52 // bfmmla v18.4s, v2.8h, v7.8h\n" ".inst 0x6e47ec9a // bfmmla v26.4s, v4.8h, v7.8h\n" "ldr q7, [x10, #0x60]\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" + ".inst 0x6e40ecce // bfmmla v14.4s, v6.8h, v0.8h\n" + ".inst 0x6e40ec56 // bfmmla v22.4s, v2.8h, v0.8h\n" "prfm pldl1keep, [x25, #0x80]\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x6e46ec9e // bfmmla v30.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x70]\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" + ".inst 0x6e40ec9e // bfmmla v30.4s, v4.8h, v0.8h\n" + "ldr q0, [x10, #0x70]\n" + ".inst 0x6e47eccb // bfmmla v11.4s, v6.8h, v7.8h\n" "prfm pldl1keep, [x23, #0x80]\n" ".inst 0x6e47ec53 // bfmmla v19.4s, v2.8h, v7.8h\n" ".inst 0x6e47ec9b // bfmmla v27.4s, v4.8h, v7.8h\n" "ldr q7, [x10, #0x80]\n" "prfm pldl1keep, [x22, #0x80]\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" + ".inst 0x6e40eccf // bfmmla v15.4s, v6.8h, v0.8h\n" + ".inst 0x6e40ec57 // bfmmla v23.4s, v2.8h, v0.8h\n" "ldr q2, [x25, #0x0]\n" - ".inst 0x6e46ec9f // bfmmla v31.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x90]\n" + ".inst 0x6e40ec9f // bfmmla v31.4s, v4.8h, v0.8h\n" + "ldr q0, [x10, #0x90]\n" "ldr q4, [x23, #0x0]\n" ".inst 0x6e47ec28 // bfmmla v8.4s, v1.8h, v7.8h\n" ".inst 0x6e47ec70 // bfmmla v16.4s, v3.8h, v7.8h\n" ".inst 0x6e47ecb8 // bfmmla v24.4s, v5.8h, v7.8h\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x6e46ec2c // bfmmla v12.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec74 // bfmmla v20.4s, v3.8h, v6.8h\n" - ".inst 0x6e46ecbc // bfmmla v28.4s, v5.8h, v6.8h\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x6e47ec29 // bfmmla v9.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec71 // bfmmla v17.4s, v3.8h, v7.8h\n" - ".inst 0x6e47ecb9 // bfmmla v25.4s, v5.8h, v7.8h\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x6e46ec2d // bfmmla v13.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec75 // bfmmla v21.4s, v3.8h, v6.8h\n" - ".inst 0x6e46ecbd // bfmmla v29.4s, v5.8h, v6.8h\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x6e47ec2a // bfmmla v10.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec72 // bfmmla v18.4s, v3.8h, v7.8h\n" - ".inst 0x6e47ecba // bfmmla v26.4s, v5.8h, v7.8h\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec76 // bfmmla v22.4s, v3.8h, v6.8h\n" - ".inst 0x6e46ecbe // bfmmla v30.4s, v5.8h, v6.8h\n" - "ldr q6, [x10, #0xf0]\n" + "ldr q6, [x10, #0xa0]\n" + ".inst 0x6e40ec2c // bfmmla v12.4s, v1.8h, v0.8h\n" + ".inst 0x6e40ec74 // bfmmla v20.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ecbc // bfmmla v28.4s, v5.8h, v0.8h\n" + "ldr q0, [x10, #0xb0]\n" + ".inst 0x6e46ec29 // bfmmla v9.4s, v1.8h, v6.8h\n" + ".inst 0x6e46ec71 // bfmmla v17.4s, v3.8h, v6.8h\n" + ".inst 0x6e46ecb9 // bfmmla v25.4s, v5.8h, v6.8h\n" + "ldr q6, [x10, #0xc0]\n" + ".inst 0x6e40ec2d // bfmmla v13.4s, v1.8h, v0.8h\n" + ".inst 0x6e40ec75 // bfmmla v21.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ecbd // bfmmla v29.4s, v5.8h, v0.8h\n" + "ldr q0, [x10, #0xd0]\n" + ".inst 0x6e46ec2a // bfmmla v10.4s, v1.8h, v6.8h\n" + ".inst 0x6e46ec72 // bfmmla v18.4s, v3.8h, v6.8h\n" + ".inst 0x6e46ecba // bfmmla v26.4s, v5.8h, v6.8h\n" + "ldr q6, [x10, #0xe0]\n" + ".inst 0x6e40ec2e // bfmmla v14.4s, v1.8h, v0.8h\n" + ".inst 0x6e40ec76 // bfmmla v22.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ecbe // bfmmla v30.4s, v5.8h, v0.8h\n" + "ldr q0, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x6e47ec2b // bfmmla v11.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec73 // bfmmla v19.4s, v3.8h, v7.8h\n" - ".inst 0x6e47ecbb // bfmmla v27.4s, v5.8h, v7.8h\n" + ".inst 0x6e46ec2b // bfmmla v11.4s, v1.8h, v6.8h\n" + ".inst 0x6e46ec73 // bfmmla v19.4s, v3.8h, v6.8h\n" + ".inst 0x6e46ecbb // bfmmla v27.4s, v5.8h, v6.8h\n" "ldr q7, [x10, #0x0]\n" - ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" + ".inst 0x6e40ec2f // bfmmla v15.4s, v1.8h, v0.8h\n" "ldr q1, [x26, #0x0]\n" - ".inst 0x6e46ec77 // bfmmla v23.4s, v3.8h, v6.8h\n" + ".inst 0x6e40ec77 // bfmmla v23.4s, v3.8h, v0.8h\n" "ldr q3, [x24, #0x0]\n" - ".inst 0x6e46ecbf // bfmmla v31.4s, v5.8h, v6.8h\n" + ".inst 0x6e40ecbf // bfmmla v31.4s, v5.8h, v0.8h\n" "ldr q5, [x22, #0x0]\n" "bge 167b\n" "168:" // Height 5: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v6.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" + ".inst 0x6e47ecc8 // bfmmla v8.4s, v6.8h, v7.8h\n" "trn1 v2.2d, v3.2d, v4.2d\n" "trn2 v3.2d, v3.2d, v4.2d\n" ".inst 0x6e47ec50 // bfmmla v16.4s, v2.8h, v7.8h\n" "add x26, x26, #0x10\n" - "trn1 v4.2d, v5.2d, v6.2d\n" - "trn2 v5.2d, v5.2d, v6.2d\n" - "ldr q6, [x10, #0x10]\n" + "trn1 v4.2d, v5.2d, v0.2d\n" + "trn2 v5.2d, v5.2d, v0.2d\n" + "ldr q0, [x10, #0x10]\n" ".inst 0x6e47ec98 // bfmmla v24.4s, v4.8h, v7.8h\n" "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" + ".inst 0x6e40eccc // bfmmla v12.4s, v6.8h, v0.8h\n" + ".inst 0x6e40ec54 // bfmmla v20.4s, v2.8h, v0.8h\n" "add x25, x25, #0x10\n" - ".inst 0x6e46ec9c // bfmmla v28.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" + ".inst 0x6e40ec9c // bfmmla v28.4s, v4.8h, v0.8h\n" + "ldr q0, [x10, #0x30]\n" + ".inst 0x6e47ecc9 // bfmmla v9.4s, v6.8h, v7.8h\n" "add x24, x24, #0x10\n" ".inst 0x6e47ec51 // bfmmla v17.4s, v2.8h, v7.8h\n" ".inst 0x6e47ec99 // bfmmla v25.4s, v4.8h, v7.8h\n" "ldr q7, [x10, #0x40]\n" "add x23, x23, #0x10\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" + ".inst 0x6e40eccd // bfmmla v13.4s, v6.8h, v0.8h\n" + ".inst 0x6e40ec55 // bfmmla v21.4s, v2.8h, v0.8h\n" "add x22, x22, #0x10\n" "sub x27, x27, #0x8\n" - ".inst 0x6e46ec9d // bfmmla v29.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" + ".inst 0x6e40ec9d // bfmmla v29.4s, v4.8h, v0.8h\n" + "ldr q0, [x10, #0x50]\n" + ".inst 0x6e47ecca // bfmmla v10.4s, v6.8h, v7.8h\n" "prfm pldl1keep, [x26, #0x80]\n" ".inst 0x6e47ec52 // bfmmla v18.4s, v2.8h, v7.8h\n" ".inst 0x6e47ec9a // bfmmla v26.4s, v4.8h, v7.8h\n" "ldr q7, [x10, #0x60]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" + ".inst 0x6e40ecce // bfmmla v14.4s, v6.8h, v0.8h\n" + ".inst 0x6e40ec56 // bfmmla v22.4s, v2.8h, v0.8h\n" "prfm pldl1keep, [x24, #0x80]\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x6e46ec9e // bfmmla v30.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x70]\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" + ".inst 0x6e40ec9e // bfmmla v30.4s, v4.8h, v0.8h\n" + "ldr q0, [x10, #0x70]\n" + ".inst 0x6e47eccb // bfmmla v11.4s, v6.8h, v7.8h\n" "prfm pldl1keep, [x22, #0x80]\n" ".inst 0x6e47ec53 // bfmmla v19.4s, v2.8h, v7.8h\n" ".inst 0x6e47ec9b // bfmmla v27.4s, v4.8h, v7.8h\n" "ldr q7, [x10, #0x80]\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9f // bfmmla v31.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x90]\n" + ".inst 0x6e40eccf // bfmmla v15.4s, v6.8h, v0.8h\n" + ".inst 0x6e40ec57 // bfmmla v23.4s, v2.8h, v0.8h\n" + ".inst 0x6e40ec9f // bfmmla v31.4s, v4.8h, v0.8h\n" + "ldr q2, [x10, #0x90]\n" ".inst 0x6e47ec28 // bfmmla v8.4s, v1.8h, v7.8h\n" ".inst 0x6e47ec70 // bfmmla v16.4s, v3.8h, v7.8h\n" ".inst 0x6e47ecb8 // bfmmla v24.4s, v5.8h, v7.8h\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x6e46ec2c // bfmmla v12.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec74 // bfmmla v20.4s, v3.8h, v6.8h\n" - ".inst 0x6e46ecbc // bfmmla v28.4s, v5.8h, v6.8h\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x6e47ec29 // bfmmla v9.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec71 // bfmmla v17.4s, v3.8h, v7.8h\n" - ".inst 0x6e47ecb9 // bfmmla v25.4s, v5.8h, v7.8h\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x6e46ec2d // bfmmla v13.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec75 // bfmmla v21.4s, v3.8h, v6.8h\n" - ".inst 0x6e46ecbd // bfmmla v29.4s, v5.8h, v6.8h\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x6e47ec2a // bfmmla v10.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec72 // bfmmla v18.4s, v3.8h, v7.8h\n" - ".inst 0x6e47ecba // bfmmla v26.4s, v5.8h, v7.8h\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec76 // bfmmla v22.4s, v3.8h, v6.8h\n" - ".inst 0x6e46ecbe // bfmmla v30.4s, v5.8h, v6.8h\n" + "ldr q0, [x10, #0xa0]\n" + ".inst 0x6e42ec2c // bfmmla v12.4s, v1.8h, v2.8h\n" + ".inst 0x6e42ec74 // bfmmla v20.4s, v3.8h, v2.8h\n" + ".inst 0x6e42ecbc // bfmmla v28.4s, v5.8h, v2.8h\n" + "ldr q2, [x10, #0xb0]\n" + ".inst 0x6e40ec29 // bfmmla v9.4s, v1.8h, v0.8h\n" + ".inst 0x6e40ec71 // bfmmla v17.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ecb9 // bfmmla v25.4s, v5.8h, v0.8h\n" + "ldr q0, [x10, #0xc0]\n" + ".inst 0x6e42ec2d // bfmmla v13.4s, v1.8h, v2.8h\n" + ".inst 0x6e42ec75 // bfmmla v21.4s, v3.8h, v2.8h\n" + ".inst 0x6e42ecbd // bfmmla v29.4s, v5.8h, v2.8h\n" + "ldr q2, [x10, #0xd0]\n" + ".inst 0x6e40ec2a // bfmmla v10.4s, v1.8h, v0.8h\n" + ".inst 0x6e40ec72 // bfmmla v18.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ecba // bfmmla v26.4s, v5.8h, v0.8h\n" + "ldr q0, [x10, #0xe0]\n" + ".inst 0x6e42ec2e // bfmmla v14.4s, v1.8h, v2.8h\n" + ".inst 0x6e42ec76 // bfmmla v22.4s, v3.8h, v2.8h\n" + ".inst 0x6e42ecbe // bfmmla v30.4s, v5.8h, v2.8h\n" "ldr q6, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x6e47ec2b // bfmmla v11.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec73 // bfmmla v19.4s, v3.8h, v7.8h\n" - ".inst 0x6e47ecbb // bfmmla v27.4s, v5.8h, v7.8h\n" + ".inst 0x6e40ec2b // bfmmla v11.4s, v1.8h, v0.8h\n" + ".inst 0x6e40ec73 // bfmmla v19.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ecbb // bfmmla v27.4s, v5.8h, v0.8h\n" ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" ".inst 0x6e46ec77 // bfmmla v23.4s, v3.8h, v6.8h\n" ".inst 0x6e46ecbf // bfmmla v31.4s, v5.8h, v6.8h\n" @@ -2502,48 +2501,48 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "blt 171f\n" "170:" // Height 5: Multiply loop: Odd block loop "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "ldr d3, [x24], #0x8\n" - "ldr d4, [x23], #0x8\n" - "trn1 v2.2d, v3.2d, v4.2d\n" + "ldr d0, [x25], #0x8\n" + "trn1 v4.2d, v1.2d, v0.2d\n" + "ldr d1, [x24], #0x8\n" + "ldr d0, [x23], #0x8\n" + "trn1 v3.2d, v1.2d, v0.2d\n" "sub x27, x27, #0x4\n" - "ldr d5, [x22], #0x8\n" - "ldr q6, [x10, #0x0]\n" - "trn1 v4.2d, v5.2d, v7.2d\n" - ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6e46ec50 // bfmmla v16.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec98 // bfmmla v24.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec54 // bfmmla v20.4s, v2.8h, v7.8h\n" + "ldr d0, [x22], #0x8\n" + "ldr q1, [x10, #0x0]\n" + "trn1 v2.2d, v0.2d, v2.2d\n" + ".inst 0x6e41ec88 // bfmmla v8.4s, v4.8h, v1.8h\n" + "ldr q0, [x10, #0x10]\n" + ".inst 0x6e41ec70 // bfmmla v16.4s, v3.8h, v1.8h\n" + ".inst 0x6e41ec58 // bfmmla v24.4s, v2.8h, v1.8h\n" + "ldr q1, [x10, #0x20]\n" + ".inst 0x6e40ec8c // bfmmla v12.4s, v4.8h, v0.8h\n" + ".inst 0x6e40ec74 // bfmmla v20.4s, v3.8h, v0.8h\n" "cmp x27, #0x4\n" - ".inst 0x6e47ec9c // bfmmla v28.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec51 // bfmmla v17.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec99 // bfmmla v25.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec55 // bfmmla v21.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9d // bfmmla v29.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec52 // bfmmla v18.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9a // bfmmla v26.4s, v4.8h, v6.8h\n" + ".inst 0x6e40ec5c // bfmmla v28.4s, v2.8h, v0.8h\n" + "ldr q0, [x10, #0x30]\n" + ".inst 0x6e41ec89 // bfmmla v9.4s, v4.8h, v1.8h\n" + ".inst 0x6e41ec71 // bfmmla v17.4s, v3.8h, v1.8h\n" + ".inst 0x6e41ec59 // bfmmla v25.4s, v2.8h, v1.8h\n" + "ldr q1, [x10, #0x40]\n" + ".inst 0x6e40ec8d // bfmmla v13.4s, v4.8h, v0.8h\n" + ".inst 0x6e40ec75 // bfmmla v21.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec5d // bfmmla v29.4s, v2.8h, v0.8h\n" + "ldr q0, [x10, #0x50]\n" + ".inst 0x6e41ec8a // bfmmla v10.4s, v4.8h, v1.8h\n" + ".inst 0x6e41ec72 // bfmmla v18.4s, v3.8h, v1.8h\n" + ".inst 0x6e41ec5a // bfmmla v26.4s, v2.8h, v1.8h\n" "ldr q6, [x10, #0x60]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec56 // bfmmla v22.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9e // bfmmla v30.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" + ".inst 0x6e40ec8e // bfmmla v14.4s, v4.8h, v0.8h\n" + ".inst 0x6e40ec76 // bfmmla v22.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec5e // bfmmla v30.4s, v2.8h, v0.8h\n" + "ldr q0, [x10, #0x70]\n" + ".inst 0x6e46ec8b // bfmmla v11.4s, v4.8h, v6.8h\n" "add x10, x10, #0x80\n" - ".inst 0x6e46ec53 // bfmmla v19.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9b // bfmmla v27.4s, v4.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9f // bfmmla v31.4s, v4.8h, v7.8h\n" + ".inst 0x6e46ec73 // bfmmla v19.4s, v3.8h, v6.8h\n" + ".inst 0x6e46ec5b // bfmmla v27.4s, v2.8h, v6.8h\n" + ".inst 0x6e40ec8f // bfmmla v15.4s, v4.8h, v0.8h\n" + ".inst 0x6e40ec77 // bfmmla v23.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec5f // bfmmla v31.4s, v2.8h, v0.8h\n" "bge 170b\n" "171:" // Height 5: Multiply loop: Skip odd blocks "cbz x27, 174f\n" @@ -2567,42 +2566,42 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "ldr h4, [x23, #0x0]\n" "ldr h5, [x22, #0x0]\n" "173:" // Height 5: Multiply loop: Ragged operand read: Done - "ldr q7, [x10, #0x0]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - "trn1 v4.2d, v5.2d, v6.2d\n" - "ldr q6, [x10, #0x10]\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec50 // bfmmla v16.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec98 // bfmmla v24.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9c // bfmmla v28.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec51 // bfmmla v17.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec99 // bfmmla v25.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9d // bfmmla v29.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec52 // bfmmla v18.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9a // bfmmla v26.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9e // bfmmla v30.4s, v4.8h, v6.8h\n" + "ldr q6, [x10, #0x0]\n" + "trn1 v7.2d, v1.2d, v2.2d\n" + "trn1 v3.2d, v3.2d, v4.2d\n" + "trn1 v2.2d, v5.2d, v0.2d\n" + "ldr q1, [x10, #0x10]\n" + ".inst 0x6e46ece8 // bfmmla v8.4s, v7.8h, v6.8h\n" + ".inst 0x6e46ec70 // bfmmla v16.4s, v3.8h, v6.8h\n" + ".inst 0x6e46ec58 // bfmmla v24.4s, v2.8h, v6.8h\n" + "ldr q0, [x10, #0x20]\n" + ".inst 0x6e41ecec // bfmmla v12.4s, v7.8h, v1.8h\n" + ".inst 0x6e41ec74 // bfmmla v20.4s, v3.8h, v1.8h\n" + ".inst 0x6e41ec5c // bfmmla v28.4s, v2.8h, v1.8h\n" + "ldr q1, [x10, #0x30]\n" + ".inst 0x6e40ece9 // bfmmla v9.4s, v7.8h, v0.8h\n" + ".inst 0x6e40ec71 // bfmmla v17.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec59 // bfmmla v25.4s, v2.8h, v0.8h\n" + "ldr q0, [x10, #0x40]\n" + ".inst 0x6e41eced // bfmmla v13.4s, v7.8h, v1.8h\n" + ".inst 0x6e41ec75 // bfmmla v21.4s, v3.8h, v1.8h\n" + ".inst 0x6e41ec5d // bfmmla v29.4s, v2.8h, v1.8h\n" + "ldr q1, [x10, #0x50]\n" + ".inst 0x6e40ecea // bfmmla v10.4s, v7.8h, v0.8h\n" + ".inst 0x6e40ec72 // bfmmla v18.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec5a // bfmmla v26.4s, v2.8h, v0.8h\n" + "ldr q0, [x10, #0x60]\n" + ".inst 0x6e41ecee // bfmmla v14.4s, v7.8h, v1.8h\n" + ".inst 0x6e41ec76 // bfmmla v22.4s, v3.8h, v1.8h\n" + ".inst 0x6e41ec5e // bfmmla v30.4s, v2.8h, v1.8h\n" "ldr q6, [x10, #0x70]\n" "add x10, x10, #0x80\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec53 // bfmmla v19.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9b // bfmmla v27.4s, v4.8h, v7.8h\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9f // bfmmla v31.4s, v4.8h, v6.8h\n" + ".inst 0x6e40eceb // bfmmla v11.4s, v7.8h, v0.8h\n" + ".inst 0x6e40ec73 // bfmmla v19.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec5b // bfmmla v27.4s, v2.8h, v0.8h\n" + ".inst 0x6e46ecef // bfmmla v15.4s, v7.8h, v6.8h\n" + ".inst 0x6e46ec77 // bfmmla v23.4s, v3.8h, v6.8h\n" + ".inst 0x6e46ec5f // bfmmla v31.4s, v2.8h, v6.8h\n" "174:" // Height 5: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -3088,16 +3087,16 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "201:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 202f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 203f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -3109,11 +3108,11 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "b 203f\n" "202:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" - "add x22, x23, x20, LSL #1\n" - "add x21, x22, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" + "add x22, x23, x21, LSL #1\n" + "add x21, x22, x21, LSL #1\n" "203:" // Height 6: input setup done "cmp x27, #0x8\n" "blt 206f\n" @@ -3180,42 +3179,42 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "ldr q2, [x25, #0x0]\n" "prfm pldl1keep, [x21, #0x80]\n" ".inst 0x6e46ec9f // bfmmla v31.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x90]\n" + "ldr q0, [x10, #0x90]\n" "ldr q4, [x23, #0x0]\n" ".inst 0x6e47ec28 // bfmmla v8.4s, v1.8h, v7.8h\n" ".inst 0x6e47ec70 // bfmmla v16.4s, v3.8h, v7.8h\n" ".inst 0x6e47ecb8 // bfmmla v24.4s, v5.8h, v7.8h\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x6e46ec2c // bfmmla v12.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec74 // bfmmla v20.4s, v3.8h, v6.8h\n" - ".inst 0x6e46ecbc // bfmmla v28.4s, v5.8h, v6.8h\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x6e47ec29 // bfmmla v9.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec71 // bfmmla v17.4s, v3.8h, v7.8h\n" - ".inst 0x6e47ecb9 // bfmmla v25.4s, v5.8h, v7.8h\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x6e46ec2d // bfmmla v13.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec75 // bfmmla v21.4s, v3.8h, v6.8h\n" - ".inst 0x6e46ecbd // bfmmla v29.4s, v5.8h, v6.8h\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x6e47ec2a // bfmmla v10.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec72 // bfmmla v18.4s, v3.8h, v7.8h\n" - ".inst 0x6e47ecba // bfmmla v26.4s, v5.8h, v7.8h\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec76 // bfmmla v22.4s, v3.8h, v6.8h\n" - ".inst 0x6e46ecbe // bfmmla v30.4s, v5.8h, v6.8h\n" - "ldr q6, [x10, #0xf0]\n" + "ldr q6, [x10, #0xa0]\n" + ".inst 0x6e40ec2c // bfmmla v12.4s, v1.8h, v0.8h\n" + ".inst 0x6e40ec74 // bfmmla v20.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ecbc // bfmmla v28.4s, v5.8h, v0.8h\n" + "ldr q0, [x10, #0xb0]\n" + ".inst 0x6e46ec29 // bfmmla v9.4s, v1.8h, v6.8h\n" + ".inst 0x6e46ec71 // bfmmla v17.4s, v3.8h, v6.8h\n" + ".inst 0x6e46ecb9 // bfmmla v25.4s, v5.8h, v6.8h\n" + "ldr q6, [x10, #0xc0]\n" + ".inst 0x6e40ec2d // bfmmla v13.4s, v1.8h, v0.8h\n" + ".inst 0x6e40ec75 // bfmmla v21.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ecbd // bfmmla v29.4s, v5.8h, v0.8h\n" + "ldr q0, [x10, #0xd0]\n" + ".inst 0x6e46ec2a // bfmmla v10.4s, v1.8h, v6.8h\n" + ".inst 0x6e46ec72 // bfmmla v18.4s, v3.8h, v6.8h\n" + ".inst 0x6e46ecba // bfmmla v26.4s, v5.8h, v6.8h\n" + "ldr q6, [x10, #0xe0]\n" + ".inst 0x6e40ec2e // bfmmla v14.4s, v1.8h, v0.8h\n" + ".inst 0x6e40ec76 // bfmmla v22.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ecbe // bfmmla v30.4s, v5.8h, v0.8h\n" + "ldr q0, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x6e47ec2b // bfmmla v11.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec73 // bfmmla v19.4s, v3.8h, v7.8h\n" - ".inst 0x6e47ecbb // bfmmla v27.4s, v5.8h, v7.8h\n" + ".inst 0x6e46ec2b // bfmmla v11.4s, v1.8h, v6.8h\n" + ".inst 0x6e46ec73 // bfmmla v19.4s, v3.8h, v6.8h\n" + ".inst 0x6e46ecbb // bfmmla v27.4s, v5.8h, v6.8h\n" "ldr q7, [x10, #0x0]\n" - ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" + ".inst 0x6e40ec2f // bfmmla v15.4s, v1.8h, v0.8h\n" "ldr q1, [x26, #0x0]\n" - ".inst 0x6e46ec77 // bfmmla v23.4s, v3.8h, v6.8h\n" + ".inst 0x6e40ec77 // bfmmla v23.4s, v3.8h, v0.8h\n" "ldr q3, [x24, #0x0]\n" - ".inst 0x6e46ecbf // bfmmla v31.4s, v5.8h, v6.8h\n" + ".inst 0x6e40ecbf // bfmmla v31.4s, v5.8h, v0.8h\n" "ldr q5, [x22, #0x0]\n" "ldr q6, [x21, #0x0]\n" "bge 204b\n" @@ -3271,35 +3270,35 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" ".inst 0x6e46ec9f // bfmmla v31.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x90]\n" + "ldr q2, [x10, #0x90]\n" ".inst 0x6e47ec28 // bfmmla v8.4s, v1.8h, v7.8h\n" ".inst 0x6e47ec70 // bfmmla v16.4s, v3.8h, v7.8h\n" ".inst 0x6e47ecb8 // bfmmla v24.4s, v5.8h, v7.8h\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x6e46ec2c // bfmmla v12.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec74 // bfmmla v20.4s, v3.8h, v6.8h\n" - ".inst 0x6e46ecbc // bfmmla v28.4s, v5.8h, v6.8h\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x6e47ec29 // bfmmla v9.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec71 // bfmmla v17.4s, v3.8h, v7.8h\n" - ".inst 0x6e47ecb9 // bfmmla v25.4s, v5.8h, v7.8h\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x6e46ec2d // bfmmla v13.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec75 // bfmmla v21.4s, v3.8h, v6.8h\n" - ".inst 0x6e46ecbd // bfmmla v29.4s, v5.8h, v6.8h\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x6e47ec2a // bfmmla v10.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec72 // bfmmla v18.4s, v3.8h, v7.8h\n" - ".inst 0x6e47ecba // bfmmla v26.4s, v5.8h, v7.8h\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - ".inst 0x6e46ec76 // bfmmla v22.4s, v3.8h, v6.8h\n" - ".inst 0x6e46ecbe // bfmmla v30.4s, v5.8h, v6.8h\n" + "ldr q0, [x10, #0xa0]\n" + ".inst 0x6e42ec2c // bfmmla v12.4s, v1.8h, v2.8h\n" + ".inst 0x6e42ec74 // bfmmla v20.4s, v3.8h, v2.8h\n" + ".inst 0x6e42ecbc // bfmmla v28.4s, v5.8h, v2.8h\n" + "ldr q2, [x10, #0xb0]\n" + ".inst 0x6e40ec29 // bfmmla v9.4s, v1.8h, v0.8h\n" + ".inst 0x6e40ec71 // bfmmla v17.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ecb9 // bfmmla v25.4s, v5.8h, v0.8h\n" + "ldr q0, [x10, #0xc0]\n" + ".inst 0x6e42ec2d // bfmmla v13.4s, v1.8h, v2.8h\n" + ".inst 0x6e42ec75 // bfmmla v21.4s, v3.8h, v2.8h\n" + ".inst 0x6e42ecbd // bfmmla v29.4s, v5.8h, v2.8h\n" + "ldr q2, [x10, #0xd0]\n" + ".inst 0x6e40ec2a // bfmmla v10.4s, v1.8h, v0.8h\n" + ".inst 0x6e40ec72 // bfmmla v18.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ecba // bfmmla v26.4s, v5.8h, v0.8h\n" + "ldr q0, [x10, #0xe0]\n" + ".inst 0x6e42ec2e // bfmmla v14.4s, v1.8h, v2.8h\n" + ".inst 0x6e42ec76 // bfmmla v22.4s, v3.8h, v2.8h\n" + ".inst 0x6e42ecbe // bfmmla v30.4s, v5.8h, v2.8h\n" "ldr q6, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x6e47ec2b // bfmmla v11.4s, v1.8h, v7.8h\n" - ".inst 0x6e47ec73 // bfmmla v19.4s, v3.8h, v7.8h\n" - ".inst 0x6e47ecbb // bfmmla v27.4s, v5.8h, v7.8h\n" + ".inst 0x6e40ec2b // bfmmla v11.4s, v1.8h, v0.8h\n" + ".inst 0x6e40ec73 // bfmmla v19.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ecbb // bfmmla v27.4s, v5.8h, v0.8h\n" ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" ".inst 0x6e46ec77 // bfmmla v23.4s, v3.8h, v6.8h\n" ".inst 0x6e46ecbf // bfmmla v31.4s, v5.8h, v6.8h\n" @@ -3309,49 +3308,49 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "blt 208f\n" "207:" // Height 6: Multiply loop: Odd block loop "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" + "ldr d0, [x25], #0x8\n" + "trn1 v4.2d, v1.2d, v0.2d\n" "sub x27, x27, #0x4\n" - "ldr d3, [x24], #0x8\n" - "ldr d4, [x23], #0x8\n" - "trn1 v2.2d, v3.2d, v4.2d\n" + "ldr d1, [x24], #0x8\n" + "ldr d0, [x23], #0x8\n" + "trn1 v3.2d, v1.2d, v0.2d\n" "cmp x27, #0x4\n" - "ldr d5, [x22], #0x8\n" - "ldr d7, [x21], #0x8\n" - "trn1 v4.2d, v5.2d, v7.2d\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec50 // bfmmla v16.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec98 // bfmmla v24.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec54 // bfmmla v20.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9c // bfmmla v28.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec51 // bfmmla v17.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec99 // bfmmla v25.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec55 // bfmmla v21.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9d // bfmmla v29.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec52 // bfmmla v18.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9a // bfmmla v26.4s, v4.8h, v6.8h\n" + "ldr d1, [x22], #0x8\n" + "ldr d0, [x21], #0x8\n" + "trn1 v2.2d, v1.2d, v0.2d\n" + "ldr q1, [x10, #0x0]\n" + "ldr q0, [x10, #0x10]\n" + ".inst 0x6e41ec88 // bfmmla v8.4s, v4.8h, v1.8h\n" + ".inst 0x6e41ec70 // bfmmla v16.4s, v3.8h, v1.8h\n" + ".inst 0x6e41ec58 // bfmmla v24.4s, v2.8h, v1.8h\n" + "ldr q1, [x10, #0x20]\n" + ".inst 0x6e40ec8c // bfmmla v12.4s, v4.8h, v0.8h\n" + ".inst 0x6e40ec74 // bfmmla v20.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec5c // bfmmla v28.4s, v2.8h, v0.8h\n" + "ldr q0, [x10, #0x30]\n" + ".inst 0x6e41ec89 // bfmmla v9.4s, v4.8h, v1.8h\n" + ".inst 0x6e41ec71 // bfmmla v17.4s, v3.8h, v1.8h\n" + ".inst 0x6e41ec59 // bfmmla v25.4s, v2.8h, v1.8h\n" + "ldr q1, [x10, #0x40]\n" + ".inst 0x6e40ec8d // bfmmla v13.4s, v4.8h, v0.8h\n" + ".inst 0x6e40ec75 // bfmmla v21.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec5d // bfmmla v29.4s, v2.8h, v0.8h\n" + "ldr q0, [x10, #0x50]\n" + ".inst 0x6e41ec8a // bfmmla v10.4s, v4.8h, v1.8h\n" + ".inst 0x6e41ec72 // bfmmla v18.4s, v3.8h, v1.8h\n" + ".inst 0x6e41ec5a // bfmmla v26.4s, v2.8h, v1.8h\n" "ldr q6, [x10, #0x60]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec56 // bfmmla v22.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9e // bfmmla v30.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x70]\n" + ".inst 0x6e40ec8e // bfmmla v14.4s, v4.8h, v0.8h\n" + ".inst 0x6e40ec76 // bfmmla v22.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec5e // bfmmla v30.4s, v2.8h, v0.8h\n" + "ldr q0, [x10, #0x70]\n" "add x10, x10, #0x80\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec53 // bfmmla v19.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9b // bfmmla v27.4s, v4.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9f // bfmmla v31.4s, v4.8h, v7.8h\n" + ".inst 0x6e46ec8b // bfmmla v11.4s, v4.8h, v6.8h\n" + ".inst 0x6e46ec73 // bfmmla v19.4s, v3.8h, v6.8h\n" + ".inst 0x6e46ec5b // bfmmla v27.4s, v2.8h, v6.8h\n" + ".inst 0x6e40ec8f // bfmmla v15.4s, v4.8h, v0.8h\n" + ".inst 0x6e40ec77 // bfmmla v23.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec5f // bfmmla v31.4s, v2.8h, v0.8h\n" "bge 207b\n" "208:" // Height 6: Multiply loop: Skip odd blocks "cbz x27, 211f\n" @@ -3378,42 +3377,42 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "ldr h5, [x22, #0x0]\n" "ldr h6, [x21, #0x0]\n" "210:" // Height 6: Multiply loop: Ragged operand read: Done - "ldr q7, [x10, #0x0]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" - "trn1 v4.2d, v5.2d, v6.2d\n" - "ldr q6, [x10, #0x10]\n" - ".inst 0x6e47ec50 // bfmmla v16.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec98 // bfmmla v24.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9c // bfmmla v28.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec51 // bfmmla v17.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec99 // bfmmla v25.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9d // bfmmla v29.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec52 // bfmmla v18.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9a // bfmmla v26.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e46ec0e // bfmmla v14.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9e // bfmmla v30.4s, v4.8h, v6.8h\n" + "ldr q0, [x10, #0x0]\n" + "trn1 v7.2d, v1.2d, v2.2d\n" + "trn1 v3.2d, v3.2d, v4.2d\n" + ".inst 0x6e40ece8 // bfmmla v8.4s, v7.8h, v0.8h\n" + "trn1 v2.2d, v5.2d, v6.2d\n" + "ldr q1, [x10, #0x10]\n" + ".inst 0x6e40ec70 // bfmmla v16.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec58 // bfmmla v24.4s, v2.8h, v0.8h\n" + "ldr q0, [x10, #0x20]\n" + ".inst 0x6e41ecec // bfmmla v12.4s, v7.8h, v1.8h\n" + ".inst 0x6e41ec74 // bfmmla v20.4s, v3.8h, v1.8h\n" + ".inst 0x6e41ec5c // bfmmla v28.4s, v2.8h, v1.8h\n" + "ldr q1, [x10, #0x30]\n" + ".inst 0x6e40ece9 // bfmmla v9.4s, v7.8h, v0.8h\n" + ".inst 0x6e40ec71 // bfmmla v17.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec59 // bfmmla v25.4s, v2.8h, v0.8h\n" + "ldr q0, [x10, #0x40]\n" + ".inst 0x6e41eced // bfmmla v13.4s, v7.8h, v1.8h\n" + ".inst 0x6e41ec75 // bfmmla v21.4s, v3.8h, v1.8h\n" + ".inst 0x6e41ec5d // bfmmla v29.4s, v2.8h, v1.8h\n" + "ldr q1, [x10, #0x50]\n" + ".inst 0x6e40ecea // bfmmla v10.4s, v7.8h, v0.8h\n" + ".inst 0x6e40ec72 // bfmmla v18.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec5a // bfmmla v26.4s, v2.8h, v0.8h\n" + "ldr q0, [x10, #0x60]\n" + ".inst 0x6e41ecee // bfmmla v14.4s, v7.8h, v1.8h\n" + ".inst 0x6e41ec76 // bfmmla v22.4s, v3.8h, v1.8h\n" + ".inst 0x6e41ec5e // bfmmla v30.4s, v2.8h, v1.8h\n" "ldr q6, [x10, #0x70]\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" + ".inst 0x6e40eceb // bfmmla v11.4s, v7.8h, v0.8h\n" "add x10, x10, #0x80\n" - ".inst 0x6e47ec53 // bfmmla v19.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9b // bfmmla v27.4s, v4.8h, v7.8h\n" - ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9f // bfmmla v31.4s, v4.8h, v6.8h\n" + ".inst 0x6e40ec73 // bfmmla v19.4s, v3.8h, v0.8h\n" + ".inst 0x6e40ec5b // bfmmla v27.4s, v2.8h, v0.8h\n" + ".inst 0x6e46ecef // bfmmla v15.4s, v7.8h, v6.8h\n" + ".inst 0x6e46ec77 // bfmmla v23.4s, v3.8h, v6.8h\n" + ".inst 0x6e46ec5f // bfmmla v31.4s, v2.8h, v6.8h\n" "211:" // Height 6: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -3678,7 +3677,6 @@ void a64_hybrid_bf16fp32_mmla_6x16 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "224:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32.hpp index 8ce3d1b9951106a366e8f04ec2281c04d5c53e1e..8b80c25bebe30ea9ae71cfe74de76984b749a1cc 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef __aarch64__ + #include "../std_transforms_fixed.hpp" #include "../performance_parameters.hpp" @@ -79,12 +79,12 @@ public: switch (ci->get_cpu_model()) { case CPUModel::A55r1: return { 6.94 }; + default: + return { 14.53 }; case CPUModel::A510: return { 8.94 }; case CPUModel::V1: return { 29.26 }; - default: - return { 14.53 }; } } @@ -108,5 +108,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32/a55.cpp index 19636548a0a1eb9f4a52d7d3e14921d1e10108be..b049ed45f90750b16291f2913eb899beedc31b6b 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32/a55.cpp @@ -244,11 +244,11 @@ void a64_hybrid_fp16_mla_6x32_a55 ( "23:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w14, [x20, x15, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 24f\n" - "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x13, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x15, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x13, [x20, #0x0]\n" "cbnz x15, 25f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x13, x13, x20, LSL #1\n" @@ -265,222 +265,222 @@ void a64_hybrid_fp16_mla_6x32_a55 ( "blt 27f\n" "26:" // Height 1: Multiply loop: Main loop head "fmla v8.8h, v6.8h, v0.h[0]\n" - "ldr d6, [x17, #0x20]\n" - "ldr x12, [x17, #0x28]\n" + "ldr d17, [x17, #0x20]\n" + "ldr x20, [x17, #0x28]\n" "fmla v9.8h, v7.8h, v0.h[0]\n" - "ldr d7, [x17, #0x30]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x38]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "ldr d6, [x17, #0x40]\n" - "ldr x12, [x17, #0x48]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "ldr d7, [x17, #0x50]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x58]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "ldr d6, [x17, #0x60]\n" - "ldr x12, [x17, #0x68]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "ldr d7, [x17, #0x70]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x78]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "ldr d6, [x17, #0x80]\n" - "ldr x12, [x17, #0x88]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "ldr d7, [x17, #0x90]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x98]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "ldr d6, [x17, #0xa0]\n" - "ldr x12, [x17, #0xa8]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "ldr d7, [x17, #0xb0]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0xb8]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "ldr d6, [x17, #0xc0]\n" - "ldr x12, [x17, #0xc8]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "ldr d7, [x17, #0xd0]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0xd8]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "ldr d6, [x17, #0xe0]\n" - "ldr x12, [x17, #0xe8]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "ldr d7, [x17, #0xf0]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0xf8]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "ldr d6, [x17, #0x100]\n" - "ldr x12, [x17, #0x108]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "ldr d7, [x17, #0x110]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x118]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "ldr d6, [x17, #0x120]\n" - "ldr x12, [x17, #0x128]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "ldr d7, [x17, #0x130]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x138]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "ldr d6, [x17, #0x140]\n" - "ldr x12, [x17, #0x148]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "ldr d7, [x17, #0x150]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x158]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "ldr d6, [x17, #0x160]\n" - "ldr x12, [x17, #0x168]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "ldr d7, [x17, #0x170]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x178]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "ldr d6, [x17, #0x180]\n" - "ldr x12, [x17, #0x188]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "ldr d7, [x17, #0x190]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x198]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "ldr d6, [x17, #0x1a0]\n" - "ldr x12, [x17, #0x1a8]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "ldr d7, [x17, #0x1b0]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x1b8]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "ldr d6, [x17, #0x1c0]\n" - "ldr x12, [x17, #0x1c8]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "ldr d7, [x17, #0x1d0]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x1d8]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "ldr d6, [x17, #0x1e0]\n" - "ldr x12, [x17, #0x1e8]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "ldr d7, [x17, #0x1f0]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x1f8]\n" - "mov v7.d[1], x11\n" + "ldr d16, [x17, #0x30]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0x38]\n" + "mov v16.d[1], x20\n" + "fmla v10.8h, v17.8h, v0.h[0]\n" + "ldr d17, [x17, #0x40]\n" + "ldr x20, [x17, #0x48]\n" + "fmla v11.8h, v16.8h, v0.h[0]\n" + "ldr d16, [x17, #0x50]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0x58]\n" + "mov v16.d[1], x20\n" + "fmla v8.8h, v17.8h, v0.h[1]\n" + "ldr d17, [x17, #0x60]\n" + "ldr x20, [x17, #0x68]\n" + "fmla v9.8h, v16.8h, v0.h[1]\n" + "ldr d16, [x17, #0x70]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0x78]\n" + "mov v16.d[1], x20\n" + "fmla v10.8h, v17.8h, v0.h[1]\n" + "ldr d17, [x17, #0x80]\n" + "ldr x20, [x17, #0x88]\n" + "fmla v11.8h, v16.8h, v0.h[1]\n" + "ldr d16, [x17, #0x90]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0x98]\n" + "mov v16.d[1], x20\n" + "fmla v8.8h, v17.8h, v0.h[2]\n" + "ldr d17, [x17, #0xa0]\n" + "ldr x20, [x17, #0xa8]\n" + "fmla v9.8h, v16.8h, v0.h[2]\n" + "ldr d16, [x17, #0xb0]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0xb8]\n" + "mov v16.d[1], x20\n" + "fmla v10.8h, v17.8h, v0.h[2]\n" + "ldr d17, [x17, #0xc0]\n" + "ldr x20, [x17, #0xc8]\n" + "fmla v11.8h, v16.8h, v0.h[2]\n" + "ldr d16, [x17, #0xd0]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0xd8]\n" + "mov v16.d[1], x20\n" + "fmla v8.8h, v17.8h, v0.h[3]\n" + "ldr d17, [x17, #0xe0]\n" + "ldr x20, [x17, #0xe8]\n" + "fmla v9.8h, v16.8h, v0.h[3]\n" + "ldr d16, [x17, #0xf0]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0xf8]\n" + "mov v16.d[1], x20\n" + "fmla v10.8h, v17.8h, v0.h[3]\n" + "ldr d17, [x17, #0x100]\n" + "ldr x20, [x17, #0x108]\n" + "fmla v11.8h, v16.8h, v0.h[3]\n" + "ldr d16, [x17, #0x110]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0x118]\n" + "mov v16.d[1], x20\n" + "fmla v8.8h, v17.8h, v0.h[4]\n" + "ldr d17, [x17, #0x120]\n" + "ldr x20, [x17, #0x128]\n" + "fmla v9.8h, v16.8h, v0.h[4]\n" + "ldr d16, [x17, #0x130]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0x138]\n" + "mov v16.d[1], x20\n" + "fmla v10.8h, v17.8h, v0.h[4]\n" + "ldr d17, [x17, #0x140]\n" + "ldr x20, [x17, #0x148]\n" + "fmla v11.8h, v16.8h, v0.h[4]\n" + "ldr d16, [x17, #0x150]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0x158]\n" + "mov v16.d[1], x20\n" + "fmla v8.8h, v17.8h, v0.h[5]\n" + "ldr d17, [x17, #0x160]\n" + "ldr x20, [x17, #0x168]\n" + "fmla v9.8h, v16.8h, v0.h[5]\n" + "ldr d16, [x17, #0x170]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0x178]\n" + "mov v16.d[1], x20\n" + "fmla v10.8h, v17.8h, v0.h[5]\n" + "ldr d17, [x17, #0x180]\n" + "ldr x20, [x17, #0x188]\n" + "fmla v11.8h, v16.8h, v0.h[5]\n" + "ldr d16, [x17, #0x190]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0x198]\n" + "mov v16.d[1], x20\n" + "fmla v8.8h, v17.8h, v0.h[6]\n" + "ldr d17, [x17, #0x1a0]\n" + "ldr x20, [x17, #0x1a8]\n" + "fmla v9.8h, v16.8h, v0.h[6]\n" + "ldr d16, [x17, #0x1b0]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0x1b8]\n" + "mov v16.d[1], x20\n" + "fmla v10.8h, v17.8h, v0.h[6]\n" + "ldr d17, [x17, #0x1c0]\n" + "ldr x20, [x17, #0x1c8]\n" + "fmla v11.8h, v16.8h, v0.h[6]\n" + "ldr d16, [x17, #0x1d0]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0x1d8]\n" + "mov v16.d[1], x20\n" + "fmla v8.8h, v17.8h, v0.h[7]\n" + "ldr d17, [x17, #0x1e0]\n" + "ldr x20, [x17, #0x1e8]\n" + "fmla v9.8h, v16.8h, v0.h[7]\n" + "ldr d16, [x17, #0x1f0]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0x1f8]\n" + "mov v16.d[1], x20\n" "add x13, x13, #0x10\n" "add x17, x17, #0x200\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" + "fmla v10.8h, v17.8h, v0.h[7]\n" "ldr d6, [x17, #0x0]\n" - "ldr x12, [x17, #0x8]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" + "ldr x20, [x17, #0x8]\n" + "fmla v11.8h, v16.8h, v0.h[7]\n" "ldr d0, [x13, #0x0]\n" "sub x14, x14, #0x8\n" "ldr d7, [x17, #0x10]\n" "cmp x14, #0x10\n" - "ldr x10, [x13, #0x8]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x18]\n" - "mov v0.d[1], x10\n" - "mov v7.d[1], x11\n" + "ldr x21, [x13, #0x8]\n" + "mov v6.d[1], x20\n" + "ldr x20, [x17, #0x18]\n" + "mov v0.d[1], x21\n" + "mov v7.d[1], x20\n" "prfm pldl1keep, [x13, #0x80]\n" "bge 26b\n" "27:" // Height 1: Multiply loop: Single iteration only "fmla v8.8h, v6.8h, v0.h[0]\n" - "ldr q6, [x17, #0x20]\n" + "ldr q17, [x17, #0x20]\n" "fmla v9.8h, v7.8h, v0.h[0]\n" - "ldr q7, [x17, #0x30]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "ldr q6, [x17, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "ldr q7, [x17, #0x50]\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "ldr q6, [x17, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "ldr q7, [x17, #0x70]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "ldr q6, [x17, #0x80]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "ldr q7, [x17, #0x90]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "ldr q6, [x17, #0xa0]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "ldr q7, [x17, #0xb0]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "ldr q6, [x17, #0xc0]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "ldr q7, [x17, #0xd0]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "ldr q6, [x17, #0xe0]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "ldr q7, [x17, #0xf0]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "ldr q6, [x17, #0x100]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "ldr q7, [x17, #0x110]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "ldr q6, [x17, #0x120]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "ldr q7, [x17, #0x130]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "ldr q6, [x17, #0x140]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "ldr q7, [x17, #0x150]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "ldr q6, [x17, #0x160]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "ldr q7, [x17, #0x170]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "ldr q6, [x17, #0x180]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "ldr q7, [x17, #0x190]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "ldr q6, [x17, #0x1a0]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "ldr q7, [x17, #0x1b0]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "ldr q6, [x17, #0x1c0]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "ldr q7, [x17, #0x1d0]\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "ldr q6, [x17, #0x1e0]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "ldr q7, [x17, #0x1f0]\n" + "ldr q16, [x17, #0x30]\n" + "fmla v10.8h, v17.8h, v0.h[0]\n" + "ldr q17, [x17, #0x40]\n" + "fmla v11.8h, v16.8h, v0.h[0]\n" + "ldr q16, [x17, #0x50]\n" + "fmla v8.8h, v17.8h, v0.h[1]\n" + "ldr q17, [x17, #0x60]\n" + "fmla v9.8h, v16.8h, v0.h[1]\n" + "ldr q16, [x17, #0x70]\n" + "fmla v10.8h, v17.8h, v0.h[1]\n" + "ldr q17, [x17, #0x80]\n" + "fmla v11.8h, v16.8h, v0.h[1]\n" + "ldr q16, [x17, #0x90]\n" + "fmla v8.8h, v17.8h, v0.h[2]\n" + "ldr q17, [x17, #0xa0]\n" + "fmla v9.8h, v16.8h, v0.h[2]\n" + "ldr q16, [x17, #0xb0]\n" + "fmla v10.8h, v17.8h, v0.h[2]\n" + "ldr q17, [x17, #0xc0]\n" + "fmla v11.8h, v16.8h, v0.h[2]\n" + "ldr q16, [x17, #0xd0]\n" + "fmla v8.8h, v17.8h, v0.h[3]\n" + "ldr q17, [x17, #0xe0]\n" + "fmla v9.8h, v16.8h, v0.h[3]\n" + "ldr q16, [x17, #0xf0]\n" + "fmla v10.8h, v17.8h, v0.h[3]\n" + "ldr q17, [x17, #0x100]\n" + "fmla v11.8h, v16.8h, v0.h[3]\n" + "ldr q16, [x17, #0x110]\n" + "fmla v8.8h, v17.8h, v0.h[4]\n" + "ldr q17, [x17, #0x120]\n" + "fmla v9.8h, v16.8h, v0.h[4]\n" + "ldr q16, [x17, #0x130]\n" + "fmla v10.8h, v17.8h, v0.h[4]\n" + "ldr q17, [x17, #0x140]\n" + "fmla v11.8h, v16.8h, v0.h[4]\n" + "ldr q16, [x17, #0x150]\n" + "fmla v8.8h, v17.8h, v0.h[5]\n" + "ldr q17, [x17, #0x160]\n" + "fmla v9.8h, v16.8h, v0.h[5]\n" + "ldr q16, [x17, #0x170]\n" + "fmla v10.8h, v17.8h, v0.h[5]\n" + "ldr q17, [x17, #0x180]\n" + "fmla v11.8h, v16.8h, v0.h[5]\n" + "ldr q16, [x17, #0x190]\n" + "fmla v8.8h, v17.8h, v0.h[6]\n" + "ldr q17, [x17, #0x1a0]\n" + "fmla v9.8h, v16.8h, v0.h[6]\n" + "ldr q16, [x17, #0x1b0]\n" + "fmla v10.8h, v17.8h, v0.h[6]\n" + "ldr q17, [x17, #0x1c0]\n" + "fmla v11.8h, v16.8h, v0.h[6]\n" + "ldr q16, [x17, #0x1d0]\n" + "fmla v8.8h, v17.8h, v0.h[7]\n" + "ldr q17, [x17, #0x1e0]\n" + "fmla v9.8h, v16.8h, v0.h[7]\n" + "ldr q16, [x17, #0x1f0]\n" "add x13, x13, #0x10\n" "sub x14, x14, #0x8\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" + "fmla v10.8h, v17.8h, v0.h[7]\n" "prfm pldl1keep, [x13, #0x80]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" + "fmla v11.8h, v16.8h, v0.h[7]\n" "add x17, x17, #0x200\n" "28:" // Height 1: Multiply loop: Main loop skip "cbz x14, 30f\n" "29:" // Height 1: Multiply loop: Odd block loop "ldr h0, [x13], #0x2\n" "sub x14, x14, #0x1\n" - "ldr q6, [x17, #0x0]\n" - "fmla v8.8h, v6.8h, v0.h[0]\n" - "ldr q7, [x17, #0x10]\n" - "fmla v9.8h, v7.8h, v0.h[0]\n" - "ldr q6, [x17, #0x20]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "ldr q7, [x17, #0x30]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" + "ldr q16, [x17, #0x0]\n" + "fmla v8.8h, v16.8h, v0.h[0]\n" + "ldr q16, [x17, #0x10]\n" + "fmla v9.8h, v16.8h, v0.h[0]\n" + "ldr q16, [x17, #0x20]\n" + "fmla v10.8h, v16.8h, v0.h[0]\n" + "ldr q16, [x17, #0x30]\n" + "fmla v11.8h, v16.8h, v0.h[0]\n" "add x17, x17, #0x40\n" "cbnz x14, 29b\n" "30:" // Height 1: Multiply loop: No odd multiplies @@ -491,17 +491,17 @@ void a64_hybrid_fp16_mla_6x32_a55 ( "prfm pstl1keep, [x16, #0x0]\n" "tbz %x[flags], #1, 31f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v0.8h }, [x20]\n" - "fmin v8.8h, v8.8h, v0.8h\n" - "fmin v9.8h, v9.8h, v0.8h\n" - "fmin v10.8h, v10.8h, v0.8h\n" - "fmin v11.8h, v11.8h, v0.8h\n" + "ld1r { v16.8h }, [x20]\n" + "fmin v8.8h, v8.8h, v16.8h\n" + "fmin v9.8h, v9.8h, v16.8h\n" + "fmin v10.8h, v10.8h, v16.8h\n" + "fmin v11.8h, v11.8h, v16.8h\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.8h }, [x20]\n" - "fmax v8.8h, v8.8h, v0.8h\n" - "fmax v9.8h, v9.8h, v0.8h\n" - "fmax v10.8h, v10.8h, v0.8h\n" - "fmax v11.8h, v11.8h, v0.8h\n" + "ld1r { v16.8h }, [x20]\n" + "fmax v8.8h, v8.8h, v16.8h\n" + "fmax v9.8h, v9.8h, v16.8h\n" + "fmax v10.8h, v10.8h, v16.8h\n" + "fmax v11.8h, v11.8h, v16.8h\n" "31:" // Height 1: No activation "cmp x8, #0x20\n" "bge 48f\n" @@ -799,324 +799,324 @@ void a64_hybrid_fp16_mla_6x32_a55 ( "72:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w14, [x20, x15, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 73f\n" - "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x13, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x15, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x13, [x20, #0x0]\n" + "ldr x12, [x20, #0x8]\n" "cbnz x15, 74f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x13, x13, x20, LSL #1\n" - "add x9, x9, x20, LSL #1\n" + "add x12, x12, x20, LSL #1\n" "b 74f\n" "73:" // Height 2: setup direct input "mov x13, %x[input_ptr]\n" - "add x9, x13, x20, LSL #1\n" + "add x12, x13, x21, LSL #1\n" "74:" // Height 2: input setup done "cmp x14, #0x8\n" "blt 77f\n" "ldr q0, [x13, #0x0]\n" "cmp x14, #0x10\n" - "ldr q1, [x9, #0x0]\n" + "ldr q1, [x12, #0x0]\n" "ldr q6, [x17, #0x0]\n" "ldr q7, [x17, #0x10]\n" "blt 76f\n" "75:" // Height 2: Multiply loop: Main loop head "fmla v8.8h, v6.8h, v0.h[0]\n" - "ldr x12, [x17, #0x28]\n" + "ldr x21, [x17, #0x28]\n" "fmla v12.8h, v6.8h, v1.h[0]\n" - "ldr d6, [x17, #0x20]\n" + "ldr d17, [x17, #0x20]\n" "fmla v9.8h, v7.8h, v0.h[0]\n" - "ldr x11, [x17, #0x38]\n" + "ldr x20, [x17, #0x38]\n" "fmla v13.8h, v7.8h, v1.h[0]\n" - "ldr d7, [x17, #0x30]\n" - "mov v6.d[1], x12\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "mov v7.d[1], x11\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "ldr d6, [x17, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "ldr x12, [x17, #0x48]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "ldr d7, [x17, #0x50]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x58]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "ldr x12, [x17, #0x68]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "ldr d6, [x17, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "ldr x11, [x17, #0x78]\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "ldr d7, [x17, #0x70]\n" - "mov v6.d[1], x12\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "mov v7.d[1], x11\n" - "fmla v14.8h, v6.8h, v1.h[1]\n" - "ldr d6, [x17, #0x80]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "ldr x12, [x17, #0x88]\n" - "fmla v15.8h, v7.8h, v1.h[1]\n" - "ldr d7, [x17, #0x90]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x98]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "ldr x12, [x17, #0xa8]\n" - "fmla v12.8h, v6.8h, v1.h[2]\n" - "ldr d6, [x17, #0xa0]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "ldr x11, [x17, #0xb8]\n" - "fmla v13.8h, v7.8h, v1.h[2]\n" - "ldr d7, [x17, #0xb0]\n" - "mov v6.d[1], x12\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "mov v7.d[1], x11\n" - "fmla v14.8h, v6.8h, v1.h[2]\n" - "ldr d6, [x17, #0xc0]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "ldr x12, [x17, #0xc8]\n" - "fmla v15.8h, v7.8h, v1.h[2]\n" - "ldr d7, [x17, #0xd0]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0xd8]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "ldr x12, [x17, #0xe8]\n" - "fmla v12.8h, v6.8h, v1.h[3]\n" - "ldr d6, [x17, #0xe0]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "ldr x11, [x17, #0xf8]\n" - "fmla v13.8h, v7.8h, v1.h[3]\n" - "ldr d7, [x17, #0xf0]\n" - "mov v6.d[1], x12\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "mov v7.d[1], x11\n" - "fmla v14.8h, v6.8h, v1.h[3]\n" - "ldr d6, [x17, #0x100]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "ldr x12, [x17, #0x108]\n" - "fmla v15.8h, v7.8h, v1.h[3]\n" - "ldr d7, [x17, #0x110]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x118]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "ldr x12, [x17, #0x128]\n" - "fmla v12.8h, v6.8h, v1.h[4]\n" - "ldr d6, [x17, #0x120]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "ldr x11, [x17, #0x138]\n" - "fmla v13.8h, v7.8h, v1.h[4]\n" - "ldr d7, [x17, #0x130]\n" - "mov v6.d[1], x12\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "mov v7.d[1], x11\n" - "fmla v14.8h, v6.8h, v1.h[4]\n" - "ldr d6, [x17, #0x140]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "ldr x12, [x17, #0x148]\n" - "fmla v15.8h, v7.8h, v1.h[4]\n" - "ldr d7, [x17, #0x150]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x158]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "ldr x12, [x17, #0x168]\n" - "fmla v12.8h, v6.8h, v1.h[5]\n" - "ldr d6, [x17, #0x160]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "ldr x11, [x17, #0x178]\n" - "fmla v13.8h, v7.8h, v1.h[5]\n" - "ldr d7, [x17, #0x170]\n" - "mov v6.d[1], x12\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "mov v7.d[1], x11\n" - "fmla v14.8h, v6.8h, v1.h[5]\n" - "ldr d6, [x17, #0x180]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "ldr x12, [x17, #0x188]\n" - "fmla v15.8h, v7.8h, v1.h[5]\n" - "ldr d7, [x17, #0x190]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x198]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "ldr x12, [x17, #0x1a8]\n" - "fmla v12.8h, v6.8h, v1.h[6]\n" - "ldr d6, [x17, #0x1a0]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "ldr x11, [x17, #0x1b8]\n" - "fmla v13.8h, v7.8h, v1.h[6]\n" - "ldr d7, [x17, #0x1b0]\n" - "mov v6.d[1], x12\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "mov v7.d[1], x11\n" - "fmla v14.8h, v6.8h, v1.h[6]\n" - "ldr d6, [x17, #0x1c0]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "ldr x12, [x17, #0x1c8]\n" - "fmla v15.8h, v7.8h, v1.h[6]\n" - "ldr d7, [x17, #0x1d0]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x1d8]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "ldr x12, [x17, #0x1e8]\n" - "fmla v12.8h, v6.8h, v1.h[7]\n" - "ldr d6, [x17, #0x1e0]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "ldr x11, [x17, #0x1f8]\n" - "fmla v13.8h, v7.8h, v1.h[7]\n" - "ldr d7, [x17, #0x1f0]\n" - "mov v6.d[1], x12\n" + "ldr d16, [x17, #0x30]\n" + "mov v17.d[1], x21\n" + "fmla v10.8h, v17.8h, v0.h[0]\n" + "mov v16.d[1], x20\n" + "fmla v14.8h, v17.8h, v1.h[0]\n" + "ldr d17, [x17, #0x40]\n" + "fmla v11.8h, v16.8h, v0.h[0]\n" + "ldr x20, [x17, #0x48]\n" + "fmla v15.8h, v16.8h, v1.h[0]\n" + "ldr d16, [x17, #0x50]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0x58]\n" + "mov v16.d[1], x20\n" + "fmla v8.8h, v17.8h, v0.h[1]\n" + "ldr x21, [x17, #0x68]\n" + "fmla v12.8h, v17.8h, v1.h[1]\n" + "ldr d17, [x17, #0x60]\n" + "fmla v9.8h, v16.8h, v0.h[1]\n" + "ldr x20, [x17, #0x78]\n" + "fmla v13.8h, v16.8h, v1.h[1]\n" + "ldr d16, [x17, #0x70]\n" + "mov v17.d[1], x21\n" + "fmla v10.8h, v17.8h, v0.h[1]\n" + "mov v16.d[1], x20\n" + "fmla v14.8h, v17.8h, v1.h[1]\n" + "ldr d17, [x17, #0x80]\n" + "fmla v11.8h, v16.8h, v0.h[1]\n" + "ldr x20, [x17, #0x88]\n" + "fmla v15.8h, v16.8h, v1.h[1]\n" + "ldr d16, [x17, #0x90]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0x98]\n" + "mov v16.d[1], x20\n" + "fmla v8.8h, v17.8h, v0.h[2]\n" + "ldr x21, [x17, #0xa8]\n" + "fmla v12.8h, v17.8h, v1.h[2]\n" + "ldr d17, [x17, #0xa0]\n" + "fmla v9.8h, v16.8h, v0.h[2]\n" + "ldr x20, [x17, #0xb8]\n" + "fmla v13.8h, v16.8h, v1.h[2]\n" + "ldr d16, [x17, #0xb0]\n" + "mov v17.d[1], x21\n" + "fmla v10.8h, v17.8h, v0.h[2]\n" + "mov v16.d[1], x20\n" + "fmla v14.8h, v17.8h, v1.h[2]\n" + "ldr d17, [x17, #0xc0]\n" + "fmla v11.8h, v16.8h, v0.h[2]\n" + "ldr x20, [x17, #0xc8]\n" + "fmla v15.8h, v16.8h, v1.h[2]\n" + "ldr d16, [x17, #0xd0]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0xd8]\n" + "mov v16.d[1], x20\n" + "fmla v8.8h, v17.8h, v0.h[3]\n" + "ldr x21, [x17, #0xe8]\n" + "fmla v12.8h, v17.8h, v1.h[3]\n" + "ldr d17, [x17, #0xe0]\n" + "fmla v9.8h, v16.8h, v0.h[3]\n" + "ldr x20, [x17, #0xf8]\n" + "fmla v13.8h, v16.8h, v1.h[3]\n" + "ldr d16, [x17, #0xf0]\n" + "mov v17.d[1], x21\n" + "fmla v10.8h, v17.8h, v0.h[3]\n" + "mov v16.d[1], x20\n" + "fmla v14.8h, v17.8h, v1.h[3]\n" + "ldr d17, [x17, #0x100]\n" + "fmla v11.8h, v16.8h, v0.h[3]\n" + "ldr x20, [x17, #0x108]\n" + "fmla v15.8h, v16.8h, v1.h[3]\n" + "ldr d16, [x17, #0x110]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0x118]\n" + "mov v16.d[1], x20\n" + "fmla v8.8h, v17.8h, v0.h[4]\n" + "ldr x21, [x17, #0x128]\n" + "fmla v12.8h, v17.8h, v1.h[4]\n" + "ldr d17, [x17, #0x120]\n" + "fmla v9.8h, v16.8h, v0.h[4]\n" + "ldr x20, [x17, #0x138]\n" + "fmla v13.8h, v16.8h, v1.h[4]\n" + "ldr d16, [x17, #0x130]\n" + "mov v17.d[1], x21\n" + "fmla v10.8h, v17.8h, v0.h[4]\n" + "mov v16.d[1], x20\n" + "fmla v14.8h, v17.8h, v1.h[4]\n" + "ldr d17, [x17, #0x140]\n" + "fmla v11.8h, v16.8h, v0.h[4]\n" + "ldr x20, [x17, #0x148]\n" + "fmla v15.8h, v16.8h, v1.h[4]\n" + "ldr d16, [x17, #0x150]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0x158]\n" + "mov v16.d[1], x20\n" + "fmla v8.8h, v17.8h, v0.h[5]\n" + "ldr x21, [x17, #0x168]\n" + "fmla v12.8h, v17.8h, v1.h[5]\n" + "ldr d17, [x17, #0x160]\n" + "fmla v9.8h, v16.8h, v0.h[5]\n" + "ldr x20, [x17, #0x178]\n" + "fmla v13.8h, v16.8h, v1.h[5]\n" + "ldr d16, [x17, #0x170]\n" + "mov v17.d[1], x21\n" + "fmla v10.8h, v17.8h, v0.h[5]\n" + "mov v16.d[1], x20\n" + "fmla v14.8h, v17.8h, v1.h[5]\n" + "ldr d17, [x17, #0x180]\n" + "fmla v11.8h, v16.8h, v0.h[5]\n" + "ldr x20, [x17, #0x188]\n" + "fmla v15.8h, v16.8h, v1.h[5]\n" + "ldr d16, [x17, #0x190]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0x198]\n" + "mov v16.d[1], x20\n" + "fmla v8.8h, v17.8h, v0.h[6]\n" + "ldr x21, [x17, #0x1a8]\n" + "fmla v12.8h, v17.8h, v1.h[6]\n" + "ldr d17, [x17, #0x1a0]\n" + "fmla v9.8h, v16.8h, v0.h[6]\n" + "ldr x20, [x17, #0x1b8]\n" + "fmla v13.8h, v16.8h, v1.h[6]\n" + "ldr d16, [x17, #0x1b0]\n" + "mov v17.d[1], x21\n" + "fmla v10.8h, v17.8h, v0.h[6]\n" + "mov v16.d[1], x20\n" + "fmla v14.8h, v17.8h, v1.h[6]\n" + "ldr d17, [x17, #0x1c0]\n" + "fmla v11.8h, v16.8h, v0.h[6]\n" + "ldr x20, [x17, #0x1c8]\n" + "fmla v15.8h, v16.8h, v1.h[6]\n" + "ldr d16, [x17, #0x1d0]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0x1d8]\n" + "mov v16.d[1], x20\n" + "fmla v8.8h, v17.8h, v0.h[7]\n" + "ldr x21, [x17, #0x1e8]\n" + "fmla v12.8h, v17.8h, v1.h[7]\n" + "ldr d17, [x17, #0x1e0]\n" + "fmla v9.8h, v16.8h, v0.h[7]\n" + "ldr x20, [x17, #0x1f8]\n" + "fmla v13.8h, v16.8h, v1.h[7]\n" + "ldr d16, [x17, #0x1f0]\n" + "mov v17.d[1], x21\n" "add x13, x13, #0x10\n" - "mov v7.d[1], x11\n" - "add x9, x9, #0x10\n" + "mov v16.d[1], x20\n" + "add x12, x12, #0x10\n" "add x17, x17, #0x200\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "fmla v14.8h, v6.8h, v1.h[7]\n" + "fmla v10.8h, v17.8h, v0.h[7]\n" + "fmla v14.8h, v17.8h, v1.h[7]\n" "ldr d6, [x17, #0x0]\n" - "ldr x12, [x17, #0x8]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" + "ldr x21, [x17, #0x8]\n" + "fmla v11.8h, v16.8h, v0.h[7]\n" "ldr d0, [x13, #0x0]\n" - "fmla v15.8h, v7.8h, v1.h[7]\n" - "ldr d1, [x9, #0x0]\n" + "fmla v15.8h, v16.8h, v1.h[7]\n" + "ldr d1, [x12, #0x0]\n" "sub x14, x14, #0x8\n" "ldr d7, [x17, #0x10]\n" "cmp x14, #0x10\n" - "ldr x10, [x13, #0x8]\n" - "mov v6.d[1], x12\n" - "ldr x28, [x9, #0x8]\n" - "mov v0.d[1], x10\n" - "ldr x11, [x17, #0x18]\n" - "mov v1.d[1], x28\n" + "ldr x20, [x13, #0x8]\n" + "mov v6.d[1], x21\n" + "ldr x21, [x12, #0x8]\n" + "mov v0.d[1], x20\n" + "ldr x20, [x17, #0x18]\n" + "mov v1.d[1], x21\n" "prfm pldl1keep, [x13, #0x80]\n" - "mov v7.d[1], x11\n" - "prfm pldl1keep, [x9, #0x80]\n" + "mov v7.d[1], x20\n" + "prfm pldl1keep, [x12, #0x80]\n" "bge 75b\n" "76:" // Height 2: Multiply loop: Single iteration only "fmla v8.8h, v6.8h, v0.h[0]\n" "add x13, x13, #0x10\n" "fmla v12.8h, v6.8h, v1.h[0]\n" - "ldr q6, [x17, #0x20]\n" + "ldr q17, [x17, #0x20]\n" "fmla v9.8h, v7.8h, v0.h[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" "fmla v13.8h, v7.8h, v1.h[0]\n" - "ldr q7, [x17, #0x30]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" + "ldr q16, [x17, #0x30]\n" + "fmla v10.8h, v17.8h, v0.h[0]\n" "sub x14, x14, #0x8\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "ldr q6, [x17, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" + "fmla v14.8h, v17.8h, v1.h[0]\n" + "ldr q17, [x17, #0x40]\n" + "fmla v11.8h, v16.8h, v0.h[0]\n" "prfm pldl1keep, [x13, #0x80]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "ldr q7, [x17, #0x50]\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "prfm pldl1keep, [x9, #0x80]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "ldr q6, [x17, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "ldr q7, [x17, #0x70]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "fmla v14.8h, v6.8h, v1.h[1]\n" - "ldr q6, [x17, #0x80]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "fmla v15.8h, v7.8h, v1.h[1]\n" - "ldr q7, [x17, #0x90]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "fmla v12.8h, v6.8h, v1.h[2]\n" - "ldr q6, [x17, #0xa0]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "fmla v13.8h, v7.8h, v1.h[2]\n" - "ldr q7, [x17, #0xb0]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "fmla v14.8h, v6.8h, v1.h[2]\n" - "ldr q6, [x17, #0xc0]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "fmla v15.8h, v7.8h, v1.h[2]\n" - "ldr q7, [x17, #0xd0]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "fmla v12.8h, v6.8h, v1.h[3]\n" - "ldr q6, [x17, #0xe0]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "fmla v13.8h, v7.8h, v1.h[3]\n" - "ldr q7, [x17, #0xf0]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "fmla v14.8h, v6.8h, v1.h[3]\n" - "ldr q6, [x17, #0x100]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "fmla v15.8h, v7.8h, v1.h[3]\n" - "ldr q7, [x17, #0x110]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "fmla v12.8h, v6.8h, v1.h[4]\n" - "ldr q6, [x17, #0x120]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "fmla v13.8h, v7.8h, v1.h[4]\n" - "ldr q7, [x17, #0x130]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "fmla v14.8h, v6.8h, v1.h[4]\n" - "ldr q6, [x17, #0x140]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "fmla v15.8h, v7.8h, v1.h[4]\n" - "ldr q7, [x17, #0x150]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "fmla v12.8h, v6.8h, v1.h[5]\n" - "ldr q6, [x17, #0x160]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "fmla v13.8h, v7.8h, v1.h[5]\n" - "ldr q7, [x17, #0x170]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "fmla v14.8h, v6.8h, v1.h[5]\n" - "ldr q6, [x17, #0x180]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "fmla v15.8h, v7.8h, v1.h[5]\n" - "ldr q7, [x17, #0x190]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "fmla v12.8h, v6.8h, v1.h[6]\n" - "ldr q6, [x17, #0x1a0]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "fmla v13.8h, v7.8h, v1.h[6]\n" - "ldr q7, [x17, #0x1b0]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "fmla v14.8h, v6.8h, v1.h[6]\n" - "ldr q6, [x17, #0x1c0]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "fmla v15.8h, v7.8h, v1.h[6]\n" - "ldr q7, [x17, #0x1d0]\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "fmla v12.8h, v6.8h, v1.h[7]\n" - "ldr q6, [x17, #0x1e0]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "fmla v13.8h, v7.8h, v1.h[7]\n" - "ldr q7, [x17, #0x1f0]\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" + "fmla v15.8h, v16.8h, v1.h[0]\n" + "ldr q16, [x17, #0x50]\n" + "fmla v8.8h, v17.8h, v0.h[1]\n" + "prfm pldl1keep, [x12, #0x80]\n" + "fmla v12.8h, v17.8h, v1.h[1]\n" + "ldr q17, [x17, #0x60]\n" + "fmla v9.8h, v16.8h, v0.h[1]\n" + "fmla v13.8h, v16.8h, v1.h[1]\n" + "ldr q16, [x17, #0x70]\n" + "fmla v10.8h, v17.8h, v0.h[1]\n" + "fmla v14.8h, v17.8h, v1.h[1]\n" + "ldr q17, [x17, #0x80]\n" + "fmla v11.8h, v16.8h, v0.h[1]\n" + "fmla v15.8h, v16.8h, v1.h[1]\n" + "ldr q16, [x17, #0x90]\n" + "fmla v8.8h, v17.8h, v0.h[2]\n" + "fmla v12.8h, v17.8h, v1.h[2]\n" + "ldr q17, [x17, #0xa0]\n" + "fmla v9.8h, v16.8h, v0.h[2]\n" + "fmla v13.8h, v16.8h, v1.h[2]\n" + "ldr q16, [x17, #0xb0]\n" + "fmla v10.8h, v17.8h, v0.h[2]\n" + "fmla v14.8h, v17.8h, v1.h[2]\n" + "ldr q17, [x17, #0xc0]\n" + "fmla v11.8h, v16.8h, v0.h[2]\n" + "fmla v15.8h, v16.8h, v1.h[2]\n" + "ldr q16, [x17, #0xd0]\n" + "fmla v8.8h, v17.8h, v0.h[3]\n" + "fmla v12.8h, v17.8h, v1.h[3]\n" + "ldr q17, [x17, #0xe0]\n" + "fmla v9.8h, v16.8h, v0.h[3]\n" + "fmla v13.8h, v16.8h, v1.h[3]\n" + "ldr q16, [x17, #0xf0]\n" + "fmla v10.8h, v17.8h, v0.h[3]\n" + "fmla v14.8h, v17.8h, v1.h[3]\n" + "ldr q17, [x17, #0x100]\n" + "fmla v11.8h, v16.8h, v0.h[3]\n" + "fmla v15.8h, v16.8h, v1.h[3]\n" + "ldr q16, [x17, #0x110]\n" + "fmla v8.8h, v17.8h, v0.h[4]\n" + "fmla v12.8h, v17.8h, v1.h[4]\n" + "ldr q17, [x17, #0x120]\n" + "fmla v9.8h, v16.8h, v0.h[4]\n" + "fmla v13.8h, v16.8h, v1.h[4]\n" + "ldr q16, [x17, #0x130]\n" + "fmla v10.8h, v17.8h, v0.h[4]\n" + "fmla v14.8h, v17.8h, v1.h[4]\n" + "ldr q17, [x17, #0x140]\n" + "fmla v11.8h, v16.8h, v0.h[4]\n" + "fmla v15.8h, v16.8h, v1.h[4]\n" + "ldr q16, [x17, #0x150]\n" + "fmla v8.8h, v17.8h, v0.h[5]\n" + "fmla v12.8h, v17.8h, v1.h[5]\n" + "ldr q17, [x17, #0x160]\n" + "fmla v9.8h, v16.8h, v0.h[5]\n" + "fmla v13.8h, v16.8h, v1.h[5]\n" + "ldr q16, [x17, #0x170]\n" + "fmla v10.8h, v17.8h, v0.h[5]\n" + "fmla v14.8h, v17.8h, v1.h[5]\n" + "ldr q17, [x17, #0x180]\n" + "fmla v11.8h, v16.8h, v0.h[5]\n" + "fmla v15.8h, v16.8h, v1.h[5]\n" + "ldr q16, [x17, #0x190]\n" + "fmla v8.8h, v17.8h, v0.h[6]\n" + "fmla v12.8h, v17.8h, v1.h[6]\n" + "ldr q17, [x17, #0x1a0]\n" + "fmla v9.8h, v16.8h, v0.h[6]\n" + "fmla v13.8h, v16.8h, v1.h[6]\n" + "ldr q16, [x17, #0x1b0]\n" + "fmla v10.8h, v17.8h, v0.h[6]\n" + "fmla v14.8h, v17.8h, v1.h[6]\n" + "ldr q17, [x17, #0x1c0]\n" + "fmla v11.8h, v16.8h, v0.h[6]\n" + "fmla v15.8h, v16.8h, v1.h[6]\n" + "ldr q16, [x17, #0x1d0]\n" + "fmla v8.8h, v17.8h, v0.h[7]\n" + "fmla v12.8h, v17.8h, v1.h[7]\n" + "ldr q17, [x17, #0x1e0]\n" + "fmla v9.8h, v16.8h, v0.h[7]\n" + "fmla v13.8h, v16.8h, v1.h[7]\n" + "ldr q16, [x17, #0x1f0]\n" + "fmla v10.8h, v17.8h, v0.h[7]\n" "add x17, x17, #0x200\n" - "fmla v14.8h, v6.8h, v1.h[7]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" - "fmla v15.8h, v7.8h, v1.h[7]\n" + "fmla v14.8h, v17.8h, v1.h[7]\n" + "fmla v11.8h, v16.8h, v0.h[7]\n" + "fmla v15.8h, v16.8h, v1.h[7]\n" "77:" // Height 2: Multiply loop: Main loop skip "cbz x14, 79f\n" "78:" // Height 2: Multiply loop: Odd block loop - "ldr h0, [x13], #0x2\n" + "ldr h1, [x13], #0x2\n" "sub x14, x14, #0x1\n" - "ldr h1, [x9], #0x2\n" - "ldr q6, [x17, #0x0]\n" - "fmla v8.8h, v6.8h, v0.h[0]\n" - "ldr q7, [x17, #0x10]\n" - "fmla v12.8h, v6.8h, v1.h[0]\n" - "ldr q6, [x17, #0x20]\n" - "fmla v9.8h, v7.8h, v0.h[0]\n" - "fmla v13.8h, v7.8h, v1.h[0]\n" - "ldr q7, [x17, #0x30]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" + "ldr h0, [x12], #0x2\n" + "ldr q17, [x17, #0x0]\n" + "fmla v8.8h, v17.8h, v1.h[0]\n" + "ldr q16, [x17, #0x10]\n" + "fmla v12.8h, v17.8h, v0.h[0]\n" + "ldr q17, [x17, #0x20]\n" + "fmla v9.8h, v16.8h, v1.h[0]\n" + "fmla v13.8h, v16.8h, v0.h[0]\n" + "ldr q16, [x17, #0x30]\n" + "fmla v10.8h, v17.8h, v1.h[0]\n" "add x17, x17, #0x40\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" + "fmla v14.8h, v17.8h, v0.h[0]\n" + "fmla v11.8h, v16.8h, v1.h[0]\n" + "fmla v15.8h, v16.8h, v0.h[0]\n" "cbnz x14, 78b\n" "79:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -1129,25 +1129,25 @@ void a64_hybrid_fp16_mla_6x32_a55 ( "prfm pstl1keep, [x25, #0x0]\n" "tbz %x[flags], #1, 80f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v0.8h }, [x20]\n" - "fmin v8.8h, v8.8h, v0.8h\n" - "fmin v9.8h, v9.8h, v0.8h\n" - "fmin v10.8h, v10.8h, v0.8h\n" - "fmin v11.8h, v11.8h, v0.8h\n" - "fmin v12.8h, v12.8h, v0.8h\n" - "fmin v13.8h, v13.8h, v0.8h\n" - "fmin v14.8h, v14.8h, v0.8h\n" - "fmin v15.8h, v15.8h, v0.8h\n" + "ld1r { v16.8h }, [x20]\n" + "fmin v8.8h, v8.8h, v16.8h\n" + "fmin v9.8h, v9.8h, v16.8h\n" + "fmin v10.8h, v10.8h, v16.8h\n" + "fmin v11.8h, v11.8h, v16.8h\n" + "fmin v12.8h, v12.8h, v16.8h\n" + "fmin v13.8h, v13.8h, v16.8h\n" + "fmin v14.8h, v14.8h, v16.8h\n" + "fmin v15.8h, v15.8h, v16.8h\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.8h }, [x20]\n" - "fmax v8.8h, v8.8h, v0.8h\n" - "fmax v9.8h, v9.8h, v0.8h\n" - "fmax v10.8h, v10.8h, v0.8h\n" - "fmax v11.8h, v11.8h, v0.8h\n" - "fmax v12.8h, v12.8h, v0.8h\n" - "fmax v13.8h, v13.8h, v0.8h\n" - "fmax v14.8h, v14.8h, v0.8h\n" - "fmax v15.8h, v15.8h, v0.8h\n" + "ld1r { v16.8h }, [x20]\n" + "fmax v8.8h, v8.8h, v16.8h\n" + "fmax v9.8h, v9.8h, v16.8h\n" + "fmax v10.8h, v10.8h, v16.8h\n" + "fmax v11.8h, v11.8h, v16.8h\n" + "fmax v12.8h, v12.8h, v16.8h\n" + "fmax v13.8h, v13.8h, v16.8h\n" + "fmax v14.8h, v14.8h, v16.8h\n" + "fmax v15.8h, v15.8h, v16.8h\n" "80:" // Height 2: No activation "cmp x8, #0x20\n" "bge 97f\n" @@ -1526,404 +1526,404 @@ void a64_hybrid_fp16_mla_6x32_a55 ( "121:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w14, [x20, x15, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 122f\n" - "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x13, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" - "ldr x27, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x15, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x13, [x20, #0x0]\n" + "ldr x12, [x20, #0x8]\n" + "ldr x11, [x20, #0x10]\n" "cbnz x15, 123f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x13, x13, x20, LSL #1\n" - "add x9, x9, x20, LSL #1\n" - "add x27, x27, x20, LSL #1\n" + "add x12, x12, x20, LSL #1\n" + "add x11, x11, x20, LSL #1\n" "b 123f\n" "122:" // Height 3: setup direct input "mov x13, %x[input_ptr]\n" - "add x9, x13, x20, LSL #1\n" - "add x27, x9, x20, LSL #1\n" + "add x12, x13, x21, LSL #1\n" + "add x11, x12, x21, LSL #1\n" "123:" // Height 3: input setup done "cmp x14, #0x8\n" "blt 126f\n" "ldr q0, [x13, #0x0]\n" "cmp x14, #0x10\n" - "ldr q1, [x9, #0x0]\n" - "ldr q2, [x27, #0x0]\n" + "ldr q1, [x12, #0x0]\n" + "ldr q2, [x11, #0x0]\n" "ldr q6, [x17, #0x0]\n" "ldr q7, [x17, #0x10]\n" "blt 125f\n" "124:" // Height 3: Multiply loop: Main loop head "fmla v8.8h, v6.8h, v0.h[0]\n" - "ldr x12, [x17, #0x28]\n" + "ldr x21, [x17, #0x28]\n" "fmla v12.8h, v6.8h, v1.h[0]\n" - "ldr x11, [x17, #0x38]\n" + "ldr x20, [x17, #0x38]\n" "fmla v16.8h, v6.8h, v2.h[0]\n" - "ldr d6, [x17, #0x20]\n" + "ldr d21, [x17, #0x20]\n" "fmla v9.8h, v7.8h, v0.h[0]\n" - "mov v6.d[1], x12\n" + "mov v21.d[1], x21\n" "fmla v13.8h, v7.8h, v1.h[0]\n" - "ldr x12, [x17, #0x48]\n" + "ldr x21, [x17, #0x48]\n" "fmla v17.8h, v7.8h, v2.h[0]\n" - "ldr d7, [x17, #0x30]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "ldr x11, [x17, #0x58]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "ldr d6, [x17, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "mov v6.d[1], x12\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "ldr x12, [x17, #0x68]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "ldr d7, [x17, #0x50]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "ldr x11, [x17, #0x78]\n" - "fmla v16.8h, v6.8h, v2.h[1]\n" - "ldr d6, [x17, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "mov v6.d[1], x12\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "ldr x12, [x17, #0x88]\n" - "fmla v17.8h, v7.8h, v2.h[1]\n" - "ldr d7, [x17, #0x70]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "fmla v14.8h, v6.8h, v1.h[1]\n" - "ldr x11, [x17, #0x98]\n" - "fmla v18.8h, v6.8h, v2.h[1]\n" - "ldr d6, [x17, #0x80]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "mov v6.d[1], x12\n" - "fmla v15.8h, v7.8h, v1.h[1]\n" - "ldr x12, [x17, #0xa8]\n" - "fmla v19.8h, v7.8h, v2.h[1]\n" - "ldr d7, [x17, #0x90]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "fmla v12.8h, v6.8h, v1.h[2]\n" - "ldr x11, [x17, #0xb8]\n" - "fmla v16.8h, v6.8h, v2.h[2]\n" - "ldr d6, [x17, #0xa0]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "mov v6.d[1], x12\n" - "fmla v13.8h, v7.8h, v1.h[2]\n" - "ldr x12, [x17, #0xc8]\n" - "fmla v17.8h, v7.8h, v2.h[2]\n" - "ldr d7, [x17, #0xb0]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "fmla v14.8h, v6.8h, v1.h[2]\n" - "ldr x11, [x17, #0xd8]\n" - "fmla v18.8h, v6.8h, v2.h[2]\n" - "ldr d6, [x17, #0xc0]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "mov v6.d[1], x12\n" - "fmla v15.8h, v7.8h, v1.h[2]\n" - "ldr x12, [x17, #0xe8]\n" - "fmla v19.8h, v7.8h, v2.h[2]\n" - "ldr d7, [x17, #0xd0]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "fmla v12.8h, v6.8h, v1.h[3]\n" - "ldr x11, [x17, #0xf8]\n" - "fmla v16.8h, v6.8h, v2.h[3]\n" - "ldr d6, [x17, #0xe0]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "mov v6.d[1], x12\n" - "fmla v13.8h, v7.8h, v1.h[3]\n" - "ldr x12, [x17, #0x108]\n" - "fmla v17.8h, v7.8h, v2.h[3]\n" - "ldr d7, [x17, #0xf0]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "fmla v14.8h, v6.8h, v1.h[3]\n" - "ldr x11, [x17, #0x118]\n" - "fmla v18.8h, v6.8h, v2.h[3]\n" - "ldr d6, [x17, #0x100]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "mov v6.d[1], x12\n" - "fmla v15.8h, v7.8h, v1.h[3]\n" - "ldr x12, [x17, #0x128]\n" - "fmla v19.8h, v7.8h, v2.h[3]\n" - "ldr d7, [x17, #0x110]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "fmla v12.8h, v6.8h, v1.h[4]\n" - "ldr x11, [x17, #0x138]\n" - "fmla v16.8h, v6.8h, v2.h[4]\n" - "ldr d6, [x17, #0x120]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "mov v6.d[1], x12\n" - "fmla v13.8h, v7.8h, v1.h[4]\n" - "ldr x12, [x17, #0x148]\n" - "fmla v17.8h, v7.8h, v2.h[4]\n" - "ldr d7, [x17, #0x130]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "fmla v14.8h, v6.8h, v1.h[4]\n" - "ldr x11, [x17, #0x158]\n" - "fmla v18.8h, v6.8h, v2.h[4]\n" - "ldr d6, [x17, #0x140]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "mov v6.d[1], x12\n" - "fmla v15.8h, v7.8h, v1.h[4]\n" - "ldr x12, [x17, #0x168]\n" - "fmla v19.8h, v7.8h, v2.h[4]\n" - "ldr d7, [x17, #0x150]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "fmla v12.8h, v6.8h, v1.h[5]\n" - "ldr x11, [x17, #0x178]\n" - "fmla v16.8h, v6.8h, v2.h[5]\n" - "ldr d6, [x17, #0x160]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "mov v6.d[1], x12\n" - "fmla v13.8h, v7.8h, v1.h[5]\n" - "ldr x12, [x17, #0x188]\n" - "fmla v17.8h, v7.8h, v2.h[5]\n" - "ldr d7, [x17, #0x170]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "fmla v14.8h, v6.8h, v1.h[5]\n" - "ldr x11, [x17, #0x198]\n" - "fmla v18.8h, v6.8h, v2.h[5]\n" - "ldr d6, [x17, #0x180]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "mov v6.d[1], x12\n" - "fmla v15.8h, v7.8h, v1.h[5]\n" - "ldr x12, [x17, #0x1a8]\n" - "fmla v19.8h, v7.8h, v2.h[5]\n" - "ldr d7, [x17, #0x190]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "fmla v12.8h, v6.8h, v1.h[6]\n" - "ldr x11, [x17, #0x1b8]\n" - "fmla v16.8h, v6.8h, v2.h[6]\n" - "ldr d6, [x17, #0x1a0]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "mov v6.d[1], x12\n" - "fmla v13.8h, v7.8h, v1.h[6]\n" - "ldr x12, [x17, #0x1c8]\n" - "fmla v17.8h, v7.8h, v2.h[6]\n" - "ldr d7, [x17, #0x1b0]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "fmla v14.8h, v6.8h, v1.h[6]\n" - "ldr x11, [x17, #0x1d8]\n" - "fmla v18.8h, v6.8h, v2.h[6]\n" - "ldr d6, [x17, #0x1c0]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "mov v6.d[1], x12\n" - "fmla v15.8h, v7.8h, v1.h[6]\n" - "ldr x12, [x17, #0x1e8]\n" - "fmla v19.8h, v7.8h, v2.h[6]\n" - "ldr d7, [x17, #0x1d0]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "fmla v12.8h, v6.8h, v1.h[7]\n" - "ldr x11, [x17, #0x1f8]\n" - "fmla v16.8h, v6.8h, v2.h[7]\n" - "ldr d6, [x17, #0x1e0]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "mov v6.d[1], x12\n" - "fmla v13.8h, v7.8h, v1.h[7]\n" + "ldr d20, [x17, #0x30]\n" + "mov v20.d[1], x20\n" + "fmla v10.8h, v21.8h, v0.h[0]\n" + "fmla v14.8h, v21.8h, v1.h[0]\n" + "ldr x20, [x17, #0x58]\n" + "fmla v18.8h, v21.8h, v2.h[0]\n" + "ldr d21, [x17, #0x40]\n" + "fmla v11.8h, v20.8h, v0.h[0]\n" + "mov v21.d[1], x21\n" + "fmla v15.8h, v20.8h, v1.h[0]\n" + "ldr x21, [x17, #0x68]\n" + "fmla v19.8h, v20.8h, v2.h[0]\n" + "ldr d20, [x17, #0x50]\n" + "mov v20.d[1], x20\n" + "fmla v8.8h, v21.8h, v0.h[1]\n" + "fmla v12.8h, v21.8h, v1.h[1]\n" + "ldr x20, [x17, #0x78]\n" + "fmla v16.8h, v21.8h, v2.h[1]\n" + "ldr d21, [x17, #0x60]\n" + "fmla v9.8h, v20.8h, v0.h[1]\n" + "mov v21.d[1], x21\n" + "fmla v13.8h, v20.8h, v1.h[1]\n" + "ldr x21, [x17, #0x88]\n" + "fmla v17.8h, v20.8h, v2.h[1]\n" + "ldr d20, [x17, #0x70]\n" + "mov v20.d[1], x20\n" + "fmla v10.8h, v21.8h, v0.h[1]\n" + "fmla v14.8h, v21.8h, v1.h[1]\n" + "ldr x20, [x17, #0x98]\n" + "fmla v18.8h, v21.8h, v2.h[1]\n" + "ldr d21, [x17, #0x80]\n" + "fmla v11.8h, v20.8h, v0.h[1]\n" + "mov v21.d[1], x21\n" + "fmla v15.8h, v20.8h, v1.h[1]\n" + "ldr x21, [x17, #0xa8]\n" + "fmla v19.8h, v20.8h, v2.h[1]\n" + "ldr d20, [x17, #0x90]\n" + "mov v20.d[1], x20\n" + "fmla v8.8h, v21.8h, v0.h[2]\n" + "fmla v12.8h, v21.8h, v1.h[2]\n" + "ldr x20, [x17, #0xb8]\n" + "fmla v16.8h, v21.8h, v2.h[2]\n" + "ldr d21, [x17, #0xa0]\n" + "fmla v9.8h, v20.8h, v0.h[2]\n" + "mov v21.d[1], x21\n" + "fmla v13.8h, v20.8h, v1.h[2]\n" + "ldr x21, [x17, #0xc8]\n" + "fmla v17.8h, v20.8h, v2.h[2]\n" + "ldr d20, [x17, #0xb0]\n" + "mov v20.d[1], x20\n" + "fmla v10.8h, v21.8h, v0.h[2]\n" + "fmla v14.8h, v21.8h, v1.h[2]\n" + "ldr x20, [x17, #0xd8]\n" + "fmla v18.8h, v21.8h, v2.h[2]\n" + "ldr d21, [x17, #0xc0]\n" + "fmla v11.8h, v20.8h, v0.h[2]\n" + "mov v21.d[1], x21\n" + "fmla v15.8h, v20.8h, v1.h[2]\n" + "ldr x21, [x17, #0xe8]\n" + "fmla v19.8h, v20.8h, v2.h[2]\n" + "ldr d20, [x17, #0xd0]\n" + "mov v20.d[1], x20\n" + "fmla v8.8h, v21.8h, v0.h[3]\n" + "fmla v12.8h, v21.8h, v1.h[3]\n" + "ldr x20, [x17, #0xf8]\n" + "fmla v16.8h, v21.8h, v2.h[3]\n" + "ldr d21, [x17, #0xe0]\n" + "fmla v9.8h, v20.8h, v0.h[3]\n" + "mov v21.d[1], x21\n" + "fmla v13.8h, v20.8h, v1.h[3]\n" + "ldr x21, [x17, #0x108]\n" + "fmla v17.8h, v20.8h, v2.h[3]\n" + "ldr d20, [x17, #0xf0]\n" + "mov v20.d[1], x20\n" + "fmla v10.8h, v21.8h, v0.h[3]\n" + "fmla v14.8h, v21.8h, v1.h[3]\n" + "ldr x20, [x17, #0x118]\n" + "fmla v18.8h, v21.8h, v2.h[3]\n" + "ldr d21, [x17, #0x100]\n" + "fmla v11.8h, v20.8h, v0.h[3]\n" + "mov v21.d[1], x21\n" + "fmla v15.8h, v20.8h, v1.h[3]\n" + "ldr x21, [x17, #0x128]\n" + "fmla v19.8h, v20.8h, v2.h[3]\n" + "ldr d20, [x17, #0x110]\n" + "mov v20.d[1], x20\n" + "fmla v8.8h, v21.8h, v0.h[4]\n" + "fmla v12.8h, v21.8h, v1.h[4]\n" + "ldr x20, [x17, #0x138]\n" + "fmla v16.8h, v21.8h, v2.h[4]\n" + "ldr d21, [x17, #0x120]\n" + "fmla v9.8h, v20.8h, v0.h[4]\n" + "mov v21.d[1], x21\n" + "fmla v13.8h, v20.8h, v1.h[4]\n" + "ldr x21, [x17, #0x148]\n" + "fmla v17.8h, v20.8h, v2.h[4]\n" + "ldr d20, [x17, #0x130]\n" + "mov v20.d[1], x20\n" + "fmla v10.8h, v21.8h, v0.h[4]\n" + "fmla v14.8h, v21.8h, v1.h[4]\n" + "ldr x20, [x17, #0x158]\n" + "fmla v18.8h, v21.8h, v2.h[4]\n" + "ldr d21, [x17, #0x140]\n" + "fmla v11.8h, v20.8h, v0.h[4]\n" + "mov v21.d[1], x21\n" + "fmla v15.8h, v20.8h, v1.h[4]\n" + "ldr x21, [x17, #0x168]\n" + "fmla v19.8h, v20.8h, v2.h[4]\n" + "ldr d20, [x17, #0x150]\n" + "mov v20.d[1], x20\n" + "fmla v8.8h, v21.8h, v0.h[5]\n" + "fmla v12.8h, v21.8h, v1.h[5]\n" + "ldr x20, [x17, #0x178]\n" + "fmla v16.8h, v21.8h, v2.h[5]\n" + "ldr d21, [x17, #0x160]\n" + "fmla v9.8h, v20.8h, v0.h[5]\n" + "mov v21.d[1], x21\n" + "fmla v13.8h, v20.8h, v1.h[5]\n" + "ldr x21, [x17, #0x188]\n" + "fmla v17.8h, v20.8h, v2.h[5]\n" + "ldr d20, [x17, #0x170]\n" + "mov v20.d[1], x20\n" + "fmla v10.8h, v21.8h, v0.h[5]\n" + "fmla v14.8h, v21.8h, v1.h[5]\n" + "ldr x20, [x17, #0x198]\n" + "fmla v18.8h, v21.8h, v2.h[5]\n" + "ldr d21, [x17, #0x180]\n" + "fmla v11.8h, v20.8h, v0.h[5]\n" + "mov v21.d[1], x21\n" + "fmla v15.8h, v20.8h, v1.h[5]\n" + "ldr x21, [x17, #0x1a8]\n" + "fmla v19.8h, v20.8h, v2.h[5]\n" + "ldr d20, [x17, #0x190]\n" + "mov v20.d[1], x20\n" + "fmla v8.8h, v21.8h, v0.h[6]\n" + "fmla v12.8h, v21.8h, v1.h[6]\n" + "ldr x20, [x17, #0x1b8]\n" + "fmla v16.8h, v21.8h, v2.h[6]\n" + "ldr d21, [x17, #0x1a0]\n" + "fmla v9.8h, v20.8h, v0.h[6]\n" + "mov v21.d[1], x21\n" + "fmla v13.8h, v20.8h, v1.h[6]\n" + "ldr x21, [x17, #0x1c8]\n" + "fmla v17.8h, v20.8h, v2.h[6]\n" + "ldr d20, [x17, #0x1b0]\n" + "mov v20.d[1], x20\n" + "fmla v10.8h, v21.8h, v0.h[6]\n" + "fmla v14.8h, v21.8h, v1.h[6]\n" + "ldr x20, [x17, #0x1d8]\n" + "fmla v18.8h, v21.8h, v2.h[6]\n" + "ldr d21, [x17, #0x1c0]\n" + "fmla v11.8h, v20.8h, v0.h[6]\n" + "mov v21.d[1], x21\n" + "fmla v15.8h, v20.8h, v1.h[6]\n" + "ldr x21, [x17, #0x1e8]\n" + "fmla v19.8h, v20.8h, v2.h[6]\n" + "ldr d20, [x17, #0x1d0]\n" + "mov v20.d[1], x20\n" + "fmla v8.8h, v21.8h, v0.h[7]\n" + "fmla v12.8h, v21.8h, v1.h[7]\n" + "ldr x20, [x17, #0x1f8]\n" + "fmla v16.8h, v21.8h, v2.h[7]\n" + "ldr d21, [x17, #0x1e0]\n" + "fmla v9.8h, v20.8h, v0.h[7]\n" + "mov v21.d[1], x21\n" + "fmla v13.8h, v20.8h, v1.h[7]\n" "add x13, x13, #0x10\n" - "fmla v17.8h, v7.8h, v2.h[7]\n" - "ldr d7, [x17, #0x1f0]\n" - "mov v7.d[1], x11\n" - "add x9, x9, #0x10\n" - "add x27, x27, #0x10\n" + "fmla v17.8h, v20.8h, v2.h[7]\n" + "ldr d20, [x17, #0x1f0]\n" + "mov v20.d[1], x20\n" + "add x12, x12, #0x10\n" + "add x11, x11, #0x10\n" "add x17, x17, #0x200\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "ldr x12, [x17, #0x8]\n" - "fmla v14.8h, v6.8h, v1.h[7]\n" - "ldr x10, [x13, #0x8]\n" - "fmla v18.8h, v6.8h, v2.h[7]\n" + "fmla v10.8h, v21.8h, v0.h[7]\n" + "ldr x20, [x17, #0x8]\n" + "fmla v14.8h, v21.8h, v1.h[7]\n" + "ldr x23, [x13, #0x8]\n" + "fmla v18.8h, v21.8h, v2.h[7]\n" "ldr d6, [x17, #0x0]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" + "fmla v11.8h, v20.8h, v0.h[7]\n" "ldr d0, [x13, #0x0]\n" - "fmla v15.8h, v7.8h, v1.h[7]\n" - "ldr d1, [x9, #0x0]\n" - "ldr x28, [x9, #0x8]\n" - "fmla v19.8h, v7.8h, v2.h[7]\n" - "ldr d2, [x27, #0x0]\n" + "fmla v15.8h, v20.8h, v1.h[7]\n" + "ldr d1, [x12, #0x0]\n" + "ldr x22, [x12, #0x8]\n" + "fmla v19.8h, v20.8h, v2.h[7]\n" + "ldr d2, [x11, #0x0]\n" "sub x14, x14, #0x8\n" "ldr d7, [x17, #0x10]\n" "cmp x14, #0x10\n" - "ldr x26, [x27, #0x8]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x18]\n" - "mov v0.d[1], x10\n" + "ldr x21, [x11, #0x8]\n" + "mov v6.d[1], x20\n" + "ldr x20, [x17, #0x18]\n" + "mov v0.d[1], x23\n" "prfm pldl1keep, [x13, #0x80]\n" - "mov v1.d[1], x28\n" - "prfm pldl1keep, [x9, #0x80]\n" - "mov v2.d[1], x26\n" - "prfm pldl1keep, [x27, #0x80]\n" - "mov v7.d[1], x11\n" + "mov v1.d[1], x22\n" + "prfm pldl1keep, [x12, #0x80]\n" + "mov v2.d[1], x21\n" + "prfm pldl1keep, [x11, #0x80]\n" + "mov v7.d[1], x20\n" "bge 124b\n" "125:" // Height 3: Multiply loop: Single iteration only "fmla v8.8h, v6.8h, v0.h[0]\n" "add x13, x13, #0x10\n" "fmla v12.8h, v6.8h, v1.h[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" "fmla v16.8h, v6.8h, v2.h[0]\n" - "ldr q6, [x17, #0x20]\n" + "ldr q21, [x17, #0x20]\n" "fmla v9.8h, v7.8h, v0.h[0]\n" - "add x27, x27, #0x10\n" + "add x11, x11, #0x10\n" "fmla v13.8h, v7.8h, v1.h[0]\n" "sub x14, x14, #0x8\n" "fmla v17.8h, v7.8h, v2.h[0]\n" - "ldr q7, [x17, #0x30]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" + "ldr q20, [x17, #0x30]\n" + "fmla v10.8h, v21.8h, v0.h[0]\n" "prfm pldl1keep, [x13, #0x80]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "prfm pldl1keep, [x9, #0x80]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "ldr q6, [x17, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "prfm pldl1keep, [x27, #0x80]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "ldr q7, [x17, #0x50]\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "fmla v16.8h, v6.8h, v2.h[1]\n" - "ldr q6, [x17, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "fmla v17.8h, v7.8h, v2.h[1]\n" - "ldr q7, [x17, #0x70]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "fmla v14.8h, v6.8h, v1.h[1]\n" - "fmla v18.8h, v6.8h, v2.h[1]\n" - "ldr q6, [x17, #0x80]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "fmla v15.8h, v7.8h, v1.h[1]\n" - "fmla v19.8h, v7.8h, v2.h[1]\n" - "ldr q7, [x17, #0x90]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "fmla v12.8h, v6.8h, v1.h[2]\n" - "fmla v16.8h, v6.8h, v2.h[2]\n" - "ldr q6, [x17, #0xa0]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "fmla v13.8h, v7.8h, v1.h[2]\n" - "fmla v17.8h, v7.8h, v2.h[2]\n" - "ldr q7, [x17, #0xb0]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "fmla v14.8h, v6.8h, v1.h[2]\n" - "fmla v18.8h, v6.8h, v2.h[2]\n" - "ldr q6, [x17, #0xc0]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "fmla v15.8h, v7.8h, v1.h[2]\n" - "fmla v19.8h, v7.8h, v2.h[2]\n" - "ldr q7, [x17, #0xd0]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "fmla v12.8h, v6.8h, v1.h[3]\n" - "fmla v16.8h, v6.8h, v2.h[3]\n" - "ldr q6, [x17, #0xe0]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "fmla v13.8h, v7.8h, v1.h[3]\n" - "fmla v17.8h, v7.8h, v2.h[3]\n" - "ldr q7, [x17, #0xf0]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "fmla v14.8h, v6.8h, v1.h[3]\n" - "fmla v18.8h, v6.8h, v2.h[3]\n" - "ldr q6, [x17, #0x100]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "fmla v15.8h, v7.8h, v1.h[3]\n" - "fmla v19.8h, v7.8h, v2.h[3]\n" - "ldr q7, [x17, #0x110]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "fmla v12.8h, v6.8h, v1.h[4]\n" - "fmla v16.8h, v6.8h, v2.h[4]\n" - "ldr q6, [x17, #0x120]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "fmla v13.8h, v7.8h, v1.h[4]\n" - "fmla v17.8h, v7.8h, v2.h[4]\n" - "ldr q7, [x17, #0x130]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "fmla v14.8h, v6.8h, v1.h[4]\n" - "fmla v18.8h, v6.8h, v2.h[4]\n" - "ldr q6, [x17, #0x140]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "fmla v15.8h, v7.8h, v1.h[4]\n" - "fmla v19.8h, v7.8h, v2.h[4]\n" - "ldr q7, [x17, #0x150]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "fmla v12.8h, v6.8h, v1.h[5]\n" - "fmla v16.8h, v6.8h, v2.h[5]\n" - "ldr q6, [x17, #0x160]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "fmla v13.8h, v7.8h, v1.h[5]\n" - "fmla v17.8h, v7.8h, v2.h[5]\n" - "ldr q7, [x17, #0x170]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "fmla v14.8h, v6.8h, v1.h[5]\n" - "fmla v18.8h, v6.8h, v2.h[5]\n" - "ldr q6, [x17, #0x180]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "fmla v15.8h, v7.8h, v1.h[5]\n" - "fmla v19.8h, v7.8h, v2.h[5]\n" - "ldr q7, [x17, #0x190]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "fmla v12.8h, v6.8h, v1.h[6]\n" - "fmla v16.8h, v6.8h, v2.h[6]\n" - "ldr q6, [x17, #0x1a0]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "fmla v13.8h, v7.8h, v1.h[6]\n" - "fmla v17.8h, v7.8h, v2.h[6]\n" - "ldr q7, [x17, #0x1b0]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "fmla v14.8h, v6.8h, v1.h[6]\n" - "fmla v18.8h, v6.8h, v2.h[6]\n" - "ldr q6, [x17, #0x1c0]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "fmla v15.8h, v7.8h, v1.h[6]\n" - "fmla v19.8h, v7.8h, v2.h[6]\n" - "ldr q7, [x17, #0x1d0]\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "fmla v12.8h, v6.8h, v1.h[7]\n" - "fmla v16.8h, v6.8h, v2.h[7]\n" - "ldr q6, [x17, #0x1e0]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "fmla v13.8h, v7.8h, v1.h[7]\n" - "fmla v17.8h, v7.8h, v2.h[7]\n" - "ldr q7, [x17, #0x1f0]\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" + "fmla v14.8h, v21.8h, v1.h[0]\n" + "prfm pldl1keep, [x12, #0x80]\n" + "fmla v18.8h, v21.8h, v2.h[0]\n" + "ldr q21, [x17, #0x40]\n" + "fmla v11.8h, v20.8h, v0.h[0]\n" + "prfm pldl1keep, [x11, #0x80]\n" + "fmla v15.8h, v20.8h, v1.h[0]\n" + "fmla v19.8h, v20.8h, v2.h[0]\n" + "ldr q20, [x17, #0x50]\n" + "fmla v8.8h, v21.8h, v0.h[1]\n" + "fmla v12.8h, v21.8h, v1.h[1]\n" + "fmla v16.8h, v21.8h, v2.h[1]\n" + "ldr q21, [x17, #0x60]\n" + "fmla v9.8h, v20.8h, v0.h[1]\n" + "fmla v13.8h, v20.8h, v1.h[1]\n" + "fmla v17.8h, v20.8h, v2.h[1]\n" + "ldr q20, [x17, #0x70]\n" + "fmla v10.8h, v21.8h, v0.h[1]\n" + "fmla v14.8h, v21.8h, v1.h[1]\n" + "fmla v18.8h, v21.8h, v2.h[1]\n" + "ldr q21, [x17, #0x80]\n" + "fmla v11.8h, v20.8h, v0.h[1]\n" + "fmla v15.8h, v20.8h, v1.h[1]\n" + "fmla v19.8h, v20.8h, v2.h[1]\n" + "ldr q20, [x17, #0x90]\n" + "fmla v8.8h, v21.8h, v0.h[2]\n" + "fmla v12.8h, v21.8h, v1.h[2]\n" + "fmla v16.8h, v21.8h, v2.h[2]\n" + "ldr q21, [x17, #0xa0]\n" + "fmla v9.8h, v20.8h, v0.h[2]\n" + "fmla v13.8h, v20.8h, v1.h[2]\n" + "fmla v17.8h, v20.8h, v2.h[2]\n" + "ldr q20, [x17, #0xb0]\n" + "fmla v10.8h, v21.8h, v0.h[2]\n" + "fmla v14.8h, v21.8h, v1.h[2]\n" + "fmla v18.8h, v21.8h, v2.h[2]\n" + "ldr q21, [x17, #0xc0]\n" + "fmla v11.8h, v20.8h, v0.h[2]\n" + "fmla v15.8h, v20.8h, v1.h[2]\n" + "fmla v19.8h, v20.8h, v2.h[2]\n" + "ldr q20, [x17, #0xd0]\n" + "fmla v8.8h, v21.8h, v0.h[3]\n" + "fmla v12.8h, v21.8h, v1.h[3]\n" + "fmla v16.8h, v21.8h, v2.h[3]\n" + "ldr q21, [x17, #0xe0]\n" + "fmla v9.8h, v20.8h, v0.h[3]\n" + "fmla v13.8h, v20.8h, v1.h[3]\n" + "fmla v17.8h, v20.8h, v2.h[3]\n" + "ldr q20, [x17, #0xf0]\n" + "fmla v10.8h, v21.8h, v0.h[3]\n" + "fmla v14.8h, v21.8h, v1.h[3]\n" + "fmla v18.8h, v21.8h, v2.h[3]\n" + "ldr q21, [x17, #0x100]\n" + "fmla v11.8h, v20.8h, v0.h[3]\n" + "fmla v15.8h, v20.8h, v1.h[3]\n" + "fmla v19.8h, v20.8h, v2.h[3]\n" + "ldr q20, [x17, #0x110]\n" + "fmla v8.8h, v21.8h, v0.h[4]\n" + "fmla v12.8h, v21.8h, v1.h[4]\n" + "fmla v16.8h, v21.8h, v2.h[4]\n" + "ldr q21, [x17, #0x120]\n" + "fmla v9.8h, v20.8h, v0.h[4]\n" + "fmla v13.8h, v20.8h, v1.h[4]\n" + "fmla v17.8h, v20.8h, v2.h[4]\n" + "ldr q20, [x17, #0x130]\n" + "fmla v10.8h, v21.8h, v0.h[4]\n" + "fmla v14.8h, v21.8h, v1.h[4]\n" + "fmla v18.8h, v21.8h, v2.h[4]\n" + "ldr q21, [x17, #0x140]\n" + "fmla v11.8h, v20.8h, v0.h[4]\n" + "fmla v15.8h, v20.8h, v1.h[4]\n" + "fmla v19.8h, v20.8h, v2.h[4]\n" + "ldr q20, [x17, #0x150]\n" + "fmla v8.8h, v21.8h, v0.h[5]\n" + "fmla v12.8h, v21.8h, v1.h[5]\n" + "fmla v16.8h, v21.8h, v2.h[5]\n" + "ldr q21, [x17, #0x160]\n" + "fmla v9.8h, v20.8h, v0.h[5]\n" + "fmla v13.8h, v20.8h, v1.h[5]\n" + "fmla v17.8h, v20.8h, v2.h[5]\n" + "ldr q20, [x17, #0x170]\n" + "fmla v10.8h, v21.8h, v0.h[5]\n" + "fmla v14.8h, v21.8h, v1.h[5]\n" + "fmla v18.8h, v21.8h, v2.h[5]\n" + "ldr q21, [x17, #0x180]\n" + "fmla v11.8h, v20.8h, v0.h[5]\n" + "fmla v15.8h, v20.8h, v1.h[5]\n" + "fmla v19.8h, v20.8h, v2.h[5]\n" + "ldr q20, [x17, #0x190]\n" + "fmla v8.8h, v21.8h, v0.h[6]\n" + "fmla v12.8h, v21.8h, v1.h[6]\n" + "fmla v16.8h, v21.8h, v2.h[6]\n" + "ldr q21, [x17, #0x1a0]\n" + "fmla v9.8h, v20.8h, v0.h[6]\n" + "fmla v13.8h, v20.8h, v1.h[6]\n" + "fmla v17.8h, v20.8h, v2.h[6]\n" + "ldr q20, [x17, #0x1b0]\n" + "fmla v10.8h, v21.8h, v0.h[6]\n" + "fmla v14.8h, v21.8h, v1.h[6]\n" + "fmla v18.8h, v21.8h, v2.h[6]\n" + "ldr q21, [x17, #0x1c0]\n" + "fmla v11.8h, v20.8h, v0.h[6]\n" + "fmla v15.8h, v20.8h, v1.h[6]\n" + "fmla v19.8h, v20.8h, v2.h[6]\n" + "ldr q20, [x17, #0x1d0]\n" + "fmla v8.8h, v21.8h, v0.h[7]\n" + "fmla v12.8h, v21.8h, v1.h[7]\n" + "fmla v16.8h, v21.8h, v2.h[7]\n" + "ldr q21, [x17, #0x1e0]\n" + "fmla v9.8h, v20.8h, v0.h[7]\n" + "fmla v13.8h, v20.8h, v1.h[7]\n" + "fmla v17.8h, v20.8h, v2.h[7]\n" + "ldr q20, [x17, #0x1f0]\n" + "fmla v10.8h, v21.8h, v0.h[7]\n" "add x17, x17, #0x200\n" - "fmla v14.8h, v6.8h, v1.h[7]\n" - "fmla v18.8h, v6.8h, v2.h[7]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" - "fmla v15.8h, v7.8h, v1.h[7]\n" - "fmla v19.8h, v7.8h, v2.h[7]\n" + "fmla v14.8h, v21.8h, v1.h[7]\n" + "fmla v18.8h, v21.8h, v2.h[7]\n" + "fmla v11.8h, v20.8h, v0.h[7]\n" + "fmla v15.8h, v20.8h, v1.h[7]\n" + "fmla v19.8h, v20.8h, v2.h[7]\n" "126:" // Height 3: Multiply loop: Main loop skip "cbz x14, 128f\n" "127:" // Height 3: Multiply loop: Odd block loop - "ldr h0, [x13], #0x2\n" + "ldr h2, [x13], #0x2\n" "sub x14, x14, #0x1\n" - "ldr h1, [x9], #0x2\n" - "ldr h2, [x27], #0x2\n" - "ldr q6, [x17, #0x0]\n" - "fmla v8.8h, v6.8h, v0.h[0]\n" - "ldr q7, [x17, #0x10]\n" - "fmla v12.8h, v6.8h, v1.h[0]\n" - "fmla v16.8h, v6.8h, v2.h[0]\n" - "ldr q6, [x17, #0x20]\n" - "fmla v9.8h, v7.8h, v0.h[0]\n" - "fmla v13.8h, v7.8h, v1.h[0]\n" - "fmla v17.8h, v7.8h, v2.h[0]\n" - "ldr q7, [x17, #0x30]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" + "ldr h1, [x12], #0x2\n" + "ldr h0, [x11], #0x2\n" + "ldr q21, [x17, #0x0]\n" + "fmla v8.8h, v21.8h, v2.h[0]\n" + "ldr q20, [x17, #0x10]\n" + "fmla v12.8h, v21.8h, v1.h[0]\n" + "fmla v16.8h, v21.8h, v0.h[0]\n" + "ldr q21, [x17, #0x20]\n" + "fmla v9.8h, v20.8h, v2.h[0]\n" + "fmla v13.8h, v20.8h, v1.h[0]\n" + "fmla v17.8h, v20.8h, v0.h[0]\n" + "ldr q20, [x17, #0x30]\n" + "fmla v10.8h, v21.8h, v2.h[0]\n" "add x17, x17, #0x40\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" + "fmla v14.8h, v21.8h, v1.h[0]\n" + "fmla v18.8h, v21.8h, v0.h[0]\n" + "fmla v11.8h, v20.8h, v2.h[0]\n" + "fmla v15.8h, v20.8h, v1.h[0]\n" + "fmla v19.8h, v20.8h, v0.h[0]\n" "cbnz x14, 127b\n" "128:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -1938,33 +1938,33 @@ void a64_hybrid_fp16_mla_6x32_a55 ( "prfm pstl1keep, [x24, #0x0]\n" "tbz %x[flags], #1, 129f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v0.8h }, [x20]\n" - "fmin v8.8h, v8.8h, v0.8h\n" - "fmin v9.8h, v9.8h, v0.8h\n" - "fmin v10.8h, v10.8h, v0.8h\n" - "fmin v11.8h, v11.8h, v0.8h\n" - "fmin v12.8h, v12.8h, v0.8h\n" - "fmin v13.8h, v13.8h, v0.8h\n" - "fmin v14.8h, v14.8h, v0.8h\n" - "fmin v15.8h, v15.8h, v0.8h\n" - "fmin v16.8h, v16.8h, v0.8h\n" - "fmin v17.8h, v17.8h, v0.8h\n" - "fmin v18.8h, v18.8h, v0.8h\n" - "fmin v19.8h, v19.8h, v0.8h\n" + "ld1r { v20.8h }, [x20]\n" + "fmin v8.8h, v8.8h, v20.8h\n" + "fmin v9.8h, v9.8h, v20.8h\n" + "fmin v10.8h, v10.8h, v20.8h\n" + "fmin v11.8h, v11.8h, v20.8h\n" + "fmin v12.8h, v12.8h, v20.8h\n" + "fmin v13.8h, v13.8h, v20.8h\n" + "fmin v14.8h, v14.8h, v20.8h\n" + "fmin v15.8h, v15.8h, v20.8h\n" + "fmin v16.8h, v16.8h, v20.8h\n" + "fmin v17.8h, v17.8h, v20.8h\n" + "fmin v18.8h, v18.8h, v20.8h\n" + "fmin v19.8h, v19.8h, v20.8h\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.8h }, [x20]\n" - "fmax v8.8h, v8.8h, v0.8h\n" - "fmax v9.8h, v9.8h, v0.8h\n" - "fmax v10.8h, v10.8h, v0.8h\n" - "fmax v11.8h, v11.8h, v0.8h\n" - "fmax v12.8h, v12.8h, v0.8h\n" - "fmax v13.8h, v13.8h, v0.8h\n" - "fmax v14.8h, v14.8h, v0.8h\n" - "fmax v15.8h, v15.8h, v0.8h\n" - "fmax v16.8h, v16.8h, v0.8h\n" - "fmax v17.8h, v17.8h, v0.8h\n" - "fmax v18.8h, v18.8h, v0.8h\n" - "fmax v19.8h, v19.8h, v0.8h\n" + "ld1r { v20.8h }, [x20]\n" + "fmax v8.8h, v8.8h, v20.8h\n" + "fmax v9.8h, v9.8h, v20.8h\n" + "fmax v10.8h, v10.8h, v20.8h\n" + "fmax v11.8h, v11.8h, v20.8h\n" + "fmax v12.8h, v12.8h, v20.8h\n" + "fmax v13.8h, v13.8h, v20.8h\n" + "fmax v14.8h, v14.8h, v20.8h\n" + "fmax v15.8h, v15.8h, v20.8h\n" + "fmax v16.8h, v16.8h, v20.8h\n" + "fmax v17.8h, v17.8h, v20.8h\n" + "fmax v18.8h, v18.8h, v20.8h\n" + "fmax v19.8h, v19.8h, v20.8h\n" "129:" // Height 3: No activation "cmp x8, #0x20\n" "bge 146f\n" @@ -2424,484 +2424,484 @@ void a64_hybrid_fp16_mla_6x32_a55 ( "170:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w14, [x20, x15, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 171f\n" - "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x13, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" - "ldr x27, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x15, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x13, [x20, #0x0]\n" + "ldr x12, [x20, #0x8]\n" + "ldr x11, [x20, #0x10]\n" + "ldr x10, [x20, #0x18]\n" "cbnz x15, 172f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x13, x13, x20, LSL #1\n" - "add x9, x9, x20, LSL #1\n" - "add x27, x27, x20, LSL #1\n" - "add x25, x25, x20, LSL #1\n" + "add x12, x12, x20, LSL #1\n" + "add x11, x11, x20, LSL #1\n" + "add x10, x10, x20, LSL #1\n" "b 172f\n" "171:" // Height 4: setup direct input "mov x13, %x[input_ptr]\n" - "add x9, x13, x20, LSL #1\n" - "add x27, x9, x20, LSL #1\n" - "add x25, x27, x20, LSL #1\n" + "add x12, x13, x21, LSL #1\n" + "add x11, x12, x21, LSL #1\n" + "add x10, x11, x21, LSL #1\n" "172:" // Height 4: input setup done "cmp x14, #0x8\n" "blt 175f\n" "ldr q0, [x13, #0x0]\n" "cmp x14, #0x10\n" - "ldr q1, [x9, #0x0]\n" - "ldr q2, [x27, #0x0]\n" - "ldr q3, [x25, #0x0]\n" + "ldr q1, [x12, #0x0]\n" + "ldr q2, [x11, #0x0]\n" + "ldr q3, [x10, #0x0]\n" "ldr q6, [x17, #0x0]\n" "ldr q7, [x17, #0x10]\n" "blt 174f\n" "173:" // Height 4: Multiply loop: Main loop head "fmla v8.8h, v6.8h, v0.h[0]\n" - "ldr x12, [x17, #0x28]\n" + "ldr x21, [x17, #0x28]\n" "fmla v12.8h, v6.8h, v1.h[0]\n" - "ldr x11, [x17, #0x38]\n" + "ldr x20, [x17, #0x38]\n" "fmla v16.8h, v6.8h, v2.h[0]\n" "add x13, x13, #0x10\n" "fmla v20.8h, v6.8h, v3.h[0]\n" - "ldr d6, [x17, #0x20]\n" + "ldr d25, [x17, #0x20]\n" "fmla v9.8h, v7.8h, v0.h[0]\n" - "mov v6.d[1], x12\n" + "mov v25.d[1], x21\n" "fmla v13.8h, v7.8h, v1.h[0]\n" - "ldr x12, [x17, #0x48]\n" + "ldr x21, [x17, #0x48]\n" "fmla v17.8h, v7.8h, v2.h[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" "fmla v21.8h, v7.8h, v3.h[0]\n" - "ldr d7, [x17, #0x30]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "ldr x11, [x17, #0x58]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "add x27, x27, #0x10\n" - "fmla v22.8h, v6.8h, v3.h[0]\n" - "ldr d6, [x17, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "mov v6.d[1], x12\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "ldr x12, [x17, #0x68]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "add x25, x25, #0x10\n" - "fmla v23.8h, v7.8h, v3.h[0]\n" - "ldr d7, [x17, #0x50]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "ldr x11, [x17, #0x78]\n" - "fmla v16.8h, v6.8h, v2.h[1]\n" - "ldr x10, [x13, #0x8]\n" - "fmla v20.8h, v6.8h, v3.h[1]\n" - "ldr d6, [x17, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "mov v6.d[1], x12\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "ldr x12, [x17, #0x88]\n" - "fmla v17.8h, v7.8h, v2.h[1]\n" - "ldr x28, [x9, #0x8]\n" - "fmla v21.8h, v7.8h, v3.h[1]\n" - "ldr d7, [x17, #0x70]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "fmla v14.8h, v6.8h, v1.h[1]\n" - "ldr x11, [x17, #0x98]\n" - "fmla v18.8h, v6.8h, v2.h[1]\n" - "ldr x26, [x27, #0x8]\n" - "fmla v22.8h, v6.8h, v3.h[1]\n" - "ldr d6, [x17, #0x80]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "mov v6.d[1], x12\n" - "fmla v15.8h, v7.8h, v1.h[1]\n" - "ldr x12, [x17, #0xa8]\n" - "fmla v19.8h, v7.8h, v2.h[1]\n" - "ldr x24, [x25, #0x8]\n" - "fmla v23.8h, v7.8h, v3.h[1]\n" - "ldr d7, [x17, #0x90]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "fmla v12.8h, v6.8h, v1.h[2]\n" - "ldr x11, [x17, #0xb8]\n" - "fmla v16.8h, v6.8h, v2.h[2]\n" + "ldr d24, [x17, #0x30]\n" + "mov v24.d[1], x20\n" + "fmla v10.8h, v25.8h, v0.h[0]\n" + "fmla v14.8h, v25.8h, v1.h[0]\n" + "ldr x20, [x17, #0x58]\n" + "fmla v18.8h, v25.8h, v2.h[0]\n" + "add x11, x11, #0x10\n" + "fmla v22.8h, v25.8h, v3.h[0]\n" + "ldr d25, [x17, #0x40]\n" + "fmla v11.8h, v24.8h, v0.h[0]\n" + "mov v25.d[1], x21\n" + "fmla v15.8h, v24.8h, v1.h[0]\n" + "ldr x21, [x17, #0x68]\n" + "fmla v19.8h, v24.8h, v2.h[0]\n" + "add x10, x10, #0x10\n" + "fmla v23.8h, v24.8h, v3.h[0]\n" + "ldr d24, [x17, #0x50]\n" + "mov v24.d[1], x20\n" + "fmla v8.8h, v25.8h, v0.h[1]\n" + "fmla v12.8h, v25.8h, v1.h[1]\n" + "ldr x20, [x17, #0x78]\n" + "fmla v16.8h, v25.8h, v2.h[1]\n" + "ldr x25, [x13, #0x8]\n" + "fmla v20.8h, v25.8h, v3.h[1]\n" + "ldr d25, [x17, #0x60]\n" + "fmla v9.8h, v24.8h, v0.h[1]\n" + "mov v25.d[1], x21\n" + "fmla v13.8h, v24.8h, v1.h[1]\n" + "ldr x21, [x17, #0x88]\n" + "fmla v17.8h, v24.8h, v2.h[1]\n" + "ldr x24, [x12, #0x8]\n" + "fmla v21.8h, v24.8h, v3.h[1]\n" + "ldr d24, [x17, #0x70]\n" + "mov v24.d[1], x20\n" + "fmla v10.8h, v25.8h, v0.h[1]\n" + "fmla v14.8h, v25.8h, v1.h[1]\n" + "ldr x20, [x17, #0x98]\n" + "fmla v18.8h, v25.8h, v2.h[1]\n" + "ldr x23, [x11, #0x8]\n" + "fmla v22.8h, v25.8h, v3.h[1]\n" + "ldr d25, [x17, #0x80]\n" + "fmla v11.8h, v24.8h, v0.h[1]\n" + "mov v25.d[1], x21\n" + "fmla v15.8h, v24.8h, v1.h[1]\n" + "ldr x21, [x17, #0xa8]\n" + "fmla v19.8h, v24.8h, v2.h[1]\n" + "ldr x22, [x10, #0x8]\n" + "fmla v23.8h, v24.8h, v3.h[1]\n" + "ldr d24, [x17, #0x90]\n" + "mov v24.d[1], x20\n" + "fmla v8.8h, v25.8h, v0.h[2]\n" + "fmla v12.8h, v25.8h, v1.h[2]\n" + "ldr x20, [x17, #0xb8]\n" + "fmla v16.8h, v25.8h, v2.h[2]\n" "sub x14, x14, #0x8\n" - "fmla v20.8h, v6.8h, v3.h[2]\n" - "ldr d6, [x17, #0xa0]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "mov v6.d[1], x12\n" - "fmla v13.8h, v7.8h, v1.h[2]\n" - "ldr x12, [x17, #0xc8]\n" - "fmla v17.8h, v7.8h, v2.h[2]\n" + "fmla v20.8h, v25.8h, v3.h[2]\n" + "ldr d25, [x17, #0xa0]\n" + "fmla v9.8h, v24.8h, v0.h[2]\n" + "mov v25.d[1], x21\n" + "fmla v13.8h, v24.8h, v1.h[2]\n" + "ldr x21, [x17, #0xc8]\n" + "fmla v17.8h, v24.8h, v2.h[2]\n" "cmp x14, #0x10\n" - "fmla v21.8h, v7.8h, v3.h[2]\n" - "ldr d7, [x17, #0xb0]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "fmla v14.8h, v6.8h, v1.h[2]\n" - "ldr x11, [x17, #0xd8]\n" - "fmla v18.8h, v6.8h, v2.h[2]\n" + "fmla v21.8h, v24.8h, v3.h[2]\n" + "ldr d24, [x17, #0xb0]\n" + "mov v24.d[1], x20\n" + "fmla v10.8h, v25.8h, v0.h[2]\n" + "fmla v14.8h, v25.8h, v1.h[2]\n" + "ldr x20, [x17, #0xd8]\n" + "fmla v18.8h, v25.8h, v2.h[2]\n" "prfm pldl1keep, [x13, #0x80]\n" - "fmla v22.8h, v6.8h, v3.h[2]\n" - "ldr d6, [x17, #0xc0]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "mov v6.d[1], x12\n" - "fmla v15.8h, v7.8h, v1.h[2]\n" - "ldr x12, [x17, #0xe8]\n" - "fmla v19.8h, v7.8h, v2.h[2]\n" - "prfm pldl1keep, [x9, #0x80]\n" - "fmla v23.8h, v7.8h, v3.h[2]\n" - "ldr d7, [x17, #0xd0]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "fmla v12.8h, v6.8h, v1.h[3]\n" - "ldr x11, [x17, #0xf8]\n" - "fmla v16.8h, v6.8h, v2.h[3]\n" - "prfm pldl1keep, [x27, #0x80]\n" - "fmla v20.8h, v6.8h, v3.h[3]\n" - "ldr d6, [x17, #0xe0]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "mov v6.d[1], x12\n" - "fmla v13.8h, v7.8h, v1.h[3]\n" - "ldr x12, [x17, #0x108]\n" - "fmla v17.8h, v7.8h, v2.h[3]\n" - "prfm pldl1keep, [x25, #0x80]\n" - "fmla v21.8h, v7.8h, v3.h[3]\n" - "ldr d7, [x17, #0xf0]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "fmla v14.8h, v6.8h, v1.h[3]\n" - "ldr x11, [x17, #0x118]\n" - "fmla v18.8h, v6.8h, v2.h[3]\n" - "fmla v22.8h, v6.8h, v3.h[3]\n" - "ldr d6, [x17, #0x100]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "mov v6.d[1], x12\n" - "fmla v15.8h, v7.8h, v1.h[3]\n" - "ldr x12, [x17, #0x128]\n" - "fmla v19.8h, v7.8h, v2.h[3]\n" - "fmla v23.8h, v7.8h, v3.h[3]\n" - "ldr d7, [x17, #0x110]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "fmla v12.8h, v6.8h, v1.h[4]\n" - "ldr x11, [x17, #0x138]\n" - "fmla v16.8h, v6.8h, v2.h[4]\n" - "fmla v20.8h, v6.8h, v3.h[4]\n" - "ldr d6, [x17, #0x120]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "mov v6.d[1], x12\n" - "fmla v13.8h, v7.8h, v1.h[4]\n" - "ldr x12, [x17, #0x148]\n" - "fmla v17.8h, v7.8h, v2.h[4]\n" - "fmla v21.8h, v7.8h, v3.h[4]\n" - "ldr d7, [x17, #0x130]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "fmla v14.8h, v6.8h, v1.h[4]\n" - "ldr x11, [x17, #0x158]\n" - "fmla v18.8h, v6.8h, v2.h[4]\n" - "fmla v22.8h, v6.8h, v3.h[4]\n" - "ldr d6, [x17, #0x140]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "mov v6.d[1], x12\n" - "fmla v15.8h, v7.8h, v1.h[4]\n" - "ldr x12, [x17, #0x168]\n" - "fmla v19.8h, v7.8h, v2.h[4]\n" - "fmla v23.8h, v7.8h, v3.h[4]\n" - "ldr d7, [x17, #0x150]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "fmla v12.8h, v6.8h, v1.h[5]\n" - "ldr x11, [x17, #0x178]\n" - "fmla v16.8h, v6.8h, v2.h[5]\n" - "fmla v20.8h, v6.8h, v3.h[5]\n" - "ldr d6, [x17, #0x160]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "mov v6.d[1], x12\n" - "fmla v13.8h, v7.8h, v1.h[5]\n" - "ldr x12, [x17, #0x188]\n" - "fmla v17.8h, v7.8h, v2.h[5]\n" - "fmla v21.8h, v7.8h, v3.h[5]\n" - "ldr d7, [x17, #0x170]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "fmla v14.8h, v6.8h, v1.h[5]\n" - "ldr x11, [x17, #0x198]\n" - "fmla v18.8h, v6.8h, v2.h[5]\n" - "fmla v22.8h, v6.8h, v3.h[5]\n" - "ldr d6, [x17, #0x180]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "mov v6.d[1], x12\n" - "fmla v15.8h, v7.8h, v1.h[5]\n" - "ldr x12, [x17, #0x1a8]\n" - "fmla v19.8h, v7.8h, v2.h[5]\n" - "fmla v23.8h, v7.8h, v3.h[5]\n" - "ldr d7, [x17, #0x190]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "fmla v12.8h, v6.8h, v1.h[6]\n" - "ldr x11, [x17, #0x1b8]\n" - "fmla v16.8h, v6.8h, v2.h[6]\n" - "fmla v20.8h, v6.8h, v3.h[6]\n" - "ldr d6, [x17, #0x1a0]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "mov v6.d[1], x12\n" - "fmla v13.8h, v7.8h, v1.h[6]\n" - "ldr x12, [x17, #0x1c8]\n" - "fmla v17.8h, v7.8h, v2.h[6]\n" - "fmla v21.8h, v7.8h, v3.h[6]\n" - "ldr d7, [x17, #0x1b0]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "fmla v14.8h, v6.8h, v1.h[6]\n" - "ldr x11, [x17, #0x1d8]\n" - "fmla v18.8h, v6.8h, v2.h[6]\n" - "fmla v22.8h, v6.8h, v3.h[6]\n" - "ldr d6, [x17, #0x1c0]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "mov v6.d[1], x12\n" - "fmla v15.8h, v7.8h, v1.h[6]\n" - "ldr x12, [x17, #0x1e8]\n" - "fmla v19.8h, v7.8h, v2.h[6]\n" - "fmla v23.8h, v7.8h, v3.h[6]\n" - "ldr d7, [x17, #0x1d0]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "fmla v12.8h, v6.8h, v1.h[7]\n" - "ldr x11, [x17, #0x1f8]\n" - "fmla v16.8h, v6.8h, v2.h[7]\n" - "fmla v20.8h, v6.8h, v3.h[7]\n" - "ldr d6, [x17, #0x1e0]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "mov v6.d[1], x12\n" - "fmla v13.8h, v7.8h, v1.h[7]\n" - "fmla v17.8h, v7.8h, v2.h[7]\n" - "fmla v21.8h, v7.8h, v3.h[7]\n" - "ldr d7, [x17, #0x1f0]\n" - "mov v7.d[1], x11\n" + "fmla v22.8h, v25.8h, v3.h[2]\n" + "ldr d25, [x17, #0xc0]\n" + "fmla v11.8h, v24.8h, v0.h[2]\n" + "mov v25.d[1], x21\n" + "fmla v15.8h, v24.8h, v1.h[2]\n" + "ldr x21, [x17, #0xe8]\n" + "fmla v19.8h, v24.8h, v2.h[2]\n" + "prfm pldl1keep, [x12, #0x80]\n" + "fmla v23.8h, v24.8h, v3.h[2]\n" + "ldr d24, [x17, #0xd0]\n" + "mov v24.d[1], x20\n" + "fmla v8.8h, v25.8h, v0.h[3]\n" + "fmla v12.8h, v25.8h, v1.h[3]\n" + "ldr x20, [x17, #0xf8]\n" + "fmla v16.8h, v25.8h, v2.h[3]\n" + "prfm pldl1keep, [x11, #0x80]\n" + "fmla v20.8h, v25.8h, v3.h[3]\n" + "ldr d25, [x17, #0xe0]\n" + "fmla v9.8h, v24.8h, v0.h[3]\n" + "mov v25.d[1], x21\n" + "fmla v13.8h, v24.8h, v1.h[3]\n" + "ldr x21, [x17, #0x108]\n" + "fmla v17.8h, v24.8h, v2.h[3]\n" + "prfm pldl1keep, [x10, #0x80]\n" + "fmla v21.8h, v24.8h, v3.h[3]\n" + "ldr d24, [x17, #0xf0]\n" + "mov v24.d[1], x20\n" + "fmla v10.8h, v25.8h, v0.h[3]\n" + "fmla v14.8h, v25.8h, v1.h[3]\n" + "ldr x20, [x17, #0x118]\n" + "fmla v18.8h, v25.8h, v2.h[3]\n" + "fmla v22.8h, v25.8h, v3.h[3]\n" + "ldr d25, [x17, #0x100]\n" + "fmla v11.8h, v24.8h, v0.h[3]\n" + "mov v25.d[1], x21\n" + "fmla v15.8h, v24.8h, v1.h[3]\n" + "ldr x21, [x17, #0x128]\n" + "fmla v19.8h, v24.8h, v2.h[3]\n" + "fmla v23.8h, v24.8h, v3.h[3]\n" + "ldr d24, [x17, #0x110]\n" + "mov v24.d[1], x20\n" + "fmla v8.8h, v25.8h, v0.h[4]\n" + "fmla v12.8h, v25.8h, v1.h[4]\n" + "ldr x20, [x17, #0x138]\n" + "fmla v16.8h, v25.8h, v2.h[4]\n" + "fmla v20.8h, v25.8h, v3.h[4]\n" + "ldr d25, [x17, #0x120]\n" + "fmla v9.8h, v24.8h, v0.h[4]\n" + "mov v25.d[1], x21\n" + "fmla v13.8h, v24.8h, v1.h[4]\n" + "ldr x21, [x17, #0x148]\n" + "fmla v17.8h, v24.8h, v2.h[4]\n" + "fmla v21.8h, v24.8h, v3.h[4]\n" + "ldr d24, [x17, #0x130]\n" + "mov v24.d[1], x20\n" + "fmla v10.8h, v25.8h, v0.h[4]\n" + "fmla v14.8h, v25.8h, v1.h[4]\n" + "ldr x20, [x17, #0x158]\n" + "fmla v18.8h, v25.8h, v2.h[4]\n" + "fmla v22.8h, v25.8h, v3.h[4]\n" + "ldr d25, [x17, #0x140]\n" + "fmla v11.8h, v24.8h, v0.h[4]\n" + "mov v25.d[1], x21\n" + "fmla v15.8h, v24.8h, v1.h[4]\n" + "ldr x21, [x17, #0x168]\n" + "fmla v19.8h, v24.8h, v2.h[4]\n" + "fmla v23.8h, v24.8h, v3.h[4]\n" + "ldr d24, [x17, #0x150]\n" + "mov v24.d[1], x20\n" + "fmla v8.8h, v25.8h, v0.h[5]\n" + "fmla v12.8h, v25.8h, v1.h[5]\n" + "ldr x20, [x17, #0x178]\n" + "fmla v16.8h, v25.8h, v2.h[5]\n" + "fmla v20.8h, v25.8h, v3.h[5]\n" + "ldr d25, [x17, #0x160]\n" + "fmla v9.8h, v24.8h, v0.h[5]\n" + "mov v25.d[1], x21\n" + "fmla v13.8h, v24.8h, v1.h[5]\n" + "ldr x21, [x17, #0x188]\n" + "fmla v17.8h, v24.8h, v2.h[5]\n" + "fmla v21.8h, v24.8h, v3.h[5]\n" + "ldr d24, [x17, #0x170]\n" + "mov v24.d[1], x20\n" + "fmla v10.8h, v25.8h, v0.h[5]\n" + "fmla v14.8h, v25.8h, v1.h[5]\n" + "ldr x20, [x17, #0x198]\n" + "fmla v18.8h, v25.8h, v2.h[5]\n" + "fmla v22.8h, v25.8h, v3.h[5]\n" + "ldr d25, [x17, #0x180]\n" + "fmla v11.8h, v24.8h, v0.h[5]\n" + "mov v25.d[1], x21\n" + "fmla v15.8h, v24.8h, v1.h[5]\n" + "ldr x21, [x17, #0x1a8]\n" + "fmla v19.8h, v24.8h, v2.h[5]\n" + "fmla v23.8h, v24.8h, v3.h[5]\n" + "ldr d24, [x17, #0x190]\n" + "mov v24.d[1], x20\n" + "fmla v8.8h, v25.8h, v0.h[6]\n" + "fmla v12.8h, v25.8h, v1.h[6]\n" + "ldr x20, [x17, #0x1b8]\n" + "fmla v16.8h, v25.8h, v2.h[6]\n" + "fmla v20.8h, v25.8h, v3.h[6]\n" + "ldr d25, [x17, #0x1a0]\n" + "fmla v9.8h, v24.8h, v0.h[6]\n" + "mov v25.d[1], x21\n" + "fmla v13.8h, v24.8h, v1.h[6]\n" + "ldr x21, [x17, #0x1c8]\n" + "fmla v17.8h, v24.8h, v2.h[6]\n" + "fmla v21.8h, v24.8h, v3.h[6]\n" + "ldr d24, [x17, #0x1b0]\n" + "mov v24.d[1], x20\n" + "fmla v10.8h, v25.8h, v0.h[6]\n" + "fmla v14.8h, v25.8h, v1.h[6]\n" + "ldr x20, [x17, #0x1d8]\n" + "fmla v18.8h, v25.8h, v2.h[6]\n" + "fmla v22.8h, v25.8h, v3.h[6]\n" + "ldr d25, [x17, #0x1c0]\n" + "fmla v11.8h, v24.8h, v0.h[6]\n" + "mov v25.d[1], x21\n" + "fmla v15.8h, v24.8h, v1.h[6]\n" + "ldr x21, [x17, #0x1e8]\n" + "fmla v19.8h, v24.8h, v2.h[6]\n" + "fmla v23.8h, v24.8h, v3.h[6]\n" + "ldr d24, [x17, #0x1d0]\n" + "mov v24.d[1], x20\n" + "fmla v8.8h, v25.8h, v0.h[7]\n" + "fmla v12.8h, v25.8h, v1.h[7]\n" + "ldr x20, [x17, #0x1f8]\n" + "fmla v16.8h, v25.8h, v2.h[7]\n" + "fmla v20.8h, v25.8h, v3.h[7]\n" + "ldr d25, [x17, #0x1e0]\n" + "fmla v9.8h, v24.8h, v0.h[7]\n" + "mov v25.d[1], x21\n" + "fmla v13.8h, v24.8h, v1.h[7]\n" + "fmla v17.8h, v24.8h, v2.h[7]\n" + "fmla v21.8h, v24.8h, v3.h[7]\n" + "ldr d24, [x17, #0x1f0]\n" + "mov v24.d[1], x20\n" "add x17, x17, #0x200\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "ldr x12, [x17, #0x8]\n" - "fmla v14.8h, v6.8h, v1.h[7]\n" - "ldr x11, [x17, #0x18]\n" - "fmla v18.8h, v6.8h, v2.h[7]\n" - "fmla v22.8h, v6.8h, v3.h[7]\n" + "fmla v10.8h, v25.8h, v0.h[7]\n" + "ldr x21, [x17, #0x8]\n" + "fmla v14.8h, v25.8h, v1.h[7]\n" + "ldr x20, [x17, #0x18]\n" + "fmla v18.8h, v25.8h, v2.h[7]\n" + "fmla v22.8h, v25.8h, v3.h[7]\n" "ldr d6, [x17, #0x0]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" + "fmla v11.8h, v24.8h, v0.h[7]\n" "ldr d0, [x13, #0x0]\n" - "fmla v15.8h, v7.8h, v1.h[7]\n" - "ldr d1, [x9, #0x0]\n" - "fmla v19.8h, v7.8h, v2.h[7]\n" - "ldr d2, [x27, #0x0]\n" - "fmla v23.8h, v7.8h, v3.h[7]\n" - "ldr d3, [x25, #0x0]\n" + "fmla v15.8h, v24.8h, v1.h[7]\n" + "ldr d1, [x12, #0x0]\n" + "fmla v19.8h, v24.8h, v2.h[7]\n" + "ldr d2, [x11, #0x0]\n" + "fmla v23.8h, v24.8h, v3.h[7]\n" + "ldr d3, [x10, #0x0]\n" "ldr d7, [x17, #0x10]\n" - "mov v6.d[1], x12\n" - "mov v0.d[1], x10\n" - "mov v1.d[1], x28\n" - "mov v2.d[1], x26\n" - "mov v3.d[1], x24\n" - "mov v7.d[1], x11\n" + "mov v6.d[1], x21\n" + "mov v0.d[1], x25\n" + "mov v1.d[1], x24\n" + "mov v2.d[1], x23\n" + "mov v3.d[1], x22\n" + "mov v7.d[1], x20\n" "bge 173b\n" "174:" // Height 4: Multiply loop: Single iteration only "fmla v8.8h, v6.8h, v0.h[0]\n" "add x13, x13, #0x10\n" "fmla v12.8h, v6.8h, v1.h[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" "fmla v16.8h, v6.8h, v2.h[0]\n" - "add x27, x27, #0x10\n" + "add x11, x11, #0x10\n" "fmla v20.8h, v6.8h, v3.h[0]\n" - "ldr q6, [x17, #0x20]\n" + "ldr q25, [x17, #0x20]\n" "fmla v9.8h, v7.8h, v0.h[0]\n" - "add x25, x25, #0x10\n" + "add x10, x10, #0x10\n" "fmla v13.8h, v7.8h, v1.h[0]\n" "sub x14, x14, #0x8\n" "fmla v17.8h, v7.8h, v2.h[0]\n" "prfm pldl1keep, [x13, #0x80]\n" "fmla v21.8h, v7.8h, v3.h[0]\n" - "ldr q7, [x17, #0x30]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "prfm pldl1keep, [x9, #0x80]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "prfm pldl1keep, [x27, #0x80]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "prfm pldl1keep, [x25, #0x80]\n" - "fmla v22.8h, v6.8h, v3.h[0]\n" - "ldr q6, [x17, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "fmla v23.8h, v7.8h, v3.h[0]\n" - "ldr q7, [x17, #0x50]\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "fmla v16.8h, v6.8h, v2.h[1]\n" - "fmla v20.8h, v6.8h, v3.h[1]\n" - "ldr q6, [x17, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "fmla v17.8h, v7.8h, v2.h[1]\n" - "fmla v21.8h, v7.8h, v3.h[1]\n" - "ldr q7, [x17, #0x70]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "fmla v14.8h, v6.8h, v1.h[1]\n" - "fmla v18.8h, v6.8h, v2.h[1]\n" - "fmla v22.8h, v6.8h, v3.h[1]\n" - "ldr q6, [x17, #0x80]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "fmla v15.8h, v7.8h, v1.h[1]\n" - "fmla v19.8h, v7.8h, v2.h[1]\n" - "fmla v23.8h, v7.8h, v3.h[1]\n" - "ldr q7, [x17, #0x90]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "fmla v12.8h, v6.8h, v1.h[2]\n" - "fmla v16.8h, v6.8h, v2.h[2]\n" - "fmla v20.8h, v6.8h, v3.h[2]\n" - "ldr q6, [x17, #0xa0]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "fmla v13.8h, v7.8h, v1.h[2]\n" - "fmla v17.8h, v7.8h, v2.h[2]\n" - "fmla v21.8h, v7.8h, v3.h[2]\n" - "ldr q7, [x17, #0xb0]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "fmla v14.8h, v6.8h, v1.h[2]\n" - "fmla v18.8h, v6.8h, v2.h[2]\n" - "fmla v22.8h, v6.8h, v3.h[2]\n" - "ldr q6, [x17, #0xc0]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "fmla v15.8h, v7.8h, v1.h[2]\n" - "fmla v19.8h, v7.8h, v2.h[2]\n" - "fmla v23.8h, v7.8h, v3.h[2]\n" - "ldr q7, [x17, #0xd0]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "fmla v12.8h, v6.8h, v1.h[3]\n" - "fmla v16.8h, v6.8h, v2.h[3]\n" - "fmla v20.8h, v6.8h, v3.h[3]\n" - "ldr q6, [x17, #0xe0]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "fmla v13.8h, v7.8h, v1.h[3]\n" - "fmla v17.8h, v7.8h, v2.h[3]\n" - "fmla v21.8h, v7.8h, v3.h[3]\n" - "ldr q7, [x17, #0xf0]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "fmla v14.8h, v6.8h, v1.h[3]\n" - "fmla v18.8h, v6.8h, v2.h[3]\n" - "fmla v22.8h, v6.8h, v3.h[3]\n" - "ldr q6, [x17, #0x100]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "fmla v15.8h, v7.8h, v1.h[3]\n" - "fmla v19.8h, v7.8h, v2.h[3]\n" - "fmla v23.8h, v7.8h, v3.h[3]\n" - "ldr q7, [x17, #0x110]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "fmla v12.8h, v6.8h, v1.h[4]\n" - "fmla v16.8h, v6.8h, v2.h[4]\n" - "fmla v20.8h, v6.8h, v3.h[4]\n" - "ldr q6, [x17, #0x120]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "fmla v13.8h, v7.8h, v1.h[4]\n" - "fmla v17.8h, v7.8h, v2.h[4]\n" - "fmla v21.8h, v7.8h, v3.h[4]\n" - "ldr q7, [x17, #0x130]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "fmla v14.8h, v6.8h, v1.h[4]\n" - "fmla v18.8h, v6.8h, v2.h[4]\n" - "fmla v22.8h, v6.8h, v3.h[4]\n" - "ldr q6, [x17, #0x140]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "fmla v15.8h, v7.8h, v1.h[4]\n" - "fmla v19.8h, v7.8h, v2.h[4]\n" - "fmla v23.8h, v7.8h, v3.h[4]\n" - "ldr q7, [x17, #0x150]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "fmla v12.8h, v6.8h, v1.h[5]\n" - "fmla v16.8h, v6.8h, v2.h[5]\n" - "fmla v20.8h, v6.8h, v3.h[5]\n" - "ldr q6, [x17, #0x160]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "fmla v13.8h, v7.8h, v1.h[5]\n" - "fmla v17.8h, v7.8h, v2.h[5]\n" - "fmla v21.8h, v7.8h, v3.h[5]\n" - "ldr q7, [x17, #0x170]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "fmla v14.8h, v6.8h, v1.h[5]\n" - "fmla v18.8h, v6.8h, v2.h[5]\n" - "fmla v22.8h, v6.8h, v3.h[5]\n" - "ldr q6, [x17, #0x180]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "fmla v15.8h, v7.8h, v1.h[5]\n" - "fmla v19.8h, v7.8h, v2.h[5]\n" - "fmla v23.8h, v7.8h, v3.h[5]\n" - "ldr q7, [x17, #0x190]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "fmla v12.8h, v6.8h, v1.h[6]\n" - "fmla v16.8h, v6.8h, v2.h[6]\n" - "fmla v20.8h, v6.8h, v3.h[6]\n" - "ldr q6, [x17, #0x1a0]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "fmla v13.8h, v7.8h, v1.h[6]\n" - "fmla v17.8h, v7.8h, v2.h[6]\n" - "fmla v21.8h, v7.8h, v3.h[6]\n" - "ldr q7, [x17, #0x1b0]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "fmla v14.8h, v6.8h, v1.h[6]\n" - "fmla v18.8h, v6.8h, v2.h[6]\n" - "fmla v22.8h, v6.8h, v3.h[6]\n" - "ldr q6, [x17, #0x1c0]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "fmla v15.8h, v7.8h, v1.h[6]\n" - "fmla v19.8h, v7.8h, v2.h[6]\n" - "fmla v23.8h, v7.8h, v3.h[6]\n" - "ldr q7, [x17, #0x1d0]\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "fmla v12.8h, v6.8h, v1.h[7]\n" - "fmla v16.8h, v6.8h, v2.h[7]\n" - "fmla v20.8h, v6.8h, v3.h[7]\n" - "ldr q6, [x17, #0x1e0]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "fmla v13.8h, v7.8h, v1.h[7]\n" - "fmla v17.8h, v7.8h, v2.h[7]\n" - "fmla v21.8h, v7.8h, v3.h[7]\n" - "ldr q7, [x17, #0x1f0]\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" + "ldr q24, [x17, #0x30]\n" + "fmla v10.8h, v25.8h, v0.h[0]\n" + "prfm pldl1keep, [x12, #0x80]\n" + "fmla v14.8h, v25.8h, v1.h[0]\n" + "prfm pldl1keep, [x11, #0x80]\n" + "fmla v18.8h, v25.8h, v2.h[0]\n" + "prfm pldl1keep, [x10, #0x80]\n" + "fmla v22.8h, v25.8h, v3.h[0]\n" + "ldr q25, [x17, #0x40]\n" + "fmla v11.8h, v24.8h, v0.h[0]\n" + "fmla v15.8h, v24.8h, v1.h[0]\n" + "fmla v19.8h, v24.8h, v2.h[0]\n" + "fmla v23.8h, v24.8h, v3.h[0]\n" + "ldr q24, [x17, #0x50]\n" + "fmla v8.8h, v25.8h, v0.h[1]\n" + "fmla v12.8h, v25.8h, v1.h[1]\n" + "fmla v16.8h, v25.8h, v2.h[1]\n" + "fmla v20.8h, v25.8h, v3.h[1]\n" + "ldr q25, [x17, #0x60]\n" + "fmla v9.8h, v24.8h, v0.h[1]\n" + "fmla v13.8h, v24.8h, v1.h[1]\n" + "fmla v17.8h, v24.8h, v2.h[1]\n" + "fmla v21.8h, v24.8h, v3.h[1]\n" + "ldr q24, [x17, #0x70]\n" + "fmla v10.8h, v25.8h, v0.h[1]\n" + "fmla v14.8h, v25.8h, v1.h[1]\n" + "fmla v18.8h, v25.8h, v2.h[1]\n" + "fmla v22.8h, v25.8h, v3.h[1]\n" + "ldr q25, [x17, #0x80]\n" + "fmla v11.8h, v24.8h, v0.h[1]\n" + "fmla v15.8h, v24.8h, v1.h[1]\n" + "fmla v19.8h, v24.8h, v2.h[1]\n" + "fmla v23.8h, v24.8h, v3.h[1]\n" + "ldr q24, [x17, #0x90]\n" + "fmla v8.8h, v25.8h, v0.h[2]\n" + "fmla v12.8h, v25.8h, v1.h[2]\n" + "fmla v16.8h, v25.8h, v2.h[2]\n" + "fmla v20.8h, v25.8h, v3.h[2]\n" + "ldr q25, [x17, #0xa0]\n" + "fmla v9.8h, v24.8h, v0.h[2]\n" + "fmla v13.8h, v24.8h, v1.h[2]\n" + "fmla v17.8h, v24.8h, v2.h[2]\n" + "fmla v21.8h, v24.8h, v3.h[2]\n" + "ldr q24, [x17, #0xb0]\n" + "fmla v10.8h, v25.8h, v0.h[2]\n" + "fmla v14.8h, v25.8h, v1.h[2]\n" + "fmla v18.8h, v25.8h, v2.h[2]\n" + "fmla v22.8h, v25.8h, v3.h[2]\n" + "ldr q25, [x17, #0xc0]\n" + "fmla v11.8h, v24.8h, v0.h[2]\n" + "fmla v15.8h, v24.8h, v1.h[2]\n" + "fmla v19.8h, v24.8h, v2.h[2]\n" + "fmla v23.8h, v24.8h, v3.h[2]\n" + "ldr q24, [x17, #0xd0]\n" + "fmla v8.8h, v25.8h, v0.h[3]\n" + "fmla v12.8h, v25.8h, v1.h[3]\n" + "fmla v16.8h, v25.8h, v2.h[3]\n" + "fmla v20.8h, v25.8h, v3.h[3]\n" + "ldr q25, [x17, #0xe0]\n" + "fmla v9.8h, v24.8h, v0.h[3]\n" + "fmla v13.8h, v24.8h, v1.h[3]\n" + "fmla v17.8h, v24.8h, v2.h[3]\n" + "fmla v21.8h, v24.8h, v3.h[3]\n" + "ldr q24, [x17, #0xf0]\n" + "fmla v10.8h, v25.8h, v0.h[3]\n" + "fmla v14.8h, v25.8h, v1.h[3]\n" + "fmla v18.8h, v25.8h, v2.h[3]\n" + "fmla v22.8h, v25.8h, v3.h[3]\n" + "ldr q25, [x17, #0x100]\n" + "fmla v11.8h, v24.8h, v0.h[3]\n" + "fmla v15.8h, v24.8h, v1.h[3]\n" + "fmla v19.8h, v24.8h, v2.h[3]\n" + "fmla v23.8h, v24.8h, v3.h[3]\n" + "ldr q24, [x17, #0x110]\n" + "fmla v8.8h, v25.8h, v0.h[4]\n" + "fmla v12.8h, v25.8h, v1.h[4]\n" + "fmla v16.8h, v25.8h, v2.h[4]\n" + "fmla v20.8h, v25.8h, v3.h[4]\n" + "ldr q25, [x17, #0x120]\n" + "fmla v9.8h, v24.8h, v0.h[4]\n" + "fmla v13.8h, v24.8h, v1.h[4]\n" + "fmla v17.8h, v24.8h, v2.h[4]\n" + "fmla v21.8h, v24.8h, v3.h[4]\n" + "ldr q24, [x17, #0x130]\n" + "fmla v10.8h, v25.8h, v0.h[4]\n" + "fmla v14.8h, v25.8h, v1.h[4]\n" + "fmla v18.8h, v25.8h, v2.h[4]\n" + "fmla v22.8h, v25.8h, v3.h[4]\n" + "ldr q25, [x17, #0x140]\n" + "fmla v11.8h, v24.8h, v0.h[4]\n" + "fmla v15.8h, v24.8h, v1.h[4]\n" + "fmla v19.8h, v24.8h, v2.h[4]\n" + "fmla v23.8h, v24.8h, v3.h[4]\n" + "ldr q24, [x17, #0x150]\n" + "fmla v8.8h, v25.8h, v0.h[5]\n" + "fmla v12.8h, v25.8h, v1.h[5]\n" + "fmla v16.8h, v25.8h, v2.h[5]\n" + "fmla v20.8h, v25.8h, v3.h[5]\n" + "ldr q25, [x17, #0x160]\n" + "fmla v9.8h, v24.8h, v0.h[5]\n" + "fmla v13.8h, v24.8h, v1.h[5]\n" + "fmla v17.8h, v24.8h, v2.h[5]\n" + "fmla v21.8h, v24.8h, v3.h[5]\n" + "ldr q24, [x17, #0x170]\n" + "fmla v10.8h, v25.8h, v0.h[5]\n" + "fmla v14.8h, v25.8h, v1.h[5]\n" + "fmla v18.8h, v25.8h, v2.h[5]\n" + "fmla v22.8h, v25.8h, v3.h[5]\n" + "ldr q25, [x17, #0x180]\n" + "fmla v11.8h, v24.8h, v0.h[5]\n" + "fmla v15.8h, v24.8h, v1.h[5]\n" + "fmla v19.8h, v24.8h, v2.h[5]\n" + "fmla v23.8h, v24.8h, v3.h[5]\n" + "ldr q24, [x17, #0x190]\n" + "fmla v8.8h, v25.8h, v0.h[6]\n" + "fmla v12.8h, v25.8h, v1.h[6]\n" + "fmla v16.8h, v25.8h, v2.h[6]\n" + "fmla v20.8h, v25.8h, v3.h[6]\n" + "ldr q25, [x17, #0x1a0]\n" + "fmla v9.8h, v24.8h, v0.h[6]\n" + "fmla v13.8h, v24.8h, v1.h[6]\n" + "fmla v17.8h, v24.8h, v2.h[6]\n" + "fmla v21.8h, v24.8h, v3.h[6]\n" + "ldr q24, [x17, #0x1b0]\n" + "fmla v10.8h, v25.8h, v0.h[6]\n" + "fmla v14.8h, v25.8h, v1.h[6]\n" + "fmla v18.8h, v25.8h, v2.h[6]\n" + "fmla v22.8h, v25.8h, v3.h[6]\n" + "ldr q25, [x17, #0x1c0]\n" + "fmla v11.8h, v24.8h, v0.h[6]\n" + "fmla v15.8h, v24.8h, v1.h[6]\n" + "fmla v19.8h, v24.8h, v2.h[6]\n" + "fmla v23.8h, v24.8h, v3.h[6]\n" + "ldr q24, [x17, #0x1d0]\n" + "fmla v8.8h, v25.8h, v0.h[7]\n" + "fmla v12.8h, v25.8h, v1.h[7]\n" + "fmla v16.8h, v25.8h, v2.h[7]\n" + "fmla v20.8h, v25.8h, v3.h[7]\n" + "ldr q25, [x17, #0x1e0]\n" + "fmla v9.8h, v24.8h, v0.h[7]\n" + "fmla v13.8h, v24.8h, v1.h[7]\n" + "fmla v17.8h, v24.8h, v2.h[7]\n" + "fmla v21.8h, v24.8h, v3.h[7]\n" + "ldr q24, [x17, #0x1f0]\n" + "fmla v10.8h, v25.8h, v0.h[7]\n" "add x17, x17, #0x200\n" - "fmla v14.8h, v6.8h, v1.h[7]\n" - "fmla v18.8h, v6.8h, v2.h[7]\n" - "fmla v22.8h, v6.8h, v3.h[7]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" - "fmla v15.8h, v7.8h, v1.h[7]\n" - "fmla v19.8h, v7.8h, v2.h[7]\n" - "fmla v23.8h, v7.8h, v3.h[7]\n" + "fmla v14.8h, v25.8h, v1.h[7]\n" + "fmla v18.8h, v25.8h, v2.h[7]\n" + "fmla v22.8h, v25.8h, v3.h[7]\n" + "fmla v11.8h, v24.8h, v0.h[7]\n" + "fmla v15.8h, v24.8h, v1.h[7]\n" + "fmla v19.8h, v24.8h, v2.h[7]\n" + "fmla v23.8h, v24.8h, v3.h[7]\n" "175:" // Height 4: Multiply loop: Main loop skip "cbz x14, 177f\n" "176:" // Height 4: Multiply loop: Odd block loop - "ldr h0, [x13], #0x2\n" + "ldr h3, [x13], #0x2\n" "sub x14, x14, #0x1\n" - "ldr h1, [x9], #0x2\n" - "ldr h2, [x27], #0x2\n" - "ldr h3, [x25], #0x2\n" - "ldr q6, [x17, #0x0]\n" - "fmla v8.8h, v6.8h, v0.h[0]\n" - "ldr q7, [x17, #0x10]\n" - "fmla v12.8h, v6.8h, v1.h[0]\n" - "fmla v16.8h, v6.8h, v2.h[0]\n" - "fmla v20.8h, v6.8h, v3.h[0]\n" - "ldr q6, [x17, #0x20]\n" - "fmla v9.8h, v7.8h, v0.h[0]\n" - "fmla v13.8h, v7.8h, v1.h[0]\n" - "fmla v17.8h, v7.8h, v2.h[0]\n" - "fmla v21.8h, v7.8h, v3.h[0]\n" - "ldr q7, [x17, #0x30]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" + "ldr h2, [x12], #0x2\n" + "ldr h1, [x11], #0x2\n" + "ldr h0, [x10], #0x2\n" + "ldr q25, [x17, #0x0]\n" + "fmla v8.8h, v25.8h, v3.h[0]\n" + "ldr q24, [x17, #0x10]\n" + "fmla v12.8h, v25.8h, v2.h[0]\n" + "fmla v16.8h, v25.8h, v1.h[0]\n" + "fmla v20.8h, v25.8h, v0.h[0]\n" + "ldr q25, [x17, #0x20]\n" + "fmla v9.8h, v24.8h, v3.h[0]\n" + "fmla v13.8h, v24.8h, v2.h[0]\n" + "fmla v17.8h, v24.8h, v1.h[0]\n" + "fmla v21.8h, v24.8h, v0.h[0]\n" + "ldr q24, [x17, #0x30]\n" + "fmla v10.8h, v25.8h, v3.h[0]\n" "add x17, x17, #0x40\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "fmla v22.8h, v6.8h, v3.h[0]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "fmla v23.8h, v7.8h, v3.h[0]\n" + "fmla v14.8h, v25.8h, v2.h[0]\n" + "fmla v18.8h, v25.8h, v1.h[0]\n" + "fmla v22.8h, v25.8h, v0.h[0]\n" + "fmla v11.8h, v24.8h, v3.h[0]\n" + "fmla v15.8h, v24.8h, v2.h[0]\n" + "fmla v19.8h, v24.8h, v1.h[0]\n" + "fmla v23.8h, v24.8h, v0.h[0]\n" "cbnz x14, 176b\n" "177:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -2918,41 +2918,41 @@ void a64_hybrid_fp16_mla_6x32_a55 ( "prfm pstl1keep, [x23, #0x0]\n" "tbz %x[flags], #1, 178f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v0.8h }, [x20]\n" - "fmin v8.8h, v8.8h, v0.8h\n" - "fmin v9.8h, v9.8h, v0.8h\n" - "fmin v10.8h, v10.8h, v0.8h\n" - "fmin v11.8h, v11.8h, v0.8h\n" - "fmin v12.8h, v12.8h, v0.8h\n" - "fmin v13.8h, v13.8h, v0.8h\n" - "fmin v14.8h, v14.8h, v0.8h\n" - "fmin v15.8h, v15.8h, v0.8h\n" - "fmin v16.8h, v16.8h, v0.8h\n" - "fmin v17.8h, v17.8h, v0.8h\n" - "fmin v18.8h, v18.8h, v0.8h\n" - "fmin v19.8h, v19.8h, v0.8h\n" - "fmin v20.8h, v20.8h, v0.8h\n" - "fmin v21.8h, v21.8h, v0.8h\n" - "fmin v22.8h, v22.8h, v0.8h\n" - "fmin v23.8h, v23.8h, v0.8h\n" + "ld1r { v24.8h }, [x20]\n" + "fmin v8.8h, v8.8h, v24.8h\n" + "fmin v9.8h, v9.8h, v24.8h\n" + "fmin v10.8h, v10.8h, v24.8h\n" + "fmin v11.8h, v11.8h, v24.8h\n" + "fmin v12.8h, v12.8h, v24.8h\n" + "fmin v13.8h, v13.8h, v24.8h\n" + "fmin v14.8h, v14.8h, v24.8h\n" + "fmin v15.8h, v15.8h, v24.8h\n" + "fmin v16.8h, v16.8h, v24.8h\n" + "fmin v17.8h, v17.8h, v24.8h\n" + "fmin v18.8h, v18.8h, v24.8h\n" + "fmin v19.8h, v19.8h, v24.8h\n" + "fmin v20.8h, v20.8h, v24.8h\n" + "fmin v21.8h, v21.8h, v24.8h\n" + "fmin v22.8h, v22.8h, v24.8h\n" + "fmin v23.8h, v23.8h, v24.8h\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.8h }, [x20]\n" - "fmax v8.8h, v8.8h, v0.8h\n" - "fmax v9.8h, v9.8h, v0.8h\n" - "fmax v10.8h, v10.8h, v0.8h\n" - "fmax v11.8h, v11.8h, v0.8h\n" - "fmax v12.8h, v12.8h, v0.8h\n" - "fmax v13.8h, v13.8h, v0.8h\n" - "fmax v14.8h, v14.8h, v0.8h\n" - "fmax v15.8h, v15.8h, v0.8h\n" - "fmax v16.8h, v16.8h, v0.8h\n" - "fmax v17.8h, v17.8h, v0.8h\n" - "fmax v18.8h, v18.8h, v0.8h\n" - "fmax v19.8h, v19.8h, v0.8h\n" - "fmax v20.8h, v20.8h, v0.8h\n" - "fmax v21.8h, v21.8h, v0.8h\n" - "fmax v22.8h, v22.8h, v0.8h\n" - "fmax v23.8h, v23.8h, v0.8h\n" + "ld1r { v24.8h }, [x20]\n" + "fmax v8.8h, v8.8h, v24.8h\n" + "fmax v9.8h, v9.8h, v24.8h\n" + "fmax v10.8h, v10.8h, v24.8h\n" + "fmax v11.8h, v11.8h, v24.8h\n" + "fmax v12.8h, v12.8h, v24.8h\n" + "fmax v13.8h, v13.8h, v24.8h\n" + "fmax v14.8h, v14.8h, v24.8h\n" + "fmax v15.8h, v15.8h, v24.8h\n" + "fmax v16.8h, v16.8h, v24.8h\n" + "fmax v17.8h, v17.8h, v24.8h\n" + "fmax v18.8h, v18.8h, v24.8h\n" + "fmax v19.8h, v19.8h, v24.8h\n" + "fmax v20.8h, v20.8h, v24.8h\n" + "fmax v21.8h, v21.8h, v24.8h\n" + "fmax v22.8h, v22.8h, v24.8h\n" + "fmax v23.8h, v23.8h, v24.8h\n" "178:" // Height 4: No activation "cmp x8, #0x20\n" "bge 195f\n" @@ -3382,675 +3382,675 @@ void a64_hybrid_fp16_mla_6x32_a55 ( "ld1 { v25.h }[2], [x22]\n" "b 215f\n" "210:" // Height 5: Partial accumulate: partial_1_8 - "mov x20, #0x10\n" - "tbz x8, #0, 215f\n" - "ldr h9, [x16, #0x0]\n" - "ldr h13, [x25, #0x0]\n" - "ldr h17, [x24, #0x0]\n" - "ldr h21, [x23, #0x0]\n" - "ldr h25, [x22, #0x0]\n" - "b 215f\n" - "211:" // Height 5: Partial accumulate: partial_4_0 - "tbz x8, #2, 213f\n" - "ldr d8, [x16], #0x8\n" - "ldr d12, [x25], #0x8\n" - "ldr d16, [x24], #0x8\n" - "ldr d20, [x23], #0x8\n" - "ldr d24, [x22], #0x8\n" - "tbz x8, #1, 212f\n" - "ld1 { v8.s }[2], [x16], #0x4\n" - "mov x20, #0xc\n" - "ld1 { v12.s }[2], [x25], #0x4\n" - "ld1 { v16.s }[2], [x24], #0x4\n" - "ld1 { v20.s }[2], [x23], #0x4\n" - "ld1 { v24.s }[2], [x22], #0x4\n" - "tbz x8, #0, 215f\n" - "ld1 { v8.h }[6], [x16]\n" - "ld1 { v12.h }[6], [x25]\n" - "ld1 { v16.h }[6], [x24]\n" - "ld1 { v20.h }[6], [x23]\n" - "ld1 { v24.h }[6], [x22]\n" - "b 215f\n" - "212:" // Height 5: Partial accumulate: partial_1_4 - "mov x20, #0x8\n" - "tbz x8, #0, 215f\n" - "ld1 { v8.h }[4], [x16]\n" - "ld1 { v12.h }[4], [x25]\n" - "ld1 { v16.h }[4], [x24]\n" - "ld1 { v20.h }[4], [x23]\n" - "ld1 { v24.h }[4], [x22]\n" - "b 215f\n" - "213:" // Height 5: Partial accumulate: partial_2_0 - "tbz x8, #1, 214f\n" - "ldr s8, [x16], #0x4\n" - "mov x20, #0x4\n" - "ldr s12, [x25], #0x4\n" - "ldr s16, [x24], #0x4\n" - "ldr s20, [x23], #0x4\n" - "ldr s24, [x22], #0x4\n" - "tbz x8, #0, 215f\n" - "ld1 { v8.h }[2], [x16]\n" - "ld1 { v12.h }[2], [x25]\n" - "ld1 { v16.h }[2], [x24]\n" - "ld1 { v20.h }[2], [x23]\n" - "ld1 { v24.h }[2], [x22]\n" - "b 215f\n" - "214:" // Height 5: Partial accumulate: partial_1_0 - "ldr h8, [x16, #0x0]\n" - "mov x20, #0x0\n" - "ldr h12, [x25, #0x0]\n" - "ldr h16, [x24, #0x0]\n" - "ldr h20, [x23, #0x0]\n" - "ldr h24, [x22, #0x0]\n" - "215:" // Height 5: Partial accumulate: Done - "sub x16, x16, x20\n" - "b 218f\n" - "216:" // Height 5: full accumulate - "ldr q8, [x16, #0x0]\n" - "ldr q9, [x16, #0x10]\n" - "ldr q10, [x16, #0x20]\n" - "ldr q11, [x16, #0x30]\n" - "ldr q12, [x25, #0x0]\n" - "ldr q13, [x25, #0x10]\n" - "ldr q14, [x25, #0x20]\n" - "ldr q15, [x25, #0x30]\n" - "ldr q16, [x24, #0x0]\n" - "ldr q17, [x24, #0x10]\n" - "ldr q18, [x24, #0x20]\n" - "ldr q19, [x24, #0x30]\n" - "ldr q20, [x23, #0x0]\n" - "ldr q21, [x23, #0x10]\n" - "ldr q22, [x23, #0x20]\n" - "ldr q23, [x23, #0x30]\n" - "ldr q24, [x22, #0x0]\n" - "ldr q25, [x22, #0x10]\n" - "ldr q26, [x22, #0x20]\n" - "ldr q27, [x22, #0x30]\n" - "b 218f\n" - "217:" // Height 5: no accumulate - "movi v8.16b, #0x0\n" - "movi v9.16b, #0x0\n" - "movi v10.16b, #0x0\n" - "movi v11.16b, #0x0\n" - "movi v12.16b, #0x0\n" - "movi v13.16b, #0x0\n" - "movi v14.16b, #0x0\n" - "movi v15.16b, #0x0\n" - "movi v16.16b, #0x0\n" - "movi v17.16b, #0x0\n" - "movi v18.16b, #0x0\n" - "movi v19.16b, #0x0\n" - "movi v20.16b, #0x0\n" - "movi v21.16b, #0x0\n" - "movi v22.16b, #0x0\n" - "movi v23.16b, #0x0\n" - "movi v24.16b, #0x0\n" - "movi v25.16b, #0x0\n" - "movi v26.16b, #0x0\n" - "movi v27.16b, #0x0\n" - "218:" // Height 5: setup done - "mov x15, #0x0\n" - "219:" // Height 5: String loop - "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr w14, [x20, x15, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" - "tbz %x[flags], #3, 220f\n" - "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x13, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" - "ldr x27, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" - "ldr x23, [x21, #0x20]\n" - "cbnz x15, 221f\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x13, x13, x20, LSL #1\n" - "add x9, x9, x20, LSL #1\n" - "add x27, x27, x20, LSL #1\n" - "add x25, x25, x20, LSL #1\n" - "add x23, x23, x20, LSL #1\n" - "b 221f\n" - "220:" // Height 5: setup direct input - "mov x13, %x[input_ptr]\n" - "add x9, x13, x20, LSL #1\n" - "add x27, x9, x20, LSL #1\n" - "add x25, x27, x20, LSL #1\n" - "add x23, x25, x20, LSL #1\n" - "221:" // Height 5: input setup done - "cmp x14, #0x8\n" - "blt 224f\n" - "ldr q0, [x13, #0x0]\n" - "cmp x14, #0x10\n" - "ldr q1, [x9, #0x0]\n" - "ldr q2, [x27, #0x0]\n" - "ldr q3, [x25, #0x0]\n" - "ldr q4, [x23, #0x0]\n" - "ldr q6, [x17, #0x0]\n" - "ldr q7, [x17, #0x10]\n" - "blt 223f\n" - "222:" // Height 5: Multiply loop: Main loop head - "fmla v8.8h, v6.8h, v0.h[0]\n" - "ldr x12, [x17, #0x28]\n" - "fmla v12.8h, v6.8h, v1.h[0]\n" - "ldr x11, [x17, #0x38]\n" - "fmla v16.8h, v6.8h, v2.h[0]\n" - "add x13, x13, #0x10\n" - "fmla v20.8h, v6.8h, v3.h[0]\n" - "add x9, x9, #0x10\n" - "fmla v24.8h, v6.8h, v4.h[0]\n" - "ldr d6, [x17, #0x20]\n" - "fmla v9.8h, v7.8h, v0.h[0]\n" - "mov v6.d[1], x12\n" - "fmla v13.8h, v7.8h, v1.h[0]\n" - "ldr x12, [x17, #0x48]\n" - "fmla v17.8h, v7.8h, v2.h[0]\n" - "add x27, x27, #0x10\n" - "fmla v21.8h, v7.8h, v3.h[0]\n" - "add x25, x25, #0x10\n" - "fmla v25.8h, v7.8h, v4.h[0]\n" - "ldr d7, [x17, #0x30]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "ldr x11, [x17, #0x58]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "add x23, x23, #0x10\n" - "fmla v22.8h, v6.8h, v3.h[0]\n" - "ldr x10, [x13, #0x8]\n" - "fmla v26.8h, v6.8h, v4.h[0]\n" - "ldr d6, [x17, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "mov v6.d[1], x12\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "ldr x12, [x17, #0x68]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "ldr x28, [x9, #0x8]\n" - "fmla v23.8h, v7.8h, v3.h[0]\n" - "ldr x26, [x27, #0x8]\n" - "fmla v27.8h, v7.8h, v4.h[0]\n" - "ldr d7, [x17, #0x50]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "ldr x11, [x17, #0x78]\n" - "fmla v16.8h, v6.8h, v2.h[1]\n" - "ldr x24, [x25, #0x8]\n" - "fmla v20.8h, v6.8h, v3.h[1]\n" - "ldr x22, [x23, #0x8]\n" - "fmla v24.8h, v6.8h, v4.h[1]\n" - "ldr d6, [x17, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "mov v6.d[1], x12\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "ldr x12, [x17, #0x88]\n" - "fmla v17.8h, v7.8h, v2.h[1]\n" - "sub x14, x14, #0x8\n" - "fmla v21.8h, v7.8h, v3.h[1]\n" - "cmp x14, #0x10\n" - "fmla v25.8h, v7.8h, v4.h[1]\n" - "ldr d7, [x17, #0x70]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "fmla v14.8h, v6.8h, v1.h[1]\n" - "ldr x11, [x17, #0x98]\n" - "fmla v18.8h, v6.8h, v2.h[1]\n" - "prfm pldl1keep, [x13, #0x80]\n" - "fmla v22.8h, v6.8h, v3.h[1]\n" - "prfm pldl1keep, [x9, #0x80]\n" - "fmla v26.8h, v6.8h, v4.h[1]\n" - "ldr d6, [x17, #0x80]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "mov v6.d[1], x12\n" - "fmla v15.8h, v7.8h, v1.h[1]\n" - "ldr x12, [x17, #0xa8]\n" - "fmla v19.8h, v7.8h, v2.h[1]\n" - "prfm pldl1keep, [x27, #0x80]\n" - "fmla v23.8h, v7.8h, v3.h[1]\n" - "prfm pldl1keep, [x25, #0x80]\n" - "fmla v27.8h, v7.8h, v4.h[1]\n" - "ldr d7, [x17, #0x90]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "fmla v12.8h, v6.8h, v1.h[2]\n" - "ldr x11, [x17, #0xb8]\n" - "fmla v16.8h, v6.8h, v2.h[2]\n" - "prfm pldl1keep, [x23, #0x80]\n" - "fmla v20.8h, v6.8h, v3.h[2]\n" - "fmla v24.8h, v6.8h, v4.h[2]\n" - "ldr d6, [x17, #0xa0]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "mov v6.d[1], x12\n" - "fmla v13.8h, v7.8h, v1.h[2]\n" - "ldr x12, [x17, #0xc8]\n" - "fmla v17.8h, v7.8h, v2.h[2]\n" - "fmla v21.8h, v7.8h, v3.h[2]\n" - "fmla v25.8h, v7.8h, v4.h[2]\n" - "ldr d7, [x17, #0xb0]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "fmla v14.8h, v6.8h, v1.h[2]\n" - "ldr x11, [x17, #0xd8]\n" - "fmla v18.8h, v6.8h, v2.h[2]\n" - "fmla v22.8h, v6.8h, v3.h[2]\n" - "fmla v26.8h, v6.8h, v4.h[2]\n" - "ldr d6, [x17, #0xc0]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "mov v6.d[1], x12\n" - "fmla v15.8h, v7.8h, v1.h[2]\n" - "ldr x12, [x17, #0xe8]\n" - "fmla v19.8h, v7.8h, v2.h[2]\n" - "fmla v23.8h, v7.8h, v3.h[2]\n" - "fmla v27.8h, v7.8h, v4.h[2]\n" - "ldr d7, [x17, #0xd0]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "fmla v12.8h, v6.8h, v1.h[3]\n" - "ldr x11, [x17, #0xf8]\n" - "fmla v16.8h, v6.8h, v2.h[3]\n" - "fmla v20.8h, v6.8h, v3.h[3]\n" - "fmla v24.8h, v6.8h, v4.h[3]\n" - "ldr d6, [x17, #0xe0]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "mov v6.d[1], x12\n" - "fmla v13.8h, v7.8h, v1.h[3]\n" - "ldr x12, [x17, #0x108]\n" - "fmla v17.8h, v7.8h, v2.h[3]\n" - "fmla v21.8h, v7.8h, v3.h[3]\n" - "fmla v25.8h, v7.8h, v4.h[3]\n" - "ldr d7, [x17, #0xf0]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "fmla v14.8h, v6.8h, v1.h[3]\n" - "ldr x11, [x17, #0x118]\n" - "fmla v18.8h, v6.8h, v2.h[3]\n" - "fmla v22.8h, v6.8h, v3.h[3]\n" - "fmla v26.8h, v6.8h, v4.h[3]\n" - "ldr d6, [x17, #0x100]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "mov v6.d[1], x12\n" - "fmla v15.8h, v7.8h, v1.h[3]\n" - "ldr x12, [x17, #0x128]\n" - "fmla v19.8h, v7.8h, v2.h[3]\n" - "fmla v23.8h, v7.8h, v3.h[3]\n" - "fmla v27.8h, v7.8h, v4.h[3]\n" - "ldr d7, [x17, #0x110]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "fmla v12.8h, v6.8h, v1.h[4]\n" - "ldr x11, [x17, #0x138]\n" - "fmla v16.8h, v6.8h, v2.h[4]\n" - "fmla v20.8h, v6.8h, v3.h[4]\n" - "fmla v24.8h, v6.8h, v4.h[4]\n" - "ldr d6, [x17, #0x120]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "mov v6.d[1], x12\n" - "fmla v13.8h, v7.8h, v1.h[4]\n" - "ldr x12, [x17, #0x148]\n" - "fmla v17.8h, v7.8h, v2.h[4]\n" - "fmla v21.8h, v7.8h, v3.h[4]\n" - "fmla v25.8h, v7.8h, v4.h[4]\n" - "ldr d7, [x17, #0x130]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "fmla v14.8h, v6.8h, v1.h[4]\n" - "ldr x11, [x17, #0x158]\n" - "fmla v18.8h, v6.8h, v2.h[4]\n" - "fmla v22.8h, v6.8h, v3.h[4]\n" - "fmla v26.8h, v6.8h, v4.h[4]\n" - "ldr d6, [x17, #0x140]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "mov v6.d[1], x12\n" - "fmla v15.8h, v7.8h, v1.h[4]\n" - "ldr x12, [x17, #0x168]\n" - "fmla v19.8h, v7.8h, v2.h[4]\n" - "fmla v23.8h, v7.8h, v3.h[4]\n" - "fmla v27.8h, v7.8h, v4.h[4]\n" - "ldr d7, [x17, #0x150]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "fmla v12.8h, v6.8h, v1.h[5]\n" - "ldr x11, [x17, #0x178]\n" - "fmla v16.8h, v6.8h, v2.h[5]\n" - "fmla v20.8h, v6.8h, v3.h[5]\n" - "fmla v24.8h, v6.8h, v4.h[5]\n" - "ldr d6, [x17, #0x160]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "mov v6.d[1], x12\n" - "fmla v13.8h, v7.8h, v1.h[5]\n" - "ldr x12, [x17, #0x188]\n" - "fmla v17.8h, v7.8h, v2.h[5]\n" - "fmla v21.8h, v7.8h, v3.h[5]\n" - "fmla v25.8h, v7.8h, v4.h[5]\n" - "ldr d7, [x17, #0x170]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "fmla v14.8h, v6.8h, v1.h[5]\n" - "ldr x11, [x17, #0x198]\n" - "fmla v18.8h, v6.8h, v2.h[5]\n" - "fmla v22.8h, v6.8h, v3.h[5]\n" - "fmla v26.8h, v6.8h, v4.h[5]\n" - "ldr d6, [x17, #0x180]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "mov v6.d[1], x12\n" - "fmla v15.8h, v7.8h, v1.h[5]\n" - "ldr x12, [x17, #0x1a8]\n" - "fmla v19.8h, v7.8h, v2.h[5]\n" - "fmla v23.8h, v7.8h, v3.h[5]\n" - "fmla v27.8h, v7.8h, v4.h[5]\n" - "ldr d7, [x17, #0x190]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "fmla v12.8h, v6.8h, v1.h[6]\n" - "ldr x11, [x17, #0x1b8]\n" - "fmla v16.8h, v6.8h, v2.h[6]\n" - "fmla v20.8h, v6.8h, v3.h[6]\n" - "fmla v24.8h, v6.8h, v4.h[6]\n" - "ldr d6, [x17, #0x1a0]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "mov v6.d[1], x12\n" - "fmla v13.8h, v7.8h, v1.h[6]\n" - "ldr x12, [x17, #0x1c8]\n" - "fmla v17.8h, v7.8h, v2.h[6]\n" - "fmla v21.8h, v7.8h, v3.h[6]\n" - "fmla v25.8h, v7.8h, v4.h[6]\n" - "ldr d7, [x17, #0x1b0]\n" - "mov v7.d[1], x11\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "fmla v14.8h, v6.8h, v1.h[6]\n" - "ldr x11, [x17, #0x1d8]\n" - "fmla v18.8h, v6.8h, v2.h[6]\n" - "fmla v22.8h, v6.8h, v3.h[6]\n" - "fmla v26.8h, v6.8h, v4.h[6]\n" - "ldr d6, [x17, #0x1c0]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "mov v6.d[1], x12\n" - "fmla v15.8h, v7.8h, v1.h[6]\n" - "ldr x12, [x17, #0x1e8]\n" - "fmla v19.8h, v7.8h, v2.h[6]\n" - "fmla v23.8h, v7.8h, v3.h[6]\n" - "fmla v27.8h, v7.8h, v4.h[6]\n" - "ldr d7, [x17, #0x1d0]\n" - "mov v7.d[1], x11\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "fmla v12.8h, v6.8h, v1.h[7]\n" - "ldr x11, [x17, #0x1f8]\n" - "fmla v16.8h, v6.8h, v2.h[7]\n" - "fmla v20.8h, v6.8h, v3.h[7]\n" - "fmla v24.8h, v6.8h, v4.h[7]\n" - "ldr d6, [x17, #0x1e0]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "mov v6.d[1], x12\n" - "fmla v13.8h, v7.8h, v1.h[7]\n" - "fmla v17.8h, v7.8h, v2.h[7]\n" - "fmla v21.8h, v7.8h, v3.h[7]\n" - "fmla v25.8h, v7.8h, v4.h[7]\n" - "ldr d7, [x17, #0x1f0]\n" - "mov v7.d[1], x11\n" - "add x17, x17, #0x200\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "ldr x12, [x17, #0x8]\n" - "fmla v14.8h, v6.8h, v1.h[7]\n" - "ldr x11, [x17, #0x18]\n" - "fmla v18.8h, v6.8h, v2.h[7]\n" - "fmla v22.8h, v6.8h, v3.h[7]\n" - "fmla v26.8h, v6.8h, v4.h[7]\n" - "ldr d6, [x17, #0x0]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" - "ldr d0, [x13, #0x0]\n" - "fmla v15.8h, v7.8h, v1.h[7]\n" - "ldr d1, [x9, #0x0]\n" - "fmla v19.8h, v7.8h, v2.h[7]\n" - "ldr d2, [x27, #0x0]\n" - "fmla v23.8h, v7.8h, v3.h[7]\n" - "ldr d3, [x25, #0x0]\n" - "fmla v27.8h, v7.8h, v4.h[7]\n" - "ldr d4, [x23, #0x0]\n" - "ldr d7, [x17, #0x10]\n" - "mov v6.d[1], x12\n" - "mov v0.d[1], x10\n" - "mov v1.d[1], x28\n" - "mov v2.d[1], x26\n" - "mov v3.d[1], x24\n" - "mov v4.d[1], x22\n" - "mov v7.d[1], x11\n" - "bge 222b\n" - "223:" // Height 5: Multiply loop: Single iteration only - "fmla v8.8h, v6.8h, v0.h[0]\n" - "add x13, x13, #0x10\n" - "fmla v12.8h, v6.8h, v1.h[0]\n" - "add x9, x9, #0x10\n" - "fmla v16.8h, v6.8h, v2.h[0]\n" - "add x27, x27, #0x10\n" - "fmla v20.8h, v6.8h, v3.h[0]\n" - "add x25, x25, #0x10\n" - "fmla v24.8h, v6.8h, v4.h[0]\n" - "ldr q6, [x17, #0x20]\n" - "fmla v9.8h, v7.8h, v0.h[0]\n" - "add x23, x23, #0x10\n" - "fmla v13.8h, v7.8h, v1.h[0]\n" - "sub x14, x14, #0x8\n" - "fmla v17.8h, v7.8h, v2.h[0]\n" - "prfm pldl1keep, [x13, #0x80]\n" - "fmla v21.8h, v7.8h, v3.h[0]\n" - "prfm pldl1keep, [x9, #0x80]\n" - "fmla v25.8h, v7.8h, v4.h[0]\n" - "ldr q7, [x17, #0x30]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "prfm pldl1keep, [x27, #0x80]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "prfm pldl1keep, [x25, #0x80]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "prfm pldl1keep, [x23, #0x80]\n" - "fmla v22.8h, v6.8h, v3.h[0]\n" - "fmla v26.8h, v6.8h, v4.h[0]\n" - "ldr q6, [x17, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "fmla v23.8h, v7.8h, v3.h[0]\n" - "fmla v27.8h, v7.8h, v4.h[0]\n" - "ldr q7, [x17, #0x50]\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "fmla v16.8h, v6.8h, v2.h[1]\n" - "fmla v20.8h, v6.8h, v3.h[1]\n" - "fmla v24.8h, v6.8h, v4.h[1]\n" - "ldr q6, [x17, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "fmla v17.8h, v7.8h, v2.h[1]\n" - "fmla v21.8h, v7.8h, v3.h[1]\n" - "fmla v25.8h, v7.8h, v4.h[1]\n" - "ldr q7, [x17, #0x70]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "fmla v14.8h, v6.8h, v1.h[1]\n" - "fmla v18.8h, v6.8h, v2.h[1]\n" - "fmla v22.8h, v6.8h, v3.h[1]\n" - "fmla v26.8h, v6.8h, v4.h[1]\n" - "ldr q6, [x17, #0x80]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "fmla v15.8h, v7.8h, v1.h[1]\n" - "fmla v19.8h, v7.8h, v2.h[1]\n" - "fmla v23.8h, v7.8h, v3.h[1]\n" - "fmla v27.8h, v7.8h, v4.h[1]\n" - "ldr q7, [x17, #0x90]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "fmla v12.8h, v6.8h, v1.h[2]\n" - "fmla v16.8h, v6.8h, v2.h[2]\n" - "fmla v20.8h, v6.8h, v3.h[2]\n" - "fmla v24.8h, v6.8h, v4.h[2]\n" - "ldr q6, [x17, #0xa0]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "fmla v13.8h, v7.8h, v1.h[2]\n" - "fmla v17.8h, v7.8h, v2.h[2]\n" - "fmla v21.8h, v7.8h, v3.h[2]\n" - "fmla v25.8h, v7.8h, v4.h[2]\n" - "ldr q7, [x17, #0xb0]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "fmla v14.8h, v6.8h, v1.h[2]\n" - "fmla v18.8h, v6.8h, v2.h[2]\n" - "fmla v22.8h, v6.8h, v3.h[2]\n" - "fmla v26.8h, v6.8h, v4.h[2]\n" - "ldr q6, [x17, #0xc0]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "fmla v15.8h, v7.8h, v1.h[2]\n" - "fmla v19.8h, v7.8h, v2.h[2]\n" - "fmla v23.8h, v7.8h, v3.h[2]\n" - "fmla v27.8h, v7.8h, v4.h[2]\n" - "ldr q7, [x17, #0xd0]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "fmla v12.8h, v6.8h, v1.h[3]\n" - "fmla v16.8h, v6.8h, v2.h[3]\n" - "fmla v20.8h, v6.8h, v3.h[3]\n" - "fmla v24.8h, v6.8h, v4.h[3]\n" - "ldr q6, [x17, #0xe0]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "fmla v13.8h, v7.8h, v1.h[3]\n" - "fmla v17.8h, v7.8h, v2.h[3]\n" - "fmla v21.8h, v7.8h, v3.h[3]\n" - "fmla v25.8h, v7.8h, v4.h[3]\n" - "ldr q7, [x17, #0xf0]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "fmla v14.8h, v6.8h, v1.h[3]\n" - "fmla v18.8h, v6.8h, v2.h[3]\n" - "fmla v22.8h, v6.8h, v3.h[3]\n" - "fmla v26.8h, v6.8h, v4.h[3]\n" - "ldr q6, [x17, #0x100]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "fmla v15.8h, v7.8h, v1.h[3]\n" - "fmla v19.8h, v7.8h, v2.h[3]\n" - "fmla v23.8h, v7.8h, v3.h[3]\n" - "fmla v27.8h, v7.8h, v4.h[3]\n" - "ldr q7, [x17, #0x110]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "fmla v12.8h, v6.8h, v1.h[4]\n" - "fmla v16.8h, v6.8h, v2.h[4]\n" - "fmla v20.8h, v6.8h, v3.h[4]\n" - "fmla v24.8h, v6.8h, v4.h[4]\n" - "ldr q6, [x17, #0x120]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "fmla v13.8h, v7.8h, v1.h[4]\n" - "fmla v17.8h, v7.8h, v2.h[4]\n" - "fmla v21.8h, v7.8h, v3.h[4]\n" - "fmla v25.8h, v7.8h, v4.h[4]\n" - "ldr q7, [x17, #0x130]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "fmla v14.8h, v6.8h, v1.h[4]\n" - "fmla v18.8h, v6.8h, v2.h[4]\n" - "fmla v22.8h, v6.8h, v3.h[4]\n" - "fmla v26.8h, v6.8h, v4.h[4]\n" - "ldr q6, [x17, #0x140]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "fmla v15.8h, v7.8h, v1.h[4]\n" - "fmla v19.8h, v7.8h, v2.h[4]\n" - "fmla v23.8h, v7.8h, v3.h[4]\n" - "fmla v27.8h, v7.8h, v4.h[4]\n" - "ldr q7, [x17, #0x150]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "fmla v12.8h, v6.8h, v1.h[5]\n" - "fmla v16.8h, v6.8h, v2.h[5]\n" - "fmla v20.8h, v6.8h, v3.h[5]\n" - "fmla v24.8h, v6.8h, v4.h[5]\n" - "ldr q6, [x17, #0x160]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "fmla v13.8h, v7.8h, v1.h[5]\n" - "fmla v17.8h, v7.8h, v2.h[5]\n" - "fmla v21.8h, v7.8h, v3.h[5]\n" - "fmla v25.8h, v7.8h, v4.h[5]\n" - "ldr q7, [x17, #0x170]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "fmla v14.8h, v6.8h, v1.h[5]\n" - "fmla v18.8h, v6.8h, v2.h[5]\n" - "fmla v22.8h, v6.8h, v3.h[5]\n" - "fmla v26.8h, v6.8h, v4.h[5]\n" - "ldr q6, [x17, #0x180]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "fmla v15.8h, v7.8h, v1.h[5]\n" - "fmla v19.8h, v7.8h, v2.h[5]\n" - "fmla v23.8h, v7.8h, v3.h[5]\n" - "fmla v27.8h, v7.8h, v4.h[5]\n" - "ldr q7, [x17, #0x190]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "fmla v12.8h, v6.8h, v1.h[6]\n" - "fmla v16.8h, v6.8h, v2.h[6]\n" - "fmla v20.8h, v6.8h, v3.h[6]\n" - "fmla v24.8h, v6.8h, v4.h[6]\n" - "ldr q6, [x17, #0x1a0]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "fmla v13.8h, v7.8h, v1.h[6]\n" - "fmla v17.8h, v7.8h, v2.h[6]\n" - "fmla v21.8h, v7.8h, v3.h[6]\n" - "fmla v25.8h, v7.8h, v4.h[6]\n" - "ldr q7, [x17, #0x1b0]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "fmla v14.8h, v6.8h, v1.h[6]\n" - "fmla v18.8h, v6.8h, v2.h[6]\n" - "fmla v22.8h, v6.8h, v3.h[6]\n" - "fmla v26.8h, v6.8h, v4.h[6]\n" - "ldr q6, [x17, #0x1c0]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "fmla v15.8h, v7.8h, v1.h[6]\n" - "fmla v19.8h, v7.8h, v2.h[6]\n" - "fmla v23.8h, v7.8h, v3.h[6]\n" - "fmla v27.8h, v7.8h, v4.h[6]\n" - "ldr q7, [x17, #0x1d0]\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "fmla v12.8h, v6.8h, v1.h[7]\n" - "fmla v16.8h, v6.8h, v2.h[7]\n" - "fmla v20.8h, v6.8h, v3.h[7]\n" - "fmla v24.8h, v6.8h, v4.h[7]\n" - "ldr q6, [x17, #0x1e0]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "fmla v13.8h, v7.8h, v1.h[7]\n" - "fmla v17.8h, v7.8h, v2.h[7]\n" - "fmla v21.8h, v7.8h, v3.h[7]\n" - "fmla v25.8h, v7.8h, v4.h[7]\n" - "ldr q7, [x17, #0x1f0]\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "add x17, x17, #0x200\n" - "fmla v14.8h, v6.8h, v1.h[7]\n" - "fmla v18.8h, v6.8h, v2.h[7]\n" - "fmla v22.8h, v6.8h, v3.h[7]\n" - "fmla v26.8h, v6.8h, v4.h[7]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" - "fmla v15.8h, v7.8h, v1.h[7]\n" - "fmla v19.8h, v7.8h, v2.h[7]\n" - "fmla v23.8h, v7.8h, v3.h[7]\n" - "fmla v27.8h, v7.8h, v4.h[7]\n" - "224:" // Height 5: Multiply loop: Main loop skip - "cbz x14, 226f\n" - "225:" // Height 5: Multiply loop: Odd block loop - "ldr h0, [x13], #0x2\n" - "sub x14, x14, #0x1\n" - "ldr h1, [x9], #0x2\n" - "ldr h2, [x27], #0x2\n" - "ldr h3, [x25], #0x2\n" - "ldr h4, [x23], #0x2\n" + "mov x20, #0x10\n" + "tbz x8, #0, 215f\n" + "ldr h9, [x16, #0x0]\n" + "ldr h13, [x25, #0x0]\n" + "ldr h17, [x24, #0x0]\n" + "ldr h21, [x23, #0x0]\n" + "ldr h25, [x22, #0x0]\n" + "b 215f\n" + "211:" // Height 5: Partial accumulate: partial_4_0 + "tbz x8, #2, 213f\n" + "ldr d8, [x16], #0x8\n" + "ldr d12, [x25], #0x8\n" + "ldr d16, [x24], #0x8\n" + "ldr d20, [x23], #0x8\n" + "ldr d24, [x22], #0x8\n" + "tbz x8, #1, 212f\n" + "ld1 { v8.s }[2], [x16], #0x4\n" + "mov x20, #0xc\n" + "ld1 { v12.s }[2], [x25], #0x4\n" + "ld1 { v16.s }[2], [x24], #0x4\n" + "ld1 { v20.s }[2], [x23], #0x4\n" + "ld1 { v24.s }[2], [x22], #0x4\n" + "tbz x8, #0, 215f\n" + "ld1 { v8.h }[6], [x16]\n" + "ld1 { v12.h }[6], [x25]\n" + "ld1 { v16.h }[6], [x24]\n" + "ld1 { v20.h }[6], [x23]\n" + "ld1 { v24.h }[6], [x22]\n" + "b 215f\n" + "212:" // Height 5: Partial accumulate: partial_1_4 + "mov x20, #0x8\n" + "tbz x8, #0, 215f\n" + "ld1 { v8.h }[4], [x16]\n" + "ld1 { v12.h }[4], [x25]\n" + "ld1 { v16.h }[4], [x24]\n" + "ld1 { v20.h }[4], [x23]\n" + "ld1 { v24.h }[4], [x22]\n" + "b 215f\n" + "213:" // Height 5: Partial accumulate: partial_2_0 + "tbz x8, #1, 214f\n" + "ldr s8, [x16], #0x4\n" + "mov x20, #0x4\n" + "ldr s12, [x25], #0x4\n" + "ldr s16, [x24], #0x4\n" + "ldr s20, [x23], #0x4\n" + "ldr s24, [x22], #0x4\n" + "tbz x8, #0, 215f\n" + "ld1 { v8.h }[2], [x16]\n" + "ld1 { v12.h }[2], [x25]\n" + "ld1 { v16.h }[2], [x24]\n" + "ld1 { v20.h }[2], [x23]\n" + "ld1 { v24.h }[2], [x22]\n" + "b 215f\n" + "214:" // Height 5: Partial accumulate: partial_1_0 + "ldr h8, [x16, #0x0]\n" + "mov x20, #0x0\n" + "ldr h12, [x25, #0x0]\n" + "ldr h16, [x24, #0x0]\n" + "ldr h20, [x23, #0x0]\n" + "ldr h24, [x22, #0x0]\n" + "215:" // Height 5: Partial accumulate: Done + "sub x16, x16, x20\n" + "b 218f\n" + "216:" // Height 5: full accumulate + "ldr q8, [x16, #0x0]\n" + "ldr q9, [x16, #0x10]\n" + "ldr q10, [x16, #0x20]\n" + "ldr q11, [x16, #0x30]\n" + "ldr q12, [x25, #0x0]\n" + "ldr q13, [x25, #0x10]\n" + "ldr q14, [x25, #0x20]\n" + "ldr q15, [x25, #0x30]\n" + "ldr q16, [x24, #0x0]\n" + "ldr q17, [x24, #0x10]\n" + "ldr q18, [x24, #0x20]\n" + "ldr q19, [x24, #0x30]\n" + "ldr q20, [x23, #0x0]\n" + "ldr q21, [x23, #0x10]\n" + "ldr q22, [x23, #0x20]\n" + "ldr q23, [x23, #0x30]\n" + "ldr q24, [x22, #0x0]\n" + "ldr q25, [x22, #0x10]\n" + "ldr q26, [x22, #0x20]\n" + "ldr q27, [x22, #0x30]\n" + "b 218f\n" + "217:" // Height 5: no accumulate + "movi v8.16b, #0x0\n" + "movi v9.16b, #0x0\n" + "movi v10.16b, #0x0\n" + "movi v11.16b, #0x0\n" + "movi v12.16b, #0x0\n" + "movi v13.16b, #0x0\n" + "movi v14.16b, #0x0\n" + "movi v15.16b, #0x0\n" + "movi v16.16b, #0x0\n" + "movi v17.16b, #0x0\n" + "movi v18.16b, #0x0\n" + "movi v19.16b, #0x0\n" + "movi v20.16b, #0x0\n" + "movi v21.16b, #0x0\n" + "movi v22.16b, #0x0\n" + "movi v23.16b, #0x0\n" + "movi v24.16b, #0x0\n" + "movi v25.16b, #0x0\n" + "movi v26.16b, #0x0\n" + "movi v27.16b, #0x0\n" + "218:" // Height 5: setup done + "mov x15, #0x0\n" + "219:" // Height 5: String loop + "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" + "ldr w14, [x20, x15, LSL #0x2]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" + "tbz %x[flags], #3, 220f\n" + "ldr x20, [%x[input_ptr], x15, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x13, [x20, #0x0]\n" + "ldr x12, [x20, #0x8]\n" + "ldr x11, [x20, #0x10]\n" + "ldr x10, [x20, #0x18]\n" + "ldr x9, [x20, #0x20]\n" + "cbnz x15, 221f\n" + "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" + "add x13, x13, x20, LSL #1\n" + "add x12, x12, x20, LSL #1\n" + "add x11, x11, x20, LSL #1\n" + "add x10, x10, x20, LSL #1\n" + "add x9, x9, x20, LSL #1\n" + "b 221f\n" + "220:" // Height 5: setup direct input + "mov x13, %x[input_ptr]\n" + "add x12, x13, x21, LSL #1\n" + "add x11, x12, x21, LSL #1\n" + "add x10, x11, x21, LSL #1\n" + "add x9, x10, x21, LSL #1\n" + "221:" // Height 5: input setup done + "cmp x14, #0x8\n" + "blt 224f\n" + "ldr q0, [x13, #0x0]\n" + "cmp x14, #0x10\n" + "ldr q1, [x12, #0x0]\n" + "ldr q2, [x11, #0x0]\n" + "ldr q3, [x10, #0x0]\n" + "ldr q4, [x9, #0x0]\n" "ldr q6, [x17, #0x0]\n" - "fmla v8.8h, v6.8h, v0.h[0]\n" "ldr q7, [x17, #0x10]\n" + "blt 223f\n" + "222:" // Height 5: Multiply loop: Main loop head + "fmla v8.8h, v6.8h, v0.h[0]\n" + "ldr x21, [x17, #0x28]\n" "fmla v12.8h, v6.8h, v1.h[0]\n" + "ldr x20, [x17, #0x38]\n" "fmla v16.8h, v6.8h, v2.h[0]\n" + "add x13, x13, #0x10\n" "fmla v20.8h, v6.8h, v3.h[0]\n" + "add x12, x12, #0x10\n" "fmla v24.8h, v6.8h, v4.h[0]\n" - "ldr q6, [x17, #0x20]\n" + "ldr d29, [x17, #0x20]\n" "fmla v9.8h, v7.8h, v0.h[0]\n" + "mov v29.d[1], x21\n" "fmla v13.8h, v7.8h, v1.h[0]\n" + "ldr x21, [x17, #0x48]\n" "fmla v17.8h, v7.8h, v2.h[0]\n" + "add x11, x11, #0x10\n" "fmla v21.8h, v7.8h, v3.h[0]\n" + "add x10, x10, #0x10\n" "fmla v25.8h, v7.8h, v4.h[0]\n" - "ldr q7, [x17, #0x30]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" + "ldr d28, [x17, #0x30]\n" + "mov v28.d[1], x20\n" + "fmla v10.8h, v29.8h, v0.h[0]\n" + "fmla v14.8h, v29.8h, v1.h[0]\n" + "ldr x20, [x17, #0x58]\n" + "fmla v18.8h, v29.8h, v2.h[0]\n" + "add x9, x9, #0x10\n" + "fmla v22.8h, v29.8h, v3.h[0]\n" + "ldr x26, [x13, #0x8]\n" + "fmla v26.8h, v29.8h, v4.h[0]\n" + "ldr d29, [x17, #0x40]\n" + "fmla v11.8h, v28.8h, v0.h[0]\n" + "mov v29.d[1], x21\n" + "fmla v15.8h, v28.8h, v1.h[0]\n" + "ldr x21, [x17, #0x68]\n" + "fmla v19.8h, v28.8h, v2.h[0]\n" + "ldr x25, [x12, #0x8]\n" + "fmla v23.8h, v28.8h, v3.h[0]\n" + "ldr x24, [x11, #0x8]\n" + "fmla v27.8h, v28.8h, v4.h[0]\n" + "ldr d28, [x17, #0x50]\n" + "mov v28.d[1], x20\n" + "fmla v8.8h, v29.8h, v0.h[1]\n" + "fmla v12.8h, v29.8h, v1.h[1]\n" + "ldr x20, [x17, #0x78]\n" + "fmla v16.8h, v29.8h, v2.h[1]\n" + "ldr x23, [x10, #0x8]\n" + "fmla v20.8h, v29.8h, v3.h[1]\n" + "ldr x22, [x9, #0x8]\n" + "fmla v24.8h, v29.8h, v4.h[1]\n" + "ldr d29, [x17, #0x60]\n" + "fmla v9.8h, v28.8h, v0.h[1]\n" + "mov v29.d[1], x21\n" + "fmla v13.8h, v28.8h, v1.h[1]\n" + "ldr x21, [x17, #0x88]\n" + "fmla v17.8h, v28.8h, v2.h[1]\n" + "sub x14, x14, #0x8\n" + "fmla v21.8h, v28.8h, v3.h[1]\n" + "cmp x14, #0x10\n" + "fmla v25.8h, v28.8h, v4.h[1]\n" + "ldr d28, [x17, #0x70]\n" + "mov v28.d[1], x20\n" + "fmla v10.8h, v29.8h, v0.h[1]\n" + "fmla v14.8h, v29.8h, v1.h[1]\n" + "ldr x20, [x17, #0x98]\n" + "fmla v18.8h, v29.8h, v2.h[1]\n" + "prfm pldl1keep, [x13, #0x80]\n" + "fmla v22.8h, v29.8h, v3.h[1]\n" + "prfm pldl1keep, [x12, #0x80]\n" + "fmla v26.8h, v29.8h, v4.h[1]\n" + "ldr d29, [x17, #0x80]\n" + "fmla v11.8h, v28.8h, v0.h[1]\n" + "mov v29.d[1], x21\n" + "fmla v15.8h, v28.8h, v1.h[1]\n" + "ldr x21, [x17, #0xa8]\n" + "fmla v19.8h, v28.8h, v2.h[1]\n" + "prfm pldl1keep, [x11, #0x80]\n" + "fmla v23.8h, v28.8h, v3.h[1]\n" + "prfm pldl1keep, [x10, #0x80]\n" + "fmla v27.8h, v28.8h, v4.h[1]\n" + "ldr d28, [x17, #0x90]\n" + "mov v28.d[1], x20\n" + "fmla v8.8h, v29.8h, v0.h[2]\n" + "fmla v12.8h, v29.8h, v1.h[2]\n" + "ldr x20, [x17, #0xb8]\n" + "fmla v16.8h, v29.8h, v2.h[2]\n" + "prfm pldl1keep, [x9, #0x80]\n" + "fmla v20.8h, v29.8h, v3.h[2]\n" + "fmla v24.8h, v29.8h, v4.h[2]\n" + "ldr d29, [x17, #0xa0]\n" + "fmla v9.8h, v28.8h, v0.h[2]\n" + "mov v29.d[1], x21\n" + "fmla v13.8h, v28.8h, v1.h[2]\n" + "ldr x21, [x17, #0xc8]\n" + "fmla v17.8h, v28.8h, v2.h[2]\n" + "fmla v21.8h, v28.8h, v3.h[2]\n" + "fmla v25.8h, v28.8h, v4.h[2]\n" + "ldr d28, [x17, #0xb0]\n" + "mov v28.d[1], x20\n" + "fmla v10.8h, v29.8h, v0.h[2]\n" + "fmla v14.8h, v29.8h, v1.h[2]\n" + "ldr x20, [x17, #0xd8]\n" + "fmla v18.8h, v29.8h, v2.h[2]\n" + "fmla v22.8h, v29.8h, v3.h[2]\n" + "fmla v26.8h, v29.8h, v4.h[2]\n" + "ldr d29, [x17, #0xc0]\n" + "fmla v11.8h, v28.8h, v0.h[2]\n" + "mov v29.d[1], x21\n" + "fmla v15.8h, v28.8h, v1.h[2]\n" + "ldr x21, [x17, #0xe8]\n" + "fmla v19.8h, v28.8h, v2.h[2]\n" + "fmla v23.8h, v28.8h, v3.h[2]\n" + "fmla v27.8h, v28.8h, v4.h[2]\n" + "ldr d28, [x17, #0xd0]\n" + "mov v28.d[1], x20\n" + "fmla v8.8h, v29.8h, v0.h[3]\n" + "fmla v12.8h, v29.8h, v1.h[3]\n" + "ldr x20, [x17, #0xf8]\n" + "fmla v16.8h, v29.8h, v2.h[3]\n" + "fmla v20.8h, v29.8h, v3.h[3]\n" + "fmla v24.8h, v29.8h, v4.h[3]\n" + "ldr d29, [x17, #0xe0]\n" + "fmla v9.8h, v28.8h, v0.h[3]\n" + "mov v29.d[1], x21\n" + "fmla v13.8h, v28.8h, v1.h[3]\n" + "ldr x21, [x17, #0x108]\n" + "fmla v17.8h, v28.8h, v2.h[3]\n" + "fmla v21.8h, v28.8h, v3.h[3]\n" + "fmla v25.8h, v28.8h, v4.h[3]\n" + "ldr d28, [x17, #0xf0]\n" + "mov v28.d[1], x20\n" + "fmla v10.8h, v29.8h, v0.h[3]\n" + "fmla v14.8h, v29.8h, v1.h[3]\n" + "ldr x20, [x17, #0x118]\n" + "fmla v18.8h, v29.8h, v2.h[3]\n" + "fmla v22.8h, v29.8h, v3.h[3]\n" + "fmla v26.8h, v29.8h, v4.h[3]\n" + "ldr d29, [x17, #0x100]\n" + "fmla v11.8h, v28.8h, v0.h[3]\n" + "mov v29.d[1], x21\n" + "fmla v15.8h, v28.8h, v1.h[3]\n" + "ldr x21, [x17, #0x128]\n" + "fmla v19.8h, v28.8h, v2.h[3]\n" + "fmla v23.8h, v28.8h, v3.h[3]\n" + "fmla v27.8h, v28.8h, v4.h[3]\n" + "ldr d28, [x17, #0x110]\n" + "mov v28.d[1], x20\n" + "fmla v8.8h, v29.8h, v0.h[4]\n" + "fmla v12.8h, v29.8h, v1.h[4]\n" + "ldr x20, [x17, #0x138]\n" + "fmla v16.8h, v29.8h, v2.h[4]\n" + "fmla v20.8h, v29.8h, v3.h[4]\n" + "fmla v24.8h, v29.8h, v4.h[4]\n" + "ldr d29, [x17, #0x120]\n" + "fmla v9.8h, v28.8h, v0.h[4]\n" + "mov v29.d[1], x21\n" + "fmla v13.8h, v28.8h, v1.h[4]\n" + "ldr x21, [x17, #0x148]\n" + "fmla v17.8h, v28.8h, v2.h[4]\n" + "fmla v21.8h, v28.8h, v3.h[4]\n" + "fmla v25.8h, v28.8h, v4.h[4]\n" + "ldr d28, [x17, #0x130]\n" + "mov v28.d[1], x20\n" + "fmla v10.8h, v29.8h, v0.h[4]\n" + "fmla v14.8h, v29.8h, v1.h[4]\n" + "ldr x20, [x17, #0x158]\n" + "fmla v18.8h, v29.8h, v2.h[4]\n" + "fmla v22.8h, v29.8h, v3.h[4]\n" + "fmla v26.8h, v29.8h, v4.h[4]\n" + "ldr d29, [x17, #0x140]\n" + "fmla v11.8h, v28.8h, v0.h[4]\n" + "mov v29.d[1], x21\n" + "fmla v15.8h, v28.8h, v1.h[4]\n" + "ldr x21, [x17, #0x168]\n" + "fmla v19.8h, v28.8h, v2.h[4]\n" + "fmla v23.8h, v28.8h, v3.h[4]\n" + "fmla v27.8h, v28.8h, v4.h[4]\n" + "ldr d28, [x17, #0x150]\n" + "mov v28.d[1], x20\n" + "fmla v8.8h, v29.8h, v0.h[5]\n" + "fmla v12.8h, v29.8h, v1.h[5]\n" + "ldr x20, [x17, #0x178]\n" + "fmla v16.8h, v29.8h, v2.h[5]\n" + "fmla v20.8h, v29.8h, v3.h[5]\n" + "fmla v24.8h, v29.8h, v4.h[5]\n" + "ldr d29, [x17, #0x160]\n" + "fmla v9.8h, v28.8h, v0.h[5]\n" + "mov v29.d[1], x21\n" + "fmla v13.8h, v28.8h, v1.h[5]\n" + "ldr x21, [x17, #0x188]\n" + "fmla v17.8h, v28.8h, v2.h[5]\n" + "fmla v21.8h, v28.8h, v3.h[5]\n" + "fmla v25.8h, v28.8h, v4.h[5]\n" + "ldr d28, [x17, #0x170]\n" + "mov v28.d[1], x20\n" + "fmla v10.8h, v29.8h, v0.h[5]\n" + "fmla v14.8h, v29.8h, v1.h[5]\n" + "ldr x20, [x17, #0x198]\n" + "fmla v18.8h, v29.8h, v2.h[5]\n" + "fmla v22.8h, v29.8h, v3.h[5]\n" + "fmla v26.8h, v29.8h, v4.h[5]\n" + "ldr d29, [x17, #0x180]\n" + "fmla v11.8h, v28.8h, v0.h[5]\n" + "mov v29.d[1], x21\n" + "fmla v15.8h, v28.8h, v1.h[5]\n" + "ldr x21, [x17, #0x1a8]\n" + "fmla v19.8h, v28.8h, v2.h[5]\n" + "fmla v23.8h, v28.8h, v3.h[5]\n" + "fmla v27.8h, v28.8h, v4.h[5]\n" + "ldr d28, [x17, #0x190]\n" + "mov v28.d[1], x20\n" + "fmla v8.8h, v29.8h, v0.h[6]\n" + "fmla v12.8h, v29.8h, v1.h[6]\n" + "ldr x20, [x17, #0x1b8]\n" + "fmla v16.8h, v29.8h, v2.h[6]\n" + "fmla v20.8h, v29.8h, v3.h[6]\n" + "fmla v24.8h, v29.8h, v4.h[6]\n" + "ldr d29, [x17, #0x1a0]\n" + "fmla v9.8h, v28.8h, v0.h[6]\n" + "mov v29.d[1], x21\n" + "fmla v13.8h, v28.8h, v1.h[6]\n" + "ldr x21, [x17, #0x1c8]\n" + "fmla v17.8h, v28.8h, v2.h[6]\n" + "fmla v21.8h, v28.8h, v3.h[6]\n" + "fmla v25.8h, v28.8h, v4.h[6]\n" + "ldr d28, [x17, #0x1b0]\n" + "mov v28.d[1], x20\n" + "fmla v10.8h, v29.8h, v0.h[6]\n" + "fmla v14.8h, v29.8h, v1.h[6]\n" + "ldr x20, [x17, #0x1d8]\n" + "fmla v18.8h, v29.8h, v2.h[6]\n" + "fmla v22.8h, v29.8h, v3.h[6]\n" + "fmla v26.8h, v29.8h, v4.h[6]\n" + "ldr d29, [x17, #0x1c0]\n" + "fmla v11.8h, v28.8h, v0.h[6]\n" + "mov v29.d[1], x21\n" + "fmla v15.8h, v28.8h, v1.h[6]\n" + "ldr x21, [x17, #0x1e8]\n" + "fmla v19.8h, v28.8h, v2.h[6]\n" + "fmla v23.8h, v28.8h, v3.h[6]\n" + "fmla v27.8h, v28.8h, v4.h[6]\n" + "ldr d28, [x17, #0x1d0]\n" + "mov v28.d[1], x20\n" + "fmla v8.8h, v29.8h, v0.h[7]\n" + "fmla v12.8h, v29.8h, v1.h[7]\n" + "ldr x20, [x17, #0x1f8]\n" + "fmla v16.8h, v29.8h, v2.h[7]\n" + "fmla v20.8h, v29.8h, v3.h[7]\n" + "fmla v24.8h, v29.8h, v4.h[7]\n" + "ldr d29, [x17, #0x1e0]\n" + "fmla v9.8h, v28.8h, v0.h[7]\n" + "mov v29.d[1], x21\n" + "fmla v13.8h, v28.8h, v1.h[7]\n" + "fmla v17.8h, v28.8h, v2.h[7]\n" + "fmla v21.8h, v28.8h, v3.h[7]\n" + "fmla v25.8h, v28.8h, v4.h[7]\n" + "ldr d28, [x17, #0x1f0]\n" + "mov v28.d[1], x20\n" + "add x17, x17, #0x200\n" + "fmla v10.8h, v29.8h, v0.h[7]\n" + "ldr x21, [x17, #0x8]\n" + "fmla v14.8h, v29.8h, v1.h[7]\n" + "ldr x20, [x17, #0x18]\n" + "fmla v18.8h, v29.8h, v2.h[7]\n" + "fmla v22.8h, v29.8h, v3.h[7]\n" + "fmla v26.8h, v29.8h, v4.h[7]\n" + "ldr d6, [x17, #0x0]\n" + "fmla v11.8h, v28.8h, v0.h[7]\n" + "ldr d0, [x13, #0x0]\n" + "fmla v15.8h, v28.8h, v1.h[7]\n" + "ldr d1, [x12, #0x0]\n" + "fmla v19.8h, v28.8h, v2.h[7]\n" + "ldr d2, [x11, #0x0]\n" + "fmla v23.8h, v28.8h, v3.h[7]\n" + "ldr d3, [x10, #0x0]\n" + "fmla v27.8h, v28.8h, v4.h[7]\n" + "ldr d4, [x9, #0x0]\n" + "ldr d7, [x17, #0x10]\n" + "mov v6.d[1], x21\n" + "mov v0.d[1], x26\n" + "mov v1.d[1], x25\n" + "mov v2.d[1], x24\n" + "mov v3.d[1], x23\n" + "mov v4.d[1], x22\n" + "mov v7.d[1], x20\n" + "bge 222b\n" + "223:" // Height 5: Multiply loop: Single iteration only + "fmla v8.8h, v6.8h, v0.h[0]\n" + "add x13, x13, #0x10\n" + "fmla v12.8h, v6.8h, v1.h[0]\n" + "add x12, x12, #0x10\n" + "fmla v16.8h, v6.8h, v2.h[0]\n" + "add x11, x11, #0x10\n" + "fmla v20.8h, v6.8h, v3.h[0]\n" + "add x10, x10, #0x10\n" + "fmla v24.8h, v6.8h, v4.h[0]\n" + "ldr q29, [x17, #0x20]\n" + "fmla v9.8h, v7.8h, v0.h[0]\n" + "add x9, x9, #0x10\n" + "fmla v13.8h, v7.8h, v1.h[0]\n" + "sub x14, x14, #0x8\n" + "fmla v17.8h, v7.8h, v2.h[0]\n" + "prfm pldl1keep, [x13, #0x80]\n" + "fmla v21.8h, v7.8h, v3.h[0]\n" + "prfm pldl1keep, [x12, #0x80]\n" + "fmla v25.8h, v7.8h, v4.h[0]\n" + "ldr q28, [x17, #0x30]\n" + "fmla v10.8h, v29.8h, v0.h[0]\n" + "prfm pldl1keep, [x11, #0x80]\n" + "fmla v14.8h, v29.8h, v1.h[0]\n" + "prfm pldl1keep, [x10, #0x80]\n" + "fmla v18.8h, v29.8h, v2.h[0]\n" + "prfm pldl1keep, [x9, #0x80]\n" + "fmla v22.8h, v29.8h, v3.h[0]\n" + "fmla v26.8h, v29.8h, v4.h[0]\n" + "ldr q29, [x17, #0x40]\n" + "fmla v11.8h, v28.8h, v0.h[0]\n" + "fmla v15.8h, v28.8h, v1.h[0]\n" + "fmla v19.8h, v28.8h, v2.h[0]\n" + "fmla v23.8h, v28.8h, v3.h[0]\n" + "fmla v27.8h, v28.8h, v4.h[0]\n" + "ldr q28, [x17, #0x50]\n" + "fmla v8.8h, v29.8h, v0.h[1]\n" + "fmla v12.8h, v29.8h, v1.h[1]\n" + "fmla v16.8h, v29.8h, v2.h[1]\n" + "fmla v20.8h, v29.8h, v3.h[1]\n" + "fmla v24.8h, v29.8h, v4.h[1]\n" + "ldr q29, [x17, #0x60]\n" + "fmla v9.8h, v28.8h, v0.h[1]\n" + "fmla v13.8h, v28.8h, v1.h[1]\n" + "fmla v17.8h, v28.8h, v2.h[1]\n" + "fmla v21.8h, v28.8h, v3.h[1]\n" + "fmla v25.8h, v28.8h, v4.h[1]\n" + "ldr q28, [x17, #0x70]\n" + "fmla v10.8h, v29.8h, v0.h[1]\n" + "fmla v14.8h, v29.8h, v1.h[1]\n" + "fmla v18.8h, v29.8h, v2.h[1]\n" + "fmla v22.8h, v29.8h, v3.h[1]\n" + "fmla v26.8h, v29.8h, v4.h[1]\n" + "ldr q29, [x17, #0x80]\n" + "fmla v11.8h, v28.8h, v0.h[1]\n" + "fmla v15.8h, v28.8h, v1.h[1]\n" + "fmla v19.8h, v28.8h, v2.h[1]\n" + "fmla v23.8h, v28.8h, v3.h[1]\n" + "fmla v27.8h, v28.8h, v4.h[1]\n" + "ldr q28, [x17, #0x90]\n" + "fmla v8.8h, v29.8h, v0.h[2]\n" + "fmla v12.8h, v29.8h, v1.h[2]\n" + "fmla v16.8h, v29.8h, v2.h[2]\n" + "fmla v20.8h, v29.8h, v3.h[2]\n" + "fmla v24.8h, v29.8h, v4.h[2]\n" + "ldr q29, [x17, #0xa0]\n" + "fmla v9.8h, v28.8h, v0.h[2]\n" + "fmla v13.8h, v28.8h, v1.h[2]\n" + "fmla v17.8h, v28.8h, v2.h[2]\n" + "fmla v21.8h, v28.8h, v3.h[2]\n" + "fmla v25.8h, v28.8h, v4.h[2]\n" + "ldr q28, [x17, #0xb0]\n" + "fmla v10.8h, v29.8h, v0.h[2]\n" + "fmla v14.8h, v29.8h, v1.h[2]\n" + "fmla v18.8h, v29.8h, v2.h[2]\n" + "fmla v22.8h, v29.8h, v3.h[2]\n" + "fmla v26.8h, v29.8h, v4.h[2]\n" + "ldr q29, [x17, #0xc0]\n" + "fmla v11.8h, v28.8h, v0.h[2]\n" + "fmla v15.8h, v28.8h, v1.h[2]\n" + "fmla v19.8h, v28.8h, v2.h[2]\n" + "fmla v23.8h, v28.8h, v3.h[2]\n" + "fmla v27.8h, v28.8h, v4.h[2]\n" + "ldr q28, [x17, #0xd0]\n" + "fmla v8.8h, v29.8h, v0.h[3]\n" + "fmla v12.8h, v29.8h, v1.h[3]\n" + "fmla v16.8h, v29.8h, v2.h[3]\n" + "fmla v20.8h, v29.8h, v3.h[3]\n" + "fmla v24.8h, v29.8h, v4.h[3]\n" + "ldr q29, [x17, #0xe0]\n" + "fmla v9.8h, v28.8h, v0.h[3]\n" + "fmla v13.8h, v28.8h, v1.h[3]\n" + "fmla v17.8h, v28.8h, v2.h[3]\n" + "fmla v21.8h, v28.8h, v3.h[3]\n" + "fmla v25.8h, v28.8h, v4.h[3]\n" + "ldr q28, [x17, #0xf0]\n" + "fmla v10.8h, v29.8h, v0.h[3]\n" + "fmla v14.8h, v29.8h, v1.h[3]\n" + "fmla v18.8h, v29.8h, v2.h[3]\n" + "fmla v22.8h, v29.8h, v3.h[3]\n" + "fmla v26.8h, v29.8h, v4.h[3]\n" + "ldr q29, [x17, #0x100]\n" + "fmla v11.8h, v28.8h, v0.h[3]\n" + "fmla v15.8h, v28.8h, v1.h[3]\n" + "fmla v19.8h, v28.8h, v2.h[3]\n" + "fmla v23.8h, v28.8h, v3.h[3]\n" + "fmla v27.8h, v28.8h, v4.h[3]\n" + "ldr q28, [x17, #0x110]\n" + "fmla v8.8h, v29.8h, v0.h[4]\n" + "fmla v12.8h, v29.8h, v1.h[4]\n" + "fmla v16.8h, v29.8h, v2.h[4]\n" + "fmla v20.8h, v29.8h, v3.h[4]\n" + "fmla v24.8h, v29.8h, v4.h[4]\n" + "ldr q29, [x17, #0x120]\n" + "fmla v9.8h, v28.8h, v0.h[4]\n" + "fmla v13.8h, v28.8h, v1.h[4]\n" + "fmla v17.8h, v28.8h, v2.h[4]\n" + "fmla v21.8h, v28.8h, v3.h[4]\n" + "fmla v25.8h, v28.8h, v4.h[4]\n" + "ldr q28, [x17, #0x130]\n" + "fmla v10.8h, v29.8h, v0.h[4]\n" + "fmla v14.8h, v29.8h, v1.h[4]\n" + "fmla v18.8h, v29.8h, v2.h[4]\n" + "fmla v22.8h, v29.8h, v3.h[4]\n" + "fmla v26.8h, v29.8h, v4.h[4]\n" + "ldr q29, [x17, #0x140]\n" + "fmla v11.8h, v28.8h, v0.h[4]\n" + "fmla v15.8h, v28.8h, v1.h[4]\n" + "fmla v19.8h, v28.8h, v2.h[4]\n" + "fmla v23.8h, v28.8h, v3.h[4]\n" + "fmla v27.8h, v28.8h, v4.h[4]\n" + "ldr q28, [x17, #0x150]\n" + "fmla v8.8h, v29.8h, v0.h[5]\n" + "fmla v12.8h, v29.8h, v1.h[5]\n" + "fmla v16.8h, v29.8h, v2.h[5]\n" + "fmla v20.8h, v29.8h, v3.h[5]\n" + "fmla v24.8h, v29.8h, v4.h[5]\n" + "ldr q29, [x17, #0x160]\n" + "fmla v9.8h, v28.8h, v0.h[5]\n" + "fmla v13.8h, v28.8h, v1.h[5]\n" + "fmla v17.8h, v28.8h, v2.h[5]\n" + "fmla v21.8h, v28.8h, v3.h[5]\n" + "fmla v25.8h, v28.8h, v4.h[5]\n" + "ldr q28, [x17, #0x170]\n" + "fmla v10.8h, v29.8h, v0.h[5]\n" + "fmla v14.8h, v29.8h, v1.h[5]\n" + "fmla v18.8h, v29.8h, v2.h[5]\n" + "fmla v22.8h, v29.8h, v3.h[5]\n" + "fmla v26.8h, v29.8h, v4.h[5]\n" + "ldr q29, [x17, #0x180]\n" + "fmla v11.8h, v28.8h, v0.h[5]\n" + "fmla v15.8h, v28.8h, v1.h[5]\n" + "fmla v19.8h, v28.8h, v2.h[5]\n" + "fmla v23.8h, v28.8h, v3.h[5]\n" + "fmla v27.8h, v28.8h, v4.h[5]\n" + "ldr q28, [x17, #0x190]\n" + "fmla v8.8h, v29.8h, v0.h[6]\n" + "fmla v12.8h, v29.8h, v1.h[6]\n" + "fmla v16.8h, v29.8h, v2.h[6]\n" + "fmla v20.8h, v29.8h, v3.h[6]\n" + "fmla v24.8h, v29.8h, v4.h[6]\n" + "ldr q29, [x17, #0x1a0]\n" + "fmla v9.8h, v28.8h, v0.h[6]\n" + "fmla v13.8h, v28.8h, v1.h[6]\n" + "fmla v17.8h, v28.8h, v2.h[6]\n" + "fmla v21.8h, v28.8h, v3.h[6]\n" + "fmla v25.8h, v28.8h, v4.h[6]\n" + "ldr q28, [x17, #0x1b0]\n" + "fmla v10.8h, v29.8h, v0.h[6]\n" + "fmla v14.8h, v29.8h, v1.h[6]\n" + "fmla v18.8h, v29.8h, v2.h[6]\n" + "fmla v22.8h, v29.8h, v3.h[6]\n" + "fmla v26.8h, v29.8h, v4.h[6]\n" + "ldr q29, [x17, #0x1c0]\n" + "fmla v11.8h, v28.8h, v0.h[6]\n" + "fmla v15.8h, v28.8h, v1.h[6]\n" + "fmla v19.8h, v28.8h, v2.h[6]\n" + "fmla v23.8h, v28.8h, v3.h[6]\n" + "fmla v27.8h, v28.8h, v4.h[6]\n" + "ldr q28, [x17, #0x1d0]\n" + "fmla v8.8h, v29.8h, v0.h[7]\n" + "fmla v12.8h, v29.8h, v1.h[7]\n" + "fmla v16.8h, v29.8h, v2.h[7]\n" + "fmla v20.8h, v29.8h, v3.h[7]\n" + "fmla v24.8h, v29.8h, v4.h[7]\n" + "ldr q29, [x17, #0x1e0]\n" + "fmla v9.8h, v28.8h, v0.h[7]\n" + "fmla v13.8h, v28.8h, v1.h[7]\n" + "fmla v17.8h, v28.8h, v2.h[7]\n" + "fmla v21.8h, v28.8h, v3.h[7]\n" + "fmla v25.8h, v28.8h, v4.h[7]\n" + "ldr q28, [x17, #0x1f0]\n" + "fmla v10.8h, v29.8h, v0.h[7]\n" + "add x17, x17, #0x200\n" + "fmla v14.8h, v29.8h, v1.h[7]\n" + "fmla v18.8h, v29.8h, v2.h[7]\n" + "fmla v22.8h, v29.8h, v3.h[7]\n" + "fmla v26.8h, v29.8h, v4.h[7]\n" + "fmla v11.8h, v28.8h, v0.h[7]\n" + "fmla v15.8h, v28.8h, v1.h[7]\n" + "fmla v19.8h, v28.8h, v2.h[7]\n" + "fmla v23.8h, v28.8h, v3.h[7]\n" + "fmla v27.8h, v28.8h, v4.h[7]\n" + "224:" // Height 5: Multiply loop: Main loop skip + "cbz x14, 226f\n" + "225:" // Height 5: Multiply loop: Odd block loop + "ldr h4, [x13], #0x2\n" + "sub x14, x14, #0x1\n" + "ldr h3, [x12], #0x2\n" + "ldr h2, [x11], #0x2\n" + "ldr h1, [x10], #0x2\n" + "ldr h0, [x9], #0x2\n" + "ldr q29, [x17, #0x0]\n" + "fmla v8.8h, v29.8h, v4.h[0]\n" + "ldr q28, [x17, #0x10]\n" + "fmla v12.8h, v29.8h, v3.h[0]\n" + "fmla v16.8h, v29.8h, v2.h[0]\n" + "fmla v20.8h, v29.8h, v1.h[0]\n" + "fmla v24.8h, v29.8h, v0.h[0]\n" + "ldr q29, [x17, #0x20]\n" + "fmla v9.8h, v28.8h, v4.h[0]\n" + "fmla v13.8h, v28.8h, v3.h[0]\n" + "fmla v17.8h, v28.8h, v2.h[0]\n" + "fmla v21.8h, v28.8h, v1.h[0]\n" + "fmla v25.8h, v28.8h, v0.h[0]\n" + "ldr q28, [x17, #0x30]\n" + "fmla v10.8h, v29.8h, v4.h[0]\n" "add x17, x17, #0x40\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "fmla v22.8h, v6.8h, v3.h[0]\n" - "fmla v26.8h, v6.8h, v4.h[0]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "fmla v23.8h, v7.8h, v3.h[0]\n" - "fmla v27.8h, v7.8h, v4.h[0]\n" + "fmla v14.8h, v29.8h, v3.h[0]\n" + "fmla v18.8h, v29.8h, v2.h[0]\n" + "fmla v22.8h, v29.8h, v1.h[0]\n" + "fmla v26.8h, v29.8h, v0.h[0]\n" + "fmla v11.8h, v28.8h, v4.h[0]\n" + "fmla v15.8h, v28.8h, v3.h[0]\n" + "fmla v19.8h, v28.8h, v2.h[0]\n" + "fmla v23.8h, v28.8h, v1.h[0]\n" + "fmla v27.8h, v28.8h, v0.h[0]\n" "cbnz x14, 225b\n" "226:" // Height 5: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -4069,49 +4069,49 @@ void a64_hybrid_fp16_mla_6x32_a55 ( "prfm pstl1keep, [x22, #0x0]\n" "tbz %x[flags], #1, 227f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v0.8h }, [x20]\n" - "fmin v8.8h, v8.8h, v0.8h\n" - "fmin v9.8h, v9.8h, v0.8h\n" - "fmin v10.8h, v10.8h, v0.8h\n" - "fmin v11.8h, v11.8h, v0.8h\n" - "fmin v12.8h, v12.8h, v0.8h\n" - "fmin v13.8h, v13.8h, v0.8h\n" - "fmin v14.8h, v14.8h, v0.8h\n" - "fmin v15.8h, v15.8h, v0.8h\n" - "fmin v16.8h, v16.8h, v0.8h\n" - "fmin v17.8h, v17.8h, v0.8h\n" - "fmin v18.8h, v18.8h, v0.8h\n" - "fmin v19.8h, v19.8h, v0.8h\n" - "fmin v20.8h, v20.8h, v0.8h\n" - "fmin v21.8h, v21.8h, v0.8h\n" - "fmin v22.8h, v22.8h, v0.8h\n" - "fmin v23.8h, v23.8h, v0.8h\n" - "fmin v24.8h, v24.8h, v0.8h\n" - "fmin v25.8h, v25.8h, v0.8h\n" - "fmin v26.8h, v26.8h, v0.8h\n" - "fmin v27.8h, v27.8h, v0.8h\n" + "ld1r { v28.8h }, [x20]\n" + "fmin v8.8h, v8.8h, v28.8h\n" + "fmin v9.8h, v9.8h, v28.8h\n" + "fmin v10.8h, v10.8h, v28.8h\n" + "fmin v11.8h, v11.8h, v28.8h\n" + "fmin v12.8h, v12.8h, v28.8h\n" + "fmin v13.8h, v13.8h, v28.8h\n" + "fmin v14.8h, v14.8h, v28.8h\n" + "fmin v15.8h, v15.8h, v28.8h\n" + "fmin v16.8h, v16.8h, v28.8h\n" + "fmin v17.8h, v17.8h, v28.8h\n" + "fmin v18.8h, v18.8h, v28.8h\n" + "fmin v19.8h, v19.8h, v28.8h\n" + "fmin v20.8h, v20.8h, v28.8h\n" + "fmin v21.8h, v21.8h, v28.8h\n" + "fmin v22.8h, v22.8h, v28.8h\n" + "fmin v23.8h, v23.8h, v28.8h\n" + "fmin v24.8h, v24.8h, v28.8h\n" + "fmin v25.8h, v25.8h, v28.8h\n" + "fmin v26.8h, v26.8h, v28.8h\n" + "fmin v27.8h, v27.8h, v28.8h\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.8h }, [x20]\n" - "fmax v8.8h, v8.8h, v0.8h\n" - "fmax v9.8h, v9.8h, v0.8h\n" - "fmax v10.8h, v10.8h, v0.8h\n" - "fmax v11.8h, v11.8h, v0.8h\n" - "fmax v12.8h, v12.8h, v0.8h\n" - "fmax v13.8h, v13.8h, v0.8h\n" - "fmax v14.8h, v14.8h, v0.8h\n" - "fmax v15.8h, v15.8h, v0.8h\n" - "fmax v16.8h, v16.8h, v0.8h\n" - "fmax v17.8h, v17.8h, v0.8h\n" - "fmax v18.8h, v18.8h, v0.8h\n" - "fmax v19.8h, v19.8h, v0.8h\n" - "fmax v20.8h, v20.8h, v0.8h\n" - "fmax v21.8h, v21.8h, v0.8h\n" - "fmax v22.8h, v22.8h, v0.8h\n" - "fmax v23.8h, v23.8h, v0.8h\n" - "fmax v24.8h, v24.8h, v0.8h\n" - "fmax v25.8h, v25.8h, v0.8h\n" - "fmax v26.8h, v26.8h, v0.8h\n" - "fmax v27.8h, v27.8h, v0.8h\n" + "ld1r { v28.8h }, [x20]\n" + "fmax v8.8h, v8.8h, v28.8h\n" + "fmax v9.8h, v9.8h, v28.8h\n" + "fmax v10.8h, v10.8h, v28.8h\n" + "fmax v11.8h, v11.8h, v28.8h\n" + "fmax v12.8h, v12.8h, v28.8h\n" + "fmax v13.8h, v13.8h, v28.8h\n" + "fmax v14.8h, v14.8h, v28.8h\n" + "fmax v15.8h, v15.8h, v28.8h\n" + "fmax v16.8h, v16.8h, v28.8h\n" + "fmax v17.8h, v17.8h, v28.8h\n" + "fmax v18.8h, v18.8h, v28.8h\n" + "fmax v19.8h, v19.8h, v28.8h\n" + "fmax v20.8h, v20.8h, v28.8h\n" + "fmax v21.8h, v21.8h, v28.8h\n" + "fmax v22.8h, v22.8h, v28.8h\n" + "fmax v23.8h, v23.8h, v28.8h\n" + "fmax v24.8h, v24.8h, v28.8h\n" + "fmax v25.8h, v25.8h, v28.8h\n" + "fmax v26.8h, v26.8h, v28.8h\n" + "fmax v27.8h, v27.8h, v28.8h\n" "227:" // Height 5: No activation "cmp x8, #0x20\n" "bge 244f\n" @@ -4736,98 +4736,98 @@ void a64_hybrid_fp16_mla_6x32_a55 ( "268:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w14, [x20, x15, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 269f\n" - "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x13, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" - "ldr x27, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" - "ldr x23, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x15, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x13, [x20, #0x0]\n" + "ldr x12, [x20, #0x8]\n" + "ldr x11, [x20, #0x10]\n" + "ldr x10, [x20, #0x18]\n" + "ldr x9, [x20, #0x20]\n" + "ldr x28, [x20, #0x28]\n" "cbnz x15, 270f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x13, x13, x20, LSL #1\n" + "add x12, x12, x20, LSL #1\n" + "add x11, x11, x20, LSL #1\n" + "add x10, x10, x20, LSL #1\n" "add x9, x9, x20, LSL #1\n" - "add x27, x27, x20, LSL #1\n" - "add x25, x25, x20, LSL #1\n" - "add x23, x23, x20, LSL #1\n" - "add x21, x21, x20, LSL #1\n" + "add x28, x28, x20, LSL #1\n" "b 270f\n" "269:" // Height 6: setup direct input "mov x13, %x[input_ptr]\n" - "add x9, x13, x20, LSL #1\n" - "add x27, x9, x20, LSL #1\n" - "add x25, x27, x20, LSL #1\n" - "add x23, x25, x20, LSL #1\n" - "add x21, x23, x20, LSL #1\n" + "add x12, x13, x21, LSL #1\n" + "add x11, x12, x21, LSL #1\n" + "add x10, x11, x21, LSL #1\n" + "add x9, x10, x21, LSL #1\n" + "add x28, x9, x21, LSL #1\n" "270:" // Height 6: input setup done "cmp x14, #0x8\n" "blt 273f\n" "ldr q0, [x13, #0x0]\n" "cmp x14, #0x10\n" - "ldr q1, [x9, #0x0]\n" - "ldr q2, [x27, #0x0]\n" - "ldr q3, [x25, #0x0]\n" - "ldr q4, [x23, #0x0]\n" - "ldr q5, [x21, #0x0]\n" + "ldr q1, [x12, #0x0]\n" + "ldr q2, [x11, #0x0]\n" + "ldr q3, [x10, #0x0]\n" + "ldr q4, [x9, #0x0]\n" + "ldr q5, [x28, #0x0]\n" "ldr q6, [x17, #0x0]\n" "ldr q7, [x17, #0x10]\n" "blt 272f\n" "271:" // Height 6: Multiply loop: Main loop head "fmla v8.8h, v6.8h, v0.h[0]\n" - "ldr x12, [x17, #0x28]\n" + "ldr x21, [x17, #0x28]\n" "fmla v12.8h, v6.8h, v1.h[0]\n" - "ldr x11, [x17, #0x38]\n" + "ldr x20, [x17, #0x38]\n" "fmla v16.8h, v6.8h, v2.h[0]\n" "add x13, x13, #0x10\n" "fmla v20.8h, v6.8h, v3.h[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" "fmla v24.8h, v6.8h, v4.h[0]\n" - "add x27, x27, #0x10\n" + "add x11, x11, #0x10\n" "fmla v28.8h, v6.8h, v5.h[0]\n" "ldr d6, [x17, #0x20]\n" "fmla v9.8h, v7.8h, v0.h[0]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" "fmla v13.8h, v7.8h, v1.h[0]\n" - "ldr x12, [x17, #0x48]\n" + "ldr x21, [x17, #0x48]\n" "fmla v17.8h, v7.8h, v2.h[0]\n" - "add x25, x25, #0x10\n" + "add x10, x10, #0x10\n" "fmla v21.8h, v7.8h, v3.h[0]\n" - "add x23, x23, #0x10\n" + "add x9, x9, #0x10\n" "fmla v25.8h, v7.8h, v4.h[0]\n" - "add x21, x21, #0x10\n" + "add x28, x28, #0x10\n" "fmla v29.8h, v7.8h, v5.h[0]\n" "ldr d7, [x17, #0x30]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "fmla v10.8h, v6.8h, v0.h[0]\n" "fmla v14.8h, v6.8h, v1.h[0]\n" - "ldr x11, [x17, #0x58]\n" + "ldr x20, [x17, #0x58]\n" "fmla v18.8h, v6.8h, v2.h[0]\n" - "ldr x10, [x13, #0x8]\n" + "ldr x27, [x13, #0x8]\n" "fmla v22.8h, v6.8h, v3.h[0]\n" - "ldr x28, [x9, #0x8]\n" + "ldr x26, [x12, #0x8]\n" "fmla v26.8h, v6.8h, v4.h[0]\n" - "ldr x26, [x27, #0x8]\n" + "ldr x25, [x11, #0x8]\n" "fmla v30.8h, v6.8h, v5.h[0]\n" "ldr d6, [x17, #0x40]\n" "fmla v11.8h, v7.8h, v0.h[0]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" "fmla v15.8h, v7.8h, v1.h[0]\n" - "ldr x12, [x17, #0x68]\n" + "ldr x21, [x17, #0x68]\n" "fmla v19.8h, v7.8h, v2.h[0]\n" - "ldr x24, [x25, #0x8]\n" + "ldr x24, [x10, #0x8]\n" "fmla v23.8h, v7.8h, v3.h[0]\n" - "ldr x22, [x23, #0x8]\n" + "ldr x23, [x9, #0x8]\n" "fmla v27.8h, v7.8h, v4.h[0]\n" - "ldr x20, [x21, #0x8]\n" + "ldr x22, [x28, #0x8]\n" "fmla v31.8h, v7.8h, v5.h[0]\n" "ldr d7, [x17, #0x50]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "fmla v8.8h, v6.8h, v0.h[1]\n" "fmla v12.8h, v6.8h, v1.h[1]\n" - "ldr x11, [x17, #0x78]\n" + "ldr x20, [x17, #0x78]\n" "fmla v16.8h, v6.8h, v2.h[1]\n" "sub x14, x14, #0x8\n" "fmla v20.8h, v6.8h, v3.h[1]\n" @@ -4837,240 +4837,240 @@ void a64_hybrid_fp16_mla_6x32_a55 ( "fmla v28.8h, v6.8h, v5.h[1]\n" "ldr d6, [x17, #0x60]\n" "fmla v9.8h, v7.8h, v0.h[1]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" "fmla v13.8h, v7.8h, v1.h[1]\n" - "ldr x12, [x17, #0x88]\n" + "ldr x21, [x17, #0x88]\n" "fmla v17.8h, v7.8h, v2.h[1]\n" - "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x12, #0x80]\n" "fmla v21.8h, v7.8h, v3.h[1]\n" - "prfm pldl1keep, [x27, #0x80]\n" + "prfm pldl1keep, [x11, #0x80]\n" "fmla v25.8h, v7.8h, v4.h[1]\n" - "prfm pldl1keep, [x25, #0x80]\n" + "prfm pldl1keep, [x10, #0x80]\n" "fmla v29.8h, v7.8h, v5.h[1]\n" "ldr d7, [x17, #0x70]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "fmla v10.8h, v6.8h, v0.h[1]\n" "fmla v14.8h, v6.8h, v1.h[1]\n" - "ldr x11, [x17, #0x98]\n" + "ldr x20, [x17, #0x98]\n" "fmla v18.8h, v6.8h, v2.h[1]\n" - "prfm pldl1keep, [x23, #0x80]\n" + "prfm pldl1keep, [x9, #0x80]\n" "fmla v22.8h, v6.8h, v3.h[1]\n" - "prfm pldl1keep, [x21, #0x80]\n" + "prfm pldl1keep, [x28, #0x80]\n" "fmla v26.8h, v6.8h, v4.h[1]\n" "fmla v30.8h, v6.8h, v5.h[1]\n" "ldr d6, [x17, #0x80]\n" "fmla v11.8h, v7.8h, v0.h[1]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" "fmla v15.8h, v7.8h, v1.h[1]\n" - "ldr x12, [x17, #0xa8]\n" + "ldr x21, [x17, #0xa8]\n" "fmla v19.8h, v7.8h, v2.h[1]\n" "fmla v23.8h, v7.8h, v3.h[1]\n" "fmla v27.8h, v7.8h, v4.h[1]\n" "fmla v31.8h, v7.8h, v5.h[1]\n" "ldr d7, [x17, #0x90]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "fmla v8.8h, v6.8h, v0.h[2]\n" "fmla v12.8h, v6.8h, v1.h[2]\n" - "ldr x11, [x17, #0xb8]\n" + "ldr x20, [x17, #0xb8]\n" "fmla v16.8h, v6.8h, v2.h[2]\n" "fmla v20.8h, v6.8h, v3.h[2]\n" "fmla v24.8h, v6.8h, v4.h[2]\n" "fmla v28.8h, v6.8h, v5.h[2]\n" "ldr d6, [x17, #0xa0]\n" "fmla v9.8h, v7.8h, v0.h[2]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" "fmla v13.8h, v7.8h, v1.h[2]\n" - "ldr x12, [x17, #0xc8]\n" + "ldr x21, [x17, #0xc8]\n" "fmla v17.8h, v7.8h, v2.h[2]\n" "fmla v21.8h, v7.8h, v3.h[2]\n" "fmla v25.8h, v7.8h, v4.h[2]\n" "fmla v29.8h, v7.8h, v5.h[2]\n" "ldr d7, [x17, #0xb0]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "fmla v10.8h, v6.8h, v0.h[2]\n" "fmla v14.8h, v6.8h, v1.h[2]\n" - "ldr x11, [x17, #0xd8]\n" + "ldr x20, [x17, #0xd8]\n" "fmla v18.8h, v6.8h, v2.h[2]\n" "fmla v22.8h, v6.8h, v3.h[2]\n" "fmla v26.8h, v6.8h, v4.h[2]\n" "fmla v30.8h, v6.8h, v5.h[2]\n" "ldr d6, [x17, #0xc0]\n" "fmla v11.8h, v7.8h, v0.h[2]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" "fmla v15.8h, v7.8h, v1.h[2]\n" - "ldr x12, [x17, #0xe8]\n" + "ldr x21, [x17, #0xe8]\n" "fmla v19.8h, v7.8h, v2.h[2]\n" "fmla v23.8h, v7.8h, v3.h[2]\n" "fmla v27.8h, v7.8h, v4.h[2]\n" "fmla v31.8h, v7.8h, v5.h[2]\n" "ldr d7, [x17, #0xd0]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "fmla v8.8h, v6.8h, v0.h[3]\n" "fmla v12.8h, v6.8h, v1.h[3]\n" - "ldr x11, [x17, #0xf8]\n" + "ldr x20, [x17, #0xf8]\n" "fmla v16.8h, v6.8h, v2.h[3]\n" "fmla v20.8h, v6.8h, v3.h[3]\n" "fmla v24.8h, v6.8h, v4.h[3]\n" "fmla v28.8h, v6.8h, v5.h[3]\n" "ldr d6, [x17, #0xe0]\n" "fmla v9.8h, v7.8h, v0.h[3]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" "fmla v13.8h, v7.8h, v1.h[3]\n" - "ldr x12, [x17, #0x108]\n" + "ldr x21, [x17, #0x108]\n" "fmla v17.8h, v7.8h, v2.h[3]\n" "fmla v21.8h, v7.8h, v3.h[3]\n" "fmla v25.8h, v7.8h, v4.h[3]\n" "fmla v29.8h, v7.8h, v5.h[3]\n" "ldr d7, [x17, #0xf0]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "fmla v10.8h, v6.8h, v0.h[3]\n" "fmla v14.8h, v6.8h, v1.h[3]\n" - "ldr x11, [x17, #0x118]\n" + "ldr x20, [x17, #0x118]\n" "fmla v18.8h, v6.8h, v2.h[3]\n" "fmla v22.8h, v6.8h, v3.h[3]\n" "fmla v26.8h, v6.8h, v4.h[3]\n" "fmla v30.8h, v6.8h, v5.h[3]\n" "ldr d6, [x17, #0x100]\n" "fmla v11.8h, v7.8h, v0.h[3]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" "fmla v15.8h, v7.8h, v1.h[3]\n" - "ldr x12, [x17, #0x128]\n" + "ldr x21, [x17, #0x128]\n" "fmla v19.8h, v7.8h, v2.h[3]\n" "fmla v23.8h, v7.8h, v3.h[3]\n" "fmla v27.8h, v7.8h, v4.h[3]\n" "fmla v31.8h, v7.8h, v5.h[3]\n" "ldr d7, [x17, #0x110]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "fmla v8.8h, v6.8h, v0.h[4]\n" "fmla v12.8h, v6.8h, v1.h[4]\n" - "ldr x11, [x17, #0x138]\n" + "ldr x20, [x17, #0x138]\n" "fmla v16.8h, v6.8h, v2.h[4]\n" "fmla v20.8h, v6.8h, v3.h[4]\n" "fmla v24.8h, v6.8h, v4.h[4]\n" "fmla v28.8h, v6.8h, v5.h[4]\n" "ldr d6, [x17, #0x120]\n" "fmla v9.8h, v7.8h, v0.h[4]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" "fmla v13.8h, v7.8h, v1.h[4]\n" - "ldr x12, [x17, #0x148]\n" + "ldr x21, [x17, #0x148]\n" "fmla v17.8h, v7.8h, v2.h[4]\n" "fmla v21.8h, v7.8h, v3.h[4]\n" "fmla v25.8h, v7.8h, v4.h[4]\n" "fmla v29.8h, v7.8h, v5.h[4]\n" "ldr d7, [x17, #0x130]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "fmla v10.8h, v6.8h, v0.h[4]\n" "fmla v14.8h, v6.8h, v1.h[4]\n" - "ldr x11, [x17, #0x158]\n" + "ldr x20, [x17, #0x158]\n" "fmla v18.8h, v6.8h, v2.h[4]\n" "fmla v22.8h, v6.8h, v3.h[4]\n" "fmla v26.8h, v6.8h, v4.h[4]\n" "fmla v30.8h, v6.8h, v5.h[4]\n" "ldr d6, [x17, #0x140]\n" "fmla v11.8h, v7.8h, v0.h[4]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" "fmla v15.8h, v7.8h, v1.h[4]\n" - "ldr x12, [x17, #0x168]\n" + "ldr x21, [x17, #0x168]\n" "fmla v19.8h, v7.8h, v2.h[4]\n" "fmla v23.8h, v7.8h, v3.h[4]\n" "fmla v27.8h, v7.8h, v4.h[4]\n" "fmla v31.8h, v7.8h, v5.h[4]\n" "ldr d7, [x17, #0x150]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "fmla v8.8h, v6.8h, v0.h[5]\n" "fmla v12.8h, v6.8h, v1.h[5]\n" - "ldr x11, [x17, #0x178]\n" + "ldr x20, [x17, #0x178]\n" "fmla v16.8h, v6.8h, v2.h[5]\n" "fmla v20.8h, v6.8h, v3.h[5]\n" "fmla v24.8h, v6.8h, v4.h[5]\n" "fmla v28.8h, v6.8h, v5.h[5]\n" "ldr d6, [x17, #0x160]\n" "fmla v9.8h, v7.8h, v0.h[5]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" "fmla v13.8h, v7.8h, v1.h[5]\n" - "ldr x12, [x17, #0x188]\n" + "ldr x21, [x17, #0x188]\n" "fmla v17.8h, v7.8h, v2.h[5]\n" "fmla v21.8h, v7.8h, v3.h[5]\n" "fmla v25.8h, v7.8h, v4.h[5]\n" "fmla v29.8h, v7.8h, v5.h[5]\n" "ldr d7, [x17, #0x170]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "fmla v10.8h, v6.8h, v0.h[5]\n" "fmla v14.8h, v6.8h, v1.h[5]\n" - "ldr x11, [x17, #0x198]\n" + "ldr x20, [x17, #0x198]\n" "fmla v18.8h, v6.8h, v2.h[5]\n" "fmla v22.8h, v6.8h, v3.h[5]\n" "fmla v26.8h, v6.8h, v4.h[5]\n" "fmla v30.8h, v6.8h, v5.h[5]\n" "ldr d6, [x17, #0x180]\n" "fmla v11.8h, v7.8h, v0.h[5]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" "fmla v15.8h, v7.8h, v1.h[5]\n" - "ldr x12, [x17, #0x1a8]\n" + "ldr x21, [x17, #0x1a8]\n" "fmla v19.8h, v7.8h, v2.h[5]\n" "fmla v23.8h, v7.8h, v3.h[5]\n" "fmla v27.8h, v7.8h, v4.h[5]\n" "fmla v31.8h, v7.8h, v5.h[5]\n" "ldr d7, [x17, #0x190]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "fmla v8.8h, v6.8h, v0.h[6]\n" "fmla v12.8h, v6.8h, v1.h[6]\n" - "ldr x11, [x17, #0x1b8]\n" + "ldr x20, [x17, #0x1b8]\n" "fmla v16.8h, v6.8h, v2.h[6]\n" "fmla v20.8h, v6.8h, v3.h[6]\n" "fmla v24.8h, v6.8h, v4.h[6]\n" "fmla v28.8h, v6.8h, v5.h[6]\n" "ldr d6, [x17, #0x1a0]\n" "fmla v9.8h, v7.8h, v0.h[6]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" "fmla v13.8h, v7.8h, v1.h[6]\n" - "ldr x12, [x17, #0x1c8]\n" + "ldr x21, [x17, #0x1c8]\n" "fmla v17.8h, v7.8h, v2.h[6]\n" "fmla v21.8h, v7.8h, v3.h[6]\n" "fmla v25.8h, v7.8h, v4.h[6]\n" "fmla v29.8h, v7.8h, v5.h[6]\n" "ldr d7, [x17, #0x1b0]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "fmla v10.8h, v6.8h, v0.h[6]\n" "fmla v14.8h, v6.8h, v1.h[6]\n" - "ldr x11, [x17, #0x1d8]\n" + "ldr x20, [x17, #0x1d8]\n" "fmla v18.8h, v6.8h, v2.h[6]\n" "fmla v22.8h, v6.8h, v3.h[6]\n" "fmla v26.8h, v6.8h, v4.h[6]\n" "fmla v30.8h, v6.8h, v5.h[6]\n" "ldr d6, [x17, #0x1c0]\n" "fmla v11.8h, v7.8h, v0.h[6]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" "fmla v15.8h, v7.8h, v1.h[6]\n" - "ldr x12, [x17, #0x1e8]\n" + "ldr x21, [x17, #0x1e8]\n" "fmla v19.8h, v7.8h, v2.h[6]\n" "fmla v23.8h, v7.8h, v3.h[6]\n" "fmla v27.8h, v7.8h, v4.h[6]\n" "fmla v31.8h, v7.8h, v5.h[6]\n" "ldr d7, [x17, #0x1d0]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "fmla v8.8h, v6.8h, v0.h[7]\n" "fmla v12.8h, v6.8h, v1.h[7]\n" - "ldr x11, [x17, #0x1f8]\n" + "ldr x20, [x17, #0x1f8]\n" "fmla v16.8h, v6.8h, v2.h[7]\n" "fmla v20.8h, v6.8h, v3.h[7]\n" "fmla v24.8h, v6.8h, v4.h[7]\n" "fmla v28.8h, v6.8h, v5.h[7]\n" "ldr d6, [x17, #0x1e0]\n" "fmla v9.8h, v7.8h, v0.h[7]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" "fmla v13.8h, v7.8h, v1.h[7]\n" "fmla v17.8h, v7.8h, v2.h[7]\n" "fmla v21.8h, v7.8h, v3.h[7]\n" "fmla v25.8h, v7.8h, v4.h[7]\n" "fmla v29.8h, v7.8h, v5.h[7]\n" "ldr d7, [x17, #0x1f0]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "add x17, x17, #0x200\n" "fmla v10.8h, v6.8h, v0.h[7]\n" - "ldr x12, [x17, #0x8]\n" + "ldr x21, [x17, #0x8]\n" "fmla v14.8h, v6.8h, v1.h[7]\n" - "ldr x11, [x17, #0x18]\n" + "ldr x20, [x17, #0x18]\n" "fmla v18.8h, v6.8h, v2.h[7]\n" "fmla v22.8h, v6.8h, v3.h[7]\n" "fmla v26.8h, v6.8h, v4.h[7]\n" @@ -5079,56 +5079,56 @@ void a64_hybrid_fp16_mla_6x32_a55 ( "fmla v11.8h, v7.8h, v0.h[7]\n" "ldr d0, [x13, #0x0]\n" "fmla v15.8h, v7.8h, v1.h[7]\n" - "ldr d1, [x9, #0x0]\n" + "ldr d1, [x12, #0x0]\n" "fmla v19.8h, v7.8h, v2.h[7]\n" - "ldr d2, [x27, #0x0]\n" + "ldr d2, [x11, #0x0]\n" "fmla v23.8h, v7.8h, v3.h[7]\n" - "ldr d3, [x25, #0x0]\n" + "ldr d3, [x10, #0x0]\n" "fmla v27.8h, v7.8h, v4.h[7]\n" - "ldr d4, [x23, #0x0]\n" + "ldr d4, [x9, #0x0]\n" "fmla v31.8h, v7.8h, v5.h[7]\n" - "ldr d5, [x21, #0x0]\n" + "ldr d5, [x28, #0x0]\n" "ldr d7, [x17, #0x10]\n" - "mov v6.d[1], x12\n" - "mov v0.d[1], x10\n" - "mov v1.d[1], x28\n" - "mov v2.d[1], x26\n" + "mov v6.d[1], x21\n" + "mov v0.d[1], x27\n" + "mov v1.d[1], x26\n" + "mov v2.d[1], x25\n" "mov v3.d[1], x24\n" - "mov v4.d[1], x22\n" - "mov v5.d[1], x20\n" - "mov v7.d[1], x11\n" + "mov v4.d[1], x23\n" + "mov v5.d[1], x22\n" + "mov v7.d[1], x20\n" "bge 271b\n" "272:" // Height 6: Multiply loop: Single iteration only "fmla v8.8h, v6.8h, v0.h[0]\n" "add x13, x13, #0x10\n" "fmla v12.8h, v6.8h, v1.h[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" "fmla v16.8h, v6.8h, v2.h[0]\n" - "add x27, x27, #0x10\n" + "add x11, x11, #0x10\n" "fmla v20.8h, v6.8h, v3.h[0]\n" - "add x25, x25, #0x10\n" + "add x10, x10, #0x10\n" "fmla v24.8h, v6.8h, v4.h[0]\n" - "add x23, x23, #0x10\n" + "add x9, x9, #0x10\n" "fmla v28.8h, v6.8h, v5.h[0]\n" "ldr q6, [x17, #0x20]\n" "fmla v9.8h, v7.8h, v0.h[0]\n" - "add x21, x21, #0x10\n" + "add x28, x28, #0x10\n" "fmla v13.8h, v7.8h, v1.h[0]\n" "sub x14, x14, #0x8\n" "fmla v17.8h, v7.8h, v2.h[0]\n" "prfm pldl1keep, [x13, #0x80]\n" "fmla v21.8h, v7.8h, v3.h[0]\n" - "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x12, #0x80]\n" "fmla v25.8h, v7.8h, v4.h[0]\n" - "prfm pldl1keep, [x27, #0x80]\n" + "prfm pldl1keep, [x11, #0x80]\n" "fmla v29.8h, v7.8h, v5.h[0]\n" "ldr q7, [x17, #0x30]\n" "fmla v10.8h, v6.8h, v0.h[0]\n" - "prfm pldl1keep, [x25, #0x80]\n" + "prfm pldl1keep, [x10, #0x80]\n" "fmla v14.8h, v6.8h, v1.h[0]\n" - "prfm pldl1keep, [x23, #0x80]\n" + "prfm pldl1keep, [x9, #0x80]\n" "fmla v18.8h, v6.8h, v2.h[0]\n" - "prfm pldl1keep, [x21, #0x80]\n" + "prfm pldl1keep, [x28, #0x80]\n" "fmla v22.8h, v6.8h, v3.h[0]\n" "fmla v26.8h, v6.8h, v4.h[0]\n" "fmla v30.8h, v6.8h, v5.h[0]\n" @@ -5338,42 +5338,42 @@ void a64_hybrid_fp16_mla_6x32_a55 ( "273:" // Height 6: Multiply loop: Main loop skip "cbz x14, 275f\n" "274:" // Height 6: Multiply loop: Odd block loop - "ldr h0, [x13], #0x2\n" + "ldr h7, [x13], #0x2\n" "sub x14, x14, #0x1\n" - "ldr h1, [x9], #0x2\n" - "ldr h2, [x27], #0x2\n" - "ldr h3, [x25], #0x2\n" - "ldr h4, [x23], #0x2\n" - "ldr h5, [x21], #0x2\n" - "ldr q6, [x17, #0x0]\n" - "fmla v8.8h, v6.8h, v0.h[0]\n" - "ldr q7, [x17, #0x10]\n" - "fmla v12.8h, v6.8h, v1.h[0]\n" - "fmla v16.8h, v6.8h, v2.h[0]\n" - "fmla v20.8h, v6.8h, v3.h[0]\n" - "fmla v24.8h, v6.8h, v4.h[0]\n" - "fmla v28.8h, v6.8h, v5.h[0]\n" - "ldr q6, [x17, #0x20]\n" - "fmla v9.8h, v7.8h, v0.h[0]\n" - "fmla v13.8h, v7.8h, v1.h[0]\n" - "fmla v17.8h, v7.8h, v2.h[0]\n" - "fmla v21.8h, v7.8h, v3.h[0]\n" - "fmla v25.8h, v7.8h, v4.h[0]\n" - "fmla v29.8h, v7.8h, v5.h[0]\n" - "ldr q7, [x17, #0x30]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" + "ldr h6, [x12], #0x2\n" + "ldr h5, [x11], #0x2\n" + "ldr h4, [x10], #0x2\n" + "ldr h3, [x9], #0x2\n" + "ldr h2, [x28], #0x2\n" + "ldr q1, [x17, #0x0]\n" + "fmla v8.8h, v1.8h, v7.h[0]\n" + "ldr q0, [x17, #0x10]\n" + "fmla v12.8h, v1.8h, v6.h[0]\n" + "fmla v16.8h, v1.8h, v5.h[0]\n" + "fmla v20.8h, v1.8h, v4.h[0]\n" + "fmla v24.8h, v1.8h, v3.h[0]\n" + "fmla v28.8h, v1.8h, v2.h[0]\n" + "ldr q1, [x17, #0x20]\n" + "fmla v9.8h, v0.8h, v7.h[0]\n" + "fmla v13.8h, v0.8h, v6.h[0]\n" + "fmla v17.8h, v0.8h, v5.h[0]\n" + "fmla v21.8h, v0.8h, v4.h[0]\n" + "fmla v25.8h, v0.8h, v3.h[0]\n" + "fmla v29.8h, v0.8h, v2.h[0]\n" + "ldr q0, [x17, #0x30]\n" + "fmla v10.8h, v1.8h, v7.h[0]\n" "add x17, x17, #0x40\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "fmla v22.8h, v6.8h, v3.h[0]\n" - "fmla v26.8h, v6.8h, v4.h[0]\n" - "fmla v30.8h, v6.8h, v5.h[0]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "fmla v23.8h, v7.8h, v3.h[0]\n" - "fmla v27.8h, v7.8h, v4.h[0]\n" - "fmla v31.8h, v7.8h, v5.h[0]\n" + "fmla v14.8h, v1.8h, v6.h[0]\n" + "fmla v18.8h, v1.8h, v5.h[0]\n" + "fmla v22.8h, v1.8h, v4.h[0]\n" + "fmla v26.8h, v1.8h, v3.h[0]\n" + "fmla v30.8h, v1.8h, v2.h[0]\n" + "fmla v11.8h, v0.8h, v7.h[0]\n" + "fmla v15.8h, v0.8h, v6.h[0]\n" + "fmla v19.8h, v0.8h, v5.h[0]\n" + "fmla v23.8h, v0.8h, v4.h[0]\n" + "fmla v27.8h, v0.8h, v3.h[0]\n" + "fmla v31.8h, v0.8h, v2.h[0]\n" "cbnz x14, 274b\n" "275:" // Height 6: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -5743,7 +5743,6 @@ void a64_hybrid_fp16_mla_6x32_a55 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "296:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32/generic.cpp index 335308751f211c302e38202c86c655af72ec50f3..8e5f600c838f0a469213d3a301634f9bcb553901 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp16_mla_6x32/generic.cpp @@ -244,11 +244,11 @@ void a64_hybrid_fp16_mla_6x32 ( "23:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 24f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 25f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -265,69 +265,69 @@ void a64_hybrid_fp16_mla_6x32 ( "blt 27f\n" "26:" // Height 1: Multiply loop: Main loop head "fmla v8.8h, v6.8h, v0.h[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q17, [x10, #0x20]\n" "fmla v9.8h, v7.8h, v0.h[0]\n" - "ldr q7, [x10, #0x30]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "ldr q6, [x10, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "ldr q7, [x10, #0x50]\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "ldr q7, [x10, #0x70]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "ldr q6, [x10, #0x80]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "ldr q7, [x10, #0x90]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "ldr q6, [x10, #0xa0]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "ldr q7, [x10, #0xb0]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "ldr q6, [x10, #0xc0]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "ldr q7, [x10, #0xd0]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "ldr q6, [x10, #0xe0]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "ldr q7, [x10, #0xf0]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "ldr q6, [x10, #0x100]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "ldr q7, [x10, #0x110]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "ldr q6, [x10, #0x120]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "ldr q7, [x10, #0x130]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "ldr q6, [x10, #0x140]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "ldr q7, [x10, #0x150]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "ldr q6, [x10, #0x160]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "ldr q7, [x10, #0x170]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "ldr q6, [x10, #0x180]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "ldr q7, [x10, #0x190]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "ldr q6, [x10, #0x1a0]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "ldr q7, [x10, #0x1b0]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "ldr q6, [x10, #0x1c0]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "ldr q7, [x10, #0x1d0]\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "ldr q6, [x10, #0x1e0]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "ldr q7, [x10, #0x1f0]\n" + "ldr q16, [x10, #0x30]\n" + "fmla v10.8h, v17.8h, v0.h[0]\n" + "ldr q17, [x10, #0x40]\n" + "fmla v11.8h, v16.8h, v0.h[0]\n" + "ldr q16, [x10, #0x50]\n" + "fmla v8.8h, v17.8h, v0.h[1]\n" + "ldr q17, [x10, #0x60]\n" + "fmla v9.8h, v16.8h, v0.h[1]\n" + "ldr q16, [x10, #0x70]\n" + "fmla v10.8h, v17.8h, v0.h[1]\n" + "ldr q17, [x10, #0x80]\n" + "fmla v11.8h, v16.8h, v0.h[1]\n" + "ldr q16, [x10, #0x90]\n" + "fmla v8.8h, v17.8h, v0.h[2]\n" + "ldr q17, [x10, #0xa0]\n" + "fmla v9.8h, v16.8h, v0.h[2]\n" + "ldr q16, [x10, #0xb0]\n" + "fmla v10.8h, v17.8h, v0.h[2]\n" + "ldr q17, [x10, #0xc0]\n" + "fmla v11.8h, v16.8h, v0.h[2]\n" + "ldr q16, [x10, #0xd0]\n" + "fmla v8.8h, v17.8h, v0.h[3]\n" + "ldr q17, [x10, #0xe0]\n" + "fmla v9.8h, v16.8h, v0.h[3]\n" + "ldr q16, [x10, #0xf0]\n" + "fmla v10.8h, v17.8h, v0.h[3]\n" + "ldr q17, [x10, #0x100]\n" + "fmla v11.8h, v16.8h, v0.h[3]\n" + "ldr q16, [x10, #0x110]\n" + "fmla v8.8h, v17.8h, v0.h[4]\n" + "ldr q17, [x10, #0x120]\n" + "fmla v9.8h, v16.8h, v0.h[4]\n" + "ldr q16, [x10, #0x130]\n" + "fmla v10.8h, v17.8h, v0.h[4]\n" + "ldr q17, [x10, #0x140]\n" + "fmla v11.8h, v16.8h, v0.h[4]\n" + "ldr q16, [x10, #0x150]\n" + "fmla v8.8h, v17.8h, v0.h[5]\n" + "ldr q17, [x10, #0x160]\n" + "fmla v9.8h, v16.8h, v0.h[5]\n" + "ldr q16, [x10, #0x170]\n" + "fmla v10.8h, v17.8h, v0.h[5]\n" + "ldr q17, [x10, #0x180]\n" + "fmla v11.8h, v16.8h, v0.h[5]\n" + "ldr q16, [x10, #0x190]\n" + "fmla v8.8h, v17.8h, v0.h[6]\n" + "ldr q17, [x10, #0x1a0]\n" + "fmla v9.8h, v16.8h, v0.h[6]\n" + "ldr q16, [x10, #0x1b0]\n" + "fmla v10.8h, v17.8h, v0.h[6]\n" + "ldr q17, [x10, #0x1c0]\n" + "fmla v11.8h, v16.8h, v0.h[6]\n" + "ldr q16, [x10, #0x1d0]\n" + "fmla v8.8h, v17.8h, v0.h[7]\n" + "ldr q17, [x10, #0x1e0]\n" + "fmla v9.8h, v16.8h, v0.h[7]\n" + "ldr q16, [x10, #0x1f0]\n" "sub x27, x27, #0x8\n" "add x26, x26, #0x10\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" + "fmla v10.8h, v17.8h, v0.h[7]\n" + "fmla v11.8h, v16.8h, v0.h[7]\n" "ldr q0, [x26, #0x0]\n" "cmp x27, #0x10\n" "add x10, x10, #0x200\n" @@ -337,84 +337,84 @@ void a64_hybrid_fp16_mla_6x32 ( "bge 26b\n" "27:" // Height 1: Multiply loop: Single iteration only "fmla v8.8h, v6.8h, v0.h[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q17, [x10, #0x20]\n" "fmla v9.8h, v7.8h, v0.h[0]\n" - "ldr q7, [x10, #0x30]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "ldr q6, [x10, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "ldr q7, [x10, #0x50]\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "ldr q7, [x10, #0x70]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "ldr q6, [x10, #0x80]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "ldr q7, [x10, #0x90]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "ldr q6, [x10, #0xa0]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "ldr q7, [x10, #0xb0]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "ldr q6, [x10, #0xc0]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "ldr q7, [x10, #0xd0]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "ldr q6, [x10, #0xe0]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "ldr q7, [x10, #0xf0]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "ldr q6, [x10, #0x100]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "ldr q7, [x10, #0x110]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "ldr q6, [x10, #0x120]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "ldr q7, [x10, #0x130]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "ldr q6, [x10, #0x140]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "ldr q7, [x10, #0x150]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "ldr q6, [x10, #0x160]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "ldr q7, [x10, #0x170]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "ldr q6, [x10, #0x180]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "ldr q7, [x10, #0x190]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "ldr q6, [x10, #0x1a0]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "ldr q7, [x10, #0x1b0]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "ldr q6, [x10, #0x1c0]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "ldr q7, [x10, #0x1d0]\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "ldr q6, [x10, #0x1e0]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "ldr q7, [x10, #0x1f0]\n" + "ldr q16, [x10, #0x30]\n" + "fmla v10.8h, v17.8h, v0.h[0]\n" + "ldr q17, [x10, #0x40]\n" + "fmla v11.8h, v16.8h, v0.h[0]\n" + "ldr q16, [x10, #0x50]\n" + "fmla v8.8h, v17.8h, v0.h[1]\n" + "ldr q17, [x10, #0x60]\n" + "fmla v9.8h, v16.8h, v0.h[1]\n" + "ldr q16, [x10, #0x70]\n" + "fmla v10.8h, v17.8h, v0.h[1]\n" + "ldr q17, [x10, #0x80]\n" + "fmla v11.8h, v16.8h, v0.h[1]\n" + "ldr q16, [x10, #0x90]\n" + "fmla v8.8h, v17.8h, v0.h[2]\n" + "ldr q17, [x10, #0xa0]\n" + "fmla v9.8h, v16.8h, v0.h[2]\n" + "ldr q16, [x10, #0xb0]\n" + "fmla v10.8h, v17.8h, v0.h[2]\n" + "ldr q17, [x10, #0xc0]\n" + "fmla v11.8h, v16.8h, v0.h[2]\n" + "ldr q16, [x10, #0xd0]\n" + "fmla v8.8h, v17.8h, v0.h[3]\n" + "ldr q17, [x10, #0xe0]\n" + "fmla v9.8h, v16.8h, v0.h[3]\n" + "ldr q16, [x10, #0xf0]\n" + "fmla v10.8h, v17.8h, v0.h[3]\n" + "ldr q17, [x10, #0x100]\n" + "fmla v11.8h, v16.8h, v0.h[3]\n" + "ldr q16, [x10, #0x110]\n" + "fmla v8.8h, v17.8h, v0.h[4]\n" + "ldr q17, [x10, #0x120]\n" + "fmla v9.8h, v16.8h, v0.h[4]\n" + "ldr q16, [x10, #0x130]\n" + "fmla v10.8h, v17.8h, v0.h[4]\n" + "ldr q17, [x10, #0x140]\n" + "fmla v11.8h, v16.8h, v0.h[4]\n" + "ldr q16, [x10, #0x150]\n" + "fmla v8.8h, v17.8h, v0.h[5]\n" + "ldr q17, [x10, #0x160]\n" + "fmla v9.8h, v16.8h, v0.h[5]\n" + "ldr q16, [x10, #0x170]\n" + "fmla v10.8h, v17.8h, v0.h[5]\n" + "ldr q17, [x10, #0x180]\n" + "fmla v11.8h, v16.8h, v0.h[5]\n" + "ldr q16, [x10, #0x190]\n" + "fmla v8.8h, v17.8h, v0.h[6]\n" + "ldr q17, [x10, #0x1a0]\n" + "fmla v9.8h, v16.8h, v0.h[6]\n" + "ldr q16, [x10, #0x1b0]\n" + "fmla v10.8h, v17.8h, v0.h[6]\n" + "ldr q17, [x10, #0x1c0]\n" + "fmla v11.8h, v16.8h, v0.h[6]\n" + "ldr q16, [x10, #0x1d0]\n" + "fmla v8.8h, v17.8h, v0.h[7]\n" + "ldr q17, [x10, #0x1e0]\n" + "fmla v9.8h, v16.8h, v0.h[7]\n" + "ldr q16, [x10, #0x1f0]\n" "add x26, x26, #0x10\n" "sub x27, x27, #0x8\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" + "fmla v10.8h, v17.8h, v0.h[7]\n" + "fmla v11.8h, v16.8h, v0.h[7]\n" "prfm pldl1keep, [x26, #0x80]\n" "add x10, x10, #0x200\n" "28:" // Height 1: Multiply loop: Main loop skip "cbz x27, 30f\n" "29:" // Height 1: Multiply loop: Odd block loop "ldr h0, [x26], #0x2\n" - "ldr q6, [x10, #0x0]\n" - "fmla v8.8h, v6.8h, v0.h[0]\n" + "ldr q16, [x10, #0x0]\n" + "fmla v8.8h, v16.8h, v0.h[0]\n" "sub x27, x27, #0x1\n" - "ldr q7, [x10, #0x10]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.8h, v7.8h, v0.h[0]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "ldr q7, [x10, #0x30]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" + "ldr q17, [x10, #0x10]\n" + "ldr q16, [x10, #0x20]\n" + "fmla v9.8h, v17.8h, v0.h[0]\n" + "fmla v10.8h, v16.8h, v0.h[0]\n" + "ldr q16, [x10, #0x30]\n" + "fmla v11.8h, v16.8h, v0.h[0]\n" "add x10, x10, #0x40\n" "cbnz x27, 29b\n" "30:" // Height 1: Multiply loop: No odd multiplies @@ -425,17 +425,17 @@ void a64_hybrid_fp16_mla_6x32 ( "prfm pstl1keep, [x9, #0x0]\n" "tbz %x[flags], #1, 31f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.8h }, [x20]\n" + "ld1r { v17.8h }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.8h }, [x20]\n" - "fmin v8.8h, v8.8h, v1.8h\n" - "fmin v9.8h, v9.8h, v1.8h\n" - "fmin v10.8h, v10.8h, v1.8h\n" - "fmin v11.8h, v11.8h, v1.8h\n" - "fmax v8.8h, v8.8h, v0.8h\n" - "fmax v9.8h, v9.8h, v0.8h\n" - "fmax v10.8h, v10.8h, v0.8h\n" - "fmax v11.8h, v11.8h, v0.8h\n" + "ld1r { v16.8h }, [x20]\n" + "fmin v8.8h, v8.8h, v17.8h\n" + "fmin v9.8h, v9.8h, v17.8h\n" + "fmin v10.8h, v10.8h, v17.8h\n" + "fmin v11.8h, v11.8h, v17.8h\n" + "fmax v8.8h, v8.8h, v16.8h\n" + "fmax v9.8h, v9.8h, v16.8h\n" + "fmax v10.8h, v10.8h, v16.8h\n" + "fmax v11.8h, v11.8h, v16.8h\n" "31:" // Height 1: No activation "cmp x11, #0x20\n" "bge 48f\n" @@ -733,12 +733,12 @@ void a64_hybrid_fp16_mla_6x32 ( "72:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 73f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 74f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -746,7 +746,7 @@ void a64_hybrid_fp16_mla_6x32 ( "b 74f\n" "73:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" "74:" // Height 2: input setup done "cmp x27, #0x8\n" "blt 77f\n" @@ -759,230 +759,230 @@ void a64_hybrid_fp16_mla_6x32 ( "75:" // Height 2: Multiply loop: Main loop head "fmla v8.8h, v6.8h, v0.h[0]\n" "fmla v12.8h, v6.8h, v1.h[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q17, [x10, #0x20]\n" "sub x27, x27, #0x8\n" "fmla v9.8h, v7.8h, v0.h[0]\n" "fmla v13.8h, v7.8h, v1.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q16, [x10, #0x30]\n" "add x26, x26, #0x10\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "ldr q6, [x10, #0x40]\n" + "fmla v10.8h, v17.8h, v0.h[0]\n" + "fmla v14.8h, v17.8h, v1.h[0]\n" + "ldr q17, [x10, #0x40]\n" "add x25, x25, #0x10\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "ldr q7, [x10, #0x50]\n" + "fmla v11.8h, v16.8h, v0.h[0]\n" + "fmla v15.8h, v16.8h, v1.h[0]\n" + "ldr q16, [x10, #0x50]\n" "cmp x27, #0x10\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "ldr q6, [x10, #0x60]\n" + "fmla v8.8h, v17.8h, v0.h[1]\n" + "fmla v12.8h, v17.8h, v1.h[1]\n" + "ldr q17, [x10, #0x60]\n" "prfm pldl1keep, [x26, #0x80]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "ldr q7, [x10, #0x70]\n" + "fmla v9.8h, v16.8h, v0.h[1]\n" + "fmla v13.8h, v16.8h, v1.h[1]\n" + "ldr q16, [x10, #0x70]\n" "prfm pldl1keep, [x25, #0x80]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "fmla v14.8h, v6.8h, v1.h[1]\n" - "ldr q6, [x10, #0x80]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "fmla v15.8h, v7.8h, v1.h[1]\n" - "ldr q7, [x10, #0x90]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "fmla v12.8h, v6.8h, v1.h[2]\n" - "ldr q6, [x10, #0xa0]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "fmla v13.8h, v7.8h, v1.h[2]\n" - "ldr q7, [x10, #0xb0]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "fmla v14.8h, v6.8h, v1.h[2]\n" - "ldr q6, [x10, #0xc0]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "fmla v15.8h, v7.8h, v1.h[2]\n" - "ldr q7, [x10, #0xd0]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "fmla v12.8h, v6.8h, v1.h[3]\n" - "ldr q6, [x10, #0xe0]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "fmla v13.8h, v7.8h, v1.h[3]\n" - "ldr q7, [x10, #0xf0]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "fmla v14.8h, v6.8h, v1.h[3]\n" - "ldr q6, [x10, #0x100]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "fmla v15.8h, v7.8h, v1.h[3]\n" - "ldr q7, [x10, #0x110]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "fmla v12.8h, v6.8h, v1.h[4]\n" - "ldr q6, [x10, #0x120]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "fmla v13.8h, v7.8h, v1.h[4]\n" - "ldr q7, [x10, #0x130]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "fmla v14.8h, v6.8h, v1.h[4]\n" - "ldr q6, [x10, #0x140]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "fmla v15.8h, v7.8h, v1.h[4]\n" - "ldr q7, [x10, #0x150]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "fmla v12.8h, v6.8h, v1.h[5]\n" - "ldr q6, [x10, #0x160]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "fmla v13.8h, v7.8h, v1.h[5]\n" - "ldr q7, [x10, #0x170]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "fmla v14.8h, v6.8h, v1.h[5]\n" - "ldr q6, [x10, #0x180]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "fmla v15.8h, v7.8h, v1.h[5]\n" - "ldr q7, [x10, #0x190]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "fmla v12.8h, v6.8h, v1.h[6]\n" - "ldr q6, [x10, #0x1a0]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "fmla v13.8h, v7.8h, v1.h[6]\n" - "ldr q7, [x10, #0x1b0]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "fmla v14.8h, v6.8h, v1.h[6]\n" - "ldr q6, [x10, #0x1c0]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "fmla v15.8h, v7.8h, v1.h[6]\n" - "ldr q7, [x10, #0x1d0]\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "fmla v12.8h, v6.8h, v1.h[7]\n" - "ldr q6, [x10, #0x1e0]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "fmla v13.8h, v7.8h, v1.h[7]\n" - "ldr q7, [x10, #0x1f0]\n" + "fmla v10.8h, v17.8h, v0.h[1]\n" + "fmla v14.8h, v17.8h, v1.h[1]\n" + "ldr q17, [x10, #0x80]\n" + "fmla v11.8h, v16.8h, v0.h[1]\n" + "fmla v15.8h, v16.8h, v1.h[1]\n" + "ldr q16, [x10, #0x90]\n" + "fmla v8.8h, v17.8h, v0.h[2]\n" + "fmla v12.8h, v17.8h, v1.h[2]\n" + "ldr q17, [x10, #0xa0]\n" + "fmla v9.8h, v16.8h, v0.h[2]\n" + "fmla v13.8h, v16.8h, v1.h[2]\n" + "ldr q16, [x10, #0xb0]\n" + "fmla v10.8h, v17.8h, v0.h[2]\n" + "fmla v14.8h, v17.8h, v1.h[2]\n" + "ldr q17, [x10, #0xc0]\n" + "fmla v11.8h, v16.8h, v0.h[2]\n" + "fmla v15.8h, v16.8h, v1.h[2]\n" + "ldr q16, [x10, #0xd0]\n" + "fmla v8.8h, v17.8h, v0.h[3]\n" + "fmla v12.8h, v17.8h, v1.h[3]\n" + "ldr q17, [x10, #0xe0]\n" + "fmla v9.8h, v16.8h, v0.h[3]\n" + "fmla v13.8h, v16.8h, v1.h[3]\n" + "ldr q16, [x10, #0xf0]\n" + "fmla v10.8h, v17.8h, v0.h[3]\n" + "fmla v14.8h, v17.8h, v1.h[3]\n" + "ldr q17, [x10, #0x100]\n" + "fmla v11.8h, v16.8h, v0.h[3]\n" + "fmla v15.8h, v16.8h, v1.h[3]\n" + "ldr q16, [x10, #0x110]\n" + "fmla v8.8h, v17.8h, v0.h[4]\n" + "fmla v12.8h, v17.8h, v1.h[4]\n" + "ldr q17, [x10, #0x120]\n" + "fmla v9.8h, v16.8h, v0.h[4]\n" + "fmla v13.8h, v16.8h, v1.h[4]\n" + "ldr q16, [x10, #0x130]\n" + "fmla v10.8h, v17.8h, v0.h[4]\n" + "fmla v14.8h, v17.8h, v1.h[4]\n" + "ldr q17, [x10, #0x140]\n" + "fmla v11.8h, v16.8h, v0.h[4]\n" + "fmla v15.8h, v16.8h, v1.h[4]\n" + "ldr q16, [x10, #0x150]\n" + "fmla v8.8h, v17.8h, v0.h[5]\n" + "fmla v12.8h, v17.8h, v1.h[5]\n" + "ldr q17, [x10, #0x160]\n" + "fmla v9.8h, v16.8h, v0.h[5]\n" + "fmla v13.8h, v16.8h, v1.h[5]\n" + "ldr q16, [x10, #0x170]\n" + "fmla v10.8h, v17.8h, v0.h[5]\n" + "fmla v14.8h, v17.8h, v1.h[5]\n" + "ldr q17, [x10, #0x180]\n" + "fmla v11.8h, v16.8h, v0.h[5]\n" + "fmla v15.8h, v16.8h, v1.h[5]\n" + "ldr q16, [x10, #0x190]\n" + "fmla v8.8h, v17.8h, v0.h[6]\n" + "fmla v12.8h, v17.8h, v1.h[6]\n" + "ldr q17, [x10, #0x1a0]\n" + "fmla v9.8h, v16.8h, v0.h[6]\n" + "fmla v13.8h, v16.8h, v1.h[6]\n" + "ldr q16, [x10, #0x1b0]\n" + "fmla v10.8h, v17.8h, v0.h[6]\n" + "fmla v14.8h, v17.8h, v1.h[6]\n" + "ldr q17, [x10, #0x1c0]\n" + "fmla v11.8h, v16.8h, v0.h[6]\n" + "fmla v15.8h, v16.8h, v1.h[6]\n" + "ldr q16, [x10, #0x1d0]\n" + "fmla v8.8h, v17.8h, v0.h[7]\n" + "fmla v12.8h, v17.8h, v1.h[7]\n" + "ldr q17, [x10, #0x1e0]\n" + "fmla v9.8h, v16.8h, v0.h[7]\n" + "fmla v13.8h, v16.8h, v1.h[7]\n" + "ldr q16, [x10, #0x1f0]\n" "add x10, x10, #0x200\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "fmla v14.8h, v6.8h, v1.h[7]\n" + "fmla v10.8h, v17.8h, v0.h[7]\n" + "fmla v14.8h, v17.8h, v1.h[7]\n" "ldr q6, [x10, #0x0]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" + "fmla v11.8h, v16.8h, v0.h[7]\n" "ldr q0, [x26, #0x0]\n" - "fmla v15.8h, v7.8h, v1.h[7]\n" + "fmla v15.8h, v16.8h, v1.h[7]\n" "ldr q1, [x25, #0x0]\n" "ldr q7, [x10, #0x10]\n" "bge 75b\n" "76:" // Height 2: Multiply loop: Single iteration only "fmla v8.8h, v6.8h, v0.h[0]\n" "fmla v12.8h, v6.8h, v1.h[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q17, [x10, #0x20]\n" "add x26, x26, #0x10\n" "fmla v9.8h, v7.8h, v0.h[0]\n" "fmla v13.8h, v7.8h, v1.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q16, [x10, #0x30]\n" "add x25, x25, #0x10\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "ldr q6, [x10, #0x40]\n" + "fmla v10.8h, v17.8h, v0.h[0]\n" + "fmla v14.8h, v17.8h, v1.h[0]\n" + "ldr q17, [x10, #0x40]\n" "sub x27, x27, #0x8\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "ldr q7, [x10, #0x50]\n" + "fmla v11.8h, v16.8h, v0.h[0]\n" + "fmla v15.8h, v16.8h, v1.h[0]\n" + "ldr q16, [x10, #0x50]\n" "prfm pldl1keep, [x26, #0x80]\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "ldr q6, [x10, #0x60]\n" + "fmla v8.8h, v17.8h, v0.h[1]\n" + "fmla v12.8h, v17.8h, v1.h[1]\n" + "ldr q17, [x10, #0x60]\n" "prfm pldl1keep, [x25, #0x80]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "ldr q7, [x10, #0x70]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "fmla v14.8h, v6.8h, v1.h[1]\n" - "ldr q6, [x10, #0x80]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "fmla v15.8h, v7.8h, v1.h[1]\n" - "ldr q7, [x10, #0x90]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "fmla v12.8h, v6.8h, v1.h[2]\n" - "ldr q6, [x10, #0xa0]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "fmla v13.8h, v7.8h, v1.h[2]\n" - "ldr q7, [x10, #0xb0]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "fmla v14.8h, v6.8h, v1.h[2]\n" - "ldr q6, [x10, #0xc0]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "fmla v15.8h, v7.8h, v1.h[2]\n" - "ldr q7, [x10, #0xd0]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "fmla v12.8h, v6.8h, v1.h[3]\n" - "ldr q6, [x10, #0xe0]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "fmla v13.8h, v7.8h, v1.h[3]\n" - "ldr q7, [x10, #0xf0]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "fmla v14.8h, v6.8h, v1.h[3]\n" - "ldr q6, [x10, #0x100]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "fmla v15.8h, v7.8h, v1.h[3]\n" - "ldr q7, [x10, #0x110]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "fmla v12.8h, v6.8h, v1.h[4]\n" - "ldr q6, [x10, #0x120]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "fmla v13.8h, v7.8h, v1.h[4]\n" - "ldr q7, [x10, #0x130]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "fmla v14.8h, v6.8h, v1.h[4]\n" - "ldr q6, [x10, #0x140]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "fmla v15.8h, v7.8h, v1.h[4]\n" - "ldr q7, [x10, #0x150]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "fmla v12.8h, v6.8h, v1.h[5]\n" - "ldr q6, [x10, #0x160]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "fmla v13.8h, v7.8h, v1.h[5]\n" - "ldr q7, [x10, #0x170]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "fmla v14.8h, v6.8h, v1.h[5]\n" - "ldr q6, [x10, #0x180]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "fmla v15.8h, v7.8h, v1.h[5]\n" - "ldr q7, [x10, #0x190]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "fmla v12.8h, v6.8h, v1.h[6]\n" - "ldr q6, [x10, #0x1a0]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "fmla v13.8h, v7.8h, v1.h[6]\n" - "ldr q7, [x10, #0x1b0]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "fmla v14.8h, v6.8h, v1.h[6]\n" - "ldr q6, [x10, #0x1c0]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "fmla v15.8h, v7.8h, v1.h[6]\n" - "ldr q7, [x10, #0x1d0]\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "fmla v12.8h, v6.8h, v1.h[7]\n" - "ldr q6, [x10, #0x1e0]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "fmla v13.8h, v7.8h, v1.h[7]\n" - "ldr q7, [x10, #0x1f0]\n" + "fmla v9.8h, v16.8h, v0.h[1]\n" + "fmla v13.8h, v16.8h, v1.h[1]\n" + "ldr q16, [x10, #0x70]\n" + "fmla v10.8h, v17.8h, v0.h[1]\n" + "fmla v14.8h, v17.8h, v1.h[1]\n" + "ldr q17, [x10, #0x80]\n" + "fmla v11.8h, v16.8h, v0.h[1]\n" + "fmla v15.8h, v16.8h, v1.h[1]\n" + "ldr q16, [x10, #0x90]\n" + "fmla v8.8h, v17.8h, v0.h[2]\n" + "fmla v12.8h, v17.8h, v1.h[2]\n" + "ldr q17, [x10, #0xa0]\n" + "fmla v9.8h, v16.8h, v0.h[2]\n" + "fmla v13.8h, v16.8h, v1.h[2]\n" + "ldr q16, [x10, #0xb0]\n" + "fmla v10.8h, v17.8h, v0.h[2]\n" + "fmla v14.8h, v17.8h, v1.h[2]\n" + "ldr q17, [x10, #0xc0]\n" + "fmla v11.8h, v16.8h, v0.h[2]\n" + "fmla v15.8h, v16.8h, v1.h[2]\n" + "ldr q16, [x10, #0xd0]\n" + "fmla v8.8h, v17.8h, v0.h[3]\n" + "fmla v12.8h, v17.8h, v1.h[3]\n" + "ldr q17, [x10, #0xe0]\n" + "fmla v9.8h, v16.8h, v0.h[3]\n" + "fmla v13.8h, v16.8h, v1.h[3]\n" + "ldr q16, [x10, #0xf0]\n" + "fmla v10.8h, v17.8h, v0.h[3]\n" + "fmla v14.8h, v17.8h, v1.h[3]\n" + "ldr q17, [x10, #0x100]\n" + "fmla v11.8h, v16.8h, v0.h[3]\n" + "fmla v15.8h, v16.8h, v1.h[3]\n" + "ldr q16, [x10, #0x110]\n" + "fmla v8.8h, v17.8h, v0.h[4]\n" + "fmla v12.8h, v17.8h, v1.h[4]\n" + "ldr q17, [x10, #0x120]\n" + "fmla v9.8h, v16.8h, v0.h[4]\n" + "fmla v13.8h, v16.8h, v1.h[4]\n" + "ldr q16, [x10, #0x130]\n" + "fmla v10.8h, v17.8h, v0.h[4]\n" + "fmla v14.8h, v17.8h, v1.h[4]\n" + "ldr q17, [x10, #0x140]\n" + "fmla v11.8h, v16.8h, v0.h[4]\n" + "fmla v15.8h, v16.8h, v1.h[4]\n" + "ldr q16, [x10, #0x150]\n" + "fmla v8.8h, v17.8h, v0.h[5]\n" + "fmla v12.8h, v17.8h, v1.h[5]\n" + "ldr q17, [x10, #0x160]\n" + "fmla v9.8h, v16.8h, v0.h[5]\n" + "fmla v13.8h, v16.8h, v1.h[5]\n" + "ldr q16, [x10, #0x170]\n" + "fmla v10.8h, v17.8h, v0.h[5]\n" + "fmla v14.8h, v17.8h, v1.h[5]\n" + "ldr q17, [x10, #0x180]\n" + "fmla v11.8h, v16.8h, v0.h[5]\n" + "fmla v15.8h, v16.8h, v1.h[5]\n" + "ldr q16, [x10, #0x190]\n" + "fmla v8.8h, v17.8h, v0.h[6]\n" + "fmla v12.8h, v17.8h, v1.h[6]\n" + "ldr q17, [x10, #0x1a0]\n" + "fmla v9.8h, v16.8h, v0.h[6]\n" + "fmla v13.8h, v16.8h, v1.h[6]\n" + "ldr q16, [x10, #0x1b0]\n" + "fmla v10.8h, v17.8h, v0.h[6]\n" + "fmla v14.8h, v17.8h, v1.h[6]\n" + "ldr q17, [x10, #0x1c0]\n" + "fmla v11.8h, v16.8h, v0.h[6]\n" + "fmla v15.8h, v16.8h, v1.h[6]\n" + "ldr q16, [x10, #0x1d0]\n" + "fmla v8.8h, v17.8h, v0.h[7]\n" + "fmla v12.8h, v17.8h, v1.h[7]\n" + "ldr q17, [x10, #0x1e0]\n" + "fmla v9.8h, v16.8h, v0.h[7]\n" + "fmla v13.8h, v16.8h, v1.h[7]\n" + "ldr q16, [x10, #0x1f0]\n" "add x10, x10, #0x200\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "fmla v14.8h, v6.8h, v1.h[7]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" - "fmla v15.8h, v7.8h, v1.h[7]\n" + "fmla v10.8h, v17.8h, v0.h[7]\n" + "fmla v14.8h, v17.8h, v1.h[7]\n" + "fmla v11.8h, v16.8h, v0.h[7]\n" + "fmla v15.8h, v16.8h, v1.h[7]\n" "77:" // Height 2: Multiply loop: Main loop skip "cbz x27, 79f\n" "78:" // Height 2: Multiply loop: Odd block loop - "ldr h0, [x26], #0x2\n" - "ldr h1, [x25], #0x2\n" + "ldr h1, [x26], #0x2\n" + "ldr h0, [x25], #0x2\n" "sub x27, x27, #0x1\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - "fmla v8.8h, v6.8h, v0.h[0]\n" - "fmla v12.8h, v6.8h, v1.h[0]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.8h, v7.8h, v0.h[0]\n" - "fmla v13.8h, v7.8h, v1.h[0]\n" - "ldr q7, [x10, #0x30]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" + "ldr q17, [x10, #0x0]\n" + "ldr q16, [x10, #0x10]\n" + "fmla v8.8h, v17.8h, v1.h[0]\n" + "fmla v12.8h, v17.8h, v0.h[0]\n" + "ldr q17, [x10, #0x20]\n" + "fmla v9.8h, v16.8h, v1.h[0]\n" + "fmla v13.8h, v16.8h, v0.h[0]\n" + "ldr q16, [x10, #0x30]\n" + "fmla v10.8h, v17.8h, v1.h[0]\n" + "fmla v14.8h, v17.8h, v0.h[0]\n" "add x10, x10, #0x40\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" + "fmla v11.8h, v16.8h, v1.h[0]\n" + "fmla v15.8h, v16.8h, v0.h[0]\n" "cbnz x27, 78b\n" "79:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -995,25 +995,25 @@ void a64_hybrid_fp16_mla_6x32 ( "prfm pstl1keep, [x25, #0x0]\n" "tbz %x[flags], #1, 80f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.8h }, [x20]\n" + "ld1r { v17.8h }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.8h }, [x20]\n" - "fmin v8.8h, v8.8h, v1.8h\n" - "fmin v9.8h, v9.8h, v1.8h\n" - "fmin v10.8h, v10.8h, v1.8h\n" - "fmin v11.8h, v11.8h, v1.8h\n" - "fmin v12.8h, v12.8h, v1.8h\n" - "fmin v13.8h, v13.8h, v1.8h\n" - "fmin v14.8h, v14.8h, v1.8h\n" - "fmin v15.8h, v15.8h, v1.8h\n" - "fmax v8.8h, v8.8h, v0.8h\n" - "fmax v9.8h, v9.8h, v0.8h\n" - "fmax v10.8h, v10.8h, v0.8h\n" - "fmax v11.8h, v11.8h, v0.8h\n" - "fmax v12.8h, v12.8h, v0.8h\n" - "fmax v13.8h, v13.8h, v0.8h\n" - "fmax v14.8h, v14.8h, v0.8h\n" - "fmax v15.8h, v15.8h, v0.8h\n" + "ld1r { v16.8h }, [x20]\n" + "fmin v8.8h, v8.8h, v17.8h\n" + "fmin v9.8h, v9.8h, v17.8h\n" + "fmin v10.8h, v10.8h, v17.8h\n" + "fmin v11.8h, v11.8h, v17.8h\n" + "fmin v12.8h, v12.8h, v17.8h\n" + "fmin v13.8h, v13.8h, v17.8h\n" + "fmin v14.8h, v14.8h, v17.8h\n" + "fmin v15.8h, v15.8h, v17.8h\n" + "fmax v8.8h, v8.8h, v16.8h\n" + "fmax v9.8h, v9.8h, v16.8h\n" + "fmax v10.8h, v10.8h, v16.8h\n" + "fmax v11.8h, v11.8h, v16.8h\n" + "fmax v12.8h, v12.8h, v16.8h\n" + "fmax v13.8h, v13.8h, v16.8h\n" + "fmax v14.8h, v14.8h, v16.8h\n" + "fmax v15.8h, v15.8h, v16.8h\n" "80:" // Height 2: No activation "cmp x11, #0x20\n" "bge 97f\n" @@ -1392,13 +1392,13 @@ void a64_hybrid_fp16_mla_6x32 ( "121:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 122f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 123f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -1407,8 +1407,8 @@ void a64_hybrid_fp16_mla_6x32 ( "b 123f\n" "122:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" "123:" // Height 3: input setup done "cmp x27, #0x8\n" "blt 126f\n" @@ -1425,139 +1425,139 @@ void a64_hybrid_fp16_mla_6x32 ( "sub x27, x27, #0x8\n" "add x26, x26, #0x10\n" "fmla v16.8h, v6.8h, v2.h[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q21, [x10, #0x20]\n" "fmla v9.8h, v7.8h, v0.h[0]\n" "add x25, x25, #0x10\n" "fmla v13.8h, v7.8h, v1.h[0]\n" "fmla v17.8h, v7.8h, v2.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q20, [x10, #0x30]\n" "add x24, x24, #0x10\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" + "fmla v10.8h, v21.8h, v0.h[0]\n" + "fmla v14.8h, v21.8h, v1.h[0]\n" "cmp x27, #0x10\n" "prfm pldl1keep, [x26, #0x80]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "ldr q6, [x10, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" + "fmla v18.8h, v21.8h, v2.h[0]\n" + "ldr q21, [x10, #0x40]\n" + "fmla v11.8h, v20.8h, v0.h[0]\n" "prfm pldl1keep, [x25, #0x80]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "ldr q7, [x10, #0x50]\n" + "fmla v15.8h, v20.8h, v1.h[0]\n" + "fmla v19.8h, v20.8h, v2.h[0]\n" + "ldr q20, [x10, #0x50]\n" "prfm pldl1keep, [x24, #0x80]\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "fmla v16.8h, v6.8h, v2.h[1]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "fmla v17.8h, v7.8h, v2.h[1]\n" - "ldr q7, [x10, #0x70]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "fmla v14.8h, v6.8h, v1.h[1]\n" - "fmla v18.8h, v6.8h, v2.h[1]\n" - "ldr q6, [x10, #0x80]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "fmla v15.8h, v7.8h, v1.h[1]\n" - "fmla v19.8h, v7.8h, v2.h[1]\n" - "ldr q7, [x10, #0x90]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "fmla v12.8h, v6.8h, v1.h[2]\n" - "fmla v16.8h, v6.8h, v2.h[2]\n" - "ldr q6, [x10, #0xa0]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "fmla v13.8h, v7.8h, v1.h[2]\n" - "fmla v17.8h, v7.8h, v2.h[2]\n" - "ldr q7, [x10, #0xb0]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "fmla v14.8h, v6.8h, v1.h[2]\n" - "fmla v18.8h, v6.8h, v2.h[2]\n" - "ldr q6, [x10, #0xc0]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "fmla v15.8h, v7.8h, v1.h[2]\n" - "fmla v19.8h, v7.8h, v2.h[2]\n" - "ldr q7, [x10, #0xd0]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "fmla v12.8h, v6.8h, v1.h[3]\n" - "fmla v16.8h, v6.8h, v2.h[3]\n" - "ldr q6, [x10, #0xe0]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "fmla v13.8h, v7.8h, v1.h[3]\n" - "fmla v17.8h, v7.8h, v2.h[3]\n" - "ldr q7, [x10, #0xf0]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "fmla v14.8h, v6.8h, v1.h[3]\n" - "fmla v18.8h, v6.8h, v2.h[3]\n" - "ldr q6, [x10, #0x100]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "fmla v15.8h, v7.8h, v1.h[3]\n" - "fmla v19.8h, v7.8h, v2.h[3]\n" - "ldr q7, [x10, #0x110]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "fmla v12.8h, v6.8h, v1.h[4]\n" - "fmla v16.8h, v6.8h, v2.h[4]\n" - "ldr q6, [x10, #0x120]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "fmla v13.8h, v7.8h, v1.h[4]\n" - "fmla v17.8h, v7.8h, v2.h[4]\n" - "ldr q7, [x10, #0x130]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "fmla v14.8h, v6.8h, v1.h[4]\n" - "fmla v18.8h, v6.8h, v2.h[4]\n" - "ldr q6, [x10, #0x140]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "fmla v15.8h, v7.8h, v1.h[4]\n" - "fmla v19.8h, v7.8h, v2.h[4]\n" - "ldr q7, [x10, #0x150]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "fmla v12.8h, v6.8h, v1.h[5]\n" - "fmla v16.8h, v6.8h, v2.h[5]\n" - "ldr q6, [x10, #0x160]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "fmla v13.8h, v7.8h, v1.h[5]\n" - "fmla v17.8h, v7.8h, v2.h[5]\n" - "ldr q7, [x10, #0x170]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "fmla v14.8h, v6.8h, v1.h[5]\n" - "fmla v18.8h, v6.8h, v2.h[5]\n" - "ldr q6, [x10, #0x180]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "fmla v15.8h, v7.8h, v1.h[5]\n" - "fmla v19.8h, v7.8h, v2.h[5]\n" - "ldr q7, [x10, #0x190]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "fmla v12.8h, v6.8h, v1.h[6]\n" - "fmla v16.8h, v6.8h, v2.h[6]\n" - "ldr q6, [x10, #0x1a0]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "fmla v13.8h, v7.8h, v1.h[6]\n" - "fmla v17.8h, v7.8h, v2.h[6]\n" - "ldr q7, [x10, #0x1b0]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "fmla v14.8h, v6.8h, v1.h[6]\n" - "fmla v18.8h, v6.8h, v2.h[6]\n" - "ldr q6, [x10, #0x1c0]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "fmla v15.8h, v7.8h, v1.h[6]\n" - "fmla v19.8h, v7.8h, v2.h[6]\n" - "ldr q7, [x10, #0x1d0]\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "fmla v12.8h, v6.8h, v1.h[7]\n" - "fmla v16.8h, v6.8h, v2.h[7]\n" - "ldr q6, [x10, #0x1e0]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "fmla v13.8h, v7.8h, v1.h[7]\n" - "fmla v17.8h, v7.8h, v2.h[7]\n" - "ldr q7, [x10, #0x1f0]\n" + "fmla v8.8h, v21.8h, v0.h[1]\n" + "fmla v12.8h, v21.8h, v1.h[1]\n" + "fmla v16.8h, v21.8h, v2.h[1]\n" + "ldr q21, [x10, #0x60]\n" + "fmla v9.8h, v20.8h, v0.h[1]\n" + "fmla v13.8h, v20.8h, v1.h[1]\n" + "fmla v17.8h, v20.8h, v2.h[1]\n" + "ldr q20, [x10, #0x70]\n" + "fmla v10.8h, v21.8h, v0.h[1]\n" + "fmla v14.8h, v21.8h, v1.h[1]\n" + "fmla v18.8h, v21.8h, v2.h[1]\n" + "ldr q21, [x10, #0x80]\n" + "fmla v11.8h, v20.8h, v0.h[1]\n" + "fmla v15.8h, v20.8h, v1.h[1]\n" + "fmla v19.8h, v20.8h, v2.h[1]\n" + "ldr q20, [x10, #0x90]\n" + "fmla v8.8h, v21.8h, v0.h[2]\n" + "fmla v12.8h, v21.8h, v1.h[2]\n" + "fmla v16.8h, v21.8h, v2.h[2]\n" + "ldr q21, [x10, #0xa0]\n" + "fmla v9.8h, v20.8h, v0.h[2]\n" + "fmla v13.8h, v20.8h, v1.h[2]\n" + "fmla v17.8h, v20.8h, v2.h[2]\n" + "ldr q20, [x10, #0xb0]\n" + "fmla v10.8h, v21.8h, v0.h[2]\n" + "fmla v14.8h, v21.8h, v1.h[2]\n" + "fmla v18.8h, v21.8h, v2.h[2]\n" + "ldr q21, [x10, #0xc0]\n" + "fmla v11.8h, v20.8h, v0.h[2]\n" + "fmla v15.8h, v20.8h, v1.h[2]\n" + "fmla v19.8h, v20.8h, v2.h[2]\n" + "ldr q20, [x10, #0xd0]\n" + "fmla v8.8h, v21.8h, v0.h[3]\n" + "fmla v12.8h, v21.8h, v1.h[3]\n" + "fmla v16.8h, v21.8h, v2.h[3]\n" + "ldr q21, [x10, #0xe0]\n" + "fmla v9.8h, v20.8h, v0.h[3]\n" + "fmla v13.8h, v20.8h, v1.h[3]\n" + "fmla v17.8h, v20.8h, v2.h[3]\n" + "ldr q20, [x10, #0xf0]\n" + "fmla v10.8h, v21.8h, v0.h[3]\n" + "fmla v14.8h, v21.8h, v1.h[3]\n" + "fmla v18.8h, v21.8h, v2.h[3]\n" + "ldr q21, [x10, #0x100]\n" + "fmla v11.8h, v20.8h, v0.h[3]\n" + "fmla v15.8h, v20.8h, v1.h[3]\n" + "fmla v19.8h, v20.8h, v2.h[3]\n" + "ldr q20, [x10, #0x110]\n" + "fmla v8.8h, v21.8h, v0.h[4]\n" + "fmla v12.8h, v21.8h, v1.h[4]\n" + "fmla v16.8h, v21.8h, v2.h[4]\n" + "ldr q21, [x10, #0x120]\n" + "fmla v9.8h, v20.8h, v0.h[4]\n" + "fmla v13.8h, v20.8h, v1.h[4]\n" + "fmla v17.8h, v20.8h, v2.h[4]\n" + "ldr q20, [x10, #0x130]\n" + "fmla v10.8h, v21.8h, v0.h[4]\n" + "fmla v14.8h, v21.8h, v1.h[4]\n" + "fmla v18.8h, v21.8h, v2.h[4]\n" + "ldr q21, [x10, #0x140]\n" + "fmla v11.8h, v20.8h, v0.h[4]\n" + "fmla v15.8h, v20.8h, v1.h[4]\n" + "fmla v19.8h, v20.8h, v2.h[4]\n" + "ldr q20, [x10, #0x150]\n" + "fmla v8.8h, v21.8h, v0.h[5]\n" + "fmla v12.8h, v21.8h, v1.h[5]\n" + "fmla v16.8h, v21.8h, v2.h[5]\n" + "ldr q21, [x10, #0x160]\n" + "fmla v9.8h, v20.8h, v0.h[5]\n" + "fmla v13.8h, v20.8h, v1.h[5]\n" + "fmla v17.8h, v20.8h, v2.h[5]\n" + "ldr q20, [x10, #0x170]\n" + "fmla v10.8h, v21.8h, v0.h[5]\n" + "fmla v14.8h, v21.8h, v1.h[5]\n" + "fmla v18.8h, v21.8h, v2.h[5]\n" + "ldr q21, [x10, #0x180]\n" + "fmla v11.8h, v20.8h, v0.h[5]\n" + "fmla v15.8h, v20.8h, v1.h[5]\n" + "fmla v19.8h, v20.8h, v2.h[5]\n" + "ldr q20, [x10, #0x190]\n" + "fmla v8.8h, v21.8h, v0.h[6]\n" + "fmla v12.8h, v21.8h, v1.h[6]\n" + "fmla v16.8h, v21.8h, v2.h[6]\n" + "ldr q21, [x10, #0x1a0]\n" + "fmla v9.8h, v20.8h, v0.h[6]\n" + "fmla v13.8h, v20.8h, v1.h[6]\n" + "fmla v17.8h, v20.8h, v2.h[6]\n" + "ldr q20, [x10, #0x1b0]\n" + "fmla v10.8h, v21.8h, v0.h[6]\n" + "fmla v14.8h, v21.8h, v1.h[6]\n" + "fmla v18.8h, v21.8h, v2.h[6]\n" + "ldr q21, [x10, #0x1c0]\n" + "fmla v11.8h, v20.8h, v0.h[6]\n" + "fmla v15.8h, v20.8h, v1.h[6]\n" + "fmla v19.8h, v20.8h, v2.h[6]\n" + "ldr q20, [x10, #0x1d0]\n" + "fmla v8.8h, v21.8h, v0.h[7]\n" + "fmla v12.8h, v21.8h, v1.h[7]\n" + "fmla v16.8h, v21.8h, v2.h[7]\n" + "ldr q21, [x10, #0x1e0]\n" + "fmla v9.8h, v20.8h, v0.h[7]\n" + "fmla v13.8h, v20.8h, v1.h[7]\n" + "fmla v17.8h, v20.8h, v2.h[7]\n" + "ldr q20, [x10, #0x1f0]\n" "add x10, x10, #0x200\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "fmla v14.8h, v6.8h, v1.h[7]\n" - "fmla v18.8h, v6.8h, v2.h[7]\n" + "fmla v10.8h, v21.8h, v0.h[7]\n" + "fmla v14.8h, v21.8h, v1.h[7]\n" + "fmla v18.8h, v21.8h, v2.h[7]\n" "ldr q6, [x10, #0x0]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" + "fmla v11.8h, v20.8h, v0.h[7]\n" "ldr q0, [x26, #0x0]\n" - "fmla v15.8h, v7.8h, v1.h[7]\n" + "fmla v15.8h, v20.8h, v1.h[7]\n" "ldr q1, [x25, #0x0]\n" - "fmla v19.8h, v7.8h, v2.h[7]\n" + "fmla v19.8h, v20.8h, v2.h[7]\n" "ldr q2, [x24, #0x0]\n" "ldr q7, [x10, #0x10]\n" "bge 124b\n" @@ -1567,159 +1567,159 @@ void a64_hybrid_fp16_mla_6x32 ( "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" "fmla v16.8h, v6.8h, v2.h[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q21, [x10, #0x20]\n" "fmla v9.8h, v7.8h, v0.h[0]\n" "add x24, x24, #0x10\n" "fmla v13.8h, v7.8h, v1.h[0]\n" "fmla v17.8h, v7.8h, v2.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q20, [x10, #0x30]\n" "sub x27, x27, #0x8\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" + "fmla v10.8h, v21.8h, v0.h[0]\n" + "fmla v14.8h, v21.8h, v1.h[0]\n" "prfm pldl1keep, [x26, #0x80]\n" "prfm pldl1keep, [x25, #0x80]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "ldr q6, [x10, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" + "fmla v18.8h, v21.8h, v2.h[0]\n" + "ldr q21, [x10, #0x40]\n" + "fmla v11.8h, v20.8h, v0.h[0]\n" "prfm pldl1keep, [x24, #0x80]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "ldr q7, [x10, #0x50]\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "fmla v16.8h, v6.8h, v2.h[1]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "fmla v17.8h, v7.8h, v2.h[1]\n" - "ldr q7, [x10, #0x70]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "fmla v14.8h, v6.8h, v1.h[1]\n" - "fmla v18.8h, v6.8h, v2.h[1]\n" - "ldr q6, [x10, #0x80]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "fmla v15.8h, v7.8h, v1.h[1]\n" - "fmla v19.8h, v7.8h, v2.h[1]\n" - "ldr q7, [x10, #0x90]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "fmla v12.8h, v6.8h, v1.h[2]\n" - "fmla v16.8h, v6.8h, v2.h[2]\n" - "ldr q6, [x10, #0xa0]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "fmla v13.8h, v7.8h, v1.h[2]\n" - "fmla v17.8h, v7.8h, v2.h[2]\n" - "ldr q7, [x10, #0xb0]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "fmla v14.8h, v6.8h, v1.h[2]\n" - "fmla v18.8h, v6.8h, v2.h[2]\n" - "ldr q6, [x10, #0xc0]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "fmla v15.8h, v7.8h, v1.h[2]\n" - "fmla v19.8h, v7.8h, v2.h[2]\n" - "ldr q7, [x10, #0xd0]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "fmla v12.8h, v6.8h, v1.h[3]\n" - "fmla v16.8h, v6.8h, v2.h[3]\n" - "ldr q6, [x10, #0xe0]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "fmla v13.8h, v7.8h, v1.h[3]\n" - "fmla v17.8h, v7.8h, v2.h[3]\n" - "ldr q7, [x10, #0xf0]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "fmla v14.8h, v6.8h, v1.h[3]\n" - "fmla v18.8h, v6.8h, v2.h[3]\n" - "ldr q6, [x10, #0x100]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "fmla v15.8h, v7.8h, v1.h[3]\n" - "fmla v19.8h, v7.8h, v2.h[3]\n" - "ldr q7, [x10, #0x110]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "fmla v12.8h, v6.8h, v1.h[4]\n" - "fmla v16.8h, v6.8h, v2.h[4]\n" - "ldr q6, [x10, #0x120]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "fmla v13.8h, v7.8h, v1.h[4]\n" - "fmla v17.8h, v7.8h, v2.h[4]\n" - "ldr q7, [x10, #0x130]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "fmla v14.8h, v6.8h, v1.h[4]\n" - "fmla v18.8h, v6.8h, v2.h[4]\n" - "ldr q6, [x10, #0x140]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "fmla v15.8h, v7.8h, v1.h[4]\n" - "fmla v19.8h, v7.8h, v2.h[4]\n" - "ldr q7, [x10, #0x150]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "fmla v12.8h, v6.8h, v1.h[5]\n" - "fmla v16.8h, v6.8h, v2.h[5]\n" - "ldr q6, [x10, #0x160]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "fmla v13.8h, v7.8h, v1.h[5]\n" - "fmla v17.8h, v7.8h, v2.h[5]\n" - "ldr q7, [x10, #0x170]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "fmla v14.8h, v6.8h, v1.h[5]\n" - "fmla v18.8h, v6.8h, v2.h[5]\n" - "ldr q6, [x10, #0x180]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "fmla v15.8h, v7.8h, v1.h[5]\n" - "fmla v19.8h, v7.8h, v2.h[5]\n" - "ldr q7, [x10, #0x190]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "fmla v12.8h, v6.8h, v1.h[6]\n" - "fmla v16.8h, v6.8h, v2.h[6]\n" - "ldr q6, [x10, #0x1a0]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "fmla v13.8h, v7.8h, v1.h[6]\n" - "fmla v17.8h, v7.8h, v2.h[6]\n" - "ldr q7, [x10, #0x1b0]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "fmla v14.8h, v6.8h, v1.h[6]\n" - "fmla v18.8h, v6.8h, v2.h[6]\n" - "ldr q6, [x10, #0x1c0]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "fmla v15.8h, v7.8h, v1.h[6]\n" - "fmla v19.8h, v7.8h, v2.h[6]\n" - "ldr q7, [x10, #0x1d0]\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "fmla v12.8h, v6.8h, v1.h[7]\n" - "fmla v16.8h, v6.8h, v2.h[7]\n" - "ldr q6, [x10, #0x1e0]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "fmla v13.8h, v7.8h, v1.h[7]\n" - "fmla v17.8h, v7.8h, v2.h[7]\n" - "ldr q7, [x10, #0x1f0]\n" + "fmla v15.8h, v20.8h, v1.h[0]\n" + "fmla v19.8h, v20.8h, v2.h[0]\n" + "ldr q20, [x10, #0x50]\n" + "fmla v8.8h, v21.8h, v0.h[1]\n" + "fmla v12.8h, v21.8h, v1.h[1]\n" + "fmla v16.8h, v21.8h, v2.h[1]\n" + "ldr q21, [x10, #0x60]\n" + "fmla v9.8h, v20.8h, v0.h[1]\n" + "fmla v13.8h, v20.8h, v1.h[1]\n" + "fmla v17.8h, v20.8h, v2.h[1]\n" + "ldr q20, [x10, #0x70]\n" + "fmla v10.8h, v21.8h, v0.h[1]\n" + "fmla v14.8h, v21.8h, v1.h[1]\n" + "fmla v18.8h, v21.8h, v2.h[1]\n" + "ldr q21, [x10, #0x80]\n" + "fmla v11.8h, v20.8h, v0.h[1]\n" + "fmla v15.8h, v20.8h, v1.h[1]\n" + "fmla v19.8h, v20.8h, v2.h[1]\n" + "ldr q20, [x10, #0x90]\n" + "fmla v8.8h, v21.8h, v0.h[2]\n" + "fmla v12.8h, v21.8h, v1.h[2]\n" + "fmla v16.8h, v21.8h, v2.h[2]\n" + "ldr q21, [x10, #0xa0]\n" + "fmla v9.8h, v20.8h, v0.h[2]\n" + "fmla v13.8h, v20.8h, v1.h[2]\n" + "fmla v17.8h, v20.8h, v2.h[2]\n" + "ldr q20, [x10, #0xb0]\n" + "fmla v10.8h, v21.8h, v0.h[2]\n" + "fmla v14.8h, v21.8h, v1.h[2]\n" + "fmla v18.8h, v21.8h, v2.h[2]\n" + "ldr q21, [x10, #0xc0]\n" + "fmla v11.8h, v20.8h, v0.h[2]\n" + "fmla v15.8h, v20.8h, v1.h[2]\n" + "fmla v19.8h, v20.8h, v2.h[2]\n" + "ldr q20, [x10, #0xd0]\n" + "fmla v8.8h, v21.8h, v0.h[3]\n" + "fmla v12.8h, v21.8h, v1.h[3]\n" + "fmla v16.8h, v21.8h, v2.h[3]\n" + "ldr q21, [x10, #0xe0]\n" + "fmla v9.8h, v20.8h, v0.h[3]\n" + "fmla v13.8h, v20.8h, v1.h[3]\n" + "fmla v17.8h, v20.8h, v2.h[3]\n" + "ldr q20, [x10, #0xf0]\n" + "fmla v10.8h, v21.8h, v0.h[3]\n" + "fmla v14.8h, v21.8h, v1.h[3]\n" + "fmla v18.8h, v21.8h, v2.h[3]\n" + "ldr q21, [x10, #0x100]\n" + "fmla v11.8h, v20.8h, v0.h[3]\n" + "fmla v15.8h, v20.8h, v1.h[3]\n" + "fmla v19.8h, v20.8h, v2.h[3]\n" + "ldr q20, [x10, #0x110]\n" + "fmla v8.8h, v21.8h, v0.h[4]\n" + "fmla v12.8h, v21.8h, v1.h[4]\n" + "fmla v16.8h, v21.8h, v2.h[4]\n" + "ldr q21, [x10, #0x120]\n" + "fmla v9.8h, v20.8h, v0.h[4]\n" + "fmla v13.8h, v20.8h, v1.h[4]\n" + "fmla v17.8h, v20.8h, v2.h[4]\n" + "ldr q20, [x10, #0x130]\n" + "fmla v10.8h, v21.8h, v0.h[4]\n" + "fmla v14.8h, v21.8h, v1.h[4]\n" + "fmla v18.8h, v21.8h, v2.h[4]\n" + "ldr q21, [x10, #0x140]\n" + "fmla v11.8h, v20.8h, v0.h[4]\n" + "fmla v15.8h, v20.8h, v1.h[4]\n" + "fmla v19.8h, v20.8h, v2.h[4]\n" + "ldr q20, [x10, #0x150]\n" + "fmla v8.8h, v21.8h, v0.h[5]\n" + "fmla v12.8h, v21.8h, v1.h[5]\n" + "fmla v16.8h, v21.8h, v2.h[5]\n" + "ldr q21, [x10, #0x160]\n" + "fmla v9.8h, v20.8h, v0.h[5]\n" + "fmla v13.8h, v20.8h, v1.h[5]\n" + "fmla v17.8h, v20.8h, v2.h[5]\n" + "ldr q20, [x10, #0x170]\n" + "fmla v10.8h, v21.8h, v0.h[5]\n" + "fmla v14.8h, v21.8h, v1.h[5]\n" + "fmla v18.8h, v21.8h, v2.h[5]\n" + "ldr q21, [x10, #0x180]\n" + "fmla v11.8h, v20.8h, v0.h[5]\n" + "fmla v15.8h, v20.8h, v1.h[5]\n" + "fmla v19.8h, v20.8h, v2.h[5]\n" + "ldr q20, [x10, #0x190]\n" + "fmla v8.8h, v21.8h, v0.h[6]\n" + "fmla v12.8h, v21.8h, v1.h[6]\n" + "fmla v16.8h, v21.8h, v2.h[6]\n" + "ldr q21, [x10, #0x1a0]\n" + "fmla v9.8h, v20.8h, v0.h[6]\n" + "fmla v13.8h, v20.8h, v1.h[6]\n" + "fmla v17.8h, v20.8h, v2.h[6]\n" + "ldr q20, [x10, #0x1b0]\n" + "fmla v10.8h, v21.8h, v0.h[6]\n" + "fmla v14.8h, v21.8h, v1.h[6]\n" + "fmla v18.8h, v21.8h, v2.h[6]\n" + "ldr q21, [x10, #0x1c0]\n" + "fmla v11.8h, v20.8h, v0.h[6]\n" + "fmla v15.8h, v20.8h, v1.h[6]\n" + "fmla v19.8h, v20.8h, v2.h[6]\n" + "ldr q20, [x10, #0x1d0]\n" + "fmla v8.8h, v21.8h, v0.h[7]\n" + "fmla v12.8h, v21.8h, v1.h[7]\n" + "fmla v16.8h, v21.8h, v2.h[7]\n" + "ldr q21, [x10, #0x1e0]\n" + "fmla v9.8h, v20.8h, v0.h[7]\n" + "fmla v13.8h, v20.8h, v1.h[7]\n" + "fmla v17.8h, v20.8h, v2.h[7]\n" + "ldr q20, [x10, #0x1f0]\n" "add x10, x10, #0x200\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "fmla v14.8h, v6.8h, v1.h[7]\n" - "fmla v18.8h, v6.8h, v2.h[7]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" - "fmla v15.8h, v7.8h, v1.h[7]\n" - "fmla v19.8h, v7.8h, v2.h[7]\n" + "fmla v10.8h, v21.8h, v0.h[7]\n" + "fmla v14.8h, v21.8h, v1.h[7]\n" + "fmla v18.8h, v21.8h, v2.h[7]\n" + "fmla v11.8h, v20.8h, v0.h[7]\n" + "fmla v15.8h, v20.8h, v1.h[7]\n" + "fmla v19.8h, v20.8h, v2.h[7]\n" "126:" // Height 3: Multiply loop: Main loop skip "cbz x27, 128f\n" "127:" // Height 3: Multiply loop: Odd block loop - "ldr h0, [x26], #0x2\n" + "ldr h2, [x26], #0x2\n" "ldr h1, [x25], #0x2\n" "sub x27, x27, #0x1\n" - "ldr h2, [x24], #0x2\n" - "ldr q6, [x10, #0x0]\n" - "fmla v8.8h, v6.8h, v0.h[0]\n" - "fmla v12.8h, v6.8h, v1.h[0]\n" - "ldr q7, [x10, #0x10]\n" - "fmla v16.8h, v6.8h, v2.h[0]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.8h, v7.8h, v0.h[0]\n" - "fmla v13.8h, v7.8h, v1.h[0]\n" - "fmla v17.8h, v7.8h, v2.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr h0, [x24], #0x2\n" + "ldr q21, [x10, #0x0]\n" + "fmla v8.8h, v21.8h, v2.h[0]\n" + "fmla v12.8h, v21.8h, v1.h[0]\n" + "ldr q20, [x10, #0x10]\n" + "fmla v16.8h, v21.8h, v0.h[0]\n" + "ldr q21, [x10, #0x20]\n" + "fmla v9.8h, v20.8h, v2.h[0]\n" + "fmla v13.8h, v20.8h, v1.h[0]\n" + "fmla v17.8h, v20.8h, v0.h[0]\n" + "ldr q20, [x10, #0x30]\n" "add x10, x10, #0x40\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" + "fmla v10.8h, v21.8h, v2.h[0]\n" + "fmla v14.8h, v21.8h, v1.h[0]\n" + "fmla v18.8h, v21.8h, v0.h[0]\n" + "fmla v11.8h, v20.8h, v2.h[0]\n" + "fmla v15.8h, v20.8h, v1.h[0]\n" + "fmla v19.8h, v20.8h, v0.h[0]\n" "cbnz x27, 127b\n" "128:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -1734,33 +1734,33 @@ void a64_hybrid_fp16_mla_6x32 ( "prfm pstl1keep, [x24, #0x0]\n" "tbz %x[flags], #1, 129f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.8h }, [x20]\n" + "ld1r { v21.8h }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.8h }, [x20]\n" - "fmin v8.8h, v8.8h, v1.8h\n" - "fmin v9.8h, v9.8h, v1.8h\n" - "fmin v10.8h, v10.8h, v1.8h\n" - "fmin v11.8h, v11.8h, v1.8h\n" - "fmin v12.8h, v12.8h, v1.8h\n" - "fmin v13.8h, v13.8h, v1.8h\n" - "fmin v14.8h, v14.8h, v1.8h\n" - "fmin v15.8h, v15.8h, v1.8h\n" - "fmin v16.8h, v16.8h, v1.8h\n" - "fmin v17.8h, v17.8h, v1.8h\n" - "fmin v18.8h, v18.8h, v1.8h\n" - "fmin v19.8h, v19.8h, v1.8h\n" - "fmax v8.8h, v8.8h, v0.8h\n" - "fmax v9.8h, v9.8h, v0.8h\n" - "fmax v10.8h, v10.8h, v0.8h\n" - "fmax v11.8h, v11.8h, v0.8h\n" - "fmax v12.8h, v12.8h, v0.8h\n" - "fmax v13.8h, v13.8h, v0.8h\n" - "fmax v14.8h, v14.8h, v0.8h\n" - "fmax v15.8h, v15.8h, v0.8h\n" - "fmax v16.8h, v16.8h, v0.8h\n" - "fmax v17.8h, v17.8h, v0.8h\n" - "fmax v18.8h, v18.8h, v0.8h\n" - "fmax v19.8h, v19.8h, v0.8h\n" + "ld1r { v20.8h }, [x20]\n" + "fmin v8.8h, v8.8h, v21.8h\n" + "fmin v9.8h, v9.8h, v21.8h\n" + "fmin v10.8h, v10.8h, v21.8h\n" + "fmin v11.8h, v11.8h, v21.8h\n" + "fmin v12.8h, v12.8h, v21.8h\n" + "fmin v13.8h, v13.8h, v21.8h\n" + "fmin v14.8h, v14.8h, v21.8h\n" + "fmin v15.8h, v15.8h, v21.8h\n" + "fmin v16.8h, v16.8h, v21.8h\n" + "fmin v17.8h, v17.8h, v21.8h\n" + "fmin v18.8h, v18.8h, v21.8h\n" + "fmin v19.8h, v19.8h, v21.8h\n" + "fmax v8.8h, v8.8h, v20.8h\n" + "fmax v9.8h, v9.8h, v20.8h\n" + "fmax v10.8h, v10.8h, v20.8h\n" + "fmax v11.8h, v11.8h, v20.8h\n" + "fmax v12.8h, v12.8h, v20.8h\n" + "fmax v13.8h, v13.8h, v20.8h\n" + "fmax v14.8h, v14.8h, v20.8h\n" + "fmax v15.8h, v15.8h, v20.8h\n" + "fmax v16.8h, v16.8h, v20.8h\n" + "fmax v17.8h, v17.8h, v20.8h\n" + "fmax v18.8h, v18.8h, v20.8h\n" + "fmax v19.8h, v19.8h, v20.8h\n" "129:" // Height 3: No activation "cmp x11, #0x20\n" "bge 146f\n" @@ -2220,14 +2220,14 @@ void a64_hybrid_fp16_mla_6x32 ( "170:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 171f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 172f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -2237,9 +2237,9 @@ void a64_hybrid_fp16_mla_6x32 ( "b 172f\n" "171:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" "172:" // Height 4: input setup done "cmp x27, #0x8\n" "blt 175f\n" @@ -2258,7 +2258,7 @@ void a64_hybrid_fp16_mla_6x32 ( "add x26, x26, #0x10\n" "fmla v16.8h, v6.8h, v2.h[0]\n" "fmla v20.8h, v6.8h, v3.h[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q25, [x10, #0x20]\n" "add x25, x25, #0x10\n" "fmla v9.8h, v7.8h, v0.h[0]\n" "fmla v13.8h, v7.8h, v1.h[0]\n" @@ -2266,165 +2266,165 @@ void a64_hybrid_fp16_mla_6x32 ( "add x23, x23, #0x10\n" "fmla v17.8h, v7.8h, v2.h[0]\n" "fmla v21.8h, v7.8h, v3.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q24, [x10, #0x30]\n" "cmp x27, #0x10\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" + "fmla v10.8h, v25.8h, v0.h[0]\n" + "fmla v14.8h, v25.8h, v1.h[0]\n" "prfm pldl1keep, [x26, #0x80]\n" "prfm pldl1keep, [x25, #0x80]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "fmla v22.8h, v6.8h, v3.h[0]\n" - "ldr q6, [x10, #0x40]\n" + "fmla v18.8h, v25.8h, v2.h[0]\n" + "fmla v22.8h, v25.8h, v3.h[0]\n" + "ldr q25, [x10, #0x40]\n" "prfm pldl1keep, [x24, #0x80]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" + "fmla v11.8h, v24.8h, v0.h[0]\n" + "fmla v15.8h, v24.8h, v1.h[0]\n" "prfm pldl1keep, [x23, #0x80]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "fmla v23.8h, v7.8h, v3.h[0]\n" - "ldr q7, [x10, #0x50]\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "fmla v16.8h, v6.8h, v2.h[1]\n" - "fmla v20.8h, v6.8h, v3.h[1]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "fmla v17.8h, v7.8h, v2.h[1]\n" - "fmla v21.8h, v7.8h, v3.h[1]\n" - "ldr q7, [x10, #0x70]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "fmla v14.8h, v6.8h, v1.h[1]\n" - "fmla v18.8h, v6.8h, v2.h[1]\n" - "fmla v22.8h, v6.8h, v3.h[1]\n" - "ldr q6, [x10, #0x80]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "fmla v15.8h, v7.8h, v1.h[1]\n" - "fmla v19.8h, v7.8h, v2.h[1]\n" - "fmla v23.8h, v7.8h, v3.h[1]\n" - "ldr q7, [x10, #0x90]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "fmla v12.8h, v6.8h, v1.h[2]\n" - "fmla v16.8h, v6.8h, v2.h[2]\n" - "fmla v20.8h, v6.8h, v3.h[2]\n" - "ldr q6, [x10, #0xa0]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "fmla v13.8h, v7.8h, v1.h[2]\n" - "fmla v17.8h, v7.8h, v2.h[2]\n" - "fmla v21.8h, v7.8h, v3.h[2]\n" - "ldr q7, [x10, #0xb0]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "fmla v14.8h, v6.8h, v1.h[2]\n" - "fmla v18.8h, v6.8h, v2.h[2]\n" - "fmla v22.8h, v6.8h, v3.h[2]\n" - "ldr q6, [x10, #0xc0]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "fmla v15.8h, v7.8h, v1.h[2]\n" - "fmla v19.8h, v7.8h, v2.h[2]\n" - "fmla v23.8h, v7.8h, v3.h[2]\n" - "ldr q7, [x10, #0xd0]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "fmla v12.8h, v6.8h, v1.h[3]\n" - "fmla v16.8h, v6.8h, v2.h[3]\n" - "fmla v20.8h, v6.8h, v3.h[3]\n" - "ldr q6, [x10, #0xe0]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "fmla v13.8h, v7.8h, v1.h[3]\n" - "fmla v17.8h, v7.8h, v2.h[3]\n" - "fmla v21.8h, v7.8h, v3.h[3]\n" - "ldr q7, [x10, #0xf0]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "fmla v14.8h, v6.8h, v1.h[3]\n" - "fmla v18.8h, v6.8h, v2.h[3]\n" - "fmla v22.8h, v6.8h, v3.h[3]\n" - "ldr q6, [x10, #0x100]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "fmla v15.8h, v7.8h, v1.h[3]\n" - "fmla v19.8h, v7.8h, v2.h[3]\n" - "fmla v23.8h, v7.8h, v3.h[3]\n" - "ldr q7, [x10, #0x110]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "fmla v12.8h, v6.8h, v1.h[4]\n" - "fmla v16.8h, v6.8h, v2.h[4]\n" - "fmla v20.8h, v6.8h, v3.h[4]\n" - "ldr q6, [x10, #0x120]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "fmla v13.8h, v7.8h, v1.h[4]\n" - "fmla v17.8h, v7.8h, v2.h[4]\n" - "fmla v21.8h, v7.8h, v3.h[4]\n" - "ldr q7, [x10, #0x130]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "fmla v14.8h, v6.8h, v1.h[4]\n" - "fmla v18.8h, v6.8h, v2.h[4]\n" - "fmla v22.8h, v6.8h, v3.h[4]\n" - "ldr q6, [x10, #0x140]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "fmla v15.8h, v7.8h, v1.h[4]\n" - "fmla v19.8h, v7.8h, v2.h[4]\n" - "fmla v23.8h, v7.8h, v3.h[4]\n" - "ldr q7, [x10, #0x150]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "fmla v12.8h, v6.8h, v1.h[5]\n" - "fmla v16.8h, v6.8h, v2.h[5]\n" - "fmla v20.8h, v6.8h, v3.h[5]\n" - "ldr q6, [x10, #0x160]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "fmla v13.8h, v7.8h, v1.h[5]\n" - "fmla v17.8h, v7.8h, v2.h[5]\n" - "fmla v21.8h, v7.8h, v3.h[5]\n" - "ldr q7, [x10, #0x170]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "fmla v14.8h, v6.8h, v1.h[5]\n" - "fmla v18.8h, v6.8h, v2.h[5]\n" - "fmla v22.8h, v6.8h, v3.h[5]\n" - "ldr q6, [x10, #0x180]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "fmla v15.8h, v7.8h, v1.h[5]\n" - "fmla v19.8h, v7.8h, v2.h[5]\n" - "fmla v23.8h, v7.8h, v3.h[5]\n" - "ldr q7, [x10, #0x190]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "fmla v12.8h, v6.8h, v1.h[6]\n" - "fmla v16.8h, v6.8h, v2.h[6]\n" - "fmla v20.8h, v6.8h, v3.h[6]\n" - "ldr q6, [x10, #0x1a0]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "fmla v13.8h, v7.8h, v1.h[6]\n" - "fmla v17.8h, v7.8h, v2.h[6]\n" - "fmla v21.8h, v7.8h, v3.h[6]\n" - "ldr q7, [x10, #0x1b0]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "fmla v14.8h, v6.8h, v1.h[6]\n" - "fmla v18.8h, v6.8h, v2.h[6]\n" - "fmla v22.8h, v6.8h, v3.h[6]\n" - "ldr q6, [x10, #0x1c0]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "fmla v15.8h, v7.8h, v1.h[6]\n" - "fmla v19.8h, v7.8h, v2.h[6]\n" - "fmla v23.8h, v7.8h, v3.h[6]\n" - "ldr q7, [x10, #0x1d0]\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "fmla v12.8h, v6.8h, v1.h[7]\n" - "fmla v16.8h, v6.8h, v2.h[7]\n" - "fmla v20.8h, v6.8h, v3.h[7]\n" - "ldr q6, [x10, #0x1e0]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "fmla v13.8h, v7.8h, v1.h[7]\n" - "fmla v17.8h, v7.8h, v2.h[7]\n" - "fmla v21.8h, v7.8h, v3.h[7]\n" - "ldr q7, [x10, #0x1f0]\n" + "fmla v19.8h, v24.8h, v2.h[0]\n" + "fmla v23.8h, v24.8h, v3.h[0]\n" + "ldr q24, [x10, #0x50]\n" + "fmla v8.8h, v25.8h, v0.h[1]\n" + "fmla v12.8h, v25.8h, v1.h[1]\n" + "fmla v16.8h, v25.8h, v2.h[1]\n" + "fmla v20.8h, v25.8h, v3.h[1]\n" + "ldr q25, [x10, #0x60]\n" + "fmla v9.8h, v24.8h, v0.h[1]\n" + "fmla v13.8h, v24.8h, v1.h[1]\n" + "fmla v17.8h, v24.8h, v2.h[1]\n" + "fmla v21.8h, v24.8h, v3.h[1]\n" + "ldr q24, [x10, #0x70]\n" + "fmla v10.8h, v25.8h, v0.h[1]\n" + "fmla v14.8h, v25.8h, v1.h[1]\n" + "fmla v18.8h, v25.8h, v2.h[1]\n" + "fmla v22.8h, v25.8h, v3.h[1]\n" + "ldr q25, [x10, #0x80]\n" + "fmla v11.8h, v24.8h, v0.h[1]\n" + "fmla v15.8h, v24.8h, v1.h[1]\n" + "fmla v19.8h, v24.8h, v2.h[1]\n" + "fmla v23.8h, v24.8h, v3.h[1]\n" + "ldr q24, [x10, #0x90]\n" + "fmla v8.8h, v25.8h, v0.h[2]\n" + "fmla v12.8h, v25.8h, v1.h[2]\n" + "fmla v16.8h, v25.8h, v2.h[2]\n" + "fmla v20.8h, v25.8h, v3.h[2]\n" + "ldr q25, [x10, #0xa0]\n" + "fmla v9.8h, v24.8h, v0.h[2]\n" + "fmla v13.8h, v24.8h, v1.h[2]\n" + "fmla v17.8h, v24.8h, v2.h[2]\n" + "fmla v21.8h, v24.8h, v3.h[2]\n" + "ldr q24, [x10, #0xb0]\n" + "fmla v10.8h, v25.8h, v0.h[2]\n" + "fmla v14.8h, v25.8h, v1.h[2]\n" + "fmla v18.8h, v25.8h, v2.h[2]\n" + "fmla v22.8h, v25.8h, v3.h[2]\n" + "ldr q25, [x10, #0xc0]\n" + "fmla v11.8h, v24.8h, v0.h[2]\n" + "fmla v15.8h, v24.8h, v1.h[2]\n" + "fmla v19.8h, v24.8h, v2.h[2]\n" + "fmla v23.8h, v24.8h, v3.h[2]\n" + "ldr q24, [x10, #0xd0]\n" + "fmla v8.8h, v25.8h, v0.h[3]\n" + "fmla v12.8h, v25.8h, v1.h[3]\n" + "fmla v16.8h, v25.8h, v2.h[3]\n" + "fmla v20.8h, v25.8h, v3.h[3]\n" + "ldr q25, [x10, #0xe0]\n" + "fmla v9.8h, v24.8h, v0.h[3]\n" + "fmla v13.8h, v24.8h, v1.h[3]\n" + "fmla v17.8h, v24.8h, v2.h[3]\n" + "fmla v21.8h, v24.8h, v3.h[3]\n" + "ldr q24, [x10, #0xf0]\n" + "fmla v10.8h, v25.8h, v0.h[3]\n" + "fmla v14.8h, v25.8h, v1.h[3]\n" + "fmla v18.8h, v25.8h, v2.h[3]\n" + "fmla v22.8h, v25.8h, v3.h[3]\n" + "ldr q25, [x10, #0x100]\n" + "fmla v11.8h, v24.8h, v0.h[3]\n" + "fmla v15.8h, v24.8h, v1.h[3]\n" + "fmla v19.8h, v24.8h, v2.h[3]\n" + "fmla v23.8h, v24.8h, v3.h[3]\n" + "ldr q24, [x10, #0x110]\n" + "fmla v8.8h, v25.8h, v0.h[4]\n" + "fmla v12.8h, v25.8h, v1.h[4]\n" + "fmla v16.8h, v25.8h, v2.h[4]\n" + "fmla v20.8h, v25.8h, v3.h[4]\n" + "ldr q25, [x10, #0x120]\n" + "fmla v9.8h, v24.8h, v0.h[4]\n" + "fmla v13.8h, v24.8h, v1.h[4]\n" + "fmla v17.8h, v24.8h, v2.h[4]\n" + "fmla v21.8h, v24.8h, v3.h[4]\n" + "ldr q24, [x10, #0x130]\n" + "fmla v10.8h, v25.8h, v0.h[4]\n" + "fmla v14.8h, v25.8h, v1.h[4]\n" + "fmla v18.8h, v25.8h, v2.h[4]\n" + "fmla v22.8h, v25.8h, v3.h[4]\n" + "ldr q25, [x10, #0x140]\n" + "fmla v11.8h, v24.8h, v0.h[4]\n" + "fmla v15.8h, v24.8h, v1.h[4]\n" + "fmla v19.8h, v24.8h, v2.h[4]\n" + "fmla v23.8h, v24.8h, v3.h[4]\n" + "ldr q24, [x10, #0x150]\n" + "fmla v8.8h, v25.8h, v0.h[5]\n" + "fmla v12.8h, v25.8h, v1.h[5]\n" + "fmla v16.8h, v25.8h, v2.h[5]\n" + "fmla v20.8h, v25.8h, v3.h[5]\n" + "ldr q25, [x10, #0x160]\n" + "fmla v9.8h, v24.8h, v0.h[5]\n" + "fmla v13.8h, v24.8h, v1.h[5]\n" + "fmla v17.8h, v24.8h, v2.h[5]\n" + "fmla v21.8h, v24.8h, v3.h[5]\n" + "ldr q24, [x10, #0x170]\n" + "fmla v10.8h, v25.8h, v0.h[5]\n" + "fmla v14.8h, v25.8h, v1.h[5]\n" + "fmla v18.8h, v25.8h, v2.h[5]\n" + "fmla v22.8h, v25.8h, v3.h[5]\n" + "ldr q25, [x10, #0x180]\n" + "fmla v11.8h, v24.8h, v0.h[5]\n" + "fmla v15.8h, v24.8h, v1.h[5]\n" + "fmla v19.8h, v24.8h, v2.h[5]\n" + "fmla v23.8h, v24.8h, v3.h[5]\n" + "ldr q24, [x10, #0x190]\n" + "fmla v8.8h, v25.8h, v0.h[6]\n" + "fmla v12.8h, v25.8h, v1.h[6]\n" + "fmla v16.8h, v25.8h, v2.h[6]\n" + "fmla v20.8h, v25.8h, v3.h[6]\n" + "ldr q25, [x10, #0x1a0]\n" + "fmla v9.8h, v24.8h, v0.h[6]\n" + "fmla v13.8h, v24.8h, v1.h[6]\n" + "fmla v17.8h, v24.8h, v2.h[6]\n" + "fmla v21.8h, v24.8h, v3.h[6]\n" + "ldr q24, [x10, #0x1b0]\n" + "fmla v10.8h, v25.8h, v0.h[6]\n" + "fmla v14.8h, v25.8h, v1.h[6]\n" + "fmla v18.8h, v25.8h, v2.h[6]\n" + "fmla v22.8h, v25.8h, v3.h[6]\n" + "ldr q25, [x10, #0x1c0]\n" + "fmla v11.8h, v24.8h, v0.h[6]\n" + "fmla v15.8h, v24.8h, v1.h[6]\n" + "fmla v19.8h, v24.8h, v2.h[6]\n" + "fmla v23.8h, v24.8h, v3.h[6]\n" + "ldr q24, [x10, #0x1d0]\n" + "fmla v8.8h, v25.8h, v0.h[7]\n" + "fmla v12.8h, v25.8h, v1.h[7]\n" + "fmla v16.8h, v25.8h, v2.h[7]\n" + "fmla v20.8h, v25.8h, v3.h[7]\n" + "ldr q25, [x10, #0x1e0]\n" + "fmla v9.8h, v24.8h, v0.h[7]\n" + "fmla v13.8h, v24.8h, v1.h[7]\n" + "fmla v17.8h, v24.8h, v2.h[7]\n" + "fmla v21.8h, v24.8h, v3.h[7]\n" + "ldr q24, [x10, #0x1f0]\n" "add x10, x10, #0x200\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "fmla v14.8h, v6.8h, v1.h[7]\n" - "fmla v18.8h, v6.8h, v2.h[7]\n" - "fmla v22.8h, v6.8h, v3.h[7]\n" + "fmla v10.8h, v25.8h, v0.h[7]\n" + "fmla v14.8h, v25.8h, v1.h[7]\n" + "fmla v18.8h, v25.8h, v2.h[7]\n" + "fmla v22.8h, v25.8h, v3.h[7]\n" "ldr q6, [x10, #0x0]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" + "fmla v11.8h, v24.8h, v0.h[7]\n" "ldr q0, [x26, #0x0]\n" - "fmla v15.8h, v7.8h, v1.h[7]\n" + "fmla v15.8h, v24.8h, v1.h[7]\n" "ldr q1, [x25, #0x0]\n" - "fmla v19.8h, v7.8h, v2.h[7]\n" + "fmla v19.8h, v24.8h, v2.h[7]\n" "ldr q2, [x24, #0x0]\n" - "fmla v23.8h, v7.8h, v3.h[7]\n" + "fmla v23.8h, v24.8h, v3.h[7]\n" "ldr q3, [x23, #0x0]\n" "ldr q7, [x10, #0x10]\n" "bge 173b\n" @@ -2435,7 +2435,7 @@ void a64_hybrid_fp16_mla_6x32 ( "add x25, x25, #0x10\n" "fmla v16.8h, v6.8h, v2.h[0]\n" "fmla v20.8h, v6.8h, v3.h[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q25, [x10, #0x20]\n" "add x24, x24, #0x10\n" "fmla v9.8h, v7.8h, v0.h[0]\n" "fmla v13.8h, v7.8h, v1.h[0]\n" @@ -2443,189 +2443,189 @@ void a64_hybrid_fp16_mla_6x32 ( "sub x27, x27, #0x8\n" "fmla v17.8h, v7.8h, v2.h[0]\n" "fmla v21.8h, v7.8h, v3.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q24, [x10, #0x30]\n" "prfm pldl1keep, [x26, #0x80]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" + "fmla v10.8h, v25.8h, v0.h[0]\n" + "fmla v14.8h, v25.8h, v1.h[0]\n" "prfm pldl1keep, [x25, #0x80]\n" "prfm pldl1keep, [x24, #0x80]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "fmla v22.8h, v6.8h, v3.h[0]\n" - "ldr q6, [x10, #0x40]\n" + "fmla v18.8h, v25.8h, v2.h[0]\n" + "fmla v22.8h, v25.8h, v3.h[0]\n" + "ldr q25, [x10, #0x40]\n" "prfm pldl1keep, [x23, #0x80]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "fmla v23.8h, v7.8h, v3.h[0]\n" - "ldr q7, [x10, #0x50]\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "fmla v16.8h, v6.8h, v2.h[1]\n" - "fmla v20.8h, v6.8h, v3.h[1]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "fmla v17.8h, v7.8h, v2.h[1]\n" - "fmla v21.8h, v7.8h, v3.h[1]\n" - "ldr q7, [x10, #0x70]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "fmla v14.8h, v6.8h, v1.h[1]\n" - "fmla v18.8h, v6.8h, v2.h[1]\n" - "fmla v22.8h, v6.8h, v3.h[1]\n" - "ldr q6, [x10, #0x80]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "fmla v15.8h, v7.8h, v1.h[1]\n" - "fmla v19.8h, v7.8h, v2.h[1]\n" - "fmla v23.8h, v7.8h, v3.h[1]\n" - "ldr q7, [x10, #0x90]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "fmla v12.8h, v6.8h, v1.h[2]\n" - "fmla v16.8h, v6.8h, v2.h[2]\n" - "fmla v20.8h, v6.8h, v3.h[2]\n" - "ldr q6, [x10, #0xa0]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "fmla v13.8h, v7.8h, v1.h[2]\n" - "fmla v17.8h, v7.8h, v2.h[2]\n" - "fmla v21.8h, v7.8h, v3.h[2]\n" - "ldr q7, [x10, #0xb0]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "fmla v14.8h, v6.8h, v1.h[2]\n" - "fmla v18.8h, v6.8h, v2.h[2]\n" - "fmla v22.8h, v6.8h, v3.h[2]\n" - "ldr q6, [x10, #0xc0]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "fmla v15.8h, v7.8h, v1.h[2]\n" - "fmla v19.8h, v7.8h, v2.h[2]\n" - "fmla v23.8h, v7.8h, v3.h[2]\n" - "ldr q7, [x10, #0xd0]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "fmla v12.8h, v6.8h, v1.h[3]\n" - "fmla v16.8h, v6.8h, v2.h[3]\n" - "fmla v20.8h, v6.8h, v3.h[3]\n" - "ldr q6, [x10, #0xe0]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "fmla v13.8h, v7.8h, v1.h[3]\n" - "fmla v17.8h, v7.8h, v2.h[3]\n" - "fmla v21.8h, v7.8h, v3.h[3]\n" - "ldr q7, [x10, #0xf0]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "fmla v14.8h, v6.8h, v1.h[3]\n" - "fmla v18.8h, v6.8h, v2.h[3]\n" - "fmla v22.8h, v6.8h, v3.h[3]\n" - "ldr q6, [x10, #0x100]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "fmla v15.8h, v7.8h, v1.h[3]\n" - "fmla v19.8h, v7.8h, v2.h[3]\n" - "fmla v23.8h, v7.8h, v3.h[3]\n" - "ldr q7, [x10, #0x110]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "fmla v12.8h, v6.8h, v1.h[4]\n" - "fmla v16.8h, v6.8h, v2.h[4]\n" - "fmla v20.8h, v6.8h, v3.h[4]\n" - "ldr q6, [x10, #0x120]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "fmla v13.8h, v7.8h, v1.h[4]\n" - "fmla v17.8h, v7.8h, v2.h[4]\n" - "fmla v21.8h, v7.8h, v3.h[4]\n" - "ldr q7, [x10, #0x130]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "fmla v14.8h, v6.8h, v1.h[4]\n" - "fmla v18.8h, v6.8h, v2.h[4]\n" - "fmla v22.8h, v6.8h, v3.h[4]\n" - "ldr q6, [x10, #0x140]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "fmla v15.8h, v7.8h, v1.h[4]\n" - "fmla v19.8h, v7.8h, v2.h[4]\n" - "fmla v23.8h, v7.8h, v3.h[4]\n" - "ldr q7, [x10, #0x150]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "fmla v12.8h, v6.8h, v1.h[5]\n" - "fmla v16.8h, v6.8h, v2.h[5]\n" - "fmla v20.8h, v6.8h, v3.h[5]\n" - "ldr q6, [x10, #0x160]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "fmla v13.8h, v7.8h, v1.h[5]\n" - "fmla v17.8h, v7.8h, v2.h[5]\n" - "fmla v21.8h, v7.8h, v3.h[5]\n" - "ldr q7, [x10, #0x170]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "fmla v14.8h, v6.8h, v1.h[5]\n" - "fmla v18.8h, v6.8h, v2.h[5]\n" - "fmla v22.8h, v6.8h, v3.h[5]\n" - "ldr q6, [x10, #0x180]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "fmla v15.8h, v7.8h, v1.h[5]\n" - "fmla v19.8h, v7.8h, v2.h[5]\n" - "fmla v23.8h, v7.8h, v3.h[5]\n" - "ldr q7, [x10, #0x190]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "fmla v12.8h, v6.8h, v1.h[6]\n" - "fmla v16.8h, v6.8h, v2.h[6]\n" - "fmla v20.8h, v6.8h, v3.h[6]\n" - "ldr q6, [x10, #0x1a0]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "fmla v13.8h, v7.8h, v1.h[6]\n" - "fmla v17.8h, v7.8h, v2.h[6]\n" - "fmla v21.8h, v7.8h, v3.h[6]\n" - "ldr q7, [x10, #0x1b0]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "fmla v14.8h, v6.8h, v1.h[6]\n" - "fmla v18.8h, v6.8h, v2.h[6]\n" - "fmla v22.8h, v6.8h, v3.h[6]\n" - "ldr q6, [x10, #0x1c0]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "fmla v15.8h, v7.8h, v1.h[6]\n" - "fmla v19.8h, v7.8h, v2.h[6]\n" - "fmla v23.8h, v7.8h, v3.h[6]\n" - "ldr q7, [x10, #0x1d0]\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "fmla v12.8h, v6.8h, v1.h[7]\n" - "fmla v16.8h, v6.8h, v2.h[7]\n" - "fmla v20.8h, v6.8h, v3.h[7]\n" - "ldr q6, [x10, #0x1e0]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "fmla v13.8h, v7.8h, v1.h[7]\n" - "fmla v17.8h, v7.8h, v2.h[7]\n" - "fmla v21.8h, v7.8h, v3.h[7]\n" - "ldr q7, [x10, #0x1f0]\n" + "fmla v11.8h, v24.8h, v0.h[0]\n" + "fmla v15.8h, v24.8h, v1.h[0]\n" + "fmla v19.8h, v24.8h, v2.h[0]\n" + "fmla v23.8h, v24.8h, v3.h[0]\n" + "ldr q24, [x10, #0x50]\n" + "fmla v8.8h, v25.8h, v0.h[1]\n" + "fmla v12.8h, v25.8h, v1.h[1]\n" + "fmla v16.8h, v25.8h, v2.h[1]\n" + "fmla v20.8h, v25.8h, v3.h[1]\n" + "ldr q25, [x10, #0x60]\n" + "fmla v9.8h, v24.8h, v0.h[1]\n" + "fmla v13.8h, v24.8h, v1.h[1]\n" + "fmla v17.8h, v24.8h, v2.h[1]\n" + "fmla v21.8h, v24.8h, v3.h[1]\n" + "ldr q24, [x10, #0x70]\n" + "fmla v10.8h, v25.8h, v0.h[1]\n" + "fmla v14.8h, v25.8h, v1.h[1]\n" + "fmla v18.8h, v25.8h, v2.h[1]\n" + "fmla v22.8h, v25.8h, v3.h[1]\n" + "ldr q25, [x10, #0x80]\n" + "fmla v11.8h, v24.8h, v0.h[1]\n" + "fmla v15.8h, v24.8h, v1.h[1]\n" + "fmla v19.8h, v24.8h, v2.h[1]\n" + "fmla v23.8h, v24.8h, v3.h[1]\n" + "ldr q24, [x10, #0x90]\n" + "fmla v8.8h, v25.8h, v0.h[2]\n" + "fmla v12.8h, v25.8h, v1.h[2]\n" + "fmla v16.8h, v25.8h, v2.h[2]\n" + "fmla v20.8h, v25.8h, v3.h[2]\n" + "ldr q25, [x10, #0xa0]\n" + "fmla v9.8h, v24.8h, v0.h[2]\n" + "fmla v13.8h, v24.8h, v1.h[2]\n" + "fmla v17.8h, v24.8h, v2.h[2]\n" + "fmla v21.8h, v24.8h, v3.h[2]\n" + "ldr q24, [x10, #0xb0]\n" + "fmla v10.8h, v25.8h, v0.h[2]\n" + "fmla v14.8h, v25.8h, v1.h[2]\n" + "fmla v18.8h, v25.8h, v2.h[2]\n" + "fmla v22.8h, v25.8h, v3.h[2]\n" + "ldr q25, [x10, #0xc0]\n" + "fmla v11.8h, v24.8h, v0.h[2]\n" + "fmla v15.8h, v24.8h, v1.h[2]\n" + "fmla v19.8h, v24.8h, v2.h[2]\n" + "fmla v23.8h, v24.8h, v3.h[2]\n" + "ldr q24, [x10, #0xd0]\n" + "fmla v8.8h, v25.8h, v0.h[3]\n" + "fmla v12.8h, v25.8h, v1.h[3]\n" + "fmla v16.8h, v25.8h, v2.h[3]\n" + "fmla v20.8h, v25.8h, v3.h[3]\n" + "ldr q25, [x10, #0xe0]\n" + "fmla v9.8h, v24.8h, v0.h[3]\n" + "fmla v13.8h, v24.8h, v1.h[3]\n" + "fmla v17.8h, v24.8h, v2.h[3]\n" + "fmla v21.8h, v24.8h, v3.h[3]\n" + "ldr q24, [x10, #0xf0]\n" + "fmla v10.8h, v25.8h, v0.h[3]\n" + "fmla v14.8h, v25.8h, v1.h[3]\n" + "fmla v18.8h, v25.8h, v2.h[3]\n" + "fmla v22.8h, v25.8h, v3.h[3]\n" + "ldr q25, [x10, #0x100]\n" + "fmla v11.8h, v24.8h, v0.h[3]\n" + "fmla v15.8h, v24.8h, v1.h[3]\n" + "fmla v19.8h, v24.8h, v2.h[3]\n" + "fmla v23.8h, v24.8h, v3.h[3]\n" + "ldr q24, [x10, #0x110]\n" + "fmla v8.8h, v25.8h, v0.h[4]\n" + "fmla v12.8h, v25.8h, v1.h[4]\n" + "fmla v16.8h, v25.8h, v2.h[4]\n" + "fmla v20.8h, v25.8h, v3.h[4]\n" + "ldr q25, [x10, #0x120]\n" + "fmla v9.8h, v24.8h, v0.h[4]\n" + "fmla v13.8h, v24.8h, v1.h[4]\n" + "fmla v17.8h, v24.8h, v2.h[4]\n" + "fmla v21.8h, v24.8h, v3.h[4]\n" + "ldr q24, [x10, #0x130]\n" + "fmla v10.8h, v25.8h, v0.h[4]\n" + "fmla v14.8h, v25.8h, v1.h[4]\n" + "fmla v18.8h, v25.8h, v2.h[4]\n" + "fmla v22.8h, v25.8h, v3.h[4]\n" + "ldr q25, [x10, #0x140]\n" + "fmla v11.8h, v24.8h, v0.h[4]\n" + "fmla v15.8h, v24.8h, v1.h[4]\n" + "fmla v19.8h, v24.8h, v2.h[4]\n" + "fmla v23.8h, v24.8h, v3.h[4]\n" + "ldr q24, [x10, #0x150]\n" + "fmla v8.8h, v25.8h, v0.h[5]\n" + "fmla v12.8h, v25.8h, v1.h[5]\n" + "fmla v16.8h, v25.8h, v2.h[5]\n" + "fmla v20.8h, v25.8h, v3.h[5]\n" + "ldr q25, [x10, #0x160]\n" + "fmla v9.8h, v24.8h, v0.h[5]\n" + "fmla v13.8h, v24.8h, v1.h[5]\n" + "fmla v17.8h, v24.8h, v2.h[5]\n" + "fmla v21.8h, v24.8h, v3.h[5]\n" + "ldr q24, [x10, #0x170]\n" + "fmla v10.8h, v25.8h, v0.h[5]\n" + "fmla v14.8h, v25.8h, v1.h[5]\n" + "fmla v18.8h, v25.8h, v2.h[5]\n" + "fmla v22.8h, v25.8h, v3.h[5]\n" + "ldr q25, [x10, #0x180]\n" + "fmla v11.8h, v24.8h, v0.h[5]\n" + "fmla v15.8h, v24.8h, v1.h[5]\n" + "fmla v19.8h, v24.8h, v2.h[5]\n" + "fmla v23.8h, v24.8h, v3.h[5]\n" + "ldr q24, [x10, #0x190]\n" + "fmla v8.8h, v25.8h, v0.h[6]\n" + "fmla v12.8h, v25.8h, v1.h[6]\n" + "fmla v16.8h, v25.8h, v2.h[6]\n" + "fmla v20.8h, v25.8h, v3.h[6]\n" + "ldr q25, [x10, #0x1a0]\n" + "fmla v9.8h, v24.8h, v0.h[6]\n" + "fmla v13.8h, v24.8h, v1.h[6]\n" + "fmla v17.8h, v24.8h, v2.h[6]\n" + "fmla v21.8h, v24.8h, v3.h[6]\n" + "ldr q24, [x10, #0x1b0]\n" + "fmla v10.8h, v25.8h, v0.h[6]\n" + "fmla v14.8h, v25.8h, v1.h[6]\n" + "fmla v18.8h, v25.8h, v2.h[6]\n" + "fmla v22.8h, v25.8h, v3.h[6]\n" + "ldr q25, [x10, #0x1c0]\n" + "fmla v11.8h, v24.8h, v0.h[6]\n" + "fmla v15.8h, v24.8h, v1.h[6]\n" + "fmla v19.8h, v24.8h, v2.h[6]\n" + "fmla v23.8h, v24.8h, v3.h[6]\n" + "ldr q24, [x10, #0x1d0]\n" + "fmla v8.8h, v25.8h, v0.h[7]\n" + "fmla v12.8h, v25.8h, v1.h[7]\n" + "fmla v16.8h, v25.8h, v2.h[7]\n" + "fmla v20.8h, v25.8h, v3.h[7]\n" + "ldr q25, [x10, #0x1e0]\n" + "fmla v9.8h, v24.8h, v0.h[7]\n" + "fmla v13.8h, v24.8h, v1.h[7]\n" + "fmla v17.8h, v24.8h, v2.h[7]\n" + "fmla v21.8h, v24.8h, v3.h[7]\n" + "ldr q24, [x10, #0x1f0]\n" "add x10, x10, #0x200\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "fmla v14.8h, v6.8h, v1.h[7]\n" - "fmla v18.8h, v6.8h, v2.h[7]\n" - "fmla v22.8h, v6.8h, v3.h[7]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" - "fmla v15.8h, v7.8h, v1.h[7]\n" - "fmla v19.8h, v7.8h, v2.h[7]\n" - "fmla v23.8h, v7.8h, v3.h[7]\n" + "fmla v10.8h, v25.8h, v0.h[7]\n" + "fmla v14.8h, v25.8h, v1.h[7]\n" + "fmla v18.8h, v25.8h, v2.h[7]\n" + "fmla v22.8h, v25.8h, v3.h[7]\n" + "fmla v11.8h, v24.8h, v0.h[7]\n" + "fmla v15.8h, v24.8h, v1.h[7]\n" + "fmla v19.8h, v24.8h, v2.h[7]\n" + "fmla v23.8h, v24.8h, v3.h[7]\n" "175:" // Height 4: Multiply loop: Main loop skip "cbz x27, 177f\n" "176:" // Height 4: Multiply loop: Odd block loop - "ldr h0, [x26], #0x2\n" - "ldr h1, [x25], #0x2\n" + "ldr h3, [x26], #0x2\n" + "ldr h2, [x25], #0x2\n" "sub x27, x27, #0x1\n" - "ldr h2, [x24], #0x2\n" - "ldr h3, [x23], #0x2\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - "fmla v8.8h, v6.8h, v0.h[0]\n" - "fmla v12.8h, v6.8h, v1.h[0]\n" - "fmla v16.8h, v6.8h, v2.h[0]\n" - "fmla v20.8h, v6.8h, v3.h[0]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.8h, v7.8h, v0.h[0]\n" - "fmla v13.8h, v7.8h, v1.h[0]\n" - "fmla v17.8h, v7.8h, v2.h[0]\n" - "fmla v21.8h, v7.8h, v3.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr h1, [x24], #0x2\n" + "ldr h0, [x23], #0x2\n" + "ldr q25, [x10, #0x0]\n" + "ldr q24, [x10, #0x10]\n" + "fmla v8.8h, v25.8h, v3.h[0]\n" + "fmla v12.8h, v25.8h, v2.h[0]\n" + "fmla v16.8h, v25.8h, v1.h[0]\n" + "fmla v20.8h, v25.8h, v0.h[0]\n" + "ldr q25, [x10, #0x20]\n" + "fmla v9.8h, v24.8h, v3.h[0]\n" + "fmla v13.8h, v24.8h, v2.h[0]\n" + "fmla v17.8h, v24.8h, v1.h[0]\n" + "fmla v21.8h, v24.8h, v0.h[0]\n" + "ldr q24, [x10, #0x30]\n" "add x10, x10, #0x40\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "fmla v22.8h, v6.8h, v3.h[0]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "fmla v23.8h, v7.8h, v3.h[0]\n" + "fmla v10.8h, v25.8h, v3.h[0]\n" + "fmla v14.8h, v25.8h, v2.h[0]\n" + "fmla v18.8h, v25.8h, v1.h[0]\n" + "fmla v22.8h, v25.8h, v0.h[0]\n" + "fmla v11.8h, v24.8h, v3.h[0]\n" + "fmla v15.8h, v24.8h, v2.h[0]\n" + "fmla v19.8h, v24.8h, v1.h[0]\n" + "fmla v23.8h, v24.8h, v0.h[0]\n" "cbnz x27, 176b\n" "177:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -2642,41 +2642,41 @@ void a64_hybrid_fp16_mla_6x32 ( "prfm pstl1keep, [x23, #0x0]\n" "tbz %x[flags], #1, 178f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.8h }, [x20]\n" + "ld1r { v25.8h }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.8h }, [x20]\n" - "fmin v8.8h, v8.8h, v1.8h\n" - "fmin v9.8h, v9.8h, v1.8h\n" - "fmin v10.8h, v10.8h, v1.8h\n" - "fmin v11.8h, v11.8h, v1.8h\n" - "fmin v12.8h, v12.8h, v1.8h\n" - "fmin v13.8h, v13.8h, v1.8h\n" - "fmin v14.8h, v14.8h, v1.8h\n" - "fmin v15.8h, v15.8h, v1.8h\n" - "fmin v16.8h, v16.8h, v1.8h\n" - "fmin v17.8h, v17.8h, v1.8h\n" - "fmin v18.8h, v18.8h, v1.8h\n" - "fmin v19.8h, v19.8h, v1.8h\n" - "fmin v20.8h, v20.8h, v1.8h\n" - "fmin v21.8h, v21.8h, v1.8h\n" - "fmin v22.8h, v22.8h, v1.8h\n" - "fmin v23.8h, v23.8h, v1.8h\n" - "fmax v8.8h, v8.8h, v0.8h\n" - "fmax v9.8h, v9.8h, v0.8h\n" - "fmax v10.8h, v10.8h, v0.8h\n" - "fmax v11.8h, v11.8h, v0.8h\n" - "fmax v12.8h, v12.8h, v0.8h\n" - "fmax v13.8h, v13.8h, v0.8h\n" - "fmax v14.8h, v14.8h, v0.8h\n" - "fmax v15.8h, v15.8h, v0.8h\n" - "fmax v16.8h, v16.8h, v0.8h\n" - "fmax v17.8h, v17.8h, v0.8h\n" - "fmax v18.8h, v18.8h, v0.8h\n" - "fmax v19.8h, v19.8h, v0.8h\n" - "fmax v20.8h, v20.8h, v0.8h\n" - "fmax v21.8h, v21.8h, v0.8h\n" - "fmax v22.8h, v22.8h, v0.8h\n" - "fmax v23.8h, v23.8h, v0.8h\n" + "ld1r { v24.8h }, [x20]\n" + "fmin v8.8h, v8.8h, v25.8h\n" + "fmin v9.8h, v9.8h, v25.8h\n" + "fmin v10.8h, v10.8h, v25.8h\n" + "fmin v11.8h, v11.8h, v25.8h\n" + "fmin v12.8h, v12.8h, v25.8h\n" + "fmin v13.8h, v13.8h, v25.8h\n" + "fmin v14.8h, v14.8h, v25.8h\n" + "fmin v15.8h, v15.8h, v25.8h\n" + "fmin v16.8h, v16.8h, v25.8h\n" + "fmin v17.8h, v17.8h, v25.8h\n" + "fmin v18.8h, v18.8h, v25.8h\n" + "fmin v19.8h, v19.8h, v25.8h\n" + "fmin v20.8h, v20.8h, v25.8h\n" + "fmin v21.8h, v21.8h, v25.8h\n" + "fmin v22.8h, v22.8h, v25.8h\n" + "fmin v23.8h, v23.8h, v25.8h\n" + "fmax v8.8h, v8.8h, v24.8h\n" + "fmax v9.8h, v9.8h, v24.8h\n" + "fmax v10.8h, v10.8h, v24.8h\n" + "fmax v11.8h, v11.8h, v24.8h\n" + "fmax v12.8h, v12.8h, v24.8h\n" + "fmax v13.8h, v13.8h, v24.8h\n" + "fmax v14.8h, v14.8h, v24.8h\n" + "fmax v15.8h, v15.8h, v24.8h\n" + "fmax v16.8h, v16.8h, v24.8h\n" + "fmax v17.8h, v17.8h, v24.8h\n" + "fmax v18.8h, v18.8h, v24.8h\n" + "fmax v19.8h, v19.8h, v24.8h\n" + "fmax v20.8h, v20.8h, v24.8h\n" + "fmax v21.8h, v21.8h, v24.8h\n" + "fmax v22.8h, v22.8h, v24.8h\n" + "fmax v23.8h, v23.8h, v24.8h\n" "178:" // Height 4: No activation "cmp x11, #0x20\n" "bge 195f\n" @@ -3124,583 +3124,583 @@ void a64_hybrid_fp16_mla_6x32 ( "tbz x11, #1, 212f\n" "ld1 { v8.s }[2], [x9], #0x4\n" "ld1 { v12.s }[2], [x25], #0x4\n" - "mov x20, #0xc\n" - "ld1 { v16.s }[2], [x24], #0x4\n" - "ld1 { v20.s }[2], [x23], #0x4\n" - "ld1 { v24.s }[2], [x22], #0x4\n" - "tbz x11, #0, 215f\n" - "ld1 { v8.h }[6], [x9]\n" - "ld1 { v12.h }[6], [x25]\n" - "ld1 { v16.h }[6], [x24]\n" - "ld1 { v20.h }[6], [x23]\n" - "ld1 { v24.h }[6], [x22]\n" - "b 215f\n" - "212:" // Height 5: Partial accumulate: partial_1_4 - "mov x20, #0x8\n" - "tbz x11, #0, 215f\n" - "ld1 { v8.h }[4], [x9]\n" - "ld1 { v12.h }[4], [x25]\n" - "ld1 { v16.h }[4], [x24]\n" - "ld1 { v20.h }[4], [x23]\n" - "ld1 { v24.h }[4], [x22]\n" - "b 215f\n" - "213:" // Height 5: Partial accumulate: partial_2_0 - "tbz x11, #1, 214f\n" - "ldr s8, [x9], #0x4\n" - "ldr s12, [x25], #0x4\n" - "mov x20, #0x4\n" - "ldr s16, [x24], #0x4\n" - "ldr s20, [x23], #0x4\n" - "ldr s24, [x22], #0x4\n" - "tbz x11, #0, 215f\n" - "ld1 { v8.h }[2], [x9]\n" - "ld1 { v12.h }[2], [x25]\n" - "ld1 { v16.h }[2], [x24]\n" - "ld1 { v20.h }[2], [x23]\n" - "ld1 { v24.h }[2], [x22]\n" - "b 215f\n" - "214:" // Height 5: Partial accumulate: partial_1_0 - "ldr h8, [x9, #0x0]\n" - "ldr h12, [x25, #0x0]\n" - "mov x20, #0x0\n" - "ldr h16, [x24, #0x0]\n" - "ldr h20, [x23, #0x0]\n" - "ldr h24, [x22, #0x0]\n" - "215:" // Height 5: Partial accumulate: Done - "sub x9, x9, x20\n" - "b 218f\n" - "216:" // Height 5: full accumulate - "ldr q8, [x9, #0x0]\n" - "ldr q9, [x9, #0x10]\n" - "ldr q10, [x9, #0x20]\n" - "ldr q11, [x9, #0x30]\n" - "ldr q12, [x25, #0x0]\n" - "ldr q13, [x25, #0x10]\n" - "ldr q14, [x25, #0x20]\n" - "ldr q15, [x25, #0x30]\n" - "ldr q16, [x24, #0x0]\n" - "ldr q17, [x24, #0x10]\n" - "ldr q18, [x24, #0x20]\n" - "ldr q19, [x24, #0x30]\n" - "ldr q20, [x23, #0x0]\n" - "ldr q21, [x23, #0x10]\n" - "ldr q22, [x23, #0x20]\n" - "ldr q23, [x23, #0x30]\n" - "ldr q24, [x22, #0x0]\n" - "ldr q25, [x22, #0x10]\n" - "ldr q26, [x22, #0x20]\n" - "ldr q27, [x22, #0x30]\n" - "b 218f\n" - "217:" // Height 5: no accumulate - "movi v8.16b, #0x0\n" - "movi v9.16b, #0x0\n" - "movi v10.16b, #0x0\n" - "movi v11.16b, #0x0\n" - "movi v12.16b, #0x0\n" - "movi v13.16b, #0x0\n" - "movi v14.16b, #0x0\n" - "movi v15.16b, #0x0\n" - "movi v16.16b, #0x0\n" - "movi v17.16b, #0x0\n" - "movi v18.16b, #0x0\n" - "movi v19.16b, #0x0\n" - "movi v20.16b, #0x0\n" - "movi v21.16b, #0x0\n" - "movi v22.16b, #0x0\n" - "movi v23.16b, #0x0\n" - "movi v24.16b, #0x0\n" - "movi v25.16b, #0x0\n" - "movi v26.16b, #0x0\n" - "movi v27.16b, #0x0\n" - "218:" // Height 5: setup done - "mov x28, #0x0\n" - "219:" // Height 5: String loop - "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" - "tbz %x[flags], #3, 220f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "cbnz x28, 221f\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x26, x26, x20, LSL #1\n" - "add x25, x25, x20, LSL #1\n" - "add x24, x24, x20, LSL #1\n" - "add x23, x23, x20, LSL #1\n" - "add x22, x22, x20, LSL #1\n" - "b 221f\n" - "220:" // Height 5: setup direct input - "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" - "add x22, x23, x20, LSL #1\n" - "221:" // Height 5: input setup done - "cmp x27, #0x8\n" - "blt 224f\n" - "ldr q0, [x26, #0x0]\n" - "ldr q1, [x25, #0x0]\n" - "cmp x27, #0x10\n" - "ldr q2, [x24, #0x0]\n" - "ldr q3, [x23, #0x0]\n" - "ldr q4, [x22, #0x0]\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - "blt 223f\n" - "222:" // Height 5: Multiply loop: Main loop head - "fmla v8.8h, v6.8h, v0.h[0]\n" - "fmla v12.8h, v6.8h, v1.h[0]\n" - "sub x27, x27, #0x8\n" - "add x26, x26, #0x10\n" - "fmla v16.8h, v6.8h, v2.h[0]\n" - "fmla v20.8h, v6.8h, v3.h[0]\n" - "add x25, x25, #0x10\n" - "add x24, x24, #0x10\n" - "fmla v24.8h, v6.8h, v4.h[0]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.8h, v7.8h, v0.h[0]\n" - "add x23, x23, #0x10\n" - "fmla v13.8h, v7.8h, v1.h[0]\n" - "fmla v17.8h, v7.8h, v2.h[0]\n" - "add x22, x22, #0x10\n" - "cmp x27, #0x10\n" - "fmla v21.8h, v7.8h, v3.h[0]\n" - "fmla v25.8h, v7.8h, v4.h[0]\n" - "ldr q7, [x10, #0x30]\n" - "prfm pldl1keep, [x26, #0x80]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "prfm pldl1keep, [x25, #0x80]\n" - "prfm pldl1keep, [x24, #0x80]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "fmla v22.8h, v6.8h, v3.h[0]\n" - "prfm pldl1keep, [x23, #0x80]\n" - "prfm pldl1keep, [x22, #0x80]\n" - "fmla v26.8h, v6.8h, v4.h[0]\n" - "ldr q6, [x10, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "fmla v23.8h, v7.8h, v3.h[0]\n" - "fmla v27.8h, v7.8h, v4.h[0]\n" - "ldr q7, [x10, #0x50]\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "fmla v16.8h, v6.8h, v2.h[1]\n" - "fmla v20.8h, v6.8h, v3.h[1]\n" - "fmla v24.8h, v6.8h, v4.h[1]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "fmla v17.8h, v7.8h, v2.h[1]\n" - "fmla v21.8h, v7.8h, v3.h[1]\n" - "fmla v25.8h, v7.8h, v4.h[1]\n" - "ldr q7, [x10, #0x70]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "fmla v14.8h, v6.8h, v1.h[1]\n" - "fmla v18.8h, v6.8h, v2.h[1]\n" - "fmla v22.8h, v6.8h, v3.h[1]\n" - "fmla v26.8h, v6.8h, v4.h[1]\n" - "ldr q6, [x10, #0x80]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "fmla v15.8h, v7.8h, v1.h[1]\n" - "fmla v19.8h, v7.8h, v2.h[1]\n" - "fmla v23.8h, v7.8h, v3.h[1]\n" - "fmla v27.8h, v7.8h, v4.h[1]\n" - "ldr q7, [x10, #0x90]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "fmla v12.8h, v6.8h, v1.h[2]\n" - "fmla v16.8h, v6.8h, v2.h[2]\n" - "fmla v20.8h, v6.8h, v3.h[2]\n" - "fmla v24.8h, v6.8h, v4.h[2]\n" - "ldr q6, [x10, #0xa0]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "fmla v13.8h, v7.8h, v1.h[2]\n" - "fmla v17.8h, v7.8h, v2.h[2]\n" - "fmla v21.8h, v7.8h, v3.h[2]\n" - "fmla v25.8h, v7.8h, v4.h[2]\n" - "ldr q7, [x10, #0xb0]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "fmla v14.8h, v6.8h, v1.h[2]\n" - "fmla v18.8h, v6.8h, v2.h[2]\n" - "fmla v22.8h, v6.8h, v3.h[2]\n" - "fmla v26.8h, v6.8h, v4.h[2]\n" - "ldr q6, [x10, #0xc0]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "fmla v15.8h, v7.8h, v1.h[2]\n" - "fmla v19.8h, v7.8h, v2.h[2]\n" - "fmla v23.8h, v7.8h, v3.h[2]\n" - "fmla v27.8h, v7.8h, v4.h[2]\n" - "ldr q7, [x10, #0xd0]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "fmla v12.8h, v6.8h, v1.h[3]\n" - "fmla v16.8h, v6.8h, v2.h[3]\n" - "fmla v20.8h, v6.8h, v3.h[3]\n" - "fmla v24.8h, v6.8h, v4.h[3]\n" - "ldr q6, [x10, #0xe0]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "fmla v13.8h, v7.8h, v1.h[3]\n" - "fmla v17.8h, v7.8h, v2.h[3]\n" - "fmla v21.8h, v7.8h, v3.h[3]\n" - "fmla v25.8h, v7.8h, v4.h[3]\n" - "ldr q7, [x10, #0xf0]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "fmla v14.8h, v6.8h, v1.h[3]\n" - "fmla v18.8h, v6.8h, v2.h[3]\n" - "fmla v22.8h, v6.8h, v3.h[3]\n" - "fmla v26.8h, v6.8h, v4.h[3]\n" - "ldr q6, [x10, #0x100]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "fmla v15.8h, v7.8h, v1.h[3]\n" - "fmla v19.8h, v7.8h, v2.h[3]\n" - "fmla v23.8h, v7.8h, v3.h[3]\n" - "fmla v27.8h, v7.8h, v4.h[3]\n" - "ldr q7, [x10, #0x110]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "fmla v12.8h, v6.8h, v1.h[4]\n" - "fmla v16.8h, v6.8h, v2.h[4]\n" - "fmla v20.8h, v6.8h, v3.h[4]\n" - "fmla v24.8h, v6.8h, v4.h[4]\n" - "ldr q6, [x10, #0x120]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "fmla v13.8h, v7.8h, v1.h[4]\n" - "fmla v17.8h, v7.8h, v2.h[4]\n" - "fmla v21.8h, v7.8h, v3.h[4]\n" - "fmla v25.8h, v7.8h, v4.h[4]\n" - "ldr q7, [x10, #0x130]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "fmla v14.8h, v6.8h, v1.h[4]\n" - "fmla v18.8h, v6.8h, v2.h[4]\n" - "fmla v22.8h, v6.8h, v3.h[4]\n" - "fmla v26.8h, v6.8h, v4.h[4]\n" - "ldr q6, [x10, #0x140]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "fmla v15.8h, v7.8h, v1.h[4]\n" - "fmla v19.8h, v7.8h, v2.h[4]\n" - "fmla v23.8h, v7.8h, v3.h[4]\n" - "fmla v27.8h, v7.8h, v4.h[4]\n" - "ldr q7, [x10, #0x150]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "fmla v12.8h, v6.8h, v1.h[5]\n" - "fmla v16.8h, v6.8h, v2.h[5]\n" - "fmla v20.8h, v6.8h, v3.h[5]\n" - "fmla v24.8h, v6.8h, v4.h[5]\n" - "ldr q6, [x10, #0x160]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "fmla v13.8h, v7.8h, v1.h[5]\n" - "fmla v17.8h, v7.8h, v2.h[5]\n" - "fmla v21.8h, v7.8h, v3.h[5]\n" - "fmla v25.8h, v7.8h, v4.h[5]\n" - "ldr q7, [x10, #0x170]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "fmla v14.8h, v6.8h, v1.h[5]\n" - "fmla v18.8h, v6.8h, v2.h[5]\n" - "fmla v22.8h, v6.8h, v3.h[5]\n" - "fmla v26.8h, v6.8h, v4.h[5]\n" - "ldr q6, [x10, #0x180]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "fmla v15.8h, v7.8h, v1.h[5]\n" - "fmla v19.8h, v7.8h, v2.h[5]\n" - "fmla v23.8h, v7.8h, v3.h[5]\n" - "fmla v27.8h, v7.8h, v4.h[5]\n" - "ldr q7, [x10, #0x190]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "fmla v12.8h, v6.8h, v1.h[6]\n" - "fmla v16.8h, v6.8h, v2.h[6]\n" - "fmla v20.8h, v6.8h, v3.h[6]\n" - "fmla v24.8h, v6.8h, v4.h[6]\n" - "ldr q6, [x10, #0x1a0]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "fmla v13.8h, v7.8h, v1.h[6]\n" - "fmla v17.8h, v7.8h, v2.h[6]\n" - "fmla v21.8h, v7.8h, v3.h[6]\n" - "fmla v25.8h, v7.8h, v4.h[6]\n" - "ldr q7, [x10, #0x1b0]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "fmla v14.8h, v6.8h, v1.h[6]\n" - "fmla v18.8h, v6.8h, v2.h[6]\n" - "fmla v22.8h, v6.8h, v3.h[6]\n" - "fmla v26.8h, v6.8h, v4.h[6]\n" - "ldr q6, [x10, #0x1c0]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "fmla v15.8h, v7.8h, v1.h[6]\n" - "fmla v19.8h, v7.8h, v2.h[6]\n" - "fmla v23.8h, v7.8h, v3.h[6]\n" - "fmla v27.8h, v7.8h, v4.h[6]\n" - "ldr q7, [x10, #0x1d0]\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "fmla v12.8h, v6.8h, v1.h[7]\n" - "fmla v16.8h, v6.8h, v2.h[7]\n" - "fmla v20.8h, v6.8h, v3.h[7]\n" - "fmla v24.8h, v6.8h, v4.h[7]\n" - "ldr q6, [x10, #0x1e0]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "fmla v13.8h, v7.8h, v1.h[7]\n" - "fmla v17.8h, v7.8h, v2.h[7]\n" - "fmla v21.8h, v7.8h, v3.h[7]\n" - "fmla v25.8h, v7.8h, v4.h[7]\n" - "ldr q7, [x10, #0x1f0]\n" - "add x10, x10, #0x200\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "fmla v14.8h, v6.8h, v1.h[7]\n" - "fmla v18.8h, v6.8h, v2.h[7]\n" - "fmla v22.8h, v6.8h, v3.h[7]\n" - "fmla v26.8h, v6.8h, v4.h[7]\n" - "ldr q6, [x10, #0x0]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" - "ldr q0, [x26, #0x0]\n" - "fmla v15.8h, v7.8h, v1.h[7]\n" - "ldr q1, [x25, #0x0]\n" - "fmla v19.8h, v7.8h, v2.h[7]\n" - "ldr q2, [x24, #0x0]\n" - "fmla v23.8h, v7.8h, v3.h[7]\n" - "ldr q3, [x23, #0x0]\n" - "fmla v27.8h, v7.8h, v4.h[7]\n" - "ldr q4, [x22, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - "bge 222b\n" - "223:" // Height 5: Multiply loop: Single iteration only - "fmla v8.8h, v6.8h, v0.h[0]\n" - "fmla v12.8h, v6.8h, v1.h[0]\n" - "add x26, x26, #0x10\n" - "add x25, x25, #0x10\n" - "fmla v16.8h, v6.8h, v2.h[0]\n" - "fmla v20.8h, v6.8h, v3.h[0]\n" - "add x24, x24, #0x10\n" - "add x23, x23, #0x10\n" - "fmla v24.8h, v6.8h, v4.h[0]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.8h, v7.8h, v0.h[0]\n" - "add x22, x22, #0x10\n" - "fmla v13.8h, v7.8h, v1.h[0]\n" - "fmla v17.8h, v7.8h, v2.h[0]\n" - "sub x27, x27, #0x8\n" - "prfm pldl1keep, [x26, #0x80]\n" - "fmla v21.8h, v7.8h, v3.h[0]\n" - "fmla v25.8h, v7.8h, v4.h[0]\n" - "ldr q7, [x10, #0x30]\n" - "prfm pldl1keep, [x25, #0x80]\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "prfm pldl1keep, [x24, #0x80]\n" - "prfm pldl1keep, [x23, #0x80]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "fmla v22.8h, v6.8h, v3.h[0]\n" - "prfm pldl1keep, [x22, #0x80]\n" - "fmla v26.8h, v6.8h, v4.h[0]\n" - "ldr q6, [x10, #0x40]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "fmla v23.8h, v7.8h, v3.h[0]\n" - "fmla v27.8h, v7.8h, v4.h[0]\n" - "ldr q7, [x10, #0x50]\n" - "fmla v8.8h, v6.8h, v0.h[1]\n" - "fmla v12.8h, v6.8h, v1.h[1]\n" - "fmla v16.8h, v6.8h, v2.h[1]\n" - "fmla v20.8h, v6.8h, v3.h[1]\n" - "fmla v24.8h, v6.8h, v4.h[1]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.8h, v7.8h, v0.h[1]\n" - "fmla v13.8h, v7.8h, v1.h[1]\n" - "fmla v17.8h, v7.8h, v2.h[1]\n" - "fmla v21.8h, v7.8h, v3.h[1]\n" - "fmla v25.8h, v7.8h, v4.h[1]\n" - "ldr q7, [x10, #0x70]\n" - "fmla v10.8h, v6.8h, v0.h[1]\n" - "fmla v14.8h, v6.8h, v1.h[1]\n" - "fmla v18.8h, v6.8h, v2.h[1]\n" - "fmla v22.8h, v6.8h, v3.h[1]\n" - "fmla v26.8h, v6.8h, v4.h[1]\n" - "ldr q6, [x10, #0x80]\n" - "fmla v11.8h, v7.8h, v0.h[1]\n" - "fmla v15.8h, v7.8h, v1.h[1]\n" - "fmla v19.8h, v7.8h, v2.h[1]\n" - "fmla v23.8h, v7.8h, v3.h[1]\n" - "fmla v27.8h, v7.8h, v4.h[1]\n" - "ldr q7, [x10, #0x90]\n" - "fmla v8.8h, v6.8h, v0.h[2]\n" - "fmla v12.8h, v6.8h, v1.h[2]\n" - "fmla v16.8h, v6.8h, v2.h[2]\n" - "fmla v20.8h, v6.8h, v3.h[2]\n" - "fmla v24.8h, v6.8h, v4.h[2]\n" - "ldr q6, [x10, #0xa0]\n" - "fmla v9.8h, v7.8h, v0.h[2]\n" - "fmla v13.8h, v7.8h, v1.h[2]\n" - "fmla v17.8h, v7.8h, v2.h[2]\n" - "fmla v21.8h, v7.8h, v3.h[2]\n" - "fmla v25.8h, v7.8h, v4.h[2]\n" - "ldr q7, [x10, #0xb0]\n" - "fmla v10.8h, v6.8h, v0.h[2]\n" - "fmla v14.8h, v6.8h, v1.h[2]\n" - "fmla v18.8h, v6.8h, v2.h[2]\n" - "fmla v22.8h, v6.8h, v3.h[2]\n" - "fmla v26.8h, v6.8h, v4.h[2]\n" - "ldr q6, [x10, #0xc0]\n" - "fmla v11.8h, v7.8h, v0.h[2]\n" - "fmla v15.8h, v7.8h, v1.h[2]\n" - "fmla v19.8h, v7.8h, v2.h[2]\n" - "fmla v23.8h, v7.8h, v3.h[2]\n" - "fmla v27.8h, v7.8h, v4.h[2]\n" - "ldr q7, [x10, #0xd0]\n" - "fmla v8.8h, v6.8h, v0.h[3]\n" - "fmla v12.8h, v6.8h, v1.h[3]\n" - "fmla v16.8h, v6.8h, v2.h[3]\n" - "fmla v20.8h, v6.8h, v3.h[3]\n" - "fmla v24.8h, v6.8h, v4.h[3]\n" - "ldr q6, [x10, #0xe0]\n" - "fmla v9.8h, v7.8h, v0.h[3]\n" - "fmla v13.8h, v7.8h, v1.h[3]\n" - "fmla v17.8h, v7.8h, v2.h[3]\n" - "fmla v21.8h, v7.8h, v3.h[3]\n" - "fmla v25.8h, v7.8h, v4.h[3]\n" - "ldr q7, [x10, #0xf0]\n" - "fmla v10.8h, v6.8h, v0.h[3]\n" - "fmla v14.8h, v6.8h, v1.h[3]\n" - "fmla v18.8h, v6.8h, v2.h[3]\n" - "fmla v22.8h, v6.8h, v3.h[3]\n" - "fmla v26.8h, v6.8h, v4.h[3]\n" - "ldr q6, [x10, #0x100]\n" - "fmla v11.8h, v7.8h, v0.h[3]\n" - "fmla v15.8h, v7.8h, v1.h[3]\n" - "fmla v19.8h, v7.8h, v2.h[3]\n" - "fmla v23.8h, v7.8h, v3.h[3]\n" - "fmla v27.8h, v7.8h, v4.h[3]\n" - "ldr q7, [x10, #0x110]\n" - "fmla v8.8h, v6.8h, v0.h[4]\n" - "fmla v12.8h, v6.8h, v1.h[4]\n" - "fmla v16.8h, v6.8h, v2.h[4]\n" - "fmla v20.8h, v6.8h, v3.h[4]\n" - "fmla v24.8h, v6.8h, v4.h[4]\n" - "ldr q6, [x10, #0x120]\n" - "fmla v9.8h, v7.8h, v0.h[4]\n" - "fmla v13.8h, v7.8h, v1.h[4]\n" - "fmla v17.8h, v7.8h, v2.h[4]\n" - "fmla v21.8h, v7.8h, v3.h[4]\n" - "fmla v25.8h, v7.8h, v4.h[4]\n" - "ldr q7, [x10, #0x130]\n" - "fmla v10.8h, v6.8h, v0.h[4]\n" - "fmla v14.8h, v6.8h, v1.h[4]\n" - "fmla v18.8h, v6.8h, v2.h[4]\n" - "fmla v22.8h, v6.8h, v3.h[4]\n" - "fmla v26.8h, v6.8h, v4.h[4]\n" - "ldr q6, [x10, #0x140]\n" - "fmla v11.8h, v7.8h, v0.h[4]\n" - "fmla v15.8h, v7.8h, v1.h[4]\n" - "fmla v19.8h, v7.8h, v2.h[4]\n" - "fmla v23.8h, v7.8h, v3.h[4]\n" - "fmla v27.8h, v7.8h, v4.h[4]\n" - "ldr q7, [x10, #0x150]\n" - "fmla v8.8h, v6.8h, v0.h[5]\n" - "fmla v12.8h, v6.8h, v1.h[5]\n" - "fmla v16.8h, v6.8h, v2.h[5]\n" - "fmla v20.8h, v6.8h, v3.h[5]\n" - "fmla v24.8h, v6.8h, v4.h[5]\n" - "ldr q6, [x10, #0x160]\n" - "fmla v9.8h, v7.8h, v0.h[5]\n" - "fmla v13.8h, v7.8h, v1.h[5]\n" - "fmla v17.8h, v7.8h, v2.h[5]\n" - "fmla v21.8h, v7.8h, v3.h[5]\n" - "fmla v25.8h, v7.8h, v4.h[5]\n" - "ldr q7, [x10, #0x170]\n" - "fmla v10.8h, v6.8h, v0.h[5]\n" - "fmla v14.8h, v6.8h, v1.h[5]\n" - "fmla v18.8h, v6.8h, v2.h[5]\n" - "fmla v22.8h, v6.8h, v3.h[5]\n" - "fmla v26.8h, v6.8h, v4.h[5]\n" - "ldr q6, [x10, #0x180]\n" - "fmla v11.8h, v7.8h, v0.h[5]\n" - "fmla v15.8h, v7.8h, v1.h[5]\n" - "fmla v19.8h, v7.8h, v2.h[5]\n" - "fmla v23.8h, v7.8h, v3.h[5]\n" - "fmla v27.8h, v7.8h, v4.h[5]\n" - "ldr q7, [x10, #0x190]\n" - "fmla v8.8h, v6.8h, v0.h[6]\n" - "fmla v12.8h, v6.8h, v1.h[6]\n" - "fmla v16.8h, v6.8h, v2.h[6]\n" - "fmla v20.8h, v6.8h, v3.h[6]\n" - "fmla v24.8h, v6.8h, v4.h[6]\n" - "ldr q6, [x10, #0x1a0]\n" - "fmla v9.8h, v7.8h, v0.h[6]\n" - "fmla v13.8h, v7.8h, v1.h[6]\n" - "fmla v17.8h, v7.8h, v2.h[6]\n" - "fmla v21.8h, v7.8h, v3.h[6]\n" - "fmla v25.8h, v7.8h, v4.h[6]\n" - "ldr q7, [x10, #0x1b0]\n" - "fmla v10.8h, v6.8h, v0.h[6]\n" - "fmla v14.8h, v6.8h, v1.h[6]\n" - "fmla v18.8h, v6.8h, v2.h[6]\n" - "fmla v22.8h, v6.8h, v3.h[6]\n" - "fmla v26.8h, v6.8h, v4.h[6]\n" - "ldr q6, [x10, #0x1c0]\n" - "fmla v11.8h, v7.8h, v0.h[6]\n" - "fmla v15.8h, v7.8h, v1.h[6]\n" - "fmla v19.8h, v7.8h, v2.h[6]\n" - "fmla v23.8h, v7.8h, v3.h[6]\n" - "fmla v27.8h, v7.8h, v4.h[6]\n" - "ldr q7, [x10, #0x1d0]\n" - "fmla v8.8h, v6.8h, v0.h[7]\n" - "fmla v12.8h, v6.8h, v1.h[7]\n" - "fmla v16.8h, v6.8h, v2.h[7]\n" - "fmla v20.8h, v6.8h, v3.h[7]\n" - "fmla v24.8h, v6.8h, v4.h[7]\n" - "ldr q6, [x10, #0x1e0]\n" - "fmla v9.8h, v7.8h, v0.h[7]\n" - "fmla v13.8h, v7.8h, v1.h[7]\n" - "fmla v17.8h, v7.8h, v2.h[7]\n" - "fmla v21.8h, v7.8h, v3.h[7]\n" - "fmla v25.8h, v7.8h, v4.h[7]\n" - "ldr q7, [x10, #0x1f0]\n" - "add x10, x10, #0x200\n" - "fmla v10.8h, v6.8h, v0.h[7]\n" - "fmla v14.8h, v6.8h, v1.h[7]\n" - "fmla v18.8h, v6.8h, v2.h[7]\n" - "fmla v22.8h, v6.8h, v3.h[7]\n" - "fmla v26.8h, v6.8h, v4.h[7]\n" - "fmla v11.8h, v7.8h, v0.h[7]\n" - "fmla v15.8h, v7.8h, v1.h[7]\n" - "fmla v19.8h, v7.8h, v2.h[7]\n" - "fmla v23.8h, v7.8h, v3.h[7]\n" - "fmla v27.8h, v7.8h, v4.h[7]\n" - "224:" // Height 5: Multiply loop: Main loop skip - "cbz x27, 226f\n" - "225:" // Height 5: Multiply loop: Odd block loop - "ldr h0, [x26], #0x2\n" - "ldr h1, [x25], #0x2\n" - "sub x27, x27, #0x1\n" - "ldr h2, [x24], #0x2\n" - "ldr h3, [x23], #0x2\n" - "ldr h4, [x22], #0x2\n" + "mov x20, #0xc\n" + "ld1 { v16.s }[2], [x24], #0x4\n" + "ld1 { v20.s }[2], [x23], #0x4\n" + "ld1 { v24.s }[2], [x22], #0x4\n" + "tbz x11, #0, 215f\n" + "ld1 { v8.h }[6], [x9]\n" + "ld1 { v12.h }[6], [x25]\n" + "ld1 { v16.h }[6], [x24]\n" + "ld1 { v20.h }[6], [x23]\n" + "ld1 { v24.h }[6], [x22]\n" + "b 215f\n" + "212:" // Height 5: Partial accumulate: partial_1_4 + "mov x20, #0x8\n" + "tbz x11, #0, 215f\n" + "ld1 { v8.h }[4], [x9]\n" + "ld1 { v12.h }[4], [x25]\n" + "ld1 { v16.h }[4], [x24]\n" + "ld1 { v20.h }[4], [x23]\n" + "ld1 { v24.h }[4], [x22]\n" + "b 215f\n" + "213:" // Height 5: Partial accumulate: partial_2_0 + "tbz x11, #1, 214f\n" + "ldr s8, [x9], #0x4\n" + "ldr s12, [x25], #0x4\n" + "mov x20, #0x4\n" + "ldr s16, [x24], #0x4\n" + "ldr s20, [x23], #0x4\n" + "ldr s24, [x22], #0x4\n" + "tbz x11, #0, 215f\n" + "ld1 { v8.h }[2], [x9]\n" + "ld1 { v12.h }[2], [x25]\n" + "ld1 { v16.h }[2], [x24]\n" + "ld1 { v20.h }[2], [x23]\n" + "ld1 { v24.h }[2], [x22]\n" + "b 215f\n" + "214:" // Height 5: Partial accumulate: partial_1_0 + "ldr h8, [x9, #0x0]\n" + "ldr h12, [x25, #0x0]\n" + "mov x20, #0x0\n" + "ldr h16, [x24, #0x0]\n" + "ldr h20, [x23, #0x0]\n" + "ldr h24, [x22, #0x0]\n" + "215:" // Height 5: Partial accumulate: Done + "sub x9, x9, x20\n" + "b 218f\n" + "216:" // Height 5: full accumulate + "ldr q8, [x9, #0x0]\n" + "ldr q9, [x9, #0x10]\n" + "ldr q10, [x9, #0x20]\n" + "ldr q11, [x9, #0x30]\n" + "ldr q12, [x25, #0x0]\n" + "ldr q13, [x25, #0x10]\n" + "ldr q14, [x25, #0x20]\n" + "ldr q15, [x25, #0x30]\n" + "ldr q16, [x24, #0x0]\n" + "ldr q17, [x24, #0x10]\n" + "ldr q18, [x24, #0x20]\n" + "ldr q19, [x24, #0x30]\n" + "ldr q20, [x23, #0x0]\n" + "ldr q21, [x23, #0x10]\n" + "ldr q22, [x23, #0x20]\n" + "ldr q23, [x23, #0x30]\n" + "ldr q24, [x22, #0x0]\n" + "ldr q25, [x22, #0x10]\n" + "ldr q26, [x22, #0x20]\n" + "ldr q27, [x22, #0x30]\n" + "b 218f\n" + "217:" // Height 5: no accumulate + "movi v8.16b, #0x0\n" + "movi v9.16b, #0x0\n" + "movi v10.16b, #0x0\n" + "movi v11.16b, #0x0\n" + "movi v12.16b, #0x0\n" + "movi v13.16b, #0x0\n" + "movi v14.16b, #0x0\n" + "movi v15.16b, #0x0\n" + "movi v16.16b, #0x0\n" + "movi v17.16b, #0x0\n" + "movi v18.16b, #0x0\n" + "movi v19.16b, #0x0\n" + "movi v20.16b, #0x0\n" + "movi v21.16b, #0x0\n" + "movi v22.16b, #0x0\n" + "movi v23.16b, #0x0\n" + "movi v24.16b, #0x0\n" + "movi v25.16b, #0x0\n" + "movi v26.16b, #0x0\n" + "movi v27.16b, #0x0\n" + "218:" // Height 5: setup done + "mov x28, #0x0\n" + "219:" // Height 5: String loop + "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" + "ldr w27, [x20, x28, LSL #0x2]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" + "tbz %x[flags], #3, 220f\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "cbnz x28, 221f\n" + "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" + "add x26, x26, x20, LSL #1\n" + "add x25, x25, x20, LSL #1\n" + "add x24, x24, x20, LSL #1\n" + "add x23, x23, x20, LSL #1\n" + "add x22, x22, x20, LSL #1\n" + "b 221f\n" + "220:" // Height 5: setup direct input + "mov x26, %x[input_ptr]\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" + "add x22, x23, x21, LSL #1\n" + "221:" // Height 5: input setup done + "cmp x27, #0x8\n" + "blt 224f\n" + "ldr q0, [x26, #0x0]\n" + "ldr q1, [x25, #0x0]\n" + "cmp x27, #0x10\n" + "ldr q2, [x24, #0x0]\n" + "ldr q3, [x23, #0x0]\n" + "ldr q4, [x22, #0x0]\n" "ldr q6, [x10, #0x0]\n" + "ldr q7, [x10, #0x10]\n" + "blt 223f\n" + "222:" // Height 5: Multiply loop: Main loop head "fmla v8.8h, v6.8h, v0.h[0]\n" "fmla v12.8h, v6.8h, v1.h[0]\n" + "sub x27, x27, #0x8\n" + "add x26, x26, #0x10\n" + "fmla v16.8h, v6.8h, v2.h[0]\n" + "fmla v20.8h, v6.8h, v3.h[0]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + "fmla v24.8h, v6.8h, v4.h[0]\n" + "ldr q29, [x10, #0x20]\n" + "fmla v9.8h, v7.8h, v0.h[0]\n" + "add x23, x23, #0x10\n" + "fmla v13.8h, v7.8h, v1.h[0]\n" + "fmla v17.8h, v7.8h, v2.h[0]\n" + "add x22, x22, #0x10\n" + "cmp x27, #0x10\n" + "fmla v21.8h, v7.8h, v3.h[0]\n" + "fmla v25.8h, v7.8h, v4.h[0]\n" + "ldr q28, [x10, #0x30]\n" + "prfm pldl1keep, [x26, #0x80]\n" + "fmla v10.8h, v29.8h, v0.h[0]\n" + "fmla v14.8h, v29.8h, v1.h[0]\n" + "prfm pldl1keep, [x25, #0x80]\n" + "prfm pldl1keep, [x24, #0x80]\n" + "fmla v18.8h, v29.8h, v2.h[0]\n" + "fmla v22.8h, v29.8h, v3.h[0]\n" + "prfm pldl1keep, [x23, #0x80]\n" + "prfm pldl1keep, [x22, #0x80]\n" + "fmla v26.8h, v29.8h, v4.h[0]\n" + "ldr q29, [x10, #0x40]\n" + "fmla v11.8h, v28.8h, v0.h[0]\n" + "fmla v15.8h, v28.8h, v1.h[0]\n" + "fmla v19.8h, v28.8h, v2.h[0]\n" + "fmla v23.8h, v28.8h, v3.h[0]\n" + "fmla v27.8h, v28.8h, v4.h[0]\n" + "ldr q28, [x10, #0x50]\n" + "fmla v8.8h, v29.8h, v0.h[1]\n" + "fmla v12.8h, v29.8h, v1.h[1]\n" + "fmla v16.8h, v29.8h, v2.h[1]\n" + "fmla v20.8h, v29.8h, v3.h[1]\n" + "fmla v24.8h, v29.8h, v4.h[1]\n" + "ldr q29, [x10, #0x60]\n" + "fmla v9.8h, v28.8h, v0.h[1]\n" + "fmla v13.8h, v28.8h, v1.h[1]\n" + "fmla v17.8h, v28.8h, v2.h[1]\n" + "fmla v21.8h, v28.8h, v3.h[1]\n" + "fmla v25.8h, v28.8h, v4.h[1]\n" + "ldr q28, [x10, #0x70]\n" + "fmla v10.8h, v29.8h, v0.h[1]\n" + "fmla v14.8h, v29.8h, v1.h[1]\n" + "fmla v18.8h, v29.8h, v2.h[1]\n" + "fmla v22.8h, v29.8h, v3.h[1]\n" + "fmla v26.8h, v29.8h, v4.h[1]\n" + "ldr q29, [x10, #0x80]\n" + "fmla v11.8h, v28.8h, v0.h[1]\n" + "fmla v15.8h, v28.8h, v1.h[1]\n" + "fmla v19.8h, v28.8h, v2.h[1]\n" + "fmla v23.8h, v28.8h, v3.h[1]\n" + "fmla v27.8h, v28.8h, v4.h[1]\n" + "ldr q28, [x10, #0x90]\n" + "fmla v8.8h, v29.8h, v0.h[2]\n" + "fmla v12.8h, v29.8h, v1.h[2]\n" + "fmla v16.8h, v29.8h, v2.h[2]\n" + "fmla v20.8h, v29.8h, v3.h[2]\n" + "fmla v24.8h, v29.8h, v4.h[2]\n" + "ldr q29, [x10, #0xa0]\n" + "fmla v9.8h, v28.8h, v0.h[2]\n" + "fmla v13.8h, v28.8h, v1.h[2]\n" + "fmla v17.8h, v28.8h, v2.h[2]\n" + "fmla v21.8h, v28.8h, v3.h[2]\n" + "fmla v25.8h, v28.8h, v4.h[2]\n" + "ldr q28, [x10, #0xb0]\n" + "fmla v10.8h, v29.8h, v0.h[2]\n" + "fmla v14.8h, v29.8h, v1.h[2]\n" + "fmla v18.8h, v29.8h, v2.h[2]\n" + "fmla v22.8h, v29.8h, v3.h[2]\n" + "fmla v26.8h, v29.8h, v4.h[2]\n" + "ldr q29, [x10, #0xc0]\n" + "fmla v11.8h, v28.8h, v0.h[2]\n" + "fmla v15.8h, v28.8h, v1.h[2]\n" + "fmla v19.8h, v28.8h, v2.h[2]\n" + "fmla v23.8h, v28.8h, v3.h[2]\n" + "fmla v27.8h, v28.8h, v4.h[2]\n" + "ldr q28, [x10, #0xd0]\n" + "fmla v8.8h, v29.8h, v0.h[3]\n" + "fmla v12.8h, v29.8h, v1.h[3]\n" + "fmla v16.8h, v29.8h, v2.h[3]\n" + "fmla v20.8h, v29.8h, v3.h[3]\n" + "fmla v24.8h, v29.8h, v4.h[3]\n" + "ldr q29, [x10, #0xe0]\n" + "fmla v9.8h, v28.8h, v0.h[3]\n" + "fmla v13.8h, v28.8h, v1.h[3]\n" + "fmla v17.8h, v28.8h, v2.h[3]\n" + "fmla v21.8h, v28.8h, v3.h[3]\n" + "fmla v25.8h, v28.8h, v4.h[3]\n" + "ldr q28, [x10, #0xf0]\n" + "fmla v10.8h, v29.8h, v0.h[3]\n" + "fmla v14.8h, v29.8h, v1.h[3]\n" + "fmla v18.8h, v29.8h, v2.h[3]\n" + "fmla v22.8h, v29.8h, v3.h[3]\n" + "fmla v26.8h, v29.8h, v4.h[3]\n" + "ldr q29, [x10, #0x100]\n" + "fmla v11.8h, v28.8h, v0.h[3]\n" + "fmla v15.8h, v28.8h, v1.h[3]\n" + "fmla v19.8h, v28.8h, v2.h[3]\n" + "fmla v23.8h, v28.8h, v3.h[3]\n" + "fmla v27.8h, v28.8h, v4.h[3]\n" + "ldr q28, [x10, #0x110]\n" + "fmla v8.8h, v29.8h, v0.h[4]\n" + "fmla v12.8h, v29.8h, v1.h[4]\n" + "fmla v16.8h, v29.8h, v2.h[4]\n" + "fmla v20.8h, v29.8h, v3.h[4]\n" + "fmla v24.8h, v29.8h, v4.h[4]\n" + "ldr q29, [x10, #0x120]\n" + "fmla v9.8h, v28.8h, v0.h[4]\n" + "fmla v13.8h, v28.8h, v1.h[4]\n" + "fmla v17.8h, v28.8h, v2.h[4]\n" + "fmla v21.8h, v28.8h, v3.h[4]\n" + "fmla v25.8h, v28.8h, v4.h[4]\n" + "ldr q28, [x10, #0x130]\n" + "fmla v10.8h, v29.8h, v0.h[4]\n" + "fmla v14.8h, v29.8h, v1.h[4]\n" + "fmla v18.8h, v29.8h, v2.h[4]\n" + "fmla v22.8h, v29.8h, v3.h[4]\n" + "fmla v26.8h, v29.8h, v4.h[4]\n" + "ldr q29, [x10, #0x140]\n" + "fmla v11.8h, v28.8h, v0.h[4]\n" + "fmla v15.8h, v28.8h, v1.h[4]\n" + "fmla v19.8h, v28.8h, v2.h[4]\n" + "fmla v23.8h, v28.8h, v3.h[4]\n" + "fmla v27.8h, v28.8h, v4.h[4]\n" + "ldr q28, [x10, #0x150]\n" + "fmla v8.8h, v29.8h, v0.h[5]\n" + "fmla v12.8h, v29.8h, v1.h[5]\n" + "fmla v16.8h, v29.8h, v2.h[5]\n" + "fmla v20.8h, v29.8h, v3.h[5]\n" + "fmla v24.8h, v29.8h, v4.h[5]\n" + "ldr q29, [x10, #0x160]\n" + "fmla v9.8h, v28.8h, v0.h[5]\n" + "fmla v13.8h, v28.8h, v1.h[5]\n" + "fmla v17.8h, v28.8h, v2.h[5]\n" + "fmla v21.8h, v28.8h, v3.h[5]\n" + "fmla v25.8h, v28.8h, v4.h[5]\n" + "ldr q28, [x10, #0x170]\n" + "fmla v10.8h, v29.8h, v0.h[5]\n" + "fmla v14.8h, v29.8h, v1.h[5]\n" + "fmla v18.8h, v29.8h, v2.h[5]\n" + "fmla v22.8h, v29.8h, v3.h[5]\n" + "fmla v26.8h, v29.8h, v4.h[5]\n" + "ldr q29, [x10, #0x180]\n" + "fmla v11.8h, v28.8h, v0.h[5]\n" + "fmla v15.8h, v28.8h, v1.h[5]\n" + "fmla v19.8h, v28.8h, v2.h[5]\n" + "fmla v23.8h, v28.8h, v3.h[5]\n" + "fmla v27.8h, v28.8h, v4.h[5]\n" + "ldr q28, [x10, #0x190]\n" + "fmla v8.8h, v29.8h, v0.h[6]\n" + "fmla v12.8h, v29.8h, v1.h[6]\n" + "fmla v16.8h, v29.8h, v2.h[6]\n" + "fmla v20.8h, v29.8h, v3.h[6]\n" + "fmla v24.8h, v29.8h, v4.h[6]\n" + "ldr q29, [x10, #0x1a0]\n" + "fmla v9.8h, v28.8h, v0.h[6]\n" + "fmla v13.8h, v28.8h, v1.h[6]\n" + "fmla v17.8h, v28.8h, v2.h[6]\n" + "fmla v21.8h, v28.8h, v3.h[6]\n" + "fmla v25.8h, v28.8h, v4.h[6]\n" + "ldr q28, [x10, #0x1b0]\n" + "fmla v10.8h, v29.8h, v0.h[6]\n" + "fmla v14.8h, v29.8h, v1.h[6]\n" + "fmla v18.8h, v29.8h, v2.h[6]\n" + "fmla v22.8h, v29.8h, v3.h[6]\n" + "fmla v26.8h, v29.8h, v4.h[6]\n" + "ldr q29, [x10, #0x1c0]\n" + "fmla v11.8h, v28.8h, v0.h[6]\n" + "fmla v15.8h, v28.8h, v1.h[6]\n" + "fmla v19.8h, v28.8h, v2.h[6]\n" + "fmla v23.8h, v28.8h, v3.h[6]\n" + "fmla v27.8h, v28.8h, v4.h[6]\n" + "ldr q28, [x10, #0x1d0]\n" + "fmla v8.8h, v29.8h, v0.h[7]\n" + "fmla v12.8h, v29.8h, v1.h[7]\n" + "fmla v16.8h, v29.8h, v2.h[7]\n" + "fmla v20.8h, v29.8h, v3.h[7]\n" + "fmla v24.8h, v29.8h, v4.h[7]\n" + "ldr q29, [x10, #0x1e0]\n" + "fmla v9.8h, v28.8h, v0.h[7]\n" + "fmla v13.8h, v28.8h, v1.h[7]\n" + "fmla v17.8h, v28.8h, v2.h[7]\n" + "fmla v21.8h, v28.8h, v3.h[7]\n" + "fmla v25.8h, v28.8h, v4.h[7]\n" + "ldr q28, [x10, #0x1f0]\n" + "add x10, x10, #0x200\n" + "fmla v10.8h, v29.8h, v0.h[7]\n" + "fmla v14.8h, v29.8h, v1.h[7]\n" + "fmla v18.8h, v29.8h, v2.h[7]\n" + "fmla v22.8h, v29.8h, v3.h[7]\n" + "fmla v26.8h, v29.8h, v4.h[7]\n" + "ldr q6, [x10, #0x0]\n" + "fmla v11.8h, v28.8h, v0.h[7]\n" + "ldr q0, [x26, #0x0]\n" + "fmla v15.8h, v28.8h, v1.h[7]\n" + "ldr q1, [x25, #0x0]\n" + "fmla v19.8h, v28.8h, v2.h[7]\n" + "ldr q2, [x24, #0x0]\n" + "fmla v23.8h, v28.8h, v3.h[7]\n" + "ldr q3, [x23, #0x0]\n" + "fmla v27.8h, v28.8h, v4.h[7]\n" + "ldr q4, [x22, #0x0]\n" "ldr q7, [x10, #0x10]\n" + "bge 222b\n" + "223:" // Height 5: Multiply loop: Single iteration only + "fmla v8.8h, v6.8h, v0.h[0]\n" + "fmla v12.8h, v6.8h, v1.h[0]\n" + "add x26, x26, #0x10\n" + "add x25, x25, #0x10\n" "fmla v16.8h, v6.8h, v2.h[0]\n" "fmla v20.8h, v6.8h, v3.h[0]\n" + "add x24, x24, #0x10\n" + "add x23, x23, #0x10\n" "fmla v24.8h, v6.8h, v4.h[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q29, [x10, #0x20]\n" "fmla v9.8h, v7.8h, v0.h[0]\n" + "add x22, x22, #0x10\n" "fmla v13.8h, v7.8h, v1.h[0]\n" "fmla v17.8h, v7.8h, v2.h[0]\n" + "sub x27, x27, #0x8\n" + "prfm pldl1keep, [x26, #0x80]\n" "fmla v21.8h, v7.8h, v3.h[0]\n" "fmla v25.8h, v7.8h, v4.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q28, [x10, #0x30]\n" + "prfm pldl1keep, [x25, #0x80]\n" + "fmla v10.8h, v29.8h, v0.h[0]\n" + "fmla v14.8h, v29.8h, v1.h[0]\n" + "prfm pldl1keep, [x24, #0x80]\n" + "prfm pldl1keep, [x23, #0x80]\n" + "fmla v18.8h, v29.8h, v2.h[0]\n" + "fmla v22.8h, v29.8h, v3.h[0]\n" + "prfm pldl1keep, [x22, #0x80]\n" + "fmla v26.8h, v29.8h, v4.h[0]\n" + "ldr q29, [x10, #0x40]\n" + "fmla v11.8h, v28.8h, v0.h[0]\n" + "fmla v15.8h, v28.8h, v1.h[0]\n" + "fmla v19.8h, v28.8h, v2.h[0]\n" + "fmla v23.8h, v28.8h, v3.h[0]\n" + "fmla v27.8h, v28.8h, v4.h[0]\n" + "ldr q28, [x10, #0x50]\n" + "fmla v8.8h, v29.8h, v0.h[1]\n" + "fmla v12.8h, v29.8h, v1.h[1]\n" + "fmla v16.8h, v29.8h, v2.h[1]\n" + "fmla v20.8h, v29.8h, v3.h[1]\n" + "fmla v24.8h, v29.8h, v4.h[1]\n" + "ldr q29, [x10, #0x60]\n" + "fmla v9.8h, v28.8h, v0.h[1]\n" + "fmla v13.8h, v28.8h, v1.h[1]\n" + "fmla v17.8h, v28.8h, v2.h[1]\n" + "fmla v21.8h, v28.8h, v3.h[1]\n" + "fmla v25.8h, v28.8h, v4.h[1]\n" + "ldr q28, [x10, #0x70]\n" + "fmla v10.8h, v29.8h, v0.h[1]\n" + "fmla v14.8h, v29.8h, v1.h[1]\n" + "fmla v18.8h, v29.8h, v2.h[1]\n" + "fmla v22.8h, v29.8h, v3.h[1]\n" + "fmla v26.8h, v29.8h, v4.h[1]\n" + "ldr q29, [x10, #0x80]\n" + "fmla v11.8h, v28.8h, v0.h[1]\n" + "fmla v15.8h, v28.8h, v1.h[1]\n" + "fmla v19.8h, v28.8h, v2.h[1]\n" + "fmla v23.8h, v28.8h, v3.h[1]\n" + "fmla v27.8h, v28.8h, v4.h[1]\n" + "ldr q28, [x10, #0x90]\n" + "fmla v8.8h, v29.8h, v0.h[2]\n" + "fmla v12.8h, v29.8h, v1.h[2]\n" + "fmla v16.8h, v29.8h, v2.h[2]\n" + "fmla v20.8h, v29.8h, v3.h[2]\n" + "fmla v24.8h, v29.8h, v4.h[2]\n" + "ldr q29, [x10, #0xa0]\n" + "fmla v9.8h, v28.8h, v0.h[2]\n" + "fmla v13.8h, v28.8h, v1.h[2]\n" + "fmla v17.8h, v28.8h, v2.h[2]\n" + "fmla v21.8h, v28.8h, v3.h[2]\n" + "fmla v25.8h, v28.8h, v4.h[2]\n" + "ldr q28, [x10, #0xb0]\n" + "fmla v10.8h, v29.8h, v0.h[2]\n" + "fmla v14.8h, v29.8h, v1.h[2]\n" + "fmla v18.8h, v29.8h, v2.h[2]\n" + "fmla v22.8h, v29.8h, v3.h[2]\n" + "fmla v26.8h, v29.8h, v4.h[2]\n" + "ldr q29, [x10, #0xc0]\n" + "fmla v11.8h, v28.8h, v0.h[2]\n" + "fmla v15.8h, v28.8h, v1.h[2]\n" + "fmla v19.8h, v28.8h, v2.h[2]\n" + "fmla v23.8h, v28.8h, v3.h[2]\n" + "fmla v27.8h, v28.8h, v4.h[2]\n" + "ldr q28, [x10, #0xd0]\n" + "fmla v8.8h, v29.8h, v0.h[3]\n" + "fmla v12.8h, v29.8h, v1.h[3]\n" + "fmla v16.8h, v29.8h, v2.h[3]\n" + "fmla v20.8h, v29.8h, v3.h[3]\n" + "fmla v24.8h, v29.8h, v4.h[3]\n" + "ldr q29, [x10, #0xe0]\n" + "fmla v9.8h, v28.8h, v0.h[3]\n" + "fmla v13.8h, v28.8h, v1.h[3]\n" + "fmla v17.8h, v28.8h, v2.h[3]\n" + "fmla v21.8h, v28.8h, v3.h[3]\n" + "fmla v25.8h, v28.8h, v4.h[3]\n" + "ldr q28, [x10, #0xf0]\n" + "fmla v10.8h, v29.8h, v0.h[3]\n" + "fmla v14.8h, v29.8h, v1.h[3]\n" + "fmla v18.8h, v29.8h, v2.h[3]\n" + "fmla v22.8h, v29.8h, v3.h[3]\n" + "fmla v26.8h, v29.8h, v4.h[3]\n" + "ldr q29, [x10, #0x100]\n" + "fmla v11.8h, v28.8h, v0.h[3]\n" + "fmla v15.8h, v28.8h, v1.h[3]\n" + "fmla v19.8h, v28.8h, v2.h[3]\n" + "fmla v23.8h, v28.8h, v3.h[3]\n" + "fmla v27.8h, v28.8h, v4.h[3]\n" + "ldr q28, [x10, #0x110]\n" + "fmla v8.8h, v29.8h, v0.h[4]\n" + "fmla v12.8h, v29.8h, v1.h[4]\n" + "fmla v16.8h, v29.8h, v2.h[4]\n" + "fmla v20.8h, v29.8h, v3.h[4]\n" + "fmla v24.8h, v29.8h, v4.h[4]\n" + "ldr q29, [x10, #0x120]\n" + "fmla v9.8h, v28.8h, v0.h[4]\n" + "fmla v13.8h, v28.8h, v1.h[4]\n" + "fmla v17.8h, v28.8h, v2.h[4]\n" + "fmla v21.8h, v28.8h, v3.h[4]\n" + "fmla v25.8h, v28.8h, v4.h[4]\n" + "ldr q28, [x10, #0x130]\n" + "fmla v10.8h, v29.8h, v0.h[4]\n" + "fmla v14.8h, v29.8h, v1.h[4]\n" + "fmla v18.8h, v29.8h, v2.h[4]\n" + "fmla v22.8h, v29.8h, v3.h[4]\n" + "fmla v26.8h, v29.8h, v4.h[4]\n" + "ldr q29, [x10, #0x140]\n" + "fmla v11.8h, v28.8h, v0.h[4]\n" + "fmla v15.8h, v28.8h, v1.h[4]\n" + "fmla v19.8h, v28.8h, v2.h[4]\n" + "fmla v23.8h, v28.8h, v3.h[4]\n" + "fmla v27.8h, v28.8h, v4.h[4]\n" + "ldr q28, [x10, #0x150]\n" + "fmla v8.8h, v29.8h, v0.h[5]\n" + "fmla v12.8h, v29.8h, v1.h[5]\n" + "fmla v16.8h, v29.8h, v2.h[5]\n" + "fmla v20.8h, v29.8h, v3.h[5]\n" + "fmla v24.8h, v29.8h, v4.h[5]\n" + "ldr q29, [x10, #0x160]\n" + "fmla v9.8h, v28.8h, v0.h[5]\n" + "fmla v13.8h, v28.8h, v1.h[5]\n" + "fmla v17.8h, v28.8h, v2.h[5]\n" + "fmla v21.8h, v28.8h, v3.h[5]\n" + "fmla v25.8h, v28.8h, v4.h[5]\n" + "ldr q28, [x10, #0x170]\n" + "fmla v10.8h, v29.8h, v0.h[5]\n" + "fmla v14.8h, v29.8h, v1.h[5]\n" + "fmla v18.8h, v29.8h, v2.h[5]\n" + "fmla v22.8h, v29.8h, v3.h[5]\n" + "fmla v26.8h, v29.8h, v4.h[5]\n" + "ldr q29, [x10, #0x180]\n" + "fmla v11.8h, v28.8h, v0.h[5]\n" + "fmla v15.8h, v28.8h, v1.h[5]\n" + "fmla v19.8h, v28.8h, v2.h[5]\n" + "fmla v23.8h, v28.8h, v3.h[5]\n" + "fmla v27.8h, v28.8h, v4.h[5]\n" + "ldr q28, [x10, #0x190]\n" + "fmla v8.8h, v29.8h, v0.h[6]\n" + "fmla v12.8h, v29.8h, v1.h[6]\n" + "fmla v16.8h, v29.8h, v2.h[6]\n" + "fmla v20.8h, v29.8h, v3.h[6]\n" + "fmla v24.8h, v29.8h, v4.h[6]\n" + "ldr q29, [x10, #0x1a0]\n" + "fmla v9.8h, v28.8h, v0.h[6]\n" + "fmla v13.8h, v28.8h, v1.h[6]\n" + "fmla v17.8h, v28.8h, v2.h[6]\n" + "fmla v21.8h, v28.8h, v3.h[6]\n" + "fmla v25.8h, v28.8h, v4.h[6]\n" + "ldr q28, [x10, #0x1b0]\n" + "fmla v10.8h, v29.8h, v0.h[6]\n" + "fmla v14.8h, v29.8h, v1.h[6]\n" + "fmla v18.8h, v29.8h, v2.h[6]\n" + "fmla v22.8h, v29.8h, v3.h[6]\n" + "fmla v26.8h, v29.8h, v4.h[6]\n" + "ldr q29, [x10, #0x1c0]\n" + "fmla v11.8h, v28.8h, v0.h[6]\n" + "fmla v15.8h, v28.8h, v1.h[6]\n" + "fmla v19.8h, v28.8h, v2.h[6]\n" + "fmla v23.8h, v28.8h, v3.h[6]\n" + "fmla v27.8h, v28.8h, v4.h[6]\n" + "ldr q28, [x10, #0x1d0]\n" + "fmla v8.8h, v29.8h, v0.h[7]\n" + "fmla v12.8h, v29.8h, v1.h[7]\n" + "fmla v16.8h, v29.8h, v2.h[7]\n" + "fmla v20.8h, v29.8h, v3.h[7]\n" + "fmla v24.8h, v29.8h, v4.h[7]\n" + "ldr q29, [x10, #0x1e0]\n" + "fmla v9.8h, v28.8h, v0.h[7]\n" + "fmla v13.8h, v28.8h, v1.h[7]\n" + "fmla v17.8h, v28.8h, v2.h[7]\n" + "fmla v21.8h, v28.8h, v3.h[7]\n" + "fmla v25.8h, v28.8h, v4.h[7]\n" + "ldr q28, [x10, #0x1f0]\n" + "add x10, x10, #0x200\n" + "fmla v10.8h, v29.8h, v0.h[7]\n" + "fmla v14.8h, v29.8h, v1.h[7]\n" + "fmla v18.8h, v29.8h, v2.h[7]\n" + "fmla v22.8h, v29.8h, v3.h[7]\n" + "fmla v26.8h, v29.8h, v4.h[7]\n" + "fmla v11.8h, v28.8h, v0.h[7]\n" + "fmla v15.8h, v28.8h, v1.h[7]\n" + "fmla v19.8h, v28.8h, v2.h[7]\n" + "fmla v23.8h, v28.8h, v3.h[7]\n" + "fmla v27.8h, v28.8h, v4.h[7]\n" + "224:" // Height 5: Multiply loop: Main loop skip + "cbz x27, 226f\n" + "225:" // Height 5: Multiply loop: Odd block loop + "ldr h4, [x26], #0x2\n" + "ldr h3, [x25], #0x2\n" + "sub x27, x27, #0x1\n" + "ldr h2, [x24], #0x2\n" + "ldr h1, [x23], #0x2\n" + "ldr h0, [x22], #0x2\n" + "ldr q29, [x10, #0x0]\n" + "fmla v8.8h, v29.8h, v4.h[0]\n" + "fmla v12.8h, v29.8h, v3.h[0]\n" + "ldr q28, [x10, #0x10]\n" + "fmla v16.8h, v29.8h, v2.h[0]\n" + "fmla v20.8h, v29.8h, v1.h[0]\n" + "fmla v24.8h, v29.8h, v0.h[0]\n" + "ldr q29, [x10, #0x20]\n" + "fmla v9.8h, v28.8h, v4.h[0]\n" + "fmla v13.8h, v28.8h, v3.h[0]\n" + "fmla v17.8h, v28.8h, v2.h[0]\n" + "fmla v21.8h, v28.8h, v1.h[0]\n" + "fmla v25.8h, v28.8h, v0.h[0]\n" + "ldr q28, [x10, #0x30]\n" "add x10, x10, #0x40\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "fmla v22.8h, v6.8h, v3.h[0]\n" - "fmla v26.8h, v6.8h, v4.h[0]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "fmla v23.8h, v7.8h, v3.h[0]\n" - "fmla v27.8h, v7.8h, v4.h[0]\n" + "fmla v10.8h, v29.8h, v4.h[0]\n" + "fmla v14.8h, v29.8h, v3.h[0]\n" + "fmla v18.8h, v29.8h, v2.h[0]\n" + "fmla v22.8h, v29.8h, v1.h[0]\n" + "fmla v26.8h, v29.8h, v0.h[0]\n" + "fmla v11.8h, v28.8h, v4.h[0]\n" + "fmla v15.8h, v28.8h, v3.h[0]\n" + "fmla v19.8h, v28.8h, v2.h[0]\n" + "fmla v23.8h, v28.8h, v1.h[0]\n" + "fmla v27.8h, v28.8h, v0.h[0]\n" "cbnz x27, 225b\n" "226:" // Height 5: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -3719,49 +3719,49 @@ void a64_hybrid_fp16_mla_6x32 ( "prfm pstl1keep, [x22, #0x0]\n" "tbz %x[flags], #1, 227f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.8h }, [x20]\n" + "ld1r { v29.8h }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.8h }, [x20]\n" - "fmin v8.8h, v8.8h, v1.8h\n" - "fmin v9.8h, v9.8h, v1.8h\n" - "fmin v10.8h, v10.8h, v1.8h\n" - "fmin v11.8h, v11.8h, v1.8h\n" - "fmin v12.8h, v12.8h, v1.8h\n" - "fmin v13.8h, v13.8h, v1.8h\n" - "fmin v14.8h, v14.8h, v1.8h\n" - "fmin v15.8h, v15.8h, v1.8h\n" - "fmin v16.8h, v16.8h, v1.8h\n" - "fmin v17.8h, v17.8h, v1.8h\n" - "fmin v18.8h, v18.8h, v1.8h\n" - "fmin v19.8h, v19.8h, v1.8h\n" - "fmin v20.8h, v20.8h, v1.8h\n" - "fmin v21.8h, v21.8h, v1.8h\n" - "fmin v22.8h, v22.8h, v1.8h\n" - "fmin v23.8h, v23.8h, v1.8h\n" - "fmin v24.8h, v24.8h, v1.8h\n" - "fmin v25.8h, v25.8h, v1.8h\n" - "fmin v26.8h, v26.8h, v1.8h\n" - "fmin v27.8h, v27.8h, v1.8h\n" - "fmax v8.8h, v8.8h, v0.8h\n" - "fmax v9.8h, v9.8h, v0.8h\n" - "fmax v10.8h, v10.8h, v0.8h\n" - "fmax v11.8h, v11.8h, v0.8h\n" - "fmax v12.8h, v12.8h, v0.8h\n" - "fmax v13.8h, v13.8h, v0.8h\n" - "fmax v14.8h, v14.8h, v0.8h\n" - "fmax v15.8h, v15.8h, v0.8h\n" - "fmax v16.8h, v16.8h, v0.8h\n" - "fmax v17.8h, v17.8h, v0.8h\n" - "fmax v18.8h, v18.8h, v0.8h\n" - "fmax v19.8h, v19.8h, v0.8h\n" - "fmax v20.8h, v20.8h, v0.8h\n" - "fmax v21.8h, v21.8h, v0.8h\n" - "fmax v22.8h, v22.8h, v0.8h\n" - "fmax v23.8h, v23.8h, v0.8h\n" - "fmax v24.8h, v24.8h, v0.8h\n" - "fmax v25.8h, v25.8h, v0.8h\n" - "fmax v26.8h, v26.8h, v0.8h\n" - "fmax v27.8h, v27.8h, v0.8h\n" + "ld1r { v28.8h }, [x20]\n" + "fmin v8.8h, v8.8h, v29.8h\n" + "fmin v9.8h, v9.8h, v29.8h\n" + "fmin v10.8h, v10.8h, v29.8h\n" + "fmin v11.8h, v11.8h, v29.8h\n" + "fmin v12.8h, v12.8h, v29.8h\n" + "fmin v13.8h, v13.8h, v29.8h\n" + "fmin v14.8h, v14.8h, v29.8h\n" + "fmin v15.8h, v15.8h, v29.8h\n" + "fmin v16.8h, v16.8h, v29.8h\n" + "fmin v17.8h, v17.8h, v29.8h\n" + "fmin v18.8h, v18.8h, v29.8h\n" + "fmin v19.8h, v19.8h, v29.8h\n" + "fmin v20.8h, v20.8h, v29.8h\n" + "fmin v21.8h, v21.8h, v29.8h\n" + "fmin v22.8h, v22.8h, v29.8h\n" + "fmin v23.8h, v23.8h, v29.8h\n" + "fmin v24.8h, v24.8h, v29.8h\n" + "fmin v25.8h, v25.8h, v29.8h\n" + "fmin v26.8h, v26.8h, v29.8h\n" + "fmin v27.8h, v27.8h, v29.8h\n" + "fmax v8.8h, v8.8h, v28.8h\n" + "fmax v9.8h, v9.8h, v28.8h\n" + "fmax v10.8h, v10.8h, v28.8h\n" + "fmax v11.8h, v11.8h, v28.8h\n" + "fmax v12.8h, v12.8h, v28.8h\n" + "fmax v13.8h, v13.8h, v28.8h\n" + "fmax v14.8h, v14.8h, v28.8h\n" + "fmax v15.8h, v15.8h, v28.8h\n" + "fmax v16.8h, v16.8h, v28.8h\n" + "fmax v17.8h, v17.8h, v28.8h\n" + "fmax v18.8h, v18.8h, v28.8h\n" + "fmax v19.8h, v19.8h, v28.8h\n" + "fmax v20.8h, v20.8h, v28.8h\n" + "fmax v21.8h, v21.8h, v28.8h\n" + "fmax v22.8h, v22.8h, v28.8h\n" + "fmax v23.8h, v23.8h, v28.8h\n" + "fmax v24.8h, v24.8h, v28.8h\n" + "fmax v25.8h, v25.8h, v28.8h\n" + "fmax v26.8h, v26.8h, v28.8h\n" + "fmax v27.8h, v27.8h, v28.8h\n" "227:" // Height 5: No activation "cmp x11, #0x20\n" "bge 244f\n" @@ -4386,16 +4386,16 @@ void a64_hybrid_fp16_mla_6x32 ( "268:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 269f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 270f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -4407,11 +4407,11 @@ void a64_hybrid_fp16_mla_6x32 ( "b 270f\n" "269:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" - "add x22, x23, x20, LSL #1\n" - "add x21, x22, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" + "add x22, x23, x21, LSL #1\n" + "add x21, x22, x21, LSL #1\n" "270:" // Height 6: input setup done "cmp x27, #0x8\n" "blt 273f\n" @@ -4912,42 +4912,42 @@ void a64_hybrid_fp16_mla_6x32 ( "273:" // Height 6: Multiply loop: Main loop skip "cbz x27, 275f\n" "274:" // Height 6: Multiply loop: Odd block loop - "ldr h0, [x26], #0x2\n" - "ldr h1, [x25], #0x2\n" + "ldr h7, [x26], #0x2\n" + "ldr h6, [x25], #0x2\n" "sub x27, x27, #0x1\n" - "ldr h2, [x24], #0x2\n" - "ldr h3, [x23], #0x2\n" - "ldr h4, [x22], #0x2\n" - "ldr h5, [x21], #0x2\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - "fmla v8.8h, v6.8h, v0.h[0]\n" - "fmla v12.8h, v6.8h, v1.h[0]\n" - "fmla v16.8h, v6.8h, v2.h[0]\n" - "fmla v20.8h, v6.8h, v3.h[0]\n" - "fmla v24.8h, v6.8h, v4.h[0]\n" - "fmla v28.8h, v6.8h, v5.h[0]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.8h, v7.8h, v0.h[0]\n" - "fmla v13.8h, v7.8h, v1.h[0]\n" - "fmla v17.8h, v7.8h, v2.h[0]\n" - "fmla v21.8h, v7.8h, v3.h[0]\n" - "fmla v25.8h, v7.8h, v4.h[0]\n" - "fmla v29.8h, v7.8h, v5.h[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr h5, [x24], #0x2\n" + "ldr h4, [x23], #0x2\n" + "ldr h3, [x22], #0x2\n" + "ldr h2, [x21], #0x2\n" + "ldr q1, [x10, #0x0]\n" + "ldr q0, [x10, #0x10]\n" + "fmla v8.8h, v1.8h, v7.h[0]\n" + "fmla v12.8h, v1.8h, v6.h[0]\n" + "fmla v16.8h, v1.8h, v5.h[0]\n" + "fmla v20.8h, v1.8h, v4.h[0]\n" + "fmla v24.8h, v1.8h, v3.h[0]\n" + "fmla v28.8h, v1.8h, v2.h[0]\n" + "ldr q1, [x10, #0x20]\n" + "fmla v9.8h, v0.8h, v7.h[0]\n" + "fmla v13.8h, v0.8h, v6.h[0]\n" + "fmla v17.8h, v0.8h, v5.h[0]\n" + "fmla v21.8h, v0.8h, v4.h[0]\n" + "fmla v25.8h, v0.8h, v3.h[0]\n" + "fmla v29.8h, v0.8h, v2.h[0]\n" + "ldr q0, [x10, #0x30]\n" "add x10, x10, #0x40\n" - "fmla v10.8h, v6.8h, v0.h[0]\n" - "fmla v14.8h, v6.8h, v1.h[0]\n" - "fmla v18.8h, v6.8h, v2.h[0]\n" - "fmla v22.8h, v6.8h, v3.h[0]\n" - "fmla v26.8h, v6.8h, v4.h[0]\n" - "fmla v30.8h, v6.8h, v5.h[0]\n" - "fmla v11.8h, v7.8h, v0.h[0]\n" - "fmla v15.8h, v7.8h, v1.h[0]\n" - "fmla v19.8h, v7.8h, v2.h[0]\n" - "fmla v23.8h, v7.8h, v3.h[0]\n" - "fmla v27.8h, v7.8h, v4.h[0]\n" - "fmla v31.8h, v7.8h, v5.h[0]\n" + "fmla v10.8h, v1.8h, v7.h[0]\n" + "fmla v14.8h, v1.8h, v6.h[0]\n" + "fmla v18.8h, v1.8h, v5.h[0]\n" + "fmla v22.8h, v1.8h, v4.h[0]\n" + "fmla v26.8h, v1.8h, v3.h[0]\n" + "fmla v30.8h, v1.8h, v2.h[0]\n" + "fmla v11.8h, v0.8h, v7.h[0]\n" + "fmla v15.8h, v0.8h, v6.h[0]\n" + "fmla v19.8h, v0.8h, v5.h[0]\n" + "fmla v23.8h, v0.8h, v4.h[0]\n" + "fmla v27.8h, v0.8h, v3.h[0]\n" + "fmla v31.8h, v0.8h, v2.h[0]\n" "cbnz x27, 274b\n" "275:" // Height 6: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -5317,7 +5317,6 @@ void a64_hybrid_fp16_mla_6x32 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "296:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x24.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x24.hpp index e155bfb11182885a3b38304ef80f1b6fd9f97447..171929e65ef02ce0c43e3731c66a223a3312e0c2 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x24.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x24.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef __aarch64__ + #include "../std_transforms_fixed.hpp" #include "../performance_parameters.hpp" @@ -113,5 +113,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x24/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x24/a55.cpp index 700d803f82dbe36e8dc476dee9e57c1acbb23f85..9ceda8fd0c692b4ea9cae6634036a27303e75d16 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x24/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x24/a55.cpp @@ -92,7 +92,6 @@ void a64_hybrid_fp32_mla_4x24_a55 ( break; } __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x4\n" "bge 124f\n" @@ -223,11 +222,11 @@ void a64_hybrid_fp32_mla_4x24_a55 ( "19:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w12, [x20, x13, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 20f\n" - "ldr x21, [%x[input_ptr], x13, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x11, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x13, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x11, [x20, #0x0]\n" "cbnz x13, 21f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x11, x11, x20, LSL #2\n" @@ -246,176 +245,176 @@ void a64_hybrid_fp32_mla_4x24_a55 ( "blt 23f\n" "22:" // Height 1: Multiply loop: Main loop head "fmla v8.4s, v4.4s, v0.s[0]\n" - "ldr d4, [x15, #0x40]\n" - "ldr x10, [x15, #0x48]\n" + "ldr d19, [x15, #0x40]\n" + "ldr x20, [x15, #0x48]\n" "fmla v9.4s, v5.4s, v0.s[0]\n" - "ldr d5, [x15, #0x50]\n" + "ldr d18, [x15, #0x50]\n" "fmla v10.4s, v6.4s, v0.s[0]\n" - "ldr d6, [x15, #0x60]\n" + "ldr d17, [x15, #0x60]\n" "fmla v11.4s, v7.4s, v0.s[0]\n" - "ldr d7, [x15, #0x70]\n" - "mov v4.d[1], x10\n" - "ldr x9, [x15, #0x58]\n" - "mov v5.d[1], x9\n" - "ldr x28, [x15, #0x68]\n" - "mov v6.d[1], x28\n" - "ldr x27, [x15, #0x78]\n" - "mov v7.d[1], x27\n" - "fmla v12.4s, v4.4s, v0.s[0]\n" - "ldr d4, [x15, #0x80]\n" - "ldr x10, [x15, #0x88]\n" - "fmla v13.4s, v5.4s, v0.s[0]\n" - "ldr d5, [x15, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "ldr d6, [x15, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "ldr d7, [x15, #0xb0]\n" - "mov v4.d[1], x10\n" - "ldr x9, [x15, #0x98]\n" - "mov v5.d[1], x9\n" - "ldr x28, [x15, #0xa8]\n" - "mov v6.d[1], x28\n" - "ldr x27, [x15, #0xb8]\n" - "mov v7.d[1], x27\n" - "fmla v10.4s, v4.4s, v0.s[1]\n" - "ldr d4, [x15, #0xc0]\n" - "ldr x10, [x15, #0xc8]\n" - "fmla v11.4s, v5.4s, v0.s[1]\n" - "ldr d5, [x15, #0xd0]\n" - "fmla v12.4s, v6.4s, v0.s[1]\n" - "ldr d6, [x15, #0xe0]\n" - "fmla v13.4s, v7.4s, v0.s[1]\n" - "ldr d7, [x15, #0xf0]\n" - "mov v4.d[1], x10\n" - "ldr x9, [x15, #0xd8]\n" - "mov v5.d[1], x9\n" - "ldr x28, [x15, #0xe8]\n" - "mov v6.d[1], x28\n" - "ldr x27, [x15, #0xf8]\n" - "mov v7.d[1], x27\n" - "fmla v8.4s, v4.4s, v0.s[2]\n" - "ldr d4, [x15, #0x100]\n" - "ldr x10, [x15, #0x108]\n" - "fmla v9.4s, v5.4s, v0.s[2]\n" - "ldr d5, [x15, #0x110]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "ldr d6, [x15, #0x120]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "ldr d7, [x15, #0x130]\n" - "mov v4.d[1], x10\n" - "ldr x9, [x15, #0x118]\n" - "mov v5.d[1], x9\n" - "ldr x28, [x15, #0x128]\n" - "mov v6.d[1], x28\n" - "ldr x27, [x15, #0x138]\n" - "mov v7.d[1], x27\n" - "fmla v12.4s, v4.4s, v0.s[2]\n" - "ldr d4, [x15, #0x140]\n" - "ldr x10, [x15, #0x148]\n" - "fmla v13.4s, v5.4s, v0.s[2]\n" - "ldr d5, [x15, #0x150]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "ldr d6, [x15, #0x160]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "ldr d7, [x15, #0x170]\n" - "mov v4.d[1], x10\n" - "ldr x9, [x15, #0x158]\n" - "mov v5.d[1], x9\n" - "ldr x28, [x15, #0x168]\n" - "mov v6.d[1], x28\n" - "ldr x27, [x15, #0x178]\n" - "mov v7.d[1], x27\n" + "ldr d16, [x15, #0x70]\n" + "mov v19.d[1], x20\n" + "ldr x20, [x15, #0x58]\n" + "mov v18.d[1], x20\n" + "ldr x20, [x15, #0x68]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x15, #0x78]\n" + "mov v16.d[1], x20\n" + "fmla v12.4s, v19.4s, v0.s[0]\n" + "ldr d19, [x15, #0x80]\n" + "ldr x20, [x15, #0x88]\n" + "fmla v13.4s, v18.4s, v0.s[0]\n" + "ldr d18, [x15, #0x90]\n" + "fmla v8.4s, v17.4s, v0.s[1]\n" + "ldr d17, [x15, #0xa0]\n" + "fmla v9.4s, v16.4s, v0.s[1]\n" + "ldr d16, [x15, #0xb0]\n" + "mov v19.d[1], x20\n" + "ldr x20, [x15, #0x98]\n" + "mov v18.d[1], x20\n" + "ldr x20, [x15, #0xa8]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x15, #0xb8]\n" + "mov v16.d[1], x20\n" + "fmla v10.4s, v19.4s, v0.s[1]\n" + "ldr d19, [x15, #0xc0]\n" + "ldr x20, [x15, #0xc8]\n" + "fmla v11.4s, v18.4s, v0.s[1]\n" + "ldr d18, [x15, #0xd0]\n" + "fmla v12.4s, v17.4s, v0.s[1]\n" + "ldr d17, [x15, #0xe0]\n" + "fmla v13.4s, v16.4s, v0.s[1]\n" + "ldr d16, [x15, #0xf0]\n" + "mov v19.d[1], x20\n" + "ldr x20, [x15, #0xd8]\n" + "mov v18.d[1], x20\n" + "ldr x20, [x15, #0xe8]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x15, #0xf8]\n" + "mov v16.d[1], x20\n" + "fmla v8.4s, v19.4s, v0.s[2]\n" + "ldr d19, [x15, #0x100]\n" + "ldr x20, [x15, #0x108]\n" + "fmla v9.4s, v18.4s, v0.s[2]\n" + "ldr d18, [x15, #0x110]\n" + "fmla v10.4s, v17.4s, v0.s[2]\n" + "ldr d17, [x15, #0x120]\n" + "fmla v11.4s, v16.4s, v0.s[2]\n" + "ldr d16, [x15, #0x130]\n" + "mov v19.d[1], x20\n" + "ldr x20, [x15, #0x118]\n" + "mov v18.d[1], x20\n" + "ldr x20, [x15, #0x128]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x15, #0x138]\n" + "mov v16.d[1], x20\n" + "fmla v12.4s, v19.4s, v0.s[2]\n" + "ldr d19, [x15, #0x140]\n" + "ldr x20, [x15, #0x148]\n" + "fmla v13.4s, v18.4s, v0.s[2]\n" + "ldr d18, [x15, #0x150]\n" + "fmla v8.4s, v17.4s, v0.s[3]\n" + "ldr d17, [x15, #0x160]\n" + "fmla v9.4s, v16.4s, v0.s[3]\n" + "ldr d16, [x15, #0x170]\n" + "mov v19.d[1], x20\n" + "ldr x20, [x15, #0x158]\n" + "mov v18.d[1], x20\n" + "ldr x20, [x15, #0x168]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x15, #0x178]\n" + "mov v16.d[1], x20\n" "add x11, x11, #0x10\n" "add x15, x15, #0x180\n" - "fmla v10.4s, v4.4s, v0.s[3]\n" + "fmla v10.4s, v19.4s, v0.s[3]\n" "ldr d4, [x15, #0x0]\n" - "ldr x10, [x15, #0x8]\n" - "fmla v11.4s, v5.4s, v0.s[3]\n" + "ldr x20, [x15, #0x8]\n" + "fmla v11.4s, v18.4s, v0.s[3]\n" "ldr d5, [x15, #0x10]\n" - "fmla v12.4s, v6.4s, v0.s[3]\n" + "fmla v12.4s, v17.4s, v0.s[3]\n" "ldr d6, [x15, #0x20]\n" - "fmla v13.4s, v7.4s, v0.s[3]\n" + "fmla v13.4s, v16.4s, v0.s[3]\n" "ldr d0, [x11, #0x0]\n" "sub x12, x12, #0x4\n" "ldr d7, [x15, #0x30]\n" "cmp x12, #0x8\n" - "ldr x9, [x15, #0x18]\n" - "mov v4.d[1], x10\n" - "ldr x28, [x15, #0x28]\n" - "mov v5.d[1], x9\n" - "ldr x26, [x11, #0x8]\n" - "mov v6.d[1], x28\n" - "ldr x27, [x15, #0x38]\n" - "mov v0.d[1], x26\n" - "mov v7.d[1], x27\n" + "ldr x21, [x15, #0x18]\n" + "mov v4.d[1], x20\n" + "ldr x20, [x15, #0x28]\n" + "mov v5.d[1], x21\n" + "ldr x21, [x11, #0x8]\n" + "mov v6.d[1], x20\n" + "ldr x20, [x15, #0x38]\n" + "mov v0.d[1], x21\n" + "mov v7.d[1], x20\n" "prfm pldl1keep, [x11, #0x80]\n" "bge 22b\n" "23:" // Height 1: Multiply loop: Single iteration only "fmla v8.4s, v4.4s, v0.s[0]\n" - "ldr q4, [x15, #0x40]\n" + "ldr q19, [x15, #0x40]\n" "fmla v9.4s, v5.4s, v0.s[0]\n" - "ldr q5, [x15, #0x50]\n" + "ldr q18, [x15, #0x50]\n" "fmla v10.4s, v6.4s, v0.s[0]\n" - "ldr q6, [x15, #0x60]\n" + "ldr q17, [x15, #0x60]\n" "fmla v11.4s, v7.4s, v0.s[0]\n" - "ldr q7, [x15, #0x70]\n" - "fmla v12.4s, v4.4s, v0.s[0]\n" - "ldr q4, [x15, #0x80]\n" - "fmla v13.4s, v5.4s, v0.s[0]\n" - "ldr q5, [x15, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "ldr q6, [x15, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "ldr q7, [x15, #0xb0]\n" - "fmla v10.4s, v4.4s, v0.s[1]\n" - "ldr q4, [x15, #0xc0]\n" - "fmla v11.4s, v5.4s, v0.s[1]\n" - "ldr q5, [x15, #0xd0]\n" - "fmla v12.4s, v6.4s, v0.s[1]\n" - "ldr q6, [x15, #0xe0]\n" - "fmla v13.4s, v7.4s, v0.s[1]\n" - "ldr q7, [x15, #0xf0]\n" - "fmla v8.4s, v4.4s, v0.s[2]\n" - "ldr q4, [x15, #0x100]\n" - "fmla v9.4s, v5.4s, v0.s[2]\n" - "ldr q5, [x15, #0x110]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "ldr q6, [x15, #0x120]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "ldr q7, [x15, #0x130]\n" - "fmla v12.4s, v4.4s, v0.s[2]\n" - "ldr q4, [x15, #0x140]\n" - "fmla v13.4s, v5.4s, v0.s[2]\n" - "ldr q5, [x15, #0x150]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "ldr q6, [x15, #0x160]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "ldr q7, [x15, #0x170]\n" + "ldr q16, [x15, #0x70]\n" + "fmla v12.4s, v19.4s, v0.s[0]\n" + "ldr q19, [x15, #0x80]\n" + "fmla v13.4s, v18.4s, v0.s[0]\n" + "ldr q18, [x15, #0x90]\n" + "fmla v8.4s, v17.4s, v0.s[1]\n" + "ldr q17, [x15, #0xa0]\n" + "fmla v9.4s, v16.4s, v0.s[1]\n" + "ldr q16, [x15, #0xb0]\n" + "fmla v10.4s, v19.4s, v0.s[1]\n" + "ldr q19, [x15, #0xc0]\n" + "fmla v11.4s, v18.4s, v0.s[1]\n" + "ldr q18, [x15, #0xd0]\n" + "fmla v12.4s, v17.4s, v0.s[1]\n" + "ldr q17, [x15, #0xe0]\n" + "fmla v13.4s, v16.4s, v0.s[1]\n" + "ldr q16, [x15, #0xf0]\n" + "fmla v8.4s, v19.4s, v0.s[2]\n" + "ldr q19, [x15, #0x100]\n" + "fmla v9.4s, v18.4s, v0.s[2]\n" + "ldr q18, [x15, #0x110]\n" + "fmla v10.4s, v17.4s, v0.s[2]\n" + "ldr q17, [x15, #0x120]\n" + "fmla v11.4s, v16.4s, v0.s[2]\n" + "ldr q16, [x15, #0x130]\n" + "fmla v12.4s, v19.4s, v0.s[2]\n" + "ldr q19, [x15, #0x140]\n" + "fmla v13.4s, v18.4s, v0.s[2]\n" + "ldr q18, [x15, #0x150]\n" + "fmla v8.4s, v17.4s, v0.s[3]\n" + "ldr q17, [x15, #0x160]\n" + "fmla v9.4s, v16.4s, v0.s[3]\n" + "ldr q16, [x15, #0x170]\n" "add x11, x11, #0x10\n" "sub x12, x12, #0x4\n" - "fmla v10.4s, v4.4s, v0.s[3]\n" + "fmla v10.4s, v19.4s, v0.s[3]\n" "prfm pldl1keep, [x11, #0x80]\n" - "fmla v11.4s, v5.4s, v0.s[3]\n" + "fmla v11.4s, v18.4s, v0.s[3]\n" "add x15, x15, #0x180\n" - "fmla v12.4s, v6.4s, v0.s[3]\n" - "fmla v13.4s, v7.4s, v0.s[3]\n" + "fmla v12.4s, v17.4s, v0.s[3]\n" + "fmla v13.4s, v16.4s, v0.s[3]\n" "24:" // Height 1: Multiply loop: Main loop skip "cbz x12, 26f\n" "25:" // Height 1: Multiply loop: Odd block loop - "ldr s0, [x11], #0x4\n" + "ldr s17, [x11], #0x4\n" "sub x12, x12, #0x1\n" - "ldr q4, [x15, #0x0]\n" - "fmla v8.4s, v4.4s, v0.s[0]\n" - "ldr q5, [x15, #0x10]\n" - "fmla v9.4s, v5.4s, v0.s[0]\n" - "ldr q6, [x15, #0x20]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "ldr q7, [x15, #0x30]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "ldr q4, [x15, #0x40]\n" - "fmla v12.4s, v4.4s, v0.s[0]\n" - "ldr q5, [x15, #0x50]\n" - "fmla v13.4s, v5.4s, v0.s[0]\n" + "ldr q16, [x15, #0x0]\n" + "fmla v8.4s, v16.4s, v17.s[0]\n" + "ldr q16, [x15, #0x10]\n" + "fmla v9.4s, v16.4s, v17.s[0]\n" + "ldr q16, [x15, #0x20]\n" + "fmla v10.4s, v16.4s, v17.s[0]\n" + "ldr q16, [x15, #0x30]\n" + "fmla v11.4s, v16.4s, v17.s[0]\n" + "ldr q16, [x15, #0x40]\n" + "fmla v12.4s, v16.4s, v17.s[0]\n" + "ldr q16, [x15, #0x50]\n" + "fmla v13.4s, v16.4s, v17.s[0]\n" "add x15, x15, #0x60\n" "cbnz x12, 25b\n" "26:" // Height 1: Multiply loop: No odd multiplies @@ -426,21 +425,21 @@ void a64_hybrid_fp32_mla_4x24_a55 ( "prfm pstl1keep, [x14, #0x0]\n" "tbz %x[flags], #1, 27f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v0.4s\n" - "fmin v9.4s, v9.4s, v0.4s\n" - "fmin v10.4s, v10.4s, v0.4s\n" - "fmin v11.4s, v11.4s, v0.4s\n" - "fmin v12.4s, v12.4s, v0.4s\n" - "fmin v13.4s, v13.4s, v0.4s\n" + "ld1r { v16.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v16.4s\n" + "fmin v9.4s, v9.4s, v16.4s\n" + "fmin v10.4s, v10.4s, v16.4s\n" + "fmin v11.4s, v11.4s, v16.4s\n" + "fmin v12.4s, v12.4s, v16.4s\n" + "fmin v13.4s, v13.4s, v16.4s\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" + "ld1r { v16.4s }, [x20]\n" + "fmax v8.4s, v8.4s, v16.4s\n" + "fmax v9.4s, v9.4s, v16.4s\n" + "fmax v10.4s, v10.4s, v16.4s\n" + "fmax v11.4s, v11.4s, v16.4s\n" + "fmax v12.4s, v12.4s, v16.4s\n" + "fmax v13.4s, v13.4s, v16.4s\n" "27:" // Height 1: No activation "cmp x16, #0x18\n" "bge 40f\n" @@ -701,26 +700,26 @@ void a64_hybrid_fp32_mla_4x24_a55 ( "60:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w12, [x20, x13, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 61f\n" - "ldr x21, [%x[input_ptr], x13, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x11, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x13, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x11, [x20, #0x0]\n" + "ldr x10, [x20, #0x8]\n" "cbnz x13, 62f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x11, x11, x20, LSL #2\n" - "add x25, x25, x20, LSL #2\n" + "add x10, x10, x20, LSL #2\n" "b 62f\n" "61:" // Height 2: setup direct input "mov x11, %x[input_ptr]\n" - "add x25, x11, x20, LSL #2\n" + "add x10, x11, x21, LSL #2\n" "62:" // Height 2: input setup done "cmp x12, #0x4\n" "blt 65f\n" "ldr q0, [x11, #0x0]\n" "cmp x12, #0x8\n" - "ldr q1, [x25, #0x0]\n" + "ldr q1, [x10, #0x0]\n" "ldr q4, [x15, #0x0]\n" "ldr q5, [x15, #0x10]\n" "ldr q6, [x15, #0x20]\n" @@ -728,239 +727,239 @@ void a64_hybrid_fp32_mla_4x24_a55 ( "blt 64f\n" "63:" // Height 2: Multiply loop: Main loop head "fmla v8.4s, v4.4s, v0.s[0]\n" - "ldr x10, [x15, #0x48]\n" + "ldr x23, [x15, #0x48]\n" "fmla v14.4s, v4.4s, v1.s[0]\n" - "ldr d4, [x15, #0x40]\n" + "ldr d23, [x15, #0x40]\n" "fmla v9.4s, v5.4s, v0.s[0]\n" - "ldr x9, [x15, #0x58]\n" + "ldr x22, [x15, #0x58]\n" "fmla v15.4s, v5.4s, v1.s[0]\n" - "ldr d5, [x15, #0x50]\n" + "ldr d22, [x15, #0x50]\n" "fmla v10.4s, v6.4s, v0.s[0]\n" - "ldr x28, [x15, #0x68]\n" + "ldr x21, [x15, #0x68]\n" "fmla v16.4s, v6.4s, v1.s[0]\n" - "ldr d6, [x15, #0x60]\n" + "ldr d21, [x15, #0x60]\n" "fmla v11.4s, v7.4s, v0.s[0]\n" - "ldr x27, [x15, #0x78]\n" + "ldr x20, [x15, #0x78]\n" "fmla v17.4s, v7.4s, v1.s[0]\n" - "ldr d7, [x15, #0x70]\n" - "mov v4.d[1], x10\n" - "fmla v12.4s, v4.4s, v0.s[0]\n" - "mov v5.d[1], x9\n" - "fmla v18.4s, v4.4s, v1.s[0]\n" - "ldr d4, [x15, #0x80]\n" - "mov v6.d[1], x28\n" - "mov v7.d[1], x27\n" - "ldr x10, [x15, #0x88]\n" - "fmla v13.4s, v5.4s, v0.s[0]\n" - "ldr x9, [x15, #0x98]\n" - "fmla v19.4s, v5.4s, v1.s[0]\n" - "ldr d5, [x15, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "ldr x28, [x15, #0xa8]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "ldr d6, [x15, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "ldr x27, [x15, #0xb8]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "ldr d7, [x15, #0xb0]\n" - "mov v4.d[1], x10\n" - "fmla v10.4s, v4.4s, v0.s[1]\n" - "mov v5.d[1], x9\n" - "fmla v16.4s, v4.4s, v1.s[1]\n" - "ldr d4, [x15, #0xc0]\n" - "mov v6.d[1], x28\n" - "mov v7.d[1], x27\n" - "ldr x10, [x15, #0xc8]\n" - "fmla v11.4s, v5.4s, v0.s[1]\n" - "ldr x9, [x15, #0xd8]\n" - "fmla v17.4s, v5.4s, v1.s[1]\n" - "ldr d5, [x15, #0xd0]\n" - "fmla v12.4s, v6.4s, v0.s[1]\n" - "ldr x28, [x15, #0xe8]\n" - "fmla v18.4s, v6.4s, v1.s[1]\n" - "ldr d6, [x15, #0xe0]\n" - "fmla v13.4s, v7.4s, v0.s[1]\n" - "ldr x27, [x15, #0xf8]\n" - "fmla v19.4s, v7.4s, v1.s[1]\n" - "ldr d7, [x15, #0xf0]\n" - "mov v4.d[1], x10\n" - "fmla v8.4s, v4.4s, v0.s[2]\n" - "mov v5.d[1], x9\n" - "fmla v14.4s, v4.4s, v1.s[2]\n" - "ldr d4, [x15, #0x100]\n" - "mov v6.d[1], x28\n" - "mov v7.d[1], x27\n" - "ldr x10, [x15, #0x108]\n" - "fmla v9.4s, v5.4s, v0.s[2]\n" - "ldr x9, [x15, #0x118]\n" - "fmla v15.4s, v5.4s, v1.s[2]\n" - "ldr d5, [x15, #0x110]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "ldr x28, [x15, #0x128]\n" - "fmla v16.4s, v6.4s, v1.s[2]\n" - "ldr d6, [x15, #0x120]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "ldr x27, [x15, #0x138]\n" - "fmla v17.4s, v7.4s, v1.s[2]\n" - "ldr d7, [x15, #0x130]\n" - "mov v4.d[1], x10\n" - "fmla v12.4s, v4.4s, v0.s[2]\n" - "mov v5.d[1], x9\n" - "fmla v18.4s, v4.4s, v1.s[2]\n" - "ldr d4, [x15, #0x140]\n" - "mov v6.d[1], x28\n" - "mov v7.d[1], x27\n" - "ldr x10, [x15, #0x148]\n" - "fmla v13.4s, v5.4s, v0.s[2]\n" - "ldr x9, [x15, #0x158]\n" - "fmla v19.4s, v5.4s, v1.s[2]\n" - "ldr d5, [x15, #0x150]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "ldr x28, [x15, #0x168]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "ldr d6, [x15, #0x160]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "ldr x27, [x15, #0x178]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" - "ldr d7, [x15, #0x170]\n" - "mov v4.d[1], x10\n" + "ldr d20, [x15, #0x70]\n" + "mov v23.d[1], x23\n" + "fmla v12.4s, v23.4s, v0.s[0]\n" + "mov v22.d[1], x22\n" + "fmla v18.4s, v23.4s, v1.s[0]\n" + "ldr d23, [x15, #0x80]\n" + "mov v21.d[1], x21\n" + "mov v20.d[1], x20\n" + "ldr x23, [x15, #0x88]\n" + "fmla v13.4s, v22.4s, v0.s[0]\n" + "ldr x22, [x15, #0x98]\n" + "fmla v19.4s, v22.4s, v1.s[0]\n" + "ldr d22, [x15, #0x90]\n" + "fmla v8.4s, v21.4s, v0.s[1]\n" + "ldr x21, [x15, #0xa8]\n" + "fmla v14.4s, v21.4s, v1.s[1]\n" + "ldr d21, [x15, #0xa0]\n" + "fmla v9.4s, v20.4s, v0.s[1]\n" + "ldr x20, [x15, #0xb8]\n" + "fmla v15.4s, v20.4s, v1.s[1]\n" + "ldr d20, [x15, #0xb0]\n" + "mov v23.d[1], x23\n" + "fmla v10.4s, v23.4s, v0.s[1]\n" + "mov v22.d[1], x22\n" + "fmla v16.4s, v23.4s, v1.s[1]\n" + "ldr d23, [x15, #0xc0]\n" + "mov v21.d[1], x21\n" + "mov v20.d[1], x20\n" + "ldr x23, [x15, #0xc8]\n" + "fmla v11.4s, v22.4s, v0.s[1]\n" + "ldr x22, [x15, #0xd8]\n" + "fmla v17.4s, v22.4s, v1.s[1]\n" + "ldr d22, [x15, #0xd0]\n" + "fmla v12.4s, v21.4s, v0.s[1]\n" + "ldr x21, [x15, #0xe8]\n" + "fmla v18.4s, v21.4s, v1.s[1]\n" + "ldr d21, [x15, #0xe0]\n" + "fmla v13.4s, v20.4s, v0.s[1]\n" + "ldr x20, [x15, #0xf8]\n" + "fmla v19.4s, v20.4s, v1.s[1]\n" + "ldr d20, [x15, #0xf0]\n" + "mov v23.d[1], x23\n" + "fmla v8.4s, v23.4s, v0.s[2]\n" + "mov v22.d[1], x22\n" + "fmla v14.4s, v23.4s, v1.s[2]\n" + "ldr d23, [x15, #0x100]\n" + "mov v21.d[1], x21\n" + "mov v20.d[1], x20\n" + "ldr x23, [x15, #0x108]\n" + "fmla v9.4s, v22.4s, v0.s[2]\n" + "ldr x22, [x15, #0x118]\n" + "fmla v15.4s, v22.4s, v1.s[2]\n" + "ldr d22, [x15, #0x110]\n" + "fmla v10.4s, v21.4s, v0.s[2]\n" + "ldr x21, [x15, #0x128]\n" + "fmla v16.4s, v21.4s, v1.s[2]\n" + "ldr d21, [x15, #0x120]\n" + "fmla v11.4s, v20.4s, v0.s[2]\n" + "ldr x20, [x15, #0x138]\n" + "fmla v17.4s, v20.4s, v1.s[2]\n" + "ldr d20, [x15, #0x130]\n" + "mov v23.d[1], x23\n" + "fmla v12.4s, v23.4s, v0.s[2]\n" + "mov v22.d[1], x22\n" + "fmla v18.4s, v23.4s, v1.s[2]\n" + "ldr d23, [x15, #0x140]\n" + "mov v21.d[1], x21\n" + "mov v20.d[1], x20\n" + "ldr x23, [x15, #0x148]\n" + "fmla v13.4s, v22.4s, v0.s[2]\n" + "ldr x22, [x15, #0x158]\n" + "fmla v19.4s, v22.4s, v1.s[2]\n" + "ldr d22, [x15, #0x150]\n" + "fmla v8.4s, v21.4s, v0.s[3]\n" + "ldr x21, [x15, #0x168]\n" + "fmla v14.4s, v21.4s, v1.s[3]\n" + "ldr d21, [x15, #0x160]\n" + "fmla v9.4s, v20.4s, v0.s[3]\n" + "ldr x20, [x15, #0x178]\n" + "fmla v15.4s, v20.4s, v1.s[3]\n" + "ldr d20, [x15, #0x170]\n" + "mov v23.d[1], x23\n" "add x11, x11, #0x10\n" - "mov v5.d[1], x9\n" - "add x25, x25, #0x10\n" - "mov v6.d[1], x28\n" + "mov v22.d[1], x22\n" + "add x10, x10, #0x10\n" + "mov v21.d[1], x21\n" "add x15, x15, #0x180\n" - "mov v7.d[1], x27\n" - "fmla v10.4s, v4.4s, v0.s[3]\n" - "fmla v16.4s, v4.4s, v1.s[3]\n" + "mov v20.d[1], x20\n" + "fmla v10.4s, v23.4s, v0.s[3]\n" + "fmla v16.4s, v23.4s, v1.s[3]\n" "ldr d4, [x15, #0x0]\n" - "ldr x10, [x15, #0x8]\n" - "fmla v11.4s, v5.4s, v0.s[3]\n" - "fmla v17.4s, v5.4s, v1.s[3]\n" + "ldr x21, [x15, #0x8]\n" + "fmla v11.4s, v22.4s, v0.s[3]\n" + "fmla v17.4s, v22.4s, v1.s[3]\n" "ldr d5, [x15, #0x10]\n" - "ldr x9, [x15, #0x18]\n" - "fmla v12.4s, v6.4s, v0.s[3]\n" - "fmla v18.4s, v6.4s, v1.s[3]\n" + "ldr x20, [x15, #0x18]\n" + "fmla v12.4s, v21.4s, v0.s[3]\n" + "fmla v18.4s, v21.4s, v1.s[3]\n" "ldr d6, [x15, #0x20]\n" - "ldr x28, [x15, #0x28]\n" - "fmla v13.4s, v7.4s, v0.s[3]\n" + "ldr x23, [x15, #0x28]\n" + "fmla v13.4s, v20.4s, v0.s[3]\n" "ldr d0, [x11, #0x0]\n" - "fmla v19.4s, v7.4s, v1.s[3]\n" - "ldr d1, [x25, #0x0]\n" + "fmla v19.4s, v20.4s, v1.s[3]\n" + "ldr d1, [x10, #0x0]\n" "sub x12, x12, #0x4\n" "ldr d7, [x15, #0x30]\n" "cmp x12, #0x8\n" - "ldr x26, [x11, #0x8]\n" - "mov v4.d[1], x10\n" - "ldr x24, [x25, #0x8]\n" - "mov v5.d[1], x9\n" - "ldr x27, [x15, #0x38]\n" - "mov v6.d[1], x28\n" + "ldr x22, [x11, #0x8]\n" + "mov v4.d[1], x21\n" + "ldr x21, [x10, #0x8]\n" + "mov v5.d[1], x20\n" + "ldr x20, [x15, #0x38]\n" + "mov v6.d[1], x23\n" "prfm pldl1keep, [x11, #0x80]\n" - "mov v0.d[1], x26\n" - "prfm pldl1keep, [x25, #0x80]\n" - "mov v1.d[1], x24\n" - "mov v7.d[1], x27\n" + "mov v0.d[1], x22\n" + "prfm pldl1keep, [x10, #0x80]\n" + "mov v1.d[1], x21\n" + "mov v7.d[1], x20\n" "bge 63b\n" "64:" // Height 2: Multiply loop: Single iteration only "fmla v8.4s, v4.4s, v0.s[0]\n" "add x11, x11, #0x10\n" "fmla v14.4s, v4.4s, v1.s[0]\n" - "ldr q4, [x15, #0x40]\n" + "ldr q23, [x15, #0x40]\n" "fmla v9.4s, v5.4s, v0.s[0]\n" - "add x25, x25, #0x10\n" + "add x10, x10, #0x10\n" "fmla v15.4s, v5.4s, v1.s[0]\n" - "ldr q5, [x15, #0x50]\n" + "ldr q22, [x15, #0x50]\n" "fmla v10.4s, v6.4s, v0.s[0]\n" "sub x12, x12, #0x4\n" "fmla v16.4s, v6.4s, v1.s[0]\n" - "ldr q6, [x15, #0x60]\n" + "ldr q21, [x15, #0x60]\n" "fmla v11.4s, v7.4s, v0.s[0]\n" "prfm pldl1keep, [x11, #0x80]\n" "fmla v17.4s, v7.4s, v1.s[0]\n" - "ldr q7, [x15, #0x70]\n" - "fmla v12.4s, v4.4s, v0.s[0]\n" - "prfm pldl1keep, [x25, #0x80]\n" - "fmla v18.4s, v4.4s, v1.s[0]\n" - "ldr q4, [x15, #0x80]\n" - "fmla v13.4s, v5.4s, v0.s[0]\n" - "fmla v19.4s, v5.4s, v1.s[0]\n" - "ldr q5, [x15, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "ldr q6, [x15, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "ldr q7, [x15, #0xb0]\n" - "fmla v10.4s, v4.4s, v0.s[1]\n" - "fmla v16.4s, v4.4s, v1.s[1]\n" - "ldr q4, [x15, #0xc0]\n" - "fmla v11.4s, v5.4s, v0.s[1]\n" - "fmla v17.4s, v5.4s, v1.s[1]\n" - "ldr q5, [x15, #0xd0]\n" - "fmla v12.4s, v6.4s, v0.s[1]\n" - "fmla v18.4s, v6.4s, v1.s[1]\n" - "ldr q6, [x15, #0xe0]\n" - "fmla v13.4s, v7.4s, v0.s[1]\n" - "fmla v19.4s, v7.4s, v1.s[1]\n" - "ldr q7, [x15, #0xf0]\n" - "fmla v8.4s, v4.4s, v0.s[2]\n" - "fmla v14.4s, v4.4s, v1.s[2]\n" - "ldr q4, [x15, #0x100]\n" - "fmla v9.4s, v5.4s, v0.s[2]\n" - "fmla v15.4s, v5.4s, v1.s[2]\n" - "ldr q5, [x15, #0x110]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v16.4s, v6.4s, v1.s[2]\n" - "ldr q6, [x15, #0x120]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "fmla v17.4s, v7.4s, v1.s[2]\n" - "ldr q7, [x15, #0x130]\n" - "fmla v12.4s, v4.4s, v0.s[2]\n" - "fmla v18.4s, v4.4s, v1.s[2]\n" - "ldr q4, [x15, #0x140]\n" - "fmla v13.4s, v5.4s, v0.s[2]\n" - "fmla v19.4s, v5.4s, v1.s[2]\n" - "ldr q5, [x15, #0x150]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "ldr q6, [x15, #0x160]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" - "ldr q7, [x15, #0x170]\n" - "fmla v10.4s, v4.4s, v0.s[3]\n" + "ldr q20, [x15, #0x70]\n" + "fmla v12.4s, v23.4s, v0.s[0]\n" + "prfm pldl1keep, [x10, #0x80]\n" + "fmla v18.4s, v23.4s, v1.s[0]\n" + "ldr q23, [x15, #0x80]\n" + "fmla v13.4s, v22.4s, v0.s[0]\n" + "fmla v19.4s, v22.4s, v1.s[0]\n" + "ldr q22, [x15, #0x90]\n" + "fmla v8.4s, v21.4s, v0.s[1]\n" + "fmla v14.4s, v21.4s, v1.s[1]\n" + "ldr q21, [x15, #0xa0]\n" + "fmla v9.4s, v20.4s, v0.s[1]\n" + "fmla v15.4s, v20.4s, v1.s[1]\n" + "ldr q20, [x15, #0xb0]\n" + "fmla v10.4s, v23.4s, v0.s[1]\n" + "fmla v16.4s, v23.4s, v1.s[1]\n" + "ldr q23, [x15, #0xc0]\n" + "fmla v11.4s, v22.4s, v0.s[1]\n" + "fmla v17.4s, v22.4s, v1.s[1]\n" + "ldr q22, [x15, #0xd0]\n" + "fmla v12.4s, v21.4s, v0.s[1]\n" + "fmla v18.4s, v21.4s, v1.s[1]\n" + "ldr q21, [x15, #0xe0]\n" + "fmla v13.4s, v20.4s, v0.s[1]\n" + "fmla v19.4s, v20.4s, v1.s[1]\n" + "ldr q20, [x15, #0xf0]\n" + "fmla v8.4s, v23.4s, v0.s[2]\n" + "fmla v14.4s, v23.4s, v1.s[2]\n" + "ldr q23, [x15, #0x100]\n" + "fmla v9.4s, v22.4s, v0.s[2]\n" + "fmla v15.4s, v22.4s, v1.s[2]\n" + "ldr q22, [x15, #0x110]\n" + "fmla v10.4s, v21.4s, v0.s[2]\n" + "fmla v16.4s, v21.4s, v1.s[2]\n" + "ldr q21, [x15, #0x120]\n" + "fmla v11.4s, v20.4s, v0.s[2]\n" + "fmla v17.4s, v20.4s, v1.s[2]\n" + "ldr q20, [x15, #0x130]\n" + "fmla v12.4s, v23.4s, v0.s[2]\n" + "fmla v18.4s, v23.4s, v1.s[2]\n" + "ldr q23, [x15, #0x140]\n" + "fmla v13.4s, v22.4s, v0.s[2]\n" + "fmla v19.4s, v22.4s, v1.s[2]\n" + "ldr q22, [x15, #0x150]\n" + "fmla v8.4s, v21.4s, v0.s[3]\n" + "fmla v14.4s, v21.4s, v1.s[3]\n" + "ldr q21, [x15, #0x160]\n" + "fmla v9.4s, v20.4s, v0.s[3]\n" + "fmla v15.4s, v20.4s, v1.s[3]\n" + "ldr q20, [x15, #0x170]\n" + "fmla v10.4s, v23.4s, v0.s[3]\n" "add x15, x15, #0x180\n" - "fmla v16.4s, v4.4s, v1.s[3]\n" - "fmla v11.4s, v5.4s, v0.s[3]\n" - "fmla v17.4s, v5.4s, v1.s[3]\n" - "fmla v12.4s, v6.4s, v0.s[3]\n" - "fmla v18.4s, v6.4s, v1.s[3]\n" - "fmla v13.4s, v7.4s, v0.s[3]\n" - "fmla v19.4s, v7.4s, v1.s[3]\n" + "fmla v16.4s, v23.4s, v1.s[3]\n" + "fmla v11.4s, v22.4s, v0.s[3]\n" + "fmla v17.4s, v22.4s, v1.s[3]\n" + "fmla v12.4s, v21.4s, v0.s[3]\n" + "fmla v18.4s, v21.4s, v1.s[3]\n" + "fmla v13.4s, v20.4s, v0.s[3]\n" + "fmla v19.4s, v20.4s, v1.s[3]\n" "65:" // Height 2: Multiply loop: Main loop skip "cbz x12, 67f\n" "66:" // Height 2: Multiply loop: Odd block loop - "ldr s0, [x11], #0x4\n" + "ldr s25, [x11], #0x4\n" "sub x12, x12, #0x1\n" - "ldr s1, [x25], #0x4\n" - "ldr q4, [x15, #0x0]\n" - "fmla v8.4s, v4.4s, v0.s[0]\n" - "ldr q5, [x15, #0x10]\n" - "fmla v14.4s, v4.4s, v1.s[0]\n" - "ldr q6, [x15, #0x20]\n" - "fmla v9.4s, v5.4s, v0.s[0]\n" - "ldr q7, [x15, #0x30]\n" - "fmla v15.4s, v5.4s, v1.s[0]\n" - "ldr q4, [x15, #0x40]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "ldr q5, [x15, #0x50]\n" - "fmla v16.4s, v6.4s, v1.s[0]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" + "ldr s24, [x10], #0x4\n" + "ldr q21, [x15, #0x0]\n" + "fmla v8.4s, v21.4s, v25.s[0]\n" + "ldr q20, [x15, #0x10]\n" + "fmla v14.4s, v21.4s, v24.s[0]\n" + "ldr q23, [x15, #0x20]\n" + "fmla v9.4s, v20.4s, v25.s[0]\n" + "ldr q22, [x15, #0x30]\n" + "fmla v15.4s, v20.4s, v24.s[0]\n" + "ldr q21, [x15, #0x40]\n" + "fmla v10.4s, v23.4s, v25.s[0]\n" + "ldr q20, [x15, #0x50]\n" + "fmla v16.4s, v23.4s, v24.s[0]\n" + "fmla v11.4s, v22.4s, v25.s[0]\n" "add x15, x15, #0x60\n" - "fmla v17.4s, v7.4s, v1.s[0]\n" - "fmla v12.4s, v4.4s, v0.s[0]\n" - "fmla v18.4s, v4.4s, v1.s[0]\n" - "fmla v13.4s, v5.4s, v0.s[0]\n" - "fmla v19.4s, v5.4s, v1.s[0]\n" + "fmla v17.4s, v22.4s, v24.s[0]\n" + "fmla v12.4s, v21.4s, v25.s[0]\n" + "fmla v18.4s, v21.4s, v24.s[0]\n" + "fmla v13.4s, v20.4s, v25.s[0]\n" + "fmla v19.4s, v20.4s, v24.s[0]\n" "cbnz x12, 66b\n" "67:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -973,33 +972,33 @@ void a64_hybrid_fp32_mla_4x24_a55 ( "prfm pstl1keep, [x23, #0x0]\n" "tbz %x[flags], #1, 68f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v0.4s\n" - "fmin v9.4s, v9.4s, v0.4s\n" - "fmin v10.4s, v10.4s, v0.4s\n" - "fmin v11.4s, v11.4s, v0.4s\n" - "fmin v12.4s, v12.4s, v0.4s\n" - "fmin v13.4s, v13.4s, v0.4s\n" - "fmin v14.4s, v14.4s, v0.4s\n" - "fmin v15.4s, v15.4s, v0.4s\n" - "fmin v16.4s, v16.4s, v0.4s\n" - "fmin v17.4s, v17.4s, v0.4s\n" - "fmin v18.4s, v18.4s, v0.4s\n" - "fmin v19.4s, v19.4s, v0.4s\n" + "ld1r { v20.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v20.4s\n" + "fmin v9.4s, v9.4s, v20.4s\n" + "fmin v10.4s, v10.4s, v20.4s\n" + "fmin v11.4s, v11.4s, v20.4s\n" + "fmin v12.4s, v12.4s, v20.4s\n" + "fmin v13.4s, v13.4s, v20.4s\n" + "fmin v14.4s, v14.4s, v20.4s\n" + "fmin v15.4s, v15.4s, v20.4s\n" + "fmin v16.4s, v16.4s, v20.4s\n" + "fmin v17.4s, v17.4s, v20.4s\n" + "fmin v18.4s, v18.4s, v20.4s\n" + "fmin v19.4s, v19.4s, v20.4s\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" - "fmax v16.4s, v16.4s, v0.4s\n" - "fmax v17.4s, v17.4s, v0.4s\n" - "fmax v18.4s, v18.4s, v0.4s\n" - "fmax v19.4s, v19.4s, v0.4s\n" + "ld1r { v20.4s }, [x20]\n" + "fmax v8.4s, v8.4s, v20.4s\n" + "fmax v9.4s, v9.4s, v20.4s\n" + "fmax v10.4s, v10.4s, v20.4s\n" + "fmax v11.4s, v11.4s, v20.4s\n" + "fmax v12.4s, v12.4s, v20.4s\n" + "fmax v13.4s, v13.4s, v20.4s\n" + "fmax v14.4s, v14.4s, v20.4s\n" + "fmax v15.4s, v15.4s, v20.4s\n" + "fmax v16.4s, v16.4s, v20.4s\n" + "fmax v17.4s, v17.4s, v20.4s\n" + "fmax v18.4s, v18.4s, v20.4s\n" + "fmax v19.4s, v19.4s, v20.4s\n" "68:" // Height 2: No activation "cmp x16, #0x18\n" "bge 81f\n" @@ -1339,30 +1338,30 @@ void a64_hybrid_fp32_mla_4x24_a55 ( "101:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w12, [x20, x13, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 102f\n" - "ldr x21, [%x[input_ptr], x13, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x11, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x23, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x13, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x11, [x20, #0x0]\n" + "ldr x10, [x20, #0x8]\n" + "ldr x9, [x20, #0x10]\n" "cbnz x13, 103f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x11, x11, x20, LSL #2\n" - "add x25, x25, x20, LSL #2\n" - "add x23, x23, x20, LSL #2\n" + "add x10, x10, x20, LSL #2\n" + "add x9, x9, x20, LSL #2\n" "b 103f\n" "102:" // Height 3: setup direct input "mov x11, %x[input_ptr]\n" - "add x25, x11, x20, LSL #2\n" - "add x23, x25, x20, LSL #2\n" + "add x10, x11, x21, LSL #2\n" + "add x9, x10, x21, LSL #2\n" "103:" // Height 3: input setup done "cmp x12, #0x4\n" "blt 106f\n" "ldr q0, [x11, #0x0]\n" "cmp x12, #0x8\n" - "ldr q1, [x25, #0x0]\n" - "ldr q2, [x23, #0x0]\n" + "ldr q1, [x10, #0x0]\n" + "ldr q2, [x9, #0x0]\n" "ldr q4, [x15, #0x0]\n" "ldr q5, [x15, #0x10]\n" "ldr q6, [x15, #0x20]\n" @@ -1370,301 +1369,301 @@ void a64_hybrid_fp32_mla_4x24_a55 ( "blt 105f\n" "104:" // Height 3: Multiply loop: Main loop head "fmla v8.4s, v4.4s, v0.s[0]\n" - "ldr x10, [x15, #0x48]\n" + "ldr x23, [x15, #0x48]\n" "fmla v14.4s, v4.4s, v1.s[0]\n" - "ldr x9, [x15, #0x58]\n" + "ldr x22, [x15, #0x58]\n" "fmla v20.4s, v4.4s, v2.s[0]\n" - "ldr d4, [x15, #0x40]\n" + "ldr d29, [x15, #0x40]\n" "fmla v9.4s, v5.4s, v0.s[0]\n" - "ldr x28, [x15, #0x68]\n" + "ldr x21, [x15, #0x68]\n" "fmla v15.4s, v5.4s, v1.s[0]\n" - "ldr x27, [x15, #0x78]\n" + "ldr x20, [x15, #0x78]\n" "fmla v21.4s, v5.4s, v2.s[0]\n" - "ldr d5, [x15, #0x50]\n" + "ldr d28, [x15, #0x50]\n" "fmla v10.4s, v6.4s, v0.s[0]\n" - "mov v4.d[1], x10\n" + "mov v29.d[1], x23\n" "fmla v16.4s, v6.4s, v1.s[0]\n" - "mov v5.d[1], x9\n" + "mov v28.d[1], x22\n" "fmla v22.4s, v6.4s, v2.s[0]\n" - "ldr d6, [x15, #0x60]\n" + "ldr d27, [x15, #0x60]\n" "fmla v11.4s, v7.4s, v0.s[0]\n" - "mov v6.d[1], x28\n" + "mov v27.d[1], x21\n" "fmla v17.4s, v7.4s, v1.s[0]\n" - "ldr x10, [x15, #0x88]\n" + "ldr x23, [x15, #0x88]\n" "fmla v23.4s, v7.4s, v2.s[0]\n" - "ldr d7, [x15, #0x70]\n" - "mov v7.d[1], x27\n" - "fmla v12.4s, v4.4s, v0.s[0]\n" - "fmla v18.4s, v4.4s, v1.s[0]\n" - "ldr x9, [x15, #0x98]\n" - "fmla v24.4s, v4.4s, v2.s[0]\n" - "ldr d4, [x15, #0x80]\n" - "fmla v13.4s, v5.4s, v0.s[0]\n" - "ldr x28, [x15, #0xa8]\n" - "fmla v19.4s, v5.4s, v1.s[0]\n" - "ldr x27, [x15, #0xb8]\n" - "fmla v25.4s, v5.4s, v2.s[0]\n" - "ldr d5, [x15, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "mov v4.d[1], x10\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "mov v5.d[1], x9\n" - "fmla v20.4s, v6.4s, v2.s[1]\n" - "ldr d6, [x15, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "mov v6.d[1], x28\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "ldr x10, [x15, #0xc8]\n" - "fmla v21.4s, v7.4s, v2.s[1]\n" - "ldr d7, [x15, #0xb0]\n" - "mov v7.d[1], x27\n" - "fmla v10.4s, v4.4s, v0.s[1]\n" - "fmla v16.4s, v4.4s, v1.s[1]\n" - "ldr x9, [x15, #0xd8]\n" - "fmla v22.4s, v4.4s, v2.s[1]\n" - "ldr d4, [x15, #0xc0]\n" - "fmla v11.4s, v5.4s, v0.s[1]\n" - "ldr x28, [x15, #0xe8]\n" - "fmla v17.4s, v5.4s, v1.s[1]\n" - "ldr x27, [x15, #0xf8]\n" - "fmla v23.4s, v5.4s, v2.s[1]\n" - "ldr d5, [x15, #0xd0]\n" - "fmla v12.4s, v6.4s, v0.s[1]\n" - "mov v4.d[1], x10\n" - "fmla v18.4s, v6.4s, v1.s[1]\n" - "mov v5.d[1], x9\n" - "fmla v24.4s, v6.4s, v2.s[1]\n" - "ldr d6, [x15, #0xe0]\n" - "fmla v13.4s, v7.4s, v0.s[1]\n" - "mov v6.d[1], x28\n" - "fmla v19.4s, v7.4s, v1.s[1]\n" - "ldr x10, [x15, #0x108]\n" - "fmla v25.4s, v7.4s, v2.s[1]\n" - "ldr d7, [x15, #0xf0]\n" - "mov v7.d[1], x27\n" - "fmla v8.4s, v4.4s, v0.s[2]\n" - "fmla v14.4s, v4.4s, v1.s[2]\n" - "ldr x9, [x15, #0x118]\n" - "fmla v20.4s, v4.4s, v2.s[2]\n" - "ldr d4, [x15, #0x100]\n" - "fmla v9.4s, v5.4s, v0.s[2]\n" - "ldr x28, [x15, #0x128]\n" - "fmla v15.4s, v5.4s, v1.s[2]\n" - "ldr x27, [x15, #0x138]\n" - "fmla v21.4s, v5.4s, v2.s[2]\n" - "ldr d5, [x15, #0x110]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "mov v4.d[1], x10\n" - "fmla v16.4s, v6.4s, v1.s[2]\n" - "mov v5.d[1], x9\n" - "fmla v22.4s, v6.4s, v2.s[2]\n" - "ldr d6, [x15, #0x120]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "mov v6.d[1], x28\n" - "fmla v17.4s, v7.4s, v1.s[2]\n" - "ldr x10, [x15, #0x148]\n" - "fmla v23.4s, v7.4s, v2.s[2]\n" - "ldr d7, [x15, #0x130]\n" - "mov v7.d[1], x27\n" - "fmla v12.4s, v4.4s, v0.s[2]\n" - "fmla v18.4s, v4.4s, v1.s[2]\n" - "ldr x9, [x15, #0x158]\n" - "fmla v24.4s, v4.4s, v2.s[2]\n" - "ldr d4, [x15, #0x140]\n" - "fmla v13.4s, v5.4s, v0.s[2]\n" - "ldr x28, [x15, #0x168]\n" - "fmla v19.4s, v5.4s, v1.s[2]\n" - "ldr x27, [x15, #0x178]\n" - "fmla v25.4s, v5.4s, v2.s[2]\n" - "ldr d5, [x15, #0x150]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "mov v4.d[1], x10\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "mov v5.d[1], x9\n" - "fmla v20.4s, v6.4s, v2.s[3]\n" - "ldr d6, [x15, #0x160]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "mov v6.d[1], x28\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" + "ldr d26, [x15, #0x70]\n" + "mov v26.d[1], x20\n" + "fmla v12.4s, v29.4s, v0.s[0]\n" + "fmla v18.4s, v29.4s, v1.s[0]\n" + "ldr x22, [x15, #0x98]\n" + "fmla v24.4s, v29.4s, v2.s[0]\n" + "ldr d29, [x15, #0x80]\n" + "fmla v13.4s, v28.4s, v0.s[0]\n" + "ldr x21, [x15, #0xa8]\n" + "fmla v19.4s, v28.4s, v1.s[0]\n" + "ldr x20, [x15, #0xb8]\n" + "fmla v25.4s, v28.4s, v2.s[0]\n" + "ldr d28, [x15, #0x90]\n" + "fmla v8.4s, v27.4s, v0.s[1]\n" + "mov v29.d[1], x23\n" + "fmla v14.4s, v27.4s, v1.s[1]\n" + "mov v28.d[1], x22\n" + "fmla v20.4s, v27.4s, v2.s[1]\n" + "ldr d27, [x15, #0xa0]\n" + "fmla v9.4s, v26.4s, v0.s[1]\n" + "mov v27.d[1], x21\n" + "fmla v15.4s, v26.4s, v1.s[1]\n" + "ldr x23, [x15, #0xc8]\n" + "fmla v21.4s, v26.4s, v2.s[1]\n" + "ldr d26, [x15, #0xb0]\n" + "mov v26.d[1], x20\n" + "fmla v10.4s, v29.4s, v0.s[1]\n" + "fmla v16.4s, v29.4s, v1.s[1]\n" + "ldr x22, [x15, #0xd8]\n" + "fmla v22.4s, v29.4s, v2.s[1]\n" + "ldr d29, [x15, #0xc0]\n" + "fmla v11.4s, v28.4s, v0.s[1]\n" + "ldr x21, [x15, #0xe8]\n" + "fmla v17.4s, v28.4s, v1.s[1]\n" + "ldr x20, [x15, #0xf8]\n" + "fmla v23.4s, v28.4s, v2.s[1]\n" + "ldr d28, [x15, #0xd0]\n" + "fmla v12.4s, v27.4s, v0.s[1]\n" + "mov v29.d[1], x23\n" + "fmla v18.4s, v27.4s, v1.s[1]\n" + "mov v28.d[1], x22\n" + "fmla v24.4s, v27.4s, v2.s[1]\n" + "ldr d27, [x15, #0xe0]\n" + "fmla v13.4s, v26.4s, v0.s[1]\n" + "mov v27.d[1], x21\n" + "fmla v19.4s, v26.4s, v1.s[1]\n" + "ldr x23, [x15, #0x108]\n" + "fmla v25.4s, v26.4s, v2.s[1]\n" + "ldr d26, [x15, #0xf0]\n" + "mov v26.d[1], x20\n" + "fmla v8.4s, v29.4s, v0.s[2]\n" + "fmla v14.4s, v29.4s, v1.s[2]\n" + "ldr x22, [x15, #0x118]\n" + "fmla v20.4s, v29.4s, v2.s[2]\n" + "ldr d29, [x15, #0x100]\n" + "fmla v9.4s, v28.4s, v0.s[2]\n" + "ldr x21, [x15, #0x128]\n" + "fmla v15.4s, v28.4s, v1.s[2]\n" + "ldr x20, [x15, #0x138]\n" + "fmla v21.4s, v28.4s, v2.s[2]\n" + "ldr d28, [x15, #0x110]\n" + "fmla v10.4s, v27.4s, v0.s[2]\n" + "mov v29.d[1], x23\n" + "fmla v16.4s, v27.4s, v1.s[2]\n" + "mov v28.d[1], x22\n" + "fmla v22.4s, v27.4s, v2.s[2]\n" + "ldr d27, [x15, #0x120]\n" + "fmla v11.4s, v26.4s, v0.s[2]\n" + "mov v27.d[1], x21\n" + "fmla v17.4s, v26.4s, v1.s[2]\n" + "ldr x23, [x15, #0x148]\n" + "fmla v23.4s, v26.4s, v2.s[2]\n" + "ldr d26, [x15, #0x130]\n" + "mov v26.d[1], x20\n" + "fmla v12.4s, v29.4s, v0.s[2]\n" + "fmla v18.4s, v29.4s, v1.s[2]\n" + "ldr x22, [x15, #0x158]\n" + "fmla v24.4s, v29.4s, v2.s[2]\n" + "ldr d29, [x15, #0x140]\n" + "fmla v13.4s, v28.4s, v0.s[2]\n" + "ldr x21, [x15, #0x168]\n" + "fmla v19.4s, v28.4s, v1.s[2]\n" + "ldr x20, [x15, #0x178]\n" + "fmla v25.4s, v28.4s, v2.s[2]\n" + "ldr d28, [x15, #0x150]\n" + "fmla v8.4s, v27.4s, v0.s[3]\n" + "mov v29.d[1], x23\n" + "fmla v14.4s, v27.4s, v1.s[3]\n" + "mov v28.d[1], x22\n" + "fmla v20.4s, v27.4s, v2.s[3]\n" + "ldr d27, [x15, #0x160]\n" + "fmla v9.4s, v26.4s, v0.s[3]\n" + "mov v27.d[1], x21\n" + "fmla v15.4s, v26.4s, v1.s[3]\n" "add x11, x11, #0x10\n" - "fmla v21.4s, v7.4s, v2.s[3]\n" - "ldr d7, [x15, #0x170]\n" - "mov v7.d[1], x27\n" - "add x25, x25, #0x10\n" - "add x23, x23, #0x10\n" + "fmla v21.4s, v26.4s, v2.s[3]\n" + "ldr d26, [x15, #0x170]\n" + "mov v26.d[1], x20\n" + "add x10, x10, #0x10\n" + "add x9, x9, #0x10\n" "add x15, x15, #0x180\n" - "fmla v10.4s, v4.4s, v0.s[3]\n" - "ldr x10, [x15, #0x8]\n" - "fmla v16.4s, v4.4s, v1.s[3]\n" - "ldr x9, [x15, #0x18]\n" - "fmla v22.4s, v4.4s, v2.s[3]\n" + "fmla v10.4s, v29.4s, v0.s[3]\n" + "ldr x26, [x15, #0x8]\n" + "fmla v16.4s, v29.4s, v1.s[3]\n" + "ldr x25, [x15, #0x18]\n" + "fmla v22.4s, v29.4s, v2.s[3]\n" "ldr d4, [x15, #0x0]\n" - "fmla v11.4s, v5.4s, v0.s[3]\n" - "ldr x28, [x15, #0x28]\n" - "fmla v17.4s, v5.4s, v1.s[3]\n" - "ldr x26, [x11, #0x8]\n" - "fmla v23.4s, v5.4s, v2.s[3]\n" + "fmla v11.4s, v28.4s, v0.s[3]\n" + "ldr x24, [x15, #0x28]\n" + "fmla v17.4s, v28.4s, v1.s[3]\n" + "ldr x23, [x11, #0x8]\n" + "fmla v23.4s, v28.4s, v2.s[3]\n" "ldr d5, [x15, #0x10]\n" - "fmla v12.4s, v6.4s, v0.s[3]\n" - "ldr x24, [x25, #0x8]\n" - "fmla v18.4s, v6.4s, v1.s[3]\n" - "ldr x22, [x23, #0x8]\n" - "fmla v24.4s, v6.4s, v2.s[3]\n" + "fmla v12.4s, v27.4s, v0.s[3]\n" + "ldr x22, [x10, #0x8]\n" + "fmla v18.4s, v27.4s, v1.s[3]\n" + "ldr x21, [x9, #0x8]\n" + "fmla v24.4s, v27.4s, v2.s[3]\n" "ldr d6, [x15, #0x20]\n" - "fmla v13.4s, v7.4s, v0.s[3]\n" + "fmla v13.4s, v26.4s, v0.s[3]\n" "ldr d0, [x11, #0x0]\n" - "fmla v19.4s, v7.4s, v1.s[3]\n" - "ldr d1, [x25, #0x0]\n" - "fmla v25.4s, v7.4s, v2.s[3]\n" - "ldr d2, [x23, #0x0]\n" + "fmla v19.4s, v26.4s, v1.s[3]\n" + "ldr d1, [x10, #0x0]\n" + "fmla v25.4s, v26.4s, v2.s[3]\n" + "ldr d2, [x9, #0x0]\n" "ldr d7, [x15, #0x30]\n" "sub x12, x12, #0x4\n" - "ldr x27, [x15, #0x38]\n" + "ldr x20, [x15, #0x38]\n" "cmp x12, #0x8\n" "prfm pldl1keep, [x11, #0x80]\n" - "mov v4.d[1], x10\n" - "prfm pldl1keep, [x25, #0x80]\n" - "mov v5.d[1], x9\n" - "prfm pldl1keep, [x23, #0x80]\n" - "mov v6.d[1], x28\n" - "mov v0.d[1], x26\n" - "mov v1.d[1], x24\n" - "mov v2.d[1], x22\n" - "mov v7.d[1], x27\n" + "mov v4.d[1], x26\n" + "prfm pldl1keep, [x10, #0x80]\n" + "mov v5.d[1], x25\n" + "prfm pldl1keep, [x9, #0x80]\n" + "mov v6.d[1], x24\n" + "mov v0.d[1], x23\n" + "mov v1.d[1], x22\n" + "mov v2.d[1], x21\n" + "mov v7.d[1], x20\n" "bge 104b\n" "105:" // Height 3: Multiply loop: Single iteration only "fmla v8.4s, v4.4s, v0.s[0]\n" "add x11, x11, #0x10\n" "fmla v14.4s, v4.4s, v1.s[0]\n" - "add x25, x25, #0x10\n" + "add x10, x10, #0x10\n" "fmla v20.4s, v4.4s, v2.s[0]\n" - "ldr q4, [x15, #0x40]\n" + "ldr q29, [x15, #0x40]\n" "fmla v9.4s, v5.4s, v0.s[0]\n" - "add x23, x23, #0x10\n" + "add x9, x9, #0x10\n" "fmla v15.4s, v5.4s, v1.s[0]\n" "sub x12, x12, #0x4\n" "fmla v21.4s, v5.4s, v2.s[0]\n" - "ldr q5, [x15, #0x50]\n" + "ldr q28, [x15, #0x50]\n" "fmla v10.4s, v6.4s, v0.s[0]\n" "prfm pldl1keep, [x11, #0x80]\n" "fmla v16.4s, v6.4s, v1.s[0]\n" - "prfm pldl1keep, [x25, #0x80]\n" + "prfm pldl1keep, [x10, #0x80]\n" "fmla v22.4s, v6.4s, v2.s[0]\n" - "ldr q6, [x15, #0x60]\n" + "ldr q27, [x15, #0x60]\n" "fmla v11.4s, v7.4s, v0.s[0]\n" - "prfm pldl1keep, [x23, #0x80]\n" + "prfm pldl1keep, [x9, #0x80]\n" "fmla v17.4s, v7.4s, v1.s[0]\n" "fmla v23.4s, v7.4s, v2.s[0]\n" - "ldr q7, [x15, #0x70]\n" - "fmla v12.4s, v4.4s, v0.s[0]\n" - "fmla v18.4s, v4.4s, v1.s[0]\n" - "fmla v24.4s, v4.4s, v2.s[0]\n" - "ldr q4, [x15, #0x80]\n" - "fmla v13.4s, v5.4s, v0.s[0]\n" - "fmla v19.4s, v5.4s, v1.s[0]\n" - "fmla v25.4s, v5.4s, v2.s[0]\n" - "ldr q5, [x15, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "fmla v20.4s, v6.4s, v2.s[1]\n" - "ldr q6, [x15, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "fmla v21.4s, v7.4s, v2.s[1]\n" - "ldr q7, [x15, #0xb0]\n" - "fmla v10.4s, v4.4s, v0.s[1]\n" - "fmla v16.4s, v4.4s, v1.s[1]\n" - "fmla v22.4s, v4.4s, v2.s[1]\n" - "ldr q4, [x15, #0xc0]\n" - "fmla v11.4s, v5.4s, v0.s[1]\n" - "fmla v17.4s, v5.4s, v1.s[1]\n" - "fmla v23.4s, v5.4s, v2.s[1]\n" - "ldr q5, [x15, #0xd0]\n" - "fmla v12.4s, v6.4s, v0.s[1]\n" - "fmla v18.4s, v6.4s, v1.s[1]\n" - "fmla v24.4s, v6.4s, v2.s[1]\n" - "ldr q6, [x15, #0xe0]\n" - "fmla v13.4s, v7.4s, v0.s[1]\n" - "fmla v19.4s, v7.4s, v1.s[1]\n" - "fmla v25.4s, v7.4s, v2.s[1]\n" - "ldr q7, [x15, #0xf0]\n" - "fmla v8.4s, v4.4s, v0.s[2]\n" - "fmla v14.4s, v4.4s, v1.s[2]\n" - "fmla v20.4s, v4.4s, v2.s[2]\n" - "ldr q4, [x15, #0x100]\n" - "fmla v9.4s, v5.4s, v0.s[2]\n" - "fmla v15.4s, v5.4s, v1.s[2]\n" - "fmla v21.4s, v5.4s, v2.s[2]\n" - "ldr q5, [x15, #0x110]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v16.4s, v6.4s, v1.s[2]\n" - "fmla v22.4s, v6.4s, v2.s[2]\n" - "ldr q6, [x15, #0x120]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "fmla v17.4s, v7.4s, v1.s[2]\n" - "fmla v23.4s, v7.4s, v2.s[2]\n" - "ldr q7, [x15, #0x130]\n" - "fmla v12.4s, v4.4s, v0.s[2]\n" - "fmla v18.4s, v4.4s, v1.s[2]\n" - "fmla v24.4s, v4.4s, v2.s[2]\n" - "ldr q4, [x15, #0x140]\n" - "fmla v13.4s, v5.4s, v0.s[2]\n" - "fmla v19.4s, v5.4s, v1.s[2]\n" - "fmla v25.4s, v5.4s, v2.s[2]\n" - "ldr q5, [x15, #0x150]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "fmla v20.4s, v6.4s, v2.s[3]\n" - "ldr q6, [x15, #0x160]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" - "fmla v21.4s, v7.4s, v2.s[3]\n" - "ldr q7, [x15, #0x170]\n" - "fmla v10.4s, v4.4s, v0.s[3]\n" + "ldr q26, [x15, #0x70]\n" + "fmla v12.4s, v29.4s, v0.s[0]\n" + "fmla v18.4s, v29.4s, v1.s[0]\n" + "fmla v24.4s, v29.4s, v2.s[0]\n" + "ldr q29, [x15, #0x80]\n" + "fmla v13.4s, v28.4s, v0.s[0]\n" + "fmla v19.4s, v28.4s, v1.s[0]\n" + "fmla v25.4s, v28.4s, v2.s[0]\n" + "ldr q28, [x15, #0x90]\n" + "fmla v8.4s, v27.4s, v0.s[1]\n" + "fmla v14.4s, v27.4s, v1.s[1]\n" + "fmla v20.4s, v27.4s, v2.s[1]\n" + "ldr q27, [x15, #0xa0]\n" + "fmla v9.4s, v26.4s, v0.s[1]\n" + "fmla v15.4s, v26.4s, v1.s[1]\n" + "fmla v21.4s, v26.4s, v2.s[1]\n" + "ldr q26, [x15, #0xb0]\n" + "fmla v10.4s, v29.4s, v0.s[1]\n" + "fmla v16.4s, v29.4s, v1.s[1]\n" + "fmla v22.4s, v29.4s, v2.s[1]\n" + "ldr q29, [x15, #0xc0]\n" + "fmla v11.4s, v28.4s, v0.s[1]\n" + "fmla v17.4s, v28.4s, v1.s[1]\n" + "fmla v23.4s, v28.4s, v2.s[1]\n" + "ldr q28, [x15, #0xd0]\n" + "fmla v12.4s, v27.4s, v0.s[1]\n" + "fmla v18.4s, v27.4s, v1.s[1]\n" + "fmla v24.4s, v27.4s, v2.s[1]\n" + "ldr q27, [x15, #0xe0]\n" + "fmla v13.4s, v26.4s, v0.s[1]\n" + "fmla v19.4s, v26.4s, v1.s[1]\n" + "fmla v25.4s, v26.4s, v2.s[1]\n" + "ldr q26, [x15, #0xf0]\n" + "fmla v8.4s, v29.4s, v0.s[2]\n" + "fmla v14.4s, v29.4s, v1.s[2]\n" + "fmla v20.4s, v29.4s, v2.s[2]\n" + "ldr q29, [x15, #0x100]\n" + "fmla v9.4s, v28.4s, v0.s[2]\n" + "fmla v15.4s, v28.4s, v1.s[2]\n" + "fmla v21.4s, v28.4s, v2.s[2]\n" + "ldr q28, [x15, #0x110]\n" + "fmla v10.4s, v27.4s, v0.s[2]\n" + "fmla v16.4s, v27.4s, v1.s[2]\n" + "fmla v22.4s, v27.4s, v2.s[2]\n" + "ldr q27, [x15, #0x120]\n" + "fmla v11.4s, v26.4s, v0.s[2]\n" + "fmla v17.4s, v26.4s, v1.s[2]\n" + "fmla v23.4s, v26.4s, v2.s[2]\n" + "ldr q26, [x15, #0x130]\n" + "fmla v12.4s, v29.4s, v0.s[2]\n" + "fmla v18.4s, v29.4s, v1.s[2]\n" + "fmla v24.4s, v29.4s, v2.s[2]\n" + "ldr q29, [x15, #0x140]\n" + "fmla v13.4s, v28.4s, v0.s[2]\n" + "fmla v19.4s, v28.4s, v1.s[2]\n" + "fmla v25.4s, v28.4s, v2.s[2]\n" + "ldr q28, [x15, #0x150]\n" + "fmla v8.4s, v27.4s, v0.s[3]\n" + "fmla v14.4s, v27.4s, v1.s[3]\n" + "fmla v20.4s, v27.4s, v2.s[3]\n" + "ldr q27, [x15, #0x160]\n" + "fmla v9.4s, v26.4s, v0.s[3]\n" + "fmla v15.4s, v26.4s, v1.s[3]\n" + "fmla v21.4s, v26.4s, v2.s[3]\n" + "ldr q26, [x15, #0x170]\n" + "fmla v10.4s, v29.4s, v0.s[3]\n" "add x15, x15, #0x180\n" - "fmla v16.4s, v4.4s, v1.s[3]\n" - "fmla v22.4s, v4.4s, v2.s[3]\n" - "fmla v11.4s, v5.4s, v0.s[3]\n" - "fmla v17.4s, v5.4s, v1.s[3]\n" - "fmla v23.4s, v5.4s, v2.s[3]\n" - "fmla v12.4s, v6.4s, v0.s[3]\n" - "fmla v18.4s, v6.4s, v1.s[3]\n" - "fmla v24.4s, v6.4s, v2.s[3]\n" - "fmla v13.4s, v7.4s, v0.s[3]\n" - "fmla v19.4s, v7.4s, v1.s[3]\n" - "fmla v25.4s, v7.4s, v2.s[3]\n" + "fmla v16.4s, v29.4s, v1.s[3]\n" + "fmla v22.4s, v29.4s, v2.s[3]\n" + "fmla v11.4s, v28.4s, v0.s[3]\n" + "fmla v17.4s, v28.4s, v1.s[3]\n" + "fmla v23.4s, v28.4s, v2.s[3]\n" + "fmla v12.4s, v27.4s, v0.s[3]\n" + "fmla v18.4s, v27.4s, v1.s[3]\n" + "fmla v24.4s, v27.4s, v2.s[3]\n" + "fmla v13.4s, v26.4s, v0.s[3]\n" + "fmla v19.4s, v26.4s, v1.s[3]\n" + "fmla v25.4s, v26.4s, v2.s[3]\n" "106:" // Height 3: Multiply loop: Main loop skip "cbz x12, 108f\n" "107:" // Height 3: Multiply loop: Odd block loop "ldr s0, [x11], #0x4\n" "sub x12, x12, #0x1\n" - "ldr s1, [x25], #0x4\n" - "ldr s2, [x23], #0x4\n" - "ldr q4, [x15, #0x0]\n" - "fmla v8.4s, v4.4s, v0.s[0]\n" - "ldr q5, [x15, #0x10]\n" - "fmla v14.4s, v4.4s, v1.s[0]\n" - "ldr q6, [x15, #0x20]\n" - "fmla v20.4s, v4.4s, v2.s[0]\n" - "ldr q7, [x15, #0x30]\n" - "fmla v9.4s, v5.4s, v0.s[0]\n" - "ldr q4, [x15, #0x40]\n" - "fmla v15.4s, v5.4s, v1.s[0]\n" - "fmla v21.4s, v5.4s, v2.s[0]\n" - "ldr q5, [x15, #0x50]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" + "ldr s31, [x10], #0x4\n" + "ldr s30, [x9], #0x4\n" + "ldr q27, [x15, #0x0]\n" + "fmla v8.4s, v27.4s, v0.s[0]\n" + "ldr q26, [x15, #0x10]\n" + "fmla v14.4s, v27.4s, v31.s[0]\n" + "ldr q29, [x15, #0x20]\n" + "fmla v20.4s, v27.4s, v30.s[0]\n" + "ldr q28, [x15, #0x30]\n" + "fmla v9.4s, v26.4s, v0.s[0]\n" + "ldr q27, [x15, #0x40]\n" + "fmla v15.4s, v26.4s, v31.s[0]\n" + "fmla v21.4s, v26.4s, v30.s[0]\n" + "ldr q26, [x15, #0x50]\n" + "fmla v10.4s, v29.4s, v0.s[0]\n" "add x15, x15, #0x60\n" - "fmla v16.4s, v6.4s, v1.s[0]\n" - "fmla v22.4s, v6.4s, v2.s[0]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v17.4s, v7.4s, v1.s[0]\n" - "fmla v23.4s, v7.4s, v2.s[0]\n" - "fmla v12.4s, v4.4s, v0.s[0]\n" - "fmla v18.4s, v4.4s, v1.s[0]\n" - "fmla v24.4s, v4.4s, v2.s[0]\n" - "fmla v13.4s, v5.4s, v0.s[0]\n" - "fmla v19.4s, v5.4s, v1.s[0]\n" - "fmla v25.4s, v5.4s, v2.s[0]\n" + "fmla v16.4s, v29.4s, v31.s[0]\n" + "fmla v22.4s, v29.4s, v30.s[0]\n" + "fmla v11.4s, v28.4s, v0.s[0]\n" + "fmla v17.4s, v28.4s, v31.s[0]\n" + "fmla v23.4s, v28.4s, v30.s[0]\n" + "fmla v12.4s, v27.4s, v0.s[0]\n" + "fmla v18.4s, v27.4s, v31.s[0]\n" + "fmla v24.4s, v27.4s, v30.s[0]\n" + "fmla v13.4s, v26.4s, v0.s[0]\n" + "fmla v19.4s, v26.4s, v31.s[0]\n" + "fmla v25.4s, v26.4s, v30.s[0]\n" "cbnz x12, 107b\n" "108:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -1679,45 +1678,45 @@ void a64_hybrid_fp32_mla_4x24_a55 ( "prfm pstl1keep, [x22, #0x0]\n" "tbz %x[flags], #1, 109f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v0.4s\n" - "fmin v9.4s, v9.4s, v0.4s\n" - "fmin v10.4s, v10.4s, v0.4s\n" - "fmin v11.4s, v11.4s, v0.4s\n" - "fmin v12.4s, v12.4s, v0.4s\n" - "fmin v13.4s, v13.4s, v0.4s\n" - "fmin v14.4s, v14.4s, v0.4s\n" - "fmin v15.4s, v15.4s, v0.4s\n" - "fmin v16.4s, v16.4s, v0.4s\n" - "fmin v17.4s, v17.4s, v0.4s\n" - "fmin v18.4s, v18.4s, v0.4s\n" - "fmin v19.4s, v19.4s, v0.4s\n" - "fmin v20.4s, v20.4s, v0.4s\n" - "fmin v21.4s, v21.4s, v0.4s\n" - "fmin v22.4s, v22.4s, v0.4s\n" - "fmin v23.4s, v23.4s, v0.4s\n" - "fmin v24.4s, v24.4s, v0.4s\n" - "fmin v25.4s, v25.4s, v0.4s\n" + "ld1r { v26.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v26.4s\n" + "fmin v9.4s, v9.4s, v26.4s\n" + "fmin v10.4s, v10.4s, v26.4s\n" + "fmin v11.4s, v11.4s, v26.4s\n" + "fmin v12.4s, v12.4s, v26.4s\n" + "fmin v13.4s, v13.4s, v26.4s\n" + "fmin v14.4s, v14.4s, v26.4s\n" + "fmin v15.4s, v15.4s, v26.4s\n" + "fmin v16.4s, v16.4s, v26.4s\n" + "fmin v17.4s, v17.4s, v26.4s\n" + "fmin v18.4s, v18.4s, v26.4s\n" + "fmin v19.4s, v19.4s, v26.4s\n" + "fmin v20.4s, v20.4s, v26.4s\n" + "fmin v21.4s, v21.4s, v26.4s\n" + "fmin v22.4s, v22.4s, v26.4s\n" + "fmin v23.4s, v23.4s, v26.4s\n" + "fmin v24.4s, v24.4s, v26.4s\n" + "fmin v25.4s, v25.4s, v26.4s\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" - "fmax v16.4s, v16.4s, v0.4s\n" - "fmax v17.4s, v17.4s, v0.4s\n" - "fmax v18.4s, v18.4s, v0.4s\n" - "fmax v19.4s, v19.4s, v0.4s\n" - "fmax v20.4s, v20.4s, v0.4s\n" - "fmax v21.4s, v21.4s, v0.4s\n" - "fmax v22.4s, v22.4s, v0.4s\n" - "fmax v23.4s, v23.4s, v0.4s\n" - "fmax v24.4s, v24.4s, v0.4s\n" - "fmax v25.4s, v25.4s, v0.4s\n" + "ld1r { v26.4s }, [x20]\n" + "fmax v8.4s, v8.4s, v26.4s\n" + "fmax v9.4s, v9.4s, v26.4s\n" + "fmax v10.4s, v10.4s, v26.4s\n" + "fmax v11.4s, v11.4s, v26.4s\n" + "fmax v12.4s, v12.4s, v26.4s\n" + "fmax v13.4s, v13.4s, v26.4s\n" + "fmax v14.4s, v14.4s, v26.4s\n" + "fmax v15.4s, v15.4s, v26.4s\n" + "fmax v16.4s, v16.4s, v26.4s\n" + "fmax v17.4s, v17.4s, v26.4s\n" + "fmax v18.4s, v18.4s, v26.4s\n" + "fmax v19.4s, v19.4s, v26.4s\n" + "fmax v20.4s, v20.4s, v26.4s\n" + "fmax v21.4s, v21.4s, v26.4s\n" + "fmax v22.4s, v22.4s, v26.4s\n" + "fmax v23.4s, v23.4s, v26.4s\n" + "fmax v24.4s, v24.4s, v26.4s\n" + "fmax v25.4s, v25.4s, v26.4s\n" "109:" // Height 3: No activation "cmp x16, #0x18\n" "bge 122f\n" @@ -2139,34 +2138,34 @@ void a64_hybrid_fp32_mla_4x24_a55 ( "142:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w12, [x20, x13, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 143f\n" - "ldr x21, [%x[input_ptr], x13, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x11, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x23, [x21, #0x10]\n" - "ldr x21, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x13, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x11, [x20, #0x0]\n" + "ldr x10, [x20, #0x8]\n" + "ldr x9, [x20, #0x10]\n" + "ldr x28, [x20, #0x18]\n" "cbnz x13, 144f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x11, x11, x20, LSL #2\n" - "add x25, x25, x20, LSL #2\n" - "add x23, x23, x20, LSL #2\n" - "add x21, x21, x20, LSL #2\n" + "add x10, x10, x20, LSL #2\n" + "add x9, x9, x20, LSL #2\n" + "add x28, x28, x20, LSL #2\n" "b 144f\n" "143:" // Height 4: setup direct input "mov x11, %x[input_ptr]\n" - "add x25, x11, x20, LSL #2\n" - "add x23, x25, x20, LSL #2\n" - "add x21, x23, x20, LSL #2\n" + "add x10, x11, x21, LSL #2\n" + "add x9, x10, x21, LSL #2\n" + "add x28, x9, x21, LSL #2\n" "144:" // Height 4: input setup done "cmp x12, #0x4\n" "blt 147f\n" "ldr q0, [x11, #0x0]\n" "cmp x12, #0x8\n" - "ldr q1, [x25, #0x0]\n" - "ldr q2, [x23, #0x0]\n" - "ldr q3, [x21, #0x0]\n" + "ldr q1, [x10, #0x0]\n" + "ldr q2, [x9, #0x0]\n" + "ldr q3, [x28, #0x0]\n" "ldr q4, [x15, #0x0]\n" "ldr q5, [x15, #0x10]\n" "ldr q6, [x15, #0x20]\n" @@ -2174,177 +2173,177 @@ void a64_hybrid_fp32_mla_4x24_a55 ( "blt 146f\n" "145:" // Height 4: Multiply loop: Main loop head "fmla v8.4s, v4.4s, v0.s[0]\n" - "ldr x10, [x15, #0x48]\n" + "ldr x23, [x15, #0x48]\n" "fmla v14.4s, v4.4s, v1.s[0]\n" - "ldr x9, [x15, #0x58]\n" + "ldr x22, [x15, #0x58]\n" "fmla v20.4s, v4.4s, v2.s[0]\n" - "ldr x28, [x15, #0x68]\n" + "ldr x21, [x15, #0x68]\n" "fmla v26.4s, v4.4s, v3.s[0]\n" "ldr d4, [x15, #0x40]\n" "fmla v9.4s, v5.4s, v0.s[0]\n" - "ldr x27, [x15, #0x78]\n" + "ldr x20, [x15, #0x78]\n" "fmla v15.4s, v5.4s, v1.s[0]\n" - "mov v4.d[1], x10\n" + "mov v4.d[1], x23\n" "fmla v21.4s, v5.4s, v2.s[0]\n" - "ldr x10, [x15, #0x88]\n" + "ldr x23, [x15, #0x88]\n" "fmla v27.4s, v5.4s, v3.s[0]\n" "ldr d5, [x15, #0x50]\n" "fmla v10.4s, v6.4s, v0.s[0]\n" - "mov v5.d[1], x9\n" + "mov v5.d[1], x22\n" "fmla v16.4s, v6.4s, v1.s[0]\n" - "ldr x9, [x15, #0x98]\n" + "ldr x22, [x15, #0x98]\n" "fmla v22.4s, v6.4s, v2.s[0]\n" "add x11, x11, #0x10\n" "fmla v28.4s, v6.4s, v3.s[0]\n" "ldr d6, [x15, #0x60]\n" "fmla v11.4s, v7.4s, v0.s[0]\n" - "mov v6.d[1], x28\n" + "mov v6.d[1], x21\n" "fmla v17.4s, v7.4s, v1.s[0]\n" - "ldr x28, [x15, #0xa8]\n" + "ldr x21, [x15, #0xa8]\n" "fmla v23.4s, v7.4s, v2.s[0]\n" - "add x25, x25, #0x10\n" + "add x10, x10, #0x10\n" "fmla v29.4s, v7.4s, v3.s[0]\n" "ldr d7, [x15, #0x70]\n" - "mov v7.d[1], x27\n" + "mov v7.d[1], x20\n" "fmla v12.4s, v4.4s, v0.s[0]\n" "fmla v18.4s, v4.4s, v1.s[0]\n" - "ldr x27, [x15, #0xb8]\n" + "ldr x20, [x15, #0xb8]\n" "fmla v24.4s, v4.4s, v2.s[0]\n" - "add x23, x23, #0x10\n" + "add x9, x9, #0x10\n" "fmla v30.4s, v4.4s, v3.s[0]\n" "ldr d4, [x15, #0x80]\n" "fmla v13.4s, v5.4s, v0.s[0]\n" - "mov v4.d[1], x10\n" + "mov v4.d[1], x23\n" "fmla v19.4s, v5.4s, v1.s[0]\n" - "ldr x10, [x15, #0xc8]\n" + "ldr x23, [x15, #0xc8]\n" "fmla v25.4s, v5.4s, v2.s[0]\n" - "add x21, x21, #0x10\n" + "add x28, x28, #0x10\n" "fmla v31.4s, v5.4s, v3.s[0]\n" "ldr d5, [x15, #0x90]\n" "fmla v8.4s, v6.4s, v0.s[1]\n" - "mov v5.d[1], x9\n" + "mov v5.d[1], x22\n" "fmla v14.4s, v6.4s, v1.s[1]\n" - "ldr x9, [x15, #0xd8]\n" + "ldr x22, [x15, #0xd8]\n" "fmla v20.4s, v6.4s, v2.s[1]\n" - "ldr x26, [x11, #0x8]\n" + "ldr x27, [x11, #0x8]\n" "fmla v26.4s, v6.4s, v3.s[1]\n" "ldr d6, [x15, #0xa0]\n" "fmla v9.4s, v7.4s, v0.s[1]\n" - "mov v6.d[1], x28\n" + "mov v6.d[1], x21\n" "fmla v15.4s, v7.4s, v1.s[1]\n" - "ldr x28, [x15, #0xe8]\n" + "ldr x21, [x15, #0xe8]\n" "fmla v21.4s, v7.4s, v2.s[1]\n" - "ldr x24, [x25, #0x8]\n" + "ldr x26, [x10, #0x8]\n" "fmla v27.4s, v7.4s, v3.s[1]\n" "ldr d7, [x15, #0xb0]\n" - "mov v7.d[1], x27\n" + "mov v7.d[1], x20\n" "fmla v10.4s, v4.4s, v0.s[1]\n" "fmla v16.4s, v4.4s, v1.s[1]\n" - "ldr x27, [x15, #0xf8]\n" + "ldr x20, [x15, #0xf8]\n" "fmla v22.4s, v4.4s, v2.s[1]\n" - "ldr x22, [x23, #0x8]\n" + "ldr x25, [x9, #0x8]\n" "fmla v28.4s, v4.4s, v3.s[1]\n" "ldr d4, [x15, #0xc0]\n" "fmla v11.4s, v5.4s, v0.s[1]\n" - "mov v4.d[1], x10\n" + "mov v4.d[1], x23\n" "fmla v17.4s, v5.4s, v1.s[1]\n" - "ldr x10, [x15, #0x108]\n" + "ldr x23, [x15, #0x108]\n" "fmla v23.4s, v5.4s, v2.s[1]\n" - "ldr x20, [x21, #0x8]\n" + "ldr x24, [x28, #0x8]\n" "fmla v29.4s, v5.4s, v3.s[1]\n" "ldr d5, [x15, #0xd0]\n" "fmla v12.4s, v6.4s, v0.s[1]\n" - "mov v5.d[1], x9\n" + "mov v5.d[1], x22\n" "fmla v18.4s, v6.4s, v1.s[1]\n" - "ldr x9, [x15, #0x118]\n" + "ldr x22, [x15, #0x118]\n" "fmla v24.4s, v6.4s, v2.s[1]\n" "sub x12, x12, #0x4\n" "fmla v30.4s, v6.4s, v3.s[1]\n" "ldr d6, [x15, #0xe0]\n" "fmla v13.4s, v7.4s, v0.s[1]\n" - "mov v6.d[1], x28\n" + "mov v6.d[1], x21\n" "fmla v19.4s, v7.4s, v1.s[1]\n" - "ldr x28, [x15, #0x128]\n" + "ldr x21, [x15, #0x128]\n" "fmla v25.4s, v7.4s, v2.s[1]\n" "cmp x12, #0x8\n" "fmla v31.4s, v7.4s, v3.s[1]\n" "ldr d7, [x15, #0xf0]\n" - "mov v7.d[1], x27\n" + "mov v7.d[1], x20\n" "fmla v8.4s, v4.4s, v0.s[2]\n" "fmla v14.4s, v4.4s, v1.s[2]\n" - "ldr x27, [x15, #0x138]\n" + "ldr x20, [x15, #0x138]\n" "fmla v20.4s, v4.4s, v2.s[2]\n" "prfm pldl1keep, [x11, #0x80]\n" "fmla v26.4s, v4.4s, v3.s[2]\n" "ldr d4, [x15, #0x100]\n" "fmla v9.4s, v5.4s, v0.s[2]\n" - "mov v4.d[1], x10\n" + "mov v4.d[1], x23\n" "fmla v15.4s, v5.4s, v1.s[2]\n" - "ldr x10, [x15, #0x148]\n" + "ldr x23, [x15, #0x148]\n" "fmla v21.4s, v5.4s, v2.s[2]\n" - "prfm pldl1keep, [x25, #0x80]\n" + "prfm pldl1keep, [x10, #0x80]\n" "fmla v27.4s, v5.4s, v3.s[2]\n" "ldr d5, [x15, #0x110]\n" "fmla v10.4s, v6.4s, v0.s[2]\n" - "mov v5.d[1], x9\n" + "mov v5.d[1], x22\n" "fmla v16.4s, v6.4s, v1.s[2]\n" - "ldr x9, [x15, #0x158]\n" + "ldr x22, [x15, #0x158]\n" "fmla v22.4s, v6.4s, v2.s[2]\n" - "prfm pldl1keep, [x23, #0x80]\n" + "prfm pldl1keep, [x9, #0x80]\n" "fmla v28.4s, v6.4s, v3.s[2]\n" "ldr d6, [x15, #0x120]\n" "fmla v11.4s, v7.4s, v0.s[2]\n" - "mov v6.d[1], x28\n" + "mov v6.d[1], x21\n" "fmla v17.4s, v7.4s, v1.s[2]\n" - "ldr x28, [x15, #0x168]\n" + "ldr x21, [x15, #0x168]\n" "fmla v23.4s, v7.4s, v2.s[2]\n" - "prfm pldl1keep, [x21, #0x80]\n" + "prfm pldl1keep, [x28, #0x80]\n" "fmla v29.4s, v7.4s, v3.s[2]\n" "ldr d7, [x15, #0x130]\n" - "mov v7.d[1], x27\n" + "mov v7.d[1], x20\n" "fmla v12.4s, v4.4s, v0.s[2]\n" "fmla v18.4s, v4.4s, v1.s[2]\n" - "ldr x27, [x15, #0x178]\n" + "ldr x20, [x15, #0x178]\n" "fmla v24.4s, v4.4s, v2.s[2]\n" "fmla v30.4s, v4.4s, v3.s[2]\n" "ldr d4, [x15, #0x140]\n" "fmla v13.4s, v5.4s, v0.s[2]\n" - "mov v4.d[1], x10\n" + "mov v4.d[1], x23\n" "fmla v19.4s, v5.4s, v1.s[2]\n" "fmla v25.4s, v5.4s, v2.s[2]\n" "fmla v31.4s, v5.4s, v3.s[2]\n" "ldr d5, [x15, #0x150]\n" "fmla v8.4s, v6.4s, v0.s[3]\n" - "mov v5.d[1], x9\n" + "mov v5.d[1], x22\n" "fmla v14.4s, v6.4s, v1.s[3]\n" "fmla v20.4s, v6.4s, v2.s[3]\n" "fmla v26.4s, v6.4s, v3.s[3]\n" "ldr d6, [x15, #0x160]\n" "fmla v9.4s, v7.4s, v0.s[3]\n" - "mov v6.d[1], x28\n" + "mov v6.d[1], x21\n" "fmla v15.4s, v7.4s, v1.s[3]\n" "fmla v21.4s, v7.4s, v2.s[3]\n" "fmla v27.4s, v7.4s, v3.s[3]\n" "ldr d7, [x15, #0x170]\n" - "mov v7.d[1], x27\n" + "mov v7.d[1], x20\n" "add x15, x15, #0x180\n" "fmla v10.4s, v4.4s, v0.s[3]\n" - "ldr x10, [x15, #0x8]\n" + "ldr x23, [x15, #0x8]\n" "fmla v16.4s, v4.4s, v1.s[3]\n" - "ldr x9, [x15, #0x18]\n" + "ldr x22, [x15, #0x18]\n" "fmla v22.4s, v4.4s, v2.s[3]\n" - "ldr x28, [x15, #0x28]\n" + "ldr x21, [x15, #0x28]\n" "fmla v28.4s, v4.4s, v3.s[3]\n" "ldr d4, [x15, #0x0]\n" "fmla v11.4s, v5.4s, v0.s[3]\n" - "ldr x27, [x15, #0x38]\n" + "ldr x20, [x15, #0x38]\n" "fmla v17.4s, v5.4s, v1.s[3]\n" - "mov v4.d[1], x10\n" + "mov v4.d[1], x23\n" "fmla v23.4s, v5.4s, v2.s[3]\n" "fmla v29.4s, v5.4s, v3.s[3]\n" "ldr d5, [x15, #0x10]\n" "fmla v12.4s, v6.4s, v0.s[3]\n" - "mov v5.d[1], x9\n" + "mov v5.d[1], x22\n" "fmla v18.4s, v6.4s, v1.s[3]\n" "fmla v24.4s, v6.4s, v2.s[3]\n" "fmla v30.4s, v6.4s, v3.s[3]\n" @@ -2352,30 +2351,30 @@ void a64_hybrid_fp32_mla_4x24_a55 ( "fmla v13.4s, v7.4s, v0.s[3]\n" "ldr d0, [x11, #0x0]\n" "fmla v19.4s, v7.4s, v1.s[3]\n" - "ldr d1, [x25, #0x0]\n" + "ldr d1, [x10, #0x0]\n" "fmla v25.4s, v7.4s, v2.s[3]\n" - "ldr d2, [x23, #0x0]\n" + "ldr d2, [x9, #0x0]\n" "fmla v31.4s, v7.4s, v3.s[3]\n" - "ldr d3, [x21, #0x0]\n" + "ldr d3, [x28, #0x0]\n" "ldr d7, [x15, #0x30]\n" - "mov v6.d[1], x28\n" - "mov v0.d[1], x26\n" - "mov v1.d[1], x24\n" - "mov v2.d[1], x22\n" - "mov v3.d[1], x20\n" - "mov v7.d[1], x27\n" + "mov v6.d[1], x21\n" + "mov v0.d[1], x27\n" + "mov v1.d[1], x26\n" + "mov v2.d[1], x25\n" + "mov v3.d[1], x24\n" + "mov v7.d[1], x20\n" "bge 145b\n" "146:" // Height 4: Multiply loop: Single iteration only "fmla v8.4s, v4.4s, v0.s[0]\n" "add x11, x11, #0x10\n" "fmla v14.4s, v4.4s, v1.s[0]\n" - "add x25, x25, #0x10\n" + "add x10, x10, #0x10\n" "fmla v20.4s, v4.4s, v2.s[0]\n" - "add x23, x23, #0x10\n" + "add x9, x9, #0x10\n" "fmla v26.4s, v4.4s, v3.s[0]\n" "ldr q4, [x15, #0x40]\n" "fmla v9.4s, v5.4s, v0.s[0]\n" - "add x21, x21, #0x10\n" + "add x28, x28, #0x10\n" "fmla v15.4s, v5.4s, v1.s[0]\n" "sub x12, x12, #0x4\n" "fmla v21.4s, v5.4s, v2.s[0]\n" @@ -2383,11 +2382,11 @@ void a64_hybrid_fp32_mla_4x24_a55 ( "fmla v27.4s, v5.4s, v3.s[0]\n" "ldr q5, [x15, #0x50]\n" "fmla v10.4s, v6.4s, v0.s[0]\n" - "prfm pldl1keep, [x25, #0x80]\n" + "prfm pldl1keep, [x10, #0x80]\n" "fmla v16.4s, v6.4s, v1.s[0]\n" - "prfm pldl1keep, [x23, #0x80]\n" + "prfm pldl1keep, [x9, #0x80]\n" "fmla v22.4s, v6.4s, v2.s[0]\n" - "prfm pldl1keep, [x21, #0x80]\n" + "prfm pldl1keep, [x28, #0x80]\n" "fmla v28.4s, v6.4s, v3.s[0]\n" "ldr q6, [x15, #0x60]\n" "fmla v11.4s, v7.4s, v0.s[0]\n" @@ -2495,42 +2494,42 @@ void a64_hybrid_fp32_mla_4x24_a55 ( "147:" // Height 4: Multiply loop: Main loop skip "cbz x12, 149f\n" "148:" // Height 4: Multiply loop: Odd block loop - "ldr s0, [x11], #0x4\n" + "ldr s7, [x11], #0x4\n" "sub x12, x12, #0x1\n" - "ldr s1, [x25], #0x4\n" - "ldr s2, [x23], #0x4\n" - "ldr s3, [x21], #0x4\n" - "ldr q4, [x15, #0x0]\n" - "fmla v8.4s, v4.4s, v0.s[0]\n" - "ldr q5, [x15, #0x10]\n" - "fmla v14.4s, v4.4s, v1.s[0]\n" - "ldr q6, [x15, #0x20]\n" - "fmla v20.4s, v4.4s, v2.s[0]\n" - "ldr q7, [x15, #0x30]\n" - "fmla v26.4s, v4.4s, v3.s[0]\n" - "ldr q4, [x15, #0x40]\n" - "fmla v9.4s, v5.4s, v0.s[0]\n" - "fmla v15.4s, v5.4s, v1.s[0]\n" - "fmla v21.4s, v5.4s, v2.s[0]\n" - "fmla v27.4s, v5.4s, v3.s[0]\n" - "ldr q5, [x15, #0x50]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" + "ldr s6, [x10], #0x4\n" + "ldr s5, [x9], #0x4\n" + "ldr s4, [x28], #0x4\n" + "ldr q1, [x15, #0x0]\n" + "fmla v8.4s, v1.4s, v7.s[0]\n" + "ldr q0, [x15, #0x10]\n" + "fmla v14.4s, v1.4s, v6.s[0]\n" + "ldr q3, [x15, #0x20]\n" + "fmla v20.4s, v1.4s, v5.s[0]\n" + "ldr q2, [x15, #0x30]\n" + "fmla v26.4s, v1.4s, v4.s[0]\n" + "ldr q1, [x15, #0x40]\n" + "fmla v9.4s, v0.4s, v7.s[0]\n" + "fmla v15.4s, v0.4s, v6.s[0]\n" + "fmla v21.4s, v0.4s, v5.s[0]\n" + "fmla v27.4s, v0.4s, v4.s[0]\n" + "ldr q0, [x15, #0x50]\n" + "fmla v10.4s, v3.4s, v7.s[0]\n" "add x15, x15, #0x60\n" - "fmla v16.4s, v6.4s, v1.s[0]\n" - "fmla v22.4s, v6.4s, v2.s[0]\n" - "fmla v28.4s, v6.4s, v3.s[0]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v17.4s, v7.4s, v1.s[0]\n" - "fmla v23.4s, v7.4s, v2.s[0]\n" - "fmla v29.4s, v7.4s, v3.s[0]\n" - "fmla v12.4s, v4.4s, v0.s[0]\n" - "fmla v18.4s, v4.4s, v1.s[0]\n" - "fmla v24.4s, v4.4s, v2.s[0]\n" - "fmla v30.4s, v4.4s, v3.s[0]\n" - "fmla v13.4s, v5.4s, v0.s[0]\n" - "fmla v19.4s, v5.4s, v1.s[0]\n" - "fmla v25.4s, v5.4s, v2.s[0]\n" - "fmla v31.4s, v5.4s, v3.s[0]\n" + "fmla v16.4s, v3.4s, v6.s[0]\n" + "fmla v22.4s, v3.4s, v5.s[0]\n" + "fmla v28.4s, v3.4s, v4.s[0]\n" + "fmla v11.4s, v2.4s, v7.s[0]\n" + "fmla v17.4s, v2.4s, v6.s[0]\n" + "fmla v23.4s, v2.4s, v5.s[0]\n" + "fmla v29.4s, v2.4s, v4.s[0]\n" + "fmla v12.4s, v1.4s, v7.s[0]\n" + "fmla v18.4s, v1.4s, v6.s[0]\n" + "fmla v24.4s, v1.4s, v5.s[0]\n" + "fmla v30.4s, v1.4s, v4.s[0]\n" + "fmla v13.4s, v0.4s, v7.s[0]\n" + "fmla v19.4s, v0.4s, v6.s[0]\n" + "fmla v25.4s, v0.4s, v5.s[0]\n" + "fmla v31.4s, v0.4s, v4.s[0]\n" "cbnz x12, 148b\n" "149:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -2796,7 +2795,6 @@ void a64_hybrid_fp32_mla_4x24_a55 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "166:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x24/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x24/generic.cpp index 5fb71c95b7012b482c92ca5b9b2b43bb50152169..dbd45460e8738c7454dc4e309a2ba2101003531d 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x24/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_4x24/generic.cpp @@ -92,7 +92,6 @@ void a64_hybrid_fp32_mla_4x24 ( break; } __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x4\n" "bge 124f\n" @@ -223,11 +222,11 @@ void a64_hybrid_fp32_mla_4x24 ( "19:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 20f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" "cbnz x26, 21f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20, LSL #2\n" @@ -246,126 +245,126 @@ void a64_hybrid_fp32_mla_4x24 ( "blt 23f\n" "22:" // Height 1: Multiply loop: Main loop head "fmla v8.4s, v4.4s, v0.s[0]\n" - "ldr q4, [x28, #0x40]\n" + "ldr q19, [x28, #0x40]\n" "fmla v9.4s, v5.4s, v0.s[0]\n" - "ldr q5, [x28, #0x50]\n" + "ldr q18, [x28, #0x50]\n" "fmla v10.4s, v6.4s, v0.s[0]\n" - "ldr q6, [x28, #0x60]\n" + "ldr q17, [x28, #0x60]\n" "fmla v11.4s, v7.4s, v0.s[0]\n" - "ldr q7, [x28, #0x70]\n" - "fmla v12.4s, v4.4s, v0.s[0]\n" - "ldr q4, [x28, #0x80]\n" - "fmla v13.4s, v5.4s, v0.s[0]\n" - "ldr q5, [x28, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "ldr q6, [x28, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "ldr q7, [x28, #0xb0]\n" - "fmla v10.4s, v4.4s, v0.s[1]\n" - "ldr q4, [x28, #0xc0]\n" - "fmla v11.4s, v5.4s, v0.s[1]\n" - "ldr q5, [x28, #0xd0]\n" - "fmla v12.4s, v6.4s, v0.s[1]\n" - "ldr q6, [x28, #0xe0]\n" - "fmla v13.4s, v7.4s, v0.s[1]\n" - "ldr q7, [x28, #0xf0]\n" - "fmla v8.4s, v4.4s, v0.s[2]\n" - "ldr q4, [x28, #0x100]\n" - "fmla v9.4s, v5.4s, v0.s[2]\n" - "ldr q5, [x28, #0x110]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "ldr q6, [x28, #0x120]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "ldr q7, [x28, #0x130]\n" - "fmla v12.4s, v4.4s, v0.s[2]\n" - "ldr q4, [x28, #0x140]\n" - "fmla v13.4s, v5.4s, v0.s[2]\n" - "ldr q5, [x28, #0x150]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "ldr q6, [x28, #0x160]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "ldr q7, [x28, #0x170]\n" + "ldr q16, [x28, #0x70]\n" + "fmla v12.4s, v19.4s, v0.s[0]\n" + "ldr q19, [x28, #0x80]\n" + "fmla v13.4s, v18.4s, v0.s[0]\n" + "ldr q18, [x28, #0x90]\n" + "fmla v8.4s, v17.4s, v0.s[1]\n" + "ldr q17, [x28, #0xa0]\n" + "fmla v9.4s, v16.4s, v0.s[1]\n" + "ldr q16, [x28, #0xb0]\n" + "fmla v10.4s, v19.4s, v0.s[1]\n" + "ldr q19, [x28, #0xc0]\n" + "fmla v11.4s, v18.4s, v0.s[1]\n" + "ldr q18, [x28, #0xd0]\n" + "fmla v12.4s, v17.4s, v0.s[1]\n" + "ldr q17, [x28, #0xe0]\n" + "fmla v13.4s, v16.4s, v0.s[1]\n" + "ldr q16, [x28, #0xf0]\n" + "fmla v8.4s, v19.4s, v0.s[2]\n" + "ldr q19, [x28, #0x100]\n" + "fmla v9.4s, v18.4s, v0.s[2]\n" + "ldr q18, [x28, #0x110]\n" + "fmla v10.4s, v17.4s, v0.s[2]\n" + "ldr q17, [x28, #0x120]\n" + "fmla v11.4s, v16.4s, v0.s[2]\n" + "ldr q16, [x28, #0x130]\n" + "fmla v12.4s, v19.4s, v0.s[2]\n" + "ldr q19, [x28, #0x140]\n" + "fmla v13.4s, v18.4s, v0.s[2]\n" + "ldr q18, [x28, #0x150]\n" + "fmla v8.4s, v17.4s, v0.s[3]\n" + "ldr q17, [x28, #0x160]\n" + "fmla v9.4s, v16.4s, v0.s[3]\n" + "ldr q16, [x28, #0x170]\n" "sub x25, x25, #0x4\n" "add x24, x24, #0x10\n" - "fmla v10.4s, v4.4s, v0.s[3]\n" - "fmla v11.4s, v5.4s, v0.s[3]\n" + "fmla v10.4s, v19.4s, v0.s[3]\n" + "fmla v11.4s, v18.4s, v0.s[3]\n" "cmp x25, #0x8\n" "add x28, x28, #0x180\n" "ldr q4, [x28, #0x0]\n" "ldr q5, [x28, #0x10]\n" - "fmla v12.4s, v6.4s, v0.s[3]\n" + "fmla v12.4s, v17.4s, v0.s[3]\n" "ldr q6, [x28, #0x20]\n" - "fmla v13.4s, v7.4s, v0.s[3]\n" + "fmla v13.4s, v16.4s, v0.s[3]\n" "ldr q0, [x24, #0x0]\n" "ldr q7, [x28, #0x30]\n" "prfm pldl1keep, [x24, #0x80]\n" "bge 22b\n" "23:" // Height 1: Multiply loop: Single iteration only "fmla v8.4s, v4.4s, v0.s[0]\n" - "ldr q4, [x28, #0x40]\n" + "ldr q19, [x28, #0x40]\n" "fmla v9.4s, v5.4s, v0.s[0]\n" - "ldr q5, [x28, #0x50]\n" + "ldr q18, [x28, #0x50]\n" "fmla v10.4s, v6.4s, v0.s[0]\n" - "ldr q6, [x28, #0x60]\n" + "ldr q17, [x28, #0x60]\n" "fmla v11.4s, v7.4s, v0.s[0]\n" - "ldr q7, [x28, #0x70]\n" - "fmla v12.4s, v4.4s, v0.s[0]\n" - "ldr q4, [x28, #0x80]\n" - "fmla v13.4s, v5.4s, v0.s[0]\n" - "ldr q5, [x28, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "ldr q6, [x28, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "ldr q7, [x28, #0xb0]\n" - "fmla v10.4s, v4.4s, v0.s[1]\n" - "ldr q4, [x28, #0xc0]\n" - "fmla v11.4s, v5.4s, v0.s[1]\n" - "ldr q5, [x28, #0xd0]\n" - "fmla v12.4s, v6.4s, v0.s[1]\n" - "ldr q6, [x28, #0xe0]\n" - "fmla v13.4s, v7.4s, v0.s[1]\n" - "ldr q7, [x28, #0xf0]\n" - "fmla v8.4s, v4.4s, v0.s[2]\n" - "ldr q4, [x28, #0x100]\n" - "fmla v9.4s, v5.4s, v0.s[2]\n" - "ldr q5, [x28, #0x110]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "ldr q6, [x28, #0x120]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "ldr q7, [x28, #0x130]\n" - "fmla v12.4s, v4.4s, v0.s[2]\n" - "ldr q4, [x28, #0x140]\n" - "fmla v13.4s, v5.4s, v0.s[2]\n" - "ldr q5, [x28, #0x150]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "ldr q6, [x28, #0x160]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "ldr q7, [x28, #0x170]\n" + "ldr q16, [x28, #0x70]\n" + "fmla v12.4s, v19.4s, v0.s[0]\n" + "ldr q19, [x28, #0x80]\n" + "fmla v13.4s, v18.4s, v0.s[0]\n" + "ldr q18, [x28, #0x90]\n" + "fmla v8.4s, v17.4s, v0.s[1]\n" + "ldr q17, [x28, #0xa0]\n" + "fmla v9.4s, v16.4s, v0.s[1]\n" + "ldr q16, [x28, #0xb0]\n" + "fmla v10.4s, v19.4s, v0.s[1]\n" + "ldr q19, [x28, #0xc0]\n" + "fmla v11.4s, v18.4s, v0.s[1]\n" + "ldr q18, [x28, #0xd0]\n" + "fmla v12.4s, v17.4s, v0.s[1]\n" + "ldr q17, [x28, #0xe0]\n" + "fmla v13.4s, v16.4s, v0.s[1]\n" + "ldr q16, [x28, #0xf0]\n" + "fmla v8.4s, v19.4s, v0.s[2]\n" + "ldr q19, [x28, #0x100]\n" + "fmla v9.4s, v18.4s, v0.s[2]\n" + "ldr q18, [x28, #0x110]\n" + "fmla v10.4s, v17.4s, v0.s[2]\n" + "ldr q17, [x28, #0x120]\n" + "fmla v11.4s, v16.4s, v0.s[2]\n" + "ldr q16, [x28, #0x130]\n" + "fmla v12.4s, v19.4s, v0.s[2]\n" + "ldr q19, [x28, #0x140]\n" + "fmla v13.4s, v18.4s, v0.s[2]\n" + "ldr q18, [x28, #0x150]\n" + "fmla v8.4s, v17.4s, v0.s[3]\n" + "ldr q17, [x28, #0x160]\n" + "fmla v9.4s, v16.4s, v0.s[3]\n" + "ldr q16, [x28, #0x170]\n" "add x24, x24, #0x10\n" "sub x25, x25, #0x4\n" - "fmla v10.4s, v4.4s, v0.s[3]\n" - "fmla v11.4s, v5.4s, v0.s[3]\n" + "fmla v10.4s, v19.4s, v0.s[3]\n" + "fmla v11.4s, v18.4s, v0.s[3]\n" "prfm pldl1keep, [x24, #0x80]\n" "add x28, x28, #0x180\n" - "fmla v12.4s, v6.4s, v0.s[3]\n" - "fmla v13.4s, v7.4s, v0.s[3]\n" + "fmla v12.4s, v17.4s, v0.s[3]\n" + "fmla v13.4s, v16.4s, v0.s[3]\n" "24:" // Height 1: Multiply loop: Main loop skip "cbz x25, 26f\n" "25:" // Height 1: Multiply loop: Odd block loop - "ldr s0, [x24], #0x4\n" - "ldr q4, [x28, #0x0]\n" - "fmla v8.4s, v4.4s, v0.s[0]\n" + "ldr s18, [x24], #0x4\n" + "ldr q16, [x28, #0x0]\n" + "fmla v8.4s, v16.4s, v18.s[0]\n" "sub x25, x25, #0x1\n" - "ldr q5, [x28, #0x10]\n" - "ldr q6, [x28, #0x20]\n" - "fmla v9.4s, v5.4s, v0.s[0]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "ldr q7, [x28, #0x30]\n" - "ldr q4, [x28, #0x40]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v12.4s, v4.4s, v0.s[0]\n" - "ldr q5, [x28, #0x50]\n" - "fmla v13.4s, v5.4s, v0.s[0]\n" + "ldr q17, [x28, #0x10]\n" + "ldr q16, [x28, #0x20]\n" + "fmla v9.4s, v17.4s, v18.s[0]\n" + "fmla v10.4s, v16.4s, v18.s[0]\n" + "ldr q17, [x28, #0x30]\n" + "ldr q16, [x28, #0x40]\n" + "fmla v11.4s, v17.4s, v18.s[0]\n" + "fmla v12.4s, v16.4s, v18.s[0]\n" + "ldr q16, [x28, #0x50]\n" + "fmla v13.4s, v16.4s, v18.s[0]\n" "add x28, x28, #0x60\n" "cbnz x25, 25b\n" "26:" // Height 1: Multiply loop: No odd multiplies @@ -376,21 +375,21 @@ void a64_hybrid_fp32_mla_4x24 ( "prfm pstl1keep, [x27, #0x0]\n" "tbz %x[flags], #1, 27f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v17.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" + "ld1r { v16.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v17.4s\n" + "fmin v9.4s, v9.4s, v17.4s\n" + "fmin v10.4s, v10.4s, v17.4s\n" + "fmin v11.4s, v11.4s, v17.4s\n" + "fmin v12.4s, v12.4s, v17.4s\n" + "fmin v13.4s, v13.4s, v17.4s\n" + "fmax v8.4s, v8.4s, v16.4s\n" + "fmax v9.4s, v9.4s, v16.4s\n" + "fmax v10.4s, v10.4s, v16.4s\n" + "fmax v11.4s, v11.4s, v16.4s\n" + "fmax v12.4s, v12.4s, v16.4s\n" + "fmax v13.4s, v13.4s, v16.4s\n" "27:" // Height 1: No activation "cmp x9, #0x18\n" "bge 40f\n" @@ -651,12 +650,12 @@ void a64_hybrid_fp32_mla_4x24 ( "60:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 61f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" "cbnz x26, 62f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20, LSL #2\n" @@ -664,7 +663,7 @@ void a64_hybrid_fp32_mla_4x24 ( "b 62f\n" "61:" // Height 2: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20, LSL #2\n" + "add x23, x24, x21, LSL #2\n" "62:" // Height 2: input setup done "cmp x25, #0x4\n" "blt 65f\n" @@ -679,186 +678,186 @@ void a64_hybrid_fp32_mla_4x24 ( "63:" // Height 2: Multiply loop: Main loop head "fmla v8.4s, v4.4s, v0.s[0]\n" "fmla v14.4s, v4.4s, v1.s[0]\n" - "ldr q4, [x28, #0x40]\n" + "ldr q23, [x28, #0x40]\n" "sub x25, x25, #0x4\n" "fmla v9.4s, v5.4s, v0.s[0]\n" "fmla v15.4s, v5.4s, v1.s[0]\n" - "ldr q5, [x28, #0x50]\n" + "ldr q22, [x28, #0x50]\n" "add x24, x24, #0x10\n" "fmla v10.4s, v6.4s, v0.s[0]\n" "fmla v16.4s, v6.4s, v1.s[0]\n" - "ldr q6, [x28, #0x60]\n" + "ldr q21, [x28, #0x60]\n" "add x23, x23, #0x10\n" "fmla v11.4s, v7.4s, v0.s[0]\n" "fmla v17.4s, v7.4s, v1.s[0]\n" - "ldr q7, [x28, #0x70]\n" + "ldr q20, [x28, #0x70]\n" "cmp x25, #0x8\n" - "fmla v12.4s, v4.4s, v0.s[0]\n" - "fmla v18.4s, v4.4s, v1.s[0]\n" - "ldr q4, [x28, #0x80]\n" + "fmla v12.4s, v23.4s, v0.s[0]\n" + "fmla v18.4s, v23.4s, v1.s[0]\n" + "ldr q23, [x28, #0x80]\n" "prfm pldl1keep, [x24, #0x80]\n" - "fmla v13.4s, v5.4s, v0.s[0]\n" - "fmla v19.4s, v5.4s, v1.s[0]\n" - "ldr q5, [x28, #0x90]\n" + "fmla v13.4s, v22.4s, v0.s[0]\n" + "fmla v19.4s, v22.4s, v1.s[0]\n" + "ldr q22, [x28, #0x90]\n" "prfm pldl1keep, [x23, #0x80]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "ldr q6, [x28, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "ldr q7, [x28, #0xb0]\n" - "fmla v10.4s, v4.4s, v0.s[1]\n" - "fmla v16.4s, v4.4s, v1.s[1]\n" - "ldr q4, [x28, #0xc0]\n" - "fmla v11.4s, v5.4s, v0.s[1]\n" - "fmla v17.4s, v5.4s, v1.s[1]\n" - "ldr q5, [x28, #0xd0]\n" - "fmla v12.4s, v6.4s, v0.s[1]\n" - "fmla v18.4s, v6.4s, v1.s[1]\n" - "ldr q6, [x28, #0xe0]\n" - "fmla v13.4s, v7.4s, v0.s[1]\n" - "fmla v19.4s, v7.4s, v1.s[1]\n" - "ldr q7, [x28, #0xf0]\n" - "fmla v8.4s, v4.4s, v0.s[2]\n" - "fmla v14.4s, v4.4s, v1.s[2]\n" - "ldr q4, [x28, #0x100]\n" - "fmla v9.4s, v5.4s, v0.s[2]\n" - "fmla v15.4s, v5.4s, v1.s[2]\n" - "ldr q5, [x28, #0x110]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v16.4s, v6.4s, v1.s[2]\n" - "ldr q6, [x28, #0x120]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "fmla v17.4s, v7.4s, v1.s[2]\n" - "ldr q7, [x28, #0x130]\n" - "fmla v12.4s, v4.4s, v0.s[2]\n" - "fmla v18.4s, v4.4s, v1.s[2]\n" - "ldr q4, [x28, #0x140]\n" - "fmla v13.4s, v5.4s, v0.s[2]\n" - "fmla v19.4s, v5.4s, v1.s[2]\n" - "ldr q5, [x28, #0x150]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "ldr q6, [x28, #0x160]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" - "ldr q7, [x28, #0x170]\n" + "fmla v8.4s, v21.4s, v0.s[1]\n" + "fmla v14.4s, v21.4s, v1.s[1]\n" + "ldr q21, [x28, #0xa0]\n" + "fmla v9.4s, v20.4s, v0.s[1]\n" + "fmla v15.4s, v20.4s, v1.s[1]\n" + "ldr q20, [x28, #0xb0]\n" + "fmla v10.4s, v23.4s, v0.s[1]\n" + "fmla v16.4s, v23.4s, v1.s[1]\n" + "ldr q23, [x28, #0xc0]\n" + "fmla v11.4s, v22.4s, v0.s[1]\n" + "fmla v17.4s, v22.4s, v1.s[1]\n" + "ldr q22, [x28, #0xd0]\n" + "fmla v12.4s, v21.4s, v0.s[1]\n" + "fmla v18.4s, v21.4s, v1.s[1]\n" + "ldr q21, [x28, #0xe0]\n" + "fmla v13.4s, v20.4s, v0.s[1]\n" + "fmla v19.4s, v20.4s, v1.s[1]\n" + "ldr q20, [x28, #0xf0]\n" + "fmla v8.4s, v23.4s, v0.s[2]\n" + "fmla v14.4s, v23.4s, v1.s[2]\n" + "ldr q23, [x28, #0x100]\n" + "fmla v9.4s, v22.4s, v0.s[2]\n" + "fmla v15.4s, v22.4s, v1.s[2]\n" + "ldr q22, [x28, #0x110]\n" + "fmla v10.4s, v21.4s, v0.s[2]\n" + "fmla v16.4s, v21.4s, v1.s[2]\n" + "ldr q21, [x28, #0x120]\n" + "fmla v11.4s, v20.4s, v0.s[2]\n" + "fmla v17.4s, v20.4s, v1.s[2]\n" + "ldr q20, [x28, #0x130]\n" + "fmla v12.4s, v23.4s, v0.s[2]\n" + "fmla v18.4s, v23.4s, v1.s[2]\n" + "ldr q23, [x28, #0x140]\n" + "fmla v13.4s, v22.4s, v0.s[2]\n" + "fmla v19.4s, v22.4s, v1.s[2]\n" + "ldr q22, [x28, #0x150]\n" + "fmla v8.4s, v21.4s, v0.s[3]\n" + "fmla v14.4s, v21.4s, v1.s[3]\n" + "ldr q21, [x28, #0x160]\n" + "fmla v9.4s, v20.4s, v0.s[3]\n" + "fmla v15.4s, v20.4s, v1.s[3]\n" + "ldr q20, [x28, #0x170]\n" "add x28, x28, #0x180\n" - "fmla v10.4s, v4.4s, v0.s[3]\n" - "fmla v16.4s, v4.4s, v1.s[3]\n" + "fmla v10.4s, v23.4s, v0.s[3]\n" + "fmla v16.4s, v23.4s, v1.s[3]\n" "ldr q4, [x28, #0x0]\n" - "fmla v11.4s, v5.4s, v0.s[3]\n" - "fmla v17.4s, v5.4s, v1.s[3]\n" + "fmla v11.4s, v22.4s, v0.s[3]\n" + "fmla v17.4s, v22.4s, v1.s[3]\n" "ldr q5, [x28, #0x10]\n" - "fmla v12.4s, v6.4s, v0.s[3]\n" - "fmla v18.4s, v6.4s, v1.s[3]\n" + "fmla v12.4s, v21.4s, v0.s[3]\n" + "fmla v18.4s, v21.4s, v1.s[3]\n" "ldr q6, [x28, #0x20]\n" - "fmla v13.4s, v7.4s, v0.s[3]\n" + "fmla v13.4s, v20.4s, v0.s[3]\n" "ldr q0, [x24, #0x0]\n" - "fmla v19.4s, v7.4s, v1.s[3]\n" + "fmla v19.4s, v20.4s, v1.s[3]\n" "ldr q1, [x23, #0x0]\n" "ldr q7, [x28, #0x30]\n" "bge 63b\n" "64:" // Height 2: Multiply loop: Single iteration only "fmla v8.4s, v4.4s, v0.s[0]\n" "fmla v14.4s, v4.4s, v1.s[0]\n" - "ldr q4, [x28, #0x40]\n" + "ldr q23, [x28, #0x40]\n" "add x24, x24, #0x10\n" "fmla v9.4s, v5.4s, v0.s[0]\n" "fmla v15.4s, v5.4s, v1.s[0]\n" - "ldr q5, [x28, #0x50]\n" + "ldr q22, [x28, #0x50]\n" "add x23, x23, #0x10\n" "fmla v10.4s, v6.4s, v0.s[0]\n" "fmla v16.4s, v6.4s, v1.s[0]\n" - "ldr q6, [x28, #0x60]\n" + "ldr q21, [x28, #0x60]\n" "sub x25, x25, #0x4\n" "fmla v11.4s, v7.4s, v0.s[0]\n" "fmla v17.4s, v7.4s, v1.s[0]\n" - "ldr q7, [x28, #0x70]\n" + "ldr q20, [x28, #0x70]\n" "prfm pldl1keep, [x24, #0x80]\n" - "fmla v12.4s, v4.4s, v0.s[0]\n" - "fmla v18.4s, v4.4s, v1.s[0]\n" - "ldr q4, [x28, #0x80]\n" + "fmla v12.4s, v23.4s, v0.s[0]\n" + "fmla v18.4s, v23.4s, v1.s[0]\n" + "ldr q23, [x28, #0x80]\n" "prfm pldl1keep, [x23, #0x80]\n" - "fmla v13.4s, v5.4s, v0.s[0]\n" - "fmla v19.4s, v5.4s, v1.s[0]\n" - "ldr q5, [x28, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "ldr q6, [x28, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "ldr q7, [x28, #0xb0]\n" - "fmla v10.4s, v4.4s, v0.s[1]\n" - "fmla v16.4s, v4.4s, v1.s[1]\n" - "ldr q4, [x28, #0xc0]\n" - "fmla v11.4s, v5.4s, v0.s[1]\n" - "fmla v17.4s, v5.4s, v1.s[1]\n" - "ldr q5, [x28, #0xd0]\n" - "fmla v12.4s, v6.4s, v0.s[1]\n" - "fmla v18.4s, v6.4s, v1.s[1]\n" - "ldr q6, [x28, #0xe0]\n" - "fmla v13.4s, v7.4s, v0.s[1]\n" - "fmla v19.4s, v7.4s, v1.s[1]\n" - "ldr q7, [x28, #0xf0]\n" - "fmla v8.4s, v4.4s, v0.s[2]\n" - "fmla v14.4s, v4.4s, v1.s[2]\n" - "ldr q4, [x28, #0x100]\n" - "fmla v9.4s, v5.4s, v0.s[2]\n" - "fmla v15.4s, v5.4s, v1.s[2]\n" - "ldr q5, [x28, #0x110]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v16.4s, v6.4s, v1.s[2]\n" - "ldr q6, [x28, #0x120]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "fmla v17.4s, v7.4s, v1.s[2]\n" - "ldr q7, [x28, #0x130]\n" - "fmla v12.4s, v4.4s, v0.s[2]\n" - "fmla v18.4s, v4.4s, v1.s[2]\n" - "ldr q4, [x28, #0x140]\n" - "fmla v13.4s, v5.4s, v0.s[2]\n" - "fmla v19.4s, v5.4s, v1.s[2]\n" - "ldr q5, [x28, #0x150]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "ldr q6, [x28, #0x160]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" - "ldr q7, [x28, #0x170]\n" + "fmla v13.4s, v22.4s, v0.s[0]\n" + "fmla v19.4s, v22.4s, v1.s[0]\n" + "ldr q22, [x28, #0x90]\n" + "fmla v8.4s, v21.4s, v0.s[1]\n" + "fmla v14.4s, v21.4s, v1.s[1]\n" + "ldr q21, [x28, #0xa0]\n" + "fmla v9.4s, v20.4s, v0.s[1]\n" + "fmla v15.4s, v20.4s, v1.s[1]\n" + "ldr q20, [x28, #0xb0]\n" + "fmla v10.4s, v23.4s, v0.s[1]\n" + "fmla v16.4s, v23.4s, v1.s[1]\n" + "ldr q23, [x28, #0xc0]\n" + "fmla v11.4s, v22.4s, v0.s[1]\n" + "fmla v17.4s, v22.4s, v1.s[1]\n" + "ldr q22, [x28, #0xd0]\n" + "fmla v12.4s, v21.4s, v0.s[1]\n" + "fmla v18.4s, v21.4s, v1.s[1]\n" + "ldr q21, [x28, #0xe0]\n" + "fmla v13.4s, v20.4s, v0.s[1]\n" + "fmla v19.4s, v20.4s, v1.s[1]\n" + "ldr q20, [x28, #0xf0]\n" + "fmla v8.4s, v23.4s, v0.s[2]\n" + "fmla v14.4s, v23.4s, v1.s[2]\n" + "ldr q23, [x28, #0x100]\n" + "fmla v9.4s, v22.4s, v0.s[2]\n" + "fmla v15.4s, v22.4s, v1.s[2]\n" + "ldr q22, [x28, #0x110]\n" + "fmla v10.4s, v21.4s, v0.s[2]\n" + "fmla v16.4s, v21.4s, v1.s[2]\n" + "ldr q21, [x28, #0x120]\n" + "fmla v11.4s, v20.4s, v0.s[2]\n" + "fmla v17.4s, v20.4s, v1.s[2]\n" + "ldr q20, [x28, #0x130]\n" + "fmla v12.4s, v23.4s, v0.s[2]\n" + "fmla v18.4s, v23.4s, v1.s[2]\n" + "ldr q23, [x28, #0x140]\n" + "fmla v13.4s, v22.4s, v0.s[2]\n" + "fmla v19.4s, v22.4s, v1.s[2]\n" + "ldr q22, [x28, #0x150]\n" + "fmla v8.4s, v21.4s, v0.s[3]\n" + "fmla v14.4s, v21.4s, v1.s[3]\n" + "ldr q21, [x28, #0x160]\n" + "fmla v9.4s, v20.4s, v0.s[3]\n" + "fmla v15.4s, v20.4s, v1.s[3]\n" + "ldr q20, [x28, #0x170]\n" "add x28, x28, #0x180\n" - "fmla v10.4s, v4.4s, v0.s[3]\n" - "fmla v16.4s, v4.4s, v1.s[3]\n" - "fmla v11.4s, v5.4s, v0.s[3]\n" - "fmla v17.4s, v5.4s, v1.s[3]\n" - "fmla v12.4s, v6.4s, v0.s[3]\n" - "fmla v18.4s, v6.4s, v1.s[3]\n" - "fmla v13.4s, v7.4s, v0.s[3]\n" - "fmla v19.4s, v7.4s, v1.s[3]\n" + "fmla v10.4s, v23.4s, v0.s[3]\n" + "fmla v16.4s, v23.4s, v1.s[3]\n" + "fmla v11.4s, v22.4s, v0.s[3]\n" + "fmla v17.4s, v22.4s, v1.s[3]\n" + "fmla v12.4s, v21.4s, v0.s[3]\n" + "fmla v18.4s, v21.4s, v1.s[3]\n" + "fmla v13.4s, v20.4s, v0.s[3]\n" + "fmla v19.4s, v20.4s, v1.s[3]\n" "65:" // Height 2: Multiply loop: Main loop skip "cbz x25, 67f\n" "66:" // Height 2: Multiply loop: Odd block loop - "ldr s0, [x24], #0x4\n" - "ldr s1, [x23], #0x4\n" + "ldr s25, [x24], #0x4\n" + "ldr s24, [x23], #0x4\n" "sub x25, x25, #0x1\n" - "ldr q4, [x28, #0x0]\n" - "ldr q5, [x28, #0x10]\n" - "fmla v8.4s, v4.4s, v0.s[0]\n" - "fmla v14.4s, v4.4s, v1.s[0]\n" - "ldr q6, [x28, #0x20]\n" - "ldr q7, [x28, #0x30]\n" - "fmla v9.4s, v5.4s, v0.s[0]\n" - "fmla v15.4s, v5.4s, v1.s[0]\n" - "ldr q4, [x28, #0x40]\n" - "ldr q5, [x28, #0x50]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v16.4s, v6.4s, v1.s[0]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v17.4s, v7.4s, v1.s[0]\n" + "ldr q21, [x28, #0x0]\n" + "ldr q20, [x28, #0x10]\n" + "fmla v8.4s, v21.4s, v25.s[0]\n" + "fmla v14.4s, v21.4s, v24.s[0]\n" + "ldr q23, [x28, #0x20]\n" + "ldr q22, [x28, #0x30]\n" + "fmla v9.4s, v20.4s, v25.s[0]\n" + "fmla v15.4s, v20.4s, v24.s[0]\n" + "ldr q21, [x28, #0x40]\n" + "ldr q20, [x28, #0x50]\n" + "fmla v10.4s, v23.4s, v25.s[0]\n" + "fmla v16.4s, v23.4s, v24.s[0]\n" + "fmla v11.4s, v22.4s, v25.s[0]\n" + "fmla v17.4s, v22.4s, v24.s[0]\n" "add x28, x28, #0x60\n" - "fmla v12.4s, v4.4s, v0.s[0]\n" - "fmla v18.4s, v4.4s, v1.s[0]\n" - "fmla v13.4s, v5.4s, v0.s[0]\n" - "fmla v19.4s, v5.4s, v1.s[0]\n" + "fmla v12.4s, v21.4s, v25.s[0]\n" + "fmla v18.4s, v21.4s, v24.s[0]\n" + "fmla v13.4s, v20.4s, v25.s[0]\n" + "fmla v19.4s, v20.4s, v24.s[0]\n" "cbnz x25, 66b\n" "67:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -871,33 +870,33 @@ void a64_hybrid_fp32_mla_4x24 ( "prfm pstl1keep, [x23, #0x0]\n" "tbz %x[flags], #1, 68f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v21.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v15.4s, v15.4s, v1.4s\n" - "fmin v16.4s, v16.4s, v1.4s\n" - "fmin v17.4s, v17.4s, v1.4s\n" - "fmin v18.4s, v18.4s, v1.4s\n" - "fmin v19.4s, v19.4s, v1.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" - "fmax v16.4s, v16.4s, v0.4s\n" - "fmax v17.4s, v17.4s, v0.4s\n" - "fmax v18.4s, v18.4s, v0.4s\n" - "fmax v19.4s, v19.4s, v0.4s\n" + "ld1r { v20.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v21.4s\n" + "fmin v9.4s, v9.4s, v21.4s\n" + "fmin v10.4s, v10.4s, v21.4s\n" + "fmin v11.4s, v11.4s, v21.4s\n" + "fmin v12.4s, v12.4s, v21.4s\n" + "fmin v13.4s, v13.4s, v21.4s\n" + "fmin v14.4s, v14.4s, v21.4s\n" + "fmin v15.4s, v15.4s, v21.4s\n" + "fmin v16.4s, v16.4s, v21.4s\n" + "fmin v17.4s, v17.4s, v21.4s\n" + "fmin v18.4s, v18.4s, v21.4s\n" + "fmin v19.4s, v19.4s, v21.4s\n" + "fmax v8.4s, v8.4s, v20.4s\n" + "fmax v9.4s, v9.4s, v20.4s\n" + "fmax v10.4s, v10.4s, v20.4s\n" + "fmax v11.4s, v11.4s, v20.4s\n" + "fmax v12.4s, v12.4s, v20.4s\n" + "fmax v13.4s, v13.4s, v20.4s\n" + "fmax v14.4s, v14.4s, v20.4s\n" + "fmax v15.4s, v15.4s, v20.4s\n" + "fmax v16.4s, v16.4s, v20.4s\n" + "fmax v17.4s, v17.4s, v20.4s\n" + "fmax v18.4s, v18.4s, v20.4s\n" + "fmax v19.4s, v19.4s, v20.4s\n" "68:" // Height 2: No activation "cmp x9, #0x18\n" "bge 81f\n" @@ -1237,13 +1236,13 @@ void a64_hybrid_fp32_mla_4x24 ( "101:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 102f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" "cbnz x26, 103f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20, LSL #2\n" @@ -1252,8 +1251,8 @@ void a64_hybrid_fp32_mla_4x24 ( "b 103f\n" "102:" // Height 3: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" "103:" // Height 3: input setup done "cmp x25, #0x4\n" "blt 106f\n" @@ -1272,107 +1271,107 @@ void a64_hybrid_fp32_mla_4x24 ( "sub x25, x25, #0x4\n" "add x24, x24, #0x10\n" "fmla v20.4s, v4.4s, v2.s[0]\n" - "ldr q4, [x28, #0x40]\n" + "ldr q29, [x28, #0x40]\n" "fmla v9.4s, v5.4s, v0.s[0]\n" "add x23, x23, #0x10\n" "fmla v15.4s, v5.4s, v1.s[0]\n" "fmla v21.4s, v5.4s, v2.s[0]\n" - "ldr q5, [x28, #0x50]\n" + "ldr q28, [x28, #0x50]\n" "add x22, x22, #0x10\n" "fmla v10.4s, v6.4s, v0.s[0]\n" "fmla v16.4s, v6.4s, v1.s[0]\n" "cmp x25, #0x8\n" "prfm pldl1keep, [x24, #0x80]\n" "fmla v22.4s, v6.4s, v2.s[0]\n" - "ldr q6, [x28, #0x60]\n" + "ldr q27, [x28, #0x60]\n" "fmla v11.4s, v7.4s, v0.s[0]\n" "prfm pldl1keep, [x23, #0x80]\n" "fmla v17.4s, v7.4s, v1.s[0]\n" "fmla v23.4s, v7.4s, v2.s[0]\n" - "ldr q7, [x28, #0x70]\n" + "ldr q26, [x28, #0x70]\n" "prfm pldl1keep, [x22, #0x80]\n" - "fmla v12.4s, v4.4s, v0.s[0]\n" - "fmla v18.4s, v4.4s, v1.s[0]\n" - "fmla v24.4s, v4.4s, v2.s[0]\n" - "ldr q4, [x28, #0x80]\n" - "fmla v13.4s, v5.4s, v0.s[0]\n" - "fmla v19.4s, v5.4s, v1.s[0]\n" - "fmla v25.4s, v5.4s, v2.s[0]\n" - "ldr q5, [x28, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "fmla v20.4s, v6.4s, v2.s[1]\n" - "ldr q6, [x28, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "fmla v21.4s, v7.4s, v2.s[1]\n" - "ldr q7, [x28, #0xb0]\n" - "fmla v10.4s, v4.4s, v0.s[1]\n" - "fmla v16.4s, v4.4s, v1.s[1]\n" - "fmla v22.4s, v4.4s, v2.s[1]\n" - "ldr q4, [x28, #0xc0]\n" - "fmla v11.4s, v5.4s, v0.s[1]\n" - "fmla v17.4s, v5.4s, v1.s[1]\n" - "fmla v23.4s, v5.4s, v2.s[1]\n" - "ldr q5, [x28, #0xd0]\n" - "fmla v12.4s, v6.4s, v0.s[1]\n" - "fmla v18.4s, v6.4s, v1.s[1]\n" - "fmla v24.4s, v6.4s, v2.s[1]\n" - "ldr q6, [x28, #0xe0]\n" - "fmla v13.4s, v7.4s, v0.s[1]\n" - "fmla v19.4s, v7.4s, v1.s[1]\n" - "fmla v25.4s, v7.4s, v2.s[1]\n" - "ldr q7, [x28, #0xf0]\n" - "fmla v8.4s, v4.4s, v0.s[2]\n" - "fmla v14.4s, v4.4s, v1.s[2]\n" - "fmla v20.4s, v4.4s, v2.s[2]\n" - "ldr q4, [x28, #0x100]\n" - "fmla v9.4s, v5.4s, v0.s[2]\n" - "fmla v15.4s, v5.4s, v1.s[2]\n" - "fmla v21.4s, v5.4s, v2.s[2]\n" - "ldr q5, [x28, #0x110]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v16.4s, v6.4s, v1.s[2]\n" - "fmla v22.4s, v6.4s, v2.s[2]\n" - "ldr q6, [x28, #0x120]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "fmla v17.4s, v7.4s, v1.s[2]\n" - "fmla v23.4s, v7.4s, v2.s[2]\n" - "ldr q7, [x28, #0x130]\n" - "fmla v12.4s, v4.4s, v0.s[2]\n" - "fmla v18.4s, v4.4s, v1.s[2]\n" - "fmla v24.4s, v4.4s, v2.s[2]\n" - "ldr q4, [x28, #0x140]\n" - "fmla v13.4s, v5.4s, v0.s[2]\n" - "fmla v19.4s, v5.4s, v1.s[2]\n" - "fmla v25.4s, v5.4s, v2.s[2]\n" - "ldr q5, [x28, #0x150]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "fmla v20.4s, v6.4s, v2.s[3]\n" - "ldr q6, [x28, #0x160]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" - "fmla v21.4s, v7.4s, v2.s[3]\n" - "ldr q7, [x28, #0x170]\n" + "fmla v12.4s, v29.4s, v0.s[0]\n" + "fmla v18.4s, v29.4s, v1.s[0]\n" + "fmla v24.4s, v29.4s, v2.s[0]\n" + "ldr q29, [x28, #0x80]\n" + "fmla v13.4s, v28.4s, v0.s[0]\n" + "fmla v19.4s, v28.4s, v1.s[0]\n" + "fmla v25.4s, v28.4s, v2.s[0]\n" + "ldr q28, [x28, #0x90]\n" + "fmla v8.4s, v27.4s, v0.s[1]\n" + "fmla v14.4s, v27.4s, v1.s[1]\n" + "fmla v20.4s, v27.4s, v2.s[1]\n" + "ldr q27, [x28, #0xa0]\n" + "fmla v9.4s, v26.4s, v0.s[1]\n" + "fmla v15.4s, v26.4s, v1.s[1]\n" + "fmla v21.4s, v26.4s, v2.s[1]\n" + "ldr q26, [x28, #0xb0]\n" + "fmla v10.4s, v29.4s, v0.s[1]\n" + "fmla v16.4s, v29.4s, v1.s[1]\n" + "fmla v22.4s, v29.4s, v2.s[1]\n" + "ldr q29, [x28, #0xc0]\n" + "fmla v11.4s, v28.4s, v0.s[1]\n" + "fmla v17.4s, v28.4s, v1.s[1]\n" + "fmla v23.4s, v28.4s, v2.s[1]\n" + "ldr q28, [x28, #0xd0]\n" + "fmla v12.4s, v27.4s, v0.s[1]\n" + "fmla v18.4s, v27.4s, v1.s[1]\n" + "fmla v24.4s, v27.4s, v2.s[1]\n" + "ldr q27, [x28, #0xe0]\n" + "fmla v13.4s, v26.4s, v0.s[1]\n" + "fmla v19.4s, v26.4s, v1.s[1]\n" + "fmla v25.4s, v26.4s, v2.s[1]\n" + "ldr q26, [x28, #0xf0]\n" + "fmla v8.4s, v29.4s, v0.s[2]\n" + "fmla v14.4s, v29.4s, v1.s[2]\n" + "fmla v20.4s, v29.4s, v2.s[2]\n" + "ldr q29, [x28, #0x100]\n" + "fmla v9.4s, v28.4s, v0.s[2]\n" + "fmla v15.4s, v28.4s, v1.s[2]\n" + "fmla v21.4s, v28.4s, v2.s[2]\n" + "ldr q28, [x28, #0x110]\n" + "fmla v10.4s, v27.4s, v0.s[2]\n" + "fmla v16.4s, v27.4s, v1.s[2]\n" + "fmla v22.4s, v27.4s, v2.s[2]\n" + "ldr q27, [x28, #0x120]\n" + "fmla v11.4s, v26.4s, v0.s[2]\n" + "fmla v17.4s, v26.4s, v1.s[2]\n" + "fmla v23.4s, v26.4s, v2.s[2]\n" + "ldr q26, [x28, #0x130]\n" + "fmla v12.4s, v29.4s, v0.s[2]\n" + "fmla v18.4s, v29.4s, v1.s[2]\n" + "fmla v24.4s, v29.4s, v2.s[2]\n" + "ldr q29, [x28, #0x140]\n" + "fmla v13.4s, v28.4s, v0.s[2]\n" + "fmla v19.4s, v28.4s, v1.s[2]\n" + "fmla v25.4s, v28.4s, v2.s[2]\n" + "ldr q28, [x28, #0x150]\n" + "fmla v8.4s, v27.4s, v0.s[3]\n" + "fmla v14.4s, v27.4s, v1.s[3]\n" + "fmla v20.4s, v27.4s, v2.s[3]\n" + "ldr q27, [x28, #0x160]\n" + "fmla v9.4s, v26.4s, v0.s[3]\n" + "fmla v15.4s, v26.4s, v1.s[3]\n" + "fmla v21.4s, v26.4s, v2.s[3]\n" + "ldr q26, [x28, #0x170]\n" "add x28, x28, #0x180\n" - "fmla v10.4s, v4.4s, v0.s[3]\n" - "fmla v16.4s, v4.4s, v1.s[3]\n" - "fmla v22.4s, v4.4s, v2.s[3]\n" + "fmla v10.4s, v29.4s, v0.s[3]\n" + "fmla v16.4s, v29.4s, v1.s[3]\n" + "fmla v22.4s, v29.4s, v2.s[3]\n" "ldr q4, [x28, #0x0]\n" - "fmla v11.4s, v5.4s, v0.s[3]\n" - "fmla v17.4s, v5.4s, v1.s[3]\n" - "fmla v23.4s, v5.4s, v2.s[3]\n" + "fmla v11.4s, v28.4s, v0.s[3]\n" + "fmla v17.4s, v28.4s, v1.s[3]\n" + "fmla v23.4s, v28.4s, v2.s[3]\n" "ldr q5, [x28, #0x10]\n" - "fmla v12.4s, v6.4s, v0.s[3]\n" - "fmla v18.4s, v6.4s, v1.s[3]\n" - "fmla v24.4s, v6.4s, v2.s[3]\n" + "fmla v12.4s, v27.4s, v0.s[3]\n" + "fmla v18.4s, v27.4s, v1.s[3]\n" + "fmla v24.4s, v27.4s, v2.s[3]\n" "ldr q6, [x28, #0x20]\n" - "fmla v13.4s, v7.4s, v0.s[3]\n" + "fmla v13.4s, v26.4s, v0.s[3]\n" "ldr q0, [x24, #0x0]\n" - "fmla v19.4s, v7.4s, v1.s[3]\n" + "fmla v19.4s, v26.4s, v1.s[3]\n" "ldr q1, [x23, #0x0]\n" - "fmla v25.4s, v7.4s, v2.s[3]\n" + "fmla v25.4s, v26.4s, v2.s[3]\n" "ldr q2, [x22, #0x0]\n" "ldr q7, [x28, #0x30]\n" "bge 104b\n" @@ -1382,133 +1381,133 @@ void a64_hybrid_fp32_mla_4x24 ( "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" "fmla v20.4s, v4.4s, v2.s[0]\n" - "ldr q4, [x28, #0x40]\n" + "ldr q29, [x28, #0x40]\n" "fmla v9.4s, v5.4s, v0.s[0]\n" "add x22, x22, #0x10\n" "fmla v15.4s, v5.4s, v1.s[0]\n" "fmla v21.4s, v5.4s, v2.s[0]\n" - "ldr q5, [x28, #0x50]\n" + "ldr q28, [x28, #0x50]\n" "sub x25, x25, #0x4\n" "fmla v10.4s, v6.4s, v0.s[0]\n" "fmla v16.4s, v6.4s, v1.s[0]\n" "prfm pldl1keep, [x24, #0x80]\n" "prfm pldl1keep, [x23, #0x80]\n" "fmla v22.4s, v6.4s, v2.s[0]\n" - "ldr q6, [x28, #0x60]\n" + "ldr q27, [x28, #0x60]\n" "fmla v11.4s, v7.4s, v0.s[0]\n" "prfm pldl1keep, [x22, #0x80]\n" "fmla v17.4s, v7.4s, v1.s[0]\n" "fmla v23.4s, v7.4s, v2.s[0]\n" - "ldr q7, [x28, #0x70]\n" - "fmla v12.4s, v4.4s, v0.s[0]\n" - "fmla v18.4s, v4.4s, v1.s[0]\n" - "fmla v24.4s, v4.4s, v2.s[0]\n" - "ldr q4, [x28, #0x80]\n" - "fmla v13.4s, v5.4s, v0.s[0]\n" - "fmla v19.4s, v5.4s, v1.s[0]\n" - "fmla v25.4s, v5.4s, v2.s[0]\n" - "ldr q5, [x28, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "fmla v20.4s, v6.4s, v2.s[1]\n" - "ldr q6, [x28, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "fmla v21.4s, v7.4s, v2.s[1]\n" - "ldr q7, [x28, #0xb0]\n" - "fmla v10.4s, v4.4s, v0.s[1]\n" - "fmla v16.4s, v4.4s, v1.s[1]\n" - "fmla v22.4s, v4.4s, v2.s[1]\n" - "ldr q4, [x28, #0xc0]\n" - "fmla v11.4s, v5.4s, v0.s[1]\n" - "fmla v17.4s, v5.4s, v1.s[1]\n" - "fmla v23.4s, v5.4s, v2.s[1]\n" - "ldr q5, [x28, #0xd0]\n" - "fmla v12.4s, v6.4s, v0.s[1]\n" - "fmla v18.4s, v6.4s, v1.s[1]\n" - "fmla v24.4s, v6.4s, v2.s[1]\n" - "ldr q6, [x28, #0xe0]\n" - "fmla v13.4s, v7.4s, v0.s[1]\n" - "fmla v19.4s, v7.4s, v1.s[1]\n" - "fmla v25.4s, v7.4s, v2.s[1]\n" - "ldr q7, [x28, #0xf0]\n" - "fmla v8.4s, v4.4s, v0.s[2]\n" - "fmla v14.4s, v4.4s, v1.s[2]\n" - "fmla v20.4s, v4.4s, v2.s[2]\n" - "ldr q4, [x28, #0x100]\n" - "fmla v9.4s, v5.4s, v0.s[2]\n" - "fmla v15.4s, v5.4s, v1.s[2]\n" - "fmla v21.4s, v5.4s, v2.s[2]\n" - "ldr q5, [x28, #0x110]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v16.4s, v6.4s, v1.s[2]\n" - "fmla v22.4s, v6.4s, v2.s[2]\n" - "ldr q6, [x28, #0x120]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "fmla v17.4s, v7.4s, v1.s[2]\n" - "fmla v23.4s, v7.4s, v2.s[2]\n" - "ldr q7, [x28, #0x130]\n" - "fmla v12.4s, v4.4s, v0.s[2]\n" - "fmla v18.4s, v4.4s, v1.s[2]\n" - "fmla v24.4s, v4.4s, v2.s[2]\n" - "ldr q4, [x28, #0x140]\n" - "fmla v13.4s, v5.4s, v0.s[2]\n" - "fmla v19.4s, v5.4s, v1.s[2]\n" - "fmla v25.4s, v5.4s, v2.s[2]\n" - "ldr q5, [x28, #0x150]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "fmla v20.4s, v6.4s, v2.s[3]\n" - "ldr q6, [x28, #0x160]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" - "fmla v21.4s, v7.4s, v2.s[3]\n" - "ldr q7, [x28, #0x170]\n" + "ldr q26, [x28, #0x70]\n" + "fmla v12.4s, v29.4s, v0.s[0]\n" + "fmla v18.4s, v29.4s, v1.s[0]\n" + "fmla v24.4s, v29.4s, v2.s[0]\n" + "ldr q29, [x28, #0x80]\n" + "fmla v13.4s, v28.4s, v0.s[0]\n" + "fmla v19.4s, v28.4s, v1.s[0]\n" + "fmla v25.4s, v28.4s, v2.s[0]\n" + "ldr q28, [x28, #0x90]\n" + "fmla v8.4s, v27.4s, v0.s[1]\n" + "fmla v14.4s, v27.4s, v1.s[1]\n" + "fmla v20.4s, v27.4s, v2.s[1]\n" + "ldr q27, [x28, #0xa0]\n" + "fmla v9.4s, v26.4s, v0.s[1]\n" + "fmla v15.4s, v26.4s, v1.s[1]\n" + "fmla v21.4s, v26.4s, v2.s[1]\n" + "ldr q26, [x28, #0xb0]\n" + "fmla v10.4s, v29.4s, v0.s[1]\n" + "fmla v16.4s, v29.4s, v1.s[1]\n" + "fmla v22.4s, v29.4s, v2.s[1]\n" + "ldr q29, [x28, #0xc0]\n" + "fmla v11.4s, v28.4s, v0.s[1]\n" + "fmla v17.4s, v28.4s, v1.s[1]\n" + "fmla v23.4s, v28.4s, v2.s[1]\n" + "ldr q28, [x28, #0xd0]\n" + "fmla v12.4s, v27.4s, v0.s[1]\n" + "fmla v18.4s, v27.4s, v1.s[1]\n" + "fmla v24.4s, v27.4s, v2.s[1]\n" + "ldr q27, [x28, #0xe0]\n" + "fmla v13.4s, v26.4s, v0.s[1]\n" + "fmla v19.4s, v26.4s, v1.s[1]\n" + "fmla v25.4s, v26.4s, v2.s[1]\n" + "ldr q26, [x28, #0xf0]\n" + "fmla v8.4s, v29.4s, v0.s[2]\n" + "fmla v14.4s, v29.4s, v1.s[2]\n" + "fmla v20.4s, v29.4s, v2.s[2]\n" + "ldr q29, [x28, #0x100]\n" + "fmla v9.4s, v28.4s, v0.s[2]\n" + "fmla v15.4s, v28.4s, v1.s[2]\n" + "fmla v21.4s, v28.4s, v2.s[2]\n" + "ldr q28, [x28, #0x110]\n" + "fmla v10.4s, v27.4s, v0.s[2]\n" + "fmla v16.4s, v27.4s, v1.s[2]\n" + "fmla v22.4s, v27.4s, v2.s[2]\n" + "ldr q27, [x28, #0x120]\n" + "fmla v11.4s, v26.4s, v0.s[2]\n" + "fmla v17.4s, v26.4s, v1.s[2]\n" + "fmla v23.4s, v26.4s, v2.s[2]\n" + "ldr q26, [x28, #0x130]\n" + "fmla v12.4s, v29.4s, v0.s[2]\n" + "fmla v18.4s, v29.4s, v1.s[2]\n" + "fmla v24.4s, v29.4s, v2.s[2]\n" + "ldr q29, [x28, #0x140]\n" + "fmla v13.4s, v28.4s, v0.s[2]\n" + "fmla v19.4s, v28.4s, v1.s[2]\n" + "fmla v25.4s, v28.4s, v2.s[2]\n" + "ldr q28, [x28, #0x150]\n" + "fmla v8.4s, v27.4s, v0.s[3]\n" + "fmla v14.4s, v27.4s, v1.s[3]\n" + "fmla v20.4s, v27.4s, v2.s[3]\n" + "ldr q27, [x28, #0x160]\n" + "fmla v9.4s, v26.4s, v0.s[3]\n" + "fmla v15.4s, v26.4s, v1.s[3]\n" + "fmla v21.4s, v26.4s, v2.s[3]\n" + "ldr q26, [x28, #0x170]\n" "add x28, x28, #0x180\n" - "fmla v10.4s, v4.4s, v0.s[3]\n" - "fmla v16.4s, v4.4s, v1.s[3]\n" - "fmla v22.4s, v4.4s, v2.s[3]\n" - "fmla v11.4s, v5.4s, v0.s[3]\n" - "fmla v17.4s, v5.4s, v1.s[3]\n" - "fmla v23.4s, v5.4s, v2.s[3]\n" - "fmla v12.4s, v6.4s, v0.s[3]\n" - "fmla v18.4s, v6.4s, v1.s[3]\n" - "fmla v24.4s, v6.4s, v2.s[3]\n" - "fmla v13.4s, v7.4s, v0.s[3]\n" - "fmla v19.4s, v7.4s, v1.s[3]\n" - "fmla v25.4s, v7.4s, v2.s[3]\n" + "fmla v10.4s, v29.4s, v0.s[3]\n" + "fmla v16.4s, v29.4s, v1.s[3]\n" + "fmla v22.4s, v29.4s, v2.s[3]\n" + "fmla v11.4s, v28.4s, v0.s[3]\n" + "fmla v17.4s, v28.4s, v1.s[3]\n" + "fmla v23.4s, v28.4s, v2.s[3]\n" + "fmla v12.4s, v27.4s, v0.s[3]\n" + "fmla v18.4s, v27.4s, v1.s[3]\n" + "fmla v24.4s, v27.4s, v2.s[3]\n" + "fmla v13.4s, v26.4s, v0.s[3]\n" + "fmla v19.4s, v26.4s, v1.s[3]\n" + "fmla v25.4s, v26.4s, v2.s[3]\n" "106:" // Height 3: Multiply loop: Main loop skip "cbz x25, 108f\n" "107:" // Height 3: Multiply loop: Odd block loop "ldr s0, [x24], #0x4\n" - "ldr s1, [x23], #0x4\n" + "ldr s31, [x23], #0x4\n" "sub x25, x25, #0x1\n" - "ldr s2, [x22], #0x4\n" - "ldr q4, [x28, #0x0]\n" - "fmla v8.4s, v4.4s, v0.s[0]\n" - "fmla v14.4s, v4.4s, v1.s[0]\n" - "ldr q5, [x28, #0x10]\n" - "ldr q6, [x28, #0x20]\n" - "fmla v20.4s, v4.4s, v2.s[0]\n" - "fmla v9.4s, v5.4s, v0.s[0]\n" - "ldr q7, [x28, #0x30]\n" - "ldr q4, [x28, #0x40]\n" - "fmla v15.4s, v5.4s, v1.s[0]\n" - "fmla v21.4s, v5.4s, v2.s[0]\n" - "ldr q5, [x28, #0x50]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v16.4s, v6.4s, v1.s[0]\n" + "ldr s30, [x22], #0x4\n" + "ldr q27, [x28, #0x0]\n" + "fmla v8.4s, v27.4s, v0.s[0]\n" + "fmla v14.4s, v27.4s, v31.s[0]\n" + "ldr q26, [x28, #0x10]\n" + "ldr q29, [x28, #0x20]\n" + "fmla v20.4s, v27.4s, v30.s[0]\n" + "fmla v9.4s, v26.4s, v0.s[0]\n" + "ldr q28, [x28, #0x30]\n" + "ldr q27, [x28, #0x40]\n" + "fmla v15.4s, v26.4s, v31.s[0]\n" + "fmla v21.4s, v26.4s, v30.s[0]\n" + "ldr q26, [x28, #0x50]\n" + "fmla v10.4s, v29.4s, v0.s[0]\n" + "fmla v16.4s, v29.4s, v31.s[0]\n" "add x28, x28, #0x60\n" - "fmla v22.4s, v6.4s, v2.s[0]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v17.4s, v7.4s, v1.s[0]\n" - "fmla v23.4s, v7.4s, v2.s[0]\n" - "fmla v12.4s, v4.4s, v0.s[0]\n" - "fmla v18.4s, v4.4s, v1.s[0]\n" - "fmla v24.4s, v4.4s, v2.s[0]\n" - "fmla v13.4s, v5.4s, v0.s[0]\n" - "fmla v19.4s, v5.4s, v1.s[0]\n" - "fmla v25.4s, v5.4s, v2.s[0]\n" + "fmla v22.4s, v29.4s, v30.s[0]\n" + "fmla v11.4s, v28.4s, v0.s[0]\n" + "fmla v17.4s, v28.4s, v31.s[0]\n" + "fmla v23.4s, v28.4s, v30.s[0]\n" + "fmla v12.4s, v27.4s, v0.s[0]\n" + "fmla v18.4s, v27.4s, v31.s[0]\n" + "fmla v24.4s, v27.4s, v30.s[0]\n" + "fmla v13.4s, v26.4s, v0.s[0]\n" + "fmla v19.4s, v26.4s, v31.s[0]\n" + "fmla v25.4s, v26.4s, v30.s[0]\n" "cbnz x25, 107b\n" "108:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -1523,45 +1522,45 @@ void a64_hybrid_fp32_mla_4x24 ( "prfm pstl1keep, [x22, #0x0]\n" "tbz %x[flags], #1, 109f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v27.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v15.4s, v15.4s, v1.4s\n" - "fmin v16.4s, v16.4s, v1.4s\n" - "fmin v17.4s, v17.4s, v1.4s\n" - "fmin v18.4s, v18.4s, v1.4s\n" - "fmin v19.4s, v19.4s, v1.4s\n" - "fmin v20.4s, v20.4s, v1.4s\n" - "fmin v21.4s, v21.4s, v1.4s\n" - "fmin v22.4s, v22.4s, v1.4s\n" - "fmin v23.4s, v23.4s, v1.4s\n" - "fmin v24.4s, v24.4s, v1.4s\n" - "fmin v25.4s, v25.4s, v1.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" - "fmax v16.4s, v16.4s, v0.4s\n" - "fmax v17.4s, v17.4s, v0.4s\n" - "fmax v18.4s, v18.4s, v0.4s\n" - "fmax v19.4s, v19.4s, v0.4s\n" - "fmax v20.4s, v20.4s, v0.4s\n" - "fmax v21.4s, v21.4s, v0.4s\n" - "fmax v22.4s, v22.4s, v0.4s\n" - "fmax v23.4s, v23.4s, v0.4s\n" - "fmax v24.4s, v24.4s, v0.4s\n" - "fmax v25.4s, v25.4s, v0.4s\n" + "ld1r { v26.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v27.4s\n" + "fmin v9.4s, v9.4s, v27.4s\n" + "fmin v10.4s, v10.4s, v27.4s\n" + "fmin v11.4s, v11.4s, v27.4s\n" + "fmin v12.4s, v12.4s, v27.4s\n" + "fmin v13.4s, v13.4s, v27.4s\n" + "fmin v14.4s, v14.4s, v27.4s\n" + "fmin v15.4s, v15.4s, v27.4s\n" + "fmin v16.4s, v16.4s, v27.4s\n" + "fmin v17.4s, v17.4s, v27.4s\n" + "fmin v18.4s, v18.4s, v27.4s\n" + "fmin v19.4s, v19.4s, v27.4s\n" + "fmin v20.4s, v20.4s, v27.4s\n" + "fmin v21.4s, v21.4s, v27.4s\n" + "fmin v22.4s, v22.4s, v27.4s\n" + "fmin v23.4s, v23.4s, v27.4s\n" + "fmin v24.4s, v24.4s, v27.4s\n" + "fmin v25.4s, v25.4s, v27.4s\n" + "fmax v8.4s, v8.4s, v26.4s\n" + "fmax v9.4s, v9.4s, v26.4s\n" + "fmax v10.4s, v10.4s, v26.4s\n" + "fmax v11.4s, v11.4s, v26.4s\n" + "fmax v12.4s, v12.4s, v26.4s\n" + "fmax v13.4s, v13.4s, v26.4s\n" + "fmax v14.4s, v14.4s, v26.4s\n" + "fmax v15.4s, v15.4s, v26.4s\n" + "fmax v16.4s, v16.4s, v26.4s\n" + "fmax v17.4s, v17.4s, v26.4s\n" + "fmax v18.4s, v18.4s, v26.4s\n" + "fmax v19.4s, v19.4s, v26.4s\n" + "fmax v20.4s, v20.4s, v26.4s\n" + "fmax v21.4s, v21.4s, v26.4s\n" + "fmax v22.4s, v22.4s, v26.4s\n" + "fmax v23.4s, v23.4s, v26.4s\n" + "fmax v24.4s, v24.4s, v26.4s\n" + "fmax v25.4s, v25.4s, v26.4s\n" "109:" // Height 3: No activation "cmp x9, #0x18\n" "bge 122f\n" @@ -1983,14 +1982,14 @@ void a64_hybrid_fp32_mla_4x24 ( "142:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 143f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" - "ldr x21, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" + "ldr x21, [x20, #0x18]\n" "cbnz x26, 144f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20, LSL #2\n" @@ -2000,9 +1999,9 @@ void a64_hybrid_fp32_mla_4x24 ( "b 144f\n" "143:" // Height 4: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "add x21, x22, x20, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" + "add x21, x22, x21, LSL #2\n" "144:" // Height 4: input setup done "cmp x25, #0x4\n" "blt 147f\n" @@ -2283,42 +2282,42 @@ void a64_hybrid_fp32_mla_4x24 ( "147:" // Height 4: Multiply loop: Main loop skip "cbz x25, 149f\n" "148:" // Height 4: Multiply loop: Odd block loop - "ldr s0, [x24], #0x4\n" - "ldr s1, [x23], #0x4\n" + "ldr s7, [x24], #0x4\n" + "ldr s6, [x23], #0x4\n" "sub x25, x25, #0x1\n" - "ldr s2, [x22], #0x4\n" - "ldr s3, [x21], #0x4\n" - "ldr q4, [x28, #0x0]\n" - "ldr q5, [x28, #0x10]\n" - "fmla v8.4s, v4.4s, v0.s[0]\n" - "fmla v14.4s, v4.4s, v1.s[0]\n" - "ldr q6, [x28, #0x20]\n" - "ldr q7, [x28, #0x30]\n" - "fmla v20.4s, v4.4s, v2.s[0]\n" - "fmla v26.4s, v4.4s, v3.s[0]\n" - "ldr q4, [x28, #0x40]\n" - "fmla v9.4s, v5.4s, v0.s[0]\n" - "fmla v15.4s, v5.4s, v1.s[0]\n" - "fmla v21.4s, v5.4s, v2.s[0]\n" - "fmla v27.4s, v5.4s, v3.s[0]\n" - "ldr q5, [x28, #0x50]\n" + "ldr s5, [x22], #0x4\n" + "ldr s4, [x21], #0x4\n" + "ldr q1, [x28, #0x0]\n" + "ldr q0, [x28, #0x10]\n" + "fmla v8.4s, v1.4s, v7.s[0]\n" + "fmla v14.4s, v1.4s, v6.s[0]\n" + "ldr q3, [x28, #0x20]\n" + "ldr q2, [x28, #0x30]\n" + "fmla v20.4s, v1.4s, v5.s[0]\n" + "fmla v26.4s, v1.4s, v4.s[0]\n" + "ldr q1, [x28, #0x40]\n" + "fmla v9.4s, v0.4s, v7.s[0]\n" + "fmla v15.4s, v0.4s, v6.s[0]\n" + "fmla v21.4s, v0.4s, v5.s[0]\n" + "fmla v27.4s, v0.4s, v4.s[0]\n" + "ldr q0, [x28, #0x50]\n" "add x28, x28, #0x60\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v16.4s, v6.4s, v1.s[0]\n" - "fmla v22.4s, v6.4s, v2.s[0]\n" - "fmla v28.4s, v6.4s, v3.s[0]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v17.4s, v7.4s, v1.s[0]\n" - "fmla v23.4s, v7.4s, v2.s[0]\n" - "fmla v29.4s, v7.4s, v3.s[0]\n" - "fmla v12.4s, v4.4s, v0.s[0]\n" - "fmla v18.4s, v4.4s, v1.s[0]\n" - "fmla v24.4s, v4.4s, v2.s[0]\n" - "fmla v30.4s, v4.4s, v3.s[0]\n" - "fmla v13.4s, v5.4s, v0.s[0]\n" - "fmla v19.4s, v5.4s, v1.s[0]\n" - "fmla v25.4s, v5.4s, v2.s[0]\n" - "fmla v31.4s, v5.4s, v3.s[0]\n" + "fmla v10.4s, v3.4s, v7.s[0]\n" + "fmla v16.4s, v3.4s, v6.s[0]\n" + "fmla v22.4s, v3.4s, v5.s[0]\n" + "fmla v28.4s, v3.4s, v4.s[0]\n" + "fmla v11.4s, v2.4s, v7.s[0]\n" + "fmla v17.4s, v2.4s, v6.s[0]\n" + "fmla v23.4s, v2.4s, v5.s[0]\n" + "fmla v29.4s, v2.4s, v4.s[0]\n" + "fmla v12.4s, v1.4s, v7.s[0]\n" + "fmla v18.4s, v1.4s, v6.s[0]\n" + "fmla v24.4s, v1.4s, v5.s[0]\n" + "fmla v30.4s, v1.4s, v4.s[0]\n" + "fmla v13.4s, v0.4s, v7.s[0]\n" + "fmla v19.4s, v0.4s, v6.s[0]\n" + "fmla v25.4s, v0.4s, v5.s[0]\n" + "fmla v31.4s, v0.4s, v4.s[0]\n" "cbnz x25, 148b\n" "149:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -2584,7 +2583,6 @@ void a64_hybrid_fp32_mla_4x24 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "166:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16.hpp index 4cfa18bb84e3f34697b2db64560f794cf3d55906..759729de5e607f522632df5effc1083b49933a78 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef __aarch64__ + #include "../std_transforms_fixed.hpp" #include "../performance_parameters.hpp" @@ -113,5 +113,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16/a55.cpp index 985d57d9b6598742ac24174c216202042f3c45f7..ddbc840829408239a76e9ef381bdbeaa6f32e572 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16/a55.cpp @@ -92,7 +92,6 @@ void a64_hybrid_fp32_mla_6x16_a55 ( break; } __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x6\n" "bge 166f\n" @@ -189,11 +188,11 @@ void a64_hybrid_fp32_mla_6x16_a55 ( "15:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w14, [x20, x15, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 16f\n" - "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x13, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x15, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x13, [x20, #0x0]\n" "cbnz x15, 17f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x13, x13, x20, LSL #2\n" @@ -210,126 +209,126 @@ void a64_hybrid_fp32_mla_6x16_a55 ( "blt 19f\n" "18:" // Height 1: Multiply loop: Main loop head "fmla v8.4s, v6.4s, v0.s[0]\n" - "ldr d6, [x17, #0x20]\n" - "ldr x12, [x17, #0x28]\n" + "ldr d17, [x17, #0x20]\n" + "ldr x20, [x17, #0x28]\n" "fmla v9.4s, v7.4s, v0.s[0]\n" - "ldr d7, [x17, #0x30]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x38]\n" - "mov v7.d[1], x11\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "ldr d6, [x17, #0x40]\n" - "ldr x12, [x17, #0x48]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "ldr d7, [x17, #0x50]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x58]\n" - "mov v7.d[1], x11\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "ldr d6, [x17, #0x60]\n" - "ldr x12, [x17, #0x68]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "ldr d7, [x17, #0x70]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x78]\n" - "mov v7.d[1], x11\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "ldr d6, [x17, #0x80]\n" - "ldr x12, [x17, #0x88]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "ldr d7, [x17, #0x90]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x98]\n" - "mov v7.d[1], x11\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "ldr d6, [x17, #0xa0]\n" - "ldr x12, [x17, #0xa8]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "ldr d7, [x17, #0xb0]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0xb8]\n" - "mov v7.d[1], x11\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "ldr d6, [x17, #0xc0]\n" - "ldr x12, [x17, #0xc8]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "ldr d7, [x17, #0xd0]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0xd8]\n" - "mov v7.d[1], x11\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "ldr d6, [x17, #0xe0]\n" - "ldr x12, [x17, #0xe8]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "ldr d7, [x17, #0xf0]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0xf8]\n" - "mov v7.d[1], x11\n" + "ldr d16, [x17, #0x30]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0x38]\n" + "mov v16.d[1], x20\n" + "fmla v10.4s, v17.4s, v0.s[0]\n" + "ldr d17, [x17, #0x40]\n" + "ldr x20, [x17, #0x48]\n" + "fmla v11.4s, v16.4s, v0.s[0]\n" + "ldr d16, [x17, #0x50]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0x58]\n" + "mov v16.d[1], x20\n" + "fmla v8.4s, v17.4s, v0.s[1]\n" + "ldr d17, [x17, #0x60]\n" + "ldr x20, [x17, #0x68]\n" + "fmla v9.4s, v16.4s, v0.s[1]\n" + "ldr d16, [x17, #0x70]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0x78]\n" + "mov v16.d[1], x20\n" + "fmla v10.4s, v17.4s, v0.s[1]\n" + "ldr d17, [x17, #0x80]\n" + "ldr x20, [x17, #0x88]\n" + "fmla v11.4s, v16.4s, v0.s[1]\n" + "ldr d16, [x17, #0x90]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0x98]\n" + "mov v16.d[1], x20\n" + "fmla v8.4s, v17.4s, v0.s[2]\n" + "ldr d17, [x17, #0xa0]\n" + "ldr x20, [x17, #0xa8]\n" + "fmla v9.4s, v16.4s, v0.s[2]\n" + "ldr d16, [x17, #0xb0]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0xb8]\n" + "mov v16.d[1], x20\n" + "fmla v10.4s, v17.4s, v0.s[2]\n" + "ldr d17, [x17, #0xc0]\n" + "ldr x20, [x17, #0xc8]\n" + "fmla v11.4s, v16.4s, v0.s[2]\n" + "ldr d16, [x17, #0xd0]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0xd8]\n" + "mov v16.d[1], x20\n" + "fmla v8.4s, v17.4s, v0.s[3]\n" + "ldr d17, [x17, #0xe0]\n" + "ldr x20, [x17, #0xe8]\n" + "fmla v9.4s, v16.4s, v0.s[3]\n" + "ldr d16, [x17, #0xf0]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0xf8]\n" + "mov v16.d[1], x20\n" "add x13, x13, #0x10\n" "add x17, x17, #0x100\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" + "fmla v10.4s, v17.4s, v0.s[3]\n" "ldr d6, [x17, #0x0]\n" - "ldr x12, [x17, #0x8]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" + "ldr x20, [x17, #0x8]\n" + "fmla v11.4s, v16.4s, v0.s[3]\n" "ldr d0, [x13, #0x0]\n" "sub x14, x14, #0x4\n" "ldr d7, [x17, #0x10]\n" "cmp x14, #0x8\n" - "ldr x10, [x13, #0x8]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x18]\n" - "mov v0.d[1], x10\n" - "mov v7.d[1], x11\n" + "ldr x21, [x13, #0x8]\n" + "mov v6.d[1], x20\n" + "ldr x20, [x17, #0x18]\n" + "mov v0.d[1], x21\n" + "mov v7.d[1], x20\n" "prfm pldl1keep, [x13, #0x80]\n" "bge 18b\n" "19:" // Height 1: Multiply loop: Single iteration only "fmla v8.4s, v6.4s, v0.s[0]\n" - "ldr q6, [x17, #0x20]\n" + "ldr q17, [x17, #0x20]\n" "fmla v9.4s, v7.4s, v0.s[0]\n" - "ldr q7, [x17, #0x30]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "ldr q6, [x17, #0x40]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "ldr q7, [x17, #0x50]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "ldr q6, [x17, #0x60]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "ldr q7, [x17, #0x70]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "ldr q6, [x17, #0x80]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "ldr q7, [x17, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "ldr q6, [x17, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "ldr q7, [x17, #0xb0]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "ldr q6, [x17, #0xc0]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "ldr q7, [x17, #0xd0]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "ldr q6, [x17, #0xe0]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "ldr q7, [x17, #0xf0]\n" + "ldr q16, [x17, #0x30]\n" + "fmla v10.4s, v17.4s, v0.s[0]\n" + "ldr q17, [x17, #0x40]\n" + "fmla v11.4s, v16.4s, v0.s[0]\n" + "ldr q16, [x17, #0x50]\n" + "fmla v8.4s, v17.4s, v0.s[1]\n" + "ldr q17, [x17, #0x60]\n" + "fmla v9.4s, v16.4s, v0.s[1]\n" + "ldr q16, [x17, #0x70]\n" + "fmla v10.4s, v17.4s, v0.s[1]\n" + "ldr q17, [x17, #0x80]\n" + "fmla v11.4s, v16.4s, v0.s[1]\n" + "ldr q16, [x17, #0x90]\n" + "fmla v8.4s, v17.4s, v0.s[2]\n" + "ldr q17, [x17, #0xa0]\n" + "fmla v9.4s, v16.4s, v0.s[2]\n" + "ldr q16, [x17, #0xb0]\n" + "fmla v10.4s, v17.4s, v0.s[2]\n" + "ldr q17, [x17, #0xc0]\n" + "fmla v11.4s, v16.4s, v0.s[2]\n" + "ldr q16, [x17, #0xd0]\n" + "fmla v8.4s, v17.4s, v0.s[3]\n" + "ldr q17, [x17, #0xe0]\n" + "fmla v9.4s, v16.4s, v0.s[3]\n" + "ldr q16, [x17, #0xf0]\n" "add x13, x13, #0x10\n" "sub x14, x14, #0x4\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" + "fmla v10.4s, v17.4s, v0.s[3]\n" "prfm pldl1keep, [x13, #0x80]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" + "fmla v11.4s, v16.4s, v0.s[3]\n" "add x17, x17, #0x100\n" "20:" // Height 1: Multiply loop: Main loop skip "cbz x14, 22f\n" "21:" // Height 1: Multiply loop: Odd block loop - "ldr s0, [x13], #0x4\n" + "ldr s17, [x13], #0x4\n" "sub x14, x14, #0x1\n" - "ldr q6, [x17, #0x0]\n" - "fmla v8.4s, v6.4s, v0.s[0]\n" - "ldr q7, [x17, #0x10]\n" - "fmla v9.4s, v7.4s, v0.s[0]\n" - "ldr q6, [x17, #0x20]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "ldr q7, [x17, #0x30]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" + "ldr q16, [x17, #0x0]\n" + "fmla v8.4s, v16.4s, v17.s[0]\n" + "ldr q16, [x17, #0x10]\n" + "fmla v9.4s, v16.4s, v17.s[0]\n" + "ldr q16, [x17, #0x20]\n" + "fmla v10.4s, v16.4s, v17.s[0]\n" + "ldr q16, [x17, #0x30]\n" + "fmla v11.4s, v16.4s, v17.s[0]\n" "add x17, x17, #0x40\n" "cbnz x14, 21b\n" "22:" // Height 1: Multiply loop: No odd multiplies @@ -340,17 +339,17 @@ void a64_hybrid_fp32_mla_6x16_a55 ( "prfm pstl1keep, [x16, #0x0]\n" "tbz %x[flags], #1, 23f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v0.4s\n" - "fmin v9.4s, v9.4s, v0.4s\n" - "fmin v10.4s, v10.4s, v0.4s\n" - "fmin v11.4s, v11.4s, v0.4s\n" + "ld1r { v16.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v16.4s\n" + "fmin v9.4s, v9.4s, v16.4s\n" + "fmin v10.4s, v10.4s, v16.4s\n" + "fmin v11.4s, v11.4s, v16.4s\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" + "ld1r { v16.4s }, [x20]\n" + "fmax v8.4s, v8.4s, v16.4s\n" + "fmax v9.4s, v9.4s, v16.4s\n" + "fmax v10.4s, v10.4s, v16.4s\n" + "fmax v11.4s, v11.4s, v16.4s\n" "23:" // Height 1: No activation "cmp x8, #0x10\n" "bge 32f\n" @@ -528,196 +527,196 @@ void a64_hybrid_fp32_mla_6x16_a55 ( "48:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w14, [x20, x15, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 49f\n" - "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x13, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x15, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x13, [x20, #0x0]\n" + "ldr x12, [x20, #0x8]\n" "cbnz x15, 50f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x13, x13, x20, LSL #2\n" - "add x9, x9, x20, LSL #2\n" + "add x12, x12, x20, LSL #2\n" "b 50f\n" "49:" // Height 2: setup direct input "mov x13, %x[input_ptr]\n" - "add x9, x13, x20, LSL #2\n" + "add x12, x13, x21, LSL #2\n" "50:" // Height 2: input setup done "cmp x14, #0x4\n" "blt 53f\n" "ldr q0, [x13, #0x0]\n" "cmp x14, #0x8\n" - "ldr q1, [x9, #0x0]\n" + "ldr q1, [x12, #0x0]\n" "ldr q6, [x17, #0x0]\n" "ldr q7, [x17, #0x10]\n" "blt 52f\n" "51:" // Height 2: Multiply loop: Main loop head "fmla v8.4s, v6.4s, v0.s[0]\n" - "ldr x12, [x17, #0x28]\n" + "ldr x21, [x17, #0x28]\n" "fmla v12.4s, v6.4s, v1.s[0]\n" - "ldr d6, [x17, #0x20]\n" + "ldr d17, [x17, #0x20]\n" "fmla v9.4s, v7.4s, v0.s[0]\n" - "ldr x11, [x17, #0x38]\n" + "ldr x20, [x17, #0x38]\n" "fmla v13.4s, v7.4s, v1.s[0]\n" - "ldr d7, [x17, #0x30]\n" - "mov v6.d[1], x12\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "mov v7.d[1], x11\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "ldr d6, [x17, #0x40]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "ldr x12, [x17, #0x48]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "ldr d7, [x17, #0x50]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x58]\n" - "mov v7.d[1], x11\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "ldr x12, [x17, #0x68]\n" - "fmla v12.4s, v6.4s, v1.s[1]\n" - "ldr d6, [x17, #0x60]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "ldr x11, [x17, #0x78]\n" - "fmla v13.4s, v7.4s, v1.s[1]\n" - "ldr d7, [x17, #0x70]\n" - "mov v6.d[1], x12\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "mov v7.d[1], x11\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "ldr d6, [x17, #0x80]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "ldr x12, [x17, #0x88]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "ldr d7, [x17, #0x90]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x98]\n" - "mov v7.d[1], x11\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "ldr x12, [x17, #0xa8]\n" - "fmla v12.4s, v6.4s, v1.s[2]\n" - "ldr d6, [x17, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "ldr x11, [x17, #0xb8]\n" - "fmla v13.4s, v7.4s, v1.s[2]\n" - "ldr d7, [x17, #0xb0]\n" - "mov v6.d[1], x12\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "mov v7.d[1], x11\n" - "fmla v14.4s, v6.4s, v1.s[2]\n" - "ldr d6, [x17, #0xc0]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "ldr x12, [x17, #0xc8]\n" - "fmla v15.4s, v7.4s, v1.s[2]\n" - "ldr d7, [x17, #0xd0]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0xd8]\n" - "mov v7.d[1], x11\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "ldr x12, [x17, #0xe8]\n" - "fmla v12.4s, v6.4s, v1.s[3]\n" - "ldr d6, [x17, #0xe0]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "ldr x11, [x17, #0xf8]\n" - "fmla v13.4s, v7.4s, v1.s[3]\n" - "ldr d7, [x17, #0xf0]\n" - "mov v6.d[1], x12\n" + "ldr d16, [x17, #0x30]\n" + "mov v17.d[1], x21\n" + "fmla v10.4s, v17.4s, v0.s[0]\n" + "mov v16.d[1], x20\n" + "fmla v14.4s, v17.4s, v1.s[0]\n" + "ldr d17, [x17, #0x40]\n" + "fmla v11.4s, v16.4s, v0.s[0]\n" + "ldr x20, [x17, #0x48]\n" + "fmla v15.4s, v16.4s, v1.s[0]\n" + "ldr d16, [x17, #0x50]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0x58]\n" + "mov v16.d[1], x20\n" + "fmla v8.4s, v17.4s, v0.s[1]\n" + "ldr x21, [x17, #0x68]\n" + "fmla v12.4s, v17.4s, v1.s[1]\n" + "ldr d17, [x17, #0x60]\n" + "fmla v9.4s, v16.4s, v0.s[1]\n" + "ldr x20, [x17, #0x78]\n" + "fmla v13.4s, v16.4s, v1.s[1]\n" + "ldr d16, [x17, #0x70]\n" + "mov v17.d[1], x21\n" + "fmla v10.4s, v17.4s, v0.s[1]\n" + "mov v16.d[1], x20\n" + "fmla v14.4s, v17.4s, v1.s[1]\n" + "ldr d17, [x17, #0x80]\n" + "fmla v11.4s, v16.4s, v0.s[1]\n" + "ldr x20, [x17, #0x88]\n" + "fmla v15.4s, v16.4s, v1.s[1]\n" + "ldr d16, [x17, #0x90]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0x98]\n" + "mov v16.d[1], x20\n" + "fmla v8.4s, v17.4s, v0.s[2]\n" + "ldr x21, [x17, #0xa8]\n" + "fmla v12.4s, v17.4s, v1.s[2]\n" + "ldr d17, [x17, #0xa0]\n" + "fmla v9.4s, v16.4s, v0.s[2]\n" + "ldr x20, [x17, #0xb8]\n" + "fmla v13.4s, v16.4s, v1.s[2]\n" + "ldr d16, [x17, #0xb0]\n" + "mov v17.d[1], x21\n" + "fmla v10.4s, v17.4s, v0.s[2]\n" + "mov v16.d[1], x20\n" + "fmla v14.4s, v17.4s, v1.s[2]\n" + "ldr d17, [x17, #0xc0]\n" + "fmla v11.4s, v16.4s, v0.s[2]\n" + "ldr x20, [x17, #0xc8]\n" + "fmla v15.4s, v16.4s, v1.s[2]\n" + "ldr d16, [x17, #0xd0]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x17, #0xd8]\n" + "mov v16.d[1], x20\n" + "fmla v8.4s, v17.4s, v0.s[3]\n" + "ldr x21, [x17, #0xe8]\n" + "fmla v12.4s, v17.4s, v1.s[3]\n" + "ldr d17, [x17, #0xe0]\n" + "fmla v9.4s, v16.4s, v0.s[3]\n" + "ldr x20, [x17, #0xf8]\n" + "fmla v13.4s, v16.4s, v1.s[3]\n" + "ldr d16, [x17, #0xf0]\n" + "mov v17.d[1], x21\n" "add x13, x13, #0x10\n" - "mov v7.d[1], x11\n" - "add x9, x9, #0x10\n" + "mov v16.d[1], x20\n" + "add x12, x12, #0x10\n" "add x17, x17, #0x100\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" + "fmla v10.4s, v17.4s, v0.s[3]\n" + "fmla v14.4s, v17.4s, v1.s[3]\n" "ldr d6, [x17, #0x0]\n" - "ldr x12, [x17, #0x8]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" + "ldr x21, [x17, #0x8]\n" + "fmla v11.4s, v16.4s, v0.s[3]\n" "ldr d0, [x13, #0x0]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" - "ldr d1, [x9, #0x0]\n" + "fmla v15.4s, v16.4s, v1.s[3]\n" + "ldr d1, [x12, #0x0]\n" "sub x14, x14, #0x4\n" "ldr d7, [x17, #0x10]\n" "cmp x14, #0x8\n" - "ldr x10, [x13, #0x8]\n" - "mov v6.d[1], x12\n" - "ldr x28, [x9, #0x8]\n" - "mov v0.d[1], x10\n" - "ldr x11, [x17, #0x18]\n" - "mov v1.d[1], x28\n" + "ldr x20, [x13, #0x8]\n" + "mov v6.d[1], x21\n" + "ldr x21, [x12, #0x8]\n" + "mov v0.d[1], x20\n" + "ldr x20, [x17, #0x18]\n" + "mov v1.d[1], x21\n" "prfm pldl1keep, [x13, #0x80]\n" - "mov v7.d[1], x11\n" - "prfm pldl1keep, [x9, #0x80]\n" + "mov v7.d[1], x20\n" + "prfm pldl1keep, [x12, #0x80]\n" "bge 51b\n" "52:" // Height 2: Multiply loop: Single iteration only "fmla v8.4s, v6.4s, v0.s[0]\n" "add x13, x13, #0x10\n" "fmla v12.4s, v6.4s, v1.s[0]\n" - "ldr q6, [x17, #0x20]\n" + "ldr q17, [x17, #0x20]\n" "fmla v9.4s, v7.4s, v0.s[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" "fmla v13.4s, v7.4s, v1.s[0]\n" - "ldr q7, [x17, #0x30]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" + "ldr q16, [x17, #0x30]\n" + "fmla v10.4s, v17.4s, v0.s[0]\n" "sub x14, x14, #0x4\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "ldr q6, [x17, #0x40]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" + "fmla v14.4s, v17.4s, v1.s[0]\n" + "ldr q17, [x17, #0x40]\n" + "fmla v11.4s, v16.4s, v0.s[0]\n" "prfm pldl1keep, [x13, #0x80]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "ldr q7, [x17, #0x50]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "prfm pldl1keep, [x9, #0x80]\n" - "fmla v12.4s, v6.4s, v1.s[1]\n" - "ldr q6, [x17, #0x60]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v13.4s, v7.4s, v1.s[1]\n" - "ldr q7, [x17, #0x70]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "ldr q6, [x17, #0x80]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "ldr q7, [x17, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "fmla v12.4s, v6.4s, v1.s[2]\n" - "ldr q6, [x17, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "fmla v13.4s, v7.4s, v1.s[2]\n" - "ldr q7, [x17, #0xb0]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v14.4s, v6.4s, v1.s[2]\n" - "ldr q6, [x17, #0xc0]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "fmla v15.4s, v7.4s, v1.s[2]\n" - "ldr q7, [x17, #0xd0]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v12.4s, v6.4s, v1.s[3]\n" - "ldr q6, [x17, #0xe0]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "fmla v13.4s, v7.4s, v1.s[3]\n" - "ldr q7, [x17, #0xf0]\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" + "fmla v15.4s, v16.4s, v1.s[0]\n" + "ldr q16, [x17, #0x50]\n" + "fmla v8.4s, v17.4s, v0.s[1]\n" + "prfm pldl1keep, [x12, #0x80]\n" + "fmla v12.4s, v17.4s, v1.s[1]\n" + "ldr q17, [x17, #0x60]\n" + "fmla v9.4s, v16.4s, v0.s[1]\n" + "fmla v13.4s, v16.4s, v1.s[1]\n" + "ldr q16, [x17, #0x70]\n" + "fmla v10.4s, v17.4s, v0.s[1]\n" + "fmla v14.4s, v17.4s, v1.s[1]\n" + "ldr q17, [x17, #0x80]\n" + "fmla v11.4s, v16.4s, v0.s[1]\n" + "fmla v15.4s, v16.4s, v1.s[1]\n" + "ldr q16, [x17, #0x90]\n" + "fmla v8.4s, v17.4s, v0.s[2]\n" + "fmla v12.4s, v17.4s, v1.s[2]\n" + "ldr q17, [x17, #0xa0]\n" + "fmla v9.4s, v16.4s, v0.s[2]\n" + "fmla v13.4s, v16.4s, v1.s[2]\n" + "ldr q16, [x17, #0xb0]\n" + "fmla v10.4s, v17.4s, v0.s[2]\n" + "fmla v14.4s, v17.4s, v1.s[2]\n" + "ldr q17, [x17, #0xc0]\n" + "fmla v11.4s, v16.4s, v0.s[2]\n" + "fmla v15.4s, v16.4s, v1.s[2]\n" + "ldr q16, [x17, #0xd0]\n" + "fmla v8.4s, v17.4s, v0.s[3]\n" + "fmla v12.4s, v17.4s, v1.s[3]\n" + "ldr q17, [x17, #0xe0]\n" + "fmla v9.4s, v16.4s, v0.s[3]\n" + "fmla v13.4s, v16.4s, v1.s[3]\n" + "ldr q16, [x17, #0xf0]\n" + "fmla v10.4s, v17.4s, v0.s[3]\n" "add x17, x17, #0x100\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" + "fmla v14.4s, v17.4s, v1.s[3]\n" + "fmla v11.4s, v16.4s, v0.s[3]\n" + "fmla v15.4s, v16.4s, v1.s[3]\n" "53:" // Height 2: Multiply loop: Main loop skip "cbz x14, 55f\n" "54:" // Height 2: Multiply loop: Odd block loop - "ldr s0, [x13], #0x4\n" + "ldr s19, [x13], #0x4\n" "sub x14, x14, #0x1\n" - "ldr s1, [x9], #0x4\n" - "ldr q6, [x17, #0x0]\n" - "fmla v8.4s, v6.4s, v0.s[0]\n" - "ldr q7, [x17, #0x10]\n" - "fmla v12.4s, v6.4s, v1.s[0]\n" - "ldr q6, [x17, #0x20]\n" - "fmla v9.4s, v7.4s, v0.s[0]\n" - "fmla v13.4s, v7.4s, v1.s[0]\n" - "ldr q7, [x17, #0x30]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" + "ldr s18, [x12], #0x4\n" + "ldr q17, [x17, #0x0]\n" + "fmla v8.4s, v17.4s, v19.s[0]\n" + "ldr q16, [x17, #0x10]\n" + "fmla v12.4s, v17.4s, v18.s[0]\n" + "ldr q17, [x17, #0x20]\n" + "fmla v9.4s, v16.4s, v19.s[0]\n" + "fmla v13.4s, v16.4s, v18.s[0]\n" + "ldr q16, [x17, #0x30]\n" + "fmla v10.4s, v17.4s, v19.s[0]\n" "add x17, x17, #0x40\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" + "fmla v14.4s, v17.4s, v18.s[0]\n" + "fmla v11.4s, v16.4s, v19.s[0]\n" + "fmla v15.4s, v16.4s, v18.s[0]\n" "cbnz x14, 54b\n" "55:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -730,25 +729,25 @@ void a64_hybrid_fp32_mla_6x16_a55 ( "prfm pstl1keep, [x25, #0x0]\n" "tbz %x[flags], #1, 56f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v0.4s\n" - "fmin v9.4s, v9.4s, v0.4s\n" - "fmin v10.4s, v10.4s, v0.4s\n" - "fmin v11.4s, v11.4s, v0.4s\n" - "fmin v12.4s, v12.4s, v0.4s\n" - "fmin v13.4s, v13.4s, v0.4s\n" - "fmin v14.4s, v14.4s, v0.4s\n" - "fmin v15.4s, v15.4s, v0.4s\n" + "ld1r { v16.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v16.4s\n" + "fmin v9.4s, v9.4s, v16.4s\n" + "fmin v10.4s, v10.4s, v16.4s\n" + "fmin v11.4s, v11.4s, v16.4s\n" + "fmin v12.4s, v12.4s, v16.4s\n" + "fmin v13.4s, v13.4s, v16.4s\n" + "fmin v14.4s, v14.4s, v16.4s\n" + "fmin v15.4s, v15.4s, v16.4s\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" + "ld1r { v16.4s }, [x20]\n" + "fmax v8.4s, v8.4s, v16.4s\n" + "fmax v9.4s, v9.4s, v16.4s\n" + "fmax v10.4s, v10.4s, v16.4s\n" + "fmax v11.4s, v11.4s, v16.4s\n" + "fmax v12.4s, v12.4s, v16.4s\n" + "fmax v13.4s, v13.4s, v16.4s\n" + "fmax v14.4s, v14.4s, v16.4s\n" + "fmax v15.4s, v15.4s, v16.4s\n" "56:" // Height 2: No activation "cmp x8, #0x10\n" "bge 65f\n" @@ -975,244 +974,244 @@ void a64_hybrid_fp32_mla_6x16_a55 ( "81:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w14, [x20, x15, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 82f\n" - "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x13, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" - "ldr x27, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x15, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x13, [x20, #0x0]\n" + "ldr x12, [x20, #0x8]\n" + "ldr x11, [x20, #0x10]\n" "cbnz x15, 83f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x13, x13, x20, LSL #2\n" - "add x9, x9, x20, LSL #2\n" - "add x27, x27, x20, LSL #2\n" + "add x12, x12, x20, LSL #2\n" + "add x11, x11, x20, LSL #2\n" "b 83f\n" "82:" // Height 3: setup direct input "mov x13, %x[input_ptr]\n" - "add x9, x13, x20, LSL #2\n" - "add x27, x9, x20, LSL #2\n" + "add x12, x13, x21, LSL #2\n" + "add x11, x12, x21, LSL #2\n" "83:" // Height 3: input setup done "cmp x14, #0x4\n" "blt 86f\n" "ldr q0, [x13, #0x0]\n" "cmp x14, #0x8\n" - "ldr q1, [x9, #0x0]\n" - "ldr q2, [x27, #0x0]\n" + "ldr q1, [x12, #0x0]\n" + "ldr q2, [x11, #0x0]\n" "ldr q6, [x17, #0x0]\n" "ldr q7, [x17, #0x10]\n" "blt 85f\n" "84:" // Height 3: Multiply loop: Main loop head "fmla v8.4s, v6.4s, v0.s[0]\n" - "ldr x12, [x17, #0x28]\n" + "ldr x21, [x17, #0x28]\n" "fmla v12.4s, v6.4s, v1.s[0]\n" - "ldr x11, [x17, #0x38]\n" + "ldr x20, [x17, #0x38]\n" "fmla v16.4s, v6.4s, v2.s[0]\n" - "ldr d6, [x17, #0x20]\n" + "ldr d21, [x17, #0x20]\n" "fmla v9.4s, v7.4s, v0.s[0]\n" - "mov v6.d[1], x12\n" + "mov v21.d[1], x21\n" "fmla v13.4s, v7.4s, v1.s[0]\n" - "ldr x12, [x17, #0x48]\n" + "ldr x21, [x17, #0x48]\n" "fmla v17.4s, v7.4s, v2.s[0]\n" - "ldr d7, [x17, #0x30]\n" - "mov v7.d[1], x11\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "ldr x11, [x17, #0x58]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "ldr d6, [x17, #0x40]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "mov v6.d[1], x12\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "ldr x12, [x17, #0x68]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "ldr d7, [x17, #0x50]\n" - "mov v7.d[1], x11\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v12.4s, v6.4s, v1.s[1]\n" - "ldr x11, [x17, #0x78]\n" - "fmla v16.4s, v6.4s, v2.s[1]\n" - "ldr d6, [x17, #0x60]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "mov v6.d[1], x12\n" - "fmla v13.4s, v7.4s, v1.s[1]\n" - "ldr x12, [x17, #0x88]\n" - "fmla v17.4s, v7.4s, v2.s[1]\n" - "ldr d7, [x17, #0x70]\n" - "mov v7.d[1], x11\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "ldr x11, [x17, #0x98]\n" - "fmla v18.4s, v6.4s, v2.s[1]\n" - "ldr d6, [x17, #0x80]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "mov v6.d[1], x12\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "ldr x12, [x17, #0xa8]\n" - "fmla v19.4s, v7.4s, v2.s[1]\n" - "ldr d7, [x17, #0x90]\n" - "mov v7.d[1], x11\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "fmla v12.4s, v6.4s, v1.s[2]\n" - "ldr x11, [x17, #0xb8]\n" - "fmla v16.4s, v6.4s, v2.s[2]\n" - "ldr d6, [x17, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "mov v6.d[1], x12\n" - "fmla v13.4s, v7.4s, v1.s[2]\n" - "ldr x12, [x17, #0xc8]\n" - "fmla v17.4s, v7.4s, v2.s[2]\n" - "ldr d7, [x17, #0xb0]\n" - "mov v7.d[1], x11\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v14.4s, v6.4s, v1.s[2]\n" - "ldr x11, [x17, #0xd8]\n" - "fmla v18.4s, v6.4s, v2.s[2]\n" - "ldr d6, [x17, #0xc0]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "mov v6.d[1], x12\n" - "fmla v15.4s, v7.4s, v1.s[2]\n" - "ldr x12, [x17, #0xe8]\n" - "fmla v19.4s, v7.4s, v2.s[2]\n" - "ldr d7, [x17, #0xd0]\n" - "mov v7.d[1], x11\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v12.4s, v6.4s, v1.s[3]\n" - "ldr x11, [x17, #0xf8]\n" - "fmla v16.4s, v6.4s, v2.s[3]\n" - "ldr d6, [x17, #0xe0]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "mov v6.d[1], x12\n" - "fmla v13.4s, v7.4s, v1.s[3]\n" + "ldr d20, [x17, #0x30]\n" + "mov v20.d[1], x20\n" + "fmla v10.4s, v21.4s, v0.s[0]\n" + "fmla v14.4s, v21.4s, v1.s[0]\n" + "ldr x20, [x17, #0x58]\n" + "fmla v18.4s, v21.4s, v2.s[0]\n" + "ldr d21, [x17, #0x40]\n" + "fmla v11.4s, v20.4s, v0.s[0]\n" + "mov v21.d[1], x21\n" + "fmla v15.4s, v20.4s, v1.s[0]\n" + "ldr x21, [x17, #0x68]\n" + "fmla v19.4s, v20.4s, v2.s[0]\n" + "ldr d20, [x17, #0x50]\n" + "mov v20.d[1], x20\n" + "fmla v8.4s, v21.4s, v0.s[1]\n" + "fmla v12.4s, v21.4s, v1.s[1]\n" + "ldr x20, [x17, #0x78]\n" + "fmla v16.4s, v21.4s, v2.s[1]\n" + "ldr d21, [x17, #0x60]\n" + "fmla v9.4s, v20.4s, v0.s[1]\n" + "mov v21.d[1], x21\n" + "fmla v13.4s, v20.4s, v1.s[1]\n" + "ldr x21, [x17, #0x88]\n" + "fmla v17.4s, v20.4s, v2.s[1]\n" + "ldr d20, [x17, #0x70]\n" + "mov v20.d[1], x20\n" + "fmla v10.4s, v21.4s, v0.s[1]\n" + "fmla v14.4s, v21.4s, v1.s[1]\n" + "ldr x20, [x17, #0x98]\n" + "fmla v18.4s, v21.4s, v2.s[1]\n" + "ldr d21, [x17, #0x80]\n" + "fmla v11.4s, v20.4s, v0.s[1]\n" + "mov v21.d[1], x21\n" + "fmla v15.4s, v20.4s, v1.s[1]\n" + "ldr x21, [x17, #0xa8]\n" + "fmla v19.4s, v20.4s, v2.s[1]\n" + "ldr d20, [x17, #0x90]\n" + "mov v20.d[1], x20\n" + "fmla v8.4s, v21.4s, v0.s[2]\n" + "fmla v12.4s, v21.4s, v1.s[2]\n" + "ldr x20, [x17, #0xb8]\n" + "fmla v16.4s, v21.4s, v2.s[2]\n" + "ldr d21, [x17, #0xa0]\n" + "fmla v9.4s, v20.4s, v0.s[2]\n" + "mov v21.d[1], x21\n" + "fmla v13.4s, v20.4s, v1.s[2]\n" + "ldr x21, [x17, #0xc8]\n" + "fmla v17.4s, v20.4s, v2.s[2]\n" + "ldr d20, [x17, #0xb0]\n" + "mov v20.d[1], x20\n" + "fmla v10.4s, v21.4s, v0.s[2]\n" + "fmla v14.4s, v21.4s, v1.s[2]\n" + "ldr x20, [x17, #0xd8]\n" + "fmla v18.4s, v21.4s, v2.s[2]\n" + "ldr d21, [x17, #0xc0]\n" + "fmla v11.4s, v20.4s, v0.s[2]\n" + "mov v21.d[1], x21\n" + "fmla v15.4s, v20.4s, v1.s[2]\n" + "ldr x21, [x17, #0xe8]\n" + "fmla v19.4s, v20.4s, v2.s[2]\n" + "ldr d20, [x17, #0xd0]\n" + "mov v20.d[1], x20\n" + "fmla v8.4s, v21.4s, v0.s[3]\n" + "fmla v12.4s, v21.4s, v1.s[3]\n" + "ldr x20, [x17, #0xf8]\n" + "fmla v16.4s, v21.4s, v2.s[3]\n" + "ldr d21, [x17, #0xe0]\n" + "fmla v9.4s, v20.4s, v0.s[3]\n" + "mov v21.d[1], x21\n" + "fmla v13.4s, v20.4s, v1.s[3]\n" "add x13, x13, #0x10\n" - "fmla v17.4s, v7.4s, v2.s[3]\n" - "ldr d7, [x17, #0xf0]\n" - "mov v7.d[1], x11\n" - "add x9, x9, #0x10\n" - "add x27, x27, #0x10\n" + "fmla v17.4s, v20.4s, v2.s[3]\n" + "ldr d20, [x17, #0xf0]\n" + "mov v20.d[1], x20\n" + "add x12, x12, #0x10\n" + "add x11, x11, #0x10\n" "add x17, x17, #0x100\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" - "ldr x12, [x17, #0x8]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "ldr x10, [x13, #0x8]\n" - "fmla v18.4s, v6.4s, v2.s[3]\n" + "fmla v10.4s, v21.4s, v0.s[3]\n" + "ldr x20, [x17, #0x8]\n" + "fmla v14.4s, v21.4s, v1.s[3]\n" + "ldr x23, [x13, #0x8]\n" + "fmla v18.4s, v21.4s, v2.s[3]\n" "ldr d6, [x17, #0x0]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" + "fmla v11.4s, v20.4s, v0.s[3]\n" "ldr d0, [x13, #0x0]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" - "ldr d1, [x9, #0x0]\n" - "ldr x28, [x9, #0x8]\n" - "fmla v19.4s, v7.4s, v2.s[3]\n" - "ldr d2, [x27, #0x0]\n" + "fmla v15.4s, v20.4s, v1.s[3]\n" + "ldr d1, [x12, #0x0]\n" + "ldr x22, [x12, #0x8]\n" + "fmla v19.4s, v20.4s, v2.s[3]\n" + "ldr d2, [x11, #0x0]\n" "sub x14, x14, #0x4\n" "ldr d7, [x17, #0x10]\n" "cmp x14, #0x8\n" - "ldr x26, [x27, #0x8]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x17, #0x18]\n" - "mov v0.d[1], x10\n" + "ldr x21, [x11, #0x8]\n" + "mov v6.d[1], x20\n" + "ldr x20, [x17, #0x18]\n" + "mov v0.d[1], x23\n" "prfm pldl1keep, [x13, #0x80]\n" - "mov v1.d[1], x28\n" - "prfm pldl1keep, [x9, #0x80]\n" - "mov v2.d[1], x26\n" - "prfm pldl1keep, [x27, #0x80]\n" - "mov v7.d[1], x11\n" + "mov v1.d[1], x22\n" + "prfm pldl1keep, [x12, #0x80]\n" + "mov v2.d[1], x21\n" + "prfm pldl1keep, [x11, #0x80]\n" + "mov v7.d[1], x20\n" "bge 84b\n" "85:" // Height 3: Multiply loop: Single iteration only "fmla v8.4s, v6.4s, v0.s[0]\n" "add x13, x13, #0x10\n" "fmla v12.4s, v6.4s, v1.s[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" "fmla v16.4s, v6.4s, v2.s[0]\n" - "ldr q6, [x17, #0x20]\n" + "ldr q21, [x17, #0x20]\n" "fmla v9.4s, v7.4s, v0.s[0]\n" - "add x27, x27, #0x10\n" + "add x11, x11, #0x10\n" "fmla v13.4s, v7.4s, v1.s[0]\n" "sub x14, x14, #0x4\n" "fmla v17.4s, v7.4s, v2.s[0]\n" - "ldr q7, [x17, #0x30]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" + "ldr q20, [x17, #0x30]\n" + "fmla v10.4s, v21.4s, v0.s[0]\n" "prfm pldl1keep, [x13, #0x80]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "prfm pldl1keep, [x9, #0x80]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "ldr q6, [x17, #0x40]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "prfm pldl1keep, [x27, #0x80]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "ldr q7, [x17, #0x50]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v12.4s, v6.4s, v1.s[1]\n" - "fmla v16.4s, v6.4s, v2.s[1]\n" - "ldr q6, [x17, #0x60]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v13.4s, v7.4s, v1.s[1]\n" - "fmla v17.4s, v7.4s, v2.s[1]\n" - "ldr q7, [x17, #0x70]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "fmla v18.4s, v6.4s, v2.s[1]\n" - "ldr q6, [x17, #0x80]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "fmla v19.4s, v7.4s, v2.s[1]\n" - "ldr q7, [x17, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "fmla v12.4s, v6.4s, v1.s[2]\n" - "fmla v16.4s, v6.4s, v2.s[2]\n" - "ldr q6, [x17, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "fmla v13.4s, v7.4s, v1.s[2]\n" - "fmla v17.4s, v7.4s, v2.s[2]\n" - "ldr q7, [x17, #0xb0]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v14.4s, v6.4s, v1.s[2]\n" - "fmla v18.4s, v6.4s, v2.s[2]\n" - "ldr q6, [x17, #0xc0]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "fmla v15.4s, v7.4s, v1.s[2]\n" - "fmla v19.4s, v7.4s, v2.s[2]\n" - "ldr q7, [x17, #0xd0]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v12.4s, v6.4s, v1.s[3]\n" - "fmla v16.4s, v6.4s, v2.s[3]\n" - "ldr q6, [x17, #0xe0]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "fmla v13.4s, v7.4s, v1.s[3]\n" - "fmla v17.4s, v7.4s, v2.s[3]\n" - "ldr q7, [x17, #0xf0]\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" + "fmla v14.4s, v21.4s, v1.s[0]\n" + "prfm pldl1keep, [x12, #0x80]\n" + "fmla v18.4s, v21.4s, v2.s[0]\n" + "ldr q21, [x17, #0x40]\n" + "fmla v11.4s, v20.4s, v0.s[0]\n" + "prfm pldl1keep, [x11, #0x80]\n" + "fmla v15.4s, v20.4s, v1.s[0]\n" + "fmla v19.4s, v20.4s, v2.s[0]\n" + "ldr q20, [x17, #0x50]\n" + "fmla v8.4s, v21.4s, v0.s[1]\n" + "fmla v12.4s, v21.4s, v1.s[1]\n" + "fmla v16.4s, v21.4s, v2.s[1]\n" + "ldr q21, [x17, #0x60]\n" + "fmla v9.4s, v20.4s, v0.s[1]\n" + "fmla v13.4s, v20.4s, v1.s[1]\n" + "fmla v17.4s, v20.4s, v2.s[1]\n" + "ldr q20, [x17, #0x70]\n" + "fmla v10.4s, v21.4s, v0.s[1]\n" + "fmla v14.4s, v21.4s, v1.s[1]\n" + "fmla v18.4s, v21.4s, v2.s[1]\n" + "ldr q21, [x17, #0x80]\n" + "fmla v11.4s, v20.4s, v0.s[1]\n" + "fmla v15.4s, v20.4s, v1.s[1]\n" + "fmla v19.4s, v20.4s, v2.s[1]\n" + "ldr q20, [x17, #0x90]\n" + "fmla v8.4s, v21.4s, v0.s[2]\n" + "fmla v12.4s, v21.4s, v1.s[2]\n" + "fmla v16.4s, v21.4s, v2.s[2]\n" + "ldr q21, [x17, #0xa0]\n" + "fmla v9.4s, v20.4s, v0.s[2]\n" + "fmla v13.4s, v20.4s, v1.s[2]\n" + "fmla v17.4s, v20.4s, v2.s[2]\n" + "ldr q20, [x17, #0xb0]\n" + "fmla v10.4s, v21.4s, v0.s[2]\n" + "fmla v14.4s, v21.4s, v1.s[2]\n" + "fmla v18.4s, v21.4s, v2.s[2]\n" + "ldr q21, [x17, #0xc0]\n" + "fmla v11.4s, v20.4s, v0.s[2]\n" + "fmla v15.4s, v20.4s, v1.s[2]\n" + "fmla v19.4s, v20.4s, v2.s[2]\n" + "ldr q20, [x17, #0xd0]\n" + "fmla v8.4s, v21.4s, v0.s[3]\n" + "fmla v12.4s, v21.4s, v1.s[3]\n" + "fmla v16.4s, v21.4s, v2.s[3]\n" + "ldr q21, [x17, #0xe0]\n" + "fmla v9.4s, v20.4s, v0.s[3]\n" + "fmla v13.4s, v20.4s, v1.s[3]\n" + "fmla v17.4s, v20.4s, v2.s[3]\n" + "ldr q20, [x17, #0xf0]\n" + "fmla v10.4s, v21.4s, v0.s[3]\n" "add x17, x17, #0x100\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "fmla v18.4s, v6.4s, v2.s[3]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" - "fmla v19.4s, v7.4s, v2.s[3]\n" + "fmla v14.4s, v21.4s, v1.s[3]\n" + "fmla v18.4s, v21.4s, v2.s[3]\n" + "fmla v11.4s, v20.4s, v0.s[3]\n" + "fmla v15.4s, v20.4s, v1.s[3]\n" + "fmla v19.4s, v20.4s, v2.s[3]\n" "86:" // Height 3: Multiply loop: Main loop skip - "cbz x14, 88f\n" - "87:" // Height 3: Multiply loop: Odd block loop - "ldr s0, [x13], #0x4\n" - "sub x14, x14, #0x1\n" - "ldr s1, [x9], #0x4\n" - "ldr s2, [x27], #0x4\n" - "ldr q6, [x17, #0x0]\n" - "fmla v8.4s, v6.4s, v0.s[0]\n" - "ldr q7, [x17, #0x10]\n" - "fmla v12.4s, v6.4s, v1.s[0]\n" - "fmla v16.4s, v6.4s, v2.s[0]\n" - "ldr q6, [x17, #0x20]\n" - "fmla v9.4s, v7.4s, v0.s[0]\n" - "fmla v13.4s, v7.4s, v1.s[0]\n" - "fmla v17.4s, v7.4s, v2.s[0]\n" - "ldr q7, [x17, #0x30]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "add x17, x17, #0x40\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" + "cbz x14, 88f\n" + "87:" // Height 3: Multiply loop: Odd block loop + "ldr s24, [x13], #0x4\n" + "sub x14, x14, #0x1\n" + "ldr s23, [x12], #0x4\n" + "ldr s22, [x11], #0x4\n" + "ldr q21, [x17, #0x0]\n" + "fmla v8.4s, v21.4s, v24.s[0]\n" + "ldr q20, [x17, #0x10]\n" + "fmla v12.4s, v21.4s, v23.s[0]\n" + "fmla v16.4s, v21.4s, v22.s[0]\n" + "ldr q21, [x17, #0x20]\n" + "fmla v9.4s, v20.4s, v24.s[0]\n" + "fmla v13.4s, v20.4s, v23.s[0]\n" + "fmla v17.4s, v20.4s, v22.s[0]\n" + "ldr q20, [x17, #0x30]\n" + "fmla v10.4s, v21.4s, v24.s[0]\n" + "add x17, x17, #0x40\n" + "fmla v14.4s, v21.4s, v23.s[0]\n" + "fmla v18.4s, v21.4s, v22.s[0]\n" + "fmla v11.4s, v20.4s, v24.s[0]\n" + "fmla v15.4s, v20.4s, v23.s[0]\n" + "fmla v19.4s, v20.4s, v22.s[0]\n" "cbnz x14, 87b\n" "88:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -1227,33 +1226,33 @@ void a64_hybrid_fp32_mla_6x16_a55 ( "prfm pstl1keep, [x24, #0x0]\n" "tbz %x[flags], #1, 89f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v0.4s\n" - "fmin v9.4s, v9.4s, v0.4s\n" - "fmin v10.4s, v10.4s, v0.4s\n" - "fmin v11.4s, v11.4s, v0.4s\n" - "fmin v12.4s, v12.4s, v0.4s\n" - "fmin v13.4s, v13.4s, v0.4s\n" - "fmin v14.4s, v14.4s, v0.4s\n" - "fmin v15.4s, v15.4s, v0.4s\n" - "fmin v16.4s, v16.4s, v0.4s\n" - "fmin v17.4s, v17.4s, v0.4s\n" - "fmin v18.4s, v18.4s, v0.4s\n" - "fmin v19.4s, v19.4s, v0.4s\n" + "ld1r { v20.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v20.4s\n" + "fmin v9.4s, v9.4s, v20.4s\n" + "fmin v10.4s, v10.4s, v20.4s\n" + "fmin v11.4s, v11.4s, v20.4s\n" + "fmin v12.4s, v12.4s, v20.4s\n" + "fmin v13.4s, v13.4s, v20.4s\n" + "fmin v14.4s, v14.4s, v20.4s\n" + "fmin v15.4s, v15.4s, v20.4s\n" + "fmin v16.4s, v16.4s, v20.4s\n" + "fmin v17.4s, v17.4s, v20.4s\n" + "fmin v18.4s, v18.4s, v20.4s\n" + "fmin v19.4s, v19.4s, v20.4s\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" - "fmax v16.4s, v16.4s, v0.4s\n" - "fmax v17.4s, v17.4s, v0.4s\n" - "fmax v18.4s, v18.4s, v0.4s\n" - "fmax v19.4s, v19.4s, v0.4s\n" + "ld1r { v20.4s }, [x20]\n" + "fmax v8.4s, v8.4s, v20.4s\n" + "fmax v9.4s, v9.4s, v20.4s\n" + "fmax v10.4s, v10.4s, v20.4s\n" + "fmax v11.4s, v11.4s, v20.4s\n" + "fmax v12.4s, v12.4s, v20.4s\n" + "fmax v13.4s, v13.4s, v20.4s\n" + "fmax v14.4s, v14.4s, v20.4s\n" + "fmax v15.4s, v15.4s, v20.4s\n" + "fmax v16.4s, v16.4s, v20.4s\n" + "fmax v17.4s, v17.4s, v20.4s\n" + "fmax v18.4s, v18.4s, v20.4s\n" + "fmax v19.4s, v19.4s, v20.4s\n" "89:" // Height 3: No activation "cmp x8, #0x10\n" "bge 98f\n" @@ -1529,292 +1528,292 @@ void a64_hybrid_fp32_mla_6x16_a55 ( "114:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w14, [x20, x15, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 115f\n" - "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x13, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" - "ldr x27, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x15, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x13, [x20, #0x0]\n" + "ldr x12, [x20, #0x8]\n" + "ldr x11, [x20, #0x10]\n" + "ldr x10, [x20, #0x18]\n" "cbnz x15, 116f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x13, x13, x20, LSL #2\n" - "add x9, x9, x20, LSL #2\n" - "add x27, x27, x20, LSL #2\n" - "add x25, x25, x20, LSL #2\n" + "add x12, x12, x20, LSL #2\n" + "add x11, x11, x20, LSL #2\n" + "add x10, x10, x20, LSL #2\n" "b 116f\n" "115:" // Height 4: setup direct input "mov x13, %x[input_ptr]\n" - "add x9, x13, x20, LSL #2\n" - "add x27, x9, x20, LSL #2\n" - "add x25, x27, x20, LSL #2\n" + "add x12, x13, x21, LSL #2\n" + "add x11, x12, x21, LSL #2\n" + "add x10, x11, x21, LSL #2\n" "116:" // Height 4: input setup done "cmp x14, #0x4\n" "blt 119f\n" "ldr q0, [x13, #0x0]\n" "cmp x14, #0x8\n" - "ldr q1, [x9, #0x0]\n" - "ldr q2, [x27, #0x0]\n" - "ldr q3, [x25, #0x0]\n" + "ldr q1, [x12, #0x0]\n" + "ldr q2, [x11, #0x0]\n" + "ldr q3, [x10, #0x0]\n" "ldr q6, [x17, #0x0]\n" "ldr q7, [x17, #0x10]\n" "blt 118f\n" "117:" // Height 4: Multiply loop: Main loop head "fmla v8.4s, v6.4s, v0.s[0]\n" - "ldr x12, [x17, #0x28]\n" + "ldr x21, [x17, #0x28]\n" "fmla v12.4s, v6.4s, v1.s[0]\n" - "ldr x11, [x17, #0x38]\n" + "ldr x20, [x17, #0x38]\n" "fmla v16.4s, v6.4s, v2.s[0]\n" "add x13, x13, #0x10\n" "fmla v20.4s, v6.4s, v3.s[0]\n" - "ldr d6, [x17, #0x20]\n" + "ldr d25, [x17, #0x20]\n" "fmla v9.4s, v7.4s, v0.s[0]\n" - "mov v6.d[1], x12\n" + "mov v25.d[1], x21\n" "fmla v13.4s, v7.4s, v1.s[0]\n" - "ldr x12, [x17, #0x48]\n" + "ldr x21, [x17, #0x48]\n" "fmla v17.4s, v7.4s, v2.s[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" "fmla v21.4s, v7.4s, v3.s[0]\n" - "ldr d7, [x17, #0x30]\n" - "mov v7.d[1], x11\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "ldr x11, [x17, #0x58]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "add x27, x27, #0x10\n" - "fmla v22.4s, v6.4s, v3.s[0]\n" - "ldr d6, [x17, #0x40]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "mov v6.d[1], x12\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "ldr x12, [x17, #0x68]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "add x25, x25, #0x10\n" - "fmla v23.4s, v7.4s, v3.s[0]\n" - "ldr d7, [x17, #0x50]\n" - "mov v7.d[1], x11\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v12.4s, v6.4s, v1.s[1]\n" - "ldr x11, [x17, #0x78]\n" - "fmla v16.4s, v6.4s, v2.s[1]\n" - "ldr x10, [x13, #0x8]\n" - "fmla v20.4s, v6.4s, v3.s[1]\n" - "ldr d6, [x17, #0x60]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "mov v6.d[1], x12\n" - "fmla v13.4s, v7.4s, v1.s[1]\n" - "ldr x12, [x17, #0x88]\n" - "fmla v17.4s, v7.4s, v2.s[1]\n" - "ldr x28, [x9, #0x8]\n" - "fmla v21.4s, v7.4s, v3.s[1]\n" - "ldr d7, [x17, #0x70]\n" - "mov v7.d[1], x11\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "ldr x11, [x17, #0x98]\n" - "fmla v18.4s, v6.4s, v2.s[1]\n" - "ldr x26, [x27, #0x8]\n" - "fmla v22.4s, v6.4s, v3.s[1]\n" - "ldr d6, [x17, #0x80]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "mov v6.d[1], x12\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "ldr x12, [x17, #0xa8]\n" - "fmla v19.4s, v7.4s, v2.s[1]\n" - "ldr x24, [x25, #0x8]\n" - "fmla v23.4s, v7.4s, v3.s[1]\n" - "ldr d7, [x17, #0x90]\n" - "mov v7.d[1], x11\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "fmla v12.4s, v6.4s, v1.s[2]\n" - "ldr x11, [x17, #0xb8]\n" - "fmla v16.4s, v6.4s, v2.s[2]\n" + "ldr d24, [x17, #0x30]\n" + "mov v24.d[1], x20\n" + "fmla v10.4s, v25.4s, v0.s[0]\n" + "fmla v14.4s, v25.4s, v1.s[0]\n" + "ldr x20, [x17, #0x58]\n" + "fmla v18.4s, v25.4s, v2.s[0]\n" + "add x11, x11, #0x10\n" + "fmla v22.4s, v25.4s, v3.s[0]\n" + "ldr d25, [x17, #0x40]\n" + "fmla v11.4s, v24.4s, v0.s[0]\n" + "mov v25.d[1], x21\n" + "fmla v15.4s, v24.4s, v1.s[0]\n" + "ldr x21, [x17, #0x68]\n" + "fmla v19.4s, v24.4s, v2.s[0]\n" + "add x10, x10, #0x10\n" + "fmla v23.4s, v24.4s, v3.s[0]\n" + "ldr d24, [x17, #0x50]\n" + "mov v24.d[1], x20\n" + "fmla v8.4s, v25.4s, v0.s[1]\n" + "fmla v12.4s, v25.4s, v1.s[1]\n" + "ldr x20, [x17, #0x78]\n" + "fmla v16.4s, v25.4s, v2.s[1]\n" + "ldr x25, [x13, #0x8]\n" + "fmla v20.4s, v25.4s, v3.s[1]\n" + "ldr d25, [x17, #0x60]\n" + "fmla v9.4s, v24.4s, v0.s[1]\n" + "mov v25.d[1], x21\n" + "fmla v13.4s, v24.4s, v1.s[1]\n" + "ldr x21, [x17, #0x88]\n" + "fmla v17.4s, v24.4s, v2.s[1]\n" + "ldr x24, [x12, #0x8]\n" + "fmla v21.4s, v24.4s, v3.s[1]\n" + "ldr d24, [x17, #0x70]\n" + "mov v24.d[1], x20\n" + "fmla v10.4s, v25.4s, v0.s[1]\n" + "fmla v14.4s, v25.4s, v1.s[1]\n" + "ldr x20, [x17, #0x98]\n" + "fmla v18.4s, v25.4s, v2.s[1]\n" + "ldr x23, [x11, #0x8]\n" + "fmla v22.4s, v25.4s, v3.s[1]\n" + "ldr d25, [x17, #0x80]\n" + "fmla v11.4s, v24.4s, v0.s[1]\n" + "mov v25.d[1], x21\n" + "fmla v15.4s, v24.4s, v1.s[1]\n" + "ldr x21, [x17, #0xa8]\n" + "fmla v19.4s, v24.4s, v2.s[1]\n" + "ldr x22, [x10, #0x8]\n" + "fmla v23.4s, v24.4s, v3.s[1]\n" + "ldr d24, [x17, #0x90]\n" + "mov v24.d[1], x20\n" + "fmla v8.4s, v25.4s, v0.s[2]\n" + "fmla v12.4s, v25.4s, v1.s[2]\n" + "ldr x20, [x17, #0xb8]\n" + "fmla v16.4s, v25.4s, v2.s[2]\n" "sub x14, x14, #0x4\n" - "fmla v20.4s, v6.4s, v3.s[2]\n" - "ldr d6, [x17, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "mov v6.d[1], x12\n" - "fmla v13.4s, v7.4s, v1.s[2]\n" - "ldr x12, [x17, #0xc8]\n" - "fmla v17.4s, v7.4s, v2.s[2]\n" + "fmla v20.4s, v25.4s, v3.s[2]\n" + "ldr d25, [x17, #0xa0]\n" + "fmla v9.4s, v24.4s, v0.s[2]\n" + "mov v25.d[1], x21\n" + "fmla v13.4s, v24.4s, v1.s[2]\n" + "ldr x21, [x17, #0xc8]\n" + "fmla v17.4s, v24.4s, v2.s[2]\n" "cmp x14, #0x8\n" - "fmla v21.4s, v7.4s, v3.s[2]\n" - "ldr d7, [x17, #0xb0]\n" - "mov v7.d[1], x11\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v14.4s, v6.4s, v1.s[2]\n" - "ldr x11, [x17, #0xd8]\n" - "fmla v18.4s, v6.4s, v2.s[2]\n" + "fmla v21.4s, v24.4s, v3.s[2]\n" + "ldr d24, [x17, #0xb0]\n" + "mov v24.d[1], x20\n" + "fmla v10.4s, v25.4s, v0.s[2]\n" + "fmla v14.4s, v25.4s, v1.s[2]\n" + "ldr x20, [x17, #0xd8]\n" + "fmla v18.4s, v25.4s, v2.s[2]\n" "prfm pldl1keep, [x13, #0x80]\n" - "fmla v22.4s, v6.4s, v3.s[2]\n" - "ldr d6, [x17, #0xc0]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "mov v6.d[1], x12\n" - "fmla v15.4s, v7.4s, v1.s[2]\n" - "ldr x12, [x17, #0xe8]\n" - "fmla v19.4s, v7.4s, v2.s[2]\n" - "prfm pldl1keep, [x9, #0x80]\n" - "fmla v23.4s, v7.4s, v3.s[2]\n" - "ldr d7, [x17, #0xd0]\n" - "mov v7.d[1], x11\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v12.4s, v6.4s, v1.s[3]\n" - "ldr x11, [x17, #0xf8]\n" - "fmla v16.4s, v6.4s, v2.s[3]\n" - "prfm pldl1keep, [x27, #0x80]\n" - "fmla v20.4s, v6.4s, v3.s[3]\n" - "ldr d6, [x17, #0xe0]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "mov v6.d[1], x12\n" - "fmla v13.4s, v7.4s, v1.s[3]\n" - "prfm pldl1keep, [x25, #0x80]\n" - "fmla v17.4s, v7.4s, v2.s[3]\n" - "fmla v21.4s, v7.4s, v3.s[3]\n" - "ldr d7, [x17, #0xf0]\n" - "mov v7.d[1], x11\n" + "fmla v22.4s, v25.4s, v3.s[2]\n" + "ldr d25, [x17, #0xc0]\n" + "fmla v11.4s, v24.4s, v0.s[2]\n" + "mov v25.d[1], x21\n" + "fmla v15.4s, v24.4s, v1.s[2]\n" + "ldr x21, [x17, #0xe8]\n" + "fmla v19.4s, v24.4s, v2.s[2]\n" + "prfm pldl1keep, [x12, #0x80]\n" + "fmla v23.4s, v24.4s, v3.s[2]\n" + "ldr d24, [x17, #0xd0]\n" + "mov v24.d[1], x20\n" + "fmla v8.4s, v25.4s, v0.s[3]\n" + "fmla v12.4s, v25.4s, v1.s[3]\n" + "ldr x20, [x17, #0xf8]\n" + "fmla v16.4s, v25.4s, v2.s[3]\n" + "prfm pldl1keep, [x11, #0x80]\n" + "fmla v20.4s, v25.4s, v3.s[3]\n" + "ldr d25, [x17, #0xe0]\n" + "fmla v9.4s, v24.4s, v0.s[3]\n" + "mov v25.d[1], x21\n" + "fmla v13.4s, v24.4s, v1.s[3]\n" + "prfm pldl1keep, [x10, #0x80]\n" + "fmla v17.4s, v24.4s, v2.s[3]\n" + "fmla v21.4s, v24.4s, v3.s[3]\n" + "ldr d24, [x17, #0xf0]\n" + "mov v24.d[1], x20\n" "add x17, x17, #0x100\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" - "ldr x12, [x17, #0x8]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "ldr x11, [x17, #0x18]\n" - "fmla v18.4s, v6.4s, v2.s[3]\n" - "fmla v22.4s, v6.4s, v3.s[3]\n" + "fmla v10.4s, v25.4s, v0.s[3]\n" + "ldr x21, [x17, #0x8]\n" + "fmla v14.4s, v25.4s, v1.s[3]\n" + "ldr x20, [x17, #0x18]\n" + "fmla v18.4s, v25.4s, v2.s[3]\n" + "fmla v22.4s, v25.4s, v3.s[3]\n" "ldr d6, [x17, #0x0]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" + "fmla v11.4s, v24.4s, v0.s[3]\n" "ldr d0, [x13, #0x0]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" - "ldr d1, [x9, #0x0]\n" - "fmla v19.4s, v7.4s, v2.s[3]\n" - "ldr d2, [x27, #0x0]\n" - "fmla v23.4s, v7.4s, v3.s[3]\n" - "ldr d3, [x25, #0x0]\n" + "fmla v15.4s, v24.4s, v1.s[3]\n" + "ldr d1, [x12, #0x0]\n" + "fmla v19.4s, v24.4s, v2.s[3]\n" + "ldr d2, [x11, #0x0]\n" + "fmla v23.4s, v24.4s, v3.s[3]\n" + "ldr d3, [x10, #0x0]\n" "ldr d7, [x17, #0x10]\n" - "mov v6.d[1], x12\n" - "mov v0.d[1], x10\n" - "mov v1.d[1], x28\n" - "mov v2.d[1], x26\n" - "mov v3.d[1], x24\n" - "mov v7.d[1], x11\n" + "mov v6.d[1], x21\n" + "mov v0.d[1], x25\n" + "mov v1.d[1], x24\n" + "mov v2.d[1], x23\n" + "mov v3.d[1], x22\n" + "mov v7.d[1], x20\n" "bge 117b\n" "118:" // Height 4: Multiply loop: Single iteration only "fmla v8.4s, v6.4s, v0.s[0]\n" "add x13, x13, #0x10\n" "fmla v12.4s, v6.4s, v1.s[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" "fmla v16.4s, v6.4s, v2.s[0]\n" - "add x27, x27, #0x10\n" + "add x11, x11, #0x10\n" "fmla v20.4s, v6.4s, v3.s[0]\n" - "ldr q6, [x17, #0x20]\n" + "ldr q25, [x17, #0x20]\n" "fmla v9.4s, v7.4s, v0.s[0]\n" - "add x25, x25, #0x10\n" + "add x10, x10, #0x10\n" "fmla v13.4s, v7.4s, v1.s[0]\n" "sub x14, x14, #0x4\n" "fmla v17.4s, v7.4s, v2.s[0]\n" "prfm pldl1keep, [x13, #0x80]\n" "fmla v21.4s, v7.4s, v3.s[0]\n" - "ldr q7, [x17, #0x30]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "prfm pldl1keep, [x9, #0x80]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "prfm pldl1keep, [x27, #0x80]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "prfm pldl1keep, [x25, #0x80]\n" - "fmla v22.4s, v6.4s, v3.s[0]\n" - "ldr q6, [x17, #0x40]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "fmla v23.4s, v7.4s, v3.s[0]\n" - "ldr q7, [x17, #0x50]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v12.4s, v6.4s, v1.s[1]\n" - "fmla v16.4s, v6.4s, v2.s[1]\n" - "fmla v20.4s, v6.4s, v3.s[1]\n" - "ldr q6, [x17, #0x60]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v13.4s, v7.4s, v1.s[1]\n" - "fmla v17.4s, v7.4s, v2.s[1]\n" - "fmla v21.4s, v7.4s, v3.s[1]\n" - "ldr q7, [x17, #0x70]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "fmla v18.4s, v6.4s, v2.s[1]\n" - "fmla v22.4s, v6.4s, v3.s[1]\n" - "ldr q6, [x17, #0x80]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "fmla v19.4s, v7.4s, v2.s[1]\n" - "fmla v23.4s, v7.4s, v3.s[1]\n" - "ldr q7, [x17, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "fmla v12.4s, v6.4s, v1.s[2]\n" - "fmla v16.4s, v6.4s, v2.s[2]\n" - "fmla v20.4s, v6.4s, v3.s[2]\n" - "ldr q6, [x17, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "fmla v13.4s, v7.4s, v1.s[2]\n" - "fmla v17.4s, v7.4s, v2.s[2]\n" - "fmla v21.4s, v7.4s, v3.s[2]\n" - "ldr q7, [x17, #0xb0]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v14.4s, v6.4s, v1.s[2]\n" - "fmla v18.4s, v6.4s, v2.s[2]\n" - "fmla v22.4s, v6.4s, v3.s[2]\n" - "ldr q6, [x17, #0xc0]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "fmla v15.4s, v7.4s, v1.s[2]\n" - "fmla v19.4s, v7.4s, v2.s[2]\n" - "fmla v23.4s, v7.4s, v3.s[2]\n" - "ldr q7, [x17, #0xd0]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v12.4s, v6.4s, v1.s[3]\n" - "fmla v16.4s, v6.4s, v2.s[3]\n" - "fmla v20.4s, v6.4s, v3.s[3]\n" - "ldr q6, [x17, #0xe0]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "fmla v13.4s, v7.4s, v1.s[3]\n" - "fmla v17.4s, v7.4s, v2.s[3]\n" - "fmla v21.4s, v7.4s, v3.s[3]\n" - "ldr q7, [x17, #0xf0]\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" + "ldr q24, [x17, #0x30]\n" + "fmla v10.4s, v25.4s, v0.s[0]\n" + "prfm pldl1keep, [x12, #0x80]\n" + "fmla v14.4s, v25.4s, v1.s[0]\n" + "prfm pldl1keep, [x11, #0x80]\n" + "fmla v18.4s, v25.4s, v2.s[0]\n" + "prfm pldl1keep, [x10, #0x80]\n" + "fmla v22.4s, v25.4s, v3.s[0]\n" + "ldr q25, [x17, #0x40]\n" + "fmla v11.4s, v24.4s, v0.s[0]\n" + "fmla v15.4s, v24.4s, v1.s[0]\n" + "fmla v19.4s, v24.4s, v2.s[0]\n" + "fmla v23.4s, v24.4s, v3.s[0]\n" + "ldr q24, [x17, #0x50]\n" + "fmla v8.4s, v25.4s, v0.s[1]\n" + "fmla v12.4s, v25.4s, v1.s[1]\n" + "fmla v16.4s, v25.4s, v2.s[1]\n" + "fmla v20.4s, v25.4s, v3.s[1]\n" + "ldr q25, [x17, #0x60]\n" + "fmla v9.4s, v24.4s, v0.s[1]\n" + "fmla v13.4s, v24.4s, v1.s[1]\n" + "fmla v17.4s, v24.4s, v2.s[1]\n" + "fmla v21.4s, v24.4s, v3.s[1]\n" + "ldr q24, [x17, #0x70]\n" + "fmla v10.4s, v25.4s, v0.s[1]\n" + "fmla v14.4s, v25.4s, v1.s[1]\n" + "fmla v18.4s, v25.4s, v2.s[1]\n" + "fmla v22.4s, v25.4s, v3.s[1]\n" + "ldr q25, [x17, #0x80]\n" + "fmla v11.4s, v24.4s, v0.s[1]\n" + "fmla v15.4s, v24.4s, v1.s[1]\n" + "fmla v19.4s, v24.4s, v2.s[1]\n" + "fmla v23.4s, v24.4s, v3.s[1]\n" + "ldr q24, [x17, #0x90]\n" + "fmla v8.4s, v25.4s, v0.s[2]\n" + "fmla v12.4s, v25.4s, v1.s[2]\n" + "fmla v16.4s, v25.4s, v2.s[2]\n" + "fmla v20.4s, v25.4s, v3.s[2]\n" + "ldr q25, [x17, #0xa0]\n" + "fmla v9.4s, v24.4s, v0.s[2]\n" + "fmla v13.4s, v24.4s, v1.s[2]\n" + "fmla v17.4s, v24.4s, v2.s[2]\n" + "fmla v21.4s, v24.4s, v3.s[2]\n" + "ldr q24, [x17, #0xb0]\n" + "fmla v10.4s, v25.4s, v0.s[2]\n" + "fmla v14.4s, v25.4s, v1.s[2]\n" + "fmla v18.4s, v25.4s, v2.s[2]\n" + "fmla v22.4s, v25.4s, v3.s[2]\n" + "ldr q25, [x17, #0xc0]\n" + "fmla v11.4s, v24.4s, v0.s[2]\n" + "fmla v15.4s, v24.4s, v1.s[2]\n" + "fmla v19.4s, v24.4s, v2.s[2]\n" + "fmla v23.4s, v24.4s, v3.s[2]\n" + "ldr q24, [x17, #0xd0]\n" + "fmla v8.4s, v25.4s, v0.s[3]\n" + "fmla v12.4s, v25.4s, v1.s[3]\n" + "fmla v16.4s, v25.4s, v2.s[3]\n" + "fmla v20.4s, v25.4s, v3.s[3]\n" + "ldr q25, [x17, #0xe0]\n" + "fmla v9.4s, v24.4s, v0.s[3]\n" + "fmla v13.4s, v24.4s, v1.s[3]\n" + "fmla v17.4s, v24.4s, v2.s[3]\n" + "fmla v21.4s, v24.4s, v3.s[3]\n" + "ldr q24, [x17, #0xf0]\n" + "fmla v10.4s, v25.4s, v0.s[3]\n" "add x17, x17, #0x100\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "fmla v18.4s, v6.4s, v2.s[3]\n" - "fmla v22.4s, v6.4s, v3.s[3]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" - "fmla v19.4s, v7.4s, v2.s[3]\n" - "fmla v23.4s, v7.4s, v3.s[3]\n" + "fmla v14.4s, v25.4s, v1.s[3]\n" + "fmla v18.4s, v25.4s, v2.s[3]\n" + "fmla v22.4s, v25.4s, v3.s[3]\n" + "fmla v11.4s, v24.4s, v0.s[3]\n" + "fmla v15.4s, v24.4s, v1.s[3]\n" + "fmla v19.4s, v24.4s, v2.s[3]\n" + "fmla v23.4s, v24.4s, v3.s[3]\n" "119:" // Height 4: Multiply loop: Main loop skip "cbz x14, 121f\n" "120:" // Height 4: Multiply loop: Odd block loop - "ldr s0, [x13], #0x4\n" + "ldr s29, [x13], #0x4\n" "sub x14, x14, #0x1\n" - "ldr s1, [x9], #0x4\n" - "ldr s2, [x27], #0x4\n" - "ldr s3, [x25], #0x4\n" - "ldr q6, [x17, #0x0]\n" - "fmla v8.4s, v6.4s, v0.s[0]\n" - "ldr q7, [x17, #0x10]\n" - "fmla v12.4s, v6.4s, v1.s[0]\n" - "fmla v16.4s, v6.4s, v2.s[0]\n" - "fmla v20.4s, v6.4s, v3.s[0]\n" - "ldr q6, [x17, #0x20]\n" - "fmla v9.4s, v7.4s, v0.s[0]\n" - "fmla v13.4s, v7.4s, v1.s[0]\n" - "fmla v17.4s, v7.4s, v2.s[0]\n" - "fmla v21.4s, v7.4s, v3.s[0]\n" - "ldr q7, [x17, #0x30]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" + "ldr s28, [x12], #0x4\n" + "ldr s27, [x11], #0x4\n" + "ldr s26, [x10], #0x4\n" + "ldr q25, [x17, #0x0]\n" + "fmla v8.4s, v25.4s, v29.s[0]\n" + "ldr q24, [x17, #0x10]\n" + "fmla v12.4s, v25.4s, v28.s[0]\n" + "fmla v16.4s, v25.4s, v27.s[0]\n" + "fmla v20.4s, v25.4s, v26.s[0]\n" + "ldr q25, [x17, #0x20]\n" + "fmla v9.4s, v24.4s, v29.s[0]\n" + "fmla v13.4s, v24.4s, v28.s[0]\n" + "fmla v17.4s, v24.4s, v27.s[0]\n" + "fmla v21.4s, v24.4s, v26.s[0]\n" + "ldr q24, [x17, #0x30]\n" + "fmla v10.4s, v25.4s, v29.s[0]\n" "add x17, x17, #0x40\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "fmla v22.4s, v6.4s, v3.s[0]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "fmla v23.4s, v7.4s, v3.s[0]\n" + "fmla v14.4s, v25.4s, v28.s[0]\n" + "fmla v18.4s, v25.4s, v27.s[0]\n" + "fmla v22.4s, v25.4s, v26.s[0]\n" + "fmla v11.4s, v24.4s, v29.s[0]\n" + "fmla v15.4s, v24.4s, v28.s[0]\n" + "fmla v19.4s, v24.4s, v27.s[0]\n" + "fmla v23.4s, v24.4s, v26.s[0]\n" "cbnz x14, 120b\n" "121:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -1831,41 +1830,41 @@ void a64_hybrid_fp32_mla_6x16_a55 ( "prfm pstl1keep, [x23, #0x0]\n" "tbz %x[flags], #1, 122f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v0.4s\n" - "fmin v9.4s, v9.4s, v0.4s\n" - "fmin v10.4s, v10.4s, v0.4s\n" - "fmin v11.4s, v11.4s, v0.4s\n" - "fmin v12.4s, v12.4s, v0.4s\n" - "fmin v13.4s, v13.4s, v0.4s\n" - "fmin v14.4s, v14.4s, v0.4s\n" - "fmin v15.4s, v15.4s, v0.4s\n" - "fmin v16.4s, v16.4s, v0.4s\n" - "fmin v17.4s, v17.4s, v0.4s\n" - "fmin v18.4s, v18.4s, v0.4s\n" - "fmin v19.4s, v19.4s, v0.4s\n" - "fmin v20.4s, v20.4s, v0.4s\n" - "fmin v21.4s, v21.4s, v0.4s\n" - "fmin v22.4s, v22.4s, v0.4s\n" - "fmin v23.4s, v23.4s, v0.4s\n" + "ld1r { v24.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v24.4s\n" + "fmin v9.4s, v9.4s, v24.4s\n" + "fmin v10.4s, v10.4s, v24.4s\n" + "fmin v11.4s, v11.4s, v24.4s\n" + "fmin v12.4s, v12.4s, v24.4s\n" + "fmin v13.4s, v13.4s, v24.4s\n" + "fmin v14.4s, v14.4s, v24.4s\n" + "fmin v15.4s, v15.4s, v24.4s\n" + "fmin v16.4s, v16.4s, v24.4s\n" + "fmin v17.4s, v17.4s, v24.4s\n" + "fmin v18.4s, v18.4s, v24.4s\n" + "fmin v19.4s, v19.4s, v24.4s\n" + "fmin v20.4s, v20.4s, v24.4s\n" + "fmin v21.4s, v21.4s, v24.4s\n" + "fmin v22.4s, v22.4s, v24.4s\n" + "fmin v23.4s, v23.4s, v24.4s\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" - "fmax v16.4s, v16.4s, v0.4s\n" - "fmax v17.4s, v17.4s, v0.4s\n" - "fmax v18.4s, v18.4s, v0.4s\n" - "fmax v19.4s, v19.4s, v0.4s\n" - "fmax v20.4s, v20.4s, v0.4s\n" - "fmax v21.4s, v21.4s, v0.4s\n" - "fmax v22.4s, v22.4s, v0.4s\n" - "fmax v23.4s, v23.4s, v0.4s\n" + "ld1r { v24.4s }, [x20]\n" + "fmax v8.4s, v8.4s, v24.4s\n" + "fmax v9.4s, v9.4s, v24.4s\n" + "fmax v10.4s, v10.4s, v24.4s\n" + "fmax v11.4s, v11.4s, v24.4s\n" + "fmax v12.4s, v12.4s, v24.4s\n" + "fmax v13.4s, v13.4s, v24.4s\n" + "fmax v14.4s, v14.4s, v24.4s\n" + "fmax v15.4s, v15.4s, v24.4s\n" + "fmax v16.4s, v16.4s, v24.4s\n" + "fmax v17.4s, v17.4s, v24.4s\n" + "fmax v18.4s, v18.4s, v24.4s\n" + "fmax v19.4s, v19.4s, v24.4s\n" + "fmax v20.4s, v20.4s, v24.4s\n" + "fmax v21.4s, v21.4s, v24.4s\n" + "fmax v22.4s, v22.4s, v24.4s\n" + "fmax v23.4s, v23.4s, v24.4s\n" "122:" // Height 4: No activation "cmp x8, #0x10\n" "bge 131f\n" @@ -2190,340 +2189,340 @@ void a64_hybrid_fp32_mla_6x16_a55 ( "147:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w14, [x20, x15, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 148f\n" - "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x13, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" - "ldr x27, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" - "ldr x23, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x15, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x13, [x20, #0x0]\n" + "ldr x12, [x20, #0x8]\n" + "ldr x11, [x20, #0x10]\n" + "ldr x10, [x20, #0x18]\n" + "ldr x9, [x20, #0x20]\n" "cbnz x15, 149f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x13, x13, x20, LSL #2\n" + "add x12, x12, x20, LSL #2\n" + "add x11, x11, x20, LSL #2\n" + "add x10, x10, x20, LSL #2\n" "add x9, x9, x20, LSL #2\n" - "add x27, x27, x20, LSL #2\n" - "add x25, x25, x20, LSL #2\n" - "add x23, x23, x20, LSL #2\n" "b 149f\n" "148:" // Height 5: setup direct input "mov x13, %x[input_ptr]\n" - "add x9, x13, x20, LSL #2\n" - "add x27, x9, x20, LSL #2\n" - "add x25, x27, x20, LSL #2\n" - "add x23, x25, x20, LSL #2\n" + "add x12, x13, x21, LSL #2\n" + "add x11, x12, x21, LSL #2\n" + "add x10, x11, x21, LSL #2\n" + "add x9, x10, x21, LSL #2\n" "149:" // Height 5: input setup done "cmp x14, #0x4\n" "blt 152f\n" "ldr q0, [x13, #0x0]\n" "cmp x14, #0x8\n" - "ldr q1, [x9, #0x0]\n" - "ldr q2, [x27, #0x0]\n" - "ldr q3, [x25, #0x0]\n" - "ldr q4, [x23, #0x0]\n" + "ldr q1, [x12, #0x0]\n" + "ldr q2, [x11, #0x0]\n" + "ldr q3, [x10, #0x0]\n" + "ldr q4, [x9, #0x0]\n" "ldr q6, [x17, #0x0]\n" "ldr q7, [x17, #0x10]\n" "blt 151f\n" "150:" // Height 5: Multiply loop: Main loop head "fmla v8.4s, v6.4s, v0.s[0]\n" - "ldr x12, [x17, #0x28]\n" + "ldr x21, [x17, #0x28]\n" "fmla v12.4s, v6.4s, v1.s[0]\n" - "ldr x11, [x17, #0x38]\n" + "ldr x20, [x17, #0x38]\n" "fmla v16.4s, v6.4s, v2.s[0]\n" "add x13, x13, #0x10\n" "fmla v20.4s, v6.4s, v3.s[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" "fmla v24.4s, v6.4s, v4.s[0]\n" - "ldr d6, [x17, #0x20]\n" + "ldr d29, [x17, #0x20]\n" "fmla v9.4s, v7.4s, v0.s[0]\n" - "mov v6.d[1], x12\n" + "mov v29.d[1], x21\n" "fmla v13.4s, v7.4s, v1.s[0]\n" - "ldr x12, [x17, #0x48]\n" + "ldr x21, [x17, #0x48]\n" "fmla v17.4s, v7.4s, v2.s[0]\n" - "add x27, x27, #0x10\n" + "add x11, x11, #0x10\n" "fmla v21.4s, v7.4s, v3.s[0]\n" - "add x25, x25, #0x10\n" + "add x10, x10, #0x10\n" "fmla v25.4s, v7.4s, v4.s[0]\n" - "ldr d7, [x17, #0x30]\n" - "mov v7.d[1], x11\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "ldr x11, [x17, #0x58]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "add x23, x23, #0x10\n" - "fmla v22.4s, v6.4s, v3.s[0]\n" - "ldr x10, [x13, #0x8]\n" - "fmla v26.4s, v6.4s, v4.s[0]\n" - "ldr d6, [x17, #0x40]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "mov v6.d[1], x12\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "ldr x12, [x17, #0x68]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "ldr x28, [x9, #0x8]\n" - "fmla v23.4s, v7.4s, v3.s[0]\n" - "ldr x26, [x27, #0x8]\n" - "fmla v27.4s, v7.4s, v4.s[0]\n" - "ldr d7, [x17, #0x50]\n" - "mov v7.d[1], x11\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v12.4s, v6.4s, v1.s[1]\n" - "ldr x11, [x17, #0x78]\n" - "fmla v16.4s, v6.4s, v2.s[1]\n" - "ldr x24, [x25, #0x8]\n" - "fmla v20.4s, v6.4s, v3.s[1]\n" - "ldr x22, [x23, #0x8]\n" - "fmla v24.4s, v6.4s, v4.s[1]\n" - "ldr d6, [x17, #0x60]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "mov v6.d[1], x12\n" - "fmla v13.4s, v7.4s, v1.s[1]\n" - "ldr x12, [x17, #0x88]\n" - "fmla v17.4s, v7.4s, v2.s[1]\n" + "ldr d28, [x17, #0x30]\n" + "mov v28.d[1], x20\n" + "fmla v10.4s, v29.4s, v0.s[0]\n" + "fmla v14.4s, v29.4s, v1.s[0]\n" + "ldr x20, [x17, #0x58]\n" + "fmla v18.4s, v29.4s, v2.s[0]\n" + "add x9, x9, #0x10\n" + "fmla v22.4s, v29.4s, v3.s[0]\n" + "ldr x26, [x13, #0x8]\n" + "fmla v26.4s, v29.4s, v4.s[0]\n" + "ldr d29, [x17, #0x40]\n" + "fmla v11.4s, v28.4s, v0.s[0]\n" + "mov v29.d[1], x21\n" + "fmla v15.4s, v28.4s, v1.s[0]\n" + "ldr x21, [x17, #0x68]\n" + "fmla v19.4s, v28.4s, v2.s[0]\n" + "ldr x25, [x12, #0x8]\n" + "fmla v23.4s, v28.4s, v3.s[0]\n" + "ldr x24, [x11, #0x8]\n" + "fmla v27.4s, v28.4s, v4.s[0]\n" + "ldr d28, [x17, #0x50]\n" + "mov v28.d[1], x20\n" + "fmla v8.4s, v29.4s, v0.s[1]\n" + "fmla v12.4s, v29.4s, v1.s[1]\n" + "ldr x20, [x17, #0x78]\n" + "fmla v16.4s, v29.4s, v2.s[1]\n" + "ldr x23, [x10, #0x8]\n" + "fmla v20.4s, v29.4s, v3.s[1]\n" + "ldr x22, [x9, #0x8]\n" + "fmla v24.4s, v29.4s, v4.s[1]\n" + "ldr d29, [x17, #0x60]\n" + "fmla v9.4s, v28.4s, v0.s[1]\n" + "mov v29.d[1], x21\n" + "fmla v13.4s, v28.4s, v1.s[1]\n" + "ldr x21, [x17, #0x88]\n" + "fmla v17.4s, v28.4s, v2.s[1]\n" "sub x14, x14, #0x4\n" - "fmla v21.4s, v7.4s, v3.s[1]\n" + "fmla v21.4s, v28.4s, v3.s[1]\n" "cmp x14, #0x8\n" - "fmla v25.4s, v7.4s, v4.s[1]\n" - "ldr d7, [x17, #0x70]\n" - "mov v7.d[1], x11\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "ldr x11, [x17, #0x98]\n" - "fmla v18.4s, v6.4s, v2.s[1]\n" + "fmla v25.4s, v28.4s, v4.s[1]\n" + "ldr d28, [x17, #0x70]\n" + "mov v28.d[1], x20\n" + "fmla v10.4s, v29.4s, v0.s[1]\n" + "fmla v14.4s, v29.4s, v1.s[1]\n" + "ldr x20, [x17, #0x98]\n" + "fmla v18.4s, v29.4s, v2.s[1]\n" "prfm pldl1keep, [x13, #0x80]\n" - "fmla v22.4s, v6.4s, v3.s[1]\n" + "fmla v22.4s, v29.4s, v3.s[1]\n" + "prfm pldl1keep, [x12, #0x80]\n" + "fmla v26.4s, v29.4s, v4.s[1]\n" + "ldr d29, [x17, #0x80]\n" + "fmla v11.4s, v28.4s, v0.s[1]\n" + "mov v29.d[1], x21\n" + "fmla v15.4s, v28.4s, v1.s[1]\n" + "ldr x21, [x17, #0xa8]\n" + "fmla v19.4s, v28.4s, v2.s[1]\n" + "prfm pldl1keep, [x11, #0x80]\n" + "fmla v23.4s, v28.4s, v3.s[1]\n" + "prfm pldl1keep, [x10, #0x80]\n" + "fmla v27.4s, v28.4s, v4.s[1]\n" + "ldr d28, [x17, #0x90]\n" + "mov v28.d[1], x20\n" + "fmla v8.4s, v29.4s, v0.s[2]\n" + "fmla v12.4s, v29.4s, v1.s[2]\n" + "ldr x20, [x17, #0xb8]\n" + "fmla v16.4s, v29.4s, v2.s[2]\n" "prfm pldl1keep, [x9, #0x80]\n" - "fmla v26.4s, v6.4s, v4.s[1]\n" - "ldr d6, [x17, #0x80]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "mov v6.d[1], x12\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "ldr x12, [x17, #0xa8]\n" - "fmla v19.4s, v7.4s, v2.s[1]\n" - "prfm pldl1keep, [x27, #0x80]\n" - "fmla v23.4s, v7.4s, v3.s[1]\n" - "prfm pldl1keep, [x25, #0x80]\n" - "fmla v27.4s, v7.4s, v4.s[1]\n" - "ldr d7, [x17, #0x90]\n" - "mov v7.d[1], x11\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "fmla v12.4s, v6.4s, v1.s[2]\n" - "ldr x11, [x17, #0xb8]\n" - "fmla v16.4s, v6.4s, v2.s[2]\n" - "prfm pldl1keep, [x23, #0x80]\n" - "fmla v20.4s, v6.4s, v3.s[2]\n" - "fmla v24.4s, v6.4s, v4.s[2]\n" - "ldr d6, [x17, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "mov v6.d[1], x12\n" - "fmla v13.4s, v7.4s, v1.s[2]\n" - "ldr x12, [x17, #0xc8]\n" - "fmla v17.4s, v7.4s, v2.s[2]\n" - "fmla v21.4s, v7.4s, v3.s[2]\n" - "fmla v25.4s, v7.4s, v4.s[2]\n" - "ldr d7, [x17, #0xb0]\n" - "mov v7.d[1], x11\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v14.4s, v6.4s, v1.s[2]\n" - "ldr x11, [x17, #0xd8]\n" - "fmla v18.4s, v6.4s, v2.s[2]\n" - "fmla v22.4s, v6.4s, v3.s[2]\n" - "fmla v26.4s, v6.4s, v4.s[2]\n" - "ldr d6, [x17, #0xc0]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "mov v6.d[1], x12\n" - "fmla v15.4s, v7.4s, v1.s[2]\n" - "ldr x12, [x17, #0xe8]\n" - "fmla v19.4s, v7.4s, v2.s[2]\n" - "fmla v23.4s, v7.4s, v3.s[2]\n" - "fmla v27.4s, v7.4s, v4.s[2]\n" - "ldr d7, [x17, #0xd0]\n" - "mov v7.d[1], x11\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v12.4s, v6.4s, v1.s[3]\n" - "ldr x11, [x17, #0xf8]\n" - "fmla v16.4s, v6.4s, v2.s[3]\n" - "fmla v20.4s, v6.4s, v3.s[3]\n" - "fmla v24.4s, v6.4s, v4.s[3]\n" - "ldr d6, [x17, #0xe0]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "mov v6.d[1], x12\n" - "fmla v13.4s, v7.4s, v1.s[3]\n" - "fmla v17.4s, v7.4s, v2.s[3]\n" - "fmla v21.4s, v7.4s, v3.s[3]\n" - "fmla v25.4s, v7.4s, v4.s[3]\n" - "ldr d7, [x17, #0xf0]\n" - "mov v7.d[1], x11\n" + "fmla v20.4s, v29.4s, v3.s[2]\n" + "fmla v24.4s, v29.4s, v4.s[2]\n" + "ldr d29, [x17, #0xa0]\n" + "fmla v9.4s, v28.4s, v0.s[2]\n" + "mov v29.d[1], x21\n" + "fmla v13.4s, v28.4s, v1.s[2]\n" + "ldr x21, [x17, #0xc8]\n" + "fmla v17.4s, v28.4s, v2.s[2]\n" + "fmla v21.4s, v28.4s, v3.s[2]\n" + "fmla v25.4s, v28.4s, v4.s[2]\n" + "ldr d28, [x17, #0xb0]\n" + "mov v28.d[1], x20\n" + "fmla v10.4s, v29.4s, v0.s[2]\n" + "fmla v14.4s, v29.4s, v1.s[2]\n" + "ldr x20, [x17, #0xd8]\n" + "fmla v18.4s, v29.4s, v2.s[2]\n" + "fmla v22.4s, v29.4s, v3.s[2]\n" + "fmla v26.4s, v29.4s, v4.s[2]\n" + "ldr d29, [x17, #0xc0]\n" + "fmla v11.4s, v28.4s, v0.s[2]\n" + "mov v29.d[1], x21\n" + "fmla v15.4s, v28.4s, v1.s[2]\n" + "ldr x21, [x17, #0xe8]\n" + "fmla v19.4s, v28.4s, v2.s[2]\n" + "fmla v23.4s, v28.4s, v3.s[2]\n" + "fmla v27.4s, v28.4s, v4.s[2]\n" + "ldr d28, [x17, #0xd0]\n" + "mov v28.d[1], x20\n" + "fmla v8.4s, v29.4s, v0.s[3]\n" + "fmla v12.4s, v29.4s, v1.s[3]\n" + "ldr x20, [x17, #0xf8]\n" + "fmla v16.4s, v29.4s, v2.s[3]\n" + "fmla v20.4s, v29.4s, v3.s[3]\n" + "fmla v24.4s, v29.4s, v4.s[3]\n" + "ldr d29, [x17, #0xe0]\n" + "fmla v9.4s, v28.4s, v0.s[3]\n" + "mov v29.d[1], x21\n" + "fmla v13.4s, v28.4s, v1.s[3]\n" + "fmla v17.4s, v28.4s, v2.s[3]\n" + "fmla v21.4s, v28.4s, v3.s[3]\n" + "fmla v25.4s, v28.4s, v4.s[3]\n" + "ldr d28, [x17, #0xf0]\n" + "mov v28.d[1], x20\n" "add x17, x17, #0x100\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" - "ldr x12, [x17, #0x8]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "ldr x11, [x17, #0x18]\n" - "fmla v18.4s, v6.4s, v2.s[3]\n" - "fmla v22.4s, v6.4s, v3.s[3]\n" - "fmla v26.4s, v6.4s, v4.s[3]\n" + "fmla v10.4s, v29.4s, v0.s[3]\n" + "ldr x21, [x17, #0x8]\n" + "fmla v14.4s, v29.4s, v1.s[3]\n" + "ldr x20, [x17, #0x18]\n" + "fmla v18.4s, v29.4s, v2.s[3]\n" + "fmla v22.4s, v29.4s, v3.s[3]\n" + "fmla v26.4s, v29.4s, v4.s[3]\n" "ldr d6, [x17, #0x0]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" + "fmla v11.4s, v28.4s, v0.s[3]\n" "ldr d0, [x13, #0x0]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" - "ldr d1, [x9, #0x0]\n" - "fmla v19.4s, v7.4s, v2.s[3]\n" - "ldr d2, [x27, #0x0]\n" - "fmla v23.4s, v7.4s, v3.s[3]\n" - "ldr d3, [x25, #0x0]\n" - "fmla v27.4s, v7.4s, v4.s[3]\n" - "ldr d4, [x23, #0x0]\n" + "fmla v15.4s, v28.4s, v1.s[3]\n" + "ldr d1, [x12, #0x0]\n" + "fmla v19.4s, v28.4s, v2.s[3]\n" + "ldr d2, [x11, #0x0]\n" + "fmla v23.4s, v28.4s, v3.s[3]\n" + "ldr d3, [x10, #0x0]\n" + "fmla v27.4s, v28.4s, v4.s[3]\n" + "ldr d4, [x9, #0x0]\n" "ldr d7, [x17, #0x10]\n" - "mov v6.d[1], x12\n" - "mov v0.d[1], x10\n" - "mov v1.d[1], x28\n" - "mov v2.d[1], x26\n" - "mov v3.d[1], x24\n" + "mov v6.d[1], x21\n" + "mov v0.d[1], x26\n" + "mov v1.d[1], x25\n" + "mov v2.d[1], x24\n" + "mov v3.d[1], x23\n" "mov v4.d[1], x22\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "bge 150b\n" "151:" // Height 5: Multiply loop: Single iteration only "fmla v8.4s, v6.4s, v0.s[0]\n" "add x13, x13, #0x10\n" "fmla v12.4s, v6.4s, v1.s[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" "fmla v16.4s, v6.4s, v2.s[0]\n" - "add x27, x27, #0x10\n" + "add x11, x11, #0x10\n" "fmla v20.4s, v6.4s, v3.s[0]\n" - "add x25, x25, #0x10\n" + "add x10, x10, #0x10\n" "fmla v24.4s, v6.4s, v4.s[0]\n" - "ldr q6, [x17, #0x20]\n" + "ldr q29, [x17, #0x20]\n" "fmla v9.4s, v7.4s, v0.s[0]\n" - "add x23, x23, #0x10\n" + "add x9, x9, #0x10\n" "fmla v13.4s, v7.4s, v1.s[0]\n" "sub x14, x14, #0x4\n" "fmla v17.4s, v7.4s, v2.s[0]\n" "prfm pldl1keep, [x13, #0x80]\n" "fmla v21.4s, v7.4s, v3.s[0]\n" - "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x12, #0x80]\n" "fmla v25.4s, v7.4s, v4.s[0]\n" - "ldr q7, [x17, #0x30]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "prfm pldl1keep, [x27, #0x80]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "prfm pldl1keep, [x25, #0x80]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "prfm pldl1keep, [x23, #0x80]\n" - "fmla v22.4s, v6.4s, v3.s[0]\n" - "fmla v26.4s, v6.4s, v4.s[0]\n" - "ldr q6, [x17, #0x40]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "fmla v23.4s, v7.4s, v3.s[0]\n" - "fmla v27.4s, v7.4s, v4.s[0]\n" - "ldr q7, [x17, #0x50]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v12.4s, v6.4s, v1.s[1]\n" - "fmla v16.4s, v6.4s, v2.s[1]\n" - "fmla v20.4s, v6.4s, v3.s[1]\n" - "fmla v24.4s, v6.4s, v4.s[1]\n" - "ldr q6, [x17, #0x60]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v13.4s, v7.4s, v1.s[1]\n" - "fmla v17.4s, v7.4s, v2.s[1]\n" - "fmla v21.4s, v7.4s, v3.s[1]\n" - "fmla v25.4s, v7.4s, v4.s[1]\n" - "ldr q7, [x17, #0x70]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "fmla v18.4s, v6.4s, v2.s[1]\n" - "fmla v22.4s, v6.4s, v3.s[1]\n" - "fmla v26.4s, v6.4s, v4.s[1]\n" - "ldr q6, [x17, #0x80]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "fmla v19.4s, v7.4s, v2.s[1]\n" - "fmla v23.4s, v7.4s, v3.s[1]\n" - "fmla v27.4s, v7.4s, v4.s[1]\n" - "ldr q7, [x17, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "fmla v12.4s, v6.4s, v1.s[2]\n" - "fmla v16.4s, v6.4s, v2.s[2]\n" - "fmla v20.4s, v6.4s, v3.s[2]\n" - "fmla v24.4s, v6.4s, v4.s[2]\n" - "ldr q6, [x17, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "fmla v13.4s, v7.4s, v1.s[2]\n" - "fmla v17.4s, v7.4s, v2.s[2]\n" - "fmla v21.4s, v7.4s, v3.s[2]\n" - "fmla v25.4s, v7.4s, v4.s[2]\n" - "ldr q7, [x17, #0xb0]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v14.4s, v6.4s, v1.s[2]\n" - "fmla v18.4s, v6.4s, v2.s[2]\n" - "fmla v22.4s, v6.4s, v3.s[2]\n" - "fmla v26.4s, v6.4s, v4.s[2]\n" - "ldr q6, [x17, #0xc0]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "fmla v15.4s, v7.4s, v1.s[2]\n" - "fmla v19.4s, v7.4s, v2.s[2]\n" - "fmla v23.4s, v7.4s, v3.s[2]\n" - "fmla v27.4s, v7.4s, v4.s[2]\n" - "ldr q7, [x17, #0xd0]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v12.4s, v6.4s, v1.s[3]\n" - "fmla v16.4s, v6.4s, v2.s[3]\n" - "fmla v20.4s, v6.4s, v3.s[3]\n" - "fmla v24.4s, v6.4s, v4.s[3]\n" - "ldr q6, [x17, #0xe0]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "fmla v13.4s, v7.4s, v1.s[3]\n" - "fmla v17.4s, v7.4s, v2.s[3]\n" - "fmla v21.4s, v7.4s, v3.s[3]\n" - "fmla v25.4s, v7.4s, v4.s[3]\n" - "ldr q7, [x17, #0xf0]\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" + "ldr q28, [x17, #0x30]\n" + "fmla v10.4s, v29.4s, v0.s[0]\n" + "prfm pldl1keep, [x11, #0x80]\n" + "fmla v14.4s, v29.4s, v1.s[0]\n" + "prfm pldl1keep, [x10, #0x80]\n" + "fmla v18.4s, v29.4s, v2.s[0]\n" + "prfm pldl1keep, [x9, #0x80]\n" + "fmla v22.4s, v29.4s, v3.s[0]\n" + "fmla v26.4s, v29.4s, v4.s[0]\n" + "ldr q29, [x17, #0x40]\n" + "fmla v11.4s, v28.4s, v0.s[0]\n" + "fmla v15.4s, v28.4s, v1.s[0]\n" + "fmla v19.4s, v28.4s, v2.s[0]\n" + "fmla v23.4s, v28.4s, v3.s[0]\n" + "fmla v27.4s, v28.4s, v4.s[0]\n" + "ldr q28, [x17, #0x50]\n" + "fmla v8.4s, v29.4s, v0.s[1]\n" + "fmla v12.4s, v29.4s, v1.s[1]\n" + "fmla v16.4s, v29.4s, v2.s[1]\n" + "fmla v20.4s, v29.4s, v3.s[1]\n" + "fmla v24.4s, v29.4s, v4.s[1]\n" + "ldr q29, [x17, #0x60]\n" + "fmla v9.4s, v28.4s, v0.s[1]\n" + "fmla v13.4s, v28.4s, v1.s[1]\n" + "fmla v17.4s, v28.4s, v2.s[1]\n" + "fmla v21.4s, v28.4s, v3.s[1]\n" + "fmla v25.4s, v28.4s, v4.s[1]\n" + "ldr q28, [x17, #0x70]\n" + "fmla v10.4s, v29.4s, v0.s[1]\n" + "fmla v14.4s, v29.4s, v1.s[1]\n" + "fmla v18.4s, v29.4s, v2.s[1]\n" + "fmla v22.4s, v29.4s, v3.s[1]\n" + "fmla v26.4s, v29.4s, v4.s[1]\n" + "ldr q29, [x17, #0x80]\n" + "fmla v11.4s, v28.4s, v0.s[1]\n" + "fmla v15.4s, v28.4s, v1.s[1]\n" + "fmla v19.4s, v28.4s, v2.s[1]\n" + "fmla v23.4s, v28.4s, v3.s[1]\n" + "fmla v27.4s, v28.4s, v4.s[1]\n" + "ldr q28, [x17, #0x90]\n" + "fmla v8.4s, v29.4s, v0.s[2]\n" + "fmla v12.4s, v29.4s, v1.s[2]\n" + "fmla v16.4s, v29.4s, v2.s[2]\n" + "fmla v20.4s, v29.4s, v3.s[2]\n" + "fmla v24.4s, v29.4s, v4.s[2]\n" + "ldr q29, [x17, #0xa0]\n" + "fmla v9.4s, v28.4s, v0.s[2]\n" + "fmla v13.4s, v28.4s, v1.s[2]\n" + "fmla v17.4s, v28.4s, v2.s[2]\n" + "fmla v21.4s, v28.4s, v3.s[2]\n" + "fmla v25.4s, v28.4s, v4.s[2]\n" + "ldr q28, [x17, #0xb0]\n" + "fmla v10.4s, v29.4s, v0.s[2]\n" + "fmla v14.4s, v29.4s, v1.s[2]\n" + "fmla v18.4s, v29.4s, v2.s[2]\n" + "fmla v22.4s, v29.4s, v3.s[2]\n" + "fmla v26.4s, v29.4s, v4.s[2]\n" + "ldr q29, [x17, #0xc0]\n" + "fmla v11.4s, v28.4s, v0.s[2]\n" + "fmla v15.4s, v28.4s, v1.s[2]\n" + "fmla v19.4s, v28.4s, v2.s[2]\n" + "fmla v23.4s, v28.4s, v3.s[2]\n" + "fmla v27.4s, v28.4s, v4.s[2]\n" + "ldr q28, [x17, #0xd0]\n" + "fmla v8.4s, v29.4s, v0.s[3]\n" + "fmla v12.4s, v29.4s, v1.s[3]\n" + "fmla v16.4s, v29.4s, v2.s[3]\n" + "fmla v20.4s, v29.4s, v3.s[3]\n" + "fmla v24.4s, v29.4s, v4.s[3]\n" + "ldr q29, [x17, #0xe0]\n" + "fmla v9.4s, v28.4s, v0.s[3]\n" + "fmla v13.4s, v28.4s, v1.s[3]\n" + "fmla v17.4s, v28.4s, v2.s[3]\n" + "fmla v21.4s, v28.4s, v3.s[3]\n" + "fmla v25.4s, v28.4s, v4.s[3]\n" + "ldr q28, [x17, #0xf0]\n" + "fmla v10.4s, v29.4s, v0.s[3]\n" "add x17, x17, #0x100\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "fmla v18.4s, v6.4s, v2.s[3]\n" - "fmla v22.4s, v6.4s, v3.s[3]\n" - "fmla v26.4s, v6.4s, v4.s[3]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" - "fmla v19.4s, v7.4s, v2.s[3]\n" - "fmla v23.4s, v7.4s, v3.s[3]\n" - "fmla v27.4s, v7.4s, v4.s[3]\n" + "fmla v14.4s, v29.4s, v1.s[3]\n" + "fmla v18.4s, v29.4s, v2.s[3]\n" + "fmla v22.4s, v29.4s, v3.s[3]\n" + "fmla v26.4s, v29.4s, v4.s[3]\n" + "fmla v11.4s, v28.4s, v0.s[3]\n" + "fmla v15.4s, v28.4s, v1.s[3]\n" + "fmla v19.4s, v28.4s, v2.s[3]\n" + "fmla v23.4s, v28.4s, v3.s[3]\n" + "fmla v27.4s, v28.4s, v4.s[3]\n" "152:" // Height 5: Multiply loop: Main loop skip "cbz x14, 154f\n" "153:" // Height 5: Multiply loop: Odd block loop - "ldr s0, [x13], #0x4\n" + "ldr s2, [x13], #0x4\n" "sub x14, x14, #0x1\n" - "ldr s1, [x9], #0x4\n" - "ldr s2, [x27], #0x4\n" - "ldr s3, [x25], #0x4\n" - "ldr s4, [x23], #0x4\n" - "ldr q6, [x17, #0x0]\n" - "fmla v8.4s, v6.4s, v0.s[0]\n" - "ldr q7, [x17, #0x10]\n" - "fmla v12.4s, v6.4s, v1.s[0]\n" - "fmla v16.4s, v6.4s, v2.s[0]\n" - "fmla v20.4s, v6.4s, v3.s[0]\n" - "fmla v24.4s, v6.4s, v4.s[0]\n" - "ldr q6, [x17, #0x20]\n" - "fmla v9.4s, v7.4s, v0.s[0]\n" - "fmla v13.4s, v7.4s, v1.s[0]\n" - "fmla v17.4s, v7.4s, v2.s[0]\n" - "fmla v21.4s, v7.4s, v3.s[0]\n" - "fmla v25.4s, v7.4s, v4.s[0]\n" - "ldr q7, [x17, #0x30]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" + "ldr s1, [x12], #0x4\n" + "ldr s0, [x11], #0x4\n" + "ldr s31, [x10], #0x4\n" + "ldr s30, [x9], #0x4\n" + "ldr q29, [x17, #0x0]\n" + "fmla v8.4s, v29.4s, v2.s[0]\n" + "ldr q28, [x17, #0x10]\n" + "fmla v12.4s, v29.4s, v1.s[0]\n" + "fmla v16.4s, v29.4s, v0.s[0]\n" + "fmla v20.4s, v29.4s, v31.s[0]\n" + "fmla v24.4s, v29.4s, v30.s[0]\n" + "ldr q29, [x17, #0x20]\n" + "fmla v9.4s, v28.4s, v2.s[0]\n" + "fmla v13.4s, v28.4s, v1.s[0]\n" + "fmla v17.4s, v28.4s, v0.s[0]\n" + "fmla v21.4s, v28.4s, v31.s[0]\n" + "fmla v25.4s, v28.4s, v30.s[0]\n" + "ldr q28, [x17, #0x30]\n" + "fmla v10.4s, v29.4s, v2.s[0]\n" "add x17, x17, #0x40\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "fmla v22.4s, v6.4s, v3.s[0]\n" - "fmla v26.4s, v6.4s, v4.s[0]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "fmla v23.4s, v7.4s, v3.s[0]\n" - "fmla v27.4s, v7.4s, v4.s[0]\n" + "fmla v14.4s, v29.4s, v1.s[0]\n" + "fmla v18.4s, v29.4s, v0.s[0]\n" + "fmla v22.4s, v29.4s, v31.s[0]\n" + "fmla v26.4s, v29.4s, v30.s[0]\n" + "fmla v11.4s, v28.4s, v2.s[0]\n" + "fmla v15.4s, v28.4s, v1.s[0]\n" + "fmla v19.4s, v28.4s, v0.s[0]\n" + "fmla v23.4s, v28.4s, v31.s[0]\n" + "fmla v27.4s, v28.4s, v30.s[0]\n" "cbnz x14, 153b\n" "154:" // Height 5: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -2542,49 +2541,49 @@ void a64_hybrid_fp32_mla_6x16_a55 ( "prfm pstl1keep, [x22, #0x0]\n" "tbz %x[flags], #1, 155f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v0.4s\n" - "fmin v9.4s, v9.4s, v0.4s\n" - "fmin v10.4s, v10.4s, v0.4s\n" - "fmin v11.4s, v11.4s, v0.4s\n" - "fmin v12.4s, v12.4s, v0.4s\n" - "fmin v13.4s, v13.4s, v0.4s\n" - "fmin v14.4s, v14.4s, v0.4s\n" - "fmin v15.4s, v15.4s, v0.4s\n" - "fmin v16.4s, v16.4s, v0.4s\n" - "fmin v17.4s, v17.4s, v0.4s\n" - "fmin v18.4s, v18.4s, v0.4s\n" - "fmin v19.4s, v19.4s, v0.4s\n" - "fmin v20.4s, v20.4s, v0.4s\n" - "fmin v21.4s, v21.4s, v0.4s\n" - "fmin v22.4s, v22.4s, v0.4s\n" - "fmin v23.4s, v23.4s, v0.4s\n" - "fmin v24.4s, v24.4s, v0.4s\n" - "fmin v25.4s, v25.4s, v0.4s\n" - "fmin v26.4s, v26.4s, v0.4s\n" - "fmin v27.4s, v27.4s, v0.4s\n" + "ld1r { v28.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v28.4s\n" + "fmin v9.4s, v9.4s, v28.4s\n" + "fmin v10.4s, v10.4s, v28.4s\n" + "fmin v11.4s, v11.4s, v28.4s\n" + "fmin v12.4s, v12.4s, v28.4s\n" + "fmin v13.4s, v13.4s, v28.4s\n" + "fmin v14.4s, v14.4s, v28.4s\n" + "fmin v15.4s, v15.4s, v28.4s\n" + "fmin v16.4s, v16.4s, v28.4s\n" + "fmin v17.4s, v17.4s, v28.4s\n" + "fmin v18.4s, v18.4s, v28.4s\n" + "fmin v19.4s, v19.4s, v28.4s\n" + "fmin v20.4s, v20.4s, v28.4s\n" + "fmin v21.4s, v21.4s, v28.4s\n" + "fmin v22.4s, v22.4s, v28.4s\n" + "fmin v23.4s, v23.4s, v28.4s\n" + "fmin v24.4s, v24.4s, v28.4s\n" + "fmin v25.4s, v25.4s, v28.4s\n" + "fmin v26.4s, v26.4s, v28.4s\n" + "fmin v27.4s, v27.4s, v28.4s\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" - "fmax v16.4s, v16.4s, v0.4s\n" - "fmax v17.4s, v17.4s, v0.4s\n" - "fmax v18.4s, v18.4s, v0.4s\n" - "fmax v19.4s, v19.4s, v0.4s\n" - "fmax v20.4s, v20.4s, v0.4s\n" - "fmax v21.4s, v21.4s, v0.4s\n" - "fmax v22.4s, v22.4s, v0.4s\n" - "fmax v23.4s, v23.4s, v0.4s\n" - "fmax v24.4s, v24.4s, v0.4s\n" - "fmax v25.4s, v25.4s, v0.4s\n" - "fmax v26.4s, v26.4s, v0.4s\n" - "fmax v27.4s, v27.4s, v0.4s\n" + "ld1r { v28.4s }, [x20]\n" + "fmax v8.4s, v8.4s, v28.4s\n" + "fmax v9.4s, v9.4s, v28.4s\n" + "fmax v10.4s, v10.4s, v28.4s\n" + "fmax v11.4s, v11.4s, v28.4s\n" + "fmax v12.4s, v12.4s, v28.4s\n" + "fmax v13.4s, v13.4s, v28.4s\n" + "fmax v14.4s, v14.4s, v28.4s\n" + "fmax v15.4s, v15.4s, v28.4s\n" + "fmax v16.4s, v16.4s, v28.4s\n" + "fmax v17.4s, v17.4s, v28.4s\n" + "fmax v18.4s, v18.4s, v28.4s\n" + "fmax v19.4s, v19.4s, v28.4s\n" + "fmax v20.4s, v20.4s, v28.4s\n" + "fmax v21.4s, v21.4s, v28.4s\n" + "fmax v22.4s, v22.4s, v28.4s\n" + "fmax v23.4s, v23.4s, v28.4s\n" + "fmax v24.4s, v24.4s, v28.4s\n" + "fmax v25.4s, v25.4s, v28.4s\n" + "fmax v26.4s, v26.4s, v28.4s\n" + "fmax v27.4s, v27.4s, v28.4s\n" "155:" // Height 5: No activation "cmp x8, #0x10\n" "bge 164f\n" @@ -2961,98 +2960,98 @@ void a64_hybrid_fp32_mla_6x16_a55 ( "180:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w14, [x20, x15, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 181f\n" - "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x13, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" - "ldr x27, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" - "ldr x23, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x15, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x13, [x20, #0x0]\n" + "ldr x12, [x20, #0x8]\n" + "ldr x11, [x20, #0x10]\n" + "ldr x10, [x20, #0x18]\n" + "ldr x9, [x20, #0x20]\n" + "ldr x28, [x20, #0x28]\n" "cbnz x15, 182f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x13, x13, x20, LSL #2\n" + "add x12, x12, x20, LSL #2\n" + "add x11, x11, x20, LSL #2\n" + "add x10, x10, x20, LSL #2\n" "add x9, x9, x20, LSL #2\n" - "add x27, x27, x20, LSL #2\n" - "add x25, x25, x20, LSL #2\n" - "add x23, x23, x20, LSL #2\n" - "add x21, x21, x20, LSL #2\n" + "add x28, x28, x20, LSL #2\n" "b 182f\n" "181:" // Height 6: setup direct input "mov x13, %x[input_ptr]\n" - "add x9, x13, x20, LSL #2\n" - "add x27, x9, x20, LSL #2\n" - "add x25, x27, x20, LSL #2\n" - "add x23, x25, x20, LSL #2\n" - "add x21, x23, x20, LSL #2\n" + "add x12, x13, x21, LSL #2\n" + "add x11, x12, x21, LSL #2\n" + "add x10, x11, x21, LSL #2\n" + "add x9, x10, x21, LSL #2\n" + "add x28, x9, x21, LSL #2\n" "182:" // Height 6: input setup done "cmp x14, #0x4\n" "blt 185f\n" "ldr q0, [x13, #0x0]\n" "cmp x14, #0x8\n" - "ldr q1, [x9, #0x0]\n" - "ldr q2, [x27, #0x0]\n" - "ldr q3, [x25, #0x0]\n" - "ldr q4, [x23, #0x0]\n" - "ldr q5, [x21, #0x0]\n" + "ldr q1, [x12, #0x0]\n" + "ldr q2, [x11, #0x0]\n" + "ldr q3, [x10, #0x0]\n" + "ldr q4, [x9, #0x0]\n" + "ldr q5, [x28, #0x0]\n" "ldr q6, [x17, #0x0]\n" "ldr q7, [x17, #0x10]\n" "blt 184f\n" "183:" // Height 6: Multiply loop: Main loop head "fmla v8.4s, v6.4s, v0.s[0]\n" - "ldr x12, [x17, #0x28]\n" + "ldr x21, [x17, #0x28]\n" "fmla v12.4s, v6.4s, v1.s[0]\n" - "ldr x11, [x17, #0x38]\n" + "ldr x20, [x17, #0x38]\n" "fmla v16.4s, v6.4s, v2.s[0]\n" "add x13, x13, #0x10\n" "fmla v20.4s, v6.4s, v3.s[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" "fmla v24.4s, v6.4s, v4.s[0]\n" - "add x27, x27, #0x10\n" + "add x11, x11, #0x10\n" "fmla v28.4s, v6.4s, v5.s[0]\n" "ldr d6, [x17, #0x20]\n" "fmla v9.4s, v7.4s, v0.s[0]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" "fmla v13.4s, v7.4s, v1.s[0]\n" - "ldr x12, [x17, #0x48]\n" + "ldr x21, [x17, #0x48]\n" "fmla v17.4s, v7.4s, v2.s[0]\n" - "add x25, x25, #0x10\n" + "add x10, x10, #0x10\n" "fmla v21.4s, v7.4s, v3.s[0]\n" - "add x23, x23, #0x10\n" + "add x9, x9, #0x10\n" "fmla v25.4s, v7.4s, v4.s[0]\n" - "add x21, x21, #0x10\n" + "add x28, x28, #0x10\n" "fmla v29.4s, v7.4s, v5.s[0]\n" "ldr d7, [x17, #0x30]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "fmla v10.4s, v6.4s, v0.s[0]\n" "fmla v14.4s, v6.4s, v1.s[0]\n" - "ldr x11, [x17, #0x58]\n" + "ldr x20, [x17, #0x58]\n" "fmla v18.4s, v6.4s, v2.s[0]\n" - "ldr x10, [x13, #0x8]\n" + "ldr x27, [x13, #0x8]\n" "fmla v22.4s, v6.4s, v3.s[0]\n" - "ldr x28, [x9, #0x8]\n" + "ldr x26, [x12, #0x8]\n" "fmla v26.4s, v6.4s, v4.s[0]\n" - "ldr x26, [x27, #0x8]\n" + "ldr x25, [x11, #0x8]\n" "fmla v30.4s, v6.4s, v5.s[0]\n" "ldr d6, [x17, #0x40]\n" "fmla v11.4s, v7.4s, v0.s[0]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" "fmla v15.4s, v7.4s, v1.s[0]\n" - "ldr x12, [x17, #0x68]\n" + "ldr x21, [x17, #0x68]\n" "fmla v19.4s, v7.4s, v2.s[0]\n" - "ldr x24, [x25, #0x8]\n" + "ldr x24, [x10, #0x8]\n" "fmla v23.4s, v7.4s, v3.s[0]\n" - "ldr x22, [x23, #0x8]\n" + "ldr x23, [x9, #0x8]\n" "fmla v27.4s, v7.4s, v4.s[0]\n" - "ldr x20, [x21, #0x8]\n" + "ldr x22, [x28, #0x8]\n" "fmla v31.4s, v7.4s, v5.s[0]\n" "ldr d7, [x17, #0x50]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "fmla v8.4s, v6.4s, v0.s[1]\n" "fmla v12.4s, v6.4s, v1.s[1]\n" - "ldr x11, [x17, #0x78]\n" + "ldr x20, [x17, #0x78]\n" "fmla v16.4s, v6.4s, v2.s[1]\n" "sub x14, x14, #0x4\n" "fmla v20.4s, v6.4s, v3.s[1]\n" @@ -3062,96 +3061,96 @@ void a64_hybrid_fp32_mla_6x16_a55 ( "fmla v28.4s, v6.4s, v5.s[1]\n" "ldr d6, [x17, #0x60]\n" "fmla v9.4s, v7.4s, v0.s[1]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" "fmla v13.4s, v7.4s, v1.s[1]\n" - "ldr x12, [x17, #0x88]\n" + "ldr x21, [x17, #0x88]\n" "fmla v17.4s, v7.4s, v2.s[1]\n" - "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x12, #0x80]\n" "fmla v21.4s, v7.4s, v3.s[1]\n" - "prfm pldl1keep, [x27, #0x80]\n" + "prfm pldl1keep, [x11, #0x80]\n" "fmla v25.4s, v7.4s, v4.s[1]\n" - "prfm pldl1keep, [x25, #0x80]\n" + "prfm pldl1keep, [x10, #0x80]\n" "fmla v29.4s, v7.4s, v5.s[1]\n" "ldr d7, [x17, #0x70]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "fmla v10.4s, v6.4s, v0.s[1]\n" "fmla v14.4s, v6.4s, v1.s[1]\n" - "ldr x11, [x17, #0x98]\n" + "ldr x20, [x17, #0x98]\n" "fmla v18.4s, v6.4s, v2.s[1]\n" - "prfm pldl1keep, [x23, #0x80]\n" + "prfm pldl1keep, [x9, #0x80]\n" "fmla v22.4s, v6.4s, v3.s[1]\n" - "prfm pldl1keep, [x21, #0x80]\n" + "prfm pldl1keep, [x28, #0x80]\n" "fmla v26.4s, v6.4s, v4.s[1]\n" "fmla v30.4s, v6.4s, v5.s[1]\n" "ldr d6, [x17, #0x80]\n" "fmla v11.4s, v7.4s, v0.s[1]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" "fmla v15.4s, v7.4s, v1.s[1]\n" - "ldr x12, [x17, #0xa8]\n" + "ldr x21, [x17, #0xa8]\n" "fmla v19.4s, v7.4s, v2.s[1]\n" "fmla v23.4s, v7.4s, v3.s[1]\n" "fmla v27.4s, v7.4s, v4.s[1]\n" "fmla v31.4s, v7.4s, v5.s[1]\n" "ldr d7, [x17, #0x90]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "fmla v8.4s, v6.4s, v0.s[2]\n" "fmla v12.4s, v6.4s, v1.s[2]\n" - "ldr x11, [x17, #0xb8]\n" + "ldr x20, [x17, #0xb8]\n" "fmla v16.4s, v6.4s, v2.s[2]\n" "fmla v20.4s, v6.4s, v3.s[2]\n" "fmla v24.4s, v6.4s, v4.s[2]\n" "fmla v28.4s, v6.4s, v5.s[2]\n" "ldr d6, [x17, #0xa0]\n" "fmla v9.4s, v7.4s, v0.s[2]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" "fmla v13.4s, v7.4s, v1.s[2]\n" - "ldr x12, [x17, #0xc8]\n" + "ldr x21, [x17, #0xc8]\n" "fmla v17.4s, v7.4s, v2.s[2]\n" "fmla v21.4s, v7.4s, v3.s[2]\n" "fmla v25.4s, v7.4s, v4.s[2]\n" "fmla v29.4s, v7.4s, v5.s[2]\n" "ldr d7, [x17, #0xb0]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "fmla v10.4s, v6.4s, v0.s[2]\n" "fmla v14.4s, v6.4s, v1.s[2]\n" - "ldr x11, [x17, #0xd8]\n" + "ldr x20, [x17, #0xd8]\n" "fmla v18.4s, v6.4s, v2.s[2]\n" "fmla v22.4s, v6.4s, v3.s[2]\n" "fmla v26.4s, v6.4s, v4.s[2]\n" "fmla v30.4s, v6.4s, v5.s[2]\n" "ldr d6, [x17, #0xc0]\n" "fmla v11.4s, v7.4s, v0.s[2]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" "fmla v15.4s, v7.4s, v1.s[2]\n" - "ldr x12, [x17, #0xe8]\n" + "ldr x21, [x17, #0xe8]\n" "fmla v19.4s, v7.4s, v2.s[2]\n" "fmla v23.4s, v7.4s, v3.s[2]\n" "fmla v27.4s, v7.4s, v4.s[2]\n" "fmla v31.4s, v7.4s, v5.s[2]\n" "ldr d7, [x17, #0xd0]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "fmla v8.4s, v6.4s, v0.s[3]\n" "fmla v12.4s, v6.4s, v1.s[3]\n" - "ldr x11, [x17, #0xf8]\n" + "ldr x20, [x17, #0xf8]\n" "fmla v16.4s, v6.4s, v2.s[3]\n" "fmla v20.4s, v6.4s, v3.s[3]\n" "fmla v24.4s, v6.4s, v4.s[3]\n" "fmla v28.4s, v6.4s, v5.s[3]\n" "ldr d6, [x17, #0xe0]\n" "fmla v9.4s, v7.4s, v0.s[3]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" "fmla v13.4s, v7.4s, v1.s[3]\n" "fmla v17.4s, v7.4s, v2.s[3]\n" "fmla v21.4s, v7.4s, v3.s[3]\n" "fmla v25.4s, v7.4s, v4.s[3]\n" "fmla v29.4s, v7.4s, v5.s[3]\n" "ldr d7, [x17, #0xf0]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "add x17, x17, #0x100\n" "fmla v10.4s, v6.4s, v0.s[3]\n" - "ldr x12, [x17, #0x8]\n" + "ldr x21, [x17, #0x8]\n" "fmla v14.4s, v6.4s, v1.s[3]\n" - "ldr x11, [x17, #0x18]\n" + "ldr x20, [x17, #0x18]\n" "fmla v18.4s, v6.4s, v2.s[3]\n" "fmla v22.4s, v6.4s, v3.s[3]\n" "fmla v26.4s, v6.4s, v4.s[3]\n" @@ -3160,56 +3159,56 @@ void a64_hybrid_fp32_mla_6x16_a55 ( "fmla v11.4s, v7.4s, v0.s[3]\n" "ldr d0, [x13, #0x0]\n" "fmla v15.4s, v7.4s, v1.s[3]\n" - "ldr d1, [x9, #0x0]\n" + "ldr d1, [x12, #0x0]\n" "fmla v19.4s, v7.4s, v2.s[3]\n" - "ldr d2, [x27, #0x0]\n" + "ldr d2, [x11, #0x0]\n" "fmla v23.4s, v7.4s, v3.s[3]\n" - "ldr d3, [x25, #0x0]\n" + "ldr d3, [x10, #0x0]\n" "fmla v27.4s, v7.4s, v4.s[3]\n" - "ldr d4, [x23, #0x0]\n" + "ldr d4, [x9, #0x0]\n" "fmla v31.4s, v7.4s, v5.s[3]\n" - "ldr d5, [x21, #0x0]\n" + "ldr d5, [x28, #0x0]\n" "ldr d7, [x17, #0x10]\n" - "mov v6.d[1], x12\n" - "mov v0.d[1], x10\n" - "mov v1.d[1], x28\n" - "mov v2.d[1], x26\n" + "mov v6.d[1], x21\n" + "mov v0.d[1], x27\n" + "mov v1.d[1], x26\n" + "mov v2.d[1], x25\n" "mov v3.d[1], x24\n" - "mov v4.d[1], x22\n" - "mov v5.d[1], x20\n" - "mov v7.d[1], x11\n" + "mov v4.d[1], x23\n" + "mov v5.d[1], x22\n" + "mov v7.d[1], x20\n" "bge 183b\n" "184:" // Height 6: Multiply loop: Single iteration only "fmla v8.4s, v6.4s, v0.s[0]\n" "add x13, x13, #0x10\n" "fmla v12.4s, v6.4s, v1.s[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" "fmla v16.4s, v6.4s, v2.s[0]\n" - "add x27, x27, #0x10\n" + "add x11, x11, #0x10\n" "fmla v20.4s, v6.4s, v3.s[0]\n" - "add x25, x25, #0x10\n" + "add x10, x10, #0x10\n" "fmla v24.4s, v6.4s, v4.s[0]\n" - "add x23, x23, #0x10\n" + "add x9, x9, #0x10\n" "fmla v28.4s, v6.4s, v5.s[0]\n" "ldr q6, [x17, #0x20]\n" "fmla v9.4s, v7.4s, v0.s[0]\n" - "add x21, x21, #0x10\n" + "add x28, x28, #0x10\n" "fmla v13.4s, v7.4s, v1.s[0]\n" "sub x14, x14, #0x4\n" "fmla v17.4s, v7.4s, v2.s[0]\n" "prfm pldl1keep, [x13, #0x80]\n" "fmla v21.4s, v7.4s, v3.s[0]\n" - "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x12, #0x80]\n" "fmla v25.4s, v7.4s, v4.s[0]\n" - "prfm pldl1keep, [x27, #0x80]\n" + "prfm pldl1keep, [x11, #0x80]\n" "fmla v29.4s, v7.4s, v5.s[0]\n" "ldr q7, [x17, #0x30]\n" "fmla v10.4s, v6.4s, v0.s[0]\n" - "prfm pldl1keep, [x25, #0x80]\n" + "prfm pldl1keep, [x10, #0x80]\n" "fmla v14.4s, v6.4s, v1.s[0]\n" - "prfm pldl1keep, [x23, #0x80]\n" + "prfm pldl1keep, [x9, #0x80]\n" "fmla v18.4s, v6.4s, v2.s[0]\n" - "prfm pldl1keep, [x21, #0x80]\n" + "prfm pldl1keep, [x28, #0x80]\n" "fmla v22.4s, v6.4s, v3.s[0]\n" "fmla v26.4s, v6.4s, v4.s[0]\n" "fmla v30.4s, v6.4s, v5.s[0]\n" @@ -3307,42 +3306,42 @@ void a64_hybrid_fp32_mla_6x16_a55 ( "185:" // Height 6: Multiply loop: Main loop skip "cbz x14, 187f\n" "186:" // Height 6: Multiply loop: Odd block loop - "ldr s0, [x13], #0x4\n" + "ldr s7, [x13], #0x4\n" "sub x14, x14, #0x1\n" - "ldr s1, [x9], #0x4\n" - "ldr s2, [x27], #0x4\n" - "ldr s3, [x25], #0x4\n" - "ldr s4, [x23], #0x4\n" - "ldr s5, [x21], #0x4\n" - "ldr q6, [x17, #0x0]\n" - "fmla v8.4s, v6.4s, v0.s[0]\n" - "ldr q7, [x17, #0x10]\n" - "fmla v12.4s, v6.4s, v1.s[0]\n" - "fmla v16.4s, v6.4s, v2.s[0]\n" - "fmla v20.4s, v6.4s, v3.s[0]\n" - "fmla v24.4s, v6.4s, v4.s[0]\n" - "fmla v28.4s, v6.4s, v5.s[0]\n" - "ldr q6, [x17, #0x20]\n" - "fmla v9.4s, v7.4s, v0.s[0]\n" - "fmla v13.4s, v7.4s, v1.s[0]\n" - "fmla v17.4s, v7.4s, v2.s[0]\n" - "fmla v21.4s, v7.4s, v3.s[0]\n" - "fmla v25.4s, v7.4s, v4.s[0]\n" - "fmla v29.4s, v7.4s, v5.s[0]\n" - "ldr q7, [x17, #0x30]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" + "ldr s6, [x12], #0x4\n" + "ldr s5, [x11], #0x4\n" + "ldr s4, [x10], #0x4\n" + "ldr s3, [x9], #0x4\n" + "ldr s2, [x28], #0x4\n" + "ldr q1, [x17, #0x0]\n" + "fmla v8.4s, v1.4s, v7.s[0]\n" + "ldr q0, [x17, #0x10]\n" + "fmla v12.4s, v1.4s, v6.s[0]\n" + "fmla v16.4s, v1.4s, v5.s[0]\n" + "fmla v20.4s, v1.4s, v4.s[0]\n" + "fmla v24.4s, v1.4s, v3.s[0]\n" + "fmla v28.4s, v1.4s, v2.s[0]\n" + "ldr q1, [x17, #0x20]\n" + "fmla v9.4s, v0.4s, v7.s[0]\n" + "fmla v13.4s, v0.4s, v6.s[0]\n" + "fmla v17.4s, v0.4s, v5.s[0]\n" + "fmla v21.4s, v0.4s, v4.s[0]\n" + "fmla v25.4s, v0.4s, v3.s[0]\n" + "fmla v29.4s, v0.4s, v2.s[0]\n" + "ldr q0, [x17, #0x30]\n" + "fmla v10.4s, v1.4s, v7.s[0]\n" "add x17, x17, #0x40\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "fmla v22.4s, v6.4s, v3.s[0]\n" - "fmla v26.4s, v6.4s, v4.s[0]\n" - "fmla v30.4s, v6.4s, v5.s[0]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "fmla v23.4s, v7.4s, v3.s[0]\n" - "fmla v27.4s, v7.4s, v4.s[0]\n" - "fmla v31.4s, v7.4s, v5.s[0]\n" + "fmla v14.4s, v1.4s, v6.s[0]\n" + "fmla v18.4s, v1.4s, v5.s[0]\n" + "fmla v22.4s, v1.4s, v4.s[0]\n" + "fmla v26.4s, v1.4s, v3.s[0]\n" + "fmla v30.4s, v1.4s, v2.s[0]\n" + "fmla v11.4s, v0.4s, v7.s[0]\n" + "fmla v15.4s, v0.4s, v6.s[0]\n" + "fmla v19.4s, v0.4s, v5.s[0]\n" + "fmla v23.4s, v0.4s, v4.s[0]\n" + "fmla v27.4s, v0.4s, v3.s[0]\n" + "fmla v31.4s, v0.4s, v2.s[0]\n" "cbnz x14, 186b\n" "187:" // Height 6: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -3584,7 +3583,6 @@ void a64_hybrid_fp32_mla_6x16_a55 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "200:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16/generic.cpp index c5e4388aa9b578b958c059c88985fc58fd0208ae..bb84a50282fd8141ce09304925c394cca134c257 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_6x16/generic.cpp @@ -92,7 +92,6 @@ void a64_hybrid_fp32_mla_6x16 ( break; } __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x6\n" "bge 166f\n" @@ -189,11 +188,11 @@ void a64_hybrid_fp32_mla_6x16 ( "15:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 16f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 17f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -210,37 +209,37 @@ void a64_hybrid_fp32_mla_6x16 ( "blt 19f\n" "18:" // Height 1: Multiply loop: Main loop head "fmla v8.4s, v6.4s, v0.s[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q17, [x10, #0x20]\n" "fmla v9.4s, v7.4s, v0.s[0]\n" - "ldr q7, [x10, #0x30]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "ldr q6, [x10, #0x40]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "ldr q7, [x10, #0x50]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "ldr q7, [x10, #0x70]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "ldr q6, [x10, #0x80]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "ldr q7, [x10, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "ldr q6, [x10, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "ldr q7, [x10, #0xb0]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "ldr q6, [x10, #0xc0]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "ldr q7, [x10, #0xd0]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "ldr q6, [x10, #0xe0]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "ldr q7, [x10, #0xf0]\n" + "ldr q16, [x10, #0x30]\n" + "fmla v10.4s, v17.4s, v0.s[0]\n" + "ldr q17, [x10, #0x40]\n" + "fmla v11.4s, v16.4s, v0.s[0]\n" + "ldr q16, [x10, #0x50]\n" + "fmla v8.4s, v17.4s, v0.s[1]\n" + "ldr q17, [x10, #0x60]\n" + "fmla v9.4s, v16.4s, v0.s[1]\n" + "ldr q16, [x10, #0x70]\n" + "fmla v10.4s, v17.4s, v0.s[1]\n" + "ldr q17, [x10, #0x80]\n" + "fmla v11.4s, v16.4s, v0.s[1]\n" + "ldr q16, [x10, #0x90]\n" + "fmla v8.4s, v17.4s, v0.s[2]\n" + "ldr q17, [x10, #0xa0]\n" + "fmla v9.4s, v16.4s, v0.s[2]\n" + "ldr q16, [x10, #0xb0]\n" + "fmla v10.4s, v17.4s, v0.s[2]\n" + "ldr q17, [x10, #0xc0]\n" + "fmla v11.4s, v16.4s, v0.s[2]\n" + "ldr q16, [x10, #0xd0]\n" + "fmla v8.4s, v17.4s, v0.s[3]\n" + "ldr q17, [x10, #0xe0]\n" + "fmla v9.4s, v16.4s, v0.s[3]\n" + "ldr q16, [x10, #0xf0]\n" "sub x27, x27, #0x4\n" "add x26, x26, #0x10\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" + "fmla v10.4s, v17.4s, v0.s[3]\n" + "fmla v11.4s, v16.4s, v0.s[3]\n" "ldr q0, [x26, #0x0]\n" "cmp x27, #0x8\n" "add x10, x10, #0x100\n" @@ -250,52 +249,52 @@ void a64_hybrid_fp32_mla_6x16 ( "bge 18b\n" "19:" // Height 1: Multiply loop: Single iteration only "fmla v8.4s, v6.4s, v0.s[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q17, [x10, #0x20]\n" "fmla v9.4s, v7.4s, v0.s[0]\n" - "ldr q7, [x10, #0x30]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "ldr q6, [x10, #0x40]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "ldr q7, [x10, #0x50]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "ldr q7, [x10, #0x70]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "ldr q6, [x10, #0x80]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "ldr q7, [x10, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "ldr q6, [x10, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "ldr q7, [x10, #0xb0]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "ldr q6, [x10, #0xc0]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "ldr q7, [x10, #0xd0]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "ldr q6, [x10, #0xe0]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "ldr q7, [x10, #0xf0]\n" + "ldr q16, [x10, #0x30]\n" + "fmla v10.4s, v17.4s, v0.s[0]\n" + "ldr q17, [x10, #0x40]\n" + "fmla v11.4s, v16.4s, v0.s[0]\n" + "ldr q16, [x10, #0x50]\n" + "fmla v8.4s, v17.4s, v0.s[1]\n" + "ldr q17, [x10, #0x60]\n" + "fmla v9.4s, v16.4s, v0.s[1]\n" + "ldr q16, [x10, #0x70]\n" + "fmla v10.4s, v17.4s, v0.s[1]\n" + "ldr q17, [x10, #0x80]\n" + "fmla v11.4s, v16.4s, v0.s[1]\n" + "ldr q16, [x10, #0x90]\n" + "fmla v8.4s, v17.4s, v0.s[2]\n" + "ldr q17, [x10, #0xa0]\n" + "fmla v9.4s, v16.4s, v0.s[2]\n" + "ldr q16, [x10, #0xb0]\n" + "fmla v10.4s, v17.4s, v0.s[2]\n" + "ldr q17, [x10, #0xc0]\n" + "fmla v11.4s, v16.4s, v0.s[2]\n" + "ldr q16, [x10, #0xd0]\n" + "fmla v8.4s, v17.4s, v0.s[3]\n" + "ldr q17, [x10, #0xe0]\n" + "fmla v9.4s, v16.4s, v0.s[3]\n" + "ldr q16, [x10, #0xf0]\n" "add x26, x26, #0x10\n" "sub x27, x27, #0x4\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" + "fmla v10.4s, v17.4s, v0.s[3]\n" + "fmla v11.4s, v16.4s, v0.s[3]\n" "prfm pldl1keep, [x26, #0x80]\n" "add x10, x10, #0x100\n" "20:" // Height 1: Multiply loop: Main loop skip "cbz x27, 22f\n" "21:" // Height 1: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr q6, [x10, #0x0]\n" - "fmla v8.4s, v6.4s, v0.s[0]\n" + "ldr s18, [x26], #0x4\n" + "ldr q16, [x10, #0x0]\n" + "fmla v8.4s, v16.4s, v18.s[0]\n" "sub x27, x27, #0x1\n" - "ldr q7, [x10, #0x10]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.4s, v7.4s, v0.s[0]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "ldr q7, [x10, #0x30]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" + "ldr q17, [x10, #0x10]\n" + "ldr q16, [x10, #0x20]\n" + "fmla v9.4s, v17.4s, v18.s[0]\n" + "fmla v10.4s, v16.4s, v18.s[0]\n" + "ldr q16, [x10, #0x30]\n" + "fmla v11.4s, v16.4s, v18.s[0]\n" "add x10, x10, #0x40\n" "cbnz x27, 21b\n" "22:" // Height 1: Multiply loop: No odd multiplies @@ -306,17 +305,17 @@ void a64_hybrid_fp32_mla_6x16 ( "prfm pstl1keep, [x9, #0x0]\n" "tbz %x[flags], #1, 23f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v17.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" + "ld1r { v16.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v17.4s\n" + "fmin v9.4s, v9.4s, v17.4s\n" + "fmin v10.4s, v10.4s, v17.4s\n" + "fmin v11.4s, v11.4s, v17.4s\n" + "fmax v8.4s, v8.4s, v16.4s\n" + "fmax v9.4s, v9.4s, v16.4s\n" + "fmax v10.4s, v10.4s, v16.4s\n" + "fmax v11.4s, v11.4s, v16.4s\n" "23:" // Height 1: No activation "cmp x11, #0x10\n" "bge 32f\n" @@ -494,12 +493,12 @@ void a64_hybrid_fp32_mla_6x16 ( "48:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 49f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 50f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -507,7 +506,7 @@ void a64_hybrid_fp32_mla_6x16 ( "b 50f\n" "49:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" "50:" // Height 2: input setup done "cmp x27, #0x4\n" "blt 53f\n" @@ -520,134 +519,134 @@ void a64_hybrid_fp32_mla_6x16 ( "51:" // Height 2: Multiply loop: Main loop head "fmla v8.4s, v6.4s, v0.s[0]\n" "fmla v12.4s, v6.4s, v1.s[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q17, [x10, #0x20]\n" "sub x27, x27, #0x4\n" "fmla v9.4s, v7.4s, v0.s[0]\n" "fmla v13.4s, v7.4s, v1.s[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q16, [x10, #0x30]\n" "add x26, x26, #0x10\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "ldr q6, [x10, #0x40]\n" + "fmla v10.4s, v17.4s, v0.s[0]\n" + "fmla v14.4s, v17.4s, v1.s[0]\n" + "ldr q17, [x10, #0x40]\n" "add x25, x25, #0x10\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "ldr q7, [x10, #0x50]\n" + "fmla v11.4s, v16.4s, v0.s[0]\n" + "fmla v15.4s, v16.4s, v1.s[0]\n" + "ldr q16, [x10, #0x50]\n" "cmp x27, #0x8\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v12.4s, v6.4s, v1.s[1]\n" - "ldr q6, [x10, #0x60]\n" + "fmla v8.4s, v17.4s, v0.s[1]\n" + "fmla v12.4s, v17.4s, v1.s[1]\n" + "ldr q17, [x10, #0x60]\n" "prfm pldl1keep, [x26, #0x80]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v13.4s, v7.4s, v1.s[1]\n" - "ldr q7, [x10, #0x70]\n" + "fmla v9.4s, v16.4s, v0.s[1]\n" + "fmla v13.4s, v16.4s, v1.s[1]\n" + "ldr q16, [x10, #0x70]\n" "prfm pldl1keep, [x25, #0x80]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "ldr q6, [x10, #0x80]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "ldr q7, [x10, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "fmla v12.4s, v6.4s, v1.s[2]\n" - "ldr q6, [x10, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "fmla v13.4s, v7.4s, v1.s[2]\n" - "ldr q7, [x10, #0xb0]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v14.4s, v6.4s, v1.s[2]\n" - "ldr q6, [x10, #0xc0]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "fmla v15.4s, v7.4s, v1.s[2]\n" - "ldr q7, [x10, #0xd0]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v12.4s, v6.4s, v1.s[3]\n" - "ldr q6, [x10, #0xe0]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "fmla v13.4s, v7.4s, v1.s[3]\n" - "ldr q7, [x10, #0xf0]\n" + "fmla v10.4s, v17.4s, v0.s[1]\n" + "fmla v14.4s, v17.4s, v1.s[1]\n" + "ldr q17, [x10, #0x80]\n" + "fmla v11.4s, v16.4s, v0.s[1]\n" + "fmla v15.4s, v16.4s, v1.s[1]\n" + "ldr q16, [x10, #0x90]\n" + "fmla v8.4s, v17.4s, v0.s[2]\n" + "fmla v12.4s, v17.4s, v1.s[2]\n" + "ldr q17, [x10, #0xa0]\n" + "fmla v9.4s, v16.4s, v0.s[2]\n" + "fmla v13.4s, v16.4s, v1.s[2]\n" + "ldr q16, [x10, #0xb0]\n" + "fmla v10.4s, v17.4s, v0.s[2]\n" + "fmla v14.4s, v17.4s, v1.s[2]\n" + "ldr q17, [x10, #0xc0]\n" + "fmla v11.4s, v16.4s, v0.s[2]\n" + "fmla v15.4s, v16.4s, v1.s[2]\n" + "ldr q16, [x10, #0xd0]\n" + "fmla v8.4s, v17.4s, v0.s[3]\n" + "fmla v12.4s, v17.4s, v1.s[3]\n" + "ldr q17, [x10, #0xe0]\n" + "fmla v9.4s, v16.4s, v0.s[3]\n" + "fmla v13.4s, v16.4s, v1.s[3]\n" + "ldr q16, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" + "fmla v10.4s, v17.4s, v0.s[3]\n" + "fmla v14.4s, v17.4s, v1.s[3]\n" "ldr q6, [x10, #0x0]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" + "fmla v11.4s, v16.4s, v0.s[3]\n" "ldr q0, [x26, #0x0]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" + "fmla v15.4s, v16.4s, v1.s[3]\n" "ldr q1, [x25, #0x0]\n" "ldr q7, [x10, #0x10]\n" "bge 51b\n" "52:" // Height 2: Multiply loop: Single iteration only "fmla v8.4s, v6.4s, v0.s[0]\n" "fmla v12.4s, v6.4s, v1.s[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q17, [x10, #0x20]\n" "add x26, x26, #0x10\n" "fmla v9.4s, v7.4s, v0.s[0]\n" "fmla v13.4s, v7.4s, v1.s[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q16, [x10, #0x30]\n" "add x25, x25, #0x10\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "ldr q6, [x10, #0x40]\n" + "fmla v10.4s, v17.4s, v0.s[0]\n" + "fmla v14.4s, v17.4s, v1.s[0]\n" + "ldr q17, [x10, #0x40]\n" "sub x27, x27, #0x4\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "ldr q7, [x10, #0x50]\n" + "fmla v11.4s, v16.4s, v0.s[0]\n" + "fmla v15.4s, v16.4s, v1.s[0]\n" + "ldr q16, [x10, #0x50]\n" "prfm pldl1keep, [x26, #0x80]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v12.4s, v6.4s, v1.s[1]\n" - "ldr q6, [x10, #0x60]\n" + "fmla v8.4s, v17.4s, v0.s[1]\n" + "fmla v12.4s, v17.4s, v1.s[1]\n" + "ldr q17, [x10, #0x60]\n" "prfm pldl1keep, [x25, #0x80]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v13.4s, v7.4s, v1.s[1]\n" - "ldr q7, [x10, #0x70]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "ldr q6, [x10, #0x80]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "ldr q7, [x10, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "fmla v12.4s, v6.4s, v1.s[2]\n" - "ldr q6, [x10, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "fmla v13.4s, v7.4s, v1.s[2]\n" - "ldr q7, [x10, #0xb0]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v14.4s, v6.4s, v1.s[2]\n" - "ldr q6, [x10, #0xc0]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "fmla v15.4s, v7.4s, v1.s[2]\n" - "ldr q7, [x10, #0xd0]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v12.4s, v6.4s, v1.s[3]\n" - "ldr q6, [x10, #0xe0]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "fmla v13.4s, v7.4s, v1.s[3]\n" - "ldr q7, [x10, #0xf0]\n" + "fmla v9.4s, v16.4s, v0.s[1]\n" + "fmla v13.4s, v16.4s, v1.s[1]\n" + "ldr q16, [x10, #0x70]\n" + "fmla v10.4s, v17.4s, v0.s[1]\n" + "fmla v14.4s, v17.4s, v1.s[1]\n" + "ldr q17, [x10, #0x80]\n" + "fmla v11.4s, v16.4s, v0.s[1]\n" + "fmla v15.4s, v16.4s, v1.s[1]\n" + "ldr q16, [x10, #0x90]\n" + "fmla v8.4s, v17.4s, v0.s[2]\n" + "fmla v12.4s, v17.4s, v1.s[2]\n" + "ldr q17, [x10, #0xa0]\n" + "fmla v9.4s, v16.4s, v0.s[2]\n" + "fmla v13.4s, v16.4s, v1.s[2]\n" + "ldr q16, [x10, #0xb0]\n" + "fmla v10.4s, v17.4s, v0.s[2]\n" + "fmla v14.4s, v17.4s, v1.s[2]\n" + "ldr q17, [x10, #0xc0]\n" + "fmla v11.4s, v16.4s, v0.s[2]\n" + "fmla v15.4s, v16.4s, v1.s[2]\n" + "ldr q16, [x10, #0xd0]\n" + "fmla v8.4s, v17.4s, v0.s[3]\n" + "fmla v12.4s, v17.4s, v1.s[3]\n" + "ldr q17, [x10, #0xe0]\n" + "fmla v9.4s, v16.4s, v0.s[3]\n" + "fmla v13.4s, v16.4s, v1.s[3]\n" + "ldr q16, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" + "fmla v10.4s, v17.4s, v0.s[3]\n" + "fmla v14.4s, v17.4s, v1.s[3]\n" + "fmla v11.4s, v16.4s, v0.s[3]\n" + "fmla v15.4s, v16.4s, v1.s[3]\n" "53:" // Height 2: Multiply loop: Main loop skip "cbz x27, 55f\n" "54:" // Height 2: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s19, [x26], #0x4\n" + "ldr s18, [x25], #0x4\n" "sub x27, x27, #0x1\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - "fmla v8.4s, v6.4s, v0.s[0]\n" - "fmla v12.4s, v6.4s, v1.s[0]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.4s, v7.4s, v0.s[0]\n" - "fmla v13.4s, v7.4s, v1.s[0]\n" - "ldr q7, [x10, #0x30]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" + "ldr q17, [x10, #0x0]\n" + "ldr q16, [x10, #0x10]\n" + "fmla v8.4s, v17.4s, v19.s[0]\n" + "fmla v12.4s, v17.4s, v18.s[0]\n" + "ldr q17, [x10, #0x20]\n" + "fmla v9.4s, v16.4s, v19.s[0]\n" + "fmla v13.4s, v16.4s, v18.s[0]\n" + "ldr q16, [x10, #0x30]\n" + "fmla v10.4s, v17.4s, v19.s[0]\n" + "fmla v14.4s, v17.4s, v18.s[0]\n" "add x10, x10, #0x40\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" + "fmla v11.4s, v16.4s, v19.s[0]\n" + "fmla v15.4s, v16.4s, v18.s[0]\n" "cbnz x27, 54b\n" "55:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -660,25 +659,25 @@ void a64_hybrid_fp32_mla_6x16 ( "prfm pstl1keep, [x25, #0x0]\n" "tbz %x[flags], #1, 56f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v17.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v15.4s, v15.4s, v1.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" + "ld1r { v16.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v17.4s\n" + "fmin v9.4s, v9.4s, v17.4s\n" + "fmin v10.4s, v10.4s, v17.4s\n" + "fmin v11.4s, v11.4s, v17.4s\n" + "fmin v12.4s, v12.4s, v17.4s\n" + "fmin v13.4s, v13.4s, v17.4s\n" + "fmin v14.4s, v14.4s, v17.4s\n" + "fmin v15.4s, v15.4s, v17.4s\n" + "fmax v8.4s, v8.4s, v16.4s\n" + "fmax v9.4s, v9.4s, v16.4s\n" + "fmax v10.4s, v10.4s, v16.4s\n" + "fmax v11.4s, v11.4s, v16.4s\n" + "fmax v12.4s, v12.4s, v16.4s\n" + "fmax v13.4s, v13.4s, v16.4s\n" + "fmax v14.4s, v14.4s, v16.4s\n" + "fmax v15.4s, v15.4s, v16.4s\n" "56:" // Height 2: No activation "cmp x11, #0x10\n" "bge 65f\n" @@ -905,13 +904,13 @@ void a64_hybrid_fp32_mla_6x16 ( "81:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 82f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 83f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -920,8 +919,8 @@ void a64_hybrid_fp32_mla_6x16 ( "b 83f\n" "82:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" "83:" // Height 3: input setup done "cmp x27, #0x4\n" "blt 86f\n" @@ -938,75 +937,75 @@ void a64_hybrid_fp32_mla_6x16 ( "sub x27, x27, #0x4\n" "add x26, x26, #0x10\n" "fmla v16.4s, v6.4s, v2.s[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q21, [x10, #0x20]\n" "fmla v9.4s, v7.4s, v0.s[0]\n" "add x25, x25, #0x10\n" "fmla v13.4s, v7.4s, v1.s[0]\n" "fmla v17.4s, v7.4s, v2.s[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q20, [x10, #0x30]\n" "add x24, x24, #0x10\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" + "fmla v10.4s, v21.4s, v0.s[0]\n" + "fmla v14.4s, v21.4s, v1.s[0]\n" "cmp x27, #0x8\n" "prfm pldl1keep, [x26, #0x80]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "ldr q6, [x10, #0x40]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" + "fmla v18.4s, v21.4s, v2.s[0]\n" + "ldr q21, [x10, #0x40]\n" + "fmla v11.4s, v20.4s, v0.s[0]\n" "prfm pldl1keep, [x25, #0x80]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "ldr q7, [x10, #0x50]\n" + "fmla v15.4s, v20.4s, v1.s[0]\n" + "fmla v19.4s, v20.4s, v2.s[0]\n" + "ldr q20, [x10, #0x50]\n" "prfm pldl1keep, [x24, #0x80]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v12.4s, v6.4s, v1.s[1]\n" - "fmla v16.4s, v6.4s, v2.s[1]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v13.4s, v7.4s, v1.s[1]\n" - "fmla v17.4s, v7.4s, v2.s[1]\n" - "ldr q7, [x10, #0x70]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "fmla v18.4s, v6.4s, v2.s[1]\n" - "ldr q6, [x10, #0x80]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "fmla v19.4s, v7.4s, v2.s[1]\n" - "ldr q7, [x10, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "fmla v12.4s, v6.4s, v1.s[2]\n" - "fmla v16.4s, v6.4s, v2.s[2]\n" - "ldr q6, [x10, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "fmla v13.4s, v7.4s, v1.s[2]\n" - "fmla v17.4s, v7.4s, v2.s[2]\n" - "ldr q7, [x10, #0xb0]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v14.4s, v6.4s, v1.s[2]\n" - "fmla v18.4s, v6.4s, v2.s[2]\n" - "ldr q6, [x10, #0xc0]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "fmla v15.4s, v7.4s, v1.s[2]\n" - "fmla v19.4s, v7.4s, v2.s[2]\n" - "ldr q7, [x10, #0xd0]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v12.4s, v6.4s, v1.s[3]\n" - "fmla v16.4s, v6.4s, v2.s[3]\n" - "ldr q6, [x10, #0xe0]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "fmla v13.4s, v7.4s, v1.s[3]\n" - "fmla v17.4s, v7.4s, v2.s[3]\n" - "ldr q7, [x10, #0xf0]\n" + "fmla v8.4s, v21.4s, v0.s[1]\n" + "fmla v12.4s, v21.4s, v1.s[1]\n" + "fmla v16.4s, v21.4s, v2.s[1]\n" + "ldr q21, [x10, #0x60]\n" + "fmla v9.4s, v20.4s, v0.s[1]\n" + "fmla v13.4s, v20.4s, v1.s[1]\n" + "fmla v17.4s, v20.4s, v2.s[1]\n" + "ldr q20, [x10, #0x70]\n" + "fmla v10.4s, v21.4s, v0.s[1]\n" + "fmla v14.4s, v21.4s, v1.s[1]\n" + "fmla v18.4s, v21.4s, v2.s[1]\n" + "ldr q21, [x10, #0x80]\n" + "fmla v11.4s, v20.4s, v0.s[1]\n" + "fmla v15.4s, v20.4s, v1.s[1]\n" + "fmla v19.4s, v20.4s, v2.s[1]\n" + "ldr q20, [x10, #0x90]\n" + "fmla v8.4s, v21.4s, v0.s[2]\n" + "fmla v12.4s, v21.4s, v1.s[2]\n" + "fmla v16.4s, v21.4s, v2.s[2]\n" + "ldr q21, [x10, #0xa0]\n" + "fmla v9.4s, v20.4s, v0.s[2]\n" + "fmla v13.4s, v20.4s, v1.s[2]\n" + "fmla v17.4s, v20.4s, v2.s[2]\n" + "ldr q20, [x10, #0xb0]\n" + "fmla v10.4s, v21.4s, v0.s[2]\n" + "fmla v14.4s, v21.4s, v1.s[2]\n" + "fmla v18.4s, v21.4s, v2.s[2]\n" + "ldr q21, [x10, #0xc0]\n" + "fmla v11.4s, v20.4s, v0.s[2]\n" + "fmla v15.4s, v20.4s, v1.s[2]\n" + "fmla v19.4s, v20.4s, v2.s[2]\n" + "ldr q20, [x10, #0xd0]\n" + "fmla v8.4s, v21.4s, v0.s[3]\n" + "fmla v12.4s, v21.4s, v1.s[3]\n" + "fmla v16.4s, v21.4s, v2.s[3]\n" + "ldr q21, [x10, #0xe0]\n" + "fmla v9.4s, v20.4s, v0.s[3]\n" + "fmla v13.4s, v20.4s, v1.s[3]\n" + "fmla v17.4s, v20.4s, v2.s[3]\n" + "ldr q20, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "fmla v18.4s, v6.4s, v2.s[3]\n" + "fmla v10.4s, v21.4s, v0.s[3]\n" + "fmla v14.4s, v21.4s, v1.s[3]\n" + "fmla v18.4s, v21.4s, v2.s[3]\n" "ldr q6, [x10, #0x0]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" + "fmla v11.4s, v20.4s, v0.s[3]\n" "ldr q0, [x26, #0x0]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" + "fmla v15.4s, v20.4s, v1.s[3]\n" "ldr q1, [x25, #0x0]\n" - "fmla v19.4s, v7.4s, v2.s[3]\n" + "fmla v19.4s, v20.4s, v2.s[3]\n" "ldr q2, [x24, #0x0]\n" "ldr q7, [x10, #0x10]\n" "bge 84b\n" @@ -1016,95 +1015,95 @@ void a64_hybrid_fp32_mla_6x16 ( "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" "fmla v16.4s, v6.4s, v2.s[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q21, [x10, #0x20]\n" "fmla v9.4s, v7.4s, v0.s[0]\n" "add x24, x24, #0x10\n" "fmla v13.4s, v7.4s, v1.s[0]\n" "fmla v17.4s, v7.4s, v2.s[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q20, [x10, #0x30]\n" "sub x27, x27, #0x4\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" + "fmla v10.4s, v21.4s, v0.s[0]\n" + "fmla v14.4s, v21.4s, v1.s[0]\n" "prfm pldl1keep, [x26, #0x80]\n" "prfm pldl1keep, [x25, #0x80]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "ldr q6, [x10, #0x40]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" + "fmla v18.4s, v21.4s, v2.s[0]\n" + "ldr q21, [x10, #0x40]\n" + "fmla v11.4s, v20.4s, v0.s[0]\n" "prfm pldl1keep, [x24, #0x80]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "ldr q7, [x10, #0x50]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v12.4s, v6.4s, v1.s[1]\n" - "fmla v16.4s, v6.4s, v2.s[1]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v13.4s, v7.4s, v1.s[1]\n" - "fmla v17.4s, v7.4s, v2.s[1]\n" - "ldr q7, [x10, #0x70]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "fmla v18.4s, v6.4s, v2.s[1]\n" - "ldr q6, [x10, #0x80]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "fmla v19.4s, v7.4s, v2.s[1]\n" - "ldr q7, [x10, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "fmla v12.4s, v6.4s, v1.s[2]\n" - "fmla v16.4s, v6.4s, v2.s[2]\n" - "ldr q6, [x10, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "fmla v13.4s, v7.4s, v1.s[2]\n" - "fmla v17.4s, v7.4s, v2.s[2]\n" - "ldr q7, [x10, #0xb0]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v14.4s, v6.4s, v1.s[2]\n" - "fmla v18.4s, v6.4s, v2.s[2]\n" - "ldr q6, [x10, #0xc0]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "fmla v15.4s, v7.4s, v1.s[2]\n" - "fmla v19.4s, v7.4s, v2.s[2]\n" - "ldr q7, [x10, #0xd0]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v12.4s, v6.4s, v1.s[3]\n" - "fmla v16.4s, v6.4s, v2.s[3]\n" - "ldr q6, [x10, #0xe0]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "fmla v13.4s, v7.4s, v1.s[3]\n" - "fmla v17.4s, v7.4s, v2.s[3]\n" - "ldr q7, [x10, #0xf0]\n" + "fmla v15.4s, v20.4s, v1.s[0]\n" + "fmla v19.4s, v20.4s, v2.s[0]\n" + "ldr q20, [x10, #0x50]\n" + "fmla v8.4s, v21.4s, v0.s[1]\n" + "fmla v12.4s, v21.4s, v1.s[1]\n" + "fmla v16.4s, v21.4s, v2.s[1]\n" + "ldr q21, [x10, #0x60]\n" + "fmla v9.4s, v20.4s, v0.s[1]\n" + "fmla v13.4s, v20.4s, v1.s[1]\n" + "fmla v17.4s, v20.4s, v2.s[1]\n" + "ldr q20, [x10, #0x70]\n" + "fmla v10.4s, v21.4s, v0.s[1]\n" + "fmla v14.4s, v21.4s, v1.s[1]\n" + "fmla v18.4s, v21.4s, v2.s[1]\n" + "ldr q21, [x10, #0x80]\n" + "fmla v11.4s, v20.4s, v0.s[1]\n" + "fmla v15.4s, v20.4s, v1.s[1]\n" + "fmla v19.4s, v20.4s, v2.s[1]\n" + "ldr q20, [x10, #0x90]\n" + "fmla v8.4s, v21.4s, v0.s[2]\n" + "fmla v12.4s, v21.4s, v1.s[2]\n" + "fmla v16.4s, v21.4s, v2.s[2]\n" + "ldr q21, [x10, #0xa0]\n" + "fmla v9.4s, v20.4s, v0.s[2]\n" + "fmla v13.4s, v20.4s, v1.s[2]\n" + "fmla v17.4s, v20.4s, v2.s[2]\n" + "ldr q20, [x10, #0xb0]\n" + "fmla v10.4s, v21.4s, v0.s[2]\n" + "fmla v14.4s, v21.4s, v1.s[2]\n" + "fmla v18.4s, v21.4s, v2.s[2]\n" + "ldr q21, [x10, #0xc0]\n" + "fmla v11.4s, v20.4s, v0.s[2]\n" + "fmla v15.4s, v20.4s, v1.s[2]\n" + "fmla v19.4s, v20.4s, v2.s[2]\n" + "ldr q20, [x10, #0xd0]\n" + "fmla v8.4s, v21.4s, v0.s[3]\n" + "fmla v12.4s, v21.4s, v1.s[3]\n" + "fmla v16.4s, v21.4s, v2.s[3]\n" + "ldr q21, [x10, #0xe0]\n" + "fmla v9.4s, v20.4s, v0.s[3]\n" + "fmla v13.4s, v20.4s, v1.s[3]\n" + "fmla v17.4s, v20.4s, v2.s[3]\n" + "ldr q20, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "fmla v18.4s, v6.4s, v2.s[3]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" - "fmla v19.4s, v7.4s, v2.s[3]\n" + "fmla v10.4s, v21.4s, v0.s[3]\n" + "fmla v14.4s, v21.4s, v1.s[3]\n" + "fmla v18.4s, v21.4s, v2.s[3]\n" + "fmla v11.4s, v20.4s, v0.s[3]\n" + "fmla v15.4s, v20.4s, v1.s[3]\n" + "fmla v19.4s, v20.4s, v2.s[3]\n" "86:" // Height 3: Multiply loop: Main loop skip "cbz x27, 88f\n" "87:" // Height 3: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s24, [x26], #0x4\n" + "ldr s23, [x25], #0x4\n" "sub x27, x27, #0x1\n" - "ldr s2, [x24], #0x4\n" - "ldr q6, [x10, #0x0]\n" - "fmla v8.4s, v6.4s, v0.s[0]\n" - "fmla v12.4s, v6.4s, v1.s[0]\n" - "ldr q7, [x10, #0x10]\n" - "fmla v16.4s, v6.4s, v2.s[0]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.4s, v7.4s, v0.s[0]\n" - "fmla v13.4s, v7.4s, v1.s[0]\n" - "fmla v17.4s, v7.4s, v2.s[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr s22, [x24], #0x4\n" + "ldr q21, [x10, #0x0]\n" + "fmla v8.4s, v21.4s, v24.s[0]\n" + "fmla v12.4s, v21.4s, v23.s[0]\n" + "ldr q20, [x10, #0x10]\n" + "fmla v16.4s, v21.4s, v22.s[0]\n" + "ldr q21, [x10, #0x20]\n" + "fmla v9.4s, v20.4s, v24.s[0]\n" + "fmla v13.4s, v20.4s, v23.s[0]\n" + "fmla v17.4s, v20.4s, v22.s[0]\n" + "ldr q20, [x10, #0x30]\n" "add x10, x10, #0x40\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" + "fmla v10.4s, v21.4s, v24.s[0]\n" + "fmla v14.4s, v21.4s, v23.s[0]\n" + "fmla v18.4s, v21.4s, v22.s[0]\n" + "fmla v11.4s, v20.4s, v24.s[0]\n" + "fmla v15.4s, v20.4s, v23.s[0]\n" + "fmla v19.4s, v20.4s, v22.s[0]\n" "cbnz x27, 87b\n" "88:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -1119,33 +1118,33 @@ void a64_hybrid_fp32_mla_6x16 ( "prfm pstl1keep, [x24, #0x0]\n" "tbz %x[flags], #1, 89f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v21.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v15.4s, v15.4s, v1.4s\n" - "fmin v16.4s, v16.4s, v1.4s\n" - "fmin v17.4s, v17.4s, v1.4s\n" - "fmin v18.4s, v18.4s, v1.4s\n" - "fmin v19.4s, v19.4s, v1.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" - "fmax v16.4s, v16.4s, v0.4s\n" - "fmax v17.4s, v17.4s, v0.4s\n" - "fmax v18.4s, v18.4s, v0.4s\n" - "fmax v19.4s, v19.4s, v0.4s\n" + "ld1r { v20.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v21.4s\n" + "fmin v9.4s, v9.4s, v21.4s\n" + "fmin v10.4s, v10.4s, v21.4s\n" + "fmin v11.4s, v11.4s, v21.4s\n" + "fmin v12.4s, v12.4s, v21.4s\n" + "fmin v13.4s, v13.4s, v21.4s\n" + "fmin v14.4s, v14.4s, v21.4s\n" + "fmin v15.4s, v15.4s, v21.4s\n" + "fmin v16.4s, v16.4s, v21.4s\n" + "fmin v17.4s, v17.4s, v21.4s\n" + "fmin v18.4s, v18.4s, v21.4s\n" + "fmin v19.4s, v19.4s, v21.4s\n" + "fmax v8.4s, v8.4s, v20.4s\n" + "fmax v9.4s, v9.4s, v20.4s\n" + "fmax v10.4s, v10.4s, v20.4s\n" + "fmax v11.4s, v11.4s, v20.4s\n" + "fmax v12.4s, v12.4s, v20.4s\n" + "fmax v13.4s, v13.4s, v20.4s\n" + "fmax v14.4s, v14.4s, v20.4s\n" + "fmax v15.4s, v15.4s, v20.4s\n" + "fmax v16.4s, v16.4s, v20.4s\n" + "fmax v17.4s, v17.4s, v20.4s\n" + "fmax v18.4s, v18.4s, v20.4s\n" + "fmax v19.4s, v19.4s, v20.4s\n" "89:" // Height 3: No activation "cmp x11, #0x10\n" "bge 98f\n" @@ -1421,14 +1420,14 @@ void a64_hybrid_fp32_mla_6x16 ( "114:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 115f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 116f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -1438,9 +1437,9 @@ void a64_hybrid_fp32_mla_6x16 ( "b 116f\n" "115:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" "116:" // Height 4: input setup done "cmp x27, #0x4\n" "blt 119f\n" @@ -1459,7 +1458,7 @@ void a64_hybrid_fp32_mla_6x16 ( "add x26, x26, #0x10\n" "fmla v16.4s, v6.4s, v2.s[0]\n" "fmla v20.4s, v6.4s, v3.s[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q25, [x10, #0x20]\n" "add x25, x25, #0x10\n" "fmla v9.4s, v7.4s, v0.s[0]\n" "fmla v13.4s, v7.4s, v1.s[0]\n" @@ -1467,85 +1466,85 @@ void a64_hybrid_fp32_mla_6x16 ( "add x23, x23, #0x10\n" "fmla v17.4s, v7.4s, v2.s[0]\n" "fmla v21.4s, v7.4s, v3.s[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q24, [x10, #0x30]\n" "cmp x27, #0x8\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" + "fmla v10.4s, v25.4s, v0.s[0]\n" + "fmla v14.4s, v25.4s, v1.s[0]\n" "prfm pldl1keep, [x26, #0x80]\n" "prfm pldl1keep, [x25, #0x80]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "fmla v22.4s, v6.4s, v3.s[0]\n" - "ldr q6, [x10, #0x40]\n" + "fmla v18.4s, v25.4s, v2.s[0]\n" + "fmla v22.4s, v25.4s, v3.s[0]\n" + "ldr q25, [x10, #0x40]\n" "prfm pldl1keep, [x24, #0x80]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" + "fmla v11.4s, v24.4s, v0.s[0]\n" + "fmla v15.4s, v24.4s, v1.s[0]\n" "prfm pldl1keep, [x23, #0x80]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "fmla v23.4s, v7.4s, v3.s[0]\n" - "ldr q7, [x10, #0x50]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v12.4s, v6.4s, v1.s[1]\n" - "fmla v16.4s, v6.4s, v2.s[1]\n" - "fmla v20.4s, v6.4s, v3.s[1]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v13.4s, v7.4s, v1.s[1]\n" - "fmla v17.4s, v7.4s, v2.s[1]\n" - "fmla v21.4s, v7.4s, v3.s[1]\n" - "ldr q7, [x10, #0x70]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "fmla v18.4s, v6.4s, v2.s[1]\n" - "fmla v22.4s, v6.4s, v3.s[1]\n" - "ldr q6, [x10, #0x80]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "fmla v19.4s, v7.4s, v2.s[1]\n" - "fmla v23.4s, v7.4s, v3.s[1]\n" - "ldr q7, [x10, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "fmla v12.4s, v6.4s, v1.s[2]\n" - "fmla v16.4s, v6.4s, v2.s[2]\n" - "fmla v20.4s, v6.4s, v3.s[2]\n" - "ldr q6, [x10, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "fmla v13.4s, v7.4s, v1.s[2]\n" - "fmla v17.4s, v7.4s, v2.s[2]\n" - "fmla v21.4s, v7.4s, v3.s[2]\n" - "ldr q7, [x10, #0xb0]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v14.4s, v6.4s, v1.s[2]\n" - "fmla v18.4s, v6.4s, v2.s[2]\n" - "fmla v22.4s, v6.4s, v3.s[2]\n" - "ldr q6, [x10, #0xc0]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "fmla v15.4s, v7.4s, v1.s[2]\n" - "fmla v19.4s, v7.4s, v2.s[2]\n" - "fmla v23.4s, v7.4s, v3.s[2]\n" - "ldr q7, [x10, #0xd0]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v12.4s, v6.4s, v1.s[3]\n" - "fmla v16.4s, v6.4s, v2.s[3]\n" - "fmla v20.4s, v6.4s, v3.s[3]\n" - "ldr q6, [x10, #0xe0]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "fmla v13.4s, v7.4s, v1.s[3]\n" - "fmla v17.4s, v7.4s, v2.s[3]\n" - "fmla v21.4s, v7.4s, v3.s[3]\n" - "ldr q7, [x10, #0xf0]\n" + "fmla v19.4s, v24.4s, v2.s[0]\n" + "fmla v23.4s, v24.4s, v3.s[0]\n" + "ldr q24, [x10, #0x50]\n" + "fmla v8.4s, v25.4s, v0.s[1]\n" + "fmla v12.4s, v25.4s, v1.s[1]\n" + "fmla v16.4s, v25.4s, v2.s[1]\n" + "fmla v20.4s, v25.4s, v3.s[1]\n" + "ldr q25, [x10, #0x60]\n" + "fmla v9.4s, v24.4s, v0.s[1]\n" + "fmla v13.4s, v24.4s, v1.s[1]\n" + "fmla v17.4s, v24.4s, v2.s[1]\n" + "fmla v21.4s, v24.4s, v3.s[1]\n" + "ldr q24, [x10, #0x70]\n" + "fmla v10.4s, v25.4s, v0.s[1]\n" + "fmla v14.4s, v25.4s, v1.s[1]\n" + "fmla v18.4s, v25.4s, v2.s[1]\n" + "fmla v22.4s, v25.4s, v3.s[1]\n" + "ldr q25, [x10, #0x80]\n" + "fmla v11.4s, v24.4s, v0.s[1]\n" + "fmla v15.4s, v24.4s, v1.s[1]\n" + "fmla v19.4s, v24.4s, v2.s[1]\n" + "fmla v23.4s, v24.4s, v3.s[1]\n" + "ldr q24, [x10, #0x90]\n" + "fmla v8.4s, v25.4s, v0.s[2]\n" + "fmla v12.4s, v25.4s, v1.s[2]\n" + "fmla v16.4s, v25.4s, v2.s[2]\n" + "fmla v20.4s, v25.4s, v3.s[2]\n" + "ldr q25, [x10, #0xa0]\n" + "fmla v9.4s, v24.4s, v0.s[2]\n" + "fmla v13.4s, v24.4s, v1.s[2]\n" + "fmla v17.4s, v24.4s, v2.s[2]\n" + "fmla v21.4s, v24.4s, v3.s[2]\n" + "ldr q24, [x10, #0xb0]\n" + "fmla v10.4s, v25.4s, v0.s[2]\n" + "fmla v14.4s, v25.4s, v1.s[2]\n" + "fmla v18.4s, v25.4s, v2.s[2]\n" + "fmla v22.4s, v25.4s, v3.s[2]\n" + "ldr q25, [x10, #0xc0]\n" + "fmla v11.4s, v24.4s, v0.s[2]\n" + "fmla v15.4s, v24.4s, v1.s[2]\n" + "fmla v19.4s, v24.4s, v2.s[2]\n" + "fmla v23.4s, v24.4s, v3.s[2]\n" + "ldr q24, [x10, #0xd0]\n" + "fmla v8.4s, v25.4s, v0.s[3]\n" + "fmla v12.4s, v25.4s, v1.s[3]\n" + "fmla v16.4s, v25.4s, v2.s[3]\n" + "fmla v20.4s, v25.4s, v3.s[3]\n" + "ldr q25, [x10, #0xe0]\n" + "fmla v9.4s, v24.4s, v0.s[3]\n" + "fmla v13.4s, v24.4s, v1.s[3]\n" + "fmla v17.4s, v24.4s, v2.s[3]\n" + "fmla v21.4s, v24.4s, v3.s[3]\n" + "ldr q24, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "fmla v18.4s, v6.4s, v2.s[3]\n" - "fmla v22.4s, v6.4s, v3.s[3]\n" + "fmla v10.4s, v25.4s, v0.s[3]\n" + "fmla v14.4s, v25.4s, v1.s[3]\n" + "fmla v18.4s, v25.4s, v2.s[3]\n" + "fmla v22.4s, v25.4s, v3.s[3]\n" "ldr q6, [x10, #0x0]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" + "fmla v11.4s, v24.4s, v0.s[3]\n" "ldr q0, [x26, #0x0]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" + "fmla v15.4s, v24.4s, v1.s[3]\n" "ldr q1, [x25, #0x0]\n" - "fmla v19.4s, v7.4s, v2.s[3]\n" + "fmla v19.4s, v24.4s, v2.s[3]\n" "ldr q2, [x24, #0x0]\n" - "fmla v23.4s, v7.4s, v3.s[3]\n" + "fmla v23.4s, v24.4s, v3.s[3]\n" "ldr q3, [x23, #0x0]\n" "ldr q7, [x10, #0x10]\n" "bge 117b\n" @@ -1556,7 +1555,7 @@ void a64_hybrid_fp32_mla_6x16 ( "add x25, x25, #0x10\n" "fmla v16.4s, v6.4s, v2.s[0]\n" "fmla v20.4s, v6.4s, v3.s[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q25, [x10, #0x20]\n" "add x24, x24, #0x10\n" "fmla v9.4s, v7.4s, v0.s[0]\n" "fmla v13.4s, v7.4s, v1.s[0]\n" @@ -1564,109 +1563,109 @@ void a64_hybrid_fp32_mla_6x16 ( "sub x27, x27, #0x4\n" "fmla v17.4s, v7.4s, v2.s[0]\n" "fmla v21.4s, v7.4s, v3.s[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q24, [x10, #0x30]\n" "prfm pldl1keep, [x26, #0x80]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" + "fmla v10.4s, v25.4s, v0.s[0]\n" + "fmla v14.4s, v25.4s, v1.s[0]\n" "prfm pldl1keep, [x25, #0x80]\n" "prfm pldl1keep, [x24, #0x80]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "fmla v22.4s, v6.4s, v3.s[0]\n" - "ldr q6, [x10, #0x40]\n" + "fmla v18.4s, v25.4s, v2.s[0]\n" + "fmla v22.4s, v25.4s, v3.s[0]\n" + "ldr q25, [x10, #0x40]\n" "prfm pldl1keep, [x23, #0x80]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "fmla v23.4s, v7.4s, v3.s[0]\n" - "ldr q7, [x10, #0x50]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v12.4s, v6.4s, v1.s[1]\n" - "fmla v16.4s, v6.4s, v2.s[1]\n" - "fmla v20.4s, v6.4s, v3.s[1]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v13.4s, v7.4s, v1.s[1]\n" - "fmla v17.4s, v7.4s, v2.s[1]\n" - "fmla v21.4s, v7.4s, v3.s[1]\n" - "ldr q7, [x10, #0x70]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "fmla v18.4s, v6.4s, v2.s[1]\n" - "fmla v22.4s, v6.4s, v3.s[1]\n" - "ldr q6, [x10, #0x80]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "fmla v19.4s, v7.4s, v2.s[1]\n" - "fmla v23.4s, v7.4s, v3.s[1]\n" - "ldr q7, [x10, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "fmla v12.4s, v6.4s, v1.s[2]\n" - "fmla v16.4s, v6.4s, v2.s[2]\n" - "fmla v20.4s, v6.4s, v3.s[2]\n" - "ldr q6, [x10, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "fmla v13.4s, v7.4s, v1.s[2]\n" - "fmla v17.4s, v7.4s, v2.s[2]\n" - "fmla v21.4s, v7.4s, v3.s[2]\n" - "ldr q7, [x10, #0xb0]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v14.4s, v6.4s, v1.s[2]\n" - "fmla v18.4s, v6.4s, v2.s[2]\n" - "fmla v22.4s, v6.4s, v3.s[2]\n" - "ldr q6, [x10, #0xc0]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "fmla v15.4s, v7.4s, v1.s[2]\n" - "fmla v19.4s, v7.4s, v2.s[2]\n" - "fmla v23.4s, v7.4s, v3.s[2]\n" - "ldr q7, [x10, #0xd0]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v12.4s, v6.4s, v1.s[3]\n" - "fmla v16.4s, v6.4s, v2.s[3]\n" - "fmla v20.4s, v6.4s, v3.s[3]\n" - "ldr q6, [x10, #0xe0]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "fmla v13.4s, v7.4s, v1.s[3]\n" - "fmla v17.4s, v7.4s, v2.s[3]\n" - "fmla v21.4s, v7.4s, v3.s[3]\n" - "ldr q7, [x10, #0xf0]\n" + "fmla v11.4s, v24.4s, v0.s[0]\n" + "fmla v15.4s, v24.4s, v1.s[0]\n" + "fmla v19.4s, v24.4s, v2.s[0]\n" + "fmla v23.4s, v24.4s, v3.s[0]\n" + "ldr q24, [x10, #0x50]\n" + "fmla v8.4s, v25.4s, v0.s[1]\n" + "fmla v12.4s, v25.4s, v1.s[1]\n" + "fmla v16.4s, v25.4s, v2.s[1]\n" + "fmla v20.4s, v25.4s, v3.s[1]\n" + "ldr q25, [x10, #0x60]\n" + "fmla v9.4s, v24.4s, v0.s[1]\n" + "fmla v13.4s, v24.4s, v1.s[1]\n" + "fmla v17.4s, v24.4s, v2.s[1]\n" + "fmla v21.4s, v24.4s, v3.s[1]\n" + "ldr q24, [x10, #0x70]\n" + "fmla v10.4s, v25.4s, v0.s[1]\n" + "fmla v14.4s, v25.4s, v1.s[1]\n" + "fmla v18.4s, v25.4s, v2.s[1]\n" + "fmla v22.4s, v25.4s, v3.s[1]\n" + "ldr q25, [x10, #0x80]\n" + "fmla v11.4s, v24.4s, v0.s[1]\n" + "fmla v15.4s, v24.4s, v1.s[1]\n" + "fmla v19.4s, v24.4s, v2.s[1]\n" + "fmla v23.4s, v24.4s, v3.s[1]\n" + "ldr q24, [x10, #0x90]\n" + "fmla v8.4s, v25.4s, v0.s[2]\n" + "fmla v12.4s, v25.4s, v1.s[2]\n" + "fmla v16.4s, v25.4s, v2.s[2]\n" + "fmla v20.4s, v25.4s, v3.s[2]\n" + "ldr q25, [x10, #0xa0]\n" + "fmla v9.4s, v24.4s, v0.s[2]\n" + "fmla v13.4s, v24.4s, v1.s[2]\n" + "fmla v17.4s, v24.4s, v2.s[2]\n" + "fmla v21.4s, v24.4s, v3.s[2]\n" + "ldr q24, [x10, #0xb0]\n" + "fmla v10.4s, v25.4s, v0.s[2]\n" + "fmla v14.4s, v25.4s, v1.s[2]\n" + "fmla v18.4s, v25.4s, v2.s[2]\n" + "fmla v22.4s, v25.4s, v3.s[2]\n" + "ldr q25, [x10, #0xc0]\n" + "fmla v11.4s, v24.4s, v0.s[2]\n" + "fmla v15.4s, v24.4s, v1.s[2]\n" + "fmla v19.4s, v24.4s, v2.s[2]\n" + "fmla v23.4s, v24.4s, v3.s[2]\n" + "ldr q24, [x10, #0xd0]\n" + "fmla v8.4s, v25.4s, v0.s[3]\n" + "fmla v12.4s, v25.4s, v1.s[3]\n" + "fmla v16.4s, v25.4s, v2.s[3]\n" + "fmla v20.4s, v25.4s, v3.s[3]\n" + "ldr q25, [x10, #0xe0]\n" + "fmla v9.4s, v24.4s, v0.s[3]\n" + "fmla v13.4s, v24.4s, v1.s[3]\n" + "fmla v17.4s, v24.4s, v2.s[3]\n" + "fmla v21.4s, v24.4s, v3.s[3]\n" + "ldr q24, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "fmla v18.4s, v6.4s, v2.s[3]\n" - "fmla v22.4s, v6.4s, v3.s[3]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" - "fmla v19.4s, v7.4s, v2.s[3]\n" - "fmla v23.4s, v7.4s, v3.s[3]\n" + "fmla v10.4s, v25.4s, v0.s[3]\n" + "fmla v14.4s, v25.4s, v1.s[3]\n" + "fmla v18.4s, v25.4s, v2.s[3]\n" + "fmla v22.4s, v25.4s, v3.s[3]\n" + "fmla v11.4s, v24.4s, v0.s[3]\n" + "fmla v15.4s, v24.4s, v1.s[3]\n" + "fmla v19.4s, v24.4s, v2.s[3]\n" + "fmla v23.4s, v24.4s, v3.s[3]\n" "119:" // Height 4: Multiply loop: Main loop skip "cbz x27, 121f\n" "120:" // Height 4: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s29, [x26], #0x4\n" + "ldr s28, [x25], #0x4\n" "sub x27, x27, #0x1\n" - "ldr s2, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - "fmla v8.4s, v6.4s, v0.s[0]\n" - "fmla v12.4s, v6.4s, v1.s[0]\n" - "fmla v16.4s, v6.4s, v2.s[0]\n" - "fmla v20.4s, v6.4s, v3.s[0]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.4s, v7.4s, v0.s[0]\n" - "fmla v13.4s, v7.4s, v1.s[0]\n" - "fmla v17.4s, v7.4s, v2.s[0]\n" - "fmla v21.4s, v7.4s, v3.s[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr s27, [x24], #0x4\n" + "ldr s26, [x23], #0x4\n" + "ldr q25, [x10, #0x0]\n" + "ldr q24, [x10, #0x10]\n" + "fmla v8.4s, v25.4s, v29.s[0]\n" + "fmla v12.4s, v25.4s, v28.s[0]\n" + "fmla v16.4s, v25.4s, v27.s[0]\n" + "fmla v20.4s, v25.4s, v26.s[0]\n" + "ldr q25, [x10, #0x20]\n" + "fmla v9.4s, v24.4s, v29.s[0]\n" + "fmla v13.4s, v24.4s, v28.s[0]\n" + "fmla v17.4s, v24.4s, v27.s[0]\n" + "fmla v21.4s, v24.4s, v26.s[0]\n" + "ldr q24, [x10, #0x30]\n" "add x10, x10, #0x40\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "fmla v22.4s, v6.4s, v3.s[0]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "fmla v23.4s, v7.4s, v3.s[0]\n" + "fmla v10.4s, v25.4s, v29.s[0]\n" + "fmla v14.4s, v25.4s, v28.s[0]\n" + "fmla v18.4s, v25.4s, v27.s[0]\n" + "fmla v22.4s, v25.4s, v26.s[0]\n" + "fmla v11.4s, v24.4s, v29.s[0]\n" + "fmla v15.4s, v24.4s, v28.s[0]\n" + "fmla v19.4s, v24.4s, v27.s[0]\n" + "fmla v23.4s, v24.4s, v26.s[0]\n" "cbnz x27, 120b\n" "121:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -1683,41 +1682,41 @@ void a64_hybrid_fp32_mla_6x16 ( "prfm pstl1keep, [x23, #0x0]\n" "tbz %x[flags], #1, 122f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v25.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v15.4s, v15.4s, v1.4s\n" - "fmin v16.4s, v16.4s, v1.4s\n" - "fmin v17.4s, v17.4s, v1.4s\n" - "fmin v18.4s, v18.4s, v1.4s\n" - "fmin v19.4s, v19.4s, v1.4s\n" - "fmin v20.4s, v20.4s, v1.4s\n" - "fmin v21.4s, v21.4s, v1.4s\n" - "fmin v22.4s, v22.4s, v1.4s\n" - "fmin v23.4s, v23.4s, v1.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" - "fmax v16.4s, v16.4s, v0.4s\n" - "fmax v17.4s, v17.4s, v0.4s\n" - "fmax v18.4s, v18.4s, v0.4s\n" - "fmax v19.4s, v19.4s, v0.4s\n" - "fmax v20.4s, v20.4s, v0.4s\n" - "fmax v21.4s, v21.4s, v0.4s\n" - "fmax v22.4s, v22.4s, v0.4s\n" - "fmax v23.4s, v23.4s, v0.4s\n" + "ld1r { v24.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v25.4s\n" + "fmin v9.4s, v9.4s, v25.4s\n" + "fmin v10.4s, v10.4s, v25.4s\n" + "fmin v11.4s, v11.4s, v25.4s\n" + "fmin v12.4s, v12.4s, v25.4s\n" + "fmin v13.4s, v13.4s, v25.4s\n" + "fmin v14.4s, v14.4s, v25.4s\n" + "fmin v15.4s, v15.4s, v25.4s\n" + "fmin v16.4s, v16.4s, v25.4s\n" + "fmin v17.4s, v17.4s, v25.4s\n" + "fmin v18.4s, v18.4s, v25.4s\n" + "fmin v19.4s, v19.4s, v25.4s\n" + "fmin v20.4s, v20.4s, v25.4s\n" + "fmin v21.4s, v21.4s, v25.4s\n" + "fmin v22.4s, v22.4s, v25.4s\n" + "fmin v23.4s, v23.4s, v25.4s\n" + "fmax v8.4s, v8.4s, v24.4s\n" + "fmax v9.4s, v9.4s, v24.4s\n" + "fmax v10.4s, v10.4s, v24.4s\n" + "fmax v11.4s, v11.4s, v24.4s\n" + "fmax v12.4s, v12.4s, v24.4s\n" + "fmax v13.4s, v13.4s, v24.4s\n" + "fmax v14.4s, v14.4s, v24.4s\n" + "fmax v15.4s, v15.4s, v24.4s\n" + "fmax v16.4s, v16.4s, v24.4s\n" + "fmax v17.4s, v17.4s, v24.4s\n" + "fmax v18.4s, v18.4s, v24.4s\n" + "fmax v19.4s, v19.4s, v24.4s\n" + "fmax v20.4s, v20.4s, v24.4s\n" + "fmax v21.4s, v21.4s, v24.4s\n" + "fmax v22.4s, v22.4s, v24.4s\n" + "fmax v23.4s, v23.4s, v24.4s\n" "122:" // Height 4: No activation "cmp x11, #0x10\n" "bge 131f\n" @@ -2028,168 +2027,168 @@ void a64_hybrid_fp32_mla_6x16 ( "movi v16.16b, #0x0\n" "movi v17.16b, #0x0\n" "movi v18.16b, #0x0\n" - "movi v19.16b, #0x0\n" - "movi v20.16b, #0x0\n" - "movi v21.16b, #0x0\n" - "movi v22.16b, #0x0\n" - "movi v23.16b, #0x0\n" - "movi v24.16b, #0x0\n" - "movi v25.16b, #0x0\n" - "movi v26.16b, #0x0\n" - "movi v27.16b, #0x0\n" - "146:" // Height 5: setup done - "mov x28, #0x0\n" - "147:" // Height 5: String loop - "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" - "tbz %x[flags], #3, 148f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "cbnz x28, 149f\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x26, x26, x20, LSL #2\n" - "add x25, x25, x20, LSL #2\n" - "add x24, x24, x20, LSL #2\n" - "add x23, x23, x20, LSL #2\n" - "add x22, x22, x20, LSL #2\n" - "b 149f\n" - "148:" // Height 5: setup direct input - "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "149:" // Height 5: input setup done - "cmp x27, #0x4\n" - "blt 152f\n" - "ldr q0, [x26, #0x0]\n" - "ldr q1, [x25, #0x0]\n" - "cmp x27, #0x8\n" - "ldr q2, [x24, #0x0]\n" - "ldr q3, [x23, #0x0]\n" - "ldr q4, [x22, #0x0]\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - "blt 151f\n" - "150:" // Height 5: Multiply loop: Main loop head - "fmla v8.4s, v6.4s, v0.s[0]\n" - "fmla v12.4s, v6.4s, v1.s[0]\n" - "sub x27, x27, #0x4\n" - "add x26, x26, #0x10\n" - "fmla v16.4s, v6.4s, v2.s[0]\n" - "fmla v20.4s, v6.4s, v3.s[0]\n" - "add x25, x25, #0x10\n" - "add x24, x24, #0x10\n" - "fmla v24.4s, v6.4s, v4.s[0]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.4s, v7.4s, v0.s[0]\n" - "add x23, x23, #0x10\n" - "fmla v13.4s, v7.4s, v1.s[0]\n" - "fmla v17.4s, v7.4s, v2.s[0]\n" - "add x22, x22, #0x10\n" - "cmp x27, #0x8\n" - "fmla v21.4s, v7.4s, v3.s[0]\n" - "fmla v25.4s, v7.4s, v4.s[0]\n" - "ldr q7, [x10, #0x30]\n" - "prfm pldl1keep, [x26, #0x80]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "prfm pldl1keep, [x25, #0x80]\n" - "prfm pldl1keep, [x24, #0x80]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "fmla v22.4s, v6.4s, v3.s[0]\n" - "prfm pldl1keep, [x23, #0x80]\n" - "prfm pldl1keep, [x22, #0x80]\n" - "fmla v26.4s, v6.4s, v4.s[0]\n" - "ldr q6, [x10, #0x40]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "fmla v23.4s, v7.4s, v3.s[0]\n" - "fmla v27.4s, v7.4s, v4.s[0]\n" - "ldr q7, [x10, #0x50]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v12.4s, v6.4s, v1.s[1]\n" - "fmla v16.4s, v6.4s, v2.s[1]\n" - "fmla v20.4s, v6.4s, v3.s[1]\n" - "fmla v24.4s, v6.4s, v4.s[1]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v13.4s, v7.4s, v1.s[1]\n" - "fmla v17.4s, v7.4s, v2.s[1]\n" - "fmla v21.4s, v7.4s, v3.s[1]\n" - "fmla v25.4s, v7.4s, v4.s[1]\n" - "ldr q7, [x10, #0x70]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "fmla v18.4s, v6.4s, v2.s[1]\n" - "fmla v22.4s, v6.4s, v3.s[1]\n" - "fmla v26.4s, v6.4s, v4.s[1]\n" - "ldr q6, [x10, #0x80]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "fmla v19.4s, v7.4s, v2.s[1]\n" - "fmla v23.4s, v7.4s, v3.s[1]\n" - "fmla v27.4s, v7.4s, v4.s[1]\n" - "ldr q7, [x10, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "fmla v12.4s, v6.4s, v1.s[2]\n" - "fmla v16.4s, v6.4s, v2.s[2]\n" - "fmla v20.4s, v6.4s, v3.s[2]\n" - "fmla v24.4s, v6.4s, v4.s[2]\n" - "ldr q6, [x10, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "fmla v13.4s, v7.4s, v1.s[2]\n" - "fmla v17.4s, v7.4s, v2.s[2]\n" - "fmla v21.4s, v7.4s, v3.s[2]\n" - "fmla v25.4s, v7.4s, v4.s[2]\n" - "ldr q7, [x10, #0xb0]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v14.4s, v6.4s, v1.s[2]\n" - "fmla v18.4s, v6.4s, v2.s[2]\n" - "fmla v22.4s, v6.4s, v3.s[2]\n" - "fmla v26.4s, v6.4s, v4.s[2]\n" - "ldr q6, [x10, #0xc0]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "fmla v15.4s, v7.4s, v1.s[2]\n" - "fmla v19.4s, v7.4s, v2.s[2]\n" - "fmla v23.4s, v7.4s, v3.s[2]\n" - "fmla v27.4s, v7.4s, v4.s[2]\n" - "ldr q7, [x10, #0xd0]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v12.4s, v6.4s, v1.s[3]\n" - "fmla v16.4s, v6.4s, v2.s[3]\n" - "fmla v20.4s, v6.4s, v3.s[3]\n" - "fmla v24.4s, v6.4s, v4.s[3]\n" - "ldr q6, [x10, #0xe0]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "fmla v13.4s, v7.4s, v1.s[3]\n" - "fmla v17.4s, v7.4s, v2.s[3]\n" - "fmla v21.4s, v7.4s, v3.s[3]\n" - "fmla v25.4s, v7.4s, v4.s[3]\n" - "ldr q7, [x10, #0xf0]\n" + "movi v19.16b, #0x0\n" + "movi v20.16b, #0x0\n" + "movi v21.16b, #0x0\n" + "movi v22.16b, #0x0\n" + "movi v23.16b, #0x0\n" + "movi v24.16b, #0x0\n" + "movi v25.16b, #0x0\n" + "movi v26.16b, #0x0\n" + "movi v27.16b, #0x0\n" + "146:" // Height 5: setup done + "mov x28, #0x0\n" + "147:" // Height 5: String loop + "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" + "ldr w27, [x20, x28, LSL #0x2]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" + "tbz %x[flags], #3, 148f\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "cbnz x28, 149f\n" + "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" + "add x26, x26, x20, LSL #2\n" + "add x25, x25, x20, LSL #2\n" + "add x24, x24, x20, LSL #2\n" + "add x23, x23, x20, LSL #2\n" + "add x22, x22, x20, LSL #2\n" + "b 149f\n" + "148:" // Height 5: setup direct input + "mov x26, %x[input_ptr]\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" + "149:" // Height 5: input setup done + "cmp x27, #0x4\n" + "blt 152f\n" + "ldr q0, [x26, #0x0]\n" + "ldr q1, [x25, #0x0]\n" + "cmp x27, #0x8\n" + "ldr q2, [x24, #0x0]\n" + "ldr q3, [x23, #0x0]\n" + "ldr q4, [x22, #0x0]\n" + "ldr q6, [x10, #0x0]\n" + "ldr q7, [x10, #0x10]\n" + "blt 151f\n" + "150:" // Height 5: Multiply loop: Main loop head + "fmla v8.4s, v6.4s, v0.s[0]\n" + "fmla v12.4s, v6.4s, v1.s[0]\n" + "sub x27, x27, #0x4\n" + "add x26, x26, #0x10\n" + "fmla v16.4s, v6.4s, v2.s[0]\n" + "fmla v20.4s, v6.4s, v3.s[0]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + "fmla v24.4s, v6.4s, v4.s[0]\n" + "ldr q29, [x10, #0x20]\n" + "fmla v9.4s, v7.4s, v0.s[0]\n" + "add x23, x23, #0x10\n" + "fmla v13.4s, v7.4s, v1.s[0]\n" + "fmla v17.4s, v7.4s, v2.s[0]\n" + "add x22, x22, #0x10\n" + "cmp x27, #0x8\n" + "fmla v21.4s, v7.4s, v3.s[0]\n" + "fmla v25.4s, v7.4s, v4.s[0]\n" + "ldr q28, [x10, #0x30]\n" + "prfm pldl1keep, [x26, #0x80]\n" + "fmla v10.4s, v29.4s, v0.s[0]\n" + "fmla v14.4s, v29.4s, v1.s[0]\n" + "prfm pldl1keep, [x25, #0x80]\n" + "prfm pldl1keep, [x24, #0x80]\n" + "fmla v18.4s, v29.4s, v2.s[0]\n" + "fmla v22.4s, v29.4s, v3.s[0]\n" + "prfm pldl1keep, [x23, #0x80]\n" + "prfm pldl1keep, [x22, #0x80]\n" + "fmla v26.4s, v29.4s, v4.s[0]\n" + "ldr q29, [x10, #0x40]\n" + "fmla v11.4s, v28.4s, v0.s[0]\n" + "fmla v15.4s, v28.4s, v1.s[0]\n" + "fmla v19.4s, v28.4s, v2.s[0]\n" + "fmla v23.4s, v28.4s, v3.s[0]\n" + "fmla v27.4s, v28.4s, v4.s[0]\n" + "ldr q28, [x10, #0x50]\n" + "fmla v8.4s, v29.4s, v0.s[1]\n" + "fmla v12.4s, v29.4s, v1.s[1]\n" + "fmla v16.4s, v29.4s, v2.s[1]\n" + "fmla v20.4s, v29.4s, v3.s[1]\n" + "fmla v24.4s, v29.4s, v4.s[1]\n" + "ldr q29, [x10, #0x60]\n" + "fmla v9.4s, v28.4s, v0.s[1]\n" + "fmla v13.4s, v28.4s, v1.s[1]\n" + "fmla v17.4s, v28.4s, v2.s[1]\n" + "fmla v21.4s, v28.4s, v3.s[1]\n" + "fmla v25.4s, v28.4s, v4.s[1]\n" + "ldr q28, [x10, #0x70]\n" + "fmla v10.4s, v29.4s, v0.s[1]\n" + "fmla v14.4s, v29.4s, v1.s[1]\n" + "fmla v18.4s, v29.4s, v2.s[1]\n" + "fmla v22.4s, v29.4s, v3.s[1]\n" + "fmla v26.4s, v29.4s, v4.s[1]\n" + "ldr q29, [x10, #0x80]\n" + "fmla v11.4s, v28.4s, v0.s[1]\n" + "fmla v15.4s, v28.4s, v1.s[1]\n" + "fmla v19.4s, v28.4s, v2.s[1]\n" + "fmla v23.4s, v28.4s, v3.s[1]\n" + "fmla v27.4s, v28.4s, v4.s[1]\n" + "ldr q28, [x10, #0x90]\n" + "fmla v8.4s, v29.4s, v0.s[2]\n" + "fmla v12.4s, v29.4s, v1.s[2]\n" + "fmla v16.4s, v29.4s, v2.s[2]\n" + "fmla v20.4s, v29.4s, v3.s[2]\n" + "fmla v24.4s, v29.4s, v4.s[2]\n" + "ldr q29, [x10, #0xa0]\n" + "fmla v9.4s, v28.4s, v0.s[2]\n" + "fmla v13.4s, v28.4s, v1.s[2]\n" + "fmla v17.4s, v28.4s, v2.s[2]\n" + "fmla v21.4s, v28.4s, v3.s[2]\n" + "fmla v25.4s, v28.4s, v4.s[2]\n" + "ldr q28, [x10, #0xb0]\n" + "fmla v10.4s, v29.4s, v0.s[2]\n" + "fmla v14.4s, v29.4s, v1.s[2]\n" + "fmla v18.4s, v29.4s, v2.s[2]\n" + "fmla v22.4s, v29.4s, v3.s[2]\n" + "fmla v26.4s, v29.4s, v4.s[2]\n" + "ldr q29, [x10, #0xc0]\n" + "fmla v11.4s, v28.4s, v0.s[2]\n" + "fmla v15.4s, v28.4s, v1.s[2]\n" + "fmla v19.4s, v28.4s, v2.s[2]\n" + "fmla v23.4s, v28.4s, v3.s[2]\n" + "fmla v27.4s, v28.4s, v4.s[2]\n" + "ldr q28, [x10, #0xd0]\n" + "fmla v8.4s, v29.4s, v0.s[3]\n" + "fmla v12.4s, v29.4s, v1.s[3]\n" + "fmla v16.4s, v29.4s, v2.s[3]\n" + "fmla v20.4s, v29.4s, v3.s[3]\n" + "fmla v24.4s, v29.4s, v4.s[3]\n" + "ldr q29, [x10, #0xe0]\n" + "fmla v9.4s, v28.4s, v0.s[3]\n" + "fmla v13.4s, v28.4s, v1.s[3]\n" + "fmla v17.4s, v28.4s, v2.s[3]\n" + "fmla v21.4s, v28.4s, v3.s[3]\n" + "fmla v25.4s, v28.4s, v4.s[3]\n" + "ldr q28, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "fmla v18.4s, v6.4s, v2.s[3]\n" - "fmla v22.4s, v6.4s, v3.s[3]\n" - "fmla v26.4s, v6.4s, v4.s[3]\n" + "fmla v10.4s, v29.4s, v0.s[3]\n" + "fmla v14.4s, v29.4s, v1.s[3]\n" + "fmla v18.4s, v29.4s, v2.s[3]\n" + "fmla v22.4s, v29.4s, v3.s[3]\n" + "fmla v26.4s, v29.4s, v4.s[3]\n" "ldr q6, [x10, #0x0]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" + "fmla v11.4s, v28.4s, v0.s[3]\n" "ldr q0, [x26, #0x0]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" + "fmla v15.4s, v28.4s, v1.s[3]\n" "ldr q1, [x25, #0x0]\n" - "fmla v19.4s, v7.4s, v2.s[3]\n" + "fmla v19.4s, v28.4s, v2.s[3]\n" "ldr q2, [x24, #0x0]\n" - "fmla v23.4s, v7.4s, v3.s[3]\n" + "fmla v23.4s, v28.4s, v3.s[3]\n" "ldr q3, [x23, #0x0]\n" - "fmla v27.4s, v7.4s, v4.s[3]\n" + "fmla v27.4s, v28.4s, v4.s[3]\n" "ldr q4, [x22, #0x0]\n" "ldr q7, [x10, #0x10]\n" "bge 150b\n" @@ -2203,7 +2202,7 @@ void a64_hybrid_fp32_mla_6x16 ( "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" "fmla v24.4s, v6.4s, v4.s[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q29, [x10, #0x20]\n" "fmla v9.4s, v7.4s, v0.s[0]\n" "add x22, x22, #0x10\n" "fmla v13.4s, v7.4s, v1.s[0]\n" @@ -2212,128 +2211,128 @@ void a64_hybrid_fp32_mla_6x16 ( "prfm pldl1keep, [x26, #0x80]\n" "fmla v21.4s, v7.4s, v3.s[0]\n" "fmla v25.4s, v7.4s, v4.s[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q28, [x10, #0x30]\n" "prfm pldl1keep, [x25, #0x80]\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" + "fmla v10.4s, v29.4s, v0.s[0]\n" + "fmla v14.4s, v29.4s, v1.s[0]\n" "prfm pldl1keep, [x24, #0x80]\n" "prfm pldl1keep, [x23, #0x80]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "fmla v22.4s, v6.4s, v3.s[0]\n" + "fmla v18.4s, v29.4s, v2.s[0]\n" + "fmla v22.4s, v29.4s, v3.s[0]\n" "prfm pldl1keep, [x22, #0x80]\n" - "fmla v26.4s, v6.4s, v4.s[0]\n" - "ldr q6, [x10, #0x40]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "fmla v23.4s, v7.4s, v3.s[0]\n" - "fmla v27.4s, v7.4s, v4.s[0]\n" - "ldr q7, [x10, #0x50]\n" - "fmla v8.4s, v6.4s, v0.s[1]\n" - "fmla v12.4s, v6.4s, v1.s[1]\n" - "fmla v16.4s, v6.4s, v2.s[1]\n" - "fmla v20.4s, v6.4s, v3.s[1]\n" - "fmla v24.4s, v6.4s, v4.s[1]\n" - "ldr q6, [x10, #0x60]\n" - "fmla v9.4s, v7.4s, v0.s[1]\n" - "fmla v13.4s, v7.4s, v1.s[1]\n" - "fmla v17.4s, v7.4s, v2.s[1]\n" - "fmla v21.4s, v7.4s, v3.s[1]\n" - "fmla v25.4s, v7.4s, v4.s[1]\n" - "ldr q7, [x10, #0x70]\n" - "fmla v10.4s, v6.4s, v0.s[1]\n" - "fmla v14.4s, v6.4s, v1.s[1]\n" - "fmla v18.4s, v6.4s, v2.s[1]\n" - "fmla v22.4s, v6.4s, v3.s[1]\n" - "fmla v26.4s, v6.4s, v4.s[1]\n" - "ldr q6, [x10, #0x80]\n" - "fmla v11.4s, v7.4s, v0.s[1]\n" - "fmla v15.4s, v7.4s, v1.s[1]\n" - "fmla v19.4s, v7.4s, v2.s[1]\n" - "fmla v23.4s, v7.4s, v3.s[1]\n" - "fmla v27.4s, v7.4s, v4.s[1]\n" - "ldr q7, [x10, #0x90]\n" - "fmla v8.4s, v6.4s, v0.s[2]\n" - "fmla v12.4s, v6.4s, v1.s[2]\n" - "fmla v16.4s, v6.4s, v2.s[2]\n" - "fmla v20.4s, v6.4s, v3.s[2]\n" - "fmla v24.4s, v6.4s, v4.s[2]\n" - "ldr q6, [x10, #0xa0]\n" - "fmla v9.4s, v7.4s, v0.s[2]\n" - "fmla v13.4s, v7.4s, v1.s[2]\n" - "fmla v17.4s, v7.4s, v2.s[2]\n" - "fmla v21.4s, v7.4s, v3.s[2]\n" - "fmla v25.4s, v7.4s, v4.s[2]\n" - "ldr q7, [x10, #0xb0]\n" - "fmla v10.4s, v6.4s, v0.s[2]\n" - "fmla v14.4s, v6.4s, v1.s[2]\n" - "fmla v18.4s, v6.4s, v2.s[2]\n" - "fmla v22.4s, v6.4s, v3.s[2]\n" - "fmla v26.4s, v6.4s, v4.s[2]\n" - "ldr q6, [x10, #0xc0]\n" - "fmla v11.4s, v7.4s, v0.s[2]\n" - "fmla v15.4s, v7.4s, v1.s[2]\n" - "fmla v19.4s, v7.4s, v2.s[2]\n" - "fmla v23.4s, v7.4s, v3.s[2]\n" - "fmla v27.4s, v7.4s, v4.s[2]\n" - "ldr q7, [x10, #0xd0]\n" - "fmla v8.4s, v6.4s, v0.s[3]\n" - "fmla v12.4s, v6.4s, v1.s[3]\n" - "fmla v16.4s, v6.4s, v2.s[3]\n" - "fmla v20.4s, v6.4s, v3.s[3]\n" - "fmla v24.4s, v6.4s, v4.s[3]\n" - "ldr q6, [x10, #0xe0]\n" - "fmla v9.4s, v7.4s, v0.s[3]\n" - "fmla v13.4s, v7.4s, v1.s[3]\n" - "fmla v17.4s, v7.4s, v2.s[3]\n" - "fmla v21.4s, v7.4s, v3.s[3]\n" - "fmla v25.4s, v7.4s, v4.s[3]\n" - "ldr q7, [x10, #0xf0]\n" + "fmla v26.4s, v29.4s, v4.s[0]\n" + "ldr q29, [x10, #0x40]\n" + "fmla v11.4s, v28.4s, v0.s[0]\n" + "fmla v15.4s, v28.4s, v1.s[0]\n" + "fmla v19.4s, v28.4s, v2.s[0]\n" + "fmla v23.4s, v28.4s, v3.s[0]\n" + "fmla v27.4s, v28.4s, v4.s[0]\n" + "ldr q28, [x10, #0x50]\n" + "fmla v8.4s, v29.4s, v0.s[1]\n" + "fmla v12.4s, v29.4s, v1.s[1]\n" + "fmla v16.4s, v29.4s, v2.s[1]\n" + "fmla v20.4s, v29.4s, v3.s[1]\n" + "fmla v24.4s, v29.4s, v4.s[1]\n" + "ldr q29, [x10, #0x60]\n" + "fmla v9.4s, v28.4s, v0.s[1]\n" + "fmla v13.4s, v28.4s, v1.s[1]\n" + "fmla v17.4s, v28.4s, v2.s[1]\n" + "fmla v21.4s, v28.4s, v3.s[1]\n" + "fmla v25.4s, v28.4s, v4.s[1]\n" + "ldr q28, [x10, #0x70]\n" + "fmla v10.4s, v29.4s, v0.s[1]\n" + "fmla v14.4s, v29.4s, v1.s[1]\n" + "fmla v18.4s, v29.4s, v2.s[1]\n" + "fmla v22.4s, v29.4s, v3.s[1]\n" + "fmla v26.4s, v29.4s, v4.s[1]\n" + "ldr q29, [x10, #0x80]\n" + "fmla v11.4s, v28.4s, v0.s[1]\n" + "fmla v15.4s, v28.4s, v1.s[1]\n" + "fmla v19.4s, v28.4s, v2.s[1]\n" + "fmla v23.4s, v28.4s, v3.s[1]\n" + "fmla v27.4s, v28.4s, v4.s[1]\n" + "ldr q28, [x10, #0x90]\n" + "fmla v8.4s, v29.4s, v0.s[2]\n" + "fmla v12.4s, v29.4s, v1.s[2]\n" + "fmla v16.4s, v29.4s, v2.s[2]\n" + "fmla v20.4s, v29.4s, v3.s[2]\n" + "fmla v24.4s, v29.4s, v4.s[2]\n" + "ldr q29, [x10, #0xa0]\n" + "fmla v9.4s, v28.4s, v0.s[2]\n" + "fmla v13.4s, v28.4s, v1.s[2]\n" + "fmla v17.4s, v28.4s, v2.s[2]\n" + "fmla v21.4s, v28.4s, v3.s[2]\n" + "fmla v25.4s, v28.4s, v4.s[2]\n" + "ldr q28, [x10, #0xb0]\n" + "fmla v10.4s, v29.4s, v0.s[2]\n" + "fmla v14.4s, v29.4s, v1.s[2]\n" + "fmla v18.4s, v29.4s, v2.s[2]\n" + "fmla v22.4s, v29.4s, v3.s[2]\n" + "fmla v26.4s, v29.4s, v4.s[2]\n" + "ldr q29, [x10, #0xc0]\n" + "fmla v11.4s, v28.4s, v0.s[2]\n" + "fmla v15.4s, v28.4s, v1.s[2]\n" + "fmla v19.4s, v28.4s, v2.s[2]\n" + "fmla v23.4s, v28.4s, v3.s[2]\n" + "fmla v27.4s, v28.4s, v4.s[2]\n" + "ldr q28, [x10, #0xd0]\n" + "fmla v8.4s, v29.4s, v0.s[3]\n" + "fmla v12.4s, v29.4s, v1.s[3]\n" + "fmla v16.4s, v29.4s, v2.s[3]\n" + "fmla v20.4s, v29.4s, v3.s[3]\n" + "fmla v24.4s, v29.4s, v4.s[3]\n" + "ldr q29, [x10, #0xe0]\n" + "fmla v9.4s, v28.4s, v0.s[3]\n" + "fmla v13.4s, v28.4s, v1.s[3]\n" + "fmla v17.4s, v28.4s, v2.s[3]\n" + "fmla v21.4s, v28.4s, v3.s[3]\n" + "fmla v25.4s, v28.4s, v4.s[3]\n" + "ldr q28, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - "fmla v10.4s, v6.4s, v0.s[3]\n" - "fmla v14.4s, v6.4s, v1.s[3]\n" - "fmla v18.4s, v6.4s, v2.s[3]\n" - "fmla v22.4s, v6.4s, v3.s[3]\n" - "fmla v26.4s, v6.4s, v4.s[3]\n" - "fmla v11.4s, v7.4s, v0.s[3]\n" - "fmla v15.4s, v7.4s, v1.s[3]\n" - "fmla v19.4s, v7.4s, v2.s[3]\n" - "fmla v23.4s, v7.4s, v3.s[3]\n" - "fmla v27.4s, v7.4s, v4.s[3]\n" + "fmla v10.4s, v29.4s, v0.s[3]\n" + "fmla v14.4s, v29.4s, v1.s[3]\n" + "fmla v18.4s, v29.4s, v2.s[3]\n" + "fmla v22.4s, v29.4s, v3.s[3]\n" + "fmla v26.4s, v29.4s, v4.s[3]\n" + "fmla v11.4s, v28.4s, v0.s[3]\n" + "fmla v15.4s, v28.4s, v1.s[3]\n" + "fmla v19.4s, v28.4s, v2.s[3]\n" + "fmla v23.4s, v28.4s, v3.s[3]\n" + "fmla v27.4s, v28.4s, v4.s[3]\n" "152:" // Height 5: Multiply loop: Main loop skip "cbz x27, 154f\n" "153:" // Height 5: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" + "ldr s2, [x26], #0x4\n" "ldr s1, [x25], #0x4\n" "sub x27, x27, #0x1\n" - "ldr s2, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr s4, [x22], #0x4\n" - "ldr q6, [x10, #0x0]\n" - "fmla v8.4s, v6.4s, v0.s[0]\n" - "fmla v12.4s, v6.4s, v1.s[0]\n" - "ldr q7, [x10, #0x10]\n" - "fmla v16.4s, v6.4s, v2.s[0]\n" - "fmla v20.4s, v6.4s, v3.s[0]\n" - "fmla v24.4s, v6.4s, v4.s[0]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.4s, v7.4s, v0.s[0]\n" - "fmla v13.4s, v7.4s, v1.s[0]\n" - "fmla v17.4s, v7.4s, v2.s[0]\n" - "fmla v21.4s, v7.4s, v3.s[0]\n" - "fmla v25.4s, v7.4s, v4.s[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr s0, [x24], #0x4\n" + "ldr s31, [x23], #0x4\n" + "ldr s30, [x22], #0x4\n" + "ldr q29, [x10, #0x0]\n" + "fmla v8.4s, v29.4s, v2.s[0]\n" + "fmla v12.4s, v29.4s, v1.s[0]\n" + "ldr q28, [x10, #0x10]\n" + "fmla v16.4s, v29.4s, v0.s[0]\n" + "fmla v20.4s, v29.4s, v31.s[0]\n" + "fmla v24.4s, v29.4s, v30.s[0]\n" + "ldr q29, [x10, #0x20]\n" + "fmla v9.4s, v28.4s, v2.s[0]\n" + "fmla v13.4s, v28.4s, v1.s[0]\n" + "fmla v17.4s, v28.4s, v0.s[0]\n" + "fmla v21.4s, v28.4s, v31.s[0]\n" + "fmla v25.4s, v28.4s, v30.s[0]\n" + "ldr q28, [x10, #0x30]\n" "add x10, x10, #0x40\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "fmla v22.4s, v6.4s, v3.s[0]\n" - "fmla v26.4s, v6.4s, v4.s[0]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "fmla v23.4s, v7.4s, v3.s[0]\n" - "fmla v27.4s, v7.4s, v4.s[0]\n" + "fmla v10.4s, v29.4s, v2.s[0]\n" + "fmla v14.4s, v29.4s, v1.s[0]\n" + "fmla v18.4s, v29.4s, v0.s[0]\n" + "fmla v22.4s, v29.4s, v31.s[0]\n" + "fmla v26.4s, v29.4s, v30.s[0]\n" + "fmla v11.4s, v28.4s, v2.s[0]\n" + "fmla v15.4s, v28.4s, v1.s[0]\n" + "fmla v19.4s, v28.4s, v0.s[0]\n" + "fmla v23.4s, v28.4s, v31.s[0]\n" + "fmla v27.4s, v28.4s, v30.s[0]\n" "cbnz x27, 153b\n" "154:" // Height 5: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -2352,49 +2351,49 @@ void a64_hybrid_fp32_mla_6x16 ( "prfm pstl1keep, [x22, #0x0]\n" "tbz %x[flags], #1, 155f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v29.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v15.4s, v15.4s, v1.4s\n" - "fmin v16.4s, v16.4s, v1.4s\n" - "fmin v17.4s, v17.4s, v1.4s\n" - "fmin v18.4s, v18.4s, v1.4s\n" - "fmin v19.4s, v19.4s, v1.4s\n" - "fmin v20.4s, v20.4s, v1.4s\n" - "fmin v21.4s, v21.4s, v1.4s\n" - "fmin v22.4s, v22.4s, v1.4s\n" - "fmin v23.4s, v23.4s, v1.4s\n" - "fmin v24.4s, v24.4s, v1.4s\n" - "fmin v25.4s, v25.4s, v1.4s\n" - "fmin v26.4s, v26.4s, v1.4s\n" - "fmin v27.4s, v27.4s, v1.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" - "fmax v16.4s, v16.4s, v0.4s\n" - "fmax v17.4s, v17.4s, v0.4s\n" - "fmax v18.4s, v18.4s, v0.4s\n" - "fmax v19.4s, v19.4s, v0.4s\n" - "fmax v20.4s, v20.4s, v0.4s\n" - "fmax v21.4s, v21.4s, v0.4s\n" - "fmax v22.4s, v22.4s, v0.4s\n" - "fmax v23.4s, v23.4s, v0.4s\n" - "fmax v24.4s, v24.4s, v0.4s\n" - "fmax v25.4s, v25.4s, v0.4s\n" - "fmax v26.4s, v26.4s, v0.4s\n" - "fmax v27.4s, v27.4s, v0.4s\n" + "ld1r { v28.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v29.4s\n" + "fmin v9.4s, v9.4s, v29.4s\n" + "fmin v10.4s, v10.4s, v29.4s\n" + "fmin v11.4s, v11.4s, v29.4s\n" + "fmin v12.4s, v12.4s, v29.4s\n" + "fmin v13.4s, v13.4s, v29.4s\n" + "fmin v14.4s, v14.4s, v29.4s\n" + "fmin v15.4s, v15.4s, v29.4s\n" + "fmin v16.4s, v16.4s, v29.4s\n" + "fmin v17.4s, v17.4s, v29.4s\n" + "fmin v18.4s, v18.4s, v29.4s\n" + "fmin v19.4s, v19.4s, v29.4s\n" + "fmin v20.4s, v20.4s, v29.4s\n" + "fmin v21.4s, v21.4s, v29.4s\n" + "fmin v22.4s, v22.4s, v29.4s\n" + "fmin v23.4s, v23.4s, v29.4s\n" + "fmin v24.4s, v24.4s, v29.4s\n" + "fmin v25.4s, v25.4s, v29.4s\n" + "fmin v26.4s, v26.4s, v29.4s\n" + "fmin v27.4s, v27.4s, v29.4s\n" + "fmax v8.4s, v8.4s, v28.4s\n" + "fmax v9.4s, v9.4s, v28.4s\n" + "fmax v10.4s, v10.4s, v28.4s\n" + "fmax v11.4s, v11.4s, v28.4s\n" + "fmax v12.4s, v12.4s, v28.4s\n" + "fmax v13.4s, v13.4s, v28.4s\n" + "fmax v14.4s, v14.4s, v28.4s\n" + "fmax v15.4s, v15.4s, v28.4s\n" + "fmax v16.4s, v16.4s, v28.4s\n" + "fmax v17.4s, v17.4s, v28.4s\n" + "fmax v18.4s, v18.4s, v28.4s\n" + "fmax v19.4s, v19.4s, v28.4s\n" + "fmax v20.4s, v20.4s, v28.4s\n" + "fmax v21.4s, v21.4s, v28.4s\n" + "fmax v22.4s, v22.4s, v28.4s\n" + "fmax v23.4s, v23.4s, v28.4s\n" + "fmax v24.4s, v24.4s, v28.4s\n" + "fmax v25.4s, v25.4s, v28.4s\n" + "fmax v26.4s, v26.4s, v28.4s\n" + "fmax v27.4s, v27.4s, v28.4s\n" "155:" // Height 5: No activation "cmp x11, #0x10\n" "bge 164f\n" @@ -2771,16 +2770,16 @@ void a64_hybrid_fp32_mla_6x16 ( "180:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 181f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 182f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -2792,11 +2791,11 @@ void a64_hybrid_fp32_mla_6x16 ( "b 182f\n" "181:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "add x21, x22, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" + "add x21, x22, x21, LSL #2\n" "182:" // Height 6: input setup done "cmp x27, #0x4\n" "blt 185f\n" @@ -3073,42 +3072,42 @@ void a64_hybrid_fp32_mla_6x16 ( "185:" // Height 6: Multiply loop: Main loop skip "cbz x27, 187f\n" "186:" // Height 6: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s7, [x26], #0x4\n" + "ldr s6, [x25], #0x4\n" "sub x27, x27, #0x1\n" - "ldr s2, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr s4, [x22], #0x4\n" - "ldr s5, [x21], #0x4\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - "fmla v8.4s, v6.4s, v0.s[0]\n" - "fmla v12.4s, v6.4s, v1.s[0]\n" - "fmla v16.4s, v6.4s, v2.s[0]\n" - "fmla v20.4s, v6.4s, v3.s[0]\n" - "fmla v24.4s, v6.4s, v4.s[0]\n" - "fmla v28.4s, v6.4s, v5.s[0]\n" - "ldr q6, [x10, #0x20]\n" - "fmla v9.4s, v7.4s, v0.s[0]\n" - "fmla v13.4s, v7.4s, v1.s[0]\n" - "fmla v17.4s, v7.4s, v2.s[0]\n" - "fmla v21.4s, v7.4s, v3.s[0]\n" - "fmla v25.4s, v7.4s, v4.s[0]\n" - "fmla v29.4s, v7.4s, v5.s[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr s5, [x24], #0x4\n" + "ldr s4, [x23], #0x4\n" + "ldr s3, [x22], #0x4\n" + "ldr s2, [x21], #0x4\n" + "ldr q1, [x10, #0x0]\n" + "ldr q0, [x10, #0x10]\n" + "fmla v8.4s, v1.4s, v7.s[0]\n" + "fmla v12.4s, v1.4s, v6.s[0]\n" + "fmla v16.4s, v1.4s, v5.s[0]\n" + "fmla v20.4s, v1.4s, v4.s[0]\n" + "fmla v24.4s, v1.4s, v3.s[0]\n" + "fmla v28.4s, v1.4s, v2.s[0]\n" + "ldr q1, [x10, #0x20]\n" + "fmla v9.4s, v0.4s, v7.s[0]\n" + "fmla v13.4s, v0.4s, v6.s[0]\n" + "fmla v17.4s, v0.4s, v5.s[0]\n" + "fmla v21.4s, v0.4s, v4.s[0]\n" + "fmla v25.4s, v0.4s, v3.s[0]\n" + "fmla v29.4s, v0.4s, v2.s[0]\n" + "ldr q0, [x10, #0x30]\n" "add x10, x10, #0x40\n" - "fmla v10.4s, v6.4s, v0.s[0]\n" - "fmla v14.4s, v6.4s, v1.s[0]\n" - "fmla v18.4s, v6.4s, v2.s[0]\n" - "fmla v22.4s, v6.4s, v3.s[0]\n" - "fmla v26.4s, v6.4s, v4.s[0]\n" - "fmla v30.4s, v6.4s, v5.s[0]\n" - "fmla v11.4s, v7.4s, v0.s[0]\n" - "fmla v15.4s, v7.4s, v1.s[0]\n" - "fmla v19.4s, v7.4s, v2.s[0]\n" - "fmla v23.4s, v7.4s, v3.s[0]\n" - "fmla v27.4s, v7.4s, v4.s[0]\n" - "fmla v31.4s, v7.4s, v5.s[0]\n" + "fmla v10.4s, v1.4s, v7.s[0]\n" + "fmla v14.4s, v1.4s, v6.s[0]\n" + "fmla v18.4s, v1.4s, v5.s[0]\n" + "fmla v22.4s, v1.4s, v4.s[0]\n" + "fmla v26.4s, v1.4s, v3.s[0]\n" + "fmla v30.4s, v1.4s, v2.s[0]\n" + "fmla v11.4s, v0.4s, v7.s[0]\n" + "fmla v15.4s, v0.4s, v6.s[0]\n" + "fmla v19.4s, v0.4s, v5.s[0]\n" + "fmla v23.4s, v0.4s, v4.s[0]\n" + "fmla v27.4s, v0.4s, v3.s[0]\n" + "fmla v31.4s, v0.4s, v2.s[0]\n" "cbnz x27, 186b\n" "187:" // Height 6: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -3350,7 +3349,6 @@ void a64_hybrid_fp32_mla_6x16 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "200:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4.hpp index 4fad58a83d868cac5c3d11dc7ed2c54847dadc23..3ec02395d1b9771d6da99120acf3a19f94d6ee39 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef __aarch64__ + #include "../std_transforms_fixed.hpp" #define ARGLIST \ @@ -90,5 +90,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4/a55.cpp index 67e0c1e8cc11842dc985b52d6f4dd23db5a1bc0f..236865315e8634cf2417ac9d0459890db8287434 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4/a55.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021, 2023 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -92,7 +92,6 @@ void a64_hybrid_fp32_mla_8x4_a55 ( break; } __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x8\n" "bge 148f\n" @@ -105,563 +104,563 @@ void a64_hybrid_fp32_mla_8x4_a55 ( "cmp %x[M], #0x2\n" "bgt 43f\n" "beq 22f\n" - "ldr x17, [%x[args_ptr], %[offsetof_N]]\n" - "ldr x16, [%x[args_ptr], %[offsetof_B_ptr]]\n" - "mov x15, %x[bias]\n" - "mov x14, %x[output_ptr]\n" + "mov x3, %x[bias]\n" + "ldr x4, [%x[args_ptr], %[offsetof_N]]\n" + "ldr x5, [%x[args_ptr], %[offsetof_B_ptr]]\n" + "mov x6, %x[output_ptr]\n" "2:" // Height 1: Column loop - "cbz x15, 3f\n" - "ldr q24, [x15, #0x0]\n" - "add x15, x15, #0x10\n" + "cbz x3, 3f\n" + "ldr q24, [x3, #0x0]\n" + "add x3, x3, #0x10\n" "b 8f\n" "3:" // Height 1: no bias "tbz %x[flags], #0, 7f\n" - "cmp x17, #0x4\n" + "cmp x4, #0x4\n" "bge 6f\n" - "tbz x17, #1, 4f\n" - "ldr d24, [x14], #0x8\n" - "mov x8, #0x8\n" - "tbz x17, #0, 5f\n" - "ld1 { v24.s }[2], [x14]\n" + "tbz x4, #1, 4f\n" + "ldr d24, [x6], #0x8\n" + "mov x26, #0x8\n" + "tbz x4, #0, 5f\n" + "ld1 { v24.s }[2], [x6]\n" "b 5f\n" "4:" // Height 1: Partial accumulate: partial_1_0 - "ldr s24, [x14, #0x0]\n" - "mov x8, #0x0\n" + "ldr s24, [x6, #0x0]\n" + "mov x26, #0x0\n" "5:" // Height 1: Partial accumulate: Done - "sub x14, x14, x8\n" + "sub x6, x6, x26\n" "b 8f\n" "6:" // Height 1: full accumulate - "ldr q24, [x14, #0x0]\n" + "ldr q24, [x6, #0x0]\n" "b 8f\n" "7:" // Height 1: no accumulate "movi v24.16b, #0x0\n" "8:" // Height 1: setup done - "mov x13, #0x0\n" + "mov x7, #0x0\n" "9:" // Height 1: String loop - "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr x8, [%x[args_ptr], %[offsetof_input_offset]]\n" - "ldr w12, [x20, x13, LSL #0x2]\n" + "ldr x26, [%x[args_ptr], %[offsetof_string_lengths]]\n" + "ldr w8, [x26, x7, LSL #0x2]\n" + "ldr x27, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 10f\n" - "ldr x20, [%x[input_ptr], x13, LSL #0x3]\n" - "add x20, x20, x8, LSL #3\n" - "ldr x11, [x20, #0x0]\n" - "cbnz x13, 11f\n" - "ldr x8, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x11, x11, x8, LSL #2\n" + "ldr x26, [%x[input_ptr], x7, LSL #0x3]\n" + "add x26, x26, x27, LSL #3\n" + "ldr x17, [x26, #0x0]\n" + "cbnz x7, 11f\n" + "ldr x26, [%x[args_ptr], %[offsetof_input_initial_col]]\n" + "add x17, x17, x26, LSL #2\n" "b 11f\n" "10:" // Height 1: setup direct input - "mov x11, %x[input_ptr]\n" + "mov x17, %x[input_ptr]\n" "11:" // Height 1: input setup done - "cmp x12, #0x4\n" + "cmp x8, #0x4\n" "blt 14f\n" - "ldr q0, [x11, #0x0]\n" - "ldr q8, [x16, #0x0]\n" - "cmp x12, #0x8\n" + "ldr q0, [x17, #0x0]\n" + "cmp x8, #0x8\n" + "ldr q8, [x5, #0x0]\n" + "ldr q9, [x5, #0x10]\n" + "ldr q10, [x5, #0x20]\n" + "ldr q11, [x5, #0x30]\n" "blt 13f\n" "12:" // Height 1: Multiply loop: Main loop head "fmla v24.4s, v8.4s, v0.s[0]\n" - "ldr d9, [x16, #0x10]\n" - "ldr x8, [x16, #0x18]\n" - "add x11, x11, #0x10\n" - "ldr d10, [x16, #0x20]\n" - "sub x12, x12, #0x4\n" - "ldr x21, [x16, #0x28]\n" - "cmp x12, #0x8\n" - "mov v9.d[1], x8\n" - "ldr d11, [x16, #0x30]\n" - "ldr x8, [x16, #0x38]\n" - "add x16, x16, #0x40\n" + "add x17, x17, #0x10\n" "fmla v24.4s, v9.4s, v0.s[1]\n" - "mov v10.d[1], x21\n" - "prfm pldl1keep, [x11, #0x80]\n" - "mov v11.d[1], x8\n" - "ldr d8, [x16, #0x0]\n" - "ldr x26, [x16, #0x8]\n" + "add x5, x5, #0x40\n" + "ldr d8, [x5, #0x0]\n" "fmla v24.4s, v10.4s, v0.s[2]\n" - "ldr x10, [x11, #0x8]\n" - "mov v8.d[1], x26\n" + "ldr d9, [x5, #0x10]\n" "fmla v24.4s, v11.4s, v0.s[3]\n" - "ldr d0, [x11, #0x0]\n" - "mov v0.d[1], x10\n" + "ldr d0, [x17, #0x0]\n" + "sub x8, x8, #0x4\n" + "ldr d10, [x5, #0x20]\n" + "cmp x8, #0x8\n" + "ldr d11, [x5, #0x30]\n" + "ldr x26, [x5, #0x8]\n" + "mov v8.d[1], x26\n" + "ldr x26, [x5, #0x18]\n" + "mov v9.d[1], x26\n" + "ldr x26, [x17, #0x8]\n" + "mov v0.d[1], x26\n" + "ldr x26, [x5, #0x28]\n" + "mov v10.d[1], x26\n" + "ldr x26, [x5, #0x38]\n" + "mov v11.d[1], x26\n" + "prfm pldl1keep, [x17, #0x80]\n" "bge 12b\n" "13:" // Height 1: Multiply loop: Single iteration only "fmla v24.4s, v8.4s, v0.s[0]\n" - "ldr q9, [x16, #0x10]\n" - "ldr q10, [x16, #0x20]\n" - "sub x12, x12, #0x4\n" - "ldr q11, [x16, #0x30]\n" - "add x11, x11, #0x10\n" - "prfm pldl1keep, [x11, #0x80]\n" - "add x16, x16, #0x40\n" + "add x17, x17, #0x10\n" "fmla v24.4s, v9.4s, v0.s[1]\n" + "sub x8, x8, #0x4\n" "fmla v24.4s, v10.4s, v0.s[2]\n" + "prfm pldl1keep, [x17, #0x80]\n" "fmla v24.4s, v11.4s, v0.s[3]\n" + "add x5, x5, #0x40\n" "14:" // Height 1: Multiply loop: Main loop skip - "cbz x12, 16f\n" + "cbz x8, 16f\n" "15:" // Height 1: Multiply loop: Odd block loop - "ldr s0, [x11], #0x4\n" - "sub x12, x12, #0x1\n" - "ldr q12, [x16, #0x0]\n" - "add x16, x16, #0x10\n" - "fmla v24.4s, v12.4s, v0.s[0]\n" - "cbnz x12, 15b\n" + "ldr s17, [x17], #0x4\n" + "sub x8, x8, #0x1\n" + "ldr q16, [x5, #0x0]\n" + "fmla v24.4s, v16.4s, v17.s[0]\n" + "add x5, x5, #0x10\n" + "cbnz x8, 15b\n" "16:" // Height 1: Multiply loop: No odd multiplies - "ldr w8, [%x[args_ptr], %[offsetof_num_strings]]\n" - "add x13, x13, #0x1\n" - "cmp x13, x8\n" + "ldr w26, [%x[args_ptr], %[offsetof_num_strings]]\n" + "add x7, x7, #0x1\n" + "cmp x7, x26\n" "bne 9b\n" - "prfm pstl1keep, [x14, #0x0]\n" + "prfm pstl1keep, [x6, #0x0]\n" "tbz %x[flags], #1, 17f\n" - "add x8, %x[args_ptr], %[offset_min]\n" - "ld1r { v17.4s }, [x8]\n" - "add x8, %x[args_ptr], %[offset_max]\n" - "ld1r { v16.4s }, [x8]\n" + "add x26, %x[args_ptr], %[offset_max]\n" + "ld1r { v16.4s }, [x26]\n" "fmin v24.4s, v24.4s, v16.4s\n" - "fmax v24.4s, v24.4s, v17.4s\n" + "add x26, %x[args_ptr], %[offset_min]\n" + "ld1r { v16.4s }, [x26]\n" + "fmax v24.4s, v24.4s, v16.4s\n" "17:" // Height 1: No activation - "cmp x17, #0x4\n" + "cmp x4, #0x4\n" "bge 20f\n" - "tbz x17, #1, 18f\n" - "str d24, [x14], #0x8\n" - "tbz x17, #0, 19f\n" - "st1 { v24.s }[2], [x14]\n" + "tbz x4, #1, 18f\n" + "str d24, [x6], #0x8\n" + "tbz x4, #0, 19f\n" + "st1 { v24.s }[2], [x6]\n" "b 19f\n" "18:" // Height 1: Partial direct writeback: partial_1_0 - "str s24, [x14, #0x0]\n" + "str s24, [x6, #0x0]\n" "19:" // Height 1: Partial direct writeback: Done "b 21f\n" "20:" // Height 1: Full writeback - "str q24, [x14, #0x0]\n" - "add x14, x14, #0x10\n" + "str q24, [x6, #0x0]\n" + "add x6, x6, #0x10\n" "21:" // Height 1: Writeback done - "subs x17, x17, #0x4\n" + "subs x4, x4, #0x4\n" "bgt 2b\n" "b 170f\n" "22:" // Height 2 - "ldr x17, [%x[args_ptr], %[offsetof_N]]\n" - "mov x15, %x[bias]\n" - "ldr x16, [%x[args_ptr], %[offsetof_B_ptr]]\n" - "mov x14, %x[output_ptr]\n" + "mov x3, %x[bias]\n" + "ldr x4, [%x[args_ptr], %[offsetof_N]]\n" + "ldr x5, [%x[args_ptr], %[offsetof_B_ptr]]\n" + "mov x6, %x[output_ptr]\n" "23:" // Height 2: Column loop - "cbz x15, 24f\n" - "ldr q24, [x15, #0x0]\n" - "add x15, x15, #0x10\n" + "cbz x3, 24f\n" + "ldr q24, [x3, #0x0]\n" "mov v25.16b, v24.16b\n" + "add x3, x3, #0x10\n" "b 29f\n" "24:" // Height 2: no bias "tbz %x[flags], #0, 28f\n" - "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n" - "cmp x17, #0x4\n" - "add x27, x14, x8, LSL #2\n" + "ldr x26, [%x[args_ptr], %[offsetof_output_offset]]\n" + "cmp x4, #0x4\n" + "add x13, x6, x26, LSL #2\n" "bge 27f\n" - "tbz x17, #1, 25f\n" - "ldr d24, [x14], #0x8\n" - "ldr d25, [x27], #0x8\n" - "mov x8, #0x8\n" - "tbz x17, #0, 26f\n" - "ld1 { v24.s }[2], [x14]\n" - "ld1 { v25.s }[2], [x27]\n" + "tbz x4, #1, 25f\n" + "ldr d24, [x6], #0x8\n" + "mov x26, #0x8\n" + "ldr d25, [x13], #0x8\n" + "tbz x4, #0, 26f\n" + "ld1 { v24.s }[2], [x6]\n" + "ld1 { v25.s }[2], [x13]\n" "b 26f\n" "25:" // Height 2: Partial accumulate: partial_1_0 - "ldr s24, [x14, #0x0]\n" - "mov x8, #0x0\n" - "ldr s25, [x27, #0x0]\n" + "ldr s24, [x6, #0x0]\n" + "mov x26, #0x0\n" + "ldr s25, [x13, #0x0]\n" "26:" // Height 2: Partial accumulate: Done - "sub x14, x14, x8\n" + "sub x6, x6, x26\n" "b 29f\n" "27:" // Height 2: full accumulate - "ldr q24, [x14, #0x0]\n" - "ldr q25, [x27, #0x0]\n" + "ldr q24, [x6, #0x0]\n" + "ldr q25, [x13, #0x0]\n" "b 29f\n" "28:" // Height 2: no accumulate "movi v24.16b, #0x0\n" "movi v25.16b, #0x0\n" "29:" // Height 2: setup done - "mov x13, #0x0\n" + "mov x7, #0x0\n" "30:" // Height 2: String loop - "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr x8, [%x[args_ptr], %[offsetof_input_offset]]\n" - "ldr w12, [x20, x13, LSL #0x2]\n" + "ldr x26, [%x[args_ptr], %[offsetof_string_lengths]]\n" + "ldr w8, [x26, x7, LSL #0x2]\n" + "ldr x27, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 31f\n" - "ldr x20, [%x[input_ptr], x13, LSL #0x3]\n" - "add x20, x20, x8, LSL #3\n" - "ldr x11, [x20, #0x0]\n" - "ldr x9, [x20, #0x8]\n" - "cbnz x13, 32f\n" - "ldr x8, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x11, x11, x8, LSL #2\n" - "add x9, x9, x8, LSL #2\n" + "ldr x26, [%x[input_ptr], x7, LSL #0x3]\n" + "add x26, x26, x27, LSL #3\n" + "ldr x17, [x26, #0x0]\n" + "ldr x16, [x26, #0x8]\n" + "cbnz x7, 32f\n" + "ldr x26, [%x[args_ptr], %[offsetof_input_initial_col]]\n" + "add x17, x17, x26, LSL #2\n" + "add x16, x16, x26, LSL #2\n" "b 32f\n" "31:" // Height 2: setup direct input - "mov x11, %x[input_ptr]\n" - "add x9, x11, x8, LSL #2\n" + "mov x17, %x[input_ptr]\n" + "add x16, x17, x27, LSL #2\n" "32:" // Height 2: input setup done - "cmp x12, #0x4\n" + "cmp x8, #0x4\n" "blt 35f\n" - "ldr q0, [x11, #0x0]\n" - "ldr q1, [x9, #0x0]\n" - "cmp x12, #0x8\n" - "ldr q8, [x16, #0x0]\n" + "ldr q0, [x17, #0x0]\n" + "cmp x8, #0x8\n" + "ldr q1, [x16, #0x0]\n" + "ldr q8, [x5, #0x0]\n" + "ldr q9, [x5, #0x10]\n" + "ldr q10, [x5, #0x20]\n" + "ldr q11, [x5, #0x30]\n" "blt 34f\n" "33:" // Height 2: Multiply loop: Main loop head "fmla v24.4s, v8.4s, v0.s[0]\n" - "ldr d9, [x16, #0x10]\n" + "add x17, x17, #0x10\n" "fmla v25.4s, v8.4s, v1.s[0]\n" - "ldr x8, [x16, #0x18]\n" - "ldr d10, [x16, #0x20]\n" - "add x11, x11, #0x10\n" - "ldr x21, [x16, #0x28]\n" - "add x9, x9, #0x10\n" - "mov v9.d[1], x8\n" - "ldr d11, [x16, #0x30]\n" - "ldr x8, [x16, #0x38]\n" - "sub x12, x12, #0x4\n" + "add x16, x16, #0x10\n" "fmla v24.4s, v9.4s, v0.s[1]\n" - "mov v10.d[1], x21\n" + "add x5, x5, #0x40\n" + "ldr d8, [x5, #0x0]\n" "fmla v25.4s, v9.4s, v1.s[1]\n" - "prfm pldl1keep, [x11, #0x80]\n" - "mov v11.d[1], x8\n" - "prfm pldl1keep, [x9, #0x80]\n" - "ldr x10, [x11, #0x8]\n" - "cmp x12, #0x8\n" + "ldr d9, [x5, #0x10]\n" "fmla v24.4s, v10.4s, v0.s[2]\n" - "ldr x28, [x9, #0x8]\n" "fmla v25.4s, v10.4s, v1.s[2]\n" - "add x16, x16, #0x40\n" - "ldr d8, [x16, #0x0]\n" - "ldr x26, [x16, #0x8]\n" + "ldr d10, [x5, #0x20]\n" + "ldr x27, [x5, #0x8]\n" "fmla v24.4s, v11.4s, v0.s[3]\n" - "ldr d0, [x11, #0x0]\n" + "ldr d0, [x17, #0x0]\n" "fmla v25.4s, v11.4s, v1.s[3]\n" - "ldr d1, [x9, #0x0]\n" - "mov v8.d[1], x26\n" - "mov v0.d[1], x10\n" - "mov v1.d[1], x28\n" + "ldr d1, [x16, #0x0]\n" + "sub x8, x8, #0x4\n" + "ldr d11, [x5, #0x30]\n" + "cmp x8, #0x8\n" + "ldr x26, [x5, #0x18]\n" + "mov v8.d[1], x27\n" + "ldr x27, [x17, #0x8]\n" + "mov v9.d[1], x26\n" + "ldr x26, [x16, #0x8]\n" + "mov v0.d[1], x27\n" + "ldr x27, [x5, #0x28]\n" + "mov v1.d[1], x26\n" + "ldr x26, [x5, #0x38]\n" + "mov v10.d[1], x27\n" + "mov v11.d[1], x26\n" + "prfm pldl1keep, [x17, #0x80]\n" + "prfm pldl1keep, [x16, #0x80]\n" "bge 33b\n" "34:" // Height 2: Multiply loop: Single iteration only "fmla v24.4s, v8.4s, v0.s[0]\n" - "ldr q9, [x16, #0x10]\n" + "add x17, x17, #0x10\n" "fmla v25.4s, v8.4s, v1.s[0]\n" - "ldr q10, [x16, #0x20]\n" - "ldr q11, [x16, #0x30]\n" - "sub x12, x12, #0x4\n" - "add x11, x11, #0x10\n" - "add x9, x9, #0x10\n" + "add x16, x16, #0x10\n" "fmla v24.4s, v9.4s, v0.s[1]\n" - "prfm pldl1keep, [x11, #0x80]\n" + "sub x8, x8, #0x4\n" "fmla v25.4s, v9.4s, v1.s[1]\n" - "prfm pldl1keep, [x9, #0x80]\n" - "add x16, x16, #0x40\n" + "prfm pldl1keep, [x17, #0x80]\n" "fmla v24.4s, v10.4s, v0.s[2]\n" + "prfm pldl1keep, [x16, #0x80]\n" "fmla v25.4s, v10.4s, v1.s[2]\n" + "add x5, x5, #0x40\n" "fmla v24.4s, v11.4s, v0.s[3]\n" "fmla v25.4s, v11.4s, v1.s[3]\n" "35:" // Height 2: Multiply loop: Main loop skip - "cbz x12, 37f\n" + "cbz x8, 37f\n" "36:" // Height 2: Multiply loop: Odd block loop - "ldr s0, [x11], #0x4\n" - "sub x12, x12, #0x1\n" - "ldr s1, [x9], #0x4\n" - "ldr q12, [x16, #0x0]\n" - "add x16, x16, #0x10\n" - "fmla v24.4s, v12.4s, v0.s[0]\n" - "fmla v25.4s, v12.4s, v1.s[0]\n" - "cbnz x12, 36b\n" + "ldr s18, [x17], #0x4\n" + "sub x8, x8, #0x1\n" + "ldr s17, [x16], #0x4\n" + "ldr q16, [x5, #0x0]\n" + "fmla v24.4s, v16.4s, v18.s[0]\n" + "fmla v25.4s, v16.4s, v17.s[0]\n" + "add x5, x5, #0x10\n" + "cbnz x8, 36b\n" "37:" // Height 2: Multiply loop: No odd multiplies - "ldr w8, [%x[args_ptr], %[offsetof_num_strings]]\n" - "add x13, x13, #0x1\n" - "cmp x13, x8\n" + "ldr w26, [%x[args_ptr], %[offsetof_num_strings]]\n" + "add x7, x7, #0x1\n" + "cmp x7, x26\n" "bne 30b\n" - "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n" - "prfm pstl1keep, [x14, #0x0]\n" - "add x27, x14, x8, LSL #2\n" - "prfm pstl1keep, [x27, #0x0]\n" + "ldr x26, [%x[args_ptr], %[offsetof_output_offset]]\n" + "add x13, x6, x26, LSL #2\n" + "prfm pstl1keep, [x6, #0x0]\n" + "prfm pstl1keep, [x13, #0x0]\n" "tbz %x[flags], #1, 38f\n" - "add x20, %x[args_ptr], %[offset_min]\n" - "add x8, %x[args_ptr], %[offset_max]\n" - "ld1r { v17.4s }, [x20]\n" - "ld1r { v16.4s }, [x8]\n" + "add x26, %x[args_ptr], %[offset_max]\n" + "ld1r { v16.4s }, [x26]\n" "fmin v24.4s, v24.4s, v16.4s\n" "fmin v25.4s, v25.4s, v16.4s\n" - "fmax v24.4s, v24.4s, v17.4s\n" - "fmax v25.4s, v25.4s, v17.4s\n" + "add x26, %x[args_ptr], %[offset_min]\n" + "ld1r { v16.4s }, [x26]\n" + "fmax v24.4s, v24.4s, v16.4s\n" + "fmax v25.4s, v25.4s, v16.4s\n" "38:" // Height 2: No activation - "cmp x17, #0x4\n" + "cmp x4, #0x4\n" "bge 41f\n" - "tbz x17, #1, 39f\n" - "str d24, [x14], #0x8\n" - "str d25, [x27], #0x8\n" - "tbz x17, #0, 40f\n" - "st1 { v24.s }[2], [x14]\n" - "st1 { v25.s }[2], [x27]\n" + "tbz x4, #1, 39f\n" + "str d24, [x6], #0x8\n" + "str d25, [x13], #0x8\n" + "tbz x4, #0, 40f\n" + "st1 { v24.s }[2], [x6]\n" + "st1 { v25.s }[2], [x13]\n" "b 40f\n" "39:" // Height 2: Partial direct writeback: partial_1_0 - "str s24, [x14, #0x0]\n" - "str s25, [x27, #0x0]\n" + "str s24, [x6, #0x0]\n" + "str s25, [x13, #0x0]\n" "40:" // Height 2: Partial direct writeback: Done "b 42f\n" "41:" // Height 2: Full writeback - "str q24, [x14, #0x0]\n" - "add x14, x14, #0x10\n" - "str q25, [x27, #0x0]\n" + "str q24, [x6, #0x0]\n" + "add x6, x6, #0x10\n" + "str q25, [x13, #0x0]\n" "42:" // Height 2: Writeback done - "subs x17, x17, #0x4\n" + "subs x4, x4, #0x4\n" "bgt 23b\n" "b 170f\n" "43:" // Height 3 - "ldr x17, [%x[args_ptr], %[offsetof_N]]\n" - "mov x15, %x[bias]\n" - "ldr x16, [%x[args_ptr], %[offsetof_B_ptr]]\n" - "mov x14, %x[output_ptr]\n" + "mov x3, %x[bias]\n" + "ldr x4, [%x[args_ptr], %[offsetof_N]]\n" + "ldr x5, [%x[args_ptr], %[offsetof_B_ptr]]\n" + "mov x6, %x[output_ptr]\n" "44:" // Height 3: Column loop - "cbz x15, 45f\n" - "ldr q24, [x15, #0x0]\n" - "add x15, x15, #0x10\n" + "cbz x3, 45f\n" + "ldr q24, [x3, #0x0]\n" "mov v25.16b, v24.16b\n" "mov v26.16b, v24.16b\n" + "add x3, x3, #0x10\n" "b 50f\n" "45:" // Height 3: no bias "tbz %x[flags], #0, 49f\n" - "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n" - "cmp x17, #0x4\n" - "add x27, x14, x8, LSL #2\n" - "add x26, x27, x8, LSL #2\n" + "ldr x26, [%x[args_ptr], %[offsetof_output_offset]]\n" + "add x13, x6, x26, LSL #2\n" + "cmp x4, #0x4\n" + "add x12, x13, x26, LSL #2\n" "bge 48f\n" - "tbz x17, #1, 46f\n" - "ldr d24, [x14], #0x8\n" - "ldr d25, [x27], #0x8\n" - "mov x8, #0x8\n" - "ldr d26, [x26], #0x8\n" - "tbz x17, #0, 47f\n" - "ld1 { v24.s }[2], [x14]\n" - "ld1 { v25.s }[2], [x27]\n" - "ld1 { v26.s }[2], [x26]\n" + "tbz x4, #1, 46f\n" + "ldr d24, [x6], #0x8\n" + "mov x26, #0x8\n" + "ldr d25, [x13], #0x8\n" + "ldr d26, [x12], #0x8\n" + "tbz x4, #0, 47f\n" + "ld1 { v24.s }[2], [x6]\n" + "ld1 { v25.s }[2], [x13]\n" + "ld1 { v26.s }[2], [x12]\n" "b 47f\n" "46:" // Height 3: Partial accumulate: partial_1_0 - "ldr s24, [x14, #0x0]\n" - "mov x8, #0x0\n" - "ldr s25, [x27, #0x0]\n" - "ldr s26, [x26, #0x0]\n" + "ldr s24, [x6, #0x0]\n" + "mov x26, #0x0\n" + "ldr s25, [x13, #0x0]\n" + "ldr s26, [x12, #0x0]\n" "47:" // Height 3: Partial accumulate: Done - "sub x14, x14, x8\n" + "sub x6, x6, x26\n" "b 50f\n" "48:" // Height 3: full accumulate - "ldr q24, [x14, #0x0]\n" - "ldr q25, [x27, #0x0]\n" - "ldr q26, [x26, #0x0]\n" + "ldr q24, [x6, #0x0]\n" + "ldr q25, [x13, #0x0]\n" + "ldr q26, [x12, #0x0]\n" "b 50f\n" "49:" // Height 3: no accumulate "movi v24.16b, #0x0\n" "movi v25.16b, #0x0\n" "movi v26.16b, #0x0\n" "50:" // Height 3: setup done - "mov x13, #0x0\n" + "mov x7, #0x0\n" "51:" // Height 3: String loop - "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr x8, [%x[args_ptr], %[offsetof_input_offset]]\n" - "ldr w12, [x20, x13, LSL #0x2]\n" + "ldr x26, [%x[args_ptr], %[offsetof_string_lengths]]\n" + "ldr w8, [x26, x7, LSL #0x2]\n" + "ldr x27, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 52f\n" - "ldr x20, [%x[input_ptr], x13, LSL #0x3]\n" - "add x20, x20, x8, LSL #3\n" - "ldr x11, [x20, #0x0]\n" - "ldr x9, [x20, #0x8]\n" - "ldr x27, [x20, #0x10]\n" - "cbnz x13, 53f\n" - "ldr x8, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x11, x11, x8, LSL #2\n" - "add x9, x9, x8, LSL #2\n" - "add x27, x27, x8, LSL #2\n" + "ldr x26, [%x[input_ptr], x7, LSL #0x3]\n" + "add x26, x26, x27, LSL #3\n" + "ldr x17, [x26, #0x0]\n" + "ldr x16, [x26, #0x8]\n" + "ldr x15, [x26, #0x10]\n" + "cbnz x7, 53f\n" + "ldr x26, [%x[args_ptr], %[offsetof_input_initial_col]]\n" + "add x17, x17, x26, LSL #2\n" + "add x16, x16, x26, LSL #2\n" + "add x15, x15, x26, LSL #2\n" "b 53f\n" "52:" // Height 3: setup direct input - "mov x11, %x[input_ptr]\n" - "add x9, x11, x8, LSL #2\n" - "add x27, x9, x8, LSL #2\n" + "mov x17, %x[input_ptr]\n" + "add x16, x17, x27, LSL #2\n" + "add x15, x16, x27, LSL #2\n" "53:" // Height 3: input setup done - "cmp x12, #0x4\n" + "cmp x8, #0x4\n" "blt 56f\n" - "ldr q0, [x11, #0x0]\n" - "ldr q1, [x9, #0x0]\n" - "cmp x12, #0x8\n" - "ldr q2, [x27, #0x0]\n" - "ldr q8, [x16, #0x0]\n" + "ldr q0, [x17, #0x0]\n" + "cmp x8, #0x8\n" + "ldr q1, [x16, #0x0]\n" + "ldr q2, [x15, #0x0]\n" + "ldr q8, [x5, #0x0]\n" + "ldr q9, [x5, #0x10]\n" + "ldr q10, [x5, #0x20]\n" + "ldr q11, [x5, #0x30]\n" "blt 55f\n" "54:" // Height 3: Multiply loop: Main loop head "fmla v24.4s, v8.4s, v0.s[0]\n" - "ldr d9, [x16, #0x10]\n" + "add x17, x17, #0x10\n" "fmla v25.4s, v8.4s, v1.s[0]\n" - "ldr x8, [x16, #0x18]\n" + "add x16, x16, #0x10\n" "fmla v26.4s, v8.4s, v2.s[0]\n" - "ldr d10, [x16, #0x20]\n" - "ldr x21, [x16, #0x28]\n" - "add x11, x11, #0x10\n" - "mov v9.d[1], x8\n" - "ldr d11, [x16, #0x30]\n" - "ldr x8, [x16, #0x38]\n" - "add x9, x9, #0x10\n" + "add x15, x15, #0x10\n" "fmla v24.4s, v9.4s, v0.s[1]\n" - "mov v10.d[1], x21\n" + "add x5, x5, #0x40\n" + "ldr d8, [x5, #0x0]\n" "fmla v25.4s, v9.4s, v1.s[1]\n" - "prfm pldl1keep, [x11, #0x80]\n" "fmla v26.4s, v9.4s, v2.s[1]\n" - "mov v11.d[1], x8\n" - "prfm pldl1keep, [x9, #0x80]\n" - "add x27, x27, #0x10\n" + "ldr d9, [x5, #0x10]\n" "fmla v24.4s, v10.4s, v0.s[2]\n" - "prfm pldl1keep, [x27, #0x80]\n" + "ldr x28, [x5, #0x8]\n" "fmla v25.4s, v10.4s, v1.s[2]\n" - "ldr x10, [x11, #0x8]\n" + "ldr x27, [x5, #0x18]\n" "fmla v26.4s, v10.4s, v2.s[2]\n" - "ldr x28, [x9, #0x8]\n" - "ldr x26, [x27, #0x8]\n" - "sub x12, x12, #0x4\n" + "ldr d10, [x5, #0x20]\n" + "ldr x26, [x5, #0x28]\n" "fmla v24.4s, v11.4s, v0.s[3]\n" - "ldr d0, [x11, #0x0]\n" + "ldr d0, [x17, #0x0]\n" "fmla v25.4s, v11.4s, v1.s[3]\n" - "ldr d1, [x9, #0x0]\n" + "ldr d1, [x16, #0x0]\n" "fmla v26.4s, v11.4s, v2.s[3]\n" - "ldr d2, [x27, #0x0]\n" - "mov v0.d[1], x10\n" - "cmp x12, #0x8\n" + "ldr d2, [x15, #0x0]\n" + "sub x8, x8, #0x4\n" + "ldr d11, [x5, #0x30]\n" + "cmp x8, #0x8\n" + "ldr x9, [x17, #0x8]\n" + "mov v8.d[1], x28\n" + "ldr x28, [x16, #0x8]\n" + "mov v9.d[1], x27\n" + "ldr x27, [x15, #0x8]\n" + "mov v10.d[1], x26\n" + "ldr x26, [x5, #0x38]\n" + "mov v0.d[1], x9\n" "mov v1.d[1], x28\n" - "add x16, x16, #0x40\n" - "mov v2.d[1], x26\n" - "ldr d8, [x16, #0x0]\n" - "ldr x26, [x16, #0x8]\n" - "mov v8.d[1], x26\n" + "prfm pldl1keep, [x17, #0x80]\n" + "mov v2.d[1], x27\n" + "prfm pldl1keep, [x16, #0x80]\n" + "mov v11.d[1], x26\n" + "prfm pldl1keep, [x15, #0x80]\n" "bge 54b\n" "55:" // Height 3: Multiply loop: Single iteration only "fmla v24.4s, v8.4s, v0.s[0]\n" - "ldr q9, [x16, #0x10]\n" + "add x17, x17, #0x10\n" "fmla v25.4s, v8.4s, v1.s[0]\n" - "ldr q10, [x16, #0x20]\n" + "add x16, x16, #0x10\n" "fmla v26.4s, v8.4s, v2.s[0]\n" - "ldr q11, [x16, #0x30]\n" - "sub x12, x12, #0x4\n" - "add x11, x11, #0x10\n" + "add x15, x15, #0x10\n" "fmla v24.4s, v9.4s, v0.s[1]\n" - "prfm pldl1keep, [x11, #0x80]\n" + "sub x8, x8, #0x4\n" "fmla v25.4s, v9.4s, v1.s[1]\n" - "add x9, x9, #0x10\n" + "prfm pldl1keep, [x17, #0x80]\n" "fmla v26.4s, v9.4s, v2.s[1]\n" - "prfm pldl1keep, [x9, #0x80]\n" - "add x27, x27, #0x10\n" - "add x16, x16, #0x40\n" + "prfm pldl1keep, [x16, #0x80]\n" "fmla v24.4s, v10.4s, v0.s[2]\n" - "prfm pldl1keep, [x27, #0x80]\n" + "prfm pldl1keep, [x15, #0x80]\n" "fmla v25.4s, v10.4s, v1.s[2]\n" + "add x5, x5, #0x40\n" "fmla v26.4s, v10.4s, v2.s[2]\n" "fmla v24.4s, v11.4s, v0.s[3]\n" "fmla v25.4s, v11.4s, v1.s[3]\n" "fmla v26.4s, v11.4s, v2.s[3]\n" "56:" // Height 3: Multiply loop: Main loop skip - "cbz x12, 58f\n" + "cbz x8, 58f\n" "57:" // Height 3: Multiply loop: Odd block loop - "ldr s0, [x11], #0x4\n" - "sub x12, x12, #0x1\n" - "ldr s1, [x9], #0x4\n" - "ldr s2, [x27], #0x4\n" - "ldr q12, [x16, #0x0]\n" - "add x16, x16, #0x10\n" - "fmla v24.4s, v12.4s, v0.s[0]\n" - "fmla v25.4s, v12.4s, v1.s[0]\n" - "fmla v26.4s, v12.4s, v2.s[0]\n" - "cbnz x12, 57b\n" + "ldr s19, [x17], #0x4\n" + "sub x8, x8, #0x1\n" + "ldr s18, [x16], #0x4\n" + "ldr s17, [x15], #0x4\n" + "ldr q16, [x5, #0x0]\n" + "fmla v24.4s, v16.4s, v19.s[0]\n" + "fmla v25.4s, v16.4s, v18.s[0]\n" + "add x5, x5, #0x10\n" + "fmla v26.4s, v16.4s, v17.s[0]\n" + "cbnz x8, 57b\n" "58:" // Height 3: Multiply loop: No odd multiplies - "ldr w8, [%x[args_ptr], %[offsetof_num_strings]]\n" - "add x13, x13, #0x1\n" - "cmp x13, x8\n" + "ldr w26, [%x[args_ptr], %[offsetof_num_strings]]\n" + "add x7, x7, #0x1\n" + "cmp x7, x26\n" "bne 51b\n" - "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n" - "prfm pstl1keep, [x14, #0x0]\n" - "add x27, x14, x8, LSL #2\n" - "prfm pstl1keep, [x27, #0x0]\n" - "add x26, x27, x8, LSL #2\n" - "prfm pstl1keep, [x26, #0x0]\n" + "ldr x26, [%x[args_ptr], %[offsetof_output_offset]]\n" + "add x13, x6, x26, LSL #2\n" + "add x12, x13, x26, LSL #2\n" + "prfm pstl1keep, [x6, #0x0]\n" + "prfm pstl1keep, [x13, #0x0]\n" + "prfm pstl1keep, [x12, #0x0]\n" "tbz %x[flags], #1, 59f\n" - "add x20, %x[args_ptr], %[offset_min]\n" - "add x8, %x[args_ptr], %[offset_max]\n" - "ld1r { v17.4s }, [x20]\n" - "ld1r { v16.4s }, [x8]\n" + "add x26, %x[args_ptr], %[offset_max]\n" + "ld1r { v16.4s }, [x26]\n" "fmin v24.4s, v24.4s, v16.4s\n" "fmin v25.4s, v25.4s, v16.4s\n" "fmin v26.4s, v26.4s, v16.4s\n" - "fmax v24.4s, v24.4s, v17.4s\n" - "fmax v25.4s, v25.4s, v17.4s\n" - "fmax v26.4s, v26.4s, v17.4s\n" + "add x26, %x[args_ptr], %[offset_min]\n" + "ld1r { v16.4s }, [x26]\n" + "fmax v24.4s, v24.4s, v16.4s\n" + "fmax v25.4s, v25.4s, v16.4s\n" + "fmax v26.4s, v26.4s, v16.4s\n" "59:" // Height 3: No activation - "cmp x17, #0x4\n" + "cmp x4, #0x4\n" "bge 62f\n" - "tbz x17, #1, 60f\n" - "str d24, [x14], #0x8\n" - "str d25, [x27], #0x8\n" - "str d26, [x26], #0x8\n" - "tbz x17, #0, 61f\n" - "st1 { v24.s }[2], [x14]\n" - "st1 { v25.s }[2], [x27]\n" - "st1 { v26.s }[2], [x26]\n" + "tbz x4, #1, 60f\n" + "str d24, [x6], #0x8\n" + "str d25, [x13], #0x8\n" + "str d26, [x12], #0x8\n" + "tbz x4, #0, 61f\n" + "st1 { v24.s }[2], [x6]\n" + "st1 { v25.s }[2], [x13]\n" + "st1 { v26.s }[2], [x12]\n" "b 61f\n" "60:" // Height 3: Partial direct writeback: partial_1_0 - "str s24, [x14, #0x0]\n" - "str s25, [x27, #0x0]\n" - "str s26, [x26, #0x0]\n" + "str s24, [x6, #0x0]\n" + "str s25, [x13, #0x0]\n" + "str s26, [x12, #0x0]\n" "61:" // Height 3: Partial direct writeback: Done "b 63f\n" "62:" // Height 3: Full writeback - "str q24, [x14, #0x0]\n" - "add x14, x14, #0x10\n" - "str q25, [x27, #0x0]\n" - "str q26, [x26, #0x0]\n" + "str q24, [x6, #0x0]\n" + "add x6, x6, #0x10\n" + "str q25, [x13, #0x0]\n" + "str q26, [x12, #0x0]\n" "63:" // Height 3: Writeback done - "subs x17, x17, #0x4\n" + "subs x4, x4, #0x4\n" "bgt 44b\n" "b 170f\n" "64:" // Height 4 - "ldr x17, [%x[args_ptr], %[offsetof_N]]\n" - "mov x15, %x[bias]\n" - "ldr x16, [%x[args_ptr], %[offsetof_B_ptr]]\n" - "mov x14, %x[output_ptr]\n" + "mov x3, %x[bias]\n" + "ldr x4, [%x[args_ptr], %[offsetof_N]]\n" + "ldr x5, [%x[args_ptr], %[offsetof_B_ptr]]\n" + "mov x6, %x[output_ptr]\n" "65:" // Height 4: Column loop - "cbz x15, 66f\n" - "ldr q24, [x15, #0x0]\n" - "add x15, x15, #0x10\n" + "cbz x3, 66f\n" + "ldr q24, [x3, #0x0]\n" "mov v25.16b, v24.16b\n" "mov v26.16b, v24.16b\n" + "add x3, x3, #0x10\n" "mov v27.16b, v24.16b\n" "b 71f\n" "66:" // Height 4: no bias "tbz %x[flags], #0, 70f\n" - "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n" - "cmp x17, #0x4\n" - "add x27, x14, x8, LSL #2\n" - "add x26, x27, x8, LSL #2\n" - "add x25, x26, x8, LSL #2\n" + "ldr x26, [%x[args_ptr], %[offsetof_output_offset]]\n" + "add x13, x6, x26, LSL #2\n" + "add x12, x13, x26, LSL #2\n" + "cmp x4, #0x4\n" + "add x11, x12, x26, LSL #2\n" "bge 69f\n" - "tbz x17, #1, 67f\n" - "ldr d24, [x14], #0x8\n" - "ldr d25, [x27], #0x8\n" - "mov x8, #0x8\n" - "ldr d26, [x26], #0x8\n" - "ldr d27, [x25], #0x8\n" - "tbz x17, #0, 68f\n" - "ld1 { v24.s }[2], [x14]\n" - "ld1 { v25.s }[2], [x27]\n" - "ld1 { v26.s }[2], [x26]\n" - "ld1 { v27.s }[2], [x25]\n" + "tbz x4, #1, 67f\n" + "ldr d24, [x6], #0x8\n" + "mov x26, #0x8\n" + "ldr d25, [x13], #0x8\n" + "ldr d26, [x12], #0x8\n" + "ldr d27, [x11], #0x8\n" + "tbz x4, #0, 68f\n" + "ld1 { v24.s }[2], [x6]\n" + "ld1 { v25.s }[2], [x13]\n" + "ld1 { v26.s }[2], [x12]\n" + "ld1 { v27.s }[2], [x11]\n" "b 68f\n" "67:" // Height 4: Partial accumulate: partial_1_0 - "ldr s24, [x14, #0x0]\n" - "mov x8, #0x0\n" - "ldr s25, [x27, #0x0]\n" - "ldr s26, [x26, #0x0]\n" - "ldr s27, [x25, #0x0]\n" + "ldr s24, [x6, #0x0]\n" + "mov x26, #0x0\n" + "ldr s25, [x13, #0x0]\n" + "ldr s26, [x12, #0x0]\n" + "ldr s27, [x11, #0x0]\n" "68:" // Height 4: Partial accumulate: Done - "sub x14, x14, x8\n" + "sub x6, x6, x26\n" "b 71f\n" "69:" // Height 4: full accumulate - "ldr q24, [x14, #0x0]\n" - "ldr q25, [x27, #0x0]\n" - "ldr q26, [x26, #0x0]\n" - "ldr q27, [x25, #0x0]\n" + "ldr q24, [x6, #0x0]\n" + "ldr q25, [x13, #0x0]\n" + "ldr q26, [x12, #0x0]\n" + "ldr q27, [x11, #0x0]\n" "b 71f\n" "70:" // Height 4: no accumulate "movi v24.16b, #0x0\n" @@ -669,248 +668,248 @@ void a64_hybrid_fp32_mla_8x4_a55 ( "movi v26.16b, #0x0\n" "movi v27.16b, #0x0\n" "71:" // Height 4: setup done - "mov x13, #0x0\n" + "mov x7, #0x0\n" "72:" // Height 4: String loop - "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr x8, [%x[args_ptr], %[offsetof_input_offset]]\n" - "ldr w12, [x20, x13, LSL #0x2]\n" + "ldr x26, [%x[args_ptr], %[offsetof_string_lengths]]\n" + "ldr w8, [x26, x7, LSL #0x2]\n" + "ldr x27, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 73f\n" - "ldr x20, [%x[input_ptr], x13, LSL #0x3]\n" - "add x20, x20, x8, LSL #3\n" - "ldr x11, [x20, #0x0]\n" - "ldr x9, [x20, #0x8]\n" - "ldr x27, [x20, #0x10]\n" - "ldr x25, [x20, #0x18]\n" - "cbnz x13, 74f\n" - "ldr x8, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x11, x11, x8, LSL #2\n" - "add x9, x9, x8, LSL #2\n" - "add x27, x27, x8, LSL #2\n" - "add x25, x25, x8, LSL #2\n" + "ldr x26, [%x[input_ptr], x7, LSL #0x3]\n" + "add x26, x26, x27, LSL #3\n" + "ldr x17, [x26, #0x0]\n" + "ldr x16, [x26, #0x8]\n" + "ldr x15, [x26, #0x10]\n" + "ldr x14, [x26, #0x18]\n" + "cbnz x7, 74f\n" + "ldr x26, [%x[args_ptr], %[offsetof_input_initial_col]]\n" + "add x17, x17, x26, LSL #2\n" + "add x16, x16, x26, LSL #2\n" + "add x15, x15, x26, LSL #2\n" + "add x14, x14, x26, LSL #2\n" "b 74f\n" "73:" // Height 4: setup direct input - "mov x11, %x[input_ptr]\n" - "add x9, x11, x8, LSL #2\n" - "add x27, x9, x8, LSL #2\n" - "add x25, x27, x8, LSL #2\n" + "mov x17, %x[input_ptr]\n" + "add x16, x17, x27, LSL #2\n" + "add x15, x16, x27, LSL #2\n" + "add x14, x15, x27, LSL #2\n" "74:" // Height 4: input setup done - "cmp x12, #0x4\n" + "cmp x8, #0x4\n" "blt 77f\n" - "ldr q0, [x11, #0x0]\n" - "ldr q1, [x9, #0x0]\n" - "cmp x12, #0x8\n" - "ldr q2, [x27, #0x0]\n" - "ldr q3, [x25, #0x0]\n" - "ldr q8, [x16, #0x0]\n" + "ldr q0, [x17, #0x0]\n" + "cmp x8, #0x8\n" + "ldr q1, [x16, #0x0]\n" + "ldr q2, [x15, #0x0]\n" + "ldr q3, [x14, #0x0]\n" + "ldr q8, [x5, #0x0]\n" + "ldr q9, [x5, #0x10]\n" + "ldr q10, [x5, #0x20]\n" + "ldr q11, [x5, #0x30]\n" "blt 76f\n" "75:" // Height 4: Multiply loop: Main loop head "fmla v24.4s, v8.4s, v0.s[0]\n" - "ldr d9, [x16, #0x10]\n" + "add x17, x17, #0x10\n" "fmla v25.4s, v8.4s, v1.s[0]\n" - "ldr x8, [x16, #0x18]\n" + "add x16, x16, #0x10\n" "fmla v26.4s, v8.4s, v2.s[0]\n" - "ldr d10, [x16, #0x20]\n" + "add x15, x15, #0x10\n" "fmla v27.4s, v8.4s, v3.s[0]\n" - "ldr x21, [x16, #0x28]\n" - "mov v9.d[1], x8\n" - "ldr d11, [x16, #0x30]\n" - "ldr x8, [x16, #0x38]\n" - "add x11, x11, #0x10\n" + "add x14, x14, #0x10\n" "fmla v24.4s, v9.4s, v0.s[1]\n" - "mov v10.d[1], x21\n" + "add x5, x5, #0x40\n" + "ldr d8, [x5, #0x0]\n" "fmla v25.4s, v9.4s, v1.s[1]\n" - "prfm pldl1keep, [x11, #0x80]\n" "fmla v26.4s, v9.4s, v2.s[1]\n" - "mov v11.d[1], x8\n" + "ldr x27, [x5, #0x8]\n" "fmla v27.4s, v9.4s, v3.s[1]\n" - "ldr x10, [x11, #0x8]\n" + "ldr d9, [x5, #0x10]\n" "fmla v24.4s, v10.4s, v0.s[2]\n" - "add x9, x9, #0x10\n" + "ldr x26, [x5, #0x18]\n" "fmla v25.4s, v10.4s, v1.s[2]\n" - "prfm pldl1keep, [x9, #0x80]\n" + "ldr x11, [x5, #0x28]\n" "fmla v26.4s, v10.4s, v2.s[2]\n" - "ldr x28, [x9, #0x8]\n" + "ldr x10, [x17, #0x8]\n" "fmla v27.4s, v10.4s, v3.s[2]\n" - "add x27, x27, #0x10\n" + "ldr d10, [x5, #0x20]\n" "fmla v24.4s, v11.4s, v0.s[3]\n" - "prfm pldl1keep, [x27, #0x80]\n" + "ldr d0, [x17, #0x0]\n" "fmla v25.4s, v11.4s, v1.s[3]\n" - "ldr d0, [x11, #0x0]\n" + "ldr d1, [x16, #0x0]\n" + "ldr x9, [x16, #0x8]\n" "fmla v26.4s, v11.4s, v2.s[3]\n" - "ldr d1, [x9, #0x0]\n" + "ldr d2, [x15, #0x0]\n" "fmla v27.4s, v11.4s, v3.s[3]\n" - "ldr d2, [x27, #0x0]\n" + "ldr d3, [x14, #0x0]\n" + "sub x8, x8, #0x4\n" + "ldr d11, [x5, #0x30]\n" + "cmp x8, #0x8\n" + "ldr x28, [x15, #0x8]\n" + "mov v8.d[1], x27\n" + "ldr x27, [x14, #0x8]\n" + "mov v9.d[1], x26\n" + "ldr x26, [x5, #0x38]\n" + "mov v10.d[1], x11\n" + "prfm pldl1keep, [x17, #0x80]\n" "mov v0.d[1], x10\n" - "ldr x26, [x27, #0x8]\n" - "mov v1.d[1], x28\n" - "add x25, x25, #0x10\n" - "prfm pldl1keep, [x25, #0x80]\n" - "sub x12, x12, #0x4\n" - "mov v2.d[1], x26\n" - "ldr d3, [x25, #0x0]\n" - "ldr x8, [x25, #0x8]\n" - "cmp x12, #0x8\n" - "add x16, x16, #0x40\n" - "ldr d8, [x16, #0x0]\n" - "mov v3.d[1], x8\n" - "ldr x26, [x16, #0x8]\n" - "mov v8.d[1], x26\n" + "prfm pldl1keep, [x16, #0x80]\n" + "mov v1.d[1], x9\n" + "mov v2.d[1], x28\n" + "prfm pldl1keep, [x15, #0x80]\n" + "mov v3.d[1], x27\n" + "prfm pldl1keep, [x14, #0x80]\n" + "mov v11.d[1], x26\n" "bge 75b\n" "76:" // Height 4: Multiply loop: Single iteration only "fmla v24.4s, v8.4s, v0.s[0]\n" - "ldr q9, [x16, #0x10]\n" + "add x17, x17, #0x10\n" "fmla v25.4s, v8.4s, v1.s[0]\n" - "ldr q10, [x16, #0x20]\n" + "add x16, x16, #0x10\n" "fmla v26.4s, v8.4s, v2.s[0]\n" - "ldr q11, [x16, #0x30]\n" + "add x15, x15, #0x10\n" "fmla v27.4s, v8.4s, v3.s[0]\n" - "sub x12, x12, #0x4\n" + "add x14, x14, #0x10\n" "fmla v24.4s, v9.4s, v0.s[1]\n" - "add x11, x11, #0x10\n" + "sub x8, x8, #0x4\n" "fmla v25.4s, v9.4s, v1.s[1]\n" - "prfm pldl1keep, [x11, #0x80]\n" + "prfm pldl1keep, [x17, #0x80]\n" "fmla v26.4s, v9.4s, v2.s[1]\n" - "add x9, x9, #0x10\n" + "prfm pldl1keep, [x16, #0x80]\n" "fmla v27.4s, v9.4s, v3.s[1]\n" - "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x15, #0x80]\n" "fmla v24.4s, v10.4s, v0.s[2]\n" - "add x27, x27, #0x10\n" + "prfm pldl1keep, [x14, #0x80]\n" "fmla v25.4s, v10.4s, v1.s[2]\n" - "prfm pldl1keep, [x27, #0x80]\n" + "add x5, x5, #0x40\n" "fmla v26.4s, v10.4s, v2.s[2]\n" - "add x25, x25, #0x10\n" "fmla v27.4s, v10.4s, v3.s[2]\n" - "prfm pldl1keep, [x25, #0x80]\n" "fmla v24.4s, v11.4s, v0.s[3]\n" - "add x16, x16, #0x40\n" "fmla v25.4s, v11.4s, v1.s[3]\n" "fmla v26.4s, v11.4s, v2.s[3]\n" "fmla v27.4s, v11.4s, v3.s[3]\n" "77:" // Height 4: Multiply loop: Main loop skip - "cbz x12, 79f\n" + "cbz x8, 79f\n" "78:" // Height 4: Multiply loop: Odd block loop - "ldr s0, [x11], #0x4\n" - "sub x12, x12, #0x1\n" - "ldr s1, [x9], #0x4\n" - "ldr s2, [x27], #0x4\n" - "ldr s3, [x25], #0x4\n" - "ldr q12, [x16, #0x0]\n" - "add x16, x16, #0x10\n" - "fmla v24.4s, v12.4s, v0.s[0]\n" - "fmla v25.4s, v12.4s, v1.s[0]\n" - "fmla v26.4s, v12.4s, v2.s[0]\n" - "fmla v27.4s, v12.4s, v3.s[0]\n" - "cbnz x12, 78b\n" + "ldr s20, [x17], #0x4\n" + "sub x8, x8, #0x1\n" + "ldr s19, [x16], #0x4\n" + "ldr s18, [x15], #0x4\n" + "ldr s17, [x14], #0x4\n" + "ldr q16, [x5, #0x0]\n" + "fmla v24.4s, v16.4s, v20.s[0]\n" + "fmla v25.4s, v16.4s, v19.s[0]\n" + "add x5, x5, #0x10\n" + "fmla v26.4s, v16.4s, v18.s[0]\n" + "fmla v27.4s, v16.4s, v17.s[0]\n" + "cbnz x8, 78b\n" "79:" // Height 4: Multiply loop: No odd multiplies - "ldr w8, [%x[args_ptr], %[offsetof_num_strings]]\n" - "add x13, x13, #0x1\n" - "cmp x13, x8\n" + "ldr w26, [%x[args_ptr], %[offsetof_num_strings]]\n" + "add x7, x7, #0x1\n" + "cmp x7, x26\n" "bne 72b\n" - "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n" - "prfm pstl1keep, [x14, #0x0]\n" - "add x27, x14, x8, LSL #2\n" - "prfm pstl1keep, [x27, #0x0]\n" - "add x26, x27, x8, LSL #2\n" - "prfm pstl1keep, [x26, #0x0]\n" - "add x25, x26, x8, LSL #2\n" - "prfm pstl1keep, [x25, #0x0]\n" + "ldr x26, [%x[args_ptr], %[offsetof_output_offset]]\n" + "add x13, x6, x26, LSL #2\n" + "add x12, x13, x26, LSL #2\n" + "add x11, x12, x26, LSL #2\n" + "prfm pstl1keep, [x6, #0x0]\n" + "prfm pstl1keep, [x13, #0x0]\n" + "prfm pstl1keep, [x12, #0x0]\n" + "prfm pstl1keep, [x11, #0x0]\n" "tbz %x[flags], #1, 80f\n" - "add x20, %x[args_ptr], %[offset_min]\n" - "add x8, %x[args_ptr], %[offset_max]\n" - "ld1r { v17.4s }, [x20]\n" - "ld1r { v16.4s }, [x8]\n" + "add x26, %x[args_ptr], %[offset_max]\n" + "ld1r { v16.4s }, [x26]\n" "fmin v24.4s, v24.4s, v16.4s\n" "fmin v25.4s, v25.4s, v16.4s\n" "fmin v26.4s, v26.4s, v16.4s\n" "fmin v27.4s, v27.4s, v16.4s\n" - "fmax v24.4s, v24.4s, v17.4s\n" - "fmax v25.4s, v25.4s, v17.4s\n" - "fmax v26.4s, v26.4s, v17.4s\n" - "fmax v27.4s, v27.4s, v17.4s\n" + "add x26, %x[args_ptr], %[offset_min]\n" + "ld1r { v16.4s }, [x26]\n" + "fmax v24.4s, v24.4s, v16.4s\n" + "fmax v25.4s, v25.4s, v16.4s\n" + "fmax v26.4s, v26.4s, v16.4s\n" + "fmax v27.4s, v27.4s, v16.4s\n" "80:" // Height 4: No activation - "cmp x17, #0x4\n" + "cmp x4, #0x4\n" "bge 83f\n" - "tbz x17, #1, 81f\n" - "str d24, [x14], #0x8\n" - "str d25, [x27], #0x8\n" - "str d26, [x26], #0x8\n" - "str d27, [x25], #0x8\n" - "tbz x17, #0, 82f\n" - "st1 { v24.s }[2], [x14]\n" - "st1 { v25.s }[2], [x27]\n" - "st1 { v26.s }[2], [x26]\n" - "st1 { v27.s }[2], [x25]\n" + "tbz x4, #1, 81f\n" + "str d24, [x6], #0x8\n" + "str d25, [x13], #0x8\n" + "str d26, [x12], #0x8\n" + "str d27, [x11], #0x8\n" + "tbz x4, #0, 82f\n" + "st1 { v24.s }[2], [x6]\n" + "st1 { v25.s }[2], [x13]\n" + "st1 { v26.s }[2], [x12]\n" + "st1 { v27.s }[2], [x11]\n" "b 82f\n" "81:" // Height 4: Partial direct writeback: partial_1_0 - "str s24, [x14, #0x0]\n" - "str s25, [x27, #0x0]\n" - "str s26, [x26, #0x0]\n" - "str s27, [x25, #0x0]\n" + "str s24, [x6, #0x0]\n" + "str s25, [x13, #0x0]\n" + "str s26, [x12, #0x0]\n" + "str s27, [x11, #0x0]\n" "82:" // Height 4: Partial direct writeback: Done "b 84f\n" "83:" // Height 4: Full writeback - "str q24, [x14, #0x0]\n" - "add x14, x14, #0x10\n" - "str q25, [x27, #0x0]\n" - "str q26, [x26, #0x0]\n" - "str q27, [x25, #0x0]\n" + "str q24, [x6, #0x0]\n" + "add x6, x6, #0x10\n" + "str q25, [x13, #0x0]\n" + "str q26, [x12, #0x0]\n" + "str q27, [x11, #0x0]\n" "84:" // Height 4: Writeback done - "subs x17, x17, #0x4\n" + "subs x4, x4, #0x4\n" "bgt 65b\n" "b 170f\n" "85:" // Height 5 - "ldr x17, [%x[args_ptr], %[offsetof_N]]\n" - "mov x15, %x[bias]\n" - "ldr x16, [%x[args_ptr], %[offsetof_B_ptr]]\n" - "mov x14, %x[output_ptr]\n" + "mov x3, %x[bias]\n" + "ldr x4, [%x[args_ptr], %[offsetof_N]]\n" + "ldr x5, [%x[args_ptr], %[offsetof_B_ptr]]\n" + "mov x6, %x[output_ptr]\n" "86:" // Height 5: Column loop - "cbz x15, 87f\n" - "ldr q24, [x15, #0x0]\n" - "add x15, x15, #0x10\n" + "cbz x3, 87f\n" + "ldr q24, [x3, #0x0]\n" "mov v25.16b, v24.16b\n" "mov v26.16b, v24.16b\n" + "add x3, x3, #0x10\n" "mov v27.16b, v24.16b\n" "mov v28.16b, v24.16b\n" "b 92f\n" "87:" // Height 5: no bias "tbz %x[flags], #0, 91f\n" - "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n" - "cmp x17, #0x4\n" - "add x27, x14, x8, LSL #2\n" - "add x26, x27, x8, LSL #2\n" - "add x25, x26, x8, LSL #2\n" - "add x24, x25, x8, LSL #2\n" + "ldr x26, [%x[args_ptr], %[offsetof_output_offset]]\n" + "add x13, x6, x26, LSL #2\n" + "add x12, x13, x26, LSL #2\n" + "add x11, x12, x26, LSL #2\n" + "cmp x4, #0x4\n" + "add x10, x11, x26, LSL #2\n" "bge 90f\n" - "tbz x17, #1, 88f\n" - "ldr d24, [x14], #0x8\n" - "ldr d25, [x27], #0x8\n" - "mov x8, #0x8\n" - "ldr d26, [x26], #0x8\n" - "ldr d27, [x25], #0x8\n" - "ldr d28, [x24], #0x8\n" - "tbz x17, #0, 89f\n" - "ld1 { v24.s }[2], [x14]\n" - "ld1 { v25.s }[2], [x27]\n" - "ld1 { v26.s }[2], [x26]\n" - "ld1 { v27.s }[2], [x25]\n" - "ld1 { v28.s }[2], [x24]\n" + "tbz x4, #1, 88f\n" + "ldr d24, [x6], #0x8\n" + "mov x26, #0x8\n" + "ldr d25, [x13], #0x8\n" + "ldr d26, [x12], #0x8\n" + "ldr d27, [x11], #0x8\n" + "ldr d28, [x10], #0x8\n" + "tbz x4, #0, 89f\n" + "ld1 { v24.s }[2], [x6]\n" + "ld1 { v25.s }[2], [x13]\n" + "ld1 { v26.s }[2], [x12]\n" + "ld1 { v27.s }[2], [x11]\n" + "ld1 { v28.s }[2], [x10]\n" "b 89f\n" "88:" // Height 5: Partial accumulate: partial_1_0 - "ldr s24, [x14, #0x0]\n" - "mov x8, #0x0\n" - "ldr s25, [x27, #0x0]\n" - "ldr s26, [x26, #0x0]\n" - "ldr s27, [x25, #0x0]\n" - "ldr s28, [x24, #0x0]\n" + "ldr s24, [x6, #0x0]\n" + "mov x26, #0x0\n" + "ldr s25, [x13, #0x0]\n" + "ldr s26, [x12, #0x0]\n" + "ldr s27, [x11, #0x0]\n" + "ldr s28, [x10, #0x0]\n" "89:" // Height 5: Partial accumulate: Done - "sub x14, x14, x8\n" + "sub x6, x6, x26\n" "b 92f\n" "90:" // Height 5: full accumulate - "ldr q24, [x14, #0x0]\n" - "ldr q25, [x27, #0x0]\n" - "ldr q26, [x26, #0x0]\n" - "ldr q27, [x25, #0x0]\n" - "ldr q28, [x24, #0x0]\n" + "ldr q24, [x6, #0x0]\n" + "ldr q25, [x13, #0x0]\n" + "ldr q26, [x12, #0x0]\n" + "ldr q27, [x11, #0x0]\n" + "ldr q28, [x10, #0x0]\n" "b 92f\n" "91:" // Height 5: no accumulate "movi v24.16b, #0x0\n" @@ -919,283 +918,283 @@ void a64_hybrid_fp32_mla_8x4_a55 ( "movi v27.16b, #0x0\n" "movi v28.16b, #0x0\n" "92:" // Height 5: setup done - "mov x13, #0x0\n" + "mov x7, #0x0\n" "93:" // Height 5: String loop - "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr x8, [%x[args_ptr], %[offsetof_input_offset]]\n" - "ldr w12, [x20, x13, LSL #0x2]\n" + "ldr x26, [%x[args_ptr], %[offsetof_string_lengths]]\n" + "ldr w8, [x26, x7, LSL #0x2]\n" + "ldr x27, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 94f\n" - "ldr x20, [%x[input_ptr], x13, LSL #0x3]\n" - "add x20, x20, x8, LSL #3\n" - "ldr x11, [x20, #0x0]\n" - "ldr x9, [x20, #0x8]\n" - "ldr x27, [x20, #0x10]\n" - "ldr x25, [x20, #0x18]\n" - "ldr x24, [x20, #0x20]\n" - "cbnz x13, 95f\n" - "ldr x8, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x11, x11, x8, LSL #2\n" - "add x9, x9, x8, LSL #2\n" - "add x27, x27, x8, LSL #2\n" - "add x25, x25, x8, LSL #2\n" - "add x24, x24, x8, LSL #2\n" + "ldr x26, [%x[input_ptr], x7, LSL #0x3]\n" + "add x26, x26, x27, LSL #3\n" + "ldr x17, [x26, #0x0]\n" + "ldr x16, [x26, #0x8]\n" + "ldr x15, [x26, #0x10]\n" + "ldr x14, [x26, #0x18]\n" + "ldr x13, [x26, #0x20]\n" + "cbnz x7, 95f\n" + "ldr x26, [%x[args_ptr], %[offsetof_input_initial_col]]\n" + "add x17, x17, x26, LSL #2\n" + "add x16, x16, x26, LSL #2\n" + "add x15, x15, x26, LSL #2\n" + "add x14, x14, x26, LSL #2\n" + "add x13, x13, x26, LSL #2\n" "b 95f\n" "94:" // Height 5: setup direct input - "mov x11, %x[input_ptr]\n" - "add x9, x11, x8, LSL #2\n" - "add x27, x9, x8, LSL #2\n" - "add x25, x27, x8, LSL #2\n" - "add x24, x25, x8, LSL #2\n" + "mov x17, %x[input_ptr]\n" + "add x16, x17, x27, LSL #2\n" + "add x15, x16, x27, LSL #2\n" + "add x14, x15, x27, LSL #2\n" + "add x13, x14, x27, LSL #2\n" "95:" // Height 5: input setup done - "cmp x12, #0x4\n" + "cmp x8, #0x4\n" "blt 98f\n" - "ldr q0, [x11, #0x0]\n" - "ldr q1, [x9, #0x0]\n" - "cmp x12, #0x8\n" - "ldr q2, [x27, #0x0]\n" - "ldr q3, [x25, #0x0]\n" - "ldr q4, [x24, #0x0]\n" - "ldr q8, [x16, #0x0]\n" + "ldr q0, [x17, #0x0]\n" + "cmp x8, #0x8\n" + "ldr q1, [x16, #0x0]\n" + "ldr q2, [x15, #0x0]\n" + "ldr q3, [x14, #0x0]\n" + "ldr q4, [x13, #0x0]\n" + "ldr q8, [x5, #0x0]\n" + "ldr q9, [x5, #0x10]\n" + "ldr q10, [x5, #0x20]\n" + "ldr q11, [x5, #0x30]\n" "blt 97f\n" "96:" // Height 5: Multiply loop: Main loop head "fmla v24.4s, v8.4s, v0.s[0]\n" - "ldr d9, [x16, #0x10]\n" + "add x17, x17, #0x10\n" "fmla v25.4s, v8.4s, v1.s[0]\n" - "ldr x8, [x16, #0x18]\n" + "add x16, x16, #0x10\n" "fmla v26.4s, v8.4s, v2.s[0]\n" - "ldr d10, [x16, #0x20]\n" + "add x15, x15, #0x10\n" "fmla v27.4s, v8.4s, v3.s[0]\n" - "ldr x21, [x16, #0x28]\n" + "add x14, x14, #0x10\n" "fmla v28.4s, v8.4s, v4.s[0]\n" - "mov v9.d[1], x8\n" - "ldr d11, [x16, #0x30]\n" - "add x11, x11, #0x10\n" + "add x13, x13, #0x10\n" "fmla v24.4s, v9.4s, v0.s[1]\n" - "mov v10.d[1], x21\n" + "add x5, x5, #0x40\n" + "ldr d8, [x5, #0x0]\n" "fmla v25.4s, v9.4s, v1.s[1]\n" - "ldr x8, [x16, #0x38]\n" "fmla v26.4s, v9.4s, v2.s[1]\n" - "prfm pldl1keep, [x11, #0x80]\n" + "ldr x27, [x5, #0x8]\n" "fmla v27.4s, v9.4s, v3.s[1]\n" - "ldr x10, [x11, #0x8]\n" + "ldr x26, [x5, #0x18]\n" "fmla v28.4s, v9.4s, v4.s[1]\n" - "mov v11.d[1], x8\n" + "ldr d9, [x5, #0x10]\n" "fmla v24.4s, v10.4s, v0.s[2]\n" - "add x9, x9, #0x10\n" + "ldr x12, [x5, #0x28]\n" "fmla v25.4s, v10.4s, v1.s[2]\n" - "prfm pldl1keep, [x9, #0x80]\n" + "ldr x11, [x17, #0x8]\n" "fmla v26.4s, v10.4s, v2.s[2]\n" - "ldr x28, [x9, #0x8]\n" + "ldr x10, [x16, #0x8]\n" "fmla v27.4s, v10.4s, v3.s[2]\n" - "add x27, x27, #0x10\n" + "ldr x9, [x15, #0x8]\n" "fmla v28.4s, v10.4s, v4.s[2]\n" - "prfm pldl1keep, [x27, #0x80]\n" + "ldr d10, [x5, #0x20]\n" "fmla v24.4s, v11.4s, v0.s[3]\n" - "ldr d0, [x11, #0x0]\n" + "ldr d0, [x17, #0x0]\n" "fmla v25.4s, v11.4s, v1.s[3]\n" - "ldr d1, [x9, #0x0]\n" + "ldr d1, [x16, #0x0]\n" "fmla v26.4s, v11.4s, v2.s[3]\n" - "ldr d2, [x27, #0x0]\n" + "ldr d2, [x15, #0x0]\n" "fmla v27.4s, v11.4s, v3.s[3]\n" - "mov v0.d[1], x10\n" + "ldr d3, [x14, #0x0]\n" + "ldr x28, [x14, #0x8]\n" "fmla v28.4s, v11.4s, v4.s[3]\n" - "mov v1.d[1], x28\n" - "ldr x26, [x27, #0x8]\n" - "add x25, x25, #0x10\n" - "prfm pldl1keep, [x25, #0x80]\n" - "add x24, x24, #0x10\n" - "prfm pldl1keep, [x24, #0x80]\n" - "sub x12, x12, #0x4\n" - "mov v2.d[1], x26\n" - "ldr d3, [x25, #0x0]\n" - "ldr x8, [x25, #0x8]\n" - "cmp x12, #0x8\n" - "ldr d4, [x24, #0x0]\n" - "add x16, x16, #0x40\n" - "ldr x21, [x24, #0x8]\n" - "mov v3.d[1], x8\n" - "ldr d8, [x16, #0x0]\n" - "ldr x26, [x16, #0x8]\n" - "mov v4.d[1], x21\n" - "mov v8.d[1], x26\n" + "ldr d4, [x13, #0x0]\n" + "sub x8, x8, #0x4\n" + "ldr d11, [x5, #0x30]\n" + "cmp x8, #0x8\n" + "mov v8.d[1], x27\n" + "ldr x27, [x13, #0x8]\n" + "mov v9.d[1], x26\n" + "ldr x26, [x5, #0x38]\n" + "prfm pldl1keep, [x17, #0x80]\n" + "mov v10.d[1], x12\n" + "prfm pldl1keep, [x16, #0x80]\n" + "mov v0.d[1], x11\n" + "prfm pldl1keep, [x15, #0x80]\n" + "mov v1.d[1], x10\n" + "prfm pldl1keep, [x14, #0x80]\n" + "mov v2.d[1], x9\n" + "mov v3.d[1], x28\n" + "prfm pldl1keep, [x13, #0x80]\n" + "mov v4.d[1], x27\n" + "mov v11.d[1], x26\n" "bge 96b\n" "97:" // Height 5: Multiply loop: Single iteration only "fmla v24.4s, v8.4s, v0.s[0]\n" - "ldr q9, [x16, #0x10]\n" + "add x17, x17, #0x10\n" "fmla v25.4s, v8.4s, v1.s[0]\n" - "ldr q10, [x16, #0x20]\n" + "add x16, x16, #0x10\n" "fmla v26.4s, v8.4s, v2.s[0]\n" - "ldr q11, [x16, #0x30]\n" + "add x15, x15, #0x10\n" "fmla v27.4s, v8.4s, v3.s[0]\n" - "sub x12, x12, #0x4\n" + "add x14, x14, #0x10\n" "fmla v28.4s, v8.4s, v4.s[0]\n" - "add x11, x11, #0x10\n" + "add x13, x13, #0x10\n" "fmla v24.4s, v9.4s, v0.s[1]\n" - "prfm pldl1keep, [x11, #0x80]\n" + "sub x8, x8, #0x4\n" "fmla v25.4s, v9.4s, v1.s[1]\n" - "add x9, x9, #0x10\n" + "prfm pldl1keep, [x17, #0x80]\n" "fmla v26.4s, v9.4s, v2.s[1]\n" - "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x16, #0x80]\n" "fmla v27.4s, v9.4s, v3.s[1]\n" - "add x27, x27, #0x10\n" + "prfm pldl1keep, [x15, #0x80]\n" "fmla v28.4s, v9.4s, v4.s[1]\n" - "prfm pldl1keep, [x27, #0x80]\n" + "prfm pldl1keep, [x14, #0x80]\n" "fmla v24.4s, v10.4s, v0.s[2]\n" - "add x25, x25, #0x10\n" + "prfm pldl1keep, [x13, #0x80]\n" "fmla v25.4s, v10.4s, v1.s[2]\n" - "prfm pldl1keep, [x25, #0x80]\n" + "add x5, x5, #0x40\n" "fmla v26.4s, v10.4s, v2.s[2]\n" - "add x24, x24, #0x10\n" "fmla v27.4s, v10.4s, v3.s[2]\n" - "prfm pldl1keep, [x24, #0x80]\n" "fmla v28.4s, v10.4s, v4.s[2]\n" - "add x16, x16, #0x40\n" "fmla v24.4s, v11.4s, v0.s[3]\n" "fmla v25.4s, v11.4s, v1.s[3]\n" "fmla v26.4s, v11.4s, v2.s[3]\n" "fmla v27.4s, v11.4s, v3.s[3]\n" "fmla v28.4s, v11.4s, v4.s[3]\n" "98:" // Height 5: Multiply loop: Main loop skip - "cbz x12, 100f\n" + "cbz x8, 100f\n" "99:" // Height 5: Multiply loop: Odd block loop - "ldr s0, [x11], #0x4\n" - "sub x12, x12, #0x1\n" - "ldr s1, [x9], #0x4\n" - "ldr s2, [x27], #0x4\n" - "ldr s3, [x25], #0x4\n" - "ldr s4, [x24], #0x4\n" - "ldr q12, [x16, #0x0]\n" - "add x16, x16, #0x10\n" - "fmla v24.4s, v12.4s, v0.s[0]\n" - "fmla v25.4s, v12.4s, v1.s[0]\n" - "fmla v26.4s, v12.4s, v2.s[0]\n" - "fmla v27.4s, v12.4s, v3.s[0]\n" - "fmla v28.4s, v12.4s, v4.s[0]\n" - "cbnz x12, 99b\n" + "ldr s21, [x17], #0x4\n" + "sub x8, x8, #0x1\n" + "ldr s20, [x16], #0x4\n" + "ldr s19, [x15], #0x4\n" + "ldr s18, [x14], #0x4\n" + "ldr s17, [x13], #0x4\n" + "ldr q16, [x5, #0x0]\n" + "fmla v24.4s, v16.4s, v21.s[0]\n" + "fmla v25.4s, v16.4s, v20.s[0]\n" + "add x5, x5, #0x10\n" + "fmla v26.4s, v16.4s, v19.s[0]\n" + "fmla v27.4s, v16.4s, v18.s[0]\n" + "fmla v28.4s, v16.4s, v17.s[0]\n" + "cbnz x8, 99b\n" "100:" // Height 5: Multiply loop: No odd multiplies - "ldr w8, [%x[args_ptr], %[offsetof_num_strings]]\n" - "add x13, x13, #0x1\n" - "cmp x13, x8\n" + "ldr w26, [%x[args_ptr], %[offsetof_num_strings]]\n" + "add x7, x7, #0x1\n" + "cmp x7, x26\n" "bne 93b\n" - "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n" - "prfm pstl1keep, [x14, #0x0]\n" - "add x27, x14, x8, LSL #2\n" - "prfm pstl1keep, [x27, #0x0]\n" - "add x26, x27, x8, LSL #2\n" - "prfm pstl1keep, [x26, #0x0]\n" - "add x25, x26, x8, LSL #2\n" - "prfm pstl1keep, [x25, #0x0]\n" - "add x24, x25, x8, LSL #2\n" - "prfm pstl1keep, [x24, #0x0]\n" + "ldr x26, [%x[args_ptr], %[offsetof_output_offset]]\n" + "add x13, x6, x26, LSL #2\n" + "add x12, x13, x26, LSL #2\n" + "add x11, x12, x26, LSL #2\n" + "add x10, x11, x26, LSL #2\n" + "prfm pstl1keep, [x6, #0x0]\n" + "prfm pstl1keep, [x13, #0x0]\n" + "prfm pstl1keep, [x12, #0x0]\n" + "prfm pstl1keep, [x11, #0x0]\n" + "prfm pstl1keep, [x10, #0x0]\n" "tbz %x[flags], #1, 101f\n" - "add x20, %x[args_ptr], %[offset_min]\n" - "add x8, %x[args_ptr], %[offset_max]\n" - "ld1r { v17.4s }, [x20]\n" - "ld1r { v16.4s }, [x8]\n" + "add x26, %x[args_ptr], %[offset_max]\n" + "ld1r { v16.4s }, [x26]\n" "fmin v24.4s, v24.4s, v16.4s\n" "fmin v25.4s, v25.4s, v16.4s\n" "fmin v26.4s, v26.4s, v16.4s\n" "fmin v27.4s, v27.4s, v16.4s\n" "fmin v28.4s, v28.4s, v16.4s\n" - "fmax v24.4s, v24.4s, v17.4s\n" - "fmax v25.4s, v25.4s, v17.4s\n" - "fmax v26.4s, v26.4s, v17.4s\n" - "fmax v27.4s, v27.4s, v17.4s\n" - "fmax v28.4s, v28.4s, v17.4s\n" + "add x26, %x[args_ptr], %[offset_min]\n" + "ld1r { v16.4s }, [x26]\n" + "fmax v24.4s, v24.4s, v16.4s\n" + "fmax v25.4s, v25.4s, v16.4s\n" + "fmax v26.4s, v26.4s, v16.4s\n" + "fmax v27.4s, v27.4s, v16.4s\n" + "fmax v28.4s, v28.4s, v16.4s\n" "101:" // Height 5: No activation - "cmp x17, #0x4\n" + "cmp x4, #0x4\n" "bge 104f\n" - "tbz x17, #1, 102f\n" - "str d24, [x14], #0x8\n" - "str d25, [x27], #0x8\n" - "str d26, [x26], #0x8\n" - "str d27, [x25], #0x8\n" - "str d28, [x24], #0x8\n" - "tbz x17, #0, 103f\n" - "st1 { v24.s }[2], [x14]\n" - "st1 { v25.s }[2], [x27]\n" - "st1 { v26.s }[2], [x26]\n" - "st1 { v27.s }[2], [x25]\n" - "st1 { v28.s }[2], [x24]\n" + "tbz x4, #1, 102f\n" + "str d24, [x6], #0x8\n" + "str d25, [x13], #0x8\n" + "str d26, [x12], #0x8\n" + "str d27, [x11], #0x8\n" + "str d28, [x10], #0x8\n" + "tbz x4, #0, 103f\n" + "st1 { v24.s }[2], [x6]\n" + "st1 { v25.s }[2], [x13]\n" + "st1 { v26.s }[2], [x12]\n" + "st1 { v27.s }[2], [x11]\n" + "st1 { v28.s }[2], [x10]\n" "b 103f\n" "102:" // Height 5: Partial direct writeback: partial_1_0 - "str s24, [x14, #0x0]\n" - "str s25, [x27, #0x0]\n" - "str s26, [x26, #0x0]\n" - "str s27, [x25, #0x0]\n" - "str s28, [x24, #0x0]\n" + "str s24, [x6, #0x0]\n" + "str s25, [x13, #0x0]\n" + "str s26, [x12, #0x0]\n" + "str s27, [x11, #0x0]\n" + "str s28, [x10, #0x0]\n" "103:" // Height 5: Partial direct writeback: Done "b 105f\n" "104:" // Height 5: Full writeback - "str q24, [x14, #0x0]\n" - "add x14, x14, #0x10\n" - "str q25, [x27, #0x0]\n" - "str q26, [x26, #0x0]\n" - "str q27, [x25, #0x0]\n" - "str q28, [x24, #0x0]\n" + "str q24, [x6, #0x0]\n" + "add x6, x6, #0x10\n" + "str q25, [x13, #0x0]\n" + "str q26, [x12, #0x0]\n" + "str q27, [x11, #0x0]\n" + "str q28, [x10, #0x0]\n" "105:" // Height 5: Writeback done - "subs x17, x17, #0x4\n" + "subs x4, x4, #0x4\n" "bgt 86b\n" "b 170f\n" "106:" // Height 6 - "ldr x17, [%x[args_ptr], %[offsetof_N]]\n" - "mov x15, %x[bias]\n" - "ldr x16, [%x[args_ptr], %[offsetof_B_ptr]]\n" - "mov x14, %x[output_ptr]\n" + "mov x3, %x[bias]\n" + "ldr x4, [%x[args_ptr], %[offsetof_N]]\n" + "ldr x5, [%x[args_ptr], %[offsetof_B_ptr]]\n" + "mov x6, %x[output_ptr]\n" "107:" // Height 6: Column loop - "cbz x15, 108f\n" - "ldr q24, [x15, #0x0]\n" - "add x15, x15, #0x10\n" + "cbz x3, 108f\n" + "ldr q24, [x3, #0x0]\n" "mov v25.16b, v24.16b\n" "mov v26.16b, v24.16b\n" + "add x3, x3, #0x10\n" "mov v27.16b, v24.16b\n" "mov v28.16b, v24.16b\n" "mov v29.16b, v24.16b\n" "b 113f\n" "108:" // Height 6: no bias "tbz %x[flags], #0, 112f\n" - "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n" - "cmp x17, #0x4\n" - "add x27, x14, x8, LSL #2\n" - "add x26, x27, x8, LSL #2\n" - "add x25, x26, x8, LSL #2\n" - "add x24, x25, x8, LSL #2\n" - "add x23, x24, x8, LSL #2\n" + "ldr x26, [%x[args_ptr], %[offsetof_output_offset]]\n" + "add x13, x6, x26, LSL #2\n" + "add x12, x13, x26, LSL #2\n" + "add x11, x12, x26, LSL #2\n" + "add x10, x11, x26, LSL #2\n" + "cmp x4, #0x4\n" + "add x9, x10, x26, LSL #2\n" "bge 111f\n" - "tbz x17, #1, 109f\n" - "ldr d24, [x14], #0x8\n" - "ldr d25, [x27], #0x8\n" - "mov x8, #0x8\n" - "ldr d26, [x26], #0x8\n" - "ldr d27, [x25], #0x8\n" - "ldr d28, [x24], #0x8\n" - "ldr d29, [x23], #0x8\n" - "tbz x17, #0, 110f\n" - "ld1 { v24.s }[2], [x14]\n" - "ld1 { v25.s }[2], [x27]\n" - "ld1 { v26.s }[2], [x26]\n" - "ld1 { v27.s }[2], [x25]\n" - "ld1 { v28.s }[2], [x24]\n" - "ld1 { v29.s }[2], [x23]\n" + "tbz x4, #1, 109f\n" + "ldr d24, [x6], #0x8\n" + "mov x26, #0x8\n" + "ldr d25, [x13], #0x8\n" + "ldr d26, [x12], #0x8\n" + "ldr d27, [x11], #0x8\n" + "ldr d28, [x10], #0x8\n" + "ldr d29, [x9], #0x8\n" + "tbz x4, #0, 110f\n" + "ld1 { v24.s }[2], [x6]\n" + "ld1 { v25.s }[2], [x13]\n" + "ld1 { v26.s }[2], [x12]\n" + "ld1 { v27.s }[2], [x11]\n" + "ld1 { v28.s }[2], [x10]\n" + "ld1 { v29.s }[2], [x9]\n" "b 110f\n" "109:" // Height 6: Partial accumulate: partial_1_0 - "ldr s24, [x14, #0x0]\n" - "mov x8, #0x0\n" - "ldr s25, [x27, #0x0]\n" - "ldr s26, [x26, #0x0]\n" - "ldr s27, [x25, #0x0]\n" - "ldr s28, [x24, #0x0]\n" - "ldr s29, [x23, #0x0]\n" + "ldr s24, [x6, #0x0]\n" + "mov x26, #0x0\n" + "ldr s25, [x13, #0x0]\n" + "ldr s26, [x12, #0x0]\n" + "ldr s27, [x11, #0x0]\n" + "ldr s28, [x10, #0x0]\n" + "ldr s29, [x9, #0x0]\n" "110:" // Height 6: Partial accumulate: Done - "sub x14, x14, x8\n" + "sub x6, x6, x26\n" "b 113f\n" "111:" // Height 6: full accumulate - "ldr q24, [x14, #0x0]\n" - "ldr q25, [x27, #0x0]\n" - "ldr q26, [x26, #0x0]\n" - "ldr q27, [x25, #0x0]\n" - "ldr q28, [x24, #0x0]\n" - "ldr q29, [x23, #0x0]\n" + "ldr q24, [x6, #0x0]\n" + "ldr q25, [x13, #0x0]\n" + "ldr q26, [x12, #0x0]\n" + "ldr q27, [x11, #0x0]\n" + "ldr q28, [x10, #0x0]\n" + "ldr q29, [x9, #0x0]\n" "b 113f\n" "112:" // Height 6: no accumulate "movi v24.16b, #0x0\n" @@ -1205,154 +1204,154 @@ void a64_hybrid_fp32_mla_8x4_a55 ( "movi v28.16b, #0x0\n" "movi v29.16b, #0x0\n" "113:" // Height 6: setup done - "mov x13, #0x0\n" + "mov x7, #0x0\n" "114:" // Height 6: String loop - "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr x8, [%x[args_ptr], %[offsetof_input_offset]]\n" - "ldr w12, [x20, x13, LSL #0x2]\n" + "ldr x26, [%x[args_ptr], %[offsetof_string_lengths]]\n" + "ldr w8, [x26, x7, LSL #0x2]\n" + "ldr x27, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 115f\n" - "ldr x20, [%x[input_ptr], x13, LSL #0x3]\n" - "add x20, x20, x8, LSL #3\n" - "ldr x11, [x20, #0x0]\n" - "ldr x9, [x20, #0x8]\n" - "ldr x27, [x20, #0x10]\n" - "ldr x25, [x20, #0x18]\n" - "ldr x24, [x20, #0x20]\n" - "ldr x23, [x20, #0x28]\n" - "cbnz x13, 116f\n" - "ldr x8, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x11, x11, x8, LSL #2\n" - "add x9, x9, x8, LSL #2\n" - "add x27, x27, x8, LSL #2\n" - "add x25, x25, x8, LSL #2\n" - "add x24, x24, x8, LSL #2\n" - "add x23, x23, x8, LSL #2\n" + "ldr x26, [%x[input_ptr], x7, LSL #0x3]\n" + "add x26, x26, x27, LSL #3\n" + "ldr x17, [x26, #0x0]\n" + "ldr x16, [x26, #0x8]\n" + "ldr x15, [x26, #0x10]\n" + "ldr x14, [x26, #0x18]\n" + "ldr x13, [x26, #0x20]\n" + "ldr x12, [x26, #0x28]\n" + "cbnz x7, 116f\n" + "ldr x26, [%x[args_ptr], %[offsetof_input_initial_col]]\n" + "add x17, x17, x26, LSL #2\n" + "add x16, x16, x26, LSL #2\n" + "add x15, x15, x26, LSL #2\n" + "add x14, x14, x26, LSL #2\n" + "add x13, x13, x26, LSL #2\n" + "add x12, x12, x26, LSL #2\n" "b 116f\n" "115:" // Height 6: setup direct input - "mov x11, %x[input_ptr]\n" - "add x9, x11, x8, LSL #2\n" - "add x27, x9, x8, LSL #2\n" - "add x25, x27, x8, LSL #2\n" - "add x24, x25, x8, LSL #2\n" - "add x23, x24, x8, LSL #2\n" + "mov x17, %x[input_ptr]\n" + "add x16, x17, x27, LSL #2\n" + "add x15, x16, x27, LSL #2\n" + "add x14, x15, x27, LSL #2\n" + "add x13, x14, x27, LSL #2\n" + "add x12, x13, x27, LSL #2\n" "116:" // Height 6: input setup done - "cmp x12, #0x4\n" + "cmp x8, #0x4\n" "blt 119f\n" - "ldr q0, [x11, #0x0]\n" - "ldr q1, [x9, #0x0]\n" - "cmp x12, #0x8\n" - "ldr q2, [x27, #0x0]\n" - "ldr q3, [x25, #0x0]\n" - "ldr q4, [x24, #0x0]\n" - "ldr q5, [x23, #0x0]\n" - "ldr q8, [x16, #0x0]\n" + "ldr q0, [x17, #0x0]\n" + "cmp x8, #0x8\n" + "ldr q1, [x16, #0x0]\n" + "ldr q2, [x15, #0x0]\n" + "ldr q3, [x14, #0x0]\n" + "ldr q4, [x13, #0x0]\n" + "ldr q5, [x12, #0x0]\n" + "ldr q8, [x5, #0x0]\n" + "ldr q9, [x5, #0x10]\n" + "ldr q10, [x5, #0x20]\n" + "ldr q11, [x5, #0x30]\n" "blt 118f\n" "117:" // Height 6: Multiply loop: Main loop head "fmla v24.4s, v8.4s, v0.s[0]\n" - "ldr d9, [x16, #0x10]\n" + "add x17, x17, #0x10\n" "fmla v25.4s, v8.4s, v1.s[0]\n" - "ldr x8, [x16, #0x18]\n" + "add x16, x16, #0x10\n" "fmla v26.4s, v8.4s, v2.s[0]\n" - "ldr d10, [x16, #0x20]\n" + "add x15, x15, #0x10\n" "fmla v27.4s, v8.4s, v3.s[0]\n" - "ldr x21, [x16, #0x28]\n" + "add x14, x14, #0x10\n" "fmla v28.4s, v8.4s, v4.s[0]\n" - "mov v9.d[1], x8\n" + "add x13, x13, #0x10\n" "fmla v29.4s, v8.4s, v5.s[0]\n" - "ldr d11, [x16, #0x30]\n" + "add x12, x12, #0x10\n" "fmla v24.4s, v9.4s, v0.s[1]\n" - "mov v10.d[1], x21\n" + "add x5, x5, #0x40\n" + "ldr d8, [x5, #0x0]\n" "fmla v25.4s, v9.4s, v1.s[1]\n" - "ldr x8, [x16, #0x38]\n" "fmla v26.4s, v9.4s, v2.s[1]\n" - "add x11, x11, #0x10\n" + "ldr x9, [x5, #0x8]\n" "fmla v27.4s, v9.4s, v3.s[1]\n" - "prfm pldl1keep, [x11, #0x80]\n" + "ldr x28, [x5, #0x18]\n" "fmla v28.4s, v9.4s, v4.s[1]\n" - "mov v11.d[1], x8\n" + "ldr x27, [x5, #0x28]\n" "fmla v29.4s, v9.4s, v5.s[1]\n" - "ldr x10, [x11, #0x8]\n" + "ldr d9, [x5, #0x10]\n" "fmla v24.4s, v10.4s, v0.s[2]\n" - "add x9, x9, #0x10\n" + "ldr x26, [x17, #0x8]\n" "fmla v25.4s, v10.4s, v1.s[2]\n" - "prfm pldl1keep, [x9, #0x80]\n" + "ldr x11, [x16, #0x8]\n" "fmla v26.4s, v10.4s, v2.s[2]\n" - "ldr x28, [x9, #0x8]\n" + "ldr x10, [x15, #0x8]\n" "fmla v27.4s, v10.4s, v3.s[2]\n" - "add x27, x27, #0x10\n" + "sub x8, x8, #0x4\n" "fmla v28.4s, v10.4s, v4.s[2]\n" - "prfm pldl1keep, [x27, #0x80]\n" + "cmp x8, #0x8\n" "fmla v29.4s, v10.4s, v5.s[2]\n" - "ldr x26, [x27, #0x8]\n" + "ldr d10, [x5, #0x20]\n" "fmla v24.4s, v11.4s, v0.s[3]\n" - "ldr d0, [x11, #0x0]\n" + "ldr d0, [x17, #0x0]\n" "fmla v25.4s, v11.4s, v1.s[3]\n" - "ldr d1, [x9, #0x0]\n" + "ldr d1, [x16, #0x0]\n" "fmla v26.4s, v11.4s, v2.s[3]\n" - "ldr d2, [x27, #0x0]\n" + "ldr d2, [x15, #0x0]\n" "fmla v27.4s, v11.4s, v3.s[3]\n" - "mov v0.d[1], x10\n" + "ldr d3, [x14, #0x0]\n" "fmla v28.4s, v11.4s, v4.s[3]\n" - "mov v1.d[1], x28\n" + "ldr d4, [x13, #0x0]\n" "fmla v29.4s, v11.4s, v5.s[3]\n" - "mov v2.d[1], x26\n" - "add x25, x25, #0x10\n" - "add x24, x24, #0x10\n" - "prfm pldl1keep, [x25, #0x80]\n" - "add x23, x23, #0x10\n" - "prfm pldl1keep, [x24, #0x80]\n" - "sub x12, x12, #0x4\n" - "prfm pldl1keep, [x23, #0x80]\n" - "cmp x12, #0x8\n" - "ldr d3, [x25, #0x0]\n" - "add x16, x16, #0x40\n" - "ldr x8, [x25, #0x8]\n" - "ldr d4, [x24, #0x0]\n" - "ldr x21, [x24, #0x8]\n" - "mov v3.d[1], x8\n" - "ldr d5, [x23, #0x0]\n" - "ldr x8, [x23, #0x8]\n" - "mov v4.d[1], x21\n" - "ldr d8, [x16, #0x0]\n" - "ldr x26, [x16, #0x8]\n" - "mov v5.d[1], x8\n" - "mov v8.d[1], x26\n" + "ldr d5, [x12, #0x0]\n" + "ldr d11, [x5, #0x30]\n" + "mov v8.d[1], x9\n" + "ldr x9, [x14, #0x8]\n" + "mov v9.d[1], x28\n" + "ldr x28, [x13, #0x8]\n" + "mov v10.d[1], x27\n" + "ldr x27, [x12, #0x8]\n" + "mov v0.d[1], x26\n" + "ldr x26, [x5, #0x38]\n" + "mov v1.d[1], x11\n" + "prfm pldl1keep, [x17, #0x80]\n" + "mov v2.d[1], x10\n" + "prfm pldl1keep, [x16, #0x80]\n" + "mov v3.d[1], x9\n" + "prfm pldl1keep, [x15, #0x80]\n" + "mov v4.d[1], x28\n" + "prfm pldl1keep, [x14, #0x80]\n" + "mov v5.d[1], x27\n" + "prfm pldl1keep, [x13, #0x80]\n" + "mov v11.d[1], x26\n" + "prfm pldl1keep, [x12, #0x80]\n" "bge 117b\n" "118:" // Height 6: Multiply loop: Single iteration only "fmla v24.4s, v8.4s, v0.s[0]\n" - "ldr q9, [x16, #0x10]\n" + "add x17, x17, #0x10\n" "fmla v25.4s, v8.4s, v1.s[0]\n" - "ldr q10, [x16, #0x20]\n" + "add x16, x16, #0x10\n" "fmla v26.4s, v8.4s, v2.s[0]\n" - "ldr q11, [x16, #0x30]\n" + "add x15, x15, #0x10\n" "fmla v27.4s, v8.4s, v3.s[0]\n" - "sub x12, x12, #0x4\n" + "add x14, x14, #0x10\n" "fmla v28.4s, v8.4s, v4.s[0]\n" - "add x11, x11, #0x10\n" + "add x13, x13, #0x10\n" "fmla v29.4s, v8.4s, v5.s[0]\n" - "prfm pldl1keep, [x11, #0x80]\n" + "add x12, x12, #0x10\n" "fmla v24.4s, v9.4s, v0.s[1]\n" - "add x9, x9, #0x10\n" + "sub x8, x8, #0x4\n" "fmla v25.4s, v9.4s, v1.s[1]\n" - "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x17, #0x80]\n" "fmla v26.4s, v9.4s, v2.s[1]\n" - "add x27, x27, #0x10\n" + "prfm pldl1keep, [x16, #0x80]\n" "fmla v27.4s, v9.4s, v3.s[1]\n" - "prfm pldl1keep, [x27, #0x80]\n" + "prfm pldl1keep, [x15, #0x80]\n" "fmla v28.4s, v9.4s, v4.s[1]\n" - "add x25, x25, #0x10\n" + "prfm pldl1keep, [x14, #0x80]\n" "fmla v29.4s, v9.4s, v5.s[1]\n" - "prfm pldl1keep, [x25, #0x80]\n" + "prfm pldl1keep, [x13, #0x80]\n" "fmla v24.4s, v10.4s, v0.s[2]\n" - "add x24, x24, #0x10\n" + "prfm pldl1keep, [x12, #0x80]\n" "fmla v25.4s, v10.4s, v1.s[2]\n" - "prfm pldl1keep, [x24, #0x80]\n" + "add x5, x5, #0x40\n" "fmla v26.4s, v10.4s, v2.s[2]\n" - "add x23, x23, #0x10\n" "fmla v27.4s, v10.4s, v3.s[2]\n" - "prfm pldl1keep, [x23, #0x80]\n" "fmla v28.4s, v10.4s, v4.s[2]\n" - "add x16, x16, #0x40\n" "fmla v29.4s, v10.4s, v5.s[2]\n" "fmla v24.4s, v11.4s, v0.s[3]\n" "fmla v25.4s, v11.4s, v1.s[3]\n" @@ -1361,108 +1360,108 @@ void a64_hybrid_fp32_mla_8x4_a55 ( "fmla v28.4s, v11.4s, v4.s[3]\n" "fmla v29.4s, v11.4s, v5.s[3]\n" "119:" // Height 6: Multiply loop: Main loop skip - "cbz x12, 121f\n" + "cbz x8, 121f\n" "120:" // Height 6: Multiply loop: Odd block loop - "ldr s0, [x11], #0x4\n" - "sub x12, x12, #0x1\n" - "ldr s1, [x9], #0x4\n" - "ldr s2, [x27], #0x4\n" - "ldr s3, [x25], #0x4\n" - "ldr s4, [x24], #0x4\n" - "ldr s5, [x23], #0x4\n" - "ldr q12, [x16, #0x0]\n" - "add x16, x16, #0x10\n" - "fmla v24.4s, v12.4s, v0.s[0]\n" - "fmla v25.4s, v12.4s, v1.s[0]\n" - "fmla v26.4s, v12.4s, v2.s[0]\n" - "fmla v27.4s, v12.4s, v3.s[0]\n" - "fmla v28.4s, v12.4s, v4.s[0]\n" - "fmla v29.4s, v12.4s, v5.s[0]\n" - "cbnz x12, 120b\n" + "ldr s22, [x17], #0x4\n" + "sub x8, x8, #0x1\n" + "ldr s21, [x16], #0x4\n" + "ldr s20, [x15], #0x4\n" + "ldr s19, [x14], #0x4\n" + "ldr s18, [x13], #0x4\n" + "ldr s17, [x12], #0x4\n" + "ldr q16, [x5, #0x0]\n" + "fmla v24.4s, v16.4s, v22.s[0]\n" + "fmla v25.4s, v16.4s, v21.s[0]\n" + "add x5, x5, #0x10\n" + "fmla v26.4s, v16.4s, v20.s[0]\n" + "fmla v27.4s, v16.4s, v19.s[0]\n" + "fmla v28.4s, v16.4s, v18.s[0]\n" + "fmla v29.4s, v16.4s, v17.s[0]\n" + "cbnz x8, 120b\n" "121:" // Height 6: Multiply loop: No odd multiplies - "ldr w8, [%x[args_ptr], %[offsetof_num_strings]]\n" - "add x13, x13, #0x1\n" - "cmp x13, x8\n" + "ldr w26, [%x[args_ptr], %[offsetof_num_strings]]\n" + "add x7, x7, #0x1\n" + "cmp x7, x26\n" "bne 114b\n" - "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n" - "prfm pstl1keep, [x14, #0x0]\n" - "add x27, x14, x8, LSL #2\n" - "prfm pstl1keep, [x27, #0x0]\n" - "add x26, x27, x8, LSL #2\n" - "prfm pstl1keep, [x26, #0x0]\n" - "add x25, x26, x8, LSL #2\n" - "prfm pstl1keep, [x25, #0x0]\n" - "add x24, x25, x8, LSL #2\n" - "prfm pstl1keep, [x24, #0x0]\n" - "add x23, x24, x8, LSL #2\n" - "prfm pstl1keep, [x23, #0x0]\n" + "ldr x26, [%x[args_ptr], %[offsetof_output_offset]]\n" + "add x13, x6, x26, LSL #2\n" + "add x12, x13, x26, LSL #2\n" + "add x11, x12, x26, LSL #2\n" + "add x10, x11, x26, LSL #2\n" + "add x9, x10, x26, LSL #2\n" + "prfm pstl1keep, [x6, #0x0]\n" + "prfm pstl1keep, [x13, #0x0]\n" + "prfm pstl1keep, [x12, #0x0]\n" + "prfm pstl1keep, [x11, #0x0]\n" + "prfm pstl1keep, [x10, #0x0]\n" + "prfm pstl1keep, [x9, #0x0]\n" "tbz %x[flags], #1, 122f\n" - "add x20, %x[args_ptr], %[offset_min]\n" - "add x8, %x[args_ptr], %[offset_max]\n" - "ld1r { v17.4s }, [x20]\n" - "ld1r { v16.4s }, [x8]\n" + "add x26, %x[args_ptr], %[offset_max]\n" + "ld1r { v16.4s }, [x26]\n" "fmin v24.4s, v24.4s, v16.4s\n" "fmin v25.4s, v25.4s, v16.4s\n" "fmin v26.4s, v26.4s, v16.4s\n" "fmin v27.4s, v27.4s, v16.4s\n" "fmin v28.4s, v28.4s, v16.4s\n" "fmin v29.4s, v29.4s, v16.4s\n" - "fmax v24.4s, v24.4s, v17.4s\n" - "fmax v25.4s, v25.4s, v17.4s\n" - "fmax v26.4s, v26.4s, v17.4s\n" - "fmax v27.4s, v27.4s, v17.4s\n" - "fmax v28.4s, v28.4s, v17.4s\n" - "fmax v29.4s, v29.4s, v17.4s\n" + "add x26, %x[args_ptr], %[offset_min]\n" + "ld1r { v16.4s }, [x26]\n" + "fmax v24.4s, v24.4s, v16.4s\n" + "fmax v25.4s, v25.4s, v16.4s\n" + "fmax v26.4s, v26.4s, v16.4s\n" + "fmax v27.4s, v27.4s, v16.4s\n" + "fmax v28.4s, v28.4s, v16.4s\n" + "fmax v29.4s, v29.4s, v16.4s\n" "122:" // Height 6: No activation - "cmp x17, #0x4\n" + "cmp x4, #0x4\n" "bge 125f\n" - "tbz x17, #1, 123f\n" - "str d24, [x14], #0x8\n" - "str d25, [x27], #0x8\n" - "str d26, [x26], #0x8\n" - "str d27, [x25], #0x8\n" - "str d28, [x24], #0x8\n" - "str d29, [x23], #0x8\n" - "tbz x17, #0, 124f\n" - "st1 { v24.s }[2], [x14]\n" - "st1 { v25.s }[2], [x27]\n" - "st1 { v26.s }[2], [x26]\n" - "st1 { v27.s }[2], [x25]\n" - "st1 { v28.s }[2], [x24]\n" - "st1 { v29.s }[2], [x23]\n" + "tbz x4, #1, 123f\n" + "str d24, [x6], #0x8\n" + "str d25, [x13], #0x8\n" + "str d26, [x12], #0x8\n" + "str d27, [x11], #0x8\n" + "str d28, [x10], #0x8\n" + "str d29, [x9], #0x8\n" + "tbz x4, #0, 124f\n" + "st1 { v24.s }[2], [x6]\n" + "st1 { v25.s }[2], [x13]\n" + "st1 { v26.s }[2], [x12]\n" + "st1 { v27.s }[2], [x11]\n" + "st1 { v28.s }[2], [x10]\n" + "st1 { v29.s }[2], [x9]\n" "b 124f\n" "123:" // Height 6: Partial direct writeback: partial_1_0 - "str s24, [x14, #0x0]\n" - "str s25, [x27, #0x0]\n" - "str s26, [x26, #0x0]\n" - "str s27, [x25, #0x0]\n" - "str s28, [x24, #0x0]\n" - "str s29, [x23, #0x0]\n" + "str s24, [x6, #0x0]\n" + "str s25, [x13, #0x0]\n" + "str s26, [x12, #0x0]\n" + "str s27, [x11, #0x0]\n" + "str s28, [x10, #0x0]\n" + "str s29, [x9, #0x0]\n" "124:" // Height 6: Partial direct writeback: Done "b 126f\n" "125:" // Height 6: Full writeback - "str q24, [x14, #0x0]\n" - "add x14, x14, #0x10\n" - "str q25, [x27, #0x0]\n" - "str q26, [x26, #0x0]\n" - "str q27, [x25, #0x0]\n" - "str q28, [x24, #0x0]\n" - "str q29, [x23, #0x0]\n" + "str q24, [x6, #0x0]\n" + "add x6, x6, #0x10\n" + "str q25, [x13, #0x0]\n" + "str q26, [x12, #0x0]\n" + "str q27, [x11, #0x0]\n" + "str q28, [x10, #0x0]\n" + "str q29, [x9, #0x0]\n" "126:" // Height 6: Writeback done - "subs x17, x17, #0x4\n" + "subs x4, x4, #0x4\n" "bgt 107b\n" "b 170f\n" "127:" // Height 7 - "ldr x17, [%x[args_ptr], %[offsetof_N]]\n" - "mov x15, %x[bias]\n" - "ldr x16, [%x[args_ptr], %[offsetof_B_ptr]]\n" - "mov x14, %x[output_ptr]\n" + "mov x3, %x[bias]\n" + "ldr x4, [%x[args_ptr], %[offsetof_N]]\n" + "ldr x5, [%x[args_ptr], %[offsetof_B_ptr]]\n" + "mov x6, %x[output_ptr]\n" "128:" // Height 7: Column loop - "cbz x15, 129f\n" - "ldr q24, [x15, #0x0]\n" - "add x15, x15, #0x10\n" + "cbz x3, 129f\n" + "ldr q24, [x3, #0x0]\n" "mov v25.16b, v24.16b\n" "mov v26.16b, v24.16b\n" + "add x3, x3, #0x10\n" "mov v27.16b, v24.16b\n" "mov v28.16b, v24.16b\n" "mov v29.16b, v24.16b\n" @@ -1470,53 +1469,53 @@ void a64_hybrid_fp32_mla_8x4_a55 ( "b 134f\n" "129:" // Height 7: no bias "tbz %x[flags], #0, 133f\n" - "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n" - "cmp x17, #0x4\n" - "add x27, x14, x8, LSL #2\n" - "add x26, x27, x8, LSL #2\n" - "add x25, x26, x8, LSL #2\n" - "add x24, x25, x8, LSL #2\n" - "add x23, x24, x8, LSL #2\n" - "add x22, x23, x8, LSL #2\n" + "ldr x26, [%x[args_ptr], %[offsetof_output_offset]]\n" + "add x13, x6, x26, LSL #2\n" + "add x12, x13, x26, LSL #2\n" + "add x11, x12, x26, LSL #2\n" + "add x10, x11, x26, LSL #2\n" + "add x9, x10, x26, LSL #2\n" + "cmp x4, #0x4\n" + "add x28, x9, x26, LSL #2\n" "bge 132f\n" - "tbz x17, #1, 130f\n" - "ldr d24, [x14], #0x8\n" - "ldr d25, [x27], #0x8\n" - "mov x8, #0x8\n" - "ldr d26, [x26], #0x8\n" - "ldr d27, [x25], #0x8\n" - "ldr d28, [x24], #0x8\n" - "ldr d29, [x23], #0x8\n" - "ldr d30, [x22], #0x8\n" - "tbz x17, #0, 131f\n" - "ld1 { v24.s }[2], [x14]\n" - "ld1 { v25.s }[2], [x27]\n" - "ld1 { v26.s }[2], [x26]\n" - "ld1 { v27.s }[2], [x25]\n" - "ld1 { v28.s }[2], [x24]\n" - "ld1 { v29.s }[2], [x23]\n" - "ld1 { v30.s }[2], [x22]\n" + "tbz x4, #1, 130f\n" + "ldr d24, [x6], #0x8\n" + "mov x26, #0x8\n" + "ldr d25, [x13], #0x8\n" + "ldr d26, [x12], #0x8\n" + "ldr d27, [x11], #0x8\n" + "ldr d28, [x10], #0x8\n" + "ldr d29, [x9], #0x8\n" + "ldr d30, [x28], #0x8\n" + "tbz x4, #0, 131f\n" + "ld1 { v24.s }[2], [x6]\n" + "ld1 { v25.s }[2], [x13]\n" + "ld1 { v26.s }[2], [x12]\n" + "ld1 { v27.s }[2], [x11]\n" + "ld1 { v28.s }[2], [x10]\n" + "ld1 { v29.s }[2], [x9]\n" + "ld1 { v30.s }[2], [x28]\n" "b 131f\n" "130:" // Height 7: Partial accumulate: partial_1_0 - "ldr s24, [x14, #0x0]\n" - "mov x8, #0x0\n" - "ldr s25, [x27, #0x0]\n" - "ldr s26, [x26, #0x0]\n" - "ldr s27, [x25, #0x0]\n" - "ldr s28, [x24, #0x0]\n" - "ldr s29, [x23, #0x0]\n" - "ldr s30, [x22, #0x0]\n" + "ldr s24, [x6, #0x0]\n" + "mov x26, #0x0\n" + "ldr s25, [x13, #0x0]\n" + "ldr s26, [x12, #0x0]\n" + "ldr s27, [x11, #0x0]\n" + "ldr s28, [x10, #0x0]\n" + "ldr s29, [x9, #0x0]\n" + "ldr s30, [x28, #0x0]\n" "131:" // Height 7: Partial accumulate: Done - "sub x14, x14, x8\n" + "sub x6, x6, x26\n" "b 134f\n" "132:" // Height 7: full accumulate - "ldr q24, [x14, #0x0]\n" - "ldr q25, [x27, #0x0]\n" - "ldr q26, [x26, #0x0]\n" - "ldr q27, [x25, #0x0]\n" - "ldr q28, [x24, #0x0]\n" - "ldr q29, [x23, #0x0]\n" - "ldr q30, [x22, #0x0]\n" + "ldr q24, [x6, #0x0]\n" + "ldr q25, [x13, #0x0]\n" + "ldr q26, [x12, #0x0]\n" + "ldr q27, [x11, #0x0]\n" + "ldr q28, [x10, #0x0]\n" + "ldr q29, [x9, #0x0]\n" + "ldr q30, [x28, #0x0]\n" "b 134f\n" "133:" // Height 7: no accumulate "movi v24.16b, #0x0\n" @@ -1527,171 +1526,171 @@ void a64_hybrid_fp32_mla_8x4_a55 ( "movi v29.16b, #0x0\n" "movi v30.16b, #0x0\n" "134:" // Height 7: setup done - "mov x13, #0x0\n" + "mov x7, #0x0\n" "135:" // Height 7: String loop - "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr x8, [%x[args_ptr], %[offsetof_input_offset]]\n" - "ldr w12, [x20, x13, LSL #0x2]\n" + "ldr x26, [%x[args_ptr], %[offsetof_string_lengths]]\n" + "ldr w8, [x26, x7, LSL #0x2]\n" + "ldr x27, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 136f\n" - "ldr x20, [%x[input_ptr], x13, LSL #0x3]\n" - "add x20, x20, x8, LSL #3\n" - "ldr x11, [x20, #0x0]\n" - "ldr x9, [x20, #0x8]\n" - "ldr x27, [x20, #0x10]\n" - "ldr x25, [x20, #0x18]\n" - "ldr x24, [x20, #0x20]\n" - "ldr x23, [x20, #0x28]\n" - "ldr x22, [x20, #0x30]\n" - "cbnz x13, 137f\n" - "ldr x8, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x11, x11, x8, LSL #2\n" - "add x9, x9, x8, LSL #2\n" - "add x27, x27, x8, LSL #2\n" - "add x25, x25, x8, LSL #2\n" - "add x24, x24, x8, LSL #2\n" - "add x23, x23, x8, LSL #2\n" - "add x22, x22, x8, LSL #2\n" + "ldr x26, [%x[input_ptr], x7, LSL #0x3]\n" + "add x26, x26, x27, LSL #3\n" + "ldr x17, [x26, #0x0]\n" + "ldr x16, [x26, #0x8]\n" + "ldr x15, [x26, #0x10]\n" + "ldr x14, [x26, #0x18]\n" + "ldr x13, [x26, #0x20]\n" + "ldr x12, [x26, #0x28]\n" + "ldr x11, [x26, #0x30]\n" + "cbnz x7, 137f\n" + "ldr x26, [%x[args_ptr], %[offsetof_input_initial_col]]\n" + "add x17, x17, x26, LSL #2\n" + "add x16, x16, x26, LSL #2\n" + "add x15, x15, x26, LSL #2\n" + "add x14, x14, x26, LSL #2\n" + "add x13, x13, x26, LSL #2\n" + "add x12, x12, x26, LSL #2\n" + "add x11, x11, x26, LSL #2\n" "b 137f\n" "136:" // Height 7: setup direct input - "mov x11, %x[input_ptr]\n" - "add x9, x11, x8, LSL #2\n" - "add x27, x9, x8, LSL #2\n" - "add x25, x27, x8, LSL #2\n" - "add x24, x25, x8, LSL #2\n" - "add x23, x24, x8, LSL #2\n" - "add x22, x23, x8, LSL #2\n" + "mov x17, %x[input_ptr]\n" + "add x16, x17, x27, LSL #2\n" + "add x15, x16, x27, LSL #2\n" + "add x14, x15, x27, LSL #2\n" + "add x13, x14, x27, LSL #2\n" + "add x12, x13, x27, LSL #2\n" + "add x11, x12, x27, LSL #2\n" "137:" // Height 7: input setup done - "cmp x12, #0x4\n" + "cmp x8, #0x4\n" "blt 140f\n" - "ldr q0, [x11, #0x0]\n" - "ldr q1, [x9, #0x0]\n" - "cmp x12, #0x8\n" - "ldr q2, [x27, #0x0]\n" - "ldr q3, [x25, #0x0]\n" - "ldr q4, [x24, #0x0]\n" - "ldr q5, [x23, #0x0]\n" - "ldr q6, [x22, #0x0]\n" - "ldr q8, [x16, #0x0]\n" + "ldr q0, [x17, #0x0]\n" + "cmp x8, #0x8\n" + "ldr q1, [x16, #0x0]\n" + "ldr q2, [x15, #0x0]\n" + "ldr q3, [x14, #0x0]\n" + "ldr q4, [x13, #0x0]\n" + "ldr q5, [x12, #0x0]\n" + "ldr q6, [x11, #0x0]\n" + "ldr q8, [x5, #0x0]\n" + "ldr q9, [x5, #0x10]\n" + "ldr q10, [x5, #0x20]\n" + "ldr q11, [x5, #0x30]\n" "blt 139f\n" "138:" // Height 7: Multiply loop: Main loop head "fmla v24.4s, v8.4s, v0.s[0]\n" - "ldr d9, [x16, #0x10]\n" + "add x17, x17, #0x10\n" "fmla v25.4s, v8.4s, v1.s[0]\n" - "ldr x8, [x16, #0x18]\n" + "add x16, x16, #0x10\n" "fmla v26.4s, v8.4s, v2.s[0]\n" - "ldr d10, [x16, #0x20]\n" + "add x15, x15, #0x10\n" "fmla v27.4s, v8.4s, v3.s[0]\n" - "ldr x21, [x16, #0x28]\n" + "add x14, x14, #0x10\n" "fmla v28.4s, v8.4s, v4.s[0]\n" - "mov v9.d[1], x8\n" + "add x13, x13, #0x10\n" "fmla v29.4s, v8.4s, v5.s[0]\n" - "ldr d11, [x16, #0x30]\n" + "add x12, x12, #0x10\n" "fmla v30.4s, v8.4s, v6.s[0]\n" - "mov v10.d[1], x21\n" + "add x11, x11, #0x10\n" "fmla v24.4s, v9.4s, v0.s[1]\n" - "ldr x8, [x16, #0x38]\n" + "add x5, x5, #0x40\n" + "ldr d8, [x5, #0x0]\n" "fmla v25.4s, v9.4s, v1.s[1]\n" - "add x11, x11, #0x10\n" "fmla v26.4s, v9.4s, v2.s[1]\n" - "prfm pldl1keep, [x11, #0x80]\n" + "ldr x26, [x5, #0x8]\n" "fmla v27.4s, v9.4s, v3.s[1]\n" - "mov v11.d[1], x8\n" + "ldr x10, [x5, #0x18]\n" "fmla v28.4s, v9.4s, v4.s[1]\n" - "ldr x10, [x11, #0x8]\n" + "ldr x9, [x5, #0x28]\n" "fmla v29.4s, v9.4s, v5.s[1]\n" - "add x9, x9, #0x10\n" + "ldr x28, [x17, #0x8]\n" "fmla v30.4s, v9.4s, v6.s[1]\n" - "prfm pldl1keep, [x9, #0x80]\n" + "ldr d9, [x5, #0x10]\n" "fmla v24.4s, v10.4s, v0.s[2]\n" - "ldr x28, [x9, #0x8]\n" + "ldr x27, [x16, #0x8]\n" "fmla v25.4s, v10.4s, v1.s[2]\n" - "add x27, x27, #0x10\n" + "sub x8, x8, #0x4\n" "fmla v26.4s, v10.4s, v2.s[2]\n" - "prfm pldl1keep, [x27, #0x80]\n" + "cmp x8, #0x8\n" "fmla v27.4s, v10.4s, v3.s[2]\n" - "ldr x26, [x27, #0x8]\n" + "mov v8.d[1], x26\n" "fmla v28.4s, v10.4s, v4.s[2]\n" - "add x25, x25, #0x10\n" + "ldr x26, [x15, #0x8]\n" "fmla v29.4s, v10.4s, v5.s[2]\n" - "prfm pldl1keep, [x25, #0x80]\n" + "prfm pldl1keep, [x17, #0x80]\n" "fmla v30.4s, v10.4s, v6.s[2]\n" - "ldr x8, [x25, #0x8]\n" + "ldr d10, [x5, #0x20]\n" "fmla v24.4s, v11.4s, v0.s[3]\n" - "ldr d0, [x11, #0x0]\n" + "ldr d0, [x17, #0x0]\n" "fmla v25.4s, v11.4s, v1.s[3]\n" - "ldr d1, [x9, #0x0]\n" + "ldr d1, [x16, #0x0]\n" "fmla v26.4s, v11.4s, v2.s[3]\n" - "ldr d2, [x27, #0x0]\n" + "ldr d2, [x15, #0x0]\n" "fmla v27.4s, v11.4s, v3.s[3]\n" - "mov v0.d[1], x10\n" + "ldr d3, [x14, #0x0]\n" "fmla v28.4s, v11.4s, v4.s[3]\n" - "mov v1.d[1], x28\n" + "ldr d4, [x13, #0x0]\n" "fmla v29.4s, v11.4s, v5.s[3]\n" - "mov v2.d[1], x26\n" + "ldr d5, [x12, #0x0]\n" "fmla v30.4s, v11.4s, v6.s[3]\n" - "ldr d3, [x25, #0x0]\n" - "add x24, x24, #0x10\n" - "add x23, x23, #0x10\n" - "prfm pldl1keep, [x24, #0x80]\n" - "add x22, x22, #0x10\n" - "mov v3.d[1], x8\n" - "prfm pldl1keep, [x23, #0x80]\n" - "prfm pldl1keep, [x22, #0x80]\n" - "sub x12, x12, #0x4\n" - "ldr d4, [x24, #0x0]\n" - "cmp x12, #0x8\n" - "ldr x21, [x24, #0x8]\n" - "add x16, x16, #0x40\n" - "ldr d8, [x16, #0x0]\n" - "ldr x26, [x16, #0x8]\n" - "mov v4.d[1], x21\n" - "ldr d5, [x23, #0x0]\n" - "ldr x8, [x23, #0x8]\n" - "mov v8.d[1], x26\n" - "ldr d6, [x22, #0x0]\n" - "ldr x21, [x22, #0x8]\n" - "mov v5.d[1], x8\n" - "mov v6.d[1], x21\n" + "ldr d6, [x11, #0x0]\n" + "ldr d11, [x5, #0x30]\n" + "mov v9.d[1], x10\n" + "ldr x10, [x14, #0x8]\n" + "mov v10.d[1], x9\n" + "ldr x9, [x13, #0x8]\n" + "mov v0.d[1], x28\n" + "ldr x28, [x12, #0x8]\n" + "mov v1.d[1], x27\n" + "ldr x27, [x11, #0x8]\n" + "mov v2.d[1], x26\n" + "ldr x26, [x5, #0x38]\n" + "mov v3.d[1], x10\n" + "prfm pldl1keep, [x16, #0x80]\n" + "mov v4.d[1], x9\n" + "prfm pldl1keep, [x15, #0x80]\n" + "mov v5.d[1], x28\n" + "prfm pldl1keep, [x14, #0x80]\n" + "mov v6.d[1], x27\n" + "prfm pldl1keep, [x13, #0x80]\n" + "mov v11.d[1], x26\n" + "prfm pldl1keep, [x12, #0x80]\n" + "prfm pldl1keep, [x11, #0x80]\n" "bge 138b\n" "139:" // Height 7: Multiply loop: Single iteration only "fmla v24.4s, v8.4s, v0.s[0]\n" - "ldr q9, [x16, #0x10]\n" + "add x17, x17, #0x10\n" "fmla v25.4s, v8.4s, v1.s[0]\n" - "ldr q10, [x16, #0x20]\n" + "add x16, x16, #0x10\n" "fmla v26.4s, v8.4s, v2.s[0]\n" - "ldr q11, [x16, #0x30]\n" + "add x15, x15, #0x10\n" "fmla v27.4s, v8.4s, v3.s[0]\n" - "sub x12, x12, #0x4\n" + "add x14, x14, #0x10\n" "fmla v28.4s, v8.4s, v4.s[0]\n" - "add x11, x11, #0x10\n" + "add x13, x13, #0x10\n" "fmla v29.4s, v8.4s, v5.s[0]\n" - "prfm pldl1keep, [x11, #0x80]\n" + "add x12, x12, #0x10\n" "fmla v30.4s, v8.4s, v6.s[0]\n" - "add x9, x9, #0x10\n" + "add x11, x11, #0x10\n" "fmla v24.4s, v9.4s, v0.s[1]\n" - "prfm pldl1keep, [x9, #0x80]\n" + "sub x8, x8, #0x4\n" "fmla v25.4s, v9.4s, v1.s[1]\n" - "add x27, x27, #0x10\n" + "prfm pldl1keep, [x17, #0x80]\n" "fmla v26.4s, v9.4s, v2.s[1]\n" - "prfm pldl1keep, [x27, #0x80]\n" + "prfm pldl1keep, [x16, #0x80]\n" "fmla v27.4s, v9.4s, v3.s[1]\n" - "add x25, x25, #0x10\n" + "prfm pldl1keep, [x15, #0x80]\n" "fmla v28.4s, v9.4s, v4.s[1]\n" - "prfm pldl1keep, [x25, #0x80]\n" + "prfm pldl1keep, [x14, #0x80]\n" "fmla v29.4s, v9.4s, v5.s[1]\n" - "add x24, x24, #0x10\n" + "prfm pldl1keep, [x13, #0x80]\n" "fmla v30.4s, v9.4s, v6.s[1]\n" - "prfm pldl1keep, [x24, #0x80]\n" + "prfm pldl1keep, [x12, #0x80]\n" "fmla v24.4s, v10.4s, v0.s[2]\n" - "add x23, x23, #0x10\n" + "prfm pldl1keep, [x11, #0x80]\n" "fmla v25.4s, v10.4s, v1.s[2]\n" - "prfm pldl1keep, [x23, #0x80]\n" + "add x5, x5, #0x40\n" "fmla v26.4s, v10.4s, v2.s[2]\n" - "add x22, x22, #0x10\n" "fmla v27.4s, v10.4s, v3.s[2]\n" - "prfm pldl1keep, [x22, #0x80]\n" "fmla v28.4s, v10.4s, v4.s[2]\n" - "add x16, x16, #0x40\n" "fmla v29.4s, v10.4s, v5.s[2]\n" "fmla v30.4s, v10.4s, v6.s[2]\n" "fmla v24.4s, v11.4s, v0.s[3]\n" @@ -1702,50 +1701,48 @@ void a64_hybrid_fp32_mla_8x4_a55 ( "fmla v29.4s, v11.4s, v5.s[3]\n" "fmla v30.4s, v11.4s, v6.s[3]\n" "140:" // Height 7: Multiply loop: Main loop skip - "cbz x12, 142f\n" + "cbz x8, 142f\n" "141:" // Height 7: Multiply loop: Odd block loop - "ldr s0, [x11], #0x4\n" - "sub x12, x12, #0x1\n" - "ldr s1, [x9], #0x4\n" - "ldr s2, [x27], #0x4\n" - "ldr s3, [x25], #0x4\n" - "ldr s4, [x24], #0x4\n" - "ldr s5, [x23], #0x4\n" - "ldr s6, [x22], #0x4\n" - "ldr q12, [x16, #0x0]\n" - "add x16, x16, #0x10\n" - "fmla v24.4s, v12.4s, v0.s[0]\n" - "fmla v25.4s, v12.4s, v1.s[0]\n" - "fmla v26.4s, v12.4s, v2.s[0]\n" - "fmla v27.4s, v12.4s, v3.s[0]\n" - "fmla v28.4s, v12.4s, v4.s[0]\n" - "fmla v29.4s, v12.4s, v5.s[0]\n" - "fmla v30.4s, v12.4s, v6.s[0]\n" - "cbnz x12, 141b\n" + "ldr s23, [x17], #0x4\n" + "sub x8, x8, #0x1\n" + "ldr s22, [x16], #0x4\n" + "ldr s21, [x15], #0x4\n" + "ldr s20, [x14], #0x4\n" + "ldr s19, [x13], #0x4\n" + "ldr s18, [x12], #0x4\n" + "ldr s17, [x11], #0x4\n" + "ldr q16, [x5, #0x0]\n" + "fmla v24.4s, v16.4s, v23.s[0]\n" + "fmla v25.4s, v16.4s, v22.s[0]\n" + "add x5, x5, #0x10\n" + "fmla v26.4s, v16.4s, v21.s[0]\n" + "fmla v27.4s, v16.4s, v20.s[0]\n" + "fmla v28.4s, v16.4s, v19.s[0]\n" + "fmla v29.4s, v16.4s, v18.s[0]\n" + "fmla v30.4s, v16.4s, v17.s[0]\n" + "cbnz x8, 141b\n" "142:" // Height 7: Multiply loop: No odd multiplies - "ldr w8, [%x[args_ptr], %[offsetof_num_strings]]\n" - "add x13, x13, #0x1\n" - "cmp x13, x8\n" + "ldr w26, [%x[args_ptr], %[offsetof_num_strings]]\n" + "add x7, x7, #0x1\n" + "cmp x7, x26\n" "bne 135b\n" - "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n" - "prfm pstl1keep, [x14, #0x0]\n" - "add x27, x14, x8, LSL #2\n" - "prfm pstl1keep, [x27, #0x0]\n" - "add x26, x27, x8, LSL #2\n" - "prfm pstl1keep, [x26, #0x0]\n" - "add x25, x26, x8, LSL #2\n" - "prfm pstl1keep, [x25, #0x0]\n" - "add x24, x25, x8, LSL #2\n" - "prfm pstl1keep, [x24, #0x0]\n" - "add x23, x24, x8, LSL #2\n" - "prfm pstl1keep, [x23, #0x0]\n" - "add x22, x23, x8, LSL #2\n" - "prfm pstl1keep, [x22, #0x0]\n" + "ldr x26, [%x[args_ptr], %[offsetof_output_offset]]\n" + "add x13, x6, x26, LSL #2\n" + "add x12, x13, x26, LSL #2\n" + "add x11, x12, x26, LSL #2\n" + "add x10, x11, x26, LSL #2\n" + "add x9, x10, x26, LSL #2\n" + "add x28, x9, x26, LSL #2\n" + "prfm pstl1keep, [x6, #0x0]\n" + "prfm pstl1keep, [x13, #0x0]\n" + "prfm pstl1keep, [x12, #0x0]\n" + "prfm pstl1keep, [x11, #0x0]\n" + "prfm pstl1keep, [x10, #0x0]\n" + "prfm pstl1keep, [x9, #0x0]\n" + "prfm pstl1keep, [x28, #0x0]\n" "tbz %x[flags], #1, 143f\n" - "add x20, %x[args_ptr], %[offset_min]\n" - "add x8, %x[args_ptr], %[offset_max]\n" - "ld1r { v17.4s }, [x20]\n" - "ld1r { v16.4s }, [x8]\n" + "add x26, %x[args_ptr], %[offset_max]\n" + "ld1r { v16.4s }, [x26]\n" "fmin v24.4s, v24.4s, v16.4s\n" "fmin v25.4s, v25.4s, v16.4s\n" "fmin v26.4s, v26.4s, v16.4s\n" @@ -1753,70 +1750,72 @@ void a64_hybrid_fp32_mla_8x4_a55 ( "fmin v28.4s, v28.4s, v16.4s\n" "fmin v29.4s, v29.4s, v16.4s\n" "fmin v30.4s, v30.4s, v16.4s\n" - "fmax v24.4s, v24.4s, v17.4s\n" - "fmax v25.4s, v25.4s, v17.4s\n" - "fmax v26.4s, v26.4s, v17.4s\n" - "fmax v27.4s, v27.4s, v17.4s\n" - "fmax v28.4s, v28.4s, v17.4s\n" - "fmax v29.4s, v29.4s, v17.4s\n" - "fmax v30.4s, v30.4s, v17.4s\n" + "add x26, %x[args_ptr], %[offset_min]\n" + "ld1r { v16.4s }, [x26]\n" + "fmax v24.4s, v24.4s, v16.4s\n" + "fmax v25.4s, v25.4s, v16.4s\n" + "fmax v26.4s, v26.4s, v16.4s\n" + "fmax v27.4s, v27.4s, v16.4s\n" + "fmax v28.4s, v28.4s, v16.4s\n" + "fmax v29.4s, v29.4s, v16.4s\n" + "fmax v30.4s, v30.4s, v16.4s\n" "143:" // Height 7: No activation - "cmp x17, #0x4\n" + "cmp x4, #0x4\n" "bge 146f\n" - "tbz x17, #1, 144f\n" - "str d24, [x14], #0x8\n" - "str d25, [x27], #0x8\n" - "str d26, [x26], #0x8\n" - "str d27, [x25], #0x8\n" - "str d28, [x24], #0x8\n" - "str d29, [x23], #0x8\n" - "str d30, [x22], #0x8\n" - "tbz x17, #0, 145f\n" - "st1 { v24.s }[2], [x14]\n" - "st1 { v25.s }[2], [x27]\n" - "st1 { v26.s }[2], [x26]\n" - "st1 { v27.s }[2], [x25]\n" - "st1 { v28.s }[2], [x24]\n" - "st1 { v29.s }[2], [x23]\n" - "st1 { v30.s }[2], [x22]\n" + "tbz x4, #1, 144f\n" + "str d24, [x6], #0x8\n" + "str d25, [x13], #0x8\n" + "str d26, [x12], #0x8\n" + "str d27, [x11], #0x8\n" + "str d28, [x10], #0x8\n" + "str d29, [x9], #0x8\n" + "str d30, [x28], #0x8\n" + "tbz x4, #0, 145f\n" + "st1 { v24.s }[2], [x6]\n" + "st1 { v25.s }[2], [x13]\n" + "st1 { v26.s }[2], [x12]\n" + "st1 { v27.s }[2], [x11]\n" + "st1 { v28.s }[2], [x10]\n" + "st1 { v29.s }[2], [x9]\n" + "st1 { v30.s }[2], [x28]\n" "b 145f\n" "144:" // Height 7: Partial direct writeback: partial_1_0 - "str s24, [x14, #0x0]\n" - "str s25, [x27, #0x0]\n" - "str s26, [x26, #0x0]\n" - "str s27, [x25, #0x0]\n" - "str s28, [x24, #0x0]\n" - "str s29, [x23, #0x0]\n" - "str s30, [x22, #0x0]\n" + "str s24, [x6, #0x0]\n" + "str s25, [x13, #0x0]\n" + "str s26, [x12, #0x0]\n" + "str s27, [x11, #0x0]\n" + "str s28, [x10, #0x0]\n" + "str s29, [x9, #0x0]\n" + "str s30, [x28, #0x0]\n" "145:" // Height 7: Partial direct writeback: Done "b 147f\n" "146:" // Height 7: Full writeback - "str q24, [x14, #0x0]\n" - "add x14, x14, #0x10\n" - "str q25, [x27, #0x0]\n" - "str q26, [x26, #0x0]\n" - "str q27, [x25, #0x0]\n" - "str q28, [x24, #0x0]\n" - "str q29, [x23, #0x0]\n" - "str q30, [x22, #0x0]\n" + "str q24, [x6, #0x0]\n" + "add x6, x6, #0x10\n" + "str q25, [x13, #0x0]\n" + "str q26, [x12, #0x0]\n" + "str q27, [x11, #0x0]\n" + "str q28, [x10, #0x0]\n" + "str q29, [x9, #0x0]\n" + "str q30, [x28, #0x0]\n" "147:" // Height 7: Writeback done - "subs x17, x17, #0x4\n" + "subs x4, x4, #0x4\n" "bgt 128b\n" "b 170f\n" "148:" // Height 8 - "ldr x17, [%x[args_ptr], %[offsetof_N]]\n" - "mov x15, %x[bias]\n" - "ldr x16, [%x[args_ptr], %[offsetof_B_ptr]]\n" - "mov x14, %x[output_ptr]\n" - "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "mov x8, #0x20\n" - "madd %x[output_ptr], x20, x8, %x[output_ptr]\n" + "ldr x27, [%x[args_ptr], %[offsetof_output_offset]]\n" + "mov x26, #0x20\n" + "mov x3, %x[bias]\n" + "ldr x4, [%x[args_ptr], %[offsetof_N]]\n" + "ldr x5, [%x[args_ptr], %[offsetof_B_ptr]]\n" + "mov x6, %x[output_ptr]\n" + "madd %x[output_ptr], x27, x26, %x[output_ptr]\n" "149:" // Height 8: Column loop - "cbz x15, 150f\n" - "ldr q24, [x15, #0x0]\n" - "add x15, x15, #0x10\n" + "cbz x3, 150f\n" + "ldr q24, [x3, #0x0]\n" "mov v25.16b, v24.16b\n" "mov v26.16b, v24.16b\n" + "add x3, x3, #0x10\n" "mov v27.16b, v24.16b\n" "mov v28.16b, v24.16b\n" "mov v29.16b, v24.16b\n" @@ -1825,58 +1824,58 @@ void a64_hybrid_fp32_mla_8x4_a55 ( "b 155f\n" "150:" // Height 8: no bias "tbz %x[flags], #0, 154f\n" - "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n" - "cmp x17, #0x4\n" - "add x27, x14, x8, LSL #2\n" - "add x26, x27, x8, LSL #2\n" - "add x25, x26, x8, LSL #2\n" - "add x24, x25, x8, LSL #2\n" - "add x23, x24, x8, LSL #2\n" - "add x22, x23, x8, LSL #2\n" - "add x21, x22, x8, LSL #2\n" + "ldr x26, [%x[args_ptr], %[offsetof_output_offset]]\n" + "add x13, x6, x26, LSL #2\n" + "add x12, x13, x26, LSL #2\n" + "add x11, x12, x26, LSL #2\n" + "add x10, x11, x26, LSL #2\n" + "add x9, x10, x26, LSL #2\n" + "add x28, x9, x26, LSL #2\n" + "cmp x4, #0x4\n" + "add x27, x28, x26, LSL #2\n" "bge 153f\n" - "tbz x17, #1, 151f\n" - "ldr d24, [x14], #0x8\n" - "ldr d25, [x27], #0x8\n" - "mov x8, #0x8\n" - "ldr d26, [x26], #0x8\n" - "ldr d27, [x25], #0x8\n" - "ldr d28, [x24], #0x8\n" - "ldr d29, [x23], #0x8\n" - "ldr d30, [x22], #0x8\n" - "ldr d31, [x21], #0x8\n" - "tbz x17, #0, 152f\n" - "ld1 { v24.s }[2], [x14]\n" - "ld1 { v25.s }[2], [x27]\n" - "ld1 { v26.s }[2], [x26]\n" - "ld1 { v27.s }[2], [x25]\n" - "ld1 { v28.s }[2], [x24]\n" - "ld1 { v29.s }[2], [x23]\n" - "ld1 { v30.s }[2], [x22]\n" - "ld1 { v31.s }[2], [x21]\n" + "tbz x4, #1, 151f\n" + "ldr d24, [x6], #0x8\n" + "mov x26, #0x8\n" + "ldr d25, [x13], #0x8\n" + "ldr d26, [x12], #0x8\n" + "ldr d27, [x11], #0x8\n" + "ldr d28, [x10], #0x8\n" + "ldr d29, [x9], #0x8\n" + "ldr d30, [x28], #0x8\n" + "ldr d31, [x27], #0x8\n" + "tbz x4, #0, 152f\n" + "ld1 { v24.s }[2], [x6]\n" + "ld1 { v25.s }[2], [x13]\n" + "ld1 { v26.s }[2], [x12]\n" + "ld1 { v27.s }[2], [x11]\n" + "ld1 { v28.s }[2], [x10]\n" + "ld1 { v29.s }[2], [x9]\n" + "ld1 { v30.s }[2], [x28]\n" + "ld1 { v31.s }[2], [x27]\n" "b 152f\n" "151:" // Height 8: Partial accumulate: partial_1_0 - "ldr s24, [x14, #0x0]\n" - "mov x8, #0x0\n" - "ldr s25, [x27, #0x0]\n" - "ldr s26, [x26, #0x0]\n" - "ldr s27, [x25, #0x0]\n" - "ldr s28, [x24, #0x0]\n" - "ldr s29, [x23, #0x0]\n" - "ldr s30, [x22, #0x0]\n" - "ldr s31, [x21, #0x0]\n" + "ldr s24, [x6, #0x0]\n" + "mov x26, #0x0\n" + "ldr s25, [x13, #0x0]\n" + "ldr s26, [x12, #0x0]\n" + "ldr s27, [x11, #0x0]\n" + "ldr s28, [x10, #0x0]\n" + "ldr s29, [x9, #0x0]\n" + "ldr s30, [x28, #0x0]\n" + "ldr s31, [x27, #0x0]\n" "152:" // Height 8: Partial accumulate: Done - "sub x14, x14, x8\n" + "sub x6, x6, x26\n" "b 155f\n" "153:" // Height 8: full accumulate - "ldr q24, [x14, #0x0]\n" - "ldr q25, [x27, #0x0]\n" - "ldr q26, [x26, #0x0]\n" - "ldr q27, [x25, #0x0]\n" - "ldr q28, [x24, #0x0]\n" - "ldr q29, [x23, #0x0]\n" - "ldr q30, [x22, #0x0]\n" - "ldr q31, [x21, #0x0]\n" + "ldr q24, [x6, #0x0]\n" + "ldr q25, [x13, #0x0]\n" + "ldr q26, [x12, #0x0]\n" + "ldr q27, [x11, #0x0]\n" + "ldr q28, [x10, #0x0]\n" + "ldr q29, [x9, #0x0]\n" + "ldr q30, [x28, #0x0]\n" + "ldr q31, [x27, #0x0]\n" "b 155f\n" "154:" // Height 8: no accumulate "movi v24.16b, #0x0\n" @@ -1888,188 +1887,188 @@ void a64_hybrid_fp32_mla_8x4_a55 ( "movi v30.16b, #0x0\n" "movi v31.16b, #0x0\n" "155:" // Height 8: setup done - "mov x13, #0x0\n" + "mov x7, #0x0\n" "156:" // Height 8: String loop - "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr x8, [%x[args_ptr], %[offsetof_input_offset]]\n" - "ldr w12, [x20, x13, LSL #0x2]\n" + "ldr x26, [%x[args_ptr], %[offsetof_string_lengths]]\n" + "ldr w8, [x26, x7, LSL #0x2]\n" + "ldr x27, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 157f\n" - "ldr x20, [%x[input_ptr], x13, LSL #0x3]\n" - "add x20, x20, x8, LSL #3\n" - "ldr x11, [x20, #0x0]\n" - "ldr x9, [x20, #0x8]\n" - "ldr x27, [x20, #0x10]\n" - "ldr x25, [x20, #0x18]\n" - "ldr x24, [x20, #0x20]\n" - "ldr x23, [x20, #0x28]\n" - "ldr x22, [x20, #0x30]\n" - "ldr x20, [x20, #0x38]\n" - "cbnz x13, 158f\n" - "ldr x8, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x11, x11, x8, LSL #2\n" - "add x9, x9, x8, LSL #2\n" - "add x27, x27, x8, LSL #2\n" - "add x25, x25, x8, LSL #2\n" - "add x24, x24, x8, LSL #2\n" - "add x23, x23, x8, LSL #2\n" - "add x22, x22, x8, LSL #2\n" - "add x20, x20, x8, LSL #2\n" + "ldr x26, [%x[input_ptr], x7, LSL #0x3]\n" + "add x26, x26, x27, LSL #3\n" + "ldr x17, [x26, #0x0]\n" + "ldr x16, [x26, #0x8]\n" + "ldr x15, [x26, #0x10]\n" + "ldr x14, [x26, #0x18]\n" + "ldr x13, [x26, #0x20]\n" + "ldr x12, [x26, #0x28]\n" + "ldr x11, [x26, #0x30]\n" + "ldr x27, [x26, #0x38]\n" + "cbnz x7, 158f\n" + "ldr x26, [%x[args_ptr], %[offsetof_input_initial_col]]\n" + "add x17, x17, x26, LSL #2\n" + "add x16, x16, x26, LSL #2\n" + "add x15, x15, x26, LSL #2\n" + "add x14, x14, x26, LSL #2\n" + "add x13, x13, x26, LSL #2\n" + "add x12, x12, x26, LSL #2\n" + "add x11, x11, x26, LSL #2\n" + "add x27, x27, x26, LSL #2\n" "b 158f\n" "157:" // Height 8: setup direct input - "mov x11, %x[input_ptr]\n" - "add x9, x11, x8, LSL #2\n" - "add x27, x9, x8, LSL #2\n" - "add x25, x27, x8, LSL #2\n" - "add x24, x25, x8, LSL #2\n" - "add x23, x24, x8, LSL #2\n" - "add x22, x23, x8, LSL #2\n" - "add x20, x22, x8, LSL #2\n" + "mov x17, %x[input_ptr]\n" + "add x16, x17, x27, LSL #2\n" + "add x15, x16, x27, LSL #2\n" + "add x14, x15, x27, LSL #2\n" + "add x13, x14, x27, LSL #2\n" + "add x12, x13, x27, LSL #2\n" + "add x11, x12, x27, LSL #2\n" + "add x27, x11, x27, LSL #2\n" "158:" // Height 8: input setup done - "cmp x12, #0x4\n" + "cmp x8, #0x4\n" "blt 161f\n" - "ldr q0, [x11, #0x0]\n" - "ldr q1, [x9, #0x0]\n" - "cmp x12, #0x8\n" - "ldr q2, [x27, #0x0]\n" - "ldr q3, [x25, #0x0]\n" - "ldr q4, [x24, #0x0]\n" - "ldr q5, [x23, #0x0]\n" - "ldr q6, [x22, #0x0]\n" - "ldr q7, [x20, #0x0]\n" - "ldr q8, [x16, #0x0]\n" + "ldr q0, [x17, #0x0]\n" + "cmp x8, #0x8\n" + "ldr q1, [x16, #0x0]\n" + "ldr q2, [x15, #0x0]\n" + "ldr q3, [x14, #0x0]\n" + "ldr q4, [x13, #0x0]\n" + "ldr q5, [x12, #0x0]\n" + "ldr q6, [x11, #0x0]\n" + "ldr q7, [x27, #0x0]\n" + "ldr q8, [x5, #0x0]\n" + "ldr q9, [x5, #0x10]\n" + "ldr q10, [x5, #0x20]\n" + "ldr q11, [x5, #0x30]\n" "blt 160f\n" "159:" // Height 8: Multiply loop: Main loop head "fmla v24.4s, v8.4s, v0.s[0]\n" - "ldr d9, [x16, #0x10]\n" + "add x17, x17, #0x10\n" "fmla v25.4s, v8.4s, v1.s[0]\n" - "ldr x8, [x16, #0x18]\n" + "add x16, x16, #0x10\n" "fmla v26.4s, v8.4s, v2.s[0]\n" - "ldr d10, [x16, #0x20]\n" + "add x15, x15, #0x10\n" "fmla v27.4s, v8.4s, v3.s[0]\n" - "ldr x21, [x16, #0x28]\n" + "add x14, x14, #0x10\n" "fmla v28.4s, v8.4s, v4.s[0]\n" - "mov v9.d[1], x8\n" + "add x13, x13, #0x10\n" "fmla v29.4s, v8.4s, v5.s[0]\n" - "ldr d11, [x16, #0x30]\n" + "add x12, x12, #0x10\n" "fmla v30.4s, v8.4s, v6.s[0]\n" - "mov v10.d[1], x21\n" + "add x11, x11, #0x10\n" "fmla v31.4s, v8.4s, v7.s[0]\n" - "ldr x8, [x16, #0x38]\n" + "add x27, x27, #0x10\n" "fmla v24.4s, v9.4s, v0.s[1]\n" - "add x11, x11, #0x10\n" + "add x5, x5, #0x40\n" + "ldr d8, [x5, #0x0]\n" "fmla v25.4s, v9.4s, v1.s[1]\n" - "prfm pldl1keep, [x11, #0x80]\n" "fmla v26.4s, v9.4s, v2.s[1]\n" - "mov v11.d[1], x8\n" + "ldr x26, [x5, #0x8]\n" "fmla v27.4s, v9.4s, v3.s[1]\n" - "ldr x10, [x11, #0x8]\n" + "sub x8, x8, #0x4\n" "fmla v28.4s, v9.4s, v4.s[1]\n" - "add x9, x9, #0x10\n" + "cmp x8, #0x8\n" "fmla v29.4s, v9.4s, v5.s[1]\n" - "prfm pldl1keep, [x9, #0x80]\n" + "mov v8.d[1], x26\n" "fmla v30.4s, v9.4s, v6.s[1]\n" - "ldr x28, [x9, #0x8]\n" + "ldr x26, [x5, #0x18]\n" "fmla v31.4s, v9.4s, v7.s[1]\n" - "add x27, x27, #0x10\n" + "ldr d9, [x5, #0x10]\n" "fmla v24.4s, v10.4s, v0.s[2]\n" - "prfm pldl1keep, [x27, #0x80]\n" + "prfm pldl1keep, [x17, #0x80]\n" "fmla v25.4s, v10.4s, v1.s[2]\n" - "ldr x26, [x27, #0x8]\n" + "prfm pldl1keep, [x16, #0x80]\n" "fmla v26.4s, v10.4s, v2.s[2]\n" - "add x25, x25, #0x10\n" + "prfm pldl1keep, [x15, #0x80]\n" "fmla v27.4s, v10.4s, v3.s[2]\n" - "prfm pldl1keep, [x25, #0x80]\n" + "mov v9.d[1], x26\n" "fmla v28.4s, v10.4s, v4.s[2]\n" - "ldr x8, [x25, #0x8]\n" + "ldr x26, [x5, #0x28]\n" "fmla v29.4s, v10.4s, v5.s[2]\n" - "add x24, x24, #0x10\n" + "prfm pldl1keep, [x14, #0x80]\n" "fmla v30.4s, v10.4s, v6.s[2]\n" - "prfm pldl1keep, [x24, #0x80]\n" + "prfm pldl1keep, [x13, #0x80]\n" "fmla v31.4s, v10.4s, v7.s[2]\n" - "ldr x21, [x24, #0x8]\n" + "ldr d10, [x5, #0x20]\n" "fmla v24.4s, v11.4s, v0.s[3]\n" - "ldr d0, [x11, #0x0]\n" + "ldr d0, [x17, #0x0]\n" "fmla v25.4s, v11.4s, v1.s[3]\n" - "ldr d1, [x9, #0x0]\n" + "ldr d1, [x16, #0x0]\n" "fmla v26.4s, v11.4s, v2.s[3]\n" - "ldr d2, [x27, #0x0]\n" + "ldr d2, [x15, #0x0]\n" "fmla v27.4s, v11.4s, v3.s[3]\n" - "mov v0.d[1], x10\n" + "ldr d3, [x14, #0x0]\n" "fmla v28.4s, v11.4s, v4.s[3]\n" - "mov v1.d[1], x28\n" + "ldr d4, [x13, #0x0]\n" "fmla v29.4s, v11.4s, v5.s[3]\n" - "mov v2.d[1], x26\n" + "ldr d5, [x12, #0x0]\n" "fmla v30.4s, v11.4s, v6.s[3]\n" - "ldr d3, [x25, #0x0]\n" + "ldr d6, [x11, #0x0]\n" "fmla v31.4s, v11.4s, v7.s[3]\n" - "ldr d4, [x24, #0x0]\n" - "add x23, x23, #0x10\n" - "add x22, x22, #0x10\n" - "mov v3.d[1], x8\n" - "prfm pldl1keep, [x23, #0x80]\n" - "mov v4.d[1], x21\n" - "prfm pldl1keep, [x22, #0x80]\n" - "ldr d5, [x23, #0x0]\n" - "add x20, x20, #0x10\n" - "prfm pldl1keep, [x20, #0x80]\n" - "sub x12, x12, #0x4\n" - "ldr x8, [x23, #0x8]\n" - "cmp x12, #0x8\n" - "ldr d6, [x22, #0x0]\n" - "add x16, x16, #0x40\n" - "ldr d8, [x16, #0x0]\n" - "mov v5.d[1], x8\n" + "ldr d7, [x27, #0x0]\n" + "ldr d11, [x5, #0x30]\n" + "mov v10.d[1], x26\n" + "ldr x26, [x17, #0x8]\n" + "mov v0.d[1], x26\n" "ldr x26, [x16, #0x8]\n" - "ldr x21, [x22, #0x8]\n" - "ldr d7, [x20, #0x0]\n" - "mov v8.d[1], x26\n" - "ldr x8, [x20, #0x8]\n" - "mov v6.d[1], x21\n" - "mov v7.d[1], x8\n" + "mov v1.d[1], x26\n" + "ldr x26, [x15, #0x8]\n" + "mov v2.d[1], x26\n" + "ldr x26, [x14, #0x8]\n" + "mov v3.d[1], x26\n" + "ldr x26, [x13, #0x8]\n" + "mov v4.d[1], x26\n" + "ldr x26, [x12, #0x8]\n" + "mov v5.d[1], x26\n" + "ldr x26, [x11, #0x8]\n" + "mov v6.d[1], x26\n" + "ldr x26, [x27, #0x8]\n" + "mov v7.d[1], x26\n" + "ldr x26, [x5, #0x38]\n" + "mov v11.d[1], x26\n" + "prfm pldl1keep, [x12, #0x80]\n" + "prfm pldl1keep, [x11, #0x80]\n" + "prfm pldl1keep, [x27, #0x80]\n" "bge 159b\n" "160:" // Height 8: Multiply loop: Single iteration only "fmla v24.4s, v8.4s, v0.s[0]\n" - "ldr q9, [x16, #0x10]\n" + "add x17, x17, #0x10\n" "fmla v25.4s, v8.4s, v1.s[0]\n" - "ldr q10, [x16, #0x20]\n" + "add x16, x16, #0x10\n" "fmla v26.4s, v8.4s, v2.s[0]\n" - "ldr q11, [x16, #0x30]\n" + "add x15, x15, #0x10\n" "fmla v27.4s, v8.4s, v3.s[0]\n" - "sub x12, x12, #0x4\n" + "add x14, x14, #0x10\n" "fmla v28.4s, v8.4s, v4.s[0]\n" - "add x11, x11, #0x10\n" + "add x13, x13, #0x10\n" "fmla v29.4s, v8.4s, v5.s[0]\n" - "prfm pldl1keep, [x11, #0x80]\n" + "add x12, x12, #0x10\n" "fmla v30.4s, v8.4s, v6.s[0]\n" - "add x9, x9, #0x10\n" + "add x11, x11, #0x10\n" "fmla v31.4s, v8.4s, v7.s[0]\n" - "prfm pldl1keep, [x9, #0x80]\n" - "fmla v24.4s, v9.4s, v0.s[1]\n" "add x27, x27, #0x10\n" + "fmla v24.4s, v9.4s, v0.s[1]\n" + "sub x8, x8, #0x4\n" "fmla v25.4s, v9.4s, v1.s[1]\n" - "prfm pldl1keep, [x27, #0x80]\n" + "prfm pldl1keep, [x17, #0x80]\n" "fmla v26.4s, v9.4s, v2.s[1]\n" - "add x25, x25, #0x10\n" + "prfm pldl1keep, [x16, #0x80]\n" "fmla v27.4s, v9.4s, v3.s[1]\n" - "prfm pldl1keep, [x25, #0x80]\n" + "prfm pldl1keep, [x15, #0x80]\n" "fmla v28.4s, v9.4s, v4.s[1]\n" - "add x24, x24, #0x10\n" + "prfm pldl1keep, [x14, #0x80]\n" "fmla v29.4s, v9.4s, v5.s[1]\n" - "prfm pldl1keep, [x24, #0x80]\n" + "prfm pldl1keep, [x13, #0x80]\n" "fmla v30.4s, v9.4s, v6.s[1]\n" - "add x23, x23, #0x10\n" + "prfm pldl1keep, [x12, #0x80]\n" "fmla v31.4s, v9.4s, v7.s[1]\n" - "prfm pldl1keep, [x23, #0x80]\n" + "prfm pldl1keep, [x11, #0x80]\n" "fmla v24.4s, v10.4s, v0.s[2]\n" - "add x22, x22, #0x10\n" + "prfm pldl1keep, [x27, #0x80]\n" "fmla v25.4s, v10.4s, v1.s[2]\n" - "prfm pldl1keep, [x22, #0x80]\n" + "add x5, x5, #0x40\n" "fmla v26.4s, v10.4s, v2.s[2]\n" - "add x20, x20, #0x10\n" "fmla v27.4s, v10.4s, v3.s[2]\n" - "prfm pldl1keep, [x20, #0x80]\n" "fmla v28.4s, v10.4s, v4.s[2]\n" - "add x16, x16, #0x40\n" "fmla v29.4s, v10.4s, v5.s[2]\n" "fmla v30.4s, v10.4s, v6.s[2]\n" "fmla v31.4s, v10.4s, v7.s[2]\n" @@ -2082,54 +2081,52 @@ void a64_hybrid_fp32_mla_8x4_a55 ( "fmla v30.4s, v11.4s, v6.s[3]\n" "fmla v31.4s, v11.4s, v7.s[3]\n" "161:" // Height 8: Multiply loop: Main loop skip - "cbz x12, 163f\n" + "cbz x8, 163f\n" "162:" // Height 8: Multiply loop: Odd block loop - "ldr s0, [x11], #0x4\n" - "sub x12, x12, #0x1\n" - "ldr s1, [x9], #0x4\n" - "ldr s2, [x27], #0x4\n" - "ldr s3, [x25], #0x4\n" - "ldr s4, [x24], #0x4\n" - "ldr s5, [x23], #0x4\n" - "ldr s6, [x22], #0x4\n" - "ldr s7, [x20], #0x4\n" - "ldr q12, [x16, #0x0]\n" - "add x16, x16, #0x10\n" - "fmla v24.4s, v12.4s, v0.s[0]\n" - "fmla v25.4s, v12.4s, v1.s[0]\n" - "fmla v26.4s, v12.4s, v2.s[0]\n" - "fmla v27.4s, v12.4s, v3.s[0]\n" - "fmla v28.4s, v12.4s, v4.s[0]\n" - "fmla v29.4s, v12.4s, v5.s[0]\n" - "fmla v30.4s, v12.4s, v6.s[0]\n" - "fmla v31.4s, v12.4s, v7.s[0]\n" - "cbnz x12, 162b\n" + "ldr s0, [x17], #0x4\n" + "sub x8, x8, #0x1\n" + "ldr s23, [x16], #0x4\n" + "ldr s22, [x15], #0x4\n" + "ldr s21, [x14], #0x4\n" + "ldr s20, [x13], #0x4\n" + "ldr s19, [x12], #0x4\n" + "ldr s18, [x11], #0x4\n" + "ldr s17, [x27], #0x4\n" + "ldr q16, [x5, #0x0]\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v23.s[0]\n" + "add x5, x5, #0x10\n" + "fmla v26.4s, v16.4s, v22.s[0]\n" + "fmla v27.4s, v16.4s, v21.s[0]\n" + "fmla v28.4s, v16.4s, v20.s[0]\n" + "fmla v29.4s, v16.4s, v19.s[0]\n" + "fmla v30.4s, v16.4s, v18.s[0]\n" + "fmla v31.4s, v16.4s, v17.s[0]\n" + "cbnz x8, 162b\n" "163:" // Height 8: Multiply loop: No odd multiplies - "ldr w8, [%x[args_ptr], %[offsetof_num_strings]]\n" - "add x13, x13, #0x1\n" - "cmp x13, x8\n" + "ldr w26, [%x[args_ptr], %[offsetof_num_strings]]\n" + "add x7, x7, #0x1\n" + "cmp x7, x26\n" "bne 156b\n" - "ldr x8, [%x[args_ptr], %[offsetof_output_offset]]\n" - "prfm pstl1keep, [x14, #0x0]\n" - "add x27, x14, x8, LSL #2\n" + "ldr x26, [%x[args_ptr], %[offsetof_output_offset]]\n" + "add x13, x6, x26, LSL #2\n" + "add x12, x13, x26, LSL #2\n" + "add x11, x12, x26, LSL #2\n" + "add x10, x11, x26, LSL #2\n" + "add x9, x10, x26, LSL #2\n" + "add x28, x9, x26, LSL #2\n" + "add x27, x28, x26, LSL #2\n" + "prfm pstl1keep, [x6, #0x0]\n" + "prfm pstl1keep, [x13, #0x0]\n" + "prfm pstl1keep, [x12, #0x0]\n" + "prfm pstl1keep, [x11, #0x0]\n" + "prfm pstl1keep, [x10, #0x0]\n" + "prfm pstl1keep, [x9, #0x0]\n" + "prfm pstl1keep, [x28, #0x0]\n" "prfm pstl1keep, [x27, #0x0]\n" - "add x26, x27, x8, LSL #2\n" - "prfm pstl1keep, [x26, #0x0]\n" - "add x25, x26, x8, LSL #2\n" - "prfm pstl1keep, [x25, #0x0]\n" - "add x24, x25, x8, LSL #2\n" - "prfm pstl1keep, [x24, #0x0]\n" - "add x23, x24, x8, LSL #2\n" - "prfm pstl1keep, [x23, #0x0]\n" - "add x22, x23, x8, LSL #2\n" - "prfm pstl1keep, [x22, #0x0]\n" - "add x21, x22, x8, LSL #2\n" - "prfm pstl1keep, [x21, #0x0]\n" "tbz %x[flags], #1, 164f\n" - "add x20, %x[args_ptr], %[offset_min]\n" - "add x8, %x[args_ptr], %[offset_max]\n" - "ld1r { v17.4s }, [x20]\n" - "ld1r { v16.4s }, [x8]\n" + "add x26, %x[args_ptr], %[offset_max]\n" + "ld1r { v16.4s }, [x26]\n" "fmin v24.4s, v24.4s, v16.4s\n" "fmin v25.4s, v25.4s, v16.4s\n" "fmin v26.4s, v26.4s, v16.4s\n" @@ -2138,76 +2135,77 @@ void a64_hybrid_fp32_mla_8x4_a55 ( "fmin v29.4s, v29.4s, v16.4s\n" "fmin v30.4s, v30.4s, v16.4s\n" "fmin v31.4s, v31.4s, v16.4s\n" - "fmax v24.4s, v24.4s, v17.4s\n" - "fmax v25.4s, v25.4s, v17.4s\n" - "fmax v26.4s, v26.4s, v17.4s\n" - "fmax v27.4s, v27.4s, v17.4s\n" - "fmax v28.4s, v28.4s, v17.4s\n" - "fmax v29.4s, v29.4s, v17.4s\n" - "fmax v30.4s, v30.4s, v17.4s\n" - "fmax v31.4s, v31.4s, v17.4s\n" + "add x26, %x[args_ptr], %[offset_min]\n" + "ld1r { v16.4s }, [x26]\n" + "fmax v24.4s, v24.4s, v16.4s\n" + "fmax v25.4s, v25.4s, v16.4s\n" + "fmax v26.4s, v26.4s, v16.4s\n" + "fmax v27.4s, v27.4s, v16.4s\n" + "fmax v28.4s, v28.4s, v16.4s\n" + "fmax v29.4s, v29.4s, v16.4s\n" + "fmax v30.4s, v30.4s, v16.4s\n" + "fmax v31.4s, v31.4s, v16.4s\n" "164:" // Height 8: No activation - "cmp x17, #0x4\n" + "cmp x4, #0x4\n" "bge 167f\n" - "tbz x17, #1, 165f\n" - "str d24, [x14], #0x8\n" - "str d25, [x27], #0x8\n" - "str d26, [x26], #0x8\n" - "str d27, [x25], #0x8\n" - "str d28, [x24], #0x8\n" - "str d29, [x23], #0x8\n" - "str d30, [x22], #0x8\n" - "str d31, [x21], #0x8\n" - "tbz x17, #0, 166f\n" - "st1 { v24.s }[2], [x14]\n" - "st1 { v25.s }[2], [x27]\n" - "st1 { v26.s }[2], [x26]\n" - "st1 { v27.s }[2], [x25]\n" - "st1 { v28.s }[2], [x24]\n" - "st1 { v29.s }[2], [x23]\n" - "st1 { v30.s }[2], [x22]\n" - "st1 { v31.s }[2], [x21]\n" + "tbz x4, #1, 165f\n" + "str d24, [x6], #0x8\n" + "str d25, [x13], #0x8\n" + "str d26, [x12], #0x8\n" + "str d27, [x11], #0x8\n" + "str d28, [x10], #0x8\n" + "str d29, [x9], #0x8\n" + "str d30, [x28], #0x8\n" + "str d31, [x27], #0x8\n" + "tbz x4, #0, 166f\n" + "st1 { v24.s }[2], [x6]\n" + "st1 { v25.s }[2], [x13]\n" + "st1 { v26.s }[2], [x12]\n" + "st1 { v27.s }[2], [x11]\n" + "st1 { v28.s }[2], [x10]\n" + "st1 { v29.s }[2], [x9]\n" + "st1 { v30.s }[2], [x28]\n" + "st1 { v31.s }[2], [x27]\n" "b 166f\n" "165:" // Height 8: Partial direct writeback: partial_1_0 - "str s24, [x14, #0x0]\n" - "str s25, [x27, #0x0]\n" - "str s26, [x26, #0x0]\n" - "str s27, [x25, #0x0]\n" - "str s28, [x24, #0x0]\n" - "str s29, [x23, #0x0]\n" - "str s30, [x22, #0x0]\n" - "str s31, [x21, #0x0]\n" + "str s24, [x6, #0x0]\n" + "str s25, [x13, #0x0]\n" + "str s26, [x12, #0x0]\n" + "str s27, [x11, #0x0]\n" + "str s28, [x10, #0x0]\n" + "str s29, [x9, #0x0]\n" + "str s30, [x28, #0x0]\n" + "str s31, [x27, #0x0]\n" "166:" // Height 8: Partial direct writeback: Done "b 168f\n" "167:" // Height 8: Full writeback - "str q24, [x14, #0x0]\n" - "add x14, x14, #0x10\n" - "str q25, [x27, #0x0]\n" - "str q26, [x26, #0x0]\n" - "str q27, [x25, #0x0]\n" - "str q28, [x24, #0x0]\n" - "str q29, [x23, #0x0]\n" - "str q30, [x22, #0x0]\n" - "str q31, [x21, #0x0]\n" + "str q24, [x6, #0x0]\n" + "add x6, x6, #0x10\n" + "str q25, [x13, #0x0]\n" + "str q26, [x12, #0x0]\n" + "str q27, [x11, #0x0]\n" + "str q28, [x10, #0x0]\n" + "str q29, [x9, #0x0]\n" + "str q30, [x28, #0x0]\n" + "str q31, [x27, #0x0]\n" "168:" // Height 8: Writeback done - "subs x17, x17, #0x4\n" + "subs x4, x4, #0x4\n" "bgt 149b\n" "subs %x[M], %x[M], #0x8\n" "beq 170f\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x27, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 169f\n" - "add x20, x20, #0x8\n" - "str x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "add x27, x27, #0x8\n" + "str x27, [%x[args_ptr], %[offsetof_input_offset]]\n" "b 1b\n" "169:" // Update direct input - "mov x8, #0x20\n" - "madd %x[input_ptr], x8, x20, %x[input_ptr]\n" + "mov x26, #0x20\n" + "madd %x[input_ptr], x26, x27, %x[input_ptr]\n" "b 1b\n" "170:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v16", "v17", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x8", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4/generic.cpp index bd22336c8d3618dc54ab2ddcc678b93b9e3a83ca..004e5d7f23d87347e1d8fd1cf7976d588d352b1b 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32_mla_8x4/generic.cpp @@ -92,7 +92,6 @@ void a64_hybrid_fp32_mla_8x4 ( break; } __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x8\n" "bge 148f\n" @@ -140,11 +139,11 @@ void a64_hybrid_fp32_mla_8x4 ( "9:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w9, [x20, x10, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 10f\n" - "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x28, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x10, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x28, [x20, #0x0]\n" "cbnz x10, 11f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x28, x28, x20, LSL #2\n" @@ -189,10 +188,10 @@ void a64_hybrid_fp32_mla_8x4 ( "14:" // Height 1: Multiply loop: Main loop skip "cbz x9, 16f\n" "15:" // Height 1: Multiply loop: Odd block loop - "ldr s0, [x28], #0x4\n" - "ldr q12, [x12, #0x0]\n" + "ldr s17, [x28], #0x4\n" + "ldr q16, [x12, #0x0]\n" "sub x9, x9, #0x1\n" - "fmla v24.4s, v12.4s, v0.s[0]\n" + "fmla v24.4s, v16.4s, v17.s[0]\n" "add x12, x12, #0x10\n" "cbnz x9, 15b\n" "16:" // Height 1: Multiply loop: No odd multiplies @@ -271,12 +270,12 @@ void a64_hybrid_fp32_mla_8x4 ( "30:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w9, [x20, x10, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 31f\n" - "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x28, [x21, #0x0]\n" - "ldr x27, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x10, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x28, [x20, #0x0]\n" + "ldr x27, [x20, #0x8]\n" "cbnz x10, 32f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x28, x28, x20, LSL #2\n" @@ -284,7 +283,7 @@ void a64_hybrid_fp32_mla_8x4 ( "b 32f\n" "31:" // Height 2: setup direct input "mov x28, %x[input_ptr]\n" - "add x27, x28, x20, LSL #2\n" + "add x27, x28, x21, LSL #2\n" "32:" // Height 2: input setup done "cmp x9, #0x4\n" "blt 35f\n" @@ -337,12 +336,12 @@ void a64_hybrid_fp32_mla_8x4 ( "35:" // Height 2: Multiply loop: Main loop skip "cbz x9, 37f\n" "36:" // Height 2: Multiply loop: Odd block loop - "ldr s0, [x28], #0x4\n" - "ldr s1, [x27], #0x4\n" + "ldr s18, [x28], #0x4\n" + "ldr s17, [x27], #0x4\n" "sub x9, x9, #0x1\n" - "ldr q12, [x12, #0x0]\n" - "fmla v24.4s, v12.4s, v0.s[0]\n" - "fmla v25.4s, v12.4s, v1.s[0]\n" + "ldr q16, [x12, #0x0]\n" + "fmla v24.4s, v16.4s, v18.s[0]\n" + "fmla v25.4s, v16.4s, v17.s[0]\n" "add x12, x12, #0x10\n" "cbnz x9, 36b\n" "37:" // Height 2: Multiply loop: No odd multiplies @@ -437,13 +436,13 @@ void a64_hybrid_fp32_mla_8x4 ( "51:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w9, [x20, x10, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 52f\n" - "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x28, [x21, #0x0]\n" - "ldr x27, [x21, #0x8]\n" - "ldr x26, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x10, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x28, [x20, #0x0]\n" + "ldr x27, [x20, #0x8]\n" + "ldr x26, [x20, #0x10]\n" "cbnz x10, 53f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x28, x28, x20, LSL #2\n" @@ -452,8 +451,8 @@ void a64_hybrid_fp32_mla_8x4 ( "b 53f\n" "52:" // Height 3: setup direct input "mov x28, %x[input_ptr]\n" - "add x27, x28, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" + "add x27, x28, x21, LSL #2\n" + "add x26, x27, x21, LSL #2\n" "53:" // Height 3: input setup done "cmp x9, #0x4\n" "blt 56f\n" @@ -520,14 +519,14 @@ void a64_hybrid_fp32_mla_8x4 ( "56:" // Height 3: Multiply loop: Main loop skip "cbz x9, 58f\n" "57:" // Height 3: Multiply loop: Odd block loop - "ldr s0, [x28], #0x4\n" - "ldr s1, [x27], #0x4\n" + "ldr s19, [x28], #0x4\n" + "ldr s18, [x27], #0x4\n" "sub x9, x9, #0x1\n" - "ldr s2, [x26], #0x4\n" - "ldr q12, [x12, #0x0]\n" - "fmla v24.4s, v12.4s, v0.s[0]\n" - "fmla v25.4s, v12.4s, v1.s[0]\n" - "fmla v26.4s, v12.4s, v2.s[0]\n" + "ldr s17, [x26], #0x4\n" + "ldr q16, [x12, #0x0]\n" + "fmla v24.4s, v16.4s, v19.s[0]\n" + "fmla v25.4s, v16.4s, v18.s[0]\n" + "fmla v26.4s, v16.4s, v17.s[0]\n" "add x12, x12, #0x10\n" "cbnz x9, 57b\n" "58:" // Height 3: Multiply loop: No odd multiplies @@ -637,14 +636,14 @@ void a64_hybrid_fp32_mla_8x4 ( "72:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w9, [x20, x10, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 73f\n" - "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x28, [x21, #0x0]\n" - "ldr x27, [x21, #0x8]\n" - "ldr x26, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x10, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x28, [x20, #0x0]\n" + "ldr x27, [x20, #0x8]\n" + "ldr x26, [x20, #0x10]\n" + "ldr x25, [x20, #0x18]\n" "cbnz x10, 74f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x28, x28, x20, LSL #2\n" @@ -654,9 +653,9 @@ void a64_hybrid_fp32_mla_8x4 ( "b 74f\n" "73:" // Height 4: setup direct input "mov x28, %x[input_ptr]\n" - "add x27, x28, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" - "add x25, x26, x20, LSL #2\n" + "add x27, x28, x21, LSL #2\n" + "add x26, x27, x21, LSL #2\n" + "add x25, x26, x21, LSL #2\n" "74:" // Height 4: input setup done "cmp x9, #0x4\n" "blt 77f\n" @@ -737,17 +736,17 @@ void a64_hybrid_fp32_mla_8x4 ( "77:" // Height 4: Multiply loop: Main loop skip "cbz x9, 79f\n" "78:" // Height 4: Multiply loop: Odd block loop - "ldr s0, [x28], #0x4\n" - "ldr s1, [x27], #0x4\n" + "ldr s20, [x28], #0x4\n" + "ldr s19, [x27], #0x4\n" "sub x9, x9, #0x1\n" - "ldr s2, [x26], #0x4\n" - "ldr s3, [x25], #0x4\n" - "ldr q12, [x12, #0x0]\n" - "fmla v24.4s, v12.4s, v0.s[0]\n" - "fmla v25.4s, v12.4s, v1.s[0]\n" + "ldr s18, [x26], #0x4\n" + "ldr s17, [x25], #0x4\n" + "ldr q16, [x12, #0x0]\n" + "fmla v24.4s, v16.4s, v20.s[0]\n" + "fmla v25.4s, v16.4s, v19.s[0]\n" "add x12, x12, #0x10\n" - "fmla v26.4s, v12.4s, v2.s[0]\n" - "fmla v27.4s, v12.4s, v3.s[0]\n" + "fmla v26.4s, v16.4s, v18.s[0]\n" + "fmla v27.4s, v16.4s, v17.s[0]\n" "cbnz x9, 78b\n" "79:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -871,15 +870,15 @@ void a64_hybrid_fp32_mla_8x4 ( "93:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w9, [x20, x10, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 94f\n" - "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x28, [x21, #0x0]\n" - "ldr x27, [x21, #0x8]\n" - "ldr x26, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" - "ldr x24, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x10, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x28, [x20, #0x0]\n" + "ldr x27, [x20, #0x8]\n" + "ldr x26, [x20, #0x10]\n" + "ldr x25, [x20, #0x18]\n" + "ldr x24, [x20, #0x20]\n" "cbnz x10, 95f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x28, x28, x20, LSL #2\n" @@ -890,10 +889,10 @@ void a64_hybrid_fp32_mla_8x4 ( "b 95f\n" "94:" // Height 5: setup direct input "mov x28, %x[input_ptr]\n" - "add x27, x28, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" + "add x27, x28, x21, LSL #2\n" + "add x26, x27, x21, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" "95:" // Height 5: input setup done "cmp x9, #0x4\n" "blt 98f\n" @@ -988,19 +987,19 @@ void a64_hybrid_fp32_mla_8x4 ( "98:" // Height 5: Multiply loop: Main loop skip "cbz x9, 100f\n" "99:" // Height 5: Multiply loop: Odd block loop - "ldr s0, [x28], #0x4\n" - "ldr s1, [x27], #0x4\n" + "ldr s21, [x28], #0x4\n" + "ldr s20, [x27], #0x4\n" "sub x9, x9, #0x1\n" - "ldr s2, [x26], #0x4\n" - "ldr s3, [x25], #0x4\n" - "ldr s4, [x24], #0x4\n" - "ldr q12, [x12, #0x0]\n" - "fmla v24.4s, v12.4s, v0.s[0]\n" - "fmla v25.4s, v12.4s, v1.s[0]\n" - "fmla v26.4s, v12.4s, v2.s[0]\n" - "fmla v27.4s, v12.4s, v3.s[0]\n" + "ldr s19, [x26], #0x4\n" + "ldr s18, [x25], #0x4\n" + "ldr s17, [x24], #0x4\n" + "ldr q16, [x12, #0x0]\n" + "fmla v24.4s, v16.4s, v21.s[0]\n" + "fmla v25.4s, v16.4s, v20.s[0]\n" + "fmla v26.4s, v16.4s, v19.s[0]\n" + "fmla v27.4s, v16.4s, v18.s[0]\n" "add x12, x12, #0x10\n" - "fmla v28.4s, v12.4s, v4.s[0]\n" + "fmla v28.4s, v16.4s, v17.s[0]\n" "cbnz x9, 99b\n" "100:" // Height 5: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -1139,16 +1138,16 @@ void a64_hybrid_fp32_mla_8x4 ( "114:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w9, [x20, x10, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 115f\n" - "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x28, [x21, #0x0]\n" - "ldr x27, [x21, #0x8]\n" - "ldr x26, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" - "ldr x24, [x21, #0x20]\n" - "ldr x23, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x10, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x28, [x20, #0x0]\n" + "ldr x27, [x20, #0x8]\n" + "ldr x26, [x20, #0x10]\n" + "ldr x25, [x20, #0x18]\n" + "ldr x24, [x20, #0x20]\n" + "ldr x23, [x20, #0x28]\n" "cbnz x10, 116f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x28, x28, x20, LSL #2\n" @@ -1160,11 +1159,11 @@ void a64_hybrid_fp32_mla_8x4 ( "b 116f\n" "115:" // Height 6: setup direct input "mov x28, %x[input_ptr]\n" - "add x27, x28, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x27, x28, x21, LSL #2\n" + "add x26, x27, x21, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" "116:" // Height 6: input setup done "cmp x9, #0x4\n" "blt 119f\n" @@ -1273,21 +1272,21 @@ void a64_hybrid_fp32_mla_8x4 ( "119:" // Height 6: Multiply loop: Main loop skip "cbz x9, 121f\n" "120:" // Height 6: Multiply loop: Odd block loop - "ldr s0, [x28], #0x4\n" - "ldr s1, [x27], #0x4\n" + "ldr s22, [x28], #0x4\n" + "ldr s21, [x27], #0x4\n" "sub x9, x9, #0x1\n" - "ldr s2, [x26], #0x4\n" - "ldr s3, [x25], #0x4\n" - "ldr s4, [x24], #0x4\n" - "ldr s5, [x23], #0x4\n" - "ldr q12, [x12, #0x0]\n" - "fmla v24.4s, v12.4s, v0.s[0]\n" - "fmla v25.4s, v12.4s, v1.s[0]\n" + "ldr s20, [x26], #0x4\n" + "ldr s19, [x25], #0x4\n" + "ldr s18, [x24], #0x4\n" + "ldr s17, [x23], #0x4\n" + "ldr q16, [x12, #0x0]\n" + "fmla v24.4s, v16.4s, v22.s[0]\n" + "fmla v25.4s, v16.4s, v21.s[0]\n" "add x12, x12, #0x10\n" - "fmla v26.4s, v12.4s, v2.s[0]\n" - "fmla v27.4s, v12.4s, v3.s[0]\n" - "fmla v28.4s, v12.4s, v4.s[0]\n" - "fmla v29.4s, v12.4s, v5.s[0]\n" + "fmla v26.4s, v16.4s, v20.s[0]\n" + "fmla v27.4s, v16.4s, v19.s[0]\n" + "fmla v28.4s, v16.4s, v18.s[0]\n" + "fmla v29.4s, v16.4s, v17.s[0]\n" "cbnz x9, 120b\n" "121:" // Height 6: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -1441,17 +1440,17 @@ void a64_hybrid_fp32_mla_8x4 ( "135:" // Height 7: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w9, [x20, x10, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 136f\n" - "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x28, [x21, #0x0]\n" - "ldr x27, [x21, #0x8]\n" - "ldr x26, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" - "ldr x24, [x21, #0x20]\n" - "ldr x23, [x21, #0x28]\n" - "ldr x22, [x21, #0x30]\n" + "ldr x20, [%x[input_ptr], x10, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x28, [x20, #0x0]\n" + "ldr x27, [x20, #0x8]\n" + "ldr x26, [x20, #0x10]\n" + "ldr x25, [x20, #0x18]\n" + "ldr x24, [x20, #0x20]\n" + "ldr x23, [x20, #0x28]\n" + "ldr x22, [x20, #0x30]\n" "cbnz x10, 137f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x28, x28, x20, LSL #2\n" @@ -1464,12 +1463,12 @@ void a64_hybrid_fp32_mla_8x4 ( "b 137f\n" "136:" // Height 7: setup direct input "mov x28, %x[input_ptr]\n" - "add x27, x28, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" + "add x27, x28, x21, LSL #2\n" + "add x26, x27, x21, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" "137:" // Height 7: input setup done "cmp x9, #0x4\n" "blt 140f\n" @@ -1592,23 +1591,23 @@ void a64_hybrid_fp32_mla_8x4 ( "140:" // Height 7: Multiply loop: Main loop skip "cbz x9, 142f\n" "141:" // Height 7: Multiply loop: Odd block loop - "ldr s0, [x28], #0x4\n" - "ldr s1, [x27], #0x4\n" + "ldr s23, [x28], #0x4\n" + "ldr s22, [x27], #0x4\n" "sub x9, x9, #0x1\n" - "ldr s2, [x26], #0x4\n" - "ldr s3, [x25], #0x4\n" - "ldr s4, [x24], #0x4\n" - "ldr s5, [x23], #0x4\n" - "ldr s6, [x22], #0x4\n" - "ldr q12, [x12, #0x0]\n" - "fmla v24.4s, v12.4s, v0.s[0]\n" - "fmla v25.4s, v12.4s, v1.s[0]\n" - "fmla v26.4s, v12.4s, v2.s[0]\n" - "fmla v27.4s, v12.4s, v3.s[0]\n" + "ldr s21, [x26], #0x4\n" + "ldr s20, [x25], #0x4\n" + "ldr s19, [x24], #0x4\n" + "ldr s18, [x23], #0x4\n" + "ldr s17, [x22], #0x4\n" + "ldr q16, [x12, #0x0]\n" + "fmla v24.4s, v16.4s, v23.s[0]\n" + "fmla v25.4s, v16.4s, v22.s[0]\n" + "fmla v26.4s, v16.4s, v21.s[0]\n" + "fmla v27.4s, v16.4s, v20.s[0]\n" "add x12, x12, #0x10\n" - "fmla v28.4s, v12.4s, v4.s[0]\n" - "fmla v29.4s, v12.4s, v5.s[0]\n" - "fmla v30.4s, v12.4s, v6.s[0]\n" + "fmla v28.4s, v16.4s, v19.s[0]\n" + "fmla v29.4s, v16.4s, v18.s[0]\n" + "fmla v30.4s, v16.4s, v17.s[0]\n" "cbnz x9, 141b\n" "142:" // Height 7: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -1780,18 +1779,18 @@ void a64_hybrid_fp32_mla_8x4 ( "156:" // Height 8: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w9, [x20, x10, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 157f\n" - "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x28, [x21, #0x0]\n" - "ldr x27, [x21, #0x8]\n" - "ldr x26, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" - "ldr x24, [x21, #0x20]\n" - "ldr x23, [x21, #0x28]\n" - "ldr x22, [x21, #0x30]\n" - "ldr x21, [x21, #0x38]\n" + "ldr x20, [%x[input_ptr], x10, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x28, [x20, #0x0]\n" + "ldr x27, [x20, #0x8]\n" + "ldr x26, [x20, #0x10]\n" + "ldr x25, [x20, #0x18]\n" + "ldr x24, [x20, #0x20]\n" + "ldr x23, [x20, #0x28]\n" + "ldr x22, [x20, #0x30]\n" + "ldr x21, [x20, #0x38]\n" "cbnz x10, 158f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x28, x28, x20, LSL #2\n" @@ -1805,13 +1804,13 @@ void a64_hybrid_fp32_mla_8x4 ( "b 158f\n" "157:" // Height 8: setup direct input "mov x28, %x[input_ptr]\n" - "add x27, x28, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "add x21, x22, x20, LSL #2\n" + "add x27, x28, x21, LSL #2\n" + "add x26, x27, x21, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" + "add x21, x22, x21, LSL #2\n" "158:" // Height 8: input setup done "cmp x9, #0x4\n" "blt 161f\n" @@ -1949,24 +1948,24 @@ void a64_hybrid_fp32_mla_8x4 ( "cbz x9, 163f\n" "162:" // Height 8: Multiply loop: Odd block loop "ldr s0, [x28], #0x4\n" - "ldr s1, [x27], #0x4\n" + "ldr s23, [x27], #0x4\n" "sub x9, x9, #0x1\n" - "ldr s2, [x26], #0x4\n" - "ldr s3, [x25], #0x4\n" - "ldr s4, [x24], #0x4\n" - "ldr s5, [x23], #0x4\n" - "ldr s6, [x22], #0x4\n" - "ldr s7, [x21], #0x4\n" - "ldr q12, [x12, #0x0]\n" - "fmla v24.4s, v12.4s, v0.s[0]\n" - "fmla v25.4s, v12.4s, v1.s[0]\n" + "ldr s22, [x26], #0x4\n" + "ldr s21, [x25], #0x4\n" + "ldr s20, [x24], #0x4\n" + "ldr s19, [x23], #0x4\n" + "ldr s18, [x22], #0x4\n" + "ldr s17, [x21], #0x4\n" + "ldr q16, [x12, #0x0]\n" + "fmla v24.4s, v16.4s, v0.s[0]\n" + "fmla v25.4s, v16.4s, v23.s[0]\n" "add x12, x12, #0x10\n" - "fmla v26.4s, v12.4s, v2.s[0]\n" - "fmla v27.4s, v12.4s, v3.s[0]\n" - "fmla v28.4s, v12.4s, v4.s[0]\n" - "fmla v29.4s, v12.4s, v5.s[0]\n" - "fmla v30.4s, v12.4s, v6.s[0]\n" - "fmla v31.4s, v12.4s, v7.s[0]\n" + "fmla v26.4s, v16.4s, v22.s[0]\n" + "fmla v27.4s, v16.4s, v21.s[0]\n" + "fmla v28.4s, v16.4s, v20.s[0]\n" + "fmla v29.4s, v16.4s, v19.s[0]\n" + "fmla v30.4s, v16.4s, v18.s[0]\n" + "fmla v31.4s, v16.4s, v17.s[0]\n" "cbnz x9, 162b\n" "163:" // Height 8: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -2068,10 +2067,9 @@ void a64_hybrid_fp32_mla_8x4 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "170:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v16", "v17", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32bf16fp32_mmla_4x24.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32bf16fp32_mmla_4x24.hpp index e6e7950979e3b3d806cc1256b472eb7ab6f44516..f31dd7afd0bfc89809a20ff3459aa3569d62ac2e 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32bf16fp32_mmla_4x24.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32bf16fp32_mmla_4x24.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef __aarch64__ + #include "../std_transforms_fixed.hpp" #include "../bfloat.hpp" #include "../performance_parameters.hpp" @@ -99,5 +99,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32bf16fp32_mmla_4x24/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32bf16fp32_mmla_4x24/generic.cpp index a0ea96822afbf21b7903bc85bd3d1a66593cc30b..0e468b196adb2d766a9ca90992fce5d983e70cef 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32bf16fp32_mmla_4x24/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32bf16fp32_mmla_4x24/generic.cpp @@ -93,7 +93,6 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( break; } __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x4\n" "bge 130f\n" @@ -255,11 +254,11 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( "20:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 21f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" "cbnz x26, 22f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20, LSL #2\n" @@ -279,31 +278,31 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( "23:" // Height 1: Multiply loop: Main loop head ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" ".inst 0x6e44ec08 // bfmmla v8.4s, v0.8h, v4.8h\n" - "ldr q4, [x28, #0x40]\n" + "ldr q24, [x28, #0x40]\n" ".inst 0x6e45ec0e // bfmmla v14.4s, v0.8h, v5.8h\n" - "ldr q5, [x28, #0x50]\n" + "ldr q23, [x28, #0x50]\n" ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - "ldr q6, [x28, #0x60]\n" + "ldr q22, [x28, #0x60]\n" ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - "ldr q7, [x28, #0x70]\n" - ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" - "ldr q4, [x28, #0x80]\n" - ".inst 0x6e45ec10 // bfmmla v16.4s, v0.8h, v5.8h\n" - "ldr q5, [x28, #0x90]\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - "ldr q6, [x28, #0xa0]\n" - ".inst 0x6e47ec11 // bfmmla v17.4s, v0.8h, v7.8h\n" - "ldr q7, [x28, #0xb0]\n" + "ldr q21, [x28, #0x70]\n" + ".inst 0x6e58ec0a // bfmmla v10.4s, v0.8h, v24.8h\n" + "ldr q24, [x28, #0x80]\n" + ".inst 0x6e57ec10 // bfmmla v16.4s, v0.8h, v23.8h\n" + "ldr q23, [x28, #0x90]\n" + ".inst 0x6e56ec0b // bfmmla v11.4s, v0.8h, v22.8h\n" + "ldr q22, [x28, #0xa0]\n" + ".inst 0x6e55ec11 // bfmmla v17.4s, v0.8h, v21.8h\n" + "ldr q21, [x28, #0xb0]\n" "sub x25, x25, #0x4\n" "cmp x25, #0x8\n" - ".inst 0x6e44ec0c // bfmmla v12.4s, v0.8h, v4.8h\n" + ".inst 0x6e58ec0c // bfmmla v12.4s, v0.8h, v24.8h\n" "add x28, x28, #0xc0\n" "ldr q4, [x28, #0x0]\n" - ".inst 0x6e45ec12 // bfmmla v18.4s, v0.8h, v5.8h\n" + ".inst 0x6e57ec12 // bfmmla v18.4s, v0.8h, v23.8h\n" "ldr q5, [x28, #0x10]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" + ".inst 0x6e56ec0d // bfmmla v13.4s, v0.8h, v22.8h\n" "ldr q6, [x28, #0x20]\n" - ".inst 0x6e47ec13 // bfmmla v19.4s, v0.8h, v7.8h\n" + ".inst 0x6e55ec13 // bfmmla v19.4s, v0.8h, v21.8h\n" "ldr q7, [x28, #0x30]\n" "prfm pldl1keep, [x24, #0x80]\n" "ld1 { v0.4s }, [x24], #0x10\n" @@ -311,28 +310,28 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( "24:" // Height 1: Multiply loop: Single iteration only ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" ".inst 0x6e44ec08 // bfmmla v8.4s, v0.8h, v4.8h\n" - "ldr q4, [x28, #0x40]\n" + "ldr q23, [x28, #0x40]\n" ".inst 0x6e45ec0e // bfmmla v14.4s, v0.8h, v5.8h\n" - "ldr q5, [x28, #0x50]\n" + "ldr q25, [x28, #0x50]\n" ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - "ldr q6, [x28, #0x60]\n" + "ldr q21, [x28, #0x60]\n" ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - "ldr q7, [x28, #0x70]\n" - ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" - "ldr q4, [x28, #0x80]\n" - ".inst 0x6e45ec10 // bfmmla v16.4s, v0.8h, v5.8h\n" - "ldr q5, [x28, #0x90]\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - "ldr q6, [x28, #0xa0]\n" - ".inst 0x6e47ec11 // bfmmla v17.4s, v0.8h, v7.8h\n" - "ldr q7, [x28, #0xb0]\n" + "ldr q24, [x28, #0x70]\n" + ".inst 0x6e57ec0a // bfmmla v10.4s, v0.8h, v23.8h\n" + "ldr q23, [x28, #0x80]\n" + ".inst 0x6e59ec10 // bfmmla v16.4s, v0.8h, v25.8h\n" + "ldr q22, [x28, #0x90]\n" + ".inst 0x6e55ec0b // bfmmla v11.4s, v0.8h, v21.8h\n" + "ldr q21, [x28, #0xa0]\n" + ".inst 0x6e58ec11 // bfmmla v17.4s, v0.8h, v24.8h\n" + "ldr q5, [x28, #0xb0]\n" "sub x25, x25, #0x4\n" - ".inst 0x6e44ec0c // bfmmla v12.4s, v0.8h, v4.8h\n" - ".inst 0x6e45ec12 // bfmmla v18.4s, v0.8h, v5.8h\n" + ".inst 0x6e57ec0c // bfmmla v12.4s, v0.8h, v23.8h\n" + ".inst 0x6e56ec12 // bfmmla v18.4s, v0.8h, v22.8h\n" "prfm pldl1keep, [x24, #0x80]\n" "add x28, x28, #0xc0\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec13 // bfmmla v19.4s, v0.8h, v7.8h\n" + ".inst 0x6e55ec0d // bfmmla v13.4s, v0.8h, v21.8h\n" + ".inst 0x6e45ec13 // bfmmla v19.4s, v0.8h, v5.8h\n" "25:" // Height 1: Multiply loop: Main loop skip "cbz x25, 28f\n" "cbz x25, 28f\n" @@ -344,31 +343,31 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( "26:" // Height 1: Multiply loop: Ragged operand read: partial_1_0 "ldr s0, [x24, #0x0]\n" "27:" // Height 1: Multiply loop: Ragged operand read: Done - "ldr q4, [x28, #0x0]\n" - "ldr q5, [x28, #0x10]\n" + "ldr q21, [x28, #0x0]\n" + "ldr q1, [x28, #0x10]\n" ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" - ".inst 0x6e44ec08 // bfmmla v8.4s, v0.8h, v4.8h\n" - "ldr q6, [x28, #0x20]\n" - "ldr q7, [x28, #0x30]\n" - ".inst 0x6e45ec0e // bfmmla v14.4s, v0.8h, v5.8h\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - "ldr q4, [x28, #0x40]\n" - "ldr q5, [x28, #0x50]\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" - "ldr q6, [x28, #0x60]\n" - "ldr q7, [x28, #0x70]\n" - ".inst 0x6e45ec10 // bfmmla v16.4s, v0.8h, v5.8h\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - "ldr q4, [x28, #0x80]\n" - "ldr q5, [x28, #0x90]\n" - ".inst 0x6e47ec11 // bfmmla v17.4s, v0.8h, v7.8h\n" - ".inst 0x6e44ec0c // bfmmla v12.4s, v0.8h, v4.8h\n" - "ldr q6, [x28, #0xa0]\n" - "ldr q7, [x28, #0xb0]\n" - ".inst 0x6e45ec12 // bfmmla v18.4s, v0.8h, v5.8h\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec13 // bfmmla v19.4s, v0.8h, v7.8h\n" + ".inst 0x6e55ec08 // bfmmla v8.4s, v0.8h, v21.8h\n" + "ldr q21, [x28, #0x20]\n" + "ldr q22, [x28, #0x30]\n" + ".inst 0x6e41ec0e // bfmmla v14.4s, v0.8h, v1.8h\n" + ".inst 0x6e55ec09 // bfmmla v9.4s, v0.8h, v21.8h\n" + "ldr q21, [x28, #0x40]\n" + "ldr q23, [x28, #0x50]\n" + ".inst 0x6e56ec0f // bfmmla v15.4s, v0.8h, v22.8h\n" + ".inst 0x6e55ec0a // bfmmla v10.4s, v0.8h, v21.8h\n" + "ldr q21, [x28, #0x60]\n" + "ldr q22, [x28, #0x70]\n" + ".inst 0x6e57ec10 // bfmmla v16.4s, v0.8h, v23.8h\n" + ".inst 0x6e55ec0b // bfmmla v11.4s, v0.8h, v21.8h\n" + "ldr q21, [x28, #0x80]\n" + "ldr q23, [x28, #0x90]\n" + ".inst 0x6e56ec11 // bfmmla v17.4s, v0.8h, v22.8h\n" + ".inst 0x6e55ec0c // bfmmla v12.4s, v0.8h, v21.8h\n" + "ldr q22, [x28, #0xa0]\n" + "ldr q21, [x28, #0xb0]\n" + ".inst 0x6e57ec12 // bfmmla v18.4s, v0.8h, v23.8h\n" + ".inst 0x6e56ec0d // bfmmla v13.4s, v0.8h, v22.8h\n" + ".inst 0x6e55ec13 // bfmmla v19.4s, v0.8h, v21.8h\n" "add x28, x28, #0xc0\n" "28:" // Height 1: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -384,21 +383,21 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( "uzp1 v13.2d, v13.2d, v19.2d\n" "tbz %x[flags], #1, 29f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v22.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" + "ld1r { v21.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v22.4s\n" + "fmin v9.4s, v9.4s, v22.4s\n" + "fmin v10.4s, v10.4s, v22.4s\n" + "fmin v11.4s, v11.4s, v22.4s\n" + "fmin v12.4s, v12.4s, v22.4s\n" + "fmin v13.4s, v13.4s, v22.4s\n" + "fmax v8.4s, v8.4s, v21.4s\n" + "fmax v9.4s, v9.4s, v21.4s\n" + "fmax v10.4s, v10.4s, v21.4s\n" + "fmax v11.4s, v11.4s, v21.4s\n" + "fmax v12.4s, v12.4s, v21.4s\n" + "fmax v13.4s, v13.4s, v21.4s\n" "29:" // Height 1: No activation "cmp x9, #0x18\n" "bge 42f\n" @@ -678,12 +677,12 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( "63:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 64f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" "cbnz x26, 65f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20, LSL #2\n" @@ -691,7 +690,7 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( "b 65f\n" "64:" // Height 2: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20, LSL #2\n" + "add x23, x24, x21, LSL #2\n" "65:" // Height 2: input setup done "cmp x25, #0x4\n" "blt 68f\n" @@ -707,31 +706,31 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" ".inst 0x4ea16820 // bfcvtn2 v0.8h, v1.4s\n" ".inst 0x6e44ec08 // bfmmla v8.4s, v0.8h, v4.8h\n" - "ldr q4, [x28, #0x40]\n" + "ldr q3, [x28, #0x40]\n" ".inst 0x6e45ec0e // bfmmla v14.4s, v0.8h, v5.8h\n" - "ldr q5, [x28, #0x50]\n" + "ldr q23, [x28, #0x50]\n" ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - "ldr q6, [x28, #0x60]\n" + "ldr q22, [x28, #0x60]\n" ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - "ldr q7, [x28, #0x70]\n" - ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" - "ldr q4, [x28, #0x80]\n" - ".inst 0x6e45ec10 // bfmmla v16.4s, v0.8h, v5.8h\n" - "ldr q5, [x28, #0x90]\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - "ldr q6, [x28, #0xa0]\n" - ".inst 0x6e47ec11 // bfmmla v17.4s, v0.8h, v7.8h\n" - "ldr q7, [x28, #0xb0]\n" + "ldr q21, [x28, #0x70]\n" + ".inst 0x6e43ec0a // bfmmla v10.4s, v0.8h, v3.8h\n" + "ldr q1, [x28, #0x80]\n" + ".inst 0x6e57ec10 // bfmmla v16.4s, v0.8h, v23.8h\n" + "ldr q23, [x28, #0x90]\n" + ".inst 0x6e56ec0b // bfmmla v11.4s, v0.8h, v22.8h\n" + "ldr q22, [x28, #0xa0]\n" + ".inst 0x6e55ec11 // bfmmla v17.4s, v0.8h, v21.8h\n" + "ldr q21, [x28, #0xb0]\n" "sub x25, x25, #0x4\n" "cmp x25, #0x8\n" "add x28, x28, #0xc0\n" - ".inst 0x6e44ec0c // bfmmla v12.4s, v0.8h, v4.8h\n" + ".inst 0x6e41ec0c // bfmmla v12.4s, v0.8h, v1.8h\n" "ldr q4, [x28, #0x0]\n" - ".inst 0x6e45ec12 // bfmmla v18.4s, v0.8h, v5.8h\n" + ".inst 0x6e57ec12 // bfmmla v18.4s, v0.8h, v23.8h\n" "ldr q5, [x28, #0x10]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" + ".inst 0x6e56ec0d // bfmmla v13.4s, v0.8h, v22.8h\n" "ldr q6, [x28, #0x20]\n" - ".inst 0x6e47ec13 // bfmmla v19.4s, v0.8h, v7.8h\n" + ".inst 0x6e55ec13 // bfmmla v19.4s, v0.8h, v21.8h\n" "ldr q7, [x28, #0x30]\n" "prfm pldl1keep, [x24, #0x80]\n" "ld1 { v0.4s }, [x24], #0x10\n" @@ -742,28 +741,28 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" ".inst 0x4ea16820 // bfcvtn2 v0.8h, v1.4s\n" ".inst 0x6e44ec08 // bfmmla v8.4s, v0.8h, v4.8h\n" - "ldr q4, [x28, #0x40]\n" + "ldr q24, [x28, #0x40]\n" ".inst 0x6e45ec0e // bfmmla v14.4s, v0.8h, v5.8h\n" - "ldr q5, [x28, #0x50]\n" + "ldr q23, [x28, #0x50]\n" ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - "ldr q6, [x28, #0x60]\n" + "ldr q22, [x28, #0x60]\n" ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - "ldr q7, [x28, #0x70]\n" - ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" - "ldr q4, [x28, #0x80]\n" - ".inst 0x6e45ec10 // bfmmla v16.4s, v0.8h, v5.8h\n" - "ldr q5, [x28, #0x90]\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - "ldr q6, [x28, #0xa0]\n" - ".inst 0x6e47ec11 // bfmmla v17.4s, v0.8h, v7.8h\n" - "ldr q7, [x28, #0xb0]\n" + "ldr q21, [x28, #0x70]\n" + ".inst 0x6e58ec0a // bfmmla v10.4s, v0.8h, v24.8h\n" + "ldr q24, [x28, #0x80]\n" + ".inst 0x6e57ec10 // bfmmla v16.4s, v0.8h, v23.8h\n" + "ldr q23, [x28, #0x90]\n" + ".inst 0x6e56ec0b // bfmmla v11.4s, v0.8h, v22.8h\n" + "ldr q22, [x28, #0xa0]\n" + ".inst 0x6e55ec11 // bfmmla v17.4s, v0.8h, v21.8h\n" + "ldr q21, [x28, #0xb0]\n" "sub x25, x25, #0x4\n" - ".inst 0x6e44ec0c // bfmmla v12.4s, v0.8h, v4.8h\n" - ".inst 0x6e45ec12 // bfmmla v18.4s, v0.8h, v5.8h\n" + ".inst 0x6e58ec0c // bfmmla v12.4s, v0.8h, v24.8h\n" + ".inst 0x6e57ec12 // bfmmla v18.4s, v0.8h, v23.8h\n" "prfm pldl1keep, [x24, #0x80]\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec13 // bfmmla v19.4s, v0.8h, v7.8h\n" + ".inst 0x6e56ec0d // bfmmla v13.4s, v0.8h, v22.8h\n" + ".inst 0x6e55ec13 // bfmmla v19.4s, v0.8h, v21.8h\n" "add x28, x28, #0xc0\n" "68:" // Height 2: Multiply loop: Main loop skip "cbz x25, 71f\n" @@ -779,32 +778,32 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( "ldr s0, [x24, #0x0]\n" "ldr s1, [x23, #0x0]\n" "70:" // Height 2: Multiply loop: Ragged operand read: Done - "ldr q4, [x28, #0x0]\n" - "ldr q5, [x28, #0x10]\n" + "ldr q24, [x28, #0x0]\n" + "ldr q23, [x28, #0x10]\n" ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" ".inst 0x4ea16820 // bfcvtn2 v0.8h, v1.4s\n" - "ldr q6, [x28, #0x20]\n" - "ldr q7, [x28, #0x30]\n" - ".inst 0x6e44ec08 // bfmmla v8.4s, v0.8h, v4.8h\n" - ".inst 0x6e45ec0e // bfmmla v14.4s, v0.8h, v5.8h\n" - "ldr q4, [x28, #0x40]\n" - "ldr q5, [x28, #0x50]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - "ldr q6, [x28, #0x60]\n" - "ldr q7, [x28, #0x70]\n" - ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" - ".inst 0x6e45ec10 // bfmmla v16.4s, v0.8h, v5.8h\n" - "ldr q4, [x28, #0x80]\n" - "ldr q5, [x28, #0x90]\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec11 // bfmmla v17.4s, v0.8h, v7.8h\n" - "ldr q6, [x28, #0xa0]\n" - "ldr q7, [x28, #0xb0]\n" - ".inst 0x6e44ec0c // bfmmla v12.4s, v0.8h, v4.8h\n" - ".inst 0x6e45ec12 // bfmmla v18.4s, v0.8h, v5.8h\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec13 // bfmmla v19.4s, v0.8h, v7.8h\n" + "ldr q22, [x28, #0x20]\n" + "ldr q21, [x28, #0x30]\n" + ".inst 0x6e58ec08 // bfmmla v8.4s, v0.8h, v24.8h\n" + ".inst 0x6e57ec0e // bfmmla v14.4s, v0.8h, v23.8h\n" + "ldr q24, [x28, #0x40]\n" + "ldr q23, [x28, #0x50]\n" + ".inst 0x6e56ec09 // bfmmla v9.4s, v0.8h, v22.8h\n" + ".inst 0x6e55ec0f // bfmmla v15.4s, v0.8h, v21.8h\n" + "ldr q22, [x28, #0x60]\n" + "ldr q21, [x28, #0x70]\n" + ".inst 0x6e58ec0a // bfmmla v10.4s, v0.8h, v24.8h\n" + ".inst 0x6e57ec10 // bfmmla v16.4s, v0.8h, v23.8h\n" + "ldr q24, [x28, #0x80]\n" + "ldr q23, [x28, #0x90]\n" + ".inst 0x6e56ec0b // bfmmla v11.4s, v0.8h, v22.8h\n" + ".inst 0x6e55ec11 // bfmmla v17.4s, v0.8h, v21.8h\n" + "ldr q22, [x28, #0xa0]\n" + "ldr q21, [x28, #0xb0]\n" + ".inst 0x6e58ec0c // bfmmla v12.4s, v0.8h, v24.8h\n" + ".inst 0x6e57ec12 // bfmmla v18.4s, v0.8h, v23.8h\n" + ".inst 0x6e56ec0d // bfmmla v13.4s, v0.8h, v22.8h\n" + ".inst 0x6e55ec13 // bfmmla v19.4s, v0.8h, v21.8h\n" "add x28, x28, #0xc0\n" "71:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -829,33 +828,33 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( "uzp2 v13.2d, v13.2d, v19.2d\n" "tbz %x[flags], #1, 72f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v22.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v4.4s, v4.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v15.4s, v15.4s, v1.4s\n" - "fmin v16.4s, v16.4s, v1.4s\n" - "fmin v17.4s, v17.4s, v1.4s\n" - "fmin v18.4s, v18.4s, v1.4s\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmax v4.4s, v4.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" - "fmax v16.4s, v16.4s, v0.4s\n" - "fmax v17.4s, v17.4s, v0.4s\n" - "fmax v18.4s, v18.4s, v0.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" + "ld1r { v21.4s }, [x20]\n" + "fmin v4.4s, v4.4s, v22.4s\n" + "fmin v14.4s, v14.4s, v22.4s\n" + "fmin v15.4s, v15.4s, v22.4s\n" + "fmin v16.4s, v16.4s, v22.4s\n" + "fmin v17.4s, v17.4s, v22.4s\n" + "fmin v18.4s, v18.4s, v22.4s\n" + "fmin v8.4s, v8.4s, v22.4s\n" + "fmin v9.4s, v9.4s, v22.4s\n" + "fmin v10.4s, v10.4s, v22.4s\n" + "fmin v11.4s, v11.4s, v22.4s\n" + "fmin v12.4s, v12.4s, v22.4s\n" + "fmin v13.4s, v13.4s, v22.4s\n" + "fmax v4.4s, v4.4s, v21.4s\n" + "fmax v14.4s, v14.4s, v21.4s\n" + "fmax v15.4s, v15.4s, v21.4s\n" + "fmax v16.4s, v16.4s, v21.4s\n" + "fmax v17.4s, v17.4s, v21.4s\n" + "fmax v18.4s, v18.4s, v21.4s\n" + "fmax v8.4s, v8.4s, v21.4s\n" + "fmax v9.4s, v9.4s, v21.4s\n" + "fmax v10.4s, v10.4s, v21.4s\n" + "fmax v11.4s, v11.4s, v21.4s\n" + "fmax v12.4s, v12.4s, v21.4s\n" + "fmax v13.4s, v13.4s, v21.4s\n" "72:" // Height 2: No activation "cmp x9, #0x18\n" "bge 85f\n" @@ -1238,13 +1237,13 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( "106:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 107f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" "cbnz x26, 108f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20, LSL #2\n" @@ -1253,8 +1252,8 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( "b 108f\n" "107:" // Height 3: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" "108:" // Height 3: input setup done "cmp x25, #0x4\n" "blt 111f\n" @@ -1285,7 +1284,7 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" "prfm pldl1keep, [x24, #0x80]\n" ".inst 0x6e47ec5b // bfmmla v27.4s, v2.8h, v7.8h\n" - "ldr q7, [x28, #0x70]\n" + "ldr q3, [x28, #0x70]\n" ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" "prfm pldl1keep, [x23, #0x80]\n" "ld1 { v1.4s }, [x23], #0x10\n" @@ -1298,9 +1297,9 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( "prfm pldl1keep, [x22, #0x80]\n" ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" "ldr q6, [x28, #0xa0]\n" - ".inst 0x6e47ec11 // bfmmla v17.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec5d // bfmmla v29.4s, v2.8h, v7.8h\n" - "ldr q7, [x28, #0xb0]\n" + ".inst 0x6e43ec11 // bfmmla v17.4s, v0.8h, v3.8h\n" + ".inst 0x6e43ec5d // bfmmla v29.4s, v2.8h, v3.8h\n" + "ldr q3, [x28, #0xb0]\n" "add x28, x28, #0xc0\n" ".inst 0x6e44ec0c // bfmmla v12.4s, v0.8h, v4.8h\n" ".inst 0x6e44ec58 // bfmmla v24.4s, v2.8h, v4.8h\n" @@ -1311,9 +1310,9 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" ".inst 0x6e46ec59 // bfmmla v25.4s, v2.8h, v6.8h\n" "ldr q6, [x28, #0x20]\n" - ".inst 0x6e47ec13 // bfmmla v19.4s, v0.8h, v7.8h\n" + ".inst 0x6e43ec13 // bfmmla v19.4s, v0.8h, v3.8h\n" "ld1 { v0.4s }, [x24], #0x10\n" - ".inst 0x6e47ec5f // bfmmla v31.4s, v2.8h, v7.8h\n" + ".inst 0x6e43ec5f // bfmmla v31.4s, v2.8h, v3.8h\n" "ld1 { v2.4s }, [x22], #0x10\n" "ldr q7, [x28, #0x30]\n" "bge 109b\n" @@ -1324,10 +1323,10 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( "sub x25, x25, #0x4\n" ".inst 0x0ea16842 // bfcvtn v2.4h, v2.4s\n" ".inst 0x6e44ec54 // bfmmla v20.4s, v2.8h, v4.8h\n" - "ldr q4, [x28, #0x40]\n" + "ldr q3, [x28, #0x40]\n" ".inst 0x6e45ec0e // bfmmla v14.4s, v0.8h, v5.8h\n" ".inst 0x6e45ec5a // bfmmla v26.4s, v2.8h, v5.8h\n" - "ldr q5, [x28, #0x50]\n" + "ldr q4, [x28, #0x50]\n" ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" "prfm pldl1keep, [x24, #0x80]\n" ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" @@ -1335,29 +1334,29 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" "prfm pldl1keep, [x23, #0x80]\n" ".inst 0x6e47ec5b // bfmmla v27.4s, v2.8h, v7.8h\n" - "ldr q7, [x28, #0x70]\n" - ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" + "ldr q1, [x28, #0x70]\n" + ".inst 0x6e43ec0a // bfmmla v10.4s, v0.8h, v3.8h\n" "prfm pldl1keep, [x22, #0x80]\n" - ".inst 0x6e44ec56 // bfmmla v22.4s, v2.8h, v4.8h\n" - "ldr q4, [x28, #0x80]\n" - ".inst 0x6e45ec10 // bfmmla v16.4s, v0.8h, v5.8h\n" - ".inst 0x6e45ec5c // bfmmla v28.4s, v2.8h, v5.8h\n" - "ldr q5, [x28, #0x90]\n" + ".inst 0x6e43ec56 // bfmmla v22.4s, v2.8h, v3.8h\n" + "ldr q5, [x28, #0x80]\n" + ".inst 0x6e44ec10 // bfmmla v16.4s, v0.8h, v4.8h\n" + ".inst 0x6e44ec5c // bfmmla v28.4s, v2.8h, v4.8h\n" + "ldr q4, [x28, #0x90]\n" ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" - "ldr q6, [x28, #0xa0]\n" - ".inst 0x6e47ec11 // bfmmla v17.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec5d // bfmmla v29.4s, v2.8h, v7.8h\n" - "ldr q7, [x28, #0xb0]\n" + "ldr q3, [x28, #0xa0]\n" + ".inst 0x6e41ec11 // bfmmla v17.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec5d // bfmmla v29.4s, v2.8h, v1.8h\n" + "ldr q1, [x28, #0xb0]\n" "add x28, x28, #0xc0\n" - ".inst 0x6e44ec0c // bfmmla v12.4s, v0.8h, v4.8h\n" - ".inst 0x6e44ec58 // bfmmla v24.4s, v2.8h, v4.8h\n" - ".inst 0x6e45ec12 // bfmmla v18.4s, v0.8h, v5.8h\n" - ".inst 0x6e45ec5e // bfmmla v30.4s, v2.8h, v5.8h\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec59 // bfmmla v25.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec13 // bfmmla v19.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec5f // bfmmla v31.4s, v2.8h, v7.8h\n" + ".inst 0x6e45ec0c // bfmmla v12.4s, v0.8h, v5.8h\n" + ".inst 0x6e45ec58 // bfmmla v24.4s, v2.8h, v5.8h\n" + ".inst 0x6e44ec12 // bfmmla v18.4s, v0.8h, v4.8h\n" + ".inst 0x6e44ec5e // bfmmla v30.4s, v2.8h, v4.8h\n" + ".inst 0x6e43ec0d // bfmmla v13.4s, v0.8h, v3.8h\n" + ".inst 0x6e43ec59 // bfmmla v25.4s, v2.8h, v3.8h\n" + ".inst 0x6e41ec13 // bfmmla v19.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec5f // bfmmla v31.4s, v2.8h, v1.8h\n" "111:" // Height 3: Multiply loop: Main loop skip "cbz x25, 114f\n" "cbz x25, 114f\n" @@ -1375,46 +1374,46 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( "ldr s1, [x23, #0x0]\n" "ldr s2, [x22, #0x0]\n" "113:" // Height 3: Multiply loop: Ragged operand read: Done - "ldr q4, [x28, #0x0]\n" - "ldr q5, [x28, #0x10]\n" + "ldr q5, [x28, #0x0]\n" + "ldr q4, [x28, #0x10]\n" ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" ".inst 0x4ea16820 // bfcvtn2 v0.8h, v1.4s\n" - "ldr q6, [x28, #0x20]\n" - "ldr q7, [x28, #0x30]\n" + "ldr q3, [x28, #0x20]\n" + "ldr q1, [x28, #0x30]\n" ".inst 0x0ea16842 // bfcvtn v2.4h, v2.4s\n" - ".inst 0x6e44ec08 // bfmmla v8.4s, v0.8h, v4.8h\n" - ".inst 0x6e44ec54 // bfmmla v20.4s, v2.8h, v4.8h\n" - "ldr q4, [x28, #0x40]\n" - ".inst 0x6e45ec0e // bfmmla v14.4s, v0.8h, v5.8h\n" - ".inst 0x6e45ec5a // bfmmla v26.4s, v2.8h, v5.8h\n" - "ldr q5, [x28, #0x50]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" - "ldr q6, [x28, #0x60]\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec5b // bfmmla v27.4s, v2.8h, v7.8h\n" - "ldr q7, [x28, #0x70]\n" - ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" - ".inst 0x6e44ec56 // bfmmla v22.4s, v2.8h, v4.8h\n" - "ldr q4, [x28, #0x80]\n" - ".inst 0x6e45ec10 // bfmmla v16.4s, v0.8h, v5.8h\n" - ".inst 0x6e45ec5c // bfmmla v28.4s, v2.8h, v5.8h\n" - "ldr q5, [x28, #0x90]\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" - "ldr q6, [x28, #0xa0]\n" - ".inst 0x6e47ec11 // bfmmla v17.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec5d // bfmmla v29.4s, v2.8h, v7.8h\n" - "ldr q7, [x28, #0xb0]\n" - ".inst 0x6e44ec0c // bfmmla v12.4s, v0.8h, v4.8h\n" + ".inst 0x6e45ec08 // bfmmla v8.4s, v0.8h, v5.8h\n" + ".inst 0x6e45ec54 // bfmmla v20.4s, v2.8h, v5.8h\n" + "ldr q5, [x28, #0x40]\n" + ".inst 0x6e44ec0e // bfmmla v14.4s, v0.8h, v4.8h\n" + ".inst 0x6e44ec5a // bfmmla v26.4s, v2.8h, v4.8h\n" + "ldr q4, [x28, #0x50]\n" + ".inst 0x6e43ec09 // bfmmla v9.4s, v0.8h, v3.8h\n" + ".inst 0x6e43ec55 // bfmmla v21.4s, v2.8h, v3.8h\n" + "ldr q3, [x28, #0x60]\n" + ".inst 0x6e41ec0f // bfmmla v15.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec5b // bfmmla v27.4s, v2.8h, v1.8h\n" + "ldr q1, [x28, #0x70]\n" + ".inst 0x6e45ec0a // bfmmla v10.4s, v0.8h, v5.8h\n" + ".inst 0x6e45ec56 // bfmmla v22.4s, v2.8h, v5.8h\n" + "ldr q5, [x28, #0x80]\n" + ".inst 0x6e44ec10 // bfmmla v16.4s, v0.8h, v4.8h\n" + ".inst 0x6e44ec5c // bfmmla v28.4s, v2.8h, v4.8h\n" + "ldr q4, [x28, #0x90]\n" + ".inst 0x6e43ec0b // bfmmla v11.4s, v0.8h, v3.8h\n" + ".inst 0x6e43ec57 // bfmmla v23.4s, v2.8h, v3.8h\n" + "ldr q3, [x28, #0xa0]\n" + ".inst 0x6e41ec11 // bfmmla v17.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec5d // bfmmla v29.4s, v2.8h, v1.8h\n" + "ldr q1, [x28, #0xb0]\n" + ".inst 0x6e45ec0c // bfmmla v12.4s, v0.8h, v5.8h\n" "add x28, x28, #0xc0\n" - ".inst 0x6e44ec58 // bfmmla v24.4s, v2.8h, v4.8h\n" - ".inst 0x6e45ec12 // bfmmla v18.4s, v0.8h, v5.8h\n" - ".inst 0x6e45ec5e // bfmmla v30.4s, v2.8h, v5.8h\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec59 // bfmmla v25.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec13 // bfmmla v19.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec5f // bfmmla v31.4s, v2.8h, v7.8h\n" + ".inst 0x6e45ec58 // bfmmla v24.4s, v2.8h, v5.8h\n" + ".inst 0x6e44ec12 // bfmmla v18.4s, v0.8h, v4.8h\n" + ".inst 0x6e44ec5e // bfmmla v30.4s, v2.8h, v4.8h\n" + ".inst 0x6e43ec0d // bfmmla v13.4s, v0.8h, v3.8h\n" + ".inst 0x6e43ec59 // bfmmla v25.4s, v2.8h, v3.8h\n" + ".inst 0x6e41ec13 // bfmmla v19.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec5f // bfmmla v31.4s, v2.8h, v1.8h\n" "114:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x26, x26, #0x1\n" @@ -1937,14 +1936,14 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( "149:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 150f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" - "ldr x21, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" + "ldr x21, [x20, #0x18]\n" "cbnz x26, 151f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20, LSL #2\n" @@ -1954,9 +1953,9 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( "b 151f\n" "150:" // Height 4: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "add x21, x22, x20, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" + "add x21, x22, x21, LSL #2\n" "151:" // Height 4: input setup done "cmp x25, #0x4\n" "blt 154f\n" @@ -2033,39 +2032,39 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( ".inst 0x6e44ec08 // bfmmla v8.4s, v0.8h, v4.8h\n" "prfm pldl1keep, [x23, #0x80]\n" ".inst 0x6e44ec54 // bfmmla v20.4s, v2.8h, v4.8h\n" - "ldr q4, [x28, #0x40]\n" + "ldr q3, [x28, #0x40]\n" ".inst 0x6e45ec0e // bfmmla v14.4s, v0.8h, v5.8h\n" "prfm pldl1keep, [x22, #0x80]\n" ".inst 0x6e45ec5a // bfmmla v26.4s, v2.8h, v5.8h\n" - "ldr q5, [x28, #0x50]\n" + "ldr q4, [x28, #0x50]\n" ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" "prfm pldl1keep, [x21, #0x80]\n" ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" "ldr q6, [x28, #0x60]\n" ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" ".inst 0x6e47ec5b // bfmmla v27.4s, v2.8h, v7.8h\n" - "ldr q7, [x28, #0x70]\n" - ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" - ".inst 0x6e44ec56 // bfmmla v22.4s, v2.8h, v4.8h\n" - "ldr q4, [x28, #0x80]\n" - ".inst 0x6e45ec10 // bfmmla v16.4s, v0.8h, v5.8h\n" - ".inst 0x6e45ec5c // bfmmla v28.4s, v2.8h, v5.8h\n" - "ldr q5, [x28, #0x90]\n" + "ldr q1, [x28, #0x70]\n" + ".inst 0x6e43ec0a // bfmmla v10.4s, v0.8h, v3.8h\n" + ".inst 0x6e43ec56 // bfmmla v22.4s, v2.8h, v3.8h\n" + "ldr q5, [x28, #0x80]\n" + ".inst 0x6e44ec10 // bfmmla v16.4s, v0.8h, v4.8h\n" + ".inst 0x6e44ec5c // bfmmla v28.4s, v2.8h, v4.8h\n" + "ldr q4, [x28, #0x90]\n" ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" - "ldr q6, [x28, #0xa0]\n" - ".inst 0x6e47ec11 // bfmmla v17.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec5d // bfmmla v29.4s, v2.8h, v7.8h\n" - "ldr q7, [x28, #0xb0]\n" + "ldr q3, [x28, #0xa0]\n" + ".inst 0x6e41ec11 // bfmmla v17.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec5d // bfmmla v29.4s, v2.8h, v1.8h\n" + "ldr q1, [x28, #0xb0]\n" "add x28, x28, #0xc0\n" - ".inst 0x6e44ec0c // bfmmla v12.4s, v0.8h, v4.8h\n" - ".inst 0x6e44ec58 // bfmmla v24.4s, v2.8h, v4.8h\n" - ".inst 0x6e45ec12 // bfmmla v18.4s, v0.8h, v5.8h\n" - ".inst 0x6e45ec5e // bfmmla v30.4s, v2.8h, v5.8h\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec59 // bfmmla v25.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec13 // bfmmla v19.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec5f // bfmmla v31.4s, v2.8h, v7.8h\n" + ".inst 0x6e45ec0c // bfmmla v12.4s, v0.8h, v5.8h\n" + ".inst 0x6e45ec58 // bfmmla v24.4s, v2.8h, v5.8h\n" + ".inst 0x6e44ec12 // bfmmla v18.4s, v0.8h, v4.8h\n" + ".inst 0x6e44ec5e // bfmmla v30.4s, v2.8h, v4.8h\n" + ".inst 0x6e43ec0d // bfmmla v13.4s, v0.8h, v3.8h\n" + ".inst 0x6e43ec59 // bfmmla v25.4s, v2.8h, v3.8h\n" + ".inst 0x6e41ec13 // bfmmla v19.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec5f // bfmmla v31.4s, v2.8h, v1.8h\n" "154:" // Height 4: Multiply loop: Main loop skip "cbz x25, 157f\n" "cbz x25, 157f\n" @@ -2086,47 +2085,47 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( "ldr s2, [x22, #0x0]\n" "ldr s3, [x21, #0x0]\n" "156:" // Height 4: Multiply loop: Ragged operand read: Done - "ldr q4, [x28, #0x0]\n" - "ldr q5, [x28, #0x10]\n" + "ldr q5, [x28, #0x0]\n" + "ldr q4, [x28, #0x10]\n" ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" ".inst 0x0ea16842 // bfcvtn v2.4h, v2.4s\n" - "ldr q6, [x28, #0x20]\n" - "ldr q7, [x28, #0x30]\n" + "ldr q7, [x28, #0x20]\n" + "ldr q6, [x28, #0x30]\n" ".inst 0x4ea16820 // bfcvtn2 v0.8h, v1.4s\n" ".inst 0x4ea16862 // bfcvtn2 v2.8h, v3.4s\n" - ".inst 0x6e44ec08 // bfmmla v8.4s, v0.8h, v4.8h\n" - ".inst 0x6e44ec54 // bfmmla v20.4s, v2.8h, v4.8h\n" - "ldr q4, [x28, #0x40]\n" - ".inst 0x6e45ec0e // bfmmla v14.4s, v0.8h, v5.8h\n" - ".inst 0x6e45ec5a // bfmmla v26.4s, v2.8h, v5.8h\n" - "ldr q5, [x28, #0x50]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" - "ldr q6, [x28, #0x60]\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec5b // bfmmla v27.4s, v2.8h, v7.8h\n" - "ldr q7, [x28, #0x70]\n" - ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" - ".inst 0x6e44ec56 // bfmmla v22.4s, v2.8h, v4.8h\n" - "ldr q4, [x28, #0x80]\n" - ".inst 0x6e45ec10 // bfmmla v16.4s, v0.8h, v5.8h\n" - ".inst 0x6e45ec5c // bfmmla v28.4s, v2.8h, v5.8h\n" - "ldr q5, [x28, #0x90]\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec57 // bfmmla v23.4s, v2.8h, v6.8h\n" - "ldr q6, [x28, #0xa0]\n" - ".inst 0x6e47ec11 // bfmmla v17.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec5d // bfmmla v29.4s, v2.8h, v7.8h\n" - "ldr q7, [x28, #0xb0]\n" + ".inst 0x6e45ec08 // bfmmla v8.4s, v0.8h, v5.8h\n" + ".inst 0x6e45ec54 // bfmmla v20.4s, v2.8h, v5.8h\n" + "ldr q5, [x28, #0x40]\n" + ".inst 0x6e44ec0e // bfmmla v14.4s, v0.8h, v4.8h\n" + ".inst 0x6e44ec5a // bfmmla v26.4s, v2.8h, v4.8h\n" + "ldr q4, [x28, #0x50]\n" + ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" + ".inst 0x6e47ec55 // bfmmla v21.4s, v2.8h, v7.8h\n" + "ldr q3, [x28, #0x60]\n" + ".inst 0x6e46ec0f // bfmmla v15.4s, v0.8h, v6.8h\n" + ".inst 0x6e46ec5b // bfmmla v27.4s, v2.8h, v6.8h\n" + "ldr q1, [x28, #0x70]\n" + ".inst 0x6e45ec0a // bfmmla v10.4s, v0.8h, v5.8h\n" + ".inst 0x6e45ec56 // bfmmla v22.4s, v2.8h, v5.8h\n" + "ldr q5, [x28, #0x80]\n" + ".inst 0x6e44ec10 // bfmmla v16.4s, v0.8h, v4.8h\n" + ".inst 0x6e44ec5c // bfmmla v28.4s, v2.8h, v4.8h\n" + "ldr q4, [x28, #0x90]\n" + ".inst 0x6e43ec0b // bfmmla v11.4s, v0.8h, v3.8h\n" + ".inst 0x6e43ec57 // bfmmla v23.4s, v2.8h, v3.8h\n" + "ldr q3, [x28, #0xa0]\n" + ".inst 0x6e41ec11 // bfmmla v17.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec5d // bfmmla v29.4s, v2.8h, v1.8h\n" + "ldr q1, [x28, #0xb0]\n" "add x28, x28, #0xc0\n" - ".inst 0x6e44ec0c // bfmmla v12.4s, v0.8h, v4.8h\n" - ".inst 0x6e44ec58 // bfmmla v24.4s, v2.8h, v4.8h\n" - ".inst 0x6e45ec12 // bfmmla v18.4s, v0.8h, v5.8h\n" - ".inst 0x6e45ec5e // bfmmla v30.4s, v2.8h, v5.8h\n" - ".inst 0x6e46ec0d // bfmmla v13.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec59 // bfmmla v25.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec13 // bfmmla v19.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec5f // bfmmla v31.4s, v2.8h, v7.8h\n" + ".inst 0x6e45ec0c // bfmmla v12.4s, v0.8h, v5.8h\n" + ".inst 0x6e45ec58 // bfmmla v24.4s, v2.8h, v5.8h\n" + ".inst 0x6e44ec12 // bfmmla v18.4s, v0.8h, v4.8h\n" + ".inst 0x6e44ec5e // bfmmla v30.4s, v2.8h, v4.8h\n" + ".inst 0x6e43ec0d // bfmmla v13.4s, v0.8h, v3.8h\n" + ".inst 0x6e43ec59 // bfmmla v25.4s, v2.8h, v3.8h\n" + ".inst 0x6e41ec13 // bfmmla v19.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec5f // bfmmla v31.4s, v2.8h, v1.8h\n" "157:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x26, x26, #0x1\n" @@ -2415,7 +2414,6 @@ void a64_hybrid_fp32bf16fp32_mmla_4x24 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "174:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32bf16fp32_mmla_6x16.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32bf16fp32_mmla_6x16.hpp index 39ffcbef123a6c3c2b40690e2e755a83fb77b620..71e16d68b59d81bad13f31614264760279831d35 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32bf16fp32_mmla_6x16.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32bf16fp32_mmla_6x16.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef __aarch64__ + #include "../std_transforms_fixed.hpp" #include "../bfloat.hpp" #include "../performance_parameters.hpp" @@ -99,5 +99,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32bf16fp32_mmla_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32bf16fp32_mmla_6x16/generic.cpp index 4993777d620f9f2245a0d7dd1a68e290e2563d8d..5693c3f39750c102f7b1e9f9dcef21c8916afb45 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32bf16fp32_mmla_6x16/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_fp32bf16fp32_mmla_6x16/generic.cpp @@ -93,7 +93,6 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( break; } __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x6\n" "bge 176f\n" @@ -211,11 +210,11 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "16:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 17f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 18f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -233,23 +232,23 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "19:" // Height 1: Multiply loop: Main loop head ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x20]\n" + "ldr q18, [x10, #0x20]\n" ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x70]\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x6e52ec09 // bfmmla v9.4s, v0.8h, v18.8h\n" + "ldr q18, [x10, #0x40]\n" + ".inst 0x6e51ec0d // bfmmla v13.4s, v0.8h, v17.8h\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x6e52ec0a // bfmmla v10.4s, v0.8h, v18.8h\n" + "ldr q18, [x10, #0x60]\n" + ".inst 0x6e51ec0e // bfmmla v14.4s, v0.8h, v17.8h\n" + "ldr q17, [x10, #0x70]\n" "sub x27, x27, #0x4\n" "cmp x27, #0x8\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" + ".inst 0x6e52ec0b // bfmmla v11.4s, v0.8h, v18.8h\n" "add x10, x10, #0x80\n" "ldr q6, [x10, #0x0]\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" + ".inst 0x6e51ec0f // bfmmla v15.4s, v0.8h, v17.8h\n" "ldr q7, [x10, #0x10]\n" "prfm pldl1keep, [x26, #0x80]\n" "ld1 { v0.4s }, [x26], #0x10\n" @@ -257,20 +256,20 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "20:" // Height 1: Multiply loop: Single iteration only ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x20]\n" + "ldr q18, [x10, #0x20]\n" ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x70]\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x6e52ec09 // bfmmla v9.4s, v0.8h, v18.8h\n" + "ldr q18, [x10, #0x40]\n" + ".inst 0x6e51ec0d // bfmmla v13.4s, v0.8h, v17.8h\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x6e52ec0a // bfmmla v10.4s, v0.8h, v18.8h\n" + "ldr q18, [x10, #0x60]\n" + ".inst 0x6e51ec0e // bfmmla v14.4s, v0.8h, v17.8h\n" + "ldr q17, [x10, #0x70]\n" "sub x27, x27, #0x4\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" + ".inst 0x6e52ec0b // bfmmla v11.4s, v0.8h, v18.8h\n" + ".inst 0x6e51ec0f // bfmmla v15.4s, v0.8h, v17.8h\n" "prfm pldl1keep, [x26, #0x80]\n" "add x10, x10, #0x80\n" "21:" // Height 1: Multiply loop: Main loop skip @@ -284,23 +283,23 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "22:" // Height 1: Multiply loop: Ragged operand read: partial_1_0 "ldr s0, [x26, #0x0]\n" "23:" // Height 1: Multiply loop: Ragged operand read: Done - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" + "ldr q18, [x10, #0x0]\n" + "ldr q17, [x10, #0x10]\n" ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" - ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" + ".inst 0x6e52ec08 // bfmmla v8.4s, v0.8h, v18.8h\n" + "ldr q18, [x10, #0x20]\n" + ".inst 0x6e51ec0c // bfmmla v12.4s, v0.8h, v17.8h\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x6e52ec09 // bfmmla v9.4s, v0.8h, v18.8h\n" + "ldr q18, [x10, #0x40]\n" + ".inst 0x6e51ec0d // bfmmla v13.4s, v0.8h, v17.8h\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x6e52ec0a // bfmmla v10.4s, v0.8h, v18.8h\n" + "ldr q18, [x10, #0x60]\n" + ".inst 0x6e51ec0e // bfmmla v14.4s, v0.8h, v17.8h\n" + "ldr q17, [x10, #0x70]\n" + ".inst 0x6e52ec0b // bfmmla v11.4s, v0.8h, v18.8h\n" + ".inst 0x6e51ec0f // bfmmla v15.4s, v0.8h, v17.8h\n" "add x10, x10, #0x80\n" "24:" // Height 1: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -314,17 +313,17 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "uzp1 v11.2d, v11.2d, v15.2d\n" "tbz %x[flags], #1, 25f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v18.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" + "ld1r { v17.4s }, [x20]\n" + "fmin v8.4s, v8.4s, v18.4s\n" + "fmin v9.4s, v9.4s, v18.4s\n" + "fmin v10.4s, v10.4s, v18.4s\n" + "fmin v11.4s, v11.4s, v18.4s\n" + "fmax v8.4s, v8.4s, v17.4s\n" + "fmax v9.4s, v9.4s, v17.4s\n" + "fmax v10.4s, v10.4s, v17.4s\n" + "fmax v11.4s, v11.4s, v17.4s\n" "25:" // Height 1: No activation "cmp x11, #0x10\n" "bge 34f\n" @@ -515,12 +514,12 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "51:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 52f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 53f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -528,7 +527,7 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "b 53f\n" "52:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" "53:" // Height 2: input setup done "cmp x27, #0x4\n" "blt 56f\n" @@ -542,23 +541,23 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" ".inst 0x4ea16820 // bfcvtn2 v0.8h, v1.4s\n" ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x20]\n" + "ldr q18, [x10, #0x20]\n" ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x70]\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x6e52ec09 // bfmmla v9.4s, v0.8h, v18.8h\n" + "ldr q18, [x10, #0x40]\n" + ".inst 0x6e51ec0d // bfmmla v13.4s, v0.8h, v17.8h\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x6e52ec0a // bfmmla v10.4s, v0.8h, v18.8h\n" + "ldr q18, [x10, #0x60]\n" + ".inst 0x6e51ec0e // bfmmla v14.4s, v0.8h, v17.8h\n" + "ldr q17, [x10, #0x70]\n" "sub x27, x27, #0x4\n" "cmp x27, #0x8\n" "add x10, x10, #0x80\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" + ".inst 0x6e52ec0b // bfmmla v11.4s, v0.8h, v18.8h\n" "ldr q6, [x10, #0x0]\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" + ".inst 0x6e51ec0f // bfmmla v15.4s, v0.8h, v17.8h\n" "ldr q7, [x10, #0x10]\n" "prfm pldl1keep, [x26, #0x80]\n" "ld1 { v0.4s }, [x26], #0x10\n" @@ -569,20 +568,20 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" ".inst 0x4ea16820 // bfcvtn2 v0.8h, v1.4s\n" ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x20]\n" + "ldr q18, [x10, #0x20]\n" ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x70]\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x6e52ec09 // bfmmla v9.4s, v0.8h, v18.8h\n" + "ldr q18, [x10, #0x40]\n" + ".inst 0x6e51ec0d // bfmmla v13.4s, v0.8h, v17.8h\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x6e52ec0a // bfmmla v10.4s, v0.8h, v18.8h\n" + "ldr q18, [x10, #0x60]\n" + ".inst 0x6e51ec0e // bfmmla v14.4s, v0.8h, v17.8h\n" + "ldr q17, [x10, #0x70]\n" "sub x27, x27, #0x4\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" + ".inst 0x6e52ec0b // bfmmla v11.4s, v0.8h, v18.8h\n" + ".inst 0x6e51ec0f // bfmmla v15.4s, v0.8h, v17.8h\n" "prfm pldl1keep, [x26, #0x80]\n" "prfm pldl1keep, [x25, #0x80]\n" "add x10, x10, #0x80\n" @@ -600,24 +599,24 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "ldr s0, [x26, #0x0]\n" "ldr s1, [x25, #0x0]\n" "58:" // Height 2: Multiply loop: Ragged operand read: Done - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" + "ldr q18, [x10, #0x0]\n" + "ldr q17, [x10, #0x10]\n" ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" ".inst 0x4ea16820 // bfcvtn2 v0.8h, v1.4s\n" - ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" + ".inst 0x6e52ec08 // bfmmla v8.4s, v0.8h, v18.8h\n" + "ldr q18, [x10, #0x20]\n" + ".inst 0x6e51ec0c // bfmmla v12.4s, v0.8h, v17.8h\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x6e52ec09 // bfmmla v9.4s, v0.8h, v18.8h\n" + "ldr q18, [x10, #0x40]\n" + ".inst 0x6e51ec0d // bfmmla v13.4s, v0.8h, v17.8h\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x6e52ec0a // bfmmla v10.4s, v0.8h, v18.8h\n" + "ldr q18, [x10, #0x60]\n" + ".inst 0x6e51ec0e // bfmmla v14.4s, v0.8h, v17.8h\n" + "ldr q17, [x10, #0x70]\n" + ".inst 0x6e52ec0b // bfmmla v11.4s, v0.8h, v18.8h\n" + ".inst 0x6e51ec0f // bfmmla v15.4s, v0.8h, v17.8h\n" "add x10, x10, #0x80\n" "59:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -638,25 +637,25 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "uzp2 v11.2d, v11.2d, v15.2d\n" "tbz %x[flags], #1, 60f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v18.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v6.4s, v6.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmax v6.4s, v6.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" + "ld1r { v17.4s }, [x20]\n" + "fmin v6.4s, v6.4s, v18.4s\n" + "fmin v12.4s, v12.4s, v18.4s\n" + "fmin v13.4s, v13.4s, v18.4s\n" + "fmin v14.4s, v14.4s, v18.4s\n" + "fmin v8.4s, v8.4s, v18.4s\n" + "fmin v9.4s, v9.4s, v18.4s\n" + "fmin v10.4s, v10.4s, v18.4s\n" + "fmin v11.4s, v11.4s, v18.4s\n" + "fmax v6.4s, v6.4s, v17.4s\n" + "fmax v12.4s, v12.4s, v17.4s\n" + "fmax v13.4s, v13.4s, v17.4s\n" + "fmax v14.4s, v14.4s, v17.4s\n" + "fmax v8.4s, v8.4s, v17.4s\n" + "fmax v9.4s, v9.4s, v17.4s\n" + "fmax v10.4s, v10.4s, v17.4s\n" + "fmax v11.4s, v11.4s, v17.4s\n" "60:" // Height 2: No activation "cmp x11, #0x10\n" "bge 69f\n" @@ -912,13 +911,13 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "86:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 87f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 88f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -927,8 +926,8 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "b 88f\n" "87:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" "88:" // Height 3: input setup done "cmp x27, #0x4\n" "blt 91f\n" @@ -946,34 +945,34 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "sub x27, x27, #0x4\n" ".inst 0x0ea16842 // bfcvtn v2.4h, v2.4s\n" ".inst 0x6e46ec50 // bfmmla v16.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x20]\n" + "ldr q26, [x10, #0x20]\n" ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" ".inst 0x6e47ec54 // bfmmla v20.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e5aec09 // bfmmla v9.4s, v0.8h, v26.8h\n" "cmp x27, #0x8\n" - ".inst 0x6e46ec51 // bfmmla v17.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" + ".inst 0x6e5aec51 // bfmmla v17.4s, v2.8h, v26.8h\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x6e59ec0d // bfmmla v13.4s, v0.8h, v25.8h\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x6e47ec55 // bfmmla v21.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" + ".inst 0x6e59ec55 // bfmmla v21.4s, v2.8h, v25.8h\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x6e5aec0a // bfmmla v10.4s, v0.8h, v26.8h\n" "prfm pldl1keep, [x25, #0x80]\n" "ld1 { v1.4s }, [x25], #0x10\n" - ".inst 0x6e46ec52 // bfmmla v18.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec56 // bfmmla v22.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x70]\n" + ".inst 0x6e5aec52 // bfmmla v18.4s, v2.8h, v26.8h\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x6e59ec0e // bfmmla v14.4s, v0.8h, v25.8h\n" + ".inst 0x6e59ec56 // bfmmla v22.4s, v2.8h, v25.8h\n" + "ldr q25, [x10, #0x70]\n" "add x10, x10, #0x80\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec53 // bfmmla v19.4s, v2.8h, v6.8h\n" + ".inst 0x6e5aec0b // bfmmla v11.4s, v0.8h, v26.8h\n" + ".inst 0x6e5aec53 // bfmmla v19.4s, v2.8h, v26.8h\n" "ldr q6, [x10, #0x0]\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" + ".inst 0x6e59ec0f // bfmmla v15.4s, v0.8h, v25.8h\n" "ld1 { v0.4s }, [x26], #0x10\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" + ".inst 0x6e59ec57 // bfmmla v23.4s, v2.8h, v25.8h\n" "ld1 { v2.4s }, [x24], #0x10\n" "ldr q7, [x10, #0x10]\n" "bge 89b\n" @@ -984,30 +983,30 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "sub x27, x27, #0x4\n" ".inst 0x0ea16842 // bfcvtn v2.4h, v2.4s\n" ".inst 0x6e46ec50 // bfmmla v16.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x20]\n" + "ldr q26, [x10, #0x20]\n" ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" ".inst 0x6e47ec54 // bfmmla v20.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e5aec09 // bfmmla v9.4s, v0.8h, v26.8h\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x6e46ec51 // bfmmla v17.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" + ".inst 0x6e5aec51 // bfmmla v17.4s, v2.8h, v26.8h\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x6e59ec0d // bfmmla v13.4s, v0.8h, v25.8h\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x6e47ec55 // bfmmla v21.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" + ".inst 0x6e59ec55 // bfmmla v21.4s, v2.8h, v25.8h\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x6e5aec0a // bfmmla v10.4s, v0.8h, v26.8h\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x6e46ec52 // bfmmla v18.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec56 // bfmmla v22.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x70]\n" + ".inst 0x6e5aec52 // bfmmla v18.4s, v2.8h, v26.8h\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x6e59ec0e // bfmmla v14.4s, v0.8h, v25.8h\n" + ".inst 0x6e59ec56 // bfmmla v22.4s, v2.8h, v25.8h\n" + "ldr q25, [x10, #0x70]\n" "add x10, x10, #0x80\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec53 // bfmmla v19.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" + ".inst 0x6e5aec0b // bfmmla v11.4s, v0.8h, v26.8h\n" + ".inst 0x6e5aec53 // bfmmla v19.4s, v2.8h, v26.8h\n" + ".inst 0x6e59ec0f // bfmmla v15.4s, v0.8h, v25.8h\n" + ".inst 0x6e59ec57 // bfmmla v23.4s, v2.8h, v25.8h\n" "91:" // Height 3: Multiply loop: Main loop skip "cbz x27, 94f\n" "cbz x27, 94f\n" @@ -1025,34 +1024,34 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "ldr s1, [x25, #0x0]\n" "ldr s2, [x24, #0x0]\n" "93:" // Height 3: Multiply loop: Ragged operand read: Done - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" + "ldr q26, [x10, #0x0]\n" + "ldr q25, [x10, #0x10]\n" ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" ".inst 0x4ea16820 // bfcvtn2 v0.8h, v1.4s\n" ".inst 0x0ea16842 // bfcvtn v2.4h, v2.4s\n" - ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec50 // bfmmla v16.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec54 // bfmmla v20.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec51 // bfmmla v17.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec55 // bfmmla v21.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec52 // bfmmla v18.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec56 // bfmmla v22.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x70]\n" + ".inst 0x6e5aec08 // bfmmla v8.4s, v0.8h, v26.8h\n" + ".inst 0x6e5aec50 // bfmmla v16.4s, v2.8h, v26.8h\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x6e59ec0c // bfmmla v12.4s, v0.8h, v25.8h\n" + ".inst 0x6e59ec54 // bfmmla v20.4s, v2.8h, v25.8h\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e5aec09 // bfmmla v9.4s, v0.8h, v26.8h\n" + ".inst 0x6e5aec51 // bfmmla v17.4s, v2.8h, v26.8h\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x6e59ec0d // bfmmla v13.4s, v0.8h, v25.8h\n" + ".inst 0x6e59ec55 // bfmmla v21.4s, v2.8h, v25.8h\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x6e5aec0a // bfmmla v10.4s, v0.8h, v26.8h\n" + ".inst 0x6e5aec52 // bfmmla v18.4s, v2.8h, v26.8h\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x6e59ec0e // bfmmla v14.4s, v0.8h, v25.8h\n" + ".inst 0x6e59ec56 // bfmmla v22.4s, v2.8h, v25.8h\n" + "ldr q25, [x10, #0x70]\n" "add x10, x10, #0x80\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec53 // bfmmla v19.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" + ".inst 0x6e5aec0b // bfmmla v11.4s, v0.8h, v26.8h\n" + ".inst 0x6e5aec53 // bfmmla v19.4s, v2.8h, v26.8h\n" + ".inst 0x6e59ec0f // bfmmla v15.4s, v0.8h, v25.8h\n" + ".inst 0x6e59ec57 // bfmmla v23.4s, v2.8h, v25.8h\n" "94:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1078,33 +1077,33 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "uzp1 v19.2d, v19.2d, v23.2d\n" "tbz %x[flags], #1, 95f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v26.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v6.4s, v6.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v16.4s, v16.4s, v1.4s\n" - "fmin v17.4s, v17.4s, v1.4s\n" - "fmin v18.4s, v18.4s, v1.4s\n" - "fmin v19.4s, v19.4s, v1.4s\n" - "fmax v6.4s, v6.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v16.4s, v16.4s, v0.4s\n" - "fmax v17.4s, v17.4s, v0.4s\n" - "fmax v18.4s, v18.4s, v0.4s\n" - "fmax v19.4s, v19.4s, v0.4s\n" + "ld1r { v25.4s }, [x20]\n" + "fmin v6.4s, v6.4s, v26.4s\n" + "fmin v12.4s, v12.4s, v26.4s\n" + "fmin v13.4s, v13.4s, v26.4s\n" + "fmin v14.4s, v14.4s, v26.4s\n" + "fmin v8.4s, v8.4s, v26.4s\n" + "fmin v9.4s, v9.4s, v26.4s\n" + "fmin v10.4s, v10.4s, v26.4s\n" + "fmin v11.4s, v11.4s, v26.4s\n" + "fmin v16.4s, v16.4s, v26.4s\n" + "fmin v17.4s, v17.4s, v26.4s\n" + "fmin v18.4s, v18.4s, v26.4s\n" + "fmin v19.4s, v19.4s, v26.4s\n" + "fmax v6.4s, v6.4s, v25.4s\n" + "fmax v12.4s, v12.4s, v25.4s\n" + "fmax v13.4s, v13.4s, v25.4s\n" + "fmax v14.4s, v14.4s, v25.4s\n" + "fmax v8.4s, v8.4s, v25.4s\n" + "fmax v9.4s, v9.4s, v25.4s\n" + "fmax v10.4s, v10.4s, v25.4s\n" + "fmax v11.4s, v11.4s, v25.4s\n" + "fmax v16.4s, v16.4s, v25.4s\n" + "fmax v17.4s, v17.4s, v25.4s\n" + "fmax v18.4s, v18.4s, v25.4s\n" + "fmax v19.4s, v19.4s, v25.4s\n" "95:" // Height 3: No activation "cmp x11, #0x10\n" "bge 104f\n" @@ -1401,14 +1400,14 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "121:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 122f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 123f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -1418,9 +1417,9 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "b 123f\n" "122:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" "123:" // Height 4: input setup done "cmp x27, #0x4\n" "blt 126f\n" @@ -1442,34 +1441,34 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" "prfm pldl1keep, [x26, #0x80]\n" ".inst 0x6e46ec50 // bfmmla v16.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x20]\n" + "ldr q26, [x10, #0x20]\n" ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" "prfm pldl1keep, [x25, #0x80]\n" "ld1 { v1.4s }, [x25], #0x10\n" ".inst 0x6e47ec54 // bfmmla v20.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec51 // bfmmla v17.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e5aec09 // bfmmla v9.4s, v0.8h, v26.8h\n" + ".inst 0x6e5aec51 // bfmmla v17.4s, v2.8h, v26.8h\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x6e59ec0d // bfmmla v13.4s, v0.8h, v25.8h\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x6e47ec55 // bfmmla v21.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" + ".inst 0x6e59ec55 // bfmmla v21.4s, v2.8h, v25.8h\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x6e5aec0a // bfmmla v10.4s, v0.8h, v26.8h\n" "prfm pldl1keep, [x23, #0x80]\n" "ld1 { v3.4s }, [x23], #0x10\n" - ".inst 0x6e46ec52 // bfmmla v18.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec56 // bfmmla v22.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x70]\n" + ".inst 0x6e5aec52 // bfmmla v18.4s, v2.8h, v26.8h\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x6e59ec0e // bfmmla v14.4s, v0.8h, v25.8h\n" + ".inst 0x6e59ec56 // bfmmla v22.4s, v2.8h, v25.8h\n" + "ldr q25, [x10, #0x70]\n" "add x10, x10, #0x80\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec53 // bfmmla v19.4s, v2.8h, v6.8h\n" + ".inst 0x6e5aec0b // bfmmla v11.4s, v0.8h, v26.8h\n" + ".inst 0x6e5aec53 // bfmmla v19.4s, v2.8h, v26.8h\n" "ldr q6, [x10, #0x0]\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" + ".inst 0x6e59ec0f // bfmmla v15.4s, v0.8h, v25.8h\n" "ld1 { v0.4s }, [x26], #0x10\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" + ".inst 0x6e59ec57 // bfmmla v23.4s, v2.8h, v25.8h\n" "ld1 { v2.4s }, [x24], #0x10\n" "ldr q7, [x10, #0x10]\n" "bge 124b\n" @@ -1483,29 +1482,29 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" "prfm pldl1keep, [x25, #0x80]\n" ".inst 0x6e46ec50 // bfmmla v16.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x20]\n" + "ldr q26, [x10, #0x20]\n" ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" "prfm pldl1keep, [x24, #0x80]\n" ".inst 0x6e47ec54 // bfmmla v20.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e5aec09 // bfmmla v9.4s, v0.8h, v26.8h\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x6e46ec51 // bfmmla v17.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec55 // bfmmla v21.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec52 // bfmmla v18.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec56 // bfmmla v22.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x70]\n" + ".inst 0x6e5aec51 // bfmmla v17.4s, v2.8h, v26.8h\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x6e59ec0d // bfmmla v13.4s, v0.8h, v25.8h\n" + ".inst 0x6e59ec55 // bfmmla v21.4s, v2.8h, v25.8h\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x6e5aec0a // bfmmla v10.4s, v0.8h, v26.8h\n" + ".inst 0x6e5aec52 // bfmmla v18.4s, v2.8h, v26.8h\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x6e59ec0e // bfmmla v14.4s, v0.8h, v25.8h\n" + ".inst 0x6e59ec56 // bfmmla v22.4s, v2.8h, v25.8h\n" + "ldr q25, [x10, #0x70]\n" "add x10, x10, #0x80\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec53 // bfmmla v19.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" + ".inst 0x6e5aec0b // bfmmla v11.4s, v0.8h, v26.8h\n" + ".inst 0x6e5aec53 // bfmmla v19.4s, v2.8h, v26.8h\n" + ".inst 0x6e59ec0f // bfmmla v15.4s, v0.8h, v25.8h\n" + ".inst 0x6e59ec57 // bfmmla v23.4s, v2.8h, v25.8h\n" "126:" // Height 4: Multiply loop: Main loop skip "cbz x27, 129f\n" "cbz x27, 129f\n" @@ -1526,35 +1525,35 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "ldr s2, [x24, #0x0]\n" "ldr s3, [x23, #0x0]\n" "128:" // Height 4: Multiply loop: Ragged operand read: Done - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" + "ldr q26, [x10, #0x0]\n" + "ldr q25, [x10, #0x10]\n" ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" ".inst 0x0ea16842 // bfcvtn v2.4h, v2.4s\n" ".inst 0x4ea16820 // bfcvtn2 v0.8h, v1.4s\n" ".inst 0x4ea16862 // bfcvtn2 v2.8h, v3.4s\n" - ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec50 // bfmmla v16.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec54 // bfmmla v20.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec51 // bfmmla v17.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec55 // bfmmla v21.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec52 // bfmmla v18.4s, v2.8h, v6.8h\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec56 // bfmmla v22.4s, v2.8h, v7.8h\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" + ".inst 0x6e5aec08 // bfmmla v8.4s, v0.8h, v26.8h\n" + ".inst 0x6e5aec50 // bfmmla v16.4s, v2.8h, v26.8h\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x6e59ec0c // bfmmla v12.4s, v0.8h, v25.8h\n" + ".inst 0x6e59ec54 // bfmmla v20.4s, v2.8h, v25.8h\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e5aec09 // bfmmla v9.4s, v0.8h, v26.8h\n" + ".inst 0x6e5aec51 // bfmmla v17.4s, v2.8h, v26.8h\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x6e59ec0d // bfmmla v13.4s, v0.8h, v25.8h\n" + ".inst 0x6e59ec55 // bfmmla v21.4s, v2.8h, v25.8h\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x6e5aec0a // bfmmla v10.4s, v0.8h, v26.8h\n" + ".inst 0x6e5aec52 // bfmmla v18.4s, v2.8h, v26.8h\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x6e59ec0e // bfmmla v14.4s, v0.8h, v25.8h\n" + ".inst 0x6e59ec56 // bfmmla v22.4s, v2.8h, v25.8h\n" + "ldr q25, [x10, #0x70]\n" + ".inst 0x6e5aec0b // bfmmla v11.4s, v0.8h, v26.8h\n" "add x10, x10, #0x80\n" - ".inst 0x6e46ec53 // bfmmla v19.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" + ".inst 0x6e5aec53 // bfmmla v19.4s, v2.8h, v26.8h\n" + ".inst 0x6e59ec0f // bfmmla v15.4s, v0.8h, v25.8h\n" + ".inst 0x6e59ec57 // bfmmla v23.4s, v2.8h, v25.8h\n" "129:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1586,41 +1585,41 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "uzp2 v19.2d, v19.2d, v23.2d\n" "tbz %x[flags], #1, 130f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1r { v1.4s }, [x20]\n" + "ld1r { v26.4s }, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1r { v0.4s }, [x20]\n" - "fmin v6.4s, v6.4s, v1.4s\n" - "fmin v12.4s, v12.4s, v1.4s\n" - "fmin v13.4s, v13.4s, v1.4s\n" - "fmin v14.4s, v14.4s, v1.4s\n" - "fmin v8.4s, v8.4s, v1.4s\n" - "fmin v9.4s, v9.4s, v1.4s\n" - "fmin v10.4s, v10.4s, v1.4s\n" - "fmin v11.4s, v11.4s, v1.4s\n" - "fmin v15.4s, v15.4s, v1.4s\n" - "fmin v20.4s, v20.4s, v1.4s\n" - "fmin v21.4s, v21.4s, v1.4s\n" - "fmin v22.4s, v22.4s, v1.4s\n" - "fmin v16.4s, v16.4s, v1.4s\n" - "fmin v17.4s, v17.4s, v1.4s\n" - "fmin v18.4s, v18.4s, v1.4s\n" - "fmin v19.4s, v19.4s, v1.4s\n" - "fmax v6.4s, v6.4s, v0.4s\n" - "fmax v12.4s, v12.4s, v0.4s\n" - "fmax v13.4s, v13.4s, v0.4s\n" - "fmax v14.4s, v14.4s, v0.4s\n" - "fmax v8.4s, v8.4s, v0.4s\n" - "fmax v9.4s, v9.4s, v0.4s\n" - "fmax v10.4s, v10.4s, v0.4s\n" - "fmax v11.4s, v11.4s, v0.4s\n" - "fmax v15.4s, v15.4s, v0.4s\n" - "fmax v20.4s, v20.4s, v0.4s\n" - "fmax v21.4s, v21.4s, v0.4s\n" - "fmax v22.4s, v22.4s, v0.4s\n" - "fmax v16.4s, v16.4s, v0.4s\n" - "fmax v17.4s, v17.4s, v0.4s\n" - "fmax v18.4s, v18.4s, v0.4s\n" - "fmax v19.4s, v19.4s, v0.4s\n" + "ld1r { v25.4s }, [x20]\n" + "fmin v6.4s, v6.4s, v26.4s\n" + "fmin v12.4s, v12.4s, v26.4s\n" + "fmin v13.4s, v13.4s, v26.4s\n" + "fmin v14.4s, v14.4s, v26.4s\n" + "fmin v8.4s, v8.4s, v26.4s\n" + "fmin v9.4s, v9.4s, v26.4s\n" + "fmin v10.4s, v10.4s, v26.4s\n" + "fmin v11.4s, v11.4s, v26.4s\n" + "fmin v15.4s, v15.4s, v26.4s\n" + "fmin v20.4s, v20.4s, v26.4s\n" + "fmin v21.4s, v21.4s, v26.4s\n" + "fmin v22.4s, v22.4s, v26.4s\n" + "fmin v16.4s, v16.4s, v26.4s\n" + "fmin v17.4s, v17.4s, v26.4s\n" + "fmin v18.4s, v18.4s, v26.4s\n" + "fmin v19.4s, v19.4s, v26.4s\n" + "fmax v6.4s, v6.4s, v25.4s\n" + "fmax v12.4s, v12.4s, v25.4s\n" + "fmax v13.4s, v13.4s, v25.4s\n" + "fmax v14.4s, v14.4s, v25.4s\n" + "fmax v8.4s, v8.4s, v25.4s\n" + "fmax v9.4s, v9.4s, v25.4s\n" + "fmax v10.4s, v10.4s, v25.4s\n" + "fmax v11.4s, v11.4s, v25.4s\n" + "fmax v15.4s, v15.4s, v25.4s\n" + "fmax v20.4s, v20.4s, v25.4s\n" + "fmax v21.4s, v21.4s, v25.4s\n" + "fmax v22.4s, v22.4s, v25.4s\n" + "fmax v16.4s, v16.4s, v25.4s\n" + "fmax v17.4s, v17.4s, v25.4s\n" + "fmax v18.4s, v18.4s, v25.4s\n" + "fmax v19.4s, v19.4s, v25.4s\n" "130:" // Height 4: No activation "cmp x11, #0x10\n" "bge 139f\n" @@ -1982,15 +1981,15 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "156:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 157f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 158f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -2001,10 +2000,10 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "b 158f\n" "157:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" "158:" // Height 5: input setup done "cmp x27, #0x4\n" "blt 161f\n" @@ -2029,43 +2028,43 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( ".inst 0x0ea16884 // bfcvtn v4.4h, v4.4s\n" ".inst 0x6e46ec50 // bfmmla v16.4s, v2.8h, v6.8h\n" ".inst 0x6e46ec98 // bfmmla v24.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x20]\n" + "ldr q3, [x10, #0x20]\n" ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" ".inst 0x6e47ec54 // bfmmla v20.4s, v2.8h, v7.8h\n" "prfm pldl1keep, [x25, #0x80]\n" "ld1 { v1.4s }, [x25], #0x10\n" ".inst 0x6e47ec9c // bfmmla v28.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" + "ldr q5, [x10, #0x30]\n" + ".inst 0x6e43ec09 // bfmmla v9.4s, v0.8h, v3.8h\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x6e46ec51 // bfmmla v17.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec99 // bfmmla v25.4s, v4.8h, v6.8h\n" + ".inst 0x6e43ec51 // bfmmla v17.4s, v2.8h, v3.8h\n" + ".inst 0x6e43ec99 // bfmmla v25.4s, v4.8h, v3.8h\n" "ldr q6, [x10, #0x40]\n" "prfm pldl1keep, [x23, #0x80]\n" "ld1 { v3.4s }, [x23], #0x10\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec55 // bfmmla v21.4s, v2.8h, v7.8h\n" + ".inst 0x6e45ec0d // bfmmla v13.4s, v0.8h, v5.8h\n" + ".inst 0x6e45ec55 // bfmmla v21.4s, v2.8h, v5.8h\n" "prfm pldl1keep, [x22, #0x80]\n" - ".inst 0x6e47ec9d // bfmmla v29.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x50]\n" + ".inst 0x6e45ec9d // bfmmla v29.4s, v4.8h, v5.8h\n" + "ldr q5, [x10, #0x50]\n" ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" ".inst 0x6e46ec52 // bfmmla v18.4s, v2.8h, v6.8h\n" ".inst 0x6e46ec9a // bfmmla v26.4s, v4.8h, v6.8h\n" "ldr q6, [x10, #0x60]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec56 // bfmmla v22.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9e // bfmmla v30.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x70]\n" + ".inst 0x6e45ec0e // bfmmla v14.4s, v0.8h, v5.8h\n" + ".inst 0x6e45ec56 // bfmmla v22.4s, v2.8h, v5.8h\n" + ".inst 0x6e45ec9e // bfmmla v30.4s, v4.8h, v5.8h\n" + "ldr q5, [x10, #0x70]\n" "add x10, x10, #0x80\n" ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" ".inst 0x6e46ec53 // bfmmla v19.4s, v2.8h, v6.8h\n" ".inst 0x6e46ec9b // bfmmla v27.4s, v4.8h, v6.8h\n" "ldr q6, [x10, #0x0]\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" + ".inst 0x6e45ec0f // bfmmla v15.4s, v0.8h, v5.8h\n" "ld1 { v0.4s }, [x26], #0x10\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" + ".inst 0x6e45ec57 // bfmmla v23.4s, v2.8h, v5.8h\n" "ld1 { v2.4s }, [x24], #0x10\n" - ".inst 0x6e47ec9f // bfmmla v31.4s, v4.8h, v7.8h\n" + ".inst 0x6e45ec9f // bfmmla v31.4s, v4.8h, v5.8h\n" "ld1 { v4.4s }, [x22], #0x10\n" "ldr q7, [x10, #0x10]\n" "bge 159b\n" @@ -2081,37 +2080,37 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( ".inst 0x0ea16884 // bfcvtn v4.4h, v4.4s\n" ".inst 0x6e46ec50 // bfmmla v16.4s, v2.8h, v6.8h\n" ".inst 0x6e46ec98 // bfmmla v24.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x20]\n" + "ldr q3, [x10, #0x20]\n" ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" ".inst 0x6e47ec54 // bfmmla v20.4s, v2.8h, v7.8h\n" "prfm pldl1keep, [x24, #0x80]\n" "prfm pldl1keep, [x23, #0x80]\n" ".inst 0x6e47ec9c // bfmmla v28.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" + "ldr q1, [x10, #0x30]\n" + ".inst 0x6e43ec09 // bfmmla v9.4s, v0.8h, v3.8h\n" "prfm pldl1keep, [x22, #0x80]\n" - ".inst 0x6e46ec51 // bfmmla v17.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec99 // bfmmla v25.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec55 // bfmmla v21.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9d // bfmmla v29.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec52 // bfmmla v18.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9a // bfmmla v26.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec56 // bfmmla v22.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9e // bfmmla v30.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x70]\n" + ".inst 0x6e43ec51 // bfmmla v17.4s, v2.8h, v3.8h\n" + ".inst 0x6e43ec99 // bfmmla v25.4s, v4.8h, v3.8h\n" + "ldr q3, [x10, #0x40]\n" + ".inst 0x6e41ec0d // bfmmla v13.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec55 // bfmmla v21.4s, v2.8h, v1.8h\n" + ".inst 0x6e41ec9d // bfmmla v29.4s, v4.8h, v1.8h\n" + "ldr q1, [x10, #0x50]\n" + ".inst 0x6e43ec0a // bfmmla v10.4s, v0.8h, v3.8h\n" + ".inst 0x6e43ec52 // bfmmla v18.4s, v2.8h, v3.8h\n" + ".inst 0x6e43ec9a // bfmmla v26.4s, v4.8h, v3.8h\n" + "ldr q3, [x10, #0x60]\n" + ".inst 0x6e41ec0e // bfmmla v14.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec56 // bfmmla v22.4s, v2.8h, v1.8h\n" + ".inst 0x6e41ec9e // bfmmla v30.4s, v4.8h, v1.8h\n" + "ldr q1, [x10, #0x70]\n" "add x10, x10, #0x80\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec53 // bfmmla v19.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9b // bfmmla v27.4s, v4.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9f // bfmmla v31.4s, v4.8h, v7.8h\n" + ".inst 0x6e43ec0b // bfmmla v11.4s, v0.8h, v3.8h\n" + ".inst 0x6e43ec53 // bfmmla v19.4s, v2.8h, v3.8h\n" + ".inst 0x6e43ec9b // bfmmla v27.4s, v4.8h, v3.8h\n" + ".inst 0x6e41ec0f // bfmmla v15.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec57 // bfmmla v23.4s, v2.8h, v1.8h\n" + ".inst 0x6e41ec9f // bfmmla v31.4s, v4.8h, v1.8h\n" "161:" // Height 5: Multiply loop: Main loop skip "cbz x27, 164f\n" "cbz x27, 164f\n" @@ -2136,7 +2135,7 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "ldr s4, [x22, #0x0]\n" "163:" // Height 5: Multiply loop: Ragged operand read: Done "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" + "ldr q5, [x10, #0x10]\n" ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" ".inst 0x0ea16842 // bfcvtn v2.4h, v2.4s\n" ".inst 0x4ea16820 // bfcvtn2 v0.8h, v1.4s\n" @@ -2145,34 +2144,34 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( ".inst 0x0ea16884 // bfcvtn v4.4h, v4.4s\n" ".inst 0x6e46ec50 // bfmmla v16.4s, v2.8h, v6.8h\n" ".inst 0x6e46ec98 // bfmmla v24.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec54 // bfmmla v20.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9c // bfmmla v28.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec51 // bfmmla v17.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec99 // bfmmla v25.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec55 // bfmmla v21.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9d // bfmmla v29.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec52 // bfmmla v18.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9a // bfmmla v26.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec56 // bfmmla v22.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9e // bfmmla v30.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" + "ldr q3, [x10, #0x20]\n" + ".inst 0x6e45ec0c // bfmmla v12.4s, v0.8h, v5.8h\n" + ".inst 0x6e45ec54 // bfmmla v20.4s, v2.8h, v5.8h\n" + ".inst 0x6e45ec9c // bfmmla v28.4s, v4.8h, v5.8h\n" + "ldr q1, [x10, #0x30]\n" + ".inst 0x6e43ec09 // bfmmla v9.4s, v0.8h, v3.8h\n" + ".inst 0x6e43ec51 // bfmmla v17.4s, v2.8h, v3.8h\n" + ".inst 0x6e43ec99 // bfmmla v25.4s, v4.8h, v3.8h\n" + "ldr q3, [x10, #0x40]\n" + ".inst 0x6e41ec0d // bfmmla v13.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec55 // bfmmla v21.4s, v2.8h, v1.8h\n" + ".inst 0x6e41ec9d // bfmmla v29.4s, v4.8h, v1.8h\n" + "ldr q1, [x10, #0x50]\n" + ".inst 0x6e43ec0a // bfmmla v10.4s, v0.8h, v3.8h\n" + ".inst 0x6e43ec52 // bfmmla v18.4s, v2.8h, v3.8h\n" + ".inst 0x6e43ec9a // bfmmla v26.4s, v4.8h, v3.8h\n" + "ldr q3, [x10, #0x60]\n" + ".inst 0x6e41ec0e // bfmmla v14.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec56 // bfmmla v22.4s, v2.8h, v1.8h\n" + ".inst 0x6e41ec9e // bfmmla v30.4s, v4.8h, v1.8h\n" + "ldr q1, [x10, #0x70]\n" + ".inst 0x6e43ec0b // bfmmla v11.4s, v0.8h, v3.8h\n" "add x10, x10, #0x80\n" - ".inst 0x6e46ec53 // bfmmla v19.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9b // bfmmla v27.4s, v4.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9f // bfmmla v31.4s, v4.8h, v7.8h\n" + ".inst 0x6e43ec53 // bfmmla v19.4s, v2.8h, v3.8h\n" + ".inst 0x6e43ec9b // bfmmla v27.4s, v4.8h, v3.8h\n" + ".inst 0x6e41ec0f // bfmmla v15.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec57 // bfmmla v23.4s, v2.8h, v1.8h\n" + ".inst 0x6e41ec9f // bfmmla v31.4s, v4.8h, v1.8h\n" "164:" // Height 5: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -2658,16 +2657,16 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "191:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 192f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 193f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -2679,11 +2678,11 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "b 193f\n" "192:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "add x21, x22, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" + "add x21, x22, x21, LSL #2\n" "193:" // Height 6: input setup done "cmp x27, #0x4\n" "blt 196f\n" @@ -2716,7 +2715,7 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" ".inst 0x6e47ec54 // bfmmla v20.4s, v2.8h, v7.8h\n" ".inst 0x6e47ec9c // bfmmla v28.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x30]\n" + "ldr q5, [x10, #0x30]\n" "prfm pldl1keep, [x24, #0x80]\n" ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" ".inst 0x6e46ec51 // bfmmla v17.4s, v2.8h, v6.8h\n" @@ -2724,10 +2723,10 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "ld1 { v3.4s }, [x23], #0x10\n" ".inst 0x6e46ec99 // bfmmla v25.4s, v4.8h, v6.8h\n" "ldr q6, [x10, #0x40]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" + ".inst 0x6e45ec0d // bfmmla v13.4s, v0.8h, v5.8h\n" "prfm pldl1keep, [x22, #0x80]\n" - ".inst 0x6e47ec55 // bfmmla v21.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9d // bfmmla v29.4s, v4.8h, v7.8h\n" + ".inst 0x6e45ec55 // bfmmla v21.4s, v2.8h, v5.8h\n" + ".inst 0x6e45ec9d // bfmmla v29.4s, v4.8h, v5.8h\n" "ldr q7, [x10, #0x50]\n" "prfm pldl1keep, [x21, #0x80]\n" "ld1 { v5.4s }, [x21], #0x10\n" @@ -2766,37 +2765,37 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( ".inst 0x6e46ec50 // bfmmla v16.4s, v2.8h, v6.8h\n" "prfm pldl1keep, [x24, #0x80]\n" ".inst 0x6e46ec98 // bfmmla v24.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x20]\n" + "ldr q3, [x10, #0x20]\n" ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" "prfm pldl1keep, [x23, #0x80]\n" ".inst 0x6e47ec54 // bfmmla v20.4s, v2.8h, v7.8h\n" ".inst 0x6e47ec9c // bfmmla v28.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x30]\n" + "ldr q1, [x10, #0x30]\n" "prfm pldl1keep, [x22, #0x80]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec51 // bfmmla v17.4s, v2.8h, v6.8h\n" + ".inst 0x6e43ec09 // bfmmla v9.4s, v0.8h, v3.8h\n" + ".inst 0x6e43ec51 // bfmmla v17.4s, v2.8h, v3.8h\n" "prfm pldl1keep, [x21, #0x80]\n" - ".inst 0x6e46ec99 // bfmmla v25.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec55 // bfmmla v21.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9d // bfmmla v29.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec52 // bfmmla v18.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9a // bfmmla v26.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec56 // bfmmla v22.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9e // bfmmla v30.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x70]\n" + ".inst 0x6e43ec99 // bfmmla v25.4s, v4.8h, v3.8h\n" + "ldr q3, [x10, #0x40]\n" + ".inst 0x6e41ec0d // bfmmla v13.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec55 // bfmmla v21.4s, v2.8h, v1.8h\n" + ".inst 0x6e41ec9d // bfmmla v29.4s, v4.8h, v1.8h\n" + "ldr q1, [x10, #0x50]\n" + ".inst 0x6e43ec0a // bfmmla v10.4s, v0.8h, v3.8h\n" + ".inst 0x6e43ec52 // bfmmla v18.4s, v2.8h, v3.8h\n" + ".inst 0x6e43ec9a // bfmmla v26.4s, v4.8h, v3.8h\n" + "ldr q3, [x10, #0x60]\n" + ".inst 0x6e41ec0e // bfmmla v14.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec56 // bfmmla v22.4s, v2.8h, v1.8h\n" + ".inst 0x6e41ec9e // bfmmla v30.4s, v4.8h, v1.8h\n" + "ldr q1, [x10, #0x70]\n" "add x10, x10, #0x80\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec53 // bfmmla v19.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9b // bfmmla v27.4s, v4.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9f // bfmmla v31.4s, v4.8h, v7.8h\n" + ".inst 0x6e43ec0b // bfmmla v11.4s, v0.8h, v3.8h\n" + ".inst 0x6e43ec53 // bfmmla v19.4s, v2.8h, v3.8h\n" + ".inst 0x6e43ec9b // bfmmla v27.4s, v4.8h, v3.8h\n" + ".inst 0x6e41ec0f // bfmmla v15.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec57 // bfmmla v23.4s, v2.8h, v1.8h\n" + ".inst 0x6e41ec9f // bfmmla v31.4s, v4.8h, v1.8h\n" "196:" // Height 6: Multiply loop: Main loop skip "cbz x27, 199f\n" "cbz x27, 199f\n" @@ -2823,45 +2822,45 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "ldr s4, [x22, #0x0]\n" "ldr s5, [x21, #0x0]\n" "198:" // Height 6: Multiply loop: Ragged operand read: Done - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" + "ldr q7, [x10, #0x0]\n" + "ldr q6, [x10, #0x10]\n" ".inst 0x0ea16800 // bfcvtn v0.4h, v0.4s\n" ".inst 0x0ea16842 // bfcvtn v2.4h, v2.4s\n" ".inst 0x0ea16884 // bfcvtn v4.4h, v4.4s\n" ".inst 0x4ea16820 // bfcvtn2 v0.8h, v1.4s\n" - ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" + ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" ".inst 0x4ea16862 // bfcvtn2 v2.8h, v3.4s\n" ".inst 0x4ea168a4 // bfcvtn2 v4.8h, v5.4s\n" - ".inst 0x6e46ec50 // bfmmla v16.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec98 // bfmmla v24.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec54 // bfmmla v20.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9c // bfmmla v28.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec51 // bfmmla v17.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec99 // bfmmla v25.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec55 // bfmmla v21.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9d // bfmmla v29.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec52 // bfmmla v18.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9a // bfmmla v26.4s, v4.8h, v6.8h\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6e47ec0e // bfmmla v14.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec56 // bfmmla v22.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9e // bfmmla v30.4s, v4.8h, v7.8h\n" - "ldr q7, [x10, #0x70]\n" + ".inst 0x6e47ec50 // bfmmla v16.4s, v2.8h, v7.8h\n" + ".inst 0x6e47ec98 // bfmmla v24.4s, v4.8h, v7.8h\n" + "ldr q3, [x10, #0x20]\n" + ".inst 0x6e46ec0c // bfmmla v12.4s, v0.8h, v6.8h\n" + ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" + ".inst 0x6e46ec9c // bfmmla v28.4s, v4.8h, v6.8h\n" + "ldr q1, [x10, #0x30]\n" + ".inst 0x6e43ec09 // bfmmla v9.4s, v0.8h, v3.8h\n" + ".inst 0x6e43ec51 // bfmmla v17.4s, v2.8h, v3.8h\n" + ".inst 0x6e43ec99 // bfmmla v25.4s, v4.8h, v3.8h\n" + "ldr q3, [x10, #0x40]\n" + ".inst 0x6e41ec0d // bfmmla v13.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec55 // bfmmla v21.4s, v2.8h, v1.8h\n" + ".inst 0x6e41ec9d // bfmmla v29.4s, v4.8h, v1.8h\n" + "ldr q1, [x10, #0x50]\n" + ".inst 0x6e43ec0a // bfmmla v10.4s, v0.8h, v3.8h\n" + ".inst 0x6e43ec52 // bfmmla v18.4s, v2.8h, v3.8h\n" + ".inst 0x6e43ec9a // bfmmla v26.4s, v4.8h, v3.8h\n" + "ldr q3, [x10, #0x60]\n" + ".inst 0x6e41ec0e // bfmmla v14.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec56 // bfmmla v22.4s, v2.8h, v1.8h\n" + ".inst 0x6e41ec9e // bfmmla v30.4s, v4.8h, v1.8h\n" + "ldr q1, [x10, #0x70]\n" "add x10, x10, #0x80\n" - ".inst 0x6e46ec0b // bfmmla v11.4s, v0.8h, v6.8h\n" - ".inst 0x6e46ec53 // bfmmla v19.4s, v2.8h, v6.8h\n" - ".inst 0x6e46ec9b // bfmmla v27.4s, v4.8h, v6.8h\n" - ".inst 0x6e47ec0f // bfmmla v15.4s, v0.8h, v7.8h\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" - ".inst 0x6e47ec9f // bfmmla v31.4s, v4.8h, v7.8h\n" + ".inst 0x6e43ec0b // bfmmla v11.4s, v0.8h, v3.8h\n" + ".inst 0x6e43ec53 // bfmmla v19.4s, v2.8h, v3.8h\n" + ".inst 0x6e43ec9b // bfmmla v27.4s, v4.8h, v3.8h\n" + ".inst 0x6e41ec0f // bfmmla v15.4s, v0.8h, v1.8h\n" + ".inst 0x6e41ec57 // bfmmla v23.4s, v2.8h, v1.8h\n" + ".inst 0x6e41ec9f // bfmmla v31.4s, v4.8h, v1.8h\n" "199:" // Height 6: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -3126,7 +3125,6 @@ void a64_hybrid_fp32bf16fp32_mmla_6x16 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "212:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16.hpp index 905a60265c58aaae123b07b3817653380c4cc260..bfc9c7e8f96556030d0842300a582d8ee2f738bc 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef __aarch64__ + #include "../std_transforms_fixed.hpp" #include "../performance_parameters.hpp" @@ -108,5 +108,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/a55.cpp index b31b80586ce65cdaf1a8bb326e68a5d611b7b9b9..eac0e7167e0fa217e6ecc73d970eda67bc62ab5d 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/a55.cpp @@ -78,329 +78,328 @@ void a64_hybrid_s8qa_dot_4x16_a55 ( flags |= 0x20; } __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x4\n" "bge 91f\n" "cmp %x[M], #0x2\n" "bgt 61f\n" "beq 31f\n" - "mov x16, %x[col_bias]\n" + "mov x15, %x[col_bias]\n" "movi v11.4s, #0x0\n" "movi v15.16b, #0x1\n" "bic %x[flags], %x[flags], #0x80000000\n" - "ldr x15, [%x[args_ptr], %[offsetof_N]]\n" - "mov x14, %x[output_ptr]\n" - "ldr x13, [%x[args_ptr], %[offsetof_B_ptr]]\n" + "ldr x14, [%x[args_ptr], %[offsetof_N]]\n" + "mov x13, %x[output_ptr]\n" + "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n" "2:" // Height 1: Column loop "movi v16.4s, #0x0\n" "movi v17.4s, #0x0\n" "movi v18.4s, #0x0\n" "movi v19.4s, #0x0\n" "3:" // Height 1: setup done - "mov x12, #0x0\n" + "mov x11, #0x0\n" "4:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr w11, [x20, x12, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr w10, [x20, x11, LSL #0x2]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 5f\n" - "ldr x21, [%x[input_ptr], x12, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x10, [x21, #0x0]\n" - "cbnz x12, 6f\n" + "ldr x20, [%x[input_ptr], x11, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x9, [x20, #0x0]\n" + "cbnz x11, 6f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x10, x10, x20\n" + "add x9, x9, x20\n" "b 6f\n" "5:" // Height 1: setup direct input - "mov x10, %x[input_ptr]\n" + "mov x9, %x[input_ptr]\n" "6:" // Height 1: input setup done - "cmp x11, #0x10\n" + "cmp x10, #0x10\n" "blt 11f\n" - "ldr q0, [x10, #0x0]\n" - "cmp x11, #0x20\n" - "ldr q4, [x13, #0x0]\n" - "ldr q5, [x13, #0x10]\n" - "ldr q6, [x13, #0x20]\n" - "ldr q7, [x13, #0x30]\n" - "ldr q8, [x13, #0x40]\n" - "ldr q9, [x13, #0x50]\n" - "ldr q10, [x13, #0x60]\n" + "ldr q0, [x9, #0x0]\n" + "cmp x10, #0x20\n" + "ldr q4, [x12, #0x0]\n" + "ldr q5, [x12, #0x10]\n" + "ldr q6, [x12, #0x20]\n" + "ldr q7, [x12, #0x30]\n" + "ldr q8, [x12, #0x40]\n" + "ldr q9, [x12, #0x50]\n" + "ldr q10, [x12, #0x60]\n" "blt 9f\n" "7:" // Height 1: Multiply loop: Main loop head ".inst 0x4f80e090 // sdot v16.4s, v4.16b, v0.4b[0]\n" - "ldr d4, [x13, #0x70]\n" - "ldr x9, [x13, #0x78]\n" + "ldr d21, [x12, #0x70]\n" + "ldr x20, [x12, #0x78]\n" ".inst 0x4f80e0b1 // sdot v17.4s, v5.16b, v0.4b[0]\n" - "ldr d5, [x13, #0x80]\n" + "ldr d20, [x12, #0x80]\n" ".inst 0x4f80e0d2 // sdot v18.4s, v6.16b, v0.4b[0]\n" - "ldr d6, [x13, #0x90]\n" + "ldr d26, [x12, #0x90]\n" ".inst 0x4f80e0f3 // sdot v19.4s, v7.16b, v0.4b[0]\n" - "ldr d7, [x13, #0xa0]\n" - "mov v4.d[1], x9\n" - "ldr x28, [x13, #0x88]\n" + "ldr d25, [x12, #0xa0]\n" + "mov v21.d[1], x20\n" + "ldr x20, [x12, #0x88]\n" ".inst 0x4fa0e110 // sdot v16.4s, v8.16b, v0.4b[1]\n" - "ldr d8, [x13, #0xb0]\n" + "ldr d24, [x12, #0xb0]\n" ".inst 0x4fa0e131 // sdot v17.4s, v9.16b, v0.4b[1]\n" - "ldr d9, [x13, #0xc0]\n" + "ldr d23, [x12, #0xc0]\n" ".inst 0x4fa0e152 // sdot v18.4s, v10.16b, v0.4b[1]\n" - "ldr d10, [x13, #0xd0]\n" - ".inst 0x4fa0e093 // sdot v19.4s, v4.16b, v0.4b[1]\n" - "ldr d4, [x13, #0xe0]\n" - "mov v5.d[1], x28\n" - "ldr x27, [x13, #0x98]\n" - "mov v6.d[1], x27\n" - "ldr x26, [x13, #0xa8]\n" - "mov v7.d[1], x26\n" - "ldr x25, [x13, #0xb8]\n" - "mov v8.d[1], x25\n" - "ldr x24, [x13, #0xc8]\n" - ".inst 0x4f80e8b0 // sdot v16.4s, v5.16b, v0.4b[2]\n" - "ldr d5, [x13, #0xf0]\n" - ".inst 0x4f80e8d1 // sdot v17.4s, v6.16b, v0.4b[2]\n" - "ldr x20, [x13, #0xd8]\n" - ".inst 0x4f80e8f2 // sdot v18.4s, v7.16b, v0.4b[2]\n" - "ldr x9, [x13, #0xe8]\n" - ".inst 0x4f80e913 // sdot v19.4s, v8.16b, v0.4b[2]\n" - "ldr x28, [x13, #0xf8]\n" - "mov v9.d[1], x24\n" - "mov v10.d[1], x20\n" - "add x10, x10, #0x10\n" - "mov v4.d[1], x9\n" - "add x13, x13, #0x100\n" - "mov v5.d[1], x28\n" - ".inst 0x4fa0e930 // sdot v16.4s, v9.16b, v0.4b[3]\n" - ".inst 0x4fa0e951 // sdot v17.4s, v10.16b, v0.4b[3]\n" - ".inst 0x4fa0e892 // sdot v18.4s, v4.16b, v0.4b[3]\n" - ".inst 0x4fa0e8b3 // sdot v19.4s, v5.16b, v0.4b[3]\n" + "ldr d22, [x12, #0xd0]\n" + ".inst 0x4fa0e2b3 // sdot v19.4s, v21.16b, v0.4b[1]\n" + "ldr d21, [x12, #0xe0]\n" + "mov v20.d[1], x20\n" + "ldr x20, [x12, #0x98]\n" + "mov v26.d[1], x20\n" + "ldr x20, [x12, #0xa8]\n" + "mov v25.d[1], x20\n" + "ldr x20, [x12, #0xb8]\n" + "mov v24.d[1], x20\n" + "ldr x23, [x12, #0xc8]\n" + ".inst 0x4f80ea90 // sdot v16.4s, v20.16b, v0.4b[2]\n" + "ldr d20, [x12, #0xf0]\n" + ".inst 0x4f80eb51 // sdot v17.4s, v26.16b, v0.4b[2]\n" + "ldr x22, [x12, #0xd8]\n" + ".inst 0x4f80eb32 // sdot v18.4s, v25.16b, v0.4b[2]\n" + "ldr x21, [x12, #0xe8]\n" + ".inst 0x4f80eb13 // sdot v19.4s, v24.16b, v0.4b[2]\n" + "ldr x20, [x12, #0xf8]\n" + "mov v23.d[1], x23\n" + "mov v22.d[1], x22\n" + "add x9, x9, #0x10\n" + "mov v21.d[1], x21\n" + "add x12, x12, #0x100\n" + "mov v20.d[1], x20\n" + ".inst 0x4fa0eaf0 // sdot v16.4s, v23.16b, v0.4b[3]\n" + ".inst 0x4fa0ead1 // sdot v17.4s, v22.16b, v0.4b[3]\n" + ".inst 0x4fa0eab2 // sdot v18.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa0ea93 // sdot v19.4s, v20.16b, v0.4b[3]\n" "tbnz %x[flags], #31, 8f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" "8:" // Height 1: Multiply loop: unique 1: skip row sum - "ldr q0, [x10, #0x0]\n" - "sub x11, x11, #0x10\n" - "ldr q4, [x13, #0x0]\n" - "cmp x11, #0x20\n" - "ldr q5, [x13, #0x10]\n" - "ldr q6, [x13, #0x20]\n" - "ldr q7, [x13, #0x30]\n" - "ldr q8, [x13, #0x40]\n" - "ldr q9, [x13, #0x50]\n" - "ldr q10, [x13, #0x60]\n" - "prfm pldl1keep, [x10, #0x80]\n" + "ldr q0, [x9, #0x0]\n" + "sub x10, x10, #0x10\n" + "ldr q4, [x12, #0x0]\n" + "cmp x10, #0x20\n" + "ldr q5, [x12, #0x10]\n" + "ldr q6, [x12, #0x20]\n" + "ldr q7, [x12, #0x30]\n" + "ldr q8, [x12, #0x40]\n" + "ldr q9, [x12, #0x50]\n" + "ldr q10, [x12, #0x60]\n" + "prfm pldl1keep, [x9, #0x80]\n" "bge 7b\n" "9:" // Height 1: Multiply loop: Single iteration only ".inst 0x4f80e090 // sdot v16.4s, v4.16b, v0.4b[0]\n" - "ldr q4, [x13, #0x70]\n" + "ldr q21, [x12, #0x70]\n" ".inst 0x4f80e0b1 // sdot v17.4s, v5.16b, v0.4b[0]\n" - "ldr q5, [x13, #0x80]\n" + "ldr q20, [x12, #0x80]\n" ".inst 0x4f80e0d2 // sdot v18.4s, v6.16b, v0.4b[0]\n" - "ldr q6, [x13, #0x90]\n" + "ldr q26, [x12, #0x90]\n" ".inst 0x4f80e0f3 // sdot v19.4s, v7.16b, v0.4b[0]\n" - "ldr q7, [x13, #0xa0]\n" + "ldr q25, [x12, #0xa0]\n" ".inst 0x4fa0e110 // sdot v16.4s, v8.16b, v0.4b[1]\n" - "ldr q8, [x13, #0xb0]\n" + "ldr q24, [x12, #0xb0]\n" ".inst 0x4fa0e131 // sdot v17.4s, v9.16b, v0.4b[1]\n" - "ldr q9, [x13, #0xc0]\n" + "ldr q23, [x12, #0xc0]\n" ".inst 0x4fa0e152 // sdot v18.4s, v10.16b, v0.4b[1]\n" - "ldr q10, [x13, #0xd0]\n" - ".inst 0x4fa0e093 // sdot v19.4s, v4.16b, v0.4b[1]\n" - "ldr q4, [x13, #0xe0]\n" - ".inst 0x4f80e8b0 // sdot v16.4s, v5.16b, v0.4b[2]\n" - "ldr q5, [x13, #0xf0]\n" - ".inst 0x4f80e8d1 // sdot v17.4s, v6.16b, v0.4b[2]\n" - "sub x11, x11, #0x10\n" - ".inst 0x4f80e8f2 // sdot v18.4s, v7.16b, v0.4b[2]\n" - "add x10, x10, #0x10\n" - ".inst 0x4f80e913 // sdot v19.4s, v8.16b, v0.4b[2]\n" - "add x13, x13, #0x100\n" - ".inst 0x4fa0e930 // sdot v16.4s, v9.16b, v0.4b[3]\n" - ".inst 0x4fa0e951 // sdot v17.4s, v10.16b, v0.4b[3]\n" - ".inst 0x4fa0e892 // sdot v18.4s, v4.16b, v0.4b[3]\n" - ".inst 0x4fa0e8b3 // sdot v19.4s, v5.16b, v0.4b[3]\n" + "ldr q22, [x12, #0xd0]\n" + ".inst 0x4fa0e2b3 // sdot v19.4s, v21.16b, v0.4b[1]\n" + "ldr q21, [x12, #0xe0]\n" + ".inst 0x4f80ea90 // sdot v16.4s, v20.16b, v0.4b[2]\n" + "ldr q20, [x12, #0xf0]\n" + ".inst 0x4f80eb51 // sdot v17.4s, v26.16b, v0.4b[2]\n" + "sub x10, x10, #0x10\n" + ".inst 0x4f80eb32 // sdot v18.4s, v25.16b, v0.4b[2]\n" + "add x9, x9, #0x10\n" + ".inst 0x4f80eb13 // sdot v19.4s, v24.16b, v0.4b[2]\n" + "add x12, x12, #0x100\n" + ".inst 0x4fa0eaf0 // sdot v16.4s, v23.16b, v0.4b[3]\n" + ".inst 0x4fa0ead1 // sdot v17.4s, v22.16b, v0.4b[3]\n" + ".inst 0x4fa0eab2 // sdot v18.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa0ea93 // sdot v19.4s, v20.16b, v0.4b[3]\n" "tbnz %x[flags], #31, 10f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" "10:" // Height 1: Multiply loop: unique 2: skip row sum - "prfm pldl1keep, [x10, #0x80]\n" + "prfm pldl1keep, [x9, #0x80]\n" "11:" // Height 1: Multiply loop: Main loop skip - "cbz x11, 18f\n" - "cmp x11, #0x4\n" + "cbz x10, 18f\n" + "cmp x10, #0x4\n" "blt 14f\n" "12:" // Height 1: Multiply loop: Odd block loop - "ldr s0, [x10], #0x4\n" + "ldr s0, [x9], #0x4\n" "tbnz %x[flags], #31, 13f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" "13:" // Height 1: Multiply loop: unique 3: skip row sum - "ldr q6, [x13, #0x0]\n" - "sub x11, x11, #0x4\n" - "ldr q7, [x13, #0x10]\n" - "cmp x11, #0x4\n" - "ldr q8, [x13, #0x20]\n" - ".inst 0x4f80e0d0 // sdot v16.4s, v6.16b, v0.4b[0]\n" - "ldr q9, [x13, #0x30]\n" - ".inst 0x4f80e0f1 // sdot v17.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f80e112 // sdot v18.4s, v8.16b, v0.4b[0]\n" - "add x13, x13, #0x40\n" - ".inst 0x4f80e133 // sdot v19.4s, v9.16b, v0.4b[0]\n" + "ldr q20, [x12, #0x0]\n" + "sub x10, x10, #0x4\n" + "ldr q22, [x12, #0x10]\n" + "cmp x10, #0x4\n" + "ldr q21, [x12, #0x20]\n" + ".inst 0x4f80e290 // sdot v16.4s, v20.16b, v0.4b[0]\n" + "ldr q20, [x12, #0x30]\n" + ".inst 0x4f80e2d1 // sdot v17.4s, v22.16b, v0.4b[0]\n" + ".inst 0x4f80e2b2 // sdot v18.4s, v21.16b, v0.4b[0]\n" + "add x12, x12, #0x40\n" + ".inst 0x4f80e293 // sdot v19.4s, v20.16b, v0.4b[0]\n" "bge 12b\n" "14:" // Height 1: Multiply loop: Skip odd blocks - "cbz x11, 18f\n" - "tbz x11, #1, 15f\n" - "ldr h0, [x10], #0x2\n" - "tbz x11, #0, 16f\n" - "ld1 { v0.b }[2], [x10]\n" + "cbz x10, 18f\n" + "tbz x10, #1, 15f\n" + "ldr h0, [x9], #0x2\n" + "tbz x10, #0, 16f\n" + "ld1 { v0.b }[2], [x9]\n" "b 16f\n" "15:" // Height 1: Multiply loop: Ragged operand read: partial_1_0 - "ldr b0, [x10, #0x0]\n" + "ldr b0, [x9, #0x0]\n" "16:" // Height 1: Multiply loop: Ragged operand read: Done "tbnz %x[flags], #31, 17f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" "17:" // Height 1: Multiply loop: unique 4: skip row sum - "ldr q10, [x13, #0x0]\n" - ".inst 0x4f80e150 // sdot v16.4s, v10.16b, v0.4b[0]\n" - "ldr q4, [x13, #0x10]\n" - ".inst 0x4f80e091 // sdot v17.4s, v4.16b, v0.4b[0]\n" - "ldr q5, [x13, #0x20]\n" - ".inst 0x4f80e0b2 // sdot v18.4s, v5.16b, v0.4b[0]\n" - "ldr q6, [x13, #0x30]\n" - ".inst 0x4f80e0d3 // sdot v19.4s, v6.16b, v0.4b[0]\n" - "add x13, x13, #0x40\n" + "ldr q20, [x12, #0x0]\n" + ".inst 0x4f80e290 // sdot v16.4s, v20.16b, v0.4b[0]\n" + "ldr q20, [x12, #0x10]\n" + ".inst 0x4f80e291 // sdot v17.4s, v20.16b, v0.4b[0]\n" + "ldr q20, [x12, #0x20]\n" + ".inst 0x4f80e292 // sdot v18.4s, v20.16b, v0.4b[0]\n" + "ldr q20, [x12, #0x30]\n" + ".inst 0x4f80e293 // sdot v19.4s, v20.16b, v0.4b[0]\n" + "add x12, x12, #0x40\n" "18:" // Height 1: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" - "add x12, x12, #0x1\n" - "cmp x12, x20\n" + "add x11, x11, #0x1\n" + "cmp x11, x20\n" "bne 4b\n" - "prfm pstl1keep, [x14, #0x0]\n" + "prfm pstl1keep, [x13, #0x0]\n" "tbnz %x[flags], #31, 19f\n" "addp v11.4s, v11.4s, v11.4s\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1r { v1.4s }, [x23]\n" - "neg v1.4s, v1.4s\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1r { v20.4s }, [x20]\n" + "neg v20.4s, v20.4s\n" "addp v11.4s, v11.4s, v11.4s\n" - "mul v11.4s, v11.4s, v1.4s\n" + "mul v11.4s, v11.4s, v20.4s\n" "19:" // Height 1: skip row sum fixup - "ldr q0, [x16, #0x0]\n" + "ldr q23, [x15, #0x0]\n" "add v16.4s, v16.4s, v11.4s\n" - "ldr q1, [x16, #0x10]\n" + "ldr q22, [x15, #0x10]\n" "add v17.4s, v17.4s, v11.4s\n" - "ldr q2, [x16, #0x20]\n" + "ldr q21, [x15, #0x20]\n" "add v18.4s, v18.4s, v11.4s\n" - "ldr q3, [x16, #0x30]\n" + "ldr q20, [x15, #0x30]\n" "add v19.4s, v19.4s, v11.4s\n" - "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" - "add x23, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x23]\n" + "add v16.4s, v16.4s, v23.4s\n" + "add v17.4s, v17.4s, v22.4s\n" + "add v18.4s, v18.4s, v21.4s\n" + "add v19.4s, v19.4s, v20.4s\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v20.4s }, [x20]\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x23]\n" - "sqrdmulh v16.4s, v16.4s, v4.4s\n" - "sqrdmulh v17.4s, v17.4s, v4.4s\n" - "sqrdmulh v18.4s, v18.4s, v4.4s\n" - "sqrdmulh v19.4s, v19.4s, v4.4s\n" - "add x16, x16, #0x40\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "sqrdmulh v16.4s, v16.4s, v20.4s\n" + "sqrdmulh v17.4s, v17.4s, v20.4s\n" + "sqrdmulh v18.4s, v18.4s, v20.4s\n" + "sqrdmulh v19.4s, v19.4s, v20.4s\n" + "add x15, x15, #0x40\n" "tbz %x[flags], #5, 20f\n" - "and v4.16b, v16.16b, v0.16b\n" - "and v5.16b, v17.16b, v0.16b\n" - "and v6.16b, v18.16b, v0.16b\n" - "and v7.16b, v19.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" + "and v23.16b, v16.16b, v0.16b\n" + "and v22.16b, v17.16b, v0.16b\n" + "and v21.16b, v18.16b, v0.16b\n" + "and v20.16b, v19.16b, v0.16b\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "sshr v21.4s, v21.4s, #0x1f\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v23.4s\n" + "sqadd v17.4s, v17.4s, v22.4s\n" + "sqadd v18.4s, v18.4s, v21.4s\n" + "sqadd v19.4s, v19.4s, v20.4s\n" "20:" // Height 1: no shift correction "srshl v16.4s, v16.4s, v0.4s\n" "srshl v17.4s, v17.4s, v0.4s\n" "srshl v18.4s, v18.4s, v0.4s\n" "srshl v19.4s, v19.4s, v0.4s\n" - "add x23, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x23]\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x23]\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "add x23, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x23]\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v20.4s }, [x20]\n" + "add v16.4s, v16.4s, v20.4s\n" + "add v17.4s, v17.4s, v20.4s\n" + "add v18.4s, v18.4s, v20.4s\n" + "add v19.4s, v19.4s, v20.4s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v20.4s }, [x20]\n" + "smin v16.4s, v16.4s, v20.4s\n" + "smin v17.4s, v17.4s, v20.4s\n" + "smin v18.4s, v18.4s, v20.4s\n" + "smin v19.4s, v19.4s, v20.4s\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v20.4s }, [x20]\n" + "smax v16.4s, v16.4s, v20.4s\n" + "smax v17.4s, v17.4s, v20.4s\n" + "smax v18.4s, v18.4s, v20.4s\n" + "smax v19.4s, v19.4s, v20.4s\n" "uzp1 v16.8h, v16.8h, v17.8h\n" "uzp1 v17.8h, v18.8h, v19.8h\n" - "cmp x15, #0x10\n" + "cmp x14, #0x10\n" "uzp1 v16.16b, v16.16b, v17.16b\n" "bge 29f\n" - "tbz x15, #3, 24f\n" - "str d16, [x14], #0x8\n" - "tbz x15, #2, 22f\n" - "st1 { v16.s }[2], [x14], #0x4\n" - "tbz x15, #1, 21f\n" - "st1 { v16.h }[6], [x14], #0x2\n" - "tbz x15, #0, 28f\n" - "st1 { v16.b }[14], [x14]\n" + "tbz x14, #3, 24f\n" + "str d16, [x13], #0x8\n" + "tbz x14, #2, 22f\n" + "st1 { v16.s }[2], [x13], #0x4\n" + "tbz x14, #1, 21f\n" + "st1 { v16.h }[6], [x13], #0x2\n" + "tbz x14, #0, 28f\n" + "st1 { v16.b }[14], [x13]\n" "b 28f\n" "21:" // Height 1: Partial direct writeback: partial_1_12 - "tbz x15, #0, 28f\n" - "st1 { v16.b }[12], [x14]\n" + "tbz x14, #0, 28f\n" + "st1 { v16.b }[12], [x13]\n" "b 28f\n" "22:" // Height 1: Partial direct writeback: partial_2_8 - "tbz x15, #1, 23f\n" - "st1 { v16.h }[4], [x14], #0x2\n" - "tbz x15, #0, 28f\n" - "st1 { v16.b }[10], [x14]\n" + "tbz x14, #1, 23f\n" + "st1 { v16.h }[4], [x13], #0x2\n" + "tbz x14, #0, 28f\n" + "st1 { v16.b }[10], [x13]\n" "b 28f\n" "23:" // Height 1: Partial direct writeback: partial_1_8 - "tbz x15, #0, 28f\n" - "st1 { v16.b }[8], [x14]\n" + "tbz x14, #0, 28f\n" + "st1 { v16.b }[8], [x13]\n" "b 28f\n" "24:" // Height 1: Partial direct writeback: partial_4_0 - "tbz x15, #2, 26f\n" - "str s16, [x14], #0x4\n" - "tbz x15, #1, 25f\n" - "st1 { v16.h }[2], [x14], #0x2\n" - "tbz x15, #0, 28f\n" - "st1 { v16.b }[6], [x14]\n" + "tbz x14, #2, 26f\n" + "str s16, [x13], #0x4\n" + "tbz x14, #1, 25f\n" + "st1 { v16.h }[2], [x13], #0x2\n" + "tbz x14, #0, 28f\n" + "st1 { v16.b }[6], [x13]\n" "b 28f\n" "25:" // Height 1: Partial direct writeback: partial_1_4 - "tbz x15, #0, 28f\n" - "st1 { v16.b }[4], [x14]\n" + "tbz x14, #0, 28f\n" + "st1 { v16.b }[4], [x13]\n" "b 28f\n" "26:" // Height 1: Partial direct writeback: partial_2_0 - "tbz x15, #1, 27f\n" - "str h16, [x14], #0x2\n" - "tbz x15, #0, 28f\n" - "st1 { v16.b }[2], [x14]\n" + "tbz x14, #1, 27f\n" + "str h16, [x13], #0x2\n" + "tbz x14, #0, 28f\n" + "st1 { v16.b }[2], [x13]\n" "b 28f\n" "27:" // Height 1: Partial direct writeback: partial_1_0 - "str b16, [x14, #0x0]\n" + "str b16, [x13, #0x0]\n" "28:" // Height 1: Partial direct writeback: Done "b 30f\n" "29:" // Height 1: Full writeback - "str q16, [x14, #0x0]\n" - "add x14, x14, #0x10\n" + "str q16, [x13, #0x0]\n" + "add x13, x13, #0x10\n" "30:" // Height 1: Writeback done - "subs x15, x15, #0x10\n" + "subs x14, x14, #0x10\n" "bgt 2b\n" "b 122f\n" "31:" // Height 2 - "mov x16, %x[col_bias]\n" + "mov x15, %x[col_bias]\n" "movi v11.4s, #0x0\n" "movi v12.4s, #0x0\n" "bic %x[flags], %x[flags], #0x80000000\n" "movi v15.16b, #0x1\n" - "ldr x15, [%x[args_ptr], %[offsetof_N]]\n" - "ldr x13, [%x[args_ptr], %[offsetof_B_ptr]]\n" - "mov x14, %x[output_ptr]\n" + "ldr x14, [%x[args_ptr], %[offsetof_N]]\n" + "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n" + "mov x13, %x[output_ptr]\n" "32:" // Height 2: Column loop "movi v16.4s, #0x0\n" "movi v17.4s, #0x0\n" @@ -411,307 +410,307 @@ void a64_hybrid_s8qa_dot_4x16_a55 ( "movi v22.4s, #0x0\n" "movi v23.4s, #0x0\n" "33:" // Height 2: setup done - "mov x12, #0x0\n" + "mov x11, #0x0\n" "34:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr w11, [x20, x12, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr w10, [x20, x11, LSL #0x2]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 35f\n" - "ldr x21, [%x[input_ptr], x12, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x10, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "cbnz x12, 36f\n" + "ldr x20, [%x[input_ptr], x11, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x9, [x20, #0x0]\n" + "ldr x28, [x20, #0x8]\n" + "cbnz x11, 36f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x10, x10, x20\n" - "add x23, x23, x20\n" + "add x9, x9, x20\n" + "add x28, x28, x20\n" "b 36f\n" "35:" // Height 2: setup direct input - "mov x10, %x[input_ptr]\n" - "add x23, x10, x20\n" + "mov x9, %x[input_ptr]\n" + "add x28, x9, x21\n" "36:" // Height 2: input setup done - "cmp x11, #0x10\n" + "cmp x10, #0x10\n" "blt 41f\n" - "ldr q0, [x10, #0x0]\n" - "cmp x11, #0x20\n" - "ldr q1, [x23, #0x0]\n" - "ldr q4, [x13, #0x0]\n" - "ldr q5, [x13, #0x10]\n" - "ldr q6, [x13, #0x20]\n" - "ldr q7, [x13, #0x30]\n" - "ldr q8, [x13, #0x40]\n" - "ldr q9, [x13, #0x50]\n" - "ldr q10, [x13, #0x60]\n" + "ldr q0, [x9, #0x0]\n" + "cmp x10, #0x20\n" + "ldr q1, [x28, #0x0]\n" + "ldr q4, [x12, #0x0]\n" + "ldr q5, [x12, #0x10]\n" + "ldr q6, [x12, #0x20]\n" + "ldr q7, [x12, #0x30]\n" + "ldr q8, [x12, #0x40]\n" + "ldr q9, [x12, #0x50]\n" + "ldr q10, [x12, #0x60]\n" "blt 39f\n" "37:" // Height 2: Multiply loop: Main loop head ".inst 0x4f80e090 // sdot v16.4s, v4.16b, v0.4b[0]\n" - "ldr x9, [x13, #0x78]\n" + "ldr x20, [x12, #0x78]\n" ".inst 0x4f81e094 // sdot v20.4s, v4.16b, v1.4b[0]\n" - "ldr d4, [x13, #0x70]\n" + "ldr d25, [x12, #0x70]\n" ".inst 0x4f80e0b1 // sdot v17.4s, v5.16b, v0.4b[0]\n" - "mov v4.d[1], x9\n" + "mov v25.d[1], x20\n" ".inst 0x4f81e0b5 // sdot v21.4s, v5.16b, v1.4b[0]\n" - "ldr d5, [x13, #0x80]\n" + "ldr d24, [x12, #0x80]\n" ".inst 0x4f80e0d2 // sdot v18.4s, v6.16b, v0.4b[0]\n" - "ldr x28, [x13, #0x88]\n" + "ldr x23, [x12, #0x88]\n" ".inst 0x4f81e0d6 // sdot v22.4s, v6.16b, v1.4b[0]\n" - "ldr d6, [x13, #0x90]\n" + "ldr d30, [x12, #0x90]\n" ".inst 0x4f80e0f3 // sdot v19.4s, v7.16b, v0.4b[0]\n" - "ldr x27, [x13, #0x98]\n" + "ldr x22, [x12, #0x98]\n" ".inst 0x4f81e0f7 // sdot v23.4s, v7.16b, v1.4b[0]\n" - "ldr d7, [x13, #0xa0]\n" - "ldr x26, [x13, #0xa8]\n" + "ldr d29, [x12, #0xa0]\n" + "ldr x21, [x12, #0xa8]\n" ".inst 0x4fa0e110 // sdot v16.4s, v8.16b, v0.4b[1]\n" ".inst 0x4fa1e114 // sdot v20.4s, v8.16b, v1.4b[1]\n" - "ldr d8, [x13, #0xb0]\n" - "ldr x25, [x13, #0xb8]\n" + "ldr d28, [x12, #0xb0]\n" + "ldr x20, [x12, #0xb8]\n" ".inst 0x4fa0e131 // sdot v17.4s, v9.16b, v0.4b[1]\n" ".inst 0x4fa1e135 // sdot v21.4s, v9.16b, v1.4b[1]\n" - "ldr d9, [x13, #0xc0]\n" + "ldr d27, [x12, #0xc0]\n" ".inst 0x4fa0e152 // sdot v18.4s, v10.16b, v0.4b[1]\n" - "mov v5.d[1], x28\n" + "mov v24.d[1], x23\n" ".inst 0x4fa1e156 // sdot v22.4s, v10.16b, v1.4b[1]\n" - "ldr d10, [x13, #0xd0]\n" - ".inst 0x4fa0e093 // sdot v19.4s, v4.16b, v0.4b[1]\n" - "mov v6.d[1], x27\n" - ".inst 0x4fa1e097 // sdot v23.4s, v4.16b, v1.4b[1]\n" - "ldr d4, [x13, #0xe0]\n" - "mov v7.d[1], x26\n" - "ldr x24, [x13, #0xc8]\n" - "mov v8.d[1], x25\n" - "ldr x20, [x13, #0xd8]\n" - "ldr x9, [x13, #0xe8]\n" - ".inst 0x4f80e8b0 // sdot v16.4s, v5.16b, v0.4b[2]\n" - ".inst 0x4f81e8b4 // sdot v20.4s, v5.16b, v1.4b[2]\n" - "ldr d5, [x13, #0xf0]\n" - "ldr x28, [x13, #0xf8]\n" - ".inst 0x4f80e8d1 // sdot v17.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8d5 // sdot v21.4s, v6.16b, v1.4b[2]\n" - "mov v9.d[1], x24\n" - ".inst 0x4f80e8f2 // sdot v18.4s, v7.16b, v0.4b[2]\n" - "mov v10.d[1], x20\n" - ".inst 0x4f81e8f6 // sdot v22.4s, v7.16b, v1.4b[2]\n" - "mov v4.d[1], x9\n" - ".inst 0x4f80e913 // sdot v19.4s, v8.16b, v0.4b[2]\n" - "mov v5.d[1], x28\n" - ".inst 0x4f81e917 // sdot v23.4s, v8.16b, v1.4b[2]\n" - "add x10, x10, #0x10\n" - "add x23, x23, #0x10\n" - "add x13, x13, #0x100\n" - ".inst 0x4fa0e930 // sdot v16.4s, v9.16b, v0.4b[3]\n" - ".inst 0x4fa1e934 // sdot v20.4s, v9.16b, v1.4b[3]\n" - ".inst 0x4fa0e951 // sdot v17.4s, v10.16b, v0.4b[3]\n" - ".inst 0x4fa1e955 // sdot v21.4s, v10.16b, v1.4b[3]\n" - ".inst 0x4fa0e892 // sdot v18.4s, v4.16b, v0.4b[3]\n" - ".inst 0x4fa1e896 // sdot v22.4s, v4.16b, v1.4b[3]\n" - ".inst 0x4fa0e8b3 // sdot v19.4s, v5.16b, v0.4b[3]\n" - ".inst 0x4fa1e8b7 // sdot v23.4s, v5.16b, v1.4b[3]\n" + "ldr d26, [x12, #0xd0]\n" + ".inst 0x4fa0e333 // sdot v19.4s, v25.16b, v0.4b[1]\n" + "mov v30.d[1], x22\n" + ".inst 0x4fa1e337 // sdot v23.4s, v25.16b, v1.4b[1]\n" + "ldr d25, [x12, #0xe0]\n" + "mov v29.d[1], x21\n" + "ldr x23, [x12, #0xc8]\n" + "mov v28.d[1], x20\n" + "ldr x22, [x12, #0xd8]\n" + "ldr x21, [x12, #0xe8]\n" + ".inst 0x4f80eb10 // sdot v16.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f81eb14 // sdot v20.4s, v24.16b, v1.4b[2]\n" + "ldr d24, [x12, #0xf0]\n" + "ldr x20, [x12, #0xf8]\n" + ".inst 0x4f80ebd1 // sdot v17.4s, v30.16b, v0.4b[2]\n" + ".inst 0x4f81ebd5 // sdot v21.4s, v30.16b, v1.4b[2]\n" + "mov v27.d[1], x23\n" + ".inst 0x4f80ebb2 // sdot v18.4s, v29.16b, v0.4b[2]\n" + "mov v26.d[1], x22\n" + ".inst 0x4f81ebb6 // sdot v22.4s, v29.16b, v1.4b[2]\n" + "mov v25.d[1], x21\n" + ".inst 0x4f80eb93 // sdot v19.4s, v28.16b, v0.4b[2]\n" + "mov v24.d[1], x20\n" + ".inst 0x4f81eb97 // sdot v23.4s, v28.16b, v1.4b[2]\n" + "add x9, x9, #0x10\n" + "add x28, x28, #0x10\n" + "add x12, x12, #0x100\n" + ".inst 0x4fa0eb70 // sdot v16.4s, v27.16b, v0.4b[3]\n" + ".inst 0x4fa1eb74 // sdot v20.4s, v27.16b, v1.4b[3]\n" + ".inst 0x4fa0eb51 // sdot v17.4s, v26.16b, v0.4b[3]\n" + ".inst 0x4fa1eb55 // sdot v21.4s, v26.16b, v1.4b[3]\n" + ".inst 0x4fa0eb32 // sdot v18.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa1eb36 // sdot v22.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa0eb13 // sdot v19.4s, v24.16b, v0.4b[3]\n" + ".inst 0x4fa1eb17 // sdot v23.4s, v24.16b, v1.4b[3]\n" "tbnz %x[flags], #31, 38f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n" "38:" // Height 2: Multiply loop: unique 5: skip row sum - "ldr q0, [x10, #0x0]\n" - "sub x11, x11, #0x10\n" - "ldr q1, [x23, #0x0]\n" - "cmp x11, #0x20\n" - "ldr q4, [x13, #0x0]\n" - "ldr q5, [x13, #0x10]\n" - "ldr q6, [x13, #0x20]\n" - "ldr q7, [x13, #0x30]\n" - "ldr q8, [x13, #0x40]\n" - "ldr q9, [x13, #0x50]\n" - "ldr q10, [x13, #0x60]\n" - "prfm pldl1keep, [x10, #0x80]\n" - "prfm pldl1keep, [x23, #0x80]\n" + "ldr q0, [x9, #0x0]\n" + "sub x10, x10, #0x10\n" + "ldr q1, [x28, #0x0]\n" + "cmp x10, #0x20\n" + "ldr q4, [x12, #0x0]\n" + "ldr q5, [x12, #0x10]\n" + "ldr q6, [x12, #0x20]\n" + "ldr q7, [x12, #0x30]\n" + "ldr q8, [x12, #0x40]\n" + "ldr q9, [x12, #0x50]\n" + "ldr q10, [x12, #0x60]\n" + "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x28, #0x80]\n" "bge 37b\n" "39:" // Height 2: Multiply loop: Single iteration only ".inst 0x4f80e090 // sdot v16.4s, v4.16b, v0.4b[0]\n" - "sub x11, x11, #0x10\n" + "sub x10, x10, #0x10\n" ".inst 0x4f81e094 // sdot v20.4s, v4.16b, v1.4b[0]\n" - "ldr q4, [x13, #0x70]\n" + "ldr q25, [x12, #0x70]\n" ".inst 0x4f80e0b1 // sdot v17.4s, v5.16b, v0.4b[0]\n" - "add x10, x10, #0x10\n" + "add x9, x9, #0x10\n" ".inst 0x4f81e0b5 // sdot v21.4s, v5.16b, v1.4b[0]\n" - "ldr q5, [x13, #0x80]\n" + "ldr q24, [x12, #0x80]\n" ".inst 0x4f80e0d2 // sdot v18.4s, v6.16b, v0.4b[0]\n" - "add x23, x23, #0x10\n" + "add x28, x28, #0x10\n" ".inst 0x4f81e0d6 // sdot v22.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x13, #0x90]\n" + "ldr q30, [x12, #0x90]\n" ".inst 0x4f80e0f3 // sdot v19.4s, v7.16b, v0.4b[0]\n" ".inst 0x4f81e0f7 // sdot v23.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x13, #0xa0]\n" + "ldr q29, [x12, #0xa0]\n" ".inst 0x4fa0e110 // sdot v16.4s, v8.16b, v0.4b[1]\n" ".inst 0x4fa1e114 // sdot v20.4s, v8.16b, v1.4b[1]\n" - "ldr q8, [x13, #0xb0]\n" + "ldr q28, [x12, #0xb0]\n" ".inst 0x4fa0e131 // sdot v17.4s, v9.16b, v0.4b[1]\n" ".inst 0x4fa1e135 // sdot v21.4s, v9.16b, v1.4b[1]\n" - "ldr q9, [x13, #0xc0]\n" + "ldr q27, [x12, #0xc0]\n" ".inst 0x4fa0e152 // sdot v18.4s, v10.16b, v0.4b[1]\n" ".inst 0x4fa1e156 // sdot v22.4s, v10.16b, v1.4b[1]\n" - "ldr q10, [x13, #0xd0]\n" - ".inst 0x4fa0e093 // sdot v19.4s, v4.16b, v0.4b[1]\n" - ".inst 0x4fa1e097 // sdot v23.4s, v4.16b, v1.4b[1]\n" - "ldr q4, [x13, #0xe0]\n" - ".inst 0x4f80e8b0 // sdot v16.4s, v5.16b, v0.4b[2]\n" - ".inst 0x4f81e8b4 // sdot v20.4s, v5.16b, v1.4b[2]\n" - "ldr q5, [x13, #0xf0]\n" - ".inst 0x4f80e8d1 // sdot v17.4s, v6.16b, v0.4b[2]\n" - "add x13, x13, #0x100\n" - ".inst 0x4f81e8d5 // sdot v21.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f80e8f2 // sdot v18.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8f6 // sdot v22.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f80e913 // sdot v19.4s, v8.16b, v0.4b[2]\n" - ".inst 0x4f81e917 // sdot v23.4s, v8.16b, v1.4b[2]\n" - ".inst 0x4fa0e930 // sdot v16.4s, v9.16b, v0.4b[3]\n" - ".inst 0x4fa1e934 // sdot v20.4s, v9.16b, v1.4b[3]\n" - ".inst 0x4fa0e951 // sdot v17.4s, v10.16b, v0.4b[3]\n" - ".inst 0x4fa1e955 // sdot v21.4s, v10.16b, v1.4b[3]\n" - ".inst 0x4fa0e892 // sdot v18.4s, v4.16b, v0.4b[3]\n" - ".inst 0x4fa1e896 // sdot v22.4s, v4.16b, v1.4b[3]\n" - ".inst 0x4fa0e8b3 // sdot v19.4s, v5.16b, v0.4b[3]\n" - ".inst 0x4fa1e8b7 // sdot v23.4s, v5.16b, v1.4b[3]\n" + "ldr q26, [x12, #0xd0]\n" + ".inst 0x4fa0e333 // sdot v19.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa1e337 // sdot v23.4s, v25.16b, v1.4b[1]\n" + "ldr q25, [x12, #0xe0]\n" + ".inst 0x4f80eb10 // sdot v16.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f81eb14 // sdot v20.4s, v24.16b, v1.4b[2]\n" + "ldr q24, [x12, #0xf0]\n" + ".inst 0x4f80ebd1 // sdot v17.4s, v30.16b, v0.4b[2]\n" + "add x12, x12, #0x100\n" + ".inst 0x4f81ebd5 // sdot v21.4s, v30.16b, v1.4b[2]\n" + ".inst 0x4f80ebb2 // sdot v18.4s, v29.16b, v0.4b[2]\n" + ".inst 0x4f81ebb6 // sdot v22.4s, v29.16b, v1.4b[2]\n" + ".inst 0x4f80eb93 // sdot v19.4s, v28.16b, v0.4b[2]\n" + ".inst 0x4f81eb97 // sdot v23.4s, v28.16b, v1.4b[2]\n" + ".inst 0x4fa0eb70 // sdot v16.4s, v27.16b, v0.4b[3]\n" + ".inst 0x4fa1eb74 // sdot v20.4s, v27.16b, v1.4b[3]\n" + ".inst 0x4fa0eb51 // sdot v17.4s, v26.16b, v0.4b[3]\n" + ".inst 0x4fa1eb55 // sdot v21.4s, v26.16b, v1.4b[3]\n" + ".inst 0x4fa0eb32 // sdot v18.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa1eb36 // sdot v22.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa0eb13 // sdot v19.4s, v24.16b, v0.4b[3]\n" + ".inst 0x4fa1eb17 // sdot v23.4s, v24.16b, v1.4b[3]\n" "tbnz %x[flags], #31, 40f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n" "40:" // Height 2: Multiply loop: unique 6: skip row sum - "prfm pldl1keep, [x10, #0x80]\n" - "prfm pldl1keep, [x23, #0x80]\n" + "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x28, #0x80]\n" "41:" // Height 2: Multiply loop: Main loop skip - "cbz x11, 48f\n" - "cmp x11, #0x4\n" + "cbz x10, 48f\n" + "cmp x10, #0x4\n" "blt 44f\n" "42:" // Height 2: Multiply loop: Odd block loop - "ldr s0, [x10], #0x4\n" - "ldr s1, [x23], #0x4\n" + "ldr s0, [x9], #0x4\n" + "ldr s1, [x28], #0x4\n" "tbnz %x[flags], #31, 43f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n" "43:" // Height 2: Multiply loop: unique 7: skip row sum - "ldr q6, [x13, #0x0]\n" - "sub x11, x11, #0x4\n" - "ldr q7, [x13, #0x10]\n" - "cmp x11, #0x4\n" - "ldr q8, [x13, #0x20]\n" - ".inst 0x4f80e0d0 // sdot v16.4s, v6.16b, v0.4b[0]\n" - "ldr q9, [x13, #0x30]\n" - ".inst 0x4f81e0d4 // sdot v20.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f80e0f1 // sdot v17.4s, v7.16b, v0.4b[0]\n" - "add x13, x13, #0x40\n" - ".inst 0x4f81e0f5 // sdot v21.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f80e112 // sdot v18.4s, v8.16b, v0.4b[0]\n" - ".inst 0x4f81e116 // sdot v22.4s, v8.16b, v1.4b[0]\n" - ".inst 0x4f80e133 // sdot v19.4s, v9.16b, v0.4b[0]\n" - ".inst 0x4f81e137 // sdot v23.4s, v9.16b, v1.4b[0]\n" + "ldr q27, [x12, #0x0]\n" + "sub x10, x10, #0x4\n" + "ldr q26, [x12, #0x10]\n" + "cmp x10, #0x4\n" + "ldr q25, [x12, #0x20]\n" + ".inst 0x4f80e370 // sdot v16.4s, v27.16b, v0.4b[0]\n" + "ldr q24, [x12, #0x30]\n" + ".inst 0x4f81e374 // sdot v20.4s, v27.16b, v1.4b[0]\n" + ".inst 0x4f80e351 // sdot v17.4s, v26.16b, v0.4b[0]\n" + "add x12, x12, #0x40\n" + ".inst 0x4f81e355 // sdot v21.4s, v26.16b, v1.4b[0]\n" + ".inst 0x4f80e332 // sdot v18.4s, v25.16b, v0.4b[0]\n" + ".inst 0x4f81e336 // sdot v22.4s, v25.16b, v1.4b[0]\n" + ".inst 0x4f80e313 // sdot v19.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f81e317 // sdot v23.4s, v24.16b, v1.4b[0]\n" "bge 42b\n" "44:" // Height 2: Multiply loop: Skip odd blocks - "cbz x11, 48f\n" - "tbz x11, #1, 45f\n" - "ldr h0, [x10], #0x2\n" - "ldr h1, [x23], #0x2\n" - "tbz x11, #0, 46f\n" - "ld1 { v0.b }[2], [x10]\n" - "ld1 { v1.b }[2], [x23]\n" + "cbz x10, 48f\n" + "tbz x10, #1, 45f\n" + "ldr h0, [x9], #0x2\n" + "ldr h1, [x28], #0x2\n" + "tbz x10, #0, 46f\n" + "ld1 { v0.b }[2], [x9]\n" + "ld1 { v1.b }[2], [x28]\n" "b 46f\n" "45:" // Height 2: Multiply loop: Ragged operand read: partial_1_0 - "ldr b0, [x10, #0x0]\n" - "ldr b1, [x23, #0x0]\n" + "ldr b0, [x9, #0x0]\n" + "ldr b1, [x28, #0x0]\n" "46:" // Height 2: Multiply loop: Ragged operand read: Done "tbnz %x[flags], #31, 47f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n" "47:" // Height 2: Multiply loop: unique 8: skip row sum - "ldr q10, [x13, #0x0]\n" - ".inst 0x4f80e150 // sdot v16.4s, v10.16b, v0.4b[0]\n" - "ldr q4, [x13, #0x10]\n" - ".inst 0x4f81e154 // sdot v20.4s, v10.16b, v1.4b[0]\n" - "ldr q5, [x13, #0x20]\n" - ".inst 0x4f80e091 // sdot v17.4s, v4.16b, v0.4b[0]\n" - "ldr q6, [x13, #0x30]\n" - ".inst 0x4f81e095 // sdot v21.4s, v4.16b, v1.4b[0]\n" - ".inst 0x4f80e0b2 // sdot v18.4s, v5.16b, v0.4b[0]\n" - "add x13, x13, #0x40\n" - ".inst 0x4f81e0b6 // sdot v22.4s, v5.16b, v1.4b[0]\n" - ".inst 0x4f80e0d3 // sdot v19.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0d7 // sdot v23.4s, v6.16b, v1.4b[0]\n" + "ldr q24, [x12, #0x0]\n" + ".inst 0x4f80e310 // sdot v16.4s, v24.16b, v0.4b[0]\n" + "ldr q26, [x12, #0x10]\n" + ".inst 0x4f81e314 // sdot v20.4s, v24.16b, v1.4b[0]\n" + "ldr q25, [x12, #0x20]\n" + ".inst 0x4f80e351 // sdot v17.4s, v26.16b, v0.4b[0]\n" + "ldr q24, [x12, #0x30]\n" + ".inst 0x4f81e355 // sdot v21.4s, v26.16b, v1.4b[0]\n" + ".inst 0x4f80e332 // sdot v18.4s, v25.16b, v0.4b[0]\n" + "add x12, x12, #0x40\n" + ".inst 0x4f81e336 // sdot v22.4s, v25.16b, v1.4b[0]\n" + ".inst 0x4f80e313 // sdot v19.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f81e317 // sdot v23.4s, v24.16b, v1.4b[0]\n" "48:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" - "add x12, x12, #0x1\n" - "cmp x12, x20\n" + "add x11, x11, #0x1\n" + "cmp x11, x20\n" "bne 34b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x22, x14, x20\n" - "prfm pstl1keep, [x14, #0x0]\n" - "prfm pstl1keep, [x22, #0x0]\n" + "add x23, x13, x20\n" + "prfm pstl1keep, [x13, #0x0]\n" + "prfm pstl1keep, [x23, #0x0]\n" "tbnz %x[flags], #31, 49f\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v12.4s, v12.4s, v12.4s\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1r { v2.4s }, [x23]\n" - "neg v2.4s, v2.4s\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1r { v24.4s }, [x20]\n" + "neg v24.4s, v24.4s\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v12.4s, v12.4s, v12.4s\n" - "mul v11.4s, v11.4s, v2.4s\n" - "mul v12.4s, v12.4s, v2.4s\n" + "mul v11.4s, v11.4s, v24.4s\n" + "mul v12.4s, v12.4s, v24.4s\n" "49:" // Height 2: skip row sum fixup - "ldr q0, [x16, #0x0]\n" + "ldr q27, [x15, #0x0]\n" "add v16.4s, v16.4s, v11.4s\n" - "ldr q1, [x16, #0x10]\n" + "ldr q26, [x15, #0x10]\n" "add v17.4s, v17.4s, v11.4s\n" - "ldr q2, [x16, #0x20]\n" + "ldr q25, [x15, #0x20]\n" "add v18.4s, v18.4s, v11.4s\n" - "ldr q3, [x16, #0x30]\n" + "ldr q24, [x15, #0x30]\n" "add v19.4s, v19.4s, v11.4s\n" "add v20.4s, v20.4s, v12.4s\n" "add v21.4s, v21.4s, v12.4s\n" "add v22.4s, v22.4s, v12.4s\n" "add v23.4s, v23.4s, v12.4s\n" - "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" - "add v20.4s, v20.4s, v0.4s\n" - "add v21.4s, v21.4s, v1.4s\n" - "add v22.4s, v22.4s, v2.4s\n" - "add v23.4s, v23.4s, v3.4s\n" - "add x23, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x23]\n" + "add v16.4s, v16.4s, v27.4s\n" + "add v17.4s, v17.4s, v26.4s\n" + "add v18.4s, v18.4s, v25.4s\n" + "add v19.4s, v19.4s, v24.4s\n" + "add v20.4s, v20.4s, v27.4s\n" + "add v21.4s, v21.4s, v26.4s\n" + "add v22.4s, v22.4s, v25.4s\n" + "add v23.4s, v23.4s, v24.4s\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v24.4s }, [x20]\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x23]\n" - "sqrdmulh v16.4s, v16.4s, v4.4s\n" - "sqrdmulh v17.4s, v17.4s, v4.4s\n" - "sqrdmulh v18.4s, v18.4s, v4.4s\n" - "sqrdmulh v19.4s, v19.4s, v4.4s\n" - "sqrdmulh v20.4s, v20.4s, v4.4s\n" - "sqrdmulh v21.4s, v21.4s, v4.4s\n" - "sqrdmulh v22.4s, v22.4s, v4.4s\n" - "sqrdmulh v23.4s, v23.4s, v4.4s\n" - "add x16, x16, #0x40\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "sqrdmulh v16.4s, v16.4s, v24.4s\n" + "sqrdmulh v17.4s, v17.4s, v24.4s\n" + "sqrdmulh v18.4s, v18.4s, v24.4s\n" + "sqrdmulh v19.4s, v19.4s, v24.4s\n" + "sqrdmulh v20.4s, v20.4s, v24.4s\n" + "sqrdmulh v21.4s, v21.4s, v24.4s\n" + "sqrdmulh v22.4s, v22.4s, v24.4s\n" + "sqrdmulh v23.4s, v23.4s, v24.4s\n" + "add x15, x15, #0x40\n" "tbz %x[flags], #5, 50f\n" - "and v4.16b, v16.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "and v5.16b, v17.16b, v0.16b\n" - "and v6.16b, v18.16b, v0.16b\n" - "and v7.16b, v19.16b, v0.16b\n" - "and v8.16b, v20.16b, v0.16b\n" - "and v9.16b, v21.16b, v0.16b\n" - "and v10.16b, v22.16b, v0.16b\n" - "and v4.16b, v23.16b, v0.16b\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v9.4s, v9.4s, #0x1f\n" - "sshr v10.4s, v10.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" - "sqadd v20.4s, v20.4s, v8.4s\n" - "sqadd v21.4s, v21.4s, v9.4s\n" - "sqadd v22.4s, v22.4s, v10.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" + "and v24.16b, v16.16b, v0.16b\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v24.4s\n" + "and v30.16b, v17.16b, v0.16b\n" + "and v29.16b, v18.16b, v0.16b\n" + "and v28.16b, v19.16b, v0.16b\n" + "and v27.16b, v20.16b, v0.16b\n" + "and v26.16b, v21.16b, v0.16b\n" + "and v25.16b, v22.16b, v0.16b\n" + "and v24.16b, v23.16b, v0.16b\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sshr v27.4s, v27.4s, #0x1f\n" + "sshr v26.4s, v26.4s, #0x1f\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sqadd v17.4s, v17.4s, v30.4s\n" + "sqadd v18.4s, v18.4s, v29.4s\n" + "sqadd v19.4s, v19.4s, v28.4s\n" + "sqadd v20.4s, v20.4s, v27.4s\n" + "sqadd v21.4s, v21.4s, v26.4s\n" + "sqadd v22.4s, v22.4s, v25.4s\n" + "sqadd v23.4s, v23.4s, v24.4s\n" "50:" // Height 2: no shift correction "srshl v16.4s, v16.4s, v0.4s\n" "srshl v17.4s, v17.4s, v0.4s\n" @@ -721,122 +720,122 @@ void a64_hybrid_s8qa_dot_4x16_a55 ( "srshl v21.4s, v21.4s, v0.4s\n" "srshl v22.4s, v22.4s, v0.4s\n" "srshl v23.4s, v23.4s, v0.4s\n" - "add x23, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x23]\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v23.4s, v23.4s, v4.4s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x23]\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "add x23, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x23]\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v24.4s }, [x20]\n" + "add v16.4s, v16.4s, v24.4s\n" + "add v17.4s, v17.4s, v24.4s\n" + "add v18.4s, v18.4s, v24.4s\n" + "add v19.4s, v19.4s, v24.4s\n" + "add v20.4s, v20.4s, v24.4s\n" + "add v21.4s, v21.4s, v24.4s\n" + "add v22.4s, v22.4s, v24.4s\n" + "add v23.4s, v23.4s, v24.4s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v24.4s }, [x20]\n" + "smin v16.4s, v16.4s, v24.4s\n" + "smin v17.4s, v17.4s, v24.4s\n" + "smin v18.4s, v18.4s, v24.4s\n" + "smin v19.4s, v19.4s, v24.4s\n" + "smin v20.4s, v20.4s, v24.4s\n" + "smin v21.4s, v21.4s, v24.4s\n" + "smin v22.4s, v22.4s, v24.4s\n" + "smin v23.4s, v23.4s, v24.4s\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v24.4s }, [x20]\n" + "smax v16.4s, v16.4s, v24.4s\n" + "smax v17.4s, v17.4s, v24.4s\n" + "smax v18.4s, v18.4s, v24.4s\n" + "smax v19.4s, v19.4s, v24.4s\n" + "smax v20.4s, v20.4s, v24.4s\n" + "smax v21.4s, v21.4s, v24.4s\n" + "smax v22.4s, v22.4s, v24.4s\n" + "smax v23.4s, v23.4s, v24.4s\n" "uzp1 v16.8h, v16.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v18.8h, v18.8h, v19.8h\n" "uzp1 v20.8h, v20.8h, v21.8h\n" - "uzp1 v21.8h, v22.8h, v23.8h\n" - "cmp x15, #0x10\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" - "uzp1 v20.16b, v20.16b, v21.16b\n" + "uzp1 v17.8h, v22.8h, v23.8h\n" + "cmp x14, #0x10\n" + "uzp1 v16.16b, v16.16b, v18.16b\n" + "uzp1 v20.16b, v20.16b, v17.16b\n" "bge 59f\n" - "tbz x15, #3, 54f\n" - "str d16, [x14], #0x8\n" - "str d20, [x22], #0x8\n" - "tbz x15, #2, 52f\n" - "st1 { v16.s }[2], [x14], #0x4\n" - "st1 { v20.s }[2], [x22], #0x4\n" - "tbz x15, #1, 51f\n" - "st1 { v16.h }[6], [x14], #0x2\n" - "st1 { v20.h }[6], [x22], #0x2\n" - "tbz x15, #0, 58f\n" - "st1 { v16.b }[14], [x14]\n" - "st1 { v20.b }[14], [x22]\n" + "tbz x14, #3, 54f\n" + "str d16, [x13], #0x8\n" + "str d20, [x23], #0x8\n" + "tbz x14, #2, 52f\n" + "st1 { v16.s }[2], [x13], #0x4\n" + "st1 { v20.s }[2], [x23], #0x4\n" + "tbz x14, #1, 51f\n" + "st1 { v16.h }[6], [x13], #0x2\n" + "st1 { v20.h }[6], [x23], #0x2\n" + "tbz x14, #0, 58f\n" + "st1 { v16.b }[14], [x13]\n" + "st1 { v20.b }[14], [x23]\n" "b 58f\n" "51:" // Height 2: Partial direct writeback: partial_1_12 - "tbz x15, #0, 58f\n" - "st1 { v16.b }[12], [x14]\n" - "st1 { v20.b }[12], [x22]\n" + "tbz x14, #0, 58f\n" + "st1 { v16.b }[12], [x13]\n" + "st1 { v20.b }[12], [x23]\n" "b 58f\n" "52:" // Height 2: Partial direct writeback: partial_2_8 - "tbz x15, #1, 53f\n" - "st1 { v16.h }[4], [x14], #0x2\n" - "st1 { v20.h }[4], [x22], #0x2\n" - "tbz x15, #0, 58f\n" - "st1 { v16.b }[10], [x14]\n" - "st1 { v20.b }[10], [x22]\n" + "tbz x14, #1, 53f\n" + "st1 { v16.h }[4], [x13], #0x2\n" + "st1 { v20.h }[4], [x23], #0x2\n" + "tbz x14, #0, 58f\n" + "st1 { v16.b }[10], [x13]\n" + "st1 { v20.b }[10], [x23]\n" "b 58f\n" "53:" // Height 2: Partial direct writeback: partial_1_8 - "tbz x15, #0, 58f\n" - "st1 { v16.b }[8], [x14]\n" - "st1 { v20.b }[8], [x22]\n" + "tbz x14, #0, 58f\n" + "st1 { v16.b }[8], [x13]\n" + "st1 { v20.b }[8], [x23]\n" "b 58f\n" "54:" // Height 2: Partial direct writeback: partial_4_0 - "tbz x15, #2, 56f\n" - "str s16, [x14], #0x4\n" - "str s20, [x22], #0x4\n" - "tbz x15, #1, 55f\n" - "st1 { v16.h }[2], [x14], #0x2\n" - "st1 { v20.h }[2], [x22], #0x2\n" - "tbz x15, #0, 58f\n" - "st1 { v16.b }[6], [x14]\n" - "st1 { v20.b }[6], [x22]\n" + "tbz x14, #2, 56f\n" + "str s16, [x13], #0x4\n" + "str s20, [x23], #0x4\n" + "tbz x14, #1, 55f\n" + "st1 { v16.h }[2], [x13], #0x2\n" + "st1 { v20.h }[2], [x23], #0x2\n" + "tbz x14, #0, 58f\n" + "st1 { v16.b }[6], [x13]\n" + "st1 { v20.b }[6], [x23]\n" "b 58f\n" "55:" // Height 2: Partial direct writeback: partial_1_4 - "tbz x15, #0, 58f\n" - "st1 { v16.b }[4], [x14]\n" - "st1 { v20.b }[4], [x22]\n" + "tbz x14, #0, 58f\n" + "st1 { v16.b }[4], [x13]\n" + "st1 { v20.b }[4], [x23]\n" "b 58f\n" "56:" // Height 2: Partial direct writeback: partial_2_0 - "tbz x15, #1, 57f\n" - "str h16, [x14], #0x2\n" - "str h20, [x22], #0x2\n" - "tbz x15, #0, 58f\n" - "st1 { v16.b }[2], [x14]\n" - "st1 { v20.b }[2], [x22]\n" + "tbz x14, #1, 57f\n" + "str h16, [x13], #0x2\n" + "str h20, [x23], #0x2\n" + "tbz x14, #0, 58f\n" + "st1 { v16.b }[2], [x13]\n" + "st1 { v20.b }[2], [x23]\n" "b 58f\n" "57:" // Height 2: Partial direct writeback: partial_1_0 - "str b16, [x14, #0x0]\n" - "str b20, [x22, #0x0]\n" + "str b16, [x13, #0x0]\n" + "str b20, [x23, #0x0]\n" "58:" // Height 2: Partial direct writeback: Done "b 60f\n" "59:" // Height 2: Full writeback - "str q16, [x14, #0x0]\n" - "add x14, x14, #0x10\n" - "str q20, [x22, #0x0]\n" + "str q16, [x13, #0x0]\n" + "add x13, x13, #0x10\n" + "str q20, [x23, #0x0]\n" "60:" // Height 2: Writeback done - "subs x15, x15, #0x10\n" + "subs x14, x14, #0x10\n" "bgt 32b\n" "b 122f\n" "61:" // Height 3 - "mov x16, %x[col_bias]\n" + "mov x15, %x[col_bias]\n" "movi v11.4s, #0x0\n" "movi v12.4s, #0x0\n" "bic %x[flags], %x[flags], #0x80000000\n" "movi v13.4s, #0x0\n" - "ldr x15, [%x[args_ptr], %[offsetof_N]]\n" + "ldr x14, [%x[args_ptr], %[offsetof_N]]\n" "movi v15.16b, #0x1\n" - "ldr x13, [%x[args_ptr], %[offsetof_B_ptr]]\n" - "mov x14, %x[output_ptr]\n" + "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n" + "mov x13, %x[output_ptr]\n" "62:" // Height 3: Column loop "movi v16.4s, #0x0\n" "movi v17.4s, #0x0\n" @@ -851,317 +850,317 @@ void a64_hybrid_s8qa_dot_4x16_a55 ( "movi v26.4s, #0x0\n" "movi v27.4s, #0x0\n" "63:" // Height 3: setup done - "mov x12, #0x0\n" + "mov x11, #0x0\n" "64:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr w11, [x20, x12, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr w10, [x20, x11, LSL #0x2]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 65f\n" - "ldr x21, [%x[input_ptr], x12, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x10, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" - "cbnz x12, 66f\n" + "ldr x20, [%x[input_ptr], x11, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x9, [x20, #0x0]\n" + "ldr x28, [x20, #0x8]\n" + "ldr x27, [x20, #0x10]\n" + "cbnz x11, 66f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x10, x10, x20\n" - "add x23, x23, x20\n" - "add x22, x22, x20\n" + "add x9, x9, x20\n" + "add x28, x28, x20\n" + "add x27, x27, x20\n" "b 66f\n" "65:" // Height 3: setup direct input - "mov x10, %x[input_ptr]\n" - "add x23, x10, x20\n" - "add x22, x23, x20\n" + "mov x9, %x[input_ptr]\n" + "add x28, x9, x21\n" + "add x27, x28, x21\n" "66:" // Height 3: input setup done - "cmp x11, #0x10\n" + "cmp x10, #0x10\n" "blt 71f\n" - "ldr q0, [x10, #0x0]\n" - "cmp x11, #0x20\n" - "ldr q1, [x23, #0x0]\n" - "ldr q2, [x22, #0x0]\n" - "ldr q4, [x13, #0x0]\n" - "ldr q5, [x13, #0x10]\n" - "ldr q6, [x13, #0x20]\n" - "ldr q7, [x13, #0x30]\n" - "ldr q8, [x13, #0x40]\n" - "ldr q9, [x13, #0x50]\n" - "ldr q10, [x13, #0x60]\n" + "ldr q0, [x9, #0x0]\n" + "cmp x10, #0x20\n" + "ldr q1, [x28, #0x0]\n" + "ldr q2, [x27, #0x0]\n" + "ldr q4, [x12, #0x0]\n" + "ldr q5, [x12, #0x10]\n" + "ldr q6, [x12, #0x20]\n" + "ldr q7, [x12, #0x30]\n" + "ldr q8, [x12, #0x40]\n" + "ldr q9, [x12, #0x50]\n" + "ldr q10, [x12, #0x60]\n" "blt 69f\n" "67:" // Height 3: Multiply loop: Main loop head ".inst 0x4f80e090 // sdot v16.4s, v4.16b, v0.4b[0]\n" - "ldr x9, [x13, #0x78]\n" + "ldr x20, [x12, #0x78]\n" ".inst 0x4f81e094 // sdot v20.4s, v4.16b, v1.4b[0]\n" - "ldr x28, [x13, #0x88]\n" + "ldr x23, [x12, #0x88]\n" ".inst 0x4f82e098 // sdot v24.4s, v4.16b, v2.4b[0]\n" - "ldr d4, [x13, #0x70]\n" + "ldr d29, [x12, #0x70]\n" ".inst 0x4f80e0b1 // sdot v17.4s, v5.16b, v0.4b[0]\n" - "mov v4.d[1], x9\n" + "mov v29.d[1], x20\n" ".inst 0x4f81e0b5 // sdot v21.4s, v5.16b, v1.4b[0]\n" - "ldr x27, [x13, #0x98]\n" + "ldr x22, [x12, #0x98]\n" ".inst 0x4f82e0b9 // sdot v25.4s, v5.16b, v2.4b[0]\n" - "ldr d5, [x13, #0x80]\n" + "ldr d28, [x12, #0x80]\n" ".inst 0x4f80e0d2 // sdot v18.4s, v6.16b, v0.4b[0]\n" - "ldr x26, [x13, #0xa8]\n" + "ldr x21, [x12, #0xa8]\n" ".inst 0x4f81e0d6 // sdot v22.4s, v6.16b, v1.4b[0]\n" - "ldr x25, [x13, #0xb8]\n" + "ldr x20, [x12, #0xb8]\n" ".inst 0x4f82e0da // sdot v26.4s, v6.16b, v2.4b[0]\n" - "ldr d6, [x13, #0x90]\n" + "ldr d5, [x12, #0x90]\n" ".inst 0x4f80e0f3 // sdot v19.4s, v7.16b, v0.4b[0]\n" - "mov v5.d[1], x28\n" + "mov v28.d[1], x23\n" ".inst 0x4f81e0f7 // sdot v23.4s, v7.16b, v1.4b[0]\n" - "mov v6.d[1], x27\n" + "mov v5.d[1], x22\n" ".inst 0x4f82e0fb // sdot v27.4s, v7.16b, v2.4b[0]\n" - "ldr d7, [x13, #0xa0]\n" + "ldr d4, [x12, #0xa0]\n" ".inst 0x4fa0e110 // sdot v16.4s, v8.16b, v0.4b[1]\n" - "mov v7.d[1], x26\n" + "mov v4.d[1], x21\n" ".inst 0x4fa1e114 // sdot v20.4s, v8.16b, v1.4b[1]\n" - "ldr x24, [x13, #0xc8]\n" + "ldr x23, [x12, #0xc8]\n" ".inst 0x4fa2e118 // sdot v24.4s, v8.16b, v2.4b[1]\n" - "ldr d8, [x13, #0xb0]\n" + "ldr d3, [x12, #0xb0]\n" ".inst 0x4fa0e131 // sdot v17.4s, v9.16b, v0.4b[1]\n" - "mov v8.d[1], x25\n" + "mov v3.d[1], x20\n" ".inst 0x4fa1e135 // sdot v21.4s, v9.16b, v1.4b[1]\n" - "ldr x20, [x13, #0xd8]\n" + "ldr x22, [x12, #0xd8]\n" ".inst 0x4fa2e139 // sdot v25.4s, v9.16b, v2.4b[1]\n" - "ldr d9, [x13, #0xc0]\n" + "ldr d31, [x12, #0xc0]\n" ".inst 0x4fa0e152 // sdot v18.4s, v10.16b, v0.4b[1]\n" - "ldr x9, [x13, #0xe8]\n" + "ldr x21, [x12, #0xe8]\n" ".inst 0x4fa1e156 // sdot v22.4s, v10.16b, v1.4b[1]\n" - "ldr x28, [x13, #0xf8]\n" + "ldr x20, [x12, #0xf8]\n" ".inst 0x4fa2e15a // sdot v26.4s, v10.16b, v2.4b[1]\n" - "ldr d10, [x13, #0xd0]\n" - ".inst 0x4fa0e093 // sdot v19.4s, v4.16b, v0.4b[1]\n" - "mov v9.d[1], x24\n" - ".inst 0x4fa1e097 // sdot v23.4s, v4.16b, v1.4b[1]\n" - "mov v10.d[1], x20\n" - ".inst 0x4fa2e09b // sdot v27.4s, v4.16b, v2.4b[1]\n" - "ldr d4, [x13, #0xe0]\n" - ".inst 0x4f80e8b0 // sdot v16.4s, v5.16b, v0.4b[2]\n" - "mov v4.d[1], x9\n" - ".inst 0x4f81e8b4 // sdot v20.4s, v5.16b, v1.4b[2]\n" - "add x10, x10, #0x10\n" - ".inst 0x4f82e8b8 // sdot v24.4s, v5.16b, v2.4b[2]\n" - "ldr d5, [x13, #0xf0]\n" - ".inst 0x4f80e8d1 // sdot v17.4s, v6.16b, v0.4b[2]\n" - "mov v5.d[1], x28\n" - ".inst 0x4f81e8d5 // sdot v21.4s, v6.16b, v1.4b[2]\n" - "add x23, x23, #0x10\n" - ".inst 0x4f82e8d9 // sdot v25.4s, v6.16b, v2.4b[2]\n" - "add x22, x22, #0x10\n" - ".inst 0x4f80e8f2 // sdot v18.4s, v7.16b, v0.4b[2]\n" - "add x13, x13, #0x100\n" - ".inst 0x4f81e8f6 // sdot v22.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8fa // sdot v26.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f80e913 // sdot v19.4s, v8.16b, v0.4b[2]\n" - ".inst 0x4f81e917 // sdot v23.4s, v8.16b, v1.4b[2]\n" - ".inst 0x4f82e91b // sdot v27.4s, v8.16b, v2.4b[2]\n" - ".inst 0x4fa0e930 // sdot v16.4s, v9.16b, v0.4b[3]\n" - ".inst 0x4fa1e934 // sdot v20.4s, v9.16b, v1.4b[3]\n" - ".inst 0x4fa2e938 // sdot v24.4s, v9.16b, v2.4b[3]\n" - ".inst 0x4fa0e951 // sdot v17.4s, v10.16b, v0.4b[3]\n" - ".inst 0x4fa1e955 // sdot v21.4s, v10.16b, v1.4b[3]\n" - ".inst 0x4fa2e959 // sdot v25.4s, v10.16b, v2.4b[3]\n" - ".inst 0x4fa0e892 // sdot v18.4s, v4.16b, v0.4b[3]\n" - ".inst 0x4fa1e896 // sdot v22.4s, v4.16b, v1.4b[3]\n" - ".inst 0x4fa2e89a // sdot v26.4s, v4.16b, v2.4b[3]\n" - ".inst 0x4fa0e8b3 // sdot v19.4s, v5.16b, v0.4b[3]\n" - ".inst 0x4fa1e8b7 // sdot v23.4s, v5.16b, v1.4b[3]\n" - ".inst 0x4fa2e8bb // sdot v27.4s, v5.16b, v2.4b[3]\n" + "ldr d30, [x12, #0xd0]\n" + ".inst 0x4fa0e3b3 // sdot v19.4s, v29.16b, v0.4b[1]\n" + "mov v31.d[1], x23\n" + ".inst 0x4fa1e3b7 // sdot v23.4s, v29.16b, v1.4b[1]\n" + "mov v30.d[1], x22\n" + ".inst 0x4fa2e3bb // sdot v27.4s, v29.16b, v2.4b[1]\n" + "ldr d29, [x12, #0xe0]\n" + ".inst 0x4f80eb90 // sdot v16.4s, v28.16b, v0.4b[2]\n" + "mov v29.d[1], x21\n" + ".inst 0x4f81eb94 // sdot v20.4s, v28.16b, v1.4b[2]\n" + "add x9, x9, #0x10\n" + ".inst 0x4f82eb98 // sdot v24.4s, v28.16b, v2.4b[2]\n" + "ldr d28, [x12, #0xf0]\n" + ".inst 0x4f80e8b1 // sdot v17.4s, v5.16b, v0.4b[2]\n" + "mov v28.d[1], x20\n" + ".inst 0x4f81e8b5 // sdot v21.4s, v5.16b, v1.4b[2]\n" + "add x28, x28, #0x10\n" + ".inst 0x4f82e8b9 // sdot v25.4s, v5.16b, v2.4b[2]\n" + "add x27, x27, #0x10\n" + ".inst 0x4f80e892 // sdot v18.4s, v4.16b, v0.4b[2]\n" + "add x12, x12, #0x100\n" + ".inst 0x4f81e896 // sdot v22.4s, v4.16b, v1.4b[2]\n" + ".inst 0x4f82e89a // sdot v26.4s, v4.16b, v2.4b[2]\n" + ".inst 0x4f80e873 // sdot v19.4s, v3.16b, v0.4b[2]\n" + ".inst 0x4f81e877 // sdot v23.4s, v3.16b, v1.4b[2]\n" + ".inst 0x4f82e87b // sdot v27.4s, v3.16b, v2.4b[2]\n" + ".inst 0x4fa0ebf0 // sdot v16.4s, v31.16b, v0.4b[3]\n" + ".inst 0x4fa1ebf4 // sdot v20.4s, v31.16b, v1.4b[3]\n" + ".inst 0x4fa2ebf8 // sdot v24.4s, v31.16b, v2.4b[3]\n" + ".inst 0x4fa0ebd1 // sdot v17.4s, v30.16b, v0.4b[3]\n" + ".inst 0x4fa1ebd5 // sdot v21.4s, v30.16b, v1.4b[3]\n" + ".inst 0x4fa2ebd9 // sdot v25.4s, v30.16b, v2.4b[3]\n" + ".inst 0x4fa0ebb2 // sdot v18.4s, v29.16b, v0.4b[3]\n" + ".inst 0x4fa1ebb6 // sdot v22.4s, v29.16b, v1.4b[3]\n" + ".inst 0x4fa2ebba // sdot v26.4s, v29.16b, v2.4b[3]\n" + ".inst 0x4fa0eb93 // sdot v19.4s, v28.16b, v0.4b[3]\n" + ".inst 0x4fa1eb97 // sdot v23.4s, v28.16b, v1.4b[3]\n" + ".inst 0x4fa2eb9b // sdot v27.4s, v28.16b, v2.4b[3]\n" "tbnz %x[flags], #31, 68f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n" ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n" "68:" // Height 3: Multiply loop: unique 9: skip row sum - "ldr q0, [x10, #0x0]\n" - "sub x11, x11, #0x10\n" - "ldr q1, [x23, #0x0]\n" - "cmp x11, #0x20\n" - "ldr q2, [x22, #0x0]\n" - "ldr q4, [x13, #0x0]\n" - "ldr q5, [x13, #0x10]\n" - "ldr q6, [x13, #0x20]\n" - "ldr q7, [x13, #0x30]\n" - "ldr q8, [x13, #0x40]\n" - "ldr q9, [x13, #0x50]\n" - "ldr q10, [x13, #0x60]\n" - "prfm pldl1keep, [x10, #0x80]\n" - "prfm pldl1keep, [x23, #0x80]\n" - "prfm pldl1keep, [x22, #0x80]\n" + "ldr q0, [x9, #0x0]\n" + "sub x10, x10, #0x10\n" + "ldr q1, [x28, #0x0]\n" + "cmp x10, #0x20\n" + "ldr q2, [x27, #0x0]\n" + "ldr q4, [x12, #0x0]\n" + "ldr q5, [x12, #0x10]\n" + "ldr q6, [x12, #0x20]\n" + "ldr q7, [x12, #0x30]\n" + "ldr q8, [x12, #0x40]\n" + "ldr q9, [x12, #0x50]\n" + "ldr q10, [x12, #0x60]\n" + "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x28, #0x80]\n" + "prfm pldl1keep, [x27, #0x80]\n" "bge 67b\n" "69:" // Height 3: Multiply loop: Single iteration only ".inst 0x4f80e090 // sdot v16.4s, v4.16b, v0.4b[0]\n" - "sub x11, x11, #0x10\n" + "sub x10, x10, #0x10\n" ".inst 0x4f81e094 // sdot v20.4s, v4.16b, v1.4b[0]\n" - "add x10, x10, #0x10\n" + "add x9, x9, #0x10\n" ".inst 0x4f82e098 // sdot v24.4s, v4.16b, v2.4b[0]\n" - "ldr q4, [x13, #0x70]\n" + "ldr q29, [x12, #0x70]\n" ".inst 0x4f80e0b1 // sdot v17.4s, v5.16b, v0.4b[0]\n" - "add x23, x23, #0x10\n" + "add x28, x28, #0x10\n" ".inst 0x4f81e0b5 // sdot v21.4s, v5.16b, v1.4b[0]\n" - "add x22, x22, #0x10\n" + "add x27, x27, #0x10\n" ".inst 0x4f82e0b9 // sdot v25.4s, v5.16b, v2.4b[0]\n" - "ldr q5, [x13, #0x80]\n" + "ldr q28, [x12, #0x80]\n" ".inst 0x4f80e0d2 // sdot v18.4s, v6.16b, v0.4b[0]\n" ".inst 0x4f81e0d6 // sdot v22.4s, v6.16b, v1.4b[0]\n" ".inst 0x4f82e0da // sdot v26.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x13, #0x90]\n" + "ldr q5, [x12, #0x90]\n" ".inst 0x4f80e0f3 // sdot v19.4s, v7.16b, v0.4b[0]\n" ".inst 0x4f81e0f7 // sdot v23.4s, v7.16b, v1.4b[0]\n" ".inst 0x4f82e0fb // sdot v27.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x13, #0xa0]\n" + "ldr q4, [x12, #0xa0]\n" ".inst 0x4fa0e110 // sdot v16.4s, v8.16b, v0.4b[1]\n" ".inst 0x4fa1e114 // sdot v20.4s, v8.16b, v1.4b[1]\n" ".inst 0x4fa2e118 // sdot v24.4s, v8.16b, v2.4b[1]\n" - "ldr q8, [x13, #0xb0]\n" + "ldr q3, [x12, #0xb0]\n" ".inst 0x4fa0e131 // sdot v17.4s, v9.16b, v0.4b[1]\n" ".inst 0x4fa1e135 // sdot v21.4s, v9.16b, v1.4b[1]\n" ".inst 0x4fa2e139 // sdot v25.4s, v9.16b, v2.4b[1]\n" - "ldr q9, [x13, #0xc0]\n" + "ldr q31, [x12, #0xc0]\n" ".inst 0x4fa0e152 // sdot v18.4s, v10.16b, v0.4b[1]\n" ".inst 0x4fa1e156 // sdot v22.4s, v10.16b, v1.4b[1]\n" ".inst 0x4fa2e15a // sdot v26.4s, v10.16b, v2.4b[1]\n" - "ldr q10, [x13, #0xd0]\n" - ".inst 0x4fa0e093 // sdot v19.4s, v4.16b, v0.4b[1]\n" - ".inst 0x4fa1e097 // sdot v23.4s, v4.16b, v1.4b[1]\n" - ".inst 0x4fa2e09b // sdot v27.4s, v4.16b, v2.4b[1]\n" - "ldr q4, [x13, #0xe0]\n" - ".inst 0x4f80e8b0 // sdot v16.4s, v5.16b, v0.4b[2]\n" - ".inst 0x4f81e8b4 // sdot v20.4s, v5.16b, v1.4b[2]\n" - ".inst 0x4f82e8b8 // sdot v24.4s, v5.16b, v2.4b[2]\n" - "ldr q5, [x13, #0xf0]\n" - ".inst 0x4f80e8d1 // sdot v17.4s, v6.16b, v0.4b[2]\n" - "add x13, x13, #0x100\n" - ".inst 0x4f81e8d5 // sdot v21.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d9 // sdot v25.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f80e8f2 // sdot v18.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8f6 // sdot v22.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8fa // sdot v26.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f80e913 // sdot v19.4s, v8.16b, v0.4b[2]\n" - ".inst 0x4f81e917 // sdot v23.4s, v8.16b, v1.4b[2]\n" - ".inst 0x4f82e91b // sdot v27.4s, v8.16b, v2.4b[2]\n" - ".inst 0x4fa0e930 // sdot v16.4s, v9.16b, v0.4b[3]\n" - ".inst 0x4fa1e934 // sdot v20.4s, v9.16b, v1.4b[3]\n" - ".inst 0x4fa2e938 // sdot v24.4s, v9.16b, v2.4b[3]\n" - ".inst 0x4fa0e951 // sdot v17.4s, v10.16b, v0.4b[3]\n" - ".inst 0x4fa1e955 // sdot v21.4s, v10.16b, v1.4b[3]\n" - ".inst 0x4fa2e959 // sdot v25.4s, v10.16b, v2.4b[3]\n" - ".inst 0x4fa0e892 // sdot v18.4s, v4.16b, v0.4b[3]\n" - ".inst 0x4fa1e896 // sdot v22.4s, v4.16b, v1.4b[3]\n" - ".inst 0x4fa2e89a // sdot v26.4s, v4.16b, v2.4b[3]\n" - ".inst 0x4fa0e8b3 // sdot v19.4s, v5.16b, v0.4b[3]\n" - ".inst 0x4fa1e8b7 // sdot v23.4s, v5.16b, v1.4b[3]\n" - ".inst 0x4fa2e8bb // sdot v27.4s, v5.16b, v2.4b[3]\n" + "ldr q30, [x12, #0xd0]\n" + ".inst 0x4fa0e3b3 // sdot v19.4s, v29.16b, v0.4b[1]\n" + ".inst 0x4fa1e3b7 // sdot v23.4s, v29.16b, v1.4b[1]\n" + ".inst 0x4fa2e3bb // sdot v27.4s, v29.16b, v2.4b[1]\n" + "ldr q29, [x12, #0xe0]\n" + ".inst 0x4f80eb90 // sdot v16.4s, v28.16b, v0.4b[2]\n" + ".inst 0x4f81eb94 // sdot v20.4s, v28.16b, v1.4b[2]\n" + ".inst 0x4f82eb98 // sdot v24.4s, v28.16b, v2.4b[2]\n" + "ldr q28, [x12, #0xf0]\n" + ".inst 0x4f80e8b1 // sdot v17.4s, v5.16b, v0.4b[2]\n" + "add x12, x12, #0x100\n" + ".inst 0x4f81e8b5 // sdot v21.4s, v5.16b, v1.4b[2]\n" + ".inst 0x4f82e8b9 // sdot v25.4s, v5.16b, v2.4b[2]\n" + ".inst 0x4f80e892 // sdot v18.4s, v4.16b, v0.4b[2]\n" + ".inst 0x4f81e896 // sdot v22.4s, v4.16b, v1.4b[2]\n" + ".inst 0x4f82e89a // sdot v26.4s, v4.16b, v2.4b[2]\n" + ".inst 0x4f80e873 // sdot v19.4s, v3.16b, v0.4b[2]\n" + ".inst 0x4f81e877 // sdot v23.4s, v3.16b, v1.4b[2]\n" + ".inst 0x4f82e87b // sdot v27.4s, v3.16b, v2.4b[2]\n" + ".inst 0x4fa0ebf0 // sdot v16.4s, v31.16b, v0.4b[3]\n" + ".inst 0x4fa1ebf4 // sdot v20.4s, v31.16b, v1.4b[3]\n" + ".inst 0x4fa2ebf8 // sdot v24.4s, v31.16b, v2.4b[3]\n" + ".inst 0x4fa0ebd1 // sdot v17.4s, v30.16b, v0.4b[3]\n" + ".inst 0x4fa1ebd5 // sdot v21.4s, v30.16b, v1.4b[3]\n" + ".inst 0x4fa2ebd9 // sdot v25.4s, v30.16b, v2.4b[3]\n" + ".inst 0x4fa0ebb2 // sdot v18.4s, v29.16b, v0.4b[3]\n" + ".inst 0x4fa1ebb6 // sdot v22.4s, v29.16b, v1.4b[3]\n" + ".inst 0x4fa2ebba // sdot v26.4s, v29.16b, v2.4b[3]\n" + ".inst 0x4fa0eb93 // sdot v19.4s, v28.16b, v0.4b[3]\n" + ".inst 0x4fa1eb97 // sdot v23.4s, v28.16b, v1.4b[3]\n" + ".inst 0x4fa2eb9b // sdot v27.4s, v28.16b, v2.4b[3]\n" "tbnz %x[flags], #31, 70f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n" ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n" "70:" // Height 3: Multiply loop: unique 10: skip row sum - "prfm pldl1keep, [x10, #0x80]\n" - "prfm pldl1keep, [x23, #0x80]\n" - "prfm pldl1keep, [x22, #0x80]\n" + "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x28, #0x80]\n" + "prfm pldl1keep, [x27, #0x80]\n" "71:" // Height 3: Multiply loop: Main loop skip - "cbz x11, 78f\n" - "cmp x11, #0x4\n" + "cbz x10, 78f\n" + "cmp x10, #0x4\n" "blt 74f\n" "72:" // Height 3: Multiply loop: Odd block loop - "ldr s0, [x10], #0x4\n" - "ldr s1, [x23], #0x4\n" - "ldr s2, [x22], #0x4\n" + "ldr s0, [x9], #0x4\n" + "ldr s1, [x28], #0x4\n" + "ldr s2, [x27], #0x4\n" "tbnz %x[flags], #31, 73f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n" ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n" "73:" // Height 3: Multiply loop: unique 11: skip row sum - "ldr q6, [x13, #0x0]\n" - "sub x11, x11, #0x4\n" - "ldr q7, [x13, #0x10]\n" - "cmp x11, #0x4\n" - "ldr q8, [x13, #0x20]\n" - ".inst 0x4f80e0d0 // sdot v16.4s, v6.16b, v0.4b[0]\n" - "ldr q9, [x13, #0x30]\n" - ".inst 0x4f81e0d4 // sdot v20.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d8 // sdot v24.4s, v6.16b, v2.4b[0]\n" - "add x13, x13, #0x40\n" - ".inst 0x4f80e0f1 // sdot v17.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0f5 // sdot v21.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f9 // sdot v25.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f80e112 // sdot v18.4s, v8.16b, v0.4b[0]\n" - ".inst 0x4f81e116 // sdot v22.4s, v8.16b, v1.4b[0]\n" - ".inst 0x4f82e11a // sdot v26.4s, v8.16b, v2.4b[0]\n" - ".inst 0x4f80e133 // sdot v19.4s, v9.16b, v0.4b[0]\n" - ".inst 0x4f81e137 // sdot v23.4s, v9.16b, v1.4b[0]\n" - ".inst 0x4f82e13b // sdot v27.4s, v9.16b, v2.4b[0]\n" + "ldr q31, [x12, #0x0]\n" + "sub x10, x10, #0x4\n" + "ldr q30, [x12, #0x10]\n" + "cmp x10, #0x4\n" + "ldr q29, [x12, #0x20]\n" + ".inst 0x4f80e3f0 // sdot v16.4s, v31.16b, v0.4b[0]\n" + "ldr q28, [x12, #0x30]\n" + ".inst 0x4f81e3f4 // sdot v20.4s, v31.16b, v1.4b[0]\n" + ".inst 0x4f82e3f8 // sdot v24.4s, v31.16b, v2.4b[0]\n" + "add x12, x12, #0x40\n" + ".inst 0x4f80e3d1 // sdot v17.4s, v30.16b, v0.4b[0]\n" + ".inst 0x4f81e3d5 // sdot v21.4s, v30.16b, v1.4b[0]\n" + ".inst 0x4f82e3d9 // sdot v25.4s, v30.16b, v2.4b[0]\n" + ".inst 0x4f80e3b2 // sdot v18.4s, v29.16b, v0.4b[0]\n" + ".inst 0x4f81e3b6 // sdot v22.4s, v29.16b, v1.4b[0]\n" + ".inst 0x4f82e3ba // sdot v26.4s, v29.16b, v2.4b[0]\n" + ".inst 0x4f80e393 // sdot v19.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f81e397 // sdot v23.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f82e39b // sdot v27.4s, v28.16b, v2.4b[0]\n" "bge 72b\n" "74:" // Height 3: Multiply loop: Skip odd blocks - "cbz x11, 78f\n" - "tbz x11, #1, 75f\n" - "ldr h0, [x10], #0x2\n" - "ldr h1, [x23], #0x2\n" - "ldr h2, [x22], #0x2\n" - "tbz x11, #0, 76f\n" - "ld1 { v0.b }[2], [x10]\n" - "ld1 { v1.b }[2], [x23]\n" - "ld1 { v2.b }[2], [x22]\n" + "cbz x10, 78f\n" + "tbz x10, #1, 75f\n" + "ldr h0, [x9], #0x2\n" + "ldr h1, [x28], #0x2\n" + "ldr h2, [x27], #0x2\n" + "tbz x10, #0, 76f\n" + "ld1 { v0.b }[2], [x9]\n" + "ld1 { v1.b }[2], [x28]\n" + "ld1 { v2.b }[2], [x27]\n" "b 76f\n" "75:" // Height 3: Multiply loop: Ragged operand read: partial_1_0 - "ldr b0, [x10, #0x0]\n" - "ldr b1, [x23, #0x0]\n" - "ldr b2, [x22, #0x0]\n" + "ldr b0, [x9, #0x0]\n" + "ldr b1, [x28, #0x0]\n" + "ldr b2, [x27, #0x0]\n" "76:" // Height 3: Multiply loop: Ragged operand read: Done "tbnz %x[flags], #31, 77f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n" ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n" "77:" // Height 3: Multiply loop: unique 12: skip row sum - "ldr q10, [x13, #0x0]\n" - ".inst 0x4f80e150 // sdot v16.4s, v10.16b, v0.4b[0]\n" - "ldr q4, [x13, #0x10]\n" - ".inst 0x4f81e154 // sdot v20.4s, v10.16b, v1.4b[0]\n" - "ldr q5, [x13, #0x20]\n" - ".inst 0x4f82e158 // sdot v24.4s, v10.16b, v2.4b[0]\n" - "ldr q6, [x13, #0x30]\n" - ".inst 0x4f80e091 // sdot v17.4s, v4.16b, v0.4b[0]\n" - ".inst 0x4f81e095 // sdot v21.4s, v4.16b, v1.4b[0]\n" - "add x13, x13, #0x40\n" - ".inst 0x4f82e099 // sdot v25.4s, v4.16b, v2.4b[0]\n" - ".inst 0x4f80e0b2 // sdot v18.4s, v5.16b, v0.4b[0]\n" - ".inst 0x4f81e0b6 // sdot v22.4s, v5.16b, v1.4b[0]\n" - ".inst 0x4f82e0ba // sdot v26.4s, v5.16b, v2.4b[0]\n" - ".inst 0x4f80e0d3 // sdot v19.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0d7 // sdot v23.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0db // sdot v27.4s, v6.16b, v2.4b[0]\n" + "ldr q28, [x12, #0x0]\n" + ".inst 0x4f80e390 // sdot v16.4s, v28.16b, v0.4b[0]\n" + "ldr q30, [x12, #0x10]\n" + ".inst 0x4f81e394 // sdot v20.4s, v28.16b, v1.4b[0]\n" + "ldr q29, [x12, #0x20]\n" + ".inst 0x4f82e398 // sdot v24.4s, v28.16b, v2.4b[0]\n" + "ldr q28, [x12, #0x30]\n" + ".inst 0x4f80e3d1 // sdot v17.4s, v30.16b, v0.4b[0]\n" + ".inst 0x4f81e3d5 // sdot v21.4s, v30.16b, v1.4b[0]\n" + "add x12, x12, #0x40\n" + ".inst 0x4f82e3d9 // sdot v25.4s, v30.16b, v2.4b[0]\n" + ".inst 0x4f80e3b2 // sdot v18.4s, v29.16b, v0.4b[0]\n" + ".inst 0x4f81e3b6 // sdot v22.4s, v29.16b, v1.4b[0]\n" + ".inst 0x4f82e3ba // sdot v26.4s, v29.16b, v2.4b[0]\n" + ".inst 0x4f80e393 // sdot v19.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f81e397 // sdot v23.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f82e39b // sdot v27.4s, v28.16b, v2.4b[0]\n" "78:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" - "add x12, x12, #0x1\n" - "cmp x12, x20\n" + "add x11, x11, #0x1\n" + "cmp x11, x20\n" "bne 64b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x22, x14, x20\n" - "add x21, x22, x20\n" - "prfm pstl1keep, [x14, #0x0]\n" + "add x23, x13, x20\n" + "add x22, x23, x20\n" + "prfm pstl1keep, [x13, #0x0]\n" + "prfm pstl1keep, [x23, #0x0]\n" "prfm pstl1keep, [x22, #0x0]\n" - "prfm pstl1keep, [x21, #0x0]\n" "tbnz %x[flags], #31, 79f\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v12.4s, v12.4s, v12.4s\n" "addp v13.4s, v13.4s, v13.4s\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1r { v3.4s }, [x23]\n" - "neg v3.4s, v3.4s\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1r { v28.4s }, [x20]\n" + "neg v28.4s, v28.4s\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v12.4s, v12.4s, v12.4s\n" "addp v13.4s, v13.4s, v13.4s\n" - "mul v11.4s, v11.4s, v3.4s\n" - "mul v12.4s, v12.4s, v3.4s\n" - "mul v13.4s, v13.4s, v3.4s\n" + "mul v11.4s, v11.4s, v28.4s\n" + "mul v12.4s, v12.4s, v28.4s\n" + "mul v13.4s, v13.4s, v28.4s\n" "79:" // Height 3: skip row sum fixup - "ldr q0, [x16, #0x0]\n" + "ldr q31, [x15, #0x0]\n" "add v16.4s, v16.4s, v11.4s\n" - "ldr q1, [x16, #0x10]\n" + "ldr q30, [x15, #0x10]\n" "add v17.4s, v17.4s, v11.4s\n" - "ldr q2, [x16, #0x20]\n" + "ldr q29, [x15, #0x20]\n" "add v18.4s, v18.4s, v11.4s\n" - "ldr q3, [x16, #0x30]\n" + "ldr q28, [x15, #0x30]\n" "add v19.4s, v19.4s, v11.4s\n" "add v20.4s, v20.4s, v12.4s\n" "add v21.4s, v21.4s, v12.4s\n" @@ -1171,73 +1170,73 @@ void a64_hybrid_s8qa_dot_4x16_a55 ( "add v25.4s, v25.4s, v13.4s\n" "add v26.4s, v26.4s, v13.4s\n" "add v27.4s, v27.4s, v13.4s\n" - "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" - "add v20.4s, v20.4s, v0.4s\n" - "add v21.4s, v21.4s, v1.4s\n" - "add v22.4s, v22.4s, v2.4s\n" - "add v23.4s, v23.4s, v3.4s\n" - "add v24.4s, v24.4s, v0.4s\n" - "add v25.4s, v25.4s, v1.4s\n" - "add v26.4s, v26.4s, v2.4s\n" - "add v27.4s, v27.4s, v3.4s\n" - "add x23, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x23]\n" + "add v16.4s, v16.4s, v31.4s\n" + "add v17.4s, v17.4s, v30.4s\n" + "add v18.4s, v18.4s, v29.4s\n" + "add v19.4s, v19.4s, v28.4s\n" + "add v20.4s, v20.4s, v31.4s\n" + "add v21.4s, v21.4s, v30.4s\n" + "add v22.4s, v22.4s, v29.4s\n" + "add v23.4s, v23.4s, v28.4s\n" + "add v24.4s, v24.4s, v31.4s\n" + "add v25.4s, v25.4s, v30.4s\n" + "add v26.4s, v26.4s, v29.4s\n" + "add v27.4s, v27.4s, v28.4s\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v28.4s }, [x20]\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x23]\n" - "sqrdmulh v16.4s, v16.4s, v4.4s\n" - "sqrdmulh v17.4s, v17.4s, v4.4s\n" - "sqrdmulh v18.4s, v18.4s, v4.4s\n" - "sqrdmulh v19.4s, v19.4s, v4.4s\n" - "sqrdmulh v20.4s, v20.4s, v4.4s\n" - "sqrdmulh v21.4s, v21.4s, v4.4s\n" - "sqrdmulh v22.4s, v22.4s, v4.4s\n" - "sqrdmulh v23.4s, v23.4s, v4.4s\n" - "sqrdmulh v24.4s, v24.4s, v4.4s\n" - "sqrdmulh v25.4s, v25.4s, v4.4s\n" - "sqrdmulh v26.4s, v26.4s, v4.4s\n" - "sqrdmulh v27.4s, v27.4s, v4.4s\n" - "add x16, x16, #0x40\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "sqrdmulh v16.4s, v16.4s, v28.4s\n" + "sqrdmulh v17.4s, v17.4s, v28.4s\n" + "sqrdmulh v18.4s, v18.4s, v28.4s\n" + "sqrdmulh v19.4s, v19.4s, v28.4s\n" + "sqrdmulh v20.4s, v20.4s, v28.4s\n" + "sqrdmulh v21.4s, v21.4s, v28.4s\n" + "sqrdmulh v22.4s, v22.4s, v28.4s\n" + "sqrdmulh v23.4s, v23.4s, v28.4s\n" + "sqrdmulh v24.4s, v24.4s, v28.4s\n" + "sqrdmulh v25.4s, v25.4s, v28.4s\n" + "sqrdmulh v26.4s, v26.4s, v28.4s\n" + "sqrdmulh v27.4s, v27.4s, v28.4s\n" + "add x15, x15, #0x40\n" "tbz %x[flags], #5, 80f\n" - "and v4.16b, v16.16b, v0.16b\n" - "and v5.16b, v17.16b, v0.16b\n" - "and v6.16b, v18.16b, v0.16b\n" - "and v7.16b, v19.16b, v0.16b\n" - "and v8.16b, v20.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" - "sqadd v20.4s, v20.4s, v8.4s\n" - "and v9.16b, v21.16b, v0.16b\n" - "and v10.16b, v22.16b, v0.16b\n" - "and v4.16b, v23.16b, v0.16b\n" - "and v5.16b, v24.16b, v0.16b\n" - "and v6.16b, v25.16b, v0.16b\n" - "and v7.16b, v26.16b, v0.16b\n" - "and v8.16b, v27.16b, v0.16b\n" - "sshr v9.4s, v9.4s, #0x1f\n" - "sshr v10.4s, v10.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sqadd v21.4s, v21.4s, v9.4s\n" - "sqadd v22.4s, v22.4s, v10.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v24.4s, v24.4s, v5.4s\n" - "sqadd v25.4s, v25.4s, v6.4s\n" - "sqadd v26.4s, v26.4s, v7.4s\n" - "sqadd v27.4s, v27.4s, v8.4s\n" + "and v1.16b, v16.16b, v0.16b\n" + "and v31.16b, v17.16b, v0.16b\n" + "and v30.16b, v18.16b, v0.16b\n" + "and v29.16b, v19.16b, v0.16b\n" + "and v28.16b, v20.16b, v0.16b\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v1.4s\n" + "sqadd v17.4s, v17.4s, v31.4s\n" + "sqadd v18.4s, v18.4s, v30.4s\n" + "sqadd v19.4s, v19.4s, v29.4s\n" + "sqadd v20.4s, v20.4s, v28.4s\n" + "and v3.16b, v21.16b, v0.16b\n" + "and v2.16b, v22.16b, v0.16b\n" + "and v1.16b, v23.16b, v0.16b\n" + "and v31.16b, v24.16b, v0.16b\n" + "and v30.16b, v25.16b, v0.16b\n" + "and v29.16b, v26.16b, v0.16b\n" + "and v28.16b, v27.16b, v0.16b\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sqadd v21.4s, v21.4s, v3.4s\n" + "sqadd v22.4s, v22.4s, v2.4s\n" + "sqadd v23.4s, v23.4s, v1.4s\n" + "sqadd v24.4s, v24.4s, v31.4s\n" + "sqadd v25.4s, v25.4s, v30.4s\n" + "sqadd v26.4s, v26.4s, v29.4s\n" + "sqadd v27.4s, v27.4s, v28.4s\n" "80:" // Height 3: no shift correction "srshl v16.4s, v16.4s, v0.4s\n" "srshl v17.4s, v17.4s, v0.4s\n" @@ -1251,156 +1250,156 @@ void a64_hybrid_s8qa_dot_4x16_a55 ( "srshl v25.4s, v25.4s, v0.4s\n" "srshl v26.4s, v26.4s, v0.4s\n" "srshl v27.4s, v27.4s, v0.4s\n" - "add x23, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x23]\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v23.4s, v23.4s, v4.4s\n" - "add v24.4s, v24.4s, v4.4s\n" - "add v25.4s, v25.4s, v4.4s\n" - "add v26.4s, v26.4s, v4.4s\n" - "add v27.4s, v27.4s, v4.4s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x23]\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "smin v24.4s, v24.4s, v6.4s\n" - "smin v25.4s, v25.4s, v6.4s\n" - "smin v26.4s, v26.4s, v6.4s\n" - "smin v27.4s, v27.4s, v6.4s\n" - "add x23, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x23]\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" - "smax v24.4s, v24.4s, v5.4s\n" - "smax v25.4s, v25.4s, v5.4s\n" - "smax v26.4s, v26.4s, v5.4s\n" - "smax v27.4s, v27.4s, v5.4s\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v28.4s }, [x20]\n" + "add v16.4s, v16.4s, v28.4s\n" + "add v17.4s, v17.4s, v28.4s\n" + "add v18.4s, v18.4s, v28.4s\n" + "add v19.4s, v19.4s, v28.4s\n" + "add v20.4s, v20.4s, v28.4s\n" + "add v21.4s, v21.4s, v28.4s\n" + "add v22.4s, v22.4s, v28.4s\n" + "add v23.4s, v23.4s, v28.4s\n" + "add v24.4s, v24.4s, v28.4s\n" + "add v25.4s, v25.4s, v28.4s\n" + "add v26.4s, v26.4s, v28.4s\n" + "add v27.4s, v27.4s, v28.4s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v28.4s }, [x20]\n" + "smin v16.4s, v16.4s, v28.4s\n" + "smin v17.4s, v17.4s, v28.4s\n" + "smin v18.4s, v18.4s, v28.4s\n" + "smin v19.4s, v19.4s, v28.4s\n" + "smin v20.4s, v20.4s, v28.4s\n" + "smin v21.4s, v21.4s, v28.4s\n" + "smin v22.4s, v22.4s, v28.4s\n" + "smin v23.4s, v23.4s, v28.4s\n" + "smin v24.4s, v24.4s, v28.4s\n" + "smin v25.4s, v25.4s, v28.4s\n" + "smin v26.4s, v26.4s, v28.4s\n" + "smin v27.4s, v27.4s, v28.4s\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v28.4s }, [x20]\n" + "smax v16.4s, v16.4s, v28.4s\n" + "smax v17.4s, v17.4s, v28.4s\n" + "smax v18.4s, v18.4s, v28.4s\n" + "smax v19.4s, v19.4s, v28.4s\n" + "smax v20.4s, v20.4s, v28.4s\n" + "smax v21.4s, v21.4s, v28.4s\n" + "smax v22.4s, v22.4s, v28.4s\n" + "smax v23.4s, v23.4s, v28.4s\n" + "smax v24.4s, v24.4s, v28.4s\n" + "smax v25.4s, v25.4s, v28.4s\n" + "smax v26.4s, v26.4s, v28.4s\n" + "smax v27.4s, v27.4s, v28.4s\n" "uzp1 v16.8h, v16.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v19.8h, v18.8h, v19.8h\n" "uzp1 v20.8h, v20.8h, v21.8h\n" - "uzp1 v21.8h, v22.8h, v23.8h\n" + "uzp1 v18.8h, v22.8h, v23.8h\n" "uzp1 v24.8h, v24.8h, v25.8h\n" - "uzp1 v25.8h, v26.8h, v27.8h\n" - "cmp x15, #0x10\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" - "uzp1 v20.16b, v20.16b, v21.16b\n" - "uzp1 v24.16b, v24.16b, v25.16b\n" + "uzp1 v17.8h, v26.8h, v27.8h\n" + "cmp x14, #0x10\n" + "uzp1 v16.16b, v16.16b, v19.16b\n" + "uzp1 v20.16b, v20.16b, v18.16b\n" + "uzp1 v24.16b, v24.16b, v17.16b\n" "bge 89f\n" - "tbz x15, #3, 84f\n" - "str d16, [x14], #0x8\n" - "str d20, [x22], #0x8\n" - "str d24, [x21], #0x8\n" - "tbz x15, #2, 82f\n" - "st1 { v16.s }[2], [x14], #0x4\n" - "st1 { v20.s }[2], [x22], #0x4\n" - "st1 { v24.s }[2], [x21], #0x4\n" - "tbz x15, #1, 81f\n" - "st1 { v16.h }[6], [x14], #0x2\n" - "st1 { v20.h }[6], [x22], #0x2\n" - "st1 { v24.h }[6], [x21], #0x2\n" - "tbz x15, #0, 88f\n" - "st1 { v16.b }[14], [x14]\n" - "st1 { v20.b }[14], [x22]\n" - "st1 { v24.b }[14], [x21]\n" + "tbz x14, #3, 84f\n" + "str d16, [x13], #0x8\n" + "str d20, [x23], #0x8\n" + "str d24, [x22], #0x8\n" + "tbz x14, #2, 82f\n" + "st1 { v16.s }[2], [x13], #0x4\n" + "st1 { v20.s }[2], [x23], #0x4\n" + "st1 { v24.s }[2], [x22], #0x4\n" + "tbz x14, #1, 81f\n" + "st1 { v16.h }[6], [x13], #0x2\n" + "st1 { v20.h }[6], [x23], #0x2\n" + "st1 { v24.h }[6], [x22], #0x2\n" + "tbz x14, #0, 88f\n" + "st1 { v16.b }[14], [x13]\n" + "st1 { v20.b }[14], [x23]\n" + "st1 { v24.b }[14], [x22]\n" "b 88f\n" "81:" // Height 3: Partial direct writeback: partial_1_12 - "tbz x15, #0, 88f\n" - "st1 { v16.b }[12], [x14]\n" - "st1 { v20.b }[12], [x22]\n" - "st1 { v24.b }[12], [x21]\n" + "tbz x14, #0, 88f\n" + "st1 { v16.b }[12], [x13]\n" + "st1 { v20.b }[12], [x23]\n" + "st1 { v24.b }[12], [x22]\n" "b 88f\n" "82:" // Height 3: Partial direct writeback: partial_2_8 - "tbz x15, #1, 83f\n" - "st1 { v16.h }[4], [x14], #0x2\n" - "st1 { v20.h }[4], [x22], #0x2\n" - "st1 { v24.h }[4], [x21], #0x2\n" - "tbz x15, #0, 88f\n" - "st1 { v16.b }[10], [x14]\n" - "st1 { v20.b }[10], [x22]\n" - "st1 { v24.b }[10], [x21]\n" + "tbz x14, #1, 83f\n" + "st1 { v16.h }[4], [x13], #0x2\n" + "st1 { v20.h }[4], [x23], #0x2\n" + "st1 { v24.h }[4], [x22], #0x2\n" + "tbz x14, #0, 88f\n" + "st1 { v16.b }[10], [x13]\n" + "st1 { v20.b }[10], [x23]\n" + "st1 { v24.b }[10], [x22]\n" "b 88f\n" "83:" // Height 3: Partial direct writeback: partial_1_8 - "tbz x15, #0, 88f\n" - "st1 { v16.b }[8], [x14]\n" - "st1 { v20.b }[8], [x22]\n" - "st1 { v24.b }[8], [x21]\n" + "tbz x14, #0, 88f\n" + "st1 { v16.b }[8], [x13]\n" + "st1 { v20.b }[8], [x23]\n" + "st1 { v24.b }[8], [x22]\n" "b 88f\n" "84:" // Height 3: Partial direct writeback: partial_4_0 - "tbz x15, #2, 86f\n" - "str s16, [x14], #0x4\n" - "str s20, [x22], #0x4\n" - "str s24, [x21], #0x4\n" - "tbz x15, #1, 85f\n" - "st1 { v16.h }[2], [x14], #0x2\n" - "st1 { v20.h }[2], [x22], #0x2\n" - "st1 { v24.h }[2], [x21], #0x2\n" - "tbz x15, #0, 88f\n" - "st1 { v16.b }[6], [x14]\n" - "st1 { v20.b }[6], [x22]\n" - "st1 { v24.b }[6], [x21]\n" + "tbz x14, #2, 86f\n" + "str s16, [x13], #0x4\n" + "str s20, [x23], #0x4\n" + "str s24, [x22], #0x4\n" + "tbz x14, #1, 85f\n" + "st1 { v16.h }[2], [x13], #0x2\n" + "st1 { v20.h }[2], [x23], #0x2\n" + "st1 { v24.h }[2], [x22], #0x2\n" + "tbz x14, #0, 88f\n" + "st1 { v16.b }[6], [x13]\n" + "st1 { v20.b }[6], [x23]\n" + "st1 { v24.b }[6], [x22]\n" "b 88f\n" "85:" // Height 3: Partial direct writeback: partial_1_4 - "tbz x15, #0, 88f\n" - "st1 { v16.b }[4], [x14]\n" - "st1 { v20.b }[4], [x22]\n" - "st1 { v24.b }[4], [x21]\n" + "tbz x14, #0, 88f\n" + "st1 { v16.b }[4], [x13]\n" + "st1 { v20.b }[4], [x23]\n" + "st1 { v24.b }[4], [x22]\n" "b 88f\n" "86:" // Height 3: Partial direct writeback: partial_2_0 - "tbz x15, #1, 87f\n" - "str h16, [x14], #0x2\n" - "str h20, [x22], #0x2\n" - "str h24, [x21], #0x2\n" - "tbz x15, #0, 88f\n" - "st1 { v16.b }[2], [x14]\n" - "st1 { v20.b }[2], [x22]\n" - "st1 { v24.b }[2], [x21]\n" + "tbz x14, #1, 87f\n" + "str h16, [x13], #0x2\n" + "str h20, [x23], #0x2\n" + "str h24, [x22], #0x2\n" + "tbz x14, #0, 88f\n" + "st1 { v16.b }[2], [x13]\n" + "st1 { v20.b }[2], [x23]\n" + "st1 { v24.b }[2], [x22]\n" "b 88f\n" "87:" // Height 3: Partial direct writeback: partial_1_0 - "str b16, [x14, #0x0]\n" - "str b20, [x22, #0x0]\n" - "str b24, [x21, #0x0]\n" + "str b16, [x13, #0x0]\n" + "str b20, [x23, #0x0]\n" + "str b24, [x22, #0x0]\n" "88:" // Height 3: Partial direct writeback: Done "b 90f\n" "89:" // Height 3: Full writeback - "str q16, [x14, #0x0]\n" - "add x14, x14, #0x10\n" - "str q20, [x22, #0x0]\n" - "str q24, [x21, #0x0]\n" + "str q16, [x13, #0x0]\n" + "add x13, x13, #0x10\n" + "str q20, [x23, #0x0]\n" + "str q24, [x22, #0x0]\n" "90:" // Height 3: Writeback done - "subs x15, x15, #0x10\n" + "subs x14, x14, #0x10\n" "bgt 62b\n" "b 122f\n" "91:" // Height 4 "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n" "mov x20, #0x4\n" - "mov x16, %x[col_bias]\n" + "mov x15, %x[col_bias]\n" "movi v11.4s, #0x0\n" "movi v12.4s, #0x0\n" "bic %x[flags], %x[flags], #0x80000000\n" "movi v13.4s, #0x0\n" - "ldr x15, [%x[args_ptr], %[offsetof_N]]\n" + "ldr x14, [%x[args_ptr], %[offsetof_N]]\n" "movi v14.4s, #0x0\n" - "ldr x13, [%x[args_ptr], %[offsetof_B_ptr]]\n" + "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n" "movi v15.16b, #0x1\n" - "mov x14, %x[output_ptr]\n" + "mov x13, %x[output_ptr]\n" "madd %x[output_ptr], x21, x20, %x[output_ptr]\n" "92:" // Height 4: Column loop "movi v16.4s, #0x0\n" @@ -1420,117 +1419,117 @@ void a64_hybrid_s8qa_dot_4x16_a55 ( "movi v30.4s, #0x0\n" "movi v31.4s, #0x0\n" "93:" // Height 4: setup done - "mov x12, #0x0\n" + "mov x11, #0x0\n" "94:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr w11, [x20, x12, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr w10, [x20, x11, LSL #0x2]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 95f\n" - "ldr x21, [%x[input_ptr], x12, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x10, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" - "ldr x21, [x21, #0x18]\n" - "cbnz x12, 96f\n" + "ldr x20, [%x[input_ptr], x11, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x9, [x20, #0x0]\n" + "ldr x28, [x20, #0x8]\n" + "ldr x27, [x20, #0x10]\n" + "ldr x26, [x20, #0x18]\n" + "cbnz x11, 96f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x10, x10, x20\n" - "add x23, x23, x20\n" - "add x22, x22, x20\n" - "add x21, x21, x20\n" + "add x9, x9, x20\n" + "add x28, x28, x20\n" + "add x27, x27, x20\n" + "add x26, x26, x20\n" "b 96f\n" "95:" // Height 4: setup direct input - "mov x10, %x[input_ptr]\n" - "add x23, x10, x20\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" + "mov x9, %x[input_ptr]\n" + "add x28, x9, x21\n" + "add x27, x28, x21\n" + "add x26, x27, x21\n" "96:" // Height 4: input setup done - "cmp x11, #0x10\n" + "cmp x10, #0x10\n" "blt 101f\n" - "ldr q0, [x10, #0x0]\n" - "cmp x11, #0x20\n" - "ldr q1, [x23, #0x0]\n" - "ldr q2, [x22, #0x0]\n" - "ldr q3, [x21, #0x0]\n" - "ldr q4, [x13, #0x0]\n" - "ldr q5, [x13, #0x10]\n" - "ldr q6, [x13, #0x20]\n" - "ldr q7, [x13, #0x30]\n" - "ldr q8, [x13, #0x40]\n" - "ldr q9, [x13, #0x50]\n" - "ldr q10, [x13, #0x60]\n" + "ldr q0, [x9, #0x0]\n" + "cmp x10, #0x20\n" + "ldr q1, [x28, #0x0]\n" + "ldr q2, [x27, #0x0]\n" + "ldr q3, [x26, #0x0]\n" + "ldr q4, [x12, #0x0]\n" + "ldr q5, [x12, #0x10]\n" + "ldr q6, [x12, #0x20]\n" + "ldr q7, [x12, #0x30]\n" + "ldr q8, [x12, #0x40]\n" + "ldr q9, [x12, #0x50]\n" + "ldr q10, [x12, #0x60]\n" "blt 99f\n" "97:" // Height 4: Multiply loop: Main loop head ".inst 0x4f80e090 // sdot v16.4s, v4.16b, v0.4b[0]\n" - "ldr x9, [x13, #0x78]\n" + "ldr x22, [x12, #0x78]\n" ".inst 0x4f81e094 // sdot v20.4s, v4.16b, v1.4b[0]\n" - "ldr x28, [x13, #0x88]\n" + "ldr x21, [x12, #0x88]\n" ".inst 0x4f82e098 // sdot v24.4s, v4.16b, v2.4b[0]\n" - "ldr x27, [x13, #0x98]\n" + "ldr x20, [x12, #0x98]\n" ".inst 0x4f83e09c // sdot v28.4s, v4.16b, v3.4b[0]\n" - "ldr d4, [x13, #0x70]\n" + "ldr d4, [x12, #0x70]\n" ".inst 0x4f80e0b1 // sdot v17.4s, v5.16b, v0.4b[0]\n" - "mov v4.d[1], x9\n" + "mov v4.d[1], x22\n" ".inst 0x4f81e0b5 // sdot v21.4s, v5.16b, v1.4b[0]\n" - "ldr x26, [x13, #0xa8]\n" + "ldr x25, [x12, #0xa8]\n" ".inst 0x4f82e0b9 // sdot v25.4s, v5.16b, v2.4b[0]\n" - "ldr x25, [x13, #0xb8]\n" + "ldr x24, [x12, #0xb8]\n" ".inst 0x4f83e0bd // sdot v29.4s, v5.16b, v3.4b[0]\n" - "ldr d5, [x13, #0x80]\n" + "ldr d5, [x12, #0x80]\n" ".inst 0x4f80e0d2 // sdot v18.4s, v6.16b, v0.4b[0]\n" - "mov v5.d[1], x28\n" + "mov v5.d[1], x21\n" ".inst 0x4f81e0d6 // sdot v22.4s, v6.16b, v1.4b[0]\n" - "ldr x24, [x13, #0xc8]\n" + "ldr x23, [x12, #0xc8]\n" ".inst 0x4f82e0da // sdot v26.4s, v6.16b, v2.4b[0]\n" - "ldr x20, [x13, #0xd8]\n" + "ldr x22, [x12, #0xd8]\n" ".inst 0x4f83e0de // sdot v30.4s, v6.16b, v3.4b[0]\n" - "ldr d6, [x13, #0x90]\n" + "ldr d6, [x12, #0x90]\n" ".inst 0x4f80e0f3 // sdot v19.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x27\n" + "mov v6.d[1], x20\n" ".inst 0x4f81e0f7 // sdot v23.4s, v7.16b, v1.4b[0]\n" - "ldr x9, [x13, #0xe8]\n" + "ldr x21, [x12, #0xe8]\n" ".inst 0x4f82e0fb // sdot v27.4s, v7.16b, v2.4b[0]\n" - "ldr x28, [x13, #0xf8]\n" + "ldr x20, [x12, #0xf8]\n" ".inst 0x4f83e0ff // sdot v31.4s, v7.16b, v3.4b[0]\n" - "ldr d7, [x13, #0xa0]\n" + "ldr d7, [x12, #0xa0]\n" ".inst 0x4fa0e110 // sdot v16.4s, v8.16b, v0.4b[1]\n" - "mov v7.d[1], x26\n" + "mov v7.d[1], x25\n" ".inst 0x4fa1e114 // sdot v20.4s, v8.16b, v1.4b[1]\n" - "add x10, x10, #0x10\n" + "add x9, x9, #0x10\n" ".inst 0x4fa2e118 // sdot v24.4s, v8.16b, v2.4b[1]\n" - "add x23, x23, #0x10\n" + "add x28, x28, #0x10\n" ".inst 0x4fa3e11c // sdot v28.4s, v8.16b, v3.4b[1]\n" - "ldr d8, [x13, #0xb0]\n" + "ldr d8, [x12, #0xb0]\n" ".inst 0x4fa0e131 // sdot v17.4s, v9.16b, v0.4b[1]\n" - "mov v8.d[1], x25\n" + "mov v8.d[1], x24\n" ".inst 0x4fa1e135 // sdot v21.4s, v9.16b, v1.4b[1]\n" - "add x22, x22, #0x10\n" + "add x27, x27, #0x10\n" ".inst 0x4fa2e139 // sdot v25.4s, v9.16b, v2.4b[1]\n" - "add x21, x21, #0x10\n" + "add x26, x26, #0x10\n" ".inst 0x4fa3e13d // sdot v29.4s, v9.16b, v3.4b[1]\n" - "ldr d9, [x13, #0xc0]\n" + "ldr d9, [x12, #0xc0]\n" ".inst 0x4fa0e152 // sdot v18.4s, v10.16b, v0.4b[1]\n" - "mov v9.d[1], x24\n" + "mov v9.d[1], x23\n" ".inst 0x4fa1e156 // sdot v22.4s, v10.16b, v1.4b[1]\n" ".inst 0x4fa2e15a // sdot v26.4s, v10.16b, v2.4b[1]\n" ".inst 0x4fa3e15e // sdot v30.4s, v10.16b, v3.4b[1]\n" - "ldr d10, [x13, #0xd0]\n" + "ldr d10, [x12, #0xd0]\n" ".inst 0x4fa0e093 // sdot v19.4s, v4.16b, v0.4b[1]\n" - "mov v10.d[1], x20\n" + "mov v10.d[1], x22\n" ".inst 0x4fa1e097 // sdot v23.4s, v4.16b, v1.4b[1]\n" ".inst 0x4fa2e09b // sdot v27.4s, v4.16b, v2.4b[1]\n" ".inst 0x4fa3e09f // sdot v31.4s, v4.16b, v3.4b[1]\n" - "ldr d4, [x13, #0xe0]\n" + "ldr d4, [x12, #0xe0]\n" ".inst 0x4f80e8b0 // sdot v16.4s, v5.16b, v0.4b[2]\n" - "mov v4.d[1], x9\n" + "mov v4.d[1], x21\n" ".inst 0x4f81e8b4 // sdot v20.4s, v5.16b, v1.4b[2]\n" ".inst 0x4f82e8b8 // sdot v24.4s, v5.16b, v2.4b[2]\n" ".inst 0x4f83e8bc // sdot v28.4s, v5.16b, v3.4b[2]\n" - "ldr d5, [x13, #0xf0]\n" + "ldr d5, [x12, #0xf0]\n" ".inst 0x4f80e8d1 // sdot v17.4s, v6.16b, v0.4b[2]\n" - "mov v5.d[1], x28\n" + "mov v5.d[1], x20\n" ".inst 0x4f81e8d5 // sdot v21.4s, v6.16b, v1.4b[2]\n" - "add x13, x13, #0x100\n" + "add x12, x12, #0x100\n" ".inst 0x4f82e8d9 // sdot v25.4s, v6.16b, v2.4b[2]\n" ".inst 0x4f83e8dd // sdot v29.4s, v6.16b, v3.4b[2]\n" ".inst 0x4f80e8f2 // sdot v18.4s, v7.16b, v0.4b[2]\n" @@ -1563,77 +1562,77 @@ void a64_hybrid_s8qa_dot_4x16_a55 ( ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n" ".inst 0x4e8f946e // sdot v14.4s, v3.16b, v15.16b\n" "98:" // Height 4: Multiply loop: unique 13: skip row sum - "ldr q0, [x10, #0x0]\n" - "sub x11, x11, #0x10\n" - "ldr q1, [x23, #0x0]\n" - "cmp x11, #0x20\n" - "ldr q2, [x22, #0x0]\n" - "ldr q3, [x21, #0x0]\n" - "ldr q4, [x13, #0x0]\n" - "ldr q5, [x13, #0x10]\n" - "ldr q6, [x13, #0x20]\n" - "ldr q7, [x13, #0x30]\n" - "ldr q8, [x13, #0x40]\n" - "ldr q9, [x13, #0x50]\n" - "ldr q10, [x13, #0x60]\n" - "prfm pldl1keep, [x10, #0x80]\n" - "prfm pldl1keep, [x23, #0x80]\n" - "prfm pldl1keep, [x22, #0x80]\n" - "prfm pldl1keep, [x21, #0x80]\n" + "ldr q0, [x9, #0x0]\n" + "sub x10, x10, #0x10\n" + "ldr q1, [x28, #0x0]\n" + "cmp x10, #0x20\n" + "ldr q2, [x27, #0x0]\n" + "ldr q3, [x26, #0x0]\n" + "ldr q4, [x12, #0x0]\n" + "ldr q5, [x12, #0x10]\n" + "ldr q6, [x12, #0x20]\n" + "ldr q7, [x12, #0x30]\n" + "ldr q8, [x12, #0x40]\n" + "ldr q9, [x12, #0x50]\n" + "ldr q10, [x12, #0x60]\n" + "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x28, #0x80]\n" + "prfm pldl1keep, [x27, #0x80]\n" + "prfm pldl1keep, [x26, #0x80]\n" "bge 97b\n" "99:" // Height 4: Multiply loop: Single iteration only ".inst 0x4f80e090 // sdot v16.4s, v4.16b, v0.4b[0]\n" - "sub x11, x11, #0x10\n" + "sub x10, x10, #0x10\n" ".inst 0x4f81e094 // sdot v20.4s, v4.16b, v1.4b[0]\n" - "add x10, x10, #0x10\n" + "add x9, x9, #0x10\n" ".inst 0x4f82e098 // sdot v24.4s, v4.16b, v2.4b[0]\n" - "add x23, x23, #0x10\n" + "add x28, x28, #0x10\n" ".inst 0x4f83e09c // sdot v28.4s, v4.16b, v3.4b[0]\n" - "ldr q4, [x13, #0x70]\n" + "ldr q4, [x12, #0x70]\n" ".inst 0x4f80e0b1 // sdot v17.4s, v5.16b, v0.4b[0]\n" - "add x22, x22, #0x10\n" + "add x27, x27, #0x10\n" ".inst 0x4f81e0b5 // sdot v21.4s, v5.16b, v1.4b[0]\n" - "add x21, x21, #0x10\n" + "add x26, x26, #0x10\n" ".inst 0x4f82e0b9 // sdot v25.4s, v5.16b, v2.4b[0]\n" ".inst 0x4f83e0bd // sdot v29.4s, v5.16b, v3.4b[0]\n" - "ldr q5, [x13, #0x80]\n" + "ldr q5, [x12, #0x80]\n" ".inst 0x4f80e0d2 // sdot v18.4s, v6.16b, v0.4b[0]\n" ".inst 0x4f81e0d6 // sdot v22.4s, v6.16b, v1.4b[0]\n" ".inst 0x4f82e0da // sdot v26.4s, v6.16b, v2.4b[0]\n" ".inst 0x4f83e0de // sdot v30.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x13, #0x90]\n" + "ldr q6, [x12, #0x90]\n" ".inst 0x4f80e0f3 // sdot v19.4s, v7.16b, v0.4b[0]\n" ".inst 0x4f81e0f7 // sdot v23.4s, v7.16b, v1.4b[0]\n" ".inst 0x4f82e0fb // sdot v27.4s, v7.16b, v2.4b[0]\n" ".inst 0x4f83e0ff // sdot v31.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x13, #0xa0]\n" + "ldr q7, [x12, #0xa0]\n" ".inst 0x4fa0e110 // sdot v16.4s, v8.16b, v0.4b[1]\n" ".inst 0x4fa1e114 // sdot v20.4s, v8.16b, v1.4b[1]\n" ".inst 0x4fa2e118 // sdot v24.4s, v8.16b, v2.4b[1]\n" ".inst 0x4fa3e11c // sdot v28.4s, v8.16b, v3.4b[1]\n" - "ldr q8, [x13, #0xb0]\n" + "ldr q8, [x12, #0xb0]\n" ".inst 0x4fa0e131 // sdot v17.4s, v9.16b, v0.4b[1]\n" ".inst 0x4fa1e135 // sdot v21.4s, v9.16b, v1.4b[1]\n" ".inst 0x4fa2e139 // sdot v25.4s, v9.16b, v2.4b[1]\n" ".inst 0x4fa3e13d // sdot v29.4s, v9.16b, v3.4b[1]\n" - "ldr q9, [x13, #0xc0]\n" + "ldr q9, [x12, #0xc0]\n" ".inst 0x4fa0e152 // sdot v18.4s, v10.16b, v0.4b[1]\n" ".inst 0x4fa1e156 // sdot v22.4s, v10.16b, v1.4b[1]\n" ".inst 0x4fa2e15a // sdot v26.4s, v10.16b, v2.4b[1]\n" ".inst 0x4fa3e15e // sdot v30.4s, v10.16b, v3.4b[1]\n" - "ldr q10, [x13, #0xd0]\n" + "ldr q10, [x12, #0xd0]\n" ".inst 0x4fa0e093 // sdot v19.4s, v4.16b, v0.4b[1]\n" ".inst 0x4fa1e097 // sdot v23.4s, v4.16b, v1.4b[1]\n" ".inst 0x4fa2e09b // sdot v27.4s, v4.16b, v2.4b[1]\n" ".inst 0x4fa3e09f // sdot v31.4s, v4.16b, v3.4b[1]\n" - "ldr q4, [x13, #0xe0]\n" + "ldr q4, [x12, #0xe0]\n" ".inst 0x4f80e8b0 // sdot v16.4s, v5.16b, v0.4b[2]\n" ".inst 0x4f81e8b4 // sdot v20.4s, v5.16b, v1.4b[2]\n" ".inst 0x4f82e8b8 // sdot v24.4s, v5.16b, v2.4b[2]\n" ".inst 0x4f83e8bc // sdot v28.4s, v5.16b, v3.4b[2]\n" - "ldr q5, [x13, #0xf0]\n" + "ldr q5, [x12, #0xf0]\n" ".inst 0x4f80e8d1 // sdot v17.4s, v6.16b, v0.4b[2]\n" - "add x13, x13, #0x100\n" + "add x12, x12, #0x100\n" ".inst 0x4f81e8d5 // sdot v21.4s, v6.16b, v1.4b[2]\n" ".inst 0x4f82e8d9 // sdot v25.4s, v6.16b, v2.4b[2]\n" ".inst 0x4f83e8dd // sdot v29.4s, v6.16b, v3.4b[2]\n" @@ -1667,67 +1666,67 @@ void a64_hybrid_s8qa_dot_4x16_a55 ( ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n" ".inst 0x4e8f946e // sdot v14.4s, v3.16b, v15.16b\n" "100:" // Height 4: Multiply loop: unique 14: skip row sum - "prfm pldl1keep, [x10, #0x80]\n" - "prfm pldl1keep, [x23, #0x80]\n" - "prfm pldl1keep, [x22, #0x80]\n" - "prfm pldl1keep, [x21, #0x80]\n" + "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x28, #0x80]\n" + "prfm pldl1keep, [x27, #0x80]\n" + "prfm pldl1keep, [x26, #0x80]\n" "101:" // Height 4: Multiply loop: Main loop skip - "cbz x11, 108f\n" - "cmp x11, #0x4\n" + "cbz x10, 108f\n" + "cmp x10, #0x4\n" "blt 104f\n" "102:" // Height 4: Multiply loop: Odd block loop - "ldr s0, [x10], #0x4\n" - "ldr s1, [x23], #0x4\n" - "ldr s2, [x22], #0x4\n" - "ldr s3, [x21], #0x4\n" + "ldr s0, [x9], #0x4\n" + "ldr s1, [x28], #0x4\n" + "ldr s2, [x27], #0x4\n" + "ldr s3, [x26], #0x4\n" "tbnz %x[flags], #31, 103f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n" ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n" ".inst 0x4e8f946e // sdot v14.4s, v3.16b, v15.16b\n" "103:" // Height 4: Multiply loop: unique 15: skip row sum - "ldr q6, [x13, #0x0]\n" - "sub x11, x11, #0x4\n" - "ldr q7, [x13, #0x10]\n" - "cmp x11, #0x4\n" - "ldr q8, [x13, #0x20]\n" - ".inst 0x4f80e0d0 // sdot v16.4s, v6.16b, v0.4b[0]\n" - "ldr q9, [x13, #0x30]\n" - ".inst 0x4f81e0d4 // sdot v20.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d8 // sdot v24.4s, v6.16b, v2.4b[0]\n" - "add x13, x13, #0x40\n" - ".inst 0x4f83e0dc // sdot v28.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f80e0f1 // sdot v17.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0f5 // sdot v21.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f9 // sdot v25.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0fd // sdot v29.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f80e112 // sdot v18.4s, v8.16b, v0.4b[0]\n" - ".inst 0x4f81e116 // sdot v22.4s, v8.16b, v1.4b[0]\n" - ".inst 0x4f82e11a // sdot v26.4s, v8.16b, v2.4b[0]\n" - ".inst 0x4f83e11e // sdot v30.4s, v8.16b, v3.4b[0]\n" - ".inst 0x4f80e133 // sdot v19.4s, v9.16b, v0.4b[0]\n" - ".inst 0x4f81e137 // sdot v23.4s, v9.16b, v1.4b[0]\n" - ".inst 0x4f82e13b // sdot v27.4s, v9.16b, v2.4b[0]\n" - ".inst 0x4f83e13f // sdot v31.4s, v9.16b, v3.4b[0]\n" + "ldr q7, [x12, #0x0]\n" + "sub x10, x10, #0x4\n" + "ldr q6, [x12, #0x10]\n" + "cmp x10, #0x4\n" + "ldr q5, [x12, #0x20]\n" + ".inst 0x4f80e0f0 // sdot v16.4s, v7.16b, v0.4b[0]\n" + "ldr q4, [x12, #0x30]\n" + ".inst 0x4f81e0f4 // sdot v20.4s, v7.16b, v1.4b[0]\n" + ".inst 0x4f82e0f8 // sdot v24.4s, v7.16b, v2.4b[0]\n" + "add x12, x12, #0x40\n" + ".inst 0x4f83e0fc // sdot v28.4s, v7.16b, v3.4b[0]\n" + ".inst 0x4f80e0d1 // sdot v17.4s, v6.16b, v0.4b[0]\n" + ".inst 0x4f81e0d5 // sdot v21.4s, v6.16b, v1.4b[0]\n" + ".inst 0x4f82e0d9 // sdot v25.4s, v6.16b, v2.4b[0]\n" + ".inst 0x4f83e0dd // sdot v29.4s, v6.16b, v3.4b[0]\n" + ".inst 0x4f80e0b2 // sdot v18.4s, v5.16b, v0.4b[0]\n" + ".inst 0x4f81e0b6 // sdot v22.4s, v5.16b, v1.4b[0]\n" + ".inst 0x4f82e0ba // sdot v26.4s, v5.16b, v2.4b[0]\n" + ".inst 0x4f83e0be // sdot v30.4s, v5.16b, v3.4b[0]\n" + ".inst 0x4f80e093 // sdot v19.4s, v4.16b, v0.4b[0]\n" + ".inst 0x4f81e097 // sdot v23.4s, v4.16b, v1.4b[0]\n" + ".inst 0x4f82e09b // sdot v27.4s, v4.16b, v2.4b[0]\n" + ".inst 0x4f83e09f // sdot v31.4s, v4.16b, v3.4b[0]\n" "bge 102b\n" "104:" // Height 4: Multiply loop: Skip odd blocks - "cbz x11, 108f\n" - "tbz x11, #1, 105f\n" - "ldr h0, [x10], #0x2\n" - "ldr h1, [x23], #0x2\n" - "ldr h2, [x22], #0x2\n" - "ldr h3, [x21], #0x2\n" - "tbz x11, #0, 106f\n" - "ld1 { v0.b }[2], [x10]\n" - "ld1 { v1.b }[2], [x23]\n" - "ld1 { v2.b }[2], [x22]\n" - "ld1 { v3.b }[2], [x21]\n" + "cbz x10, 108f\n" + "tbz x10, #1, 105f\n" + "ldr h0, [x9], #0x2\n" + "ldr h1, [x28], #0x2\n" + "ldr h2, [x27], #0x2\n" + "ldr h3, [x26], #0x2\n" + "tbz x10, #0, 106f\n" + "ld1 { v0.b }[2], [x9]\n" + "ld1 { v1.b }[2], [x28]\n" + "ld1 { v2.b }[2], [x27]\n" + "ld1 { v3.b }[2], [x26]\n" "b 106f\n" "105:" // Height 4: Multiply loop: Ragged operand read: partial_1_0 - "ldr b0, [x10, #0x0]\n" - "ldr b1, [x23, #0x0]\n" - "ldr b2, [x22, #0x0]\n" - "ldr b3, [x21, #0x0]\n" + "ldr b0, [x9, #0x0]\n" + "ldr b1, [x28, #0x0]\n" + "ldr b2, [x27, #0x0]\n" + "ldr b3, [x26, #0x0]\n" "106:" // Height 4: Multiply loop: Ragged operand read: Done "tbnz %x[flags], #31, 107f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" @@ -1735,64 +1734,64 @@ void a64_hybrid_s8qa_dot_4x16_a55 ( ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n" ".inst 0x4e8f946e // sdot v14.4s, v3.16b, v15.16b\n" "107:" // Height 4: Multiply loop: unique 16: skip row sum - "ldr q10, [x13, #0x0]\n" - ".inst 0x4f80e150 // sdot v16.4s, v10.16b, v0.4b[0]\n" - "ldr q4, [x13, #0x10]\n" - ".inst 0x4f81e154 // sdot v20.4s, v10.16b, v1.4b[0]\n" - "ldr q5, [x13, #0x20]\n" - ".inst 0x4f82e158 // sdot v24.4s, v10.16b, v2.4b[0]\n" - "ldr q6, [x13, #0x30]\n" - ".inst 0x4f83e15c // sdot v28.4s, v10.16b, v3.4b[0]\n" - ".inst 0x4f80e091 // sdot v17.4s, v4.16b, v0.4b[0]\n" - "add x13, x13, #0x40\n" - ".inst 0x4f81e095 // sdot v21.4s, v4.16b, v1.4b[0]\n" - ".inst 0x4f82e099 // sdot v25.4s, v4.16b, v2.4b[0]\n" - ".inst 0x4f83e09d // sdot v29.4s, v4.16b, v3.4b[0]\n" + "ldr q7, [x12, #0x0]\n" + ".inst 0x4f80e0f0 // sdot v16.4s, v7.16b, v0.4b[0]\n" + "ldr q6, [x12, #0x10]\n" + ".inst 0x4f81e0f4 // sdot v20.4s, v7.16b, v1.4b[0]\n" + "ldr q5, [x12, #0x20]\n" + ".inst 0x4f82e0f8 // sdot v24.4s, v7.16b, v2.4b[0]\n" + "ldr q4, [x12, #0x30]\n" + ".inst 0x4f83e0fc // sdot v28.4s, v7.16b, v3.4b[0]\n" + ".inst 0x4f80e0d1 // sdot v17.4s, v6.16b, v0.4b[0]\n" + "add x12, x12, #0x40\n" + ".inst 0x4f81e0d5 // sdot v21.4s, v6.16b, v1.4b[0]\n" + ".inst 0x4f82e0d9 // sdot v25.4s, v6.16b, v2.4b[0]\n" + ".inst 0x4f83e0dd // sdot v29.4s, v6.16b, v3.4b[0]\n" ".inst 0x4f80e0b2 // sdot v18.4s, v5.16b, v0.4b[0]\n" ".inst 0x4f81e0b6 // sdot v22.4s, v5.16b, v1.4b[0]\n" ".inst 0x4f82e0ba // sdot v26.4s, v5.16b, v2.4b[0]\n" ".inst 0x4f83e0be // sdot v30.4s, v5.16b, v3.4b[0]\n" - ".inst 0x4f80e0d3 // sdot v19.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0d7 // sdot v23.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0db // sdot v27.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0df // sdot v31.4s, v6.16b, v3.4b[0]\n" + ".inst 0x4f80e093 // sdot v19.4s, v4.16b, v0.4b[0]\n" + ".inst 0x4f81e097 // sdot v23.4s, v4.16b, v1.4b[0]\n" + ".inst 0x4f82e09b // sdot v27.4s, v4.16b, v2.4b[0]\n" + ".inst 0x4f83e09f // sdot v31.4s, v4.16b, v3.4b[0]\n" "108:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" - "add x12, x12, #0x1\n" - "cmp x12, x20\n" + "add x11, x11, #0x1\n" + "cmp x11, x20\n" "bne 94b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x22, x14, x20\n" + "add x23, x13, x20\n" + "add x22, x23, x20\n" "add x21, x22, x20\n" - "add x20, x21, x20\n" - "prfm pstl1keep, [x14, #0x0]\n" + "prfm pstl1keep, [x13, #0x0]\n" + "prfm pstl1keep, [x23, #0x0]\n" "prfm pstl1keep, [x22, #0x0]\n" "prfm pstl1keep, [x21, #0x0]\n" - "prfm pstl1keep, [x20, #0x0]\n" "tbnz %x[flags], #31, 109f\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v12.4s, v12.4s, v12.4s\n" "addp v13.4s, v13.4s, v13.4s\n" "addp v14.4s, v14.4s, v14.4s\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1r { v4.4s }, [x23]\n" - "neg v4.4s, v4.4s\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1r { v0.4s }, [x20]\n" + "neg v0.4s, v0.4s\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v12.4s, v12.4s, v12.4s\n" "addp v13.4s, v13.4s, v13.4s\n" "addp v14.4s, v14.4s, v14.4s\n" - "mul v11.4s, v11.4s, v4.4s\n" - "mul v12.4s, v12.4s, v4.4s\n" - "mul v13.4s, v13.4s, v4.4s\n" - "mul v14.4s, v14.4s, v4.4s\n" + "mul v11.4s, v11.4s, v0.4s\n" + "mul v12.4s, v12.4s, v0.4s\n" + "mul v13.4s, v13.4s, v0.4s\n" + "mul v14.4s, v14.4s, v0.4s\n" "109:" // Height 4: skip row sum fixup - "ldr q0, [x16, #0x0]\n" + "ldr q3, [x15, #0x0]\n" "add v16.4s, v16.4s, v11.4s\n" - "ldr q1, [x16, #0x10]\n" + "ldr q2, [x15, #0x10]\n" "add v17.4s, v17.4s, v11.4s\n" - "ldr q2, [x16, #0x20]\n" + "ldr q1, [x15, #0x20]\n" "add v18.4s, v18.4s, v11.4s\n" - "ldr q3, [x16, #0x30]\n" + "ldr q0, [x15, #0x30]\n" "add v19.4s, v19.4s, v11.4s\n" "add v20.4s, v20.4s, v12.4s\n" "add v21.4s, v21.4s, v12.4s\n" @@ -1806,93 +1805,93 @@ void a64_hybrid_s8qa_dot_4x16_a55 ( "add v29.4s, v29.4s, v14.4s\n" "add v30.4s, v30.4s, v14.4s\n" "add v31.4s, v31.4s, v14.4s\n" - "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" - "add v20.4s, v20.4s, v0.4s\n" - "add v21.4s, v21.4s, v1.4s\n" - "add v22.4s, v22.4s, v2.4s\n" - "add v23.4s, v23.4s, v3.4s\n" - "add v24.4s, v24.4s, v0.4s\n" - "add v25.4s, v25.4s, v1.4s\n" - "add v26.4s, v26.4s, v2.4s\n" - "add v27.4s, v27.4s, v3.4s\n" - "add v28.4s, v28.4s, v0.4s\n" - "add v29.4s, v29.4s, v1.4s\n" - "add v30.4s, v30.4s, v2.4s\n" - "add v31.4s, v31.4s, v3.4s\n" - "add x23, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x23]\n" + "add v16.4s, v16.4s, v3.4s\n" + "add v17.4s, v17.4s, v2.4s\n" + "add v18.4s, v18.4s, v1.4s\n" + "add v19.4s, v19.4s, v0.4s\n" + "add v20.4s, v20.4s, v3.4s\n" + "add v21.4s, v21.4s, v2.4s\n" + "add v22.4s, v22.4s, v1.4s\n" + "add v23.4s, v23.4s, v0.4s\n" + "add v24.4s, v24.4s, v3.4s\n" + "add v25.4s, v25.4s, v2.4s\n" + "add v26.4s, v26.4s, v1.4s\n" + "add v27.4s, v27.4s, v0.4s\n" + "add v28.4s, v28.4s, v3.4s\n" + "add v29.4s, v29.4s, v2.4s\n" + "add v30.4s, v30.4s, v1.4s\n" + "add v31.4s, v31.4s, v0.4s\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v1.4s }, [x20]\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x23]\n" - "sqrdmulh v16.4s, v16.4s, v4.4s\n" - "sqrdmulh v17.4s, v17.4s, v4.4s\n" - "sqrdmulh v18.4s, v18.4s, v4.4s\n" - "sqrdmulh v19.4s, v19.4s, v4.4s\n" - "sqrdmulh v20.4s, v20.4s, v4.4s\n" - "sqrdmulh v21.4s, v21.4s, v4.4s\n" - "sqrdmulh v22.4s, v22.4s, v4.4s\n" - "sqrdmulh v23.4s, v23.4s, v4.4s\n" - "sqrdmulh v24.4s, v24.4s, v4.4s\n" - "sqrdmulh v25.4s, v25.4s, v4.4s\n" - "sqrdmulh v26.4s, v26.4s, v4.4s\n" - "sqrdmulh v27.4s, v27.4s, v4.4s\n" - "sqrdmulh v28.4s, v28.4s, v4.4s\n" - "sqrdmulh v29.4s, v29.4s, v4.4s\n" - "sqrdmulh v30.4s, v30.4s, v4.4s\n" - "sqrdmulh v31.4s, v31.4s, v4.4s\n" - "add x16, x16, #0x40\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "sqrdmulh v16.4s, v16.4s, v1.4s\n" + "sqrdmulh v17.4s, v17.4s, v1.4s\n" + "sqrdmulh v18.4s, v18.4s, v1.4s\n" + "sqrdmulh v19.4s, v19.4s, v1.4s\n" + "sqrdmulh v20.4s, v20.4s, v1.4s\n" + "sqrdmulh v21.4s, v21.4s, v1.4s\n" + "sqrdmulh v22.4s, v22.4s, v1.4s\n" + "sqrdmulh v23.4s, v23.4s, v1.4s\n" + "sqrdmulh v24.4s, v24.4s, v1.4s\n" + "sqrdmulh v25.4s, v25.4s, v1.4s\n" + "sqrdmulh v26.4s, v26.4s, v1.4s\n" + "sqrdmulh v27.4s, v27.4s, v1.4s\n" + "sqrdmulh v28.4s, v28.4s, v1.4s\n" + "sqrdmulh v29.4s, v29.4s, v1.4s\n" + "sqrdmulh v30.4s, v30.4s, v1.4s\n" + "sqrdmulh v31.4s, v31.4s, v1.4s\n" + "add x15, x15, #0x40\n" "tbz %x[flags], #5, 110f\n" - "and v4.16b, v16.16b, v0.16b\n" - "and v5.16b, v17.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "and v6.16b, v18.16b, v0.16b\n" - "and v7.16b, v19.16b, v0.16b\n" - "and v8.16b, v20.16b, v0.16b\n" - "and v9.16b, v21.16b, v0.16b\n" - "and v10.16b, v22.16b, v0.16b\n" - "and v4.16b, v23.16b, v0.16b\n" - "and v5.16b, v24.16b, v0.16b\n" - "sshr v6.4s, v6.4s, #0x1f\n" + "and v2.16b, v16.16b, v0.16b\n" + "and v1.16b, v17.16b, v0.16b\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v2.4s\n" + "sqadd v17.4s, v17.4s, v1.4s\n" + "and v7.16b, v18.16b, v0.16b\n" + "and v6.16b, v19.16b, v0.16b\n" + "and v5.16b, v20.16b, v0.16b\n" + "and v4.16b, v21.16b, v0.16b\n" + "and v3.16b, v22.16b, v0.16b\n" + "and v2.16b, v23.16b, v0.16b\n" + "and v1.16b, v24.16b, v0.16b\n" "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v9.4s, v9.4s, #0x1f\n" - "sshr v10.4s, v10.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" - "sqadd v20.4s, v20.4s, v8.4s\n" - "sqadd v21.4s, v21.4s, v9.4s\n" - "sqadd v22.4s, v22.4s, v10.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v24.4s, v24.4s, v5.4s\n" - "and v6.16b, v25.16b, v0.16b\n" - "and v7.16b, v26.16b, v0.16b\n" - "and v8.16b, v27.16b, v0.16b\n" - "and v9.16b, v28.16b, v0.16b\n" - "and v10.16b, v29.16b, v0.16b\n" - "and v4.16b, v30.16b, v0.16b\n" - "and v5.16b, v31.16b, v0.16b\n" "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v9.4s, v9.4s, #0x1f\n" - "sshr v10.4s, v10.4s, #0x1f\n" + "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sqadd v18.4s, v18.4s, v7.4s\n" + "sqadd v19.4s, v19.4s, v6.4s\n" + "sqadd v20.4s, v20.4s, v5.4s\n" + "sqadd v21.4s, v21.4s, v4.4s\n" + "sqadd v22.4s, v22.4s, v3.4s\n" + "sqadd v23.4s, v23.4s, v2.4s\n" + "sqadd v24.4s, v24.4s, v1.4s\n" + "and v7.16b, v25.16b, v0.16b\n" + "and v6.16b, v26.16b, v0.16b\n" + "and v5.16b, v27.16b, v0.16b\n" + "and v4.16b, v28.16b, v0.16b\n" + "and v3.16b, v29.16b, v0.16b\n" + "and v2.16b, v30.16b, v0.16b\n" + "and v1.16b, v31.16b, v0.16b\n" + "sshr v7.4s, v7.4s, #0x1f\n" + "sshr v6.4s, v6.4s, #0x1f\n" "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v25.4s, v25.4s, v6.4s\n" - "sqadd v26.4s, v26.4s, v7.4s\n" - "sqadd v27.4s, v27.4s, v8.4s\n" - "sqadd v28.4s, v28.4s, v9.4s\n" - "sqadd v29.4s, v29.4s, v10.4s\n" - "sqadd v30.4s, v30.4s, v4.4s\n" - "sqadd v31.4s, v31.4s, v5.4s\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sqadd v25.4s, v25.4s, v7.4s\n" + "sqadd v26.4s, v26.4s, v6.4s\n" + "sqadd v27.4s, v27.4s, v5.4s\n" + "sqadd v28.4s, v28.4s, v4.4s\n" + "sqadd v29.4s, v29.4s, v3.4s\n" + "sqadd v30.4s, v30.4s, v2.4s\n" + "sqadd v31.4s, v31.4s, v1.4s\n" "110:" // Height 4: no shift correction "srshl v16.4s, v16.4s, v0.4s\n" "srshl v17.4s, v17.4s, v0.4s\n" @@ -1910,172 +1909,172 @@ void a64_hybrid_s8qa_dot_4x16_a55 ( "srshl v29.4s, v29.4s, v0.4s\n" "srshl v30.4s, v30.4s, v0.4s\n" "srshl v31.4s, v31.4s, v0.4s\n" - "add x23, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x23]\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v23.4s, v23.4s, v4.4s\n" - "add v24.4s, v24.4s, v4.4s\n" - "add v25.4s, v25.4s, v4.4s\n" - "add v26.4s, v26.4s, v4.4s\n" - "add v27.4s, v27.4s, v4.4s\n" - "add v28.4s, v28.4s, v4.4s\n" - "add v29.4s, v29.4s, v4.4s\n" - "add v30.4s, v30.4s, v4.4s\n" - "add v31.4s, v31.4s, v4.4s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x23]\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "smin v24.4s, v24.4s, v6.4s\n" - "smin v25.4s, v25.4s, v6.4s\n" - "smin v26.4s, v26.4s, v6.4s\n" - "smin v27.4s, v27.4s, v6.4s\n" - "smin v28.4s, v28.4s, v6.4s\n" - "smin v29.4s, v29.4s, v6.4s\n" - "smin v30.4s, v30.4s, v6.4s\n" - "smin v31.4s, v31.4s, v6.4s\n" - "add x23, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x23]\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" - "smax v24.4s, v24.4s, v5.4s\n" - "smax v25.4s, v25.4s, v5.4s\n" - "smax v26.4s, v26.4s, v5.4s\n" - "smax v27.4s, v27.4s, v5.4s\n" - "smax v28.4s, v28.4s, v5.4s\n" - "smax v29.4s, v29.4s, v5.4s\n" - "smax v30.4s, v30.4s, v5.4s\n" - "smax v31.4s, v31.4s, v5.4s\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v0.4s }, [x20]\n" + "add v16.4s, v16.4s, v0.4s\n" + "add v17.4s, v17.4s, v0.4s\n" + "add v18.4s, v18.4s, v0.4s\n" + "add v19.4s, v19.4s, v0.4s\n" + "add v20.4s, v20.4s, v0.4s\n" + "add v21.4s, v21.4s, v0.4s\n" + "add v22.4s, v22.4s, v0.4s\n" + "add v23.4s, v23.4s, v0.4s\n" + "add v24.4s, v24.4s, v0.4s\n" + "add v25.4s, v25.4s, v0.4s\n" + "add v26.4s, v26.4s, v0.4s\n" + "add v27.4s, v27.4s, v0.4s\n" + "add v28.4s, v28.4s, v0.4s\n" + "add v29.4s, v29.4s, v0.4s\n" + "add v30.4s, v30.4s, v0.4s\n" + "add v31.4s, v31.4s, v0.4s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v0.4s }, [x20]\n" + "smin v16.4s, v16.4s, v0.4s\n" + "smin v17.4s, v17.4s, v0.4s\n" + "smin v18.4s, v18.4s, v0.4s\n" + "smin v19.4s, v19.4s, v0.4s\n" + "smin v20.4s, v20.4s, v0.4s\n" + "smin v21.4s, v21.4s, v0.4s\n" + "smin v22.4s, v22.4s, v0.4s\n" + "smin v23.4s, v23.4s, v0.4s\n" + "smin v24.4s, v24.4s, v0.4s\n" + "smin v25.4s, v25.4s, v0.4s\n" + "smin v26.4s, v26.4s, v0.4s\n" + "smin v27.4s, v27.4s, v0.4s\n" + "smin v28.4s, v28.4s, v0.4s\n" + "smin v29.4s, v29.4s, v0.4s\n" + "smin v30.4s, v30.4s, v0.4s\n" + "smin v31.4s, v31.4s, v0.4s\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v0.4s }, [x20]\n" + "smax v16.4s, v16.4s, v0.4s\n" + "smax v17.4s, v17.4s, v0.4s\n" + "smax v18.4s, v18.4s, v0.4s\n" + "smax v19.4s, v19.4s, v0.4s\n" + "smax v20.4s, v20.4s, v0.4s\n" + "smax v21.4s, v21.4s, v0.4s\n" + "smax v22.4s, v22.4s, v0.4s\n" + "smax v23.4s, v23.4s, v0.4s\n" + "smax v24.4s, v24.4s, v0.4s\n" + "smax v25.4s, v25.4s, v0.4s\n" + "smax v26.4s, v26.4s, v0.4s\n" + "smax v27.4s, v27.4s, v0.4s\n" + "smax v28.4s, v28.4s, v0.4s\n" + "smax v29.4s, v29.4s, v0.4s\n" + "smax v30.4s, v30.4s, v0.4s\n" + "smax v31.4s, v31.4s, v0.4s\n" "uzp1 v16.8h, v16.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v0.8h, v18.8h, v19.8h\n" "uzp1 v20.8h, v20.8h, v21.8h\n" - "uzp1 v21.8h, v22.8h, v23.8h\n" + "uzp1 v19.8h, v22.8h, v23.8h\n" "uzp1 v24.8h, v24.8h, v25.8h\n" - "uzp1 v25.8h, v26.8h, v27.8h\n" + "uzp1 v18.8h, v26.8h, v27.8h\n" "uzp1 v28.8h, v28.8h, v29.8h\n" - "uzp1 v29.8h, v30.8h, v31.8h\n" - "cmp x15, #0x10\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" - "uzp1 v20.16b, v20.16b, v21.16b\n" - "uzp1 v24.16b, v24.16b, v25.16b\n" - "uzp1 v28.16b, v28.16b, v29.16b\n" + "uzp1 v17.8h, v30.8h, v31.8h\n" + "cmp x14, #0x10\n" + "uzp1 v16.16b, v16.16b, v0.16b\n" + "uzp1 v20.16b, v20.16b, v19.16b\n" + "uzp1 v24.16b, v24.16b, v18.16b\n" + "uzp1 v28.16b, v28.16b, v17.16b\n" "bge 119f\n" - "tbz x15, #3, 114f\n" - "str d16, [x14], #0x8\n" - "str d20, [x22], #0x8\n" - "str d24, [x21], #0x8\n" - "str d28, [x20], #0x8\n" - "tbz x15, #2, 112f\n" - "st1 { v16.s }[2], [x14], #0x4\n" - "st1 { v20.s }[2], [x22], #0x4\n" - "st1 { v24.s }[2], [x21], #0x4\n" - "st1 { v28.s }[2], [x20], #0x4\n" - "tbz x15, #1, 111f\n" - "st1 { v16.h }[6], [x14], #0x2\n" - "st1 { v20.h }[6], [x22], #0x2\n" - "st1 { v24.h }[6], [x21], #0x2\n" - "st1 { v28.h }[6], [x20], #0x2\n" - "tbz x15, #0, 118f\n" - "st1 { v16.b }[14], [x14]\n" - "st1 { v20.b }[14], [x22]\n" - "st1 { v24.b }[14], [x21]\n" - "st1 { v28.b }[14], [x20]\n" + "tbz x14, #3, 114f\n" + "str d16, [x13], #0x8\n" + "str d20, [x23], #0x8\n" + "str d24, [x22], #0x8\n" + "str d28, [x21], #0x8\n" + "tbz x14, #2, 112f\n" + "st1 { v16.s }[2], [x13], #0x4\n" + "st1 { v20.s }[2], [x23], #0x4\n" + "st1 { v24.s }[2], [x22], #0x4\n" + "st1 { v28.s }[2], [x21], #0x4\n" + "tbz x14, #1, 111f\n" + "st1 { v16.h }[6], [x13], #0x2\n" + "st1 { v20.h }[6], [x23], #0x2\n" + "st1 { v24.h }[6], [x22], #0x2\n" + "st1 { v28.h }[6], [x21], #0x2\n" + "tbz x14, #0, 118f\n" + "st1 { v16.b }[14], [x13]\n" + "st1 { v20.b }[14], [x23]\n" + "st1 { v24.b }[14], [x22]\n" + "st1 { v28.b }[14], [x21]\n" "b 118f\n" "111:" // Height 4: Partial direct writeback: partial_1_12 - "tbz x15, #0, 118f\n" - "st1 { v16.b }[12], [x14]\n" - "st1 { v20.b }[12], [x22]\n" - "st1 { v24.b }[12], [x21]\n" - "st1 { v28.b }[12], [x20]\n" + "tbz x14, #0, 118f\n" + "st1 { v16.b }[12], [x13]\n" + "st1 { v20.b }[12], [x23]\n" + "st1 { v24.b }[12], [x22]\n" + "st1 { v28.b }[12], [x21]\n" "b 118f\n" "112:" // Height 4: Partial direct writeback: partial_2_8 - "tbz x15, #1, 113f\n" - "st1 { v16.h }[4], [x14], #0x2\n" - "st1 { v20.h }[4], [x22], #0x2\n" - "st1 { v24.h }[4], [x21], #0x2\n" - "st1 { v28.h }[4], [x20], #0x2\n" - "tbz x15, #0, 118f\n" - "st1 { v16.b }[10], [x14]\n" - "st1 { v20.b }[10], [x22]\n" - "st1 { v24.b }[10], [x21]\n" - "st1 { v28.b }[10], [x20]\n" + "tbz x14, #1, 113f\n" + "st1 { v16.h }[4], [x13], #0x2\n" + "st1 { v20.h }[4], [x23], #0x2\n" + "st1 { v24.h }[4], [x22], #0x2\n" + "st1 { v28.h }[4], [x21], #0x2\n" + "tbz x14, #0, 118f\n" + "st1 { v16.b }[10], [x13]\n" + "st1 { v20.b }[10], [x23]\n" + "st1 { v24.b }[10], [x22]\n" + "st1 { v28.b }[10], [x21]\n" "b 118f\n" "113:" // Height 4: Partial direct writeback: partial_1_8 - "tbz x15, #0, 118f\n" - "st1 { v16.b }[8], [x14]\n" - "st1 { v20.b }[8], [x22]\n" - "st1 { v24.b }[8], [x21]\n" - "st1 { v28.b }[8], [x20]\n" + "tbz x14, #0, 118f\n" + "st1 { v16.b }[8], [x13]\n" + "st1 { v20.b }[8], [x23]\n" + "st1 { v24.b }[8], [x22]\n" + "st1 { v28.b }[8], [x21]\n" "b 118f\n" "114:" // Height 4: Partial direct writeback: partial_4_0 - "tbz x15, #2, 116f\n" - "str s16, [x14], #0x4\n" - "str s20, [x22], #0x4\n" - "str s24, [x21], #0x4\n" - "str s28, [x20], #0x4\n" - "tbz x15, #1, 115f\n" - "st1 { v16.h }[2], [x14], #0x2\n" - "st1 { v20.h }[2], [x22], #0x2\n" - "st1 { v24.h }[2], [x21], #0x2\n" - "st1 { v28.h }[2], [x20], #0x2\n" - "tbz x15, #0, 118f\n" - "st1 { v16.b }[6], [x14]\n" - "st1 { v20.b }[6], [x22]\n" - "st1 { v24.b }[6], [x21]\n" - "st1 { v28.b }[6], [x20]\n" + "tbz x14, #2, 116f\n" + "str s16, [x13], #0x4\n" + "str s20, [x23], #0x4\n" + "str s24, [x22], #0x4\n" + "str s28, [x21], #0x4\n" + "tbz x14, #1, 115f\n" + "st1 { v16.h }[2], [x13], #0x2\n" + "st1 { v20.h }[2], [x23], #0x2\n" + "st1 { v24.h }[2], [x22], #0x2\n" + "st1 { v28.h }[2], [x21], #0x2\n" + "tbz x14, #0, 118f\n" + "st1 { v16.b }[6], [x13]\n" + "st1 { v20.b }[6], [x23]\n" + "st1 { v24.b }[6], [x22]\n" + "st1 { v28.b }[6], [x21]\n" "b 118f\n" "115:" // Height 4: Partial direct writeback: partial_1_4 - "tbz x15, #0, 118f\n" - "st1 { v16.b }[4], [x14]\n" - "st1 { v20.b }[4], [x22]\n" - "st1 { v24.b }[4], [x21]\n" - "st1 { v28.b }[4], [x20]\n" + "tbz x14, #0, 118f\n" + "st1 { v16.b }[4], [x13]\n" + "st1 { v20.b }[4], [x23]\n" + "st1 { v24.b }[4], [x22]\n" + "st1 { v28.b }[4], [x21]\n" "b 118f\n" "116:" // Height 4: Partial direct writeback: partial_2_0 - "tbz x15, #1, 117f\n" - "str h16, [x14], #0x2\n" - "str h20, [x22], #0x2\n" - "str h24, [x21], #0x2\n" - "str h28, [x20], #0x2\n" - "tbz x15, #0, 118f\n" - "st1 { v16.b }[2], [x14]\n" - "st1 { v20.b }[2], [x22]\n" - "st1 { v24.b }[2], [x21]\n" - "st1 { v28.b }[2], [x20]\n" + "tbz x14, #1, 117f\n" + "str h16, [x13], #0x2\n" + "str h20, [x23], #0x2\n" + "str h24, [x22], #0x2\n" + "str h28, [x21], #0x2\n" + "tbz x14, #0, 118f\n" + "st1 { v16.b }[2], [x13]\n" + "st1 { v20.b }[2], [x23]\n" + "st1 { v24.b }[2], [x22]\n" + "st1 { v28.b }[2], [x21]\n" "b 118f\n" "117:" // Height 4: Partial direct writeback: partial_1_0 - "str b16, [x14, #0x0]\n" - "str b20, [x22, #0x0]\n" - "str b24, [x21, #0x0]\n" - "str b28, [x20, #0x0]\n" + "str b16, [x13, #0x0]\n" + "str b20, [x23, #0x0]\n" + "str b24, [x22, #0x0]\n" + "str b28, [x21, #0x0]\n" "118:" // Height 4: Partial direct writeback: Done "b 120f\n" "119:" // Height 4: Full writeback - "str q16, [x14, #0x0]\n" - "add x14, x14, #0x10\n" - "str q20, [x22, #0x0]\n" - "str q24, [x21, #0x0]\n" - "str q28, [x20, #0x0]\n" + "str q16, [x13, #0x0]\n" + "add x13, x13, #0x10\n" + "str q20, [x23, #0x0]\n" + "str q24, [x22, #0x0]\n" + "str q28, [x21, #0x0]\n" "120:" // Height 4: Writeback done - "subs x15, x15, #0x10\n" + "subs x14, x14, #0x10\n" "bgt 92b\n" "subs %x[M], %x[M], #0x4\n" "beq 122f\n" @@ -2089,10 +2088,9 @@ void a64_hybrid_s8qa_dot_4x16_a55 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "122:" // Exit - : [M] "+&r" (M), [flags] "+&r" (flags), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/generic.cpp index 485a47dc67f5ce91897eff6867f78dbd94b3b9f4..3b773a6827f3938dcab380ebc61f9f5b1bc3c139 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/generic.cpp @@ -78,7 +78,6 @@ void a64_hybrid_s8qa_dot_4x16 ( flags |= 0x20; } __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x4\n" "bge 91f\n" @@ -102,11 +101,11 @@ void a64_hybrid_s8qa_dot_4x16 ( "4:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 5f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" "cbnz x26, 6f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -128,32 +127,32 @@ void a64_hybrid_s8qa_dot_4x16 ( "blt 9f\n" "7:" // Height 1: Multiply loop: Main loop head ".inst 0x4f80e090 // sdot v16.4s, v4.16b, v0.4b[0]\n" - "ldr q4, [x28, #0x70]\n" + "ldr q21, [x28, #0x70]\n" ".inst 0x4f80e0b1 // sdot v17.4s, v5.16b, v0.4b[0]\n" - "ldr q5, [x28, #0x80]\n" + "ldr q20, [x28, #0x80]\n" ".inst 0x4f80e0d2 // sdot v18.4s, v6.16b, v0.4b[0]\n" - "ldr q6, [x28, #0x90]\n" + "ldr q26, [x28, #0x90]\n" ".inst 0x4f80e0f3 // sdot v19.4s, v7.16b, v0.4b[0]\n" - "ldr q7, [x28, #0xa0]\n" + "ldr q25, [x28, #0xa0]\n" ".inst 0x4fa0e110 // sdot v16.4s, v8.16b, v0.4b[1]\n" - "ldr q8, [x28, #0xb0]\n" + "ldr q24, [x28, #0xb0]\n" ".inst 0x4fa0e131 // sdot v17.4s, v9.16b, v0.4b[1]\n" - "ldr q9, [x28, #0xc0]\n" + "ldr q23, [x28, #0xc0]\n" ".inst 0x4fa0e152 // sdot v18.4s, v10.16b, v0.4b[1]\n" - "ldr q10, [x28, #0xd0]\n" - ".inst 0x4fa0e093 // sdot v19.4s, v4.16b, v0.4b[1]\n" - "ldr q4, [x28, #0xe0]\n" - ".inst 0x4f80e8b0 // sdot v16.4s, v5.16b, v0.4b[2]\n" - "ldr q5, [x28, #0xf0]\n" - ".inst 0x4f80e8d1 // sdot v17.4s, v6.16b, v0.4b[2]\n" + "ldr q22, [x28, #0xd0]\n" + ".inst 0x4fa0e2b3 // sdot v19.4s, v21.16b, v0.4b[1]\n" + "ldr q21, [x28, #0xe0]\n" + ".inst 0x4f80ea90 // sdot v16.4s, v20.16b, v0.4b[2]\n" + "ldr q20, [x28, #0xf0]\n" + ".inst 0x4f80eb51 // sdot v17.4s, v26.16b, v0.4b[2]\n" "add x24, x24, #0x10\n" - ".inst 0x4f80e8f2 // sdot v18.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f80e913 // sdot v19.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80eb32 // sdot v18.4s, v25.16b, v0.4b[2]\n" + ".inst 0x4f80eb13 // sdot v19.4s, v24.16b, v0.4b[2]\n" "add x28, x28, #0x100\n" - ".inst 0x4fa0e930 // sdot v16.4s, v9.16b, v0.4b[3]\n" - ".inst 0x4fa0e951 // sdot v17.4s, v10.16b, v0.4b[3]\n" - ".inst 0x4fa0e892 // sdot v18.4s, v4.16b, v0.4b[3]\n" - ".inst 0x4fa0e8b3 // sdot v19.4s, v5.16b, v0.4b[3]\n" + ".inst 0x4fa0eaf0 // sdot v16.4s, v23.16b, v0.4b[3]\n" + ".inst 0x4fa0ead1 // sdot v17.4s, v22.16b, v0.4b[3]\n" + ".inst 0x4fa0eab2 // sdot v18.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa0ea93 // sdot v19.4s, v20.16b, v0.4b[3]\n" "tbnz %x[flags], #31, 8f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" "8:" // Height 1: Multiply loop: unique 1: skip row sum @@ -171,33 +170,33 @@ void a64_hybrid_s8qa_dot_4x16 ( "bge 7b\n" "9:" // Height 1: Multiply loop: Single iteration only ".inst 0x4f80e090 // sdot v16.4s, v4.16b, v0.4b[0]\n" - "ldr q4, [x28, #0x70]\n" + "ldr q21, [x28, #0x70]\n" ".inst 0x4f80e0b1 // sdot v17.4s, v5.16b, v0.4b[0]\n" - "ldr q5, [x28, #0x80]\n" + "ldr q20, [x28, #0x80]\n" ".inst 0x4f80e0d2 // sdot v18.4s, v6.16b, v0.4b[0]\n" - "ldr q6, [x28, #0x90]\n" + "ldr q26, [x28, #0x90]\n" ".inst 0x4f80e0f3 // sdot v19.4s, v7.16b, v0.4b[0]\n" - "ldr q7, [x28, #0xa0]\n" + "ldr q25, [x28, #0xa0]\n" ".inst 0x4fa0e110 // sdot v16.4s, v8.16b, v0.4b[1]\n" - "ldr q8, [x28, #0xb0]\n" + "ldr q24, [x28, #0xb0]\n" ".inst 0x4fa0e131 // sdot v17.4s, v9.16b, v0.4b[1]\n" - "ldr q9, [x28, #0xc0]\n" + "ldr q23, [x28, #0xc0]\n" ".inst 0x4fa0e152 // sdot v18.4s, v10.16b, v0.4b[1]\n" - "ldr q10, [x28, #0xd0]\n" - ".inst 0x4fa0e093 // sdot v19.4s, v4.16b, v0.4b[1]\n" - "ldr q4, [x28, #0xe0]\n" - ".inst 0x4f80e8b0 // sdot v16.4s, v5.16b, v0.4b[2]\n" - "ldr q5, [x28, #0xf0]\n" - ".inst 0x4f80e8d1 // sdot v17.4s, v6.16b, v0.4b[2]\n" + "ldr q22, [x28, #0xd0]\n" + ".inst 0x4fa0e2b3 // sdot v19.4s, v21.16b, v0.4b[1]\n" + "ldr q21, [x28, #0xe0]\n" + ".inst 0x4f80ea90 // sdot v16.4s, v20.16b, v0.4b[2]\n" + "ldr q20, [x28, #0xf0]\n" + ".inst 0x4f80eb51 // sdot v17.4s, v26.16b, v0.4b[2]\n" "sub x25, x25, #0x10\n" - ".inst 0x4f80e8f2 // sdot v18.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f80e913 // sdot v19.4s, v8.16b, v0.4b[2]\n" + ".inst 0x4f80eb32 // sdot v18.4s, v25.16b, v0.4b[2]\n" + ".inst 0x4f80eb13 // sdot v19.4s, v24.16b, v0.4b[2]\n" "add x24, x24, #0x10\n" "add x28, x28, #0x100\n" - ".inst 0x4fa0e930 // sdot v16.4s, v9.16b, v0.4b[3]\n" - ".inst 0x4fa0e951 // sdot v17.4s, v10.16b, v0.4b[3]\n" - ".inst 0x4fa0e892 // sdot v18.4s, v4.16b, v0.4b[3]\n" - ".inst 0x4fa0e8b3 // sdot v19.4s, v5.16b, v0.4b[3]\n" + ".inst 0x4fa0eaf0 // sdot v16.4s, v23.16b, v0.4b[3]\n" + ".inst 0x4fa0ead1 // sdot v17.4s, v22.16b, v0.4b[3]\n" + ".inst 0x4fa0eab2 // sdot v18.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa0ea93 // sdot v19.4s, v20.16b, v0.4b[3]\n" "tbnz %x[flags], #31, 10f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" "10:" // Height 1: Multiply loop: unique 2: skip row sum @@ -211,16 +210,16 @@ void a64_hybrid_s8qa_dot_4x16 ( "tbnz %x[flags], #31, 13f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" "13:" // Height 1: Multiply loop: unique 3: skip row sum - "ldr q6, [x28, #0x0]\n" - "ldr q7, [x28, #0x10]\n" + "ldr q23, [x28, #0x0]\n" + "ldr q22, [x28, #0x10]\n" "sub x25, x25, #0x4\n" "cmp x25, #0x4\n" - "ldr q8, [x28, #0x20]\n" - "ldr q9, [x28, #0x30]\n" - ".inst 0x4f80e0d0 // sdot v16.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f80e0f1 // sdot v17.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f80e112 // sdot v18.4s, v8.16b, v0.4b[0]\n" - ".inst 0x4f80e133 // sdot v19.4s, v9.16b, v0.4b[0]\n" + "ldr q21, [x28, #0x20]\n" + "ldr q20, [x28, #0x30]\n" + ".inst 0x4f80e2f0 // sdot v16.4s, v23.16b, v0.4b[0]\n" + ".inst 0x4f80e2d1 // sdot v17.4s, v22.16b, v0.4b[0]\n" + ".inst 0x4f80e2b2 // sdot v18.4s, v21.16b, v0.4b[0]\n" + ".inst 0x4f80e293 // sdot v19.4s, v20.16b, v0.4b[0]\n" "add x28, x28, #0x40\n" "bge 12b\n" "14:" // Height 1: Multiply loop: Skip odd blocks @@ -236,14 +235,14 @@ void a64_hybrid_s8qa_dot_4x16 ( "tbnz %x[flags], #31, 17f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" "17:" // Height 1: Multiply loop: unique 4: skip row sum - "ldr q10, [x28, #0x0]\n" - "ldr q4, [x28, #0x10]\n" - ".inst 0x4f80e150 // sdot v16.4s, v10.16b, v0.4b[0]\n" - ".inst 0x4f80e091 // sdot v17.4s, v4.16b, v0.4b[0]\n" - "ldr q5, [x28, #0x20]\n" - "ldr q6, [x28, #0x30]\n" - ".inst 0x4f80e0b2 // sdot v18.4s, v5.16b, v0.4b[0]\n" - ".inst 0x4f80e0d3 // sdot v19.4s, v6.16b, v0.4b[0]\n" + "ldr q21, [x28, #0x0]\n" + "ldr q20, [x28, #0x10]\n" + ".inst 0x4f80e2b0 // sdot v16.4s, v21.16b, v0.4b[0]\n" + ".inst 0x4f80e291 // sdot v17.4s, v20.16b, v0.4b[0]\n" + "ldr q21, [x28, #0x20]\n" + "ldr q20, [x28, #0x30]\n" + ".inst 0x4f80e2b2 // sdot v18.4s, v21.16b, v0.4b[0]\n" + ".inst 0x4f80e293 // sdot v19.4s, v20.16b, v0.4b[0]\n" "add x28, x28, #0x40\n" "18:" // Height 1: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -252,72 +251,72 @@ void a64_hybrid_s8qa_dot_4x16 ( "bne 4b\n" "prfm pstl1keep, [x27, #0x0]\n" "tbnz %x[flags], #31, 19f\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1r { v1.4s }, [x23]\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1r { v20.4s }, [x20]\n" "addp v11.4s, v11.4s, v11.4s\n" - "neg v1.4s, v1.4s\n" + "neg v20.4s, v20.4s\n" "addp v11.4s, v11.4s, v11.4s\n" - "mul v11.4s, v11.4s, v1.4s\n" + "mul v11.4s, v11.4s, v20.4s\n" "19:" // Height 1: skip row sum fixup - "ldr q0, [x10, #0x0]\n" - "ldr q1, [x10, #0x10]\n" + "ldr q24, [x10, #0x0]\n" + "ldr q23, [x10, #0x10]\n" "add v16.4s, v16.4s, v11.4s\n" "add v17.4s, v17.4s, v11.4s\n" - "ldr q2, [x10, #0x20]\n" - "ldr q3, [x10, #0x30]\n" + "ldr q22, [x10, #0x20]\n" + "ldr q21, [x10, #0x30]\n" "add v18.4s, v18.4s, v11.4s\n" "add v19.4s, v19.4s, v11.4s\n" - "add x23, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v20.4s }, [x20]\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x23]\n" - "add v19.4s, v19.4s, v3.4s\n" - "sqrdmulh v16.4s, v16.4s, v4.4s\n" + "add v16.4s, v16.4s, v24.4s\n" + "add v17.4s, v17.4s, v23.4s\n" + "add v18.4s, v18.4s, v22.4s\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "add v19.4s, v19.4s, v21.4s\n" + "sqrdmulh v16.4s, v16.4s, v20.4s\n" "add x10, x10, #0x40\n" - "sqrdmulh v17.4s, v17.4s, v4.4s\n" - "sqrdmulh v18.4s, v18.4s, v4.4s\n" - "sqrdmulh v19.4s, v19.4s, v4.4s\n" + "sqrdmulh v17.4s, v17.4s, v20.4s\n" + "sqrdmulh v18.4s, v18.4s, v20.4s\n" + "sqrdmulh v19.4s, v19.4s, v20.4s\n" "tbz %x[flags], #5, 20f\n" - "and v4.16b, v16.16b, v0.16b\n" - "and v5.16b, v17.16b, v0.16b\n" - "and v6.16b, v18.16b, v0.16b\n" - "and v7.16b, v19.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" + "and v23.16b, v16.16b, v0.16b\n" + "and v22.16b, v17.16b, v0.16b\n" + "and v21.16b, v18.16b, v0.16b\n" + "and v20.16b, v19.16b, v0.16b\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "sshr v21.4s, v21.4s, #0x1f\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v23.4s\n" + "sqadd v17.4s, v17.4s, v22.4s\n" + "sqadd v18.4s, v18.4s, v21.4s\n" + "sqadd v19.4s, v19.4s, v20.4s\n" "20:" // Height 1: no shift correction - "add x23, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v22.4s }, [x20]\n" "srshl v16.4s, v16.4s, v0.4s\n" "srshl v17.4s, v17.4s, v0.4s\n" "srshl v18.4s, v18.4s, v0.4s\n" "srshl v19.4s, v19.4s, v0.4s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x23]\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add x23, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x23]\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v21.4s }, [x20]\n" + "add v16.4s, v16.4s, v22.4s\n" + "add v17.4s, v17.4s, v22.4s\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v20.4s }, [x20]\n" + "add v18.4s, v18.4s, v22.4s\n" + "add v19.4s, v19.4s, v22.4s\n" "cmp x9, #0x10\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" + "smin v16.4s, v16.4s, v21.4s\n" + "smin v17.4s, v17.4s, v21.4s\n" + "smin v18.4s, v18.4s, v21.4s\n" + "smin v19.4s, v19.4s, v21.4s\n" + "smax v16.4s, v16.4s, v20.4s\n" + "smax v17.4s, v17.4s, v20.4s\n" + "smax v18.4s, v18.4s, v20.4s\n" + "smax v19.4s, v19.4s, v20.4s\n" "uzp1 v16.8h, v16.8h, v17.8h\n" "uzp1 v17.8h, v18.8h, v19.8h\n" "uzp1 v16.16b, v16.16b, v17.16b\n" @@ -397,12 +396,12 @@ void a64_hybrid_s8qa_dot_4x16 ( "34:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 35f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" "cbnz x26, 36f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -410,7 +409,7 @@ void a64_hybrid_s8qa_dot_4x16 ( "b 36f\n" "35:" // Height 2: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20\n" + "add x23, x24, x21\n" "36:" // Height 2: input setup done "cmp x25, #0x10\n" "blt 41f\n" @@ -428,48 +427,48 @@ void a64_hybrid_s8qa_dot_4x16 ( "37:" // Height 2: Multiply loop: Main loop head ".inst 0x4f80e090 // sdot v16.4s, v4.16b, v0.4b[0]\n" ".inst 0x4f81e094 // sdot v20.4s, v4.16b, v1.4b[0]\n" - "ldr q4, [x28, #0x70]\n" + "ldr q25, [x28, #0x70]\n" "add x24, x24, #0x10\n" ".inst 0x4f80e0b1 // sdot v17.4s, v5.16b, v0.4b[0]\n" ".inst 0x4f81e0b5 // sdot v21.4s, v5.16b, v1.4b[0]\n" - "ldr q5, [x28, #0x80]\n" + "ldr q24, [x28, #0x80]\n" "add x23, x23, #0x10\n" ".inst 0x4f80e0d2 // sdot v18.4s, v6.16b, v0.4b[0]\n" ".inst 0x4f81e0d6 // sdot v22.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x28, #0x90]\n" + "ldr q30, [x28, #0x90]\n" ".inst 0x4f80e0f3 // sdot v19.4s, v7.16b, v0.4b[0]\n" ".inst 0x4f81e0f7 // sdot v23.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x28, #0xa0]\n" + "ldr q29, [x28, #0xa0]\n" ".inst 0x4fa0e110 // sdot v16.4s, v8.16b, v0.4b[1]\n" ".inst 0x4fa1e114 // sdot v20.4s, v8.16b, v1.4b[1]\n" - "ldr q8, [x28, #0xb0]\n" + "ldr q28, [x28, #0xb0]\n" ".inst 0x4fa0e131 // sdot v17.4s, v9.16b, v0.4b[1]\n" ".inst 0x4fa1e135 // sdot v21.4s, v9.16b, v1.4b[1]\n" - "ldr q9, [x28, #0xc0]\n" + "ldr q27, [x28, #0xc0]\n" ".inst 0x4fa0e152 // sdot v18.4s, v10.16b, v0.4b[1]\n" ".inst 0x4fa1e156 // sdot v22.4s, v10.16b, v1.4b[1]\n" - "ldr q10, [x28, #0xd0]\n" - ".inst 0x4fa0e093 // sdot v19.4s, v4.16b, v0.4b[1]\n" - ".inst 0x4fa1e097 // sdot v23.4s, v4.16b, v1.4b[1]\n" - "ldr q4, [x28, #0xe0]\n" - ".inst 0x4f80e8b0 // sdot v16.4s, v5.16b, v0.4b[2]\n" - ".inst 0x4f81e8b4 // sdot v20.4s, v5.16b, v1.4b[2]\n" - "ldr q5, [x28, #0xf0]\n" + "ldr q26, [x28, #0xd0]\n" + ".inst 0x4fa0e333 // sdot v19.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa1e337 // sdot v23.4s, v25.16b, v1.4b[1]\n" + "ldr q25, [x28, #0xe0]\n" + ".inst 0x4f80eb10 // sdot v16.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f81eb14 // sdot v20.4s, v24.16b, v1.4b[2]\n" + "ldr q24, [x28, #0xf0]\n" "add x28, x28, #0x100\n" - ".inst 0x4f80e8d1 // sdot v17.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8d5 // sdot v21.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f80e8f2 // sdot v18.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8f6 // sdot v22.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f80e913 // sdot v19.4s, v8.16b, v0.4b[2]\n" - ".inst 0x4f81e917 // sdot v23.4s, v8.16b, v1.4b[2]\n" - ".inst 0x4fa0e930 // sdot v16.4s, v9.16b, v0.4b[3]\n" - ".inst 0x4fa1e934 // sdot v20.4s, v9.16b, v1.4b[3]\n" - ".inst 0x4fa0e951 // sdot v17.4s, v10.16b, v0.4b[3]\n" - ".inst 0x4fa1e955 // sdot v21.4s, v10.16b, v1.4b[3]\n" - ".inst 0x4fa0e892 // sdot v18.4s, v4.16b, v0.4b[3]\n" - ".inst 0x4fa1e896 // sdot v22.4s, v4.16b, v1.4b[3]\n" - ".inst 0x4fa0e8b3 // sdot v19.4s, v5.16b, v0.4b[3]\n" - ".inst 0x4fa1e8b7 // sdot v23.4s, v5.16b, v1.4b[3]\n" + ".inst 0x4f80ebd1 // sdot v17.4s, v30.16b, v0.4b[2]\n" + ".inst 0x4f81ebd5 // sdot v21.4s, v30.16b, v1.4b[2]\n" + ".inst 0x4f80ebb2 // sdot v18.4s, v29.16b, v0.4b[2]\n" + ".inst 0x4f81ebb6 // sdot v22.4s, v29.16b, v1.4b[2]\n" + ".inst 0x4f80eb93 // sdot v19.4s, v28.16b, v0.4b[2]\n" + ".inst 0x4f81eb97 // sdot v23.4s, v28.16b, v1.4b[2]\n" + ".inst 0x4fa0eb70 // sdot v16.4s, v27.16b, v0.4b[3]\n" + ".inst 0x4fa1eb74 // sdot v20.4s, v27.16b, v1.4b[3]\n" + ".inst 0x4fa0eb51 // sdot v17.4s, v26.16b, v0.4b[3]\n" + ".inst 0x4fa1eb55 // sdot v21.4s, v26.16b, v1.4b[3]\n" + ".inst 0x4fa0eb32 // sdot v18.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa1eb36 // sdot v22.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa0eb13 // sdot v19.4s, v24.16b, v0.4b[3]\n" + ".inst 0x4fa1eb17 // sdot v23.4s, v24.16b, v1.4b[3]\n" "tbnz %x[flags], #31, 38f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n" @@ -491,49 +490,49 @@ void a64_hybrid_s8qa_dot_4x16 ( "39:" // Height 2: Multiply loop: Single iteration only ".inst 0x4f80e090 // sdot v16.4s, v4.16b, v0.4b[0]\n" ".inst 0x4f81e094 // sdot v20.4s, v4.16b, v1.4b[0]\n" - "ldr q4, [x28, #0x70]\n" + "ldr q25, [x28, #0x70]\n" "sub x25, x25, #0x10\n" ".inst 0x4f80e0b1 // sdot v17.4s, v5.16b, v0.4b[0]\n" ".inst 0x4f81e0b5 // sdot v21.4s, v5.16b, v1.4b[0]\n" - "ldr q5, [x28, #0x80]\n" + "ldr q24, [x28, #0x80]\n" "add x24, x24, #0x10\n" ".inst 0x4f80e0d2 // sdot v18.4s, v6.16b, v0.4b[0]\n" ".inst 0x4f81e0d6 // sdot v22.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x28, #0x90]\n" + "ldr q30, [x28, #0x90]\n" "add x23, x23, #0x10\n" ".inst 0x4f80e0f3 // sdot v19.4s, v7.16b, v0.4b[0]\n" ".inst 0x4f81e0f7 // sdot v23.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x28, #0xa0]\n" + "ldr q29, [x28, #0xa0]\n" ".inst 0x4fa0e110 // sdot v16.4s, v8.16b, v0.4b[1]\n" ".inst 0x4fa1e114 // sdot v20.4s, v8.16b, v1.4b[1]\n" - "ldr q8, [x28, #0xb0]\n" + "ldr q28, [x28, #0xb0]\n" ".inst 0x4fa0e131 // sdot v17.4s, v9.16b, v0.4b[1]\n" ".inst 0x4fa1e135 // sdot v21.4s, v9.16b, v1.4b[1]\n" - "ldr q9, [x28, #0xc0]\n" + "ldr q27, [x28, #0xc0]\n" ".inst 0x4fa0e152 // sdot v18.4s, v10.16b, v0.4b[1]\n" ".inst 0x4fa1e156 // sdot v22.4s, v10.16b, v1.4b[1]\n" - "ldr q10, [x28, #0xd0]\n" - ".inst 0x4fa0e093 // sdot v19.4s, v4.16b, v0.4b[1]\n" - ".inst 0x4fa1e097 // sdot v23.4s, v4.16b, v1.4b[1]\n" - "ldr q4, [x28, #0xe0]\n" - ".inst 0x4f80e8b0 // sdot v16.4s, v5.16b, v0.4b[2]\n" - ".inst 0x4f81e8b4 // sdot v20.4s, v5.16b, v1.4b[2]\n" - "ldr q5, [x28, #0xf0]\n" + "ldr q26, [x28, #0xd0]\n" + ".inst 0x4fa0e333 // sdot v19.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa1e337 // sdot v23.4s, v25.16b, v1.4b[1]\n" + "ldr q25, [x28, #0xe0]\n" + ".inst 0x4f80eb10 // sdot v16.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f81eb14 // sdot v20.4s, v24.16b, v1.4b[2]\n" + "ldr q24, [x28, #0xf0]\n" "add x28, x28, #0x100\n" - ".inst 0x4f80e8d1 // sdot v17.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8d5 // sdot v21.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f80e8f2 // sdot v18.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8f6 // sdot v22.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f80e913 // sdot v19.4s, v8.16b, v0.4b[2]\n" - ".inst 0x4f81e917 // sdot v23.4s, v8.16b, v1.4b[2]\n" - ".inst 0x4fa0e930 // sdot v16.4s, v9.16b, v0.4b[3]\n" - ".inst 0x4fa1e934 // sdot v20.4s, v9.16b, v1.4b[3]\n" - ".inst 0x4fa0e951 // sdot v17.4s, v10.16b, v0.4b[3]\n" - ".inst 0x4fa1e955 // sdot v21.4s, v10.16b, v1.4b[3]\n" - ".inst 0x4fa0e892 // sdot v18.4s, v4.16b, v0.4b[3]\n" - ".inst 0x4fa1e896 // sdot v22.4s, v4.16b, v1.4b[3]\n" - ".inst 0x4fa0e8b3 // sdot v19.4s, v5.16b, v0.4b[3]\n" - ".inst 0x4fa1e8b7 // sdot v23.4s, v5.16b, v1.4b[3]\n" + ".inst 0x4f80ebd1 // sdot v17.4s, v30.16b, v0.4b[2]\n" + ".inst 0x4f81ebd5 // sdot v21.4s, v30.16b, v1.4b[2]\n" + ".inst 0x4f80ebb2 // sdot v18.4s, v29.16b, v0.4b[2]\n" + ".inst 0x4f81ebb6 // sdot v22.4s, v29.16b, v1.4b[2]\n" + ".inst 0x4f80eb93 // sdot v19.4s, v28.16b, v0.4b[2]\n" + ".inst 0x4f81eb97 // sdot v23.4s, v28.16b, v1.4b[2]\n" + ".inst 0x4fa0eb70 // sdot v16.4s, v27.16b, v0.4b[3]\n" + ".inst 0x4fa1eb74 // sdot v20.4s, v27.16b, v1.4b[3]\n" + ".inst 0x4fa0eb51 // sdot v17.4s, v26.16b, v0.4b[3]\n" + ".inst 0x4fa1eb55 // sdot v21.4s, v26.16b, v1.4b[3]\n" + ".inst 0x4fa0eb32 // sdot v18.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa1eb36 // sdot v22.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa0eb13 // sdot v19.4s, v24.16b, v0.4b[3]\n" + ".inst 0x4fa1eb17 // sdot v23.4s, v24.16b, v1.4b[3]\n" "tbnz %x[flags], #31, 40f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n" @@ -551,21 +550,21 @@ void a64_hybrid_s8qa_dot_4x16 ( ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n" "43:" // Height 2: Multiply loop: unique 7: skip row sum - "ldr q6, [x28, #0x0]\n" - "ldr q7, [x28, #0x10]\n" + "ldr q27, [x28, #0x0]\n" + "ldr q26, [x28, #0x10]\n" "sub x25, x25, #0x4\n" "cmp x25, #0x4\n" - "ldr q8, [x28, #0x20]\n" - "ldr q9, [x28, #0x30]\n" - ".inst 0x4f80e0d0 // sdot v16.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0d4 // sdot v20.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f80e0f1 // sdot v17.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0f5 // sdot v21.4s, v7.16b, v1.4b[0]\n" + "ldr q25, [x28, #0x20]\n" + "ldr q24, [x28, #0x30]\n" + ".inst 0x4f80e370 // sdot v16.4s, v27.16b, v0.4b[0]\n" + ".inst 0x4f81e374 // sdot v20.4s, v27.16b, v1.4b[0]\n" + ".inst 0x4f80e351 // sdot v17.4s, v26.16b, v0.4b[0]\n" + ".inst 0x4f81e355 // sdot v21.4s, v26.16b, v1.4b[0]\n" "add x28, x28, #0x40\n" - ".inst 0x4f80e112 // sdot v18.4s, v8.16b, v0.4b[0]\n" - ".inst 0x4f81e116 // sdot v22.4s, v8.16b, v1.4b[0]\n" - ".inst 0x4f80e133 // sdot v19.4s, v9.16b, v0.4b[0]\n" - ".inst 0x4f81e137 // sdot v23.4s, v9.16b, v1.4b[0]\n" + ".inst 0x4f80e332 // sdot v18.4s, v25.16b, v0.4b[0]\n" + ".inst 0x4f81e336 // sdot v22.4s, v25.16b, v1.4b[0]\n" + ".inst 0x4f80e313 // sdot v19.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f81e317 // sdot v23.4s, v24.16b, v1.4b[0]\n" "bge 42b\n" "44:" // Height 2: Multiply loop: Skip odd blocks "cbz x25, 48f\n" @@ -584,209 +583,209 @@ void a64_hybrid_s8qa_dot_4x16 ( ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n" "47:" // Height 2: Multiply loop: unique 8: skip row sum - "ldr q10, [x28, #0x0]\n" - "ldr q4, [x28, #0x10]\n" - ".inst 0x4f80e150 // sdot v16.4s, v10.16b, v0.4b[0]\n" - ".inst 0x4f81e154 // sdot v20.4s, v10.16b, v1.4b[0]\n" - "ldr q5, [x28, #0x20]\n" - "ldr q6, [x28, #0x30]\n" - ".inst 0x4f80e091 // sdot v17.4s, v4.16b, v0.4b[0]\n" - ".inst 0x4f81e095 // sdot v21.4s, v4.16b, v1.4b[0]\n" - ".inst 0x4f80e0b2 // sdot v18.4s, v5.16b, v0.4b[0]\n" - ".inst 0x4f81e0b6 // sdot v22.4s, v5.16b, v1.4b[0]\n" + "ldr q24, [x28, #0x0]\n" + "ldr q26, [x28, #0x10]\n" + ".inst 0x4f80e310 // sdot v16.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f81e314 // sdot v20.4s, v24.16b, v1.4b[0]\n" + "ldr q25, [x28, #0x20]\n" + "ldr q24, [x28, #0x30]\n" + ".inst 0x4f80e351 // sdot v17.4s, v26.16b, v0.4b[0]\n" + ".inst 0x4f81e355 // sdot v21.4s, v26.16b, v1.4b[0]\n" + ".inst 0x4f80e332 // sdot v18.4s, v25.16b, v0.4b[0]\n" + ".inst 0x4f81e336 // sdot v22.4s, v25.16b, v1.4b[0]\n" "add x28, x28, #0x40\n" - ".inst 0x4f80e0d3 // sdot v19.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0d7 // sdot v23.4s, v6.16b, v1.4b[0]\n" + ".inst 0x4f80e313 // sdot v19.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f81e317 // sdot v23.4s, v24.16b, v1.4b[0]\n" "48:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x26, x26, #0x1\n" "cmp x26, x20\n" "bne 34b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x22, x27, x20\n" + "add x23, x27, x20\n" "prfm pstl1keep, [x27, #0x0]\n" - "prfm pstl1keep, [x22, #0x0]\n" + "prfm pstl1keep, [x23, #0x0]\n" "tbnz %x[flags], #31, 49f\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1r { v2.4s }, [x23]\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1r { v24.4s }, [x20]\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v12.4s, v12.4s, v12.4s\n" - "neg v2.4s, v2.4s\n" + "neg v24.4s, v24.4s\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v12.4s, v12.4s, v12.4s\n" - "mul v11.4s, v11.4s, v2.4s\n" - "mul v12.4s, v12.4s, v2.4s\n" + "mul v11.4s, v11.4s, v24.4s\n" + "mul v12.4s, v12.4s, v24.4s\n" "49:" // Height 2: skip row sum fixup - "ldr q0, [x10, #0x0]\n" - "ldr q1, [x10, #0x10]\n" + "ldr q28, [x10, #0x0]\n" + "ldr q27, [x10, #0x10]\n" "add v16.4s, v16.4s, v11.4s\n" "add v17.4s, v17.4s, v11.4s\n" - "ldr q2, [x10, #0x20]\n" - "ldr q3, [x10, #0x30]\n" + "ldr q26, [x10, #0x20]\n" + "ldr q25, [x10, #0x30]\n" "add v18.4s, v18.4s, v11.4s\n" "add v19.4s, v19.4s, v11.4s\n" "add v20.4s, v20.4s, v12.4s\n" "add v21.4s, v21.4s, v12.4s\n" - "add x23, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v24.4s }, [x20]\n" "add v22.4s, v22.4s, v12.4s\n" "add v23.4s, v23.4s, v12.4s\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" - "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "add v16.4s, v16.4s, v28.4s\n" + "add v17.4s, v17.4s, v27.4s\n" "add x10, x10, #0x40\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" - "add v20.4s, v20.4s, v0.4s\n" - "ld1r { v0.4s }, [x23]\n" - "add v21.4s, v21.4s, v1.4s\n" - "add v22.4s, v22.4s, v2.4s\n" - "add v23.4s, v23.4s, v3.4s\n" - "sqrdmulh v16.4s, v16.4s, v4.4s\n" - "sqrdmulh v17.4s, v17.4s, v4.4s\n" - "sqrdmulh v18.4s, v18.4s, v4.4s\n" - "sqrdmulh v19.4s, v19.4s, v4.4s\n" - "sqrdmulh v20.4s, v20.4s, v4.4s\n" - "sqrdmulh v21.4s, v21.4s, v4.4s\n" - "sqrdmulh v22.4s, v22.4s, v4.4s\n" - "sqrdmulh v23.4s, v23.4s, v4.4s\n" + "add v18.4s, v18.4s, v26.4s\n" + "add v19.4s, v19.4s, v25.4s\n" + "add v20.4s, v20.4s, v28.4s\n" + "ld1r { v0.4s }, [x20]\n" + "add v21.4s, v21.4s, v27.4s\n" + "add v22.4s, v22.4s, v26.4s\n" + "add v23.4s, v23.4s, v25.4s\n" + "sqrdmulh v16.4s, v16.4s, v24.4s\n" + "sqrdmulh v17.4s, v17.4s, v24.4s\n" + "sqrdmulh v18.4s, v18.4s, v24.4s\n" + "sqrdmulh v19.4s, v19.4s, v24.4s\n" + "sqrdmulh v20.4s, v20.4s, v24.4s\n" + "sqrdmulh v21.4s, v21.4s, v24.4s\n" + "sqrdmulh v22.4s, v22.4s, v24.4s\n" + "sqrdmulh v23.4s, v23.4s, v24.4s\n" "tbz %x[flags], #5, 50f\n" - "and v4.16b, v16.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "and v5.16b, v17.16b, v0.16b\n" - "and v6.16b, v18.16b, v0.16b\n" - "and v7.16b, v19.16b, v0.16b\n" - "and v8.16b, v20.16b, v0.16b\n" - "and v9.16b, v21.16b, v0.16b\n" - "and v10.16b, v22.16b, v0.16b\n" - "and v4.16b, v23.16b, v0.16b\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v9.4s, v9.4s, #0x1f\n" - "sshr v10.4s, v10.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" - "sqadd v20.4s, v20.4s, v8.4s\n" - "sqadd v21.4s, v21.4s, v9.4s\n" - "sqadd v22.4s, v22.4s, v10.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" + "and v24.16b, v16.16b, v0.16b\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v24.4s\n" + "and v30.16b, v17.16b, v0.16b\n" + "and v29.16b, v18.16b, v0.16b\n" + "and v28.16b, v19.16b, v0.16b\n" + "and v27.16b, v20.16b, v0.16b\n" + "and v26.16b, v21.16b, v0.16b\n" + "and v25.16b, v22.16b, v0.16b\n" + "and v24.16b, v23.16b, v0.16b\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sshr v27.4s, v27.4s, #0x1f\n" + "sshr v26.4s, v26.4s, #0x1f\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sqadd v17.4s, v17.4s, v30.4s\n" + "sqadd v18.4s, v18.4s, v29.4s\n" + "sqadd v19.4s, v19.4s, v28.4s\n" + "sqadd v20.4s, v20.4s, v27.4s\n" + "sqadd v21.4s, v21.4s, v26.4s\n" + "sqadd v22.4s, v22.4s, v25.4s\n" + "sqadd v23.4s, v23.4s, v24.4s\n" "50:" // Height 2: no shift correction - "add x23, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v26.4s }, [x20]\n" "srshl v16.4s, v16.4s, v0.4s\n" "srshl v17.4s, v17.4s, v0.4s\n" "srshl v18.4s, v18.4s, v0.4s\n" "srshl v19.4s, v19.4s, v0.4s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x23]\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v25.4s }, [x20]\n" "srshl v20.4s, v20.4s, v0.4s\n" "srshl v21.4s, v21.4s, v0.4s\n" - "add x23, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x23]\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v24.4s }, [x20]\n" "srshl v22.4s, v22.4s, v0.4s\n" "srshl v23.4s, v23.4s, v0.4s\n" "cmp x9, #0x10\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v23.4s, v23.4s, v4.4s\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" + "add v16.4s, v16.4s, v26.4s\n" + "add v17.4s, v17.4s, v26.4s\n" + "add v18.4s, v18.4s, v26.4s\n" + "add v19.4s, v19.4s, v26.4s\n" + "add v20.4s, v20.4s, v26.4s\n" + "add v21.4s, v21.4s, v26.4s\n" + "add v22.4s, v22.4s, v26.4s\n" + "add v23.4s, v23.4s, v26.4s\n" + "smin v16.4s, v16.4s, v25.4s\n" + "smin v17.4s, v17.4s, v25.4s\n" + "smin v18.4s, v18.4s, v25.4s\n" + "smin v19.4s, v19.4s, v25.4s\n" + "smin v20.4s, v20.4s, v25.4s\n" + "smin v21.4s, v21.4s, v25.4s\n" + "smin v22.4s, v22.4s, v25.4s\n" + "smin v23.4s, v23.4s, v25.4s\n" + "smax v16.4s, v16.4s, v24.4s\n" + "smax v17.4s, v17.4s, v24.4s\n" + "smax v18.4s, v18.4s, v24.4s\n" + "smax v19.4s, v19.4s, v24.4s\n" + "smax v20.4s, v20.4s, v24.4s\n" + "smax v21.4s, v21.4s, v24.4s\n" + "smax v22.4s, v22.4s, v24.4s\n" + "smax v23.4s, v23.4s, v24.4s\n" "uzp1 v16.8h, v16.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v18.8h, v18.8h, v19.8h\n" "uzp1 v20.8h, v20.8h, v21.8h\n" - "uzp1 v21.8h, v22.8h, v23.8h\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" - "uzp1 v20.16b, v20.16b, v21.16b\n" + "uzp1 v17.8h, v22.8h, v23.8h\n" + "uzp1 v16.16b, v16.16b, v18.16b\n" + "uzp1 v20.16b, v20.16b, v17.16b\n" "bge 59f\n" "tbz x9, #3, 54f\n" "str d16, [x27], #0x8\n" - "str d20, [x22], #0x8\n" + "str d20, [x23], #0x8\n" "tbz x9, #2, 52f\n" "st1 { v16.s }[2], [x27], #0x4\n" - "st1 { v20.s }[2], [x22], #0x4\n" + "st1 { v20.s }[2], [x23], #0x4\n" "tbz x9, #1, 51f\n" "st1 { v16.h }[6], [x27], #0x2\n" - "st1 { v20.h }[6], [x22], #0x2\n" + "st1 { v20.h }[6], [x23], #0x2\n" "tbz x9, #0, 58f\n" "st1 { v16.b }[14], [x27]\n" - "st1 { v20.b }[14], [x22]\n" + "st1 { v20.b }[14], [x23]\n" "b 58f\n" "51:" // Height 2: Partial direct writeback: partial_1_12 "tbz x9, #0, 58f\n" "st1 { v16.b }[12], [x27]\n" - "st1 { v20.b }[12], [x22]\n" + "st1 { v20.b }[12], [x23]\n" "b 58f\n" "52:" // Height 2: Partial direct writeback: partial_2_8 "tbz x9, #1, 53f\n" "st1 { v16.h }[4], [x27], #0x2\n" - "st1 { v20.h }[4], [x22], #0x2\n" + "st1 { v20.h }[4], [x23], #0x2\n" "tbz x9, #0, 58f\n" "st1 { v16.b }[10], [x27]\n" - "st1 { v20.b }[10], [x22]\n" + "st1 { v20.b }[10], [x23]\n" "b 58f\n" "53:" // Height 2: Partial direct writeback: partial_1_8 "tbz x9, #0, 58f\n" "st1 { v16.b }[8], [x27]\n" - "st1 { v20.b }[8], [x22]\n" + "st1 { v20.b }[8], [x23]\n" "b 58f\n" "54:" // Height 2: Partial direct writeback: partial_4_0 "tbz x9, #2, 56f\n" "str s16, [x27], #0x4\n" - "str s20, [x22], #0x4\n" + "str s20, [x23], #0x4\n" "tbz x9, #1, 55f\n" "st1 { v16.h }[2], [x27], #0x2\n" - "st1 { v20.h }[2], [x22], #0x2\n" + "st1 { v20.h }[2], [x23], #0x2\n" "tbz x9, #0, 58f\n" "st1 { v16.b }[6], [x27]\n" - "st1 { v20.b }[6], [x22]\n" + "st1 { v20.b }[6], [x23]\n" "b 58f\n" "55:" // Height 2: Partial direct writeback: partial_1_4 "tbz x9, #0, 58f\n" "st1 { v16.b }[4], [x27]\n" - "st1 { v20.b }[4], [x22]\n" + "st1 { v20.b }[4], [x23]\n" "b 58f\n" "56:" // Height 2: Partial direct writeback: partial_2_0 "tbz x9, #1, 57f\n" "str h16, [x27], #0x2\n" - "str h20, [x22], #0x2\n" + "str h20, [x23], #0x2\n" "tbz x9, #0, 58f\n" "st1 { v16.b }[2], [x27]\n" - "st1 { v20.b }[2], [x22]\n" + "st1 { v20.b }[2], [x23]\n" "b 58f\n" "57:" // Height 2: Partial direct writeback: partial_1_0 "str b16, [x27, #0x0]\n" - "str b20, [x22, #0x0]\n" + "str b20, [x23, #0x0]\n" "58:" // Height 2: Partial direct writeback: Done "b 60f\n" "59:" // Height 2: Full writeback "str q16, [x27, #0x0]\n" "add x27, x27, #0x10\n" - "str q20, [x22, #0x0]\n" + "str q20, [x23, #0x0]\n" "60:" // Height 2: Writeback done "subs x9, x9, #0x10\n" "bgt 32b\n" @@ -819,13 +818,13 @@ void a64_hybrid_s8qa_dot_4x16 ( "64:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 65f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" "cbnz x26, 66f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -834,8 +833,8 @@ void a64_hybrid_s8qa_dot_4x16 ( "b 66f\n" "65:" // Height 3: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" "66:" // Height 3: input setup done "cmp x25, #0x10\n" "blt 71f\n" @@ -857,62 +856,62 @@ void a64_hybrid_s8qa_dot_4x16 ( "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" ".inst 0x4f82e098 // sdot v24.4s, v4.16b, v2.4b[0]\n" - "ldr q4, [x28, #0x70]\n" + "ldr q29, [x28, #0x70]\n" ".inst 0x4f80e0b1 // sdot v17.4s, v5.16b, v0.4b[0]\n" "add x22, x22, #0x10\n" ".inst 0x4f81e0b5 // sdot v21.4s, v5.16b, v1.4b[0]\n" ".inst 0x4f82e0b9 // sdot v25.4s, v5.16b, v2.4b[0]\n" - "ldr q5, [x28, #0x80]\n" + "ldr q28, [x28, #0x80]\n" ".inst 0x4f80e0d2 // sdot v18.4s, v6.16b, v0.4b[0]\n" ".inst 0x4f81e0d6 // sdot v22.4s, v6.16b, v1.4b[0]\n" ".inst 0x4f82e0da // sdot v26.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x28, #0x90]\n" + "ldr q5, [x28, #0x90]\n" ".inst 0x4f80e0f3 // sdot v19.4s, v7.16b, v0.4b[0]\n" ".inst 0x4f81e0f7 // sdot v23.4s, v7.16b, v1.4b[0]\n" ".inst 0x4f82e0fb // sdot v27.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x28, #0xa0]\n" + "ldr q4, [x28, #0xa0]\n" ".inst 0x4fa0e110 // sdot v16.4s, v8.16b, v0.4b[1]\n" ".inst 0x4fa1e114 // sdot v20.4s, v8.16b, v1.4b[1]\n" ".inst 0x4fa2e118 // sdot v24.4s, v8.16b, v2.4b[1]\n" - "ldr q8, [x28, #0xb0]\n" + "ldr q3, [x28, #0xb0]\n" ".inst 0x4fa0e131 // sdot v17.4s, v9.16b, v0.4b[1]\n" ".inst 0x4fa1e135 // sdot v21.4s, v9.16b, v1.4b[1]\n" ".inst 0x4fa2e139 // sdot v25.4s, v9.16b, v2.4b[1]\n" - "ldr q9, [x28, #0xc0]\n" + "ldr q31, [x28, #0xc0]\n" ".inst 0x4fa0e152 // sdot v18.4s, v10.16b, v0.4b[1]\n" ".inst 0x4fa1e156 // sdot v22.4s, v10.16b, v1.4b[1]\n" ".inst 0x4fa2e15a // sdot v26.4s, v10.16b, v2.4b[1]\n" - "ldr q10, [x28, #0xd0]\n" - ".inst 0x4fa0e093 // sdot v19.4s, v4.16b, v0.4b[1]\n" - ".inst 0x4fa1e097 // sdot v23.4s, v4.16b, v1.4b[1]\n" - ".inst 0x4fa2e09b // sdot v27.4s, v4.16b, v2.4b[1]\n" - "ldr q4, [x28, #0xe0]\n" - ".inst 0x4f80e8b0 // sdot v16.4s, v5.16b, v0.4b[2]\n" - ".inst 0x4f81e8b4 // sdot v20.4s, v5.16b, v1.4b[2]\n" - ".inst 0x4f82e8b8 // sdot v24.4s, v5.16b, v2.4b[2]\n" - "ldr q5, [x28, #0xf0]\n" - ".inst 0x4f80e8d1 // sdot v17.4s, v6.16b, v0.4b[2]\n" + "ldr q30, [x28, #0xd0]\n" + ".inst 0x4fa0e3b3 // sdot v19.4s, v29.16b, v0.4b[1]\n" + ".inst 0x4fa1e3b7 // sdot v23.4s, v29.16b, v1.4b[1]\n" + ".inst 0x4fa2e3bb // sdot v27.4s, v29.16b, v2.4b[1]\n" + "ldr q29, [x28, #0xe0]\n" + ".inst 0x4f80eb90 // sdot v16.4s, v28.16b, v0.4b[2]\n" + ".inst 0x4f81eb94 // sdot v20.4s, v28.16b, v1.4b[2]\n" + ".inst 0x4f82eb98 // sdot v24.4s, v28.16b, v2.4b[2]\n" + "ldr q28, [x28, #0xf0]\n" + ".inst 0x4f80e8b1 // sdot v17.4s, v5.16b, v0.4b[2]\n" "add x28, x28, #0x100\n" - ".inst 0x4f81e8d5 // sdot v21.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d9 // sdot v25.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f80e8f2 // sdot v18.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8f6 // sdot v22.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8fa // sdot v26.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f80e913 // sdot v19.4s, v8.16b, v0.4b[2]\n" - ".inst 0x4f81e917 // sdot v23.4s, v8.16b, v1.4b[2]\n" - ".inst 0x4f82e91b // sdot v27.4s, v8.16b, v2.4b[2]\n" - ".inst 0x4fa0e930 // sdot v16.4s, v9.16b, v0.4b[3]\n" - ".inst 0x4fa1e934 // sdot v20.4s, v9.16b, v1.4b[3]\n" - ".inst 0x4fa2e938 // sdot v24.4s, v9.16b, v2.4b[3]\n" - ".inst 0x4fa0e951 // sdot v17.4s, v10.16b, v0.4b[3]\n" - ".inst 0x4fa1e955 // sdot v21.4s, v10.16b, v1.4b[3]\n" - ".inst 0x4fa2e959 // sdot v25.4s, v10.16b, v2.4b[3]\n" - ".inst 0x4fa0e892 // sdot v18.4s, v4.16b, v0.4b[3]\n" - ".inst 0x4fa1e896 // sdot v22.4s, v4.16b, v1.4b[3]\n" - ".inst 0x4fa2e89a // sdot v26.4s, v4.16b, v2.4b[3]\n" - ".inst 0x4fa0e8b3 // sdot v19.4s, v5.16b, v0.4b[3]\n" - ".inst 0x4fa1e8b7 // sdot v23.4s, v5.16b, v1.4b[3]\n" - ".inst 0x4fa2e8bb // sdot v27.4s, v5.16b, v2.4b[3]\n" + ".inst 0x4f81e8b5 // sdot v21.4s, v5.16b, v1.4b[2]\n" + ".inst 0x4f82e8b9 // sdot v25.4s, v5.16b, v2.4b[2]\n" + ".inst 0x4f80e892 // sdot v18.4s, v4.16b, v0.4b[2]\n" + ".inst 0x4f81e896 // sdot v22.4s, v4.16b, v1.4b[2]\n" + ".inst 0x4f82e89a // sdot v26.4s, v4.16b, v2.4b[2]\n" + ".inst 0x4f80e873 // sdot v19.4s, v3.16b, v0.4b[2]\n" + ".inst 0x4f81e877 // sdot v23.4s, v3.16b, v1.4b[2]\n" + ".inst 0x4f82e87b // sdot v27.4s, v3.16b, v2.4b[2]\n" + ".inst 0x4fa0ebf0 // sdot v16.4s, v31.16b, v0.4b[3]\n" + ".inst 0x4fa1ebf4 // sdot v20.4s, v31.16b, v1.4b[3]\n" + ".inst 0x4fa2ebf8 // sdot v24.4s, v31.16b, v2.4b[3]\n" + ".inst 0x4fa0ebd1 // sdot v17.4s, v30.16b, v0.4b[3]\n" + ".inst 0x4fa1ebd5 // sdot v21.4s, v30.16b, v1.4b[3]\n" + ".inst 0x4fa2ebd9 // sdot v25.4s, v30.16b, v2.4b[3]\n" + ".inst 0x4fa0ebb2 // sdot v18.4s, v29.16b, v0.4b[3]\n" + ".inst 0x4fa1ebb6 // sdot v22.4s, v29.16b, v1.4b[3]\n" + ".inst 0x4fa2ebba // sdot v26.4s, v29.16b, v2.4b[3]\n" + ".inst 0x4fa0eb93 // sdot v19.4s, v28.16b, v0.4b[3]\n" + ".inst 0x4fa1eb97 // sdot v23.4s, v28.16b, v1.4b[3]\n" + ".inst 0x4fa2eb9b // sdot v27.4s, v28.16b, v2.4b[3]\n" "tbnz %x[flags], #31, 68f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n" @@ -940,63 +939,63 @@ void a64_hybrid_s8qa_dot_4x16 ( "sub x25, x25, #0x10\n" "add x24, x24, #0x10\n" ".inst 0x4f82e098 // sdot v24.4s, v4.16b, v2.4b[0]\n" - "ldr q4, [x28, #0x70]\n" + "ldr q29, [x28, #0x70]\n" ".inst 0x4f80e0b1 // sdot v17.4s, v5.16b, v0.4b[0]\n" "add x23, x23, #0x10\n" ".inst 0x4f81e0b5 // sdot v21.4s, v5.16b, v1.4b[0]\n" ".inst 0x4f82e0b9 // sdot v25.4s, v5.16b, v2.4b[0]\n" - "ldr q5, [x28, #0x80]\n" + "ldr q28, [x28, #0x80]\n" "add x22, x22, #0x10\n" ".inst 0x4f80e0d2 // sdot v18.4s, v6.16b, v0.4b[0]\n" ".inst 0x4f81e0d6 // sdot v22.4s, v6.16b, v1.4b[0]\n" ".inst 0x4f82e0da // sdot v26.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x28, #0x90]\n" + "ldr q5, [x28, #0x90]\n" ".inst 0x4f80e0f3 // sdot v19.4s, v7.16b, v0.4b[0]\n" ".inst 0x4f81e0f7 // sdot v23.4s, v7.16b, v1.4b[0]\n" ".inst 0x4f82e0fb // sdot v27.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x28, #0xa0]\n" + "ldr q4, [x28, #0xa0]\n" ".inst 0x4fa0e110 // sdot v16.4s, v8.16b, v0.4b[1]\n" ".inst 0x4fa1e114 // sdot v20.4s, v8.16b, v1.4b[1]\n" ".inst 0x4fa2e118 // sdot v24.4s, v8.16b, v2.4b[1]\n" - "ldr q8, [x28, #0xb0]\n" + "ldr q3, [x28, #0xb0]\n" ".inst 0x4fa0e131 // sdot v17.4s, v9.16b, v0.4b[1]\n" ".inst 0x4fa1e135 // sdot v21.4s, v9.16b, v1.4b[1]\n" ".inst 0x4fa2e139 // sdot v25.4s, v9.16b, v2.4b[1]\n" - "ldr q9, [x28, #0xc0]\n" + "ldr q31, [x28, #0xc0]\n" ".inst 0x4fa0e152 // sdot v18.4s, v10.16b, v0.4b[1]\n" ".inst 0x4fa1e156 // sdot v22.4s, v10.16b, v1.4b[1]\n" ".inst 0x4fa2e15a // sdot v26.4s, v10.16b, v2.4b[1]\n" - "ldr q10, [x28, #0xd0]\n" - ".inst 0x4fa0e093 // sdot v19.4s, v4.16b, v0.4b[1]\n" - ".inst 0x4fa1e097 // sdot v23.4s, v4.16b, v1.4b[1]\n" - ".inst 0x4fa2e09b // sdot v27.4s, v4.16b, v2.4b[1]\n" - "ldr q4, [x28, #0xe0]\n" - ".inst 0x4f80e8b0 // sdot v16.4s, v5.16b, v0.4b[2]\n" - ".inst 0x4f81e8b4 // sdot v20.4s, v5.16b, v1.4b[2]\n" - ".inst 0x4f82e8b8 // sdot v24.4s, v5.16b, v2.4b[2]\n" - "ldr q5, [x28, #0xf0]\n" - ".inst 0x4f80e8d1 // sdot v17.4s, v6.16b, v0.4b[2]\n" + "ldr q30, [x28, #0xd0]\n" + ".inst 0x4fa0e3b3 // sdot v19.4s, v29.16b, v0.4b[1]\n" + ".inst 0x4fa1e3b7 // sdot v23.4s, v29.16b, v1.4b[1]\n" + ".inst 0x4fa2e3bb // sdot v27.4s, v29.16b, v2.4b[1]\n" + "ldr q29, [x28, #0xe0]\n" + ".inst 0x4f80eb90 // sdot v16.4s, v28.16b, v0.4b[2]\n" + ".inst 0x4f81eb94 // sdot v20.4s, v28.16b, v1.4b[2]\n" + ".inst 0x4f82eb98 // sdot v24.4s, v28.16b, v2.4b[2]\n" + "ldr q28, [x28, #0xf0]\n" + ".inst 0x4f80e8b1 // sdot v17.4s, v5.16b, v0.4b[2]\n" "add x28, x28, #0x100\n" - ".inst 0x4f81e8d5 // sdot v21.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d9 // sdot v25.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f80e8f2 // sdot v18.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8f6 // sdot v22.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8fa // sdot v26.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f80e913 // sdot v19.4s, v8.16b, v0.4b[2]\n" - ".inst 0x4f81e917 // sdot v23.4s, v8.16b, v1.4b[2]\n" - ".inst 0x4f82e91b // sdot v27.4s, v8.16b, v2.4b[2]\n" - ".inst 0x4fa0e930 // sdot v16.4s, v9.16b, v0.4b[3]\n" - ".inst 0x4fa1e934 // sdot v20.4s, v9.16b, v1.4b[3]\n" - ".inst 0x4fa2e938 // sdot v24.4s, v9.16b, v2.4b[3]\n" - ".inst 0x4fa0e951 // sdot v17.4s, v10.16b, v0.4b[3]\n" - ".inst 0x4fa1e955 // sdot v21.4s, v10.16b, v1.4b[3]\n" - ".inst 0x4fa2e959 // sdot v25.4s, v10.16b, v2.4b[3]\n" - ".inst 0x4fa0e892 // sdot v18.4s, v4.16b, v0.4b[3]\n" - ".inst 0x4fa1e896 // sdot v22.4s, v4.16b, v1.4b[3]\n" - ".inst 0x4fa2e89a // sdot v26.4s, v4.16b, v2.4b[3]\n" - ".inst 0x4fa0e8b3 // sdot v19.4s, v5.16b, v0.4b[3]\n" - ".inst 0x4fa1e8b7 // sdot v23.4s, v5.16b, v1.4b[3]\n" - ".inst 0x4fa2e8bb // sdot v27.4s, v5.16b, v2.4b[3]\n" + ".inst 0x4f81e8b5 // sdot v21.4s, v5.16b, v1.4b[2]\n" + ".inst 0x4f82e8b9 // sdot v25.4s, v5.16b, v2.4b[2]\n" + ".inst 0x4f80e892 // sdot v18.4s, v4.16b, v0.4b[2]\n" + ".inst 0x4f81e896 // sdot v22.4s, v4.16b, v1.4b[2]\n" + ".inst 0x4f82e89a // sdot v26.4s, v4.16b, v2.4b[2]\n" + ".inst 0x4f80e873 // sdot v19.4s, v3.16b, v0.4b[2]\n" + ".inst 0x4f81e877 // sdot v23.4s, v3.16b, v1.4b[2]\n" + ".inst 0x4f82e87b // sdot v27.4s, v3.16b, v2.4b[2]\n" + ".inst 0x4fa0ebf0 // sdot v16.4s, v31.16b, v0.4b[3]\n" + ".inst 0x4fa1ebf4 // sdot v20.4s, v31.16b, v1.4b[3]\n" + ".inst 0x4fa2ebf8 // sdot v24.4s, v31.16b, v2.4b[3]\n" + ".inst 0x4fa0ebd1 // sdot v17.4s, v30.16b, v0.4b[3]\n" + ".inst 0x4fa1ebd5 // sdot v21.4s, v30.16b, v1.4b[3]\n" + ".inst 0x4fa2ebd9 // sdot v25.4s, v30.16b, v2.4b[3]\n" + ".inst 0x4fa0ebb2 // sdot v18.4s, v29.16b, v0.4b[3]\n" + ".inst 0x4fa1ebb6 // sdot v22.4s, v29.16b, v1.4b[3]\n" + ".inst 0x4fa2ebba // sdot v26.4s, v29.16b, v2.4b[3]\n" + ".inst 0x4fa0eb93 // sdot v19.4s, v28.16b, v0.4b[3]\n" + ".inst 0x4fa1eb97 // sdot v23.4s, v28.16b, v1.4b[3]\n" + ".inst 0x4fa2eb9b // sdot v27.4s, v28.16b, v2.4b[3]\n" "tbnz %x[flags], #31, 70f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n" @@ -1018,25 +1017,25 @@ void a64_hybrid_s8qa_dot_4x16 ( ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n" ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n" "73:" // Height 3: Multiply loop: unique 11: skip row sum - "ldr q6, [x28, #0x0]\n" - "ldr q7, [x28, #0x10]\n" + "ldr q31, [x28, #0x0]\n" + "ldr q30, [x28, #0x10]\n" "sub x25, x25, #0x4\n" "cmp x25, #0x4\n" - "ldr q8, [x28, #0x20]\n" - "ldr q9, [x28, #0x30]\n" - ".inst 0x4f80e0d0 // sdot v16.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0d4 // sdot v20.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d8 // sdot v24.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f80e0f1 // sdot v17.4s, v7.16b, v0.4b[0]\n" + "ldr q29, [x28, #0x20]\n" + "ldr q28, [x28, #0x30]\n" + ".inst 0x4f80e3f0 // sdot v16.4s, v31.16b, v0.4b[0]\n" + ".inst 0x4f81e3f4 // sdot v20.4s, v31.16b, v1.4b[0]\n" + ".inst 0x4f82e3f8 // sdot v24.4s, v31.16b, v2.4b[0]\n" + ".inst 0x4f80e3d1 // sdot v17.4s, v30.16b, v0.4b[0]\n" "add x28, x28, #0x40\n" - ".inst 0x4f81e0f5 // sdot v21.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f9 // sdot v25.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f80e112 // sdot v18.4s, v8.16b, v0.4b[0]\n" - ".inst 0x4f81e116 // sdot v22.4s, v8.16b, v1.4b[0]\n" - ".inst 0x4f82e11a // sdot v26.4s, v8.16b, v2.4b[0]\n" - ".inst 0x4f80e133 // sdot v19.4s, v9.16b, v0.4b[0]\n" - ".inst 0x4f81e137 // sdot v23.4s, v9.16b, v1.4b[0]\n" - ".inst 0x4f82e13b // sdot v27.4s, v9.16b, v2.4b[0]\n" + ".inst 0x4f81e3d5 // sdot v21.4s, v30.16b, v1.4b[0]\n" + ".inst 0x4f82e3d9 // sdot v25.4s, v30.16b, v2.4b[0]\n" + ".inst 0x4f80e3b2 // sdot v18.4s, v29.16b, v0.4b[0]\n" + ".inst 0x4f81e3b6 // sdot v22.4s, v29.16b, v1.4b[0]\n" + ".inst 0x4f82e3ba // sdot v26.4s, v29.16b, v2.4b[0]\n" + ".inst 0x4f80e393 // sdot v19.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f81e397 // sdot v23.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f82e39b // sdot v27.4s, v28.16b, v2.4b[0]\n" "bge 72b\n" "74:" // Height 3: Multiply loop: Skip odd blocks "cbz x25, 78f\n" @@ -1059,144 +1058,144 @@ void a64_hybrid_s8qa_dot_4x16 ( ".inst 0x4e8f942c // sdot v12.4s, v1.16b, v15.16b\n" ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n" "77:" // Height 3: Multiply loop: unique 12: skip row sum - "ldr q10, [x28, #0x0]\n" - "ldr q4, [x28, #0x10]\n" - ".inst 0x4f80e150 // sdot v16.4s, v10.16b, v0.4b[0]\n" - ".inst 0x4f81e154 // sdot v20.4s, v10.16b, v1.4b[0]\n" - "ldr q5, [x28, #0x20]\n" - "ldr q6, [x28, #0x30]\n" - ".inst 0x4f82e158 // sdot v24.4s, v10.16b, v2.4b[0]\n" - ".inst 0x4f80e091 // sdot v17.4s, v4.16b, v0.4b[0]\n" - ".inst 0x4f81e095 // sdot v21.4s, v4.16b, v1.4b[0]\n" - ".inst 0x4f82e099 // sdot v25.4s, v4.16b, v2.4b[0]\n" + "ldr q31, [x28, #0x0]\n" + "ldr q30, [x28, #0x10]\n" + ".inst 0x4f80e3f0 // sdot v16.4s, v31.16b, v0.4b[0]\n" + ".inst 0x4f81e3f4 // sdot v20.4s, v31.16b, v1.4b[0]\n" + "ldr q29, [x28, #0x20]\n" + "ldr q28, [x28, #0x30]\n" + ".inst 0x4f82e3f8 // sdot v24.4s, v31.16b, v2.4b[0]\n" + ".inst 0x4f80e3d1 // sdot v17.4s, v30.16b, v0.4b[0]\n" + ".inst 0x4f81e3d5 // sdot v21.4s, v30.16b, v1.4b[0]\n" + ".inst 0x4f82e3d9 // sdot v25.4s, v30.16b, v2.4b[0]\n" "add x28, x28, #0x40\n" - ".inst 0x4f80e0b2 // sdot v18.4s, v5.16b, v0.4b[0]\n" - ".inst 0x4f81e0b6 // sdot v22.4s, v5.16b, v1.4b[0]\n" - ".inst 0x4f82e0ba // sdot v26.4s, v5.16b, v2.4b[0]\n" - ".inst 0x4f80e0d3 // sdot v19.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0d7 // sdot v23.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0db // sdot v27.4s, v6.16b, v2.4b[0]\n" + ".inst 0x4f80e3b2 // sdot v18.4s, v29.16b, v0.4b[0]\n" + ".inst 0x4f81e3b6 // sdot v22.4s, v29.16b, v1.4b[0]\n" + ".inst 0x4f82e3ba // sdot v26.4s, v29.16b, v2.4b[0]\n" + ".inst 0x4f80e393 // sdot v19.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f81e397 // sdot v23.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f82e39b // sdot v27.4s, v28.16b, v2.4b[0]\n" "78:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x26, x26, #0x1\n" "cmp x26, x20\n" "bne 64b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x22, x27, x20\n" - "add x21, x22, x20\n" + "add x23, x27, x20\n" + "add x22, x23, x20\n" "prfm pstl1keep, [x27, #0x0]\n" + "prfm pstl1keep, [x23, #0x0]\n" "prfm pstl1keep, [x22, #0x0]\n" - "prfm pstl1keep, [x21, #0x0]\n" "tbnz %x[flags], #31, 79f\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1r { v3.4s }, [x23]\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1r { v28.4s }, [x20]\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v12.4s, v12.4s, v12.4s\n" "addp v13.4s, v13.4s, v13.4s\n" - "neg v3.4s, v3.4s\n" + "neg v28.4s, v28.4s\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v12.4s, v12.4s, v12.4s\n" "addp v13.4s, v13.4s, v13.4s\n" - "mul v11.4s, v11.4s, v3.4s\n" - "mul v12.4s, v12.4s, v3.4s\n" - "mul v13.4s, v13.4s, v3.4s\n" + "mul v11.4s, v11.4s, v28.4s\n" + "mul v12.4s, v12.4s, v28.4s\n" + "mul v13.4s, v13.4s, v28.4s\n" "79:" // Height 3: skip row sum fixup "ldr q0, [x10, #0x0]\n" - "ldr q1, [x10, #0x10]\n" + "ldr q31, [x10, #0x10]\n" "add v16.4s, v16.4s, v11.4s\n" "add v17.4s, v17.4s, v11.4s\n" - "ldr q2, [x10, #0x20]\n" - "ldr q3, [x10, #0x30]\n" + "ldr q30, [x10, #0x20]\n" + "ldr q29, [x10, #0x30]\n" "add v18.4s, v18.4s, v11.4s\n" "add v19.4s, v19.4s, v11.4s\n" "add v20.4s, v20.4s, v12.4s\n" "add v21.4s, v21.4s, v12.4s\n" - "add x23, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v28.4s }, [x20]\n" "add v22.4s, v22.4s, v12.4s\n" "add v23.4s, v23.4s, v12.4s\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" "add v24.4s, v24.4s, v13.4s\n" "add v25.4s, v25.4s, v13.4s\n" "add x10, x10, #0x40\n" "add v26.4s, v26.4s, v13.4s\n" "add v27.4s, v27.4s, v13.4s\n" "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" + "add v17.4s, v17.4s, v31.4s\n" + "add v18.4s, v18.4s, v30.4s\n" + "add v19.4s, v19.4s, v29.4s\n" "add v20.4s, v20.4s, v0.4s\n" - "add v21.4s, v21.4s, v1.4s\n" - "add v22.4s, v22.4s, v2.4s\n" - "add v23.4s, v23.4s, v3.4s\n" + "add v21.4s, v21.4s, v31.4s\n" + "add v22.4s, v22.4s, v30.4s\n" + "add v23.4s, v23.4s, v29.4s\n" "add v24.4s, v24.4s, v0.4s\n" - "ld1r { v0.4s }, [x23]\n" - "add v25.4s, v25.4s, v1.4s\n" - "add v26.4s, v26.4s, v2.4s\n" - "add v27.4s, v27.4s, v3.4s\n" - "sqrdmulh v16.4s, v16.4s, v4.4s\n" - "sqrdmulh v17.4s, v17.4s, v4.4s\n" - "sqrdmulh v18.4s, v18.4s, v4.4s\n" - "sqrdmulh v19.4s, v19.4s, v4.4s\n" - "sqrdmulh v20.4s, v20.4s, v4.4s\n" - "sqrdmulh v21.4s, v21.4s, v4.4s\n" - "sqrdmulh v22.4s, v22.4s, v4.4s\n" - "sqrdmulh v23.4s, v23.4s, v4.4s\n" - "sqrdmulh v24.4s, v24.4s, v4.4s\n" - "sqrdmulh v25.4s, v25.4s, v4.4s\n" - "sqrdmulh v26.4s, v26.4s, v4.4s\n" - "sqrdmulh v27.4s, v27.4s, v4.4s\n" + "ld1r { v0.4s }, [x20]\n" + "add v25.4s, v25.4s, v31.4s\n" + "add v26.4s, v26.4s, v30.4s\n" + "add v27.4s, v27.4s, v29.4s\n" + "sqrdmulh v16.4s, v16.4s, v28.4s\n" + "sqrdmulh v17.4s, v17.4s, v28.4s\n" + "sqrdmulh v18.4s, v18.4s, v28.4s\n" + "sqrdmulh v19.4s, v19.4s, v28.4s\n" + "sqrdmulh v20.4s, v20.4s, v28.4s\n" + "sqrdmulh v21.4s, v21.4s, v28.4s\n" + "sqrdmulh v22.4s, v22.4s, v28.4s\n" + "sqrdmulh v23.4s, v23.4s, v28.4s\n" + "sqrdmulh v24.4s, v24.4s, v28.4s\n" + "sqrdmulh v25.4s, v25.4s, v28.4s\n" + "sqrdmulh v26.4s, v26.4s, v28.4s\n" + "sqrdmulh v27.4s, v27.4s, v28.4s\n" "tbz %x[flags], #5, 80f\n" - "and v4.16b, v16.16b, v0.16b\n" - "and v5.16b, v17.16b, v0.16b\n" - "and v6.16b, v18.16b, v0.16b\n" - "and v7.16b, v19.16b, v0.16b\n" - "and v8.16b, v20.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" - "sqadd v20.4s, v20.4s, v8.4s\n" - "and v9.16b, v21.16b, v0.16b\n" - "and v10.16b, v22.16b, v0.16b\n" - "and v4.16b, v23.16b, v0.16b\n" - "and v5.16b, v24.16b, v0.16b\n" - "and v6.16b, v25.16b, v0.16b\n" - "and v7.16b, v26.16b, v0.16b\n" - "and v8.16b, v27.16b, v0.16b\n" - "sshr v9.4s, v9.4s, #0x1f\n" - "sshr v10.4s, v10.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sqadd v21.4s, v21.4s, v9.4s\n" - "sqadd v22.4s, v22.4s, v10.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v24.4s, v24.4s, v5.4s\n" - "sqadd v25.4s, v25.4s, v6.4s\n" - "sqadd v26.4s, v26.4s, v7.4s\n" - "sqadd v27.4s, v27.4s, v8.4s\n" + "and v1.16b, v16.16b, v0.16b\n" + "and v31.16b, v17.16b, v0.16b\n" + "and v30.16b, v18.16b, v0.16b\n" + "and v29.16b, v19.16b, v0.16b\n" + "and v28.16b, v20.16b, v0.16b\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v1.4s\n" + "sqadd v17.4s, v17.4s, v31.4s\n" + "sqadd v18.4s, v18.4s, v30.4s\n" + "sqadd v19.4s, v19.4s, v29.4s\n" + "sqadd v20.4s, v20.4s, v28.4s\n" + "and v3.16b, v21.16b, v0.16b\n" + "and v2.16b, v22.16b, v0.16b\n" + "and v1.16b, v23.16b, v0.16b\n" + "and v31.16b, v24.16b, v0.16b\n" + "and v30.16b, v25.16b, v0.16b\n" + "and v29.16b, v26.16b, v0.16b\n" + "and v28.16b, v27.16b, v0.16b\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sqadd v21.4s, v21.4s, v3.4s\n" + "sqadd v22.4s, v22.4s, v2.4s\n" + "sqadd v23.4s, v23.4s, v1.4s\n" + "sqadd v24.4s, v24.4s, v31.4s\n" + "sqadd v25.4s, v25.4s, v30.4s\n" + "sqadd v26.4s, v26.4s, v29.4s\n" + "sqadd v27.4s, v27.4s, v28.4s\n" "80:" // Height 3: no shift correction - "add x23, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v30.4s }, [x20]\n" "srshl v16.4s, v16.4s, v0.4s\n" "srshl v17.4s, v17.4s, v0.4s\n" "srshl v18.4s, v18.4s, v0.4s\n" "srshl v19.4s, v19.4s, v0.4s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x23]\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v29.4s }, [x20]\n" "srshl v20.4s, v20.4s, v0.4s\n" "srshl v21.4s, v21.4s, v0.4s\n" - "add x23, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x23]\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v28.4s }, [x20]\n" "srshl v22.4s, v22.4s, v0.4s\n" "srshl v23.4s, v23.4s, v0.4s\n" "cmp x9, #0x10\n" @@ -1204,132 +1203,132 @@ void a64_hybrid_s8qa_dot_4x16 ( "srshl v25.4s, v25.4s, v0.4s\n" "srshl v26.4s, v26.4s, v0.4s\n" "srshl v27.4s, v27.4s, v0.4s\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v23.4s, v23.4s, v4.4s\n" - "add v24.4s, v24.4s, v4.4s\n" - "add v25.4s, v25.4s, v4.4s\n" - "add v26.4s, v26.4s, v4.4s\n" - "add v27.4s, v27.4s, v4.4s\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "smin v24.4s, v24.4s, v6.4s\n" - "smin v25.4s, v25.4s, v6.4s\n" - "smin v26.4s, v26.4s, v6.4s\n" - "smin v27.4s, v27.4s, v6.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" - "smax v24.4s, v24.4s, v5.4s\n" - "smax v25.4s, v25.4s, v5.4s\n" - "smax v26.4s, v26.4s, v5.4s\n" - "smax v27.4s, v27.4s, v5.4s\n" + "add v16.4s, v16.4s, v30.4s\n" + "add v17.4s, v17.4s, v30.4s\n" + "add v18.4s, v18.4s, v30.4s\n" + "add v19.4s, v19.4s, v30.4s\n" + "add v20.4s, v20.4s, v30.4s\n" + "add v21.4s, v21.4s, v30.4s\n" + "add v22.4s, v22.4s, v30.4s\n" + "add v23.4s, v23.4s, v30.4s\n" + "add v24.4s, v24.4s, v30.4s\n" + "add v25.4s, v25.4s, v30.4s\n" + "add v26.4s, v26.4s, v30.4s\n" + "add v27.4s, v27.4s, v30.4s\n" + "smin v16.4s, v16.4s, v29.4s\n" + "smin v17.4s, v17.4s, v29.4s\n" + "smin v18.4s, v18.4s, v29.4s\n" + "smin v19.4s, v19.4s, v29.4s\n" + "smin v20.4s, v20.4s, v29.4s\n" + "smin v21.4s, v21.4s, v29.4s\n" + "smin v22.4s, v22.4s, v29.4s\n" + "smin v23.4s, v23.4s, v29.4s\n" + "smin v24.4s, v24.4s, v29.4s\n" + "smin v25.4s, v25.4s, v29.4s\n" + "smin v26.4s, v26.4s, v29.4s\n" + "smin v27.4s, v27.4s, v29.4s\n" + "smax v16.4s, v16.4s, v28.4s\n" + "smax v17.4s, v17.4s, v28.4s\n" + "smax v18.4s, v18.4s, v28.4s\n" + "smax v19.4s, v19.4s, v28.4s\n" + "smax v20.4s, v20.4s, v28.4s\n" + "smax v21.4s, v21.4s, v28.4s\n" + "smax v22.4s, v22.4s, v28.4s\n" + "smax v23.4s, v23.4s, v28.4s\n" + "smax v24.4s, v24.4s, v28.4s\n" + "smax v25.4s, v25.4s, v28.4s\n" + "smax v26.4s, v26.4s, v28.4s\n" + "smax v27.4s, v27.4s, v28.4s\n" "uzp1 v16.8h, v16.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v19.8h, v18.8h, v19.8h\n" "uzp1 v20.8h, v20.8h, v21.8h\n" - "uzp1 v21.8h, v22.8h, v23.8h\n" + "uzp1 v18.8h, v22.8h, v23.8h\n" "uzp1 v24.8h, v24.8h, v25.8h\n" - "uzp1 v25.8h, v26.8h, v27.8h\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" - "uzp1 v20.16b, v20.16b, v21.16b\n" - "uzp1 v24.16b, v24.16b, v25.16b\n" + "uzp1 v17.8h, v26.8h, v27.8h\n" + "uzp1 v16.16b, v16.16b, v19.16b\n" + "uzp1 v20.16b, v20.16b, v18.16b\n" + "uzp1 v24.16b, v24.16b, v17.16b\n" "bge 89f\n" "tbz x9, #3, 84f\n" "str d16, [x27], #0x8\n" - "str d20, [x22], #0x8\n" - "str d24, [x21], #0x8\n" + "str d20, [x23], #0x8\n" + "str d24, [x22], #0x8\n" "tbz x9, #2, 82f\n" "st1 { v16.s }[2], [x27], #0x4\n" - "st1 { v20.s }[2], [x22], #0x4\n" - "st1 { v24.s }[2], [x21], #0x4\n" + "st1 { v20.s }[2], [x23], #0x4\n" + "st1 { v24.s }[2], [x22], #0x4\n" "tbz x9, #1, 81f\n" "st1 { v16.h }[6], [x27], #0x2\n" - "st1 { v20.h }[6], [x22], #0x2\n" - "st1 { v24.h }[6], [x21], #0x2\n" + "st1 { v20.h }[6], [x23], #0x2\n" + "st1 { v24.h }[6], [x22], #0x2\n" "tbz x9, #0, 88f\n" "st1 { v16.b }[14], [x27]\n" - "st1 { v20.b }[14], [x22]\n" - "st1 { v24.b }[14], [x21]\n" + "st1 { v20.b }[14], [x23]\n" + "st1 { v24.b }[14], [x22]\n" "b 88f\n" "81:" // Height 3: Partial direct writeback: partial_1_12 "tbz x9, #0, 88f\n" "st1 { v16.b }[12], [x27]\n" - "st1 { v20.b }[12], [x22]\n" - "st1 { v24.b }[12], [x21]\n" + "st1 { v20.b }[12], [x23]\n" + "st1 { v24.b }[12], [x22]\n" "b 88f\n" "82:" // Height 3: Partial direct writeback: partial_2_8 "tbz x9, #1, 83f\n" "st1 { v16.h }[4], [x27], #0x2\n" - "st1 { v20.h }[4], [x22], #0x2\n" - "st1 { v24.h }[4], [x21], #0x2\n" + "st1 { v20.h }[4], [x23], #0x2\n" + "st1 { v24.h }[4], [x22], #0x2\n" "tbz x9, #0, 88f\n" "st1 { v16.b }[10], [x27]\n" - "st1 { v20.b }[10], [x22]\n" - "st1 { v24.b }[10], [x21]\n" + "st1 { v20.b }[10], [x23]\n" + "st1 { v24.b }[10], [x22]\n" "b 88f\n" "83:" // Height 3: Partial direct writeback: partial_1_8 "tbz x9, #0, 88f\n" "st1 { v16.b }[8], [x27]\n" - "st1 { v20.b }[8], [x22]\n" - "st1 { v24.b }[8], [x21]\n" + "st1 { v20.b }[8], [x23]\n" + "st1 { v24.b }[8], [x22]\n" "b 88f\n" "84:" // Height 3: Partial direct writeback: partial_4_0 "tbz x9, #2, 86f\n" "str s16, [x27], #0x4\n" - "str s20, [x22], #0x4\n" - "str s24, [x21], #0x4\n" + "str s20, [x23], #0x4\n" + "str s24, [x22], #0x4\n" "tbz x9, #1, 85f\n" "st1 { v16.h }[2], [x27], #0x2\n" - "st1 { v20.h }[2], [x22], #0x2\n" - "st1 { v24.h }[2], [x21], #0x2\n" + "st1 { v20.h }[2], [x23], #0x2\n" + "st1 { v24.h }[2], [x22], #0x2\n" "tbz x9, #0, 88f\n" "st1 { v16.b }[6], [x27]\n" - "st1 { v20.b }[6], [x22]\n" - "st1 { v24.b }[6], [x21]\n" + "st1 { v20.b }[6], [x23]\n" + "st1 { v24.b }[6], [x22]\n" "b 88f\n" "85:" // Height 3: Partial direct writeback: partial_1_4 "tbz x9, #0, 88f\n" "st1 { v16.b }[4], [x27]\n" - "st1 { v20.b }[4], [x22]\n" - "st1 { v24.b }[4], [x21]\n" + "st1 { v20.b }[4], [x23]\n" + "st1 { v24.b }[4], [x22]\n" "b 88f\n" "86:" // Height 3: Partial direct writeback: partial_2_0 "tbz x9, #1, 87f\n" "str h16, [x27], #0x2\n" - "str h20, [x22], #0x2\n" - "str h24, [x21], #0x2\n" + "str h20, [x23], #0x2\n" + "str h24, [x22], #0x2\n" "tbz x9, #0, 88f\n" "st1 { v16.b }[2], [x27]\n" - "st1 { v20.b }[2], [x22]\n" - "st1 { v24.b }[2], [x21]\n" + "st1 { v20.b }[2], [x23]\n" + "st1 { v24.b }[2], [x22]\n" "b 88f\n" "87:" // Height 3: Partial direct writeback: partial_1_0 "str b16, [x27, #0x0]\n" - "str b20, [x22, #0x0]\n" - "str b24, [x21, #0x0]\n" + "str b20, [x23, #0x0]\n" + "str b24, [x22, #0x0]\n" "88:" // Height 3: Partial direct writeback: Done "b 90f\n" "89:" // Height 3: Full writeback "str q16, [x27, #0x0]\n" "add x27, x27, #0x10\n" - "str q20, [x22, #0x0]\n" - "str q24, [x21, #0x0]\n" + "str q20, [x23, #0x0]\n" + "str q24, [x22, #0x0]\n" "90:" // Height 3: Writeback done "subs x9, x9, #0x10\n" "bgt 62b\n" @@ -1370,14 +1369,14 @@ void a64_hybrid_s8qa_dot_4x16 ( "94:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 95f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" - "ldr x21, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" + "ldr x21, [x20, #0x18]\n" "cbnz x26, 96f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -1387,9 +1386,9 @@ void a64_hybrid_s8qa_dot_4x16 ( "b 96f\n" "95:" // Height 4: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" + "add x21, x22, x21\n" "96:" // Height 4: input setup done "cmp x25, #0x10\n" "blt 101f\n" @@ -1614,29 +1613,29 @@ void a64_hybrid_s8qa_dot_4x16 ( ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n" ".inst 0x4e8f946e // sdot v14.4s, v3.16b, v15.16b\n" "103:" // Height 4: Multiply loop: unique 15: skip row sum - "ldr q6, [x28, #0x0]\n" - "ldr q7, [x28, #0x10]\n" + "ldr q7, [x28, #0x0]\n" + "ldr q6, [x28, #0x10]\n" "sub x25, x25, #0x4\n" "cmp x25, #0x4\n" - "ldr q8, [x28, #0x20]\n" - "ldr q9, [x28, #0x30]\n" - ".inst 0x4f80e0d0 // sdot v16.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0d4 // sdot v20.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d8 // sdot v24.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0dc // sdot v28.4s, v6.16b, v3.4b[0]\n" + "ldr q5, [x28, #0x20]\n" + "ldr q4, [x28, #0x30]\n" + ".inst 0x4f80e0f0 // sdot v16.4s, v7.16b, v0.4b[0]\n" + ".inst 0x4f81e0f4 // sdot v20.4s, v7.16b, v1.4b[0]\n" + ".inst 0x4f82e0f8 // sdot v24.4s, v7.16b, v2.4b[0]\n" + ".inst 0x4f83e0fc // sdot v28.4s, v7.16b, v3.4b[0]\n" "add x28, x28, #0x40\n" - ".inst 0x4f80e0f1 // sdot v17.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0f5 // sdot v21.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f9 // sdot v25.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0fd // sdot v29.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f80e112 // sdot v18.4s, v8.16b, v0.4b[0]\n" - ".inst 0x4f81e116 // sdot v22.4s, v8.16b, v1.4b[0]\n" - ".inst 0x4f82e11a // sdot v26.4s, v8.16b, v2.4b[0]\n" - ".inst 0x4f83e11e // sdot v30.4s, v8.16b, v3.4b[0]\n" - ".inst 0x4f80e133 // sdot v19.4s, v9.16b, v0.4b[0]\n" - ".inst 0x4f81e137 // sdot v23.4s, v9.16b, v1.4b[0]\n" - ".inst 0x4f82e13b // sdot v27.4s, v9.16b, v2.4b[0]\n" - ".inst 0x4f83e13f // sdot v31.4s, v9.16b, v3.4b[0]\n" + ".inst 0x4f80e0d1 // sdot v17.4s, v6.16b, v0.4b[0]\n" + ".inst 0x4f81e0d5 // sdot v21.4s, v6.16b, v1.4b[0]\n" + ".inst 0x4f82e0d9 // sdot v25.4s, v6.16b, v2.4b[0]\n" + ".inst 0x4f83e0dd // sdot v29.4s, v6.16b, v3.4b[0]\n" + ".inst 0x4f80e0b2 // sdot v18.4s, v5.16b, v0.4b[0]\n" + ".inst 0x4f81e0b6 // sdot v22.4s, v5.16b, v1.4b[0]\n" + ".inst 0x4f82e0ba // sdot v26.4s, v5.16b, v2.4b[0]\n" + ".inst 0x4f83e0be // sdot v30.4s, v5.16b, v3.4b[0]\n" + ".inst 0x4f80e093 // sdot v19.4s, v4.16b, v0.4b[0]\n" + ".inst 0x4f81e097 // sdot v23.4s, v4.16b, v1.4b[0]\n" + ".inst 0x4f82e09b // sdot v27.4s, v4.16b, v2.4b[0]\n" + ".inst 0x4f83e09f // sdot v31.4s, v4.16b, v3.4b[0]\n" "bge 102b\n" "104:" // Height 4: Multiply loop: Skip odd blocks "cbz x25, 108f\n" @@ -1663,73 +1662,73 @@ void a64_hybrid_s8qa_dot_4x16 ( ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n" ".inst 0x4e8f946e // sdot v14.4s, v3.16b, v15.16b\n" "107:" // Height 4: Multiply loop: unique 16: skip row sum - "ldr q10, [x28, #0x0]\n" - "ldr q4, [x28, #0x10]\n" - ".inst 0x4f80e150 // sdot v16.4s, v10.16b, v0.4b[0]\n" - ".inst 0x4f81e154 // sdot v20.4s, v10.16b, v1.4b[0]\n" + "ldr q7, [x28, #0x0]\n" + "ldr q6, [x28, #0x10]\n" + ".inst 0x4f80e0f0 // sdot v16.4s, v7.16b, v0.4b[0]\n" + ".inst 0x4f81e0f4 // sdot v20.4s, v7.16b, v1.4b[0]\n" "ldr q5, [x28, #0x20]\n" - "ldr q6, [x28, #0x30]\n" - ".inst 0x4f82e158 // sdot v24.4s, v10.16b, v2.4b[0]\n" - ".inst 0x4f83e15c // sdot v28.4s, v10.16b, v3.4b[0]\n" - ".inst 0x4f80e091 // sdot v17.4s, v4.16b, v0.4b[0]\n" - ".inst 0x4f81e095 // sdot v21.4s, v4.16b, v1.4b[0]\n" + "ldr q4, [x28, #0x30]\n" + ".inst 0x4f82e0f8 // sdot v24.4s, v7.16b, v2.4b[0]\n" + ".inst 0x4f83e0fc // sdot v28.4s, v7.16b, v3.4b[0]\n" + ".inst 0x4f80e0d1 // sdot v17.4s, v6.16b, v0.4b[0]\n" + ".inst 0x4f81e0d5 // sdot v21.4s, v6.16b, v1.4b[0]\n" "add x28, x28, #0x40\n" - ".inst 0x4f82e099 // sdot v25.4s, v4.16b, v2.4b[0]\n" - ".inst 0x4f83e09d // sdot v29.4s, v4.16b, v3.4b[0]\n" + ".inst 0x4f82e0d9 // sdot v25.4s, v6.16b, v2.4b[0]\n" + ".inst 0x4f83e0dd // sdot v29.4s, v6.16b, v3.4b[0]\n" ".inst 0x4f80e0b2 // sdot v18.4s, v5.16b, v0.4b[0]\n" ".inst 0x4f81e0b6 // sdot v22.4s, v5.16b, v1.4b[0]\n" ".inst 0x4f82e0ba // sdot v26.4s, v5.16b, v2.4b[0]\n" ".inst 0x4f83e0be // sdot v30.4s, v5.16b, v3.4b[0]\n" - ".inst 0x4f80e0d3 // sdot v19.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0d7 // sdot v23.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0db // sdot v27.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0df // sdot v31.4s, v6.16b, v3.4b[0]\n" + ".inst 0x4f80e093 // sdot v19.4s, v4.16b, v0.4b[0]\n" + ".inst 0x4f81e097 // sdot v23.4s, v4.16b, v1.4b[0]\n" + ".inst 0x4f82e09b // sdot v27.4s, v4.16b, v2.4b[0]\n" + ".inst 0x4f83e09f // sdot v31.4s, v4.16b, v3.4b[0]\n" "108:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x26, x26, #0x1\n" "cmp x26, x20\n" "bne 94b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x22, x27, x20\n" - "add x21, x22, x20\n" + "add x23, x27, x20\n" + "add x22, x23, x20\n" "prfm pstl1keep, [x27, #0x0]\n" - "add x20, x21, x20\n" + "add x21, x22, x20\n" + "prfm pstl1keep, [x23, #0x0]\n" "prfm pstl1keep, [x22, #0x0]\n" "prfm pstl1keep, [x21, #0x0]\n" - "prfm pstl1keep, [x20, #0x0]\n" "tbnz %x[flags], #31, 109f\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1r { v0.4s }, [x20]\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v12.4s, v12.4s, v12.4s\n" "addp v13.4s, v13.4s, v13.4s\n" "addp v14.4s, v14.4s, v14.4s\n" - "neg v4.4s, v4.4s\n" + "neg v0.4s, v0.4s\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v12.4s, v12.4s, v12.4s\n" "addp v13.4s, v13.4s, v13.4s\n" "addp v14.4s, v14.4s, v14.4s\n" - "mul v11.4s, v11.4s, v4.4s\n" - "mul v12.4s, v12.4s, v4.4s\n" - "mul v13.4s, v13.4s, v4.4s\n" - "mul v14.4s, v14.4s, v4.4s\n" + "mul v11.4s, v11.4s, v0.4s\n" + "mul v12.4s, v12.4s, v0.4s\n" + "mul v13.4s, v13.4s, v0.4s\n" + "mul v14.4s, v14.4s, v0.4s\n" "109:" // Height 4: skip row sum fixup "ldr q0, [x10, #0x0]\n" - "ldr q1, [x10, #0x10]\n" + "ldr q4, [x10, #0x10]\n" "add v16.4s, v16.4s, v11.4s\n" "add v17.4s, v17.4s, v11.4s\n" - "ldr q2, [x10, #0x20]\n" - "ldr q3, [x10, #0x30]\n" + "ldr q3, [x10, #0x20]\n" + "ldr q2, [x10, #0x30]\n" "add v18.4s, v18.4s, v11.4s\n" "add v19.4s, v19.4s, v11.4s\n" "add v20.4s, v20.4s, v12.4s\n" "add v21.4s, v21.4s, v12.4s\n" - "add x23, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v1.4s }, [x20]\n" "add v22.4s, v22.4s, v12.4s\n" "add v23.4s, v23.4s, v12.4s\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" "add v24.4s, v24.4s, v13.4s\n" "add v25.4s, v25.4s, v13.4s\n" "add x10, x10, #0x40\n" @@ -1740,100 +1739,100 @@ void a64_hybrid_s8qa_dot_4x16 ( "add v30.4s, v30.4s, v14.4s\n" "add v31.4s, v31.4s, v14.4s\n" "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" + "add v17.4s, v17.4s, v4.4s\n" + "add v18.4s, v18.4s, v3.4s\n" + "add v19.4s, v19.4s, v2.4s\n" "add v20.4s, v20.4s, v0.4s\n" - "add v21.4s, v21.4s, v1.4s\n" - "add v22.4s, v22.4s, v2.4s\n" - "add v23.4s, v23.4s, v3.4s\n" + "add v21.4s, v21.4s, v4.4s\n" + "add v22.4s, v22.4s, v3.4s\n" + "add v23.4s, v23.4s, v2.4s\n" "add v24.4s, v24.4s, v0.4s\n" - "add v25.4s, v25.4s, v1.4s\n" - "add v26.4s, v26.4s, v2.4s\n" - "add v27.4s, v27.4s, v3.4s\n" + "add v25.4s, v25.4s, v4.4s\n" + "add v26.4s, v26.4s, v3.4s\n" + "add v27.4s, v27.4s, v2.4s\n" "add v28.4s, v28.4s, v0.4s\n" - "ld1r { v0.4s }, [x23]\n" - "add v29.4s, v29.4s, v1.4s\n" - "add v30.4s, v30.4s, v2.4s\n" - "add v31.4s, v31.4s, v3.4s\n" - "sqrdmulh v16.4s, v16.4s, v4.4s\n" - "sqrdmulh v17.4s, v17.4s, v4.4s\n" - "sqrdmulh v18.4s, v18.4s, v4.4s\n" - "sqrdmulh v19.4s, v19.4s, v4.4s\n" - "sqrdmulh v20.4s, v20.4s, v4.4s\n" - "sqrdmulh v21.4s, v21.4s, v4.4s\n" - "sqrdmulh v22.4s, v22.4s, v4.4s\n" - "sqrdmulh v23.4s, v23.4s, v4.4s\n" - "sqrdmulh v24.4s, v24.4s, v4.4s\n" - "sqrdmulh v25.4s, v25.4s, v4.4s\n" - "sqrdmulh v26.4s, v26.4s, v4.4s\n" - "sqrdmulh v27.4s, v27.4s, v4.4s\n" - "sqrdmulh v28.4s, v28.4s, v4.4s\n" - "sqrdmulh v29.4s, v29.4s, v4.4s\n" - "sqrdmulh v30.4s, v30.4s, v4.4s\n" - "sqrdmulh v31.4s, v31.4s, v4.4s\n" + "ld1r { v0.4s }, [x20]\n" + "add v29.4s, v29.4s, v4.4s\n" + "add v30.4s, v30.4s, v3.4s\n" + "add v31.4s, v31.4s, v2.4s\n" + "sqrdmulh v16.4s, v16.4s, v1.4s\n" + "sqrdmulh v17.4s, v17.4s, v1.4s\n" + "sqrdmulh v18.4s, v18.4s, v1.4s\n" + "sqrdmulh v19.4s, v19.4s, v1.4s\n" + "sqrdmulh v20.4s, v20.4s, v1.4s\n" + "sqrdmulh v21.4s, v21.4s, v1.4s\n" + "sqrdmulh v22.4s, v22.4s, v1.4s\n" + "sqrdmulh v23.4s, v23.4s, v1.4s\n" + "sqrdmulh v24.4s, v24.4s, v1.4s\n" + "sqrdmulh v25.4s, v25.4s, v1.4s\n" + "sqrdmulh v26.4s, v26.4s, v1.4s\n" + "sqrdmulh v27.4s, v27.4s, v1.4s\n" + "sqrdmulh v28.4s, v28.4s, v1.4s\n" + "sqrdmulh v29.4s, v29.4s, v1.4s\n" + "sqrdmulh v30.4s, v30.4s, v1.4s\n" + "sqrdmulh v31.4s, v31.4s, v1.4s\n" "tbz %x[flags], #5, 110f\n" - "and v4.16b, v16.16b, v0.16b\n" - "and v5.16b, v17.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "and v6.16b, v18.16b, v0.16b\n" - "and v7.16b, v19.16b, v0.16b\n" - "and v8.16b, v20.16b, v0.16b\n" - "and v9.16b, v21.16b, v0.16b\n" - "and v10.16b, v22.16b, v0.16b\n" - "and v4.16b, v23.16b, v0.16b\n" - "and v5.16b, v24.16b, v0.16b\n" - "sshr v6.4s, v6.4s, #0x1f\n" + "and v2.16b, v16.16b, v0.16b\n" + "and v1.16b, v17.16b, v0.16b\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v2.4s\n" + "sqadd v17.4s, v17.4s, v1.4s\n" + "and v7.16b, v18.16b, v0.16b\n" + "and v6.16b, v19.16b, v0.16b\n" + "and v5.16b, v20.16b, v0.16b\n" + "and v4.16b, v21.16b, v0.16b\n" + "and v3.16b, v22.16b, v0.16b\n" + "and v2.16b, v23.16b, v0.16b\n" + "and v1.16b, v24.16b, v0.16b\n" "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v9.4s, v9.4s, #0x1f\n" - "sshr v10.4s, v10.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" - "sqadd v20.4s, v20.4s, v8.4s\n" - "sqadd v21.4s, v21.4s, v9.4s\n" - "sqadd v22.4s, v22.4s, v10.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v24.4s, v24.4s, v5.4s\n" - "and v6.16b, v25.16b, v0.16b\n" - "and v7.16b, v26.16b, v0.16b\n" - "and v8.16b, v27.16b, v0.16b\n" - "and v9.16b, v28.16b, v0.16b\n" - "and v10.16b, v29.16b, v0.16b\n" - "and v4.16b, v30.16b, v0.16b\n" - "and v5.16b, v31.16b, v0.16b\n" "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v9.4s, v9.4s, #0x1f\n" - "sshr v10.4s, v10.4s, #0x1f\n" + "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sqadd v18.4s, v18.4s, v7.4s\n" + "sqadd v19.4s, v19.4s, v6.4s\n" + "sqadd v20.4s, v20.4s, v5.4s\n" + "sqadd v21.4s, v21.4s, v4.4s\n" + "sqadd v22.4s, v22.4s, v3.4s\n" + "sqadd v23.4s, v23.4s, v2.4s\n" + "sqadd v24.4s, v24.4s, v1.4s\n" + "and v7.16b, v25.16b, v0.16b\n" + "and v6.16b, v26.16b, v0.16b\n" + "and v5.16b, v27.16b, v0.16b\n" + "and v4.16b, v28.16b, v0.16b\n" + "and v3.16b, v29.16b, v0.16b\n" + "and v2.16b, v30.16b, v0.16b\n" + "and v1.16b, v31.16b, v0.16b\n" + "sshr v7.4s, v7.4s, #0x1f\n" + "sshr v6.4s, v6.4s, #0x1f\n" "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v25.4s, v25.4s, v6.4s\n" - "sqadd v26.4s, v26.4s, v7.4s\n" - "sqadd v27.4s, v27.4s, v8.4s\n" - "sqadd v28.4s, v28.4s, v9.4s\n" - "sqadd v29.4s, v29.4s, v10.4s\n" - "sqadd v30.4s, v30.4s, v4.4s\n" - "sqadd v31.4s, v31.4s, v5.4s\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sqadd v25.4s, v25.4s, v7.4s\n" + "sqadd v26.4s, v26.4s, v6.4s\n" + "sqadd v27.4s, v27.4s, v5.4s\n" + "sqadd v28.4s, v28.4s, v4.4s\n" + "sqadd v29.4s, v29.4s, v3.4s\n" + "sqadd v30.4s, v30.4s, v2.4s\n" + "sqadd v31.4s, v31.4s, v1.4s\n" "110:" // Height 4: no shift correction - "add x23, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v3.4s }, [x20]\n" "srshl v16.4s, v16.4s, v0.4s\n" "srshl v17.4s, v17.4s, v0.4s\n" "srshl v18.4s, v18.4s, v0.4s\n" "srshl v19.4s, v19.4s, v0.4s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x23]\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v2.4s }, [x20]\n" "srshl v20.4s, v20.4s, v0.4s\n" "srshl v21.4s, v21.4s, v0.4s\n" - "add x23, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x23]\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v1.4s }, [x20]\n" "srshl v22.4s, v22.4s, v0.4s\n" "srshl v23.4s, v23.4s, v0.4s\n" "cmp x9, #0x10\n" @@ -1845,163 +1844,163 @@ void a64_hybrid_s8qa_dot_4x16 ( "srshl v29.4s, v29.4s, v0.4s\n" "srshl v30.4s, v30.4s, v0.4s\n" "srshl v31.4s, v31.4s, v0.4s\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v23.4s, v23.4s, v4.4s\n" - "add v24.4s, v24.4s, v4.4s\n" - "add v25.4s, v25.4s, v4.4s\n" - "add v26.4s, v26.4s, v4.4s\n" - "add v27.4s, v27.4s, v4.4s\n" - "add v28.4s, v28.4s, v4.4s\n" - "add v29.4s, v29.4s, v4.4s\n" - "add v30.4s, v30.4s, v4.4s\n" - "add v31.4s, v31.4s, v4.4s\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "smin v24.4s, v24.4s, v6.4s\n" - "smin v25.4s, v25.4s, v6.4s\n" - "smin v26.4s, v26.4s, v6.4s\n" - "smin v27.4s, v27.4s, v6.4s\n" - "smin v28.4s, v28.4s, v6.4s\n" - "smin v29.4s, v29.4s, v6.4s\n" - "smin v30.4s, v30.4s, v6.4s\n" - "smin v31.4s, v31.4s, v6.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" - "smax v24.4s, v24.4s, v5.4s\n" - "smax v25.4s, v25.4s, v5.4s\n" - "smax v26.4s, v26.4s, v5.4s\n" - "smax v27.4s, v27.4s, v5.4s\n" - "smax v28.4s, v28.4s, v5.4s\n" - "smax v29.4s, v29.4s, v5.4s\n" - "smax v30.4s, v30.4s, v5.4s\n" - "smax v31.4s, v31.4s, v5.4s\n" + "add v16.4s, v16.4s, v3.4s\n" + "add v17.4s, v17.4s, v3.4s\n" + "add v18.4s, v18.4s, v3.4s\n" + "add v19.4s, v19.4s, v3.4s\n" + "add v20.4s, v20.4s, v3.4s\n" + "add v21.4s, v21.4s, v3.4s\n" + "add v22.4s, v22.4s, v3.4s\n" + "add v23.4s, v23.4s, v3.4s\n" + "add v24.4s, v24.4s, v3.4s\n" + "add v25.4s, v25.4s, v3.4s\n" + "add v26.4s, v26.4s, v3.4s\n" + "add v27.4s, v27.4s, v3.4s\n" + "add v28.4s, v28.4s, v3.4s\n" + "add v29.4s, v29.4s, v3.4s\n" + "add v30.4s, v30.4s, v3.4s\n" + "add v31.4s, v31.4s, v3.4s\n" + "smin v16.4s, v16.4s, v2.4s\n" + "smin v17.4s, v17.4s, v2.4s\n" + "smin v18.4s, v18.4s, v2.4s\n" + "smin v19.4s, v19.4s, v2.4s\n" + "smin v20.4s, v20.4s, v2.4s\n" + "smin v21.4s, v21.4s, v2.4s\n" + "smin v22.4s, v22.4s, v2.4s\n" + "smin v23.4s, v23.4s, v2.4s\n" + "smin v24.4s, v24.4s, v2.4s\n" + "smin v25.4s, v25.4s, v2.4s\n" + "smin v26.4s, v26.4s, v2.4s\n" + "smin v27.4s, v27.4s, v2.4s\n" + "smin v28.4s, v28.4s, v2.4s\n" + "smin v29.4s, v29.4s, v2.4s\n" + "smin v30.4s, v30.4s, v2.4s\n" + "smin v31.4s, v31.4s, v2.4s\n" + "smax v16.4s, v16.4s, v1.4s\n" + "smax v17.4s, v17.4s, v1.4s\n" + "smax v18.4s, v18.4s, v1.4s\n" + "smax v19.4s, v19.4s, v1.4s\n" + "smax v20.4s, v20.4s, v1.4s\n" + "smax v21.4s, v21.4s, v1.4s\n" + "smax v22.4s, v22.4s, v1.4s\n" + "smax v23.4s, v23.4s, v1.4s\n" + "smax v24.4s, v24.4s, v1.4s\n" + "smax v25.4s, v25.4s, v1.4s\n" + "smax v26.4s, v26.4s, v1.4s\n" + "smax v27.4s, v27.4s, v1.4s\n" + "smax v28.4s, v28.4s, v1.4s\n" + "smax v29.4s, v29.4s, v1.4s\n" + "smax v30.4s, v30.4s, v1.4s\n" + "smax v31.4s, v31.4s, v1.4s\n" "uzp1 v16.8h, v16.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v0.8h, v18.8h, v19.8h\n" "uzp1 v20.8h, v20.8h, v21.8h\n" - "uzp1 v21.8h, v22.8h, v23.8h\n" + "uzp1 v19.8h, v22.8h, v23.8h\n" "uzp1 v24.8h, v24.8h, v25.8h\n" - "uzp1 v25.8h, v26.8h, v27.8h\n" + "uzp1 v18.8h, v26.8h, v27.8h\n" "uzp1 v28.8h, v28.8h, v29.8h\n" - "uzp1 v29.8h, v30.8h, v31.8h\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" - "uzp1 v20.16b, v20.16b, v21.16b\n" - "uzp1 v24.16b, v24.16b, v25.16b\n" - "uzp1 v28.16b, v28.16b, v29.16b\n" + "uzp1 v17.8h, v30.8h, v31.8h\n" + "uzp1 v16.16b, v16.16b, v0.16b\n" + "uzp1 v20.16b, v20.16b, v19.16b\n" + "uzp1 v24.16b, v24.16b, v18.16b\n" + "uzp1 v28.16b, v28.16b, v17.16b\n" "bge 119f\n" "tbz x9, #3, 114f\n" "str d16, [x27], #0x8\n" - "str d20, [x22], #0x8\n" - "str d24, [x21], #0x8\n" - "str d28, [x20], #0x8\n" + "str d20, [x23], #0x8\n" + "str d24, [x22], #0x8\n" + "str d28, [x21], #0x8\n" "tbz x9, #2, 112f\n" "st1 { v16.s }[2], [x27], #0x4\n" - "st1 { v20.s }[2], [x22], #0x4\n" - "st1 { v24.s }[2], [x21], #0x4\n" - "st1 { v28.s }[2], [x20], #0x4\n" + "st1 { v20.s }[2], [x23], #0x4\n" + "st1 { v24.s }[2], [x22], #0x4\n" + "st1 { v28.s }[2], [x21], #0x4\n" "tbz x9, #1, 111f\n" "st1 { v16.h }[6], [x27], #0x2\n" - "st1 { v20.h }[6], [x22], #0x2\n" - "st1 { v24.h }[6], [x21], #0x2\n" - "st1 { v28.h }[6], [x20], #0x2\n" + "st1 { v20.h }[6], [x23], #0x2\n" + "st1 { v24.h }[6], [x22], #0x2\n" + "st1 { v28.h }[6], [x21], #0x2\n" "tbz x9, #0, 118f\n" "st1 { v16.b }[14], [x27]\n" - "st1 { v20.b }[14], [x22]\n" - "st1 { v24.b }[14], [x21]\n" - "st1 { v28.b }[14], [x20]\n" + "st1 { v20.b }[14], [x23]\n" + "st1 { v24.b }[14], [x22]\n" + "st1 { v28.b }[14], [x21]\n" "b 118f\n" "111:" // Height 4: Partial direct writeback: partial_1_12 "tbz x9, #0, 118f\n" "st1 { v16.b }[12], [x27]\n" - "st1 { v20.b }[12], [x22]\n" - "st1 { v24.b }[12], [x21]\n" - "st1 { v28.b }[12], [x20]\n" + "st1 { v20.b }[12], [x23]\n" + "st1 { v24.b }[12], [x22]\n" + "st1 { v28.b }[12], [x21]\n" "b 118f\n" "112:" // Height 4: Partial direct writeback: partial_2_8 "tbz x9, #1, 113f\n" "st1 { v16.h }[4], [x27], #0x2\n" - "st1 { v20.h }[4], [x22], #0x2\n" - "st1 { v24.h }[4], [x21], #0x2\n" - "st1 { v28.h }[4], [x20], #0x2\n" + "st1 { v20.h }[4], [x23], #0x2\n" + "st1 { v24.h }[4], [x22], #0x2\n" + "st1 { v28.h }[4], [x21], #0x2\n" "tbz x9, #0, 118f\n" "st1 { v16.b }[10], [x27]\n" - "st1 { v20.b }[10], [x22]\n" - "st1 { v24.b }[10], [x21]\n" - "st1 { v28.b }[10], [x20]\n" + "st1 { v20.b }[10], [x23]\n" + "st1 { v24.b }[10], [x22]\n" + "st1 { v28.b }[10], [x21]\n" "b 118f\n" "113:" // Height 4: Partial direct writeback: partial_1_8 "tbz x9, #0, 118f\n" "st1 { v16.b }[8], [x27]\n" - "st1 { v20.b }[8], [x22]\n" - "st1 { v24.b }[8], [x21]\n" - "st1 { v28.b }[8], [x20]\n" + "st1 { v20.b }[8], [x23]\n" + "st1 { v24.b }[8], [x22]\n" + "st1 { v28.b }[8], [x21]\n" "b 118f\n" "114:" // Height 4: Partial direct writeback: partial_4_0 "tbz x9, #2, 116f\n" "str s16, [x27], #0x4\n" - "str s20, [x22], #0x4\n" - "str s24, [x21], #0x4\n" - "str s28, [x20], #0x4\n" + "str s20, [x23], #0x4\n" + "str s24, [x22], #0x4\n" + "str s28, [x21], #0x4\n" "tbz x9, #1, 115f\n" "st1 { v16.h }[2], [x27], #0x2\n" - "st1 { v20.h }[2], [x22], #0x2\n" - "st1 { v24.h }[2], [x21], #0x2\n" - "st1 { v28.h }[2], [x20], #0x2\n" + "st1 { v20.h }[2], [x23], #0x2\n" + "st1 { v24.h }[2], [x22], #0x2\n" + "st1 { v28.h }[2], [x21], #0x2\n" "tbz x9, #0, 118f\n" "st1 { v16.b }[6], [x27]\n" - "st1 { v20.b }[6], [x22]\n" - "st1 { v24.b }[6], [x21]\n" - "st1 { v28.b }[6], [x20]\n" + "st1 { v20.b }[6], [x23]\n" + "st1 { v24.b }[6], [x22]\n" + "st1 { v28.b }[6], [x21]\n" "b 118f\n" "115:" // Height 4: Partial direct writeback: partial_1_4 "tbz x9, #0, 118f\n" "st1 { v16.b }[4], [x27]\n" - "st1 { v20.b }[4], [x22]\n" - "st1 { v24.b }[4], [x21]\n" - "st1 { v28.b }[4], [x20]\n" + "st1 { v20.b }[4], [x23]\n" + "st1 { v24.b }[4], [x22]\n" + "st1 { v28.b }[4], [x21]\n" "b 118f\n" "116:" // Height 4: Partial direct writeback: partial_2_0 "tbz x9, #1, 117f\n" "str h16, [x27], #0x2\n" - "str h20, [x22], #0x2\n" - "str h24, [x21], #0x2\n" - "str h28, [x20], #0x2\n" + "str h20, [x23], #0x2\n" + "str h24, [x22], #0x2\n" + "str h28, [x21], #0x2\n" "tbz x9, #0, 118f\n" "st1 { v16.b }[2], [x27]\n" - "st1 { v20.b }[2], [x22]\n" - "st1 { v24.b }[2], [x21]\n" - "st1 { v28.b }[2], [x20]\n" + "st1 { v20.b }[2], [x23]\n" + "st1 { v24.b }[2], [x22]\n" + "st1 { v28.b }[2], [x21]\n" "b 118f\n" "117:" // Height 4: Partial direct writeback: partial_1_0 "str b16, [x27, #0x0]\n" - "str b20, [x22, #0x0]\n" - "str b24, [x21, #0x0]\n" - "str b28, [x20, #0x0]\n" + "str b20, [x23, #0x0]\n" + "str b24, [x22, #0x0]\n" + "str b28, [x21, #0x0]\n" "118:" // Height 4: Partial direct writeback: Done "b 120f\n" "119:" // Height 4: Full writeback "str q16, [x27, #0x0]\n" "add x27, x27, #0x10\n" - "str q20, [x22, #0x0]\n" - "str q24, [x21, #0x0]\n" - "str q28, [x20, #0x0]\n" + "str q20, [x23, #0x0]\n" + "str q24, [x22, #0x0]\n" + "str q28, [x21, #0x0]\n" "120:" // Height 4: Writeback done "subs x9, x9, #0x10\n" "bgt 92b\n" @@ -2017,7 +2016,6 @@ void a64_hybrid_s8qa_dot_4x16 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "122:" // Exit - : [M] "+&r" (M), [flags] "+&r" (flags), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_mmla_4x16.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_mmla_4x16.hpp index 69ea87bc9ee72d8d91c37f52b2277dda0132ecbf..55ea68d1b5b0615730fc05332600203b002e831d 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_mmla_4x16.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_mmla_4x16.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef __aarch64__ + #include "../std_transforms_fixed.hpp" #include "../performance_parameters.hpp" @@ -98,5 +98,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_mmla_4x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_mmla_4x16/generic.cpp index 69d01a265e9259ed21edcfae067cb39f7bf68210..883bd5afddb5d5de34568c1fd84c4756de7a3560 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_mmla_4x16/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_mmla_4x16/generic.cpp @@ -78,7 +78,6 @@ void a64_hybrid_s8qa_mmla_4x16 ( flags |= 0x20; } __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x4\n" "bge 97f\n" @@ -106,11 +105,11 @@ void a64_hybrid_s8qa_mmla_4x16 ( "4:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 5f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" "cbnz x26, 6f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -131,35 +130,35 @@ void a64_hybrid_s8qa_mmla_4x16 ( "ldr q4, [x28, #0x60]\n" "blt 9f\n" "7:" // Height 1: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v0.2d, v1.2d, v27.2d\n" ".inst 0x4e85a410 // smmla v16.4s, v0.16b, v5.16b\n" - "ldr q5, [x28, #0x70]\n" - "trn2 v1.2d, v1.2d, v2.2d\n" + "ldr q25, [x28, #0x70]\n" + "trn2 v1.2d, v1.2d, v27.2d\n" ".inst 0x4e86a414 // smmla v20.4s, v0.16b, v6.16b\n" - "ldr q6, [x28, #0x80]\n" + "ldr q24, [x28, #0x80]\n" ".inst 0x4e87a411 // smmla v17.4s, v0.16b, v7.16b\n" - "ldr q7, [x28, #0x90]\n" + "ldr q30, [x28, #0x90]\n" ".inst 0x4e88a415 // smmla v21.4s, v0.16b, v8.16b\n" - "ldr q8, [x28, #0xa0]\n" + "ldr q29, [x28, #0xa0]\n" ".inst 0x4e89a412 // smmla v18.4s, v0.16b, v9.16b\n" - "ldr q9, [x28, #0xb0]\n" + "ldr q28, [x28, #0xb0]\n" ".inst 0x4e8aa416 // smmla v22.4s, v0.16b, v10.16b\n" - "ldr q10, [x28, #0xc0]\n" + "ldr q27, [x28, #0xc0]\n" ".inst 0x4e84a413 // smmla v19.4s, v0.16b, v4.16b\n" - "ldr q4, [x28, #0xd0]\n" - ".inst 0x4e85a417 // smmla v23.4s, v0.16b, v5.16b\n" - "ldr q5, [x28, #0xe0]\n" - ".inst 0x4e86a430 // smmla v16.4s, v1.16b, v6.16b\n" - "ldr q6, [x28, #0xf0]\n" - ".inst 0x4e87a434 // smmla v20.4s, v1.16b, v7.16b\n" + "ldr q26, [x28, #0xd0]\n" + ".inst 0x4e99a417 // smmla v23.4s, v0.16b, v25.16b\n" + "ldr q25, [x28, #0xe0]\n" + ".inst 0x4e98a430 // smmla v16.4s, v1.16b, v24.16b\n" + "ldr q24, [x28, #0xf0]\n" + ".inst 0x4e9ea434 // smmla v20.4s, v1.16b, v30.16b\n" "add x24, x24, #0x10\n" - ".inst 0x4e88a431 // smmla v17.4s, v1.16b, v8.16b\n" + ".inst 0x4e9da431 // smmla v17.4s, v1.16b, v29.16b\n" "add x28, x28, #0x100\n" - ".inst 0x4e89a435 // smmla v21.4s, v1.16b, v9.16b\n" - ".inst 0x4e8aa432 // smmla v18.4s, v1.16b, v10.16b\n" - ".inst 0x4e84a436 // smmla v22.4s, v1.16b, v4.16b\n" - ".inst 0x4e85a433 // smmla v19.4s, v1.16b, v5.16b\n" - ".inst 0x4e86a437 // smmla v23.4s, v1.16b, v6.16b\n" + ".inst 0x4e9ca435 // smmla v21.4s, v1.16b, v28.16b\n" + ".inst 0x4e9ba432 // smmla v18.4s, v1.16b, v27.16b\n" + ".inst 0x4e9aa436 // smmla v22.4s, v1.16b, v26.16b\n" + ".inst 0x4e99a433 // smmla v19.4s, v1.16b, v25.16b\n" + ".inst 0x4e98a437 // smmla v23.4s, v1.16b, v24.16b\n" "tbnz %x[flags], #31, 8f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f942b // sdot v11.4s, v1.16b, v15.16b\n" @@ -177,36 +176,36 @@ void a64_hybrid_s8qa_mmla_4x16 ( "prfm pldl1keep, [x24, #0x80]\n" "bge 7b\n" "9:" // Height 1: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v0.2d, v1.2d, v24.2d\n" ".inst 0x4e85a410 // smmla v16.4s, v0.16b, v5.16b\n" - "ldr q5, [x28, #0x70]\n" - "trn2 v1.2d, v1.2d, v2.2d\n" + "ldr q25, [x28, #0x70]\n" + "trn2 v1.2d, v1.2d, v24.2d\n" ".inst 0x4e86a414 // smmla v20.4s, v0.16b, v6.16b\n" - "ldr q6, [x28, #0x80]\n" + "ldr q24, [x28, #0x80]\n" ".inst 0x4e87a411 // smmla v17.4s, v0.16b, v7.16b\n" - "ldr q7, [x28, #0x90]\n" + "ldr q30, [x28, #0x90]\n" ".inst 0x4e88a415 // smmla v21.4s, v0.16b, v8.16b\n" - "ldr q8, [x28, #0xa0]\n" + "ldr q29, [x28, #0xa0]\n" ".inst 0x4e89a412 // smmla v18.4s, v0.16b, v9.16b\n" - "ldr q9, [x28, #0xb0]\n" + "ldr q28, [x28, #0xb0]\n" ".inst 0x4e8aa416 // smmla v22.4s, v0.16b, v10.16b\n" - "ldr q10, [x28, #0xc0]\n" + "ldr q27, [x28, #0xc0]\n" ".inst 0x4e84a413 // smmla v19.4s, v0.16b, v4.16b\n" - "ldr q4, [x28, #0xd0]\n" - ".inst 0x4e85a417 // smmla v23.4s, v0.16b, v5.16b\n" - "ldr q5, [x28, #0xe0]\n" - ".inst 0x4e86a430 // smmla v16.4s, v1.16b, v6.16b\n" - "ldr q6, [x28, #0xf0]\n" + "ldr q26, [x28, #0xd0]\n" + ".inst 0x4e99a417 // smmla v23.4s, v0.16b, v25.16b\n" + "ldr q25, [x28, #0xe0]\n" + ".inst 0x4e98a430 // smmla v16.4s, v1.16b, v24.16b\n" + "ldr q24, [x28, #0xf0]\n" "sub x25, x25, #0x10\n" - ".inst 0x4e87a434 // smmla v20.4s, v1.16b, v7.16b\n" - ".inst 0x4e88a431 // smmla v17.4s, v1.16b, v8.16b\n" + ".inst 0x4e9ea434 // smmla v20.4s, v1.16b, v30.16b\n" + ".inst 0x4e9da431 // smmla v17.4s, v1.16b, v29.16b\n" "add x24, x24, #0x10\n" "add x28, x28, #0x100\n" - ".inst 0x4e89a435 // smmla v21.4s, v1.16b, v9.16b\n" - ".inst 0x4e8aa432 // smmla v18.4s, v1.16b, v10.16b\n" - ".inst 0x4e84a436 // smmla v22.4s, v1.16b, v4.16b\n" - ".inst 0x4e85a433 // smmla v19.4s, v1.16b, v5.16b\n" - ".inst 0x4e86a437 // smmla v23.4s, v1.16b, v6.16b\n" + ".inst 0x4e9ca435 // smmla v21.4s, v1.16b, v28.16b\n" + ".inst 0x4e9ba432 // smmla v18.4s, v1.16b, v27.16b\n" + ".inst 0x4e9aa436 // smmla v22.4s, v1.16b, v26.16b\n" + ".inst 0x4e99a433 // smmla v19.4s, v1.16b, v25.16b\n" + ".inst 0x4e98a437 // smmla v23.4s, v1.16b, v24.16b\n" "tbnz %x[flags], #31, 10f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f942b // sdot v11.4s, v1.16b, v15.16b\n" @@ -217,29 +216,29 @@ void a64_hybrid_s8qa_mmla_4x16 ( "cmp x25, #0x8\n" "blt 14f\n" "12:" // Height 1: Multiply loop: Odd block loop - "ldr d1, [x24], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" + "ldr d25, [x24], #0x8\n" + "trn1 v0.2d, v25.2d, v24.2d\n" "tbnz %x[flags], #31, 13f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" "13:" // Height 1: Multiply loop: unique 3: skip row sum - "ldr q8, [x28, #0x0]\n" - "ldr q9, [x28, #0x10]\n" - ".inst 0x4e88a410 // smmla v16.4s, v0.16b, v8.16b\n" + "ldr q24, [x28, #0x0]\n" + "ldr q26, [x28, #0x10]\n" + ".inst 0x4e98a410 // smmla v16.4s, v0.16b, v24.16b\n" "sub x25, x25, #0x8\n" - "ldr q10, [x28, #0x20]\n" - "ldr q4, [x28, #0x30]\n" + "ldr q25, [x28, #0x20]\n" + "ldr q24, [x28, #0x30]\n" "cmp x25, #0x8\n" - ".inst 0x4e89a414 // smmla v20.4s, v0.16b, v9.16b\n" - "ldr q5, [x28, #0x40]\n" - "ldr q6, [x28, #0x50]\n" - ".inst 0x4e8aa411 // smmla v17.4s, v0.16b, v10.16b\n" - ".inst 0x4e84a415 // smmla v21.4s, v0.16b, v4.16b\n" - "ldr q7, [x28, #0x60]\n" - "ldr q8, [x28, #0x70]\n" - ".inst 0x4e85a412 // smmla v18.4s, v0.16b, v5.16b\n" - ".inst 0x4e86a416 // smmla v22.4s, v0.16b, v6.16b\n" - ".inst 0x4e87a413 // smmla v19.4s, v0.16b, v7.16b\n" - ".inst 0x4e88a417 // smmla v23.4s, v0.16b, v8.16b\n" + ".inst 0x4e9aa414 // smmla v20.4s, v0.16b, v26.16b\n" + "ldr q27, [x28, #0x40]\n" + "ldr q26, [x28, #0x50]\n" + ".inst 0x4e99a411 // smmla v17.4s, v0.16b, v25.16b\n" + ".inst 0x4e98a415 // smmla v21.4s, v0.16b, v24.16b\n" + "ldr q25, [x28, #0x60]\n" + "ldr q24, [x28, #0x70]\n" + ".inst 0x4e9ba412 // smmla v18.4s, v0.16b, v27.16b\n" + ".inst 0x4e9aa416 // smmla v22.4s, v0.16b, v26.16b\n" + ".inst 0x4e99a413 // smmla v19.4s, v0.16b, v25.16b\n" + ".inst 0x4e98a417 // smmla v23.4s, v0.16b, v24.16b\n" "add x28, x28, #0x80\n" "bge 12b\n" "14:" // Height 1: Multiply loop: Skip odd blocks @@ -264,26 +263,26 @@ void a64_hybrid_s8qa_mmla_4x16 ( "17:" // Height 1: Multiply loop: Ragged operand read: partial_1_0 "ldr b1, [x24, #0x0]\n" "18:" // Height 1: Multiply loop: Ragged operand read: Done - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v0.2d, v1.2d, v24.2d\n" "tbnz %x[flags], #31, 19f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" "19:" // Height 1: Multiply loop: unique 4: skip row sum - "ldr q10, [x28, #0x0]\n" - "ldr q4, [x28, #0x10]\n" - ".inst 0x4e8aa410 // smmla v16.4s, v0.16b, v10.16b\n" - ".inst 0x4e84a414 // smmla v20.4s, v0.16b, v4.16b\n" - "ldr q5, [x28, #0x20]\n" - "ldr q6, [x28, #0x30]\n" - ".inst 0x4e85a411 // smmla v17.4s, v0.16b, v5.16b\n" - ".inst 0x4e86a415 // smmla v21.4s, v0.16b, v6.16b\n" - "ldr q7, [x28, #0x40]\n" - "ldr q8, [x28, #0x50]\n" - ".inst 0x4e87a412 // smmla v18.4s, v0.16b, v7.16b\n" - ".inst 0x4e88a416 // smmla v22.4s, v0.16b, v8.16b\n" - "ldr q9, [x28, #0x60]\n" - "ldr q10, [x28, #0x70]\n" - ".inst 0x4e89a413 // smmla v19.4s, v0.16b, v9.16b\n" - ".inst 0x4e8aa417 // smmla v23.4s, v0.16b, v10.16b\n" + "ldr q25, [x28, #0x0]\n" + "ldr q24, [x28, #0x10]\n" + ".inst 0x4e99a410 // smmla v16.4s, v0.16b, v25.16b\n" + ".inst 0x4e98a414 // smmla v20.4s, v0.16b, v24.16b\n" + "ldr q25, [x28, #0x20]\n" + "ldr q24, [x28, #0x30]\n" + ".inst 0x4e99a411 // smmla v17.4s, v0.16b, v25.16b\n" + ".inst 0x4e98a415 // smmla v21.4s, v0.16b, v24.16b\n" + "ldr q25, [x28, #0x40]\n" + "ldr q24, [x28, #0x50]\n" + ".inst 0x4e99a412 // smmla v18.4s, v0.16b, v25.16b\n" + ".inst 0x4e98a416 // smmla v22.4s, v0.16b, v24.16b\n" + "ldr q25, [x28, #0x60]\n" + "ldr q24, [x28, #0x70]\n" + ".inst 0x4e99a413 // smmla v19.4s, v0.16b, v25.16b\n" + ".inst 0x4e98a417 // smmla v23.4s, v0.16b, v24.16b\n" "add x28, x28, #0x80\n" "20:" // Height 1: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -297,75 +296,75 @@ void a64_hybrid_s8qa_mmla_4x16 ( "uzp1 v19.2d, v19.2d, v23.2d\n" "mov v23.16b, v16.16b\n" "tbnz %x[flags], #31, 21f\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1r { v1.4s }, [x23]\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1r { v16.4s }, [x20]\n" "addp v11.4s, v11.4s, v11.4s\n" - "neg v1.4s, v1.4s\n" + "neg v16.4s, v16.4s\n" "dup v11.4s, v11.s[0]\n" - "mul v11.4s, v11.4s, v1.4s\n" + "mul v11.4s, v11.4s, v16.4s\n" "21:" // Height 1: skip row sum fixup - "ldr q0, [x10, #0x0]\n" - "ldr q1, [x10, #0x10]\n" + "ldr q24, [x10, #0x0]\n" + "ldr q22, [x10, #0x10]\n" "add v23.4s, v23.4s, v11.4s\n" "add v17.4s, v17.4s, v11.4s\n" - "ldr q2, [x10, #0x20]\n" - "ldr q3, [x10, #0x30]\n" + "ldr q21, [x10, #0x20]\n" + "ldr q20, [x10, #0x30]\n" "add v18.4s, v18.4s, v11.4s\n" "add v19.4s, v19.4s, v11.4s\n" - "add x23, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v16.4s }, [x20]\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add v23.4s, v23.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x23]\n" - "add v19.4s, v19.4s, v3.4s\n" - "sqrdmulh v23.4s, v23.4s, v4.4s\n" + "add v23.4s, v23.4s, v24.4s\n" + "add v17.4s, v17.4s, v22.4s\n" + "add v18.4s, v18.4s, v21.4s\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "add v19.4s, v19.4s, v20.4s\n" + "sqrdmulh v23.4s, v23.4s, v16.4s\n" "add x10, x10, #0x40\n" - "sqrdmulh v17.4s, v17.4s, v4.4s\n" - "sqrdmulh v18.4s, v18.4s, v4.4s\n" - "sqrdmulh v19.4s, v19.4s, v4.4s\n" + "sqrdmulh v17.4s, v17.4s, v16.4s\n" + "sqrdmulh v18.4s, v18.4s, v16.4s\n" + "sqrdmulh v19.4s, v19.4s, v16.4s\n" "tbz %x[flags], #5, 22f\n" - "and v4.16b, v23.16b, v0.16b\n" - "and v5.16b, v17.16b, v0.16b\n" - "and v6.16b, v18.16b, v0.16b\n" - "and v7.16b, v19.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" + "and v22.16b, v23.16b, v0.16b\n" + "and v21.16b, v17.16b, v0.16b\n" + "and v20.16b, v18.16b, v0.16b\n" + "and v16.16b, v19.16b, v0.16b\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "sshr v21.4s, v21.4s, #0x1f\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v23.4s, v23.4s, v22.4s\n" + "sqadd v17.4s, v17.4s, v21.4s\n" + "sqadd v18.4s, v18.4s, v20.4s\n" + "sqadd v19.4s, v19.4s, v16.4s\n" "22:" // Height 1: no shift correction - "add x23, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v21.4s }, [x20]\n" "srshl v23.4s, v23.4s, v0.4s\n" "srshl v17.4s, v17.4s, v0.4s\n" "srshl v18.4s, v18.4s, v0.4s\n" "srshl v19.4s, v19.4s, v0.4s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x23]\n" - "add v23.4s, v23.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add x23, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x23]\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v20.4s }, [x20]\n" + "add v23.4s, v23.4s, v21.4s\n" + "add v17.4s, v17.4s, v21.4s\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v16.4s }, [x20]\n" + "add v18.4s, v18.4s, v21.4s\n" + "add v19.4s, v19.4s, v21.4s\n" "cmp x9, #0x10\n" - "smin v23.4s, v23.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" + "smin v23.4s, v23.4s, v20.4s\n" + "smin v17.4s, v17.4s, v20.4s\n" + "smin v18.4s, v18.4s, v20.4s\n" + "smin v19.4s, v19.4s, v20.4s\n" + "smax v23.4s, v23.4s, v16.4s\n" + "smax v17.4s, v17.4s, v16.4s\n" + "smax v18.4s, v18.4s, v16.4s\n" + "smax v19.4s, v19.4s, v16.4s\n" "uzp1 v23.8h, v23.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" - "uzp1 v23.16b, v23.16b, v17.16b\n" + "uzp1 v16.8h, v18.8h, v19.8h\n" + "uzp1 v23.16b, v23.16b, v16.16b\n" "bge 31f\n" "tbz x9, #3, 26f\n" "str d23, [x27], #0x8\n" @@ -442,12 +441,12 @@ void a64_hybrid_s8qa_mmla_4x16 ( "36:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 37f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" "cbnz x26, 38f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -455,7 +454,7 @@ void a64_hybrid_s8qa_mmla_4x16 ( "b 38f\n" "37:" // Height 2: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20\n" + "add x23, x24, x21\n" "38:" // Height 2: input setup done "cmp x25, #0x10\n" "blt 43f\n" @@ -473,34 +472,34 @@ void a64_hybrid_s8qa_mmla_4x16 ( "39:" // Height 2: Multiply loop: Main loop head "trn1 v0.2d, v1.2d, v2.2d\n" ".inst 0x4e85a410 // smmla v16.4s, v0.16b, v5.16b\n" - "ldr q5, [x28, #0x70]\n" + "ldr q25, [x28, #0x70]\n" ".inst 0x4e86a414 // smmla v20.4s, v0.16b, v6.16b\n" - "ldr q6, [x28, #0x80]\n" + "ldr q24, [x28, #0x80]\n" "trn2 v1.2d, v1.2d, v2.2d\n" ".inst 0x4e87a411 // smmla v17.4s, v0.16b, v7.16b\n" - "ldr q7, [x28, #0x90]\n" + "ldr q30, [x28, #0x90]\n" ".inst 0x4e88a415 // smmla v21.4s, v0.16b, v8.16b\n" - "ldr q8, [x28, #0xa0]\n" + "ldr q29, [x28, #0xa0]\n" ".inst 0x4e89a412 // smmla v18.4s, v0.16b, v9.16b\n" - "ldr q9, [x28, #0xb0]\n" + "ldr q28, [x28, #0xb0]\n" ".inst 0x4e8aa416 // smmla v22.4s, v0.16b, v10.16b\n" - "ldr q10, [x28, #0xc0]\n" + "ldr q27, [x28, #0xc0]\n" ".inst 0x4e84a413 // smmla v19.4s, v0.16b, v4.16b\n" - "ldr q4, [x28, #0xd0]\n" - ".inst 0x4e85a417 // smmla v23.4s, v0.16b, v5.16b\n" - "ldr q5, [x28, #0xe0]\n" - ".inst 0x4e86a430 // smmla v16.4s, v1.16b, v6.16b\n" - "ldr q6, [x28, #0xf0]\n" - ".inst 0x4e87a434 // smmla v20.4s, v1.16b, v7.16b\n" + "ldr q26, [x28, #0xd0]\n" + ".inst 0x4e99a417 // smmla v23.4s, v0.16b, v25.16b\n" + "ldr q25, [x28, #0xe0]\n" + ".inst 0x4e98a430 // smmla v16.4s, v1.16b, v24.16b\n" + "ldr q24, [x28, #0xf0]\n" + ".inst 0x4e9ea434 // smmla v20.4s, v1.16b, v30.16b\n" "add x24, x24, #0x10\n" - ".inst 0x4e88a431 // smmla v17.4s, v1.16b, v8.16b\n" + ".inst 0x4e9da431 // smmla v17.4s, v1.16b, v29.16b\n" "add x23, x23, #0x10\n" - ".inst 0x4e89a435 // smmla v21.4s, v1.16b, v9.16b\n" + ".inst 0x4e9ca435 // smmla v21.4s, v1.16b, v28.16b\n" "add x28, x28, #0x100\n" - ".inst 0x4e8aa432 // smmla v18.4s, v1.16b, v10.16b\n" - ".inst 0x4e84a436 // smmla v22.4s, v1.16b, v4.16b\n" - ".inst 0x4e85a433 // smmla v19.4s, v1.16b, v5.16b\n" - ".inst 0x4e86a437 // smmla v23.4s, v1.16b, v6.16b\n" + ".inst 0x4e9ba432 // smmla v18.4s, v1.16b, v27.16b\n" + ".inst 0x4e9aa436 // smmla v22.4s, v1.16b, v26.16b\n" + ".inst 0x4e99a433 // smmla v19.4s, v1.16b, v25.16b\n" + ".inst 0x4e98a437 // smmla v23.4s, v1.16b, v24.16b\n" "tbnz %x[flags], #31, 40f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f942b // sdot v11.4s, v1.16b, v15.16b\n" @@ -522,35 +521,35 @@ void a64_hybrid_s8qa_mmla_4x16 ( "41:" // Height 2: Multiply loop: Single iteration only "trn1 v0.2d, v1.2d, v2.2d\n" ".inst 0x4e85a410 // smmla v16.4s, v0.16b, v5.16b\n" - "ldr q5, [x28, #0x70]\n" + "ldr q25, [x28, #0x70]\n" ".inst 0x4e86a414 // smmla v20.4s, v0.16b, v6.16b\n" - "ldr q6, [x28, #0x80]\n" + "ldr q24, [x28, #0x80]\n" "trn2 v1.2d, v1.2d, v2.2d\n" ".inst 0x4e87a411 // smmla v17.4s, v0.16b, v7.16b\n" - "ldr q7, [x28, #0x90]\n" + "ldr q30, [x28, #0x90]\n" ".inst 0x4e88a415 // smmla v21.4s, v0.16b, v8.16b\n" - "ldr q8, [x28, #0xa0]\n" + "ldr q29, [x28, #0xa0]\n" ".inst 0x4e89a412 // smmla v18.4s, v0.16b, v9.16b\n" - "ldr q9, [x28, #0xb0]\n" + "ldr q28, [x28, #0xb0]\n" ".inst 0x4e8aa416 // smmla v22.4s, v0.16b, v10.16b\n" - "ldr q10, [x28, #0xc0]\n" + "ldr q27, [x28, #0xc0]\n" ".inst 0x4e84a413 // smmla v19.4s, v0.16b, v4.16b\n" - "ldr q4, [x28, #0xd0]\n" - ".inst 0x4e85a417 // smmla v23.4s, v0.16b, v5.16b\n" - "ldr q5, [x28, #0xe0]\n" - ".inst 0x4e86a430 // smmla v16.4s, v1.16b, v6.16b\n" - "ldr q6, [x28, #0xf0]\n" + "ldr q26, [x28, #0xd0]\n" + ".inst 0x4e99a417 // smmla v23.4s, v0.16b, v25.16b\n" + "ldr q25, [x28, #0xe0]\n" + ".inst 0x4e98a430 // smmla v16.4s, v1.16b, v24.16b\n" + "ldr q24, [x28, #0xf0]\n" "sub x25, x25, #0x10\n" - ".inst 0x4e87a434 // smmla v20.4s, v1.16b, v7.16b\n" - ".inst 0x4e88a431 // smmla v17.4s, v1.16b, v8.16b\n" + ".inst 0x4e9ea434 // smmla v20.4s, v1.16b, v30.16b\n" + ".inst 0x4e9da431 // smmla v17.4s, v1.16b, v29.16b\n" "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" - ".inst 0x4e89a435 // smmla v21.4s, v1.16b, v9.16b\n" - ".inst 0x4e8aa432 // smmla v18.4s, v1.16b, v10.16b\n" + ".inst 0x4e9ca435 // smmla v21.4s, v1.16b, v28.16b\n" + ".inst 0x4e9ba432 // smmla v18.4s, v1.16b, v27.16b\n" "add x28, x28, #0x100\n" - ".inst 0x4e84a436 // smmla v22.4s, v1.16b, v4.16b\n" - ".inst 0x4e85a433 // smmla v19.4s, v1.16b, v5.16b\n" - ".inst 0x4e86a437 // smmla v23.4s, v1.16b, v6.16b\n" + ".inst 0x4e9aa436 // smmla v22.4s, v1.16b, v26.16b\n" + ".inst 0x4e99a433 // smmla v19.4s, v1.16b, v25.16b\n" + ".inst 0x4e98a437 // smmla v23.4s, v1.16b, v24.16b\n" "tbnz %x[flags], #31, 42f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f942b // sdot v11.4s, v1.16b, v15.16b\n" @@ -562,30 +561,30 @@ void a64_hybrid_s8qa_mmla_4x16 ( "cmp x25, #0x8\n" "blt 46f\n" "44:" // Height 2: Multiply loop: Odd block loop - "ldr d1, [x24], #0x8\n" - "ldr d2, [x23], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" + "ldr d25, [x24], #0x8\n" + "ldr d24, [x23], #0x8\n" + "trn1 v0.2d, v25.2d, v24.2d\n" "tbnz %x[flags], #31, 45f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" "45:" // Height 2: Multiply loop: unique 7: skip row sum - "ldr q8, [x28, #0x0]\n" - "ldr q9, [x28, #0x10]\n" - ".inst 0x4e88a410 // smmla v16.4s, v0.16b, v8.16b\n" + "ldr q24, [x28, #0x0]\n" + "ldr q26, [x28, #0x10]\n" + ".inst 0x4e98a410 // smmla v16.4s, v0.16b, v24.16b\n" "sub x25, x25, #0x8\n" - "ldr q10, [x28, #0x20]\n" - "ldr q4, [x28, #0x30]\n" + "ldr q25, [x28, #0x20]\n" + "ldr q24, [x28, #0x30]\n" "cmp x25, #0x8\n" - ".inst 0x4e89a414 // smmla v20.4s, v0.16b, v9.16b\n" - "ldr q5, [x28, #0x40]\n" - "ldr q6, [x28, #0x50]\n" - ".inst 0x4e8aa411 // smmla v17.4s, v0.16b, v10.16b\n" - ".inst 0x4e84a415 // smmla v21.4s, v0.16b, v4.16b\n" - "ldr q7, [x28, #0x60]\n" - "ldr q8, [x28, #0x70]\n" - ".inst 0x4e85a412 // smmla v18.4s, v0.16b, v5.16b\n" - ".inst 0x4e86a416 // smmla v22.4s, v0.16b, v6.16b\n" - ".inst 0x4e87a413 // smmla v19.4s, v0.16b, v7.16b\n" - ".inst 0x4e88a417 // smmla v23.4s, v0.16b, v8.16b\n" + ".inst 0x4e9aa414 // smmla v20.4s, v0.16b, v26.16b\n" + "ldr q27, [x28, #0x40]\n" + "ldr q26, [x28, #0x50]\n" + ".inst 0x4e99a411 // smmla v17.4s, v0.16b, v25.16b\n" + ".inst 0x4e98a415 // smmla v21.4s, v0.16b, v24.16b\n" + "ldr q25, [x28, #0x60]\n" + "ldr q24, [x28, #0x70]\n" + ".inst 0x4e9ba412 // smmla v18.4s, v0.16b, v27.16b\n" + ".inst 0x4e9aa416 // smmla v22.4s, v0.16b, v26.16b\n" + ".inst 0x4e99a413 // smmla v19.4s, v0.16b, v25.16b\n" + ".inst 0x4e98a417 // smmla v23.4s, v0.16b, v24.16b\n" "add x28, x28, #0x80\n" "bge 44b\n" "46:" // Height 2: Multiply loop: Skip odd blocks @@ -621,22 +620,22 @@ void a64_hybrid_s8qa_mmla_4x16 ( "tbnz %x[flags], #31, 51f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" "51:" // Height 2: Multiply loop: unique 8: skip row sum - "ldr q10, [x28, #0x0]\n" - "ldr q4, [x28, #0x10]\n" - ".inst 0x4e8aa410 // smmla v16.4s, v0.16b, v10.16b\n" - ".inst 0x4e84a414 // smmla v20.4s, v0.16b, v4.16b\n" - "ldr q5, [x28, #0x20]\n" - "ldr q6, [x28, #0x30]\n" - ".inst 0x4e85a411 // smmla v17.4s, v0.16b, v5.16b\n" - ".inst 0x4e86a415 // smmla v21.4s, v0.16b, v6.16b\n" - "ldr q7, [x28, #0x40]\n" - "ldr q8, [x28, #0x50]\n" - ".inst 0x4e87a412 // smmla v18.4s, v0.16b, v7.16b\n" - ".inst 0x4e88a416 // smmla v22.4s, v0.16b, v8.16b\n" - "ldr q9, [x28, #0x60]\n" - "ldr q10, [x28, #0x70]\n" - ".inst 0x4e89a413 // smmla v19.4s, v0.16b, v9.16b\n" - ".inst 0x4e8aa417 // smmla v23.4s, v0.16b, v10.16b\n" + "ldr q25, [x28, #0x0]\n" + "ldr q24, [x28, #0x10]\n" + ".inst 0x4e99a410 // smmla v16.4s, v0.16b, v25.16b\n" + ".inst 0x4e98a414 // smmla v20.4s, v0.16b, v24.16b\n" + "ldr q25, [x28, #0x20]\n" + "ldr q24, [x28, #0x30]\n" + ".inst 0x4e99a411 // smmla v17.4s, v0.16b, v25.16b\n" + ".inst 0x4e98a415 // smmla v21.4s, v0.16b, v24.16b\n" + "ldr q25, [x28, #0x40]\n" + "ldr q24, [x28, #0x50]\n" + ".inst 0x4e99a412 // smmla v18.4s, v0.16b, v25.16b\n" + ".inst 0x4e98a416 // smmla v22.4s, v0.16b, v24.16b\n" + "ldr q25, [x28, #0x60]\n" + "ldr q24, [x28, #0x70]\n" + ".inst 0x4e99a413 // smmla v19.4s, v0.16b, v25.16b\n" + ".inst 0x4e98a417 // smmla v23.4s, v0.16b, v24.16b\n" "add x28, x28, #0x80\n" "52:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -644,127 +643,127 @@ void a64_hybrid_s8qa_mmla_4x16 ( "cmp x26, x20\n" "bne 36b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "uzp1 v4.2d, v16.2d, v20.2d\n" - "add x22, x27, x20\n" + "uzp1 v24.2d, v16.2d, v20.2d\n" + "add x23, x27, x20\n" "uzp2 v16.2d, v16.2d, v20.2d\n" "uzp1 v20.2d, v17.2d, v21.2d\n" "uzp2 v17.2d, v17.2d, v21.2d\n" "prfm pstl1keep, [x27, #0x0]\n" - "prfm pstl1keep, [x22, #0x0]\n" + "prfm pstl1keep, [x23, #0x0]\n" "uzp1 v21.2d, v18.2d, v22.2d\n" "uzp2 v18.2d, v18.2d, v22.2d\n" "uzp1 v22.2d, v19.2d, v23.2d\n" "uzp2 v19.2d, v19.2d, v23.2d\n" - "mov v23.16b, v4.16b\n" + "mov v23.16b, v24.16b\n" "tbnz %x[flags], #31, 53f\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1r { v2.4s }, [x23]\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1r { v24.4s }, [x20]\n" "addp v11.4s, v11.4s, v11.4s\n" - "neg v2.4s, v2.4s\n" + "neg v24.4s, v24.4s\n" "dup v12.4s, v11.s[3]\n" "dup v11.4s, v11.s[0]\n" - "mul v11.4s, v11.4s, v2.4s\n" - "mul v12.4s, v12.4s, v2.4s\n" + "mul v11.4s, v11.4s, v24.4s\n" + "mul v12.4s, v12.4s, v24.4s\n" "53:" // Height 2: skip row sum fixup - "ldr q0, [x10, #0x0]\n" - "ldr q1, [x10, #0x10]\n" + "ldr q28, [x10, #0x0]\n" + "ldr q27, [x10, #0x10]\n" "add v23.4s, v23.4s, v11.4s\n" "add v20.4s, v20.4s, v11.4s\n" - "ldr q2, [x10, #0x20]\n" - "ldr q3, [x10, #0x30]\n" + "ldr q26, [x10, #0x20]\n" + "ldr q25, [x10, #0x30]\n" "add v21.4s, v21.4s, v11.4s\n" "add v22.4s, v22.4s, v11.4s\n" "add v16.4s, v16.4s, v12.4s\n" "add v17.4s, v17.4s, v12.4s\n" - "add x23, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v24.4s }, [x20]\n" "add v18.4s, v18.4s, v12.4s\n" "add v19.4s, v19.4s, v12.4s\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" - "add v23.4s, v23.4s, v0.4s\n" - "add v20.4s, v20.4s, v1.4s\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "add v23.4s, v23.4s, v28.4s\n" + "add v20.4s, v20.4s, v27.4s\n" "add x10, x10, #0x40\n" - "add v21.4s, v21.4s, v2.4s\n" - "add v22.4s, v22.4s, v3.4s\n" - "add v16.4s, v16.4s, v0.4s\n" - "ld1r { v0.4s }, [x23]\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" - "sqrdmulh v23.4s, v23.4s, v4.4s\n" - "sqrdmulh v20.4s, v20.4s, v4.4s\n" - "sqrdmulh v21.4s, v21.4s, v4.4s\n" - "sqrdmulh v22.4s, v22.4s, v4.4s\n" - "sqrdmulh v16.4s, v16.4s, v4.4s\n" - "sqrdmulh v17.4s, v17.4s, v4.4s\n" - "sqrdmulh v18.4s, v18.4s, v4.4s\n" - "sqrdmulh v19.4s, v19.4s, v4.4s\n" + "add v21.4s, v21.4s, v26.4s\n" + "add v22.4s, v22.4s, v25.4s\n" + "add v16.4s, v16.4s, v28.4s\n" + "ld1r { v0.4s }, [x20]\n" + "add v17.4s, v17.4s, v27.4s\n" + "add v18.4s, v18.4s, v26.4s\n" + "add v19.4s, v19.4s, v25.4s\n" + "sqrdmulh v23.4s, v23.4s, v24.4s\n" + "sqrdmulh v20.4s, v20.4s, v24.4s\n" + "sqrdmulh v21.4s, v21.4s, v24.4s\n" + "sqrdmulh v22.4s, v22.4s, v24.4s\n" + "sqrdmulh v16.4s, v16.4s, v24.4s\n" + "sqrdmulh v17.4s, v17.4s, v24.4s\n" + "sqrdmulh v18.4s, v18.4s, v24.4s\n" + "sqrdmulh v19.4s, v19.4s, v24.4s\n" "tbz %x[flags], #5, 54f\n" - "and v4.16b, v23.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "and v5.16b, v20.16b, v0.16b\n" - "and v6.16b, v21.16b, v0.16b\n" - "and v7.16b, v22.16b, v0.16b\n" - "and v8.16b, v16.16b, v0.16b\n" - "and v9.16b, v17.16b, v0.16b\n" - "and v10.16b, v18.16b, v0.16b\n" - "and v4.16b, v19.16b, v0.16b\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v9.4s, v9.4s, #0x1f\n" - "sshr v10.4s, v10.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v20.4s, v20.4s, v5.4s\n" - "sqadd v21.4s, v21.4s, v6.4s\n" - "sqadd v22.4s, v22.4s, v7.4s\n" - "sqadd v16.4s, v16.4s, v8.4s\n" - "sqadd v17.4s, v17.4s, v9.4s\n" - "sqadd v18.4s, v18.4s, v10.4s\n" - "sqadd v19.4s, v19.4s, v4.4s\n" + "and v24.16b, v23.16b, v0.16b\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sqadd v23.4s, v23.4s, v24.4s\n" + "and v30.16b, v20.16b, v0.16b\n" + "and v29.16b, v21.16b, v0.16b\n" + "and v28.16b, v22.16b, v0.16b\n" + "and v27.16b, v16.16b, v0.16b\n" + "and v26.16b, v17.16b, v0.16b\n" + "and v25.16b, v18.16b, v0.16b\n" + "and v24.16b, v19.16b, v0.16b\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sshr v27.4s, v27.4s, #0x1f\n" + "sshr v26.4s, v26.4s, #0x1f\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v30.4s\n" + "sqadd v21.4s, v21.4s, v29.4s\n" + "sqadd v22.4s, v22.4s, v28.4s\n" + "sqadd v16.4s, v16.4s, v27.4s\n" + "sqadd v17.4s, v17.4s, v26.4s\n" + "sqadd v18.4s, v18.4s, v25.4s\n" + "sqadd v19.4s, v19.4s, v24.4s\n" "54:" // Height 2: no shift correction - "add x23, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v26.4s }, [x20]\n" "srshl v23.4s, v23.4s, v0.4s\n" "srshl v20.4s, v20.4s, v0.4s\n" "srshl v21.4s, v21.4s, v0.4s\n" "srshl v22.4s, v22.4s, v0.4s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x23]\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v25.4s }, [x20]\n" "srshl v16.4s, v16.4s, v0.4s\n" "srshl v17.4s, v17.4s, v0.4s\n" - "add x23, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x23]\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v24.4s }, [x20]\n" "srshl v18.4s, v18.4s, v0.4s\n" "srshl v19.4s, v19.4s, v0.4s\n" "cmp x9, #0x10\n" - "add v23.4s, v23.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" + "add v23.4s, v23.4s, v26.4s\n" + "add v20.4s, v20.4s, v26.4s\n" + "add v21.4s, v21.4s, v26.4s\n" + "add v22.4s, v22.4s, v26.4s\n" + "add v16.4s, v16.4s, v26.4s\n" + "add v17.4s, v17.4s, v26.4s\n" + "add v18.4s, v18.4s, v26.4s\n" + "add v19.4s, v19.4s, v26.4s\n" + "smin v23.4s, v23.4s, v25.4s\n" + "smin v20.4s, v20.4s, v25.4s\n" + "smin v21.4s, v21.4s, v25.4s\n" + "smin v22.4s, v22.4s, v25.4s\n" + "smin v16.4s, v16.4s, v25.4s\n" + "smin v17.4s, v17.4s, v25.4s\n" + "smin v18.4s, v18.4s, v25.4s\n" + "smin v19.4s, v19.4s, v25.4s\n" + "smax v23.4s, v23.4s, v24.4s\n" + "smax v20.4s, v20.4s, v24.4s\n" + "smax v21.4s, v21.4s, v24.4s\n" + "smax v22.4s, v22.4s, v24.4s\n" + "smax v16.4s, v16.4s, v24.4s\n" + "smax v17.4s, v17.4s, v24.4s\n" + "smax v18.4s, v18.4s, v24.4s\n" + "smax v19.4s, v19.4s, v24.4s\n" "uzp1 v23.8h, v23.8h, v20.8h\n" "uzp1 v20.8h, v21.8h, v22.8h\n" "uzp1 v16.8h, v16.8h, v17.8h\n" @@ -774,68 +773,68 @@ void a64_hybrid_s8qa_mmla_4x16 ( "bge 63f\n" "tbz x9, #3, 58f\n" "str d23, [x27], #0x8\n" - "str d16, [x22], #0x8\n" + "str d16, [x23], #0x8\n" "tbz x9, #2, 56f\n" "st1 { v23.s }[2], [x27], #0x4\n" - "st1 { v16.s }[2], [x22], #0x4\n" + "st1 { v16.s }[2], [x23], #0x4\n" "tbz x9, #1, 55f\n" "st1 { v23.h }[6], [x27], #0x2\n" - "st1 { v16.h }[6], [x22], #0x2\n" + "st1 { v16.h }[6], [x23], #0x2\n" "tbz x9, #0, 62f\n" "st1 { v23.b }[14], [x27]\n" - "st1 { v16.b }[14], [x22]\n" + "st1 { v16.b }[14], [x23]\n" "b 62f\n" "55:" // Height 2: Partial direct writeback: partial_1_12 "tbz x9, #0, 62f\n" "st1 { v23.b }[12], [x27]\n" - "st1 { v16.b }[12], [x22]\n" + "st1 { v16.b }[12], [x23]\n" "b 62f\n" "56:" // Height 2: Partial direct writeback: partial_2_8 "tbz x9, #1, 57f\n" "st1 { v23.h }[4], [x27], #0x2\n" - "st1 { v16.h }[4], [x22], #0x2\n" + "st1 { v16.h }[4], [x23], #0x2\n" "tbz x9, #0, 62f\n" "st1 { v23.b }[10], [x27]\n" - "st1 { v16.b }[10], [x22]\n" + "st1 { v16.b }[10], [x23]\n" "b 62f\n" "57:" // Height 2: Partial direct writeback: partial_1_8 "tbz x9, #0, 62f\n" "st1 { v23.b }[8], [x27]\n" - "st1 { v16.b }[8], [x22]\n" + "st1 { v16.b }[8], [x23]\n" "b 62f\n" "58:" // Height 2: Partial direct writeback: partial_4_0 "tbz x9, #2, 60f\n" "str s23, [x27], #0x4\n" - "str s16, [x22], #0x4\n" + "str s16, [x23], #0x4\n" "tbz x9, #1, 59f\n" "st1 { v23.h }[2], [x27], #0x2\n" - "st1 { v16.h }[2], [x22], #0x2\n" + "st1 { v16.h }[2], [x23], #0x2\n" "tbz x9, #0, 62f\n" "st1 { v23.b }[6], [x27]\n" - "st1 { v16.b }[6], [x22]\n" + "st1 { v16.b }[6], [x23]\n" "b 62f\n" "59:" // Height 2: Partial direct writeback: partial_1_4 "tbz x9, #0, 62f\n" "st1 { v23.b }[4], [x27]\n" - "st1 { v16.b }[4], [x22]\n" + "st1 { v16.b }[4], [x23]\n" "b 62f\n" "60:" // Height 2: Partial direct writeback: partial_2_0 "tbz x9, #1, 61f\n" "str h23, [x27], #0x2\n" - "str h16, [x22], #0x2\n" + "str h16, [x23], #0x2\n" "tbz x9, #0, 62f\n" "st1 { v23.b }[2], [x27]\n" - "st1 { v16.b }[2], [x22]\n" + "st1 { v16.b }[2], [x23]\n" "b 62f\n" "61:" // Height 2: Partial direct writeback: partial_1_0 "str b23, [x27, #0x0]\n" - "str b16, [x22, #0x0]\n" + "str b16, [x23, #0x0]\n" "62:" // Height 2: Partial direct writeback: Done "b 64f\n" "63:" // Height 2: Full writeback "str q23, [x27, #0x0]\n" "add x27, x27, #0x10\n" - "str q16, [x22, #0x0]\n" + "str q16, [x23, #0x0]\n" "64:" // Height 2: Writeback done "subs x9, x9, #0x10\n" "bgt 34b\n" @@ -872,13 +871,13 @@ void a64_hybrid_s8qa_mmla_4x16 ( "68:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 69f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" "cbnz x26, 70f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -887,8 +886,8 @@ void a64_hybrid_s8qa_mmla_4x16 ( "b 70f\n" "69:" // Height 3: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" "70:" // Height 3: input setup done "cmp x25, #0x10\n" "blt 75f\n" @@ -909,12 +908,12 @@ void a64_hybrid_s8qa_mmla_4x16 ( ".inst 0x4e85a410 // smmla v16.4s, v0.16b, v5.16b\n" "trn1 v2.2d, v3.2d, v4.2d\n" ".inst 0x4e85a458 // smmla v24.4s, v2.16b, v5.16b\n" - "ldr q5, [x28, #0x70]\n" + "ldr q14, [x28, #0x70]\n" ".inst 0x4e86a414 // smmla v20.4s, v0.16b, v6.16b\n" "trn2 v3.2d, v3.2d, v4.2d\n" - "ldr q4, [x28, #0x60]\n" + "ldr q5, [x28, #0x60]\n" ".inst 0x4e86a45c // smmla v28.4s, v2.16b, v6.16b\n" - "ldr q6, [x28, #0x80]\n" + "ldr q4, [x28, #0x80]\n" ".inst 0x4e87a411 // smmla v17.4s, v0.16b, v7.16b\n" ".inst 0x4e87a459 // smmla v25.4s, v2.16b, v7.16b\n" "ldr q7, [x28, #0x90]\n" @@ -930,15 +929,15 @@ void a64_hybrid_s8qa_mmla_4x16 ( ".inst 0x4e8aa416 // smmla v22.4s, v0.16b, v10.16b\n" ".inst 0x4e8aa45e // smmla v30.4s, v2.16b, v10.16b\n" "ldr q10, [x28, #0xc0]\n" - ".inst 0x4e84a413 // smmla v19.4s, v0.16b, v4.16b\n" - ".inst 0x4e84a45b // smmla v27.4s, v2.16b, v4.16b\n" - "ldr q4, [x28, #0xd0]\n" - ".inst 0x4e85a417 // smmla v23.4s, v0.16b, v5.16b\n" - ".inst 0x4e85a45f // smmla v31.4s, v2.16b, v5.16b\n" + ".inst 0x4e85a413 // smmla v19.4s, v0.16b, v5.16b\n" + ".inst 0x4e85a45b // smmla v27.4s, v2.16b, v5.16b\n" + "ldr q6, [x28, #0xd0]\n" + ".inst 0x4e8ea417 // smmla v23.4s, v0.16b, v14.16b\n" + ".inst 0x4e8ea45f // smmla v31.4s, v2.16b, v14.16b\n" "ldr q5, [x28, #0xe0]\n" - ".inst 0x4e86a430 // smmla v16.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a478 // smmla v24.4s, v3.16b, v6.16b\n" - "ldr q6, [x28, #0xf0]\n" + ".inst 0x4e84a430 // smmla v16.4s, v1.16b, v4.16b\n" + ".inst 0x4e84a478 // smmla v24.4s, v3.16b, v4.16b\n" + "ldr q4, [x28, #0xf0]\n" "add x28, x28, #0x100\n" ".inst 0x4e87a434 // smmla v20.4s, v1.16b, v7.16b\n" ".inst 0x4e87a47c // smmla v28.4s, v3.16b, v7.16b\n" @@ -948,12 +947,12 @@ void a64_hybrid_s8qa_mmla_4x16 ( ".inst 0x4e89a47d // smmla v29.4s, v3.16b, v9.16b\n" ".inst 0x4e8aa432 // smmla v18.4s, v1.16b, v10.16b\n" ".inst 0x4e8aa47a // smmla v26.4s, v3.16b, v10.16b\n" - ".inst 0x4e84a436 // smmla v22.4s, v1.16b, v4.16b\n" - ".inst 0x4e84a47e // smmla v30.4s, v3.16b, v4.16b\n" + ".inst 0x4e86a436 // smmla v22.4s, v1.16b, v6.16b\n" + ".inst 0x4e86a47e // smmla v30.4s, v3.16b, v6.16b\n" ".inst 0x4e85a433 // smmla v19.4s, v1.16b, v5.16b\n" ".inst 0x4e85a47b // smmla v27.4s, v3.16b, v5.16b\n" - ".inst 0x4e86a437 // smmla v23.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a47f // smmla v31.4s, v3.16b, v6.16b\n" + ".inst 0x4e84a437 // smmla v23.4s, v1.16b, v4.16b\n" + ".inst 0x4e84a47f // smmla v31.4s, v3.16b, v4.16b\n" "tbnz %x[flags], #31, 72f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n" @@ -981,12 +980,12 @@ void a64_hybrid_s8qa_mmla_4x16 ( ".inst 0x4e85a410 // smmla v16.4s, v0.16b, v5.16b\n" "trn1 v2.2d, v3.2d, v4.2d\n" ".inst 0x4e85a458 // smmla v24.4s, v2.16b, v5.16b\n" - "ldr q5, [x28, #0x70]\n" + "ldr q14, [x28, #0x70]\n" ".inst 0x4e86a414 // smmla v20.4s, v0.16b, v6.16b\n" "trn2 v3.2d, v3.2d, v4.2d\n" - "ldr q4, [x28, #0x60]\n" + "ldr q5, [x28, #0x60]\n" ".inst 0x4e86a45c // smmla v28.4s, v2.16b, v6.16b\n" - "ldr q6, [x28, #0x80]\n" + "ldr q4, [x28, #0x80]\n" ".inst 0x4e87a411 // smmla v17.4s, v0.16b, v7.16b\n" ".inst 0x4e87a459 // smmla v25.4s, v2.16b, v7.16b\n" "ldr q7, [x28, #0x90]\n" @@ -1003,15 +1002,15 @@ void a64_hybrid_s8qa_mmla_4x16 ( ".inst 0x4e8aa45e // smmla v30.4s, v2.16b, v10.16b\n" "ldr q10, [x28, #0xc0]\n" "add x22, x22, #0x10\n" - ".inst 0x4e84a413 // smmla v19.4s, v0.16b, v4.16b\n" - ".inst 0x4e84a45b // smmla v27.4s, v2.16b, v4.16b\n" - "ldr q4, [x28, #0xd0]\n" - ".inst 0x4e85a417 // smmla v23.4s, v0.16b, v5.16b\n" - ".inst 0x4e85a45f // smmla v31.4s, v2.16b, v5.16b\n" + ".inst 0x4e85a413 // smmla v19.4s, v0.16b, v5.16b\n" + ".inst 0x4e85a45b // smmla v27.4s, v2.16b, v5.16b\n" + "ldr q6, [x28, #0xd0]\n" + ".inst 0x4e8ea417 // smmla v23.4s, v0.16b, v14.16b\n" + ".inst 0x4e8ea45f // smmla v31.4s, v2.16b, v14.16b\n" "ldr q5, [x28, #0xe0]\n" - ".inst 0x4e86a430 // smmla v16.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a478 // smmla v24.4s, v3.16b, v6.16b\n" - "ldr q6, [x28, #0xf0]\n" + ".inst 0x4e84a430 // smmla v16.4s, v1.16b, v4.16b\n" + ".inst 0x4e84a478 // smmla v24.4s, v3.16b, v4.16b\n" + "ldr q4, [x28, #0xf0]\n" "add x28, x28, #0x100\n" ".inst 0x4e87a434 // smmla v20.4s, v1.16b, v7.16b\n" ".inst 0x4e87a47c // smmla v28.4s, v3.16b, v7.16b\n" @@ -1021,12 +1020,12 @@ void a64_hybrid_s8qa_mmla_4x16 ( ".inst 0x4e89a47d // smmla v29.4s, v3.16b, v9.16b\n" ".inst 0x4e8aa432 // smmla v18.4s, v1.16b, v10.16b\n" ".inst 0x4e8aa47a // smmla v26.4s, v3.16b, v10.16b\n" - ".inst 0x4e84a436 // smmla v22.4s, v1.16b, v4.16b\n" - ".inst 0x4e84a47e // smmla v30.4s, v3.16b, v4.16b\n" + ".inst 0x4e86a436 // smmla v22.4s, v1.16b, v6.16b\n" + ".inst 0x4e86a47e // smmla v30.4s, v3.16b, v6.16b\n" ".inst 0x4e85a433 // smmla v19.4s, v1.16b, v5.16b\n" ".inst 0x4e85a47b // smmla v27.4s, v3.16b, v5.16b\n" - ".inst 0x4e86a437 // smmla v23.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a47f // smmla v31.4s, v3.16b, v6.16b\n" + ".inst 0x4e84a437 // smmla v23.4s, v1.16b, v4.16b\n" + ".inst 0x4e84a47f // smmla v31.4s, v3.16b, v4.16b\n" "tbnz %x[flags], #31, 74f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n" @@ -1042,41 +1041,41 @@ void a64_hybrid_s8qa_mmla_4x16 ( "blt 78f\n" "76:" // Height 3: Multiply loop: Odd block loop "ldr d1, [x24], #0x8\n" - "ldr d2, [x23], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "ldr d3, [x22], #0x8\n" - "trn1 v2.2d, v3.2d, v7.2d\n" + "ldr d0, [x23], #0x8\n" + "trn1 v0.2d, v1.2d, v0.2d\n" + "ldr d1, [x22], #0x8\n" + "trn1 v2.2d, v1.2d, v2.2d\n" "tbnz %x[flags], #31, 77f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n" "77:" // Height 3: Multiply loop: unique 11: skip row sum - "ldr q8, [x28, #0x0]\n" - "ldr q9, [x28, #0x10]\n" - ".inst 0x4e88a410 // smmla v16.4s, v0.16b, v8.16b\n" - ".inst 0x4e88a458 // smmla v24.4s, v2.16b, v8.16b\n" - "ldr q10, [x28, #0x20]\n" - "ldr q4, [x28, #0x30]\n" + "ldr q3, [x28, #0x0]\n" + "ldr q1, [x28, #0x10]\n" + ".inst 0x4e83a410 // smmla v16.4s, v0.16b, v3.16b\n" + ".inst 0x4e83a458 // smmla v24.4s, v2.16b, v3.16b\n" + "ldr q7, [x28, #0x20]\n" + "ldr q6, [x28, #0x30]\n" "sub x25, x25, #0x8\n" "cmp x25, #0x8\n" "ldr q5, [x28, #0x40]\n" - "ldr q6, [x28, #0x50]\n" - ".inst 0x4e89a414 // smmla v20.4s, v0.16b, v9.16b\n" - ".inst 0x4e89a45c // smmla v28.4s, v2.16b, v9.16b\n" - "ldr q7, [x28, #0x60]\n" - "ldr q8, [x28, #0x70]\n" - ".inst 0x4e8aa411 // smmla v17.4s, v0.16b, v10.16b\n" - ".inst 0x4e8aa459 // smmla v25.4s, v2.16b, v10.16b\n" - ".inst 0x4e84a415 // smmla v21.4s, v0.16b, v4.16b\n" - ".inst 0x4e84a45d // smmla v29.4s, v2.16b, v4.16b\n" + "ldr q4, [x28, #0x50]\n" + ".inst 0x4e81a414 // smmla v20.4s, v0.16b, v1.16b\n" + ".inst 0x4e81a45c // smmla v28.4s, v2.16b, v1.16b\n" + "ldr q3, [x28, #0x60]\n" + "ldr q1, [x28, #0x70]\n" + ".inst 0x4e87a411 // smmla v17.4s, v0.16b, v7.16b\n" + ".inst 0x4e87a459 // smmla v25.4s, v2.16b, v7.16b\n" + ".inst 0x4e86a415 // smmla v21.4s, v0.16b, v6.16b\n" + ".inst 0x4e86a45d // smmla v29.4s, v2.16b, v6.16b\n" "add x28, x28, #0x80\n" ".inst 0x4e85a412 // smmla v18.4s, v0.16b, v5.16b\n" ".inst 0x4e85a45a // smmla v26.4s, v2.16b, v5.16b\n" - ".inst 0x4e86a416 // smmla v22.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a45e // smmla v30.4s, v2.16b, v6.16b\n" - ".inst 0x4e87a413 // smmla v19.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a45b // smmla v27.4s, v2.16b, v7.16b\n" - ".inst 0x4e88a417 // smmla v23.4s, v0.16b, v8.16b\n" - ".inst 0x4e88a45f // smmla v31.4s, v2.16b, v8.16b\n" + ".inst 0x4e84a416 // smmla v22.4s, v0.16b, v4.16b\n" + ".inst 0x4e84a45e // smmla v30.4s, v2.16b, v4.16b\n" + ".inst 0x4e83a413 // smmla v19.4s, v0.16b, v3.16b\n" + ".inst 0x4e83a45b // smmla v27.4s, v2.16b, v3.16b\n" + ".inst 0x4e81a417 // smmla v23.4s, v0.16b, v1.16b\n" + ".inst 0x4e81a45f // smmla v31.4s, v2.16b, v1.16b\n" "bge 76b\n" "78:" // Height 3: Multiply loop: Skip odd blocks "cbz x25, 84f\n" @@ -1115,52 +1114,52 @@ void a64_hybrid_s8qa_mmla_4x16 ( "ldr b3, [x22, #0x0]\n" "82:" // Height 3: Multiply loop: Ragged operand read: Done "trn1 v0.2d, v1.2d, v2.2d\n" - "trn1 v2.2d, v3.2d, v9.2d\n" + "trn1 v2.2d, v3.2d, v4.2d\n" "tbnz %x[flags], #31, 83f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n" "83:" // Height 3: Multiply loop: unique 12: skip row sum - "ldr q10, [x28, #0x0]\n" - "ldr q4, [x28, #0x10]\n" - ".inst 0x4e8aa410 // smmla v16.4s, v0.16b, v10.16b\n" - ".inst 0x4e8aa458 // smmla v24.4s, v2.16b, v10.16b\n" - "ldr q5, [x28, #0x20]\n" + "ldr q1, [x28, #0x0]\n" + "ldr q3, [x28, #0x10]\n" + ".inst 0x4e81a410 // smmla v16.4s, v0.16b, v1.16b\n" + ".inst 0x4e81a458 // smmla v24.4s, v2.16b, v1.16b\n" + "ldr q1, [x28, #0x20]\n" "ldr q6, [x28, #0x30]\n" - ".inst 0x4e84a414 // smmla v20.4s, v0.16b, v4.16b\n" - ".inst 0x4e84a45c // smmla v28.4s, v2.16b, v4.16b\n" - "ldr q7, [x28, #0x40]\n" - "ldr q8, [x28, #0x50]\n" - ".inst 0x4e85a411 // smmla v17.4s, v0.16b, v5.16b\n" - ".inst 0x4e85a459 // smmla v25.4s, v2.16b, v5.16b\n" - "ldr q9, [x28, #0x60]\n" - "ldr q10, [x28, #0x70]\n" + ".inst 0x4e83a414 // smmla v20.4s, v0.16b, v3.16b\n" + ".inst 0x4e83a45c // smmla v28.4s, v2.16b, v3.16b\n" + "ldr q5, [x28, #0x40]\n" + "ldr q4, [x28, #0x50]\n" + ".inst 0x4e81a411 // smmla v17.4s, v0.16b, v1.16b\n" + ".inst 0x4e81a459 // smmla v25.4s, v2.16b, v1.16b\n" + "ldr q3, [x28, #0x60]\n" + "ldr q1, [x28, #0x70]\n" ".inst 0x4e86a415 // smmla v21.4s, v0.16b, v6.16b\n" ".inst 0x4e86a45d // smmla v29.4s, v2.16b, v6.16b\n" - ".inst 0x4e87a412 // smmla v18.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a45a // smmla v26.4s, v2.16b, v7.16b\n" + ".inst 0x4e85a412 // smmla v18.4s, v0.16b, v5.16b\n" + ".inst 0x4e85a45a // smmla v26.4s, v2.16b, v5.16b\n" "add x28, x28, #0x80\n" - ".inst 0x4e88a416 // smmla v22.4s, v0.16b, v8.16b\n" - ".inst 0x4e88a45e // smmla v30.4s, v2.16b, v8.16b\n" - ".inst 0x4e89a413 // smmla v19.4s, v0.16b, v9.16b\n" - ".inst 0x4e89a45b // smmla v27.4s, v2.16b, v9.16b\n" - ".inst 0x4e8aa417 // smmla v23.4s, v0.16b, v10.16b\n" - ".inst 0x4e8aa45f // smmla v31.4s, v2.16b, v10.16b\n" + ".inst 0x4e84a416 // smmla v22.4s, v0.16b, v4.16b\n" + ".inst 0x4e84a45e // smmla v30.4s, v2.16b, v4.16b\n" + ".inst 0x4e83a413 // smmla v19.4s, v0.16b, v3.16b\n" + ".inst 0x4e83a45b // smmla v27.4s, v2.16b, v3.16b\n" + ".inst 0x4e81a417 // smmla v23.4s, v0.16b, v1.16b\n" + ".inst 0x4e81a45f // smmla v31.4s, v2.16b, v1.16b\n" "84:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x26, x26, #0x1\n" "cmp x26, x20\n" "bne 68b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "uzp1 v4.2d, v16.2d, v20.2d\n" - "add x22, x27, x20\n" - "add x21, x22, x20\n" + "uzp1 v0.2d, v16.2d, v20.2d\n" + "add x23, x27, x20\n" + "add x22, x23, x20\n" "uzp2 v16.2d, v16.2d, v20.2d\n" "uzp1 v20.2d, v17.2d, v21.2d\n" "prfm pstl1keep, [x27, #0x0]\n" - "prfm pstl1keep, [x22, #0x0]\n" + "prfm pstl1keep, [x23, #0x0]\n" "uzp2 v17.2d, v17.2d, v21.2d\n" "uzp1 v21.2d, v18.2d, v22.2d\n" - "prfm pstl1keep, [x21, #0x0]\n" + "prfm pstl1keep, [x22, #0x0]\n" "uzp2 v18.2d, v18.2d, v22.2d\n" "uzp1 v22.2d, v19.2d, v23.2d\n" "uzp2 v19.2d, v19.2d, v23.2d\n" @@ -1168,116 +1167,116 @@ void a64_hybrid_s8qa_mmla_4x16 ( "uzp1 v25.2d, v25.2d, v29.2d\n" "uzp1 v26.2d, v26.2d, v30.2d\n" "uzp1 v27.2d, v27.2d, v31.2d\n" - "mov v31.16b, v4.16b\n" + "mov v31.16b, v0.16b\n" "tbnz %x[flags], #31, 85f\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1r { v3.4s }, [x23]\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1r { v23.4s }, [x20]\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v13.4s, v13.4s, v13.4s\n" - "neg v3.4s, v3.4s\n" + "neg v23.4s, v23.4s\n" "dup v12.4s, v11.s[3]\n" "dup v11.4s, v11.s[0]\n" "dup v13.4s, v13.s[0]\n" - "mul v11.4s, v11.4s, v3.4s\n" - "mul v12.4s, v12.4s, v3.4s\n" - "mul v13.4s, v13.4s, v3.4s\n" + "mul v11.4s, v11.4s, v23.4s\n" + "mul v12.4s, v12.4s, v23.4s\n" + "mul v13.4s, v13.4s, v23.4s\n" "85:" // Height 3: skip row sum fixup "ldr q0, [x10, #0x0]\n" - "ldr q1, [x10, #0x10]\n" + "ldr q30, [x10, #0x10]\n" "add v31.4s, v31.4s, v11.4s\n" "add v20.4s, v20.4s, v11.4s\n" - "ldr q2, [x10, #0x20]\n" - "ldr q3, [x10, #0x30]\n" + "ldr q29, [x10, #0x20]\n" + "ldr q28, [x10, #0x30]\n" "add v21.4s, v21.4s, v11.4s\n" "add v22.4s, v22.4s, v11.4s\n" "add v16.4s, v16.4s, v12.4s\n" "add v17.4s, v17.4s, v12.4s\n" - "add x23, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v23.4s }, [x20]\n" "add v18.4s, v18.4s, v12.4s\n" "add v19.4s, v19.4s, v12.4s\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" "add v24.4s, v24.4s, v13.4s\n" "add v25.4s, v25.4s, v13.4s\n" "add x10, x10, #0x40\n" "add v26.4s, v26.4s, v13.4s\n" "add v27.4s, v27.4s, v13.4s\n" "add v31.4s, v31.4s, v0.4s\n" - "add v20.4s, v20.4s, v1.4s\n" - "add v21.4s, v21.4s, v2.4s\n" - "add v22.4s, v22.4s, v3.4s\n" + "add v20.4s, v20.4s, v30.4s\n" + "add v21.4s, v21.4s, v29.4s\n" + "add v22.4s, v22.4s, v28.4s\n" "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" + "add v17.4s, v17.4s, v30.4s\n" + "add v18.4s, v18.4s, v29.4s\n" + "add v19.4s, v19.4s, v28.4s\n" "add v24.4s, v24.4s, v0.4s\n" - "ld1r { v0.4s }, [x23]\n" - "add v25.4s, v25.4s, v1.4s\n" - "add v26.4s, v26.4s, v2.4s\n" - "add v27.4s, v27.4s, v3.4s\n" - "sqrdmulh v31.4s, v31.4s, v4.4s\n" - "sqrdmulh v20.4s, v20.4s, v4.4s\n" - "sqrdmulh v21.4s, v21.4s, v4.4s\n" - "sqrdmulh v22.4s, v22.4s, v4.4s\n" - "sqrdmulh v16.4s, v16.4s, v4.4s\n" - "sqrdmulh v17.4s, v17.4s, v4.4s\n" - "sqrdmulh v18.4s, v18.4s, v4.4s\n" - "sqrdmulh v19.4s, v19.4s, v4.4s\n" - "sqrdmulh v24.4s, v24.4s, v4.4s\n" - "sqrdmulh v25.4s, v25.4s, v4.4s\n" - "sqrdmulh v26.4s, v26.4s, v4.4s\n" - "sqrdmulh v27.4s, v27.4s, v4.4s\n" + "ld1r { v0.4s }, [x20]\n" + "add v25.4s, v25.4s, v30.4s\n" + "add v26.4s, v26.4s, v29.4s\n" + "add v27.4s, v27.4s, v28.4s\n" + "sqrdmulh v31.4s, v31.4s, v23.4s\n" + "sqrdmulh v20.4s, v20.4s, v23.4s\n" + "sqrdmulh v21.4s, v21.4s, v23.4s\n" + "sqrdmulh v22.4s, v22.4s, v23.4s\n" + "sqrdmulh v16.4s, v16.4s, v23.4s\n" + "sqrdmulh v17.4s, v17.4s, v23.4s\n" + "sqrdmulh v18.4s, v18.4s, v23.4s\n" + "sqrdmulh v19.4s, v19.4s, v23.4s\n" + "sqrdmulh v24.4s, v24.4s, v23.4s\n" + "sqrdmulh v25.4s, v25.4s, v23.4s\n" + "sqrdmulh v26.4s, v26.4s, v23.4s\n" + "sqrdmulh v27.4s, v27.4s, v23.4s\n" "tbz %x[flags], #5, 86f\n" - "and v4.16b, v31.16b, v0.16b\n" - "and v5.16b, v20.16b, v0.16b\n" - "and v6.16b, v21.16b, v0.16b\n" - "and v7.16b, v22.16b, v0.16b\n" - "and v8.16b, v16.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sqadd v31.4s, v31.4s, v4.4s\n" - "sqadd v20.4s, v20.4s, v5.4s\n" - "sqadd v21.4s, v21.4s, v6.4s\n" - "sqadd v22.4s, v22.4s, v7.4s\n" - "sqadd v16.4s, v16.4s, v8.4s\n" - "and v9.16b, v17.16b, v0.16b\n" - "and v10.16b, v18.16b, v0.16b\n" - "and v4.16b, v19.16b, v0.16b\n" - "and v5.16b, v24.16b, v0.16b\n" - "and v6.16b, v25.16b, v0.16b\n" - "and v7.16b, v26.16b, v0.16b\n" - "and v8.16b, v27.16b, v0.16b\n" - "sshr v9.4s, v9.4s, #0x1f\n" - "sshr v10.4s, v10.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sqadd v17.4s, v17.4s, v9.4s\n" - "sqadd v18.4s, v18.4s, v10.4s\n" - "sqadd v19.4s, v19.4s, v4.4s\n" - "sqadd v24.4s, v24.4s, v5.4s\n" - "sqadd v25.4s, v25.4s, v6.4s\n" - "sqadd v26.4s, v26.4s, v7.4s\n" - "sqadd v27.4s, v27.4s, v8.4s\n" + "and v1.16b, v31.16b, v0.16b\n" + "and v30.16b, v20.16b, v0.16b\n" + "and v29.16b, v21.16b, v0.16b\n" + "and v28.16b, v22.16b, v0.16b\n" + "and v23.16b, v16.16b, v0.16b\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "sqadd v31.4s, v31.4s, v1.4s\n" + "sqadd v20.4s, v20.4s, v30.4s\n" + "sqadd v21.4s, v21.4s, v29.4s\n" + "sqadd v22.4s, v22.4s, v28.4s\n" + "sqadd v16.4s, v16.4s, v23.4s\n" + "and v3.16b, v17.16b, v0.16b\n" + "and v2.16b, v18.16b, v0.16b\n" + "and v1.16b, v19.16b, v0.16b\n" + "and v30.16b, v24.16b, v0.16b\n" + "and v29.16b, v25.16b, v0.16b\n" + "and v28.16b, v26.16b, v0.16b\n" + "and v23.16b, v27.16b, v0.16b\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "sqadd v17.4s, v17.4s, v3.4s\n" + "sqadd v18.4s, v18.4s, v2.4s\n" + "sqadd v19.4s, v19.4s, v1.4s\n" + "sqadd v24.4s, v24.4s, v30.4s\n" + "sqadd v25.4s, v25.4s, v29.4s\n" + "sqadd v26.4s, v26.4s, v28.4s\n" + "sqadd v27.4s, v27.4s, v23.4s\n" "86:" // Height 3: no shift correction - "add x23, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v29.4s }, [x20]\n" "srshl v31.4s, v31.4s, v0.4s\n" "srshl v20.4s, v20.4s, v0.4s\n" "srshl v21.4s, v21.4s, v0.4s\n" "srshl v22.4s, v22.4s, v0.4s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x23]\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v28.4s }, [x20]\n" "srshl v16.4s, v16.4s, v0.4s\n" "srshl v17.4s, v17.4s, v0.4s\n" - "add x23, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x23]\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v23.4s }, [x20]\n" "srshl v18.4s, v18.4s, v0.4s\n" "srshl v19.4s, v19.4s, v0.4s\n" "cmp x9, #0x10\n" @@ -1285,132 +1284,132 @@ void a64_hybrid_s8qa_mmla_4x16 ( "srshl v25.4s, v25.4s, v0.4s\n" "srshl v26.4s, v26.4s, v0.4s\n" "srshl v27.4s, v27.4s, v0.4s\n" - "add v31.4s, v31.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add v24.4s, v24.4s, v4.4s\n" - "add v25.4s, v25.4s, v4.4s\n" - "add v26.4s, v26.4s, v4.4s\n" - "add v27.4s, v27.4s, v4.4s\n" - "smin v31.4s, v31.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smin v24.4s, v24.4s, v6.4s\n" - "smin v25.4s, v25.4s, v6.4s\n" - "smin v26.4s, v26.4s, v6.4s\n" - "smin v27.4s, v27.4s, v6.4s\n" - "smax v31.4s, v31.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" - "smax v24.4s, v24.4s, v5.4s\n" - "smax v25.4s, v25.4s, v5.4s\n" - "smax v26.4s, v26.4s, v5.4s\n" - "smax v27.4s, v27.4s, v5.4s\n" + "add v31.4s, v31.4s, v29.4s\n" + "add v20.4s, v20.4s, v29.4s\n" + "add v21.4s, v21.4s, v29.4s\n" + "add v22.4s, v22.4s, v29.4s\n" + "add v16.4s, v16.4s, v29.4s\n" + "add v17.4s, v17.4s, v29.4s\n" + "add v18.4s, v18.4s, v29.4s\n" + "add v19.4s, v19.4s, v29.4s\n" + "add v24.4s, v24.4s, v29.4s\n" + "add v25.4s, v25.4s, v29.4s\n" + "add v26.4s, v26.4s, v29.4s\n" + "add v27.4s, v27.4s, v29.4s\n" + "smin v31.4s, v31.4s, v28.4s\n" + "smin v20.4s, v20.4s, v28.4s\n" + "smin v21.4s, v21.4s, v28.4s\n" + "smin v22.4s, v22.4s, v28.4s\n" + "smin v16.4s, v16.4s, v28.4s\n" + "smin v17.4s, v17.4s, v28.4s\n" + "smin v18.4s, v18.4s, v28.4s\n" + "smin v19.4s, v19.4s, v28.4s\n" + "smin v24.4s, v24.4s, v28.4s\n" + "smin v25.4s, v25.4s, v28.4s\n" + "smin v26.4s, v26.4s, v28.4s\n" + "smin v27.4s, v27.4s, v28.4s\n" + "smax v31.4s, v31.4s, v23.4s\n" + "smax v20.4s, v20.4s, v23.4s\n" + "smax v21.4s, v21.4s, v23.4s\n" + "smax v22.4s, v22.4s, v23.4s\n" + "smax v16.4s, v16.4s, v23.4s\n" + "smax v17.4s, v17.4s, v23.4s\n" + "smax v18.4s, v18.4s, v23.4s\n" + "smax v19.4s, v19.4s, v23.4s\n" + "smax v24.4s, v24.4s, v23.4s\n" + "smax v25.4s, v25.4s, v23.4s\n" + "smax v26.4s, v26.4s, v23.4s\n" + "smax v27.4s, v27.4s, v23.4s\n" "uzp1 v31.8h, v31.8h, v20.8h\n" "uzp1 v20.8h, v21.8h, v22.8h\n" "uzp1 v16.8h, v16.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v18.8h, v18.8h, v19.8h\n" "uzp1 v24.8h, v24.8h, v25.8h\n" - "uzp1 v25.8h, v26.8h, v27.8h\n" + "uzp1 v17.8h, v26.8h, v27.8h\n" "uzp1 v31.16b, v31.16b, v20.16b\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" - "uzp1 v24.16b, v24.16b, v25.16b\n" + "uzp1 v16.16b, v16.16b, v18.16b\n" + "uzp1 v24.16b, v24.16b, v17.16b\n" "bge 95f\n" "tbz x9, #3, 90f\n" "str d31, [x27], #0x8\n" - "str d16, [x22], #0x8\n" - "str d24, [x21], #0x8\n" + "str d16, [x23], #0x8\n" + "str d24, [x22], #0x8\n" "tbz x9, #2, 88f\n" "st1 { v31.s }[2], [x27], #0x4\n" - "st1 { v16.s }[2], [x22], #0x4\n" - "st1 { v24.s }[2], [x21], #0x4\n" + "st1 { v16.s }[2], [x23], #0x4\n" + "st1 { v24.s }[2], [x22], #0x4\n" "tbz x9, #1, 87f\n" "st1 { v31.h }[6], [x27], #0x2\n" - "st1 { v16.h }[6], [x22], #0x2\n" - "st1 { v24.h }[6], [x21], #0x2\n" + "st1 { v16.h }[6], [x23], #0x2\n" + "st1 { v24.h }[6], [x22], #0x2\n" "tbz x9, #0, 94f\n" "st1 { v31.b }[14], [x27]\n" - "st1 { v16.b }[14], [x22]\n" - "st1 { v24.b }[14], [x21]\n" + "st1 { v16.b }[14], [x23]\n" + "st1 { v24.b }[14], [x22]\n" "b 94f\n" "87:" // Height 3: Partial direct writeback: partial_1_12 "tbz x9, #0, 94f\n" "st1 { v31.b }[12], [x27]\n" - "st1 { v16.b }[12], [x22]\n" - "st1 { v24.b }[12], [x21]\n" + "st1 { v16.b }[12], [x23]\n" + "st1 { v24.b }[12], [x22]\n" "b 94f\n" "88:" // Height 3: Partial direct writeback: partial_2_8 "tbz x9, #1, 89f\n" "st1 { v31.h }[4], [x27], #0x2\n" - "st1 { v16.h }[4], [x22], #0x2\n" - "st1 { v24.h }[4], [x21], #0x2\n" + "st1 { v16.h }[4], [x23], #0x2\n" + "st1 { v24.h }[4], [x22], #0x2\n" "tbz x9, #0, 94f\n" "st1 { v31.b }[10], [x27]\n" - "st1 { v16.b }[10], [x22]\n" - "st1 { v24.b }[10], [x21]\n" + "st1 { v16.b }[10], [x23]\n" + "st1 { v24.b }[10], [x22]\n" "b 94f\n" "89:" // Height 3: Partial direct writeback: partial_1_8 "tbz x9, #0, 94f\n" "st1 { v31.b }[8], [x27]\n" - "st1 { v16.b }[8], [x22]\n" - "st1 { v24.b }[8], [x21]\n" + "st1 { v16.b }[8], [x23]\n" + "st1 { v24.b }[8], [x22]\n" "b 94f\n" "90:" // Height 3: Partial direct writeback: partial_4_0 "tbz x9, #2, 92f\n" "str s31, [x27], #0x4\n" - "str s16, [x22], #0x4\n" - "str s24, [x21], #0x4\n" + "str s16, [x23], #0x4\n" + "str s24, [x22], #0x4\n" "tbz x9, #1, 91f\n" "st1 { v31.h }[2], [x27], #0x2\n" - "st1 { v16.h }[2], [x22], #0x2\n" - "st1 { v24.h }[2], [x21], #0x2\n" + "st1 { v16.h }[2], [x23], #0x2\n" + "st1 { v24.h }[2], [x22], #0x2\n" "tbz x9, #0, 94f\n" "st1 { v31.b }[6], [x27]\n" - "st1 { v16.b }[6], [x22]\n" - "st1 { v24.b }[6], [x21]\n" + "st1 { v16.b }[6], [x23]\n" + "st1 { v24.b }[6], [x22]\n" "b 94f\n" "91:" // Height 3: Partial direct writeback: partial_1_4 "tbz x9, #0, 94f\n" "st1 { v31.b }[4], [x27]\n" - "st1 { v16.b }[4], [x22]\n" - "st1 { v24.b }[4], [x21]\n" + "st1 { v16.b }[4], [x23]\n" + "st1 { v24.b }[4], [x22]\n" "b 94f\n" "92:" // Height 3: Partial direct writeback: partial_2_0 "tbz x9, #1, 93f\n" "str h31, [x27], #0x2\n" - "str h16, [x22], #0x2\n" - "str h24, [x21], #0x2\n" + "str h16, [x23], #0x2\n" + "str h24, [x22], #0x2\n" "tbz x9, #0, 94f\n" "st1 { v31.b }[2], [x27]\n" - "st1 { v16.b }[2], [x22]\n" - "st1 { v24.b }[2], [x21]\n" + "st1 { v16.b }[2], [x23]\n" + "st1 { v24.b }[2], [x22]\n" "b 94f\n" "93:" // Height 3: Partial direct writeback: partial_1_0 "str b31, [x27, #0x0]\n" - "str b16, [x22, #0x0]\n" - "str b24, [x21, #0x0]\n" + "str b16, [x23, #0x0]\n" + "str b24, [x22, #0x0]\n" "94:" // Height 3: Partial direct writeback: Done "b 96f\n" "95:" // Height 3: Full writeback "str q31, [x27, #0x0]\n" "add x27, x27, #0x10\n" - "str q16, [x22, #0x0]\n" - "str q24, [x21, #0x0]\n" + "str q16, [x23, #0x0]\n" + "str q24, [x22, #0x0]\n" "96:" // Height 3: Writeback done "subs x9, x9, #0x10\n" "bgt 66b\n" @@ -1451,14 +1450,14 @@ void a64_hybrid_s8qa_mmla_4x16 ( "100:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 101f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" - "ldr x21, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" + "ldr x21, [x20, #0x18]\n" "cbnz x26, 102f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -1468,9 +1467,9 @@ void a64_hybrid_s8qa_mmla_4x16 ( "b 102f\n" "101:" // Height 4: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" + "add x21, x22, x21\n" "102:" // Height 4: input setup done "cmp x25, #0x10\n" "blt 107f\n" @@ -1630,42 +1629,42 @@ void a64_hybrid_s8qa_mmla_4x16 ( "blt 110f\n" "108:" // Height 4: Multiply loop: Odd block loop "ldr d1, [x24], #0x8\n" - "ldr d2, [x23], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "ldr d3, [x22], #0x8\n" - "ldr d7, [x21], #0x8\n" - "trn1 v2.2d, v3.2d, v7.2d\n" + "ldr d0, [x23], #0x8\n" + "trn1 v0.2d, v1.2d, v0.2d\n" + "ldr d2, [x22], #0x8\n" + "ldr d1, [x21], #0x8\n" + "trn1 v2.2d, v2.2d, v1.2d\n" "tbnz %x[flags], #31, 109f\n" ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n" "109:" // Height 4: Multiply loop: unique 15: skip row sum - "ldr q8, [x28, #0x0]\n" - "ldr q9, [x28, #0x10]\n" - ".inst 0x4e88a410 // smmla v16.4s, v0.16b, v8.16b\n" - ".inst 0x4e88a458 // smmla v24.4s, v2.16b, v8.16b\n" - "ldr q10, [x28, #0x20]\n" - "ldr q4, [x28, #0x30]\n" + "ldr q3, [x28, #0x0]\n" + "ldr q1, [x28, #0x10]\n" + ".inst 0x4e83a410 // smmla v16.4s, v0.16b, v3.16b\n" + ".inst 0x4e83a458 // smmla v24.4s, v2.16b, v3.16b\n" + "ldr q7, [x28, #0x20]\n" + "ldr q6, [x28, #0x30]\n" "sub x25, x25, #0x8\n" "cmp x25, #0x8\n" "ldr q5, [x28, #0x40]\n" - "ldr q6, [x28, #0x50]\n" - ".inst 0x4e89a414 // smmla v20.4s, v0.16b, v9.16b\n" - ".inst 0x4e89a45c // smmla v28.4s, v2.16b, v9.16b\n" - "ldr q7, [x28, #0x60]\n" - "ldr q8, [x28, #0x70]\n" - ".inst 0x4e8aa411 // smmla v17.4s, v0.16b, v10.16b\n" - ".inst 0x4e8aa459 // smmla v25.4s, v2.16b, v10.16b\n" - ".inst 0x4e84a415 // smmla v21.4s, v0.16b, v4.16b\n" - ".inst 0x4e84a45d // smmla v29.4s, v2.16b, v4.16b\n" + "ldr q4, [x28, #0x50]\n" + ".inst 0x4e81a414 // smmla v20.4s, v0.16b, v1.16b\n" + ".inst 0x4e81a45c // smmla v28.4s, v2.16b, v1.16b\n" + "ldr q3, [x28, #0x60]\n" + "ldr q1, [x28, #0x70]\n" + ".inst 0x4e87a411 // smmla v17.4s, v0.16b, v7.16b\n" + ".inst 0x4e87a459 // smmla v25.4s, v2.16b, v7.16b\n" + ".inst 0x4e86a415 // smmla v21.4s, v0.16b, v6.16b\n" + ".inst 0x4e86a45d // smmla v29.4s, v2.16b, v6.16b\n" "add x28, x28, #0x80\n" ".inst 0x4e85a412 // smmla v18.4s, v0.16b, v5.16b\n" ".inst 0x4e85a45a // smmla v26.4s, v2.16b, v5.16b\n" - ".inst 0x4e86a416 // smmla v22.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a45e // smmla v30.4s, v2.16b, v6.16b\n" - ".inst 0x4e87a413 // smmla v19.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a45b // smmla v27.4s, v2.16b, v7.16b\n" - ".inst 0x4e88a417 // smmla v23.4s, v0.16b, v8.16b\n" - ".inst 0x4e88a45f // smmla v31.4s, v2.16b, v8.16b\n" + ".inst 0x4e84a416 // smmla v22.4s, v0.16b, v4.16b\n" + ".inst 0x4e84a45e // smmla v30.4s, v2.16b, v4.16b\n" + ".inst 0x4e83a413 // smmla v19.4s, v0.16b, v3.16b\n" + ".inst 0x4e83a45b // smmla v27.4s, v2.16b, v3.16b\n" + ".inst 0x4e81a417 // smmla v23.4s, v0.16b, v1.16b\n" + ".inst 0x4e81a45f // smmla v31.4s, v2.16b, v1.16b\n" "bge 108b\n" "110:" // Height 4: Multiply loop: Skip odd blocks "cbz x25, 116f\n" @@ -1716,51 +1715,51 @@ void a64_hybrid_s8qa_mmla_4x16 ( ".inst 0x4e8f940b // sdot v11.4s, v0.16b, v15.16b\n" ".inst 0x4e8f944d // sdot v13.4s, v2.16b, v15.16b\n" "115:" // Height 4: Multiply loop: unique 16: skip row sum - "ldr q10, [x28, #0x0]\n" - "ldr q4, [x28, #0x10]\n" - ".inst 0x4e8aa410 // smmla v16.4s, v0.16b, v10.16b\n" - ".inst 0x4e8aa458 // smmla v24.4s, v2.16b, v10.16b\n" - "ldr q5, [x28, #0x20]\n" + "ldr q1, [x28, #0x0]\n" + "ldr q3, [x28, #0x10]\n" + ".inst 0x4e81a410 // smmla v16.4s, v0.16b, v1.16b\n" + ".inst 0x4e81a458 // smmla v24.4s, v2.16b, v1.16b\n" + "ldr q1, [x28, #0x20]\n" "ldr q6, [x28, #0x30]\n" - ".inst 0x4e84a414 // smmla v20.4s, v0.16b, v4.16b\n" - ".inst 0x4e84a45c // smmla v28.4s, v2.16b, v4.16b\n" - "ldr q7, [x28, #0x40]\n" - "ldr q8, [x28, #0x50]\n" - ".inst 0x4e85a411 // smmla v17.4s, v0.16b, v5.16b\n" - ".inst 0x4e85a459 // smmla v25.4s, v2.16b, v5.16b\n" - "ldr q9, [x28, #0x60]\n" - "ldr q10, [x28, #0x70]\n" + ".inst 0x4e83a414 // smmla v20.4s, v0.16b, v3.16b\n" + ".inst 0x4e83a45c // smmla v28.4s, v2.16b, v3.16b\n" + "ldr q5, [x28, #0x40]\n" + "ldr q4, [x28, #0x50]\n" + ".inst 0x4e81a411 // smmla v17.4s, v0.16b, v1.16b\n" + ".inst 0x4e81a459 // smmla v25.4s, v2.16b, v1.16b\n" + "ldr q3, [x28, #0x60]\n" + "ldr q1, [x28, #0x70]\n" ".inst 0x4e86a415 // smmla v21.4s, v0.16b, v6.16b\n" ".inst 0x4e86a45d // smmla v29.4s, v2.16b, v6.16b\n" - ".inst 0x4e87a412 // smmla v18.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a45a // smmla v26.4s, v2.16b, v7.16b\n" + ".inst 0x4e85a412 // smmla v18.4s, v0.16b, v5.16b\n" + ".inst 0x4e85a45a // smmla v26.4s, v2.16b, v5.16b\n" "add x28, x28, #0x80\n" - ".inst 0x4e88a416 // smmla v22.4s, v0.16b, v8.16b\n" - ".inst 0x4e88a45e // smmla v30.4s, v2.16b, v8.16b\n" - ".inst 0x4e89a413 // smmla v19.4s, v0.16b, v9.16b\n" - ".inst 0x4e89a45b // smmla v27.4s, v2.16b, v9.16b\n" - ".inst 0x4e8aa417 // smmla v23.4s, v0.16b, v10.16b\n" - ".inst 0x4e8aa45f // smmla v31.4s, v2.16b, v10.16b\n" + ".inst 0x4e84a416 // smmla v22.4s, v0.16b, v4.16b\n" + ".inst 0x4e84a45e // smmla v30.4s, v2.16b, v4.16b\n" + ".inst 0x4e83a413 // smmla v19.4s, v0.16b, v3.16b\n" + ".inst 0x4e83a45b // smmla v27.4s, v2.16b, v3.16b\n" + ".inst 0x4e81a417 // smmla v23.4s, v0.16b, v1.16b\n" + ".inst 0x4e81a45f // smmla v31.4s, v2.16b, v1.16b\n" "116:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x26, x26, #0x1\n" "cmp x26, x20\n" "bne 100b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "uzp1 v4.2d, v16.2d, v20.2d\n" - "add x22, x27, x20\n" + "uzp1 v0.2d, v16.2d, v20.2d\n" + "add x23, x27, x20\n" + "add x22, x23, x20\n" "add x21, x22, x20\n" - "add x20, x21, x20\n" "uzp2 v16.2d, v16.2d, v20.2d\n" "uzp1 v20.2d, v17.2d, v21.2d\n" "prfm pstl1keep, [x27, #0x0]\n" "uzp2 v17.2d, v17.2d, v21.2d\n" "uzp1 v21.2d, v18.2d, v22.2d\n" + "prfm pstl1keep, [x23, #0x0]\n" "prfm pstl1keep, [x22, #0x0]\n" - "prfm pstl1keep, [x21, #0x0]\n" "uzp2 v18.2d, v18.2d, v22.2d\n" "uzp1 v22.2d, v19.2d, v23.2d\n" - "prfm pstl1keep, [x20, #0x0]\n" + "prfm pstl1keep, [x21, #0x0]\n" "uzp2 v19.2d, v19.2d, v23.2d\n" "uzp1 v23.2d, v24.2d, v28.2d\n" "uzp2 v24.2d, v24.2d, v28.2d\n" @@ -1770,38 +1769,38 @@ void a64_hybrid_s8qa_mmla_4x16 ( "uzp2 v26.2d, v26.2d, v30.2d\n" "uzp1 v30.2d, v27.2d, v31.2d\n" "uzp2 v27.2d, v27.2d, v31.2d\n" - "mov v31.16b, v4.16b\n" + "mov v31.16b, v0.16b\n" "tbnz %x[flags], #31, 117f\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1r { v0.4s }, [x20]\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v13.4s, v13.4s, v13.4s\n" - "neg v4.4s, v4.4s\n" + "neg v0.4s, v0.4s\n" "dup v12.4s, v11.s[3]\n" "dup v11.4s, v11.s[0]\n" "dup v14.4s, v13.s[3]\n" "dup v13.4s, v13.s[0]\n" - "mul v11.4s, v11.4s, v4.4s\n" - "mul v12.4s, v12.4s, v4.4s\n" - "mul v13.4s, v13.4s, v4.4s\n" - "mul v14.4s, v14.4s, v4.4s\n" + "mul v11.4s, v11.4s, v0.4s\n" + "mul v12.4s, v12.4s, v0.4s\n" + "mul v13.4s, v13.4s, v0.4s\n" + "mul v14.4s, v14.4s, v0.4s\n" "117:" // Height 4: skip row sum fixup "ldr q0, [x10, #0x0]\n" - "ldr q1, [x10, #0x10]\n" + "ldr q4, [x10, #0x10]\n" "add v31.4s, v31.4s, v11.4s\n" "add v20.4s, v20.4s, v11.4s\n" - "ldr q2, [x10, #0x20]\n" - "ldr q3, [x10, #0x30]\n" + "ldr q3, [x10, #0x20]\n" + "ldr q2, [x10, #0x30]\n" "add v21.4s, v21.4s, v11.4s\n" "add v22.4s, v22.4s, v11.4s\n" "add v16.4s, v16.4s, v12.4s\n" "add v17.4s, v17.4s, v12.4s\n" - "add x23, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v1.4s }, [x20]\n" "add v18.4s, v18.4s, v12.4s\n" "add v19.4s, v19.4s, v12.4s\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" "add v23.4s, v23.4s, v13.4s\n" "add v28.4s, v28.4s, v13.4s\n" "add x10, x10, #0x40\n" @@ -1812,100 +1811,100 @@ void a64_hybrid_s8qa_mmla_4x16 ( "add v26.4s, v26.4s, v14.4s\n" "add v27.4s, v27.4s, v14.4s\n" "add v31.4s, v31.4s, v0.4s\n" - "add v20.4s, v20.4s, v1.4s\n" - "add v21.4s, v21.4s, v2.4s\n" - "add v22.4s, v22.4s, v3.4s\n" + "add v20.4s, v20.4s, v4.4s\n" + "add v21.4s, v21.4s, v3.4s\n" + "add v22.4s, v22.4s, v2.4s\n" "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" + "add v17.4s, v17.4s, v4.4s\n" + "add v18.4s, v18.4s, v3.4s\n" + "add v19.4s, v19.4s, v2.4s\n" "add v23.4s, v23.4s, v0.4s\n" - "add v28.4s, v28.4s, v1.4s\n" - "add v29.4s, v29.4s, v2.4s\n" - "add v30.4s, v30.4s, v3.4s\n" + "add v28.4s, v28.4s, v4.4s\n" + "add v29.4s, v29.4s, v3.4s\n" + "add v30.4s, v30.4s, v2.4s\n" "add v24.4s, v24.4s, v0.4s\n" - "ld1r { v0.4s }, [x23]\n" - "add v25.4s, v25.4s, v1.4s\n" - "add v26.4s, v26.4s, v2.4s\n" - "add v27.4s, v27.4s, v3.4s\n" - "sqrdmulh v31.4s, v31.4s, v4.4s\n" - "sqrdmulh v20.4s, v20.4s, v4.4s\n" - "sqrdmulh v21.4s, v21.4s, v4.4s\n" - "sqrdmulh v22.4s, v22.4s, v4.4s\n" - "sqrdmulh v16.4s, v16.4s, v4.4s\n" - "sqrdmulh v17.4s, v17.4s, v4.4s\n" - "sqrdmulh v18.4s, v18.4s, v4.4s\n" - "sqrdmulh v19.4s, v19.4s, v4.4s\n" - "sqrdmulh v23.4s, v23.4s, v4.4s\n" - "sqrdmulh v28.4s, v28.4s, v4.4s\n" - "sqrdmulh v29.4s, v29.4s, v4.4s\n" - "sqrdmulh v30.4s, v30.4s, v4.4s\n" - "sqrdmulh v24.4s, v24.4s, v4.4s\n" - "sqrdmulh v25.4s, v25.4s, v4.4s\n" - "sqrdmulh v26.4s, v26.4s, v4.4s\n" - "sqrdmulh v27.4s, v27.4s, v4.4s\n" + "ld1r { v0.4s }, [x20]\n" + "add v25.4s, v25.4s, v4.4s\n" + "add v26.4s, v26.4s, v3.4s\n" + "add v27.4s, v27.4s, v2.4s\n" + "sqrdmulh v31.4s, v31.4s, v1.4s\n" + "sqrdmulh v20.4s, v20.4s, v1.4s\n" + "sqrdmulh v21.4s, v21.4s, v1.4s\n" + "sqrdmulh v22.4s, v22.4s, v1.4s\n" + "sqrdmulh v16.4s, v16.4s, v1.4s\n" + "sqrdmulh v17.4s, v17.4s, v1.4s\n" + "sqrdmulh v18.4s, v18.4s, v1.4s\n" + "sqrdmulh v19.4s, v19.4s, v1.4s\n" + "sqrdmulh v23.4s, v23.4s, v1.4s\n" + "sqrdmulh v28.4s, v28.4s, v1.4s\n" + "sqrdmulh v29.4s, v29.4s, v1.4s\n" + "sqrdmulh v30.4s, v30.4s, v1.4s\n" + "sqrdmulh v24.4s, v24.4s, v1.4s\n" + "sqrdmulh v25.4s, v25.4s, v1.4s\n" + "sqrdmulh v26.4s, v26.4s, v1.4s\n" + "sqrdmulh v27.4s, v27.4s, v1.4s\n" "tbz %x[flags], #5, 118f\n" - "and v4.16b, v31.16b, v0.16b\n" - "and v5.16b, v20.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v31.4s, v31.4s, v4.4s\n" - "sqadd v20.4s, v20.4s, v5.4s\n" - "and v6.16b, v21.16b, v0.16b\n" - "and v7.16b, v22.16b, v0.16b\n" - "and v8.16b, v16.16b, v0.16b\n" - "and v9.16b, v17.16b, v0.16b\n" - "and v10.16b, v18.16b, v0.16b\n" - "and v4.16b, v19.16b, v0.16b\n" - "and v5.16b, v23.16b, v0.16b\n" - "sshr v6.4s, v6.4s, #0x1f\n" + "and v2.16b, v31.16b, v0.16b\n" + "and v1.16b, v20.16b, v0.16b\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sqadd v31.4s, v31.4s, v2.4s\n" + "sqadd v20.4s, v20.4s, v1.4s\n" + "and v7.16b, v21.16b, v0.16b\n" + "and v6.16b, v22.16b, v0.16b\n" + "and v5.16b, v16.16b, v0.16b\n" + "and v4.16b, v17.16b, v0.16b\n" + "and v3.16b, v18.16b, v0.16b\n" + "and v2.16b, v19.16b, v0.16b\n" + "and v1.16b, v23.16b, v0.16b\n" "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v9.4s, v9.4s, #0x1f\n" - "sshr v10.4s, v10.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v21.4s, v21.4s, v6.4s\n" - "sqadd v22.4s, v22.4s, v7.4s\n" - "sqadd v16.4s, v16.4s, v8.4s\n" - "sqadd v17.4s, v17.4s, v9.4s\n" - "sqadd v18.4s, v18.4s, v10.4s\n" - "sqadd v19.4s, v19.4s, v4.4s\n" - "sqadd v23.4s, v23.4s, v5.4s\n" - "and v6.16b, v28.16b, v0.16b\n" - "and v7.16b, v29.16b, v0.16b\n" - "and v8.16b, v30.16b, v0.16b\n" - "and v9.16b, v24.16b, v0.16b\n" - "and v10.16b, v25.16b, v0.16b\n" - "and v4.16b, v26.16b, v0.16b\n" - "and v5.16b, v27.16b, v0.16b\n" "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v9.4s, v9.4s, #0x1f\n" - "sshr v10.4s, v10.4s, #0x1f\n" + "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sqadd v21.4s, v21.4s, v7.4s\n" + "sqadd v22.4s, v22.4s, v6.4s\n" + "sqadd v16.4s, v16.4s, v5.4s\n" + "sqadd v17.4s, v17.4s, v4.4s\n" + "sqadd v18.4s, v18.4s, v3.4s\n" + "sqadd v19.4s, v19.4s, v2.4s\n" + "sqadd v23.4s, v23.4s, v1.4s\n" + "and v7.16b, v28.16b, v0.16b\n" + "and v6.16b, v29.16b, v0.16b\n" + "and v5.16b, v30.16b, v0.16b\n" + "and v4.16b, v24.16b, v0.16b\n" + "and v3.16b, v25.16b, v0.16b\n" + "and v2.16b, v26.16b, v0.16b\n" + "and v1.16b, v27.16b, v0.16b\n" + "sshr v7.4s, v7.4s, #0x1f\n" + "sshr v6.4s, v6.4s, #0x1f\n" "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v28.4s, v28.4s, v6.4s\n" - "sqadd v29.4s, v29.4s, v7.4s\n" - "sqadd v30.4s, v30.4s, v8.4s\n" - "sqadd v24.4s, v24.4s, v9.4s\n" - "sqadd v25.4s, v25.4s, v10.4s\n" - "sqadd v26.4s, v26.4s, v4.4s\n" - "sqadd v27.4s, v27.4s, v5.4s\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sqadd v28.4s, v28.4s, v7.4s\n" + "sqadd v29.4s, v29.4s, v6.4s\n" + "sqadd v30.4s, v30.4s, v5.4s\n" + "sqadd v24.4s, v24.4s, v4.4s\n" + "sqadd v25.4s, v25.4s, v3.4s\n" + "sqadd v26.4s, v26.4s, v2.4s\n" + "sqadd v27.4s, v27.4s, v1.4s\n" "118:" // Height 4: no shift correction - "add x23, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v3.4s }, [x20]\n" "srshl v31.4s, v31.4s, v0.4s\n" "srshl v20.4s, v20.4s, v0.4s\n" "srshl v21.4s, v21.4s, v0.4s\n" "srshl v22.4s, v22.4s, v0.4s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x23]\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v2.4s }, [x20]\n" "srshl v16.4s, v16.4s, v0.4s\n" "srshl v17.4s, v17.4s, v0.4s\n" - "add x23, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x23]\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v1.4s }, [x20]\n" "srshl v18.4s, v18.4s, v0.4s\n" "srshl v19.4s, v19.4s, v0.4s\n" "cmp x9, #0x10\n" @@ -1917,163 +1916,163 @@ void a64_hybrid_s8qa_mmla_4x16 ( "srshl v25.4s, v25.4s, v0.4s\n" "srshl v26.4s, v26.4s, v0.4s\n" "srshl v27.4s, v27.4s, v0.4s\n" - "add v31.4s, v31.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add v23.4s, v23.4s, v4.4s\n" - "add v28.4s, v28.4s, v4.4s\n" - "add v29.4s, v29.4s, v4.4s\n" - "add v30.4s, v30.4s, v4.4s\n" - "add v24.4s, v24.4s, v4.4s\n" - "add v25.4s, v25.4s, v4.4s\n" - "add v26.4s, v26.4s, v4.4s\n" - "add v27.4s, v27.4s, v4.4s\n" - "smin v31.4s, v31.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "smin v28.4s, v28.4s, v6.4s\n" - "smin v29.4s, v29.4s, v6.4s\n" - "smin v30.4s, v30.4s, v6.4s\n" - "smin v24.4s, v24.4s, v6.4s\n" - "smin v25.4s, v25.4s, v6.4s\n" - "smin v26.4s, v26.4s, v6.4s\n" - "smin v27.4s, v27.4s, v6.4s\n" - "smax v31.4s, v31.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" - "smax v28.4s, v28.4s, v5.4s\n" - "smax v29.4s, v29.4s, v5.4s\n" - "smax v30.4s, v30.4s, v5.4s\n" - "smax v24.4s, v24.4s, v5.4s\n" - "smax v25.4s, v25.4s, v5.4s\n" - "smax v26.4s, v26.4s, v5.4s\n" - "smax v27.4s, v27.4s, v5.4s\n" + "add v31.4s, v31.4s, v3.4s\n" + "add v20.4s, v20.4s, v3.4s\n" + "add v21.4s, v21.4s, v3.4s\n" + "add v22.4s, v22.4s, v3.4s\n" + "add v16.4s, v16.4s, v3.4s\n" + "add v17.4s, v17.4s, v3.4s\n" + "add v18.4s, v18.4s, v3.4s\n" + "add v19.4s, v19.4s, v3.4s\n" + "add v23.4s, v23.4s, v3.4s\n" + "add v28.4s, v28.4s, v3.4s\n" + "add v29.4s, v29.4s, v3.4s\n" + "add v30.4s, v30.4s, v3.4s\n" + "add v24.4s, v24.4s, v3.4s\n" + "add v25.4s, v25.4s, v3.4s\n" + "add v26.4s, v26.4s, v3.4s\n" + "add v27.4s, v27.4s, v3.4s\n" + "smin v31.4s, v31.4s, v2.4s\n" + "smin v20.4s, v20.4s, v2.4s\n" + "smin v21.4s, v21.4s, v2.4s\n" + "smin v22.4s, v22.4s, v2.4s\n" + "smin v16.4s, v16.4s, v2.4s\n" + "smin v17.4s, v17.4s, v2.4s\n" + "smin v18.4s, v18.4s, v2.4s\n" + "smin v19.4s, v19.4s, v2.4s\n" + "smin v23.4s, v23.4s, v2.4s\n" + "smin v28.4s, v28.4s, v2.4s\n" + "smin v29.4s, v29.4s, v2.4s\n" + "smin v30.4s, v30.4s, v2.4s\n" + "smin v24.4s, v24.4s, v2.4s\n" + "smin v25.4s, v25.4s, v2.4s\n" + "smin v26.4s, v26.4s, v2.4s\n" + "smin v27.4s, v27.4s, v2.4s\n" + "smax v31.4s, v31.4s, v1.4s\n" + "smax v20.4s, v20.4s, v1.4s\n" + "smax v21.4s, v21.4s, v1.4s\n" + "smax v22.4s, v22.4s, v1.4s\n" + "smax v16.4s, v16.4s, v1.4s\n" + "smax v17.4s, v17.4s, v1.4s\n" + "smax v18.4s, v18.4s, v1.4s\n" + "smax v19.4s, v19.4s, v1.4s\n" + "smax v23.4s, v23.4s, v1.4s\n" + "smax v28.4s, v28.4s, v1.4s\n" + "smax v29.4s, v29.4s, v1.4s\n" + "smax v30.4s, v30.4s, v1.4s\n" + "smax v24.4s, v24.4s, v1.4s\n" + "smax v25.4s, v25.4s, v1.4s\n" + "smax v26.4s, v26.4s, v1.4s\n" + "smax v27.4s, v27.4s, v1.4s\n" "uzp1 v31.8h, v31.8h, v20.8h\n" "uzp1 v20.8h, v21.8h, v22.8h\n" "uzp1 v16.8h, v16.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v19.8h, v18.8h, v19.8h\n" "uzp1 v23.8h, v23.8h, v28.8h\n" - "uzp1 v28.8h, v29.8h, v30.8h\n" + "uzp1 v18.8h, v29.8h, v30.8h\n" "uzp1 v24.8h, v24.8h, v25.8h\n" - "uzp1 v25.8h, v26.8h, v27.8h\n" + "uzp1 v17.8h, v26.8h, v27.8h\n" "uzp1 v31.16b, v31.16b, v20.16b\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" - "uzp1 v23.16b, v23.16b, v28.16b\n" - "uzp1 v24.16b, v24.16b, v25.16b\n" + "uzp1 v16.16b, v16.16b, v19.16b\n" + "uzp1 v23.16b, v23.16b, v18.16b\n" + "uzp1 v24.16b, v24.16b, v17.16b\n" "bge 127f\n" "tbz x9, #3, 122f\n" "str d31, [x27], #0x8\n" - "str d16, [x22], #0x8\n" - "str d23, [x21], #0x8\n" - "str d24, [x20], #0x8\n" + "str d16, [x23], #0x8\n" + "str d23, [x22], #0x8\n" + "str d24, [x21], #0x8\n" "tbz x9, #2, 120f\n" "st1 { v31.s }[2], [x27], #0x4\n" - "st1 { v16.s }[2], [x22], #0x4\n" - "st1 { v23.s }[2], [x21], #0x4\n" - "st1 { v24.s }[2], [x20], #0x4\n" + "st1 { v16.s }[2], [x23], #0x4\n" + "st1 { v23.s }[2], [x22], #0x4\n" + "st1 { v24.s }[2], [x21], #0x4\n" "tbz x9, #1, 119f\n" "st1 { v31.h }[6], [x27], #0x2\n" - "st1 { v16.h }[6], [x22], #0x2\n" - "st1 { v23.h }[6], [x21], #0x2\n" - "st1 { v24.h }[6], [x20], #0x2\n" + "st1 { v16.h }[6], [x23], #0x2\n" + "st1 { v23.h }[6], [x22], #0x2\n" + "st1 { v24.h }[6], [x21], #0x2\n" "tbz x9, #0, 126f\n" "st1 { v31.b }[14], [x27]\n" - "st1 { v16.b }[14], [x22]\n" - "st1 { v23.b }[14], [x21]\n" - "st1 { v24.b }[14], [x20]\n" + "st1 { v16.b }[14], [x23]\n" + "st1 { v23.b }[14], [x22]\n" + "st1 { v24.b }[14], [x21]\n" "b 126f\n" "119:" // Height 4: Partial direct writeback: partial_1_12 "tbz x9, #0, 126f\n" "st1 { v31.b }[12], [x27]\n" - "st1 { v16.b }[12], [x22]\n" - "st1 { v23.b }[12], [x21]\n" - "st1 { v24.b }[12], [x20]\n" + "st1 { v16.b }[12], [x23]\n" + "st1 { v23.b }[12], [x22]\n" + "st1 { v24.b }[12], [x21]\n" "b 126f\n" "120:" // Height 4: Partial direct writeback: partial_2_8 "tbz x9, #1, 121f\n" "st1 { v31.h }[4], [x27], #0x2\n" - "st1 { v16.h }[4], [x22], #0x2\n" - "st1 { v23.h }[4], [x21], #0x2\n" - "st1 { v24.h }[4], [x20], #0x2\n" + "st1 { v16.h }[4], [x23], #0x2\n" + "st1 { v23.h }[4], [x22], #0x2\n" + "st1 { v24.h }[4], [x21], #0x2\n" "tbz x9, #0, 126f\n" "st1 { v31.b }[10], [x27]\n" - "st1 { v16.b }[10], [x22]\n" - "st1 { v23.b }[10], [x21]\n" - "st1 { v24.b }[10], [x20]\n" + "st1 { v16.b }[10], [x23]\n" + "st1 { v23.b }[10], [x22]\n" + "st1 { v24.b }[10], [x21]\n" "b 126f\n" "121:" // Height 4: Partial direct writeback: partial_1_8 "tbz x9, #0, 126f\n" "st1 { v31.b }[8], [x27]\n" - "st1 { v16.b }[8], [x22]\n" - "st1 { v23.b }[8], [x21]\n" - "st1 { v24.b }[8], [x20]\n" + "st1 { v16.b }[8], [x23]\n" + "st1 { v23.b }[8], [x22]\n" + "st1 { v24.b }[8], [x21]\n" "b 126f\n" "122:" // Height 4: Partial direct writeback: partial_4_0 "tbz x9, #2, 124f\n" "str s31, [x27], #0x4\n" - "str s16, [x22], #0x4\n" - "str s23, [x21], #0x4\n" - "str s24, [x20], #0x4\n" + "str s16, [x23], #0x4\n" + "str s23, [x22], #0x4\n" + "str s24, [x21], #0x4\n" "tbz x9, #1, 123f\n" "st1 { v31.h }[2], [x27], #0x2\n" - "st1 { v16.h }[2], [x22], #0x2\n" - "st1 { v23.h }[2], [x21], #0x2\n" - "st1 { v24.h }[2], [x20], #0x2\n" + "st1 { v16.h }[2], [x23], #0x2\n" + "st1 { v23.h }[2], [x22], #0x2\n" + "st1 { v24.h }[2], [x21], #0x2\n" "tbz x9, #0, 126f\n" "st1 { v31.b }[6], [x27]\n" - "st1 { v16.b }[6], [x22]\n" - "st1 { v23.b }[6], [x21]\n" - "st1 { v24.b }[6], [x20]\n" + "st1 { v16.b }[6], [x23]\n" + "st1 { v23.b }[6], [x22]\n" + "st1 { v24.b }[6], [x21]\n" "b 126f\n" "123:" // Height 4: Partial direct writeback: partial_1_4 "tbz x9, #0, 126f\n" "st1 { v31.b }[4], [x27]\n" - "st1 { v16.b }[4], [x22]\n" - "st1 { v23.b }[4], [x21]\n" - "st1 { v24.b }[4], [x20]\n" + "st1 { v16.b }[4], [x23]\n" + "st1 { v23.b }[4], [x22]\n" + "st1 { v24.b }[4], [x21]\n" "b 126f\n" "124:" // Height 4: Partial direct writeback: partial_2_0 "tbz x9, #1, 125f\n" "str h31, [x27], #0x2\n" - "str h16, [x22], #0x2\n" - "str h23, [x21], #0x2\n" - "str h24, [x20], #0x2\n" + "str h16, [x23], #0x2\n" + "str h23, [x22], #0x2\n" + "str h24, [x21], #0x2\n" "tbz x9, #0, 126f\n" "st1 { v31.b }[2], [x27]\n" - "st1 { v16.b }[2], [x22]\n" - "st1 { v23.b }[2], [x21]\n" - "st1 { v24.b }[2], [x20]\n" + "st1 { v16.b }[2], [x23]\n" + "st1 { v23.b }[2], [x22]\n" + "st1 { v24.b }[2], [x21]\n" "b 126f\n" "125:" // Height 4: Partial direct writeback: partial_1_0 "str b31, [x27, #0x0]\n" - "str b16, [x22, #0x0]\n" - "str b23, [x21, #0x0]\n" - "str b24, [x20, #0x0]\n" + "str b16, [x23, #0x0]\n" + "str b23, [x22, #0x0]\n" + "str b24, [x21, #0x0]\n" "126:" // Height 4: Partial direct writeback: Done "b 128f\n" "127:" // Height 4: Full writeback "str q31, [x27, #0x0]\n" "add x27, x27, #0x10\n" - "str q16, [x22, #0x0]\n" - "str q23, [x21, #0x0]\n" - "str q24, [x20, #0x0]\n" + "str q16, [x23, #0x0]\n" + "str q23, [x22, #0x0]\n" + "str q24, [x21, #0x0]\n" "128:" // Height 4: Writeback done "subs x9, x9, #0x10\n" "bgt 98b\n" @@ -2089,7 +2088,6 @@ void a64_hybrid_s8qa_mmla_4x16 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "130:" // Exit - : [M] "+&r" (M), [flags] "+&r" (flags), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16.hpp index b028a8a9a33c94d294ff530b5ca7730cdb4edd8d..2b7531d1e2f51376eb3ce47ae5b4929a9c637dfa 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef __aarch64__ + #include "../std_transforms_fixed.hpp" #include "../performance_parameters.hpp" @@ -108,5 +108,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16/a55.cpp index b97b63cdce2dc8db5c67dc891ca04da9a7922a8d..38a57b074186d008f7e183470a73d8cef3c13097 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16/a55.cpp @@ -85,7 +85,6 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( flags |= 0x20; } __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x6\n" "bge 136f\n" @@ -111,11 +110,11 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "4:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w13, [x20, x14, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 5f\n" - "ldr x21, [%x[input_ptr], x14, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x12, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x12, [x20, #0x0]\n" "cbnz x14, 6f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x12, x12, x20\n" @@ -132,129 +131,129 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "blt 8f\n" "7:" // Height 1: Multiply loop: Main loop head ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr d6, [x15, #0x20]\n" + "ldr d17, [x15, #0x20]\n" "ldr x20, [x15, #0x28]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "ldr d7, [x15, #0x30]\n" - "mov v6.d[1], x20\n" - "ldr x11, [x15, #0x38]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - "ldr d6, [x15, #0x40]\n" + "ldr d16, [x15, #0x30]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x15, #0x38]\n" + "mov v16.d[1], x20\n" + ".inst 0x4f80e22a // sdot v10.4s, v17.16b, v0.4b[0]\n" + "ldr d17, [x15, #0x40]\n" "ldr x20, [x15, #0x48]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - "ldr d7, [x15, #0x50]\n" - "mov v6.d[1], x20\n" - "ldr x11, [x15, #0x58]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - "ldr d6, [x15, #0x60]\n" + ".inst 0x4f80e20b // sdot v11.4s, v16.16b, v0.4b[0]\n" + "ldr d16, [x15, #0x50]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x15, #0x58]\n" + "mov v16.d[1], x20\n" + ".inst 0x4fa0e228 // sdot v8.4s, v17.16b, v0.4b[1]\n" + "ldr d17, [x15, #0x60]\n" "ldr x20, [x15, #0x68]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - "ldr d7, [x15, #0x70]\n" - "mov v6.d[1], x20\n" - "ldr x11, [x15, #0x78]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - "ldr d6, [x15, #0x80]\n" + ".inst 0x4fa0e209 // sdot v9.4s, v16.16b, v0.4b[1]\n" + "ldr d16, [x15, #0x70]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x15, #0x78]\n" + "mov v16.d[1], x20\n" + ".inst 0x4fa0e22a // sdot v10.4s, v17.16b, v0.4b[1]\n" + "ldr d17, [x15, #0x80]\n" "ldr x20, [x15, #0x88]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - "ldr d7, [x15, #0x90]\n" - "mov v6.d[1], x20\n" - "ldr x11, [x15, #0x98]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - "ldr d6, [x15, #0xa0]\n" + ".inst 0x4fa0e20b // sdot v11.4s, v16.16b, v0.4b[1]\n" + "ldr d16, [x15, #0x90]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x15, #0x98]\n" + "mov v16.d[1], x20\n" + ".inst 0x4f80ea28 // sdot v8.4s, v17.16b, v0.4b[2]\n" + "ldr d17, [x15, #0xa0]\n" "ldr x20, [x15, #0xa8]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - "ldr d7, [x15, #0xb0]\n" - "mov v6.d[1], x20\n" - "ldr x11, [x15, #0xb8]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - "ldr d6, [x15, #0xc0]\n" + ".inst 0x4f80ea09 // sdot v9.4s, v16.16b, v0.4b[2]\n" + "ldr d16, [x15, #0xb0]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x15, #0xb8]\n" + "mov v16.d[1], x20\n" + ".inst 0x4f80ea2a // sdot v10.4s, v17.16b, v0.4b[2]\n" + "ldr d17, [x15, #0xc0]\n" "ldr x20, [x15, #0xc8]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - "ldr d7, [x15, #0xd0]\n" - "mov v6.d[1], x20\n" - "ldr x11, [x15, #0xd8]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - "ldr d6, [x15, #0xe0]\n" + ".inst 0x4f80ea0b // sdot v11.4s, v16.16b, v0.4b[2]\n" + "ldr d16, [x15, #0xd0]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x15, #0xd8]\n" + "mov v16.d[1], x20\n" + ".inst 0x4fa0ea28 // sdot v8.4s, v17.16b, v0.4b[3]\n" + "ldr d17, [x15, #0xe0]\n" "ldr x20, [x15, #0xe8]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - "ldr d7, [x15, #0xf0]\n" - "mov v6.d[1], x20\n" - "ldr x11, [x15, #0xf8]\n" - "mov v7.d[1], x11\n" + ".inst 0x4fa0ea09 // sdot v9.4s, v16.16b, v0.4b[3]\n" + "ldr d16, [x15, #0xf0]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x15, #0xf8]\n" + "mov v16.d[1], x20\n" "add x12, x12, #0x10\n" "add x15, x15, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" + ".inst 0x4fa0ea2a // sdot v10.4s, v17.16b, v0.4b[3]\n" "ldr d6, [x15, #0x0]\n" "ldr x20, [x15, #0x8]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x4fa0ea0b // sdot v11.4s, v16.16b, v0.4b[3]\n" "ldr d0, [x12, #0x0]\n" "sub x13, x13, #0x10\n" "ldr d7, [x15, #0x10]\n" "cmp x13, #0x20\n" - "ldr x10, [x12, #0x8]\n" + "ldr x21, [x12, #0x8]\n" "mov v6.d[1], x20\n" - "ldr x11, [x15, #0x18]\n" - "mov v0.d[1], x10\n" - "mov v7.d[1], x11\n" + "ldr x20, [x15, #0x18]\n" + "mov v0.d[1], x21\n" + "mov v7.d[1], x20\n" "prfm pldl1keep, [x12, #0x80]\n" "bge 7b\n" "8:" // Height 1: Multiply loop: Single iteration only ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q6, [x15, #0x20]\n" + "ldr q17, [x15, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "ldr q7, [x15, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - "ldr q6, [x15, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - "ldr q7, [x15, #0x50]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - "ldr q6, [x15, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - "ldr q7, [x15, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - "ldr q6, [x15, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - "ldr q7, [x15, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - "ldr q6, [x15, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - "ldr q7, [x15, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - "ldr q6, [x15, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - "ldr q7, [x15, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - "ldr q6, [x15, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - "ldr q7, [x15, #0xf0]\n" + "ldr q16, [x15, #0x30]\n" + ".inst 0x4f80e22a // sdot v10.4s, v17.16b, v0.4b[0]\n" + "ldr q17, [x15, #0x40]\n" + ".inst 0x4f80e20b // sdot v11.4s, v16.16b, v0.4b[0]\n" + "ldr q16, [x15, #0x50]\n" + ".inst 0x4fa0e228 // sdot v8.4s, v17.16b, v0.4b[1]\n" + "ldr q17, [x15, #0x60]\n" + ".inst 0x4fa0e209 // sdot v9.4s, v16.16b, v0.4b[1]\n" + "ldr q16, [x15, #0x70]\n" + ".inst 0x4fa0e22a // sdot v10.4s, v17.16b, v0.4b[1]\n" + "ldr q17, [x15, #0x80]\n" + ".inst 0x4fa0e20b // sdot v11.4s, v16.16b, v0.4b[1]\n" + "ldr q16, [x15, #0x90]\n" + ".inst 0x4f80ea28 // sdot v8.4s, v17.16b, v0.4b[2]\n" + "ldr q17, [x15, #0xa0]\n" + ".inst 0x4f80ea09 // sdot v9.4s, v16.16b, v0.4b[2]\n" + "ldr q16, [x15, #0xb0]\n" + ".inst 0x4f80ea2a // sdot v10.4s, v17.16b, v0.4b[2]\n" + "ldr q17, [x15, #0xc0]\n" + ".inst 0x4f80ea0b // sdot v11.4s, v16.16b, v0.4b[2]\n" + "ldr q16, [x15, #0xd0]\n" + ".inst 0x4fa0ea28 // sdot v8.4s, v17.16b, v0.4b[3]\n" + "ldr q17, [x15, #0xe0]\n" + ".inst 0x4fa0ea09 // sdot v9.4s, v16.16b, v0.4b[3]\n" + "ldr q16, [x15, #0xf0]\n" "add x12, x12, #0x10\n" "sub x13, x13, #0x10\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" + ".inst 0x4fa0ea2a // sdot v10.4s, v17.16b, v0.4b[3]\n" "prfm pldl1keep, [x12, #0x80]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x4fa0ea0b // sdot v11.4s, v16.16b, v0.4b[3]\n" "add x15, x15, #0x100\n" "9:" // Height 1: Multiply loop: Main loop skip "cbz x13, 14f\n" "cmp x13, #0x4\n" "blt 11f\n" "10:" // Height 1: Multiply loop: Odd block loop - "ldr s0, [x12], #0x4\n" + "ldr s18, [x12], #0x4\n" "sub x13, x13, #0x4\n" - "ldr q6, [x15, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x15, #0x10]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "ldr q6, [x15, #0x20]\n" + "ldr q16, [x15, #0x0]\n" + ".inst 0x4f92e208 // sdot v8.4s, v16.16b, v18.4b[0]\n" + "ldr q16, [x15, #0x10]\n" + ".inst 0x4f92e209 // sdot v9.4s, v16.16b, v18.4b[0]\n" + "ldr q17, [x15, #0x20]\n" "cmp x13, #0x4\n" - "ldr q7, [x15, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" + "ldr q16, [x15, #0x30]\n" + ".inst 0x4f92e22a // sdot v10.4s, v17.16b, v18.4b[0]\n" + ".inst 0x4f92e20b // sdot v11.4s, v16.16b, v18.4b[0]\n" "add x15, x15, #0x40\n" "bge 10b\n" "11:" // Height 1: Multiply loop: Skip odd blocks @@ -267,28 +266,28 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "12:" // Height 1: Multiply loop: Ragged operand read: partial_1_0 "ldr b0, [x12, #0x0]\n" "13:" // Height 1: Multiply loop: Ragged operand read: Done - "ldr q6, [x15, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x15, #0x10]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "ldr q6, [x15, #0x20]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x15, #0x30]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" + "ldr q16, [x15, #0x0]\n" + ".inst 0x4f80e208 // sdot v8.4s, v16.16b, v0.4b[0]\n" + "ldr q16, [x15, #0x10]\n" + ".inst 0x4f80e209 // sdot v9.4s, v16.16b, v0.4b[0]\n" + "ldr q16, [x15, #0x20]\n" + ".inst 0x4f80e20a // sdot v10.4s, v16.16b, v0.4b[0]\n" + "ldr q16, [x15, #0x30]\n" + ".inst 0x4f80e20b // sdot v11.4s, v16.16b, v0.4b[0]\n" "add x15, x15, #0x40\n" "14:" // Height 1: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x14, x14, #0x1\n" "cmp x14, x20\n" "bne 4b\n" - "ldr q0, [x6, #0x0]\n" - "add v8.4s, v8.4s, v0.4s\n" - "ldr q1, [x6, #0x10]\n" - "add v9.4s, v9.4s, v1.4s\n" - "ldr q2, [x6, #0x20]\n" - "add v10.4s, v10.4s, v2.4s\n" - "ldr q3, [x6, #0x30]\n" - "add v11.4s, v11.4s, v3.4s\n" + "ldr q16, [x6, #0x0]\n" + "add v8.4s, v8.4s, v16.4s\n" + "ldr q16, [x6, #0x10]\n" + "add v9.4s, v9.4s, v16.4s\n" + "ldr q16, [x6, #0x20]\n" + "add v10.4s, v10.4s, v16.4s\n" + "ldr q16, [x6, #0x30]\n" + "add v11.4s, v11.4s, v16.4s\n" "prfm pstl1keep, [x17, #0x0]\n" "add x6, x6, #0x40\n" "tbz %x[flags], #4, 15f\n" @@ -304,10 +303,10 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "add x7, x7, #0x40\n" "b 16f\n" "15:" // Height 1: per layer parameters - "add x25, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x25]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v4.4s }, [x20]\n" "mov v1.16b, v0.16b\n" "mov v5.16b, v4.16b\n" "mov v2.16b, v0.16b\n" @@ -320,45 +319,45 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "sqrdmulh v10.4s, v10.4s, v6.4s\n" "sqrdmulh v11.4s, v11.4s, v7.4s\n" "tbz %x[flags], #5, 17f\n" - "and v4.16b, v8.16b, v0.16b\n" - "and v5.16b, v9.16b, v1.16b\n" - "and v6.16b, v10.16b, v2.16b\n" - "and v7.16b, v11.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v4.4s\n" - "sqadd v9.4s, v9.4s, v5.4s\n" - "sqadd v10.4s, v10.4s, v6.4s\n" - "sqadd v11.4s, v11.4s, v7.4s\n" + "and v19.16b, v8.16b, v0.16b\n" + "and v18.16b, v9.16b, v1.16b\n" + "and v17.16b, v10.16b, v2.16b\n" + "and v16.16b, v11.16b, v3.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v8.4s, v8.4s, v19.4s\n" + "sqadd v9.4s, v9.4s, v18.4s\n" + "sqadd v10.4s, v10.4s, v17.4s\n" + "sqadd v11.4s, v11.4s, v16.4s\n" "17:" // Height 1: no shift correction "srshl v8.4s, v8.4s, v0.4s\n" "srshl v9.4s, v9.4s, v1.4s\n" "srshl v10.4s, v10.4s, v2.4s\n" "srshl v11.4s, v11.4s, v3.4s\n" - "add x25, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x25]\n" - "add v8.4s, v8.4s, v4.4s\n" - "add v9.4s, v9.4s, v4.4s\n" - "add v10.4s, v10.4s, v4.4s\n" - "add v11.4s, v11.4s, v4.4s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x25]\n" - "smin v8.4s, v8.4s, v6.4s\n" - "smin v9.4s, v9.4s, v6.4s\n" - "smin v10.4s, v10.4s, v6.4s\n" - "smin v11.4s, v11.4s, v6.4s\n" - "add x25, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x25]\n" - "smax v8.4s, v8.4s, v5.4s\n" - "smax v9.4s, v9.4s, v5.4s\n" - "smax v10.4s, v10.4s, v5.4s\n" - "smax v11.4s, v11.4s, v5.4s\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v16.4s }, [x20]\n" + "add v8.4s, v8.4s, v16.4s\n" + "add v9.4s, v9.4s, v16.4s\n" + "add v10.4s, v10.4s, v16.4s\n" + "add v11.4s, v11.4s, v16.4s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v16.4s }, [x20]\n" + "smin v8.4s, v8.4s, v16.4s\n" + "smin v9.4s, v9.4s, v16.4s\n" + "smin v10.4s, v10.4s, v16.4s\n" + "smin v11.4s, v11.4s, v16.4s\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v16.4s }, [x20]\n" + "smax v8.4s, v8.4s, v16.4s\n" + "smax v9.4s, v9.4s, v16.4s\n" + "smax v10.4s, v10.4s, v16.4s\n" + "smax v11.4s, v11.4s, v16.4s\n" "uzp1 v8.8h, v8.8h, v9.8h\n" - "uzp1 v9.8h, v10.8h, v11.8h\n" + "uzp1 v16.8h, v10.8h, v11.8h\n" "cmp x16, #0x10\n" - "uzp1 v8.16b, v8.16b, v9.16b\n" + "uzp1 v8.16b, v8.16b, v16.16b\n" "bge 26f\n" "tbz x16, #3, 21f\n" "str d8, [x17], #0x8\n" @@ -433,247 +432,247 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "31:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w13, [x20, x14, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 32f\n" - "ldr x21, [%x[input_ptr], x14, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x12, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x12, [x20, #0x0]\n" + "ldr x11, [x20, #0x8]\n" "cbnz x14, 33f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x12, x12, x20\n" - "add x9, x9, x20\n" + "add x11, x11, x20\n" "b 33f\n" "32:" // Height 2: setup direct input "mov x12, %x[input_ptr]\n" - "add x9, x12, x20\n" + "add x11, x12, x21\n" "33:" // Height 2: input setup done "cmp x13, #0x10\n" "blt 36f\n" "ldr q0, [x12, #0x0]\n" "cmp x13, #0x20\n" - "ldr q1, [x9, #0x0]\n" + "ldr q1, [x11, #0x0]\n" "ldr q6, [x15, #0x0]\n" "ldr q7, [x15, #0x10]\n" "blt 35f\n" "34:" // Height 2: Multiply loop: Main loop head ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr x20, [x15, #0x28]\n" + "ldr x21, [x15, #0x28]\n" ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr d6, [x15, #0x20]\n" + "ldr d17, [x15, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "ldr x11, [x15, #0x38]\n" + "ldr x20, [x15, #0x38]\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "ldr d7, [x15, #0x30]\n" - "mov v6.d[1], x20\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "ldr d6, [x15, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" + "ldr d16, [x15, #0x30]\n" + "mov v17.d[1], x21\n" + ".inst 0x4f80e22a // sdot v10.4s, v17.16b, v0.4b[0]\n" + "mov v16.d[1], x20\n" + ".inst 0x4f81e22e // sdot v14.4s, v17.16b, v1.4b[0]\n" + "ldr d17, [x15, #0x40]\n" + ".inst 0x4f80e20b // sdot v11.4s, v16.16b, v0.4b[0]\n" "ldr x20, [x15, #0x48]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - "ldr d7, [x15, #0x50]\n" - "mov v6.d[1], x20\n" - "ldr x11, [x15, #0x58]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - "ldr x20, [x15, #0x68]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - "ldr d6, [x15, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - "ldr x11, [x15, #0x78]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - "ldr d7, [x15, #0x70]\n" - "mov v6.d[1], x20\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - "ldr d6, [x15, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" + ".inst 0x4f81e20f // sdot v15.4s, v16.16b, v1.4b[0]\n" + "ldr d16, [x15, #0x50]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x15, #0x58]\n" + "mov v16.d[1], x20\n" + ".inst 0x4fa0e228 // sdot v8.4s, v17.16b, v0.4b[1]\n" + "ldr x21, [x15, #0x68]\n" + ".inst 0x4fa1e22c // sdot v12.4s, v17.16b, v1.4b[1]\n" + "ldr d17, [x15, #0x60]\n" + ".inst 0x4fa0e209 // sdot v9.4s, v16.16b, v0.4b[1]\n" + "ldr x20, [x15, #0x78]\n" + ".inst 0x4fa1e20d // sdot v13.4s, v16.16b, v1.4b[1]\n" + "ldr d16, [x15, #0x70]\n" + "mov v17.d[1], x21\n" + ".inst 0x4fa0e22a // sdot v10.4s, v17.16b, v0.4b[1]\n" + "mov v16.d[1], x20\n" + ".inst 0x4fa1e22e // sdot v14.4s, v17.16b, v1.4b[1]\n" + "ldr d17, [x15, #0x80]\n" + ".inst 0x4fa0e20b // sdot v11.4s, v16.16b, v0.4b[1]\n" "ldr x20, [x15, #0x88]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - "ldr d7, [x15, #0x90]\n" - "mov v6.d[1], x20\n" - "ldr x11, [x15, #0x98]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - "ldr x20, [x15, #0xa8]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - "ldr d6, [x15, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - "ldr x11, [x15, #0xb8]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - "ldr d7, [x15, #0xb0]\n" - "mov v6.d[1], x20\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - "ldr d6, [x15, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" + ".inst 0x4fa1e20f // sdot v15.4s, v16.16b, v1.4b[1]\n" + "ldr d16, [x15, #0x90]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x15, #0x98]\n" + "mov v16.d[1], x20\n" + ".inst 0x4f80ea28 // sdot v8.4s, v17.16b, v0.4b[2]\n" + "ldr x21, [x15, #0xa8]\n" + ".inst 0x4f81ea2c // sdot v12.4s, v17.16b, v1.4b[2]\n" + "ldr d17, [x15, #0xa0]\n" + ".inst 0x4f80ea09 // sdot v9.4s, v16.16b, v0.4b[2]\n" + "ldr x20, [x15, #0xb8]\n" + ".inst 0x4f81ea0d // sdot v13.4s, v16.16b, v1.4b[2]\n" + "ldr d16, [x15, #0xb0]\n" + "mov v17.d[1], x21\n" + ".inst 0x4f80ea2a // sdot v10.4s, v17.16b, v0.4b[2]\n" + "mov v16.d[1], x20\n" + ".inst 0x4f81ea2e // sdot v14.4s, v17.16b, v1.4b[2]\n" + "ldr d17, [x15, #0xc0]\n" + ".inst 0x4f80ea0b // sdot v11.4s, v16.16b, v0.4b[2]\n" "ldr x20, [x15, #0xc8]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - "ldr d7, [x15, #0xd0]\n" - "mov v6.d[1], x20\n" - "ldr x11, [x15, #0xd8]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - "ldr x20, [x15, #0xe8]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - "ldr d6, [x15, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - "ldr x11, [x15, #0xf8]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - "ldr d7, [x15, #0xf0]\n" - "mov v6.d[1], x20\n" + ".inst 0x4f81ea0f // sdot v15.4s, v16.16b, v1.4b[2]\n" + "ldr d16, [x15, #0xd0]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x15, #0xd8]\n" + "mov v16.d[1], x20\n" + ".inst 0x4fa0ea28 // sdot v8.4s, v17.16b, v0.4b[3]\n" + "ldr x21, [x15, #0xe8]\n" + ".inst 0x4fa1ea2c // sdot v12.4s, v17.16b, v1.4b[3]\n" + "ldr d17, [x15, #0xe0]\n" + ".inst 0x4fa0ea09 // sdot v9.4s, v16.16b, v0.4b[3]\n" + "ldr x20, [x15, #0xf8]\n" + ".inst 0x4fa1ea0d // sdot v13.4s, v16.16b, v1.4b[3]\n" + "ldr d16, [x15, #0xf0]\n" + "mov v17.d[1], x21\n" "add x12, x12, #0x10\n" - "mov v7.d[1], x11\n" - "add x9, x9, #0x10\n" + "mov v16.d[1], x20\n" + "add x11, x11, #0x10\n" "add x15, x15, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" + ".inst 0x4fa0ea2a // sdot v10.4s, v17.16b, v0.4b[3]\n" + ".inst 0x4fa1ea2e // sdot v14.4s, v17.16b, v1.4b[3]\n" "ldr d6, [x15, #0x0]\n" - "ldr x20, [x15, #0x8]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" + "ldr x21, [x15, #0x8]\n" + ".inst 0x4fa0ea0b // sdot v11.4s, v16.16b, v0.4b[3]\n" "ldr d0, [x12, #0x0]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" - "ldr d1, [x9, #0x0]\n" + ".inst 0x4fa1ea0f // sdot v15.4s, v16.16b, v1.4b[3]\n" + "ldr d1, [x11, #0x0]\n" "sub x13, x13, #0x10\n" "ldr d7, [x15, #0x10]\n" "cmp x13, #0x20\n" - "ldr x10, [x12, #0x8]\n" - "mov v6.d[1], x20\n" - "ldr x28, [x9, #0x8]\n" - "mov v0.d[1], x10\n" - "ldr x11, [x15, #0x18]\n" - "mov v1.d[1], x28\n" + "ldr x20, [x12, #0x8]\n" + "mov v6.d[1], x21\n" + "ldr x21, [x11, #0x8]\n" + "mov v0.d[1], x20\n" + "ldr x20, [x15, #0x18]\n" + "mov v1.d[1], x21\n" "prfm pldl1keep, [x12, #0x80]\n" - "mov v7.d[1], x11\n" - "prfm pldl1keep, [x9, #0x80]\n" + "mov v7.d[1], x20\n" + "prfm pldl1keep, [x11, #0x80]\n" "bge 34b\n" "35:" // Height 2: Multiply loop: Single iteration only ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" "add x12, x12, #0x10\n" ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x15, #0x20]\n" + "ldr q17, [x15, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "add x9, x9, #0x10\n" + "add x11, x11, #0x10\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x15, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q16, [x15, #0x30]\n" + ".inst 0x4f80e22a // sdot v10.4s, v17.16b, v0.4b[0]\n" "sub x13, x13, #0x10\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x15, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" + ".inst 0x4f81e22e // sdot v14.4s, v17.16b, v1.4b[0]\n" + "ldr q17, [x15, #0x40]\n" + ".inst 0x4f80e20b // sdot v11.4s, v16.16b, v0.4b[0]\n" "prfm pldl1keep, [x12, #0x80]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x15, #0x50]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - "prfm pldl1keep, [x9, #0x80]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - "ldr q6, [x15, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - "ldr q7, [x15, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - "ldr q6, [x15, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - "ldr q7, [x15, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - "ldr q6, [x15, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - "ldr q7, [x15, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - "ldr q6, [x15, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - "ldr q7, [x15, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - "ldr q6, [x15, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - "ldr q7, [x15, #0xf0]\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" + ".inst 0x4f81e20f // sdot v15.4s, v16.16b, v1.4b[0]\n" + "ldr q16, [x15, #0x50]\n" + ".inst 0x4fa0e228 // sdot v8.4s, v17.16b, v0.4b[1]\n" + "prfm pldl1keep, [x11, #0x80]\n" + ".inst 0x4fa1e22c // sdot v12.4s, v17.16b, v1.4b[1]\n" + "ldr q17, [x15, #0x60]\n" + ".inst 0x4fa0e209 // sdot v9.4s, v16.16b, v0.4b[1]\n" + ".inst 0x4fa1e20d // sdot v13.4s, v16.16b, v1.4b[1]\n" + "ldr q16, [x15, #0x70]\n" + ".inst 0x4fa0e22a // sdot v10.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e22e // sdot v14.4s, v17.16b, v1.4b[1]\n" + "ldr q17, [x15, #0x80]\n" + ".inst 0x4fa0e20b // sdot v11.4s, v16.16b, v0.4b[1]\n" + ".inst 0x4fa1e20f // sdot v15.4s, v16.16b, v1.4b[1]\n" + "ldr q16, [x15, #0x90]\n" + ".inst 0x4f80ea28 // sdot v8.4s, v17.16b, v0.4b[2]\n" + ".inst 0x4f81ea2c // sdot v12.4s, v17.16b, v1.4b[2]\n" + "ldr q17, [x15, #0xa0]\n" + ".inst 0x4f80ea09 // sdot v9.4s, v16.16b, v0.4b[2]\n" + ".inst 0x4f81ea0d // sdot v13.4s, v16.16b, v1.4b[2]\n" + "ldr q16, [x15, #0xb0]\n" + ".inst 0x4f80ea2a // sdot v10.4s, v17.16b, v0.4b[2]\n" + ".inst 0x4f81ea2e // sdot v14.4s, v17.16b, v1.4b[2]\n" + "ldr q17, [x15, #0xc0]\n" + ".inst 0x4f80ea0b // sdot v11.4s, v16.16b, v0.4b[2]\n" + ".inst 0x4f81ea0f // sdot v15.4s, v16.16b, v1.4b[2]\n" + "ldr q16, [x15, #0xd0]\n" + ".inst 0x4fa0ea28 // sdot v8.4s, v17.16b, v0.4b[3]\n" + ".inst 0x4fa1ea2c // sdot v12.4s, v17.16b, v1.4b[3]\n" + "ldr q17, [x15, #0xe0]\n" + ".inst 0x4fa0ea09 // sdot v9.4s, v16.16b, v0.4b[3]\n" + ".inst 0x4fa1ea0d // sdot v13.4s, v16.16b, v1.4b[3]\n" + "ldr q16, [x15, #0xf0]\n" + ".inst 0x4fa0ea2a // sdot v10.4s, v17.16b, v0.4b[3]\n" "add x15, x15, #0x100\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" + ".inst 0x4fa1ea2e // sdot v14.4s, v17.16b, v1.4b[3]\n" + ".inst 0x4fa0ea0b // sdot v11.4s, v16.16b, v0.4b[3]\n" + ".inst 0x4fa1ea0f // sdot v15.4s, v16.16b, v1.4b[3]\n" "36:" // Height 2: Multiply loop: Main loop skip "cbz x13, 41f\n" "cmp x13, #0x4\n" "blt 38f\n" "37:" // Height 2: Multiply loop: Odd block loop - "ldr s0, [x12], #0x4\n" + "ldr s19, [x12], #0x4\n" "sub x13, x13, #0x4\n" - "ldr s1, [x9], #0x4\n" + "ldr s18, [x11], #0x4\n" "cmp x13, #0x4\n" - "ldr q6, [x15, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x15, #0x10]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x15, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x15, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q17, [x15, #0x0]\n" + ".inst 0x4f93e228 // sdot v8.4s, v17.16b, v19.4b[0]\n" + "ldr q16, [x15, #0x10]\n" + ".inst 0x4f92e22c // sdot v12.4s, v17.16b, v18.4b[0]\n" + "ldr q17, [x15, #0x20]\n" + ".inst 0x4f93e209 // sdot v9.4s, v16.16b, v19.4b[0]\n" + ".inst 0x4f92e20d // sdot v13.4s, v16.16b, v18.4b[0]\n" + "ldr q16, [x15, #0x30]\n" + ".inst 0x4f93e22a // sdot v10.4s, v17.16b, v19.4b[0]\n" "add x15, x15, #0x40\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" + ".inst 0x4f92e22e // sdot v14.4s, v17.16b, v18.4b[0]\n" + ".inst 0x4f93e20b // sdot v11.4s, v16.16b, v19.4b[0]\n" + ".inst 0x4f92e20f // sdot v15.4s, v16.16b, v18.4b[0]\n" "bge 37b\n" "38:" // Height 2: Multiply loop: Skip odd blocks "cbz x13, 41f\n" "tbz x13, #1, 39f\n" "ldr h0, [x12], #0x2\n" - "ldr h1, [x9], #0x2\n" + "ldr h1, [x11], #0x2\n" "tbz x13, #0, 40f\n" "ld1 { v0.b }[2], [x12]\n" - "ld1 { v1.b }[2], [x9]\n" + "ld1 { v1.b }[2], [x11]\n" "b 40f\n" "39:" // Height 2: Multiply loop: Ragged operand read: partial_1_0 "ldr b0, [x12, #0x0]\n" - "ldr b1, [x9, #0x0]\n" + "ldr b1, [x11, #0x0]\n" "40:" // Height 2: Multiply loop: Ragged operand read: Done - "ldr q6, [x15, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x15, #0x10]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x15, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x15, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q17, [x15, #0x0]\n" + ".inst 0x4f80e228 // sdot v8.4s, v17.16b, v0.4b[0]\n" + "ldr q16, [x15, #0x10]\n" + ".inst 0x4f81e22c // sdot v12.4s, v17.16b, v1.4b[0]\n" + "ldr q17, [x15, #0x20]\n" + ".inst 0x4f80e209 // sdot v9.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f81e20d // sdot v13.4s, v16.16b, v1.4b[0]\n" + "ldr q16, [x15, #0x30]\n" + ".inst 0x4f80e22a // sdot v10.4s, v17.16b, v0.4b[0]\n" "add x15, x15, #0x40\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" + ".inst 0x4f81e22e // sdot v14.4s, v17.16b, v1.4b[0]\n" + ".inst 0x4f80e20b // sdot v11.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f81e20f // sdot v15.4s, v16.16b, v1.4b[0]\n" "41:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x14, x14, #0x1\n" "cmp x14, x20\n" "bne 31b\n" - "ldr q0, [x6, #0x0]\n" - "add v8.4s, v8.4s, v0.4s\n" - "ldr q1, [x6, #0x10]\n" - "add v9.4s, v9.4s, v1.4s\n" - "ldr q2, [x6, #0x20]\n" - "add v10.4s, v10.4s, v2.4s\n" - "ldr q3, [x6, #0x30]\n" - "add v11.4s, v11.4s, v3.4s\n" + "ldr q19, [x6, #0x0]\n" + "add v8.4s, v8.4s, v19.4s\n" + "ldr q18, [x6, #0x10]\n" + "add v9.4s, v9.4s, v18.4s\n" + "ldr q17, [x6, #0x20]\n" + "add v10.4s, v10.4s, v17.4s\n" + "ldr q16, [x6, #0x30]\n" + "add v11.4s, v11.4s, v16.4s\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x17, x20\n" + "add x25, x17, x20\n" "prfm pstl1keep, [x17, #0x0]\n" - "add v12.4s, v12.4s, v0.4s\n" - "prfm pstl1keep, [x24, #0x0]\n" - "add v13.4s, v13.4s, v1.4s\n" - "add v14.4s, v14.4s, v2.4s\n" - "add v15.4s, v15.4s, v3.4s\n" + "add v12.4s, v12.4s, v19.4s\n" + "prfm pstl1keep, [x25, #0x0]\n" + "add v13.4s, v13.4s, v18.4s\n" + "add v14.4s, v14.4s, v17.4s\n" + "add v15.4s, v15.4s, v16.4s\n" "add x6, x6, #0x40\n" "tbz %x[flags], #4, 42f\n" "ldr q0, [x8, #0x0]\n" @@ -688,10 +687,10 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "add x7, x7, #0x40\n" "b 43f\n" "42:" // Height 2: per layer parameters - "add x25, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x25]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v4.4s }, [x20]\n" "mov v1.16b, v0.16b\n" "mov v5.16b, v4.16b\n" "mov v2.16b, v0.16b\n" @@ -708,30 +707,30 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "sqrdmulh v14.4s, v14.4s, v6.4s\n" "sqrdmulh v15.4s, v15.4s, v7.4s\n" "tbz %x[flags], #5, 44f\n" - "and v4.16b, v8.16b, v0.16b\n" - "and v5.16b, v9.16b, v1.16b\n" - "and v6.16b, v10.16b, v2.16b\n" - "and v7.16b, v11.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v4.4s\n" - "sqadd v9.4s, v9.4s, v5.4s\n" - "sqadd v10.4s, v10.4s, v6.4s\n" - "sqadd v11.4s, v11.4s, v7.4s\n" - "and v4.16b, v12.16b, v0.16b\n" - "and v5.16b, v13.16b, v1.16b\n" - "and v6.16b, v14.16b, v2.16b\n" - "and v7.16b, v15.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v12.4s, v12.4s, v4.4s\n" - "sqadd v13.4s, v13.4s, v5.4s\n" - "sqadd v14.4s, v14.4s, v6.4s\n" - "sqadd v15.4s, v15.4s, v7.4s\n" + "and v19.16b, v8.16b, v0.16b\n" + "and v18.16b, v9.16b, v1.16b\n" + "and v17.16b, v10.16b, v2.16b\n" + "and v16.16b, v11.16b, v3.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v8.4s, v8.4s, v19.4s\n" + "sqadd v9.4s, v9.4s, v18.4s\n" + "sqadd v10.4s, v10.4s, v17.4s\n" + "sqadd v11.4s, v11.4s, v16.4s\n" + "and v19.16b, v12.16b, v0.16b\n" + "and v18.16b, v13.16b, v1.16b\n" + "and v17.16b, v14.16b, v2.16b\n" + "and v16.16b, v15.16b, v3.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v12.4s, v12.4s, v19.4s\n" + "sqadd v13.4s, v13.4s, v18.4s\n" + "sqadd v14.4s, v14.4s, v17.4s\n" + "sqadd v15.4s, v15.4s, v16.4s\n" "44:" // Height 2: no shift correction "srshl v8.4s, v8.4s, v0.4s\n" "srshl v9.4s, v9.4s, v1.4s\n" @@ -741,108 +740,108 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "srshl v13.4s, v13.4s, v1.4s\n" "srshl v14.4s, v14.4s, v2.4s\n" "srshl v15.4s, v15.4s, v3.4s\n" - "add x25, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x25]\n" - "add v8.4s, v8.4s, v4.4s\n" - "add v9.4s, v9.4s, v4.4s\n" - "add v10.4s, v10.4s, v4.4s\n" - "add v11.4s, v11.4s, v4.4s\n" - "add v12.4s, v12.4s, v4.4s\n" - "add v13.4s, v13.4s, v4.4s\n" - "add v14.4s, v14.4s, v4.4s\n" - "add v15.4s, v15.4s, v4.4s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x25]\n" - "smin v8.4s, v8.4s, v6.4s\n" - "smin v9.4s, v9.4s, v6.4s\n" - "smin v10.4s, v10.4s, v6.4s\n" - "smin v11.4s, v11.4s, v6.4s\n" - "smin v12.4s, v12.4s, v6.4s\n" - "smin v13.4s, v13.4s, v6.4s\n" - "smin v14.4s, v14.4s, v6.4s\n" - "smin v15.4s, v15.4s, v6.4s\n" - "add x25, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x25]\n" - "smax v8.4s, v8.4s, v5.4s\n" - "smax v9.4s, v9.4s, v5.4s\n" - "smax v10.4s, v10.4s, v5.4s\n" - "smax v11.4s, v11.4s, v5.4s\n" - "smax v12.4s, v12.4s, v5.4s\n" - "smax v13.4s, v13.4s, v5.4s\n" - "smax v14.4s, v14.4s, v5.4s\n" - "smax v15.4s, v15.4s, v5.4s\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v16.4s }, [x20]\n" + "add v8.4s, v8.4s, v16.4s\n" + "add v9.4s, v9.4s, v16.4s\n" + "add v10.4s, v10.4s, v16.4s\n" + "add v11.4s, v11.4s, v16.4s\n" + "add v12.4s, v12.4s, v16.4s\n" + "add v13.4s, v13.4s, v16.4s\n" + "add v14.4s, v14.4s, v16.4s\n" + "add v15.4s, v15.4s, v16.4s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v16.4s }, [x20]\n" + "smin v8.4s, v8.4s, v16.4s\n" + "smin v9.4s, v9.4s, v16.4s\n" + "smin v10.4s, v10.4s, v16.4s\n" + "smin v11.4s, v11.4s, v16.4s\n" + "smin v12.4s, v12.4s, v16.4s\n" + "smin v13.4s, v13.4s, v16.4s\n" + "smin v14.4s, v14.4s, v16.4s\n" + "smin v15.4s, v15.4s, v16.4s\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v16.4s }, [x20]\n" + "smax v8.4s, v8.4s, v16.4s\n" + "smax v9.4s, v9.4s, v16.4s\n" + "smax v10.4s, v10.4s, v16.4s\n" + "smax v11.4s, v11.4s, v16.4s\n" + "smax v12.4s, v12.4s, v16.4s\n" + "smax v13.4s, v13.4s, v16.4s\n" + "smax v14.4s, v14.4s, v16.4s\n" + "smax v15.4s, v15.4s, v16.4s\n" "uzp1 v8.8h, v8.8h, v9.8h\n" - "uzp1 v9.8h, v10.8h, v11.8h\n" + "uzp1 v17.8h, v10.8h, v11.8h\n" "uzp1 v12.8h, v12.8h, v13.8h\n" - "uzp1 v13.8h, v14.8h, v15.8h\n" + "uzp1 v16.8h, v14.8h, v15.8h\n" "cmp x16, #0x10\n" - "uzp1 v8.16b, v8.16b, v9.16b\n" - "uzp1 v12.16b, v12.16b, v13.16b\n" + "uzp1 v8.16b, v8.16b, v17.16b\n" + "uzp1 v12.16b, v12.16b, v16.16b\n" "bge 53f\n" "tbz x16, #3, 48f\n" "str d8, [x17], #0x8\n" - "str d12, [x24], #0x8\n" + "str d12, [x25], #0x8\n" "tbz x16, #2, 46f\n" "st1 { v8.s }[2], [x17], #0x4\n" - "st1 { v12.s }[2], [x24], #0x4\n" + "st1 { v12.s }[2], [x25], #0x4\n" "tbz x16, #1, 45f\n" "st1 { v8.h }[6], [x17], #0x2\n" - "st1 { v12.h }[6], [x24], #0x2\n" + "st1 { v12.h }[6], [x25], #0x2\n" "tbz x16, #0, 52f\n" "st1 { v8.b }[14], [x17]\n" - "st1 { v12.b }[14], [x24]\n" + "st1 { v12.b }[14], [x25]\n" "b 52f\n" "45:" // Height 2: Partial direct writeback: partial_1_12 "tbz x16, #0, 52f\n" "st1 { v8.b }[12], [x17]\n" - "st1 { v12.b }[12], [x24]\n" + "st1 { v12.b }[12], [x25]\n" "b 52f\n" "46:" // Height 2: Partial direct writeback: partial_2_8 "tbz x16, #1, 47f\n" "st1 { v8.h }[4], [x17], #0x2\n" - "st1 { v12.h }[4], [x24], #0x2\n" + "st1 { v12.h }[4], [x25], #0x2\n" "tbz x16, #0, 52f\n" "st1 { v8.b }[10], [x17]\n" - "st1 { v12.b }[10], [x24]\n" + "st1 { v12.b }[10], [x25]\n" "b 52f\n" "47:" // Height 2: Partial direct writeback: partial_1_8 "tbz x16, #0, 52f\n" "st1 { v8.b }[8], [x17]\n" - "st1 { v12.b }[8], [x24]\n" + "st1 { v12.b }[8], [x25]\n" "b 52f\n" "48:" // Height 2: Partial direct writeback: partial_4_0 "tbz x16, #2, 50f\n" "str s8, [x17], #0x4\n" - "str s12, [x24], #0x4\n" + "str s12, [x25], #0x4\n" "tbz x16, #1, 49f\n" "st1 { v8.h }[2], [x17], #0x2\n" - "st1 { v12.h }[2], [x24], #0x2\n" + "st1 { v12.h }[2], [x25], #0x2\n" "tbz x16, #0, 52f\n" "st1 { v8.b }[6], [x17]\n" - "st1 { v12.b }[6], [x24]\n" + "st1 { v12.b }[6], [x25]\n" "b 52f\n" "49:" // Height 2: Partial direct writeback: partial_1_4 "tbz x16, #0, 52f\n" "st1 { v8.b }[4], [x17]\n" - "st1 { v12.b }[4], [x24]\n" + "st1 { v12.b }[4], [x25]\n" "b 52f\n" "50:" // Height 2: Partial direct writeback: partial_2_0 "tbz x16, #1, 51f\n" "str h8, [x17], #0x2\n" - "str h12, [x24], #0x2\n" + "str h12, [x25], #0x2\n" "tbz x16, #0, 52f\n" "st1 { v8.b }[2], [x17]\n" - "st1 { v12.b }[2], [x24]\n" + "st1 { v12.b }[2], [x25]\n" "b 52f\n" "51:" // Height 2: Partial direct writeback: partial_1_0 "str b8, [x17, #0x0]\n" - "str b12, [x24, #0x0]\n" + "str b12, [x25, #0x0]\n" "52:" // Height 2: Partial direct writeback: Done "b 54f\n" "53:" // Height 2: Full writeback "str q8, [x17, #0x0]\n" "add x17, x17, #0x10\n" - "str q12, [x24, #0x0]\n" + "str q12, [x25, #0x0]\n" "54:" // Height 2: Writeback done "subs x16, x16, #0x10\n" "bgt 29b\n" @@ -872,308 +871,308 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "58:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w13, [x20, x14, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 59f\n" - "ldr x21, [%x[input_ptr], x14, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x12, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" - "ldr x27, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x12, [x20, #0x0]\n" + "ldr x11, [x20, #0x8]\n" + "ldr x10, [x20, #0x10]\n" "cbnz x14, 60f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x12, x12, x20\n" - "add x9, x9, x20\n" - "add x27, x27, x20\n" + "add x11, x11, x20\n" + "add x10, x10, x20\n" "b 60f\n" "59:" // Height 3: setup direct input "mov x12, %x[input_ptr]\n" - "add x9, x12, x20\n" - "add x27, x9, x20\n" + "add x11, x12, x21\n" + "add x10, x11, x21\n" "60:" // Height 3: input setup done "cmp x13, #0x10\n" "blt 63f\n" "ldr q0, [x12, #0x0]\n" "cmp x13, #0x20\n" - "ldr q1, [x9, #0x0]\n" - "ldr q2, [x27, #0x0]\n" + "ldr q1, [x11, #0x0]\n" + "ldr q2, [x10, #0x0]\n" "ldr q6, [x15, #0x0]\n" "ldr q7, [x15, #0x10]\n" "blt 62f\n" "61:" // Height 3: Multiply loop: Main loop head ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr x20, [x15, #0x28]\n" + "ldr x21, [x15, #0x28]\n" ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr x11, [x15, #0x38]\n" + "ldr x20, [x15, #0x38]\n" ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - "ldr d6, [x15, #0x20]\n" + "ldr d21, [x15, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x20\n" + "mov v21.d[1], x21\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "ldr x20, [x15, #0x48]\n" + "ldr x21, [x15, #0x48]\n" ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - "ldr d7, [x15, #0x30]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "ldr x11, [x15, #0x58]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - "ldr d6, [x15, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x20\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - "ldr x20, [x15, #0x68]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - "ldr d7, [x15, #0x50]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - "ldr x11, [x15, #0x78]\n" - ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" - "ldr d6, [x15, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" + "ldr d20, [x15, #0x30]\n" + "mov v20.d[1], x20\n" + ".inst 0x4f80e2aa // sdot v10.4s, v21.16b, v0.4b[0]\n" + ".inst 0x4f81e2ae // sdot v14.4s, v21.16b, v1.4b[0]\n" + "ldr x20, [x15, #0x58]\n" + ".inst 0x4f82e2b2 // sdot v18.4s, v21.16b, v2.4b[0]\n" + "ldr d21, [x15, #0x40]\n" + ".inst 0x4f80e28b // sdot v11.4s, v20.16b, v0.4b[0]\n" + "mov v21.d[1], x21\n" + ".inst 0x4f81e28f // sdot v15.4s, v20.16b, v1.4b[0]\n" + "ldr x21, [x15, #0x68]\n" + ".inst 0x4f82e293 // sdot v19.4s, v20.16b, v2.4b[0]\n" + "ldr d20, [x15, #0x50]\n" + "mov v20.d[1], x20\n" + ".inst 0x4fa0e2a8 // sdot v8.4s, v21.16b, v0.4b[1]\n" + ".inst 0x4fa1e2ac // sdot v12.4s, v21.16b, v1.4b[1]\n" + "ldr x20, [x15, #0x78]\n" + ".inst 0x4fa2e2b0 // sdot v16.4s, v21.16b, v2.4b[1]\n" + "ldr d21, [x15, #0x60]\n" + ".inst 0x4fa0e289 // sdot v9.4s, v20.16b, v0.4b[1]\n" + "mov v21.d[1], x21\n" + ".inst 0x4fa1e28d // sdot v13.4s, v20.16b, v1.4b[1]\n" + "ldr x21, [x15, #0x88]\n" + ".inst 0x4fa2e291 // sdot v17.4s, v20.16b, v2.4b[1]\n" + "ldr d20, [x15, #0x70]\n" + "mov v20.d[1], x20\n" + ".inst 0x4fa0e2aa // sdot v10.4s, v21.16b, v0.4b[1]\n" + ".inst 0x4fa1e2ae // sdot v14.4s, v21.16b, v1.4b[1]\n" + "ldr x20, [x15, #0x98]\n" + ".inst 0x4fa2e2b2 // sdot v18.4s, v21.16b, v2.4b[1]\n" + "ldr d21, [x15, #0x80]\n" + ".inst 0x4fa0e28b // sdot v11.4s, v20.16b, v0.4b[1]\n" + "mov v21.d[1], x21\n" + ".inst 0x4fa1e28f // sdot v15.4s, v20.16b, v1.4b[1]\n" + "ldr x21, [x15, #0xa8]\n" + ".inst 0x4fa2e293 // sdot v19.4s, v20.16b, v2.4b[1]\n" + "ldr d20, [x15, #0x90]\n" + "mov v20.d[1], x20\n" + ".inst 0x4f80eaa8 // sdot v8.4s, v21.16b, v0.4b[2]\n" + ".inst 0x4f81eaac // sdot v12.4s, v21.16b, v1.4b[2]\n" + "ldr x20, [x15, #0xb8]\n" + ".inst 0x4f82eab0 // sdot v16.4s, v21.16b, v2.4b[2]\n" + "ldr d21, [x15, #0xa0]\n" + ".inst 0x4f80ea89 // sdot v9.4s, v20.16b, v0.4b[2]\n" + "mov v21.d[1], x21\n" + ".inst 0x4f81ea8d // sdot v13.4s, v20.16b, v1.4b[2]\n" + "ldr x21, [x15, #0xc8]\n" + ".inst 0x4f82ea91 // sdot v17.4s, v20.16b, v2.4b[2]\n" + "ldr d20, [x15, #0xb0]\n" + "mov v20.d[1], x20\n" + ".inst 0x4f80eaaa // sdot v10.4s, v21.16b, v0.4b[2]\n" + ".inst 0x4f81eaae // sdot v14.4s, v21.16b, v1.4b[2]\n" + "ldr x20, [x15, #0xd8]\n" + ".inst 0x4f82eab2 // sdot v18.4s, v21.16b, v2.4b[2]\n" + "ldr d21, [x15, #0xc0]\n" + ".inst 0x4f80ea8b // sdot v11.4s, v20.16b, v0.4b[2]\n" + "mov v21.d[1], x21\n" + ".inst 0x4f81ea8f // sdot v15.4s, v20.16b, v1.4b[2]\n" + "ldr x21, [x15, #0xe8]\n" + ".inst 0x4f82ea93 // sdot v19.4s, v20.16b, v2.4b[2]\n" + "ldr d20, [x15, #0xd0]\n" + "mov v20.d[1], x20\n" + ".inst 0x4fa0eaa8 // sdot v8.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa1eaac // sdot v12.4s, v21.16b, v1.4b[3]\n" + "ldr x20, [x15, #0xf8]\n" + ".inst 0x4fa2eab0 // sdot v16.4s, v21.16b, v2.4b[3]\n" + "ldr d21, [x15, #0xe0]\n" + ".inst 0x4fa0ea89 // sdot v9.4s, v20.16b, v0.4b[3]\n" + "mov v21.d[1], x21\n" + ".inst 0x4fa1ea8d // sdot v13.4s, v20.16b, v1.4b[3]\n" + "add x12, x12, #0x10\n" + ".inst 0x4fa2ea91 // sdot v17.4s, v20.16b, v2.4b[3]\n" + "ldr d20, [x15, #0xf0]\n" + "mov v20.d[1], x20\n" + "add x11, x11, #0x10\n" + "add x10, x10, #0x10\n" + "add x15, x15, #0x100\n" + ".inst 0x4fa0eaaa // sdot v10.4s, v21.16b, v0.4b[3]\n" + "ldr x20, [x15, #0x8]\n" + ".inst 0x4fa1eaae // sdot v14.4s, v21.16b, v1.4b[3]\n" + "ldr x23, [x12, #0x8]\n" + ".inst 0x4fa2eab2 // sdot v18.4s, v21.16b, v2.4b[3]\n" + "ldr d6, [x15, #0x0]\n" + ".inst 0x4fa0ea8b // sdot v11.4s, v20.16b, v0.4b[3]\n" + "ldr d0, [x12, #0x0]\n" + ".inst 0x4fa1ea8f // sdot v15.4s, v20.16b, v1.4b[3]\n" + "ldr d1, [x11, #0x0]\n" + "ldr x22, [x11, #0x8]\n" + ".inst 0x4fa2ea93 // sdot v19.4s, v20.16b, v2.4b[3]\n" + "ldr d2, [x10, #0x0]\n" + "sub x13, x13, #0x10\n" + "ldr d7, [x15, #0x10]\n" + "cmp x13, #0x20\n" + "ldr x21, [x10, #0x8]\n" "mov v6.d[1], x20\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - "ldr x20, [x15, #0x88]\n" - ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - "ldr d7, [x15, #0x70]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - "ldr x11, [x15, #0x98]\n" - ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - "ldr d6, [x15, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - "mov v6.d[1], x20\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - "ldr x20, [x15, #0xa8]\n" - ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" - "ldr d7, [x15, #0x90]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - "ldr x11, [x15, #0xb8]\n" - ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" - "ldr d6, [x15, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - "mov v6.d[1], x20\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - "ldr x20, [x15, #0xc8]\n" - ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" - "ldr d7, [x15, #0xb0]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - "ldr x11, [x15, #0xd8]\n" - ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" - "ldr d6, [x15, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - "mov v6.d[1], x20\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - "ldr x20, [x15, #0xe8]\n" - ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" - "ldr d7, [x15, #0xd0]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - "ldr x11, [x15, #0xf8]\n" - ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" - "ldr d6, [x15, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - "mov v6.d[1], x20\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - "add x12, x12, #0x10\n" - ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" - "ldr d7, [x15, #0xf0]\n" - "mov v7.d[1], x11\n" - "add x9, x9, #0x10\n" - "add x27, x27, #0x10\n" - "add x15, x15, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - "ldr x20, [x15, #0x8]\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - "ldr x10, [x12, #0x8]\n" - ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" - "ldr d6, [x15, #0x0]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" - "ldr d0, [x12, #0x0]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" - "ldr d1, [x9, #0x0]\n" - "ldr x28, [x9, #0x8]\n" - ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" - "ldr d2, [x27, #0x0]\n" - "sub x13, x13, #0x10\n" - "ldr d7, [x15, #0x10]\n" - "cmp x13, #0x20\n" - "ldr x26, [x27, #0x8]\n" - "mov v6.d[1], x20\n" - "ldr x11, [x15, #0x18]\n" - "mov v0.d[1], x10\n" + "ldr x20, [x15, #0x18]\n" + "mov v0.d[1], x23\n" "prfm pldl1keep, [x12, #0x80]\n" - "mov v1.d[1], x28\n" - "prfm pldl1keep, [x9, #0x80]\n" - "mov v2.d[1], x26\n" - "prfm pldl1keep, [x27, #0x80]\n" - "mov v7.d[1], x11\n" + "mov v1.d[1], x22\n" + "prfm pldl1keep, [x11, #0x80]\n" + "mov v2.d[1], x21\n" + "prfm pldl1keep, [x10, #0x80]\n" + "mov v7.d[1], x20\n" "bge 61b\n" "62:" // Height 3: Multiply loop: Single iteration only ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" "add x12, x12, #0x10\n" ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "add x9, x9, #0x10\n" + "add x11, x11, #0x10\n" ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x15, #0x20]\n" + "ldr q21, [x15, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "add x27, x27, #0x10\n" + "add x10, x10, #0x10\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" "sub x13, x13, #0x10\n" ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x15, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q20, [x15, #0x30]\n" + ".inst 0x4f80e2aa // sdot v10.4s, v21.16b, v0.4b[0]\n" "prfm pldl1keep, [x12, #0x80]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "prfm pldl1keep, [x9, #0x80]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x15, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - "prfm pldl1keep, [x27, #0x80]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x15, #0x50]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" - "ldr q6, [x15, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - "ldr q7, [x15, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - "ldr q6, [x15, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" - "ldr q7, [x15, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" - "ldr q6, [x15, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" - "ldr q7, [x15, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" - "ldr q6, [x15, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" - "ldr q7, [x15, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" - "ldr q6, [x15, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" - "ldr q7, [x15, #0xf0]\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" + ".inst 0x4f81e2ae // sdot v14.4s, v21.16b, v1.4b[0]\n" + "prfm pldl1keep, [x11, #0x80]\n" + ".inst 0x4f82e2b2 // sdot v18.4s, v21.16b, v2.4b[0]\n" + "ldr q21, [x15, #0x40]\n" + ".inst 0x4f80e28b // sdot v11.4s, v20.16b, v0.4b[0]\n" + "prfm pldl1keep, [x10, #0x80]\n" + ".inst 0x4f81e28f // sdot v15.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f82e293 // sdot v19.4s, v20.16b, v2.4b[0]\n" + "ldr q20, [x15, #0x50]\n" + ".inst 0x4fa0e2a8 // sdot v8.4s, v21.16b, v0.4b[1]\n" + ".inst 0x4fa1e2ac // sdot v12.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa2e2b0 // sdot v16.4s, v21.16b, v2.4b[1]\n" + "ldr q21, [x15, #0x60]\n" + ".inst 0x4fa0e289 // sdot v9.4s, v20.16b, v0.4b[1]\n" + ".inst 0x4fa1e28d // sdot v13.4s, v20.16b, v1.4b[1]\n" + ".inst 0x4fa2e291 // sdot v17.4s, v20.16b, v2.4b[1]\n" + "ldr q20, [x15, #0x70]\n" + ".inst 0x4fa0e2aa // sdot v10.4s, v21.16b, v0.4b[1]\n" + ".inst 0x4fa1e2ae // sdot v14.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa2e2b2 // sdot v18.4s, v21.16b, v2.4b[1]\n" + "ldr q21, [x15, #0x80]\n" + ".inst 0x4fa0e28b // sdot v11.4s, v20.16b, v0.4b[1]\n" + ".inst 0x4fa1e28f // sdot v15.4s, v20.16b, v1.4b[1]\n" + ".inst 0x4fa2e293 // sdot v19.4s, v20.16b, v2.4b[1]\n" + "ldr q20, [x15, #0x90]\n" + ".inst 0x4f80eaa8 // sdot v8.4s, v21.16b, v0.4b[2]\n" + ".inst 0x4f81eaac // sdot v12.4s, v21.16b, v1.4b[2]\n" + ".inst 0x4f82eab0 // sdot v16.4s, v21.16b, v2.4b[2]\n" + "ldr q21, [x15, #0xa0]\n" + ".inst 0x4f80ea89 // sdot v9.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f81ea8d // sdot v13.4s, v20.16b, v1.4b[2]\n" + ".inst 0x4f82ea91 // sdot v17.4s, v20.16b, v2.4b[2]\n" + "ldr q20, [x15, #0xb0]\n" + ".inst 0x4f80eaaa // sdot v10.4s, v21.16b, v0.4b[2]\n" + ".inst 0x4f81eaae // sdot v14.4s, v21.16b, v1.4b[2]\n" + ".inst 0x4f82eab2 // sdot v18.4s, v21.16b, v2.4b[2]\n" + "ldr q21, [x15, #0xc0]\n" + ".inst 0x4f80ea8b // sdot v11.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f81ea8f // sdot v15.4s, v20.16b, v1.4b[2]\n" + ".inst 0x4f82ea93 // sdot v19.4s, v20.16b, v2.4b[2]\n" + "ldr q20, [x15, #0xd0]\n" + ".inst 0x4fa0eaa8 // sdot v8.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa1eaac // sdot v12.4s, v21.16b, v1.4b[3]\n" + ".inst 0x4fa2eab0 // sdot v16.4s, v21.16b, v2.4b[3]\n" + "ldr q21, [x15, #0xe0]\n" + ".inst 0x4fa0ea89 // sdot v9.4s, v20.16b, v0.4b[3]\n" + ".inst 0x4fa1ea8d // sdot v13.4s, v20.16b, v1.4b[3]\n" + ".inst 0x4fa2ea91 // sdot v17.4s, v20.16b, v2.4b[3]\n" + "ldr q20, [x15, #0xf0]\n" + ".inst 0x4fa0eaaa // sdot v10.4s, v21.16b, v0.4b[3]\n" "add x15, x15, #0x100\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" + ".inst 0x4fa1eaae // sdot v14.4s, v21.16b, v1.4b[3]\n" + ".inst 0x4fa2eab2 // sdot v18.4s, v21.16b, v2.4b[3]\n" + ".inst 0x4fa0ea8b // sdot v11.4s, v20.16b, v0.4b[3]\n" + ".inst 0x4fa1ea8f // sdot v15.4s, v20.16b, v1.4b[3]\n" + ".inst 0x4fa2ea93 // sdot v19.4s, v20.16b, v2.4b[3]\n" "63:" // Height 3: Multiply loop: Main loop skip "cbz x13, 68f\n" "cmp x13, #0x4\n" "blt 65f\n" "64:" // Height 3: Multiply loop: Odd block loop - "ldr s0, [x12], #0x4\n" + "ldr s24, [x12], #0x4\n" "sub x13, x13, #0x4\n" - "ldr s1, [x9], #0x4\n" + "ldr s23, [x11], #0x4\n" "cmp x13, #0x4\n" - "ldr s2, [x27], #0x4\n" - "ldr q6, [x15, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x15, #0x10]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x15, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x15, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr s22, [x10], #0x4\n" + "ldr q21, [x15, #0x0]\n" + ".inst 0x4f98e2a8 // sdot v8.4s, v21.16b, v24.4b[0]\n" + "ldr q20, [x15, #0x10]\n" + ".inst 0x4f97e2ac // sdot v12.4s, v21.16b, v23.4b[0]\n" + ".inst 0x4f96e2b0 // sdot v16.4s, v21.16b, v22.4b[0]\n" + "ldr q21, [x15, #0x20]\n" + ".inst 0x4f98e289 // sdot v9.4s, v20.16b, v24.4b[0]\n" + ".inst 0x4f97e28d // sdot v13.4s, v20.16b, v23.4b[0]\n" + ".inst 0x4f96e291 // sdot v17.4s, v20.16b, v22.4b[0]\n" + "ldr q20, [x15, #0x30]\n" + ".inst 0x4f98e2aa // sdot v10.4s, v21.16b, v24.4b[0]\n" "add x15, x15, #0x40\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" + ".inst 0x4f97e2ae // sdot v14.4s, v21.16b, v23.4b[0]\n" + ".inst 0x4f96e2b2 // sdot v18.4s, v21.16b, v22.4b[0]\n" + ".inst 0x4f98e28b // sdot v11.4s, v20.16b, v24.4b[0]\n" + ".inst 0x4f97e28f // sdot v15.4s, v20.16b, v23.4b[0]\n" + ".inst 0x4f96e293 // sdot v19.4s, v20.16b, v22.4b[0]\n" "bge 64b\n" "65:" // Height 3: Multiply loop: Skip odd blocks "cbz x13, 68f\n" "tbz x13, #1, 66f\n" "ldr h0, [x12], #0x2\n" - "ldr h1, [x9], #0x2\n" - "ldr h2, [x27], #0x2\n" + "ldr h1, [x11], #0x2\n" + "ldr h2, [x10], #0x2\n" "tbz x13, #0, 67f\n" "ld1 { v0.b }[2], [x12]\n" - "ld1 { v1.b }[2], [x9]\n" - "ld1 { v2.b }[2], [x27]\n" + "ld1 { v1.b }[2], [x11]\n" + "ld1 { v2.b }[2], [x10]\n" "b 67f\n" "66:" // Height 3: Multiply loop: Ragged operand read: partial_1_0 "ldr b0, [x12, #0x0]\n" - "ldr b1, [x9, #0x0]\n" - "ldr b2, [x27, #0x0]\n" + "ldr b1, [x11, #0x0]\n" + "ldr b2, [x10, #0x0]\n" "67:" // Height 3: Multiply loop: Ragged operand read: Done - "ldr q6, [x15, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x15, #0x10]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x15, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x15, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q21, [x15, #0x0]\n" + ".inst 0x4f80e2a8 // sdot v8.4s, v21.16b, v0.4b[0]\n" + "ldr q20, [x15, #0x10]\n" + ".inst 0x4f81e2ac // sdot v12.4s, v21.16b, v1.4b[0]\n" + ".inst 0x4f82e2b0 // sdot v16.4s, v21.16b, v2.4b[0]\n" + "ldr q21, [x15, #0x20]\n" + ".inst 0x4f80e289 // sdot v9.4s, v20.16b, v0.4b[0]\n" + ".inst 0x4f81e28d // sdot v13.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f82e291 // sdot v17.4s, v20.16b, v2.4b[0]\n" + "ldr q20, [x15, #0x30]\n" + ".inst 0x4f80e2aa // sdot v10.4s, v21.16b, v0.4b[0]\n" "add x15, x15, #0x40\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" + ".inst 0x4f81e2ae // sdot v14.4s, v21.16b, v1.4b[0]\n" + ".inst 0x4f82e2b2 // sdot v18.4s, v21.16b, v2.4b[0]\n" + ".inst 0x4f80e28b // sdot v11.4s, v20.16b, v0.4b[0]\n" + ".inst 0x4f81e28f // sdot v15.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f82e293 // sdot v19.4s, v20.16b, v2.4b[0]\n" "68:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x14, x14, #0x1\n" "cmp x14, x20\n" "bne 58b\n" - "ldr q0, [x6, #0x0]\n" - "add v8.4s, v8.4s, v0.4s\n" - "ldr q1, [x6, #0x10]\n" - "add v9.4s, v9.4s, v1.4s\n" - "ldr q2, [x6, #0x20]\n" - "add v10.4s, v10.4s, v2.4s\n" - "ldr q3, [x6, #0x30]\n" - "add v11.4s, v11.4s, v3.4s\n" + "ldr q23, [x6, #0x0]\n" + "add v8.4s, v8.4s, v23.4s\n" + "ldr q22, [x6, #0x10]\n" + "add v9.4s, v9.4s, v22.4s\n" + "ldr q21, [x6, #0x20]\n" + "add v10.4s, v10.4s, v21.4s\n" + "ldr q20, [x6, #0x30]\n" + "add v11.4s, v11.4s, v20.4s\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x17, x20\n" - "add x23, x24, x20\n" + "add x25, x17, x20\n" + "add x24, x25, x20\n" "prfm pstl1keep, [x17, #0x0]\n" + "prfm pstl1keep, [x25, #0x0]\n" + "add v12.4s, v12.4s, v23.4s\n" "prfm pstl1keep, [x24, #0x0]\n" - "add v12.4s, v12.4s, v0.4s\n" - "prfm pstl1keep, [x23, #0x0]\n" - "add v13.4s, v13.4s, v1.4s\n" - "add v14.4s, v14.4s, v2.4s\n" - "add v15.4s, v15.4s, v3.4s\n" - "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" + "add v13.4s, v13.4s, v22.4s\n" + "add v14.4s, v14.4s, v21.4s\n" + "add v15.4s, v15.4s, v20.4s\n" + "add v16.4s, v16.4s, v23.4s\n" + "add v17.4s, v17.4s, v22.4s\n" + "add v18.4s, v18.4s, v21.4s\n" + "add v19.4s, v19.4s, v20.4s\n" "add x6, x6, #0x40\n" "tbz %x[flags], #4, 69f\n" "ldr q0, [x8, #0x0]\n" @@ -1188,10 +1187,10 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "add x7, x7, #0x40\n" "b 70f\n" "69:" // Height 3: per layer parameters - "add x25, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x25]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v4.4s }, [x20]\n" "mov v1.16b, v0.16b\n" "mov v5.16b, v4.16b\n" "mov v2.16b, v0.16b\n" @@ -1212,42 +1211,42 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "sqrdmulh v18.4s, v18.4s, v6.4s\n" "sqrdmulh v19.4s, v19.4s, v7.4s\n" "tbz %x[flags], #5, 71f\n" - "and v4.16b, v8.16b, v0.16b\n" - "and v5.16b, v9.16b, v1.16b\n" - "and v6.16b, v10.16b, v2.16b\n" - "and v7.16b, v11.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v4.4s\n" - "sqadd v9.4s, v9.4s, v5.4s\n" - "sqadd v10.4s, v10.4s, v6.4s\n" - "sqadd v11.4s, v11.4s, v7.4s\n" - "and v4.16b, v12.16b, v0.16b\n" - "and v5.16b, v13.16b, v1.16b\n" - "and v6.16b, v14.16b, v2.16b\n" - "and v7.16b, v15.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v12.4s, v12.4s, v4.4s\n" - "sqadd v13.4s, v13.4s, v5.4s\n" - "sqadd v14.4s, v14.4s, v6.4s\n" - "sqadd v15.4s, v15.4s, v7.4s\n" - "and v4.16b, v16.16b, v0.16b\n" - "and v5.16b, v17.16b, v1.16b\n" - "and v6.16b, v18.16b, v2.16b\n" - "and v7.16b, v19.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" + "and v23.16b, v8.16b, v0.16b\n" + "and v22.16b, v9.16b, v1.16b\n" + "and v21.16b, v10.16b, v2.16b\n" + "and v20.16b, v11.16b, v3.16b\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "sshr v21.4s, v21.4s, #0x1f\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "sqadd v8.4s, v8.4s, v23.4s\n" + "sqadd v9.4s, v9.4s, v22.4s\n" + "sqadd v10.4s, v10.4s, v21.4s\n" + "sqadd v11.4s, v11.4s, v20.4s\n" + "and v23.16b, v12.16b, v0.16b\n" + "and v22.16b, v13.16b, v1.16b\n" + "and v21.16b, v14.16b, v2.16b\n" + "and v20.16b, v15.16b, v3.16b\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "sshr v21.4s, v21.4s, #0x1f\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "sqadd v12.4s, v12.4s, v23.4s\n" + "sqadd v13.4s, v13.4s, v22.4s\n" + "sqadd v14.4s, v14.4s, v21.4s\n" + "sqadd v15.4s, v15.4s, v20.4s\n" + "and v23.16b, v16.16b, v0.16b\n" + "and v22.16b, v17.16b, v1.16b\n" + "and v21.16b, v18.16b, v2.16b\n" + "and v20.16b, v19.16b, v3.16b\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "sshr v21.4s, v21.4s, #0x1f\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v23.4s\n" + "sqadd v17.4s, v17.4s, v22.4s\n" + "sqadd v18.4s, v18.4s, v21.4s\n" + "sqadd v19.4s, v19.4s, v20.4s\n" "71:" // Height 3: no shift correction "srshl v8.4s, v8.4s, v0.4s\n" "srshl v9.4s, v9.4s, v1.4s\n" @@ -1261,139 +1260,139 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "srshl v17.4s, v17.4s, v1.4s\n" "srshl v18.4s, v18.4s, v2.4s\n" "srshl v19.4s, v19.4s, v3.4s\n" - "add x25, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x25]\n" - "add v8.4s, v8.4s, v4.4s\n" - "add v9.4s, v9.4s, v4.4s\n" - "add v10.4s, v10.4s, v4.4s\n" - "add v11.4s, v11.4s, v4.4s\n" - "add v12.4s, v12.4s, v4.4s\n" - "add v13.4s, v13.4s, v4.4s\n" - "add v14.4s, v14.4s, v4.4s\n" - "add v15.4s, v15.4s, v4.4s\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x25]\n" - "smin v8.4s, v8.4s, v6.4s\n" - "smin v9.4s, v9.4s, v6.4s\n" - "smin v10.4s, v10.4s, v6.4s\n" - "smin v11.4s, v11.4s, v6.4s\n" - "smin v12.4s, v12.4s, v6.4s\n" - "smin v13.4s, v13.4s, v6.4s\n" - "smin v14.4s, v14.4s, v6.4s\n" - "smin v15.4s, v15.4s, v6.4s\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "add x25, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x25]\n" - "smax v8.4s, v8.4s, v5.4s\n" - "smax v9.4s, v9.4s, v5.4s\n" - "smax v10.4s, v10.4s, v5.4s\n" - "smax v11.4s, v11.4s, v5.4s\n" - "smax v12.4s, v12.4s, v5.4s\n" - "smax v13.4s, v13.4s, v5.4s\n" - "smax v14.4s, v14.4s, v5.4s\n" - "smax v15.4s, v15.4s, v5.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v20.4s }, [x20]\n" + "add v8.4s, v8.4s, v20.4s\n" + "add v9.4s, v9.4s, v20.4s\n" + "add v10.4s, v10.4s, v20.4s\n" + "add v11.4s, v11.4s, v20.4s\n" + "add v12.4s, v12.4s, v20.4s\n" + "add v13.4s, v13.4s, v20.4s\n" + "add v14.4s, v14.4s, v20.4s\n" + "add v15.4s, v15.4s, v20.4s\n" + "add v16.4s, v16.4s, v20.4s\n" + "add v17.4s, v17.4s, v20.4s\n" + "add v18.4s, v18.4s, v20.4s\n" + "add v19.4s, v19.4s, v20.4s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v20.4s }, [x20]\n" + "smin v8.4s, v8.4s, v20.4s\n" + "smin v9.4s, v9.4s, v20.4s\n" + "smin v10.4s, v10.4s, v20.4s\n" + "smin v11.4s, v11.4s, v20.4s\n" + "smin v12.4s, v12.4s, v20.4s\n" + "smin v13.4s, v13.4s, v20.4s\n" + "smin v14.4s, v14.4s, v20.4s\n" + "smin v15.4s, v15.4s, v20.4s\n" + "smin v16.4s, v16.4s, v20.4s\n" + "smin v17.4s, v17.4s, v20.4s\n" + "smin v18.4s, v18.4s, v20.4s\n" + "smin v19.4s, v19.4s, v20.4s\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v20.4s }, [x20]\n" + "smax v8.4s, v8.4s, v20.4s\n" + "smax v9.4s, v9.4s, v20.4s\n" + "smax v10.4s, v10.4s, v20.4s\n" + "smax v11.4s, v11.4s, v20.4s\n" + "smax v12.4s, v12.4s, v20.4s\n" + "smax v13.4s, v13.4s, v20.4s\n" + "smax v14.4s, v14.4s, v20.4s\n" + "smax v15.4s, v15.4s, v20.4s\n" + "smax v16.4s, v16.4s, v20.4s\n" + "smax v17.4s, v17.4s, v20.4s\n" + "smax v18.4s, v18.4s, v20.4s\n" + "smax v19.4s, v19.4s, v20.4s\n" "uzp1 v8.8h, v8.8h, v9.8h\n" - "uzp1 v9.8h, v10.8h, v11.8h\n" + "uzp1 v21.8h, v10.8h, v11.8h\n" "uzp1 v12.8h, v12.8h, v13.8h\n" - "uzp1 v13.8h, v14.8h, v15.8h\n" + "uzp1 v20.8h, v14.8h, v15.8h\n" "uzp1 v16.8h, v16.8h, v17.8h\n" "uzp1 v17.8h, v18.8h, v19.8h\n" "cmp x16, #0x10\n" - "uzp1 v8.16b, v8.16b, v9.16b\n" - "uzp1 v12.16b, v12.16b, v13.16b\n" + "uzp1 v8.16b, v8.16b, v21.16b\n" + "uzp1 v12.16b, v12.16b, v20.16b\n" "uzp1 v16.16b, v16.16b, v17.16b\n" "bge 80f\n" "tbz x16, #3, 75f\n" "str d8, [x17], #0x8\n" - "str d12, [x24], #0x8\n" - "str d16, [x23], #0x8\n" + "str d12, [x25], #0x8\n" + "str d16, [x24], #0x8\n" "tbz x16, #2, 73f\n" "st1 { v8.s }[2], [x17], #0x4\n" - "st1 { v12.s }[2], [x24], #0x4\n" - "st1 { v16.s }[2], [x23], #0x4\n" + "st1 { v12.s }[2], [x25], #0x4\n" + "st1 { v16.s }[2], [x24], #0x4\n" "tbz x16, #1, 72f\n" "st1 { v8.h }[6], [x17], #0x2\n" - "st1 { v12.h }[6], [x24], #0x2\n" - "st1 { v16.h }[6], [x23], #0x2\n" + "st1 { v12.h }[6], [x25], #0x2\n" + "st1 { v16.h }[6], [x24], #0x2\n" "tbz x16, #0, 79f\n" "st1 { v8.b }[14], [x17]\n" - "st1 { v12.b }[14], [x24]\n" - "st1 { v16.b }[14], [x23]\n" + "st1 { v12.b }[14], [x25]\n" + "st1 { v16.b }[14], [x24]\n" "b 79f\n" "72:" // Height 3: Partial direct writeback: partial_1_12 "tbz x16, #0, 79f\n" "st1 { v8.b }[12], [x17]\n" - "st1 { v12.b }[12], [x24]\n" - "st1 { v16.b }[12], [x23]\n" + "st1 { v12.b }[12], [x25]\n" + "st1 { v16.b }[12], [x24]\n" "b 79f\n" "73:" // Height 3: Partial direct writeback: partial_2_8 "tbz x16, #1, 74f\n" "st1 { v8.h }[4], [x17], #0x2\n" - "st1 { v12.h }[4], [x24], #0x2\n" - "st1 { v16.h }[4], [x23], #0x2\n" + "st1 { v12.h }[4], [x25], #0x2\n" + "st1 { v16.h }[4], [x24], #0x2\n" "tbz x16, #0, 79f\n" "st1 { v8.b }[10], [x17]\n" - "st1 { v12.b }[10], [x24]\n" - "st1 { v16.b }[10], [x23]\n" + "st1 { v12.b }[10], [x25]\n" + "st1 { v16.b }[10], [x24]\n" "b 79f\n" "74:" // Height 3: Partial direct writeback: partial_1_8 "tbz x16, #0, 79f\n" "st1 { v8.b }[8], [x17]\n" - "st1 { v12.b }[8], [x24]\n" - "st1 { v16.b }[8], [x23]\n" + "st1 { v12.b }[8], [x25]\n" + "st1 { v16.b }[8], [x24]\n" "b 79f\n" "75:" // Height 3: Partial direct writeback: partial_4_0 "tbz x16, #2, 77f\n" "str s8, [x17], #0x4\n" - "str s12, [x24], #0x4\n" - "str s16, [x23], #0x4\n" + "str s12, [x25], #0x4\n" + "str s16, [x24], #0x4\n" "tbz x16, #1, 76f\n" "st1 { v8.h }[2], [x17], #0x2\n" - "st1 { v12.h }[2], [x24], #0x2\n" - "st1 { v16.h }[2], [x23], #0x2\n" + "st1 { v12.h }[2], [x25], #0x2\n" + "st1 { v16.h }[2], [x24], #0x2\n" "tbz x16, #0, 79f\n" "st1 { v8.b }[6], [x17]\n" - "st1 { v12.b }[6], [x24]\n" - "st1 { v16.b }[6], [x23]\n" + "st1 { v12.b }[6], [x25]\n" + "st1 { v16.b }[6], [x24]\n" "b 79f\n" "76:" // Height 3: Partial direct writeback: partial_1_4 "tbz x16, #0, 79f\n" "st1 { v8.b }[4], [x17]\n" - "st1 { v12.b }[4], [x24]\n" - "st1 { v16.b }[4], [x23]\n" + "st1 { v12.b }[4], [x25]\n" + "st1 { v16.b }[4], [x24]\n" "b 79f\n" "77:" // Height 3: Partial direct writeback: partial_2_0 "tbz x16, #1, 78f\n" "str h8, [x17], #0x2\n" - "str h12, [x24], #0x2\n" - "str h16, [x23], #0x2\n" + "str h12, [x25], #0x2\n" + "str h16, [x24], #0x2\n" "tbz x16, #0, 79f\n" "st1 { v8.b }[2], [x17]\n" - "st1 { v12.b }[2], [x24]\n" - "st1 { v16.b }[2], [x23]\n" + "st1 { v12.b }[2], [x25]\n" + "st1 { v16.b }[2], [x24]\n" "b 79f\n" "78:" // Height 3: Partial direct writeback: partial_1_0 "str b8, [x17, #0x0]\n" - "str b12, [x24, #0x0]\n" - "str b16, [x23, #0x0]\n" + "str b12, [x25, #0x0]\n" + "str b16, [x24, #0x0]\n" "79:" // Height 3: Partial direct writeback: Done "b 81f\n" "80:" // Height 3: Full writeback "str q8, [x17, #0x0]\n" "add x17, x17, #0x10\n" - "str q12, [x24, #0x0]\n" - "str q16, [x23, #0x0]\n" + "str q12, [x25, #0x0]\n" + "str q16, [x24, #0x0]\n" "81:" // Height 3: Writeback done "subs x16, x16, #0x10\n" "bgt 56b\n" @@ -1427,369 +1426,369 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "85:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w13, [x20, x14, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 86f\n" - "ldr x21, [%x[input_ptr], x14, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x12, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" - "ldr x27, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x12, [x20, #0x0]\n" + "ldr x11, [x20, #0x8]\n" + "ldr x10, [x20, #0x10]\n" + "ldr x9, [x20, #0x18]\n" "cbnz x14, 87f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x12, x12, x20\n" + "add x11, x11, x20\n" + "add x10, x10, x20\n" "add x9, x9, x20\n" - "add x27, x27, x20\n" - "add x25, x25, x20\n" "b 87f\n" "86:" // Height 4: setup direct input "mov x12, %x[input_ptr]\n" - "add x9, x12, x20\n" - "add x27, x9, x20\n" - "add x25, x27, x20\n" + "add x11, x12, x21\n" + "add x10, x11, x21\n" + "add x9, x10, x21\n" "87:" // Height 4: input setup done "cmp x13, #0x10\n" "blt 90f\n" "ldr q0, [x12, #0x0]\n" "cmp x13, #0x20\n" - "ldr q1, [x9, #0x0]\n" - "ldr q2, [x27, #0x0]\n" - "ldr q3, [x25, #0x0]\n" + "ldr q1, [x11, #0x0]\n" + "ldr q2, [x10, #0x0]\n" + "ldr q3, [x9, #0x0]\n" "ldr q6, [x15, #0x0]\n" "ldr q7, [x15, #0x10]\n" "blt 89f\n" "88:" // Height 4: Multiply loop: Main loop head ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr x20, [x15, #0x28]\n" + "ldr x21, [x15, #0x28]\n" ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr x11, [x15, #0x38]\n" + "ldr x20, [x15, #0x38]\n" ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" "add x12, x12, #0x10\n" ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "ldr d6, [x15, #0x20]\n" + "ldr d25, [x15, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x20\n" + "mov v25.d[1], x21\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "ldr x20, [x15, #0x48]\n" + "ldr x21, [x15, #0x48]\n" ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - "add x9, x9, #0x10\n" + "add x11, x11, #0x10\n" ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - "ldr d7, [x15, #0x30]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "ldr x11, [x15, #0x58]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - "add x27, x27, #0x10\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - "ldr d6, [x15, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x20\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - "ldr x20, [x15, #0x68]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - "add x25, x25, #0x10\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - "ldr d7, [x15, #0x50]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - "ldr x11, [x15, #0x78]\n" - ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" - "ldr x10, [x12, #0x8]\n" - ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n" - "ldr d6, [x15, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - "mov v6.d[1], x20\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - "ldr x20, [x15, #0x88]\n" - ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - "ldr x28, [x9, #0x8]\n" - ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n" - "ldr d7, [x15, #0x70]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - "ldr x11, [x15, #0x98]\n" - ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - "ldr x26, [x27, #0x8]\n" - ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n" - "ldr d6, [x15, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - "mov v6.d[1], x20\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - "ldr x20, [x15, #0xa8]\n" - ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" - "ldr x24, [x25, #0x8]\n" - ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n" - "ldr d7, [x15, #0x90]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - "ldr x11, [x15, #0xb8]\n" - ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" + "ldr d24, [x15, #0x30]\n" + "mov v24.d[1], x20\n" + ".inst 0x4f80e32a // sdot v10.4s, v25.16b, v0.4b[0]\n" + ".inst 0x4f81e32e // sdot v14.4s, v25.16b, v1.4b[0]\n" + "ldr x20, [x15, #0x58]\n" + ".inst 0x4f82e332 // sdot v18.4s, v25.16b, v2.4b[0]\n" + "add x10, x10, #0x10\n" + ".inst 0x4f83e336 // sdot v22.4s, v25.16b, v3.4b[0]\n" + "ldr d25, [x15, #0x40]\n" + ".inst 0x4f80e30b // sdot v11.4s, v24.16b, v0.4b[0]\n" + "mov v25.d[1], x21\n" + ".inst 0x4f81e30f // sdot v15.4s, v24.16b, v1.4b[0]\n" + "ldr x21, [x15, #0x68]\n" + ".inst 0x4f82e313 // sdot v19.4s, v24.16b, v2.4b[0]\n" + "add x9, x9, #0x10\n" + ".inst 0x4f83e317 // sdot v23.4s, v24.16b, v3.4b[0]\n" + "ldr d24, [x15, #0x50]\n" + "mov v24.d[1], x20\n" + ".inst 0x4fa0e328 // sdot v8.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa1e32c // sdot v12.4s, v25.16b, v1.4b[1]\n" + "ldr x20, [x15, #0x78]\n" + ".inst 0x4fa2e330 // sdot v16.4s, v25.16b, v2.4b[1]\n" + "ldr x25, [x12, #0x8]\n" + ".inst 0x4fa3e334 // sdot v20.4s, v25.16b, v3.4b[1]\n" + "ldr d25, [x15, #0x60]\n" + ".inst 0x4fa0e309 // sdot v9.4s, v24.16b, v0.4b[1]\n" + "mov v25.d[1], x21\n" + ".inst 0x4fa1e30d // sdot v13.4s, v24.16b, v1.4b[1]\n" + "ldr x21, [x15, #0x88]\n" + ".inst 0x4fa2e311 // sdot v17.4s, v24.16b, v2.4b[1]\n" + "ldr x24, [x11, #0x8]\n" + ".inst 0x4fa3e315 // sdot v21.4s, v24.16b, v3.4b[1]\n" + "ldr d24, [x15, #0x70]\n" + "mov v24.d[1], x20\n" + ".inst 0x4fa0e32a // sdot v10.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa1e32e // sdot v14.4s, v25.16b, v1.4b[1]\n" + "ldr x20, [x15, #0x98]\n" + ".inst 0x4fa2e332 // sdot v18.4s, v25.16b, v2.4b[1]\n" + "ldr x23, [x10, #0x8]\n" + ".inst 0x4fa3e336 // sdot v22.4s, v25.16b, v3.4b[1]\n" + "ldr d25, [x15, #0x80]\n" + ".inst 0x4fa0e30b // sdot v11.4s, v24.16b, v0.4b[1]\n" + "mov v25.d[1], x21\n" + ".inst 0x4fa1e30f // sdot v15.4s, v24.16b, v1.4b[1]\n" + "ldr x21, [x15, #0xa8]\n" + ".inst 0x4fa2e313 // sdot v19.4s, v24.16b, v2.4b[1]\n" + "ldr x22, [x9, #0x8]\n" + ".inst 0x4fa3e317 // sdot v23.4s, v24.16b, v3.4b[1]\n" + "ldr d24, [x15, #0x90]\n" + "mov v24.d[1], x20\n" + ".inst 0x4f80eb28 // sdot v8.4s, v25.16b, v0.4b[2]\n" + ".inst 0x4f81eb2c // sdot v12.4s, v25.16b, v1.4b[2]\n" + "ldr x20, [x15, #0xb8]\n" + ".inst 0x4f82eb30 // sdot v16.4s, v25.16b, v2.4b[2]\n" "sub x13, x13, #0x10\n" - ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n" - "ldr d6, [x15, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - "mov v6.d[1], x20\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - "ldr x20, [x15, #0xc8]\n" - ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" + ".inst 0x4f83eb34 // sdot v20.4s, v25.16b, v3.4b[2]\n" + "ldr d25, [x15, #0xa0]\n" + ".inst 0x4f80eb09 // sdot v9.4s, v24.16b, v0.4b[2]\n" + "mov v25.d[1], x21\n" + ".inst 0x4f81eb0d // sdot v13.4s, v24.16b, v1.4b[2]\n" + "ldr x21, [x15, #0xc8]\n" + ".inst 0x4f82eb11 // sdot v17.4s, v24.16b, v2.4b[2]\n" "cmp x13, #0x20\n" - ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n" - "ldr d7, [x15, #0xb0]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - "ldr x11, [x15, #0xd8]\n" - ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" + ".inst 0x4f83eb15 // sdot v21.4s, v24.16b, v3.4b[2]\n" + "ldr d24, [x15, #0xb0]\n" + "mov v24.d[1], x20\n" + ".inst 0x4f80eb2a // sdot v10.4s, v25.16b, v0.4b[2]\n" + ".inst 0x4f81eb2e // sdot v14.4s, v25.16b, v1.4b[2]\n" + "ldr x20, [x15, #0xd8]\n" + ".inst 0x4f82eb32 // sdot v18.4s, v25.16b, v2.4b[2]\n" "prfm pldl1keep, [x12, #0x80]\n" - ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n" - "ldr d6, [x15, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - "mov v6.d[1], x20\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - "ldr x20, [x15, #0xe8]\n" - ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" + ".inst 0x4f83eb36 // sdot v22.4s, v25.16b, v3.4b[2]\n" + "ldr d25, [x15, #0xc0]\n" + ".inst 0x4f80eb0b // sdot v11.4s, v24.16b, v0.4b[2]\n" + "mov v25.d[1], x21\n" + ".inst 0x4f81eb0f // sdot v15.4s, v24.16b, v1.4b[2]\n" + "ldr x21, [x15, #0xe8]\n" + ".inst 0x4f82eb13 // sdot v19.4s, v24.16b, v2.4b[2]\n" + "prfm pldl1keep, [x11, #0x80]\n" + ".inst 0x4f83eb17 // sdot v23.4s, v24.16b, v3.4b[2]\n" + "ldr d24, [x15, #0xd0]\n" + "mov v24.d[1], x20\n" + ".inst 0x4fa0eb28 // sdot v8.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa1eb2c // sdot v12.4s, v25.16b, v1.4b[3]\n" + "ldr x20, [x15, #0xf8]\n" + ".inst 0x4fa2eb30 // sdot v16.4s, v25.16b, v2.4b[3]\n" + "prfm pldl1keep, [x10, #0x80]\n" + ".inst 0x4fa3eb34 // sdot v20.4s, v25.16b, v3.4b[3]\n" + "ldr d25, [x15, #0xe0]\n" + ".inst 0x4fa0eb09 // sdot v9.4s, v24.16b, v0.4b[3]\n" + "mov v25.d[1], x21\n" + ".inst 0x4fa1eb0d // sdot v13.4s, v24.16b, v1.4b[3]\n" "prfm pldl1keep, [x9, #0x80]\n" - ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n" - "ldr d7, [x15, #0xd0]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - "ldr x11, [x15, #0xf8]\n" - ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" - "prfm pldl1keep, [x27, #0x80]\n" - ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n" - "ldr d6, [x15, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - "mov v6.d[1], x20\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" - ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n" - "ldr d7, [x15, #0xf0]\n" - "mov v7.d[1], x11\n" + ".inst 0x4fa2eb11 // sdot v17.4s, v24.16b, v2.4b[3]\n" + ".inst 0x4fa3eb15 // sdot v21.4s, v24.16b, v3.4b[3]\n" + "ldr d24, [x15, #0xf0]\n" + "mov v24.d[1], x20\n" "add x15, x15, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - "ldr x20, [x15, #0x8]\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - "ldr x11, [x15, #0x18]\n" - ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n" + ".inst 0x4fa0eb2a // sdot v10.4s, v25.16b, v0.4b[3]\n" + "ldr x21, [x15, #0x8]\n" + ".inst 0x4fa1eb2e // sdot v14.4s, v25.16b, v1.4b[3]\n" + "ldr x20, [x15, #0x18]\n" + ".inst 0x4fa2eb32 // sdot v18.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa3eb36 // sdot v22.4s, v25.16b, v3.4b[3]\n" "ldr d6, [x15, #0x0]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x4fa0eb0b // sdot v11.4s, v24.16b, v0.4b[3]\n" "ldr d0, [x12, #0x0]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" - "ldr d1, [x9, #0x0]\n" - ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" - "ldr d2, [x27, #0x0]\n" - ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n" - "ldr d3, [x25, #0x0]\n" + ".inst 0x4fa1eb0f // sdot v15.4s, v24.16b, v1.4b[3]\n" + "ldr d1, [x11, #0x0]\n" + ".inst 0x4fa2eb13 // sdot v19.4s, v24.16b, v2.4b[3]\n" + "ldr d2, [x10, #0x0]\n" + ".inst 0x4fa3eb17 // sdot v23.4s, v24.16b, v3.4b[3]\n" + "ldr d3, [x9, #0x0]\n" "ldr d7, [x15, #0x10]\n" - "mov v6.d[1], x20\n" - "mov v0.d[1], x10\n" - "mov v1.d[1], x28\n" - "mov v2.d[1], x26\n" - "mov v3.d[1], x24\n" - "mov v7.d[1], x11\n" + "mov v6.d[1], x21\n" + "mov v0.d[1], x25\n" + "mov v1.d[1], x24\n" + "mov v2.d[1], x23\n" + "mov v3.d[1], x22\n" + "mov v7.d[1], x20\n" "bge 88b\n" "89:" // Height 4: Multiply loop: Single iteration only ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" "add x12, x12, #0x10\n" ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "add x9, x9, #0x10\n" + "add x11, x11, #0x10\n" ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - "add x27, x27, #0x10\n" + "add x10, x10, #0x10\n" ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x15, #0x20]\n" + "ldr q25, [x15, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "add x25, x25, #0x10\n" + "add x9, x9, #0x10\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" "sub x13, x13, #0x10\n" ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" "prfm pldl1keep, [x12, #0x80]\n" ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x15, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q24, [x15, #0x30]\n" + ".inst 0x4f80e32a // sdot v10.4s, v25.16b, v0.4b[0]\n" + "prfm pldl1keep, [x11, #0x80]\n" + ".inst 0x4f81e32e // sdot v14.4s, v25.16b, v1.4b[0]\n" + "prfm pldl1keep, [x10, #0x80]\n" + ".inst 0x4f82e332 // sdot v18.4s, v25.16b, v2.4b[0]\n" "prfm pldl1keep, [x9, #0x80]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "prfm pldl1keep, [x27, #0x80]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x15, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x15, #0x50]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" - ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n" - "ldr q6, [x15, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n" - "ldr q7, [x15, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n" - "ldr q6, [x15, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" - ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n" - "ldr q7, [x15, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n" - "ldr q6, [x15, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n" - "ldr q7, [x15, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n" - "ldr q6, [x15, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n" - "ldr q7, [x15, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n" - "ldr q6, [x15, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" - ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n" - "ldr q7, [x15, #0xf0]\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" + ".inst 0x4f83e336 // sdot v22.4s, v25.16b, v3.4b[0]\n" + "ldr q25, [x15, #0x40]\n" + ".inst 0x4f80e30b // sdot v11.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f81e30f // sdot v15.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f82e313 // sdot v19.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f83e317 // sdot v23.4s, v24.16b, v3.4b[0]\n" + "ldr q24, [x15, #0x50]\n" + ".inst 0x4fa0e328 // sdot v8.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa1e32c // sdot v12.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa2e330 // sdot v16.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa3e334 // sdot v20.4s, v25.16b, v3.4b[1]\n" + "ldr q25, [x15, #0x60]\n" + ".inst 0x4fa0e309 // sdot v9.4s, v24.16b, v0.4b[1]\n" + ".inst 0x4fa1e30d // sdot v13.4s, v24.16b, v1.4b[1]\n" + ".inst 0x4fa2e311 // sdot v17.4s, v24.16b, v2.4b[1]\n" + ".inst 0x4fa3e315 // sdot v21.4s, v24.16b, v3.4b[1]\n" + "ldr q24, [x15, #0x70]\n" + ".inst 0x4fa0e32a // sdot v10.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa1e32e // sdot v14.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa2e332 // sdot v18.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa3e336 // sdot v22.4s, v25.16b, v3.4b[1]\n" + "ldr q25, [x15, #0x80]\n" + ".inst 0x4fa0e30b // sdot v11.4s, v24.16b, v0.4b[1]\n" + ".inst 0x4fa1e30f // sdot v15.4s, v24.16b, v1.4b[1]\n" + ".inst 0x4fa2e313 // sdot v19.4s, v24.16b, v2.4b[1]\n" + ".inst 0x4fa3e317 // sdot v23.4s, v24.16b, v3.4b[1]\n" + "ldr q24, [x15, #0x90]\n" + ".inst 0x4f80eb28 // sdot v8.4s, v25.16b, v0.4b[2]\n" + ".inst 0x4f81eb2c // sdot v12.4s, v25.16b, v1.4b[2]\n" + ".inst 0x4f82eb30 // sdot v16.4s, v25.16b, v2.4b[2]\n" + ".inst 0x4f83eb34 // sdot v20.4s, v25.16b, v3.4b[2]\n" + "ldr q25, [x15, #0xa0]\n" + ".inst 0x4f80eb09 // sdot v9.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f81eb0d // sdot v13.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f82eb11 // sdot v17.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f83eb15 // sdot v21.4s, v24.16b, v3.4b[2]\n" + "ldr q24, [x15, #0xb0]\n" + ".inst 0x4f80eb2a // sdot v10.4s, v25.16b, v0.4b[2]\n" + ".inst 0x4f81eb2e // sdot v14.4s, v25.16b, v1.4b[2]\n" + ".inst 0x4f82eb32 // sdot v18.4s, v25.16b, v2.4b[2]\n" + ".inst 0x4f83eb36 // sdot v22.4s, v25.16b, v3.4b[2]\n" + "ldr q25, [x15, #0xc0]\n" + ".inst 0x4f80eb0b // sdot v11.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f81eb0f // sdot v15.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f82eb13 // sdot v19.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f83eb17 // sdot v23.4s, v24.16b, v3.4b[2]\n" + "ldr q24, [x15, #0xd0]\n" + ".inst 0x4fa0eb28 // sdot v8.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa1eb2c // sdot v12.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa2eb30 // sdot v16.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa3eb34 // sdot v20.4s, v25.16b, v3.4b[3]\n" + "ldr q25, [x15, #0xe0]\n" + ".inst 0x4fa0eb09 // sdot v9.4s, v24.16b, v0.4b[3]\n" + ".inst 0x4fa1eb0d // sdot v13.4s, v24.16b, v1.4b[3]\n" + ".inst 0x4fa2eb11 // sdot v17.4s, v24.16b, v2.4b[3]\n" + ".inst 0x4fa3eb15 // sdot v21.4s, v24.16b, v3.4b[3]\n" + "ldr q24, [x15, #0xf0]\n" + ".inst 0x4fa0eb2a // sdot v10.4s, v25.16b, v0.4b[3]\n" "add x15, x15, #0x100\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" - ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n" + ".inst 0x4fa1eb2e // sdot v14.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa2eb32 // sdot v18.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa3eb36 // sdot v22.4s, v25.16b, v3.4b[3]\n" + ".inst 0x4fa0eb0b // sdot v11.4s, v24.16b, v0.4b[3]\n" + ".inst 0x4fa1eb0f // sdot v15.4s, v24.16b, v1.4b[3]\n" + ".inst 0x4fa2eb13 // sdot v19.4s, v24.16b, v2.4b[3]\n" + ".inst 0x4fa3eb17 // sdot v23.4s, v24.16b, v3.4b[3]\n" "90:" // Height 4: Multiply loop: Main loop skip "cbz x13, 95f\n" "cmp x13, #0x4\n" "blt 92f\n" "91:" // Height 4: Multiply loop: Odd block loop - "ldr s0, [x12], #0x4\n" + "ldr s29, [x12], #0x4\n" "sub x13, x13, #0x4\n" - "ldr s1, [x9], #0x4\n" + "ldr s28, [x11], #0x4\n" "cmp x13, #0x4\n" - "ldr s2, [x27], #0x4\n" - "ldr s3, [x25], #0x4\n" - "ldr q6, [x15, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x15, #0x10]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x15, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x15, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr s27, [x10], #0x4\n" + "ldr s26, [x9], #0x4\n" + "ldr q25, [x15, #0x0]\n" + ".inst 0x4f9de328 // sdot v8.4s, v25.16b, v29.4b[0]\n" + "ldr q24, [x15, #0x10]\n" + ".inst 0x4f9ce32c // sdot v12.4s, v25.16b, v28.4b[0]\n" + ".inst 0x4f9be330 // sdot v16.4s, v25.16b, v27.4b[0]\n" + ".inst 0x4f9ae334 // sdot v20.4s, v25.16b, v26.4b[0]\n" + "ldr q25, [x15, #0x20]\n" + ".inst 0x4f9de309 // sdot v9.4s, v24.16b, v29.4b[0]\n" + ".inst 0x4f9ce30d // sdot v13.4s, v24.16b, v28.4b[0]\n" + ".inst 0x4f9be311 // sdot v17.4s, v24.16b, v27.4b[0]\n" + ".inst 0x4f9ae315 // sdot v21.4s, v24.16b, v26.4b[0]\n" + "ldr q24, [x15, #0x30]\n" + ".inst 0x4f9de32a // sdot v10.4s, v25.16b, v29.4b[0]\n" "add x15, x15, #0x40\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" + ".inst 0x4f9ce32e // sdot v14.4s, v25.16b, v28.4b[0]\n" + ".inst 0x4f9be332 // sdot v18.4s, v25.16b, v27.4b[0]\n" + ".inst 0x4f9ae336 // sdot v22.4s, v25.16b, v26.4b[0]\n" + ".inst 0x4f9de30b // sdot v11.4s, v24.16b, v29.4b[0]\n" + ".inst 0x4f9ce30f // sdot v15.4s, v24.16b, v28.4b[0]\n" + ".inst 0x4f9be313 // sdot v19.4s, v24.16b, v27.4b[0]\n" + ".inst 0x4f9ae317 // sdot v23.4s, v24.16b, v26.4b[0]\n" "bge 91b\n" "92:" // Height 4: Multiply loop: Skip odd blocks "cbz x13, 95f\n" "tbz x13, #1, 93f\n" "ldr h0, [x12], #0x2\n" - "ldr h1, [x9], #0x2\n" - "ldr h2, [x27], #0x2\n" - "ldr h3, [x25], #0x2\n" + "ldr h1, [x11], #0x2\n" + "ldr h2, [x10], #0x2\n" + "ldr h3, [x9], #0x2\n" "tbz x13, #0, 94f\n" "ld1 { v0.b }[2], [x12]\n" - "ld1 { v1.b }[2], [x9]\n" - "ld1 { v2.b }[2], [x27]\n" - "ld1 { v3.b }[2], [x25]\n" + "ld1 { v1.b }[2], [x11]\n" + "ld1 { v2.b }[2], [x10]\n" + "ld1 { v3.b }[2], [x9]\n" "b 94f\n" "93:" // Height 4: Multiply loop: Ragged operand read: partial_1_0 "ldr b0, [x12, #0x0]\n" - "ldr b1, [x9, #0x0]\n" - "ldr b2, [x27, #0x0]\n" - "ldr b3, [x25, #0x0]\n" + "ldr b1, [x11, #0x0]\n" + "ldr b2, [x10, #0x0]\n" + "ldr b3, [x9, #0x0]\n" "94:" // Height 4: Multiply loop: Ragged operand read: Done - "ldr q6, [x15, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x15, #0x10]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x15, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x15, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q25, [x15, #0x0]\n" + ".inst 0x4f80e328 // sdot v8.4s, v25.16b, v0.4b[0]\n" + "ldr q24, [x15, #0x10]\n" + ".inst 0x4f81e32c // sdot v12.4s, v25.16b, v1.4b[0]\n" + ".inst 0x4f82e330 // sdot v16.4s, v25.16b, v2.4b[0]\n" + ".inst 0x4f83e334 // sdot v20.4s, v25.16b, v3.4b[0]\n" + "ldr q25, [x15, #0x20]\n" + ".inst 0x4f80e309 // sdot v9.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f81e30d // sdot v13.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f82e311 // sdot v17.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f83e315 // sdot v21.4s, v24.16b, v3.4b[0]\n" + "ldr q24, [x15, #0x30]\n" + ".inst 0x4f80e32a // sdot v10.4s, v25.16b, v0.4b[0]\n" "add x15, x15, #0x40\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" + ".inst 0x4f81e32e // sdot v14.4s, v25.16b, v1.4b[0]\n" + ".inst 0x4f82e332 // sdot v18.4s, v25.16b, v2.4b[0]\n" + ".inst 0x4f83e336 // sdot v22.4s, v25.16b, v3.4b[0]\n" + ".inst 0x4f80e30b // sdot v11.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f81e30f // sdot v15.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f82e313 // sdot v19.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f83e317 // sdot v23.4s, v24.16b, v3.4b[0]\n" "95:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x14, x14, #0x1\n" "cmp x14, x20\n" "bne 85b\n" - "ldr q0, [x6, #0x0]\n" - "add v8.4s, v8.4s, v0.4s\n" - "ldr q1, [x6, #0x10]\n" - "add v9.4s, v9.4s, v1.4s\n" - "ldr q2, [x6, #0x20]\n" - "add v10.4s, v10.4s, v2.4s\n" - "ldr q3, [x6, #0x30]\n" - "add v11.4s, v11.4s, v3.4s\n" + "ldr q27, [x6, #0x0]\n" + "add v8.4s, v8.4s, v27.4s\n" + "ldr q26, [x6, #0x10]\n" + "add v9.4s, v9.4s, v26.4s\n" + "ldr q25, [x6, #0x20]\n" + "add v10.4s, v10.4s, v25.4s\n" + "ldr q24, [x6, #0x30]\n" + "add v11.4s, v11.4s, v24.4s\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x17, x20\n" + "add x25, x17, x20\n" + "add x24, x25, x20\n" "add x23, x24, x20\n" - "add x22, x23, x20\n" "prfm pstl1keep, [x17, #0x0]\n" - "add v12.4s, v12.4s, v0.4s\n" + "add v12.4s, v12.4s, v27.4s\n" + "prfm pstl1keep, [x25, #0x0]\n" + "add v13.4s, v13.4s, v26.4s\n" "prfm pstl1keep, [x24, #0x0]\n" - "add v13.4s, v13.4s, v1.4s\n" + "add v14.4s, v14.4s, v25.4s\n" "prfm pstl1keep, [x23, #0x0]\n" - "add v14.4s, v14.4s, v2.4s\n" - "prfm pstl1keep, [x22, #0x0]\n" - "add v15.4s, v15.4s, v3.4s\n" - "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" - "add v20.4s, v20.4s, v0.4s\n" - "add v21.4s, v21.4s, v1.4s\n" - "add v22.4s, v22.4s, v2.4s\n" - "add v23.4s, v23.4s, v3.4s\n" + "add v15.4s, v15.4s, v24.4s\n" + "add v16.4s, v16.4s, v27.4s\n" + "add v17.4s, v17.4s, v26.4s\n" + "add v18.4s, v18.4s, v25.4s\n" + "add v19.4s, v19.4s, v24.4s\n" + "add v20.4s, v20.4s, v27.4s\n" + "add v21.4s, v21.4s, v26.4s\n" + "add v22.4s, v22.4s, v25.4s\n" + "add v23.4s, v23.4s, v24.4s\n" "add x6, x6, #0x40\n" "tbz %x[flags], #4, 96f\n" "ldr q0, [x8, #0x0]\n" @@ -1804,10 +1803,10 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "add x7, x7, #0x40\n" "b 97f\n" "96:" // Height 4: per layer parameters - "add x25, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x25]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v4.4s }, [x20]\n" "mov v1.16b, v0.16b\n" "mov v5.16b, v4.16b\n" "mov v2.16b, v0.16b\n" @@ -1832,54 +1831,54 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "sqrdmulh v22.4s, v22.4s, v6.4s\n" "sqrdmulh v23.4s, v23.4s, v7.4s\n" "tbz %x[flags], #5, 98f\n" - "and v4.16b, v8.16b, v0.16b\n" - "and v5.16b, v9.16b, v1.16b\n" - "and v6.16b, v10.16b, v2.16b\n" - "and v7.16b, v11.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v4.4s\n" - "sqadd v9.4s, v9.4s, v5.4s\n" - "sqadd v10.4s, v10.4s, v6.4s\n" - "sqadd v11.4s, v11.4s, v7.4s\n" - "and v4.16b, v12.16b, v0.16b\n" - "and v5.16b, v13.16b, v1.16b\n" - "and v6.16b, v14.16b, v2.16b\n" - "and v7.16b, v15.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v12.4s, v12.4s, v4.4s\n" - "sqadd v13.4s, v13.4s, v5.4s\n" - "sqadd v14.4s, v14.4s, v6.4s\n" - "sqadd v15.4s, v15.4s, v7.4s\n" - "and v4.16b, v16.16b, v0.16b\n" - "and v5.16b, v17.16b, v1.16b\n" - "and v6.16b, v18.16b, v2.16b\n" - "and v7.16b, v19.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" - "and v4.16b, v20.16b, v0.16b\n" - "and v5.16b, v21.16b, v1.16b\n" - "and v6.16b, v22.16b, v2.16b\n" - "and v7.16b, v23.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v20.4s, v20.4s, v4.4s\n" - "sqadd v21.4s, v21.4s, v5.4s\n" - "sqadd v22.4s, v22.4s, v6.4s\n" - "sqadd v23.4s, v23.4s, v7.4s\n" + "and v27.16b, v8.16b, v0.16b\n" + "and v26.16b, v9.16b, v1.16b\n" + "and v25.16b, v10.16b, v2.16b\n" + "and v24.16b, v11.16b, v3.16b\n" + "sshr v27.4s, v27.4s, #0x1f\n" + "sshr v26.4s, v26.4s, #0x1f\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sqadd v8.4s, v8.4s, v27.4s\n" + "sqadd v9.4s, v9.4s, v26.4s\n" + "sqadd v10.4s, v10.4s, v25.4s\n" + "sqadd v11.4s, v11.4s, v24.4s\n" + "and v27.16b, v12.16b, v0.16b\n" + "and v26.16b, v13.16b, v1.16b\n" + "and v25.16b, v14.16b, v2.16b\n" + "and v24.16b, v15.16b, v3.16b\n" + "sshr v27.4s, v27.4s, #0x1f\n" + "sshr v26.4s, v26.4s, #0x1f\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sqadd v12.4s, v12.4s, v27.4s\n" + "sqadd v13.4s, v13.4s, v26.4s\n" + "sqadd v14.4s, v14.4s, v25.4s\n" + "sqadd v15.4s, v15.4s, v24.4s\n" + "and v27.16b, v16.16b, v0.16b\n" + "and v26.16b, v17.16b, v1.16b\n" + "and v25.16b, v18.16b, v2.16b\n" + "and v24.16b, v19.16b, v3.16b\n" + "sshr v27.4s, v27.4s, #0x1f\n" + "sshr v26.4s, v26.4s, #0x1f\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v27.4s\n" + "sqadd v17.4s, v17.4s, v26.4s\n" + "sqadd v18.4s, v18.4s, v25.4s\n" + "sqadd v19.4s, v19.4s, v24.4s\n" + "and v27.16b, v20.16b, v0.16b\n" + "and v26.16b, v21.16b, v1.16b\n" + "and v25.16b, v22.16b, v2.16b\n" + "and v24.16b, v23.16b, v3.16b\n" + "sshr v27.4s, v27.4s, #0x1f\n" + "sshr v26.4s, v26.4s, #0x1f\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v27.4s\n" + "sqadd v21.4s, v21.4s, v26.4s\n" + "sqadd v22.4s, v22.4s, v25.4s\n" + "sqadd v23.4s, v23.4s, v24.4s\n" "98:" // Height 4: no shift correction "srshl v8.4s, v8.4s, v0.4s\n" "srshl v9.4s, v9.4s, v1.4s\n" @@ -1897,170 +1896,170 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "srshl v21.4s, v21.4s, v1.4s\n" "srshl v22.4s, v22.4s, v2.4s\n" "srshl v23.4s, v23.4s, v3.4s\n" - "add x25, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x25]\n" - "add v8.4s, v8.4s, v4.4s\n" - "add v9.4s, v9.4s, v4.4s\n" - "add v10.4s, v10.4s, v4.4s\n" - "add v11.4s, v11.4s, v4.4s\n" - "add v12.4s, v12.4s, v4.4s\n" - "add v13.4s, v13.4s, v4.4s\n" - "add v14.4s, v14.4s, v4.4s\n" - "add v15.4s, v15.4s, v4.4s\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v23.4s, v23.4s, v4.4s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x25]\n" - "smin v8.4s, v8.4s, v6.4s\n" - "smin v9.4s, v9.4s, v6.4s\n" - "smin v10.4s, v10.4s, v6.4s\n" - "smin v11.4s, v11.4s, v6.4s\n" - "smin v12.4s, v12.4s, v6.4s\n" - "smin v13.4s, v13.4s, v6.4s\n" - "smin v14.4s, v14.4s, v6.4s\n" - "smin v15.4s, v15.4s, v6.4s\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "add x25, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x25]\n" - "smax v8.4s, v8.4s, v5.4s\n" - "smax v9.4s, v9.4s, v5.4s\n" - "smax v10.4s, v10.4s, v5.4s\n" - "smax v11.4s, v11.4s, v5.4s\n" - "smax v12.4s, v12.4s, v5.4s\n" - "smax v13.4s, v13.4s, v5.4s\n" - "smax v14.4s, v14.4s, v5.4s\n" - "smax v15.4s, v15.4s, v5.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v24.4s }, [x20]\n" + "add v8.4s, v8.4s, v24.4s\n" + "add v9.4s, v9.4s, v24.4s\n" + "add v10.4s, v10.4s, v24.4s\n" + "add v11.4s, v11.4s, v24.4s\n" + "add v12.4s, v12.4s, v24.4s\n" + "add v13.4s, v13.4s, v24.4s\n" + "add v14.4s, v14.4s, v24.4s\n" + "add v15.4s, v15.4s, v24.4s\n" + "add v16.4s, v16.4s, v24.4s\n" + "add v17.4s, v17.4s, v24.4s\n" + "add v18.4s, v18.4s, v24.4s\n" + "add v19.4s, v19.4s, v24.4s\n" + "add v20.4s, v20.4s, v24.4s\n" + "add v21.4s, v21.4s, v24.4s\n" + "add v22.4s, v22.4s, v24.4s\n" + "add v23.4s, v23.4s, v24.4s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v24.4s }, [x20]\n" + "smin v8.4s, v8.4s, v24.4s\n" + "smin v9.4s, v9.4s, v24.4s\n" + "smin v10.4s, v10.4s, v24.4s\n" + "smin v11.4s, v11.4s, v24.4s\n" + "smin v12.4s, v12.4s, v24.4s\n" + "smin v13.4s, v13.4s, v24.4s\n" + "smin v14.4s, v14.4s, v24.4s\n" + "smin v15.4s, v15.4s, v24.4s\n" + "smin v16.4s, v16.4s, v24.4s\n" + "smin v17.4s, v17.4s, v24.4s\n" + "smin v18.4s, v18.4s, v24.4s\n" + "smin v19.4s, v19.4s, v24.4s\n" + "smin v20.4s, v20.4s, v24.4s\n" + "smin v21.4s, v21.4s, v24.4s\n" + "smin v22.4s, v22.4s, v24.4s\n" + "smin v23.4s, v23.4s, v24.4s\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v24.4s }, [x20]\n" + "smax v8.4s, v8.4s, v24.4s\n" + "smax v9.4s, v9.4s, v24.4s\n" + "smax v10.4s, v10.4s, v24.4s\n" + "smax v11.4s, v11.4s, v24.4s\n" + "smax v12.4s, v12.4s, v24.4s\n" + "smax v13.4s, v13.4s, v24.4s\n" + "smax v14.4s, v14.4s, v24.4s\n" + "smax v15.4s, v15.4s, v24.4s\n" + "smax v16.4s, v16.4s, v24.4s\n" + "smax v17.4s, v17.4s, v24.4s\n" + "smax v18.4s, v18.4s, v24.4s\n" + "smax v19.4s, v19.4s, v24.4s\n" + "smax v20.4s, v20.4s, v24.4s\n" + "smax v21.4s, v21.4s, v24.4s\n" + "smax v22.4s, v22.4s, v24.4s\n" + "smax v23.4s, v23.4s, v24.4s\n" "uzp1 v8.8h, v8.8h, v9.8h\n" - "uzp1 v9.8h, v10.8h, v11.8h\n" + "uzp1 v25.8h, v10.8h, v11.8h\n" "uzp1 v12.8h, v12.8h, v13.8h\n" - "uzp1 v13.8h, v14.8h, v15.8h\n" + "uzp1 v24.8h, v14.8h, v15.8h\n" "uzp1 v16.8h, v16.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v18.8h, v18.8h, v19.8h\n" "uzp1 v20.8h, v20.8h, v21.8h\n" - "uzp1 v21.8h, v22.8h, v23.8h\n" + "uzp1 v17.8h, v22.8h, v23.8h\n" "cmp x16, #0x10\n" - "uzp1 v8.16b, v8.16b, v9.16b\n" - "uzp1 v12.16b, v12.16b, v13.16b\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" - "uzp1 v20.16b, v20.16b, v21.16b\n" + "uzp1 v8.16b, v8.16b, v25.16b\n" + "uzp1 v12.16b, v12.16b, v24.16b\n" + "uzp1 v16.16b, v16.16b, v18.16b\n" + "uzp1 v20.16b, v20.16b, v17.16b\n" "bge 107f\n" "tbz x16, #3, 102f\n" "str d8, [x17], #0x8\n" - "str d12, [x24], #0x8\n" - "str d16, [x23], #0x8\n" - "str d20, [x22], #0x8\n" + "str d12, [x25], #0x8\n" + "str d16, [x24], #0x8\n" + "str d20, [x23], #0x8\n" "tbz x16, #2, 100f\n" "st1 { v8.s }[2], [x17], #0x4\n" - "st1 { v12.s }[2], [x24], #0x4\n" - "st1 { v16.s }[2], [x23], #0x4\n" - "st1 { v20.s }[2], [x22], #0x4\n" + "st1 { v12.s }[2], [x25], #0x4\n" + "st1 { v16.s }[2], [x24], #0x4\n" + "st1 { v20.s }[2], [x23], #0x4\n" "tbz x16, #1, 99f\n" "st1 { v8.h }[6], [x17], #0x2\n" - "st1 { v12.h }[6], [x24], #0x2\n" - "st1 { v16.h }[6], [x23], #0x2\n" - "st1 { v20.h }[6], [x22], #0x2\n" + "st1 { v12.h }[6], [x25], #0x2\n" + "st1 { v16.h }[6], [x24], #0x2\n" + "st1 { v20.h }[6], [x23], #0x2\n" "tbz x16, #0, 106f\n" "st1 { v8.b }[14], [x17]\n" - "st1 { v12.b }[14], [x24]\n" - "st1 { v16.b }[14], [x23]\n" - "st1 { v20.b }[14], [x22]\n" + "st1 { v12.b }[14], [x25]\n" + "st1 { v16.b }[14], [x24]\n" + "st1 { v20.b }[14], [x23]\n" "b 106f\n" "99:" // Height 4: Partial direct writeback: partial_1_12 "tbz x16, #0, 106f\n" "st1 { v8.b }[12], [x17]\n" - "st1 { v12.b }[12], [x24]\n" - "st1 { v16.b }[12], [x23]\n" - "st1 { v20.b }[12], [x22]\n" + "st1 { v12.b }[12], [x25]\n" + "st1 { v16.b }[12], [x24]\n" + "st1 { v20.b }[12], [x23]\n" "b 106f\n" "100:" // Height 4: Partial direct writeback: partial_2_8 "tbz x16, #1, 101f\n" "st1 { v8.h }[4], [x17], #0x2\n" - "st1 { v12.h }[4], [x24], #0x2\n" - "st1 { v16.h }[4], [x23], #0x2\n" - "st1 { v20.h }[4], [x22], #0x2\n" + "st1 { v12.h }[4], [x25], #0x2\n" + "st1 { v16.h }[4], [x24], #0x2\n" + "st1 { v20.h }[4], [x23], #0x2\n" "tbz x16, #0, 106f\n" "st1 { v8.b }[10], [x17]\n" - "st1 { v12.b }[10], [x24]\n" - "st1 { v16.b }[10], [x23]\n" - "st1 { v20.b }[10], [x22]\n" + "st1 { v12.b }[10], [x25]\n" + "st1 { v16.b }[10], [x24]\n" + "st1 { v20.b }[10], [x23]\n" "b 106f\n" "101:" // Height 4: Partial direct writeback: partial_1_8 "tbz x16, #0, 106f\n" "st1 { v8.b }[8], [x17]\n" - "st1 { v12.b }[8], [x24]\n" - "st1 { v16.b }[8], [x23]\n" - "st1 { v20.b }[8], [x22]\n" + "st1 { v12.b }[8], [x25]\n" + "st1 { v16.b }[8], [x24]\n" + "st1 { v20.b }[8], [x23]\n" "b 106f\n" "102:" // Height 4: Partial direct writeback: partial_4_0 "tbz x16, #2, 104f\n" "str s8, [x17], #0x4\n" - "str s12, [x24], #0x4\n" - "str s16, [x23], #0x4\n" - "str s20, [x22], #0x4\n" + "str s12, [x25], #0x4\n" + "str s16, [x24], #0x4\n" + "str s20, [x23], #0x4\n" "tbz x16, #1, 103f\n" "st1 { v8.h }[2], [x17], #0x2\n" - "st1 { v12.h }[2], [x24], #0x2\n" - "st1 { v16.h }[2], [x23], #0x2\n" - "st1 { v20.h }[2], [x22], #0x2\n" + "st1 { v12.h }[2], [x25], #0x2\n" + "st1 { v16.h }[2], [x24], #0x2\n" + "st1 { v20.h }[2], [x23], #0x2\n" "tbz x16, #0, 106f\n" "st1 { v8.b }[6], [x17]\n" - "st1 { v12.b }[6], [x24]\n" - "st1 { v16.b }[6], [x23]\n" - "st1 { v20.b }[6], [x22]\n" + "st1 { v12.b }[6], [x25]\n" + "st1 { v16.b }[6], [x24]\n" + "st1 { v20.b }[6], [x23]\n" "b 106f\n" "103:" // Height 4: Partial direct writeback: partial_1_4 "tbz x16, #0, 106f\n" "st1 { v8.b }[4], [x17]\n" - "st1 { v12.b }[4], [x24]\n" - "st1 { v16.b }[4], [x23]\n" - "st1 { v20.b }[4], [x22]\n" + "st1 { v12.b }[4], [x25]\n" + "st1 { v16.b }[4], [x24]\n" + "st1 { v20.b }[4], [x23]\n" "b 106f\n" "104:" // Height 4: Partial direct writeback: partial_2_0 "tbz x16, #1, 105f\n" "str h8, [x17], #0x2\n" - "str h12, [x24], #0x2\n" - "str h16, [x23], #0x2\n" - "str h20, [x22], #0x2\n" + "str h12, [x25], #0x2\n" + "str h16, [x24], #0x2\n" + "str h20, [x23], #0x2\n" "tbz x16, #0, 106f\n" "st1 { v8.b }[2], [x17]\n" - "st1 { v12.b }[2], [x24]\n" - "st1 { v16.b }[2], [x23]\n" - "st1 { v20.b }[2], [x22]\n" + "st1 { v12.b }[2], [x25]\n" + "st1 { v16.b }[2], [x24]\n" + "st1 { v20.b }[2], [x23]\n" "b 106f\n" "105:" // Height 4: Partial direct writeback: partial_1_0 "str b8, [x17, #0x0]\n" - "str b12, [x24, #0x0]\n" - "str b16, [x23, #0x0]\n" - "str b20, [x22, #0x0]\n" + "str b12, [x25, #0x0]\n" + "str b16, [x24, #0x0]\n" + "str b20, [x23, #0x0]\n" "106:" // Height 4: Partial direct writeback: Done "b 108f\n" "107:" // Height 4: Full writeback "str q8, [x17, #0x0]\n" "add x17, x17, #0x10\n" - "str q12, [x24, #0x0]\n" - "str q16, [x23, #0x0]\n" - "str q20, [x22, #0x0]\n" + "str q12, [x25, #0x0]\n" + "str q16, [x24, #0x0]\n" + "str q20, [x23, #0x0]\n" "108:" // Height 4: Writeback done "subs x16, x16, #0x10\n" "bgt 83b\n" @@ -2089,439 +2088,439 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "movi v21.4s, #0x0\n" "movi v22.4s, #0x0\n" "movi v23.4s, #0x0\n" - "movi v24.4s, #0x0\n" - "movi v25.4s, #0x0\n" - "movi v26.4s, #0x0\n" - "movi v27.4s, #0x0\n" - "111:" // Height 5: setup done - "mov x14, #0x0\n" - "112:" // Height 5: String loop - "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr w13, [x20, x14, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" - "tbz %x[flags], #3, 113f\n" - "ldr x21, [%x[input_ptr], x14, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x12, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" - "ldr x27, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" - "ldr x23, [x21, #0x20]\n" - "cbnz x14, 114f\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x12, x12, x20\n" - "add x9, x9, x20\n" - "add x27, x27, x20\n" - "add x25, x25, x20\n" - "add x23, x23, x20\n" - "b 114f\n" - "113:" // Height 5: setup direct input - "mov x12, %x[input_ptr]\n" - "add x9, x12, x20\n" - "add x27, x9, x20\n" - "add x25, x27, x20\n" - "add x23, x25, x20\n" - "114:" // Height 5: input setup done - "cmp x13, #0x10\n" - "blt 117f\n" - "ldr q0, [x12, #0x0]\n" - "cmp x13, #0x20\n" - "ldr q1, [x9, #0x0]\n" - "ldr q2, [x27, #0x0]\n" - "ldr q3, [x25, #0x0]\n" - "ldr q4, [x23, #0x0]\n" - "ldr q6, [x15, #0x0]\n" - "ldr q7, [x15, #0x10]\n" - "blt 116f\n" - "115:" // Height 5: Multiply loop: Main loop head - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr x20, [x15, #0x28]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr x11, [x15, #0x38]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - "add x12, x12, #0x10\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "add x9, x9, #0x10\n" - ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - "ldr d6, [x15, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x20\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "ldr x20, [x15, #0x48]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - "add x27, x27, #0x10\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - "add x25, x25, #0x10\n" - ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - "ldr d7, [x15, #0x30]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "ldr x11, [x15, #0x58]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - "add x23, x23, #0x10\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - "ldr x10, [x12, #0x8]\n" - ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - "ldr d6, [x15, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x20\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - "ldr x20, [x15, #0x68]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - "ldr x28, [x9, #0x8]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - "ldr x26, [x27, #0x8]\n" - ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" - "ldr d7, [x15, #0x50]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - "ldr x11, [x15, #0x78]\n" - ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" - "ldr x24, [x25, #0x8]\n" - ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n" - "ldr x22, [x23, #0x8]\n" - ".inst 0x4fa4e0d8 // sdot v24.4s, v6.16b, v4.4b[1]\n" - "ldr d6, [x15, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - "mov v6.d[1], x20\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - "ldr x20, [x15, #0x88]\n" - ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - "sub x13, x13, #0x10\n" - ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n" - "cmp x13, #0x20\n" - ".inst 0x4fa4e0f9 // sdot v25.4s, v7.16b, v4.4b[1]\n" - "ldr d7, [x15, #0x70]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - "ldr x11, [x15, #0x98]\n" - ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - "prfm pldl1keep, [x12, #0x80]\n" - ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n" - "prfm pldl1keep, [x9, #0x80]\n" - ".inst 0x4fa4e0da // sdot v26.4s, v6.16b, v4.4b[1]\n" - "ldr d6, [x15, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - "mov v6.d[1], x20\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - "ldr x20, [x15, #0xa8]\n" - ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" - "prfm pldl1keep, [x27, #0x80]\n" - ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n" - "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4fa4e0fb // sdot v27.4s, v7.16b, v4.4b[1]\n" - "ldr d7, [x15, #0x90]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - "ldr x11, [x15, #0xb8]\n" - ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" - "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n" - ".inst 0x4f84e8d8 // sdot v24.4s, v6.16b, v4.4b[2]\n" - "ldr d6, [x15, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - "mov v6.d[1], x20\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - "ldr x20, [x15, #0xc8]\n" - ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n" - ".inst 0x4f84e8f9 // sdot v25.4s, v7.16b, v4.4b[2]\n" - "ldr d7, [x15, #0xb0]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - "ldr x11, [x15, #0xd8]\n" - ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n" - ".inst 0x4f84e8da // sdot v26.4s, v6.16b, v4.4b[2]\n" - "ldr d6, [x15, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - "mov v6.d[1], x20\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - "ldr x20, [x15, #0xe8]\n" - ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n" - ".inst 0x4f84e8fb // sdot v27.4s, v7.16b, v4.4b[2]\n" - "ldr d7, [x15, #0xd0]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - "ldr x11, [x15, #0xf8]\n" - ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n" - ".inst 0x4fa4e8d8 // sdot v24.4s, v6.16b, v4.4b[3]\n" - "ldr d6, [x15, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - "mov v6.d[1], x20\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" - ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n" - ".inst 0x4fa4e8f9 // sdot v25.4s, v7.16b, v4.4b[3]\n" - "ldr d7, [x15, #0xf0]\n" - "mov v7.d[1], x11\n" - "add x15, x15, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - "ldr x20, [x15, #0x8]\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - "ldr x11, [x15, #0x18]\n" - ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n" - ".inst 0x4fa4e8da // sdot v26.4s, v6.16b, v4.4b[3]\n" - "ldr d6, [x15, #0x0]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" - "ldr d0, [x12, #0x0]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" - "ldr d1, [x9, #0x0]\n" - ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" - "ldr d2, [x27, #0x0]\n" - ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n" - "ldr d3, [x25, #0x0]\n" - ".inst 0x4fa4e8fb // sdot v27.4s, v7.16b, v4.4b[3]\n" - "ldr d4, [x23, #0x0]\n" - "ldr d7, [x15, #0x10]\n" - "mov v6.d[1], x20\n" - "mov v0.d[1], x10\n" - "mov v1.d[1], x28\n" - "mov v2.d[1], x26\n" - "mov v3.d[1], x24\n" - "mov v4.d[1], x22\n" - "mov v7.d[1], x11\n" - "bge 115b\n" - "116:" // Height 5: Multiply loop: Single iteration only - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "add x12, x12, #0x10\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "add x9, x9, #0x10\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - "add x27, x27, #0x10\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "add x25, x25, #0x10\n" - ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x15, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "add x23, x23, #0x10\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "sub x13, x13, #0x10\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - "prfm pldl1keep, [x12, #0x80]\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - "prfm pldl1keep, [x9, #0x80]\n" - ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x15, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - "prfm pldl1keep, [x27, #0x80]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x15, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x15, #0x50]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" - ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n" - ".inst 0x4fa4e0d8 // sdot v24.4s, v6.16b, v4.4b[1]\n" - "ldr q6, [x15, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n" - ".inst 0x4fa4e0f9 // sdot v25.4s, v7.16b, v4.4b[1]\n" - "ldr q7, [x15, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n" - ".inst 0x4fa4e0da // sdot v26.4s, v6.16b, v4.4b[1]\n" - "ldr q6, [x15, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" - ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n" - ".inst 0x4fa4e0fb // sdot v27.4s, v7.16b, v4.4b[1]\n" - "ldr q7, [x15, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n" - ".inst 0x4f84e8d8 // sdot v24.4s, v6.16b, v4.4b[2]\n" - "ldr q6, [x15, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n" - ".inst 0x4f84e8f9 // sdot v25.4s, v7.16b, v4.4b[2]\n" - "ldr q7, [x15, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n" - ".inst 0x4f84e8da // sdot v26.4s, v6.16b, v4.4b[2]\n" - "ldr q6, [x15, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n" - ".inst 0x4f84e8fb // sdot v27.4s, v7.16b, v4.4b[2]\n" - "ldr q7, [x15, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n" - ".inst 0x4fa4e8d8 // sdot v24.4s, v6.16b, v4.4b[3]\n" - "ldr q6, [x15, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" - ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n" - ".inst 0x4fa4e8f9 // sdot v25.4s, v7.16b, v4.4b[3]\n" - "ldr q7, [x15, #0xf0]\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - "add x15, x15, #0x100\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n" - ".inst 0x4fa4e8da // sdot v26.4s, v6.16b, v4.4b[3]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" - ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n" - ".inst 0x4fa4e8fb // sdot v27.4s, v7.16b, v4.4b[3]\n" - "117:" // Height 5: Multiply loop: Main loop skip - "cbz x13, 122f\n" - "cmp x13, #0x4\n" - "blt 119f\n" - "118:" // Height 5: Multiply loop: Odd block loop - "ldr s0, [x12], #0x4\n" - "sub x13, x13, #0x4\n" - "ldr s1, [x9], #0x4\n" - "cmp x13, #0x4\n" - "ldr s2, [x27], #0x4\n" - "ldr s3, [x25], #0x4\n" - "ldr s4, [x23], #0x4\n" + "movi v24.4s, #0x0\n" + "movi v25.4s, #0x0\n" + "movi v26.4s, #0x0\n" + "movi v27.4s, #0x0\n" + "111:" // Height 5: setup done + "mov x14, #0x0\n" + "112:" // Height 5: String loop + "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" + "ldr w13, [x20, x14, LSL #0x2]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" + "tbz %x[flags], #3, 113f\n" + "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x12, [x20, #0x0]\n" + "ldr x11, [x20, #0x8]\n" + "ldr x10, [x20, #0x10]\n" + "ldr x9, [x20, #0x18]\n" + "ldr x28, [x20, #0x20]\n" + "cbnz x14, 114f\n" + "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" + "add x12, x12, x20\n" + "add x11, x11, x20\n" + "add x10, x10, x20\n" + "add x9, x9, x20\n" + "add x28, x28, x20\n" + "b 114f\n" + "113:" // Height 5: setup direct input + "mov x12, %x[input_ptr]\n" + "add x11, x12, x21\n" + "add x10, x11, x21\n" + "add x9, x10, x21\n" + "add x28, x9, x21\n" + "114:" // Height 5: input setup done + "cmp x13, #0x10\n" + "blt 117f\n" + "ldr q0, [x12, #0x0]\n" + "cmp x13, #0x20\n" + "ldr q1, [x11, #0x0]\n" + "ldr q2, [x10, #0x0]\n" + "ldr q3, [x9, #0x0]\n" + "ldr q4, [x28, #0x0]\n" "ldr q6, [x15, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" "ldr q7, [x15, #0x10]\n" + "blt 116f\n" + "115:" // Height 5: Multiply loop: Main loop head + ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" + "ldr x21, [x15, #0x28]\n" ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" + "ldr x20, [x15, #0x38]\n" ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" + "add x12, x12, #0x10\n" ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" + "add x11, x11, #0x10\n" ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x15, #0x20]\n" + "ldr d29, [x15, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" + "mov v29.d[1], x21\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" + "ldr x21, [x15, #0x48]\n" ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" + "add x10, x10, #0x10\n" ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" + "add x9, x9, #0x10\n" ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x15, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr d28, [x15, #0x30]\n" + "mov v28.d[1], x20\n" + ".inst 0x4f80e3aa // sdot v10.4s, v29.16b, v0.4b[0]\n" + ".inst 0x4f81e3ae // sdot v14.4s, v29.16b, v1.4b[0]\n" + "ldr x20, [x15, #0x58]\n" + ".inst 0x4f82e3b2 // sdot v18.4s, v29.16b, v2.4b[0]\n" + "add x28, x28, #0x10\n" + ".inst 0x4f83e3b6 // sdot v22.4s, v29.16b, v3.4b[0]\n" + "ldr x26, [x12, #0x8]\n" + ".inst 0x4f84e3ba // sdot v26.4s, v29.16b, v4.4b[0]\n" + "ldr d29, [x15, #0x40]\n" + ".inst 0x4f80e38b // sdot v11.4s, v28.16b, v0.4b[0]\n" + "mov v29.d[1], x21\n" + ".inst 0x4f81e38f // sdot v15.4s, v28.16b, v1.4b[0]\n" + "ldr x21, [x15, #0x68]\n" + ".inst 0x4f82e393 // sdot v19.4s, v28.16b, v2.4b[0]\n" + "ldr x25, [x11, #0x8]\n" + ".inst 0x4f83e397 // sdot v23.4s, v28.16b, v3.4b[0]\n" + "ldr x24, [x10, #0x8]\n" + ".inst 0x4f84e39b // sdot v27.4s, v28.16b, v4.4b[0]\n" + "ldr d28, [x15, #0x50]\n" + "mov v28.d[1], x20\n" + ".inst 0x4fa0e3a8 // sdot v8.4s, v29.16b, v0.4b[1]\n" + ".inst 0x4fa1e3ac // sdot v12.4s, v29.16b, v1.4b[1]\n" + "ldr x20, [x15, #0x78]\n" + ".inst 0x4fa2e3b0 // sdot v16.4s, v29.16b, v2.4b[1]\n" + "ldr x23, [x9, #0x8]\n" + ".inst 0x4fa3e3b4 // sdot v20.4s, v29.16b, v3.4b[1]\n" + "ldr x22, [x28, #0x8]\n" + ".inst 0x4fa4e3b8 // sdot v24.4s, v29.16b, v4.4b[1]\n" + "ldr d29, [x15, #0x60]\n" + ".inst 0x4fa0e389 // sdot v9.4s, v28.16b, v0.4b[1]\n" + "mov v29.d[1], x21\n" + ".inst 0x4fa1e38d // sdot v13.4s, v28.16b, v1.4b[1]\n" + "ldr x21, [x15, #0x88]\n" + ".inst 0x4fa2e391 // sdot v17.4s, v28.16b, v2.4b[1]\n" + "sub x13, x13, #0x10\n" + ".inst 0x4fa3e395 // sdot v21.4s, v28.16b, v3.4b[1]\n" + "cmp x13, #0x20\n" + ".inst 0x4fa4e399 // sdot v25.4s, v28.16b, v4.4b[1]\n" + "ldr d28, [x15, #0x70]\n" + "mov v28.d[1], x20\n" + ".inst 0x4fa0e3aa // sdot v10.4s, v29.16b, v0.4b[1]\n" + ".inst 0x4fa1e3ae // sdot v14.4s, v29.16b, v1.4b[1]\n" + "ldr x20, [x15, #0x98]\n" + ".inst 0x4fa2e3b2 // sdot v18.4s, v29.16b, v2.4b[1]\n" + "prfm pldl1keep, [x12, #0x80]\n" + ".inst 0x4fa3e3b6 // sdot v22.4s, v29.16b, v3.4b[1]\n" + "prfm pldl1keep, [x11, #0x80]\n" + ".inst 0x4fa4e3ba // sdot v26.4s, v29.16b, v4.4b[1]\n" + "ldr d29, [x15, #0x80]\n" + ".inst 0x4fa0e38b // sdot v11.4s, v28.16b, v0.4b[1]\n" + "mov v29.d[1], x21\n" + ".inst 0x4fa1e38f // sdot v15.4s, v28.16b, v1.4b[1]\n" + "ldr x21, [x15, #0xa8]\n" + ".inst 0x4fa2e393 // sdot v19.4s, v28.16b, v2.4b[1]\n" + "prfm pldl1keep, [x10, #0x80]\n" + ".inst 0x4fa3e397 // sdot v23.4s, v28.16b, v3.4b[1]\n" + "prfm pldl1keep, [x9, #0x80]\n" + ".inst 0x4fa4e39b // sdot v27.4s, v28.16b, v4.4b[1]\n" + "ldr d28, [x15, #0x90]\n" + "mov v28.d[1], x20\n" + ".inst 0x4f80eba8 // sdot v8.4s, v29.16b, v0.4b[2]\n" + ".inst 0x4f81ebac // sdot v12.4s, v29.16b, v1.4b[2]\n" + "ldr x20, [x15, #0xb8]\n" + ".inst 0x4f82ebb0 // sdot v16.4s, v29.16b, v2.4b[2]\n" + "prfm pldl1keep, [x28, #0x80]\n" + ".inst 0x4f83ebb4 // sdot v20.4s, v29.16b, v3.4b[2]\n" + ".inst 0x4f84ebb8 // sdot v24.4s, v29.16b, v4.4b[2]\n" + "ldr d29, [x15, #0xa0]\n" + ".inst 0x4f80eb89 // sdot v9.4s, v28.16b, v0.4b[2]\n" + "mov v29.d[1], x21\n" + ".inst 0x4f81eb8d // sdot v13.4s, v28.16b, v1.4b[2]\n" + "ldr x21, [x15, #0xc8]\n" + ".inst 0x4f82eb91 // sdot v17.4s, v28.16b, v2.4b[2]\n" + ".inst 0x4f83eb95 // sdot v21.4s, v28.16b, v3.4b[2]\n" + ".inst 0x4f84eb99 // sdot v25.4s, v28.16b, v4.4b[2]\n" + "ldr d28, [x15, #0xb0]\n" + "mov v28.d[1], x20\n" + ".inst 0x4f80ebaa // sdot v10.4s, v29.16b, v0.4b[2]\n" + ".inst 0x4f81ebae // sdot v14.4s, v29.16b, v1.4b[2]\n" + "ldr x20, [x15, #0xd8]\n" + ".inst 0x4f82ebb2 // sdot v18.4s, v29.16b, v2.4b[2]\n" + ".inst 0x4f83ebb6 // sdot v22.4s, v29.16b, v3.4b[2]\n" + ".inst 0x4f84ebba // sdot v26.4s, v29.16b, v4.4b[2]\n" + "ldr d29, [x15, #0xc0]\n" + ".inst 0x4f80eb8b // sdot v11.4s, v28.16b, v0.4b[2]\n" + "mov v29.d[1], x21\n" + ".inst 0x4f81eb8f // sdot v15.4s, v28.16b, v1.4b[2]\n" + "ldr x21, [x15, #0xe8]\n" + ".inst 0x4f82eb93 // sdot v19.4s, v28.16b, v2.4b[2]\n" + ".inst 0x4f83eb97 // sdot v23.4s, v28.16b, v3.4b[2]\n" + ".inst 0x4f84eb9b // sdot v27.4s, v28.16b, v4.4b[2]\n" + "ldr d28, [x15, #0xd0]\n" + "mov v28.d[1], x20\n" + ".inst 0x4fa0eba8 // sdot v8.4s, v29.16b, v0.4b[3]\n" + ".inst 0x4fa1ebac // sdot v12.4s, v29.16b, v1.4b[3]\n" + "ldr x20, [x15, #0xf8]\n" + ".inst 0x4fa2ebb0 // sdot v16.4s, v29.16b, v2.4b[3]\n" + ".inst 0x4fa3ebb4 // sdot v20.4s, v29.16b, v3.4b[3]\n" + ".inst 0x4fa4ebb8 // sdot v24.4s, v29.16b, v4.4b[3]\n" + "ldr d29, [x15, #0xe0]\n" + ".inst 0x4fa0eb89 // sdot v9.4s, v28.16b, v0.4b[3]\n" + "mov v29.d[1], x21\n" + ".inst 0x4fa1eb8d // sdot v13.4s, v28.16b, v1.4b[3]\n" + ".inst 0x4fa2eb91 // sdot v17.4s, v28.16b, v2.4b[3]\n" + ".inst 0x4fa3eb95 // sdot v21.4s, v28.16b, v3.4b[3]\n" + ".inst 0x4fa4eb99 // sdot v25.4s, v28.16b, v4.4b[3]\n" + "ldr d28, [x15, #0xf0]\n" + "mov v28.d[1], x20\n" + "add x15, x15, #0x100\n" + ".inst 0x4fa0ebaa // sdot v10.4s, v29.16b, v0.4b[3]\n" + "ldr x21, [x15, #0x8]\n" + ".inst 0x4fa1ebae // sdot v14.4s, v29.16b, v1.4b[3]\n" + "ldr x20, [x15, #0x18]\n" + ".inst 0x4fa2ebb2 // sdot v18.4s, v29.16b, v2.4b[3]\n" + ".inst 0x4fa3ebb6 // sdot v22.4s, v29.16b, v3.4b[3]\n" + ".inst 0x4fa4ebba // sdot v26.4s, v29.16b, v4.4b[3]\n" + "ldr d6, [x15, #0x0]\n" + ".inst 0x4fa0eb8b // sdot v11.4s, v28.16b, v0.4b[3]\n" + "ldr d0, [x12, #0x0]\n" + ".inst 0x4fa1eb8f // sdot v15.4s, v28.16b, v1.4b[3]\n" + "ldr d1, [x11, #0x0]\n" + ".inst 0x4fa2eb93 // sdot v19.4s, v28.16b, v2.4b[3]\n" + "ldr d2, [x10, #0x0]\n" + ".inst 0x4fa3eb97 // sdot v23.4s, v28.16b, v3.4b[3]\n" + "ldr d3, [x9, #0x0]\n" + ".inst 0x4fa4eb9b // sdot v27.4s, v28.16b, v4.4b[3]\n" + "ldr d4, [x28, #0x0]\n" + "ldr d7, [x15, #0x10]\n" + "mov v6.d[1], x21\n" + "mov v0.d[1], x26\n" + "mov v1.d[1], x25\n" + "mov v2.d[1], x24\n" + "mov v3.d[1], x23\n" + "mov v4.d[1], x22\n" + "mov v7.d[1], x20\n" + "bge 115b\n" + "116:" // Height 5: Multiply loop: Single iteration only + ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" + "add x12, x12, #0x10\n" + ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" + "add x11, x11, #0x10\n" + ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" + "add x10, x10, #0x10\n" + ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" + "add x9, x9, #0x10\n" + ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" + "ldr q29, [x15, #0x20]\n" + ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" + "add x28, x28, #0x10\n" + ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" + "sub x13, x13, #0x10\n" + ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" + "prfm pldl1keep, [x12, #0x80]\n" + ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" + "prfm pldl1keep, [x11, #0x80]\n" + ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" + "ldr q28, [x15, #0x30]\n" + ".inst 0x4f80e3aa // sdot v10.4s, v29.16b, v0.4b[0]\n" + "prfm pldl1keep, [x10, #0x80]\n" + ".inst 0x4f81e3ae // sdot v14.4s, v29.16b, v1.4b[0]\n" + "prfm pldl1keep, [x9, #0x80]\n" + ".inst 0x4f82e3b2 // sdot v18.4s, v29.16b, v2.4b[0]\n" + "prfm pldl1keep, [x28, #0x80]\n" + ".inst 0x4f83e3b6 // sdot v22.4s, v29.16b, v3.4b[0]\n" + ".inst 0x4f84e3ba // sdot v26.4s, v29.16b, v4.4b[0]\n" + "ldr q29, [x15, #0x40]\n" + ".inst 0x4f80e38b // sdot v11.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f81e38f // sdot v15.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f82e393 // sdot v19.4s, v28.16b, v2.4b[0]\n" + ".inst 0x4f83e397 // sdot v23.4s, v28.16b, v3.4b[0]\n" + ".inst 0x4f84e39b // sdot v27.4s, v28.16b, v4.4b[0]\n" + "ldr q28, [x15, #0x50]\n" + ".inst 0x4fa0e3a8 // sdot v8.4s, v29.16b, v0.4b[1]\n" + ".inst 0x4fa1e3ac // sdot v12.4s, v29.16b, v1.4b[1]\n" + ".inst 0x4fa2e3b0 // sdot v16.4s, v29.16b, v2.4b[1]\n" + ".inst 0x4fa3e3b4 // sdot v20.4s, v29.16b, v3.4b[1]\n" + ".inst 0x4fa4e3b8 // sdot v24.4s, v29.16b, v4.4b[1]\n" + "ldr q29, [x15, #0x60]\n" + ".inst 0x4fa0e389 // sdot v9.4s, v28.16b, v0.4b[1]\n" + ".inst 0x4fa1e38d // sdot v13.4s, v28.16b, v1.4b[1]\n" + ".inst 0x4fa2e391 // sdot v17.4s, v28.16b, v2.4b[1]\n" + ".inst 0x4fa3e395 // sdot v21.4s, v28.16b, v3.4b[1]\n" + ".inst 0x4fa4e399 // sdot v25.4s, v28.16b, v4.4b[1]\n" + "ldr q28, [x15, #0x70]\n" + ".inst 0x4fa0e3aa // sdot v10.4s, v29.16b, v0.4b[1]\n" + ".inst 0x4fa1e3ae // sdot v14.4s, v29.16b, v1.4b[1]\n" + ".inst 0x4fa2e3b2 // sdot v18.4s, v29.16b, v2.4b[1]\n" + ".inst 0x4fa3e3b6 // sdot v22.4s, v29.16b, v3.4b[1]\n" + ".inst 0x4fa4e3ba // sdot v26.4s, v29.16b, v4.4b[1]\n" + "ldr q29, [x15, #0x80]\n" + ".inst 0x4fa0e38b // sdot v11.4s, v28.16b, v0.4b[1]\n" + ".inst 0x4fa1e38f // sdot v15.4s, v28.16b, v1.4b[1]\n" + ".inst 0x4fa2e393 // sdot v19.4s, v28.16b, v2.4b[1]\n" + ".inst 0x4fa3e397 // sdot v23.4s, v28.16b, v3.4b[1]\n" + ".inst 0x4fa4e39b // sdot v27.4s, v28.16b, v4.4b[1]\n" + "ldr q28, [x15, #0x90]\n" + ".inst 0x4f80eba8 // sdot v8.4s, v29.16b, v0.4b[2]\n" + ".inst 0x4f81ebac // sdot v12.4s, v29.16b, v1.4b[2]\n" + ".inst 0x4f82ebb0 // sdot v16.4s, v29.16b, v2.4b[2]\n" + ".inst 0x4f83ebb4 // sdot v20.4s, v29.16b, v3.4b[2]\n" + ".inst 0x4f84ebb8 // sdot v24.4s, v29.16b, v4.4b[2]\n" + "ldr q29, [x15, #0xa0]\n" + ".inst 0x4f80eb89 // sdot v9.4s, v28.16b, v0.4b[2]\n" + ".inst 0x4f81eb8d // sdot v13.4s, v28.16b, v1.4b[2]\n" + ".inst 0x4f82eb91 // sdot v17.4s, v28.16b, v2.4b[2]\n" + ".inst 0x4f83eb95 // sdot v21.4s, v28.16b, v3.4b[2]\n" + ".inst 0x4f84eb99 // sdot v25.4s, v28.16b, v4.4b[2]\n" + "ldr q28, [x15, #0xb0]\n" + ".inst 0x4f80ebaa // sdot v10.4s, v29.16b, v0.4b[2]\n" + ".inst 0x4f81ebae // sdot v14.4s, v29.16b, v1.4b[2]\n" + ".inst 0x4f82ebb2 // sdot v18.4s, v29.16b, v2.4b[2]\n" + ".inst 0x4f83ebb6 // sdot v22.4s, v29.16b, v3.4b[2]\n" + ".inst 0x4f84ebba // sdot v26.4s, v29.16b, v4.4b[2]\n" + "ldr q29, [x15, #0xc0]\n" + ".inst 0x4f80eb8b // sdot v11.4s, v28.16b, v0.4b[2]\n" + ".inst 0x4f81eb8f // sdot v15.4s, v28.16b, v1.4b[2]\n" + ".inst 0x4f82eb93 // sdot v19.4s, v28.16b, v2.4b[2]\n" + ".inst 0x4f83eb97 // sdot v23.4s, v28.16b, v3.4b[2]\n" + ".inst 0x4f84eb9b // sdot v27.4s, v28.16b, v4.4b[2]\n" + "ldr q28, [x15, #0xd0]\n" + ".inst 0x4fa0eba8 // sdot v8.4s, v29.16b, v0.4b[3]\n" + ".inst 0x4fa1ebac // sdot v12.4s, v29.16b, v1.4b[3]\n" + ".inst 0x4fa2ebb0 // sdot v16.4s, v29.16b, v2.4b[3]\n" + ".inst 0x4fa3ebb4 // sdot v20.4s, v29.16b, v3.4b[3]\n" + ".inst 0x4fa4ebb8 // sdot v24.4s, v29.16b, v4.4b[3]\n" + "ldr q29, [x15, #0xe0]\n" + ".inst 0x4fa0eb89 // sdot v9.4s, v28.16b, v0.4b[3]\n" + ".inst 0x4fa1eb8d // sdot v13.4s, v28.16b, v1.4b[3]\n" + ".inst 0x4fa2eb91 // sdot v17.4s, v28.16b, v2.4b[3]\n" + ".inst 0x4fa3eb95 // sdot v21.4s, v28.16b, v3.4b[3]\n" + ".inst 0x4fa4eb99 // sdot v25.4s, v28.16b, v4.4b[3]\n" + "ldr q28, [x15, #0xf0]\n" + ".inst 0x4fa0ebaa // sdot v10.4s, v29.16b, v0.4b[3]\n" + "add x15, x15, #0x100\n" + ".inst 0x4fa1ebae // sdot v14.4s, v29.16b, v1.4b[3]\n" + ".inst 0x4fa2ebb2 // sdot v18.4s, v29.16b, v2.4b[3]\n" + ".inst 0x4fa3ebb6 // sdot v22.4s, v29.16b, v3.4b[3]\n" + ".inst 0x4fa4ebba // sdot v26.4s, v29.16b, v4.4b[3]\n" + ".inst 0x4fa0eb8b // sdot v11.4s, v28.16b, v0.4b[3]\n" + ".inst 0x4fa1eb8f // sdot v15.4s, v28.16b, v1.4b[3]\n" + ".inst 0x4fa2eb93 // sdot v19.4s, v28.16b, v2.4b[3]\n" + ".inst 0x4fa3eb97 // sdot v23.4s, v28.16b, v3.4b[3]\n" + ".inst 0x4fa4eb9b // sdot v27.4s, v28.16b, v4.4b[3]\n" + "117:" // Height 5: Multiply loop: Main loop skip + "cbz x13, 122f\n" + "cmp x13, #0x4\n" + "blt 119f\n" + "118:" // Height 5: Multiply loop: Odd block loop + "ldr s2, [x12], #0x4\n" + "sub x13, x13, #0x4\n" + "ldr s1, [x11], #0x4\n" + "cmp x13, #0x4\n" + "ldr s0, [x10], #0x4\n" + "ldr s31, [x9], #0x4\n" + "ldr s30, [x28], #0x4\n" + "ldr q29, [x15, #0x0]\n" + ".inst 0x4f82e3a8 // sdot v8.4s, v29.16b, v2.4b[0]\n" + "ldr q28, [x15, #0x10]\n" + ".inst 0x4f81e3ac // sdot v12.4s, v29.16b, v1.4b[0]\n" + ".inst 0x4f80e3b0 // sdot v16.4s, v29.16b, v0.4b[0]\n" + ".inst 0x4f9fe3b4 // sdot v20.4s, v29.16b, v31.4b[0]\n" + ".inst 0x4f9ee3b8 // sdot v24.4s, v29.16b, v30.4b[0]\n" + "ldr q29, [x15, #0x20]\n" + ".inst 0x4f82e389 // sdot v9.4s, v28.16b, v2.4b[0]\n" + ".inst 0x4f81e38d // sdot v13.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f80e391 // sdot v17.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f9fe395 // sdot v21.4s, v28.16b, v31.4b[0]\n" + ".inst 0x4f9ee399 // sdot v25.4s, v28.16b, v30.4b[0]\n" + "ldr q28, [x15, #0x30]\n" + ".inst 0x4f82e3aa // sdot v10.4s, v29.16b, v2.4b[0]\n" "add x15, x15, #0x40\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" + ".inst 0x4f81e3ae // sdot v14.4s, v29.16b, v1.4b[0]\n" + ".inst 0x4f80e3b2 // sdot v18.4s, v29.16b, v0.4b[0]\n" + ".inst 0x4f9fe3b6 // sdot v22.4s, v29.16b, v31.4b[0]\n" + ".inst 0x4f9ee3ba // sdot v26.4s, v29.16b, v30.4b[0]\n" + ".inst 0x4f82e38b // sdot v11.4s, v28.16b, v2.4b[0]\n" + ".inst 0x4f81e38f // sdot v15.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f80e393 // sdot v19.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f9fe397 // sdot v23.4s, v28.16b, v31.4b[0]\n" + ".inst 0x4f9ee39b // sdot v27.4s, v28.16b, v30.4b[0]\n" "bge 118b\n" "119:" // Height 5: Multiply loop: Skip odd blocks "cbz x13, 122f\n" "tbz x13, #1, 120f\n" "ldr h0, [x12], #0x2\n" - "ldr h1, [x9], #0x2\n" - "ldr h2, [x27], #0x2\n" - "ldr h3, [x25], #0x2\n" - "ldr h4, [x23], #0x2\n" + "ldr h1, [x11], #0x2\n" + "ldr h2, [x10], #0x2\n" + "ldr h3, [x9], #0x2\n" + "ldr h4, [x28], #0x2\n" "tbz x13, #0, 121f\n" "ld1 { v0.b }[2], [x12]\n" - "ld1 { v1.b }[2], [x9]\n" - "ld1 { v2.b }[2], [x27]\n" - "ld1 { v3.b }[2], [x25]\n" - "ld1 { v4.b }[2], [x23]\n" + "ld1 { v1.b }[2], [x11]\n" + "ld1 { v2.b }[2], [x10]\n" + "ld1 { v3.b }[2], [x9]\n" + "ld1 { v4.b }[2], [x28]\n" "b 121f\n" "120:" // Height 5: Multiply loop: Ragged operand read: partial_1_0 "ldr b0, [x12, #0x0]\n" - "ldr b1, [x9, #0x0]\n" - "ldr b2, [x27, #0x0]\n" - "ldr b3, [x25, #0x0]\n" - "ldr b4, [x23, #0x0]\n" + "ldr b1, [x11, #0x0]\n" + "ldr b2, [x10, #0x0]\n" + "ldr b3, [x9, #0x0]\n" + "ldr b4, [x28, #0x0]\n" "121:" // Height 5: Multiply loop: Ragged operand read: Done - "ldr q6, [x15, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x15, #0x10]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x15, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x15, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q29, [x15, #0x0]\n" + ".inst 0x4f80e3a8 // sdot v8.4s, v29.16b, v0.4b[0]\n" + "ldr q28, [x15, #0x10]\n" + ".inst 0x4f81e3ac // sdot v12.4s, v29.16b, v1.4b[0]\n" + ".inst 0x4f82e3b0 // sdot v16.4s, v29.16b, v2.4b[0]\n" + ".inst 0x4f83e3b4 // sdot v20.4s, v29.16b, v3.4b[0]\n" + ".inst 0x4f84e3b8 // sdot v24.4s, v29.16b, v4.4b[0]\n" + "ldr q29, [x15, #0x20]\n" + ".inst 0x4f80e389 // sdot v9.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f81e38d // sdot v13.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f82e391 // sdot v17.4s, v28.16b, v2.4b[0]\n" + ".inst 0x4f83e395 // sdot v21.4s, v28.16b, v3.4b[0]\n" + ".inst 0x4f84e399 // sdot v25.4s, v28.16b, v4.4b[0]\n" + "ldr q28, [x15, #0x30]\n" + ".inst 0x4f80e3aa // sdot v10.4s, v29.16b, v0.4b[0]\n" "add x15, x15, #0x40\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" + ".inst 0x4f81e3ae // sdot v14.4s, v29.16b, v1.4b[0]\n" + ".inst 0x4f82e3b2 // sdot v18.4s, v29.16b, v2.4b[0]\n" + ".inst 0x4f83e3b6 // sdot v22.4s, v29.16b, v3.4b[0]\n" + ".inst 0x4f84e3ba // sdot v26.4s, v29.16b, v4.4b[0]\n" + ".inst 0x4f80e38b // sdot v11.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f81e38f // sdot v15.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f82e393 // sdot v19.4s, v28.16b, v2.4b[0]\n" + ".inst 0x4f83e397 // sdot v23.4s, v28.16b, v3.4b[0]\n" + ".inst 0x4f84e39b // sdot v27.4s, v28.16b, v4.4b[0]\n" "122:" // Height 5: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x14, x14, #0x1\n" "cmp x14, x20\n" "bne 112b\n" - "ldr q0, [x6, #0x0]\n" - "add v8.4s, v8.4s, v0.4s\n" - "ldr q1, [x6, #0x10]\n" - "add v9.4s, v9.4s, v1.4s\n" - "ldr q2, [x6, #0x20]\n" - "add v10.4s, v10.4s, v2.4s\n" - "ldr q3, [x6, #0x30]\n" - "add v11.4s, v11.4s, v3.4s\n" + "ldr q31, [x6, #0x0]\n" + "add v8.4s, v8.4s, v31.4s\n" + "ldr q30, [x6, #0x10]\n" + "add v9.4s, v9.4s, v30.4s\n" + "ldr q29, [x6, #0x20]\n" + "add v10.4s, v10.4s, v29.4s\n" + "ldr q28, [x6, #0x30]\n" + "add v11.4s, v11.4s, v28.4s\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x17, x20\n" + "add x25, x17, x20\n" + "add x24, x25, x20\n" "add x23, x24, x20\n" "add x22, x23, x20\n" - "add x21, x22, x20\n" "prfm pstl1keep, [x17, #0x0]\n" + "prfm pstl1keep, [x25, #0x0]\n" + "add v12.4s, v12.4s, v31.4s\n" "prfm pstl1keep, [x24, #0x0]\n" - "add v12.4s, v12.4s, v0.4s\n" + "add v13.4s, v13.4s, v30.4s\n" "prfm pstl1keep, [x23, #0x0]\n" - "add v13.4s, v13.4s, v1.4s\n" + "add v14.4s, v14.4s, v29.4s\n" "prfm pstl1keep, [x22, #0x0]\n" - "add v14.4s, v14.4s, v2.4s\n" - "prfm pstl1keep, [x21, #0x0]\n" - "add v15.4s, v15.4s, v3.4s\n" - "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" - "add v20.4s, v20.4s, v0.4s\n" - "add v21.4s, v21.4s, v1.4s\n" - "add v22.4s, v22.4s, v2.4s\n" - "add v23.4s, v23.4s, v3.4s\n" - "add v24.4s, v24.4s, v0.4s\n" - "add v25.4s, v25.4s, v1.4s\n" - "add v26.4s, v26.4s, v2.4s\n" - "add v27.4s, v27.4s, v3.4s\n" + "add v15.4s, v15.4s, v28.4s\n" + "add v16.4s, v16.4s, v31.4s\n" + "add v17.4s, v17.4s, v30.4s\n" + "add v18.4s, v18.4s, v29.4s\n" + "add v19.4s, v19.4s, v28.4s\n" + "add v20.4s, v20.4s, v31.4s\n" + "add v21.4s, v21.4s, v30.4s\n" + "add v22.4s, v22.4s, v29.4s\n" + "add v23.4s, v23.4s, v28.4s\n" + "add v24.4s, v24.4s, v31.4s\n" + "add v25.4s, v25.4s, v30.4s\n" + "add v26.4s, v26.4s, v29.4s\n" + "add v27.4s, v27.4s, v28.4s\n" "add x6, x6, #0x40\n" "tbz %x[flags], #4, 123f\n" "ldr q0, [x8, #0x0]\n" @@ -2536,10 +2535,10 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "add x7, x7, #0x40\n" "b 124f\n" "123:" // Height 5: per layer parameters - "add x25, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x25]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v4.4s }, [x20]\n" "mov v1.16b, v0.16b\n" "mov v5.16b, v4.16b\n" "mov v2.16b, v0.16b\n" @@ -2568,66 +2567,66 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "sqrdmulh v26.4s, v26.4s, v6.4s\n" "sqrdmulh v27.4s, v27.4s, v7.4s\n" "tbz %x[flags], #5, 125f\n" - "and v4.16b, v8.16b, v0.16b\n" - "and v5.16b, v9.16b, v1.16b\n" - "and v6.16b, v10.16b, v2.16b\n" - "and v7.16b, v11.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v4.4s\n" - "sqadd v9.4s, v9.4s, v5.4s\n" - "sqadd v10.4s, v10.4s, v6.4s\n" - "sqadd v11.4s, v11.4s, v7.4s\n" - "and v4.16b, v12.16b, v0.16b\n" - "and v5.16b, v13.16b, v1.16b\n" - "and v6.16b, v14.16b, v2.16b\n" - "and v7.16b, v15.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v12.4s, v12.4s, v4.4s\n" - "sqadd v13.4s, v13.4s, v5.4s\n" - "sqadd v14.4s, v14.4s, v6.4s\n" - "sqadd v15.4s, v15.4s, v7.4s\n" - "and v4.16b, v16.16b, v0.16b\n" - "and v5.16b, v17.16b, v1.16b\n" - "and v6.16b, v18.16b, v2.16b\n" - "and v7.16b, v19.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" - "and v4.16b, v20.16b, v0.16b\n" - "and v5.16b, v21.16b, v1.16b\n" - "and v6.16b, v22.16b, v2.16b\n" - "and v7.16b, v23.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v20.4s, v20.4s, v4.4s\n" - "sqadd v21.4s, v21.4s, v5.4s\n" - "sqadd v22.4s, v22.4s, v6.4s\n" - "sqadd v23.4s, v23.4s, v7.4s\n" - "and v4.16b, v24.16b, v0.16b\n" - "and v5.16b, v25.16b, v1.16b\n" - "and v6.16b, v26.16b, v2.16b\n" - "and v7.16b, v27.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v24.4s, v24.4s, v4.4s\n" - "sqadd v25.4s, v25.4s, v5.4s\n" - "sqadd v26.4s, v26.4s, v6.4s\n" - "sqadd v27.4s, v27.4s, v7.4s\n" + "and v31.16b, v8.16b, v0.16b\n" + "and v30.16b, v9.16b, v1.16b\n" + "and v29.16b, v10.16b, v2.16b\n" + "and v28.16b, v11.16b, v3.16b\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sqadd v8.4s, v8.4s, v31.4s\n" + "sqadd v9.4s, v9.4s, v30.4s\n" + "sqadd v10.4s, v10.4s, v29.4s\n" + "sqadd v11.4s, v11.4s, v28.4s\n" + "and v31.16b, v12.16b, v0.16b\n" + "and v30.16b, v13.16b, v1.16b\n" + "and v29.16b, v14.16b, v2.16b\n" + "and v28.16b, v15.16b, v3.16b\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sqadd v12.4s, v12.4s, v31.4s\n" + "sqadd v13.4s, v13.4s, v30.4s\n" + "sqadd v14.4s, v14.4s, v29.4s\n" + "sqadd v15.4s, v15.4s, v28.4s\n" + "and v31.16b, v16.16b, v0.16b\n" + "and v30.16b, v17.16b, v1.16b\n" + "and v29.16b, v18.16b, v2.16b\n" + "and v28.16b, v19.16b, v3.16b\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v31.4s\n" + "sqadd v17.4s, v17.4s, v30.4s\n" + "sqadd v18.4s, v18.4s, v29.4s\n" + "sqadd v19.4s, v19.4s, v28.4s\n" + "and v31.16b, v20.16b, v0.16b\n" + "and v30.16b, v21.16b, v1.16b\n" + "and v29.16b, v22.16b, v2.16b\n" + "and v28.16b, v23.16b, v3.16b\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v31.4s\n" + "sqadd v21.4s, v21.4s, v30.4s\n" + "sqadd v22.4s, v22.4s, v29.4s\n" + "sqadd v23.4s, v23.4s, v28.4s\n" + "and v31.16b, v24.16b, v0.16b\n" + "and v30.16b, v25.16b, v1.16b\n" + "and v29.16b, v26.16b, v2.16b\n" + "and v28.16b, v27.16b, v3.16b\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sqadd v24.4s, v24.4s, v31.4s\n" + "sqadd v25.4s, v25.4s, v30.4s\n" + "sqadd v26.4s, v26.4s, v29.4s\n" + "sqadd v27.4s, v27.4s, v28.4s\n" "125:" // Height 5: no shift correction "srshl v8.4s, v8.4s, v0.4s\n" "srshl v9.4s, v9.4s, v1.4s\n" @@ -2649,201 +2648,201 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "srshl v25.4s, v25.4s, v1.4s\n" "srshl v26.4s, v26.4s, v2.4s\n" "srshl v27.4s, v27.4s, v3.4s\n" - "add x25, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x25]\n" - "add v8.4s, v8.4s, v4.4s\n" - "add v9.4s, v9.4s, v4.4s\n" - "add v10.4s, v10.4s, v4.4s\n" - "add v11.4s, v11.4s, v4.4s\n" - "add v12.4s, v12.4s, v4.4s\n" - "add v13.4s, v13.4s, v4.4s\n" - "add v14.4s, v14.4s, v4.4s\n" - "add v15.4s, v15.4s, v4.4s\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v23.4s, v23.4s, v4.4s\n" - "add v24.4s, v24.4s, v4.4s\n" - "add v25.4s, v25.4s, v4.4s\n" - "add v26.4s, v26.4s, v4.4s\n" - "add v27.4s, v27.4s, v4.4s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x25]\n" - "smin v8.4s, v8.4s, v6.4s\n" - "smin v9.4s, v9.4s, v6.4s\n" - "smin v10.4s, v10.4s, v6.4s\n" - "smin v11.4s, v11.4s, v6.4s\n" - "smin v12.4s, v12.4s, v6.4s\n" - "smin v13.4s, v13.4s, v6.4s\n" - "smin v14.4s, v14.4s, v6.4s\n" - "smin v15.4s, v15.4s, v6.4s\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "smin v24.4s, v24.4s, v6.4s\n" - "smin v25.4s, v25.4s, v6.4s\n" - "smin v26.4s, v26.4s, v6.4s\n" - "smin v27.4s, v27.4s, v6.4s\n" - "add x25, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x25]\n" - "smax v8.4s, v8.4s, v5.4s\n" - "smax v9.4s, v9.4s, v5.4s\n" - "smax v10.4s, v10.4s, v5.4s\n" - "smax v11.4s, v11.4s, v5.4s\n" - "smax v12.4s, v12.4s, v5.4s\n" - "smax v13.4s, v13.4s, v5.4s\n" - "smax v14.4s, v14.4s, v5.4s\n" - "smax v15.4s, v15.4s, v5.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" - "smax v24.4s, v24.4s, v5.4s\n" - "smax v25.4s, v25.4s, v5.4s\n" - "smax v26.4s, v26.4s, v5.4s\n" - "smax v27.4s, v27.4s, v5.4s\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v28.4s }, [x20]\n" + "add v8.4s, v8.4s, v28.4s\n" + "add v9.4s, v9.4s, v28.4s\n" + "add v10.4s, v10.4s, v28.4s\n" + "add v11.4s, v11.4s, v28.4s\n" + "add v12.4s, v12.4s, v28.4s\n" + "add v13.4s, v13.4s, v28.4s\n" + "add v14.4s, v14.4s, v28.4s\n" + "add v15.4s, v15.4s, v28.4s\n" + "add v16.4s, v16.4s, v28.4s\n" + "add v17.4s, v17.4s, v28.4s\n" + "add v18.4s, v18.4s, v28.4s\n" + "add v19.4s, v19.4s, v28.4s\n" + "add v20.4s, v20.4s, v28.4s\n" + "add v21.4s, v21.4s, v28.4s\n" + "add v22.4s, v22.4s, v28.4s\n" + "add v23.4s, v23.4s, v28.4s\n" + "add v24.4s, v24.4s, v28.4s\n" + "add v25.4s, v25.4s, v28.4s\n" + "add v26.4s, v26.4s, v28.4s\n" + "add v27.4s, v27.4s, v28.4s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v28.4s }, [x20]\n" + "smin v8.4s, v8.4s, v28.4s\n" + "smin v9.4s, v9.4s, v28.4s\n" + "smin v10.4s, v10.4s, v28.4s\n" + "smin v11.4s, v11.4s, v28.4s\n" + "smin v12.4s, v12.4s, v28.4s\n" + "smin v13.4s, v13.4s, v28.4s\n" + "smin v14.4s, v14.4s, v28.4s\n" + "smin v15.4s, v15.4s, v28.4s\n" + "smin v16.4s, v16.4s, v28.4s\n" + "smin v17.4s, v17.4s, v28.4s\n" + "smin v18.4s, v18.4s, v28.4s\n" + "smin v19.4s, v19.4s, v28.4s\n" + "smin v20.4s, v20.4s, v28.4s\n" + "smin v21.4s, v21.4s, v28.4s\n" + "smin v22.4s, v22.4s, v28.4s\n" + "smin v23.4s, v23.4s, v28.4s\n" + "smin v24.4s, v24.4s, v28.4s\n" + "smin v25.4s, v25.4s, v28.4s\n" + "smin v26.4s, v26.4s, v28.4s\n" + "smin v27.4s, v27.4s, v28.4s\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v28.4s }, [x20]\n" + "smax v8.4s, v8.4s, v28.4s\n" + "smax v9.4s, v9.4s, v28.4s\n" + "smax v10.4s, v10.4s, v28.4s\n" + "smax v11.4s, v11.4s, v28.4s\n" + "smax v12.4s, v12.4s, v28.4s\n" + "smax v13.4s, v13.4s, v28.4s\n" + "smax v14.4s, v14.4s, v28.4s\n" + "smax v15.4s, v15.4s, v28.4s\n" + "smax v16.4s, v16.4s, v28.4s\n" + "smax v17.4s, v17.4s, v28.4s\n" + "smax v18.4s, v18.4s, v28.4s\n" + "smax v19.4s, v19.4s, v28.4s\n" + "smax v20.4s, v20.4s, v28.4s\n" + "smax v21.4s, v21.4s, v28.4s\n" + "smax v22.4s, v22.4s, v28.4s\n" + "smax v23.4s, v23.4s, v28.4s\n" + "smax v24.4s, v24.4s, v28.4s\n" + "smax v25.4s, v25.4s, v28.4s\n" + "smax v26.4s, v26.4s, v28.4s\n" + "smax v27.4s, v27.4s, v28.4s\n" "uzp1 v8.8h, v8.8h, v9.8h\n" - "uzp1 v9.8h, v10.8h, v11.8h\n" + "uzp1 v29.8h, v10.8h, v11.8h\n" "uzp1 v12.8h, v12.8h, v13.8h\n" - "uzp1 v13.8h, v14.8h, v15.8h\n" + "uzp1 v28.8h, v14.8h, v15.8h\n" "uzp1 v16.8h, v16.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v19.8h, v18.8h, v19.8h\n" "uzp1 v20.8h, v20.8h, v21.8h\n" - "uzp1 v21.8h, v22.8h, v23.8h\n" + "uzp1 v18.8h, v22.8h, v23.8h\n" "uzp1 v24.8h, v24.8h, v25.8h\n" - "uzp1 v25.8h, v26.8h, v27.8h\n" + "uzp1 v17.8h, v26.8h, v27.8h\n" "cmp x16, #0x10\n" - "uzp1 v8.16b, v8.16b, v9.16b\n" - "uzp1 v12.16b, v12.16b, v13.16b\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" - "uzp1 v20.16b, v20.16b, v21.16b\n" - "uzp1 v24.16b, v24.16b, v25.16b\n" + "uzp1 v8.16b, v8.16b, v29.16b\n" + "uzp1 v12.16b, v12.16b, v28.16b\n" + "uzp1 v16.16b, v16.16b, v19.16b\n" + "uzp1 v20.16b, v20.16b, v18.16b\n" + "uzp1 v24.16b, v24.16b, v17.16b\n" "bge 134f\n" "tbz x16, #3, 129f\n" "str d8, [x17], #0x8\n" - "str d12, [x24], #0x8\n" - "str d16, [x23], #0x8\n" - "str d20, [x22], #0x8\n" - "str d24, [x21], #0x8\n" + "str d12, [x25], #0x8\n" + "str d16, [x24], #0x8\n" + "str d20, [x23], #0x8\n" + "str d24, [x22], #0x8\n" "tbz x16, #2, 127f\n" "st1 { v8.s }[2], [x17], #0x4\n" - "st1 { v12.s }[2], [x24], #0x4\n" - "st1 { v16.s }[2], [x23], #0x4\n" - "st1 { v20.s }[2], [x22], #0x4\n" - "st1 { v24.s }[2], [x21], #0x4\n" + "st1 { v12.s }[2], [x25], #0x4\n" + "st1 { v16.s }[2], [x24], #0x4\n" + "st1 { v20.s }[2], [x23], #0x4\n" + "st1 { v24.s }[2], [x22], #0x4\n" "tbz x16, #1, 126f\n" "st1 { v8.h }[6], [x17], #0x2\n" - "st1 { v12.h }[6], [x24], #0x2\n" - "st1 { v16.h }[6], [x23], #0x2\n" - "st1 { v20.h }[6], [x22], #0x2\n" - "st1 { v24.h }[6], [x21], #0x2\n" + "st1 { v12.h }[6], [x25], #0x2\n" + "st1 { v16.h }[6], [x24], #0x2\n" + "st1 { v20.h }[6], [x23], #0x2\n" + "st1 { v24.h }[6], [x22], #0x2\n" "tbz x16, #0, 133f\n" "st1 { v8.b }[14], [x17]\n" - "st1 { v12.b }[14], [x24]\n" - "st1 { v16.b }[14], [x23]\n" - "st1 { v20.b }[14], [x22]\n" - "st1 { v24.b }[14], [x21]\n" + "st1 { v12.b }[14], [x25]\n" + "st1 { v16.b }[14], [x24]\n" + "st1 { v20.b }[14], [x23]\n" + "st1 { v24.b }[14], [x22]\n" "b 133f\n" "126:" // Height 5: Partial direct writeback: partial_1_12 "tbz x16, #0, 133f\n" "st1 { v8.b }[12], [x17]\n" - "st1 { v12.b }[12], [x24]\n" - "st1 { v16.b }[12], [x23]\n" - "st1 { v20.b }[12], [x22]\n" - "st1 { v24.b }[12], [x21]\n" + "st1 { v12.b }[12], [x25]\n" + "st1 { v16.b }[12], [x24]\n" + "st1 { v20.b }[12], [x23]\n" + "st1 { v24.b }[12], [x22]\n" "b 133f\n" "127:" // Height 5: Partial direct writeback: partial_2_8 "tbz x16, #1, 128f\n" "st1 { v8.h }[4], [x17], #0x2\n" - "st1 { v12.h }[4], [x24], #0x2\n" - "st1 { v16.h }[4], [x23], #0x2\n" - "st1 { v20.h }[4], [x22], #0x2\n" - "st1 { v24.h }[4], [x21], #0x2\n" + "st1 { v12.h }[4], [x25], #0x2\n" + "st1 { v16.h }[4], [x24], #0x2\n" + "st1 { v20.h }[4], [x23], #0x2\n" + "st1 { v24.h }[4], [x22], #0x2\n" "tbz x16, #0, 133f\n" "st1 { v8.b }[10], [x17]\n" - "st1 { v12.b }[10], [x24]\n" - "st1 { v16.b }[10], [x23]\n" - "st1 { v20.b }[10], [x22]\n" - "st1 { v24.b }[10], [x21]\n" + "st1 { v12.b }[10], [x25]\n" + "st1 { v16.b }[10], [x24]\n" + "st1 { v20.b }[10], [x23]\n" + "st1 { v24.b }[10], [x22]\n" "b 133f\n" "128:" // Height 5: Partial direct writeback: partial_1_8 "tbz x16, #0, 133f\n" "st1 { v8.b }[8], [x17]\n" - "st1 { v12.b }[8], [x24]\n" - "st1 { v16.b }[8], [x23]\n" - "st1 { v20.b }[8], [x22]\n" - "st1 { v24.b }[8], [x21]\n" + "st1 { v12.b }[8], [x25]\n" + "st1 { v16.b }[8], [x24]\n" + "st1 { v20.b }[8], [x23]\n" + "st1 { v24.b }[8], [x22]\n" "b 133f\n" "129:" // Height 5: Partial direct writeback: partial_4_0 "tbz x16, #2, 131f\n" "str s8, [x17], #0x4\n" - "str s12, [x24], #0x4\n" - "str s16, [x23], #0x4\n" - "str s20, [x22], #0x4\n" - "str s24, [x21], #0x4\n" + "str s12, [x25], #0x4\n" + "str s16, [x24], #0x4\n" + "str s20, [x23], #0x4\n" + "str s24, [x22], #0x4\n" "tbz x16, #1, 130f\n" "st1 { v8.h }[2], [x17], #0x2\n" - "st1 { v12.h }[2], [x24], #0x2\n" - "st1 { v16.h }[2], [x23], #0x2\n" - "st1 { v20.h }[2], [x22], #0x2\n" - "st1 { v24.h }[2], [x21], #0x2\n" + "st1 { v12.h }[2], [x25], #0x2\n" + "st1 { v16.h }[2], [x24], #0x2\n" + "st1 { v20.h }[2], [x23], #0x2\n" + "st1 { v24.h }[2], [x22], #0x2\n" "tbz x16, #0, 133f\n" "st1 { v8.b }[6], [x17]\n" - "st1 { v12.b }[6], [x24]\n" - "st1 { v16.b }[6], [x23]\n" - "st1 { v20.b }[6], [x22]\n" - "st1 { v24.b }[6], [x21]\n" + "st1 { v12.b }[6], [x25]\n" + "st1 { v16.b }[6], [x24]\n" + "st1 { v20.b }[6], [x23]\n" + "st1 { v24.b }[6], [x22]\n" "b 133f\n" "130:" // Height 5: Partial direct writeback: partial_1_4 "tbz x16, #0, 133f\n" "st1 { v8.b }[4], [x17]\n" - "st1 { v12.b }[4], [x24]\n" - "st1 { v16.b }[4], [x23]\n" - "st1 { v20.b }[4], [x22]\n" - "st1 { v24.b }[4], [x21]\n" + "st1 { v12.b }[4], [x25]\n" + "st1 { v16.b }[4], [x24]\n" + "st1 { v20.b }[4], [x23]\n" + "st1 { v24.b }[4], [x22]\n" "b 133f\n" "131:" // Height 5: Partial direct writeback: partial_2_0 "tbz x16, #1, 132f\n" "str h8, [x17], #0x2\n" - "str h12, [x24], #0x2\n" - "str h16, [x23], #0x2\n" - "str h20, [x22], #0x2\n" - "str h24, [x21], #0x2\n" + "str h12, [x25], #0x2\n" + "str h16, [x24], #0x2\n" + "str h20, [x23], #0x2\n" + "str h24, [x22], #0x2\n" "tbz x16, #0, 133f\n" "st1 { v8.b }[2], [x17]\n" - "st1 { v12.b }[2], [x24]\n" - "st1 { v16.b }[2], [x23]\n" - "st1 { v20.b }[2], [x22]\n" - "st1 { v24.b }[2], [x21]\n" + "st1 { v12.b }[2], [x25]\n" + "st1 { v16.b }[2], [x24]\n" + "st1 { v20.b }[2], [x23]\n" + "st1 { v24.b }[2], [x22]\n" "b 133f\n" "132:" // Height 5: Partial direct writeback: partial_1_0 "str b8, [x17, #0x0]\n" - "str b12, [x24, #0x0]\n" - "str b16, [x23, #0x0]\n" - "str b20, [x22, #0x0]\n" - "str b24, [x21, #0x0]\n" + "str b12, [x25, #0x0]\n" + "str b16, [x24, #0x0]\n" + "str b20, [x23, #0x0]\n" + "str b24, [x22, #0x0]\n" "133:" // Height 5: Partial direct writeback: Done "b 135f\n" "134:" // Height 5: Full writeback "str q8, [x17, #0x0]\n" "add x17, x17, #0x10\n" - "str q12, [x24, #0x0]\n" - "str q16, [x23, #0x0]\n" - "str q20, [x22, #0x0]\n" - "str q24, [x21, #0x0]\n" + "str q12, [x25, #0x0]\n" + "str q16, [x24, #0x0]\n" + "str q20, [x23, #0x0]\n" + "str q24, [x22, #0x0]\n" "135:" // Height 5: Writeback done "subs x16, x16, #0x10\n" "bgt 110b\n" @@ -2888,191 +2887,191 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "139:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w13, [x20, x14, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 140f\n" - "ldr x21, [%x[input_ptr], x14, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x12, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" - "ldr x27, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" - "ldr x23, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x14, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x12, [x20, #0x0]\n" + "ldr x11, [x20, #0x8]\n" + "ldr x10, [x20, #0x10]\n" + "ldr x9, [x20, #0x18]\n" + "ldr x28, [x20, #0x20]\n" + "ldr x27, [x20, #0x28]\n" "cbnz x14, 141f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x12, x12, x20\n" + "add x11, x11, x20\n" + "add x10, x10, x20\n" "add x9, x9, x20\n" + "add x28, x28, x20\n" "add x27, x27, x20\n" - "add x25, x25, x20\n" - "add x23, x23, x20\n" - "add x21, x21, x20\n" "b 141f\n" "140:" // Height 6: setup direct input "mov x12, %x[input_ptr]\n" - "add x9, x12, x20\n" - "add x27, x9, x20\n" - "add x25, x27, x20\n" - "add x23, x25, x20\n" - "add x21, x23, x20\n" + "add x11, x12, x21\n" + "add x10, x11, x21\n" + "add x9, x10, x21\n" + "add x28, x9, x21\n" + "add x27, x28, x21\n" "141:" // Height 6: input setup done "cmp x13, #0x10\n" "blt 144f\n" "ldr q0, [x12, #0x0]\n" "cmp x13, #0x20\n" - "ldr q1, [x9, #0x0]\n" - "ldr q2, [x27, #0x0]\n" - "ldr q3, [x25, #0x0]\n" - "ldr q4, [x23, #0x0]\n" - "ldr q5, [x21, #0x0]\n" + "ldr q1, [x11, #0x0]\n" + "ldr q2, [x10, #0x0]\n" + "ldr q3, [x9, #0x0]\n" + "ldr q4, [x28, #0x0]\n" + "ldr q5, [x27, #0x0]\n" "ldr q6, [x15, #0x0]\n" "ldr q7, [x15, #0x10]\n" "blt 143f\n" "142:" // Height 6: Multiply loop: Main loop head ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr x20, [x15, #0x28]\n" + "ldr x21, [x15, #0x28]\n" ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr x11, [x15, #0x38]\n" + "ldr x20, [x15, #0x38]\n" ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" "add x12, x12, #0x10\n" ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "add x9, x9, #0x10\n" + "add x11, x11, #0x10\n" ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - "add x27, x27, #0x10\n" + "add x10, x10, #0x10\n" ".inst 0x4f85e0dc // sdot v28.4s, v6.16b, v5.4b[0]\n" "ldr d6, [x15, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x20\n" + "mov v6.d[1], x21\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "ldr x20, [x15, #0x48]\n" + "ldr x21, [x15, #0x48]\n" ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - "add x25, x25, #0x10\n" + "add x9, x9, #0x10\n" ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - "add x23, x23, #0x10\n" + "add x28, x28, #0x10\n" ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - "add x21, x21, #0x10\n" + "add x27, x27, #0x10\n" ".inst 0x4f85e0fd // sdot v29.4s, v7.16b, v5.4b[0]\n" "ldr d7, [x15, #0x30]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "ldr x11, [x15, #0x58]\n" + "ldr x20, [x15, #0x58]\n" ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - "ldr x10, [x12, #0x8]\n" + "ldr x26, [x12, #0x8]\n" ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - "ldr x28, [x9, #0x8]\n" + "ldr x25, [x11, #0x8]\n" ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - "ldr x26, [x27, #0x8]\n" + "ldr x24, [x10, #0x8]\n" ".inst 0x4f85e0de // sdot v30.4s, v6.16b, v5.4b[0]\n" "ldr d6, [x15, #0x40]\n" ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x20\n" + "mov v6.d[1], x21\n" ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - "ldr x20, [x15, #0x68]\n" + "ldr x21, [x15, #0x68]\n" ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - "ldr x24, [x25, #0x8]\n" + "ldr x23, [x9, #0x8]\n" ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" "sub x13, x13, #0x10\n" ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" "cmp x13, #0x20\n" ".inst 0x4f85e0ff // sdot v31.4s, v7.16b, v5.4b[0]\n" "ldr d7, [x15, #0x50]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - "ldr x11, [x15, #0x78]\n" + "ldr x20, [x15, #0x78]\n" ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" "prfm pldl1keep, [x12, #0x80]\n" ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n" - "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x11, #0x80]\n" ".inst 0x4fa4e0d8 // sdot v24.4s, v6.16b, v4.4b[1]\n" - "prfm pldl1keep, [x27, #0x80]\n" + "prfm pldl1keep, [x10, #0x80]\n" ".inst 0x4fa5e0dc // sdot v28.4s, v6.16b, v5.4b[1]\n" "ldr d6, [x15, #0x60]\n" ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - "mov v6.d[1], x20\n" + "mov v6.d[1], x21\n" ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - "ldr x20, [x15, #0x88]\n" + "ldr x21, [x15, #0x88]\n" ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - "prfm pldl1keep, [x25, #0x80]\n" + "prfm pldl1keep, [x9, #0x80]\n" ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n" - "prfm pldl1keep, [x23, #0x80]\n" + "prfm pldl1keep, [x28, #0x80]\n" ".inst 0x4fa4e0f9 // sdot v25.4s, v7.16b, v4.4b[1]\n" - "prfm pldl1keep, [x21, #0x80]\n" + "prfm pldl1keep, [x27, #0x80]\n" ".inst 0x4fa5e0fd // sdot v29.4s, v7.16b, v5.4b[1]\n" "ldr d7, [x15, #0x70]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - "ldr x11, [x15, #0x98]\n" + "ldr x20, [x15, #0x98]\n" ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n" ".inst 0x4fa4e0da // sdot v26.4s, v6.16b, v4.4b[1]\n" ".inst 0x4fa5e0de // sdot v30.4s, v6.16b, v5.4b[1]\n" "ldr d6, [x15, #0x80]\n" ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - "mov v6.d[1], x20\n" + "mov v6.d[1], x21\n" ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - "ldr x20, [x15, #0xa8]\n" + "ldr x21, [x15, #0xa8]\n" ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n" ".inst 0x4fa4e0fb // sdot v27.4s, v7.16b, v4.4b[1]\n" ".inst 0x4fa5e0ff // sdot v31.4s, v7.16b, v5.4b[1]\n" "ldr d7, [x15, #0x90]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - "ldr x11, [x15, #0xb8]\n" + "ldr x20, [x15, #0xb8]\n" ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n" ".inst 0x4f84e8d8 // sdot v24.4s, v6.16b, v4.4b[2]\n" ".inst 0x4f85e8dc // sdot v28.4s, v6.16b, v5.4b[2]\n" "ldr d6, [x15, #0xa0]\n" ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - "mov v6.d[1], x20\n" + "mov v6.d[1], x21\n" ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - "ldr x20, [x15, #0xc8]\n" + "ldr x21, [x15, #0xc8]\n" ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n" ".inst 0x4f84e8f9 // sdot v25.4s, v7.16b, v4.4b[2]\n" ".inst 0x4f85e8fd // sdot v29.4s, v7.16b, v5.4b[2]\n" "ldr d7, [x15, #0xb0]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - "ldr x11, [x15, #0xd8]\n" + "ldr x20, [x15, #0xd8]\n" ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n" ".inst 0x4f84e8da // sdot v26.4s, v6.16b, v4.4b[2]\n" ".inst 0x4f85e8de // sdot v30.4s, v6.16b, v5.4b[2]\n" "ldr d6, [x15, #0xc0]\n" ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - "mov v6.d[1], x20\n" + "mov v6.d[1], x21\n" ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - "ldr x20, [x15, #0xe8]\n" + "ldr x21, [x15, #0xe8]\n" ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n" ".inst 0x4f84e8fb // sdot v27.4s, v7.16b, v4.4b[2]\n" ".inst 0x4f85e8ff // sdot v31.4s, v7.16b, v5.4b[2]\n" "ldr d7, [x15, #0xd0]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - "ldr x11, [x15, #0xf8]\n" + "ldr x20, [x15, #0xf8]\n" ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n" ".inst 0x4fa4e8d8 // sdot v24.4s, v6.16b, v4.4b[3]\n" ".inst 0x4fa5e8dc // sdot v28.4s, v6.16b, v5.4b[3]\n" "ldr d6, [x15, #0xe0]\n" ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - "mov v6.d[1], x20\n" + "mov v6.d[1], x21\n" ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - "ldr x22, [x23, #0x8]\n" + "ldr x22, [x28, #0x8]\n" ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n" ".inst 0x4fa4e8f9 // sdot v25.4s, v7.16b, v4.4b[3]\n" ".inst 0x4fa5e8fd // sdot v29.4s, v7.16b, v5.4b[3]\n" "ldr d7, [x15, #0xf0]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "add x15, x15, #0x100\n" ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" "ldr x20, [x15, #0x8]\n" @@ -3085,58 +3084,58 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" "ldr d0, [x12, #0x0]\n" ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" - "ldr d1, [x9, #0x0]\n" + "ldr d1, [x11, #0x0]\n" ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" - "ldr d2, [x27, #0x0]\n" + "ldr d2, [x10, #0x0]\n" ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n" - "ldr d3, [x25, #0x0]\n" + "ldr d3, [x9, #0x0]\n" ".inst 0x4fa4e8fb // sdot v27.4s, v7.16b, v4.4b[3]\n" - "ldr d4, [x23, #0x0]\n" + "ldr d4, [x28, #0x0]\n" ".inst 0x4fa5e8ff // sdot v31.4s, v7.16b, v5.4b[3]\n" - "ldr d5, [x21, #0x0]\n" + "ldr d5, [x27, #0x0]\n" "ldr d7, [x15, #0x10]\n" "mov v6.d[1], x20\n" - "ldr x20, [x21, #0x8]\n" - "mov v0.d[1], x10\n" - "ldr x11, [x15, #0x18]\n" - "mov v1.d[1], x28\n" - "mov v2.d[1], x26\n" - "mov v3.d[1], x24\n" + "ldr x21, [x27, #0x8]\n" + "mov v0.d[1], x26\n" + "ldr x20, [x15, #0x18]\n" + "mov v1.d[1], x25\n" + "mov v2.d[1], x24\n" + "mov v3.d[1], x23\n" "mov v4.d[1], x22\n" - "mov v5.d[1], x20\n" - "mov v7.d[1], x11\n" + "mov v5.d[1], x21\n" + "mov v7.d[1], x20\n" "bge 142b\n" "143:" // Height 6: Multiply loop: Single iteration only ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" "add x12, x12, #0x10\n" ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "add x9, x9, #0x10\n" + "add x11, x11, #0x10\n" ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - "add x27, x27, #0x10\n" + "add x10, x10, #0x10\n" ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "add x25, x25, #0x10\n" + "add x9, x9, #0x10\n" ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - "add x23, x23, #0x10\n" + "add x28, x28, #0x10\n" ".inst 0x4f85e0dc // sdot v28.4s, v6.16b, v5.4b[0]\n" "ldr q6, [x15, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "add x21, x21, #0x10\n" + "add x27, x27, #0x10\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" "sub x13, x13, #0x10\n" ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" "prfm pldl1keep, [x12, #0x80]\n" ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x11, #0x80]\n" ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - "prfm pldl1keep, [x27, #0x80]\n" + "prfm pldl1keep, [x10, #0x80]\n" ".inst 0x4f85e0fd // sdot v29.4s, v7.16b, v5.4b[0]\n" "ldr q7, [x15, #0x30]\n" ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - "prfm pldl1keep, [x25, #0x80]\n" + "prfm pldl1keep, [x9, #0x80]\n" ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "prfm pldl1keep, [x23, #0x80]\n" + "prfm pldl1keep, [x28, #0x80]\n" ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - "prfm pldl1keep, [x21, #0x80]\n" + "prfm pldl1keep, [x27, #0x80]\n" ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" ".inst 0x4f85e0de // sdot v30.4s, v6.16b, v5.4b[0]\n" @@ -3236,143 +3235,143 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "cmp x13, #0x4\n" "blt 146f\n" "145:" // Height 6: Multiply loop: Odd block loop - "ldr s0, [x12], #0x4\n" + "ldr s7, [x12], #0x4\n" "sub x13, x13, #0x4\n" - "ldr s1, [x9], #0x4\n" + "ldr s6, [x11], #0x4\n" "cmp x13, #0x4\n" + "ldr s5, [x10], #0x4\n" + "ldr s4, [x9], #0x4\n" + "ldr s3, [x28], #0x4\n" "ldr s2, [x27], #0x4\n" - "ldr s3, [x25], #0x4\n" - "ldr s4, [x23], #0x4\n" - "ldr s5, [x21], #0x4\n" - "ldr q6, [x15, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x15, #0x10]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - ".inst 0x4f85e0dc // sdot v28.4s, v6.16b, v5.4b[0]\n" - "ldr q6, [x15, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - ".inst 0x4f85e0fd // sdot v29.4s, v7.16b, v5.4b[0]\n" - "ldr q7, [x15, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q1, [x15, #0x0]\n" + ".inst 0x4f87e028 // sdot v8.4s, v1.16b, v7.4b[0]\n" + "ldr q0, [x15, #0x10]\n" + ".inst 0x4f86e02c // sdot v12.4s, v1.16b, v6.4b[0]\n" + ".inst 0x4f85e030 // sdot v16.4s, v1.16b, v5.4b[0]\n" + ".inst 0x4f84e034 // sdot v20.4s, v1.16b, v4.4b[0]\n" + ".inst 0x4f83e038 // sdot v24.4s, v1.16b, v3.4b[0]\n" + ".inst 0x4f82e03c // sdot v28.4s, v1.16b, v2.4b[0]\n" + "ldr q1, [x15, #0x20]\n" + ".inst 0x4f87e009 // sdot v9.4s, v0.16b, v7.4b[0]\n" + ".inst 0x4f86e00d // sdot v13.4s, v0.16b, v6.4b[0]\n" + ".inst 0x4f85e011 // sdot v17.4s, v0.16b, v5.4b[0]\n" + ".inst 0x4f84e015 // sdot v21.4s, v0.16b, v4.4b[0]\n" + ".inst 0x4f83e019 // sdot v25.4s, v0.16b, v3.4b[0]\n" + ".inst 0x4f82e01d // sdot v29.4s, v0.16b, v2.4b[0]\n" + "ldr q0, [x15, #0x30]\n" + ".inst 0x4f87e02a // sdot v10.4s, v1.16b, v7.4b[0]\n" "add x15, x15, #0x40\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - ".inst 0x4f85e0de // sdot v30.4s, v6.16b, v5.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" - ".inst 0x4f85e0ff // sdot v31.4s, v7.16b, v5.4b[0]\n" + ".inst 0x4f86e02e // sdot v14.4s, v1.16b, v6.4b[0]\n" + ".inst 0x4f85e032 // sdot v18.4s, v1.16b, v5.4b[0]\n" + ".inst 0x4f84e036 // sdot v22.4s, v1.16b, v4.4b[0]\n" + ".inst 0x4f83e03a // sdot v26.4s, v1.16b, v3.4b[0]\n" + ".inst 0x4f82e03e // sdot v30.4s, v1.16b, v2.4b[0]\n" + ".inst 0x4f87e00b // sdot v11.4s, v0.16b, v7.4b[0]\n" + ".inst 0x4f86e00f // sdot v15.4s, v0.16b, v6.4b[0]\n" + ".inst 0x4f85e013 // sdot v19.4s, v0.16b, v5.4b[0]\n" + ".inst 0x4f84e017 // sdot v23.4s, v0.16b, v4.4b[0]\n" + ".inst 0x4f83e01b // sdot v27.4s, v0.16b, v3.4b[0]\n" + ".inst 0x4f82e01f // sdot v31.4s, v0.16b, v2.4b[0]\n" "bge 145b\n" "146:" // Height 6: Multiply loop: Skip odd blocks "cbz x13, 149f\n" "tbz x13, #1, 147f\n" "ldr h0, [x12], #0x2\n" - "ldr h1, [x9], #0x2\n" - "ldr h2, [x27], #0x2\n" - "ldr h3, [x25], #0x2\n" - "ldr h4, [x23], #0x2\n" - "ldr h5, [x21], #0x2\n" + "ldr h1, [x11], #0x2\n" + "ldr h2, [x10], #0x2\n" + "ldr h3, [x9], #0x2\n" + "ldr h4, [x28], #0x2\n" + "ldr h5, [x27], #0x2\n" "tbz x13, #0, 148f\n" "ld1 { v0.b }[2], [x12]\n" - "ld1 { v1.b }[2], [x9]\n" - "ld1 { v2.b }[2], [x27]\n" - "ld1 { v3.b }[2], [x25]\n" - "ld1 { v4.b }[2], [x23]\n" - "ld1 { v5.b }[2], [x21]\n" + "ld1 { v1.b }[2], [x11]\n" + "ld1 { v2.b }[2], [x10]\n" + "ld1 { v3.b }[2], [x9]\n" + "ld1 { v4.b }[2], [x28]\n" + "ld1 { v5.b }[2], [x27]\n" "b 148f\n" "147:" // Height 6: Multiply loop: Ragged operand read: partial_1_0 "ldr b0, [x12, #0x0]\n" - "ldr b1, [x9, #0x0]\n" - "ldr b2, [x27, #0x0]\n" - "ldr b3, [x25, #0x0]\n" - "ldr b4, [x23, #0x0]\n" - "ldr b5, [x21, #0x0]\n" + "ldr b1, [x11, #0x0]\n" + "ldr b2, [x10, #0x0]\n" + "ldr b3, [x9, #0x0]\n" + "ldr b4, [x28, #0x0]\n" + "ldr b5, [x27, #0x0]\n" "148:" // Height 6: Multiply loop: Ragged operand read: Done - "ldr q6, [x15, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x15, #0x10]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - ".inst 0x4f85e0dc // sdot v28.4s, v6.16b, v5.4b[0]\n" - "ldr q6, [x15, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - ".inst 0x4f85e0fd // sdot v29.4s, v7.16b, v5.4b[0]\n" - "ldr q7, [x15, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q7, [x15, #0x0]\n" + ".inst 0x4f80e0e8 // sdot v8.4s, v7.16b, v0.4b[0]\n" + "ldr q6, [x15, #0x10]\n" + ".inst 0x4f81e0ec // sdot v12.4s, v7.16b, v1.4b[0]\n" + ".inst 0x4f82e0f0 // sdot v16.4s, v7.16b, v2.4b[0]\n" + ".inst 0x4f83e0f4 // sdot v20.4s, v7.16b, v3.4b[0]\n" + ".inst 0x4f84e0f8 // sdot v24.4s, v7.16b, v4.4b[0]\n" + ".inst 0x4f85e0fc // sdot v28.4s, v7.16b, v5.4b[0]\n" + "ldr q7, [x15, #0x20]\n" + ".inst 0x4f80e0c9 // sdot v9.4s, v6.16b, v0.4b[0]\n" + ".inst 0x4f81e0cd // sdot v13.4s, v6.16b, v1.4b[0]\n" + ".inst 0x4f82e0d1 // sdot v17.4s, v6.16b, v2.4b[0]\n" + ".inst 0x4f83e0d5 // sdot v21.4s, v6.16b, v3.4b[0]\n" + ".inst 0x4f84e0d9 // sdot v25.4s, v6.16b, v4.4b[0]\n" + ".inst 0x4f85e0dd // sdot v29.4s, v6.16b, v5.4b[0]\n" + "ldr q6, [x15, #0x30]\n" + ".inst 0x4f80e0ea // sdot v10.4s, v7.16b, v0.4b[0]\n" "add x15, x15, #0x40\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - ".inst 0x4f85e0de // sdot v30.4s, v6.16b, v5.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" - ".inst 0x4f85e0ff // sdot v31.4s, v7.16b, v5.4b[0]\n" + ".inst 0x4f81e0ee // sdot v14.4s, v7.16b, v1.4b[0]\n" + ".inst 0x4f82e0f2 // sdot v18.4s, v7.16b, v2.4b[0]\n" + ".inst 0x4f83e0f6 // sdot v22.4s, v7.16b, v3.4b[0]\n" + ".inst 0x4f84e0fa // sdot v26.4s, v7.16b, v4.4b[0]\n" + ".inst 0x4f85e0fe // sdot v30.4s, v7.16b, v5.4b[0]\n" + ".inst 0x4f80e0cb // sdot v11.4s, v6.16b, v0.4b[0]\n" + ".inst 0x4f81e0cf // sdot v15.4s, v6.16b, v1.4b[0]\n" + ".inst 0x4f82e0d3 // sdot v19.4s, v6.16b, v2.4b[0]\n" + ".inst 0x4f83e0d7 // sdot v23.4s, v6.16b, v3.4b[0]\n" + ".inst 0x4f84e0db // sdot v27.4s, v6.16b, v4.4b[0]\n" + ".inst 0x4f85e0df // sdot v31.4s, v6.16b, v5.4b[0]\n" "149:" // Height 6: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x14, x14, #0x1\n" "cmp x14, x20\n" "bne 139b\n" - "ldr q0, [x6, #0x0]\n" - "add v8.4s, v8.4s, v0.4s\n" - "ldr q1, [x6, #0x10]\n" - "add v9.4s, v9.4s, v1.4s\n" - "ldr q2, [x6, #0x20]\n" - "add v10.4s, v10.4s, v2.4s\n" - "ldr q3, [x6, #0x30]\n" - "add v11.4s, v11.4s, v3.4s\n" + "ldr q3, [x6, #0x0]\n" + "add v8.4s, v8.4s, v3.4s\n" + "ldr q2, [x6, #0x10]\n" + "add v9.4s, v9.4s, v2.4s\n" + "ldr q1, [x6, #0x20]\n" + "add v10.4s, v10.4s, v1.4s\n" + "ldr q0, [x6, #0x30]\n" + "add v11.4s, v11.4s, v0.4s\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x17, x20\n" + "add x25, x17, x20\n" + "add x24, x25, x20\n" "add x23, x24, x20\n" "add x22, x23, x20\n" "add x21, x22, x20\n" - "add x20, x21, x20\n" "prfm pstl1keep, [x17, #0x0]\n" - "add v12.4s, v12.4s, v0.4s\n" + "add v12.4s, v12.4s, v3.4s\n" + "prfm pstl1keep, [x25, #0x0]\n" + "add v13.4s, v13.4s, v2.4s\n" "prfm pstl1keep, [x24, #0x0]\n" - "add v13.4s, v13.4s, v1.4s\n" + "add v14.4s, v14.4s, v1.4s\n" "prfm pstl1keep, [x23, #0x0]\n" - "add v14.4s, v14.4s, v2.4s\n" + "add v15.4s, v15.4s, v0.4s\n" "prfm pstl1keep, [x22, #0x0]\n" - "add v15.4s, v15.4s, v3.4s\n" + "add v16.4s, v16.4s, v3.4s\n" "prfm pstl1keep, [x21, #0x0]\n" - "add v16.4s, v16.4s, v0.4s\n" - "prfm pstl1keep, [x20, #0x0]\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" - "add v20.4s, v20.4s, v0.4s\n" - "add v21.4s, v21.4s, v1.4s\n" - "add v22.4s, v22.4s, v2.4s\n" - "add v23.4s, v23.4s, v3.4s\n" - "add v24.4s, v24.4s, v0.4s\n" - "add v25.4s, v25.4s, v1.4s\n" - "add v26.4s, v26.4s, v2.4s\n" - "add v27.4s, v27.4s, v3.4s\n" - "add v28.4s, v28.4s, v0.4s\n" - "add v29.4s, v29.4s, v1.4s\n" - "add v30.4s, v30.4s, v2.4s\n" - "add v31.4s, v31.4s, v3.4s\n" + "add v17.4s, v17.4s, v2.4s\n" + "add v18.4s, v18.4s, v1.4s\n" + "add v19.4s, v19.4s, v0.4s\n" + "add v20.4s, v20.4s, v3.4s\n" + "add v21.4s, v21.4s, v2.4s\n" + "add v22.4s, v22.4s, v1.4s\n" + "add v23.4s, v23.4s, v0.4s\n" + "add v24.4s, v24.4s, v3.4s\n" + "add v25.4s, v25.4s, v2.4s\n" + "add v26.4s, v26.4s, v1.4s\n" + "add v27.4s, v27.4s, v0.4s\n" + "add v28.4s, v28.4s, v3.4s\n" + "add v29.4s, v29.4s, v2.4s\n" + "add v30.4s, v30.4s, v1.4s\n" + "add v31.4s, v31.4s, v0.4s\n" "add x6, x6, #0x40\n" "tbz %x[flags], #4, 150f\n" "ldr q0, [x8, #0x0]\n" @@ -3387,10 +3386,10 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "add x7, x7, #0x40\n" "b 151f\n" "150:" // Height 6: per layer parameters - "add x25, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x25]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v4.4s }, [x20]\n" "mov v1.16b, v0.16b\n" "mov v5.16b, v4.16b\n" "mov v2.16b, v0.16b\n" @@ -3423,78 +3422,78 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "sqrdmulh v30.4s, v30.4s, v6.4s\n" "sqrdmulh v31.4s, v31.4s, v7.4s\n" "tbz %x[flags], #5, 152f\n" - "and v4.16b, v8.16b, v0.16b\n" - "and v5.16b, v9.16b, v1.16b\n" - "and v6.16b, v10.16b, v2.16b\n" - "and v7.16b, v11.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" + "and v7.16b, v8.16b, v0.16b\n" + "and v6.16b, v9.16b, v1.16b\n" + "and v5.16b, v10.16b, v2.16b\n" + "and v4.16b, v11.16b, v3.16b\n" "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v4.4s\n" - "sqadd v9.4s, v9.4s, v5.4s\n" - "sqadd v10.4s, v10.4s, v6.4s\n" - "sqadd v11.4s, v11.4s, v7.4s\n" - "and v4.16b, v12.16b, v0.16b\n" - "and v5.16b, v13.16b, v1.16b\n" - "and v6.16b, v14.16b, v2.16b\n" - "and v7.16b, v15.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v12.4s, v12.4s, v4.4s\n" - "sqadd v13.4s, v13.4s, v5.4s\n" - "sqadd v14.4s, v14.4s, v6.4s\n" - "sqadd v15.4s, v15.4s, v7.4s\n" - "and v4.16b, v16.16b, v0.16b\n" - "and v5.16b, v17.16b, v1.16b\n" - "and v6.16b, v18.16b, v2.16b\n" - "and v7.16b, v19.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" - "and v4.16b, v20.16b, v0.16b\n" - "and v5.16b, v21.16b, v1.16b\n" - "and v6.16b, v22.16b, v2.16b\n" - "and v7.16b, v23.16b, v3.16b\n" "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" + "sqadd v8.4s, v8.4s, v7.4s\n" + "sqadd v9.4s, v9.4s, v6.4s\n" + "sqadd v10.4s, v10.4s, v5.4s\n" + "sqadd v11.4s, v11.4s, v4.4s\n" + "and v7.16b, v12.16b, v0.16b\n" + "and v6.16b, v13.16b, v1.16b\n" + "and v5.16b, v14.16b, v2.16b\n" + "and v4.16b, v15.16b, v3.16b\n" "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v20.4s, v20.4s, v4.4s\n" - "sqadd v21.4s, v21.4s, v5.4s\n" - "sqadd v22.4s, v22.4s, v6.4s\n" - "sqadd v23.4s, v23.4s, v7.4s\n" - "and v4.16b, v24.16b, v0.16b\n" - "and v5.16b, v25.16b, v1.16b\n" - "and v6.16b, v26.16b, v2.16b\n" - "and v7.16b, v27.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" "sshr v6.4s, v6.4s, #0x1f\n" + "sshr v5.4s, v5.4s, #0x1f\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sqadd v12.4s, v12.4s, v7.4s\n" + "sqadd v13.4s, v13.4s, v6.4s\n" + "sqadd v14.4s, v14.4s, v5.4s\n" + "sqadd v15.4s, v15.4s, v4.4s\n" + "and v7.16b, v16.16b, v0.16b\n" + "and v6.16b, v17.16b, v1.16b\n" + "and v5.16b, v18.16b, v2.16b\n" + "and v4.16b, v19.16b, v3.16b\n" "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v24.4s, v24.4s, v4.4s\n" - "sqadd v25.4s, v25.4s, v5.4s\n" - "sqadd v26.4s, v26.4s, v6.4s\n" - "sqadd v27.4s, v27.4s, v7.4s\n" - "and v4.16b, v28.16b, v0.16b\n" - "and v5.16b, v29.16b, v1.16b\n" - "and v6.16b, v30.16b, v2.16b\n" - "and v7.16b, v31.16b, v3.16b\n" + "sshr v6.4s, v6.4s, #0x1f\n" + "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v7.4s\n" + "sqadd v17.4s, v17.4s, v6.4s\n" + "sqadd v18.4s, v18.4s, v5.4s\n" + "sqadd v19.4s, v19.4s, v4.4s\n" + "and v7.16b, v20.16b, v0.16b\n" + "and v6.16b, v21.16b, v1.16b\n" + "and v5.16b, v22.16b, v2.16b\n" + "and v4.16b, v23.16b, v3.16b\n" + "sshr v7.4s, v7.4s, #0x1f\n" + "sshr v6.4s, v6.4s, #0x1f\n" "sshr v5.4s, v5.4s, #0x1f\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v7.4s\n" + "sqadd v21.4s, v21.4s, v6.4s\n" + "sqadd v22.4s, v22.4s, v5.4s\n" + "sqadd v23.4s, v23.4s, v4.4s\n" + "and v7.16b, v24.16b, v0.16b\n" + "and v6.16b, v25.16b, v1.16b\n" + "and v5.16b, v26.16b, v2.16b\n" + "and v4.16b, v27.16b, v3.16b\n" + "sshr v7.4s, v7.4s, #0x1f\n" "sshr v6.4s, v6.4s, #0x1f\n" + "sshr v5.4s, v5.4s, #0x1f\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sqadd v24.4s, v24.4s, v7.4s\n" + "sqadd v25.4s, v25.4s, v6.4s\n" + "sqadd v26.4s, v26.4s, v5.4s\n" + "sqadd v27.4s, v27.4s, v4.4s\n" + "and v7.16b, v28.16b, v0.16b\n" + "and v6.16b, v29.16b, v1.16b\n" + "and v5.16b, v30.16b, v2.16b\n" + "and v4.16b, v31.16b, v3.16b\n" "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v28.4s, v28.4s, v4.4s\n" - "sqadd v29.4s, v29.4s, v5.4s\n" - "sqadd v30.4s, v30.4s, v6.4s\n" - "sqadd v31.4s, v31.4s, v7.4s\n" + "sshr v6.4s, v6.4s, #0x1f\n" + "sshr v5.4s, v5.4s, #0x1f\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sqadd v28.4s, v28.4s, v7.4s\n" + "sqadd v29.4s, v29.4s, v6.4s\n" + "sqadd v30.4s, v30.4s, v5.4s\n" + "sqadd v31.4s, v31.4s, v4.4s\n" "152:" // Height 6: no shift correction "srshl v8.4s, v8.4s, v0.4s\n" "srshl v9.4s, v9.4s, v1.4s\n" @@ -3520,232 +3519,232 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "srshl v29.4s, v29.4s, v1.4s\n" "srshl v30.4s, v30.4s, v2.4s\n" "srshl v31.4s, v31.4s, v3.4s\n" - "add x25, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x25]\n" - "add v8.4s, v8.4s, v4.4s\n" - "add v9.4s, v9.4s, v4.4s\n" - "add v10.4s, v10.4s, v4.4s\n" - "add v11.4s, v11.4s, v4.4s\n" - "add v12.4s, v12.4s, v4.4s\n" - "add v13.4s, v13.4s, v4.4s\n" - "add v14.4s, v14.4s, v4.4s\n" - "add v15.4s, v15.4s, v4.4s\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v23.4s, v23.4s, v4.4s\n" - "add v24.4s, v24.4s, v4.4s\n" - "add v25.4s, v25.4s, v4.4s\n" - "add v26.4s, v26.4s, v4.4s\n" - "add v27.4s, v27.4s, v4.4s\n" - "add v28.4s, v28.4s, v4.4s\n" - "add v29.4s, v29.4s, v4.4s\n" - "add v30.4s, v30.4s, v4.4s\n" - "add v31.4s, v31.4s, v4.4s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x25]\n" - "smin v8.4s, v8.4s, v6.4s\n" - "smin v9.4s, v9.4s, v6.4s\n" - "smin v10.4s, v10.4s, v6.4s\n" - "smin v11.4s, v11.4s, v6.4s\n" - "smin v12.4s, v12.4s, v6.4s\n" - "smin v13.4s, v13.4s, v6.4s\n" - "smin v14.4s, v14.4s, v6.4s\n" - "smin v15.4s, v15.4s, v6.4s\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "smin v24.4s, v24.4s, v6.4s\n" - "smin v25.4s, v25.4s, v6.4s\n" - "smin v26.4s, v26.4s, v6.4s\n" - "smin v27.4s, v27.4s, v6.4s\n" - "smin v28.4s, v28.4s, v6.4s\n" - "smin v29.4s, v29.4s, v6.4s\n" - "smin v30.4s, v30.4s, v6.4s\n" - "smin v31.4s, v31.4s, v6.4s\n" - "add x25, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x25]\n" - "smax v8.4s, v8.4s, v5.4s\n" - "smax v9.4s, v9.4s, v5.4s\n" - "smax v10.4s, v10.4s, v5.4s\n" - "smax v11.4s, v11.4s, v5.4s\n" - "smax v12.4s, v12.4s, v5.4s\n" - "smax v13.4s, v13.4s, v5.4s\n" - "smax v14.4s, v14.4s, v5.4s\n" - "smax v15.4s, v15.4s, v5.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" - "smax v24.4s, v24.4s, v5.4s\n" - "smax v25.4s, v25.4s, v5.4s\n" - "smax v26.4s, v26.4s, v5.4s\n" - "smax v27.4s, v27.4s, v5.4s\n" - "smax v28.4s, v28.4s, v5.4s\n" - "smax v29.4s, v29.4s, v5.4s\n" - "smax v30.4s, v30.4s, v5.4s\n" - "smax v31.4s, v31.4s, v5.4s\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v0.4s }, [x20]\n" + "add v8.4s, v8.4s, v0.4s\n" + "add v9.4s, v9.4s, v0.4s\n" + "add v10.4s, v10.4s, v0.4s\n" + "add v11.4s, v11.4s, v0.4s\n" + "add v12.4s, v12.4s, v0.4s\n" + "add v13.4s, v13.4s, v0.4s\n" + "add v14.4s, v14.4s, v0.4s\n" + "add v15.4s, v15.4s, v0.4s\n" + "add v16.4s, v16.4s, v0.4s\n" + "add v17.4s, v17.4s, v0.4s\n" + "add v18.4s, v18.4s, v0.4s\n" + "add v19.4s, v19.4s, v0.4s\n" + "add v20.4s, v20.4s, v0.4s\n" + "add v21.4s, v21.4s, v0.4s\n" + "add v22.4s, v22.4s, v0.4s\n" + "add v23.4s, v23.4s, v0.4s\n" + "add v24.4s, v24.4s, v0.4s\n" + "add v25.4s, v25.4s, v0.4s\n" + "add v26.4s, v26.4s, v0.4s\n" + "add v27.4s, v27.4s, v0.4s\n" + "add v28.4s, v28.4s, v0.4s\n" + "add v29.4s, v29.4s, v0.4s\n" + "add v30.4s, v30.4s, v0.4s\n" + "add v31.4s, v31.4s, v0.4s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v0.4s }, [x20]\n" + "smin v8.4s, v8.4s, v0.4s\n" + "smin v9.4s, v9.4s, v0.4s\n" + "smin v10.4s, v10.4s, v0.4s\n" + "smin v11.4s, v11.4s, v0.4s\n" + "smin v12.4s, v12.4s, v0.4s\n" + "smin v13.4s, v13.4s, v0.4s\n" + "smin v14.4s, v14.4s, v0.4s\n" + "smin v15.4s, v15.4s, v0.4s\n" + "smin v16.4s, v16.4s, v0.4s\n" + "smin v17.4s, v17.4s, v0.4s\n" + "smin v18.4s, v18.4s, v0.4s\n" + "smin v19.4s, v19.4s, v0.4s\n" + "smin v20.4s, v20.4s, v0.4s\n" + "smin v21.4s, v21.4s, v0.4s\n" + "smin v22.4s, v22.4s, v0.4s\n" + "smin v23.4s, v23.4s, v0.4s\n" + "smin v24.4s, v24.4s, v0.4s\n" + "smin v25.4s, v25.4s, v0.4s\n" + "smin v26.4s, v26.4s, v0.4s\n" + "smin v27.4s, v27.4s, v0.4s\n" + "smin v28.4s, v28.4s, v0.4s\n" + "smin v29.4s, v29.4s, v0.4s\n" + "smin v30.4s, v30.4s, v0.4s\n" + "smin v31.4s, v31.4s, v0.4s\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v0.4s }, [x20]\n" + "smax v8.4s, v8.4s, v0.4s\n" + "smax v9.4s, v9.4s, v0.4s\n" + "smax v10.4s, v10.4s, v0.4s\n" + "smax v11.4s, v11.4s, v0.4s\n" + "smax v12.4s, v12.4s, v0.4s\n" + "smax v13.4s, v13.4s, v0.4s\n" + "smax v14.4s, v14.4s, v0.4s\n" + "smax v15.4s, v15.4s, v0.4s\n" + "smax v16.4s, v16.4s, v0.4s\n" + "smax v17.4s, v17.4s, v0.4s\n" + "smax v18.4s, v18.4s, v0.4s\n" + "smax v19.4s, v19.4s, v0.4s\n" + "smax v20.4s, v20.4s, v0.4s\n" + "smax v21.4s, v21.4s, v0.4s\n" + "smax v22.4s, v22.4s, v0.4s\n" + "smax v23.4s, v23.4s, v0.4s\n" + "smax v24.4s, v24.4s, v0.4s\n" + "smax v25.4s, v25.4s, v0.4s\n" + "smax v26.4s, v26.4s, v0.4s\n" + "smax v27.4s, v27.4s, v0.4s\n" + "smax v28.4s, v28.4s, v0.4s\n" + "smax v29.4s, v29.4s, v0.4s\n" + "smax v30.4s, v30.4s, v0.4s\n" + "smax v31.4s, v31.4s, v0.4s\n" "uzp1 v8.8h, v8.8h, v9.8h\n" - "uzp1 v9.8h, v10.8h, v11.8h\n" + "uzp1 v2.8h, v10.8h, v11.8h\n" "uzp1 v12.8h, v12.8h, v13.8h\n" - "uzp1 v13.8h, v14.8h, v15.8h\n" + "uzp1 v1.8h, v14.8h, v15.8h\n" "uzp1 v16.8h, v16.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v0.8h, v18.8h, v19.8h\n" "uzp1 v20.8h, v20.8h, v21.8h\n" - "uzp1 v21.8h, v22.8h, v23.8h\n" + "uzp1 v19.8h, v22.8h, v23.8h\n" "uzp1 v24.8h, v24.8h, v25.8h\n" - "uzp1 v25.8h, v26.8h, v27.8h\n" + "uzp1 v18.8h, v26.8h, v27.8h\n" "uzp1 v28.8h, v28.8h, v29.8h\n" - "uzp1 v29.8h, v30.8h, v31.8h\n" + "uzp1 v17.8h, v30.8h, v31.8h\n" "cmp x16, #0x10\n" - "uzp1 v8.16b, v8.16b, v9.16b\n" - "uzp1 v12.16b, v12.16b, v13.16b\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" - "uzp1 v20.16b, v20.16b, v21.16b\n" - "uzp1 v24.16b, v24.16b, v25.16b\n" - "uzp1 v28.16b, v28.16b, v29.16b\n" + "uzp1 v8.16b, v8.16b, v2.16b\n" + "uzp1 v12.16b, v12.16b, v1.16b\n" + "uzp1 v16.16b, v16.16b, v0.16b\n" + "uzp1 v20.16b, v20.16b, v19.16b\n" + "uzp1 v24.16b, v24.16b, v18.16b\n" + "uzp1 v28.16b, v28.16b, v17.16b\n" "bge 161f\n" "tbz x16, #3, 156f\n" "str d8, [x17], #0x8\n" - "str d12, [x24], #0x8\n" - "str d16, [x23], #0x8\n" - "str d20, [x22], #0x8\n" - "str d24, [x21], #0x8\n" - "str d28, [x20], #0x8\n" + "str d12, [x25], #0x8\n" + "str d16, [x24], #0x8\n" + "str d20, [x23], #0x8\n" + "str d24, [x22], #0x8\n" + "str d28, [x21], #0x8\n" "tbz x16, #2, 154f\n" "st1 { v8.s }[2], [x17], #0x4\n" - "st1 { v12.s }[2], [x24], #0x4\n" - "st1 { v16.s }[2], [x23], #0x4\n" - "st1 { v20.s }[2], [x22], #0x4\n" - "st1 { v24.s }[2], [x21], #0x4\n" - "st1 { v28.s }[2], [x20], #0x4\n" + "st1 { v12.s }[2], [x25], #0x4\n" + "st1 { v16.s }[2], [x24], #0x4\n" + "st1 { v20.s }[2], [x23], #0x4\n" + "st1 { v24.s }[2], [x22], #0x4\n" + "st1 { v28.s }[2], [x21], #0x4\n" "tbz x16, #1, 153f\n" "st1 { v8.h }[6], [x17], #0x2\n" - "st1 { v12.h }[6], [x24], #0x2\n" - "st1 { v16.h }[6], [x23], #0x2\n" - "st1 { v20.h }[6], [x22], #0x2\n" - "st1 { v24.h }[6], [x21], #0x2\n" - "st1 { v28.h }[6], [x20], #0x2\n" + "st1 { v12.h }[6], [x25], #0x2\n" + "st1 { v16.h }[6], [x24], #0x2\n" + "st1 { v20.h }[6], [x23], #0x2\n" + "st1 { v24.h }[6], [x22], #0x2\n" + "st1 { v28.h }[6], [x21], #0x2\n" "tbz x16, #0, 160f\n" "st1 { v8.b }[14], [x17]\n" - "st1 { v12.b }[14], [x24]\n" - "st1 { v16.b }[14], [x23]\n" - "st1 { v20.b }[14], [x22]\n" - "st1 { v24.b }[14], [x21]\n" - "st1 { v28.b }[14], [x20]\n" + "st1 { v12.b }[14], [x25]\n" + "st1 { v16.b }[14], [x24]\n" + "st1 { v20.b }[14], [x23]\n" + "st1 { v24.b }[14], [x22]\n" + "st1 { v28.b }[14], [x21]\n" "b 160f\n" "153:" // Height 6: Partial direct writeback: partial_1_12 "tbz x16, #0, 160f\n" "st1 { v8.b }[12], [x17]\n" - "st1 { v12.b }[12], [x24]\n" - "st1 { v16.b }[12], [x23]\n" - "st1 { v20.b }[12], [x22]\n" - "st1 { v24.b }[12], [x21]\n" - "st1 { v28.b }[12], [x20]\n" + "st1 { v12.b }[12], [x25]\n" + "st1 { v16.b }[12], [x24]\n" + "st1 { v20.b }[12], [x23]\n" + "st1 { v24.b }[12], [x22]\n" + "st1 { v28.b }[12], [x21]\n" "b 160f\n" "154:" // Height 6: Partial direct writeback: partial_2_8 "tbz x16, #1, 155f\n" "st1 { v8.h }[4], [x17], #0x2\n" - "st1 { v12.h }[4], [x24], #0x2\n" - "st1 { v16.h }[4], [x23], #0x2\n" - "st1 { v20.h }[4], [x22], #0x2\n" - "st1 { v24.h }[4], [x21], #0x2\n" - "st1 { v28.h }[4], [x20], #0x2\n" + "st1 { v12.h }[4], [x25], #0x2\n" + "st1 { v16.h }[4], [x24], #0x2\n" + "st1 { v20.h }[4], [x23], #0x2\n" + "st1 { v24.h }[4], [x22], #0x2\n" + "st1 { v28.h }[4], [x21], #0x2\n" "tbz x16, #0, 160f\n" "st1 { v8.b }[10], [x17]\n" - "st1 { v12.b }[10], [x24]\n" - "st1 { v16.b }[10], [x23]\n" - "st1 { v20.b }[10], [x22]\n" - "st1 { v24.b }[10], [x21]\n" - "st1 { v28.b }[10], [x20]\n" + "st1 { v12.b }[10], [x25]\n" + "st1 { v16.b }[10], [x24]\n" + "st1 { v20.b }[10], [x23]\n" + "st1 { v24.b }[10], [x22]\n" + "st1 { v28.b }[10], [x21]\n" "b 160f\n" "155:" // Height 6: Partial direct writeback: partial_1_8 "tbz x16, #0, 160f\n" "st1 { v8.b }[8], [x17]\n" - "st1 { v12.b }[8], [x24]\n" - "st1 { v16.b }[8], [x23]\n" - "st1 { v20.b }[8], [x22]\n" - "st1 { v24.b }[8], [x21]\n" - "st1 { v28.b }[8], [x20]\n" + "st1 { v12.b }[8], [x25]\n" + "st1 { v16.b }[8], [x24]\n" + "st1 { v20.b }[8], [x23]\n" + "st1 { v24.b }[8], [x22]\n" + "st1 { v28.b }[8], [x21]\n" "b 160f\n" "156:" // Height 6: Partial direct writeback: partial_4_0 "tbz x16, #2, 158f\n" "str s8, [x17], #0x4\n" - "str s12, [x24], #0x4\n" - "str s16, [x23], #0x4\n" - "str s20, [x22], #0x4\n" - "str s24, [x21], #0x4\n" - "str s28, [x20], #0x4\n" + "str s12, [x25], #0x4\n" + "str s16, [x24], #0x4\n" + "str s20, [x23], #0x4\n" + "str s24, [x22], #0x4\n" + "str s28, [x21], #0x4\n" "tbz x16, #1, 157f\n" "st1 { v8.h }[2], [x17], #0x2\n" - "st1 { v12.h }[2], [x24], #0x2\n" - "st1 { v16.h }[2], [x23], #0x2\n" - "st1 { v20.h }[2], [x22], #0x2\n" - "st1 { v24.h }[2], [x21], #0x2\n" - "st1 { v28.h }[2], [x20], #0x2\n" + "st1 { v12.h }[2], [x25], #0x2\n" + "st1 { v16.h }[2], [x24], #0x2\n" + "st1 { v20.h }[2], [x23], #0x2\n" + "st1 { v24.h }[2], [x22], #0x2\n" + "st1 { v28.h }[2], [x21], #0x2\n" "tbz x16, #0, 160f\n" "st1 { v8.b }[6], [x17]\n" - "st1 { v12.b }[6], [x24]\n" - "st1 { v16.b }[6], [x23]\n" - "st1 { v20.b }[6], [x22]\n" - "st1 { v24.b }[6], [x21]\n" - "st1 { v28.b }[6], [x20]\n" + "st1 { v12.b }[6], [x25]\n" + "st1 { v16.b }[6], [x24]\n" + "st1 { v20.b }[6], [x23]\n" + "st1 { v24.b }[6], [x22]\n" + "st1 { v28.b }[6], [x21]\n" "b 160f\n" "157:" // Height 6: Partial direct writeback: partial_1_4 "tbz x16, #0, 160f\n" "st1 { v8.b }[4], [x17]\n" - "st1 { v12.b }[4], [x24]\n" - "st1 { v16.b }[4], [x23]\n" - "st1 { v20.b }[4], [x22]\n" - "st1 { v24.b }[4], [x21]\n" - "st1 { v28.b }[4], [x20]\n" + "st1 { v12.b }[4], [x25]\n" + "st1 { v16.b }[4], [x24]\n" + "st1 { v20.b }[4], [x23]\n" + "st1 { v24.b }[4], [x22]\n" + "st1 { v28.b }[4], [x21]\n" "b 160f\n" "158:" // Height 6: Partial direct writeback: partial_2_0 "tbz x16, #1, 159f\n" "str h8, [x17], #0x2\n" - "str h12, [x24], #0x2\n" - "str h16, [x23], #0x2\n" - "str h20, [x22], #0x2\n" - "str h24, [x21], #0x2\n" - "str h28, [x20], #0x2\n" + "str h12, [x25], #0x2\n" + "str h16, [x24], #0x2\n" + "str h20, [x23], #0x2\n" + "str h24, [x22], #0x2\n" + "str h28, [x21], #0x2\n" "tbz x16, #0, 160f\n" "st1 { v8.b }[2], [x17]\n" - "st1 { v12.b }[2], [x24]\n" - "st1 { v16.b }[2], [x23]\n" - "st1 { v20.b }[2], [x22]\n" - "st1 { v24.b }[2], [x21]\n" - "st1 { v28.b }[2], [x20]\n" + "st1 { v12.b }[2], [x25]\n" + "st1 { v16.b }[2], [x24]\n" + "st1 { v20.b }[2], [x23]\n" + "st1 { v24.b }[2], [x22]\n" + "st1 { v28.b }[2], [x21]\n" "b 160f\n" "159:" // Height 6: Partial direct writeback: partial_1_0 "str b8, [x17, #0x0]\n" - "str b12, [x24, #0x0]\n" - "str b16, [x23, #0x0]\n" - "str b20, [x22, #0x0]\n" - "str b24, [x21, #0x0]\n" - "str b28, [x20, #0x0]\n" + "str b12, [x25, #0x0]\n" + "str b16, [x24, #0x0]\n" + "str b20, [x23, #0x0]\n" + "str b24, [x22, #0x0]\n" + "str b28, [x21, #0x0]\n" "160:" // Height 6: Partial direct writeback: Done "b 162f\n" "161:" // Height 6: Full writeback "str q8, [x17, #0x0]\n" "add x17, x17, #0x10\n" - "str q12, [x24, #0x0]\n" - "str q16, [x23, #0x0]\n" - "str q20, [x22, #0x0]\n" - "str q24, [x21, #0x0]\n" - "str q28, [x20, #0x0]\n" + "str q12, [x25, #0x0]\n" + "str q16, [x24, #0x0]\n" + "str q20, [x23, #0x0]\n" + "str q24, [x22, #0x0]\n" + "str q28, [x21, #0x0]\n" "162:" // Height 6: Writeback done "subs x16, x16, #0x10\n" "bgt 137b\n" @@ -3761,7 +3760,6 @@ void a64_hybrid_s8qs_dot_6x16_a55 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "164:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [flags] "r" (flags), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_multiplier_ptr] "I" (offsetof(KernelArgs, multiplier_ptr)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_shift_ptr] "I" (offsetof(KernelArgs, shift_ptr)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16/generic.cpp index 598d1524e83e6072a7cdfc16f30bdfa4276a25d9..f3942328a6bba95a0f31a6182312f3fb5425367d 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_dot_6x16/generic.cpp @@ -85,7 +85,6 @@ void a64_hybrid_s8qs_dot_6x16 ( flags |= 0x20; } __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x6\n" "bge 136f\n" @@ -111,11 +110,11 @@ void a64_hybrid_s8qs_dot_6x16 ( "4:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 5f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 6f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -132,37 +131,37 @@ void a64_hybrid_s8qs_dot_6x16 ( "blt 8f\n" "7:" // Height 1: Multiply loop: Main loop head ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q6, [x9, #0x20]\n" + "ldr q17, [x9, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "ldr q7, [x9, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - "ldr q6, [x9, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - "ldr q7, [x9, #0x50]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - "ldr q6, [x9, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - "ldr q7, [x9, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - "ldr q6, [x9, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - "ldr q7, [x9, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - "ldr q6, [x9, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - "ldr q7, [x9, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - "ldr q6, [x9, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - "ldr q7, [x9, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - "ldr q6, [x9, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - "ldr q7, [x9, #0xf0]\n" + "ldr q16, [x9, #0x30]\n" + ".inst 0x4f80e22a // sdot v10.4s, v17.16b, v0.4b[0]\n" + "ldr q17, [x9, #0x40]\n" + ".inst 0x4f80e20b // sdot v11.4s, v16.16b, v0.4b[0]\n" + "ldr q16, [x9, #0x50]\n" + ".inst 0x4fa0e228 // sdot v8.4s, v17.16b, v0.4b[1]\n" + "ldr q17, [x9, #0x60]\n" + ".inst 0x4fa0e209 // sdot v9.4s, v16.16b, v0.4b[1]\n" + "ldr q16, [x9, #0x70]\n" + ".inst 0x4fa0e22a // sdot v10.4s, v17.16b, v0.4b[1]\n" + "ldr q17, [x9, #0x80]\n" + ".inst 0x4fa0e20b // sdot v11.4s, v16.16b, v0.4b[1]\n" + "ldr q16, [x9, #0x90]\n" + ".inst 0x4f80ea28 // sdot v8.4s, v17.16b, v0.4b[2]\n" + "ldr q17, [x9, #0xa0]\n" + ".inst 0x4f80ea09 // sdot v9.4s, v16.16b, v0.4b[2]\n" + "ldr q16, [x9, #0xb0]\n" + ".inst 0x4f80ea2a // sdot v10.4s, v17.16b, v0.4b[2]\n" + "ldr q17, [x9, #0xc0]\n" + ".inst 0x4f80ea0b // sdot v11.4s, v16.16b, v0.4b[2]\n" + "ldr q16, [x9, #0xd0]\n" + ".inst 0x4fa0ea28 // sdot v8.4s, v17.16b, v0.4b[3]\n" + "ldr q17, [x9, #0xe0]\n" + ".inst 0x4fa0ea09 // sdot v9.4s, v16.16b, v0.4b[3]\n" + "ldr q16, [x9, #0xf0]\n" "sub x27, x27, #0x10\n" "add x26, x26, #0x10\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x4fa0ea2a // sdot v10.4s, v17.16b, v0.4b[3]\n" + ".inst 0x4fa0ea0b // sdot v11.4s, v16.16b, v0.4b[3]\n" "ldr q0, [x26, #0x0]\n" "cmp x27, #0x20\n" "add x9, x9, #0x100\n" @@ -172,37 +171,37 @@ void a64_hybrid_s8qs_dot_6x16 ( "bge 7b\n" "8:" // Height 1: Multiply loop: Single iteration only ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q6, [x9, #0x20]\n" + "ldr q17, [x9, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "ldr q7, [x9, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - "ldr q6, [x9, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - "ldr q7, [x9, #0x50]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - "ldr q6, [x9, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - "ldr q7, [x9, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - "ldr q6, [x9, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - "ldr q7, [x9, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - "ldr q6, [x9, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - "ldr q7, [x9, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - "ldr q6, [x9, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - "ldr q7, [x9, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - "ldr q6, [x9, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - "ldr q7, [x9, #0xf0]\n" + "ldr q16, [x9, #0x30]\n" + ".inst 0x4f80e22a // sdot v10.4s, v17.16b, v0.4b[0]\n" + "ldr q17, [x9, #0x40]\n" + ".inst 0x4f80e20b // sdot v11.4s, v16.16b, v0.4b[0]\n" + "ldr q16, [x9, #0x50]\n" + ".inst 0x4fa0e228 // sdot v8.4s, v17.16b, v0.4b[1]\n" + "ldr q17, [x9, #0x60]\n" + ".inst 0x4fa0e209 // sdot v9.4s, v16.16b, v0.4b[1]\n" + "ldr q16, [x9, #0x70]\n" + ".inst 0x4fa0e22a // sdot v10.4s, v17.16b, v0.4b[1]\n" + "ldr q17, [x9, #0x80]\n" + ".inst 0x4fa0e20b // sdot v11.4s, v16.16b, v0.4b[1]\n" + "ldr q16, [x9, #0x90]\n" + ".inst 0x4f80ea28 // sdot v8.4s, v17.16b, v0.4b[2]\n" + "ldr q17, [x9, #0xa0]\n" + ".inst 0x4f80ea09 // sdot v9.4s, v16.16b, v0.4b[2]\n" + "ldr q16, [x9, #0xb0]\n" + ".inst 0x4f80ea2a // sdot v10.4s, v17.16b, v0.4b[2]\n" + "ldr q17, [x9, #0xc0]\n" + ".inst 0x4f80ea0b // sdot v11.4s, v16.16b, v0.4b[2]\n" + "ldr q16, [x9, #0xd0]\n" + ".inst 0x4fa0ea28 // sdot v8.4s, v17.16b, v0.4b[3]\n" + "ldr q17, [x9, #0xe0]\n" + ".inst 0x4fa0ea09 // sdot v9.4s, v16.16b, v0.4b[3]\n" + "ldr q16, [x9, #0xf0]\n" "add x26, x26, #0x10\n" "sub x27, x27, #0x10\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x4fa0ea2a // sdot v10.4s, v17.16b, v0.4b[3]\n" + ".inst 0x4fa0ea0b // sdot v11.4s, v16.16b, v0.4b[3]\n" "prfm pldl1keep, [x26, #0x80]\n" "add x9, x9, #0x100\n" "9:" // Height 1: Multiply loop: Main loop skip @@ -210,17 +209,17 @@ void a64_hybrid_s8qs_dot_6x16 ( "cmp x27, #0x4\n" "blt 11f\n" "10:" // Height 1: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr q6, [x9, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" + "ldr s18, [x26], #0x4\n" + "ldr q16, [x9, #0x0]\n" + ".inst 0x4f92e208 // sdot v8.4s, v16.16b, v18.4b[0]\n" "sub x27, x27, #0x4\n" - "ldr q7, [x9, #0x10]\n" - "ldr q6, [x9, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" + "ldr q16, [x9, #0x10]\n" + "ldr q17, [x9, #0x20]\n" + ".inst 0x4f92e209 // sdot v9.4s, v16.16b, v18.4b[0]\n" "cmp x27, #0x4\n" - "ldr q7, [x9, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" + "ldr q16, [x9, #0x30]\n" + ".inst 0x4f92e22a // sdot v10.4s, v17.16b, v18.4b[0]\n" + ".inst 0x4f92e20b // sdot v11.4s, v16.16b, v18.4b[0]\n" "add x9, x9, #0x40\n" "bge 10b\n" "11:" // Height 1: Multiply loop: Skip odd blocks @@ -233,28 +232,28 @@ void a64_hybrid_s8qs_dot_6x16 ( "12:" // Height 1: Multiply loop: Ragged operand read: partial_1_0 "ldr b0, [x26, #0x0]\n" "13:" // Height 1: Multiply loop: Ragged operand read: Done - "ldr q6, [x9, #0x0]\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "ldr q6, [x9, #0x20]\n" - "ldr q7, [x9, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" + "ldr q17, [x9, #0x0]\n" + "ldr q16, [x9, #0x10]\n" + ".inst 0x4f80e228 // sdot v8.4s, v17.16b, v0.4b[0]\n" + ".inst 0x4f80e209 // sdot v9.4s, v16.16b, v0.4b[0]\n" + "ldr q17, [x9, #0x20]\n" + "ldr q16, [x9, #0x30]\n" + ".inst 0x4f80e22a // sdot v10.4s, v17.16b, v0.4b[0]\n" + ".inst 0x4f80e20b // sdot v11.4s, v16.16b, v0.4b[0]\n" "add x9, x9, #0x40\n" "14:" // Height 1: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 4b\n" - "ldr q0, [x14, #0x0]\n" - "ldr q1, [x14, #0x10]\n" - "add v8.4s, v8.4s, v0.4s\n" - "add v9.4s, v9.4s, v1.4s\n" - "ldr q2, [x14, #0x20]\n" - "ldr q3, [x14, #0x30]\n" - "add v10.4s, v10.4s, v2.4s\n" - "add v11.4s, v11.4s, v3.4s\n" + "ldr q17, [x14, #0x0]\n" + "ldr q16, [x14, #0x10]\n" + "add v8.4s, v8.4s, v17.4s\n" + "add v9.4s, v9.4s, v16.4s\n" + "ldr q17, [x14, #0x20]\n" + "ldr q16, [x14, #0x30]\n" + "add v10.4s, v10.4s, v17.4s\n" + "add v11.4s, v11.4s, v16.4s\n" "prfm pstl1keep, [x11, #0x0]\n" "add x14, x14, #0x40\n" "tbz %x[flags], #4, 15f\n" @@ -270,10 +269,10 @@ void a64_hybrid_s8qs_dot_6x16 ( "add x13, x13, #0x40\n" "b 16f\n" "15:" // Height 1: per layer parameters - "add x25, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x25]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v4.4s }, [x20]\n" "mov v1.16b, v0.16b\n" "mov v5.16b, v4.16b\n" "mov v2.16b, v0.16b\n" @@ -286,45 +285,45 @@ void a64_hybrid_s8qs_dot_6x16 ( "sqrdmulh v10.4s, v10.4s, v6.4s\n" "sqrdmulh v11.4s, v11.4s, v7.4s\n" "tbz %x[flags], #5, 17f\n" - "and v4.16b, v8.16b, v0.16b\n" - "and v5.16b, v9.16b, v1.16b\n" - "and v6.16b, v10.16b, v2.16b\n" - "and v7.16b, v11.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v4.4s\n" - "sqadd v9.4s, v9.4s, v5.4s\n" - "sqadd v10.4s, v10.4s, v6.4s\n" - "sqadd v11.4s, v11.4s, v7.4s\n" + "and v19.16b, v8.16b, v0.16b\n" + "and v18.16b, v9.16b, v1.16b\n" + "and v17.16b, v10.16b, v2.16b\n" + "and v16.16b, v11.16b, v3.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v8.4s, v8.4s, v19.4s\n" + "sqadd v9.4s, v9.4s, v18.4s\n" + "sqadd v10.4s, v10.4s, v17.4s\n" + "sqadd v11.4s, v11.4s, v16.4s\n" "17:" // Height 1: no shift correction - "add x25, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v18.4s }, [x20]\n" "srshl v8.4s, v8.4s, v0.4s\n" "srshl v9.4s, v9.4s, v1.4s\n" "srshl v10.4s, v10.4s, v2.4s\n" "srshl v11.4s, v11.4s, v3.4s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x25]\n" - "add v8.4s, v8.4s, v4.4s\n" - "add v9.4s, v9.4s, v4.4s\n" - "add x25, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x25]\n" - "add v10.4s, v10.4s, v4.4s\n" - "add v11.4s, v11.4s, v4.4s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v17.4s }, [x20]\n" + "add v8.4s, v8.4s, v18.4s\n" + "add v9.4s, v9.4s, v18.4s\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v16.4s }, [x20]\n" + "add v10.4s, v10.4s, v18.4s\n" + "add v11.4s, v11.4s, v18.4s\n" "cmp x10, #0x10\n" - "smin v8.4s, v8.4s, v6.4s\n" - "smin v9.4s, v9.4s, v6.4s\n" - "smin v10.4s, v10.4s, v6.4s\n" - "smin v11.4s, v11.4s, v6.4s\n" - "smax v8.4s, v8.4s, v5.4s\n" - "smax v9.4s, v9.4s, v5.4s\n" - "smax v10.4s, v10.4s, v5.4s\n" - "smax v11.4s, v11.4s, v5.4s\n" + "smin v8.4s, v8.4s, v17.4s\n" + "smin v9.4s, v9.4s, v17.4s\n" + "smin v10.4s, v10.4s, v17.4s\n" + "smin v11.4s, v11.4s, v17.4s\n" + "smax v8.4s, v8.4s, v16.4s\n" + "smax v9.4s, v9.4s, v16.4s\n" + "smax v10.4s, v10.4s, v16.4s\n" + "smax v11.4s, v11.4s, v16.4s\n" "uzp1 v8.8h, v8.8h, v9.8h\n" - "uzp1 v9.8h, v10.8h, v11.8h\n" - "uzp1 v8.16b, v8.16b, v9.16b\n" + "uzp1 v16.8h, v10.8h, v11.8h\n" + "uzp1 v8.16b, v8.16b, v16.16b\n" "bge 26f\n" "tbz x10, #3, 21f\n" "str d8, [x11], #0x8\n" @@ -399,12 +398,12 @@ void a64_hybrid_s8qs_dot_6x16 ( "31:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 32f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 33f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -412,7 +411,7 @@ void a64_hybrid_s8qs_dot_6x16 ( "b 33f\n" "32:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" + "add x25, x26, x21\n" "33:" // Height 2: input setup done "cmp x27, #0x10\n" "blt 36f\n" @@ -425,137 +424,137 @@ void a64_hybrid_s8qs_dot_6x16 ( "34:" // Height 2: Multiply loop: Main loop head ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x9, #0x20]\n" + "ldr q17, [x9, #0x20]\n" "sub x27, x27, #0x10\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x9, #0x30]\n" + "ldr q16, [x9, #0x30]\n" "add x26, x26, #0x10\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x9, #0x40]\n" + ".inst 0x4f80e22a // sdot v10.4s, v17.16b, v0.4b[0]\n" + ".inst 0x4f81e22e // sdot v14.4s, v17.16b, v1.4b[0]\n" + "ldr q17, [x9, #0x40]\n" "add x25, x25, #0x10\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x9, #0x50]\n" + ".inst 0x4f80e20b // sdot v11.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f81e20f // sdot v15.4s, v16.16b, v1.4b[0]\n" + "ldr q16, [x9, #0x50]\n" "cmp x27, #0x20\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - "ldr q6, [x9, #0x60]\n" + ".inst 0x4fa0e228 // sdot v8.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e22c // sdot v12.4s, v17.16b, v1.4b[1]\n" + "ldr q17, [x9, #0x60]\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - "ldr q7, [x9, #0x70]\n" + ".inst 0x4fa0e209 // sdot v9.4s, v16.16b, v0.4b[1]\n" + ".inst 0x4fa1e20d // sdot v13.4s, v16.16b, v1.4b[1]\n" + "ldr q16, [x9, #0x70]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - "ldr q6, [x9, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - "ldr q7, [x9, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - "ldr q6, [x9, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - "ldr q7, [x9, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - "ldr q6, [x9, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - "ldr q7, [x9, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - "ldr q6, [x9, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - "ldr q7, [x9, #0xf0]\n" + ".inst 0x4fa0e22a // sdot v10.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e22e // sdot v14.4s, v17.16b, v1.4b[1]\n" + "ldr q17, [x9, #0x80]\n" + ".inst 0x4fa0e20b // sdot v11.4s, v16.16b, v0.4b[1]\n" + ".inst 0x4fa1e20f // sdot v15.4s, v16.16b, v1.4b[1]\n" + "ldr q16, [x9, #0x90]\n" + ".inst 0x4f80ea28 // sdot v8.4s, v17.16b, v0.4b[2]\n" + ".inst 0x4f81ea2c // sdot v12.4s, v17.16b, v1.4b[2]\n" + "ldr q17, [x9, #0xa0]\n" + ".inst 0x4f80ea09 // sdot v9.4s, v16.16b, v0.4b[2]\n" + ".inst 0x4f81ea0d // sdot v13.4s, v16.16b, v1.4b[2]\n" + "ldr q16, [x9, #0xb0]\n" + ".inst 0x4f80ea2a // sdot v10.4s, v17.16b, v0.4b[2]\n" + ".inst 0x4f81ea2e // sdot v14.4s, v17.16b, v1.4b[2]\n" + "ldr q17, [x9, #0xc0]\n" + ".inst 0x4f80ea0b // sdot v11.4s, v16.16b, v0.4b[2]\n" + ".inst 0x4f81ea0f // sdot v15.4s, v16.16b, v1.4b[2]\n" + "ldr q16, [x9, #0xd0]\n" + ".inst 0x4fa0ea28 // sdot v8.4s, v17.16b, v0.4b[3]\n" + ".inst 0x4fa1ea2c // sdot v12.4s, v17.16b, v1.4b[3]\n" + "ldr q17, [x9, #0xe0]\n" + ".inst 0x4fa0ea09 // sdot v9.4s, v16.16b, v0.4b[3]\n" + ".inst 0x4fa1ea0d // sdot v13.4s, v16.16b, v1.4b[3]\n" + "ldr q16, [x9, #0xf0]\n" "add x9, x9, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" + ".inst 0x4fa0ea2a // sdot v10.4s, v17.16b, v0.4b[3]\n" + ".inst 0x4fa1ea2e // sdot v14.4s, v17.16b, v1.4b[3]\n" "ldr q6, [x9, #0x0]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x4fa0ea0b // sdot v11.4s, v16.16b, v0.4b[3]\n" "ldr q0, [x26, #0x0]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" + ".inst 0x4fa1ea0f // sdot v15.4s, v16.16b, v1.4b[3]\n" "ldr q1, [x25, #0x0]\n" "ldr q7, [x9, #0x10]\n" "bge 34b\n" "35:" // Height 2: Multiply loop: Single iteration only ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x9, #0x20]\n" + "ldr q17, [x9, #0x20]\n" "add x26, x26, #0x10\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x9, #0x30]\n" + "ldr q16, [x9, #0x30]\n" "add x25, x25, #0x10\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x9, #0x40]\n" + ".inst 0x4f80e22a // sdot v10.4s, v17.16b, v0.4b[0]\n" + ".inst 0x4f81e22e // sdot v14.4s, v17.16b, v1.4b[0]\n" + "ldr q17, [x9, #0x40]\n" "sub x27, x27, #0x10\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x9, #0x50]\n" + ".inst 0x4f80e20b // sdot v11.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f81e20f // sdot v15.4s, v16.16b, v1.4b[0]\n" + "ldr q16, [x9, #0x50]\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - "ldr q6, [x9, #0x60]\n" + ".inst 0x4fa0e228 // sdot v8.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e22c // sdot v12.4s, v17.16b, v1.4b[1]\n" + "ldr q17, [x9, #0x60]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - "ldr q7, [x9, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - "ldr q6, [x9, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - "ldr q7, [x9, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - "ldr q6, [x9, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - "ldr q7, [x9, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - "ldr q6, [x9, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - "ldr q7, [x9, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - "ldr q6, [x9, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - "ldr q7, [x9, #0xf0]\n" + ".inst 0x4fa0e209 // sdot v9.4s, v16.16b, v0.4b[1]\n" + ".inst 0x4fa1e20d // sdot v13.4s, v16.16b, v1.4b[1]\n" + "ldr q16, [x9, #0x70]\n" + ".inst 0x4fa0e22a // sdot v10.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e22e // sdot v14.4s, v17.16b, v1.4b[1]\n" + "ldr q17, [x9, #0x80]\n" + ".inst 0x4fa0e20b // sdot v11.4s, v16.16b, v0.4b[1]\n" + ".inst 0x4fa1e20f // sdot v15.4s, v16.16b, v1.4b[1]\n" + "ldr q16, [x9, #0x90]\n" + ".inst 0x4f80ea28 // sdot v8.4s, v17.16b, v0.4b[2]\n" + ".inst 0x4f81ea2c // sdot v12.4s, v17.16b, v1.4b[2]\n" + "ldr q17, [x9, #0xa0]\n" + ".inst 0x4f80ea09 // sdot v9.4s, v16.16b, v0.4b[2]\n" + ".inst 0x4f81ea0d // sdot v13.4s, v16.16b, v1.4b[2]\n" + "ldr q16, [x9, #0xb0]\n" + ".inst 0x4f80ea2a // sdot v10.4s, v17.16b, v0.4b[2]\n" + ".inst 0x4f81ea2e // sdot v14.4s, v17.16b, v1.4b[2]\n" + "ldr q17, [x9, #0xc0]\n" + ".inst 0x4f80ea0b // sdot v11.4s, v16.16b, v0.4b[2]\n" + ".inst 0x4f81ea0f // sdot v15.4s, v16.16b, v1.4b[2]\n" + "ldr q16, [x9, #0xd0]\n" + ".inst 0x4fa0ea28 // sdot v8.4s, v17.16b, v0.4b[3]\n" + ".inst 0x4fa1ea2c // sdot v12.4s, v17.16b, v1.4b[3]\n" + "ldr q17, [x9, #0xe0]\n" + ".inst 0x4fa0ea09 // sdot v9.4s, v16.16b, v0.4b[3]\n" + ".inst 0x4fa1ea0d // sdot v13.4s, v16.16b, v1.4b[3]\n" + "ldr q16, [x9, #0xf0]\n" "add x9, x9, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" + ".inst 0x4fa0ea2a // sdot v10.4s, v17.16b, v0.4b[3]\n" + ".inst 0x4fa1ea2e // sdot v14.4s, v17.16b, v1.4b[3]\n" + ".inst 0x4fa0ea0b // sdot v11.4s, v16.16b, v0.4b[3]\n" + ".inst 0x4fa1ea0f // sdot v15.4s, v16.16b, v1.4b[3]\n" "36:" // Height 2: Multiply loop: Main loop skip "cbz x27, 41f\n" "cmp x27, #0x4\n" "blt 38f\n" "37:" // Height 2: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s19, [x26], #0x4\n" + "ldr s18, [x25], #0x4\n" "sub x27, x27, #0x4\n" "cmp x27, #0x4\n" - "ldr q6, [x9, #0x0]\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x9, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x9, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" + "ldr q17, [x9, #0x0]\n" + "ldr q16, [x9, #0x10]\n" + ".inst 0x4f93e228 // sdot v8.4s, v17.16b, v19.4b[0]\n" + ".inst 0x4f92e22c // sdot v12.4s, v17.16b, v18.4b[0]\n" + "ldr q17, [x9, #0x20]\n" + ".inst 0x4f93e209 // sdot v9.4s, v16.16b, v19.4b[0]\n" + ".inst 0x4f92e20d // sdot v13.4s, v16.16b, v18.4b[0]\n" + "ldr q16, [x9, #0x30]\n" + ".inst 0x4f93e22a // sdot v10.4s, v17.16b, v19.4b[0]\n" + ".inst 0x4f92e22e // sdot v14.4s, v17.16b, v18.4b[0]\n" "add x9, x9, #0x40\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" + ".inst 0x4f93e20b // sdot v11.4s, v16.16b, v19.4b[0]\n" + ".inst 0x4f92e20f // sdot v15.4s, v16.16b, v18.4b[0]\n" "bge 37b\n" "38:" // Height 2: Multiply loop: Skip odd blocks "cbz x27, 41f\n" @@ -570,41 +569,41 @@ void a64_hybrid_s8qs_dot_6x16 ( "ldr b0, [x26, #0x0]\n" "ldr b1, [x25, #0x0]\n" "40:" // Height 2: Multiply loop: Ragged operand read: Done - "ldr q6, [x9, #0x0]\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x9, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x9, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" + "ldr q17, [x9, #0x0]\n" + "ldr q16, [x9, #0x10]\n" + ".inst 0x4f80e228 // sdot v8.4s, v17.16b, v0.4b[0]\n" + ".inst 0x4f81e22c // sdot v12.4s, v17.16b, v1.4b[0]\n" + "ldr q17, [x9, #0x20]\n" + ".inst 0x4f80e209 // sdot v9.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f81e20d // sdot v13.4s, v16.16b, v1.4b[0]\n" + "ldr q16, [x9, #0x30]\n" + ".inst 0x4f80e22a // sdot v10.4s, v17.16b, v0.4b[0]\n" + ".inst 0x4f81e22e // sdot v14.4s, v17.16b, v1.4b[0]\n" "add x9, x9, #0x40\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" + ".inst 0x4f80e20b // sdot v11.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f81e20f // sdot v15.4s, v16.16b, v1.4b[0]\n" "41:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 31b\n" - "ldr q0, [x14, #0x0]\n" - "ldr q1, [x14, #0x10]\n" - "add v8.4s, v8.4s, v0.4s\n" - "add v9.4s, v9.4s, v1.4s\n" - "ldr q2, [x14, #0x20]\n" - "ldr q3, [x14, #0x30]\n" - "add v10.4s, v10.4s, v2.4s\n" - "add v11.4s, v11.4s, v3.4s\n" + "ldr q19, [x14, #0x0]\n" + "ldr q18, [x14, #0x10]\n" + "add v8.4s, v8.4s, v19.4s\n" + "add v9.4s, v9.4s, v18.4s\n" + "ldr q17, [x14, #0x20]\n" + "ldr q16, [x14, #0x30]\n" + "add v10.4s, v10.4s, v17.4s\n" + "add v11.4s, v11.4s, v16.4s\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x11, x20\n" + "add x25, x11, x20\n" "prfm pstl1keep, [x11, #0x0]\n" - "add v12.4s, v12.4s, v0.4s\n" - "prfm pstl1keep, [x24, #0x0]\n" - "add v13.4s, v13.4s, v1.4s\n" - "add v14.4s, v14.4s, v2.4s\n" + "add v12.4s, v12.4s, v19.4s\n" + "prfm pstl1keep, [x25, #0x0]\n" + "add v13.4s, v13.4s, v18.4s\n" + "add v14.4s, v14.4s, v17.4s\n" "add x14, x14, #0x40\n" - "add v15.4s, v15.4s, v3.4s\n" + "add v15.4s, v15.4s, v16.4s\n" "tbz %x[flags], #4, 42f\n" "ldr q0, [x12, #0x0]\n" "ldr q4, [x13, #0x0]\n" @@ -618,10 +617,10 @@ void a64_hybrid_s8qs_dot_6x16 ( "add x13, x13, #0x40\n" "b 43f\n" "42:" // Height 2: per layer parameters - "add x25, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x25]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v4.4s }, [x20]\n" "mov v1.16b, v0.16b\n" "mov v5.16b, v4.16b\n" "mov v2.16b, v0.16b\n" @@ -638,141 +637,141 @@ void a64_hybrid_s8qs_dot_6x16 ( "sqrdmulh v14.4s, v14.4s, v6.4s\n" "sqrdmulh v15.4s, v15.4s, v7.4s\n" "tbz %x[flags], #5, 44f\n" - "and v4.16b, v8.16b, v0.16b\n" - "and v5.16b, v9.16b, v1.16b\n" - "and v6.16b, v10.16b, v2.16b\n" - "and v7.16b, v11.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v4.4s\n" - "sqadd v9.4s, v9.4s, v5.4s\n" - "sqadd v10.4s, v10.4s, v6.4s\n" - "sqadd v11.4s, v11.4s, v7.4s\n" - "and v4.16b, v12.16b, v0.16b\n" - "and v5.16b, v13.16b, v1.16b\n" - "and v6.16b, v14.16b, v2.16b\n" - "and v7.16b, v15.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v12.4s, v12.4s, v4.4s\n" - "sqadd v13.4s, v13.4s, v5.4s\n" - "sqadd v14.4s, v14.4s, v6.4s\n" - "sqadd v15.4s, v15.4s, v7.4s\n" + "and v19.16b, v8.16b, v0.16b\n" + "and v18.16b, v9.16b, v1.16b\n" + "and v17.16b, v10.16b, v2.16b\n" + "and v16.16b, v11.16b, v3.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v8.4s, v8.4s, v19.4s\n" + "sqadd v9.4s, v9.4s, v18.4s\n" + "sqadd v10.4s, v10.4s, v17.4s\n" + "sqadd v11.4s, v11.4s, v16.4s\n" + "and v19.16b, v12.16b, v0.16b\n" + "and v18.16b, v13.16b, v1.16b\n" + "and v17.16b, v14.16b, v2.16b\n" + "and v16.16b, v15.16b, v3.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v12.4s, v12.4s, v19.4s\n" + "sqadd v13.4s, v13.4s, v18.4s\n" + "sqadd v14.4s, v14.4s, v17.4s\n" + "sqadd v15.4s, v15.4s, v16.4s\n" "44:" // Height 2: no shift correction - "add x25, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v18.4s }, [x20]\n" "srshl v8.4s, v8.4s, v0.4s\n" "srshl v9.4s, v9.4s, v1.4s\n" "srshl v10.4s, v10.4s, v2.4s\n" "srshl v11.4s, v11.4s, v3.4s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x25]\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v17.4s }, [x20]\n" "srshl v12.4s, v12.4s, v0.4s\n" "srshl v13.4s, v13.4s, v1.4s\n" - "add x25, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x25]\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v16.4s }, [x20]\n" "srshl v14.4s, v14.4s, v2.4s\n" "srshl v15.4s, v15.4s, v3.4s\n" "cmp x10, #0x10\n" - "add v8.4s, v8.4s, v4.4s\n" - "add v9.4s, v9.4s, v4.4s\n" - "add v10.4s, v10.4s, v4.4s\n" - "add v11.4s, v11.4s, v4.4s\n" - "add v12.4s, v12.4s, v4.4s\n" - "add v13.4s, v13.4s, v4.4s\n" - "add v14.4s, v14.4s, v4.4s\n" - "add v15.4s, v15.4s, v4.4s\n" - "smin v8.4s, v8.4s, v6.4s\n" - "smin v9.4s, v9.4s, v6.4s\n" - "smin v10.4s, v10.4s, v6.4s\n" - "smin v11.4s, v11.4s, v6.4s\n" - "smin v12.4s, v12.4s, v6.4s\n" - "smin v13.4s, v13.4s, v6.4s\n" - "smin v14.4s, v14.4s, v6.4s\n" - "smin v15.4s, v15.4s, v6.4s\n" - "smax v8.4s, v8.4s, v5.4s\n" - "smax v9.4s, v9.4s, v5.4s\n" - "smax v10.4s, v10.4s, v5.4s\n" - "smax v11.4s, v11.4s, v5.4s\n" - "smax v12.4s, v12.4s, v5.4s\n" - "smax v13.4s, v13.4s, v5.4s\n" - "smax v14.4s, v14.4s, v5.4s\n" - "smax v15.4s, v15.4s, v5.4s\n" + "add v8.4s, v8.4s, v18.4s\n" + "add v9.4s, v9.4s, v18.4s\n" + "add v10.4s, v10.4s, v18.4s\n" + "add v11.4s, v11.4s, v18.4s\n" + "add v12.4s, v12.4s, v18.4s\n" + "add v13.4s, v13.4s, v18.4s\n" + "add v14.4s, v14.4s, v18.4s\n" + "add v15.4s, v15.4s, v18.4s\n" + "smin v8.4s, v8.4s, v17.4s\n" + "smin v9.4s, v9.4s, v17.4s\n" + "smin v10.4s, v10.4s, v17.4s\n" + "smin v11.4s, v11.4s, v17.4s\n" + "smin v12.4s, v12.4s, v17.4s\n" + "smin v13.4s, v13.4s, v17.4s\n" + "smin v14.4s, v14.4s, v17.4s\n" + "smin v15.4s, v15.4s, v17.4s\n" + "smax v8.4s, v8.4s, v16.4s\n" + "smax v9.4s, v9.4s, v16.4s\n" + "smax v10.4s, v10.4s, v16.4s\n" + "smax v11.4s, v11.4s, v16.4s\n" + "smax v12.4s, v12.4s, v16.4s\n" + "smax v13.4s, v13.4s, v16.4s\n" + "smax v14.4s, v14.4s, v16.4s\n" + "smax v15.4s, v15.4s, v16.4s\n" "uzp1 v8.8h, v8.8h, v9.8h\n" - "uzp1 v9.8h, v10.8h, v11.8h\n" + "uzp1 v17.8h, v10.8h, v11.8h\n" "uzp1 v12.8h, v12.8h, v13.8h\n" - "uzp1 v13.8h, v14.8h, v15.8h\n" - "uzp1 v8.16b, v8.16b, v9.16b\n" - "uzp1 v12.16b, v12.16b, v13.16b\n" + "uzp1 v16.8h, v14.8h, v15.8h\n" + "uzp1 v8.16b, v8.16b, v17.16b\n" + "uzp1 v12.16b, v12.16b, v16.16b\n" "bge 53f\n" "tbz x10, #3, 48f\n" "str d8, [x11], #0x8\n" - "str d12, [x24], #0x8\n" + "str d12, [x25], #0x8\n" "tbz x10, #2, 46f\n" "st1 { v8.s }[2], [x11], #0x4\n" - "st1 { v12.s }[2], [x24], #0x4\n" + "st1 { v12.s }[2], [x25], #0x4\n" "tbz x10, #1, 45f\n" "st1 { v8.h }[6], [x11], #0x2\n" - "st1 { v12.h }[6], [x24], #0x2\n" + "st1 { v12.h }[6], [x25], #0x2\n" "tbz x10, #0, 52f\n" "st1 { v8.b }[14], [x11]\n" - "st1 { v12.b }[14], [x24]\n" + "st1 { v12.b }[14], [x25]\n" "b 52f\n" "45:" // Height 2: Partial direct writeback: partial_1_12 "tbz x10, #0, 52f\n" "st1 { v8.b }[12], [x11]\n" - "st1 { v12.b }[12], [x24]\n" + "st1 { v12.b }[12], [x25]\n" "b 52f\n" "46:" // Height 2: Partial direct writeback: partial_2_8 "tbz x10, #1, 47f\n" "st1 { v8.h }[4], [x11], #0x2\n" - "st1 { v12.h }[4], [x24], #0x2\n" + "st1 { v12.h }[4], [x25], #0x2\n" "tbz x10, #0, 52f\n" "st1 { v8.b }[10], [x11]\n" - "st1 { v12.b }[10], [x24]\n" + "st1 { v12.b }[10], [x25]\n" "b 52f\n" "47:" // Height 2: Partial direct writeback: partial_1_8 "tbz x10, #0, 52f\n" "st1 { v8.b }[8], [x11]\n" - "st1 { v12.b }[8], [x24]\n" + "st1 { v12.b }[8], [x25]\n" "b 52f\n" "48:" // Height 2: Partial direct writeback: partial_4_0 "tbz x10, #2, 50f\n" "str s8, [x11], #0x4\n" - "str s12, [x24], #0x4\n" + "str s12, [x25], #0x4\n" "tbz x10, #1, 49f\n" "st1 { v8.h }[2], [x11], #0x2\n" - "st1 { v12.h }[2], [x24], #0x2\n" + "st1 { v12.h }[2], [x25], #0x2\n" "tbz x10, #0, 52f\n" "st1 { v8.b }[6], [x11]\n" - "st1 { v12.b }[6], [x24]\n" + "st1 { v12.b }[6], [x25]\n" "b 52f\n" "49:" // Height 2: Partial direct writeback: partial_1_4 "tbz x10, #0, 52f\n" "st1 { v8.b }[4], [x11]\n" - "st1 { v12.b }[4], [x24]\n" + "st1 { v12.b }[4], [x25]\n" "b 52f\n" "50:" // Height 2: Partial direct writeback: partial_2_0 "tbz x10, #1, 51f\n" "str h8, [x11], #0x2\n" - "str h12, [x24], #0x2\n" + "str h12, [x25], #0x2\n" "tbz x10, #0, 52f\n" "st1 { v8.b }[2], [x11]\n" - "st1 { v12.b }[2], [x24]\n" + "st1 { v12.b }[2], [x25]\n" "b 52f\n" "51:" // Height 2: Partial direct writeback: partial_1_0 "str b8, [x11, #0x0]\n" - "str b12, [x24, #0x0]\n" + "str b12, [x25, #0x0]\n" "52:" // Height 2: Partial direct writeback: Done "b 54f\n" "53:" // Height 2: Full writeback "str q8, [x11, #0x0]\n" "add x11, x11, #0x10\n" - "str q12, [x24, #0x0]\n" + "str q12, [x25, #0x0]\n" "54:" // Height 2: Writeback done "subs x10, x10, #0x10\n" "bgt 29b\n" @@ -802,13 +801,13 @@ void a64_hybrid_s8qs_dot_6x16 ( "58:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 59f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 60f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -817,8 +816,8 @@ void a64_hybrid_s8qs_dot_6x16 ( "b 60f\n" "59:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" "60:" // Height 3: input setup done "cmp x27, #0x10\n" "blt 63f\n" @@ -835,75 +834,75 @@ void a64_hybrid_s8qs_dot_6x16 ( "sub x27, x27, #0x10\n" "add x26, x26, #0x10\n" ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x9, #0x20]\n" + "ldr q21, [x9, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" "add x25, x25, #0x10\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x9, #0x30]\n" + "ldr q20, [x9, #0x30]\n" "add x24, x24, #0x10\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" + ".inst 0x4f80e2aa // sdot v10.4s, v21.16b, v0.4b[0]\n" + ".inst 0x4f81e2ae // sdot v14.4s, v21.16b, v1.4b[0]\n" "cmp x27, #0x20\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x9, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" + ".inst 0x4f82e2b2 // sdot v18.4s, v21.16b, v2.4b[0]\n" + "ldr q21, [x9, #0x40]\n" + ".inst 0x4f80e28b // sdot v11.4s, v20.16b, v0.4b[0]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x9, #0x50]\n" + ".inst 0x4f81e28f // sdot v15.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f82e293 // sdot v19.4s, v20.16b, v2.4b[0]\n" + "ldr q20, [x9, #0x50]\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" - "ldr q6, [x9, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - "ldr q7, [x9, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - "ldr q6, [x9, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" - "ldr q7, [x9, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" - "ldr q6, [x9, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" - "ldr q7, [x9, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" - "ldr q6, [x9, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" - "ldr q7, [x9, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" - "ldr q6, [x9, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" - "ldr q7, [x9, #0xf0]\n" + ".inst 0x4fa0e2a8 // sdot v8.4s, v21.16b, v0.4b[1]\n" + ".inst 0x4fa1e2ac // sdot v12.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa2e2b0 // sdot v16.4s, v21.16b, v2.4b[1]\n" + "ldr q21, [x9, #0x60]\n" + ".inst 0x4fa0e289 // sdot v9.4s, v20.16b, v0.4b[1]\n" + ".inst 0x4fa1e28d // sdot v13.4s, v20.16b, v1.4b[1]\n" + ".inst 0x4fa2e291 // sdot v17.4s, v20.16b, v2.4b[1]\n" + "ldr q20, [x9, #0x70]\n" + ".inst 0x4fa0e2aa // sdot v10.4s, v21.16b, v0.4b[1]\n" + ".inst 0x4fa1e2ae // sdot v14.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa2e2b2 // sdot v18.4s, v21.16b, v2.4b[1]\n" + "ldr q21, [x9, #0x80]\n" + ".inst 0x4fa0e28b // sdot v11.4s, v20.16b, v0.4b[1]\n" + ".inst 0x4fa1e28f // sdot v15.4s, v20.16b, v1.4b[1]\n" + ".inst 0x4fa2e293 // sdot v19.4s, v20.16b, v2.4b[1]\n" + "ldr q20, [x9, #0x90]\n" + ".inst 0x4f80eaa8 // sdot v8.4s, v21.16b, v0.4b[2]\n" + ".inst 0x4f81eaac // sdot v12.4s, v21.16b, v1.4b[2]\n" + ".inst 0x4f82eab0 // sdot v16.4s, v21.16b, v2.4b[2]\n" + "ldr q21, [x9, #0xa0]\n" + ".inst 0x4f80ea89 // sdot v9.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f81ea8d // sdot v13.4s, v20.16b, v1.4b[2]\n" + ".inst 0x4f82ea91 // sdot v17.4s, v20.16b, v2.4b[2]\n" + "ldr q20, [x9, #0xb0]\n" + ".inst 0x4f80eaaa // sdot v10.4s, v21.16b, v0.4b[2]\n" + ".inst 0x4f81eaae // sdot v14.4s, v21.16b, v1.4b[2]\n" + ".inst 0x4f82eab2 // sdot v18.4s, v21.16b, v2.4b[2]\n" + "ldr q21, [x9, #0xc0]\n" + ".inst 0x4f80ea8b // sdot v11.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f81ea8f // sdot v15.4s, v20.16b, v1.4b[2]\n" + ".inst 0x4f82ea93 // sdot v19.4s, v20.16b, v2.4b[2]\n" + "ldr q20, [x9, #0xd0]\n" + ".inst 0x4fa0eaa8 // sdot v8.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa1eaac // sdot v12.4s, v21.16b, v1.4b[3]\n" + ".inst 0x4fa2eab0 // sdot v16.4s, v21.16b, v2.4b[3]\n" + "ldr q21, [x9, #0xe0]\n" + ".inst 0x4fa0ea89 // sdot v9.4s, v20.16b, v0.4b[3]\n" + ".inst 0x4fa1ea8d // sdot v13.4s, v20.16b, v1.4b[3]\n" + ".inst 0x4fa2ea91 // sdot v17.4s, v20.16b, v2.4b[3]\n" + "ldr q20, [x9, #0xf0]\n" "add x9, x9, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" + ".inst 0x4fa0eaaa // sdot v10.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa1eaae // sdot v14.4s, v21.16b, v1.4b[3]\n" + ".inst 0x4fa2eab2 // sdot v18.4s, v21.16b, v2.4b[3]\n" "ldr q6, [x9, #0x0]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x4fa0ea8b // sdot v11.4s, v20.16b, v0.4b[3]\n" "ldr q0, [x26, #0x0]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" + ".inst 0x4fa1ea8f // sdot v15.4s, v20.16b, v1.4b[3]\n" "ldr q1, [x25, #0x0]\n" - ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" + ".inst 0x4fa2ea93 // sdot v19.4s, v20.16b, v2.4b[3]\n" "ldr q2, [x24, #0x0]\n" "ldr q7, [x9, #0x10]\n" "bge 61b\n" @@ -913,98 +912,98 @@ void a64_hybrid_s8qs_dot_6x16 ( "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x9, #0x20]\n" + "ldr q21, [x9, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" "add x24, x24, #0x10\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x9, #0x30]\n" + "ldr q20, [x9, #0x30]\n" "sub x27, x27, #0x10\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" + ".inst 0x4f80e2aa // sdot v10.4s, v21.16b, v0.4b[0]\n" + ".inst 0x4f81e2ae // sdot v14.4s, v21.16b, v1.4b[0]\n" "prfm pldl1keep, [x26, #0x80]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x9, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" + ".inst 0x4f82e2b2 // sdot v18.4s, v21.16b, v2.4b[0]\n" + "ldr q21, [x9, #0x40]\n" + ".inst 0x4f80e28b // sdot v11.4s, v20.16b, v0.4b[0]\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x9, #0x50]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" - "ldr q6, [x9, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - "ldr q7, [x9, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - "ldr q6, [x9, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" - "ldr q7, [x9, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" - "ldr q6, [x9, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" - "ldr q7, [x9, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" - "ldr q6, [x9, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" - "ldr q7, [x9, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" - "ldr q6, [x9, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" - "ldr q7, [x9, #0xf0]\n" + ".inst 0x4f81e28f // sdot v15.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f82e293 // sdot v19.4s, v20.16b, v2.4b[0]\n" + "ldr q20, [x9, #0x50]\n" + ".inst 0x4fa0e2a8 // sdot v8.4s, v21.16b, v0.4b[1]\n" + ".inst 0x4fa1e2ac // sdot v12.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa2e2b0 // sdot v16.4s, v21.16b, v2.4b[1]\n" + "ldr q21, [x9, #0x60]\n" + ".inst 0x4fa0e289 // sdot v9.4s, v20.16b, v0.4b[1]\n" + ".inst 0x4fa1e28d // sdot v13.4s, v20.16b, v1.4b[1]\n" + ".inst 0x4fa2e291 // sdot v17.4s, v20.16b, v2.4b[1]\n" + "ldr q20, [x9, #0x70]\n" + ".inst 0x4fa0e2aa // sdot v10.4s, v21.16b, v0.4b[1]\n" + ".inst 0x4fa1e2ae // sdot v14.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa2e2b2 // sdot v18.4s, v21.16b, v2.4b[1]\n" + "ldr q21, [x9, #0x80]\n" + ".inst 0x4fa0e28b // sdot v11.4s, v20.16b, v0.4b[1]\n" + ".inst 0x4fa1e28f // sdot v15.4s, v20.16b, v1.4b[1]\n" + ".inst 0x4fa2e293 // sdot v19.4s, v20.16b, v2.4b[1]\n" + "ldr q20, [x9, #0x90]\n" + ".inst 0x4f80eaa8 // sdot v8.4s, v21.16b, v0.4b[2]\n" + ".inst 0x4f81eaac // sdot v12.4s, v21.16b, v1.4b[2]\n" + ".inst 0x4f82eab0 // sdot v16.4s, v21.16b, v2.4b[2]\n" + "ldr q21, [x9, #0xa0]\n" + ".inst 0x4f80ea89 // sdot v9.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f81ea8d // sdot v13.4s, v20.16b, v1.4b[2]\n" + ".inst 0x4f82ea91 // sdot v17.4s, v20.16b, v2.4b[2]\n" + "ldr q20, [x9, #0xb0]\n" + ".inst 0x4f80eaaa // sdot v10.4s, v21.16b, v0.4b[2]\n" + ".inst 0x4f81eaae // sdot v14.4s, v21.16b, v1.4b[2]\n" + ".inst 0x4f82eab2 // sdot v18.4s, v21.16b, v2.4b[2]\n" + "ldr q21, [x9, #0xc0]\n" + ".inst 0x4f80ea8b // sdot v11.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f81ea8f // sdot v15.4s, v20.16b, v1.4b[2]\n" + ".inst 0x4f82ea93 // sdot v19.4s, v20.16b, v2.4b[2]\n" + "ldr q20, [x9, #0xd0]\n" + ".inst 0x4fa0eaa8 // sdot v8.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa1eaac // sdot v12.4s, v21.16b, v1.4b[3]\n" + ".inst 0x4fa2eab0 // sdot v16.4s, v21.16b, v2.4b[3]\n" + "ldr q21, [x9, #0xe0]\n" + ".inst 0x4fa0ea89 // sdot v9.4s, v20.16b, v0.4b[3]\n" + ".inst 0x4fa1ea8d // sdot v13.4s, v20.16b, v1.4b[3]\n" + ".inst 0x4fa2ea91 // sdot v17.4s, v20.16b, v2.4b[3]\n" + "ldr q20, [x9, #0xf0]\n" "add x9, x9, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" + ".inst 0x4fa0eaaa // sdot v10.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa1eaae // sdot v14.4s, v21.16b, v1.4b[3]\n" + ".inst 0x4fa2eab2 // sdot v18.4s, v21.16b, v2.4b[3]\n" + ".inst 0x4fa0ea8b // sdot v11.4s, v20.16b, v0.4b[3]\n" + ".inst 0x4fa1ea8f // sdot v15.4s, v20.16b, v1.4b[3]\n" + ".inst 0x4fa2ea93 // sdot v19.4s, v20.16b, v2.4b[3]\n" "63:" // Height 3: Multiply loop: Main loop skip "cbz x27, 68f\n" "cmp x27, #0x4\n" "blt 65f\n" "64:" // Height 3: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s24, [x26], #0x4\n" + "ldr s23, [x25], #0x4\n" "sub x27, x27, #0x4\n" "cmp x27, #0x4\n" - "ldr s2, [x24], #0x4\n" - "ldr q6, [x9, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x9, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x9, #0x30]\n" + "ldr s22, [x24], #0x4\n" + "ldr q21, [x9, #0x0]\n" + ".inst 0x4f98e2a8 // sdot v8.4s, v21.16b, v24.4b[0]\n" + ".inst 0x4f97e2ac // sdot v12.4s, v21.16b, v23.4b[0]\n" + "ldr q20, [x9, #0x10]\n" + ".inst 0x4f96e2b0 // sdot v16.4s, v21.16b, v22.4b[0]\n" + "ldr q21, [x9, #0x20]\n" + ".inst 0x4f98e289 // sdot v9.4s, v20.16b, v24.4b[0]\n" + ".inst 0x4f97e28d // sdot v13.4s, v20.16b, v23.4b[0]\n" + ".inst 0x4f96e291 // sdot v17.4s, v20.16b, v22.4b[0]\n" + "ldr q20, [x9, #0x30]\n" "add x9, x9, #0x40\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" + ".inst 0x4f98e2aa // sdot v10.4s, v21.16b, v24.4b[0]\n" + ".inst 0x4f97e2ae // sdot v14.4s, v21.16b, v23.4b[0]\n" + ".inst 0x4f96e2b2 // sdot v18.4s, v21.16b, v22.4b[0]\n" + ".inst 0x4f98e28b // sdot v11.4s, v20.16b, v24.4b[0]\n" + ".inst 0x4f97e28f // sdot v15.4s, v20.16b, v23.4b[0]\n" + ".inst 0x4f96e293 // sdot v19.4s, v20.16b, v22.4b[0]\n" "bge 64b\n" "65:" // Height 3: Multiply loop: Skip odd blocks "cbz x27, 68f\n" @@ -1022,51 +1021,51 @@ void a64_hybrid_s8qs_dot_6x16 ( "ldr b1, [x25, #0x0]\n" "ldr b2, [x24, #0x0]\n" "67:" // Height 3: Multiply loop: Ragged operand read: Done - "ldr q6, [x9, #0x0]\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x9, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x9, #0x30]\n" + "ldr q21, [x9, #0x0]\n" + "ldr q20, [x9, #0x10]\n" + ".inst 0x4f80e2a8 // sdot v8.4s, v21.16b, v0.4b[0]\n" + ".inst 0x4f81e2ac // sdot v12.4s, v21.16b, v1.4b[0]\n" + ".inst 0x4f82e2b0 // sdot v16.4s, v21.16b, v2.4b[0]\n" + "ldr q21, [x9, #0x20]\n" + ".inst 0x4f80e289 // sdot v9.4s, v20.16b, v0.4b[0]\n" + ".inst 0x4f81e28d // sdot v13.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f82e291 // sdot v17.4s, v20.16b, v2.4b[0]\n" + "ldr q20, [x9, #0x30]\n" "add x9, x9, #0x40\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" + ".inst 0x4f80e2aa // sdot v10.4s, v21.16b, v0.4b[0]\n" + ".inst 0x4f81e2ae // sdot v14.4s, v21.16b, v1.4b[0]\n" + ".inst 0x4f82e2b2 // sdot v18.4s, v21.16b, v2.4b[0]\n" + ".inst 0x4f80e28b // sdot v11.4s, v20.16b, v0.4b[0]\n" + ".inst 0x4f81e28f // sdot v15.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f82e293 // sdot v19.4s, v20.16b, v2.4b[0]\n" "68:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 58b\n" - "ldr q0, [x14, #0x0]\n" - "ldr q1, [x14, #0x10]\n" - "add v8.4s, v8.4s, v0.4s\n" - "add v9.4s, v9.4s, v1.4s\n" - "ldr q2, [x14, #0x20]\n" - "ldr q3, [x14, #0x30]\n" - "add v10.4s, v10.4s, v2.4s\n" - "add v11.4s, v11.4s, v3.4s\n" + "ldr q23, [x14, #0x0]\n" + "ldr q22, [x14, #0x10]\n" + "add v8.4s, v8.4s, v23.4s\n" + "add v9.4s, v9.4s, v22.4s\n" + "ldr q21, [x14, #0x20]\n" + "ldr q20, [x14, #0x30]\n" + "add v10.4s, v10.4s, v21.4s\n" + "add v11.4s, v11.4s, v20.4s\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x11, x20\n" - "add x23, x24, x20\n" + "add x25, x11, x20\n" + "add x24, x25, x20\n" "prfm pstl1keep, [x11, #0x0]\n" + "prfm pstl1keep, [x25, #0x0]\n" "prfm pstl1keep, [x24, #0x0]\n" - "prfm pstl1keep, [x23, #0x0]\n" - "add v12.4s, v12.4s, v0.4s\n" - "add v13.4s, v13.4s, v1.4s\n" - "add v14.4s, v14.4s, v2.4s\n" - "add v15.4s, v15.4s, v3.4s\n" + "add v12.4s, v12.4s, v23.4s\n" + "add v13.4s, v13.4s, v22.4s\n" + "add v14.4s, v14.4s, v21.4s\n" + "add v15.4s, v15.4s, v20.4s\n" "add x14, x14, #0x40\n" - "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" + "add v16.4s, v16.4s, v23.4s\n" + "add v17.4s, v17.4s, v22.4s\n" + "add v18.4s, v18.4s, v21.4s\n" + "add v19.4s, v19.4s, v20.4s\n" "tbz %x[flags], #4, 69f\n" "ldr q0, [x12, #0x0]\n" "ldr q4, [x13, #0x0]\n" @@ -1080,10 +1079,10 @@ void a64_hybrid_s8qs_dot_6x16 ( "add x13, x13, #0x40\n" "b 70f\n" "69:" // Height 3: per layer parameters - "add x25, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x25]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v4.4s }, [x20]\n" "mov v1.16b, v0.16b\n" "mov v5.16b, v4.16b\n" "mov v2.16b, v0.16b\n" @@ -1104,55 +1103,55 @@ void a64_hybrid_s8qs_dot_6x16 ( "sqrdmulh v18.4s, v18.4s, v6.4s\n" "sqrdmulh v19.4s, v19.4s, v7.4s\n" "tbz %x[flags], #5, 71f\n" - "and v4.16b, v8.16b, v0.16b\n" - "and v5.16b, v9.16b, v1.16b\n" - "and v6.16b, v10.16b, v2.16b\n" - "and v7.16b, v11.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v4.4s\n" - "sqadd v9.4s, v9.4s, v5.4s\n" - "sqadd v10.4s, v10.4s, v6.4s\n" - "sqadd v11.4s, v11.4s, v7.4s\n" - "and v4.16b, v12.16b, v0.16b\n" - "and v5.16b, v13.16b, v1.16b\n" - "and v6.16b, v14.16b, v2.16b\n" - "and v7.16b, v15.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v12.4s, v12.4s, v4.4s\n" - "sqadd v13.4s, v13.4s, v5.4s\n" - "sqadd v14.4s, v14.4s, v6.4s\n" - "sqadd v15.4s, v15.4s, v7.4s\n" - "and v4.16b, v16.16b, v0.16b\n" - "and v5.16b, v17.16b, v1.16b\n" - "and v6.16b, v18.16b, v2.16b\n" - "and v7.16b, v19.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" + "and v23.16b, v8.16b, v0.16b\n" + "and v22.16b, v9.16b, v1.16b\n" + "and v21.16b, v10.16b, v2.16b\n" + "and v20.16b, v11.16b, v3.16b\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "sshr v21.4s, v21.4s, #0x1f\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "sqadd v8.4s, v8.4s, v23.4s\n" + "sqadd v9.4s, v9.4s, v22.4s\n" + "sqadd v10.4s, v10.4s, v21.4s\n" + "sqadd v11.4s, v11.4s, v20.4s\n" + "and v23.16b, v12.16b, v0.16b\n" + "and v22.16b, v13.16b, v1.16b\n" + "and v21.16b, v14.16b, v2.16b\n" + "and v20.16b, v15.16b, v3.16b\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "sshr v21.4s, v21.4s, #0x1f\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "sqadd v12.4s, v12.4s, v23.4s\n" + "sqadd v13.4s, v13.4s, v22.4s\n" + "sqadd v14.4s, v14.4s, v21.4s\n" + "sqadd v15.4s, v15.4s, v20.4s\n" + "and v23.16b, v16.16b, v0.16b\n" + "and v22.16b, v17.16b, v1.16b\n" + "and v21.16b, v18.16b, v2.16b\n" + "and v20.16b, v19.16b, v3.16b\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "sshr v21.4s, v21.4s, #0x1f\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v23.4s\n" + "sqadd v17.4s, v17.4s, v22.4s\n" + "sqadd v18.4s, v18.4s, v21.4s\n" + "sqadd v19.4s, v19.4s, v20.4s\n" "71:" // Height 3: no shift correction - "add x25, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v22.4s }, [x20]\n" "srshl v8.4s, v8.4s, v0.4s\n" "srshl v9.4s, v9.4s, v1.4s\n" "srshl v10.4s, v10.4s, v2.4s\n" "srshl v11.4s, v11.4s, v3.4s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x25]\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v21.4s }, [x20]\n" "srshl v12.4s, v12.4s, v0.4s\n" "srshl v13.4s, v13.4s, v1.4s\n" - "add x25, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x25]\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v20.4s }, [x20]\n" "srshl v14.4s, v14.4s, v2.4s\n" "srshl v15.4s, v15.4s, v3.4s\n" "cmp x10, #0x10\n" @@ -1160,132 +1159,132 @@ void a64_hybrid_s8qs_dot_6x16 ( "srshl v17.4s, v17.4s, v1.4s\n" "srshl v18.4s, v18.4s, v2.4s\n" "srshl v19.4s, v19.4s, v3.4s\n" - "add v8.4s, v8.4s, v4.4s\n" - "add v9.4s, v9.4s, v4.4s\n" - "add v10.4s, v10.4s, v4.4s\n" - "add v11.4s, v11.4s, v4.4s\n" - "add v12.4s, v12.4s, v4.4s\n" - "add v13.4s, v13.4s, v4.4s\n" - "add v14.4s, v14.4s, v4.4s\n" - "add v15.4s, v15.4s, v4.4s\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "smin v8.4s, v8.4s, v6.4s\n" - "smin v9.4s, v9.4s, v6.4s\n" - "smin v10.4s, v10.4s, v6.4s\n" - "smin v11.4s, v11.4s, v6.4s\n" - "smin v12.4s, v12.4s, v6.4s\n" - "smin v13.4s, v13.4s, v6.4s\n" - "smin v14.4s, v14.4s, v6.4s\n" - "smin v15.4s, v15.4s, v6.4s\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smax v8.4s, v8.4s, v5.4s\n" - "smax v9.4s, v9.4s, v5.4s\n" - "smax v10.4s, v10.4s, v5.4s\n" - "smax v11.4s, v11.4s, v5.4s\n" - "smax v12.4s, v12.4s, v5.4s\n" - "smax v13.4s, v13.4s, v5.4s\n" - "smax v14.4s, v14.4s, v5.4s\n" - "smax v15.4s, v15.4s, v5.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" + "add v8.4s, v8.4s, v22.4s\n" + "add v9.4s, v9.4s, v22.4s\n" + "add v10.4s, v10.4s, v22.4s\n" + "add v11.4s, v11.4s, v22.4s\n" + "add v12.4s, v12.4s, v22.4s\n" + "add v13.4s, v13.4s, v22.4s\n" + "add v14.4s, v14.4s, v22.4s\n" + "add v15.4s, v15.4s, v22.4s\n" + "add v16.4s, v16.4s, v22.4s\n" + "add v17.4s, v17.4s, v22.4s\n" + "add v18.4s, v18.4s, v22.4s\n" + "add v19.4s, v19.4s, v22.4s\n" + "smin v8.4s, v8.4s, v21.4s\n" + "smin v9.4s, v9.4s, v21.4s\n" + "smin v10.4s, v10.4s, v21.4s\n" + "smin v11.4s, v11.4s, v21.4s\n" + "smin v12.4s, v12.4s, v21.4s\n" + "smin v13.4s, v13.4s, v21.4s\n" + "smin v14.4s, v14.4s, v21.4s\n" + "smin v15.4s, v15.4s, v21.4s\n" + "smin v16.4s, v16.4s, v21.4s\n" + "smin v17.4s, v17.4s, v21.4s\n" + "smin v18.4s, v18.4s, v21.4s\n" + "smin v19.4s, v19.4s, v21.4s\n" + "smax v8.4s, v8.4s, v20.4s\n" + "smax v9.4s, v9.4s, v20.4s\n" + "smax v10.4s, v10.4s, v20.4s\n" + "smax v11.4s, v11.4s, v20.4s\n" + "smax v12.4s, v12.4s, v20.4s\n" + "smax v13.4s, v13.4s, v20.4s\n" + "smax v14.4s, v14.4s, v20.4s\n" + "smax v15.4s, v15.4s, v20.4s\n" + "smax v16.4s, v16.4s, v20.4s\n" + "smax v17.4s, v17.4s, v20.4s\n" + "smax v18.4s, v18.4s, v20.4s\n" + "smax v19.4s, v19.4s, v20.4s\n" "uzp1 v8.8h, v8.8h, v9.8h\n" - "uzp1 v9.8h, v10.8h, v11.8h\n" + "uzp1 v21.8h, v10.8h, v11.8h\n" "uzp1 v12.8h, v12.8h, v13.8h\n" - "uzp1 v13.8h, v14.8h, v15.8h\n" + "uzp1 v20.8h, v14.8h, v15.8h\n" "uzp1 v16.8h, v16.8h, v17.8h\n" "uzp1 v17.8h, v18.8h, v19.8h\n" - "uzp1 v8.16b, v8.16b, v9.16b\n" - "uzp1 v12.16b, v12.16b, v13.16b\n" + "uzp1 v8.16b, v8.16b, v21.16b\n" + "uzp1 v12.16b, v12.16b, v20.16b\n" "uzp1 v16.16b, v16.16b, v17.16b\n" "bge 80f\n" "tbz x10, #3, 75f\n" "str d8, [x11], #0x8\n" - "str d12, [x24], #0x8\n" - "str d16, [x23], #0x8\n" + "str d12, [x25], #0x8\n" + "str d16, [x24], #0x8\n" "tbz x10, #2, 73f\n" "st1 { v8.s }[2], [x11], #0x4\n" - "st1 { v12.s }[2], [x24], #0x4\n" - "st1 { v16.s }[2], [x23], #0x4\n" + "st1 { v12.s }[2], [x25], #0x4\n" + "st1 { v16.s }[2], [x24], #0x4\n" "tbz x10, #1, 72f\n" "st1 { v8.h }[6], [x11], #0x2\n" - "st1 { v12.h }[6], [x24], #0x2\n" - "st1 { v16.h }[6], [x23], #0x2\n" + "st1 { v12.h }[6], [x25], #0x2\n" + "st1 { v16.h }[6], [x24], #0x2\n" "tbz x10, #0, 79f\n" "st1 { v8.b }[14], [x11]\n" - "st1 { v12.b }[14], [x24]\n" - "st1 { v16.b }[14], [x23]\n" + "st1 { v12.b }[14], [x25]\n" + "st1 { v16.b }[14], [x24]\n" "b 79f\n" "72:" // Height 3: Partial direct writeback: partial_1_12 "tbz x10, #0, 79f\n" "st1 { v8.b }[12], [x11]\n" - "st1 { v12.b }[12], [x24]\n" - "st1 { v16.b }[12], [x23]\n" + "st1 { v12.b }[12], [x25]\n" + "st1 { v16.b }[12], [x24]\n" "b 79f\n" "73:" // Height 3: Partial direct writeback: partial_2_8 "tbz x10, #1, 74f\n" "st1 { v8.h }[4], [x11], #0x2\n" - "st1 { v12.h }[4], [x24], #0x2\n" - "st1 { v16.h }[4], [x23], #0x2\n" + "st1 { v12.h }[4], [x25], #0x2\n" + "st1 { v16.h }[4], [x24], #0x2\n" "tbz x10, #0, 79f\n" "st1 { v8.b }[10], [x11]\n" - "st1 { v12.b }[10], [x24]\n" - "st1 { v16.b }[10], [x23]\n" + "st1 { v12.b }[10], [x25]\n" + "st1 { v16.b }[10], [x24]\n" "b 79f\n" "74:" // Height 3: Partial direct writeback: partial_1_8 "tbz x10, #0, 79f\n" "st1 { v8.b }[8], [x11]\n" - "st1 { v12.b }[8], [x24]\n" - "st1 { v16.b }[8], [x23]\n" + "st1 { v12.b }[8], [x25]\n" + "st1 { v16.b }[8], [x24]\n" "b 79f\n" "75:" // Height 3: Partial direct writeback: partial_4_0 "tbz x10, #2, 77f\n" "str s8, [x11], #0x4\n" - "str s12, [x24], #0x4\n" - "str s16, [x23], #0x4\n" + "str s12, [x25], #0x4\n" + "str s16, [x24], #0x4\n" "tbz x10, #1, 76f\n" "st1 { v8.h }[2], [x11], #0x2\n" - "st1 { v12.h }[2], [x24], #0x2\n" - "st1 { v16.h }[2], [x23], #0x2\n" + "st1 { v12.h }[2], [x25], #0x2\n" + "st1 { v16.h }[2], [x24], #0x2\n" "tbz x10, #0, 79f\n" "st1 { v8.b }[6], [x11]\n" - "st1 { v12.b }[6], [x24]\n" - "st1 { v16.b }[6], [x23]\n" + "st1 { v12.b }[6], [x25]\n" + "st1 { v16.b }[6], [x24]\n" "b 79f\n" "76:" // Height 3: Partial direct writeback: partial_1_4 "tbz x10, #0, 79f\n" "st1 { v8.b }[4], [x11]\n" - "st1 { v12.b }[4], [x24]\n" - "st1 { v16.b }[4], [x23]\n" + "st1 { v12.b }[4], [x25]\n" + "st1 { v16.b }[4], [x24]\n" "b 79f\n" "77:" // Height 3: Partial direct writeback: partial_2_0 "tbz x10, #1, 78f\n" "str h8, [x11], #0x2\n" - "str h12, [x24], #0x2\n" - "str h16, [x23], #0x2\n" + "str h12, [x25], #0x2\n" + "str h16, [x24], #0x2\n" "tbz x10, #0, 79f\n" "st1 { v8.b }[2], [x11]\n" - "st1 { v12.b }[2], [x24]\n" - "st1 { v16.b }[2], [x23]\n" + "st1 { v12.b }[2], [x25]\n" + "st1 { v16.b }[2], [x24]\n" "b 79f\n" "78:" // Height 3: Partial direct writeback: partial_1_0 "str b8, [x11, #0x0]\n" - "str b12, [x24, #0x0]\n" - "str b16, [x23, #0x0]\n" + "str b12, [x25, #0x0]\n" + "str b16, [x24, #0x0]\n" "79:" // Height 3: Partial direct writeback: Done "b 81f\n" "80:" // Height 3: Full writeback "str q8, [x11, #0x0]\n" "add x11, x11, #0x10\n" - "str q12, [x24, #0x0]\n" - "str q16, [x23, #0x0]\n" + "str q12, [x25, #0x0]\n" + "str q16, [x24, #0x0]\n" "81:" // Height 3: Writeback done "subs x10, x10, #0x10\n" "bgt 56b\n" @@ -1319,14 +1318,14 @@ void a64_hybrid_s8qs_dot_6x16 ( "85:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 86f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 87f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -1336,9 +1335,9 @@ void a64_hybrid_s8qs_dot_6x16 ( "b 87f\n" "86:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" "87:" // Height 4: input setup done "cmp x27, #0x10\n" "blt 90f\n" @@ -1357,7 +1356,7 @@ void a64_hybrid_s8qs_dot_6x16 ( "add x26, x26, #0x10\n" ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x9, #0x20]\n" + "ldr q25, [x9, #0x20]\n" "add x25, x25, #0x10\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" @@ -1365,85 +1364,85 @@ void a64_hybrid_s8qs_dot_6x16 ( "add x23, x23, #0x10\n" ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x9, #0x30]\n" + "ldr q24, [x9, #0x30]\n" "cmp x27, #0x20\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" + ".inst 0x4f80e32a // sdot v10.4s, v25.16b, v0.4b[0]\n" + ".inst 0x4f81e32e // sdot v14.4s, v25.16b, v1.4b[0]\n" "prfm pldl1keep, [x26, #0x80]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x9, #0x40]\n" + ".inst 0x4f82e332 // sdot v18.4s, v25.16b, v2.4b[0]\n" + ".inst 0x4f83e336 // sdot v22.4s, v25.16b, v3.4b[0]\n" + "ldr q25, [x9, #0x40]\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" + ".inst 0x4f80e30b // sdot v11.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f81e30f // sdot v15.4s, v24.16b, v1.4b[0]\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x9, #0x50]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" - ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n" - "ldr q6, [x9, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n" - "ldr q7, [x9, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n" - "ldr q6, [x9, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" - ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n" - "ldr q7, [x9, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n" - "ldr q6, [x9, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n" - "ldr q7, [x9, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n" - "ldr q6, [x9, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n" - "ldr q7, [x9, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n" - "ldr q6, [x9, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" - ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n" - "ldr q7, [x9, #0xf0]\n" + ".inst 0x4f82e313 // sdot v19.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f83e317 // sdot v23.4s, v24.16b, v3.4b[0]\n" + "ldr q24, [x9, #0x50]\n" + ".inst 0x4fa0e328 // sdot v8.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa1e32c // sdot v12.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa2e330 // sdot v16.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa3e334 // sdot v20.4s, v25.16b, v3.4b[1]\n" + "ldr q25, [x9, #0x60]\n" + ".inst 0x4fa0e309 // sdot v9.4s, v24.16b, v0.4b[1]\n" + ".inst 0x4fa1e30d // sdot v13.4s, v24.16b, v1.4b[1]\n" + ".inst 0x4fa2e311 // sdot v17.4s, v24.16b, v2.4b[1]\n" + ".inst 0x4fa3e315 // sdot v21.4s, v24.16b, v3.4b[1]\n" + "ldr q24, [x9, #0x70]\n" + ".inst 0x4fa0e32a // sdot v10.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa1e32e // sdot v14.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa2e332 // sdot v18.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa3e336 // sdot v22.4s, v25.16b, v3.4b[1]\n" + "ldr q25, [x9, #0x80]\n" + ".inst 0x4fa0e30b // sdot v11.4s, v24.16b, v0.4b[1]\n" + ".inst 0x4fa1e30f // sdot v15.4s, v24.16b, v1.4b[1]\n" + ".inst 0x4fa2e313 // sdot v19.4s, v24.16b, v2.4b[1]\n" + ".inst 0x4fa3e317 // sdot v23.4s, v24.16b, v3.4b[1]\n" + "ldr q24, [x9, #0x90]\n" + ".inst 0x4f80eb28 // sdot v8.4s, v25.16b, v0.4b[2]\n" + ".inst 0x4f81eb2c // sdot v12.4s, v25.16b, v1.4b[2]\n" + ".inst 0x4f82eb30 // sdot v16.4s, v25.16b, v2.4b[2]\n" + ".inst 0x4f83eb34 // sdot v20.4s, v25.16b, v3.4b[2]\n" + "ldr q25, [x9, #0xa0]\n" + ".inst 0x4f80eb09 // sdot v9.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f81eb0d // sdot v13.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f82eb11 // sdot v17.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f83eb15 // sdot v21.4s, v24.16b, v3.4b[2]\n" + "ldr q24, [x9, #0xb0]\n" + ".inst 0x4f80eb2a // sdot v10.4s, v25.16b, v0.4b[2]\n" + ".inst 0x4f81eb2e // sdot v14.4s, v25.16b, v1.4b[2]\n" + ".inst 0x4f82eb32 // sdot v18.4s, v25.16b, v2.4b[2]\n" + ".inst 0x4f83eb36 // sdot v22.4s, v25.16b, v3.4b[2]\n" + "ldr q25, [x9, #0xc0]\n" + ".inst 0x4f80eb0b // sdot v11.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f81eb0f // sdot v15.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f82eb13 // sdot v19.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f83eb17 // sdot v23.4s, v24.16b, v3.4b[2]\n" + "ldr q24, [x9, #0xd0]\n" + ".inst 0x4fa0eb28 // sdot v8.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa1eb2c // sdot v12.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa2eb30 // sdot v16.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa3eb34 // sdot v20.4s, v25.16b, v3.4b[3]\n" + "ldr q25, [x9, #0xe0]\n" + ".inst 0x4fa0eb09 // sdot v9.4s, v24.16b, v0.4b[3]\n" + ".inst 0x4fa1eb0d // sdot v13.4s, v24.16b, v1.4b[3]\n" + ".inst 0x4fa2eb11 // sdot v17.4s, v24.16b, v2.4b[3]\n" + ".inst 0x4fa3eb15 // sdot v21.4s, v24.16b, v3.4b[3]\n" + "ldr q24, [x9, #0xf0]\n" "add x9, x9, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n" + ".inst 0x4fa0eb2a // sdot v10.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa1eb2e // sdot v14.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa2eb32 // sdot v18.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa3eb36 // sdot v22.4s, v25.16b, v3.4b[3]\n" "ldr q6, [x9, #0x0]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x4fa0eb0b // sdot v11.4s, v24.16b, v0.4b[3]\n" "ldr q0, [x26, #0x0]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" + ".inst 0x4fa1eb0f // sdot v15.4s, v24.16b, v1.4b[3]\n" "ldr q1, [x25, #0x0]\n" - ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" + ".inst 0x4fa2eb13 // sdot v19.4s, v24.16b, v2.4b[3]\n" "ldr q2, [x24, #0x0]\n" - ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n" + ".inst 0x4fa3eb17 // sdot v23.4s, v24.16b, v3.4b[3]\n" "ldr q3, [x23, #0x0]\n" "ldr q7, [x9, #0x10]\n" "bge 88b\n" @@ -1454,7 +1453,7 @@ void a64_hybrid_s8qs_dot_6x16 ( "add x25, x25, #0x10\n" ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x9, #0x20]\n" + "ldr q25, [x9, #0x20]\n" "add x24, x24, #0x10\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" @@ -1462,112 +1461,112 @@ void a64_hybrid_s8qs_dot_6x16 ( "sub x27, x27, #0x10\n" ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x9, #0x30]\n" + "ldr q24, [x9, #0x30]\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" + ".inst 0x4f80e32a // sdot v10.4s, v25.16b, v0.4b[0]\n" + ".inst 0x4f81e32e // sdot v14.4s, v25.16b, v1.4b[0]\n" "prfm pldl1keep, [x25, #0x80]\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x9, #0x40]\n" + ".inst 0x4f82e332 // sdot v18.4s, v25.16b, v2.4b[0]\n" + ".inst 0x4f83e336 // sdot v22.4s, v25.16b, v3.4b[0]\n" + "ldr q25, [x9, #0x40]\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x9, #0x50]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" - ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n" - "ldr q6, [x9, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n" - "ldr q7, [x9, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n" - "ldr q6, [x9, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" - ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n" - "ldr q7, [x9, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n" - "ldr q6, [x9, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n" - "ldr q7, [x9, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n" - "ldr q6, [x9, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n" - "ldr q7, [x9, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n" - "ldr q6, [x9, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" - ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n" - "ldr q7, [x9, #0xf0]\n" + ".inst 0x4f80e30b // sdot v11.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f81e30f // sdot v15.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f82e313 // sdot v19.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f83e317 // sdot v23.4s, v24.16b, v3.4b[0]\n" + "ldr q24, [x9, #0x50]\n" + ".inst 0x4fa0e328 // sdot v8.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa1e32c // sdot v12.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa2e330 // sdot v16.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa3e334 // sdot v20.4s, v25.16b, v3.4b[1]\n" + "ldr q25, [x9, #0x60]\n" + ".inst 0x4fa0e309 // sdot v9.4s, v24.16b, v0.4b[1]\n" + ".inst 0x4fa1e30d // sdot v13.4s, v24.16b, v1.4b[1]\n" + ".inst 0x4fa2e311 // sdot v17.4s, v24.16b, v2.4b[1]\n" + ".inst 0x4fa3e315 // sdot v21.4s, v24.16b, v3.4b[1]\n" + "ldr q24, [x9, #0x70]\n" + ".inst 0x4fa0e32a // sdot v10.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa1e32e // sdot v14.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa2e332 // sdot v18.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa3e336 // sdot v22.4s, v25.16b, v3.4b[1]\n" + "ldr q25, [x9, #0x80]\n" + ".inst 0x4fa0e30b // sdot v11.4s, v24.16b, v0.4b[1]\n" + ".inst 0x4fa1e30f // sdot v15.4s, v24.16b, v1.4b[1]\n" + ".inst 0x4fa2e313 // sdot v19.4s, v24.16b, v2.4b[1]\n" + ".inst 0x4fa3e317 // sdot v23.4s, v24.16b, v3.4b[1]\n" + "ldr q24, [x9, #0x90]\n" + ".inst 0x4f80eb28 // sdot v8.4s, v25.16b, v0.4b[2]\n" + ".inst 0x4f81eb2c // sdot v12.4s, v25.16b, v1.4b[2]\n" + ".inst 0x4f82eb30 // sdot v16.4s, v25.16b, v2.4b[2]\n" + ".inst 0x4f83eb34 // sdot v20.4s, v25.16b, v3.4b[2]\n" + "ldr q25, [x9, #0xa0]\n" + ".inst 0x4f80eb09 // sdot v9.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f81eb0d // sdot v13.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f82eb11 // sdot v17.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f83eb15 // sdot v21.4s, v24.16b, v3.4b[2]\n" + "ldr q24, [x9, #0xb0]\n" + ".inst 0x4f80eb2a // sdot v10.4s, v25.16b, v0.4b[2]\n" + ".inst 0x4f81eb2e // sdot v14.4s, v25.16b, v1.4b[2]\n" + ".inst 0x4f82eb32 // sdot v18.4s, v25.16b, v2.4b[2]\n" + ".inst 0x4f83eb36 // sdot v22.4s, v25.16b, v3.4b[2]\n" + "ldr q25, [x9, #0xc0]\n" + ".inst 0x4f80eb0b // sdot v11.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f81eb0f // sdot v15.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f82eb13 // sdot v19.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f83eb17 // sdot v23.4s, v24.16b, v3.4b[2]\n" + "ldr q24, [x9, #0xd0]\n" + ".inst 0x4fa0eb28 // sdot v8.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa1eb2c // sdot v12.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa2eb30 // sdot v16.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa3eb34 // sdot v20.4s, v25.16b, v3.4b[3]\n" + "ldr q25, [x9, #0xe0]\n" + ".inst 0x4fa0eb09 // sdot v9.4s, v24.16b, v0.4b[3]\n" + ".inst 0x4fa1eb0d // sdot v13.4s, v24.16b, v1.4b[3]\n" + ".inst 0x4fa2eb11 // sdot v17.4s, v24.16b, v2.4b[3]\n" + ".inst 0x4fa3eb15 // sdot v21.4s, v24.16b, v3.4b[3]\n" + "ldr q24, [x9, #0xf0]\n" "add x9, x9, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" - ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n" + ".inst 0x4fa0eb2a // sdot v10.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa1eb2e // sdot v14.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa2eb32 // sdot v18.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa3eb36 // sdot v22.4s, v25.16b, v3.4b[3]\n" + ".inst 0x4fa0eb0b // sdot v11.4s, v24.16b, v0.4b[3]\n" + ".inst 0x4fa1eb0f // sdot v15.4s, v24.16b, v1.4b[3]\n" + ".inst 0x4fa2eb13 // sdot v19.4s, v24.16b, v2.4b[3]\n" + ".inst 0x4fa3eb17 // sdot v23.4s, v24.16b, v3.4b[3]\n" "90:" // Height 4: Multiply loop: Main loop skip "cbz x27, 95f\n" "cmp x27, #0x4\n" "blt 92f\n" "91:" // Height 4: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s29, [x26], #0x4\n" + "ldr s28, [x25], #0x4\n" "sub x27, x27, #0x4\n" "cmp x27, #0x4\n" - "ldr s2, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr q6, [x9, #0x0]\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x9, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x9, #0x30]\n" + "ldr s27, [x24], #0x4\n" + "ldr s26, [x23], #0x4\n" + "ldr q25, [x9, #0x0]\n" + "ldr q24, [x9, #0x10]\n" + ".inst 0x4f9de328 // sdot v8.4s, v25.16b, v29.4b[0]\n" + ".inst 0x4f9ce32c // sdot v12.4s, v25.16b, v28.4b[0]\n" + ".inst 0x4f9be330 // sdot v16.4s, v25.16b, v27.4b[0]\n" + ".inst 0x4f9ae334 // sdot v20.4s, v25.16b, v26.4b[0]\n" + "ldr q25, [x9, #0x20]\n" + ".inst 0x4f9de309 // sdot v9.4s, v24.16b, v29.4b[0]\n" + ".inst 0x4f9ce30d // sdot v13.4s, v24.16b, v28.4b[0]\n" + ".inst 0x4f9be311 // sdot v17.4s, v24.16b, v27.4b[0]\n" + ".inst 0x4f9ae315 // sdot v21.4s, v24.16b, v26.4b[0]\n" + "ldr q24, [x9, #0x30]\n" "add x9, x9, #0x40\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" + ".inst 0x4f9de32a // sdot v10.4s, v25.16b, v29.4b[0]\n" + ".inst 0x4f9ce32e // sdot v14.4s, v25.16b, v28.4b[0]\n" + ".inst 0x4f9be332 // sdot v18.4s, v25.16b, v27.4b[0]\n" + ".inst 0x4f9ae336 // sdot v22.4s, v25.16b, v26.4b[0]\n" + ".inst 0x4f9de30b // sdot v11.4s, v24.16b, v29.4b[0]\n" + ".inst 0x4f9ce30f // sdot v15.4s, v24.16b, v28.4b[0]\n" + ".inst 0x4f9be313 // sdot v19.4s, v24.16b, v27.4b[0]\n" + ".inst 0x4f9ae317 // sdot v23.4s, v24.16b, v26.4b[0]\n" "bge 91b\n" "92:" // Height 4: Multiply loop: Skip odd blocks "cbz x27, 95f\n" @@ -1588,61 +1587,61 @@ void a64_hybrid_s8qs_dot_6x16 ( "ldr b2, [x24, #0x0]\n" "ldr b3, [x23, #0x0]\n" "94:" // Height 4: Multiply loop: Ragged operand read: Done - "ldr q6, [x9, #0x0]\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x9, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x9, #0x30]\n" + "ldr q25, [x9, #0x0]\n" + "ldr q24, [x9, #0x10]\n" + ".inst 0x4f80e328 // sdot v8.4s, v25.16b, v0.4b[0]\n" + ".inst 0x4f81e32c // sdot v12.4s, v25.16b, v1.4b[0]\n" + ".inst 0x4f82e330 // sdot v16.4s, v25.16b, v2.4b[0]\n" + ".inst 0x4f83e334 // sdot v20.4s, v25.16b, v3.4b[0]\n" + "ldr q25, [x9, #0x20]\n" + ".inst 0x4f80e309 // sdot v9.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f81e30d // sdot v13.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f82e311 // sdot v17.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f83e315 // sdot v21.4s, v24.16b, v3.4b[0]\n" + "ldr q24, [x9, #0x30]\n" "add x9, x9, #0x40\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" + ".inst 0x4f80e32a // sdot v10.4s, v25.16b, v0.4b[0]\n" + ".inst 0x4f81e32e // sdot v14.4s, v25.16b, v1.4b[0]\n" + ".inst 0x4f82e332 // sdot v18.4s, v25.16b, v2.4b[0]\n" + ".inst 0x4f83e336 // sdot v22.4s, v25.16b, v3.4b[0]\n" + ".inst 0x4f80e30b // sdot v11.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f81e30f // sdot v15.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f82e313 // sdot v19.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f83e317 // sdot v23.4s, v24.16b, v3.4b[0]\n" "95:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 85b\n" - "ldr q0, [x14, #0x0]\n" - "ldr q1, [x14, #0x10]\n" - "add v8.4s, v8.4s, v0.4s\n" - "add v9.4s, v9.4s, v1.4s\n" - "ldr q2, [x14, #0x20]\n" - "ldr q3, [x14, #0x30]\n" - "add v10.4s, v10.4s, v2.4s\n" - "add v11.4s, v11.4s, v3.4s\n" + "ldr q27, [x14, #0x0]\n" + "ldr q26, [x14, #0x10]\n" + "add v8.4s, v8.4s, v27.4s\n" + "add v9.4s, v9.4s, v26.4s\n" + "ldr q25, [x14, #0x20]\n" + "ldr q24, [x14, #0x30]\n" + "add v10.4s, v10.4s, v25.4s\n" + "add v11.4s, v11.4s, v24.4s\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x11, x20\n" - "add x23, x24, x20\n" + "add x25, x11, x20\n" + "add x24, x25, x20\n" "prfm pstl1keep, [x11, #0x0]\n" - "add x22, x23, x20\n" + "add x23, x24, x20\n" + "prfm pstl1keep, [x25, #0x0]\n" "prfm pstl1keep, [x24, #0x0]\n" + "add v12.4s, v12.4s, v27.4s\n" "prfm pstl1keep, [x23, #0x0]\n" - "add v12.4s, v12.4s, v0.4s\n" - "prfm pstl1keep, [x22, #0x0]\n" - "add v13.4s, v13.4s, v1.4s\n" - "add v14.4s, v14.4s, v2.4s\n" + "add v13.4s, v13.4s, v26.4s\n" + "add v14.4s, v14.4s, v25.4s\n" "add x14, x14, #0x40\n" - "add v15.4s, v15.4s, v3.4s\n" - "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" - "add v20.4s, v20.4s, v0.4s\n" - "add v21.4s, v21.4s, v1.4s\n" - "add v22.4s, v22.4s, v2.4s\n" - "add v23.4s, v23.4s, v3.4s\n" + "add v15.4s, v15.4s, v24.4s\n" + "add v16.4s, v16.4s, v27.4s\n" + "add v17.4s, v17.4s, v26.4s\n" + "add v18.4s, v18.4s, v25.4s\n" + "add v19.4s, v19.4s, v24.4s\n" + "add v20.4s, v20.4s, v27.4s\n" + "add v21.4s, v21.4s, v26.4s\n" + "add v22.4s, v22.4s, v25.4s\n" + "add v23.4s, v23.4s, v24.4s\n" "tbz %x[flags], #4, 96f\n" "ldr q0, [x12, #0x0]\n" "ldr q4, [x13, #0x0]\n" @@ -1656,10 +1655,10 @@ void a64_hybrid_s8qs_dot_6x16 ( "add x13, x13, #0x40\n" "b 97f\n" "96:" // Height 4: per layer parameters - "add x25, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x25]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v4.4s }, [x20]\n" "mov v1.16b, v0.16b\n" "mov v5.16b, v4.16b\n" "mov v2.16b, v0.16b\n" @@ -1684,67 +1683,67 @@ void a64_hybrid_s8qs_dot_6x16 ( "sqrdmulh v22.4s, v22.4s, v6.4s\n" "sqrdmulh v23.4s, v23.4s, v7.4s\n" "tbz %x[flags], #5, 98f\n" - "and v4.16b, v8.16b, v0.16b\n" - "and v5.16b, v9.16b, v1.16b\n" - "and v6.16b, v10.16b, v2.16b\n" - "and v7.16b, v11.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v4.4s\n" - "sqadd v9.4s, v9.4s, v5.4s\n" - "sqadd v10.4s, v10.4s, v6.4s\n" - "sqadd v11.4s, v11.4s, v7.4s\n" - "and v4.16b, v12.16b, v0.16b\n" - "and v5.16b, v13.16b, v1.16b\n" - "and v6.16b, v14.16b, v2.16b\n" - "and v7.16b, v15.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v12.4s, v12.4s, v4.4s\n" - "sqadd v13.4s, v13.4s, v5.4s\n" - "sqadd v14.4s, v14.4s, v6.4s\n" - "sqadd v15.4s, v15.4s, v7.4s\n" - "and v4.16b, v16.16b, v0.16b\n" - "and v5.16b, v17.16b, v1.16b\n" - "and v6.16b, v18.16b, v2.16b\n" - "and v7.16b, v19.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" - "and v4.16b, v20.16b, v0.16b\n" - "and v5.16b, v21.16b, v1.16b\n" - "and v6.16b, v22.16b, v2.16b\n" - "and v7.16b, v23.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v20.4s, v20.4s, v4.4s\n" - "sqadd v21.4s, v21.4s, v5.4s\n" - "sqadd v22.4s, v22.4s, v6.4s\n" - "sqadd v23.4s, v23.4s, v7.4s\n" + "and v27.16b, v8.16b, v0.16b\n" + "and v26.16b, v9.16b, v1.16b\n" + "and v25.16b, v10.16b, v2.16b\n" + "and v24.16b, v11.16b, v3.16b\n" + "sshr v27.4s, v27.4s, #0x1f\n" + "sshr v26.4s, v26.4s, #0x1f\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sqadd v8.4s, v8.4s, v27.4s\n" + "sqadd v9.4s, v9.4s, v26.4s\n" + "sqadd v10.4s, v10.4s, v25.4s\n" + "sqadd v11.4s, v11.4s, v24.4s\n" + "and v27.16b, v12.16b, v0.16b\n" + "and v26.16b, v13.16b, v1.16b\n" + "and v25.16b, v14.16b, v2.16b\n" + "and v24.16b, v15.16b, v3.16b\n" + "sshr v27.4s, v27.4s, #0x1f\n" + "sshr v26.4s, v26.4s, #0x1f\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sqadd v12.4s, v12.4s, v27.4s\n" + "sqadd v13.4s, v13.4s, v26.4s\n" + "sqadd v14.4s, v14.4s, v25.4s\n" + "sqadd v15.4s, v15.4s, v24.4s\n" + "and v27.16b, v16.16b, v0.16b\n" + "and v26.16b, v17.16b, v1.16b\n" + "and v25.16b, v18.16b, v2.16b\n" + "and v24.16b, v19.16b, v3.16b\n" + "sshr v27.4s, v27.4s, #0x1f\n" + "sshr v26.4s, v26.4s, #0x1f\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v27.4s\n" + "sqadd v17.4s, v17.4s, v26.4s\n" + "sqadd v18.4s, v18.4s, v25.4s\n" + "sqadd v19.4s, v19.4s, v24.4s\n" + "and v27.16b, v20.16b, v0.16b\n" + "and v26.16b, v21.16b, v1.16b\n" + "and v25.16b, v22.16b, v2.16b\n" + "and v24.16b, v23.16b, v3.16b\n" + "sshr v27.4s, v27.4s, #0x1f\n" + "sshr v26.4s, v26.4s, #0x1f\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v27.4s\n" + "sqadd v21.4s, v21.4s, v26.4s\n" + "sqadd v22.4s, v22.4s, v25.4s\n" + "sqadd v23.4s, v23.4s, v24.4s\n" "98:" // Height 4: no shift correction - "add x25, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v26.4s }, [x20]\n" "srshl v8.4s, v8.4s, v0.4s\n" "srshl v9.4s, v9.4s, v1.4s\n" "srshl v10.4s, v10.4s, v2.4s\n" "srshl v11.4s, v11.4s, v3.4s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x25]\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v25.4s }, [x20]\n" "srshl v12.4s, v12.4s, v0.4s\n" "srshl v13.4s, v13.4s, v1.4s\n" - "add x25, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x25]\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v24.4s }, [x20]\n" "srshl v14.4s, v14.4s, v2.4s\n" "srshl v15.4s, v15.4s, v3.4s\n" "cmp x10, #0x10\n" @@ -1756,163 +1755,163 @@ void a64_hybrid_s8qs_dot_6x16 ( "srshl v21.4s, v21.4s, v1.4s\n" "srshl v22.4s, v22.4s, v2.4s\n" "srshl v23.4s, v23.4s, v3.4s\n" - "add v8.4s, v8.4s, v4.4s\n" - "add v9.4s, v9.4s, v4.4s\n" - "add v10.4s, v10.4s, v4.4s\n" - "add v11.4s, v11.4s, v4.4s\n" - "add v12.4s, v12.4s, v4.4s\n" - "add v13.4s, v13.4s, v4.4s\n" - "add v14.4s, v14.4s, v4.4s\n" - "add v15.4s, v15.4s, v4.4s\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v23.4s, v23.4s, v4.4s\n" - "smin v8.4s, v8.4s, v6.4s\n" - "smin v9.4s, v9.4s, v6.4s\n" - "smin v10.4s, v10.4s, v6.4s\n" - "smin v11.4s, v11.4s, v6.4s\n" - "smin v12.4s, v12.4s, v6.4s\n" - "smin v13.4s, v13.4s, v6.4s\n" - "smin v14.4s, v14.4s, v6.4s\n" - "smin v15.4s, v15.4s, v6.4s\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "smax v8.4s, v8.4s, v5.4s\n" - "smax v9.4s, v9.4s, v5.4s\n" - "smax v10.4s, v10.4s, v5.4s\n" - "smax v11.4s, v11.4s, v5.4s\n" - "smax v12.4s, v12.4s, v5.4s\n" - "smax v13.4s, v13.4s, v5.4s\n" - "smax v14.4s, v14.4s, v5.4s\n" - "smax v15.4s, v15.4s, v5.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" + "add v8.4s, v8.4s, v26.4s\n" + "add v9.4s, v9.4s, v26.4s\n" + "add v10.4s, v10.4s, v26.4s\n" + "add v11.4s, v11.4s, v26.4s\n" + "add v12.4s, v12.4s, v26.4s\n" + "add v13.4s, v13.4s, v26.4s\n" + "add v14.4s, v14.4s, v26.4s\n" + "add v15.4s, v15.4s, v26.4s\n" + "add v16.4s, v16.4s, v26.4s\n" + "add v17.4s, v17.4s, v26.4s\n" + "add v18.4s, v18.4s, v26.4s\n" + "add v19.4s, v19.4s, v26.4s\n" + "add v20.4s, v20.4s, v26.4s\n" + "add v21.4s, v21.4s, v26.4s\n" + "add v22.4s, v22.4s, v26.4s\n" + "add v23.4s, v23.4s, v26.4s\n" + "smin v8.4s, v8.4s, v25.4s\n" + "smin v9.4s, v9.4s, v25.4s\n" + "smin v10.4s, v10.4s, v25.4s\n" + "smin v11.4s, v11.4s, v25.4s\n" + "smin v12.4s, v12.4s, v25.4s\n" + "smin v13.4s, v13.4s, v25.4s\n" + "smin v14.4s, v14.4s, v25.4s\n" + "smin v15.4s, v15.4s, v25.4s\n" + "smin v16.4s, v16.4s, v25.4s\n" + "smin v17.4s, v17.4s, v25.4s\n" + "smin v18.4s, v18.4s, v25.4s\n" + "smin v19.4s, v19.4s, v25.4s\n" + "smin v20.4s, v20.4s, v25.4s\n" + "smin v21.4s, v21.4s, v25.4s\n" + "smin v22.4s, v22.4s, v25.4s\n" + "smin v23.4s, v23.4s, v25.4s\n" + "smax v8.4s, v8.4s, v24.4s\n" + "smax v9.4s, v9.4s, v24.4s\n" + "smax v10.4s, v10.4s, v24.4s\n" + "smax v11.4s, v11.4s, v24.4s\n" + "smax v12.4s, v12.4s, v24.4s\n" + "smax v13.4s, v13.4s, v24.4s\n" + "smax v14.4s, v14.4s, v24.4s\n" + "smax v15.4s, v15.4s, v24.4s\n" + "smax v16.4s, v16.4s, v24.4s\n" + "smax v17.4s, v17.4s, v24.4s\n" + "smax v18.4s, v18.4s, v24.4s\n" + "smax v19.4s, v19.4s, v24.4s\n" + "smax v20.4s, v20.4s, v24.4s\n" + "smax v21.4s, v21.4s, v24.4s\n" + "smax v22.4s, v22.4s, v24.4s\n" + "smax v23.4s, v23.4s, v24.4s\n" "uzp1 v8.8h, v8.8h, v9.8h\n" - "uzp1 v9.8h, v10.8h, v11.8h\n" + "uzp1 v25.8h, v10.8h, v11.8h\n" "uzp1 v12.8h, v12.8h, v13.8h\n" - "uzp1 v13.8h, v14.8h, v15.8h\n" + "uzp1 v24.8h, v14.8h, v15.8h\n" "uzp1 v16.8h, v16.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v18.8h, v18.8h, v19.8h\n" "uzp1 v20.8h, v20.8h, v21.8h\n" - "uzp1 v21.8h, v22.8h, v23.8h\n" - "uzp1 v8.16b, v8.16b, v9.16b\n" - "uzp1 v12.16b, v12.16b, v13.16b\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" - "uzp1 v20.16b, v20.16b, v21.16b\n" + "uzp1 v17.8h, v22.8h, v23.8h\n" + "uzp1 v8.16b, v8.16b, v25.16b\n" + "uzp1 v12.16b, v12.16b, v24.16b\n" + "uzp1 v16.16b, v16.16b, v18.16b\n" + "uzp1 v20.16b, v20.16b, v17.16b\n" "bge 107f\n" "tbz x10, #3, 102f\n" "str d8, [x11], #0x8\n" - "str d12, [x24], #0x8\n" - "str d16, [x23], #0x8\n" - "str d20, [x22], #0x8\n" + "str d12, [x25], #0x8\n" + "str d16, [x24], #0x8\n" + "str d20, [x23], #0x8\n" "tbz x10, #2, 100f\n" "st1 { v8.s }[2], [x11], #0x4\n" - "st1 { v12.s }[2], [x24], #0x4\n" - "st1 { v16.s }[2], [x23], #0x4\n" - "st1 { v20.s }[2], [x22], #0x4\n" + "st1 { v12.s }[2], [x25], #0x4\n" + "st1 { v16.s }[2], [x24], #0x4\n" + "st1 { v20.s }[2], [x23], #0x4\n" "tbz x10, #1, 99f\n" "st1 { v8.h }[6], [x11], #0x2\n" - "st1 { v12.h }[6], [x24], #0x2\n" - "st1 { v16.h }[6], [x23], #0x2\n" - "st1 { v20.h }[6], [x22], #0x2\n" + "st1 { v12.h }[6], [x25], #0x2\n" + "st1 { v16.h }[6], [x24], #0x2\n" + "st1 { v20.h }[6], [x23], #0x2\n" "tbz x10, #0, 106f\n" "st1 { v8.b }[14], [x11]\n" - "st1 { v12.b }[14], [x24]\n" - "st1 { v16.b }[14], [x23]\n" - "st1 { v20.b }[14], [x22]\n" + "st1 { v12.b }[14], [x25]\n" + "st1 { v16.b }[14], [x24]\n" + "st1 { v20.b }[14], [x23]\n" "b 106f\n" "99:" // Height 4: Partial direct writeback: partial_1_12 "tbz x10, #0, 106f\n" "st1 { v8.b }[12], [x11]\n" - "st1 { v12.b }[12], [x24]\n" - "st1 { v16.b }[12], [x23]\n" - "st1 { v20.b }[12], [x22]\n" + "st1 { v12.b }[12], [x25]\n" + "st1 { v16.b }[12], [x24]\n" + "st1 { v20.b }[12], [x23]\n" "b 106f\n" "100:" // Height 4: Partial direct writeback: partial_2_8 "tbz x10, #1, 101f\n" "st1 { v8.h }[4], [x11], #0x2\n" - "st1 { v12.h }[4], [x24], #0x2\n" - "st1 { v16.h }[4], [x23], #0x2\n" - "st1 { v20.h }[4], [x22], #0x2\n" + "st1 { v12.h }[4], [x25], #0x2\n" + "st1 { v16.h }[4], [x24], #0x2\n" + "st1 { v20.h }[4], [x23], #0x2\n" "tbz x10, #0, 106f\n" "st1 { v8.b }[10], [x11]\n" - "st1 { v12.b }[10], [x24]\n" - "st1 { v16.b }[10], [x23]\n" - "st1 { v20.b }[10], [x22]\n" + "st1 { v12.b }[10], [x25]\n" + "st1 { v16.b }[10], [x24]\n" + "st1 { v20.b }[10], [x23]\n" "b 106f\n" "101:" // Height 4: Partial direct writeback: partial_1_8 "tbz x10, #0, 106f\n" "st1 { v8.b }[8], [x11]\n" - "st1 { v12.b }[8], [x24]\n" - "st1 { v16.b }[8], [x23]\n" - "st1 { v20.b }[8], [x22]\n" + "st1 { v12.b }[8], [x25]\n" + "st1 { v16.b }[8], [x24]\n" + "st1 { v20.b }[8], [x23]\n" "b 106f\n" "102:" // Height 4: Partial direct writeback: partial_4_0 "tbz x10, #2, 104f\n" "str s8, [x11], #0x4\n" - "str s12, [x24], #0x4\n" - "str s16, [x23], #0x4\n" - "str s20, [x22], #0x4\n" + "str s12, [x25], #0x4\n" + "str s16, [x24], #0x4\n" + "str s20, [x23], #0x4\n" "tbz x10, #1, 103f\n" "st1 { v8.h }[2], [x11], #0x2\n" - "st1 { v12.h }[2], [x24], #0x2\n" - "st1 { v16.h }[2], [x23], #0x2\n" - "st1 { v20.h }[2], [x22], #0x2\n" + "st1 { v12.h }[2], [x25], #0x2\n" + "st1 { v16.h }[2], [x24], #0x2\n" + "st1 { v20.h }[2], [x23], #0x2\n" "tbz x10, #0, 106f\n" "st1 { v8.b }[6], [x11]\n" - "st1 { v12.b }[6], [x24]\n" - "st1 { v16.b }[6], [x23]\n" - "st1 { v20.b }[6], [x22]\n" + "st1 { v12.b }[6], [x25]\n" + "st1 { v16.b }[6], [x24]\n" + "st1 { v20.b }[6], [x23]\n" "b 106f\n" "103:" // Height 4: Partial direct writeback: partial_1_4 "tbz x10, #0, 106f\n" "st1 { v8.b }[4], [x11]\n" - "st1 { v12.b }[4], [x24]\n" - "st1 { v16.b }[4], [x23]\n" - "st1 { v20.b }[4], [x22]\n" + "st1 { v12.b }[4], [x25]\n" + "st1 { v16.b }[4], [x24]\n" + "st1 { v20.b }[4], [x23]\n" "b 106f\n" "104:" // Height 4: Partial direct writeback: partial_2_0 "tbz x10, #1, 105f\n" "str h8, [x11], #0x2\n" - "str h12, [x24], #0x2\n" - "str h16, [x23], #0x2\n" - "str h20, [x22], #0x2\n" + "str h12, [x25], #0x2\n" + "str h16, [x24], #0x2\n" + "str h20, [x23], #0x2\n" "tbz x10, #0, 106f\n" "st1 { v8.b }[2], [x11]\n" - "st1 { v12.b }[2], [x24]\n" - "st1 { v16.b }[2], [x23]\n" - "st1 { v20.b }[2], [x22]\n" + "st1 { v12.b }[2], [x25]\n" + "st1 { v16.b }[2], [x24]\n" + "st1 { v20.b }[2], [x23]\n" "b 106f\n" "105:" // Height 4: Partial direct writeback: partial_1_0 "str b8, [x11, #0x0]\n" - "str b12, [x24, #0x0]\n" - "str b16, [x23, #0x0]\n" - "str b20, [x22, #0x0]\n" + "str b12, [x25, #0x0]\n" + "str b16, [x24, #0x0]\n" + "str b20, [x23, #0x0]\n" "106:" // Height 4: Partial direct writeback: Done "b 108f\n" "107:" // Height 4: Full writeback "str q8, [x11, #0x0]\n" "add x11, x11, #0x10\n" - "str q12, [x24, #0x0]\n" - "str q16, [x23, #0x0]\n" - "str q20, [x22, #0x0]\n" + "str q12, [x25, #0x0]\n" + "str q16, [x24, #0x0]\n" + "str q20, [x23, #0x0]\n" "108:" // Height 4: Writeback done "subs x10, x10, #0x10\n" "bgt 83b\n" @@ -1950,15 +1949,15 @@ void a64_hybrid_s8qs_dot_6x16 ( "112:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 113f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 114f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -1969,10 +1968,10 @@ void a64_hybrid_s8qs_dot_6x16 ( "b 114f\n" "113:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" "114:" // Height 5: input setup done "cmp x27, #0x10\n" "blt 117f\n" @@ -1995,7 +1994,7 @@ void a64_hybrid_s8qs_dot_6x16 ( "add x25, x25, #0x10\n" "add x24, x24, #0x10\n" ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x9, #0x20]\n" + "ldr q29, [x9, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" "add x23, x23, #0x10\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" @@ -2004,100 +2003,100 @@ void a64_hybrid_s8qs_dot_6x16 ( "cmp x27, #0x20\n" ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x9, #0x30]\n" + "ldr q28, [x9, #0x30]\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" + ".inst 0x4f80e3aa // sdot v10.4s, v29.16b, v0.4b[0]\n" + ".inst 0x4f81e3ae // sdot v14.4s, v29.16b, v1.4b[0]\n" "prfm pldl1keep, [x25, #0x80]\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" + ".inst 0x4f82e3b2 // sdot v18.4s, v29.16b, v2.4b[0]\n" + ".inst 0x4f83e3b6 // sdot v22.4s, v29.16b, v3.4b[0]\n" "prfm pldl1keep, [x23, #0x80]\n" "prfm pldl1keep, [x22, #0x80]\n" - ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x9, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x9, #0x50]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" - ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n" - ".inst 0x4fa4e0d8 // sdot v24.4s, v6.16b, v4.4b[1]\n" - "ldr q6, [x9, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n" - ".inst 0x4fa4e0f9 // sdot v25.4s, v7.16b, v4.4b[1]\n" - "ldr q7, [x9, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n" - ".inst 0x4fa4e0da // sdot v26.4s, v6.16b, v4.4b[1]\n" - "ldr q6, [x9, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" - ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n" - ".inst 0x4fa4e0fb // sdot v27.4s, v7.16b, v4.4b[1]\n" - "ldr q7, [x9, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n" - ".inst 0x4f84e8d8 // sdot v24.4s, v6.16b, v4.4b[2]\n" - "ldr q6, [x9, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n" - ".inst 0x4f84e8f9 // sdot v25.4s, v7.16b, v4.4b[2]\n" - "ldr q7, [x9, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n" - ".inst 0x4f84e8da // sdot v26.4s, v6.16b, v4.4b[2]\n" - "ldr q6, [x9, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n" - ".inst 0x4f84e8fb // sdot v27.4s, v7.16b, v4.4b[2]\n" - "ldr q7, [x9, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n" - ".inst 0x4fa4e8d8 // sdot v24.4s, v6.16b, v4.4b[3]\n" - "ldr q6, [x9, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" - ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n" - ".inst 0x4fa4e8f9 // sdot v25.4s, v7.16b, v4.4b[3]\n" - "ldr q7, [x9, #0xf0]\n" + ".inst 0x4f84e3ba // sdot v26.4s, v29.16b, v4.4b[0]\n" + "ldr q29, [x9, #0x40]\n" + ".inst 0x4f80e38b // sdot v11.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f81e38f // sdot v15.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f82e393 // sdot v19.4s, v28.16b, v2.4b[0]\n" + ".inst 0x4f83e397 // sdot v23.4s, v28.16b, v3.4b[0]\n" + ".inst 0x4f84e39b // sdot v27.4s, v28.16b, v4.4b[0]\n" + "ldr q28, [x9, #0x50]\n" + ".inst 0x4fa0e3a8 // sdot v8.4s, v29.16b, v0.4b[1]\n" + ".inst 0x4fa1e3ac // sdot v12.4s, v29.16b, v1.4b[1]\n" + ".inst 0x4fa2e3b0 // sdot v16.4s, v29.16b, v2.4b[1]\n" + ".inst 0x4fa3e3b4 // sdot v20.4s, v29.16b, v3.4b[1]\n" + ".inst 0x4fa4e3b8 // sdot v24.4s, v29.16b, v4.4b[1]\n" + "ldr q29, [x9, #0x60]\n" + ".inst 0x4fa0e389 // sdot v9.4s, v28.16b, v0.4b[1]\n" + ".inst 0x4fa1e38d // sdot v13.4s, v28.16b, v1.4b[1]\n" + ".inst 0x4fa2e391 // sdot v17.4s, v28.16b, v2.4b[1]\n" + ".inst 0x4fa3e395 // sdot v21.4s, v28.16b, v3.4b[1]\n" + ".inst 0x4fa4e399 // sdot v25.4s, v28.16b, v4.4b[1]\n" + "ldr q28, [x9, #0x70]\n" + ".inst 0x4fa0e3aa // sdot v10.4s, v29.16b, v0.4b[1]\n" + ".inst 0x4fa1e3ae // sdot v14.4s, v29.16b, v1.4b[1]\n" + ".inst 0x4fa2e3b2 // sdot v18.4s, v29.16b, v2.4b[1]\n" + ".inst 0x4fa3e3b6 // sdot v22.4s, v29.16b, v3.4b[1]\n" + ".inst 0x4fa4e3ba // sdot v26.4s, v29.16b, v4.4b[1]\n" + "ldr q29, [x9, #0x80]\n" + ".inst 0x4fa0e38b // sdot v11.4s, v28.16b, v0.4b[1]\n" + ".inst 0x4fa1e38f // sdot v15.4s, v28.16b, v1.4b[1]\n" + ".inst 0x4fa2e393 // sdot v19.4s, v28.16b, v2.4b[1]\n" + ".inst 0x4fa3e397 // sdot v23.4s, v28.16b, v3.4b[1]\n" + ".inst 0x4fa4e39b // sdot v27.4s, v28.16b, v4.4b[1]\n" + "ldr q28, [x9, #0x90]\n" + ".inst 0x4f80eba8 // sdot v8.4s, v29.16b, v0.4b[2]\n" + ".inst 0x4f81ebac // sdot v12.4s, v29.16b, v1.4b[2]\n" + ".inst 0x4f82ebb0 // sdot v16.4s, v29.16b, v2.4b[2]\n" + ".inst 0x4f83ebb4 // sdot v20.4s, v29.16b, v3.4b[2]\n" + ".inst 0x4f84ebb8 // sdot v24.4s, v29.16b, v4.4b[2]\n" + "ldr q29, [x9, #0xa0]\n" + ".inst 0x4f80eb89 // sdot v9.4s, v28.16b, v0.4b[2]\n" + ".inst 0x4f81eb8d // sdot v13.4s, v28.16b, v1.4b[2]\n" + ".inst 0x4f82eb91 // sdot v17.4s, v28.16b, v2.4b[2]\n" + ".inst 0x4f83eb95 // sdot v21.4s, v28.16b, v3.4b[2]\n" + ".inst 0x4f84eb99 // sdot v25.4s, v28.16b, v4.4b[2]\n" + "ldr q28, [x9, #0xb0]\n" + ".inst 0x4f80ebaa // sdot v10.4s, v29.16b, v0.4b[2]\n" + ".inst 0x4f81ebae // sdot v14.4s, v29.16b, v1.4b[2]\n" + ".inst 0x4f82ebb2 // sdot v18.4s, v29.16b, v2.4b[2]\n" + ".inst 0x4f83ebb6 // sdot v22.4s, v29.16b, v3.4b[2]\n" + ".inst 0x4f84ebba // sdot v26.4s, v29.16b, v4.4b[2]\n" + "ldr q29, [x9, #0xc0]\n" + ".inst 0x4f80eb8b // sdot v11.4s, v28.16b, v0.4b[2]\n" + ".inst 0x4f81eb8f // sdot v15.4s, v28.16b, v1.4b[2]\n" + ".inst 0x4f82eb93 // sdot v19.4s, v28.16b, v2.4b[2]\n" + ".inst 0x4f83eb97 // sdot v23.4s, v28.16b, v3.4b[2]\n" + ".inst 0x4f84eb9b // sdot v27.4s, v28.16b, v4.4b[2]\n" + "ldr q28, [x9, #0xd0]\n" + ".inst 0x4fa0eba8 // sdot v8.4s, v29.16b, v0.4b[3]\n" + ".inst 0x4fa1ebac // sdot v12.4s, v29.16b, v1.4b[3]\n" + ".inst 0x4fa2ebb0 // sdot v16.4s, v29.16b, v2.4b[3]\n" + ".inst 0x4fa3ebb4 // sdot v20.4s, v29.16b, v3.4b[3]\n" + ".inst 0x4fa4ebb8 // sdot v24.4s, v29.16b, v4.4b[3]\n" + "ldr q29, [x9, #0xe0]\n" + ".inst 0x4fa0eb89 // sdot v9.4s, v28.16b, v0.4b[3]\n" + ".inst 0x4fa1eb8d // sdot v13.4s, v28.16b, v1.4b[3]\n" + ".inst 0x4fa2eb91 // sdot v17.4s, v28.16b, v2.4b[3]\n" + ".inst 0x4fa3eb95 // sdot v21.4s, v28.16b, v3.4b[3]\n" + ".inst 0x4fa4eb99 // sdot v25.4s, v28.16b, v4.4b[3]\n" + "ldr q28, [x9, #0xf0]\n" "add x9, x9, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n" - ".inst 0x4fa4e8da // sdot v26.4s, v6.16b, v4.4b[3]\n" + ".inst 0x4fa0ebaa // sdot v10.4s, v29.16b, v0.4b[3]\n" + ".inst 0x4fa1ebae // sdot v14.4s, v29.16b, v1.4b[3]\n" + ".inst 0x4fa2ebb2 // sdot v18.4s, v29.16b, v2.4b[3]\n" + ".inst 0x4fa3ebb6 // sdot v22.4s, v29.16b, v3.4b[3]\n" + ".inst 0x4fa4ebba // sdot v26.4s, v29.16b, v4.4b[3]\n" "ldr q6, [x9, #0x0]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x4fa0eb8b // sdot v11.4s, v28.16b, v0.4b[3]\n" "ldr q0, [x26, #0x0]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" + ".inst 0x4fa1eb8f // sdot v15.4s, v28.16b, v1.4b[3]\n" "ldr q1, [x25, #0x0]\n" - ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" + ".inst 0x4fa2eb93 // sdot v19.4s, v28.16b, v2.4b[3]\n" "ldr q2, [x24, #0x0]\n" - ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n" + ".inst 0x4fa3eb97 // sdot v23.4s, v28.16b, v3.4b[3]\n" "ldr q3, [x23, #0x0]\n" - ".inst 0x4fa4e8fb // sdot v27.4s, v7.16b, v4.4b[3]\n" + ".inst 0x4fa4eb9b // sdot v27.4s, v28.16b, v4.4b[3]\n" "ldr q4, [x22, #0x0]\n" "ldr q7, [x9, #0x10]\n" "bge 115b\n" @@ -2111,7 +2110,7 @@ void a64_hybrid_s8qs_dot_6x16 ( "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x9, #0x20]\n" + "ldr q29, [x9, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" "add x22, x22, #0x10\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" @@ -2120,131 +2119,131 @@ void a64_hybrid_s8qs_dot_6x16 ( "prfm pldl1keep, [x26, #0x80]\n" ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x9, #0x30]\n" + "ldr q28, [x9, #0x30]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" + ".inst 0x4f80e3aa // sdot v10.4s, v29.16b, v0.4b[0]\n" + ".inst 0x4f81e3ae // sdot v14.4s, v29.16b, v1.4b[0]\n" "prfm pldl1keep, [x24, #0x80]\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" + ".inst 0x4f82e3b2 // sdot v18.4s, v29.16b, v2.4b[0]\n" + ".inst 0x4f83e3b6 // sdot v22.4s, v29.16b, v3.4b[0]\n" "prfm pldl1keep, [x22, #0x80]\n" - ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x9, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x9, #0x50]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" - ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n" - ".inst 0x4fa4e0d8 // sdot v24.4s, v6.16b, v4.4b[1]\n" - "ldr q6, [x9, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n" - ".inst 0x4fa4e0f9 // sdot v25.4s, v7.16b, v4.4b[1]\n" - "ldr q7, [x9, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n" - ".inst 0x4fa4e0da // sdot v26.4s, v6.16b, v4.4b[1]\n" - "ldr q6, [x9, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" - ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n" - ".inst 0x4fa4e0fb // sdot v27.4s, v7.16b, v4.4b[1]\n" - "ldr q7, [x9, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n" - ".inst 0x4f84e8d8 // sdot v24.4s, v6.16b, v4.4b[2]\n" - "ldr q6, [x9, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n" - ".inst 0x4f84e8f9 // sdot v25.4s, v7.16b, v4.4b[2]\n" - "ldr q7, [x9, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n" - ".inst 0x4f84e8da // sdot v26.4s, v6.16b, v4.4b[2]\n" - "ldr q6, [x9, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n" - ".inst 0x4f84e8fb // sdot v27.4s, v7.16b, v4.4b[2]\n" - "ldr q7, [x9, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n" - ".inst 0x4fa4e8d8 // sdot v24.4s, v6.16b, v4.4b[3]\n" - "ldr q6, [x9, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" - ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n" - ".inst 0x4fa4e8f9 // sdot v25.4s, v7.16b, v4.4b[3]\n" - "ldr q7, [x9, #0xf0]\n" + ".inst 0x4f84e3ba // sdot v26.4s, v29.16b, v4.4b[0]\n" + "ldr q29, [x9, #0x40]\n" + ".inst 0x4f80e38b // sdot v11.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f81e38f // sdot v15.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f82e393 // sdot v19.4s, v28.16b, v2.4b[0]\n" + ".inst 0x4f83e397 // sdot v23.4s, v28.16b, v3.4b[0]\n" + ".inst 0x4f84e39b // sdot v27.4s, v28.16b, v4.4b[0]\n" + "ldr q28, [x9, #0x50]\n" + ".inst 0x4fa0e3a8 // sdot v8.4s, v29.16b, v0.4b[1]\n" + ".inst 0x4fa1e3ac // sdot v12.4s, v29.16b, v1.4b[1]\n" + ".inst 0x4fa2e3b0 // sdot v16.4s, v29.16b, v2.4b[1]\n" + ".inst 0x4fa3e3b4 // sdot v20.4s, v29.16b, v3.4b[1]\n" + ".inst 0x4fa4e3b8 // sdot v24.4s, v29.16b, v4.4b[1]\n" + "ldr q29, [x9, #0x60]\n" + ".inst 0x4fa0e389 // sdot v9.4s, v28.16b, v0.4b[1]\n" + ".inst 0x4fa1e38d // sdot v13.4s, v28.16b, v1.4b[1]\n" + ".inst 0x4fa2e391 // sdot v17.4s, v28.16b, v2.4b[1]\n" + ".inst 0x4fa3e395 // sdot v21.4s, v28.16b, v3.4b[1]\n" + ".inst 0x4fa4e399 // sdot v25.4s, v28.16b, v4.4b[1]\n" + "ldr q28, [x9, #0x70]\n" + ".inst 0x4fa0e3aa // sdot v10.4s, v29.16b, v0.4b[1]\n" + ".inst 0x4fa1e3ae // sdot v14.4s, v29.16b, v1.4b[1]\n" + ".inst 0x4fa2e3b2 // sdot v18.4s, v29.16b, v2.4b[1]\n" + ".inst 0x4fa3e3b6 // sdot v22.4s, v29.16b, v3.4b[1]\n" + ".inst 0x4fa4e3ba // sdot v26.4s, v29.16b, v4.4b[1]\n" + "ldr q29, [x9, #0x80]\n" + ".inst 0x4fa0e38b // sdot v11.4s, v28.16b, v0.4b[1]\n" + ".inst 0x4fa1e38f // sdot v15.4s, v28.16b, v1.4b[1]\n" + ".inst 0x4fa2e393 // sdot v19.4s, v28.16b, v2.4b[1]\n" + ".inst 0x4fa3e397 // sdot v23.4s, v28.16b, v3.4b[1]\n" + ".inst 0x4fa4e39b // sdot v27.4s, v28.16b, v4.4b[1]\n" + "ldr q28, [x9, #0x90]\n" + ".inst 0x4f80eba8 // sdot v8.4s, v29.16b, v0.4b[2]\n" + ".inst 0x4f81ebac // sdot v12.4s, v29.16b, v1.4b[2]\n" + ".inst 0x4f82ebb0 // sdot v16.4s, v29.16b, v2.4b[2]\n" + ".inst 0x4f83ebb4 // sdot v20.4s, v29.16b, v3.4b[2]\n" + ".inst 0x4f84ebb8 // sdot v24.4s, v29.16b, v4.4b[2]\n" + "ldr q29, [x9, #0xa0]\n" + ".inst 0x4f80eb89 // sdot v9.4s, v28.16b, v0.4b[2]\n" + ".inst 0x4f81eb8d // sdot v13.4s, v28.16b, v1.4b[2]\n" + ".inst 0x4f82eb91 // sdot v17.4s, v28.16b, v2.4b[2]\n" + ".inst 0x4f83eb95 // sdot v21.4s, v28.16b, v3.4b[2]\n" + ".inst 0x4f84eb99 // sdot v25.4s, v28.16b, v4.4b[2]\n" + "ldr q28, [x9, #0xb0]\n" + ".inst 0x4f80ebaa // sdot v10.4s, v29.16b, v0.4b[2]\n" + ".inst 0x4f81ebae // sdot v14.4s, v29.16b, v1.4b[2]\n" + ".inst 0x4f82ebb2 // sdot v18.4s, v29.16b, v2.4b[2]\n" + ".inst 0x4f83ebb6 // sdot v22.4s, v29.16b, v3.4b[2]\n" + ".inst 0x4f84ebba // sdot v26.4s, v29.16b, v4.4b[2]\n" + "ldr q29, [x9, #0xc0]\n" + ".inst 0x4f80eb8b // sdot v11.4s, v28.16b, v0.4b[2]\n" + ".inst 0x4f81eb8f // sdot v15.4s, v28.16b, v1.4b[2]\n" + ".inst 0x4f82eb93 // sdot v19.4s, v28.16b, v2.4b[2]\n" + ".inst 0x4f83eb97 // sdot v23.4s, v28.16b, v3.4b[2]\n" + ".inst 0x4f84eb9b // sdot v27.4s, v28.16b, v4.4b[2]\n" + "ldr q28, [x9, #0xd0]\n" + ".inst 0x4fa0eba8 // sdot v8.4s, v29.16b, v0.4b[3]\n" + ".inst 0x4fa1ebac // sdot v12.4s, v29.16b, v1.4b[3]\n" + ".inst 0x4fa2ebb0 // sdot v16.4s, v29.16b, v2.4b[3]\n" + ".inst 0x4fa3ebb4 // sdot v20.4s, v29.16b, v3.4b[3]\n" + ".inst 0x4fa4ebb8 // sdot v24.4s, v29.16b, v4.4b[3]\n" + "ldr q29, [x9, #0xe0]\n" + ".inst 0x4fa0eb89 // sdot v9.4s, v28.16b, v0.4b[3]\n" + ".inst 0x4fa1eb8d // sdot v13.4s, v28.16b, v1.4b[3]\n" + ".inst 0x4fa2eb91 // sdot v17.4s, v28.16b, v2.4b[3]\n" + ".inst 0x4fa3eb95 // sdot v21.4s, v28.16b, v3.4b[3]\n" + ".inst 0x4fa4eb99 // sdot v25.4s, v28.16b, v4.4b[3]\n" + "ldr q28, [x9, #0xf0]\n" "add x9, x9, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n" - ".inst 0x4fa4e8da // sdot v26.4s, v6.16b, v4.4b[3]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" - ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n" - ".inst 0x4fa4e8fb // sdot v27.4s, v7.16b, v4.4b[3]\n" + ".inst 0x4fa0ebaa // sdot v10.4s, v29.16b, v0.4b[3]\n" + ".inst 0x4fa1ebae // sdot v14.4s, v29.16b, v1.4b[3]\n" + ".inst 0x4fa2ebb2 // sdot v18.4s, v29.16b, v2.4b[3]\n" + ".inst 0x4fa3ebb6 // sdot v22.4s, v29.16b, v3.4b[3]\n" + ".inst 0x4fa4ebba // sdot v26.4s, v29.16b, v4.4b[3]\n" + ".inst 0x4fa0eb8b // sdot v11.4s, v28.16b, v0.4b[3]\n" + ".inst 0x4fa1eb8f // sdot v15.4s, v28.16b, v1.4b[3]\n" + ".inst 0x4fa2eb93 // sdot v19.4s, v28.16b, v2.4b[3]\n" + ".inst 0x4fa3eb97 // sdot v23.4s, v28.16b, v3.4b[3]\n" + ".inst 0x4fa4eb9b // sdot v27.4s, v28.16b, v4.4b[3]\n" "117:" // Height 5: Multiply loop: Main loop skip "cbz x27, 122f\n" "cmp x27, #0x4\n" "blt 119f\n" "118:" // Height 5: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" + "ldr s2, [x26], #0x4\n" "ldr s1, [x25], #0x4\n" "sub x27, x27, #0x4\n" "cmp x27, #0x4\n" - "ldr s2, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr s4, [x22], #0x4\n" - "ldr q6, [x9, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x9, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x9, #0x30]\n" + "ldr s0, [x24], #0x4\n" + "ldr s31, [x23], #0x4\n" + "ldr s30, [x22], #0x4\n" + "ldr q29, [x9, #0x0]\n" + ".inst 0x4f82e3a8 // sdot v8.4s, v29.16b, v2.4b[0]\n" + ".inst 0x4f81e3ac // sdot v12.4s, v29.16b, v1.4b[0]\n" + "ldr q28, [x9, #0x10]\n" + ".inst 0x4f80e3b0 // sdot v16.4s, v29.16b, v0.4b[0]\n" + ".inst 0x4f9fe3b4 // sdot v20.4s, v29.16b, v31.4b[0]\n" + ".inst 0x4f9ee3b8 // sdot v24.4s, v29.16b, v30.4b[0]\n" + "ldr q29, [x9, #0x20]\n" + ".inst 0x4f82e389 // sdot v9.4s, v28.16b, v2.4b[0]\n" + ".inst 0x4f81e38d // sdot v13.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f80e391 // sdot v17.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f9fe395 // sdot v21.4s, v28.16b, v31.4b[0]\n" + ".inst 0x4f9ee399 // sdot v25.4s, v28.16b, v30.4b[0]\n" + "ldr q28, [x9, #0x30]\n" "add x9, x9, #0x40\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" + ".inst 0x4f82e3aa // sdot v10.4s, v29.16b, v2.4b[0]\n" + ".inst 0x4f81e3ae // sdot v14.4s, v29.16b, v1.4b[0]\n" + ".inst 0x4f80e3b2 // sdot v18.4s, v29.16b, v0.4b[0]\n" + ".inst 0x4f9fe3b6 // sdot v22.4s, v29.16b, v31.4b[0]\n" + ".inst 0x4f9ee3ba // sdot v26.4s, v29.16b, v30.4b[0]\n" + ".inst 0x4f82e38b // sdot v11.4s, v28.16b, v2.4b[0]\n" + ".inst 0x4f81e38f // sdot v15.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f80e393 // sdot v19.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f9fe397 // sdot v23.4s, v28.16b, v31.4b[0]\n" + ".inst 0x4f9ee39b // sdot v27.4s, v28.16b, v30.4b[0]\n" "bge 118b\n" "119:" // Height 5: Multiply loop: Skip odd blocks "cbz x27, 122f\n" @@ -2268,71 +2267,71 @@ void a64_hybrid_s8qs_dot_6x16 ( "ldr b3, [x23, #0x0]\n" "ldr b4, [x22, #0x0]\n" "121:" // Height 5: Multiply loop: Ragged operand read: Done - "ldr q6, [x9, #0x0]\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x9, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x9, #0x30]\n" + "ldr q29, [x9, #0x0]\n" + "ldr q28, [x9, #0x10]\n" + ".inst 0x4f80e3a8 // sdot v8.4s, v29.16b, v0.4b[0]\n" + ".inst 0x4f81e3ac // sdot v12.4s, v29.16b, v1.4b[0]\n" + ".inst 0x4f82e3b0 // sdot v16.4s, v29.16b, v2.4b[0]\n" + ".inst 0x4f83e3b4 // sdot v20.4s, v29.16b, v3.4b[0]\n" + ".inst 0x4f84e3b8 // sdot v24.4s, v29.16b, v4.4b[0]\n" + "ldr q29, [x9, #0x20]\n" + ".inst 0x4f80e389 // sdot v9.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f81e38d // sdot v13.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f82e391 // sdot v17.4s, v28.16b, v2.4b[0]\n" + ".inst 0x4f83e395 // sdot v21.4s, v28.16b, v3.4b[0]\n" + ".inst 0x4f84e399 // sdot v25.4s, v28.16b, v4.4b[0]\n" + "ldr q28, [x9, #0x30]\n" "add x9, x9, #0x40\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" + ".inst 0x4f80e3aa // sdot v10.4s, v29.16b, v0.4b[0]\n" + ".inst 0x4f81e3ae // sdot v14.4s, v29.16b, v1.4b[0]\n" + ".inst 0x4f82e3b2 // sdot v18.4s, v29.16b, v2.4b[0]\n" + ".inst 0x4f83e3b6 // sdot v22.4s, v29.16b, v3.4b[0]\n" + ".inst 0x4f84e3ba // sdot v26.4s, v29.16b, v4.4b[0]\n" + ".inst 0x4f80e38b // sdot v11.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f81e38f // sdot v15.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f82e393 // sdot v19.4s, v28.16b, v2.4b[0]\n" + ".inst 0x4f83e397 // sdot v23.4s, v28.16b, v3.4b[0]\n" + ".inst 0x4f84e39b // sdot v27.4s, v28.16b, v4.4b[0]\n" "122:" // Height 5: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 112b\n" - "ldr q0, [x14, #0x0]\n" - "ldr q1, [x14, #0x10]\n" - "add v8.4s, v8.4s, v0.4s\n" - "add v9.4s, v9.4s, v1.4s\n" - "ldr q2, [x14, #0x20]\n" - "ldr q3, [x14, #0x30]\n" - "add v10.4s, v10.4s, v2.4s\n" - "add v11.4s, v11.4s, v3.4s\n" + "ldr q31, [x14, #0x0]\n" + "ldr q30, [x14, #0x10]\n" + "add v8.4s, v8.4s, v31.4s\n" + "add v9.4s, v9.4s, v30.4s\n" + "ldr q29, [x14, #0x20]\n" + "ldr q28, [x14, #0x30]\n" + "add v10.4s, v10.4s, v29.4s\n" + "add v11.4s, v11.4s, v28.4s\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x11, x20\n" - "add x23, x24, x20\n" + "add x25, x11, x20\n" + "add x24, x25, x20\n" "prfm pstl1keep, [x11, #0x0]\n" + "add x23, x24, x20\n" "add x22, x23, x20\n" - "add x21, x22, x20\n" + "prfm pstl1keep, [x25, #0x0]\n" "prfm pstl1keep, [x24, #0x0]\n" "prfm pstl1keep, [x23, #0x0]\n" "prfm pstl1keep, [x22, #0x0]\n" - "prfm pstl1keep, [x21, #0x0]\n" - "add v12.4s, v12.4s, v0.4s\n" - "add v13.4s, v13.4s, v1.4s\n" - "add v14.4s, v14.4s, v2.4s\n" - "add v15.4s, v15.4s, v3.4s\n" + "add v12.4s, v12.4s, v31.4s\n" + "add v13.4s, v13.4s, v30.4s\n" + "add v14.4s, v14.4s, v29.4s\n" + "add v15.4s, v15.4s, v28.4s\n" "add x14, x14, #0x40\n" - "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" - "add v20.4s, v20.4s, v0.4s\n" - "add v21.4s, v21.4s, v1.4s\n" - "add v22.4s, v22.4s, v2.4s\n" - "add v23.4s, v23.4s, v3.4s\n" - "add v24.4s, v24.4s, v0.4s\n" - "add v25.4s, v25.4s, v1.4s\n" - "add v26.4s, v26.4s, v2.4s\n" - "add v27.4s, v27.4s, v3.4s\n" + "add v16.4s, v16.4s, v31.4s\n" + "add v17.4s, v17.4s, v30.4s\n" + "add v18.4s, v18.4s, v29.4s\n" + "add v19.4s, v19.4s, v28.4s\n" + "add v20.4s, v20.4s, v31.4s\n" + "add v21.4s, v21.4s, v30.4s\n" + "add v22.4s, v22.4s, v29.4s\n" + "add v23.4s, v23.4s, v28.4s\n" + "add v24.4s, v24.4s, v31.4s\n" + "add v25.4s, v25.4s, v30.4s\n" + "add v26.4s, v26.4s, v29.4s\n" + "add v27.4s, v27.4s, v28.4s\n" "tbz %x[flags], #4, 123f\n" "ldr q0, [x12, #0x0]\n" "ldr q4, [x13, #0x0]\n" @@ -2346,10 +2345,10 @@ void a64_hybrid_s8qs_dot_6x16 ( "add x13, x13, #0x40\n" "b 124f\n" "123:" // Height 5: per layer parameters - "add x25, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x25]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v4.4s }, [x20]\n" "mov v1.16b, v0.16b\n" "mov v5.16b, v4.16b\n" "mov v2.16b, v0.16b\n" @@ -2378,79 +2377,79 @@ void a64_hybrid_s8qs_dot_6x16 ( "sqrdmulh v26.4s, v26.4s, v6.4s\n" "sqrdmulh v27.4s, v27.4s, v7.4s\n" "tbz %x[flags], #5, 125f\n" - "and v4.16b, v8.16b, v0.16b\n" - "and v5.16b, v9.16b, v1.16b\n" - "and v6.16b, v10.16b, v2.16b\n" - "and v7.16b, v11.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v4.4s\n" - "sqadd v9.4s, v9.4s, v5.4s\n" - "sqadd v10.4s, v10.4s, v6.4s\n" - "sqadd v11.4s, v11.4s, v7.4s\n" - "and v4.16b, v12.16b, v0.16b\n" - "and v5.16b, v13.16b, v1.16b\n" - "and v6.16b, v14.16b, v2.16b\n" - "and v7.16b, v15.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v12.4s, v12.4s, v4.4s\n" - "sqadd v13.4s, v13.4s, v5.4s\n" - "sqadd v14.4s, v14.4s, v6.4s\n" - "sqadd v15.4s, v15.4s, v7.4s\n" - "and v4.16b, v16.16b, v0.16b\n" - "and v5.16b, v17.16b, v1.16b\n" - "and v6.16b, v18.16b, v2.16b\n" - "and v7.16b, v19.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" - "and v4.16b, v20.16b, v0.16b\n" - "and v5.16b, v21.16b, v1.16b\n" - "and v6.16b, v22.16b, v2.16b\n" - "and v7.16b, v23.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v20.4s, v20.4s, v4.4s\n" - "sqadd v21.4s, v21.4s, v5.4s\n" - "sqadd v22.4s, v22.4s, v6.4s\n" - "sqadd v23.4s, v23.4s, v7.4s\n" - "and v4.16b, v24.16b, v0.16b\n" - "and v5.16b, v25.16b, v1.16b\n" - "and v6.16b, v26.16b, v2.16b\n" - "and v7.16b, v27.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v24.4s, v24.4s, v4.4s\n" - "sqadd v25.4s, v25.4s, v5.4s\n" - "sqadd v26.4s, v26.4s, v6.4s\n" - "sqadd v27.4s, v27.4s, v7.4s\n" + "and v31.16b, v8.16b, v0.16b\n" + "and v30.16b, v9.16b, v1.16b\n" + "and v29.16b, v10.16b, v2.16b\n" + "and v28.16b, v11.16b, v3.16b\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sqadd v8.4s, v8.4s, v31.4s\n" + "sqadd v9.4s, v9.4s, v30.4s\n" + "sqadd v10.4s, v10.4s, v29.4s\n" + "sqadd v11.4s, v11.4s, v28.4s\n" + "and v31.16b, v12.16b, v0.16b\n" + "and v30.16b, v13.16b, v1.16b\n" + "and v29.16b, v14.16b, v2.16b\n" + "and v28.16b, v15.16b, v3.16b\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sqadd v12.4s, v12.4s, v31.4s\n" + "sqadd v13.4s, v13.4s, v30.4s\n" + "sqadd v14.4s, v14.4s, v29.4s\n" + "sqadd v15.4s, v15.4s, v28.4s\n" + "and v31.16b, v16.16b, v0.16b\n" + "and v30.16b, v17.16b, v1.16b\n" + "and v29.16b, v18.16b, v2.16b\n" + "and v28.16b, v19.16b, v3.16b\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v31.4s\n" + "sqadd v17.4s, v17.4s, v30.4s\n" + "sqadd v18.4s, v18.4s, v29.4s\n" + "sqadd v19.4s, v19.4s, v28.4s\n" + "and v31.16b, v20.16b, v0.16b\n" + "and v30.16b, v21.16b, v1.16b\n" + "and v29.16b, v22.16b, v2.16b\n" + "and v28.16b, v23.16b, v3.16b\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v31.4s\n" + "sqadd v21.4s, v21.4s, v30.4s\n" + "sqadd v22.4s, v22.4s, v29.4s\n" + "sqadd v23.4s, v23.4s, v28.4s\n" + "and v31.16b, v24.16b, v0.16b\n" + "and v30.16b, v25.16b, v1.16b\n" + "and v29.16b, v26.16b, v2.16b\n" + "and v28.16b, v27.16b, v3.16b\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sqadd v24.4s, v24.4s, v31.4s\n" + "sqadd v25.4s, v25.4s, v30.4s\n" + "sqadd v26.4s, v26.4s, v29.4s\n" + "sqadd v27.4s, v27.4s, v28.4s\n" "125:" // Height 5: no shift correction - "add x25, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v30.4s }, [x20]\n" "srshl v8.4s, v8.4s, v0.4s\n" "srshl v9.4s, v9.4s, v1.4s\n" "srshl v10.4s, v10.4s, v2.4s\n" "srshl v11.4s, v11.4s, v3.4s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x25]\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v29.4s }, [x20]\n" "srshl v12.4s, v12.4s, v0.4s\n" "srshl v13.4s, v13.4s, v1.4s\n" - "add x25, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x25]\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v28.4s }, [x20]\n" "srshl v14.4s, v14.4s, v2.4s\n" "srshl v15.4s, v15.4s, v3.4s\n" "cmp x10, #0x10\n" @@ -2466,194 +2465,194 @@ void a64_hybrid_s8qs_dot_6x16 ( "srshl v25.4s, v25.4s, v1.4s\n" "srshl v26.4s, v26.4s, v2.4s\n" "srshl v27.4s, v27.4s, v3.4s\n" - "add v8.4s, v8.4s, v4.4s\n" - "add v9.4s, v9.4s, v4.4s\n" - "add v10.4s, v10.4s, v4.4s\n" - "add v11.4s, v11.4s, v4.4s\n" - "add v12.4s, v12.4s, v4.4s\n" - "add v13.4s, v13.4s, v4.4s\n" - "add v14.4s, v14.4s, v4.4s\n" - "add v15.4s, v15.4s, v4.4s\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v23.4s, v23.4s, v4.4s\n" - "add v24.4s, v24.4s, v4.4s\n" - "add v25.4s, v25.4s, v4.4s\n" - "add v26.4s, v26.4s, v4.4s\n" - "add v27.4s, v27.4s, v4.4s\n" - "smin v8.4s, v8.4s, v6.4s\n" - "smin v9.4s, v9.4s, v6.4s\n" - "smin v10.4s, v10.4s, v6.4s\n" - "smin v11.4s, v11.4s, v6.4s\n" - "smin v12.4s, v12.4s, v6.4s\n" - "smin v13.4s, v13.4s, v6.4s\n" - "smin v14.4s, v14.4s, v6.4s\n" - "smin v15.4s, v15.4s, v6.4s\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "smin v24.4s, v24.4s, v6.4s\n" - "smin v25.4s, v25.4s, v6.4s\n" - "smin v26.4s, v26.4s, v6.4s\n" - "smin v27.4s, v27.4s, v6.4s\n" - "smax v8.4s, v8.4s, v5.4s\n" - "smax v9.4s, v9.4s, v5.4s\n" - "smax v10.4s, v10.4s, v5.4s\n" - "smax v11.4s, v11.4s, v5.4s\n" - "smax v12.4s, v12.4s, v5.4s\n" - "smax v13.4s, v13.4s, v5.4s\n" - "smax v14.4s, v14.4s, v5.4s\n" - "smax v15.4s, v15.4s, v5.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" - "smax v24.4s, v24.4s, v5.4s\n" - "smax v25.4s, v25.4s, v5.4s\n" - "smax v26.4s, v26.4s, v5.4s\n" - "smax v27.4s, v27.4s, v5.4s\n" + "add v8.4s, v8.4s, v30.4s\n" + "add v9.4s, v9.4s, v30.4s\n" + "add v10.4s, v10.4s, v30.4s\n" + "add v11.4s, v11.4s, v30.4s\n" + "add v12.4s, v12.4s, v30.4s\n" + "add v13.4s, v13.4s, v30.4s\n" + "add v14.4s, v14.4s, v30.4s\n" + "add v15.4s, v15.4s, v30.4s\n" + "add v16.4s, v16.4s, v30.4s\n" + "add v17.4s, v17.4s, v30.4s\n" + "add v18.4s, v18.4s, v30.4s\n" + "add v19.4s, v19.4s, v30.4s\n" + "add v20.4s, v20.4s, v30.4s\n" + "add v21.4s, v21.4s, v30.4s\n" + "add v22.4s, v22.4s, v30.4s\n" + "add v23.4s, v23.4s, v30.4s\n" + "add v24.4s, v24.4s, v30.4s\n" + "add v25.4s, v25.4s, v30.4s\n" + "add v26.4s, v26.4s, v30.4s\n" + "add v27.4s, v27.4s, v30.4s\n" + "smin v8.4s, v8.4s, v29.4s\n" + "smin v9.4s, v9.4s, v29.4s\n" + "smin v10.4s, v10.4s, v29.4s\n" + "smin v11.4s, v11.4s, v29.4s\n" + "smin v12.4s, v12.4s, v29.4s\n" + "smin v13.4s, v13.4s, v29.4s\n" + "smin v14.4s, v14.4s, v29.4s\n" + "smin v15.4s, v15.4s, v29.4s\n" + "smin v16.4s, v16.4s, v29.4s\n" + "smin v17.4s, v17.4s, v29.4s\n" + "smin v18.4s, v18.4s, v29.4s\n" + "smin v19.4s, v19.4s, v29.4s\n" + "smin v20.4s, v20.4s, v29.4s\n" + "smin v21.4s, v21.4s, v29.4s\n" + "smin v22.4s, v22.4s, v29.4s\n" + "smin v23.4s, v23.4s, v29.4s\n" + "smin v24.4s, v24.4s, v29.4s\n" + "smin v25.4s, v25.4s, v29.4s\n" + "smin v26.4s, v26.4s, v29.4s\n" + "smin v27.4s, v27.4s, v29.4s\n" + "smax v8.4s, v8.4s, v28.4s\n" + "smax v9.4s, v9.4s, v28.4s\n" + "smax v10.4s, v10.4s, v28.4s\n" + "smax v11.4s, v11.4s, v28.4s\n" + "smax v12.4s, v12.4s, v28.4s\n" + "smax v13.4s, v13.4s, v28.4s\n" + "smax v14.4s, v14.4s, v28.4s\n" + "smax v15.4s, v15.4s, v28.4s\n" + "smax v16.4s, v16.4s, v28.4s\n" + "smax v17.4s, v17.4s, v28.4s\n" + "smax v18.4s, v18.4s, v28.4s\n" + "smax v19.4s, v19.4s, v28.4s\n" + "smax v20.4s, v20.4s, v28.4s\n" + "smax v21.4s, v21.4s, v28.4s\n" + "smax v22.4s, v22.4s, v28.4s\n" + "smax v23.4s, v23.4s, v28.4s\n" + "smax v24.4s, v24.4s, v28.4s\n" + "smax v25.4s, v25.4s, v28.4s\n" + "smax v26.4s, v26.4s, v28.4s\n" + "smax v27.4s, v27.4s, v28.4s\n" "uzp1 v8.8h, v8.8h, v9.8h\n" - "uzp1 v9.8h, v10.8h, v11.8h\n" + "uzp1 v29.8h, v10.8h, v11.8h\n" "uzp1 v12.8h, v12.8h, v13.8h\n" - "uzp1 v13.8h, v14.8h, v15.8h\n" + "uzp1 v28.8h, v14.8h, v15.8h\n" "uzp1 v16.8h, v16.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v19.8h, v18.8h, v19.8h\n" "uzp1 v20.8h, v20.8h, v21.8h\n" - "uzp1 v21.8h, v22.8h, v23.8h\n" + "uzp1 v18.8h, v22.8h, v23.8h\n" "uzp1 v24.8h, v24.8h, v25.8h\n" - "uzp1 v25.8h, v26.8h, v27.8h\n" - "uzp1 v8.16b, v8.16b, v9.16b\n" - "uzp1 v12.16b, v12.16b, v13.16b\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" - "uzp1 v20.16b, v20.16b, v21.16b\n" - "uzp1 v24.16b, v24.16b, v25.16b\n" + "uzp1 v17.8h, v26.8h, v27.8h\n" + "uzp1 v8.16b, v8.16b, v29.16b\n" + "uzp1 v12.16b, v12.16b, v28.16b\n" + "uzp1 v16.16b, v16.16b, v19.16b\n" + "uzp1 v20.16b, v20.16b, v18.16b\n" + "uzp1 v24.16b, v24.16b, v17.16b\n" "bge 134f\n" "tbz x10, #3, 129f\n" "str d8, [x11], #0x8\n" - "str d12, [x24], #0x8\n" - "str d16, [x23], #0x8\n" - "str d20, [x22], #0x8\n" - "str d24, [x21], #0x8\n" + "str d12, [x25], #0x8\n" + "str d16, [x24], #0x8\n" + "str d20, [x23], #0x8\n" + "str d24, [x22], #0x8\n" "tbz x10, #2, 127f\n" "st1 { v8.s }[2], [x11], #0x4\n" - "st1 { v12.s }[2], [x24], #0x4\n" - "st1 { v16.s }[2], [x23], #0x4\n" - "st1 { v20.s }[2], [x22], #0x4\n" - "st1 { v24.s }[2], [x21], #0x4\n" + "st1 { v12.s }[2], [x25], #0x4\n" + "st1 { v16.s }[2], [x24], #0x4\n" + "st1 { v20.s }[2], [x23], #0x4\n" + "st1 { v24.s }[2], [x22], #0x4\n" "tbz x10, #1, 126f\n" "st1 { v8.h }[6], [x11], #0x2\n" - "st1 { v12.h }[6], [x24], #0x2\n" - "st1 { v16.h }[6], [x23], #0x2\n" - "st1 { v20.h }[6], [x22], #0x2\n" - "st1 { v24.h }[6], [x21], #0x2\n" + "st1 { v12.h }[6], [x25], #0x2\n" + "st1 { v16.h }[6], [x24], #0x2\n" + "st1 { v20.h }[6], [x23], #0x2\n" + "st1 { v24.h }[6], [x22], #0x2\n" "tbz x10, #0, 133f\n" "st1 { v8.b }[14], [x11]\n" - "st1 { v12.b }[14], [x24]\n" - "st1 { v16.b }[14], [x23]\n" - "st1 { v20.b }[14], [x22]\n" - "st1 { v24.b }[14], [x21]\n" + "st1 { v12.b }[14], [x25]\n" + "st1 { v16.b }[14], [x24]\n" + "st1 { v20.b }[14], [x23]\n" + "st1 { v24.b }[14], [x22]\n" "b 133f\n" "126:" // Height 5: Partial direct writeback: partial_1_12 "tbz x10, #0, 133f\n" "st1 { v8.b }[12], [x11]\n" - "st1 { v12.b }[12], [x24]\n" - "st1 { v16.b }[12], [x23]\n" - "st1 { v20.b }[12], [x22]\n" - "st1 { v24.b }[12], [x21]\n" + "st1 { v12.b }[12], [x25]\n" + "st1 { v16.b }[12], [x24]\n" + "st1 { v20.b }[12], [x23]\n" + "st1 { v24.b }[12], [x22]\n" "b 133f\n" "127:" // Height 5: Partial direct writeback: partial_2_8 "tbz x10, #1, 128f\n" "st1 { v8.h }[4], [x11], #0x2\n" - "st1 { v12.h }[4], [x24], #0x2\n" - "st1 { v16.h }[4], [x23], #0x2\n" - "st1 { v20.h }[4], [x22], #0x2\n" - "st1 { v24.h }[4], [x21], #0x2\n" + "st1 { v12.h }[4], [x25], #0x2\n" + "st1 { v16.h }[4], [x24], #0x2\n" + "st1 { v20.h }[4], [x23], #0x2\n" + "st1 { v24.h }[4], [x22], #0x2\n" "tbz x10, #0, 133f\n" "st1 { v8.b }[10], [x11]\n" - "st1 { v12.b }[10], [x24]\n" - "st1 { v16.b }[10], [x23]\n" - "st1 { v20.b }[10], [x22]\n" - "st1 { v24.b }[10], [x21]\n" + "st1 { v12.b }[10], [x25]\n" + "st1 { v16.b }[10], [x24]\n" + "st1 { v20.b }[10], [x23]\n" + "st1 { v24.b }[10], [x22]\n" "b 133f\n" "128:" // Height 5: Partial direct writeback: partial_1_8 "tbz x10, #0, 133f\n" "st1 { v8.b }[8], [x11]\n" - "st1 { v12.b }[8], [x24]\n" - "st1 { v16.b }[8], [x23]\n" - "st1 { v20.b }[8], [x22]\n" - "st1 { v24.b }[8], [x21]\n" + "st1 { v12.b }[8], [x25]\n" + "st1 { v16.b }[8], [x24]\n" + "st1 { v20.b }[8], [x23]\n" + "st1 { v24.b }[8], [x22]\n" "b 133f\n" "129:" // Height 5: Partial direct writeback: partial_4_0 "tbz x10, #2, 131f\n" "str s8, [x11], #0x4\n" - "str s12, [x24], #0x4\n" - "str s16, [x23], #0x4\n" - "str s20, [x22], #0x4\n" - "str s24, [x21], #0x4\n" + "str s12, [x25], #0x4\n" + "str s16, [x24], #0x4\n" + "str s20, [x23], #0x4\n" + "str s24, [x22], #0x4\n" "tbz x10, #1, 130f\n" "st1 { v8.h }[2], [x11], #0x2\n" - "st1 { v12.h }[2], [x24], #0x2\n" - "st1 { v16.h }[2], [x23], #0x2\n" - "st1 { v20.h }[2], [x22], #0x2\n" - "st1 { v24.h }[2], [x21], #0x2\n" + "st1 { v12.h }[2], [x25], #0x2\n" + "st1 { v16.h }[2], [x24], #0x2\n" + "st1 { v20.h }[2], [x23], #0x2\n" + "st1 { v24.h }[2], [x22], #0x2\n" "tbz x10, #0, 133f\n" "st1 { v8.b }[6], [x11]\n" - "st1 { v12.b }[6], [x24]\n" - "st1 { v16.b }[6], [x23]\n" - "st1 { v20.b }[6], [x22]\n" - "st1 { v24.b }[6], [x21]\n" + "st1 { v12.b }[6], [x25]\n" + "st1 { v16.b }[6], [x24]\n" + "st1 { v20.b }[6], [x23]\n" + "st1 { v24.b }[6], [x22]\n" "b 133f\n" "130:" // Height 5: Partial direct writeback: partial_1_4 "tbz x10, #0, 133f\n" "st1 { v8.b }[4], [x11]\n" - "st1 { v12.b }[4], [x24]\n" - "st1 { v16.b }[4], [x23]\n" - "st1 { v20.b }[4], [x22]\n" - "st1 { v24.b }[4], [x21]\n" + "st1 { v12.b }[4], [x25]\n" + "st1 { v16.b }[4], [x24]\n" + "st1 { v20.b }[4], [x23]\n" + "st1 { v24.b }[4], [x22]\n" "b 133f\n" "131:" // Height 5: Partial direct writeback: partial_2_0 "tbz x10, #1, 132f\n" "str h8, [x11], #0x2\n" - "str h12, [x24], #0x2\n" - "str h16, [x23], #0x2\n" - "str h20, [x22], #0x2\n" - "str h24, [x21], #0x2\n" + "str h12, [x25], #0x2\n" + "str h16, [x24], #0x2\n" + "str h20, [x23], #0x2\n" + "str h24, [x22], #0x2\n" "tbz x10, #0, 133f\n" "st1 { v8.b }[2], [x11]\n" - "st1 { v12.b }[2], [x24]\n" - "st1 { v16.b }[2], [x23]\n" - "st1 { v20.b }[2], [x22]\n" - "st1 { v24.b }[2], [x21]\n" + "st1 { v12.b }[2], [x25]\n" + "st1 { v16.b }[2], [x24]\n" + "st1 { v20.b }[2], [x23]\n" + "st1 { v24.b }[2], [x22]\n" "b 133f\n" "132:" // Height 5: Partial direct writeback: partial_1_0 "str b8, [x11, #0x0]\n" - "str b12, [x24, #0x0]\n" - "str b16, [x23, #0x0]\n" - "str b20, [x22, #0x0]\n" - "str b24, [x21, #0x0]\n" + "str b12, [x25, #0x0]\n" + "str b16, [x24, #0x0]\n" + "str b20, [x23, #0x0]\n" + "str b24, [x22, #0x0]\n" "133:" // Height 5: Partial direct writeback: Done "b 135f\n" "134:" // Height 5: Full writeback "str q8, [x11, #0x0]\n" "add x11, x11, #0x10\n" - "str q12, [x24, #0x0]\n" - "str q16, [x23, #0x0]\n" - "str q20, [x22, #0x0]\n" - "str q24, [x21, #0x0]\n" + "str q12, [x25, #0x0]\n" + "str q16, [x24, #0x0]\n" + "str q20, [x23, #0x0]\n" + "str q24, [x22, #0x0]\n" "135:" // Height 5: Writeback done "subs x10, x10, #0x10\n" "bgt 110b\n" @@ -2698,16 +2697,16 @@ void a64_hybrid_s8qs_dot_6x16 ( "139:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 140f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 141f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -2719,11 +2718,11 @@ void a64_hybrid_s8qs_dot_6x16 ( "b 141f\n" "140:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" + "add x21, x22, x21\n" "141:" // Height 6: input setup done "cmp x27, #0x10\n" "blt 144f\n" @@ -3002,43 +3001,43 @@ void a64_hybrid_s8qs_dot_6x16 ( "cmp x27, #0x4\n" "blt 146f\n" "145:" // Height 6: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s7, [x26], #0x4\n" + "ldr s6, [x25], #0x4\n" "sub x27, x27, #0x4\n" "cmp x27, #0x4\n" - "ldr s2, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr s4, [x22], #0x4\n" - "ldr s5, [x21], #0x4\n" - "ldr q6, [x9, #0x0]\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - ".inst 0x4f85e0dc // sdot v28.4s, v6.16b, v5.4b[0]\n" - "ldr q6, [x9, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - ".inst 0x4f85e0fd // sdot v29.4s, v7.16b, v5.4b[0]\n" - "ldr q7, [x9, #0x30]\n" + "ldr s5, [x24], #0x4\n" + "ldr s4, [x23], #0x4\n" + "ldr s3, [x22], #0x4\n" + "ldr s2, [x21], #0x4\n" + "ldr q1, [x9, #0x0]\n" + "ldr q0, [x9, #0x10]\n" + ".inst 0x4f87e028 // sdot v8.4s, v1.16b, v7.4b[0]\n" + ".inst 0x4f86e02c // sdot v12.4s, v1.16b, v6.4b[0]\n" + ".inst 0x4f85e030 // sdot v16.4s, v1.16b, v5.4b[0]\n" + ".inst 0x4f84e034 // sdot v20.4s, v1.16b, v4.4b[0]\n" + ".inst 0x4f83e038 // sdot v24.4s, v1.16b, v3.4b[0]\n" + ".inst 0x4f82e03c // sdot v28.4s, v1.16b, v2.4b[0]\n" + "ldr q1, [x9, #0x20]\n" + ".inst 0x4f87e009 // sdot v9.4s, v0.16b, v7.4b[0]\n" + ".inst 0x4f86e00d // sdot v13.4s, v0.16b, v6.4b[0]\n" + ".inst 0x4f85e011 // sdot v17.4s, v0.16b, v5.4b[0]\n" + ".inst 0x4f84e015 // sdot v21.4s, v0.16b, v4.4b[0]\n" + ".inst 0x4f83e019 // sdot v25.4s, v0.16b, v3.4b[0]\n" + ".inst 0x4f82e01d // sdot v29.4s, v0.16b, v2.4b[0]\n" + "ldr q0, [x9, #0x30]\n" "add x9, x9, #0x40\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - ".inst 0x4f85e0de // sdot v30.4s, v6.16b, v5.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" - ".inst 0x4f85e0ff // sdot v31.4s, v7.16b, v5.4b[0]\n" + ".inst 0x4f87e02a // sdot v10.4s, v1.16b, v7.4b[0]\n" + ".inst 0x4f86e02e // sdot v14.4s, v1.16b, v6.4b[0]\n" + ".inst 0x4f85e032 // sdot v18.4s, v1.16b, v5.4b[0]\n" + ".inst 0x4f84e036 // sdot v22.4s, v1.16b, v4.4b[0]\n" + ".inst 0x4f83e03a // sdot v26.4s, v1.16b, v3.4b[0]\n" + ".inst 0x4f82e03e // sdot v30.4s, v1.16b, v2.4b[0]\n" + ".inst 0x4f87e00b // sdot v11.4s, v0.16b, v7.4b[0]\n" + ".inst 0x4f86e00f // sdot v15.4s, v0.16b, v6.4b[0]\n" + ".inst 0x4f85e013 // sdot v19.4s, v0.16b, v5.4b[0]\n" + ".inst 0x4f84e017 // sdot v23.4s, v0.16b, v4.4b[0]\n" + ".inst 0x4f83e01b // sdot v27.4s, v0.16b, v3.4b[0]\n" + ".inst 0x4f82e01f // sdot v31.4s, v0.16b, v2.4b[0]\n" "bge 145b\n" "146:" // Height 6: Multiply loop: Skip odd blocks "cbz x27, 149f\n" @@ -3065,81 +3064,81 @@ void a64_hybrid_s8qs_dot_6x16 ( "ldr b4, [x22, #0x0]\n" "ldr b5, [x21, #0x0]\n" "148:" // Height 6: Multiply loop: Ragged operand read: Done - "ldr q6, [x9, #0x0]\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - ".inst 0x4f85e0dc // sdot v28.4s, v6.16b, v5.4b[0]\n" - "ldr q6, [x9, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - ".inst 0x4f85e0fd // sdot v29.4s, v7.16b, v5.4b[0]\n" - "ldr q7, [x9, #0x30]\n" + "ldr q7, [x9, #0x0]\n" + "ldr q6, [x9, #0x10]\n" + ".inst 0x4f80e0e8 // sdot v8.4s, v7.16b, v0.4b[0]\n" + ".inst 0x4f81e0ec // sdot v12.4s, v7.16b, v1.4b[0]\n" + ".inst 0x4f82e0f0 // sdot v16.4s, v7.16b, v2.4b[0]\n" + ".inst 0x4f83e0f4 // sdot v20.4s, v7.16b, v3.4b[0]\n" + ".inst 0x4f84e0f8 // sdot v24.4s, v7.16b, v4.4b[0]\n" + ".inst 0x4f85e0fc // sdot v28.4s, v7.16b, v5.4b[0]\n" + "ldr q7, [x9, #0x20]\n" + ".inst 0x4f80e0c9 // sdot v9.4s, v6.16b, v0.4b[0]\n" + ".inst 0x4f81e0cd // sdot v13.4s, v6.16b, v1.4b[0]\n" + ".inst 0x4f82e0d1 // sdot v17.4s, v6.16b, v2.4b[0]\n" + ".inst 0x4f83e0d5 // sdot v21.4s, v6.16b, v3.4b[0]\n" + ".inst 0x4f84e0d9 // sdot v25.4s, v6.16b, v4.4b[0]\n" + ".inst 0x4f85e0dd // sdot v29.4s, v6.16b, v5.4b[0]\n" + "ldr q6, [x9, #0x30]\n" "add x9, x9, #0x40\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - ".inst 0x4f85e0de // sdot v30.4s, v6.16b, v5.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" - ".inst 0x4f85e0ff // sdot v31.4s, v7.16b, v5.4b[0]\n" + ".inst 0x4f80e0ea // sdot v10.4s, v7.16b, v0.4b[0]\n" + ".inst 0x4f81e0ee // sdot v14.4s, v7.16b, v1.4b[0]\n" + ".inst 0x4f82e0f2 // sdot v18.4s, v7.16b, v2.4b[0]\n" + ".inst 0x4f83e0f6 // sdot v22.4s, v7.16b, v3.4b[0]\n" + ".inst 0x4f84e0fa // sdot v26.4s, v7.16b, v4.4b[0]\n" + ".inst 0x4f85e0fe // sdot v30.4s, v7.16b, v5.4b[0]\n" + ".inst 0x4f80e0cb // sdot v11.4s, v6.16b, v0.4b[0]\n" + ".inst 0x4f81e0cf // sdot v15.4s, v6.16b, v1.4b[0]\n" + ".inst 0x4f82e0d3 // sdot v19.4s, v6.16b, v2.4b[0]\n" + ".inst 0x4f83e0d7 // sdot v23.4s, v6.16b, v3.4b[0]\n" + ".inst 0x4f84e0db // sdot v27.4s, v6.16b, v4.4b[0]\n" + ".inst 0x4f85e0df // sdot v31.4s, v6.16b, v5.4b[0]\n" "149:" // Height 6: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 139b\n" - "ldr q0, [x14, #0x0]\n" - "ldr q1, [x14, #0x10]\n" - "add v8.4s, v8.4s, v0.4s\n" - "add v9.4s, v9.4s, v1.4s\n" - "ldr q2, [x14, #0x20]\n" - "ldr q3, [x14, #0x30]\n" - "add v10.4s, v10.4s, v2.4s\n" - "add v11.4s, v11.4s, v3.4s\n" + "ldr q3, [x14, #0x0]\n" + "ldr q2, [x14, #0x10]\n" + "add v8.4s, v8.4s, v3.4s\n" + "add v9.4s, v9.4s, v2.4s\n" + "ldr q1, [x14, #0x20]\n" + "ldr q0, [x14, #0x30]\n" + "add v10.4s, v10.4s, v1.4s\n" + "add v11.4s, v11.4s, v0.4s\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x11, x20\n" - "add x23, x24, x20\n" + "add x25, x11, x20\n" + "add x24, x25, x20\n" "prfm pstl1keep, [x11, #0x0]\n" + "add x23, x24, x20\n" "add x22, x23, x20\n" - "add x21, x22, x20\n" + "prfm pstl1keep, [x25, #0x0]\n" "prfm pstl1keep, [x24, #0x0]\n" + "add x21, x22, x20\n" "prfm pstl1keep, [x23, #0x0]\n" - "add x20, x21, x20\n" "prfm pstl1keep, [x22, #0x0]\n" + "add v12.4s, v12.4s, v3.4s\n" "prfm pstl1keep, [x21, #0x0]\n" - "add v12.4s, v12.4s, v0.4s\n" - "prfm pstl1keep, [x20, #0x0]\n" - "add v13.4s, v13.4s, v1.4s\n" - "add v14.4s, v14.4s, v2.4s\n" + "add v13.4s, v13.4s, v2.4s\n" + "add v14.4s, v14.4s, v1.4s\n" "add x14, x14, #0x40\n" - "add v15.4s, v15.4s, v3.4s\n" - "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" - "add v20.4s, v20.4s, v0.4s\n" - "add v21.4s, v21.4s, v1.4s\n" - "add v22.4s, v22.4s, v2.4s\n" - "add v23.4s, v23.4s, v3.4s\n" - "add v24.4s, v24.4s, v0.4s\n" - "add v25.4s, v25.4s, v1.4s\n" - "add v26.4s, v26.4s, v2.4s\n" - "add v27.4s, v27.4s, v3.4s\n" - "add v28.4s, v28.4s, v0.4s\n" - "add v29.4s, v29.4s, v1.4s\n" - "add v30.4s, v30.4s, v2.4s\n" - "add v31.4s, v31.4s, v3.4s\n" + "add v15.4s, v15.4s, v0.4s\n" + "add v16.4s, v16.4s, v3.4s\n" + "add v17.4s, v17.4s, v2.4s\n" + "add v18.4s, v18.4s, v1.4s\n" + "add v19.4s, v19.4s, v0.4s\n" + "add v20.4s, v20.4s, v3.4s\n" + "add v21.4s, v21.4s, v2.4s\n" + "add v22.4s, v22.4s, v1.4s\n" + "add v23.4s, v23.4s, v0.4s\n" + "add v24.4s, v24.4s, v3.4s\n" + "add v25.4s, v25.4s, v2.4s\n" + "add v26.4s, v26.4s, v1.4s\n" + "add v27.4s, v27.4s, v0.4s\n" + "add v28.4s, v28.4s, v3.4s\n" + "add v29.4s, v29.4s, v2.4s\n" + "add v30.4s, v30.4s, v1.4s\n" + "add v31.4s, v31.4s, v0.4s\n" "tbz %x[flags], #4, 150f\n" "ldr q0, [x12, #0x0]\n" "ldr q4, [x13, #0x0]\n" @@ -3153,10 +3152,10 @@ void a64_hybrid_s8qs_dot_6x16 ( "add x13, x13, #0x40\n" "b 151f\n" "150:" // Height 6: per layer parameters - "add x25, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x25]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v4.4s }, [x20]\n" "mov v1.16b, v0.16b\n" "mov v5.16b, v4.16b\n" "mov v2.16b, v0.16b\n" @@ -3189,91 +3188,91 @@ void a64_hybrid_s8qs_dot_6x16 ( "sqrdmulh v30.4s, v30.4s, v6.4s\n" "sqrdmulh v31.4s, v31.4s, v7.4s\n" "tbz %x[flags], #5, 152f\n" - "and v4.16b, v8.16b, v0.16b\n" - "and v5.16b, v9.16b, v1.16b\n" - "and v6.16b, v10.16b, v2.16b\n" - "and v7.16b, v11.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" + "and v7.16b, v8.16b, v0.16b\n" + "and v6.16b, v9.16b, v1.16b\n" + "and v5.16b, v10.16b, v2.16b\n" + "and v4.16b, v11.16b, v3.16b\n" "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v4.4s\n" - "sqadd v9.4s, v9.4s, v5.4s\n" - "sqadd v10.4s, v10.4s, v6.4s\n" - "sqadd v11.4s, v11.4s, v7.4s\n" - "and v4.16b, v12.16b, v0.16b\n" - "and v5.16b, v13.16b, v1.16b\n" - "and v6.16b, v14.16b, v2.16b\n" - "and v7.16b, v15.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v12.4s, v12.4s, v4.4s\n" - "sqadd v13.4s, v13.4s, v5.4s\n" - "sqadd v14.4s, v14.4s, v6.4s\n" - "sqadd v15.4s, v15.4s, v7.4s\n" - "and v4.16b, v16.16b, v0.16b\n" - "and v5.16b, v17.16b, v1.16b\n" - "and v6.16b, v18.16b, v2.16b\n" - "and v7.16b, v19.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" - "and v4.16b, v20.16b, v0.16b\n" - "and v5.16b, v21.16b, v1.16b\n" - "and v6.16b, v22.16b, v2.16b\n" - "and v7.16b, v23.16b, v3.16b\n" "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" + "sqadd v8.4s, v8.4s, v7.4s\n" + "sqadd v9.4s, v9.4s, v6.4s\n" + "sqadd v10.4s, v10.4s, v5.4s\n" + "sqadd v11.4s, v11.4s, v4.4s\n" + "and v7.16b, v12.16b, v0.16b\n" + "and v6.16b, v13.16b, v1.16b\n" + "and v5.16b, v14.16b, v2.16b\n" + "and v4.16b, v15.16b, v3.16b\n" "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v20.4s, v20.4s, v4.4s\n" - "sqadd v21.4s, v21.4s, v5.4s\n" - "sqadd v22.4s, v22.4s, v6.4s\n" - "sqadd v23.4s, v23.4s, v7.4s\n" - "and v4.16b, v24.16b, v0.16b\n" - "and v5.16b, v25.16b, v1.16b\n" - "and v6.16b, v26.16b, v2.16b\n" - "and v7.16b, v27.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" "sshr v6.4s, v6.4s, #0x1f\n" + "sshr v5.4s, v5.4s, #0x1f\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sqadd v12.4s, v12.4s, v7.4s\n" + "sqadd v13.4s, v13.4s, v6.4s\n" + "sqadd v14.4s, v14.4s, v5.4s\n" + "sqadd v15.4s, v15.4s, v4.4s\n" + "and v7.16b, v16.16b, v0.16b\n" + "and v6.16b, v17.16b, v1.16b\n" + "and v5.16b, v18.16b, v2.16b\n" + "and v4.16b, v19.16b, v3.16b\n" "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v24.4s, v24.4s, v4.4s\n" - "sqadd v25.4s, v25.4s, v5.4s\n" - "sqadd v26.4s, v26.4s, v6.4s\n" - "sqadd v27.4s, v27.4s, v7.4s\n" - "and v4.16b, v28.16b, v0.16b\n" - "and v5.16b, v29.16b, v1.16b\n" - "and v6.16b, v30.16b, v2.16b\n" - "and v7.16b, v31.16b, v3.16b\n" + "sshr v6.4s, v6.4s, #0x1f\n" + "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v7.4s\n" + "sqadd v17.4s, v17.4s, v6.4s\n" + "sqadd v18.4s, v18.4s, v5.4s\n" + "sqadd v19.4s, v19.4s, v4.4s\n" + "and v7.16b, v20.16b, v0.16b\n" + "and v6.16b, v21.16b, v1.16b\n" + "and v5.16b, v22.16b, v2.16b\n" + "and v4.16b, v23.16b, v3.16b\n" + "sshr v7.4s, v7.4s, #0x1f\n" + "sshr v6.4s, v6.4s, #0x1f\n" "sshr v5.4s, v5.4s, #0x1f\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v7.4s\n" + "sqadd v21.4s, v21.4s, v6.4s\n" + "sqadd v22.4s, v22.4s, v5.4s\n" + "sqadd v23.4s, v23.4s, v4.4s\n" + "and v7.16b, v24.16b, v0.16b\n" + "and v6.16b, v25.16b, v1.16b\n" + "and v5.16b, v26.16b, v2.16b\n" + "and v4.16b, v27.16b, v3.16b\n" + "sshr v7.4s, v7.4s, #0x1f\n" "sshr v6.4s, v6.4s, #0x1f\n" + "sshr v5.4s, v5.4s, #0x1f\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sqadd v24.4s, v24.4s, v7.4s\n" + "sqadd v25.4s, v25.4s, v6.4s\n" + "sqadd v26.4s, v26.4s, v5.4s\n" + "sqadd v27.4s, v27.4s, v4.4s\n" + "and v7.16b, v28.16b, v0.16b\n" + "and v6.16b, v29.16b, v1.16b\n" + "and v5.16b, v30.16b, v2.16b\n" + "and v4.16b, v31.16b, v3.16b\n" "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v28.4s, v28.4s, v4.4s\n" - "sqadd v29.4s, v29.4s, v5.4s\n" - "sqadd v30.4s, v30.4s, v6.4s\n" - "sqadd v31.4s, v31.4s, v7.4s\n" + "sshr v6.4s, v6.4s, #0x1f\n" + "sshr v5.4s, v5.4s, #0x1f\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sqadd v28.4s, v28.4s, v7.4s\n" + "sqadd v29.4s, v29.4s, v6.4s\n" + "sqadd v30.4s, v30.4s, v5.4s\n" + "sqadd v31.4s, v31.4s, v4.4s\n" "152:" // Height 6: no shift correction - "add x25, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v6.4s }, [x20]\n" "srshl v8.4s, v8.4s, v0.4s\n" "srshl v9.4s, v9.4s, v1.4s\n" "srshl v10.4s, v10.4s, v2.4s\n" "srshl v11.4s, v11.4s, v3.4s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x25]\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v5.4s }, [x20]\n" "srshl v12.4s, v12.4s, v0.4s\n" "srshl v13.4s, v13.4s, v1.4s\n" - "add x25, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x25]\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v4.4s }, [x20]\n" "srshl v14.4s, v14.4s, v2.4s\n" "srshl v15.4s, v15.4s, v3.4s\n" "cmp x10, #0x10\n" @@ -3293,225 +3292,225 @@ void a64_hybrid_s8qs_dot_6x16 ( "srshl v29.4s, v29.4s, v1.4s\n" "srshl v30.4s, v30.4s, v2.4s\n" "srshl v31.4s, v31.4s, v3.4s\n" - "add v8.4s, v8.4s, v4.4s\n" - "add v9.4s, v9.4s, v4.4s\n" - "add v10.4s, v10.4s, v4.4s\n" - "add v11.4s, v11.4s, v4.4s\n" - "add v12.4s, v12.4s, v4.4s\n" - "add v13.4s, v13.4s, v4.4s\n" - "add v14.4s, v14.4s, v4.4s\n" - "add v15.4s, v15.4s, v4.4s\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v23.4s, v23.4s, v4.4s\n" - "add v24.4s, v24.4s, v4.4s\n" - "add v25.4s, v25.4s, v4.4s\n" - "add v26.4s, v26.4s, v4.4s\n" - "add v27.4s, v27.4s, v4.4s\n" - "add v28.4s, v28.4s, v4.4s\n" - "add v29.4s, v29.4s, v4.4s\n" - "add v30.4s, v30.4s, v4.4s\n" - "add v31.4s, v31.4s, v4.4s\n" - "smin v8.4s, v8.4s, v6.4s\n" - "smin v9.4s, v9.4s, v6.4s\n" - "smin v10.4s, v10.4s, v6.4s\n" - "smin v11.4s, v11.4s, v6.4s\n" - "smin v12.4s, v12.4s, v6.4s\n" - "smin v13.4s, v13.4s, v6.4s\n" - "smin v14.4s, v14.4s, v6.4s\n" - "smin v15.4s, v15.4s, v6.4s\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "smin v24.4s, v24.4s, v6.4s\n" - "smin v25.4s, v25.4s, v6.4s\n" - "smin v26.4s, v26.4s, v6.4s\n" - "smin v27.4s, v27.4s, v6.4s\n" - "smin v28.4s, v28.4s, v6.4s\n" - "smin v29.4s, v29.4s, v6.4s\n" - "smin v30.4s, v30.4s, v6.4s\n" - "smin v31.4s, v31.4s, v6.4s\n" - "smax v8.4s, v8.4s, v5.4s\n" - "smax v9.4s, v9.4s, v5.4s\n" - "smax v10.4s, v10.4s, v5.4s\n" - "smax v11.4s, v11.4s, v5.4s\n" - "smax v12.4s, v12.4s, v5.4s\n" - "smax v13.4s, v13.4s, v5.4s\n" - "smax v14.4s, v14.4s, v5.4s\n" - "smax v15.4s, v15.4s, v5.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" - "smax v24.4s, v24.4s, v5.4s\n" - "smax v25.4s, v25.4s, v5.4s\n" - "smax v26.4s, v26.4s, v5.4s\n" - "smax v27.4s, v27.4s, v5.4s\n" - "smax v28.4s, v28.4s, v5.4s\n" - "smax v29.4s, v29.4s, v5.4s\n" - "smax v30.4s, v30.4s, v5.4s\n" - "smax v31.4s, v31.4s, v5.4s\n" + "add v8.4s, v8.4s, v6.4s\n" + "add v9.4s, v9.4s, v6.4s\n" + "add v10.4s, v10.4s, v6.4s\n" + "add v11.4s, v11.4s, v6.4s\n" + "add v12.4s, v12.4s, v6.4s\n" + "add v13.4s, v13.4s, v6.4s\n" + "add v14.4s, v14.4s, v6.4s\n" + "add v15.4s, v15.4s, v6.4s\n" + "add v16.4s, v16.4s, v6.4s\n" + "add v17.4s, v17.4s, v6.4s\n" + "add v18.4s, v18.4s, v6.4s\n" + "add v19.4s, v19.4s, v6.4s\n" + "add v20.4s, v20.4s, v6.4s\n" + "add v21.4s, v21.4s, v6.4s\n" + "add v22.4s, v22.4s, v6.4s\n" + "add v23.4s, v23.4s, v6.4s\n" + "add v24.4s, v24.4s, v6.4s\n" + "add v25.4s, v25.4s, v6.4s\n" + "add v26.4s, v26.4s, v6.4s\n" + "add v27.4s, v27.4s, v6.4s\n" + "add v28.4s, v28.4s, v6.4s\n" + "add v29.4s, v29.4s, v6.4s\n" + "add v30.4s, v30.4s, v6.4s\n" + "add v31.4s, v31.4s, v6.4s\n" + "smin v8.4s, v8.4s, v5.4s\n" + "smin v9.4s, v9.4s, v5.4s\n" + "smin v10.4s, v10.4s, v5.4s\n" + "smin v11.4s, v11.4s, v5.4s\n" + "smin v12.4s, v12.4s, v5.4s\n" + "smin v13.4s, v13.4s, v5.4s\n" + "smin v14.4s, v14.4s, v5.4s\n" + "smin v15.4s, v15.4s, v5.4s\n" + "smin v16.4s, v16.4s, v5.4s\n" + "smin v17.4s, v17.4s, v5.4s\n" + "smin v18.4s, v18.4s, v5.4s\n" + "smin v19.4s, v19.4s, v5.4s\n" + "smin v20.4s, v20.4s, v5.4s\n" + "smin v21.4s, v21.4s, v5.4s\n" + "smin v22.4s, v22.4s, v5.4s\n" + "smin v23.4s, v23.4s, v5.4s\n" + "smin v24.4s, v24.4s, v5.4s\n" + "smin v25.4s, v25.4s, v5.4s\n" + "smin v26.4s, v26.4s, v5.4s\n" + "smin v27.4s, v27.4s, v5.4s\n" + "smin v28.4s, v28.4s, v5.4s\n" + "smin v29.4s, v29.4s, v5.4s\n" + "smin v30.4s, v30.4s, v5.4s\n" + "smin v31.4s, v31.4s, v5.4s\n" + "smax v8.4s, v8.4s, v4.4s\n" + "smax v9.4s, v9.4s, v4.4s\n" + "smax v10.4s, v10.4s, v4.4s\n" + "smax v11.4s, v11.4s, v4.4s\n" + "smax v12.4s, v12.4s, v4.4s\n" + "smax v13.4s, v13.4s, v4.4s\n" + "smax v14.4s, v14.4s, v4.4s\n" + "smax v15.4s, v15.4s, v4.4s\n" + "smax v16.4s, v16.4s, v4.4s\n" + "smax v17.4s, v17.4s, v4.4s\n" + "smax v18.4s, v18.4s, v4.4s\n" + "smax v19.4s, v19.4s, v4.4s\n" + "smax v20.4s, v20.4s, v4.4s\n" + "smax v21.4s, v21.4s, v4.4s\n" + "smax v22.4s, v22.4s, v4.4s\n" + "smax v23.4s, v23.4s, v4.4s\n" + "smax v24.4s, v24.4s, v4.4s\n" + "smax v25.4s, v25.4s, v4.4s\n" + "smax v26.4s, v26.4s, v4.4s\n" + "smax v27.4s, v27.4s, v4.4s\n" + "smax v28.4s, v28.4s, v4.4s\n" + "smax v29.4s, v29.4s, v4.4s\n" + "smax v30.4s, v30.4s, v4.4s\n" + "smax v31.4s, v31.4s, v4.4s\n" "uzp1 v8.8h, v8.8h, v9.8h\n" - "uzp1 v9.8h, v10.8h, v11.8h\n" + "uzp1 v2.8h, v10.8h, v11.8h\n" "uzp1 v12.8h, v12.8h, v13.8h\n" - "uzp1 v13.8h, v14.8h, v15.8h\n" + "uzp1 v1.8h, v14.8h, v15.8h\n" "uzp1 v16.8h, v16.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v0.8h, v18.8h, v19.8h\n" "uzp1 v20.8h, v20.8h, v21.8h\n" - "uzp1 v21.8h, v22.8h, v23.8h\n" + "uzp1 v19.8h, v22.8h, v23.8h\n" "uzp1 v24.8h, v24.8h, v25.8h\n" - "uzp1 v25.8h, v26.8h, v27.8h\n" + "uzp1 v18.8h, v26.8h, v27.8h\n" "uzp1 v28.8h, v28.8h, v29.8h\n" - "uzp1 v29.8h, v30.8h, v31.8h\n" - "uzp1 v8.16b, v8.16b, v9.16b\n" - "uzp1 v12.16b, v12.16b, v13.16b\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" - "uzp1 v20.16b, v20.16b, v21.16b\n" - "uzp1 v24.16b, v24.16b, v25.16b\n" - "uzp1 v28.16b, v28.16b, v29.16b\n" + "uzp1 v17.8h, v30.8h, v31.8h\n" + "uzp1 v8.16b, v8.16b, v2.16b\n" + "uzp1 v12.16b, v12.16b, v1.16b\n" + "uzp1 v16.16b, v16.16b, v0.16b\n" + "uzp1 v20.16b, v20.16b, v19.16b\n" + "uzp1 v24.16b, v24.16b, v18.16b\n" + "uzp1 v28.16b, v28.16b, v17.16b\n" "bge 161f\n" "tbz x10, #3, 156f\n" "str d8, [x11], #0x8\n" - "str d12, [x24], #0x8\n" - "str d16, [x23], #0x8\n" - "str d20, [x22], #0x8\n" - "str d24, [x21], #0x8\n" - "str d28, [x20], #0x8\n" + "str d12, [x25], #0x8\n" + "str d16, [x24], #0x8\n" + "str d20, [x23], #0x8\n" + "str d24, [x22], #0x8\n" + "str d28, [x21], #0x8\n" "tbz x10, #2, 154f\n" "st1 { v8.s }[2], [x11], #0x4\n" - "st1 { v12.s }[2], [x24], #0x4\n" - "st1 { v16.s }[2], [x23], #0x4\n" - "st1 { v20.s }[2], [x22], #0x4\n" - "st1 { v24.s }[2], [x21], #0x4\n" - "st1 { v28.s }[2], [x20], #0x4\n" + "st1 { v12.s }[2], [x25], #0x4\n" + "st1 { v16.s }[2], [x24], #0x4\n" + "st1 { v20.s }[2], [x23], #0x4\n" + "st1 { v24.s }[2], [x22], #0x4\n" + "st1 { v28.s }[2], [x21], #0x4\n" "tbz x10, #1, 153f\n" "st1 { v8.h }[6], [x11], #0x2\n" - "st1 { v12.h }[6], [x24], #0x2\n" - "st1 { v16.h }[6], [x23], #0x2\n" - "st1 { v20.h }[6], [x22], #0x2\n" - "st1 { v24.h }[6], [x21], #0x2\n" - "st1 { v28.h }[6], [x20], #0x2\n" + "st1 { v12.h }[6], [x25], #0x2\n" + "st1 { v16.h }[6], [x24], #0x2\n" + "st1 { v20.h }[6], [x23], #0x2\n" + "st1 { v24.h }[6], [x22], #0x2\n" + "st1 { v28.h }[6], [x21], #0x2\n" "tbz x10, #0, 160f\n" "st1 { v8.b }[14], [x11]\n" - "st1 { v12.b }[14], [x24]\n" - "st1 { v16.b }[14], [x23]\n" - "st1 { v20.b }[14], [x22]\n" - "st1 { v24.b }[14], [x21]\n" - "st1 { v28.b }[14], [x20]\n" + "st1 { v12.b }[14], [x25]\n" + "st1 { v16.b }[14], [x24]\n" + "st1 { v20.b }[14], [x23]\n" + "st1 { v24.b }[14], [x22]\n" + "st1 { v28.b }[14], [x21]\n" "b 160f\n" "153:" // Height 6: Partial direct writeback: partial_1_12 "tbz x10, #0, 160f\n" "st1 { v8.b }[12], [x11]\n" - "st1 { v12.b }[12], [x24]\n" - "st1 { v16.b }[12], [x23]\n" - "st1 { v20.b }[12], [x22]\n" - "st1 { v24.b }[12], [x21]\n" - "st1 { v28.b }[12], [x20]\n" + "st1 { v12.b }[12], [x25]\n" + "st1 { v16.b }[12], [x24]\n" + "st1 { v20.b }[12], [x23]\n" + "st1 { v24.b }[12], [x22]\n" + "st1 { v28.b }[12], [x21]\n" "b 160f\n" "154:" // Height 6: Partial direct writeback: partial_2_8 "tbz x10, #1, 155f\n" "st1 { v8.h }[4], [x11], #0x2\n" - "st1 { v12.h }[4], [x24], #0x2\n" - "st1 { v16.h }[4], [x23], #0x2\n" - "st1 { v20.h }[4], [x22], #0x2\n" - "st1 { v24.h }[4], [x21], #0x2\n" - "st1 { v28.h }[4], [x20], #0x2\n" + "st1 { v12.h }[4], [x25], #0x2\n" + "st1 { v16.h }[4], [x24], #0x2\n" + "st1 { v20.h }[4], [x23], #0x2\n" + "st1 { v24.h }[4], [x22], #0x2\n" + "st1 { v28.h }[4], [x21], #0x2\n" "tbz x10, #0, 160f\n" "st1 { v8.b }[10], [x11]\n" - "st1 { v12.b }[10], [x24]\n" - "st1 { v16.b }[10], [x23]\n" - "st1 { v20.b }[10], [x22]\n" - "st1 { v24.b }[10], [x21]\n" - "st1 { v28.b }[10], [x20]\n" + "st1 { v12.b }[10], [x25]\n" + "st1 { v16.b }[10], [x24]\n" + "st1 { v20.b }[10], [x23]\n" + "st1 { v24.b }[10], [x22]\n" + "st1 { v28.b }[10], [x21]\n" "b 160f\n" "155:" // Height 6: Partial direct writeback: partial_1_8 "tbz x10, #0, 160f\n" "st1 { v8.b }[8], [x11]\n" - "st1 { v12.b }[8], [x24]\n" - "st1 { v16.b }[8], [x23]\n" - "st1 { v20.b }[8], [x22]\n" - "st1 { v24.b }[8], [x21]\n" - "st1 { v28.b }[8], [x20]\n" + "st1 { v12.b }[8], [x25]\n" + "st1 { v16.b }[8], [x24]\n" + "st1 { v20.b }[8], [x23]\n" + "st1 { v24.b }[8], [x22]\n" + "st1 { v28.b }[8], [x21]\n" "b 160f\n" "156:" // Height 6: Partial direct writeback: partial_4_0 "tbz x10, #2, 158f\n" "str s8, [x11], #0x4\n" - "str s12, [x24], #0x4\n" - "str s16, [x23], #0x4\n" - "str s20, [x22], #0x4\n" - "str s24, [x21], #0x4\n" - "str s28, [x20], #0x4\n" + "str s12, [x25], #0x4\n" + "str s16, [x24], #0x4\n" + "str s20, [x23], #0x4\n" + "str s24, [x22], #0x4\n" + "str s28, [x21], #0x4\n" "tbz x10, #1, 157f\n" "st1 { v8.h }[2], [x11], #0x2\n" - "st1 { v12.h }[2], [x24], #0x2\n" - "st1 { v16.h }[2], [x23], #0x2\n" - "st1 { v20.h }[2], [x22], #0x2\n" - "st1 { v24.h }[2], [x21], #0x2\n" - "st1 { v28.h }[2], [x20], #0x2\n" + "st1 { v12.h }[2], [x25], #0x2\n" + "st1 { v16.h }[2], [x24], #0x2\n" + "st1 { v20.h }[2], [x23], #0x2\n" + "st1 { v24.h }[2], [x22], #0x2\n" + "st1 { v28.h }[2], [x21], #0x2\n" "tbz x10, #0, 160f\n" "st1 { v8.b }[6], [x11]\n" - "st1 { v12.b }[6], [x24]\n" - "st1 { v16.b }[6], [x23]\n" - "st1 { v20.b }[6], [x22]\n" - "st1 { v24.b }[6], [x21]\n" - "st1 { v28.b }[6], [x20]\n" + "st1 { v12.b }[6], [x25]\n" + "st1 { v16.b }[6], [x24]\n" + "st1 { v20.b }[6], [x23]\n" + "st1 { v24.b }[6], [x22]\n" + "st1 { v28.b }[6], [x21]\n" "b 160f\n" "157:" // Height 6: Partial direct writeback: partial_1_4 "tbz x10, #0, 160f\n" "st1 { v8.b }[4], [x11]\n" - "st1 { v12.b }[4], [x24]\n" - "st1 { v16.b }[4], [x23]\n" - "st1 { v20.b }[4], [x22]\n" - "st1 { v24.b }[4], [x21]\n" - "st1 { v28.b }[4], [x20]\n" + "st1 { v12.b }[4], [x25]\n" + "st1 { v16.b }[4], [x24]\n" + "st1 { v20.b }[4], [x23]\n" + "st1 { v24.b }[4], [x22]\n" + "st1 { v28.b }[4], [x21]\n" "b 160f\n" "158:" // Height 6: Partial direct writeback: partial_2_0 "tbz x10, #1, 159f\n" "str h8, [x11], #0x2\n" - "str h12, [x24], #0x2\n" - "str h16, [x23], #0x2\n" - "str h20, [x22], #0x2\n" - "str h24, [x21], #0x2\n" - "str h28, [x20], #0x2\n" + "str h12, [x25], #0x2\n" + "str h16, [x24], #0x2\n" + "str h20, [x23], #0x2\n" + "str h24, [x22], #0x2\n" + "str h28, [x21], #0x2\n" "tbz x10, #0, 160f\n" "st1 { v8.b }[2], [x11]\n" - "st1 { v12.b }[2], [x24]\n" - "st1 { v16.b }[2], [x23]\n" - "st1 { v20.b }[2], [x22]\n" - "st1 { v24.b }[2], [x21]\n" - "st1 { v28.b }[2], [x20]\n" + "st1 { v12.b }[2], [x25]\n" + "st1 { v16.b }[2], [x24]\n" + "st1 { v20.b }[2], [x23]\n" + "st1 { v24.b }[2], [x22]\n" + "st1 { v28.b }[2], [x21]\n" "b 160f\n" "159:" // Height 6: Partial direct writeback: partial_1_0 "str b8, [x11, #0x0]\n" - "str b12, [x24, #0x0]\n" - "str b16, [x23, #0x0]\n" - "str b20, [x22, #0x0]\n" - "str b24, [x21, #0x0]\n" - "str b28, [x20, #0x0]\n" + "str b12, [x25, #0x0]\n" + "str b16, [x24, #0x0]\n" + "str b20, [x23, #0x0]\n" + "str b24, [x22, #0x0]\n" + "str b28, [x21, #0x0]\n" "160:" // Height 6: Partial direct writeback: Done "b 162f\n" "161:" // Height 6: Full writeback "str q8, [x11, #0x0]\n" "add x11, x11, #0x10\n" - "str q12, [x24, #0x0]\n" - "str q16, [x23, #0x0]\n" - "str q20, [x22, #0x0]\n" - "str q24, [x21, #0x0]\n" - "str q28, [x20, #0x0]\n" + "str q12, [x25, #0x0]\n" + "str q16, [x24, #0x0]\n" + "str q20, [x23, #0x0]\n" + "str q24, [x22, #0x0]\n" + "str q28, [x21, #0x0]\n" "162:" // Height 6: Writeback done "subs x10, x10, #0x10\n" "bgt 137b\n" @@ -3527,7 +3526,6 @@ void a64_hybrid_s8qs_dot_6x16 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "164:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [flags] "r" (flags), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_multiplier_ptr] "I" (offsetof(KernelArgs, multiplier_ptr)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_shift_ptr] "I" (offsetof(KernelArgs, shift_ptr)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_mmla_6x16.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_mmla_6x16.hpp index 7eacdceae78163d09ed9b6a0017eef1218bb93ef..d0d5f1b80df74cab13d57ddf9fbb762c4ba21f0e 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_mmla_6x16.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_mmla_6x16.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef __aarch64__ + #include "../std_transforms_fixed.hpp" #include "../performance_parameters.hpp" @@ -98,5 +98,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_mmla_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_mmla_6x16/generic.cpp index fc525531b22b690364d640bee1a230a83fb888fe..0771829d375a88a4470c8efcf5fb4b1792cff9c2 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_mmla_6x16/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qs_mmla_6x16/generic.cpp @@ -85,7 +85,6 @@ void a64_hybrid_s8qs_mmla_6x16 ( flags |= 0x20; } __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x6\n" "bge 146f\n" @@ -115,11 +114,11 @@ void a64_hybrid_s8qs_mmla_6x16 ( "4:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 5f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 6f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -135,41 +134,41 @@ void a64_hybrid_s8qs_mmla_6x16 ( "ldr q6, [x9, #0x10]\n" "blt 8f\n" "7:" // Height 1: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x70]\n" - "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x80]\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x90]\n" - ".inst 0x4e87a428 // smmla v8.4s, v1.16b, v7.16b\n" - "ldr q7, [x9, #0xa0]\n" - ".inst 0x4e86a42c // smmla v12.4s, v1.16b, v6.16b\n" - "ldr q6, [x9, #0xb0]\n" - ".inst 0x4e87a429 // smmla v9.4s, v1.16b, v7.16b\n" - "ldr q7, [x9, #0xc0]\n" - ".inst 0x4e86a42d // smmla v13.4s, v1.16b, v6.16b\n" - "ldr q6, [x9, #0xd0]\n" - ".inst 0x4e87a42a // smmla v10.4s, v1.16b, v7.16b\n" - "ldr q7, [x9, #0xe0]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - "ldr q6, [x9, #0xf0]\n" + "trn1 v18.2d, v1.2d, v21.2d\n" + ".inst 0x4e87a648 // smmla v8.4s, v18.16b, v7.16b\n" + "ldr q17, [x9, #0x20]\n" + ".inst 0x4e86a64c // smmla v12.4s, v18.16b, v6.16b\n" + "ldr q16, [x9, #0x30]\n" + ".inst 0x4e91a649 // smmla v9.4s, v18.16b, v17.16b\n" + "ldr q17, [x9, #0x40]\n" + ".inst 0x4e90a64d // smmla v13.4s, v18.16b, v16.16b\n" + "ldr q16, [x9, #0x50]\n" + ".inst 0x4e91a64a // smmla v10.4s, v18.16b, v17.16b\n" + "ldr q17, [x9, #0x60]\n" + ".inst 0x4e90a64e // smmla v14.4s, v18.16b, v16.16b\n" + "ldr q16, [x9, #0x70]\n" + "trn2 v1.2d, v1.2d, v21.2d\n" + ".inst 0x4e91a64b // smmla v11.4s, v18.16b, v17.16b\n" + "ldr q17, [x9, #0x80]\n" + ".inst 0x4e90a64f // smmla v15.4s, v18.16b, v16.16b\n" + "ldr q16, [x9, #0x90]\n" + ".inst 0x4e91a428 // smmla v8.4s, v1.16b, v17.16b\n" + "ldr q17, [x9, #0xa0]\n" + ".inst 0x4e90a42c // smmla v12.4s, v1.16b, v16.16b\n" + "ldr q16, [x9, #0xb0]\n" + ".inst 0x4e91a429 // smmla v9.4s, v1.16b, v17.16b\n" + "ldr q17, [x9, #0xc0]\n" + ".inst 0x4e90a42d // smmla v13.4s, v1.16b, v16.16b\n" + "ldr q16, [x9, #0xd0]\n" + ".inst 0x4e91a42a // smmla v10.4s, v1.16b, v17.16b\n" + "ldr q17, [x9, #0xe0]\n" + ".inst 0x4e90a42e // smmla v14.4s, v1.16b, v16.16b\n" + "ldr q16, [x9, #0xf0]\n" "sub x27, x27, #0x10\n" "add x26, x26, #0x10\n" "cmp x27, #0x20\n" - ".inst 0x4e87a42b // smmla v11.4s, v1.16b, v7.16b\n" - ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" + ".inst 0x4e91a42b // smmla v11.4s, v1.16b, v17.16b\n" + ".inst 0x4e90a42f // smmla v15.4s, v1.16b, v16.16b\n" "ldr q1, [x26, #0x0]\n" "add x9, x9, #0x100\n" "ldr q7, [x9, #0x0]\n" @@ -177,40 +176,40 @@ void a64_hybrid_s8qs_mmla_6x16 ( "prfm pldl1keep, [x26, #0x80]\n" "bge 7b\n" "8:" // Height 1: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x70]\n" - "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x80]\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x90]\n" - ".inst 0x4e87a428 // smmla v8.4s, v1.16b, v7.16b\n" - "ldr q7, [x9, #0xa0]\n" - ".inst 0x4e86a42c // smmla v12.4s, v1.16b, v6.16b\n" - "ldr q6, [x9, #0xb0]\n" - ".inst 0x4e87a429 // smmla v9.4s, v1.16b, v7.16b\n" - "ldr q7, [x9, #0xc0]\n" - ".inst 0x4e86a42d // smmla v13.4s, v1.16b, v6.16b\n" - "ldr q6, [x9, #0xd0]\n" - ".inst 0x4e87a42a // smmla v10.4s, v1.16b, v7.16b\n" - "ldr q7, [x9, #0xe0]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - "ldr q6, [x9, #0xf0]\n" + "trn1 v18.2d, v1.2d, v19.2d\n" + ".inst 0x4e87a648 // smmla v8.4s, v18.16b, v7.16b\n" + "ldr q17, [x9, #0x20]\n" + ".inst 0x4e86a64c // smmla v12.4s, v18.16b, v6.16b\n" + "ldr q16, [x9, #0x30]\n" + ".inst 0x4e91a649 // smmla v9.4s, v18.16b, v17.16b\n" + "ldr q17, [x9, #0x40]\n" + ".inst 0x4e90a64d // smmla v13.4s, v18.16b, v16.16b\n" + "ldr q16, [x9, #0x50]\n" + ".inst 0x4e91a64a // smmla v10.4s, v18.16b, v17.16b\n" + "ldr q17, [x9, #0x60]\n" + ".inst 0x4e90a64e // smmla v14.4s, v18.16b, v16.16b\n" + "ldr q16, [x9, #0x70]\n" + "trn2 v1.2d, v1.2d, v19.2d\n" + ".inst 0x4e91a64b // smmla v11.4s, v18.16b, v17.16b\n" + "ldr q17, [x9, #0x80]\n" + ".inst 0x4e90a64f // smmla v15.4s, v18.16b, v16.16b\n" + "ldr q16, [x9, #0x90]\n" + ".inst 0x4e91a428 // smmla v8.4s, v1.16b, v17.16b\n" + "ldr q17, [x9, #0xa0]\n" + ".inst 0x4e90a42c // smmla v12.4s, v1.16b, v16.16b\n" + "ldr q16, [x9, #0xb0]\n" + ".inst 0x4e91a429 // smmla v9.4s, v1.16b, v17.16b\n" + "ldr q17, [x9, #0xc0]\n" + ".inst 0x4e90a42d // smmla v13.4s, v1.16b, v16.16b\n" + "ldr q16, [x9, #0xd0]\n" + ".inst 0x4e91a42a // smmla v10.4s, v1.16b, v17.16b\n" + "ldr q17, [x9, #0xe0]\n" + ".inst 0x4e90a42e // smmla v14.4s, v1.16b, v16.16b\n" + "ldr q16, [x9, #0xf0]\n" "add x26, x26, #0x10\n" "sub x27, x27, #0x10\n" - ".inst 0x4e87a42b // smmla v11.4s, v1.16b, v7.16b\n" - ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" + ".inst 0x4e91a42b // smmla v11.4s, v1.16b, v17.16b\n" + ".inst 0x4e90a42f // smmla v15.4s, v1.16b, v16.16b\n" "prfm pldl1keep, [x26, #0x80]\n" "add x9, x9, #0x100\n" "9:" // Height 1: Multiply loop: Main loop skip @@ -218,26 +217,26 @@ void a64_hybrid_s8qs_mmla_6x16 ( "cmp x27, #0x8\n" "blt 11f\n" "10:" // Height 1: Multiply loop: Odd block loop - "ldr d1, [x26], #0x8\n" - "ldr q6, [x9, #0x0]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x4e86a408 // smmla v8.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x20]\n" - ".inst 0x4e87a40c // smmla v12.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x30]\n" - ".inst 0x4e86a409 // smmla v9.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x40]\n" - ".inst 0x4e87a40d // smmla v13.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x50]\n" - ".inst 0x4e86a40a // smmla v10.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x60]\n" - ".inst 0x4e87a40e // smmla v14.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x70]\n" + "ldr d18, [x26], #0x8\n" + "ldr q17, [x9, #0x0]\n" + "trn1 v18.2d, v18.2d, v16.2d\n" + "ldr q31, [x9, #0x10]\n" + ".inst 0x4e91a648 // smmla v8.4s, v18.16b, v17.16b\n" + "ldr q17, [x9, #0x20]\n" + ".inst 0x4e9fa64c // smmla v12.4s, v18.16b, v31.16b\n" + "ldr q16, [x9, #0x30]\n" + ".inst 0x4e91a649 // smmla v9.4s, v18.16b, v17.16b\n" + "ldr q17, [x9, #0x40]\n" + ".inst 0x4e90a64d // smmla v13.4s, v18.16b, v16.16b\n" + "ldr q16, [x9, #0x50]\n" + ".inst 0x4e91a64a // smmla v10.4s, v18.16b, v17.16b\n" + "ldr q17, [x9, #0x60]\n" + ".inst 0x4e90a64e // smmla v14.4s, v18.16b, v16.16b\n" + "ldr q16, [x9, #0x70]\n" "sub x27, x27, #0x8\n" "cmp x27, #0x8\n" - ".inst 0x4e86a40b // smmla v11.4s, v0.16b, v6.16b\n" - ".inst 0x4e87a40f // smmla v15.4s, v0.16b, v7.16b\n" + ".inst 0x4e91a64b // smmla v11.4s, v18.16b, v17.16b\n" + ".inst 0x4e90a64f // smmla v15.4s, v18.16b, v16.16b\n" "add x9, x9, #0x80\n" "bge 10b\n" "11:" // Height 1: Multiply loop: Skip odd blocks @@ -262,44 +261,44 @@ void a64_hybrid_s8qs_mmla_6x16 ( "14:" // Height 1: Multiply loop: Ragged operand read: partial_1_0 "ldr b1, [x26, #0x0]\n" "15:" // Height 1: Multiply loop: Ragged operand read: Done - "ldr q7, [x9, #0x0]\n" - "ldr q6, [x9, #0x10]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x70]\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" + "ldr q17, [x9, #0x0]\n" + "ldr q19, [x9, #0x10]\n" + "trn1 v18.2d, v1.2d, v16.2d\n" + ".inst 0x4e91a648 // smmla v8.4s, v18.16b, v17.16b\n" + "ldr q17, [x9, #0x20]\n" + ".inst 0x4e93a64c // smmla v12.4s, v18.16b, v19.16b\n" + "ldr q16, [x9, #0x30]\n" + ".inst 0x4e91a649 // smmla v9.4s, v18.16b, v17.16b\n" + "ldr q17, [x9, #0x40]\n" + ".inst 0x4e90a64d // smmla v13.4s, v18.16b, v16.16b\n" + "ldr q16, [x9, #0x50]\n" + ".inst 0x4e91a64a // smmla v10.4s, v18.16b, v17.16b\n" + "ldr q17, [x9, #0x60]\n" + ".inst 0x4e90a64e // smmla v14.4s, v18.16b, v16.16b\n" + "ldr q16, [x9, #0x70]\n" + ".inst 0x4e91a64b // smmla v11.4s, v18.16b, v17.16b\n" + ".inst 0x4e90a64f // smmla v15.4s, v18.16b, v16.16b\n" "add x9, x9, #0x80\n" "16:" // Height 1: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 4b\n" - "ldr q0, [x14, #0x0]\n" - "ldr q1, [x14, #0x10]\n" + "ldr q19, [x14, #0x0]\n" + "ldr q18, [x14, #0x10]\n" "uzp1 v8.2d, v8.2d, v12.2d\n" "uzp1 v9.2d, v9.2d, v13.2d\n" - "ldr q2, [x14, #0x20]\n" - "ldr q3, [x14, #0x30]\n" + "ldr q17, [x14, #0x20]\n" + "ldr q16, [x14, #0x30]\n" "uzp1 v10.2d, v10.2d, v14.2d\n" "uzp1 v11.2d, v11.2d, v15.2d\n" "mov v15.16b, v8.16b\n" "prfm pstl1keep, [x11, #0x0]\n" - "add v15.4s, v15.4s, v0.4s\n" + "add v15.4s, v15.4s, v19.4s\n" "add x14, x14, #0x40\n" - "add v9.4s, v9.4s, v1.4s\n" - "add v10.4s, v10.4s, v2.4s\n" - "add v11.4s, v11.4s, v3.4s\n" + "add v9.4s, v9.4s, v18.4s\n" + "add v10.4s, v10.4s, v17.4s\n" + "add v11.4s, v11.4s, v16.4s\n" "tbz %x[flags], #4, 17f\n" "ldr q0, [x12, #0x0]\n" "ldr q4, [x13, #0x0]\n" @@ -313,10 +312,10 @@ void a64_hybrid_s8qs_mmla_6x16 ( "add x13, x13, #0x40\n" "b 18f\n" "17:" // Height 1: per layer parameters - "add x25, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x25]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v4.4s }, [x20]\n" "mov v1.16b, v0.16b\n" "mov v5.16b, v4.16b\n" "mov v2.16b, v0.16b\n" @@ -329,45 +328,45 @@ void a64_hybrid_s8qs_mmla_6x16 ( "sqrdmulh v10.4s, v10.4s, v6.4s\n" "sqrdmulh v11.4s, v11.4s, v7.4s\n" "tbz %x[flags], #5, 19f\n" - "and v4.16b, v15.16b, v0.16b\n" - "and v5.16b, v9.16b, v1.16b\n" - "and v6.16b, v10.16b, v2.16b\n" - "and v7.16b, v11.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v15.4s, v15.4s, v4.4s\n" - "sqadd v9.4s, v9.4s, v5.4s\n" - "sqadd v10.4s, v10.4s, v6.4s\n" - "sqadd v11.4s, v11.4s, v7.4s\n" + "and v17.16b, v15.16b, v0.16b\n" + "and v16.16b, v9.16b, v1.16b\n" + "and v25.16b, v10.16b, v2.16b\n" + "and v18.16b, v11.16b, v3.16b\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sqadd v15.4s, v15.4s, v17.4s\n" + "sqadd v9.4s, v9.4s, v16.4s\n" + "sqadd v10.4s, v10.4s, v25.4s\n" + "sqadd v11.4s, v11.4s, v18.4s\n" "19:" // Height 1: no shift correction - "add x25, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v18.4s }, [x20]\n" "srshl v15.4s, v15.4s, v0.4s\n" "srshl v9.4s, v9.4s, v1.4s\n" "srshl v10.4s, v10.4s, v2.4s\n" "srshl v11.4s, v11.4s, v3.4s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x25]\n" - "add v15.4s, v15.4s, v4.4s\n" - "add v9.4s, v9.4s, v4.4s\n" - "add x25, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x25]\n" - "add v10.4s, v10.4s, v4.4s\n" - "add v11.4s, v11.4s, v4.4s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v17.4s }, [x20]\n" + "add v15.4s, v15.4s, v18.4s\n" + "add v9.4s, v9.4s, v18.4s\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v16.4s }, [x20]\n" + "add v10.4s, v10.4s, v18.4s\n" + "add v11.4s, v11.4s, v18.4s\n" "cmp x10, #0x10\n" - "smin v15.4s, v15.4s, v6.4s\n" - "smin v9.4s, v9.4s, v6.4s\n" - "smin v10.4s, v10.4s, v6.4s\n" - "smin v11.4s, v11.4s, v6.4s\n" - "smax v15.4s, v15.4s, v5.4s\n" - "smax v9.4s, v9.4s, v5.4s\n" - "smax v10.4s, v10.4s, v5.4s\n" - "smax v11.4s, v11.4s, v5.4s\n" + "smin v15.4s, v15.4s, v17.4s\n" + "smin v9.4s, v9.4s, v17.4s\n" + "smin v10.4s, v10.4s, v17.4s\n" + "smin v11.4s, v11.4s, v17.4s\n" + "smax v15.4s, v15.4s, v16.4s\n" + "smax v9.4s, v9.4s, v16.4s\n" + "smax v10.4s, v10.4s, v16.4s\n" + "smax v11.4s, v11.4s, v16.4s\n" "uzp1 v15.8h, v15.8h, v9.8h\n" - "uzp1 v9.8h, v10.8h, v11.8h\n" - "uzp1 v15.16b, v15.16b, v9.16b\n" + "uzp1 v16.8h, v10.8h, v11.8h\n" + "uzp1 v15.16b, v15.16b, v16.16b\n" "bge 28f\n" "tbz x10, #3, 23f\n" "str d15, [x11], #0x8\n" @@ -442,12 +441,12 @@ void a64_hybrid_s8qs_mmla_6x16 ( "33:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 34f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 35f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -455,7 +454,7 @@ void a64_hybrid_s8qs_mmla_6x16 ( "b 35f\n" "34:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" + "add x25, x26, x21\n" "35:" // Height 2: input setup done "cmp x27, #0x10\n" "blt 38f\n" @@ -466,85 +465,85 @@ void a64_hybrid_s8qs_mmla_6x16 ( "ldr q6, [x9, #0x10]\n" "blt 37f\n" "36:" // Height 2: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x70]\n" + "trn1 v18.2d, v1.2d, v2.2d\n" + ".inst 0x4e87a648 // smmla v8.4s, v18.16b, v7.16b\n" + "ldr q17, [x9, #0x20]\n" + ".inst 0x4e86a64c // smmla v12.4s, v18.16b, v6.16b\n" + "ldr q16, [x9, #0x30]\n" + ".inst 0x4e91a649 // smmla v9.4s, v18.16b, v17.16b\n" + "ldr q17, [x9, #0x40]\n" + ".inst 0x4e90a64d // smmla v13.4s, v18.16b, v16.16b\n" + "ldr q16, [x9, #0x50]\n" + ".inst 0x4e91a64a // smmla v10.4s, v18.16b, v17.16b\n" + "ldr q17, [x9, #0x60]\n" + ".inst 0x4e90a64e // smmla v14.4s, v18.16b, v16.16b\n" + "ldr q16, [x9, #0x70]\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x80]\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x90]\n" - ".inst 0x4e87a428 // smmla v8.4s, v1.16b, v7.16b\n" - "ldr q7, [x9, #0xa0]\n" - ".inst 0x4e86a42c // smmla v12.4s, v1.16b, v6.16b\n" - "ldr q6, [x9, #0xb0]\n" - ".inst 0x4e87a429 // smmla v9.4s, v1.16b, v7.16b\n" - "ldr q7, [x9, #0xc0]\n" - ".inst 0x4e86a42d // smmla v13.4s, v1.16b, v6.16b\n" - "ldr q6, [x9, #0xd0]\n" - ".inst 0x4e87a42a // smmla v10.4s, v1.16b, v7.16b\n" - "ldr q7, [x9, #0xe0]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - "ldr q6, [x9, #0xf0]\n" + ".inst 0x4e91a64b // smmla v11.4s, v18.16b, v17.16b\n" + "ldr q17, [x9, #0x80]\n" + ".inst 0x4e90a64f // smmla v15.4s, v18.16b, v16.16b\n" + "ldr q16, [x9, #0x90]\n" + ".inst 0x4e91a428 // smmla v8.4s, v1.16b, v17.16b\n" + "ldr q17, [x9, #0xa0]\n" + ".inst 0x4e90a42c // smmla v12.4s, v1.16b, v16.16b\n" + "ldr q16, [x9, #0xb0]\n" + ".inst 0x4e91a429 // smmla v9.4s, v1.16b, v17.16b\n" + "ldr q17, [x9, #0xc0]\n" + ".inst 0x4e90a42d // smmla v13.4s, v1.16b, v16.16b\n" + "ldr q16, [x9, #0xd0]\n" + ".inst 0x4e91a42a // smmla v10.4s, v1.16b, v17.16b\n" + "ldr q17, [x9, #0xe0]\n" + ".inst 0x4e90a42e // smmla v14.4s, v1.16b, v16.16b\n" + "ldr q16, [x9, #0xf0]\n" "sub x27, x27, #0x10\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" "ldr q2, [x25, #0x0]\n" "cmp x27, #0x20\n" - ".inst 0x4e87a42b // smmla v11.4s, v1.16b, v7.16b\n" + ".inst 0x4e91a42b // smmla v11.4s, v1.16b, v17.16b\n" "add x9, x9, #0x100\n" "ldr q7, [x9, #0x0]\n" - ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" + ".inst 0x4e90a42f // smmla v15.4s, v1.16b, v16.16b\n" "ldr q1, [x26, #0x0]\n" "ldr q6, [x9, #0x10]\n" "prfm pldl1keep, [x26, #0x80]\n" "prfm pldl1keep, [x25, #0x80]\n" "bge 36b\n" "37:" // Height 2: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x70]\n" + "trn1 v18.2d, v1.2d, v2.2d\n" + ".inst 0x4e87a648 // smmla v8.4s, v18.16b, v7.16b\n" + "ldr q17, [x9, #0x20]\n" + ".inst 0x4e86a64c // smmla v12.4s, v18.16b, v6.16b\n" + "ldr q16, [x9, #0x30]\n" + ".inst 0x4e91a649 // smmla v9.4s, v18.16b, v17.16b\n" + "ldr q17, [x9, #0x40]\n" + ".inst 0x4e90a64d // smmla v13.4s, v18.16b, v16.16b\n" + "ldr q16, [x9, #0x50]\n" + ".inst 0x4e91a64a // smmla v10.4s, v18.16b, v17.16b\n" + "ldr q17, [x9, #0x60]\n" + ".inst 0x4e90a64e // smmla v14.4s, v18.16b, v16.16b\n" + "ldr q16, [x9, #0x70]\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x80]\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x90]\n" - ".inst 0x4e87a428 // smmla v8.4s, v1.16b, v7.16b\n" - "ldr q7, [x9, #0xa0]\n" - ".inst 0x4e86a42c // smmla v12.4s, v1.16b, v6.16b\n" - "ldr q6, [x9, #0xb0]\n" - ".inst 0x4e87a429 // smmla v9.4s, v1.16b, v7.16b\n" - "ldr q7, [x9, #0xc0]\n" - ".inst 0x4e86a42d // smmla v13.4s, v1.16b, v6.16b\n" - "ldr q6, [x9, #0xd0]\n" - ".inst 0x4e87a42a // smmla v10.4s, v1.16b, v7.16b\n" - "ldr q7, [x9, #0xe0]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - "ldr q6, [x9, #0xf0]\n" + ".inst 0x4e91a64b // smmla v11.4s, v18.16b, v17.16b\n" + "ldr q17, [x9, #0x80]\n" + ".inst 0x4e90a64f // smmla v15.4s, v18.16b, v16.16b\n" + "ldr q16, [x9, #0x90]\n" + ".inst 0x4e91a428 // smmla v8.4s, v1.16b, v17.16b\n" + "ldr q17, [x9, #0xa0]\n" + ".inst 0x4e90a42c // smmla v12.4s, v1.16b, v16.16b\n" + "ldr q16, [x9, #0xb0]\n" + ".inst 0x4e91a429 // smmla v9.4s, v1.16b, v17.16b\n" + "ldr q17, [x9, #0xc0]\n" + ".inst 0x4e90a42d // smmla v13.4s, v1.16b, v16.16b\n" + "ldr q16, [x9, #0xd0]\n" + ".inst 0x4e91a42a // smmla v10.4s, v1.16b, v17.16b\n" + "ldr q17, [x9, #0xe0]\n" + ".inst 0x4e90a42e // smmla v14.4s, v1.16b, v16.16b\n" + "ldr q16, [x9, #0xf0]\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" - ".inst 0x4e87a42b // smmla v11.4s, v1.16b, v7.16b\n" - ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" + ".inst 0x4e91a42b // smmla v11.4s, v1.16b, v17.16b\n" + ".inst 0x4e90a42f // smmla v15.4s, v1.16b, v16.16b\n" "sub x27, x27, #0x10\n" "prfm pldl1keep, [x26, #0x80]\n" "prfm pldl1keep, [x25, #0x80]\n" @@ -554,27 +553,27 @@ void a64_hybrid_s8qs_mmla_6x16 ( "cmp x27, #0x8\n" "blt 40f\n" "39:" // Height 2: Multiply loop: Odd block loop - "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" + "ldr d17, [x26], #0x8\n" + "ldr d16, [x25], #0x8\n" + "trn1 v18.2d, v17.2d, v16.2d\n" "sub x27, x27, #0x8\n" - "ldr q6, [x9, #0x0]\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x4e86a408 // smmla v8.4s, v0.16b, v6.16b\n" - ".inst 0x4e87a40c // smmla v12.4s, v0.16b, v7.16b\n" - "ldr q6, [x9, #0x20]\n" - "ldr q7, [x9, #0x30]\n" - ".inst 0x4e86a409 // smmla v9.4s, v0.16b, v6.16b\n" - ".inst 0x4e87a40d // smmla v13.4s, v0.16b, v7.16b\n" - "ldr q6, [x9, #0x40]\n" - "ldr q7, [x9, #0x50]\n" - ".inst 0x4e86a40a // smmla v10.4s, v0.16b, v6.16b\n" - ".inst 0x4e87a40e // smmla v14.4s, v0.16b, v7.16b\n" - "ldr q6, [x9, #0x60]\n" - "ldr q7, [x9, #0x70]\n" + "ldr q17, [x9, #0x0]\n" + "ldr q16, [x9, #0x10]\n" + ".inst 0x4e91a648 // smmla v8.4s, v18.16b, v17.16b\n" + ".inst 0x4e90a64c // smmla v12.4s, v18.16b, v16.16b\n" + "ldr q17, [x9, #0x20]\n" + "ldr q16, [x9, #0x30]\n" + ".inst 0x4e91a649 // smmla v9.4s, v18.16b, v17.16b\n" + ".inst 0x4e90a64d // smmla v13.4s, v18.16b, v16.16b\n" + "ldr q17, [x9, #0x40]\n" + "ldr q16, [x9, #0x50]\n" + ".inst 0x4e91a64a // smmla v10.4s, v18.16b, v17.16b\n" + ".inst 0x4e90a64e // smmla v14.4s, v18.16b, v16.16b\n" + "ldr q17, [x9, #0x60]\n" + "ldr q16, [x9, #0x70]\n" "cmp x27, #0x8\n" - ".inst 0x4e86a40b // smmla v11.4s, v0.16b, v6.16b\n" - ".inst 0x4e87a40f // smmla v15.4s, v0.16b, v7.16b\n" + ".inst 0x4e91a64b // smmla v11.4s, v18.16b, v17.16b\n" + ".inst 0x4e90a64f // smmla v15.4s, v18.16b, v16.16b\n" "add x9, x9, #0x80\n" "bge 39b\n" "40:" // Height 2: Multiply loop: Skip odd blocks @@ -606,55 +605,55 @@ void a64_hybrid_s8qs_mmla_6x16 ( "ldr b1, [x26, #0x0]\n" "ldr b2, [x25, #0x0]\n" "44:" // Height 2: Multiply loop: Ragged operand read: Done - "ldr q7, [x9, #0x0]\n" - "ldr q6, [x9, #0x10]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" - "ldr q7, [x9, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" - "ldr q6, [x9, #0x70]\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" + "ldr q17, [x9, #0x0]\n" + "ldr q16, [x9, #0x10]\n" + "trn1 v18.2d, v1.2d, v2.2d\n" + ".inst 0x4e91a648 // smmla v8.4s, v18.16b, v17.16b\n" + "ldr q17, [x9, #0x20]\n" + ".inst 0x4e90a64c // smmla v12.4s, v18.16b, v16.16b\n" + "ldr q16, [x9, #0x30]\n" + ".inst 0x4e91a649 // smmla v9.4s, v18.16b, v17.16b\n" + "ldr q17, [x9, #0x40]\n" + ".inst 0x4e90a64d // smmla v13.4s, v18.16b, v16.16b\n" + "ldr q16, [x9, #0x50]\n" + ".inst 0x4e91a64a // smmla v10.4s, v18.16b, v17.16b\n" + "ldr q17, [x9, #0x60]\n" + ".inst 0x4e90a64e // smmla v14.4s, v18.16b, v16.16b\n" + "ldr q16, [x9, #0x70]\n" + ".inst 0x4e91a64b // smmla v11.4s, v18.16b, v17.16b\n" + ".inst 0x4e90a64f // smmla v15.4s, v18.16b, v16.16b\n" "add x9, x9, #0x80\n" "45:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 33b\n" - "ldr q0, [x14, #0x0]\n" - "ldr q1, [x14, #0x10]\n" - "uzp1 v7.2d, v8.2d, v12.2d\n" + "ldr q19, [x14, #0x0]\n" + "ldr q18, [x14, #0x10]\n" + "uzp1 v17.2d, v8.2d, v12.2d\n" "uzp2 v8.2d, v8.2d, v12.2d\n" - "ldr q2, [x14, #0x20]\n" - "ldr q3, [x14, #0x30]\n" + "ldr q5, [x14, #0x20]\n" + "ldr q16, [x14, #0x30]\n" "uzp1 v12.2d, v9.2d, v13.2d\n" "uzp2 v9.2d, v9.2d, v13.2d\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "uzp1 v13.2d, v10.2d, v14.2d\n" "uzp2 v10.2d, v10.2d, v14.2d\n" - "add x24, x11, x20\n" + "add x25, x11, x20\n" "uzp1 v14.2d, v11.2d, v15.2d\n" "uzp2 v11.2d, v11.2d, v15.2d\n" "prfm pstl1keep, [x11, #0x0]\n" - "prfm pstl1keep, [x24, #0x0]\n" - "mov v15.16b, v7.16b\n" - "add v15.4s, v15.4s, v0.4s\n" + "prfm pstl1keep, [x25, #0x0]\n" + "mov v15.16b, v17.16b\n" + "add v15.4s, v15.4s, v19.4s\n" "add x14, x14, #0x40\n" - "add v12.4s, v12.4s, v1.4s\n" - "add v13.4s, v13.4s, v2.4s\n" - "add v14.4s, v14.4s, v3.4s\n" - "add v8.4s, v8.4s, v0.4s\n" - "add v9.4s, v9.4s, v1.4s\n" - "add v10.4s, v10.4s, v2.4s\n" - "add v11.4s, v11.4s, v3.4s\n" + "add v12.4s, v12.4s, v18.4s\n" + "add v13.4s, v13.4s, v5.4s\n" + "add v14.4s, v14.4s, v16.4s\n" + "add v8.4s, v8.4s, v19.4s\n" + "add v9.4s, v9.4s, v18.4s\n" + "add v10.4s, v10.4s, v5.4s\n" + "add v11.4s, v11.4s, v16.4s\n" "tbz %x[flags], #4, 46f\n" "ldr q0, [x12, #0x0]\n" "ldr q4, [x13, #0x0]\n" @@ -668,10 +667,10 @@ void a64_hybrid_s8qs_mmla_6x16 ( "add x13, x13, #0x40\n" "b 47f\n" "46:" // Height 2: per layer parameters - "add x25, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x25]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v4.4s }, [x20]\n" "mov v1.16b, v0.16b\n" "mov v5.16b, v4.16b\n" "mov v2.16b, v0.16b\n" @@ -688,141 +687,141 @@ void a64_hybrid_s8qs_mmla_6x16 ( "sqrdmulh v10.4s, v10.4s, v6.4s\n" "sqrdmulh v11.4s, v11.4s, v7.4s\n" "tbz %x[flags], #5, 48f\n" - "and v4.16b, v15.16b, v0.16b\n" - "and v5.16b, v12.16b, v1.16b\n" - "and v6.16b, v13.16b, v2.16b\n" - "and v7.16b, v14.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v15.4s, v15.4s, v4.4s\n" - "sqadd v12.4s, v12.4s, v5.4s\n" - "sqadd v13.4s, v13.4s, v6.4s\n" - "sqadd v14.4s, v14.4s, v7.4s\n" - "and v4.16b, v8.16b, v0.16b\n" - "and v5.16b, v9.16b, v1.16b\n" - "and v6.16b, v10.16b, v2.16b\n" - "and v7.16b, v11.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v4.4s\n" - "sqadd v9.4s, v9.4s, v5.4s\n" - "sqadd v10.4s, v10.4s, v6.4s\n" - "sqadd v11.4s, v11.4s, v7.4s\n" + "and v19.16b, v15.16b, v0.16b\n" + "and v18.16b, v12.16b, v1.16b\n" + "and v17.16b, v13.16b, v2.16b\n" + "and v16.16b, v14.16b, v3.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v15.4s, v15.4s, v19.4s\n" + "sqadd v12.4s, v12.4s, v18.4s\n" + "sqadd v13.4s, v13.4s, v17.4s\n" + "sqadd v14.4s, v14.4s, v16.4s\n" + "and v19.16b, v8.16b, v0.16b\n" + "and v18.16b, v9.16b, v1.16b\n" + "and v17.16b, v10.16b, v2.16b\n" + "and v16.16b, v11.16b, v3.16b\n" + "sshr v19.4s, v19.4s, #0x1f\n" + "sshr v18.4s, v18.4s, #0x1f\n" + "sshr v17.4s, v17.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v8.4s, v8.4s, v19.4s\n" + "sqadd v9.4s, v9.4s, v18.4s\n" + "sqadd v10.4s, v10.4s, v17.4s\n" + "sqadd v11.4s, v11.4s, v16.4s\n" "48:" // Height 2: no shift correction - "add x25, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v18.4s }, [x20]\n" "srshl v15.4s, v15.4s, v0.4s\n" "srshl v12.4s, v12.4s, v1.4s\n" "srshl v13.4s, v13.4s, v2.4s\n" "srshl v14.4s, v14.4s, v3.4s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x25]\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v17.4s }, [x20]\n" "srshl v8.4s, v8.4s, v0.4s\n" "srshl v9.4s, v9.4s, v1.4s\n" - "add x25, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x25]\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v16.4s }, [x20]\n" "srshl v10.4s, v10.4s, v2.4s\n" "srshl v11.4s, v11.4s, v3.4s\n" "cmp x10, #0x10\n" - "add v15.4s, v15.4s, v4.4s\n" - "add v12.4s, v12.4s, v4.4s\n" - "add v13.4s, v13.4s, v4.4s\n" - "add v14.4s, v14.4s, v4.4s\n" - "add v8.4s, v8.4s, v4.4s\n" - "add v9.4s, v9.4s, v4.4s\n" - "add v10.4s, v10.4s, v4.4s\n" - "add v11.4s, v11.4s, v4.4s\n" - "smin v15.4s, v15.4s, v6.4s\n" - "smin v12.4s, v12.4s, v6.4s\n" - "smin v13.4s, v13.4s, v6.4s\n" - "smin v14.4s, v14.4s, v6.4s\n" - "smin v8.4s, v8.4s, v6.4s\n" - "smin v9.4s, v9.4s, v6.4s\n" - "smin v10.4s, v10.4s, v6.4s\n" - "smin v11.4s, v11.4s, v6.4s\n" - "smax v15.4s, v15.4s, v5.4s\n" - "smax v12.4s, v12.4s, v5.4s\n" - "smax v13.4s, v13.4s, v5.4s\n" - "smax v14.4s, v14.4s, v5.4s\n" - "smax v8.4s, v8.4s, v5.4s\n" - "smax v9.4s, v9.4s, v5.4s\n" - "smax v10.4s, v10.4s, v5.4s\n" - "smax v11.4s, v11.4s, v5.4s\n" + "add v15.4s, v15.4s, v18.4s\n" + "add v12.4s, v12.4s, v18.4s\n" + "add v13.4s, v13.4s, v18.4s\n" + "add v14.4s, v14.4s, v18.4s\n" + "add v8.4s, v8.4s, v18.4s\n" + "add v9.4s, v9.4s, v18.4s\n" + "add v10.4s, v10.4s, v18.4s\n" + "add v11.4s, v11.4s, v18.4s\n" + "smin v15.4s, v15.4s, v17.4s\n" + "smin v12.4s, v12.4s, v17.4s\n" + "smin v13.4s, v13.4s, v17.4s\n" + "smin v14.4s, v14.4s, v17.4s\n" + "smin v8.4s, v8.4s, v17.4s\n" + "smin v9.4s, v9.4s, v17.4s\n" + "smin v10.4s, v10.4s, v17.4s\n" + "smin v11.4s, v11.4s, v17.4s\n" + "smax v15.4s, v15.4s, v16.4s\n" + "smax v12.4s, v12.4s, v16.4s\n" + "smax v13.4s, v13.4s, v16.4s\n" + "smax v14.4s, v14.4s, v16.4s\n" + "smax v8.4s, v8.4s, v16.4s\n" + "smax v9.4s, v9.4s, v16.4s\n" + "smax v10.4s, v10.4s, v16.4s\n" + "smax v11.4s, v11.4s, v16.4s\n" "uzp1 v15.8h, v15.8h, v12.8h\n" - "uzp1 v12.8h, v13.8h, v14.8h\n" + "uzp1 v17.8h, v13.8h, v14.8h\n" "uzp1 v8.8h, v8.8h, v9.8h\n" - "uzp1 v9.8h, v10.8h, v11.8h\n" - "uzp1 v15.16b, v15.16b, v12.16b\n" - "uzp1 v8.16b, v8.16b, v9.16b\n" + "uzp1 v16.8h, v10.8h, v11.8h\n" + "uzp1 v15.16b, v15.16b, v17.16b\n" + "uzp1 v8.16b, v8.16b, v16.16b\n" "bge 57f\n" "tbz x10, #3, 52f\n" "str d15, [x11], #0x8\n" - "str d8, [x24], #0x8\n" + "str d8, [x25], #0x8\n" "tbz x10, #2, 50f\n" "st1 { v15.s }[2], [x11], #0x4\n" - "st1 { v8.s }[2], [x24], #0x4\n" + "st1 { v8.s }[2], [x25], #0x4\n" "tbz x10, #1, 49f\n" "st1 { v15.h }[6], [x11], #0x2\n" - "st1 { v8.h }[6], [x24], #0x2\n" + "st1 { v8.h }[6], [x25], #0x2\n" "tbz x10, #0, 56f\n" "st1 { v15.b }[14], [x11]\n" - "st1 { v8.b }[14], [x24]\n" + "st1 { v8.b }[14], [x25]\n" "b 56f\n" "49:" // Height 2: Partial direct writeback: partial_1_12 "tbz x10, #0, 56f\n" "st1 { v15.b }[12], [x11]\n" - "st1 { v8.b }[12], [x24]\n" + "st1 { v8.b }[12], [x25]\n" "b 56f\n" "50:" // Height 2: Partial direct writeback: partial_2_8 "tbz x10, #1, 51f\n" "st1 { v15.h }[4], [x11], #0x2\n" - "st1 { v8.h }[4], [x24], #0x2\n" + "st1 { v8.h }[4], [x25], #0x2\n" "tbz x10, #0, 56f\n" "st1 { v15.b }[10], [x11]\n" - "st1 { v8.b }[10], [x24]\n" + "st1 { v8.b }[10], [x25]\n" "b 56f\n" "51:" // Height 2: Partial direct writeback: partial_1_8 "tbz x10, #0, 56f\n" "st1 { v15.b }[8], [x11]\n" - "st1 { v8.b }[8], [x24]\n" + "st1 { v8.b }[8], [x25]\n" "b 56f\n" "52:" // Height 2: Partial direct writeback: partial_4_0 "tbz x10, #2, 54f\n" "str s15, [x11], #0x4\n" - "str s8, [x24], #0x4\n" + "str s8, [x25], #0x4\n" "tbz x10, #1, 53f\n" "st1 { v15.h }[2], [x11], #0x2\n" - "st1 { v8.h }[2], [x24], #0x2\n" + "st1 { v8.h }[2], [x25], #0x2\n" "tbz x10, #0, 56f\n" "st1 { v15.b }[6], [x11]\n" - "st1 { v8.b }[6], [x24]\n" + "st1 { v8.b }[6], [x25]\n" "b 56f\n" "53:" // Height 2: Partial direct writeback: partial_1_4 "tbz x10, #0, 56f\n" "st1 { v15.b }[4], [x11]\n" - "st1 { v8.b }[4], [x24]\n" + "st1 { v8.b }[4], [x25]\n" "b 56f\n" "54:" // Height 2: Partial direct writeback: partial_2_0 "tbz x10, #1, 55f\n" "str h15, [x11], #0x2\n" - "str h8, [x24], #0x2\n" + "str h8, [x25], #0x2\n" "tbz x10, #0, 56f\n" "st1 { v15.b }[2], [x11]\n" - "st1 { v8.b }[2], [x24]\n" + "st1 { v8.b }[2], [x25]\n" "b 56f\n" "55:" // Height 2: Partial direct writeback: partial_1_0 "str b15, [x11, #0x0]\n" - "str b8, [x24, #0x0]\n" + "str b8, [x25, #0x0]\n" "56:" // Height 2: Partial direct writeback: Done "b 58f\n" "57:" // Height 2: Full writeback "str q15, [x11, #0x0]\n" "add x11, x11, #0x10\n" - "str q8, [x24, #0x0]\n" + "str q8, [x25, #0x0]\n" "58:" // Height 2: Writeback done "subs x10, x10, #0x10\n" "bgt 31b\n" @@ -856,13 +855,13 @@ void a64_hybrid_s8qs_mmla_6x16 ( "62:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 63f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 64f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -871,8 +870,8 @@ void a64_hybrid_s8qs_mmla_6x16 ( "b 64f\n" "63:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" "64:" // Height 3: input setup done "cmp x27, #0x10\n" "blt 67f\n" @@ -884,167 +883,167 @@ void a64_hybrid_s8qs_mmla_6x16 ( "ldr q6, [x9, #0x10]\n" "blt 66f\n" "65:" // Height 3: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v27.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x4e87a450 // smmla v16.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" - "trn2 v3.2d, v3.2d, v4.2d\n" - ".inst 0x4e87a451 // smmla v17.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" + ".inst 0x4e87a768 // smmla v8.4s, v27.16b, v7.16b\n" + "trn1 v26.2d, v3.2d, v28.2d\n" + ".inst 0x4e87a750 // smmla v16.4s, v26.16b, v7.16b\n" + "ldr q25, [x9, #0x20]\n" + ".inst 0x4e86a76c // smmla v12.4s, v27.16b, v6.16b\n" + ".inst 0x4e86a754 // smmla v20.4s, v26.16b, v6.16b\n" + "ldr q24, [x9, #0x30]\n" + ".inst 0x4e99a769 // smmla v9.4s, v27.16b, v25.16b\n" + "trn2 v3.2d, v3.2d, v28.2d\n" + ".inst 0x4e99a751 // smmla v17.4s, v26.16b, v25.16b\n" + "ldr q25, [x9, #0x40]\n" + ".inst 0x4e98a76d // smmla v13.4s, v27.16b, v24.16b\n" "sub x27, x27, #0x10\n" - ".inst 0x4e86a455 // smmla v21.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" + ".inst 0x4e98a755 // smmla v21.4s, v26.16b, v24.16b\n" + "ldr q24, [x9, #0x50]\n" + ".inst 0x4e99a76a // smmla v10.4s, v27.16b, v25.16b\n" "add x26, x26, #0x10\n" - ".inst 0x4e87a452 // smmla v18.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" + ".inst 0x4e99a752 // smmla v18.4s, v26.16b, v25.16b\n" + "ldr q25, [x9, #0x60]\n" + ".inst 0x4e98a76e // smmla v14.4s, v27.16b, v24.16b\n" "add x25, x25, #0x10\n" - ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x70]\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" + ".inst 0x4e98a756 // smmla v22.4s, v26.16b, v24.16b\n" + "ldr q24, [x9, #0x70]\n" + ".inst 0x4e99a76b // smmla v11.4s, v27.16b, v25.16b\n" "add x24, x24, #0x10\n" - ".inst 0x4e87a453 // smmla v19.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x80]\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" + ".inst 0x4e99a753 // smmla v19.4s, v26.16b, v25.16b\n" + "ldr q25, [x9, #0x80]\n" + ".inst 0x4e98a76f // smmla v15.4s, v27.16b, v24.16b\n" "cmp x27, #0x20\n" - ".inst 0x4e86a457 // smmla v23.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x90]\n" + ".inst 0x4e98a757 // smmla v23.4s, v26.16b, v24.16b\n" + "ldr q24, [x9, #0x90]\n" "ldr q2, [x25, #0x0]\n" - ".inst 0x4e87a428 // smmla v8.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a470 // smmla v16.4s, v3.16b, v7.16b\n" - "ldr q7, [x9, #0xa0]\n" - ".inst 0x4e86a42c // smmla v12.4s, v1.16b, v6.16b\n" + ".inst 0x4e99a428 // smmla v8.4s, v1.16b, v25.16b\n" + ".inst 0x4e99a470 // smmla v16.4s, v3.16b, v25.16b\n" + "ldr q25, [x9, #0xa0]\n" + ".inst 0x4e98a42c // smmla v12.4s, v1.16b, v24.16b\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x4e86a474 // smmla v20.4s, v3.16b, v6.16b\n" - "ldr q6, [x9, #0xb0]\n" - ".inst 0x4e87a429 // smmla v9.4s, v1.16b, v7.16b\n" + ".inst 0x4e98a474 // smmla v20.4s, v3.16b, v24.16b\n" + "ldr q24, [x9, #0xb0]\n" + ".inst 0x4e99a429 // smmla v9.4s, v1.16b, v25.16b\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4e87a471 // smmla v17.4s, v3.16b, v7.16b\n" - "ldr q7, [x9, #0xc0]\n" - ".inst 0x4e86a42d // smmla v13.4s, v1.16b, v6.16b\n" + ".inst 0x4e99a471 // smmla v17.4s, v3.16b, v25.16b\n" + "ldr q25, [x9, #0xc0]\n" + ".inst 0x4e98a42d // smmla v13.4s, v1.16b, v24.16b\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x4e86a475 // smmla v21.4s, v3.16b, v6.16b\n" - "ldr q6, [x9, #0xd0]\n" - ".inst 0x4e87a42a // smmla v10.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a472 // smmla v18.4s, v3.16b, v7.16b\n" - "ldr q7, [x9, #0xe0]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a476 // smmla v22.4s, v3.16b, v6.16b\n" - "ldr q6, [x9, #0xf0]\n" + ".inst 0x4e98a475 // smmla v21.4s, v3.16b, v24.16b\n" + "ldr q24, [x9, #0xd0]\n" + ".inst 0x4e99a42a // smmla v10.4s, v1.16b, v25.16b\n" + ".inst 0x4e99a472 // smmla v18.4s, v3.16b, v25.16b\n" + "ldr q25, [x9, #0xe0]\n" + ".inst 0x4e98a42e // smmla v14.4s, v1.16b, v24.16b\n" + ".inst 0x4e98a476 // smmla v22.4s, v3.16b, v24.16b\n" + "ldr q24, [x9, #0xf0]\n" "add x9, x9, #0x100\n" - ".inst 0x4e87a42b // smmla v11.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a473 // smmla v19.4s, v3.16b, v7.16b\n" + ".inst 0x4e99a42b // smmla v11.4s, v1.16b, v25.16b\n" + ".inst 0x4e99a473 // smmla v19.4s, v3.16b, v25.16b\n" "ldr q7, [x9, #0x0]\n" - ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" + ".inst 0x4e98a42f // smmla v15.4s, v1.16b, v24.16b\n" "ldr q1, [x26, #0x0]\n" - ".inst 0x4e86a477 // smmla v23.4s, v3.16b, v6.16b\n" + ".inst 0x4e98a477 // smmla v23.4s, v3.16b, v24.16b\n" "ldr q3, [x24, #0x0]\n" "ldr q6, [x9, #0x10]\n" "bge 65b\n" "66:" // Height 3: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v27.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x4e87a450 // smmla v16.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" - "trn2 v3.2d, v3.2d, v4.2d\n" - ".inst 0x4e87a451 // smmla v17.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" + ".inst 0x4e87a768 // smmla v8.4s, v27.16b, v7.16b\n" + "trn1 v26.2d, v3.2d, v25.2d\n" + ".inst 0x4e87a750 // smmla v16.4s, v26.16b, v7.16b\n" + "ldr q24, [x9, #0x20]\n" + ".inst 0x4e86a76c // smmla v12.4s, v27.16b, v6.16b\n" + ".inst 0x4e86a754 // smmla v20.4s, v26.16b, v6.16b\n" + "ldr q0, [x9, #0x30]\n" + ".inst 0x4e98a769 // smmla v9.4s, v27.16b, v24.16b\n" + "trn2 v3.2d, v3.2d, v25.2d\n" + ".inst 0x4e98a751 // smmla v17.4s, v26.16b, v24.16b\n" + "ldr q25, [x9, #0x40]\n" + ".inst 0x4e80a76d // smmla v13.4s, v27.16b, v0.16b\n" "add x26, x26, #0x10\n" - ".inst 0x4e86a455 // smmla v21.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" + ".inst 0x4e80a755 // smmla v21.4s, v26.16b, v0.16b\n" + "ldr q24, [x9, #0x50]\n" + ".inst 0x4e99a76a // smmla v10.4s, v27.16b, v25.16b\n" "add x25, x25, #0x10\n" - ".inst 0x4e87a452 // smmla v18.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" + ".inst 0x4e99a752 // smmla v18.4s, v26.16b, v25.16b\n" + "ldr q25, [x9, #0x60]\n" + ".inst 0x4e98a76e // smmla v14.4s, v27.16b, v24.16b\n" "add x24, x24, #0x10\n" - ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x70]\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" + ".inst 0x4e98a756 // smmla v22.4s, v26.16b, v24.16b\n" + "ldr q24, [x9, #0x70]\n" + ".inst 0x4e99a76b // smmla v11.4s, v27.16b, v25.16b\n" "sub x27, x27, #0x10\n" - ".inst 0x4e87a453 // smmla v19.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x80]\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" + ".inst 0x4e99a753 // smmla v19.4s, v26.16b, v25.16b\n" + "ldr q25, [x9, #0x80]\n" + ".inst 0x4e98a76f // smmla v15.4s, v27.16b, v24.16b\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x4e86a457 // smmla v23.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x90]\n" - ".inst 0x4e87a428 // smmla v8.4s, v1.16b, v7.16b\n" + ".inst 0x4e98a757 // smmla v23.4s, v26.16b, v24.16b\n" + "ldr q24, [x9, #0x90]\n" + ".inst 0x4e99a428 // smmla v8.4s, v1.16b, v25.16b\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4e87a470 // smmla v16.4s, v3.16b, v7.16b\n" - "ldr q7, [x9, #0xa0]\n" - ".inst 0x4e86a42c // smmla v12.4s, v1.16b, v6.16b\n" + ".inst 0x4e99a470 // smmla v16.4s, v3.16b, v25.16b\n" + "ldr q25, [x9, #0xa0]\n" + ".inst 0x4e98a42c // smmla v12.4s, v1.16b, v24.16b\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x4e86a474 // smmla v20.4s, v3.16b, v6.16b\n" - "ldr q6, [x9, #0xb0]\n" - ".inst 0x4e87a429 // smmla v9.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a471 // smmla v17.4s, v3.16b, v7.16b\n" - "ldr q7, [x9, #0xc0]\n" - ".inst 0x4e86a42d // smmla v13.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a475 // smmla v21.4s, v3.16b, v6.16b\n" - "ldr q6, [x9, #0xd0]\n" - ".inst 0x4e87a42a // smmla v10.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a472 // smmla v18.4s, v3.16b, v7.16b\n" - "ldr q7, [x9, #0xe0]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a476 // smmla v22.4s, v3.16b, v6.16b\n" - "ldr q6, [x9, #0xf0]\n" + ".inst 0x4e98a474 // smmla v20.4s, v3.16b, v24.16b\n" + "ldr q24, [x9, #0xb0]\n" + ".inst 0x4e99a429 // smmla v9.4s, v1.16b, v25.16b\n" + ".inst 0x4e99a471 // smmla v17.4s, v3.16b, v25.16b\n" + "ldr q25, [x9, #0xc0]\n" + ".inst 0x4e98a42d // smmla v13.4s, v1.16b, v24.16b\n" + ".inst 0x4e98a475 // smmla v21.4s, v3.16b, v24.16b\n" + "ldr q24, [x9, #0xd0]\n" + ".inst 0x4e99a42a // smmla v10.4s, v1.16b, v25.16b\n" + ".inst 0x4e99a472 // smmla v18.4s, v3.16b, v25.16b\n" + "ldr q25, [x9, #0xe0]\n" + ".inst 0x4e98a42e // smmla v14.4s, v1.16b, v24.16b\n" + ".inst 0x4e98a476 // smmla v22.4s, v3.16b, v24.16b\n" + "ldr q24, [x9, #0xf0]\n" "add x9, x9, #0x100\n" - ".inst 0x4e87a42b // smmla v11.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a473 // smmla v19.4s, v3.16b, v7.16b\n" - ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a477 // smmla v23.4s, v3.16b, v6.16b\n" + ".inst 0x4e99a42b // smmla v11.4s, v1.16b, v25.16b\n" + ".inst 0x4e99a473 // smmla v19.4s, v3.16b, v25.16b\n" + ".inst 0x4e98a42f // smmla v15.4s, v1.16b, v24.16b\n" + ".inst 0x4e98a477 // smmla v23.4s, v3.16b, v24.16b\n" "67:" // Height 3: Multiply loop: Main loop skip "cbz x27, 74f\n" "cmp x27, #0x8\n" "blt 69f\n" "68:" // Height 3: Multiply loop: Odd block loop - "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "ldr d3, [x24], #0x8\n" - "ldr q6, [x9, #0x0]\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x4e86a408 // smmla v8.4s, v0.16b, v6.16b\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x4e86a450 // smmla v16.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x20]\n" - ".inst 0x4e87a40c // smmla v12.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a454 // smmla v20.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x30]\n" - ".inst 0x4e86a409 // smmla v9.4s, v0.16b, v6.16b\n" + "ldr d25, [x26], #0x8\n" + "ldr d24, [x25], #0x8\n" + "trn1 v27.2d, v25.2d, v24.2d\n" + "ldr d24, [x24], #0x8\n" + "ldr q25, [x9, #0x0]\n" + "trn1 v26.2d, v24.2d, v26.2d\n" + ".inst 0x4e99a768 // smmla v8.4s, v27.16b, v25.16b\n" + "ldr q24, [x9, #0x10]\n" + ".inst 0x4e99a750 // smmla v16.4s, v26.16b, v25.16b\n" + "ldr q25, [x9, #0x20]\n" + ".inst 0x4e98a76c // smmla v12.4s, v27.16b, v24.16b\n" + ".inst 0x4e98a754 // smmla v20.4s, v26.16b, v24.16b\n" + "ldr q24, [x9, #0x30]\n" + ".inst 0x4e99a769 // smmla v9.4s, v27.16b, v25.16b\n" "sub x27, x27, #0x8\n" - ".inst 0x4e86a451 // smmla v17.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x40]\n" - ".inst 0x4e87a40d // smmla v13.4s, v0.16b, v7.16b\n" + ".inst 0x4e99a751 // smmla v17.4s, v26.16b, v25.16b\n" + "ldr q25, [x9, #0x40]\n" + ".inst 0x4e98a76d // smmla v13.4s, v27.16b, v24.16b\n" "cmp x27, #0x8\n" - ".inst 0x4e87a455 // smmla v21.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x50]\n" - ".inst 0x4e86a40a // smmla v10.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a452 // smmla v18.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x60]\n" - ".inst 0x4e87a40e // smmla v14.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a456 // smmla v22.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x70]\n" - ".inst 0x4e86a40b // smmla v11.4s, v0.16b, v6.16b\n" + ".inst 0x4e98a755 // smmla v21.4s, v26.16b, v24.16b\n" + "ldr q24, [x9, #0x50]\n" + ".inst 0x4e99a76a // smmla v10.4s, v27.16b, v25.16b\n" + ".inst 0x4e99a752 // smmla v18.4s, v26.16b, v25.16b\n" + "ldr q25, [x9, #0x60]\n" + ".inst 0x4e98a76e // smmla v14.4s, v27.16b, v24.16b\n" + ".inst 0x4e98a756 // smmla v22.4s, v26.16b, v24.16b\n" + "ldr q24, [x9, #0x70]\n" + ".inst 0x4e99a76b // smmla v11.4s, v27.16b, v25.16b\n" "add x9, x9, #0x80\n" - ".inst 0x4e86a453 // smmla v19.4s, v2.16b, v6.16b\n" - ".inst 0x4e87a40f // smmla v15.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a457 // smmla v23.4s, v2.16b, v7.16b\n" + ".inst 0x4e99a753 // smmla v19.4s, v26.16b, v25.16b\n" + ".inst 0x4e98a76f // smmla v15.4s, v27.16b, v24.16b\n" + ".inst 0x4e98a757 // smmla v23.4s, v26.16b, v24.16b\n" "bge 68b\n" "69:" // Height 3: Multiply loop: Skip odd blocks "cbz x27, 74f\n" @@ -1082,74 +1081,74 @@ void a64_hybrid_s8qs_mmla_6x16 ( "ldr b2, [x25, #0x0]\n" "ldr b3, [x24, #0x0]\n" "73:" // Height 3: Multiply loop: Ragged operand read: Done - "ldr q7, [x9, #0x0]\n" - "ldr q6, [x9, #0x10]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a450 // smmla v16.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a451 // smmla v17.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a455 // smmla v21.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a452 // smmla v18.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x70]\n" + "ldr q25, [x9, #0x0]\n" + "ldr q28, [x9, #0x10]\n" + "trn1 v27.2d, v1.2d, v2.2d\n" + "trn1 v26.2d, v3.2d, v24.2d\n" + ".inst 0x4e99a768 // smmla v8.4s, v27.16b, v25.16b\n" + ".inst 0x4e99a750 // smmla v16.4s, v26.16b, v25.16b\n" + "ldr q25, [x9, #0x20]\n" + ".inst 0x4e9ca76c // smmla v12.4s, v27.16b, v28.16b\n" + ".inst 0x4e9ca754 // smmla v20.4s, v26.16b, v28.16b\n" + "ldr q24, [x9, #0x30]\n" + ".inst 0x4e99a769 // smmla v9.4s, v27.16b, v25.16b\n" + ".inst 0x4e99a751 // smmla v17.4s, v26.16b, v25.16b\n" + "ldr q25, [x9, #0x40]\n" + ".inst 0x4e98a76d // smmla v13.4s, v27.16b, v24.16b\n" + ".inst 0x4e98a755 // smmla v21.4s, v26.16b, v24.16b\n" + "ldr q24, [x9, #0x50]\n" + ".inst 0x4e99a76a // smmla v10.4s, v27.16b, v25.16b\n" + ".inst 0x4e99a752 // smmla v18.4s, v26.16b, v25.16b\n" + "ldr q25, [x9, #0x60]\n" + ".inst 0x4e98a76e // smmla v14.4s, v27.16b, v24.16b\n" + ".inst 0x4e98a756 // smmla v22.4s, v26.16b, v24.16b\n" + "ldr q24, [x9, #0x70]\n" "add x9, x9, #0x80\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a453 // smmla v19.4s, v2.16b, v7.16b\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a457 // smmla v23.4s, v2.16b, v6.16b\n" + ".inst 0x4e99a76b // smmla v11.4s, v27.16b, v25.16b\n" + ".inst 0x4e99a753 // smmla v19.4s, v26.16b, v25.16b\n" + ".inst 0x4e98a76f // smmla v15.4s, v27.16b, v24.16b\n" + ".inst 0x4e98a757 // smmla v23.4s, v26.16b, v24.16b\n" "74:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 62b\n" - "ldr q0, [x14, #0x0]\n" - "ldr q1, [x14, #0x10]\n" - "uzp1 v7.2d, v8.2d, v12.2d\n" + "ldr q28, [x14, #0x0]\n" + "ldr q27, [x14, #0x10]\n" + "uzp1 v26.2d, v8.2d, v12.2d\n" "uzp2 v8.2d, v8.2d, v12.2d\n" - "ldr q2, [x14, #0x20]\n" - "ldr q3, [x14, #0x30]\n" + "ldr q25, [x14, #0x20]\n" + "ldr q24, [x14, #0x30]\n" "uzp1 v12.2d, v9.2d, v13.2d\n" "uzp2 v9.2d, v9.2d, v13.2d\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "uzp1 v13.2d, v10.2d, v14.2d\n" "uzp2 v10.2d, v10.2d, v14.2d\n" - "add x24, x11, x20\n" + "add x25, x11, x20\n" "uzp1 v14.2d, v11.2d, v15.2d\n" "uzp2 v11.2d, v11.2d, v15.2d\n" - "add x23, x24, x20\n" + "add x24, x25, x20\n" "prfm pstl1keep, [x11, #0x0]\n" "uzp1 v16.2d, v16.2d, v20.2d\n" "uzp1 v17.2d, v17.2d, v21.2d\n" + "prfm pstl1keep, [x25, #0x0]\n" "prfm pstl1keep, [x24, #0x0]\n" - "prfm pstl1keep, [x23, #0x0]\n" "uzp1 v18.2d, v18.2d, v22.2d\n" "uzp1 v19.2d, v19.2d, v23.2d\n" "add x14, x14, #0x40\n" - "mov v23.16b, v7.16b\n" - "add v23.4s, v23.4s, v0.4s\n" - "add v12.4s, v12.4s, v1.4s\n" - "add v13.4s, v13.4s, v2.4s\n" - "add v14.4s, v14.4s, v3.4s\n" - "add v8.4s, v8.4s, v0.4s\n" - "add v9.4s, v9.4s, v1.4s\n" - "add v10.4s, v10.4s, v2.4s\n" - "add v11.4s, v11.4s, v3.4s\n" - "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" + "mov v23.16b, v26.16b\n" + "add v23.4s, v23.4s, v28.4s\n" + "add v12.4s, v12.4s, v27.4s\n" + "add v13.4s, v13.4s, v25.4s\n" + "add v14.4s, v14.4s, v24.4s\n" + "add v8.4s, v8.4s, v28.4s\n" + "add v9.4s, v9.4s, v27.4s\n" + "add v10.4s, v10.4s, v25.4s\n" + "add v11.4s, v11.4s, v24.4s\n" + "add v16.4s, v16.4s, v28.4s\n" + "add v17.4s, v17.4s, v27.4s\n" + "add v18.4s, v18.4s, v25.4s\n" + "add v19.4s, v19.4s, v24.4s\n" "tbz %x[flags], #4, 75f\n" "ldr q0, [x12, #0x0]\n" "ldr q4, [x13, #0x0]\n" @@ -1163,10 +1162,10 @@ void a64_hybrid_s8qs_mmla_6x16 ( "add x13, x13, #0x40\n" "b 76f\n" "75:" // Height 3: per layer parameters - "add x25, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x25]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v4.4s }, [x20]\n" "mov v1.16b, v0.16b\n" "mov v5.16b, v4.16b\n" "mov v2.16b, v0.16b\n" @@ -1187,55 +1186,55 @@ void a64_hybrid_s8qs_mmla_6x16 ( "sqrdmulh v18.4s, v18.4s, v6.4s\n" "sqrdmulh v19.4s, v19.4s, v7.4s\n" "tbz %x[flags], #5, 77f\n" - "and v4.16b, v23.16b, v0.16b\n" - "and v5.16b, v12.16b, v1.16b\n" - "and v6.16b, v13.16b, v2.16b\n" - "and v7.16b, v14.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v12.4s, v12.4s, v5.4s\n" - "sqadd v13.4s, v13.4s, v6.4s\n" - "sqadd v14.4s, v14.4s, v7.4s\n" - "and v4.16b, v8.16b, v0.16b\n" - "and v5.16b, v9.16b, v1.16b\n" - "and v6.16b, v10.16b, v2.16b\n" - "and v7.16b, v11.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v4.4s\n" - "sqadd v9.4s, v9.4s, v5.4s\n" - "sqadd v10.4s, v10.4s, v6.4s\n" - "sqadd v11.4s, v11.4s, v7.4s\n" - "and v4.16b, v16.16b, v0.16b\n" - "and v5.16b, v17.16b, v1.16b\n" - "and v6.16b, v18.16b, v2.16b\n" - "and v7.16b, v19.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" + "and v24.16b, v23.16b, v0.16b\n" + "and v22.16b, v12.16b, v1.16b\n" + "and v21.16b, v13.16b, v2.16b\n" + "and v20.16b, v14.16b, v3.16b\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "sshr v21.4s, v21.4s, #0x1f\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "sqadd v23.4s, v23.4s, v24.4s\n" + "sqadd v12.4s, v12.4s, v22.4s\n" + "sqadd v13.4s, v13.4s, v21.4s\n" + "sqadd v14.4s, v14.4s, v20.4s\n" + "and v24.16b, v8.16b, v0.16b\n" + "and v22.16b, v9.16b, v1.16b\n" + "and v21.16b, v10.16b, v2.16b\n" + "and v20.16b, v11.16b, v3.16b\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "sshr v21.4s, v21.4s, #0x1f\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "sqadd v8.4s, v8.4s, v24.4s\n" + "sqadd v9.4s, v9.4s, v22.4s\n" + "sqadd v10.4s, v10.4s, v21.4s\n" + "sqadd v11.4s, v11.4s, v20.4s\n" + "and v24.16b, v16.16b, v0.16b\n" + "and v22.16b, v17.16b, v1.16b\n" + "and v21.16b, v18.16b, v2.16b\n" + "and v20.16b, v19.16b, v3.16b\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "sshr v21.4s, v21.4s, #0x1f\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v24.4s\n" + "sqadd v17.4s, v17.4s, v22.4s\n" + "sqadd v18.4s, v18.4s, v21.4s\n" + "sqadd v19.4s, v19.4s, v20.4s\n" "77:" // Height 3: no shift correction - "add x25, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v22.4s }, [x20]\n" "srshl v23.4s, v23.4s, v0.4s\n" "srshl v12.4s, v12.4s, v1.4s\n" "srshl v13.4s, v13.4s, v2.4s\n" "srshl v14.4s, v14.4s, v3.4s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x25]\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v21.4s }, [x20]\n" "srshl v8.4s, v8.4s, v0.4s\n" "srshl v9.4s, v9.4s, v1.4s\n" - "add x25, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x25]\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v20.4s }, [x20]\n" "srshl v10.4s, v10.4s, v2.4s\n" "srshl v11.4s, v11.4s, v3.4s\n" "cmp x10, #0x10\n" @@ -1243,132 +1242,132 @@ void a64_hybrid_s8qs_mmla_6x16 ( "srshl v17.4s, v17.4s, v1.4s\n" "srshl v18.4s, v18.4s, v2.4s\n" "srshl v19.4s, v19.4s, v3.4s\n" - "add v23.4s, v23.4s, v4.4s\n" - "add v12.4s, v12.4s, v4.4s\n" - "add v13.4s, v13.4s, v4.4s\n" - "add v14.4s, v14.4s, v4.4s\n" - "add v8.4s, v8.4s, v4.4s\n" - "add v9.4s, v9.4s, v4.4s\n" - "add v10.4s, v10.4s, v4.4s\n" - "add v11.4s, v11.4s, v4.4s\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "smin v12.4s, v12.4s, v6.4s\n" - "smin v13.4s, v13.4s, v6.4s\n" - "smin v14.4s, v14.4s, v6.4s\n" - "smin v8.4s, v8.4s, v6.4s\n" - "smin v9.4s, v9.4s, v6.4s\n" - "smin v10.4s, v10.4s, v6.4s\n" - "smin v11.4s, v11.4s, v6.4s\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" - "smax v12.4s, v12.4s, v5.4s\n" - "smax v13.4s, v13.4s, v5.4s\n" - "smax v14.4s, v14.4s, v5.4s\n" - "smax v8.4s, v8.4s, v5.4s\n" - "smax v9.4s, v9.4s, v5.4s\n" - "smax v10.4s, v10.4s, v5.4s\n" - "smax v11.4s, v11.4s, v5.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" + "add v23.4s, v23.4s, v22.4s\n" + "add v12.4s, v12.4s, v22.4s\n" + "add v13.4s, v13.4s, v22.4s\n" + "add v14.4s, v14.4s, v22.4s\n" + "add v8.4s, v8.4s, v22.4s\n" + "add v9.4s, v9.4s, v22.4s\n" + "add v10.4s, v10.4s, v22.4s\n" + "add v11.4s, v11.4s, v22.4s\n" + "add v16.4s, v16.4s, v22.4s\n" + "add v17.4s, v17.4s, v22.4s\n" + "add v18.4s, v18.4s, v22.4s\n" + "add v19.4s, v19.4s, v22.4s\n" + "smin v23.4s, v23.4s, v21.4s\n" + "smin v12.4s, v12.4s, v21.4s\n" + "smin v13.4s, v13.4s, v21.4s\n" + "smin v14.4s, v14.4s, v21.4s\n" + "smin v8.4s, v8.4s, v21.4s\n" + "smin v9.4s, v9.4s, v21.4s\n" + "smin v10.4s, v10.4s, v21.4s\n" + "smin v11.4s, v11.4s, v21.4s\n" + "smin v16.4s, v16.4s, v21.4s\n" + "smin v17.4s, v17.4s, v21.4s\n" + "smin v18.4s, v18.4s, v21.4s\n" + "smin v19.4s, v19.4s, v21.4s\n" + "smax v23.4s, v23.4s, v20.4s\n" + "smax v12.4s, v12.4s, v20.4s\n" + "smax v13.4s, v13.4s, v20.4s\n" + "smax v14.4s, v14.4s, v20.4s\n" + "smax v8.4s, v8.4s, v20.4s\n" + "smax v9.4s, v9.4s, v20.4s\n" + "smax v10.4s, v10.4s, v20.4s\n" + "smax v11.4s, v11.4s, v20.4s\n" + "smax v16.4s, v16.4s, v20.4s\n" + "smax v17.4s, v17.4s, v20.4s\n" + "smax v18.4s, v18.4s, v20.4s\n" + "smax v19.4s, v19.4s, v20.4s\n" "uzp1 v23.8h, v23.8h, v12.8h\n" - "uzp1 v12.8h, v13.8h, v14.8h\n" + "uzp1 v21.8h, v13.8h, v14.8h\n" "uzp1 v8.8h, v8.8h, v9.8h\n" - "uzp1 v9.8h, v10.8h, v11.8h\n" + "uzp1 v20.8h, v10.8h, v11.8h\n" "uzp1 v16.8h, v16.8h, v17.8h\n" "uzp1 v17.8h, v18.8h, v19.8h\n" - "uzp1 v23.16b, v23.16b, v12.16b\n" - "uzp1 v8.16b, v8.16b, v9.16b\n" + "uzp1 v23.16b, v23.16b, v21.16b\n" + "uzp1 v8.16b, v8.16b, v20.16b\n" "uzp1 v16.16b, v16.16b, v17.16b\n" "bge 86f\n" "tbz x10, #3, 81f\n" "str d23, [x11], #0x8\n" - "str d8, [x24], #0x8\n" - "str d16, [x23], #0x8\n" + "str d8, [x25], #0x8\n" + "str d16, [x24], #0x8\n" "tbz x10, #2, 79f\n" "st1 { v23.s }[2], [x11], #0x4\n" - "st1 { v8.s }[2], [x24], #0x4\n" - "st1 { v16.s }[2], [x23], #0x4\n" + "st1 { v8.s }[2], [x25], #0x4\n" + "st1 { v16.s }[2], [x24], #0x4\n" "tbz x10, #1, 78f\n" "st1 { v23.h }[6], [x11], #0x2\n" - "st1 { v8.h }[6], [x24], #0x2\n" - "st1 { v16.h }[6], [x23], #0x2\n" + "st1 { v8.h }[6], [x25], #0x2\n" + "st1 { v16.h }[6], [x24], #0x2\n" "tbz x10, #0, 85f\n" "st1 { v23.b }[14], [x11]\n" - "st1 { v8.b }[14], [x24]\n" - "st1 { v16.b }[14], [x23]\n" + "st1 { v8.b }[14], [x25]\n" + "st1 { v16.b }[14], [x24]\n" "b 85f\n" "78:" // Height 3: Partial direct writeback: partial_1_12 "tbz x10, #0, 85f\n" "st1 { v23.b }[12], [x11]\n" - "st1 { v8.b }[12], [x24]\n" - "st1 { v16.b }[12], [x23]\n" + "st1 { v8.b }[12], [x25]\n" + "st1 { v16.b }[12], [x24]\n" "b 85f\n" "79:" // Height 3: Partial direct writeback: partial_2_8 "tbz x10, #1, 80f\n" "st1 { v23.h }[4], [x11], #0x2\n" - "st1 { v8.h }[4], [x24], #0x2\n" - "st1 { v16.h }[4], [x23], #0x2\n" + "st1 { v8.h }[4], [x25], #0x2\n" + "st1 { v16.h }[4], [x24], #0x2\n" "tbz x10, #0, 85f\n" "st1 { v23.b }[10], [x11]\n" - "st1 { v8.b }[10], [x24]\n" - "st1 { v16.b }[10], [x23]\n" + "st1 { v8.b }[10], [x25]\n" + "st1 { v16.b }[10], [x24]\n" "b 85f\n" "80:" // Height 3: Partial direct writeback: partial_1_8 "tbz x10, #0, 85f\n" "st1 { v23.b }[8], [x11]\n" - "st1 { v8.b }[8], [x24]\n" - "st1 { v16.b }[8], [x23]\n" + "st1 { v8.b }[8], [x25]\n" + "st1 { v16.b }[8], [x24]\n" "b 85f\n" "81:" // Height 3: Partial direct writeback: partial_4_0 "tbz x10, #2, 83f\n" "str s23, [x11], #0x4\n" - "str s8, [x24], #0x4\n" - "str s16, [x23], #0x4\n" + "str s8, [x25], #0x4\n" + "str s16, [x24], #0x4\n" "tbz x10, #1, 82f\n" "st1 { v23.h }[2], [x11], #0x2\n" - "st1 { v8.h }[2], [x24], #0x2\n" - "st1 { v16.h }[2], [x23], #0x2\n" + "st1 { v8.h }[2], [x25], #0x2\n" + "st1 { v16.h }[2], [x24], #0x2\n" "tbz x10, #0, 85f\n" "st1 { v23.b }[6], [x11]\n" - "st1 { v8.b }[6], [x24]\n" - "st1 { v16.b }[6], [x23]\n" + "st1 { v8.b }[6], [x25]\n" + "st1 { v16.b }[6], [x24]\n" "b 85f\n" "82:" // Height 3: Partial direct writeback: partial_1_4 "tbz x10, #0, 85f\n" "st1 { v23.b }[4], [x11]\n" - "st1 { v8.b }[4], [x24]\n" - "st1 { v16.b }[4], [x23]\n" + "st1 { v8.b }[4], [x25]\n" + "st1 { v16.b }[4], [x24]\n" "b 85f\n" "83:" // Height 3: Partial direct writeback: partial_2_0 "tbz x10, #1, 84f\n" "str h23, [x11], #0x2\n" - "str h8, [x24], #0x2\n" - "str h16, [x23], #0x2\n" + "str h8, [x25], #0x2\n" + "str h16, [x24], #0x2\n" "tbz x10, #0, 85f\n" "st1 { v23.b }[2], [x11]\n" - "st1 { v8.b }[2], [x24]\n" - "st1 { v16.b }[2], [x23]\n" + "st1 { v8.b }[2], [x25]\n" + "st1 { v16.b }[2], [x24]\n" "b 85f\n" "84:" // Height 3: Partial direct writeback: partial_1_0 "str b23, [x11, #0x0]\n" - "str b8, [x24, #0x0]\n" - "str b16, [x23, #0x0]\n" + "str b8, [x25, #0x0]\n" + "str b16, [x24, #0x0]\n" "85:" // Height 3: Partial direct writeback: Done "b 87f\n" "86:" // Height 3: Full writeback "str q23, [x11, #0x0]\n" "add x11, x11, #0x10\n" - "str q8, [x24, #0x0]\n" - "str q16, [x23, #0x0]\n" + "str q8, [x25, #0x0]\n" + "str q16, [x24, #0x0]\n" "87:" // Height 3: Writeback done "subs x10, x10, #0x10\n" "bgt 60b\n" @@ -1402,14 +1401,14 @@ void a64_hybrid_s8qs_mmla_6x16 ( "91:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 92f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 93f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -1419,9 +1418,9 @@ void a64_hybrid_s8qs_mmla_6x16 ( "b 93f\n" "92:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" "93:" // Height 4: input setup done "cmp x27, #0x10\n" "blt 96f\n" @@ -1434,173 +1433,173 @@ void a64_hybrid_s8qs_mmla_6x16 ( "ldr q6, [x9, #0x10]\n" "blt 95f\n" "94:" // Height 4: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v27.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" + ".inst 0x4e87a768 // smmla v8.4s, v27.16b, v7.16b\n" "sub x27, x27, #0x10\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x4e87a450 // smmla v16.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" + "trn1 v26.2d, v3.2d, v4.2d\n" + ".inst 0x4e87a750 // smmla v16.4s, v26.16b, v7.16b\n" + "ldr q25, [x9, #0x20]\n" + ".inst 0x4e86a76c // smmla v12.4s, v27.16b, v6.16b\n" + ".inst 0x4e86a754 // smmla v20.4s, v26.16b, v6.16b\n" + "ldr q24, [x9, #0x30]\n" + ".inst 0x4e99a769 // smmla v9.4s, v27.16b, v25.16b\n" "trn2 v3.2d, v3.2d, v4.2d\n" - ".inst 0x4e87a451 // smmla v17.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" + ".inst 0x4e99a751 // smmla v17.4s, v26.16b, v25.16b\n" + "ldr q25, [x9, #0x40]\n" + ".inst 0x4e98a76d // smmla v13.4s, v27.16b, v24.16b\n" "add x26, x26, #0x10\n" - ".inst 0x4e86a455 // smmla v21.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" + ".inst 0x4e98a755 // smmla v21.4s, v26.16b, v24.16b\n" + "ldr q24, [x9, #0x50]\n" + ".inst 0x4e99a76a // smmla v10.4s, v27.16b, v25.16b\n" "add x25, x25, #0x10\n" - ".inst 0x4e87a452 // smmla v18.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" + ".inst 0x4e99a752 // smmla v18.4s, v26.16b, v25.16b\n" + "ldr q25, [x9, #0x60]\n" + ".inst 0x4e98a76e // smmla v14.4s, v27.16b, v24.16b\n" "add x24, x24, #0x10\n" - ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x70]\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" + ".inst 0x4e98a756 // smmla v22.4s, v26.16b, v24.16b\n" + "ldr q24, [x9, #0x70]\n" + ".inst 0x4e99a76b // smmla v11.4s, v27.16b, v25.16b\n" "add x23, x23, #0x10\n" "ldr q4, [x23, #0x0]\n" - ".inst 0x4e87a453 // smmla v19.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x80]\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a457 // smmla v23.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x90]\n" + ".inst 0x4e99a753 // smmla v19.4s, v26.16b, v25.16b\n" + "ldr q25, [x9, #0x80]\n" + ".inst 0x4e98a76f // smmla v15.4s, v27.16b, v24.16b\n" + ".inst 0x4e98a757 // smmla v23.4s, v26.16b, v24.16b\n" + "ldr q24, [x9, #0x90]\n" "ldr q2, [x25, #0x0]\n" - ".inst 0x4e87a428 // smmla v8.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a470 // smmla v16.4s, v3.16b, v7.16b\n" - "ldr q7, [x9, #0xa0]\n" - ".inst 0x4e86a42c // smmla v12.4s, v1.16b, v6.16b\n" + ".inst 0x4e99a428 // smmla v8.4s, v1.16b, v25.16b\n" + ".inst 0x4e99a470 // smmla v16.4s, v3.16b, v25.16b\n" + "ldr q25, [x9, #0xa0]\n" + ".inst 0x4e98a42c // smmla v12.4s, v1.16b, v24.16b\n" "cmp x27, #0x20\n" - ".inst 0x4e86a474 // smmla v20.4s, v3.16b, v6.16b\n" - "ldr q6, [x9, #0xb0]\n" - ".inst 0x4e87a429 // smmla v9.4s, v1.16b, v7.16b\n" + ".inst 0x4e98a474 // smmla v20.4s, v3.16b, v24.16b\n" + "ldr q24, [x9, #0xb0]\n" + ".inst 0x4e99a429 // smmla v9.4s, v1.16b, v25.16b\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x4e87a471 // smmla v17.4s, v3.16b, v7.16b\n" - "ldr q7, [x9, #0xc0]\n" - ".inst 0x4e86a42d // smmla v13.4s, v1.16b, v6.16b\n" + ".inst 0x4e99a471 // smmla v17.4s, v3.16b, v25.16b\n" + "ldr q25, [x9, #0xc0]\n" + ".inst 0x4e98a42d // smmla v13.4s, v1.16b, v24.16b\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4e86a475 // smmla v21.4s, v3.16b, v6.16b\n" - "ldr q6, [x9, #0xd0]\n" - ".inst 0x4e87a42a // smmla v10.4s, v1.16b, v7.16b\n" + ".inst 0x4e98a475 // smmla v21.4s, v3.16b, v24.16b\n" + "ldr q24, [x9, #0xd0]\n" + ".inst 0x4e99a42a // smmla v10.4s, v1.16b, v25.16b\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x4e87a472 // smmla v18.4s, v3.16b, v7.16b\n" - "ldr q7, [x9, #0xe0]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" + ".inst 0x4e99a472 // smmla v18.4s, v3.16b, v25.16b\n" + "ldr q25, [x9, #0xe0]\n" + ".inst 0x4e98a42e // smmla v14.4s, v1.16b, v24.16b\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x4e86a476 // smmla v22.4s, v3.16b, v6.16b\n" - "ldr q6, [x9, #0xf0]\n" + ".inst 0x4e98a476 // smmla v22.4s, v3.16b, v24.16b\n" + "ldr q24, [x9, #0xf0]\n" "add x9, x9, #0x100\n" - ".inst 0x4e87a42b // smmla v11.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a473 // smmla v19.4s, v3.16b, v7.16b\n" + ".inst 0x4e99a42b // smmla v11.4s, v1.16b, v25.16b\n" + ".inst 0x4e99a473 // smmla v19.4s, v3.16b, v25.16b\n" "ldr q7, [x9, #0x0]\n" - ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" + ".inst 0x4e98a42f // smmla v15.4s, v1.16b, v24.16b\n" "ldr q1, [x26, #0x0]\n" - ".inst 0x4e86a477 // smmla v23.4s, v3.16b, v6.16b\n" + ".inst 0x4e98a477 // smmla v23.4s, v3.16b, v24.16b\n" "ldr q3, [x24, #0x0]\n" "ldr q6, [x9, #0x10]\n" "bge 94b\n" "95:" // Height 4: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v27.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" + ".inst 0x4e87a768 // smmla v8.4s, v27.16b, v7.16b\n" "add x26, x26, #0x10\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x4e87a450 // smmla v16.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" + "trn1 v26.2d, v3.2d, v4.2d\n" + ".inst 0x4e87a750 // smmla v16.4s, v26.16b, v7.16b\n" + "ldr q25, [x9, #0x20]\n" + ".inst 0x4e86a76c // smmla v12.4s, v27.16b, v6.16b\n" + ".inst 0x4e86a754 // smmla v20.4s, v26.16b, v6.16b\n" + "ldr q24, [x9, #0x30]\n" + ".inst 0x4e99a769 // smmla v9.4s, v27.16b, v25.16b\n" "trn2 v3.2d, v3.2d, v4.2d\n" - ".inst 0x4e87a451 // smmla v17.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" + ".inst 0x4e99a751 // smmla v17.4s, v26.16b, v25.16b\n" + "ldr q25, [x9, #0x40]\n" + ".inst 0x4e98a76d // smmla v13.4s, v27.16b, v24.16b\n" "add x25, x25, #0x10\n" - ".inst 0x4e86a455 // smmla v21.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" + ".inst 0x4e98a755 // smmla v21.4s, v26.16b, v24.16b\n" + "ldr q24, [x9, #0x50]\n" + ".inst 0x4e99a76a // smmla v10.4s, v27.16b, v25.16b\n" "add x24, x24, #0x10\n" - ".inst 0x4e87a452 // smmla v18.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" + ".inst 0x4e99a752 // smmla v18.4s, v26.16b, v25.16b\n" + "ldr q25, [x9, #0x60]\n" + ".inst 0x4e98a76e // smmla v14.4s, v27.16b, v24.16b\n" "add x23, x23, #0x10\n" - ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x70]\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" + ".inst 0x4e98a756 // smmla v22.4s, v26.16b, v24.16b\n" + "ldr q24, [x9, #0x70]\n" + ".inst 0x4e99a76b // smmla v11.4s, v27.16b, v25.16b\n" "sub x27, x27, #0x10\n" - ".inst 0x4e87a453 // smmla v19.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x80]\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" + ".inst 0x4e99a753 // smmla v19.4s, v26.16b, v25.16b\n" + "ldr q25, [x9, #0x80]\n" + ".inst 0x4e98a76f // smmla v15.4s, v27.16b, v24.16b\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x4e86a457 // smmla v23.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x90]\n" - ".inst 0x4e87a428 // smmla v8.4s, v1.16b, v7.16b\n" + ".inst 0x4e98a757 // smmla v23.4s, v26.16b, v24.16b\n" + "ldr q24, [x9, #0x90]\n" + ".inst 0x4e99a428 // smmla v8.4s, v1.16b, v25.16b\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4e87a470 // smmla v16.4s, v3.16b, v7.16b\n" - "ldr q7, [x9, #0xa0]\n" - ".inst 0x4e86a42c // smmla v12.4s, v1.16b, v6.16b\n" + ".inst 0x4e99a470 // smmla v16.4s, v3.16b, v25.16b\n" + "ldr q25, [x9, #0xa0]\n" + ".inst 0x4e98a42c // smmla v12.4s, v1.16b, v24.16b\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x4e86a474 // smmla v20.4s, v3.16b, v6.16b\n" - "ldr q6, [x9, #0xb0]\n" - ".inst 0x4e87a429 // smmla v9.4s, v1.16b, v7.16b\n" + ".inst 0x4e98a474 // smmla v20.4s, v3.16b, v24.16b\n" + "ldr q24, [x9, #0xb0]\n" + ".inst 0x4e99a429 // smmla v9.4s, v1.16b, v25.16b\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x4e87a471 // smmla v17.4s, v3.16b, v7.16b\n" - "ldr q7, [x9, #0xc0]\n" - ".inst 0x4e86a42d // smmla v13.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a475 // smmla v21.4s, v3.16b, v6.16b\n" - "ldr q6, [x9, #0xd0]\n" - ".inst 0x4e87a42a // smmla v10.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a472 // smmla v18.4s, v3.16b, v7.16b\n" - "ldr q7, [x9, #0xe0]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a476 // smmla v22.4s, v3.16b, v6.16b\n" - "ldr q6, [x9, #0xf0]\n" + ".inst 0x4e99a471 // smmla v17.4s, v3.16b, v25.16b\n" + "ldr q25, [x9, #0xc0]\n" + ".inst 0x4e98a42d // smmla v13.4s, v1.16b, v24.16b\n" + ".inst 0x4e98a475 // smmla v21.4s, v3.16b, v24.16b\n" + "ldr q24, [x9, #0xd0]\n" + ".inst 0x4e99a42a // smmla v10.4s, v1.16b, v25.16b\n" + ".inst 0x4e99a472 // smmla v18.4s, v3.16b, v25.16b\n" + "ldr q25, [x9, #0xe0]\n" + ".inst 0x4e98a42e // smmla v14.4s, v1.16b, v24.16b\n" + ".inst 0x4e98a476 // smmla v22.4s, v3.16b, v24.16b\n" + "ldr q24, [x9, #0xf0]\n" "add x9, x9, #0x100\n" - ".inst 0x4e87a42b // smmla v11.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a473 // smmla v19.4s, v3.16b, v7.16b\n" - ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a477 // smmla v23.4s, v3.16b, v6.16b\n" + ".inst 0x4e99a42b // smmla v11.4s, v1.16b, v25.16b\n" + ".inst 0x4e99a473 // smmla v19.4s, v3.16b, v25.16b\n" + ".inst 0x4e98a42f // smmla v15.4s, v1.16b, v24.16b\n" + ".inst 0x4e98a477 // smmla v23.4s, v3.16b, v24.16b\n" "96:" // Height 4: Multiply loop: Main loop skip "cbz x27, 103f\n" "cmp x27, #0x8\n" "blt 98f\n" "97:" // Height 4: Multiply loop: Odd block loop - "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" + "ldr d25, [x26], #0x8\n" + "ldr d24, [x25], #0x8\n" + "trn1 v27.2d, v25.2d, v24.2d\n" "sub x27, x27, #0x8\n" - "ldr d3, [x24], #0x8\n" - "ldr d4, [x23], #0x8\n" - "trn1 v2.2d, v3.2d, v4.2d\n" + "ldr d25, [x24], #0x8\n" + "ldr d24, [x23], #0x8\n" + "trn1 v26.2d, v25.2d, v24.2d\n" "cmp x27, #0x8\n" - "ldr q6, [x9, #0x0]\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x4e86a408 // smmla v8.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a450 // smmla v16.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x20]\n" - ".inst 0x4e87a40c // smmla v12.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a454 // smmla v20.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x30]\n" - ".inst 0x4e86a409 // smmla v9.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a451 // smmla v17.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x40]\n" - ".inst 0x4e87a40d // smmla v13.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a455 // smmla v21.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x50]\n" - ".inst 0x4e86a40a // smmla v10.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a452 // smmla v18.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x60]\n" - ".inst 0x4e87a40e // smmla v14.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a456 // smmla v22.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x70]\n" + "ldr q25, [x9, #0x0]\n" + "ldr q24, [x9, #0x10]\n" + ".inst 0x4e99a768 // smmla v8.4s, v27.16b, v25.16b\n" + ".inst 0x4e99a750 // smmla v16.4s, v26.16b, v25.16b\n" + "ldr q25, [x9, #0x20]\n" + ".inst 0x4e98a76c // smmla v12.4s, v27.16b, v24.16b\n" + ".inst 0x4e98a754 // smmla v20.4s, v26.16b, v24.16b\n" + "ldr q24, [x9, #0x30]\n" + ".inst 0x4e99a769 // smmla v9.4s, v27.16b, v25.16b\n" + ".inst 0x4e99a751 // smmla v17.4s, v26.16b, v25.16b\n" + "ldr q25, [x9, #0x40]\n" + ".inst 0x4e98a76d // smmla v13.4s, v27.16b, v24.16b\n" + ".inst 0x4e98a755 // smmla v21.4s, v26.16b, v24.16b\n" + "ldr q24, [x9, #0x50]\n" + ".inst 0x4e99a76a // smmla v10.4s, v27.16b, v25.16b\n" + ".inst 0x4e99a752 // smmla v18.4s, v26.16b, v25.16b\n" + "ldr q25, [x9, #0x60]\n" + ".inst 0x4e98a76e // smmla v14.4s, v27.16b, v24.16b\n" + ".inst 0x4e98a756 // smmla v22.4s, v26.16b, v24.16b\n" + "ldr q24, [x9, #0x70]\n" "add x9, x9, #0x80\n" - ".inst 0x4e86a40b // smmla v11.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a453 // smmla v19.4s, v2.16b, v6.16b\n" - ".inst 0x4e87a40f // smmla v15.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a457 // smmla v23.4s, v2.16b, v7.16b\n" + ".inst 0x4e99a76b // smmla v11.4s, v27.16b, v25.16b\n" + ".inst 0x4e99a753 // smmla v19.4s, v26.16b, v25.16b\n" + ".inst 0x4e98a76f // smmla v15.4s, v27.16b, v24.16b\n" + ".inst 0x4e98a757 // smmla v23.4s, v26.16b, v24.16b\n" "bge 97b\n" "98:" // Height 4: Multiply loop: Skip odd blocks "cbz x27, 103f\n" @@ -1645,84 +1644,84 @@ void a64_hybrid_s8qs_mmla_6x16 ( "ldr b3, [x24, #0x0]\n" "ldr b4, [x23, #0x0]\n" "102:" // Height 4: Multiply loop: Ragged operand read: Done - "ldr q7, [x9, #0x0]\n" - "ldr q6, [x9, #0x10]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a450 // smmla v16.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a451 // smmla v17.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a455 // smmla v21.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a452 // smmla v18.4s, v2.16b, v7.16b\n" - "ldr q7, [x9, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n" - "ldr q6, [x9, #0x70]\n" + "ldr q25, [x9, #0x0]\n" + "ldr q24, [x9, #0x10]\n" + "trn1 v27.2d, v1.2d, v2.2d\n" + "trn1 v26.2d, v3.2d, v4.2d\n" + ".inst 0x4e99a768 // smmla v8.4s, v27.16b, v25.16b\n" + ".inst 0x4e99a750 // smmla v16.4s, v26.16b, v25.16b\n" + "ldr q25, [x9, #0x20]\n" + ".inst 0x4e98a76c // smmla v12.4s, v27.16b, v24.16b\n" + ".inst 0x4e98a754 // smmla v20.4s, v26.16b, v24.16b\n" + "ldr q24, [x9, #0x30]\n" + ".inst 0x4e99a769 // smmla v9.4s, v27.16b, v25.16b\n" + ".inst 0x4e99a751 // smmla v17.4s, v26.16b, v25.16b\n" + "ldr q25, [x9, #0x40]\n" + ".inst 0x4e98a76d // smmla v13.4s, v27.16b, v24.16b\n" + ".inst 0x4e98a755 // smmla v21.4s, v26.16b, v24.16b\n" + "ldr q24, [x9, #0x50]\n" + ".inst 0x4e99a76a // smmla v10.4s, v27.16b, v25.16b\n" + ".inst 0x4e99a752 // smmla v18.4s, v26.16b, v25.16b\n" + "ldr q25, [x9, #0x60]\n" + ".inst 0x4e98a76e // smmla v14.4s, v27.16b, v24.16b\n" + ".inst 0x4e98a756 // smmla v22.4s, v26.16b, v24.16b\n" + "ldr q24, [x9, #0x70]\n" "add x9, x9, #0x80\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a453 // smmla v19.4s, v2.16b, v7.16b\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a457 // smmla v23.4s, v2.16b, v6.16b\n" + ".inst 0x4e99a76b // smmla v11.4s, v27.16b, v25.16b\n" + ".inst 0x4e99a753 // smmla v19.4s, v26.16b, v25.16b\n" + ".inst 0x4e98a76f // smmla v15.4s, v27.16b, v24.16b\n" + ".inst 0x4e98a757 // smmla v23.4s, v26.16b, v24.16b\n" "103:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 91b\n" - "ldr q0, [x14, #0x0]\n" - "ldr q1, [x14, #0x10]\n" - "uzp1 v7.2d, v8.2d, v12.2d\n" + "ldr q28, [x14, #0x0]\n" + "ldr q27, [x14, #0x10]\n" + "uzp1 v26.2d, v8.2d, v12.2d\n" "uzp2 v8.2d, v8.2d, v12.2d\n" - "ldr q2, [x14, #0x20]\n" - "ldr q3, [x14, #0x30]\n" + "ldr q25, [x14, #0x20]\n" + "ldr q24, [x14, #0x30]\n" "uzp1 v12.2d, v9.2d, v13.2d\n" "uzp2 v9.2d, v9.2d, v13.2d\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "uzp1 v13.2d, v10.2d, v14.2d\n" "uzp2 v10.2d, v10.2d, v14.2d\n" - "add x24, x11, x20\n" + "add x25, x11, x20\n" "uzp1 v14.2d, v11.2d, v15.2d\n" "uzp2 v11.2d, v11.2d, v15.2d\n" + "add x24, x25, x20\n" "add x23, x24, x20\n" - "add x22, x23, x20\n" "uzp1 v15.2d, v16.2d, v20.2d\n" "uzp2 v16.2d, v16.2d, v20.2d\n" "prfm pstl1keep, [x11, #0x0]\n" - "prfm pstl1keep, [x24, #0x0]\n" + "prfm pstl1keep, [x25, #0x0]\n" "uzp1 v20.2d, v17.2d, v21.2d\n" "uzp2 v17.2d, v17.2d, v21.2d\n" + "prfm pstl1keep, [x24, #0x0]\n" "prfm pstl1keep, [x23, #0x0]\n" - "prfm pstl1keep, [x22, #0x0]\n" "uzp1 v21.2d, v18.2d, v22.2d\n" "uzp2 v18.2d, v18.2d, v22.2d\n" "add x14, x14, #0x40\n" "uzp1 v22.2d, v19.2d, v23.2d\n" "uzp2 v19.2d, v19.2d, v23.2d\n" - "mov v23.16b, v7.16b\n" - "add v23.4s, v23.4s, v0.4s\n" - "add v12.4s, v12.4s, v1.4s\n" - "add v13.4s, v13.4s, v2.4s\n" - "add v14.4s, v14.4s, v3.4s\n" - "add v8.4s, v8.4s, v0.4s\n" - "add v9.4s, v9.4s, v1.4s\n" - "add v10.4s, v10.4s, v2.4s\n" - "add v11.4s, v11.4s, v3.4s\n" - "add v15.4s, v15.4s, v0.4s\n" - "add v20.4s, v20.4s, v1.4s\n" - "add v21.4s, v21.4s, v2.4s\n" - "add v22.4s, v22.4s, v3.4s\n" - "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" + "mov v23.16b, v26.16b\n" + "add v23.4s, v23.4s, v28.4s\n" + "add v12.4s, v12.4s, v27.4s\n" + "add v13.4s, v13.4s, v25.4s\n" + "add v14.4s, v14.4s, v24.4s\n" + "add v8.4s, v8.4s, v28.4s\n" + "add v9.4s, v9.4s, v27.4s\n" + "add v10.4s, v10.4s, v25.4s\n" + "add v11.4s, v11.4s, v24.4s\n" + "add v15.4s, v15.4s, v28.4s\n" + "add v20.4s, v20.4s, v27.4s\n" + "add v21.4s, v21.4s, v25.4s\n" + "add v22.4s, v22.4s, v24.4s\n" + "add v16.4s, v16.4s, v28.4s\n" + "add v17.4s, v17.4s, v27.4s\n" + "add v18.4s, v18.4s, v25.4s\n" + "add v19.4s, v19.4s, v24.4s\n" "tbz %x[flags], #4, 104f\n" "ldr q0, [x12, #0x0]\n" "ldr q4, [x13, #0x0]\n" @@ -1736,10 +1735,10 @@ void a64_hybrid_s8qs_mmla_6x16 ( "add x13, x13, #0x40\n" "b 105f\n" "104:" // Height 4: per layer parameters - "add x25, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x25]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v4.4s }, [x20]\n" "mov v1.16b, v0.16b\n" "mov v5.16b, v4.16b\n" "mov v2.16b, v0.16b\n" @@ -1763,68 +1762,68 @@ void a64_hybrid_s8qs_mmla_6x16 ( "sqrdmulh v17.4s, v17.4s, v5.4s\n" "sqrdmulh v18.4s, v18.4s, v6.4s\n" "sqrdmulh v19.4s, v19.4s, v7.4s\n" - "tbz %x[flags], #5, 106f\n" - "and v4.16b, v23.16b, v0.16b\n" - "and v5.16b, v12.16b, v1.16b\n" - "and v6.16b, v13.16b, v2.16b\n" - "and v7.16b, v14.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v12.4s, v12.4s, v5.4s\n" - "sqadd v13.4s, v13.4s, v6.4s\n" - "sqadd v14.4s, v14.4s, v7.4s\n" - "and v4.16b, v8.16b, v0.16b\n" - "and v5.16b, v9.16b, v1.16b\n" - "and v6.16b, v10.16b, v2.16b\n" - "and v7.16b, v11.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v4.4s\n" - "sqadd v9.4s, v9.4s, v5.4s\n" - "sqadd v10.4s, v10.4s, v6.4s\n" - "sqadd v11.4s, v11.4s, v7.4s\n" - "and v4.16b, v15.16b, v0.16b\n" - "and v5.16b, v20.16b, v1.16b\n" - "and v6.16b, v21.16b, v2.16b\n" - "and v7.16b, v22.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v15.4s, v15.4s, v4.4s\n" - "sqadd v20.4s, v20.4s, v5.4s\n" - "sqadd v21.4s, v21.4s, v6.4s\n" - "sqadd v22.4s, v22.4s, v7.4s\n" - "and v4.16b, v16.16b, v0.16b\n" - "and v5.16b, v17.16b, v1.16b\n" - "and v6.16b, v18.16b, v2.16b\n" - "and v7.16b, v19.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" + "tbz %x[flags], #5, 106f\n" + "and v27.16b, v23.16b, v0.16b\n" + "and v26.16b, v12.16b, v1.16b\n" + "and v25.16b, v13.16b, v2.16b\n" + "and v24.16b, v14.16b, v3.16b\n" + "sshr v27.4s, v27.4s, #0x1f\n" + "sshr v26.4s, v26.4s, #0x1f\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sqadd v23.4s, v23.4s, v27.4s\n" + "sqadd v12.4s, v12.4s, v26.4s\n" + "sqadd v13.4s, v13.4s, v25.4s\n" + "sqadd v14.4s, v14.4s, v24.4s\n" + "and v27.16b, v8.16b, v0.16b\n" + "and v26.16b, v9.16b, v1.16b\n" + "and v25.16b, v10.16b, v2.16b\n" + "and v24.16b, v11.16b, v3.16b\n" + "sshr v27.4s, v27.4s, #0x1f\n" + "sshr v26.4s, v26.4s, #0x1f\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sqadd v8.4s, v8.4s, v27.4s\n" + "sqadd v9.4s, v9.4s, v26.4s\n" + "sqadd v10.4s, v10.4s, v25.4s\n" + "sqadd v11.4s, v11.4s, v24.4s\n" + "and v27.16b, v15.16b, v0.16b\n" + "and v26.16b, v20.16b, v1.16b\n" + "and v25.16b, v21.16b, v2.16b\n" + "and v24.16b, v22.16b, v3.16b\n" + "sshr v27.4s, v27.4s, #0x1f\n" + "sshr v26.4s, v26.4s, #0x1f\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sqadd v15.4s, v15.4s, v27.4s\n" + "sqadd v20.4s, v20.4s, v26.4s\n" + "sqadd v21.4s, v21.4s, v25.4s\n" + "sqadd v22.4s, v22.4s, v24.4s\n" + "and v27.16b, v16.16b, v0.16b\n" + "and v26.16b, v17.16b, v1.16b\n" + "and v25.16b, v18.16b, v2.16b\n" + "and v24.16b, v19.16b, v3.16b\n" + "sshr v27.4s, v27.4s, #0x1f\n" + "sshr v26.4s, v26.4s, #0x1f\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v27.4s\n" + "sqadd v17.4s, v17.4s, v26.4s\n" + "sqadd v18.4s, v18.4s, v25.4s\n" + "sqadd v19.4s, v19.4s, v24.4s\n" "106:" // Height 4: no shift correction - "add x25, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v26.4s }, [x20]\n" "srshl v23.4s, v23.4s, v0.4s\n" "srshl v12.4s, v12.4s, v1.4s\n" "srshl v13.4s, v13.4s, v2.4s\n" "srshl v14.4s, v14.4s, v3.4s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x25]\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v25.4s }, [x20]\n" "srshl v8.4s, v8.4s, v0.4s\n" "srshl v9.4s, v9.4s, v1.4s\n" - "add x25, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x25]\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v24.4s }, [x20]\n" "srshl v10.4s, v10.4s, v2.4s\n" "srshl v11.4s, v11.4s, v3.4s\n" "cmp x10, #0x10\n" @@ -1836,163 +1835,163 @@ void a64_hybrid_s8qs_mmla_6x16 ( "srshl v17.4s, v17.4s, v1.4s\n" "srshl v18.4s, v18.4s, v2.4s\n" "srshl v19.4s, v19.4s, v3.4s\n" - "add v23.4s, v23.4s, v4.4s\n" - "add v12.4s, v12.4s, v4.4s\n" - "add v13.4s, v13.4s, v4.4s\n" - "add v14.4s, v14.4s, v4.4s\n" - "add v8.4s, v8.4s, v4.4s\n" - "add v9.4s, v9.4s, v4.4s\n" - "add v10.4s, v10.4s, v4.4s\n" - "add v11.4s, v11.4s, v4.4s\n" - "add v15.4s, v15.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "smin v12.4s, v12.4s, v6.4s\n" - "smin v13.4s, v13.4s, v6.4s\n" - "smin v14.4s, v14.4s, v6.4s\n" - "smin v8.4s, v8.4s, v6.4s\n" - "smin v9.4s, v9.4s, v6.4s\n" - "smin v10.4s, v10.4s, v6.4s\n" - "smin v11.4s, v11.4s, v6.4s\n" - "smin v15.4s, v15.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" - "smax v12.4s, v12.4s, v5.4s\n" - "smax v13.4s, v13.4s, v5.4s\n" - "smax v14.4s, v14.4s, v5.4s\n" - "smax v8.4s, v8.4s, v5.4s\n" - "smax v9.4s, v9.4s, v5.4s\n" - "smax v10.4s, v10.4s, v5.4s\n" - "smax v11.4s, v11.4s, v5.4s\n" - "smax v15.4s, v15.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" + "add v23.4s, v23.4s, v26.4s\n" + "add v12.4s, v12.4s, v26.4s\n" + "add v13.4s, v13.4s, v26.4s\n" + "add v14.4s, v14.4s, v26.4s\n" + "add v8.4s, v8.4s, v26.4s\n" + "add v9.4s, v9.4s, v26.4s\n" + "add v10.4s, v10.4s, v26.4s\n" + "add v11.4s, v11.4s, v26.4s\n" + "add v15.4s, v15.4s, v26.4s\n" + "add v20.4s, v20.4s, v26.4s\n" + "add v21.4s, v21.4s, v26.4s\n" + "add v22.4s, v22.4s, v26.4s\n" + "add v16.4s, v16.4s, v26.4s\n" + "add v17.4s, v17.4s, v26.4s\n" + "add v18.4s, v18.4s, v26.4s\n" + "add v19.4s, v19.4s, v26.4s\n" + "smin v23.4s, v23.4s, v25.4s\n" + "smin v12.4s, v12.4s, v25.4s\n" + "smin v13.4s, v13.4s, v25.4s\n" + "smin v14.4s, v14.4s, v25.4s\n" + "smin v8.4s, v8.4s, v25.4s\n" + "smin v9.4s, v9.4s, v25.4s\n" + "smin v10.4s, v10.4s, v25.4s\n" + "smin v11.4s, v11.4s, v25.4s\n" + "smin v15.4s, v15.4s, v25.4s\n" + "smin v20.4s, v20.4s, v25.4s\n" + "smin v21.4s, v21.4s, v25.4s\n" + "smin v22.4s, v22.4s, v25.4s\n" + "smin v16.4s, v16.4s, v25.4s\n" + "smin v17.4s, v17.4s, v25.4s\n" + "smin v18.4s, v18.4s, v25.4s\n" + "smin v19.4s, v19.4s, v25.4s\n" + "smax v23.4s, v23.4s, v24.4s\n" + "smax v12.4s, v12.4s, v24.4s\n" + "smax v13.4s, v13.4s, v24.4s\n" + "smax v14.4s, v14.4s, v24.4s\n" + "smax v8.4s, v8.4s, v24.4s\n" + "smax v9.4s, v9.4s, v24.4s\n" + "smax v10.4s, v10.4s, v24.4s\n" + "smax v11.4s, v11.4s, v24.4s\n" + "smax v15.4s, v15.4s, v24.4s\n" + "smax v20.4s, v20.4s, v24.4s\n" + "smax v21.4s, v21.4s, v24.4s\n" + "smax v22.4s, v22.4s, v24.4s\n" + "smax v16.4s, v16.4s, v24.4s\n" + "smax v17.4s, v17.4s, v24.4s\n" + "smax v18.4s, v18.4s, v24.4s\n" + "smax v19.4s, v19.4s, v24.4s\n" "uzp1 v23.8h, v23.8h, v12.8h\n" - "uzp1 v12.8h, v13.8h, v14.8h\n" + "uzp1 v25.8h, v13.8h, v14.8h\n" "uzp1 v8.8h, v8.8h, v9.8h\n" - "uzp1 v9.8h, v10.8h, v11.8h\n" + "uzp1 v24.8h, v10.8h, v11.8h\n" "uzp1 v15.8h, v15.8h, v20.8h\n" "uzp1 v20.8h, v21.8h, v22.8h\n" "uzp1 v16.8h, v16.8h, v17.8h\n" "uzp1 v17.8h, v18.8h, v19.8h\n" - "uzp1 v23.16b, v23.16b, v12.16b\n" - "uzp1 v8.16b, v8.16b, v9.16b\n" + "uzp1 v23.16b, v23.16b, v25.16b\n" + "uzp1 v8.16b, v8.16b, v24.16b\n" "uzp1 v15.16b, v15.16b, v20.16b\n" "uzp1 v16.16b, v16.16b, v17.16b\n" "bge 115f\n" "tbz x10, #3, 110f\n" "str d23, [x11], #0x8\n" - "str d8, [x24], #0x8\n" - "str d15, [x23], #0x8\n" - "str d16, [x22], #0x8\n" + "str d8, [x25], #0x8\n" + "str d15, [x24], #0x8\n" + "str d16, [x23], #0x8\n" "tbz x10, #2, 108f\n" "st1 { v23.s }[2], [x11], #0x4\n" - "st1 { v8.s }[2], [x24], #0x4\n" - "st1 { v15.s }[2], [x23], #0x4\n" - "st1 { v16.s }[2], [x22], #0x4\n" + "st1 { v8.s }[2], [x25], #0x4\n" + "st1 { v15.s }[2], [x24], #0x4\n" + "st1 { v16.s }[2], [x23], #0x4\n" "tbz x10, #1, 107f\n" "st1 { v23.h }[6], [x11], #0x2\n" - "st1 { v8.h }[6], [x24], #0x2\n" - "st1 { v15.h }[6], [x23], #0x2\n" - "st1 { v16.h }[6], [x22], #0x2\n" + "st1 { v8.h }[6], [x25], #0x2\n" + "st1 { v15.h }[6], [x24], #0x2\n" + "st1 { v16.h }[6], [x23], #0x2\n" "tbz x10, #0, 114f\n" "st1 { v23.b }[14], [x11]\n" - "st1 { v8.b }[14], [x24]\n" - "st1 { v15.b }[14], [x23]\n" - "st1 { v16.b }[14], [x22]\n" + "st1 { v8.b }[14], [x25]\n" + "st1 { v15.b }[14], [x24]\n" + "st1 { v16.b }[14], [x23]\n" "b 114f\n" "107:" // Height 4: Partial direct writeback: partial_1_12 "tbz x10, #0, 114f\n" "st1 { v23.b }[12], [x11]\n" - "st1 { v8.b }[12], [x24]\n" - "st1 { v15.b }[12], [x23]\n" - "st1 { v16.b }[12], [x22]\n" + "st1 { v8.b }[12], [x25]\n" + "st1 { v15.b }[12], [x24]\n" + "st1 { v16.b }[12], [x23]\n" "b 114f\n" "108:" // Height 4: Partial direct writeback: partial_2_8 "tbz x10, #1, 109f\n" "st1 { v23.h }[4], [x11], #0x2\n" - "st1 { v8.h }[4], [x24], #0x2\n" - "st1 { v15.h }[4], [x23], #0x2\n" - "st1 { v16.h }[4], [x22], #0x2\n" + "st1 { v8.h }[4], [x25], #0x2\n" + "st1 { v15.h }[4], [x24], #0x2\n" + "st1 { v16.h }[4], [x23], #0x2\n" "tbz x10, #0, 114f\n" "st1 { v23.b }[10], [x11]\n" - "st1 { v8.b }[10], [x24]\n" - "st1 { v15.b }[10], [x23]\n" - "st1 { v16.b }[10], [x22]\n" + "st1 { v8.b }[10], [x25]\n" + "st1 { v15.b }[10], [x24]\n" + "st1 { v16.b }[10], [x23]\n" "b 114f\n" "109:" // Height 4: Partial direct writeback: partial_1_8 "tbz x10, #0, 114f\n" "st1 { v23.b }[8], [x11]\n" - "st1 { v8.b }[8], [x24]\n" - "st1 { v15.b }[8], [x23]\n" - "st1 { v16.b }[8], [x22]\n" + "st1 { v8.b }[8], [x25]\n" + "st1 { v15.b }[8], [x24]\n" + "st1 { v16.b }[8], [x23]\n" "b 114f\n" "110:" // Height 4: Partial direct writeback: partial_4_0 "tbz x10, #2, 112f\n" "str s23, [x11], #0x4\n" - "str s8, [x24], #0x4\n" - "str s15, [x23], #0x4\n" - "str s16, [x22], #0x4\n" + "str s8, [x25], #0x4\n" + "str s15, [x24], #0x4\n" + "str s16, [x23], #0x4\n" "tbz x10, #1, 111f\n" "st1 { v23.h }[2], [x11], #0x2\n" - "st1 { v8.h }[2], [x24], #0x2\n" - "st1 { v15.h }[2], [x23], #0x2\n" - "st1 { v16.h }[2], [x22], #0x2\n" + "st1 { v8.h }[2], [x25], #0x2\n" + "st1 { v15.h }[2], [x24], #0x2\n" + "st1 { v16.h }[2], [x23], #0x2\n" "tbz x10, #0, 114f\n" "st1 { v23.b }[6], [x11]\n" - "st1 { v8.b }[6], [x24]\n" - "st1 { v15.b }[6], [x23]\n" - "st1 { v16.b }[6], [x22]\n" + "st1 { v8.b }[6], [x25]\n" + "st1 { v15.b }[6], [x24]\n" + "st1 { v16.b }[6], [x23]\n" "b 114f\n" "111:" // Height 4: Partial direct writeback: partial_1_4 "tbz x10, #0, 114f\n" "st1 { v23.b }[4], [x11]\n" - "st1 { v8.b }[4], [x24]\n" - "st1 { v15.b }[4], [x23]\n" - "st1 { v16.b }[4], [x22]\n" + "st1 { v8.b }[4], [x25]\n" + "st1 { v15.b }[4], [x24]\n" + "st1 { v16.b }[4], [x23]\n" "b 114f\n" "112:" // Height 4: Partial direct writeback: partial_2_0 "tbz x10, #1, 113f\n" "str h23, [x11], #0x2\n" - "str h8, [x24], #0x2\n" - "str h15, [x23], #0x2\n" - "str h16, [x22], #0x2\n" + "str h8, [x25], #0x2\n" + "str h15, [x24], #0x2\n" + "str h16, [x23], #0x2\n" "tbz x10, #0, 114f\n" "st1 { v23.b }[2], [x11]\n" - "st1 { v8.b }[2], [x24]\n" - "st1 { v15.b }[2], [x23]\n" - "st1 { v16.b }[2], [x22]\n" + "st1 { v8.b }[2], [x25]\n" + "st1 { v15.b }[2], [x24]\n" + "st1 { v16.b }[2], [x23]\n" "b 114f\n" "113:" // Height 4: Partial direct writeback: partial_1_0 "str b23, [x11, #0x0]\n" - "str b8, [x24, #0x0]\n" - "str b15, [x23, #0x0]\n" - "str b16, [x22, #0x0]\n" + "str b8, [x25, #0x0]\n" + "str b15, [x24, #0x0]\n" + "str b16, [x23, #0x0]\n" "114:" // Height 4: Partial direct writeback: Done "b 116f\n" "115:" // Height 4: Full writeback "str q23, [x11, #0x0]\n" "add x11, x11, #0x10\n" - "str q8, [x24, #0x0]\n" - "str q15, [x23, #0x0]\n" - "str q16, [x22, #0x0]\n" + "str q8, [x25, #0x0]\n" + "str q15, [x24, #0x0]\n" + "str q16, [x23, #0x0]\n" "116:" // Height 4: Writeback done "subs x10, x10, #0x10\n" "bgt 89b\n" @@ -2034,15 +2033,15 @@ void a64_hybrid_s8qs_mmla_6x16 ( "120:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 121f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 122f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -2053,10 +2052,10 @@ void a64_hybrid_s8qs_mmla_6x16 ( "b 122f\n" "121:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" "122:" // Height 5: input setup done "cmp x27, #0x10\n" "blt 125f\n" @@ -2120,42 +2119,42 @@ void a64_hybrid_s8qs_mmla_6x16 ( ".inst 0x4e86a457 // smmla v23.4s, v2.16b, v6.16b\n" "ldr q2, [x25, #0x0]\n" ".inst 0x4e86a49f // smmla v31.4s, v4.16b, v6.16b\n" - "ldr q6, [x9, #0x90]\n" + "ldr q0, [x9, #0x90]\n" "ldr q4, [x23, #0x0]\n" ".inst 0x4e87a428 // smmla v8.4s, v1.16b, v7.16b\n" ".inst 0x4e87a470 // smmla v16.4s, v3.16b, v7.16b\n" ".inst 0x4e87a4b8 // smmla v24.4s, v5.16b, v7.16b\n" - "ldr q7, [x9, #0xa0]\n" - ".inst 0x4e86a42c // smmla v12.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a474 // smmla v20.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4bc // smmla v28.4s, v5.16b, v6.16b\n" - "ldr q6, [x9, #0xb0]\n" - ".inst 0x4e87a429 // smmla v9.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a471 // smmla v17.4s, v3.16b, v7.16b\n" - ".inst 0x4e87a4b9 // smmla v25.4s, v5.16b, v7.16b\n" - "ldr q7, [x9, #0xc0]\n" - ".inst 0x4e86a42d // smmla v13.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a475 // smmla v21.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4bd // smmla v29.4s, v5.16b, v6.16b\n" - "ldr q6, [x9, #0xd0]\n" - ".inst 0x4e87a42a // smmla v10.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a472 // smmla v18.4s, v3.16b, v7.16b\n" - ".inst 0x4e87a4ba // smmla v26.4s, v5.16b, v7.16b\n" - "ldr q7, [x9, #0xe0]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a476 // smmla v22.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4be // smmla v30.4s, v5.16b, v6.16b\n" - "ldr q6, [x9, #0xf0]\n" + "ldr q6, [x9, #0xa0]\n" + ".inst 0x4e80a42c // smmla v12.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a474 // smmla v20.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4bc // smmla v28.4s, v5.16b, v0.16b\n" + "ldr q0, [x9, #0xb0]\n" + ".inst 0x4e86a429 // smmla v9.4s, v1.16b, v6.16b\n" + ".inst 0x4e86a471 // smmla v17.4s, v3.16b, v6.16b\n" + ".inst 0x4e86a4b9 // smmla v25.4s, v5.16b, v6.16b\n" + "ldr q6, [x9, #0xc0]\n" + ".inst 0x4e80a42d // smmla v13.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a475 // smmla v21.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4bd // smmla v29.4s, v5.16b, v0.16b\n" + "ldr q0, [x9, #0xd0]\n" + ".inst 0x4e86a42a // smmla v10.4s, v1.16b, v6.16b\n" + ".inst 0x4e86a472 // smmla v18.4s, v3.16b, v6.16b\n" + ".inst 0x4e86a4ba // smmla v26.4s, v5.16b, v6.16b\n" + "ldr q6, [x9, #0xe0]\n" + ".inst 0x4e80a42e // smmla v14.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a476 // smmla v22.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4be // smmla v30.4s, v5.16b, v0.16b\n" + "ldr q0, [x9, #0xf0]\n" "add x9, x9, #0x100\n" - ".inst 0x4e87a42b // smmla v11.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a473 // smmla v19.4s, v3.16b, v7.16b\n" - ".inst 0x4e87a4bb // smmla v27.4s, v5.16b, v7.16b\n" + ".inst 0x4e86a42b // smmla v11.4s, v1.16b, v6.16b\n" + ".inst 0x4e86a473 // smmla v19.4s, v3.16b, v6.16b\n" + ".inst 0x4e86a4bb // smmla v27.4s, v5.16b, v6.16b\n" "ldr q7, [x9, #0x0]\n" - ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" + ".inst 0x4e80a42f // smmla v15.4s, v1.16b, v0.16b\n" "ldr q1, [x26, #0x0]\n" - ".inst 0x4e86a477 // smmla v23.4s, v3.16b, v6.16b\n" + ".inst 0x4e80a477 // smmla v23.4s, v3.16b, v0.16b\n" "ldr q3, [x24, #0x0]\n" - ".inst 0x4e86a4bf // smmla v31.4s, v5.16b, v6.16b\n" + ".inst 0x4e80a4bf // smmla v31.4s, v5.16b, v0.16b\n" "ldr q5, [x22, #0x0]\n" "bge 123b\n" "124:" // Height 5: Multiply loop: Single iteration only @@ -2208,86 +2207,86 @@ void a64_hybrid_s8qs_mmla_6x16 ( ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" ".inst 0x4e86a457 // smmla v23.4s, v2.16b, v6.16b\n" ".inst 0x4e86a49f // smmla v31.4s, v4.16b, v6.16b\n" - "ldr q6, [x9, #0x90]\n" + "ldr q0, [x9, #0x90]\n" ".inst 0x4e87a428 // smmla v8.4s, v1.16b, v7.16b\n" ".inst 0x4e87a470 // smmla v16.4s, v3.16b, v7.16b\n" ".inst 0x4e87a4b8 // smmla v24.4s, v5.16b, v7.16b\n" - "ldr q7, [x9, #0xa0]\n" - ".inst 0x4e86a42c // smmla v12.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a474 // smmla v20.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4bc // smmla v28.4s, v5.16b, v6.16b\n" - "ldr q6, [x9, #0xb0]\n" - ".inst 0x4e87a429 // smmla v9.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a471 // smmla v17.4s, v3.16b, v7.16b\n" - ".inst 0x4e87a4b9 // smmla v25.4s, v5.16b, v7.16b\n" - "ldr q7, [x9, #0xc0]\n" - ".inst 0x4e86a42d // smmla v13.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a475 // smmla v21.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4bd // smmla v29.4s, v5.16b, v6.16b\n" - "ldr q6, [x9, #0xd0]\n" - ".inst 0x4e87a42a // smmla v10.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a472 // smmla v18.4s, v3.16b, v7.16b\n" - ".inst 0x4e87a4ba // smmla v26.4s, v5.16b, v7.16b\n" - "ldr q7, [x9, #0xe0]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a476 // smmla v22.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4be // smmla v30.4s, v5.16b, v6.16b\n" - "ldr q6, [x9, #0xf0]\n" + "ldr q2, [x9, #0xa0]\n" + ".inst 0x4e80a42c // smmla v12.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a474 // smmla v20.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4bc // smmla v28.4s, v5.16b, v0.16b\n" + "ldr q0, [x9, #0xb0]\n" + ".inst 0x4e82a429 // smmla v9.4s, v1.16b, v2.16b\n" + ".inst 0x4e82a471 // smmla v17.4s, v3.16b, v2.16b\n" + ".inst 0x4e82a4b9 // smmla v25.4s, v5.16b, v2.16b\n" + "ldr q2, [x9, #0xc0]\n" + ".inst 0x4e80a42d // smmla v13.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a475 // smmla v21.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4bd // smmla v29.4s, v5.16b, v0.16b\n" + "ldr q0, [x9, #0xd0]\n" + ".inst 0x4e82a42a // smmla v10.4s, v1.16b, v2.16b\n" + ".inst 0x4e82a472 // smmla v18.4s, v3.16b, v2.16b\n" + ".inst 0x4e82a4ba // smmla v26.4s, v5.16b, v2.16b\n" + "ldr q2, [x9, #0xe0]\n" + ".inst 0x4e80a42e // smmla v14.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a476 // smmla v22.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4be // smmla v30.4s, v5.16b, v0.16b\n" + "ldr q0, [x9, #0xf0]\n" "add x9, x9, #0x100\n" - ".inst 0x4e87a42b // smmla v11.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a473 // smmla v19.4s, v3.16b, v7.16b\n" - ".inst 0x4e87a4bb // smmla v27.4s, v5.16b, v7.16b\n" - ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a477 // smmla v23.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4bf // smmla v31.4s, v5.16b, v6.16b\n" + ".inst 0x4e82a42b // smmla v11.4s, v1.16b, v2.16b\n" + ".inst 0x4e82a473 // smmla v19.4s, v3.16b, v2.16b\n" + ".inst 0x4e82a4bb // smmla v27.4s, v5.16b, v2.16b\n" + ".inst 0x4e80a42f // smmla v15.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a477 // smmla v23.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4bf // smmla v31.4s, v5.16b, v0.16b\n" "125:" // Height 5: Multiply loop: Main loop skip "cbz x27, 132f\n" "cmp x27, #0x8\n" "blt 127f\n" "126:" // Height 5: Multiply loop: Odd block loop "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "ldr d3, [x24], #0x8\n" - "ldr d4, [x23], #0x8\n" - "trn1 v2.2d, v3.2d, v4.2d\n" + "ldr d0, [x25], #0x8\n" + "trn1 v4.2d, v1.2d, v0.2d\n" + "ldr d1, [x24], #0x8\n" + "ldr d0, [x23], #0x8\n" + "trn1 v3.2d, v1.2d, v0.2d\n" "sub x27, x27, #0x8\n" - "ldr d5, [x22], #0x8\n" - "ldr q6, [x9, #0x0]\n" - "trn1 v4.2d, v5.2d, v7.2d\n" - ".inst 0x4e86a408 // smmla v8.4s, v0.16b, v6.16b\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x4e86a450 // smmla v16.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a498 // smmla v24.4s, v4.16b, v6.16b\n" - "ldr q6, [x9, #0x20]\n" - ".inst 0x4e87a40c // smmla v12.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a454 // smmla v20.4s, v2.16b, v7.16b\n" + "ldr d0, [x22], #0x8\n" + "ldr q1, [x9, #0x0]\n" + "trn1 v2.2d, v0.2d, v2.2d\n" + ".inst 0x4e81a488 // smmla v8.4s, v4.16b, v1.16b\n" + "ldr q0, [x9, #0x10]\n" + ".inst 0x4e81a470 // smmla v16.4s, v3.16b, v1.16b\n" + ".inst 0x4e81a458 // smmla v24.4s, v2.16b, v1.16b\n" + "ldr q1, [x9, #0x20]\n" + ".inst 0x4e80a48c // smmla v12.4s, v4.16b, v0.16b\n" + ".inst 0x4e80a474 // smmla v20.4s, v3.16b, v0.16b\n" "cmp x27, #0x8\n" - ".inst 0x4e87a49c // smmla v28.4s, v4.16b, v7.16b\n" - "ldr q7, [x9, #0x30]\n" - ".inst 0x4e86a409 // smmla v9.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a451 // smmla v17.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a499 // smmla v25.4s, v4.16b, v6.16b\n" - "ldr q6, [x9, #0x40]\n" - ".inst 0x4e87a40d // smmla v13.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a455 // smmla v21.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a49d // smmla v29.4s, v4.16b, v7.16b\n" - "ldr q7, [x9, #0x50]\n" - ".inst 0x4e86a40a // smmla v10.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a452 // smmla v18.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49a // smmla v26.4s, v4.16b, v6.16b\n" - "ldr q6, [x9, #0x60]\n" - ".inst 0x4e87a40e // smmla v14.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a456 // smmla v22.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a49e // smmla v30.4s, v4.16b, v7.16b\n" - "ldr q7, [x9, #0x70]\n" - ".inst 0x4e86a40b // smmla v11.4s, v0.16b, v6.16b\n" + ".inst 0x4e80a45c // smmla v28.4s, v2.16b, v0.16b\n" + "ldr q0, [x9, #0x30]\n" + ".inst 0x4e81a489 // smmla v9.4s, v4.16b, v1.16b\n" + ".inst 0x4e81a471 // smmla v17.4s, v3.16b, v1.16b\n" + ".inst 0x4e81a459 // smmla v25.4s, v2.16b, v1.16b\n" + "ldr q1, [x9, #0x40]\n" + ".inst 0x4e80a48d // smmla v13.4s, v4.16b, v0.16b\n" + ".inst 0x4e80a475 // smmla v21.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a45d // smmla v29.4s, v2.16b, v0.16b\n" + "ldr q0, [x9, #0x50]\n" + ".inst 0x4e81a48a // smmla v10.4s, v4.16b, v1.16b\n" + ".inst 0x4e81a472 // smmla v18.4s, v3.16b, v1.16b\n" + ".inst 0x4e81a45a // smmla v26.4s, v2.16b, v1.16b\n" + "ldr q1, [x9, #0x60]\n" + ".inst 0x4e80a48e // smmla v14.4s, v4.16b, v0.16b\n" + ".inst 0x4e80a476 // smmla v22.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a45e // smmla v30.4s, v2.16b, v0.16b\n" + "ldr q0, [x9, #0x70]\n" + ".inst 0x4e81a48b // smmla v11.4s, v4.16b, v1.16b\n" "add x9, x9, #0x80\n" - ".inst 0x4e86a453 // smmla v19.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49b // smmla v27.4s, v4.16b, v6.16b\n" - ".inst 0x4e87a40f // smmla v15.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a457 // smmla v23.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a49f // smmla v31.4s, v4.16b, v7.16b\n" + ".inst 0x4e81a473 // smmla v19.4s, v3.16b, v1.16b\n" + ".inst 0x4e81a45b // smmla v27.4s, v2.16b, v1.16b\n" + ".inst 0x4e80a48f // smmla v15.4s, v4.16b, v0.16b\n" + ".inst 0x4e80a477 // smmla v23.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a45f // smmla v31.4s, v2.16b, v0.16b\n" "bge 126b\n" "127:" // Height 5: Multiply loop: Skip odd blocks "cbz x27, 132f\n" @@ -2340,74 +2339,74 @@ void a64_hybrid_s8qs_mmla_6x16 ( "ldr b5, [x22, #0x0]\n" "131:" // Height 5: Multiply loop: Ragged operand read: Done "ldr q7, [x9, #0x0]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - "trn1 v4.2d, v5.2d, v6.2d\n" - "ldr q6, [x9, #0x10]\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a450 // smmla v16.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a498 // smmla v24.4s, v4.16b, v7.16b\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49c // smmla v28.4s, v4.16b, v6.16b\n" - "ldr q6, [x9, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a451 // smmla v17.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a499 // smmla v25.4s, v4.16b, v7.16b\n" - "ldr q7, [x9, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a455 // smmla v21.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49d // smmla v29.4s, v4.16b, v6.16b\n" - "ldr q6, [x9, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a452 // smmla v18.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a49a // smmla v26.4s, v4.16b, v7.16b\n" - "ldr q7, [x9, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49e // smmla v30.4s, v4.16b, v6.16b\n" - "ldr q6, [x9, #0x70]\n" + "trn1 v6.2d, v1.2d, v2.2d\n" + "trn1 v3.2d, v3.2d, v4.2d\n" + "trn1 v2.2d, v5.2d, v0.2d\n" + "ldr q0, [x9, #0x10]\n" + ".inst 0x4e87a4c8 // smmla v8.4s, v6.16b, v7.16b\n" + ".inst 0x4e87a470 // smmla v16.4s, v3.16b, v7.16b\n" + ".inst 0x4e87a458 // smmla v24.4s, v2.16b, v7.16b\n" + "ldr q1, [x9, #0x20]\n" + ".inst 0x4e80a4cc // smmla v12.4s, v6.16b, v0.16b\n" + ".inst 0x4e80a474 // smmla v20.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a45c // smmla v28.4s, v2.16b, v0.16b\n" + "ldr q0, [x9, #0x30]\n" + ".inst 0x4e81a4c9 // smmla v9.4s, v6.16b, v1.16b\n" + ".inst 0x4e81a471 // smmla v17.4s, v3.16b, v1.16b\n" + ".inst 0x4e81a459 // smmla v25.4s, v2.16b, v1.16b\n" + "ldr q1, [x9, #0x40]\n" + ".inst 0x4e80a4cd // smmla v13.4s, v6.16b, v0.16b\n" + ".inst 0x4e80a475 // smmla v21.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a45d // smmla v29.4s, v2.16b, v0.16b\n" + "ldr q0, [x9, #0x50]\n" + ".inst 0x4e81a4ca // smmla v10.4s, v6.16b, v1.16b\n" + ".inst 0x4e81a472 // smmla v18.4s, v3.16b, v1.16b\n" + ".inst 0x4e81a45a // smmla v26.4s, v2.16b, v1.16b\n" + "ldr q1, [x9, #0x60]\n" + ".inst 0x4e80a4ce // smmla v14.4s, v6.16b, v0.16b\n" + ".inst 0x4e80a476 // smmla v22.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a45e // smmla v30.4s, v2.16b, v0.16b\n" + "ldr q0, [x9, #0x70]\n" "add x9, x9, #0x80\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a453 // smmla v19.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a49b // smmla v27.4s, v4.16b, v7.16b\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a457 // smmla v23.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49f // smmla v31.4s, v4.16b, v6.16b\n" + ".inst 0x4e81a4cb // smmla v11.4s, v6.16b, v1.16b\n" + ".inst 0x4e81a473 // smmla v19.4s, v3.16b, v1.16b\n" + ".inst 0x4e81a45b // smmla v27.4s, v2.16b, v1.16b\n" + ".inst 0x4e80a4cf // smmla v15.4s, v6.16b, v0.16b\n" + ".inst 0x4e80a477 // smmla v23.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a45f // smmla v31.4s, v2.16b, v0.16b\n" "132:" // Height 5: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 120b\n" - "ldr q0, [x14, #0x0]\n" - "ldr q1, [x14, #0x10]\n" - "uzp1 v7.2d, v8.2d, v12.2d\n" + "ldr q4, [x14, #0x0]\n" + "ldr q3, [x14, #0x10]\n" + "uzp1 v2.2d, v8.2d, v12.2d\n" "uzp2 v8.2d, v8.2d, v12.2d\n" - "ldr q2, [x14, #0x20]\n" - "ldr q3, [x14, #0x30]\n" + "ldr q1, [x14, #0x20]\n" + "ldr q0, [x14, #0x30]\n" "uzp1 v12.2d, v9.2d, v13.2d\n" "uzp2 v9.2d, v9.2d, v13.2d\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x11, x20\n" + "add x25, x11, x20\n" "uzp1 v13.2d, v10.2d, v14.2d\n" "uzp2 v10.2d, v10.2d, v14.2d\n" "uzp1 v14.2d, v11.2d, v15.2d\n" "uzp2 v11.2d, v11.2d, v15.2d\n" + "add x24, x25, x20\n" "add x23, x24, x20\n" - "add x22, x23, x20\n" "uzp1 v15.2d, v16.2d, v20.2d\n" "uzp2 v16.2d, v16.2d, v20.2d\n" - "add x21, x22, x20\n" + "add x22, x23, x20\n" "prfm pstl1keep, [x11, #0x0]\n" "uzp1 v20.2d, v17.2d, v21.2d\n" "uzp2 v17.2d, v17.2d, v21.2d\n" + "prfm pstl1keep, [x25, #0x0]\n" "prfm pstl1keep, [x24, #0x0]\n" - "prfm pstl1keep, [x23, #0x0]\n" "uzp1 v21.2d, v18.2d, v22.2d\n" "uzp2 v18.2d, v18.2d, v22.2d\n" + "prfm pstl1keep, [x23, #0x0]\n" "prfm pstl1keep, [x22, #0x0]\n" - "prfm pstl1keep, [x21, #0x0]\n" "uzp1 v22.2d, v19.2d, v23.2d\n" "uzp2 v19.2d, v19.2d, v23.2d\n" "add x14, x14, #0x40\n" @@ -2415,27 +2414,27 @@ void a64_hybrid_s8qs_mmla_6x16 ( "uzp1 v25.2d, v25.2d, v29.2d\n" "uzp1 v26.2d, v26.2d, v30.2d\n" "uzp1 v27.2d, v27.2d, v31.2d\n" - "mov v31.16b, v7.16b\n" - "add v31.4s, v31.4s, v0.4s\n" - "add v12.4s, v12.4s, v1.4s\n" - "add v13.4s, v13.4s, v2.4s\n" - "add v14.4s, v14.4s, v3.4s\n" - "add v8.4s, v8.4s, v0.4s\n" - "add v9.4s, v9.4s, v1.4s\n" - "add v10.4s, v10.4s, v2.4s\n" - "add v11.4s, v11.4s, v3.4s\n" - "add v15.4s, v15.4s, v0.4s\n" - "add v20.4s, v20.4s, v1.4s\n" - "add v21.4s, v21.4s, v2.4s\n" - "add v22.4s, v22.4s, v3.4s\n" - "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" - "add v24.4s, v24.4s, v0.4s\n" - "add v25.4s, v25.4s, v1.4s\n" - "add v26.4s, v26.4s, v2.4s\n" - "add v27.4s, v27.4s, v3.4s\n" + "mov v31.16b, v2.16b\n" + "add v31.4s, v31.4s, v4.4s\n" + "add v12.4s, v12.4s, v3.4s\n" + "add v13.4s, v13.4s, v1.4s\n" + "add v14.4s, v14.4s, v0.4s\n" + "add v8.4s, v8.4s, v4.4s\n" + "add v9.4s, v9.4s, v3.4s\n" + "add v10.4s, v10.4s, v1.4s\n" + "add v11.4s, v11.4s, v0.4s\n" + "add v15.4s, v15.4s, v4.4s\n" + "add v20.4s, v20.4s, v3.4s\n" + "add v21.4s, v21.4s, v1.4s\n" + "add v22.4s, v22.4s, v0.4s\n" + "add v16.4s, v16.4s, v4.4s\n" + "add v17.4s, v17.4s, v3.4s\n" + "add v18.4s, v18.4s, v1.4s\n" + "add v19.4s, v19.4s, v0.4s\n" + "add v24.4s, v24.4s, v4.4s\n" + "add v25.4s, v25.4s, v3.4s\n" + "add v26.4s, v26.4s, v1.4s\n" + "add v27.4s, v27.4s, v0.4s\n" "tbz %x[flags], #4, 133f\n" "ldr q0, [x12, #0x0]\n" "ldr q4, [x13, #0x0]\n" @@ -2449,10 +2448,10 @@ void a64_hybrid_s8qs_mmla_6x16 ( "add x13, x13, #0x40\n" "b 134f\n" "133:" // Height 5: per layer parameters - "add x25, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x25]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v4.4s }, [x20]\n" "mov v1.16b, v0.16b\n" "mov v5.16b, v4.16b\n" "mov v2.16b, v0.16b\n" @@ -2481,79 +2480,79 @@ void a64_hybrid_s8qs_mmla_6x16 ( "sqrdmulh v26.4s, v26.4s, v6.4s\n" "sqrdmulh v27.4s, v27.4s, v7.4s\n" "tbz %x[flags], #5, 135f\n" - "and v4.16b, v31.16b, v0.16b\n" - "and v5.16b, v12.16b, v1.16b\n" - "and v6.16b, v13.16b, v2.16b\n" - "and v7.16b, v14.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v31.4s, v31.4s, v4.4s\n" - "sqadd v12.4s, v12.4s, v5.4s\n" - "sqadd v13.4s, v13.4s, v6.4s\n" - "sqadd v14.4s, v14.4s, v7.4s\n" - "and v4.16b, v8.16b, v0.16b\n" - "and v5.16b, v9.16b, v1.16b\n" - "and v6.16b, v10.16b, v2.16b\n" - "and v7.16b, v11.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v4.4s\n" - "sqadd v9.4s, v9.4s, v5.4s\n" - "sqadd v10.4s, v10.4s, v6.4s\n" - "sqadd v11.4s, v11.4s, v7.4s\n" - "and v4.16b, v15.16b, v0.16b\n" - "and v5.16b, v20.16b, v1.16b\n" - "and v6.16b, v21.16b, v2.16b\n" - "and v7.16b, v22.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v15.4s, v15.4s, v4.4s\n" - "sqadd v20.4s, v20.4s, v5.4s\n" - "sqadd v21.4s, v21.4s, v6.4s\n" - "sqadd v22.4s, v22.4s, v7.4s\n" - "and v4.16b, v16.16b, v0.16b\n" - "and v5.16b, v17.16b, v1.16b\n" - "and v6.16b, v18.16b, v2.16b\n" - "and v7.16b, v19.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" - "and v4.16b, v24.16b, v0.16b\n" - "and v5.16b, v25.16b, v1.16b\n" - "and v6.16b, v26.16b, v2.16b\n" - "and v7.16b, v27.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v24.4s, v24.4s, v4.4s\n" - "sqadd v25.4s, v25.4s, v5.4s\n" - "sqadd v26.4s, v26.4s, v6.4s\n" - "sqadd v27.4s, v27.4s, v7.4s\n" + "and v30.16b, v31.16b, v0.16b\n" + "and v29.16b, v12.16b, v1.16b\n" + "and v28.16b, v13.16b, v2.16b\n" + "and v23.16b, v14.16b, v3.16b\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "sqadd v31.4s, v31.4s, v30.4s\n" + "sqadd v12.4s, v12.4s, v29.4s\n" + "sqadd v13.4s, v13.4s, v28.4s\n" + "sqadd v14.4s, v14.4s, v23.4s\n" + "and v30.16b, v8.16b, v0.16b\n" + "and v29.16b, v9.16b, v1.16b\n" + "and v28.16b, v10.16b, v2.16b\n" + "and v23.16b, v11.16b, v3.16b\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "sqadd v8.4s, v8.4s, v30.4s\n" + "sqadd v9.4s, v9.4s, v29.4s\n" + "sqadd v10.4s, v10.4s, v28.4s\n" + "sqadd v11.4s, v11.4s, v23.4s\n" + "and v30.16b, v15.16b, v0.16b\n" + "and v29.16b, v20.16b, v1.16b\n" + "and v28.16b, v21.16b, v2.16b\n" + "and v23.16b, v22.16b, v3.16b\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "sqadd v15.4s, v15.4s, v30.4s\n" + "sqadd v20.4s, v20.4s, v29.4s\n" + "sqadd v21.4s, v21.4s, v28.4s\n" + "sqadd v22.4s, v22.4s, v23.4s\n" + "and v30.16b, v16.16b, v0.16b\n" + "and v29.16b, v17.16b, v1.16b\n" + "and v28.16b, v18.16b, v2.16b\n" + "and v23.16b, v19.16b, v3.16b\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v30.4s\n" + "sqadd v17.4s, v17.4s, v29.4s\n" + "sqadd v18.4s, v18.4s, v28.4s\n" + "sqadd v19.4s, v19.4s, v23.4s\n" + "and v30.16b, v24.16b, v0.16b\n" + "and v29.16b, v25.16b, v1.16b\n" + "and v28.16b, v26.16b, v2.16b\n" + "and v23.16b, v27.16b, v3.16b\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "sqadd v24.4s, v24.4s, v30.4s\n" + "sqadd v25.4s, v25.4s, v29.4s\n" + "sqadd v26.4s, v26.4s, v28.4s\n" + "sqadd v27.4s, v27.4s, v23.4s\n" "135:" // Height 5: no shift correction - "add x25, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v29.4s }, [x20]\n" "srshl v31.4s, v31.4s, v0.4s\n" "srshl v12.4s, v12.4s, v1.4s\n" "srshl v13.4s, v13.4s, v2.4s\n" "srshl v14.4s, v14.4s, v3.4s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x25]\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v28.4s }, [x20]\n" "srshl v8.4s, v8.4s, v0.4s\n" "srshl v9.4s, v9.4s, v1.4s\n" - "add x25, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x25]\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v23.4s }, [x20]\n" "srshl v10.4s, v10.4s, v2.4s\n" "srshl v11.4s, v11.4s, v3.4s\n" "cmp x10, #0x10\n" @@ -2569,194 +2568,194 @@ void a64_hybrid_s8qs_mmla_6x16 ( "srshl v25.4s, v25.4s, v1.4s\n" "srshl v26.4s, v26.4s, v2.4s\n" "srshl v27.4s, v27.4s, v3.4s\n" - "add v31.4s, v31.4s, v4.4s\n" - "add v12.4s, v12.4s, v4.4s\n" - "add v13.4s, v13.4s, v4.4s\n" - "add v14.4s, v14.4s, v4.4s\n" - "add v8.4s, v8.4s, v4.4s\n" - "add v9.4s, v9.4s, v4.4s\n" - "add v10.4s, v10.4s, v4.4s\n" - "add v11.4s, v11.4s, v4.4s\n" - "add v15.4s, v15.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add v24.4s, v24.4s, v4.4s\n" - "add v25.4s, v25.4s, v4.4s\n" - "add v26.4s, v26.4s, v4.4s\n" - "add v27.4s, v27.4s, v4.4s\n" - "smin v31.4s, v31.4s, v6.4s\n" - "smin v12.4s, v12.4s, v6.4s\n" - "smin v13.4s, v13.4s, v6.4s\n" - "smin v14.4s, v14.4s, v6.4s\n" - "smin v8.4s, v8.4s, v6.4s\n" - "smin v9.4s, v9.4s, v6.4s\n" - "smin v10.4s, v10.4s, v6.4s\n" - "smin v11.4s, v11.4s, v6.4s\n" - "smin v15.4s, v15.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smin v24.4s, v24.4s, v6.4s\n" - "smin v25.4s, v25.4s, v6.4s\n" - "smin v26.4s, v26.4s, v6.4s\n" - "smin v27.4s, v27.4s, v6.4s\n" - "smax v31.4s, v31.4s, v5.4s\n" - "smax v12.4s, v12.4s, v5.4s\n" - "smax v13.4s, v13.4s, v5.4s\n" - "smax v14.4s, v14.4s, v5.4s\n" - "smax v8.4s, v8.4s, v5.4s\n" - "smax v9.4s, v9.4s, v5.4s\n" - "smax v10.4s, v10.4s, v5.4s\n" - "smax v11.4s, v11.4s, v5.4s\n" - "smax v15.4s, v15.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" - "smax v24.4s, v24.4s, v5.4s\n" - "smax v25.4s, v25.4s, v5.4s\n" - "smax v26.4s, v26.4s, v5.4s\n" - "smax v27.4s, v27.4s, v5.4s\n" + "add v31.4s, v31.4s, v29.4s\n" + "add v12.4s, v12.4s, v29.4s\n" + "add v13.4s, v13.4s, v29.4s\n" + "add v14.4s, v14.4s, v29.4s\n" + "add v8.4s, v8.4s, v29.4s\n" + "add v9.4s, v9.4s, v29.4s\n" + "add v10.4s, v10.4s, v29.4s\n" + "add v11.4s, v11.4s, v29.4s\n" + "add v15.4s, v15.4s, v29.4s\n" + "add v20.4s, v20.4s, v29.4s\n" + "add v21.4s, v21.4s, v29.4s\n" + "add v22.4s, v22.4s, v29.4s\n" + "add v16.4s, v16.4s, v29.4s\n" + "add v17.4s, v17.4s, v29.4s\n" + "add v18.4s, v18.4s, v29.4s\n" + "add v19.4s, v19.4s, v29.4s\n" + "add v24.4s, v24.4s, v29.4s\n" + "add v25.4s, v25.4s, v29.4s\n" + "add v26.4s, v26.4s, v29.4s\n" + "add v27.4s, v27.4s, v29.4s\n" + "smin v31.4s, v31.4s, v28.4s\n" + "smin v12.4s, v12.4s, v28.4s\n" + "smin v13.4s, v13.4s, v28.4s\n" + "smin v14.4s, v14.4s, v28.4s\n" + "smin v8.4s, v8.4s, v28.4s\n" + "smin v9.4s, v9.4s, v28.4s\n" + "smin v10.4s, v10.4s, v28.4s\n" + "smin v11.4s, v11.4s, v28.4s\n" + "smin v15.4s, v15.4s, v28.4s\n" + "smin v20.4s, v20.4s, v28.4s\n" + "smin v21.4s, v21.4s, v28.4s\n" + "smin v22.4s, v22.4s, v28.4s\n" + "smin v16.4s, v16.4s, v28.4s\n" + "smin v17.4s, v17.4s, v28.4s\n" + "smin v18.4s, v18.4s, v28.4s\n" + "smin v19.4s, v19.4s, v28.4s\n" + "smin v24.4s, v24.4s, v28.4s\n" + "smin v25.4s, v25.4s, v28.4s\n" + "smin v26.4s, v26.4s, v28.4s\n" + "smin v27.4s, v27.4s, v28.4s\n" + "smax v31.4s, v31.4s, v23.4s\n" + "smax v12.4s, v12.4s, v23.4s\n" + "smax v13.4s, v13.4s, v23.4s\n" + "smax v14.4s, v14.4s, v23.4s\n" + "smax v8.4s, v8.4s, v23.4s\n" + "smax v9.4s, v9.4s, v23.4s\n" + "smax v10.4s, v10.4s, v23.4s\n" + "smax v11.4s, v11.4s, v23.4s\n" + "smax v15.4s, v15.4s, v23.4s\n" + "smax v20.4s, v20.4s, v23.4s\n" + "smax v21.4s, v21.4s, v23.4s\n" + "smax v22.4s, v22.4s, v23.4s\n" + "smax v16.4s, v16.4s, v23.4s\n" + "smax v17.4s, v17.4s, v23.4s\n" + "smax v18.4s, v18.4s, v23.4s\n" + "smax v19.4s, v19.4s, v23.4s\n" + "smax v24.4s, v24.4s, v23.4s\n" + "smax v25.4s, v25.4s, v23.4s\n" + "smax v26.4s, v26.4s, v23.4s\n" + "smax v27.4s, v27.4s, v23.4s\n" "uzp1 v31.8h, v31.8h, v12.8h\n" - "uzp1 v12.8h, v13.8h, v14.8h\n" + "uzp1 v28.8h, v13.8h, v14.8h\n" "uzp1 v8.8h, v8.8h, v9.8h\n" - "uzp1 v9.8h, v10.8h, v11.8h\n" + "uzp1 v23.8h, v10.8h, v11.8h\n" "uzp1 v15.8h, v15.8h, v20.8h\n" "uzp1 v20.8h, v21.8h, v22.8h\n" "uzp1 v16.8h, v16.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v18.8h, v18.8h, v19.8h\n" "uzp1 v24.8h, v24.8h, v25.8h\n" - "uzp1 v25.8h, v26.8h, v27.8h\n" - "uzp1 v31.16b, v31.16b, v12.16b\n" - "uzp1 v8.16b, v8.16b, v9.16b\n" + "uzp1 v17.8h, v26.8h, v27.8h\n" + "uzp1 v31.16b, v31.16b, v28.16b\n" + "uzp1 v8.16b, v8.16b, v23.16b\n" "uzp1 v15.16b, v15.16b, v20.16b\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" - "uzp1 v24.16b, v24.16b, v25.16b\n" + "uzp1 v16.16b, v16.16b, v18.16b\n" + "uzp1 v24.16b, v24.16b, v17.16b\n" "bge 144f\n" "tbz x10, #3, 139f\n" "str d31, [x11], #0x8\n" - "str d8, [x24], #0x8\n" - "str d15, [x23], #0x8\n" - "str d16, [x22], #0x8\n" - "str d24, [x21], #0x8\n" + "str d8, [x25], #0x8\n" + "str d15, [x24], #0x8\n" + "str d16, [x23], #0x8\n" + "str d24, [x22], #0x8\n" "tbz x10, #2, 137f\n" "st1 { v31.s }[2], [x11], #0x4\n" - "st1 { v8.s }[2], [x24], #0x4\n" - "st1 { v15.s }[2], [x23], #0x4\n" - "st1 { v16.s }[2], [x22], #0x4\n" - "st1 { v24.s }[2], [x21], #0x4\n" + "st1 { v8.s }[2], [x25], #0x4\n" + "st1 { v15.s }[2], [x24], #0x4\n" + "st1 { v16.s }[2], [x23], #0x4\n" + "st1 { v24.s }[2], [x22], #0x4\n" "tbz x10, #1, 136f\n" "st1 { v31.h }[6], [x11], #0x2\n" - "st1 { v8.h }[6], [x24], #0x2\n" - "st1 { v15.h }[6], [x23], #0x2\n" - "st1 { v16.h }[6], [x22], #0x2\n" - "st1 { v24.h }[6], [x21], #0x2\n" + "st1 { v8.h }[6], [x25], #0x2\n" + "st1 { v15.h }[6], [x24], #0x2\n" + "st1 { v16.h }[6], [x23], #0x2\n" + "st1 { v24.h }[6], [x22], #0x2\n" "tbz x10, #0, 143f\n" "st1 { v31.b }[14], [x11]\n" - "st1 { v8.b }[14], [x24]\n" - "st1 { v15.b }[14], [x23]\n" - "st1 { v16.b }[14], [x22]\n" - "st1 { v24.b }[14], [x21]\n" + "st1 { v8.b }[14], [x25]\n" + "st1 { v15.b }[14], [x24]\n" + "st1 { v16.b }[14], [x23]\n" + "st1 { v24.b }[14], [x22]\n" "b 143f\n" "136:" // Height 5: Partial direct writeback: partial_1_12 "tbz x10, #0, 143f\n" "st1 { v31.b }[12], [x11]\n" - "st1 { v8.b }[12], [x24]\n" - "st1 { v15.b }[12], [x23]\n" - "st1 { v16.b }[12], [x22]\n" - "st1 { v24.b }[12], [x21]\n" + "st1 { v8.b }[12], [x25]\n" + "st1 { v15.b }[12], [x24]\n" + "st1 { v16.b }[12], [x23]\n" + "st1 { v24.b }[12], [x22]\n" "b 143f\n" "137:" // Height 5: Partial direct writeback: partial_2_8 "tbz x10, #1, 138f\n" "st1 { v31.h }[4], [x11], #0x2\n" - "st1 { v8.h }[4], [x24], #0x2\n" - "st1 { v15.h }[4], [x23], #0x2\n" - "st1 { v16.h }[4], [x22], #0x2\n" - "st1 { v24.h }[4], [x21], #0x2\n" + "st1 { v8.h }[4], [x25], #0x2\n" + "st1 { v15.h }[4], [x24], #0x2\n" + "st1 { v16.h }[4], [x23], #0x2\n" + "st1 { v24.h }[4], [x22], #0x2\n" "tbz x10, #0, 143f\n" "st1 { v31.b }[10], [x11]\n" - "st1 { v8.b }[10], [x24]\n" - "st1 { v15.b }[10], [x23]\n" - "st1 { v16.b }[10], [x22]\n" - "st1 { v24.b }[10], [x21]\n" + "st1 { v8.b }[10], [x25]\n" + "st1 { v15.b }[10], [x24]\n" + "st1 { v16.b }[10], [x23]\n" + "st1 { v24.b }[10], [x22]\n" "b 143f\n" "138:" // Height 5: Partial direct writeback: partial_1_8 "tbz x10, #0, 143f\n" "st1 { v31.b }[8], [x11]\n" - "st1 { v8.b }[8], [x24]\n" - "st1 { v15.b }[8], [x23]\n" - "st1 { v16.b }[8], [x22]\n" - "st1 { v24.b }[8], [x21]\n" + "st1 { v8.b }[8], [x25]\n" + "st1 { v15.b }[8], [x24]\n" + "st1 { v16.b }[8], [x23]\n" + "st1 { v24.b }[8], [x22]\n" "b 143f\n" "139:" // Height 5: Partial direct writeback: partial_4_0 "tbz x10, #2, 141f\n" "str s31, [x11], #0x4\n" - "str s8, [x24], #0x4\n" - "str s15, [x23], #0x4\n" - "str s16, [x22], #0x4\n" - "str s24, [x21], #0x4\n" + "str s8, [x25], #0x4\n" + "str s15, [x24], #0x4\n" + "str s16, [x23], #0x4\n" + "str s24, [x22], #0x4\n" "tbz x10, #1, 140f\n" "st1 { v31.h }[2], [x11], #0x2\n" - "st1 { v8.h }[2], [x24], #0x2\n" - "st1 { v15.h }[2], [x23], #0x2\n" - "st1 { v16.h }[2], [x22], #0x2\n" - "st1 { v24.h }[2], [x21], #0x2\n" + "st1 { v8.h }[2], [x25], #0x2\n" + "st1 { v15.h }[2], [x24], #0x2\n" + "st1 { v16.h }[2], [x23], #0x2\n" + "st1 { v24.h }[2], [x22], #0x2\n" "tbz x10, #0, 143f\n" "st1 { v31.b }[6], [x11]\n" - "st1 { v8.b }[6], [x24]\n" - "st1 { v15.b }[6], [x23]\n" - "st1 { v16.b }[6], [x22]\n" - "st1 { v24.b }[6], [x21]\n" + "st1 { v8.b }[6], [x25]\n" + "st1 { v15.b }[6], [x24]\n" + "st1 { v16.b }[6], [x23]\n" + "st1 { v24.b }[6], [x22]\n" "b 143f\n" "140:" // Height 5: Partial direct writeback: partial_1_4 "tbz x10, #0, 143f\n" "st1 { v31.b }[4], [x11]\n" - "st1 { v8.b }[4], [x24]\n" - "st1 { v15.b }[4], [x23]\n" - "st1 { v16.b }[4], [x22]\n" - "st1 { v24.b }[4], [x21]\n" + "st1 { v8.b }[4], [x25]\n" + "st1 { v15.b }[4], [x24]\n" + "st1 { v16.b }[4], [x23]\n" + "st1 { v24.b }[4], [x22]\n" "b 143f\n" "141:" // Height 5: Partial direct writeback: partial_2_0 "tbz x10, #1, 142f\n" "str h31, [x11], #0x2\n" - "str h8, [x24], #0x2\n" - "str h15, [x23], #0x2\n" - "str h16, [x22], #0x2\n" - "str h24, [x21], #0x2\n" + "str h8, [x25], #0x2\n" + "str h15, [x24], #0x2\n" + "str h16, [x23], #0x2\n" + "str h24, [x22], #0x2\n" "tbz x10, #0, 143f\n" "st1 { v31.b }[2], [x11]\n" - "st1 { v8.b }[2], [x24]\n" - "st1 { v15.b }[2], [x23]\n" - "st1 { v16.b }[2], [x22]\n" - "st1 { v24.b }[2], [x21]\n" + "st1 { v8.b }[2], [x25]\n" + "st1 { v15.b }[2], [x24]\n" + "st1 { v16.b }[2], [x23]\n" + "st1 { v24.b }[2], [x22]\n" "b 143f\n" "142:" // Height 5: Partial direct writeback: partial_1_0 "str b31, [x11, #0x0]\n" - "str b8, [x24, #0x0]\n" - "str b15, [x23, #0x0]\n" - "str b16, [x22, #0x0]\n" - "str b24, [x21, #0x0]\n" + "str b8, [x25, #0x0]\n" + "str b15, [x24, #0x0]\n" + "str b16, [x23, #0x0]\n" + "str b24, [x22, #0x0]\n" "143:" // Height 5: Partial direct writeback: Done "b 145f\n" "144:" // Height 5: Full writeback "str q31, [x11, #0x0]\n" "add x11, x11, #0x10\n" - "str q8, [x24, #0x0]\n" - "str q15, [x23, #0x0]\n" - "str q16, [x22, #0x0]\n" - "str q24, [x21, #0x0]\n" + "str q8, [x25, #0x0]\n" + "str q15, [x24, #0x0]\n" + "str q16, [x23, #0x0]\n" + "str q24, [x22, #0x0]\n" "145:" // Height 5: Writeback done "subs x10, x10, #0x10\n" "bgt 118b\n" @@ -2801,16 +2800,16 @@ void a64_hybrid_s8qs_mmla_6x16 ( "149:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 150f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 151f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -2822,11 +2821,11 @@ void a64_hybrid_s8qs_mmla_6x16 ( "b 151f\n" "150:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" + "add x21, x22, x21\n" "151:" // Height 6: input setup done "cmp x27, #0x10\n" "blt 154f\n" @@ -2893,42 +2892,42 @@ void a64_hybrid_s8qs_mmla_6x16 ( "ldr q2, [x25, #0x0]\n" "prfm pldl1keep, [x21, #0x80]\n" ".inst 0x4e86a49f // smmla v31.4s, v4.16b, v6.16b\n" - "ldr q6, [x9, #0x90]\n" + "ldr q0, [x9, #0x90]\n" "ldr q4, [x23, #0x0]\n" ".inst 0x4e87a428 // smmla v8.4s, v1.16b, v7.16b\n" ".inst 0x4e87a470 // smmla v16.4s, v3.16b, v7.16b\n" ".inst 0x4e87a4b8 // smmla v24.4s, v5.16b, v7.16b\n" - "ldr q7, [x9, #0xa0]\n" - ".inst 0x4e86a42c // smmla v12.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a474 // smmla v20.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4bc // smmla v28.4s, v5.16b, v6.16b\n" - "ldr q6, [x9, #0xb0]\n" - ".inst 0x4e87a429 // smmla v9.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a471 // smmla v17.4s, v3.16b, v7.16b\n" - ".inst 0x4e87a4b9 // smmla v25.4s, v5.16b, v7.16b\n" - "ldr q7, [x9, #0xc0]\n" - ".inst 0x4e86a42d // smmla v13.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a475 // smmla v21.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4bd // smmla v29.4s, v5.16b, v6.16b\n" - "ldr q6, [x9, #0xd0]\n" - ".inst 0x4e87a42a // smmla v10.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a472 // smmla v18.4s, v3.16b, v7.16b\n" - ".inst 0x4e87a4ba // smmla v26.4s, v5.16b, v7.16b\n" - "ldr q7, [x9, #0xe0]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a476 // smmla v22.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4be // smmla v30.4s, v5.16b, v6.16b\n" - "ldr q6, [x9, #0xf0]\n" + "ldr q6, [x9, #0xa0]\n" + ".inst 0x4e80a42c // smmla v12.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a474 // smmla v20.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4bc // smmla v28.4s, v5.16b, v0.16b\n" + "ldr q0, [x9, #0xb0]\n" + ".inst 0x4e86a429 // smmla v9.4s, v1.16b, v6.16b\n" + ".inst 0x4e86a471 // smmla v17.4s, v3.16b, v6.16b\n" + ".inst 0x4e86a4b9 // smmla v25.4s, v5.16b, v6.16b\n" + "ldr q6, [x9, #0xc0]\n" + ".inst 0x4e80a42d // smmla v13.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a475 // smmla v21.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4bd // smmla v29.4s, v5.16b, v0.16b\n" + "ldr q0, [x9, #0xd0]\n" + ".inst 0x4e86a42a // smmla v10.4s, v1.16b, v6.16b\n" + ".inst 0x4e86a472 // smmla v18.4s, v3.16b, v6.16b\n" + ".inst 0x4e86a4ba // smmla v26.4s, v5.16b, v6.16b\n" + "ldr q6, [x9, #0xe0]\n" + ".inst 0x4e80a42e // smmla v14.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a476 // smmla v22.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4be // smmla v30.4s, v5.16b, v0.16b\n" + "ldr q0, [x9, #0xf0]\n" "add x9, x9, #0x100\n" - ".inst 0x4e87a42b // smmla v11.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a473 // smmla v19.4s, v3.16b, v7.16b\n" - ".inst 0x4e87a4bb // smmla v27.4s, v5.16b, v7.16b\n" + ".inst 0x4e86a42b // smmla v11.4s, v1.16b, v6.16b\n" + ".inst 0x4e86a473 // smmla v19.4s, v3.16b, v6.16b\n" + ".inst 0x4e86a4bb // smmla v27.4s, v5.16b, v6.16b\n" "ldr q7, [x9, #0x0]\n" - ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" + ".inst 0x4e80a42f // smmla v15.4s, v1.16b, v0.16b\n" "ldr q1, [x26, #0x0]\n" - ".inst 0x4e86a477 // smmla v23.4s, v3.16b, v6.16b\n" + ".inst 0x4e80a477 // smmla v23.4s, v3.16b, v0.16b\n" "ldr q3, [x24, #0x0]\n" - ".inst 0x4e86a4bf // smmla v31.4s, v5.16b, v6.16b\n" + ".inst 0x4e80a4bf // smmla v31.4s, v5.16b, v0.16b\n" "ldr q5, [x22, #0x0]\n" "ldr q6, [x21, #0x0]\n" "bge 152b\n" @@ -2984,87 +2983,87 @@ void a64_hybrid_s8qs_mmla_6x16 ( ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" ".inst 0x4e86a457 // smmla v23.4s, v2.16b, v6.16b\n" ".inst 0x4e86a49f // smmla v31.4s, v4.16b, v6.16b\n" - "ldr q6, [x9, #0x90]\n" + "ldr q0, [x9, #0x90]\n" ".inst 0x4e87a428 // smmla v8.4s, v1.16b, v7.16b\n" ".inst 0x4e87a470 // smmla v16.4s, v3.16b, v7.16b\n" ".inst 0x4e87a4b8 // smmla v24.4s, v5.16b, v7.16b\n" - "ldr q7, [x9, #0xa0]\n" - ".inst 0x4e86a42c // smmla v12.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a474 // smmla v20.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4bc // smmla v28.4s, v5.16b, v6.16b\n" - "ldr q6, [x9, #0xb0]\n" - ".inst 0x4e87a429 // smmla v9.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a471 // smmla v17.4s, v3.16b, v7.16b\n" - ".inst 0x4e87a4b9 // smmla v25.4s, v5.16b, v7.16b\n" - "ldr q7, [x9, #0xc0]\n" - ".inst 0x4e86a42d // smmla v13.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a475 // smmla v21.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4bd // smmla v29.4s, v5.16b, v6.16b\n" - "ldr q6, [x9, #0xd0]\n" - ".inst 0x4e87a42a // smmla v10.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a472 // smmla v18.4s, v3.16b, v7.16b\n" - ".inst 0x4e87a4ba // smmla v26.4s, v5.16b, v7.16b\n" - "ldr q7, [x9, #0xe0]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a476 // smmla v22.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4be // smmla v30.4s, v5.16b, v6.16b\n" - "ldr q6, [x9, #0xf0]\n" + "ldr q2, [x9, #0xa0]\n" + ".inst 0x4e80a42c // smmla v12.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a474 // smmla v20.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4bc // smmla v28.4s, v5.16b, v0.16b\n" + "ldr q0, [x9, #0xb0]\n" + ".inst 0x4e82a429 // smmla v9.4s, v1.16b, v2.16b\n" + ".inst 0x4e82a471 // smmla v17.4s, v3.16b, v2.16b\n" + ".inst 0x4e82a4b9 // smmla v25.4s, v5.16b, v2.16b\n" + "ldr q2, [x9, #0xc0]\n" + ".inst 0x4e80a42d // smmla v13.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a475 // smmla v21.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4bd // smmla v29.4s, v5.16b, v0.16b\n" + "ldr q0, [x9, #0xd0]\n" + ".inst 0x4e82a42a // smmla v10.4s, v1.16b, v2.16b\n" + ".inst 0x4e82a472 // smmla v18.4s, v3.16b, v2.16b\n" + ".inst 0x4e82a4ba // smmla v26.4s, v5.16b, v2.16b\n" + "ldr q2, [x9, #0xe0]\n" + ".inst 0x4e80a42e // smmla v14.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a476 // smmla v22.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4be // smmla v30.4s, v5.16b, v0.16b\n" + "ldr q0, [x9, #0xf0]\n" "add x9, x9, #0x100\n" - ".inst 0x4e87a42b // smmla v11.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a473 // smmla v19.4s, v3.16b, v7.16b\n" - ".inst 0x4e87a4bb // smmla v27.4s, v5.16b, v7.16b\n" - ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a477 // smmla v23.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4bf // smmla v31.4s, v5.16b, v6.16b\n" + ".inst 0x4e82a42b // smmla v11.4s, v1.16b, v2.16b\n" + ".inst 0x4e82a473 // smmla v19.4s, v3.16b, v2.16b\n" + ".inst 0x4e82a4bb // smmla v27.4s, v5.16b, v2.16b\n" + ".inst 0x4e80a42f // smmla v15.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a477 // smmla v23.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4bf // smmla v31.4s, v5.16b, v0.16b\n" "154:" // Height 6: Multiply loop: Main loop skip "cbz x27, 161f\n" "cmp x27, #0x8\n" "blt 156f\n" "155:" // Height 6: Multiply loop: Odd block loop "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" + "ldr d0, [x25], #0x8\n" + "trn1 v4.2d, v1.2d, v0.2d\n" "sub x27, x27, #0x8\n" - "ldr d3, [x24], #0x8\n" - "ldr d4, [x23], #0x8\n" - "trn1 v2.2d, v3.2d, v4.2d\n" + "ldr d1, [x24], #0x8\n" + "ldr d0, [x23], #0x8\n" + "trn1 v3.2d, v1.2d, v0.2d\n" "cmp x27, #0x8\n" - "ldr d5, [x22], #0x8\n" - "ldr d7, [x21], #0x8\n" - "trn1 v4.2d, v5.2d, v7.2d\n" - "ldr q6, [x9, #0x0]\n" - "ldr q7, [x9, #0x10]\n" - ".inst 0x4e86a408 // smmla v8.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a450 // smmla v16.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a498 // smmla v24.4s, v4.16b, v6.16b\n" - "ldr q6, [x9, #0x20]\n" - ".inst 0x4e87a40c // smmla v12.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a454 // smmla v20.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a49c // smmla v28.4s, v4.16b, v7.16b\n" - "ldr q7, [x9, #0x30]\n" - ".inst 0x4e86a409 // smmla v9.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a451 // smmla v17.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a499 // smmla v25.4s, v4.16b, v6.16b\n" - "ldr q6, [x9, #0x40]\n" - ".inst 0x4e87a40d // smmla v13.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a455 // smmla v21.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a49d // smmla v29.4s, v4.16b, v7.16b\n" - "ldr q7, [x9, #0x50]\n" - ".inst 0x4e86a40a // smmla v10.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a452 // smmla v18.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49a // smmla v26.4s, v4.16b, v6.16b\n" - "ldr q6, [x9, #0x60]\n" - ".inst 0x4e87a40e // smmla v14.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a456 // smmla v22.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a49e // smmla v30.4s, v4.16b, v7.16b\n" - "ldr q7, [x9, #0x70]\n" + "ldr d1, [x22], #0x8\n" + "ldr d0, [x21], #0x8\n" + "trn1 v2.2d, v1.2d, v0.2d\n" + "ldr q1, [x9, #0x0]\n" + "ldr q0, [x9, #0x10]\n" + ".inst 0x4e81a488 // smmla v8.4s, v4.16b, v1.16b\n" + ".inst 0x4e81a470 // smmla v16.4s, v3.16b, v1.16b\n" + ".inst 0x4e81a458 // smmla v24.4s, v2.16b, v1.16b\n" + "ldr q1, [x9, #0x20]\n" + ".inst 0x4e80a48c // smmla v12.4s, v4.16b, v0.16b\n" + ".inst 0x4e80a474 // smmla v20.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a45c // smmla v28.4s, v2.16b, v0.16b\n" + "ldr q0, [x9, #0x30]\n" + ".inst 0x4e81a489 // smmla v9.4s, v4.16b, v1.16b\n" + ".inst 0x4e81a471 // smmla v17.4s, v3.16b, v1.16b\n" + ".inst 0x4e81a459 // smmla v25.4s, v2.16b, v1.16b\n" + "ldr q1, [x9, #0x40]\n" + ".inst 0x4e80a48d // smmla v13.4s, v4.16b, v0.16b\n" + ".inst 0x4e80a475 // smmla v21.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a45d // smmla v29.4s, v2.16b, v0.16b\n" + "ldr q0, [x9, #0x50]\n" + ".inst 0x4e81a48a // smmla v10.4s, v4.16b, v1.16b\n" + ".inst 0x4e81a472 // smmla v18.4s, v3.16b, v1.16b\n" + ".inst 0x4e81a45a // smmla v26.4s, v2.16b, v1.16b\n" + "ldr q1, [x9, #0x60]\n" + ".inst 0x4e80a48e // smmla v14.4s, v4.16b, v0.16b\n" + ".inst 0x4e80a476 // smmla v22.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a45e // smmla v30.4s, v2.16b, v0.16b\n" + "ldr q0, [x9, #0x70]\n" "add x9, x9, #0x80\n" - ".inst 0x4e86a40b // smmla v11.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a453 // smmla v19.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49b // smmla v27.4s, v4.16b, v6.16b\n" - ".inst 0x4e87a40f // smmla v15.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a457 // smmla v23.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a49f // smmla v31.4s, v4.16b, v7.16b\n" + ".inst 0x4e81a48b // smmla v11.4s, v4.16b, v1.16b\n" + ".inst 0x4e81a473 // smmla v19.4s, v3.16b, v1.16b\n" + ".inst 0x4e81a45b // smmla v27.4s, v2.16b, v1.16b\n" + ".inst 0x4e80a48f // smmla v15.4s, v4.16b, v0.16b\n" + ".inst 0x4e80a477 // smmla v23.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a45f // smmla v31.4s, v2.16b, v0.16b\n" "bge 155b\n" "156:" // Height 6: Multiply loop: Skip odd blocks "cbz x27, 161f\n" @@ -3124,77 +3123,77 @@ void a64_hybrid_s8qs_mmla_6x16 ( "ldr b6, [x21, #0x0]\n" "160:" // Height 6: Multiply loop: Ragged operand read: Done "ldr q7, [x9, #0x0]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" - "trn1 v4.2d, v5.2d, v6.2d\n" - "ldr q6, [x9, #0x10]\n" - ".inst 0x4e87a450 // smmla v16.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a498 // smmla v24.4s, v4.16b, v7.16b\n" - "ldr q7, [x9, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49c // smmla v28.4s, v4.16b, v6.16b\n" - "ldr q6, [x9, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a451 // smmla v17.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a499 // smmla v25.4s, v4.16b, v7.16b\n" - "ldr q7, [x9, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a455 // smmla v21.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49d // smmla v29.4s, v4.16b, v6.16b\n" - "ldr q6, [x9, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a452 // smmla v18.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a49a // smmla v26.4s, v4.16b, v7.16b\n" - "ldr q7, [x9, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49e // smmla v30.4s, v4.16b, v6.16b\n" - "ldr q6, [x9, #0x70]\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" + "trn1 v2.2d, v1.2d, v2.2d\n" + "trn1 v4.2d, v3.2d, v4.2d\n" + ".inst 0x4e87a448 // smmla v8.4s, v2.16b, v7.16b\n" + "trn1 v3.2d, v5.2d, v6.2d\n" + "ldr q0, [x9, #0x10]\n" + ".inst 0x4e87a490 // smmla v16.4s, v4.16b, v7.16b\n" + ".inst 0x4e87a478 // smmla v24.4s, v3.16b, v7.16b\n" + "ldr q1, [x9, #0x20]\n" + ".inst 0x4e80a44c // smmla v12.4s, v2.16b, v0.16b\n" + ".inst 0x4e80a494 // smmla v20.4s, v4.16b, v0.16b\n" + ".inst 0x4e80a47c // smmla v28.4s, v3.16b, v0.16b\n" + "ldr q0, [x9, #0x30]\n" + ".inst 0x4e81a449 // smmla v9.4s, v2.16b, v1.16b\n" + ".inst 0x4e81a491 // smmla v17.4s, v4.16b, v1.16b\n" + ".inst 0x4e81a479 // smmla v25.4s, v3.16b, v1.16b\n" + "ldr q1, [x9, #0x40]\n" + ".inst 0x4e80a44d // smmla v13.4s, v2.16b, v0.16b\n" + ".inst 0x4e80a495 // smmla v21.4s, v4.16b, v0.16b\n" + ".inst 0x4e80a47d // smmla v29.4s, v3.16b, v0.16b\n" + "ldr q0, [x9, #0x50]\n" + ".inst 0x4e81a44a // smmla v10.4s, v2.16b, v1.16b\n" + ".inst 0x4e81a492 // smmla v18.4s, v4.16b, v1.16b\n" + ".inst 0x4e81a47a // smmla v26.4s, v3.16b, v1.16b\n" + "ldr q1, [x9, #0x60]\n" + ".inst 0x4e80a44e // smmla v14.4s, v2.16b, v0.16b\n" + ".inst 0x4e80a496 // smmla v22.4s, v4.16b, v0.16b\n" + ".inst 0x4e80a47e // smmla v30.4s, v3.16b, v0.16b\n" + "ldr q0, [x9, #0x70]\n" + ".inst 0x4e81a44b // smmla v11.4s, v2.16b, v1.16b\n" "add x9, x9, #0x80\n" - ".inst 0x4e87a453 // smmla v19.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a49b // smmla v27.4s, v4.16b, v7.16b\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a457 // smmla v23.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49f // smmla v31.4s, v4.16b, v6.16b\n" + ".inst 0x4e81a493 // smmla v19.4s, v4.16b, v1.16b\n" + ".inst 0x4e81a47b // smmla v27.4s, v3.16b, v1.16b\n" + ".inst 0x4e80a44f // smmla v15.4s, v2.16b, v0.16b\n" + ".inst 0x4e80a497 // smmla v23.4s, v4.16b, v0.16b\n" + ".inst 0x4e80a47f // smmla v31.4s, v3.16b, v0.16b\n" "161:" // Height 6: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 149b\n" - "ldr q0, [x14, #0x0]\n" - "ldr q1, [x14, #0x10]\n" - "uzp1 v7.2d, v8.2d, v12.2d\n" + "ldr q4, [x14, #0x0]\n" + "ldr q3, [x14, #0x10]\n" + "uzp1 v2.2d, v8.2d, v12.2d\n" "uzp2 v8.2d, v8.2d, v12.2d\n" - "ldr q2, [x14, #0x20]\n" - "ldr q3, [x14, #0x30]\n" + "ldr q1, [x14, #0x20]\n" + "ldr q0, [x14, #0x30]\n" "uzp1 v12.2d, v9.2d, v13.2d\n" "uzp2 v9.2d, v9.2d, v13.2d\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x11, x20\n" - "add x23, x24, x20\n" + "add x25, x11, x20\n" + "add x24, x25, x20\n" "uzp1 v13.2d, v10.2d, v14.2d\n" "uzp2 v10.2d, v10.2d, v14.2d\n" "uzp1 v14.2d, v11.2d, v15.2d\n" + "add x23, x24, x20\n" "add x22, x23, x20\n" - "add x21, x22, x20\n" "uzp2 v11.2d, v11.2d, v15.2d\n" "uzp1 v15.2d, v16.2d, v20.2d\n" - "add x20, x21, x20\n" + "add x21, x22, x20\n" "prfm pstl1keep, [x11, #0x0]\n" "uzp2 v16.2d, v16.2d, v20.2d\n" "uzp1 v20.2d, v17.2d, v21.2d\n" + "prfm pstl1keep, [x25, #0x0]\n" "prfm pstl1keep, [x24, #0x0]\n" - "prfm pstl1keep, [x23, #0x0]\n" "uzp2 v17.2d, v17.2d, v21.2d\n" "uzp1 v21.2d, v18.2d, v22.2d\n" + "prfm pstl1keep, [x23, #0x0]\n" "prfm pstl1keep, [x22, #0x0]\n" - "prfm pstl1keep, [x21, #0x0]\n" "uzp2 v18.2d, v18.2d, v22.2d\n" "uzp1 v22.2d, v19.2d, v23.2d\n" - "prfm pstl1keep, [x20, #0x0]\n" + "prfm pstl1keep, [x21, #0x0]\n" "add x14, x14, #0x40\n" "uzp2 v19.2d, v19.2d, v23.2d\n" "uzp1 v23.2d, v24.2d, v28.2d\n" @@ -3205,31 +3204,31 @@ void a64_hybrid_s8qs_mmla_6x16 ( "uzp2 v26.2d, v26.2d, v30.2d\n" "uzp1 v30.2d, v27.2d, v31.2d\n" "uzp2 v27.2d, v27.2d, v31.2d\n" - "mov v31.16b, v7.16b\n" - "add v31.4s, v31.4s, v0.4s\n" - "add v12.4s, v12.4s, v1.4s\n" - "add v13.4s, v13.4s, v2.4s\n" - "add v14.4s, v14.4s, v3.4s\n" - "add v8.4s, v8.4s, v0.4s\n" - "add v9.4s, v9.4s, v1.4s\n" - "add v10.4s, v10.4s, v2.4s\n" - "add v11.4s, v11.4s, v3.4s\n" - "add v15.4s, v15.4s, v0.4s\n" - "add v20.4s, v20.4s, v1.4s\n" - "add v21.4s, v21.4s, v2.4s\n" - "add v22.4s, v22.4s, v3.4s\n" - "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" - "add v23.4s, v23.4s, v0.4s\n" - "add v28.4s, v28.4s, v1.4s\n" - "add v29.4s, v29.4s, v2.4s\n" - "add v30.4s, v30.4s, v3.4s\n" - "add v24.4s, v24.4s, v0.4s\n" - "add v25.4s, v25.4s, v1.4s\n" - "add v26.4s, v26.4s, v2.4s\n" - "add v27.4s, v27.4s, v3.4s\n" + "mov v31.16b, v2.16b\n" + "add v31.4s, v31.4s, v4.4s\n" + "add v12.4s, v12.4s, v3.4s\n" + "add v13.4s, v13.4s, v1.4s\n" + "add v14.4s, v14.4s, v0.4s\n" + "add v8.4s, v8.4s, v4.4s\n" + "add v9.4s, v9.4s, v3.4s\n" + "add v10.4s, v10.4s, v1.4s\n" + "add v11.4s, v11.4s, v0.4s\n" + "add v15.4s, v15.4s, v4.4s\n" + "add v20.4s, v20.4s, v3.4s\n" + "add v21.4s, v21.4s, v1.4s\n" + "add v22.4s, v22.4s, v0.4s\n" + "add v16.4s, v16.4s, v4.4s\n" + "add v17.4s, v17.4s, v3.4s\n" + "add v18.4s, v18.4s, v1.4s\n" + "add v19.4s, v19.4s, v0.4s\n" + "add v23.4s, v23.4s, v4.4s\n" + "add v28.4s, v28.4s, v3.4s\n" + "add v29.4s, v29.4s, v1.4s\n" + "add v30.4s, v30.4s, v0.4s\n" + "add v24.4s, v24.4s, v4.4s\n" + "add v25.4s, v25.4s, v3.4s\n" + "add v26.4s, v26.4s, v1.4s\n" + "add v27.4s, v27.4s, v0.4s\n" "tbz %x[flags], #4, 162f\n" "ldr q0, [x12, #0x0]\n" "ldr q4, [x13, #0x0]\n" @@ -3243,10 +3242,10 @@ void a64_hybrid_s8qs_mmla_6x16 ( "add x13, x13, #0x40\n" "b 163f\n" "162:" // Height 6: per layer parameters - "add x25, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x25]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v4.4s }, [x20]\n" "mov v1.16b, v0.16b\n" "mov v5.16b, v4.16b\n" "mov v2.16b, v0.16b\n" @@ -3279,91 +3278,91 @@ void a64_hybrid_s8qs_mmla_6x16 ( "sqrdmulh v26.4s, v26.4s, v6.4s\n" "sqrdmulh v27.4s, v27.4s, v7.4s\n" "tbz %x[flags], #5, 164f\n" - "and v4.16b, v31.16b, v0.16b\n" - "and v5.16b, v12.16b, v1.16b\n" - "and v6.16b, v13.16b, v2.16b\n" - "and v7.16b, v14.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" + "and v7.16b, v31.16b, v0.16b\n" + "and v6.16b, v12.16b, v1.16b\n" + "and v5.16b, v13.16b, v2.16b\n" + "and v4.16b, v14.16b, v3.16b\n" "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v31.4s, v31.4s, v4.4s\n" - "sqadd v12.4s, v12.4s, v5.4s\n" - "sqadd v13.4s, v13.4s, v6.4s\n" - "sqadd v14.4s, v14.4s, v7.4s\n" - "and v4.16b, v8.16b, v0.16b\n" - "and v5.16b, v9.16b, v1.16b\n" - "and v6.16b, v10.16b, v2.16b\n" - "and v7.16b, v11.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v8.4s, v8.4s, v4.4s\n" - "sqadd v9.4s, v9.4s, v5.4s\n" - "sqadd v10.4s, v10.4s, v6.4s\n" - "sqadd v11.4s, v11.4s, v7.4s\n" - "and v4.16b, v15.16b, v0.16b\n" - "and v5.16b, v20.16b, v1.16b\n" - "and v6.16b, v21.16b, v2.16b\n" - "and v7.16b, v22.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v15.4s, v15.4s, v4.4s\n" - "sqadd v20.4s, v20.4s, v5.4s\n" - "sqadd v21.4s, v21.4s, v6.4s\n" - "sqadd v22.4s, v22.4s, v7.4s\n" - "and v4.16b, v16.16b, v0.16b\n" - "and v5.16b, v17.16b, v1.16b\n" - "and v6.16b, v18.16b, v2.16b\n" - "and v7.16b, v19.16b, v3.16b\n" "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" + "sqadd v31.4s, v31.4s, v7.4s\n" + "sqadd v12.4s, v12.4s, v6.4s\n" + "sqadd v13.4s, v13.4s, v5.4s\n" + "sqadd v14.4s, v14.4s, v4.4s\n" + "and v7.16b, v8.16b, v0.16b\n" + "and v6.16b, v9.16b, v1.16b\n" + "and v5.16b, v10.16b, v2.16b\n" + "and v4.16b, v11.16b, v3.16b\n" "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" - "and v4.16b, v23.16b, v0.16b\n" - "and v5.16b, v28.16b, v1.16b\n" - "and v6.16b, v29.16b, v2.16b\n" - "and v7.16b, v30.16b, v3.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" "sshr v6.4s, v6.4s, #0x1f\n" + "sshr v5.4s, v5.4s, #0x1f\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sqadd v8.4s, v8.4s, v7.4s\n" + "sqadd v9.4s, v9.4s, v6.4s\n" + "sqadd v10.4s, v10.4s, v5.4s\n" + "sqadd v11.4s, v11.4s, v4.4s\n" + "and v7.16b, v15.16b, v0.16b\n" + "and v6.16b, v20.16b, v1.16b\n" + "and v5.16b, v21.16b, v2.16b\n" + "and v4.16b, v22.16b, v3.16b\n" "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v28.4s, v28.4s, v5.4s\n" - "sqadd v29.4s, v29.4s, v6.4s\n" - "sqadd v30.4s, v30.4s, v7.4s\n" - "and v4.16b, v24.16b, v0.16b\n" - "and v5.16b, v25.16b, v1.16b\n" - "and v6.16b, v26.16b, v2.16b\n" - "and v7.16b, v27.16b, v3.16b\n" + "sshr v6.4s, v6.4s, #0x1f\n" + "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sqadd v15.4s, v15.4s, v7.4s\n" + "sqadd v20.4s, v20.4s, v6.4s\n" + "sqadd v21.4s, v21.4s, v5.4s\n" + "sqadd v22.4s, v22.4s, v4.4s\n" + "and v7.16b, v16.16b, v0.16b\n" + "and v6.16b, v17.16b, v1.16b\n" + "and v5.16b, v18.16b, v2.16b\n" + "and v4.16b, v19.16b, v3.16b\n" + "sshr v7.4s, v7.4s, #0x1f\n" + "sshr v6.4s, v6.4s, #0x1f\n" "sshr v5.4s, v5.4s, #0x1f\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v7.4s\n" + "sqadd v17.4s, v17.4s, v6.4s\n" + "sqadd v18.4s, v18.4s, v5.4s\n" + "sqadd v19.4s, v19.4s, v4.4s\n" + "and v7.16b, v23.16b, v0.16b\n" + "and v6.16b, v28.16b, v1.16b\n" + "and v5.16b, v29.16b, v2.16b\n" + "and v4.16b, v30.16b, v3.16b\n" + "sshr v7.4s, v7.4s, #0x1f\n" "sshr v6.4s, v6.4s, #0x1f\n" + "sshr v5.4s, v5.4s, #0x1f\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sqadd v23.4s, v23.4s, v7.4s\n" + "sqadd v28.4s, v28.4s, v6.4s\n" + "sqadd v29.4s, v29.4s, v5.4s\n" + "sqadd v30.4s, v30.4s, v4.4s\n" + "and v7.16b, v24.16b, v0.16b\n" + "and v6.16b, v25.16b, v1.16b\n" + "and v5.16b, v26.16b, v2.16b\n" + "and v4.16b, v27.16b, v3.16b\n" "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v24.4s, v24.4s, v4.4s\n" - "sqadd v25.4s, v25.4s, v5.4s\n" - "sqadd v26.4s, v26.4s, v6.4s\n" - "sqadd v27.4s, v27.4s, v7.4s\n" + "sshr v6.4s, v6.4s, #0x1f\n" + "sshr v5.4s, v5.4s, #0x1f\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sqadd v24.4s, v24.4s, v7.4s\n" + "sqadd v25.4s, v25.4s, v6.4s\n" + "sqadd v26.4s, v26.4s, v5.4s\n" + "sqadd v27.4s, v27.4s, v4.4s\n" "164:" // Height 6: no shift correction - "add x25, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x25]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v6.4s }, [x20]\n" "srshl v31.4s, v31.4s, v0.4s\n" "srshl v12.4s, v12.4s, v1.4s\n" "srshl v13.4s, v13.4s, v2.4s\n" "srshl v14.4s, v14.4s, v3.4s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x25]\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v5.4s }, [x20]\n" "srshl v8.4s, v8.4s, v0.4s\n" "srshl v9.4s, v9.4s, v1.4s\n" - "add x25, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x25]\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v4.4s }, [x20]\n" "srshl v10.4s, v10.4s, v2.4s\n" "srshl v11.4s, v11.4s, v3.4s\n" "cmp x10, #0x10\n" @@ -3383,225 +3382,225 @@ void a64_hybrid_s8qs_mmla_6x16 ( "srshl v25.4s, v25.4s, v1.4s\n" "srshl v26.4s, v26.4s, v2.4s\n" "srshl v27.4s, v27.4s, v3.4s\n" - "add v31.4s, v31.4s, v4.4s\n" - "add v12.4s, v12.4s, v4.4s\n" - "add v13.4s, v13.4s, v4.4s\n" - "add v14.4s, v14.4s, v4.4s\n" - "add v8.4s, v8.4s, v4.4s\n" - "add v9.4s, v9.4s, v4.4s\n" - "add v10.4s, v10.4s, v4.4s\n" - "add v11.4s, v11.4s, v4.4s\n" - "add v15.4s, v15.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add v23.4s, v23.4s, v4.4s\n" - "add v28.4s, v28.4s, v4.4s\n" - "add v29.4s, v29.4s, v4.4s\n" - "add v30.4s, v30.4s, v4.4s\n" - "add v24.4s, v24.4s, v4.4s\n" - "add v25.4s, v25.4s, v4.4s\n" - "add v26.4s, v26.4s, v4.4s\n" - "add v27.4s, v27.4s, v4.4s\n" - "smin v31.4s, v31.4s, v6.4s\n" - "smin v12.4s, v12.4s, v6.4s\n" - "smin v13.4s, v13.4s, v6.4s\n" - "smin v14.4s, v14.4s, v6.4s\n" - "smin v8.4s, v8.4s, v6.4s\n" - "smin v9.4s, v9.4s, v6.4s\n" - "smin v10.4s, v10.4s, v6.4s\n" - "smin v11.4s, v11.4s, v6.4s\n" - "smin v15.4s, v15.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "smin v28.4s, v28.4s, v6.4s\n" - "smin v29.4s, v29.4s, v6.4s\n" - "smin v30.4s, v30.4s, v6.4s\n" - "smin v24.4s, v24.4s, v6.4s\n" - "smin v25.4s, v25.4s, v6.4s\n" - "smin v26.4s, v26.4s, v6.4s\n" - "smin v27.4s, v27.4s, v6.4s\n" - "smax v31.4s, v31.4s, v5.4s\n" - "smax v12.4s, v12.4s, v5.4s\n" - "smax v13.4s, v13.4s, v5.4s\n" - "smax v14.4s, v14.4s, v5.4s\n" - "smax v8.4s, v8.4s, v5.4s\n" - "smax v9.4s, v9.4s, v5.4s\n" - "smax v10.4s, v10.4s, v5.4s\n" - "smax v11.4s, v11.4s, v5.4s\n" - "smax v15.4s, v15.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" - "smax v28.4s, v28.4s, v5.4s\n" - "smax v29.4s, v29.4s, v5.4s\n" - "smax v30.4s, v30.4s, v5.4s\n" - "smax v24.4s, v24.4s, v5.4s\n" - "smax v25.4s, v25.4s, v5.4s\n" - "smax v26.4s, v26.4s, v5.4s\n" - "smax v27.4s, v27.4s, v5.4s\n" + "add v31.4s, v31.4s, v6.4s\n" + "add v12.4s, v12.4s, v6.4s\n" + "add v13.4s, v13.4s, v6.4s\n" + "add v14.4s, v14.4s, v6.4s\n" + "add v8.4s, v8.4s, v6.4s\n" + "add v9.4s, v9.4s, v6.4s\n" + "add v10.4s, v10.4s, v6.4s\n" + "add v11.4s, v11.4s, v6.4s\n" + "add v15.4s, v15.4s, v6.4s\n" + "add v20.4s, v20.4s, v6.4s\n" + "add v21.4s, v21.4s, v6.4s\n" + "add v22.4s, v22.4s, v6.4s\n" + "add v16.4s, v16.4s, v6.4s\n" + "add v17.4s, v17.4s, v6.4s\n" + "add v18.4s, v18.4s, v6.4s\n" + "add v19.4s, v19.4s, v6.4s\n" + "add v23.4s, v23.4s, v6.4s\n" + "add v28.4s, v28.4s, v6.4s\n" + "add v29.4s, v29.4s, v6.4s\n" + "add v30.4s, v30.4s, v6.4s\n" + "add v24.4s, v24.4s, v6.4s\n" + "add v25.4s, v25.4s, v6.4s\n" + "add v26.4s, v26.4s, v6.4s\n" + "add v27.4s, v27.4s, v6.4s\n" + "smin v31.4s, v31.4s, v5.4s\n" + "smin v12.4s, v12.4s, v5.4s\n" + "smin v13.4s, v13.4s, v5.4s\n" + "smin v14.4s, v14.4s, v5.4s\n" + "smin v8.4s, v8.4s, v5.4s\n" + "smin v9.4s, v9.4s, v5.4s\n" + "smin v10.4s, v10.4s, v5.4s\n" + "smin v11.4s, v11.4s, v5.4s\n" + "smin v15.4s, v15.4s, v5.4s\n" + "smin v20.4s, v20.4s, v5.4s\n" + "smin v21.4s, v21.4s, v5.4s\n" + "smin v22.4s, v22.4s, v5.4s\n" + "smin v16.4s, v16.4s, v5.4s\n" + "smin v17.4s, v17.4s, v5.4s\n" + "smin v18.4s, v18.4s, v5.4s\n" + "smin v19.4s, v19.4s, v5.4s\n" + "smin v23.4s, v23.4s, v5.4s\n" + "smin v28.4s, v28.4s, v5.4s\n" + "smin v29.4s, v29.4s, v5.4s\n" + "smin v30.4s, v30.4s, v5.4s\n" + "smin v24.4s, v24.4s, v5.4s\n" + "smin v25.4s, v25.4s, v5.4s\n" + "smin v26.4s, v26.4s, v5.4s\n" + "smin v27.4s, v27.4s, v5.4s\n" + "smax v31.4s, v31.4s, v4.4s\n" + "smax v12.4s, v12.4s, v4.4s\n" + "smax v13.4s, v13.4s, v4.4s\n" + "smax v14.4s, v14.4s, v4.4s\n" + "smax v8.4s, v8.4s, v4.4s\n" + "smax v9.4s, v9.4s, v4.4s\n" + "smax v10.4s, v10.4s, v4.4s\n" + "smax v11.4s, v11.4s, v4.4s\n" + "smax v15.4s, v15.4s, v4.4s\n" + "smax v20.4s, v20.4s, v4.4s\n" + "smax v21.4s, v21.4s, v4.4s\n" + "smax v22.4s, v22.4s, v4.4s\n" + "smax v16.4s, v16.4s, v4.4s\n" + "smax v17.4s, v17.4s, v4.4s\n" + "smax v18.4s, v18.4s, v4.4s\n" + "smax v19.4s, v19.4s, v4.4s\n" + "smax v23.4s, v23.4s, v4.4s\n" + "smax v28.4s, v28.4s, v4.4s\n" + "smax v29.4s, v29.4s, v4.4s\n" + "smax v30.4s, v30.4s, v4.4s\n" + "smax v24.4s, v24.4s, v4.4s\n" + "smax v25.4s, v25.4s, v4.4s\n" + "smax v26.4s, v26.4s, v4.4s\n" + "smax v27.4s, v27.4s, v4.4s\n" "uzp1 v31.8h, v31.8h, v12.8h\n" - "uzp1 v12.8h, v13.8h, v14.8h\n" + "uzp1 v1.8h, v13.8h, v14.8h\n" "uzp1 v8.8h, v8.8h, v9.8h\n" - "uzp1 v9.8h, v10.8h, v11.8h\n" + "uzp1 v0.8h, v10.8h, v11.8h\n" "uzp1 v15.8h, v15.8h, v20.8h\n" "uzp1 v20.8h, v21.8h, v22.8h\n" "uzp1 v16.8h, v16.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v19.8h, v18.8h, v19.8h\n" "uzp1 v23.8h, v23.8h, v28.8h\n" - "uzp1 v28.8h, v29.8h, v30.8h\n" + "uzp1 v18.8h, v29.8h, v30.8h\n" "uzp1 v24.8h, v24.8h, v25.8h\n" - "uzp1 v25.8h, v26.8h, v27.8h\n" - "uzp1 v31.16b, v31.16b, v12.16b\n" - "uzp1 v8.16b, v8.16b, v9.16b\n" + "uzp1 v17.8h, v26.8h, v27.8h\n" + "uzp1 v31.16b, v31.16b, v1.16b\n" + "uzp1 v8.16b, v8.16b, v0.16b\n" "uzp1 v15.16b, v15.16b, v20.16b\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" - "uzp1 v23.16b, v23.16b, v28.16b\n" - "uzp1 v24.16b, v24.16b, v25.16b\n" + "uzp1 v16.16b, v16.16b, v19.16b\n" + "uzp1 v23.16b, v23.16b, v18.16b\n" + "uzp1 v24.16b, v24.16b, v17.16b\n" "bge 173f\n" "tbz x10, #3, 168f\n" "str d31, [x11], #0x8\n" - "str d8, [x24], #0x8\n" - "str d15, [x23], #0x8\n" - "str d16, [x22], #0x8\n" - "str d23, [x21], #0x8\n" - "str d24, [x20], #0x8\n" + "str d8, [x25], #0x8\n" + "str d15, [x24], #0x8\n" + "str d16, [x23], #0x8\n" + "str d23, [x22], #0x8\n" + "str d24, [x21], #0x8\n" "tbz x10, #2, 166f\n" "st1 { v31.s }[2], [x11], #0x4\n" - "st1 { v8.s }[2], [x24], #0x4\n" - "st1 { v15.s }[2], [x23], #0x4\n" - "st1 { v16.s }[2], [x22], #0x4\n" - "st1 { v23.s }[2], [x21], #0x4\n" - "st1 { v24.s }[2], [x20], #0x4\n" + "st1 { v8.s }[2], [x25], #0x4\n" + "st1 { v15.s }[2], [x24], #0x4\n" + "st1 { v16.s }[2], [x23], #0x4\n" + "st1 { v23.s }[2], [x22], #0x4\n" + "st1 { v24.s }[2], [x21], #0x4\n" "tbz x10, #1, 165f\n" "st1 { v31.h }[6], [x11], #0x2\n" - "st1 { v8.h }[6], [x24], #0x2\n" - "st1 { v15.h }[6], [x23], #0x2\n" - "st1 { v16.h }[6], [x22], #0x2\n" - "st1 { v23.h }[6], [x21], #0x2\n" - "st1 { v24.h }[6], [x20], #0x2\n" + "st1 { v8.h }[6], [x25], #0x2\n" + "st1 { v15.h }[6], [x24], #0x2\n" + "st1 { v16.h }[6], [x23], #0x2\n" + "st1 { v23.h }[6], [x22], #0x2\n" + "st1 { v24.h }[6], [x21], #0x2\n" "tbz x10, #0, 172f\n" "st1 { v31.b }[14], [x11]\n" - "st1 { v8.b }[14], [x24]\n" - "st1 { v15.b }[14], [x23]\n" - "st1 { v16.b }[14], [x22]\n" - "st1 { v23.b }[14], [x21]\n" - "st1 { v24.b }[14], [x20]\n" + "st1 { v8.b }[14], [x25]\n" + "st1 { v15.b }[14], [x24]\n" + "st1 { v16.b }[14], [x23]\n" + "st1 { v23.b }[14], [x22]\n" + "st1 { v24.b }[14], [x21]\n" "b 172f\n" "165:" // Height 6: Partial direct writeback: partial_1_12 "tbz x10, #0, 172f\n" "st1 { v31.b }[12], [x11]\n" - "st1 { v8.b }[12], [x24]\n" - "st1 { v15.b }[12], [x23]\n" - "st1 { v16.b }[12], [x22]\n" - "st1 { v23.b }[12], [x21]\n" - "st1 { v24.b }[12], [x20]\n" + "st1 { v8.b }[12], [x25]\n" + "st1 { v15.b }[12], [x24]\n" + "st1 { v16.b }[12], [x23]\n" + "st1 { v23.b }[12], [x22]\n" + "st1 { v24.b }[12], [x21]\n" "b 172f\n" "166:" // Height 6: Partial direct writeback: partial_2_8 "tbz x10, #1, 167f\n" "st1 { v31.h }[4], [x11], #0x2\n" - "st1 { v8.h }[4], [x24], #0x2\n" - "st1 { v15.h }[4], [x23], #0x2\n" - "st1 { v16.h }[4], [x22], #0x2\n" - "st1 { v23.h }[4], [x21], #0x2\n" - "st1 { v24.h }[4], [x20], #0x2\n" + "st1 { v8.h }[4], [x25], #0x2\n" + "st1 { v15.h }[4], [x24], #0x2\n" + "st1 { v16.h }[4], [x23], #0x2\n" + "st1 { v23.h }[4], [x22], #0x2\n" + "st1 { v24.h }[4], [x21], #0x2\n" "tbz x10, #0, 172f\n" "st1 { v31.b }[10], [x11]\n" - "st1 { v8.b }[10], [x24]\n" - "st1 { v15.b }[10], [x23]\n" - "st1 { v16.b }[10], [x22]\n" - "st1 { v23.b }[10], [x21]\n" - "st1 { v24.b }[10], [x20]\n" + "st1 { v8.b }[10], [x25]\n" + "st1 { v15.b }[10], [x24]\n" + "st1 { v16.b }[10], [x23]\n" + "st1 { v23.b }[10], [x22]\n" + "st1 { v24.b }[10], [x21]\n" "b 172f\n" "167:" // Height 6: Partial direct writeback: partial_1_8 "tbz x10, #0, 172f\n" "st1 { v31.b }[8], [x11]\n" - "st1 { v8.b }[8], [x24]\n" - "st1 { v15.b }[8], [x23]\n" - "st1 { v16.b }[8], [x22]\n" - "st1 { v23.b }[8], [x21]\n" - "st1 { v24.b }[8], [x20]\n" + "st1 { v8.b }[8], [x25]\n" + "st1 { v15.b }[8], [x24]\n" + "st1 { v16.b }[8], [x23]\n" + "st1 { v23.b }[8], [x22]\n" + "st1 { v24.b }[8], [x21]\n" "b 172f\n" "168:" // Height 6: Partial direct writeback: partial_4_0 "tbz x10, #2, 170f\n" "str s31, [x11], #0x4\n" - "str s8, [x24], #0x4\n" - "str s15, [x23], #0x4\n" - "str s16, [x22], #0x4\n" - "str s23, [x21], #0x4\n" - "str s24, [x20], #0x4\n" + "str s8, [x25], #0x4\n" + "str s15, [x24], #0x4\n" + "str s16, [x23], #0x4\n" + "str s23, [x22], #0x4\n" + "str s24, [x21], #0x4\n" "tbz x10, #1, 169f\n" "st1 { v31.h }[2], [x11], #0x2\n" - "st1 { v8.h }[2], [x24], #0x2\n" - "st1 { v15.h }[2], [x23], #0x2\n" - "st1 { v16.h }[2], [x22], #0x2\n" - "st1 { v23.h }[2], [x21], #0x2\n" - "st1 { v24.h }[2], [x20], #0x2\n" + "st1 { v8.h }[2], [x25], #0x2\n" + "st1 { v15.h }[2], [x24], #0x2\n" + "st1 { v16.h }[2], [x23], #0x2\n" + "st1 { v23.h }[2], [x22], #0x2\n" + "st1 { v24.h }[2], [x21], #0x2\n" "tbz x10, #0, 172f\n" "st1 { v31.b }[6], [x11]\n" - "st1 { v8.b }[6], [x24]\n" - "st1 { v15.b }[6], [x23]\n" - "st1 { v16.b }[6], [x22]\n" - "st1 { v23.b }[6], [x21]\n" - "st1 { v24.b }[6], [x20]\n" + "st1 { v8.b }[6], [x25]\n" + "st1 { v15.b }[6], [x24]\n" + "st1 { v16.b }[6], [x23]\n" + "st1 { v23.b }[6], [x22]\n" + "st1 { v24.b }[6], [x21]\n" "b 172f\n" "169:" // Height 6: Partial direct writeback: partial_1_4 "tbz x10, #0, 172f\n" "st1 { v31.b }[4], [x11]\n" - "st1 { v8.b }[4], [x24]\n" - "st1 { v15.b }[4], [x23]\n" - "st1 { v16.b }[4], [x22]\n" - "st1 { v23.b }[4], [x21]\n" - "st1 { v24.b }[4], [x20]\n" + "st1 { v8.b }[4], [x25]\n" + "st1 { v15.b }[4], [x24]\n" + "st1 { v16.b }[4], [x23]\n" + "st1 { v23.b }[4], [x22]\n" + "st1 { v24.b }[4], [x21]\n" "b 172f\n" "170:" // Height 6: Partial direct writeback: partial_2_0 "tbz x10, #1, 171f\n" "str h31, [x11], #0x2\n" - "str h8, [x24], #0x2\n" - "str h15, [x23], #0x2\n" - "str h16, [x22], #0x2\n" - "str h23, [x21], #0x2\n" - "str h24, [x20], #0x2\n" + "str h8, [x25], #0x2\n" + "str h15, [x24], #0x2\n" + "str h16, [x23], #0x2\n" + "str h23, [x22], #0x2\n" + "str h24, [x21], #0x2\n" "tbz x10, #0, 172f\n" "st1 { v31.b }[2], [x11]\n" - "st1 { v8.b }[2], [x24]\n" - "st1 { v15.b }[2], [x23]\n" - "st1 { v16.b }[2], [x22]\n" - "st1 { v23.b }[2], [x21]\n" - "st1 { v24.b }[2], [x20]\n" + "st1 { v8.b }[2], [x25]\n" + "st1 { v15.b }[2], [x24]\n" + "st1 { v16.b }[2], [x23]\n" + "st1 { v23.b }[2], [x22]\n" + "st1 { v24.b }[2], [x21]\n" "b 172f\n" "171:" // Height 6: Partial direct writeback: partial_1_0 "str b31, [x11, #0x0]\n" - "str b8, [x24, #0x0]\n" - "str b15, [x23, #0x0]\n" - "str b16, [x22, #0x0]\n" - "str b23, [x21, #0x0]\n" - "str b24, [x20, #0x0]\n" + "str b8, [x25, #0x0]\n" + "str b15, [x24, #0x0]\n" + "str b16, [x23, #0x0]\n" + "str b23, [x22, #0x0]\n" + "str b24, [x21, #0x0]\n" "172:" // Height 6: Partial direct writeback: Done "b 174f\n" "173:" // Height 6: Full writeback "str q31, [x11, #0x0]\n" "add x11, x11, #0x10\n" - "str q8, [x24, #0x0]\n" - "str q15, [x23, #0x0]\n" - "str q16, [x22, #0x0]\n" - "str q23, [x21, #0x0]\n" - "str q24, [x20, #0x0]\n" + "str q8, [x25, #0x0]\n" + "str q15, [x24, #0x0]\n" + "str q16, [x23, #0x0]\n" + "str q23, [x22, #0x0]\n" + "str q24, [x21, #0x0]\n" "174:" // Height 6: Writeback done "subs x10, x10, #0x10\n" "bgt 147b\n" @@ -3617,7 +3616,6 @@ void a64_hybrid_s8qs_mmla_6x16 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "176:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [flags] "r" (flags), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_multiplier_ptr] "I" (offsetof(KernelArgs, multiplier_ptr)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_shift_ptr] "I" (offsetof(KernelArgs, shift_ptr)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16.hpp index 48ce67613e2225129f56fa18ec588a7641a5680d..a02fbe8f28b6a7e5c00bffd3efd9b42c968c9ee0 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef __aarch64__ + #include "../std_transforms_fixed.hpp" #include "../performance_parameters.hpp" @@ -79,12 +79,12 @@ public: switch (ci->get_cpu_model()) { default: return { 31.65 }; - case CPUModel::A55r1: - return { 9.217 }; case CPUModel::A510: return { 15.87 }; case CPUModel::V1: return { 54.50 }; + case CPUModel::A55r1: + return { 9.217 }; } } @@ -97,7 +97,7 @@ public: case CPUModel::A510: return { 16.66, 3.92, 0.48 }; case CPUModel::V1: - return { 55.40, 19.21, 0.93 }; + return { 42.62, 16.32, 0.83 }; } } @@ -121,5 +121,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16/a55.cpp index 8046b2ebb0272180b23d1b679671933b99e2ef22..289d38c3b6a85e47e2b76a277ddb2fe6182a3643 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16/a55.cpp @@ -77,7 +77,6 @@ void a64_hybrid_s8s32_dot_6x16_a55 ( ka.N = N; ka.B_ptr = B_ptr; __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x6\n" "bge 171f\n" @@ -165,11 +164,11 @@ void a64_hybrid_s8s32_dot_6x16_a55 ( "14:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w14, [x20, x15, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 15f\n" - "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x13, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x15, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x13, [x20, #0x0]\n" "cbnz x15, 16f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x13, x13, x20\n" @@ -186,129 +185,129 @@ void a64_hybrid_s8s32_dot_6x16_a55 ( "blt 18f\n" "17:" // Height 1: Multiply loop: Main loop head ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr d6, [x16, #0x20]\n" - "ldr x12, [x16, #0x28]\n" + "ldr d17, [x16, #0x20]\n" + "ldr x20, [x16, #0x28]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "ldr d7, [x16, #0x30]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x16, #0x38]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - "ldr d6, [x16, #0x40]\n" - "ldr x12, [x16, #0x48]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - "ldr d7, [x16, #0x50]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x16, #0x58]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - "ldr d6, [x16, #0x60]\n" - "ldr x12, [x16, #0x68]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - "ldr d7, [x16, #0x70]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x16, #0x78]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - "ldr d6, [x16, #0x80]\n" - "ldr x12, [x16, #0x88]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - "ldr d7, [x16, #0x90]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x16, #0x98]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - "ldr d6, [x16, #0xa0]\n" - "ldr x12, [x16, #0xa8]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - "ldr d7, [x16, #0xb0]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x16, #0xb8]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - "ldr d6, [x16, #0xc0]\n" - "ldr x12, [x16, #0xc8]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - "ldr d7, [x16, #0xd0]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x16, #0xd8]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - "ldr d6, [x16, #0xe0]\n" - "ldr x12, [x16, #0xe8]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - "ldr d7, [x16, #0xf0]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x16, #0xf8]\n" - "mov v7.d[1], x11\n" + "ldr d16, [x16, #0x30]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x16, #0x38]\n" + "mov v16.d[1], x20\n" + ".inst 0x4f80e22a // sdot v10.4s, v17.16b, v0.4b[0]\n" + "ldr d17, [x16, #0x40]\n" + "ldr x20, [x16, #0x48]\n" + ".inst 0x4f80e20b // sdot v11.4s, v16.16b, v0.4b[0]\n" + "ldr d16, [x16, #0x50]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x16, #0x58]\n" + "mov v16.d[1], x20\n" + ".inst 0x4fa0e228 // sdot v8.4s, v17.16b, v0.4b[1]\n" + "ldr d17, [x16, #0x60]\n" + "ldr x20, [x16, #0x68]\n" + ".inst 0x4fa0e209 // sdot v9.4s, v16.16b, v0.4b[1]\n" + "ldr d16, [x16, #0x70]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x16, #0x78]\n" + "mov v16.d[1], x20\n" + ".inst 0x4fa0e22a // sdot v10.4s, v17.16b, v0.4b[1]\n" + "ldr d17, [x16, #0x80]\n" + "ldr x20, [x16, #0x88]\n" + ".inst 0x4fa0e20b // sdot v11.4s, v16.16b, v0.4b[1]\n" + "ldr d16, [x16, #0x90]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x16, #0x98]\n" + "mov v16.d[1], x20\n" + ".inst 0x4f80ea28 // sdot v8.4s, v17.16b, v0.4b[2]\n" + "ldr d17, [x16, #0xa0]\n" + "ldr x20, [x16, #0xa8]\n" + ".inst 0x4f80ea09 // sdot v9.4s, v16.16b, v0.4b[2]\n" + "ldr d16, [x16, #0xb0]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x16, #0xb8]\n" + "mov v16.d[1], x20\n" + ".inst 0x4f80ea2a // sdot v10.4s, v17.16b, v0.4b[2]\n" + "ldr d17, [x16, #0xc0]\n" + "ldr x20, [x16, #0xc8]\n" + ".inst 0x4f80ea0b // sdot v11.4s, v16.16b, v0.4b[2]\n" + "ldr d16, [x16, #0xd0]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x16, #0xd8]\n" + "mov v16.d[1], x20\n" + ".inst 0x4fa0ea28 // sdot v8.4s, v17.16b, v0.4b[3]\n" + "ldr d17, [x16, #0xe0]\n" + "ldr x20, [x16, #0xe8]\n" + ".inst 0x4fa0ea09 // sdot v9.4s, v16.16b, v0.4b[3]\n" + "ldr d16, [x16, #0xf0]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x16, #0xf8]\n" + "mov v16.d[1], x20\n" "add x13, x13, #0x10\n" "add x16, x16, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" + ".inst 0x4fa0ea2a // sdot v10.4s, v17.16b, v0.4b[3]\n" "ldr d6, [x16, #0x0]\n" - "ldr x12, [x16, #0x8]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" + "ldr x20, [x16, #0x8]\n" + ".inst 0x4fa0ea0b // sdot v11.4s, v16.16b, v0.4b[3]\n" "ldr d0, [x13, #0x0]\n" "sub x14, x14, #0x10\n" "ldr d7, [x16, #0x10]\n" "cmp x14, #0x20\n" - "ldr x10, [x13, #0x8]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x16, #0x18]\n" - "mov v0.d[1], x10\n" - "mov v7.d[1], x11\n" + "ldr x21, [x13, #0x8]\n" + "mov v6.d[1], x20\n" + "ldr x20, [x16, #0x18]\n" + "mov v0.d[1], x21\n" + "mov v7.d[1], x20\n" "prfm pldl1keep, [x13, #0x80]\n" "bge 17b\n" "18:" // Height 1: Multiply loop: Single iteration only ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q6, [x16, #0x20]\n" + "ldr q17, [x16, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - "ldr q6, [x16, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x50]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - "ldr q6, [x16, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - "ldr q7, [x16, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - "ldr q6, [x16, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - "ldr q7, [x16, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - "ldr q6, [x16, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - "ldr q7, [x16, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - "ldr q6, [x16, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - "ldr q7, [x16, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - "ldr q6, [x16, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - "ldr q7, [x16, #0xf0]\n" + "ldr q16, [x16, #0x30]\n" + ".inst 0x4f80e22a // sdot v10.4s, v17.16b, v0.4b[0]\n" + "ldr q17, [x16, #0x40]\n" + ".inst 0x4f80e20b // sdot v11.4s, v16.16b, v0.4b[0]\n" + "ldr q16, [x16, #0x50]\n" + ".inst 0x4fa0e228 // sdot v8.4s, v17.16b, v0.4b[1]\n" + "ldr q17, [x16, #0x60]\n" + ".inst 0x4fa0e209 // sdot v9.4s, v16.16b, v0.4b[1]\n" + "ldr q16, [x16, #0x70]\n" + ".inst 0x4fa0e22a // sdot v10.4s, v17.16b, v0.4b[1]\n" + "ldr q17, [x16, #0x80]\n" + ".inst 0x4fa0e20b // sdot v11.4s, v16.16b, v0.4b[1]\n" + "ldr q16, [x16, #0x90]\n" + ".inst 0x4f80ea28 // sdot v8.4s, v17.16b, v0.4b[2]\n" + "ldr q17, [x16, #0xa0]\n" + ".inst 0x4f80ea09 // sdot v9.4s, v16.16b, v0.4b[2]\n" + "ldr q16, [x16, #0xb0]\n" + ".inst 0x4f80ea2a // sdot v10.4s, v17.16b, v0.4b[2]\n" + "ldr q17, [x16, #0xc0]\n" + ".inst 0x4f80ea0b // sdot v11.4s, v16.16b, v0.4b[2]\n" + "ldr q16, [x16, #0xd0]\n" + ".inst 0x4fa0ea28 // sdot v8.4s, v17.16b, v0.4b[3]\n" + "ldr q17, [x16, #0xe0]\n" + ".inst 0x4fa0ea09 // sdot v9.4s, v16.16b, v0.4b[3]\n" + "ldr q16, [x16, #0xf0]\n" "add x13, x13, #0x10\n" "sub x14, x14, #0x10\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" + ".inst 0x4fa0ea2a // sdot v10.4s, v17.16b, v0.4b[3]\n" "prfm pldl1keep, [x13, #0x80]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x4fa0ea0b // sdot v11.4s, v16.16b, v0.4b[3]\n" "add x16, x16, #0x100\n" "19:" // Height 1: Multiply loop: Main loop skip "cbz x14, 24f\n" "cmp x14, #0x4\n" "blt 21f\n" "20:" // Height 1: Multiply loop: Odd block loop - "ldr s0, [x13], #0x4\n" + "ldr s18, [x13], #0x4\n" "sub x14, x14, #0x4\n" - "ldr q6, [x16, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x10]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "ldr q6, [x16, #0x20]\n" + "ldr q16, [x16, #0x0]\n" + ".inst 0x4f92e208 // sdot v8.4s, v16.16b, v18.4b[0]\n" + "ldr q16, [x16, #0x10]\n" + ".inst 0x4f92e209 // sdot v9.4s, v16.16b, v18.4b[0]\n" + "ldr q17, [x16, #0x20]\n" "cmp x14, #0x4\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" + "ldr q16, [x16, #0x30]\n" + ".inst 0x4f92e22a // sdot v10.4s, v17.16b, v18.4b[0]\n" + ".inst 0x4f92e20b // sdot v11.4s, v16.16b, v18.4b[0]\n" "add x16, x16, #0x40\n" "bge 20b\n" "21:" // Height 1: Multiply loop: Skip odd blocks @@ -321,14 +320,14 @@ void a64_hybrid_s8s32_dot_6x16_a55 ( "22:" // Height 1: Multiply loop: Ragged operand read: partial_1_0 "ldr b0, [x13, #0x0]\n" "23:" // Height 1: Multiply loop: Ragged operand read: Done - "ldr q6, [x16, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x10]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "ldr q6, [x16, #0x20]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" + "ldr q16, [x16, #0x0]\n" + ".inst 0x4f80e208 // sdot v8.4s, v16.16b, v0.4b[0]\n" + "ldr q16, [x16, #0x10]\n" + ".inst 0x4f80e209 // sdot v9.4s, v16.16b, v0.4b[0]\n" + "ldr q16, [x16, #0x20]\n" + ".inst 0x4f80e20a // sdot v10.4s, v16.16b, v0.4b[0]\n" + "ldr q16, [x16, #0x30]\n" + ".inst 0x4f80e20b // sdot v11.4s, v16.16b, v0.4b[0]\n" "add x16, x16, #0x40\n" "24:" // Height 1: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -499,226 +498,226 @@ void a64_hybrid_s8s32_dot_6x16_a55 ( "48:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w14, [x20, x15, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 49f\n" - "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x13, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x15, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x13, [x20, #0x0]\n" + "ldr x12, [x20, #0x8]\n" "cbnz x15, 50f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x13, x13, x20\n" - "add x9, x9, x20\n" + "add x12, x12, x20\n" "b 50f\n" "49:" // Height 2: setup direct input "mov x13, %x[input_ptr]\n" - "add x9, x13, x20\n" + "add x12, x13, x21\n" "50:" // Height 2: input setup done "cmp x14, #0x10\n" "blt 53f\n" "ldr q0, [x13, #0x0]\n" "cmp x14, #0x20\n" - "ldr q1, [x9, #0x0]\n" + "ldr q1, [x12, #0x0]\n" "ldr q6, [x16, #0x0]\n" "ldr q7, [x16, #0x10]\n" "blt 52f\n" "51:" // Height 2: Multiply loop: Main loop head ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr x12, [x16, #0x28]\n" + "ldr x21, [x16, #0x28]\n" ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr d6, [x16, #0x20]\n" + "ldr d17, [x16, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "ldr x11, [x16, #0x38]\n" + "ldr x20, [x16, #0x38]\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "ldr d7, [x16, #0x30]\n" - "mov v6.d[1], x12\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "ldr d6, [x16, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - "ldr x12, [x16, #0x48]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - "ldr d7, [x16, #0x50]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x16, #0x58]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - "ldr x12, [x16, #0x68]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - "ldr d6, [x16, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - "ldr x11, [x16, #0x78]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - "ldr d7, [x16, #0x70]\n" - "mov v6.d[1], x12\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - "ldr d6, [x16, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - "ldr x12, [x16, #0x88]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - "ldr d7, [x16, #0x90]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x16, #0x98]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - "ldr x12, [x16, #0xa8]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - "ldr d6, [x16, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - "ldr x11, [x16, #0xb8]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - "ldr d7, [x16, #0xb0]\n" - "mov v6.d[1], x12\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - "ldr d6, [x16, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - "ldr x12, [x16, #0xc8]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - "ldr d7, [x16, #0xd0]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x16, #0xd8]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - "ldr x12, [x16, #0xe8]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - "ldr d6, [x16, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - "ldr x11, [x16, #0xf8]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - "ldr d7, [x16, #0xf0]\n" - "mov v6.d[1], x12\n" + "ldr d16, [x16, #0x30]\n" + "mov v17.d[1], x21\n" + ".inst 0x4f80e22a // sdot v10.4s, v17.16b, v0.4b[0]\n" + "mov v16.d[1], x20\n" + ".inst 0x4f81e22e // sdot v14.4s, v17.16b, v1.4b[0]\n" + "ldr d17, [x16, #0x40]\n" + ".inst 0x4f80e20b // sdot v11.4s, v16.16b, v0.4b[0]\n" + "ldr x20, [x16, #0x48]\n" + ".inst 0x4f81e20f // sdot v15.4s, v16.16b, v1.4b[0]\n" + "ldr d16, [x16, #0x50]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x16, #0x58]\n" + "mov v16.d[1], x20\n" + ".inst 0x4fa0e228 // sdot v8.4s, v17.16b, v0.4b[1]\n" + "ldr x21, [x16, #0x68]\n" + ".inst 0x4fa1e22c // sdot v12.4s, v17.16b, v1.4b[1]\n" + "ldr d17, [x16, #0x60]\n" + ".inst 0x4fa0e209 // sdot v9.4s, v16.16b, v0.4b[1]\n" + "ldr x20, [x16, #0x78]\n" + ".inst 0x4fa1e20d // sdot v13.4s, v16.16b, v1.4b[1]\n" + "ldr d16, [x16, #0x70]\n" + "mov v17.d[1], x21\n" + ".inst 0x4fa0e22a // sdot v10.4s, v17.16b, v0.4b[1]\n" + "mov v16.d[1], x20\n" + ".inst 0x4fa1e22e // sdot v14.4s, v17.16b, v1.4b[1]\n" + "ldr d17, [x16, #0x80]\n" + ".inst 0x4fa0e20b // sdot v11.4s, v16.16b, v0.4b[1]\n" + "ldr x20, [x16, #0x88]\n" + ".inst 0x4fa1e20f // sdot v15.4s, v16.16b, v1.4b[1]\n" + "ldr d16, [x16, #0x90]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x16, #0x98]\n" + "mov v16.d[1], x20\n" + ".inst 0x4f80ea28 // sdot v8.4s, v17.16b, v0.4b[2]\n" + "ldr x21, [x16, #0xa8]\n" + ".inst 0x4f81ea2c // sdot v12.4s, v17.16b, v1.4b[2]\n" + "ldr d17, [x16, #0xa0]\n" + ".inst 0x4f80ea09 // sdot v9.4s, v16.16b, v0.4b[2]\n" + "ldr x20, [x16, #0xb8]\n" + ".inst 0x4f81ea0d // sdot v13.4s, v16.16b, v1.4b[2]\n" + "ldr d16, [x16, #0xb0]\n" + "mov v17.d[1], x21\n" + ".inst 0x4f80ea2a // sdot v10.4s, v17.16b, v0.4b[2]\n" + "mov v16.d[1], x20\n" + ".inst 0x4f81ea2e // sdot v14.4s, v17.16b, v1.4b[2]\n" + "ldr d17, [x16, #0xc0]\n" + ".inst 0x4f80ea0b // sdot v11.4s, v16.16b, v0.4b[2]\n" + "ldr x20, [x16, #0xc8]\n" + ".inst 0x4f81ea0f // sdot v15.4s, v16.16b, v1.4b[2]\n" + "ldr d16, [x16, #0xd0]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x16, #0xd8]\n" + "mov v16.d[1], x20\n" + ".inst 0x4fa0ea28 // sdot v8.4s, v17.16b, v0.4b[3]\n" + "ldr x21, [x16, #0xe8]\n" + ".inst 0x4fa1ea2c // sdot v12.4s, v17.16b, v1.4b[3]\n" + "ldr d17, [x16, #0xe0]\n" + ".inst 0x4fa0ea09 // sdot v9.4s, v16.16b, v0.4b[3]\n" + "ldr x20, [x16, #0xf8]\n" + ".inst 0x4fa1ea0d // sdot v13.4s, v16.16b, v1.4b[3]\n" + "ldr d16, [x16, #0xf0]\n" + "mov v17.d[1], x21\n" "add x13, x13, #0x10\n" - "mov v7.d[1], x11\n" - "add x9, x9, #0x10\n" + "mov v16.d[1], x20\n" + "add x12, x12, #0x10\n" "add x16, x16, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" + ".inst 0x4fa0ea2a // sdot v10.4s, v17.16b, v0.4b[3]\n" + ".inst 0x4fa1ea2e // sdot v14.4s, v17.16b, v1.4b[3]\n" "ldr d6, [x16, #0x0]\n" - "ldr x12, [x16, #0x8]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" + "ldr x21, [x16, #0x8]\n" + ".inst 0x4fa0ea0b // sdot v11.4s, v16.16b, v0.4b[3]\n" "ldr d0, [x13, #0x0]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" - "ldr d1, [x9, #0x0]\n" + ".inst 0x4fa1ea0f // sdot v15.4s, v16.16b, v1.4b[3]\n" + "ldr d1, [x12, #0x0]\n" "sub x14, x14, #0x10\n" "ldr d7, [x16, #0x10]\n" "cmp x14, #0x20\n" - "ldr x10, [x13, #0x8]\n" - "mov v6.d[1], x12\n" - "ldr x28, [x9, #0x8]\n" - "mov v0.d[1], x10\n" - "ldr x11, [x16, #0x18]\n" - "mov v1.d[1], x28\n" + "ldr x20, [x13, #0x8]\n" + "mov v6.d[1], x21\n" + "ldr x21, [x12, #0x8]\n" + "mov v0.d[1], x20\n" + "ldr x20, [x16, #0x18]\n" + "mov v1.d[1], x21\n" "prfm pldl1keep, [x13, #0x80]\n" - "mov v7.d[1], x11\n" - "prfm pldl1keep, [x9, #0x80]\n" + "mov v7.d[1], x20\n" + "prfm pldl1keep, [x12, #0x80]\n" "bge 51b\n" "52:" // Height 2: Multiply loop: Single iteration only ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" "add x13, x13, #0x10\n" ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x16, #0x20]\n" + "ldr q17, [x16, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q16, [x16, #0x30]\n" + ".inst 0x4f80e22a // sdot v10.4s, v17.16b, v0.4b[0]\n" "sub x14, x14, #0x10\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x16, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" + ".inst 0x4f81e22e // sdot v14.4s, v17.16b, v1.4b[0]\n" + "ldr q17, [x16, #0x40]\n" + ".inst 0x4f80e20b // sdot v11.4s, v16.16b, v0.4b[0]\n" "prfm pldl1keep, [x13, #0x80]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x16, #0x50]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - "prfm pldl1keep, [x9, #0x80]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - "ldr q6, [x16, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - "ldr q7, [x16, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - "ldr q6, [x16, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - "ldr q7, [x16, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - "ldr q6, [x16, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - "ldr q7, [x16, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - "ldr q6, [x16, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - "ldr q7, [x16, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - "ldr q6, [x16, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - "ldr q7, [x16, #0xf0]\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" + ".inst 0x4f81e20f // sdot v15.4s, v16.16b, v1.4b[0]\n" + "ldr q16, [x16, #0x50]\n" + ".inst 0x4fa0e228 // sdot v8.4s, v17.16b, v0.4b[1]\n" + "prfm pldl1keep, [x12, #0x80]\n" + ".inst 0x4fa1e22c // sdot v12.4s, v17.16b, v1.4b[1]\n" + "ldr q17, [x16, #0x60]\n" + ".inst 0x4fa0e209 // sdot v9.4s, v16.16b, v0.4b[1]\n" + ".inst 0x4fa1e20d // sdot v13.4s, v16.16b, v1.4b[1]\n" + "ldr q16, [x16, #0x70]\n" + ".inst 0x4fa0e22a // sdot v10.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e22e // sdot v14.4s, v17.16b, v1.4b[1]\n" + "ldr q17, [x16, #0x80]\n" + ".inst 0x4fa0e20b // sdot v11.4s, v16.16b, v0.4b[1]\n" + ".inst 0x4fa1e20f // sdot v15.4s, v16.16b, v1.4b[1]\n" + "ldr q16, [x16, #0x90]\n" + ".inst 0x4f80ea28 // sdot v8.4s, v17.16b, v0.4b[2]\n" + ".inst 0x4f81ea2c // sdot v12.4s, v17.16b, v1.4b[2]\n" + "ldr q17, [x16, #0xa0]\n" + ".inst 0x4f80ea09 // sdot v9.4s, v16.16b, v0.4b[2]\n" + ".inst 0x4f81ea0d // sdot v13.4s, v16.16b, v1.4b[2]\n" + "ldr q16, [x16, #0xb0]\n" + ".inst 0x4f80ea2a // sdot v10.4s, v17.16b, v0.4b[2]\n" + ".inst 0x4f81ea2e // sdot v14.4s, v17.16b, v1.4b[2]\n" + "ldr q17, [x16, #0xc0]\n" + ".inst 0x4f80ea0b // sdot v11.4s, v16.16b, v0.4b[2]\n" + ".inst 0x4f81ea0f // sdot v15.4s, v16.16b, v1.4b[2]\n" + "ldr q16, [x16, #0xd0]\n" + ".inst 0x4fa0ea28 // sdot v8.4s, v17.16b, v0.4b[3]\n" + ".inst 0x4fa1ea2c // sdot v12.4s, v17.16b, v1.4b[3]\n" + "ldr q17, [x16, #0xe0]\n" + ".inst 0x4fa0ea09 // sdot v9.4s, v16.16b, v0.4b[3]\n" + ".inst 0x4fa1ea0d // sdot v13.4s, v16.16b, v1.4b[3]\n" + "ldr q16, [x16, #0xf0]\n" + ".inst 0x4fa0ea2a // sdot v10.4s, v17.16b, v0.4b[3]\n" "add x16, x16, #0x100\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" + ".inst 0x4fa1ea2e // sdot v14.4s, v17.16b, v1.4b[3]\n" + ".inst 0x4fa0ea0b // sdot v11.4s, v16.16b, v0.4b[3]\n" + ".inst 0x4fa1ea0f // sdot v15.4s, v16.16b, v1.4b[3]\n" "53:" // Height 2: Multiply loop: Main loop skip "cbz x14, 58f\n" "cmp x14, #0x4\n" "blt 55f\n" "54:" // Height 2: Multiply loop: Odd block loop - "ldr s0, [x13], #0x4\n" + "ldr s19, [x13], #0x4\n" "sub x14, x14, #0x4\n" - "ldr s1, [x9], #0x4\n" + "ldr s18, [x12], #0x4\n" "cmp x14, #0x4\n" - "ldr q6, [x16, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x10]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x16, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q17, [x16, #0x0]\n" + ".inst 0x4f93e228 // sdot v8.4s, v17.16b, v19.4b[0]\n" + "ldr q16, [x16, #0x10]\n" + ".inst 0x4f92e22c // sdot v12.4s, v17.16b, v18.4b[0]\n" + "ldr q17, [x16, #0x20]\n" + ".inst 0x4f93e209 // sdot v9.4s, v16.16b, v19.4b[0]\n" + ".inst 0x4f92e20d // sdot v13.4s, v16.16b, v18.4b[0]\n" + "ldr q16, [x16, #0x30]\n" + ".inst 0x4f93e22a // sdot v10.4s, v17.16b, v19.4b[0]\n" "add x16, x16, #0x40\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" + ".inst 0x4f92e22e // sdot v14.4s, v17.16b, v18.4b[0]\n" + ".inst 0x4f93e20b // sdot v11.4s, v16.16b, v19.4b[0]\n" + ".inst 0x4f92e20f // sdot v15.4s, v16.16b, v18.4b[0]\n" "bge 54b\n" "55:" // Height 2: Multiply loop: Skip odd blocks "cbz x14, 58f\n" "tbz x14, #1, 56f\n" "ldr h0, [x13], #0x2\n" - "ldr h1, [x9], #0x2\n" + "ldr h1, [x12], #0x2\n" "tbz x14, #0, 57f\n" "ld1 { v0.b }[2], [x13]\n" - "ld1 { v1.b }[2], [x9]\n" + "ld1 { v1.b }[2], [x12]\n" "b 57f\n" "56:" // Height 2: Multiply loop: Ragged operand read: partial_1_0 "ldr b0, [x13, #0x0]\n" - "ldr b1, [x9, #0x0]\n" + "ldr b1, [x12, #0x0]\n" "57:" // Height 2: Multiply loop: Ragged operand read: Done - "ldr q6, [x16, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x10]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x16, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q17, [x16, #0x0]\n" + ".inst 0x4f80e228 // sdot v8.4s, v17.16b, v0.4b[0]\n" + "ldr q16, [x16, #0x10]\n" + ".inst 0x4f81e22c // sdot v12.4s, v17.16b, v1.4b[0]\n" + "ldr q17, [x16, #0x20]\n" + ".inst 0x4f80e209 // sdot v9.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f81e20d // sdot v13.4s, v16.16b, v1.4b[0]\n" + "ldr q16, [x16, #0x30]\n" + ".inst 0x4f80e22a // sdot v10.4s, v17.16b, v0.4b[0]\n" "add x16, x16, #0x40\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" + ".inst 0x4f81e22e // sdot v14.4s, v17.16b, v1.4b[0]\n" + ".inst 0x4f80e20b // sdot v11.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f81e20f // sdot v15.4s, v16.16b, v1.4b[0]\n" "58:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x15, x15, #0x1\n" @@ -936,281 +935,281 @@ void a64_hybrid_s8s32_dot_6x16_a55 ( "82:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w14, [x20, x15, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 83f\n" - "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x13, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" - "ldr x27, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x15, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x13, [x20, #0x0]\n" + "ldr x12, [x20, #0x8]\n" + "ldr x11, [x20, #0x10]\n" "cbnz x15, 84f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x13, x13, x20\n" - "add x9, x9, x20\n" - "add x27, x27, x20\n" + "add x12, x12, x20\n" + "add x11, x11, x20\n" "b 84f\n" "83:" // Height 3: setup direct input "mov x13, %x[input_ptr]\n" - "add x9, x13, x20\n" - "add x27, x9, x20\n" + "add x12, x13, x21\n" + "add x11, x12, x21\n" "84:" // Height 3: input setup done "cmp x14, #0x10\n" "blt 87f\n" "ldr q0, [x13, #0x0]\n" "cmp x14, #0x20\n" - "ldr q1, [x9, #0x0]\n" - "ldr q2, [x27, #0x0]\n" + "ldr q1, [x12, #0x0]\n" + "ldr q2, [x11, #0x0]\n" "ldr q6, [x16, #0x0]\n" "ldr q7, [x16, #0x10]\n" "blt 86f\n" "85:" // Height 3: Multiply loop: Main loop head ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr x12, [x16, #0x28]\n" + "ldr x21, [x16, #0x28]\n" ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr x11, [x16, #0x38]\n" + "ldr x20, [x16, #0x38]\n" ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - "ldr d6, [x16, #0x20]\n" + "ldr d21, [x16, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x12\n" + "mov v21.d[1], x21\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "ldr x12, [x16, #0x48]\n" + "ldr x21, [x16, #0x48]\n" ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - "ldr d7, [x16, #0x30]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "ldr x11, [x16, #0x58]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - "ldr d6, [x16, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x12\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - "ldr x12, [x16, #0x68]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - "ldr d7, [x16, #0x50]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - "ldr x11, [x16, #0x78]\n" - ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" - "ldr d6, [x16, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - "mov v6.d[1], x12\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - "ldr x12, [x16, #0x88]\n" - ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - "ldr d7, [x16, #0x70]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - "ldr x11, [x16, #0x98]\n" - ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - "ldr d6, [x16, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - "mov v6.d[1], x12\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - "ldr x12, [x16, #0xa8]\n" - ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" - "ldr d7, [x16, #0x90]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - "ldr x11, [x16, #0xb8]\n" - ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" - "ldr d6, [x16, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - "mov v6.d[1], x12\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - "ldr x12, [x16, #0xc8]\n" - ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" - "ldr d7, [x16, #0xb0]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - "ldr x11, [x16, #0xd8]\n" - ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" - "ldr d6, [x16, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - "mov v6.d[1], x12\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - "ldr x12, [x16, #0xe8]\n" - ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" - "ldr d7, [x16, #0xd0]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - "ldr x11, [x16, #0xf8]\n" - ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" - "ldr d6, [x16, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - "mov v6.d[1], x12\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" + "ldr d20, [x16, #0x30]\n" + "mov v20.d[1], x20\n" + ".inst 0x4f80e2aa // sdot v10.4s, v21.16b, v0.4b[0]\n" + ".inst 0x4f81e2ae // sdot v14.4s, v21.16b, v1.4b[0]\n" + "ldr x20, [x16, #0x58]\n" + ".inst 0x4f82e2b2 // sdot v18.4s, v21.16b, v2.4b[0]\n" + "ldr d21, [x16, #0x40]\n" + ".inst 0x4f80e28b // sdot v11.4s, v20.16b, v0.4b[0]\n" + "mov v21.d[1], x21\n" + ".inst 0x4f81e28f // sdot v15.4s, v20.16b, v1.4b[0]\n" + "ldr x21, [x16, #0x68]\n" + ".inst 0x4f82e293 // sdot v19.4s, v20.16b, v2.4b[0]\n" + "ldr d20, [x16, #0x50]\n" + "mov v20.d[1], x20\n" + ".inst 0x4fa0e2a8 // sdot v8.4s, v21.16b, v0.4b[1]\n" + ".inst 0x4fa1e2ac // sdot v12.4s, v21.16b, v1.4b[1]\n" + "ldr x20, [x16, #0x78]\n" + ".inst 0x4fa2e2b0 // sdot v16.4s, v21.16b, v2.4b[1]\n" + "ldr d21, [x16, #0x60]\n" + ".inst 0x4fa0e289 // sdot v9.4s, v20.16b, v0.4b[1]\n" + "mov v21.d[1], x21\n" + ".inst 0x4fa1e28d // sdot v13.4s, v20.16b, v1.4b[1]\n" + "ldr x21, [x16, #0x88]\n" + ".inst 0x4fa2e291 // sdot v17.4s, v20.16b, v2.4b[1]\n" + "ldr d20, [x16, #0x70]\n" + "mov v20.d[1], x20\n" + ".inst 0x4fa0e2aa // sdot v10.4s, v21.16b, v0.4b[1]\n" + ".inst 0x4fa1e2ae // sdot v14.4s, v21.16b, v1.4b[1]\n" + "ldr x20, [x16, #0x98]\n" + ".inst 0x4fa2e2b2 // sdot v18.4s, v21.16b, v2.4b[1]\n" + "ldr d21, [x16, #0x80]\n" + ".inst 0x4fa0e28b // sdot v11.4s, v20.16b, v0.4b[1]\n" + "mov v21.d[1], x21\n" + ".inst 0x4fa1e28f // sdot v15.4s, v20.16b, v1.4b[1]\n" + "ldr x21, [x16, #0xa8]\n" + ".inst 0x4fa2e293 // sdot v19.4s, v20.16b, v2.4b[1]\n" + "ldr d20, [x16, #0x90]\n" + "mov v20.d[1], x20\n" + ".inst 0x4f80eaa8 // sdot v8.4s, v21.16b, v0.4b[2]\n" + ".inst 0x4f81eaac // sdot v12.4s, v21.16b, v1.4b[2]\n" + "ldr x20, [x16, #0xb8]\n" + ".inst 0x4f82eab0 // sdot v16.4s, v21.16b, v2.4b[2]\n" + "ldr d21, [x16, #0xa0]\n" + ".inst 0x4f80ea89 // sdot v9.4s, v20.16b, v0.4b[2]\n" + "mov v21.d[1], x21\n" + ".inst 0x4f81ea8d // sdot v13.4s, v20.16b, v1.4b[2]\n" + "ldr x21, [x16, #0xc8]\n" + ".inst 0x4f82ea91 // sdot v17.4s, v20.16b, v2.4b[2]\n" + "ldr d20, [x16, #0xb0]\n" + "mov v20.d[1], x20\n" + ".inst 0x4f80eaaa // sdot v10.4s, v21.16b, v0.4b[2]\n" + ".inst 0x4f81eaae // sdot v14.4s, v21.16b, v1.4b[2]\n" + "ldr x20, [x16, #0xd8]\n" + ".inst 0x4f82eab2 // sdot v18.4s, v21.16b, v2.4b[2]\n" + "ldr d21, [x16, #0xc0]\n" + ".inst 0x4f80ea8b // sdot v11.4s, v20.16b, v0.4b[2]\n" + "mov v21.d[1], x21\n" + ".inst 0x4f81ea8f // sdot v15.4s, v20.16b, v1.4b[2]\n" + "ldr x21, [x16, #0xe8]\n" + ".inst 0x4f82ea93 // sdot v19.4s, v20.16b, v2.4b[2]\n" + "ldr d20, [x16, #0xd0]\n" + "mov v20.d[1], x20\n" + ".inst 0x4fa0eaa8 // sdot v8.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa1eaac // sdot v12.4s, v21.16b, v1.4b[3]\n" + "ldr x20, [x16, #0xf8]\n" + ".inst 0x4fa2eab0 // sdot v16.4s, v21.16b, v2.4b[3]\n" + "ldr d21, [x16, #0xe0]\n" + ".inst 0x4fa0ea89 // sdot v9.4s, v20.16b, v0.4b[3]\n" + "mov v21.d[1], x21\n" + ".inst 0x4fa1ea8d // sdot v13.4s, v20.16b, v1.4b[3]\n" "add x13, x13, #0x10\n" - ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" - "ldr d7, [x16, #0xf0]\n" - "mov v7.d[1], x11\n" - "add x9, x9, #0x10\n" - "add x27, x27, #0x10\n" + ".inst 0x4fa2ea91 // sdot v17.4s, v20.16b, v2.4b[3]\n" + "ldr d20, [x16, #0xf0]\n" + "mov v20.d[1], x20\n" + "add x12, x12, #0x10\n" + "add x11, x11, #0x10\n" "add x16, x16, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - "ldr x12, [x16, #0x8]\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - "ldr x10, [x13, #0x8]\n" - ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" + ".inst 0x4fa0eaaa // sdot v10.4s, v21.16b, v0.4b[3]\n" + "ldr x20, [x16, #0x8]\n" + ".inst 0x4fa1eaae // sdot v14.4s, v21.16b, v1.4b[3]\n" + "ldr x23, [x13, #0x8]\n" + ".inst 0x4fa2eab2 // sdot v18.4s, v21.16b, v2.4b[3]\n" "ldr d6, [x16, #0x0]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x4fa0ea8b // sdot v11.4s, v20.16b, v0.4b[3]\n" "ldr d0, [x13, #0x0]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" - "ldr d1, [x9, #0x0]\n" - "ldr x28, [x9, #0x8]\n" - ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" - "ldr d2, [x27, #0x0]\n" + ".inst 0x4fa1ea8f // sdot v15.4s, v20.16b, v1.4b[3]\n" + "ldr d1, [x12, #0x0]\n" + "ldr x22, [x12, #0x8]\n" + ".inst 0x4fa2ea93 // sdot v19.4s, v20.16b, v2.4b[3]\n" + "ldr d2, [x11, #0x0]\n" "sub x14, x14, #0x10\n" "ldr d7, [x16, #0x10]\n" "cmp x14, #0x20\n" - "ldr x26, [x27, #0x8]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x16, #0x18]\n" - "mov v0.d[1], x10\n" + "ldr x21, [x11, #0x8]\n" + "mov v6.d[1], x20\n" + "ldr x20, [x16, #0x18]\n" + "mov v0.d[1], x23\n" "prfm pldl1keep, [x13, #0x80]\n" - "mov v1.d[1], x28\n" - "prfm pldl1keep, [x9, #0x80]\n" - "mov v2.d[1], x26\n" - "prfm pldl1keep, [x27, #0x80]\n" - "mov v7.d[1], x11\n" + "mov v1.d[1], x22\n" + "prfm pldl1keep, [x12, #0x80]\n" + "mov v2.d[1], x21\n" + "prfm pldl1keep, [x11, #0x80]\n" + "mov v7.d[1], x20\n" "bge 85b\n" "86:" // Height 3: Multiply loop: Single iteration only ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" "add x13, x13, #0x10\n" ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x16, #0x20]\n" + "ldr q21, [x16, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "add x27, x27, #0x10\n" + "add x11, x11, #0x10\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" "sub x14, x14, #0x10\n" ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q20, [x16, #0x30]\n" + ".inst 0x4f80e2aa // sdot v10.4s, v21.16b, v0.4b[0]\n" "prfm pldl1keep, [x13, #0x80]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "prfm pldl1keep, [x9, #0x80]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x16, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - "prfm pldl1keep, [x27, #0x80]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x16, #0x50]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" - "ldr q6, [x16, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - "ldr q7, [x16, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - "ldr q6, [x16, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" - "ldr q7, [x16, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" - "ldr q6, [x16, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" - "ldr q7, [x16, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" - "ldr q6, [x16, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" - "ldr q7, [x16, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" - "ldr q6, [x16, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" - "ldr q7, [x16, #0xf0]\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" + ".inst 0x4f81e2ae // sdot v14.4s, v21.16b, v1.4b[0]\n" + "prfm pldl1keep, [x12, #0x80]\n" + ".inst 0x4f82e2b2 // sdot v18.4s, v21.16b, v2.4b[0]\n" + "ldr q21, [x16, #0x40]\n" + ".inst 0x4f80e28b // sdot v11.4s, v20.16b, v0.4b[0]\n" + "prfm pldl1keep, [x11, #0x80]\n" + ".inst 0x4f81e28f // sdot v15.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f82e293 // sdot v19.4s, v20.16b, v2.4b[0]\n" + "ldr q20, [x16, #0x50]\n" + ".inst 0x4fa0e2a8 // sdot v8.4s, v21.16b, v0.4b[1]\n" + ".inst 0x4fa1e2ac // sdot v12.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa2e2b0 // sdot v16.4s, v21.16b, v2.4b[1]\n" + "ldr q21, [x16, #0x60]\n" + ".inst 0x4fa0e289 // sdot v9.4s, v20.16b, v0.4b[1]\n" + ".inst 0x4fa1e28d // sdot v13.4s, v20.16b, v1.4b[1]\n" + ".inst 0x4fa2e291 // sdot v17.4s, v20.16b, v2.4b[1]\n" + "ldr q20, [x16, #0x70]\n" + ".inst 0x4fa0e2aa // sdot v10.4s, v21.16b, v0.4b[1]\n" + ".inst 0x4fa1e2ae // sdot v14.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa2e2b2 // sdot v18.4s, v21.16b, v2.4b[1]\n" + "ldr q21, [x16, #0x80]\n" + ".inst 0x4fa0e28b // sdot v11.4s, v20.16b, v0.4b[1]\n" + ".inst 0x4fa1e28f // sdot v15.4s, v20.16b, v1.4b[1]\n" + ".inst 0x4fa2e293 // sdot v19.4s, v20.16b, v2.4b[1]\n" + "ldr q20, [x16, #0x90]\n" + ".inst 0x4f80eaa8 // sdot v8.4s, v21.16b, v0.4b[2]\n" + ".inst 0x4f81eaac // sdot v12.4s, v21.16b, v1.4b[2]\n" + ".inst 0x4f82eab0 // sdot v16.4s, v21.16b, v2.4b[2]\n" + "ldr q21, [x16, #0xa0]\n" + ".inst 0x4f80ea89 // sdot v9.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f81ea8d // sdot v13.4s, v20.16b, v1.4b[2]\n" + ".inst 0x4f82ea91 // sdot v17.4s, v20.16b, v2.4b[2]\n" + "ldr q20, [x16, #0xb0]\n" + ".inst 0x4f80eaaa // sdot v10.4s, v21.16b, v0.4b[2]\n" + ".inst 0x4f81eaae // sdot v14.4s, v21.16b, v1.4b[2]\n" + ".inst 0x4f82eab2 // sdot v18.4s, v21.16b, v2.4b[2]\n" + "ldr q21, [x16, #0xc0]\n" + ".inst 0x4f80ea8b // sdot v11.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f81ea8f // sdot v15.4s, v20.16b, v1.4b[2]\n" + ".inst 0x4f82ea93 // sdot v19.4s, v20.16b, v2.4b[2]\n" + "ldr q20, [x16, #0xd0]\n" + ".inst 0x4fa0eaa8 // sdot v8.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa1eaac // sdot v12.4s, v21.16b, v1.4b[3]\n" + ".inst 0x4fa2eab0 // sdot v16.4s, v21.16b, v2.4b[3]\n" + "ldr q21, [x16, #0xe0]\n" + ".inst 0x4fa0ea89 // sdot v9.4s, v20.16b, v0.4b[3]\n" + ".inst 0x4fa1ea8d // sdot v13.4s, v20.16b, v1.4b[3]\n" + ".inst 0x4fa2ea91 // sdot v17.4s, v20.16b, v2.4b[3]\n" + "ldr q20, [x16, #0xf0]\n" + ".inst 0x4fa0eaaa // sdot v10.4s, v21.16b, v0.4b[3]\n" "add x16, x16, #0x100\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" + ".inst 0x4fa1eaae // sdot v14.4s, v21.16b, v1.4b[3]\n" + ".inst 0x4fa2eab2 // sdot v18.4s, v21.16b, v2.4b[3]\n" + ".inst 0x4fa0ea8b // sdot v11.4s, v20.16b, v0.4b[3]\n" + ".inst 0x4fa1ea8f // sdot v15.4s, v20.16b, v1.4b[3]\n" + ".inst 0x4fa2ea93 // sdot v19.4s, v20.16b, v2.4b[3]\n" "87:" // Height 3: Multiply loop: Main loop skip "cbz x14, 92f\n" "cmp x14, #0x4\n" "blt 89f\n" "88:" // Height 3: Multiply loop: Odd block loop - "ldr s0, [x13], #0x4\n" + "ldr s24, [x13], #0x4\n" "sub x14, x14, #0x4\n" - "ldr s1, [x9], #0x4\n" + "ldr s23, [x12], #0x4\n" "cmp x14, #0x4\n" - "ldr s2, [x27], #0x4\n" - "ldr q6, [x16, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x10]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x16, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr s22, [x11], #0x4\n" + "ldr q21, [x16, #0x0]\n" + ".inst 0x4f98e2a8 // sdot v8.4s, v21.16b, v24.4b[0]\n" + "ldr q20, [x16, #0x10]\n" + ".inst 0x4f97e2ac // sdot v12.4s, v21.16b, v23.4b[0]\n" + ".inst 0x4f96e2b0 // sdot v16.4s, v21.16b, v22.4b[0]\n" + "ldr q21, [x16, #0x20]\n" + ".inst 0x4f98e289 // sdot v9.4s, v20.16b, v24.4b[0]\n" + ".inst 0x4f97e28d // sdot v13.4s, v20.16b, v23.4b[0]\n" + ".inst 0x4f96e291 // sdot v17.4s, v20.16b, v22.4b[0]\n" + "ldr q20, [x16, #0x30]\n" + ".inst 0x4f98e2aa // sdot v10.4s, v21.16b, v24.4b[0]\n" "add x16, x16, #0x40\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" + ".inst 0x4f97e2ae // sdot v14.4s, v21.16b, v23.4b[0]\n" + ".inst 0x4f96e2b2 // sdot v18.4s, v21.16b, v22.4b[0]\n" + ".inst 0x4f98e28b // sdot v11.4s, v20.16b, v24.4b[0]\n" + ".inst 0x4f97e28f // sdot v15.4s, v20.16b, v23.4b[0]\n" + ".inst 0x4f96e293 // sdot v19.4s, v20.16b, v22.4b[0]\n" "bge 88b\n" "89:" // Height 3: Multiply loop: Skip odd blocks "cbz x14, 92f\n" "tbz x14, #1, 90f\n" "ldr h0, [x13], #0x2\n" - "ldr h1, [x9], #0x2\n" - "ldr h2, [x27], #0x2\n" + "ldr h1, [x12], #0x2\n" + "ldr h2, [x11], #0x2\n" "tbz x14, #0, 91f\n" "ld1 { v0.b }[2], [x13]\n" - "ld1 { v1.b }[2], [x9]\n" - "ld1 { v2.b }[2], [x27]\n" + "ld1 { v1.b }[2], [x12]\n" + "ld1 { v2.b }[2], [x11]\n" "b 91f\n" "90:" // Height 3: Multiply loop: Ragged operand read: partial_1_0 "ldr b0, [x13, #0x0]\n" - "ldr b1, [x9, #0x0]\n" - "ldr b2, [x27, #0x0]\n" + "ldr b1, [x12, #0x0]\n" + "ldr b2, [x11, #0x0]\n" "91:" // Height 3: Multiply loop: Ragged operand read: Done - "ldr q6, [x16, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x10]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x16, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q21, [x16, #0x0]\n" + ".inst 0x4f80e2a8 // sdot v8.4s, v21.16b, v0.4b[0]\n" + "ldr q20, [x16, #0x10]\n" + ".inst 0x4f81e2ac // sdot v12.4s, v21.16b, v1.4b[0]\n" + ".inst 0x4f82e2b0 // sdot v16.4s, v21.16b, v2.4b[0]\n" + "ldr q21, [x16, #0x20]\n" + ".inst 0x4f80e289 // sdot v9.4s, v20.16b, v0.4b[0]\n" + ".inst 0x4f81e28d // sdot v13.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f82e291 // sdot v17.4s, v20.16b, v2.4b[0]\n" + "ldr q20, [x16, #0x30]\n" + ".inst 0x4f80e2aa // sdot v10.4s, v21.16b, v0.4b[0]\n" "add x16, x16, #0x40\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" + ".inst 0x4f81e2ae // sdot v14.4s, v21.16b, v1.4b[0]\n" + ".inst 0x4f82e2b2 // sdot v18.4s, v21.16b, v2.4b[0]\n" + ".inst 0x4f80e28b // sdot v11.4s, v20.16b, v0.4b[0]\n" + ".inst 0x4f81e28f // sdot v15.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f82e293 // sdot v19.4s, v20.16b, v2.4b[0]\n" "92:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x15, x15, #0x1\n" @@ -1475,336 +1474,336 @@ void a64_hybrid_s8s32_dot_6x16_a55 ( "116:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w14, [x20, x15, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 117f\n" - "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x13, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" - "ldr x27, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x15, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x13, [x20, #0x0]\n" + "ldr x12, [x20, #0x8]\n" + "ldr x11, [x20, #0x10]\n" + "ldr x10, [x20, #0x18]\n" "cbnz x15, 118f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x13, x13, x20\n" - "add x9, x9, x20\n" - "add x27, x27, x20\n" - "add x25, x25, x20\n" + "add x12, x12, x20\n" + "add x11, x11, x20\n" + "add x10, x10, x20\n" "b 118f\n" "117:" // Height 4: setup direct input "mov x13, %x[input_ptr]\n" - "add x9, x13, x20\n" - "add x27, x9, x20\n" - "add x25, x27, x20\n" + "add x12, x13, x21\n" + "add x11, x12, x21\n" + "add x10, x11, x21\n" "118:" // Height 4: input setup done "cmp x14, #0x10\n" "blt 121f\n" "ldr q0, [x13, #0x0]\n" "cmp x14, #0x20\n" - "ldr q1, [x9, #0x0]\n" - "ldr q2, [x27, #0x0]\n" - "ldr q3, [x25, #0x0]\n" + "ldr q1, [x12, #0x0]\n" + "ldr q2, [x11, #0x0]\n" + "ldr q3, [x10, #0x0]\n" "ldr q6, [x16, #0x0]\n" "ldr q7, [x16, #0x10]\n" "blt 120f\n" "119:" // Height 4: Multiply loop: Main loop head ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr x12, [x16, #0x28]\n" + "ldr x21, [x16, #0x28]\n" ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr x11, [x16, #0x38]\n" + "ldr x20, [x16, #0x38]\n" ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" "add x13, x13, #0x10\n" ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "ldr d6, [x16, #0x20]\n" + "ldr d25, [x16, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x12\n" + "mov v25.d[1], x21\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "ldr x12, [x16, #0x48]\n" + "ldr x21, [x16, #0x48]\n" ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - "ldr d7, [x16, #0x30]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "ldr x11, [x16, #0x58]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - "add x27, x27, #0x10\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - "ldr d6, [x16, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x12\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - "ldr x12, [x16, #0x68]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - "add x25, x25, #0x10\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - "ldr d7, [x16, #0x50]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - "ldr x11, [x16, #0x78]\n" - ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" - "ldr x10, [x13, #0x8]\n" - ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n" - "ldr d6, [x16, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - "mov v6.d[1], x12\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - "ldr x12, [x16, #0x88]\n" - ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - "ldr x28, [x9, #0x8]\n" - ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n" - "ldr d7, [x16, #0x70]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - "ldr x11, [x16, #0x98]\n" - ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - "ldr x26, [x27, #0x8]\n" - ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n" - "ldr d6, [x16, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - "mov v6.d[1], x12\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - "ldr x12, [x16, #0xa8]\n" - ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" - "ldr x24, [x25, #0x8]\n" - ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n" - "ldr d7, [x16, #0x90]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - "ldr x11, [x16, #0xb8]\n" - ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" + "ldr d24, [x16, #0x30]\n" + "mov v24.d[1], x20\n" + ".inst 0x4f80e32a // sdot v10.4s, v25.16b, v0.4b[0]\n" + ".inst 0x4f81e32e // sdot v14.4s, v25.16b, v1.4b[0]\n" + "ldr x20, [x16, #0x58]\n" + ".inst 0x4f82e332 // sdot v18.4s, v25.16b, v2.4b[0]\n" + "add x11, x11, #0x10\n" + ".inst 0x4f83e336 // sdot v22.4s, v25.16b, v3.4b[0]\n" + "ldr d25, [x16, #0x40]\n" + ".inst 0x4f80e30b // sdot v11.4s, v24.16b, v0.4b[0]\n" + "mov v25.d[1], x21\n" + ".inst 0x4f81e30f // sdot v15.4s, v24.16b, v1.4b[0]\n" + "ldr x21, [x16, #0x68]\n" + ".inst 0x4f82e313 // sdot v19.4s, v24.16b, v2.4b[0]\n" + "add x10, x10, #0x10\n" + ".inst 0x4f83e317 // sdot v23.4s, v24.16b, v3.4b[0]\n" + "ldr d24, [x16, #0x50]\n" + "mov v24.d[1], x20\n" + ".inst 0x4fa0e328 // sdot v8.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa1e32c // sdot v12.4s, v25.16b, v1.4b[1]\n" + "ldr x20, [x16, #0x78]\n" + ".inst 0x4fa2e330 // sdot v16.4s, v25.16b, v2.4b[1]\n" + "ldr x25, [x13, #0x8]\n" + ".inst 0x4fa3e334 // sdot v20.4s, v25.16b, v3.4b[1]\n" + "ldr d25, [x16, #0x60]\n" + ".inst 0x4fa0e309 // sdot v9.4s, v24.16b, v0.4b[1]\n" + "mov v25.d[1], x21\n" + ".inst 0x4fa1e30d // sdot v13.4s, v24.16b, v1.4b[1]\n" + "ldr x21, [x16, #0x88]\n" + ".inst 0x4fa2e311 // sdot v17.4s, v24.16b, v2.4b[1]\n" + "ldr x24, [x12, #0x8]\n" + ".inst 0x4fa3e315 // sdot v21.4s, v24.16b, v3.4b[1]\n" + "ldr d24, [x16, #0x70]\n" + "mov v24.d[1], x20\n" + ".inst 0x4fa0e32a // sdot v10.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa1e32e // sdot v14.4s, v25.16b, v1.4b[1]\n" + "ldr x20, [x16, #0x98]\n" + ".inst 0x4fa2e332 // sdot v18.4s, v25.16b, v2.4b[1]\n" + "ldr x23, [x11, #0x8]\n" + ".inst 0x4fa3e336 // sdot v22.4s, v25.16b, v3.4b[1]\n" + "ldr d25, [x16, #0x80]\n" + ".inst 0x4fa0e30b // sdot v11.4s, v24.16b, v0.4b[1]\n" + "mov v25.d[1], x21\n" + ".inst 0x4fa1e30f // sdot v15.4s, v24.16b, v1.4b[1]\n" + "ldr x21, [x16, #0xa8]\n" + ".inst 0x4fa2e313 // sdot v19.4s, v24.16b, v2.4b[1]\n" + "ldr x22, [x10, #0x8]\n" + ".inst 0x4fa3e317 // sdot v23.4s, v24.16b, v3.4b[1]\n" + "ldr d24, [x16, #0x90]\n" + "mov v24.d[1], x20\n" + ".inst 0x4f80eb28 // sdot v8.4s, v25.16b, v0.4b[2]\n" + ".inst 0x4f81eb2c // sdot v12.4s, v25.16b, v1.4b[2]\n" + "ldr x20, [x16, #0xb8]\n" + ".inst 0x4f82eb30 // sdot v16.4s, v25.16b, v2.4b[2]\n" "sub x14, x14, #0x10\n" - ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n" - "ldr d6, [x16, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - "mov v6.d[1], x12\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - "ldr x12, [x16, #0xc8]\n" - ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" + ".inst 0x4f83eb34 // sdot v20.4s, v25.16b, v3.4b[2]\n" + "ldr d25, [x16, #0xa0]\n" + ".inst 0x4f80eb09 // sdot v9.4s, v24.16b, v0.4b[2]\n" + "mov v25.d[1], x21\n" + ".inst 0x4f81eb0d // sdot v13.4s, v24.16b, v1.4b[2]\n" + "ldr x21, [x16, #0xc8]\n" + ".inst 0x4f82eb11 // sdot v17.4s, v24.16b, v2.4b[2]\n" "cmp x14, #0x20\n" - ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n" - "ldr d7, [x16, #0xb0]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - "ldr x11, [x16, #0xd8]\n" - ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" + ".inst 0x4f83eb15 // sdot v21.4s, v24.16b, v3.4b[2]\n" + "ldr d24, [x16, #0xb0]\n" + "mov v24.d[1], x20\n" + ".inst 0x4f80eb2a // sdot v10.4s, v25.16b, v0.4b[2]\n" + ".inst 0x4f81eb2e // sdot v14.4s, v25.16b, v1.4b[2]\n" + "ldr x20, [x16, #0xd8]\n" + ".inst 0x4f82eb32 // sdot v18.4s, v25.16b, v2.4b[2]\n" "prfm pldl1keep, [x13, #0x80]\n" - ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n" - "ldr d6, [x16, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - "mov v6.d[1], x12\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - "ldr x12, [x16, #0xe8]\n" - ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" - "prfm pldl1keep, [x9, #0x80]\n" - ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n" - "ldr d7, [x16, #0xd0]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - "ldr x11, [x16, #0xf8]\n" - ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" - "prfm pldl1keep, [x27, #0x80]\n" - ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n" - "ldr d6, [x16, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - "mov v6.d[1], x12\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" - ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n" - "ldr d7, [x16, #0xf0]\n" - "mov v7.d[1], x11\n" + ".inst 0x4f83eb36 // sdot v22.4s, v25.16b, v3.4b[2]\n" + "ldr d25, [x16, #0xc0]\n" + ".inst 0x4f80eb0b // sdot v11.4s, v24.16b, v0.4b[2]\n" + "mov v25.d[1], x21\n" + ".inst 0x4f81eb0f // sdot v15.4s, v24.16b, v1.4b[2]\n" + "ldr x21, [x16, #0xe8]\n" + ".inst 0x4f82eb13 // sdot v19.4s, v24.16b, v2.4b[2]\n" + "prfm pldl1keep, [x12, #0x80]\n" + ".inst 0x4f83eb17 // sdot v23.4s, v24.16b, v3.4b[2]\n" + "ldr d24, [x16, #0xd0]\n" + "mov v24.d[1], x20\n" + ".inst 0x4fa0eb28 // sdot v8.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa1eb2c // sdot v12.4s, v25.16b, v1.4b[3]\n" + "ldr x20, [x16, #0xf8]\n" + ".inst 0x4fa2eb30 // sdot v16.4s, v25.16b, v2.4b[3]\n" + "prfm pldl1keep, [x11, #0x80]\n" + ".inst 0x4fa3eb34 // sdot v20.4s, v25.16b, v3.4b[3]\n" + "ldr d25, [x16, #0xe0]\n" + ".inst 0x4fa0eb09 // sdot v9.4s, v24.16b, v0.4b[3]\n" + "mov v25.d[1], x21\n" + ".inst 0x4fa1eb0d // sdot v13.4s, v24.16b, v1.4b[3]\n" + "prfm pldl1keep, [x10, #0x80]\n" + ".inst 0x4fa2eb11 // sdot v17.4s, v24.16b, v2.4b[3]\n" + ".inst 0x4fa3eb15 // sdot v21.4s, v24.16b, v3.4b[3]\n" + "ldr d24, [x16, #0xf0]\n" + "mov v24.d[1], x20\n" "add x16, x16, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - "ldr x12, [x16, #0x8]\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - "ldr x11, [x16, #0x18]\n" - ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n" + ".inst 0x4fa0eb2a // sdot v10.4s, v25.16b, v0.4b[3]\n" + "ldr x21, [x16, #0x8]\n" + ".inst 0x4fa1eb2e // sdot v14.4s, v25.16b, v1.4b[3]\n" + "ldr x20, [x16, #0x18]\n" + ".inst 0x4fa2eb32 // sdot v18.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa3eb36 // sdot v22.4s, v25.16b, v3.4b[3]\n" "ldr d6, [x16, #0x0]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x4fa0eb0b // sdot v11.4s, v24.16b, v0.4b[3]\n" "ldr d0, [x13, #0x0]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" - "ldr d1, [x9, #0x0]\n" - ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" - "ldr d2, [x27, #0x0]\n" - ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n" - "ldr d3, [x25, #0x0]\n" + ".inst 0x4fa1eb0f // sdot v15.4s, v24.16b, v1.4b[3]\n" + "ldr d1, [x12, #0x0]\n" + ".inst 0x4fa2eb13 // sdot v19.4s, v24.16b, v2.4b[3]\n" + "ldr d2, [x11, #0x0]\n" + ".inst 0x4fa3eb17 // sdot v23.4s, v24.16b, v3.4b[3]\n" + "ldr d3, [x10, #0x0]\n" "ldr d7, [x16, #0x10]\n" - "mov v6.d[1], x12\n" - "mov v0.d[1], x10\n" - "mov v1.d[1], x28\n" - "mov v2.d[1], x26\n" - "mov v3.d[1], x24\n" - "mov v7.d[1], x11\n" + "mov v6.d[1], x21\n" + "mov v0.d[1], x25\n" + "mov v1.d[1], x24\n" + "mov v2.d[1], x23\n" + "mov v3.d[1], x22\n" + "mov v7.d[1], x20\n" "bge 119b\n" "120:" // Height 4: Multiply loop: Single iteration only ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" "add x13, x13, #0x10\n" ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - "add x27, x27, #0x10\n" + "add x11, x11, #0x10\n" ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x16, #0x20]\n" + "ldr q25, [x16, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "add x25, x25, #0x10\n" + "add x10, x10, #0x10\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" "sub x14, x14, #0x10\n" ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" "prfm pldl1keep, [x13, #0x80]\n" ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - "prfm pldl1keep, [x9, #0x80]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "prfm pldl1keep, [x27, #0x80]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x16, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x16, #0x50]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" - ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n" - "ldr q6, [x16, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n" - "ldr q7, [x16, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n" - "ldr q6, [x16, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" - ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n" - "ldr q7, [x16, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n" - "ldr q6, [x16, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n" - "ldr q7, [x16, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n" - "ldr q6, [x16, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n" - "ldr q7, [x16, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n" - "ldr q6, [x16, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" - ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n" - "ldr q7, [x16, #0xf0]\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" + "ldr q24, [x16, #0x30]\n" + ".inst 0x4f80e32a // sdot v10.4s, v25.16b, v0.4b[0]\n" + "prfm pldl1keep, [x12, #0x80]\n" + ".inst 0x4f81e32e // sdot v14.4s, v25.16b, v1.4b[0]\n" + "prfm pldl1keep, [x11, #0x80]\n" + ".inst 0x4f82e332 // sdot v18.4s, v25.16b, v2.4b[0]\n" + "prfm pldl1keep, [x10, #0x80]\n" + ".inst 0x4f83e336 // sdot v22.4s, v25.16b, v3.4b[0]\n" + "ldr q25, [x16, #0x40]\n" + ".inst 0x4f80e30b // sdot v11.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f81e30f // sdot v15.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f82e313 // sdot v19.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f83e317 // sdot v23.4s, v24.16b, v3.4b[0]\n" + "ldr q24, [x16, #0x50]\n" + ".inst 0x4fa0e328 // sdot v8.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa1e32c // sdot v12.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa2e330 // sdot v16.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa3e334 // sdot v20.4s, v25.16b, v3.4b[1]\n" + "ldr q25, [x16, #0x60]\n" + ".inst 0x4fa0e309 // sdot v9.4s, v24.16b, v0.4b[1]\n" + ".inst 0x4fa1e30d // sdot v13.4s, v24.16b, v1.4b[1]\n" + ".inst 0x4fa2e311 // sdot v17.4s, v24.16b, v2.4b[1]\n" + ".inst 0x4fa3e315 // sdot v21.4s, v24.16b, v3.4b[1]\n" + "ldr q24, [x16, #0x70]\n" + ".inst 0x4fa0e32a // sdot v10.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa1e32e // sdot v14.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa2e332 // sdot v18.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa3e336 // sdot v22.4s, v25.16b, v3.4b[1]\n" + "ldr q25, [x16, #0x80]\n" + ".inst 0x4fa0e30b // sdot v11.4s, v24.16b, v0.4b[1]\n" + ".inst 0x4fa1e30f // sdot v15.4s, v24.16b, v1.4b[1]\n" + ".inst 0x4fa2e313 // sdot v19.4s, v24.16b, v2.4b[1]\n" + ".inst 0x4fa3e317 // sdot v23.4s, v24.16b, v3.4b[1]\n" + "ldr q24, [x16, #0x90]\n" + ".inst 0x4f80eb28 // sdot v8.4s, v25.16b, v0.4b[2]\n" + ".inst 0x4f81eb2c // sdot v12.4s, v25.16b, v1.4b[2]\n" + ".inst 0x4f82eb30 // sdot v16.4s, v25.16b, v2.4b[2]\n" + ".inst 0x4f83eb34 // sdot v20.4s, v25.16b, v3.4b[2]\n" + "ldr q25, [x16, #0xa0]\n" + ".inst 0x4f80eb09 // sdot v9.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f81eb0d // sdot v13.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f82eb11 // sdot v17.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f83eb15 // sdot v21.4s, v24.16b, v3.4b[2]\n" + "ldr q24, [x16, #0xb0]\n" + ".inst 0x4f80eb2a // sdot v10.4s, v25.16b, v0.4b[2]\n" + ".inst 0x4f81eb2e // sdot v14.4s, v25.16b, v1.4b[2]\n" + ".inst 0x4f82eb32 // sdot v18.4s, v25.16b, v2.4b[2]\n" + ".inst 0x4f83eb36 // sdot v22.4s, v25.16b, v3.4b[2]\n" + "ldr q25, [x16, #0xc0]\n" + ".inst 0x4f80eb0b // sdot v11.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f81eb0f // sdot v15.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f82eb13 // sdot v19.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f83eb17 // sdot v23.4s, v24.16b, v3.4b[2]\n" + "ldr q24, [x16, #0xd0]\n" + ".inst 0x4fa0eb28 // sdot v8.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa1eb2c // sdot v12.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa2eb30 // sdot v16.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa3eb34 // sdot v20.4s, v25.16b, v3.4b[3]\n" + "ldr q25, [x16, #0xe0]\n" + ".inst 0x4fa0eb09 // sdot v9.4s, v24.16b, v0.4b[3]\n" + ".inst 0x4fa1eb0d // sdot v13.4s, v24.16b, v1.4b[3]\n" + ".inst 0x4fa2eb11 // sdot v17.4s, v24.16b, v2.4b[3]\n" + ".inst 0x4fa3eb15 // sdot v21.4s, v24.16b, v3.4b[3]\n" + "ldr q24, [x16, #0xf0]\n" + ".inst 0x4fa0eb2a // sdot v10.4s, v25.16b, v0.4b[3]\n" "add x16, x16, #0x100\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" - ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n" + ".inst 0x4fa1eb2e // sdot v14.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa2eb32 // sdot v18.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa3eb36 // sdot v22.4s, v25.16b, v3.4b[3]\n" + ".inst 0x4fa0eb0b // sdot v11.4s, v24.16b, v0.4b[3]\n" + ".inst 0x4fa1eb0f // sdot v15.4s, v24.16b, v1.4b[3]\n" + ".inst 0x4fa2eb13 // sdot v19.4s, v24.16b, v2.4b[3]\n" + ".inst 0x4fa3eb17 // sdot v23.4s, v24.16b, v3.4b[3]\n" "121:" // Height 4: Multiply loop: Main loop skip "cbz x14, 126f\n" "cmp x14, #0x4\n" "blt 123f\n" "122:" // Height 4: Multiply loop: Odd block loop - "ldr s0, [x13], #0x4\n" + "ldr s29, [x13], #0x4\n" "sub x14, x14, #0x4\n" - "ldr s1, [x9], #0x4\n" + "ldr s28, [x12], #0x4\n" "cmp x14, #0x4\n" - "ldr s2, [x27], #0x4\n" - "ldr s3, [x25], #0x4\n" - "ldr q6, [x16, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x10]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x16, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr s27, [x11], #0x4\n" + "ldr s26, [x10], #0x4\n" + "ldr q25, [x16, #0x0]\n" + ".inst 0x4f9de328 // sdot v8.4s, v25.16b, v29.4b[0]\n" + "ldr q24, [x16, #0x10]\n" + ".inst 0x4f9ce32c // sdot v12.4s, v25.16b, v28.4b[0]\n" + ".inst 0x4f9be330 // sdot v16.4s, v25.16b, v27.4b[0]\n" + ".inst 0x4f9ae334 // sdot v20.4s, v25.16b, v26.4b[0]\n" + "ldr q25, [x16, #0x20]\n" + ".inst 0x4f9de309 // sdot v9.4s, v24.16b, v29.4b[0]\n" + ".inst 0x4f9ce30d // sdot v13.4s, v24.16b, v28.4b[0]\n" + ".inst 0x4f9be311 // sdot v17.4s, v24.16b, v27.4b[0]\n" + ".inst 0x4f9ae315 // sdot v21.4s, v24.16b, v26.4b[0]\n" + "ldr q24, [x16, #0x30]\n" + ".inst 0x4f9de32a // sdot v10.4s, v25.16b, v29.4b[0]\n" "add x16, x16, #0x40\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" + ".inst 0x4f9ce32e // sdot v14.4s, v25.16b, v28.4b[0]\n" + ".inst 0x4f9be332 // sdot v18.4s, v25.16b, v27.4b[0]\n" + ".inst 0x4f9ae336 // sdot v22.4s, v25.16b, v26.4b[0]\n" + ".inst 0x4f9de30b // sdot v11.4s, v24.16b, v29.4b[0]\n" + ".inst 0x4f9ce30f // sdot v15.4s, v24.16b, v28.4b[0]\n" + ".inst 0x4f9be313 // sdot v19.4s, v24.16b, v27.4b[0]\n" + ".inst 0x4f9ae317 // sdot v23.4s, v24.16b, v26.4b[0]\n" "bge 122b\n" "123:" // Height 4: Multiply loop: Skip odd blocks "cbz x14, 126f\n" "tbz x14, #1, 124f\n" "ldr h0, [x13], #0x2\n" - "ldr h1, [x9], #0x2\n" - "ldr h2, [x27], #0x2\n" - "ldr h3, [x25], #0x2\n" + "ldr h1, [x12], #0x2\n" + "ldr h2, [x11], #0x2\n" + "ldr h3, [x10], #0x2\n" "tbz x14, #0, 125f\n" "ld1 { v0.b }[2], [x13]\n" - "ld1 { v1.b }[2], [x9]\n" - "ld1 { v2.b }[2], [x27]\n" - "ld1 { v3.b }[2], [x25]\n" + "ld1 { v1.b }[2], [x12]\n" + "ld1 { v2.b }[2], [x11]\n" + "ld1 { v3.b }[2], [x10]\n" "b 125f\n" "124:" // Height 4: Multiply loop: Ragged operand read: partial_1_0 "ldr b0, [x13, #0x0]\n" - "ldr b1, [x9, #0x0]\n" - "ldr b2, [x27, #0x0]\n" - "ldr b3, [x25, #0x0]\n" + "ldr b1, [x12, #0x0]\n" + "ldr b2, [x11, #0x0]\n" + "ldr b3, [x10, #0x0]\n" "125:" // Height 4: Multiply loop: Ragged operand read: Done - "ldr q6, [x16, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x10]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x16, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q25, [x16, #0x0]\n" + ".inst 0x4f80e328 // sdot v8.4s, v25.16b, v0.4b[0]\n" + "ldr q24, [x16, #0x10]\n" + ".inst 0x4f81e32c // sdot v12.4s, v25.16b, v1.4b[0]\n" + ".inst 0x4f82e330 // sdot v16.4s, v25.16b, v2.4b[0]\n" + ".inst 0x4f83e334 // sdot v20.4s, v25.16b, v3.4b[0]\n" + "ldr q25, [x16, #0x20]\n" + ".inst 0x4f80e309 // sdot v9.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f81e30d // sdot v13.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f82e311 // sdot v17.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f83e315 // sdot v21.4s, v24.16b, v3.4b[0]\n" + "ldr q24, [x16, #0x30]\n" + ".inst 0x4f80e32a // sdot v10.4s, v25.16b, v0.4b[0]\n" "add x16, x16, #0x40\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" + ".inst 0x4f81e32e // sdot v14.4s, v25.16b, v1.4b[0]\n" + ".inst 0x4f82e332 // sdot v18.4s, v25.16b, v2.4b[0]\n" + ".inst 0x4f83e336 // sdot v22.4s, v25.16b, v3.4b[0]\n" + ".inst 0x4f80e30b // sdot v11.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f81e30f // sdot v15.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f82e313 // sdot v19.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f83e317 // sdot v23.4s, v24.16b, v3.4b[0]\n" "126:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x15, x15, #0x1\n" @@ -2108,399 +2107,399 @@ void a64_hybrid_s8s32_dot_6x16_a55 ( "movi v22.4s, #0x0\n" "movi v23.4s, #0x0\n" "movi v24.4s, #0x0\n" - "movi v25.4s, #0x0\n" - "movi v26.4s, #0x0\n" - "movi v27.4s, #0x0\n" - "149:" // Height 5: setup done - "mov x15, #0x0\n" - "150:" // Height 5: String loop - "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr w14, [x20, x15, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" - "tbz %x[flags], #3, 151f\n" - "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x13, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" - "ldr x27, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" - "ldr x23, [x21, #0x20]\n" - "cbnz x15, 152f\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x13, x13, x20\n" - "add x9, x9, x20\n" - "add x27, x27, x20\n" - "add x25, x25, x20\n" - "add x23, x23, x20\n" - "b 152f\n" - "151:" // Height 5: setup direct input - "mov x13, %x[input_ptr]\n" - "add x9, x13, x20\n" - "add x27, x9, x20\n" - "add x25, x27, x20\n" - "add x23, x25, x20\n" - "152:" // Height 5: input setup done - "cmp x14, #0x10\n" - "blt 155f\n" - "ldr q0, [x13, #0x0]\n" - "cmp x14, #0x20\n" - "ldr q1, [x9, #0x0]\n" - "ldr q2, [x27, #0x0]\n" - "ldr q3, [x25, #0x0]\n" - "ldr q4, [x23, #0x0]\n" - "ldr q6, [x16, #0x0]\n" - "ldr q7, [x16, #0x10]\n" - "blt 154f\n" - "153:" // Height 5: Multiply loop: Main loop head - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr x12, [x16, #0x28]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr x11, [x16, #0x38]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - "add x13, x13, #0x10\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "add x9, x9, #0x10\n" - ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - "ldr d6, [x16, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x12\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "ldr x12, [x16, #0x48]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - "add x27, x27, #0x10\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - "add x25, x25, #0x10\n" - ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - "ldr d7, [x16, #0x30]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "ldr x11, [x16, #0x58]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - "add x23, x23, #0x10\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - "ldr x10, [x13, #0x8]\n" - ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - "ldr d6, [x16, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x12\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - "ldr x12, [x16, #0x68]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - "ldr x28, [x9, #0x8]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - "ldr x26, [x27, #0x8]\n" - ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" - "ldr d7, [x16, #0x50]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - "ldr x11, [x16, #0x78]\n" - ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" - "ldr x24, [x25, #0x8]\n" - ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n" - "ldr x22, [x23, #0x8]\n" - ".inst 0x4fa4e0d8 // sdot v24.4s, v6.16b, v4.4b[1]\n" - "ldr d6, [x16, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - "mov v6.d[1], x12\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - "ldr x12, [x16, #0x88]\n" - ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - "sub x14, x14, #0x10\n" - ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n" - "cmp x14, #0x20\n" - ".inst 0x4fa4e0f9 // sdot v25.4s, v7.16b, v4.4b[1]\n" - "ldr d7, [x16, #0x70]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - "ldr x11, [x16, #0x98]\n" - ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - "prfm pldl1keep, [x13, #0x80]\n" - ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n" - "prfm pldl1keep, [x9, #0x80]\n" - ".inst 0x4fa4e0da // sdot v26.4s, v6.16b, v4.4b[1]\n" - "ldr d6, [x16, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - "mov v6.d[1], x12\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - "ldr x12, [x16, #0xa8]\n" - ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" - "prfm pldl1keep, [x27, #0x80]\n" - ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n" - "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4fa4e0fb // sdot v27.4s, v7.16b, v4.4b[1]\n" - "ldr d7, [x16, #0x90]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - "ldr x11, [x16, #0xb8]\n" - ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" - "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n" - ".inst 0x4f84e8d8 // sdot v24.4s, v6.16b, v4.4b[2]\n" - "ldr d6, [x16, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - "mov v6.d[1], x12\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - "ldr x12, [x16, #0xc8]\n" - ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n" - ".inst 0x4f84e8f9 // sdot v25.4s, v7.16b, v4.4b[2]\n" - "ldr d7, [x16, #0xb0]\n" - "mov v7.d[1], x11\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - "ldr x11, [x16, #0xd8]\n" - ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n" - ".inst 0x4f84e8da // sdot v26.4s, v6.16b, v4.4b[2]\n" - "ldr d6, [x16, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - "mov v6.d[1], x12\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - "ldr x12, [x16, #0xe8]\n" - ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n" - ".inst 0x4f84e8fb // sdot v27.4s, v7.16b, v4.4b[2]\n" - "ldr d7, [x16, #0xd0]\n" - "mov v7.d[1], x11\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - "ldr x11, [x16, #0xf8]\n" - ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n" - ".inst 0x4fa4e8d8 // sdot v24.4s, v6.16b, v4.4b[3]\n" - "ldr d6, [x16, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - "mov v6.d[1], x12\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" - ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n" - ".inst 0x4fa4e8f9 // sdot v25.4s, v7.16b, v4.4b[3]\n" - "ldr d7, [x16, #0xf0]\n" - "mov v7.d[1], x11\n" + "movi v25.4s, #0x0\n" + "movi v26.4s, #0x0\n" + "movi v27.4s, #0x0\n" + "149:" // Height 5: setup done + "mov x15, #0x0\n" + "150:" // Height 5: String loop + "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" + "ldr w14, [x20, x15, LSL #0x2]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" + "tbz %x[flags], #3, 151f\n" + "ldr x20, [%x[input_ptr], x15, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x13, [x20, #0x0]\n" + "ldr x12, [x20, #0x8]\n" + "ldr x11, [x20, #0x10]\n" + "ldr x10, [x20, #0x18]\n" + "ldr x9, [x20, #0x20]\n" + "cbnz x15, 152f\n" + "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" + "add x13, x13, x20\n" + "add x12, x12, x20\n" + "add x11, x11, x20\n" + "add x10, x10, x20\n" + "add x9, x9, x20\n" + "b 152f\n" + "151:" // Height 5: setup direct input + "mov x13, %x[input_ptr]\n" + "add x12, x13, x21\n" + "add x11, x12, x21\n" + "add x10, x11, x21\n" + "add x9, x10, x21\n" + "152:" // Height 5: input setup done + "cmp x14, #0x10\n" + "blt 155f\n" + "ldr q0, [x13, #0x0]\n" + "cmp x14, #0x20\n" + "ldr q1, [x12, #0x0]\n" + "ldr q2, [x11, #0x0]\n" + "ldr q3, [x10, #0x0]\n" + "ldr q4, [x9, #0x0]\n" + "ldr q6, [x16, #0x0]\n" + "ldr q7, [x16, #0x10]\n" + "blt 154f\n" + "153:" // Height 5: Multiply loop: Main loop head + ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" + "ldr x21, [x16, #0x28]\n" + ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" + "ldr x20, [x16, #0x38]\n" + ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" + "add x13, x13, #0x10\n" + ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" + "add x12, x12, #0x10\n" + ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" + "ldr d29, [x16, #0x20]\n" + ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" + "mov v29.d[1], x21\n" + ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" + "ldr x21, [x16, #0x48]\n" + ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" + "add x11, x11, #0x10\n" + ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" + "add x10, x10, #0x10\n" + ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" + "ldr d28, [x16, #0x30]\n" + "mov v28.d[1], x20\n" + ".inst 0x4f80e3aa // sdot v10.4s, v29.16b, v0.4b[0]\n" + ".inst 0x4f81e3ae // sdot v14.4s, v29.16b, v1.4b[0]\n" + "ldr x20, [x16, #0x58]\n" + ".inst 0x4f82e3b2 // sdot v18.4s, v29.16b, v2.4b[0]\n" + "add x9, x9, #0x10\n" + ".inst 0x4f83e3b6 // sdot v22.4s, v29.16b, v3.4b[0]\n" + "ldr x26, [x13, #0x8]\n" + ".inst 0x4f84e3ba // sdot v26.4s, v29.16b, v4.4b[0]\n" + "ldr d29, [x16, #0x40]\n" + ".inst 0x4f80e38b // sdot v11.4s, v28.16b, v0.4b[0]\n" + "mov v29.d[1], x21\n" + ".inst 0x4f81e38f // sdot v15.4s, v28.16b, v1.4b[0]\n" + "ldr x21, [x16, #0x68]\n" + ".inst 0x4f82e393 // sdot v19.4s, v28.16b, v2.4b[0]\n" + "ldr x25, [x12, #0x8]\n" + ".inst 0x4f83e397 // sdot v23.4s, v28.16b, v3.4b[0]\n" + "ldr x24, [x11, #0x8]\n" + ".inst 0x4f84e39b // sdot v27.4s, v28.16b, v4.4b[0]\n" + "ldr d28, [x16, #0x50]\n" + "mov v28.d[1], x20\n" + ".inst 0x4fa0e3a8 // sdot v8.4s, v29.16b, v0.4b[1]\n" + ".inst 0x4fa1e3ac // sdot v12.4s, v29.16b, v1.4b[1]\n" + "ldr x20, [x16, #0x78]\n" + ".inst 0x4fa2e3b0 // sdot v16.4s, v29.16b, v2.4b[1]\n" + "ldr x23, [x10, #0x8]\n" + ".inst 0x4fa3e3b4 // sdot v20.4s, v29.16b, v3.4b[1]\n" + "ldr x22, [x9, #0x8]\n" + ".inst 0x4fa4e3b8 // sdot v24.4s, v29.16b, v4.4b[1]\n" + "ldr d29, [x16, #0x60]\n" + ".inst 0x4fa0e389 // sdot v9.4s, v28.16b, v0.4b[1]\n" + "mov v29.d[1], x21\n" + ".inst 0x4fa1e38d // sdot v13.4s, v28.16b, v1.4b[1]\n" + "ldr x21, [x16, #0x88]\n" + ".inst 0x4fa2e391 // sdot v17.4s, v28.16b, v2.4b[1]\n" + "sub x14, x14, #0x10\n" + ".inst 0x4fa3e395 // sdot v21.4s, v28.16b, v3.4b[1]\n" + "cmp x14, #0x20\n" + ".inst 0x4fa4e399 // sdot v25.4s, v28.16b, v4.4b[1]\n" + "ldr d28, [x16, #0x70]\n" + "mov v28.d[1], x20\n" + ".inst 0x4fa0e3aa // sdot v10.4s, v29.16b, v0.4b[1]\n" + ".inst 0x4fa1e3ae // sdot v14.4s, v29.16b, v1.4b[1]\n" + "ldr x20, [x16, #0x98]\n" + ".inst 0x4fa2e3b2 // sdot v18.4s, v29.16b, v2.4b[1]\n" + "prfm pldl1keep, [x13, #0x80]\n" + ".inst 0x4fa3e3b6 // sdot v22.4s, v29.16b, v3.4b[1]\n" + "prfm pldl1keep, [x12, #0x80]\n" + ".inst 0x4fa4e3ba // sdot v26.4s, v29.16b, v4.4b[1]\n" + "ldr d29, [x16, #0x80]\n" + ".inst 0x4fa0e38b // sdot v11.4s, v28.16b, v0.4b[1]\n" + "mov v29.d[1], x21\n" + ".inst 0x4fa1e38f // sdot v15.4s, v28.16b, v1.4b[1]\n" + "ldr x21, [x16, #0xa8]\n" + ".inst 0x4fa2e393 // sdot v19.4s, v28.16b, v2.4b[1]\n" + "prfm pldl1keep, [x11, #0x80]\n" + ".inst 0x4fa3e397 // sdot v23.4s, v28.16b, v3.4b[1]\n" + "prfm pldl1keep, [x10, #0x80]\n" + ".inst 0x4fa4e39b // sdot v27.4s, v28.16b, v4.4b[1]\n" + "ldr d28, [x16, #0x90]\n" + "mov v28.d[1], x20\n" + ".inst 0x4f80eba8 // sdot v8.4s, v29.16b, v0.4b[2]\n" + ".inst 0x4f81ebac // sdot v12.4s, v29.16b, v1.4b[2]\n" + "ldr x20, [x16, #0xb8]\n" + ".inst 0x4f82ebb0 // sdot v16.4s, v29.16b, v2.4b[2]\n" + "prfm pldl1keep, [x9, #0x80]\n" + ".inst 0x4f83ebb4 // sdot v20.4s, v29.16b, v3.4b[2]\n" + ".inst 0x4f84ebb8 // sdot v24.4s, v29.16b, v4.4b[2]\n" + "ldr d29, [x16, #0xa0]\n" + ".inst 0x4f80eb89 // sdot v9.4s, v28.16b, v0.4b[2]\n" + "mov v29.d[1], x21\n" + ".inst 0x4f81eb8d // sdot v13.4s, v28.16b, v1.4b[2]\n" + "ldr x21, [x16, #0xc8]\n" + ".inst 0x4f82eb91 // sdot v17.4s, v28.16b, v2.4b[2]\n" + ".inst 0x4f83eb95 // sdot v21.4s, v28.16b, v3.4b[2]\n" + ".inst 0x4f84eb99 // sdot v25.4s, v28.16b, v4.4b[2]\n" + "ldr d28, [x16, #0xb0]\n" + "mov v28.d[1], x20\n" + ".inst 0x4f80ebaa // sdot v10.4s, v29.16b, v0.4b[2]\n" + ".inst 0x4f81ebae // sdot v14.4s, v29.16b, v1.4b[2]\n" + "ldr x20, [x16, #0xd8]\n" + ".inst 0x4f82ebb2 // sdot v18.4s, v29.16b, v2.4b[2]\n" + ".inst 0x4f83ebb6 // sdot v22.4s, v29.16b, v3.4b[2]\n" + ".inst 0x4f84ebba // sdot v26.4s, v29.16b, v4.4b[2]\n" + "ldr d29, [x16, #0xc0]\n" + ".inst 0x4f80eb8b // sdot v11.4s, v28.16b, v0.4b[2]\n" + "mov v29.d[1], x21\n" + ".inst 0x4f81eb8f // sdot v15.4s, v28.16b, v1.4b[2]\n" + "ldr x21, [x16, #0xe8]\n" + ".inst 0x4f82eb93 // sdot v19.4s, v28.16b, v2.4b[2]\n" + ".inst 0x4f83eb97 // sdot v23.4s, v28.16b, v3.4b[2]\n" + ".inst 0x4f84eb9b // sdot v27.4s, v28.16b, v4.4b[2]\n" + "ldr d28, [x16, #0xd0]\n" + "mov v28.d[1], x20\n" + ".inst 0x4fa0eba8 // sdot v8.4s, v29.16b, v0.4b[3]\n" + ".inst 0x4fa1ebac // sdot v12.4s, v29.16b, v1.4b[3]\n" + "ldr x20, [x16, #0xf8]\n" + ".inst 0x4fa2ebb0 // sdot v16.4s, v29.16b, v2.4b[3]\n" + ".inst 0x4fa3ebb4 // sdot v20.4s, v29.16b, v3.4b[3]\n" + ".inst 0x4fa4ebb8 // sdot v24.4s, v29.16b, v4.4b[3]\n" + "ldr d29, [x16, #0xe0]\n" + ".inst 0x4fa0eb89 // sdot v9.4s, v28.16b, v0.4b[3]\n" + "mov v29.d[1], x21\n" + ".inst 0x4fa1eb8d // sdot v13.4s, v28.16b, v1.4b[3]\n" + ".inst 0x4fa2eb91 // sdot v17.4s, v28.16b, v2.4b[3]\n" + ".inst 0x4fa3eb95 // sdot v21.4s, v28.16b, v3.4b[3]\n" + ".inst 0x4fa4eb99 // sdot v25.4s, v28.16b, v4.4b[3]\n" + "ldr d28, [x16, #0xf0]\n" + "mov v28.d[1], x20\n" "add x16, x16, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - "ldr x12, [x16, #0x8]\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - "ldr x11, [x16, #0x18]\n" - ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n" - ".inst 0x4fa4e8da // sdot v26.4s, v6.16b, v4.4b[3]\n" + ".inst 0x4fa0ebaa // sdot v10.4s, v29.16b, v0.4b[3]\n" + "ldr x21, [x16, #0x8]\n" + ".inst 0x4fa1ebae // sdot v14.4s, v29.16b, v1.4b[3]\n" + "ldr x20, [x16, #0x18]\n" + ".inst 0x4fa2ebb2 // sdot v18.4s, v29.16b, v2.4b[3]\n" + ".inst 0x4fa3ebb6 // sdot v22.4s, v29.16b, v3.4b[3]\n" + ".inst 0x4fa4ebba // sdot v26.4s, v29.16b, v4.4b[3]\n" "ldr d6, [x16, #0x0]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x4fa0eb8b // sdot v11.4s, v28.16b, v0.4b[3]\n" "ldr d0, [x13, #0x0]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" - "ldr d1, [x9, #0x0]\n" - ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" - "ldr d2, [x27, #0x0]\n" - ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n" - "ldr d3, [x25, #0x0]\n" - ".inst 0x4fa4e8fb // sdot v27.4s, v7.16b, v4.4b[3]\n" - "ldr d4, [x23, #0x0]\n" + ".inst 0x4fa1eb8f // sdot v15.4s, v28.16b, v1.4b[3]\n" + "ldr d1, [x12, #0x0]\n" + ".inst 0x4fa2eb93 // sdot v19.4s, v28.16b, v2.4b[3]\n" + "ldr d2, [x11, #0x0]\n" + ".inst 0x4fa3eb97 // sdot v23.4s, v28.16b, v3.4b[3]\n" + "ldr d3, [x10, #0x0]\n" + ".inst 0x4fa4eb9b // sdot v27.4s, v28.16b, v4.4b[3]\n" + "ldr d4, [x9, #0x0]\n" "ldr d7, [x16, #0x10]\n" - "mov v6.d[1], x12\n" - "mov v0.d[1], x10\n" - "mov v1.d[1], x28\n" - "mov v2.d[1], x26\n" - "mov v3.d[1], x24\n" + "mov v6.d[1], x21\n" + "mov v0.d[1], x26\n" + "mov v1.d[1], x25\n" + "mov v2.d[1], x24\n" + "mov v3.d[1], x23\n" "mov v4.d[1], x22\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "bge 153b\n" "154:" // Height 5: Multiply loop: Single iteration only ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" "add x13, x13, #0x10\n" ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - "add x27, x27, #0x10\n" + "add x11, x11, #0x10\n" ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "add x25, x25, #0x10\n" + "add x10, x10, #0x10\n" ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x16, #0x20]\n" + "ldr q29, [x16, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "add x23, x23, #0x10\n" + "add x9, x9, #0x10\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" "sub x14, x14, #0x10\n" ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" "prfm pldl1keep, [x13, #0x80]\n" ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x12, #0x80]\n" ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - "prfm pldl1keep, [x27, #0x80]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x16, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x16, #0x50]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" - ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n" - ".inst 0x4fa4e0d8 // sdot v24.4s, v6.16b, v4.4b[1]\n" - "ldr q6, [x16, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n" - ".inst 0x4fa4e0f9 // sdot v25.4s, v7.16b, v4.4b[1]\n" - "ldr q7, [x16, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n" - ".inst 0x4fa4e0da // sdot v26.4s, v6.16b, v4.4b[1]\n" - "ldr q6, [x16, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" - ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n" - ".inst 0x4fa4e0fb // sdot v27.4s, v7.16b, v4.4b[1]\n" - "ldr q7, [x16, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n" - ".inst 0x4f84e8d8 // sdot v24.4s, v6.16b, v4.4b[2]\n" - "ldr q6, [x16, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n" - ".inst 0x4f84e8f9 // sdot v25.4s, v7.16b, v4.4b[2]\n" - "ldr q7, [x16, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n" - ".inst 0x4f84e8da // sdot v26.4s, v6.16b, v4.4b[2]\n" - "ldr q6, [x16, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n" - ".inst 0x4f84e8fb // sdot v27.4s, v7.16b, v4.4b[2]\n" - "ldr q7, [x16, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n" - ".inst 0x4fa4e8d8 // sdot v24.4s, v6.16b, v4.4b[3]\n" - "ldr q6, [x16, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" - ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n" - ".inst 0x4fa4e8f9 // sdot v25.4s, v7.16b, v4.4b[3]\n" - "ldr q7, [x16, #0xf0]\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" + "ldr q28, [x16, #0x30]\n" + ".inst 0x4f80e3aa // sdot v10.4s, v29.16b, v0.4b[0]\n" + "prfm pldl1keep, [x11, #0x80]\n" + ".inst 0x4f81e3ae // sdot v14.4s, v29.16b, v1.4b[0]\n" + "prfm pldl1keep, [x10, #0x80]\n" + ".inst 0x4f82e3b2 // sdot v18.4s, v29.16b, v2.4b[0]\n" + "prfm pldl1keep, [x9, #0x80]\n" + ".inst 0x4f83e3b6 // sdot v22.4s, v29.16b, v3.4b[0]\n" + ".inst 0x4f84e3ba // sdot v26.4s, v29.16b, v4.4b[0]\n" + "ldr q29, [x16, #0x40]\n" + ".inst 0x4f80e38b // sdot v11.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f81e38f // sdot v15.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f82e393 // sdot v19.4s, v28.16b, v2.4b[0]\n" + ".inst 0x4f83e397 // sdot v23.4s, v28.16b, v3.4b[0]\n" + ".inst 0x4f84e39b // sdot v27.4s, v28.16b, v4.4b[0]\n" + "ldr q28, [x16, #0x50]\n" + ".inst 0x4fa0e3a8 // sdot v8.4s, v29.16b, v0.4b[1]\n" + ".inst 0x4fa1e3ac // sdot v12.4s, v29.16b, v1.4b[1]\n" + ".inst 0x4fa2e3b0 // sdot v16.4s, v29.16b, v2.4b[1]\n" + ".inst 0x4fa3e3b4 // sdot v20.4s, v29.16b, v3.4b[1]\n" + ".inst 0x4fa4e3b8 // sdot v24.4s, v29.16b, v4.4b[1]\n" + "ldr q29, [x16, #0x60]\n" + ".inst 0x4fa0e389 // sdot v9.4s, v28.16b, v0.4b[1]\n" + ".inst 0x4fa1e38d // sdot v13.4s, v28.16b, v1.4b[1]\n" + ".inst 0x4fa2e391 // sdot v17.4s, v28.16b, v2.4b[1]\n" + ".inst 0x4fa3e395 // sdot v21.4s, v28.16b, v3.4b[1]\n" + ".inst 0x4fa4e399 // sdot v25.4s, v28.16b, v4.4b[1]\n" + "ldr q28, [x16, #0x70]\n" + ".inst 0x4fa0e3aa // sdot v10.4s, v29.16b, v0.4b[1]\n" + ".inst 0x4fa1e3ae // sdot v14.4s, v29.16b, v1.4b[1]\n" + ".inst 0x4fa2e3b2 // sdot v18.4s, v29.16b, v2.4b[1]\n" + ".inst 0x4fa3e3b6 // sdot v22.4s, v29.16b, v3.4b[1]\n" + ".inst 0x4fa4e3ba // sdot v26.4s, v29.16b, v4.4b[1]\n" + "ldr q29, [x16, #0x80]\n" + ".inst 0x4fa0e38b // sdot v11.4s, v28.16b, v0.4b[1]\n" + ".inst 0x4fa1e38f // sdot v15.4s, v28.16b, v1.4b[1]\n" + ".inst 0x4fa2e393 // sdot v19.4s, v28.16b, v2.4b[1]\n" + ".inst 0x4fa3e397 // sdot v23.4s, v28.16b, v3.4b[1]\n" + ".inst 0x4fa4e39b // sdot v27.4s, v28.16b, v4.4b[1]\n" + "ldr q28, [x16, #0x90]\n" + ".inst 0x4f80eba8 // sdot v8.4s, v29.16b, v0.4b[2]\n" + ".inst 0x4f81ebac // sdot v12.4s, v29.16b, v1.4b[2]\n" + ".inst 0x4f82ebb0 // sdot v16.4s, v29.16b, v2.4b[2]\n" + ".inst 0x4f83ebb4 // sdot v20.4s, v29.16b, v3.4b[2]\n" + ".inst 0x4f84ebb8 // sdot v24.4s, v29.16b, v4.4b[2]\n" + "ldr q29, [x16, #0xa0]\n" + ".inst 0x4f80eb89 // sdot v9.4s, v28.16b, v0.4b[2]\n" + ".inst 0x4f81eb8d // sdot v13.4s, v28.16b, v1.4b[2]\n" + ".inst 0x4f82eb91 // sdot v17.4s, v28.16b, v2.4b[2]\n" + ".inst 0x4f83eb95 // sdot v21.4s, v28.16b, v3.4b[2]\n" + ".inst 0x4f84eb99 // sdot v25.4s, v28.16b, v4.4b[2]\n" + "ldr q28, [x16, #0xb0]\n" + ".inst 0x4f80ebaa // sdot v10.4s, v29.16b, v0.4b[2]\n" + ".inst 0x4f81ebae // sdot v14.4s, v29.16b, v1.4b[2]\n" + ".inst 0x4f82ebb2 // sdot v18.4s, v29.16b, v2.4b[2]\n" + ".inst 0x4f83ebb6 // sdot v22.4s, v29.16b, v3.4b[2]\n" + ".inst 0x4f84ebba // sdot v26.4s, v29.16b, v4.4b[2]\n" + "ldr q29, [x16, #0xc0]\n" + ".inst 0x4f80eb8b // sdot v11.4s, v28.16b, v0.4b[2]\n" + ".inst 0x4f81eb8f // sdot v15.4s, v28.16b, v1.4b[2]\n" + ".inst 0x4f82eb93 // sdot v19.4s, v28.16b, v2.4b[2]\n" + ".inst 0x4f83eb97 // sdot v23.4s, v28.16b, v3.4b[2]\n" + ".inst 0x4f84eb9b // sdot v27.4s, v28.16b, v4.4b[2]\n" + "ldr q28, [x16, #0xd0]\n" + ".inst 0x4fa0eba8 // sdot v8.4s, v29.16b, v0.4b[3]\n" + ".inst 0x4fa1ebac // sdot v12.4s, v29.16b, v1.4b[3]\n" + ".inst 0x4fa2ebb0 // sdot v16.4s, v29.16b, v2.4b[3]\n" + ".inst 0x4fa3ebb4 // sdot v20.4s, v29.16b, v3.4b[3]\n" + ".inst 0x4fa4ebb8 // sdot v24.4s, v29.16b, v4.4b[3]\n" + "ldr q29, [x16, #0xe0]\n" + ".inst 0x4fa0eb89 // sdot v9.4s, v28.16b, v0.4b[3]\n" + ".inst 0x4fa1eb8d // sdot v13.4s, v28.16b, v1.4b[3]\n" + ".inst 0x4fa2eb91 // sdot v17.4s, v28.16b, v2.4b[3]\n" + ".inst 0x4fa3eb95 // sdot v21.4s, v28.16b, v3.4b[3]\n" + ".inst 0x4fa4eb99 // sdot v25.4s, v28.16b, v4.4b[3]\n" + "ldr q28, [x16, #0xf0]\n" + ".inst 0x4fa0ebaa // sdot v10.4s, v29.16b, v0.4b[3]\n" "add x16, x16, #0x100\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n" - ".inst 0x4fa4e8da // sdot v26.4s, v6.16b, v4.4b[3]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" - ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n" - ".inst 0x4fa4e8fb // sdot v27.4s, v7.16b, v4.4b[3]\n" + ".inst 0x4fa1ebae // sdot v14.4s, v29.16b, v1.4b[3]\n" + ".inst 0x4fa2ebb2 // sdot v18.4s, v29.16b, v2.4b[3]\n" + ".inst 0x4fa3ebb6 // sdot v22.4s, v29.16b, v3.4b[3]\n" + ".inst 0x4fa4ebba // sdot v26.4s, v29.16b, v4.4b[3]\n" + ".inst 0x4fa0eb8b // sdot v11.4s, v28.16b, v0.4b[3]\n" + ".inst 0x4fa1eb8f // sdot v15.4s, v28.16b, v1.4b[3]\n" + ".inst 0x4fa2eb93 // sdot v19.4s, v28.16b, v2.4b[3]\n" + ".inst 0x4fa3eb97 // sdot v23.4s, v28.16b, v3.4b[3]\n" + ".inst 0x4fa4eb9b // sdot v27.4s, v28.16b, v4.4b[3]\n" "155:" // Height 5: Multiply loop: Main loop skip "cbz x14, 160f\n" "cmp x14, #0x4\n" "blt 157f\n" "156:" // Height 5: Multiply loop: Odd block loop - "ldr s0, [x13], #0x4\n" + "ldr s2, [x13], #0x4\n" "sub x14, x14, #0x4\n" - "ldr s1, [x9], #0x4\n" + "ldr s1, [x12], #0x4\n" "cmp x14, #0x4\n" - "ldr s2, [x27], #0x4\n" - "ldr s3, [x25], #0x4\n" - "ldr s4, [x23], #0x4\n" - "ldr q6, [x16, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x10]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x16, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr s0, [x11], #0x4\n" + "ldr s31, [x10], #0x4\n" + "ldr s30, [x9], #0x4\n" + "ldr q29, [x16, #0x0]\n" + ".inst 0x4f82e3a8 // sdot v8.4s, v29.16b, v2.4b[0]\n" + "ldr q28, [x16, #0x10]\n" + ".inst 0x4f81e3ac // sdot v12.4s, v29.16b, v1.4b[0]\n" + ".inst 0x4f80e3b0 // sdot v16.4s, v29.16b, v0.4b[0]\n" + ".inst 0x4f9fe3b4 // sdot v20.4s, v29.16b, v31.4b[0]\n" + ".inst 0x4f9ee3b8 // sdot v24.4s, v29.16b, v30.4b[0]\n" + "ldr q29, [x16, #0x20]\n" + ".inst 0x4f82e389 // sdot v9.4s, v28.16b, v2.4b[0]\n" + ".inst 0x4f81e38d // sdot v13.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f80e391 // sdot v17.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f9fe395 // sdot v21.4s, v28.16b, v31.4b[0]\n" + ".inst 0x4f9ee399 // sdot v25.4s, v28.16b, v30.4b[0]\n" + "ldr q28, [x16, #0x30]\n" + ".inst 0x4f82e3aa // sdot v10.4s, v29.16b, v2.4b[0]\n" "add x16, x16, #0x40\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" + ".inst 0x4f81e3ae // sdot v14.4s, v29.16b, v1.4b[0]\n" + ".inst 0x4f80e3b2 // sdot v18.4s, v29.16b, v0.4b[0]\n" + ".inst 0x4f9fe3b6 // sdot v22.4s, v29.16b, v31.4b[0]\n" + ".inst 0x4f9ee3ba // sdot v26.4s, v29.16b, v30.4b[0]\n" + ".inst 0x4f82e38b // sdot v11.4s, v28.16b, v2.4b[0]\n" + ".inst 0x4f81e38f // sdot v15.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f80e393 // sdot v19.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f9fe397 // sdot v23.4s, v28.16b, v31.4b[0]\n" + ".inst 0x4f9ee39b // sdot v27.4s, v28.16b, v30.4b[0]\n" "bge 156b\n" "157:" // Height 5: Multiply loop: Skip odd blocks "cbz x14, 160f\n" "tbz x14, #1, 158f\n" "ldr h0, [x13], #0x2\n" - "ldr h1, [x9], #0x2\n" - "ldr h2, [x27], #0x2\n" - "ldr h3, [x25], #0x2\n" - "ldr h4, [x23], #0x2\n" + "ldr h1, [x12], #0x2\n" + "ldr h2, [x11], #0x2\n" + "ldr h3, [x10], #0x2\n" + "ldr h4, [x9], #0x2\n" "tbz x14, #0, 159f\n" "ld1 { v0.b }[2], [x13]\n" - "ld1 { v1.b }[2], [x9]\n" - "ld1 { v2.b }[2], [x27]\n" - "ld1 { v3.b }[2], [x25]\n" - "ld1 { v4.b }[2], [x23]\n" + "ld1 { v1.b }[2], [x12]\n" + "ld1 { v2.b }[2], [x11]\n" + "ld1 { v3.b }[2], [x10]\n" + "ld1 { v4.b }[2], [x9]\n" "b 159f\n" "158:" // Height 5: Multiply loop: Ragged operand read: partial_1_0 "ldr b0, [x13, #0x0]\n" - "ldr b1, [x9, #0x0]\n" - "ldr b2, [x27, #0x0]\n" - "ldr b3, [x25, #0x0]\n" - "ldr b4, [x23, #0x0]\n" + "ldr b1, [x12, #0x0]\n" + "ldr b2, [x11, #0x0]\n" + "ldr b3, [x10, #0x0]\n" + "ldr b4, [x9, #0x0]\n" "159:" // Height 5: Multiply loop: Ragged operand read: Done - "ldr q6, [x16, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x10]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x16, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q29, [x16, #0x0]\n" + ".inst 0x4f80e3a8 // sdot v8.4s, v29.16b, v0.4b[0]\n" + "ldr q28, [x16, #0x10]\n" + ".inst 0x4f81e3ac // sdot v12.4s, v29.16b, v1.4b[0]\n" + ".inst 0x4f82e3b0 // sdot v16.4s, v29.16b, v2.4b[0]\n" + ".inst 0x4f83e3b4 // sdot v20.4s, v29.16b, v3.4b[0]\n" + ".inst 0x4f84e3b8 // sdot v24.4s, v29.16b, v4.4b[0]\n" + "ldr q29, [x16, #0x20]\n" + ".inst 0x4f80e389 // sdot v9.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f81e38d // sdot v13.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f82e391 // sdot v17.4s, v28.16b, v2.4b[0]\n" + ".inst 0x4f83e395 // sdot v21.4s, v28.16b, v3.4b[0]\n" + ".inst 0x4f84e399 // sdot v25.4s, v28.16b, v4.4b[0]\n" + "ldr q28, [x16, #0x30]\n" + ".inst 0x4f80e3aa // sdot v10.4s, v29.16b, v0.4b[0]\n" "add x16, x16, #0x40\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" + ".inst 0x4f81e3ae // sdot v14.4s, v29.16b, v1.4b[0]\n" + ".inst 0x4f82e3b2 // sdot v18.4s, v29.16b, v2.4b[0]\n" + ".inst 0x4f83e3b6 // sdot v22.4s, v29.16b, v3.4b[0]\n" + ".inst 0x4f84e3ba // sdot v26.4s, v29.16b, v4.4b[0]\n" + ".inst 0x4f80e38b // sdot v11.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f81e38f // sdot v15.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f82e393 // sdot v19.4s, v28.16b, v2.4b[0]\n" + ".inst 0x4f83e397 // sdot v23.4s, v28.16b, v3.4b[0]\n" + ".inst 0x4f84e39b // sdot v27.4s, v28.16b, v4.4b[0]\n" "160:" // Height 5: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x15, x15, #0x1\n" @@ -2862,98 +2861,98 @@ void a64_hybrid_s8s32_dot_6x16_a55 ( "184:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w14, [x20, x15, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 185f\n" - "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x13, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" - "ldr x27, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" - "ldr x23, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x15, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x13, [x20, #0x0]\n" + "ldr x12, [x20, #0x8]\n" + "ldr x11, [x20, #0x10]\n" + "ldr x10, [x20, #0x18]\n" + "ldr x9, [x20, #0x20]\n" + "ldr x28, [x20, #0x28]\n" "cbnz x15, 186f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x13, x13, x20\n" + "add x12, x12, x20\n" + "add x11, x11, x20\n" + "add x10, x10, x20\n" "add x9, x9, x20\n" - "add x27, x27, x20\n" - "add x25, x25, x20\n" - "add x23, x23, x20\n" - "add x21, x21, x20\n" + "add x28, x28, x20\n" "b 186f\n" "185:" // Height 6: setup direct input "mov x13, %x[input_ptr]\n" - "add x9, x13, x20\n" - "add x27, x9, x20\n" - "add x25, x27, x20\n" - "add x23, x25, x20\n" - "add x21, x23, x20\n" + "add x12, x13, x21\n" + "add x11, x12, x21\n" + "add x10, x11, x21\n" + "add x9, x10, x21\n" + "add x28, x9, x21\n" "186:" // Height 6: input setup done "cmp x14, #0x10\n" "blt 189f\n" "ldr q0, [x13, #0x0]\n" "cmp x14, #0x20\n" - "ldr q1, [x9, #0x0]\n" - "ldr q2, [x27, #0x0]\n" - "ldr q3, [x25, #0x0]\n" - "ldr q4, [x23, #0x0]\n" - "ldr q5, [x21, #0x0]\n" + "ldr q1, [x12, #0x0]\n" + "ldr q2, [x11, #0x0]\n" + "ldr q3, [x10, #0x0]\n" + "ldr q4, [x9, #0x0]\n" + "ldr q5, [x28, #0x0]\n" "ldr q6, [x16, #0x0]\n" "ldr q7, [x16, #0x10]\n" "blt 188f\n" "187:" // Height 6: Multiply loop: Main loop head ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr x12, [x16, #0x28]\n" + "ldr x21, [x16, #0x28]\n" ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr x11, [x16, #0x38]\n" + "ldr x20, [x16, #0x38]\n" ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" "add x13, x13, #0x10\n" ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - "add x27, x27, #0x10\n" + "add x11, x11, #0x10\n" ".inst 0x4f85e0dc // sdot v28.4s, v6.16b, v5.4b[0]\n" "ldr d6, [x16, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "ldr x12, [x16, #0x48]\n" + "ldr x21, [x16, #0x48]\n" ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - "add x25, x25, #0x10\n" + "add x10, x10, #0x10\n" ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - "add x23, x23, #0x10\n" + "add x9, x9, #0x10\n" ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - "add x21, x21, #0x10\n" + "add x28, x28, #0x10\n" ".inst 0x4f85e0fd // sdot v29.4s, v7.16b, v5.4b[0]\n" "ldr d7, [x16, #0x30]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "ldr x11, [x16, #0x58]\n" + "ldr x20, [x16, #0x58]\n" ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - "ldr x10, [x13, #0x8]\n" + "ldr x27, [x13, #0x8]\n" ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - "ldr x28, [x9, #0x8]\n" + "ldr x26, [x12, #0x8]\n" ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - "ldr x26, [x27, #0x8]\n" + "ldr x25, [x11, #0x8]\n" ".inst 0x4f85e0de // sdot v30.4s, v6.16b, v5.4b[0]\n" "ldr d6, [x16, #0x40]\n" ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - "ldr x12, [x16, #0x68]\n" + "ldr x21, [x16, #0x68]\n" ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - "ldr x24, [x25, #0x8]\n" + "ldr x24, [x10, #0x8]\n" ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - "ldr x22, [x23, #0x8]\n" + "ldr x23, [x9, #0x8]\n" ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" - "ldr x20, [x21, #0x8]\n" + "ldr x22, [x28, #0x8]\n" ".inst 0x4f85e0ff // sdot v31.4s, v7.16b, v5.4b[0]\n" "ldr d7, [x16, #0x50]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - "ldr x11, [x16, #0x78]\n" + "ldr x20, [x16, #0x78]\n" ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" "sub x14, x14, #0x10\n" ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n" @@ -2963,96 +2962,96 @@ void a64_hybrid_s8s32_dot_6x16_a55 ( ".inst 0x4fa5e0dc // sdot v28.4s, v6.16b, v5.4b[1]\n" "ldr d6, [x16, #0x60]\n" ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - "ldr x12, [x16, #0x88]\n" + "ldr x21, [x16, #0x88]\n" ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x12, #0x80]\n" ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n" - "prfm pldl1keep, [x27, #0x80]\n" + "prfm pldl1keep, [x11, #0x80]\n" ".inst 0x4fa4e0f9 // sdot v25.4s, v7.16b, v4.4b[1]\n" - "prfm pldl1keep, [x25, #0x80]\n" + "prfm pldl1keep, [x10, #0x80]\n" ".inst 0x4fa5e0fd // sdot v29.4s, v7.16b, v5.4b[1]\n" "ldr d7, [x16, #0x70]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - "ldr x11, [x16, #0x98]\n" + "ldr x20, [x16, #0x98]\n" ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - "prfm pldl1keep, [x23, #0x80]\n" + "prfm pldl1keep, [x9, #0x80]\n" ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n" - "prfm pldl1keep, [x21, #0x80]\n" + "prfm pldl1keep, [x28, #0x80]\n" ".inst 0x4fa4e0da // sdot v26.4s, v6.16b, v4.4b[1]\n" ".inst 0x4fa5e0de // sdot v30.4s, v6.16b, v5.4b[1]\n" "ldr d6, [x16, #0x80]\n" ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - "ldr x12, [x16, #0xa8]\n" + "ldr x21, [x16, #0xa8]\n" ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n" ".inst 0x4fa4e0fb // sdot v27.4s, v7.16b, v4.4b[1]\n" ".inst 0x4fa5e0ff // sdot v31.4s, v7.16b, v5.4b[1]\n" "ldr d7, [x16, #0x90]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - "ldr x11, [x16, #0xb8]\n" + "ldr x20, [x16, #0xb8]\n" ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n" ".inst 0x4f84e8d8 // sdot v24.4s, v6.16b, v4.4b[2]\n" ".inst 0x4f85e8dc // sdot v28.4s, v6.16b, v5.4b[2]\n" "ldr d6, [x16, #0xa0]\n" ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - "ldr x12, [x16, #0xc8]\n" + "ldr x21, [x16, #0xc8]\n" ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n" ".inst 0x4f84e8f9 // sdot v25.4s, v7.16b, v4.4b[2]\n" ".inst 0x4f85e8fd // sdot v29.4s, v7.16b, v5.4b[2]\n" "ldr d7, [x16, #0xb0]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - "ldr x11, [x16, #0xd8]\n" + "ldr x20, [x16, #0xd8]\n" ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n" ".inst 0x4f84e8da // sdot v26.4s, v6.16b, v4.4b[2]\n" ".inst 0x4f85e8de // sdot v30.4s, v6.16b, v5.4b[2]\n" "ldr d6, [x16, #0xc0]\n" ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - "ldr x12, [x16, #0xe8]\n" + "ldr x21, [x16, #0xe8]\n" ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n" ".inst 0x4f84e8fb // sdot v27.4s, v7.16b, v4.4b[2]\n" ".inst 0x4f85e8ff // sdot v31.4s, v7.16b, v5.4b[2]\n" "ldr d7, [x16, #0xd0]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - "ldr x11, [x16, #0xf8]\n" + "ldr x20, [x16, #0xf8]\n" ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n" ".inst 0x4fa4e8d8 // sdot v24.4s, v6.16b, v4.4b[3]\n" ".inst 0x4fa5e8dc // sdot v28.4s, v6.16b, v5.4b[3]\n" "ldr d6, [x16, #0xe0]\n" ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n" ".inst 0x4fa4e8f9 // sdot v25.4s, v7.16b, v4.4b[3]\n" ".inst 0x4fa5e8fd // sdot v29.4s, v7.16b, v5.4b[3]\n" "ldr d7, [x16, #0xf0]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "add x16, x16, #0x100\n" ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - "ldr x12, [x16, #0x8]\n" + "ldr x21, [x16, #0x8]\n" ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - "ldr x11, [x16, #0x18]\n" + "ldr x20, [x16, #0x18]\n" ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n" ".inst 0x4fa4e8da // sdot v26.4s, v6.16b, v4.4b[3]\n" @@ -3061,56 +3060,56 @@ void a64_hybrid_s8s32_dot_6x16_a55 ( ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" "ldr d0, [x13, #0x0]\n" ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" - "ldr d1, [x9, #0x0]\n" + "ldr d1, [x12, #0x0]\n" ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" - "ldr d2, [x27, #0x0]\n" + "ldr d2, [x11, #0x0]\n" ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n" - "ldr d3, [x25, #0x0]\n" + "ldr d3, [x10, #0x0]\n" ".inst 0x4fa4e8fb // sdot v27.4s, v7.16b, v4.4b[3]\n" - "ldr d4, [x23, #0x0]\n" + "ldr d4, [x9, #0x0]\n" ".inst 0x4fa5e8ff // sdot v31.4s, v7.16b, v5.4b[3]\n" - "ldr d5, [x21, #0x0]\n" + "ldr d5, [x28, #0x0]\n" "ldr d7, [x16, #0x10]\n" - "mov v6.d[1], x12\n" - "mov v0.d[1], x10\n" - "mov v1.d[1], x28\n" - "mov v2.d[1], x26\n" + "mov v6.d[1], x21\n" + "mov v0.d[1], x27\n" + "mov v1.d[1], x26\n" + "mov v2.d[1], x25\n" "mov v3.d[1], x24\n" - "mov v4.d[1], x22\n" - "mov v5.d[1], x20\n" - "mov v7.d[1], x11\n" + "mov v4.d[1], x23\n" + "mov v5.d[1], x22\n" + "mov v7.d[1], x20\n" "bge 187b\n" "188:" // Height 6: Multiply loop: Single iteration only ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" "add x13, x13, #0x10\n" ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - "add x27, x27, #0x10\n" + "add x11, x11, #0x10\n" ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "add x25, x25, #0x10\n" + "add x10, x10, #0x10\n" ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - "add x23, x23, #0x10\n" + "add x9, x9, #0x10\n" ".inst 0x4f85e0dc // sdot v28.4s, v6.16b, v5.4b[0]\n" "ldr q6, [x16, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "add x21, x21, #0x10\n" + "add x28, x28, #0x10\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" "sub x14, x14, #0x10\n" ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" "prfm pldl1keep, [x13, #0x80]\n" ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x12, #0x80]\n" ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - "prfm pldl1keep, [x27, #0x80]\n" + "prfm pldl1keep, [x11, #0x80]\n" ".inst 0x4f85e0fd // sdot v29.4s, v7.16b, v5.4b[0]\n" "ldr q7, [x16, #0x30]\n" ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - "prfm pldl1keep, [x25, #0x80]\n" + "prfm pldl1keep, [x10, #0x80]\n" ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "prfm pldl1keep, [x23, #0x80]\n" + "prfm pldl1keep, [x9, #0x80]\n" ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - "prfm pldl1keep, [x21, #0x80]\n" + "prfm pldl1keep, [x28, #0x80]\n" ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" ".inst 0x4f85e0de // sdot v30.4s, v6.16b, v5.4b[0]\n" @@ -3210,98 +3209,98 @@ void a64_hybrid_s8s32_dot_6x16_a55 ( "cmp x14, #0x4\n" "blt 191f\n" "190:" // Height 6: Multiply loop: Odd block loop - "ldr s0, [x13], #0x4\n" + "ldr s7, [x13], #0x4\n" "sub x14, x14, #0x4\n" - "ldr s1, [x9], #0x4\n" + "ldr s6, [x12], #0x4\n" "cmp x14, #0x4\n" - "ldr s2, [x27], #0x4\n" - "ldr s3, [x25], #0x4\n" - "ldr s4, [x23], #0x4\n" - "ldr s5, [x21], #0x4\n" - "ldr q6, [x16, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x10]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - ".inst 0x4f85e0dc // sdot v28.4s, v6.16b, v5.4b[0]\n" - "ldr q6, [x16, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - ".inst 0x4f85e0fd // sdot v29.4s, v7.16b, v5.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr s5, [x11], #0x4\n" + "ldr s4, [x10], #0x4\n" + "ldr s3, [x9], #0x4\n" + "ldr s2, [x28], #0x4\n" + "ldr q1, [x16, #0x0]\n" + ".inst 0x4f87e028 // sdot v8.4s, v1.16b, v7.4b[0]\n" + "ldr q0, [x16, #0x10]\n" + ".inst 0x4f86e02c // sdot v12.4s, v1.16b, v6.4b[0]\n" + ".inst 0x4f85e030 // sdot v16.4s, v1.16b, v5.4b[0]\n" + ".inst 0x4f84e034 // sdot v20.4s, v1.16b, v4.4b[0]\n" + ".inst 0x4f83e038 // sdot v24.4s, v1.16b, v3.4b[0]\n" + ".inst 0x4f82e03c // sdot v28.4s, v1.16b, v2.4b[0]\n" + "ldr q1, [x16, #0x20]\n" + ".inst 0x4f87e009 // sdot v9.4s, v0.16b, v7.4b[0]\n" + ".inst 0x4f86e00d // sdot v13.4s, v0.16b, v6.4b[0]\n" + ".inst 0x4f85e011 // sdot v17.4s, v0.16b, v5.4b[0]\n" + ".inst 0x4f84e015 // sdot v21.4s, v0.16b, v4.4b[0]\n" + ".inst 0x4f83e019 // sdot v25.4s, v0.16b, v3.4b[0]\n" + ".inst 0x4f82e01d // sdot v29.4s, v0.16b, v2.4b[0]\n" + "ldr q0, [x16, #0x30]\n" + ".inst 0x4f87e02a // sdot v10.4s, v1.16b, v7.4b[0]\n" "add x16, x16, #0x40\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - ".inst 0x4f85e0de // sdot v30.4s, v6.16b, v5.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" - ".inst 0x4f85e0ff // sdot v31.4s, v7.16b, v5.4b[0]\n" + ".inst 0x4f86e02e // sdot v14.4s, v1.16b, v6.4b[0]\n" + ".inst 0x4f85e032 // sdot v18.4s, v1.16b, v5.4b[0]\n" + ".inst 0x4f84e036 // sdot v22.4s, v1.16b, v4.4b[0]\n" + ".inst 0x4f83e03a // sdot v26.4s, v1.16b, v3.4b[0]\n" + ".inst 0x4f82e03e // sdot v30.4s, v1.16b, v2.4b[0]\n" + ".inst 0x4f87e00b // sdot v11.4s, v0.16b, v7.4b[0]\n" + ".inst 0x4f86e00f // sdot v15.4s, v0.16b, v6.4b[0]\n" + ".inst 0x4f85e013 // sdot v19.4s, v0.16b, v5.4b[0]\n" + ".inst 0x4f84e017 // sdot v23.4s, v0.16b, v4.4b[0]\n" + ".inst 0x4f83e01b // sdot v27.4s, v0.16b, v3.4b[0]\n" + ".inst 0x4f82e01f // sdot v31.4s, v0.16b, v2.4b[0]\n" "bge 190b\n" "191:" // Height 6: Multiply loop: Skip odd blocks "cbz x14, 194f\n" "tbz x14, #1, 192f\n" "ldr h0, [x13], #0x2\n" - "ldr h1, [x9], #0x2\n" - "ldr h2, [x27], #0x2\n" - "ldr h3, [x25], #0x2\n" - "ldr h4, [x23], #0x2\n" - "ldr h5, [x21], #0x2\n" + "ldr h1, [x12], #0x2\n" + "ldr h2, [x11], #0x2\n" + "ldr h3, [x10], #0x2\n" + "ldr h4, [x9], #0x2\n" + "ldr h5, [x28], #0x2\n" "tbz x14, #0, 193f\n" "ld1 { v0.b }[2], [x13]\n" - "ld1 { v1.b }[2], [x9]\n" - "ld1 { v2.b }[2], [x27]\n" - "ld1 { v3.b }[2], [x25]\n" - "ld1 { v4.b }[2], [x23]\n" - "ld1 { v5.b }[2], [x21]\n" + "ld1 { v1.b }[2], [x12]\n" + "ld1 { v2.b }[2], [x11]\n" + "ld1 { v3.b }[2], [x10]\n" + "ld1 { v4.b }[2], [x9]\n" + "ld1 { v5.b }[2], [x28]\n" "b 193f\n" "192:" // Height 6: Multiply loop: Ragged operand read: partial_1_0 "ldr b0, [x13, #0x0]\n" - "ldr b1, [x9, #0x0]\n" - "ldr b2, [x27, #0x0]\n" - "ldr b3, [x25, #0x0]\n" - "ldr b4, [x23, #0x0]\n" - "ldr b5, [x21, #0x0]\n" + "ldr b1, [x12, #0x0]\n" + "ldr b2, [x11, #0x0]\n" + "ldr b3, [x10, #0x0]\n" + "ldr b4, [x9, #0x0]\n" + "ldr b5, [x28, #0x0]\n" "193:" // Height 6: Multiply loop: Ragged operand read: Done - "ldr q6, [x16, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x10]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - ".inst 0x4f85e0dc // sdot v28.4s, v6.16b, v5.4b[0]\n" - "ldr q6, [x16, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - ".inst 0x4f85e0fd // sdot v29.4s, v7.16b, v5.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q7, [x16, #0x0]\n" + ".inst 0x4f80e0e8 // sdot v8.4s, v7.16b, v0.4b[0]\n" + "ldr q6, [x16, #0x10]\n" + ".inst 0x4f81e0ec // sdot v12.4s, v7.16b, v1.4b[0]\n" + ".inst 0x4f82e0f0 // sdot v16.4s, v7.16b, v2.4b[0]\n" + ".inst 0x4f83e0f4 // sdot v20.4s, v7.16b, v3.4b[0]\n" + ".inst 0x4f84e0f8 // sdot v24.4s, v7.16b, v4.4b[0]\n" + ".inst 0x4f85e0fc // sdot v28.4s, v7.16b, v5.4b[0]\n" + "ldr q7, [x16, #0x20]\n" + ".inst 0x4f80e0c9 // sdot v9.4s, v6.16b, v0.4b[0]\n" + ".inst 0x4f81e0cd // sdot v13.4s, v6.16b, v1.4b[0]\n" + ".inst 0x4f82e0d1 // sdot v17.4s, v6.16b, v2.4b[0]\n" + ".inst 0x4f83e0d5 // sdot v21.4s, v6.16b, v3.4b[0]\n" + ".inst 0x4f84e0d9 // sdot v25.4s, v6.16b, v4.4b[0]\n" + ".inst 0x4f85e0dd // sdot v29.4s, v6.16b, v5.4b[0]\n" + "ldr q6, [x16, #0x30]\n" + ".inst 0x4f80e0ea // sdot v10.4s, v7.16b, v0.4b[0]\n" "add x16, x16, #0x40\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - ".inst 0x4f85e0de // sdot v30.4s, v6.16b, v5.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" - ".inst 0x4f85e0ff // sdot v31.4s, v7.16b, v5.4b[0]\n" + ".inst 0x4f81e0ee // sdot v14.4s, v7.16b, v1.4b[0]\n" + ".inst 0x4f82e0f2 // sdot v18.4s, v7.16b, v2.4b[0]\n" + ".inst 0x4f83e0f6 // sdot v22.4s, v7.16b, v3.4b[0]\n" + ".inst 0x4f84e0fa // sdot v26.4s, v7.16b, v4.4b[0]\n" + ".inst 0x4f85e0fe // sdot v30.4s, v7.16b, v5.4b[0]\n" + ".inst 0x4f80e0cb // sdot v11.4s, v6.16b, v0.4b[0]\n" + ".inst 0x4f81e0cf // sdot v15.4s, v6.16b, v1.4b[0]\n" + ".inst 0x4f82e0d3 // sdot v19.4s, v6.16b, v2.4b[0]\n" + ".inst 0x4f83e0d7 // sdot v23.4s, v6.16b, v3.4b[0]\n" + ".inst 0x4f84e0db // sdot v27.4s, v6.16b, v4.4b[0]\n" + ".inst 0x4f85e0df // sdot v31.4s, v6.16b, v5.4b[0]\n" "194:" // Height 6: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x15, x15, #0x1\n" @@ -3488,7 +3487,6 @@ void a64_hybrid_s8s32_dot_6x16_a55 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "206:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16/generic.cpp index ddf776107ababd252a1350b789d369202f6e20ec..452d647bb4b982727be0200860a5df6e7b7113b6 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_6x16/generic.cpp @@ -77,7 +77,6 @@ void a64_hybrid_s8s32_dot_6x16 ( ka.N = N; ka.B_ptr = B_ptr; __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x6\n" "bge 171f\n" @@ -165,11 +164,11 @@ void a64_hybrid_s8s32_dot_6x16 ( "14:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 15f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 16f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -186,37 +185,37 @@ void a64_hybrid_s8s32_dot_6x16 ( "blt 18f\n" "17:" // Height 1: Multiply loop: Main loop head ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q17, [x10, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - "ldr q7, [x10, #0xf0]\n" + "ldr q16, [x10, #0x30]\n" + ".inst 0x4f80e22a // sdot v10.4s, v17.16b, v0.4b[0]\n" + "ldr q17, [x10, #0x40]\n" + ".inst 0x4f80e20b // sdot v11.4s, v16.16b, v0.4b[0]\n" + "ldr q16, [x10, #0x50]\n" + ".inst 0x4fa0e228 // sdot v8.4s, v17.16b, v0.4b[1]\n" + "ldr q17, [x10, #0x60]\n" + ".inst 0x4fa0e209 // sdot v9.4s, v16.16b, v0.4b[1]\n" + "ldr q16, [x10, #0x70]\n" + ".inst 0x4fa0e22a // sdot v10.4s, v17.16b, v0.4b[1]\n" + "ldr q17, [x10, #0x80]\n" + ".inst 0x4fa0e20b // sdot v11.4s, v16.16b, v0.4b[1]\n" + "ldr q16, [x10, #0x90]\n" + ".inst 0x4f80ea28 // sdot v8.4s, v17.16b, v0.4b[2]\n" + "ldr q17, [x10, #0xa0]\n" + ".inst 0x4f80ea09 // sdot v9.4s, v16.16b, v0.4b[2]\n" + "ldr q16, [x10, #0xb0]\n" + ".inst 0x4f80ea2a // sdot v10.4s, v17.16b, v0.4b[2]\n" + "ldr q17, [x10, #0xc0]\n" + ".inst 0x4f80ea0b // sdot v11.4s, v16.16b, v0.4b[2]\n" + "ldr q16, [x10, #0xd0]\n" + ".inst 0x4fa0ea28 // sdot v8.4s, v17.16b, v0.4b[3]\n" + "ldr q17, [x10, #0xe0]\n" + ".inst 0x4fa0ea09 // sdot v9.4s, v16.16b, v0.4b[3]\n" + "ldr q16, [x10, #0xf0]\n" "sub x27, x27, #0x10\n" "add x26, x26, #0x10\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x4fa0ea2a // sdot v10.4s, v17.16b, v0.4b[3]\n" + ".inst 0x4fa0ea0b // sdot v11.4s, v16.16b, v0.4b[3]\n" "ldr q0, [x26, #0x0]\n" "cmp x27, #0x20\n" "add x10, x10, #0x100\n" @@ -226,37 +225,37 @@ void a64_hybrid_s8s32_dot_6x16 ( "bge 17b\n" "18:" // Height 1: Multiply loop: Single iteration only ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q17, [x10, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - "ldr q7, [x10, #0xf0]\n" + "ldr q16, [x10, #0x30]\n" + ".inst 0x4f80e22a // sdot v10.4s, v17.16b, v0.4b[0]\n" + "ldr q17, [x10, #0x40]\n" + ".inst 0x4f80e20b // sdot v11.4s, v16.16b, v0.4b[0]\n" + "ldr q16, [x10, #0x50]\n" + ".inst 0x4fa0e228 // sdot v8.4s, v17.16b, v0.4b[1]\n" + "ldr q17, [x10, #0x60]\n" + ".inst 0x4fa0e209 // sdot v9.4s, v16.16b, v0.4b[1]\n" + "ldr q16, [x10, #0x70]\n" + ".inst 0x4fa0e22a // sdot v10.4s, v17.16b, v0.4b[1]\n" + "ldr q17, [x10, #0x80]\n" + ".inst 0x4fa0e20b // sdot v11.4s, v16.16b, v0.4b[1]\n" + "ldr q16, [x10, #0x90]\n" + ".inst 0x4f80ea28 // sdot v8.4s, v17.16b, v0.4b[2]\n" + "ldr q17, [x10, #0xa0]\n" + ".inst 0x4f80ea09 // sdot v9.4s, v16.16b, v0.4b[2]\n" + "ldr q16, [x10, #0xb0]\n" + ".inst 0x4f80ea2a // sdot v10.4s, v17.16b, v0.4b[2]\n" + "ldr q17, [x10, #0xc0]\n" + ".inst 0x4f80ea0b // sdot v11.4s, v16.16b, v0.4b[2]\n" + "ldr q16, [x10, #0xd0]\n" + ".inst 0x4fa0ea28 // sdot v8.4s, v17.16b, v0.4b[3]\n" + "ldr q17, [x10, #0xe0]\n" + ".inst 0x4fa0ea09 // sdot v9.4s, v16.16b, v0.4b[3]\n" + "ldr q16, [x10, #0xf0]\n" "add x26, x26, #0x10\n" "sub x27, x27, #0x10\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x4fa0ea2a // sdot v10.4s, v17.16b, v0.4b[3]\n" + ".inst 0x4fa0ea0b // sdot v11.4s, v16.16b, v0.4b[3]\n" "prfm pldl1keep, [x26, #0x80]\n" "add x10, x10, #0x100\n" "19:" // Height 1: Multiply loop: Main loop skip @@ -264,17 +263,17 @@ void a64_hybrid_s8s32_dot_6x16 ( "cmp x27, #0x4\n" "blt 21f\n" "20:" // Height 1: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr q6, [x10, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" + "ldr s18, [x26], #0x4\n" + "ldr q16, [x10, #0x0]\n" + ".inst 0x4f92e208 // sdot v8.4s, v16.16b, v18.4b[0]\n" "sub x27, x27, #0x4\n" - "ldr q7, [x10, #0x10]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" + "ldr q16, [x10, #0x10]\n" + "ldr q17, [x10, #0x20]\n" + ".inst 0x4f92e209 // sdot v9.4s, v16.16b, v18.4b[0]\n" "cmp x27, #0x4\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" + "ldr q16, [x10, #0x30]\n" + ".inst 0x4f92e22a // sdot v10.4s, v17.16b, v18.4b[0]\n" + ".inst 0x4f92e20b // sdot v11.4s, v16.16b, v18.4b[0]\n" "add x10, x10, #0x40\n" "bge 20b\n" "21:" // Height 1: Multiply loop: Skip odd blocks @@ -287,14 +286,14 @@ void a64_hybrid_s8s32_dot_6x16 ( "22:" // Height 1: Multiply loop: Ragged operand read: partial_1_0 "ldr b0, [x26, #0x0]\n" "23:" // Height 1: Multiply loop: Ragged operand read: Done - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "ldr q6, [x10, #0x20]\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" + "ldr q17, [x10, #0x0]\n" + "ldr q16, [x10, #0x10]\n" + ".inst 0x4f80e228 // sdot v8.4s, v17.16b, v0.4b[0]\n" + ".inst 0x4f80e209 // sdot v9.4s, v16.16b, v0.4b[0]\n" + "ldr q17, [x10, #0x20]\n" + "ldr q16, [x10, #0x30]\n" + ".inst 0x4f80e22a // sdot v10.4s, v17.16b, v0.4b[0]\n" + ".inst 0x4f80e20b // sdot v11.4s, v16.16b, v0.4b[0]\n" "add x10, x10, #0x40\n" "24:" // Height 1: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -465,12 +464,12 @@ void a64_hybrid_s8s32_dot_6x16 ( "48:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 49f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 50f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -478,7 +477,7 @@ void a64_hybrid_s8s32_dot_6x16 ( "b 50f\n" "49:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" + "add x25, x26, x21\n" "50:" // Height 2: input setup done "cmp x27, #0x10\n" "blt 53f\n" @@ -491,137 +490,137 @@ void a64_hybrid_s8s32_dot_6x16 ( "51:" // Height 2: Multiply loop: Main loop head ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q17, [x10, #0x20]\n" "sub x27, x27, #0x10\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q16, [x10, #0x30]\n" "add x26, x26, #0x10\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x10, #0x40]\n" + ".inst 0x4f80e22a // sdot v10.4s, v17.16b, v0.4b[0]\n" + ".inst 0x4f81e22e // sdot v14.4s, v17.16b, v1.4b[0]\n" + "ldr q17, [x10, #0x40]\n" "add x25, x25, #0x10\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x10, #0x50]\n" + ".inst 0x4f80e20b // sdot v11.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f81e20f // sdot v15.4s, v16.16b, v1.4b[0]\n" + "ldr q16, [x10, #0x50]\n" "cmp x27, #0x20\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - "ldr q6, [x10, #0x60]\n" + ".inst 0x4fa0e228 // sdot v8.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e22c // sdot v12.4s, v17.16b, v1.4b[1]\n" + "ldr q17, [x10, #0x60]\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - "ldr q7, [x10, #0x70]\n" + ".inst 0x4fa0e209 // sdot v9.4s, v16.16b, v0.4b[1]\n" + ".inst 0x4fa1e20d // sdot v13.4s, v16.16b, v1.4b[1]\n" + "ldr q16, [x10, #0x70]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - "ldr q7, [x10, #0xf0]\n" + ".inst 0x4fa0e22a // sdot v10.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e22e // sdot v14.4s, v17.16b, v1.4b[1]\n" + "ldr q17, [x10, #0x80]\n" + ".inst 0x4fa0e20b // sdot v11.4s, v16.16b, v0.4b[1]\n" + ".inst 0x4fa1e20f // sdot v15.4s, v16.16b, v1.4b[1]\n" + "ldr q16, [x10, #0x90]\n" + ".inst 0x4f80ea28 // sdot v8.4s, v17.16b, v0.4b[2]\n" + ".inst 0x4f81ea2c // sdot v12.4s, v17.16b, v1.4b[2]\n" + "ldr q17, [x10, #0xa0]\n" + ".inst 0x4f80ea09 // sdot v9.4s, v16.16b, v0.4b[2]\n" + ".inst 0x4f81ea0d // sdot v13.4s, v16.16b, v1.4b[2]\n" + "ldr q16, [x10, #0xb0]\n" + ".inst 0x4f80ea2a // sdot v10.4s, v17.16b, v0.4b[2]\n" + ".inst 0x4f81ea2e // sdot v14.4s, v17.16b, v1.4b[2]\n" + "ldr q17, [x10, #0xc0]\n" + ".inst 0x4f80ea0b // sdot v11.4s, v16.16b, v0.4b[2]\n" + ".inst 0x4f81ea0f // sdot v15.4s, v16.16b, v1.4b[2]\n" + "ldr q16, [x10, #0xd0]\n" + ".inst 0x4fa0ea28 // sdot v8.4s, v17.16b, v0.4b[3]\n" + ".inst 0x4fa1ea2c // sdot v12.4s, v17.16b, v1.4b[3]\n" + "ldr q17, [x10, #0xe0]\n" + ".inst 0x4fa0ea09 // sdot v9.4s, v16.16b, v0.4b[3]\n" + ".inst 0x4fa1ea0d // sdot v13.4s, v16.16b, v1.4b[3]\n" + "ldr q16, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" + ".inst 0x4fa0ea2a // sdot v10.4s, v17.16b, v0.4b[3]\n" + ".inst 0x4fa1ea2e // sdot v14.4s, v17.16b, v1.4b[3]\n" "ldr q6, [x10, #0x0]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x4fa0ea0b // sdot v11.4s, v16.16b, v0.4b[3]\n" "ldr q0, [x26, #0x0]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" + ".inst 0x4fa1ea0f // sdot v15.4s, v16.16b, v1.4b[3]\n" "ldr q1, [x25, #0x0]\n" "ldr q7, [x10, #0x10]\n" "bge 51b\n" "52:" // Height 2: Multiply loop: Single iteration only ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q17, [x10, #0x20]\n" "add x26, x26, #0x10\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q16, [x10, #0x30]\n" "add x25, x25, #0x10\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x10, #0x40]\n" + ".inst 0x4f80e22a // sdot v10.4s, v17.16b, v0.4b[0]\n" + ".inst 0x4f81e22e // sdot v14.4s, v17.16b, v1.4b[0]\n" + "ldr q17, [x10, #0x40]\n" "sub x27, x27, #0x10\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x10, #0x50]\n" + ".inst 0x4f80e20b // sdot v11.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f81e20f // sdot v15.4s, v16.16b, v1.4b[0]\n" + "ldr q16, [x10, #0x50]\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - "ldr q6, [x10, #0x60]\n" + ".inst 0x4fa0e228 // sdot v8.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e22c // sdot v12.4s, v17.16b, v1.4b[1]\n" + "ldr q17, [x10, #0x60]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - "ldr q7, [x10, #0xf0]\n" + ".inst 0x4fa0e209 // sdot v9.4s, v16.16b, v0.4b[1]\n" + ".inst 0x4fa1e20d // sdot v13.4s, v16.16b, v1.4b[1]\n" + "ldr q16, [x10, #0x70]\n" + ".inst 0x4fa0e22a // sdot v10.4s, v17.16b, v0.4b[1]\n" + ".inst 0x4fa1e22e // sdot v14.4s, v17.16b, v1.4b[1]\n" + "ldr q17, [x10, #0x80]\n" + ".inst 0x4fa0e20b // sdot v11.4s, v16.16b, v0.4b[1]\n" + ".inst 0x4fa1e20f // sdot v15.4s, v16.16b, v1.4b[1]\n" + "ldr q16, [x10, #0x90]\n" + ".inst 0x4f80ea28 // sdot v8.4s, v17.16b, v0.4b[2]\n" + ".inst 0x4f81ea2c // sdot v12.4s, v17.16b, v1.4b[2]\n" + "ldr q17, [x10, #0xa0]\n" + ".inst 0x4f80ea09 // sdot v9.4s, v16.16b, v0.4b[2]\n" + ".inst 0x4f81ea0d // sdot v13.4s, v16.16b, v1.4b[2]\n" + "ldr q16, [x10, #0xb0]\n" + ".inst 0x4f80ea2a // sdot v10.4s, v17.16b, v0.4b[2]\n" + ".inst 0x4f81ea2e // sdot v14.4s, v17.16b, v1.4b[2]\n" + "ldr q17, [x10, #0xc0]\n" + ".inst 0x4f80ea0b // sdot v11.4s, v16.16b, v0.4b[2]\n" + ".inst 0x4f81ea0f // sdot v15.4s, v16.16b, v1.4b[2]\n" + "ldr q16, [x10, #0xd0]\n" + ".inst 0x4fa0ea28 // sdot v8.4s, v17.16b, v0.4b[3]\n" + ".inst 0x4fa1ea2c // sdot v12.4s, v17.16b, v1.4b[3]\n" + "ldr q17, [x10, #0xe0]\n" + ".inst 0x4fa0ea09 // sdot v9.4s, v16.16b, v0.4b[3]\n" + ".inst 0x4fa1ea0d // sdot v13.4s, v16.16b, v1.4b[3]\n" + "ldr q16, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" + ".inst 0x4fa0ea2a // sdot v10.4s, v17.16b, v0.4b[3]\n" + ".inst 0x4fa1ea2e // sdot v14.4s, v17.16b, v1.4b[3]\n" + ".inst 0x4fa0ea0b // sdot v11.4s, v16.16b, v0.4b[3]\n" + ".inst 0x4fa1ea0f // sdot v15.4s, v16.16b, v1.4b[3]\n" "53:" // Height 2: Multiply loop: Main loop skip "cbz x27, 58f\n" "cmp x27, #0x4\n" "blt 55f\n" "54:" // Height 2: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s19, [x26], #0x4\n" + "ldr s18, [x25], #0x4\n" "sub x27, x27, #0x4\n" "cmp x27, #0x4\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" + "ldr q17, [x10, #0x0]\n" + "ldr q16, [x10, #0x10]\n" + ".inst 0x4f93e228 // sdot v8.4s, v17.16b, v19.4b[0]\n" + ".inst 0x4f92e22c // sdot v12.4s, v17.16b, v18.4b[0]\n" + "ldr q17, [x10, #0x20]\n" + ".inst 0x4f93e209 // sdot v9.4s, v16.16b, v19.4b[0]\n" + ".inst 0x4f92e20d // sdot v13.4s, v16.16b, v18.4b[0]\n" + "ldr q16, [x10, #0x30]\n" + ".inst 0x4f93e22a // sdot v10.4s, v17.16b, v19.4b[0]\n" + ".inst 0x4f92e22e // sdot v14.4s, v17.16b, v18.4b[0]\n" "add x10, x10, #0x40\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" + ".inst 0x4f93e20b // sdot v11.4s, v16.16b, v19.4b[0]\n" + ".inst 0x4f92e20f // sdot v15.4s, v16.16b, v18.4b[0]\n" "bge 54b\n" "55:" // Height 2: Multiply loop: Skip odd blocks "cbz x27, 58f\n" @@ -636,19 +635,19 @@ void a64_hybrid_s8s32_dot_6x16 ( "ldr b0, [x26, #0x0]\n" "ldr b1, [x25, #0x0]\n" "57:" // Height 2: Multiply loop: Ragged operand read: Done - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" + "ldr q17, [x10, #0x0]\n" + "ldr q16, [x10, #0x10]\n" + ".inst 0x4f80e228 // sdot v8.4s, v17.16b, v0.4b[0]\n" + ".inst 0x4f81e22c // sdot v12.4s, v17.16b, v1.4b[0]\n" + "ldr q17, [x10, #0x20]\n" + ".inst 0x4f80e209 // sdot v9.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f81e20d // sdot v13.4s, v16.16b, v1.4b[0]\n" + "ldr q16, [x10, #0x30]\n" + ".inst 0x4f80e22a // sdot v10.4s, v17.16b, v0.4b[0]\n" + ".inst 0x4f81e22e // sdot v14.4s, v17.16b, v1.4b[0]\n" "add x10, x10, #0x40\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" + ".inst 0x4f80e20b // sdot v11.4s, v16.16b, v0.4b[0]\n" + ".inst 0x4f81e20f // sdot v15.4s, v16.16b, v1.4b[0]\n" "58:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -866,13 +865,13 @@ void a64_hybrid_s8s32_dot_6x16 ( "82:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 83f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 84f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -881,8 +880,8 @@ void a64_hybrid_s8s32_dot_6x16 ( "b 84f\n" "83:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" "84:" // Height 3: input setup done "cmp x27, #0x10\n" "blt 87f\n" @@ -899,75 +898,75 @@ void a64_hybrid_s8s32_dot_6x16 ( "sub x27, x27, #0x10\n" "add x26, x26, #0x10\n" ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q21, [x10, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" "add x25, x25, #0x10\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q20, [x10, #0x30]\n" "add x24, x24, #0x10\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" + ".inst 0x4f80e2aa // sdot v10.4s, v21.16b, v0.4b[0]\n" + ".inst 0x4f81e2ae // sdot v14.4s, v21.16b, v1.4b[0]\n" "cmp x27, #0x20\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" + ".inst 0x4f82e2b2 // sdot v18.4s, v21.16b, v2.4b[0]\n" + "ldr q21, [x10, #0x40]\n" + ".inst 0x4f80e28b // sdot v11.4s, v20.16b, v0.4b[0]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x10, #0x50]\n" + ".inst 0x4f81e28f // sdot v15.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f82e293 // sdot v19.4s, v20.16b, v2.4b[0]\n" + "ldr q20, [x10, #0x50]\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" - "ldr q7, [x10, #0xf0]\n" + ".inst 0x4fa0e2a8 // sdot v8.4s, v21.16b, v0.4b[1]\n" + ".inst 0x4fa1e2ac // sdot v12.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa2e2b0 // sdot v16.4s, v21.16b, v2.4b[1]\n" + "ldr q21, [x10, #0x60]\n" + ".inst 0x4fa0e289 // sdot v9.4s, v20.16b, v0.4b[1]\n" + ".inst 0x4fa1e28d // sdot v13.4s, v20.16b, v1.4b[1]\n" + ".inst 0x4fa2e291 // sdot v17.4s, v20.16b, v2.4b[1]\n" + "ldr q20, [x10, #0x70]\n" + ".inst 0x4fa0e2aa // sdot v10.4s, v21.16b, v0.4b[1]\n" + ".inst 0x4fa1e2ae // sdot v14.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa2e2b2 // sdot v18.4s, v21.16b, v2.4b[1]\n" + "ldr q21, [x10, #0x80]\n" + ".inst 0x4fa0e28b // sdot v11.4s, v20.16b, v0.4b[1]\n" + ".inst 0x4fa1e28f // sdot v15.4s, v20.16b, v1.4b[1]\n" + ".inst 0x4fa2e293 // sdot v19.4s, v20.16b, v2.4b[1]\n" + "ldr q20, [x10, #0x90]\n" + ".inst 0x4f80eaa8 // sdot v8.4s, v21.16b, v0.4b[2]\n" + ".inst 0x4f81eaac // sdot v12.4s, v21.16b, v1.4b[2]\n" + ".inst 0x4f82eab0 // sdot v16.4s, v21.16b, v2.4b[2]\n" + "ldr q21, [x10, #0xa0]\n" + ".inst 0x4f80ea89 // sdot v9.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f81ea8d // sdot v13.4s, v20.16b, v1.4b[2]\n" + ".inst 0x4f82ea91 // sdot v17.4s, v20.16b, v2.4b[2]\n" + "ldr q20, [x10, #0xb0]\n" + ".inst 0x4f80eaaa // sdot v10.4s, v21.16b, v0.4b[2]\n" + ".inst 0x4f81eaae // sdot v14.4s, v21.16b, v1.4b[2]\n" + ".inst 0x4f82eab2 // sdot v18.4s, v21.16b, v2.4b[2]\n" + "ldr q21, [x10, #0xc0]\n" + ".inst 0x4f80ea8b // sdot v11.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f81ea8f // sdot v15.4s, v20.16b, v1.4b[2]\n" + ".inst 0x4f82ea93 // sdot v19.4s, v20.16b, v2.4b[2]\n" + "ldr q20, [x10, #0xd0]\n" + ".inst 0x4fa0eaa8 // sdot v8.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa1eaac // sdot v12.4s, v21.16b, v1.4b[3]\n" + ".inst 0x4fa2eab0 // sdot v16.4s, v21.16b, v2.4b[3]\n" + "ldr q21, [x10, #0xe0]\n" + ".inst 0x4fa0ea89 // sdot v9.4s, v20.16b, v0.4b[3]\n" + ".inst 0x4fa1ea8d // sdot v13.4s, v20.16b, v1.4b[3]\n" + ".inst 0x4fa2ea91 // sdot v17.4s, v20.16b, v2.4b[3]\n" + "ldr q20, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" + ".inst 0x4fa0eaaa // sdot v10.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa1eaae // sdot v14.4s, v21.16b, v1.4b[3]\n" + ".inst 0x4fa2eab2 // sdot v18.4s, v21.16b, v2.4b[3]\n" "ldr q6, [x10, #0x0]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x4fa0ea8b // sdot v11.4s, v20.16b, v0.4b[3]\n" "ldr q0, [x26, #0x0]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" + ".inst 0x4fa1ea8f // sdot v15.4s, v20.16b, v1.4b[3]\n" "ldr q1, [x25, #0x0]\n" - ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" + ".inst 0x4fa2ea93 // sdot v19.4s, v20.16b, v2.4b[3]\n" "ldr q2, [x24, #0x0]\n" "ldr q7, [x10, #0x10]\n" "bge 85b\n" @@ -977,98 +976,98 @@ void a64_hybrid_s8s32_dot_6x16 ( "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q21, [x10, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" "add x24, x24, #0x10\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q20, [x10, #0x30]\n" "sub x27, x27, #0x10\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" + ".inst 0x4f80e2aa // sdot v10.4s, v21.16b, v0.4b[0]\n" + ".inst 0x4f81e2ae // sdot v14.4s, v21.16b, v1.4b[0]\n" "prfm pldl1keep, [x26, #0x80]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" + ".inst 0x4f82e2b2 // sdot v18.4s, v21.16b, v2.4b[0]\n" + "ldr q21, [x10, #0x40]\n" + ".inst 0x4f80e28b // sdot v11.4s, v20.16b, v0.4b[0]\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" - "ldr q7, [x10, #0xf0]\n" + ".inst 0x4f81e28f // sdot v15.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f82e293 // sdot v19.4s, v20.16b, v2.4b[0]\n" + "ldr q20, [x10, #0x50]\n" + ".inst 0x4fa0e2a8 // sdot v8.4s, v21.16b, v0.4b[1]\n" + ".inst 0x4fa1e2ac // sdot v12.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa2e2b0 // sdot v16.4s, v21.16b, v2.4b[1]\n" + "ldr q21, [x10, #0x60]\n" + ".inst 0x4fa0e289 // sdot v9.4s, v20.16b, v0.4b[1]\n" + ".inst 0x4fa1e28d // sdot v13.4s, v20.16b, v1.4b[1]\n" + ".inst 0x4fa2e291 // sdot v17.4s, v20.16b, v2.4b[1]\n" + "ldr q20, [x10, #0x70]\n" + ".inst 0x4fa0e2aa // sdot v10.4s, v21.16b, v0.4b[1]\n" + ".inst 0x4fa1e2ae // sdot v14.4s, v21.16b, v1.4b[1]\n" + ".inst 0x4fa2e2b2 // sdot v18.4s, v21.16b, v2.4b[1]\n" + "ldr q21, [x10, #0x80]\n" + ".inst 0x4fa0e28b // sdot v11.4s, v20.16b, v0.4b[1]\n" + ".inst 0x4fa1e28f // sdot v15.4s, v20.16b, v1.4b[1]\n" + ".inst 0x4fa2e293 // sdot v19.4s, v20.16b, v2.4b[1]\n" + "ldr q20, [x10, #0x90]\n" + ".inst 0x4f80eaa8 // sdot v8.4s, v21.16b, v0.4b[2]\n" + ".inst 0x4f81eaac // sdot v12.4s, v21.16b, v1.4b[2]\n" + ".inst 0x4f82eab0 // sdot v16.4s, v21.16b, v2.4b[2]\n" + "ldr q21, [x10, #0xa0]\n" + ".inst 0x4f80ea89 // sdot v9.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f81ea8d // sdot v13.4s, v20.16b, v1.4b[2]\n" + ".inst 0x4f82ea91 // sdot v17.4s, v20.16b, v2.4b[2]\n" + "ldr q20, [x10, #0xb0]\n" + ".inst 0x4f80eaaa // sdot v10.4s, v21.16b, v0.4b[2]\n" + ".inst 0x4f81eaae // sdot v14.4s, v21.16b, v1.4b[2]\n" + ".inst 0x4f82eab2 // sdot v18.4s, v21.16b, v2.4b[2]\n" + "ldr q21, [x10, #0xc0]\n" + ".inst 0x4f80ea8b // sdot v11.4s, v20.16b, v0.4b[2]\n" + ".inst 0x4f81ea8f // sdot v15.4s, v20.16b, v1.4b[2]\n" + ".inst 0x4f82ea93 // sdot v19.4s, v20.16b, v2.4b[2]\n" + "ldr q20, [x10, #0xd0]\n" + ".inst 0x4fa0eaa8 // sdot v8.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa1eaac // sdot v12.4s, v21.16b, v1.4b[3]\n" + ".inst 0x4fa2eab0 // sdot v16.4s, v21.16b, v2.4b[3]\n" + "ldr q21, [x10, #0xe0]\n" + ".inst 0x4fa0ea89 // sdot v9.4s, v20.16b, v0.4b[3]\n" + ".inst 0x4fa1ea8d // sdot v13.4s, v20.16b, v1.4b[3]\n" + ".inst 0x4fa2ea91 // sdot v17.4s, v20.16b, v2.4b[3]\n" + "ldr q20, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" + ".inst 0x4fa0eaaa // sdot v10.4s, v21.16b, v0.4b[3]\n" + ".inst 0x4fa1eaae // sdot v14.4s, v21.16b, v1.4b[3]\n" + ".inst 0x4fa2eab2 // sdot v18.4s, v21.16b, v2.4b[3]\n" + ".inst 0x4fa0ea8b // sdot v11.4s, v20.16b, v0.4b[3]\n" + ".inst 0x4fa1ea8f // sdot v15.4s, v20.16b, v1.4b[3]\n" + ".inst 0x4fa2ea93 // sdot v19.4s, v20.16b, v2.4b[3]\n" "87:" // Height 3: Multiply loop: Main loop skip "cbz x27, 92f\n" "cmp x27, #0x4\n" "blt 89f\n" "88:" // Height 3: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s24, [x26], #0x4\n" + "ldr s23, [x25], #0x4\n" "sub x27, x27, #0x4\n" "cmp x27, #0x4\n" - "ldr s2, [x24], #0x4\n" - "ldr q6, [x10, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr s22, [x24], #0x4\n" + "ldr q21, [x10, #0x0]\n" + ".inst 0x4f98e2a8 // sdot v8.4s, v21.16b, v24.4b[0]\n" + ".inst 0x4f97e2ac // sdot v12.4s, v21.16b, v23.4b[0]\n" + "ldr q20, [x10, #0x10]\n" + ".inst 0x4f96e2b0 // sdot v16.4s, v21.16b, v22.4b[0]\n" + "ldr q21, [x10, #0x20]\n" + ".inst 0x4f98e289 // sdot v9.4s, v20.16b, v24.4b[0]\n" + ".inst 0x4f97e28d // sdot v13.4s, v20.16b, v23.4b[0]\n" + ".inst 0x4f96e291 // sdot v17.4s, v20.16b, v22.4b[0]\n" + "ldr q20, [x10, #0x30]\n" "add x10, x10, #0x40\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" + ".inst 0x4f98e2aa // sdot v10.4s, v21.16b, v24.4b[0]\n" + ".inst 0x4f97e2ae // sdot v14.4s, v21.16b, v23.4b[0]\n" + ".inst 0x4f96e2b2 // sdot v18.4s, v21.16b, v22.4b[0]\n" + ".inst 0x4f98e28b // sdot v11.4s, v20.16b, v24.4b[0]\n" + ".inst 0x4f97e28f // sdot v15.4s, v20.16b, v23.4b[0]\n" + ".inst 0x4f96e293 // sdot v19.4s, v20.16b, v22.4b[0]\n" "bge 88b\n" "89:" // Height 3: Multiply loop: Skip odd blocks "cbz x27, 92f\n" @@ -1086,23 +1085,23 @@ void a64_hybrid_s8s32_dot_6x16 ( "ldr b1, [x25, #0x0]\n" "ldr b2, [x24, #0x0]\n" "91:" // Height 3: Multiply loop: Ragged operand read: Done - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q21, [x10, #0x0]\n" + "ldr q20, [x10, #0x10]\n" + ".inst 0x4f80e2a8 // sdot v8.4s, v21.16b, v0.4b[0]\n" + ".inst 0x4f81e2ac // sdot v12.4s, v21.16b, v1.4b[0]\n" + ".inst 0x4f82e2b0 // sdot v16.4s, v21.16b, v2.4b[0]\n" + "ldr q21, [x10, #0x20]\n" + ".inst 0x4f80e289 // sdot v9.4s, v20.16b, v0.4b[0]\n" + ".inst 0x4f81e28d // sdot v13.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f82e291 // sdot v17.4s, v20.16b, v2.4b[0]\n" + "ldr q20, [x10, #0x30]\n" "add x10, x10, #0x40\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" + ".inst 0x4f80e2aa // sdot v10.4s, v21.16b, v0.4b[0]\n" + ".inst 0x4f81e2ae // sdot v14.4s, v21.16b, v1.4b[0]\n" + ".inst 0x4f82e2b2 // sdot v18.4s, v21.16b, v2.4b[0]\n" + ".inst 0x4f80e28b // sdot v11.4s, v20.16b, v0.4b[0]\n" + ".inst 0x4f81e28f // sdot v15.4s, v20.16b, v1.4b[0]\n" + ".inst 0x4f82e293 // sdot v19.4s, v20.16b, v2.4b[0]\n" "92:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1367,14 +1366,14 @@ void a64_hybrid_s8s32_dot_6x16 ( "116:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 117f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 118f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -1384,9 +1383,9 @@ void a64_hybrid_s8s32_dot_6x16 ( "b 118f\n" "117:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" "118:" // Height 4: input setup done "cmp x27, #0x10\n" "blt 121f\n" @@ -1405,7 +1404,7 @@ void a64_hybrid_s8s32_dot_6x16 ( "add x26, x26, #0x10\n" ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q25, [x10, #0x20]\n" "add x25, x25, #0x10\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" @@ -1413,85 +1412,85 @@ void a64_hybrid_s8s32_dot_6x16 ( "add x23, x23, #0x10\n" ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q24, [x10, #0x30]\n" "cmp x27, #0x20\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" + ".inst 0x4f80e32a // sdot v10.4s, v25.16b, v0.4b[0]\n" + ".inst 0x4f81e32e // sdot v14.4s, v25.16b, v1.4b[0]\n" "prfm pldl1keep, [x26, #0x80]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x10, #0x40]\n" + ".inst 0x4f82e332 // sdot v18.4s, v25.16b, v2.4b[0]\n" + ".inst 0x4f83e336 // sdot v22.4s, v25.16b, v3.4b[0]\n" + "ldr q25, [x10, #0x40]\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" + ".inst 0x4f80e30b // sdot v11.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f81e30f // sdot v15.4s, v24.16b, v1.4b[0]\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" - ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" - ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" - ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n" - "ldr q7, [x10, #0xf0]\n" + ".inst 0x4f82e313 // sdot v19.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f83e317 // sdot v23.4s, v24.16b, v3.4b[0]\n" + "ldr q24, [x10, #0x50]\n" + ".inst 0x4fa0e328 // sdot v8.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa1e32c // sdot v12.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa2e330 // sdot v16.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa3e334 // sdot v20.4s, v25.16b, v3.4b[1]\n" + "ldr q25, [x10, #0x60]\n" + ".inst 0x4fa0e309 // sdot v9.4s, v24.16b, v0.4b[1]\n" + ".inst 0x4fa1e30d // sdot v13.4s, v24.16b, v1.4b[1]\n" + ".inst 0x4fa2e311 // sdot v17.4s, v24.16b, v2.4b[1]\n" + ".inst 0x4fa3e315 // sdot v21.4s, v24.16b, v3.4b[1]\n" + "ldr q24, [x10, #0x70]\n" + ".inst 0x4fa0e32a // sdot v10.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa1e32e // sdot v14.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa2e332 // sdot v18.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa3e336 // sdot v22.4s, v25.16b, v3.4b[1]\n" + "ldr q25, [x10, #0x80]\n" + ".inst 0x4fa0e30b // sdot v11.4s, v24.16b, v0.4b[1]\n" + ".inst 0x4fa1e30f // sdot v15.4s, v24.16b, v1.4b[1]\n" + ".inst 0x4fa2e313 // sdot v19.4s, v24.16b, v2.4b[1]\n" + ".inst 0x4fa3e317 // sdot v23.4s, v24.16b, v3.4b[1]\n" + "ldr q24, [x10, #0x90]\n" + ".inst 0x4f80eb28 // sdot v8.4s, v25.16b, v0.4b[2]\n" + ".inst 0x4f81eb2c // sdot v12.4s, v25.16b, v1.4b[2]\n" + ".inst 0x4f82eb30 // sdot v16.4s, v25.16b, v2.4b[2]\n" + ".inst 0x4f83eb34 // sdot v20.4s, v25.16b, v3.4b[2]\n" + "ldr q25, [x10, #0xa0]\n" + ".inst 0x4f80eb09 // sdot v9.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f81eb0d // sdot v13.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f82eb11 // sdot v17.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f83eb15 // sdot v21.4s, v24.16b, v3.4b[2]\n" + "ldr q24, [x10, #0xb0]\n" + ".inst 0x4f80eb2a // sdot v10.4s, v25.16b, v0.4b[2]\n" + ".inst 0x4f81eb2e // sdot v14.4s, v25.16b, v1.4b[2]\n" + ".inst 0x4f82eb32 // sdot v18.4s, v25.16b, v2.4b[2]\n" + ".inst 0x4f83eb36 // sdot v22.4s, v25.16b, v3.4b[2]\n" + "ldr q25, [x10, #0xc0]\n" + ".inst 0x4f80eb0b // sdot v11.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f81eb0f // sdot v15.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f82eb13 // sdot v19.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f83eb17 // sdot v23.4s, v24.16b, v3.4b[2]\n" + "ldr q24, [x10, #0xd0]\n" + ".inst 0x4fa0eb28 // sdot v8.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa1eb2c // sdot v12.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa2eb30 // sdot v16.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa3eb34 // sdot v20.4s, v25.16b, v3.4b[3]\n" + "ldr q25, [x10, #0xe0]\n" + ".inst 0x4fa0eb09 // sdot v9.4s, v24.16b, v0.4b[3]\n" + ".inst 0x4fa1eb0d // sdot v13.4s, v24.16b, v1.4b[3]\n" + ".inst 0x4fa2eb11 // sdot v17.4s, v24.16b, v2.4b[3]\n" + ".inst 0x4fa3eb15 // sdot v21.4s, v24.16b, v3.4b[3]\n" + "ldr q24, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n" + ".inst 0x4fa0eb2a // sdot v10.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa1eb2e // sdot v14.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa2eb32 // sdot v18.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa3eb36 // sdot v22.4s, v25.16b, v3.4b[3]\n" "ldr q6, [x10, #0x0]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x4fa0eb0b // sdot v11.4s, v24.16b, v0.4b[3]\n" "ldr q0, [x26, #0x0]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" + ".inst 0x4fa1eb0f // sdot v15.4s, v24.16b, v1.4b[3]\n" "ldr q1, [x25, #0x0]\n" - ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" + ".inst 0x4fa2eb13 // sdot v19.4s, v24.16b, v2.4b[3]\n" "ldr q2, [x24, #0x0]\n" - ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n" + ".inst 0x4fa3eb17 // sdot v23.4s, v24.16b, v3.4b[3]\n" "ldr q3, [x23, #0x0]\n" "ldr q7, [x10, #0x10]\n" "bge 119b\n" @@ -1502,7 +1501,7 @@ void a64_hybrid_s8s32_dot_6x16 ( "add x25, x25, #0x10\n" ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q25, [x10, #0x20]\n" "add x24, x24, #0x10\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" @@ -1510,112 +1509,112 @@ void a64_hybrid_s8s32_dot_6x16 ( "sub x27, x27, #0x10\n" ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q24, [x10, #0x30]\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" + ".inst 0x4f80e32a // sdot v10.4s, v25.16b, v0.4b[0]\n" + ".inst 0x4f81e32e // sdot v14.4s, v25.16b, v1.4b[0]\n" "prfm pldl1keep, [x25, #0x80]\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x10, #0x40]\n" + ".inst 0x4f82e332 // sdot v18.4s, v25.16b, v2.4b[0]\n" + ".inst 0x4f83e336 // sdot v22.4s, v25.16b, v3.4b[0]\n" + "ldr q25, [x10, #0x40]\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" - ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" - ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" - ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n" - "ldr q7, [x10, #0xf0]\n" + ".inst 0x4f80e30b // sdot v11.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f81e30f // sdot v15.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f82e313 // sdot v19.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f83e317 // sdot v23.4s, v24.16b, v3.4b[0]\n" + "ldr q24, [x10, #0x50]\n" + ".inst 0x4fa0e328 // sdot v8.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa1e32c // sdot v12.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa2e330 // sdot v16.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa3e334 // sdot v20.4s, v25.16b, v3.4b[1]\n" + "ldr q25, [x10, #0x60]\n" + ".inst 0x4fa0e309 // sdot v9.4s, v24.16b, v0.4b[1]\n" + ".inst 0x4fa1e30d // sdot v13.4s, v24.16b, v1.4b[1]\n" + ".inst 0x4fa2e311 // sdot v17.4s, v24.16b, v2.4b[1]\n" + ".inst 0x4fa3e315 // sdot v21.4s, v24.16b, v3.4b[1]\n" + "ldr q24, [x10, #0x70]\n" + ".inst 0x4fa0e32a // sdot v10.4s, v25.16b, v0.4b[1]\n" + ".inst 0x4fa1e32e // sdot v14.4s, v25.16b, v1.4b[1]\n" + ".inst 0x4fa2e332 // sdot v18.4s, v25.16b, v2.4b[1]\n" + ".inst 0x4fa3e336 // sdot v22.4s, v25.16b, v3.4b[1]\n" + "ldr q25, [x10, #0x80]\n" + ".inst 0x4fa0e30b // sdot v11.4s, v24.16b, v0.4b[1]\n" + ".inst 0x4fa1e30f // sdot v15.4s, v24.16b, v1.4b[1]\n" + ".inst 0x4fa2e313 // sdot v19.4s, v24.16b, v2.4b[1]\n" + ".inst 0x4fa3e317 // sdot v23.4s, v24.16b, v3.4b[1]\n" + "ldr q24, [x10, #0x90]\n" + ".inst 0x4f80eb28 // sdot v8.4s, v25.16b, v0.4b[2]\n" + ".inst 0x4f81eb2c // sdot v12.4s, v25.16b, v1.4b[2]\n" + ".inst 0x4f82eb30 // sdot v16.4s, v25.16b, v2.4b[2]\n" + ".inst 0x4f83eb34 // sdot v20.4s, v25.16b, v3.4b[2]\n" + "ldr q25, [x10, #0xa0]\n" + ".inst 0x4f80eb09 // sdot v9.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f81eb0d // sdot v13.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f82eb11 // sdot v17.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f83eb15 // sdot v21.4s, v24.16b, v3.4b[2]\n" + "ldr q24, [x10, #0xb0]\n" + ".inst 0x4f80eb2a // sdot v10.4s, v25.16b, v0.4b[2]\n" + ".inst 0x4f81eb2e // sdot v14.4s, v25.16b, v1.4b[2]\n" + ".inst 0x4f82eb32 // sdot v18.4s, v25.16b, v2.4b[2]\n" + ".inst 0x4f83eb36 // sdot v22.4s, v25.16b, v3.4b[2]\n" + "ldr q25, [x10, #0xc0]\n" + ".inst 0x4f80eb0b // sdot v11.4s, v24.16b, v0.4b[2]\n" + ".inst 0x4f81eb0f // sdot v15.4s, v24.16b, v1.4b[2]\n" + ".inst 0x4f82eb13 // sdot v19.4s, v24.16b, v2.4b[2]\n" + ".inst 0x4f83eb17 // sdot v23.4s, v24.16b, v3.4b[2]\n" + "ldr q24, [x10, #0xd0]\n" + ".inst 0x4fa0eb28 // sdot v8.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa1eb2c // sdot v12.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa2eb30 // sdot v16.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa3eb34 // sdot v20.4s, v25.16b, v3.4b[3]\n" + "ldr q25, [x10, #0xe0]\n" + ".inst 0x4fa0eb09 // sdot v9.4s, v24.16b, v0.4b[3]\n" + ".inst 0x4fa1eb0d // sdot v13.4s, v24.16b, v1.4b[3]\n" + ".inst 0x4fa2eb11 // sdot v17.4s, v24.16b, v2.4b[3]\n" + ".inst 0x4fa3eb15 // sdot v21.4s, v24.16b, v3.4b[3]\n" + "ldr q24, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" - ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n" + ".inst 0x4fa0eb2a // sdot v10.4s, v25.16b, v0.4b[3]\n" + ".inst 0x4fa1eb2e // sdot v14.4s, v25.16b, v1.4b[3]\n" + ".inst 0x4fa2eb32 // sdot v18.4s, v25.16b, v2.4b[3]\n" + ".inst 0x4fa3eb36 // sdot v22.4s, v25.16b, v3.4b[3]\n" + ".inst 0x4fa0eb0b // sdot v11.4s, v24.16b, v0.4b[3]\n" + ".inst 0x4fa1eb0f // sdot v15.4s, v24.16b, v1.4b[3]\n" + ".inst 0x4fa2eb13 // sdot v19.4s, v24.16b, v2.4b[3]\n" + ".inst 0x4fa3eb17 // sdot v23.4s, v24.16b, v3.4b[3]\n" "121:" // Height 4: Multiply loop: Main loop skip "cbz x27, 126f\n" "cmp x27, #0x4\n" "blt 123f\n" "122:" // Height 4: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s29, [x26], #0x4\n" + "ldr s28, [x25], #0x4\n" "sub x27, x27, #0x4\n" "cmp x27, #0x4\n" - "ldr s2, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr s27, [x24], #0x4\n" + "ldr s26, [x23], #0x4\n" + "ldr q25, [x10, #0x0]\n" + "ldr q24, [x10, #0x10]\n" + ".inst 0x4f9de328 // sdot v8.4s, v25.16b, v29.4b[0]\n" + ".inst 0x4f9ce32c // sdot v12.4s, v25.16b, v28.4b[0]\n" + ".inst 0x4f9be330 // sdot v16.4s, v25.16b, v27.4b[0]\n" + ".inst 0x4f9ae334 // sdot v20.4s, v25.16b, v26.4b[0]\n" + "ldr q25, [x10, #0x20]\n" + ".inst 0x4f9de309 // sdot v9.4s, v24.16b, v29.4b[0]\n" + ".inst 0x4f9ce30d // sdot v13.4s, v24.16b, v28.4b[0]\n" + ".inst 0x4f9be311 // sdot v17.4s, v24.16b, v27.4b[0]\n" + ".inst 0x4f9ae315 // sdot v21.4s, v24.16b, v26.4b[0]\n" + "ldr q24, [x10, #0x30]\n" "add x10, x10, #0x40\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" + ".inst 0x4f9de32a // sdot v10.4s, v25.16b, v29.4b[0]\n" + ".inst 0x4f9ce32e // sdot v14.4s, v25.16b, v28.4b[0]\n" + ".inst 0x4f9be332 // sdot v18.4s, v25.16b, v27.4b[0]\n" + ".inst 0x4f9ae336 // sdot v22.4s, v25.16b, v26.4b[0]\n" + ".inst 0x4f9de30b // sdot v11.4s, v24.16b, v29.4b[0]\n" + ".inst 0x4f9ce30f // sdot v15.4s, v24.16b, v28.4b[0]\n" + ".inst 0x4f9be313 // sdot v19.4s, v24.16b, v27.4b[0]\n" + ".inst 0x4f9ae317 // sdot v23.4s, v24.16b, v26.4b[0]\n" "bge 122b\n" "123:" // Height 4: Multiply loop: Skip odd blocks "cbz x27, 126f\n" @@ -1636,27 +1635,27 @@ void a64_hybrid_s8s32_dot_6x16 ( "ldr b2, [x24, #0x0]\n" "ldr b3, [x23, #0x0]\n" "125:" // Height 4: Multiply loop: Ragged operand read: Done - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q25, [x10, #0x0]\n" + "ldr q24, [x10, #0x10]\n" + ".inst 0x4f80e328 // sdot v8.4s, v25.16b, v0.4b[0]\n" + ".inst 0x4f81e32c // sdot v12.4s, v25.16b, v1.4b[0]\n" + ".inst 0x4f82e330 // sdot v16.4s, v25.16b, v2.4b[0]\n" + ".inst 0x4f83e334 // sdot v20.4s, v25.16b, v3.4b[0]\n" + "ldr q25, [x10, #0x20]\n" + ".inst 0x4f80e309 // sdot v9.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f81e30d // sdot v13.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f82e311 // sdot v17.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f83e315 // sdot v21.4s, v24.16b, v3.4b[0]\n" + "ldr q24, [x10, #0x30]\n" "add x10, x10, #0x40\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" + ".inst 0x4f80e32a // sdot v10.4s, v25.16b, v0.4b[0]\n" + ".inst 0x4f81e32e // sdot v14.4s, v25.16b, v1.4b[0]\n" + ".inst 0x4f82e332 // sdot v18.4s, v25.16b, v2.4b[0]\n" + ".inst 0x4f83e336 // sdot v22.4s, v25.16b, v3.4b[0]\n" + ".inst 0x4f80e30b // sdot v11.4s, v24.16b, v0.4b[0]\n" + ".inst 0x4f81e30f // sdot v15.4s, v24.16b, v1.4b[0]\n" + ".inst 0x4f82e313 // sdot v19.4s, v24.16b, v2.4b[0]\n" + ".inst 0x4f83e317 // sdot v23.4s, v24.16b, v3.4b[0]\n" "126:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1960,162 +1959,162 @@ void a64_hybrid_s8s32_dot_6x16 ( "movi v22.4s, #0x0\n" "movi v23.4s, #0x0\n" "movi v24.4s, #0x0\n" - "movi v25.4s, #0x0\n" - "movi v26.4s, #0x0\n" - "movi v27.4s, #0x0\n" - "149:" // Height 5: setup done - "mov x28, #0x0\n" - "150:" // Height 5: String loop - "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" - "tbz %x[flags], #3, 151f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "cbnz x28, 152f\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x26, x26, x20\n" - "add x25, x25, x20\n" - "add x24, x24, x20\n" - "add x23, x23, x20\n" - "add x22, x22, x20\n" - "b 152f\n" - "151:" // Height 5: setup direct input - "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" - "152:" // Height 5: input setup done - "cmp x27, #0x10\n" - "blt 155f\n" - "ldr q0, [x26, #0x0]\n" - "ldr q1, [x25, #0x0]\n" - "cmp x27, #0x20\n" - "ldr q2, [x24, #0x0]\n" - "ldr q3, [x23, #0x0]\n" - "ldr q4, [x22, #0x0]\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - "blt 154f\n" - "153:" // Height 5: Multiply loop: Main loop head - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "sub x27, x27, #0x10\n" - "add x26, x26, #0x10\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - "add x25, x25, #0x10\n" - "add x24, x24, #0x10\n" - ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - "add x23, x23, #0x10\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - "add x22, x22, #0x10\n" - "cmp x27, #0x20\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x10, #0x30]\n" - "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - "prfm pldl1keep, [x25, #0x80]\n" - "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - "prfm pldl1keep, [x23, #0x80]\n" - "prfm pldl1keep, [x22, #0x80]\n" - ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" - ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n" - ".inst 0x4fa4e0d8 // sdot v24.4s, v6.16b, v4.4b[1]\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n" - ".inst 0x4fa4e0f9 // sdot v25.4s, v7.16b, v4.4b[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n" - ".inst 0x4fa4e0da // sdot v26.4s, v6.16b, v4.4b[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" - ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n" - ".inst 0x4fa4e0fb // sdot v27.4s, v7.16b, v4.4b[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n" - ".inst 0x4f84e8d8 // sdot v24.4s, v6.16b, v4.4b[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n" - ".inst 0x4f84e8f9 // sdot v25.4s, v7.16b, v4.4b[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n" - ".inst 0x4f84e8da // sdot v26.4s, v6.16b, v4.4b[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n" - ".inst 0x4f84e8fb // sdot v27.4s, v7.16b, v4.4b[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n" - ".inst 0x4fa4e8d8 // sdot v24.4s, v6.16b, v4.4b[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" - ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n" - ".inst 0x4fa4e8f9 // sdot v25.4s, v7.16b, v4.4b[3]\n" - "ldr q7, [x10, #0xf0]\n" + "movi v25.4s, #0x0\n" + "movi v26.4s, #0x0\n" + "movi v27.4s, #0x0\n" + "149:" // Height 5: setup done + "mov x28, #0x0\n" + "150:" // Height 5: String loop + "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" + "ldr w27, [x20, x28, LSL #0x2]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" + "tbz %x[flags], #3, 151f\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "cbnz x28, 152f\n" + "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" + "add x26, x26, x20\n" + "add x25, x25, x20\n" + "add x24, x24, x20\n" + "add x23, x23, x20\n" + "add x22, x22, x20\n" + "b 152f\n" + "151:" // Height 5: setup direct input + "mov x26, %x[input_ptr]\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" + "152:" // Height 5: input setup done + "cmp x27, #0x10\n" + "blt 155f\n" + "ldr q0, [x26, #0x0]\n" + "ldr q1, [x25, #0x0]\n" + "cmp x27, #0x20\n" + "ldr q2, [x24, #0x0]\n" + "ldr q3, [x23, #0x0]\n" + "ldr q4, [x22, #0x0]\n" + "ldr q6, [x10, #0x0]\n" + "ldr q7, [x10, #0x10]\n" + "blt 154f\n" + "153:" // Height 5: Multiply loop: Main loop head + ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" + ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" + "sub x27, x27, #0x10\n" + "add x26, x26, #0x10\n" + ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" + ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" + "ldr q29, [x10, #0x20]\n" + ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" + "add x23, x23, #0x10\n" + ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" + ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" + "add x22, x22, #0x10\n" + "cmp x27, #0x20\n" + ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" + ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" + "ldr q28, [x10, #0x30]\n" + "prfm pldl1keep, [x26, #0x80]\n" + ".inst 0x4f80e3aa // sdot v10.4s, v29.16b, v0.4b[0]\n" + ".inst 0x4f81e3ae // sdot v14.4s, v29.16b, v1.4b[0]\n" + "prfm pldl1keep, [x25, #0x80]\n" + "prfm pldl1keep, [x24, #0x80]\n" + ".inst 0x4f82e3b2 // sdot v18.4s, v29.16b, v2.4b[0]\n" + ".inst 0x4f83e3b6 // sdot v22.4s, v29.16b, v3.4b[0]\n" + "prfm pldl1keep, [x23, #0x80]\n" + "prfm pldl1keep, [x22, #0x80]\n" + ".inst 0x4f84e3ba // sdot v26.4s, v29.16b, v4.4b[0]\n" + "ldr q29, [x10, #0x40]\n" + ".inst 0x4f80e38b // sdot v11.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f81e38f // sdot v15.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f82e393 // sdot v19.4s, v28.16b, v2.4b[0]\n" + ".inst 0x4f83e397 // sdot v23.4s, v28.16b, v3.4b[0]\n" + ".inst 0x4f84e39b // sdot v27.4s, v28.16b, v4.4b[0]\n" + "ldr q28, [x10, #0x50]\n" + ".inst 0x4fa0e3a8 // sdot v8.4s, v29.16b, v0.4b[1]\n" + ".inst 0x4fa1e3ac // sdot v12.4s, v29.16b, v1.4b[1]\n" + ".inst 0x4fa2e3b0 // sdot v16.4s, v29.16b, v2.4b[1]\n" + ".inst 0x4fa3e3b4 // sdot v20.4s, v29.16b, v3.4b[1]\n" + ".inst 0x4fa4e3b8 // sdot v24.4s, v29.16b, v4.4b[1]\n" + "ldr q29, [x10, #0x60]\n" + ".inst 0x4fa0e389 // sdot v9.4s, v28.16b, v0.4b[1]\n" + ".inst 0x4fa1e38d // sdot v13.4s, v28.16b, v1.4b[1]\n" + ".inst 0x4fa2e391 // sdot v17.4s, v28.16b, v2.4b[1]\n" + ".inst 0x4fa3e395 // sdot v21.4s, v28.16b, v3.4b[1]\n" + ".inst 0x4fa4e399 // sdot v25.4s, v28.16b, v4.4b[1]\n" + "ldr q28, [x10, #0x70]\n" + ".inst 0x4fa0e3aa // sdot v10.4s, v29.16b, v0.4b[1]\n" + ".inst 0x4fa1e3ae // sdot v14.4s, v29.16b, v1.4b[1]\n" + ".inst 0x4fa2e3b2 // sdot v18.4s, v29.16b, v2.4b[1]\n" + ".inst 0x4fa3e3b6 // sdot v22.4s, v29.16b, v3.4b[1]\n" + ".inst 0x4fa4e3ba // sdot v26.4s, v29.16b, v4.4b[1]\n" + "ldr q29, [x10, #0x80]\n" + ".inst 0x4fa0e38b // sdot v11.4s, v28.16b, v0.4b[1]\n" + ".inst 0x4fa1e38f // sdot v15.4s, v28.16b, v1.4b[1]\n" + ".inst 0x4fa2e393 // sdot v19.4s, v28.16b, v2.4b[1]\n" + ".inst 0x4fa3e397 // sdot v23.4s, v28.16b, v3.4b[1]\n" + ".inst 0x4fa4e39b // sdot v27.4s, v28.16b, v4.4b[1]\n" + "ldr q28, [x10, #0x90]\n" + ".inst 0x4f80eba8 // sdot v8.4s, v29.16b, v0.4b[2]\n" + ".inst 0x4f81ebac // sdot v12.4s, v29.16b, v1.4b[2]\n" + ".inst 0x4f82ebb0 // sdot v16.4s, v29.16b, v2.4b[2]\n" + ".inst 0x4f83ebb4 // sdot v20.4s, v29.16b, v3.4b[2]\n" + ".inst 0x4f84ebb8 // sdot v24.4s, v29.16b, v4.4b[2]\n" + "ldr q29, [x10, #0xa0]\n" + ".inst 0x4f80eb89 // sdot v9.4s, v28.16b, v0.4b[2]\n" + ".inst 0x4f81eb8d // sdot v13.4s, v28.16b, v1.4b[2]\n" + ".inst 0x4f82eb91 // sdot v17.4s, v28.16b, v2.4b[2]\n" + ".inst 0x4f83eb95 // sdot v21.4s, v28.16b, v3.4b[2]\n" + ".inst 0x4f84eb99 // sdot v25.4s, v28.16b, v4.4b[2]\n" + "ldr q28, [x10, #0xb0]\n" + ".inst 0x4f80ebaa // sdot v10.4s, v29.16b, v0.4b[2]\n" + ".inst 0x4f81ebae // sdot v14.4s, v29.16b, v1.4b[2]\n" + ".inst 0x4f82ebb2 // sdot v18.4s, v29.16b, v2.4b[2]\n" + ".inst 0x4f83ebb6 // sdot v22.4s, v29.16b, v3.4b[2]\n" + ".inst 0x4f84ebba // sdot v26.4s, v29.16b, v4.4b[2]\n" + "ldr q29, [x10, #0xc0]\n" + ".inst 0x4f80eb8b // sdot v11.4s, v28.16b, v0.4b[2]\n" + ".inst 0x4f81eb8f // sdot v15.4s, v28.16b, v1.4b[2]\n" + ".inst 0x4f82eb93 // sdot v19.4s, v28.16b, v2.4b[2]\n" + ".inst 0x4f83eb97 // sdot v23.4s, v28.16b, v3.4b[2]\n" + ".inst 0x4f84eb9b // sdot v27.4s, v28.16b, v4.4b[2]\n" + "ldr q28, [x10, #0xd0]\n" + ".inst 0x4fa0eba8 // sdot v8.4s, v29.16b, v0.4b[3]\n" + ".inst 0x4fa1ebac // sdot v12.4s, v29.16b, v1.4b[3]\n" + ".inst 0x4fa2ebb0 // sdot v16.4s, v29.16b, v2.4b[3]\n" + ".inst 0x4fa3ebb4 // sdot v20.4s, v29.16b, v3.4b[3]\n" + ".inst 0x4fa4ebb8 // sdot v24.4s, v29.16b, v4.4b[3]\n" + "ldr q29, [x10, #0xe0]\n" + ".inst 0x4fa0eb89 // sdot v9.4s, v28.16b, v0.4b[3]\n" + ".inst 0x4fa1eb8d // sdot v13.4s, v28.16b, v1.4b[3]\n" + ".inst 0x4fa2eb91 // sdot v17.4s, v28.16b, v2.4b[3]\n" + ".inst 0x4fa3eb95 // sdot v21.4s, v28.16b, v3.4b[3]\n" + ".inst 0x4fa4eb99 // sdot v25.4s, v28.16b, v4.4b[3]\n" + "ldr q28, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n" - ".inst 0x4fa4e8da // sdot v26.4s, v6.16b, v4.4b[3]\n" + ".inst 0x4fa0ebaa // sdot v10.4s, v29.16b, v0.4b[3]\n" + ".inst 0x4fa1ebae // sdot v14.4s, v29.16b, v1.4b[3]\n" + ".inst 0x4fa2ebb2 // sdot v18.4s, v29.16b, v2.4b[3]\n" + ".inst 0x4fa3ebb6 // sdot v22.4s, v29.16b, v3.4b[3]\n" + ".inst 0x4fa4ebba // sdot v26.4s, v29.16b, v4.4b[3]\n" "ldr q6, [x10, #0x0]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x4fa0eb8b // sdot v11.4s, v28.16b, v0.4b[3]\n" "ldr q0, [x26, #0x0]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" + ".inst 0x4fa1eb8f // sdot v15.4s, v28.16b, v1.4b[3]\n" "ldr q1, [x25, #0x0]\n" - ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" + ".inst 0x4fa2eb93 // sdot v19.4s, v28.16b, v2.4b[3]\n" "ldr q2, [x24, #0x0]\n" - ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n" + ".inst 0x4fa3eb97 // sdot v23.4s, v28.16b, v3.4b[3]\n" "ldr q3, [x23, #0x0]\n" - ".inst 0x4fa4e8fb // sdot v27.4s, v7.16b, v4.4b[3]\n" + ".inst 0x4fa4eb9b // sdot v27.4s, v28.16b, v4.4b[3]\n" "ldr q4, [x22, #0x0]\n" "ldr q7, [x10, #0x10]\n" "bge 153b\n" @@ -2129,7 +2128,7 @@ void a64_hybrid_s8s32_dot_6x16 ( "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q29, [x10, #0x20]\n" ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" "add x22, x22, #0x10\n" ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" @@ -2138,131 +2137,131 @@ void a64_hybrid_s8s32_dot_6x16 ( "prfm pldl1keep, [x26, #0x80]\n" ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q28, [x10, #0x30]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" + ".inst 0x4f80e3aa // sdot v10.4s, v29.16b, v0.4b[0]\n" + ".inst 0x4f81e3ae // sdot v14.4s, v29.16b, v1.4b[0]\n" "prfm pldl1keep, [x24, #0x80]\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" + ".inst 0x4f82e3b2 // sdot v18.4s, v29.16b, v2.4b[0]\n" + ".inst 0x4f83e3b6 // sdot v22.4s, v29.16b, v3.4b[0]\n" "prfm pldl1keep, [x22, #0x80]\n" - ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x4fa0e0c8 // sdot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0cc // sdot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d0 // sdot v16.4s, v6.16b, v2.4b[1]\n" - ".inst 0x4fa3e0d4 // sdot v20.4s, v6.16b, v3.4b[1]\n" - ".inst 0x4fa4e0d8 // sdot v24.4s, v6.16b, v4.4b[1]\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x4fa0e0e9 // sdot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ed // sdot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f1 // sdot v17.4s, v7.16b, v2.4b[1]\n" - ".inst 0x4fa3e0f5 // sdot v21.4s, v7.16b, v3.4b[1]\n" - ".inst 0x4fa4e0f9 // sdot v25.4s, v7.16b, v4.4b[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x4fa0e0ca // sdot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ce // sdot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x4fa2e0d2 // sdot v18.4s, v6.16b, v2.4b[1]\n" - ".inst 0x4fa3e0d6 // sdot v22.4s, v6.16b, v3.4b[1]\n" - ".inst 0x4fa4e0da // sdot v26.4s, v6.16b, v4.4b[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x4fa0e0eb // sdot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x4fa1e0ef // sdot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x4fa2e0f3 // sdot v19.4s, v7.16b, v2.4b[1]\n" - ".inst 0x4fa3e0f7 // sdot v23.4s, v7.16b, v3.4b[1]\n" - ".inst 0x4fa4e0fb // sdot v27.4s, v7.16b, v4.4b[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x4f80e8c8 // sdot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8cc // sdot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d0 // sdot v16.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d4 // sdot v20.4s, v6.16b, v3.4b[2]\n" - ".inst 0x4f84e8d8 // sdot v24.4s, v6.16b, v4.4b[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x4f80e8e9 // sdot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ed // sdot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f1 // sdot v17.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f5 // sdot v21.4s, v7.16b, v3.4b[2]\n" - ".inst 0x4f84e8f9 // sdot v25.4s, v7.16b, v4.4b[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x4f80e8ca // sdot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x4f81e8ce // sdot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x4f82e8d2 // sdot v18.4s, v6.16b, v2.4b[2]\n" - ".inst 0x4f83e8d6 // sdot v22.4s, v6.16b, v3.4b[2]\n" - ".inst 0x4f84e8da // sdot v26.4s, v6.16b, v4.4b[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x4f80e8eb // sdot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x4f81e8ef // sdot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x4f82e8f3 // sdot v19.4s, v7.16b, v2.4b[2]\n" - ".inst 0x4f83e8f7 // sdot v23.4s, v7.16b, v3.4b[2]\n" - ".inst 0x4f84e8fb // sdot v27.4s, v7.16b, v4.4b[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x4fa0e8c8 // sdot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8cc // sdot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d0 // sdot v16.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d4 // sdot v20.4s, v6.16b, v3.4b[3]\n" - ".inst 0x4fa4e8d8 // sdot v24.4s, v6.16b, v4.4b[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x4fa0e8e9 // sdot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ed // sdot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f1 // sdot v17.4s, v7.16b, v2.4b[3]\n" - ".inst 0x4fa3e8f5 // sdot v21.4s, v7.16b, v3.4b[3]\n" - ".inst 0x4fa4e8f9 // sdot v25.4s, v7.16b, v4.4b[3]\n" - "ldr q7, [x10, #0xf0]\n" + ".inst 0x4f84e3ba // sdot v26.4s, v29.16b, v4.4b[0]\n" + "ldr q29, [x10, #0x40]\n" + ".inst 0x4f80e38b // sdot v11.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f81e38f // sdot v15.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f82e393 // sdot v19.4s, v28.16b, v2.4b[0]\n" + ".inst 0x4f83e397 // sdot v23.4s, v28.16b, v3.4b[0]\n" + ".inst 0x4f84e39b // sdot v27.4s, v28.16b, v4.4b[0]\n" + "ldr q28, [x10, #0x50]\n" + ".inst 0x4fa0e3a8 // sdot v8.4s, v29.16b, v0.4b[1]\n" + ".inst 0x4fa1e3ac // sdot v12.4s, v29.16b, v1.4b[1]\n" + ".inst 0x4fa2e3b0 // sdot v16.4s, v29.16b, v2.4b[1]\n" + ".inst 0x4fa3e3b4 // sdot v20.4s, v29.16b, v3.4b[1]\n" + ".inst 0x4fa4e3b8 // sdot v24.4s, v29.16b, v4.4b[1]\n" + "ldr q29, [x10, #0x60]\n" + ".inst 0x4fa0e389 // sdot v9.4s, v28.16b, v0.4b[1]\n" + ".inst 0x4fa1e38d // sdot v13.4s, v28.16b, v1.4b[1]\n" + ".inst 0x4fa2e391 // sdot v17.4s, v28.16b, v2.4b[1]\n" + ".inst 0x4fa3e395 // sdot v21.4s, v28.16b, v3.4b[1]\n" + ".inst 0x4fa4e399 // sdot v25.4s, v28.16b, v4.4b[1]\n" + "ldr q28, [x10, #0x70]\n" + ".inst 0x4fa0e3aa // sdot v10.4s, v29.16b, v0.4b[1]\n" + ".inst 0x4fa1e3ae // sdot v14.4s, v29.16b, v1.4b[1]\n" + ".inst 0x4fa2e3b2 // sdot v18.4s, v29.16b, v2.4b[1]\n" + ".inst 0x4fa3e3b6 // sdot v22.4s, v29.16b, v3.4b[1]\n" + ".inst 0x4fa4e3ba // sdot v26.4s, v29.16b, v4.4b[1]\n" + "ldr q29, [x10, #0x80]\n" + ".inst 0x4fa0e38b // sdot v11.4s, v28.16b, v0.4b[1]\n" + ".inst 0x4fa1e38f // sdot v15.4s, v28.16b, v1.4b[1]\n" + ".inst 0x4fa2e393 // sdot v19.4s, v28.16b, v2.4b[1]\n" + ".inst 0x4fa3e397 // sdot v23.4s, v28.16b, v3.4b[1]\n" + ".inst 0x4fa4e39b // sdot v27.4s, v28.16b, v4.4b[1]\n" + "ldr q28, [x10, #0x90]\n" + ".inst 0x4f80eba8 // sdot v8.4s, v29.16b, v0.4b[2]\n" + ".inst 0x4f81ebac // sdot v12.4s, v29.16b, v1.4b[2]\n" + ".inst 0x4f82ebb0 // sdot v16.4s, v29.16b, v2.4b[2]\n" + ".inst 0x4f83ebb4 // sdot v20.4s, v29.16b, v3.4b[2]\n" + ".inst 0x4f84ebb8 // sdot v24.4s, v29.16b, v4.4b[2]\n" + "ldr q29, [x10, #0xa0]\n" + ".inst 0x4f80eb89 // sdot v9.4s, v28.16b, v0.4b[2]\n" + ".inst 0x4f81eb8d // sdot v13.4s, v28.16b, v1.4b[2]\n" + ".inst 0x4f82eb91 // sdot v17.4s, v28.16b, v2.4b[2]\n" + ".inst 0x4f83eb95 // sdot v21.4s, v28.16b, v3.4b[2]\n" + ".inst 0x4f84eb99 // sdot v25.4s, v28.16b, v4.4b[2]\n" + "ldr q28, [x10, #0xb0]\n" + ".inst 0x4f80ebaa // sdot v10.4s, v29.16b, v0.4b[2]\n" + ".inst 0x4f81ebae // sdot v14.4s, v29.16b, v1.4b[2]\n" + ".inst 0x4f82ebb2 // sdot v18.4s, v29.16b, v2.4b[2]\n" + ".inst 0x4f83ebb6 // sdot v22.4s, v29.16b, v3.4b[2]\n" + ".inst 0x4f84ebba // sdot v26.4s, v29.16b, v4.4b[2]\n" + "ldr q29, [x10, #0xc0]\n" + ".inst 0x4f80eb8b // sdot v11.4s, v28.16b, v0.4b[2]\n" + ".inst 0x4f81eb8f // sdot v15.4s, v28.16b, v1.4b[2]\n" + ".inst 0x4f82eb93 // sdot v19.4s, v28.16b, v2.4b[2]\n" + ".inst 0x4f83eb97 // sdot v23.4s, v28.16b, v3.4b[2]\n" + ".inst 0x4f84eb9b // sdot v27.4s, v28.16b, v4.4b[2]\n" + "ldr q28, [x10, #0xd0]\n" + ".inst 0x4fa0eba8 // sdot v8.4s, v29.16b, v0.4b[3]\n" + ".inst 0x4fa1ebac // sdot v12.4s, v29.16b, v1.4b[3]\n" + ".inst 0x4fa2ebb0 // sdot v16.4s, v29.16b, v2.4b[3]\n" + ".inst 0x4fa3ebb4 // sdot v20.4s, v29.16b, v3.4b[3]\n" + ".inst 0x4fa4ebb8 // sdot v24.4s, v29.16b, v4.4b[3]\n" + "ldr q29, [x10, #0xe0]\n" + ".inst 0x4fa0eb89 // sdot v9.4s, v28.16b, v0.4b[3]\n" + ".inst 0x4fa1eb8d // sdot v13.4s, v28.16b, v1.4b[3]\n" + ".inst 0x4fa2eb91 // sdot v17.4s, v28.16b, v2.4b[3]\n" + ".inst 0x4fa3eb95 // sdot v21.4s, v28.16b, v3.4b[3]\n" + ".inst 0x4fa4eb99 // sdot v25.4s, v28.16b, v4.4b[3]\n" + "ldr q28, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x4fa0e8ca // sdot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ce // sdot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x4fa2e8d2 // sdot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x4fa3e8d6 // sdot v22.4s, v6.16b, v3.4b[3]\n" - ".inst 0x4fa4e8da // sdot v26.4s, v6.16b, v4.4b[3]\n" - ".inst 0x4fa0e8eb // sdot v11.4s, v7.16b, v0.4b[3]\n" - ".inst 0x4fa1e8ef // sdot v15.4s, v7.16b, v1.4b[3]\n" - ".inst 0x4fa2e8f3 // sdot v19.4s, v7.16b, v2.4b[3]\n" - ".inst 0x4fa3e8f7 // sdot v23.4s, v7.16b, v3.4b[3]\n" - ".inst 0x4fa4e8fb // sdot v27.4s, v7.16b, v4.4b[3]\n" + ".inst 0x4fa0ebaa // sdot v10.4s, v29.16b, v0.4b[3]\n" + ".inst 0x4fa1ebae // sdot v14.4s, v29.16b, v1.4b[3]\n" + ".inst 0x4fa2ebb2 // sdot v18.4s, v29.16b, v2.4b[3]\n" + ".inst 0x4fa3ebb6 // sdot v22.4s, v29.16b, v3.4b[3]\n" + ".inst 0x4fa4ebba // sdot v26.4s, v29.16b, v4.4b[3]\n" + ".inst 0x4fa0eb8b // sdot v11.4s, v28.16b, v0.4b[3]\n" + ".inst 0x4fa1eb8f // sdot v15.4s, v28.16b, v1.4b[3]\n" + ".inst 0x4fa2eb93 // sdot v19.4s, v28.16b, v2.4b[3]\n" + ".inst 0x4fa3eb97 // sdot v23.4s, v28.16b, v3.4b[3]\n" + ".inst 0x4fa4eb9b // sdot v27.4s, v28.16b, v4.4b[3]\n" "155:" // Height 5: Multiply loop: Main loop skip "cbz x27, 160f\n" "cmp x27, #0x4\n" "blt 157f\n" "156:" // Height 5: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" + "ldr s2, [x26], #0x4\n" "ldr s1, [x25], #0x4\n" "sub x27, x27, #0x4\n" "cmp x27, #0x4\n" - "ldr s2, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr s4, [x22], #0x4\n" - "ldr q6, [x10, #0x0]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr s0, [x24], #0x4\n" + "ldr s31, [x23], #0x4\n" + "ldr s30, [x22], #0x4\n" + "ldr q29, [x10, #0x0]\n" + ".inst 0x4f82e3a8 // sdot v8.4s, v29.16b, v2.4b[0]\n" + ".inst 0x4f81e3ac // sdot v12.4s, v29.16b, v1.4b[0]\n" + "ldr q28, [x10, #0x10]\n" + ".inst 0x4f80e3b0 // sdot v16.4s, v29.16b, v0.4b[0]\n" + ".inst 0x4f9fe3b4 // sdot v20.4s, v29.16b, v31.4b[0]\n" + ".inst 0x4f9ee3b8 // sdot v24.4s, v29.16b, v30.4b[0]\n" + "ldr q29, [x10, #0x20]\n" + ".inst 0x4f82e389 // sdot v9.4s, v28.16b, v2.4b[0]\n" + ".inst 0x4f81e38d // sdot v13.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f80e391 // sdot v17.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f9fe395 // sdot v21.4s, v28.16b, v31.4b[0]\n" + ".inst 0x4f9ee399 // sdot v25.4s, v28.16b, v30.4b[0]\n" + "ldr q28, [x10, #0x30]\n" "add x10, x10, #0x40\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" + ".inst 0x4f82e3aa // sdot v10.4s, v29.16b, v2.4b[0]\n" + ".inst 0x4f81e3ae // sdot v14.4s, v29.16b, v1.4b[0]\n" + ".inst 0x4f80e3b2 // sdot v18.4s, v29.16b, v0.4b[0]\n" + ".inst 0x4f9fe3b6 // sdot v22.4s, v29.16b, v31.4b[0]\n" + ".inst 0x4f9ee3ba // sdot v26.4s, v29.16b, v30.4b[0]\n" + ".inst 0x4f82e38b // sdot v11.4s, v28.16b, v2.4b[0]\n" + ".inst 0x4f81e38f // sdot v15.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f80e393 // sdot v19.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f9fe397 // sdot v23.4s, v28.16b, v31.4b[0]\n" + ".inst 0x4f9ee39b // sdot v27.4s, v28.16b, v30.4b[0]\n" "bge 156b\n" "157:" // Height 5: Multiply loop: Skip odd blocks "cbz x27, 160f\n" @@ -2286,31 +2285,31 @@ void a64_hybrid_s8s32_dot_6x16 ( "ldr b3, [x23, #0x0]\n" "ldr b4, [x22, #0x0]\n" "159:" // Height 5: Multiply loop: Ragged operand read: Done - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q29, [x10, #0x0]\n" + "ldr q28, [x10, #0x10]\n" + ".inst 0x4f80e3a8 // sdot v8.4s, v29.16b, v0.4b[0]\n" + ".inst 0x4f81e3ac // sdot v12.4s, v29.16b, v1.4b[0]\n" + ".inst 0x4f82e3b0 // sdot v16.4s, v29.16b, v2.4b[0]\n" + ".inst 0x4f83e3b4 // sdot v20.4s, v29.16b, v3.4b[0]\n" + ".inst 0x4f84e3b8 // sdot v24.4s, v29.16b, v4.4b[0]\n" + "ldr q29, [x10, #0x20]\n" + ".inst 0x4f80e389 // sdot v9.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f81e38d // sdot v13.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f82e391 // sdot v17.4s, v28.16b, v2.4b[0]\n" + ".inst 0x4f83e395 // sdot v21.4s, v28.16b, v3.4b[0]\n" + ".inst 0x4f84e399 // sdot v25.4s, v28.16b, v4.4b[0]\n" + "ldr q28, [x10, #0x30]\n" "add x10, x10, #0x40\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" + ".inst 0x4f80e3aa // sdot v10.4s, v29.16b, v0.4b[0]\n" + ".inst 0x4f81e3ae // sdot v14.4s, v29.16b, v1.4b[0]\n" + ".inst 0x4f82e3b2 // sdot v18.4s, v29.16b, v2.4b[0]\n" + ".inst 0x4f83e3b6 // sdot v22.4s, v29.16b, v3.4b[0]\n" + ".inst 0x4f84e3ba // sdot v26.4s, v29.16b, v4.4b[0]\n" + ".inst 0x4f80e38b // sdot v11.4s, v28.16b, v0.4b[0]\n" + ".inst 0x4f81e38f // sdot v15.4s, v28.16b, v1.4b[0]\n" + ".inst 0x4f82e393 // sdot v19.4s, v28.16b, v2.4b[0]\n" + ".inst 0x4f83e397 // sdot v23.4s, v28.16b, v3.4b[0]\n" + ".inst 0x4f84e39b // sdot v27.4s, v28.16b, v4.4b[0]\n" "160:" // Height 5: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -2672,16 +2671,16 @@ void a64_hybrid_s8s32_dot_6x16 ( "184:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 185f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 186f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -2693,11 +2692,11 @@ void a64_hybrid_s8s32_dot_6x16 ( "b 186f\n" "185:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" + "add x21, x22, x21\n" "186:" // Height 6: input setup done "cmp x27, #0x10\n" "blt 189f\n" @@ -2976,43 +2975,43 @@ void a64_hybrid_s8s32_dot_6x16 ( "cmp x27, #0x4\n" "blt 191f\n" "190:" // Height 6: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s7, [x26], #0x4\n" + "ldr s6, [x25], #0x4\n" "sub x27, x27, #0x4\n" "cmp x27, #0x4\n" - "ldr s2, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr s4, [x22], #0x4\n" - "ldr s5, [x21], #0x4\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - ".inst 0x4f85e0dc // sdot v28.4s, v6.16b, v5.4b[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - ".inst 0x4f85e0fd // sdot v29.4s, v7.16b, v5.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr s5, [x24], #0x4\n" + "ldr s4, [x23], #0x4\n" + "ldr s3, [x22], #0x4\n" + "ldr s2, [x21], #0x4\n" + "ldr q1, [x10, #0x0]\n" + "ldr q0, [x10, #0x10]\n" + ".inst 0x4f87e028 // sdot v8.4s, v1.16b, v7.4b[0]\n" + ".inst 0x4f86e02c // sdot v12.4s, v1.16b, v6.4b[0]\n" + ".inst 0x4f85e030 // sdot v16.4s, v1.16b, v5.4b[0]\n" + ".inst 0x4f84e034 // sdot v20.4s, v1.16b, v4.4b[0]\n" + ".inst 0x4f83e038 // sdot v24.4s, v1.16b, v3.4b[0]\n" + ".inst 0x4f82e03c // sdot v28.4s, v1.16b, v2.4b[0]\n" + "ldr q1, [x10, #0x20]\n" + ".inst 0x4f87e009 // sdot v9.4s, v0.16b, v7.4b[0]\n" + ".inst 0x4f86e00d // sdot v13.4s, v0.16b, v6.4b[0]\n" + ".inst 0x4f85e011 // sdot v17.4s, v0.16b, v5.4b[0]\n" + ".inst 0x4f84e015 // sdot v21.4s, v0.16b, v4.4b[0]\n" + ".inst 0x4f83e019 // sdot v25.4s, v0.16b, v3.4b[0]\n" + ".inst 0x4f82e01d // sdot v29.4s, v0.16b, v2.4b[0]\n" + "ldr q0, [x10, #0x30]\n" "add x10, x10, #0x40\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - ".inst 0x4f85e0de // sdot v30.4s, v6.16b, v5.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" - ".inst 0x4f85e0ff // sdot v31.4s, v7.16b, v5.4b[0]\n" + ".inst 0x4f87e02a // sdot v10.4s, v1.16b, v7.4b[0]\n" + ".inst 0x4f86e02e // sdot v14.4s, v1.16b, v6.4b[0]\n" + ".inst 0x4f85e032 // sdot v18.4s, v1.16b, v5.4b[0]\n" + ".inst 0x4f84e036 // sdot v22.4s, v1.16b, v4.4b[0]\n" + ".inst 0x4f83e03a // sdot v26.4s, v1.16b, v3.4b[0]\n" + ".inst 0x4f82e03e // sdot v30.4s, v1.16b, v2.4b[0]\n" + ".inst 0x4f87e00b // sdot v11.4s, v0.16b, v7.4b[0]\n" + ".inst 0x4f86e00f // sdot v15.4s, v0.16b, v6.4b[0]\n" + ".inst 0x4f85e013 // sdot v19.4s, v0.16b, v5.4b[0]\n" + ".inst 0x4f84e017 // sdot v23.4s, v0.16b, v4.4b[0]\n" + ".inst 0x4f83e01b // sdot v27.4s, v0.16b, v3.4b[0]\n" + ".inst 0x4f82e01f // sdot v31.4s, v0.16b, v2.4b[0]\n" "bge 190b\n" "191:" // Height 6: Multiply loop: Skip odd blocks "cbz x27, 194f\n" @@ -3039,35 +3038,35 @@ void a64_hybrid_s8s32_dot_6x16 ( "ldr b4, [x22, #0x0]\n" "ldr b5, [x21, #0x0]\n" "193:" // Height 6: Multiply loop: Ragged operand read: Done - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4f80e0c8 // sdot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0cc // sdot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d0 // sdot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d4 // sdot v20.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0d8 // sdot v24.4s, v6.16b, v4.4b[0]\n" - ".inst 0x4f85e0dc // sdot v28.4s, v6.16b, v5.4b[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4f80e0e9 // sdot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ed // sdot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f1 // sdot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f5 // sdot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0f9 // sdot v25.4s, v7.16b, v4.4b[0]\n" - ".inst 0x4f85e0fd // sdot v29.4s, v7.16b, v5.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q7, [x10, #0x0]\n" + "ldr q6, [x10, #0x10]\n" + ".inst 0x4f80e0e8 // sdot v8.4s, v7.16b, v0.4b[0]\n" + ".inst 0x4f81e0ec // sdot v12.4s, v7.16b, v1.4b[0]\n" + ".inst 0x4f82e0f0 // sdot v16.4s, v7.16b, v2.4b[0]\n" + ".inst 0x4f83e0f4 // sdot v20.4s, v7.16b, v3.4b[0]\n" + ".inst 0x4f84e0f8 // sdot v24.4s, v7.16b, v4.4b[0]\n" + ".inst 0x4f85e0fc // sdot v28.4s, v7.16b, v5.4b[0]\n" + "ldr q7, [x10, #0x20]\n" + ".inst 0x4f80e0c9 // sdot v9.4s, v6.16b, v0.4b[0]\n" + ".inst 0x4f81e0cd // sdot v13.4s, v6.16b, v1.4b[0]\n" + ".inst 0x4f82e0d1 // sdot v17.4s, v6.16b, v2.4b[0]\n" + ".inst 0x4f83e0d5 // sdot v21.4s, v6.16b, v3.4b[0]\n" + ".inst 0x4f84e0d9 // sdot v25.4s, v6.16b, v4.4b[0]\n" + ".inst 0x4f85e0dd // sdot v29.4s, v6.16b, v5.4b[0]\n" + "ldr q6, [x10, #0x30]\n" "add x10, x10, #0x40\n" - ".inst 0x4f80e0ca // sdot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x4f81e0ce // sdot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x4f82e0d2 // sdot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x4f83e0d6 // sdot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x4f84e0da // sdot v26.4s, v6.16b, v4.4b[0]\n" - ".inst 0x4f85e0de // sdot v30.4s, v6.16b, v5.4b[0]\n" - ".inst 0x4f80e0eb // sdot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x4f81e0ef // sdot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x4f82e0f3 // sdot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x4f83e0f7 // sdot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x4f84e0fb // sdot v27.4s, v7.16b, v4.4b[0]\n" - ".inst 0x4f85e0ff // sdot v31.4s, v7.16b, v5.4b[0]\n" + ".inst 0x4f80e0ea // sdot v10.4s, v7.16b, v0.4b[0]\n" + ".inst 0x4f81e0ee // sdot v14.4s, v7.16b, v1.4b[0]\n" + ".inst 0x4f82e0f2 // sdot v18.4s, v7.16b, v2.4b[0]\n" + ".inst 0x4f83e0f6 // sdot v22.4s, v7.16b, v3.4b[0]\n" + ".inst 0x4f84e0fa // sdot v26.4s, v7.16b, v4.4b[0]\n" + ".inst 0x4f85e0fe // sdot v30.4s, v7.16b, v5.4b[0]\n" + ".inst 0x4f80e0cb // sdot v11.4s, v6.16b, v0.4b[0]\n" + ".inst 0x4f81e0cf // sdot v15.4s, v6.16b, v1.4b[0]\n" + ".inst 0x4f82e0d3 // sdot v19.4s, v6.16b, v2.4b[0]\n" + ".inst 0x4f83e0d7 // sdot v23.4s, v6.16b, v3.4b[0]\n" + ".inst 0x4f84e0db // sdot v27.4s, v6.16b, v4.4b[0]\n" + ".inst 0x4f85e0df // sdot v31.4s, v6.16b, v5.4b[0]\n" "194:" // Height 6: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -3254,7 +3253,6 @@ void a64_hybrid_s8s32_dot_6x16 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "206:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_mmla_6x16.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_mmla_6x16.hpp index 50ccb6fa3df731ab058e5934ff13d511532c4c62..4905ba56569d1f4122734e2243a1803421d22a30 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_mmla_6x16.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_mmla_6x16.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef __aarch64__ + #include "../std_transforms_fixed.hpp" #include "../performance_parameters.hpp" @@ -92,7 +92,7 @@ public: case CPUModel::A510: return { 33.62, 3.92, 0.48 }; case CPUModel::V1: - return { 86.36, 19.25, 0.92 }; + return { 63.94, 16.18, 0.83 }; } } @@ -109,5 +109,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_mmla_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_mmla_6x16/generic.cpp index f48623e12908f60e21972247b0bed1af30dd36e7..f8a76b524433a447bb1527d48fde062b34d5ef7c 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_mmla_6x16/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_mmla_6x16/generic.cpp @@ -77,7 +77,6 @@ void a64_hybrid_s8s32_mmla_6x16 ( ka.N = N; ka.B_ptr = B_ptr; __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x6\n" "bge 186f\n" @@ -178,11 +177,11 @@ void a64_hybrid_s8s32_mmla_6x16 ( "15:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 16f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 17f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -198,41 +197,41 @@ void a64_hybrid_s8s32_mmla_6x16 ( "ldr q6, [x10, #0x10]\n" "blt 19f\n" "18:" // Height 1: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" - "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x80]\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x90]\n" - ".inst 0x4e87a428 // smmla v8.4s, v1.16b, v7.16b\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x4e86a42c // smmla v12.4s, v1.16b, v6.16b\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x4e87a429 // smmla v9.4s, v1.16b, v7.16b\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x4e86a42d // smmla v13.4s, v1.16b, v6.16b\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x4e87a42a // smmla v10.4s, v1.16b, v7.16b\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - "ldr q6, [x10, #0xf0]\n" + "trn1 v19.2d, v1.2d, v20.2d\n" + ".inst 0x4e87a668 // smmla v8.4s, v19.16b, v7.16b\n" + "ldr q18, [x10, #0x20]\n" + ".inst 0x4e86a66c // smmla v12.4s, v19.16b, v6.16b\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x4e92a669 // smmla v9.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x40]\n" + ".inst 0x4e91a66d // smmla v13.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x4e92a66a // smmla v10.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x60]\n" + ".inst 0x4e91a66e // smmla v14.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x70]\n" + "trn2 v1.2d, v1.2d, v20.2d\n" + ".inst 0x4e92a66b // smmla v11.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x80]\n" + ".inst 0x4e91a66f // smmla v15.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x90]\n" + ".inst 0x4e92a428 // smmla v8.4s, v1.16b, v18.16b\n" + "ldr q18, [x10, #0xa0]\n" + ".inst 0x4e91a42c // smmla v12.4s, v1.16b, v17.16b\n" + "ldr q17, [x10, #0xb0]\n" + ".inst 0x4e92a429 // smmla v9.4s, v1.16b, v18.16b\n" + "ldr q18, [x10, #0xc0]\n" + ".inst 0x4e91a42d // smmla v13.4s, v1.16b, v17.16b\n" + "ldr q17, [x10, #0xd0]\n" + ".inst 0x4e92a42a // smmla v10.4s, v1.16b, v18.16b\n" + "ldr q18, [x10, #0xe0]\n" + ".inst 0x4e91a42e // smmla v14.4s, v1.16b, v17.16b\n" + "ldr q17, [x10, #0xf0]\n" "sub x27, x27, #0x10\n" "add x26, x26, #0x10\n" "cmp x27, #0x20\n" - ".inst 0x4e87a42b // smmla v11.4s, v1.16b, v7.16b\n" - ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" + ".inst 0x4e92a42b // smmla v11.4s, v1.16b, v18.16b\n" + ".inst 0x4e91a42f // smmla v15.4s, v1.16b, v17.16b\n" "ldr q1, [x26, #0x0]\n" "add x10, x10, #0x100\n" "ldr q7, [x10, #0x0]\n" @@ -240,40 +239,40 @@ void a64_hybrid_s8s32_mmla_6x16 ( "prfm pldl1keep, [x26, #0x80]\n" "bge 18b\n" "19:" // Height 1: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" - "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x80]\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x90]\n" - ".inst 0x4e87a428 // smmla v8.4s, v1.16b, v7.16b\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x4e86a42c // smmla v12.4s, v1.16b, v6.16b\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x4e87a429 // smmla v9.4s, v1.16b, v7.16b\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x4e86a42d // smmla v13.4s, v1.16b, v6.16b\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x4e87a42a // smmla v10.4s, v1.16b, v7.16b\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - "ldr q6, [x10, #0xf0]\n" + "trn1 v20.2d, v1.2d, v21.2d\n" + ".inst 0x4e87a688 // smmla v8.4s, v20.16b, v7.16b\n" + "ldr q18, [x10, #0x20]\n" + ".inst 0x4e86a68c // smmla v12.4s, v20.16b, v6.16b\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x4e92a689 // smmla v9.4s, v20.16b, v18.16b\n" + "ldr q18, [x10, #0x40]\n" + ".inst 0x4e91a68d // smmla v13.4s, v20.16b, v17.16b\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x4e92a68a // smmla v10.4s, v20.16b, v18.16b\n" + "ldr q19, [x10, #0x60]\n" + ".inst 0x4e91a68e // smmla v14.4s, v20.16b, v17.16b\n" + "ldr q18, [x10, #0x70]\n" + "trn2 v1.2d, v1.2d, v21.2d\n" + ".inst 0x4e93a68b // smmla v11.4s, v20.16b, v19.16b\n" + "ldr q17, [x10, #0x80]\n" + ".inst 0x4e92a68f // smmla v15.4s, v20.16b, v18.16b\n" + "ldr q19, [x10, #0x90]\n" + ".inst 0x4e91a428 // smmla v8.4s, v1.16b, v17.16b\n" + "ldr q18, [x10, #0xa0]\n" + ".inst 0x4e93a42c // smmla v12.4s, v1.16b, v19.16b\n" + "ldr q17, [x10, #0xb0]\n" + ".inst 0x4e92a429 // smmla v9.4s, v1.16b, v18.16b\n" + "ldr q18, [x10, #0xc0]\n" + ".inst 0x4e91a42d // smmla v13.4s, v1.16b, v17.16b\n" + "ldr q17, [x10, #0xd0]\n" + ".inst 0x4e92a42a // smmla v10.4s, v1.16b, v18.16b\n" + "ldr q18, [x10, #0xe0]\n" + ".inst 0x4e91a42e // smmla v14.4s, v1.16b, v17.16b\n" + "ldr q17, [x10, #0xf0]\n" "add x26, x26, #0x10\n" "sub x27, x27, #0x10\n" - ".inst 0x4e87a42b // smmla v11.4s, v1.16b, v7.16b\n" - ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" + ".inst 0x4e92a42b // smmla v11.4s, v1.16b, v18.16b\n" + ".inst 0x4e91a42f // smmla v15.4s, v1.16b, v17.16b\n" "prfm pldl1keep, [x26, #0x80]\n" "add x10, x10, #0x100\n" "20:" // Height 1: Multiply loop: Main loop skip @@ -281,26 +280,26 @@ void a64_hybrid_s8s32_mmla_6x16 ( "cmp x27, #0x8\n" "blt 22f\n" "21:" // Height 1: Multiply loop: Odd block loop - "ldr d1, [x26], #0x8\n" - "ldr q6, [x10, #0x0]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4e86a408 // smmla v8.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4e87a40c // smmla v12.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x4e86a409 // smmla v9.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x4e87a40d // smmla v13.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x4e86a40a // smmla v10.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x4e87a40e // smmla v14.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x70]\n" + "ldr d19, [x26], #0x8\n" + "ldr q18, [x10, #0x0]\n" + "trn1 v19.2d, v19.2d, v17.2d\n" + "ldr q17, [x10, #0x10]\n" + ".inst 0x4e92a668 // smmla v8.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x20]\n" + ".inst 0x4e91a66c // smmla v12.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x4e92a669 // smmla v9.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x40]\n" + ".inst 0x4e91a66d // smmla v13.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x4e92a66a // smmla v10.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x60]\n" + ".inst 0x4e91a66e // smmla v14.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x70]\n" "sub x27, x27, #0x8\n" "cmp x27, #0x8\n" - ".inst 0x4e86a40b // smmla v11.4s, v0.16b, v6.16b\n" - ".inst 0x4e87a40f // smmla v15.4s, v0.16b, v7.16b\n" + ".inst 0x4e92a66b // smmla v11.4s, v19.16b, v18.16b\n" + ".inst 0x4e91a66f // smmla v15.4s, v19.16b, v17.16b\n" "add x10, x10, #0x80\n" "bge 21b\n" "22:" // Height 1: Multiply loop: Skip odd blocks @@ -325,23 +324,23 @@ void a64_hybrid_s8s32_mmla_6x16 ( "25:" // Height 1: Multiply loop: Ragged operand read: partial_1_0 "ldr b1, [x26, #0x0]\n" "26:" // Height 1: Multiply loop: Ragged operand read: Done - "ldr q7, [x10, #0x0]\n" - "ldr q6, [x10, #0x10]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" + "ldr q23, [x10, #0x0]\n" + "ldr q18, [x10, #0x10]\n" + "trn1 v19.2d, v1.2d, v17.2d\n" + ".inst 0x4e97a668 // smmla v8.4s, v19.16b, v23.16b\n" + "ldr q17, [x10, #0x20]\n" + ".inst 0x4e92a66c // smmla v12.4s, v19.16b, v18.16b\n" + "ldr q31, [x10, #0x30]\n" + ".inst 0x4e91a669 // smmla v9.4s, v19.16b, v17.16b\n" + "ldr q20, [x10, #0x40]\n" + ".inst 0x4e9fa66d // smmla v13.4s, v19.16b, v31.16b\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x4e94a66a // smmla v10.4s, v19.16b, v20.16b\n" + "ldr q18, [x10, #0x60]\n" + ".inst 0x4e91a66e // smmla v14.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x70]\n" + ".inst 0x4e92a66b // smmla v11.4s, v19.16b, v18.16b\n" + ".inst 0x4e91a66f // smmla v15.4s, v19.16b, v17.16b\n" "add x10, x10, #0x80\n" "27:" // Height 1: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -525,12 +524,12 @@ void a64_hybrid_s8s32_mmla_6x16 ( "52:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 53f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 54f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -538,7 +537,7 @@ void a64_hybrid_s8s32_mmla_6x16 ( "b 54f\n" "53:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" + "add x25, x26, x21\n" "54:" // Height 2: input setup done "cmp x27, #0x10\n" "blt 57f\n" @@ -549,85 +548,85 @@ void a64_hybrid_s8s32_mmla_6x16 ( "ldr q6, [x10, #0x10]\n" "blt 56f\n" "55:" // Height 2: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" + "trn1 v19.2d, v1.2d, v2.2d\n" + ".inst 0x4e87a668 // smmla v8.4s, v19.16b, v7.16b\n" + "ldr q18, [x10, #0x20]\n" + ".inst 0x4e86a66c // smmla v12.4s, v19.16b, v6.16b\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x4e92a669 // smmla v9.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x40]\n" + ".inst 0x4e91a66d // smmla v13.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x4e92a66a // smmla v10.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x60]\n" + ".inst 0x4e91a66e // smmla v14.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x70]\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x80]\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x90]\n" - ".inst 0x4e87a428 // smmla v8.4s, v1.16b, v7.16b\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x4e86a42c // smmla v12.4s, v1.16b, v6.16b\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x4e87a429 // smmla v9.4s, v1.16b, v7.16b\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x4e86a42d // smmla v13.4s, v1.16b, v6.16b\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x4e87a42a // smmla v10.4s, v1.16b, v7.16b\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - "ldr q6, [x10, #0xf0]\n" + ".inst 0x4e92a66b // smmla v11.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x80]\n" + ".inst 0x4e91a66f // smmla v15.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x90]\n" + ".inst 0x4e92a428 // smmla v8.4s, v1.16b, v18.16b\n" + "ldr q18, [x10, #0xa0]\n" + ".inst 0x4e91a42c // smmla v12.4s, v1.16b, v17.16b\n" + "ldr q17, [x10, #0xb0]\n" + ".inst 0x4e92a429 // smmla v9.4s, v1.16b, v18.16b\n" + "ldr q18, [x10, #0xc0]\n" + ".inst 0x4e91a42d // smmla v13.4s, v1.16b, v17.16b\n" + "ldr q17, [x10, #0xd0]\n" + ".inst 0x4e92a42a // smmla v10.4s, v1.16b, v18.16b\n" + "ldr q18, [x10, #0xe0]\n" + ".inst 0x4e91a42e // smmla v14.4s, v1.16b, v17.16b\n" + "ldr q17, [x10, #0xf0]\n" "sub x27, x27, #0x10\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" "ldr q2, [x25, #0x0]\n" "cmp x27, #0x20\n" - ".inst 0x4e87a42b // smmla v11.4s, v1.16b, v7.16b\n" + ".inst 0x4e92a42b // smmla v11.4s, v1.16b, v18.16b\n" "add x10, x10, #0x100\n" "ldr q7, [x10, #0x0]\n" - ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" + ".inst 0x4e91a42f // smmla v15.4s, v1.16b, v17.16b\n" "ldr q1, [x26, #0x0]\n" "ldr q6, [x10, #0x10]\n" "prfm pldl1keep, [x26, #0x80]\n" "prfm pldl1keep, [x25, #0x80]\n" "bge 55b\n" "56:" // Height 2: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" + "trn1 v19.2d, v1.2d, v2.2d\n" + ".inst 0x4e87a668 // smmla v8.4s, v19.16b, v7.16b\n" + "ldr q18, [x10, #0x20]\n" + ".inst 0x4e86a66c // smmla v12.4s, v19.16b, v6.16b\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x4e92a669 // smmla v9.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x40]\n" + ".inst 0x4e91a66d // smmla v13.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x4e92a66a // smmla v10.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x60]\n" + ".inst 0x4e91a66e // smmla v14.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x70]\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x80]\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x90]\n" - ".inst 0x4e87a428 // smmla v8.4s, v1.16b, v7.16b\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x4e86a42c // smmla v12.4s, v1.16b, v6.16b\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x4e87a429 // smmla v9.4s, v1.16b, v7.16b\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x4e86a42d // smmla v13.4s, v1.16b, v6.16b\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x4e87a42a // smmla v10.4s, v1.16b, v7.16b\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - "ldr q6, [x10, #0xf0]\n" + ".inst 0x4e92a66b // smmla v11.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x80]\n" + ".inst 0x4e91a66f // smmla v15.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x90]\n" + ".inst 0x4e92a428 // smmla v8.4s, v1.16b, v18.16b\n" + "ldr q18, [x10, #0xa0]\n" + ".inst 0x4e91a42c // smmla v12.4s, v1.16b, v17.16b\n" + "ldr q17, [x10, #0xb0]\n" + ".inst 0x4e92a429 // smmla v9.4s, v1.16b, v18.16b\n" + "ldr q18, [x10, #0xc0]\n" + ".inst 0x4e91a42d // smmla v13.4s, v1.16b, v17.16b\n" + "ldr q17, [x10, #0xd0]\n" + ".inst 0x4e92a42a // smmla v10.4s, v1.16b, v18.16b\n" + "ldr q18, [x10, #0xe0]\n" + ".inst 0x4e91a42e // smmla v14.4s, v1.16b, v17.16b\n" + "ldr q17, [x10, #0xf0]\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" - ".inst 0x4e87a42b // smmla v11.4s, v1.16b, v7.16b\n" - ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" + ".inst 0x4e92a42b // smmla v11.4s, v1.16b, v18.16b\n" + ".inst 0x4e91a42f // smmla v15.4s, v1.16b, v17.16b\n" "sub x27, x27, #0x10\n" "prfm pldl1keep, [x26, #0x80]\n" "prfm pldl1keep, [x25, #0x80]\n" @@ -637,27 +636,27 @@ void a64_hybrid_s8s32_mmla_6x16 ( "cmp x27, #0x8\n" "blt 59f\n" "58:" // Height 2: Multiply loop: Odd block loop - "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" + "ldr d18, [x26], #0x8\n" + "ldr d17, [x25], #0x8\n" + "trn1 v19.2d, v18.2d, v17.2d\n" "sub x27, x27, #0x8\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4e86a408 // smmla v8.4s, v0.16b, v6.16b\n" - ".inst 0x4e87a40c // smmla v12.4s, v0.16b, v7.16b\n" - "ldr q6, [x10, #0x20]\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x4e86a409 // smmla v9.4s, v0.16b, v6.16b\n" - ".inst 0x4e87a40d // smmla v13.4s, v0.16b, v7.16b\n" - "ldr q6, [x10, #0x40]\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x4e86a40a // smmla v10.4s, v0.16b, v6.16b\n" - ".inst 0x4e87a40e // smmla v14.4s, v0.16b, v7.16b\n" - "ldr q6, [x10, #0x60]\n" - "ldr q7, [x10, #0x70]\n" + "ldr q17, [x10, #0x0]\n" + "ldr q22, [x10, #0x10]\n" + ".inst 0x4e91a668 // smmla v8.4s, v19.16b, v17.16b\n" + ".inst 0x4e96a66c // smmla v12.4s, v19.16b, v22.16b\n" + "ldr q1, [x10, #0x20]\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x4e81a669 // smmla v9.4s, v19.16b, v1.16b\n" + ".inst 0x4e91a66d // smmla v13.4s, v19.16b, v17.16b\n" + "ldr q18, [x10, #0x40]\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x4e92a66a // smmla v10.4s, v19.16b, v18.16b\n" + ".inst 0x4e91a66e // smmla v14.4s, v19.16b, v17.16b\n" + "ldr q18, [x10, #0x60]\n" + "ldr q17, [x10, #0x70]\n" "cmp x27, #0x8\n" - ".inst 0x4e86a40b // smmla v11.4s, v0.16b, v6.16b\n" - ".inst 0x4e87a40f // smmla v15.4s, v0.16b, v7.16b\n" + ".inst 0x4e92a66b // smmla v11.4s, v19.16b, v18.16b\n" + ".inst 0x4e91a66f // smmla v15.4s, v19.16b, v17.16b\n" "add x10, x10, #0x80\n" "bge 58b\n" "59:" // Height 2: Multiply loop: Skip odd blocks @@ -689,23 +688,23 @@ void a64_hybrid_s8s32_mmla_6x16 ( "ldr b1, [x26, #0x0]\n" "ldr b2, [x25, #0x0]\n" "63:" // Height 2: Multiply loop: Ragged operand read: Done - "ldr q7, [x10, #0x0]\n" - "ldr q6, [x10, #0x10]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" + "ldr q18, [x10, #0x0]\n" + "ldr q17, [x10, #0x10]\n" + "trn1 v19.2d, v1.2d, v2.2d\n" + ".inst 0x4e92a668 // smmla v8.4s, v19.16b, v18.16b\n" + "ldr q5, [x10, #0x20]\n" + ".inst 0x4e91a66c // smmla v12.4s, v19.16b, v17.16b\n" + "ldr q21, [x10, #0x30]\n" + ".inst 0x4e85a669 // smmla v9.4s, v19.16b, v5.16b\n" + "ldr q18, [x10, #0x40]\n" + ".inst 0x4e95a66d // smmla v13.4s, v19.16b, v21.16b\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x4e92a66a // smmla v10.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x60]\n" + ".inst 0x4e91a66e // smmla v14.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x70]\n" + ".inst 0x4e92a66b // smmla v11.4s, v19.16b, v18.16b\n" + ".inst 0x4e91a66f // smmla v15.4s, v19.16b, v17.16b\n" "add x10, x10, #0x80\n" "64:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -953,13 +952,13 @@ void a64_hybrid_s8s32_mmla_6x16 ( "89:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 90f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 91f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -968,8 +967,8 @@ void a64_hybrid_s8s32_mmla_6x16 ( "b 91f\n" "90:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" "91:" // Height 3: input setup done "cmp x27, #0x10\n" "blt 94f\n" @@ -981,167 +980,167 @@ void a64_hybrid_s8s32_mmla_6x16 ( "ldr q6, [x10, #0x10]\n" "blt 93f\n" "92:" // Height 3: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v28.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x4e87a450 // smmla v16.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" - "trn2 v3.2d, v3.2d, v4.2d\n" - ".inst 0x4e87a451 // smmla v17.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" + ".inst 0x4e87a788 // smmla v8.4s, v28.16b, v7.16b\n" + "trn1 v27.2d, v3.2d, v29.2d\n" + ".inst 0x4e87a770 // smmla v16.4s, v27.16b, v7.16b\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x4e86a78c // smmla v12.4s, v28.16b, v6.16b\n" + ".inst 0x4e86a774 // smmla v20.4s, v27.16b, v6.16b\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x4e9aa789 // smmla v9.4s, v28.16b, v26.16b\n" + "trn2 v3.2d, v3.2d, v29.2d\n" + ".inst 0x4e9aa771 // smmla v17.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x4e99a78d // smmla v13.4s, v28.16b, v25.16b\n" "sub x27, x27, #0x10\n" - ".inst 0x4e86a455 // smmla v21.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" + ".inst 0x4e99a775 // smmla v21.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x4e9aa78a // smmla v10.4s, v28.16b, v26.16b\n" "add x26, x26, #0x10\n" - ".inst 0x4e87a452 // smmla v18.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" + ".inst 0x4e9aa772 // smmla v18.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x4e99a78e // smmla v14.4s, v28.16b, v25.16b\n" "add x25, x25, #0x10\n" - ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" + ".inst 0x4e99a776 // smmla v22.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x70]\n" + ".inst 0x4e9aa78b // smmla v11.4s, v28.16b, v26.16b\n" "add x24, x24, #0x10\n" - ".inst 0x4e87a453 // smmla v19.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x80]\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" + ".inst 0x4e9aa773 // smmla v19.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x80]\n" + ".inst 0x4e99a78f // smmla v15.4s, v28.16b, v25.16b\n" "cmp x27, #0x20\n" - ".inst 0x4e86a457 // smmla v23.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x90]\n" + ".inst 0x4e99a777 // smmla v23.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x90]\n" "ldr q2, [x25, #0x0]\n" - ".inst 0x4e87a428 // smmla v8.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a470 // smmla v16.4s, v3.16b, v7.16b\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x4e86a42c // smmla v12.4s, v1.16b, v6.16b\n" + ".inst 0x4e9aa428 // smmla v8.4s, v1.16b, v26.16b\n" + ".inst 0x4e9aa470 // smmla v16.4s, v3.16b, v26.16b\n" + "ldr q26, [x10, #0xa0]\n" + ".inst 0x4e99a42c // smmla v12.4s, v1.16b, v25.16b\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x4e86a474 // smmla v20.4s, v3.16b, v6.16b\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x4e87a429 // smmla v9.4s, v1.16b, v7.16b\n" + ".inst 0x4e99a474 // smmla v20.4s, v3.16b, v25.16b\n" + "ldr q25, [x10, #0xb0]\n" + ".inst 0x4e9aa429 // smmla v9.4s, v1.16b, v26.16b\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4e87a471 // smmla v17.4s, v3.16b, v7.16b\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x4e86a42d // smmla v13.4s, v1.16b, v6.16b\n" + ".inst 0x4e9aa471 // smmla v17.4s, v3.16b, v26.16b\n" + "ldr q26, [x10, #0xc0]\n" + ".inst 0x4e99a42d // smmla v13.4s, v1.16b, v25.16b\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x4e86a475 // smmla v21.4s, v3.16b, v6.16b\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x4e87a42a // smmla v10.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a472 // smmla v18.4s, v3.16b, v7.16b\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a476 // smmla v22.4s, v3.16b, v6.16b\n" - "ldr q6, [x10, #0xf0]\n" + ".inst 0x4e99a475 // smmla v21.4s, v3.16b, v25.16b\n" + "ldr q25, [x10, #0xd0]\n" + ".inst 0x4e9aa42a // smmla v10.4s, v1.16b, v26.16b\n" + ".inst 0x4e9aa472 // smmla v18.4s, v3.16b, v26.16b\n" + "ldr q26, [x10, #0xe0]\n" + ".inst 0x4e99a42e // smmla v14.4s, v1.16b, v25.16b\n" + ".inst 0x4e99a476 // smmla v22.4s, v3.16b, v25.16b\n" + "ldr q25, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x4e87a42b // smmla v11.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a473 // smmla v19.4s, v3.16b, v7.16b\n" + ".inst 0x4e9aa42b // smmla v11.4s, v1.16b, v26.16b\n" + ".inst 0x4e9aa473 // smmla v19.4s, v3.16b, v26.16b\n" "ldr q7, [x10, #0x0]\n" - ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" + ".inst 0x4e99a42f // smmla v15.4s, v1.16b, v25.16b\n" "ldr q1, [x26, #0x0]\n" - ".inst 0x4e86a477 // smmla v23.4s, v3.16b, v6.16b\n" + ".inst 0x4e99a477 // smmla v23.4s, v3.16b, v25.16b\n" "ldr q3, [x24, #0x0]\n" "ldr q6, [x10, #0x10]\n" "bge 92b\n" "93:" // Height 3: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v28.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x4e87a450 // smmla v16.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" - "trn2 v3.2d, v3.2d, v4.2d\n" - ".inst 0x4e87a451 // smmla v17.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" + ".inst 0x4e87a788 // smmla v8.4s, v28.16b, v7.16b\n" + "trn1 v27.2d, v3.2d, v29.2d\n" + ".inst 0x4e87a770 // smmla v16.4s, v27.16b, v7.16b\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x4e86a78c // smmla v12.4s, v28.16b, v6.16b\n" + ".inst 0x4e86a774 // smmla v20.4s, v27.16b, v6.16b\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x4e9aa789 // smmla v9.4s, v28.16b, v26.16b\n" + "trn2 v3.2d, v3.2d, v29.2d\n" + ".inst 0x4e9aa771 // smmla v17.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x4e99a78d // smmla v13.4s, v28.16b, v25.16b\n" "add x26, x26, #0x10\n" - ".inst 0x4e86a455 // smmla v21.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" + ".inst 0x4e99a775 // smmla v21.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x4e9aa78a // smmla v10.4s, v28.16b, v26.16b\n" "add x25, x25, #0x10\n" - ".inst 0x4e87a452 // smmla v18.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" + ".inst 0x4e9aa772 // smmla v18.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x4e99a78e // smmla v14.4s, v28.16b, v25.16b\n" "add x24, x24, #0x10\n" - ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" + ".inst 0x4e99a776 // smmla v22.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x70]\n" + ".inst 0x4e9aa78b // smmla v11.4s, v28.16b, v26.16b\n" "sub x27, x27, #0x10\n" - ".inst 0x4e87a453 // smmla v19.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x80]\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" + ".inst 0x4e9aa773 // smmla v19.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x80]\n" + ".inst 0x4e99a78f // smmla v15.4s, v28.16b, v25.16b\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x4e86a457 // smmla v23.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x90]\n" - ".inst 0x4e87a428 // smmla v8.4s, v1.16b, v7.16b\n" + ".inst 0x4e99a777 // smmla v23.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x90]\n" + ".inst 0x4e9aa428 // smmla v8.4s, v1.16b, v26.16b\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4e87a470 // smmla v16.4s, v3.16b, v7.16b\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x4e86a42c // smmla v12.4s, v1.16b, v6.16b\n" + ".inst 0x4e9aa470 // smmla v16.4s, v3.16b, v26.16b\n" + "ldr q26, [x10, #0xa0]\n" + ".inst 0x4e99a42c // smmla v12.4s, v1.16b, v25.16b\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x4e86a474 // smmla v20.4s, v3.16b, v6.16b\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x4e87a429 // smmla v9.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a471 // smmla v17.4s, v3.16b, v7.16b\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x4e86a42d // smmla v13.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a475 // smmla v21.4s, v3.16b, v6.16b\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x4e87a42a // smmla v10.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a472 // smmla v18.4s, v3.16b, v7.16b\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a476 // smmla v22.4s, v3.16b, v6.16b\n" - "ldr q6, [x10, #0xf0]\n" + ".inst 0x4e99a474 // smmla v20.4s, v3.16b, v25.16b\n" + "ldr q25, [x10, #0xb0]\n" + ".inst 0x4e9aa429 // smmla v9.4s, v1.16b, v26.16b\n" + ".inst 0x4e9aa471 // smmla v17.4s, v3.16b, v26.16b\n" + "ldr q26, [x10, #0xc0]\n" + ".inst 0x4e99a42d // smmla v13.4s, v1.16b, v25.16b\n" + ".inst 0x4e99a475 // smmla v21.4s, v3.16b, v25.16b\n" + "ldr q25, [x10, #0xd0]\n" + ".inst 0x4e9aa42a // smmla v10.4s, v1.16b, v26.16b\n" + ".inst 0x4e9aa472 // smmla v18.4s, v3.16b, v26.16b\n" + "ldr q26, [x10, #0xe0]\n" + ".inst 0x4e99a42e // smmla v14.4s, v1.16b, v25.16b\n" + ".inst 0x4e99a476 // smmla v22.4s, v3.16b, v25.16b\n" + "ldr q25, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x4e87a42b // smmla v11.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a473 // smmla v19.4s, v3.16b, v7.16b\n" - ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a477 // smmla v23.4s, v3.16b, v6.16b\n" + ".inst 0x4e9aa42b // smmla v11.4s, v1.16b, v26.16b\n" + ".inst 0x4e9aa473 // smmla v19.4s, v3.16b, v26.16b\n" + ".inst 0x4e99a42f // smmla v15.4s, v1.16b, v25.16b\n" + ".inst 0x4e99a477 // smmla v23.4s, v3.16b, v25.16b\n" "94:" // Height 3: Multiply loop: Main loop skip "cbz x27, 101f\n" "cmp x27, #0x8\n" "blt 96f\n" "95:" // Height 3: Multiply loop: Odd block loop - "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "ldr d3, [x24], #0x8\n" - "ldr q6, [x10, #0x0]\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x4e86a408 // smmla v8.4s, v0.16b, v6.16b\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4e86a450 // smmla v16.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4e87a40c // smmla v12.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a454 // smmla v20.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x4e86a409 // smmla v9.4s, v0.16b, v6.16b\n" + "ldr d26, [x26], #0x8\n" + "ldr d25, [x25], #0x8\n" + "trn1 v28.2d, v26.2d, v25.2d\n" + "ldr d25, [x24], #0x8\n" + "ldr q26, [x10, #0x0]\n" + "trn1 v27.2d, v25.2d, v27.2d\n" + ".inst 0x4e9aa788 // smmla v8.4s, v28.16b, v26.16b\n" + "ldr q25, [x10, #0x10]\n" + ".inst 0x4e9aa770 // smmla v16.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x4e99a78c // smmla v12.4s, v28.16b, v25.16b\n" + ".inst 0x4e99a774 // smmla v20.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x4e9aa789 // smmla v9.4s, v28.16b, v26.16b\n" "sub x27, x27, #0x8\n" - ".inst 0x4e86a451 // smmla v17.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x4e87a40d // smmla v13.4s, v0.16b, v7.16b\n" + ".inst 0x4e9aa771 // smmla v17.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x4e99a78d // smmla v13.4s, v28.16b, v25.16b\n" "cmp x27, #0x8\n" - ".inst 0x4e87a455 // smmla v21.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x4e86a40a // smmla v10.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a452 // smmla v18.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x4e87a40e // smmla v14.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a456 // smmla v22.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x4e86a40b // smmla v11.4s, v0.16b, v6.16b\n" + ".inst 0x4e99a775 // smmla v21.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x4e9aa78a // smmla v10.4s, v28.16b, v26.16b\n" + ".inst 0x4e9aa772 // smmla v18.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x4e99a78e // smmla v14.4s, v28.16b, v25.16b\n" + ".inst 0x4e99a776 // smmla v22.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x70]\n" + ".inst 0x4e9aa78b // smmla v11.4s, v28.16b, v26.16b\n" "add x10, x10, #0x80\n" - ".inst 0x4e86a453 // smmla v19.4s, v2.16b, v6.16b\n" - ".inst 0x4e87a40f // smmla v15.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a457 // smmla v23.4s, v2.16b, v7.16b\n" + ".inst 0x4e9aa773 // smmla v19.4s, v27.16b, v26.16b\n" + ".inst 0x4e99a78f // smmla v15.4s, v28.16b, v25.16b\n" + ".inst 0x4e99a777 // smmla v23.4s, v27.16b, v25.16b\n" "bge 95b\n" "96:" // Height 3: Multiply loop: Skip odd blocks "cbz x27, 101f\n" @@ -1179,33 +1178,33 @@ void a64_hybrid_s8s32_mmla_6x16 ( "ldr b2, [x25, #0x0]\n" "ldr b3, [x24, #0x0]\n" "100:" // Height 3: Multiply loop: Ragged operand read: Done - "ldr q7, [x10, #0x0]\n" - "ldr q6, [x10, #0x10]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a450 // smmla v16.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a451 // smmla v17.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a455 // smmla v21.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a452 // smmla v18.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" + "ldr q26, [x10, #0x0]\n" + "ldr q29, [x10, #0x10]\n" + "trn1 v28.2d, v1.2d, v2.2d\n" + "trn1 v27.2d, v3.2d, v25.2d\n" + ".inst 0x4e9aa788 // smmla v8.4s, v28.16b, v26.16b\n" + ".inst 0x4e9aa770 // smmla v16.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x4e9da78c // smmla v12.4s, v28.16b, v29.16b\n" + ".inst 0x4e9da774 // smmla v20.4s, v27.16b, v29.16b\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x4e9aa789 // smmla v9.4s, v28.16b, v26.16b\n" + ".inst 0x4e9aa771 // smmla v17.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x4e99a78d // smmla v13.4s, v28.16b, v25.16b\n" + ".inst 0x4e99a775 // smmla v21.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x4e9aa78a // smmla v10.4s, v28.16b, v26.16b\n" + ".inst 0x4e9aa772 // smmla v18.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x4e99a78e // smmla v14.4s, v28.16b, v25.16b\n" + ".inst 0x4e99a776 // smmla v22.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x70]\n" "add x10, x10, #0x80\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a453 // smmla v19.4s, v2.16b, v7.16b\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a457 // smmla v23.4s, v2.16b, v6.16b\n" + ".inst 0x4e9aa78b // smmla v11.4s, v28.16b, v26.16b\n" + ".inst 0x4e9aa773 // smmla v19.4s, v27.16b, v26.16b\n" + ".inst 0x4e99a78f // smmla v15.4s, v28.16b, v25.16b\n" + ".inst 0x4e99a777 // smmla v23.4s, v27.16b, v25.16b\n" "101:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1499,14 +1498,14 @@ void a64_hybrid_s8s32_mmla_6x16 ( "126:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 127f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 128f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -1516,9 +1515,9 @@ void a64_hybrid_s8s32_mmla_6x16 ( "b 128f\n" "127:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" "128:" // Height 4: input setup done "cmp x27, #0x10\n" "blt 131f\n" @@ -1531,173 +1530,173 @@ void a64_hybrid_s8s32_mmla_6x16 ( "ldr q6, [x10, #0x10]\n" "blt 130f\n" "129:" // Height 4: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v28.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" + ".inst 0x4e87a788 // smmla v8.4s, v28.16b, v7.16b\n" "sub x27, x27, #0x10\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x4e87a450 // smmla v16.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" + "trn1 v27.2d, v3.2d, v4.2d\n" + ".inst 0x4e87a770 // smmla v16.4s, v27.16b, v7.16b\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x4e86a78c // smmla v12.4s, v28.16b, v6.16b\n" + ".inst 0x4e86a774 // smmla v20.4s, v27.16b, v6.16b\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x4e9aa789 // smmla v9.4s, v28.16b, v26.16b\n" "trn2 v3.2d, v3.2d, v4.2d\n" - ".inst 0x4e87a451 // smmla v17.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" + ".inst 0x4e9aa771 // smmla v17.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x4e99a78d // smmla v13.4s, v28.16b, v25.16b\n" "add x26, x26, #0x10\n" - ".inst 0x4e86a455 // smmla v21.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" + ".inst 0x4e99a775 // smmla v21.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x4e9aa78a // smmla v10.4s, v28.16b, v26.16b\n" "add x25, x25, #0x10\n" - ".inst 0x4e87a452 // smmla v18.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" + ".inst 0x4e9aa772 // smmla v18.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x4e99a78e // smmla v14.4s, v28.16b, v25.16b\n" "add x24, x24, #0x10\n" - ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" + ".inst 0x4e99a776 // smmla v22.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x70]\n" + ".inst 0x4e9aa78b // smmla v11.4s, v28.16b, v26.16b\n" "add x23, x23, #0x10\n" "ldr q4, [x23, #0x0]\n" - ".inst 0x4e87a453 // smmla v19.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x80]\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a457 // smmla v23.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x90]\n" + ".inst 0x4e9aa773 // smmla v19.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x80]\n" + ".inst 0x4e99a78f // smmla v15.4s, v28.16b, v25.16b\n" + ".inst 0x4e99a777 // smmla v23.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x90]\n" "ldr q2, [x25, #0x0]\n" - ".inst 0x4e87a428 // smmla v8.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a470 // smmla v16.4s, v3.16b, v7.16b\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x4e86a42c // smmla v12.4s, v1.16b, v6.16b\n" + ".inst 0x4e9aa428 // smmla v8.4s, v1.16b, v26.16b\n" + ".inst 0x4e9aa470 // smmla v16.4s, v3.16b, v26.16b\n" + "ldr q26, [x10, #0xa0]\n" + ".inst 0x4e99a42c // smmla v12.4s, v1.16b, v25.16b\n" "cmp x27, #0x20\n" - ".inst 0x4e86a474 // smmla v20.4s, v3.16b, v6.16b\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x4e87a429 // smmla v9.4s, v1.16b, v7.16b\n" + ".inst 0x4e99a474 // smmla v20.4s, v3.16b, v25.16b\n" + "ldr q25, [x10, #0xb0]\n" + ".inst 0x4e9aa429 // smmla v9.4s, v1.16b, v26.16b\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x4e87a471 // smmla v17.4s, v3.16b, v7.16b\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x4e86a42d // smmla v13.4s, v1.16b, v6.16b\n" + ".inst 0x4e9aa471 // smmla v17.4s, v3.16b, v26.16b\n" + "ldr q26, [x10, #0xc0]\n" + ".inst 0x4e99a42d // smmla v13.4s, v1.16b, v25.16b\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4e86a475 // smmla v21.4s, v3.16b, v6.16b\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x4e87a42a // smmla v10.4s, v1.16b, v7.16b\n" + ".inst 0x4e99a475 // smmla v21.4s, v3.16b, v25.16b\n" + "ldr q25, [x10, #0xd0]\n" + ".inst 0x4e9aa42a // smmla v10.4s, v1.16b, v26.16b\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x4e87a472 // smmla v18.4s, v3.16b, v7.16b\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" + ".inst 0x4e9aa472 // smmla v18.4s, v3.16b, v26.16b\n" + "ldr q26, [x10, #0xe0]\n" + ".inst 0x4e99a42e // smmla v14.4s, v1.16b, v25.16b\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x4e86a476 // smmla v22.4s, v3.16b, v6.16b\n" - "ldr q6, [x10, #0xf0]\n" + ".inst 0x4e99a476 // smmla v22.4s, v3.16b, v25.16b\n" + "ldr q25, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x4e87a42b // smmla v11.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a473 // smmla v19.4s, v3.16b, v7.16b\n" + ".inst 0x4e9aa42b // smmla v11.4s, v1.16b, v26.16b\n" + ".inst 0x4e9aa473 // smmla v19.4s, v3.16b, v26.16b\n" "ldr q7, [x10, #0x0]\n" - ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" + ".inst 0x4e99a42f // smmla v15.4s, v1.16b, v25.16b\n" "ldr q1, [x26, #0x0]\n" - ".inst 0x4e86a477 // smmla v23.4s, v3.16b, v6.16b\n" + ".inst 0x4e99a477 // smmla v23.4s, v3.16b, v25.16b\n" "ldr q3, [x24, #0x0]\n" "ldr q6, [x10, #0x10]\n" "bge 129b\n" "130:" // Height 4: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v28.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" + ".inst 0x4e87a788 // smmla v8.4s, v28.16b, v7.16b\n" "add x26, x26, #0x10\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x4e87a450 // smmla v16.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" + "trn1 v27.2d, v3.2d, v4.2d\n" + ".inst 0x4e87a770 // smmla v16.4s, v27.16b, v7.16b\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x4e86a78c // smmla v12.4s, v28.16b, v6.16b\n" + ".inst 0x4e86a774 // smmla v20.4s, v27.16b, v6.16b\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x4e9aa789 // smmla v9.4s, v28.16b, v26.16b\n" "trn2 v3.2d, v3.2d, v4.2d\n" - ".inst 0x4e87a451 // smmla v17.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" + ".inst 0x4e9aa771 // smmla v17.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x4e99a78d // smmla v13.4s, v28.16b, v25.16b\n" "add x25, x25, #0x10\n" - ".inst 0x4e86a455 // smmla v21.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" + ".inst 0x4e99a775 // smmla v21.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x4e9aa78a // smmla v10.4s, v28.16b, v26.16b\n" "add x24, x24, #0x10\n" - ".inst 0x4e87a452 // smmla v18.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" + ".inst 0x4e9aa772 // smmla v18.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x4e99a78e // smmla v14.4s, v28.16b, v25.16b\n" "add x23, x23, #0x10\n" - ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" + ".inst 0x4e99a776 // smmla v22.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x70]\n" + ".inst 0x4e9aa78b // smmla v11.4s, v28.16b, v26.16b\n" "sub x27, x27, #0x10\n" - ".inst 0x4e87a453 // smmla v19.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x80]\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" + ".inst 0x4e9aa773 // smmla v19.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x80]\n" + ".inst 0x4e99a78f // smmla v15.4s, v28.16b, v25.16b\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x4e86a457 // smmla v23.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x90]\n" - ".inst 0x4e87a428 // smmla v8.4s, v1.16b, v7.16b\n" + ".inst 0x4e99a777 // smmla v23.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x90]\n" + ".inst 0x4e9aa428 // smmla v8.4s, v1.16b, v26.16b\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4e87a470 // smmla v16.4s, v3.16b, v7.16b\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x4e86a42c // smmla v12.4s, v1.16b, v6.16b\n" + ".inst 0x4e9aa470 // smmla v16.4s, v3.16b, v26.16b\n" + "ldr q26, [x10, #0xa0]\n" + ".inst 0x4e99a42c // smmla v12.4s, v1.16b, v25.16b\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x4e86a474 // smmla v20.4s, v3.16b, v6.16b\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x4e87a429 // smmla v9.4s, v1.16b, v7.16b\n" + ".inst 0x4e99a474 // smmla v20.4s, v3.16b, v25.16b\n" + "ldr q25, [x10, #0xb0]\n" + ".inst 0x4e9aa429 // smmla v9.4s, v1.16b, v26.16b\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x4e87a471 // smmla v17.4s, v3.16b, v7.16b\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x4e86a42d // smmla v13.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a475 // smmla v21.4s, v3.16b, v6.16b\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x4e87a42a // smmla v10.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a472 // smmla v18.4s, v3.16b, v7.16b\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a476 // smmla v22.4s, v3.16b, v6.16b\n" - "ldr q6, [x10, #0xf0]\n" + ".inst 0x4e9aa471 // smmla v17.4s, v3.16b, v26.16b\n" + "ldr q26, [x10, #0xc0]\n" + ".inst 0x4e99a42d // smmla v13.4s, v1.16b, v25.16b\n" + ".inst 0x4e99a475 // smmla v21.4s, v3.16b, v25.16b\n" + "ldr q25, [x10, #0xd0]\n" + ".inst 0x4e9aa42a // smmla v10.4s, v1.16b, v26.16b\n" + ".inst 0x4e9aa472 // smmla v18.4s, v3.16b, v26.16b\n" + "ldr q26, [x10, #0xe0]\n" + ".inst 0x4e99a42e // smmla v14.4s, v1.16b, v25.16b\n" + ".inst 0x4e99a476 // smmla v22.4s, v3.16b, v25.16b\n" + "ldr q25, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x4e87a42b // smmla v11.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a473 // smmla v19.4s, v3.16b, v7.16b\n" - ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a477 // smmla v23.4s, v3.16b, v6.16b\n" + ".inst 0x4e9aa42b // smmla v11.4s, v1.16b, v26.16b\n" + ".inst 0x4e9aa473 // smmla v19.4s, v3.16b, v26.16b\n" + ".inst 0x4e99a42f // smmla v15.4s, v1.16b, v25.16b\n" + ".inst 0x4e99a477 // smmla v23.4s, v3.16b, v25.16b\n" "131:" // Height 4: Multiply loop: Main loop skip "cbz x27, 138f\n" "cmp x27, #0x8\n" "blt 133f\n" "132:" // Height 4: Multiply loop: Odd block loop - "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" + "ldr d26, [x26], #0x8\n" + "ldr d25, [x25], #0x8\n" + "trn1 v28.2d, v26.2d, v25.2d\n" "sub x27, x27, #0x8\n" - "ldr d3, [x24], #0x8\n" - "ldr d4, [x23], #0x8\n" - "trn1 v2.2d, v3.2d, v4.2d\n" + "ldr d26, [x24], #0x8\n" + "ldr d25, [x23], #0x8\n" + "trn1 v27.2d, v26.2d, v25.2d\n" "cmp x27, #0x8\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4e86a408 // smmla v8.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a450 // smmla v16.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4e87a40c // smmla v12.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a454 // smmla v20.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x4e86a409 // smmla v9.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a451 // smmla v17.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x4e87a40d // smmla v13.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a455 // smmla v21.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x4e86a40a // smmla v10.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a452 // smmla v18.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x4e87a40e // smmla v14.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a456 // smmla v22.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x70]\n" + "ldr q26, [x10, #0x0]\n" + "ldr q25, [x10, #0x10]\n" + ".inst 0x4e9aa788 // smmla v8.4s, v28.16b, v26.16b\n" + ".inst 0x4e9aa770 // smmla v16.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x4e99a78c // smmla v12.4s, v28.16b, v25.16b\n" + ".inst 0x4e99a774 // smmla v20.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x4e9aa789 // smmla v9.4s, v28.16b, v26.16b\n" + ".inst 0x4e9aa771 // smmla v17.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x4e99a78d // smmla v13.4s, v28.16b, v25.16b\n" + ".inst 0x4e99a775 // smmla v21.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x4e9aa78a // smmla v10.4s, v28.16b, v26.16b\n" + ".inst 0x4e9aa772 // smmla v18.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x4e99a78e // smmla v14.4s, v28.16b, v25.16b\n" + ".inst 0x4e99a776 // smmla v22.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x70]\n" "add x10, x10, #0x80\n" - ".inst 0x4e86a40b // smmla v11.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a453 // smmla v19.4s, v2.16b, v6.16b\n" - ".inst 0x4e87a40f // smmla v15.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a457 // smmla v23.4s, v2.16b, v7.16b\n" + ".inst 0x4e9aa78b // smmla v11.4s, v28.16b, v26.16b\n" + ".inst 0x4e9aa773 // smmla v19.4s, v27.16b, v26.16b\n" + ".inst 0x4e99a78f // smmla v15.4s, v28.16b, v25.16b\n" + ".inst 0x4e99a777 // smmla v23.4s, v27.16b, v25.16b\n" "bge 132b\n" "133:" // Height 4: Multiply loop: Skip odd blocks "cbz x27, 138f\n" @@ -1742,33 +1741,33 @@ void a64_hybrid_s8s32_mmla_6x16 ( "ldr b3, [x24, #0x0]\n" "ldr b4, [x23, #0x0]\n" "137:" // Height 4: Multiply loop: Ragged operand read: Done - "ldr q7, [x10, #0x0]\n" - "ldr q6, [x10, #0x10]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a450 // smmla v16.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a451 // smmla v17.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a455 // smmla v21.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a452 // smmla v18.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" + "ldr q26, [x10, #0x0]\n" + "ldr q25, [x10, #0x10]\n" + "trn1 v28.2d, v1.2d, v2.2d\n" + "trn1 v27.2d, v3.2d, v4.2d\n" + ".inst 0x4e9aa788 // smmla v8.4s, v28.16b, v26.16b\n" + ".inst 0x4e9aa770 // smmla v16.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x4e99a78c // smmla v12.4s, v28.16b, v25.16b\n" + ".inst 0x4e99a774 // smmla v20.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x4e9aa789 // smmla v9.4s, v28.16b, v26.16b\n" + ".inst 0x4e9aa771 // smmla v17.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x4e99a78d // smmla v13.4s, v28.16b, v25.16b\n" + ".inst 0x4e99a775 // smmla v21.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x4e9aa78a // smmla v10.4s, v28.16b, v26.16b\n" + ".inst 0x4e9aa772 // smmla v18.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x4e99a78e // smmla v14.4s, v28.16b, v25.16b\n" + ".inst 0x4e99a776 // smmla v22.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x70]\n" "add x10, x10, #0x80\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a453 // smmla v19.4s, v2.16b, v7.16b\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a457 // smmla v23.4s, v2.16b, v6.16b\n" + ".inst 0x4e9aa78b // smmla v11.4s, v28.16b, v26.16b\n" + ".inst 0x4e9aa773 // smmla v19.4s, v27.16b, v26.16b\n" + ".inst 0x4e99a78f // smmla v15.4s, v28.16b, v25.16b\n" + ".inst 0x4e99a777 // smmla v23.4s, v27.16b, v25.16b\n" "138:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -2125,15 +2124,15 @@ void a64_hybrid_s8s32_mmla_6x16 ( "163:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 164f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 165f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -2144,10 +2143,10 @@ void a64_hybrid_s8s32_mmla_6x16 ( "b 165f\n" "164:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" "165:" // Height 5: input setup done "cmp x27, #0x10\n" "blt 168f\n" @@ -2160,174 +2159,174 @@ void a64_hybrid_s8s32_mmla_6x16 ( "ldr q7, [x10, #0x0]\n" "blt 167f\n" "166:" // Height 5: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v6.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" + ".inst 0x4e87a4c8 // smmla v8.4s, v6.16b, v7.16b\n" "trn1 v2.2d, v3.2d, v4.2d\n" "trn2 v3.2d, v3.2d, v4.2d\n" ".inst 0x4e87a450 // smmla v16.4s, v2.16b, v7.16b\n" "sub x27, x27, #0x10\n" - "trn1 v4.2d, v5.2d, v6.2d\n" - "trn2 v5.2d, v5.2d, v6.2d\n" - "ldr q6, [x10, #0x10]\n" + "trn1 v4.2d, v5.2d, v0.2d\n" + "trn2 v5.2d, v5.2d, v0.2d\n" + "ldr q0, [x10, #0x10]\n" ".inst 0x4e87a498 // smmla v24.4s, v4.16b, v7.16b\n" "ldr q7, [x10, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n" + ".inst 0x4e80a4cc // smmla v12.4s, v6.16b, v0.16b\n" + ".inst 0x4e80a454 // smmla v20.4s, v2.16b, v0.16b\n" "add x26, x26, #0x10\n" - ".inst 0x4e86a49c // smmla v28.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" + ".inst 0x4e80a49c // smmla v28.4s, v4.16b, v0.16b\n" + "ldr q0, [x10, #0x30]\n" + ".inst 0x4e87a4c9 // smmla v9.4s, v6.16b, v7.16b\n" "add x25, x25, #0x10\n" ".inst 0x4e87a451 // smmla v17.4s, v2.16b, v7.16b\n" ".inst 0x4e87a499 // smmla v25.4s, v4.16b, v7.16b\n" "ldr q7, [x10, #0x40]\n" "add x24, x24, #0x10\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a455 // smmla v21.4s, v2.16b, v6.16b\n" + ".inst 0x4e80a4cd // smmla v13.4s, v6.16b, v0.16b\n" + ".inst 0x4e80a455 // smmla v21.4s, v2.16b, v0.16b\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - ".inst 0x4e86a49d // smmla v29.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" + ".inst 0x4e80a49d // smmla v29.4s, v4.16b, v0.16b\n" + "ldr q0, [x10, #0x50]\n" + ".inst 0x4e87a4ca // smmla v10.4s, v6.16b, v7.16b\n" "cmp x27, #0x20\n" ".inst 0x4e87a452 // smmla v18.4s, v2.16b, v7.16b\n" ".inst 0x4e87a49a // smmla v26.4s, v4.16b, v7.16b\n" "ldr q7, [x10, #0x60]\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n" + ".inst 0x4e80a4ce // smmla v14.4s, v6.16b, v0.16b\n" + ".inst 0x4e80a456 // smmla v22.4s, v2.16b, v0.16b\n" "prfm pldl1keep, [x25, #0x80]\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x4e86a49e // smmla v30.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" + ".inst 0x4e80a49e // smmla v30.4s, v4.16b, v0.16b\n" + "ldr q0, [x10, #0x70]\n" + ".inst 0x4e87a4cb // smmla v11.4s, v6.16b, v7.16b\n" "prfm pldl1keep, [x23, #0x80]\n" ".inst 0x4e87a453 // smmla v19.4s, v2.16b, v7.16b\n" ".inst 0x4e87a49b // smmla v27.4s, v4.16b, v7.16b\n" "ldr q7, [x10, #0x80]\n" "prfm pldl1keep, [x22, #0x80]\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a457 // smmla v23.4s, v2.16b, v6.16b\n" + ".inst 0x4e80a4cf // smmla v15.4s, v6.16b, v0.16b\n" + ".inst 0x4e80a457 // smmla v23.4s, v2.16b, v0.16b\n" "ldr q2, [x25, #0x0]\n" - ".inst 0x4e86a49f // smmla v31.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x90]\n" + ".inst 0x4e80a49f // smmla v31.4s, v4.16b, v0.16b\n" + "ldr q0, [x10, #0x90]\n" "ldr q4, [x23, #0x0]\n" ".inst 0x4e87a428 // smmla v8.4s, v1.16b, v7.16b\n" ".inst 0x4e87a470 // smmla v16.4s, v3.16b, v7.16b\n" ".inst 0x4e87a4b8 // smmla v24.4s, v5.16b, v7.16b\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x4e86a42c // smmla v12.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a474 // smmla v20.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4bc // smmla v28.4s, v5.16b, v6.16b\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x4e87a429 // smmla v9.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a471 // smmla v17.4s, v3.16b, v7.16b\n" - ".inst 0x4e87a4b9 // smmla v25.4s, v5.16b, v7.16b\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x4e86a42d // smmla v13.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a475 // smmla v21.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4bd // smmla v29.4s, v5.16b, v6.16b\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x4e87a42a // smmla v10.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a472 // smmla v18.4s, v3.16b, v7.16b\n" - ".inst 0x4e87a4ba // smmla v26.4s, v5.16b, v7.16b\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a476 // smmla v22.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4be // smmla v30.4s, v5.16b, v6.16b\n" - "ldr q6, [x10, #0xf0]\n" + "ldr q6, [x10, #0xa0]\n" + ".inst 0x4e80a42c // smmla v12.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a474 // smmla v20.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4bc // smmla v28.4s, v5.16b, v0.16b\n" + "ldr q0, [x10, #0xb0]\n" + ".inst 0x4e86a429 // smmla v9.4s, v1.16b, v6.16b\n" + ".inst 0x4e86a471 // smmla v17.4s, v3.16b, v6.16b\n" + ".inst 0x4e86a4b9 // smmla v25.4s, v5.16b, v6.16b\n" + "ldr q6, [x10, #0xc0]\n" + ".inst 0x4e80a42d // smmla v13.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a475 // smmla v21.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4bd // smmla v29.4s, v5.16b, v0.16b\n" + "ldr q0, [x10, #0xd0]\n" + ".inst 0x4e86a42a // smmla v10.4s, v1.16b, v6.16b\n" + ".inst 0x4e86a472 // smmla v18.4s, v3.16b, v6.16b\n" + ".inst 0x4e86a4ba // smmla v26.4s, v5.16b, v6.16b\n" + "ldr q6, [x10, #0xe0]\n" + ".inst 0x4e80a42e // smmla v14.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a476 // smmla v22.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4be // smmla v30.4s, v5.16b, v0.16b\n" + "ldr q0, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x4e87a42b // smmla v11.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a473 // smmla v19.4s, v3.16b, v7.16b\n" - ".inst 0x4e87a4bb // smmla v27.4s, v5.16b, v7.16b\n" + ".inst 0x4e86a42b // smmla v11.4s, v1.16b, v6.16b\n" + ".inst 0x4e86a473 // smmla v19.4s, v3.16b, v6.16b\n" + ".inst 0x4e86a4bb // smmla v27.4s, v5.16b, v6.16b\n" "ldr q7, [x10, #0x0]\n" - ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" + ".inst 0x4e80a42f // smmla v15.4s, v1.16b, v0.16b\n" "ldr q1, [x26, #0x0]\n" - ".inst 0x4e86a477 // smmla v23.4s, v3.16b, v6.16b\n" + ".inst 0x4e80a477 // smmla v23.4s, v3.16b, v0.16b\n" "ldr q3, [x24, #0x0]\n" - ".inst 0x4e86a4bf // smmla v31.4s, v5.16b, v6.16b\n" + ".inst 0x4e80a4bf // smmla v31.4s, v5.16b, v0.16b\n" "ldr q5, [x22, #0x0]\n" "bge 166b\n" "167:" // Height 5: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v6.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" + ".inst 0x4e87a4c8 // smmla v8.4s, v6.16b, v7.16b\n" "trn1 v2.2d, v3.2d, v4.2d\n" "trn2 v3.2d, v3.2d, v4.2d\n" ".inst 0x4e87a450 // smmla v16.4s, v2.16b, v7.16b\n" "add x26, x26, #0x10\n" - "trn1 v4.2d, v5.2d, v6.2d\n" - "trn2 v5.2d, v5.2d, v6.2d\n" - "ldr q6, [x10, #0x10]\n" + "trn1 v4.2d, v5.2d, v0.2d\n" + "trn2 v5.2d, v5.2d, v0.2d\n" + "ldr q0, [x10, #0x10]\n" ".inst 0x4e87a498 // smmla v24.4s, v4.16b, v7.16b\n" "ldr q7, [x10, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n" + ".inst 0x4e80a4cc // smmla v12.4s, v6.16b, v0.16b\n" + ".inst 0x4e80a454 // smmla v20.4s, v2.16b, v0.16b\n" "add x25, x25, #0x10\n" - ".inst 0x4e86a49c // smmla v28.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" + ".inst 0x4e80a49c // smmla v28.4s, v4.16b, v0.16b\n" + "ldr q0, [x10, #0x30]\n" + ".inst 0x4e87a4c9 // smmla v9.4s, v6.16b, v7.16b\n" "add x24, x24, #0x10\n" ".inst 0x4e87a451 // smmla v17.4s, v2.16b, v7.16b\n" ".inst 0x4e87a499 // smmla v25.4s, v4.16b, v7.16b\n" "ldr q7, [x10, #0x40]\n" "add x23, x23, #0x10\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a455 // smmla v21.4s, v2.16b, v6.16b\n" + ".inst 0x4e80a4cd // smmla v13.4s, v6.16b, v0.16b\n" + ".inst 0x4e80a455 // smmla v21.4s, v2.16b, v0.16b\n" "add x22, x22, #0x10\n" "sub x27, x27, #0x10\n" - ".inst 0x4e86a49d // smmla v29.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" + ".inst 0x4e80a49d // smmla v29.4s, v4.16b, v0.16b\n" + "ldr q0, [x10, #0x50]\n" + ".inst 0x4e87a4ca // smmla v10.4s, v6.16b, v7.16b\n" "prfm pldl1keep, [x26, #0x80]\n" ".inst 0x4e87a452 // smmla v18.4s, v2.16b, v7.16b\n" ".inst 0x4e87a49a // smmla v26.4s, v4.16b, v7.16b\n" "ldr q7, [x10, #0x60]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n" + ".inst 0x4e80a4ce // smmla v14.4s, v6.16b, v0.16b\n" + ".inst 0x4e80a456 // smmla v22.4s, v2.16b, v0.16b\n" "prfm pldl1keep, [x24, #0x80]\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x4e86a49e // smmla v30.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" + ".inst 0x4e80a49e // smmla v30.4s, v4.16b, v0.16b\n" + "ldr q0, [x10, #0x70]\n" + ".inst 0x4e87a4cb // smmla v11.4s, v6.16b, v7.16b\n" "prfm pldl1keep, [x22, #0x80]\n" ".inst 0x4e87a453 // smmla v19.4s, v2.16b, v7.16b\n" ".inst 0x4e87a49b // smmla v27.4s, v4.16b, v7.16b\n" "ldr q7, [x10, #0x80]\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a457 // smmla v23.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49f // smmla v31.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x90]\n" + ".inst 0x4e80a4cf // smmla v15.4s, v6.16b, v0.16b\n" + ".inst 0x4e80a457 // smmla v23.4s, v2.16b, v0.16b\n" + ".inst 0x4e80a49f // smmla v31.4s, v4.16b, v0.16b\n" + "ldr q2, [x10, #0x90]\n" ".inst 0x4e87a428 // smmla v8.4s, v1.16b, v7.16b\n" ".inst 0x4e87a470 // smmla v16.4s, v3.16b, v7.16b\n" ".inst 0x4e87a4b8 // smmla v24.4s, v5.16b, v7.16b\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x4e86a42c // smmla v12.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a474 // smmla v20.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4bc // smmla v28.4s, v5.16b, v6.16b\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x4e87a429 // smmla v9.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a471 // smmla v17.4s, v3.16b, v7.16b\n" - ".inst 0x4e87a4b9 // smmla v25.4s, v5.16b, v7.16b\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x4e86a42d // smmla v13.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a475 // smmla v21.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4bd // smmla v29.4s, v5.16b, v6.16b\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x4e87a42a // smmla v10.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a472 // smmla v18.4s, v3.16b, v7.16b\n" - ".inst 0x4e87a4ba // smmla v26.4s, v5.16b, v7.16b\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a476 // smmla v22.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4be // smmla v30.4s, v5.16b, v6.16b\n" + "ldr q0, [x10, #0xa0]\n" + ".inst 0x4e82a42c // smmla v12.4s, v1.16b, v2.16b\n" + ".inst 0x4e82a474 // smmla v20.4s, v3.16b, v2.16b\n" + ".inst 0x4e82a4bc // smmla v28.4s, v5.16b, v2.16b\n" + "ldr q2, [x10, #0xb0]\n" + ".inst 0x4e80a429 // smmla v9.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a471 // smmla v17.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4b9 // smmla v25.4s, v5.16b, v0.16b\n" + "ldr q0, [x10, #0xc0]\n" + ".inst 0x4e82a42d // smmla v13.4s, v1.16b, v2.16b\n" + ".inst 0x4e82a475 // smmla v21.4s, v3.16b, v2.16b\n" + ".inst 0x4e82a4bd // smmla v29.4s, v5.16b, v2.16b\n" + "ldr q2, [x10, #0xd0]\n" + ".inst 0x4e80a42a // smmla v10.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a472 // smmla v18.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4ba // smmla v26.4s, v5.16b, v0.16b\n" + "ldr q0, [x10, #0xe0]\n" + ".inst 0x4e82a42e // smmla v14.4s, v1.16b, v2.16b\n" + ".inst 0x4e82a476 // smmla v22.4s, v3.16b, v2.16b\n" + ".inst 0x4e82a4be // smmla v30.4s, v5.16b, v2.16b\n" "ldr q6, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x4e87a42b // smmla v11.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a473 // smmla v19.4s, v3.16b, v7.16b\n" - ".inst 0x4e87a4bb // smmla v27.4s, v5.16b, v7.16b\n" + ".inst 0x4e80a42b // smmla v11.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a473 // smmla v19.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4bb // smmla v27.4s, v5.16b, v0.16b\n" ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" ".inst 0x4e86a477 // smmla v23.4s, v3.16b, v6.16b\n" ".inst 0x4e86a4bf // smmla v31.4s, v5.16b, v6.16b\n" @@ -2337,48 +2336,48 @@ void a64_hybrid_s8s32_mmla_6x16 ( "blt 170f\n" "169:" // Height 5: Multiply loop: Odd block loop "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "ldr d3, [x24], #0x8\n" - "ldr d4, [x23], #0x8\n" - "trn1 v2.2d, v3.2d, v4.2d\n" + "ldr d0, [x25], #0x8\n" + "trn1 v4.2d, v1.2d, v0.2d\n" + "ldr d1, [x24], #0x8\n" + "ldr d0, [x23], #0x8\n" + "trn1 v3.2d, v1.2d, v0.2d\n" "sub x27, x27, #0x8\n" - "ldr d5, [x22], #0x8\n" - "ldr q6, [x10, #0x0]\n" - "trn1 v4.2d, v5.2d, v7.2d\n" - ".inst 0x4e86a408 // smmla v8.4s, v0.16b, v6.16b\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4e86a450 // smmla v16.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a498 // smmla v24.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4e87a40c // smmla v12.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a454 // smmla v20.4s, v2.16b, v7.16b\n" + "ldr d0, [x22], #0x8\n" + "ldr q1, [x10, #0x0]\n" + "trn1 v2.2d, v0.2d, v2.2d\n" + ".inst 0x4e81a488 // smmla v8.4s, v4.16b, v1.16b\n" + "ldr q0, [x10, #0x10]\n" + ".inst 0x4e81a470 // smmla v16.4s, v3.16b, v1.16b\n" + ".inst 0x4e81a458 // smmla v24.4s, v2.16b, v1.16b\n" + "ldr q1, [x10, #0x20]\n" + ".inst 0x4e80a48c // smmla v12.4s, v4.16b, v0.16b\n" + ".inst 0x4e80a474 // smmla v20.4s, v3.16b, v0.16b\n" "cmp x27, #0x8\n" - ".inst 0x4e87a49c // smmla v28.4s, v4.16b, v7.16b\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x4e86a409 // smmla v9.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a451 // smmla v17.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a499 // smmla v25.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x4e87a40d // smmla v13.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a455 // smmla v21.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a49d // smmla v29.4s, v4.16b, v7.16b\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x4e86a40a // smmla v10.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a452 // smmla v18.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49a // smmla v26.4s, v4.16b, v6.16b\n" + ".inst 0x4e80a45c // smmla v28.4s, v2.16b, v0.16b\n" + "ldr q0, [x10, #0x30]\n" + ".inst 0x4e81a489 // smmla v9.4s, v4.16b, v1.16b\n" + ".inst 0x4e81a471 // smmla v17.4s, v3.16b, v1.16b\n" + ".inst 0x4e81a459 // smmla v25.4s, v2.16b, v1.16b\n" + "ldr q1, [x10, #0x40]\n" + ".inst 0x4e80a48d // smmla v13.4s, v4.16b, v0.16b\n" + ".inst 0x4e80a475 // smmla v21.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a45d // smmla v29.4s, v2.16b, v0.16b\n" + "ldr q0, [x10, #0x50]\n" + ".inst 0x4e81a48a // smmla v10.4s, v4.16b, v1.16b\n" + ".inst 0x4e81a472 // smmla v18.4s, v3.16b, v1.16b\n" + ".inst 0x4e81a45a // smmla v26.4s, v2.16b, v1.16b\n" "ldr q6, [x10, #0x60]\n" - ".inst 0x4e87a40e // smmla v14.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a456 // smmla v22.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a49e // smmla v30.4s, v4.16b, v7.16b\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x4e86a40b // smmla v11.4s, v0.16b, v6.16b\n" + ".inst 0x4e80a48e // smmla v14.4s, v4.16b, v0.16b\n" + ".inst 0x4e80a476 // smmla v22.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a45e // smmla v30.4s, v2.16b, v0.16b\n" + "ldr q0, [x10, #0x70]\n" + ".inst 0x4e86a48b // smmla v11.4s, v4.16b, v6.16b\n" "add x10, x10, #0x80\n" - ".inst 0x4e86a453 // smmla v19.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49b // smmla v27.4s, v4.16b, v6.16b\n" - ".inst 0x4e87a40f // smmla v15.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a457 // smmla v23.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a49f // smmla v31.4s, v4.16b, v7.16b\n" + ".inst 0x4e86a473 // smmla v19.4s, v3.16b, v6.16b\n" + ".inst 0x4e86a45b // smmla v27.4s, v2.16b, v6.16b\n" + ".inst 0x4e80a48f // smmla v15.4s, v4.16b, v0.16b\n" + ".inst 0x4e80a477 // smmla v23.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a45f // smmla v31.4s, v2.16b, v0.16b\n" "bge 169b\n" "170:" // Height 5: Multiply loop: Skip odd blocks "cbz x27, 175f\n" @@ -2430,42 +2429,42 @@ void a64_hybrid_s8s32_mmla_6x16 ( "ldr b4, [x23, #0x0]\n" "ldr b5, [x22, #0x0]\n" "174:" // Height 5: Multiply loop: Ragged operand read: Done - "ldr q7, [x10, #0x0]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - "trn1 v4.2d, v5.2d, v6.2d\n" - "ldr q6, [x10, #0x10]\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a450 // smmla v16.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a498 // smmla v24.4s, v4.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49c // smmla v28.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a451 // smmla v17.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a499 // smmla v25.4s, v4.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a455 // smmla v21.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49d // smmla v29.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a452 // smmla v18.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a49a // smmla v26.4s, v4.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49e // smmla v30.4s, v4.16b, v6.16b\n" + "ldr q6, [x10, #0x0]\n" + "trn1 v7.2d, v1.2d, v2.2d\n" + "trn1 v3.2d, v3.2d, v4.2d\n" + "trn1 v2.2d, v5.2d, v0.2d\n" + "ldr q1, [x10, #0x10]\n" + ".inst 0x4e86a4e8 // smmla v8.4s, v7.16b, v6.16b\n" + ".inst 0x4e86a470 // smmla v16.4s, v3.16b, v6.16b\n" + ".inst 0x4e86a458 // smmla v24.4s, v2.16b, v6.16b\n" + "ldr q0, [x10, #0x20]\n" + ".inst 0x4e81a4ec // smmla v12.4s, v7.16b, v1.16b\n" + ".inst 0x4e81a474 // smmla v20.4s, v3.16b, v1.16b\n" + ".inst 0x4e81a45c // smmla v28.4s, v2.16b, v1.16b\n" + "ldr q1, [x10, #0x30]\n" + ".inst 0x4e80a4e9 // smmla v9.4s, v7.16b, v0.16b\n" + ".inst 0x4e80a471 // smmla v17.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a459 // smmla v25.4s, v2.16b, v0.16b\n" + "ldr q0, [x10, #0x40]\n" + ".inst 0x4e81a4ed // smmla v13.4s, v7.16b, v1.16b\n" + ".inst 0x4e81a475 // smmla v21.4s, v3.16b, v1.16b\n" + ".inst 0x4e81a45d // smmla v29.4s, v2.16b, v1.16b\n" + "ldr q1, [x10, #0x50]\n" + ".inst 0x4e80a4ea // smmla v10.4s, v7.16b, v0.16b\n" + ".inst 0x4e80a472 // smmla v18.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a45a // smmla v26.4s, v2.16b, v0.16b\n" + "ldr q0, [x10, #0x60]\n" + ".inst 0x4e81a4ee // smmla v14.4s, v7.16b, v1.16b\n" + ".inst 0x4e81a476 // smmla v22.4s, v3.16b, v1.16b\n" + ".inst 0x4e81a45e // smmla v30.4s, v2.16b, v1.16b\n" "ldr q6, [x10, #0x70]\n" "add x10, x10, #0x80\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a453 // smmla v19.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a49b // smmla v27.4s, v4.16b, v7.16b\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a457 // smmla v23.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49f // smmla v31.4s, v4.16b, v6.16b\n" + ".inst 0x4e80a4eb // smmla v11.4s, v7.16b, v0.16b\n" + ".inst 0x4e80a473 // smmla v19.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a45b // smmla v27.4s, v2.16b, v0.16b\n" + ".inst 0x4e86a4ef // smmla v15.4s, v7.16b, v6.16b\n" + ".inst 0x4e86a477 // smmla v23.4s, v3.16b, v6.16b\n" + ".inst 0x4e86a45f // smmla v31.4s, v2.16b, v6.16b\n" "175:" // Height 5: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -2872,16 +2871,16 @@ void a64_hybrid_s8s32_mmla_6x16 ( "200:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 201f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 202f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -2893,11 +2892,11 @@ void a64_hybrid_s8s32_mmla_6x16 ( "b 202f\n" "201:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" + "add x21, x22, x21\n" "202:" // Height 6: input setup done "cmp x27, #0x10\n" "blt 205f\n" @@ -2964,42 +2963,42 @@ void a64_hybrid_s8s32_mmla_6x16 ( "ldr q2, [x25, #0x0]\n" "prfm pldl1keep, [x21, #0x80]\n" ".inst 0x4e86a49f // smmla v31.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x90]\n" + "ldr q0, [x10, #0x90]\n" "ldr q4, [x23, #0x0]\n" ".inst 0x4e87a428 // smmla v8.4s, v1.16b, v7.16b\n" ".inst 0x4e87a470 // smmla v16.4s, v3.16b, v7.16b\n" ".inst 0x4e87a4b8 // smmla v24.4s, v5.16b, v7.16b\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x4e86a42c // smmla v12.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a474 // smmla v20.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4bc // smmla v28.4s, v5.16b, v6.16b\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x4e87a429 // smmla v9.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a471 // smmla v17.4s, v3.16b, v7.16b\n" - ".inst 0x4e87a4b9 // smmla v25.4s, v5.16b, v7.16b\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x4e86a42d // smmla v13.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a475 // smmla v21.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4bd // smmla v29.4s, v5.16b, v6.16b\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x4e87a42a // smmla v10.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a472 // smmla v18.4s, v3.16b, v7.16b\n" - ".inst 0x4e87a4ba // smmla v26.4s, v5.16b, v7.16b\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a476 // smmla v22.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4be // smmla v30.4s, v5.16b, v6.16b\n" - "ldr q6, [x10, #0xf0]\n" + "ldr q6, [x10, #0xa0]\n" + ".inst 0x4e80a42c // smmla v12.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a474 // smmla v20.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4bc // smmla v28.4s, v5.16b, v0.16b\n" + "ldr q0, [x10, #0xb0]\n" + ".inst 0x4e86a429 // smmla v9.4s, v1.16b, v6.16b\n" + ".inst 0x4e86a471 // smmla v17.4s, v3.16b, v6.16b\n" + ".inst 0x4e86a4b9 // smmla v25.4s, v5.16b, v6.16b\n" + "ldr q6, [x10, #0xc0]\n" + ".inst 0x4e80a42d // smmla v13.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a475 // smmla v21.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4bd // smmla v29.4s, v5.16b, v0.16b\n" + "ldr q0, [x10, #0xd0]\n" + ".inst 0x4e86a42a // smmla v10.4s, v1.16b, v6.16b\n" + ".inst 0x4e86a472 // smmla v18.4s, v3.16b, v6.16b\n" + ".inst 0x4e86a4ba // smmla v26.4s, v5.16b, v6.16b\n" + "ldr q6, [x10, #0xe0]\n" + ".inst 0x4e80a42e // smmla v14.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a476 // smmla v22.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4be // smmla v30.4s, v5.16b, v0.16b\n" + "ldr q0, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x4e87a42b // smmla v11.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a473 // smmla v19.4s, v3.16b, v7.16b\n" - ".inst 0x4e87a4bb // smmla v27.4s, v5.16b, v7.16b\n" + ".inst 0x4e86a42b // smmla v11.4s, v1.16b, v6.16b\n" + ".inst 0x4e86a473 // smmla v19.4s, v3.16b, v6.16b\n" + ".inst 0x4e86a4bb // smmla v27.4s, v5.16b, v6.16b\n" "ldr q7, [x10, #0x0]\n" - ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" + ".inst 0x4e80a42f // smmla v15.4s, v1.16b, v0.16b\n" "ldr q1, [x26, #0x0]\n" - ".inst 0x4e86a477 // smmla v23.4s, v3.16b, v6.16b\n" + ".inst 0x4e80a477 // smmla v23.4s, v3.16b, v0.16b\n" "ldr q3, [x24, #0x0]\n" - ".inst 0x4e86a4bf // smmla v31.4s, v5.16b, v6.16b\n" + ".inst 0x4e80a4bf // smmla v31.4s, v5.16b, v0.16b\n" "ldr q5, [x22, #0x0]\n" "ldr q6, [x21, #0x0]\n" "bge 203b\n" @@ -3055,35 +3054,35 @@ void a64_hybrid_s8s32_mmla_6x16 ( ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" ".inst 0x4e86a457 // smmla v23.4s, v2.16b, v6.16b\n" ".inst 0x4e86a49f // smmla v31.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x90]\n" + "ldr q2, [x10, #0x90]\n" ".inst 0x4e87a428 // smmla v8.4s, v1.16b, v7.16b\n" ".inst 0x4e87a470 // smmla v16.4s, v3.16b, v7.16b\n" ".inst 0x4e87a4b8 // smmla v24.4s, v5.16b, v7.16b\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x4e86a42c // smmla v12.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a474 // smmla v20.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4bc // smmla v28.4s, v5.16b, v6.16b\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x4e87a429 // smmla v9.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a471 // smmla v17.4s, v3.16b, v7.16b\n" - ".inst 0x4e87a4b9 // smmla v25.4s, v5.16b, v7.16b\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x4e86a42d // smmla v13.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a475 // smmla v21.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4bd // smmla v29.4s, v5.16b, v6.16b\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x4e87a42a // smmla v10.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a472 // smmla v18.4s, v3.16b, v7.16b\n" - ".inst 0x4e87a4ba // smmla v26.4s, v5.16b, v7.16b\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - ".inst 0x4e86a476 // smmla v22.4s, v3.16b, v6.16b\n" - ".inst 0x4e86a4be // smmla v30.4s, v5.16b, v6.16b\n" + "ldr q0, [x10, #0xa0]\n" + ".inst 0x4e82a42c // smmla v12.4s, v1.16b, v2.16b\n" + ".inst 0x4e82a474 // smmla v20.4s, v3.16b, v2.16b\n" + ".inst 0x4e82a4bc // smmla v28.4s, v5.16b, v2.16b\n" + "ldr q2, [x10, #0xb0]\n" + ".inst 0x4e80a429 // smmla v9.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a471 // smmla v17.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4b9 // smmla v25.4s, v5.16b, v0.16b\n" + "ldr q0, [x10, #0xc0]\n" + ".inst 0x4e82a42d // smmla v13.4s, v1.16b, v2.16b\n" + ".inst 0x4e82a475 // smmla v21.4s, v3.16b, v2.16b\n" + ".inst 0x4e82a4bd // smmla v29.4s, v5.16b, v2.16b\n" + "ldr q2, [x10, #0xd0]\n" + ".inst 0x4e80a42a // smmla v10.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a472 // smmla v18.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4ba // smmla v26.4s, v5.16b, v0.16b\n" + "ldr q0, [x10, #0xe0]\n" + ".inst 0x4e82a42e // smmla v14.4s, v1.16b, v2.16b\n" + ".inst 0x4e82a476 // smmla v22.4s, v3.16b, v2.16b\n" + ".inst 0x4e82a4be // smmla v30.4s, v5.16b, v2.16b\n" "ldr q6, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x4e87a42b // smmla v11.4s, v1.16b, v7.16b\n" - ".inst 0x4e87a473 // smmla v19.4s, v3.16b, v7.16b\n" - ".inst 0x4e87a4bb // smmla v27.4s, v5.16b, v7.16b\n" + ".inst 0x4e80a42b // smmla v11.4s, v1.16b, v0.16b\n" + ".inst 0x4e80a473 // smmla v19.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a4bb // smmla v27.4s, v5.16b, v0.16b\n" ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" ".inst 0x4e86a477 // smmla v23.4s, v3.16b, v6.16b\n" ".inst 0x4e86a4bf // smmla v31.4s, v5.16b, v6.16b\n" @@ -3093,49 +3092,49 @@ void a64_hybrid_s8s32_mmla_6x16 ( "blt 207f\n" "206:" // Height 6: Multiply loop: Odd block loop "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" + "ldr d0, [x25], #0x8\n" + "trn1 v4.2d, v1.2d, v0.2d\n" "sub x27, x27, #0x8\n" - "ldr d3, [x24], #0x8\n" - "ldr d4, [x23], #0x8\n" - "trn1 v2.2d, v3.2d, v4.2d\n" + "ldr d1, [x24], #0x8\n" + "ldr d0, [x23], #0x8\n" + "trn1 v3.2d, v1.2d, v0.2d\n" "cmp x27, #0x8\n" - "ldr d5, [x22], #0x8\n" - "ldr d7, [x21], #0x8\n" - "trn1 v4.2d, v5.2d, v7.2d\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x4e86a408 // smmla v8.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a450 // smmla v16.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a498 // smmla v24.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x4e87a40c // smmla v12.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a454 // smmla v20.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a49c // smmla v28.4s, v4.16b, v7.16b\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x4e86a409 // smmla v9.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a451 // smmla v17.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a499 // smmla v25.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x4e87a40d // smmla v13.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a455 // smmla v21.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a49d // smmla v29.4s, v4.16b, v7.16b\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x4e86a40a // smmla v10.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a452 // smmla v18.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49a // smmla v26.4s, v4.16b, v6.16b\n" + "ldr d1, [x22], #0x8\n" + "ldr d0, [x21], #0x8\n" + "trn1 v2.2d, v1.2d, v0.2d\n" + "ldr q1, [x10, #0x0]\n" + "ldr q0, [x10, #0x10]\n" + ".inst 0x4e81a488 // smmla v8.4s, v4.16b, v1.16b\n" + ".inst 0x4e81a470 // smmla v16.4s, v3.16b, v1.16b\n" + ".inst 0x4e81a458 // smmla v24.4s, v2.16b, v1.16b\n" + "ldr q1, [x10, #0x20]\n" + ".inst 0x4e80a48c // smmla v12.4s, v4.16b, v0.16b\n" + ".inst 0x4e80a474 // smmla v20.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a45c // smmla v28.4s, v2.16b, v0.16b\n" + "ldr q0, [x10, #0x30]\n" + ".inst 0x4e81a489 // smmla v9.4s, v4.16b, v1.16b\n" + ".inst 0x4e81a471 // smmla v17.4s, v3.16b, v1.16b\n" + ".inst 0x4e81a459 // smmla v25.4s, v2.16b, v1.16b\n" + "ldr q1, [x10, #0x40]\n" + ".inst 0x4e80a48d // smmla v13.4s, v4.16b, v0.16b\n" + ".inst 0x4e80a475 // smmla v21.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a45d // smmla v29.4s, v2.16b, v0.16b\n" + "ldr q0, [x10, #0x50]\n" + ".inst 0x4e81a48a // smmla v10.4s, v4.16b, v1.16b\n" + ".inst 0x4e81a472 // smmla v18.4s, v3.16b, v1.16b\n" + ".inst 0x4e81a45a // smmla v26.4s, v2.16b, v1.16b\n" "ldr q6, [x10, #0x60]\n" - ".inst 0x4e87a40e // smmla v14.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a456 // smmla v22.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a49e // smmla v30.4s, v4.16b, v7.16b\n" - "ldr q7, [x10, #0x70]\n" + ".inst 0x4e80a48e // smmla v14.4s, v4.16b, v0.16b\n" + ".inst 0x4e80a476 // smmla v22.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a45e // smmla v30.4s, v2.16b, v0.16b\n" + "ldr q0, [x10, #0x70]\n" "add x10, x10, #0x80\n" - ".inst 0x4e86a40b // smmla v11.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a453 // smmla v19.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49b // smmla v27.4s, v4.16b, v6.16b\n" - ".inst 0x4e87a40f // smmla v15.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a457 // smmla v23.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a49f // smmla v31.4s, v4.16b, v7.16b\n" + ".inst 0x4e86a48b // smmla v11.4s, v4.16b, v6.16b\n" + ".inst 0x4e86a473 // smmla v19.4s, v3.16b, v6.16b\n" + ".inst 0x4e86a45b // smmla v27.4s, v2.16b, v6.16b\n" + ".inst 0x4e80a48f // smmla v15.4s, v4.16b, v0.16b\n" + ".inst 0x4e80a477 // smmla v23.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a45f // smmla v31.4s, v2.16b, v0.16b\n" "bge 206b\n" "207:" // Height 6: Multiply loop: Skip odd blocks "cbz x27, 212f\n" @@ -3194,42 +3193,42 @@ void a64_hybrid_s8s32_mmla_6x16 ( "ldr b5, [x22, #0x0]\n" "ldr b6, [x21, #0x0]\n" "211:" // Height 6: Multiply loop: Ragged operand read: Done - "ldr q7, [x10, #0x0]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" - "trn1 v4.2d, v5.2d, v6.2d\n" - "ldr q6, [x10, #0x10]\n" - ".inst 0x4e87a450 // smmla v16.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a498 // smmla v24.4s, v4.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49c // smmla v28.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a451 // smmla v17.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a499 // smmla v25.4s, v4.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x4e86a40d // smmla v13.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a455 // smmla v21.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49d // smmla v29.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" - ".inst 0x4e87a452 // smmla v18.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a49a // smmla v26.4s, v4.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x4e86a40e // smmla v14.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49e // smmla v30.4s, v4.16b, v6.16b\n" + "ldr q0, [x10, #0x0]\n" + "trn1 v7.2d, v1.2d, v2.2d\n" + "trn1 v3.2d, v3.2d, v4.2d\n" + ".inst 0x4e80a4e8 // smmla v8.4s, v7.16b, v0.16b\n" + "trn1 v2.2d, v5.2d, v6.2d\n" + "ldr q1, [x10, #0x10]\n" + ".inst 0x4e80a470 // smmla v16.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a458 // smmla v24.4s, v2.16b, v0.16b\n" + "ldr q0, [x10, #0x20]\n" + ".inst 0x4e81a4ec // smmla v12.4s, v7.16b, v1.16b\n" + ".inst 0x4e81a474 // smmla v20.4s, v3.16b, v1.16b\n" + ".inst 0x4e81a45c // smmla v28.4s, v2.16b, v1.16b\n" + "ldr q1, [x10, #0x30]\n" + ".inst 0x4e80a4e9 // smmla v9.4s, v7.16b, v0.16b\n" + ".inst 0x4e80a471 // smmla v17.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a459 // smmla v25.4s, v2.16b, v0.16b\n" + "ldr q0, [x10, #0x40]\n" + ".inst 0x4e81a4ed // smmla v13.4s, v7.16b, v1.16b\n" + ".inst 0x4e81a475 // smmla v21.4s, v3.16b, v1.16b\n" + ".inst 0x4e81a45d // smmla v29.4s, v2.16b, v1.16b\n" + "ldr q1, [x10, #0x50]\n" + ".inst 0x4e80a4ea // smmla v10.4s, v7.16b, v0.16b\n" + ".inst 0x4e80a472 // smmla v18.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a45a // smmla v26.4s, v2.16b, v0.16b\n" + "ldr q0, [x10, #0x60]\n" + ".inst 0x4e81a4ee // smmla v14.4s, v7.16b, v1.16b\n" + ".inst 0x4e81a476 // smmla v22.4s, v3.16b, v1.16b\n" + ".inst 0x4e81a45e // smmla v30.4s, v2.16b, v1.16b\n" "ldr q6, [x10, #0x70]\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" + ".inst 0x4e80a4eb // smmla v11.4s, v7.16b, v0.16b\n" "add x10, x10, #0x80\n" - ".inst 0x4e87a453 // smmla v19.4s, v2.16b, v7.16b\n" - ".inst 0x4e87a49b // smmla v27.4s, v4.16b, v7.16b\n" - ".inst 0x4e86a40f // smmla v15.4s, v0.16b, v6.16b\n" - ".inst 0x4e86a457 // smmla v23.4s, v2.16b, v6.16b\n" - ".inst 0x4e86a49f // smmla v31.4s, v4.16b, v6.16b\n" + ".inst 0x4e80a473 // smmla v19.4s, v3.16b, v0.16b\n" + ".inst 0x4e80a45b // smmla v27.4s, v2.16b, v0.16b\n" + ".inst 0x4e86a4ef // smmla v15.4s, v7.16b, v6.16b\n" + ".inst 0x4e86a477 // smmla v23.4s, v3.16b, v6.16b\n" + ".inst 0x4e86a45f // smmla v31.4s, v2.16b, v6.16b\n" "212:" // Height 6: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -3440,7 +3439,6 @@ void a64_hybrid_s8s32_mmla_6x16 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "224:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16.hpp index ebc43425b8d73e8d07209750bd00add57d9388aa..14aba0078855a987632e15e81ed1136be95bd4fe 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef __aarch64__ + #include "../std_transforms_fixed.hpp" #include "../performance_parameters.hpp" @@ -84,7 +84,7 @@ public: case CPUModel::A510: return { 14.81 }; case CPUModel::V1: - return { 48.36 }; + return { 44.54 }; } } @@ -108,5 +108,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16/a55.cpp index b9caf545f1c20d8ebc76f05d3dc7f2a12c445d39..00d063b42648028926ba9f7ed566c09573a2ea59 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16/a55.cpp @@ -78,329 +78,328 @@ void a64_hybrid_u8qa_dot_4x16_a55 ( flags |= 0x20; } __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x4\n" "bge 91f\n" "cmp %x[M], #0x2\n" "bgt 61f\n" "beq 31f\n" - "mov x16, %x[col_bias]\n" + "mov x15, %x[col_bias]\n" "movi v11.4s, #0x0\n" "movi v15.16b, #0x1\n" "bic %x[flags], %x[flags], #0x80000000\n" - "ldr x15, [%x[args_ptr], %[offsetof_N]]\n" - "mov x14, %x[output_ptr]\n" - "ldr x13, [%x[args_ptr], %[offsetof_B_ptr]]\n" + "ldr x14, [%x[args_ptr], %[offsetof_N]]\n" + "mov x13, %x[output_ptr]\n" + "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n" "2:" // Height 1: Column loop "movi v16.4s, #0x0\n" "movi v17.4s, #0x0\n" "movi v18.4s, #0x0\n" "movi v19.4s, #0x0\n" "3:" // Height 1: setup done - "mov x12, #0x0\n" + "mov x11, #0x0\n" "4:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr w11, [x20, x12, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr w10, [x20, x11, LSL #0x2]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 5f\n" - "ldr x21, [%x[input_ptr], x12, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x10, [x21, #0x0]\n" - "cbnz x12, 6f\n" + "ldr x20, [%x[input_ptr], x11, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x9, [x20, #0x0]\n" + "cbnz x11, 6f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x10, x10, x20\n" + "add x9, x9, x20\n" "b 6f\n" "5:" // Height 1: setup direct input - "mov x10, %x[input_ptr]\n" + "mov x9, %x[input_ptr]\n" "6:" // Height 1: input setup done - "cmp x11, #0x10\n" + "cmp x10, #0x10\n" "blt 11f\n" - "ldr q0, [x10, #0x0]\n" - "cmp x11, #0x20\n" - "ldr q4, [x13, #0x0]\n" - "ldr q5, [x13, #0x10]\n" - "ldr q6, [x13, #0x20]\n" - "ldr q7, [x13, #0x30]\n" - "ldr q8, [x13, #0x40]\n" - "ldr q9, [x13, #0x50]\n" - "ldr q10, [x13, #0x60]\n" + "ldr q0, [x9, #0x0]\n" + "cmp x10, #0x20\n" + "ldr q4, [x12, #0x0]\n" + "ldr q5, [x12, #0x10]\n" + "ldr q6, [x12, #0x20]\n" + "ldr q7, [x12, #0x30]\n" + "ldr q8, [x12, #0x40]\n" + "ldr q9, [x12, #0x50]\n" + "ldr q10, [x12, #0x60]\n" "blt 9f\n" "7:" // Height 1: Multiply loop: Main loop head ".inst 0x6f80e090 // udot v16.4s, v4.16b, v0.4b[0]\n" - "ldr d4, [x13, #0x70]\n" - "ldr x9, [x13, #0x78]\n" + "ldr d21, [x12, #0x70]\n" + "ldr x20, [x12, #0x78]\n" ".inst 0x6f80e0b1 // udot v17.4s, v5.16b, v0.4b[0]\n" - "ldr d5, [x13, #0x80]\n" + "ldr d20, [x12, #0x80]\n" ".inst 0x6f80e0d2 // udot v18.4s, v6.16b, v0.4b[0]\n" - "ldr d6, [x13, #0x90]\n" + "ldr d26, [x12, #0x90]\n" ".inst 0x6f80e0f3 // udot v19.4s, v7.16b, v0.4b[0]\n" - "ldr d7, [x13, #0xa0]\n" - "mov v4.d[1], x9\n" - "ldr x28, [x13, #0x88]\n" + "ldr d25, [x12, #0xa0]\n" + "mov v21.d[1], x20\n" + "ldr x20, [x12, #0x88]\n" ".inst 0x6fa0e110 // udot v16.4s, v8.16b, v0.4b[1]\n" - "ldr d8, [x13, #0xb0]\n" + "ldr d24, [x12, #0xb0]\n" ".inst 0x6fa0e131 // udot v17.4s, v9.16b, v0.4b[1]\n" - "ldr d9, [x13, #0xc0]\n" + "ldr d23, [x12, #0xc0]\n" ".inst 0x6fa0e152 // udot v18.4s, v10.16b, v0.4b[1]\n" - "ldr d10, [x13, #0xd0]\n" - ".inst 0x6fa0e093 // udot v19.4s, v4.16b, v0.4b[1]\n" - "ldr d4, [x13, #0xe0]\n" - "mov v5.d[1], x28\n" - "ldr x27, [x13, #0x98]\n" - "mov v6.d[1], x27\n" - "ldr x26, [x13, #0xa8]\n" - "mov v7.d[1], x26\n" - "ldr x25, [x13, #0xb8]\n" - "mov v8.d[1], x25\n" - "ldr x24, [x13, #0xc8]\n" - ".inst 0x6f80e8b0 // udot v16.4s, v5.16b, v0.4b[2]\n" - "ldr d5, [x13, #0xf0]\n" - ".inst 0x6f80e8d1 // udot v17.4s, v6.16b, v0.4b[2]\n" - "ldr x20, [x13, #0xd8]\n" - ".inst 0x6f80e8f2 // udot v18.4s, v7.16b, v0.4b[2]\n" - "ldr x9, [x13, #0xe8]\n" - ".inst 0x6f80e913 // udot v19.4s, v8.16b, v0.4b[2]\n" - "ldr x28, [x13, #0xf8]\n" - "mov v9.d[1], x24\n" - "mov v10.d[1], x20\n" - "add x10, x10, #0x10\n" - "mov v4.d[1], x9\n" - "add x13, x13, #0x100\n" - "mov v5.d[1], x28\n" - ".inst 0x6fa0e930 // udot v16.4s, v9.16b, v0.4b[3]\n" - ".inst 0x6fa0e951 // udot v17.4s, v10.16b, v0.4b[3]\n" - ".inst 0x6fa0e892 // udot v18.4s, v4.16b, v0.4b[3]\n" - ".inst 0x6fa0e8b3 // udot v19.4s, v5.16b, v0.4b[3]\n" + "ldr d22, [x12, #0xd0]\n" + ".inst 0x6fa0e2b3 // udot v19.4s, v21.16b, v0.4b[1]\n" + "ldr d21, [x12, #0xe0]\n" + "mov v20.d[1], x20\n" + "ldr x20, [x12, #0x98]\n" + "mov v26.d[1], x20\n" + "ldr x20, [x12, #0xa8]\n" + "mov v25.d[1], x20\n" + "ldr x20, [x12, #0xb8]\n" + "mov v24.d[1], x20\n" + "ldr x23, [x12, #0xc8]\n" + ".inst 0x6f80ea90 // udot v16.4s, v20.16b, v0.4b[2]\n" + "ldr d20, [x12, #0xf0]\n" + ".inst 0x6f80eb51 // udot v17.4s, v26.16b, v0.4b[2]\n" + "ldr x22, [x12, #0xd8]\n" + ".inst 0x6f80eb32 // udot v18.4s, v25.16b, v0.4b[2]\n" + "ldr x21, [x12, #0xe8]\n" + ".inst 0x6f80eb13 // udot v19.4s, v24.16b, v0.4b[2]\n" + "ldr x20, [x12, #0xf8]\n" + "mov v23.d[1], x23\n" + "mov v22.d[1], x22\n" + "add x9, x9, #0x10\n" + "mov v21.d[1], x21\n" + "add x12, x12, #0x100\n" + "mov v20.d[1], x20\n" + ".inst 0x6fa0eaf0 // udot v16.4s, v23.16b, v0.4b[3]\n" + ".inst 0x6fa0ead1 // udot v17.4s, v22.16b, v0.4b[3]\n" + ".inst 0x6fa0eab2 // udot v18.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa0ea93 // udot v19.4s, v20.16b, v0.4b[3]\n" "tbnz %x[flags], #31, 8f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" "8:" // Height 1: Multiply loop: unique 1: skip row sum - "ldr q0, [x10, #0x0]\n" - "sub x11, x11, #0x10\n" - "ldr q4, [x13, #0x0]\n" - "cmp x11, #0x20\n" - "ldr q5, [x13, #0x10]\n" - "ldr q6, [x13, #0x20]\n" - "ldr q7, [x13, #0x30]\n" - "ldr q8, [x13, #0x40]\n" - "ldr q9, [x13, #0x50]\n" - "ldr q10, [x13, #0x60]\n" - "prfm pldl1keep, [x10, #0x80]\n" + "ldr q0, [x9, #0x0]\n" + "sub x10, x10, #0x10\n" + "ldr q4, [x12, #0x0]\n" + "cmp x10, #0x20\n" + "ldr q5, [x12, #0x10]\n" + "ldr q6, [x12, #0x20]\n" + "ldr q7, [x12, #0x30]\n" + "ldr q8, [x12, #0x40]\n" + "ldr q9, [x12, #0x50]\n" + "ldr q10, [x12, #0x60]\n" + "prfm pldl1keep, [x9, #0x80]\n" "bge 7b\n" "9:" // Height 1: Multiply loop: Single iteration only ".inst 0x6f80e090 // udot v16.4s, v4.16b, v0.4b[0]\n" - "ldr q4, [x13, #0x70]\n" + "ldr q21, [x12, #0x70]\n" ".inst 0x6f80e0b1 // udot v17.4s, v5.16b, v0.4b[0]\n" - "ldr q5, [x13, #0x80]\n" + "ldr q20, [x12, #0x80]\n" ".inst 0x6f80e0d2 // udot v18.4s, v6.16b, v0.4b[0]\n" - "ldr q6, [x13, #0x90]\n" + "ldr q26, [x12, #0x90]\n" ".inst 0x6f80e0f3 // udot v19.4s, v7.16b, v0.4b[0]\n" - "ldr q7, [x13, #0xa0]\n" + "ldr q25, [x12, #0xa0]\n" ".inst 0x6fa0e110 // udot v16.4s, v8.16b, v0.4b[1]\n" - "ldr q8, [x13, #0xb0]\n" + "ldr q24, [x12, #0xb0]\n" ".inst 0x6fa0e131 // udot v17.4s, v9.16b, v0.4b[1]\n" - "ldr q9, [x13, #0xc0]\n" + "ldr q23, [x12, #0xc0]\n" ".inst 0x6fa0e152 // udot v18.4s, v10.16b, v0.4b[1]\n" - "ldr q10, [x13, #0xd0]\n" - ".inst 0x6fa0e093 // udot v19.4s, v4.16b, v0.4b[1]\n" - "ldr q4, [x13, #0xe0]\n" - ".inst 0x6f80e8b0 // udot v16.4s, v5.16b, v0.4b[2]\n" - "ldr q5, [x13, #0xf0]\n" - ".inst 0x6f80e8d1 // udot v17.4s, v6.16b, v0.4b[2]\n" - "sub x11, x11, #0x10\n" - ".inst 0x6f80e8f2 // udot v18.4s, v7.16b, v0.4b[2]\n" - "add x10, x10, #0x10\n" - ".inst 0x6f80e913 // udot v19.4s, v8.16b, v0.4b[2]\n" - "add x13, x13, #0x100\n" - ".inst 0x6fa0e930 // udot v16.4s, v9.16b, v0.4b[3]\n" - ".inst 0x6fa0e951 // udot v17.4s, v10.16b, v0.4b[3]\n" - ".inst 0x6fa0e892 // udot v18.4s, v4.16b, v0.4b[3]\n" - ".inst 0x6fa0e8b3 // udot v19.4s, v5.16b, v0.4b[3]\n" + "ldr q22, [x12, #0xd0]\n" + ".inst 0x6fa0e2b3 // udot v19.4s, v21.16b, v0.4b[1]\n" + "ldr q21, [x12, #0xe0]\n" + ".inst 0x6f80ea90 // udot v16.4s, v20.16b, v0.4b[2]\n" + "ldr q20, [x12, #0xf0]\n" + ".inst 0x6f80eb51 // udot v17.4s, v26.16b, v0.4b[2]\n" + "sub x10, x10, #0x10\n" + ".inst 0x6f80eb32 // udot v18.4s, v25.16b, v0.4b[2]\n" + "add x9, x9, #0x10\n" + ".inst 0x6f80eb13 // udot v19.4s, v24.16b, v0.4b[2]\n" + "add x12, x12, #0x100\n" + ".inst 0x6fa0eaf0 // udot v16.4s, v23.16b, v0.4b[3]\n" + ".inst 0x6fa0ead1 // udot v17.4s, v22.16b, v0.4b[3]\n" + ".inst 0x6fa0eab2 // udot v18.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa0ea93 // udot v19.4s, v20.16b, v0.4b[3]\n" "tbnz %x[flags], #31, 10f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" "10:" // Height 1: Multiply loop: unique 2: skip row sum - "prfm pldl1keep, [x10, #0x80]\n" + "prfm pldl1keep, [x9, #0x80]\n" "11:" // Height 1: Multiply loop: Main loop skip - "cbz x11, 18f\n" - "cmp x11, #0x4\n" + "cbz x10, 18f\n" + "cmp x10, #0x4\n" "blt 14f\n" "12:" // Height 1: Multiply loop: Odd block loop - "ldr s0, [x10], #0x4\n" + "ldr s0, [x9], #0x4\n" "tbnz %x[flags], #31, 13f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" "13:" // Height 1: Multiply loop: unique 3: skip row sum - "ldr q6, [x13, #0x0]\n" - "sub x11, x11, #0x4\n" - "ldr q7, [x13, #0x10]\n" - "cmp x11, #0x4\n" - "ldr q8, [x13, #0x20]\n" - ".inst 0x6f80e0d0 // udot v16.4s, v6.16b, v0.4b[0]\n" - "ldr q9, [x13, #0x30]\n" - ".inst 0x6f80e0f1 // udot v17.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f80e112 // udot v18.4s, v8.16b, v0.4b[0]\n" - "add x13, x13, #0x40\n" - ".inst 0x6f80e133 // udot v19.4s, v9.16b, v0.4b[0]\n" + "ldr q20, [x12, #0x0]\n" + "sub x10, x10, #0x4\n" + "ldr q22, [x12, #0x10]\n" + "cmp x10, #0x4\n" + "ldr q21, [x12, #0x20]\n" + ".inst 0x6f80e290 // udot v16.4s, v20.16b, v0.4b[0]\n" + "ldr q20, [x12, #0x30]\n" + ".inst 0x6f80e2d1 // udot v17.4s, v22.16b, v0.4b[0]\n" + ".inst 0x6f80e2b2 // udot v18.4s, v21.16b, v0.4b[0]\n" + "add x12, x12, #0x40\n" + ".inst 0x6f80e293 // udot v19.4s, v20.16b, v0.4b[0]\n" "bge 12b\n" "14:" // Height 1: Multiply loop: Skip odd blocks - "cbz x11, 18f\n" - "tbz x11, #1, 15f\n" - "ldr h0, [x10], #0x2\n" - "tbz x11, #0, 16f\n" - "ld1 { v0.b }[2], [x10]\n" + "cbz x10, 18f\n" + "tbz x10, #1, 15f\n" + "ldr h0, [x9], #0x2\n" + "tbz x10, #0, 16f\n" + "ld1 { v0.b }[2], [x9]\n" "b 16f\n" "15:" // Height 1: Multiply loop: Ragged operand read: partial_1_0 - "ldr b0, [x10, #0x0]\n" + "ldr b0, [x9, #0x0]\n" "16:" // Height 1: Multiply loop: Ragged operand read: Done "tbnz %x[flags], #31, 17f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" "17:" // Height 1: Multiply loop: unique 4: skip row sum - "ldr q10, [x13, #0x0]\n" - ".inst 0x6f80e150 // udot v16.4s, v10.16b, v0.4b[0]\n" - "ldr q4, [x13, #0x10]\n" - ".inst 0x6f80e091 // udot v17.4s, v4.16b, v0.4b[0]\n" - "ldr q5, [x13, #0x20]\n" - ".inst 0x6f80e0b2 // udot v18.4s, v5.16b, v0.4b[0]\n" - "ldr q6, [x13, #0x30]\n" - ".inst 0x6f80e0d3 // udot v19.4s, v6.16b, v0.4b[0]\n" - "add x13, x13, #0x40\n" + "ldr q20, [x12, #0x0]\n" + ".inst 0x6f80e290 // udot v16.4s, v20.16b, v0.4b[0]\n" + "ldr q20, [x12, #0x10]\n" + ".inst 0x6f80e291 // udot v17.4s, v20.16b, v0.4b[0]\n" + "ldr q20, [x12, #0x20]\n" + ".inst 0x6f80e292 // udot v18.4s, v20.16b, v0.4b[0]\n" + "ldr q20, [x12, #0x30]\n" + ".inst 0x6f80e293 // udot v19.4s, v20.16b, v0.4b[0]\n" + "add x12, x12, #0x40\n" "18:" // Height 1: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" - "add x12, x12, #0x1\n" - "cmp x12, x20\n" + "add x11, x11, #0x1\n" + "cmp x11, x20\n" "bne 4b\n" - "prfm pstl1keep, [x14, #0x0]\n" + "prfm pstl1keep, [x13, #0x0]\n" "tbnz %x[flags], #31, 19f\n" "addp v11.4s, v11.4s, v11.4s\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1r { v1.4s }, [x23]\n" - "neg v1.4s, v1.4s\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1r { v20.4s }, [x20]\n" + "neg v20.4s, v20.4s\n" "addp v11.4s, v11.4s, v11.4s\n" - "mul v11.4s, v11.4s, v1.4s\n" + "mul v11.4s, v11.4s, v20.4s\n" "19:" // Height 1: skip row sum fixup - "ldr q0, [x16, #0x0]\n" + "ldr q23, [x15, #0x0]\n" "add v16.4s, v16.4s, v11.4s\n" - "ldr q1, [x16, #0x10]\n" + "ldr q22, [x15, #0x10]\n" "add v17.4s, v17.4s, v11.4s\n" - "ldr q2, [x16, #0x20]\n" + "ldr q21, [x15, #0x20]\n" "add v18.4s, v18.4s, v11.4s\n" - "ldr q3, [x16, #0x30]\n" + "ldr q20, [x15, #0x30]\n" "add v19.4s, v19.4s, v11.4s\n" - "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" - "add x23, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x23]\n" + "add v16.4s, v16.4s, v23.4s\n" + "add v17.4s, v17.4s, v22.4s\n" + "add v18.4s, v18.4s, v21.4s\n" + "add v19.4s, v19.4s, v20.4s\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v20.4s }, [x20]\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x23]\n" - "sqrdmulh v16.4s, v16.4s, v4.4s\n" - "sqrdmulh v17.4s, v17.4s, v4.4s\n" - "sqrdmulh v18.4s, v18.4s, v4.4s\n" - "sqrdmulh v19.4s, v19.4s, v4.4s\n" - "add x16, x16, #0x40\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "sqrdmulh v16.4s, v16.4s, v20.4s\n" + "sqrdmulh v17.4s, v17.4s, v20.4s\n" + "sqrdmulh v18.4s, v18.4s, v20.4s\n" + "sqrdmulh v19.4s, v19.4s, v20.4s\n" + "add x15, x15, #0x40\n" "tbz %x[flags], #5, 20f\n" - "and v4.16b, v16.16b, v0.16b\n" - "and v5.16b, v17.16b, v0.16b\n" - "and v6.16b, v18.16b, v0.16b\n" - "and v7.16b, v19.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" + "and v23.16b, v16.16b, v0.16b\n" + "and v22.16b, v17.16b, v0.16b\n" + "and v21.16b, v18.16b, v0.16b\n" + "and v20.16b, v19.16b, v0.16b\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "sshr v21.4s, v21.4s, #0x1f\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v23.4s\n" + "sqadd v17.4s, v17.4s, v22.4s\n" + "sqadd v18.4s, v18.4s, v21.4s\n" + "sqadd v19.4s, v19.4s, v20.4s\n" "20:" // Height 1: no shift correction "srshl v16.4s, v16.4s, v0.4s\n" "srshl v17.4s, v17.4s, v0.4s\n" "srshl v18.4s, v18.4s, v0.4s\n" "srshl v19.4s, v19.4s, v0.4s\n" - "add x23, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x23]\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x23]\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "add x23, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x23]\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v20.4s }, [x20]\n" + "add v16.4s, v16.4s, v20.4s\n" + "add v17.4s, v17.4s, v20.4s\n" + "add v18.4s, v18.4s, v20.4s\n" + "add v19.4s, v19.4s, v20.4s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v20.4s }, [x20]\n" + "smin v16.4s, v16.4s, v20.4s\n" + "smin v17.4s, v17.4s, v20.4s\n" + "smin v18.4s, v18.4s, v20.4s\n" + "smin v19.4s, v19.4s, v20.4s\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v20.4s }, [x20]\n" + "smax v16.4s, v16.4s, v20.4s\n" + "smax v17.4s, v17.4s, v20.4s\n" + "smax v18.4s, v18.4s, v20.4s\n" + "smax v19.4s, v19.4s, v20.4s\n" "uzp1 v16.8h, v16.8h, v17.8h\n" "uzp1 v17.8h, v18.8h, v19.8h\n" - "cmp x15, #0x10\n" + "cmp x14, #0x10\n" "uzp1 v16.16b, v16.16b, v17.16b\n" "bge 29f\n" - "tbz x15, #3, 24f\n" - "str d16, [x14], #0x8\n" - "tbz x15, #2, 22f\n" - "st1 { v16.s }[2], [x14], #0x4\n" - "tbz x15, #1, 21f\n" - "st1 { v16.h }[6], [x14], #0x2\n" - "tbz x15, #0, 28f\n" - "st1 { v16.b }[14], [x14]\n" + "tbz x14, #3, 24f\n" + "str d16, [x13], #0x8\n" + "tbz x14, #2, 22f\n" + "st1 { v16.s }[2], [x13], #0x4\n" + "tbz x14, #1, 21f\n" + "st1 { v16.h }[6], [x13], #0x2\n" + "tbz x14, #0, 28f\n" + "st1 { v16.b }[14], [x13]\n" "b 28f\n" "21:" // Height 1: Partial direct writeback: partial_1_12 - "tbz x15, #0, 28f\n" - "st1 { v16.b }[12], [x14]\n" + "tbz x14, #0, 28f\n" + "st1 { v16.b }[12], [x13]\n" "b 28f\n" "22:" // Height 1: Partial direct writeback: partial_2_8 - "tbz x15, #1, 23f\n" - "st1 { v16.h }[4], [x14], #0x2\n" - "tbz x15, #0, 28f\n" - "st1 { v16.b }[10], [x14]\n" + "tbz x14, #1, 23f\n" + "st1 { v16.h }[4], [x13], #0x2\n" + "tbz x14, #0, 28f\n" + "st1 { v16.b }[10], [x13]\n" "b 28f\n" "23:" // Height 1: Partial direct writeback: partial_1_8 - "tbz x15, #0, 28f\n" - "st1 { v16.b }[8], [x14]\n" + "tbz x14, #0, 28f\n" + "st1 { v16.b }[8], [x13]\n" "b 28f\n" "24:" // Height 1: Partial direct writeback: partial_4_0 - "tbz x15, #2, 26f\n" - "str s16, [x14], #0x4\n" - "tbz x15, #1, 25f\n" - "st1 { v16.h }[2], [x14], #0x2\n" - "tbz x15, #0, 28f\n" - "st1 { v16.b }[6], [x14]\n" + "tbz x14, #2, 26f\n" + "str s16, [x13], #0x4\n" + "tbz x14, #1, 25f\n" + "st1 { v16.h }[2], [x13], #0x2\n" + "tbz x14, #0, 28f\n" + "st1 { v16.b }[6], [x13]\n" "b 28f\n" "25:" // Height 1: Partial direct writeback: partial_1_4 - "tbz x15, #0, 28f\n" - "st1 { v16.b }[4], [x14]\n" + "tbz x14, #0, 28f\n" + "st1 { v16.b }[4], [x13]\n" "b 28f\n" "26:" // Height 1: Partial direct writeback: partial_2_0 - "tbz x15, #1, 27f\n" - "str h16, [x14], #0x2\n" - "tbz x15, #0, 28f\n" - "st1 { v16.b }[2], [x14]\n" + "tbz x14, #1, 27f\n" + "str h16, [x13], #0x2\n" + "tbz x14, #0, 28f\n" + "st1 { v16.b }[2], [x13]\n" "b 28f\n" "27:" // Height 1: Partial direct writeback: partial_1_0 - "str b16, [x14, #0x0]\n" + "str b16, [x13, #0x0]\n" "28:" // Height 1: Partial direct writeback: Done "b 30f\n" "29:" // Height 1: Full writeback - "str q16, [x14, #0x0]\n" - "add x14, x14, #0x10\n" + "str q16, [x13, #0x0]\n" + "add x13, x13, #0x10\n" "30:" // Height 1: Writeback done - "subs x15, x15, #0x10\n" + "subs x14, x14, #0x10\n" "bgt 2b\n" "b 122f\n" "31:" // Height 2 - "mov x16, %x[col_bias]\n" + "mov x15, %x[col_bias]\n" "movi v11.4s, #0x0\n" "movi v12.4s, #0x0\n" "bic %x[flags], %x[flags], #0x80000000\n" "movi v15.16b, #0x1\n" - "ldr x15, [%x[args_ptr], %[offsetof_N]]\n" - "ldr x13, [%x[args_ptr], %[offsetof_B_ptr]]\n" - "mov x14, %x[output_ptr]\n" + "ldr x14, [%x[args_ptr], %[offsetof_N]]\n" + "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n" + "mov x13, %x[output_ptr]\n" "32:" // Height 2: Column loop "movi v16.4s, #0x0\n" "movi v17.4s, #0x0\n" @@ -411,307 +410,307 @@ void a64_hybrid_u8qa_dot_4x16_a55 ( "movi v22.4s, #0x0\n" "movi v23.4s, #0x0\n" "33:" // Height 2: setup done - "mov x12, #0x0\n" + "mov x11, #0x0\n" "34:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr w11, [x20, x12, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr w10, [x20, x11, LSL #0x2]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 35f\n" - "ldr x21, [%x[input_ptr], x12, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x10, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "cbnz x12, 36f\n" + "ldr x20, [%x[input_ptr], x11, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x9, [x20, #0x0]\n" + "ldr x28, [x20, #0x8]\n" + "cbnz x11, 36f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x10, x10, x20\n" - "add x23, x23, x20\n" + "add x9, x9, x20\n" + "add x28, x28, x20\n" "b 36f\n" "35:" // Height 2: setup direct input - "mov x10, %x[input_ptr]\n" - "add x23, x10, x20\n" + "mov x9, %x[input_ptr]\n" + "add x28, x9, x21\n" "36:" // Height 2: input setup done - "cmp x11, #0x10\n" + "cmp x10, #0x10\n" "blt 41f\n" - "ldr q0, [x10, #0x0]\n" - "cmp x11, #0x20\n" - "ldr q1, [x23, #0x0]\n" - "ldr q4, [x13, #0x0]\n" - "ldr q5, [x13, #0x10]\n" - "ldr q6, [x13, #0x20]\n" - "ldr q7, [x13, #0x30]\n" - "ldr q8, [x13, #0x40]\n" - "ldr q9, [x13, #0x50]\n" - "ldr q10, [x13, #0x60]\n" + "ldr q0, [x9, #0x0]\n" + "cmp x10, #0x20\n" + "ldr q1, [x28, #0x0]\n" + "ldr q4, [x12, #0x0]\n" + "ldr q5, [x12, #0x10]\n" + "ldr q6, [x12, #0x20]\n" + "ldr q7, [x12, #0x30]\n" + "ldr q8, [x12, #0x40]\n" + "ldr q9, [x12, #0x50]\n" + "ldr q10, [x12, #0x60]\n" "blt 39f\n" "37:" // Height 2: Multiply loop: Main loop head ".inst 0x6f80e090 // udot v16.4s, v4.16b, v0.4b[0]\n" - "ldr x9, [x13, #0x78]\n" + "ldr x20, [x12, #0x78]\n" ".inst 0x6f81e094 // udot v20.4s, v4.16b, v1.4b[0]\n" - "ldr d4, [x13, #0x70]\n" + "ldr d25, [x12, #0x70]\n" ".inst 0x6f80e0b1 // udot v17.4s, v5.16b, v0.4b[0]\n" - "mov v4.d[1], x9\n" + "mov v25.d[1], x20\n" ".inst 0x6f81e0b5 // udot v21.4s, v5.16b, v1.4b[0]\n" - "ldr d5, [x13, #0x80]\n" + "ldr d24, [x12, #0x80]\n" ".inst 0x6f80e0d2 // udot v18.4s, v6.16b, v0.4b[0]\n" - "ldr x28, [x13, #0x88]\n" + "ldr x23, [x12, #0x88]\n" ".inst 0x6f81e0d6 // udot v22.4s, v6.16b, v1.4b[0]\n" - "ldr d6, [x13, #0x90]\n" + "ldr d30, [x12, #0x90]\n" ".inst 0x6f80e0f3 // udot v19.4s, v7.16b, v0.4b[0]\n" - "ldr x27, [x13, #0x98]\n" + "ldr x22, [x12, #0x98]\n" ".inst 0x6f81e0f7 // udot v23.4s, v7.16b, v1.4b[0]\n" - "ldr d7, [x13, #0xa0]\n" - "ldr x26, [x13, #0xa8]\n" + "ldr d29, [x12, #0xa0]\n" + "ldr x21, [x12, #0xa8]\n" ".inst 0x6fa0e110 // udot v16.4s, v8.16b, v0.4b[1]\n" ".inst 0x6fa1e114 // udot v20.4s, v8.16b, v1.4b[1]\n" - "ldr d8, [x13, #0xb0]\n" - "ldr x25, [x13, #0xb8]\n" + "ldr d28, [x12, #0xb0]\n" + "ldr x20, [x12, #0xb8]\n" ".inst 0x6fa0e131 // udot v17.4s, v9.16b, v0.4b[1]\n" ".inst 0x6fa1e135 // udot v21.4s, v9.16b, v1.4b[1]\n" - "ldr d9, [x13, #0xc0]\n" + "ldr d27, [x12, #0xc0]\n" ".inst 0x6fa0e152 // udot v18.4s, v10.16b, v0.4b[1]\n" - "mov v5.d[1], x28\n" + "mov v24.d[1], x23\n" ".inst 0x6fa1e156 // udot v22.4s, v10.16b, v1.4b[1]\n" - "ldr d10, [x13, #0xd0]\n" - ".inst 0x6fa0e093 // udot v19.4s, v4.16b, v0.4b[1]\n" - "mov v6.d[1], x27\n" - ".inst 0x6fa1e097 // udot v23.4s, v4.16b, v1.4b[1]\n" - "ldr d4, [x13, #0xe0]\n" - "mov v7.d[1], x26\n" - "ldr x24, [x13, #0xc8]\n" - "mov v8.d[1], x25\n" - "ldr x20, [x13, #0xd8]\n" - "ldr x9, [x13, #0xe8]\n" - ".inst 0x6f80e8b0 // udot v16.4s, v5.16b, v0.4b[2]\n" - ".inst 0x6f81e8b4 // udot v20.4s, v5.16b, v1.4b[2]\n" - "ldr d5, [x13, #0xf0]\n" - "ldr x28, [x13, #0xf8]\n" - ".inst 0x6f80e8d1 // udot v17.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8d5 // udot v21.4s, v6.16b, v1.4b[2]\n" - "mov v9.d[1], x24\n" - ".inst 0x6f80e8f2 // udot v18.4s, v7.16b, v0.4b[2]\n" - "mov v10.d[1], x20\n" - ".inst 0x6f81e8f6 // udot v22.4s, v7.16b, v1.4b[2]\n" - "mov v4.d[1], x9\n" - ".inst 0x6f80e913 // udot v19.4s, v8.16b, v0.4b[2]\n" - "mov v5.d[1], x28\n" - ".inst 0x6f81e917 // udot v23.4s, v8.16b, v1.4b[2]\n" - "add x10, x10, #0x10\n" - "add x23, x23, #0x10\n" - "add x13, x13, #0x100\n" - ".inst 0x6fa0e930 // udot v16.4s, v9.16b, v0.4b[3]\n" - ".inst 0x6fa1e934 // udot v20.4s, v9.16b, v1.4b[3]\n" - ".inst 0x6fa0e951 // udot v17.4s, v10.16b, v0.4b[3]\n" - ".inst 0x6fa1e955 // udot v21.4s, v10.16b, v1.4b[3]\n" - ".inst 0x6fa0e892 // udot v18.4s, v4.16b, v0.4b[3]\n" - ".inst 0x6fa1e896 // udot v22.4s, v4.16b, v1.4b[3]\n" - ".inst 0x6fa0e8b3 // udot v19.4s, v5.16b, v0.4b[3]\n" - ".inst 0x6fa1e8b7 // udot v23.4s, v5.16b, v1.4b[3]\n" + "ldr d26, [x12, #0xd0]\n" + ".inst 0x6fa0e333 // udot v19.4s, v25.16b, v0.4b[1]\n" + "mov v30.d[1], x22\n" + ".inst 0x6fa1e337 // udot v23.4s, v25.16b, v1.4b[1]\n" + "ldr d25, [x12, #0xe0]\n" + "mov v29.d[1], x21\n" + "ldr x23, [x12, #0xc8]\n" + "mov v28.d[1], x20\n" + "ldr x22, [x12, #0xd8]\n" + "ldr x21, [x12, #0xe8]\n" + ".inst 0x6f80eb10 // udot v16.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f81eb14 // udot v20.4s, v24.16b, v1.4b[2]\n" + "ldr d24, [x12, #0xf0]\n" + "ldr x20, [x12, #0xf8]\n" + ".inst 0x6f80ebd1 // udot v17.4s, v30.16b, v0.4b[2]\n" + ".inst 0x6f81ebd5 // udot v21.4s, v30.16b, v1.4b[2]\n" + "mov v27.d[1], x23\n" + ".inst 0x6f80ebb2 // udot v18.4s, v29.16b, v0.4b[2]\n" + "mov v26.d[1], x22\n" + ".inst 0x6f81ebb6 // udot v22.4s, v29.16b, v1.4b[2]\n" + "mov v25.d[1], x21\n" + ".inst 0x6f80eb93 // udot v19.4s, v28.16b, v0.4b[2]\n" + "mov v24.d[1], x20\n" + ".inst 0x6f81eb97 // udot v23.4s, v28.16b, v1.4b[2]\n" + "add x9, x9, #0x10\n" + "add x28, x28, #0x10\n" + "add x12, x12, #0x100\n" + ".inst 0x6fa0eb70 // udot v16.4s, v27.16b, v0.4b[3]\n" + ".inst 0x6fa1eb74 // udot v20.4s, v27.16b, v1.4b[3]\n" + ".inst 0x6fa0eb51 // udot v17.4s, v26.16b, v0.4b[3]\n" + ".inst 0x6fa1eb55 // udot v21.4s, v26.16b, v1.4b[3]\n" + ".inst 0x6fa0eb32 // udot v18.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa1eb36 // udot v22.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa0eb13 // udot v19.4s, v24.16b, v0.4b[3]\n" + ".inst 0x6fa1eb17 // udot v23.4s, v24.16b, v1.4b[3]\n" "tbnz %x[flags], #31, 38f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n" "38:" // Height 2: Multiply loop: unique 5: skip row sum - "ldr q0, [x10, #0x0]\n" - "sub x11, x11, #0x10\n" - "ldr q1, [x23, #0x0]\n" - "cmp x11, #0x20\n" - "ldr q4, [x13, #0x0]\n" - "ldr q5, [x13, #0x10]\n" - "ldr q6, [x13, #0x20]\n" - "ldr q7, [x13, #0x30]\n" - "ldr q8, [x13, #0x40]\n" - "ldr q9, [x13, #0x50]\n" - "ldr q10, [x13, #0x60]\n" - "prfm pldl1keep, [x10, #0x80]\n" - "prfm pldl1keep, [x23, #0x80]\n" + "ldr q0, [x9, #0x0]\n" + "sub x10, x10, #0x10\n" + "ldr q1, [x28, #0x0]\n" + "cmp x10, #0x20\n" + "ldr q4, [x12, #0x0]\n" + "ldr q5, [x12, #0x10]\n" + "ldr q6, [x12, #0x20]\n" + "ldr q7, [x12, #0x30]\n" + "ldr q8, [x12, #0x40]\n" + "ldr q9, [x12, #0x50]\n" + "ldr q10, [x12, #0x60]\n" + "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x28, #0x80]\n" "bge 37b\n" "39:" // Height 2: Multiply loop: Single iteration only ".inst 0x6f80e090 // udot v16.4s, v4.16b, v0.4b[0]\n" - "sub x11, x11, #0x10\n" + "sub x10, x10, #0x10\n" ".inst 0x6f81e094 // udot v20.4s, v4.16b, v1.4b[0]\n" - "ldr q4, [x13, #0x70]\n" + "ldr q25, [x12, #0x70]\n" ".inst 0x6f80e0b1 // udot v17.4s, v5.16b, v0.4b[0]\n" - "add x10, x10, #0x10\n" + "add x9, x9, #0x10\n" ".inst 0x6f81e0b5 // udot v21.4s, v5.16b, v1.4b[0]\n" - "ldr q5, [x13, #0x80]\n" + "ldr q24, [x12, #0x80]\n" ".inst 0x6f80e0d2 // udot v18.4s, v6.16b, v0.4b[0]\n" - "add x23, x23, #0x10\n" + "add x28, x28, #0x10\n" ".inst 0x6f81e0d6 // udot v22.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x13, #0x90]\n" + "ldr q30, [x12, #0x90]\n" ".inst 0x6f80e0f3 // udot v19.4s, v7.16b, v0.4b[0]\n" ".inst 0x6f81e0f7 // udot v23.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x13, #0xa0]\n" + "ldr q29, [x12, #0xa0]\n" ".inst 0x6fa0e110 // udot v16.4s, v8.16b, v0.4b[1]\n" ".inst 0x6fa1e114 // udot v20.4s, v8.16b, v1.4b[1]\n" - "ldr q8, [x13, #0xb0]\n" + "ldr q28, [x12, #0xb0]\n" ".inst 0x6fa0e131 // udot v17.4s, v9.16b, v0.4b[1]\n" ".inst 0x6fa1e135 // udot v21.4s, v9.16b, v1.4b[1]\n" - "ldr q9, [x13, #0xc0]\n" + "ldr q27, [x12, #0xc0]\n" ".inst 0x6fa0e152 // udot v18.4s, v10.16b, v0.4b[1]\n" ".inst 0x6fa1e156 // udot v22.4s, v10.16b, v1.4b[1]\n" - "ldr q10, [x13, #0xd0]\n" - ".inst 0x6fa0e093 // udot v19.4s, v4.16b, v0.4b[1]\n" - ".inst 0x6fa1e097 // udot v23.4s, v4.16b, v1.4b[1]\n" - "ldr q4, [x13, #0xe0]\n" - ".inst 0x6f80e8b0 // udot v16.4s, v5.16b, v0.4b[2]\n" - ".inst 0x6f81e8b4 // udot v20.4s, v5.16b, v1.4b[2]\n" - "ldr q5, [x13, #0xf0]\n" - ".inst 0x6f80e8d1 // udot v17.4s, v6.16b, v0.4b[2]\n" - "add x13, x13, #0x100\n" - ".inst 0x6f81e8d5 // udot v21.4s, v6.16b, v1.4b[2]\n" - ".inst 0x6f80e8f2 // udot v18.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8f6 // udot v22.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f80e913 // udot v19.4s, v8.16b, v0.4b[2]\n" - ".inst 0x6f81e917 // udot v23.4s, v8.16b, v1.4b[2]\n" - ".inst 0x6fa0e930 // udot v16.4s, v9.16b, v0.4b[3]\n" - ".inst 0x6fa1e934 // udot v20.4s, v9.16b, v1.4b[3]\n" - ".inst 0x6fa0e951 // udot v17.4s, v10.16b, v0.4b[3]\n" - ".inst 0x6fa1e955 // udot v21.4s, v10.16b, v1.4b[3]\n" - ".inst 0x6fa0e892 // udot v18.4s, v4.16b, v0.4b[3]\n" - ".inst 0x6fa1e896 // udot v22.4s, v4.16b, v1.4b[3]\n" - ".inst 0x6fa0e8b3 // udot v19.4s, v5.16b, v0.4b[3]\n" - ".inst 0x6fa1e8b7 // udot v23.4s, v5.16b, v1.4b[3]\n" + "ldr q26, [x12, #0xd0]\n" + ".inst 0x6fa0e333 // udot v19.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa1e337 // udot v23.4s, v25.16b, v1.4b[1]\n" + "ldr q25, [x12, #0xe0]\n" + ".inst 0x6f80eb10 // udot v16.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f81eb14 // udot v20.4s, v24.16b, v1.4b[2]\n" + "ldr q24, [x12, #0xf0]\n" + ".inst 0x6f80ebd1 // udot v17.4s, v30.16b, v0.4b[2]\n" + "add x12, x12, #0x100\n" + ".inst 0x6f81ebd5 // udot v21.4s, v30.16b, v1.4b[2]\n" + ".inst 0x6f80ebb2 // udot v18.4s, v29.16b, v0.4b[2]\n" + ".inst 0x6f81ebb6 // udot v22.4s, v29.16b, v1.4b[2]\n" + ".inst 0x6f80eb93 // udot v19.4s, v28.16b, v0.4b[2]\n" + ".inst 0x6f81eb97 // udot v23.4s, v28.16b, v1.4b[2]\n" + ".inst 0x6fa0eb70 // udot v16.4s, v27.16b, v0.4b[3]\n" + ".inst 0x6fa1eb74 // udot v20.4s, v27.16b, v1.4b[3]\n" + ".inst 0x6fa0eb51 // udot v17.4s, v26.16b, v0.4b[3]\n" + ".inst 0x6fa1eb55 // udot v21.4s, v26.16b, v1.4b[3]\n" + ".inst 0x6fa0eb32 // udot v18.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa1eb36 // udot v22.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa0eb13 // udot v19.4s, v24.16b, v0.4b[3]\n" + ".inst 0x6fa1eb17 // udot v23.4s, v24.16b, v1.4b[3]\n" "tbnz %x[flags], #31, 40f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n" "40:" // Height 2: Multiply loop: unique 6: skip row sum - "prfm pldl1keep, [x10, #0x80]\n" - "prfm pldl1keep, [x23, #0x80]\n" + "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x28, #0x80]\n" "41:" // Height 2: Multiply loop: Main loop skip - "cbz x11, 48f\n" - "cmp x11, #0x4\n" + "cbz x10, 48f\n" + "cmp x10, #0x4\n" "blt 44f\n" "42:" // Height 2: Multiply loop: Odd block loop - "ldr s0, [x10], #0x4\n" - "ldr s1, [x23], #0x4\n" + "ldr s0, [x9], #0x4\n" + "ldr s1, [x28], #0x4\n" "tbnz %x[flags], #31, 43f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n" "43:" // Height 2: Multiply loop: unique 7: skip row sum - "ldr q6, [x13, #0x0]\n" - "sub x11, x11, #0x4\n" - "ldr q7, [x13, #0x10]\n" - "cmp x11, #0x4\n" - "ldr q8, [x13, #0x20]\n" - ".inst 0x6f80e0d0 // udot v16.4s, v6.16b, v0.4b[0]\n" - "ldr q9, [x13, #0x30]\n" - ".inst 0x6f81e0d4 // udot v20.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f80e0f1 // udot v17.4s, v7.16b, v0.4b[0]\n" - "add x13, x13, #0x40\n" - ".inst 0x6f81e0f5 // udot v21.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f80e112 // udot v18.4s, v8.16b, v0.4b[0]\n" - ".inst 0x6f81e116 // udot v22.4s, v8.16b, v1.4b[0]\n" - ".inst 0x6f80e133 // udot v19.4s, v9.16b, v0.4b[0]\n" - ".inst 0x6f81e137 // udot v23.4s, v9.16b, v1.4b[0]\n" + "ldr q27, [x12, #0x0]\n" + "sub x10, x10, #0x4\n" + "ldr q26, [x12, #0x10]\n" + "cmp x10, #0x4\n" + "ldr q25, [x12, #0x20]\n" + ".inst 0x6f80e370 // udot v16.4s, v27.16b, v0.4b[0]\n" + "ldr q24, [x12, #0x30]\n" + ".inst 0x6f81e374 // udot v20.4s, v27.16b, v1.4b[0]\n" + ".inst 0x6f80e351 // udot v17.4s, v26.16b, v0.4b[0]\n" + "add x12, x12, #0x40\n" + ".inst 0x6f81e355 // udot v21.4s, v26.16b, v1.4b[0]\n" + ".inst 0x6f80e332 // udot v18.4s, v25.16b, v0.4b[0]\n" + ".inst 0x6f81e336 // udot v22.4s, v25.16b, v1.4b[0]\n" + ".inst 0x6f80e313 // udot v19.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f81e317 // udot v23.4s, v24.16b, v1.4b[0]\n" "bge 42b\n" "44:" // Height 2: Multiply loop: Skip odd blocks - "cbz x11, 48f\n" - "tbz x11, #1, 45f\n" - "ldr h0, [x10], #0x2\n" - "ldr h1, [x23], #0x2\n" - "tbz x11, #0, 46f\n" - "ld1 { v0.b }[2], [x10]\n" - "ld1 { v1.b }[2], [x23]\n" + "cbz x10, 48f\n" + "tbz x10, #1, 45f\n" + "ldr h0, [x9], #0x2\n" + "ldr h1, [x28], #0x2\n" + "tbz x10, #0, 46f\n" + "ld1 { v0.b }[2], [x9]\n" + "ld1 { v1.b }[2], [x28]\n" "b 46f\n" "45:" // Height 2: Multiply loop: Ragged operand read: partial_1_0 - "ldr b0, [x10, #0x0]\n" - "ldr b1, [x23, #0x0]\n" + "ldr b0, [x9, #0x0]\n" + "ldr b1, [x28, #0x0]\n" "46:" // Height 2: Multiply loop: Ragged operand read: Done "tbnz %x[flags], #31, 47f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n" "47:" // Height 2: Multiply loop: unique 8: skip row sum - "ldr q10, [x13, #0x0]\n" - ".inst 0x6f80e150 // udot v16.4s, v10.16b, v0.4b[0]\n" - "ldr q4, [x13, #0x10]\n" - ".inst 0x6f81e154 // udot v20.4s, v10.16b, v1.4b[0]\n" - "ldr q5, [x13, #0x20]\n" - ".inst 0x6f80e091 // udot v17.4s, v4.16b, v0.4b[0]\n" - "ldr q6, [x13, #0x30]\n" - ".inst 0x6f81e095 // udot v21.4s, v4.16b, v1.4b[0]\n" - ".inst 0x6f80e0b2 // udot v18.4s, v5.16b, v0.4b[0]\n" - "add x13, x13, #0x40\n" - ".inst 0x6f81e0b6 // udot v22.4s, v5.16b, v1.4b[0]\n" - ".inst 0x6f80e0d3 // udot v19.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0d7 // udot v23.4s, v6.16b, v1.4b[0]\n" + "ldr q24, [x12, #0x0]\n" + ".inst 0x6f80e310 // udot v16.4s, v24.16b, v0.4b[0]\n" + "ldr q26, [x12, #0x10]\n" + ".inst 0x6f81e314 // udot v20.4s, v24.16b, v1.4b[0]\n" + "ldr q25, [x12, #0x20]\n" + ".inst 0x6f80e351 // udot v17.4s, v26.16b, v0.4b[0]\n" + "ldr q24, [x12, #0x30]\n" + ".inst 0x6f81e355 // udot v21.4s, v26.16b, v1.4b[0]\n" + ".inst 0x6f80e332 // udot v18.4s, v25.16b, v0.4b[0]\n" + "add x12, x12, #0x40\n" + ".inst 0x6f81e336 // udot v22.4s, v25.16b, v1.4b[0]\n" + ".inst 0x6f80e313 // udot v19.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f81e317 // udot v23.4s, v24.16b, v1.4b[0]\n" "48:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" - "add x12, x12, #0x1\n" - "cmp x12, x20\n" + "add x11, x11, #0x1\n" + "cmp x11, x20\n" "bne 34b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x22, x14, x20\n" - "prfm pstl1keep, [x14, #0x0]\n" - "prfm pstl1keep, [x22, #0x0]\n" + "add x23, x13, x20\n" + "prfm pstl1keep, [x13, #0x0]\n" + "prfm pstl1keep, [x23, #0x0]\n" "tbnz %x[flags], #31, 49f\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v12.4s, v12.4s, v12.4s\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1r { v2.4s }, [x23]\n" - "neg v2.4s, v2.4s\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1r { v24.4s }, [x20]\n" + "neg v24.4s, v24.4s\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v12.4s, v12.4s, v12.4s\n" - "mul v11.4s, v11.4s, v2.4s\n" - "mul v12.4s, v12.4s, v2.4s\n" + "mul v11.4s, v11.4s, v24.4s\n" + "mul v12.4s, v12.4s, v24.4s\n" "49:" // Height 2: skip row sum fixup - "ldr q0, [x16, #0x0]\n" + "ldr q27, [x15, #0x0]\n" "add v16.4s, v16.4s, v11.4s\n" - "ldr q1, [x16, #0x10]\n" + "ldr q26, [x15, #0x10]\n" "add v17.4s, v17.4s, v11.4s\n" - "ldr q2, [x16, #0x20]\n" + "ldr q25, [x15, #0x20]\n" "add v18.4s, v18.4s, v11.4s\n" - "ldr q3, [x16, #0x30]\n" + "ldr q24, [x15, #0x30]\n" "add v19.4s, v19.4s, v11.4s\n" "add v20.4s, v20.4s, v12.4s\n" "add v21.4s, v21.4s, v12.4s\n" "add v22.4s, v22.4s, v12.4s\n" "add v23.4s, v23.4s, v12.4s\n" - "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" - "add v20.4s, v20.4s, v0.4s\n" - "add v21.4s, v21.4s, v1.4s\n" - "add v22.4s, v22.4s, v2.4s\n" - "add v23.4s, v23.4s, v3.4s\n" - "add x23, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x23]\n" + "add v16.4s, v16.4s, v27.4s\n" + "add v17.4s, v17.4s, v26.4s\n" + "add v18.4s, v18.4s, v25.4s\n" + "add v19.4s, v19.4s, v24.4s\n" + "add v20.4s, v20.4s, v27.4s\n" + "add v21.4s, v21.4s, v26.4s\n" + "add v22.4s, v22.4s, v25.4s\n" + "add v23.4s, v23.4s, v24.4s\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v24.4s }, [x20]\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x23]\n" - "sqrdmulh v16.4s, v16.4s, v4.4s\n" - "sqrdmulh v17.4s, v17.4s, v4.4s\n" - "sqrdmulh v18.4s, v18.4s, v4.4s\n" - "sqrdmulh v19.4s, v19.4s, v4.4s\n" - "sqrdmulh v20.4s, v20.4s, v4.4s\n" - "sqrdmulh v21.4s, v21.4s, v4.4s\n" - "sqrdmulh v22.4s, v22.4s, v4.4s\n" - "sqrdmulh v23.4s, v23.4s, v4.4s\n" - "add x16, x16, #0x40\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "sqrdmulh v16.4s, v16.4s, v24.4s\n" + "sqrdmulh v17.4s, v17.4s, v24.4s\n" + "sqrdmulh v18.4s, v18.4s, v24.4s\n" + "sqrdmulh v19.4s, v19.4s, v24.4s\n" + "sqrdmulh v20.4s, v20.4s, v24.4s\n" + "sqrdmulh v21.4s, v21.4s, v24.4s\n" + "sqrdmulh v22.4s, v22.4s, v24.4s\n" + "sqrdmulh v23.4s, v23.4s, v24.4s\n" + "add x15, x15, #0x40\n" "tbz %x[flags], #5, 50f\n" - "and v4.16b, v16.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "and v5.16b, v17.16b, v0.16b\n" - "and v6.16b, v18.16b, v0.16b\n" - "and v7.16b, v19.16b, v0.16b\n" - "and v8.16b, v20.16b, v0.16b\n" - "and v9.16b, v21.16b, v0.16b\n" - "and v10.16b, v22.16b, v0.16b\n" - "and v4.16b, v23.16b, v0.16b\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v9.4s, v9.4s, #0x1f\n" - "sshr v10.4s, v10.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" - "sqadd v20.4s, v20.4s, v8.4s\n" - "sqadd v21.4s, v21.4s, v9.4s\n" - "sqadd v22.4s, v22.4s, v10.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" + "and v24.16b, v16.16b, v0.16b\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v24.4s\n" + "and v30.16b, v17.16b, v0.16b\n" + "and v29.16b, v18.16b, v0.16b\n" + "and v28.16b, v19.16b, v0.16b\n" + "and v27.16b, v20.16b, v0.16b\n" + "and v26.16b, v21.16b, v0.16b\n" + "and v25.16b, v22.16b, v0.16b\n" + "and v24.16b, v23.16b, v0.16b\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sshr v27.4s, v27.4s, #0x1f\n" + "sshr v26.4s, v26.4s, #0x1f\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sqadd v17.4s, v17.4s, v30.4s\n" + "sqadd v18.4s, v18.4s, v29.4s\n" + "sqadd v19.4s, v19.4s, v28.4s\n" + "sqadd v20.4s, v20.4s, v27.4s\n" + "sqadd v21.4s, v21.4s, v26.4s\n" + "sqadd v22.4s, v22.4s, v25.4s\n" + "sqadd v23.4s, v23.4s, v24.4s\n" "50:" // Height 2: no shift correction "srshl v16.4s, v16.4s, v0.4s\n" "srshl v17.4s, v17.4s, v0.4s\n" @@ -721,122 +720,122 @@ void a64_hybrid_u8qa_dot_4x16_a55 ( "srshl v21.4s, v21.4s, v0.4s\n" "srshl v22.4s, v22.4s, v0.4s\n" "srshl v23.4s, v23.4s, v0.4s\n" - "add x23, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x23]\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v23.4s, v23.4s, v4.4s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x23]\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "add x23, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x23]\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v24.4s }, [x20]\n" + "add v16.4s, v16.4s, v24.4s\n" + "add v17.4s, v17.4s, v24.4s\n" + "add v18.4s, v18.4s, v24.4s\n" + "add v19.4s, v19.4s, v24.4s\n" + "add v20.4s, v20.4s, v24.4s\n" + "add v21.4s, v21.4s, v24.4s\n" + "add v22.4s, v22.4s, v24.4s\n" + "add v23.4s, v23.4s, v24.4s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v24.4s }, [x20]\n" + "smin v16.4s, v16.4s, v24.4s\n" + "smin v17.4s, v17.4s, v24.4s\n" + "smin v18.4s, v18.4s, v24.4s\n" + "smin v19.4s, v19.4s, v24.4s\n" + "smin v20.4s, v20.4s, v24.4s\n" + "smin v21.4s, v21.4s, v24.4s\n" + "smin v22.4s, v22.4s, v24.4s\n" + "smin v23.4s, v23.4s, v24.4s\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v24.4s }, [x20]\n" + "smax v16.4s, v16.4s, v24.4s\n" + "smax v17.4s, v17.4s, v24.4s\n" + "smax v18.4s, v18.4s, v24.4s\n" + "smax v19.4s, v19.4s, v24.4s\n" + "smax v20.4s, v20.4s, v24.4s\n" + "smax v21.4s, v21.4s, v24.4s\n" + "smax v22.4s, v22.4s, v24.4s\n" + "smax v23.4s, v23.4s, v24.4s\n" "uzp1 v16.8h, v16.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v18.8h, v18.8h, v19.8h\n" "uzp1 v20.8h, v20.8h, v21.8h\n" - "uzp1 v21.8h, v22.8h, v23.8h\n" - "cmp x15, #0x10\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" - "uzp1 v20.16b, v20.16b, v21.16b\n" + "uzp1 v17.8h, v22.8h, v23.8h\n" + "cmp x14, #0x10\n" + "uzp1 v16.16b, v16.16b, v18.16b\n" + "uzp1 v20.16b, v20.16b, v17.16b\n" "bge 59f\n" - "tbz x15, #3, 54f\n" - "str d16, [x14], #0x8\n" - "str d20, [x22], #0x8\n" - "tbz x15, #2, 52f\n" - "st1 { v16.s }[2], [x14], #0x4\n" - "st1 { v20.s }[2], [x22], #0x4\n" - "tbz x15, #1, 51f\n" - "st1 { v16.h }[6], [x14], #0x2\n" - "st1 { v20.h }[6], [x22], #0x2\n" - "tbz x15, #0, 58f\n" - "st1 { v16.b }[14], [x14]\n" - "st1 { v20.b }[14], [x22]\n" + "tbz x14, #3, 54f\n" + "str d16, [x13], #0x8\n" + "str d20, [x23], #0x8\n" + "tbz x14, #2, 52f\n" + "st1 { v16.s }[2], [x13], #0x4\n" + "st1 { v20.s }[2], [x23], #0x4\n" + "tbz x14, #1, 51f\n" + "st1 { v16.h }[6], [x13], #0x2\n" + "st1 { v20.h }[6], [x23], #0x2\n" + "tbz x14, #0, 58f\n" + "st1 { v16.b }[14], [x13]\n" + "st1 { v20.b }[14], [x23]\n" "b 58f\n" "51:" // Height 2: Partial direct writeback: partial_1_12 - "tbz x15, #0, 58f\n" - "st1 { v16.b }[12], [x14]\n" - "st1 { v20.b }[12], [x22]\n" + "tbz x14, #0, 58f\n" + "st1 { v16.b }[12], [x13]\n" + "st1 { v20.b }[12], [x23]\n" "b 58f\n" "52:" // Height 2: Partial direct writeback: partial_2_8 - "tbz x15, #1, 53f\n" - "st1 { v16.h }[4], [x14], #0x2\n" - "st1 { v20.h }[4], [x22], #0x2\n" - "tbz x15, #0, 58f\n" - "st1 { v16.b }[10], [x14]\n" - "st1 { v20.b }[10], [x22]\n" + "tbz x14, #1, 53f\n" + "st1 { v16.h }[4], [x13], #0x2\n" + "st1 { v20.h }[4], [x23], #0x2\n" + "tbz x14, #0, 58f\n" + "st1 { v16.b }[10], [x13]\n" + "st1 { v20.b }[10], [x23]\n" "b 58f\n" "53:" // Height 2: Partial direct writeback: partial_1_8 - "tbz x15, #0, 58f\n" - "st1 { v16.b }[8], [x14]\n" - "st1 { v20.b }[8], [x22]\n" + "tbz x14, #0, 58f\n" + "st1 { v16.b }[8], [x13]\n" + "st1 { v20.b }[8], [x23]\n" "b 58f\n" "54:" // Height 2: Partial direct writeback: partial_4_0 - "tbz x15, #2, 56f\n" - "str s16, [x14], #0x4\n" - "str s20, [x22], #0x4\n" - "tbz x15, #1, 55f\n" - "st1 { v16.h }[2], [x14], #0x2\n" - "st1 { v20.h }[2], [x22], #0x2\n" - "tbz x15, #0, 58f\n" - "st1 { v16.b }[6], [x14]\n" - "st1 { v20.b }[6], [x22]\n" + "tbz x14, #2, 56f\n" + "str s16, [x13], #0x4\n" + "str s20, [x23], #0x4\n" + "tbz x14, #1, 55f\n" + "st1 { v16.h }[2], [x13], #0x2\n" + "st1 { v20.h }[2], [x23], #0x2\n" + "tbz x14, #0, 58f\n" + "st1 { v16.b }[6], [x13]\n" + "st1 { v20.b }[6], [x23]\n" "b 58f\n" "55:" // Height 2: Partial direct writeback: partial_1_4 - "tbz x15, #0, 58f\n" - "st1 { v16.b }[4], [x14]\n" - "st1 { v20.b }[4], [x22]\n" + "tbz x14, #0, 58f\n" + "st1 { v16.b }[4], [x13]\n" + "st1 { v20.b }[4], [x23]\n" "b 58f\n" "56:" // Height 2: Partial direct writeback: partial_2_0 - "tbz x15, #1, 57f\n" - "str h16, [x14], #0x2\n" - "str h20, [x22], #0x2\n" - "tbz x15, #0, 58f\n" - "st1 { v16.b }[2], [x14]\n" - "st1 { v20.b }[2], [x22]\n" + "tbz x14, #1, 57f\n" + "str h16, [x13], #0x2\n" + "str h20, [x23], #0x2\n" + "tbz x14, #0, 58f\n" + "st1 { v16.b }[2], [x13]\n" + "st1 { v20.b }[2], [x23]\n" "b 58f\n" "57:" // Height 2: Partial direct writeback: partial_1_0 - "str b16, [x14, #0x0]\n" - "str b20, [x22, #0x0]\n" + "str b16, [x13, #0x0]\n" + "str b20, [x23, #0x0]\n" "58:" // Height 2: Partial direct writeback: Done "b 60f\n" "59:" // Height 2: Full writeback - "str q16, [x14, #0x0]\n" - "add x14, x14, #0x10\n" - "str q20, [x22, #0x0]\n" + "str q16, [x13, #0x0]\n" + "add x13, x13, #0x10\n" + "str q20, [x23, #0x0]\n" "60:" // Height 2: Writeback done - "subs x15, x15, #0x10\n" + "subs x14, x14, #0x10\n" "bgt 32b\n" "b 122f\n" "61:" // Height 3 - "mov x16, %x[col_bias]\n" + "mov x15, %x[col_bias]\n" "movi v11.4s, #0x0\n" "movi v12.4s, #0x0\n" "bic %x[flags], %x[flags], #0x80000000\n" "movi v13.4s, #0x0\n" - "ldr x15, [%x[args_ptr], %[offsetof_N]]\n" + "ldr x14, [%x[args_ptr], %[offsetof_N]]\n" "movi v15.16b, #0x1\n" - "ldr x13, [%x[args_ptr], %[offsetof_B_ptr]]\n" - "mov x14, %x[output_ptr]\n" + "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n" + "mov x13, %x[output_ptr]\n" "62:" // Height 3: Column loop "movi v16.4s, #0x0\n" "movi v17.4s, #0x0\n" @@ -851,317 +850,317 @@ void a64_hybrid_u8qa_dot_4x16_a55 ( "movi v26.4s, #0x0\n" "movi v27.4s, #0x0\n" "63:" // Height 3: setup done - "mov x12, #0x0\n" + "mov x11, #0x0\n" "64:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr w11, [x20, x12, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr w10, [x20, x11, LSL #0x2]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 65f\n" - "ldr x21, [%x[input_ptr], x12, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x10, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" - "cbnz x12, 66f\n" + "ldr x20, [%x[input_ptr], x11, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x9, [x20, #0x0]\n" + "ldr x28, [x20, #0x8]\n" + "ldr x27, [x20, #0x10]\n" + "cbnz x11, 66f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x10, x10, x20\n" - "add x23, x23, x20\n" - "add x22, x22, x20\n" + "add x9, x9, x20\n" + "add x28, x28, x20\n" + "add x27, x27, x20\n" "b 66f\n" "65:" // Height 3: setup direct input - "mov x10, %x[input_ptr]\n" - "add x23, x10, x20\n" - "add x22, x23, x20\n" + "mov x9, %x[input_ptr]\n" + "add x28, x9, x21\n" + "add x27, x28, x21\n" "66:" // Height 3: input setup done - "cmp x11, #0x10\n" + "cmp x10, #0x10\n" "blt 71f\n" - "ldr q0, [x10, #0x0]\n" - "cmp x11, #0x20\n" - "ldr q1, [x23, #0x0]\n" - "ldr q2, [x22, #0x0]\n" - "ldr q4, [x13, #0x0]\n" - "ldr q5, [x13, #0x10]\n" - "ldr q6, [x13, #0x20]\n" - "ldr q7, [x13, #0x30]\n" - "ldr q8, [x13, #0x40]\n" - "ldr q9, [x13, #0x50]\n" - "ldr q10, [x13, #0x60]\n" + "ldr q0, [x9, #0x0]\n" + "cmp x10, #0x20\n" + "ldr q1, [x28, #0x0]\n" + "ldr q2, [x27, #0x0]\n" + "ldr q4, [x12, #0x0]\n" + "ldr q5, [x12, #0x10]\n" + "ldr q6, [x12, #0x20]\n" + "ldr q7, [x12, #0x30]\n" + "ldr q8, [x12, #0x40]\n" + "ldr q9, [x12, #0x50]\n" + "ldr q10, [x12, #0x60]\n" "blt 69f\n" "67:" // Height 3: Multiply loop: Main loop head ".inst 0x6f80e090 // udot v16.4s, v4.16b, v0.4b[0]\n" - "ldr x9, [x13, #0x78]\n" + "ldr x20, [x12, #0x78]\n" ".inst 0x6f81e094 // udot v20.4s, v4.16b, v1.4b[0]\n" - "ldr x28, [x13, #0x88]\n" + "ldr x23, [x12, #0x88]\n" ".inst 0x6f82e098 // udot v24.4s, v4.16b, v2.4b[0]\n" - "ldr d4, [x13, #0x70]\n" + "ldr d29, [x12, #0x70]\n" ".inst 0x6f80e0b1 // udot v17.4s, v5.16b, v0.4b[0]\n" - "mov v4.d[1], x9\n" + "mov v29.d[1], x20\n" ".inst 0x6f81e0b5 // udot v21.4s, v5.16b, v1.4b[0]\n" - "ldr x27, [x13, #0x98]\n" + "ldr x22, [x12, #0x98]\n" ".inst 0x6f82e0b9 // udot v25.4s, v5.16b, v2.4b[0]\n" - "ldr d5, [x13, #0x80]\n" + "ldr d28, [x12, #0x80]\n" ".inst 0x6f80e0d2 // udot v18.4s, v6.16b, v0.4b[0]\n" - "ldr x26, [x13, #0xa8]\n" + "ldr x21, [x12, #0xa8]\n" ".inst 0x6f81e0d6 // udot v22.4s, v6.16b, v1.4b[0]\n" - "ldr x25, [x13, #0xb8]\n" + "ldr x20, [x12, #0xb8]\n" ".inst 0x6f82e0da // udot v26.4s, v6.16b, v2.4b[0]\n" - "ldr d6, [x13, #0x90]\n" + "ldr d5, [x12, #0x90]\n" ".inst 0x6f80e0f3 // udot v19.4s, v7.16b, v0.4b[0]\n" - "mov v5.d[1], x28\n" + "mov v28.d[1], x23\n" ".inst 0x6f81e0f7 // udot v23.4s, v7.16b, v1.4b[0]\n" - "mov v6.d[1], x27\n" + "mov v5.d[1], x22\n" ".inst 0x6f82e0fb // udot v27.4s, v7.16b, v2.4b[0]\n" - "ldr d7, [x13, #0xa0]\n" + "ldr d4, [x12, #0xa0]\n" ".inst 0x6fa0e110 // udot v16.4s, v8.16b, v0.4b[1]\n" - "mov v7.d[1], x26\n" + "mov v4.d[1], x21\n" ".inst 0x6fa1e114 // udot v20.4s, v8.16b, v1.4b[1]\n" - "ldr x24, [x13, #0xc8]\n" + "ldr x23, [x12, #0xc8]\n" ".inst 0x6fa2e118 // udot v24.4s, v8.16b, v2.4b[1]\n" - "ldr d8, [x13, #0xb0]\n" + "ldr d3, [x12, #0xb0]\n" ".inst 0x6fa0e131 // udot v17.4s, v9.16b, v0.4b[1]\n" - "mov v8.d[1], x25\n" + "mov v3.d[1], x20\n" ".inst 0x6fa1e135 // udot v21.4s, v9.16b, v1.4b[1]\n" - "ldr x20, [x13, #0xd8]\n" + "ldr x22, [x12, #0xd8]\n" ".inst 0x6fa2e139 // udot v25.4s, v9.16b, v2.4b[1]\n" - "ldr d9, [x13, #0xc0]\n" + "ldr d31, [x12, #0xc0]\n" ".inst 0x6fa0e152 // udot v18.4s, v10.16b, v0.4b[1]\n" - "ldr x9, [x13, #0xe8]\n" + "ldr x21, [x12, #0xe8]\n" ".inst 0x6fa1e156 // udot v22.4s, v10.16b, v1.4b[1]\n" - "ldr x28, [x13, #0xf8]\n" + "ldr x20, [x12, #0xf8]\n" ".inst 0x6fa2e15a // udot v26.4s, v10.16b, v2.4b[1]\n" - "ldr d10, [x13, #0xd0]\n" - ".inst 0x6fa0e093 // udot v19.4s, v4.16b, v0.4b[1]\n" - "mov v9.d[1], x24\n" - ".inst 0x6fa1e097 // udot v23.4s, v4.16b, v1.4b[1]\n" - "mov v10.d[1], x20\n" - ".inst 0x6fa2e09b // udot v27.4s, v4.16b, v2.4b[1]\n" - "ldr d4, [x13, #0xe0]\n" - ".inst 0x6f80e8b0 // udot v16.4s, v5.16b, v0.4b[2]\n" - "mov v4.d[1], x9\n" - ".inst 0x6f81e8b4 // udot v20.4s, v5.16b, v1.4b[2]\n" - "add x10, x10, #0x10\n" - ".inst 0x6f82e8b8 // udot v24.4s, v5.16b, v2.4b[2]\n" - "ldr d5, [x13, #0xf0]\n" - ".inst 0x6f80e8d1 // udot v17.4s, v6.16b, v0.4b[2]\n" - "mov v5.d[1], x28\n" - ".inst 0x6f81e8d5 // udot v21.4s, v6.16b, v1.4b[2]\n" - "add x23, x23, #0x10\n" - ".inst 0x6f82e8d9 // udot v25.4s, v6.16b, v2.4b[2]\n" - "add x22, x22, #0x10\n" - ".inst 0x6f80e8f2 // udot v18.4s, v7.16b, v0.4b[2]\n" - "add x13, x13, #0x100\n" - ".inst 0x6f81e8f6 // udot v22.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f82e8fa // udot v26.4s, v7.16b, v2.4b[2]\n" - ".inst 0x6f80e913 // udot v19.4s, v8.16b, v0.4b[2]\n" - ".inst 0x6f81e917 // udot v23.4s, v8.16b, v1.4b[2]\n" - ".inst 0x6f82e91b // udot v27.4s, v8.16b, v2.4b[2]\n" - ".inst 0x6fa0e930 // udot v16.4s, v9.16b, v0.4b[3]\n" - ".inst 0x6fa1e934 // udot v20.4s, v9.16b, v1.4b[3]\n" - ".inst 0x6fa2e938 // udot v24.4s, v9.16b, v2.4b[3]\n" - ".inst 0x6fa0e951 // udot v17.4s, v10.16b, v0.4b[3]\n" - ".inst 0x6fa1e955 // udot v21.4s, v10.16b, v1.4b[3]\n" - ".inst 0x6fa2e959 // udot v25.4s, v10.16b, v2.4b[3]\n" - ".inst 0x6fa0e892 // udot v18.4s, v4.16b, v0.4b[3]\n" - ".inst 0x6fa1e896 // udot v22.4s, v4.16b, v1.4b[3]\n" - ".inst 0x6fa2e89a // udot v26.4s, v4.16b, v2.4b[3]\n" - ".inst 0x6fa0e8b3 // udot v19.4s, v5.16b, v0.4b[3]\n" - ".inst 0x6fa1e8b7 // udot v23.4s, v5.16b, v1.4b[3]\n" - ".inst 0x6fa2e8bb // udot v27.4s, v5.16b, v2.4b[3]\n" + "ldr d30, [x12, #0xd0]\n" + ".inst 0x6fa0e3b3 // udot v19.4s, v29.16b, v0.4b[1]\n" + "mov v31.d[1], x23\n" + ".inst 0x6fa1e3b7 // udot v23.4s, v29.16b, v1.4b[1]\n" + "mov v30.d[1], x22\n" + ".inst 0x6fa2e3bb // udot v27.4s, v29.16b, v2.4b[1]\n" + "ldr d29, [x12, #0xe0]\n" + ".inst 0x6f80eb90 // udot v16.4s, v28.16b, v0.4b[2]\n" + "mov v29.d[1], x21\n" + ".inst 0x6f81eb94 // udot v20.4s, v28.16b, v1.4b[2]\n" + "add x9, x9, #0x10\n" + ".inst 0x6f82eb98 // udot v24.4s, v28.16b, v2.4b[2]\n" + "ldr d28, [x12, #0xf0]\n" + ".inst 0x6f80e8b1 // udot v17.4s, v5.16b, v0.4b[2]\n" + "mov v28.d[1], x20\n" + ".inst 0x6f81e8b5 // udot v21.4s, v5.16b, v1.4b[2]\n" + "add x28, x28, #0x10\n" + ".inst 0x6f82e8b9 // udot v25.4s, v5.16b, v2.4b[2]\n" + "add x27, x27, #0x10\n" + ".inst 0x6f80e892 // udot v18.4s, v4.16b, v0.4b[2]\n" + "add x12, x12, #0x100\n" + ".inst 0x6f81e896 // udot v22.4s, v4.16b, v1.4b[2]\n" + ".inst 0x6f82e89a // udot v26.4s, v4.16b, v2.4b[2]\n" + ".inst 0x6f80e873 // udot v19.4s, v3.16b, v0.4b[2]\n" + ".inst 0x6f81e877 // udot v23.4s, v3.16b, v1.4b[2]\n" + ".inst 0x6f82e87b // udot v27.4s, v3.16b, v2.4b[2]\n" + ".inst 0x6fa0ebf0 // udot v16.4s, v31.16b, v0.4b[3]\n" + ".inst 0x6fa1ebf4 // udot v20.4s, v31.16b, v1.4b[3]\n" + ".inst 0x6fa2ebf8 // udot v24.4s, v31.16b, v2.4b[3]\n" + ".inst 0x6fa0ebd1 // udot v17.4s, v30.16b, v0.4b[3]\n" + ".inst 0x6fa1ebd5 // udot v21.4s, v30.16b, v1.4b[3]\n" + ".inst 0x6fa2ebd9 // udot v25.4s, v30.16b, v2.4b[3]\n" + ".inst 0x6fa0ebb2 // udot v18.4s, v29.16b, v0.4b[3]\n" + ".inst 0x6fa1ebb6 // udot v22.4s, v29.16b, v1.4b[3]\n" + ".inst 0x6fa2ebba // udot v26.4s, v29.16b, v2.4b[3]\n" + ".inst 0x6fa0eb93 // udot v19.4s, v28.16b, v0.4b[3]\n" + ".inst 0x6fa1eb97 // udot v23.4s, v28.16b, v1.4b[3]\n" + ".inst 0x6fa2eb9b // udot v27.4s, v28.16b, v2.4b[3]\n" "tbnz %x[flags], #31, 68f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n" ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n" "68:" // Height 3: Multiply loop: unique 9: skip row sum - "ldr q0, [x10, #0x0]\n" - "sub x11, x11, #0x10\n" - "ldr q1, [x23, #0x0]\n" - "cmp x11, #0x20\n" - "ldr q2, [x22, #0x0]\n" - "ldr q4, [x13, #0x0]\n" - "ldr q5, [x13, #0x10]\n" - "ldr q6, [x13, #0x20]\n" - "ldr q7, [x13, #0x30]\n" - "ldr q8, [x13, #0x40]\n" - "ldr q9, [x13, #0x50]\n" - "ldr q10, [x13, #0x60]\n" - "prfm pldl1keep, [x10, #0x80]\n" - "prfm pldl1keep, [x23, #0x80]\n" - "prfm pldl1keep, [x22, #0x80]\n" + "ldr q0, [x9, #0x0]\n" + "sub x10, x10, #0x10\n" + "ldr q1, [x28, #0x0]\n" + "cmp x10, #0x20\n" + "ldr q2, [x27, #0x0]\n" + "ldr q4, [x12, #0x0]\n" + "ldr q5, [x12, #0x10]\n" + "ldr q6, [x12, #0x20]\n" + "ldr q7, [x12, #0x30]\n" + "ldr q8, [x12, #0x40]\n" + "ldr q9, [x12, #0x50]\n" + "ldr q10, [x12, #0x60]\n" + "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x28, #0x80]\n" + "prfm pldl1keep, [x27, #0x80]\n" "bge 67b\n" "69:" // Height 3: Multiply loop: Single iteration only ".inst 0x6f80e090 // udot v16.4s, v4.16b, v0.4b[0]\n" - "sub x11, x11, #0x10\n" + "sub x10, x10, #0x10\n" ".inst 0x6f81e094 // udot v20.4s, v4.16b, v1.4b[0]\n" - "add x10, x10, #0x10\n" + "add x9, x9, #0x10\n" ".inst 0x6f82e098 // udot v24.4s, v4.16b, v2.4b[0]\n" - "ldr q4, [x13, #0x70]\n" + "ldr q29, [x12, #0x70]\n" ".inst 0x6f80e0b1 // udot v17.4s, v5.16b, v0.4b[0]\n" - "add x23, x23, #0x10\n" + "add x28, x28, #0x10\n" ".inst 0x6f81e0b5 // udot v21.4s, v5.16b, v1.4b[0]\n" - "add x22, x22, #0x10\n" + "add x27, x27, #0x10\n" ".inst 0x6f82e0b9 // udot v25.4s, v5.16b, v2.4b[0]\n" - "ldr q5, [x13, #0x80]\n" + "ldr q28, [x12, #0x80]\n" ".inst 0x6f80e0d2 // udot v18.4s, v6.16b, v0.4b[0]\n" ".inst 0x6f81e0d6 // udot v22.4s, v6.16b, v1.4b[0]\n" ".inst 0x6f82e0da // udot v26.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x13, #0x90]\n" + "ldr q5, [x12, #0x90]\n" ".inst 0x6f80e0f3 // udot v19.4s, v7.16b, v0.4b[0]\n" ".inst 0x6f81e0f7 // udot v23.4s, v7.16b, v1.4b[0]\n" ".inst 0x6f82e0fb // udot v27.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x13, #0xa0]\n" + "ldr q4, [x12, #0xa0]\n" ".inst 0x6fa0e110 // udot v16.4s, v8.16b, v0.4b[1]\n" ".inst 0x6fa1e114 // udot v20.4s, v8.16b, v1.4b[1]\n" ".inst 0x6fa2e118 // udot v24.4s, v8.16b, v2.4b[1]\n" - "ldr q8, [x13, #0xb0]\n" + "ldr q3, [x12, #0xb0]\n" ".inst 0x6fa0e131 // udot v17.4s, v9.16b, v0.4b[1]\n" ".inst 0x6fa1e135 // udot v21.4s, v9.16b, v1.4b[1]\n" ".inst 0x6fa2e139 // udot v25.4s, v9.16b, v2.4b[1]\n" - "ldr q9, [x13, #0xc0]\n" + "ldr q31, [x12, #0xc0]\n" ".inst 0x6fa0e152 // udot v18.4s, v10.16b, v0.4b[1]\n" ".inst 0x6fa1e156 // udot v22.4s, v10.16b, v1.4b[1]\n" ".inst 0x6fa2e15a // udot v26.4s, v10.16b, v2.4b[1]\n" - "ldr q10, [x13, #0xd0]\n" - ".inst 0x6fa0e093 // udot v19.4s, v4.16b, v0.4b[1]\n" - ".inst 0x6fa1e097 // udot v23.4s, v4.16b, v1.4b[1]\n" - ".inst 0x6fa2e09b // udot v27.4s, v4.16b, v2.4b[1]\n" - "ldr q4, [x13, #0xe0]\n" - ".inst 0x6f80e8b0 // udot v16.4s, v5.16b, v0.4b[2]\n" - ".inst 0x6f81e8b4 // udot v20.4s, v5.16b, v1.4b[2]\n" - ".inst 0x6f82e8b8 // udot v24.4s, v5.16b, v2.4b[2]\n" - "ldr q5, [x13, #0xf0]\n" - ".inst 0x6f80e8d1 // udot v17.4s, v6.16b, v0.4b[2]\n" - "add x13, x13, #0x100\n" - ".inst 0x6f81e8d5 // udot v21.4s, v6.16b, v1.4b[2]\n" - ".inst 0x6f82e8d9 // udot v25.4s, v6.16b, v2.4b[2]\n" - ".inst 0x6f80e8f2 // udot v18.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8f6 // udot v22.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f82e8fa // udot v26.4s, v7.16b, v2.4b[2]\n" - ".inst 0x6f80e913 // udot v19.4s, v8.16b, v0.4b[2]\n" - ".inst 0x6f81e917 // udot v23.4s, v8.16b, v1.4b[2]\n" - ".inst 0x6f82e91b // udot v27.4s, v8.16b, v2.4b[2]\n" - ".inst 0x6fa0e930 // udot v16.4s, v9.16b, v0.4b[3]\n" - ".inst 0x6fa1e934 // udot v20.4s, v9.16b, v1.4b[3]\n" - ".inst 0x6fa2e938 // udot v24.4s, v9.16b, v2.4b[3]\n" - ".inst 0x6fa0e951 // udot v17.4s, v10.16b, v0.4b[3]\n" - ".inst 0x6fa1e955 // udot v21.4s, v10.16b, v1.4b[3]\n" - ".inst 0x6fa2e959 // udot v25.4s, v10.16b, v2.4b[3]\n" - ".inst 0x6fa0e892 // udot v18.4s, v4.16b, v0.4b[3]\n" - ".inst 0x6fa1e896 // udot v22.4s, v4.16b, v1.4b[3]\n" - ".inst 0x6fa2e89a // udot v26.4s, v4.16b, v2.4b[3]\n" - ".inst 0x6fa0e8b3 // udot v19.4s, v5.16b, v0.4b[3]\n" - ".inst 0x6fa1e8b7 // udot v23.4s, v5.16b, v1.4b[3]\n" - ".inst 0x6fa2e8bb // udot v27.4s, v5.16b, v2.4b[3]\n" + "ldr q30, [x12, #0xd0]\n" + ".inst 0x6fa0e3b3 // udot v19.4s, v29.16b, v0.4b[1]\n" + ".inst 0x6fa1e3b7 // udot v23.4s, v29.16b, v1.4b[1]\n" + ".inst 0x6fa2e3bb // udot v27.4s, v29.16b, v2.4b[1]\n" + "ldr q29, [x12, #0xe0]\n" + ".inst 0x6f80eb90 // udot v16.4s, v28.16b, v0.4b[2]\n" + ".inst 0x6f81eb94 // udot v20.4s, v28.16b, v1.4b[2]\n" + ".inst 0x6f82eb98 // udot v24.4s, v28.16b, v2.4b[2]\n" + "ldr q28, [x12, #0xf0]\n" + ".inst 0x6f80e8b1 // udot v17.4s, v5.16b, v0.4b[2]\n" + "add x12, x12, #0x100\n" + ".inst 0x6f81e8b5 // udot v21.4s, v5.16b, v1.4b[2]\n" + ".inst 0x6f82e8b9 // udot v25.4s, v5.16b, v2.4b[2]\n" + ".inst 0x6f80e892 // udot v18.4s, v4.16b, v0.4b[2]\n" + ".inst 0x6f81e896 // udot v22.4s, v4.16b, v1.4b[2]\n" + ".inst 0x6f82e89a // udot v26.4s, v4.16b, v2.4b[2]\n" + ".inst 0x6f80e873 // udot v19.4s, v3.16b, v0.4b[2]\n" + ".inst 0x6f81e877 // udot v23.4s, v3.16b, v1.4b[2]\n" + ".inst 0x6f82e87b // udot v27.4s, v3.16b, v2.4b[2]\n" + ".inst 0x6fa0ebf0 // udot v16.4s, v31.16b, v0.4b[3]\n" + ".inst 0x6fa1ebf4 // udot v20.4s, v31.16b, v1.4b[3]\n" + ".inst 0x6fa2ebf8 // udot v24.4s, v31.16b, v2.4b[3]\n" + ".inst 0x6fa0ebd1 // udot v17.4s, v30.16b, v0.4b[3]\n" + ".inst 0x6fa1ebd5 // udot v21.4s, v30.16b, v1.4b[3]\n" + ".inst 0x6fa2ebd9 // udot v25.4s, v30.16b, v2.4b[3]\n" + ".inst 0x6fa0ebb2 // udot v18.4s, v29.16b, v0.4b[3]\n" + ".inst 0x6fa1ebb6 // udot v22.4s, v29.16b, v1.4b[3]\n" + ".inst 0x6fa2ebba // udot v26.4s, v29.16b, v2.4b[3]\n" + ".inst 0x6fa0eb93 // udot v19.4s, v28.16b, v0.4b[3]\n" + ".inst 0x6fa1eb97 // udot v23.4s, v28.16b, v1.4b[3]\n" + ".inst 0x6fa2eb9b // udot v27.4s, v28.16b, v2.4b[3]\n" "tbnz %x[flags], #31, 70f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n" ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n" "70:" // Height 3: Multiply loop: unique 10: skip row sum - "prfm pldl1keep, [x10, #0x80]\n" - "prfm pldl1keep, [x23, #0x80]\n" - "prfm pldl1keep, [x22, #0x80]\n" + "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x28, #0x80]\n" + "prfm pldl1keep, [x27, #0x80]\n" "71:" // Height 3: Multiply loop: Main loop skip - "cbz x11, 78f\n" - "cmp x11, #0x4\n" + "cbz x10, 78f\n" + "cmp x10, #0x4\n" "blt 74f\n" "72:" // Height 3: Multiply loop: Odd block loop - "ldr s0, [x10], #0x4\n" - "ldr s1, [x23], #0x4\n" - "ldr s2, [x22], #0x4\n" + "ldr s0, [x9], #0x4\n" + "ldr s1, [x28], #0x4\n" + "ldr s2, [x27], #0x4\n" "tbnz %x[flags], #31, 73f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n" ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n" "73:" // Height 3: Multiply loop: unique 11: skip row sum - "ldr q6, [x13, #0x0]\n" - "sub x11, x11, #0x4\n" - "ldr q7, [x13, #0x10]\n" - "cmp x11, #0x4\n" - "ldr q8, [x13, #0x20]\n" - ".inst 0x6f80e0d0 // udot v16.4s, v6.16b, v0.4b[0]\n" - "ldr q9, [x13, #0x30]\n" - ".inst 0x6f81e0d4 // udot v20.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d8 // udot v24.4s, v6.16b, v2.4b[0]\n" - "add x13, x13, #0x40\n" - ".inst 0x6f80e0f1 // udot v17.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0f5 // udot v21.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f9 // udot v25.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f80e112 // udot v18.4s, v8.16b, v0.4b[0]\n" - ".inst 0x6f81e116 // udot v22.4s, v8.16b, v1.4b[0]\n" - ".inst 0x6f82e11a // udot v26.4s, v8.16b, v2.4b[0]\n" - ".inst 0x6f80e133 // udot v19.4s, v9.16b, v0.4b[0]\n" - ".inst 0x6f81e137 // udot v23.4s, v9.16b, v1.4b[0]\n" - ".inst 0x6f82e13b // udot v27.4s, v9.16b, v2.4b[0]\n" + "ldr q31, [x12, #0x0]\n" + "sub x10, x10, #0x4\n" + "ldr q30, [x12, #0x10]\n" + "cmp x10, #0x4\n" + "ldr q29, [x12, #0x20]\n" + ".inst 0x6f80e3f0 // udot v16.4s, v31.16b, v0.4b[0]\n" + "ldr q28, [x12, #0x30]\n" + ".inst 0x6f81e3f4 // udot v20.4s, v31.16b, v1.4b[0]\n" + ".inst 0x6f82e3f8 // udot v24.4s, v31.16b, v2.4b[0]\n" + "add x12, x12, #0x40\n" + ".inst 0x6f80e3d1 // udot v17.4s, v30.16b, v0.4b[0]\n" + ".inst 0x6f81e3d5 // udot v21.4s, v30.16b, v1.4b[0]\n" + ".inst 0x6f82e3d9 // udot v25.4s, v30.16b, v2.4b[0]\n" + ".inst 0x6f80e3b2 // udot v18.4s, v29.16b, v0.4b[0]\n" + ".inst 0x6f81e3b6 // udot v22.4s, v29.16b, v1.4b[0]\n" + ".inst 0x6f82e3ba // udot v26.4s, v29.16b, v2.4b[0]\n" + ".inst 0x6f80e393 // udot v19.4s, v28.16b, v0.4b[0]\n" + ".inst 0x6f81e397 // udot v23.4s, v28.16b, v1.4b[0]\n" + ".inst 0x6f82e39b // udot v27.4s, v28.16b, v2.4b[0]\n" "bge 72b\n" "74:" // Height 3: Multiply loop: Skip odd blocks - "cbz x11, 78f\n" - "tbz x11, #1, 75f\n" - "ldr h0, [x10], #0x2\n" - "ldr h1, [x23], #0x2\n" - "ldr h2, [x22], #0x2\n" - "tbz x11, #0, 76f\n" - "ld1 { v0.b }[2], [x10]\n" - "ld1 { v1.b }[2], [x23]\n" - "ld1 { v2.b }[2], [x22]\n" + "cbz x10, 78f\n" + "tbz x10, #1, 75f\n" + "ldr h0, [x9], #0x2\n" + "ldr h1, [x28], #0x2\n" + "ldr h2, [x27], #0x2\n" + "tbz x10, #0, 76f\n" + "ld1 { v0.b }[2], [x9]\n" + "ld1 { v1.b }[2], [x28]\n" + "ld1 { v2.b }[2], [x27]\n" "b 76f\n" "75:" // Height 3: Multiply loop: Ragged operand read: partial_1_0 - "ldr b0, [x10, #0x0]\n" - "ldr b1, [x23, #0x0]\n" - "ldr b2, [x22, #0x0]\n" + "ldr b0, [x9, #0x0]\n" + "ldr b1, [x28, #0x0]\n" + "ldr b2, [x27, #0x0]\n" "76:" // Height 3: Multiply loop: Ragged operand read: Done "tbnz %x[flags], #31, 77f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n" ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n" "77:" // Height 3: Multiply loop: unique 12: skip row sum - "ldr q10, [x13, #0x0]\n" - ".inst 0x6f80e150 // udot v16.4s, v10.16b, v0.4b[0]\n" - "ldr q4, [x13, #0x10]\n" - ".inst 0x6f81e154 // udot v20.4s, v10.16b, v1.4b[0]\n" - "ldr q5, [x13, #0x20]\n" - ".inst 0x6f82e158 // udot v24.4s, v10.16b, v2.4b[0]\n" - "ldr q6, [x13, #0x30]\n" - ".inst 0x6f80e091 // udot v17.4s, v4.16b, v0.4b[0]\n" - ".inst 0x6f81e095 // udot v21.4s, v4.16b, v1.4b[0]\n" - "add x13, x13, #0x40\n" - ".inst 0x6f82e099 // udot v25.4s, v4.16b, v2.4b[0]\n" - ".inst 0x6f80e0b2 // udot v18.4s, v5.16b, v0.4b[0]\n" - ".inst 0x6f81e0b6 // udot v22.4s, v5.16b, v1.4b[0]\n" - ".inst 0x6f82e0ba // udot v26.4s, v5.16b, v2.4b[0]\n" - ".inst 0x6f80e0d3 // udot v19.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0d7 // udot v23.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0db // udot v27.4s, v6.16b, v2.4b[0]\n" + "ldr q28, [x12, #0x0]\n" + ".inst 0x6f80e390 // udot v16.4s, v28.16b, v0.4b[0]\n" + "ldr q30, [x12, #0x10]\n" + ".inst 0x6f81e394 // udot v20.4s, v28.16b, v1.4b[0]\n" + "ldr q29, [x12, #0x20]\n" + ".inst 0x6f82e398 // udot v24.4s, v28.16b, v2.4b[0]\n" + "ldr q28, [x12, #0x30]\n" + ".inst 0x6f80e3d1 // udot v17.4s, v30.16b, v0.4b[0]\n" + ".inst 0x6f81e3d5 // udot v21.4s, v30.16b, v1.4b[0]\n" + "add x12, x12, #0x40\n" + ".inst 0x6f82e3d9 // udot v25.4s, v30.16b, v2.4b[0]\n" + ".inst 0x6f80e3b2 // udot v18.4s, v29.16b, v0.4b[0]\n" + ".inst 0x6f81e3b6 // udot v22.4s, v29.16b, v1.4b[0]\n" + ".inst 0x6f82e3ba // udot v26.4s, v29.16b, v2.4b[0]\n" + ".inst 0x6f80e393 // udot v19.4s, v28.16b, v0.4b[0]\n" + ".inst 0x6f81e397 // udot v23.4s, v28.16b, v1.4b[0]\n" + ".inst 0x6f82e39b // udot v27.4s, v28.16b, v2.4b[0]\n" "78:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" - "add x12, x12, #0x1\n" - "cmp x12, x20\n" + "add x11, x11, #0x1\n" + "cmp x11, x20\n" "bne 64b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x22, x14, x20\n" - "add x21, x22, x20\n" - "prfm pstl1keep, [x14, #0x0]\n" + "add x23, x13, x20\n" + "add x22, x23, x20\n" + "prfm pstl1keep, [x13, #0x0]\n" + "prfm pstl1keep, [x23, #0x0]\n" "prfm pstl1keep, [x22, #0x0]\n" - "prfm pstl1keep, [x21, #0x0]\n" "tbnz %x[flags], #31, 79f\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v12.4s, v12.4s, v12.4s\n" "addp v13.4s, v13.4s, v13.4s\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1r { v3.4s }, [x23]\n" - "neg v3.4s, v3.4s\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1r { v28.4s }, [x20]\n" + "neg v28.4s, v28.4s\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v12.4s, v12.4s, v12.4s\n" "addp v13.4s, v13.4s, v13.4s\n" - "mul v11.4s, v11.4s, v3.4s\n" - "mul v12.4s, v12.4s, v3.4s\n" - "mul v13.4s, v13.4s, v3.4s\n" + "mul v11.4s, v11.4s, v28.4s\n" + "mul v12.4s, v12.4s, v28.4s\n" + "mul v13.4s, v13.4s, v28.4s\n" "79:" // Height 3: skip row sum fixup - "ldr q0, [x16, #0x0]\n" + "ldr q31, [x15, #0x0]\n" "add v16.4s, v16.4s, v11.4s\n" - "ldr q1, [x16, #0x10]\n" + "ldr q30, [x15, #0x10]\n" "add v17.4s, v17.4s, v11.4s\n" - "ldr q2, [x16, #0x20]\n" + "ldr q29, [x15, #0x20]\n" "add v18.4s, v18.4s, v11.4s\n" - "ldr q3, [x16, #0x30]\n" + "ldr q28, [x15, #0x30]\n" "add v19.4s, v19.4s, v11.4s\n" "add v20.4s, v20.4s, v12.4s\n" "add v21.4s, v21.4s, v12.4s\n" @@ -1171,73 +1170,73 @@ void a64_hybrid_u8qa_dot_4x16_a55 ( "add v25.4s, v25.4s, v13.4s\n" "add v26.4s, v26.4s, v13.4s\n" "add v27.4s, v27.4s, v13.4s\n" - "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" - "add v20.4s, v20.4s, v0.4s\n" - "add v21.4s, v21.4s, v1.4s\n" - "add v22.4s, v22.4s, v2.4s\n" - "add v23.4s, v23.4s, v3.4s\n" - "add v24.4s, v24.4s, v0.4s\n" - "add v25.4s, v25.4s, v1.4s\n" - "add v26.4s, v26.4s, v2.4s\n" - "add v27.4s, v27.4s, v3.4s\n" - "add x23, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x23]\n" + "add v16.4s, v16.4s, v31.4s\n" + "add v17.4s, v17.4s, v30.4s\n" + "add v18.4s, v18.4s, v29.4s\n" + "add v19.4s, v19.4s, v28.4s\n" + "add v20.4s, v20.4s, v31.4s\n" + "add v21.4s, v21.4s, v30.4s\n" + "add v22.4s, v22.4s, v29.4s\n" + "add v23.4s, v23.4s, v28.4s\n" + "add v24.4s, v24.4s, v31.4s\n" + "add v25.4s, v25.4s, v30.4s\n" + "add v26.4s, v26.4s, v29.4s\n" + "add v27.4s, v27.4s, v28.4s\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v28.4s }, [x20]\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x23]\n" - "sqrdmulh v16.4s, v16.4s, v4.4s\n" - "sqrdmulh v17.4s, v17.4s, v4.4s\n" - "sqrdmulh v18.4s, v18.4s, v4.4s\n" - "sqrdmulh v19.4s, v19.4s, v4.4s\n" - "sqrdmulh v20.4s, v20.4s, v4.4s\n" - "sqrdmulh v21.4s, v21.4s, v4.4s\n" - "sqrdmulh v22.4s, v22.4s, v4.4s\n" - "sqrdmulh v23.4s, v23.4s, v4.4s\n" - "sqrdmulh v24.4s, v24.4s, v4.4s\n" - "sqrdmulh v25.4s, v25.4s, v4.4s\n" - "sqrdmulh v26.4s, v26.4s, v4.4s\n" - "sqrdmulh v27.4s, v27.4s, v4.4s\n" - "add x16, x16, #0x40\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "sqrdmulh v16.4s, v16.4s, v28.4s\n" + "sqrdmulh v17.4s, v17.4s, v28.4s\n" + "sqrdmulh v18.4s, v18.4s, v28.4s\n" + "sqrdmulh v19.4s, v19.4s, v28.4s\n" + "sqrdmulh v20.4s, v20.4s, v28.4s\n" + "sqrdmulh v21.4s, v21.4s, v28.4s\n" + "sqrdmulh v22.4s, v22.4s, v28.4s\n" + "sqrdmulh v23.4s, v23.4s, v28.4s\n" + "sqrdmulh v24.4s, v24.4s, v28.4s\n" + "sqrdmulh v25.4s, v25.4s, v28.4s\n" + "sqrdmulh v26.4s, v26.4s, v28.4s\n" + "sqrdmulh v27.4s, v27.4s, v28.4s\n" + "add x15, x15, #0x40\n" "tbz %x[flags], #5, 80f\n" - "and v4.16b, v16.16b, v0.16b\n" - "and v5.16b, v17.16b, v0.16b\n" - "and v6.16b, v18.16b, v0.16b\n" - "and v7.16b, v19.16b, v0.16b\n" - "and v8.16b, v20.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" - "sqadd v20.4s, v20.4s, v8.4s\n" - "and v9.16b, v21.16b, v0.16b\n" - "and v10.16b, v22.16b, v0.16b\n" - "and v4.16b, v23.16b, v0.16b\n" - "and v5.16b, v24.16b, v0.16b\n" - "and v6.16b, v25.16b, v0.16b\n" - "and v7.16b, v26.16b, v0.16b\n" - "and v8.16b, v27.16b, v0.16b\n" - "sshr v9.4s, v9.4s, #0x1f\n" - "sshr v10.4s, v10.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sqadd v21.4s, v21.4s, v9.4s\n" - "sqadd v22.4s, v22.4s, v10.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v24.4s, v24.4s, v5.4s\n" - "sqadd v25.4s, v25.4s, v6.4s\n" - "sqadd v26.4s, v26.4s, v7.4s\n" - "sqadd v27.4s, v27.4s, v8.4s\n" + "and v1.16b, v16.16b, v0.16b\n" + "and v31.16b, v17.16b, v0.16b\n" + "and v30.16b, v18.16b, v0.16b\n" + "and v29.16b, v19.16b, v0.16b\n" + "and v28.16b, v20.16b, v0.16b\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v1.4s\n" + "sqadd v17.4s, v17.4s, v31.4s\n" + "sqadd v18.4s, v18.4s, v30.4s\n" + "sqadd v19.4s, v19.4s, v29.4s\n" + "sqadd v20.4s, v20.4s, v28.4s\n" + "and v3.16b, v21.16b, v0.16b\n" + "and v2.16b, v22.16b, v0.16b\n" + "and v1.16b, v23.16b, v0.16b\n" + "and v31.16b, v24.16b, v0.16b\n" + "and v30.16b, v25.16b, v0.16b\n" + "and v29.16b, v26.16b, v0.16b\n" + "and v28.16b, v27.16b, v0.16b\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sqadd v21.4s, v21.4s, v3.4s\n" + "sqadd v22.4s, v22.4s, v2.4s\n" + "sqadd v23.4s, v23.4s, v1.4s\n" + "sqadd v24.4s, v24.4s, v31.4s\n" + "sqadd v25.4s, v25.4s, v30.4s\n" + "sqadd v26.4s, v26.4s, v29.4s\n" + "sqadd v27.4s, v27.4s, v28.4s\n" "80:" // Height 3: no shift correction "srshl v16.4s, v16.4s, v0.4s\n" "srshl v17.4s, v17.4s, v0.4s\n" @@ -1251,156 +1250,156 @@ void a64_hybrid_u8qa_dot_4x16_a55 ( "srshl v25.4s, v25.4s, v0.4s\n" "srshl v26.4s, v26.4s, v0.4s\n" "srshl v27.4s, v27.4s, v0.4s\n" - "add x23, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x23]\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v23.4s, v23.4s, v4.4s\n" - "add v24.4s, v24.4s, v4.4s\n" - "add v25.4s, v25.4s, v4.4s\n" - "add v26.4s, v26.4s, v4.4s\n" - "add v27.4s, v27.4s, v4.4s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x23]\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "smin v24.4s, v24.4s, v6.4s\n" - "smin v25.4s, v25.4s, v6.4s\n" - "smin v26.4s, v26.4s, v6.4s\n" - "smin v27.4s, v27.4s, v6.4s\n" - "add x23, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x23]\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" - "smax v24.4s, v24.4s, v5.4s\n" - "smax v25.4s, v25.4s, v5.4s\n" - "smax v26.4s, v26.4s, v5.4s\n" - "smax v27.4s, v27.4s, v5.4s\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v28.4s }, [x20]\n" + "add v16.4s, v16.4s, v28.4s\n" + "add v17.4s, v17.4s, v28.4s\n" + "add v18.4s, v18.4s, v28.4s\n" + "add v19.4s, v19.4s, v28.4s\n" + "add v20.4s, v20.4s, v28.4s\n" + "add v21.4s, v21.4s, v28.4s\n" + "add v22.4s, v22.4s, v28.4s\n" + "add v23.4s, v23.4s, v28.4s\n" + "add v24.4s, v24.4s, v28.4s\n" + "add v25.4s, v25.4s, v28.4s\n" + "add v26.4s, v26.4s, v28.4s\n" + "add v27.4s, v27.4s, v28.4s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v28.4s }, [x20]\n" + "smin v16.4s, v16.4s, v28.4s\n" + "smin v17.4s, v17.4s, v28.4s\n" + "smin v18.4s, v18.4s, v28.4s\n" + "smin v19.4s, v19.4s, v28.4s\n" + "smin v20.4s, v20.4s, v28.4s\n" + "smin v21.4s, v21.4s, v28.4s\n" + "smin v22.4s, v22.4s, v28.4s\n" + "smin v23.4s, v23.4s, v28.4s\n" + "smin v24.4s, v24.4s, v28.4s\n" + "smin v25.4s, v25.4s, v28.4s\n" + "smin v26.4s, v26.4s, v28.4s\n" + "smin v27.4s, v27.4s, v28.4s\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v28.4s }, [x20]\n" + "smax v16.4s, v16.4s, v28.4s\n" + "smax v17.4s, v17.4s, v28.4s\n" + "smax v18.4s, v18.4s, v28.4s\n" + "smax v19.4s, v19.4s, v28.4s\n" + "smax v20.4s, v20.4s, v28.4s\n" + "smax v21.4s, v21.4s, v28.4s\n" + "smax v22.4s, v22.4s, v28.4s\n" + "smax v23.4s, v23.4s, v28.4s\n" + "smax v24.4s, v24.4s, v28.4s\n" + "smax v25.4s, v25.4s, v28.4s\n" + "smax v26.4s, v26.4s, v28.4s\n" + "smax v27.4s, v27.4s, v28.4s\n" "uzp1 v16.8h, v16.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v19.8h, v18.8h, v19.8h\n" "uzp1 v20.8h, v20.8h, v21.8h\n" - "uzp1 v21.8h, v22.8h, v23.8h\n" + "uzp1 v18.8h, v22.8h, v23.8h\n" "uzp1 v24.8h, v24.8h, v25.8h\n" - "uzp1 v25.8h, v26.8h, v27.8h\n" - "cmp x15, #0x10\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" - "uzp1 v20.16b, v20.16b, v21.16b\n" - "uzp1 v24.16b, v24.16b, v25.16b\n" + "uzp1 v17.8h, v26.8h, v27.8h\n" + "cmp x14, #0x10\n" + "uzp1 v16.16b, v16.16b, v19.16b\n" + "uzp1 v20.16b, v20.16b, v18.16b\n" + "uzp1 v24.16b, v24.16b, v17.16b\n" "bge 89f\n" - "tbz x15, #3, 84f\n" - "str d16, [x14], #0x8\n" - "str d20, [x22], #0x8\n" - "str d24, [x21], #0x8\n" - "tbz x15, #2, 82f\n" - "st1 { v16.s }[2], [x14], #0x4\n" - "st1 { v20.s }[2], [x22], #0x4\n" - "st1 { v24.s }[2], [x21], #0x4\n" - "tbz x15, #1, 81f\n" - "st1 { v16.h }[6], [x14], #0x2\n" - "st1 { v20.h }[6], [x22], #0x2\n" - "st1 { v24.h }[6], [x21], #0x2\n" - "tbz x15, #0, 88f\n" - "st1 { v16.b }[14], [x14]\n" - "st1 { v20.b }[14], [x22]\n" - "st1 { v24.b }[14], [x21]\n" + "tbz x14, #3, 84f\n" + "str d16, [x13], #0x8\n" + "str d20, [x23], #0x8\n" + "str d24, [x22], #0x8\n" + "tbz x14, #2, 82f\n" + "st1 { v16.s }[2], [x13], #0x4\n" + "st1 { v20.s }[2], [x23], #0x4\n" + "st1 { v24.s }[2], [x22], #0x4\n" + "tbz x14, #1, 81f\n" + "st1 { v16.h }[6], [x13], #0x2\n" + "st1 { v20.h }[6], [x23], #0x2\n" + "st1 { v24.h }[6], [x22], #0x2\n" + "tbz x14, #0, 88f\n" + "st1 { v16.b }[14], [x13]\n" + "st1 { v20.b }[14], [x23]\n" + "st1 { v24.b }[14], [x22]\n" "b 88f\n" "81:" // Height 3: Partial direct writeback: partial_1_12 - "tbz x15, #0, 88f\n" - "st1 { v16.b }[12], [x14]\n" - "st1 { v20.b }[12], [x22]\n" - "st1 { v24.b }[12], [x21]\n" + "tbz x14, #0, 88f\n" + "st1 { v16.b }[12], [x13]\n" + "st1 { v20.b }[12], [x23]\n" + "st1 { v24.b }[12], [x22]\n" "b 88f\n" "82:" // Height 3: Partial direct writeback: partial_2_8 - "tbz x15, #1, 83f\n" - "st1 { v16.h }[4], [x14], #0x2\n" - "st1 { v20.h }[4], [x22], #0x2\n" - "st1 { v24.h }[4], [x21], #0x2\n" - "tbz x15, #0, 88f\n" - "st1 { v16.b }[10], [x14]\n" - "st1 { v20.b }[10], [x22]\n" - "st1 { v24.b }[10], [x21]\n" + "tbz x14, #1, 83f\n" + "st1 { v16.h }[4], [x13], #0x2\n" + "st1 { v20.h }[4], [x23], #0x2\n" + "st1 { v24.h }[4], [x22], #0x2\n" + "tbz x14, #0, 88f\n" + "st1 { v16.b }[10], [x13]\n" + "st1 { v20.b }[10], [x23]\n" + "st1 { v24.b }[10], [x22]\n" "b 88f\n" "83:" // Height 3: Partial direct writeback: partial_1_8 - "tbz x15, #0, 88f\n" - "st1 { v16.b }[8], [x14]\n" - "st1 { v20.b }[8], [x22]\n" - "st1 { v24.b }[8], [x21]\n" + "tbz x14, #0, 88f\n" + "st1 { v16.b }[8], [x13]\n" + "st1 { v20.b }[8], [x23]\n" + "st1 { v24.b }[8], [x22]\n" "b 88f\n" "84:" // Height 3: Partial direct writeback: partial_4_0 - "tbz x15, #2, 86f\n" - "str s16, [x14], #0x4\n" - "str s20, [x22], #0x4\n" - "str s24, [x21], #0x4\n" - "tbz x15, #1, 85f\n" - "st1 { v16.h }[2], [x14], #0x2\n" - "st1 { v20.h }[2], [x22], #0x2\n" - "st1 { v24.h }[2], [x21], #0x2\n" - "tbz x15, #0, 88f\n" - "st1 { v16.b }[6], [x14]\n" - "st1 { v20.b }[6], [x22]\n" - "st1 { v24.b }[6], [x21]\n" + "tbz x14, #2, 86f\n" + "str s16, [x13], #0x4\n" + "str s20, [x23], #0x4\n" + "str s24, [x22], #0x4\n" + "tbz x14, #1, 85f\n" + "st1 { v16.h }[2], [x13], #0x2\n" + "st1 { v20.h }[2], [x23], #0x2\n" + "st1 { v24.h }[2], [x22], #0x2\n" + "tbz x14, #0, 88f\n" + "st1 { v16.b }[6], [x13]\n" + "st1 { v20.b }[6], [x23]\n" + "st1 { v24.b }[6], [x22]\n" "b 88f\n" "85:" // Height 3: Partial direct writeback: partial_1_4 - "tbz x15, #0, 88f\n" - "st1 { v16.b }[4], [x14]\n" - "st1 { v20.b }[4], [x22]\n" - "st1 { v24.b }[4], [x21]\n" + "tbz x14, #0, 88f\n" + "st1 { v16.b }[4], [x13]\n" + "st1 { v20.b }[4], [x23]\n" + "st1 { v24.b }[4], [x22]\n" "b 88f\n" "86:" // Height 3: Partial direct writeback: partial_2_0 - "tbz x15, #1, 87f\n" - "str h16, [x14], #0x2\n" - "str h20, [x22], #0x2\n" - "str h24, [x21], #0x2\n" - "tbz x15, #0, 88f\n" - "st1 { v16.b }[2], [x14]\n" - "st1 { v20.b }[2], [x22]\n" - "st1 { v24.b }[2], [x21]\n" + "tbz x14, #1, 87f\n" + "str h16, [x13], #0x2\n" + "str h20, [x23], #0x2\n" + "str h24, [x22], #0x2\n" + "tbz x14, #0, 88f\n" + "st1 { v16.b }[2], [x13]\n" + "st1 { v20.b }[2], [x23]\n" + "st1 { v24.b }[2], [x22]\n" "b 88f\n" "87:" // Height 3: Partial direct writeback: partial_1_0 - "str b16, [x14, #0x0]\n" - "str b20, [x22, #0x0]\n" - "str b24, [x21, #0x0]\n" + "str b16, [x13, #0x0]\n" + "str b20, [x23, #0x0]\n" + "str b24, [x22, #0x0]\n" "88:" // Height 3: Partial direct writeback: Done "b 90f\n" "89:" // Height 3: Full writeback - "str q16, [x14, #0x0]\n" - "add x14, x14, #0x10\n" - "str q20, [x22, #0x0]\n" - "str q24, [x21, #0x0]\n" + "str q16, [x13, #0x0]\n" + "add x13, x13, #0x10\n" + "str q20, [x23, #0x0]\n" + "str q24, [x22, #0x0]\n" "90:" // Height 3: Writeback done - "subs x15, x15, #0x10\n" + "subs x14, x14, #0x10\n" "bgt 62b\n" "b 122f\n" "91:" // Height 4 "ldr x21, [%x[args_ptr], %[offsetof_output_offset]]\n" "mov x20, #0x4\n" - "mov x16, %x[col_bias]\n" + "mov x15, %x[col_bias]\n" "movi v11.4s, #0x0\n" "movi v12.4s, #0x0\n" "bic %x[flags], %x[flags], #0x80000000\n" "movi v13.4s, #0x0\n" - "ldr x15, [%x[args_ptr], %[offsetof_N]]\n" + "ldr x14, [%x[args_ptr], %[offsetof_N]]\n" "movi v14.4s, #0x0\n" - "ldr x13, [%x[args_ptr], %[offsetof_B_ptr]]\n" + "ldr x12, [%x[args_ptr], %[offsetof_B_ptr]]\n" "movi v15.16b, #0x1\n" - "mov x14, %x[output_ptr]\n" + "mov x13, %x[output_ptr]\n" "madd %x[output_ptr], x21, x20, %x[output_ptr]\n" "92:" // Height 4: Column loop "movi v16.4s, #0x0\n" @@ -1420,117 +1419,117 @@ void a64_hybrid_u8qa_dot_4x16_a55 ( "movi v30.4s, #0x0\n" "movi v31.4s, #0x0\n" "93:" // Height 4: setup done - "mov x12, #0x0\n" + "mov x11, #0x0\n" "94:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr w11, [x20, x12, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr w10, [x20, x11, LSL #0x2]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 95f\n" - "ldr x21, [%x[input_ptr], x12, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x10, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" - "ldr x21, [x21, #0x18]\n" - "cbnz x12, 96f\n" + "ldr x20, [%x[input_ptr], x11, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x9, [x20, #0x0]\n" + "ldr x28, [x20, #0x8]\n" + "ldr x27, [x20, #0x10]\n" + "ldr x26, [x20, #0x18]\n" + "cbnz x11, 96f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x10, x10, x20\n" - "add x23, x23, x20\n" - "add x22, x22, x20\n" - "add x21, x21, x20\n" + "add x9, x9, x20\n" + "add x28, x28, x20\n" + "add x27, x27, x20\n" + "add x26, x26, x20\n" "b 96f\n" "95:" // Height 4: setup direct input - "mov x10, %x[input_ptr]\n" - "add x23, x10, x20\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" + "mov x9, %x[input_ptr]\n" + "add x28, x9, x21\n" + "add x27, x28, x21\n" + "add x26, x27, x21\n" "96:" // Height 4: input setup done - "cmp x11, #0x10\n" + "cmp x10, #0x10\n" "blt 101f\n" - "ldr q0, [x10, #0x0]\n" - "cmp x11, #0x20\n" - "ldr q1, [x23, #0x0]\n" - "ldr q2, [x22, #0x0]\n" - "ldr q3, [x21, #0x0]\n" - "ldr q4, [x13, #0x0]\n" - "ldr q5, [x13, #0x10]\n" - "ldr q6, [x13, #0x20]\n" - "ldr q7, [x13, #0x30]\n" - "ldr q8, [x13, #0x40]\n" - "ldr q9, [x13, #0x50]\n" - "ldr q10, [x13, #0x60]\n" + "ldr q0, [x9, #0x0]\n" + "cmp x10, #0x20\n" + "ldr q1, [x28, #0x0]\n" + "ldr q2, [x27, #0x0]\n" + "ldr q3, [x26, #0x0]\n" + "ldr q4, [x12, #0x0]\n" + "ldr q5, [x12, #0x10]\n" + "ldr q6, [x12, #0x20]\n" + "ldr q7, [x12, #0x30]\n" + "ldr q8, [x12, #0x40]\n" + "ldr q9, [x12, #0x50]\n" + "ldr q10, [x12, #0x60]\n" "blt 99f\n" "97:" // Height 4: Multiply loop: Main loop head ".inst 0x6f80e090 // udot v16.4s, v4.16b, v0.4b[0]\n" - "ldr x9, [x13, #0x78]\n" + "ldr x22, [x12, #0x78]\n" ".inst 0x6f81e094 // udot v20.4s, v4.16b, v1.4b[0]\n" - "ldr x28, [x13, #0x88]\n" + "ldr x21, [x12, #0x88]\n" ".inst 0x6f82e098 // udot v24.4s, v4.16b, v2.4b[0]\n" - "ldr x27, [x13, #0x98]\n" + "ldr x20, [x12, #0x98]\n" ".inst 0x6f83e09c // udot v28.4s, v4.16b, v3.4b[0]\n" - "ldr d4, [x13, #0x70]\n" + "ldr d4, [x12, #0x70]\n" ".inst 0x6f80e0b1 // udot v17.4s, v5.16b, v0.4b[0]\n" - "mov v4.d[1], x9\n" + "mov v4.d[1], x22\n" ".inst 0x6f81e0b5 // udot v21.4s, v5.16b, v1.4b[0]\n" - "ldr x26, [x13, #0xa8]\n" + "ldr x25, [x12, #0xa8]\n" ".inst 0x6f82e0b9 // udot v25.4s, v5.16b, v2.4b[0]\n" - "ldr x25, [x13, #0xb8]\n" + "ldr x24, [x12, #0xb8]\n" ".inst 0x6f83e0bd // udot v29.4s, v5.16b, v3.4b[0]\n" - "ldr d5, [x13, #0x80]\n" + "ldr d5, [x12, #0x80]\n" ".inst 0x6f80e0d2 // udot v18.4s, v6.16b, v0.4b[0]\n" - "mov v5.d[1], x28\n" + "mov v5.d[1], x21\n" ".inst 0x6f81e0d6 // udot v22.4s, v6.16b, v1.4b[0]\n" - "ldr x24, [x13, #0xc8]\n" + "ldr x23, [x12, #0xc8]\n" ".inst 0x6f82e0da // udot v26.4s, v6.16b, v2.4b[0]\n" - "ldr x20, [x13, #0xd8]\n" + "ldr x22, [x12, #0xd8]\n" ".inst 0x6f83e0de // udot v30.4s, v6.16b, v3.4b[0]\n" - "ldr d6, [x13, #0x90]\n" + "ldr d6, [x12, #0x90]\n" ".inst 0x6f80e0f3 // udot v19.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x27\n" + "mov v6.d[1], x20\n" ".inst 0x6f81e0f7 // udot v23.4s, v7.16b, v1.4b[0]\n" - "ldr x9, [x13, #0xe8]\n" + "ldr x21, [x12, #0xe8]\n" ".inst 0x6f82e0fb // udot v27.4s, v7.16b, v2.4b[0]\n" - "ldr x28, [x13, #0xf8]\n" + "ldr x20, [x12, #0xf8]\n" ".inst 0x6f83e0ff // udot v31.4s, v7.16b, v3.4b[0]\n" - "ldr d7, [x13, #0xa0]\n" + "ldr d7, [x12, #0xa0]\n" ".inst 0x6fa0e110 // udot v16.4s, v8.16b, v0.4b[1]\n" - "mov v7.d[1], x26\n" + "mov v7.d[1], x25\n" ".inst 0x6fa1e114 // udot v20.4s, v8.16b, v1.4b[1]\n" - "add x10, x10, #0x10\n" + "add x9, x9, #0x10\n" ".inst 0x6fa2e118 // udot v24.4s, v8.16b, v2.4b[1]\n" - "add x23, x23, #0x10\n" + "add x28, x28, #0x10\n" ".inst 0x6fa3e11c // udot v28.4s, v8.16b, v3.4b[1]\n" - "ldr d8, [x13, #0xb0]\n" + "ldr d8, [x12, #0xb0]\n" ".inst 0x6fa0e131 // udot v17.4s, v9.16b, v0.4b[1]\n" - "mov v8.d[1], x25\n" + "mov v8.d[1], x24\n" ".inst 0x6fa1e135 // udot v21.4s, v9.16b, v1.4b[1]\n" - "add x22, x22, #0x10\n" + "add x27, x27, #0x10\n" ".inst 0x6fa2e139 // udot v25.4s, v9.16b, v2.4b[1]\n" - "add x21, x21, #0x10\n" + "add x26, x26, #0x10\n" ".inst 0x6fa3e13d // udot v29.4s, v9.16b, v3.4b[1]\n" - "ldr d9, [x13, #0xc0]\n" + "ldr d9, [x12, #0xc0]\n" ".inst 0x6fa0e152 // udot v18.4s, v10.16b, v0.4b[1]\n" - "mov v9.d[1], x24\n" + "mov v9.d[1], x23\n" ".inst 0x6fa1e156 // udot v22.4s, v10.16b, v1.4b[1]\n" ".inst 0x6fa2e15a // udot v26.4s, v10.16b, v2.4b[1]\n" ".inst 0x6fa3e15e // udot v30.4s, v10.16b, v3.4b[1]\n" - "ldr d10, [x13, #0xd0]\n" + "ldr d10, [x12, #0xd0]\n" ".inst 0x6fa0e093 // udot v19.4s, v4.16b, v0.4b[1]\n" - "mov v10.d[1], x20\n" + "mov v10.d[1], x22\n" ".inst 0x6fa1e097 // udot v23.4s, v4.16b, v1.4b[1]\n" ".inst 0x6fa2e09b // udot v27.4s, v4.16b, v2.4b[1]\n" ".inst 0x6fa3e09f // udot v31.4s, v4.16b, v3.4b[1]\n" - "ldr d4, [x13, #0xe0]\n" + "ldr d4, [x12, #0xe0]\n" ".inst 0x6f80e8b0 // udot v16.4s, v5.16b, v0.4b[2]\n" - "mov v4.d[1], x9\n" + "mov v4.d[1], x21\n" ".inst 0x6f81e8b4 // udot v20.4s, v5.16b, v1.4b[2]\n" ".inst 0x6f82e8b8 // udot v24.4s, v5.16b, v2.4b[2]\n" ".inst 0x6f83e8bc // udot v28.4s, v5.16b, v3.4b[2]\n" - "ldr d5, [x13, #0xf0]\n" + "ldr d5, [x12, #0xf0]\n" ".inst 0x6f80e8d1 // udot v17.4s, v6.16b, v0.4b[2]\n" - "mov v5.d[1], x28\n" + "mov v5.d[1], x20\n" ".inst 0x6f81e8d5 // udot v21.4s, v6.16b, v1.4b[2]\n" - "add x13, x13, #0x100\n" + "add x12, x12, #0x100\n" ".inst 0x6f82e8d9 // udot v25.4s, v6.16b, v2.4b[2]\n" ".inst 0x6f83e8dd // udot v29.4s, v6.16b, v3.4b[2]\n" ".inst 0x6f80e8f2 // udot v18.4s, v7.16b, v0.4b[2]\n" @@ -1563,77 +1562,77 @@ void a64_hybrid_u8qa_dot_4x16_a55 ( ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n" ".inst 0x6e8f946e // udot v14.4s, v3.16b, v15.16b\n" "98:" // Height 4: Multiply loop: unique 13: skip row sum - "ldr q0, [x10, #0x0]\n" - "sub x11, x11, #0x10\n" - "ldr q1, [x23, #0x0]\n" - "cmp x11, #0x20\n" - "ldr q2, [x22, #0x0]\n" - "ldr q3, [x21, #0x0]\n" - "ldr q4, [x13, #0x0]\n" - "ldr q5, [x13, #0x10]\n" - "ldr q6, [x13, #0x20]\n" - "ldr q7, [x13, #0x30]\n" - "ldr q8, [x13, #0x40]\n" - "ldr q9, [x13, #0x50]\n" - "ldr q10, [x13, #0x60]\n" - "prfm pldl1keep, [x10, #0x80]\n" - "prfm pldl1keep, [x23, #0x80]\n" - "prfm pldl1keep, [x22, #0x80]\n" - "prfm pldl1keep, [x21, #0x80]\n" + "ldr q0, [x9, #0x0]\n" + "sub x10, x10, #0x10\n" + "ldr q1, [x28, #0x0]\n" + "cmp x10, #0x20\n" + "ldr q2, [x27, #0x0]\n" + "ldr q3, [x26, #0x0]\n" + "ldr q4, [x12, #0x0]\n" + "ldr q5, [x12, #0x10]\n" + "ldr q6, [x12, #0x20]\n" + "ldr q7, [x12, #0x30]\n" + "ldr q8, [x12, #0x40]\n" + "ldr q9, [x12, #0x50]\n" + "ldr q10, [x12, #0x60]\n" + "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x28, #0x80]\n" + "prfm pldl1keep, [x27, #0x80]\n" + "prfm pldl1keep, [x26, #0x80]\n" "bge 97b\n" "99:" // Height 4: Multiply loop: Single iteration only ".inst 0x6f80e090 // udot v16.4s, v4.16b, v0.4b[0]\n" - "sub x11, x11, #0x10\n" + "sub x10, x10, #0x10\n" ".inst 0x6f81e094 // udot v20.4s, v4.16b, v1.4b[0]\n" - "add x10, x10, #0x10\n" + "add x9, x9, #0x10\n" ".inst 0x6f82e098 // udot v24.4s, v4.16b, v2.4b[0]\n" - "add x23, x23, #0x10\n" + "add x28, x28, #0x10\n" ".inst 0x6f83e09c // udot v28.4s, v4.16b, v3.4b[0]\n" - "ldr q4, [x13, #0x70]\n" + "ldr q4, [x12, #0x70]\n" ".inst 0x6f80e0b1 // udot v17.4s, v5.16b, v0.4b[0]\n" - "add x22, x22, #0x10\n" + "add x27, x27, #0x10\n" ".inst 0x6f81e0b5 // udot v21.4s, v5.16b, v1.4b[0]\n" - "add x21, x21, #0x10\n" + "add x26, x26, #0x10\n" ".inst 0x6f82e0b9 // udot v25.4s, v5.16b, v2.4b[0]\n" ".inst 0x6f83e0bd // udot v29.4s, v5.16b, v3.4b[0]\n" - "ldr q5, [x13, #0x80]\n" + "ldr q5, [x12, #0x80]\n" ".inst 0x6f80e0d2 // udot v18.4s, v6.16b, v0.4b[0]\n" ".inst 0x6f81e0d6 // udot v22.4s, v6.16b, v1.4b[0]\n" ".inst 0x6f82e0da // udot v26.4s, v6.16b, v2.4b[0]\n" ".inst 0x6f83e0de // udot v30.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x13, #0x90]\n" + "ldr q6, [x12, #0x90]\n" ".inst 0x6f80e0f3 // udot v19.4s, v7.16b, v0.4b[0]\n" ".inst 0x6f81e0f7 // udot v23.4s, v7.16b, v1.4b[0]\n" ".inst 0x6f82e0fb // udot v27.4s, v7.16b, v2.4b[0]\n" ".inst 0x6f83e0ff // udot v31.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x13, #0xa0]\n" + "ldr q7, [x12, #0xa0]\n" ".inst 0x6fa0e110 // udot v16.4s, v8.16b, v0.4b[1]\n" ".inst 0x6fa1e114 // udot v20.4s, v8.16b, v1.4b[1]\n" ".inst 0x6fa2e118 // udot v24.4s, v8.16b, v2.4b[1]\n" ".inst 0x6fa3e11c // udot v28.4s, v8.16b, v3.4b[1]\n" - "ldr q8, [x13, #0xb0]\n" + "ldr q8, [x12, #0xb0]\n" ".inst 0x6fa0e131 // udot v17.4s, v9.16b, v0.4b[1]\n" ".inst 0x6fa1e135 // udot v21.4s, v9.16b, v1.4b[1]\n" ".inst 0x6fa2e139 // udot v25.4s, v9.16b, v2.4b[1]\n" ".inst 0x6fa3e13d // udot v29.4s, v9.16b, v3.4b[1]\n" - "ldr q9, [x13, #0xc0]\n" + "ldr q9, [x12, #0xc0]\n" ".inst 0x6fa0e152 // udot v18.4s, v10.16b, v0.4b[1]\n" ".inst 0x6fa1e156 // udot v22.4s, v10.16b, v1.4b[1]\n" ".inst 0x6fa2e15a // udot v26.4s, v10.16b, v2.4b[1]\n" ".inst 0x6fa3e15e // udot v30.4s, v10.16b, v3.4b[1]\n" - "ldr q10, [x13, #0xd0]\n" + "ldr q10, [x12, #0xd0]\n" ".inst 0x6fa0e093 // udot v19.4s, v4.16b, v0.4b[1]\n" ".inst 0x6fa1e097 // udot v23.4s, v4.16b, v1.4b[1]\n" ".inst 0x6fa2e09b // udot v27.4s, v4.16b, v2.4b[1]\n" ".inst 0x6fa3e09f // udot v31.4s, v4.16b, v3.4b[1]\n" - "ldr q4, [x13, #0xe0]\n" + "ldr q4, [x12, #0xe0]\n" ".inst 0x6f80e8b0 // udot v16.4s, v5.16b, v0.4b[2]\n" ".inst 0x6f81e8b4 // udot v20.4s, v5.16b, v1.4b[2]\n" ".inst 0x6f82e8b8 // udot v24.4s, v5.16b, v2.4b[2]\n" ".inst 0x6f83e8bc // udot v28.4s, v5.16b, v3.4b[2]\n" - "ldr q5, [x13, #0xf0]\n" + "ldr q5, [x12, #0xf0]\n" ".inst 0x6f80e8d1 // udot v17.4s, v6.16b, v0.4b[2]\n" - "add x13, x13, #0x100\n" + "add x12, x12, #0x100\n" ".inst 0x6f81e8d5 // udot v21.4s, v6.16b, v1.4b[2]\n" ".inst 0x6f82e8d9 // udot v25.4s, v6.16b, v2.4b[2]\n" ".inst 0x6f83e8dd // udot v29.4s, v6.16b, v3.4b[2]\n" @@ -1667,67 +1666,67 @@ void a64_hybrid_u8qa_dot_4x16_a55 ( ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n" ".inst 0x6e8f946e // udot v14.4s, v3.16b, v15.16b\n" "100:" // Height 4: Multiply loop: unique 14: skip row sum - "prfm pldl1keep, [x10, #0x80]\n" - "prfm pldl1keep, [x23, #0x80]\n" - "prfm pldl1keep, [x22, #0x80]\n" - "prfm pldl1keep, [x21, #0x80]\n" + "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x28, #0x80]\n" + "prfm pldl1keep, [x27, #0x80]\n" + "prfm pldl1keep, [x26, #0x80]\n" "101:" // Height 4: Multiply loop: Main loop skip - "cbz x11, 108f\n" - "cmp x11, #0x4\n" + "cbz x10, 108f\n" + "cmp x10, #0x4\n" "blt 104f\n" "102:" // Height 4: Multiply loop: Odd block loop - "ldr s0, [x10], #0x4\n" - "ldr s1, [x23], #0x4\n" - "ldr s2, [x22], #0x4\n" - "ldr s3, [x21], #0x4\n" + "ldr s0, [x9], #0x4\n" + "ldr s1, [x28], #0x4\n" + "ldr s2, [x27], #0x4\n" + "ldr s3, [x26], #0x4\n" "tbnz %x[flags], #31, 103f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n" ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n" ".inst 0x6e8f946e // udot v14.4s, v3.16b, v15.16b\n" "103:" // Height 4: Multiply loop: unique 15: skip row sum - "ldr q6, [x13, #0x0]\n" - "sub x11, x11, #0x4\n" - "ldr q7, [x13, #0x10]\n" - "cmp x11, #0x4\n" - "ldr q8, [x13, #0x20]\n" - ".inst 0x6f80e0d0 // udot v16.4s, v6.16b, v0.4b[0]\n" - "ldr q9, [x13, #0x30]\n" - ".inst 0x6f81e0d4 // udot v20.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d8 // udot v24.4s, v6.16b, v2.4b[0]\n" - "add x13, x13, #0x40\n" - ".inst 0x6f83e0dc // udot v28.4s, v6.16b, v3.4b[0]\n" - ".inst 0x6f80e0f1 // udot v17.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0f5 // udot v21.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f9 // udot v25.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0fd // udot v29.4s, v7.16b, v3.4b[0]\n" - ".inst 0x6f80e112 // udot v18.4s, v8.16b, v0.4b[0]\n" - ".inst 0x6f81e116 // udot v22.4s, v8.16b, v1.4b[0]\n" - ".inst 0x6f82e11a // udot v26.4s, v8.16b, v2.4b[0]\n" - ".inst 0x6f83e11e // udot v30.4s, v8.16b, v3.4b[0]\n" - ".inst 0x6f80e133 // udot v19.4s, v9.16b, v0.4b[0]\n" - ".inst 0x6f81e137 // udot v23.4s, v9.16b, v1.4b[0]\n" - ".inst 0x6f82e13b // udot v27.4s, v9.16b, v2.4b[0]\n" - ".inst 0x6f83e13f // udot v31.4s, v9.16b, v3.4b[0]\n" + "ldr q7, [x12, #0x0]\n" + "sub x10, x10, #0x4\n" + "ldr q6, [x12, #0x10]\n" + "cmp x10, #0x4\n" + "ldr q5, [x12, #0x20]\n" + ".inst 0x6f80e0f0 // udot v16.4s, v7.16b, v0.4b[0]\n" + "ldr q4, [x12, #0x30]\n" + ".inst 0x6f81e0f4 // udot v20.4s, v7.16b, v1.4b[0]\n" + ".inst 0x6f82e0f8 // udot v24.4s, v7.16b, v2.4b[0]\n" + "add x12, x12, #0x40\n" + ".inst 0x6f83e0fc // udot v28.4s, v7.16b, v3.4b[0]\n" + ".inst 0x6f80e0d1 // udot v17.4s, v6.16b, v0.4b[0]\n" + ".inst 0x6f81e0d5 // udot v21.4s, v6.16b, v1.4b[0]\n" + ".inst 0x6f82e0d9 // udot v25.4s, v6.16b, v2.4b[0]\n" + ".inst 0x6f83e0dd // udot v29.4s, v6.16b, v3.4b[0]\n" + ".inst 0x6f80e0b2 // udot v18.4s, v5.16b, v0.4b[0]\n" + ".inst 0x6f81e0b6 // udot v22.4s, v5.16b, v1.4b[0]\n" + ".inst 0x6f82e0ba // udot v26.4s, v5.16b, v2.4b[0]\n" + ".inst 0x6f83e0be // udot v30.4s, v5.16b, v3.4b[0]\n" + ".inst 0x6f80e093 // udot v19.4s, v4.16b, v0.4b[0]\n" + ".inst 0x6f81e097 // udot v23.4s, v4.16b, v1.4b[0]\n" + ".inst 0x6f82e09b // udot v27.4s, v4.16b, v2.4b[0]\n" + ".inst 0x6f83e09f // udot v31.4s, v4.16b, v3.4b[0]\n" "bge 102b\n" "104:" // Height 4: Multiply loop: Skip odd blocks - "cbz x11, 108f\n" - "tbz x11, #1, 105f\n" - "ldr h0, [x10], #0x2\n" - "ldr h1, [x23], #0x2\n" - "ldr h2, [x22], #0x2\n" - "ldr h3, [x21], #0x2\n" - "tbz x11, #0, 106f\n" - "ld1 { v0.b }[2], [x10]\n" - "ld1 { v1.b }[2], [x23]\n" - "ld1 { v2.b }[2], [x22]\n" - "ld1 { v3.b }[2], [x21]\n" + "cbz x10, 108f\n" + "tbz x10, #1, 105f\n" + "ldr h0, [x9], #0x2\n" + "ldr h1, [x28], #0x2\n" + "ldr h2, [x27], #0x2\n" + "ldr h3, [x26], #0x2\n" + "tbz x10, #0, 106f\n" + "ld1 { v0.b }[2], [x9]\n" + "ld1 { v1.b }[2], [x28]\n" + "ld1 { v2.b }[2], [x27]\n" + "ld1 { v3.b }[2], [x26]\n" "b 106f\n" "105:" // Height 4: Multiply loop: Ragged operand read: partial_1_0 - "ldr b0, [x10, #0x0]\n" - "ldr b1, [x23, #0x0]\n" - "ldr b2, [x22, #0x0]\n" - "ldr b3, [x21, #0x0]\n" + "ldr b0, [x9, #0x0]\n" + "ldr b1, [x28, #0x0]\n" + "ldr b2, [x27, #0x0]\n" + "ldr b3, [x26, #0x0]\n" "106:" // Height 4: Multiply loop: Ragged operand read: Done "tbnz %x[flags], #31, 107f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" @@ -1735,64 +1734,64 @@ void a64_hybrid_u8qa_dot_4x16_a55 ( ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n" ".inst 0x6e8f946e // udot v14.4s, v3.16b, v15.16b\n" "107:" // Height 4: Multiply loop: unique 16: skip row sum - "ldr q10, [x13, #0x0]\n" - ".inst 0x6f80e150 // udot v16.4s, v10.16b, v0.4b[0]\n" - "ldr q4, [x13, #0x10]\n" - ".inst 0x6f81e154 // udot v20.4s, v10.16b, v1.4b[0]\n" - "ldr q5, [x13, #0x20]\n" - ".inst 0x6f82e158 // udot v24.4s, v10.16b, v2.4b[0]\n" - "ldr q6, [x13, #0x30]\n" - ".inst 0x6f83e15c // udot v28.4s, v10.16b, v3.4b[0]\n" - ".inst 0x6f80e091 // udot v17.4s, v4.16b, v0.4b[0]\n" - "add x13, x13, #0x40\n" - ".inst 0x6f81e095 // udot v21.4s, v4.16b, v1.4b[0]\n" - ".inst 0x6f82e099 // udot v25.4s, v4.16b, v2.4b[0]\n" - ".inst 0x6f83e09d // udot v29.4s, v4.16b, v3.4b[0]\n" + "ldr q7, [x12, #0x0]\n" + ".inst 0x6f80e0f0 // udot v16.4s, v7.16b, v0.4b[0]\n" + "ldr q6, [x12, #0x10]\n" + ".inst 0x6f81e0f4 // udot v20.4s, v7.16b, v1.4b[0]\n" + "ldr q5, [x12, #0x20]\n" + ".inst 0x6f82e0f8 // udot v24.4s, v7.16b, v2.4b[0]\n" + "ldr q4, [x12, #0x30]\n" + ".inst 0x6f83e0fc // udot v28.4s, v7.16b, v3.4b[0]\n" + ".inst 0x6f80e0d1 // udot v17.4s, v6.16b, v0.4b[0]\n" + "add x12, x12, #0x40\n" + ".inst 0x6f81e0d5 // udot v21.4s, v6.16b, v1.4b[0]\n" + ".inst 0x6f82e0d9 // udot v25.4s, v6.16b, v2.4b[0]\n" + ".inst 0x6f83e0dd // udot v29.4s, v6.16b, v3.4b[0]\n" ".inst 0x6f80e0b2 // udot v18.4s, v5.16b, v0.4b[0]\n" ".inst 0x6f81e0b6 // udot v22.4s, v5.16b, v1.4b[0]\n" ".inst 0x6f82e0ba // udot v26.4s, v5.16b, v2.4b[0]\n" ".inst 0x6f83e0be // udot v30.4s, v5.16b, v3.4b[0]\n" - ".inst 0x6f80e0d3 // udot v19.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0d7 // udot v23.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0db // udot v27.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0df // udot v31.4s, v6.16b, v3.4b[0]\n" + ".inst 0x6f80e093 // udot v19.4s, v4.16b, v0.4b[0]\n" + ".inst 0x6f81e097 // udot v23.4s, v4.16b, v1.4b[0]\n" + ".inst 0x6f82e09b // udot v27.4s, v4.16b, v2.4b[0]\n" + ".inst 0x6f83e09f // udot v31.4s, v4.16b, v3.4b[0]\n" "108:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" - "add x12, x12, #0x1\n" - "cmp x12, x20\n" + "add x11, x11, #0x1\n" + "cmp x11, x20\n" "bne 94b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x22, x14, x20\n" + "add x23, x13, x20\n" + "add x22, x23, x20\n" "add x21, x22, x20\n" - "add x20, x21, x20\n" - "prfm pstl1keep, [x14, #0x0]\n" + "prfm pstl1keep, [x13, #0x0]\n" + "prfm pstl1keep, [x23, #0x0]\n" "prfm pstl1keep, [x22, #0x0]\n" "prfm pstl1keep, [x21, #0x0]\n" - "prfm pstl1keep, [x20, #0x0]\n" "tbnz %x[flags], #31, 109f\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v12.4s, v12.4s, v12.4s\n" "addp v13.4s, v13.4s, v13.4s\n" "addp v14.4s, v14.4s, v14.4s\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1r { v4.4s }, [x23]\n" - "neg v4.4s, v4.4s\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1r { v0.4s }, [x20]\n" + "neg v0.4s, v0.4s\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v12.4s, v12.4s, v12.4s\n" "addp v13.4s, v13.4s, v13.4s\n" "addp v14.4s, v14.4s, v14.4s\n" - "mul v11.4s, v11.4s, v4.4s\n" - "mul v12.4s, v12.4s, v4.4s\n" - "mul v13.4s, v13.4s, v4.4s\n" - "mul v14.4s, v14.4s, v4.4s\n" + "mul v11.4s, v11.4s, v0.4s\n" + "mul v12.4s, v12.4s, v0.4s\n" + "mul v13.4s, v13.4s, v0.4s\n" + "mul v14.4s, v14.4s, v0.4s\n" "109:" // Height 4: skip row sum fixup - "ldr q0, [x16, #0x0]\n" + "ldr q3, [x15, #0x0]\n" "add v16.4s, v16.4s, v11.4s\n" - "ldr q1, [x16, #0x10]\n" + "ldr q2, [x15, #0x10]\n" "add v17.4s, v17.4s, v11.4s\n" - "ldr q2, [x16, #0x20]\n" + "ldr q1, [x15, #0x20]\n" "add v18.4s, v18.4s, v11.4s\n" - "ldr q3, [x16, #0x30]\n" + "ldr q0, [x15, #0x30]\n" "add v19.4s, v19.4s, v11.4s\n" "add v20.4s, v20.4s, v12.4s\n" "add v21.4s, v21.4s, v12.4s\n" @@ -1806,93 +1805,93 @@ void a64_hybrid_u8qa_dot_4x16_a55 ( "add v29.4s, v29.4s, v14.4s\n" "add v30.4s, v30.4s, v14.4s\n" "add v31.4s, v31.4s, v14.4s\n" - "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" - "add v20.4s, v20.4s, v0.4s\n" - "add v21.4s, v21.4s, v1.4s\n" - "add v22.4s, v22.4s, v2.4s\n" - "add v23.4s, v23.4s, v3.4s\n" - "add v24.4s, v24.4s, v0.4s\n" - "add v25.4s, v25.4s, v1.4s\n" - "add v26.4s, v26.4s, v2.4s\n" - "add v27.4s, v27.4s, v3.4s\n" - "add v28.4s, v28.4s, v0.4s\n" - "add v29.4s, v29.4s, v1.4s\n" - "add v30.4s, v30.4s, v2.4s\n" - "add v31.4s, v31.4s, v3.4s\n" - "add x23, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x23]\n" + "add v16.4s, v16.4s, v3.4s\n" + "add v17.4s, v17.4s, v2.4s\n" + "add v18.4s, v18.4s, v1.4s\n" + "add v19.4s, v19.4s, v0.4s\n" + "add v20.4s, v20.4s, v3.4s\n" + "add v21.4s, v21.4s, v2.4s\n" + "add v22.4s, v22.4s, v1.4s\n" + "add v23.4s, v23.4s, v0.4s\n" + "add v24.4s, v24.4s, v3.4s\n" + "add v25.4s, v25.4s, v2.4s\n" + "add v26.4s, v26.4s, v1.4s\n" + "add v27.4s, v27.4s, v0.4s\n" + "add v28.4s, v28.4s, v3.4s\n" + "add v29.4s, v29.4s, v2.4s\n" + "add v30.4s, v30.4s, v1.4s\n" + "add v31.4s, v31.4s, v0.4s\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v1.4s }, [x20]\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x23]\n" - "sqrdmulh v16.4s, v16.4s, v4.4s\n" - "sqrdmulh v17.4s, v17.4s, v4.4s\n" - "sqrdmulh v18.4s, v18.4s, v4.4s\n" - "sqrdmulh v19.4s, v19.4s, v4.4s\n" - "sqrdmulh v20.4s, v20.4s, v4.4s\n" - "sqrdmulh v21.4s, v21.4s, v4.4s\n" - "sqrdmulh v22.4s, v22.4s, v4.4s\n" - "sqrdmulh v23.4s, v23.4s, v4.4s\n" - "sqrdmulh v24.4s, v24.4s, v4.4s\n" - "sqrdmulh v25.4s, v25.4s, v4.4s\n" - "sqrdmulh v26.4s, v26.4s, v4.4s\n" - "sqrdmulh v27.4s, v27.4s, v4.4s\n" - "sqrdmulh v28.4s, v28.4s, v4.4s\n" - "sqrdmulh v29.4s, v29.4s, v4.4s\n" - "sqrdmulh v30.4s, v30.4s, v4.4s\n" - "sqrdmulh v31.4s, v31.4s, v4.4s\n" - "add x16, x16, #0x40\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "sqrdmulh v16.4s, v16.4s, v1.4s\n" + "sqrdmulh v17.4s, v17.4s, v1.4s\n" + "sqrdmulh v18.4s, v18.4s, v1.4s\n" + "sqrdmulh v19.4s, v19.4s, v1.4s\n" + "sqrdmulh v20.4s, v20.4s, v1.4s\n" + "sqrdmulh v21.4s, v21.4s, v1.4s\n" + "sqrdmulh v22.4s, v22.4s, v1.4s\n" + "sqrdmulh v23.4s, v23.4s, v1.4s\n" + "sqrdmulh v24.4s, v24.4s, v1.4s\n" + "sqrdmulh v25.4s, v25.4s, v1.4s\n" + "sqrdmulh v26.4s, v26.4s, v1.4s\n" + "sqrdmulh v27.4s, v27.4s, v1.4s\n" + "sqrdmulh v28.4s, v28.4s, v1.4s\n" + "sqrdmulh v29.4s, v29.4s, v1.4s\n" + "sqrdmulh v30.4s, v30.4s, v1.4s\n" + "sqrdmulh v31.4s, v31.4s, v1.4s\n" + "add x15, x15, #0x40\n" "tbz %x[flags], #5, 110f\n" - "and v4.16b, v16.16b, v0.16b\n" - "and v5.16b, v17.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "and v6.16b, v18.16b, v0.16b\n" - "and v7.16b, v19.16b, v0.16b\n" - "and v8.16b, v20.16b, v0.16b\n" - "and v9.16b, v21.16b, v0.16b\n" - "and v10.16b, v22.16b, v0.16b\n" - "and v4.16b, v23.16b, v0.16b\n" - "and v5.16b, v24.16b, v0.16b\n" - "sshr v6.4s, v6.4s, #0x1f\n" + "and v2.16b, v16.16b, v0.16b\n" + "and v1.16b, v17.16b, v0.16b\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v2.4s\n" + "sqadd v17.4s, v17.4s, v1.4s\n" + "and v7.16b, v18.16b, v0.16b\n" + "and v6.16b, v19.16b, v0.16b\n" + "and v5.16b, v20.16b, v0.16b\n" + "and v4.16b, v21.16b, v0.16b\n" + "and v3.16b, v22.16b, v0.16b\n" + "and v2.16b, v23.16b, v0.16b\n" + "and v1.16b, v24.16b, v0.16b\n" "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v9.4s, v9.4s, #0x1f\n" - "sshr v10.4s, v10.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" - "sqadd v20.4s, v20.4s, v8.4s\n" - "sqadd v21.4s, v21.4s, v9.4s\n" - "sqadd v22.4s, v22.4s, v10.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v24.4s, v24.4s, v5.4s\n" - "and v6.16b, v25.16b, v0.16b\n" - "and v7.16b, v26.16b, v0.16b\n" - "and v8.16b, v27.16b, v0.16b\n" - "and v9.16b, v28.16b, v0.16b\n" - "and v10.16b, v29.16b, v0.16b\n" - "and v4.16b, v30.16b, v0.16b\n" - "and v5.16b, v31.16b, v0.16b\n" "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v9.4s, v9.4s, #0x1f\n" - "sshr v10.4s, v10.4s, #0x1f\n" + "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sqadd v18.4s, v18.4s, v7.4s\n" + "sqadd v19.4s, v19.4s, v6.4s\n" + "sqadd v20.4s, v20.4s, v5.4s\n" + "sqadd v21.4s, v21.4s, v4.4s\n" + "sqadd v22.4s, v22.4s, v3.4s\n" + "sqadd v23.4s, v23.4s, v2.4s\n" + "sqadd v24.4s, v24.4s, v1.4s\n" + "and v7.16b, v25.16b, v0.16b\n" + "and v6.16b, v26.16b, v0.16b\n" + "and v5.16b, v27.16b, v0.16b\n" + "and v4.16b, v28.16b, v0.16b\n" + "and v3.16b, v29.16b, v0.16b\n" + "and v2.16b, v30.16b, v0.16b\n" + "and v1.16b, v31.16b, v0.16b\n" + "sshr v7.4s, v7.4s, #0x1f\n" + "sshr v6.4s, v6.4s, #0x1f\n" "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v25.4s, v25.4s, v6.4s\n" - "sqadd v26.4s, v26.4s, v7.4s\n" - "sqadd v27.4s, v27.4s, v8.4s\n" - "sqadd v28.4s, v28.4s, v9.4s\n" - "sqadd v29.4s, v29.4s, v10.4s\n" - "sqadd v30.4s, v30.4s, v4.4s\n" - "sqadd v31.4s, v31.4s, v5.4s\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sqadd v25.4s, v25.4s, v7.4s\n" + "sqadd v26.4s, v26.4s, v6.4s\n" + "sqadd v27.4s, v27.4s, v5.4s\n" + "sqadd v28.4s, v28.4s, v4.4s\n" + "sqadd v29.4s, v29.4s, v3.4s\n" + "sqadd v30.4s, v30.4s, v2.4s\n" + "sqadd v31.4s, v31.4s, v1.4s\n" "110:" // Height 4: no shift correction "srshl v16.4s, v16.4s, v0.4s\n" "srshl v17.4s, v17.4s, v0.4s\n" @@ -1910,172 +1909,172 @@ void a64_hybrid_u8qa_dot_4x16_a55 ( "srshl v29.4s, v29.4s, v0.4s\n" "srshl v30.4s, v30.4s, v0.4s\n" "srshl v31.4s, v31.4s, v0.4s\n" - "add x23, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x23]\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v23.4s, v23.4s, v4.4s\n" - "add v24.4s, v24.4s, v4.4s\n" - "add v25.4s, v25.4s, v4.4s\n" - "add v26.4s, v26.4s, v4.4s\n" - "add v27.4s, v27.4s, v4.4s\n" - "add v28.4s, v28.4s, v4.4s\n" - "add v29.4s, v29.4s, v4.4s\n" - "add v30.4s, v30.4s, v4.4s\n" - "add v31.4s, v31.4s, v4.4s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x23]\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "smin v24.4s, v24.4s, v6.4s\n" - "smin v25.4s, v25.4s, v6.4s\n" - "smin v26.4s, v26.4s, v6.4s\n" - "smin v27.4s, v27.4s, v6.4s\n" - "smin v28.4s, v28.4s, v6.4s\n" - "smin v29.4s, v29.4s, v6.4s\n" - "smin v30.4s, v30.4s, v6.4s\n" - "smin v31.4s, v31.4s, v6.4s\n" - "add x23, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x23]\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" - "smax v24.4s, v24.4s, v5.4s\n" - "smax v25.4s, v25.4s, v5.4s\n" - "smax v26.4s, v26.4s, v5.4s\n" - "smax v27.4s, v27.4s, v5.4s\n" - "smax v28.4s, v28.4s, v5.4s\n" - "smax v29.4s, v29.4s, v5.4s\n" - "smax v30.4s, v30.4s, v5.4s\n" - "smax v31.4s, v31.4s, v5.4s\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v0.4s }, [x20]\n" + "add v16.4s, v16.4s, v0.4s\n" + "add v17.4s, v17.4s, v0.4s\n" + "add v18.4s, v18.4s, v0.4s\n" + "add v19.4s, v19.4s, v0.4s\n" + "add v20.4s, v20.4s, v0.4s\n" + "add v21.4s, v21.4s, v0.4s\n" + "add v22.4s, v22.4s, v0.4s\n" + "add v23.4s, v23.4s, v0.4s\n" + "add v24.4s, v24.4s, v0.4s\n" + "add v25.4s, v25.4s, v0.4s\n" + "add v26.4s, v26.4s, v0.4s\n" + "add v27.4s, v27.4s, v0.4s\n" + "add v28.4s, v28.4s, v0.4s\n" + "add v29.4s, v29.4s, v0.4s\n" + "add v30.4s, v30.4s, v0.4s\n" + "add v31.4s, v31.4s, v0.4s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v0.4s }, [x20]\n" + "smin v16.4s, v16.4s, v0.4s\n" + "smin v17.4s, v17.4s, v0.4s\n" + "smin v18.4s, v18.4s, v0.4s\n" + "smin v19.4s, v19.4s, v0.4s\n" + "smin v20.4s, v20.4s, v0.4s\n" + "smin v21.4s, v21.4s, v0.4s\n" + "smin v22.4s, v22.4s, v0.4s\n" + "smin v23.4s, v23.4s, v0.4s\n" + "smin v24.4s, v24.4s, v0.4s\n" + "smin v25.4s, v25.4s, v0.4s\n" + "smin v26.4s, v26.4s, v0.4s\n" + "smin v27.4s, v27.4s, v0.4s\n" + "smin v28.4s, v28.4s, v0.4s\n" + "smin v29.4s, v29.4s, v0.4s\n" + "smin v30.4s, v30.4s, v0.4s\n" + "smin v31.4s, v31.4s, v0.4s\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v0.4s }, [x20]\n" + "smax v16.4s, v16.4s, v0.4s\n" + "smax v17.4s, v17.4s, v0.4s\n" + "smax v18.4s, v18.4s, v0.4s\n" + "smax v19.4s, v19.4s, v0.4s\n" + "smax v20.4s, v20.4s, v0.4s\n" + "smax v21.4s, v21.4s, v0.4s\n" + "smax v22.4s, v22.4s, v0.4s\n" + "smax v23.4s, v23.4s, v0.4s\n" + "smax v24.4s, v24.4s, v0.4s\n" + "smax v25.4s, v25.4s, v0.4s\n" + "smax v26.4s, v26.4s, v0.4s\n" + "smax v27.4s, v27.4s, v0.4s\n" + "smax v28.4s, v28.4s, v0.4s\n" + "smax v29.4s, v29.4s, v0.4s\n" + "smax v30.4s, v30.4s, v0.4s\n" + "smax v31.4s, v31.4s, v0.4s\n" "uzp1 v16.8h, v16.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v0.8h, v18.8h, v19.8h\n" "uzp1 v20.8h, v20.8h, v21.8h\n" - "uzp1 v21.8h, v22.8h, v23.8h\n" + "uzp1 v19.8h, v22.8h, v23.8h\n" "uzp1 v24.8h, v24.8h, v25.8h\n" - "uzp1 v25.8h, v26.8h, v27.8h\n" + "uzp1 v18.8h, v26.8h, v27.8h\n" "uzp1 v28.8h, v28.8h, v29.8h\n" - "uzp1 v29.8h, v30.8h, v31.8h\n" - "cmp x15, #0x10\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" - "uzp1 v20.16b, v20.16b, v21.16b\n" - "uzp1 v24.16b, v24.16b, v25.16b\n" - "uzp1 v28.16b, v28.16b, v29.16b\n" + "uzp1 v17.8h, v30.8h, v31.8h\n" + "cmp x14, #0x10\n" + "uzp1 v16.16b, v16.16b, v0.16b\n" + "uzp1 v20.16b, v20.16b, v19.16b\n" + "uzp1 v24.16b, v24.16b, v18.16b\n" + "uzp1 v28.16b, v28.16b, v17.16b\n" "bge 119f\n" - "tbz x15, #3, 114f\n" - "str d16, [x14], #0x8\n" - "str d20, [x22], #0x8\n" - "str d24, [x21], #0x8\n" - "str d28, [x20], #0x8\n" - "tbz x15, #2, 112f\n" - "st1 { v16.s }[2], [x14], #0x4\n" - "st1 { v20.s }[2], [x22], #0x4\n" - "st1 { v24.s }[2], [x21], #0x4\n" - "st1 { v28.s }[2], [x20], #0x4\n" - "tbz x15, #1, 111f\n" - "st1 { v16.h }[6], [x14], #0x2\n" - "st1 { v20.h }[6], [x22], #0x2\n" - "st1 { v24.h }[6], [x21], #0x2\n" - "st1 { v28.h }[6], [x20], #0x2\n" - "tbz x15, #0, 118f\n" - "st1 { v16.b }[14], [x14]\n" - "st1 { v20.b }[14], [x22]\n" - "st1 { v24.b }[14], [x21]\n" - "st1 { v28.b }[14], [x20]\n" + "tbz x14, #3, 114f\n" + "str d16, [x13], #0x8\n" + "str d20, [x23], #0x8\n" + "str d24, [x22], #0x8\n" + "str d28, [x21], #0x8\n" + "tbz x14, #2, 112f\n" + "st1 { v16.s }[2], [x13], #0x4\n" + "st1 { v20.s }[2], [x23], #0x4\n" + "st1 { v24.s }[2], [x22], #0x4\n" + "st1 { v28.s }[2], [x21], #0x4\n" + "tbz x14, #1, 111f\n" + "st1 { v16.h }[6], [x13], #0x2\n" + "st1 { v20.h }[6], [x23], #0x2\n" + "st1 { v24.h }[6], [x22], #0x2\n" + "st1 { v28.h }[6], [x21], #0x2\n" + "tbz x14, #0, 118f\n" + "st1 { v16.b }[14], [x13]\n" + "st1 { v20.b }[14], [x23]\n" + "st1 { v24.b }[14], [x22]\n" + "st1 { v28.b }[14], [x21]\n" "b 118f\n" "111:" // Height 4: Partial direct writeback: partial_1_12 - "tbz x15, #0, 118f\n" - "st1 { v16.b }[12], [x14]\n" - "st1 { v20.b }[12], [x22]\n" - "st1 { v24.b }[12], [x21]\n" - "st1 { v28.b }[12], [x20]\n" + "tbz x14, #0, 118f\n" + "st1 { v16.b }[12], [x13]\n" + "st1 { v20.b }[12], [x23]\n" + "st1 { v24.b }[12], [x22]\n" + "st1 { v28.b }[12], [x21]\n" "b 118f\n" "112:" // Height 4: Partial direct writeback: partial_2_8 - "tbz x15, #1, 113f\n" - "st1 { v16.h }[4], [x14], #0x2\n" - "st1 { v20.h }[4], [x22], #0x2\n" - "st1 { v24.h }[4], [x21], #0x2\n" - "st1 { v28.h }[4], [x20], #0x2\n" - "tbz x15, #0, 118f\n" - "st1 { v16.b }[10], [x14]\n" - "st1 { v20.b }[10], [x22]\n" - "st1 { v24.b }[10], [x21]\n" - "st1 { v28.b }[10], [x20]\n" + "tbz x14, #1, 113f\n" + "st1 { v16.h }[4], [x13], #0x2\n" + "st1 { v20.h }[4], [x23], #0x2\n" + "st1 { v24.h }[4], [x22], #0x2\n" + "st1 { v28.h }[4], [x21], #0x2\n" + "tbz x14, #0, 118f\n" + "st1 { v16.b }[10], [x13]\n" + "st1 { v20.b }[10], [x23]\n" + "st1 { v24.b }[10], [x22]\n" + "st1 { v28.b }[10], [x21]\n" "b 118f\n" "113:" // Height 4: Partial direct writeback: partial_1_8 - "tbz x15, #0, 118f\n" - "st1 { v16.b }[8], [x14]\n" - "st1 { v20.b }[8], [x22]\n" - "st1 { v24.b }[8], [x21]\n" - "st1 { v28.b }[8], [x20]\n" + "tbz x14, #0, 118f\n" + "st1 { v16.b }[8], [x13]\n" + "st1 { v20.b }[8], [x23]\n" + "st1 { v24.b }[8], [x22]\n" + "st1 { v28.b }[8], [x21]\n" "b 118f\n" "114:" // Height 4: Partial direct writeback: partial_4_0 - "tbz x15, #2, 116f\n" - "str s16, [x14], #0x4\n" - "str s20, [x22], #0x4\n" - "str s24, [x21], #0x4\n" - "str s28, [x20], #0x4\n" - "tbz x15, #1, 115f\n" - "st1 { v16.h }[2], [x14], #0x2\n" - "st1 { v20.h }[2], [x22], #0x2\n" - "st1 { v24.h }[2], [x21], #0x2\n" - "st1 { v28.h }[2], [x20], #0x2\n" - "tbz x15, #0, 118f\n" - "st1 { v16.b }[6], [x14]\n" - "st1 { v20.b }[6], [x22]\n" - "st1 { v24.b }[6], [x21]\n" - "st1 { v28.b }[6], [x20]\n" + "tbz x14, #2, 116f\n" + "str s16, [x13], #0x4\n" + "str s20, [x23], #0x4\n" + "str s24, [x22], #0x4\n" + "str s28, [x21], #0x4\n" + "tbz x14, #1, 115f\n" + "st1 { v16.h }[2], [x13], #0x2\n" + "st1 { v20.h }[2], [x23], #0x2\n" + "st1 { v24.h }[2], [x22], #0x2\n" + "st1 { v28.h }[2], [x21], #0x2\n" + "tbz x14, #0, 118f\n" + "st1 { v16.b }[6], [x13]\n" + "st1 { v20.b }[6], [x23]\n" + "st1 { v24.b }[6], [x22]\n" + "st1 { v28.b }[6], [x21]\n" "b 118f\n" "115:" // Height 4: Partial direct writeback: partial_1_4 - "tbz x15, #0, 118f\n" - "st1 { v16.b }[4], [x14]\n" - "st1 { v20.b }[4], [x22]\n" - "st1 { v24.b }[4], [x21]\n" - "st1 { v28.b }[4], [x20]\n" + "tbz x14, #0, 118f\n" + "st1 { v16.b }[4], [x13]\n" + "st1 { v20.b }[4], [x23]\n" + "st1 { v24.b }[4], [x22]\n" + "st1 { v28.b }[4], [x21]\n" "b 118f\n" "116:" // Height 4: Partial direct writeback: partial_2_0 - "tbz x15, #1, 117f\n" - "str h16, [x14], #0x2\n" - "str h20, [x22], #0x2\n" - "str h24, [x21], #0x2\n" - "str h28, [x20], #0x2\n" - "tbz x15, #0, 118f\n" - "st1 { v16.b }[2], [x14]\n" - "st1 { v20.b }[2], [x22]\n" - "st1 { v24.b }[2], [x21]\n" - "st1 { v28.b }[2], [x20]\n" + "tbz x14, #1, 117f\n" + "str h16, [x13], #0x2\n" + "str h20, [x23], #0x2\n" + "str h24, [x22], #0x2\n" + "str h28, [x21], #0x2\n" + "tbz x14, #0, 118f\n" + "st1 { v16.b }[2], [x13]\n" + "st1 { v20.b }[2], [x23]\n" + "st1 { v24.b }[2], [x22]\n" + "st1 { v28.b }[2], [x21]\n" "b 118f\n" "117:" // Height 4: Partial direct writeback: partial_1_0 - "str b16, [x14, #0x0]\n" - "str b20, [x22, #0x0]\n" - "str b24, [x21, #0x0]\n" - "str b28, [x20, #0x0]\n" + "str b16, [x13, #0x0]\n" + "str b20, [x23, #0x0]\n" + "str b24, [x22, #0x0]\n" + "str b28, [x21, #0x0]\n" "118:" // Height 4: Partial direct writeback: Done "b 120f\n" "119:" // Height 4: Full writeback - "str q16, [x14, #0x0]\n" - "add x14, x14, #0x10\n" - "str q20, [x22, #0x0]\n" - "str q24, [x21, #0x0]\n" - "str q28, [x20, #0x0]\n" + "str q16, [x13, #0x0]\n" + "add x13, x13, #0x10\n" + "str q20, [x23, #0x0]\n" + "str q24, [x22, #0x0]\n" + "str q28, [x21, #0x0]\n" "120:" // Height 4: Writeback done - "subs x15, x15, #0x10\n" + "subs x14, x14, #0x10\n" "bgt 92b\n" "subs %x[M], %x[M], #0x4\n" "beq 122f\n" @@ -2089,10 +2088,9 @@ void a64_hybrid_u8qa_dot_4x16_a55 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "122:" // Exit - : [M] "+&r" (M), [flags] "+&r" (flags), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp) - : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" + : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16/generic.cpp index 31fbf88603197f5637bffdc52113972828d6662e..ebe583b5d44f6161e9659187e33c9fc57e5552d9 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_dot_4x16/generic.cpp @@ -78,7 +78,6 @@ void a64_hybrid_u8qa_dot_4x16 ( flags |= 0x20; } __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x4\n" "bge 91f\n" @@ -102,11 +101,11 @@ void a64_hybrid_u8qa_dot_4x16 ( "4:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 5f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" "cbnz x26, 6f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -128,32 +127,32 @@ void a64_hybrid_u8qa_dot_4x16 ( "blt 9f\n" "7:" // Height 1: Multiply loop: Main loop head ".inst 0x6f80e090 // udot v16.4s, v4.16b, v0.4b[0]\n" - "ldr q4, [x28, #0x70]\n" + "ldr q21, [x28, #0x70]\n" ".inst 0x6f80e0b1 // udot v17.4s, v5.16b, v0.4b[0]\n" - "ldr q5, [x28, #0x80]\n" + "ldr q20, [x28, #0x80]\n" ".inst 0x6f80e0d2 // udot v18.4s, v6.16b, v0.4b[0]\n" - "ldr q6, [x28, #0x90]\n" + "ldr q26, [x28, #0x90]\n" ".inst 0x6f80e0f3 // udot v19.4s, v7.16b, v0.4b[0]\n" - "ldr q7, [x28, #0xa0]\n" + "ldr q25, [x28, #0xa0]\n" ".inst 0x6fa0e110 // udot v16.4s, v8.16b, v0.4b[1]\n" - "ldr q8, [x28, #0xb0]\n" + "ldr q24, [x28, #0xb0]\n" ".inst 0x6fa0e131 // udot v17.4s, v9.16b, v0.4b[1]\n" - "ldr q9, [x28, #0xc0]\n" + "ldr q23, [x28, #0xc0]\n" ".inst 0x6fa0e152 // udot v18.4s, v10.16b, v0.4b[1]\n" - "ldr q10, [x28, #0xd0]\n" - ".inst 0x6fa0e093 // udot v19.4s, v4.16b, v0.4b[1]\n" - "ldr q4, [x28, #0xe0]\n" - ".inst 0x6f80e8b0 // udot v16.4s, v5.16b, v0.4b[2]\n" - "ldr q5, [x28, #0xf0]\n" - ".inst 0x6f80e8d1 // udot v17.4s, v6.16b, v0.4b[2]\n" + "ldr q22, [x28, #0xd0]\n" + ".inst 0x6fa0e2b3 // udot v19.4s, v21.16b, v0.4b[1]\n" + "ldr q21, [x28, #0xe0]\n" + ".inst 0x6f80ea90 // udot v16.4s, v20.16b, v0.4b[2]\n" + "ldr q20, [x28, #0xf0]\n" + ".inst 0x6f80eb51 // udot v17.4s, v26.16b, v0.4b[2]\n" "add x24, x24, #0x10\n" - ".inst 0x6f80e8f2 // udot v18.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f80e913 // udot v19.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80eb32 // udot v18.4s, v25.16b, v0.4b[2]\n" + ".inst 0x6f80eb13 // udot v19.4s, v24.16b, v0.4b[2]\n" "add x28, x28, #0x100\n" - ".inst 0x6fa0e930 // udot v16.4s, v9.16b, v0.4b[3]\n" - ".inst 0x6fa0e951 // udot v17.4s, v10.16b, v0.4b[3]\n" - ".inst 0x6fa0e892 // udot v18.4s, v4.16b, v0.4b[3]\n" - ".inst 0x6fa0e8b3 // udot v19.4s, v5.16b, v0.4b[3]\n" + ".inst 0x6fa0eaf0 // udot v16.4s, v23.16b, v0.4b[3]\n" + ".inst 0x6fa0ead1 // udot v17.4s, v22.16b, v0.4b[3]\n" + ".inst 0x6fa0eab2 // udot v18.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa0ea93 // udot v19.4s, v20.16b, v0.4b[3]\n" "tbnz %x[flags], #31, 8f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" "8:" // Height 1: Multiply loop: unique 1: skip row sum @@ -171,33 +170,33 @@ void a64_hybrid_u8qa_dot_4x16 ( "bge 7b\n" "9:" // Height 1: Multiply loop: Single iteration only ".inst 0x6f80e090 // udot v16.4s, v4.16b, v0.4b[0]\n" - "ldr q4, [x28, #0x70]\n" + "ldr q21, [x28, #0x70]\n" ".inst 0x6f80e0b1 // udot v17.4s, v5.16b, v0.4b[0]\n" - "ldr q5, [x28, #0x80]\n" + "ldr q20, [x28, #0x80]\n" ".inst 0x6f80e0d2 // udot v18.4s, v6.16b, v0.4b[0]\n" - "ldr q6, [x28, #0x90]\n" + "ldr q26, [x28, #0x90]\n" ".inst 0x6f80e0f3 // udot v19.4s, v7.16b, v0.4b[0]\n" - "ldr q7, [x28, #0xa0]\n" + "ldr q25, [x28, #0xa0]\n" ".inst 0x6fa0e110 // udot v16.4s, v8.16b, v0.4b[1]\n" - "ldr q8, [x28, #0xb0]\n" + "ldr q24, [x28, #0xb0]\n" ".inst 0x6fa0e131 // udot v17.4s, v9.16b, v0.4b[1]\n" - "ldr q9, [x28, #0xc0]\n" + "ldr q23, [x28, #0xc0]\n" ".inst 0x6fa0e152 // udot v18.4s, v10.16b, v0.4b[1]\n" - "ldr q10, [x28, #0xd0]\n" - ".inst 0x6fa0e093 // udot v19.4s, v4.16b, v0.4b[1]\n" - "ldr q4, [x28, #0xe0]\n" - ".inst 0x6f80e8b0 // udot v16.4s, v5.16b, v0.4b[2]\n" - "ldr q5, [x28, #0xf0]\n" - ".inst 0x6f80e8d1 // udot v17.4s, v6.16b, v0.4b[2]\n" + "ldr q22, [x28, #0xd0]\n" + ".inst 0x6fa0e2b3 // udot v19.4s, v21.16b, v0.4b[1]\n" + "ldr q21, [x28, #0xe0]\n" + ".inst 0x6f80ea90 // udot v16.4s, v20.16b, v0.4b[2]\n" + "ldr q20, [x28, #0xf0]\n" + ".inst 0x6f80eb51 // udot v17.4s, v26.16b, v0.4b[2]\n" "sub x25, x25, #0x10\n" - ".inst 0x6f80e8f2 // udot v18.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f80e913 // udot v19.4s, v8.16b, v0.4b[2]\n" + ".inst 0x6f80eb32 // udot v18.4s, v25.16b, v0.4b[2]\n" + ".inst 0x6f80eb13 // udot v19.4s, v24.16b, v0.4b[2]\n" "add x24, x24, #0x10\n" "add x28, x28, #0x100\n" - ".inst 0x6fa0e930 // udot v16.4s, v9.16b, v0.4b[3]\n" - ".inst 0x6fa0e951 // udot v17.4s, v10.16b, v0.4b[3]\n" - ".inst 0x6fa0e892 // udot v18.4s, v4.16b, v0.4b[3]\n" - ".inst 0x6fa0e8b3 // udot v19.4s, v5.16b, v0.4b[3]\n" + ".inst 0x6fa0eaf0 // udot v16.4s, v23.16b, v0.4b[3]\n" + ".inst 0x6fa0ead1 // udot v17.4s, v22.16b, v0.4b[3]\n" + ".inst 0x6fa0eab2 // udot v18.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa0ea93 // udot v19.4s, v20.16b, v0.4b[3]\n" "tbnz %x[flags], #31, 10f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" "10:" // Height 1: Multiply loop: unique 2: skip row sum @@ -211,16 +210,16 @@ void a64_hybrid_u8qa_dot_4x16 ( "tbnz %x[flags], #31, 13f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" "13:" // Height 1: Multiply loop: unique 3: skip row sum - "ldr q6, [x28, #0x0]\n" - "ldr q7, [x28, #0x10]\n" + "ldr q23, [x28, #0x0]\n" + "ldr q22, [x28, #0x10]\n" "sub x25, x25, #0x4\n" "cmp x25, #0x4\n" - "ldr q8, [x28, #0x20]\n" - "ldr q9, [x28, #0x30]\n" - ".inst 0x6f80e0d0 // udot v16.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f80e0f1 // udot v17.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f80e112 // udot v18.4s, v8.16b, v0.4b[0]\n" - ".inst 0x6f80e133 // udot v19.4s, v9.16b, v0.4b[0]\n" + "ldr q21, [x28, #0x20]\n" + "ldr q20, [x28, #0x30]\n" + ".inst 0x6f80e2f0 // udot v16.4s, v23.16b, v0.4b[0]\n" + ".inst 0x6f80e2d1 // udot v17.4s, v22.16b, v0.4b[0]\n" + ".inst 0x6f80e2b2 // udot v18.4s, v21.16b, v0.4b[0]\n" + ".inst 0x6f80e293 // udot v19.4s, v20.16b, v0.4b[0]\n" "add x28, x28, #0x40\n" "bge 12b\n" "14:" // Height 1: Multiply loop: Skip odd blocks @@ -236,14 +235,14 @@ void a64_hybrid_u8qa_dot_4x16 ( "tbnz %x[flags], #31, 17f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" "17:" // Height 1: Multiply loop: unique 4: skip row sum - "ldr q10, [x28, #0x0]\n" - "ldr q4, [x28, #0x10]\n" - ".inst 0x6f80e150 // udot v16.4s, v10.16b, v0.4b[0]\n" - ".inst 0x6f80e091 // udot v17.4s, v4.16b, v0.4b[0]\n" - "ldr q5, [x28, #0x20]\n" - "ldr q6, [x28, #0x30]\n" - ".inst 0x6f80e0b2 // udot v18.4s, v5.16b, v0.4b[0]\n" - ".inst 0x6f80e0d3 // udot v19.4s, v6.16b, v0.4b[0]\n" + "ldr q21, [x28, #0x0]\n" + "ldr q20, [x28, #0x10]\n" + ".inst 0x6f80e2b0 // udot v16.4s, v21.16b, v0.4b[0]\n" + ".inst 0x6f80e291 // udot v17.4s, v20.16b, v0.4b[0]\n" + "ldr q21, [x28, #0x20]\n" + "ldr q20, [x28, #0x30]\n" + ".inst 0x6f80e2b2 // udot v18.4s, v21.16b, v0.4b[0]\n" + ".inst 0x6f80e293 // udot v19.4s, v20.16b, v0.4b[0]\n" "add x28, x28, #0x40\n" "18:" // Height 1: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -252,72 +251,72 @@ void a64_hybrid_u8qa_dot_4x16 ( "bne 4b\n" "prfm pstl1keep, [x27, #0x0]\n" "tbnz %x[flags], #31, 19f\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1r { v1.4s }, [x23]\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1r { v20.4s }, [x20]\n" "addp v11.4s, v11.4s, v11.4s\n" - "neg v1.4s, v1.4s\n" + "neg v20.4s, v20.4s\n" "addp v11.4s, v11.4s, v11.4s\n" - "mul v11.4s, v11.4s, v1.4s\n" + "mul v11.4s, v11.4s, v20.4s\n" "19:" // Height 1: skip row sum fixup - "ldr q0, [x10, #0x0]\n" - "ldr q1, [x10, #0x10]\n" + "ldr q24, [x10, #0x0]\n" + "ldr q23, [x10, #0x10]\n" "add v16.4s, v16.4s, v11.4s\n" "add v17.4s, v17.4s, v11.4s\n" - "ldr q2, [x10, #0x20]\n" - "ldr q3, [x10, #0x30]\n" + "ldr q22, [x10, #0x20]\n" + "ldr q21, [x10, #0x30]\n" "add v18.4s, v18.4s, v11.4s\n" "add v19.4s, v19.4s, v11.4s\n" - "add x23, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v20.4s }, [x20]\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x23]\n" - "add v19.4s, v19.4s, v3.4s\n" - "sqrdmulh v16.4s, v16.4s, v4.4s\n" + "add v16.4s, v16.4s, v24.4s\n" + "add v17.4s, v17.4s, v23.4s\n" + "add v18.4s, v18.4s, v22.4s\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "add v19.4s, v19.4s, v21.4s\n" + "sqrdmulh v16.4s, v16.4s, v20.4s\n" "add x10, x10, #0x40\n" - "sqrdmulh v17.4s, v17.4s, v4.4s\n" - "sqrdmulh v18.4s, v18.4s, v4.4s\n" - "sqrdmulh v19.4s, v19.4s, v4.4s\n" + "sqrdmulh v17.4s, v17.4s, v20.4s\n" + "sqrdmulh v18.4s, v18.4s, v20.4s\n" + "sqrdmulh v19.4s, v19.4s, v20.4s\n" "tbz %x[flags], #5, 20f\n" - "and v4.16b, v16.16b, v0.16b\n" - "and v5.16b, v17.16b, v0.16b\n" - "and v6.16b, v18.16b, v0.16b\n" - "and v7.16b, v19.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" + "and v23.16b, v16.16b, v0.16b\n" + "and v22.16b, v17.16b, v0.16b\n" + "and v21.16b, v18.16b, v0.16b\n" + "and v20.16b, v19.16b, v0.16b\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "sshr v21.4s, v21.4s, #0x1f\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v23.4s\n" + "sqadd v17.4s, v17.4s, v22.4s\n" + "sqadd v18.4s, v18.4s, v21.4s\n" + "sqadd v19.4s, v19.4s, v20.4s\n" "20:" // Height 1: no shift correction - "add x23, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v22.4s }, [x20]\n" "srshl v16.4s, v16.4s, v0.4s\n" "srshl v17.4s, v17.4s, v0.4s\n" "srshl v18.4s, v18.4s, v0.4s\n" "srshl v19.4s, v19.4s, v0.4s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x23]\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add x23, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x23]\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v21.4s }, [x20]\n" + "add v16.4s, v16.4s, v22.4s\n" + "add v17.4s, v17.4s, v22.4s\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v20.4s }, [x20]\n" + "add v18.4s, v18.4s, v22.4s\n" + "add v19.4s, v19.4s, v22.4s\n" "cmp x9, #0x10\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" + "smin v16.4s, v16.4s, v21.4s\n" + "smin v17.4s, v17.4s, v21.4s\n" + "smin v18.4s, v18.4s, v21.4s\n" + "smin v19.4s, v19.4s, v21.4s\n" + "smax v16.4s, v16.4s, v20.4s\n" + "smax v17.4s, v17.4s, v20.4s\n" + "smax v18.4s, v18.4s, v20.4s\n" + "smax v19.4s, v19.4s, v20.4s\n" "uzp1 v16.8h, v16.8h, v17.8h\n" "uzp1 v17.8h, v18.8h, v19.8h\n" "uzp1 v16.16b, v16.16b, v17.16b\n" @@ -397,12 +396,12 @@ void a64_hybrid_u8qa_dot_4x16 ( "34:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 35f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" "cbnz x26, 36f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -410,7 +409,7 @@ void a64_hybrid_u8qa_dot_4x16 ( "b 36f\n" "35:" // Height 2: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20\n" + "add x23, x24, x21\n" "36:" // Height 2: input setup done "cmp x25, #0x10\n" "blt 41f\n" @@ -428,48 +427,48 @@ void a64_hybrid_u8qa_dot_4x16 ( "37:" // Height 2: Multiply loop: Main loop head ".inst 0x6f80e090 // udot v16.4s, v4.16b, v0.4b[0]\n" ".inst 0x6f81e094 // udot v20.4s, v4.16b, v1.4b[0]\n" - "ldr q4, [x28, #0x70]\n" + "ldr q25, [x28, #0x70]\n" "add x24, x24, #0x10\n" ".inst 0x6f80e0b1 // udot v17.4s, v5.16b, v0.4b[0]\n" ".inst 0x6f81e0b5 // udot v21.4s, v5.16b, v1.4b[0]\n" - "ldr q5, [x28, #0x80]\n" + "ldr q24, [x28, #0x80]\n" "add x23, x23, #0x10\n" ".inst 0x6f80e0d2 // udot v18.4s, v6.16b, v0.4b[0]\n" ".inst 0x6f81e0d6 // udot v22.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x28, #0x90]\n" + "ldr q30, [x28, #0x90]\n" ".inst 0x6f80e0f3 // udot v19.4s, v7.16b, v0.4b[0]\n" ".inst 0x6f81e0f7 // udot v23.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x28, #0xa0]\n" + "ldr q29, [x28, #0xa0]\n" ".inst 0x6fa0e110 // udot v16.4s, v8.16b, v0.4b[1]\n" ".inst 0x6fa1e114 // udot v20.4s, v8.16b, v1.4b[1]\n" - "ldr q8, [x28, #0xb0]\n" + "ldr q28, [x28, #0xb0]\n" ".inst 0x6fa0e131 // udot v17.4s, v9.16b, v0.4b[1]\n" ".inst 0x6fa1e135 // udot v21.4s, v9.16b, v1.4b[1]\n" - "ldr q9, [x28, #0xc0]\n" + "ldr q27, [x28, #0xc0]\n" ".inst 0x6fa0e152 // udot v18.4s, v10.16b, v0.4b[1]\n" ".inst 0x6fa1e156 // udot v22.4s, v10.16b, v1.4b[1]\n" - "ldr q10, [x28, #0xd0]\n" - ".inst 0x6fa0e093 // udot v19.4s, v4.16b, v0.4b[1]\n" - ".inst 0x6fa1e097 // udot v23.4s, v4.16b, v1.4b[1]\n" - "ldr q4, [x28, #0xe0]\n" - ".inst 0x6f80e8b0 // udot v16.4s, v5.16b, v0.4b[2]\n" - ".inst 0x6f81e8b4 // udot v20.4s, v5.16b, v1.4b[2]\n" - "ldr q5, [x28, #0xf0]\n" + "ldr q26, [x28, #0xd0]\n" + ".inst 0x6fa0e333 // udot v19.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa1e337 // udot v23.4s, v25.16b, v1.4b[1]\n" + "ldr q25, [x28, #0xe0]\n" + ".inst 0x6f80eb10 // udot v16.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f81eb14 // udot v20.4s, v24.16b, v1.4b[2]\n" + "ldr q24, [x28, #0xf0]\n" "add x28, x28, #0x100\n" - ".inst 0x6f80e8d1 // udot v17.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8d5 // udot v21.4s, v6.16b, v1.4b[2]\n" - ".inst 0x6f80e8f2 // udot v18.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8f6 // udot v22.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f80e913 // udot v19.4s, v8.16b, v0.4b[2]\n" - ".inst 0x6f81e917 // udot v23.4s, v8.16b, v1.4b[2]\n" - ".inst 0x6fa0e930 // udot v16.4s, v9.16b, v0.4b[3]\n" - ".inst 0x6fa1e934 // udot v20.4s, v9.16b, v1.4b[3]\n" - ".inst 0x6fa0e951 // udot v17.4s, v10.16b, v0.4b[3]\n" - ".inst 0x6fa1e955 // udot v21.4s, v10.16b, v1.4b[3]\n" - ".inst 0x6fa0e892 // udot v18.4s, v4.16b, v0.4b[3]\n" - ".inst 0x6fa1e896 // udot v22.4s, v4.16b, v1.4b[3]\n" - ".inst 0x6fa0e8b3 // udot v19.4s, v5.16b, v0.4b[3]\n" - ".inst 0x6fa1e8b7 // udot v23.4s, v5.16b, v1.4b[3]\n" + ".inst 0x6f80ebd1 // udot v17.4s, v30.16b, v0.4b[2]\n" + ".inst 0x6f81ebd5 // udot v21.4s, v30.16b, v1.4b[2]\n" + ".inst 0x6f80ebb2 // udot v18.4s, v29.16b, v0.4b[2]\n" + ".inst 0x6f81ebb6 // udot v22.4s, v29.16b, v1.4b[2]\n" + ".inst 0x6f80eb93 // udot v19.4s, v28.16b, v0.4b[2]\n" + ".inst 0x6f81eb97 // udot v23.4s, v28.16b, v1.4b[2]\n" + ".inst 0x6fa0eb70 // udot v16.4s, v27.16b, v0.4b[3]\n" + ".inst 0x6fa1eb74 // udot v20.4s, v27.16b, v1.4b[3]\n" + ".inst 0x6fa0eb51 // udot v17.4s, v26.16b, v0.4b[3]\n" + ".inst 0x6fa1eb55 // udot v21.4s, v26.16b, v1.4b[3]\n" + ".inst 0x6fa0eb32 // udot v18.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa1eb36 // udot v22.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa0eb13 // udot v19.4s, v24.16b, v0.4b[3]\n" + ".inst 0x6fa1eb17 // udot v23.4s, v24.16b, v1.4b[3]\n" "tbnz %x[flags], #31, 38f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n" @@ -491,49 +490,49 @@ void a64_hybrid_u8qa_dot_4x16 ( "39:" // Height 2: Multiply loop: Single iteration only ".inst 0x6f80e090 // udot v16.4s, v4.16b, v0.4b[0]\n" ".inst 0x6f81e094 // udot v20.4s, v4.16b, v1.4b[0]\n" - "ldr q4, [x28, #0x70]\n" + "ldr q25, [x28, #0x70]\n" "sub x25, x25, #0x10\n" ".inst 0x6f80e0b1 // udot v17.4s, v5.16b, v0.4b[0]\n" ".inst 0x6f81e0b5 // udot v21.4s, v5.16b, v1.4b[0]\n" - "ldr q5, [x28, #0x80]\n" + "ldr q24, [x28, #0x80]\n" "add x24, x24, #0x10\n" ".inst 0x6f80e0d2 // udot v18.4s, v6.16b, v0.4b[0]\n" ".inst 0x6f81e0d6 // udot v22.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x28, #0x90]\n" + "ldr q30, [x28, #0x90]\n" "add x23, x23, #0x10\n" ".inst 0x6f80e0f3 // udot v19.4s, v7.16b, v0.4b[0]\n" ".inst 0x6f81e0f7 // udot v23.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x28, #0xa0]\n" + "ldr q29, [x28, #0xa0]\n" ".inst 0x6fa0e110 // udot v16.4s, v8.16b, v0.4b[1]\n" ".inst 0x6fa1e114 // udot v20.4s, v8.16b, v1.4b[1]\n" - "ldr q8, [x28, #0xb0]\n" + "ldr q28, [x28, #0xb0]\n" ".inst 0x6fa0e131 // udot v17.4s, v9.16b, v0.4b[1]\n" ".inst 0x6fa1e135 // udot v21.4s, v9.16b, v1.4b[1]\n" - "ldr q9, [x28, #0xc0]\n" + "ldr q27, [x28, #0xc0]\n" ".inst 0x6fa0e152 // udot v18.4s, v10.16b, v0.4b[1]\n" ".inst 0x6fa1e156 // udot v22.4s, v10.16b, v1.4b[1]\n" - "ldr q10, [x28, #0xd0]\n" - ".inst 0x6fa0e093 // udot v19.4s, v4.16b, v0.4b[1]\n" - ".inst 0x6fa1e097 // udot v23.4s, v4.16b, v1.4b[1]\n" - "ldr q4, [x28, #0xe0]\n" - ".inst 0x6f80e8b0 // udot v16.4s, v5.16b, v0.4b[2]\n" - ".inst 0x6f81e8b4 // udot v20.4s, v5.16b, v1.4b[2]\n" - "ldr q5, [x28, #0xf0]\n" + "ldr q26, [x28, #0xd0]\n" + ".inst 0x6fa0e333 // udot v19.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa1e337 // udot v23.4s, v25.16b, v1.4b[1]\n" + "ldr q25, [x28, #0xe0]\n" + ".inst 0x6f80eb10 // udot v16.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f81eb14 // udot v20.4s, v24.16b, v1.4b[2]\n" + "ldr q24, [x28, #0xf0]\n" "add x28, x28, #0x100\n" - ".inst 0x6f80e8d1 // udot v17.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8d5 // udot v21.4s, v6.16b, v1.4b[2]\n" - ".inst 0x6f80e8f2 // udot v18.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8f6 // udot v22.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f80e913 // udot v19.4s, v8.16b, v0.4b[2]\n" - ".inst 0x6f81e917 // udot v23.4s, v8.16b, v1.4b[2]\n" - ".inst 0x6fa0e930 // udot v16.4s, v9.16b, v0.4b[3]\n" - ".inst 0x6fa1e934 // udot v20.4s, v9.16b, v1.4b[3]\n" - ".inst 0x6fa0e951 // udot v17.4s, v10.16b, v0.4b[3]\n" - ".inst 0x6fa1e955 // udot v21.4s, v10.16b, v1.4b[3]\n" - ".inst 0x6fa0e892 // udot v18.4s, v4.16b, v0.4b[3]\n" - ".inst 0x6fa1e896 // udot v22.4s, v4.16b, v1.4b[3]\n" - ".inst 0x6fa0e8b3 // udot v19.4s, v5.16b, v0.4b[3]\n" - ".inst 0x6fa1e8b7 // udot v23.4s, v5.16b, v1.4b[3]\n" + ".inst 0x6f80ebd1 // udot v17.4s, v30.16b, v0.4b[2]\n" + ".inst 0x6f81ebd5 // udot v21.4s, v30.16b, v1.4b[2]\n" + ".inst 0x6f80ebb2 // udot v18.4s, v29.16b, v0.4b[2]\n" + ".inst 0x6f81ebb6 // udot v22.4s, v29.16b, v1.4b[2]\n" + ".inst 0x6f80eb93 // udot v19.4s, v28.16b, v0.4b[2]\n" + ".inst 0x6f81eb97 // udot v23.4s, v28.16b, v1.4b[2]\n" + ".inst 0x6fa0eb70 // udot v16.4s, v27.16b, v0.4b[3]\n" + ".inst 0x6fa1eb74 // udot v20.4s, v27.16b, v1.4b[3]\n" + ".inst 0x6fa0eb51 // udot v17.4s, v26.16b, v0.4b[3]\n" + ".inst 0x6fa1eb55 // udot v21.4s, v26.16b, v1.4b[3]\n" + ".inst 0x6fa0eb32 // udot v18.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa1eb36 // udot v22.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa0eb13 // udot v19.4s, v24.16b, v0.4b[3]\n" + ".inst 0x6fa1eb17 // udot v23.4s, v24.16b, v1.4b[3]\n" "tbnz %x[flags], #31, 40f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n" @@ -551,21 +550,21 @@ void a64_hybrid_u8qa_dot_4x16 ( ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n" "43:" // Height 2: Multiply loop: unique 7: skip row sum - "ldr q6, [x28, #0x0]\n" - "ldr q7, [x28, #0x10]\n" + "ldr q27, [x28, #0x0]\n" + "ldr q26, [x28, #0x10]\n" "sub x25, x25, #0x4\n" "cmp x25, #0x4\n" - "ldr q8, [x28, #0x20]\n" - "ldr q9, [x28, #0x30]\n" - ".inst 0x6f80e0d0 // udot v16.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0d4 // udot v20.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f80e0f1 // udot v17.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0f5 // udot v21.4s, v7.16b, v1.4b[0]\n" + "ldr q25, [x28, #0x20]\n" + "ldr q24, [x28, #0x30]\n" + ".inst 0x6f80e370 // udot v16.4s, v27.16b, v0.4b[0]\n" + ".inst 0x6f81e374 // udot v20.4s, v27.16b, v1.4b[0]\n" + ".inst 0x6f80e351 // udot v17.4s, v26.16b, v0.4b[0]\n" + ".inst 0x6f81e355 // udot v21.4s, v26.16b, v1.4b[0]\n" "add x28, x28, #0x40\n" - ".inst 0x6f80e112 // udot v18.4s, v8.16b, v0.4b[0]\n" - ".inst 0x6f81e116 // udot v22.4s, v8.16b, v1.4b[0]\n" - ".inst 0x6f80e133 // udot v19.4s, v9.16b, v0.4b[0]\n" - ".inst 0x6f81e137 // udot v23.4s, v9.16b, v1.4b[0]\n" + ".inst 0x6f80e332 // udot v18.4s, v25.16b, v0.4b[0]\n" + ".inst 0x6f81e336 // udot v22.4s, v25.16b, v1.4b[0]\n" + ".inst 0x6f80e313 // udot v19.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f81e317 // udot v23.4s, v24.16b, v1.4b[0]\n" "bge 42b\n" "44:" // Height 2: Multiply loop: Skip odd blocks "cbz x25, 48f\n" @@ -584,209 +583,209 @@ void a64_hybrid_u8qa_dot_4x16 ( ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n" "47:" // Height 2: Multiply loop: unique 8: skip row sum - "ldr q10, [x28, #0x0]\n" - "ldr q4, [x28, #0x10]\n" - ".inst 0x6f80e150 // udot v16.4s, v10.16b, v0.4b[0]\n" - ".inst 0x6f81e154 // udot v20.4s, v10.16b, v1.4b[0]\n" - "ldr q5, [x28, #0x20]\n" - "ldr q6, [x28, #0x30]\n" - ".inst 0x6f80e091 // udot v17.4s, v4.16b, v0.4b[0]\n" - ".inst 0x6f81e095 // udot v21.4s, v4.16b, v1.4b[0]\n" - ".inst 0x6f80e0b2 // udot v18.4s, v5.16b, v0.4b[0]\n" - ".inst 0x6f81e0b6 // udot v22.4s, v5.16b, v1.4b[0]\n" + "ldr q24, [x28, #0x0]\n" + "ldr q26, [x28, #0x10]\n" + ".inst 0x6f80e310 // udot v16.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f81e314 // udot v20.4s, v24.16b, v1.4b[0]\n" + "ldr q25, [x28, #0x20]\n" + "ldr q24, [x28, #0x30]\n" + ".inst 0x6f80e351 // udot v17.4s, v26.16b, v0.4b[0]\n" + ".inst 0x6f81e355 // udot v21.4s, v26.16b, v1.4b[0]\n" + ".inst 0x6f80e332 // udot v18.4s, v25.16b, v0.4b[0]\n" + ".inst 0x6f81e336 // udot v22.4s, v25.16b, v1.4b[0]\n" "add x28, x28, #0x40\n" - ".inst 0x6f80e0d3 // udot v19.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0d7 // udot v23.4s, v6.16b, v1.4b[0]\n" + ".inst 0x6f80e313 // udot v19.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f81e317 // udot v23.4s, v24.16b, v1.4b[0]\n" "48:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x26, x26, #0x1\n" "cmp x26, x20\n" "bne 34b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x22, x27, x20\n" + "add x23, x27, x20\n" "prfm pstl1keep, [x27, #0x0]\n" - "prfm pstl1keep, [x22, #0x0]\n" + "prfm pstl1keep, [x23, #0x0]\n" "tbnz %x[flags], #31, 49f\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1r { v2.4s }, [x23]\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1r { v24.4s }, [x20]\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v12.4s, v12.4s, v12.4s\n" - "neg v2.4s, v2.4s\n" + "neg v24.4s, v24.4s\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v12.4s, v12.4s, v12.4s\n" - "mul v11.4s, v11.4s, v2.4s\n" - "mul v12.4s, v12.4s, v2.4s\n" + "mul v11.4s, v11.4s, v24.4s\n" + "mul v12.4s, v12.4s, v24.4s\n" "49:" // Height 2: skip row sum fixup - "ldr q0, [x10, #0x0]\n" - "ldr q1, [x10, #0x10]\n" + "ldr q28, [x10, #0x0]\n" + "ldr q27, [x10, #0x10]\n" "add v16.4s, v16.4s, v11.4s\n" "add v17.4s, v17.4s, v11.4s\n" - "ldr q2, [x10, #0x20]\n" - "ldr q3, [x10, #0x30]\n" + "ldr q26, [x10, #0x20]\n" + "ldr q25, [x10, #0x30]\n" "add v18.4s, v18.4s, v11.4s\n" "add v19.4s, v19.4s, v11.4s\n" "add v20.4s, v20.4s, v12.4s\n" "add v21.4s, v21.4s, v12.4s\n" - "add x23, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v24.4s }, [x20]\n" "add v22.4s, v22.4s, v12.4s\n" "add v23.4s, v23.4s, v12.4s\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" - "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "add v16.4s, v16.4s, v28.4s\n" + "add v17.4s, v17.4s, v27.4s\n" "add x10, x10, #0x40\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" - "add v20.4s, v20.4s, v0.4s\n" - "ld1r { v0.4s }, [x23]\n" - "add v21.4s, v21.4s, v1.4s\n" - "add v22.4s, v22.4s, v2.4s\n" - "add v23.4s, v23.4s, v3.4s\n" - "sqrdmulh v16.4s, v16.4s, v4.4s\n" - "sqrdmulh v17.4s, v17.4s, v4.4s\n" - "sqrdmulh v18.4s, v18.4s, v4.4s\n" - "sqrdmulh v19.4s, v19.4s, v4.4s\n" - "sqrdmulh v20.4s, v20.4s, v4.4s\n" - "sqrdmulh v21.4s, v21.4s, v4.4s\n" - "sqrdmulh v22.4s, v22.4s, v4.4s\n" - "sqrdmulh v23.4s, v23.4s, v4.4s\n" + "add v18.4s, v18.4s, v26.4s\n" + "add v19.4s, v19.4s, v25.4s\n" + "add v20.4s, v20.4s, v28.4s\n" + "ld1r { v0.4s }, [x20]\n" + "add v21.4s, v21.4s, v27.4s\n" + "add v22.4s, v22.4s, v26.4s\n" + "add v23.4s, v23.4s, v25.4s\n" + "sqrdmulh v16.4s, v16.4s, v24.4s\n" + "sqrdmulh v17.4s, v17.4s, v24.4s\n" + "sqrdmulh v18.4s, v18.4s, v24.4s\n" + "sqrdmulh v19.4s, v19.4s, v24.4s\n" + "sqrdmulh v20.4s, v20.4s, v24.4s\n" + "sqrdmulh v21.4s, v21.4s, v24.4s\n" + "sqrdmulh v22.4s, v22.4s, v24.4s\n" + "sqrdmulh v23.4s, v23.4s, v24.4s\n" "tbz %x[flags], #5, 50f\n" - "and v4.16b, v16.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "and v5.16b, v17.16b, v0.16b\n" - "and v6.16b, v18.16b, v0.16b\n" - "and v7.16b, v19.16b, v0.16b\n" - "and v8.16b, v20.16b, v0.16b\n" - "and v9.16b, v21.16b, v0.16b\n" - "and v10.16b, v22.16b, v0.16b\n" - "and v4.16b, v23.16b, v0.16b\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v9.4s, v9.4s, #0x1f\n" - "sshr v10.4s, v10.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" - "sqadd v20.4s, v20.4s, v8.4s\n" - "sqadd v21.4s, v21.4s, v9.4s\n" - "sqadd v22.4s, v22.4s, v10.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" + "and v24.16b, v16.16b, v0.16b\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v24.4s\n" + "and v30.16b, v17.16b, v0.16b\n" + "and v29.16b, v18.16b, v0.16b\n" + "and v28.16b, v19.16b, v0.16b\n" + "and v27.16b, v20.16b, v0.16b\n" + "and v26.16b, v21.16b, v0.16b\n" + "and v25.16b, v22.16b, v0.16b\n" + "and v24.16b, v23.16b, v0.16b\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sshr v27.4s, v27.4s, #0x1f\n" + "sshr v26.4s, v26.4s, #0x1f\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sqadd v17.4s, v17.4s, v30.4s\n" + "sqadd v18.4s, v18.4s, v29.4s\n" + "sqadd v19.4s, v19.4s, v28.4s\n" + "sqadd v20.4s, v20.4s, v27.4s\n" + "sqadd v21.4s, v21.4s, v26.4s\n" + "sqadd v22.4s, v22.4s, v25.4s\n" + "sqadd v23.4s, v23.4s, v24.4s\n" "50:" // Height 2: no shift correction - "add x23, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v26.4s }, [x20]\n" "srshl v16.4s, v16.4s, v0.4s\n" "srshl v17.4s, v17.4s, v0.4s\n" "srshl v18.4s, v18.4s, v0.4s\n" "srshl v19.4s, v19.4s, v0.4s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x23]\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v25.4s }, [x20]\n" "srshl v20.4s, v20.4s, v0.4s\n" "srshl v21.4s, v21.4s, v0.4s\n" - "add x23, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x23]\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v24.4s }, [x20]\n" "srshl v22.4s, v22.4s, v0.4s\n" "srshl v23.4s, v23.4s, v0.4s\n" "cmp x9, #0x10\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v23.4s, v23.4s, v4.4s\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" + "add v16.4s, v16.4s, v26.4s\n" + "add v17.4s, v17.4s, v26.4s\n" + "add v18.4s, v18.4s, v26.4s\n" + "add v19.4s, v19.4s, v26.4s\n" + "add v20.4s, v20.4s, v26.4s\n" + "add v21.4s, v21.4s, v26.4s\n" + "add v22.4s, v22.4s, v26.4s\n" + "add v23.4s, v23.4s, v26.4s\n" + "smin v16.4s, v16.4s, v25.4s\n" + "smin v17.4s, v17.4s, v25.4s\n" + "smin v18.4s, v18.4s, v25.4s\n" + "smin v19.4s, v19.4s, v25.4s\n" + "smin v20.4s, v20.4s, v25.4s\n" + "smin v21.4s, v21.4s, v25.4s\n" + "smin v22.4s, v22.4s, v25.4s\n" + "smin v23.4s, v23.4s, v25.4s\n" + "smax v16.4s, v16.4s, v24.4s\n" + "smax v17.4s, v17.4s, v24.4s\n" + "smax v18.4s, v18.4s, v24.4s\n" + "smax v19.4s, v19.4s, v24.4s\n" + "smax v20.4s, v20.4s, v24.4s\n" + "smax v21.4s, v21.4s, v24.4s\n" + "smax v22.4s, v22.4s, v24.4s\n" + "smax v23.4s, v23.4s, v24.4s\n" "uzp1 v16.8h, v16.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v18.8h, v18.8h, v19.8h\n" "uzp1 v20.8h, v20.8h, v21.8h\n" - "uzp1 v21.8h, v22.8h, v23.8h\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" - "uzp1 v20.16b, v20.16b, v21.16b\n" + "uzp1 v17.8h, v22.8h, v23.8h\n" + "uzp1 v16.16b, v16.16b, v18.16b\n" + "uzp1 v20.16b, v20.16b, v17.16b\n" "bge 59f\n" "tbz x9, #3, 54f\n" "str d16, [x27], #0x8\n" - "str d20, [x22], #0x8\n" + "str d20, [x23], #0x8\n" "tbz x9, #2, 52f\n" "st1 { v16.s }[2], [x27], #0x4\n" - "st1 { v20.s }[2], [x22], #0x4\n" + "st1 { v20.s }[2], [x23], #0x4\n" "tbz x9, #1, 51f\n" "st1 { v16.h }[6], [x27], #0x2\n" - "st1 { v20.h }[6], [x22], #0x2\n" + "st1 { v20.h }[6], [x23], #0x2\n" "tbz x9, #0, 58f\n" "st1 { v16.b }[14], [x27]\n" - "st1 { v20.b }[14], [x22]\n" + "st1 { v20.b }[14], [x23]\n" "b 58f\n" "51:" // Height 2: Partial direct writeback: partial_1_12 "tbz x9, #0, 58f\n" "st1 { v16.b }[12], [x27]\n" - "st1 { v20.b }[12], [x22]\n" + "st1 { v20.b }[12], [x23]\n" "b 58f\n" "52:" // Height 2: Partial direct writeback: partial_2_8 "tbz x9, #1, 53f\n" "st1 { v16.h }[4], [x27], #0x2\n" - "st1 { v20.h }[4], [x22], #0x2\n" + "st1 { v20.h }[4], [x23], #0x2\n" "tbz x9, #0, 58f\n" "st1 { v16.b }[10], [x27]\n" - "st1 { v20.b }[10], [x22]\n" + "st1 { v20.b }[10], [x23]\n" "b 58f\n" "53:" // Height 2: Partial direct writeback: partial_1_8 "tbz x9, #0, 58f\n" "st1 { v16.b }[8], [x27]\n" - "st1 { v20.b }[8], [x22]\n" + "st1 { v20.b }[8], [x23]\n" "b 58f\n" "54:" // Height 2: Partial direct writeback: partial_4_0 "tbz x9, #2, 56f\n" "str s16, [x27], #0x4\n" - "str s20, [x22], #0x4\n" + "str s20, [x23], #0x4\n" "tbz x9, #1, 55f\n" "st1 { v16.h }[2], [x27], #0x2\n" - "st1 { v20.h }[2], [x22], #0x2\n" + "st1 { v20.h }[2], [x23], #0x2\n" "tbz x9, #0, 58f\n" "st1 { v16.b }[6], [x27]\n" - "st1 { v20.b }[6], [x22]\n" + "st1 { v20.b }[6], [x23]\n" "b 58f\n" "55:" // Height 2: Partial direct writeback: partial_1_4 "tbz x9, #0, 58f\n" "st1 { v16.b }[4], [x27]\n" - "st1 { v20.b }[4], [x22]\n" + "st1 { v20.b }[4], [x23]\n" "b 58f\n" "56:" // Height 2: Partial direct writeback: partial_2_0 "tbz x9, #1, 57f\n" "str h16, [x27], #0x2\n" - "str h20, [x22], #0x2\n" + "str h20, [x23], #0x2\n" "tbz x9, #0, 58f\n" "st1 { v16.b }[2], [x27]\n" - "st1 { v20.b }[2], [x22]\n" + "st1 { v20.b }[2], [x23]\n" "b 58f\n" "57:" // Height 2: Partial direct writeback: partial_1_0 "str b16, [x27, #0x0]\n" - "str b20, [x22, #0x0]\n" + "str b20, [x23, #0x0]\n" "58:" // Height 2: Partial direct writeback: Done "b 60f\n" "59:" // Height 2: Full writeback "str q16, [x27, #0x0]\n" "add x27, x27, #0x10\n" - "str q20, [x22, #0x0]\n" + "str q20, [x23, #0x0]\n" "60:" // Height 2: Writeback done "subs x9, x9, #0x10\n" "bgt 32b\n" @@ -819,13 +818,13 @@ void a64_hybrid_u8qa_dot_4x16 ( "64:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 65f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" "cbnz x26, 66f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -834,8 +833,8 @@ void a64_hybrid_u8qa_dot_4x16 ( "b 66f\n" "65:" // Height 3: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" "66:" // Height 3: input setup done "cmp x25, #0x10\n" "blt 71f\n" @@ -857,62 +856,62 @@ void a64_hybrid_u8qa_dot_4x16 ( "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" ".inst 0x6f82e098 // udot v24.4s, v4.16b, v2.4b[0]\n" - "ldr q4, [x28, #0x70]\n" + "ldr q29, [x28, #0x70]\n" ".inst 0x6f80e0b1 // udot v17.4s, v5.16b, v0.4b[0]\n" "add x22, x22, #0x10\n" ".inst 0x6f81e0b5 // udot v21.4s, v5.16b, v1.4b[0]\n" ".inst 0x6f82e0b9 // udot v25.4s, v5.16b, v2.4b[0]\n" - "ldr q5, [x28, #0x80]\n" + "ldr q28, [x28, #0x80]\n" ".inst 0x6f80e0d2 // udot v18.4s, v6.16b, v0.4b[0]\n" ".inst 0x6f81e0d6 // udot v22.4s, v6.16b, v1.4b[0]\n" ".inst 0x6f82e0da // udot v26.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x28, #0x90]\n" + "ldr q5, [x28, #0x90]\n" ".inst 0x6f80e0f3 // udot v19.4s, v7.16b, v0.4b[0]\n" ".inst 0x6f81e0f7 // udot v23.4s, v7.16b, v1.4b[0]\n" ".inst 0x6f82e0fb // udot v27.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x28, #0xa0]\n" + "ldr q4, [x28, #0xa0]\n" ".inst 0x6fa0e110 // udot v16.4s, v8.16b, v0.4b[1]\n" ".inst 0x6fa1e114 // udot v20.4s, v8.16b, v1.4b[1]\n" ".inst 0x6fa2e118 // udot v24.4s, v8.16b, v2.4b[1]\n" - "ldr q8, [x28, #0xb0]\n" + "ldr q3, [x28, #0xb0]\n" ".inst 0x6fa0e131 // udot v17.4s, v9.16b, v0.4b[1]\n" ".inst 0x6fa1e135 // udot v21.4s, v9.16b, v1.4b[1]\n" ".inst 0x6fa2e139 // udot v25.4s, v9.16b, v2.4b[1]\n" - "ldr q9, [x28, #0xc0]\n" + "ldr q31, [x28, #0xc0]\n" ".inst 0x6fa0e152 // udot v18.4s, v10.16b, v0.4b[1]\n" ".inst 0x6fa1e156 // udot v22.4s, v10.16b, v1.4b[1]\n" ".inst 0x6fa2e15a // udot v26.4s, v10.16b, v2.4b[1]\n" - "ldr q10, [x28, #0xd0]\n" - ".inst 0x6fa0e093 // udot v19.4s, v4.16b, v0.4b[1]\n" - ".inst 0x6fa1e097 // udot v23.4s, v4.16b, v1.4b[1]\n" - ".inst 0x6fa2e09b // udot v27.4s, v4.16b, v2.4b[1]\n" - "ldr q4, [x28, #0xe0]\n" - ".inst 0x6f80e8b0 // udot v16.4s, v5.16b, v0.4b[2]\n" - ".inst 0x6f81e8b4 // udot v20.4s, v5.16b, v1.4b[2]\n" - ".inst 0x6f82e8b8 // udot v24.4s, v5.16b, v2.4b[2]\n" - "ldr q5, [x28, #0xf0]\n" - ".inst 0x6f80e8d1 // udot v17.4s, v6.16b, v0.4b[2]\n" + "ldr q30, [x28, #0xd0]\n" + ".inst 0x6fa0e3b3 // udot v19.4s, v29.16b, v0.4b[1]\n" + ".inst 0x6fa1e3b7 // udot v23.4s, v29.16b, v1.4b[1]\n" + ".inst 0x6fa2e3bb // udot v27.4s, v29.16b, v2.4b[1]\n" + "ldr q29, [x28, #0xe0]\n" + ".inst 0x6f80eb90 // udot v16.4s, v28.16b, v0.4b[2]\n" + ".inst 0x6f81eb94 // udot v20.4s, v28.16b, v1.4b[2]\n" + ".inst 0x6f82eb98 // udot v24.4s, v28.16b, v2.4b[2]\n" + "ldr q28, [x28, #0xf0]\n" + ".inst 0x6f80e8b1 // udot v17.4s, v5.16b, v0.4b[2]\n" "add x28, x28, #0x100\n" - ".inst 0x6f81e8d5 // udot v21.4s, v6.16b, v1.4b[2]\n" - ".inst 0x6f82e8d9 // udot v25.4s, v6.16b, v2.4b[2]\n" - ".inst 0x6f80e8f2 // udot v18.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8f6 // udot v22.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f82e8fa // udot v26.4s, v7.16b, v2.4b[2]\n" - ".inst 0x6f80e913 // udot v19.4s, v8.16b, v0.4b[2]\n" - ".inst 0x6f81e917 // udot v23.4s, v8.16b, v1.4b[2]\n" - ".inst 0x6f82e91b // udot v27.4s, v8.16b, v2.4b[2]\n" - ".inst 0x6fa0e930 // udot v16.4s, v9.16b, v0.4b[3]\n" - ".inst 0x6fa1e934 // udot v20.4s, v9.16b, v1.4b[3]\n" - ".inst 0x6fa2e938 // udot v24.4s, v9.16b, v2.4b[3]\n" - ".inst 0x6fa0e951 // udot v17.4s, v10.16b, v0.4b[3]\n" - ".inst 0x6fa1e955 // udot v21.4s, v10.16b, v1.4b[3]\n" - ".inst 0x6fa2e959 // udot v25.4s, v10.16b, v2.4b[3]\n" - ".inst 0x6fa0e892 // udot v18.4s, v4.16b, v0.4b[3]\n" - ".inst 0x6fa1e896 // udot v22.4s, v4.16b, v1.4b[3]\n" - ".inst 0x6fa2e89a // udot v26.4s, v4.16b, v2.4b[3]\n" - ".inst 0x6fa0e8b3 // udot v19.4s, v5.16b, v0.4b[3]\n" - ".inst 0x6fa1e8b7 // udot v23.4s, v5.16b, v1.4b[3]\n" - ".inst 0x6fa2e8bb // udot v27.4s, v5.16b, v2.4b[3]\n" + ".inst 0x6f81e8b5 // udot v21.4s, v5.16b, v1.4b[2]\n" + ".inst 0x6f82e8b9 // udot v25.4s, v5.16b, v2.4b[2]\n" + ".inst 0x6f80e892 // udot v18.4s, v4.16b, v0.4b[2]\n" + ".inst 0x6f81e896 // udot v22.4s, v4.16b, v1.4b[2]\n" + ".inst 0x6f82e89a // udot v26.4s, v4.16b, v2.4b[2]\n" + ".inst 0x6f80e873 // udot v19.4s, v3.16b, v0.4b[2]\n" + ".inst 0x6f81e877 // udot v23.4s, v3.16b, v1.4b[2]\n" + ".inst 0x6f82e87b // udot v27.4s, v3.16b, v2.4b[2]\n" + ".inst 0x6fa0ebf0 // udot v16.4s, v31.16b, v0.4b[3]\n" + ".inst 0x6fa1ebf4 // udot v20.4s, v31.16b, v1.4b[3]\n" + ".inst 0x6fa2ebf8 // udot v24.4s, v31.16b, v2.4b[3]\n" + ".inst 0x6fa0ebd1 // udot v17.4s, v30.16b, v0.4b[3]\n" + ".inst 0x6fa1ebd5 // udot v21.4s, v30.16b, v1.4b[3]\n" + ".inst 0x6fa2ebd9 // udot v25.4s, v30.16b, v2.4b[3]\n" + ".inst 0x6fa0ebb2 // udot v18.4s, v29.16b, v0.4b[3]\n" + ".inst 0x6fa1ebb6 // udot v22.4s, v29.16b, v1.4b[3]\n" + ".inst 0x6fa2ebba // udot v26.4s, v29.16b, v2.4b[3]\n" + ".inst 0x6fa0eb93 // udot v19.4s, v28.16b, v0.4b[3]\n" + ".inst 0x6fa1eb97 // udot v23.4s, v28.16b, v1.4b[3]\n" + ".inst 0x6fa2eb9b // udot v27.4s, v28.16b, v2.4b[3]\n" "tbnz %x[flags], #31, 68f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n" @@ -940,63 +939,63 @@ void a64_hybrid_u8qa_dot_4x16 ( "sub x25, x25, #0x10\n" "add x24, x24, #0x10\n" ".inst 0x6f82e098 // udot v24.4s, v4.16b, v2.4b[0]\n" - "ldr q4, [x28, #0x70]\n" + "ldr q29, [x28, #0x70]\n" ".inst 0x6f80e0b1 // udot v17.4s, v5.16b, v0.4b[0]\n" "add x23, x23, #0x10\n" ".inst 0x6f81e0b5 // udot v21.4s, v5.16b, v1.4b[0]\n" ".inst 0x6f82e0b9 // udot v25.4s, v5.16b, v2.4b[0]\n" - "ldr q5, [x28, #0x80]\n" + "ldr q28, [x28, #0x80]\n" "add x22, x22, #0x10\n" ".inst 0x6f80e0d2 // udot v18.4s, v6.16b, v0.4b[0]\n" ".inst 0x6f81e0d6 // udot v22.4s, v6.16b, v1.4b[0]\n" ".inst 0x6f82e0da // udot v26.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x28, #0x90]\n" + "ldr q5, [x28, #0x90]\n" ".inst 0x6f80e0f3 // udot v19.4s, v7.16b, v0.4b[0]\n" ".inst 0x6f81e0f7 // udot v23.4s, v7.16b, v1.4b[0]\n" ".inst 0x6f82e0fb // udot v27.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x28, #0xa0]\n" + "ldr q4, [x28, #0xa0]\n" ".inst 0x6fa0e110 // udot v16.4s, v8.16b, v0.4b[1]\n" ".inst 0x6fa1e114 // udot v20.4s, v8.16b, v1.4b[1]\n" ".inst 0x6fa2e118 // udot v24.4s, v8.16b, v2.4b[1]\n" - "ldr q8, [x28, #0xb0]\n" + "ldr q3, [x28, #0xb0]\n" ".inst 0x6fa0e131 // udot v17.4s, v9.16b, v0.4b[1]\n" ".inst 0x6fa1e135 // udot v21.4s, v9.16b, v1.4b[1]\n" ".inst 0x6fa2e139 // udot v25.4s, v9.16b, v2.4b[1]\n" - "ldr q9, [x28, #0xc0]\n" + "ldr q31, [x28, #0xc0]\n" ".inst 0x6fa0e152 // udot v18.4s, v10.16b, v0.4b[1]\n" ".inst 0x6fa1e156 // udot v22.4s, v10.16b, v1.4b[1]\n" ".inst 0x6fa2e15a // udot v26.4s, v10.16b, v2.4b[1]\n" - "ldr q10, [x28, #0xd0]\n" - ".inst 0x6fa0e093 // udot v19.4s, v4.16b, v0.4b[1]\n" - ".inst 0x6fa1e097 // udot v23.4s, v4.16b, v1.4b[1]\n" - ".inst 0x6fa2e09b // udot v27.4s, v4.16b, v2.4b[1]\n" - "ldr q4, [x28, #0xe0]\n" - ".inst 0x6f80e8b0 // udot v16.4s, v5.16b, v0.4b[2]\n" - ".inst 0x6f81e8b4 // udot v20.4s, v5.16b, v1.4b[2]\n" - ".inst 0x6f82e8b8 // udot v24.4s, v5.16b, v2.4b[2]\n" - "ldr q5, [x28, #0xf0]\n" - ".inst 0x6f80e8d1 // udot v17.4s, v6.16b, v0.4b[2]\n" + "ldr q30, [x28, #0xd0]\n" + ".inst 0x6fa0e3b3 // udot v19.4s, v29.16b, v0.4b[1]\n" + ".inst 0x6fa1e3b7 // udot v23.4s, v29.16b, v1.4b[1]\n" + ".inst 0x6fa2e3bb // udot v27.4s, v29.16b, v2.4b[1]\n" + "ldr q29, [x28, #0xe0]\n" + ".inst 0x6f80eb90 // udot v16.4s, v28.16b, v0.4b[2]\n" + ".inst 0x6f81eb94 // udot v20.4s, v28.16b, v1.4b[2]\n" + ".inst 0x6f82eb98 // udot v24.4s, v28.16b, v2.4b[2]\n" + "ldr q28, [x28, #0xf0]\n" + ".inst 0x6f80e8b1 // udot v17.4s, v5.16b, v0.4b[2]\n" "add x28, x28, #0x100\n" - ".inst 0x6f81e8d5 // udot v21.4s, v6.16b, v1.4b[2]\n" - ".inst 0x6f82e8d9 // udot v25.4s, v6.16b, v2.4b[2]\n" - ".inst 0x6f80e8f2 // udot v18.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8f6 // udot v22.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f82e8fa // udot v26.4s, v7.16b, v2.4b[2]\n" - ".inst 0x6f80e913 // udot v19.4s, v8.16b, v0.4b[2]\n" - ".inst 0x6f81e917 // udot v23.4s, v8.16b, v1.4b[2]\n" - ".inst 0x6f82e91b // udot v27.4s, v8.16b, v2.4b[2]\n" - ".inst 0x6fa0e930 // udot v16.4s, v9.16b, v0.4b[3]\n" - ".inst 0x6fa1e934 // udot v20.4s, v9.16b, v1.4b[3]\n" - ".inst 0x6fa2e938 // udot v24.4s, v9.16b, v2.4b[3]\n" - ".inst 0x6fa0e951 // udot v17.4s, v10.16b, v0.4b[3]\n" - ".inst 0x6fa1e955 // udot v21.4s, v10.16b, v1.4b[3]\n" - ".inst 0x6fa2e959 // udot v25.4s, v10.16b, v2.4b[3]\n" - ".inst 0x6fa0e892 // udot v18.4s, v4.16b, v0.4b[3]\n" - ".inst 0x6fa1e896 // udot v22.4s, v4.16b, v1.4b[3]\n" - ".inst 0x6fa2e89a // udot v26.4s, v4.16b, v2.4b[3]\n" - ".inst 0x6fa0e8b3 // udot v19.4s, v5.16b, v0.4b[3]\n" - ".inst 0x6fa1e8b7 // udot v23.4s, v5.16b, v1.4b[3]\n" - ".inst 0x6fa2e8bb // udot v27.4s, v5.16b, v2.4b[3]\n" + ".inst 0x6f81e8b5 // udot v21.4s, v5.16b, v1.4b[2]\n" + ".inst 0x6f82e8b9 // udot v25.4s, v5.16b, v2.4b[2]\n" + ".inst 0x6f80e892 // udot v18.4s, v4.16b, v0.4b[2]\n" + ".inst 0x6f81e896 // udot v22.4s, v4.16b, v1.4b[2]\n" + ".inst 0x6f82e89a // udot v26.4s, v4.16b, v2.4b[2]\n" + ".inst 0x6f80e873 // udot v19.4s, v3.16b, v0.4b[2]\n" + ".inst 0x6f81e877 // udot v23.4s, v3.16b, v1.4b[2]\n" + ".inst 0x6f82e87b // udot v27.4s, v3.16b, v2.4b[2]\n" + ".inst 0x6fa0ebf0 // udot v16.4s, v31.16b, v0.4b[3]\n" + ".inst 0x6fa1ebf4 // udot v20.4s, v31.16b, v1.4b[3]\n" + ".inst 0x6fa2ebf8 // udot v24.4s, v31.16b, v2.4b[3]\n" + ".inst 0x6fa0ebd1 // udot v17.4s, v30.16b, v0.4b[3]\n" + ".inst 0x6fa1ebd5 // udot v21.4s, v30.16b, v1.4b[3]\n" + ".inst 0x6fa2ebd9 // udot v25.4s, v30.16b, v2.4b[3]\n" + ".inst 0x6fa0ebb2 // udot v18.4s, v29.16b, v0.4b[3]\n" + ".inst 0x6fa1ebb6 // udot v22.4s, v29.16b, v1.4b[3]\n" + ".inst 0x6fa2ebba // udot v26.4s, v29.16b, v2.4b[3]\n" + ".inst 0x6fa0eb93 // udot v19.4s, v28.16b, v0.4b[3]\n" + ".inst 0x6fa1eb97 // udot v23.4s, v28.16b, v1.4b[3]\n" + ".inst 0x6fa2eb9b // udot v27.4s, v28.16b, v2.4b[3]\n" "tbnz %x[flags], #31, 70f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n" @@ -1018,25 +1017,25 @@ void a64_hybrid_u8qa_dot_4x16 ( ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n" ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n" "73:" // Height 3: Multiply loop: unique 11: skip row sum - "ldr q6, [x28, #0x0]\n" - "ldr q7, [x28, #0x10]\n" + "ldr q31, [x28, #0x0]\n" + "ldr q30, [x28, #0x10]\n" "sub x25, x25, #0x4\n" "cmp x25, #0x4\n" - "ldr q8, [x28, #0x20]\n" - "ldr q9, [x28, #0x30]\n" - ".inst 0x6f80e0d0 // udot v16.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0d4 // udot v20.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d8 // udot v24.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f80e0f1 // udot v17.4s, v7.16b, v0.4b[0]\n" + "ldr q29, [x28, #0x20]\n" + "ldr q28, [x28, #0x30]\n" + ".inst 0x6f80e3f0 // udot v16.4s, v31.16b, v0.4b[0]\n" + ".inst 0x6f81e3f4 // udot v20.4s, v31.16b, v1.4b[0]\n" + ".inst 0x6f82e3f8 // udot v24.4s, v31.16b, v2.4b[0]\n" + ".inst 0x6f80e3d1 // udot v17.4s, v30.16b, v0.4b[0]\n" "add x28, x28, #0x40\n" - ".inst 0x6f81e0f5 // udot v21.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f9 // udot v25.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f80e112 // udot v18.4s, v8.16b, v0.4b[0]\n" - ".inst 0x6f81e116 // udot v22.4s, v8.16b, v1.4b[0]\n" - ".inst 0x6f82e11a // udot v26.4s, v8.16b, v2.4b[0]\n" - ".inst 0x6f80e133 // udot v19.4s, v9.16b, v0.4b[0]\n" - ".inst 0x6f81e137 // udot v23.4s, v9.16b, v1.4b[0]\n" - ".inst 0x6f82e13b // udot v27.4s, v9.16b, v2.4b[0]\n" + ".inst 0x6f81e3d5 // udot v21.4s, v30.16b, v1.4b[0]\n" + ".inst 0x6f82e3d9 // udot v25.4s, v30.16b, v2.4b[0]\n" + ".inst 0x6f80e3b2 // udot v18.4s, v29.16b, v0.4b[0]\n" + ".inst 0x6f81e3b6 // udot v22.4s, v29.16b, v1.4b[0]\n" + ".inst 0x6f82e3ba // udot v26.4s, v29.16b, v2.4b[0]\n" + ".inst 0x6f80e393 // udot v19.4s, v28.16b, v0.4b[0]\n" + ".inst 0x6f81e397 // udot v23.4s, v28.16b, v1.4b[0]\n" + ".inst 0x6f82e39b // udot v27.4s, v28.16b, v2.4b[0]\n" "bge 72b\n" "74:" // Height 3: Multiply loop: Skip odd blocks "cbz x25, 78f\n" @@ -1059,144 +1058,144 @@ void a64_hybrid_u8qa_dot_4x16 ( ".inst 0x6e8f942c // udot v12.4s, v1.16b, v15.16b\n" ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n" "77:" // Height 3: Multiply loop: unique 12: skip row sum - "ldr q10, [x28, #0x0]\n" - "ldr q4, [x28, #0x10]\n" - ".inst 0x6f80e150 // udot v16.4s, v10.16b, v0.4b[0]\n" - ".inst 0x6f81e154 // udot v20.4s, v10.16b, v1.4b[0]\n" - "ldr q5, [x28, #0x20]\n" - "ldr q6, [x28, #0x30]\n" - ".inst 0x6f82e158 // udot v24.4s, v10.16b, v2.4b[0]\n" - ".inst 0x6f80e091 // udot v17.4s, v4.16b, v0.4b[0]\n" - ".inst 0x6f81e095 // udot v21.4s, v4.16b, v1.4b[0]\n" - ".inst 0x6f82e099 // udot v25.4s, v4.16b, v2.4b[0]\n" + "ldr q31, [x28, #0x0]\n" + "ldr q30, [x28, #0x10]\n" + ".inst 0x6f80e3f0 // udot v16.4s, v31.16b, v0.4b[0]\n" + ".inst 0x6f81e3f4 // udot v20.4s, v31.16b, v1.4b[0]\n" + "ldr q29, [x28, #0x20]\n" + "ldr q28, [x28, #0x30]\n" + ".inst 0x6f82e3f8 // udot v24.4s, v31.16b, v2.4b[0]\n" + ".inst 0x6f80e3d1 // udot v17.4s, v30.16b, v0.4b[0]\n" + ".inst 0x6f81e3d5 // udot v21.4s, v30.16b, v1.4b[0]\n" + ".inst 0x6f82e3d9 // udot v25.4s, v30.16b, v2.4b[0]\n" "add x28, x28, #0x40\n" - ".inst 0x6f80e0b2 // udot v18.4s, v5.16b, v0.4b[0]\n" - ".inst 0x6f81e0b6 // udot v22.4s, v5.16b, v1.4b[0]\n" - ".inst 0x6f82e0ba // udot v26.4s, v5.16b, v2.4b[0]\n" - ".inst 0x6f80e0d3 // udot v19.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0d7 // udot v23.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0db // udot v27.4s, v6.16b, v2.4b[0]\n" + ".inst 0x6f80e3b2 // udot v18.4s, v29.16b, v0.4b[0]\n" + ".inst 0x6f81e3b6 // udot v22.4s, v29.16b, v1.4b[0]\n" + ".inst 0x6f82e3ba // udot v26.4s, v29.16b, v2.4b[0]\n" + ".inst 0x6f80e393 // udot v19.4s, v28.16b, v0.4b[0]\n" + ".inst 0x6f81e397 // udot v23.4s, v28.16b, v1.4b[0]\n" + ".inst 0x6f82e39b // udot v27.4s, v28.16b, v2.4b[0]\n" "78:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x26, x26, #0x1\n" "cmp x26, x20\n" "bne 64b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x22, x27, x20\n" - "add x21, x22, x20\n" + "add x23, x27, x20\n" + "add x22, x23, x20\n" "prfm pstl1keep, [x27, #0x0]\n" + "prfm pstl1keep, [x23, #0x0]\n" "prfm pstl1keep, [x22, #0x0]\n" - "prfm pstl1keep, [x21, #0x0]\n" "tbnz %x[flags], #31, 79f\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1r { v3.4s }, [x23]\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1r { v28.4s }, [x20]\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v12.4s, v12.4s, v12.4s\n" "addp v13.4s, v13.4s, v13.4s\n" - "neg v3.4s, v3.4s\n" + "neg v28.4s, v28.4s\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v12.4s, v12.4s, v12.4s\n" "addp v13.4s, v13.4s, v13.4s\n" - "mul v11.4s, v11.4s, v3.4s\n" - "mul v12.4s, v12.4s, v3.4s\n" - "mul v13.4s, v13.4s, v3.4s\n" + "mul v11.4s, v11.4s, v28.4s\n" + "mul v12.4s, v12.4s, v28.4s\n" + "mul v13.4s, v13.4s, v28.4s\n" "79:" // Height 3: skip row sum fixup "ldr q0, [x10, #0x0]\n" - "ldr q1, [x10, #0x10]\n" + "ldr q31, [x10, #0x10]\n" "add v16.4s, v16.4s, v11.4s\n" "add v17.4s, v17.4s, v11.4s\n" - "ldr q2, [x10, #0x20]\n" - "ldr q3, [x10, #0x30]\n" + "ldr q30, [x10, #0x20]\n" + "ldr q29, [x10, #0x30]\n" "add v18.4s, v18.4s, v11.4s\n" "add v19.4s, v19.4s, v11.4s\n" "add v20.4s, v20.4s, v12.4s\n" "add v21.4s, v21.4s, v12.4s\n" - "add x23, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v28.4s }, [x20]\n" "add v22.4s, v22.4s, v12.4s\n" "add v23.4s, v23.4s, v12.4s\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" "add v24.4s, v24.4s, v13.4s\n" "add v25.4s, v25.4s, v13.4s\n" "add x10, x10, #0x40\n" "add v26.4s, v26.4s, v13.4s\n" "add v27.4s, v27.4s, v13.4s\n" "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" + "add v17.4s, v17.4s, v31.4s\n" + "add v18.4s, v18.4s, v30.4s\n" + "add v19.4s, v19.4s, v29.4s\n" "add v20.4s, v20.4s, v0.4s\n" - "add v21.4s, v21.4s, v1.4s\n" - "add v22.4s, v22.4s, v2.4s\n" - "add v23.4s, v23.4s, v3.4s\n" + "add v21.4s, v21.4s, v31.4s\n" + "add v22.4s, v22.4s, v30.4s\n" + "add v23.4s, v23.4s, v29.4s\n" "add v24.4s, v24.4s, v0.4s\n" - "ld1r { v0.4s }, [x23]\n" - "add v25.4s, v25.4s, v1.4s\n" - "add v26.4s, v26.4s, v2.4s\n" - "add v27.4s, v27.4s, v3.4s\n" - "sqrdmulh v16.4s, v16.4s, v4.4s\n" - "sqrdmulh v17.4s, v17.4s, v4.4s\n" - "sqrdmulh v18.4s, v18.4s, v4.4s\n" - "sqrdmulh v19.4s, v19.4s, v4.4s\n" - "sqrdmulh v20.4s, v20.4s, v4.4s\n" - "sqrdmulh v21.4s, v21.4s, v4.4s\n" - "sqrdmulh v22.4s, v22.4s, v4.4s\n" - "sqrdmulh v23.4s, v23.4s, v4.4s\n" - "sqrdmulh v24.4s, v24.4s, v4.4s\n" - "sqrdmulh v25.4s, v25.4s, v4.4s\n" - "sqrdmulh v26.4s, v26.4s, v4.4s\n" - "sqrdmulh v27.4s, v27.4s, v4.4s\n" + "ld1r { v0.4s }, [x20]\n" + "add v25.4s, v25.4s, v31.4s\n" + "add v26.4s, v26.4s, v30.4s\n" + "add v27.4s, v27.4s, v29.4s\n" + "sqrdmulh v16.4s, v16.4s, v28.4s\n" + "sqrdmulh v17.4s, v17.4s, v28.4s\n" + "sqrdmulh v18.4s, v18.4s, v28.4s\n" + "sqrdmulh v19.4s, v19.4s, v28.4s\n" + "sqrdmulh v20.4s, v20.4s, v28.4s\n" + "sqrdmulh v21.4s, v21.4s, v28.4s\n" + "sqrdmulh v22.4s, v22.4s, v28.4s\n" + "sqrdmulh v23.4s, v23.4s, v28.4s\n" + "sqrdmulh v24.4s, v24.4s, v28.4s\n" + "sqrdmulh v25.4s, v25.4s, v28.4s\n" + "sqrdmulh v26.4s, v26.4s, v28.4s\n" + "sqrdmulh v27.4s, v27.4s, v28.4s\n" "tbz %x[flags], #5, 80f\n" - "and v4.16b, v16.16b, v0.16b\n" - "and v5.16b, v17.16b, v0.16b\n" - "and v6.16b, v18.16b, v0.16b\n" - "and v7.16b, v19.16b, v0.16b\n" - "and v8.16b, v20.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" - "sqadd v20.4s, v20.4s, v8.4s\n" - "and v9.16b, v21.16b, v0.16b\n" - "and v10.16b, v22.16b, v0.16b\n" - "and v4.16b, v23.16b, v0.16b\n" - "and v5.16b, v24.16b, v0.16b\n" - "and v6.16b, v25.16b, v0.16b\n" - "and v7.16b, v26.16b, v0.16b\n" - "and v8.16b, v27.16b, v0.16b\n" - "sshr v9.4s, v9.4s, #0x1f\n" - "sshr v10.4s, v10.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sqadd v21.4s, v21.4s, v9.4s\n" - "sqadd v22.4s, v22.4s, v10.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v24.4s, v24.4s, v5.4s\n" - "sqadd v25.4s, v25.4s, v6.4s\n" - "sqadd v26.4s, v26.4s, v7.4s\n" - "sqadd v27.4s, v27.4s, v8.4s\n" + "and v1.16b, v16.16b, v0.16b\n" + "and v31.16b, v17.16b, v0.16b\n" + "and v30.16b, v18.16b, v0.16b\n" + "and v29.16b, v19.16b, v0.16b\n" + "and v28.16b, v20.16b, v0.16b\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v1.4s\n" + "sqadd v17.4s, v17.4s, v31.4s\n" + "sqadd v18.4s, v18.4s, v30.4s\n" + "sqadd v19.4s, v19.4s, v29.4s\n" + "sqadd v20.4s, v20.4s, v28.4s\n" + "and v3.16b, v21.16b, v0.16b\n" + "and v2.16b, v22.16b, v0.16b\n" + "and v1.16b, v23.16b, v0.16b\n" + "and v31.16b, v24.16b, v0.16b\n" + "and v30.16b, v25.16b, v0.16b\n" + "and v29.16b, v26.16b, v0.16b\n" + "and v28.16b, v27.16b, v0.16b\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v31.4s, v31.4s, #0x1f\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sqadd v21.4s, v21.4s, v3.4s\n" + "sqadd v22.4s, v22.4s, v2.4s\n" + "sqadd v23.4s, v23.4s, v1.4s\n" + "sqadd v24.4s, v24.4s, v31.4s\n" + "sqadd v25.4s, v25.4s, v30.4s\n" + "sqadd v26.4s, v26.4s, v29.4s\n" + "sqadd v27.4s, v27.4s, v28.4s\n" "80:" // Height 3: no shift correction - "add x23, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v30.4s }, [x20]\n" "srshl v16.4s, v16.4s, v0.4s\n" "srshl v17.4s, v17.4s, v0.4s\n" "srshl v18.4s, v18.4s, v0.4s\n" "srshl v19.4s, v19.4s, v0.4s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x23]\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v29.4s }, [x20]\n" "srshl v20.4s, v20.4s, v0.4s\n" "srshl v21.4s, v21.4s, v0.4s\n" - "add x23, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x23]\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v28.4s }, [x20]\n" "srshl v22.4s, v22.4s, v0.4s\n" "srshl v23.4s, v23.4s, v0.4s\n" "cmp x9, #0x10\n" @@ -1204,132 +1203,132 @@ void a64_hybrid_u8qa_dot_4x16 ( "srshl v25.4s, v25.4s, v0.4s\n" "srshl v26.4s, v26.4s, v0.4s\n" "srshl v27.4s, v27.4s, v0.4s\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v23.4s, v23.4s, v4.4s\n" - "add v24.4s, v24.4s, v4.4s\n" - "add v25.4s, v25.4s, v4.4s\n" - "add v26.4s, v26.4s, v4.4s\n" - "add v27.4s, v27.4s, v4.4s\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "smin v24.4s, v24.4s, v6.4s\n" - "smin v25.4s, v25.4s, v6.4s\n" - "smin v26.4s, v26.4s, v6.4s\n" - "smin v27.4s, v27.4s, v6.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" - "smax v24.4s, v24.4s, v5.4s\n" - "smax v25.4s, v25.4s, v5.4s\n" - "smax v26.4s, v26.4s, v5.4s\n" - "smax v27.4s, v27.4s, v5.4s\n" + "add v16.4s, v16.4s, v30.4s\n" + "add v17.4s, v17.4s, v30.4s\n" + "add v18.4s, v18.4s, v30.4s\n" + "add v19.4s, v19.4s, v30.4s\n" + "add v20.4s, v20.4s, v30.4s\n" + "add v21.4s, v21.4s, v30.4s\n" + "add v22.4s, v22.4s, v30.4s\n" + "add v23.4s, v23.4s, v30.4s\n" + "add v24.4s, v24.4s, v30.4s\n" + "add v25.4s, v25.4s, v30.4s\n" + "add v26.4s, v26.4s, v30.4s\n" + "add v27.4s, v27.4s, v30.4s\n" + "smin v16.4s, v16.4s, v29.4s\n" + "smin v17.4s, v17.4s, v29.4s\n" + "smin v18.4s, v18.4s, v29.4s\n" + "smin v19.4s, v19.4s, v29.4s\n" + "smin v20.4s, v20.4s, v29.4s\n" + "smin v21.4s, v21.4s, v29.4s\n" + "smin v22.4s, v22.4s, v29.4s\n" + "smin v23.4s, v23.4s, v29.4s\n" + "smin v24.4s, v24.4s, v29.4s\n" + "smin v25.4s, v25.4s, v29.4s\n" + "smin v26.4s, v26.4s, v29.4s\n" + "smin v27.4s, v27.4s, v29.4s\n" + "smax v16.4s, v16.4s, v28.4s\n" + "smax v17.4s, v17.4s, v28.4s\n" + "smax v18.4s, v18.4s, v28.4s\n" + "smax v19.4s, v19.4s, v28.4s\n" + "smax v20.4s, v20.4s, v28.4s\n" + "smax v21.4s, v21.4s, v28.4s\n" + "smax v22.4s, v22.4s, v28.4s\n" + "smax v23.4s, v23.4s, v28.4s\n" + "smax v24.4s, v24.4s, v28.4s\n" + "smax v25.4s, v25.4s, v28.4s\n" + "smax v26.4s, v26.4s, v28.4s\n" + "smax v27.4s, v27.4s, v28.4s\n" "uzp1 v16.8h, v16.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v19.8h, v18.8h, v19.8h\n" "uzp1 v20.8h, v20.8h, v21.8h\n" - "uzp1 v21.8h, v22.8h, v23.8h\n" + "uzp1 v18.8h, v22.8h, v23.8h\n" "uzp1 v24.8h, v24.8h, v25.8h\n" - "uzp1 v25.8h, v26.8h, v27.8h\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" - "uzp1 v20.16b, v20.16b, v21.16b\n" - "uzp1 v24.16b, v24.16b, v25.16b\n" + "uzp1 v17.8h, v26.8h, v27.8h\n" + "uzp1 v16.16b, v16.16b, v19.16b\n" + "uzp1 v20.16b, v20.16b, v18.16b\n" + "uzp1 v24.16b, v24.16b, v17.16b\n" "bge 89f\n" "tbz x9, #3, 84f\n" "str d16, [x27], #0x8\n" - "str d20, [x22], #0x8\n" - "str d24, [x21], #0x8\n" + "str d20, [x23], #0x8\n" + "str d24, [x22], #0x8\n" "tbz x9, #2, 82f\n" "st1 { v16.s }[2], [x27], #0x4\n" - "st1 { v20.s }[2], [x22], #0x4\n" - "st1 { v24.s }[2], [x21], #0x4\n" + "st1 { v20.s }[2], [x23], #0x4\n" + "st1 { v24.s }[2], [x22], #0x4\n" "tbz x9, #1, 81f\n" "st1 { v16.h }[6], [x27], #0x2\n" - "st1 { v20.h }[6], [x22], #0x2\n" - "st1 { v24.h }[6], [x21], #0x2\n" + "st1 { v20.h }[6], [x23], #0x2\n" + "st1 { v24.h }[6], [x22], #0x2\n" "tbz x9, #0, 88f\n" "st1 { v16.b }[14], [x27]\n" - "st1 { v20.b }[14], [x22]\n" - "st1 { v24.b }[14], [x21]\n" + "st1 { v20.b }[14], [x23]\n" + "st1 { v24.b }[14], [x22]\n" "b 88f\n" "81:" // Height 3: Partial direct writeback: partial_1_12 "tbz x9, #0, 88f\n" "st1 { v16.b }[12], [x27]\n" - "st1 { v20.b }[12], [x22]\n" - "st1 { v24.b }[12], [x21]\n" + "st1 { v20.b }[12], [x23]\n" + "st1 { v24.b }[12], [x22]\n" "b 88f\n" "82:" // Height 3: Partial direct writeback: partial_2_8 "tbz x9, #1, 83f\n" "st1 { v16.h }[4], [x27], #0x2\n" - "st1 { v20.h }[4], [x22], #0x2\n" - "st1 { v24.h }[4], [x21], #0x2\n" + "st1 { v20.h }[4], [x23], #0x2\n" + "st1 { v24.h }[4], [x22], #0x2\n" "tbz x9, #0, 88f\n" "st1 { v16.b }[10], [x27]\n" - "st1 { v20.b }[10], [x22]\n" - "st1 { v24.b }[10], [x21]\n" + "st1 { v20.b }[10], [x23]\n" + "st1 { v24.b }[10], [x22]\n" "b 88f\n" "83:" // Height 3: Partial direct writeback: partial_1_8 "tbz x9, #0, 88f\n" "st1 { v16.b }[8], [x27]\n" - "st1 { v20.b }[8], [x22]\n" - "st1 { v24.b }[8], [x21]\n" + "st1 { v20.b }[8], [x23]\n" + "st1 { v24.b }[8], [x22]\n" "b 88f\n" "84:" // Height 3: Partial direct writeback: partial_4_0 "tbz x9, #2, 86f\n" "str s16, [x27], #0x4\n" - "str s20, [x22], #0x4\n" - "str s24, [x21], #0x4\n" + "str s20, [x23], #0x4\n" + "str s24, [x22], #0x4\n" "tbz x9, #1, 85f\n" "st1 { v16.h }[2], [x27], #0x2\n" - "st1 { v20.h }[2], [x22], #0x2\n" - "st1 { v24.h }[2], [x21], #0x2\n" + "st1 { v20.h }[2], [x23], #0x2\n" + "st1 { v24.h }[2], [x22], #0x2\n" "tbz x9, #0, 88f\n" "st1 { v16.b }[6], [x27]\n" - "st1 { v20.b }[6], [x22]\n" - "st1 { v24.b }[6], [x21]\n" + "st1 { v20.b }[6], [x23]\n" + "st1 { v24.b }[6], [x22]\n" "b 88f\n" "85:" // Height 3: Partial direct writeback: partial_1_4 "tbz x9, #0, 88f\n" "st1 { v16.b }[4], [x27]\n" - "st1 { v20.b }[4], [x22]\n" - "st1 { v24.b }[4], [x21]\n" + "st1 { v20.b }[4], [x23]\n" + "st1 { v24.b }[4], [x22]\n" "b 88f\n" "86:" // Height 3: Partial direct writeback: partial_2_0 "tbz x9, #1, 87f\n" "str h16, [x27], #0x2\n" - "str h20, [x22], #0x2\n" - "str h24, [x21], #0x2\n" + "str h20, [x23], #0x2\n" + "str h24, [x22], #0x2\n" "tbz x9, #0, 88f\n" "st1 { v16.b }[2], [x27]\n" - "st1 { v20.b }[2], [x22]\n" - "st1 { v24.b }[2], [x21]\n" + "st1 { v20.b }[2], [x23]\n" + "st1 { v24.b }[2], [x22]\n" "b 88f\n" "87:" // Height 3: Partial direct writeback: partial_1_0 "str b16, [x27, #0x0]\n" - "str b20, [x22, #0x0]\n" - "str b24, [x21, #0x0]\n" + "str b20, [x23, #0x0]\n" + "str b24, [x22, #0x0]\n" "88:" // Height 3: Partial direct writeback: Done "b 90f\n" "89:" // Height 3: Full writeback "str q16, [x27, #0x0]\n" "add x27, x27, #0x10\n" - "str q20, [x22, #0x0]\n" - "str q24, [x21, #0x0]\n" + "str q20, [x23, #0x0]\n" + "str q24, [x22, #0x0]\n" "90:" // Height 3: Writeback done "subs x9, x9, #0x10\n" "bgt 62b\n" @@ -1370,14 +1369,14 @@ void a64_hybrid_u8qa_dot_4x16 ( "94:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 95f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" - "ldr x21, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" + "ldr x21, [x20, #0x18]\n" "cbnz x26, 96f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -1387,9 +1386,9 @@ void a64_hybrid_u8qa_dot_4x16 ( "b 96f\n" "95:" // Height 4: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" + "add x21, x22, x21\n" "96:" // Height 4: input setup done "cmp x25, #0x10\n" "blt 101f\n" @@ -1614,29 +1613,29 @@ void a64_hybrid_u8qa_dot_4x16 ( ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n" ".inst 0x6e8f946e // udot v14.4s, v3.16b, v15.16b\n" "103:" // Height 4: Multiply loop: unique 15: skip row sum - "ldr q6, [x28, #0x0]\n" - "ldr q7, [x28, #0x10]\n" + "ldr q7, [x28, #0x0]\n" + "ldr q6, [x28, #0x10]\n" "sub x25, x25, #0x4\n" "cmp x25, #0x4\n" - "ldr q8, [x28, #0x20]\n" - "ldr q9, [x28, #0x30]\n" - ".inst 0x6f80e0d0 // udot v16.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0d4 // udot v20.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d8 // udot v24.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0dc // udot v28.4s, v6.16b, v3.4b[0]\n" + "ldr q5, [x28, #0x20]\n" + "ldr q4, [x28, #0x30]\n" + ".inst 0x6f80e0f0 // udot v16.4s, v7.16b, v0.4b[0]\n" + ".inst 0x6f81e0f4 // udot v20.4s, v7.16b, v1.4b[0]\n" + ".inst 0x6f82e0f8 // udot v24.4s, v7.16b, v2.4b[0]\n" + ".inst 0x6f83e0fc // udot v28.4s, v7.16b, v3.4b[0]\n" "add x28, x28, #0x40\n" - ".inst 0x6f80e0f1 // udot v17.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0f5 // udot v21.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f9 // udot v25.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0fd // udot v29.4s, v7.16b, v3.4b[0]\n" - ".inst 0x6f80e112 // udot v18.4s, v8.16b, v0.4b[0]\n" - ".inst 0x6f81e116 // udot v22.4s, v8.16b, v1.4b[0]\n" - ".inst 0x6f82e11a // udot v26.4s, v8.16b, v2.4b[0]\n" - ".inst 0x6f83e11e // udot v30.4s, v8.16b, v3.4b[0]\n" - ".inst 0x6f80e133 // udot v19.4s, v9.16b, v0.4b[0]\n" - ".inst 0x6f81e137 // udot v23.4s, v9.16b, v1.4b[0]\n" - ".inst 0x6f82e13b // udot v27.4s, v9.16b, v2.4b[0]\n" - ".inst 0x6f83e13f // udot v31.4s, v9.16b, v3.4b[0]\n" + ".inst 0x6f80e0d1 // udot v17.4s, v6.16b, v0.4b[0]\n" + ".inst 0x6f81e0d5 // udot v21.4s, v6.16b, v1.4b[0]\n" + ".inst 0x6f82e0d9 // udot v25.4s, v6.16b, v2.4b[0]\n" + ".inst 0x6f83e0dd // udot v29.4s, v6.16b, v3.4b[0]\n" + ".inst 0x6f80e0b2 // udot v18.4s, v5.16b, v0.4b[0]\n" + ".inst 0x6f81e0b6 // udot v22.4s, v5.16b, v1.4b[0]\n" + ".inst 0x6f82e0ba // udot v26.4s, v5.16b, v2.4b[0]\n" + ".inst 0x6f83e0be // udot v30.4s, v5.16b, v3.4b[0]\n" + ".inst 0x6f80e093 // udot v19.4s, v4.16b, v0.4b[0]\n" + ".inst 0x6f81e097 // udot v23.4s, v4.16b, v1.4b[0]\n" + ".inst 0x6f82e09b // udot v27.4s, v4.16b, v2.4b[0]\n" + ".inst 0x6f83e09f // udot v31.4s, v4.16b, v3.4b[0]\n" "bge 102b\n" "104:" // Height 4: Multiply loop: Skip odd blocks "cbz x25, 108f\n" @@ -1663,73 +1662,73 @@ void a64_hybrid_u8qa_dot_4x16 ( ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n" ".inst 0x6e8f946e // udot v14.4s, v3.16b, v15.16b\n" "107:" // Height 4: Multiply loop: unique 16: skip row sum - "ldr q10, [x28, #0x0]\n" - "ldr q4, [x28, #0x10]\n" - ".inst 0x6f80e150 // udot v16.4s, v10.16b, v0.4b[0]\n" - ".inst 0x6f81e154 // udot v20.4s, v10.16b, v1.4b[0]\n" + "ldr q7, [x28, #0x0]\n" + "ldr q6, [x28, #0x10]\n" + ".inst 0x6f80e0f0 // udot v16.4s, v7.16b, v0.4b[0]\n" + ".inst 0x6f81e0f4 // udot v20.4s, v7.16b, v1.4b[0]\n" "ldr q5, [x28, #0x20]\n" - "ldr q6, [x28, #0x30]\n" - ".inst 0x6f82e158 // udot v24.4s, v10.16b, v2.4b[0]\n" - ".inst 0x6f83e15c // udot v28.4s, v10.16b, v3.4b[0]\n" - ".inst 0x6f80e091 // udot v17.4s, v4.16b, v0.4b[0]\n" - ".inst 0x6f81e095 // udot v21.4s, v4.16b, v1.4b[0]\n" + "ldr q4, [x28, #0x30]\n" + ".inst 0x6f82e0f8 // udot v24.4s, v7.16b, v2.4b[0]\n" + ".inst 0x6f83e0fc // udot v28.4s, v7.16b, v3.4b[0]\n" + ".inst 0x6f80e0d1 // udot v17.4s, v6.16b, v0.4b[0]\n" + ".inst 0x6f81e0d5 // udot v21.4s, v6.16b, v1.4b[0]\n" "add x28, x28, #0x40\n" - ".inst 0x6f82e099 // udot v25.4s, v4.16b, v2.4b[0]\n" - ".inst 0x6f83e09d // udot v29.4s, v4.16b, v3.4b[0]\n" + ".inst 0x6f82e0d9 // udot v25.4s, v6.16b, v2.4b[0]\n" + ".inst 0x6f83e0dd // udot v29.4s, v6.16b, v3.4b[0]\n" ".inst 0x6f80e0b2 // udot v18.4s, v5.16b, v0.4b[0]\n" ".inst 0x6f81e0b6 // udot v22.4s, v5.16b, v1.4b[0]\n" ".inst 0x6f82e0ba // udot v26.4s, v5.16b, v2.4b[0]\n" ".inst 0x6f83e0be // udot v30.4s, v5.16b, v3.4b[0]\n" - ".inst 0x6f80e0d3 // udot v19.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0d7 // udot v23.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0db // udot v27.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0df // udot v31.4s, v6.16b, v3.4b[0]\n" + ".inst 0x6f80e093 // udot v19.4s, v4.16b, v0.4b[0]\n" + ".inst 0x6f81e097 // udot v23.4s, v4.16b, v1.4b[0]\n" + ".inst 0x6f82e09b // udot v27.4s, v4.16b, v2.4b[0]\n" + ".inst 0x6f83e09f // udot v31.4s, v4.16b, v3.4b[0]\n" "108:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x26, x26, #0x1\n" "cmp x26, x20\n" "bne 94b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x22, x27, x20\n" - "add x21, x22, x20\n" + "add x23, x27, x20\n" + "add x22, x23, x20\n" "prfm pstl1keep, [x27, #0x0]\n" - "add x20, x21, x20\n" + "add x21, x22, x20\n" + "prfm pstl1keep, [x23, #0x0]\n" "prfm pstl1keep, [x22, #0x0]\n" "prfm pstl1keep, [x21, #0x0]\n" - "prfm pstl1keep, [x20, #0x0]\n" "tbnz %x[flags], #31, 109f\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1r { v0.4s }, [x20]\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v12.4s, v12.4s, v12.4s\n" "addp v13.4s, v13.4s, v13.4s\n" "addp v14.4s, v14.4s, v14.4s\n" - "neg v4.4s, v4.4s\n" + "neg v0.4s, v0.4s\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v12.4s, v12.4s, v12.4s\n" "addp v13.4s, v13.4s, v13.4s\n" "addp v14.4s, v14.4s, v14.4s\n" - "mul v11.4s, v11.4s, v4.4s\n" - "mul v12.4s, v12.4s, v4.4s\n" - "mul v13.4s, v13.4s, v4.4s\n" - "mul v14.4s, v14.4s, v4.4s\n" + "mul v11.4s, v11.4s, v0.4s\n" + "mul v12.4s, v12.4s, v0.4s\n" + "mul v13.4s, v13.4s, v0.4s\n" + "mul v14.4s, v14.4s, v0.4s\n" "109:" // Height 4: skip row sum fixup "ldr q0, [x10, #0x0]\n" - "ldr q1, [x10, #0x10]\n" + "ldr q4, [x10, #0x10]\n" "add v16.4s, v16.4s, v11.4s\n" "add v17.4s, v17.4s, v11.4s\n" - "ldr q2, [x10, #0x20]\n" - "ldr q3, [x10, #0x30]\n" + "ldr q3, [x10, #0x20]\n" + "ldr q2, [x10, #0x30]\n" "add v18.4s, v18.4s, v11.4s\n" "add v19.4s, v19.4s, v11.4s\n" "add v20.4s, v20.4s, v12.4s\n" "add v21.4s, v21.4s, v12.4s\n" - "add x23, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v1.4s }, [x20]\n" "add v22.4s, v22.4s, v12.4s\n" "add v23.4s, v23.4s, v12.4s\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" "add v24.4s, v24.4s, v13.4s\n" "add v25.4s, v25.4s, v13.4s\n" "add x10, x10, #0x40\n" @@ -1740,100 +1739,100 @@ void a64_hybrid_u8qa_dot_4x16 ( "add v30.4s, v30.4s, v14.4s\n" "add v31.4s, v31.4s, v14.4s\n" "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" + "add v17.4s, v17.4s, v4.4s\n" + "add v18.4s, v18.4s, v3.4s\n" + "add v19.4s, v19.4s, v2.4s\n" "add v20.4s, v20.4s, v0.4s\n" - "add v21.4s, v21.4s, v1.4s\n" - "add v22.4s, v22.4s, v2.4s\n" - "add v23.4s, v23.4s, v3.4s\n" + "add v21.4s, v21.4s, v4.4s\n" + "add v22.4s, v22.4s, v3.4s\n" + "add v23.4s, v23.4s, v2.4s\n" "add v24.4s, v24.4s, v0.4s\n" - "add v25.4s, v25.4s, v1.4s\n" - "add v26.4s, v26.4s, v2.4s\n" - "add v27.4s, v27.4s, v3.4s\n" + "add v25.4s, v25.4s, v4.4s\n" + "add v26.4s, v26.4s, v3.4s\n" + "add v27.4s, v27.4s, v2.4s\n" "add v28.4s, v28.4s, v0.4s\n" - "ld1r { v0.4s }, [x23]\n" - "add v29.4s, v29.4s, v1.4s\n" - "add v30.4s, v30.4s, v2.4s\n" - "add v31.4s, v31.4s, v3.4s\n" - "sqrdmulh v16.4s, v16.4s, v4.4s\n" - "sqrdmulh v17.4s, v17.4s, v4.4s\n" - "sqrdmulh v18.4s, v18.4s, v4.4s\n" - "sqrdmulh v19.4s, v19.4s, v4.4s\n" - "sqrdmulh v20.4s, v20.4s, v4.4s\n" - "sqrdmulh v21.4s, v21.4s, v4.4s\n" - "sqrdmulh v22.4s, v22.4s, v4.4s\n" - "sqrdmulh v23.4s, v23.4s, v4.4s\n" - "sqrdmulh v24.4s, v24.4s, v4.4s\n" - "sqrdmulh v25.4s, v25.4s, v4.4s\n" - "sqrdmulh v26.4s, v26.4s, v4.4s\n" - "sqrdmulh v27.4s, v27.4s, v4.4s\n" - "sqrdmulh v28.4s, v28.4s, v4.4s\n" - "sqrdmulh v29.4s, v29.4s, v4.4s\n" - "sqrdmulh v30.4s, v30.4s, v4.4s\n" - "sqrdmulh v31.4s, v31.4s, v4.4s\n" + "ld1r { v0.4s }, [x20]\n" + "add v29.4s, v29.4s, v4.4s\n" + "add v30.4s, v30.4s, v3.4s\n" + "add v31.4s, v31.4s, v2.4s\n" + "sqrdmulh v16.4s, v16.4s, v1.4s\n" + "sqrdmulh v17.4s, v17.4s, v1.4s\n" + "sqrdmulh v18.4s, v18.4s, v1.4s\n" + "sqrdmulh v19.4s, v19.4s, v1.4s\n" + "sqrdmulh v20.4s, v20.4s, v1.4s\n" + "sqrdmulh v21.4s, v21.4s, v1.4s\n" + "sqrdmulh v22.4s, v22.4s, v1.4s\n" + "sqrdmulh v23.4s, v23.4s, v1.4s\n" + "sqrdmulh v24.4s, v24.4s, v1.4s\n" + "sqrdmulh v25.4s, v25.4s, v1.4s\n" + "sqrdmulh v26.4s, v26.4s, v1.4s\n" + "sqrdmulh v27.4s, v27.4s, v1.4s\n" + "sqrdmulh v28.4s, v28.4s, v1.4s\n" + "sqrdmulh v29.4s, v29.4s, v1.4s\n" + "sqrdmulh v30.4s, v30.4s, v1.4s\n" + "sqrdmulh v31.4s, v31.4s, v1.4s\n" "tbz %x[flags], #5, 110f\n" - "and v4.16b, v16.16b, v0.16b\n" - "and v5.16b, v17.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "and v6.16b, v18.16b, v0.16b\n" - "and v7.16b, v19.16b, v0.16b\n" - "and v8.16b, v20.16b, v0.16b\n" - "and v9.16b, v21.16b, v0.16b\n" - "and v10.16b, v22.16b, v0.16b\n" - "and v4.16b, v23.16b, v0.16b\n" - "and v5.16b, v24.16b, v0.16b\n" - "sshr v6.4s, v6.4s, #0x1f\n" + "and v2.16b, v16.16b, v0.16b\n" + "and v1.16b, v17.16b, v0.16b\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v2.4s\n" + "sqadd v17.4s, v17.4s, v1.4s\n" + "and v7.16b, v18.16b, v0.16b\n" + "and v6.16b, v19.16b, v0.16b\n" + "and v5.16b, v20.16b, v0.16b\n" + "and v4.16b, v21.16b, v0.16b\n" + "and v3.16b, v22.16b, v0.16b\n" + "and v2.16b, v23.16b, v0.16b\n" + "and v1.16b, v24.16b, v0.16b\n" "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v9.4s, v9.4s, #0x1f\n" - "sshr v10.4s, v10.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" - "sqadd v20.4s, v20.4s, v8.4s\n" - "sqadd v21.4s, v21.4s, v9.4s\n" - "sqadd v22.4s, v22.4s, v10.4s\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v24.4s, v24.4s, v5.4s\n" - "and v6.16b, v25.16b, v0.16b\n" - "and v7.16b, v26.16b, v0.16b\n" - "and v8.16b, v27.16b, v0.16b\n" - "and v9.16b, v28.16b, v0.16b\n" - "and v10.16b, v29.16b, v0.16b\n" - "and v4.16b, v30.16b, v0.16b\n" - "and v5.16b, v31.16b, v0.16b\n" "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v9.4s, v9.4s, #0x1f\n" - "sshr v10.4s, v10.4s, #0x1f\n" + "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sqadd v18.4s, v18.4s, v7.4s\n" + "sqadd v19.4s, v19.4s, v6.4s\n" + "sqadd v20.4s, v20.4s, v5.4s\n" + "sqadd v21.4s, v21.4s, v4.4s\n" + "sqadd v22.4s, v22.4s, v3.4s\n" + "sqadd v23.4s, v23.4s, v2.4s\n" + "sqadd v24.4s, v24.4s, v1.4s\n" + "and v7.16b, v25.16b, v0.16b\n" + "and v6.16b, v26.16b, v0.16b\n" + "and v5.16b, v27.16b, v0.16b\n" + "and v4.16b, v28.16b, v0.16b\n" + "and v3.16b, v29.16b, v0.16b\n" + "and v2.16b, v30.16b, v0.16b\n" + "and v1.16b, v31.16b, v0.16b\n" + "sshr v7.4s, v7.4s, #0x1f\n" + "sshr v6.4s, v6.4s, #0x1f\n" "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v25.4s, v25.4s, v6.4s\n" - "sqadd v26.4s, v26.4s, v7.4s\n" - "sqadd v27.4s, v27.4s, v8.4s\n" - "sqadd v28.4s, v28.4s, v9.4s\n" - "sqadd v29.4s, v29.4s, v10.4s\n" - "sqadd v30.4s, v30.4s, v4.4s\n" - "sqadd v31.4s, v31.4s, v5.4s\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sqadd v25.4s, v25.4s, v7.4s\n" + "sqadd v26.4s, v26.4s, v6.4s\n" + "sqadd v27.4s, v27.4s, v5.4s\n" + "sqadd v28.4s, v28.4s, v4.4s\n" + "sqadd v29.4s, v29.4s, v3.4s\n" + "sqadd v30.4s, v30.4s, v2.4s\n" + "sqadd v31.4s, v31.4s, v1.4s\n" "110:" // Height 4: no shift correction - "add x23, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v3.4s }, [x20]\n" "srshl v16.4s, v16.4s, v0.4s\n" "srshl v17.4s, v17.4s, v0.4s\n" "srshl v18.4s, v18.4s, v0.4s\n" "srshl v19.4s, v19.4s, v0.4s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x23]\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v2.4s }, [x20]\n" "srshl v20.4s, v20.4s, v0.4s\n" "srshl v21.4s, v21.4s, v0.4s\n" - "add x23, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x23]\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v1.4s }, [x20]\n" "srshl v22.4s, v22.4s, v0.4s\n" "srshl v23.4s, v23.4s, v0.4s\n" "cmp x9, #0x10\n" @@ -1845,163 +1844,163 @@ void a64_hybrid_u8qa_dot_4x16 ( "srshl v29.4s, v29.4s, v0.4s\n" "srshl v30.4s, v30.4s, v0.4s\n" "srshl v31.4s, v31.4s, v0.4s\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v23.4s, v23.4s, v4.4s\n" - "add v24.4s, v24.4s, v4.4s\n" - "add v25.4s, v25.4s, v4.4s\n" - "add v26.4s, v26.4s, v4.4s\n" - "add v27.4s, v27.4s, v4.4s\n" - "add v28.4s, v28.4s, v4.4s\n" - "add v29.4s, v29.4s, v4.4s\n" - "add v30.4s, v30.4s, v4.4s\n" - "add v31.4s, v31.4s, v4.4s\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "smin v24.4s, v24.4s, v6.4s\n" - "smin v25.4s, v25.4s, v6.4s\n" - "smin v26.4s, v26.4s, v6.4s\n" - "smin v27.4s, v27.4s, v6.4s\n" - "smin v28.4s, v28.4s, v6.4s\n" - "smin v29.4s, v29.4s, v6.4s\n" - "smin v30.4s, v30.4s, v6.4s\n" - "smin v31.4s, v31.4s, v6.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" - "smax v24.4s, v24.4s, v5.4s\n" - "smax v25.4s, v25.4s, v5.4s\n" - "smax v26.4s, v26.4s, v5.4s\n" - "smax v27.4s, v27.4s, v5.4s\n" - "smax v28.4s, v28.4s, v5.4s\n" - "smax v29.4s, v29.4s, v5.4s\n" - "smax v30.4s, v30.4s, v5.4s\n" - "smax v31.4s, v31.4s, v5.4s\n" + "add v16.4s, v16.4s, v3.4s\n" + "add v17.4s, v17.4s, v3.4s\n" + "add v18.4s, v18.4s, v3.4s\n" + "add v19.4s, v19.4s, v3.4s\n" + "add v20.4s, v20.4s, v3.4s\n" + "add v21.4s, v21.4s, v3.4s\n" + "add v22.4s, v22.4s, v3.4s\n" + "add v23.4s, v23.4s, v3.4s\n" + "add v24.4s, v24.4s, v3.4s\n" + "add v25.4s, v25.4s, v3.4s\n" + "add v26.4s, v26.4s, v3.4s\n" + "add v27.4s, v27.4s, v3.4s\n" + "add v28.4s, v28.4s, v3.4s\n" + "add v29.4s, v29.4s, v3.4s\n" + "add v30.4s, v30.4s, v3.4s\n" + "add v31.4s, v31.4s, v3.4s\n" + "smin v16.4s, v16.4s, v2.4s\n" + "smin v17.4s, v17.4s, v2.4s\n" + "smin v18.4s, v18.4s, v2.4s\n" + "smin v19.4s, v19.4s, v2.4s\n" + "smin v20.4s, v20.4s, v2.4s\n" + "smin v21.4s, v21.4s, v2.4s\n" + "smin v22.4s, v22.4s, v2.4s\n" + "smin v23.4s, v23.4s, v2.4s\n" + "smin v24.4s, v24.4s, v2.4s\n" + "smin v25.4s, v25.4s, v2.4s\n" + "smin v26.4s, v26.4s, v2.4s\n" + "smin v27.4s, v27.4s, v2.4s\n" + "smin v28.4s, v28.4s, v2.4s\n" + "smin v29.4s, v29.4s, v2.4s\n" + "smin v30.4s, v30.4s, v2.4s\n" + "smin v31.4s, v31.4s, v2.4s\n" + "smax v16.4s, v16.4s, v1.4s\n" + "smax v17.4s, v17.4s, v1.4s\n" + "smax v18.4s, v18.4s, v1.4s\n" + "smax v19.4s, v19.4s, v1.4s\n" + "smax v20.4s, v20.4s, v1.4s\n" + "smax v21.4s, v21.4s, v1.4s\n" + "smax v22.4s, v22.4s, v1.4s\n" + "smax v23.4s, v23.4s, v1.4s\n" + "smax v24.4s, v24.4s, v1.4s\n" + "smax v25.4s, v25.4s, v1.4s\n" + "smax v26.4s, v26.4s, v1.4s\n" + "smax v27.4s, v27.4s, v1.4s\n" + "smax v28.4s, v28.4s, v1.4s\n" + "smax v29.4s, v29.4s, v1.4s\n" + "smax v30.4s, v30.4s, v1.4s\n" + "smax v31.4s, v31.4s, v1.4s\n" "uzp1 v16.8h, v16.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v0.8h, v18.8h, v19.8h\n" "uzp1 v20.8h, v20.8h, v21.8h\n" - "uzp1 v21.8h, v22.8h, v23.8h\n" + "uzp1 v19.8h, v22.8h, v23.8h\n" "uzp1 v24.8h, v24.8h, v25.8h\n" - "uzp1 v25.8h, v26.8h, v27.8h\n" + "uzp1 v18.8h, v26.8h, v27.8h\n" "uzp1 v28.8h, v28.8h, v29.8h\n" - "uzp1 v29.8h, v30.8h, v31.8h\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" - "uzp1 v20.16b, v20.16b, v21.16b\n" - "uzp1 v24.16b, v24.16b, v25.16b\n" - "uzp1 v28.16b, v28.16b, v29.16b\n" + "uzp1 v17.8h, v30.8h, v31.8h\n" + "uzp1 v16.16b, v16.16b, v0.16b\n" + "uzp1 v20.16b, v20.16b, v19.16b\n" + "uzp1 v24.16b, v24.16b, v18.16b\n" + "uzp1 v28.16b, v28.16b, v17.16b\n" "bge 119f\n" "tbz x9, #3, 114f\n" "str d16, [x27], #0x8\n" - "str d20, [x22], #0x8\n" - "str d24, [x21], #0x8\n" - "str d28, [x20], #0x8\n" + "str d20, [x23], #0x8\n" + "str d24, [x22], #0x8\n" + "str d28, [x21], #0x8\n" "tbz x9, #2, 112f\n" "st1 { v16.s }[2], [x27], #0x4\n" - "st1 { v20.s }[2], [x22], #0x4\n" - "st1 { v24.s }[2], [x21], #0x4\n" - "st1 { v28.s }[2], [x20], #0x4\n" + "st1 { v20.s }[2], [x23], #0x4\n" + "st1 { v24.s }[2], [x22], #0x4\n" + "st1 { v28.s }[2], [x21], #0x4\n" "tbz x9, #1, 111f\n" "st1 { v16.h }[6], [x27], #0x2\n" - "st1 { v20.h }[6], [x22], #0x2\n" - "st1 { v24.h }[6], [x21], #0x2\n" - "st1 { v28.h }[6], [x20], #0x2\n" + "st1 { v20.h }[6], [x23], #0x2\n" + "st1 { v24.h }[6], [x22], #0x2\n" + "st1 { v28.h }[6], [x21], #0x2\n" "tbz x9, #0, 118f\n" "st1 { v16.b }[14], [x27]\n" - "st1 { v20.b }[14], [x22]\n" - "st1 { v24.b }[14], [x21]\n" - "st1 { v28.b }[14], [x20]\n" + "st1 { v20.b }[14], [x23]\n" + "st1 { v24.b }[14], [x22]\n" + "st1 { v28.b }[14], [x21]\n" "b 118f\n" "111:" // Height 4: Partial direct writeback: partial_1_12 "tbz x9, #0, 118f\n" "st1 { v16.b }[12], [x27]\n" - "st1 { v20.b }[12], [x22]\n" - "st1 { v24.b }[12], [x21]\n" - "st1 { v28.b }[12], [x20]\n" + "st1 { v20.b }[12], [x23]\n" + "st1 { v24.b }[12], [x22]\n" + "st1 { v28.b }[12], [x21]\n" "b 118f\n" "112:" // Height 4: Partial direct writeback: partial_2_8 "tbz x9, #1, 113f\n" "st1 { v16.h }[4], [x27], #0x2\n" - "st1 { v20.h }[4], [x22], #0x2\n" - "st1 { v24.h }[4], [x21], #0x2\n" - "st1 { v28.h }[4], [x20], #0x2\n" + "st1 { v20.h }[4], [x23], #0x2\n" + "st1 { v24.h }[4], [x22], #0x2\n" + "st1 { v28.h }[4], [x21], #0x2\n" "tbz x9, #0, 118f\n" "st1 { v16.b }[10], [x27]\n" - "st1 { v20.b }[10], [x22]\n" - "st1 { v24.b }[10], [x21]\n" - "st1 { v28.b }[10], [x20]\n" + "st1 { v20.b }[10], [x23]\n" + "st1 { v24.b }[10], [x22]\n" + "st1 { v28.b }[10], [x21]\n" "b 118f\n" "113:" // Height 4: Partial direct writeback: partial_1_8 "tbz x9, #0, 118f\n" "st1 { v16.b }[8], [x27]\n" - "st1 { v20.b }[8], [x22]\n" - "st1 { v24.b }[8], [x21]\n" - "st1 { v28.b }[8], [x20]\n" + "st1 { v20.b }[8], [x23]\n" + "st1 { v24.b }[8], [x22]\n" + "st1 { v28.b }[8], [x21]\n" "b 118f\n" "114:" // Height 4: Partial direct writeback: partial_4_0 "tbz x9, #2, 116f\n" "str s16, [x27], #0x4\n" - "str s20, [x22], #0x4\n" - "str s24, [x21], #0x4\n" - "str s28, [x20], #0x4\n" + "str s20, [x23], #0x4\n" + "str s24, [x22], #0x4\n" + "str s28, [x21], #0x4\n" "tbz x9, #1, 115f\n" "st1 { v16.h }[2], [x27], #0x2\n" - "st1 { v20.h }[2], [x22], #0x2\n" - "st1 { v24.h }[2], [x21], #0x2\n" - "st1 { v28.h }[2], [x20], #0x2\n" + "st1 { v20.h }[2], [x23], #0x2\n" + "st1 { v24.h }[2], [x22], #0x2\n" + "st1 { v28.h }[2], [x21], #0x2\n" "tbz x9, #0, 118f\n" "st1 { v16.b }[6], [x27]\n" - "st1 { v20.b }[6], [x22]\n" - "st1 { v24.b }[6], [x21]\n" - "st1 { v28.b }[6], [x20]\n" + "st1 { v20.b }[6], [x23]\n" + "st1 { v24.b }[6], [x22]\n" + "st1 { v28.b }[6], [x21]\n" "b 118f\n" "115:" // Height 4: Partial direct writeback: partial_1_4 "tbz x9, #0, 118f\n" "st1 { v16.b }[4], [x27]\n" - "st1 { v20.b }[4], [x22]\n" - "st1 { v24.b }[4], [x21]\n" - "st1 { v28.b }[4], [x20]\n" + "st1 { v20.b }[4], [x23]\n" + "st1 { v24.b }[4], [x22]\n" + "st1 { v28.b }[4], [x21]\n" "b 118f\n" "116:" // Height 4: Partial direct writeback: partial_2_0 "tbz x9, #1, 117f\n" "str h16, [x27], #0x2\n" - "str h20, [x22], #0x2\n" - "str h24, [x21], #0x2\n" - "str h28, [x20], #0x2\n" + "str h20, [x23], #0x2\n" + "str h24, [x22], #0x2\n" + "str h28, [x21], #0x2\n" "tbz x9, #0, 118f\n" "st1 { v16.b }[2], [x27]\n" - "st1 { v20.b }[2], [x22]\n" - "st1 { v24.b }[2], [x21]\n" - "st1 { v28.b }[2], [x20]\n" + "st1 { v20.b }[2], [x23]\n" + "st1 { v24.b }[2], [x22]\n" + "st1 { v28.b }[2], [x21]\n" "b 118f\n" "117:" // Height 4: Partial direct writeback: partial_1_0 "str b16, [x27, #0x0]\n" - "str b20, [x22, #0x0]\n" - "str b24, [x21, #0x0]\n" - "str b28, [x20, #0x0]\n" + "str b20, [x23, #0x0]\n" + "str b24, [x22, #0x0]\n" + "str b28, [x21, #0x0]\n" "118:" // Height 4: Partial direct writeback: Done "b 120f\n" "119:" // Height 4: Full writeback "str q16, [x27, #0x0]\n" "add x27, x27, #0x10\n" - "str q20, [x22, #0x0]\n" - "str q24, [x21, #0x0]\n" - "str q28, [x20, #0x0]\n" + "str q20, [x23, #0x0]\n" + "str q24, [x22, #0x0]\n" + "str q28, [x21, #0x0]\n" "120:" // Height 4: Writeback done "subs x9, x9, #0x10\n" "bgt 92b\n" @@ -2017,7 +2016,6 @@ void a64_hybrid_u8qa_dot_4x16 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "122:" // Exit - : [M] "+&r" (M), [flags] "+&r" (flags), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_mmla_4x16.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_mmla_4x16.hpp index 8a47701a4a050987765b35e03ff92092f67fd223..17e7405a0ab779faa3cc17ec6f5f59748b65f7fa 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_mmla_4x16.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_mmla_4x16.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef __aarch64__ + #include "../std_transforms_fixed.hpp" #include "../performance_parameters.hpp" @@ -81,7 +81,7 @@ public: case CPUModel::A510: return { 28.00 }; case CPUModel::V1: - return { 68.98 }; + return { 62.26 }; } } @@ -98,5 +98,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_mmla_4x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_mmla_4x16/generic.cpp index f808cb199d14eae757643f3277574712479af84f..1335b355ef278139fb98016f2ba9978c4236f608 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_mmla_4x16/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8qa_mmla_4x16/generic.cpp @@ -78,7 +78,6 @@ void a64_hybrid_u8qa_mmla_4x16 ( flags |= 0x20; } __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x4\n" "bge 97f\n" @@ -106,11 +105,11 @@ void a64_hybrid_u8qa_mmla_4x16 ( "4:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 5f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" "cbnz x26, 6f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -131,35 +130,35 @@ void a64_hybrid_u8qa_mmla_4x16 ( "ldr q4, [x28, #0x60]\n" "blt 9f\n" "7:" // Height 1: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v0.2d, v1.2d, v27.2d\n" ".inst 0x6e85a410 // ummla v16.4s, v0.16b, v5.16b\n" - "ldr q5, [x28, #0x70]\n" - "trn2 v1.2d, v1.2d, v2.2d\n" + "ldr q25, [x28, #0x70]\n" + "trn2 v1.2d, v1.2d, v27.2d\n" ".inst 0x6e86a414 // ummla v20.4s, v0.16b, v6.16b\n" - "ldr q6, [x28, #0x80]\n" + "ldr q24, [x28, #0x80]\n" ".inst 0x6e87a411 // ummla v17.4s, v0.16b, v7.16b\n" - "ldr q7, [x28, #0x90]\n" + "ldr q30, [x28, #0x90]\n" ".inst 0x6e88a415 // ummla v21.4s, v0.16b, v8.16b\n" - "ldr q8, [x28, #0xa0]\n" + "ldr q29, [x28, #0xa0]\n" ".inst 0x6e89a412 // ummla v18.4s, v0.16b, v9.16b\n" - "ldr q9, [x28, #0xb0]\n" + "ldr q28, [x28, #0xb0]\n" ".inst 0x6e8aa416 // ummla v22.4s, v0.16b, v10.16b\n" - "ldr q10, [x28, #0xc0]\n" + "ldr q27, [x28, #0xc0]\n" ".inst 0x6e84a413 // ummla v19.4s, v0.16b, v4.16b\n" - "ldr q4, [x28, #0xd0]\n" - ".inst 0x6e85a417 // ummla v23.4s, v0.16b, v5.16b\n" - "ldr q5, [x28, #0xe0]\n" - ".inst 0x6e86a430 // ummla v16.4s, v1.16b, v6.16b\n" - "ldr q6, [x28, #0xf0]\n" - ".inst 0x6e87a434 // ummla v20.4s, v1.16b, v7.16b\n" + "ldr q26, [x28, #0xd0]\n" + ".inst 0x6e99a417 // ummla v23.4s, v0.16b, v25.16b\n" + "ldr q25, [x28, #0xe0]\n" + ".inst 0x6e98a430 // ummla v16.4s, v1.16b, v24.16b\n" + "ldr q24, [x28, #0xf0]\n" + ".inst 0x6e9ea434 // ummla v20.4s, v1.16b, v30.16b\n" "add x24, x24, #0x10\n" - ".inst 0x6e88a431 // ummla v17.4s, v1.16b, v8.16b\n" + ".inst 0x6e9da431 // ummla v17.4s, v1.16b, v29.16b\n" "add x28, x28, #0x100\n" - ".inst 0x6e89a435 // ummla v21.4s, v1.16b, v9.16b\n" - ".inst 0x6e8aa432 // ummla v18.4s, v1.16b, v10.16b\n" - ".inst 0x6e84a436 // ummla v22.4s, v1.16b, v4.16b\n" - ".inst 0x6e85a433 // ummla v19.4s, v1.16b, v5.16b\n" - ".inst 0x6e86a437 // ummla v23.4s, v1.16b, v6.16b\n" + ".inst 0x6e9ca435 // ummla v21.4s, v1.16b, v28.16b\n" + ".inst 0x6e9ba432 // ummla v18.4s, v1.16b, v27.16b\n" + ".inst 0x6e9aa436 // ummla v22.4s, v1.16b, v26.16b\n" + ".inst 0x6e99a433 // ummla v19.4s, v1.16b, v25.16b\n" + ".inst 0x6e98a437 // ummla v23.4s, v1.16b, v24.16b\n" "tbnz %x[flags], #31, 8f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f942b // udot v11.4s, v1.16b, v15.16b\n" @@ -177,36 +176,36 @@ void a64_hybrid_u8qa_mmla_4x16 ( "prfm pldl1keep, [x24, #0x80]\n" "bge 7b\n" "9:" // Height 1: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v0.2d, v1.2d, v24.2d\n" ".inst 0x6e85a410 // ummla v16.4s, v0.16b, v5.16b\n" - "ldr q5, [x28, #0x70]\n" - "trn2 v1.2d, v1.2d, v2.2d\n" + "ldr q25, [x28, #0x70]\n" + "trn2 v1.2d, v1.2d, v24.2d\n" ".inst 0x6e86a414 // ummla v20.4s, v0.16b, v6.16b\n" - "ldr q6, [x28, #0x80]\n" + "ldr q24, [x28, #0x80]\n" ".inst 0x6e87a411 // ummla v17.4s, v0.16b, v7.16b\n" - "ldr q7, [x28, #0x90]\n" + "ldr q30, [x28, #0x90]\n" ".inst 0x6e88a415 // ummla v21.4s, v0.16b, v8.16b\n" - "ldr q8, [x28, #0xa0]\n" + "ldr q29, [x28, #0xa0]\n" ".inst 0x6e89a412 // ummla v18.4s, v0.16b, v9.16b\n" - "ldr q9, [x28, #0xb0]\n" + "ldr q28, [x28, #0xb0]\n" ".inst 0x6e8aa416 // ummla v22.4s, v0.16b, v10.16b\n" - "ldr q10, [x28, #0xc0]\n" + "ldr q27, [x28, #0xc0]\n" ".inst 0x6e84a413 // ummla v19.4s, v0.16b, v4.16b\n" - "ldr q4, [x28, #0xd0]\n" - ".inst 0x6e85a417 // ummla v23.4s, v0.16b, v5.16b\n" - "ldr q5, [x28, #0xe0]\n" - ".inst 0x6e86a430 // ummla v16.4s, v1.16b, v6.16b\n" - "ldr q6, [x28, #0xf0]\n" + "ldr q26, [x28, #0xd0]\n" + ".inst 0x6e99a417 // ummla v23.4s, v0.16b, v25.16b\n" + "ldr q25, [x28, #0xe0]\n" + ".inst 0x6e98a430 // ummla v16.4s, v1.16b, v24.16b\n" + "ldr q24, [x28, #0xf0]\n" "sub x25, x25, #0x10\n" - ".inst 0x6e87a434 // ummla v20.4s, v1.16b, v7.16b\n" - ".inst 0x6e88a431 // ummla v17.4s, v1.16b, v8.16b\n" + ".inst 0x6e9ea434 // ummla v20.4s, v1.16b, v30.16b\n" + ".inst 0x6e9da431 // ummla v17.4s, v1.16b, v29.16b\n" "add x24, x24, #0x10\n" "add x28, x28, #0x100\n" - ".inst 0x6e89a435 // ummla v21.4s, v1.16b, v9.16b\n" - ".inst 0x6e8aa432 // ummla v18.4s, v1.16b, v10.16b\n" - ".inst 0x6e84a436 // ummla v22.4s, v1.16b, v4.16b\n" - ".inst 0x6e85a433 // ummla v19.4s, v1.16b, v5.16b\n" - ".inst 0x6e86a437 // ummla v23.4s, v1.16b, v6.16b\n" + ".inst 0x6e9ca435 // ummla v21.4s, v1.16b, v28.16b\n" + ".inst 0x6e9ba432 // ummla v18.4s, v1.16b, v27.16b\n" + ".inst 0x6e9aa436 // ummla v22.4s, v1.16b, v26.16b\n" + ".inst 0x6e99a433 // ummla v19.4s, v1.16b, v25.16b\n" + ".inst 0x6e98a437 // ummla v23.4s, v1.16b, v24.16b\n" "tbnz %x[flags], #31, 10f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f942b // udot v11.4s, v1.16b, v15.16b\n" @@ -217,29 +216,29 @@ void a64_hybrid_u8qa_mmla_4x16 ( "cmp x25, #0x8\n" "blt 14f\n" "12:" // Height 1: Multiply loop: Odd block loop - "ldr d1, [x24], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" + "ldr d25, [x24], #0x8\n" + "trn1 v0.2d, v25.2d, v24.2d\n" "tbnz %x[flags], #31, 13f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" "13:" // Height 1: Multiply loop: unique 3: skip row sum - "ldr q8, [x28, #0x0]\n" - "ldr q9, [x28, #0x10]\n" - ".inst 0x6e88a410 // ummla v16.4s, v0.16b, v8.16b\n" + "ldr q24, [x28, #0x0]\n" + "ldr q26, [x28, #0x10]\n" + ".inst 0x6e98a410 // ummla v16.4s, v0.16b, v24.16b\n" "sub x25, x25, #0x8\n" - "ldr q10, [x28, #0x20]\n" - "ldr q4, [x28, #0x30]\n" + "ldr q25, [x28, #0x20]\n" + "ldr q24, [x28, #0x30]\n" "cmp x25, #0x8\n" - ".inst 0x6e89a414 // ummla v20.4s, v0.16b, v9.16b\n" - "ldr q5, [x28, #0x40]\n" - "ldr q6, [x28, #0x50]\n" - ".inst 0x6e8aa411 // ummla v17.4s, v0.16b, v10.16b\n" - ".inst 0x6e84a415 // ummla v21.4s, v0.16b, v4.16b\n" - "ldr q7, [x28, #0x60]\n" - "ldr q8, [x28, #0x70]\n" - ".inst 0x6e85a412 // ummla v18.4s, v0.16b, v5.16b\n" - ".inst 0x6e86a416 // ummla v22.4s, v0.16b, v6.16b\n" - ".inst 0x6e87a413 // ummla v19.4s, v0.16b, v7.16b\n" - ".inst 0x6e88a417 // ummla v23.4s, v0.16b, v8.16b\n" + ".inst 0x6e9aa414 // ummla v20.4s, v0.16b, v26.16b\n" + "ldr q27, [x28, #0x40]\n" + "ldr q26, [x28, #0x50]\n" + ".inst 0x6e99a411 // ummla v17.4s, v0.16b, v25.16b\n" + ".inst 0x6e98a415 // ummla v21.4s, v0.16b, v24.16b\n" + "ldr q25, [x28, #0x60]\n" + "ldr q24, [x28, #0x70]\n" + ".inst 0x6e9ba412 // ummla v18.4s, v0.16b, v27.16b\n" + ".inst 0x6e9aa416 // ummla v22.4s, v0.16b, v26.16b\n" + ".inst 0x6e99a413 // ummla v19.4s, v0.16b, v25.16b\n" + ".inst 0x6e98a417 // ummla v23.4s, v0.16b, v24.16b\n" "add x28, x28, #0x80\n" "bge 12b\n" "14:" // Height 1: Multiply loop: Skip odd blocks @@ -264,26 +263,26 @@ void a64_hybrid_u8qa_mmla_4x16 ( "17:" // Height 1: Multiply loop: Ragged operand read: partial_1_0 "ldr b1, [x24, #0x0]\n" "18:" // Height 1: Multiply loop: Ragged operand read: Done - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v0.2d, v1.2d, v24.2d\n" "tbnz %x[flags], #31, 19f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" "19:" // Height 1: Multiply loop: unique 4: skip row sum - "ldr q10, [x28, #0x0]\n" - "ldr q4, [x28, #0x10]\n" - ".inst 0x6e8aa410 // ummla v16.4s, v0.16b, v10.16b\n" - ".inst 0x6e84a414 // ummla v20.4s, v0.16b, v4.16b\n" - "ldr q5, [x28, #0x20]\n" - "ldr q6, [x28, #0x30]\n" - ".inst 0x6e85a411 // ummla v17.4s, v0.16b, v5.16b\n" - ".inst 0x6e86a415 // ummla v21.4s, v0.16b, v6.16b\n" - "ldr q7, [x28, #0x40]\n" - "ldr q8, [x28, #0x50]\n" - ".inst 0x6e87a412 // ummla v18.4s, v0.16b, v7.16b\n" - ".inst 0x6e88a416 // ummla v22.4s, v0.16b, v8.16b\n" - "ldr q9, [x28, #0x60]\n" - "ldr q10, [x28, #0x70]\n" - ".inst 0x6e89a413 // ummla v19.4s, v0.16b, v9.16b\n" - ".inst 0x6e8aa417 // ummla v23.4s, v0.16b, v10.16b\n" + "ldr q25, [x28, #0x0]\n" + "ldr q24, [x28, #0x10]\n" + ".inst 0x6e99a410 // ummla v16.4s, v0.16b, v25.16b\n" + ".inst 0x6e98a414 // ummla v20.4s, v0.16b, v24.16b\n" + "ldr q25, [x28, #0x20]\n" + "ldr q24, [x28, #0x30]\n" + ".inst 0x6e99a411 // ummla v17.4s, v0.16b, v25.16b\n" + ".inst 0x6e98a415 // ummla v21.4s, v0.16b, v24.16b\n" + "ldr q25, [x28, #0x40]\n" + "ldr q24, [x28, #0x50]\n" + ".inst 0x6e99a412 // ummla v18.4s, v0.16b, v25.16b\n" + ".inst 0x6e98a416 // ummla v22.4s, v0.16b, v24.16b\n" + "ldr q25, [x28, #0x60]\n" + "ldr q24, [x28, #0x70]\n" + ".inst 0x6e99a413 // ummla v19.4s, v0.16b, v25.16b\n" + ".inst 0x6e98a417 // ummla v23.4s, v0.16b, v24.16b\n" "add x28, x28, #0x80\n" "20:" // Height 1: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -297,75 +296,75 @@ void a64_hybrid_u8qa_mmla_4x16 ( "uzp1 v19.2d, v19.2d, v23.2d\n" "mov v23.16b, v16.16b\n" "tbnz %x[flags], #31, 21f\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1r { v1.4s }, [x23]\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1r { v16.4s }, [x20]\n" "addp v11.4s, v11.4s, v11.4s\n" - "neg v1.4s, v1.4s\n" + "neg v16.4s, v16.4s\n" "dup v11.4s, v11.s[0]\n" - "mul v11.4s, v11.4s, v1.4s\n" + "mul v11.4s, v11.4s, v16.4s\n" "21:" // Height 1: skip row sum fixup - "ldr q0, [x10, #0x0]\n" - "ldr q1, [x10, #0x10]\n" + "ldr q24, [x10, #0x0]\n" + "ldr q22, [x10, #0x10]\n" "add v23.4s, v23.4s, v11.4s\n" "add v17.4s, v17.4s, v11.4s\n" - "ldr q2, [x10, #0x20]\n" - "ldr q3, [x10, #0x30]\n" + "ldr q21, [x10, #0x20]\n" + "ldr q20, [x10, #0x30]\n" "add v18.4s, v18.4s, v11.4s\n" "add v19.4s, v19.4s, v11.4s\n" - "add x23, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v16.4s }, [x20]\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add v23.4s, v23.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" - "ld1r { v0.4s }, [x23]\n" - "add v19.4s, v19.4s, v3.4s\n" - "sqrdmulh v23.4s, v23.4s, v4.4s\n" + "add v23.4s, v23.4s, v24.4s\n" + "add v17.4s, v17.4s, v22.4s\n" + "add v18.4s, v18.4s, v21.4s\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1r { v0.4s }, [x20]\n" + "add v19.4s, v19.4s, v20.4s\n" + "sqrdmulh v23.4s, v23.4s, v16.4s\n" "add x10, x10, #0x40\n" - "sqrdmulh v17.4s, v17.4s, v4.4s\n" - "sqrdmulh v18.4s, v18.4s, v4.4s\n" - "sqrdmulh v19.4s, v19.4s, v4.4s\n" + "sqrdmulh v17.4s, v17.4s, v16.4s\n" + "sqrdmulh v18.4s, v18.4s, v16.4s\n" + "sqrdmulh v19.4s, v19.4s, v16.4s\n" "tbz %x[flags], #5, 22f\n" - "and v4.16b, v23.16b, v0.16b\n" - "and v5.16b, v17.16b, v0.16b\n" - "and v6.16b, v18.16b, v0.16b\n" - "and v7.16b, v19.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sqadd v19.4s, v19.4s, v7.4s\n" + "and v22.16b, v23.16b, v0.16b\n" + "and v21.16b, v17.16b, v0.16b\n" + "and v20.16b, v18.16b, v0.16b\n" + "and v16.16b, v19.16b, v0.16b\n" + "sshr v22.4s, v22.4s, #0x1f\n" + "sshr v21.4s, v21.4s, #0x1f\n" + "sshr v20.4s, v20.4s, #0x1f\n" + "sshr v16.4s, v16.4s, #0x1f\n" + "sqadd v23.4s, v23.4s, v22.4s\n" + "sqadd v17.4s, v17.4s, v21.4s\n" + "sqadd v18.4s, v18.4s, v20.4s\n" + "sqadd v19.4s, v19.4s, v16.4s\n" "22:" // Height 1: no shift correction - "add x23, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v21.4s }, [x20]\n" "srshl v23.4s, v23.4s, v0.4s\n" "srshl v17.4s, v17.4s, v0.4s\n" "srshl v18.4s, v18.4s, v0.4s\n" "srshl v19.4s, v19.4s, v0.4s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x23]\n" - "add v23.4s, v23.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add x23, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x23]\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v20.4s }, [x20]\n" + "add v23.4s, v23.4s, v21.4s\n" + "add v17.4s, v17.4s, v21.4s\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v16.4s }, [x20]\n" + "add v18.4s, v18.4s, v21.4s\n" + "add v19.4s, v19.4s, v21.4s\n" "cmp x9, #0x10\n" - "smin v23.4s, v23.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" + "smin v23.4s, v23.4s, v20.4s\n" + "smin v17.4s, v17.4s, v20.4s\n" + "smin v18.4s, v18.4s, v20.4s\n" + "smin v19.4s, v19.4s, v20.4s\n" + "smax v23.4s, v23.4s, v16.4s\n" + "smax v17.4s, v17.4s, v16.4s\n" + "smax v18.4s, v18.4s, v16.4s\n" + "smax v19.4s, v19.4s, v16.4s\n" "uzp1 v23.8h, v23.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" - "uzp1 v23.16b, v23.16b, v17.16b\n" + "uzp1 v16.8h, v18.8h, v19.8h\n" + "uzp1 v23.16b, v23.16b, v16.16b\n" "bge 31f\n" "tbz x9, #3, 26f\n" "str d23, [x27], #0x8\n" @@ -442,12 +441,12 @@ void a64_hybrid_u8qa_mmla_4x16 ( "36:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 37f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" "cbnz x26, 38f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -455,7 +454,7 @@ void a64_hybrid_u8qa_mmla_4x16 ( "b 38f\n" "37:" // Height 2: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20\n" + "add x23, x24, x21\n" "38:" // Height 2: input setup done "cmp x25, #0x10\n" "blt 43f\n" @@ -473,34 +472,34 @@ void a64_hybrid_u8qa_mmla_4x16 ( "39:" // Height 2: Multiply loop: Main loop head "trn1 v0.2d, v1.2d, v2.2d\n" ".inst 0x6e85a410 // ummla v16.4s, v0.16b, v5.16b\n" - "ldr q5, [x28, #0x70]\n" + "ldr q25, [x28, #0x70]\n" ".inst 0x6e86a414 // ummla v20.4s, v0.16b, v6.16b\n" - "ldr q6, [x28, #0x80]\n" + "ldr q24, [x28, #0x80]\n" "trn2 v1.2d, v1.2d, v2.2d\n" ".inst 0x6e87a411 // ummla v17.4s, v0.16b, v7.16b\n" - "ldr q7, [x28, #0x90]\n" + "ldr q30, [x28, #0x90]\n" ".inst 0x6e88a415 // ummla v21.4s, v0.16b, v8.16b\n" - "ldr q8, [x28, #0xa0]\n" + "ldr q29, [x28, #0xa0]\n" ".inst 0x6e89a412 // ummla v18.4s, v0.16b, v9.16b\n" - "ldr q9, [x28, #0xb0]\n" + "ldr q28, [x28, #0xb0]\n" ".inst 0x6e8aa416 // ummla v22.4s, v0.16b, v10.16b\n" - "ldr q10, [x28, #0xc0]\n" + "ldr q27, [x28, #0xc0]\n" ".inst 0x6e84a413 // ummla v19.4s, v0.16b, v4.16b\n" - "ldr q4, [x28, #0xd0]\n" - ".inst 0x6e85a417 // ummla v23.4s, v0.16b, v5.16b\n" - "ldr q5, [x28, #0xe0]\n" - ".inst 0x6e86a430 // ummla v16.4s, v1.16b, v6.16b\n" - "ldr q6, [x28, #0xf0]\n" - ".inst 0x6e87a434 // ummla v20.4s, v1.16b, v7.16b\n" + "ldr q26, [x28, #0xd0]\n" + ".inst 0x6e99a417 // ummla v23.4s, v0.16b, v25.16b\n" + "ldr q25, [x28, #0xe0]\n" + ".inst 0x6e98a430 // ummla v16.4s, v1.16b, v24.16b\n" + "ldr q24, [x28, #0xf0]\n" + ".inst 0x6e9ea434 // ummla v20.4s, v1.16b, v30.16b\n" "add x24, x24, #0x10\n" - ".inst 0x6e88a431 // ummla v17.4s, v1.16b, v8.16b\n" + ".inst 0x6e9da431 // ummla v17.4s, v1.16b, v29.16b\n" "add x23, x23, #0x10\n" - ".inst 0x6e89a435 // ummla v21.4s, v1.16b, v9.16b\n" + ".inst 0x6e9ca435 // ummla v21.4s, v1.16b, v28.16b\n" "add x28, x28, #0x100\n" - ".inst 0x6e8aa432 // ummla v18.4s, v1.16b, v10.16b\n" - ".inst 0x6e84a436 // ummla v22.4s, v1.16b, v4.16b\n" - ".inst 0x6e85a433 // ummla v19.4s, v1.16b, v5.16b\n" - ".inst 0x6e86a437 // ummla v23.4s, v1.16b, v6.16b\n" + ".inst 0x6e9ba432 // ummla v18.4s, v1.16b, v27.16b\n" + ".inst 0x6e9aa436 // ummla v22.4s, v1.16b, v26.16b\n" + ".inst 0x6e99a433 // ummla v19.4s, v1.16b, v25.16b\n" + ".inst 0x6e98a437 // ummla v23.4s, v1.16b, v24.16b\n" "tbnz %x[flags], #31, 40f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f942b // udot v11.4s, v1.16b, v15.16b\n" @@ -522,35 +521,35 @@ void a64_hybrid_u8qa_mmla_4x16 ( "41:" // Height 2: Multiply loop: Single iteration only "trn1 v0.2d, v1.2d, v2.2d\n" ".inst 0x6e85a410 // ummla v16.4s, v0.16b, v5.16b\n" - "ldr q5, [x28, #0x70]\n" + "ldr q25, [x28, #0x70]\n" ".inst 0x6e86a414 // ummla v20.4s, v0.16b, v6.16b\n" - "ldr q6, [x28, #0x80]\n" + "ldr q24, [x28, #0x80]\n" "trn2 v1.2d, v1.2d, v2.2d\n" ".inst 0x6e87a411 // ummla v17.4s, v0.16b, v7.16b\n" - "ldr q7, [x28, #0x90]\n" + "ldr q30, [x28, #0x90]\n" ".inst 0x6e88a415 // ummla v21.4s, v0.16b, v8.16b\n" - "ldr q8, [x28, #0xa0]\n" + "ldr q29, [x28, #0xa0]\n" ".inst 0x6e89a412 // ummla v18.4s, v0.16b, v9.16b\n" - "ldr q9, [x28, #0xb0]\n" + "ldr q28, [x28, #0xb0]\n" ".inst 0x6e8aa416 // ummla v22.4s, v0.16b, v10.16b\n" - "ldr q10, [x28, #0xc0]\n" + "ldr q27, [x28, #0xc0]\n" ".inst 0x6e84a413 // ummla v19.4s, v0.16b, v4.16b\n" - "ldr q4, [x28, #0xd0]\n" - ".inst 0x6e85a417 // ummla v23.4s, v0.16b, v5.16b\n" - "ldr q5, [x28, #0xe0]\n" - ".inst 0x6e86a430 // ummla v16.4s, v1.16b, v6.16b\n" - "ldr q6, [x28, #0xf0]\n" + "ldr q26, [x28, #0xd0]\n" + ".inst 0x6e99a417 // ummla v23.4s, v0.16b, v25.16b\n" + "ldr q25, [x28, #0xe0]\n" + ".inst 0x6e98a430 // ummla v16.4s, v1.16b, v24.16b\n" + "ldr q24, [x28, #0xf0]\n" "sub x25, x25, #0x10\n" - ".inst 0x6e87a434 // ummla v20.4s, v1.16b, v7.16b\n" - ".inst 0x6e88a431 // ummla v17.4s, v1.16b, v8.16b\n" + ".inst 0x6e9ea434 // ummla v20.4s, v1.16b, v30.16b\n" + ".inst 0x6e9da431 // ummla v17.4s, v1.16b, v29.16b\n" "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" - ".inst 0x6e89a435 // ummla v21.4s, v1.16b, v9.16b\n" - ".inst 0x6e8aa432 // ummla v18.4s, v1.16b, v10.16b\n" + ".inst 0x6e9ca435 // ummla v21.4s, v1.16b, v28.16b\n" + ".inst 0x6e9ba432 // ummla v18.4s, v1.16b, v27.16b\n" "add x28, x28, #0x100\n" - ".inst 0x6e84a436 // ummla v22.4s, v1.16b, v4.16b\n" - ".inst 0x6e85a433 // ummla v19.4s, v1.16b, v5.16b\n" - ".inst 0x6e86a437 // ummla v23.4s, v1.16b, v6.16b\n" + ".inst 0x6e9aa436 // ummla v22.4s, v1.16b, v26.16b\n" + ".inst 0x6e99a433 // ummla v19.4s, v1.16b, v25.16b\n" + ".inst 0x6e98a437 // ummla v23.4s, v1.16b, v24.16b\n" "tbnz %x[flags], #31, 42f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f942b // udot v11.4s, v1.16b, v15.16b\n" @@ -562,30 +561,30 @@ void a64_hybrid_u8qa_mmla_4x16 ( "cmp x25, #0x8\n" "blt 46f\n" "44:" // Height 2: Multiply loop: Odd block loop - "ldr d1, [x24], #0x8\n" - "ldr d2, [x23], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" + "ldr d25, [x24], #0x8\n" + "ldr d24, [x23], #0x8\n" + "trn1 v0.2d, v25.2d, v24.2d\n" "tbnz %x[flags], #31, 45f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" "45:" // Height 2: Multiply loop: unique 7: skip row sum - "ldr q8, [x28, #0x0]\n" - "ldr q9, [x28, #0x10]\n" - ".inst 0x6e88a410 // ummla v16.4s, v0.16b, v8.16b\n" + "ldr q24, [x28, #0x0]\n" + "ldr q26, [x28, #0x10]\n" + ".inst 0x6e98a410 // ummla v16.4s, v0.16b, v24.16b\n" "sub x25, x25, #0x8\n" - "ldr q10, [x28, #0x20]\n" - "ldr q4, [x28, #0x30]\n" + "ldr q25, [x28, #0x20]\n" + "ldr q24, [x28, #0x30]\n" "cmp x25, #0x8\n" - ".inst 0x6e89a414 // ummla v20.4s, v0.16b, v9.16b\n" - "ldr q5, [x28, #0x40]\n" - "ldr q6, [x28, #0x50]\n" - ".inst 0x6e8aa411 // ummla v17.4s, v0.16b, v10.16b\n" - ".inst 0x6e84a415 // ummla v21.4s, v0.16b, v4.16b\n" - "ldr q7, [x28, #0x60]\n" - "ldr q8, [x28, #0x70]\n" - ".inst 0x6e85a412 // ummla v18.4s, v0.16b, v5.16b\n" - ".inst 0x6e86a416 // ummla v22.4s, v0.16b, v6.16b\n" - ".inst 0x6e87a413 // ummla v19.4s, v0.16b, v7.16b\n" - ".inst 0x6e88a417 // ummla v23.4s, v0.16b, v8.16b\n" + ".inst 0x6e9aa414 // ummla v20.4s, v0.16b, v26.16b\n" + "ldr q27, [x28, #0x40]\n" + "ldr q26, [x28, #0x50]\n" + ".inst 0x6e99a411 // ummla v17.4s, v0.16b, v25.16b\n" + ".inst 0x6e98a415 // ummla v21.4s, v0.16b, v24.16b\n" + "ldr q25, [x28, #0x60]\n" + "ldr q24, [x28, #0x70]\n" + ".inst 0x6e9ba412 // ummla v18.4s, v0.16b, v27.16b\n" + ".inst 0x6e9aa416 // ummla v22.4s, v0.16b, v26.16b\n" + ".inst 0x6e99a413 // ummla v19.4s, v0.16b, v25.16b\n" + ".inst 0x6e98a417 // ummla v23.4s, v0.16b, v24.16b\n" "add x28, x28, #0x80\n" "bge 44b\n" "46:" // Height 2: Multiply loop: Skip odd blocks @@ -621,22 +620,22 @@ void a64_hybrid_u8qa_mmla_4x16 ( "tbnz %x[flags], #31, 51f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" "51:" // Height 2: Multiply loop: unique 8: skip row sum - "ldr q10, [x28, #0x0]\n" - "ldr q4, [x28, #0x10]\n" - ".inst 0x6e8aa410 // ummla v16.4s, v0.16b, v10.16b\n" - ".inst 0x6e84a414 // ummla v20.4s, v0.16b, v4.16b\n" - "ldr q5, [x28, #0x20]\n" - "ldr q6, [x28, #0x30]\n" - ".inst 0x6e85a411 // ummla v17.4s, v0.16b, v5.16b\n" - ".inst 0x6e86a415 // ummla v21.4s, v0.16b, v6.16b\n" - "ldr q7, [x28, #0x40]\n" - "ldr q8, [x28, #0x50]\n" - ".inst 0x6e87a412 // ummla v18.4s, v0.16b, v7.16b\n" - ".inst 0x6e88a416 // ummla v22.4s, v0.16b, v8.16b\n" - "ldr q9, [x28, #0x60]\n" - "ldr q10, [x28, #0x70]\n" - ".inst 0x6e89a413 // ummla v19.4s, v0.16b, v9.16b\n" - ".inst 0x6e8aa417 // ummla v23.4s, v0.16b, v10.16b\n" + "ldr q25, [x28, #0x0]\n" + "ldr q24, [x28, #0x10]\n" + ".inst 0x6e99a410 // ummla v16.4s, v0.16b, v25.16b\n" + ".inst 0x6e98a414 // ummla v20.4s, v0.16b, v24.16b\n" + "ldr q25, [x28, #0x20]\n" + "ldr q24, [x28, #0x30]\n" + ".inst 0x6e99a411 // ummla v17.4s, v0.16b, v25.16b\n" + ".inst 0x6e98a415 // ummla v21.4s, v0.16b, v24.16b\n" + "ldr q25, [x28, #0x40]\n" + "ldr q24, [x28, #0x50]\n" + ".inst 0x6e99a412 // ummla v18.4s, v0.16b, v25.16b\n" + ".inst 0x6e98a416 // ummla v22.4s, v0.16b, v24.16b\n" + "ldr q25, [x28, #0x60]\n" + "ldr q24, [x28, #0x70]\n" + ".inst 0x6e99a413 // ummla v19.4s, v0.16b, v25.16b\n" + ".inst 0x6e98a417 // ummla v23.4s, v0.16b, v24.16b\n" "add x28, x28, #0x80\n" "52:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -644,127 +643,127 @@ void a64_hybrid_u8qa_mmla_4x16 ( "cmp x26, x20\n" "bne 36b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "uzp1 v4.2d, v16.2d, v20.2d\n" - "add x22, x27, x20\n" + "uzp1 v24.2d, v16.2d, v20.2d\n" + "add x23, x27, x20\n" "uzp2 v16.2d, v16.2d, v20.2d\n" "uzp1 v20.2d, v17.2d, v21.2d\n" "uzp2 v17.2d, v17.2d, v21.2d\n" "prfm pstl1keep, [x27, #0x0]\n" - "prfm pstl1keep, [x22, #0x0]\n" + "prfm pstl1keep, [x23, #0x0]\n" "uzp1 v21.2d, v18.2d, v22.2d\n" "uzp2 v18.2d, v18.2d, v22.2d\n" "uzp1 v22.2d, v19.2d, v23.2d\n" "uzp2 v19.2d, v19.2d, v23.2d\n" - "mov v23.16b, v4.16b\n" + "mov v23.16b, v24.16b\n" "tbnz %x[flags], #31, 53f\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1r { v2.4s }, [x23]\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1r { v24.4s }, [x20]\n" "addp v11.4s, v11.4s, v11.4s\n" - "neg v2.4s, v2.4s\n" + "neg v24.4s, v24.4s\n" "dup v12.4s, v11.s[3]\n" "dup v11.4s, v11.s[0]\n" - "mul v11.4s, v11.4s, v2.4s\n" - "mul v12.4s, v12.4s, v2.4s\n" + "mul v11.4s, v11.4s, v24.4s\n" + "mul v12.4s, v12.4s, v24.4s\n" "53:" // Height 2: skip row sum fixup - "ldr q0, [x10, #0x0]\n" - "ldr q1, [x10, #0x10]\n" + "ldr q28, [x10, #0x0]\n" + "ldr q27, [x10, #0x10]\n" "add v23.4s, v23.4s, v11.4s\n" "add v20.4s, v20.4s, v11.4s\n" - "ldr q2, [x10, #0x20]\n" - "ldr q3, [x10, #0x30]\n" + "ldr q26, [x10, #0x20]\n" + "ldr q25, [x10, #0x30]\n" "add v21.4s, v21.4s, v11.4s\n" "add v22.4s, v22.4s, v11.4s\n" "add v16.4s, v16.4s, v12.4s\n" "add v17.4s, v17.4s, v12.4s\n" - "add x23, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v24.4s }, [x20]\n" "add v18.4s, v18.4s, v12.4s\n" "add v19.4s, v19.4s, v12.4s\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" - "add v23.4s, v23.4s, v0.4s\n" - "add v20.4s, v20.4s, v1.4s\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "add v23.4s, v23.4s, v28.4s\n" + "add v20.4s, v20.4s, v27.4s\n" "add x10, x10, #0x40\n" - "add v21.4s, v21.4s, v2.4s\n" - "add v22.4s, v22.4s, v3.4s\n" - "add v16.4s, v16.4s, v0.4s\n" - "ld1r { v0.4s }, [x23]\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" - "sqrdmulh v23.4s, v23.4s, v4.4s\n" - "sqrdmulh v20.4s, v20.4s, v4.4s\n" - "sqrdmulh v21.4s, v21.4s, v4.4s\n" - "sqrdmulh v22.4s, v22.4s, v4.4s\n" - "sqrdmulh v16.4s, v16.4s, v4.4s\n" - "sqrdmulh v17.4s, v17.4s, v4.4s\n" - "sqrdmulh v18.4s, v18.4s, v4.4s\n" - "sqrdmulh v19.4s, v19.4s, v4.4s\n" + "add v21.4s, v21.4s, v26.4s\n" + "add v22.4s, v22.4s, v25.4s\n" + "add v16.4s, v16.4s, v28.4s\n" + "ld1r { v0.4s }, [x20]\n" + "add v17.4s, v17.4s, v27.4s\n" + "add v18.4s, v18.4s, v26.4s\n" + "add v19.4s, v19.4s, v25.4s\n" + "sqrdmulh v23.4s, v23.4s, v24.4s\n" + "sqrdmulh v20.4s, v20.4s, v24.4s\n" + "sqrdmulh v21.4s, v21.4s, v24.4s\n" + "sqrdmulh v22.4s, v22.4s, v24.4s\n" + "sqrdmulh v16.4s, v16.4s, v24.4s\n" + "sqrdmulh v17.4s, v17.4s, v24.4s\n" + "sqrdmulh v18.4s, v18.4s, v24.4s\n" + "sqrdmulh v19.4s, v19.4s, v24.4s\n" "tbz %x[flags], #5, 54f\n" - "and v4.16b, v23.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v23.4s, v23.4s, v4.4s\n" - "and v5.16b, v20.16b, v0.16b\n" - "and v6.16b, v21.16b, v0.16b\n" - "and v7.16b, v22.16b, v0.16b\n" - "and v8.16b, v16.16b, v0.16b\n" - "and v9.16b, v17.16b, v0.16b\n" - "and v10.16b, v18.16b, v0.16b\n" - "and v4.16b, v19.16b, v0.16b\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v9.4s, v9.4s, #0x1f\n" - "sshr v10.4s, v10.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v20.4s, v20.4s, v5.4s\n" - "sqadd v21.4s, v21.4s, v6.4s\n" - "sqadd v22.4s, v22.4s, v7.4s\n" - "sqadd v16.4s, v16.4s, v8.4s\n" - "sqadd v17.4s, v17.4s, v9.4s\n" - "sqadd v18.4s, v18.4s, v10.4s\n" - "sqadd v19.4s, v19.4s, v4.4s\n" + "and v24.16b, v23.16b, v0.16b\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sqadd v23.4s, v23.4s, v24.4s\n" + "and v30.16b, v20.16b, v0.16b\n" + "and v29.16b, v21.16b, v0.16b\n" + "and v28.16b, v22.16b, v0.16b\n" + "and v27.16b, v16.16b, v0.16b\n" + "and v26.16b, v17.16b, v0.16b\n" + "and v25.16b, v18.16b, v0.16b\n" + "and v24.16b, v19.16b, v0.16b\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sshr v27.4s, v27.4s, #0x1f\n" + "sshr v26.4s, v26.4s, #0x1f\n" + "sshr v25.4s, v25.4s, #0x1f\n" + "sshr v24.4s, v24.4s, #0x1f\n" + "sqadd v20.4s, v20.4s, v30.4s\n" + "sqadd v21.4s, v21.4s, v29.4s\n" + "sqadd v22.4s, v22.4s, v28.4s\n" + "sqadd v16.4s, v16.4s, v27.4s\n" + "sqadd v17.4s, v17.4s, v26.4s\n" + "sqadd v18.4s, v18.4s, v25.4s\n" + "sqadd v19.4s, v19.4s, v24.4s\n" "54:" // Height 2: no shift correction - "add x23, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v26.4s }, [x20]\n" "srshl v23.4s, v23.4s, v0.4s\n" "srshl v20.4s, v20.4s, v0.4s\n" "srshl v21.4s, v21.4s, v0.4s\n" "srshl v22.4s, v22.4s, v0.4s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x23]\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v25.4s }, [x20]\n" "srshl v16.4s, v16.4s, v0.4s\n" "srshl v17.4s, v17.4s, v0.4s\n" - "add x23, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x23]\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v24.4s }, [x20]\n" "srshl v18.4s, v18.4s, v0.4s\n" "srshl v19.4s, v19.4s, v0.4s\n" "cmp x9, #0x10\n" - "add v23.4s, v23.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" + "add v23.4s, v23.4s, v26.4s\n" + "add v20.4s, v20.4s, v26.4s\n" + "add v21.4s, v21.4s, v26.4s\n" + "add v22.4s, v22.4s, v26.4s\n" + "add v16.4s, v16.4s, v26.4s\n" + "add v17.4s, v17.4s, v26.4s\n" + "add v18.4s, v18.4s, v26.4s\n" + "add v19.4s, v19.4s, v26.4s\n" + "smin v23.4s, v23.4s, v25.4s\n" + "smin v20.4s, v20.4s, v25.4s\n" + "smin v21.4s, v21.4s, v25.4s\n" + "smin v22.4s, v22.4s, v25.4s\n" + "smin v16.4s, v16.4s, v25.4s\n" + "smin v17.4s, v17.4s, v25.4s\n" + "smin v18.4s, v18.4s, v25.4s\n" + "smin v19.4s, v19.4s, v25.4s\n" + "smax v23.4s, v23.4s, v24.4s\n" + "smax v20.4s, v20.4s, v24.4s\n" + "smax v21.4s, v21.4s, v24.4s\n" + "smax v22.4s, v22.4s, v24.4s\n" + "smax v16.4s, v16.4s, v24.4s\n" + "smax v17.4s, v17.4s, v24.4s\n" + "smax v18.4s, v18.4s, v24.4s\n" + "smax v19.4s, v19.4s, v24.4s\n" "uzp1 v23.8h, v23.8h, v20.8h\n" "uzp1 v20.8h, v21.8h, v22.8h\n" "uzp1 v16.8h, v16.8h, v17.8h\n" @@ -774,68 +773,68 @@ void a64_hybrid_u8qa_mmla_4x16 ( "bge 63f\n" "tbz x9, #3, 58f\n" "str d23, [x27], #0x8\n" - "str d16, [x22], #0x8\n" + "str d16, [x23], #0x8\n" "tbz x9, #2, 56f\n" "st1 { v23.s }[2], [x27], #0x4\n" - "st1 { v16.s }[2], [x22], #0x4\n" + "st1 { v16.s }[2], [x23], #0x4\n" "tbz x9, #1, 55f\n" "st1 { v23.h }[6], [x27], #0x2\n" - "st1 { v16.h }[6], [x22], #0x2\n" + "st1 { v16.h }[6], [x23], #0x2\n" "tbz x9, #0, 62f\n" "st1 { v23.b }[14], [x27]\n" - "st1 { v16.b }[14], [x22]\n" + "st1 { v16.b }[14], [x23]\n" "b 62f\n" "55:" // Height 2: Partial direct writeback: partial_1_12 "tbz x9, #0, 62f\n" "st1 { v23.b }[12], [x27]\n" - "st1 { v16.b }[12], [x22]\n" + "st1 { v16.b }[12], [x23]\n" "b 62f\n" "56:" // Height 2: Partial direct writeback: partial_2_8 "tbz x9, #1, 57f\n" "st1 { v23.h }[4], [x27], #0x2\n" - "st1 { v16.h }[4], [x22], #0x2\n" + "st1 { v16.h }[4], [x23], #0x2\n" "tbz x9, #0, 62f\n" "st1 { v23.b }[10], [x27]\n" - "st1 { v16.b }[10], [x22]\n" + "st1 { v16.b }[10], [x23]\n" "b 62f\n" "57:" // Height 2: Partial direct writeback: partial_1_8 "tbz x9, #0, 62f\n" "st1 { v23.b }[8], [x27]\n" - "st1 { v16.b }[8], [x22]\n" + "st1 { v16.b }[8], [x23]\n" "b 62f\n" "58:" // Height 2: Partial direct writeback: partial_4_0 "tbz x9, #2, 60f\n" "str s23, [x27], #0x4\n" - "str s16, [x22], #0x4\n" + "str s16, [x23], #0x4\n" "tbz x9, #1, 59f\n" "st1 { v23.h }[2], [x27], #0x2\n" - "st1 { v16.h }[2], [x22], #0x2\n" + "st1 { v16.h }[2], [x23], #0x2\n" "tbz x9, #0, 62f\n" "st1 { v23.b }[6], [x27]\n" - "st1 { v16.b }[6], [x22]\n" + "st1 { v16.b }[6], [x23]\n" "b 62f\n" "59:" // Height 2: Partial direct writeback: partial_1_4 "tbz x9, #0, 62f\n" "st1 { v23.b }[4], [x27]\n" - "st1 { v16.b }[4], [x22]\n" + "st1 { v16.b }[4], [x23]\n" "b 62f\n" "60:" // Height 2: Partial direct writeback: partial_2_0 "tbz x9, #1, 61f\n" "str h23, [x27], #0x2\n" - "str h16, [x22], #0x2\n" + "str h16, [x23], #0x2\n" "tbz x9, #0, 62f\n" "st1 { v23.b }[2], [x27]\n" - "st1 { v16.b }[2], [x22]\n" + "st1 { v16.b }[2], [x23]\n" "b 62f\n" "61:" // Height 2: Partial direct writeback: partial_1_0 "str b23, [x27, #0x0]\n" - "str b16, [x22, #0x0]\n" + "str b16, [x23, #0x0]\n" "62:" // Height 2: Partial direct writeback: Done "b 64f\n" "63:" // Height 2: Full writeback "str q23, [x27, #0x0]\n" "add x27, x27, #0x10\n" - "str q16, [x22, #0x0]\n" + "str q16, [x23, #0x0]\n" "64:" // Height 2: Writeback done "subs x9, x9, #0x10\n" "bgt 34b\n" @@ -872,13 +871,13 @@ void a64_hybrid_u8qa_mmla_4x16 ( "68:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 69f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" "cbnz x26, 70f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -887,8 +886,8 @@ void a64_hybrid_u8qa_mmla_4x16 ( "b 70f\n" "69:" // Height 3: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" "70:" // Height 3: input setup done "cmp x25, #0x10\n" "blt 75f\n" @@ -909,12 +908,12 @@ void a64_hybrid_u8qa_mmla_4x16 ( ".inst 0x6e85a410 // ummla v16.4s, v0.16b, v5.16b\n" "trn1 v2.2d, v3.2d, v4.2d\n" ".inst 0x6e85a458 // ummla v24.4s, v2.16b, v5.16b\n" - "ldr q5, [x28, #0x70]\n" + "ldr q14, [x28, #0x70]\n" ".inst 0x6e86a414 // ummla v20.4s, v0.16b, v6.16b\n" "trn2 v3.2d, v3.2d, v4.2d\n" - "ldr q4, [x28, #0x60]\n" + "ldr q5, [x28, #0x60]\n" ".inst 0x6e86a45c // ummla v28.4s, v2.16b, v6.16b\n" - "ldr q6, [x28, #0x80]\n" + "ldr q4, [x28, #0x80]\n" ".inst 0x6e87a411 // ummla v17.4s, v0.16b, v7.16b\n" ".inst 0x6e87a459 // ummla v25.4s, v2.16b, v7.16b\n" "ldr q7, [x28, #0x90]\n" @@ -930,15 +929,15 @@ void a64_hybrid_u8qa_mmla_4x16 ( ".inst 0x6e8aa416 // ummla v22.4s, v0.16b, v10.16b\n" ".inst 0x6e8aa45e // ummla v30.4s, v2.16b, v10.16b\n" "ldr q10, [x28, #0xc0]\n" - ".inst 0x6e84a413 // ummla v19.4s, v0.16b, v4.16b\n" - ".inst 0x6e84a45b // ummla v27.4s, v2.16b, v4.16b\n" - "ldr q4, [x28, #0xd0]\n" - ".inst 0x6e85a417 // ummla v23.4s, v0.16b, v5.16b\n" - ".inst 0x6e85a45f // ummla v31.4s, v2.16b, v5.16b\n" + ".inst 0x6e85a413 // ummla v19.4s, v0.16b, v5.16b\n" + ".inst 0x6e85a45b // ummla v27.4s, v2.16b, v5.16b\n" + "ldr q6, [x28, #0xd0]\n" + ".inst 0x6e8ea417 // ummla v23.4s, v0.16b, v14.16b\n" + ".inst 0x6e8ea45f // ummla v31.4s, v2.16b, v14.16b\n" "ldr q5, [x28, #0xe0]\n" - ".inst 0x6e86a430 // ummla v16.4s, v1.16b, v6.16b\n" - ".inst 0x6e86a478 // ummla v24.4s, v3.16b, v6.16b\n" - "ldr q6, [x28, #0xf0]\n" + ".inst 0x6e84a430 // ummla v16.4s, v1.16b, v4.16b\n" + ".inst 0x6e84a478 // ummla v24.4s, v3.16b, v4.16b\n" + "ldr q4, [x28, #0xf0]\n" "add x28, x28, #0x100\n" ".inst 0x6e87a434 // ummla v20.4s, v1.16b, v7.16b\n" ".inst 0x6e87a47c // ummla v28.4s, v3.16b, v7.16b\n" @@ -948,12 +947,12 @@ void a64_hybrid_u8qa_mmla_4x16 ( ".inst 0x6e89a47d // ummla v29.4s, v3.16b, v9.16b\n" ".inst 0x6e8aa432 // ummla v18.4s, v1.16b, v10.16b\n" ".inst 0x6e8aa47a // ummla v26.4s, v3.16b, v10.16b\n" - ".inst 0x6e84a436 // ummla v22.4s, v1.16b, v4.16b\n" - ".inst 0x6e84a47e // ummla v30.4s, v3.16b, v4.16b\n" + ".inst 0x6e86a436 // ummla v22.4s, v1.16b, v6.16b\n" + ".inst 0x6e86a47e // ummla v30.4s, v3.16b, v6.16b\n" ".inst 0x6e85a433 // ummla v19.4s, v1.16b, v5.16b\n" ".inst 0x6e85a47b // ummla v27.4s, v3.16b, v5.16b\n" - ".inst 0x6e86a437 // ummla v23.4s, v1.16b, v6.16b\n" - ".inst 0x6e86a47f // ummla v31.4s, v3.16b, v6.16b\n" + ".inst 0x6e84a437 // ummla v23.4s, v1.16b, v4.16b\n" + ".inst 0x6e84a47f // ummla v31.4s, v3.16b, v4.16b\n" "tbnz %x[flags], #31, 72f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n" @@ -981,12 +980,12 @@ void a64_hybrid_u8qa_mmla_4x16 ( ".inst 0x6e85a410 // ummla v16.4s, v0.16b, v5.16b\n" "trn1 v2.2d, v3.2d, v4.2d\n" ".inst 0x6e85a458 // ummla v24.4s, v2.16b, v5.16b\n" - "ldr q5, [x28, #0x70]\n" + "ldr q14, [x28, #0x70]\n" ".inst 0x6e86a414 // ummla v20.4s, v0.16b, v6.16b\n" "trn2 v3.2d, v3.2d, v4.2d\n" - "ldr q4, [x28, #0x60]\n" + "ldr q5, [x28, #0x60]\n" ".inst 0x6e86a45c // ummla v28.4s, v2.16b, v6.16b\n" - "ldr q6, [x28, #0x80]\n" + "ldr q4, [x28, #0x80]\n" ".inst 0x6e87a411 // ummla v17.4s, v0.16b, v7.16b\n" ".inst 0x6e87a459 // ummla v25.4s, v2.16b, v7.16b\n" "ldr q7, [x28, #0x90]\n" @@ -1003,15 +1002,15 @@ void a64_hybrid_u8qa_mmla_4x16 ( ".inst 0x6e8aa45e // ummla v30.4s, v2.16b, v10.16b\n" "ldr q10, [x28, #0xc0]\n" "add x22, x22, #0x10\n" - ".inst 0x6e84a413 // ummla v19.4s, v0.16b, v4.16b\n" - ".inst 0x6e84a45b // ummla v27.4s, v2.16b, v4.16b\n" - "ldr q4, [x28, #0xd0]\n" - ".inst 0x6e85a417 // ummla v23.4s, v0.16b, v5.16b\n" - ".inst 0x6e85a45f // ummla v31.4s, v2.16b, v5.16b\n" + ".inst 0x6e85a413 // ummla v19.4s, v0.16b, v5.16b\n" + ".inst 0x6e85a45b // ummla v27.4s, v2.16b, v5.16b\n" + "ldr q6, [x28, #0xd0]\n" + ".inst 0x6e8ea417 // ummla v23.4s, v0.16b, v14.16b\n" + ".inst 0x6e8ea45f // ummla v31.4s, v2.16b, v14.16b\n" "ldr q5, [x28, #0xe0]\n" - ".inst 0x6e86a430 // ummla v16.4s, v1.16b, v6.16b\n" - ".inst 0x6e86a478 // ummla v24.4s, v3.16b, v6.16b\n" - "ldr q6, [x28, #0xf0]\n" + ".inst 0x6e84a430 // ummla v16.4s, v1.16b, v4.16b\n" + ".inst 0x6e84a478 // ummla v24.4s, v3.16b, v4.16b\n" + "ldr q4, [x28, #0xf0]\n" "add x28, x28, #0x100\n" ".inst 0x6e87a434 // ummla v20.4s, v1.16b, v7.16b\n" ".inst 0x6e87a47c // ummla v28.4s, v3.16b, v7.16b\n" @@ -1021,12 +1020,12 @@ void a64_hybrid_u8qa_mmla_4x16 ( ".inst 0x6e89a47d // ummla v29.4s, v3.16b, v9.16b\n" ".inst 0x6e8aa432 // ummla v18.4s, v1.16b, v10.16b\n" ".inst 0x6e8aa47a // ummla v26.4s, v3.16b, v10.16b\n" - ".inst 0x6e84a436 // ummla v22.4s, v1.16b, v4.16b\n" - ".inst 0x6e84a47e // ummla v30.4s, v3.16b, v4.16b\n" + ".inst 0x6e86a436 // ummla v22.4s, v1.16b, v6.16b\n" + ".inst 0x6e86a47e // ummla v30.4s, v3.16b, v6.16b\n" ".inst 0x6e85a433 // ummla v19.4s, v1.16b, v5.16b\n" ".inst 0x6e85a47b // ummla v27.4s, v3.16b, v5.16b\n" - ".inst 0x6e86a437 // ummla v23.4s, v1.16b, v6.16b\n" - ".inst 0x6e86a47f // ummla v31.4s, v3.16b, v6.16b\n" + ".inst 0x6e84a437 // ummla v23.4s, v1.16b, v4.16b\n" + ".inst 0x6e84a47f // ummla v31.4s, v3.16b, v4.16b\n" "tbnz %x[flags], #31, 74f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n" @@ -1042,41 +1041,41 @@ void a64_hybrid_u8qa_mmla_4x16 ( "blt 78f\n" "76:" // Height 3: Multiply loop: Odd block loop "ldr d1, [x24], #0x8\n" - "ldr d2, [x23], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "ldr d3, [x22], #0x8\n" - "trn1 v2.2d, v3.2d, v7.2d\n" + "ldr d0, [x23], #0x8\n" + "trn1 v0.2d, v1.2d, v0.2d\n" + "ldr d1, [x22], #0x8\n" + "trn1 v2.2d, v1.2d, v2.2d\n" "tbnz %x[flags], #31, 77f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n" "77:" // Height 3: Multiply loop: unique 11: skip row sum - "ldr q8, [x28, #0x0]\n" - "ldr q9, [x28, #0x10]\n" - ".inst 0x6e88a410 // ummla v16.4s, v0.16b, v8.16b\n" - ".inst 0x6e88a458 // ummla v24.4s, v2.16b, v8.16b\n" - "ldr q10, [x28, #0x20]\n" - "ldr q4, [x28, #0x30]\n" + "ldr q3, [x28, #0x0]\n" + "ldr q1, [x28, #0x10]\n" + ".inst 0x6e83a410 // ummla v16.4s, v0.16b, v3.16b\n" + ".inst 0x6e83a458 // ummla v24.4s, v2.16b, v3.16b\n" + "ldr q7, [x28, #0x20]\n" + "ldr q6, [x28, #0x30]\n" "sub x25, x25, #0x8\n" "cmp x25, #0x8\n" "ldr q5, [x28, #0x40]\n" - "ldr q6, [x28, #0x50]\n" - ".inst 0x6e89a414 // ummla v20.4s, v0.16b, v9.16b\n" - ".inst 0x6e89a45c // ummla v28.4s, v2.16b, v9.16b\n" - "ldr q7, [x28, #0x60]\n" - "ldr q8, [x28, #0x70]\n" - ".inst 0x6e8aa411 // ummla v17.4s, v0.16b, v10.16b\n" - ".inst 0x6e8aa459 // ummla v25.4s, v2.16b, v10.16b\n" - ".inst 0x6e84a415 // ummla v21.4s, v0.16b, v4.16b\n" - ".inst 0x6e84a45d // ummla v29.4s, v2.16b, v4.16b\n" + "ldr q4, [x28, #0x50]\n" + ".inst 0x6e81a414 // ummla v20.4s, v0.16b, v1.16b\n" + ".inst 0x6e81a45c // ummla v28.4s, v2.16b, v1.16b\n" + "ldr q3, [x28, #0x60]\n" + "ldr q1, [x28, #0x70]\n" + ".inst 0x6e87a411 // ummla v17.4s, v0.16b, v7.16b\n" + ".inst 0x6e87a459 // ummla v25.4s, v2.16b, v7.16b\n" + ".inst 0x6e86a415 // ummla v21.4s, v0.16b, v6.16b\n" + ".inst 0x6e86a45d // ummla v29.4s, v2.16b, v6.16b\n" "add x28, x28, #0x80\n" ".inst 0x6e85a412 // ummla v18.4s, v0.16b, v5.16b\n" ".inst 0x6e85a45a // ummla v26.4s, v2.16b, v5.16b\n" - ".inst 0x6e86a416 // ummla v22.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a45e // ummla v30.4s, v2.16b, v6.16b\n" - ".inst 0x6e87a413 // ummla v19.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a45b // ummla v27.4s, v2.16b, v7.16b\n" - ".inst 0x6e88a417 // ummla v23.4s, v0.16b, v8.16b\n" - ".inst 0x6e88a45f // ummla v31.4s, v2.16b, v8.16b\n" + ".inst 0x6e84a416 // ummla v22.4s, v0.16b, v4.16b\n" + ".inst 0x6e84a45e // ummla v30.4s, v2.16b, v4.16b\n" + ".inst 0x6e83a413 // ummla v19.4s, v0.16b, v3.16b\n" + ".inst 0x6e83a45b // ummla v27.4s, v2.16b, v3.16b\n" + ".inst 0x6e81a417 // ummla v23.4s, v0.16b, v1.16b\n" + ".inst 0x6e81a45f // ummla v31.4s, v2.16b, v1.16b\n" "bge 76b\n" "78:" // Height 3: Multiply loop: Skip odd blocks "cbz x25, 84f\n" @@ -1115,52 +1114,52 @@ void a64_hybrid_u8qa_mmla_4x16 ( "ldr b3, [x22, #0x0]\n" "82:" // Height 3: Multiply loop: Ragged operand read: Done "trn1 v0.2d, v1.2d, v2.2d\n" - "trn1 v2.2d, v3.2d, v9.2d\n" + "trn1 v2.2d, v3.2d, v4.2d\n" "tbnz %x[flags], #31, 83f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n" "83:" // Height 3: Multiply loop: unique 12: skip row sum - "ldr q10, [x28, #0x0]\n" - "ldr q4, [x28, #0x10]\n" - ".inst 0x6e8aa410 // ummla v16.4s, v0.16b, v10.16b\n" - ".inst 0x6e8aa458 // ummla v24.4s, v2.16b, v10.16b\n" - "ldr q5, [x28, #0x20]\n" + "ldr q1, [x28, #0x0]\n" + "ldr q3, [x28, #0x10]\n" + ".inst 0x6e81a410 // ummla v16.4s, v0.16b, v1.16b\n" + ".inst 0x6e81a458 // ummla v24.4s, v2.16b, v1.16b\n" + "ldr q1, [x28, #0x20]\n" "ldr q6, [x28, #0x30]\n" - ".inst 0x6e84a414 // ummla v20.4s, v0.16b, v4.16b\n" - ".inst 0x6e84a45c // ummla v28.4s, v2.16b, v4.16b\n" - "ldr q7, [x28, #0x40]\n" - "ldr q8, [x28, #0x50]\n" - ".inst 0x6e85a411 // ummla v17.4s, v0.16b, v5.16b\n" - ".inst 0x6e85a459 // ummla v25.4s, v2.16b, v5.16b\n" - "ldr q9, [x28, #0x60]\n" - "ldr q10, [x28, #0x70]\n" + ".inst 0x6e83a414 // ummla v20.4s, v0.16b, v3.16b\n" + ".inst 0x6e83a45c // ummla v28.4s, v2.16b, v3.16b\n" + "ldr q5, [x28, #0x40]\n" + "ldr q4, [x28, #0x50]\n" + ".inst 0x6e81a411 // ummla v17.4s, v0.16b, v1.16b\n" + ".inst 0x6e81a459 // ummla v25.4s, v2.16b, v1.16b\n" + "ldr q3, [x28, #0x60]\n" + "ldr q1, [x28, #0x70]\n" ".inst 0x6e86a415 // ummla v21.4s, v0.16b, v6.16b\n" ".inst 0x6e86a45d // ummla v29.4s, v2.16b, v6.16b\n" - ".inst 0x6e87a412 // ummla v18.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a45a // ummla v26.4s, v2.16b, v7.16b\n" + ".inst 0x6e85a412 // ummla v18.4s, v0.16b, v5.16b\n" + ".inst 0x6e85a45a // ummla v26.4s, v2.16b, v5.16b\n" "add x28, x28, #0x80\n" - ".inst 0x6e88a416 // ummla v22.4s, v0.16b, v8.16b\n" - ".inst 0x6e88a45e // ummla v30.4s, v2.16b, v8.16b\n" - ".inst 0x6e89a413 // ummla v19.4s, v0.16b, v9.16b\n" - ".inst 0x6e89a45b // ummla v27.4s, v2.16b, v9.16b\n" - ".inst 0x6e8aa417 // ummla v23.4s, v0.16b, v10.16b\n" - ".inst 0x6e8aa45f // ummla v31.4s, v2.16b, v10.16b\n" + ".inst 0x6e84a416 // ummla v22.4s, v0.16b, v4.16b\n" + ".inst 0x6e84a45e // ummla v30.4s, v2.16b, v4.16b\n" + ".inst 0x6e83a413 // ummla v19.4s, v0.16b, v3.16b\n" + ".inst 0x6e83a45b // ummla v27.4s, v2.16b, v3.16b\n" + ".inst 0x6e81a417 // ummla v23.4s, v0.16b, v1.16b\n" + ".inst 0x6e81a45f // ummla v31.4s, v2.16b, v1.16b\n" "84:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x26, x26, #0x1\n" "cmp x26, x20\n" "bne 68b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "uzp1 v4.2d, v16.2d, v20.2d\n" - "add x22, x27, x20\n" - "add x21, x22, x20\n" + "uzp1 v0.2d, v16.2d, v20.2d\n" + "add x23, x27, x20\n" + "add x22, x23, x20\n" "uzp2 v16.2d, v16.2d, v20.2d\n" "uzp1 v20.2d, v17.2d, v21.2d\n" "prfm pstl1keep, [x27, #0x0]\n" - "prfm pstl1keep, [x22, #0x0]\n" + "prfm pstl1keep, [x23, #0x0]\n" "uzp2 v17.2d, v17.2d, v21.2d\n" "uzp1 v21.2d, v18.2d, v22.2d\n" - "prfm pstl1keep, [x21, #0x0]\n" + "prfm pstl1keep, [x22, #0x0]\n" "uzp2 v18.2d, v18.2d, v22.2d\n" "uzp1 v22.2d, v19.2d, v23.2d\n" "uzp2 v19.2d, v19.2d, v23.2d\n" @@ -1168,116 +1167,116 @@ void a64_hybrid_u8qa_mmla_4x16 ( "uzp1 v25.2d, v25.2d, v29.2d\n" "uzp1 v26.2d, v26.2d, v30.2d\n" "uzp1 v27.2d, v27.2d, v31.2d\n" - "mov v31.16b, v4.16b\n" + "mov v31.16b, v0.16b\n" "tbnz %x[flags], #31, 85f\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1r { v3.4s }, [x23]\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1r { v23.4s }, [x20]\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v13.4s, v13.4s, v13.4s\n" - "neg v3.4s, v3.4s\n" + "neg v23.4s, v23.4s\n" "dup v12.4s, v11.s[3]\n" "dup v11.4s, v11.s[0]\n" "dup v13.4s, v13.s[0]\n" - "mul v11.4s, v11.4s, v3.4s\n" - "mul v12.4s, v12.4s, v3.4s\n" - "mul v13.4s, v13.4s, v3.4s\n" + "mul v11.4s, v11.4s, v23.4s\n" + "mul v12.4s, v12.4s, v23.4s\n" + "mul v13.4s, v13.4s, v23.4s\n" "85:" // Height 3: skip row sum fixup "ldr q0, [x10, #0x0]\n" - "ldr q1, [x10, #0x10]\n" + "ldr q30, [x10, #0x10]\n" "add v31.4s, v31.4s, v11.4s\n" "add v20.4s, v20.4s, v11.4s\n" - "ldr q2, [x10, #0x20]\n" - "ldr q3, [x10, #0x30]\n" + "ldr q29, [x10, #0x20]\n" + "ldr q28, [x10, #0x30]\n" "add v21.4s, v21.4s, v11.4s\n" "add v22.4s, v22.4s, v11.4s\n" "add v16.4s, v16.4s, v12.4s\n" "add v17.4s, v17.4s, v12.4s\n" - "add x23, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v23.4s }, [x20]\n" "add v18.4s, v18.4s, v12.4s\n" "add v19.4s, v19.4s, v12.4s\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" "add v24.4s, v24.4s, v13.4s\n" "add v25.4s, v25.4s, v13.4s\n" "add x10, x10, #0x40\n" "add v26.4s, v26.4s, v13.4s\n" "add v27.4s, v27.4s, v13.4s\n" "add v31.4s, v31.4s, v0.4s\n" - "add v20.4s, v20.4s, v1.4s\n" - "add v21.4s, v21.4s, v2.4s\n" - "add v22.4s, v22.4s, v3.4s\n" + "add v20.4s, v20.4s, v30.4s\n" + "add v21.4s, v21.4s, v29.4s\n" + "add v22.4s, v22.4s, v28.4s\n" "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" + "add v17.4s, v17.4s, v30.4s\n" + "add v18.4s, v18.4s, v29.4s\n" + "add v19.4s, v19.4s, v28.4s\n" "add v24.4s, v24.4s, v0.4s\n" - "ld1r { v0.4s }, [x23]\n" - "add v25.4s, v25.4s, v1.4s\n" - "add v26.4s, v26.4s, v2.4s\n" - "add v27.4s, v27.4s, v3.4s\n" - "sqrdmulh v31.4s, v31.4s, v4.4s\n" - "sqrdmulh v20.4s, v20.4s, v4.4s\n" - "sqrdmulh v21.4s, v21.4s, v4.4s\n" - "sqrdmulh v22.4s, v22.4s, v4.4s\n" - "sqrdmulh v16.4s, v16.4s, v4.4s\n" - "sqrdmulh v17.4s, v17.4s, v4.4s\n" - "sqrdmulh v18.4s, v18.4s, v4.4s\n" - "sqrdmulh v19.4s, v19.4s, v4.4s\n" - "sqrdmulh v24.4s, v24.4s, v4.4s\n" - "sqrdmulh v25.4s, v25.4s, v4.4s\n" - "sqrdmulh v26.4s, v26.4s, v4.4s\n" - "sqrdmulh v27.4s, v27.4s, v4.4s\n" + "ld1r { v0.4s }, [x20]\n" + "add v25.4s, v25.4s, v30.4s\n" + "add v26.4s, v26.4s, v29.4s\n" + "add v27.4s, v27.4s, v28.4s\n" + "sqrdmulh v31.4s, v31.4s, v23.4s\n" + "sqrdmulh v20.4s, v20.4s, v23.4s\n" + "sqrdmulh v21.4s, v21.4s, v23.4s\n" + "sqrdmulh v22.4s, v22.4s, v23.4s\n" + "sqrdmulh v16.4s, v16.4s, v23.4s\n" + "sqrdmulh v17.4s, v17.4s, v23.4s\n" + "sqrdmulh v18.4s, v18.4s, v23.4s\n" + "sqrdmulh v19.4s, v19.4s, v23.4s\n" + "sqrdmulh v24.4s, v24.4s, v23.4s\n" + "sqrdmulh v25.4s, v25.4s, v23.4s\n" + "sqrdmulh v26.4s, v26.4s, v23.4s\n" + "sqrdmulh v27.4s, v27.4s, v23.4s\n" "tbz %x[flags], #5, 86f\n" - "and v4.16b, v31.16b, v0.16b\n" - "and v5.16b, v20.16b, v0.16b\n" - "and v6.16b, v21.16b, v0.16b\n" - "and v7.16b, v22.16b, v0.16b\n" - "and v8.16b, v16.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sqadd v31.4s, v31.4s, v4.4s\n" - "sqadd v20.4s, v20.4s, v5.4s\n" - "sqadd v21.4s, v21.4s, v6.4s\n" - "sqadd v22.4s, v22.4s, v7.4s\n" - "sqadd v16.4s, v16.4s, v8.4s\n" - "and v9.16b, v17.16b, v0.16b\n" - "and v10.16b, v18.16b, v0.16b\n" - "and v4.16b, v19.16b, v0.16b\n" - "and v5.16b, v24.16b, v0.16b\n" - "and v6.16b, v25.16b, v0.16b\n" - "and v7.16b, v26.16b, v0.16b\n" - "and v8.16b, v27.16b, v0.16b\n" - "sshr v9.4s, v9.4s, #0x1f\n" - "sshr v10.4s, v10.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sqadd v17.4s, v17.4s, v9.4s\n" - "sqadd v18.4s, v18.4s, v10.4s\n" - "sqadd v19.4s, v19.4s, v4.4s\n" - "sqadd v24.4s, v24.4s, v5.4s\n" - "sqadd v25.4s, v25.4s, v6.4s\n" - "sqadd v26.4s, v26.4s, v7.4s\n" - "sqadd v27.4s, v27.4s, v8.4s\n" + "and v1.16b, v31.16b, v0.16b\n" + "and v30.16b, v20.16b, v0.16b\n" + "and v29.16b, v21.16b, v0.16b\n" + "and v28.16b, v22.16b, v0.16b\n" + "and v23.16b, v16.16b, v0.16b\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "sqadd v31.4s, v31.4s, v1.4s\n" + "sqadd v20.4s, v20.4s, v30.4s\n" + "sqadd v21.4s, v21.4s, v29.4s\n" + "sqadd v22.4s, v22.4s, v28.4s\n" + "sqadd v16.4s, v16.4s, v23.4s\n" + "and v3.16b, v17.16b, v0.16b\n" + "and v2.16b, v18.16b, v0.16b\n" + "and v1.16b, v19.16b, v0.16b\n" + "and v30.16b, v24.16b, v0.16b\n" + "and v29.16b, v25.16b, v0.16b\n" + "and v28.16b, v26.16b, v0.16b\n" + "and v23.16b, v27.16b, v0.16b\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sshr v30.4s, v30.4s, #0x1f\n" + "sshr v29.4s, v29.4s, #0x1f\n" + "sshr v28.4s, v28.4s, #0x1f\n" + "sshr v23.4s, v23.4s, #0x1f\n" + "sqadd v17.4s, v17.4s, v3.4s\n" + "sqadd v18.4s, v18.4s, v2.4s\n" + "sqadd v19.4s, v19.4s, v1.4s\n" + "sqadd v24.4s, v24.4s, v30.4s\n" + "sqadd v25.4s, v25.4s, v29.4s\n" + "sqadd v26.4s, v26.4s, v28.4s\n" + "sqadd v27.4s, v27.4s, v23.4s\n" "86:" // Height 3: no shift correction - "add x23, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v29.4s }, [x20]\n" "srshl v31.4s, v31.4s, v0.4s\n" "srshl v20.4s, v20.4s, v0.4s\n" "srshl v21.4s, v21.4s, v0.4s\n" "srshl v22.4s, v22.4s, v0.4s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x23]\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v28.4s }, [x20]\n" "srshl v16.4s, v16.4s, v0.4s\n" "srshl v17.4s, v17.4s, v0.4s\n" - "add x23, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x23]\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v23.4s }, [x20]\n" "srshl v18.4s, v18.4s, v0.4s\n" "srshl v19.4s, v19.4s, v0.4s\n" "cmp x9, #0x10\n" @@ -1285,132 +1284,132 @@ void a64_hybrid_u8qa_mmla_4x16 ( "srshl v25.4s, v25.4s, v0.4s\n" "srshl v26.4s, v26.4s, v0.4s\n" "srshl v27.4s, v27.4s, v0.4s\n" - "add v31.4s, v31.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add v24.4s, v24.4s, v4.4s\n" - "add v25.4s, v25.4s, v4.4s\n" - "add v26.4s, v26.4s, v4.4s\n" - "add v27.4s, v27.4s, v4.4s\n" - "smin v31.4s, v31.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smin v24.4s, v24.4s, v6.4s\n" - "smin v25.4s, v25.4s, v6.4s\n" - "smin v26.4s, v26.4s, v6.4s\n" - "smin v27.4s, v27.4s, v6.4s\n" - "smax v31.4s, v31.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" - "smax v24.4s, v24.4s, v5.4s\n" - "smax v25.4s, v25.4s, v5.4s\n" - "smax v26.4s, v26.4s, v5.4s\n" - "smax v27.4s, v27.4s, v5.4s\n" + "add v31.4s, v31.4s, v29.4s\n" + "add v20.4s, v20.4s, v29.4s\n" + "add v21.4s, v21.4s, v29.4s\n" + "add v22.4s, v22.4s, v29.4s\n" + "add v16.4s, v16.4s, v29.4s\n" + "add v17.4s, v17.4s, v29.4s\n" + "add v18.4s, v18.4s, v29.4s\n" + "add v19.4s, v19.4s, v29.4s\n" + "add v24.4s, v24.4s, v29.4s\n" + "add v25.4s, v25.4s, v29.4s\n" + "add v26.4s, v26.4s, v29.4s\n" + "add v27.4s, v27.4s, v29.4s\n" + "smin v31.4s, v31.4s, v28.4s\n" + "smin v20.4s, v20.4s, v28.4s\n" + "smin v21.4s, v21.4s, v28.4s\n" + "smin v22.4s, v22.4s, v28.4s\n" + "smin v16.4s, v16.4s, v28.4s\n" + "smin v17.4s, v17.4s, v28.4s\n" + "smin v18.4s, v18.4s, v28.4s\n" + "smin v19.4s, v19.4s, v28.4s\n" + "smin v24.4s, v24.4s, v28.4s\n" + "smin v25.4s, v25.4s, v28.4s\n" + "smin v26.4s, v26.4s, v28.4s\n" + "smin v27.4s, v27.4s, v28.4s\n" + "smax v31.4s, v31.4s, v23.4s\n" + "smax v20.4s, v20.4s, v23.4s\n" + "smax v21.4s, v21.4s, v23.4s\n" + "smax v22.4s, v22.4s, v23.4s\n" + "smax v16.4s, v16.4s, v23.4s\n" + "smax v17.4s, v17.4s, v23.4s\n" + "smax v18.4s, v18.4s, v23.4s\n" + "smax v19.4s, v19.4s, v23.4s\n" + "smax v24.4s, v24.4s, v23.4s\n" + "smax v25.4s, v25.4s, v23.4s\n" + "smax v26.4s, v26.4s, v23.4s\n" + "smax v27.4s, v27.4s, v23.4s\n" "uzp1 v31.8h, v31.8h, v20.8h\n" "uzp1 v20.8h, v21.8h, v22.8h\n" "uzp1 v16.8h, v16.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v18.8h, v18.8h, v19.8h\n" "uzp1 v24.8h, v24.8h, v25.8h\n" - "uzp1 v25.8h, v26.8h, v27.8h\n" + "uzp1 v17.8h, v26.8h, v27.8h\n" "uzp1 v31.16b, v31.16b, v20.16b\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" - "uzp1 v24.16b, v24.16b, v25.16b\n" + "uzp1 v16.16b, v16.16b, v18.16b\n" + "uzp1 v24.16b, v24.16b, v17.16b\n" "bge 95f\n" "tbz x9, #3, 90f\n" "str d31, [x27], #0x8\n" - "str d16, [x22], #0x8\n" - "str d24, [x21], #0x8\n" + "str d16, [x23], #0x8\n" + "str d24, [x22], #0x8\n" "tbz x9, #2, 88f\n" "st1 { v31.s }[2], [x27], #0x4\n" - "st1 { v16.s }[2], [x22], #0x4\n" - "st1 { v24.s }[2], [x21], #0x4\n" + "st1 { v16.s }[2], [x23], #0x4\n" + "st1 { v24.s }[2], [x22], #0x4\n" "tbz x9, #1, 87f\n" "st1 { v31.h }[6], [x27], #0x2\n" - "st1 { v16.h }[6], [x22], #0x2\n" - "st1 { v24.h }[6], [x21], #0x2\n" + "st1 { v16.h }[6], [x23], #0x2\n" + "st1 { v24.h }[6], [x22], #0x2\n" "tbz x9, #0, 94f\n" "st1 { v31.b }[14], [x27]\n" - "st1 { v16.b }[14], [x22]\n" - "st1 { v24.b }[14], [x21]\n" + "st1 { v16.b }[14], [x23]\n" + "st1 { v24.b }[14], [x22]\n" "b 94f\n" "87:" // Height 3: Partial direct writeback: partial_1_12 "tbz x9, #0, 94f\n" "st1 { v31.b }[12], [x27]\n" - "st1 { v16.b }[12], [x22]\n" - "st1 { v24.b }[12], [x21]\n" + "st1 { v16.b }[12], [x23]\n" + "st1 { v24.b }[12], [x22]\n" "b 94f\n" "88:" // Height 3: Partial direct writeback: partial_2_8 "tbz x9, #1, 89f\n" "st1 { v31.h }[4], [x27], #0x2\n" - "st1 { v16.h }[4], [x22], #0x2\n" - "st1 { v24.h }[4], [x21], #0x2\n" + "st1 { v16.h }[4], [x23], #0x2\n" + "st1 { v24.h }[4], [x22], #0x2\n" "tbz x9, #0, 94f\n" "st1 { v31.b }[10], [x27]\n" - "st1 { v16.b }[10], [x22]\n" - "st1 { v24.b }[10], [x21]\n" + "st1 { v16.b }[10], [x23]\n" + "st1 { v24.b }[10], [x22]\n" "b 94f\n" "89:" // Height 3: Partial direct writeback: partial_1_8 "tbz x9, #0, 94f\n" "st1 { v31.b }[8], [x27]\n" - "st1 { v16.b }[8], [x22]\n" - "st1 { v24.b }[8], [x21]\n" + "st1 { v16.b }[8], [x23]\n" + "st1 { v24.b }[8], [x22]\n" "b 94f\n" "90:" // Height 3: Partial direct writeback: partial_4_0 "tbz x9, #2, 92f\n" "str s31, [x27], #0x4\n" - "str s16, [x22], #0x4\n" - "str s24, [x21], #0x4\n" + "str s16, [x23], #0x4\n" + "str s24, [x22], #0x4\n" "tbz x9, #1, 91f\n" "st1 { v31.h }[2], [x27], #0x2\n" - "st1 { v16.h }[2], [x22], #0x2\n" - "st1 { v24.h }[2], [x21], #0x2\n" + "st1 { v16.h }[2], [x23], #0x2\n" + "st1 { v24.h }[2], [x22], #0x2\n" "tbz x9, #0, 94f\n" "st1 { v31.b }[6], [x27]\n" - "st1 { v16.b }[6], [x22]\n" - "st1 { v24.b }[6], [x21]\n" + "st1 { v16.b }[6], [x23]\n" + "st1 { v24.b }[6], [x22]\n" "b 94f\n" "91:" // Height 3: Partial direct writeback: partial_1_4 "tbz x9, #0, 94f\n" "st1 { v31.b }[4], [x27]\n" - "st1 { v16.b }[4], [x22]\n" - "st1 { v24.b }[4], [x21]\n" + "st1 { v16.b }[4], [x23]\n" + "st1 { v24.b }[4], [x22]\n" "b 94f\n" "92:" // Height 3: Partial direct writeback: partial_2_0 "tbz x9, #1, 93f\n" "str h31, [x27], #0x2\n" - "str h16, [x22], #0x2\n" - "str h24, [x21], #0x2\n" + "str h16, [x23], #0x2\n" + "str h24, [x22], #0x2\n" "tbz x9, #0, 94f\n" "st1 { v31.b }[2], [x27]\n" - "st1 { v16.b }[2], [x22]\n" - "st1 { v24.b }[2], [x21]\n" + "st1 { v16.b }[2], [x23]\n" + "st1 { v24.b }[2], [x22]\n" "b 94f\n" "93:" // Height 3: Partial direct writeback: partial_1_0 "str b31, [x27, #0x0]\n" - "str b16, [x22, #0x0]\n" - "str b24, [x21, #0x0]\n" + "str b16, [x23, #0x0]\n" + "str b24, [x22, #0x0]\n" "94:" // Height 3: Partial direct writeback: Done "b 96f\n" "95:" // Height 3: Full writeback "str q31, [x27, #0x0]\n" "add x27, x27, #0x10\n" - "str q16, [x22, #0x0]\n" - "str q24, [x21, #0x0]\n" + "str q16, [x23, #0x0]\n" + "str q24, [x22, #0x0]\n" "96:" // Height 3: Writeback done "subs x9, x9, #0x10\n" "bgt 66b\n" @@ -1451,14 +1450,14 @@ void a64_hybrid_u8qa_mmla_4x16 ( "100:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 101f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" - "ldr x21, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" + "ldr x21, [x20, #0x18]\n" "cbnz x26, 102f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -1468,9 +1467,9 @@ void a64_hybrid_u8qa_mmla_4x16 ( "b 102f\n" "101:" // Height 4: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" + "add x21, x22, x21\n" "102:" // Height 4: input setup done "cmp x25, #0x10\n" "blt 107f\n" @@ -1630,42 +1629,42 @@ void a64_hybrid_u8qa_mmla_4x16 ( "blt 110f\n" "108:" // Height 4: Multiply loop: Odd block loop "ldr d1, [x24], #0x8\n" - "ldr d2, [x23], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "ldr d3, [x22], #0x8\n" - "ldr d7, [x21], #0x8\n" - "trn1 v2.2d, v3.2d, v7.2d\n" + "ldr d0, [x23], #0x8\n" + "trn1 v0.2d, v1.2d, v0.2d\n" + "ldr d2, [x22], #0x8\n" + "ldr d1, [x21], #0x8\n" + "trn1 v2.2d, v2.2d, v1.2d\n" "tbnz %x[flags], #31, 109f\n" ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n" "109:" // Height 4: Multiply loop: unique 15: skip row sum - "ldr q8, [x28, #0x0]\n" - "ldr q9, [x28, #0x10]\n" - ".inst 0x6e88a410 // ummla v16.4s, v0.16b, v8.16b\n" - ".inst 0x6e88a458 // ummla v24.4s, v2.16b, v8.16b\n" - "ldr q10, [x28, #0x20]\n" - "ldr q4, [x28, #0x30]\n" + "ldr q3, [x28, #0x0]\n" + "ldr q1, [x28, #0x10]\n" + ".inst 0x6e83a410 // ummla v16.4s, v0.16b, v3.16b\n" + ".inst 0x6e83a458 // ummla v24.4s, v2.16b, v3.16b\n" + "ldr q7, [x28, #0x20]\n" + "ldr q6, [x28, #0x30]\n" "sub x25, x25, #0x8\n" "cmp x25, #0x8\n" "ldr q5, [x28, #0x40]\n" - "ldr q6, [x28, #0x50]\n" - ".inst 0x6e89a414 // ummla v20.4s, v0.16b, v9.16b\n" - ".inst 0x6e89a45c // ummla v28.4s, v2.16b, v9.16b\n" - "ldr q7, [x28, #0x60]\n" - "ldr q8, [x28, #0x70]\n" - ".inst 0x6e8aa411 // ummla v17.4s, v0.16b, v10.16b\n" - ".inst 0x6e8aa459 // ummla v25.4s, v2.16b, v10.16b\n" - ".inst 0x6e84a415 // ummla v21.4s, v0.16b, v4.16b\n" - ".inst 0x6e84a45d // ummla v29.4s, v2.16b, v4.16b\n" + "ldr q4, [x28, #0x50]\n" + ".inst 0x6e81a414 // ummla v20.4s, v0.16b, v1.16b\n" + ".inst 0x6e81a45c // ummla v28.4s, v2.16b, v1.16b\n" + "ldr q3, [x28, #0x60]\n" + "ldr q1, [x28, #0x70]\n" + ".inst 0x6e87a411 // ummla v17.4s, v0.16b, v7.16b\n" + ".inst 0x6e87a459 // ummla v25.4s, v2.16b, v7.16b\n" + ".inst 0x6e86a415 // ummla v21.4s, v0.16b, v6.16b\n" + ".inst 0x6e86a45d // ummla v29.4s, v2.16b, v6.16b\n" "add x28, x28, #0x80\n" ".inst 0x6e85a412 // ummla v18.4s, v0.16b, v5.16b\n" ".inst 0x6e85a45a // ummla v26.4s, v2.16b, v5.16b\n" - ".inst 0x6e86a416 // ummla v22.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a45e // ummla v30.4s, v2.16b, v6.16b\n" - ".inst 0x6e87a413 // ummla v19.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a45b // ummla v27.4s, v2.16b, v7.16b\n" - ".inst 0x6e88a417 // ummla v23.4s, v0.16b, v8.16b\n" - ".inst 0x6e88a45f // ummla v31.4s, v2.16b, v8.16b\n" + ".inst 0x6e84a416 // ummla v22.4s, v0.16b, v4.16b\n" + ".inst 0x6e84a45e // ummla v30.4s, v2.16b, v4.16b\n" + ".inst 0x6e83a413 // ummla v19.4s, v0.16b, v3.16b\n" + ".inst 0x6e83a45b // ummla v27.4s, v2.16b, v3.16b\n" + ".inst 0x6e81a417 // ummla v23.4s, v0.16b, v1.16b\n" + ".inst 0x6e81a45f // ummla v31.4s, v2.16b, v1.16b\n" "bge 108b\n" "110:" // Height 4: Multiply loop: Skip odd blocks "cbz x25, 116f\n" @@ -1716,51 +1715,51 @@ void a64_hybrid_u8qa_mmla_4x16 ( ".inst 0x6e8f940b // udot v11.4s, v0.16b, v15.16b\n" ".inst 0x6e8f944d // udot v13.4s, v2.16b, v15.16b\n" "115:" // Height 4: Multiply loop: unique 16: skip row sum - "ldr q10, [x28, #0x0]\n" - "ldr q4, [x28, #0x10]\n" - ".inst 0x6e8aa410 // ummla v16.4s, v0.16b, v10.16b\n" - ".inst 0x6e8aa458 // ummla v24.4s, v2.16b, v10.16b\n" - "ldr q5, [x28, #0x20]\n" + "ldr q1, [x28, #0x0]\n" + "ldr q3, [x28, #0x10]\n" + ".inst 0x6e81a410 // ummla v16.4s, v0.16b, v1.16b\n" + ".inst 0x6e81a458 // ummla v24.4s, v2.16b, v1.16b\n" + "ldr q1, [x28, #0x20]\n" "ldr q6, [x28, #0x30]\n" - ".inst 0x6e84a414 // ummla v20.4s, v0.16b, v4.16b\n" - ".inst 0x6e84a45c // ummla v28.4s, v2.16b, v4.16b\n" - "ldr q7, [x28, #0x40]\n" - "ldr q8, [x28, #0x50]\n" - ".inst 0x6e85a411 // ummla v17.4s, v0.16b, v5.16b\n" - ".inst 0x6e85a459 // ummla v25.4s, v2.16b, v5.16b\n" - "ldr q9, [x28, #0x60]\n" - "ldr q10, [x28, #0x70]\n" + ".inst 0x6e83a414 // ummla v20.4s, v0.16b, v3.16b\n" + ".inst 0x6e83a45c // ummla v28.4s, v2.16b, v3.16b\n" + "ldr q5, [x28, #0x40]\n" + "ldr q4, [x28, #0x50]\n" + ".inst 0x6e81a411 // ummla v17.4s, v0.16b, v1.16b\n" + ".inst 0x6e81a459 // ummla v25.4s, v2.16b, v1.16b\n" + "ldr q3, [x28, #0x60]\n" + "ldr q1, [x28, #0x70]\n" ".inst 0x6e86a415 // ummla v21.4s, v0.16b, v6.16b\n" ".inst 0x6e86a45d // ummla v29.4s, v2.16b, v6.16b\n" - ".inst 0x6e87a412 // ummla v18.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a45a // ummla v26.4s, v2.16b, v7.16b\n" + ".inst 0x6e85a412 // ummla v18.4s, v0.16b, v5.16b\n" + ".inst 0x6e85a45a // ummla v26.4s, v2.16b, v5.16b\n" "add x28, x28, #0x80\n" - ".inst 0x6e88a416 // ummla v22.4s, v0.16b, v8.16b\n" - ".inst 0x6e88a45e // ummla v30.4s, v2.16b, v8.16b\n" - ".inst 0x6e89a413 // ummla v19.4s, v0.16b, v9.16b\n" - ".inst 0x6e89a45b // ummla v27.4s, v2.16b, v9.16b\n" - ".inst 0x6e8aa417 // ummla v23.4s, v0.16b, v10.16b\n" - ".inst 0x6e8aa45f // ummla v31.4s, v2.16b, v10.16b\n" + ".inst 0x6e84a416 // ummla v22.4s, v0.16b, v4.16b\n" + ".inst 0x6e84a45e // ummla v30.4s, v2.16b, v4.16b\n" + ".inst 0x6e83a413 // ummla v19.4s, v0.16b, v3.16b\n" + ".inst 0x6e83a45b // ummla v27.4s, v2.16b, v3.16b\n" + ".inst 0x6e81a417 // ummla v23.4s, v0.16b, v1.16b\n" + ".inst 0x6e81a45f // ummla v31.4s, v2.16b, v1.16b\n" "116:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x26, x26, #0x1\n" "cmp x26, x20\n" "bne 100b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "uzp1 v4.2d, v16.2d, v20.2d\n" - "add x22, x27, x20\n" + "uzp1 v0.2d, v16.2d, v20.2d\n" + "add x23, x27, x20\n" + "add x22, x23, x20\n" "add x21, x22, x20\n" - "add x20, x21, x20\n" "uzp2 v16.2d, v16.2d, v20.2d\n" "uzp1 v20.2d, v17.2d, v21.2d\n" "prfm pstl1keep, [x27, #0x0]\n" "uzp2 v17.2d, v17.2d, v21.2d\n" "uzp1 v21.2d, v18.2d, v22.2d\n" + "prfm pstl1keep, [x23, #0x0]\n" "prfm pstl1keep, [x22, #0x0]\n" - "prfm pstl1keep, [x21, #0x0]\n" "uzp2 v18.2d, v18.2d, v22.2d\n" "uzp1 v22.2d, v19.2d, v23.2d\n" - "prfm pstl1keep, [x20, #0x0]\n" + "prfm pstl1keep, [x21, #0x0]\n" "uzp2 v19.2d, v19.2d, v23.2d\n" "uzp1 v23.2d, v24.2d, v28.2d\n" "uzp2 v24.2d, v24.2d, v28.2d\n" @@ -1770,38 +1769,38 @@ void a64_hybrid_u8qa_mmla_4x16 ( "uzp2 v26.2d, v26.2d, v30.2d\n" "uzp1 v30.2d, v27.2d, v31.2d\n" "uzp2 v27.2d, v27.2d, v31.2d\n" - "mov v31.16b, v4.16b\n" + "mov v31.16b, v0.16b\n" "tbnz %x[flags], #31, 117f\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1r { v0.4s }, [x20]\n" "addp v11.4s, v11.4s, v11.4s\n" "addp v13.4s, v13.4s, v13.4s\n" - "neg v4.4s, v4.4s\n" + "neg v0.4s, v0.4s\n" "dup v12.4s, v11.s[3]\n" "dup v11.4s, v11.s[0]\n" "dup v14.4s, v13.s[3]\n" "dup v13.4s, v13.s[0]\n" - "mul v11.4s, v11.4s, v4.4s\n" - "mul v12.4s, v12.4s, v4.4s\n" - "mul v13.4s, v13.4s, v4.4s\n" - "mul v14.4s, v14.4s, v4.4s\n" + "mul v11.4s, v11.4s, v0.4s\n" + "mul v12.4s, v12.4s, v0.4s\n" + "mul v13.4s, v13.4s, v0.4s\n" + "mul v14.4s, v14.4s, v0.4s\n" "117:" // Height 4: skip row sum fixup "ldr q0, [x10, #0x0]\n" - "ldr q1, [x10, #0x10]\n" + "ldr q4, [x10, #0x10]\n" "add v31.4s, v31.4s, v11.4s\n" "add v20.4s, v20.4s, v11.4s\n" - "ldr q2, [x10, #0x20]\n" - "ldr q3, [x10, #0x30]\n" + "ldr q3, [x10, #0x20]\n" + "ldr q2, [x10, #0x30]\n" "add v21.4s, v21.4s, v11.4s\n" "add v22.4s, v22.4s, v11.4s\n" "add v16.4s, v16.4s, v12.4s\n" "add v17.4s, v17.4s, v12.4s\n" - "add x23, %x[qp], %[per_layer_mul]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1r { v1.4s }, [x20]\n" "add v18.4s, v18.4s, v12.4s\n" "add v19.4s, v19.4s, v12.4s\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" "add v23.4s, v23.4s, v13.4s\n" "add v28.4s, v28.4s, v13.4s\n" "add x10, x10, #0x40\n" @@ -1812,100 +1811,100 @@ void a64_hybrid_u8qa_mmla_4x16 ( "add v26.4s, v26.4s, v14.4s\n" "add v27.4s, v27.4s, v14.4s\n" "add v31.4s, v31.4s, v0.4s\n" - "add v20.4s, v20.4s, v1.4s\n" - "add v21.4s, v21.4s, v2.4s\n" - "add v22.4s, v22.4s, v3.4s\n" + "add v20.4s, v20.4s, v4.4s\n" + "add v21.4s, v21.4s, v3.4s\n" + "add v22.4s, v22.4s, v2.4s\n" "add v16.4s, v16.4s, v0.4s\n" - "add v17.4s, v17.4s, v1.4s\n" - "add v18.4s, v18.4s, v2.4s\n" - "add v19.4s, v19.4s, v3.4s\n" + "add v17.4s, v17.4s, v4.4s\n" + "add v18.4s, v18.4s, v3.4s\n" + "add v19.4s, v19.4s, v2.4s\n" "add v23.4s, v23.4s, v0.4s\n" - "add v28.4s, v28.4s, v1.4s\n" - "add v29.4s, v29.4s, v2.4s\n" - "add v30.4s, v30.4s, v3.4s\n" + "add v28.4s, v28.4s, v4.4s\n" + "add v29.4s, v29.4s, v3.4s\n" + "add v30.4s, v30.4s, v2.4s\n" "add v24.4s, v24.4s, v0.4s\n" - "ld1r { v0.4s }, [x23]\n" - "add v25.4s, v25.4s, v1.4s\n" - "add v26.4s, v26.4s, v2.4s\n" - "add v27.4s, v27.4s, v3.4s\n" - "sqrdmulh v31.4s, v31.4s, v4.4s\n" - "sqrdmulh v20.4s, v20.4s, v4.4s\n" - "sqrdmulh v21.4s, v21.4s, v4.4s\n" - "sqrdmulh v22.4s, v22.4s, v4.4s\n" - "sqrdmulh v16.4s, v16.4s, v4.4s\n" - "sqrdmulh v17.4s, v17.4s, v4.4s\n" - "sqrdmulh v18.4s, v18.4s, v4.4s\n" - "sqrdmulh v19.4s, v19.4s, v4.4s\n" - "sqrdmulh v23.4s, v23.4s, v4.4s\n" - "sqrdmulh v28.4s, v28.4s, v4.4s\n" - "sqrdmulh v29.4s, v29.4s, v4.4s\n" - "sqrdmulh v30.4s, v30.4s, v4.4s\n" - "sqrdmulh v24.4s, v24.4s, v4.4s\n" - "sqrdmulh v25.4s, v25.4s, v4.4s\n" - "sqrdmulh v26.4s, v26.4s, v4.4s\n" - "sqrdmulh v27.4s, v27.4s, v4.4s\n" + "ld1r { v0.4s }, [x20]\n" + "add v25.4s, v25.4s, v4.4s\n" + "add v26.4s, v26.4s, v3.4s\n" + "add v27.4s, v27.4s, v2.4s\n" + "sqrdmulh v31.4s, v31.4s, v1.4s\n" + "sqrdmulh v20.4s, v20.4s, v1.4s\n" + "sqrdmulh v21.4s, v21.4s, v1.4s\n" + "sqrdmulh v22.4s, v22.4s, v1.4s\n" + "sqrdmulh v16.4s, v16.4s, v1.4s\n" + "sqrdmulh v17.4s, v17.4s, v1.4s\n" + "sqrdmulh v18.4s, v18.4s, v1.4s\n" + "sqrdmulh v19.4s, v19.4s, v1.4s\n" + "sqrdmulh v23.4s, v23.4s, v1.4s\n" + "sqrdmulh v28.4s, v28.4s, v1.4s\n" + "sqrdmulh v29.4s, v29.4s, v1.4s\n" + "sqrdmulh v30.4s, v30.4s, v1.4s\n" + "sqrdmulh v24.4s, v24.4s, v1.4s\n" + "sqrdmulh v25.4s, v25.4s, v1.4s\n" + "sqrdmulh v26.4s, v26.4s, v1.4s\n" + "sqrdmulh v27.4s, v27.4s, v1.4s\n" "tbz %x[flags], #5, 118f\n" - "and v4.16b, v31.16b, v0.16b\n" - "and v5.16b, v20.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v31.4s, v31.4s, v4.4s\n" - "sqadd v20.4s, v20.4s, v5.4s\n" - "and v6.16b, v21.16b, v0.16b\n" - "and v7.16b, v22.16b, v0.16b\n" - "and v8.16b, v16.16b, v0.16b\n" - "and v9.16b, v17.16b, v0.16b\n" - "and v10.16b, v18.16b, v0.16b\n" - "and v4.16b, v19.16b, v0.16b\n" - "and v5.16b, v23.16b, v0.16b\n" - "sshr v6.4s, v6.4s, #0x1f\n" + "and v2.16b, v31.16b, v0.16b\n" + "and v1.16b, v20.16b, v0.16b\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sqadd v31.4s, v31.4s, v2.4s\n" + "sqadd v20.4s, v20.4s, v1.4s\n" + "and v7.16b, v21.16b, v0.16b\n" + "and v6.16b, v22.16b, v0.16b\n" + "and v5.16b, v16.16b, v0.16b\n" + "and v4.16b, v17.16b, v0.16b\n" + "and v3.16b, v18.16b, v0.16b\n" + "and v2.16b, v19.16b, v0.16b\n" + "and v1.16b, v23.16b, v0.16b\n" "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v9.4s, v9.4s, #0x1f\n" - "sshr v10.4s, v10.4s, #0x1f\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v21.4s, v21.4s, v6.4s\n" - "sqadd v22.4s, v22.4s, v7.4s\n" - "sqadd v16.4s, v16.4s, v8.4s\n" - "sqadd v17.4s, v17.4s, v9.4s\n" - "sqadd v18.4s, v18.4s, v10.4s\n" - "sqadd v19.4s, v19.4s, v4.4s\n" - "sqadd v23.4s, v23.4s, v5.4s\n" - "and v6.16b, v28.16b, v0.16b\n" - "and v7.16b, v29.16b, v0.16b\n" - "and v8.16b, v30.16b, v0.16b\n" - "and v9.16b, v24.16b, v0.16b\n" - "and v10.16b, v25.16b, v0.16b\n" - "and v4.16b, v26.16b, v0.16b\n" - "and v5.16b, v27.16b, v0.16b\n" "sshr v6.4s, v6.4s, #0x1f\n" - "sshr v7.4s, v7.4s, #0x1f\n" - "sshr v8.4s, v8.4s, #0x1f\n" - "sshr v9.4s, v9.4s, #0x1f\n" - "sshr v10.4s, v10.4s, #0x1f\n" + "sshr v5.4s, v5.4s, #0x1f\n" "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sqadd v21.4s, v21.4s, v7.4s\n" + "sqadd v22.4s, v22.4s, v6.4s\n" + "sqadd v16.4s, v16.4s, v5.4s\n" + "sqadd v17.4s, v17.4s, v4.4s\n" + "sqadd v18.4s, v18.4s, v3.4s\n" + "sqadd v19.4s, v19.4s, v2.4s\n" + "sqadd v23.4s, v23.4s, v1.4s\n" + "and v7.16b, v28.16b, v0.16b\n" + "and v6.16b, v29.16b, v0.16b\n" + "and v5.16b, v30.16b, v0.16b\n" + "and v4.16b, v24.16b, v0.16b\n" + "and v3.16b, v25.16b, v0.16b\n" + "and v2.16b, v26.16b, v0.16b\n" + "and v1.16b, v27.16b, v0.16b\n" + "sshr v7.4s, v7.4s, #0x1f\n" + "sshr v6.4s, v6.4s, #0x1f\n" "sshr v5.4s, v5.4s, #0x1f\n" - "sqadd v28.4s, v28.4s, v6.4s\n" - "sqadd v29.4s, v29.4s, v7.4s\n" - "sqadd v30.4s, v30.4s, v8.4s\n" - "sqadd v24.4s, v24.4s, v9.4s\n" - "sqadd v25.4s, v25.4s, v10.4s\n" - "sqadd v26.4s, v26.4s, v4.4s\n" - "sqadd v27.4s, v27.4s, v5.4s\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v3.4s, v3.4s, #0x1f\n" + "sshr v2.4s, v2.4s, #0x1f\n" + "sshr v1.4s, v1.4s, #0x1f\n" + "sqadd v28.4s, v28.4s, v7.4s\n" + "sqadd v29.4s, v29.4s, v6.4s\n" + "sqadd v30.4s, v30.4s, v5.4s\n" + "sqadd v24.4s, v24.4s, v4.4s\n" + "sqadd v25.4s, v25.4s, v3.4s\n" + "sqadd v26.4s, v26.4s, v2.4s\n" + "sqadd v27.4s, v27.4s, v1.4s\n" "118:" // Height 4: no shift correction - "add x23, %x[qp], %[c_offset]\n" - "ld1r { v4.4s }, [x23]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1r { v3.4s }, [x20]\n" "srshl v31.4s, v31.4s, v0.4s\n" "srshl v20.4s, v20.4s, v0.4s\n" "srshl v21.4s, v21.4s, v0.4s\n" "srshl v22.4s, v22.4s, v0.4s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1r { v6.4s }, [x23]\n" + "add x20, %x[qp], %[maxval]\n" + "ld1r { v2.4s }, [x20]\n" "srshl v16.4s, v16.4s, v0.4s\n" "srshl v17.4s, v17.4s, v0.4s\n" - "add x23, %x[qp], %[minval]\n" - "ld1r { v5.4s }, [x23]\n" + "add x20, %x[qp], %[minval]\n" + "ld1r { v1.4s }, [x20]\n" "srshl v18.4s, v18.4s, v0.4s\n" "srshl v19.4s, v19.4s, v0.4s\n" "cmp x9, #0x10\n" @@ -1917,163 +1916,163 @@ void a64_hybrid_u8qa_mmla_4x16 ( "srshl v25.4s, v25.4s, v0.4s\n" "srshl v26.4s, v26.4s, v0.4s\n" "srshl v27.4s, v27.4s, v0.4s\n" - "add v31.4s, v31.4s, v4.4s\n" - "add v20.4s, v20.4s, v4.4s\n" - "add v21.4s, v21.4s, v4.4s\n" - "add v22.4s, v22.4s, v4.4s\n" - "add v16.4s, v16.4s, v4.4s\n" - "add v17.4s, v17.4s, v4.4s\n" - "add v18.4s, v18.4s, v4.4s\n" - "add v19.4s, v19.4s, v4.4s\n" - "add v23.4s, v23.4s, v4.4s\n" - "add v28.4s, v28.4s, v4.4s\n" - "add v29.4s, v29.4s, v4.4s\n" - "add v30.4s, v30.4s, v4.4s\n" - "add v24.4s, v24.4s, v4.4s\n" - "add v25.4s, v25.4s, v4.4s\n" - "add v26.4s, v26.4s, v4.4s\n" - "add v27.4s, v27.4s, v4.4s\n" - "smin v31.4s, v31.4s, v6.4s\n" - "smin v20.4s, v20.4s, v6.4s\n" - "smin v21.4s, v21.4s, v6.4s\n" - "smin v22.4s, v22.4s, v6.4s\n" - "smin v16.4s, v16.4s, v6.4s\n" - "smin v17.4s, v17.4s, v6.4s\n" - "smin v18.4s, v18.4s, v6.4s\n" - "smin v19.4s, v19.4s, v6.4s\n" - "smin v23.4s, v23.4s, v6.4s\n" - "smin v28.4s, v28.4s, v6.4s\n" - "smin v29.4s, v29.4s, v6.4s\n" - "smin v30.4s, v30.4s, v6.4s\n" - "smin v24.4s, v24.4s, v6.4s\n" - "smin v25.4s, v25.4s, v6.4s\n" - "smin v26.4s, v26.4s, v6.4s\n" - "smin v27.4s, v27.4s, v6.4s\n" - "smax v31.4s, v31.4s, v5.4s\n" - "smax v20.4s, v20.4s, v5.4s\n" - "smax v21.4s, v21.4s, v5.4s\n" - "smax v22.4s, v22.4s, v5.4s\n" - "smax v16.4s, v16.4s, v5.4s\n" - "smax v17.4s, v17.4s, v5.4s\n" - "smax v18.4s, v18.4s, v5.4s\n" - "smax v19.4s, v19.4s, v5.4s\n" - "smax v23.4s, v23.4s, v5.4s\n" - "smax v28.4s, v28.4s, v5.4s\n" - "smax v29.4s, v29.4s, v5.4s\n" - "smax v30.4s, v30.4s, v5.4s\n" - "smax v24.4s, v24.4s, v5.4s\n" - "smax v25.4s, v25.4s, v5.4s\n" - "smax v26.4s, v26.4s, v5.4s\n" - "smax v27.4s, v27.4s, v5.4s\n" + "add v31.4s, v31.4s, v3.4s\n" + "add v20.4s, v20.4s, v3.4s\n" + "add v21.4s, v21.4s, v3.4s\n" + "add v22.4s, v22.4s, v3.4s\n" + "add v16.4s, v16.4s, v3.4s\n" + "add v17.4s, v17.4s, v3.4s\n" + "add v18.4s, v18.4s, v3.4s\n" + "add v19.4s, v19.4s, v3.4s\n" + "add v23.4s, v23.4s, v3.4s\n" + "add v28.4s, v28.4s, v3.4s\n" + "add v29.4s, v29.4s, v3.4s\n" + "add v30.4s, v30.4s, v3.4s\n" + "add v24.4s, v24.4s, v3.4s\n" + "add v25.4s, v25.4s, v3.4s\n" + "add v26.4s, v26.4s, v3.4s\n" + "add v27.4s, v27.4s, v3.4s\n" + "smin v31.4s, v31.4s, v2.4s\n" + "smin v20.4s, v20.4s, v2.4s\n" + "smin v21.4s, v21.4s, v2.4s\n" + "smin v22.4s, v22.4s, v2.4s\n" + "smin v16.4s, v16.4s, v2.4s\n" + "smin v17.4s, v17.4s, v2.4s\n" + "smin v18.4s, v18.4s, v2.4s\n" + "smin v19.4s, v19.4s, v2.4s\n" + "smin v23.4s, v23.4s, v2.4s\n" + "smin v28.4s, v28.4s, v2.4s\n" + "smin v29.4s, v29.4s, v2.4s\n" + "smin v30.4s, v30.4s, v2.4s\n" + "smin v24.4s, v24.4s, v2.4s\n" + "smin v25.4s, v25.4s, v2.4s\n" + "smin v26.4s, v26.4s, v2.4s\n" + "smin v27.4s, v27.4s, v2.4s\n" + "smax v31.4s, v31.4s, v1.4s\n" + "smax v20.4s, v20.4s, v1.4s\n" + "smax v21.4s, v21.4s, v1.4s\n" + "smax v22.4s, v22.4s, v1.4s\n" + "smax v16.4s, v16.4s, v1.4s\n" + "smax v17.4s, v17.4s, v1.4s\n" + "smax v18.4s, v18.4s, v1.4s\n" + "smax v19.4s, v19.4s, v1.4s\n" + "smax v23.4s, v23.4s, v1.4s\n" + "smax v28.4s, v28.4s, v1.4s\n" + "smax v29.4s, v29.4s, v1.4s\n" + "smax v30.4s, v30.4s, v1.4s\n" + "smax v24.4s, v24.4s, v1.4s\n" + "smax v25.4s, v25.4s, v1.4s\n" + "smax v26.4s, v26.4s, v1.4s\n" + "smax v27.4s, v27.4s, v1.4s\n" "uzp1 v31.8h, v31.8h, v20.8h\n" "uzp1 v20.8h, v21.8h, v22.8h\n" "uzp1 v16.8h, v16.8h, v17.8h\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v19.8h, v18.8h, v19.8h\n" "uzp1 v23.8h, v23.8h, v28.8h\n" - "uzp1 v28.8h, v29.8h, v30.8h\n" + "uzp1 v18.8h, v29.8h, v30.8h\n" "uzp1 v24.8h, v24.8h, v25.8h\n" - "uzp1 v25.8h, v26.8h, v27.8h\n" + "uzp1 v17.8h, v26.8h, v27.8h\n" "uzp1 v31.16b, v31.16b, v20.16b\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" - "uzp1 v23.16b, v23.16b, v28.16b\n" - "uzp1 v24.16b, v24.16b, v25.16b\n" + "uzp1 v16.16b, v16.16b, v19.16b\n" + "uzp1 v23.16b, v23.16b, v18.16b\n" + "uzp1 v24.16b, v24.16b, v17.16b\n" "bge 127f\n" "tbz x9, #3, 122f\n" "str d31, [x27], #0x8\n" - "str d16, [x22], #0x8\n" - "str d23, [x21], #0x8\n" - "str d24, [x20], #0x8\n" + "str d16, [x23], #0x8\n" + "str d23, [x22], #0x8\n" + "str d24, [x21], #0x8\n" "tbz x9, #2, 120f\n" "st1 { v31.s }[2], [x27], #0x4\n" - "st1 { v16.s }[2], [x22], #0x4\n" - "st1 { v23.s }[2], [x21], #0x4\n" - "st1 { v24.s }[2], [x20], #0x4\n" + "st1 { v16.s }[2], [x23], #0x4\n" + "st1 { v23.s }[2], [x22], #0x4\n" + "st1 { v24.s }[2], [x21], #0x4\n" "tbz x9, #1, 119f\n" "st1 { v31.h }[6], [x27], #0x2\n" - "st1 { v16.h }[6], [x22], #0x2\n" - "st1 { v23.h }[6], [x21], #0x2\n" - "st1 { v24.h }[6], [x20], #0x2\n" + "st1 { v16.h }[6], [x23], #0x2\n" + "st1 { v23.h }[6], [x22], #0x2\n" + "st1 { v24.h }[6], [x21], #0x2\n" "tbz x9, #0, 126f\n" "st1 { v31.b }[14], [x27]\n" - "st1 { v16.b }[14], [x22]\n" - "st1 { v23.b }[14], [x21]\n" - "st1 { v24.b }[14], [x20]\n" + "st1 { v16.b }[14], [x23]\n" + "st1 { v23.b }[14], [x22]\n" + "st1 { v24.b }[14], [x21]\n" "b 126f\n" "119:" // Height 4: Partial direct writeback: partial_1_12 "tbz x9, #0, 126f\n" "st1 { v31.b }[12], [x27]\n" - "st1 { v16.b }[12], [x22]\n" - "st1 { v23.b }[12], [x21]\n" - "st1 { v24.b }[12], [x20]\n" + "st1 { v16.b }[12], [x23]\n" + "st1 { v23.b }[12], [x22]\n" + "st1 { v24.b }[12], [x21]\n" "b 126f\n" "120:" // Height 4: Partial direct writeback: partial_2_8 "tbz x9, #1, 121f\n" "st1 { v31.h }[4], [x27], #0x2\n" - "st1 { v16.h }[4], [x22], #0x2\n" - "st1 { v23.h }[4], [x21], #0x2\n" - "st1 { v24.h }[4], [x20], #0x2\n" + "st1 { v16.h }[4], [x23], #0x2\n" + "st1 { v23.h }[4], [x22], #0x2\n" + "st1 { v24.h }[4], [x21], #0x2\n" "tbz x9, #0, 126f\n" "st1 { v31.b }[10], [x27]\n" - "st1 { v16.b }[10], [x22]\n" - "st1 { v23.b }[10], [x21]\n" - "st1 { v24.b }[10], [x20]\n" + "st1 { v16.b }[10], [x23]\n" + "st1 { v23.b }[10], [x22]\n" + "st1 { v24.b }[10], [x21]\n" "b 126f\n" "121:" // Height 4: Partial direct writeback: partial_1_8 "tbz x9, #0, 126f\n" "st1 { v31.b }[8], [x27]\n" - "st1 { v16.b }[8], [x22]\n" - "st1 { v23.b }[8], [x21]\n" - "st1 { v24.b }[8], [x20]\n" + "st1 { v16.b }[8], [x23]\n" + "st1 { v23.b }[8], [x22]\n" + "st1 { v24.b }[8], [x21]\n" "b 126f\n" "122:" // Height 4: Partial direct writeback: partial_4_0 "tbz x9, #2, 124f\n" "str s31, [x27], #0x4\n" - "str s16, [x22], #0x4\n" - "str s23, [x21], #0x4\n" - "str s24, [x20], #0x4\n" + "str s16, [x23], #0x4\n" + "str s23, [x22], #0x4\n" + "str s24, [x21], #0x4\n" "tbz x9, #1, 123f\n" "st1 { v31.h }[2], [x27], #0x2\n" - "st1 { v16.h }[2], [x22], #0x2\n" - "st1 { v23.h }[2], [x21], #0x2\n" - "st1 { v24.h }[2], [x20], #0x2\n" + "st1 { v16.h }[2], [x23], #0x2\n" + "st1 { v23.h }[2], [x22], #0x2\n" + "st1 { v24.h }[2], [x21], #0x2\n" "tbz x9, #0, 126f\n" "st1 { v31.b }[6], [x27]\n" - "st1 { v16.b }[6], [x22]\n" - "st1 { v23.b }[6], [x21]\n" - "st1 { v24.b }[6], [x20]\n" + "st1 { v16.b }[6], [x23]\n" + "st1 { v23.b }[6], [x22]\n" + "st1 { v24.b }[6], [x21]\n" "b 126f\n" "123:" // Height 4: Partial direct writeback: partial_1_4 "tbz x9, #0, 126f\n" "st1 { v31.b }[4], [x27]\n" - "st1 { v16.b }[4], [x22]\n" - "st1 { v23.b }[4], [x21]\n" - "st1 { v24.b }[4], [x20]\n" + "st1 { v16.b }[4], [x23]\n" + "st1 { v23.b }[4], [x22]\n" + "st1 { v24.b }[4], [x21]\n" "b 126f\n" "124:" // Height 4: Partial direct writeback: partial_2_0 "tbz x9, #1, 125f\n" "str h31, [x27], #0x2\n" - "str h16, [x22], #0x2\n" - "str h23, [x21], #0x2\n" - "str h24, [x20], #0x2\n" + "str h16, [x23], #0x2\n" + "str h23, [x22], #0x2\n" + "str h24, [x21], #0x2\n" "tbz x9, #0, 126f\n" "st1 { v31.b }[2], [x27]\n" - "st1 { v16.b }[2], [x22]\n" - "st1 { v23.b }[2], [x21]\n" - "st1 { v24.b }[2], [x20]\n" + "st1 { v16.b }[2], [x23]\n" + "st1 { v23.b }[2], [x22]\n" + "st1 { v24.b }[2], [x21]\n" "b 126f\n" "125:" // Height 4: Partial direct writeback: partial_1_0 "str b31, [x27, #0x0]\n" - "str b16, [x22, #0x0]\n" - "str b23, [x21, #0x0]\n" - "str b24, [x20, #0x0]\n" + "str b16, [x23, #0x0]\n" + "str b23, [x22, #0x0]\n" + "str b24, [x21, #0x0]\n" "126:" // Height 4: Partial direct writeback: Done "b 128f\n" "127:" // Height 4: Full writeback "str q31, [x27, #0x0]\n" "add x27, x27, #0x10\n" - "str q16, [x22, #0x0]\n" - "str q23, [x21, #0x0]\n" - "str q24, [x20, #0x0]\n" + "str q16, [x23, #0x0]\n" + "str q23, [x22, #0x0]\n" + "str q24, [x21, #0x0]\n" "128:" // Height 4: Writeback done "subs x9, x9, #0x10\n" "bgt 98b\n" @@ -2089,7 +2088,6 @@ void a64_hybrid_u8qa_mmla_4x16 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "130:" // Exit - : [M] "+&r" (M), [flags] "+&r" (flags), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16.hpp index ce96c1b28f1eeb00b8429be9f7d517e07074571c..38bb7c646dc81e9a88e7de216577b9c4fb6e5e4f 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef __aarch64__ + #include "../std_transforms_fixed.hpp" #include "../performance_parameters.hpp" @@ -121,5 +121,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16/a55.cpp index 705f6525b6df60e1e0b7b3877d50ad20ca1f900a..7f0fad7fa70a423e05c1f60a4c7cd14ad99b19b2 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16/a55.cpp @@ -77,7 +77,6 @@ void a64_hybrid_u8u32_dot_6x16_a55 ( ka.N = N; ka.B_ptr = B_ptr; __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x6\n" "bge 171f\n" @@ -165,11 +164,11 @@ void a64_hybrid_u8u32_dot_6x16_a55 ( "14:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w14, [x20, x15, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 15f\n" - "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x13, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x15, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x13, [x20, #0x0]\n" "cbnz x15, 16f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x13, x13, x20\n" @@ -186,129 +185,129 @@ void a64_hybrid_u8u32_dot_6x16_a55 ( "blt 18f\n" "17:" // Height 1: Multiply loop: Main loop head ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - "ldr d6, [x16, #0x20]\n" - "ldr x12, [x16, #0x28]\n" + "ldr d17, [x16, #0x20]\n" + "ldr x20, [x16, #0x28]\n" ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - "ldr d7, [x16, #0x30]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x16, #0x38]\n" - "mov v7.d[1], x11\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - "ldr d6, [x16, #0x40]\n" - "ldr x12, [x16, #0x48]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - "ldr d7, [x16, #0x50]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x16, #0x58]\n" - "mov v7.d[1], x11\n" - ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n" - "ldr d6, [x16, #0x60]\n" - "ldr x12, [x16, #0x68]\n" - ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n" - "ldr d7, [x16, #0x70]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x16, #0x78]\n" - "mov v7.d[1], x11\n" - ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n" - "ldr d6, [x16, #0x80]\n" - "ldr x12, [x16, #0x88]\n" - ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n" - "ldr d7, [x16, #0x90]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x16, #0x98]\n" - "mov v7.d[1], x11\n" - ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n" - "ldr d6, [x16, #0xa0]\n" - "ldr x12, [x16, #0xa8]\n" - ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n" - "ldr d7, [x16, #0xb0]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x16, #0xb8]\n" - "mov v7.d[1], x11\n" - ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n" - "ldr d6, [x16, #0xc0]\n" - "ldr x12, [x16, #0xc8]\n" - ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n" - "ldr d7, [x16, #0xd0]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x16, #0xd8]\n" - "mov v7.d[1], x11\n" - ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n" - "ldr d6, [x16, #0xe0]\n" - "ldr x12, [x16, #0xe8]\n" - ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n" - "ldr d7, [x16, #0xf0]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x16, #0xf8]\n" - "mov v7.d[1], x11\n" + "ldr d16, [x16, #0x30]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x16, #0x38]\n" + "mov v16.d[1], x20\n" + ".inst 0x6f80e22a // udot v10.4s, v17.16b, v0.4b[0]\n" + "ldr d17, [x16, #0x40]\n" + "ldr x20, [x16, #0x48]\n" + ".inst 0x6f80e20b // udot v11.4s, v16.16b, v0.4b[0]\n" + "ldr d16, [x16, #0x50]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x16, #0x58]\n" + "mov v16.d[1], x20\n" + ".inst 0x6fa0e228 // udot v8.4s, v17.16b, v0.4b[1]\n" + "ldr d17, [x16, #0x60]\n" + "ldr x20, [x16, #0x68]\n" + ".inst 0x6fa0e209 // udot v9.4s, v16.16b, v0.4b[1]\n" + "ldr d16, [x16, #0x70]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x16, #0x78]\n" + "mov v16.d[1], x20\n" + ".inst 0x6fa0e22a // udot v10.4s, v17.16b, v0.4b[1]\n" + "ldr d17, [x16, #0x80]\n" + "ldr x20, [x16, #0x88]\n" + ".inst 0x6fa0e20b // udot v11.4s, v16.16b, v0.4b[1]\n" + "ldr d16, [x16, #0x90]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x16, #0x98]\n" + "mov v16.d[1], x20\n" + ".inst 0x6f80ea28 // udot v8.4s, v17.16b, v0.4b[2]\n" + "ldr d17, [x16, #0xa0]\n" + "ldr x20, [x16, #0xa8]\n" + ".inst 0x6f80ea09 // udot v9.4s, v16.16b, v0.4b[2]\n" + "ldr d16, [x16, #0xb0]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x16, #0xb8]\n" + "mov v16.d[1], x20\n" + ".inst 0x6f80ea2a // udot v10.4s, v17.16b, v0.4b[2]\n" + "ldr d17, [x16, #0xc0]\n" + "ldr x20, [x16, #0xc8]\n" + ".inst 0x6f80ea0b // udot v11.4s, v16.16b, v0.4b[2]\n" + "ldr d16, [x16, #0xd0]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x16, #0xd8]\n" + "mov v16.d[1], x20\n" + ".inst 0x6fa0ea28 // udot v8.4s, v17.16b, v0.4b[3]\n" + "ldr d17, [x16, #0xe0]\n" + "ldr x20, [x16, #0xe8]\n" + ".inst 0x6fa0ea09 // udot v9.4s, v16.16b, v0.4b[3]\n" + "ldr d16, [x16, #0xf0]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x16, #0xf8]\n" + "mov v16.d[1], x20\n" "add x13, x13, #0x10\n" "add x16, x16, #0x100\n" - ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n" + ".inst 0x6fa0ea2a // udot v10.4s, v17.16b, v0.4b[3]\n" "ldr d6, [x16, #0x0]\n" - "ldr x12, [x16, #0x8]\n" - ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n" + "ldr x20, [x16, #0x8]\n" + ".inst 0x6fa0ea0b // udot v11.4s, v16.16b, v0.4b[3]\n" "ldr d0, [x13, #0x0]\n" "sub x14, x14, #0x10\n" "ldr d7, [x16, #0x10]\n" "cmp x14, #0x20\n" - "ldr x10, [x13, #0x8]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x16, #0x18]\n" - "mov v0.d[1], x10\n" - "mov v7.d[1], x11\n" + "ldr x21, [x13, #0x8]\n" + "mov v6.d[1], x20\n" + "ldr x20, [x16, #0x18]\n" + "mov v0.d[1], x21\n" + "mov v7.d[1], x20\n" "prfm pldl1keep, [x13, #0x80]\n" "bge 17b\n" "18:" // Height 1: Multiply loop: Single iteration only ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q6, [x16, #0x20]\n" + "ldr q17, [x16, #0x20]\n" ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - "ldr q6, [x16, #0x40]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x50]\n" - ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n" - "ldr q6, [x16, #0x60]\n" - ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n" - "ldr q7, [x16, #0x70]\n" - ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n" - "ldr q6, [x16, #0x80]\n" - ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n" - "ldr q7, [x16, #0x90]\n" - ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n" - "ldr q6, [x16, #0xa0]\n" - ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n" - "ldr q7, [x16, #0xb0]\n" - ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n" - "ldr q6, [x16, #0xc0]\n" - ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n" - "ldr q7, [x16, #0xd0]\n" - ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n" - "ldr q6, [x16, #0xe0]\n" - ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n" - "ldr q7, [x16, #0xf0]\n" + "ldr q16, [x16, #0x30]\n" + ".inst 0x6f80e22a // udot v10.4s, v17.16b, v0.4b[0]\n" + "ldr q17, [x16, #0x40]\n" + ".inst 0x6f80e20b // udot v11.4s, v16.16b, v0.4b[0]\n" + "ldr q16, [x16, #0x50]\n" + ".inst 0x6fa0e228 // udot v8.4s, v17.16b, v0.4b[1]\n" + "ldr q17, [x16, #0x60]\n" + ".inst 0x6fa0e209 // udot v9.4s, v16.16b, v0.4b[1]\n" + "ldr q16, [x16, #0x70]\n" + ".inst 0x6fa0e22a // udot v10.4s, v17.16b, v0.4b[1]\n" + "ldr q17, [x16, #0x80]\n" + ".inst 0x6fa0e20b // udot v11.4s, v16.16b, v0.4b[1]\n" + "ldr q16, [x16, #0x90]\n" + ".inst 0x6f80ea28 // udot v8.4s, v17.16b, v0.4b[2]\n" + "ldr q17, [x16, #0xa0]\n" + ".inst 0x6f80ea09 // udot v9.4s, v16.16b, v0.4b[2]\n" + "ldr q16, [x16, #0xb0]\n" + ".inst 0x6f80ea2a // udot v10.4s, v17.16b, v0.4b[2]\n" + "ldr q17, [x16, #0xc0]\n" + ".inst 0x6f80ea0b // udot v11.4s, v16.16b, v0.4b[2]\n" + "ldr q16, [x16, #0xd0]\n" + ".inst 0x6fa0ea28 // udot v8.4s, v17.16b, v0.4b[3]\n" + "ldr q17, [x16, #0xe0]\n" + ".inst 0x6fa0ea09 // udot v9.4s, v16.16b, v0.4b[3]\n" + "ldr q16, [x16, #0xf0]\n" "add x13, x13, #0x10\n" "sub x14, x14, #0x10\n" - ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n" + ".inst 0x6fa0ea2a // udot v10.4s, v17.16b, v0.4b[3]\n" "prfm pldl1keep, [x13, #0x80]\n" - ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x6fa0ea0b // udot v11.4s, v16.16b, v0.4b[3]\n" "add x16, x16, #0x100\n" "19:" // Height 1: Multiply loop: Main loop skip "cbz x14, 24f\n" "cmp x14, #0x4\n" "blt 21f\n" "20:" // Height 1: Multiply loop: Odd block loop - "ldr s0, [x13], #0x4\n" + "ldr s18, [x13], #0x4\n" "sub x14, x14, #0x4\n" - "ldr q6, [x16, #0x0]\n" - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x10]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - "ldr q6, [x16, #0x20]\n" + "ldr q16, [x16, #0x0]\n" + ".inst 0x6f92e208 // udot v8.4s, v16.16b, v18.4b[0]\n" + "ldr q16, [x16, #0x10]\n" + ".inst 0x6f92e209 // udot v9.4s, v16.16b, v18.4b[0]\n" + "ldr q17, [x16, #0x20]\n" "cmp x14, #0x4\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" + "ldr q16, [x16, #0x30]\n" + ".inst 0x6f92e22a // udot v10.4s, v17.16b, v18.4b[0]\n" + ".inst 0x6f92e20b // udot v11.4s, v16.16b, v18.4b[0]\n" "add x16, x16, #0x40\n" "bge 20b\n" "21:" // Height 1: Multiply loop: Skip odd blocks @@ -321,14 +320,14 @@ void a64_hybrid_u8u32_dot_6x16_a55 ( "22:" // Height 1: Multiply loop: Ragged operand read: partial_1_0 "ldr b0, [x13, #0x0]\n" "23:" // Height 1: Multiply loop: Ragged operand read: Done - "ldr q6, [x16, #0x0]\n" - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x10]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - "ldr q6, [x16, #0x20]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" + "ldr q16, [x16, #0x0]\n" + ".inst 0x6f80e208 // udot v8.4s, v16.16b, v0.4b[0]\n" + "ldr q16, [x16, #0x10]\n" + ".inst 0x6f80e209 // udot v9.4s, v16.16b, v0.4b[0]\n" + "ldr q16, [x16, #0x20]\n" + ".inst 0x6f80e20a // udot v10.4s, v16.16b, v0.4b[0]\n" + "ldr q16, [x16, #0x30]\n" + ".inst 0x6f80e20b // udot v11.4s, v16.16b, v0.4b[0]\n" "add x16, x16, #0x40\n" "24:" // Height 1: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -499,226 +498,226 @@ void a64_hybrid_u8u32_dot_6x16_a55 ( "48:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w14, [x20, x15, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 49f\n" - "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x13, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x15, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x13, [x20, #0x0]\n" + "ldr x12, [x20, #0x8]\n" "cbnz x15, 50f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x13, x13, x20\n" - "add x9, x9, x20\n" + "add x12, x12, x20\n" "b 50f\n" "49:" // Height 2: setup direct input "mov x13, %x[input_ptr]\n" - "add x9, x13, x20\n" + "add x12, x13, x21\n" "50:" // Height 2: input setup done "cmp x14, #0x10\n" "blt 53f\n" "ldr q0, [x13, #0x0]\n" "cmp x14, #0x20\n" - "ldr q1, [x9, #0x0]\n" + "ldr q1, [x12, #0x0]\n" "ldr q6, [x16, #0x0]\n" "ldr q7, [x16, #0x10]\n" "blt 52f\n" "51:" // Height 2: Multiply loop: Main loop head ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - "ldr x12, [x16, #0x28]\n" + "ldr x21, [x16, #0x28]\n" ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - "ldr d6, [x16, #0x20]\n" + "ldr d17, [x16, #0x20]\n" ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - "ldr x11, [x16, #0x38]\n" + "ldr x20, [x16, #0x38]\n" ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - "ldr d7, [x16, #0x30]\n" - "mov v6.d[1], x12\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - "mov v7.d[1], x11\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - "ldr d6, [x16, #0x40]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - "ldr x12, [x16, #0x48]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - "ldr d7, [x16, #0x50]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x16, #0x58]\n" - "mov v7.d[1], x11\n" - ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n" - "ldr x12, [x16, #0x68]\n" - ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n" - "ldr d6, [x16, #0x60]\n" - ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n" - "ldr x11, [x16, #0x78]\n" - ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n" - "ldr d7, [x16, #0x70]\n" - "mov v6.d[1], x12\n" - ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n" - "mov v7.d[1], x11\n" - ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n" - "ldr d6, [x16, #0x80]\n" - ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n" - "ldr x12, [x16, #0x88]\n" - ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n" - "ldr d7, [x16, #0x90]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x16, #0x98]\n" - "mov v7.d[1], x11\n" - ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n" - "ldr x12, [x16, #0xa8]\n" - ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n" - "ldr d6, [x16, #0xa0]\n" - ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n" - "ldr x11, [x16, #0xb8]\n" - ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n" - "ldr d7, [x16, #0xb0]\n" - "mov v6.d[1], x12\n" - ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n" - "mov v7.d[1], x11\n" - ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n" - "ldr d6, [x16, #0xc0]\n" - ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n" - "ldr x12, [x16, #0xc8]\n" - ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n" - "ldr d7, [x16, #0xd0]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x16, #0xd8]\n" - "mov v7.d[1], x11\n" - ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n" - "ldr x12, [x16, #0xe8]\n" - ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n" - "ldr d6, [x16, #0xe0]\n" - ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n" - "ldr x11, [x16, #0xf8]\n" - ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n" - "ldr d7, [x16, #0xf0]\n" - "mov v6.d[1], x12\n" + "ldr d16, [x16, #0x30]\n" + "mov v17.d[1], x21\n" + ".inst 0x6f80e22a // udot v10.4s, v17.16b, v0.4b[0]\n" + "mov v16.d[1], x20\n" + ".inst 0x6f81e22e // udot v14.4s, v17.16b, v1.4b[0]\n" + "ldr d17, [x16, #0x40]\n" + ".inst 0x6f80e20b // udot v11.4s, v16.16b, v0.4b[0]\n" + "ldr x20, [x16, #0x48]\n" + ".inst 0x6f81e20f // udot v15.4s, v16.16b, v1.4b[0]\n" + "ldr d16, [x16, #0x50]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x16, #0x58]\n" + "mov v16.d[1], x20\n" + ".inst 0x6fa0e228 // udot v8.4s, v17.16b, v0.4b[1]\n" + "ldr x21, [x16, #0x68]\n" + ".inst 0x6fa1e22c // udot v12.4s, v17.16b, v1.4b[1]\n" + "ldr d17, [x16, #0x60]\n" + ".inst 0x6fa0e209 // udot v9.4s, v16.16b, v0.4b[1]\n" + "ldr x20, [x16, #0x78]\n" + ".inst 0x6fa1e20d // udot v13.4s, v16.16b, v1.4b[1]\n" + "ldr d16, [x16, #0x70]\n" + "mov v17.d[1], x21\n" + ".inst 0x6fa0e22a // udot v10.4s, v17.16b, v0.4b[1]\n" + "mov v16.d[1], x20\n" + ".inst 0x6fa1e22e // udot v14.4s, v17.16b, v1.4b[1]\n" + "ldr d17, [x16, #0x80]\n" + ".inst 0x6fa0e20b // udot v11.4s, v16.16b, v0.4b[1]\n" + "ldr x20, [x16, #0x88]\n" + ".inst 0x6fa1e20f // udot v15.4s, v16.16b, v1.4b[1]\n" + "ldr d16, [x16, #0x90]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x16, #0x98]\n" + "mov v16.d[1], x20\n" + ".inst 0x6f80ea28 // udot v8.4s, v17.16b, v0.4b[2]\n" + "ldr x21, [x16, #0xa8]\n" + ".inst 0x6f81ea2c // udot v12.4s, v17.16b, v1.4b[2]\n" + "ldr d17, [x16, #0xa0]\n" + ".inst 0x6f80ea09 // udot v9.4s, v16.16b, v0.4b[2]\n" + "ldr x20, [x16, #0xb8]\n" + ".inst 0x6f81ea0d // udot v13.4s, v16.16b, v1.4b[2]\n" + "ldr d16, [x16, #0xb0]\n" + "mov v17.d[1], x21\n" + ".inst 0x6f80ea2a // udot v10.4s, v17.16b, v0.4b[2]\n" + "mov v16.d[1], x20\n" + ".inst 0x6f81ea2e // udot v14.4s, v17.16b, v1.4b[2]\n" + "ldr d17, [x16, #0xc0]\n" + ".inst 0x6f80ea0b // udot v11.4s, v16.16b, v0.4b[2]\n" + "ldr x20, [x16, #0xc8]\n" + ".inst 0x6f81ea0f // udot v15.4s, v16.16b, v1.4b[2]\n" + "ldr d16, [x16, #0xd0]\n" + "mov v17.d[1], x20\n" + "ldr x20, [x16, #0xd8]\n" + "mov v16.d[1], x20\n" + ".inst 0x6fa0ea28 // udot v8.4s, v17.16b, v0.4b[3]\n" + "ldr x21, [x16, #0xe8]\n" + ".inst 0x6fa1ea2c // udot v12.4s, v17.16b, v1.4b[3]\n" + "ldr d17, [x16, #0xe0]\n" + ".inst 0x6fa0ea09 // udot v9.4s, v16.16b, v0.4b[3]\n" + "ldr x20, [x16, #0xf8]\n" + ".inst 0x6fa1ea0d // udot v13.4s, v16.16b, v1.4b[3]\n" + "ldr d16, [x16, #0xf0]\n" + "mov v17.d[1], x21\n" "add x13, x13, #0x10\n" - "mov v7.d[1], x11\n" - "add x9, x9, #0x10\n" + "mov v16.d[1], x20\n" + "add x12, x12, #0x10\n" "add x16, x16, #0x100\n" - ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n" + ".inst 0x6fa0ea2a // udot v10.4s, v17.16b, v0.4b[3]\n" + ".inst 0x6fa1ea2e // udot v14.4s, v17.16b, v1.4b[3]\n" "ldr d6, [x16, #0x0]\n" - "ldr x12, [x16, #0x8]\n" - ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n" + "ldr x21, [x16, #0x8]\n" + ".inst 0x6fa0ea0b // udot v11.4s, v16.16b, v0.4b[3]\n" "ldr d0, [x13, #0x0]\n" - ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n" - "ldr d1, [x9, #0x0]\n" + ".inst 0x6fa1ea0f // udot v15.4s, v16.16b, v1.4b[3]\n" + "ldr d1, [x12, #0x0]\n" "sub x14, x14, #0x10\n" "ldr d7, [x16, #0x10]\n" "cmp x14, #0x20\n" - "ldr x10, [x13, #0x8]\n" - "mov v6.d[1], x12\n" - "ldr x28, [x9, #0x8]\n" - "mov v0.d[1], x10\n" - "ldr x11, [x16, #0x18]\n" - "mov v1.d[1], x28\n" + "ldr x20, [x13, #0x8]\n" + "mov v6.d[1], x21\n" + "ldr x21, [x12, #0x8]\n" + "mov v0.d[1], x20\n" + "ldr x20, [x16, #0x18]\n" + "mov v1.d[1], x21\n" "prfm pldl1keep, [x13, #0x80]\n" - "mov v7.d[1], x11\n" - "prfm pldl1keep, [x9, #0x80]\n" + "mov v7.d[1], x20\n" + "prfm pldl1keep, [x12, #0x80]\n" "bge 51b\n" "52:" // Height 2: Multiply loop: Single iteration only ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" "add x13, x13, #0x10\n" ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x16, #0x20]\n" + "ldr q17, [x16, #0x20]\n" ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q16, [x16, #0x30]\n" + ".inst 0x6f80e22a // udot v10.4s, v17.16b, v0.4b[0]\n" "sub x14, x14, #0x10\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x16, #0x40]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" + ".inst 0x6f81e22e // udot v14.4s, v17.16b, v1.4b[0]\n" + "ldr q17, [x16, #0x40]\n" + ".inst 0x6f80e20b // udot v11.4s, v16.16b, v0.4b[0]\n" "prfm pldl1keep, [x13, #0x80]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x16, #0x50]\n" - ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n" - "prfm pldl1keep, [x9, #0x80]\n" - ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n" - "ldr q6, [x16, #0x60]\n" - ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n" - "ldr q7, [x16, #0x70]\n" - ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n" - "ldr q6, [x16, #0x80]\n" - ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n" - "ldr q7, [x16, #0x90]\n" - ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n" - "ldr q6, [x16, #0xa0]\n" - ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n" - "ldr q7, [x16, #0xb0]\n" - ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n" - "ldr q6, [x16, #0xc0]\n" - ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n" - "ldr q7, [x16, #0xd0]\n" - ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n" - "ldr q6, [x16, #0xe0]\n" - ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n" - "ldr q7, [x16, #0xf0]\n" - ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n" + ".inst 0x6f81e20f // udot v15.4s, v16.16b, v1.4b[0]\n" + "ldr q16, [x16, #0x50]\n" + ".inst 0x6fa0e228 // udot v8.4s, v17.16b, v0.4b[1]\n" + "prfm pldl1keep, [x12, #0x80]\n" + ".inst 0x6fa1e22c // udot v12.4s, v17.16b, v1.4b[1]\n" + "ldr q17, [x16, #0x60]\n" + ".inst 0x6fa0e209 // udot v9.4s, v16.16b, v0.4b[1]\n" + ".inst 0x6fa1e20d // udot v13.4s, v16.16b, v1.4b[1]\n" + "ldr q16, [x16, #0x70]\n" + ".inst 0x6fa0e22a // udot v10.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa1e22e // udot v14.4s, v17.16b, v1.4b[1]\n" + "ldr q17, [x16, #0x80]\n" + ".inst 0x6fa0e20b // udot v11.4s, v16.16b, v0.4b[1]\n" + ".inst 0x6fa1e20f // udot v15.4s, v16.16b, v1.4b[1]\n" + "ldr q16, [x16, #0x90]\n" + ".inst 0x6f80ea28 // udot v8.4s, v17.16b, v0.4b[2]\n" + ".inst 0x6f81ea2c // udot v12.4s, v17.16b, v1.4b[2]\n" + "ldr q17, [x16, #0xa0]\n" + ".inst 0x6f80ea09 // udot v9.4s, v16.16b, v0.4b[2]\n" + ".inst 0x6f81ea0d // udot v13.4s, v16.16b, v1.4b[2]\n" + "ldr q16, [x16, #0xb0]\n" + ".inst 0x6f80ea2a // udot v10.4s, v17.16b, v0.4b[2]\n" + ".inst 0x6f81ea2e // udot v14.4s, v17.16b, v1.4b[2]\n" + "ldr q17, [x16, #0xc0]\n" + ".inst 0x6f80ea0b // udot v11.4s, v16.16b, v0.4b[2]\n" + ".inst 0x6f81ea0f // udot v15.4s, v16.16b, v1.4b[2]\n" + "ldr q16, [x16, #0xd0]\n" + ".inst 0x6fa0ea28 // udot v8.4s, v17.16b, v0.4b[3]\n" + ".inst 0x6fa1ea2c // udot v12.4s, v17.16b, v1.4b[3]\n" + "ldr q17, [x16, #0xe0]\n" + ".inst 0x6fa0ea09 // udot v9.4s, v16.16b, v0.4b[3]\n" + ".inst 0x6fa1ea0d // udot v13.4s, v16.16b, v1.4b[3]\n" + "ldr q16, [x16, #0xf0]\n" + ".inst 0x6fa0ea2a // udot v10.4s, v17.16b, v0.4b[3]\n" "add x16, x16, #0x100\n" - ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n" + ".inst 0x6fa1ea2e // udot v14.4s, v17.16b, v1.4b[3]\n" + ".inst 0x6fa0ea0b // udot v11.4s, v16.16b, v0.4b[3]\n" + ".inst 0x6fa1ea0f // udot v15.4s, v16.16b, v1.4b[3]\n" "53:" // Height 2: Multiply loop: Main loop skip "cbz x14, 58f\n" "cmp x14, #0x4\n" "blt 55f\n" "54:" // Height 2: Multiply loop: Odd block loop - "ldr s0, [x13], #0x4\n" + "ldr s19, [x13], #0x4\n" "sub x14, x14, #0x4\n" - "ldr s1, [x9], #0x4\n" + "ldr s18, [x12], #0x4\n" "cmp x14, #0x4\n" - "ldr q6, [x16, #0x0]\n" - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x10]\n" - ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x16, #0x20]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q17, [x16, #0x0]\n" + ".inst 0x6f93e228 // udot v8.4s, v17.16b, v19.4b[0]\n" + "ldr q16, [x16, #0x10]\n" + ".inst 0x6f92e22c // udot v12.4s, v17.16b, v18.4b[0]\n" + "ldr q17, [x16, #0x20]\n" + ".inst 0x6f93e209 // udot v9.4s, v16.16b, v19.4b[0]\n" + ".inst 0x6f92e20d // udot v13.4s, v16.16b, v18.4b[0]\n" + "ldr q16, [x16, #0x30]\n" + ".inst 0x6f93e22a // udot v10.4s, v17.16b, v19.4b[0]\n" "add x16, x16, #0x40\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" + ".inst 0x6f92e22e // udot v14.4s, v17.16b, v18.4b[0]\n" + ".inst 0x6f93e20b // udot v11.4s, v16.16b, v19.4b[0]\n" + ".inst 0x6f92e20f // udot v15.4s, v16.16b, v18.4b[0]\n" "bge 54b\n" "55:" // Height 2: Multiply loop: Skip odd blocks "cbz x14, 58f\n" "tbz x14, #1, 56f\n" "ldr h0, [x13], #0x2\n" - "ldr h1, [x9], #0x2\n" + "ldr h1, [x12], #0x2\n" "tbz x14, #0, 57f\n" "ld1 { v0.b }[2], [x13]\n" - "ld1 { v1.b }[2], [x9]\n" + "ld1 { v1.b }[2], [x12]\n" "b 57f\n" "56:" // Height 2: Multiply loop: Ragged operand read: partial_1_0 "ldr b0, [x13, #0x0]\n" - "ldr b1, [x9, #0x0]\n" + "ldr b1, [x12, #0x0]\n" "57:" // Height 2: Multiply loop: Ragged operand read: Done - "ldr q6, [x16, #0x0]\n" - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x10]\n" - ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x16, #0x20]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q17, [x16, #0x0]\n" + ".inst 0x6f80e228 // udot v8.4s, v17.16b, v0.4b[0]\n" + "ldr q16, [x16, #0x10]\n" + ".inst 0x6f81e22c // udot v12.4s, v17.16b, v1.4b[0]\n" + "ldr q17, [x16, #0x20]\n" + ".inst 0x6f80e209 // udot v9.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f81e20d // udot v13.4s, v16.16b, v1.4b[0]\n" + "ldr q16, [x16, #0x30]\n" + ".inst 0x6f80e22a // udot v10.4s, v17.16b, v0.4b[0]\n" "add x16, x16, #0x40\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" + ".inst 0x6f81e22e // udot v14.4s, v17.16b, v1.4b[0]\n" + ".inst 0x6f80e20b // udot v11.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f81e20f // udot v15.4s, v16.16b, v1.4b[0]\n" "58:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x15, x15, #0x1\n" @@ -936,281 +935,281 @@ void a64_hybrid_u8u32_dot_6x16_a55 ( "82:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w14, [x20, x15, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 83f\n" - "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x13, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" - "ldr x27, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x15, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x13, [x20, #0x0]\n" + "ldr x12, [x20, #0x8]\n" + "ldr x11, [x20, #0x10]\n" "cbnz x15, 84f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x13, x13, x20\n" - "add x9, x9, x20\n" - "add x27, x27, x20\n" + "add x12, x12, x20\n" + "add x11, x11, x20\n" "b 84f\n" "83:" // Height 3: setup direct input "mov x13, %x[input_ptr]\n" - "add x9, x13, x20\n" - "add x27, x9, x20\n" + "add x12, x13, x21\n" + "add x11, x12, x21\n" "84:" // Height 3: input setup done "cmp x14, #0x10\n" "blt 87f\n" "ldr q0, [x13, #0x0]\n" "cmp x14, #0x20\n" - "ldr q1, [x9, #0x0]\n" - "ldr q2, [x27, #0x0]\n" + "ldr q1, [x12, #0x0]\n" + "ldr q2, [x11, #0x0]\n" "ldr q6, [x16, #0x0]\n" "ldr q7, [x16, #0x10]\n" "blt 86f\n" "85:" // Height 3: Multiply loop: Main loop head ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - "ldr x12, [x16, #0x28]\n" + "ldr x21, [x16, #0x28]\n" ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - "ldr x11, [x16, #0x38]\n" + "ldr x20, [x16, #0x38]\n" ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - "ldr d6, [x16, #0x20]\n" + "ldr d21, [x16, #0x20]\n" ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x12\n" + "mov v21.d[1], x21\n" ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - "ldr x12, [x16, #0x48]\n" + "ldr x21, [x16, #0x48]\n" ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" - "ldr d7, [x16, #0x30]\n" - "mov v7.d[1], x11\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - "ldr x11, [x16, #0x58]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - "ldr d6, [x16, #0x40]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x12\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - "ldr x12, [x16, #0x68]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - "ldr d7, [x16, #0x50]\n" - "mov v7.d[1], x11\n" - ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n" - "ldr x11, [x16, #0x78]\n" - ".inst 0x6fa2e0d0 // udot v16.4s, v6.16b, v2.4b[1]\n" - "ldr d6, [x16, #0x60]\n" - ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n" - "mov v6.d[1], x12\n" - ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n" - "ldr x12, [x16, #0x88]\n" - ".inst 0x6fa2e0f1 // udot v17.4s, v7.16b, v2.4b[1]\n" - "ldr d7, [x16, #0x70]\n" - "mov v7.d[1], x11\n" - ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n" - "ldr x11, [x16, #0x98]\n" - ".inst 0x6fa2e0d2 // udot v18.4s, v6.16b, v2.4b[1]\n" - "ldr d6, [x16, #0x80]\n" - ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n" - "mov v6.d[1], x12\n" - ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n" - "ldr x12, [x16, #0xa8]\n" - ".inst 0x6fa2e0f3 // udot v19.4s, v7.16b, v2.4b[1]\n" - "ldr d7, [x16, #0x90]\n" - "mov v7.d[1], x11\n" - ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n" - "ldr x11, [x16, #0xb8]\n" - ".inst 0x6f82e8d0 // udot v16.4s, v6.16b, v2.4b[2]\n" - "ldr d6, [x16, #0xa0]\n" - ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n" - "mov v6.d[1], x12\n" - ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n" - "ldr x12, [x16, #0xc8]\n" - ".inst 0x6f82e8f1 // udot v17.4s, v7.16b, v2.4b[2]\n" - "ldr d7, [x16, #0xb0]\n" - "mov v7.d[1], x11\n" - ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n" - "ldr x11, [x16, #0xd8]\n" - ".inst 0x6f82e8d2 // udot v18.4s, v6.16b, v2.4b[2]\n" - "ldr d6, [x16, #0xc0]\n" - ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n" - "mov v6.d[1], x12\n" - ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n" - "ldr x12, [x16, #0xe8]\n" - ".inst 0x6f82e8f3 // udot v19.4s, v7.16b, v2.4b[2]\n" - "ldr d7, [x16, #0xd0]\n" - "mov v7.d[1], x11\n" - ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n" - "ldr x11, [x16, #0xf8]\n" - ".inst 0x6fa2e8d0 // udot v16.4s, v6.16b, v2.4b[3]\n" - "ldr d6, [x16, #0xe0]\n" - ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n" - "mov v6.d[1], x12\n" - ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n" + "ldr d20, [x16, #0x30]\n" + "mov v20.d[1], x20\n" + ".inst 0x6f80e2aa // udot v10.4s, v21.16b, v0.4b[0]\n" + ".inst 0x6f81e2ae // udot v14.4s, v21.16b, v1.4b[0]\n" + "ldr x20, [x16, #0x58]\n" + ".inst 0x6f82e2b2 // udot v18.4s, v21.16b, v2.4b[0]\n" + "ldr d21, [x16, #0x40]\n" + ".inst 0x6f80e28b // udot v11.4s, v20.16b, v0.4b[0]\n" + "mov v21.d[1], x21\n" + ".inst 0x6f81e28f // udot v15.4s, v20.16b, v1.4b[0]\n" + "ldr x21, [x16, #0x68]\n" + ".inst 0x6f82e293 // udot v19.4s, v20.16b, v2.4b[0]\n" + "ldr d20, [x16, #0x50]\n" + "mov v20.d[1], x20\n" + ".inst 0x6fa0e2a8 // udot v8.4s, v21.16b, v0.4b[1]\n" + ".inst 0x6fa1e2ac // udot v12.4s, v21.16b, v1.4b[1]\n" + "ldr x20, [x16, #0x78]\n" + ".inst 0x6fa2e2b0 // udot v16.4s, v21.16b, v2.4b[1]\n" + "ldr d21, [x16, #0x60]\n" + ".inst 0x6fa0e289 // udot v9.4s, v20.16b, v0.4b[1]\n" + "mov v21.d[1], x21\n" + ".inst 0x6fa1e28d // udot v13.4s, v20.16b, v1.4b[1]\n" + "ldr x21, [x16, #0x88]\n" + ".inst 0x6fa2e291 // udot v17.4s, v20.16b, v2.4b[1]\n" + "ldr d20, [x16, #0x70]\n" + "mov v20.d[1], x20\n" + ".inst 0x6fa0e2aa // udot v10.4s, v21.16b, v0.4b[1]\n" + ".inst 0x6fa1e2ae // udot v14.4s, v21.16b, v1.4b[1]\n" + "ldr x20, [x16, #0x98]\n" + ".inst 0x6fa2e2b2 // udot v18.4s, v21.16b, v2.4b[1]\n" + "ldr d21, [x16, #0x80]\n" + ".inst 0x6fa0e28b // udot v11.4s, v20.16b, v0.4b[1]\n" + "mov v21.d[1], x21\n" + ".inst 0x6fa1e28f // udot v15.4s, v20.16b, v1.4b[1]\n" + "ldr x21, [x16, #0xa8]\n" + ".inst 0x6fa2e293 // udot v19.4s, v20.16b, v2.4b[1]\n" + "ldr d20, [x16, #0x90]\n" + "mov v20.d[1], x20\n" + ".inst 0x6f80eaa8 // udot v8.4s, v21.16b, v0.4b[2]\n" + ".inst 0x6f81eaac // udot v12.4s, v21.16b, v1.4b[2]\n" + "ldr x20, [x16, #0xb8]\n" + ".inst 0x6f82eab0 // udot v16.4s, v21.16b, v2.4b[2]\n" + "ldr d21, [x16, #0xa0]\n" + ".inst 0x6f80ea89 // udot v9.4s, v20.16b, v0.4b[2]\n" + "mov v21.d[1], x21\n" + ".inst 0x6f81ea8d // udot v13.4s, v20.16b, v1.4b[2]\n" + "ldr x21, [x16, #0xc8]\n" + ".inst 0x6f82ea91 // udot v17.4s, v20.16b, v2.4b[2]\n" + "ldr d20, [x16, #0xb0]\n" + "mov v20.d[1], x20\n" + ".inst 0x6f80eaaa // udot v10.4s, v21.16b, v0.4b[2]\n" + ".inst 0x6f81eaae // udot v14.4s, v21.16b, v1.4b[2]\n" + "ldr x20, [x16, #0xd8]\n" + ".inst 0x6f82eab2 // udot v18.4s, v21.16b, v2.4b[2]\n" + "ldr d21, [x16, #0xc0]\n" + ".inst 0x6f80ea8b // udot v11.4s, v20.16b, v0.4b[2]\n" + "mov v21.d[1], x21\n" + ".inst 0x6f81ea8f // udot v15.4s, v20.16b, v1.4b[2]\n" + "ldr x21, [x16, #0xe8]\n" + ".inst 0x6f82ea93 // udot v19.4s, v20.16b, v2.4b[2]\n" + "ldr d20, [x16, #0xd0]\n" + "mov v20.d[1], x20\n" + ".inst 0x6fa0eaa8 // udot v8.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa1eaac // udot v12.4s, v21.16b, v1.4b[3]\n" + "ldr x20, [x16, #0xf8]\n" + ".inst 0x6fa2eab0 // udot v16.4s, v21.16b, v2.4b[3]\n" + "ldr d21, [x16, #0xe0]\n" + ".inst 0x6fa0ea89 // udot v9.4s, v20.16b, v0.4b[3]\n" + "mov v21.d[1], x21\n" + ".inst 0x6fa1ea8d // udot v13.4s, v20.16b, v1.4b[3]\n" "add x13, x13, #0x10\n" - ".inst 0x6fa2e8f1 // udot v17.4s, v7.16b, v2.4b[3]\n" - "ldr d7, [x16, #0xf0]\n" - "mov v7.d[1], x11\n" - "add x9, x9, #0x10\n" - "add x27, x27, #0x10\n" + ".inst 0x6fa2ea91 // udot v17.4s, v20.16b, v2.4b[3]\n" + "ldr d20, [x16, #0xf0]\n" + "mov v20.d[1], x20\n" + "add x12, x12, #0x10\n" + "add x11, x11, #0x10\n" "add x16, x16, #0x100\n" - ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n" - "ldr x12, [x16, #0x8]\n" - ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n" - "ldr x10, [x13, #0x8]\n" - ".inst 0x6fa2e8d2 // udot v18.4s, v6.16b, v2.4b[3]\n" + ".inst 0x6fa0eaaa // udot v10.4s, v21.16b, v0.4b[3]\n" + "ldr x20, [x16, #0x8]\n" + ".inst 0x6fa1eaae // udot v14.4s, v21.16b, v1.4b[3]\n" + "ldr x23, [x13, #0x8]\n" + ".inst 0x6fa2eab2 // udot v18.4s, v21.16b, v2.4b[3]\n" "ldr d6, [x16, #0x0]\n" - ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x6fa0ea8b // udot v11.4s, v20.16b, v0.4b[3]\n" "ldr d0, [x13, #0x0]\n" - ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n" - "ldr d1, [x9, #0x0]\n" - "ldr x28, [x9, #0x8]\n" - ".inst 0x6fa2e8f3 // udot v19.4s, v7.16b, v2.4b[3]\n" - "ldr d2, [x27, #0x0]\n" + ".inst 0x6fa1ea8f // udot v15.4s, v20.16b, v1.4b[3]\n" + "ldr d1, [x12, #0x0]\n" + "ldr x22, [x12, #0x8]\n" + ".inst 0x6fa2ea93 // udot v19.4s, v20.16b, v2.4b[3]\n" + "ldr d2, [x11, #0x0]\n" "sub x14, x14, #0x10\n" "ldr d7, [x16, #0x10]\n" "cmp x14, #0x20\n" - "ldr x26, [x27, #0x8]\n" - "mov v6.d[1], x12\n" - "ldr x11, [x16, #0x18]\n" - "mov v0.d[1], x10\n" + "ldr x21, [x11, #0x8]\n" + "mov v6.d[1], x20\n" + "ldr x20, [x16, #0x18]\n" + "mov v0.d[1], x23\n" "prfm pldl1keep, [x13, #0x80]\n" - "mov v1.d[1], x28\n" - "prfm pldl1keep, [x9, #0x80]\n" - "mov v2.d[1], x26\n" - "prfm pldl1keep, [x27, #0x80]\n" - "mov v7.d[1], x11\n" + "mov v1.d[1], x22\n" + "prfm pldl1keep, [x12, #0x80]\n" + "mov v2.d[1], x21\n" + "prfm pldl1keep, [x11, #0x80]\n" + "mov v7.d[1], x20\n" "bge 85b\n" "86:" // Height 3: Multiply loop: Single iteration only ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" "add x13, x13, #0x10\n" ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x16, #0x20]\n" + "ldr q21, [x16, #0x20]\n" ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - "add x27, x27, #0x10\n" + "add x11, x11, #0x10\n" ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" "sub x14, x14, #0x10\n" ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q20, [x16, #0x30]\n" + ".inst 0x6f80e2aa // udot v10.4s, v21.16b, v0.4b[0]\n" "prfm pldl1keep, [x13, #0x80]\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - "prfm pldl1keep, [x9, #0x80]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x16, #0x40]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - "prfm pldl1keep, [x27, #0x80]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x16, #0x50]\n" - ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x6fa2e0d0 // udot v16.4s, v6.16b, v2.4b[1]\n" - "ldr q6, [x16, #0x60]\n" - ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x6fa2e0f1 // udot v17.4s, v7.16b, v2.4b[1]\n" - "ldr q7, [x16, #0x70]\n" - ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x6fa2e0d2 // udot v18.4s, v6.16b, v2.4b[1]\n" - "ldr q6, [x16, #0x80]\n" - ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x6fa2e0f3 // udot v19.4s, v7.16b, v2.4b[1]\n" - "ldr q7, [x16, #0x90]\n" - ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x6f82e8d0 // udot v16.4s, v6.16b, v2.4b[2]\n" - "ldr q6, [x16, #0xa0]\n" - ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f82e8f1 // udot v17.4s, v7.16b, v2.4b[2]\n" - "ldr q7, [x16, #0xb0]\n" - ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x6f82e8d2 // udot v18.4s, v6.16b, v2.4b[2]\n" - "ldr q6, [x16, #0xc0]\n" - ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f82e8f3 // udot v19.4s, v7.16b, v2.4b[2]\n" - "ldr q7, [x16, #0xd0]\n" - ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x6fa2e8d0 // udot v16.4s, v6.16b, v2.4b[3]\n" - "ldr q6, [x16, #0xe0]\n" - ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x6fa2e8f1 // udot v17.4s, v7.16b, v2.4b[3]\n" - "ldr q7, [x16, #0xf0]\n" - ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n" + ".inst 0x6f81e2ae // udot v14.4s, v21.16b, v1.4b[0]\n" + "prfm pldl1keep, [x12, #0x80]\n" + ".inst 0x6f82e2b2 // udot v18.4s, v21.16b, v2.4b[0]\n" + "ldr q21, [x16, #0x40]\n" + ".inst 0x6f80e28b // udot v11.4s, v20.16b, v0.4b[0]\n" + "prfm pldl1keep, [x11, #0x80]\n" + ".inst 0x6f81e28f // udot v15.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f82e293 // udot v19.4s, v20.16b, v2.4b[0]\n" + "ldr q20, [x16, #0x50]\n" + ".inst 0x6fa0e2a8 // udot v8.4s, v21.16b, v0.4b[1]\n" + ".inst 0x6fa1e2ac // udot v12.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa2e2b0 // udot v16.4s, v21.16b, v2.4b[1]\n" + "ldr q21, [x16, #0x60]\n" + ".inst 0x6fa0e289 // udot v9.4s, v20.16b, v0.4b[1]\n" + ".inst 0x6fa1e28d // udot v13.4s, v20.16b, v1.4b[1]\n" + ".inst 0x6fa2e291 // udot v17.4s, v20.16b, v2.4b[1]\n" + "ldr q20, [x16, #0x70]\n" + ".inst 0x6fa0e2aa // udot v10.4s, v21.16b, v0.4b[1]\n" + ".inst 0x6fa1e2ae // udot v14.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa2e2b2 // udot v18.4s, v21.16b, v2.4b[1]\n" + "ldr q21, [x16, #0x80]\n" + ".inst 0x6fa0e28b // udot v11.4s, v20.16b, v0.4b[1]\n" + ".inst 0x6fa1e28f // udot v15.4s, v20.16b, v1.4b[1]\n" + ".inst 0x6fa2e293 // udot v19.4s, v20.16b, v2.4b[1]\n" + "ldr q20, [x16, #0x90]\n" + ".inst 0x6f80eaa8 // udot v8.4s, v21.16b, v0.4b[2]\n" + ".inst 0x6f81eaac // udot v12.4s, v21.16b, v1.4b[2]\n" + ".inst 0x6f82eab0 // udot v16.4s, v21.16b, v2.4b[2]\n" + "ldr q21, [x16, #0xa0]\n" + ".inst 0x6f80ea89 // udot v9.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f81ea8d // udot v13.4s, v20.16b, v1.4b[2]\n" + ".inst 0x6f82ea91 // udot v17.4s, v20.16b, v2.4b[2]\n" + "ldr q20, [x16, #0xb0]\n" + ".inst 0x6f80eaaa // udot v10.4s, v21.16b, v0.4b[2]\n" + ".inst 0x6f81eaae // udot v14.4s, v21.16b, v1.4b[2]\n" + ".inst 0x6f82eab2 // udot v18.4s, v21.16b, v2.4b[2]\n" + "ldr q21, [x16, #0xc0]\n" + ".inst 0x6f80ea8b // udot v11.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f81ea8f // udot v15.4s, v20.16b, v1.4b[2]\n" + ".inst 0x6f82ea93 // udot v19.4s, v20.16b, v2.4b[2]\n" + "ldr q20, [x16, #0xd0]\n" + ".inst 0x6fa0eaa8 // udot v8.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa1eaac // udot v12.4s, v21.16b, v1.4b[3]\n" + ".inst 0x6fa2eab0 // udot v16.4s, v21.16b, v2.4b[3]\n" + "ldr q21, [x16, #0xe0]\n" + ".inst 0x6fa0ea89 // udot v9.4s, v20.16b, v0.4b[3]\n" + ".inst 0x6fa1ea8d // udot v13.4s, v20.16b, v1.4b[3]\n" + ".inst 0x6fa2ea91 // udot v17.4s, v20.16b, v2.4b[3]\n" + "ldr q20, [x16, #0xf0]\n" + ".inst 0x6fa0eaaa // udot v10.4s, v21.16b, v0.4b[3]\n" "add x16, x16, #0x100\n" - ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x6fa2e8d2 // udot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n" - ".inst 0x6fa2e8f3 // udot v19.4s, v7.16b, v2.4b[3]\n" + ".inst 0x6fa1eaae // udot v14.4s, v21.16b, v1.4b[3]\n" + ".inst 0x6fa2eab2 // udot v18.4s, v21.16b, v2.4b[3]\n" + ".inst 0x6fa0ea8b // udot v11.4s, v20.16b, v0.4b[3]\n" + ".inst 0x6fa1ea8f // udot v15.4s, v20.16b, v1.4b[3]\n" + ".inst 0x6fa2ea93 // udot v19.4s, v20.16b, v2.4b[3]\n" "87:" // Height 3: Multiply loop: Main loop skip "cbz x14, 92f\n" "cmp x14, #0x4\n" "blt 89f\n" "88:" // Height 3: Multiply loop: Odd block loop - "ldr s0, [x13], #0x4\n" + "ldr s24, [x13], #0x4\n" "sub x14, x14, #0x4\n" - "ldr s1, [x9], #0x4\n" + "ldr s23, [x12], #0x4\n" "cmp x14, #0x4\n" - "ldr s2, [x27], #0x4\n" - "ldr q6, [x16, #0x0]\n" - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x10]\n" - ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x16, #0x20]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" + "ldr s22, [x11], #0x4\n" + "ldr q21, [x16, #0x0]\n" + ".inst 0x6f98e2a8 // udot v8.4s, v21.16b, v24.4b[0]\n" + "ldr q20, [x16, #0x10]\n" + ".inst 0x6f97e2ac // udot v12.4s, v21.16b, v23.4b[0]\n" + ".inst 0x6f96e2b0 // udot v16.4s, v21.16b, v22.4b[0]\n" + "ldr q21, [x16, #0x20]\n" + ".inst 0x6f98e289 // udot v9.4s, v20.16b, v24.4b[0]\n" + ".inst 0x6f97e28d // udot v13.4s, v20.16b, v23.4b[0]\n" + ".inst 0x6f96e291 // udot v17.4s, v20.16b, v22.4b[0]\n" + "ldr q20, [x16, #0x30]\n" + ".inst 0x6f98e2aa // udot v10.4s, v21.16b, v24.4b[0]\n" "add x16, x16, #0x40\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" + ".inst 0x6f97e2ae // udot v14.4s, v21.16b, v23.4b[0]\n" + ".inst 0x6f96e2b2 // udot v18.4s, v21.16b, v22.4b[0]\n" + ".inst 0x6f98e28b // udot v11.4s, v20.16b, v24.4b[0]\n" + ".inst 0x6f97e28f // udot v15.4s, v20.16b, v23.4b[0]\n" + ".inst 0x6f96e293 // udot v19.4s, v20.16b, v22.4b[0]\n" "bge 88b\n" "89:" // Height 3: Multiply loop: Skip odd blocks "cbz x14, 92f\n" "tbz x14, #1, 90f\n" "ldr h0, [x13], #0x2\n" - "ldr h1, [x9], #0x2\n" - "ldr h2, [x27], #0x2\n" + "ldr h1, [x12], #0x2\n" + "ldr h2, [x11], #0x2\n" "tbz x14, #0, 91f\n" "ld1 { v0.b }[2], [x13]\n" - "ld1 { v1.b }[2], [x9]\n" - "ld1 { v2.b }[2], [x27]\n" + "ld1 { v1.b }[2], [x12]\n" + "ld1 { v2.b }[2], [x11]\n" "b 91f\n" "90:" // Height 3: Multiply loop: Ragged operand read: partial_1_0 "ldr b0, [x13, #0x0]\n" - "ldr b1, [x9, #0x0]\n" - "ldr b2, [x27, #0x0]\n" + "ldr b1, [x12, #0x0]\n" + "ldr b2, [x11, #0x0]\n" "91:" // Height 3: Multiply loop: Ragged operand read: Done - "ldr q6, [x16, #0x0]\n" - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x10]\n" - ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x16, #0x20]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q21, [x16, #0x0]\n" + ".inst 0x6f80e2a8 // udot v8.4s, v21.16b, v0.4b[0]\n" + "ldr q20, [x16, #0x10]\n" + ".inst 0x6f81e2ac // udot v12.4s, v21.16b, v1.4b[0]\n" + ".inst 0x6f82e2b0 // udot v16.4s, v21.16b, v2.4b[0]\n" + "ldr q21, [x16, #0x20]\n" + ".inst 0x6f80e289 // udot v9.4s, v20.16b, v0.4b[0]\n" + ".inst 0x6f81e28d // udot v13.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f82e291 // udot v17.4s, v20.16b, v2.4b[0]\n" + "ldr q20, [x16, #0x30]\n" + ".inst 0x6f80e2aa // udot v10.4s, v21.16b, v0.4b[0]\n" "add x16, x16, #0x40\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" + ".inst 0x6f81e2ae // udot v14.4s, v21.16b, v1.4b[0]\n" + ".inst 0x6f82e2b2 // udot v18.4s, v21.16b, v2.4b[0]\n" + ".inst 0x6f80e28b // udot v11.4s, v20.16b, v0.4b[0]\n" + ".inst 0x6f81e28f // udot v15.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f82e293 // udot v19.4s, v20.16b, v2.4b[0]\n" "92:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x15, x15, #0x1\n" @@ -1475,336 +1474,336 @@ void a64_hybrid_u8u32_dot_6x16_a55 ( "116:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w14, [x20, x15, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 117f\n" - "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x13, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" - "ldr x27, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x15, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x13, [x20, #0x0]\n" + "ldr x12, [x20, #0x8]\n" + "ldr x11, [x20, #0x10]\n" + "ldr x10, [x20, #0x18]\n" "cbnz x15, 118f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x13, x13, x20\n" - "add x9, x9, x20\n" - "add x27, x27, x20\n" - "add x25, x25, x20\n" + "add x12, x12, x20\n" + "add x11, x11, x20\n" + "add x10, x10, x20\n" "b 118f\n" "117:" // Height 4: setup direct input "mov x13, %x[input_ptr]\n" - "add x9, x13, x20\n" - "add x27, x9, x20\n" - "add x25, x27, x20\n" + "add x12, x13, x21\n" + "add x11, x12, x21\n" + "add x10, x11, x21\n" "118:" // Height 4: input setup done "cmp x14, #0x10\n" "blt 121f\n" "ldr q0, [x13, #0x0]\n" "cmp x14, #0x20\n" - "ldr q1, [x9, #0x0]\n" - "ldr q2, [x27, #0x0]\n" - "ldr q3, [x25, #0x0]\n" + "ldr q1, [x12, #0x0]\n" + "ldr q2, [x11, #0x0]\n" + "ldr q3, [x10, #0x0]\n" "ldr q6, [x16, #0x0]\n" "ldr q7, [x16, #0x10]\n" "blt 120f\n" "119:" // Height 4: Multiply loop: Main loop head ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - "ldr x12, [x16, #0x28]\n" + "ldr x21, [x16, #0x28]\n" ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - "ldr x11, [x16, #0x38]\n" + "ldr x20, [x16, #0x38]\n" ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" "add x13, x13, #0x10\n" ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n" - "ldr d6, [x16, #0x20]\n" + "ldr d25, [x16, #0x20]\n" ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x12\n" + "mov v25.d[1], x21\n" ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - "ldr x12, [x16, #0x48]\n" + "ldr x21, [x16, #0x48]\n" ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n" - "ldr d7, [x16, #0x30]\n" - "mov v7.d[1], x11\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - "ldr x11, [x16, #0x58]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - "add x27, x27, #0x10\n" - ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n" - "ldr d6, [x16, #0x40]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x12\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - "ldr x12, [x16, #0x68]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - "add x25, x25, #0x10\n" - ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n" - "ldr d7, [x16, #0x50]\n" - "mov v7.d[1], x11\n" - ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n" - "ldr x11, [x16, #0x78]\n" - ".inst 0x6fa2e0d0 // udot v16.4s, v6.16b, v2.4b[1]\n" - "ldr x10, [x13, #0x8]\n" - ".inst 0x6fa3e0d4 // udot v20.4s, v6.16b, v3.4b[1]\n" - "ldr d6, [x16, #0x60]\n" - ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n" - "mov v6.d[1], x12\n" - ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n" - "ldr x12, [x16, #0x88]\n" - ".inst 0x6fa2e0f1 // udot v17.4s, v7.16b, v2.4b[1]\n" - "ldr x28, [x9, #0x8]\n" - ".inst 0x6fa3e0f5 // udot v21.4s, v7.16b, v3.4b[1]\n" - "ldr d7, [x16, #0x70]\n" - "mov v7.d[1], x11\n" - ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n" - "ldr x11, [x16, #0x98]\n" - ".inst 0x6fa2e0d2 // udot v18.4s, v6.16b, v2.4b[1]\n" - "ldr x26, [x27, #0x8]\n" - ".inst 0x6fa3e0d6 // udot v22.4s, v6.16b, v3.4b[1]\n" - "ldr d6, [x16, #0x80]\n" - ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n" - "mov v6.d[1], x12\n" - ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n" - "ldr x12, [x16, #0xa8]\n" - ".inst 0x6fa2e0f3 // udot v19.4s, v7.16b, v2.4b[1]\n" - "ldr x24, [x25, #0x8]\n" - ".inst 0x6fa3e0f7 // udot v23.4s, v7.16b, v3.4b[1]\n" - "ldr d7, [x16, #0x90]\n" - "mov v7.d[1], x11\n" - ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n" - "ldr x11, [x16, #0xb8]\n" - ".inst 0x6f82e8d0 // udot v16.4s, v6.16b, v2.4b[2]\n" + "ldr d24, [x16, #0x30]\n" + "mov v24.d[1], x20\n" + ".inst 0x6f80e32a // udot v10.4s, v25.16b, v0.4b[0]\n" + ".inst 0x6f81e32e // udot v14.4s, v25.16b, v1.4b[0]\n" + "ldr x20, [x16, #0x58]\n" + ".inst 0x6f82e332 // udot v18.4s, v25.16b, v2.4b[0]\n" + "add x11, x11, #0x10\n" + ".inst 0x6f83e336 // udot v22.4s, v25.16b, v3.4b[0]\n" + "ldr d25, [x16, #0x40]\n" + ".inst 0x6f80e30b // udot v11.4s, v24.16b, v0.4b[0]\n" + "mov v25.d[1], x21\n" + ".inst 0x6f81e30f // udot v15.4s, v24.16b, v1.4b[0]\n" + "ldr x21, [x16, #0x68]\n" + ".inst 0x6f82e313 // udot v19.4s, v24.16b, v2.4b[0]\n" + "add x10, x10, #0x10\n" + ".inst 0x6f83e317 // udot v23.4s, v24.16b, v3.4b[0]\n" + "ldr d24, [x16, #0x50]\n" + "mov v24.d[1], x20\n" + ".inst 0x6fa0e328 // udot v8.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa1e32c // udot v12.4s, v25.16b, v1.4b[1]\n" + "ldr x20, [x16, #0x78]\n" + ".inst 0x6fa2e330 // udot v16.4s, v25.16b, v2.4b[1]\n" + "ldr x25, [x13, #0x8]\n" + ".inst 0x6fa3e334 // udot v20.4s, v25.16b, v3.4b[1]\n" + "ldr d25, [x16, #0x60]\n" + ".inst 0x6fa0e309 // udot v9.4s, v24.16b, v0.4b[1]\n" + "mov v25.d[1], x21\n" + ".inst 0x6fa1e30d // udot v13.4s, v24.16b, v1.4b[1]\n" + "ldr x21, [x16, #0x88]\n" + ".inst 0x6fa2e311 // udot v17.4s, v24.16b, v2.4b[1]\n" + "ldr x24, [x12, #0x8]\n" + ".inst 0x6fa3e315 // udot v21.4s, v24.16b, v3.4b[1]\n" + "ldr d24, [x16, #0x70]\n" + "mov v24.d[1], x20\n" + ".inst 0x6fa0e32a // udot v10.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa1e32e // udot v14.4s, v25.16b, v1.4b[1]\n" + "ldr x20, [x16, #0x98]\n" + ".inst 0x6fa2e332 // udot v18.4s, v25.16b, v2.4b[1]\n" + "ldr x23, [x11, #0x8]\n" + ".inst 0x6fa3e336 // udot v22.4s, v25.16b, v3.4b[1]\n" + "ldr d25, [x16, #0x80]\n" + ".inst 0x6fa0e30b // udot v11.4s, v24.16b, v0.4b[1]\n" + "mov v25.d[1], x21\n" + ".inst 0x6fa1e30f // udot v15.4s, v24.16b, v1.4b[1]\n" + "ldr x21, [x16, #0xa8]\n" + ".inst 0x6fa2e313 // udot v19.4s, v24.16b, v2.4b[1]\n" + "ldr x22, [x10, #0x8]\n" + ".inst 0x6fa3e317 // udot v23.4s, v24.16b, v3.4b[1]\n" + "ldr d24, [x16, #0x90]\n" + "mov v24.d[1], x20\n" + ".inst 0x6f80eb28 // udot v8.4s, v25.16b, v0.4b[2]\n" + ".inst 0x6f81eb2c // udot v12.4s, v25.16b, v1.4b[2]\n" + "ldr x20, [x16, #0xb8]\n" + ".inst 0x6f82eb30 // udot v16.4s, v25.16b, v2.4b[2]\n" "sub x14, x14, #0x10\n" - ".inst 0x6f83e8d4 // udot v20.4s, v6.16b, v3.4b[2]\n" - "ldr d6, [x16, #0xa0]\n" - ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n" - "mov v6.d[1], x12\n" - ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n" - "ldr x12, [x16, #0xc8]\n" - ".inst 0x6f82e8f1 // udot v17.4s, v7.16b, v2.4b[2]\n" + ".inst 0x6f83eb34 // udot v20.4s, v25.16b, v3.4b[2]\n" + "ldr d25, [x16, #0xa0]\n" + ".inst 0x6f80eb09 // udot v9.4s, v24.16b, v0.4b[2]\n" + "mov v25.d[1], x21\n" + ".inst 0x6f81eb0d // udot v13.4s, v24.16b, v1.4b[2]\n" + "ldr x21, [x16, #0xc8]\n" + ".inst 0x6f82eb11 // udot v17.4s, v24.16b, v2.4b[2]\n" "cmp x14, #0x20\n" - ".inst 0x6f83e8f5 // udot v21.4s, v7.16b, v3.4b[2]\n" - "ldr d7, [x16, #0xb0]\n" - "mov v7.d[1], x11\n" - ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n" - "ldr x11, [x16, #0xd8]\n" - ".inst 0x6f82e8d2 // udot v18.4s, v6.16b, v2.4b[2]\n" + ".inst 0x6f83eb15 // udot v21.4s, v24.16b, v3.4b[2]\n" + "ldr d24, [x16, #0xb0]\n" + "mov v24.d[1], x20\n" + ".inst 0x6f80eb2a // udot v10.4s, v25.16b, v0.4b[2]\n" + ".inst 0x6f81eb2e // udot v14.4s, v25.16b, v1.4b[2]\n" + "ldr x20, [x16, #0xd8]\n" + ".inst 0x6f82eb32 // udot v18.4s, v25.16b, v2.4b[2]\n" "prfm pldl1keep, [x13, #0x80]\n" - ".inst 0x6f83e8d6 // udot v22.4s, v6.16b, v3.4b[2]\n" - "ldr d6, [x16, #0xc0]\n" - ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n" - "mov v6.d[1], x12\n" - ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n" - "ldr x12, [x16, #0xe8]\n" - ".inst 0x6f82e8f3 // udot v19.4s, v7.16b, v2.4b[2]\n" - "prfm pldl1keep, [x9, #0x80]\n" - ".inst 0x6f83e8f7 // udot v23.4s, v7.16b, v3.4b[2]\n" - "ldr d7, [x16, #0xd0]\n" - "mov v7.d[1], x11\n" - ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n" - "ldr x11, [x16, #0xf8]\n" - ".inst 0x6fa2e8d0 // udot v16.4s, v6.16b, v2.4b[3]\n" - "prfm pldl1keep, [x27, #0x80]\n" - ".inst 0x6fa3e8d4 // udot v20.4s, v6.16b, v3.4b[3]\n" - "ldr d6, [x16, #0xe0]\n" - ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n" - "mov v6.d[1], x12\n" - ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n" - "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x6fa2e8f1 // udot v17.4s, v7.16b, v2.4b[3]\n" - ".inst 0x6fa3e8f5 // udot v21.4s, v7.16b, v3.4b[3]\n" - "ldr d7, [x16, #0xf0]\n" - "mov v7.d[1], x11\n" + ".inst 0x6f83eb36 // udot v22.4s, v25.16b, v3.4b[2]\n" + "ldr d25, [x16, #0xc0]\n" + ".inst 0x6f80eb0b // udot v11.4s, v24.16b, v0.4b[2]\n" + "mov v25.d[1], x21\n" + ".inst 0x6f81eb0f // udot v15.4s, v24.16b, v1.4b[2]\n" + "ldr x21, [x16, #0xe8]\n" + ".inst 0x6f82eb13 // udot v19.4s, v24.16b, v2.4b[2]\n" + "prfm pldl1keep, [x12, #0x80]\n" + ".inst 0x6f83eb17 // udot v23.4s, v24.16b, v3.4b[2]\n" + "ldr d24, [x16, #0xd0]\n" + "mov v24.d[1], x20\n" + ".inst 0x6fa0eb28 // udot v8.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa1eb2c // udot v12.4s, v25.16b, v1.4b[3]\n" + "ldr x20, [x16, #0xf8]\n" + ".inst 0x6fa2eb30 // udot v16.4s, v25.16b, v2.4b[3]\n" + "prfm pldl1keep, [x11, #0x80]\n" + ".inst 0x6fa3eb34 // udot v20.4s, v25.16b, v3.4b[3]\n" + "ldr d25, [x16, #0xe0]\n" + ".inst 0x6fa0eb09 // udot v9.4s, v24.16b, v0.4b[3]\n" + "mov v25.d[1], x21\n" + ".inst 0x6fa1eb0d // udot v13.4s, v24.16b, v1.4b[3]\n" + "prfm pldl1keep, [x10, #0x80]\n" + ".inst 0x6fa2eb11 // udot v17.4s, v24.16b, v2.4b[3]\n" + ".inst 0x6fa3eb15 // udot v21.4s, v24.16b, v3.4b[3]\n" + "ldr d24, [x16, #0xf0]\n" + "mov v24.d[1], x20\n" "add x16, x16, #0x100\n" - ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n" - "ldr x12, [x16, #0x8]\n" - ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n" - "ldr x11, [x16, #0x18]\n" - ".inst 0x6fa2e8d2 // udot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x6fa3e8d6 // udot v22.4s, v6.16b, v3.4b[3]\n" + ".inst 0x6fa0eb2a // udot v10.4s, v25.16b, v0.4b[3]\n" + "ldr x21, [x16, #0x8]\n" + ".inst 0x6fa1eb2e // udot v14.4s, v25.16b, v1.4b[3]\n" + "ldr x20, [x16, #0x18]\n" + ".inst 0x6fa2eb32 // udot v18.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa3eb36 // udot v22.4s, v25.16b, v3.4b[3]\n" "ldr d6, [x16, #0x0]\n" - ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x6fa0eb0b // udot v11.4s, v24.16b, v0.4b[3]\n" "ldr d0, [x13, #0x0]\n" - ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n" - "ldr d1, [x9, #0x0]\n" - ".inst 0x6fa2e8f3 // udot v19.4s, v7.16b, v2.4b[3]\n" - "ldr d2, [x27, #0x0]\n" - ".inst 0x6fa3e8f7 // udot v23.4s, v7.16b, v3.4b[3]\n" - "ldr d3, [x25, #0x0]\n" + ".inst 0x6fa1eb0f // udot v15.4s, v24.16b, v1.4b[3]\n" + "ldr d1, [x12, #0x0]\n" + ".inst 0x6fa2eb13 // udot v19.4s, v24.16b, v2.4b[3]\n" + "ldr d2, [x11, #0x0]\n" + ".inst 0x6fa3eb17 // udot v23.4s, v24.16b, v3.4b[3]\n" + "ldr d3, [x10, #0x0]\n" "ldr d7, [x16, #0x10]\n" - "mov v6.d[1], x12\n" - "mov v0.d[1], x10\n" - "mov v1.d[1], x28\n" - "mov v2.d[1], x26\n" - "mov v3.d[1], x24\n" - "mov v7.d[1], x11\n" + "mov v6.d[1], x21\n" + "mov v0.d[1], x25\n" + "mov v1.d[1], x24\n" + "mov v2.d[1], x23\n" + "mov v3.d[1], x22\n" + "mov v7.d[1], x20\n" "bge 119b\n" "120:" // Height 4: Multiply loop: Single iteration only ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" "add x13, x13, #0x10\n" ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - "add x27, x27, #0x10\n" + "add x11, x11, #0x10\n" ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x16, #0x20]\n" + "ldr q25, [x16, #0x20]\n" ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - "add x25, x25, #0x10\n" + "add x10, x10, #0x10\n" ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" "sub x14, x14, #0x10\n" ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" "prfm pldl1keep, [x13, #0x80]\n" ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - "prfm pldl1keep, [x9, #0x80]\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - "prfm pldl1keep, [x27, #0x80]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x16, #0x40]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x16, #0x50]\n" - ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x6fa2e0d0 // udot v16.4s, v6.16b, v2.4b[1]\n" - ".inst 0x6fa3e0d4 // udot v20.4s, v6.16b, v3.4b[1]\n" - "ldr q6, [x16, #0x60]\n" - ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x6fa2e0f1 // udot v17.4s, v7.16b, v2.4b[1]\n" - ".inst 0x6fa3e0f5 // udot v21.4s, v7.16b, v3.4b[1]\n" - "ldr q7, [x16, #0x70]\n" - ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x6fa2e0d2 // udot v18.4s, v6.16b, v2.4b[1]\n" - ".inst 0x6fa3e0d6 // udot v22.4s, v6.16b, v3.4b[1]\n" - "ldr q6, [x16, #0x80]\n" - ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x6fa2e0f3 // udot v19.4s, v7.16b, v2.4b[1]\n" - ".inst 0x6fa3e0f7 // udot v23.4s, v7.16b, v3.4b[1]\n" - "ldr q7, [x16, #0x90]\n" - ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x6f82e8d0 // udot v16.4s, v6.16b, v2.4b[2]\n" - ".inst 0x6f83e8d4 // udot v20.4s, v6.16b, v3.4b[2]\n" - "ldr q6, [x16, #0xa0]\n" - ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f82e8f1 // udot v17.4s, v7.16b, v2.4b[2]\n" - ".inst 0x6f83e8f5 // udot v21.4s, v7.16b, v3.4b[2]\n" - "ldr q7, [x16, #0xb0]\n" - ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x6f82e8d2 // udot v18.4s, v6.16b, v2.4b[2]\n" - ".inst 0x6f83e8d6 // udot v22.4s, v6.16b, v3.4b[2]\n" - "ldr q6, [x16, #0xc0]\n" - ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f82e8f3 // udot v19.4s, v7.16b, v2.4b[2]\n" - ".inst 0x6f83e8f7 // udot v23.4s, v7.16b, v3.4b[2]\n" - "ldr q7, [x16, #0xd0]\n" - ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x6fa2e8d0 // udot v16.4s, v6.16b, v2.4b[3]\n" - ".inst 0x6fa3e8d4 // udot v20.4s, v6.16b, v3.4b[3]\n" - "ldr q6, [x16, #0xe0]\n" - ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x6fa2e8f1 // udot v17.4s, v7.16b, v2.4b[3]\n" - ".inst 0x6fa3e8f5 // udot v21.4s, v7.16b, v3.4b[3]\n" - "ldr q7, [x16, #0xf0]\n" - ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n" + "ldr q24, [x16, #0x30]\n" + ".inst 0x6f80e32a // udot v10.4s, v25.16b, v0.4b[0]\n" + "prfm pldl1keep, [x12, #0x80]\n" + ".inst 0x6f81e32e // udot v14.4s, v25.16b, v1.4b[0]\n" + "prfm pldl1keep, [x11, #0x80]\n" + ".inst 0x6f82e332 // udot v18.4s, v25.16b, v2.4b[0]\n" + "prfm pldl1keep, [x10, #0x80]\n" + ".inst 0x6f83e336 // udot v22.4s, v25.16b, v3.4b[0]\n" + "ldr q25, [x16, #0x40]\n" + ".inst 0x6f80e30b // udot v11.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f81e30f // udot v15.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f82e313 // udot v19.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f83e317 // udot v23.4s, v24.16b, v3.4b[0]\n" + "ldr q24, [x16, #0x50]\n" + ".inst 0x6fa0e328 // udot v8.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa1e32c // udot v12.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa2e330 // udot v16.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa3e334 // udot v20.4s, v25.16b, v3.4b[1]\n" + "ldr q25, [x16, #0x60]\n" + ".inst 0x6fa0e309 // udot v9.4s, v24.16b, v0.4b[1]\n" + ".inst 0x6fa1e30d // udot v13.4s, v24.16b, v1.4b[1]\n" + ".inst 0x6fa2e311 // udot v17.4s, v24.16b, v2.4b[1]\n" + ".inst 0x6fa3e315 // udot v21.4s, v24.16b, v3.4b[1]\n" + "ldr q24, [x16, #0x70]\n" + ".inst 0x6fa0e32a // udot v10.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa1e32e // udot v14.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa2e332 // udot v18.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa3e336 // udot v22.4s, v25.16b, v3.4b[1]\n" + "ldr q25, [x16, #0x80]\n" + ".inst 0x6fa0e30b // udot v11.4s, v24.16b, v0.4b[1]\n" + ".inst 0x6fa1e30f // udot v15.4s, v24.16b, v1.4b[1]\n" + ".inst 0x6fa2e313 // udot v19.4s, v24.16b, v2.4b[1]\n" + ".inst 0x6fa3e317 // udot v23.4s, v24.16b, v3.4b[1]\n" + "ldr q24, [x16, #0x90]\n" + ".inst 0x6f80eb28 // udot v8.4s, v25.16b, v0.4b[2]\n" + ".inst 0x6f81eb2c // udot v12.4s, v25.16b, v1.4b[2]\n" + ".inst 0x6f82eb30 // udot v16.4s, v25.16b, v2.4b[2]\n" + ".inst 0x6f83eb34 // udot v20.4s, v25.16b, v3.4b[2]\n" + "ldr q25, [x16, #0xa0]\n" + ".inst 0x6f80eb09 // udot v9.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f81eb0d // udot v13.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f82eb11 // udot v17.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f83eb15 // udot v21.4s, v24.16b, v3.4b[2]\n" + "ldr q24, [x16, #0xb0]\n" + ".inst 0x6f80eb2a // udot v10.4s, v25.16b, v0.4b[2]\n" + ".inst 0x6f81eb2e // udot v14.4s, v25.16b, v1.4b[2]\n" + ".inst 0x6f82eb32 // udot v18.4s, v25.16b, v2.4b[2]\n" + ".inst 0x6f83eb36 // udot v22.4s, v25.16b, v3.4b[2]\n" + "ldr q25, [x16, #0xc0]\n" + ".inst 0x6f80eb0b // udot v11.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f81eb0f // udot v15.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f82eb13 // udot v19.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f83eb17 // udot v23.4s, v24.16b, v3.4b[2]\n" + "ldr q24, [x16, #0xd0]\n" + ".inst 0x6fa0eb28 // udot v8.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa1eb2c // udot v12.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa2eb30 // udot v16.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa3eb34 // udot v20.4s, v25.16b, v3.4b[3]\n" + "ldr q25, [x16, #0xe0]\n" + ".inst 0x6fa0eb09 // udot v9.4s, v24.16b, v0.4b[3]\n" + ".inst 0x6fa1eb0d // udot v13.4s, v24.16b, v1.4b[3]\n" + ".inst 0x6fa2eb11 // udot v17.4s, v24.16b, v2.4b[3]\n" + ".inst 0x6fa3eb15 // udot v21.4s, v24.16b, v3.4b[3]\n" + "ldr q24, [x16, #0xf0]\n" + ".inst 0x6fa0eb2a // udot v10.4s, v25.16b, v0.4b[3]\n" "add x16, x16, #0x100\n" - ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x6fa2e8d2 // udot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x6fa3e8d6 // udot v22.4s, v6.16b, v3.4b[3]\n" - ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n" - ".inst 0x6fa2e8f3 // udot v19.4s, v7.16b, v2.4b[3]\n" - ".inst 0x6fa3e8f7 // udot v23.4s, v7.16b, v3.4b[3]\n" + ".inst 0x6fa1eb2e // udot v14.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa2eb32 // udot v18.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa3eb36 // udot v22.4s, v25.16b, v3.4b[3]\n" + ".inst 0x6fa0eb0b // udot v11.4s, v24.16b, v0.4b[3]\n" + ".inst 0x6fa1eb0f // udot v15.4s, v24.16b, v1.4b[3]\n" + ".inst 0x6fa2eb13 // udot v19.4s, v24.16b, v2.4b[3]\n" + ".inst 0x6fa3eb17 // udot v23.4s, v24.16b, v3.4b[3]\n" "121:" // Height 4: Multiply loop: Main loop skip "cbz x14, 126f\n" "cmp x14, #0x4\n" "blt 123f\n" "122:" // Height 4: Multiply loop: Odd block loop - "ldr s0, [x13], #0x4\n" + "ldr s29, [x13], #0x4\n" "sub x14, x14, #0x4\n" - "ldr s1, [x9], #0x4\n" + "ldr s28, [x12], #0x4\n" "cmp x14, #0x4\n" - "ldr s2, [x27], #0x4\n" - "ldr s3, [x25], #0x4\n" - "ldr q6, [x16, #0x0]\n" - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x10]\n" - ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x16, #0x20]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" + "ldr s27, [x11], #0x4\n" + "ldr s26, [x10], #0x4\n" + "ldr q25, [x16, #0x0]\n" + ".inst 0x6f9de328 // udot v8.4s, v25.16b, v29.4b[0]\n" + "ldr q24, [x16, #0x10]\n" + ".inst 0x6f9ce32c // udot v12.4s, v25.16b, v28.4b[0]\n" + ".inst 0x6f9be330 // udot v16.4s, v25.16b, v27.4b[0]\n" + ".inst 0x6f9ae334 // udot v20.4s, v25.16b, v26.4b[0]\n" + "ldr q25, [x16, #0x20]\n" + ".inst 0x6f9de309 // udot v9.4s, v24.16b, v29.4b[0]\n" + ".inst 0x6f9ce30d // udot v13.4s, v24.16b, v28.4b[0]\n" + ".inst 0x6f9be311 // udot v17.4s, v24.16b, v27.4b[0]\n" + ".inst 0x6f9ae315 // udot v21.4s, v24.16b, v26.4b[0]\n" + "ldr q24, [x16, #0x30]\n" + ".inst 0x6f9de32a // udot v10.4s, v25.16b, v29.4b[0]\n" "add x16, x16, #0x40\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n" + ".inst 0x6f9ce32e // udot v14.4s, v25.16b, v28.4b[0]\n" + ".inst 0x6f9be332 // udot v18.4s, v25.16b, v27.4b[0]\n" + ".inst 0x6f9ae336 // udot v22.4s, v25.16b, v26.4b[0]\n" + ".inst 0x6f9de30b // udot v11.4s, v24.16b, v29.4b[0]\n" + ".inst 0x6f9ce30f // udot v15.4s, v24.16b, v28.4b[0]\n" + ".inst 0x6f9be313 // udot v19.4s, v24.16b, v27.4b[0]\n" + ".inst 0x6f9ae317 // udot v23.4s, v24.16b, v26.4b[0]\n" "bge 122b\n" "123:" // Height 4: Multiply loop: Skip odd blocks "cbz x14, 126f\n" "tbz x14, #1, 124f\n" "ldr h0, [x13], #0x2\n" - "ldr h1, [x9], #0x2\n" - "ldr h2, [x27], #0x2\n" - "ldr h3, [x25], #0x2\n" + "ldr h1, [x12], #0x2\n" + "ldr h2, [x11], #0x2\n" + "ldr h3, [x10], #0x2\n" "tbz x14, #0, 125f\n" "ld1 { v0.b }[2], [x13]\n" - "ld1 { v1.b }[2], [x9]\n" - "ld1 { v2.b }[2], [x27]\n" - "ld1 { v3.b }[2], [x25]\n" + "ld1 { v1.b }[2], [x12]\n" + "ld1 { v2.b }[2], [x11]\n" + "ld1 { v3.b }[2], [x10]\n" "b 125f\n" "124:" // Height 4: Multiply loop: Ragged operand read: partial_1_0 "ldr b0, [x13, #0x0]\n" - "ldr b1, [x9, #0x0]\n" - "ldr b2, [x27, #0x0]\n" - "ldr b3, [x25, #0x0]\n" + "ldr b1, [x12, #0x0]\n" + "ldr b2, [x11, #0x0]\n" + "ldr b3, [x10, #0x0]\n" "125:" // Height 4: Multiply loop: Ragged operand read: Done - "ldr q6, [x16, #0x0]\n" - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x10]\n" - ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x16, #0x20]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q25, [x16, #0x0]\n" + ".inst 0x6f80e328 // udot v8.4s, v25.16b, v0.4b[0]\n" + "ldr q24, [x16, #0x10]\n" + ".inst 0x6f81e32c // udot v12.4s, v25.16b, v1.4b[0]\n" + ".inst 0x6f82e330 // udot v16.4s, v25.16b, v2.4b[0]\n" + ".inst 0x6f83e334 // udot v20.4s, v25.16b, v3.4b[0]\n" + "ldr q25, [x16, #0x20]\n" + ".inst 0x6f80e309 // udot v9.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f81e30d // udot v13.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f82e311 // udot v17.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f83e315 // udot v21.4s, v24.16b, v3.4b[0]\n" + "ldr q24, [x16, #0x30]\n" + ".inst 0x6f80e32a // udot v10.4s, v25.16b, v0.4b[0]\n" "add x16, x16, #0x40\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n" + ".inst 0x6f81e32e // udot v14.4s, v25.16b, v1.4b[0]\n" + ".inst 0x6f82e332 // udot v18.4s, v25.16b, v2.4b[0]\n" + ".inst 0x6f83e336 // udot v22.4s, v25.16b, v3.4b[0]\n" + ".inst 0x6f80e30b // udot v11.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f81e30f // udot v15.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f82e313 // udot v19.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f83e317 // udot v23.4s, v24.16b, v3.4b[0]\n" "126:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x15, x15, #0x1\n" @@ -2108,399 +2107,399 @@ void a64_hybrid_u8u32_dot_6x16_a55 ( "movi v22.4s, #0x0\n" "movi v23.4s, #0x0\n" "movi v24.4s, #0x0\n" - "movi v25.4s, #0x0\n" - "movi v26.4s, #0x0\n" - "movi v27.4s, #0x0\n" - "149:" // Height 5: setup done - "mov x15, #0x0\n" - "150:" // Height 5: String loop - "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr w14, [x20, x15, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" - "tbz %x[flags], #3, 151f\n" - "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x13, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" - "ldr x27, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" - "ldr x23, [x21, #0x20]\n" - "cbnz x15, 152f\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x13, x13, x20\n" - "add x9, x9, x20\n" - "add x27, x27, x20\n" - "add x25, x25, x20\n" - "add x23, x23, x20\n" - "b 152f\n" - "151:" // Height 5: setup direct input - "mov x13, %x[input_ptr]\n" - "add x9, x13, x20\n" - "add x27, x9, x20\n" - "add x25, x27, x20\n" - "add x23, x25, x20\n" - "152:" // Height 5: input setup done - "cmp x14, #0x10\n" - "blt 155f\n" - "ldr q0, [x13, #0x0]\n" - "cmp x14, #0x20\n" - "ldr q1, [x9, #0x0]\n" - "ldr q2, [x27, #0x0]\n" - "ldr q3, [x25, #0x0]\n" - "ldr q4, [x23, #0x0]\n" - "ldr q6, [x16, #0x0]\n" - "ldr q7, [x16, #0x10]\n" - "blt 154f\n" - "153:" // Height 5: Multiply loop: Main loop head - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - "ldr x12, [x16, #0x28]\n" - ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - "ldr x11, [x16, #0x38]\n" - ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - "add x13, x13, #0x10\n" - ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n" - "add x9, x9, #0x10\n" - ".inst 0x6f84e0d8 // udot v24.4s, v6.16b, v4.4b[0]\n" - "ldr d6, [x16, #0x20]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x12\n" - ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - "ldr x12, [x16, #0x48]\n" - ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" - "add x27, x27, #0x10\n" - ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n" - "add x25, x25, #0x10\n" - ".inst 0x6f84e0f9 // udot v25.4s, v7.16b, v4.4b[0]\n" - "ldr d7, [x16, #0x30]\n" - "mov v7.d[1], x11\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - "ldr x11, [x16, #0x58]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - "add x23, x23, #0x10\n" - ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n" - "ldr x10, [x13, #0x8]\n" - ".inst 0x6f84e0da // udot v26.4s, v6.16b, v4.4b[0]\n" - "ldr d6, [x16, #0x40]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x12\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - "ldr x12, [x16, #0x68]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - "ldr x28, [x9, #0x8]\n" - ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n" - "ldr x26, [x27, #0x8]\n" - ".inst 0x6f84e0fb // udot v27.4s, v7.16b, v4.4b[0]\n" - "ldr d7, [x16, #0x50]\n" - "mov v7.d[1], x11\n" - ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n" - "ldr x11, [x16, #0x78]\n" - ".inst 0x6fa2e0d0 // udot v16.4s, v6.16b, v2.4b[1]\n" - "ldr x24, [x25, #0x8]\n" - ".inst 0x6fa3e0d4 // udot v20.4s, v6.16b, v3.4b[1]\n" - "ldr x22, [x23, #0x8]\n" - ".inst 0x6fa4e0d8 // udot v24.4s, v6.16b, v4.4b[1]\n" - "ldr d6, [x16, #0x60]\n" - ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n" - "mov v6.d[1], x12\n" - ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n" - "ldr x12, [x16, #0x88]\n" - ".inst 0x6fa2e0f1 // udot v17.4s, v7.16b, v2.4b[1]\n" - "sub x14, x14, #0x10\n" - ".inst 0x6fa3e0f5 // udot v21.4s, v7.16b, v3.4b[1]\n" - "cmp x14, #0x20\n" - ".inst 0x6fa4e0f9 // udot v25.4s, v7.16b, v4.4b[1]\n" - "ldr d7, [x16, #0x70]\n" - "mov v7.d[1], x11\n" - ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n" - "ldr x11, [x16, #0x98]\n" - ".inst 0x6fa2e0d2 // udot v18.4s, v6.16b, v2.4b[1]\n" - "prfm pldl1keep, [x13, #0x80]\n" - ".inst 0x6fa3e0d6 // udot v22.4s, v6.16b, v3.4b[1]\n" - "prfm pldl1keep, [x9, #0x80]\n" - ".inst 0x6fa4e0da // udot v26.4s, v6.16b, v4.4b[1]\n" - "ldr d6, [x16, #0x80]\n" - ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n" - "mov v6.d[1], x12\n" - ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n" - "ldr x12, [x16, #0xa8]\n" - ".inst 0x6fa2e0f3 // udot v19.4s, v7.16b, v2.4b[1]\n" - "prfm pldl1keep, [x27, #0x80]\n" - ".inst 0x6fa3e0f7 // udot v23.4s, v7.16b, v3.4b[1]\n" - "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x6fa4e0fb // udot v27.4s, v7.16b, v4.4b[1]\n" - "ldr d7, [x16, #0x90]\n" - "mov v7.d[1], x11\n" - ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n" - "ldr x11, [x16, #0xb8]\n" - ".inst 0x6f82e8d0 // udot v16.4s, v6.16b, v2.4b[2]\n" - "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x6f83e8d4 // udot v20.4s, v6.16b, v3.4b[2]\n" - ".inst 0x6f84e8d8 // udot v24.4s, v6.16b, v4.4b[2]\n" - "ldr d6, [x16, #0xa0]\n" - ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n" - "mov v6.d[1], x12\n" - ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n" - "ldr x12, [x16, #0xc8]\n" - ".inst 0x6f82e8f1 // udot v17.4s, v7.16b, v2.4b[2]\n" - ".inst 0x6f83e8f5 // udot v21.4s, v7.16b, v3.4b[2]\n" - ".inst 0x6f84e8f9 // udot v25.4s, v7.16b, v4.4b[2]\n" - "ldr d7, [x16, #0xb0]\n" - "mov v7.d[1], x11\n" - ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n" - "ldr x11, [x16, #0xd8]\n" - ".inst 0x6f82e8d2 // udot v18.4s, v6.16b, v2.4b[2]\n" - ".inst 0x6f83e8d6 // udot v22.4s, v6.16b, v3.4b[2]\n" - ".inst 0x6f84e8da // udot v26.4s, v6.16b, v4.4b[2]\n" - "ldr d6, [x16, #0xc0]\n" - ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n" - "mov v6.d[1], x12\n" - ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n" - "ldr x12, [x16, #0xe8]\n" - ".inst 0x6f82e8f3 // udot v19.4s, v7.16b, v2.4b[2]\n" - ".inst 0x6f83e8f7 // udot v23.4s, v7.16b, v3.4b[2]\n" - ".inst 0x6f84e8fb // udot v27.4s, v7.16b, v4.4b[2]\n" - "ldr d7, [x16, #0xd0]\n" - "mov v7.d[1], x11\n" - ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n" - "ldr x11, [x16, #0xf8]\n" - ".inst 0x6fa2e8d0 // udot v16.4s, v6.16b, v2.4b[3]\n" - ".inst 0x6fa3e8d4 // udot v20.4s, v6.16b, v3.4b[3]\n" - ".inst 0x6fa4e8d8 // udot v24.4s, v6.16b, v4.4b[3]\n" - "ldr d6, [x16, #0xe0]\n" - ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n" - "mov v6.d[1], x12\n" - ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x6fa2e8f1 // udot v17.4s, v7.16b, v2.4b[3]\n" - ".inst 0x6fa3e8f5 // udot v21.4s, v7.16b, v3.4b[3]\n" - ".inst 0x6fa4e8f9 // udot v25.4s, v7.16b, v4.4b[3]\n" - "ldr d7, [x16, #0xf0]\n" - "mov v7.d[1], x11\n" + "movi v25.4s, #0x0\n" + "movi v26.4s, #0x0\n" + "movi v27.4s, #0x0\n" + "149:" // Height 5: setup done + "mov x15, #0x0\n" + "150:" // Height 5: String loop + "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" + "ldr w14, [x20, x15, LSL #0x2]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" + "tbz %x[flags], #3, 151f\n" + "ldr x20, [%x[input_ptr], x15, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x13, [x20, #0x0]\n" + "ldr x12, [x20, #0x8]\n" + "ldr x11, [x20, #0x10]\n" + "ldr x10, [x20, #0x18]\n" + "ldr x9, [x20, #0x20]\n" + "cbnz x15, 152f\n" + "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" + "add x13, x13, x20\n" + "add x12, x12, x20\n" + "add x11, x11, x20\n" + "add x10, x10, x20\n" + "add x9, x9, x20\n" + "b 152f\n" + "151:" // Height 5: setup direct input + "mov x13, %x[input_ptr]\n" + "add x12, x13, x21\n" + "add x11, x12, x21\n" + "add x10, x11, x21\n" + "add x9, x10, x21\n" + "152:" // Height 5: input setup done + "cmp x14, #0x10\n" + "blt 155f\n" + "ldr q0, [x13, #0x0]\n" + "cmp x14, #0x20\n" + "ldr q1, [x12, #0x0]\n" + "ldr q2, [x11, #0x0]\n" + "ldr q3, [x10, #0x0]\n" + "ldr q4, [x9, #0x0]\n" + "ldr q6, [x16, #0x0]\n" + "ldr q7, [x16, #0x10]\n" + "blt 154f\n" + "153:" // Height 5: Multiply loop: Main loop head + ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" + "ldr x21, [x16, #0x28]\n" + ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" + "ldr x20, [x16, #0x38]\n" + ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" + "add x13, x13, #0x10\n" + ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n" + "add x12, x12, #0x10\n" + ".inst 0x6f84e0d8 // udot v24.4s, v6.16b, v4.4b[0]\n" + "ldr d29, [x16, #0x20]\n" + ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" + "mov v29.d[1], x21\n" + ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" + "ldr x21, [x16, #0x48]\n" + ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" + "add x11, x11, #0x10\n" + ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n" + "add x10, x10, #0x10\n" + ".inst 0x6f84e0f9 // udot v25.4s, v7.16b, v4.4b[0]\n" + "ldr d28, [x16, #0x30]\n" + "mov v28.d[1], x20\n" + ".inst 0x6f80e3aa // udot v10.4s, v29.16b, v0.4b[0]\n" + ".inst 0x6f81e3ae // udot v14.4s, v29.16b, v1.4b[0]\n" + "ldr x20, [x16, #0x58]\n" + ".inst 0x6f82e3b2 // udot v18.4s, v29.16b, v2.4b[0]\n" + "add x9, x9, #0x10\n" + ".inst 0x6f83e3b6 // udot v22.4s, v29.16b, v3.4b[0]\n" + "ldr x26, [x13, #0x8]\n" + ".inst 0x6f84e3ba // udot v26.4s, v29.16b, v4.4b[0]\n" + "ldr d29, [x16, #0x40]\n" + ".inst 0x6f80e38b // udot v11.4s, v28.16b, v0.4b[0]\n" + "mov v29.d[1], x21\n" + ".inst 0x6f81e38f // udot v15.4s, v28.16b, v1.4b[0]\n" + "ldr x21, [x16, #0x68]\n" + ".inst 0x6f82e393 // udot v19.4s, v28.16b, v2.4b[0]\n" + "ldr x25, [x12, #0x8]\n" + ".inst 0x6f83e397 // udot v23.4s, v28.16b, v3.4b[0]\n" + "ldr x24, [x11, #0x8]\n" + ".inst 0x6f84e39b // udot v27.4s, v28.16b, v4.4b[0]\n" + "ldr d28, [x16, #0x50]\n" + "mov v28.d[1], x20\n" + ".inst 0x6fa0e3a8 // udot v8.4s, v29.16b, v0.4b[1]\n" + ".inst 0x6fa1e3ac // udot v12.4s, v29.16b, v1.4b[1]\n" + "ldr x20, [x16, #0x78]\n" + ".inst 0x6fa2e3b0 // udot v16.4s, v29.16b, v2.4b[1]\n" + "ldr x23, [x10, #0x8]\n" + ".inst 0x6fa3e3b4 // udot v20.4s, v29.16b, v3.4b[1]\n" + "ldr x22, [x9, #0x8]\n" + ".inst 0x6fa4e3b8 // udot v24.4s, v29.16b, v4.4b[1]\n" + "ldr d29, [x16, #0x60]\n" + ".inst 0x6fa0e389 // udot v9.4s, v28.16b, v0.4b[1]\n" + "mov v29.d[1], x21\n" + ".inst 0x6fa1e38d // udot v13.4s, v28.16b, v1.4b[1]\n" + "ldr x21, [x16, #0x88]\n" + ".inst 0x6fa2e391 // udot v17.4s, v28.16b, v2.4b[1]\n" + "sub x14, x14, #0x10\n" + ".inst 0x6fa3e395 // udot v21.4s, v28.16b, v3.4b[1]\n" + "cmp x14, #0x20\n" + ".inst 0x6fa4e399 // udot v25.4s, v28.16b, v4.4b[1]\n" + "ldr d28, [x16, #0x70]\n" + "mov v28.d[1], x20\n" + ".inst 0x6fa0e3aa // udot v10.4s, v29.16b, v0.4b[1]\n" + ".inst 0x6fa1e3ae // udot v14.4s, v29.16b, v1.4b[1]\n" + "ldr x20, [x16, #0x98]\n" + ".inst 0x6fa2e3b2 // udot v18.4s, v29.16b, v2.4b[1]\n" + "prfm pldl1keep, [x13, #0x80]\n" + ".inst 0x6fa3e3b6 // udot v22.4s, v29.16b, v3.4b[1]\n" + "prfm pldl1keep, [x12, #0x80]\n" + ".inst 0x6fa4e3ba // udot v26.4s, v29.16b, v4.4b[1]\n" + "ldr d29, [x16, #0x80]\n" + ".inst 0x6fa0e38b // udot v11.4s, v28.16b, v0.4b[1]\n" + "mov v29.d[1], x21\n" + ".inst 0x6fa1e38f // udot v15.4s, v28.16b, v1.4b[1]\n" + "ldr x21, [x16, #0xa8]\n" + ".inst 0x6fa2e393 // udot v19.4s, v28.16b, v2.4b[1]\n" + "prfm pldl1keep, [x11, #0x80]\n" + ".inst 0x6fa3e397 // udot v23.4s, v28.16b, v3.4b[1]\n" + "prfm pldl1keep, [x10, #0x80]\n" + ".inst 0x6fa4e39b // udot v27.4s, v28.16b, v4.4b[1]\n" + "ldr d28, [x16, #0x90]\n" + "mov v28.d[1], x20\n" + ".inst 0x6f80eba8 // udot v8.4s, v29.16b, v0.4b[2]\n" + ".inst 0x6f81ebac // udot v12.4s, v29.16b, v1.4b[2]\n" + "ldr x20, [x16, #0xb8]\n" + ".inst 0x6f82ebb0 // udot v16.4s, v29.16b, v2.4b[2]\n" + "prfm pldl1keep, [x9, #0x80]\n" + ".inst 0x6f83ebb4 // udot v20.4s, v29.16b, v3.4b[2]\n" + ".inst 0x6f84ebb8 // udot v24.4s, v29.16b, v4.4b[2]\n" + "ldr d29, [x16, #0xa0]\n" + ".inst 0x6f80eb89 // udot v9.4s, v28.16b, v0.4b[2]\n" + "mov v29.d[1], x21\n" + ".inst 0x6f81eb8d // udot v13.4s, v28.16b, v1.4b[2]\n" + "ldr x21, [x16, #0xc8]\n" + ".inst 0x6f82eb91 // udot v17.4s, v28.16b, v2.4b[2]\n" + ".inst 0x6f83eb95 // udot v21.4s, v28.16b, v3.4b[2]\n" + ".inst 0x6f84eb99 // udot v25.4s, v28.16b, v4.4b[2]\n" + "ldr d28, [x16, #0xb0]\n" + "mov v28.d[1], x20\n" + ".inst 0x6f80ebaa // udot v10.4s, v29.16b, v0.4b[2]\n" + ".inst 0x6f81ebae // udot v14.4s, v29.16b, v1.4b[2]\n" + "ldr x20, [x16, #0xd8]\n" + ".inst 0x6f82ebb2 // udot v18.4s, v29.16b, v2.4b[2]\n" + ".inst 0x6f83ebb6 // udot v22.4s, v29.16b, v3.4b[2]\n" + ".inst 0x6f84ebba // udot v26.4s, v29.16b, v4.4b[2]\n" + "ldr d29, [x16, #0xc0]\n" + ".inst 0x6f80eb8b // udot v11.4s, v28.16b, v0.4b[2]\n" + "mov v29.d[1], x21\n" + ".inst 0x6f81eb8f // udot v15.4s, v28.16b, v1.4b[2]\n" + "ldr x21, [x16, #0xe8]\n" + ".inst 0x6f82eb93 // udot v19.4s, v28.16b, v2.4b[2]\n" + ".inst 0x6f83eb97 // udot v23.4s, v28.16b, v3.4b[2]\n" + ".inst 0x6f84eb9b // udot v27.4s, v28.16b, v4.4b[2]\n" + "ldr d28, [x16, #0xd0]\n" + "mov v28.d[1], x20\n" + ".inst 0x6fa0eba8 // udot v8.4s, v29.16b, v0.4b[3]\n" + ".inst 0x6fa1ebac // udot v12.4s, v29.16b, v1.4b[3]\n" + "ldr x20, [x16, #0xf8]\n" + ".inst 0x6fa2ebb0 // udot v16.4s, v29.16b, v2.4b[3]\n" + ".inst 0x6fa3ebb4 // udot v20.4s, v29.16b, v3.4b[3]\n" + ".inst 0x6fa4ebb8 // udot v24.4s, v29.16b, v4.4b[3]\n" + "ldr d29, [x16, #0xe0]\n" + ".inst 0x6fa0eb89 // udot v9.4s, v28.16b, v0.4b[3]\n" + "mov v29.d[1], x21\n" + ".inst 0x6fa1eb8d // udot v13.4s, v28.16b, v1.4b[3]\n" + ".inst 0x6fa2eb91 // udot v17.4s, v28.16b, v2.4b[3]\n" + ".inst 0x6fa3eb95 // udot v21.4s, v28.16b, v3.4b[3]\n" + ".inst 0x6fa4eb99 // udot v25.4s, v28.16b, v4.4b[3]\n" + "ldr d28, [x16, #0xf0]\n" + "mov v28.d[1], x20\n" "add x16, x16, #0x100\n" - ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n" - "ldr x12, [x16, #0x8]\n" - ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n" - "ldr x11, [x16, #0x18]\n" - ".inst 0x6fa2e8d2 // udot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x6fa3e8d6 // udot v22.4s, v6.16b, v3.4b[3]\n" - ".inst 0x6fa4e8da // udot v26.4s, v6.16b, v4.4b[3]\n" + ".inst 0x6fa0ebaa // udot v10.4s, v29.16b, v0.4b[3]\n" + "ldr x21, [x16, #0x8]\n" + ".inst 0x6fa1ebae // udot v14.4s, v29.16b, v1.4b[3]\n" + "ldr x20, [x16, #0x18]\n" + ".inst 0x6fa2ebb2 // udot v18.4s, v29.16b, v2.4b[3]\n" + ".inst 0x6fa3ebb6 // udot v22.4s, v29.16b, v3.4b[3]\n" + ".inst 0x6fa4ebba // udot v26.4s, v29.16b, v4.4b[3]\n" "ldr d6, [x16, #0x0]\n" - ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x6fa0eb8b // udot v11.4s, v28.16b, v0.4b[3]\n" "ldr d0, [x13, #0x0]\n" - ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n" - "ldr d1, [x9, #0x0]\n" - ".inst 0x6fa2e8f3 // udot v19.4s, v7.16b, v2.4b[3]\n" - "ldr d2, [x27, #0x0]\n" - ".inst 0x6fa3e8f7 // udot v23.4s, v7.16b, v3.4b[3]\n" - "ldr d3, [x25, #0x0]\n" - ".inst 0x6fa4e8fb // udot v27.4s, v7.16b, v4.4b[3]\n" - "ldr d4, [x23, #0x0]\n" + ".inst 0x6fa1eb8f // udot v15.4s, v28.16b, v1.4b[3]\n" + "ldr d1, [x12, #0x0]\n" + ".inst 0x6fa2eb93 // udot v19.4s, v28.16b, v2.4b[3]\n" + "ldr d2, [x11, #0x0]\n" + ".inst 0x6fa3eb97 // udot v23.4s, v28.16b, v3.4b[3]\n" + "ldr d3, [x10, #0x0]\n" + ".inst 0x6fa4eb9b // udot v27.4s, v28.16b, v4.4b[3]\n" + "ldr d4, [x9, #0x0]\n" "ldr d7, [x16, #0x10]\n" - "mov v6.d[1], x12\n" - "mov v0.d[1], x10\n" - "mov v1.d[1], x28\n" - "mov v2.d[1], x26\n" - "mov v3.d[1], x24\n" + "mov v6.d[1], x21\n" + "mov v0.d[1], x26\n" + "mov v1.d[1], x25\n" + "mov v2.d[1], x24\n" + "mov v3.d[1], x23\n" "mov v4.d[1], x22\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "bge 153b\n" "154:" // Height 5: Multiply loop: Single iteration only ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" "add x13, x13, #0x10\n" ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - "add x27, x27, #0x10\n" + "add x11, x11, #0x10\n" ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n" - "add x25, x25, #0x10\n" + "add x10, x10, #0x10\n" ".inst 0x6f84e0d8 // udot v24.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x16, #0x20]\n" + "ldr q29, [x16, #0x20]\n" ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - "add x23, x23, #0x10\n" + "add x9, x9, #0x10\n" ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" "sub x14, x14, #0x10\n" ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" "prfm pldl1keep, [x13, #0x80]\n" ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n" - "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x12, #0x80]\n" ".inst 0x6f84e0f9 // udot v25.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - "prfm pldl1keep, [x27, #0x80]\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x6f84e0da // udot v26.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x16, #0x40]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x6f84e0fb // udot v27.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x16, #0x50]\n" - ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x6fa2e0d0 // udot v16.4s, v6.16b, v2.4b[1]\n" - ".inst 0x6fa3e0d4 // udot v20.4s, v6.16b, v3.4b[1]\n" - ".inst 0x6fa4e0d8 // udot v24.4s, v6.16b, v4.4b[1]\n" - "ldr q6, [x16, #0x60]\n" - ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x6fa2e0f1 // udot v17.4s, v7.16b, v2.4b[1]\n" - ".inst 0x6fa3e0f5 // udot v21.4s, v7.16b, v3.4b[1]\n" - ".inst 0x6fa4e0f9 // udot v25.4s, v7.16b, v4.4b[1]\n" - "ldr q7, [x16, #0x70]\n" - ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x6fa2e0d2 // udot v18.4s, v6.16b, v2.4b[1]\n" - ".inst 0x6fa3e0d6 // udot v22.4s, v6.16b, v3.4b[1]\n" - ".inst 0x6fa4e0da // udot v26.4s, v6.16b, v4.4b[1]\n" - "ldr q6, [x16, #0x80]\n" - ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x6fa2e0f3 // udot v19.4s, v7.16b, v2.4b[1]\n" - ".inst 0x6fa3e0f7 // udot v23.4s, v7.16b, v3.4b[1]\n" - ".inst 0x6fa4e0fb // udot v27.4s, v7.16b, v4.4b[1]\n" - "ldr q7, [x16, #0x90]\n" - ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x6f82e8d0 // udot v16.4s, v6.16b, v2.4b[2]\n" - ".inst 0x6f83e8d4 // udot v20.4s, v6.16b, v3.4b[2]\n" - ".inst 0x6f84e8d8 // udot v24.4s, v6.16b, v4.4b[2]\n" - "ldr q6, [x16, #0xa0]\n" - ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f82e8f1 // udot v17.4s, v7.16b, v2.4b[2]\n" - ".inst 0x6f83e8f5 // udot v21.4s, v7.16b, v3.4b[2]\n" - ".inst 0x6f84e8f9 // udot v25.4s, v7.16b, v4.4b[2]\n" - "ldr q7, [x16, #0xb0]\n" - ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x6f82e8d2 // udot v18.4s, v6.16b, v2.4b[2]\n" - ".inst 0x6f83e8d6 // udot v22.4s, v6.16b, v3.4b[2]\n" - ".inst 0x6f84e8da // udot v26.4s, v6.16b, v4.4b[2]\n" - "ldr q6, [x16, #0xc0]\n" - ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f82e8f3 // udot v19.4s, v7.16b, v2.4b[2]\n" - ".inst 0x6f83e8f7 // udot v23.4s, v7.16b, v3.4b[2]\n" - ".inst 0x6f84e8fb // udot v27.4s, v7.16b, v4.4b[2]\n" - "ldr q7, [x16, #0xd0]\n" - ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x6fa2e8d0 // udot v16.4s, v6.16b, v2.4b[3]\n" - ".inst 0x6fa3e8d4 // udot v20.4s, v6.16b, v3.4b[3]\n" - ".inst 0x6fa4e8d8 // udot v24.4s, v6.16b, v4.4b[3]\n" - "ldr q6, [x16, #0xe0]\n" - ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x6fa2e8f1 // udot v17.4s, v7.16b, v2.4b[3]\n" - ".inst 0x6fa3e8f5 // udot v21.4s, v7.16b, v3.4b[3]\n" - ".inst 0x6fa4e8f9 // udot v25.4s, v7.16b, v4.4b[3]\n" - "ldr q7, [x16, #0xf0]\n" - ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n" + "ldr q28, [x16, #0x30]\n" + ".inst 0x6f80e3aa // udot v10.4s, v29.16b, v0.4b[0]\n" + "prfm pldl1keep, [x11, #0x80]\n" + ".inst 0x6f81e3ae // udot v14.4s, v29.16b, v1.4b[0]\n" + "prfm pldl1keep, [x10, #0x80]\n" + ".inst 0x6f82e3b2 // udot v18.4s, v29.16b, v2.4b[0]\n" + "prfm pldl1keep, [x9, #0x80]\n" + ".inst 0x6f83e3b6 // udot v22.4s, v29.16b, v3.4b[0]\n" + ".inst 0x6f84e3ba // udot v26.4s, v29.16b, v4.4b[0]\n" + "ldr q29, [x16, #0x40]\n" + ".inst 0x6f80e38b // udot v11.4s, v28.16b, v0.4b[0]\n" + ".inst 0x6f81e38f // udot v15.4s, v28.16b, v1.4b[0]\n" + ".inst 0x6f82e393 // udot v19.4s, v28.16b, v2.4b[0]\n" + ".inst 0x6f83e397 // udot v23.4s, v28.16b, v3.4b[0]\n" + ".inst 0x6f84e39b // udot v27.4s, v28.16b, v4.4b[0]\n" + "ldr q28, [x16, #0x50]\n" + ".inst 0x6fa0e3a8 // udot v8.4s, v29.16b, v0.4b[1]\n" + ".inst 0x6fa1e3ac // udot v12.4s, v29.16b, v1.4b[1]\n" + ".inst 0x6fa2e3b0 // udot v16.4s, v29.16b, v2.4b[1]\n" + ".inst 0x6fa3e3b4 // udot v20.4s, v29.16b, v3.4b[1]\n" + ".inst 0x6fa4e3b8 // udot v24.4s, v29.16b, v4.4b[1]\n" + "ldr q29, [x16, #0x60]\n" + ".inst 0x6fa0e389 // udot v9.4s, v28.16b, v0.4b[1]\n" + ".inst 0x6fa1e38d // udot v13.4s, v28.16b, v1.4b[1]\n" + ".inst 0x6fa2e391 // udot v17.4s, v28.16b, v2.4b[1]\n" + ".inst 0x6fa3e395 // udot v21.4s, v28.16b, v3.4b[1]\n" + ".inst 0x6fa4e399 // udot v25.4s, v28.16b, v4.4b[1]\n" + "ldr q28, [x16, #0x70]\n" + ".inst 0x6fa0e3aa // udot v10.4s, v29.16b, v0.4b[1]\n" + ".inst 0x6fa1e3ae // udot v14.4s, v29.16b, v1.4b[1]\n" + ".inst 0x6fa2e3b2 // udot v18.4s, v29.16b, v2.4b[1]\n" + ".inst 0x6fa3e3b6 // udot v22.4s, v29.16b, v3.4b[1]\n" + ".inst 0x6fa4e3ba // udot v26.4s, v29.16b, v4.4b[1]\n" + "ldr q29, [x16, #0x80]\n" + ".inst 0x6fa0e38b // udot v11.4s, v28.16b, v0.4b[1]\n" + ".inst 0x6fa1e38f // udot v15.4s, v28.16b, v1.4b[1]\n" + ".inst 0x6fa2e393 // udot v19.4s, v28.16b, v2.4b[1]\n" + ".inst 0x6fa3e397 // udot v23.4s, v28.16b, v3.4b[1]\n" + ".inst 0x6fa4e39b // udot v27.4s, v28.16b, v4.4b[1]\n" + "ldr q28, [x16, #0x90]\n" + ".inst 0x6f80eba8 // udot v8.4s, v29.16b, v0.4b[2]\n" + ".inst 0x6f81ebac // udot v12.4s, v29.16b, v1.4b[2]\n" + ".inst 0x6f82ebb0 // udot v16.4s, v29.16b, v2.4b[2]\n" + ".inst 0x6f83ebb4 // udot v20.4s, v29.16b, v3.4b[2]\n" + ".inst 0x6f84ebb8 // udot v24.4s, v29.16b, v4.4b[2]\n" + "ldr q29, [x16, #0xa0]\n" + ".inst 0x6f80eb89 // udot v9.4s, v28.16b, v0.4b[2]\n" + ".inst 0x6f81eb8d // udot v13.4s, v28.16b, v1.4b[2]\n" + ".inst 0x6f82eb91 // udot v17.4s, v28.16b, v2.4b[2]\n" + ".inst 0x6f83eb95 // udot v21.4s, v28.16b, v3.4b[2]\n" + ".inst 0x6f84eb99 // udot v25.4s, v28.16b, v4.4b[2]\n" + "ldr q28, [x16, #0xb0]\n" + ".inst 0x6f80ebaa // udot v10.4s, v29.16b, v0.4b[2]\n" + ".inst 0x6f81ebae // udot v14.4s, v29.16b, v1.4b[2]\n" + ".inst 0x6f82ebb2 // udot v18.4s, v29.16b, v2.4b[2]\n" + ".inst 0x6f83ebb6 // udot v22.4s, v29.16b, v3.4b[2]\n" + ".inst 0x6f84ebba // udot v26.4s, v29.16b, v4.4b[2]\n" + "ldr q29, [x16, #0xc0]\n" + ".inst 0x6f80eb8b // udot v11.4s, v28.16b, v0.4b[2]\n" + ".inst 0x6f81eb8f // udot v15.4s, v28.16b, v1.4b[2]\n" + ".inst 0x6f82eb93 // udot v19.4s, v28.16b, v2.4b[2]\n" + ".inst 0x6f83eb97 // udot v23.4s, v28.16b, v3.4b[2]\n" + ".inst 0x6f84eb9b // udot v27.4s, v28.16b, v4.4b[2]\n" + "ldr q28, [x16, #0xd0]\n" + ".inst 0x6fa0eba8 // udot v8.4s, v29.16b, v0.4b[3]\n" + ".inst 0x6fa1ebac // udot v12.4s, v29.16b, v1.4b[3]\n" + ".inst 0x6fa2ebb0 // udot v16.4s, v29.16b, v2.4b[3]\n" + ".inst 0x6fa3ebb4 // udot v20.4s, v29.16b, v3.4b[3]\n" + ".inst 0x6fa4ebb8 // udot v24.4s, v29.16b, v4.4b[3]\n" + "ldr q29, [x16, #0xe0]\n" + ".inst 0x6fa0eb89 // udot v9.4s, v28.16b, v0.4b[3]\n" + ".inst 0x6fa1eb8d // udot v13.4s, v28.16b, v1.4b[3]\n" + ".inst 0x6fa2eb91 // udot v17.4s, v28.16b, v2.4b[3]\n" + ".inst 0x6fa3eb95 // udot v21.4s, v28.16b, v3.4b[3]\n" + ".inst 0x6fa4eb99 // udot v25.4s, v28.16b, v4.4b[3]\n" + "ldr q28, [x16, #0xf0]\n" + ".inst 0x6fa0ebaa // udot v10.4s, v29.16b, v0.4b[3]\n" "add x16, x16, #0x100\n" - ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x6fa2e8d2 // udot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x6fa3e8d6 // udot v22.4s, v6.16b, v3.4b[3]\n" - ".inst 0x6fa4e8da // udot v26.4s, v6.16b, v4.4b[3]\n" - ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n" - ".inst 0x6fa2e8f3 // udot v19.4s, v7.16b, v2.4b[3]\n" - ".inst 0x6fa3e8f7 // udot v23.4s, v7.16b, v3.4b[3]\n" - ".inst 0x6fa4e8fb // udot v27.4s, v7.16b, v4.4b[3]\n" + ".inst 0x6fa1ebae // udot v14.4s, v29.16b, v1.4b[3]\n" + ".inst 0x6fa2ebb2 // udot v18.4s, v29.16b, v2.4b[3]\n" + ".inst 0x6fa3ebb6 // udot v22.4s, v29.16b, v3.4b[3]\n" + ".inst 0x6fa4ebba // udot v26.4s, v29.16b, v4.4b[3]\n" + ".inst 0x6fa0eb8b // udot v11.4s, v28.16b, v0.4b[3]\n" + ".inst 0x6fa1eb8f // udot v15.4s, v28.16b, v1.4b[3]\n" + ".inst 0x6fa2eb93 // udot v19.4s, v28.16b, v2.4b[3]\n" + ".inst 0x6fa3eb97 // udot v23.4s, v28.16b, v3.4b[3]\n" + ".inst 0x6fa4eb9b // udot v27.4s, v28.16b, v4.4b[3]\n" "155:" // Height 5: Multiply loop: Main loop skip "cbz x14, 160f\n" "cmp x14, #0x4\n" "blt 157f\n" "156:" // Height 5: Multiply loop: Odd block loop - "ldr s0, [x13], #0x4\n" + "ldr s2, [x13], #0x4\n" "sub x14, x14, #0x4\n" - "ldr s1, [x9], #0x4\n" + "ldr s1, [x12], #0x4\n" "cmp x14, #0x4\n" - "ldr s2, [x27], #0x4\n" - "ldr s3, [x25], #0x4\n" - "ldr s4, [x23], #0x4\n" - "ldr q6, [x16, #0x0]\n" - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x10]\n" - ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n" - ".inst 0x6f84e0d8 // udot v24.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x16, #0x20]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x6f84e0f9 // udot v25.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" + "ldr s0, [x11], #0x4\n" + "ldr s31, [x10], #0x4\n" + "ldr s30, [x9], #0x4\n" + "ldr q29, [x16, #0x0]\n" + ".inst 0x6f82e3a8 // udot v8.4s, v29.16b, v2.4b[0]\n" + "ldr q28, [x16, #0x10]\n" + ".inst 0x6f81e3ac // udot v12.4s, v29.16b, v1.4b[0]\n" + ".inst 0x6f80e3b0 // udot v16.4s, v29.16b, v0.4b[0]\n" + ".inst 0x6f9fe3b4 // udot v20.4s, v29.16b, v31.4b[0]\n" + ".inst 0x6f9ee3b8 // udot v24.4s, v29.16b, v30.4b[0]\n" + "ldr q29, [x16, #0x20]\n" + ".inst 0x6f82e389 // udot v9.4s, v28.16b, v2.4b[0]\n" + ".inst 0x6f81e38d // udot v13.4s, v28.16b, v1.4b[0]\n" + ".inst 0x6f80e391 // udot v17.4s, v28.16b, v0.4b[0]\n" + ".inst 0x6f9fe395 // udot v21.4s, v28.16b, v31.4b[0]\n" + ".inst 0x6f9ee399 // udot v25.4s, v28.16b, v30.4b[0]\n" + "ldr q28, [x16, #0x30]\n" + ".inst 0x6f82e3aa // udot v10.4s, v29.16b, v2.4b[0]\n" "add x16, x16, #0x40\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x6f84e0da // udot v26.4s, v6.16b, v4.4b[0]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x6f84e0fb // udot v27.4s, v7.16b, v4.4b[0]\n" + ".inst 0x6f81e3ae // udot v14.4s, v29.16b, v1.4b[0]\n" + ".inst 0x6f80e3b2 // udot v18.4s, v29.16b, v0.4b[0]\n" + ".inst 0x6f9fe3b6 // udot v22.4s, v29.16b, v31.4b[0]\n" + ".inst 0x6f9ee3ba // udot v26.4s, v29.16b, v30.4b[0]\n" + ".inst 0x6f82e38b // udot v11.4s, v28.16b, v2.4b[0]\n" + ".inst 0x6f81e38f // udot v15.4s, v28.16b, v1.4b[0]\n" + ".inst 0x6f80e393 // udot v19.4s, v28.16b, v0.4b[0]\n" + ".inst 0x6f9fe397 // udot v23.4s, v28.16b, v31.4b[0]\n" + ".inst 0x6f9ee39b // udot v27.4s, v28.16b, v30.4b[0]\n" "bge 156b\n" "157:" // Height 5: Multiply loop: Skip odd blocks "cbz x14, 160f\n" "tbz x14, #1, 158f\n" "ldr h0, [x13], #0x2\n" - "ldr h1, [x9], #0x2\n" - "ldr h2, [x27], #0x2\n" - "ldr h3, [x25], #0x2\n" - "ldr h4, [x23], #0x2\n" + "ldr h1, [x12], #0x2\n" + "ldr h2, [x11], #0x2\n" + "ldr h3, [x10], #0x2\n" + "ldr h4, [x9], #0x2\n" "tbz x14, #0, 159f\n" "ld1 { v0.b }[2], [x13]\n" - "ld1 { v1.b }[2], [x9]\n" - "ld1 { v2.b }[2], [x27]\n" - "ld1 { v3.b }[2], [x25]\n" - "ld1 { v4.b }[2], [x23]\n" + "ld1 { v1.b }[2], [x12]\n" + "ld1 { v2.b }[2], [x11]\n" + "ld1 { v3.b }[2], [x10]\n" + "ld1 { v4.b }[2], [x9]\n" "b 159f\n" "158:" // Height 5: Multiply loop: Ragged operand read: partial_1_0 "ldr b0, [x13, #0x0]\n" - "ldr b1, [x9, #0x0]\n" - "ldr b2, [x27, #0x0]\n" - "ldr b3, [x25, #0x0]\n" - "ldr b4, [x23, #0x0]\n" + "ldr b1, [x12, #0x0]\n" + "ldr b2, [x11, #0x0]\n" + "ldr b3, [x10, #0x0]\n" + "ldr b4, [x9, #0x0]\n" "159:" // Height 5: Multiply loop: Ragged operand read: Done - "ldr q6, [x16, #0x0]\n" - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x10]\n" - ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n" - ".inst 0x6f84e0d8 // udot v24.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x16, #0x20]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x6f84e0f9 // udot v25.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q29, [x16, #0x0]\n" + ".inst 0x6f80e3a8 // udot v8.4s, v29.16b, v0.4b[0]\n" + "ldr q28, [x16, #0x10]\n" + ".inst 0x6f81e3ac // udot v12.4s, v29.16b, v1.4b[0]\n" + ".inst 0x6f82e3b0 // udot v16.4s, v29.16b, v2.4b[0]\n" + ".inst 0x6f83e3b4 // udot v20.4s, v29.16b, v3.4b[0]\n" + ".inst 0x6f84e3b8 // udot v24.4s, v29.16b, v4.4b[0]\n" + "ldr q29, [x16, #0x20]\n" + ".inst 0x6f80e389 // udot v9.4s, v28.16b, v0.4b[0]\n" + ".inst 0x6f81e38d // udot v13.4s, v28.16b, v1.4b[0]\n" + ".inst 0x6f82e391 // udot v17.4s, v28.16b, v2.4b[0]\n" + ".inst 0x6f83e395 // udot v21.4s, v28.16b, v3.4b[0]\n" + ".inst 0x6f84e399 // udot v25.4s, v28.16b, v4.4b[0]\n" + "ldr q28, [x16, #0x30]\n" + ".inst 0x6f80e3aa // udot v10.4s, v29.16b, v0.4b[0]\n" "add x16, x16, #0x40\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x6f84e0da // udot v26.4s, v6.16b, v4.4b[0]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x6f84e0fb // udot v27.4s, v7.16b, v4.4b[0]\n" + ".inst 0x6f81e3ae // udot v14.4s, v29.16b, v1.4b[0]\n" + ".inst 0x6f82e3b2 // udot v18.4s, v29.16b, v2.4b[0]\n" + ".inst 0x6f83e3b6 // udot v22.4s, v29.16b, v3.4b[0]\n" + ".inst 0x6f84e3ba // udot v26.4s, v29.16b, v4.4b[0]\n" + ".inst 0x6f80e38b // udot v11.4s, v28.16b, v0.4b[0]\n" + ".inst 0x6f81e38f // udot v15.4s, v28.16b, v1.4b[0]\n" + ".inst 0x6f82e393 // udot v19.4s, v28.16b, v2.4b[0]\n" + ".inst 0x6f83e397 // udot v23.4s, v28.16b, v3.4b[0]\n" + ".inst 0x6f84e39b // udot v27.4s, v28.16b, v4.4b[0]\n" "160:" // Height 5: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x15, x15, #0x1\n" @@ -2862,98 +2861,98 @@ void a64_hybrid_u8u32_dot_6x16_a55 ( "184:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w14, [x20, x15, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 185f\n" - "ldr x21, [%x[input_ptr], x15, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x13, [x21, #0x0]\n" - "ldr x9, [x21, #0x8]\n" - "ldr x27, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" - "ldr x23, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x15, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x13, [x20, #0x0]\n" + "ldr x12, [x20, #0x8]\n" + "ldr x11, [x20, #0x10]\n" + "ldr x10, [x20, #0x18]\n" + "ldr x9, [x20, #0x20]\n" + "ldr x28, [x20, #0x28]\n" "cbnz x15, 186f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x13, x13, x20\n" + "add x12, x12, x20\n" + "add x11, x11, x20\n" + "add x10, x10, x20\n" "add x9, x9, x20\n" - "add x27, x27, x20\n" - "add x25, x25, x20\n" - "add x23, x23, x20\n" - "add x21, x21, x20\n" + "add x28, x28, x20\n" "b 186f\n" "185:" // Height 6: setup direct input "mov x13, %x[input_ptr]\n" - "add x9, x13, x20\n" - "add x27, x9, x20\n" - "add x25, x27, x20\n" - "add x23, x25, x20\n" - "add x21, x23, x20\n" + "add x12, x13, x21\n" + "add x11, x12, x21\n" + "add x10, x11, x21\n" + "add x9, x10, x21\n" + "add x28, x9, x21\n" "186:" // Height 6: input setup done "cmp x14, #0x10\n" "blt 189f\n" "ldr q0, [x13, #0x0]\n" "cmp x14, #0x20\n" - "ldr q1, [x9, #0x0]\n" - "ldr q2, [x27, #0x0]\n" - "ldr q3, [x25, #0x0]\n" - "ldr q4, [x23, #0x0]\n" - "ldr q5, [x21, #0x0]\n" + "ldr q1, [x12, #0x0]\n" + "ldr q2, [x11, #0x0]\n" + "ldr q3, [x10, #0x0]\n" + "ldr q4, [x9, #0x0]\n" + "ldr q5, [x28, #0x0]\n" "ldr q6, [x16, #0x0]\n" "ldr q7, [x16, #0x10]\n" "blt 188f\n" "187:" // Height 6: Multiply loop: Main loop head ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - "ldr x12, [x16, #0x28]\n" + "ldr x21, [x16, #0x28]\n" ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - "ldr x11, [x16, #0x38]\n" + "ldr x20, [x16, #0x38]\n" ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" "add x13, x13, #0x10\n" ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" ".inst 0x6f84e0d8 // udot v24.4s, v6.16b, v4.4b[0]\n" - "add x27, x27, #0x10\n" + "add x11, x11, #0x10\n" ".inst 0x6f85e0dc // udot v28.4s, v6.16b, v5.4b[0]\n" "ldr d6, [x16, #0x20]\n" ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - "ldr x12, [x16, #0x48]\n" + "ldr x21, [x16, #0x48]\n" ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" - "add x25, x25, #0x10\n" + "add x10, x10, #0x10\n" ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n" - "add x23, x23, #0x10\n" + "add x9, x9, #0x10\n" ".inst 0x6f84e0f9 // udot v25.4s, v7.16b, v4.4b[0]\n" - "add x21, x21, #0x10\n" + "add x28, x28, #0x10\n" ".inst 0x6f85e0fd // udot v29.4s, v7.16b, v5.4b[0]\n" "ldr d7, [x16, #0x30]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - "ldr x11, [x16, #0x58]\n" + "ldr x20, [x16, #0x58]\n" ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - "ldr x10, [x13, #0x8]\n" + "ldr x27, [x13, #0x8]\n" ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n" - "ldr x28, [x9, #0x8]\n" + "ldr x26, [x12, #0x8]\n" ".inst 0x6f84e0da // udot v26.4s, v6.16b, v4.4b[0]\n" - "ldr x26, [x27, #0x8]\n" + "ldr x25, [x11, #0x8]\n" ".inst 0x6f85e0de // udot v30.4s, v6.16b, v5.4b[0]\n" "ldr d6, [x16, #0x40]\n" ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - "ldr x12, [x16, #0x68]\n" + "ldr x21, [x16, #0x68]\n" ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - "ldr x24, [x25, #0x8]\n" + "ldr x24, [x10, #0x8]\n" ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n" - "ldr x22, [x23, #0x8]\n" + "ldr x23, [x9, #0x8]\n" ".inst 0x6f84e0fb // udot v27.4s, v7.16b, v4.4b[0]\n" - "ldr x20, [x21, #0x8]\n" + "ldr x22, [x28, #0x8]\n" ".inst 0x6f85e0ff // udot v31.4s, v7.16b, v5.4b[0]\n" "ldr d7, [x16, #0x50]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n" ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n" - "ldr x11, [x16, #0x78]\n" + "ldr x20, [x16, #0x78]\n" ".inst 0x6fa2e0d0 // udot v16.4s, v6.16b, v2.4b[1]\n" "sub x14, x14, #0x10\n" ".inst 0x6fa3e0d4 // udot v20.4s, v6.16b, v3.4b[1]\n" @@ -2963,96 +2962,96 @@ void a64_hybrid_u8u32_dot_6x16_a55 ( ".inst 0x6fa5e0dc // udot v28.4s, v6.16b, v5.4b[1]\n" "ldr d6, [x16, #0x60]\n" ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n" - "ldr x12, [x16, #0x88]\n" + "ldr x21, [x16, #0x88]\n" ".inst 0x6fa2e0f1 // udot v17.4s, v7.16b, v2.4b[1]\n" - "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x12, #0x80]\n" ".inst 0x6fa3e0f5 // udot v21.4s, v7.16b, v3.4b[1]\n" - "prfm pldl1keep, [x27, #0x80]\n" + "prfm pldl1keep, [x11, #0x80]\n" ".inst 0x6fa4e0f9 // udot v25.4s, v7.16b, v4.4b[1]\n" - "prfm pldl1keep, [x25, #0x80]\n" + "prfm pldl1keep, [x10, #0x80]\n" ".inst 0x6fa5e0fd // udot v29.4s, v7.16b, v5.4b[1]\n" "ldr d7, [x16, #0x70]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n" ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n" - "ldr x11, [x16, #0x98]\n" + "ldr x20, [x16, #0x98]\n" ".inst 0x6fa2e0d2 // udot v18.4s, v6.16b, v2.4b[1]\n" - "prfm pldl1keep, [x23, #0x80]\n" + "prfm pldl1keep, [x9, #0x80]\n" ".inst 0x6fa3e0d6 // udot v22.4s, v6.16b, v3.4b[1]\n" - "prfm pldl1keep, [x21, #0x80]\n" + "prfm pldl1keep, [x28, #0x80]\n" ".inst 0x6fa4e0da // udot v26.4s, v6.16b, v4.4b[1]\n" ".inst 0x6fa5e0de // udot v30.4s, v6.16b, v5.4b[1]\n" "ldr d6, [x16, #0x80]\n" ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n" - "ldr x12, [x16, #0xa8]\n" + "ldr x21, [x16, #0xa8]\n" ".inst 0x6fa2e0f3 // udot v19.4s, v7.16b, v2.4b[1]\n" ".inst 0x6fa3e0f7 // udot v23.4s, v7.16b, v3.4b[1]\n" ".inst 0x6fa4e0fb // udot v27.4s, v7.16b, v4.4b[1]\n" ".inst 0x6fa5e0ff // udot v31.4s, v7.16b, v5.4b[1]\n" "ldr d7, [x16, #0x90]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n" ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n" - "ldr x11, [x16, #0xb8]\n" + "ldr x20, [x16, #0xb8]\n" ".inst 0x6f82e8d0 // udot v16.4s, v6.16b, v2.4b[2]\n" ".inst 0x6f83e8d4 // udot v20.4s, v6.16b, v3.4b[2]\n" ".inst 0x6f84e8d8 // udot v24.4s, v6.16b, v4.4b[2]\n" ".inst 0x6f85e8dc // udot v28.4s, v6.16b, v5.4b[2]\n" "ldr d6, [x16, #0xa0]\n" ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n" - "ldr x12, [x16, #0xc8]\n" + "ldr x21, [x16, #0xc8]\n" ".inst 0x6f82e8f1 // udot v17.4s, v7.16b, v2.4b[2]\n" ".inst 0x6f83e8f5 // udot v21.4s, v7.16b, v3.4b[2]\n" ".inst 0x6f84e8f9 // udot v25.4s, v7.16b, v4.4b[2]\n" ".inst 0x6f85e8fd // udot v29.4s, v7.16b, v5.4b[2]\n" "ldr d7, [x16, #0xb0]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n" ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n" - "ldr x11, [x16, #0xd8]\n" + "ldr x20, [x16, #0xd8]\n" ".inst 0x6f82e8d2 // udot v18.4s, v6.16b, v2.4b[2]\n" ".inst 0x6f83e8d6 // udot v22.4s, v6.16b, v3.4b[2]\n" ".inst 0x6f84e8da // udot v26.4s, v6.16b, v4.4b[2]\n" ".inst 0x6f85e8de // udot v30.4s, v6.16b, v5.4b[2]\n" "ldr d6, [x16, #0xc0]\n" ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n" - "ldr x12, [x16, #0xe8]\n" + "ldr x21, [x16, #0xe8]\n" ".inst 0x6f82e8f3 // udot v19.4s, v7.16b, v2.4b[2]\n" ".inst 0x6f83e8f7 // udot v23.4s, v7.16b, v3.4b[2]\n" ".inst 0x6f84e8fb // udot v27.4s, v7.16b, v4.4b[2]\n" ".inst 0x6f85e8ff // udot v31.4s, v7.16b, v5.4b[2]\n" "ldr d7, [x16, #0xd0]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n" ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n" - "ldr x11, [x16, #0xf8]\n" + "ldr x20, [x16, #0xf8]\n" ".inst 0x6fa2e8d0 // udot v16.4s, v6.16b, v2.4b[3]\n" ".inst 0x6fa3e8d4 // udot v20.4s, v6.16b, v3.4b[3]\n" ".inst 0x6fa4e8d8 // udot v24.4s, v6.16b, v4.4b[3]\n" ".inst 0x6fa5e8dc // udot v28.4s, v6.16b, v5.4b[3]\n" "ldr d6, [x16, #0xe0]\n" ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n" - "mov v6.d[1], x12\n" + "mov v6.d[1], x21\n" ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n" ".inst 0x6fa2e8f1 // udot v17.4s, v7.16b, v2.4b[3]\n" ".inst 0x6fa3e8f5 // udot v21.4s, v7.16b, v3.4b[3]\n" ".inst 0x6fa4e8f9 // udot v25.4s, v7.16b, v4.4b[3]\n" ".inst 0x6fa5e8fd // udot v29.4s, v7.16b, v5.4b[3]\n" "ldr d7, [x16, #0xf0]\n" - "mov v7.d[1], x11\n" + "mov v7.d[1], x20\n" "add x16, x16, #0x100\n" ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n" - "ldr x12, [x16, #0x8]\n" + "ldr x21, [x16, #0x8]\n" ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n" - "ldr x11, [x16, #0x18]\n" + "ldr x20, [x16, #0x18]\n" ".inst 0x6fa2e8d2 // udot v18.4s, v6.16b, v2.4b[3]\n" ".inst 0x6fa3e8d6 // udot v22.4s, v6.16b, v3.4b[3]\n" ".inst 0x6fa4e8da // udot v26.4s, v6.16b, v4.4b[3]\n" @@ -3061,56 +3060,56 @@ void a64_hybrid_u8u32_dot_6x16_a55 ( ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n" "ldr d0, [x13, #0x0]\n" ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n" - "ldr d1, [x9, #0x0]\n" + "ldr d1, [x12, #0x0]\n" ".inst 0x6fa2e8f3 // udot v19.4s, v7.16b, v2.4b[3]\n" - "ldr d2, [x27, #0x0]\n" + "ldr d2, [x11, #0x0]\n" ".inst 0x6fa3e8f7 // udot v23.4s, v7.16b, v3.4b[3]\n" - "ldr d3, [x25, #0x0]\n" + "ldr d3, [x10, #0x0]\n" ".inst 0x6fa4e8fb // udot v27.4s, v7.16b, v4.4b[3]\n" - "ldr d4, [x23, #0x0]\n" + "ldr d4, [x9, #0x0]\n" ".inst 0x6fa5e8ff // udot v31.4s, v7.16b, v5.4b[3]\n" - "ldr d5, [x21, #0x0]\n" + "ldr d5, [x28, #0x0]\n" "ldr d7, [x16, #0x10]\n" - "mov v6.d[1], x12\n" - "mov v0.d[1], x10\n" - "mov v1.d[1], x28\n" - "mov v2.d[1], x26\n" + "mov v6.d[1], x21\n" + "mov v0.d[1], x27\n" + "mov v1.d[1], x26\n" + "mov v2.d[1], x25\n" "mov v3.d[1], x24\n" - "mov v4.d[1], x22\n" - "mov v5.d[1], x20\n" - "mov v7.d[1], x11\n" + "mov v4.d[1], x23\n" + "mov v5.d[1], x22\n" + "mov v7.d[1], x20\n" "bge 187b\n" "188:" // Height 6: Multiply loop: Single iteration only ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" "add x13, x13, #0x10\n" ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - "add x9, x9, #0x10\n" + "add x12, x12, #0x10\n" ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - "add x27, x27, #0x10\n" + "add x11, x11, #0x10\n" ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n" - "add x25, x25, #0x10\n" + "add x10, x10, #0x10\n" ".inst 0x6f84e0d8 // udot v24.4s, v6.16b, v4.4b[0]\n" - "add x23, x23, #0x10\n" + "add x9, x9, #0x10\n" ".inst 0x6f85e0dc // udot v28.4s, v6.16b, v5.4b[0]\n" "ldr q6, [x16, #0x20]\n" ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - "add x21, x21, #0x10\n" + "add x28, x28, #0x10\n" ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" "sub x14, x14, #0x10\n" ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" "prfm pldl1keep, [x13, #0x80]\n" ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n" - "prfm pldl1keep, [x9, #0x80]\n" + "prfm pldl1keep, [x12, #0x80]\n" ".inst 0x6f84e0f9 // udot v25.4s, v7.16b, v4.4b[0]\n" - "prfm pldl1keep, [x27, #0x80]\n" + "prfm pldl1keep, [x11, #0x80]\n" ".inst 0x6f85e0fd // udot v29.4s, v7.16b, v5.4b[0]\n" "ldr q7, [x16, #0x30]\n" ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - "prfm pldl1keep, [x25, #0x80]\n" + "prfm pldl1keep, [x10, #0x80]\n" ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - "prfm pldl1keep, [x23, #0x80]\n" + "prfm pldl1keep, [x9, #0x80]\n" ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - "prfm pldl1keep, [x21, #0x80]\n" + "prfm pldl1keep, [x28, #0x80]\n" ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n" ".inst 0x6f84e0da // udot v26.4s, v6.16b, v4.4b[0]\n" ".inst 0x6f85e0de // udot v30.4s, v6.16b, v5.4b[0]\n" @@ -3210,98 +3209,98 @@ void a64_hybrid_u8u32_dot_6x16_a55 ( "cmp x14, #0x4\n" "blt 191f\n" "190:" // Height 6: Multiply loop: Odd block loop - "ldr s0, [x13], #0x4\n" + "ldr s7, [x13], #0x4\n" "sub x14, x14, #0x4\n" - "ldr s1, [x9], #0x4\n" + "ldr s6, [x12], #0x4\n" "cmp x14, #0x4\n" - "ldr s2, [x27], #0x4\n" - "ldr s3, [x25], #0x4\n" - "ldr s4, [x23], #0x4\n" - "ldr s5, [x21], #0x4\n" - "ldr q6, [x16, #0x0]\n" - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x10]\n" - ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n" - ".inst 0x6f84e0d8 // udot v24.4s, v6.16b, v4.4b[0]\n" - ".inst 0x6f85e0dc // udot v28.4s, v6.16b, v5.4b[0]\n" - "ldr q6, [x16, #0x20]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x6f84e0f9 // udot v25.4s, v7.16b, v4.4b[0]\n" - ".inst 0x6f85e0fd // udot v29.4s, v7.16b, v5.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" + "ldr s5, [x11], #0x4\n" + "ldr s4, [x10], #0x4\n" + "ldr s3, [x9], #0x4\n" + "ldr s2, [x28], #0x4\n" + "ldr q1, [x16, #0x0]\n" + ".inst 0x6f87e028 // udot v8.4s, v1.16b, v7.4b[0]\n" + "ldr q0, [x16, #0x10]\n" + ".inst 0x6f86e02c // udot v12.4s, v1.16b, v6.4b[0]\n" + ".inst 0x6f85e030 // udot v16.4s, v1.16b, v5.4b[0]\n" + ".inst 0x6f84e034 // udot v20.4s, v1.16b, v4.4b[0]\n" + ".inst 0x6f83e038 // udot v24.4s, v1.16b, v3.4b[0]\n" + ".inst 0x6f82e03c // udot v28.4s, v1.16b, v2.4b[0]\n" + "ldr q1, [x16, #0x20]\n" + ".inst 0x6f87e009 // udot v9.4s, v0.16b, v7.4b[0]\n" + ".inst 0x6f86e00d // udot v13.4s, v0.16b, v6.4b[0]\n" + ".inst 0x6f85e011 // udot v17.4s, v0.16b, v5.4b[0]\n" + ".inst 0x6f84e015 // udot v21.4s, v0.16b, v4.4b[0]\n" + ".inst 0x6f83e019 // udot v25.4s, v0.16b, v3.4b[0]\n" + ".inst 0x6f82e01d // udot v29.4s, v0.16b, v2.4b[0]\n" + "ldr q0, [x16, #0x30]\n" + ".inst 0x6f87e02a // udot v10.4s, v1.16b, v7.4b[0]\n" "add x16, x16, #0x40\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x6f84e0da // udot v26.4s, v6.16b, v4.4b[0]\n" - ".inst 0x6f85e0de // udot v30.4s, v6.16b, v5.4b[0]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x6f84e0fb // udot v27.4s, v7.16b, v4.4b[0]\n" - ".inst 0x6f85e0ff // udot v31.4s, v7.16b, v5.4b[0]\n" + ".inst 0x6f86e02e // udot v14.4s, v1.16b, v6.4b[0]\n" + ".inst 0x6f85e032 // udot v18.4s, v1.16b, v5.4b[0]\n" + ".inst 0x6f84e036 // udot v22.4s, v1.16b, v4.4b[0]\n" + ".inst 0x6f83e03a // udot v26.4s, v1.16b, v3.4b[0]\n" + ".inst 0x6f82e03e // udot v30.4s, v1.16b, v2.4b[0]\n" + ".inst 0x6f87e00b // udot v11.4s, v0.16b, v7.4b[0]\n" + ".inst 0x6f86e00f // udot v15.4s, v0.16b, v6.4b[0]\n" + ".inst 0x6f85e013 // udot v19.4s, v0.16b, v5.4b[0]\n" + ".inst 0x6f84e017 // udot v23.4s, v0.16b, v4.4b[0]\n" + ".inst 0x6f83e01b // udot v27.4s, v0.16b, v3.4b[0]\n" + ".inst 0x6f82e01f // udot v31.4s, v0.16b, v2.4b[0]\n" "bge 190b\n" "191:" // Height 6: Multiply loop: Skip odd blocks "cbz x14, 194f\n" "tbz x14, #1, 192f\n" "ldr h0, [x13], #0x2\n" - "ldr h1, [x9], #0x2\n" - "ldr h2, [x27], #0x2\n" - "ldr h3, [x25], #0x2\n" - "ldr h4, [x23], #0x2\n" - "ldr h5, [x21], #0x2\n" + "ldr h1, [x12], #0x2\n" + "ldr h2, [x11], #0x2\n" + "ldr h3, [x10], #0x2\n" + "ldr h4, [x9], #0x2\n" + "ldr h5, [x28], #0x2\n" "tbz x14, #0, 193f\n" "ld1 { v0.b }[2], [x13]\n" - "ld1 { v1.b }[2], [x9]\n" - "ld1 { v2.b }[2], [x27]\n" - "ld1 { v3.b }[2], [x25]\n" - "ld1 { v4.b }[2], [x23]\n" - "ld1 { v5.b }[2], [x21]\n" + "ld1 { v1.b }[2], [x12]\n" + "ld1 { v2.b }[2], [x11]\n" + "ld1 { v3.b }[2], [x10]\n" + "ld1 { v4.b }[2], [x9]\n" + "ld1 { v5.b }[2], [x28]\n" "b 193f\n" "192:" // Height 6: Multiply loop: Ragged operand read: partial_1_0 "ldr b0, [x13, #0x0]\n" - "ldr b1, [x9, #0x0]\n" - "ldr b2, [x27, #0x0]\n" - "ldr b3, [x25, #0x0]\n" - "ldr b4, [x23, #0x0]\n" - "ldr b5, [x21, #0x0]\n" + "ldr b1, [x12, #0x0]\n" + "ldr b2, [x11, #0x0]\n" + "ldr b3, [x10, #0x0]\n" + "ldr b4, [x9, #0x0]\n" + "ldr b5, [x28, #0x0]\n" "193:" // Height 6: Multiply loop: Ragged operand read: Done - "ldr q6, [x16, #0x0]\n" - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q7, [x16, #0x10]\n" - ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n" - ".inst 0x6f84e0d8 // udot v24.4s, v6.16b, v4.4b[0]\n" - ".inst 0x6f85e0dc // udot v28.4s, v6.16b, v5.4b[0]\n" - "ldr q6, [x16, #0x20]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x6f84e0f9 // udot v25.4s, v7.16b, v4.4b[0]\n" - ".inst 0x6f85e0fd // udot v29.4s, v7.16b, v5.4b[0]\n" - "ldr q7, [x16, #0x30]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" + "ldr q7, [x16, #0x0]\n" + ".inst 0x6f80e0e8 // udot v8.4s, v7.16b, v0.4b[0]\n" + "ldr q6, [x16, #0x10]\n" + ".inst 0x6f81e0ec // udot v12.4s, v7.16b, v1.4b[0]\n" + ".inst 0x6f82e0f0 // udot v16.4s, v7.16b, v2.4b[0]\n" + ".inst 0x6f83e0f4 // udot v20.4s, v7.16b, v3.4b[0]\n" + ".inst 0x6f84e0f8 // udot v24.4s, v7.16b, v4.4b[0]\n" + ".inst 0x6f85e0fc // udot v28.4s, v7.16b, v5.4b[0]\n" + "ldr q7, [x16, #0x20]\n" + ".inst 0x6f80e0c9 // udot v9.4s, v6.16b, v0.4b[0]\n" + ".inst 0x6f81e0cd // udot v13.4s, v6.16b, v1.4b[0]\n" + ".inst 0x6f82e0d1 // udot v17.4s, v6.16b, v2.4b[0]\n" + ".inst 0x6f83e0d5 // udot v21.4s, v6.16b, v3.4b[0]\n" + ".inst 0x6f84e0d9 // udot v25.4s, v6.16b, v4.4b[0]\n" + ".inst 0x6f85e0dd // udot v29.4s, v6.16b, v5.4b[0]\n" + "ldr q6, [x16, #0x30]\n" + ".inst 0x6f80e0ea // udot v10.4s, v7.16b, v0.4b[0]\n" "add x16, x16, #0x40\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x6f84e0da // udot v26.4s, v6.16b, v4.4b[0]\n" - ".inst 0x6f85e0de // udot v30.4s, v6.16b, v5.4b[0]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x6f84e0fb // udot v27.4s, v7.16b, v4.4b[0]\n" - ".inst 0x6f85e0ff // udot v31.4s, v7.16b, v5.4b[0]\n" + ".inst 0x6f81e0ee // udot v14.4s, v7.16b, v1.4b[0]\n" + ".inst 0x6f82e0f2 // udot v18.4s, v7.16b, v2.4b[0]\n" + ".inst 0x6f83e0f6 // udot v22.4s, v7.16b, v3.4b[0]\n" + ".inst 0x6f84e0fa // udot v26.4s, v7.16b, v4.4b[0]\n" + ".inst 0x6f85e0fe // udot v30.4s, v7.16b, v5.4b[0]\n" + ".inst 0x6f80e0cb // udot v11.4s, v6.16b, v0.4b[0]\n" + ".inst 0x6f81e0cf // udot v15.4s, v6.16b, v1.4b[0]\n" + ".inst 0x6f82e0d3 // udot v19.4s, v6.16b, v2.4b[0]\n" + ".inst 0x6f83e0d7 // udot v23.4s, v6.16b, v3.4b[0]\n" + ".inst 0x6f84e0db // udot v27.4s, v6.16b, v4.4b[0]\n" + ".inst 0x6f85e0df // udot v31.4s, v6.16b, v5.4b[0]\n" "194:" // Height 6: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x15, x15, #0x1\n" @@ -3488,7 +3487,6 @@ void a64_hybrid_u8u32_dot_6x16_a55 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "206:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16/generic.cpp index 38131cfd4b8bbd425fefcab9c332eea956f966ca..849c6808435b0ee3a28603c1142eb93948442cb0 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_6x16/generic.cpp @@ -77,7 +77,6 @@ void a64_hybrid_u8u32_dot_6x16 ( ka.N = N; ka.B_ptr = B_ptr; __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x6\n" "bge 171f\n" @@ -165,11 +164,11 @@ void a64_hybrid_u8u32_dot_6x16 ( "14:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 15f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 16f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -186,37 +185,37 @@ void a64_hybrid_u8u32_dot_6x16 ( "blt 18f\n" "17:" // Height 1: Multiply loop: Main loop head ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q17, [x10, #0x20]\n" ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n" - "ldr q7, [x10, #0xf0]\n" + "ldr q16, [x10, #0x30]\n" + ".inst 0x6f80e22a // udot v10.4s, v17.16b, v0.4b[0]\n" + "ldr q17, [x10, #0x40]\n" + ".inst 0x6f80e20b // udot v11.4s, v16.16b, v0.4b[0]\n" + "ldr q16, [x10, #0x50]\n" + ".inst 0x6fa0e228 // udot v8.4s, v17.16b, v0.4b[1]\n" + "ldr q17, [x10, #0x60]\n" + ".inst 0x6fa0e209 // udot v9.4s, v16.16b, v0.4b[1]\n" + "ldr q16, [x10, #0x70]\n" + ".inst 0x6fa0e22a // udot v10.4s, v17.16b, v0.4b[1]\n" + "ldr q17, [x10, #0x80]\n" + ".inst 0x6fa0e20b // udot v11.4s, v16.16b, v0.4b[1]\n" + "ldr q16, [x10, #0x90]\n" + ".inst 0x6f80ea28 // udot v8.4s, v17.16b, v0.4b[2]\n" + "ldr q17, [x10, #0xa0]\n" + ".inst 0x6f80ea09 // udot v9.4s, v16.16b, v0.4b[2]\n" + "ldr q16, [x10, #0xb0]\n" + ".inst 0x6f80ea2a // udot v10.4s, v17.16b, v0.4b[2]\n" + "ldr q17, [x10, #0xc0]\n" + ".inst 0x6f80ea0b // udot v11.4s, v16.16b, v0.4b[2]\n" + "ldr q16, [x10, #0xd0]\n" + ".inst 0x6fa0ea28 // udot v8.4s, v17.16b, v0.4b[3]\n" + "ldr q17, [x10, #0xe0]\n" + ".inst 0x6fa0ea09 // udot v9.4s, v16.16b, v0.4b[3]\n" + "ldr q16, [x10, #0xf0]\n" "sub x27, x27, #0x10\n" "add x26, x26, #0x10\n" - ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x6fa0ea2a // udot v10.4s, v17.16b, v0.4b[3]\n" + ".inst 0x6fa0ea0b // udot v11.4s, v16.16b, v0.4b[3]\n" "ldr q0, [x26, #0x0]\n" "cmp x27, #0x20\n" "add x10, x10, #0x100\n" @@ -226,37 +225,37 @@ void a64_hybrid_u8u32_dot_6x16 ( "bge 17b\n" "18:" // Height 1: Multiply loop: Single iteration only ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q17, [x10, #0x20]\n" ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n" - "ldr q7, [x10, #0xf0]\n" + "ldr q16, [x10, #0x30]\n" + ".inst 0x6f80e22a // udot v10.4s, v17.16b, v0.4b[0]\n" + "ldr q17, [x10, #0x40]\n" + ".inst 0x6f80e20b // udot v11.4s, v16.16b, v0.4b[0]\n" + "ldr q16, [x10, #0x50]\n" + ".inst 0x6fa0e228 // udot v8.4s, v17.16b, v0.4b[1]\n" + "ldr q17, [x10, #0x60]\n" + ".inst 0x6fa0e209 // udot v9.4s, v16.16b, v0.4b[1]\n" + "ldr q16, [x10, #0x70]\n" + ".inst 0x6fa0e22a // udot v10.4s, v17.16b, v0.4b[1]\n" + "ldr q17, [x10, #0x80]\n" + ".inst 0x6fa0e20b // udot v11.4s, v16.16b, v0.4b[1]\n" + "ldr q16, [x10, #0x90]\n" + ".inst 0x6f80ea28 // udot v8.4s, v17.16b, v0.4b[2]\n" + "ldr q17, [x10, #0xa0]\n" + ".inst 0x6f80ea09 // udot v9.4s, v16.16b, v0.4b[2]\n" + "ldr q16, [x10, #0xb0]\n" + ".inst 0x6f80ea2a // udot v10.4s, v17.16b, v0.4b[2]\n" + "ldr q17, [x10, #0xc0]\n" + ".inst 0x6f80ea0b // udot v11.4s, v16.16b, v0.4b[2]\n" + "ldr q16, [x10, #0xd0]\n" + ".inst 0x6fa0ea28 // udot v8.4s, v17.16b, v0.4b[3]\n" + "ldr q17, [x10, #0xe0]\n" + ".inst 0x6fa0ea09 // udot v9.4s, v16.16b, v0.4b[3]\n" + "ldr q16, [x10, #0xf0]\n" "add x26, x26, #0x10\n" "sub x27, x27, #0x10\n" - ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x6fa0ea2a // udot v10.4s, v17.16b, v0.4b[3]\n" + ".inst 0x6fa0ea0b // udot v11.4s, v16.16b, v0.4b[3]\n" "prfm pldl1keep, [x26, #0x80]\n" "add x10, x10, #0x100\n" "19:" // Height 1: Multiply loop: Main loop skip @@ -264,17 +263,17 @@ void a64_hybrid_u8u32_dot_6x16 ( "cmp x27, #0x4\n" "blt 21f\n" "20:" // Height 1: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr q6, [x10, #0x0]\n" - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" + "ldr s18, [x26], #0x4\n" + "ldr q16, [x10, #0x0]\n" + ".inst 0x6f92e208 // udot v8.4s, v16.16b, v18.4b[0]\n" "sub x27, x27, #0x4\n" - "ldr q7, [x10, #0x10]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" + "ldr q16, [x10, #0x10]\n" + "ldr q17, [x10, #0x20]\n" + ".inst 0x6f92e209 // udot v9.4s, v16.16b, v18.4b[0]\n" "cmp x27, #0x4\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" + "ldr q16, [x10, #0x30]\n" + ".inst 0x6f92e22a // udot v10.4s, v17.16b, v18.4b[0]\n" + ".inst 0x6f92e20b // udot v11.4s, v16.16b, v18.4b[0]\n" "add x10, x10, #0x40\n" "bge 20b\n" "21:" // Height 1: Multiply loop: Skip odd blocks @@ -287,14 +286,14 @@ void a64_hybrid_u8u32_dot_6x16 ( "22:" // Height 1: Multiply loop: Ragged operand read: partial_1_0 "ldr b0, [x26, #0x0]\n" "23:" // Height 1: Multiply loop: Ragged operand read: Done - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - "ldr q6, [x10, #0x20]\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" + "ldr q17, [x10, #0x0]\n" + "ldr q16, [x10, #0x10]\n" + ".inst 0x6f80e228 // udot v8.4s, v17.16b, v0.4b[0]\n" + ".inst 0x6f80e209 // udot v9.4s, v16.16b, v0.4b[0]\n" + "ldr q17, [x10, #0x20]\n" + "ldr q16, [x10, #0x30]\n" + ".inst 0x6f80e22a // udot v10.4s, v17.16b, v0.4b[0]\n" + ".inst 0x6f80e20b // udot v11.4s, v16.16b, v0.4b[0]\n" "add x10, x10, #0x40\n" "24:" // Height 1: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -465,12 +464,12 @@ void a64_hybrid_u8u32_dot_6x16 ( "48:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 49f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 50f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -478,7 +477,7 @@ void a64_hybrid_u8u32_dot_6x16 ( "b 50f\n" "49:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" + "add x25, x26, x21\n" "50:" // Height 2: input setup done "cmp x27, #0x10\n" "blt 53f\n" @@ -491,137 +490,137 @@ void a64_hybrid_u8u32_dot_6x16 ( "51:" // Height 2: Multiply loop: Main loop head ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q17, [x10, #0x20]\n" "sub x27, x27, #0x10\n" ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q16, [x10, #0x30]\n" "add x26, x26, #0x10\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x10, #0x40]\n" + ".inst 0x6f80e22a // udot v10.4s, v17.16b, v0.4b[0]\n" + ".inst 0x6f81e22e // udot v14.4s, v17.16b, v1.4b[0]\n" + "ldr q17, [x10, #0x40]\n" "add x25, x25, #0x10\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x10, #0x50]\n" + ".inst 0x6f80e20b // udot v11.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f81e20f // udot v15.4s, v16.16b, v1.4b[0]\n" + "ldr q16, [x10, #0x50]\n" "cmp x27, #0x20\n" - ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n" - "ldr q6, [x10, #0x60]\n" + ".inst 0x6fa0e228 // udot v8.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa1e22c // udot v12.4s, v17.16b, v1.4b[1]\n" + "ldr q17, [x10, #0x60]\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n" - "ldr q7, [x10, #0x70]\n" + ".inst 0x6fa0e209 // udot v9.4s, v16.16b, v0.4b[1]\n" + ".inst 0x6fa1e20d // udot v13.4s, v16.16b, v1.4b[1]\n" + "ldr q16, [x10, #0x70]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n" - "ldr q7, [x10, #0xf0]\n" + ".inst 0x6fa0e22a // udot v10.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa1e22e // udot v14.4s, v17.16b, v1.4b[1]\n" + "ldr q17, [x10, #0x80]\n" + ".inst 0x6fa0e20b // udot v11.4s, v16.16b, v0.4b[1]\n" + ".inst 0x6fa1e20f // udot v15.4s, v16.16b, v1.4b[1]\n" + "ldr q16, [x10, #0x90]\n" + ".inst 0x6f80ea28 // udot v8.4s, v17.16b, v0.4b[2]\n" + ".inst 0x6f81ea2c // udot v12.4s, v17.16b, v1.4b[2]\n" + "ldr q17, [x10, #0xa0]\n" + ".inst 0x6f80ea09 // udot v9.4s, v16.16b, v0.4b[2]\n" + ".inst 0x6f81ea0d // udot v13.4s, v16.16b, v1.4b[2]\n" + "ldr q16, [x10, #0xb0]\n" + ".inst 0x6f80ea2a // udot v10.4s, v17.16b, v0.4b[2]\n" + ".inst 0x6f81ea2e // udot v14.4s, v17.16b, v1.4b[2]\n" + "ldr q17, [x10, #0xc0]\n" + ".inst 0x6f80ea0b // udot v11.4s, v16.16b, v0.4b[2]\n" + ".inst 0x6f81ea0f // udot v15.4s, v16.16b, v1.4b[2]\n" + "ldr q16, [x10, #0xd0]\n" + ".inst 0x6fa0ea28 // udot v8.4s, v17.16b, v0.4b[3]\n" + ".inst 0x6fa1ea2c // udot v12.4s, v17.16b, v1.4b[3]\n" + "ldr q17, [x10, #0xe0]\n" + ".inst 0x6fa0ea09 // udot v9.4s, v16.16b, v0.4b[3]\n" + ".inst 0x6fa1ea0d // udot v13.4s, v16.16b, v1.4b[3]\n" + "ldr q16, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n" + ".inst 0x6fa0ea2a // udot v10.4s, v17.16b, v0.4b[3]\n" + ".inst 0x6fa1ea2e // udot v14.4s, v17.16b, v1.4b[3]\n" "ldr q6, [x10, #0x0]\n" - ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x6fa0ea0b // udot v11.4s, v16.16b, v0.4b[3]\n" "ldr q0, [x26, #0x0]\n" - ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n" + ".inst 0x6fa1ea0f // udot v15.4s, v16.16b, v1.4b[3]\n" "ldr q1, [x25, #0x0]\n" "ldr q7, [x10, #0x10]\n" "bge 51b\n" "52:" // Height 2: Multiply loop: Single iteration only ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q17, [x10, #0x20]\n" "add x26, x26, #0x10\n" ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q16, [x10, #0x30]\n" "add x25, x25, #0x10\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x10, #0x40]\n" + ".inst 0x6f80e22a // udot v10.4s, v17.16b, v0.4b[0]\n" + ".inst 0x6f81e22e // udot v14.4s, v17.16b, v1.4b[0]\n" + "ldr q17, [x10, #0x40]\n" "sub x27, x27, #0x10\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x10, #0x50]\n" + ".inst 0x6f80e20b // udot v11.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f81e20f // udot v15.4s, v16.16b, v1.4b[0]\n" + "ldr q16, [x10, #0x50]\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n" - "ldr q6, [x10, #0x60]\n" + ".inst 0x6fa0e228 // udot v8.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa1e22c // udot v12.4s, v17.16b, v1.4b[1]\n" + "ldr q17, [x10, #0x60]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n" - "ldr q7, [x10, #0xf0]\n" + ".inst 0x6fa0e209 // udot v9.4s, v16.16b, v0.4b[1]\n" + ".inst 0x6fa1e20d // udot v13.4s, v16.16b, v1.4b[1]\n" + "ldr q16, [x10, #0x70]\n" + ".inst 0x6fa0e22a // udot v10.4s, v17.16b, v0.4b[1]\n" + ".inst 0x6fa1e22e // udot v14.4s, v17.16b, v1.4b[1]\n" + "ldr q17, [x10, #0x80]\n" + ".inst 0x6fa0e20b // udot v11.4s, v16.16b, v0.4b[1]\n" + ".inst 0x6fa1e20f // udot v15.4s, v16.16b, v1.4b[1]\n" + "ldr q16, [x10, #0x90]\n" + ".inst 0x6f80ea28 // udot v8.4s, v17.16b, v0.4b[2]\n" + ".inst 0x6f81ea2c // udot v12.4s, v17.16b, v1.4b[2]\n" + "ldr q17, [x10, #0xa0]\n" + ".inst 0x6f80ea09 // udot v9.4s, v16.16b, v0.4b[2]\n" + ".inst 0x6f81ea0d // udot v13.4s, v16.16b, v1.4b[2]\n" + "ldr q16, [x10, #0xb0]\n" + ".inst 0x6f80ea2a // udot v10.4s, v17.16b, v0.4b[2]\n" + ".inst 0x6f81ea2e // udot v14.4s, v17.16b, v1.4b[2]\n" + "ldr q17, [x10, #0xc0]\n" + ".inst 0x6f80ea0b // udot v11.4s, v16.16b, v0.4b[2]\n" + ".inst 0x6f81ea0f // udot v15.4s, v16.16b, v1.4b[2]\n" + "ldr q16, [x10, #0xd0]\n" + ".inst 0x6fa0ea28 // udot v8.4s, v17.16b, v0.4b[3]\n" + ".inst 0x6fa1ea2c // udot v12.4s, v17.16b, v1.4b[3]\n" + "ldr q17, [x10, #0xe0]\n" + ".inst 0x6fa0ea09 // udot v9.4s, v16.16b, v0.4b[3]\n" + ".inst 0x6fa1ea0d // udot v13.4s, v16.16b, v1.4b[3]\n" + "ldr q16, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n" + ".inst 0x6fa0ea2a // udot v10.4s, v17.16b, v0.4b[3]\n" + ".inst 0x6fa1ea2e // udot v14.4s, v17.16b, v1.4b[3]\n" + ".inst 0x6fa0ea0b // udot v11.4s, v16.16b, v0.4b[3]\n" + ".inst 0x6fa1ea0f // udot v15.4s, v16.16b, v1.4b[3]\n" "53:" // Height 2: Multiply loop: Main loop skip "cbz x27, 58f\n" "cmp x27, #0x4\n" "blt 55f\n" "54:" // Height 2: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s19, [x26], #0x4\n" + "ldr s18, [x25], #0x4\n" "sub x27, x27, #0x4\n" "cmp x27, #0x4\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" + "ldr q17, [x10, #0x0]\n" + "ldr q16, [x10, #0x10]\n" + ".inst 0x6f93e228 // udot v8.4s, v17.16b, v19.4b[0]\n" + ".inst 0x6f92e22c // udot v12.4s, v17.16b, v18.4b[0]\n" + "ldr q17, [x10, #0x20]\n" + ".inst 0x6f93e209 // udot v9.4s, v16.16b, v19.4b[0]\n" + ".inst 0x6f92e20d // udot v13.4s, v16.16b, v18.4b[0]\n" + "ldr q16, [x10, #0x30]\n" + ".inst 0x6f93e22a // udot v10.4s, v17.16b, v19.4b[0]\n" + ".inst 0x6f92e22e // udot v14.4s, v17.16b, v18.4b[0]\n" "add x10, x10, #0x40\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" + ".inst 0x6f93e20b // udot v11.4s, v16.16b, v19.4b[0]\n" + ".inst 0x6f92e20f // udot v15.4s, v16.16b, v18.4b[0]\n" "bge 54b\n" "55:" // Height 2: Multiply loop: Skip odd blocks "cbz x27, 58f\n" @@ -636,19 +635,19 @@ void a64_hybrid_u8u32_dot_6x16 ( "ldr b0, [x26, #0x0]\n" "ldr b1, [x25, #0x0]\n" "57:" // Height 2: Multiply loop: Ragged operand read: Done - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" + "ldr q17, [x10, #0x0]\n" + "ldr q16, [x10, #0x10]\n" + ".inst 0x6f80e228 // udot v8.4s, v17.16b, v0.4b[0]\n" + ".inst 0x6f81e22c // udot v12.4s, v17.16b, v1.4b[0]\n" + "ldr q17, [x10, #0x20]\n" + ".inst 0x6f80e209 // udot v9.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f81e20d // udot v13.4s, v16.16b, v1.4b[0]\n" + "ldr q16, [x10, #0x30]\n" + ".inst 0x6f80e22a // udot v10.4s, v17.16b, v0.4b[0]\n" + ".inst 0x6f81e22e // udot v14.4s, v17.16b, v1.4b[0]\n" "add x10, x10, #0x40\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" + ".inst 0x6f80e20b // udot v11.4s, v16.16b, v0.4b[0]\n" + ".inst 0x6f81e20f // udot v15.4s, v16.16b, v1.4b[0]\n" "58:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -866,13 +865,13 @@ void a64_hybrid_u8u32_dot_6x16 ( "82:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 83f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 84f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -881,8 +880,8 @@ void a64_hybrid_u8u32_dot_6x16 ( "b 84f\n" "83:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" "84:" // Height 3: input setup done "cmp x27, #0x10\n" "blt 87f\n" @@ -899,75 +898,75 @@ void a64_hybrid_u8u32_dot_6x16 ( "sub x27, x27, #0x10\n" "add x26, x26, #0x10\n" ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q21, [x10, #0x20]\n" ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" "add x25, x25, #0x10\n" ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q20, [x10, #0x30]\n" "add x24, x24, #0x10\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" + ".inst 0x6f80e2aa // udot v10.4s, v21.16b, v0.4b[0]\n" + ".inst 0x6f81e2ae // udot v14.4s, v21.16b, v1.4b[0]\n" "cmp x27, #0x20\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" + ".inst 0x6f82e2b2 // udot v18.4s, v21.16b, v2.4b[0]\n" + "ldr q21, [x10, #0x40]\n" + ".inst 0x6f80e28b // udot v11.4s, v20.16b, v0.4b[0]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x10, #0x50]\n" + ".inst 0x6f81e28f // udot v15.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f82e293 // udot v19.4s, v20.16b, v2.4b[0]\n" + "ldr q20, [x10, #0x50]\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x6fa2e0d0 // udot v16.4s, v6.16b, v2.4b[1]\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x6fa2e0f1 // udot v17.4s, v7.16b, v2.4b[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x6fa2e0d2 // udot v18.4s, v6.16b, v2.4b[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x6fa2e0f3 // udot v19.4s, v7.16b, v2.4b[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x6f82e8d0 // udot v16.4s, v6.16b, v2.4b[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f82e8f1 // udot v17.4s, v7.16b, v2.4b[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x6f82e8d2 // udot v18.4s, v6.16b, v2.4b[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f82e8f3 // udot v19.4s, v7.16b, v2.4b[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x6fa2e8d0 // udot v16.4s, v6.16b, v2.4b[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x6fa2e8f1 // udot v17.4s, v7.16b, v2.4b[3]\n" - "ldr q7, [x10, #0xf0]\n" + ".inst 0x6fa0e2a8 // udot v8.4s, v21.16b, v0.4b[1]\n" + ".inst 0x6fa1e2ac // udot v12.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa2e2b0 // udot v16.4s, v21.16b, v2.4b[1]\n" + "ldr q21, [x10, #0x60]\n" + ".inst 0x6fa0e289 // udot v9.4s, v20.16b, v0.4b[1]\n" + ".inst 0x6fa1e28d // udot v13.4s, v20.16b, v1.4b[1]\n" + ".inst 0x6fa2e291 // udot v17.4s, v20.16b, v2.4b[1]\n" + "ldr q20, [x10, #0x70]\n" + ".inst 0x6fa0e2aa // udot v10.4s, v21.16b, v0.4b[1]\n" + ".inst 0x6fa1e2ae // udot v14.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa2e2b2 // udot v18.4s, v21.16b, v2.4b[1]\n" + "ldr q21, [x10, #0x80]\n" + ".inst 0x6fa0e28b // udot v11.4s, v20.16b, v0.4b[1]\n" + ".inst 0x6fa1e28f // udot v15.4s, v20.16b, v1.4b[1]\n" + ".inst 0x6fa2e293 // udot v19.4s, v20.16b, v2.4b[1]\n" + "ldr q20, [x10, #0x90]\n" + ".inst 0x6f80eaa8 // udot v8.4s, v21.16b, v0.4b[2]\n" + ".inst 0x6f81eaac // udot v12.4s, v21.16b, v1.4b[2]\n" + ".inst 0x6f82eab0 // udot v16.4s, v21.16b, v2.4b[2]\n" + "ldr q21, [x10, #0xa0]\n" + ".inst 0x6f80ea89 // udot v9.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f81ea8d // udot v13.4s, v20.16b, v1.4b[2]\n" + ".inst 0x6f82ea91 // udot v17.4s, v20.16b, v2.4b[2]\n" + "ldr q20, [x10, #0xb0]\n" + ".inst 0x6f80eaaa // udot v10.4s, v21.16b, v0.4b[2]\n" + ".inst 0x6f81eaae // udot v14.4s, v21.16b, v1.4b[2]\n" + ".inst 0x6f82eab2 // udot v18.4s, v21.16b, v2.4b[2]\n" + "ldr q21, [x10, #0xc0]\n" + ".inst 0x6f80ea8b // udot v11.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f81ea8f // udot v15.4s, v20.16b, v1.4b[2]\n" + ".inst 0x6f82ea93 // udot v19.4s, v20.16b, v2.4b[2]\n" + "ldr q20, [x10, #0xd0]\n" + ".inst 0x6fa0eaa8 // udot v8.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa1eaac // udot v12.4s, v21.16b, v1.4b[3]\n" + ".inst 0x6fa2eab0 // udot v16.4s, v21.16b, v2.4b[3]\n" + "ldr q21, [x10, #0xe0]\n" + ".inst 0x6fa0ea89 // udot v9.4s, v20.16b, v0.4b[3]\n" + ".inst 0x6fa1ea8d // udot v13.4s, v20.16b, v1.4b[3]\n" + ".inst 0x6fa2ea91 // udot v17.4s, v20.16b, v2.4b[3]\n" + "ldr q20, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x6fa2e8d2 // udot v18.4s, v6.16b, v2.4b[3]\n" + ".inst 0x6fa0eaaa // udot v10.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa1eaae // udot v14.4s, v21.16b, v1.4b[3]\n" + ".inst 0x6fa2eab2 // udot v18.4s, v21.16b, v2.4b[3]\n" "ldr q6, [x10, #0x0]\n" - ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x6fa0ea8b // udot v11.4s, v20.16b, v0.4b[3]\n" "ldr q0, [x26, #0x0]\n" - ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n" + ".inst 0x6fa1ea8f // udot v15.4s, v20.16b, v1.4b[3]\n" "ldr q1, [x25, #0x0]\n" - ".inst 0x6fa2e8f3 // udot v19.4s, v7.16b, v2.4b[3]\n" + ".inst 0x6fa2ea93 // udot v19.4s, v20.16b, v2.4b[3]\n" "ldr q2, [x24, #0x0]\n" "ldr q7, [x10, #0x10]\n" "bge 85b\n" @@ -977,98 +976,98 @@ void a64_hybrid_u8u32_dot_6x16 ( "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q21, [x10, #0x20]\n" ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" "add x24, x24, #0x10\n" ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q20, [x10, #0x30]\n" "sub x27, x27, #0x10\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" + ".inst 0x6f80e2aa // udot v10.4s, v21.16b, v0.4b[0]\n" + ".inst 0x6f81e2ae // udot v14.4s, v21.16b, v1.4b[0]\n" "prfm pldl1keep, [x26, #0x80]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" + ".inst 0x6f82e2b2 // udot v18.4s, v21.16b, v2.4b[0]\n" + "ldr q21, [x10, #0x40]\n" + ".inst 0x6f80e28b // udot v11.4s, v20.16b, v0.4b[0]\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x6fa2e0d0 // udot v16.4s, v6.16b, v2.4b[1]\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x6fa2e0f1 // udot v17.4s, v7.16b, v2.4b[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x6fa2e0d2 // udot v18.4s, v6.16b, v2.4b[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x6fa2e0f3 // udot v19.4s, v7.16b, v2.4b[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x6f82e8d0 // udot v16.4s, v6.16b, v2.4b[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f82e8f1 // udot v17.4s, v7.16b, v2.4b[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x6f82e8d2 // udot v18.4s, v6.16b, v2.4b[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f82e8f3 // udot v19.4s, v7.16b, v2.4b[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x6fa2e8d0 // udot v16.4s, v6.16b, v2.4b[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x6fa2e8f1 // udot v17.4s, v7.16b, v2.4b[3]\n" - "ldr q7, [x10, #0xf0]\n" + ".inst 0x6f81e28f // udot v15.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f82e293 // udot v19.4s, v20.16b, v2.4b[0]\n" + "ldr q20, [x10, #0x50]\n" + ".inst 0x6fa0e2a8 // udot v8.4s, v21.16b, v0.4b[1]\n" + ".inst 0x6fa1e2ac // udot v12.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa2e2b0 // udot v16.4s, v21.16b, v2.4b[1]\n" + "ldr q21, [x10, #0x60]\n" + ".inst 0x6fa0e289 // udot v9.4s, v20.16b, v0.4b[1]\n" + ".inst 0x6fa1e28d // udot v13.4s, v20.16b, v1.4b[1]\n" + ".inst 0x6fa2e291 // udot v17.4s, v20.16b, v2.4b[1]\n" + "ldr q20, [x10, #0x70]\n" + ".inst 0x6fa0e2aa // udot v10.4s, v21.16b, v0.4b[1]\n" + ".inst 0x6fa1e2ae // udot v14.4s, v21.16b, v1.4b[1]\n" + ".inst 0x6fa2e2b2 // udot v18.4s, v21.16b, v2.4b[1]\n" + "ldr q21, [x10, #0x80]\n" + ".inst 0x6fa0e28b // udot v11.4s, v20.16b, v0.4b[1]\n" + ".inst 0x6fa1e28f // udot v15.4s, v20.16b, v1.4b[1]\n" + ".inst 0x6fa2e293 // udot v19.4s, v20.16b, v2.4b[1]\n" + "ldr q20, [x10, #0x90]\n" + ".inst 0x6f80eaa8 // udot v8.4s, v21.16b, v0.4b[2]\n" + ".inst 0x6f81eaac // udot v12.4s, v21.16b, v1.4b[2]\n" + ".inst 0x6f82eab0 // udot v16.4s, v21.16b, v2.4b[2]\n" + "ldr q21, [x10, #0xa0]\n" + ".inst 0x6f80ea89 // udot v9.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f81ea8d // udot v13.4s, v20.16b, v1.4b[2]\n" + ".inst 0x6f82ea91 // udot v17.4s, v20.16b, v2.4b[2]\n" + "ldr q20, [x10, #0xb0]\n" + ".inst 0x6f80eaaa // udot v10.4s, v21.16b, v0.4b[2]\n" + ".inst 0x6f81eaae // udot v14.4s, v21.16b, v1.4b[2]\n" + ".inst 0x6f82eab2 // udot v18.4s, v21.16b, v2.4b[2]\n" + "ldr q21, [x10, #0xc0]\n" + ".inst 0x6f80ea8b // udot v11.4s, v20.16b, v0.4b[2]\n" + ".inst 0x6f81ea8f // udot v15.4s, v20.16b, v1.4b[2]\n" + ".inst 0x6f82ea93 // udot v19.4s, v20.16b, v2.4b[2]\n" + "ldr q20, [x10, #0xd0]\n" + ".inst 0x6fa0eaa8 // udot v8.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa1eaac // udot v12.4s, v21.16b, v1.4b[3]\n" + ".inst 0x6fa2eab0 // udot v16.4s, v21.16b, v2.4b[3]\n" + "ldr q21, [x10, #0xe0]\n" + ".inst 0x6fa0ea89 // udot v9.4s, v20.16b, v0.4b[3]\n" + ".inst 0x6fa1ea8d // udot v13.4s, v20.16b, v1.4b[3]\n" + ".inst 0x6fa2ea91 // udot v17.4s, v20.16b, v2.4b[3]\n" + "ldr q20, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x6fa2e8d2 // udot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n" - ".inst 0x6fa2e8f3 // udot v19.4s, v7.16b, v2.4b[3]\n" + ".inst 0x6fa0eaaa // udot v10.4s, v21.16b, v0.4b[3]\n" + ".inst 0x6fa1eaae // udot v14.4s, v21.16b, v1.4b[3]\n" + ".inst 0x6fa2eab2 // udot v18.4s, v21.16b, v2.4b[3]\n" + ".inst 0x6fa0ea8b // udot v11.4s, v20.16b, v0.4b[3]\n" + ".inst 0x6fa1ea8f // udot v15.4s, v20.16b, v1.4b[3]\n" + ".inst 0x6fa2ea93 // udot v19.4s, v20.16b, v2.4b[3]\n" "87:" // Height 3: Multiply loop: Main loop skip "cbz x27, 92f\n" "cmp x27, #0x4\n" "blt 89f\n" "88:" // Height 3: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s24, [x26], #0x4\n" + "ldr s23, [x25], #0x4\n" "sub x27, x27, #0x4\n" "cmp x27, #0x4\n" - "ldr s2, [x24], #0x4\n" - "ldr q6, [x10, #0x0]\n" - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr s22, [x24], #0x4\n" + "ldr q21, [x10, #0x0]\n" + ".inst 0x6f98e2a8 // udot v8.4s, v21.16b, v24.4b[0]\n" + ".inst 0x6f97e2ac // udot v12.4s, v21.16b, v23.4b[0]\n" + "ldr q20, [x10, #0x10]\n" + ".inst 0x6f96e2b0 // udot v16.4s, v21.16b, v22.4b[0]\n" + "ldr q21, [x10, #0x20]\n" + ".inst 0x6f98e289 // udot v9.4s, v20.16b, v24.4b[0]\n" + ".inst 0x6f97e28d // udot v13.4s, v20.16b, v23.4b[0]\n" + ".inst 0x6f96e291 // udot v17.4s, v20.16b, v22.4b[0]\n" + "ldr q20, [x10, #0x30]\n" "add x10, x10, #0x40\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" + ".inst 0x6f98e2aa // udot v10.4s, v21.16b, v24.4b[0]\n" + ".inst 0x6f97e2ae // udot v14.4s, v21.16b, v23.4b[0]\n" + ".inst 0x6f96e2b2 // udot v18.4s, v21.16b, v22.4b[0]\n" + ".inst 0x6f98e28b // udot v11.4s, v20.16b, v24.4b[0]\n" + ".inst 0x6f97e28f // udot v15.4s, v20.16b, v23.4b[0]\n" + ".inst 0x6f96e293 // udot v19.4s, v20.16b, v22.4b[0]\n" "bge 88b\n" "89:" // Height 3: Multiply loop: Skip odd blocks "cbz x27, 92f\n" @@ -1086,23 +1085,23 @@ void a64_hybrid_u8u32_dot_6x16 ( "ldr b1, [x25, #0x0]\n" "ldr b2, [x24, #0x0]\n" "91:" // Height 3: Multiply loop: Ragged operand read: Done - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q21, [x10, #0x0]\n" + "ldr q20, [x10, #0x10]\n" + ".inst 0x6f80e2a8 // udot v8.4s, v21.16b, v0.4b[0]\n" + ".inst 0x6f81e2ac // udot v12.4s, v21.16b, v1.4b[0]\n" + ".inst 0x6f82e2b0 // udot v16.4s, v21.16b, v2.4b[0]\n" + "ldr q21, [x10, #0x20]\n" + ".inst 0x6f80e289 // udot v9.4s, v20.16b, v0.4b[0]\n" + ".inst 0x6f81e28d // udot v13.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f82e291 // udot v17.4s, v20.16b, v2.4b[0]\n" + "ldr q20, [x10, #0x30]\n" "add x10, x10, #0x40\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" + ".inst 0x6f80e2aa // udot v10.4s, v21.16b, v0.4b[0]\n" + ".inst 0x6f81e2ae // udot v14.4s, v21.16b, v1.4b[0]\n" + ".inst 0x6f82e2b2 // udot v18.4s, v21.16b, v2.4b[0]\n" + ".inst 0x6f80e28b // udot v11.4s, v20.16b, v0.4b[0]\n" + ".inst 0x6f81e28f // udot v15.4s, v20.16b, v1.4b[0]\n" + ".inst 0x6f82e293 // udot v19.4s, v20.16b, v2.4b[0]\n" "92:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1367,14 +1366,14 @@ void a64_hybrid_u8u32_dot_6x16 ( "116:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 117f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 118f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -1384,9 +1383,9 @@ void a64_hybrid_u8u32_dot_6x16 ( "b 118f\n" "117:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" "118:" // Height 4: input setup done "cmp x27, #0x10\n" "blt 121f\n" @@ -1405,7 +1404,7 @@ void a64_hybrid_u8u32_dot_6x16 ( "add x26, x26, #0x10\n" ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q25, [x10, #0x20]\n" "add x25, x25, #0x10\n" ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" @@ -1413,85 +1412,85 @@ void a64_hybrid_u8u32_dot_6x16 ( "add x23, x23, #0x10\n" ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q24, [x10, #0x30]\n" "cmp x27, #0x20\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" + ".inst 0x6f80e32a // udot v10.4s, v25.16b, v0.4b[0]\n" + ".inst 0x6f81e32e // udot v14.4s, v25.16b, v1.4b[0]\n" "prfm pldl1keep, [x26, #0x80]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x10, #0x40]\n" + ".inst 0x6f82e332 // udot v18.4s, v25.16b, v2.4b[0]\n" + ".inst 0x6f83e336 // udot v22.4s, v25.16b, v3.4b[0]\n" + "ldr q25, [x10, #0x40]\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" + ".inst 0x6f80e30b // udot v11.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f81e30f // udot v15.4s, v24.16b, v1.4b[0]\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x6fa2e0d0 // udot v16.4s, v6.16b, v2.4b[1]\n" - ".inst 0x6fa3e0d4 // udot v20.4s, v6.16b, v3.4b[1]\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x6fa2e0f1 // udot v17.4s, v7.16b, v2.4b[1]\n" - ".inst 0x6fa3e0f5 // udot v21.4s, v7.16b, v3.4b[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x6fa2e0d2 // udot v18.4s, v6.16b, v2.4b[1]\n" - ".inst 0x6fa3e0d6 // udot v22.4s, v6.16b, v3.4b[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x6fa2e0f3 // udot v19.4s, v7.16b, v2.4b[1]\n" - ".inst 0x6fa3e0f7 // udot v23.4s, v7.16b, v3.4b[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x6f82e8d0 // udot v16.4s, v6.16b, v2.4b[2]\n" - ".inst 0x6f83e8d4 // udot v20.4s, v6.16b, v3.4b[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f82e8f1 // udot v17.4s, v7.16b, v2.4b[2]\n" - ".inst 0x6f83e8f5 // udot v21.4s, v7.16b, v3.4b[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x6f82e8d2 // udot v18.4s, v6.16b, v2.4b[2]\n" - ".inst 0x6f83e8d6 // udot v22.4s, v6.16b, v3.4b[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f82e8f3 // udot v19.4s, v7.16b, v2.4b[2]\n" - ".inst 0x6f83e8f7 // udot v23.4s, v7.16b, v3.4b[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x6fa2e8d0 // udot v16.4s, v6.16b, v2.4b[3]\n" - ".inst 0x6fa3e8d4 // udot v20.4s, v6.16b, v3.4b[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x6fa2e8f1 // udot v17.4s, v7.16b, v2.4b[3]\n" - ".inst 0x6fa3e8f5 // udot v21.4s, v7.16b, v3.4b[3]\n" - "ldr q7, [x10, #0xf0]\n" + ".inst 0x6f82e313 // udot v19.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f83e317 // udot v23.4s, v24.16b, v3.4b[0]\n" + "ldr q24, [x10, #0x50]\n" + ".inst 0x6fa0e328 // udot v8.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa1e32c // udot v12.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa2e330 // udot v16.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa3e334 // udot v20.4s, v25.16b, v3.4b[1]\n" + "ldr q25, [x10, #0x60]\n" + ".inst 0x6fa0e309 // udot v9.4s, v24.16b, v0.4b[1]\n" + ".inst 0x6fa1e30d // udot v13.4s, v24.16b, v1.4b[1]\n" + ".inst 0x6fa2e311 // udot v17.4s, v24.16b, v2.4b[1]\n" + ".inst 0x6fa3e315 // udot v21.4s, v24.16b, v3.4b[1]\n" + "ldr q24, [x10, #0x70]\n" + ".inst 0x6fa0e32a // udot v10.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa1e32e // udot v14.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa2e332 // udot v18.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa3e336 // udot v22.4s, v25.16b, v3.4b[1]\n" + "ldr q25, [x10, #0x80]\n" + ".inst 0x6fa0e30b // udot v11.4s, v24.16b, v0.4b[1]\n" + ".inst 0x6fa1e30f // udot v15.4s, v24.16b, v1.4b[1]\n" + ".inst 0x6fa2e313 // udot v19.4s, v24.16b, v2.4b[1]\n" + ".inst 0x6fa3e317 // udot v23.4s, v24.16b, v3.4b[1]\n" + "ldr q24, [x10, #0x90]\n" + ".inst 0x6f80eb28 // udot v8.4s, v25.16b, v0.4b[2]\n" + ".inst 0x6f81eb2c // udot v12.4s, v25.16b, v1.4b[2]\n" + ".inst 0x6f82eb30 // udot v16.4s, v25.16b, v2.4b[2]\n" + ".inst 0x6f83eb34 // udot v20.4s, v25.16b, v3.4b[2]\n" + "ldr q25, [x10, #0xa0]\n" + ".inst 0x6f80eb09 // udot v9.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f81eb0d // udot v13.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f82eb11 // udot v17.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f83eb15 // udot v21.4s, v24.16b, v3.4b[2]\n" + "ldr q24, [x10, #0xb0]\n" + ".inst 0x6f80eb2a // udot v10.4s, v25.16b, v0.4b[2]\n" + ".inst 0x6f81eb2e // udot v14.4s, v25.16b, v1.4b[2]\n" + ".inst 0x6f82eb32 // udot v18.4s, v25.16b, v2.4b[2]\n" + ".inst 0x6f83eb36 // udot v22.4s, v25.16b, v3.4b[2]\n" + "ldr q25, [x10, #0xc0]\n" + ".inst 0x6f80eb0b // udot v11.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f81eb0f // udot v15.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f82eb13 // udot v19.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f83eb17 // udot v23.4s, v24.16b, v3.4b[2]\n" + "ldr q24, [x10, #0xd0]\n" + ".inst 0x6fa0eb28 // udot v8.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa1eb2c // udot v12.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa2eb30 // udot v16.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa3eb34 // udot v20.4s, v25.16b, v3.4b[3]\n" + "ldr q25, [x10, #0xe0]\n" + ".inst 0x6fa0eb09 // udot v9.4s, v24.16b, v0.4b[3]\n" + ".inst 0x6fa1eb0d // udot v13.4s, v24.16b, v1.4b[3]\n" + ".inst 0x6fa2eb11 // udot v17.4s, v24.16b, v2.4b[3]\n" + ".inst 0x6fa3eb15 // udot v21.4s, v24.16b, v3.4b[3]\n" + "ldr q24, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x6fa2e8d2 // udot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x6fa3e8d6 // udot v22.4s, v6.16b, v3.4b[3]\n" + ".inst 0x6fa0eb2a // udot v10.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa1eb2e // udot v14.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa2eb32 // udot v18.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa3eb36 // udot v22.4s, v25.16b, v3.4b[3]\n" "ldr q6, [x10, #0x0]\n" - ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x6fa0eb0b // udot v11.4s, v24.16b, v0.4b[3]\n" "ldr q0, [x26, #0x0]\n" - ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n" + ".inst 0x6fa1eb0f // udot v15.4s, v24.16b, v1.4b[3]\n" "ldr q1, [x25, #0x0]\n" - ".inst 0x6fa2e8f3 // udot v19.4s, v7.16b, v2.4b[3]\n" + ".inst 0x6fa2eb13 // udot v19.4s, v24.16b, v2.4b[3]\n" "ldr q2, [x24, #0x0]\n" - ".inst 0x6fa3e8f7 // udot v23.4s, v7.16b, v3.4b[3]\n" + ".inst 0x6fa3eb17 // udot v23.4s, v24.16b, v3.4b[3]\n" "ldr q3, [x23, #0x0]\n" "ldr q7, [x10, #0x10]\n" "bge 119b\n" @@ -1502,7 +1501,7 @@ void a64_hybrid_u8u32_dot_6x16 ( "add x25, x25, #0x10\n" ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q25, [x10, #0x20]\n" "add x24, x24, #0x10\n" ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" @@ -1510,112 +1509,112 @@ void a64_hybrid_u8u32_dot_6x16 ( "sub x27, x27, #0x10\n" ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q24, [x10, #0x30]\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" + ".inst 0x6f80e32a // udot v10.4s, v25.16b, v0.4b[0]\n" + ".inst 0x6f81e32e // udot v14.4s, v25.16b, v1.4b[0]\n" "prfm pldl1keep, [x25, #0x80]\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x10, #0x40]\n" + ".inst 0x6f82e332 // udot v18.4s, v25.16b, v2.4b[0]\n" + ".inst 0x6f83e336 // udot v22.4s, v25.16b, v3.4b[0]\n" + "ldr q25, [x10, #0x40]\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x6fa2e0d0 // udot v16.4s, v6.16b, v2.4b[1]\n" - ".inst 0x6fa3e0d4 // udot v20.4s, v6.16b, v3.4b[1]\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x6fa2e0f1 // udot v17.4s, v7.16b, v2.4b[1]\n" - ".inst 0x6fa3e0f5 // udot v21.4s, v7.16b, v3.4b[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x6fa2e0d2 // udot v18.4s, v6.16b, v2.4b[1]\n" - ".inst 0x6fa3e0d6 // udot v22.4s, v6.16b, v3.4b[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x6fa2e0f3 // udot v19.4s, v7.16b, v2.4b[1]\n" - ".inst 0x6fa3e0f7 // udot v23.4s, v7.16b, v3.4b[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x6f82e8d0 // udot v16.4s, v6.16b, v2.4b[2]\n" - ".inst 0x6f83e8d4 // udot v20.4s, v6.16b, v3.4b[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f82e8f1 // udot v17.4s, v7.16b, v2.4b[2]\n" - ".inst 0x6f83e8f5 // udot v21.4s, v7.16b, v3.4b[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x6f82e8d2 // udot v18.4s, v6.16b, v2.4b[2]\n" - ".inst 0x6f83e8d6 // udot v22.4s, v6.16b, v3.4b[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f82e8f3 // udot v19.4s, v7.16b, v2.4b[2]\n" - ".inst 0x6f83e8f7 // udot v23.4s, v7.16b, v3.4b[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x6fa2e8d0 // udot v16.4s, v6.16b, v2.4b[3]\n" - ".inst 0x6fa3e8d4 // udot v20.4s, v6.16b, v3.4b[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x6fa2e8f1 // udot v17.4s, v7.16b, v2.4b[3]\n" - ".inst 0x6fa3e8f5 // udot v21.4s, v7.16b, v3.4b[3]\n" - "ldr q7, [x10, #0xf0]\n" + ".inst 0x6f80e30b // udot v11.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f81e30f // udot v15.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f82e313 // udot v19.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f83e317 // udot v23.4s, v24.16b, v3.4b[0]\n" + "ldr q24, [x10, #0x50]\n" + ".inst 0x6fa0e328 // udot v8.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa1e32c // udot v12.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa2e330 // udot v16.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa3e334 // udot v20.4s, v25.16b, v3.4b[1]\n" + "ldr q25, [x10, #0x60]\n" + ".inst 0x6fa0e309 // udot v9.4s, v24.16b, v0.4b[1]\n" + ".inst 0x6fa1e30d // udot v13.4s, v24.16b, v1.4b[1]\n" + ".inst 0x6fa2e311 // udot v17.4s, v24.16b, v2.4b[1]\n" + ".inst 0x6fa3e315 // udot v21.4s, v24.16b, v3.4b[1]\n" + "ldr q24, [x10, #0x70]\n" + ".inst 0x6fa0e32a // udot v10.4s, v25.16b, v0.4b[1]\n" + ".inst 0x6fa1e32e // udot v14.4s, v25.16b, v1.4b[1]\n" + ".inst 0x6fa2e332 // udot v18.4s, v25.16b, v2.4b[1]\n" + ".inst 0x6fa3e336 // udot v22.4s, v25.16b, v3.4b[1]\n" + "ldr q25, [x10, #0x80]\n" + ".inst 0x6fa0e30b // udot v11.4s, v24.16b, v0.4b[1]\n" + ".inst 0x6fa1e30f // udot v15.4s, v24.16b, v1.4b[1]\n" + ".inst 0x6fa2e313 // udot v19.4s, v24.16b, v2.4b[1]\n" + ".inst 0x6fa3e317 // udot v23.4s, v24.16b, v3.4b[1]\n" + "ldr q24, [x10, #0x90]\n" + ".inst 0x6f80eb28 // udot v8.4s, v25.16b, v0.4b[2]\n" + ".inst 0x6f81eb2c // udot v12.4s, v25.16b, v1.4b[2]\n" + ".inst 0x6f82eb30 // udot v16.4s, v25.16b, v2.4b[2]\n" + ".inst 0x6f83eb34 // udot v20.4s, v25.16b, v3.4b[2]\n" + "ldr q25, [x10, #0xa0]\n" + ".inst 0x6f80eb09 // udot v9.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f81eb0d // udot v13.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f82eb11 // udot v17.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f83eb15 // udot v21.4s, v24.16b, v3.4b[2]\n" + "ldr q24, [x10, #0xb0]\n" + ".inst 0x6f80eb2a // udot v10.4s, v25.16b, v0.4b[2]\n" + ".inst 0x6f81eb2e // udot v14.4s, v25.16b, v1.4b[2]\n" + ".inst 0x6f82eb32 // udot v18.4s, v25.16b, v2.4b[2]\n" + ".inst 0x6f83eb36 // udot v22.4s, v25.16b, v3.4b[2]\n" + "ldr q25, [x10, #0xc0]\n" + ".inst 0x6f80eb0b // udot v11.4s, v24.16b, v0.4b[2]\n" + ".inst 0x6f81eb0f // udot v15.4s, v24.16b, v1.4b[2]\n" + ".inst 0x6f82eb13 // udot v19.4s, v24.16b, v2.4b[2]\n" + ".inst 0x6f83eb17 // udot v23.4s, v24.16b, v3.4b[2]\n" + "ldr q24, [x10, #0xd0]\n" + ".inst 0x6fa0eb28 // udot v8.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa1eb2c // udot v12.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa2eb30 // udot v16.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa3eb34 // udot v20.4s, v25.16b, v3.4b[3]\n" + "ldr q25, [x10, #0xe0]\n" + ".inst 0x6fa0eb09 // udot v9.4s, v24.16b, v0.4b[3]\n" + ".inst 0x6fa1eb0d // udot v13.4s, v24.16b, v1.4b[3]\n" + ".inst 0x6fa2eb11 // udot v17.4s, v24.16b, v2.4b[3]\n" + ".inst 0x6fa3eb15 // udot v21.4s, v24.16b, v3.4b[3]\n" + "ldr q24, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x6fa2e8d2 // udot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x6fa3e8d6 // udot v22.4s, v6.16b, v3.4b[3]\n" - ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n" - ".inst 0x6fa2e8f3 // udot v19.4s, v7.16b, v2.4b[3]\n" - ".inst 0x6fa3e8f7 // udot v23.4s, v7.16b, v3.4b[3]\n" + ".inst 0x6fa0eb2a // udot v10.4s, v25.16b, v0.4b[3]\n" + ".inst 0x6fa1eb2e // udot v14.4s, v25.16b, v1.4b[3]\n" + ".inst 0x6fa2eb32 // udot v18.4s, v25.16b, v2.4b[3]\n" + ".inst 0x6fa3eb36 // udot v22.4s, v25.16b, v3.4b[3]\n" + ".inst 0x6fa0eb0b // udot v11.4s, v24.16b, v0.4b[3]\n" + ".inst 0x6fa1eb0f // udot v15.4s, v24.16b, v1.4b[3]\n" + ".inst 0x6fa2eb13 // udot v19.4s, v24.16b, v2.4b[3]\n" + ".inst 0x6fa3eb17 // udot v23.4s, v24.16b, v3.4b[3]\n" "121:" // Height 4: Multiply loop: Main loop skip "cbz x27, 126f\n" "cmp x27, #0x4\n" "blt 123f\n" "122:" // Height 4: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s29, [x26], #0x4\n" + "ldr s28, [x25], #0x4\n" "sub x27, x27, #0x4\n" "cmp x27, #0x4\n" - "ldr s2, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr s27, [x24], #0x4\n" + "ldr s26, [x23], #0x4\n" + "ldr q25, [x10, #0x0]\n" + "ldr q24, [x10, #0x10]\n" + ".inst 0x6f9de328 // udot v8.4s, v25.16b, v29.4b[0]\n" + ".inst 0x6f9ce32c // udot v12.4s, v25.16b, v28.4b[0]\n" + ".inst 0x6f9be330 // udot v16.4s, v25.16b, v27.4b[0]\n" + ".inst 0x6f9ae334 // udot v20.4s, v25.16b, v26.4b[0]\n" + "ldr q25, [x10, #0x20]\n" + ".inst 0x6f9de309 // udot v9.4s, v24.16b, v29.4b[0]\n" + ".inst 0x6f9ce30d // udot v13.4s, v24.16b, v28.4b[0]\n" + ".inst 0x6f9be311 // udot v17.4s, v24.16b, v27.4b[0]\n" + ".inst 0x6f9ae315 // udot v21.4s, v24.16b, v26.4b[0]\n" + "ldr q24, [x10, #0x30]\n" "add x10, x10, #0x40\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n" + ".inst 0x6f9de32a // udot v10.4s, v25.16b, v29.4b[0]\n" + ".inst 0x6f9ce32e // udot v14.4s, v25.16b, v28.4b[0]\n" + ".inst 0x6f9be332 // udot v18.4s, v25.16b, v27.4b[0]\n" + ".inst 0x6f9ae336 // udot v22.4s, v25.16b, v26.4b[0]\n" + ".inst 0x6f9de30b // udot v11.4s, v24.16b, v29.4b[0]\n" + ".inst 0x6f9ce30f // udot v15.4s, v24.16b, v28.4b[0]\n" + ".inst 0x6f9be313 // udot v19.4s, v24.16b, v27.4b[0]\n" + ".inst 0x6f9ae317 // udot v23.4s, v24.16b, v26.4b[0]\n" "bge 122b\n" "123:" // Height 4: Multiply loop: Skip odd blocks "cbz x27, 126f\n" @@ -1636,27 +1635,27 @@ void a64_hybrid_u8u32_dot_6x16 ( "ldr b2, [x24, #0x0]\n" "ldr b3, [x23, #0x0]\n" "125:" // Height 4: Multiply loop: Ragged operand read: Done - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q25, [x10, #0x0]\n" + "ldr q24, [x10, #0x10]\n" + ".inst 0x6f80e328 // udot v8.4s, v25.16b, v0.4b[0]\n" + ".inst 0x6f81e32c // udot v12.4s, v25.16b, v1.4b[0]\n" + ".inst 0x6f82e330 // udot v16.4s, v25.16b, v2.4b[0]\n" + ".inst 0x6f83e334 // udot v20.4s, v25.16b, v3.4b[0]\n" + "ldr q25, [x10, #0x20]\n" + ".inst 0x6f80e309 // udot v9.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f81e30d // udot v13.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f82e311 // udot v17.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f83e315 // udot v21.4s, v24.16b, v3.4b[0]\n" + "ldr q24, [x10, #0x30]\n" "add x10, x10, #0x40\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n" + ".inst 0x6f80e32a // udot v10.4s, v25.16b, v0.4b[0]\n" + ".inst 0x6f81e32e // udot v14.4s, v25.16b, v1.4b[0]\n" + ".inst 0x6f82e332 // udot v18.4s, v25.16b, v2.4b[0]\n" + ".inst 0x6f83e336 // udot v22.4s, v25.16b, v3.4b[0]\n" + ".inst 0x6f80e30b // udot v11.4s, v24.16b, v0.4b[0]\n" + ".inst 0x6f81e30f // udot v15.4s, v24.16b, v1.4b[0]\n" + ".inst 0x6f82e313 // udot v19.4s, v24.16b, v2.4b[0]\n" + ".inst 0x6f83e317 // udot v23.4s, v24.16b, v3.4b[0]\n" "126:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1960,162 +1959,162 @@ void a64_hybrid_u8u32_dot_6x16 ( "movi v22.4s, #0x0\n" "movi v23.4s, #0x0\n" "movi v24.4s, #0x0\n" - "movi v25.4s, #0x0\n" - "movi v26.4s, #0x0\n" - "movi v27.4s, #0x0\n" - "149:" // Height 5: setup done - "mov x28, #0x0\n" - "150:" // Height 5: String loop - "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" - "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" - "tbz %x[flags], #3, 151f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "cbnz x28, 152f\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" - "add x26, x26, x20\n" - "add x25, x25, x20\n" - "add x24, x24, x20\n" - "add x23, x23, x20\n" - "add x22, x22, x20\n" - "b 152f\n" - "151:" // Height 5: setup direct input - "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" - "152:" // Height 5: input setup done - "cmp x27, #0x10\n" - "blt 155f\n" - "ldr q0, [x26, #0x0]\n" - "ldr q1, [x25, #0x0]\n" - "cmp x27, #0x20\n" - "ldr q2, [x24, #0x0]\n" - "ldr q3, [x23, #0x0]\n" - "ldr q4, [x22, #0x0]\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - "blt 154f\n" - "153:" // Height 5: Multiply loop: Main loop head - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - "sub x27, x27, #0x10\n" - "add x26, x26, #0x10\n" - ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n" - "add x25, x25, #0x10\n" - "add x24, x24, #0x10\n" - ".inst 0x6f84e0d8 // udot v24.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - "add x23, x23, #0x10\n" - ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" - "add x22, x22, #0x10\n" - "cmp x27, #0x20\n" - ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x6f84e0f9 // udot v25.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x10, #0x30]\n" - "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - "prfm pldl1keep, [x25, #0x80]\n" - "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n" - "prfm pldl1keep, [x23, #0x80]\n" - "prfm pldl1keep, [x22, #0x80]\n" - ".inst 0x6f84e0da // udot v26.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x6f84e0fb // udot v27.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x6fa2e0d0 // udot v16.4s, v6.16b, v2.4b[1]\n" - ".inst 0x6fa3e0d4 // udot v20.4s, v6.16b, v3.4b[1]\n" - ".inst 0x6fa4e0d8 // udot v24.4s, v6.16b, v4.4b[1]\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x6fa2e0f1 // udot v17.4s, v7.16b, v2.4b[1]\n" - ".inst 0x6fa3e0f5 // udot v21.4s, v7.16b, v3.4b[1]\n" - ".inst 0x6fa4e0f9 // udot v25.4s, v7.16b, v4.4b[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x6fa2e0d2 // udot v18.4s, v6.16b, v2.4b[1]\n" - ".inst 0x6fa3e0d6 // udot v22.4s, v6.16b, v3.4b[1]\n" - ".inst 0x6fa4e0da // udot v26.4s, v6.16b, v4.4b[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x6fa2e0f3 // udot v19.4s, v7.16b, v2.4b[1]\n" - ".inst 0x6fa3e0f7 // udot v23.4s, v7.16b, v3.4b[1]\n" - ".inst 0x6fa4e0fb // udot v27.4s, v7.16b, v4.4b[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x6f82e8d0 // udot v16.4s, v6.16b, v2.4b[2]\n" - ".inst 0x6f83e8d4 // udot v20.4s, v6.16b, v3.4b[2]\n" - ".inst 0x6f84e8d8 // udot v24.4s, v6.16b, v4.4b[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f82e8f1 // udot v17.4s, v7.16b, v2.4b[2]\n" - ".inst 0x6f83e8f5 // udot v21.4s, v7.16b, v3.4b[2]\n" - ".inst 0x6f84e8f9 // udot v25.4s, v7.16b, v4.4b[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x6f82e8d2 // udot v18.4s, v6.16b, v2.4b[2]\n" - ".inst 0x6f83e8d6 // udot v22.4s, v6.16b, v3.4b[2]\n" - ".inst 0x6f84e8da // udot v26.4s, v6.16b, v4.4b[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f82e8f3 // udot v19.4s, v7.16b, v2.4b[2]\n" - ".inst 0x6f83e8f7 // udot v23.4s, v7.16b, v3.4b[2]\n" - ".inst 0x6f84e8fb // udot v27.4s, v7.16b, v4.4b[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x6fa2e8d0 // udot v16.4s, v6.16b, v2.4b[3]\n" - ".inst 0x6fa3e8d4 // udot v20.4s, v6.16b, v3.4b[3]\n" - ".inst 0x6fa4e8d8 // udot v24.4s, v6.16b, v4.4b[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x6fa2e8f1 // udot v17.4s, v7.16b, v2.4b[3]\n" - ".inst 0x6fa3e8f5 // udot v21.4s, v7.16b, v3.4b[3]\n" - ".inst 0x6fa4e8f9 // udot v25.4s, v7.16b, v4.4b[3]\n" - "ldr q7, [x10, #0xf0]\n" + "movi v25.4s, #0x0\n" + "movi v26.4s, #0x0\n" + "movi v27.4s, #0x0\n" + "149:" // Height 5: setup done + "mov x28, #0x0\n" + "150:" // Height 5: String loop + "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" + "ldr w27, [x20, x28, LSL #0x2]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" + "tbz %x[flags], #3, 151f\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "cbnz x28, 152f\n" + "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" + "add x26, x26, x20\n" + "add x25, x25, x20\n" + "add x24, x24, x20\n" + "add x23, x23, x20\n" + "add x22, x22, x20\n" + "b 152f\n" + "151:" // Height 5: setup direct input + "mov x26, %x[input_ptr]\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" + "152:" // Height 5: input setup done + "cmp x27, #0x10\n" + "blt 155f\n" + "ldr q0, [x26, #0x0]\n" + "ldr q1, [x25, #0x0]\n" + "cmp x27, #0x20\n" + "ldr q2, [x24, #0x0]\n" + "ldr q3, [x23, #0x0]\n" + "ldr q4, [x22, #0x0]\n" + "ldr q6, [x10, #0x0]\n" + "ldr q7, [x10, #0x10]\n" + "blt 154f\n" + "153:" // Height 5: Multiply loop: Main loop head + ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" + ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" + "sub x27, x27, #0x10\n" + "add x26, x26, #0x10\n" + ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" + ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n" + "add x25, x25, #0x10\n" + "add x24, x24, #0x10\n" + ".inst 0x6f84e0d8 // udot v24.4s, v6.16b, v4.4b[0]\n" + "ldr q29, [x10, #0x20]\n" + ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" + "add x23, x23, #0x10\n" + ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" + ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" + "add x22, x22, #0x10\n" + "cmp x27, #0x20\n" + ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n" + ".inst 0x6f84e0f9 // udot v25.4s, v7.16b, v4.4b[0]\n" + "ldr q28, [x10, #0x30]\n" + "prfm pldl1keep, [x26, #0x80]\n" + ".inst 0x6f80e3aa // udot v10.4s, v29.16b, v0.4b[0]\n" + ".inst 0x6f81e3ae // udot v14.4s, v29.16b, v1.4b[0]\n" + "prfm pldl1keep, [x25, #0x80]\n" + "prfm pldl1keep, [x24, #0x80]\n" + ".inst 0x6f82e3b2 // udot v18.4s, v29.16b, v2.4b[0]\n" + ".inst 0x6f83e3b6 // udot v22.4s, v29.16b, v3.4b[0]\n" + "prfm pldl1keep, [x23, #0x80]\n" + "prfm pldl1keep, [x22, #0x80]\n" + ".inst 0x6f84e3ba // udot v26.4s, v29.16b, v4.4b[0]\n" + "ldr q29, [x10, #0x40]\n" + ".inst 0x6f80e38b // udot v11.4s, v28.16b, v0.4b[0]\n" + ".inst 0x6f81e38f // udot v15.4s, v28.16b, v1.4b[0]\n" + ".inst 0x6f82e393 // udot v19.4s, v28.16b, v2.4b[0]\n" + ".inst 0x6f83e397 // udot v23.4s, v28.16b, v3.4b[0]\n" + ".inst 0x6f84e39b // udot v27.4s, v28.16b, v4.4b[0]\n" + "ldr q28, [x10, #0x50]\n" + ".inst 0x6fa0e3a8 // udot v8.4s, v29.16b, v0.4b[1]\n" + ".inst 0x6fa1e3ac // udot v12.4s, v29.16b, v1.4b[1]\n" + ".inst 0x6fa2e3b0 // udot v16.4s, v29.16b, v2.4b[1]\n" + ".inst 0x6fa3e3b4 // udot v20.4s, v29.16b, v3.4b[1]\n" + ".inst 0x6fa4e3b8 // udot v24.4s, v29.16b, v4.4b[1]\n" + "ldr q29, [x10, #0x60]\n" + ".inst 0x6fa0e389 // udot v9.4s, v28.16b, v0.4b[1]\n" + ".inst 0x6fa1e38d // udot v13.4s, v28.16b, v1.4b[1]\n" + ".inst 0x6fa2e391 // udot v17.4s, v28.16b, v2.4b[1]\n" + ".inst 0x6fa3e395 // udot v21.4s, v28.16b, v3.4b[1]\n" + ".inst 0x6fa4e399 // udot v25.4s, v28.16b, v4.4b[1]\n" + "ldr q28, [x10, #0x70]\n" + ".inst 0x6fa0e3aa // udot v10.4s, v29.16b, v0.4b[1]\n" + ".inst 0x6fa1e3ae // udot v14.4s, v29.16b, v1.4b[1]\n" + ".inst 0x6fa2e3b2 // udot v18.4s, v29.16b, v2.4b[1]\n" + ".inst 0x6fa3e3b6 // udot v22.4s, v29.16b, v3.4b[1]\n" + ".inst 0x6fa4e3ba // udot v26.4s, v29.16b, v4.4b[1]\n" + "ldr q29, [x10, #0x80]\n" + ".inst 0x6fa0e38b // udot v11.4s, v28.16b, v0.4b[1]\n" + ".inst 0x6fa1e38f // udot v15.4s, v28.16b, v1.4b[1]\n" + ".inst 0x6fa2e393 // udot v19.4s, v28.16b, v2.4b[1]\n" + ".inst 0x6fa3e397 // udot v23.4s, v28.16b, v3.4b[1]\n" + ".inst 0x6fa4e39b // udot v27.4s, v28.16b, v4.4b[1]\n" + "ldr q28, [x10, #0x90]\n" + ".inst 0x6f80eba8 // udot v8.4s, v29.16b, v0.4b[2]\n" + ".inst 0x6f81ebac // udot v12.4s, v29.16b, v1.4b[2]\n" + ".inst 0x6f82ebb0 // udot v16.4s, v29.16b, v2.4b[2]\n" + ".inst 0x6f83ebb4 // udot v20.4s, v29.16b, v3.4b[2]\n" + ".inst 0x6f84ebb8 // udot v24.4s, v29.16b, v4.4b[2]\n" + "ldr q29, [x10, #0xa0]\n" + ".inst 0x6f80eb89 // udot v9.4s, v28.16b, v0.4b[2]\n" + ".inst 0x6f81eb8d // udot v13.4s, v28.16b, v1.4b[2]\n" + ".inst 0x6f82eb91 // udot v17.4s, v28.16b, v2.4b[2]\n" + ".inst 0x6f83eb95 // udot v21.4s, v28.16b, v3.4b[2]\n" + ".inst 0x6f84eb99 // udot v25.4s, v28.16b, v4.4b[2]\n" + "ldr q28, [x10, #0xb0]\n" + ".inst 0x6f80ebaa // udot v10.4s, v29.16b, v0.4b[2]\n" + ".inst 0x6f81ebae // udot v14.4s, v29.16b, v1.4b[2]\n" + ".inst 0x6f82ebb2 // udot v18.4s, v29.16b, v2.4b[2]\n" + ".inst 0x6f83ebb6 // udot v22.4s, v29.16b, v3.4b[2]\n" + ".inst 0x6f84ebba // udot v26.4s, v29.16b, v4.4b[2]\n" + "ldr q29, [x10, #0xc0]\n" + ".inst 0x6f80eb8b // udot v11.4s, v28.16b, v0.4b[2]\n" + ".inst 0x6f81eb8f // udot v15.4s, v28.16b, v1.4b[2]\n" + ".inst 0x6f82eb93 // udot v19.4s, v28.16b, v2.4b[2]\n" + ".inst 0x6f83eb97 // udot v23.4s, v28.16b, v3.4b[2]\n" + ".inst 0x6f84eb9b // udot v27.4s, v28.16b, v4.4b[2]\n" + "ldr q28, [x10, #0xd0]\n" + ".inst 0x6fa0eba8 // udot v8.4s, v29.16b, v0.4b[3]\n" + ".inst 0x6fa1ebac // udot v12.4s, v29.16b, v1.4b[3]\n" + ".inst 0x6fa2ebb0 // udot v16.4s, v29.16b, v2.4b[3]\n" + ".inst 0x6fa3ebb4 // udot v20.4s, v29.16b, v3.4b[3]\n" + ".inst 0x6fa4ebb8 // udot v24.4s, v29.16b, v4.4b[3]\n" + "ldr q29, [x10, #0xe0]\n" + ".inst 0x6fa0eb89 // udot v9.4s, v28.16b, v0.4b[3]\n" + ".inst 0x6fa1eb8d // udot v13.4s, v28.16b, v1.4b[3]\n" + ".inst 0x6fa2eb91 // udot v17.4s, v28.16b, v2.4b[3]\n" + ".inst 0x6fa3eb95 // udot v21.4s, v28.16b, v3.4b[3]\n" + ".inst 0x6fa4eb99 // udot v25.4s, v28.16b, v4.4b[3]\n" + "ldr q28, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x6fa2e8d2 // udot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x6fa3e8d6 // udot v22.4s, v6.16b, v3.4b[3]\n" - ".inst 0x6fa4e8da // udot v26.4s, v6.16b, v4.4b[3]\n" + ".inst 0x6fa0ebaa // udot v10.4s, v29.16b, v0.4b[3]\n" + ".inst 0x6fa1ebae // udot v14.4s, v29.16b, v1.4b[3]\n" + ".inst 0x6fa2ebb2 // udot v18.4s, v29.16b, v2.4b[3]\n" + ".inst 0x6fa3ebb6 // udot v22.4s, v29.16b, v3.4b[3]\n" + ".inst 0x6fa4ebba // udot v26.4s, v29.16b, v4.4b[3]\n" "ldr q6, [x10, #0x0]\n" - ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n" + ".inst 0x6fa0eb8b // udot v11.4s, v28.16b, v0.4b[3]\n" "ldr q0, [x26, #0x0]\n" - ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n" + ".inst 0x6fa1eb8f // udot v15.4s, v28.16b, v1.4b[3]\n" "ldr q1, [x25, #0x0]\n" - ".inst 0x6fa2e8f3 // udot v19.4s, v7.16b, v2.4b[3]\n" + ".inst 0x6fa2eb93 // udot v19.4s, v28.16b, v2.4b[3]\n" "ldr q2, [x24, #0x0]\n" - ".inst 0x6fa3e8f7 // udot v23.4s, v7.16b, v3.4b[3]\n" + ".inst 0x6fa3eb97 // udot v23.4s, v28.16b, v3.4b[3]\n" "ldr q3, [x23, #0x0]\n" - ".inst 0x6fa4e8fb // udot v27.4s, v7.16b, v4.4b[3]\n" + ".inst 0x6fa4eb9b // udot v27.4s, v28.16b, v4.4b[3]\n" "ldr q4, [x22, #0x0]\n" "ldr q7, [x10, #0x10]\n" "bge 153b\n" @@ -2129,7 +2128,7 @@ void a64_hybrid_u8u32_dot_6x16 ( "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" ".inst 0x6f84e0d8 // udot v24.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x10, #0x20]\n" + "ldr q29, [x10, #0x20]\n" ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" "add x22, x22, #0x10\n" ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" @@ -2138,131 +2137,131 @@ void a64_hybrid_u8u32_dot_6x16 ( "prfm pldl1keep, [x26, #0x80]\n" ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n" ".inst 0x6f84e0f9 // udot v25.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q28, [x10, #0x30]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" + ".inst 0x6f80e3aa // udot v10.4s, v29.16b, v0.4b[0]\n" + ".inst 0x6f81e3ae // udot v14.4s, v29.16b, v1.4b[0]\n" "prfm pldl1keep, [x24, #0x80]\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n" + ".inst 0x6f82e3b2 // udot v18.4s, v29.16b, v2.4b[0]\n" + ".inst 0x6f83e3b6 // udot v22.4s, v29.16b, v3.4b[0]\n" "prfm pldl1keep, [x22, #0x80]\n" - ".inst 0x6f84e0da // udot v26.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x6f84e0fb // udot v27.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6fa0e0c8 // udot v8.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0cc // udot v12.4s, v6.16b, v1.4b[1]\n" - ".inst 0x6fa2e0d0 // udot v16.4s, v6.16b, v2.4b[1]\n" - ".inst 0x6fa3e0d4 // udot v20.4s, v6.16b, v3.4b[1]\n" - ".inst 0x6fa4e0d8 // udot v24.4s, v6.16b, v4.4b[1]\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6fa0e0e9 // udot v9.4s, v7.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ed // udot v13.4s, v7.16b, v1.4b[1]\n" - ".inst 0x6fa2e0f1 // udot v17.4s, v7.16b, v2.4b[1]\n" - ".inst 0x6fa3e0f5 // udot v21.4s, v7.16b, v3.4b[1]\n" - ".inst 0x6fa4e0f9 // udot v25.4s, v7.16b, v4.4b[1]\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x6fa0e0ca // udot v10.4s, v6.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ce // udot v14.4s, v6.16b, v1.4b[1]\n" - ".inst 0x6fa2e0d2 // udot v18.4s, v6.16b, v2.4b[1]\n" - ".inst 0x6fa3e0d6 // udot v22.4s, v6.16b, v3.4b[1]\n" - ".inst 0x6fa4e0da // udot v26.4s, v6.16b, v4.4b[1]\n" - "ldr q6, [x10, #0x80]\n" - ".inst 0x6fa0e0eb // udot v11.4s, v7.16b, v0.4b[1]\n" - ".inst 0x6fa1e0ef // udot v15.4s, v7.16b, v1.4b[1]\n" - ".inst 0x6fa2e0f3 // udot v19.4s, v7.16b, v2.4b[1]\n" - ".inst 0x6fa3e0f7 // udot v23.4s, v7.16b, v3.4b[1]\n" - ".inst 0x6fa4e0fb // udot v27.4s, v7.16b, v4.4b[1]\n" - "ldr q7, [x10, #0x90]\n" - ".inst 0x6f80e8c8 // udot v8.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8cc // udot v12.4s, v6.16b, v1.4b[2]\n" - ".inst 0x6f82e8d0 // udot v16.4s, v6.16b, v2.4b[2]\n" - ".inst 0x6f83e8d4 // udot v20.4s, v6.16b, v3.4b[2]\n" - ".inst 0x6f84e8d8 // udot v24.4s, v6.16b, v4.4b[2]\n" - "ldr q6, [x10, #0xa0]\n" - ".inst 0x6f80e8e9 // udot v9.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8ed // udot v13.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f82e8f1 // udot v17.4s, v7.16b, v2.4b[2]\n" - ".inst 0x6f83e8f5 // udot v21.4s, v7.16b, v3.4b[2]\n" - ".inst 0x6f84e8f9 // udot v25.4s, v7.16b, v4.4b[2]\n" - "ldr q7, [x10, #0xb0]\n" - ".inst 0x6f80e8ca // udot v10.4s, v6.16b, v0.4b[2]\n" - ".inst 0x6f81e8ce // udot v14.4s, v6.16b, v1.4b[2]\n" - ".inst 0x6f82e8d2 // udot v18.4s, v6.16b, v2.4b[2]\n" - ".inst 0x6f83e8d6 // udot v22.4s, v6.16b, v3.4b[2]\n" - ".inst 0x6f84e8da // udot v26.4s, v6.16b, v4.4b[2]\n" - "ldr q6, [x10, #0xc0]\n" - ".inst 0x6f80e8eb // udot v11.4s, v7.16b, v0.4b[2]\n" - ".inst 0x6f81e8ef // udot v15.4s, v7.16b, v1.4b[2]\n" - ".inst 0x6f82e8f3 // udot v19.4s, v7.16b, v2.4b[2]\n" - ".inst 0x6f83e8f7 // udot v23.4s, v7.16b, v3.4b[2]\n" - ".inst 0x6f84e8fb // udot v27.4s, v7.16b, v4.4b[2]\n" - "ldr q7, [x10, #0xd0]\n" - ".inst 0x6fa0e8c8 // udot v8.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa1e8cc // udot v12.4s, v6.16b, v1.4b[3]\n" - ".inst 0x6fa2e8d0 // udot v16.4s, v6.16b, v2.4b[3]\n" - ".inst 0x6fa3e8d4 // udot v20.4s, v6.16b, v3.4b[3]\n" - ".inst 0x6fa4e8d8 // udot v24.4s, v6.16b, v4.4b[3]\n" - "ldr q6, [x10, #0xe0]\n" - ".inst 0x6fa0e8e9 // udot v9.4s, v7.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ed // udot v13.4s, v7.16b, v1.4b[3]\n" - ".inst 0x6fa2e8f1 // udot v17.4s, v7.16b, v2.4b[3]\n" - ".inst 0x6fa3e8f5 // udot v21.4s, v7.16b, v3.4b[3]\n" - ".inst 0x6fa4e8f9 // udot v25.4s, v7.16b, v4.4b[3]\n" - "ldr q7, [x10, #0xf0]\n" + ".inst 0x6f84e3ba // udot v26.4s, v29.16b, v4.4b[0]\n" + "ldr q29, [x10, #0x40]\n" + ".inst 0x6f80e38b // udot v11.4s, v28.16b, v0.4b[0]\n" + ".inst 0x6f81e38f // udot v15.4s, v28.16b, v1.4b[0]\n" + ".inst 0x6f82e393 // udot v19.4s, v28.16b, v2.4b[0]\n" + ".inst 0x6f83e397 // udot v23.4s, v28.16b, v3.4b[0]\n" + ".inst 0x6f84e39b // udot v27.4s, v28.16b, v4.4b[0]\n" + "ldr q28, [x10, #0x50]\n" + ".inst 0x6fa0e3a8 // udot v8.4s, v29.16b, v0.4b[1]\n" + ".inst 0x6fa1e3ac // udot v12.4s, v29.16b, v1.4b[1]\n" + ".inst 0x6fa2e3b0 // udot v16.4s, v29.16b, v2.4b[1]\n" + ".inst 0x6fa3e3b4 // udot v20.4s, v29.16b, v3.4b[1]\n" + ".inst 0x6fa4e3b8 // udot v24.4s, v29.16b, v4.4b[1]\n" + "ldr q29, [x10, #0x60]\n" + ".inst 0x6fa0e389 // udot v9.4s, v28.16b, v0.4b[1]\n" + ".inst 0x6fa1e38d // udot v13.4s, v28.16b, v1.4b[1]\n" + ".inst 0x6fa2e391 // udot v17.4s, v28.16b, v2.4b[1]\n" + ".inst 0x6fa3e395 // udot v21.4s, v28.16b, v3.4b[1]\n" + ".inst 0x6fa4e399 // udot v25.4s, v28.16b, v4.4b[1]\n" + "ldr q28, [x10, #0x70]\n" + ".inst 0x6fa0e3aa // udot v10.4s, v29.16b, v0.4b[1]\n" + ".inst 0x6fa1e3ae // udot v14.4s, v29.16b, v1.4b[1]\n" + ".inst 0x6fa2e3b2 // udot v18.4s, v29.16b, v2.4b[1]\n" + ".inst 0x6fa3e3b6 // udot v22.4s, v29.16b, v3.4b[1]\n" + ".inst 0x6fa4e3ba // udot v26.4s, v29.16b, v4.4b[1]\n" + "ldr q29, [x10, #0x80]\n" + ".inst 0x6fa0e38b // udot v11.4s, v28.16b, v0.4b[1]\n" + ".inst 0x6fa1e38f // udot v15.4s, v28.16b, v1.4b[1]\n" + ".inst 0x6fa2e393 // udot v19.4s, v28.16b, v2.4b[1]\n" + ".inst 0x6fa3e397 // udot v23.4s, v28.16b, v3.4b[1]\n" + ".inst 0x6fa4e39b // udot v27.4s, v28.16b, v4.4b[1]\n" + "ldr q28, [x10, #0x90]\n" + ".inst 0x6f80eba8 // udot v8.4s, v29.16b, v0.4b[2]\n" + ".inst 0x6f81ebac // udot v12.4s, v29.16b, v1.4b[2]\n" + ".inst 0x6f82ebb0 // udot v16.4s, v29.16b, v2.4b[2]\n" + ".inst 0x6f83ebb4 // udot v20.4s, v29.16b, v3.4b[2]\n" + ".inst 0x6f84ebb8 // udot v24.4s, v29.16b, v4.4b[2]\n" + "ldr q29, [x10, #0xa0]\n" + ".inst 0x6f80eb89 // udot v9.4s, v28.16b, v0.4b[2]\n" + ".inst 0x6f81eb8d // udot v13.4s, v28.16b, v1.4b[2]\n" + ".inst 0x6f82eb91 // udot v17.4s, v28.16b, v2.4b[2]\n" + ".inst 0x6f83eb95 // udot v21.4s, v28.16b, v3.4b[2]\n" + ".inst 0x6f84eb99 // udot v25.4s, v28.16b, v4.4b[2]\n" + "ldr q28, [x10, #0xb0]\n" + ".inst 0x6f80ebaa // udot v10.4s, v29.16b, v0.4b[2]\n" + ".inst 0x6f81ebae // udot v14.4s, v29.16b, v1.4b[2]\n" + ".inst 0x6f82ebb2 // udot v18.4s, v29.16b, v2.4b[2]\n" + ".inst 0x6f83ebb6 // udot v22.4s, v29.16b, v3.4b[2]\n" + ".inst 0x6f84ebba // udot v26.4s, v29.16b, v4.4b[2]\n" + "ldr q29, [x10, #0xc0]\n" + ".inst 0x6f80eb8b // udot v11.4s, v28.16b, v0.4b[2]\n" + ".inst 0x6f81eb8f // udot v15.4s, v28.16b, v1.4b[2]\n" + ".inst 0x6f82eb93 // udot v19.4s, v28.16b, v2.4b[2]\n" + ".inst 0x6f83eb97 // udot v23.4s, v28.16b, v3.4b[2]\n" + ".inst 0x6f84eb9b // udot v27.4s, v28.16b, v4.4b[2]\n" + "ldr q28, [x10, #0xd0]\n" + ".inst 0x6fa0eba8 // udot v8.4s, v29.16b, v0.4b[3]\n" + ".inst 0x6fa1ebac // udot v12.4s, v29.16b, v1.4b[3]\n" + ".inst 0x6fa2ebb0 // udot v16.4s, v29.16b, v2.4b[3]\n" + ".inst 0x6fa3ebb4 // udot v20.4s, v29.16b, v3.4b[3]\n" + ".inst 0x6fa4ebb8 // udot v24.4s, v29.16b, v4.4b[3]\n" + "ldr q29, [x10, #0xe0]\n" + ".inst 0x6fa0eb89 // udot v9.4s, v28.16b, v0.4b[3]\n" + ".inst 0x6fa1eb8d // udot v13.4s, v28.16b, v1.4b[3]\n" + ".inst 0x6fa2eb91 // udot v17.4s, v28.16b, v2.4b[3]\n" + ".inst 0x6fa3eb95 // udot v21.4s, v28.16b, v3.4b[3]\n" + ".inst 0x6fa4eb99 // udot v25.4s, v28.16b, v4.4b[3]\n" + "ldr q28, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x6fa0e8ca // udot v10.4s, v6.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ce // udot v14.4s, v6.16b, v1.4b[3]\n" - ".inst 0x6fa2e8d2 // udot v18.4s, v6.16b, v2.4b[3]\n" - ".inst 0x6fa3e8d6 // udot v22.4s, v6.16b, v3.4b[3]\n" - ".inst 0x6fa4e8da // udot v26.4s, v6.16b, v4.4b[3]\n" - ".inst 0x6fa0e8eb // udot v11.4s, v7.16b, v0.4b[3]\n" - ".inst 0x6fa1e8ef // udot v15.4s, v7.16b, v1.4b[3]\n" - ".inst 0x6fa2e8f3 // udot v19.4s, v7.16b, v2.4b[3]\n" - ".inst 0x6fa3e8f7 // udot v23.4s, v7.16b, v3.4b[3]\n" - ".inst 0x6fa4e8fb // udot v27.4s, v7.16b, v4.4b[3]\n" + ".inst 0x6fa0ebaa // udot v10.4s, v29.16b, v0.4b[3]\n" + ".inst 0x6fa1ebae // udot v14.4s, v29.16b, v1.4b[3]\n" + ".inst 0x6fa2ebb2 // udot v18.4s, v29.16b, v2.4b[3]\n" + ".inst 0x6fa3ebb6 // udot v22.4s, v29.16b, v3.4b[3]\n" + ".inst 0x6fa4ebba // udot v26.4s, v29.16b, v4.4b[3]\n" + ".inst 0x6fa0eb8b // udot v11.4s, v28.16b, v0.4b[3]\n" + ".inst 0x6fa1eb8f // udot v15.4s, v28.16b, v1.4b[3]\n" + ".inst 0x6fa2eb93 // udot v19.4s, v28.16b, v2.4b[3]\n" + ".inst 0x6fa3eb97 // udot v23.4s, v28.16b, v3.4b[3]\n" + ".inst 0x6fa4eb9b // udot v27.4s, v28.16b, v4.4b[3]\n" "155:" // Height 5: Multiply loop: Main loop skip "cbz x27, 160f\n" "cmp x27, #0x4\n" "blt 157f\n" "156:" // Height 5: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" + "ldr s2, [x26], #0x4\n" "ldr s1, [x25], #0x4\n" "sub x27, x27, #0x4\n" "cmp x27, #0x4\n" - "ldr s2, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr s4, [x22], #0x4\n" - "ldr q6, [x10, #0x0]\n" - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n" - ".inst 0x6f84e0d8 // udot v24.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x6f84e0f9 // udot v25.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr s0, [x24], #0x4\n" + "ldr s31, [x23], #0x4\n" + "ldr s30, [x22], #0x4\n" + "ldr q29, [x10, #0x0]\n" + ".inst 0x6f82e3a8 // udot v8.4s, v29.16b, v2.4b[0]\n" + ".inst 0x6f81e3ac // udot v12.4s, v29.16b, v1.4b[0]\n" + "ldr q28, [x10, #0x10]\n" + ".inst 0x6f80e3b0 // udot v16.4s, v29.16b, v0.4b[0]\n" + ".inst 0x6f9fe3b4 // udot v20.4s, v29.16b, v31.4b[0]\n" + ".inst 0x6f9ee3b8 // udot v24.4s, v29.16b, v30.4b[0]\n" + "ldr q29, [x10, #0x20]\n" + ".inst 0x6f82e389 // udot v9.4s, v28.16b, v2.4b[0]\n" + ".inst 0x6f81e38d // udot v13.4s, v28.16b, v1.4b[0]\n" + ".inst 0x6f80e391 // udot v17.4s, v28.16b, v0.4b[0]\n" + ".inst 0x6f9fe395 // udot v21.4s, v28.16b, v31.4b[0]\n" + ".inst 0x6f9ee399 // udot v25.4s, v28.16b, v30.4b[0]\n" + "ldr q28, [x10, #0x30]\n" "add x10, x10, #0x40\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x6f84e0da // udot v26.4s, v6.16b, v4.4b[0]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x6f84e0fb // udot v27.4s, v7.16b, v4.4b[0]\n" + ".inst 0x6f82e3aa // udot v10.4s, v29.16b, v2.4b[0]\n" + ".inst 0x6f81e3ae // udot v14.4s, v29.16b, v1.4b[0]\n" + ".inst 0x6f80e3b2 // udot v18.4s, v29.16b, v0.4b[0]\n" + ".inst 0x6f9fe3b6 // udot v22.4s, v29.16b, v31.4b[0]\n" + ".inst 0x6f9ee3ba // udot v26.4s, v29.16b, v30.4b[0]\n" + ".inst 0x6f82e38b // udot v11.4s, v28.16b, v2.4b[0]\n" + ".inst 0x6f81e38f // udot v15.4s, v28.16b, v1.4b[0]\n" + ".inst 0x6f80e393 // udot v19.4s, v28.16b, v0.4b[0]\n" + ".inst 0x6f9fe397 // udot v23.4s, v28.16b, v31.4b[0]\n" + ".inst 0x6f9ee39b // udot v27.4s, v28.16b, v30.4b[0]\n" "bge 156b\n" "157:" // Height 5: Multiply loop: Skip odd blocks "cbz x27, 160f\n" @@ -2286,31 +2285,31 @@ void a64_hybrid_u8u32_dot_6x16 ( "ldr b3, [x23, #0x0]\n" "ldr b4, [x22, #0x0]\n" "159:" // Height 5: Multiply loop: Ragged operand read: Done - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n" - ".inst 0x6f84e0d8 // udot v24.4s, v6.16b, v4.4b[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x6f84e0f9 // udot v25.4s, v7.16b, v4.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q29, [x10, #0x0]\n" + "ldr q28, [x10, #0x10]\n" + ".inst 0x6f80e3a8 // udot v8.4s, v29.16b, v0.4b[0]\n" + ".inst 0x6f81e3ac // udot v12.4s, v29.16b, v1.4b[0]\n" + ".inst 0x6f82e3b0 // udot v16.4s, v29.16b, v2.4b[0]\n" + ".inst 0x6f83e3b4 // udot v20.4s, v29.16b, v3.4b[0]\n" + ".inst 0x6f84e3b8 // udot v24.4s, v29.16b, v4.4b[0]\n" + "ldr q29, [x10, #0x20]\n" + ".inst 0x6f80e389 // udot v9.4s, v28.16b, v0.4b[0]\n" + ".inst 0x6f81e38d // udot v13.4s, v28.16b, v1.4b[0]\n" + ".inst 0x6f82e391 // udot v17.4s, v28.16b, v2.4b[0]\n" + ".inst 0x6f83e395 // udot v21.4s, v28.16b, v3.4b[0]\n" + ".inst 0x6f84e399 // udot v25.4s, v28.16b, v4.4b[0]\n" + "ldr q28, [x10, #0x30]\n" "add x10, x10, #0x40\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x6f84e0da // udot v26.4s, v6.16b, v4.4b[0]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x6f84e0fb // udot v27.4s, v7.16b, v4.4b[0]\n" + ".inst 0x6f80e3aa // udot v10.4s, v29.16b, v0.4b[0]\n" + ".inst 0x6f81e3ae // udot v14.4s, v29.16b, v1.4b[0]\n" + ".inst 0x6f82e3b2 // udot v18.4s, v29.16b, v2.4b[0]\n" + ".inst 0x6f83e3b6 // udot v22.4s, v29.16b, v3.4b[0]\n" + ".inst 0x6f84e3ba // udot v26.4s, v29.16b, v4.4b[0]\n" + ".inst 0x6f80e38b // udot v11.4s, v28.16b, v0.4b[0]\n" + ".inst 0x6f81e38f // udot v15.4s, v28.16b, v1.4b[0]\n" + ".inst 0x6f82e393 // udot v19.4s, v28.16b, v2.4b[0]\n" + ".inst 0x6f83e397 // udot v23.4s, v28.16b, v3.4b[0]\n" + ".inst 0x6f84e39b // udot v27.4s, v28.16b, v4.4b[0]\n" "160:" // Height 5: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -2672,16 +2671,16 @@ void a64_hybrid_u8u32_dot_6x16 ( "184:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 185f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 186f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -2693,11 +2692,11 @@ void a64_hybrid_u8u32_dot_6x16 ( "b 186f\n" "185:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" + "add x21, x22, x21\n" "186:" // Height 6: input setup done "cmp x27, #0x10\n" "blt 189f\n" @@ -2976,43 +2975,43 @@ void a64_hybrid_u8u32_dot_6x16 ( "cmp x27, #0x4\n" "blt 191f\n" "190:" // Height 6: Multiply loop: Odd block loop - "ldr s0, [x26], #0x4\n" - "ldr s1, [x25], #0x4\n" + "ldr s7, [x26], #0x4\n" + "ldr s6, [x25], #0x4\n" "sub x27, x27, #0x4\n" "cmp x27, #0x4\n" - "ldr s2, [x24], #0x4\n" - "ldr s3, [x23], #0x4\n" - "ldr s4, [x22], #0x4\n" - "ldr s5, [x21], #0x4\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n" - ".inst 0x6f84e0d8 // udot v24.4s, v6.16b, v4.4b[0]\n" - ".inst 0x6f85e0dc // udot v28.4s, v6.16b, v5.4b[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x6f84e0f9 // udot v25.4s, v7.16b, v4.4b[0]\n" - ".inst 0x6f85e0fd // udot v29.4s, v7.16b, v5.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr s5, [x24], #0x4\n" + "ldr s4, [x23], #0x4\n" + "ldr s3, [x22], #0x4\n" + "ldr s2, [x21], #0x4\n" + "ldr q1, [x10, #0x0]\n" + "ldr q0, [x10, #0x10]\n" + ".inst 0x6f87e028 // udot v8.4s, v1.16b, v7.4b[0]\n" + ".inst 0x6f86e02c // udot v12.4s, v1.16b, v6.4b[0]\n" + ".inst 0x6f85e030 // udot v16.4s, v1.16b, v5.4b[0]\n" + ".inst 0x6f84e034 // udot v20.4s, v1.16b, v4.4b[0]\n" + ".inst 0x6f83e038 // udot v24.4s, v1.16b, v3.4b[0]\n" + ".inst 0x6f82e03c // udot v28.4s, v1.16b, v2.4b[0]\n" + "ldr q1, [x10, #0x20]\n" + ".inst 0x6f87e009 // udot v9.4s, v0.16b, v7.4b[0]\n" + ".inst 0x6f86e00d // udot v13.4s, v0.16b, v6.4b[0]\n" + ".inst 0x6f85e011 // udot v17.4s, v0.16b, v5.4b[0]\n" + ".inst 0x6f84e015 // udot v21.4s, v0.16b, v4.4b[0]\n" + ".inst 0x6f83e019 // udot v25.4s, v0.16b, v3.4b[0]\n" + ".inst 0x6f82e01d // udot v29.4s, v0.16b, v2.4b[0]\n" + "ldr q0, [x10, #0x30]\n" "add x10, x10, #0x40\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x6f84e0da // udot v26.4s, v6.16b, v4.4b[0]\n" - ".inst 0x6f85e0de // udot v30.4s, v6.16b, v5.4b[0]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x6f84e0fb // udot v27.4s, v7.16b, v4.4b[0]\n" - ".inst 0x6f85e0ff // udot v31.4s, v7.16b, v5.4b[0]\n" + ".inst 0x6f87e02a // udot v10.4s, v1.16b, v7.4b[0]\n" + ".inst 0x6f86e02e // udot v14.4s, v1.16b, v6.4b[0]\n" + ".inst 0x6f85e032 // udot v18.4s, v1.16b, v5.4b[0]\n" + ".inst 0x6f84e036 // udot v22.4s, v1.16b, v4.4b[0]\n" + ".inst 0x6f83e03a // udot v26.4s, v1.16b, v3.4b[0]\n" + ".inst 0x6f82e03e // udot v30.4s, v1.16b, v2.4b[0]\n" + ".inst 0x6f87e00b // udot v11.4s, v0.16b, v7.4b[0]\n" + ".inst 0x6f86e00f // udot v15.4s, v0.16b, v6.4b[0]\n" + ".inst 0x6f85e013 // udot v19.4s, v0.16b, v5.4b[0]\n" + ".inst 0x6f84e017 // udot v23.4s, v0.16b, v4.4b[0]\n" + ".inst 0x6f83e01b // udot v27.4s, v0.16b, v3.4b[0]\n" + ".inst 0x6f82e01f // udot v31.4s, v0.16b, v2.4b[0]\n" "bge 190b\n" "191:" // Height 6: Multiply loop: Skip odd blocks "cbz x27, 194f\n" @@ -3039,35 +3038,35 @@ void a64_hybrid_u8u32_dot_6x16 ( "ldr b4, [x22, #0x0]\n" "ldr b5, [x21, #0x0]\n" "193:" // Height 6: Multiply loop: Ragged operand read: Done - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6f80e0c8 // udot v8.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0cc // udot v12.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d0 // udot v16.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d4 // udot v20.4s, v6.16b, v3.4b[0]\n" - ".inst 0x6f84e0d8 // udot v24.4s, v6.16b, v4.4b[0]\n" - ".inst 0x6f85e0dc // udot v28.4s, v6.16b, v5.4b[0]\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6f80e0e9 // udot v9.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ed // udot v13.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f1 // udot v17.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f5 // udot v21.4s, v7.16b, v3.4b[0]\n" - ".inst 0x6f84e0f9 // udot v25.4s, v7.16b, v4.4b[0]\n" - ".inst 0x6f85e0fd // udot v29.4s, v7.16b, v5.4b[0]\n" - "ldr q7, [x10, #0x30]\n" + "ldr q7, [x10, #0x0]\n" + "ldr q6, [x10, #0x10]\n" + ".inst 0x6f80e0e8 // udot v8.4s, v7.16b, v0.4b[0]\n" + ".inst 0x6f81e0ec // udot v12.4s, v7.16b, v1.4b[0]\n" + ".inst 0x6f82e0f0 // udot v16.4s, v7.16b, v2.4b[0]\n" + ".inst 0x6f83e0f4 // udot v20.4s, v7.16b, v3.4b[0]\n" + ".inst 0x6f84e0f8 // udot v24.4s, v7.16b, v4.4b[0]\n" + ".inst 0x6f85e0fc // udot v28.4s, v7.16b, v5.4b[0]\n" + "ldr q7, [x10, #0x20]\n" + ".inst 0x6f80e0c9 // udot v9.4s, v6.16b, v0.4b[0]\n" + ".inst 0x6f81e0cd // udot v13.4s, v6.16b, v1.4b[0]\n" + ".inst 0x6f82e0d1 // udot v17.4s, v6.16b, v2.4b[0]\n" + ".inst 0x6f83e0d5 // udot v21.4s, v6.16b, v3.4b[0]\n" + ".inst 0x6f84e0d9 // udot v25.4s, v6.16b, v4.4b[0]\n" + ".inst 0x6f85e0dd // udot v29.4s, v6.16b, v5.4b[0]\n" + "ldr q6, [x10, #0x30]\n" "add x10, x10, #0x40\n" - ".inst 0x6f80e0ca // udot v10.4s, v6.16b, v0.4b[0]\n" - ".inst 0x6f81e0ce // udot v14.4s, v6.16b, v1.4b[0]\n" - ".inst 0x6f82e0d2 // udot v18.4s, v6.16b, v2.4b[0]\n" - ".inst 0x6f83e0d6 // udot v22.4s, v6.16b, v3.4b[0]\n" - ".inst 0x6f84e0da // udot v26.4s, v6.16b, v4.4b[0]\n" - ".inst 0x6f85e0de // udot v30.4s, v6.16b, v5.4b[0]\n" - ".inst 0x6f80e0eb // udot v11.4s, v7.16b, v0.4b[0]\n" - ".inst 0x6f81e0ef // udot v15.4s, v7.16b, v1.4b[0]\n" - ".inst 0x6f82e0f3 // udot v19.4s, v7.16b, v2.4b[0]\n" - ".inst 0x6f83e0f7 // udot v23.4s, v7.16b, v3.4b[0]\n" - ".inst 0x6f84e0fb // udot v27.4s, v7.16b, v4.4b[0]\n" - ".inst 0x6f85e0ff // udot v31.4s, v7.16b, v5.4b[0]\n" + ".inst 0x6f80e0ea // udot v10.4s, v7.16b, v0.4b[0]\n" + ".inst 0x6f81e0ee // udot v14.4s, v7.16b, v1.4b[0]\n" + ".inst 0x6f82e0f2 // udot v18.4s, v7.16b, v2.4b[0]\n" + ".inst 0x6f83e0f6 // udot v22.4s, v7.16b, v3.4b[0]\n" + ".inst 0x6f84e0fa // udot v26.4s, v7.16b, v4.4b[0]\n" + ".inst 0x6f85e0fe // udot v30.4s, v7.16b, v5.4b[0]\n" + ".inst 0x6f80e0cb // udot v11.4s, v6.16b, v0.4b[0]\n" + ".inst 0x6f81e0cf // udot v15.4s, v6.16b, v1.4b[0]\n" + ".inst 0x6f82e0d3 // udot v19.4s, v6.16b, v2.4b[0]\n" + ".inst 0x6f83e0d7 // udot v23.4s, v6.16b, v3.4b[0]\n" + ".inst 0x6f84e0db // udot v27.4s, v6.16b, v4.4b[0]\n" + ".inst 0x6f85e0df // udot v31.4s, v6.16b, v5.4b[0]\n" "194:" // Height 6: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -3254,7 +3253,6 @@ void a64_hybrid_u8u32_dot_6x16 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "206:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_mmla_6x16.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_mmla_6x16.hpp index b5cedc7e9831208f0ddac8f2a949c3db7edc693e..e3604521080831bb700179b759b2af037a5a1781 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_mmla_6x16.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_mmla_6x16.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef __aarch64__ + #include "../std_transforms_fixed.hpp" #include "../performance_parameters.hpp" @@ -109,5 +109,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_mmla_6x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_mmla_6x16/generic.cpp index dd0c46e4dcb1e7e569d6ad6db0cca00a32f50f63..364f388e793bdc688864a545c99e443e8dd5d9f3 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_mmla_6x16/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_mmla_6x16/generic.cpp @@ -77,7 +77,6 @@ void a64_hybrid_u8u32_mmla_6x16 ( ka.N = N; ka.B_ptr = B_ptr; __asm__ __volatile__( - "1:" // Row loop "cmp %x[M], #0x6\n" "bge 186f\n" @@ -178,11 +177,11 @@ void a64_hybrid_u8u32_mmla_6x16 ( "15:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 16f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 17f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -198,41 +197,41 @@ void a64_hybrid_u8u32_mmla_6x16 ( "ldr q6, [x10, #0x10]\n" "blt 19f\n" "18:" // Height 1: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x6e87a408 // ummla v8.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e86a40c // ummla v12.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e87a409 // ummla v9.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e86a40d // ummla v13.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e87a40a // ummla v10.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e86a40e // ummla v14.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" - "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e87a40b // ummla v11.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x80]\n" - ".inst 0x6e86a40f // ummla v15.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x90]\n" - ".inst 0x6e87a428 // ummla v8.4s, v1.16b, v7.16b\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x6e86a42c // ummla v12.4s, v1.16b, v6.16b\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x6e87a429 // ummla v9.4s, v1.16b, v7.16b\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x6e86a42d // ummla v13.4s, v1.16b, v6.16b\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x6e87a42a // ummla v10.4s, v1.16b, v7.16b\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x6e86a42e // ummla v14.4s, v1.16b, v6.16b\n" - "ldr q6, [x10, #0xf0]\n" + "trn1 v19.2d, v1.2d, v20.2d\n" + ".inst 0x6e87a668 // ummla v8.4s, v19.16b, v7.16b\n" + "ldr q18, [x10, #0x20]\n" + ".inst 0x6e86a66c // ummla v12.4s, v19.16b, v6.16b\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x6e92a669 // ummla v9.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x40]\n" + ".inst 0x6e91a66d // ummla v13.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x6e92a66a // ummla v10.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x60]\n" + ".inst 0x6e91a66e // ummla v14.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x70]\n" + "trn2 v1.2d, v1.2d, v20.2d\n" + ".inst 0x6e92a66b // ummla v11.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x80]\n" + ".inst 0x6e91a66f // ummla v15.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x90]\n" + ".inst 0x6e92a428 // ummla v8.4s, v1.16b, v18.16b\n" + "ldr q18, [x10, #0xa0]\n" + ".inst 0x6e91a42c // ummla v12.4s, v1.16b, v17.16b\n" + "ldr q17, [x10, #0xb0]\n" + ".inst 0x6e92a429 // ummla v9.4s, v1.16b, v18.16b\n" + "ldr q18, [x10, #0xc0]\n" + ".inst 0x6e91a42d // ummla v13.4s, v1.16b, v17.16b\n" + "ldr q17, [x10, #0xd0]\n" + ".inst 0x6e92a42a // ummla v10.4s, v1.16b, v18.16b\n" + "ldr q18, [x10, #0xe0]\n" + ".inst 0x6e91a42e // ummla v14.4s, v1.16b, v17.16b\n" + "ldr q17, [x10, #0xf0]\n" "sub x27, x27, #0x10\n" "add x26, x26, #0x10\n" "cmp x27, #0x20\n" - ".inst 0x6e87a42b // ummla v11.4s, v1.16b, v7.16b\n" - ".inst 0x6e86a42f // ummla v15.4s, v1.16b, v6.16b\n" + ".inst 0x6e92a42b // ummla v11.4s, v1.16b, v18.16b\n" + ".inst 0x6e91a42f // ummla v15.4s, v1.16b, v17.16b\n" "ldr q1, [x26, #0x0]\n" "add x10, x10, #0x100\n" "ldr q7, [x10, #0x0]\n" @@ -240,40 +239,40 @@ void a64_hybrid_u8u32_mmla_6x16 ( "prfm pldl1keep, [x26, #0x80]\n" "bge 18b\n" "19:" // Height 1: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x6e87a408 // ummla v8.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e86a40c // ummla v12.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e87a409 // ummla v9.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e86a40d // ummla v13.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e87a40a // ummla v10.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e86a40e // ummla v14.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" - "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e87a40b // ummla v11.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x80]\n" - ".inst 0x6e86a40f // ummla v15.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x90]\n" - ".inst 0x6e87a428 // ummla v8.4s, v1.16b, v7.16b\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x6e86a42c // ummla v12.4s, v1.16b, v6.16b\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x6e87a429 // ummla v9.4s, v1.16b, v7.16b\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x6e86a42d // ummla v13.4s, v1.16b, v6.16b\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x6e87a42a // ummla v10.4s, v1.16b, v7.16b\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x6e86a42e // ummla v14.4s, v1.16b, v6.16b\n" - "ldr q6, [x10, #0xf0]\n" + "trn1 v20.2d, v1.2d, v21.2d\n" + ".inst 0x6e87a688 // ummla v8.4s, v20.16b, v7.16b\n" + "ldr q18, [x10, #0x20]\n" + ".inst 0x6e86a68c // ummla v12.4s, v20.16b, v6.16b\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x6e92a689 // ummla v9.4s, v20.16b, v18.16b\n" + "ldr q18, [x10, #0x40]\n" + ".inst 0x6e91a68d // ummla v13.4s, v20.16b, v17.16b\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x6e92a68a // ummla v10.4s, v20.16b, v18.16b\n" + "ldr q19, [x10, #0x60]\n" + ".inst 0x6e91a68e // ummla v14.4s, v20.16b, v17.16b\n" + "ldr q18, [x10, #0x70]\n" + "trn2 v1.2d, v1.2d, v21.2d\n" + ".inst 0x6e93a68b // ummla v11.4s, v20.16b, v19.16b\n" + "ldr q17, [x10, #0x80]\n" + ".inst 0x6e92a68f // ummla v15.4s, v20.16b, v18.16b\n" + "ldr q19, [x10, #0x90]\n" + ".inst 0x6e91a428 // ummla v8.4s, v1.16b, v17.16b\n" + "ldr q18, [x10, #0xa0]\n" + ".inst 0x6e93a42c // ummla v12.4s, v1.16b, v19.16b\n" + "ldr q17, [x10, #0xb0]\n" + ".inst 0x6e92a429 // ummla v9.4s, v1.16b, v18.16b\n" + "ldr q18, [x10, #0xc0]\n" + ".inst 0x6e91a42d // ummla v13.4s, v1.16b, v17.16b\n" + "ldr q17, [x10, #0xd0]\n" + ".inst 0x6e92a42a // ummla v10.4s, v1.16b, v18.16b\n" + "ldr q18, [x10, #0xe0]\n" + ".inst 0x6e91a42e // ummla v14.4s, v1.16b, v17.16b\n" + "ldr q17, [x10, #0xf0]\n" "add x26, x26, #0x10\n" "sub x27, x27, #0x10\n" - ".inst 0x6e87a42b // ummla v11.4s, v1.16b, v7.16b\n" - ".inst 0x6e86a42f // ummla v15.4s, v1.16b, v6.16b\n" + ".inst 0x6e92a42b // ummla v11.4s, v1.16b, v18.16b\n" + ".inst 0x6e91a42f // ummla v15.4s, v1.16b, v17.16b\n" "prfm pldl1keep, [x26, #0x80]\n" "add x10, x10, #0x100\n" "20:" // Height 1: Multiply loop: Main loop skip @@ -281,26 +280,26 @@ void a64_hybrid_u8u32_mmla_6x16 ( "cmp x27, #0x8\n" "blt 22f\n" "21:" // Height 1: Multiply loop: Odd block loop - "ldr d1, [x26], #0x8\n" - "ldr q6, [x10, #0x0]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6e86a408 // ummla v8.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6e87a40c // ummla v12.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e86a409 // ummla v9.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e87a40d // ummla v13.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e86a40a // ummla v10.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6e87a40e // ummla v14.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x70]\n" + "ldr d19, [x26], #0x8\n" + "ldr q18, [x10, #0x0]\n" + "trn1 v19.2d, v19.2d, v17.2d\n" + "ldr q17, [x10, #0x10]\n" + ".inst 0x6e92a668 // ummla v8.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x20]\n" + ".inst 0x6e91a66c // ummla v12.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x6e92a669 // ummla v9.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x40]\n" + ".inst 0x6e91a66d // ummla v13.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x6e92a66a // ummla v10.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x60]\n" + ".inst 0x6e91a66e // ummla v14.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x70]\n" "sub x27, x27, #0x8\n" "cmp x27, #0x8\n" - ".inst 0x6e86a40b // ummla v11.4s, v0.16b, v6.16b\n" - ".inst 0x6e87a40f // ummla v15.4s, v0.16b, v7.16b\n" + ".inst 0x6e92a66b // ummla v11.4s, v19.16b, v18.16b\n" + ".inst 0x6e91a66f // ummla v15.4s, v19.16b, v17.16b\n" "add x10, x10, #0x80\n" "bge 21b\n" "22:" // Height 1: Multiply loop: Skip odd blocks @@ -325,23 +324,23 @@ void a64_hybrid_u8u32_mmla_6x16 ( "25:" // Height 1: Multiply loop: Ragged operand read: partial_1_0 "ldr b1, [x26, #0x0]\n" "26:" // Height 1: Multiply loop: Ragged operand read: Done - "ldr q7, [x10, #0x0]\n" - "ldr q6, [x10, #0x10]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x6e87a408 // ummla v8.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e86a40c // ummla v12.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e87a409 // ummla v9.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e86a40d // ummla v13.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e87a40a // ummla v10.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e86a40e // ummla v14.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" - ".inst 0x6e87a40b // ummla v11.4s, v0.16b, v7.16b\n" - ".inst 0x6e86a40f // ummla v15.4s, v0.16b, v6.16b\n" + "ldr q23, [x10, #0x0]\n" + "ldr q18, [x10, #0x10]\n" + "trn1 v19.2d, v1.2d, v17.2d\n" + ".inst 0x6e97a668 // ummla v8.4s, v19.16b, v23.16b\n" + "ldr q17, [x10, #0x20]\n" + ".inst 0x6e92a66c // ummla v12.4s, v19.16b, v18.16b\n" + "ldr q31, [x10, #0x30]\n" + ".inst 0x6e91a669 // ummla v9.4s, v19.16b, v17.16b\n" + "ldr q20, [x10, #0x40]\n" + ".inst 0x6e9fa66d // ummla v13.4s, v19.16b, v31.16b\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x6e94a66a // ummla v10.4s, v19.16b, v20.16b\n" + "ldr q18, [x10, #0x60]\n" + ".inst 0x6e91a66e // ummla v14.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x70]\n" + ".inst 0x6e92a66b // ummla v11.4s, v19.16b, v18.16b\n" + ".inst 0x6e91a66f // ummla v15.4s, v19.16b, v17.16b\n" "add x10, x10, #0x80\n" "27:" // Height 1: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -525,12 +524,12 @@ void a64_hybrid_u8u32_mmla_6x16 ( "52:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 53f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 54f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -538,7 +537,7 @@ void a64_hybrid_u8u32_mmla_6x16 ( "b 54f\n" "53:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" + "add x25, x26, x21\n" "54:" // Height 2: input setup done "cmp x27, #0x10\n" "blt 57f\n" @@ -549,85 +548,85 @@ void a64_hybrid_u8u32_mmla_6x16 ( "ldr q6, [x10, #0x10]\n" "blt 56f\n" "55:" // Height 2: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x6e87a408 // ummla v8.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e86a40c // ummla v12.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e87a409 // ummla v9.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e86a40d // ummla v13.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e87a40a // ummla v10.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e86a40e // ummla v14.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" + "trn1 v19.2d, v1.2d, v2.2d\n" + ".inst 0x6e87a668 // ummla v8.4s, v19.16b, v7.16b\n" + "ldr q18, [x10, #0x20]\n" + ".inst 0x6e86a66c // ummla v12.4s, v19.16b, v6.16b\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x6e92a669 // ummla v9.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x40]\n" + ".inst 0x6e91a66d // ummla v13.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x6e92a66a // ummla v10.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x60]\n" + ".inst 0x6e91a66e // ummla v14.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x70]\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e87a40b // ummla v11.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x80]\n" - ".inst 0x6e86a40f // ummla v15.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x90]\n" - ".inst 0x6e87a428 // ummla v8.4s, v1.16b, v7.16b\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x6e86a42c // ummla v12.4s, v1.16b, v6.16b\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x6e87a429 // ummla v9.4s, v1.16b, v7.16b\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x6e86a42d // ummla v13.4s, v1.16b, v6.16b\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x6e87a42a // ummla v10.4s, v1.16b, v7.16b\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x6e86a42e // ummla v14.4s, v1.16b, v6.16b\n" - "ldr q6, [x10, #0xf0]\n" + ".inst 0x6e92a66b // ummla v11.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x80]\n" + ".inst 0x6e91a66f // ummla v15.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x90]\n" + ".inst 0x6e92a428 // ummla v8.4s, v1.16b, v18.16b\n" + "ldr q18, [x10, #0xa0]\n" + ".inst 0x6e91a42c // ummla v12.4s, v1.16b, v17.16b\n" + "ldr q17, [x10, #0xb0]\n" + ".inst 0x6e92a429 // ummla v9.4s, v1.16b, v18.16b\n" + "ldr q18, [x10, #0xc0]\n" + ".inst 0x6e91a42d // ummla v13.4s, v1.16b, v17.16b\n" + "ldr q17, [x10, #0xd0]\n" + ".inst 0x6e92a42a // ummla v10.4s, v1.16b, v18.16b\n" + "ldr q18, [x10, #0xe0]\n" + ".inst 0x6e91a42e // ummla v14.4s, v1.16b, v17.16b\n" + "ldr q17, [x10, #0xf0]\n" "sub x27, x27, #0x10\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" "ldr q2, [x25, #0x0]\n" "cmp x27, #0x20\n" - ".inst 0x6e87a42b // ummla v11.4s, v1.16b, v7.16b\n" + ".inst 0x6e92a42b // ummla v11.4s, v1.16b, v18.16b\n" "add x10, x10, #0x100\n" "ldr q7, [x10, #0x0]\n" - ".inst 0x6e86a42f // ummla v15.4s, v1.16b, v6.16b\n" + ".inst 0x6e91a42f // ummla v15.4s, v1.16b, v17.16b\n" "ldr q1, [x26, #0x0]\n" "ldr q6, [x10, #0x10]\n" "prfm pldl1keep, [x26, #0x80]\n" "prfm pldl1keep, [x25, #0x80]\n" "bge 55b\n" "56:" // Height 2: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x6e87a408 // ummla v8.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e86a40c // ummla v12.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e87a409 // ummla v9.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e86a40d // ummla v13.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e87a40a // ummla v10.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e86a40e // ummla v14.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" + "trn1 v19.2d, v1.2d, v2.2d\n" + ".inst 0x6e87a668 // ummla v8.4s, v19.16b, v7.16b\n" + "ldr q18, [x10, #0x20]\n" + ".inst 0x6e86a66c // ummla v12.4s, v19.16b, v6.16b\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x6e92a669 // ummla v9.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x40]\n" + ".inst 0x6e91a66d // ummla v13.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x6e92a66a // ummla v10.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x60]\n" + ".inst 0x6e91a66e // ummla v14.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x70]\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e87a40b // ummla v11.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x80]\n" - ".inst 0x6e86a40f // ummla v15.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x90]\n" - ".inst 0x6e87a428 // ummla v8.4s, v1.16b, v7.16b\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x6e86a42c // ummla v12.4s, v1.16b, v6.16b\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x6e87a429 // ummla v9.4s, v1.16b, v7.16b\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x6e86a42d // ummla v13.4s, v1.16b, v6.16b\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x6e87a42a // ummla v10.4s, v1.16b, v7.16b\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x6e86a42e // ummla v14.4s, v1.16b, v6.16b\n" - "ldr q6, [x10, #0xf0]\n" + ".inst 0x6e92a66b // ummla v11.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x80]\n" + ".inst 0x6e91a66f // ummla v15.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x90]\n" + ".inst 0x6e92a428 // ummla v8.4s, v1.16b, v18.16b\n" + "ldr q18, [x10, #0xa0]\n" + ".inst 0x6e91a42c // ummla v12.4s, v1.16b, v17.16b\n" + "ldr q17, [x10, #0xb0]\n" + ".inst 0x6e92a429 // ummla v9.4s, v1.16b, v18.16b\n" + "ldr q18, [x10, #0xc0]\n" + ".inst 0x6e91a42d // ummla v13.4s, v1.16b, v17.16b\n" + "ldr q17, [x10, #0xd0]\n" + ".inst 0x6e92a42a // ummla v10.4s, v1.16b, v18.16b\n" + "ldr q18, [x10, #0xe0]\n" + ".inst 0x6e91a42e // ummla v14.4s, v1.16b, v17.16b\n" + "ldr q17, [x10, #0xf0]\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" - ".inst 0x6e87a42b // ummla v11.4s, v1.16b, v7.16b\n" - ".inst 0x6e86a42f // ummla v15.4s, v1.16b, v6.16b\n" + ".inst 0x6e92a42b // ummla v11.4s, v1.16b, v18.16b\n" + ".inst 0x6e91a42f // ummla v15.4s, v1.16b, v17.16b\n" "sub x27, x27, #0x10\n" "prfm pldl1keep, [x26, #0x80]\n" "prfm pldl1keep, [x25, #0x80]\n" @@ -637,27 +636,27 @@ void a64_hybrid_u8u32_mmla_6x16 ( "cmp x27, #0x8\n" "blt 59f\n" "58:" // Height 2: Multiply loop: Odd block loop - "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" + "ldr d18, [x26], #0x8\n" + "ldr d17, [x25], #0x8\n" + "trn1 v19.2d, v18.2d, v17.2d\n" "sub x27, x27, #0x8\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6e86a408 // ummla v8.4s, v0.16b, v6.16b\n" - ".inst 0x6e87a40c // ummla v12.4s, v0.16b, v7.16b\n" - "ldr q6, [x10, #0x20]\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e86a409 // ummla v9.4s, v0.16b, v6.16b\n" - ".inst 0x6e87a40d // ummla v13.4s, v0.16b, v7.16b\n" - "ldr q6, [x10, #0x40]\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e86a40a // ummla v10.4s, v0.16b, v6.16b\n" - ".inst 0x6e87a40e // ummla v14.4s, v0.16b, v7.16b\n" - "ldr q6, [x10, #0x60]\n" - "ldr q7, [x10, #0x70]\n" + "ldr q17, [x10, #0x0]\n" + "ldr q22, [x10, #0x10]\n" + ".inst 0x6e91a668 // ummla v8.4s, v19.16b, v17.16b\n" + ".inst 0x6e96a66c // ummla v12.4s, v19.16b, v22.16b\n" + "ldr q1, [x10, #0x20]\n" + "ldr q17, [x10, #0x30]\n" + ".inst 0x6e81a669 // ummla v9.4s, v19.16b, v1.16b\n" + ".inst 0x6e91a66d // ummla v13.4s, v19.16b, v17.16b\n" + "ldr q18, [x10, #0x40]\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x6e92a66a // ummla v10.4s, v19.16b, v18.16b\n" + ".inst 0x6e91a66e // ummla v14.4s, v19.16b, v17.16b\n" + "ldr q18, [x10, #0x60]\n" + "ldr q17, [x10, #0x70]\n" "cmp x27, #0x8\n" - ".inst 0x6e86a40b // ummla v11.4s, v0.16b, v6.16b\n" - ".inst 0x6e87a40f // ummla v15.4s, v0.16b, v7.16b\n" + ".inst 0x6e92a66b // ummla v11.4s, v19.16b, v18.16b\n" + ".inst 0x6e91a66f // ummla v15.4s, v19.16b, v17.16b\n" "add x10, x10, #0x80\n" "bge 58b\n" "59:" // Height 2: Multiply loop: Skip odd blocks @@ -689,23 +688,23 @@ void a64_hybrid_u8u32_mmla_6x16 ( "ldr b1, [x26, #0x0]\n" "ldr b2, [x25, #0x0]\n" "63:" // Height 2: Multiply loop: Ragged operand read: Done - "ldr q7, [x10, #0x0]\n" - "ldr q6, [x10, #0x10]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - ".inst 0x6e87a408 // ummla v8.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e86a40c // ummla v12.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e87a409 // ummla v9.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e86a40d // ummla v13.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e87a40a // ummla v10.4s, v0.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e86a40e // ummla v14.4s, v0.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" - ".inst 0x6e87a40b // ummla v11.4s, v0.16b, v7.16b\n" - ".inst 0x6e86a40f // ummla v15.4s, v0.16b, v6.16b\n" + "ldr q18, [x10, #0x0]\n" + "ldr q17, [x10, #0x10]\n" + "trn1 v19.2d, v1.2d, v2.2d\n" + ".inst 0x6e92a668 // ummla v8.4s, v19.16b, v18.16b\n" + "ldr q5, [x10, #0x20]\n" + ".inst 0x6e91a66c // ummla v12.4s, v19.16b, v17.16b\n" + "ldr q21, [x10, #0x30]\n" + ".inst 0x6e85a669 // ummla v9.4s, v19.16b, v5.16b\n" + "ldr q18, [x10, #0x40]\n" + ".inst 0x6e95a66d // ummla v13.4s, v19.16b, v21.16b\n" + "ldr q17, [x10, #0x50]\n" + ".inst 0x6e92a66a // ummla v10.4s, v19.16b, v18.16b\n" + "ldr q18, [x10, #0x60]\n" + ".inst 0x6e91a66e // ummla v14.4s, v19.16b, v17.16b\n" + "ldr q17, [x10, #0x70]\n" + ".inst 0x6e92a66b // ummla v11.4s, v19.16b, v18.16b\n" + ".inst 0x6e91a66f // ummla v15.4s, v19.16b, v17.16b\n" "add x10, x10, #0x80\n" "64:" // Height 2: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -953,13 +952,13 @@ void a64_hybrid_u8u32_mmla_6x16 ( "89:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 90f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 91f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -968,8 +967,8 @@ void a64_hybrid_u8u32_mmla_6x16 ( "b 91f\n" "90:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" "91:" // Height 3: input setup done "cmp x27, #0x10\n" "blt 94f\n" @@ -981,167 +980,167 @@ void a64_hybrid_u8u32_mmla_6x16 ( "ldr q6, [x10, #0x10]\n" "blt 93f\n" "92:" // Height 3: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v28.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e87a408 // ummla v8.4s, v0.16b, v7.16b\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x6e87a450 // ummla v16.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e86a40c // ummla v12.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a454 // ummla v20.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e87a409 // ummla v9.4s, v0.16b, v7.16b\n" - "trn2 v3.2d, v3.2d, v4.2d\n" - ".inst 0x6e87a451 // ummla v17.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e86a40d // ummla v13.4s, v0.16b, v6.16b\n" + ".inst 0x6e87a788 // ummla v8.4s, v28.16b, v7.16b\n" + "trn1 v27.2d, v3.2d, v29.2d\n" + ".inst 0x6e87a770 // ummla v16.4s, v27.16b, v7.16b\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x6e86a78c // ummla v12.4s, v28.16b, v6.16b\n" + ".inst 0x6e86a774 // ummla v20.4s, v27.16b, v6.16b\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e9aa789 // ummla v9.4s, v28.16b, v26.16b\n" + "trn2 v3.2d, v3.2d, v29.2d\n" + ".inst 0x6e9aa771 // ummla v17.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x6e99a78d // ummla v13.4s, v28.16b, v25.16b\n" "sub x27, x27, #0x10\n" - ".inst 0x6e86a455 // ummla v21.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e87a40a // ummla v10.4s, v0.16b, v7.16b\n" + ".inst 0x6e99a775 // ummla v21.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x6e9aa78a // ummla v10.4s, v28.16b, v26.16b\n" "add x26, x26, #0x10\n" - ".inst 0x6e87a452 // ummla v18.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e86a40e // ummla v14.4s, v0.16b, v6.16b\n" + ".inst 0x6e9aa772 // ummla v18.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x6e99a78e // ummla v14.4s, v28.16b, v25.16b\n" "add x25, x25, #0x10\n" - ".inst 0x6e86a456 // ummla v22.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" - ".inst 0x6e87a40b // ummla v11.4s, v0.16b, v7.16b\n" + ".inst 0x6e99a776 // ummla v22.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x70]\n" + ".inst 0x6e9aa78b // ummla v11.4s, v28.16b, v26.16b\n" "add x24, x24, #0x10\n" - ".inst 0x6e87a453 // ummla v19.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x80]\n" - ".inst 0x6e86a40f // ummla v15.4s, v0.16b, v6.16b\n" + ".inst 0x6e9aa773 // ummla v19.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x80]\n" + ".inst 0x6e99a78f // ummla v15.4s, v28.16b, v25.16b\n" "cmp x27, #0x20\n" - ".inst 0x6e86a457 // ummla v23.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x90]\n" + ".inst 0x6e99a777 // ummla v23.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x90]\n" "ldr q2, [x25, #0x0]\n" - ".inst 0x6e87a428 // ummla v8.4s, v1.16b, v7.16b\n" - ".inst 0x6e87a470 // ummla v16.4s, v3.16b, v7.16b\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x6e86a42c // ummla v12.4s, v1.16b, v6.16b\n" + ".inst 0x6e9aa428 // ummla v8.4s, v1.16b, v26.16b\n" + ".inst 0x6e9aa470 // ummla v16.4s, v3.16b, v26.16b\n" + "ldr q26, [x10, #0xa0]\n" + ".inst 0x6e99a42c // ummla v12.4s, v1.16b, v25.16b\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x6e86a474 // ummla v20.4s, v3.16b, v6.16b\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x6e87a429 // ummla v9.4s, v1.16b, v7.16b\n" + ".inst 0x6e99a474 // ummla v20.4s, v3.16b, v25.16b\n" + "ldr q25, [x10, #0xb0]\n" + ".inst 0x6e9aa429 // ummla v9.4s, v1.16b, v26.16b\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x6e87a471 // ummla v17.4s, v3.16b, v7.16b\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x6e86a42d // ummla v13.4s, v1.16b, v6.16b\n" + ".inst 0x6e9aa471 // ummla v17.4s, v3.16b, v26.16b\n" + "ldr q26, [x10, #0xc0]\n" + ".inst 0x6e99a42d // ummla v13.4s, v1.16b, v25.16b\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x6e86a475 // ummla v21.4s, v3.16b, v6.16b\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x6e87a42a // ummla v10.4s, v1.16b, v7.16b\n" - ".inst 0x6e87a472 // ummla v18.4s, v3.16b, v7.16b\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x6e86a42e // ummla v14.4s, v1.16b, v6.16b\n" - ".inst 0x6e86a476 // ummla v22.4s, v3.16b, v6.16b\n" - "ldr q6, [x10, #0xf0]\n" + ".inst 0x6e99a475 // ummla v21.4s, v3.16b, v25.16b\n" + "ldr q25, [x10, #0xd0]\n" + ".inst 0x6e9aa42a // ummla v10.4s, v1.16b, v26.16b\n" + ".inst 0x6e9aa472 // ummla v18.4s, v3.16b, v26.16b\n" + "ldr q26, [x10, #0xe0]\n" + ".inst 0x6e99a42e // ummla v14.4s, v1.16b, v25.16b\n" + ".inst 0x6e99a476 // ummla v22.4s, v3.16b, v25.16b\n" + "ldr q25, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x6e87a42b // ummla v11.4s, v1.16b, v7.16b\n" - ".inst 0x6e87a473 // ummla v19.4s, v3.16b, v7.16b\n" + ".inst 0x6e9aa42b // ummla v11.4s, v1.16b, v26.16b\n" + ".inst 0x6e9aa473 // ummla v19.4s, v3.16b, v26.16b\n" "ldr q7, [x10, #0x0]\n" - ".inst 0x6e86a42f // ummla v15.4s, v1.16b, v6.16b\n" + ".inst 0x6e99a42f // ummla v15.4s, v1.16b, v25.16b\n" "ldr q1, [x26, #0x0]\n" - ".inst 0x6e86a477 // ummla v23.4s, v3.16b, v6.16b\n" + ".inst 0x6e99a477 // ummla v23.4s, v3.16b, v25.16b\n" "ldr q3, [x24, #0x0]\n" "ldr q6, [x10, #0x10]\n" "bge 92b\n" "93:" // Height 3: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v28.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e87a408 // ummla v8.4s, v0.16b, v7.16b\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x6e87a450 // ummla v16.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e86a40c // ummla v12.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a454 // ummla v20.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e87a409 // ummla v9.4s, v0.16b, v7.16b\n" - "trn2 v3.2d, v3.2d, v4.2d\n" - ".inst 0x6e87a451 // ummla v17.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e86a40d // ummla v13.4s, v0.16b, v6.16b\n" + ".inst 0x6e87a788 // ummla v8.4s, v28.16b, v7.16b\n" + "trn1 v27.2d, v3.2d, v29.2d\n" + ".inst 0x6e87a770 // ummla v16.4s, v27.16b, v7.16b\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x6e86a78c // ummla v12.4s, v28.16b, v6.16b\n" + ".inst 0x6e86a774 // ummla v20.4s, v27.16b, v6.16b\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e9aa789 // ummla v9.4s, v28.16b, v26.16b\n" + "trn2 v3.2d, v3.2d, v29.2d\n" + ".inst 0x6e9aa771 // ummla v17.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x6e99a78d // ummla v13.4s, v28.16b, v25.16b\n" "add x26, x26, #0x10\n" - ".inst 0x6e86a455 // ummla v21.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e87a40a // ummla v10.4s, v0.16b, v7.16b\n" + ".inst 0x6e99a775 // ummla v21.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x6e9aa78a // ummla v10.4s, v28.16b, v26.16b\n" "add x25, x25, #0x10\n" - ".inst 0x6e87a452 // ummla v18.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e86a40e // ummla v14.4s, v0.16b, v6.16b\n" + ".inst 0x6e9aa772 // ummla v18.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x6e99a78e // ummla v14.4s, v28.16b, v25.16b\n" "add x24, x24, #0x10\n" - ".inst 0x6e86a456 // ummla v22.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" - ".inst 0x6e87a40b // ummla v11.4s, v0.16b, v7.16b\n" + ".inst 0x6e99a776 // ummla v22.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x70]\n" + ".inst 0x6e9aa78b // ummla v11.4s, v28.16b, v26.16b\n" "sub x27, x27, #0x10\n" - ".inst 0x6e87a453 // ummla v19.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x80]\n" - ".inst 0x6e86a40f // ummla v15.4s, v0.16b, v6.16b\n" + ".inst 0x6e9aa773 // ummla v19.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x80]\n" + ".inst 0x6e99a78f // ummla v15.4s, v28.16b, v25.16b\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x6e86a457 // ummla v23.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x90]\n" - ".inst 0x6e87a428 // ummla v8.4s, v1.16b, v7.16b\n" + ".inst 0x6e99a777 // ummla v23.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x90]\n" + ".inst 0x6e9aa428 // ummla v8.4s, v1.16b, v26.16b\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x6e87a470 // ummla v16.4s, v3.16b, v7.16b\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x6e86a42c // ummla v12.4s, v1.16b, v6.16b\n" + ".inst 0x6e9aa470 // ummla v16.4s, v3.16b, v26.16b\n" + "ldr q26, [x10, #0xa0]\n" + ".inst 0x6e99a42c // ummla v12.4s, v1.16b, v25.16b\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x6e86a474 // ummla v20.4s, v3.16b, v6.16b\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x6e87a429 // ummla v9.4s, v1.16b, v7.16b\n" - ".inst 0x6e87a471 // ummla v17.4s, v3.16b, v7.16b\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x6e86a42d // ummla v13.4s, v1.16b, v6.16b\n" - ".inst 0x6e86a475 // ummla v21.4s, v3.16b, v6.16b\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x6e87a42a // ummla v10.4s, v1.16b, v7.16b\n" - ".inst 0x6e87a472 // ummla v18.4s, v3.16b, v7.16b\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x6e86a42e // ummla v14.4s, v1.16b, v6.16b\n" - ".inst 0x6e86a476 // ummla v22.4s, v3.16b, v6.16b\n" - "ldr q6, [x10, #0xf0]\n" + ".inst 0x6e99a474 // ummla v20.4s, v3.16b, v25.16b\n" + "ldr q25, [x10, #0xb0]\n" + ".inst 0x6e9aa429 // ummla v9.4s, v1.16b, v26.16b\n" + ".inst 0x6e9aa471 // ummla v17.4s, v3.16b, v26.16b\n" + "ldr q26, [x10, #0xc0]\n" + ".inst 0x6e99a42d // ummla v13.4s, v1.16b, v25.16b\n" + ".inst 0x6e99a475 // ummla v21.4s, v3.16b, v25.16b\n" + "ldr q25, [x10, #0xd0]\n" + ".inst 0x6e9aa42a // ummla v10.4s, v1.16b, v26.16b\n" + ".inst 0x6e9aa472 // ummla v18.4s, v3.16b, v26.16b\n" + "ldr q26, [x10, #0xe0]\n" + ".inst 0x6e99a42e // ummla v14.4s, v1.16b, v25.16b\n" + ".inst 0x6e99a476 // ummla v22.4s, v3.16b, v25.16b\n" + "ldr q25, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x6e87a42b // ummla v11.4s, v1.16b, v7.16b\n" - ".inst 0x6e87a473 // ummla v19.4s, v3.16b, v7.16b\n" - ".inst 0x6e86a42f // ummla v15.4s, v1.16b, v6.16b\n" - ".inst 0x6e86a477 // ummla v23.4s, v3.16b, v6.16b\n" + ".inst 0x6e9aa42b // ummla v11.4s, v1.16b, v26.16b\n" + ".inst 0x6e9aa473 // ummla v19.4s, v3.16b, v26.16b\n" + ".inst 0x6e99a42f // ummla v15.4s, v1.16b, v25.16b\n" + ".inst 0x6e99a477 // ummla v23.4s, v3.16b, v25.16b\n" "94:" // Height 3: Multiply loop: Main loop skip "cbz x27, 101f\n" "cmp x27, #0x8\n" "blt 96f\n" "95:" // Height 3: Multiply loop: Odd block loop - "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "ldr d3, [x24], #0x8\n" - "ldr q6, [x10, #0x0]\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x6e86a408 // ummla v8.4s, v0.16b, v6.16b\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6e86a450 // ummla v16.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6e87a40c // ummla v12.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a454 // ummla v20.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e86a409 // ummla v9.4s, v0.16b, v6.16b\n" + "ldr d26, [x26], #0x8\n" + "ldr d25, [x25], #0x8\n" + "trn1 v28.2d, v26.2d, v25.2d\n" + "ldr d25, [x24], #0x8\n" + "ldr q26, [x10, #0x0]\n" + "trn1 v27.2d, v25.2d, v27.2d\n" + ".inst 0x6e9aa788 // ummla v8.4s, v28.16b, v26.16b\n" + "ldr q25, [x10, #0x10]\n" + ".inst 0x6e9aa770 // ummla v16.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x6e99a78c // ummla v12.4s, v28.16b, v25.16b\n" + ".inst 0x6e99a774 // ummla v20.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e9aa789 // ummla v9.4s, v28.16b, v26.16b\n" "sub x27, x27, #0x8\n" - ".inst 0x6e86a451 // ummla v17.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e87a40d // ummla v13.4s, v0.16b, v7.16b\n" + ".inst 0x6e9aa771 // ummla v17.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x6e99a78d // ummla v13.4s, v28.16b, v25.16b\n" "cmp x27, #0x8\n" - ".inst 0x6e87a455 // ummla v21.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e86a40a // ummla v10.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a452 // ummla v18.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6e87a40e // ummla v14.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a456 // ummla v22.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x6e86a40b // ummla v11.4s, v0.16b, v6.16b\n" + ".inst 0x6e99a775 // ummla v21.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x6e9aa78a // ummla v10.4s, v28.16b, v26.16b\n" + ".inst 0x6e9aa772 // ummla v18.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x6e99a78e // ummla v14.4s, v28.16b, v25.16b\n" + ".inst 0x6e99a776 // ummla v22.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x70]\n" + ".inst 0x6e9aa78b // ummla v11.4s, v28.16b, v26.16b\n" "add x10, x10, #0x80\n" - ".inst 0x6e86a453 // ummla v19.4s, v2.16b, v6.16b\n" - ".inst 0x6e87a40f // ummla v15.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a457 // ummla v23.4s, v2.16b, v7.16b\n" + ".inst 0x6e9aa773 // ummla v19.4s, v27.16b, v26.16b\n" + ".inst 0x6e99a78f // ummla v15.4s, v28.16b, v25.16b\n" + ".inst 0x6e99a777 // ummla v23.4s, v27.16b, v25.16b\n" "bge 95b\n" "96:" // Height 3: Multiply loop: Skip odd blocks "cbz x27, 101f\n" @@ -1179,33 +1178,33 @@ void a64_hybrid_u8u32_mmla_6x16 ( "ldr b2, [x25, #0x0]\n" "ldr b3, [x24, #0x0]\n" "100:" // Height 3: Multiply loop: Ragged operand read: Done - "ldr q7, [x10, #0x0]\n" - "ldr q6, [x10, #0x10]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x6e87a408 // ummla v8.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a450 // ummla v16.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e86a40c // ummla v12.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a454 // ummla v20.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e87a409 // ummla v9.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a451 // ummla v17.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e86a40d // ummla v13.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a455 // ummla v21.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e87a40a // ummla v10.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a452 // ummla v18.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e86a40e // ummla v14.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a456 // ummla v22.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" + "ldr q26, [x10, #0x0]\n" + "ldr q29, [x10, #0x10]\n" + "trn1 v28.2d, v1.2d, v2.2d\n" + "trn1 v27.2d, v3.2d, v25.2d\n" + ".inst 0x6e9aa788 // ummla v8.4s, v28.16b, v26.16b\n" + ".inst 0x6e9aa770 // ummla v16.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x6e9da78c // ummla v12.4s, v28.16b, v29.16b\n" + ".inst 0x6e9da774 // ummla v20.4s, v27.16b, v29.16b\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e9aa789 // ummla v9.4s, v28.16b, v26.16b\n" + ".inst 0x6e9aa771 // ummla v17.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x6e99a78d // ummla v13.4s, v28.16b, v25.16b\n" + ".inst 0x6e99a775 // ummla v21.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x6e9aa78a // ummla v10.4s, v28.16b, v26.16b\n" + ".inst 0x6e9aa772 // ummla v18.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x6e99a78e // ummla v14.4s, v28.16b, v25.16b\n" + ".inst 0x6e99a776 // ummla v22.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x70]\n" "add x10, x10, #0x80\n" - ".inst 0x6e87a40b // ummla v11.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a453 // ummla v19.4s, v2.16b, v7.16b\n" - ".inst 0x6e86a40f // ummla v15.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a457 // ummla v23.4s, v2.16b, v6.16b\n" + ".inst 0x6e9aa78b // ummla v11.4s, v28.16b, v26.16b\n" + ".inst 0x6e9aa773 // ummla v19.4s, v27.16b, v26.16b\n" + ".inst 0x6e99a78f // ummla v15.4s, v28.16b, v25.16b\n" + ".inst 0x6e99a777 // ummla v23.4s, v27.16b, v25.16b\n" "101:" // Height 3: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1499,14 +1498,14 @@ void a64_hybrid_u8u32_mmla_6x16 ( "126:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 127f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 128f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -1516,9 +1515,9 @@ void a64_hybrid_u8u32_mmla_6x16 ( "b 128f\n" "127:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" "128:" // Height 4: input setup done "cmp x27, #0x10\n" "blt 131f\n" @@ -1531,173 +1530,173 @@ void a64_hybrid_u8u32_mmla_6x16 ( "ldr q6, [x10, #0x10]\n" "blt 130f\n" "129:" // Height 4: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v28.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e87a408 // ummla v8.4s, v0.16b, v7.16b\n" + ".inst 0x6e87a788 // ummla v8.4s, v28.16b, v7.16b\n" "sub x27, x27, #0x10\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x6e87a450 // ummla v16.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e86a40c // ummla v12.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a454 // ummla v20.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e87a409 // ummla v9.4s, v0.16b, v7.16b\n" + "trn1 v27.2d, v3.2d, v4.2d\n" + ".inst 0x6e87a770 // ummla v16.4s, v27.16b, v7.16b\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x6e86a78c // ummla v12.4s, v28.16b, v6.16b\n" + ".inst 0x6e86a774 // ummla v20.4s, v27.16b, v6.16b\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e9aa789 // ummla v9.4s, v28.16b, v26.16b\n" "trn2 v3.2d, v3.2d, v4.2d\n" - ".inst 0x6e87a451 // ummla v17.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e86a40d // ummla v13.4s, v0.16b, v6.16b\n" + ".inst 0x6e9aa771 // ummla v17.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x6e99a78d // ummla v13.4s, v28.16b, v25.16b\n" "add x26, x26, #0x10\n" - ".inst 0x6e86a455 // ummla v21.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e87a40a // ummla v10.4s, v0.16b, v7.16b\n" + ".inst 0x6e99a775 // ummla v21.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x6e9aa78a // ummla v10.4s, v28.16b, v26.16b\n" "add x25, x25, #0x10\n" - ".inst 0x6e87a452 // ummla v18.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e86a40e // ummla v14.4s, v0.16b, v6.16b\n" + ".inst 0x6e9aa772 // ummla v18.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x6e99a78e // ummla v14.4s, v28.16b, v25.16b\n" "add x24, x24, #0x10\n" - ".inst 0x6e86a456 // ummla v22.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" - ".inst 0x6e87a40b // ummla v11.4s, v0.16b, v7.16b\n" + ".inst 0x6e99a776 // ummla v22.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x70]\n" + ".inst 0x6e9aa78b // ummla v11.4s, v28.16b, v26.16b\n" "add x23, x23, #0x10\n" "ldr q4, [x23, #0x0]\n" - ".inst 0x6e87a453 // ummla v19.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x80]\n" - ".inst 0x6e86a40f // ummla v15.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a457 // ummla v23.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x90]\n" + ".inst 0x6e9aa773 // ummla v19.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x80]\n" + ".inst 0x6e99a78f // ummla v15.4s, v28.16b, v25.16b\n" + ".inst 0x6e99a777 // ummla v23.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x90]\n" "ldr q2, [x25, #0x0]\n" - ".inst 0x6e87a428 // ummla v8.4s, v1.16b, v7.16b\n" - ".inst 0x6e87a470 // ummla v16.4s, v3.16b, v7.16b\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x6e86a42c // ummla v12.4s, v1.16b, v6.16b\n" + ".inst 0x6e9aa428 // ummla v8.4s, v1.16b, v26.16b\n" + ".inst 0x6e9aa470 // ummla v16.4s, v3.16b, v26.16b\n" + "ldr q26, [x10, #0xa0]\n" + ".inst 0x6e99a42c // ummla v12.4s, v1.16b, v25.16b\n" "cmp x27, #0x20\n" - ".inst 0x6e86a474 // ummla v20.4s, v3.16b, v6.16b\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x6e87a429 // ummla v9.4s, v1.16b, v7.16b\n" + ".inst 0x6e99a474 // ummla v20.4s, v3.16b, v25.16b\n" + "ldr q25, [x10, #0xb0]\n" + ".inst 0x6e9aa429 // ummla v9.4s, v1.16b, v26.16b\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x6e87a471 // ummla v17.4s, v3.16b, v7.16b\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x6e86a42d // ummla v13.4s, v1.16b, v6.16b\n" + ".inst 0x6e9aa471 // ummla v17.4s, v3.16b, v26.16b\n" + "ldr q26, [x10, #0xc0]\n" + ".inst 0x6e99a42d // ummla v13.4s, v1.16b, v25.16b\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x6e86a475 // ummla v21.4s, v3.16b, v6.16b\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x6e87a42a // ummla v10.4s, v1.16b, v7.16b\n" + ".inst 0x6e99a475 // ummla v21.4s, v3.16b, v25.16b\n" + "ldr q25, [x10, #0xd0]\n" + ".inst 0x6e9aa42a // ummla v10.4s, v1.16b, v26.16b\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x6e87a472 // ummla v18.4s, v3.16b, v7.16b\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x6e86a42e // ummla v14.4s, v1.16b, v6.16b\n" + ".inst 0x6e9aa472 // ummla v18.4s, v3.16b, v26.16b\n" + "ldr q26, [x10, #0xe0]\n" + ".inst 0x6e99a42e // ummla v14.4s, v1.16b, v25.16b\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x6e86a476 // ummla v22.4s, v3.16b, v6.16b\n" - "ldr q6, [x10, #0xf0]\n" + ".inst 0x6e99a476 // ummla v22.4s, v3.16b, v25.16b\n" + "ldr q25, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x6e87a42b // ummla v11.4s, v1.16b, v7.16b\n" - ".inst 0x6e87a473 // ummla v19.4s, v3.16b, v7.16b\n" + ".inst 0x6e9aa42b // ummla v11.4s, v1.16b, v26.16b\n" + ".inst 0x6e9aa473 // ummla v19.4s, v3.16b, v26.16b\n" "ldr q7, [x10, #0x0]\n" - ".inst 0x6e86a42f // ummla v15.4s, v1.16b, v6.16b\n" + ".inst 0x6e99a42f // ummla v15.4s, v1.16b, v25.16b\n" "ldr q1, [x26, #0x0]\n" - ".inst 0x6e86a477 // ummla v23.4s, v3.16b, v6.16b\n" + ".inst 0x6e99a477 // ummla v23.4s, v3.16b, v25.16b\n" "ldr q3, [x24, #0x0]\n" "ldr q6, [x10, #0x10]\n" "bge 129b\n" "130:" // Height 4: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v28.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e87a408 // ummla v8.4s, v0.16b, v7.16b\n" + ".inst 0x6e87a788 // ummla v8.4s, v28.16b, v7.16b\n" "add x26, x26, #0x10\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x6e87a450 // ummla v16.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e86a40c // ummla v12.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a454 // ummla v20.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e87a409 // ummla v9.4s, v0.16b, v7.16b\n" + "trn1 v27.2d, v3.2d, v4.2d\n" + ".inst 0x6e87a770 // ummla v16.4s, v27.16b, v7.16b\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x6e86a78c // ummla v12.4s, v28.16b, v6.16b\n" + ".inst 0x6e86a774 // ummla v20.4s, v27.16b, v6.16b\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e9aa789 // ummla v9.4s, v28.16b, v26.16b\n" "trn2 v3.2d, v3.2d, v4.2d\n" - ".inst 0x6e87a451 // ummla v17.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e86a40d // ummla v13.4s, v0.16b, v6.16b\n" + ".inst 0x6e9aa771 // ummla v17.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x6e99a78d // ummla v13.4s, v28.16b, v25.16b\n" "add x25, x25, #0x10\n" - ".inst 0x6e86a455 // ummla v21.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e87a40a // ummla v10.4s, v0.16b, v7.16b\n" + ".inst 0x6e99a775 // ummla v21.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x6e9aa78a // ummla v10.4s, v28.16b, v26.16b\n" "add x24, x24, #0x10\n" - ".inst 0x6e87a452 // ummla v18.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e86a40e // ummla v14.4s, v0.16b, v6.16b\n" + ".inst 0x6e9aa772 // ummla v18.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x6e99a78e // ummla v14.4s, v28.16b, v25.16b\n" "add x23, x23, #0x10\n" - ".inst 0x6e86a456 // ummla v22.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" - ".inst 0x6e87a40b // ummla v11.4s, v0.16b, v7.16b\n" + ".inst 0x6e99a776 // ummla v22.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x70]\n" + ".inst 0x6e9aa78b // ummla v11.4s, v28.16b, v26.16b\n" "sub x27, x27, #0x10\n" - ".inst 0x6e87a453 // ummla v19.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x80]\n" - ".inst 0x6e86a40f // ummla v15.4s, v0.16b, v6.16b\n" + ".inst 0x6e9aa773 // ummla v19.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x80]\n" + ".inst 0x6e99a78f // ummla v15.4s, v28.16b, v25.16b\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x6e86a457 // ummla v23.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x90]\n" - ".inst 0x6e87a428 // ummla v8.4s, v1.16b, v7.16b\n" + ".inst 0x6e99a777 // ummla v23.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x90]\n" + ".inst 0x6e9aa428 // ummla v8.4s, v1.16b, v26.16b\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x6e87a470 // ummla v16.4s, v3.16b, v7.16b\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x6e86a42c // ummla v12.4s, v1.16b, v6.16b\n" + ".inst 0x6e9aa470 // ummla v16.4s, v3.16b, v26.16b\n" + "ldr q26, [x10, #0xa0]\n" + ".inst 0x6e99a42c // ummla v12.4s, v1.16b, v25.16b\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x6e86a474 // ummla v20.4s, v3.16b, v6.16b\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x6e87a429 // ummla v9.4s, v1.16b, v7.16b\n" + ".inst 0x6e99a474 // ummla v20.4s, v3.16b, v25.16b\n" + "ldr q25, [x10, #0xb0]\n" + ".inst 0x6e9aa429 // ummla v9.4s, v1.16b, v26.16b\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x6e87a471 // ummla v17.4s, v3.16b, v7.16b\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x6e86a42d // ummla v13.4s, v1.16b, v6.16b\n" - ".inst 0x6e86a475 // ummla v21.4s, v3.16b, v6.16b\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x6e87a42a // ummla v10.4s, v1.16b, v7.16b\n" - ".inst 0x6e87a472 // ummla v18.4s, v3.16b, v7.16b\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x6e86a42e // ummla v14.4s, v1.16b, v6.16b\n" - ".inst 0x6e86a476 // ummla v22.4s, v3.16b, v6.16b\n" - "ldr q6, [x10, #0xf0]\n" + ".inst 0x6e9aa471 // ummla v17.4s, v3.16b, v26.16b\n" + "ldr q26, [x10, #0xc0]\n" + ".inst 0x6e99a42d // ummla v13.4s, v1.16b, v25.16b\n" + ".inst 0x6e99a475 // ummla v21.4s, v3.16b, v25.16b\n" + "ldr q25, [x10, #0xd0]\n" + ".inst 0x6e9aa42a // ummla v10.4s, v1.16b, v26.16b\n" + ".inst 0x6e9aa472 // ummla v18.4s, v3.16b, v26.16b\n" + "ldr q26, [x10, #0xe0]\n" + ".inst 0x6e99a42e // ummla v14.4s, v1.16b, v25.16b\n" + ".inst 0x6e99a476 // ummla v22.4s, v3.16b, v25.16b\n" + "ldr q25, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x6e87a42b // ummla v11.4s, v1.16b, v7.16b\n" - ".inst 0x6e87a473 // ummla v19.4s, v3.16b, v7.16b\n" - ".inst 0x6e86a42f // ummla v15.4s, v1.16b, v6.16b\n" - ".inst 0x6e86a477 // ummla v23.4s, v3.16b, v6.16b\n" + ".inst 0x6e9aa42b // ummla v11.4s, v1.16b, v26.16b\n" + ".inst 0x6e9aa473 // ummla v19.4s, v3.16b, v26.16b\n" + ".inst 0x6e99a42f // ummla v15.4s, v1.16b, v25.16b\n" + ".inst 0x6e99a477 // ummla v23.4s, v3.16b, v25.16b\n" "131:" // Height 4: Multiply loop: Main loop skip "cbz x27, 138f\n" "cmp x27, #0x8\n" "blt 133f\n" "132:" // Height 4: Multiply loop: Odd block loop - "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" + "ldr d26, [x26], #0x8\n" + "ldr d25, [x25], #0x8\n" + "trn1 v28.2d, v26.2d, v25.2d\n" "sub x27, x27, #0x8\n" - "ldr d3, [x24], #0x8\n" - "ldr d4, [x23], #0x8\n" - "trn1 v2.2d, v3.2d, v4.2d\n" + "ldr d26, [x24], #0x8\n" + "ldr d25, [x23], #0x8\n" + "trn1 v27.2d, v26.2d, v25.2d\n" "cmp x27, #0x8\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6e86a408 // ummla v8.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a450 // ummla v16.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6e87a40c // ummla v12.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a454 // ummla v20.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e86a409 // ummla v9.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a451 // ummla v17.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e87a40d // ummla v13.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a455 // ummla v21.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e86a40a // ummla v10.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a452 // ummla v18.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x60]\n" - ".inst 0x6e87a40e // ummla v14.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a456 // ummla v22.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x70]\n" + "ldr q26, [x10, #0x0]\n" + "ldr q25, [x10, #0x10]\n" + ".inst 0x6e9aa788 // ummla v8.4s, v28.16b, v26.16b\n" + ".inst 0x6e9aa770 // ummla v16.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x6e99a78c // ummla v12.4s, v28.16b, v25.16b\n" + ".inst 0x6e99a774 // ummla v20.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e9aa789 // ummla v9.4s, v28.16b, v26.16b\n" + ".inst 0x6e9aa771 // ummla v17.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x6e99a78d // ummla v13.4s, v28.16b, v25.16b\n" + ".inst 0x6e99a775 // ummla v21.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x6e9aa78a // ummla v10.4s, v28.16b, v26.16b\n" + ".inst 0x6e9aa772 // ummla v18.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x6e99a78e // ummla v14.4s, v28.16b, v25.16b\n" + ".inst 0x6e99a776 // ummla v22.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x70]\n" "add x10, x10, #0x80\n" - ".inst 0x6e86a40b // ummla v11.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a453 // ummla v19.4s, v2.16b, v6.16b\n" - ".inst 0x6e87a40f // ummla v15.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a457 // ummla v23.4s, v2.16b, v7.16b\n" + ".inst 0x6e9aa78b // ummla v11.4s, v28.16b, v26.16b\n" + ".inst 0x6e9aa773 // ummla v19.4s, v27.16b, v26.16b\n" + ".inst 0x6e99a78f // ummla v15.4s, v28.16b, v25.16b\n" + ".inst 0x6e99a777 // ummla v23.4s, v27.16b, v25.16b\n" "bge 132b\n" "133:" // Height 4: Multiply loop: Skip odd blocks "cbz x27, 138f\n" @@ -1742,33 +1741,33 @@ void a64_hybrid_u8u32_mmla_6x16 ( "ldr b3, [x24, #0x0]\n" "ldr b4, [x23, #0x0]\n" "137:" // Height 4: Multiply loop: Ragged operand read: Done - "ldr q7, [x10, #0x0]\n" - "ldr q6, [x10, #0x10]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x6e87a408 // ummla v8.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a450 // ummla v16.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e86a40c // ummla v12.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a454 // ummla v20.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e87a409 // ummla v9.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a451 // ummla v17.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e86a40d // ummla v13.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a455 // ummla v21.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e87a40a // ummla v10.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a452 // ummla v18.4s, v2.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e86a40e // ummla v14.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a456 // ummla v22.4s, v2.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" + "ldr q26, [x10, #0x0]\n" + "ldr q25, [x10, #0x10]\n" + "trn1 v28.2d, v1.2d, v2.2d\n" + "trn1 v27.2d, v3.2d, v4.2d\n" + ".inst 0x6e9aa788 // ummla v8.4s, v28.16b, v26.16b\n" + ".inst 0x6e9aa770 // ummla v16.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x20]\n" + ".inst 0x6e99a78c // ummla v12.4s, v28.16b, v25.16b\n" + ".inst 0x6e99a774 // ummla v20.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x30]\n" + ".inst 0x6e9aa789 // ummla v9.4s, v28.16b, v26.16b\n" + ".inst 0x6e9aa771 // ummla v17.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x40]\n" + ".inst 0x6e99a78d // ummla v13.4s, v28.16b, v25.16b\n" + ".inst 0x6e99a775 // ummla v21.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x50]\n" + ".inst 0x6e9aa78a // ummla v10.4s, v28.16b, v26.16b\n" + ".inst 0x6e9aa772 // ummla v18.4s, v27.16b, v26.16b\n" + "ldr q26, [x10, #0x60]\n" + ".inst 0x6e99a78e // ummla v14.4s, v28.16b, v25.16b\n" + ".inst 0x6e99a776 // ummla v22.4s, v27.16b, v25.16b\n" + "ldr q25, [x10, #0x70]\n" "add x10, x10, #0x80\n" - ".inst 0x6e87a40b // ummla v11.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a453 // ummla v19.4s, v2.16b, v7.16b\n" - ".inst 0x6e86a40f // ummla v15.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a457 // ummla v23.4s, v2.16b, v6.16b\n" + ".inst 0x6e9aa78b // ummla v11.4s, v28.16b, v26.16b\n" + ".inst 0x6e9aa773 // ummla v19.4s, v27.16b, v26.16b\n" + ".inst 0x6e99a78f // ummla v15.4s, v28.16b, v25.16b\n" + ".inst 0x6e99a777 // ummla v23.4s, v27.16b, v25.16b\n" "138:" // Height 4: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -2125,15 +2124,15 @@ void a64_hybrid_u8u32_mmla_6x16 ( "163:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 164f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 165f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -2144,10 +2143,10 @@ void a64_hybrid_u8u32_mmla_6x16 ( "b 165f\n" "164:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" "165:" // Height 5: input setup done "cmp x27, #0x10\n" "blt 168f\n" @@ -2160,174 +2159,174 @@ void a64_hybrid_u8u32_mmla_6x16 ( "ldr q7, [x10, #0x0]\n" "blt 167f\n" "166:" // Height 5: Multiply loop: Main loop head - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v6.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e87a408 // ummla v8.4s, v0.16b, v7.16b\n" + ".inst 0x6e87a4c8 // ummla v8.4s, v6.16b, v7.16b\n" "trn1 v2.2d, v3.2d, v4.2d\n" "trn2 v3.2d, v3.2d, v4.2d\n" ".inst 0x6e87a450 // ummla v16.4s, v2.16b, v7.16b\n" "sub x27, x27, #0x10\n" - "trn1 v4.2d, v5.2d, v6.2d\n" - "trn2 v5.2d, v5.2d, v6.2d\n" - "ldr q6, [x10, #0x10]\n" + "trn1 v4.2d, v5.2d, v0.2d\n" + "trn2 v5.2d, v5.2d, v0.2d\n" + "ldr q0, [x10, #0x10]\n" ".inst 0x6e87a498 // ummla v24.4s, v4.16b, v7.16b\n" "ldr q7, [x10, #0x20]\n" - ".inst 0x6e86a40c // ummla v12.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a454 // ummla v20.4s, v2.16b, v6.16b\n" + ".inst 0x6e80a4cc // ummla v12.4s, v6.16b, v0.16b\n" + ".inst 0x6e80a454 // ummla v20.4s, v2.16b, v0.16b\n" "add x26, x26, #0x10\n" - ".inst 0x6e86a49c // ummla v28.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e87a409 // ummla v9.4s, v0.16b, v7.16b\n" + ".inst 0x6e80a49c // ummla v28.4s, v4.16b, v0.16b\n" + "ldr q0, [x10, #0x30]\n" + ".inst 0x6e87a4c9 // ummla v9.4s, v6.16b, v7.16b\n" "add x25, x25, #0x10\n" ".inst 0x6e87a451 // ummla v17.4s, v2.16b, v7.16b\n" ".inst 0x6e87a499 // ummla v25.4s, v4.16b, v7.16b\n" "ldr q7, [x10, #0x40]\n" "add x24, x24, #0x10\n" - ".inst 0x6e86a40d // ummla v13.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a455 // ummla v21.4s, v2.16b, v6.16b\n" + ".inst 0x6e80a4cd // ummla v13.4s, v6.16b, v0.16b\n" + ".inst 0x6e80a455 // ummla v21.4s, v2.16b, v0.16b\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - ".inst 0x6e86a49d // ummla v29.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e87a40a // ummla v10.4s, v0.16b, v7.16b\n" + ".inst 0x6e80a49d // ummla v29.4s, v4.16b, v0.16b\n" + "ldr q0, [x10, #0x50]\n" + ".inst 0x6e87a4ca // ummla v10.4s, v6.16b, v7.16b\n" "cmp x27, #0x20\n" ".inst 0x6e87a452 // ummla v18.4s, v2.16b, v7.16b\n" ".inst 0x6e87a49a // ummla v26.4s, v4.16b, v7.16b\n" "ldr q7, [x10, #0x60]\n" "prfm pldl1keep, [x26, #0x80]\n" - ".inst 0x6e86a40e // ummla v14.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a456 // ummla v22.4s, v2.16b, v6.16b\n" + ".inst 0x6e80a4ce // ummla v14.4s, v6.16b, v0.16b\n" + ".inst 0x6e80a456 // ummla v22.4s, v2.16b, v0.16b\n" "prfm pldl1keep, [x25, #0x80]\n" "prfm pldl1keep, [x24, #0x80]\n" - ".inst 0x6e86a49e // ummla v30.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" - ".inst 0x6e87a40b // ummla v11.4s, v0.16b, v7.16b\n" + ".inst 0x6e80a49e // ummla v30.4s, v4.16b, v0.16b\n" + "ldr q0, [x10, #0x70]\n" + ".inst 0x6e87a4cb // ummla v11.4s, v6.16b, v7.16b\n" "prfm pldl1keep, [x23, #0x80]\n" ".inst 0x6e87a453 // ummla v19.4s, v2.16b, v7.16b\n" ".inst 0x6e87a49b // ummla v27.4s, v4.16b, v7.16b\n" "ldr q7, [x10, #0x80]\n" "prfm pldl1keep, [x22, #0x80]\n" - ".inst 0x6e86a40f // ummla v15.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a457 // ummla v23.4s, v2.16b, v6.16b\n" + ".inst 0x6e80a4cf // ummla v15.4s, v6.16b, v0.16b\n" + ".inst 0x6e80a457 // ummla v23.4s, v2.16b, v0.16b\n" "ldr q2, [x25, #0x0]\n" - ".inst 0x6e86a49f // ummla v31.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x90]\n" + ".inst 0x6e80a49f // ummla v31.4s, v4.16b, v0.16b\n" + "ldr q0, [x10, #0x90]\n" "ldr q4, [x23, #0x0]\n" ".inst 0x6e87a428 // ummla v8.4s, v1.16b, v7.16b\n" ".inst 0x6e87a470 // ummla v16.4s, v3.16b, v7.16b\n" ".inst 0x6e87a4b8 // ummla v24.4s, v5.16b, v7.16b\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x6e86a42c // ummla v12.4s, v1.16b, v6.16b\n" - ".inst 0x6e86a474 // ummla v20.4s, v3.16b, v6.16b\n" - ".inst 0x6e86a4bc // ummla v28.4s, v5.16b, v6.16b\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x6e87a429 // ummla v9.4s, v1.16b, v7.16b\n" - ".inst 0x6e87a471 // ummla v17.4s, v3.16b, v7.16b\n" - ".inst 0x6e87a4b9 // ummla v25.4s, v5.16b, v7.16b\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x6e86a42d // ummla v13.4s, v1.16b, v6.16b\n" - ".inst 0x6e86a475 // ummla v21.4s, v3.16b, v6.16b\n" - ".inst 0x6e86a4bd // ummla v29.4s, v5.16b, v6.16b\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x6e87a42a // ummla v10.4s, v1.16b, v7.16b\n" - ".inst 0x6e87a472 // ummla v18.4s, v3.16b, v7.16b\n" - ".inst 0x6e87a4ba // ummla v26.4s, v5.16b, v7.16b\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x6e86a42e // ummla v14.4s, v1.16b, v6.16b\n" - ".inst 0x6e86a476 // ummla v22.4s, v3.16b, v6.16b\n" - ".inst 0x6e86a4be // ummla v30.4s, v5.16b, v6.16b\n" - "ldr q6, [x10, #0xf0]\n" + "ldr q6, [x10, #0xa0]\n" + ".inst 0x6e80a42c // ummla v12.4s, v1.16b, v0.16b\n" + ".inst 0x6e80a474 // ummla v20.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a4bc // ummla v28.4s, v5.16b, v0.16b\n" + "ldr q0, [x10, #0xb0]\n" + ".inst 0x6e86a429 // ummla v9.4s, v1.16b, v6.16b\n" + ".inst 0x6e86a471 // ummla v17.4s, v3.16b, v6.16b\n" + ".inst 0x6e86a4b9 // ummla v25.4s, v5.16b, v6.16b\n" + "ldr q6, [x10, #0xc0]\n" + ".inst 0x6e80a42d // ummla v13.4s, v1.16b, v0.16b\n" + ".inst 0x6e80a475 // ummla v21.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a4bd // ummla v29.4s, v5.16b, v0.16b\n" + "ldr q0, [x10, #0xd0]\n" + ".inst 0x6e86a42a // ummla v10.4s, v1.16b, v6.16b\n" + ".inst 0x6e86a472 // ummla v18.4s, v3.16b, v6.16b\n" + ".inst 0x6e86a4ba // ummla v26.4s, v5.16b, v6.16b\n" + "ldr q6, [x10, #0xe0]\n" + ".inst 0x6e80a42e // ummla v14.4s, v1.16b, v0.16b\n" + ".inst 0x6e80a476 // ummla v22.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a4be // ummla v30.4s, v5.16b, v0.16b\n" + "ldr q0, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x6e87a42b // ummla v11.4s, v1.16b, v7.16b\n" - ".inst 0x6e87a473 // ummla v19.4s, v3.16b, v7.16b\n" - ".inst 0x6e87a4bb // ummla v27.4s, v5.16b, v7.16b\n" + ".inst 0x6e86a42b // ummla v11.4s, v1.16b, v6.16b\n" + ".inst 0x6e86a473 // ummla v19.4s, v3.16b, v6.16b\n" + ".inst 0x6e86a4bb // ummla v27.4s, v5.16b, v6.16b\n" "ldr q7, [x10, #0x0]\n" - ".inst 0x6e86a42f // ummla v15.4s, v1.16b, v6.16b\n" + ".inst 0x6e80a42f // ummla v15.4s, v1.16b, v0.16b\n" "ldr q1, [x26, #0x0]\n" - ".inst 0x6e86a477 // ummla v23.4s, v3.16b, v6.16b\n" + ".inst 0x6e80a477 // ummla v23.4s, v3.16b, v0.16b\n" "ldr q3, [x24, #0x0]\n" - ".inst 0x6e86a4bf // ummla v31.4s, v5.16b, v6.16b\n" + ".inst 0x6e80a4bf // ummla v31.4s, v5.16b, v0.16b\n" "ldr q5, [x22, #0x0]\n" "bge 166b\n" "167:" // Height 5: Multiply loop: Single iteration only - "trn1 v0.2d, v1.2d, v2.2d\n" + "trn1 v6.2d, v1.2d, v2.2d\n" "trn2 v1.2d, v1.2d, v2.2d\n" - ".inst 0x6e87a408 // ummla v8.4s, v0.16b, v7.16b\n" + ".inst 0x6e87a4c8 // ummla v8.4s, v6.16b, v7.16b\n" "trn1 v2.2d, v3.2d, v4.2d\n" "trn2 v3.2d, v3.2d, v4.2d\n" ".inst 0x6e87a450 // ummla v16.4s, v2.16b, v7.16b\n" "add x26, x26, #0x10\n" - "trn1 v4.2d, v5.2d, v6.2d\n" - "trn2 v5.2d, v5.2d, v6.2d\n" - "ldr q6, [x10, #0x10]\n" + "trn1 v4.2d, v5.2d, v0.2d\n" + "trn2 v5.2d, v5.2d, v0.2d\n" + "ldr q0, [x10, #0x10]\n" ".inst 0x6e87a498 // ummla v24.4s, v4.16b, v7.16b\n" "ldr q7, [x10, #0x20]\n" - ".inst 0x6e86a40c // ummla v12.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a454 // ummla v20.4s, v2.16b, v6.16b\n" + ".inst 0x6e80a4cc // ummla v12.4s, v6.16b, v0.16b\n" + ".inst 0x6e80a454 // ummla v20.4s, v2.16b, v0.16b\n" "add x25, x25, #0x10\n" - ".inst 0x6e86a49c // ummla v28.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e87a409 // ummla v9.4s, v0.16b, v7.16b\n" + ".inst 0x6e80a49c // ummla v28.4s, v4.16b, v0.16b\n" + "ldr q0, [x10, #0x30]\n" + ".inst 0x6e87a4c9 // ummla v9.4s, v6.16b, v7.16b\n" "add x24, x24, #0x10\n" ".inst 0x6e87a451 // ummla v17.4s, v2.16b, v7.16b\n" ".inst 0x6e87a499 // ummla v25.4s, v4.16b, v7.16b\n" "ldr q7, [x10, #0x40]\n" "add x23, x23, #0x10\n" - ".inst 0x6e86a40d // ummla v13.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a455 // ummla v21.4s, v2.16b, v6.16b\n" + ".inst 0x6e80a4cd // ummla v13.4s, v6.16b, v0.16b\n" + ".inst 0x6e80a455 // ummla v21.4s, v2.16b, v0.16b\n" "add x22, x22, #0x10\n" "sub x27, x27, #0x10\n" - ".inst 0x6e86a49d // ummla v29.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e87a40a // ummla v10.4s, v0.16b, v7.16b\n" + ".inst 0x6e80a49d // ummla v29.4s, v4.16b, v0.16b\n" + "ldr q0, [x10, #0x50]\n" + ".inst 0x6e87a4ca // ummla v10.4s, v6.16b, v7.16b\n" "prfm pldl1keep, [x26, #0x80]\n" ".inst 0x6e87a452 // ummla v18.4s, v2.16b, v7.16b\n" ".inst 0x6e87a49a // ummla v26.4s, v4.16b, v7.16b\n" "ldr q7, [x10, #0x60]\n" "prfm pldl1keep, [x25, #0x80]\n" - ".inst 0x6e86a40e // ummla v14.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a456 // ummla v22.4s, v2.16b, v6.16b\n" + ".inst 0x6e80a4ce // ummla v14.4s, v6.16b, v0.16b\n" + ".inst 0x6e80a456 // ummla v22.4s, v2.16b, v0.16b\n" "prfm pldl1keep, [x24, #0x80]\n" "prfm pldl1keep, [x23, #0x80]\n" - ".inst 0x6e86a49e // ummla v30.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x70]\n" - ".inst 0x6e87a40b // ummla v11.4s, v0.16b, v7.16b\n" + ".inst 0x6e80a49e // ummla v30.4s, v4.16b, v0.16b\n" + "ldr q0, [x10, #0x70]\n" + ".inst 0x6e87a4cb // ummla v11.4s, v6.16b, v7.16b\n" "prfm pldl1keep, [x22, #0x80]\n" ".inst 0x6e87a453 // ummla v19.4s, v2.16b, v7.16b\n" ".inst 0x6e87a49b // ummla v27.4s, v4.16b, v7.16b\n" "ldr q7, [x10, #0x80]\n" - ".inst 0x6e86a40f // ummla v15.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a457 // ummla v23.4s, v2.16b, v6.16b\n" - ".inst 0x6e86a49f // ummla v31.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x90]\n" + ".inst 0x6e80a4cf // ummla v15.4s, v6.16b, v0.16b\n" + ".inst 0x6e80a457 // ummla v23.4s, v2.16b, v0.16b\n" + ".inst 0x6e80a49f // ummla v31.4s, v4.16b, v0.16b\n" + "ldr q2, [x10, #0x90]\n" ".inst 0x6e87a428 // ummla v8.4s, v1.16b, v7.16b\n" ".inst 0x6e87a470 // ummla v16.4s, v3.16b, v7.16b\n" ".inst 0x6e87a4b8 // ummla v24.4s, v5.16b, v7.16b\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x6e86a42c // ummla v12.4s, v1.16b, v6.16b\n" - ".inst 0x6e86a474 // ummla v20.4s, v3.16b, v6.16b\n" - ".inst 0x6e86a4bc // ummla v28.4s, v5.16b, v6.16b\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x6e87a429 // ummla v9.4s, v1.16b, v7.16b\n" - ".inst 0x6e87a471 // ummla v17.4s, v3.16b, v7.16b\n" - ".inst 0x6e87a4b9 // ummla v25.4s, v5.16b, v7.16b\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x6e86a42d // ummla v13.4s, v1.16b, v6.16b\n" - ".inst 0x6e86a475 // ummla v21.4s, v3.16b, v6.16b\n" - ".inst 0x6e86a4bd // ummla v29.4s, v5.16b, v6.16b\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x6e87a42a // ummla v10.4s, v1.16b, v7.16b\n" - ".inst 0x6e87a472 // ummla v18.4s, v3.16b, v7.16b\n" - ".inst 0x6e87a4ba // ummla v26.4s, v5.16b, v7.16b\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x6e86a42e // ummla v14.4s, v1.16b, v6.16b\n" - ".inst 0x6e86a476 // ummla v22.4s, v3.16b, v6.16b\n" - ".inst 0x6e86a4be // ummla v30.4s, v5.16b, v6.16b\n" + "ldr q0, [x10, #0xa0]\n" + ".inst 0x6e82a42c // ummla v12.4s, v1.16b, v2.16b\n" + ".inst 0x6e82a474 // ummla v20.4s, v3.16b, v2.16b\n" + ".inst 0x6e82a4bc // ummla v28.4s, v5.16b, v2.16b\n" + "ldr q2, [x10, #0xb0]\n" + ".inst 0x6e80a429 // ummla v9.4s, v1.16b, v0.16b\n" + ".inst 0x6e80a471 // ummla v17.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a4b9 // ummla v25.4s, v5.16b, v0.16b\n" + "ldr q0, [x10, #0xc0]\n" + ".inst 0x6e82a42d // ummla v13.4s, v1.16b, v2.16b\n" + ".inst 0x6e82a475 // ummla v21.4s, v3.16b, v2.16b\n" + ".inst 0x6e82a4bd // ummla v29.4s, v5.16b, v2.16b\n" + "ldr q2, [x10, #0xd0]\n" + ".inst 0x6e80a42a // ummla v10.4s, v1.16b, v0.16b\n" + ".inst 0x6e80a472 // ummla v18.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a4ba // ummla v26.4s, v5.16b, v0.16b\n" + "ldr q0, [x10, #0xe0]\n" + ".inst 0x6e82a42e // ummla v14.4s, v1.16b, v2.16b\n" + ".inst 0x6e82a476 // ummla v22.4s, v3.16b, v2.16b\n" + ".inst 0x6e82a4be // ummla v30.4s, v5.16b, v2.16b\n" "ldr q6, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x6e87a42b // ummla v11.4s, v1.16b, v7.16b\n" - ".inst 0x6e87a473 // ummla v19.4s, v3.16b, v7.16b\n" - ".inst 0x6e87a4bb // ummla v27.4s, v5.16b, v7.16b\n" + ".inst 0x6e80a42b // ummla v11.4s, v1.16b, v0.16b\n" + ".inst 0x6e80a473 // ummla v19.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a4bb // ummla v27.4s, v5.16b, v0.16b\n" ".inst 0x6e86a42f // ummla v15.4s, v1.16b, v6.16b\n" ".inst 0x6e86a477 // ummla v23.4s, v3.16b, v6.16b\n" ".inst 0x6e86a4bf // ummla v31.4s, v5.16b, v6.16b\n" @@ -2337,48 +2336,48 @@ void a64_hybrid_u8u32_mmla_6x16 ( "blt 170f\n" "169:" // Height 5: Multiply loop: Odd block loop "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "ldr d3, [x24], #0x8\n" - "ldr d4, [x23], #0x8\n" - "trn1 v2.2d, v3.2d, v4.2d\n" + "ldr d0, [x25], #0x8\n" + "trn1 v4.2d, v1.2d, v0.2d\n" + "ldr d1, [x24], #0x8\n" + "ldr d0, [x23], #0x8\n" + "trn1 v3.2d, v1.2d, v0.2d\n" "sub x27, x27, #0x8\n" - "ldr d5, [x22], #0x8\n" - "ldr q6, [x10, #0x0]\n" - "trn1 v4.2d, v5.2d, v7.2d\n" - ".inst 0x6e86a408 // ummla v8.4s, v0.16b, v6.16b\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6e86a450 // ummla v16.4s, v2.16b, v6.16b\n" - ".inst 0x6e86a498 // ummla v24.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6e87a40c // ummla v12.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a454 // ummla v20.4s, v2.16b, v7.16b\n" + "ldr d0, [x22], #0x8\n" + "ldr q1, [x10, #0x0]\n" + "trn1 v2.2d, v0.2d, v2.2d\n" + ".inst 0x6e81a488 // ummla v8.4s, v4.16b, v1.16b\n" + "ldr q0, [x10, #0x10]\n" + ".inst 0x6e81a470 // ummla v16.4s, v3.16b, v1.16b\n" + ".inst 0x6e81a458 // ummla v24.4s, v2.16b, v1.16b\n" + "ldr q1, [x10, #0x20]\n" + ".inst 0x6e80a48c // ummla v12.4s, v4.16b, v0.16b\n" + ".inst 0x6e80a474 // ummla v20.4s, v3.16b, v0.16b\n" "cmp x27, #0x8\n" - ".inst 0x6e87a49c // ummla v28.4s, v4.16b, v7.16b\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e86a409 // ummla v9.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a451 // ummla v17.4s, v2.16b, v6.16b\n" - ".inst 0x6e86a499 // ummla v25.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e87a40d // ummla v13.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a455 // ummla v21.4s, v2.16b, v7.16b\n" - ".inst 0x6e87a49d // ummla v29.4s, v4.16b, v7.16b\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e86a40a // ummla v10.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a452 // ummla v18.4s, v2.16b, v6.16b\n" - ".inst 0x6e86a49a // ummla v26.4s, v4.16b, v6.16b\n" + ".inst 0x6e80a45c // ummla v28.4s, v2.16b, v0.16b\n" + "ldr q0, [x10, #0x30]\n" + ".inst 0x6e81a489 // ummla v9.4s, v4.16b, v1.16b\n" + ".inst 0x6e81a471 // ummla v17.4s, v3.16b, v1.16b\n" + ".inst 0x6e81a459 // ummla v25.4s, v2.16b, v1.16b\n" + "ldr q1, [x10, #0x40]\n" + ".inst 0x6e80a48d // ummla v13.4s, v4.16b, v0.16b\n" + ".inst 0x6e80a475 // ummla v21.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a45d // ummla v29.4s, v2.16b, v0.16b\n" + "ldr q0, [x10, #0x50]\n" + ".inst 0x6e81a48a // ummla v10.4s, v4.16b, v1.16b\n" + ".inst 0x6e81a472 // ummla v18.4s, v3.16b, v1.16b\n" + ".inst 0x6e81a45a // ummla v26.4s, v2.16b, v1.16b\n" "ldr q6, [x10, #0x60]\n" - ".inst 0x6e87a40e // ummla v14.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a456 // ummla v22.4s, v2.16b, v7.16b\n" - ".inst 0x6e87a49e // ummla v30.4s, v4.16b, v7.16b\n" - "ldr q7, [x10, #0x70]\n" - ".inst 0x6e86a40b // ummla v11.4s, v0.16b, v6.16b\n" + ".inst 0x6e80a48e // ummla v14.4s, v4.16b, v0.16b\n" + ".inst 0x6e80a476 // ummla v22.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a45e // ummla v30.4s, v2.16b, v0.16b\n" + "ldr q0, [x10, #0x70]\n" + ".inst 0x6e86a48b // ummla v11.4s, v4.16b, v6.16b\n" "add x10, x10, #0x80\n" - ".inst 0x6e86a453 // ummla v19.4s, v2.16b, v6.16b\n" - ".inst 0x6e86a49b // ummla v27.4s, v4.16b, v6.16b\n" - ".inst 0x6e87a40f // ummla v15.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a457 // ummla v23.4s, v2.16b, v7.16b\n" - ".inst 0x6e87a49f // ummla v31.4s, v4.16b, v7.16b\n" + ".inst 0x6e86a473 // ummla v19.4s, v3.16b, v6.16b\n" + ".inst 0x6e86a45b // ummla v27.4s, v2.16b, v6.16b\n" + ".inst 0x6e80a48f // ummla v15.4s, v4.16b, v0.16b\n" + ".inst 0x6e80a477 // ummla v23.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a45f // ummla v31.4s, v2.16b, v0.16b\n" "bge 169b\n" "170:" // Height 5: Multiply loop: Skip odd blocks "cbz x27, 175f\n" @@ -2430,42 +2429,42 @@ void a64_hybrid_u8u32_mmla_6x16 ( "ldr b4, [x23, #0x0]\n" "ldr b5, [x22, #0x0]\n" "174:" // Height 5: Multiply loop: Ragged operand read: Done - "ldr q7, [x10, #0x0]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - "trn1 v4.2d, v5.2d, v6.2d\n" - "ldr q6, [x10, #0x10]\n" - ".inst 0x6e87a408 // ummla v8.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a450 // ummla v16.4s, v2.16b, v7.16b\n" - ".inst 0x6e87a498 // ummla v24.4s, v4.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e86a40c // ummla v12.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a454 // ummla v20.4s, v2.16b, v6.16b\n" - ".inst 0x6e86a49c // ummla v28.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e87a409 // ummla v9.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a451 // ummla v17.4s, v2.16b, v7.16b\n" - ".inst 0x6e87a499 // ummla v25.4s, v4.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e86a40d // ummla v13.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a455 // ummla v21.4s, v2.16b, v6.16b\n" - ".inst 0x6e86a49d // ummla v29.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e87a40a // ummla v10.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a452 // ummla v18.4s, v2.16b, v7.16b\n" - ".inst 0x6e87a49a // ummla v26.4s, v4.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e86a40e // ummla v14.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a456 // ummla v22.4s, v2.16b, v6.16b\n" - ".inst 0x6e86a49e // ummla v30.4s, v4.16b, v6.16b\n" + "ldr q6, [x10, #0x0]\n" + "trn1 v7.2d, v1.2d, v2.2d\n" + "trn1 v3.2d, v3.2d, v4.2d\n" + "trn1 v2.2d, v5.2d, v0.2d\n" + "ldr q1, [x10, #0x10]\n" + ".inst 0x6e86a4e8 // ummla v8.4s, v7.16b, v6.16b\n" + ".inst 0x6e86a470 // ummla v16.4s, v3.16b, v6.16b\n" + ".inst 0x6e86a458 // ummla v24.4s, v2.16b, v6.16b\n" + "ldr q0, [x10, #0x20]\n" + ".inst 0x6e81a4ec // ummla v12.4s, v7.16b, v1.16b\n" + ".inst 0x6e81a474 // ummla v20.4s, v3.16b, v1.16b\n" + ".inst 0x6e81a45c // ummla v28.4s, v2.16b, v1.16b\n" + "ldr q1, [x10, #0x30]\n" + ".inst 0x6e80a4e9 // ummla v9.4s, v7.16b, v0.16b\n" + ".inst 0x6e80a471 // ummla v17.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a459 // ummla v25.4s, v2.16b, v0.16b\n" + "ldr q0, [x10, #0x40]\n" + ".inst 0x6e81a4ed // ummla v13.4s, v7.16b, v1.16b\n" + ".inst 0x6e81a475 // ummla v21.4s, v3.16b, v1.16b\n" + ".inst 0x6e81a45d // ummla v29.4s, v2.16b, v1.16b\n" + "ldr q1, [x10, #0x50]\n" + ".inst 0x6e80a4ea // ummla v10.4s, v7.16b, v0.16b\n" + ".inst 0x6e80a472 // ummla v18.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a45a // ummla v26.4s, v2.16b, v0.16b\n" + "ldr q0, [x10, #0x60]\n" + ".inst 0x6e81a4ee // ummla v14.4s, v7.16b, v1.16b\n" + ".inst 0x6e81a476 // ummla v22.4s, v3.16b, v1.16b\n" + ".inst 0x6e81a45e // ummla v30.4s, v2.16b, v1.16b\n" "ldr q6, [x10, #0x70]\n" "add x10, x10, #0x80\n" - ".inst 0x6e87a40b // ummla v11.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a453 // ummla v19.4s, v2.16b, v7.16b\n" - ".inst 0x6e87a49b // ummla v27.4s, v4.16b, v7.16b\n" - ".inst 0x6e86a40f // ummla v15.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a457 // ummla v23.4s, v2.16b, v6.16b\n" - ".inst 0x6e86a49f // ummla v31.4s, v4.16b, v6.16b\n" + ".inst 0x6e80a4eb // ummla v11.4s, v7.16b, v0.16b\n" + ".inst 0x6e80a473 // ummla v19.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a45b // ummla v27.4s, v2.16b, v0.16b\n" + ".inst 0x6e86a4ef // ummla v15.4s, v7.16b, v6.16b\n" + ".inst 0x6e86a477 // ummla v23.4s, v3.16b, v6.16b\n" + ".inst 0x6e86a45f // ummla v31.4s, v2.16b, v6.16b\n" "175:" // Height 5: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -2872,16 +2871,16 @@ void a64_hybrid_u8u32_mmla_6x16 ( "200:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 201f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 202f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -2893,11 +2892,11 @@ void a64_hybrid_u8u32_mmla_6x16 ( "b 202f\n" "201:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" + "add x21, x22, x21\n" "202:" // Height 6: input setup done "cmp x27, #0x10\n" "blt 205f\n" @@ -2964,42 +2963,42 @@ void a64_hybrid_u8u32_mmla_6x16 ( "ldr q2, [x25, #0x0]\n" "prfm pldl1keep, [x21, #0x80]\n" ".inst 0x6e86a49f // ummla v31.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x90]\n" + "ldr q0, [x10, #0x90]\n" "ldr q4, [x23, #0x0]\n" ".inst 0x6e87a428 // ummla v8.4s, v1.16b, v7.16b\n" ".inst 0x6e87a470 // ummla v16.4s, v3.16b, v7.16b\n" ".inst 0x6e87a4b8 // ummla v24.4s, v5.16b, v7.16b\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x6e86a42c // ummla v12.4s, v1.16b, v6.16b\n" - ".inst 0x6e86a474 // ummla v20.4s, v3.16b, v6.16b\n" - ".inst 0x6e86a4bc // ummla v28.4s, v5.16b, v6.16b\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x6e87a429 // ummla v9.4s, v1.16b, v7.16b\n" - ".inst 0x6e87a471 // ummla v17.4s, v3.16b, v7.16b\n" - ".inst 0x6e87a4b9 // ummla v25.4s, v5.16b, v7.16b\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x6e86a42d // ummla v13.4s, v1.16b, v6.16b\n" - ".inst 0x6e86a475 // ummla v21.4s, v3.16b, v6.16b\n" - ".inst 0x6e86a4bd // ummla v29.4s, v5.16b, v6.16b\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x6e87a42a // ummla v10.4s, v1.16b, v7.16b\n" - ".inst 0x6e87a472 // ummla v18.4s, v3.16b, v7.16b\n" - ".inst 0x6e87a4ba // ummla v26.4s, v5.16b, v7.16b\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x6e86a42e // ummla v14.4s, v1.16b, v6.16b\n" - ".inst 0x6e86a476 // ummla v22.4s, v3.16b, v6.16b\n" - ".inst 0x6e86a4be // ummla v30.4s, v5.16b, v6.16b\n" - "ldr q6, [x10, #0xf0]\n" + "ldr q6, [x10, #0xa0]\n" + ".inst 0x6e80a42c // ummla v12.4s, v1.16b, v0.16b\n" + ".inst 0x6e80a474 // ummla v20.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a4bc // ummla v28.4s, v5.16b, v0.16b\n" + "ldr q0, [x10, #0xb0]\n" + ".inst 0x6e86a429 // ummla v9.4s, v1.16b, v6.16b\n" + ".inst 0x6e86a471 // ummla v17.4s, v3.16b, v6.16b\n" + ".inst 0x6e86a4b9 // ummla v25.4s, v5.16b, v6.16b\n" + "ldr q6, [x10, #0xc0]\n" + ".inst 0x6e80a42d // ummla v13.4s, v1.16b, v0.16b\n" + ".inst 0x6e80a475 // ummla v21.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a4bd // ummla v29.4s, v5.16b, v0.16b\n" + "ldr q0, [x10, #0xd0]\n" + ".inst 0x6e86a42a // ummla v10.4s, v1.16b, v6.16b\n" + ".inst 0x6e86a472 // ummla v18.4s, v3.16b, v6.16b\n" + ".inst 0x6e86a4ba // ummla v26.4s, v5.16b, v6.16b\n" + "ldr q6, [x10, #0xe0]\n" + ".inst 0x6e80a42e // ummla v14.4s, v1.16b, v0.16b\n" + ".inst 0x6e80a476 // ummla v22.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a4be // ummla v30.4s, v5.16b, v0.16b\n" + "ldr q0, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x6e87a42b // ummla v11.4s, v1.16b, v7.16b\n" - ".inst 0x6e87a473 // ummla v19.4s, v3.16b, v7.16b\n" - ".inst 0x6e87a4bb // ummla v27.4s, v5.16b, v7.16b\n" + ".inst 0x6e86a42b // ummla v11.4s, v1.16b, v6.16b\n" + ".inst 0x6e86a473 // ummla v19.4s, v3.16b, v6.16b\n" + ".inst 0x6e86a4bb // ummla v27.4s, v5.16b, v6.16b\n" "ldr q7, [x10, #0x0]\n" - ".inst 0x6e86a42f // ummla v15.4s, v1.16b, v6.16b\n" + ".inst 0x6e80a42f // ummla v15.4s, v1.16b, v0.16b\n" "ldr q1, [x26, #0x0]\n" - ".inst 0x6e86a477 // ummla v23.4s, v3.16b, v6.16b\n" + ".inst 0x6e80a477 // ummla v23.4s, v3.16b, v0.16b\n" "ldr q3, [x24, #0x0]\n" - ".inst 0x6e86a4bf // ummla v31.4s, v5.16b, v6.16b\n" + ".inst 0x6e80a4bf // ummla v31.4s, v5.16b, v0.16b\n" "ldr q5, [x22, #0x0]\n" "ldr q6, [x21, #0x0]\n" "bge 203b\n" @@ -3055,35 +3054,35 @@ void a64_hybrid_u8u32_mmla_6x16 ( ".inst 0x6e86a40f // ummla v15.4s, v0.16b, v6.16b\n" ".inst 0x6e86a457 // ummla v23.4s, v2.16b, v6.16b\n" ".inst 0x6e86a49f // ummla v31.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x90]\n" + "ldr q2, [x10, #0x90]\n" ".inst 0x6e87a428 // ummla v8.4s, v1.16b, v7.16b\n" ".inst 0x6e87a470 // ummla v16.4s, v3.16b, v7.16b\n" ".inst 0x6e87a4b8 // ummla v24.4s, v5.16b, v7.16b\n" - "ldr q7, [x10, #0xa0]\n" - ".inst 0x6e86a42c // ummla v12.4s, v1.16b, v6.16b\n" - ".inst 0x6e86a474 // ummla v20.4s, v3.16b, v6.16b\n" - ".inst 0x6e86a4bc // ummla v28.4s, v5.16b, v6.16b\n" - "ldr q6, [x10, #0xb0]\n" - ".inst 0x6e87a429 // ummla v9.4s, v1.16b, v7.16b\n" - ".inst 0x6e87a471 // ummla v17.4s, v3.16b, v7.16b\n" - ".inst 0x6e87a4b9 // ummla v25.4s, v5.16b, v7.16b\n" - "ldr q7, [x10, #0xc0]\n" - ".inst 0x6e86a42d // ummla v13.4s, v1.16b, v6.16b\n" - ".inst 0x6e86a475 // ummla v21.4s, v3.16b, v6.16b\n" - ".inst 0x6e86a4bd // ummla v29.4s, v5.16b, v6.16b\n" - "ldr q6, [x10, #0xd0]\n" - ".inst 0x6e87a42a // ummla v10.4s, v1.16b, v7.16b\n" - ".inst 0x6e87a472 // ummla v18.4s, v3.16b, v7.16b\n" - ".inst 0x6e87a4ba // ummla v26.4s, v5.16b, v7.16b\n" - "ldr q7, [x10, #0xe0]\n" - ".inst 0x6e86a42e // ummla v14.4s, v1.16b, v6.16b\n" - ".inst 0x6e86a476 // ummla v22.4s, v3.16b, v6.16b\n" - ".inst 0x6e86a4be // ummla v30.4s, v5.16b, v6.16b\n" + "ldr q0, [x10, #0xa0]\n" + ".inst 0x6e82a42c // ummla v12.4s, v1.16b, v2.16b\n" + ".inst 0x6e82a474 // ummla v20.4s, v3.16b, v2.16b\n" + ".inst 0x6e82a4bc // ummla v28.4s, v5.16b, v2.16b\n" + "ldr q2, [x10, #0xb0]\n" + ".inst 0x6e80a429 // ummla v9.4s, v1.16b, v0.16b\n" + ".inst 0x6e80a471 // ummla v17.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a4b9 // ummla v25.4s, v5.16b, v0.16b\n" + "ldr q0, [x10, #0xc0]\n" + ".inst 0x6e82a42d // ummla v13.4s, v1.16b, v2.16b\n" + ".inst 0x6e82a475 // ummla v21.4s, v3.16b, v2.16b\n" + ".inst 0x6e82a4bd // ummla v29.4s, v5.16b, v2.16b\n" + "ldr q2, [x10, #0xd0]\n" + ".inst 0x6e80a42a // ummla v10.4s, v1.16b, v0.16b\n" + ".inst 0x6e80a472 // ummla v18.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a4ba // ummla v26.4s, v5.16b, v0.16b\n" + "ldr q0, [x10, #0xe0]\n" + ".inst 0x6e82a42e // ummla v14.4s, v1.16b, v2.16b\n" + ".inst 0x6e82a476 // ummla v22.4s, v3.16b, v2.16b\n" + ".inst 0x6e82a4be // ummla v30.4s, v5.16b, v2.16b\n" "ldr q6, [x10, #0xf0]\n" "add x10, x10, #0x100\n" - ".inst 0x6e87a42b // ummla v11.4s, v1.16b, v7.16b\n" - ".inst 0x6e87a473 // ummla v19.4s, v3.16b, v7.16b\n" - ".inst 0x6e87a4bb // ummla v27.4s, v5.16b, v7.16b\n" + ".inst 0x6e80a42b // ummla v11.4s, v1.16b, v0.16b\n" + ".inst 0x6e80a473 // ummla v19.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a4bb // ummla v27.4s, v5.16b, v0.16b\n" ".inst 0x6e86a42f // ummla v15.4s, v1.16b, v6.16b\n" ".inst 0x6e86a477 // ummla v23.4s, v3.16b, v6.16b\n" ".inst 0x6e86a4bf // ummla v31.4s, v5.16b, v6.16b\n" @@ -3093,49 +3092,49 @@ void a64_hybrid_u8u32_mmla_6x16 ( "blt 207f\n" "206:" // Height 6: Multiply loop: Odd block loop "ldr d1, [x26], #0x8\n" - "ldr d2, [x25], #0x8\n" - "trn1 v0.2d, v1.2d, v2.2d\n" + "ldr d0, [x25], #0x8\n" + "trn1 v4.2d, v1.2d, v0.2d\n" "sub x27, x27, #0x8\n" - "ldr d3, [x24], #0x8\n" - "ldr d4, [x23], #0x8\n" - "trn1 v2.2d, v3.2d, v4.2d\n" + "ldr d1, [x24], #0x8\n" + "ldr d0, [x23], #0x8\n" + "trn1 v3.2d, v1.2d, v0.2d\n" "cmp x27, #0x8\n" - "ldr d5, [x22], #0x8\n" - "ldr d7, [x21], #0x8\n" - "trn1 v4.2d, v5.2d, v7.2d\n" - "ldr q6, [x10, #0x0]\n" - "ldr q7, [x10, #0x10]\n" - ".inst 0x6e86a408 // ummla v8.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a450 // ummla v16.4s, v2.16b, v6.16b\n" - ".inst 0x6e86a498 // ummla v24.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x20]\n" - ".inst 0x6e87a40c // ummla v12.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a454 // ummla v20.4s, v2.16b, v7.16b\n" - ".inst 0x6e87a49c // ummla v28.4s, v4.16b, v7.16b\n" - "ldr q7, [x10, #0x30]\n" - ".inst 0x6e86a409 // ummla v9.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a451 // ummla v17.4s, v2.16b, v6.16b\n" - ".inst 0x6e86a499 // ummla v25.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x40]\n" - ".inst 0x6e87a40d // ummla v13.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a455 // ummla v21.4s, v2.16b, v7.16b\n" - ".inst 0x6e87a49d // ummla v29.4s, v4.16b, v7.16b\n" - "ldr q7, [x10, #0x50]\n" - ".inst 0x6e86a40a // ummla v10.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a452 // ummla v18.4s, v2.16b, v6.16b\n" - ".inst 0x6e86a49a // ummla v26.4s, v4.16b, v6.16b\n" + "ldr d1, [x22], #0x8\n" + "ldr d0, [x21], #0x8\n" + "trn1 v2.2d, v1.2d, v0.2d\n" + "ldr q1, [x10, #0x0]\n" + "ldr q0, [x10, #0x10]\n" + ".inst 0x6e81a488 // ummla v8.4s, v4.16b, v1.16b\n" + ".inst 0x6e81a470 // ummla v16.4s, v3.16b, v1.16b\n" + ".inst 0x6e81a458 // ummla v24.4s, v2.16b, v1.16b\n" + "ldr q1, [x10, #0x20]\n" + ".inst 0x6e80a48c // ummla v12.4s, v4.16b, v0.16b\n" + ".inst 0x6e80a474 // ummla v20.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a45c // ummla v28.4s, v2.16b, v0.16b\n" + "ldr q0, [x10, #0x30]\n" + ".inst 0x6e81a489 // ummla v9.4s, v4.16b, v1.16b\n" + ".inst 0x6e81a471 // ummla v17.4s, v3.16b, v1.16b\n" + ".inst 0x6e81a459 // ummla v25.4s, v2.16b, v1.16b\n" + "ldr q1, [x10, #0x40]\n" + ".inst 0x6e80a48d // ummla v13.4s, v4.16b, v0.16b\n" + ".inst 0x6e80a475 // ummla v21.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a45d // ummla v29.4s, v2.16b, v0.16b\n" + "ldr q0, [x10, #0x50]\n" + ".inst 0x6e81a48a // ummla v10.4s, v4.16b, v1.16b\n" + ".inst 0x6e81a472 // ummla v18.4s, v3.16b, v1.16b\n" + ".inst 0x6e81a45a // ummla v26.4s, v2.16b, v1.16b\n" "ldr q6, [x10, #0x60]\n" - ".inst 0x6e87a40e // ummla v14.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a456 // ummla v22.4s, v2.16b, v7.16b\n" - ".inst 0x6e87a49e // ummla v30.4s, v4.16b, v7.16b\n" - "ldr q7, [x10, #0x70]\n" + ".inst 0x6e80a48e // ummla v14.4s, v4.16b, v0.16b\n" + ".inst 0x6e80a476 // ummla v22.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a45e // ummla v30.4s, v2.16b, v0.16b\n" + "ldr q0, [x10, #0x70]\n" "add x10, x10, #0x80\n" - ".inst 0x6e86a40b // ummla v11.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a453 // ummla v19.4s, v2.16b, v6.16b\n" - ".inst 0x6e86a49b // ummla v27.4s, v4.16b, v6.16b\n" - ".inst 0x6e87a40f // ummla v15.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a457 // ummla v23.4s, v2.16b, v7.16b\n" - ".inst 0x6e87a49f // ummla v31.4s, v4.16b, v7.16b\n" + ".inst 0x6e86a48b // ummla v11.4s, v4.16b, v6.16b\n" + ".inst 0x6e86a473 // ummla v19.4s, v3.16b, v6.16b\n" + ".inst 0x6e86a45b // ummla v27.4s, v2.16b, v6.16b\n" + ".inst 0x6e80a48f // ummla v15.4s, v4.16b, v0.16b\n" + ".inst 0x6e80a477 // ummla v23.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a45f // ummla v31.4s, v2.16b, v0.16b\n" "bge 206b\n" "207:" // Height 6: Multiply loop: Skip odd blocks "cbz x27, 212f\n" @@ -3194,42 +3193,42 @@ void a64_hybrid_u8u32_mmla_6x16 ( "ldr b5, [x22, #0x0]\n" "ldr b6, [x21, #0x0]\n" "211:" // Height 6: Multiply loop: Ragged operand read: Done - "ldr q7, [x10, #0x0]\n" - "trn1 v0.2d, v1.2d, v2.2d\n" - "trn1 v2.2d, v3.2d, v4.2d\n" - ".inst 0x6e87a408 // ummla v8.4s, v0.16b, v7.16b\n" - "trn1 v4.2d, v5.2d, v6.2d\n" - "ldr q6, [x10, #0x10]\n" - ".inst 0x6e87a450 // ummla v16.4s, v2.16b, v7.16b\n" - ".inst 0x6e87a498 // ummla v24.4s, v4.16b, v7.16b\n" - "ldr q7, [x10, #0x20]\n" - ".inst 0x6e86a40c // ummla v12.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a454 // ummla v20.4s, v2.16b, v6.16b\n" - ".inst 0x6e86a49c // ummla v28.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x30]\n" - ".inst 0x6e87a409 // ummla v9.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a451 // ummla v17.4s, v2.16b, v7.16b\n" - ".inst 0x6e87a499 // ummla v25.4s, v4.16b, v7.16b\n" - "ldr q7, [x10, #0x40]\n" - ".inst 0x6e86a40d // ummla v13.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a455 // ummla v21.4s, v2.16b, v6.16b\n" - ".inst 0x6e86a49d // ummla v29.4s, v4.16b, v6.16b\n" - "ldr q6, [x10, #0x50]\n" - ".inst 0x6e87a40a // ummla v10.4s, v0.16b, v7.16b\n" - ".inst 0x6e87a452 // ummla v18.4s, v2.16b, v7.16b\n" - ".inst 0x6e87a49a // ummla v26.4s, v4.16b, v7.16b\n" - "ldr q7, [x10, #0x60]\n" - ".inst 0x6e86a40e // ummla v14.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a456 // ummla v22.4s, v2.16b, v6.16b\n" - ".inst 0x6e86a49e // ummla v30.4s, v4.16b, v6.16b\n" + "ldr q0, [x10, #0x0]\n" + "trn1 v7.2d, v1.2d, v2.2d\n" + "trn1 v3.2d, v3.2d, v4.2d\n" + ".inst 0x6e80a4e8 // ummla v8.4s, v7.16b, v0.16b\n" + "trn1 v2.2d, v5.2d, v6.2d\n" + "ldr q1, [x10, #0x10]\n" + ".inst 0x6e80a470 // ummla v16.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a458 // ummla v24.4s, v2.16b, v0.16b\n" + "ldr q0, [x10, #0x20]\n" + ".inst 0x6e81a4ec // ummla v12.4s, v7.16b, v1.16b\n" + ".inst 0x6e81a474 // ummla v20.4s, v3.16b, v1.16b\n" + ".inst 0x6e81a45c // ummla v28.4s, v2.16b, v1.16b\n" + "ldr q1, [x10, #0x30]\n" + ".inst 0x6e80a4e9 // ummla v9.4s, v7.16b, v0.16b\n" + ".inst 0x6e80a471 // ummla v17.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a459 // ummla v25.4s, v2.16b, v0.16b\n" + "ldr q0, [x10, #0x40]\n" + ".inst 0x6e81a4ed // ummla v13.4s, v7.16b, v1.16b\n" + ".inst 0x6e81a475 // ummla v21.4s, v3.16b, v1.16b\n" + ".inst 0x6e81a45d // ummla v29.4s, v2.16b, v1.16b\n" + "ldr q1, [x10, #0x50]\n" + ".inst 0x6e80a4ea // ummla v10.4s, v7.16b, v0.16b\n" + ".inst 0x6e80a472 // ummla v18.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a45a // ummla v26.4s, v2.16b, v0.16b\n" + "ldr q0, [x10, #0x60]\n" + ".inst 0x6e81a4ee // ummla v14.4s, v7.16b, v1.16b\n" + ".inst 0x6e81a476 // ummla v22.4s, v3.16b, v1.16b\n" + ".inst 0x6e81a45e // ummla v30.4s, v2.16b, v1.16b\n" "ldr q6, [x10, #0x70]\n" - ".inst 0x6e87a40b // ummla v11.4s, v0.16b, v7.16b\n" + ".inst 0x6e80a4eb // ummla v11.4s, v7.16b, v0.16b\n" "add x10, x10, #0x80\n" - ".inst 0x6e87a453 // ummla v19.4s, v2.16b, v7.16b\n" - ".inst 0x6e87a49b // ummla v27.4s, v4.16b, v7.16b\n" - ".inst 0x6e86a40f // ummla v15.4s, v0.16b, v6.16b\n" - ".inst 0x6e86a457 // ummla v23.4s, v2.16b, v6.16b\n" - ".inst 0x6e86a49f // ummla v31.4s, v4.16b, v6.16b\n" + ".inst 0x6e80a473 // ummla v19.4s, v3.16b, v0.16b\n" + ".inst 0x6e80a45b // ummla v27.4s, v2.16b, v0.16b\n" + ".inst 0x6e86a4ef // ummla v15.4s, v7.16b, v6.16b\n" + ".inst 0x6e86a477 // ummla v23.4s, v3.16b, v6.16b\n" + ".inst 0x6e86a45f // ummla v31.4s, v2.16b, v6.16b\n" "212:" // Height 6: Multiply loop: No odd multiplies "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -3440,7 +3439,6 @@ void a64_hybrid_u8u32_mmla_6x16 ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "224:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_8x12.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_8x12.hpp index 153a4cc1673c591be116f7bb0a59e964da729d46..25c5bf1b44648a9f097a99f5ff5ed9368edfd5d6 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_8x12.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_8x12.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef __aarch64__ + #include "../std_transforms_fixed.hpp" #include "../bfloat.hpp" #include "../performance_parameters.hpp" @@ -56,11 +56,6 @@ public: return 12; } - static unsigned int stripe_width() - { - return 4; - } - static constexpr unsigned int k_unroll() { return 2; @@ -97,5 +92,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_8x12/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_8x12/generic.cpp index b3bde74635074c0b65ab16720d83a76b107b9d7f..5684f464b6f812924c28d77f4ae516ebf7936cd1 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_8x12/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_dot_8x12/generic.cpp @@ -29,8 +29,12 @@ namespace arm_gemm { void a64_interleaved_bf16fp32_dot_8x12( - const bfloat16 *Apanel, const bfloat16 *Bpanel, - float *Cpanel, int ablocks, int bblocks, int K) { + const bfloat16 *Apanel, + const bfloat16 *Bpanel, + float *Cpanel, + int ablocks, + int bblocks, + int K) { struct KernelArgs { size_t K = {}; @@ -43,7 +47,6 @@ void a64_interleaved_bf16fp32_dot_8x12( ka.bblocks = bblocks; __asm__ __volatile__( - "1:" // Height loop "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n" "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n" @@ -88,8 +91,8 @@ void a64_interleaved_bf16fp32_dot_8x12( "movi v31.16b, #0x0\n" "blt 4f\n" "3:" // main loop head - "ldr q2, [%x[Apanel], #0x20]\n" - "ldr q3, [%x[Apanel], #0x30]\n" + "ldr q3, [%x[Apanel], #0x20]\n" + "ldr q7, [%x[Apanel], #0x30]\n" ".inst 0x4f40f088 // bfdot v8.4s, v4.8h, v0.h[0]\n" ".inst 0x4f60f08b // bfdot v11.4s, v4.8h, v0.h[1]\n" ".inst 0x4f40f88e // bfdot v14.4s, v4.8h, v0.h[2]\n" @@ -123,35 +126,35 @@ void a64_interleaved_bf16fp32_dot_8x12( ".inst 0x4f61f0d9 // bfdot v25.4s, v6.8h, v1.h[1]\n" ".inst 0x4f41f8dc // bfdot v28.4s, v6.8h, v1.h[2]\n" ".inst 0x4f61f8df // bfdot v31.4s, v6.8h, v1.h[3]\n" - "ldr q6, [x22, #0x50]\n" + "ldr q2, [x22, #0x50]\n" "ldr q1, [%x[Apanel], #0x10]\n" "add x22, x22, #0x60\n" - ".inst 0x4f42f088 // bfdot v8.4s, v4.8h, v2.h[0]\n" - ".inst 0x4f62f08b // bfdot v11.4s, v4.8h, v2.h[1]\n" - ".inst 0x4f42f88e // bfdot v14.4s, v4.8h, v2.h[2]\n" - ".inst 0x4f62f891 // bfdot v17.4s, v4.8h, v2.h[3]\n" - ".inst 0x4f43f094 // bfdot v20.4s, v4.8h, v3.h[0]\n" - ".inst 0x4f63f097 // bfdot v23.4s, v4.8h, v3.h[1]\n" - ".inst 0x4f43f89a // bfdot v26.4s, v4.8h, v3.h[2]\n" - ".inst 0x4f63f89d // bfdot v29.4s, v4.8h, v3.h[3]\n" + ".inst 0x4f43f088 // bfdot v8.4s, v4.8h, v3.h[0]\n" + ".inst 0x4f63f08b // bfdot v11.4s, v4.8h, v3.h[1]\n" + ".inst 0x4f43f88e // bfdot v14.4s, v4.8h, v3.h[2]\n" + ".inst 0x4f63f891 // bfdot v17.4s, v4.8h, v3.h[3]\n" + ".inst 0x4f47f094 // bfdot v20.4s, v4.8h, v7.h[0]\n" + ".inst 0x4f67f097 // bfdot v23.4s, v4.8h, v7.h[1]\n" + ".inst 0x4f47f89a // bfdot v26.4s, v4.8h, v7.h[2]\n" + ".inst 0x4f67f89d // bfdot v29.4s, v4.8h, v7.h[3]\n" "ldr q4, [x22, #0x0]\n" - ".inst 0x4f42f0a9 // bfdot v9.4s, v5.8h, v2.h[0]\n" - ".inst 0x4f62f0ac // bfdot v12.4s, v5.8h, v2.h[1]\n" - ".inst 0x4f42f8af // bfdot v15.4s, v5.8h, v2.h[2]\n" - ".inst 0x4f62f8b2 // bfdot v18.4s, v5.8h, v2.h[3]\n" - ".inst 0x4f43f0b5 // bfdot v21.4s, v5.8h, v3.h[0]\n" - ".inst 0x4f63f0b8 // bfdot v24.4s, v5.8h, v3.h[1]\n" - ".inst 0x4f43f8bb // bfdot v27.4s, v5.8h, v3.h[2]\n" - ".inst 0x4f63f8be // bfdot v30.4s, v5.8h, v3.h[3]\n" + ".inst 0x4f43f0a9 // bfdot v9.4s, v5.8h, v3.h[0]\n" + ".inst 0x4f63f0ac // bfdot v12.4s, v5.8h, v3.h[1]\n" + ".inst 0x4f43f8af // bfdot v15.4s, v5.8h, v3.h[2]\n" + ".inst 0x4f63f8b2 // bfdot v18.4s, v5.8h, v3.h[3]\n" + ".inst 0x4f47f0b5 // bfdot v21.4s, v5.8h, v7.h[0]\n" + ".inst 0x4f67f0b8 // bfdot v24.4s, v5.8h, v7.h[1]\n" + ".inst 0x4f47f8bb // bfdot v27.4s, v5.8h, v7.h[2]\n" + ".inst 0x4f67f8be // bfdot v30.4s, v5.8h, v7.h[3]\n" "ldr q5, [x22, #0x10]\n" - ".inst 0x4f42f0ca // bfdot v10.4s, v6.8h, v2.h[0]\n" - ".inst 0x4f62f0cd // bfdot v13.4s, v6.8h, v2.h[1]\n" - ".inst 0x4f42f8d0 // bfdot v16.4s, v6.8h, v2.h[2]\n" - ".inst 0x4f62f8d3 // bfdot v19.4s, v6.8h, v2.h[3]\n" - ".inst 0x4f43f0d6 // bfdot v22.4s, v6.8h, v3.h[0]\n" - ".inst 0x4f63f0d9 // bfdot v25.4s, v6.8h, v3.h[1]\n" - ".inst 0x4f43f8dc // bfdot v28.4s, v6.8h, v3.h[2]\n" - ".inst 0x4f63f8df // bfdot v31.4s, v6.8h, v3.h[3]\n" + ".inst 0x4f43f04a // bfdot v10.4s, v2.8h, v3.h[0]\n" + ".inst 0x4f63f04d // bfdot v13.4s, v2.8h, v3.h[1]\n" + ".inst 0x4f43f850 // bfdot v16.4s, v2.8h, v3.h[2]\n" + ".inst 0x4f63f853 // bfdot v19.4s, v2.8h, v3.h[3]\n" + ".inst 0x4f47f056 // bfdot v22.4s, v2.8h, v7.h[0]\n" + ".inst 0x4f67f059 // bfdot v25.4s, v2.8h, v7.h[1]\n" + ".inst 0x4f47f85c // bfdot v28.4s, v2.8h, v7.h[2]\n" + ".inst 0x4f67f85f // bfdot v31.4s, v2.8h, v7.h[3]\n" "ldr q6, [x22, #0x20]\n" "bge 3b\n" "4:" // main loop skip @@ -182,37 +185,37 @@ void a64_interleaved_bf16fp32_dot_8x12( ".inst 0x4f41f8dc // bfdot v28.4s, v6.8h, v1.h[2]\n" ".inst 0x4f61f8df // bfdot v31.4s, v6.8h, v1.h[3]\n" "cbz x20, 5f\n" - "ldr q0, [%x[Apanel], #0x0]\n" - "ldr q1, [%x[Apanel], #0x10]\n" + "ldr q4, [%x[Apanel], #0x0]\n" + "ldr q3, [%x[Apanel], #0x10]\n" "add %x[Apanel], %x[Apanel], #0x20\n" - "ldr q7, [x22, #0x0]\n" - "ldr q4, [x22, #0x10]\n" - ".inst 0x4f40f0e8 // bfdot v8.4s, v7.8h, v0.h[0]\n" - "ldr q5, [x22, #0x20]\n" - ".inst 0x4f60f0eb // bfdot v11.4s, v7.8h, v0.h[1]\n" - ".inst 0x4f40f8ee // bfdot v14.4s, v7.8h, v0.h[2]\n" - ".inst 0x4f60f8f1 // bfdot v17.4s, v7.8h, v0.h[3]\n" - ".inst 0x4f41f0f4 // bfdot v20.4s, v7.8h, v1.h[0]\n" + "ldr q2, [x22, #0x0]\n" + "ldr q1, [x22, #0x10]\n" + ".inst 0x4f44f048 // bfdot v8.4s, v2.8h, v4.h[0]\n" + "ldr q0, [x22, #0x20]\n" + ".inst 0x4f64f04b // bfdot v11.4s, v2.8h, v4.h[1]\n" + ".inst 0x4f44f84e // bfdot v14.4s, v2.8h, v4.h[2]\n" + ".inst 0x4f64f851 // bfdot v17.4s, v2.8h, v4.h[3]\n" + ".inst 0x4f43f054 // bfdot v20.4s, v2.8h, v3.h[0]\n" "add x22, x22, #0x30\n" - ".inst 0x4f61f0f7 // bfdot v23.4s, v7.8h, v1.h[1]\n" - ".inst 0x4f41f8fa // bfdot v26.4s, v7.8h, v1.h[2]\n" - ".inst 0x4f61f8fd // bfdot v29.4s, v7.8h, v1.h[3]\n" - ".inst 0x4f40f089 // bfdot v9.4s, v4.8h, v0.h[0]\n" - ".inst 0x4f60f08c // bfdot v12.4s, v4.8h, v0.h[1]\n" - ".inst 0x4f40f88f // bfdot v15.4s, v4.8h, v0.h[2]\n" - ".inst 0x4f60f892 // bfdot v18.4s, v4.8h, v0.h[3]\n" - ".inst 0x4f41f095 // bfdot v21.4s, v4.8h, v1.h[0]\n" - ".inst 0x4f61f098 // bfdot v24.4s, v4.8h, v1.h[1]\n" - ".inst 0x4f41f89b // bfdot v27.4s, v4.8h, v1.h[2]\n" - ".inst 0x4f61f89e // bfdot v30.4s, v4.8h, v1.h[3]\n" - ".inst 0x4f40f0aa // bfdot v10.4s, v5.8h, v0.h[0]\n" - ".inst 0x4f60f0ad // bfdot v13.4s, v5.8h, v0.h[1]\n" - ".inst 0x4f40f8b0 // bfdot v16.4s, v5.8h, v0.h[2]\n" - ".inst 0x4f60f8b3 // bfdot v19.4s, v5.8h, v0.h[3]\n" - ".inst 0x4f41f0b6 // bfdot v22.4s, v5.8h, v1.h[0]\n" - ".inst 0x4f61f0b9 // bfdot v25.4s, v5.8h, v1.h[1]\n" - ".inst 0x4f41f8bc // bfdot v28.4s, v5.8h, v1.h[2]\n" - ".inst 0x4f61f8bf // bfdot v31.4s, v5.8h, v1.h[3]\n" + ".inst 0x4f63f057 // bfdot v23.4s, v2.8h, v3.h[1]\n" + ".inst 0x4f43f85a // bfdot v26.4s, v2.8h, v3.h[2]\n" + ".inst 0x4f63f85d // bfdot v29.4s, v2.8h, v3.h[3]\n" + ".inst 0x4f44f029 // bfdot v9.4s, v1.8h, v4.h[0]\n" + ".inst 0x4f64f02c // bfdot v12.4s, v1.8h, v4.h[1]\n" + ".inst 0x4f44f82f // bfdot v15.4s, v1.8h, v4.h[2]\n" + ".inst 0x4f64f832 // bfdot v18.4s, v1.8h, v4.h[3]\n" + ".inst 0x4f43f035 // bfdot v21.4s, v1.8h, v3.h[0]\n" + ".inst 0x4f63f038 // bfdot v24.4s, v1.8h, v3.h[1]\n" + ".inst 0x4f43f83b // bfdot v27.4s, v1.8h, v3.h[2]\n" + ".inst 0x4f63f83e // bfdot v30.4s, v1.8h, v3.h[3]\n" + ".inst 0x4f44f00a // bfdot v10.4s, v0.8h, v4.h[0]\n" + ".inst 0x4f64f00d // bfdot v13.4s, v0.8h, v4.h[1]\n" + ".inst 0x4f44f810 // bfdot v16.4s, v0.8h, v4.h[2]\n" + ".inst 0x4f64f813 // bfdot v19.4s, v0.8h, v4.h[3]\n" + ".inst 0x4f43f016 // bfdot v22.4s, v0.8h, v3.h[0]\n" + ".inst 0x4f63f019 // bfdot v25.4s, v0.8h, v3.h[1]\n" + ".inst 0x4f43f81c // bfdot v28.4s, v0.8h, v3.h[2]\n" + ".inst 0x4f63f81f // bfdot v31.4s, v0.8h, v3.h[3]\n" "5:" // multiply loop done "subs x23, x23, #0x1\n" "str q8, [%x[Cpanel], #0x0]\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12.hpp index 17c93faca2b3d053cfa26d44cc785db1fbfc6af2..66c2b92a348db0728893ea719b75abacf082715a 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022 Arm Limited. + * Copyright (c) 2019-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef __aarch64__ + #include "../std_transforms_fixed.hpp" #include "../bfloat.hpp" #include "../performance_parameters.hpp" @@ -57,11 +57,6 @@ public: return 12; } - static unsigned int stripe_width() - { - return 4; - } - static constexpr unsigned int k_unroll() { return 4; @@ -117,5 +112,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12/a510.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12/a510.cpp index cba29bc572268bc983278ae5fdd110a9c1e27cd1..bab687a9b4ac3e7a7f4815ab1604c655444b8342 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12/a510.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12/a510.cpp @@ -29,8 +29,12 @@ namespace arm_gemm { void a64_interleaved_bf16fp32_mmla_8x12_a510( - const bfloat16 *Apanel, const bfloat16 *Bpanel, - float *Cpanel, int ablocks, int bblocks, int K) { + const bfloat16 *Apanel, + const bfloat16 *Bpanel, + float *Cpanel, + int ablocks, + int bblocks, + int K) { struct KernelArgs { size_t K = {}; @@ -43,7 +47,6 @@ void a64_interleaved_bf16fp32_mmla_8x12_a510( ka.bblocks = bblocks; __asm__ __volatile__( - "1:" // Height loop "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n" "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n" @@ -82,28 +85,28 @@ void a64_interleaved_bf16fp32_mmla_8x12_a510( "movi v31.16b, #0x0\n" "blt 4f\n" "3:" // main loop head - "ld1 { v3.8h }, [%x[Apanel]], #0x10\n" + "ld1 { v6.8h }, [%x[Apanel]], #0x10\n" ".inst 0x6e44ec08 // bfmmla v8.4s, v0.8h, v4.8h\n" ".inst 0x6e45ec0b // bfmmla v11.4s, v0.8h, v5.8h\n" - "ldp q6, q7, [x22], #0x20\n" + "ldp q3, q7, [x22], #0x20\n" ".inst 0x6e44ec2e // bfmmla v14.4s, v1.8h, v4.8h\n" ".inst 0x6e45ec31 // bfmmla v17.4s, v1.8h, v5.8h\n" ".inst 0x6e44ec54 // bfmmla v20.4s, v2.8h, v4.8h\n" "sub x20, x20, #0x2\n" ".inst 0x6e45ec57 // bfmmla v23.4s, v2.8h, v5.8h\n" - ".inst 0x6e44ec7a // bfmmla v26.4s, v3.8h, v4.8h\n" + ".inst 0x6e44ecda // bfmmla v26.4s, v6.8h, v4.8h\n" "cmp x20, #0x2\n" - ".inst 0x6e45ec7d // bfmmla v29.4s, v3.8h, v5.8h\n" + ".inst 0x6e45ecdd // bfmmla v29.4s, v6.8h, v5.8h\n" "ldp q4, q5, [x22], #0x20\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" + ".inst 0x6e43ec09 // bfmmla v9.4s, v0.8h, v3.8h\n" ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" - ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" + ".inst 0x6e43ec2f // bfmmla v15.4s, v1.8h, v3.8h\n" ".inst 0x6e47ec32 // bfmmla v18.4s, v1.8h, v7.8h\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" + ".inst 0x6e43ec55 // bfmmla v21.4s, v2.8h, v3.8h\n" ".inst 0x6e47ec58 // bfmmla v24.4s, v2.8h, v7.8h\n" - ".inst 0x6e46ec7b // bfmmla v27.4s, v3.8h, v6.8h\n" - ".inst 0x6e47ec7e // bfmmla v30.4s, v3.8h, v7.8h\n" - "ldp q6, q7, [x22], #0x20\n" + ".inst 0x6e43ecdb // bfmmla v27.4s, v6.8h, v3.8h\n" + ".inst 0x6e47ecde // bfmmla v30.4s, v6.8h, v7.8h\n" + "ldp q7, q3, [x22], #0x20\n" ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" ".inst 0x6e45ec0d // bfmmla v13.4s, v0.8h, v5.8h\n" "ld1 { v0.8h }, [%x[Apanel]], #0x10\n" @@ -113,39 +116,39 @@ void a64_interleaved_bf16fp32_mmla_8x12_a510( ".inst 0x6e44ec56 // bfmmla v22.4s, v2.8h, v4.8h\n" ".inst 0x6e45ec59 // bfmmla v25.4s, v2.8h, v5.8h\n" "ld1 { v2.8h }, [%x[Apanel]], #0x10\n" - ".inst 0x6e44ec7c // bfmmla v28.4s, v3.8h, v4.8h\n" - ".inst 0x6e45ec7f // bfmmla v31.4s, v3.8h, v5.8h\n" - "ld1 { v3.8h }, [%x[Apanel]], #0x10\n" - ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" + ".inst 0x6e44ecdc // bfmmla v28.4s, v6.8h, v4.8h\n" + ".inst 0x6e45ecdf // bfmmla v31.4s, v6.8h, v5.8h\n" + "ld1 { v6.8h }, [%x[Apanel]], #0x10\n" + ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" "ldp q4, q5, [x22], #0x20\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - ".inst 0x6e47ec31 // bfmmla v17.4s, v1.8h, v7.8h\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" - ".inst 0x6e46ec7a // bfmmla v26.4s, v3.8h, v6.8h\n" - ".inst 0x6e47ec7d // bfmmla v29.4s, v3.8h, v7.8h\n" - "ldp q6, q7, [x22], #0x20\n" + ".inst 0x6e43ec0b // bfmmla v11.4s, v0.8h, v3.8h\n" + ".inst 0x6e47ec2e // bfmmla v14.4s, v1.8h, v7.8h\n" + ".inst 0x6e43ec31 // bfmmla v17.4s, v1.8h, v3.8h\n" + ".inst 0x6e47ec54 // bfmmla v20.4s, v2.8h, v7.8h\n" + ".inst 0x6e43ec57 // bfmmla v23.4s, v2.8h, v3.8h\n" + ".inst 0x6e47ecda // bfmmla v26.4s, v6.8h, v7.8h\n" + ".inst 0x6e43ecdd // bfmmla v29.4s, v6.8h, v3.8h\n" + "ldp q7, q3, [x22], #0x20\n" ".inst 0x6e44ec09 // bfmmla v9.4s, v0.8h, v4.8h\n" ".inst 0x6e45ec0c // bfmmla v12.4s, v0.8h, v5.8h\n" ".inst 0x6e44ec2f // bfmmla v15.4s, v1.8h, v4.8h\n" ".inst 0x6e45ec32 // bfmmla v18.4s, v1.8h, v5.8h\n" ".inst 0x6e44ec55 // bfmmla v21.4s, v2.8h, v4.8h\n" ".inst 0x6e45ec58 // bfmmla v24.4s, v2.8h, v5.8h\n" - ".inst 0x6e44ec7b // bfmmla v27.4s, v3.8h, v4.8h\n" - ".inst 0x6e45ec7e // bfmmla v30.4s, v3.8h, v5.8h\n" + ".inst 0x6e44ecdb // bfmmla v27.4s, v6.8h, v4.8h\n" + ".inst 0x6e45ecde // bfmmla v30.4s, v6.8h, v5.8h\n" "ldp q4, q5, [x22], #0x20\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" + ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" + ".inst 0x6e43ec0d // bfmmla v13.4s, v0.8h, v3.8h\n" "ld1 { v0.8h }, [%x[Apanel]], #0x10\n" - ".inst 0x6e46ec30 // bfmmla v16.4s, v1.8h, v6.8h\n" - ".inst 0x6e47ec33 // bfmmla v19.4s, v1.8h, v7.8h\n" + ".inst 0x6e47ec30 // bfmmla v16.4s, v1.8h, v7.8h\n" + ".inst 0x6e43ec33 // bfmmla v19.4s, v1.8h, v3.8h\n" "ld1 { v1.8h }, [%x[Apanel]], #0x10\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec59 // bfmmla v25.4s, v2.8h, v7.8h\n" + ".inst 0x6e47ec56 // bfmmla v22.4s, v2.8h, v7.8h\n" + ".inst 0x6e43ec59 // bfmmla v25.4s, v2.8h, v3.8h\n" "ld1 { v2.8h }, [%x[Apanel]], #0x10\n" - ".inst 0x6e46ec7c // bfmmla v28.4s, v3.8h, v6.8h\n" - ".inst 0x6e47ec7f // bfmmla v31.4s, v3.8h, v7.8h\n" + ".inst 0x6e47ecdc // bfmmla v28.4s, v6.8h, v7.8h\n" + ".inst 0x6e43ecdf // bfmmla v31.4s, v6.8h, v3.8h\n" "bge 3b\n" "4:" // main loop skip "ld1 { v3.8h }, [%x[Apanel]], #0x10\n" @@ -158,7 +161,7 @@ void a64_interleaved_bf16fp32_mmla_8x12_a510( ".inst 0x6e45ec57 // bfmmla v23.4s, v2.8h, v5.8h\n" ".inst 0x6e44ec7a // bfmmla v26.4s, v3.8h, v4.8h\n" ".inst 0x6e45ec7d // bfmmla v29.4s, v3.8h, v5.8h\n" - "ldp q4, q5, [x22], #0x20\n" + "ldp q5, q4, [x22], #0x20\n" ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" @@ -167,93 +170,93 @@ void a64_interleaved_bf16fp32_mmla_8x12_a510( ".inst 0x6e47ec58 // bfmmla v24.4s, v2.8h, v7.8h\n" ".inst 0x6e46ec7b // bfmmla v27.4s, v3.8h, v6.8h\n" ".inst 0x6e47ec7e // bfmmla v30.4s, v3.8h, v7.8h\n" - ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" - ".inst 0x6e45ec0d // bfmmla v13.4s, v0.8h, v5.8h\n" - ".inst 0x6e44ec30 // bfmmla v16.4s, v1.8h, v4.8h\n" - ".inst 0x6e45ec33 // bfmmla v19.4s, v1.8h, v5.8h\n" - ".inst 0x6e44ec56 // bfmmla v22.4s, v2.8h, v4.8h\n" - ".inst 0x6e45ec59 // bfmmla v25.4s, v2.8h, v5.8h\n" - ".inst 0x6e44ec7c // bfmmla v28.4s, v3.8h, v4.8h\n" - ".inst 0x6e45ec7f // bfmmla v31.4s, v3.8h, v5.8h\n" + ".inst 0x6e45ec0a // bfmmla v10.4s, v0.8h, v5.8h\n" + ".inst 0x6e44ec0d // bfmmla v13.4s, v0.8h, v4.8h\n" + ".inst 0x6e45ec30 // bfmmla v16.4s, v1.8h, v5.8h\n" + ".inst 0x6e44ec33 // bfmmla v19.4s, v1.8h, v4.8h\n" + ".inst 0x6e45ec56 // bfmmla v22.4s, v2.8h, v5.8h\n" + ".inst 0x6e44ec59 // bfmmla v25.4s, v2.8h, v4.8h\n" + ".inst 0x6e45ec7c // bfmmla v28.4s, v3.8h, v5.8h\n" + ".inst 0x6e44ec7f // bfmmla v31.4s, v3.8h, v4.8h\n" "cbz x20, 5f\n" - "ldp q6, q7, [x22], #0x20\n" - "ld1 { v0.8h }, [%x[Apanel]], #0x10\n" - "ld1 { v1.8h }, [%x[Apanel]], #0x10\n" - ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - "ld1 { v2.8h }, [%x[Apanel]], #0x10\n" - "ld1 { v3.8h }, [%x[Apanel]], #0x10\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" - "ldp q4, q5, [x22], #0x20\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - ".inst 0x6e47ec31 // bfmmla v17.4s, v1.8h, v7.8h\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" - ".inst 0x6e46ec7a // bfmmla v26.4s, v3.8h, v6.8h\n" - ".inst 0x6e47ec7d // bfmmla v29.4s, v3.8h, v7.8h\n" - "ldp q6, q7, [x22], #0x20\n" - ".inst 0x6e44ec09 // bfmmla v9.4s, v0.8h, v4.8h\n" - ".inst 0x6e45ec0c // bfmmla v12.4s, v0.8h, v5.8h\n" - ".inst 0x6e44ec2f // bfmmla v15.4s, v1.8h, v4.8h\n" - ".inst 0x6e45ec32 // bfmmla v18.4s, v1.8h, v5.8h\n" - ".inst 0x6e44ec55 // bfmmla v21.4s, v2.8h, v4.8h\n" - ".inst 0x6e45ec58 // bfmmla v24.4s, v2.8h, v5.8h\n" - ".inst 0x6e44ec7b // bfmmla v27.4s, v3.8h, v4.8h\n" - ".inst 0x6e45ec7e // bfmmla v30.4s, v3.8h, v5.8h\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - ".inst 0x6e46ec30 // bfmmla v16.4s, v1.8h, v6.8h\n" - ".inst 0x6e47ec33 // bfmmla v19.4s, v1.8h, v7.8h\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec59 // bfmmla v25.4s, v2.8h, v7.8h\n" - ".inst 0x6e46ec7c // bfmmla v28.4s, v3.8h, v6.8h\n" - ".inst 0x6e47ec7f // bfmmla v31.4s, v3.8h, v7.8h\n" + "ldp q1, q0, [x22], #0x20\n" + "ld1 { v7.8h }, [%x[Apanel]], #0x10\n" + "ld1 { v6.8h }, [%x[Apanel]], #0x10\n" + ".inst 0x6e41ece8 // bfmmla v8.4s, v7.8h, v1.8h\n" + "ld1 { v5.8h }, [%x[Apanel]], #0x10\n" + "ld1 { v4.8h }, [%x[Apanel]], #0x10\n" + ".inst 0x6e40eceb // bfmmla v11.4s, v7.8h, v0.8h\n" + "ldp q3, q2, [x22], #0x20\n" + ".inst 0x6e41ecce // bfmmla v14.4s, v6.8h, v1.8h\n" + ".inst 0x6e40ecd1 // bfmmla v17.4s, v6.8h, v0.8h\n" + ".inst 0x6e41ecb4 // bfmmla v20.4s, v5.8h, v1.8h\n" + ".inst 0x6e40ecb7 // bfmmla v23.4s, v5.8h, v0.8h\n" + ".inst 0x6e41ec9a // bfmmla v26.4s, v4.8h, v1.8h\n" + ".inst 0x6e40ec9d // bfmmla v29.4s, v4.8h, v0.8h\n" + "ldp q1, q0, [x22], #0x20\n" + ".inst 0x6e43ece9 // bfmmla v9.4s, v7.8h, v3.8h\n" + ".inst 0x6e42ecec // bfmmla v12.4s, v7.8h, v2.8h\n" + ".inst 0x6e43eccf // bfmmla v15.4s, v6.8h, v3.8h\n" + ".inst 0x6e42ecd2 // bfmmla v18.4s, v6.8h, v2.8h\n" + ".inst 0x6e43ecb5 // bfmmla v21.4s, v5.8h, v3.8h\n" + ".inst 0x6e42ecb8 // bfmmla v24.4s, v5.8h, v2.8h\n" + ".inst 0x6e43ec9b // bfmmla v27.4s, v4.8h, v3.8h\n" + ".inst 0x6e42ec9e // bfmmla v30.4s, v4.8h, v2.8h\n" + ".inst 0x6e41ecea // bfmmla v10.4s, v7.8h, v1.8h\n" + ".inst 0x6e40eced // bfmmla v13.4s, v7.8h, v0.8h\n" + ".inst 0x6e41ecd0 // bfmmla v16.4s, v6.8h, v1.8h\n" + ".inst 0x6e40ecd3 // bfmmla v19.4s, v6.8h, v0.8h\n" + ".inst 0x6e41ecb6 // bfmmla v22.4s, v5.8h, v1.8h\n" + ".inst 0x6e40ecb9 // bfmmla v25.4s, v5.8h, v0.8h\n" + ".inst 0x6e41ec9c // bfmmla v28.4s, v4.8h, v1.8h\n" + ".inst 0x6e40ec9f // bfmmla v31.4s, v4.8h, v0.8h\n" "5:" // multiply loop done "subs x23, x23, #0x1\n" - "uzp1 v4.2d, v8.2d, v11.2d\n" + "uzp1 v0.2d, v8.2d, v11.2d\n" "uzp2 v8.2d, v8.2d, v11.2d\n" - "uzp1 v11.2d, v9.2d, v12.2d\n" + "uzp1 v1.2d, v9.2d, v12.2d\n" "uzp2 v9.2d, v9.2d, v12.2d\n" - "str q4, [%x[Cpanel], #0x0]\n" - "uzp1 v12.2d, v10.2d, v13.2d\n" + "str q0, [%x[Cpanel], #0x0]\n" + "uzp1 v0.2d, v10.2d, v13.2d\n" "uzp2 v10.2d, v10.2d, v13.2d\n" - "str q11, [%x[Cpanel], #0x10]\n" - "str q12, [%x[Cpanel], #0x20]\n" - "uzp1 v13.2d, v14.2d, v17.2d\n" + "str q1, [%x[Cpanel], #0x10]\n" + "str q0, [%x[Cpanel], #0x20]\n" + "uzp1 v0.2d, v14.2d, v17.2d\n" "uzp2 v14.2d, v14.2d, v17.2d\n" "str q8, [%x[Cpanel], #0x30]\n" - "uzp1 v17.2d, v15.2d, v18.2d\n" + "uzp1 v2.2d, v15.2d, v18.2d\n" "uzp2 v15.2d, v15.2d, v18.2d\n" "str q9, [%x[Cpanel], #0x40]\n" - "uzp1 v18.2d, v16.2d, v19.2d\n" + "uzp1 v17.2d, v16.2d, v19.2d\n" "uzp2 v16.2d, v16.2d, v19.2d\n" "str q10, [%x[Cpanel], #0x50]\n" - "uzp1 v19.2d, v20.2d, v23.2d\n" + "uzp1 v1.2d, v20.2d, v23.2d\n" "uzp2 v20.2d, v20.2d, v23.2d\n" - "str q13, [%x[Cpanel], #0x60]\n" - "uzp1 v23.2d, v21.2d, v24.2d\n" + "str q0, [%x[Cpanel], #0x60]\n" + "uzp1 v0.2d, v21.2d, v24.2d\n" "uzp2 v21.2d, v21.2d, v24.2d\n" - "str q17, [%x[Cpanel], #0x70]\n" - "uzp1 v24.2d, v22.2d, v25.2d\n" + "str q2, [%x[Cpanel], #0x70]\n" + "uzp1 v23.2d, v22.2d, v25.2d\n" "uzp2 v22.2d, v22.2d, v25.2d\n" - "str q18, [%x[Cpanel], #0x80]\n" - "uzp1 v25.2d, v26.2d, v29.2d\n" + "str q17, [%x[Cpanel], #0x80]\n" + "uzp1 v19.2d, v26.2d, v29.2d\n" "uzp2 v26.2d, v26.2d, v29.2d\n" "str q14, [%x[Cpanel], #0x90]\n" - "uzp1 v29.2d, v27.2d, v30.2d\n" + "uzp1 v18.2d, v27.2d, v30.2d\n" "uzp2 v27.2d, v27.2d, v30.2d\n" "str q15, [%x[Cpanel], #0xa0]\n" - "uzp1 v30.2d, v28.2d, v31.2d\n" + "uzp1 v17.2d, v28.2d, v31.2d\n" "uzp2 v28.2d, v28.2d, v31.2d\n" "str q16, [%x[Cpanel], #0xb0]\n" - "str q19, [%x[Cpanel], #0xc0]\n" - "str q23, [%x[Cpanel], #0xd0]\n" - "str q24, [%x[Cpanel], #0xe0]\n" + "str q1, [%x[Cpanel], #0xc0]\n" + "str q0, [%x[Cpanel], #0xd0]\n" + "str q23, [%x[Cpanel], #0xe0]\n" "str q20, [%x[Cpanel], #0xf0]\n" "str q21, [%x[Cpanel], #0x100]\n" "str q22, [%x[Cpanel], #0x110]\n" - "str q25, [%x[Cpanel], #0x120]\n" - "str q29, [%x[Cpanel], #0x130]\n" - "str q30, [%x[Cpanel], #0x140]\n" + "str q19, [%x[Cpanel], #0x120]\n" + "str q18, [%x[Cpanel], #0x130]\n" + "str q17, [%x[Cpanel], #0x140]\n" "str q26, [%x[Cpanel], #0x150]\n" "str q27, [%x[Cpanel], #0x160]\n" "str q28, [%x[Cpanel], #0x170]\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12/generic.cpp index 2938639048f76d35ed864764dac8b578f957a89e..8485820c7cc71a370335c8f264932d69b69dc617 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_bf16fp32_mmla_8x12/generic.cpp @@ -29,8 +29,12 @@ namespace arm_gemm { void a64_interleaved_bf16fp32_mmla_8x12( - const bfloat16 *Apanel, const bfloat16 *Bpanel, - float *Cpanel, int ablocks, int bblocks, int K) { + const bfloat16 *Apanel, + const bfloat16 *Bpanel, + float *Cpanel, + int ablocks, + int bblocks, + int K) { struct KernelArgs { size_t K = {}; @@ -43,7 +47,6 @@ void a64_interleaved_bf16fp32_mmla_8x12( ka.bblocks = bblocks; __asm__ __volatile__( - "1:" // Height loop "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n" "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n" @@ -85,31 +88,31 @@ void a64_interleaved_bf16fp32_mmla_8x12( "movi v31.16b, #0x0\n" "blt 4f\n" "3:" // main loop head - "ldr q3, [%x[Apanel], #0x0]\n" - "ldr q6, [x22, #0x0]\n" + "ldr q6, [%x[Apanel], #0x0]\n" + "ldr q7, [x22, #0x0]\n" ".inst 0x6e44ec08 // bfmmla v8.4s, v0.8h, v4.8h\n" - "ldr q7, [x22, #0x10]\n" + "ldr q3, [x22, #0x10]\n" ".inst 0x6e45ec0b // bfmmla v11.4s, v0.8h, v5.8h\n" ".inst 0x6e44ec2e // bfmmla v14.4s, v1.8h, v4.8h\n" ".inst 0x6e45ec31 // bfmmla v17.4s, v1.8h, v5.8h\n" ".inst 0x6e44ec54 // bfmmla v20.4s, v2.8h, v4.8h\n" "sub x20, x20, #0x2\n" ".inst 0x6e45ec57 // bfmmla v23.4s, v2.8h, v5.8h\n" - ".inst 0x6e44ec7a // bfmmla v26.4s, v3.8h, v4.8h\n" + ".inst 0x6e44ecda // bfmmla v26.4s, v6.8h, v4.8h\n" "ldr q4, [x22, #0x20]\n" - ".inst 0x6e45ec7d // bfmmla v29.4s, v3.8h, v5.8h\n" + ".inst 0x6e45ecdd // bfmmla v29.4s, v6.8h, v5.8h\n" "ldr q5, [x22, #0x30]\n" - ".inst 0x6e46ec09 // bfmmla v9.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0c // bfmmla v12.4s, v0.8h, v7.8h\n" - ".inst 0x6e46ec2f // bfmmla v15.4s, v1.8h, v6.8h\n" + ".inst 0x6e47ec09 // bfmmla v9.4s, v0.8h, v7.8h\n" + ".inst 0x6e43ec0c // bfmmla v12.4s, v0.8h, v3.8h\n" + ".inst 0x6e47ec2f // bfmmla v15.4s, v1.8h, v7.8h\n" "cmp x20, #0x2\n" - ".inst 0x6e47ec32 // bfmmla v18.4s, v1.8h, v7.8h\n" - ".inst 0x6e46ec55 // bfmmla v21.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec58 // bfmmla v24.4s, v2.8h, v7.8h\n" - ".inst 0x6e46ec7b // bfmmla v27.4s, v3.8h, v6.8h\n" - "ldr q6, [x22, #0x40]\n" - ".inst 0x6e47ec7e // bfmmla v30.4s, v3.8h, v7.8h\n" - "ldr q7, [x22, #0x50]\n" + ".inst 0x6e43ec32 // bfmmla v18.4s, v1.8h, v3.8h\n" + ".inst 0x6e47ec55 // bfmmla v21.4s, v2.8h, v7.8h\n" + ".inst 0x6e43ec58 // bfmmla v24.4s, v2.8h, v3.8h\n" + ".inst 0x6e47ecdb // bfmmla v27.4s, v6.8h, v7.8h\n" + "ldr q7, [x22, #0x40]\n" + ".inst 0x6e43ecde // bfmmla v30.4s, v6.8h, v3.8h\n" + "ldr q3, [x22, #0x50]\n" ".inst 0x6e44ec0a // bfmmla v10.4s, v0.8h, v4.8h\n" ".inst 0x6e45ec0d // bfmmla v13.4s, v0.8h, v5.8h\n" "ldr q0, [%x[Apanel], #0x10]\n" @@ -119,42 +122,42 @@ void a64_interleaved_bf16fp32_mmla_8x12( ".inst 0x6e44ec56 // bfmmla v22.4s, v2.8h, v4.8h\n" ".inst 0x6e45ec59 // bfmmla v25.4s, v2.8h, v5.8h\n" "ldr q2, [%x[Apanel], #0x30]\n" - ".inst 0x6e44ec7c // bfmmla v28.4s, v3.8h, v4.8h\n" + ".inst 0x6e44ecdc // bfmmla v28.4s, v6.8h, v4.8h\n" "ldr q4, [x22, #0x60]\n" - ".inst 0x6e45ec7f // bfmmla v31.4s, v3.8h, v5.8h\n" - "ldr q3, [%x[Apanel], #0x40]\n" + ".inst 0x6e45ecdf // bfmmla v31.4s, v6.8h, v5.8h\n" + "ldr q6, [%x[Apanel], #0x40]\n" "ldr q5, [x22, #0x70]\n" - ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - ".inst 0x6e47ec31 // bfmmla v17.4s, v1.8h, v7.8h\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" - ".inst 0x6e46ec7a // bfmmla v26.4s, v3.8h, v6.8h\n" - "ldr q6, [x22, #0x80]\n" - ".inst 0x6e47ec7d // bfmmla v29.4s, v3.8h, v7.8h\n" - "ldr q7, [x22, #0x90]\n" + ".inst 0x6e47ec08 // bfmmla v8.4s, v0.8h, v7.8h\n" + ".inst 0x6e43ec0b // bfmmla v11.4s, v0.8h, v3.8h\n" + ".inst 0x6e47ec2e // bfmmla v14.4s, v1.8h, v7.8h\n" + ".inst 0x6e43ec31 // bfmmla v17.4s, v1.8h, v3.8h\n" + ".inst 0x6e47ec54 // bfmmla v20.4s, v2.8h, v7.8h\n" + ".inst 0x6e43ec57 // bfmmla v23.4s, v2.8h, v3.8h\n" + ".inst 0x6e47ecda // bfmmla v26.4s, v6.8h, v7.8h\n" + "ldr q7, [x22, #0x80]\n" + ".inst 0x6e43ecdd // bfmmla v29.4s, v6.8h, v3.8h\n" + "ldr q3, [x22, #0x90]\n" ".inst 0x6e44ec09 // bfmmla v9.4s, v0.8h, v4.8h\n" ".inst 0x6e45ec0c // bfmmla v12.4s, v0.8h, v5.8h\n" ".inst 0x6e44ec2f // bfmmla v15.4s, v1.8h, v4.8h\n" ".inst 0x6e45ec32 // bfmmla v18.4s, v1.8h, v5.8h\n" ".inst 0x6e44ec55 // bfmmla v21.4s, v2.8h, v4.8h\n" ".inst 0x6e45ec58 // bfmmla v24.4s, v2.8h, v5.8h\n" - ".inst 0x6e44ec7b // bfmmla v27.4s, v3.8h, v4.8h\n" + ".inst 0x6e44ecdb // bfmmla v27.4s, v6.8h, v4.8h\n" "ldr q4, [x22, #0xa0]\n" - ".inst 0x6e45ec7e // bfmmla v30.4s, v3.8h, v5.8h\n" + ".inst 0x6e45ecde // bfmmla v30.4s, v6.8h, v5.8h\n" "ldr q5, [x22, #0xb0]\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" + ".inst 0x6e47ec0a // bfmmla v10.4s, v0.8h, v7.8h\n" + ".inst 0x6e43ec0d // bfmmla v13.4s, v0.8h, v3.8h\n" "ldr q0, [%x[Apanel], #0x50]\n" - ".inst 0x6e46ec30 // bfmmla v16.4s, v1.8h, v6.8h\n" - ".inst 0x6e47ec33 // bfmmla v19.4s, v1.8h, v7.8h\n" + ".inst 0x6e47ec30 // bfmmla v16.4s, v1.8h, v7.8h\n" + ".inst 0x6e43ec33 // bfmmla v19.4s, v1.8h, v3.8h\n" "ldr q1, [%x[Apanel], #0x60]\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec59 // bfmmla v25.4s, v2.8h, v7.8h\n" + ".inst 0x6e47ec56 // bfmmla v22.4s, v2.8h, v7.8h\n" + ".inst 0x6e43ec59 // bfmmla v25.4s, v2.8h, v3.8h\n" "ldr q2, [%x[Apanel], #0x70]\n" - ".inst 0x6e46ec7c // bfmmla v28.4s, v3.8h, v6.8h\n" - ".inst 0x6e47ec7f // bfmmla v31.4s, v3.8h, v7.8h\n" + ".inst 0x6e47ecdc // bfmmla v28.4s, v6.8h, v7.8h\n" + ".inst 0x6e43ecdf // bfmmla v31.4s, v6.8h, v3.8h\n" "add %x[Apanel], %x[Apanel], #0x80\n" "add x22, x22, #0xc0\n" "bge 3b\n" @@ -191,89 +194,89 @@ void a64_interleaved_bf16fp32_mmla_8x12( ".inst 0x6e44ec7c // bfmmla v28.4s, v3.8h, v4.8h\n" ".inst 0x6e45ec7f // bfmmla v31.4s, v3.8h, v5.8h\n" "cbz x20, 5f\n" - "ldr q6, [x22, #0x0]\n" - "ldr q0, [%x[Apanel], #0x0]\n" - ".inst 0x6e46ec08 // bfmmla v8.4s, v0.8h, v6.8h\n" - "ldr q1, [%x[Apanel], #0x10]\n" - "ldr q7, [x22, #0x10]\n" - ".inst 0x6e47ec0b // bfmmla v11.4s, v0.8h, v7.8h\n" - "ldr q2, [%x[Apanel], #0x20]\n" - "ldr q3, [%x[Apanel], #0x30]\n" - ".inst 0x6e46ec2e // bfmmla v14.4s, v1.8h, v6.8h\n" - "ldr q4, [x22, #0x20]\n" - "ldr q5, [x22, #0x30]\n" - ".inst 0x6e47ec31 // bfmmla v17.4s, v1.8h, v7.8h\n" - ".inst 0x6e46ec54 // bfmmla v20.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec57 // bfmmla v23.4s, v2.8h, v7.8h\n" + "ldr q1, [x22, #0x0]\n" + "ldr q7, [%x[Apanel], #0x0]\n" + ".inst 0x6e41ece8 // bfmmla v8.4s, v7.8h, v1.8h\n" + "ldr q6, [%x[Apanel], #0x10]\n" + "ldr q0, [x22, #0x10]\n" + ".inst 0x6e40eceb // bfmmla v11.4s, v7.8h, v0.8h\n" + "ldr q5, [%x[Apanel], #0x20]\n" + "ldr q4, [%x[Apanel], #0x30]\n" + ".inst 0x6e41ecce // bfmmla v14.4s, v6.8h, v1.8h\n" + "ldr q3, [x22, #0x20]\n" + "ldr q2, [x22, #0x30]\n" + ".inst 0x6e40ecd1 // bfmmla v17.4s, v6.8h, v0.8h\n" + ".inst 0x6e41ecb4 // bfmmla v20.4s, v5.8h, v1.8h\n" + ".inst 0x6e40ecb7 // bfmmla v23.4s, v5.8h, v0.8h\n" "add %x[Apanel], %x[Apanel], #0x40\n" - ".inst 0x6e46ec7a // bfmmla v26.4s, v3.8h, v6.8h\n" - "ldr q6, [x22, #0x40]\n" - ".inst 0x6e47ec7d // bfmmla v29.4s, v3.8h, v7.8h\n" - "ldr q7, [x22, #0x50]\n" - ".inst 0x6e44ec09 // bfmmla v9.4s, v0.8h, v4.8h\n" - ".inst 0x6e45ec0c // bfmmla v12.4s, v0.8h, v5.8h\n" - ".inst 0x6e44ec2f // bfmmla v15.4s, v1.8h, v4.8h\n" - ".inst 0x6e45ec32 // bfmmla v18.4s, v1.8h, v5.8h\n" + ".inst 0x6e41ec9a // bfmmla v26.4s, v4.8h, v1.8h\n" + "ldr q1, [x22, #0x40]\n" + ".inst 0x6e40ec9d // bfmmla v29.4s, v4.8h, v0.8h\n" + "ldr q0, [x22, #0x50]\n" + ".inst 0x6e43ece9 // bfmmla v9.4s, v7.8h, v3.8h\n" + ".inst 0x6e42ecec // bfmmla v12.4s, v7.8h, v2.8h\n" + ".inst 0x6e43eccf // bfmmla v15.4s, v6.8h, v3.8h\n" + ".inst 0x6e42ecd2 // bfmmla v18.4s, v6.8h, v2.8h\n" "add x22, x22, #0x60\n" - ".inst 0x6e44ec55 // bfmmla v21.4s, v2.8h, v4.8h\n" - ".inst 0x6e45ec58 // bfmmla v24.4s, v2.8h, v5.8h\n" - ".inst 0x6e44ec7b // bfmmla v27.4s, v3.8h, v4.8h\n" - ".inst 0x6e45ec7e // bfmmla v30.4s, v3.8h, v5.8h\n" - ".inst 0x6e46ec0a // bfmmla v10.4s, v0.8h, v6.8h\n" - ".inst 0x6e47ec0d // bfmmla v13.4s, v0.8h, v7.8h\n" - ".inst 0x6e46ec30 // bfmmla v16.4s, v1.8h, v6.8h\n" - ".inst 0x6e47ec33 // bfmmla v19.4s, v1.8h, v7.8h\n" - ".inst 0x6e46ec56 // bfmmla v22.4s, v2.8h, v6.8h\n" - ".inst 0x6e47ec59 // bfmmla v25.4s, v2.8h, v7.8h\n" - ".inst 0x6e46ec7c // bfmmla v28.4s, v3.8h, v6.8h\n" - ".inst 0x6e47ec7f // bfmmla v31.4s, v3.8h, v7.8h\n" + ".inst 0x6e43ecb5 // bfmmla v21.4s, v5.8h, v3.8h\n" + ".inst 0x6e42ecb8 // bfmmla v24.4s, v5.8h, v2.8h\n" + ".inst 0x6e43ec9b // bfmmla v27.4s, v4.8h, v3.8h\n" + ".inst 0x6e42ec9e // bfmmla v30.4s, v4.8h, v2.8h\n" + ".inst 0x6e41ecea // bfmmla v10.4s, v7.8h, v1.8h\n" + ".inst 0x6e40eced // bfmmla v13.4s, v7.8h, v0.8h\n" + ".inst 0x6e41ecd0 // bfmmla v16.4s, v6.8h, v1.8h\n" + ".inst 0x6e40ecd3 // bfmmla v19.4s, v6.8h, v0.8h\n" + ".inst 0x6e41ecb6 // bfmmla v22.4s, v5.8h, v1.8h\n" + ".inst 0x6e40ecb9 // bfmmla v25.4s, v5.8h, v0.8h\n" + ".inst 0x6e41ec9c // bfmmla v28.4s, v4.8h, v1.8h\n" + ".inst 0x6e40ec9f // bfmmla v31.4s, v4.8h, v0.8h\n" "5:" // multiply loop done "subs x23, x23, #0x1\n" - "uzp1 v4.2d, v8.2d, v11.2d\n" + "uzp1 v0.2d, v8.2d, v11.2d\n" "uzp2 v8.2d, v8.2d, v11.2d\n" - "uzp1 v11.2d, v9.2d, v12.2d\n" + "uzp1 v1.2d, v9.2d, v12.2d\n" "uzp2 v9.2d, v9.2d, v12.2d\n" - "str q4, [%x[Cpanel], #0x0]\n" - "uzp1 v12.2d, v10.2d, v13.2d\n" + "str q0, [%x[Cpanel], #0x0]\n" + "uzp1 v0.2d, v10.2d, v13.2d\n" "uzp2 v10.2d, v10.2d, v13.2d\n" - "str q11, [%x[Cpanel], #0x10]\n" - "str q12, [%x[Cpanel], #0x20]\n" - "uzp1 v13.2d, v14.2d, v17.2d\n" + "str q1, [%x[Cpanel], #0x10]\n" + "str q0, [%x[Cpanel], #0x20]\n" + "uzp1 v0.2d, v14.2d, v17.2d\n" "uzp2 v14.2d, v14.2d, v17.2d\n" "str q8, [%x[Cpanel], #0x30]\n" - "uzp1 v17.2d, v15.2d, v18.2d\n" + "uzp1 v2.2d, v15.2d, v18.2d\n" "uzp2 v15.2d, v15.2d, v18.2d\n" "str q9, [%x[Cpanel], #0x40]\n" - "uzp1 v18.2d, v16.2d, v19.2d\n" + "uzp1 v17.2d, v16.2d, v19.2d\n" "uzp2 v16.2d, v16.2d, v19.2d\n" "str q10, [%x[Cpanel], #0x50]\n" - "uzp1 v19.2d, v20.2d, v23.2d\n" + "uzp1 v1.2d, v20.2d, v23.2d\n" "uzp2 v20.2d, v20.2d, v23.2d\n" - "str q13, [%x[Cpanel], #0x60]\n" - "uzp1 v23.2d, v21.2d, v24.2d\n" + "str q0, [%x[Cpanel], #0x60]\n" + "uzp1 v0.2d, v21.2d, v24.2d\n" "uzp2 v21.2d, v21.2d, v24.2d\n" - "str q17, [%x[Cpanel], #0x70]\n" - "uzp1 v24.2d, v22.2d, v25.2d\n" + "str q2, [%x[Cpanel], #0x70]\n" + "uzp1 v23.2d, v22.2d, v25.2d\n" "uzp2 v22.2d, v22.2d, v25.2d\n" - "str q18, [%x[Cpanel], #0x80]\n" - "uzp1 v25.2d, v26.2d, v29.2d\n" + "str q17, [%x[Cpanel], #0x80]\n" + "uzp1 v19.2d, v26.2d, v29.2d\n" "uzp2 v26.2d, v26.2d, v29.2d\n" "str q14, [%x[Cpanel], #0x90]\n" - "uzp1 v29.2d, v27.2d, v30.2d\n" + "uzp1 v18.2d, v27.2d, v30.2d\n" "uzp2 v27.2d, v27.2d, v30.2d\n" "str q15, [%x[Cpanel], #0xa0]\n" - "uzp1 v30.2d, v28.2d, v31.2d\n" + "uzp1 v17.2d, v28.2d, v31.2d\n" "uzp2 v28.2d, v28.2d, v31.2d\n" "str q16, [%x[Cpanel], #0xb0]\n" - "str q19, [%x[Cpanel], #0xc0]\n" - "str q23, [%x[Cpanel], #0xd0]\n" - "str q24, [%x[Cpanel], #0xe0]\n" + "str q1, [%x[Cpanel], #0xc0]\n" + "str q0, [%x[Cpanel], #0xd0]\n" + "str q23, [%x[Cpanel], #0xe0]\n" "str q20, [%x[Cpanel], #0xf0]\n" "str q21, [%x[Cpanel], #0x100]\n" "str q22, [%x[Cpanel], #0x110]\n" - "str q25, [%x[Cpanel], #0x120]\n" - "str q29, [%x[Cpanel], #0x130]\n" - "str q30, [%x[Cpanel], #0x140]\n" + "str q19, [%x[Cpanel], #0x120]\n" + "str q18, [%x[Cpanel], #0x130]\n" + "str q17, [%x[Cpanel], #0x140]\n" "str q26, [%x[Cpanel], #0x150]\n" "str q27, [%x[Cpanel], #0x160]\n" "str q28, [%x[Cpanel], #0x170]\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12.hpp index 4cc3ed040af57a531894f8e2759ad554a8d1acff..37a54fcfaba5651a9ba0396583f6a0dff1b381d8 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef __aarch64__ + #include "../std_transforms_fixed.hpp" #include "../performance_parameters.hpp" @@ -56,11 +56,6 @@ public: return 12; } - static unsigned int stripe_width() - { - return 4; - } - static constexpr unsigned int k_unroll() { return 8; @@ -111,11 +106,9 @@ public: break; } } - }; } // namespace arm_gemm #undef ARGLIST - #endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12/a510.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12/a510.cpp index e46cb8a67a8939069c7c085af5fbcaf681d3be14..c1d37383df04df33a9907728396a0cd9afc8882c 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12/a510.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12/a510.cpp @@ -29,8 +29,12 @@ namespace arm_gemm { void a64_interleaved_s8s32_mmla_8x12_a510( - const int8_t *Apanel, const int8_t *Bpanel, - int32_t *Cpanel, int ablocks, int bblocks, int K) { + const int8_t *Apanel, + const int8_t *Bpanel, + int32_t *Cpanel, + int ablocks, + int bblocks, + int K) { struct KernelArgs { size_t K = {}; @@ -43,7 +47,6 @@ void a64_interleaved_s8s32_mmla_8x12_a510( ka.bblocks = bblocks; __asm__ __volatile__( - "1:" // Height loop "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n" "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n" @@ -82,28 +85,28 @@ void a64_interleaved_s8s32_mmla_8x12_a510( "movi v31.4s, #0x0\n" "blt 4f\n" "3:" // main loop head - "ld1 { v3.16b }, [%x[Apanel]], #0x10\n" + "ld1 { v6.16b }, [%x[Apanel]], #0x10\n" ".inst 0x4e84a408 // smmla v8.4s, v0.16b, v4.16b\n" ".inst 0x4e85a40b // smmla v11.4s, v0.16b, v5.16b\n" - "ldp q6, q7, [x22], #0x20\n" + "ldp q3, q7, [x22], #0x20\n" ".inst 0x4e84a42e // smmla v14.4s, v1.16b, v4.16b\n" ".inst 0x4e85a431 // smmla v17.4s, v1.16b, v5.16b\n" ".inst 0x4e84a454 // smmla v20.4s, v2.16b, v4.16b\n" "sub x20, x20, #0x2\n" ".inst 0x4e85a457 // smmla v23.4s, v2.16b, v5.16b\n" - ".inst 0x4e84a47a // smmla v26.4s, v3.16b, v4.16b\n" + ".inst 0x4e84a4da // smmla v26.4s, v6.16b, v4.16b\n" "cmp x20, #0x2\n" - ".inst 0x4e85a47d // smmla v29.4s, v3.16b, v5.16b\n" + ".inst 0x4e85a4dd // smmla v29.4s, v6.16b, v5.16b\n" "ldp q4, q5, [x22], #0x20\n" - ".inst 0x4e86a409 // smmla v9.4s, v0.16b, v6.16b\n" + ".inst 0x4e83a409 // smmla v9.4s, v0.16b, v3.16b\n" ".inst 0x4e87a40c // smmla v12.4s, v0.16b, v7.16b\n" - ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" + ".inst 0x4e83a42f // smmla v15.4s, v1.16b, v3.16b\n" ".inst 0x4e87a432 // smmla v18.4s, v1.16b, v7.16b\n" - ".inst 0x4e86a455 // smmla v21.4s, v2.16b, v6.16b\n" + ".inst 0x4e83a455 // smmla v21.4s, v2.16b, v3.16b\n" ".inst 0x4e87a458 // smmla v24.4s, v2.16b, v7.16b\n" - ".inst 0x4e86a47b // smmla v27.4s, v3.16b, v6.16b\n" - ".inst 0x4e87a47e // smmla v30.4s, v3.16b, v7.16b\n" - "ldp q6, q7, [x22], #0x20\n" + ".inst 0x4e83a4db // smmla v27.4s, v6.16b, v3.16b\n" + ".inst 0x4e87a4de // smmla v30.4s, v6.16b, v7.16b\n" + "ldp q7, q3, [x22], #0x20\n" ".inst 0x4e84a40a // smmla v10.4s, v0.16b, v4.16b\n" ".inst 0x4e85a40d // smmla v13.4s, v0.16b, v5.16b\n" "ld1 { v0.16b }, [%x[Apanel]], #0x10\n" @@ -113,39 +116,39 @@ void a64_interleaved_s8s32_mmla_8x12_a510( ".inst 0x4e84a456 // smmla v22.4s, v2.16b, v4.16b\n" ".inst 0x4e85a459 // smmla v25.4s, v2.16b, v5.16b\n" "ld1 { v2.16b }, [%x[Apanel]], #0x10\n" - ".inst 0x4e84a47c // smmla v28.4s, v3.16b, v4.16b\n" - ".inst 0x4e85a47f // smmla v31.4s, v3.16b, v5.16b\n" - "ld1 { v3.16b }, [%x[Apanel]], #0x10\n" - ".inst 0x4e86a408 // smmla v8.4s, v0.16b, v6.16b\n" + ".inst 0x4e84a4dc // smmla v28.4s, v6.16b, v4.16b\n" + ".inst 0x4e85a4df // smmla v31.4s, v6.16b, v5.16b\n" + "ld1 { v6.16b }, [%x[Apanel]], #0x10\n" + ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" "ldp q4, q5, [x22], #0x20\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - ".inst 0x4e87a431 // smmla v17.4s, v1.16b, v7.16b\n" - ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n" - ".inst 0x4e87a457 // smmla v23.4s, v2.16b, v7.16b\n" - ".inst 0x4e86a47a // smmla v26.4s, v3.16b, v6.16b\n" - ".inst 0x4e87a47d // smmla v29.4s, v3.16b, v7.16b\n" - "ldp q6, q7, [x22], #0x20\n" + ".inst 0x4e83a40b // smmla v11.4s, v0.16b, v3.16b\n" + ".inst 0x4e87a42e // smmla v14.4s, v1.16b, v7.16b\n" + ".inst 0x4e83a431 // smmla v17.4s, v1.16b, v3.16b\n" + ".inst 0x4e87a454 // smmla v20.4s, v2.16b, v7.16b\n" + ".inst 0x4e83a457 // smmla v23.4s, v2.16b, v3.16b\n" + ".inst 0x4e87a4da // smmla v26.4s, v6.16b, v7.16b\n" + ".inst 0x4e83a4dd // smmla v29.4s, v6.16b, v3.16b\n" + "ldp q7, q3, [x22], #0x20\n" ".inst 0x4e84a409 // smmla v9.4s, v0.16b, v4.16b\n" ".inst 0x4e85a40c // smmla v12.4s, v0.16b, v5.16b\n" ".inst 0x4e84a42f // smmla v15.4s, v1.16b, v4.16b\n" ".inst 0x4e85a432 // smmla v18.4s, v1.16b, v5.16b\n" ".inst 0x4e84a455 // smmla v21.4s, v2.16b, v4.16b\n" ".inst 0x4e85a458 // smmla v24.4s, v2.16b, v5.16b\n" - ".inst 0x4e84a47b // smmla v27.4s, v3.16b, v4.16b\n" - ".inst 0x4e85a47e // smmla v30.4s, v3.16b, v5.16b\n" + ".inst 0x4e84a4db // smmla v27.4s, v6.16b, v4.16b\n" + ".inst 0x4e85a4de // smmla v30.4s, v6.16b, v5.16b\n" "ldp q4, q5, [x22], #0x20\n" - ".inst 0x4e86a40a // smmla v10.4s, v0.16b, v6.16b\n" - ".inst 0x4e87a40d // smmla v13.4s, v0.16b, v7.16b\n" + ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" + ".inst 0x4e83a40d // smmla v13.4s, v0.16b, v3.16b\n" "ld1 { v0.16b }, [%x[Apanel]], #0x10\n" - ".inst 0x4e86a430 // smmla v16.4s, v1.16b, v6.16b\n" - ".inst 0x4e87a433 // smmla v19.4s, v1.16b, v7.16b\n" + ".inst 0x4e87a430 // smmla v16.4s, v1.16b, v7.16b\n" + ".inst 0x4e83a433 // smmla v19.4s, v1.16b, v3.16b\n" "ld1 { v1.16b }, [%x[Apanel]], #0x10\n" - ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n" - ".inst 0x4e87a459 // smmla v25.4s, v2.16b, v7.16b\n" + ".inst 0x4e87a456 // smmla v22.4s, v2.16b, v7.16b\n" + ".inst 0x4e83a459 // smmla v25.4s, v2.16b, v3.16b\n" "ld1 { v2.16b }, [%x[Apanel]], #0x10\n" - ".inst 0x4e86a47c // smmla v28.4s, v3.16b, v6.16b\n" - ".inst 0x4e87a47f // smmla v31.4s, v3.16b, v7.16b\n" + ".inst 0x4e87a4dc // smmla v28.4s, v6.16b, v7.16b\n" + ".inst 0x4e83a4df // smmla v31.4s, v6.16b, v3.16b\n" "bge 3b\n" "4:" // main loop skip "ld1 { v3.16b }, [%x[Apanel]], #0x10\n" @@ -158,7 +161,7 @@ void a64_interleaved_s8s32_mmla_8x12_a510( ".inst 0x4e85a457 // smmla v23.4s, v2.16b, v5.16b\n" ".inst 0x4e84a47a // smmla v26.4s, v3.16b, v4.16b\n" ".inst 0x4e85a47d // smmla v29.4s, v3.16b, v5.16b\n" - "ldp q4, q5, [x22], #0x20\n" + "ldp q5, q4, [x22], #0x20\n" ".inst 0x4e86a409 // smmla v9.4s, v0.16b, v6.16b\n" ".inst 0x4e87a40c // smmla v12.4s, v0.16b, v7.16b\n" ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" @@ -167,93 +170,93 @@ void a64_interleaved_s8s32_mmla_8x12_a510( ".inst 0x4e87a458 // smmla v24.4s, v2.16b, v7.16b\n" ".inst 0x4e86a47b // smmla v27.4s, v3.16b, v6.16b\n" ".inst 0x4e87a47e // smmla v30.4s, v3.16b, v7.16b\n" - ".inst 0x4e84a40a // smmla v10.4s, v0.16b, v4.16b\n" - ".inst 0x4e85a40d // smmla v13.4s, v0.16b, v5.16b\n" - ".inst 0x4e84a430 // smmla v16.4s, v1.16b, v4.16b\n" - ".inst 0x4e85a433 // smmla v19.4s, v1.16b, v5.16b\n" - ".inst 0x4e84a456 // smmla v22.4s, v2.16b, v4.16b\n" - ".inst 0x4e85a459 // smmla v25.4s, v2.16b, v5.16b\n" - ".inst 0x4e84a47c // smmla v28.4s, v3.16b, v4.16b\n" - ".inst 0x4e85a47f // smmla v31.4s, v3.16b, v5.16b\n" + ".inst 0x4e85a40a // smmla v10.4s, v0.16b, v5.16b\n" + ".inst 0x4e84a40d // smmla v13.4s, v0.16b, v4.16b\n" + ".inst 0x4e85a430 // smmla v16.4s, v1.16b, v5.16b\n" + ".inst 0x4e84a433 // smmla v19.4s, v1.16b, v4.16b\n" + ".inst 0x4e85a456 // smmla v22.4s, v2.16b, v5.16b\n" + ".inst 0x4e84a459 // smmla v25.4s, v2.16b, v4.16b\n" + ".inst 0x4e85a47c // smmla v28.4s, v3.16b, v5.16b\n" + ".inst 0x4e84a47f // smmla v31.4s, v3.16b, v4.16b\n" "cbz x20, 5f\n" - "ldp q6, q7, [x22], #0x20\n" - "ld1 { v0.16b }, [%x[Apanel]], #0x10\n" - "ld1 { v1.16b }, [%x[Apanel]], #0x10\n" - ".inst 0x4e86a408 // smmla v8.4s, v0.16b, v6.16b\n" - "ld1 { v2.16b }, [%x[Apanel]], #0x10\n" - "ld1 { v3.16b }, [%x[Apanel]], #0x10\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" - "ldp q4, q5, [x22], #0x20\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - ".inst 0x4e87a431 // smmla v17.4s, v1.16b, v7.16b\n" - ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n" - ".inst 0x4e87a457 // smmla v23.4s, v2.16b, v7.16b\n" - ".inst 0x4e86a47a // smmla v26.4s, v3.16b, v6.16b\n" - ".inst 0x4e87a47d // smmla v29.4s, v3.16b, v7.16b\n" - "ldp q6, q7, [x22], #0x20\n" - ".inst 0x4e84a409 // smmla v9.4s, v0.16b, v4.16b\n" - ".inst 0x4e85a40c // smmla v12.4s, v0.16b, v5.16b\n" - ".inst 0x4e84a42f // smmla v15.4s, v1.16b, v4.16b\n" - ".inst 0x4e85a432 // smmla v18.4s, v1.16b, v5.16b\n" - ".inst 0x4e84a455 // smmla v21.4s, v2.16b, v4.16b\n" - ".inst 0x4e85a458 // smmla v24.4s, v2.16b, v5.16b\n" - ".inst 0x4e84a47b // smmla v27.4s, v3.16b, v4.16b\n" - ".inst 0x4e85a47e // smmla v30.4s, v3.16b, v5.16b\n" - ".inst 0x4e86a40a // smmla v10.4s, v0.16b, v6.16b\n" - ".inst 0x4e87a40d // smmla v13.4s, v0.16b, v7.16b\n" - ".inst 0x4e86a430 // smmla v16.4s, v1.16b, v6.16b\n" - ".inst 0x4e87a433 // smmla v19.4s, v1.16b, v7.16b\n" - ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n" - ".inst 0x4e87a459 // smmla v25.4s, v2.16b, v7.16b\n" - ".inst 0x4e86a47c // smmla v28.4s, v3.16b, v6.16b\n" - ".inst 0x4e87a47f // smmla v31.4s, v3.16b, v7.16b\n" + "ldp q1, q0, [x22], #0x20\n" + "ld1 { v7.16b }, [%x[Apanel]], #0x10\n" + "ld1 { v6.16b }, [%x[Apanel]], #0x10\n" + ".inst 0x4e81a4e8 // smmla v8.4s, v7.16b, v1.16b\n" + "ld1 { v5.16b }, [%x[Apanel]], #0x10\n" + "ld1 { v4.16b }, [%x[Apanel]], #0x10\n" + ".inst 0x4e80a4eb // smmla v11.4s, v7.16b, v0.16b\n" + "ldp q3, q2, [x22], #0x20\n" + ".inst 0x4e81a4ce // smmla v14.4s, v6.16b, v1.16b\n" + ".inst 0x4e80a4d1 // smmla v17.4s, v6.16b, v0.16b\n" + ".inst 0x4e81a4b4 // smmla v20.4s, v5.16b, v1.16b\n" + ".inst 0x4e80a4b7 // smmla v23.4s, v5.16b, v0.16b\n" + ".inst 0x4e81a49a // smmla v26.4s, v4.16b, v1.16b\n" + ".inst 0x4e80a49d // smmla v29.4s, v4.16b, v0.16b\n" + "ldp q1, q0, [x22], #0x20\n" + ".inst 0x4e83a4e9 // smmla v9.4s, v7.16b, v3.16b\n" + ".inst 0x4e82a4ec // smmla v12.4s, v7.16b, v2.16b\n" + ".inst 0x4e83a4cf // smmla v15.4s, v6.16b, v3.16b\n" + ".inst 0x4e82a4d2 // smmla v18.4s, v6.16b, v2.16b\n" + ".inst 0x4e83a4b5 // smmla v21.4s, v5.16b, v3.16b\n" + ".inst 0x4e82a4b8 // smmla v24.4s, v5.16b, v2.16b\n" + ".inst 0x4e83a49b // smmla v27.4s, v4.16b, v3.16b\n" + ".inst 0x4e82a49e // smmla v30.4s, v4.16b, v2.16b\n" + ".inst 0x4e81a4ea // smmla v10.4s, v7.16b, v1.16b\n" + ".inst 0x4e80a4ed // smmla v13.4s, v7.16b, v0.16b\n" + ".inst 0x4e81a4d0 // smmla v16.4s, v6.16b, v1.16b\n" + ".inst 0x4e80a4d3 // smmla v19.4s, v6.16b, v0.16b\n" + ".inst 0x4e81a4b6 // smmla v22.4s, v5.16b, v1.16b\n" + ".inst 0x4e80a4b9 // smmla v25.4s, v5.16b, v0.16b\n" + ".inst 0x4e81a49c // smmla v28.4s, v4.16b, v1.16b\n" + ".inst 0x4e80a49f // smmla v31.4s, v4.16b, v0.16b\n" "5:" // multiply loop done "subs x23, x23, #0x1\n" - "uzp1 v4.2d, v8.2d, v11.2d\n" + "uzp1 v0.2d, v8.2d, v11.2d\n" "uzp2 v8.2d, v8.2d, v11.2d\n" - "uzp1 v11.2d, v9.2d, v12.2d\n" + "uzp1 v1.2d, v9.2d, v12.2d\n" "uzp2 v9.2d, v9.2d, v12.2d\n" - "str q4, [%x[Cpanel], #0x0]\n" - "uzp1 v12.2d, v10.2d, v13.2d\n" + "str q0, [%x[Cpanel], #0x0]\n" + "uzp1 v0.2d, v10.2d, v13.2d\n" "uzp2 v10.2d, v10.2d, v13.2d\n" - "str q11, [%x[Cpanel], #0x10]\n" - "str q12, [%x[Cpanel], #0x20]\n" - "uzp1 v13.2d, v14.2d, v17.2d\n" + "str q1, [%x[Cpanel], #0x10]\n" + "str q0, [%x[Cpanel], #0x20]\n" + "uzp1 v0.2d, v14.2d, v17.2d\n" "uzp2 v14.2d, v14.2d, v17.2d\n" "str q8, [%x[Cpanel], #0x30]\n" - "uzp1 v17.2d, v15.2d, v18.2d\n" + "uzp1 v2.2d, v15.2d, v18.2d\n" "uzp2 v15.2d, v15.2d, v18.2d\n" "str q9, [%x[Cpanel], #0x40]\n" - "uzp1 v18.2d, v16.2d, v19.2d\n" + "uzp1 v17.2d, v16.2d, v19.2d\n" "uzp2 v16.2d, v16.2d, v19.2d\n" "str q10, [%x[Cpanel], #0x50]\n" - "uzp1 v19.2d, v20.2d, v23.2d\n" + "uzp1 v1.2d, v20.2d, v23.2d\n" "uzp2 v20.2d, v20.2d, v23.2d\n" - "str q13, [%x[Cpanel], #0x60]\n" - "uzp1 v23.2d, v21.2d, v24.2d\n" + "str q0, [%x[Cpanel], #0x60]\n" + "uzp1 v0.2d, v21.2d, v24.2d\n" "uzp2 v21.2d, v21.2d, v24.2d\n" - "str q17, [%x[Cpanel], #0x70]\n" - "uzp1 v24.2d, v22.2d, v25.2d\n" + "str q2, [%x[Cpanel], #0x70]\n" + "uzp1 v23.2d, v22.2d, v25.2d\n" "uzp2 v22.2d, v22.2d, v25.2d\n" - "str q18, [%x[Cpanel], #0x80]\n" - "uzp1 v25.2d, v26.2d, v29.2d\n" + "str q17, [%x[Cpanel], #0x80]\n" + "uzp1 v19.2d, v26.2d, v29.2d\n" "uzp2 v26.2d, v26.2d, v29.2d\n" "str q14, [%x[Cpanel], #0x90]\n" - "uzp1 v29.2d, v27.2d, v30.2d\n" + "uzp1 v18.2d, v27.2d, v30.2d\n" "uzp2 v27.2d, v27.2d, v30.2d\n" "str q15, [%x[Cpanel], #0xa0]\n" - "uzp1 v30.2d, v28.2d, v31.2d\n" + "uzp1 v17.2d, v28.2d, v31.2d\n" "uzp2 v28.2d, v28.2d, v31.2d\n" "str q16, [%x[Cpanel], #0xb0]\n" - "str q19, [%x[Cpanel], #0xc0]\n" - "str q23, [%x[Cpanel], #0xd0]\n" - "str q24, [%x[Cpanel], #0xe0]\n" + "str q1, [%x[Cpanel], #0xc0]\n" + "str q0, [%x[Cpanel], #0xd0]\n" + "str q23, [%x[Cpanel], #0xe0]\n" "str q20, [%x[Cpanel], #0xf0]\n" "str q21, [%x[Cpanel], #0x100]\n" "str q22, [%x[Cpanel], #0x110]\n" - "str q25, [%x[Cpanel], #0x120]\n" - "str q29, [%x[Cpanel], #0x130]\n" - "str q30, [%x[Cpanel], #0x140]\n" + "str q19, [%x[Cpanel], #0x120]\n" + "str q18, [%x[Cpanel], #0x130]\n" + "str q17, [%x[Cpanel], #0x140]\n" "str q26, [%x[Cpanel], #0x150]\n" "str q27, [%x[Cpanel], #0x160]\n" "str q28, [%x[Cpanel], #0x170]\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12/generic.cpp index fc20c2fc9d877742e5acde2fcca47604bd0a63c7..a097dc358a52bcc887a3c45f7313fc17d57627a1 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_8x12/generic.cpp @@ -29,8 +29,12 @@ namespace arm_gemm { void a64_interleaved_s8s32_mmla_8x12( - const int8_t *Apanel, const int8_t *Bpanel, - int32_t *Cpanel, int ablocks, int bblocks, int K) { + const int8_t *Apanel, + const int8_t *Bpanel, + int32_t *Cpanel, + int ablocks, + int bblocks, + int K) { struct KernelArgs { size_t K = {}; @@ -43,7 +47,6 @@ void a64_interleaved_s8s32_mmla_8x12( ka.bblocks = bblocks; __asm__ __volatile__( - "1:" // Height loop "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n" "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n" @@ -85,31 +88,31 @@ void a64_interleaved_s8s32_mmla_8x12( "movi v31.4s, #0x0\n" "blt 4f\n" "3:" // main loop head - "ldr q3, [%x[Apanel], #0x0]\n" - "ldr q6, [x22, #0x0]\n" + "ldr q6, [%x[Apanel], #0x0]\n" + "ldr q7, [x22, #0x0]\n" ".inst 0x4e84a408 // smmla v8.4s, v0.16b, v4.16b\n" - "ldr q7, [x22, #0x10]\n" + "ldr q3, [x22, #0x10]\n" ".inst 0x4e85a40b // smmla v11.4s, v0.16b, v5.16b\n" ".inst 0x4e84a42e // smmla v14.4s, v1.16b, v4.16b\n" ".inst 0x4e85a431 // smmla v17.4s, v1.16b, v5.16b\n" ".inst 0x4e84a454 // smmla v20.4s, v2.16b, v4.16b\n" "sub x20, x20, #0x2\n" ".inst 0x4e85a457 // smmla v23.4s, v2.16b, v5.16b\n" - ".inst 0x4e84a47a // smmla v26.4s, v3.16b, v4.16b\n" + ".inst 0x4e84a4da // smmla v26.4s, v6.16b, v4.16b\n" "ldr q4, [x22, #0x20]\n" - ".inst 0x4e85a47d // smmla v29.4s, v3.16b, v5.16b\n" + ".inst 0x4e85a4dd // smmla v29.4s, v6.16b, v5.16b\n" "ldr q5, [x22, #0x30]\n" - ".inst 0x4e86a409 // smmla v9.4s, v0.16b, v6.16b\n" - ".inst 0x4e87a40c // smmla v12.4s, v0.16b, v7.16b\n" - ".inst 0x4e86a42f // smmla v15.4s, v1.16b, v6.16b\n" + ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n" + ".inst 0x4e83a40c // smmla v12.4s, v0.16b, v3.16b\n" + ".inst 0x4e87a42f // smmla v15.4s, v1.16b, v7.16b\n" "cmp x20, #0x2\n" - ".inst 0x4e87a432 // smmla v18.4s, v1.16b, v7.16b\n" - ".inst 0x4e86a455 // smmla v21.4s, v2.16b, v6.16b\n" - ".inst 0x4e87a458 // smmla v24.4s, v2.16b, v7.16b\n" - ".inst 0x4e86a47b // smmla v27.4s, v3.16b, v6.16b\n" - "ldr q6, [x22, #0x40]\n" - ".inst 0x4e87a47e // smmla v30.4s, v3.16b, v7.16b\n" - "ldr q7, [x22, #0x50]\n" + ".inst 0x4e83a432 // smmla v18.4s, v1.16b, v3.16b\n" + ".inst 0x4e87a455 // smmla v21.4s, v2.16b, v7.16b\n" + ".inst 0x4e83a458 // smmla v24.4s, v2.16b, v3.16b\n" + ".inst 0x4e87a4db // smmla v27.4s, v6.16b, v7.16b\n" + "ldr q7, [x22, #0x40]\n" + ".inst 0x4e83a4de // smmla v30.4s, v6.16b, v3.16b\n" + "ldr q3, [x22, #0x50]\n" ".inst 0x4e84a40a // smmla v10.4s, v0.16b, v4.16b\n" ".inst 0x4e85a40d // smmla v13.4s, v0.16b, v5.16b\n" "ldr q0, [%x[Apanel], #0x10]\n" @@ -119,42 +122,42 @@ void a64_interleaved_s8s32_mmla_8x12( ".inst 0x4e84a456 // smmla v22.4s, v2.16b, v4.16b\n" ".inst 0x4e85a459 // smmla v25.4s, v2.16b, v5.16b\n" "ldr q2, [%x[Apanel], #0x30]\n" - ".inst 0x4e84a47c // smmla v28.4s, v3.16b, v4.16b\n" + ".inst 0x4e84a4dc // smmla v28.4s, v6.16b, v4.16b\n" "ldr q4, [x22, #0x60]\n" - ".inst 0x4e85a47f // smmla v31.4s, v3.16b, v5.16b\n" - "ldr q3, [%x[Apanel], #0x40]\n" + ".inst 0x4e85a4df // smmla v31.4s, v6.16b, v5.16b\n" + "ldr q6, [%x[Apanel], #0x40]\n" "ldr q5, [x22, #0x70]\n" - ".inst 0x4e86a408 // smmla v8.4s, v0.16b, v6.16b\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - ".inst 0x4e87a431 // smmla v17.4s, v1.16b, v7.16b\n" - ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n" - ".inst 0x4e87a457 // smmla v23.4s, v2.16b, v7.16b\n" - ".inst 0x4e86a47a // smmla v26.4s, v3.16b, v6.16b\n" - "ldr q6, [x22, #0x80]\n" - ".inst 0x4e87a47d // smmla v29.4s, v3.16b, v7.16b\n" - "ldr q7, [x22, #0x90]\n" + ".inst 0x4e87a408 // smmla v8.4s, v0.16b, v7.16b\n" + ".inst 0x4e83a40b // smmla v11.4s, v0.16b, v3.16b\n" + ".inst 0x4e87a42e // smmla v14.4s, v1.16b, v7.16b\n" + ".inst 0x4e83a431 // smmla v17.4s, v1.16b, v3.16b\n" + ".inst 0x4e87a454 // smmla v20.4s, v2.16b, v7.16b\n" + ".inst 0x4e83a457 // smmla v23.4s, v2.16b, v3.16b\n" + ".inst 0x4e87a4da // smmla v26.4s, v6.16b, v7.16b\n" + "ldr q7, [x22, #0x80]\n" + ".inst 0x4e83a4dd // smmla v29.4s, v6.16b, v3.16b\n" + "ldr q3, [x22, #0x90]\n" ".inst 0x4e84a409 // smmla v9.4s, v0.16b, v4.16b\n" ".inst 0x4e85a40c // smmla v12.4s, v0.16b, v5.16b\n" ".inst 0x4e84a42f // smmla v15.4s, v1.16b, v4.16b\n" ".inst 0x4e85a432 // smmla v18.4s, v1.16b, v5.16b\n" ".inst 0x4e84a455 // smmla v21.4s, v2.16b, v4.16b\n" ".inst 0x4e85a458 // smmla v24.4s, v2.16b, v5.16b\n" - ".inst 0x4e84a47b // smmla v27.4s, v3.16b, v4.16b\n" + ".inst 0x4e84a4db // smmla v27.4s, v6.16b, v4.16b\n" "ldr q4, [x22, #0xa0]\n" - ".inst 0x4e85a47e // smmla v30.4s, v3.16b, v5.16b\n" + ".inst 0x4e85a4de // smmla v30.4s, v6.16b, v5.16b\n" "ldr q5, [x22, #0xb0]\n" - ".inst 0x4e86a40a // smmla v10.4s, v0.16b, v6.16b\n" - ".inst 0x4e87a40d // smmla v13.4s, v0.16b, v7.16b\n" + ".inst 0x4e87a40a // smmla v10.4s, v0.16b, v7.16b\n" + ".inst 0x4e83a40d // smmla v13.4s, v0.16b, v3.16b\n" "ldr q0, [%x[Apanel], #0x50]\n" - ".inst 0x4e86a430 // smmla v16.4s, v1.16b, v6.16b\n" - ".inst 0x4e87a433 // smmla v19.4s, v1.16b, v7.16b\n" + ".inst 0x4e87a430 // smmla v16.4s, v1.16b, v7.16b\n" + ".inst 0x4e83a433 // smmla v19.4s, v1.16b, v3.16b\n" "ldr q1, [%x[Apanel], #0x60]\n" - ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n" - ".inst 0x4e87a459 // smmla v25.4s, v2.16b, v7.16b\n" + ".inst 0x4e87a456 // smmla v22.4s, v2.16b, v7.16b\n" + ".inst 0x4e83a459 // smmla v25.4s, v2.16b, v3.16b\n" "ldr q2, [%x[Apanel], #0x70]\n" - ".inst 0x4e86a47c // smmla v28.4s, v3.16b, v6.16b\n" - ".inst 0x4e87a47f // smmla v31.4s, v3.16b, v7.16b\n" + ".inst 0x4e87a4dc // smmla v28.4s, v6.16b, v7.16b\n" + ".inst 0x4e83a4df // smmla v31.4s, v6.16b, v3.16b\n" "add %x[Apanel], %x[Apanel], #0x80\n" "add x22, x22, #0xc0\n" "bge 3b\n" @@ -191,89 +194,89 @@ void a64_interleaved_s8s32_mmla_8x12( ".inst 0x4e84a47c // smmla v28.4s, v3.16b, v4.16b\n" ".inst 0x4e85a47f // smmla v31.4s, v3.16b, v5.16b\n" "cbz x20, 5f\n" - "ldr q6, [x22, #0x0]\n" - "ldr q0, [%x[Apanel], #0x0]\n" - ".inst 0x4e86a408 // smmla v8.4s, v0.16b, v6.16b\n" - "ldr q1, [%x[Apanel], #0x10]\n" - "ldr q7, [x22, #0x10]\n" - ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n" - "ldr q2, [%x[Apanel], #0x20]\n" - "ldr q3, [%x[Apanel], #0x30]\n" - ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n" - "ldr q4, [x22, #0x20]\n" - "ldr q5, [x22, #0x30]\n" - ".inst 0x4e87a431 // smmla v17.4s, v1.16b, v7.16b\n" - ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n" - ".inst 0x4e87a457 // smmla v23.4s, v2.16b, v7.16b\n" + "ldr q1, [x22, #0x0]\n" + "ldr q7, [%x[Apanel], #0x0]\n" + ".inst 0x4e81a4e8 // smmla v8.4s, v7.16b, v1.16b\n" + "ldr q6, [%x[Apanel], #0x10]\n" + "ldr q0, [x22, #0x10]\n" + ".inst 0x4e80a4eb // smmla v11.4s, v7.16b, v0.16b\n" + "ldr q5, [%x[Apanel], #0x20]\n" + "ldr q4, [%x[Apanel], #0x30]\n" + ".inst 0x4e81a4ce // smmla v14.4s, v6.16b, v1.16b\n" + "ldr q3, [x22, #0x20]\n" + "ldr q2, [x22, #0x30]\n" + ".inst 0x4e80a4d1 // smmla v17.4s, v6.16b, v0.16b\n" + ".inst 0x4e81a4b4 // smmla v20.4s, v5.16b, v1.16b\n" + ".inst 0x4e80a4b7 // smmla v23.4s, v5.16b, v0.16b\n" "add %x[Apanel], %x[Apanel], #0x40\n" - ".inst 0x4e86a47a // smmla v26.4s, v3.16b, v6.16b\n" - "ldr q6, [x22, #0x40]\n" - ".inst 0x4e87a47d // smmla v29.4s, v3.16b, v7.16b\n" - "ldr q7, [x22, #0x50]\n" - ".inst 0x4e84a409 // smmla v9.4s, v0.16b, v4.16b\n" - ".inst 0x4e85a40c // smmla v12.4s, v0.16b, v5.16b\n" - ".inst 0x4e84a42f // smmla v15.4s, v1.16b, v4.16b\n" - ".inst 0x4e85a432 // smmla v18.4s, v1.16b, v5.16b\n" + ".inst 0x4e81a49a // smmla v26.4s, v4.16b, v1.16b\n" + "ldr q1, [x22, #0x40]\n" + ".inst 0x4e80a49d // smmla v29.4s, v4.16b, v0.16b\n" + "ldr q0, [x22, #0x50]\n" + ".inst 0x4e83a4e9 // smmla v9.4s, v7.16b, v3.16b\n" + ".inst 0x4e82a4ec // smmla v12.4s, v7.16b, v2.16b\n" + ".inst 0x4e83a4cf // smmla v15.4s, v6.16b, v3.16b\n" + ".inst 0x4e82a4d2 // smmla v18.4s, v6.16b, v2.16b\n" "add x22, x22, #0x60\n" - ".inst 0x4e84a455 // smmla v21.4s, v2.16b, v4.16b\n" - ".inst 0x4e85a458 // smmla v24.4s, v2.16b, v5.16b\n" - ".inst 0x4e84a47b // smmla v27.4s, v3.16b, v4.16b\n" - ".inst 0x4e85a47e // smmla v30.4s, v3.16b, v5.16b\n" - ".inst 0x4e86a40a // smmla v10.4s, v0.16b, v6.16b\n" - ".inst 0x4e87a40d // smmla v13.4s, v0.16b, v7.16b\n" - ".inst 0x4e86a430 // smmla v16.4s, v1.16b, v6.16b\n" - ".inst 0x4e87a433 // smmla v19.4s, v1.16b, v7.16b\n" - ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n" - ".inst 0x4e87a459 // smmla v25.4s, v2.16b, v7.16b\n" - ".inst 0x4e86a47c // smmla v28.4s, v3.16b, v6.16b\n" - ".inst 0x4e87a47f // smmla v31.4s, v3.16b, v7.16b\n" + ".inst 0x4e83a4b5 // smmla v21.4s, v5.16b, v3.16b\n" + ".inst 0x4e82a4b8 // smmla v24.4s, v5.16b, v2.16b\n" + ".inst 0x4e83a49b // smmla v27.4s, v4.16b, v3.16b\n" + ".inst 0x4e82a49e // smmla v30.4s, v4.16b, v2.16b\n" + ".inst 0x4e81a4ea // smmla v10.4s, v7.16b, v1.16b\n" + ".inst 0x4e80a4ed // smmla v13.4s, v7.16b, v0.16b\n" + ".inst 0x4e81a4d0 // smmla v16.4s, v6.16b, v1.16b\n" + ".inst 0x4e80a4d3 // smmla v19.4s, v6.16b, v0.16b\n" + ".inst 0x4e81a4b6 // smmla v22.4s, v5.16b, v1.16b\n" + ".inst 0x4e80a4b9 // smmla v25.4s, v5.16b, v0.16b\n" + ".inst 0x4e81a49c // smmla v28.4s, v4.16b, v1.16b\n" + ".inst 0x4e80a49f // smmla v31.4s, v4.16b, v0.16b\n" "5:" // multiply loop done "subs x23, x23, #0x1\n" - "uzp1 v4.2d, v8.2d, v11.2d\n" + "uzp1 v0.2d, v8.2d, v11.2d\n" "uzp2 v8.2d, v8.2d, v11.2d\n" - "uzp1 v11.2d, v9.2d, v12.2d\n" + "uzp1 v1.2d, v9.2d, v12.2d\n" "uzp2 v9.2d, v9.2d, v12.2d\n" - "str q4, [%x[Cpanel], #0x0]\n" - "uzp1 v12.2d, v10.2d, v13.2d\n" + "str q0, [%x[Cpanel], #0x0]\n" + "uzp1 v0.2d, v10.2d, v13.2d\n" "uzp2 v10.2d, v10.2d, v13.2d\n" - "str q11, [%x[Cpanel], #0x10]\n" - "str q12, [%x[Cpanel], #0x20]\n" - "uzp1 v13.2d, v14.2d, v17.2d\n" + "str q1, [%x[Cpanel], #0x10]\n" + "str q0, [%x[Cpanel], #0x20]\n" + "uzp1 v0.2d, v14.2d, v17.2d\n" "uzp2 v14.2d, v14.2d, v17.2d\n" "str q8, [%x[Cpanel], #0x30]\n" - "uzp1 v17.2d, v15.2d, v18.2d\n" + "uzp1 v2.2d, v15.2d, v18.2d\n" "uzp2 v15.2d, v15.2d, v18.2d\n" "str q9, [%x[Cpanel], #0x40]\n" - "uzp1 v18.2d, v16.2d, v19.2d\n" + "uzp1 v17.2d, v16.2d, v19.2d\n" "uzp2 v16.2d, v16.2d, v19.2d\n" "str q10, [%x[Cpanel], #0x50]\n" - "uzp1 v19.2d, v20.2d, v23.2d\n" + "uzp1 v1.2d, v20.2d, v23.2d\n" "uzp2 v20.2d, v20.2d, v23.2d\n" - "str q13, [%x[Cpanel], #0x60]\n" - "uzp1 v23.2d, v21.2d, v24.2d\n" + "str q0, [%x[Cpanel], #0x60]\n" + "uzp1 v0.2d, v21.2d, v24.2d\n" "uzp2 v21.2d, v21.2d, v24.2d\n" - "str q17, [%x[Cpanel], #0x70]\n" - "uzp1 v24.2d, v22.2d, v25.2d\n" + "str q2, [%x[Cpanel], #0x70]\n" + "uzp1 v23.2d, v22.2d, v25.2d\n" "uzp2 v22.2d, v22.2d, v25.2d\n" - "str q18, [%x[Cpanel], #0x80]\n" - "uzp1 v25.2d, v26.2d, v29.2d\n" + "str q17, [%x[Cpanel], #0x80]\n" + "uzp1 v19.2d, v26.2d, v29.2d\n" "uzp2 v26.2d, v26.2d, v29.2d\n" "str q14, [%x[Cpanel], #0x90]\n" - "uzp1 v29.2d, v27.2d, v30.2d\n" + "uzp1 v18.2d, v27.2d, v30.2d\n" "uzp2 v27.2d, v27.2d, v30.2d\n" "str q15, [%x[Cpanel], #0xa0]\n" - "uzp1 v30.2d, v28.2d, v31.2d\n" + "uzp1 v17.2d, v28.2d, v31.2d\n" "uzp2 v28.2d, v28.2d, v31.2d\n" "str q16, [%x[Cpanel], #0xb0]\n" - "str q19, [%x[Cpanel], #0xc0]\n" - "str q23, [%x[Cpanel], #0xd0]\n" - "str q24, [%x[Cpanel], #0xe0]\n" + "str q1, [%x[Cpanel], #0xc0]\n" + "str q0, [%x[Cpanel], #0xd0]\n" + "str q23, [%x[Cpanel], #0xe0]\n" "str q20, [%x[Cpanel], #0xf0]\n" "str q21, [%x[Cpanel], #0x100]\n" "str q22, [%x[Cpanel], #0x110]\n" - "str q25, [%x[Cpanel], #0x120]\n" - "str q29, [%x[Cpanel], #0x130]\n" - "str q30, [%x[Cpanel], #0x140]\n" + "str q19, [%x[Cpanel], #0x120]\n" + "str q18, [%x[Cpanel], #0x130]\n" + "str q17, [%x[Cpanel], #0x140]\n" "str q26, [%x[Cpanel], #0x150]\n" "str q27, [%x[Cpanel], #0x160]\n" "str q28, [%x[Cpanel], #0x170]\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12.hpp index fa93c1d90d238ebd274717c37c65e89496af456e..0088557b8dba27afe6872901dc02f33b2a359c31 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef __aarch64__ + #include "../std_transforms_fixed.hpp" #include "../performance_parameters.hpp" @@ -56,11 +56,6 @@ public: return 12; } - static unsigned int stripe_width() - { - return 4; - } - static constexpr unsigned int k_unroll() { return 8; @@ -116,5 +111,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12/a510.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12/a510.cpp index 83301d80bb535c41cdb7df68536754a7a87ceff7..54c51954c8ac0547ea86c5187ae5380db1d853c5 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12/a510.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12/a510.cpp @@ -29,8 +29,12 @@ namespace arm_gemm { void a64_interleaved_u8u32_mmla_8x12_a510( - const uint8_t *Apanel, const uint8_t *Bpanel, - uint32_t *Cpanel, int ablocks, int bblocks, int K) { + const uint8_t *Apanel, + const uint8_t *Bpanel, + uint32_t *Cpanel, + int ablocks, + int bblocks, + int K) { struct KernelArgs { size_t K = {}; @@ -43,7 +47,6 @@ void a64_interleaved_u8u32_mmla_8x12_a510( ka.bblocks = bblocks; __asm__ __volatile__( - "1:" // Height loop "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n" "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n" @@ -82,28 +85,28 @@ void a64_interleaved_u8u32_mmla_8x12_a510( "movi v31.4s, #0x0\n" "blt 4f\n" "3:" // main loop head - "ld1 { v3.16b }, [%x[Apanel]], #0x10\n" + "ld1 { v6.16b }, [%x[Apanel]], #0x10\n" ".inst 0x6e84a408 // ummla v8.4s, v0.16b, v4.16b\n" ".inst 0x6e85a40b // ummla v11.4s, v0.16b, v5.16b\n" - "ldp q6, q7, [x22], #0x20\n" + "ldp q3, q7, [x22], #0x20\n" ".inst 0x6e84a42e // ummla v14.4s, v1.16b, v4.16b\n" ".inst 0x6e85a431 // ummla v17.4s, v1.16b, v5.16b\n" ".inst 0x6e84a454 // ummla v20.4s, v2.16b, v4.16b\n" "sub x20, x20, #0x2\n" ".inst 0x6e85a457 // ummla v23.4s, v2.16b, v5.16b\n" - ".inst 0x6e84a47a // ummla v26.4s, v3.16b, v4.16b\n" + ".inst 0x6e84a4da // ummla v26.4s, v6.16b, v4.16b\n" "cmp x20, #0x2\n" - ".inst 0x6e85a47d // ummla v29.4s, v3.16b, v5.16b\n" + ".inst 0x6e85a4dd // ummla v29.4s, v6.16b, v5.16b\n" "ldp q4, q5, [x22], #0x20\n" - ".inst 0x6e86a409 // ummla v9.4s, v0.16b, v6.16b\n" + ".inst 0x6e83a409 // ummla v9.4s, v0.16b, v3.16b\n" ".inst 0x6e87a40c // ummla v12.4s, v0.16b, v7.16b\n" - ".inst 0x6e86a42f // ummla v15.4s, v1.16b, v6.16b\n" + ".inst 0x6e83a42f // ummla v15.4s, v1.16b, v3.16b\n" ".inst 0x6e87a432 // ummla v18.4s, v1.16b, v7.16b\n" - ".inst 0x6e86a455 // ummla v21.4s, v2.16b, v6.16b\n" + ".inst 0x6e83a455 // ummla v21.4s, v2.16b, v3.16b\n" ".inst 0x6e87a458 // ummla v24.4s, v2.16b, v7.16b\n" - ".inst 0x6e86a47b // ummla v27.4s, v3.16b, v6.16b\n" - ".inst 0x6e87a47e // ummla v30.4s, v3.16b, v7.16b\n" - "ldp q6, q7, [x22], #0x20\n" + ".inst 0x6e83a4db // ummla v27.4s, v6.16b, v3.16b\n" + ".inst 0x6e87a4de // ummla v30.4s, v6.16b, v7.16b\n" + "ldp q7, q3, [x22], #0x20\n" ".inst 0x6e84a40a // ummla v10.4s, v0.16b, v4.16b\n" ".inst 0x6e85a40d // ummla v13.4s, v0.16b, v5.16b\n" "ld1 { v0.16b }, [%x[Apanel]], #0x10\n" @@ -113,39 +116,39 @@ void a64_interleaved_u8u32_mmla_8x12_a510( ".inst 0x6e84a456 // ummla v22.4s, v2.16b, v4.16b\n" ".inst 0x6e85a459 // ummla v25.4s, v2.16b, v5.16b\n" "ld1 { v2.16b }, [%x[Apanel]], #0x10\n" - ".inst 0x6e84a47c // ummla v28.4s, v3.16b, v4.16b\n" - ".inst 0x6e85a47f // ummla v31.4s, v3.16b, v5.16b\n" - "ld1 { v3.16b }, [%x[Apanel]], #0x10\n" - ".inst 0x6e86a408 // ummla v8.4s, v0.16b, v6.16b\n" + ".inst 0x6e84a4dc // ummla v28.4s, v6.16b, v4.16b\n" + ".inst 0x6e85a4df // ummla v31.4s, v6.16b, v5.16b\n" + "ld1 { v6.16b }, [%x[Apanel]], #0x10\n" + ".inst 0x6e87a408 // ummla v8.4s, v0.16b, v7.16b\n" "ldp q4, q5, [x22], #0x20\n" - ".inst 0x6e87a40b // ummla v11.4s, v0.16b, v7.16b\n" - ".inst 0x6e86a42e // ummla v14.4s, v1.16b, v6.16b\n" - ".inst 0x6e87a431 // ummla v17.4s, v1.16b, v7.16b\n" - ".inst 0x6e86a454 // ummla v20.4s, v2.16b, v6.16b\n" - ".inst 0x6e87a457 // ummla v23.4s, v2.16b, v7.16b\n" - ".inst 0x6e86a47a // ummla v26.4s, v3.16b, v6.16b\n" - ".inst 0x6e87a47d // ummla v29.4s, v3.16b, v7.16b\n" - "ldp q6, q7, [x22], #0x20\n" + ".inst 0x6e83a40b // ummla v11.4s, v0.16b, v3.16b\n" + ".inst 0x6e87a42e // ummla v14.4s, v1.16b, v7.16b\n" + ".inst 0x6e83a431 // ummla v17.4s, v1.16b, v3.16b\n" + ".inst 0x6e87a454 // ummla v20.4s, v2.16b, v7.16b\n" + ".inst 0x6e83a457 // ummla v23.4s, v2.16b, v3.16b\n" + ".inst 0x6e87a4da // ummla v26.4s, v6.16b, v7.16b\n" + ".inst 0x6e83a4dd // ummla v29.4s, v6.16b, v3.16b\n" + "ldp q7, q3, [x22], #0x20\n" ".inst 0x6e84a409 // ummla v9.4s, v0.16b, v4.16b\n" ".inst 0x6e85a40c // ummla v12.4s, v0.16b, v5.16b\n" ".inst 0x6e84a42f // ummla v15.4s, v1.16b, v4.16b\n" ".inst 0x6e85a432 // ummla v18.4s, v1.16b, v5.16b\n" ".inst 0x6e84a455 // ummla v21.4s, v2.16b, v4.16b\n" ".inst 0x6e85a458 // ummla v24.4s, v2.16b, v5.16b\n" - ".inst 0x6e84a47b // ummla v27.4s, v3.16b, v4.16b\n" - ".inst 0x6e85a47e // ummla v30.4s, v3.16b, v5.16b\n" + ".inst 0x6e84a4db // ummla v27.4s, v6.16b, v4.16b\n" + ".inst 0x6e85a4de // ummla v30.4s, v6.16b, v5.16b\n" "ldp q4, q5, [x22], #0x20\n" - ".inst 0x6e86a40a // ummla v10.4s, v0.16b, v6.16b\n" - ".inst 0x6e87a40d // ummla v13.4s, v0.16b, v7.16b\n" + ".inst 0x6e87a40a // ummla v10.4s, v0.16b, v7.16b\n" + ".inst 0x6e83a40d // ummla v13.4s, v0.16b, v3.16b\n" "ld1 { v0.16b }, [%x[Apanel]], #0x10\n" - ".inst 0x6e86a430 // ummla v16.4s, v1.16b, v6.16b\n" - ".inst 0x6e87a433 // ummla v19.4s, v1.16b, v7.16b\n" + ".inst 0x6e87a430 // ummla v16.4s, v1.16b, v7.16b\n" + ".inst 0x6e83a433 // ummla v19.4s, v1.16b, v3.16b\n" "ld1 { v1.16b }, [%x[Apanel]], #0x10\n" - ".inst 0x6e86a456 // ummla v22.4s, v2.16b, v6.16b\n" - ".inst 0x6e87a459 // ummla v25.4s, v2.16b, v7.16b\n" + ".inst 0x6e87a456 // ummla v22.4s, v2.16b, v7.16b\n" + ".inst 0x6e83a459 // ummla v25.4s, v2.16b, v3.16b\n" "ld1 { v2.16b }, [%x[Apanel]], #0x10\n" - ".inst 0x6e86a47c // ummla v28.4s, v3.16b, v6.16b\n" - ".inst 0x6e87a47f // ummla v31.4s, v3.16b, v7.16b\n" + ".inst 0x6e87a4dc // ummla v28.4s, v6.16b, v7.16b\n" + ".inst 0x6e83a4df // ummla v31.4s, v6.16b, v3.16b\n" "bge 3b\n" "4:" // main loop skip "ld1 { v3.16b }, [%x[Apanel]], #0x10\n" @@ -158,7 +161,7 @@ void a64_interleaved_u8u32_mmla_8x12_a510( ".inst 0x6e85a457 // ummla v23.4s, v2.16b, v5.16b\n" ".inst 0x6e84a47a // ummla v26.4s, v3.16b, v4.16b\n" ".inst 0x6e85a47d // ummla v29.4s, v3.16b, v5.16b\n" - "ldp q4, q5, [x22], #0x20\n" + "ldp q5, q4, [x22], #0x20\n" ".inst 0x6e86a409 // ummla v9.4s, v0.16b, v6.16b\n" ".inst 0x6e87a40c // ummla v12.4s, v0.16b, v7.16b\n" ".inst 0x6e86a42f // ummla v15.4s, v1.16b, v6.16b\n" @@ -167,93 +170,93 @@ void a64_interleaved_u8u32_mmla_8x12_a510( ".inst 0x6e87a458 // ummla v24.4s, v2.16b, v7.16b\n" ".inst 0x6e86a47b // ummla v27.4s, v3.16b, v6.16b\n" ".inst 0x6e87a47e // ummla v30.4s, v3.16b, v7.16b\n" - ".inst 0x6e84a40a // ummla v10.4s, v0.16b, v4.16b\n" - ".inst 0x6e85a40d // ummla v13.4s, v0.16b, v5.16b\n" - ".inst 0x6e84a430 // ummla v16.4s, v1.16b, v4.16b\n" - ".inst 0x6e85a433 // ummla v19.4s, v1.16b, v5.16b\n" - ".inst 0x6e84a456 // ummla v22.4s, v2.16b, v4.16b\n" - ".inst 0x6e85a459 // ummla v25.4s, v2.16b, v5.16b\n" - ".inst 0x6e84a47c // ummla v28.4s, v3.16b, v4.16b\n" - ".inst 0x6e85a47f // ummla v31.4s, v3.16b, v5.16b\n" + ".inst 0x6e85a40a // ummla v10.4s, v0.16b, v5.16b\n" + ".inst 0x6e84a40d // ummla v13.4s, v0.16b, v4.16b\n" + ".inst 0x6e85a430 // ummla v16.4s, v1.16b, v5.16b\n" + ".inst 0x6e84a433 // ummla v19.4s, v1.16b, v4.16b\n" + ".inst 0x6e85a456 // ummla v22.4s, v2.16b, v5.16b\n" + ".inst 0x6e84a459 // ummla v25.4s, v2.16b, v4.16b\n" + ".inst 0x6e85a47c // ummla v28.4s, v3.16b, v5.16b\n" + ".inst 0x6e84a47f // ummla v31.4s, v3.16b, v4.16b\n" "cbz x20, 5f\n" - "ldp q6, q7, [x22], #0x20\n" - "ld1 { v0.16b }, [%x[Apanel]], #0x10\n" - "ld1 { v1.16b }, [%x[Apanel]], #0x10\n" - ".inst 0x6e86a408 // ummla v8.4s, v0.16b, v6.16b\n" - "ld1 { v2.16b }, [%x[Apanel]], #0x10\n" - "ld1 { v3.16b }, [%x[Apanel]], #0x10\n" - ".inst 0x6e87a40b // ummla v11.4s, v0.16b, v7.16b\n" - "ldp q4, q5, [x22], #0x20\n" - ".inst 0x6e86a42e // ummla v14.4s, v1.16b, v6.16b\n" - ".inst 0x6e87a431 // ummla v17.4s, v1.16b, v7.16b\n" - ".inst 0x6e86a454 // ummla v20.4s, v2.16b, v6.16b\n" - ".inst 0x6e87a457 // ummla v23.4s, v2.16b, v7.16b\n" - ".inst 0x6e86a47a // ummla v26.4s, v3.16b, v6.16b\n" - ".inst 0x6e87a47d // ummla v29.4s, v3.16b, v7.16b\n" - "ldp q6, q7, [x22], #0x20\n" - ".inst 0x6e84a409 // ummla v9.4s, v0.16b, v4.16b\n" - ".inst 0x6e85a40c // ummla v12.4s, v0.16b, v5.16b\n" - ".inst 0x6e84a42f // ummla v15.4s, v1.16b, v4.16b\n" - ".inst 0x6e85a432 // ummla v18.4s, v1.16b, v5.16b\n" - ".inst 0x6e84a455 // ummla v21.4s, v2.16b, v4.16b\n" - ".inst 0x6e85a458 // ummla v24.4s, v2.16b, v5.16b\n" - ".inst 0x6e84a47b // ummla v27.4s, v3.16b, v4.16b\n" - ".inst 0x6e85a47e // ummla v30.4s, v3.16b, v5.16b\n" - ".inst 0x6e86a40a // ummla v10.4s, v0.16b, v6.16b\n" - ".inst 0x6e87a40d // ummla v13.4s, v0.16b, v7.16b\n" - ".inst 0x6e86a430 // ummla v16.4s, v1.16b, v6.16b\n" - ".inst 0x6e87a433 // ummla v19.4s, v1.16b, v7.16b\n" - ".inst 0x6e86a456 // ummla v22.4s, v2.16b, v6.16b\n" - ".inst 0x6e87a459 // ummla v25.4s, v2.16b, v7.16b\n" - ".inst 0x6e86a47c // ummla v28.4s, v3.16b, v6.16b\n" - ".inst 0x6e87a47f // ummla v31.4s, v3.16b, v7.16b\n" + "ldp q1, q0, [x22], #0x20\n" + "ld1 { v7.16b }, [%x[Apanel]], #0x10\n" + "ld1 { v6.16b }, [%x[Apanel]], #0x10\n" + ".inst 0x6e81a4e8 // ummla v8.4s, v7.16b, v1.16b\n" + "ld1 { v5.16b }, [%x[Apanel]], #0x10\n" + "ld1 { v4.16b }, [%x[Apanel]], #0x10\n" + ".inst 0x6e80a4eb // ummla v11.4s, v7.16b, v0.16b\n" + "ldp q3, q2, [x22], #0x20\n" + ".inst 0x6e81a4ce // ummla v14.4s, v6.16b, v1.16b\n" + ".inst 0x6e80a4d1 // ummla v17.4s, v6.16b, v0.16b\n" + ".inst 0x6e81a4b4 // ummla v20.4s, v5.16b, v1.16b\n" + ".inst 0x6e80a4b7 // ummla v23.4s, v5.16b, v0.16b\n" + ".inst 0x6e81a49a // ummla v26.4s, v4.16b, v1.16b\n" + ".inst 0x6e80a49d // ummla v29.4s, v4.16b, v0.16b\n" + "ldp q1, q0, [x22], #0x20\n" + ".inst 0x6e83a4e9 // ummla v9.4s, v7.16b, v3.16b\n" + ".inst 0x6e82a4ec // ummla v12.4s, v7.16b, v2.16b\n" + ".inst 0x6e83a4cf // ummla v15.4s, v6.16b, v3.16b\n" + ".inst 0x6e82a4d2 // ummla v18.4s, v6.16b, v2.16b\n" + ".inst 0x6e83a4b5 // ummla v21.4s, v5.16b, v3.16b\n" + ".inst 0x6e82a4b8 // ummla v24.4s, v5.16b, v2.16b\n" + ".inst 0x6e83a49b // ummla v27.4s, v4.16b, v3.16b\n" + ".inst 0x6e82a49e // ummla v30.4s, v4.16b, v2.16b\n" + ".inst 0x6e81a4ea // ummla v10.4s, v7.16b, v1.16b\n" + ".inst 0x6e80a4ed // ummla v13.4s, v7.16b, v0.16b\n" + ".inst 0x6e81a4d0 // ummla v16.4s, v6.16b, v1.16b\n" + ".inst 0x6e80a4d3 // ummla v19.4s, v6.16b, v0.16b\n" + ".inst 0x6e81a4b6 // ummla v22.4s, v5.16b, v1.16b\n" + ".inst 0x6e80a4b9 // ummla v25.4s, v5.16b, v0.16b\n" + ".inst 0x6e81a49c // ummla v28.4s, v4.16b, v1.16b\n" + ".inst 0x6e80a49f // ummla v31.4s, v4.16b, v0.16b\n" "5:" // multiply loop done "subs x23, x23, #0x1\n" - "uzp1 v4.2d, v8.2d, v11.2d\n" + "uzp1 v0.2d, v8.2d, v11.2d\n" "uzp2 v8.2d, v8.2d, v11.2d\n" - "uzp1 v11.2d, v9.2d, v12.2d\n" + "uzp1 v1.2d, v9.2d, v12.2d\n" "uzp2 v9.2d, v9.2d, v12.2d\n" - "str q4, [%x[Cpanel], #0x0]\n" - "uzp1 v12.2d, v10.2d, v13.2d\n" + "str q0, [%x[Cpanel], #0x0]\n" + "uzp1 v0.2d, v10.2d, v13.2d\n" "uzp2 v10.2d, v10.2d, v13.2d\n" - "str q11, [%x[Cpanel], #0x10]\n" - "str q12, [%x[Cpanel], #0x20]\n" - "uzp1 v13.2d, v14.2d, v17.2d\n" + "str q1, [%x[Cpanel], #0x10]\n" + "str q0, [%x[Cpanel], #0x20]\n" + "uzp1 v0.2d, v14.2d, v17.2d\n" "uzp2 v14.2d, v14.2d, v17.2d\n" "str q8, [%x[Cpanel], #0x30]\n" - "uzp1 v17.2d, v15.2d, v18.2d\n" + "uzp1 v2.2d, v15.2d, v18.2d\n" "uzp2 v15.2d, v15.2d, v18.2d\n" "str q9, [%x[Cpanel], #0x40]\n" - "uzp1 v18.2d, v16.2d, v19.2d\n" + "uzp1 v17.2d, v16.2d, v19.2d\n" "uzp2 v16.2d, v16.2d, v19.2d\n" "str q10, [%x[Cpanel], #0x50]\n" - "uzp1 v19.2d, v20.2d, v23.2d\n" + "uzp1 v1.2d, v20.2d, v23.2d\n" "uzp2 v20.2d, v20.2d, v23.2d\n" - "str q13, [%x[Cpanel], #0x60]\n" - "uzp1 v23.2d, v21.2d, v24.2d\n" + "str q0, [%x[Cpanel], #0x60]\n" + "uzp1 v0.2d, v21.2d, v24.2d\n" "uzp2 v21.2d, v21.2d, v24.2d\n" - "str q17, [%x[Cpanel], #0x70]\n" - "uzp1 v24.2d, v22.2d, v25.2d\n" + "str q2, [%x[Cpanel], #0x70]\n" + "uzp1 v23.2d, v22.2d, v25.2d\n" "uzp2 v22.2d, v22.2d, v25.2d\n" - "str q18, [%x[Cpanel], #0x80]\n" - "uzp1 v25.2d, v26.2d, v29.2d\n" + "str q17, [%x[Cpanel], #0x80]\n" + "uzp1 v19.2d, v26.2d, v29.2d\n" "uzp2 v26.2d, v26.2d, v29.2d\n" "str q14, [%x[Cpanel], #0x90]\n" - "uzp1 v29.2d, v27.2d, v30.2d\n" + "uzp1 v18.2d, v27.2d, v30.2d\n" "uzp2 v27.2d, v27.2d, v30.2d\n" "str q15, [%x[Cpanel], #0xa0]\n" - "uzp1 v30.2d, v28.2d, v31.2d\n" + "uzp1 v17.2d, v28.2d, v31.2d\n" "uzp2 v28.2d, v28.2d, v31.2d\n" "str q16, [%x[Cpanel], #0xb0]\n" - "str q19, [%x[Cpanel], #0xc0]\n" - "str q23, [%x[Cpanel], #0xd0]\n" - "str q24, [%x[Cpanel], #0xe0]\n" + "str q1, [%x[Cpanel], #0xc0]\n" + "str q0, [%x[Cpanel], #0xd0]\n" + "str q23, [%x[Cpanel], #0xe0]\n" "str q20, [%x[Cpanel], #0xf0]\n" "str q21, [%x[Cpanel], #0x100]\n" "str q22, [%x[Cpanel], #0x110]\n" - "str q25, [%x[Cpanel], #0x120]\n" - "str q29, [%x[Cpanel], #0x130]\n" - "str q30, [%x[Cpanel], #0x140]\n" + "str q19, [%x[Cpanel], #0x120]\n" + "str q18, [%x[Cpanel], #0x130]\n" + "str q17, [%x[Cpanel], #0x140]\n" "str q26, [%x[Cpanel], #0x150]\n" "str q27, [%x[Cpanel], #0x160]\n" "str q28, [%x[Cpanel], #0x170]\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12/generic.cpp index c5342197c1fdc237f55d6b3288bfbae8f9cd056b..30260b9c292fdf3e4206391813fdd675a3d000e6 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_8x12/generic.cpp @@ -29,8 +29,12 @@ namespace arm_gemm { void a64_interleaved_u8u32_mmla_8x12( - const uint8_t *Apanel, const uint8_t *Bpanel, - uint32_t *Cpanel, int ablocks, int bblocks, int K) { + const uint8_t *Apanel, + const uint8_t *Bpanel, + uint32_t *Cpanel, + int ablocks, + int bblocks, + int K) { struct KernelArgs { size_t K = {}; @@ -43,7 +47,6 @@ void a64_interleaved_u8u32_mmla_8x12( ka.bblocks = bblocks; __asm__ __volatile__( - "1:" // Height loop "ldr x23, [%x[args_ptr], %[offsetof_bblocks]]\n" "ldr x22, [%x[args_ptr], %[offsetof_Bpanel]]\n" @@ -85,31 +88,31 @@ void a64_interleaved_u8u32_mmla_8x12( "movi v31.4s, #0x0\n" "blt 4f\n" "3:" // main loop head - "ldr q3, [%x[Apanel], #0x0]\n" - "ldr q6, [x22, #0x0]\n" + "ldr q6, [%x[Apanel], #0x0]\n" + "ldr q7, [x22, #0x0]\n" ".inst 0x6e84a408 // ummla v8.4s, v0.16b, v4.16b\n" - "ldr q7, [x22, #0x10]\n" + "ldr q3, [x22, #0x10]\n" ".inst 0x6e85a40b // ummla v11.4s, v0.16b, v5.16b\n" ".inst 0x6e84a42e // ummla v14.4s, v1.16b, v4.16b\n" ".inst 0x6e85a431 // ummla v17.4s, v1.16b, v5.16b\n" ".inst 0x6e84a454 // ummla v20.4s, v2.16b, v4.16b\n" "sub x20, x20, #0x2\n" ".inst 0x6e85a457 // ummla v23.4s, v2.16b, v5.16b\n" - ".inst 0x6e84a47a // ummla v26.4s, v3.16b, v4.16b\n" + ".inst 0x6e84a4da // ummla v26.4s, v6.16b, v4.16b\n" "ldr q4, [x22, #0x20]\n" - ".inst 0x6e85a47d // ummla v29.4s, v3.16b, v5.16b\n" + ".inst 0x6e85a4dd // ummla v29.4s, v6.16b, v5.16b\n" "ldr q5, [x22, #0x30]\n" - ".inst 0x6e86a409 // ummla v9.4s, v0.16b, v6.16b\n" - ".inst 0x6e87a40c // ummla v12.4s, v0.16b, v7.16b\n" - ".inst 0x6e86a42f // ummla v15.4s, v1.16b, v6.16b\n" + ".inst 0x6e87a409 // ummla v9.4s, v0.16b, v7.16b\n" + ".inst 0x6e83a40c // ummla v12.4s, v0.16b, v3.16b\n" + ".inst 0x6e87a42f // ummla v15.4s, v1.16b, v7.16b\n" "cmp x20, #0x2\n" - ".inst 0x6e87a432 // ummla v18.4s, v1.16b, v7.16b\n" - ".inst 0x6e86a455 // ummla v21.4s, v2.16b, v6.16b\n" - ".inst 0x6e87a458 // ummla v24.4s, v2.16b, v7.16b\n" - ".inst 0x6e86a47b // ummla v27.4s, v3.16b, v6.16b\n" - "ldr q6, [x22, #0x40]\n" - ".inst 0x6e87a47e // ummla v30.4s, v3.16b, v7.16b\n" - "ldr q7, [x22, #0x50]\n" + ".inst 0x6e83a432 // ummla v18.4s, v1.16b, v3.16b\n" + ".inst 0x6e87a455 // ummla v21.4s, v2.16b, v7.16b\n" + ".inst 0x6e83a458 // ummla v24.4s, v2.16b, v3.16b\n" + ".inst 0x6e87a4db // ummla v27.4s, v6.16b, v7.16b\n" + "ldr q7, [x22, #0x40]\n" + ".inst 0x6e83a4de // ummla v30.4s, v6.16b, v3.16b\n" + "ldr q3, [x22, #0x50]\n" ".inst 0x6e84a40a // ummla v10.4s, v0.16b, v4.16b\n" ".inst 0x6e85a40d // ummla v13.4s, v0.16b, v5.16b\n" "ldr q0, [%x[Apanel], #0x10]\n" @@ -119,42 +122,42 @@ void a64_interleaved_u8u32_mmla_8x12( ".inst 0x6e84a456 // ummla v22.4s, v2.16b, v4.16b\n" ".inst 0x6e85a459 // ummla v25.4s, v2.16b, v5.16b\n" "ldr q2, [%x[Apanel], #0x30]\n" - ".inst 0x6e84a47c // ummla v28.4s, v3.16b, v4.16b\n" + ".inst 0x6e84a4dc // ummla v28.4s, v6.16b, v4.16b\n" "ldr q4, [x22, #0x60]\n" - ".inst 0x6e85a47f // ummla v31.4s, v3.16b, v5.16b\n" - "ldr q3, [%x[Apanel], #0x40]\n" + ".inst 0x6e85a4df // ummla v31.4s, v6.16b, v5.16b\n" + "ldr q6, [%x[Apanel], #0x40]\n" "ldr q5, [x22, #0x70]\n" - ".inst 0x6e86a408 // ummla v8.4s, v0.16b, v6.16b\n" - ".inst 0x6e87a40b // ummla v11.4s, v0.16b, v7.16b\n" - ".inst 0x6e86a42e // ummla v14.4s, v1.16b, v6.16b\n" - ".inst 0x6e87a431 // ummla v17.4s, v1.16b, v7.16b\n" - ".inst 0x6e86a454 // ummla v20.4s, v2.16b, v6.16b\n" - ".inst 0x6e87a457 // ummla v23.4s, v2.16b, v7.16b\n" - ".inst 0x6e86a47a // ummla v26.4s, v3.16b, v6.16b\n" - "ldr q6, [x22, #0x80]\n" - ".inst 0x6e87a47d // ummla v29.4s, v3.16b, v7.16b\n" - "ldr q7, [x22, #0x90]\n" + ".inst 0x6e87a408 // ummla v8.4s, v0.16b, v7.16b\n" + ".inst 0x6e83a40b // ummla v11.4s, v0.16b, v3.16b\n" + ".inst 0x6e87a42e // ummla v14.4s, v1.16b, v7.16b\n" + ".inst 0x6e83a431 // ummla v17.4s, v1.16b, v3.16b\n" + ".inst 0x6e87a454 // ummla v20.4s, v2.16b, v7.16b\n" + ".inst 0x6e83a457 // ummla v23.4s, v2.16b, v3.16b\n" + ".inst 0x6e87a4da // ummla v26.4s, v6.16b, v7.16b\n" + "ldr q7, [x22, #0x80]\n" + ".inst 0x6e83a4dd // ummla v29.4s, v6.16b, v3.16b\n" + "ldr q3, [x22, #0x90]\n" ".inst 0x6e84a409 // ummla v9.4s, v0.16b, v4.16b\n" ".inst 0x6e85a40c // ummla v12.4s, v0.16b, v5.16b\n" ".inst 0x6e84a42f // ummla v15.4s, v1.16b, v4.16b\n" ".inst 0x6e85a432 // ummla v18.4s, v1.16b, v5.16b\n" ".inst 0x6e84a455 // ummla v21.4s, v2.16b, v4.16b\n" ".inst 0x6e85a458 // ummla v24.4s, v2.16b, v5.16b\n" - ".inst 0x6e84a47b // ummla v27.4s, v3.16b, v4.16b\n" + ".inst 0x6e84a4db // ummla v27.4s, v6.16b, v4.16b\n" "ldr q4, [x22, #0xa0]\n" - ".inst 0x6e85a47e // ummla v30.4s, v3.16b, v5.16b\n" + ".inst 0x6e85a4de // ummla v30.4s, v6.16b, v5.16b\n" "ldr q5, [x22, #0xb0]\n" - ".inst 0x6e86a40a // ummla v10.4s, v0.16b, v6.16b\n" - ".inst 0x6e87a40d // ummla v13.4s, v0.16b, v7.16b\n" + ".inst 0x6e87a40a // ummla v10.4s, v0.16b, v7.16b\n" + ".inst 0x6e83a40d // ummla v13.4s, v0.16b, v3.16b\n" "ldr q0, [%x[Apanel], #0x50]\n" - ".inst 0x6e86a430 // ummla v16.4s, v1.16b, v6.16b\n" - ".inst 0x6e87a433 // ummla v19.4s, v1.16b, v7.16b\n" + ".inst 0x6e87a430 // ummla v16.4s, v1.16b, v7.16b\n" + ".inst 0x6e83a433 // ummla v19.4s, v1.16b, v3.16b\n" "ldr q1, [%x[Apanel], #0x60]\n" - ".inst 0x6e86a456 // ummla v22.4s, v2.16b, v6.16b\n" - ".inst 0x6e87a459 // ummla v25.4s, v2.16b, v7.16b\n" + ".inst 0x6e87a456 // ummla v22.4s, v2.16b, v7.16b\n" + ".inst 0x6e83a459 // ummla v25.4s, v2.16b, v3.16b\n" "ldr q2, [%x[Apanel], #0x70]\n" - ".inst 0x6e86a47c // ummla v28.4s, v3.16b, v6.16b\n" - ".inst 0x6e87a47f // ummla v31.4s, v3.16b, v7.16b\n" + ".inst 0x6e87a4dc // ummla v28.4s, v6.16b, v7.16b\n" + ".inst 0x6e83a4df // ummla v31.4s, v6.16b, v3.16b\n" "add %x[Apanel], %x[Apanel], #0x80\n" "add x22, x22, #0xc0\n" "bge 3b\n" @@ -191,89 +194,89 @@ void a64_interleaved_u8u32_mmla_8x12( ".inst 0x6e84a47c // ummla v28.4s, v3.16b, v4.16b\n" ".inst 0x6e85a47f // ummla v31.4s, v3.16b, v5.16b\n" "cbz x20, 5f\n" - "ldr q6, [x22, #0x0]\n" - "ldr q0, [%x[Apanel], #0x0]\n" - ".inst 0x6e86a408 // ummla v8.4s, v0.16b, v6.16b\n" - "ldr q1, [%x[Apanel], #0x10]\n" - "ldr q7, [x22, #0x10]\n" - ".inst 0x6e87a40b // ummla v11.4s, v0.16b, v7.16b\n" - "ldr q2, [%x[Apanel], #0x20]\n" - "ldr q3, [%x[Apanel], #0x30]\n" - ".inst 0x6e86a42e // ummla v14.4s, v1.16b, v6.16b\n" - "ldr q4, [x22, #0x20]\n" - "ldr q5, [x22, #0x30]\n" - ".inst 0x6e87a431 // ummla v17.4s, v1.16b, v7.16b\n" - ".inst 0x6e86a454 // ummla v20.4s, v2.16b, v6.16b\n" - ".inst 0x6e87a457 // ummla v23.4s, v2.16b, v7.16b\n" + "ldr q1, [x22, #0x0]\n" + "ldr q7, [%x[Apanel], #0x0]\n" + ".inst 0x6e81a4e8 // ummla v8.4s, v7.16b, v1.16b\n" + "ldr q6, [%x[Apanel], #0x10]\n" + "ldr q0, [x22, #0x10]\n" + ".inst 0x6e80a4eb // ummla v11.4s, v7.16b, v0.16b\n" + "ldr q5, [%x[Apanel], #0x20]\n" + "ldr q4, [%x[Apanel], #0x30]\n" + ".inst 0x6e81a4ce // ummla v14.4s, v6.16b, v1.16b\n" + "ldr q3, [x22, #0x20]\n" + "ldr q2, [x22, #0x30]\n" + ".inst 0x6e80a4d1 // ummla v17.4s, v6.16b, v0.16b\n" + ".inst 0x6e81a4b4 // ummla v20.4s, v5.16b, v1.16b\n" + ".inst 0x6e80a4b7 // ummla v23.4s, v5.16b, v0.16b\n" "add %x[Apanel], %x[Apanel], #0x40\n" - ".inst 0x6e86a47a // ummla v26.4s, v3.16b, v6.16b\n" - "ldr q6, [x22, #0x40]\n" - ".inst 0x6e87a47d // ummla v29.4s, v3.16b, v7.16b\n" - "ldr q7, [x22, #0x50]\n" - ".inst 0x6e84a409 // ummla v9.4s, v0.16b, v4.16b\n" - ".inst 0x6e85a40c // ummla v12.4s, v0.16b, v5.16b\n" - ".inst 0x6e84a42f // ummla v15.4s, v1.16b, v4.16b\n" - ".inst 0x6e85a432 // ummla v18.4s, v1.16b, v5.16b\n" + ".inst 0x6e81a49a // ummla v26.4s, v4.16b, v1.16b\n" + "ldr q1, [x22, #0x40]\n" + ".inst 0x6e80a49d // ummla v29.4s, v4.16b, v0.16b\n" + "ldr q0, [x22, #0x50]\n" + ".inst 0x6e83a4e9 // ummla v9.4s, v7.16b, v3.16b\n" + ".inst 0x6e82a4ec // ummla v12.4s, v7.16b, v2.16b\n" + ".inst 0x6e83a4cf // ummla v15.4s, v6.16b, v3.16b\n" + ".inst 0x6e82a4d2 // ummla v18.4s, v6.16b, v2.16b\n" "add x22, x22, #0x60\n" - ".inst 0x6e84a455 // ummla v21.4s, v2.16b, v4.16b\n" - ".inst 0x6e85a458 // ummla v24.4s, v2.16b, v5.16b\n" - ".inst 0x6e84a47b // ummla v27.4s, v3.16b, v4.16b\n" - ".inst 0x6e85a47e // ummla v30.4s, v3.16b, v5.16b\n" - ".inst 0x6e86a40a // ummla v10.4s, v0.16b, v6.16b\n" - ".inst 0x6e87a40d // ummla v13.4s, v0.16b, v7.16b\n" - ".inst 0x6e86a430 // ummla v16.4s, v1.16b, v6.16b\n" - ".inst 0x6e87a433 // ummla v19.4s, v1.16b, v7.16b\n" - ".inst 0x6e86a456 // ummla v22.4s, v2.16b, v6.16b\n" - ".inst 0x6e87a459 // ummla v25.4s, v2.16b, v7.16b\n" - ".inst 0x6e86a47c // ummla v28.4s, v3.16b, v6.16b\n" - ".inst 0x6e87a47f // ummla v31.4s, v3.16b, v7.16b\n" + ".inst 0x6e83a4b5 // ummla v21.4s, v5.16b, v3.16b\n" + ".inst 0x6e82a4b8 // ummla v24.4s, v5.16b, v2.16b\n" + ".inst 0x6e83a49b // ummla v27.4s, v4.16b, v3.16b\n" + ".inst 0x6e82a49e // ummla v30.4s, v4.16b, v2.16b\n" + ".inst 0x6e81a4ea // ummla v10.4s, v7.16b, v1.16b\n" + ".inst 0x6e80a4ed // ummla v13.4s, v7.16b, v0.16b\n" + ".inst 0x6e81a4d0 // ummla v16.4s, v6.16b, v1.16b\n" + ".inst 0x6e80a4d3 // ummla v19.4s, v6.16b, v0.16b\n" + ".inst 0x6e81a4b6 // ummla v22.4s, v5.16b, v1.16b\n" + ".inst 0x6e80a4b9 // ummla v25.4s, v5.16b, v0.16b\n" + ".inst 0x6e81a49c // ummla v28.4s, v4.16b, v1.16b\n" + ".inst 0x6e80a49f // ummla v31.4s, v4.16b, v0.16b\n" "5:" // multiply loop done "subs x23, x23, #0x1\n" - "uzp1 v4.2d, v8.2d, v11.2d\n" + "uzp1 v0.2d, v8.2d, v11.2d\n" "uzp2 v8.2d, v8.2d, v11.2d\n" - "uzp1 v11.2d, v9.2d, v12.2d\n" + "uzp1 v1.2d, v9.2d, v12.2d\n" "uzp2 v9.2d, v9.2d, v12.2d\n" - "str q4, [%x[Cpanel], #0x0]\n" - "uzp1 v12.2d, v10.2d, v13.2d\n" + "str q0, [%x[Cpanel], #0x0]\n" + "uzp1 v0.2d, v10.2d, v13.2d\n" "uzp2 v10.2d, v10.2d, v13.2d\n" - "str q11, [%x[Cpanel], #0x10]\n" - "str q12, [%x[Cpanel], #0x20]\n" - "uzp1 v13.2d, v14.2d, v17.2d\n" + "str q1, [%x[Cpanel], #0x10]\n" + "str q0, [%x[Cpanel], #0x20]\n" + "uzp1 v0.2d, v14.2d, v17.2d\n" "uzp2 v14.2d, v14.2d, v17.2d\n" "str q8, [%x[Cpanel], #0x30]\n" - "uzp1 v17.2d, v15.2d, v18.2d\n" + "uzp1 v2.2d, v15.2d, v18.2d\n" "uzp2 v15.2d, v15.2d, v18.2d\n" "str q9, [%x[Cpanel], #0x40]\n" - "uzp1 v18.2d, v16.2d, v19.2d\n" + "uzp1 v17.2d, v16.2d, v19.2d\n" "uzp2 v16.2d, v16.2d, v19.2d\n" "str q10, [%x[Cpanel], #0x50]\n" - "uzp1 v19.2d, v20.2d, v23.2d\n" + "uzp1 v1.2d, v20.2d, v23.2d\n" "uzp2 v20.2d, v20.2d, v23.2d\n" - "str q13, [%x[Cpanel], #0x60]\n" - "uzp1 v23.2d, v21.2d, v24.2d\n" + "str q0, [%x[Cpanel], #0x60]\n" + "uzp1 v0.2d, v21.2d, v24.2d\n" "uzp2 v21.2d, v21.2d, v24.2d\n" - "str q17, [%x[Cpanel], #0x70]\n" - "uzp1 v24.2d, v22.2d, v25.2d\n" + "str q2, [%x[Cpanel], #0x70]\n" + "uzp1 v23.2d, v22.2d, v25.2d\n" "uzp2 v22.2d, v22.2d, v25.2d\n" - "str q18, [%x[Cpanel], #0x80]\n" - "uzp1 v25.2d, v26.2d, v29.2d\n" + "str q17, [%x[Cpanel], #0x80]\n" + "uzp1 v19.2d, v26.2d, v29.2d\n" "uzp2 v26.2d, v26.2d, v29.2d\n" "str q14, [%x[Cpanel], #0x90]\n" - "uzp1 v29.2d, v27.2d, v30.2d\n" + "uzp1 v18.2d, v27.2d, v30.2d\n" "uzp2 v27.2d, v27.2d, v30.2d\n" "str q15, [%x[Cpanel], #0xa0]\n" - "uzp1 v30.2d, v28.2d, v31.2d\n" + "uzp1 v17.2d, v28.2d, v31.2d\n" "uzp2 v28.2d, v28.2d, v31.2d\n" "str q16, [%x[Cpanel], #0xb0]\n" - "str q19, [%x[Cpanel], #0xc0]\n" - "str q23, [%x[Cpanel], #0xd0]\n" - "str q24, [%x[Cpanel], #0xe0]\n" + "str q1, [%x[Cpanel], #0xc0]\n" + "str q0, [%x[Cpanel], #0xd0]\n" + "str q23, [%x[Cpanel], #0xe0]\n" "str q20, [%x[Cpanel], #0xf0]\n" "str q21, [%x[Cpanel], #0x100]\n" "str q22, [%x[Cpanel], #0x110]\n" - "str q25, [%x[Cpanel], #0x120]\n" - "str q29, [%x[Cpanel], #0x130]\n" - "str q30, [%x[Cpanel], #0x140]\n" + "str q19, [%x[Cpanel], #0x120]\n" + "str q18, [%x[Cpanel], #0x130]\n" + "str q17, [%x[Cpanel], #0x140]\n" "str q26, [%x[Cpanel], #0x150]\n" "str q27, [%x[Cpanel], #0x160]\n" "str q28, [%x[Cpanel], #0x170]\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_bf16fp32_dot_16VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_bf16fp32_dot_16VL.hpp index f86bcebe64e812db457fea6864683f33c66873b6..76f43f0933b7117598636a026988476cd7832461 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_bf16fp32_dot_16VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_bf16fp32_dot_16VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,19 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - -#ifdef __aarch64__ +#if defined(ARM_COMPUTE_ENABLE_SME2) #include "../std_transforms_sme.hpp" #include "../bfloat.hpp" @@ -84,4 +83,4 @@ public: #undef ARGLIST -#endif // __aarch64__ +#endif // defined(ARM_COMPUTE_ENABLE_SME2) diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_bf16fp32_dot_16VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_bf16fp32_dot_16VL/generic.cpp index 520eeedfecd32cfad9ad7b2ca4833beada5f4b78..db29e42ef1b5e9fb7fc02515b0b024643b8c6550 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_bf16fp32_dot_16VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_bf16fp32_dot_16VL/generic.cpp @@ -21,8 +21,8 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifdef __ARM_FEATURE_SVE -#ifdef ARM_COMPUTE_ENABLE_SME2 + +#if defined(ARM_COMPUTE_ENABLE_SME2) #include "arm_gemm.hpp" #include "../../utils.hpp" @@ -62,7 +62,7 @@ void sme2_gemv_bf16fp32_dot_16VL ( break; } __asm__ __volatile__( - "ptrue p1.b\n" + "ptrue p8.b\n" ".inst 0xd503477f // SMSTART ZA\n" "cntw x28, ALL, MUL #4\n" "add x27, %x[N], x28\n" @@ -102,311 +102,311 @@ void sme2_gemv_bf16fp32_dot_16VL ( "bgt 20f\n" "beq 12f\n" "mov x23, %x[A_ptr]\n" - "lsl x22, %x[K], #0x1\n" + "lsl x21, %x[K], #0x1\n" "mov x20, %x[N]\n" - "mov x21, %x[K]\n" - ".inst 0xf8b64af8 // rprfm pldmany, x22, [x23]\n" + "mov x22, %x[K]\n" + ".inst 0xf8b54af8 // rprfm pldmany, x21, [x23]\n" ".inst 0x25b467f0 // whilelt p8.s, XZR, x20, VLx4\n" "cbz x24, 5f\n" - ".inst 0xa040c700 // ld1w { z0.s-z3.s }, pn9.b/Z, [x24]\n" - ".inst 0xc0042c00 // mova za.d[x9, #0], { z0.d-z3.d }\n" + ".inst 0xa040c718 // ld1w { z24.s-z27.s }, pn9.b/Z, [x24]\n" + ".inst 0xc0042f00 // mova za.d[x9, #0], { z24.d-z27.d }\n" "b 6f\n" "5:" // Width 1: no bias ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "6:" // Width 1: setup done - "cmp x21, #0x8\n" + "cmp x22, #0x8\n" "ble 8f\n" "7:" // Width 1: Multiply loop: Main loop head - "whilelt p0.h, XZR, x21\n" - "ld1rqh { z10.h }, p0/Z, [x23]\n" - "sub x21, x21, #0x8\n" - ".inst 0xa040a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26]\n" - ".inst 0xc15ab018 // bfdot za.s[x9, 0], { z0.h-z3.h }, z10.h[0]\n" + "whilelt p0.h, XZR, x22\n" + "ld1rqh { z8.h }, p0/Z, [x23]\n" + "sub x22, x22, #0x8\n" + ".inst 0xa040a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26]\n" + ".inst 0xc158b298 // bfdot za.s[x9, 0], { z20.h-z23.h }, z8.h[0]\n" "addvl x26, x26, #16\n" - "cmp x21, #0x8\n" - ".inst 0xa040a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26]\n" - ".inst 0xc15ab718 // bfdot za.s[x9, 0], { z24.h-z27.h }, z10.h[1]\n" + "cmp x22, #0x8\n" + ".inst 0xa040a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26]\n" + ".inst 0xc158b498 // bfdot za.s[x9, 0], { z4.h-z7.h }, z8.h[1]\n" "addvl x26, x26, #16\n" "add x23, x23, #0x10\n" - ".inst 0xa040a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26]\n" - ".inst 0xc15ab998 // bfdot za.s[x9, 0], { z12.h-z15.h }, z10.h[2]\n" + ".inst 0xa040a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26]\n" + ".inst 0xc158bb98 // bfdot za.s[x9, 0], { z28.h-z31.h }, z8.h[2]\n" "addvl x26, x26, #16\n" - ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" - ".inst 0xc15abe18 // bfdot za.s[x9, 0], { z16.h-z19.h }, z10.h[3]\n" + ".inst 0xa040a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26]\n" + ".inst 0xc158bf18 // bfdot za.s[x9, 0], { z24.h-z27.h }, z8.h[3]\n" "addvl x26, x26, #16\n" "bgt 7b\n" "8:" // Width 1: Multiply loop: Single iteration only - "whilelt p0.h, XZR, x21\n" - "ld1rqh { z10.h }, p0/Z, [x23]\n" - "subs x21, x21, #0x2\n" - ".inst 0xa040a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26]\n" + "whilelt p0.h, XZR, x22\n" + "ld1rqh { z11.h }, p0/Z, [x23]\n" + "subs x22, x22, #0x2\n" + ".inst 0xa040a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26]\n" "add x23, x23, #0x10\n" - ".inst 0xc15ab018 // bfdot za.s[x9, 0], { z0.h-z3.h }, z10.h[0]\n" - "addvl x26, x26, #16\n" - "ble 9f\n" - ".inst 0xa040a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26]\n" - "subs x21, x21, #0x2\n" - ".inst 0xc15ab718 // bfdot za.s[x9, 0], { z24.h-z27.h }, z10.h[1]\n" + ".inst 0xc15bb398 // bfdot za.s[x9, 0], { z28.h-z31.h }, z11.h[0]\n" "addvl x26, x26, #16\n" "ble 9f\n" ".inst 0xa040a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26]\n" - "subs x21, x21, #0x2\n" - ".inst 0xc15ab998 // bfdot za.s[x9, 0], { z12.h-z15.h }, z10.h[2]\n" + "subs x22, x22, #0x2\n" + ".inst 0xc15bb598 // bfdot za.s[x9, 0], { z12.h-z15.h }, z11.h[1]\n" "addvl x26, x26, #16\n" "ble 9f\n" ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" - ".inst 0xc15abe18 // bfdot za.s[x9, 0], { z16.h-z19.h }, z10.h[3]\n" + "subs x22, x22, #0x2\n" + ".inst 0xc15bba18 // bfdot za.s[x9, 0], { z16.h-z19.h }, z11.h[2]\n" + "addvl x26, x26, #16\n" + "ble 9f\n" + ".inst 0xa040a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26]\n" + ".inst 0xc15bbc18 // bfdot za.s[x9, 0], { z0.h-z3.h }, z11.h[3]\n" "addvl x26, x26, #16\n" "9:" // Width 1: Multiply loop: multiply skip "tbz %x[flags], #1, 10f\n" "add x21, %x[args_ptr], %[offset_min]\n" "add x20, %x[args_ptr], %[offset_max]\n" ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" - "ld1rw { z0.s }, p1/Z, [x21]\n" - "ld1rw { z6.s }, p1/Z, [x20]\n" - ".inst 0xc1a6c808 // fclamp { z8.s-z11.s }, z0.s, z6.s\n" + "ld1rw { z3.s }, p1/Z, [x21]\n" + "ld1rw { z29.s }, p1/Z, [x20]\n" + ".inst 0xc1bdc868 // fclamp { z8.s-z11.s }, z3.s, z29.s\n" ".inst 0xa060c328 // st1w { z8.s-z11.s }, p8, [x25]\n" "addvl x25, x25, #4\n" "b 11f\n" "10:" // Width 1: No activation - ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" - ".inst 0xa060c328 // st1w { z8.s-z11.s }, p8, [x25]\n" + ".inst 0xc0062c0c // mova { z12.d-z15.d }, za.d[x9, #0]\n" + ".inst 0xa060c32c // st1w { z12.s-z15.s }, p8, [x25]\n" "addvl x25, x25, #4\n" "11:" // Width 1: Output done "b 36f\n" "12:" // Width 2 "mov x23, %x[A_ptr]\n" - "lsl x22, %x[K], #0x1\n" + "lsl x21, %x[K], #0x1\n" "sub x20, %x[N], x28\n" - "mov x21, %x[K]\n" - ".inst 0xf8b64af8 // rprfm pldmany, x22, [x23]\n" + "mov x22, %x[K]\n" + ".inst 0xf8b54af8 // rprfm pldmany, x21, [x23]\n" ".inst 0x25b467f0 // whilelt p8.s, XZR, x20, VLx4\n" "cbz x24, 13f\n" - ".inst 0xa040c700 // ld1w { z0.s-z3.s }, pn9.b/Z, [x24]\n" - ".inst 0xc0042c00 // mova za.d[x9, #0], { z0.d-z3.d }\n" - ".inst 0xa041c708 // ld1w { z8.s-z11.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n" - ".inst 0xc0042d01 // mova za.d[x9, #1], { z8.d-z11.d }\n" + ".inst 0xa040c704 // ld1w { z4.s-z7.s }, pn9.b/Z, [x24]\n" + ".inst 0xc0042c80 // mova za.d[x9, #0], { z4.d-z7.d }\n" + ".inst 0xa041c714 // ld1w { z20.s-z23.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n" + ".inst 0xc0042e81 // mova za.d[x9, #1], { z20.d-z23.d }\n" "b 14f\n" "13:" // Width 2: no bias ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "14:" // Width 2: setup done - "cmp x21, #0x8\n" + "cmp x22, #0x8\n" "ble 16f\n" "15:" // Width 2: Multiply loop: Main loop head - "whilelt p0.h, XZR, x21\n" - "ld1rqh { z10.h }, p0/Z, [x23]\n" - "sub x21, x21, #0x8\n" - ".inst 0xa040a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26]\n" - ".inst 0xc15ab018 // bfdot za.s[x9, 0], { z0.h-z3.h }, z10.h[0]\n" - "cmp x21, #0x8\n" + "whilelt p0.h, XZR, x22\n" + "ld1rqh { z9.h }, p0/Z, [x23]\n" + "sub x22, x22, #0x8\n" + ".inst 0xa040a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26]\n" + ".inst 0xc159b198 // bfdot za.s[x9, 0], { z12.h-z15.h }, z9.h[0]\n" + "cmp x22, #0x8\n" "add x23, x23, #0x10\n" ".inst 0xa041a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15ab099 // bfdot za.s[x9, 1], { z4.h-z7.h }, z10.h[0]\n" - "addvl x26, x26, #16\n" - ".inst 0xa040a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26]\n" - ".inst 0xc15ab718 // bfdot za.s[x9, 0], { z24.h-z27.h }, z10.h[1]\n" - ".inst 0xa041a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15ab619 // bfdot za.s[x9, 1], { z16.h-z19.h }, z10.h[1]\n" + ".inst 0xc159b099 // bfdot za.s[x9, 1], { z4.h-z7.h }, z9.h[0]\n" "addvl x26, x26, #16\n" ".inst 0xa040a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26]\n" - ".inst 0xc15ab998 // bfdot za.s[x9, 0], { z12.h-z15.h }, z10.h[2]\n" - ".inst 0xa041a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aba19 // bfdot za.s[x9, 1], { z16.h-z19.h }, z10.h[2]\n" + ".inst 0xc159b598 // bfdot za.s[x9, 0], { z12.h-z15.h }, z9.h[1]\n" + ".inst 0xa041a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc159b499 // bfdot za.s[x9, 1], { z4.h-z7.h }, z9.h[1]\n" "addvl x26, x26, #16\n" - ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" - ".inst 0xc15abe18 // bfdot za.s[x9, 0], { z16.h-z19.h }, z10.h[3]\n" - ".inst 0xa041a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15abf19 // bfdot za.s[x9, 1], { z24.h-z27.h }, z10.h[3]\n" + ".inst 0xa040a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26]\n" + ".inst 0xc159bb18 // bfdot za.s[x9, 0], { z24.h-z27.h }, z9.h[2]\n" + ".inst 0xa041a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc159b819 // bfdot za.s[x9, 1], { z0.h-z3.h }, z9.h[2]\n" + "addvl x26, x26, #16\n" + ".inst 0xa040a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26]\n" + ".inst 0xc159bc18 // bfdot za.s[x9, 0], { z0.h-z3.h }, z9.h[3]\n" + ".inst 0xa041a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc159bf99 // bfdot za.s[x9, 1], { z28.h-z31.h }, z9.h[3]\n" "addvl x26, x26, #16\n" "bgt 15b\n" "16:" // Width 2: Multiply loop: Single iteration only - "whilelt p0.h, XZR, x21\n" - "ld1rqh { z10.h }, p0/Z, [x23]\n" - "subs x21, x21, #0x2\n" - ".inst 0xa040a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26]\n" + "whilelt p0.h, XZR, x22\n" + "ld1rqh { z11.h }, p0/Z, [x23]\n" + "subs x22, x22, #0x2\n" + ".inst 0xa040a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26]\n" "add x23, x23, #0x10\n" - ".inst 0xc15ab018 // bfdot za.s[x9, 0], { z0.h-z3.h }, z10.h[0]\n" - ".inst 0xa041a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15ab099 // bfdot za.s[x9, 1], { z4.h-z7.h }, z10.h[0]\n" + ".inst 0xc15bb198 // bfdot za.s[x9, 0], { z12.h-z15.h }, z11.h[0]\n" + ".inst 0xa041a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15bb019 // bfdot za.s[x9, 1], { z0.h-z3.h }, z11.h[0]\n" "addvl x26, x26, #16\n" "ble 17f\n" ".inst 0xa040a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26]\n" - "subs x21, x21, #0x2\n" - ".inst 0xc15ab718 // bfdot za.s[x9, 0], { z24.h-z27.h }, z10.h[1]\n" - ".inst 0xa041a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15ab619 // bfdot za.s[x9, 1], { z16.h-z19.h }, z10.h[1]\n" + "subs x22, x22, #0x2\n" + ".inst 0xc15bb718 // bfdot za.s[x9, 0], { z24.h-z27.h }, z11.h[1]\n" + ".inst 0xa041a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15bb419 // bfdot za.s[x9, 1], { z0.h-z3.h }, z11.h[1]\n" "addvl x26, x26, #16\n" "ble 17f\n" ".inst 0xa040a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26]\n" - "subs x21, x21, #0x2\n" - ".inst 0xc15ab998 // bfdot za.s[x9, 0], { z12.h-z15.h }, z10.h[2]\n" - ".inst 0xa041a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aba19 // bfdot za.s[x9, 1], { z16.h-z19.h }, z10.h[2]\n" + "subs x22, x22, #0x2\n" + ".inst 0xc15bb998 // bfdot za.s[x9, 0], { z12.h-z15.h }, z11.h[2]\n" + ".inst 0xa041a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15bbb99 // bfdot za.s[x9, 1], { z28.h-z31.h }, z11.h[2]\n" "addvl x26, x26, #16\n" "ble 17f\n" ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" - ".inst 0xc15abe18 // bfdot za.s[x9, 0], { z16.h-z19.h }, z10.h[3]\n" - ".inst 0xa041a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15abf19 // bfdot za.s[x9, 1], { z24.h-z27.h }, z10.h[3]\n" + ".inst 0xc15bbe18 // bfdot za.s[x9, 0], { z16.h-z19.h }, z11.h[3]\n" + ".inst 0xa041a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15bbe99 // bfdot za.s[x9, 1], { z20.h-z23.h }, z11.h[3]\n" "addvl x26, x26, #16\n" "17:" // Width 2: Multiply loop: multiply skip "tbz %x[flags], #1, 18f\n" "add x21, %x[args_ptr], %[offset_min]\n" "add x20, %x[args_ptr], %[offset_max]\n" - ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" - "ld1rw { z0.s }, p1/Z, [x21]\n" - ".inst 0xc0062c34 // mova { z20.d-z23.d }, za.d[x9, #1]\n" - "ld1rw { z6.s }, p1/Z, [x20]\n" - ".inst 0xc1a6c808 // fclamp { z8.s-z11.s }, z0.s, z6.s\n" - ".inst 0xa060c728 // st1w { z8.s-z11.s }, pn9.b, [x25]\n" - ".inst 0xc1a6c814 // fclamp { z20.s-z23.s }, z0.s, z6.s\n" - ".inst 0xa061c334 // st1w { z20.s-z23.s }, p8, [x25, #0x4, MUL VL]\n" + ".inst 0xc0062c00 // mova { z0.d-z3.d }, za.d[x9, #0]\n" + "ld1rw { z9.s }, p1/Z, [x21]\n" + ".inst 0xc0062c24 // mova { z4.d-z7.d }, za.d[x9, #1]\n" + "ld1rw { z8.s }, p1/Z, [x20]\n" + ".inst 0xc1a8c920 // fclamp { z0.s-z3.s }, z9.s, z8.s\n" + ".inst 0xa060c720 // st1w { z0.s-z3.s }, pn9.b, [x25]\n" + ".inst 0xc1a8c924 // fclamp { z4.s-z7.s }, z9.s, z8.s\n" + ".inst 0xa061c324 // st1w { z4.s-z7.s }, p8, [x25, #0x4, MUL VL]\n" "addvl x25, x25, #8\n" "b 19f\n" "18:" // Width 2: No activation - ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" - ".inst 0xa060c728 // st1w { z8.s-z11.s }, pn9.b, [x25]\n" - ".inst 0xc0062c34 // mova { z20.d-z23.d }, za.d[x9, #1]\n" - ".inst 0xa061c334 // st1w { z20.s-z23.s }, p8, [x25, #0x4, MUL VL]\n" + ".inst 0xc0062c10 // mova { z16.d-z19.d }, za.d[x9, #0]\n" + ".inst 0xa060c730 // st1w { z16.s-z19.s }, pn9.b, [x25]\n" + ".inst 0xc0062c2c // mova { z12.d-z15.d }, za.d[x9, #1]\n" + ".inst 0xa061c32c // st1w { z12.s-z15.s }, p8, [x25, #0x4, MUL VL]\n" "addvl x25, x25, #8\n" "19:" // Width 2: Output done "b 36f\n" "20:" // Width 3 "mov x20, #0x2\n" "mov x23, %x[A_ptr]\n" - "lsl x22, %x[K], #0x1\n" + "lsl x21, %x[K], #0x1\n" "msub x20, x28, x20, %x[N]\n" - "mov x21, %x[K]\n" - ".inst 0xf8b64af8 // rprfm pldmany, x22, [x23]\n" + "mov x22, %x[K]\n" + ".inst 0xf8b54af8 // rprfm pldmany, x21, [x23]\n" ".inst 0x25b467f0 // whilelt p8.s, XZR, x20, VLx4\n" "cbz x24, 21f\n" - ".inst 0xa040c700 // ld1w { z0.s-z3.s }, pn9.b/Z, [x24]\n" - ".inst 0xc0042c00 // mova za.d[x9, #0], { z0.d-z3.d }\n" - ".inst 0xa041c708 // ld1w { z8.s-z11.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n" - ".inst 0xc0042d01 // mova za.d[x9, #1], { z8.d-z11.d }\n" - ".inst 0xa042c704 // ld1w { z4.s-z7.s }, pn9.b/Z, [x24, #0x8, MUL VL]\n" - ".inst 0xc0042c82 // mova za.d[x9, #2], { z4.d-z7.d }\n" + ".inst 0xa040c718 // ld1w { z24.s-z27.s }, pn9.b/Z, [x24]\n" + ".inst 0xc0042f00 // mova za.d[x9, #0], { z24.d-z27.d }\n" + ".inst 0xa041c704 // ld1w { z4.s-z7.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n" + ".inst 0xc0042c81 // mova za.d[x9, #1], { z4.d-z7.d }\n" + ".inst 0xa042c710 // ld1w { z16.s-z19.s }, pn9.b/Z, [x24, #0x8, MUL VL]\n" + ".inst 0xc0042e02 // mova za.d[x9, #2], { z16.d-z19.d }\n" "b 22f\n" "21:" // Width 3: no bias ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "22:" // Width 3: setup done - "cmp x21, #0x8\n" + "cmp x22, #0x8\n" "ble 24f\n" "23:" // Width 3: Multiply loop: Main loop head - "whilelt p0.h, XZR, x21\n" - "ld1rqh { z10.h }, p0/Z, [x23]\n" - "sub x21, x21, #0x8\n" + "whilelt p0.h, XZR, x22\n" + "ld1rqh { z15.h }, p0/Z, [x23]\n" + "sub x22, x22, #0x8\n" ".inst 0xa040a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26]\n" - ".inst 0xc15ab018 // bfdot za.s[x9, 0], { z0.h-z3.h }, z10.h[0]\n" - "cmp x21, #0x8\n" + ".inst 0xc15fb018 // bfdot za.s[x9, 0], { z0.h-z3.h }, z15.h[0]\n" + "cmp x22, #0x8\n" "add x23, x23, #0x10\n" ".inst 0xa041a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15ab099 // bfdot za.s[x9, 1], { z4.h-z7.h }, z10.h[0]\n" - ".inst 0xa042a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15ab29a // bfdot za.s[x9, 2], { z20.h-z23.h }, z10.h[0]\n" + ".inst 0xc15fb099 // bfdot za.s[x9, 1], { z4.h-z7.h }, z15.h[0]\n" + ".inst 0xa042a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc15fb01a // bfdot za.s[x9, 2], { z0.h-z3.h }, z15.h[0]\n" "addvl x26, x26, #16\n" - ".inst 0xa040a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26]\n" - ".inst 0xc15ab718 // bfdot za.s[x9, 0], { z24.h-z27.h }, z10.h[1]\n" - ".inst 0xa041a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15ab619 // bfdot za.s[x9, 1], { z16.h-z19.h }, z10.h[1]\n" - ".inst 0xa042a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15ab71a // bfdot za.s[x9, 2], { z24.h-z27.h }, z10.h[1]\n" + ".inst 0xa040a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26]\n" + ".inst 0xc15fb698 // bfdot za.s[x9, 0], { z20.h-z23.h }, z15.h[1]\n" + ".inst 0xa041a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15fb699 // bfdot za.s[x9, 1], { z20.h-z23.h }, z15.h[1]\n" + ".inst 0xa042a749 // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc15fb51a // bfdot za.s[x9, 2], { z8.h-z11.h }, z15.h[1]\n" "addvl x26, x26, #16\n" - ".inst 0xa040a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26]\n" - ".inst 0xc15ab998 // bfdot za.s[x9, 0], { z12.h-z15.h }, z10.h[2]\n" - ".inst 0xa041a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aba19 // bfdot za.s[x9, 1], { z16.h-z19.h }, z10.h[2]\n" - ".inst 0xa042a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15abb9a // bfdot za.s[x9, 2], { z28.h-z31.h }, z10.h[2]\n" + ".inst 0xa040a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26]\n" + ".inst 0xc15fbb18 // bfdot za.s[x9, 0], { z24.h-z27.h }, z15.h[2]\n" + ".inst 0xa041a749 // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15fb919 // bfdot za.s[x9, 1], { z8.h-z11.h }, z15.h[2]\n" + ".inst 0xa042a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc15fba1a // bfdot za.s[x9, 2], { z16.h-z19.h }, z15.h[2]\n" "addvl x26, x26, #16\n" ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" - ".inst 0xc15abe18 // bfdot za.s[x9, 0], { z16.h-z19.h }, z10.h[3]\n" - ".inst 0xa041a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15abf19 // bfdot za.s[x9, 1], { z24.h-z27.h }, z10.h[3]\n" - ".inst 0xa042a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15abe1a // bfdot za.s[x9, 2], { z16.h-z19.h }, z10.h[3]\n" + ".inst 0xc15fbe18 // bfdot za.s[x9, 0], { z16.h-z19.h }, z15.h[3]\n" + ".inst 0xa041a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15fbe19 // bfdot za.s[x9, 1], { z16.h-z19.h }, z15.h[3]\n" + ".inst 0xa042a749 // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc15fbd1a // bfdot za.s[x9, 2], { z8.h-z11.h }, z15.h[3]\n" "addvl x26, x26, #16\n" "bgt 23b\n" "24:" // Width 3: Multiply loop: Single iteration only - "whilelt p0.h, XZR, x21\n" - "ld1rqh { z10.h }, p0/Z, [x23]\n" - "subs x21, x21, #0x2\n" - ".inst 0xa040a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26]\n" + "whilelt p0.h, XZR, x22\n" + "ld1rqh { z11.h }, p0/Z, [x23]\n" + "subs x22, x22, #0x2\n" + ".inst 0xa040a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26]\n" "add x23, x23, #0x10\n" - ".inst 0xc15ab018 // bfdot za.s[x9, 0], { z0.h-z3.h }, z10.h[0]\n" - ".inst 0xa041a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15ab099 // bfdot za.s[x9, 1], { z4.h-z7.h }, z10.h[0]\n" + ".inst 0xc15bb398 // bfdot za.s[x9, 0], { z28.h-z31.h }, z11.h[0]\n" + ".inst 0xa041a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15bb019 // bfdot za.s[x9, 1], { z0.h-z3.h }, z11.h[0]\n" ".inst 0xa042a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15ab29a // bfdot za.s[x9, 2], { z20.h-z23.h }, z10.h[0]\n" - "addvl x26, x26, #16\n" - "ble 25f\n" - ".inst 0xa040a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26]\n" - "subs x21, x21, #0x2\n" - ".inst 0xc15ab718 // bfdot za.s[x9, 0], { z24.h-z27.h }, z10.h[1]\n" - ".inst 0xa041a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15ab619 // bfdot za.s[x9, 1], { z16.h-z19.h }, z10.h[1]\n" - ".inst 0xa042a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15ab71a // bfdot za.s[x9, 2], { z24.h-z27.h }, z10.h[1]\n" + ".inst 0xc15bb29a // bfdot za.s[x9, 2], { z20.h-z23.h }, z11.h[0]\n" "addvl x26, x26, #16\n" "ble 25f\n" ".inst 0xa040a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26]\n" - "subs x21, x21, #0x2\n" - ".inst 0xc15ab998 // bfdot za.s[x9, 0], { z12.h-z15.h }, z10.h[2]\n" - ".inst 0xa041a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aba19 // bfdot za.s[x9, 1], { z16.h-z19.h }, z10.h[2]\n" + "subs x22, x22, #0x2\n" + ".inst 0xc15bb598 // bfdot za.s[x9, 0], { z12.h-z15.h }, z11.h[1]\n" + ".inst 0xa041a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15bb499 // bfdot za.s[x9, 1], { z4.h-z7.h }, z11.h[1]\n" ".inst 0xa042a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15abb9a // bfdot za.s[x9, 2], { z28.h-z31.h }, z10.h[2]\n" + ".inst 0xc15bb79a // bfdot za.s[x9, 2], { z28.h-z31.h }, z11.h[1]\n" "addvl x26, x26, #16\n" "ble 25f\n" - ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" - ".inst 0xc15abe18 // bfdot za.s[x9, 0], { z16.h-z19.h }, z10.h[3]\n" - ".inst 0xa041a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15abf19 // bfdot za.s[x9, 1], { z24.h-z27.h }, z10.h[3]\n" + ".inst 0xa040a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26]\n" + "subs x22, x22, #0x2\n" + ".inst 0xc15bb898 // bfdot za.s[x9, 0], { z4.h-z7.h }, z11.h[2]\n" + ".inst 0xa041a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15bba99 // bfdot za.s[x9, 1], { z20.h-z23.h }, z11.h[2]\n" + ".inst 0xa042a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc15bb99a // bfdot za.s[x9, 2], { z12.h-z15.h }, z11.h[2]\n" + "addvl x26, x26, #16\n" + "ble 25f\n" + ".inst 0xa040a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26]\n" + ".inst 0xc15bbd98 // bfdot za.s[x9, 0], { z12.h-z15.h }, z11.h[3]\n" + ".inst 0xa041a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15bbe99 // bfdot za.s[x9, 1], { z20.h-z23.h }, z11.h[3]\n" ".inst 0xa042a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15abe1a // bfdot za.s[x9, 2], { z16.h-z19.h }, z10.h[3]\n" + ".inst 0xc15bbe1a // bfdot za.s[x9, 2], { z16.h-z19.h }, z11.h[3]\n" "addvl x26, x26, #16\n" "25:" // Width 3: Multiply loop: multiply skip "tbz %x[flags], #1, 26f\n" "add x21, %x[args_ptr], %[offset_min]\n" "add x20, %x[args_ptr], %[offset_max]\n" - ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" - "ld1rw { z0.s }, p1/Z, [x21]\n" - ".inst 0xc0062c34 // mova { z20.d-z23.d }, za.d[x9, #1]\n" - "ld1rw { z6.s }, p1/Z, [x20]\n" - ".inst 0xc1a6c808 // fclamp { z8.s-z11.s }, z0.s, z6.s\n" - ".inst 0xc0062c50 // mova { z16.d-z19.d }, za.d[x9, #2]\n" - ".inst 0xa060c728 // st1w { z8.s-z11.s }, pn9.b, [x25]\n" - ".inst 0xc1a6c814 // fclamp { z20.s-z23.s }, z0.s, z6.s\n" - ".inst 0xa061c734 // st1w { z20.s-z23.s }, pn9.b, [x25, #0x4, MUL VL]\n" - ".inst 0xc1a6c810 // fclamp { z16.s-z19.s }, z0.s, z6.s\n" - ".inst 0xa062c330 // st1w { z16.s-z19.s }, p8, [x25, #0x8, MUL VL]\n" + ".inst 0xc0062c04 // mova { z4.d-z7.d }, za.d[x9, #0]\n" + "ld1rw { z17.s }, p1/Z, [x21]\n" + ".inst 0xc0062c28 // mova { z8.d-z11.d }, za.d[x9, #1]\n" + "ld1rw { z16.s }, p1/Z, [x20]\n" + ".inst 0xc1b0ca24 // fclamp { z4.s-z7.s }, z17.s, z16.s\n" + ".inst 0xc0062c4c // mova { z12.d-z15.d }, za.d[x9, #2]\n" + ".inst 0xa060c724 // st1w { z4.s-z7.s }, pn9.b, [x25]\n" + ".inst 0xc1b0ca28 // fclamp { z8.s-z11.s }, z17.s, z16.s\n" + ".inst 0xa061c728 // st1w { z8.s-z11.s }, pn9.b, [x25, #0x4, MUL VL]\n" + ".inst 0xc1b0ca2c // fclamp { z12.s-z15.s }, z17.s, z16.s\n" + ".inst 0xa062c32c // st1w { z12.s-z15.s }, p8, [x25, #0x8, MUL VL]\n" "addvl x25, x25, #12\n" "b 27f\n" "26:" // Width 3: No activation - ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" - ".inst 0xa060c728 // st1w { z8.s-z11.s }, pn9.b, [x25]\n" - ".inst 0xc0062c34 // mova { z20.d-z23.d }, za.d[x9, #1]\n" - ".inst 0xa061c734 // st1w { z20.s-z23.s }, pn9.b, [x25, #0x4, MUL VL]\n" - ".inst 0xc0062c50 // mova { z16.d-z19.d }, za.d[x9, #2]\n" - ".inst 0xa062c330 // st1w { z16.s-z19.s }, p8, [x25, #0x8, MUL VL]\n" + ".inst 0xc0062c14 // mova { z20.d-z23.d }, za.d[x9, #0]\n" + ".inst 0xa060c734 // st1w { z20.s-z23.s }, pn9.b, [x25]\n" + ".inst 0xc0062c2c // mova { z12.d-z15.d }, za.d[x9, #1]\n" + ".inst 0xa061c72c // st1w { z12.s-z15.s }, pn9.b, [x25, #0x4, MUL VL]\n" + ".inst 0xc0062c4c // mova { z12.d-z15.d }, za.d[x9, #2]\n" + ".inst 0xa062c32c // st1w { z12.s-z15.s }, p8, [x25, #0x8, MUL VL]\n" "addvl x25, x25, #12\n" "27:" // Width 3: Output done "b 36f\n" "28:" // Width 4 "mov x20, #0x3\n" "mov x23, %x[A_ptr]\n" - "lsl x22, %x[K], #0x1\n" + "lsl x21, %x[K], #0x1\n" "msub x20, x28, x20, %x[N]\n" - "mov x21, %x[K]\n" - ".inst 0xf8b64af8 // rprfm pldmany, x22, [x23]\n" + "mov x22, %x[K]\n" + ".inst 0xf8b54af8 // rprfm pldmany, x21, [x23]\n" ".inst 0x25b467f0 // whilelt p8.s, XZR, x20, VLx4\n" "cbz x24, 29f\n" - ".inst 0xa040c700 // ld1w { z0.s-z3.s }, pn9.b/Z, [x24]\n" - ".inst 0xc0042c00 // mova za.d[x9, #0], { z0.d-z3.d }\n" - ".inst 0xa041c708 // ld1w { z8.s-z11.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n" - ".inst 0xc0042d01 // mova za.d[x9, #1], { z8.d-z11.d }\n" - ".inst 0xa042c704 // ld1w { z4.s-z7.s }, pn9.b/Z, [x24, #0x8, MUL VL]\n" - ".inst 0xc0042c82 // mova za.d[x9, #2], { z4.d-z7.d }\n" + ".inst 0xa040c704 // ld1w { z4.s-z7.s }, pn9.b/Z, [x24]\n" + ".inst 0xc0042c80 // mova za.d[x9, #0], { z4.d-z7.d }\n" + ".inst 0xa041c710 // ld1w { z16.s-z19.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n" + ".inst 0xc0042e01 // mova za.d[x9, #1], { z16.d-z19.d }\n" + ".inst 0xa042c70c // ld1w { z12.s-z15.s }, pn9.b/Z, [x24, #0x8, MUL VL]\n" + ".inst 0xc0042d82 // mova za.d[x9, #2], { z12.d-z15.d }\n" ".inst 0xa043c710 // ld1w { z16.s-z19.s }, pn9.b/Z, [x24, #0xc, MUL VL]\n" ".inst 0xc0042e03 // mova za.d[x9, #3], { z16.d-z19.d }\n" "addvl x24, x24, #16\n" @@ -414,126 +414,126 @@ void sme2_gemv_bf16fp32_dot_16VL ( "29:" // Width 4: no bias ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "30:" // Width 4: setup done - "cmp x21, #0x8\n" + "cmp x22, #0x8\n" "ble 32f\n" "31:" // Width 4: Multiply loop: Main loop head - "whilelt p0.h, XZR, x21\n" - "ld1rqh { z10.h }, p0/Z, [x23]\n" - "sub x21, x21, #0x8\n" - ".inst 0xa040a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26]\n" - ".inst 0xc15ab018 // bfdot za.s[x9, 0], { z0.h-z3.h }, z10.h[0]\n" - "cmp x21, #0x8\n" + "whilelt p0.h, XZR, x22\n" + "ld1rqh { z8.h }, p0/Z, [x23]\n" + "sub x22, x22, #0x8\n" + ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" + ".inst 0xc158b218 // bfdot za.s[x9, 0], { z16.h-z19.h }, z8.h[0]\n" + "cmp x22, #0x8\n" "add x23, x23, #0x10\n" - ".inst 0xa041a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15ab099 // bfdot za.s[x9, 1], { z4.h-z7.h }, z10.h[0]\n" - ".inst 0xa042a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15ab29a // bfdot za.s[x9, 2], { z20.h-z23.h }, z10.h[0]\n" - ".inst 0xa043a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc15ab21b // bfdot za.s[x9, 3], { z16.h-z19.h }, z10.h[0]\n" - "addvl x26, x26, #16\n" - ".inst 0xa040a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26]\n" - ".inst 0xc15ab718 // bfdot za.s[x9, 0], { z24.h-z27.h }, z10.h[1]\n" - ".inst 0xa041a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15ab619 // bfdot za.s[x9, 1], { z16.h-z19.h }, z10.h[1]\n" - ".inst 0xa042a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15ab71a // bfdot za.s[x9, 2], { z24.h-z27.h }, z10.h[1]\n" - ".inst 0xa043a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc15ab61b // bfdot za.s[x9, 3], { z16.h-z19.h }, z10.h[1]\n" + ".inst 0xa041a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc158b199 // bfdot za.s[x9, 1], { z12.h-z15.h }, z8.h[0]\n" + ".inst 0xa042a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc158b21a // bfdot za.s[x9, 2], { z16.h-z19.h }, z8.h[0]\n" + ".inst 0xa043a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc158b19b // bfdot za.s[x9, 3], { z12.h-z15.h }, z8.h[0]\n" "addvl x26, x26, #16\n" ".inst 0xa040a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26]\n" - ".inst 0xc15ab998 // bfdot za.s[x9, 0], { z12.h-z15.h }, z10.h[2]\n" - ".inst 0xa041a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aba19 // bfdot za.s[x9, 1], { z16.h-z19.h }, z10.h[2]\n" - ".inst 0xa042a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15abb9a // bfdot za.s[x9, 2], { z28.h-z31.h }, z10.h[2]\n" + ".inst 0xc158b598 // bfdot za.s[x9, 0], { z12.h-z15.h }, z8.h[1]\n" + ".inst 0xa041a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc158b699 // bfdot za.s[x9, 1], { z20.h-z23.h }, z8.h[1]\n" + ".inst 0xa042a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc158b61a // bfdot za.s[x9, 2], { z16.h-z19.h }, z8.h[1]\n" ".inst 0xa043a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc15aba9b // bfdot za.s[x9, 3], { z20.h-z23.h }, z10.h[2]\n" + ".inst 0xc158b69b // bfdot za.s[x9, 3], { z20.h-z23.h }, z8.h[1]\n" "addvl x26, x26, #16\n" - ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" - ".inst 0xc15abe18 // bfdot za.s[x9, 0], { z16.h-z19.h }, z10.h[3]\n" - ".inst 0xa041a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15abf19 // bfdot za.s[x9, 1], { z24.h-z27.h }, z10.h[3]\n" + ".inst 0xa040a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26]\n" + ".inst 0xc158b898 // bfdot za.s[x9, 0], { z4.h-z7.h }, z8.h[2]\n" + ".inst 0xa041a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc158ba19 // bfdot za.s[x9, 1], { z16.h-z19.h }, z8.h[2]\n" ".inst 0xa042a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15abe1a // bfdot za.s[x9, 2], { z16.h-z19.h }, z10.h[3]\n" - ".inst 0xa043a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc15abe1b // bfdot za.s[x9, 3], { z16.h-z19.h }, z10.h[3]\n" + ".inst 0xc158ba1a // bfdot za.s[x9, 2], { z16.h-z19.h }, z8.h[2]\n" + ".inst 0xa043a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc158b81b // bfdot za.s[x9, 3], { z0.h-z3.h }, z8.h[2]\n" + "addvl x26, x26, #16\n" + ".inst 0xa040a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26]\n" + ".inst 0xc158be98 // bfdot za.s[x9, 0], { z20.h-z23.h }, z8.h[3]\n" + ".inst 0xa041a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc158be19 // bfdot za.s[x9, 1], { z16.h-z19.h }, z8.h[3]\n" + ".inst 0xa042a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc158bc9a // bfdot za.s[x9, 2], { z4.h-z7.h }, z8.h[3]\n" + ".inst 0xa043a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc158be9b // bfdot za.s[x9, 3], { z20.h-z23.h }, z8.h[3]\n" "addvl x26, x26, #16\n" "bgt 31b\n" "32:" // Width 4: Multiply loop: Single iteration only - "whilelt p0.h, XZR, x21\n" - "ld1rqh { z10.h }, p0/Z, [x23]\n" - "subs x21, x21, #0x2\n" - ".inst 0xa040a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26]\n" + "whilelt p0.h, XZR, x22\n" + "ld1rqh { z11.h }, p0/Z, [x23]\n" + "subs x22, x22, #0x2\n" + ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" "add x23, x23, #0x10\n" - ".inst 0xc15ab018 // bfdot za.s[x9, 0], { z0.h-z3.h }, z10.h[0]\n" - ".inst 0xa041a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15ab099 // bfdot za.s[x9, 1], { z4.h-z7.h }, z10.h[0]\n" - ".inst 0xa042a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15ab29a // bfdot za.s[x9, 2], { z20.h-z23.h }, z10.h[0]\n" + ".inst 0xc15bb218 // bfdot za.s[x9, 0], { z16.h-z19.h }, z11.h[0]\n" + ".inst 0xa041a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15bb299 // bfdot za.s[x9, 1], { z20.h-z23.h }, z11.h[0]\n" + ".inst 0xa042a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc15bb39a // bfdot za.s[x9, 2], { z28.h-z31.h }, z11.h[0]\n" ".inst 0xa043a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc15ab21b // bfdot za.s[x9, 3], { z16.h-z19.h }, z10.h[0]\n" + ".inst 0xc15bb21b // bfdot za.s[x9, 3], { z16.h-z19.h }, z11.h[0]\n" "addvl x26, x26, #16\n" "ble 33f\n" - ".inst 0xa040a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26]\n" - "subs x21, x21, #0x2\n" - ".inst 0xc15ab718 // bfdot za.s[x9, 0], { z24.h-z27.h }, z10.h[1]\n" + ".inst 0xa040a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26]\n" + "subs x22, x22, #0x2\n" + ".inst 0xc15bb418 // bfdot za.s[x9, 0], { z0.h-z3.h }, z11.h[1]\n" ".inst 0xa041a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15ab619 // bfdot za.s[x9, 1], { z16.h-z19.h }, z10.h[1]\n" - ".inst 0xa042a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15ab71a // bfdot za.s[x9, 2], { z24.h-z27.h }, z10.h[1]\n" + ".inst 0xc15bb619 // bfdot za.s[x9, 1], { z16.h-z19.h }, z11.h[1]\n" + ".inst 0xa042a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc15bb61a // bfdot za.s[x9, 2], { z16.h-z19.h }, z11.h[1]\n" ".inst 0xa043a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc15ab61b // bfdot za.s[x9, 3], { z16.h-z19.h }, z10.h[1]\n" + ".inst 0xc15bb61b // bfdot za.s[x9, 3], { z16.h-z19.h }, z11.h[1]\n" "addvl x26, x26, #16\n" "ble 33f\n" - ".inst 0xa040a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26]\n" - "subs x21, x21, #0x2\n" - ".inst 0xc15ab998 // bfdot za.s[x9, 0], { z12.h-z15.h }, z10.h[2]\n" + ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" + "subs x22, x22, #0x2\n" + ".inst 0xc15bba18 // bfdot za.s[x9, 0], { z16.h-z19.h }, z11.h[2]\n" ".inst 0xa041a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aba19 // bfdot za.s[x9, 1], { z16.h-z19.h }, z10.h[2]\n" - ".inst 0xa042a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15abb9a // bfdot za.s[x9, 2], { z28.h-z31.h }, z10.h[2]\n" - ".inst 0xa043a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc15aba9b // bfdot za.s[x9, 3], { z20.h-z23.h }, z10.h[2]\n" + ".inst 0xc15bba19 // bfdot za.s[x9, 1], { z16.h-z19.h }, z11.h[2]\n" + ".inst 0xa042a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc15bba1a // bfdot za.s[x9, 2], { z16.h-z19.h }, z11.h[2]\n" + ".inst 0xa043a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc15bba1b // bfdot za.s[x9, 3], { z16.h-z19.h }, z11.h[2]\n" "addvl x26, x26, #16\n" "ble 33f\n" ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" - ".inst 0xc15abe18 // bfdot za.s[x9, 0], { z16.h-z19.h }, z10.h[3]\n" + ".inst 0xc15bbe18 // bfdot za.s[x9, 0], { z16.h-z19.h }, z11.h[3]\n" ".inst 0xa041a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15abf19 // bfdot za.s[x9, 1], { z24.h-z27.h }, z10.h[3]\n" + ".inst 0xc15bbf19 // bfdot za.s[x9, 1], { z24.h-z27.h }, z11.h[3]\n" ".inst 0xa042a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15abe1a // bfdot za.s[x9, 2], { z16.h-z19.h }, z10.h[3]\n" + ".inst 0xc15bbe1a // bfdot za.s[x9, 2], { z16.h-z19.h }, z11.h[3]\n" ".inst 0xa043a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc15abe1b // bfdot za.s[x9, 3], { z16.h-z19.h }, z10.h[3]\n" + ".inst 0xc15bbe1b // bfdot za.s[x9, 3], { z16.h-z19.h }, z11.h[3]\n" "addvl x26, x26, #16\n" "33:" // Width 4: Multiply loop: multiply skip "tbz %x[flags], #1, 34f\n" "add x21, %x[args_ptr], %[offset_min]\n" "add x20, %x[args_ptr], %[offset_max]\n" - ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" - "ld1rw { z0.s }, p1/Z, [x21]\n" - ".inst 0xc0062c34 // mova { z20.d-z23.d }, za.d[x9, #1]\n" - "ld1rw { z6.s }, p1/Z, [x20]\n" - ".inst 0xc1a6c808 // fclamp { z8.s-z11.s }, z0.s, z6.s\n" - ".inst 0xc0062c50 // mova { z16.d-z19.d }, za.d[x9, #2]\n" - ".inst 0xa060c728 // st1w { z8.s-z11.s }, pn9.b, [x25]\n" - ".inst 0xc1a6c814 // fclamp { z20.s-z23.s }, z0.s, z6.s\n" - ".inst 0xc0062c78 // mova { z24.d-z27.d }, za.d[x9, #3]\n" - ".inst 0xa061c734 // st1w { z20.s-z23.s }, pn9.b, [x25, #0x4, MUL VL]\n" - ".inst 0xc1a6c810 // fclamp { z16.s-z19.s }, z0.s, z6.s\n" - ".inst 0xa062c730 // st1w { z16.s-z19.s }, pn9.b, [x25, #0x8, MUL VL]\n" - ".inst 0xc1a6c818 // fclamp { z24.s-z27.s }, z0.s, z6.s\n" - ".inst 0xa063c338 // st1w { z24.s-z27.s }, p8, [x25, #0xc, MUL VL]\n" + ".inst 0xc0062c0c // mova { z12.d-z15.d }, za.d[x9, #0]\n" + "ld1rw { z21.s }, p1/Z, [x21]\n" + ".inst 0xc0062c38 // mova { z24.d-z27.d }, za.d[x9, #1]\n" + "ld1rw { z20.s }, p1/Z, [x20]\n" + ".inst 0xc1b4caac // fclamp { z12.s-z15.s }, z21.s, z20.s\n" + ".inst 0xc0062c40 // mova { z0.d-z3.d }, za.d[x9, #2]\n" + ".inst 0xa060c72c // st1w { z12.s-z15.s }, pn9.b, [x25]\n" + ".inst 0xc1b4cab8 // fclamp { z24.s-z27.s }, z21.s, z20.s\n" + ".inst 0xc0062c70 // mova { z16.d-z19.d }, za.d[x9, #3]\n" + ".inst 0xa061c738 // st1w { z24.s-z27.s }, pn9.b, [x25, #0x4, MUL VL]\n" + ".inst 0xc1b4caa0 // fclamp { z0.s-z3.s }, z21.s, z20.s\n" + ".inst 0xa062c720 // st1w { z0.s-z3.s }, pn9.b, [x25, #0x8, MUL VL]\n" + ".inst 0xc1b4cab0 // fclamp { z16.s-z19.s }, z21.s, z20.s\n" + ".inst 0xa063c330 // st1w { z16.s-z19.s }, p8, [x25, #0xc, MUL VL]\n" "addvl x25, x25, #16\n" "b 35f\n" "34:" // Width 4: No activation - ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" - ".inst 0xa060c728 // st1w { z8.s-z11.s }, pn9.b, [x25]\n" - ".inst 0xc0062c34 // mova { z20.d-z23.d }, za.d[x9, #1]\n" - ".inst 0xa061c734 // st1w { z20.s-z23.s }, pn9.b, [x25, #0x4, MUL VL]\n" + ".inst 0xc0062c0c // mova { z12.d-z15.d }, za.d[x9, #0]\n" + ".inst 0xa060c72c // st1w { z12.s-z15.s }, pn9.b, [x25]\n" + ".inst 0xc0062c30 // mova { z16.d-z19.d }, za.d[x9, #1]\n" + ".inst 0xa061c730 // st1w { z16.s-z19.s }, pn9.b, [x25, #0x4, MUL VL]\n" ".inst 0xc0062c50 // mova { z16.d-z19.d }, za.d[x9, #2]\n" ".inst 0xa062c730 // st1w { z16.s-z19.s }, pn9.b, [x25, #0x8, MUL VL]\n" - ".inst 0xc0062c78 // mova { z24.d-z27.d }, za.d[x9, #3]\n" - ".inst 0xa063c338 // st1w { z24.s-z27.s }, p8, [x25, #0xc, MUL VL]\n" + ".inst 0xc0062c64 // mova { z4.d-z7.d }, za.d[x9, #3]\n" + ".inst 0xa063c324 // st1w { z4.s-z7.s }, p8, [x25, #0xc, MUL VL]\n" "addvl x25, x25, #16\n" "35:" // Width 4: Output done "subs x27, x27, #0x4\n" @@ -541,7 +541,7 @@ void sme2_gemv_bf16fp32_dot_16VL ( "bgt 4b\n" "36:" // Exit ".inst 0xd503467f // SMSTOP\n" - "ptrue p1.b\n" + "ptrue p8.b\n" : [N] "+&r" (N) : [A_ptr] "r" (A_ptr), [B_ptr] "r" (B_ptr), [K] "r" (K), [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [output_ptr] "r" (output_ptr) : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -550,5 +550,4 @@ void sme2_gemv_bf16fp32_dot_16VL ( } // namespace arm_gemm -#endif // ARM_COMPUTE_ENABLE_SME2 -#endif +#endif // defined(ARM_COMPUTE_ENABLE_SME2) diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32_mla_16VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32_mla_16VL.hpp index f33cb9a33d057a5f5ac61597901c68a34d3ce852..7d98d5cb98f3ee6062d3c2fa6396e71598ed5554 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32_mla_16VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32_mla_16VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,19 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - -#ifdef __aarch64__ +#if defined(ARM_COMPUTE_ENABLE_SME2) #include "../std_transforms_sme.hpp" #define ARGLIST \ @@ -83,4 +82,4 @@ public: #undef ARGLIST -#endif // __aarch64__ +#endif // defined(ARM_COMPUTE_ENABLE_SME2) diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32_mla_16VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32_mla_16VL/generic.cpp index 9224868e6ac6c77fd9f531bf2adf116d78744ffd..d2c260536d4fecd53efe40e3909a0e177338bfd2 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32_mla_16VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32_mla_16VL/generic.cpp @@ -21,8 +21,8 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifdef __ARM_FEATURE_SVE -#ifdef ARM_COMPUTE_ENABLE_SME2 + +#if defined(ARM_COMPUTE_ENABLE_SME2) #include "arm_gemm.hpp" #include "../../utils.hpp" @@ -61,7 +61,7 @@ void sme2_gemv_fp32_mla_16VL ( break; } __asm__ __volatile__( - "ptrue p1.b\n" + "ptrue p8.b\n" ".inst 0xd503477f // SMSTART ZA\n" "cntw x28, ALL, MUL #4\n" "add x27, %x[N], x28\n" @@ -101,311 +101,311 @@ void sme2_gemv_fp32_mla_16VL ( "bgt 20f\n" "beq 12f\n" "mov x23, %x[A_ptr]\n" - "lsl x22, %x[K], #0x2\n" + "lsl x21, %x[K], #0x2\n" "mov x20, %x[N]\n" - "mov x21, %x[K]\n" - ".inst 0xf8b64af8 // rprfm pldmany, x22, [x23]\n" + "mov x22, %x[K]\n" + ".inst 0xf8b54af8 // rprfm pldmany, x21, [x23]\n" ".inst 0x25b467f0 // whilelt p8.s, XZR, x20, VLx4\n" "cbz x24, 5f\n" - ".inst 0xa040c700 // ld1w { z0.s-z3.s }, pn9.b/Z, [x24]\n" - ".inst 0xc0042c00 // mova za.d[x9, #0], { z0.d-z3.d }\n" + ".inst 0xa040c718 // ld1w { z24.s-z27.s }, pn9.b/Z, [x24]\n" + ".inst 0xc0042f00 // mova za.d[x9, #0], { z24.d-z27.d }\n" "b 6f\n" "5:" // Width 1: no bias ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "6:" // Width 1: setup done - "cmp x21, #0x4\n" + "cmp x22, #0x4\n" "ble 8f\n" "7:" // Width 1: Multiply loop: Main loop head - "whilelt p0.s, XZR, x21\n" - "ld1rqw { z10.s }, p0/Z, [x23]\n" - "sub x21, x21, #0x4\n" - ".inst 0xa040c741 // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26]\n" - ".inst 0xc15aa000 // fmla za.s[x9, 0], { z0.s-z3.s }, z10.s[0]\n" + "whilelt p0.s, XZR, x22\n" + "ld1rqw { z8.s }, p0/Z, [x23]\n" + "sub x22, x22, #0x4\n" + ".inst 0xa040c755 // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x26]\n" + ".inst 0xc158a280 // fmla za.s[x9, 0], { z20.s-z23.s }, z8.s[0]\n" "addvl x26, x26, #16\n" - "cmp x21, #0x4\n" - ".inst 0xa040c759 // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26]\n" - ".inst 0xc15aa700 // fmla za.s[x9, 0], { z24.s-z27.s }, z10.s[1]\n" + "cmp x22, #0x4\n" + ".inst 0xa040c745 // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x26]\n" + ".inst 0xc158a480 // fmla za.s[x9, 0], { z4.s-z7.s }, z8.s[1]\n" "addvl x26, x26, #16\n" "add x23, x23, #0x10\n" - ".inst 0xa040c74d // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x26]\n" - ".inst 0xc15aa980 // fmla za.s[x9, 0], { z12.s-z15.s }, z10.s[2]\n" + ".inst 0xa040c75d // ldnt1w { z28.s-z31.s }, pn9.b/Z, [x26]\n" + ".inst 0xc158ab80 // fmla za.s[x9, 0], { z28.s-z31.s }, z8.s[2]\n" "addvl x26, x26, #16\n" - ".inst 0xa040c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26]\n" - ".inst 0xc15aae00 // fmla za.s[x9, 0], { z16.s-z19.s }, z10.s[3]\n" + ".inst 0xa040c759 // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26]\n" + ".inst 0xc158af00 // fmla za.s[x9, 0], { z24.s-z27.s }, z8.s[3]\n" "addvl x26, x26, #16\n" "bgt 7b\n" "8:" // Width 1: Multiply loop: Single iteration only - "whilelt p0.s, XZR, x21\n" - "ld1rqw { z10.s }, p0/Z, [x23]\n" - "subs x21, x21, #0x1\n" - ".inst 0xa040c741 // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26]\n" + "whilelt p0.s, XZR, x22\n" + "ld1rqw { z11.s }, p0/Z, [x23]\n" + "subs x22, x22, #0x1\n" + ".inst 0xa040c75d // ldnt1w { z28.s-z31.s }, pn9.b/Z, [x26]\n" "add x23, x23, #0x10\n" - ".inst 0xc15aa000 // fmla za.s[x9, 0], { z0.s-z3.s }, z10.s[0]\n" - "addvl x26, x26, #16\n" - "ble 9f\n" - ".inst 0xa040c759 // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26]\n" - "subs x21, x21, #0x1\n" - ".inst 0xc15aa700 // fmla za.s[x9, 0], { z24.s-z27.s }, z10.s[1]\n" + ".inst 0xc15ba380 // fmla za.s[x9, 0], { z28.s-z31.s }, z11.s[0]\n" "addvl x26, x26, #16\n" "ble 9f\n" ".inst 0xa040c74d // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x26]\n" - "subs x21, x21, #0x1\n" - ".inst 0xc15aa980 // fmla za.s[x9, 0], { z12.s-z15.s }, z10.s[2]\n" + "subs x22, x22, #0x1\n" + ".inst 0xc15ba580 // fmla za.s[x9, 0], { z12.s-z15.s }, z11.s[1]\n" "addvl x26, x26, #16\n" "ble 9f\n" ".inst 0xa040c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26]\n" - ".inst 0xc15aae00 // fmla za.s[x9, 0], { z16.s-z19.s }, z10.s[3]\n" + "subs x22, x22, #0x1\n" + ".inst 0xc15baa00 // fmla za.s[x9, 0], { z16.s-z19.s }, z11.s[2]\n" + "addvl x26, x26, #16\n" + "ble 9f\n" + ".inst 0xa040c741 // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26]\n" + ".inst 0xc15bac00 // fmla za.s[x9, 0], { z0.s-z3.s }, z11.s[3]\n" "addvl x26, x26, #16\n" "9:" // Width 1: Multiply loop: multiply skip "tbz %x[flags], #1, 10f\n" "add x21, %x[args_ptr], %[offset_min]\n" "add x20, %x[args_ptr], %[offset_max]\n" ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" - "ld1rw { z0.s }, p1/Z, [x21]\n" - "ld1rw { z6.s }, p1/Z, [x20]\n" - ".inst 0xc1a6c808 // fclamp { z8.s-z11.s }, z0.s, z6.s\n" + "ld1rw { z3.s }, p1/Z, [x21]\n" + "ld1rw { z29.s }, p1/Z, [x20]\n" + ".inst 0xc1bdc868 // fclamp { z8.s-z11.s }, z3.s, z29.s\n" ".inst 0xa060c328 // st1w { z8.s-z11.s }, p8, [x25]\n" "addvl x25, x25, #4\n" "b 11f\n" "10:" // Width 1: No activation - ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" - ".inst 0xa060c328 // st1w { z8.s-z11.s }, p8, [x25]\n" + ".inst 0xc0062c0c // mova { z12.d-z15.d }, za.d[x9, #0]\n" + ".inst 0xa060c32c // st1w { z12.s-z15.s }, p8, [x25]\n" "addvl x25, x25, #4\n" "11:" // Width 1: Output done "b 36f\n" "12:" // Width 2 "mov x23, %x[A_ptr]\n" - "lsl x22, %x[K], #0x2\n" + "lsl x21, %x[K], #0x2\n" "sub x20, %x[N], x28\n" - "mov x21, %x[K]\n" - ".inst 0xf8b64af8 // rprfm pldmany, x22, [x23]\n" + "mov x22, %x[K]\n" + ".inst 0xf8b54af8 // rprfm pldmany, x21, [x23]\n" ".inst 0x25b467f0 // whilelt p8.s, XZR, x20, VLx4\n" "cbz x24, 13f\n" - ".inst 0xa040c700 // ld1w { z0.s-z3.s }, pn9.b/Z, [x24]\n" - ".inst 0xc0042c00 // mova za.d[x9, #0], { z0.d-z3.d }\n" - ".inst 0xa041c708 // ld1w { z8.s-z11.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n" - ".inst 0xc0042d01 // mova za.d[x9, #1], { z8.d-z11.d }\n" + ".inst 0xa040c704 // ld1w { z4.s-z7.s }, pn9.b/Z, [x24]\n" + ".inst 0xc0042c80 // mova za.d[x9, #0], { z4.d-z7.d }\n" + ".inst 0xa041c714 // ld1w { z20.s-z23.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n" + ".inst 0xc0042e81 // mova za.d[x9, #1], { z20.d-z23.d }\n" "b 14f\n" "13:" // Width 2: no bias ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "14:" // Width 2: setup done - "cmp x21, #0x4\n" + "cmp x22, #0x4\n" "ble 16f\n" "15:" // Width 2: Multiply loop: Main loop head - "whilelt p0.s, XZR, x21\n" - "ld1rqw { z10.s }, p0/Z, [x23]\n" - "sub x21, x21, #0x4\n" - ".inst 0xa040c741 // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26]\n" - ".inst 0xc15aa000 // fmla za.s[x9, 0], { z0.s-z3.s }, z10.s[0]\n" - "cmp x21, #0x4\n" + "whilelt p0.s, XZR, x22\n" + "ld1rqw { z9.s }, p0/Z, [x23]\n" + "sub x22, x22, #0x4\n" + ".inst 0xa040c74d // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x26]\n" + ".inst 0xc159a180 // fmla za.s[x9, 0], { z12.s-z15.s }, z9.s[0]\n" + "cmp x22, #0x4\n" "add x23, x23, #0x10\n" ".inst 0xa041c745 // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aa081 // fmla za.s[x9, 1], { z4.s-z7.s }, z10.s[0]\n" - "addvl x26, x26, #16\n" - ".inst 0xa040c759 // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26]\n" - ".inst 0xc15aa700 // fmla za.s[x9, 0], { z24.s-z27.s }, z10.s[1]\n" - ".inst 0xa041c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aa601 // fmla za.s[x9, 1], { z16.s-z19.s }, z10.s[1]\n" + ".inst 0xc159a081 // fmla za.s[x9, 1], { z4.s-z7.s }, z9.s[0]\n" "addvl x26, x26, #16\n" ".inst 0xa040c74d // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x26]\n" - ".inst 0xc15aa980 // fmla za.s[x9, 0], { z12.s-z15.s }, z10.s[2]\n" - ".inst 0xa041c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aaa01 // fmla za.s[x9, 1], { z16.s-z19.s }, z10.s[2]\n" + ".inst 0xc159a580 // fmla za.s[x9, 0], { z12.s-z15.s }, z9.s[1]\n" + ".inst 0xa041c745 // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc159a481 // fmla za.s[x9, 1], { z4.s-z7.s }, z9.s[1]\n" "addvl x26, x26, #16\n" - ".inst 0xa040c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26]\n" - ".inst 0xc15aae00 // fmla za.s[x9, 0], { z16.s-z19.s }, z10.s[3]\n" - ".inst 0xa041c759 // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aaf01 // fmla za.s[x9, 1], { z24.s-z27.s }, z10.s[3]\n" + ".inst 0xa040c759 // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26]\n" + ".inst 0xc159ab00 // fmla za.s[x9, 0], { z24.s-z27.s }, z9.s[2]\n" + ".inst 0xa041c741 // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc159a801 // fmla za.s[x9, 1], { z0.s-z3.s }, z9.s[2]\n" + "addvl x26, x26, #16\n" + ".inst 0xa040c741 // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26]\n" + ".inst 0xc159ac00 // fmla za.s[x9, 0], { z0.s-z3.s }, z9.s[3]\n" + ".inst 0xa041c75d // ldnt1w { z28.s-z31.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc159af81 // fmla za.s[x9, 1], { z28.s-z31.s }, z9.s[3]\n" "addvl x26, x26, #16\n" "bgt 15b\n" "16:" // Width 2: Multiply loop: Single iteration only - "whilelt p0.s, XZR, x21\n" - "ld1rqw { z10.s }, p0/Z, [x23]\n" - "subs x21, x21, #0x1\n" - ".inst 0xa040c741 // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26]\n" + "whilelt p0.s, XZR, x22\n" + "ld1rqw { z11.s }, p0/Z, [x23]\n" + "subs x22, x22, #0x1\n" + ".inst 0xa040c74d // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x26]\n" "add x23, x23, #0x10\n" - ".inst 0xc15aa000 // fmla za.s[x9, 0], { z0.s-z3.s }, z10.s[0]\n" - ".inst 0xa041c745 // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aa081 // fmla za.s[x9, 1], { z4.s-z7.s }, z10.s[0]\n" + ".inst 0xc15ba180 // fmla za.s[x9, 0], { z12.s-z15.s }, z11.s[0]\n" + ".inst 0xa041c741 // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15ba001 // fmla za.s[x9, 1], { z0.s-z3.s }, z11.s[0]\n" "addvl x26, x26, #16\n" "ble 17f\n" ".inst 0xa040c759 // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26]\n" - "subs x21, x21, #0x1\n" - ".inst 0xc15aa700 // fmla za.s[x9, 0], { z24.s-z27.s }, z10.s[1]\n" - ".inst 0xa041c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aa601 // fmla za.s[x9, 1], { z16.s-z19.s }, z10.s[1]\n" + "subs x22, x22, #0x1\n" + ".inst 0xc15ba700 // fmla za.s[x9, 0], { z24.s-z27.s }, z11.s[1]\n" + ".inst 0xa041c741 // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15ba401 // fmla za.s[x9, 1], { z0.s-z3.s }, z11.s[1]\n" "addvl x26, x26, #16\n" "ble 17f\n" ".inst 0xa040c74d // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x26]\n" - "subs x21, x21, #0x1\n" - ".inst 0xc15aa980 // fmla za.s[x9, 0], { z12.s-z15.s }, z10.s[2]\n" - ".inst 0xa041c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aaa01 // fmla za.s[x9, 1], { z16.s-z19.s }, z10.s[2]\n" + "subs x22, x22, #0x1\n" + ".inst 0xc15ba980 // fmla za.s[x9, 0], { z12.s-z15.s }, z11.s[2]\n" + ".inst 0xa041c75d // ldnt1w { z28.s-z31.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15bab81 // fmla za.s[x9, 1], { z28.s-z31.s }, z11.s[2]\n" "addvl x26, x26, #16\n" "ble 17f\n" ".inst 0xa040c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26]\n" - ".inst 0xc15aae00 // fmla za.s[x9, 0], { z16.s-z19.s }, z10.s[3]\n" - ".inst 0xa041c759 // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aaf01 // fmla za.s[x9, 1], { z24.s-z27.s }, z10.s[3]\n" + ".inst 0xc15bae00 // fmla za.s[x9, 0], { z16.s-z19.s }, z11.s[3]\n" + ".inst 0xa041c755 // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15bae81 // fmla za.s[x9, 1], { z20.s-z23.s }, z11.s[3]\n" "addvl x26, x26, #16\n" "17:" // Width 2: Multiply loop: multiply skip "tbz %x[flags], #1, 18f\n" "add x21, %x[args_ptr], %[offset_min]\n" "add x20, %x[args_ptr], %[offset_max]\n" - ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" - "ld1rw { z0.s }, p1/Z, [x21]\n" - ".inst 0xc0062c34 // mova { z20.d-z23.d }, za.d[x9, #1]\n" - "ld1rw { z6.s }, p1/Z, [x20]\n" - ".inst 0xc1a6c808 // fclamp { z8.s-z11.s }, z0.s, z6.s\n" - ".inst 0xa060c728 // st1w { z8.s-z11.s }, pn9.b, [x25]\n" - ".inst 0xc1a6c814 // fclamp { z20.s-z23.s }, z0.s, z6.s\n" - ".inst 0xa061c334 // st1w { z20.s-z23.s }, p8, [x25, #0x4, MUL VL]\n" + ".inst 0xc0062c00 // mova { z0.d-z3.d }, za.d[x9, #0]\n" + "ld1rw { z9.s }, p1/Z, [x21]\n" + ".inst 0xc0062c24 // mova { z4.d-z7.d }, za.d[x9, #1]\n" + "ld1rw { z8.s }, p1/Z, [x20]\n" + ".inst 0xc1a8c920 // fclamp { z0.s-z3.s }, z9.s, z8.s\n" + ".inst 0xa060c720 // st1w { z0.s-z3.s }, pn9.b, [x25]\n" + ".inst 0xc1a8c924 // fclamp { z4.s-z7.s }, z9.s, z8.s\n" + ".inst 0xa061c324 // st1w { z4.s-z7.s }, p8, [x25, #0x4, MUL VL]\n" "addvl x25, x25, #8\n" "b 19f\n" "18:" // Width 2: No activation - ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" - ".inst 0xa060c728 // st1w { z8.s-z11.s }, pn9.b, [x25]\n" - ".inst 0xc0062c34 // mova { z20.d-z23.d }, za.d[x9, #1]\n" - ".inst 0xa061c334 // st1w { z20.s-z23.s }, p8, [x25, #0x4, MUL VL]\n" + ".inst 0xc0062c10 // mova { z16.d-z19.d }, za.d[x9, #0]\n" + ".inst 0xa060c730 // st1w { z16.s-z19.s }, pn9.b, [x25]\n" + ".inst 0xc0062c2c // mova { z12.d-z15.d }, za.d[x9, #1]\n" + ".inst 0xa061c32c // st1w { z12.s-z15.s }, p8, [x25, #0x4, MUL VL]\n" "addvl x25, x25, #8\n" "19:" // Width 2: Output done "b 36f\n" "20:" // Width 3 "mov x20, #0x2\n" "mov x23, %x[A_ptr]\n" - "lsl x22, %x[K], #0x2\n" + "lsl x21, %x[K], #0x2\n" "msub x20, x28, x20, %x[N]\n" - "mov x21, %x[K]\n" - ".inst 0xf8b64af8 // rprfm pldmany, x22, [x23]\n" + "mov x22, %x[K]\n" + ".inst 0xf8b54af8 // rprfm pldmany, x21, [x23]\n" ".inst 0x25b467f0 // whilelt p8.s, XZR, x20, VLx4\n" "cbz x24, 21f\n" - ".inst 0xa040c700 // ld1w { z0.s-z3.s }, pn9.b/Z, [x24]\n" - ".inst 0xc0042c00 // mova za.d[x9, #0], { z0.d-z3.d }\n" - ".inst 0xa041c708 // ld1w { z8.s-z11.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n" - ".inst 0xc0042d01 // mova za.d[x9, #1], { z8.d-z11.d }\n" - ".inst 0xa042c704 // ld1w { z4.s-z7.s }, pn9.b/Z, [x24, #0x8, MUL VL]\n" - ".inst 0xc0042c82 // mova za.d[x9, #2], { z4.d-z7.d }\n" + ".inst 0xa040c718 // ld1w { z24.s-z27.s }, pn9.b/Z, [x24]\n" + ".inst 0xc0042f00 // mova za.d[x9, #0], { z24.d-z27.d }\n" + ".inst 0xa041c704 // ld1w { z4.s-z7.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n" + ".inst 0xc0042c81 // mova za.d[x9, #1], { z4.d-z7.d }\n" + ".inst 0xa042c710 // ld1w { z16.s-z19.s }, pn9.b/Z, [x24, #0x8, MUL VL]\n" + ".inst 0xc0042e02 // mova za.d[x9, #2], { z16.d-z19.d }\n" "b 22f\n" "21:" // Width 3: no bias ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "22:" // Width 3: setup done - "cmp x21, #0x4\n" + "cmp x22, #0x4\n" "ble 24f\n" "23:" // Width 3: Multiply loop: Main loop head - "whilelt p0.s, XZR, x21\n" - "ld1rqw { z10.s }, p0/Z, [x23]\n" - "sub x21, x21, #0x4\n" + "whilelt p0.s, XZR, x22\n" + "ld1rqw { z15.s }, p0/Z, [x23]\n" + "sub x22, x22, #0x4\n" ".inst 0xa040c741 // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26]\n" - ".inst 0xc15aa000 // fmla za.s[x9, 0], { z0.s-z3.s }, z10.s[0]\n" - "cmp x21, #0x4\n" + ".inst 0xc15fa000 // fmla za.s[x9, 0], { z0.s-z3.s }, z15.s[0]\n" + "cmp x22, #0x4\n" "add x23, x23, #0x10\n" ".inst 0xa041c745 // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aa081 // fmla za.s[x9, 1], { z4.s-z7.s }, z10.s[0]\n" - ".inst 0xa042c755 // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15aa282 // fmla za.s[x9, 2], { z20.s-z23.s }, z10.s[0]\n" + ".inst 0xc15fa081 // fmla za.s[x9, 1], { z4.s-z7.s }, z15.s[0]\n" + ".inst 0xa042c741 // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc15fa002 // fmla za.s[x9, 2], { z0.s-z3.s }, z15.s[0]\n" "addvl x26, x26, #16\n" - ".inst 0xa040c759 // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26]\n" - ".inst 0xc15aa700 // fmla za.s[x9, 0], { z24.s-z27.s }, z10.s[1]\n" - ".inst 0xa041c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aa601 // fmla za.s[x9, 1], { z16.s-z19.s }, z10.s[1]\n" - ".inst 0xa042c759 // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15aa702 // fmla za.s[x9, 2], { z24.s-z27.s }, z10.s[1]\n" + ".inst 0xa040c755 // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x26]\n" + ".inst 0xc15fa680 // fmla za.s[x9, 0], { z20.s-z23.s }, z15.s[1]\n" + ".inst 0xa041c755 // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15fa681 // fmla za.s[x9, 1], { z20.s-z23.s }, z15.s[1]\n" + ".inst 0xa042c749 // ldnt1w { z8.s-z11.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc15fa502 // fmla za.s[x9, 2], { z8.s-z11.s }, z15.s[1]\n" "addvl x26, x26, #16\n" - ".inst 0xa040c74d // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x26]\n" - ".inst 0xc15aa980 // fmla za.s[x9, 0], { z12.s-z15.s }, z10.s[2]\n" - ".inst 0xa041c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aaa01 // fmla za.s[x9, 1], { z16.s-z19.s }, z10.s[2]\n" - ".inst 0xa042c75d // ldnt1w { z28.s-z31.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15aab82 // fmla za.s[x9, 2], { z28.s-z31.s }, z10.s[2]\n" + ".inst 0xa040c759 // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26]\n" + ".inst 0xc15fab00 // fmla za.s[x9, 0], { z24.s-z27.s }, z15.s[2]\n" + ".inst 0xa041c749 // ldnt1w { z8.s-z11.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15fa901 // fmla za.s[x9, 1], { z8.s-z11.s }, z15.s[2]\n" + ".inst 0xa042c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc15faa02 // fmla za.s[x9, 2], { z16.s-z19.s }, z15.s[2]\n" "addvl x26, x26, #16\n" ".inst 0xa040c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26]\n" - ".inst 0xc15aae00 // fmla za.s[x9, 0], { z16.s-z19.s }, z10.s[3]\n" - ".inst 0xa041c759 // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aaf01 // fmla za.s[x9, 1], { z24.s-z27.s }, z10.s[3]\n" - ".inst 0xa042c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15aae02 // fmla za.s[x9, 2], { z16.s-z19.s }, z10.s[3]\n" + ".inst 0xc15fae00 // fmla za.s[x9, 0], { z16.s-z19.s }, z15.s[3]\n" + ".inst 0xa041c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15fae01 // fmla za.s[x9, 1], { z16.s-z19.s }, z15.s[3]\n" + ".inst 0xa042c749 // ldnt1w { z8.s-z11.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc15fad02 // fmla za.s[x9, 2], { z8.s-z11.s }, z15.s[3]\n" "addvl x26, x26, #16\n" "bgt 23b\n" "24:" // Width 3: Multiply loop: Single iteration only - "whilelt p0.s, XZR, x21\n" - "ld1rqw { z10.s }, p0/Z, [x23]\n" - "subs x21, x21, #0x1\n" - ".inst 0xa040c741 // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26]\n" + "whilelt p0.s, XZR, x22\n" + "ld1rqw { z11.s }, p0/Z, [x23]\n" + "subs x22, x22, #0x1\n" + ".inst 0xa040c75d // ldnt1w { z28.s-z31.s }, pn9.b/Z, [x26]\n" "add x23, x23, #0x10\n" - ".inst 0xc15aa000 // fmla za.s[x9, 0], { z0.s-z3.s }, z10.s[0]\n" - ".inst 0xa041c745 // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aa081 // fmla za.s[x9, 1], { z4.s-z7.s }, z10.s[0]\n" + ".inst 0xc15ba380 // fmla za.s[x9, 0], { z28.s-z31.s }, z11.s[0]\n" + ".inst 0xa041c741 // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15ba001 // fmla za.s[x9, 1], { z0.s-z3.s }, z11.s[0]\n" ".inst 0xa042c755 // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15aa282 // fmla za.s[x9, 2], { z20.s-z23.s }, z10.s[0]\n" - "addvl x26, x26, #16\n" - "ble 25f\n" - ".inst 0xa040c759 // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26]\n" - "subs x21, x21, #0x1\n" - ".inst 0xc15aa700 // fmla za.s[x9, 0], { z24.s-z27.s }, z10.s[1]\n" - ".inst 0xa041c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aa601 // fmla za.s[x9, 1], { z16.s-z19.s }, z10.s[1]\n" - ".inst 0xa042c759 // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15aa702 // fmla za.s[x9, 2], { z24.s-z27.s }, z10.s[1]\n" + ".inst 0xc15ba282 // fmla za.s[x9, 2], { z20.s-z23.s }, z11.s[0]\n" "addvl x26, x26, #16\n" "ble 25f\n" ".inst 0xa040c74d // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x26]\n" - "subs x21, x21, #0x1\n" - ".inst 0xc15aa980 // fmla za.s[x9, 0], { z12.s-z15.s }, z10.s[2]\n" - ".inst 0xa041c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aaa01 // fmla za.s[x9, 1], { z16.s-z19.s }, z10.s[2]\n" + "subs x22, x22, #0x1\n" + ".inst 0xc15ba580 // fmla za.s[x9, 0], { z12.s-z15.s }, z11.s[1]\n" + ".inst 0xa041c745 // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15ba481 // fmla za.s[x9, 1], { z4.s-z7.s }, z11.s[1]\n" ".inst 0xa042c75d // ldnt1w { z28.s-z31.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15aab82 // fmla za.s[x9, 2], { z28.s-z31.s }, z10.s[2]\n" + ".inst 0xc15ba782 // fmla za.s[x9, 2], { z28.s-z31.s }, z11.s[1]\n" "addvl x26, x26, #16\n" "ble 25f\n" - ".inst 0xa040c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26]\n" - ".inst 0xc15aae00 // fmla za.s[x9, 0], { z16.s-z19.s }, z10.s[3]\n" - ".inst 0xa041c759 // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aaf01 // fmla za.s[x9, 1], { z24.s-z27.s }, z10.s[3]\n" + ".inst 0xa040c745 // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x26]\n" + "subs x22, x22, #0x1\n" + ".inst 0xc15ba880 // fmla za.s[x9, 0], { z4.s-z7.s }, z11.s[2]\n" + ".inst 0xa041c755 // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15baa81 // fmla za.s[x9, 1], { z20.s-z23.s }, z11.s[2]\n" + ".inst 0xa042c74d // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc15ba982 // fmla za.s[x9, 2], { z12.s-z15.s }, z11.s[2]\n" + "addvl x26, x26, #16\n" + "ble 25f\n" + ".inst 0xa040c74d // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x26]\n" + ".inst 0xc15bad80 // fmla za.s[x9, 0], { z12.s-z15.s }, z11.s[3]\n" + ".inst 0xa041c755 // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15bae81 // fmla za.s[x9, 1], { z20.s-z23.s }, z11.s[3]\n" ".inst 0xa042c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15aae02 // fmla za.s[x9, 2], { z16.s-z19.s }, z10.s[3]\n" + ".inst 0xc15bae02 // fmla za.s[x9, 2], { z16.s-z19.s }, z11.s[3]\n" "addvl x26, x26, #16\n" "25:" // Width 3: Multiply loop: multiply skip "tbz %x[flags], #1, 26f\n" "add x21, %x[args_ptr], %[offset_min]\n" "add x20, %x[args_ptr], %[offset_max]\n" - ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" - "ld1rw { z0.s }, p1/Z, [x21]\n" - ".inst 0xc0062c34 // mova { z20.d-z23.d }, za.d[x9, #1]\n" - "ld1rw { z6.s }, p1/Z, [x20]\n" - ".inst 0xc1a6c808 // fclamp { z8.s-z11.s }, z0.s, z6.s\n" - ".inst 0xc0062c50 // mova { z16.d-z19.d }, za.d[x9, #2]\n" - ".inst 0xa060c728 // st1w { z8.s-z11.s }, pn9.b, [x25]\n" - ".inst 0xc1a6c814 // fclamp { z20.s-z23.s }, z0.s, z6.s\n" - ".inst 0xa061c734 // st1w { z20.s-z23.s }, pn9.b, [x25, #0x4, MUL VL]\n" - ".inst 0xc1a6c810 // fclamp { z16.s-z19.s }, z0.s, z6.s\n" - ".inst 0xa062c330 // st1w { z16.s-z19.s }, p8, [x25, #0x8, MUL VL]\n" + ".inst 0xc0062c04 // mova { z4.d-z7.d }, za.d[x9, #0]\n" + "ld1rw { z17.s }, p1/Z, [x21]\n" + ".inst 0xc0062c28 // mova { z8.d-z11.d }, za.d[x9, #1]\n" + "ld1rw { z16.s }, p1/Z, [x20]\n" + ".inst 0xc1b0ca24 // fclamp { z4.s-z7.s }, z17.s, z16.s\n" + ".inst 0xc0062c4c // mova { z12.d-z15.d }, za.d[x9, #2]\n" + ".inst 0xa060c724 // st1w { z4.s-z7.s }, pn9.b, [x25]\n" + ".inst 0xc1b0ca28 // fclamp { z8.s-z11.s }, z17.s, z16.s\n" + ".inst 0xa061c728 // st1w { z8.s-z11.s }, pn9.b, [x25, #0x4, MUL VL]\n" + ".inst 0xc1b0ca2c // fclamp { z12.s-z15.s }, z17.s, z16.s\n" + ".inst 0xa062c32c // st1w { z12.s-z15.s }, p8, [x25, #0x8, MUL VL]\n" "addvl x25, x25, #12\n" "b 27f\n" "26:" // Width 3: No activation - ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" - ".inst 0xa060c728 // st1w { z8.s-z11.s }, pn9.b, [x25]\n" - ".inst 0xc0062c34 // mova { z20.d-z23.d }, za.d[x9, #1]\n" - ".inst 0xa061c734 // st1w { z20.s-z23.s }, pn9.b, [x25, #0x4, MUL VL]\n" - ".inst 0xc0062c50 // mova { z16.d-z19.d }, za.d[x9, #2]\n" - ".inst 0xa062c330 // st1w { z16.s-z19.s }, p8, [x25, #0x8, MUL VL]\n" + ".inst 0xc0062c14 // mova { z20.d-z23.d }, za.d[x9, #0]\n" + ".inst 0xa060c734 // st1w { z20.s-z23.s }, pn9.b, [x25]\n" + ".inst 0xc0062c2c // mova { z12.d-z15.d }, za.d[x9, #1]\n" + ".inst 0xa061c72c // st1w { z12.s-z15.s }, pn9.b, [x25, #0x4, MUL VL]\n" + ".inst 0xc0062c4c // mova { z12.d-z15.d }, za.d[x9, #2]\n" + ".inst 0xa062c32c // st1w { z12.s-z15.s }, p8, [x25, #0x8, MUL VL]\n" "addvl x25, x25, #12\n" "27:" // Width 3: Output done "b 36f\n" "28:" // Width 4 "mov x20, #0x3\n" "mov x23, %x[A_ptr]\n" - "lsl x22, %x[K], #0x2\n" + "lsl x21, %x[K], #0x2\n" "msub x20, x28, x20, %x[N]\n" - "mov x21, %x[K]\n" - ".inst 0xf8b64af8 // rprfm pldmany, x22, [x23]\n" + "mov x22, %x[K]\n" + ".inst 0xf8b54af8 // rprfm pldmany, x21, [x23]\n" ".inst 0x25b467f0 // whilelt p8.s, XZR, x20, VLx4\n" "cbz x24, 29f\n" - ".inst 0xa040c700 // ld1w { z0.s-z3.s }, pn9.b/Z, [x24]\n" - ".inst 0xc0042c00 // mova za.d[x9, #0], { z0.d-z3.d }\n" - ".inst 0xa041c708 // ld1w { z8.s-z11.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n" - ".inst 0xc0042d01 // mova za.d[x9, #1], { z8.d-z11.d }\n" - ".inst 0xa042c704 // ld1w { z4.s-z7.s }, pn9.b/Z, [x24, #0x8, MUL VL]\n" - ".inst 0xc0042c82 // mova za.d[x9, #2], { z4.d-z7.d }\n" + ".inst 0xa040c704 // ld1w { z4.s-z7.s }, pn9.b/Z, [x24]\n" + ".inst 0xc0042c80 // mova za.d[x9, #0], { z4.d-z7.d }\n" + ".inst 0xa041c710 // ld1w { z16.s-z19.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n" + ".inst 0xc0042e01 // mova za.d[x9, #1], { z16.d-z19.d }\n" + ".inst 0xa042c70c // ld1w { z12.s-z15.s }, pn9.b/Z, [x24, #0x8, MUL VL]\n" + ".inst 0xc0042d82 // mova za.d[x9, #2], { z12.d-z15.d }\n" ".inst 0xa043c710 // ld1w { z16.s-z19.s }, pn9.b/Z, [x24, #0xc, MUL VL]\n" ".inst 0xc0042e03 // mova za.d[x9, #3], { z16.d-z19.d }\n" "addvl x24, x24, #16\n" @@ -413,126 +413,126 @@ void sme2_gemv_fp32_mla_16VL ( "29:" // Width 4: no bias ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "30:" // Width 4: setup done - "cmp x21, #0x4\n" + "cmp x22, #0x4\n" "ble 32f\n" "31:" // Width 4: Multiply loop: Main loop head - "whilelt p0.s, XZR, x21\n" - "ld1rqw { z10.s }, p0/Z, [x23]\n" - "sub x21, x21, #0x4\n" - ".inst 0xa040c741 // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26]\n" - ".inst 0xc15aa000 // fmla za.s[x9, 0], { z0.s-z3.s }, z10.s[0]\n" - "cmp x21, #0x4\n" + "whilelt p0.s, XZR, x22\n" + "ld1rqw { z8.s }, p0/Z, [x23]\n" + "sub x22, x22, #0x4\n" + ".inst 0xa040c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26]\n" + ".inst 0xc158a200 // fmla za.s[x9, 0], { z16.s-z19.s }, z8.s[0]\n" + "cmp x22, #0x4\n" "add x23, x23, #0x10\n" - ".inst 0xa041c745 // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aa081 // fmla za.s[x9, 1], { z4.s-z7.s }, z10.s[0]\n" - ".inst 0xa042c755 // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15aa282 // fmla za.s[x9, 2], { z20.s-z23.s }, z10.s[0]\n" - ".inst 0xa043c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc15aa203 // fmla za.s[x9, 3], { z16.s-z19.s }, z10.s[0]\n" - "addvl x26, x26, #16\n" - ".inst 0xa040c759 // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26]\n" - ".inst 0xc15aa700 // fmla za.s[x9, 0], { z24.s-z27.s }, z10.s[1]\n" - ".inst 0xa041c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aa601 // fmla za.s[x9, 1], { z16.s-z19.s }, z10.s[1]\n" - ".inst 0xa042c759 // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15aa702 // fmla za.s[x9, 2], { z24.s-z27.s }, z10.s[1]\n" - ".inst 0xa043c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc15aa603 // fmla za.s[x9, 3], { z16.s-z19.s }, z10.s[1]\n" + ".inst 0xa041c74d // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc158a181 // fmla za.s[x9, 1], { z12.s-z15.s }, z8.s[0]\n" + ".inst 0xa042c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc158a202 // fmla za.s[x9, 2], { z16.s-z19.s }, z8.s[0]\n" + ".inst 0xa043c74d // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc158a183 // fmla za.s[x9, 3], { z12.s-z15.s }, z8.s[0]\n" "addvl x26, x26, #16\n" ".inst 0xa040c74d // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x26]\n" - ".inst 0xc15aa980 // fmla za.s[x9, 0], { z12.s-z15.s }, z10.s[2]\n" - ".inst 0xa041c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aaa01 // fmla za.s[x9, 1], { z16.s-z19.s }, z10.s[2]\n" - ".inst 0xa042c75d // ldnt1w { z28.s-z31.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15aab82 // fmla za.s[x9, 2], { z28.s-z31.s }, z10.s[2]\n" + ".inst 0xc158a580 // fmla za.s[x9, 0], { z12.s-z15.s }, z8.s[1]\n" + ".inst 0xa041c755 // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc158a681 // fmla za.s[x9, 1], { z20.s-z23.s }, z8.s[1]\n" + ".inst 0xa042c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc158a602 // fmla za.s[x9, 2], { z16.s-z19.s }, z8.s[1]\n" ".inst 0xa043c755 // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc15aaa83 // fmla za.s[x9, 3], { z20.s-z23.s }, z10.s[2]\n" + ".inst 0xc158a683 // fmla za.s[x9, 3], { z20.s-z23.s }, z8.s[1]\n" "addvl x26, x26, #16\n" - ".inst 0xa040c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26]\n" - ".inst 0xc15aae00 // fmla za.s[x9, 0], { z16.s-z19.s }, z10.s[3]\n" - ".inst 0xa041c759 // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aaf01 // fmla za.s[x9, 1], { z24.s-z27.s }, z10.s[3]\n" + ".inst 0xa040c745 // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x26]\n" + ".inst 0xc158a880 // fmla za.s[x9, 0], { z4.s-z7.s }, z8.s[2]\n" + ".inst 0xa041c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc158aa01 // fmla za.s[x9, 1], { z16.s-z19.s }, z8.s[2]\n" ".inst 0xa042c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15aae02 // fmla za.s[x9, 2], { z16.s-z19.s }, z10.s[3]\n" - ".inst 0xa043c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc15aae03 // fmla za.s[x9, 3], { z16.s-z19.s }, z10.s[3]\n" + ".inst 0xc158aa02 // fmla za.s[x9, 2], { z16.s-z19.s }, z8.s[2]\n" + ".inst 0xa043c741 // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc158a803 // fmla za.s[x9, 3], { z0.s-z3.s }, z8.s[2]\n" + "addvl x26, x26, #16\n" + ".inst 0xa040c755 // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x26]\n" + ".inst 0xc158ae80 // fmla za.s[x9, 0], { z20.s-z23.s }, z8.s[3]\n" + ".inst 0xa041c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc158ae01 // fmla za.s[x9, 1], { z16.s-z19.s }, z8.s[3]\n" + ".inst 0xa042c745 // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc158ac82 // fmla za.s[x9, 2], { z4.s-z7.s }, z8.s[3]\n" + ".inst 0xa043c755 // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc158ae83 // fmla za.s[x9, 3], { z20.s-z23.s }, z8.s[3]\n" "addvl x26, x26, #16\n" "bgt 31b\n" "32:" // Width 4: Multiply loop: Single iteration only - "whilelt p0.s, XZR, x21\n" - "ld1rqw { z10.s }, p0/Z, [x23]\n" - "subs x21, x21, #0x1\n" - ".inst 0xa040c741 // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26]\n" + "whilelt p0.s, XZR, x22\n" + "ld1rqw { z11.s }, p0/Z, [x23]\n" + "subs x22, x22, #0x1\n" + ".inst 0xa040c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26]\n" "add x23, x23, #0x10\n" - ".inst 0xc15aa000 // fmla za.s[x9, 0], { z0.s-z3.s }, z10.s[0]\n" - ".inst 0xa041c745 // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aa081 // fmla za.s[x9, 1], { z4.s-z7.s }, z10.s[0]\n" - ".inst 0xa042c755 // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15aa282 // fmla za.s[x9, 2], { z20.s-z23.s }, z10.s[0]\n" + ".inst 0xc15ba200 // fmla za.s[x9, 0], { z16.s-z19.s }, z11.s[0]\n" + ".inst 0xa041c755 // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15ba281 // fmla za.s[x9, 1], { z20.s-z23.s }, z11.s[0]\n" + ".inst 0xa042c75d // ldnt1w { z28.s-z31.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc15ba382 // fmla za.s[x9, 2], { z28.s-z31.s }, z11.s[0]\n" ".inst 0xa043c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc15aa203 // fmla za.s[x9, 3], { z16.s-z19.s }, z10.s[0]\n" + ".inst 0xc15ba203 // fmla za.s[x9, 3], { z16.s-z19.s }, z11.s[0]\n" "addvl x26, x26, #16\n" "ble 33f\n" - ".inst 0xa040c759 // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26]\n" - "subs x21, x21, #0x1\n" - ".inst 0xc15aa700 // fmla za.s[x9, 0], { z24.s-z27.s }, z10.s[1]\n" + ".inst 0xa040c741 // ldnt1w { z0.s-z3.s }, pn9.b/Z, [x26]\n" + "subs x22, x22, #0x1\n" + ".inst 0xc15ba400 // fmla za.s[x9, 0], { z0.s-z3.s }, z11.s[1]\n" ".inst 0xa041c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aa601 // fmla za.s[x9, 1], { z16.s-z19.s }, z10.s[1]\n" - ".inst 0xa042c759 // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15aa702 // fmla za.s[x9, 2], { z24.s-z27.s }, z10.s[1]\n" + ".inst 0xc15ba601 // fmla za.s[x9, 1], { z16.s-z19.s }, z11.s[1]\n" + ".inst 0xa042c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc15ba602 // fmla za.s[x9, 2], { z16.s-z19.s }, z11.s[1]\n" ".inst 0xa043c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc15aa603 // fmla za.s[x9, 3], { z16.s-z19.s }, z10.s[1]\n" + ".inst 0xc15ba603 // fmla za.s[x9, 3], { z16.s-z19.s }, z11.s[1]\n" "addvl x26, x26, #16\n" "ble 33f\n" - ".inst 0xa040c74d // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x26]\n" - "subs x21, x21, #0x1\n" - ".inst 0xc15aa980 // fmla za.s[x9, 0], { z12.s-z15.s }, z10.s[2]\n" + ".inst 0xa040c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26]\n" + "subs x22, x22, #0x1\n" + ".inst 0xc15baa00 // fmla za.s[x9, 0], { z16.s-z19.s }, z11.s[2]\n" ".inst 0xa041c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aaa01 // fmla za.s[x9, 1], { z16.s-z19.s }, z10.s[2]\n" - ".inst 0xa042c75d // ldnt1w { z28.s-z31.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15aab82 // fmla za.s[x9, 2], { z28.s-z31.s }, z10.s[2]\n" - ".inst 0xa043c755 // ldnt1w { z20.s-z23.s }, pn9.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc15aaa83 // fmla za.s[x9, 3], { z20.s-z23.s }, z10.s[2]\n" + ".inst 0xc15baa01 // fmla za.s[x9, 1], { z16.s-z19.s }, z11.s[2]\n" + ".inst 0xa042c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc15baa02 // fmla za.s[x9, 2], { z16.s-z19.s }, z11.s[2]\n" + ".inst 0xa043c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc15baa03 // fmla za.s[x9, 3], { z16.s-z19.s }, z11.s[2]\n" "addvl x26, x26, #16\n" "ble 33f\n" ".inst 0xa040c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26]\n" - ".inst 0xc15aae00 // fmla za.s[x9, 0], { z16.s-z19.s }, z10.s[3]\n" + ".inst 0xc15bae00 // fmla za.s[x9, 0], { z16.s-z19.s }, z11.s[3]\n" ".inst 0xa041c759 // ldnt1w { z24.s-z27.s }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc15aaf01 // fmla za.s[x9, 1], { z24.s-z27.s }, z10.s[3]\n" + ".inst 0xc15baf01 // fmla za.s[x9, 1], { z24.s-z27.s }, z11.s[3]\n" ".inst 0xa042c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc15aae02 // fmla za.s[x9, 2], { z16.s-z19.s }, z10.s[3]\n" + ".inst 0xc15bae02 // fmla za.s[x9, 2], { z16.s-z19.s }, z11.s[3]\n" ".inst 0xa043c751 // ldnt1w { z16.s-z19.s }, pn9.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc15aae03 // fmla za.s[x9, 3], { z16.s-z19.s }, z10.s[3]\n" + ".inst 0xc15bae03 // fmla za.s[x9, 3], { z16.s-z19.s }, z11.s[3]\n" "addvl x26, x26, #16\n" "33:" // Width 4: Multiply loop: multiply skip "tbz %x[flags], #1, 34f\n" "add x21, %x[args_ptr], %[offset_min]\n" "add x20, %x[args_ptr], %[offset_max]\n" - ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" - "ld1rw { z0.s }, p1/Z, [x21]\n" - ".inst 0xc0062c34 // mova { z20.d-z23.d }, za.d[x9, #1]\n" - "ld1rw { z6.s }, p1/Z, [x20]\n" - ".inst 0xc1a6c808 // fclamp { z8.s-z11.s }, z0.s, z6.s\n" - ".inst 0xc0062c50 // mova { z16.d-z19.d }, za.d[x9, #2]\n" - ".inst 0xa060c728 // st1w { z8.s-z11.s }, pn9.b, [x25]\n" - ".inst 0xc1a6c814 // fclamp { z20.s-z23.s }, z0.s, z6.s\n" - ".inst 0xc0062c78 // mova { z24.d-z27.d }, za.d[x9, #3]\n" - ".inst 0xa061c734 // st1w { z20.s-z23.s }, pn9.b, [x25, #0x4, MUL VL]\n" - ".inst 0xc1a6c810 // fclamp { z16.s-z19.s }, z0.s, z6.s\n" - ".inst 0xa062c730 // st1w { z16.s-z19.s }, pn9.b, [x25, #0x8, MUL VL]\n" - ".inst 0xc1a6c818 // fclamp { z24.s-z27.s }, z0.s, z6.s\n" - ".inst 0xa063c338 // st1w { z24.s-z27.s }, p8, [x25, #0xc, MUL VL]\n" + ".inst 0xc0062c0c // mova { z12.d-z15.d }, za.d[x9, #0]\n" + "ld1rw { z21.s }, p1/Z, [x21]\n" + ".inst 0xc0062c38 // mova { z24.d-z27.d }, za.d[x9, #1]\n" + "ld1rw { z20.s }, p1/Z, [x20]\n" + ".inst 0xc1b4caac // fclamp { z12.s-z15.s }, z21.s, z20.s\n" + ".inst 0xc0062c40 // mova { z0.d-z3.d }, za.d[x9, #2]\n" + ".inst 0xa060c72c // st1w { z12.s-z15.s }, pn9.b, [x25]\n" + ".inst 0xc1b4cab8 // fclamp { z24.s-z27.s }, z21.s, z20.s\n" + ".inst 0xc0062c70 // mova { z16.d-z19.d }, za.d[x9, #3]\n" + ".inst 0xa061c738 // st1w { z24.s-z27.s }, pn9.b, [x25, #0x4, MUL VL]\n" + ".inst 0xc1b4caa0 // fclamp { z0.s-z3.s }, z21.s, z20.s\n" + ".inst 0xa062c720 // st1w { z0.s-z3.s }, pn9.b, [x25, #0x8, MUL VL]\n" + ".inst 0xc1b4cab0 // fclamp { z16.s-z19.s }, z21.s, z20.s\n" + ".inst 0xa063c330 // st1w { z16.s-z19.s }, p8, [x25, #0xc, MUL VL]\n" "addvl x25, x25, #16\n" "b 35f\n" "34:" // Width 4: No activation - ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" - ".inst 0xa060c728 // st1w { z8.s-z11.s }, pn9.b, [x25]\n" - ".inst 0xc0062c34 // mova { z20.d-z23.d }, za.d[x9, #1]\n" - ".inst 0xa061c734 // st1w { z20.s-z23.s }, pn9.b, [x25, #0x4, MUL VL]\n" + ".inst 0xc0062c0c // mova { z12.d-z15.d }, za.d[x9, #0]\n" + ".inst 0xa060c72c // st1w { z12.s-z15.s }, pn9.b, [x25]\n" + ".inst 0xc0062c30 // mova { z16.d-z19.d }, za.d[x9, #1]\n" + ".inst 0xa061c730 // st1w { z16.s-z19.s }, pn9.b, [x25, #0x4, MUL VL]\n" ".inst 0xc0062c50 // mova { z16.d-z19.d }, za.d[x9, #2]\n" ".inst 0xa062c730 // st1w { z16.s-z19.s }, pn9.b, [x25, #0x8, MUL VL]\n" - ".inst 0xc0062c78 // mova { z24.d-z27.d }, za.d[x9, #3]\n" - ".inst 0xa063c338 // st1w { z24.s-z27.s }, p8, [x25, #0xc, MUL VL]\n" + ".inst 0xc0062c64 // mova { z4.d-z7.d }, za.d[x9, #3]\n" + ".inst 0xa063c324 // st1w { z4.s-z7.s }, p8, [x25, #0xc, MUL VL]\n" "addvl x25, x25, #16\n" "35:" // Width 4: Output done "subs x27, x27, #0x4\n" @@ -540,7 +540,7 @@ void sme2_gemv_fp32_mla_16VL ( "bgt 4b\n" "36:" // Exit ".inst 0xd503467f // SMSTOP\n" - "ptrue p1.b\n" + "ptrue p8.b\n" : [N] "+&r" (N) : [A_ptr] "r" (A_ptr), [B_ptr] "r" (B_ptr), [K] "r" (K), [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [output_ptr] "r" (output_ptr) : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -549,5 +549,4 @@ void sme2_gemv_fp32_mla_16VL ( } // namespace arm_gemm -#endif // ARM_COMPUTE_ENABLE_SME2 -#endif +#endif // defined(ARM_COMPUTE_ENABLE_SME2) diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32bf16fp32_dot_16VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32bf16fp32_dot_16VL.hpp index f52fbcd57f0723aacf71ca51a0fa9d92d33e3aae..76c2bdd71e91ac1fa986a5422bcff3b585a73d70 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32bf16fp32_dot_16VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32bf16fp32_dot_16VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,19 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - -#ifdef __aarch64__ +#if defined(ARM_COMPUTE_ENABLE_SME2) #include "../std_transforms_sme.hpp" #include "../bfloat.hpp" @@ -84,4 +83,4 @@ public: #undef ARGLIST -#endif // __aarch64__ +#endif // defined(ARM_COMPUTE_ENABLE_SME2) diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32bf16fp32_dot_16VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32bf16fp32_dot_16VL/generic.cpp index 0a394b641316c88e7db3e612ba6a6952de6818c4..c6fa11016f4a78e802393fe949af969d88c4d310 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32bf16fp32_dot_16VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32bf16fp32_dot_16VL/generic.cpp @@ -21,8 +21,8 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifdef __ARM_FEATURE_SVE -#ifdef ARM_COMPUTE_ENABLE_SME2 + +#if defined(ARM_COMPUTE_ENABLE_SME2) #include "arm_gemm.hpp" #include "../../utils.hpp" @@ -62,7 +62,7 @@ void sme2_gemv_fp32bf16fp32_dot_16VL ( break; } __asm__ __volatile__( - "ptrue p2.b\n" + "ptrue p8.b\n" ".inst 0xd503477f // SMSTART ZA\n" "cntw x10, ALL, MUL #4\n" "add x28, %x[N], x10\n" @@ -103,494 +103,494 @@ void sme2_gemv_fp32bf16fp32_dot_16VL ( "bgt 20f\n" "beq 12f\n" "mov x23, %x[A_ptr]\n" - "lsl x22, %x[K], #0x2\n" + "lsl x21, %x[K], #0x2\n" "mov x20, %x[N]\n" - "mov x21, %x[K]\n" - ".inst 0xf8b64af8 // rprfm pldmany, x22, [x23]\n" + "mov x22, %x[K]\n" + ".inst 0xf8b54af8 // rprfm pldmany, x21, [x23]\n" ".inst 0x25b467f0 // whilelt p8.s, XZR, x20, VLx4\n" "cbz x24, 5f\n" - ".inst 0xa040c700 // ld1w { z0.s-z3.s }, pn9.b/Z, [x24]\n" - ".inst 0xc0042c00 // mova za.d[x9, #0], { z0.d-z3.d }\n" + ".inst 0xa040c710 // ld1w { z16.s-z19.s }, pn9.b/Z, [x24]\n" + ".inst 0xc0042e00 // mova za.d[x9, #0], { z16.d-z19.d }\n" "b 6f\n" "5:" // Width 1: no bias ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "6:" // Width 1: setup done - "cmp x21, #0x8\n" + "cmp x22, #0x8\n" "ble 8f\n" "7:" // Width 1: Multiply loop: Main loop head - "whilelt p1.s, XZR, x21\n" - "whilelt p0.s, x27, x21\n" - "ld1rqw { z0.s }, p1/Z, [x23]\n" - ".inst 0x658aa800 // bfcvt z0.h, p2/M, z0.s\n" - "ld1rqw { z11.s }, p0/Z, [x23, #16]\n" - ".inst 0x658aa96b // bfcvt z11.h, p2/M, z11.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "sub x21, x21, #0x8\n" - "uzp1 z11.h, z11.h, z11.h\n" - "trn1 z0.d, z0.d, z11.d\n" - ".inst 0xa040a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26]\n" - "addvl x26, x26, #16\n" - ".inst 0xc150b098 // bfdot za.s[x9, 0], { z4.h-z7.h }, z0.h[0]\n" - ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" - "addvl x26, x26, #16\n" - "cmp x21, #0x8\n" - ".inst 0xc150b618 // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[1]\n" - ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" + "whilelt p1.s, XZR, x22\n" + "whilelt p0.s, x27, x22\n" + "ld1rqw { z10.s }, p1/Z, [x23]\n" + ".inst 0x658aa94a // bfcvt z10.h, p2/M, z10.s\n" + "ld1rqw { z16.s }, p0/Z, [x23, #16]\n" + ".inst 0x658aaa10 // bfcvt z16.h, p2/M, z16.s\n" + "uzp1 z10.h, z10.h, z10.h\n" + "sub x22, x22, #0x8\n" + "uzp1 z16.h, z16.h, z16.h\n" + "trn1 z10.d, z10.d, z16.d\n" + ".inst 0xa040a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26]\n" + "addvl x26, x26, #16\n" + ".inst 0xc15ab198 // bfdot za.s[x9, 0], { z12.h-z15.h }, z10.h[0]\n" + ".inst 0xa040a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26]\n" + "addvl x26, x26, #16\n" + "cmp x22, #0x8\n" + ".inst 0xc15ab598 // bfdot za.s[x9, 0], { z12.h-z15.h }, z10.h[1]\n" + ".inst 0xa040a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26]\n" "addvl x26, x26, #16\n" "add x23, x23, #0x20\n" - ".inst 0xc150ba18 // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[2]\n" + ".inst 0xc15ab818 // bfdot za.s[x9, 0], { z0.h-z3.h }, z10.h[2]\n" ".inst 0xa040a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26]\n" "addvl x26, x26, #16\n" - ".inst 0xc150bf98 // bfdot za.s[x9, 0], { z28.h-z31.h }, z0.h[3]\n" + ".inst 0xc15abf98 // bfdot za.s[x9, 0], { z28.h-z31.h }, z10.h[3]\n" "bgt 7b\n" "8:" // Width 1: Multiply loop: Single iteration only - "whilelt p1.s, XZR, x21\n" - "whilelt p0.s, x27, x21\n" - "ld1rqw { z0.s }, p1/Z, [x23]\n" - ".inst 0x658aa800 // bfcvt z0.h, p2/M, z0.s\n" - "ld1rqw { z11.s }, p0/Z, [x23, #16]\n" - ".inst 0x658aa96b // bfcvt z11.h, p2/M, z11.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "subs x21, x21, #0x2\n" - "uzp1 z11.h, z11.h, z11.h\n" - "trn1 z0.d, z0.d, z11.d\n" - ".inst 0xa040a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26]\n" + "whilelt p1.s, XZR, x22\n" + "whilelt p0.s, x27, x22\n" + "ld1rqw { z15.s }, p1/Z, [x23]\n" + ".inst 0x658aa9ef // bfcvt z15.h, p2/M, z15.s\n" + "ld1rqw { z17.s }, p0/Z, [x23, #16]\n" + ".inst 0x658aaa31 // bfcvt z17.h, p2/M, z17.s\n" + "uzp1 z15.h, z15.h, z15.h\n" + "subs x22, x22, #0x2\n" + "uzp1 z17.h, z17.h, z17.h\n" + "trn1 z15.d, z15.d, z17.d\n" + ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" "add x23, x23, #0x20\n" - ".inst 0xc150b098 // bfdot za.s[x9, 0], { z4.h-z7.h }, z0.h[0]\n" + ".inst 0xc15fb218 // bfdot za.s[x9, 0], { z16.h-z19.h }, z15.h[0]\n" "addvl x26, x26, #16\n" "ble 9f\n" - ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" - "subs x21, x21, #0x2\n" - ".inst 0xc150b618 // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[1]\n" + ".inst 0xa040a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26]\n" + "subs x22, x22, #0x2\n" + ".inst 0xc15fb418 // bfdot za.s[x9, 0], { z0.h-z3.h }, z15.h[1]\n" "addvl x26, x26, #16\n" "ble 9f\n" - ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" - "subs x21, x21, #0x2\n" - ".inst 0xc150ba18 // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[2]\n" + ".inst 0xa040a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26]\n" + "subs x22, x22, #0x2\n" + ".inst 0xc15fb898 // bfdot za.s[x9, 0], { z4.h-z7.h }, z15.h[2]\n" "addvl x26, x26, #16\n" "ble 9f\n" - ".inst 0xa040a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26]\n" - ".inst 0xc150bf98 // bfdot za.s[x9, 0], { z28.h-z31.h }, z0.h[3]\n" + ".inst 0xa040a749 // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26]\n" + ".inst 0xc15fbd18 // bfdot za.s[x9, 0], { z8.h-z11.h }, z15.h[3]\n" "addvl x26, x26, #16\n" "9:" // Width 1: Multiply loop: multiply skip "tbz %x[flags], #1, 10f\n" "add x21, %x[args_ptr], %[offset_min]\n" "add x20, %x[args_ptr], %[offset_max]\n" - ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" - "ld1rw { z29.s }, p2/Z, [x21]\n" - "ld1rw { z18.s }, p2/Z, [x20]\n" - ".inst 0xc1b2cba8 // fclamp { z8.s-z11.s }, z29.s, z18.s\n" - ".inst 0xa060c328 // st1w { z8.s-z11.s }, p8, [x25]\n" + ".inst 0xc0062c00 // mova { z0.d-z3.d }, za.d[x9, #0]\n" + "ld1rw { z8.s }, p2/Z, [x21]\n" + "ld1rw { z26.s }, p2/Z, [x20]\n" + ".inst 0xc1bac900 // fclamp { z0.s-z3.s }, z8.s, z26.s\n" + ".inst 0xa060c320 // st1w { z0.s-z3.s }, p8, [x25]\n" "addvl x25, x25, #4\n" "b 11f\n" "10:" // Width 1: No activation - ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" - ".inst 0xa060c328 // st1w { z8.s-z11.s }, p8, [x25]\n" + ".inst 0xc0062c04 // mova { z4.d-z7.d }, za.d[x9, #0]\n" + ".inst 0xa060c324 // st1w { z4.s-z7.s }, p8, [x25]\n" "addvl x25, x25, #4\n" "11:" // Width 1: Output done "b 36f\n" "12:" // Width 2 "mov x23, %x[A_ptr]\n" - "lsl x22, %x[K], #0x2\n" + "lsl x21, %x[K], #0x2\n" "sub x20, %x[N], x10\n" - "mov x21, %x[K]\n" - ".inst 0xf8b64af8 // rprfm pldmany, x22, [x23]\n" + "mov x22, %x[K]\n" + ".inst 0xf8b54af8 // rprfm pldmany, x21, [x23]\n" ".inst 0x25b467f0 // whilelt p8.s, XZR, x20, VLx4\n" "cbz x24, 13f\n" - ".inst 0xa040c700 // ld1w { z0.s-z3.s }, pn9.b/Z, [x24]\n" - ".inst 0xc0042c00 // mova za.d[x9, #0], { z0.d-z3.d }\n" + ".inst 0xa040c718 // ld1w { z24.s-z27.s }, pn9.b/Z, [x24]\n" + ".inst 0xc0042f00 // mova za.d[x9, #0], { z24.d-z27.d }\n" ".inst 0xa041c710 // ld1w { z16.s-z19.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n" ".inst 0xc0042e01 // mova za.d[x9, #1], { z16.d-z19.d }\n" "b 14f\n" "13:" // Width 2: no bias ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "14:" // Width 2: setup done - "cmp x21, #0x8\n" + "cmp x22, #0x8\n" "ble 16f\n" "15:" // Width 2: Multiply loop: Main loop head - "whilelt p1.s, XZR, x21\n" - "whilelt p0.s, x27, x21\n" - "ld1rqw { z0.s }, p1/Z, [x23]\n" - ".inst 0x658aa800 // bfcvt z0.h, p2/M, z0.s\n" - "ld1rqw { z11.s }, p0/Z, [x23, #16]\n" - ".inst 0x658aa96b // bfcvt z11.h, p2/M, z11.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "sub x21, x21, #0x8\n" - "uzp1 z11.h, z11.h, z11.h\n" - "trn1 z0.d, z0.d, z11.d\n" - ".inst 0xa040a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26]\n" - "cmp x21, #0x8\n" - ".inst 0xa041a749 // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc150b098 // bfdot za.s[x9, 0], { z4.h-z7.h }, z0.h[0]\n" + "whilelt p1.s, XZR, x22\n" + "whilelt p0.s, x27, x22\n" + "ld1rqw { z13.s }, p1/Z, [x23]\n" + ".inst 0x658aa9ad // bfcvt z13.h, p2/M, z13.s\n" + "ld1rqw { z27.s }, p0/Z, [x23, #16]\n" + ".inst 0x658aab7b // bfcvt z27.h, p2/M, z27.s\n" + "uzp1 z13.h, z13.h, z13.h\n" + "sub x22, x22, #0x8\n" + "uzp1 z27.h, z27.h, z27.h\n" + "trn1 z13.d, z13.d, z27.d\n" + ".inst 0xa040a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26]\n" + "cmp x22, #0x8\n" + ".inst 0xa041a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15db298 // bfdot za.s[x9, 0], { z20.h-z23.h }, z13.h[0]\n" "addvl x26, x26, #16\n" "add x23, x23, #0x20\n" - ".inst 0xc150b119 // bfdot za.s[x9, 1], { z8.h-z11.h }, z0.h[0]\n" - ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" + ".inst 0xc15db019 // bfdot za.s[x9, 1], { z0.h-z3.h }, z13.h[0]\n" + ".inst 0xa040a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26]\n" + ".inst 0xa041a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15db698 // bfdot za.s[x9, 0], { z20.h-z23.h }, z13.h[1]\n" + "addvl x26, x26, #16\n" + ".inst 0xc15db719 // bfdot za.s[x9, 1], { z24.h-z27.h }, z13.h[1]\n" + ".inst 0xa040a749 // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26]\n" + ".inst 0xa041a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15db918 // bfdot za.s[x9, 0], { z8.h-z11.h }, z13.h[2]\n" + "addvl x26, x26, #16\n" + ".inst 0xc15dba19 // bfdot za.s[x9, 1], { z16.h-z19.h }, z13.h[2]\n" + ".inst 0xa040a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26]\n" ".inst 0xa041a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc150b618 // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[1]\n" - "addvl x26, x26, #16\n" - ".inst 0xc150b499 // bfdot za.s[x9, 1], { z4.h-z7.h }, z0.h[1]\n" - ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" - ".inst 0xa041a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc150ba18 // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[2]\n" - "addvl x26, x26, #16\n" - ".inst 0xc150ba99 // bfdot za.s[x9, 1], { z20.h-z23.h }, z0.h[2]\n" - ".inst 0xa040a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26]\n" - ".inst 0xa041a749 // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc150bf98 // bfdot za.s[x9, 0], { z28.h-z31.h }, z0.h[3]\n" + ".inst 0xc15dbc18 // bfdot za.s[x9, 0], { z0.h-z3.h }, z13.h[3]\n" "addvl x26, x26, #16\n" - ".inst 0xc150bd19 // bfdot za.s[x9, 1], { z8.h-z11.h }, z0.h[3]\n" + ".inst 0xc15dbc99 // bfdot za.s[x9, 1], { z4.h-z7.h }, z13.h[3]\n" "bgt 15b\n" "16:" // Width 2: Multiply loop: Single iteration only - "whilelt p1.s, XZR, x21\n" - "whilelt p0.s, x27, x21\n" - "ld1rqw { z0.s }, p1/Z, [x23]\n" - ".inst 0x658aa800 // bfcvt z0.h, p2/M, z0.s\n" - "ld1rqw { z11.s }, p0/Z, [x23, #16]\n" - ".inst 0x658aa96b // bfcvt z11.h, p2/M, z11.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "subs x21, x21, #0x2\n" - "uzp1 z11.h, z11.h, z11.h\n" - "trn1 z0.d, z0.d, z11.d\n" - ".inst 0xa040a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26]\n" + "whilelt p1.s, XZR, x22\n" + "whilelt p0.s, x27, x22\n" + "ld1rqw { z15.s }, p1/Z, [x23]\n" + ".inst 0x658aa9ef // bfcvt z15.h, p2/M, z15.s\n" + "ld1rqw { z5.s }, p0/Z, [x23, #16]\n" + ".inst 0x658aa8a5 // bfcvt z5.h, p2/M, z5.s\n" + "uzp1 z15.h, z15.h, z15.h\n" + "subs x22, x22, #0x2\n" + "uzp1 z5.h, z5.h, z5.h\n" + "trn1 z15.d, z15.d, z5.d\n" + ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" "add x23, x23, #0x20\n" - ".inst 0xa041a749 // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc150b098 // bfdot za.s[x9, 0], { z4.h-z7.h }, z0.h[0]\n" + ".inst 0xa041a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15fb218 // bfdot za.s[x9, 0], { z16.h-z19.h }, z15.h[0]\n" "addvl x26, x26, #16\n" - ".inst 0xc150b119 // bfdot za.s[x9, 1], { z8.h-z11.h }, z0.h[0]\n" + ".inst 0xc15fb319 // bfdot za.s[x9, 1], { z24.h-z27.h }, z15.h[0]\n" "ble 17f\n" - ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" - "subs x21, x21, #0x2\n" - ".inst 0xc150b618 // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[1]\n" + ".inst 0xa040a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26]\n" + "subs x22, x22, #0x2\n" + ".inst 0xc15fb798 // bfdot za.s[x9, 0], { z28.h-z31.h }, z15.h[1]\n" ".inst 0xa041a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc150b499 // bfdot za.s[x9, 1], { z4.h-z7.h }, z0.h[1]\n" + ".inst 0xc15fb499 // bfdot za.s[x9, 1], { z4.h-z7.h }, z15.h[1]\n" "addvl x26, x26, #16\n" "ble 17f\n" ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" - "subs x21, x21, #0x2\n" - ".inst 0xc150ba18 // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[2]\n" - ".inst 0xa041a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc150ba99 // bfdot za.s[x9, 1], { z20.h-z23.h }, z0.h[2]\n" + "subs x22, x22, #0x2\n" + ".inst 0xc15fba18 // bfdot za.s[x9, 0], { z16.h-z19.h }, z15.h[2]\n" + ".inst 0xa041a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15fba19 // bfdot za.s[x9, 1], { z16.h-z19.h }, z15.h[2]\n" "addvl x26, x26, #16\n" "ble 17f\n" ".inst 0xa040a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26]\n" - ".inst 0xc150bf98 // bfdot za.s[x9, 0], { z28.h-z31.h }, z0.h[3]\n" + ".inst 0xc15fbf98 // bfdot za.s[x9, 0], { z28.h-z31.h }, z15.h[3]\n" ".inst 0xa041a749 // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc150bd19 // bfdot za.s[x9, 1], { z8.h-z11.h }, z0.h[3]\n" + ".inst 0xc15fbd19 // bfdot za.s[x9, 1], { z8.h-z11.h }, z15.h[3]\n" "addvl x26, x26, #16\n" "17:" // Width 2: Multiply loop: multiply skip "tbz %x[flags], #1, 18f\n" "add x21, %x[args_ptr], %[offset_min]\n" "add x20, %x[args_ptr], %[offset_max]\n" - ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" - "ld1rw { z29.s }, p2/Z, [x21]\n" + ".inst 0xc0062c14 // mova { z20.d-z23.d }, za.d[x9, #0]\n" + "ld1rw { z11.s }, p2/Z, [x21]\n" ".inst 0xc0062c2c // mova { z12.d-z15.d }, za.d[x9, #1]\n" - "ld1rw { z18.s }, p2/Z, [x20]\n" - ".inst 0xc1b2cba8 // fclamp { z8.s-z11.s }, z29.s, z18.s\n" - ".inst 0xa060c728 // st1w { z8.s-z11.s }, pn9.b, [x25]\n" - ".inst 0xc1b2cbac // fclamp { z12.s-z15.s }, z29.s, z18.s\n" + "ld1rw { z28.s }, p2/Z, [x20]\n" + ".inst 0xc1bcc974 // fclamp { z20.s-z23.s }, z11.s, z28.s\n" + ".inst 0xa060c734 // st1w { z20.s-z23.s }, pn9.b, [x25]\n" + ".inst 0xc1bcc96c // fclamp { z12.s-z15.s }, z11.s, z28.s\n" ".inst 0xa061c32c // st1w { z12.s-z15.s }, p8, [x25, #0x4, MUL VL]\n" "addvl x25, x25, #8\n" "b 19f\n" "18:" // Width 2: No activation - ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" - ".inst 0xa060c728 // st1w { z8.s-z11.s }, pn9.b, [x25]\n" - ".inst 0xc0062c2c // mova { z12.d-z15.d }, za.d[x9, #1]\n" - ".inst 0xa061c32c // st1w { z12.s-z15.s }, p8, [x25, #0x4, MUL VL]\n" + ".inst 0xc0062c00 // mova { z0.d-z3.d }, za.d[x9, #0]\n" + ".inst 0xa060c720 // st1w { z0.s-z3.s }, pn9.b, [x25]\n" + ".inst 0xc0062c20 // mova { z0.d-z3.d }, za.d[x9, #1]\n" + ".inst 0xa061c320 // st1w { z0.s-z3.s }, p8, [x25, #0x4, MUL VL]\n" "addvl x25, x25, #8\n" "19:" // Width 2: Output done "b 36f\n" "20:" // Width 3 "mov x20, #0x2\n" "mov x23, %x[A_ptr]\n" - "lsl x22, %x[K], #0x2\n" + "lsl x21, %x[K], #0x2\n" "msub x20, x10, x20, %x[N]\n" - "mov x21, %x[K]\n" - ".inst 0xf8b64af8 // rprfm pldmany, x22, [x23]\n" + "mov x22, %x[K]\n" + ".inst 0xf8b54af8 // rprfm pldmany, x21, [x23]\n" ".inst 0x25b467f0 // whilelt p8.s, XZR, x20, VLx4\n" "cbz x24, 21f\n" - ".inst 0xa040c700 // ld1w { z0.s-z3.s }, pn9.b/Z, [x24]\n" - ".inst 0xc0042c00 // mova za.d[x9, #0], { z0.d-z3.d }\n" - ".inst 0xa041c710 // ld1w { z16.s-z19.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n" - ".inst 0xc0042e01 // mova za.d[x9, #1], { z16.d-z19.d }\n" - ".inst 0xa042c71c // ld1w { z28.s-z31.s }, pn9.b/Z, [x24, #0x8, MUL VL]\n" - ".inst 0xc0042f82 // mova za.d[x9, #2], { z28.d-z31.d }\n" + ".inst 0xa040c71c // ld1w { z28.s-z31.s }, pn9.b/Z, [x24]\n" + ".inst 0xc0042f80 // mova za.d[x9, #0], { z28.d-z31.d }\n" + ".inst 0xa041c704 // ld1w { z4.s-z7.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n" + ".inst 0xc0042c81 // mova za.d[x9, #1], { z4.d-z7.d }\n" + ".inst 0xa042c704 // ld1w { z4.s-z7.s }, pn9.b/Z, [x24, #0x8, MUL VL]\n" + ".inst 0xc0042c82 // mova za.d[x9, #2], { z4.d-z7.d }\n" "b 22f\n" "21:" // Width 3: no bias ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "22:" // Width 3: setup done - "cmp x21, #0x8\n" + "cmp x22, #0x8\n" "ble 24f\n" "23:" // Width 3: Multiply loop: Main loop head - "whilelt p1.s, XZR, x21\n" - "whilelt p0.s, x27, x21\n" - "ld1rqw { z0.s }, p1/Z, [x23]\n" - ".inst 0x658aa800 // bfcvt z0.h, p2/M, z0.s\n" - "ld1rqw { z11.s }, p0/Z, [x23, #16]\n" - ".inst 0x658aa96b // bfcvt z11.h, p2/M, z11.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "sub x21, x21, #0x8\n" - "uzp1 z11.h, z11.h, z11.h\n" - "trn1 z0.d, z0.d, z11.d\n" + "whilelt p1.s, XZR, x22\n" + "whilelt p0.s, x27, x22\n" + "ld1rqw { z14.s }, p1/Z, [x23]\n" + ".inst 0x658aa9ce // bfcvt z14.h, p2/M, z14.s\n" + "ld1rqw { z16.s }, p0/Z, [x23, #16]\n" + ".inst 0x658aaa10 // bfcvt z16.h, p2/M, z16.s\n" + "uzp1 z14.h, z14.h, z14.h\n" + "sub x22, x22, #0x8\n" + "uzp1 z16.h, z16.h, z16.h\n" + "trn1 z14.d, z14.d, z16.d\n" ".inst 0xa040a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26]\n" - "cmp x21, #0x8\n" - ".inst 0xa041a749 // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc150b098 // bfdot za.s[x9, 0], { z4.h-z7.h }, z0.h[0]\n" + "cmp x22, #0x8\n" + ".inst 0xa041a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15eb098 // bfdot za.s[x9, 0], { z4.h-z7.h }, z14.h[0]\n" "add x23, x23, #0x20\n" - ".inst 0xa042a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc150b119 // bfdot za.s[x9, 1], { z8.h-z11.h }, z0.h[0]\n" + ".inst 0xa042a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc15eb319 // bfdot za.s[x9, 1], { z24.h-z27.h }, z14.h[0]\n" "addvl x26, x26, #16\n" - ".inst 0xc150b21a // bfdot za.s[x9, 2], { z16.h-z19.h }, z0.h[0]\n" - ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" + ".inst 0xc15eb01a // bfdot za.s[x9, 2], { z0.h-z3.h }, z14.h[0]\n" + ".inst 0xa040a749 // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26]\n" ".inst 0xa041a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc150b618 // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[1]\n" - ".inst 0xa042a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc150b499 // bfdot za.s[x9, 1], { z4.h-z7.h }, z0.h[1]\n" - "addvl x26, x26, #16\n" - ".inst 0xc150b59a // bfdot za.s[x9, 2], { z12.h-z15.h }, z0.h[1]\n" - ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" - ".inst 0xa041a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc150ba18 // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[2]\n" + ".inst 0xc15eb518 // bfdot za.s[x9, 0], { z8.h-z11.h }, z14.h[1]\n" ".inst 0xa042a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc150ba99 // bfdot za.s[x9, 1], { z20.h-z23.h }, z0.h[2]\n" - "addvl x26, x26, #16\n" - ".inst 0xc150ba1a // bfdot za.s[x9, 2], { z16.h-z19.h }, z0.h[2]\n" - ".inst 0xa040a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26]\n" - ".inst 0xa041a749 // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc150bf98 // bfdot za.s[x9, 0], { z28.h-z31.h }, z0.h[3]\n" + ".inst 0xc15eb499 // bfdot za.s[x9, 1], { z4.h-z7.h }, z14.h[1]\n" + "addvl x26, x26, #16\n" + ".inst 0xc15eb61a // bfdot za.s[x9, 2], { z16.h-z19.h }, z14.h[1]\n" + ".inst 0xa040a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26]\n" + ".inst 0xa041a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15eb818 // bfdot za.s[x9, 0], { z0.h-z3.h }, z14.h[2]\n" + ".inst 0xa042a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc15ebb99 // bfdot za.s[x9, 1], { z28.h-z31.h }, z14.h[2]\n" + "addvl x26, x26, #16\n" + ".inst 0xc15eb81a // bfdot za.s[x9, 2], { z0.h-z3.h }, z14.h[2]\n" + ".inst 0xa040a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26]\n" + ".inst 0xa041a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15ebf18 // bfdot za.s[x9, 0], { z24.h-z27.h }, z14.h[3]\n" ".inst 0xa042a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc150bd19 // bfdot za.s[x9, 1], { z8.h-z11.h }, z0.h[3]\n" + ".inst 0xc15ebf99 // bfdot za.s[x9, 1], { z28.h-z31.h }, z14.h[3]\n" "addvl x26, x26, #16\n" - ".inst 0xc150be1a // bfdot za.s[x9, 2], { z16.h-z19.h }, z0.h[3]\n" + ".inst 0xc15ebe1a // bfdot za.s[x9, 2], { z16.h-z19.h }, z14.h[3]\n" "bgt 23b\n" "24:" // Width 3: Multiply loop: Single iteration only - "whilelt p1.s, XZR, x21\n" - "whilelt p0.s, x27, x21\n" - "ld1rqw { z0.s }, p1/Z, [x23]\n" - ".inst 0x658aa800 // bfcvt z0.h, p2/M, z0.s\n" - "ld1rqw { z11.s }, p0/Z, [x23, #16]\n" - ".inst 0x658aa96b // bfcvt z11.h, p2/M, z11.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "subs x21, x21, #0x2\n" - "uzp1 z11.h, z11.h, z11.h\n" - "trn1 z0.d, z0.d, z11.d\n" - ".inst 0xa040a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26]\n" - "add x23, x23, #0x20\n" - ".inst 0xa041a749 // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc150b098 // bfdot za.s[x9, 0], { z4.h-z7.h }, z0.h[0]\n" - ".inst 0xa042a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc150b119 // bfdot za.s[x9, 1], { z8.h-z11.h }, z0.h[0]\n" - "addvl x26, x26, #16\n" - ".inst 0xc150b21a // bfdot za.s[x9, 2], { z16.h-z19.h }, z0.h[0]\n" - "ble 25f\n" + "whilelt p1.s, XZR, x22\n" + "whilelt p0.s, x27, x22\n" + "ld1rqw { z15.s }, p1/Z, [x23]\n" + ".inst 0x658aa9ef // bfcvt z15.h, p2/M, z15.s\n" + "ld1rqw { z31.s }, p0/Z, [x23, #16]\n" + ".inst 0x658aabff // bfcvt z31.h, p2/M, z31.s\n" + "uzp1 z15.h, z15.h, z15.h\n" + "subs x22, x22, #0x2\n" + "uzp1 z31.h, z31.h, z31.h\n" + "trn1 z15.d, z15.d, z31.d\n" ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" - "subs x21, x21, #0x2\n" - ".inst 0xc150b618 // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[1]\n" - ".inst 0xa041a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc150b499 // bfdot za.s[x9, 1], { z4.h-z7.h }, z0.h[1]\n" - ".inst 0xa042a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc150b59a // bfdot za.s[x9, 2], { z12.h-z15.h }, z0.h[1]\n" + "add x23, x23, #0x20\n" + ".inst 0xa041a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15fb218 // bfdot za.s[x9, 0], { z16.h-z19.h }, z15.h[0]\n" + ".inst 0xa042a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc15fb019 // bfdot za.s[x9, 1], { z0.h-z3.h }, z15.h[0]\n" "addvl x26, x26, #16\n" + ".inst 0xc15fb09a // bfdot za.s[x9, 2], { z4.h-z7.h }, z15.h[0]\n" "ble 25f\n" - ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" - "subs x21, x21, #0x2\n" - ".inst 0xc150ba18 // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[2]\n" + ".inst 0xa040a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26]\n" + "subs x22, x22, #0x2\n" + ".inst 0xc15fb698 // bfdot za.s[x9, 0], { z20.h-z23.h }, z15.h[1]\n" ".inst 0xa041a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc150ba99 // bfdot za.s[x9, 1], { z20.h-z23.h }, z0.h[2]\n" + ".inst 0xc15fb699 // bfdot za.s[x9, 1], { z20.h-z23.h }, z15.h[1]\n" ".inst 0xa042a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc150ba1a // bfdot za.s[x9, 2], { z16.h-z19.h }, z0.h[2]\n" + ".inst 0xc15fb61a // bfdot za.s[x9, 2], { z16.h-z19.h }, z15.h[1]\n" + "addvl x26, x26, #16\n" + "ble 25f\n" + ".inst 0xa040a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26]\n" + "subs x22, x22, #0x2\n" + ".inst 0xc15fb898 // bfdot za.s[x9, 0], { z4.h-z7.h }, z15.h[2]\n" + ".inst 0xa041a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15fb819 // bfdot za.s[x9, 1], { z0.h-z3.h }, z15.h[2]\n" + ".inst 0xa042a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc15fbb1a // bfdot za.s[x9, 2], { z24.h-z27.h }, z15.h[2]\n" "addvl x26, x26, #16\n" "ble 25f\n" ".inst 0xa040a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26]\n" - ".inst 0xc150bf98 // bfdot za.s[x9, 0], { z28.h-z31.h }, z0.h[3]\n" + ".inst 0xc15fbf98 // bfdot za.s[x9, 0], { z28.h-z31.h }, z15.h[3]\n" ".inst 0xa041a749 // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc150bd19 // bfdot za.s[x9, 1], { z8.h-z11.h }, z0.h[3]\n" - ".inst 0xa042a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc150be1a // bfdot za.s[x9, 2], { z16.h-z19.h }, z0.h[3]\n" + ".inst 0xc15fbd19 // bfdot za.s[x9, 1], { z8.h-z11.h }, z15.h[3]\n" + ".inst 0xa042a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc15fbc9a // bfdot za.s[x9, 2], { z4.h-z7.h }, z15.h[3]\n" "addvl x26, x26, #16\n" "25:" // Width 3: Multiply loop: multiply skip "tbz %x[flags], #1, 26f\n" "add x21, %x[args_ptr], %[offset_min]\n" "add x20, %x[args_ptr], %[offset_max]\n" - ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" - "ld1rw { z29.s }, p2/Z, [x21]\n" - ".inst 0xc0062c2c // mova { z12.d-z15.d }, za.d[x9, #1]\n" - "ld1rw { z18.s }, p2/Z, [x20]\n" - ".inst 0xc1b2cba8 // fclamp { z8.s-z11.s }, z29.s, z18.s\n" - ".inst 0xc0062c44 // mova { z4.d-z7.d }, za.d[x9, #2]\n" - ".inst 0xa060c728 // st1w { z8.s-z11.s }, pn9.b, [x25]\n" - ".inst 0xc1b2cbac // fclamp { z12.s-z15.s }, z29.s, z18.s\n" - ".inst 0xa061c72c // st1w { z12.s-z15.s }, pn9.b, [x25, #0x4, MUL VL]\n" - ".inst 0xc1b2cba4 // fclamp { z4.s-z7.s }, z29.s, z18.s\n" - ".inst 0xa062c324 // st1w { z4.s-z7.s }, p8, [x25, #0x8, MUL VL]\n" + ".inst 0xc0062c1c // mova { z28.d-z31.d }, za.d[x9, #0]\n" + "ld1rw { z17.s }, p2/Z, [x21]\n" + ".inst 0xc0062c24 // mova { z4.d-z7.d }, za.d[x9, #1]\n" + "ld1rw { z16.s }, p2/Z, [x20]\n" + ".inst 0xc1b0ca3c // fclamp { z28.s-z31.s }, z17.s, z16.s\n" + ".inst 0xc0062c4c // mova { z12.d-z15.d }, za.d[x9, #2]\n" + ".inst 0xa060c73c // st1w { z28.s-z31.s }, pn9.b, [x25]\n" + ".inst 0xc1b0ca24 // fclamp { z4.s-z7.s }, z17.s, z16.s\n" + ".inst 0xa061c724 // st1w { z4.s-z7.s }, pn9.b, [x25, #0x4, MUL VL]\n" + ".inst 0xc1b0ca2c // fclamp { z12.s-z15.s }, z17.s, z16.s\n" + ".inst 0xa062c32c // st1w { z12.s-z15.s }, p8, [x25, #0x8, MUL VL]\n" "addvl x25, x25, #12\n" "b 27f\n" "26:" // Width 3: No activation - ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" - ".inst 0xa060c728 // st1w { z8.s-z11.s }, pn9.b, [x25]\n" - ".inst 0xc0062c2c // mova { z12.d-z15.d }, za.d[x9, #1]\n" - ".inst 0xa061c72c // st1w { z12.s-z15.s }, pn9.b, [x25, #0x4, MUL VL]\n" - ".inst 0xc0062c44 // mova { z4.d-z7.d }, za.d[x9, #2]\n" - ".inst 0xa062c324 // st1w { z4.s-z7.s }, p8, [x25, #0x8, MUL VL]\n" + ".inst 0xc0062c00 // mova { z0.d-z3.d }, za.d[x9, #0]\n" + ".inst 0xa060c720 // st1w { z0.s-z3.s }, pn9.b, [x25]\n" + ".inst 0xc0062c30 // mova { z16.d-z19.d }, za.d[x9, #1]\n" + ".inst 0xa061c730 // st1w { z16.s-z19.s }, pn9.b, [x25, #0x4, MUL VL]\n" + ".inst 0xc0062c50 // mova { z16.d-z19.d }, za.d[x9, #2]\n" + ".inst 0xa062c330 // st1w { z16.s-z19.s }, p8, [x25, #0x8, MUL VL]\n" "addvl x25, x25, #12\n" "27:" // Width 3: Output done "b 36f\n" "28:" // Width 4 "mov x20, #0x3\n" "mov x23, %x[A_ptr]\n" - "lsl x22, %x[K], #0x2\n" + "lsl x21, %x[K], #0x2\n" "msub x20, x10, x20, %x[N]\n" - "mov x21, %x[K]\n" - ".inst 0xf8b64af8 // rprfm pldmany, x22, [x23]\n" + "mov x22, %x[K]\n" + ".inst 0xf8b54af8 // rprfm pldmany, x21, [x23]\n" ".inst 0x25b467f0 // whilelt p8.s, XZR, x20, VLx4\n" "cbz x24, 29f\n" - ".inst 0xa040c700 // ld1w { z0.s-z3.s }, pn9.b/Z, [x24]\n" - ".inst 0xc0042c00 // mova za.d[x9, #0], { z0.d-z3.d }\n" - ".inst 0xa041c710 // ld1w { z16.s-z19.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n" - ".inst 0xc0042e01 // mova za.d[x9, #1], { z16.d-z19.d }\n" - ".inst 0xa042c71c // ld1w { z28.s-z31.s }, pn9.b/Z, [x24, #0x8, MUL VL]\n" - ".inst 0xc0042f82 // mova za.d[x9, #2], { z28.d-z31.d }\n" - ".inst 0xa043c710 // ld1w { z16.s-z19.s }, pn9.b/Z, [x24, #0xc, MUL VL]\n" - ".inst 0xc0042e03 // mova za.d[x9, #3], { z16.d-z19.d }\n" + ".inst 0xa040c70c // ld1w { z12.s-z15.s }, pn9.b/Z, [x24]\n" + ".inst 0xc0042d80 // mova za.d[x9, #0], { z12.d-z15.d }\n" + ".inst 0xa041c70c // ld1w { z12.s-z15.s }, pn9.b/Z, [x24, #0x4, MUL VL]\n" + ".inst 0xc0042d81 // mova za.d[x9, #1], { z12.d-z15.d }\n" + ".inst 0xa042c710 // ld1w { z16.s-z19.s }, pn9.b/Z, [x24, #0x8, MUL VL]\n" + ".inst 0xc0042e02 // mova za.d[x9, #2], { z16.d-z19.d }\n" + ".inst 0xa043c714 // ld1w { z20.s-z23.s }, pn9.b/Z, [x24, #0xc, MUL VL]\n" + ".inst 0xc0042e83 // mova za.d[x9, #3], { z20.d-z23.d }\n" "addvl x24, x24, #16\n" "b 30f\n" "29:" // Width 4: no bias ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "30:" // Width 4: setup done - "cmp x21, #0x8\n" + "cmp x22, #0x8\n" "ble 32f\n" "31:" // Width 4: Multiply loop: Main loop head - "whilelt p1.s, XZR, x21\n" - "whilelt p0.s, x27, x21\n" - "ld1rqw { z0.s }, p1/Z, [x23]\n" - ".inst 0x658aa800 // bfcvt z0.h, p2/M, z0.s\n" - "ld1rqw { z11.s }, p0/Z, [x23, #16]\n" - ".inst 0x658aa96b // bfcvt z11.h, p2/M, z11.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "sub x21, x21, #0x8\n" - "uzp1 z11.h, z11.h, z11.h\n" - "trn1 z0.d, z0.d, z11.d\n" - ".inst 0xa040a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26]\n" - "cmp x21, #0x8\n" + "whilelt p1.s, XZR, x22\n" + "whilelt p0.s, x27, x22\n" + "ld1rqw { z6.s }, p1/Z, [x23]\n" + ".inst 0x658aa8c6 // bfcvt z6.h, p2/M, z6.s\n" + "ld1rqw { z16.s }, p0/Z, [x23, #16]\n" + ".inst 0x658aaa10 // bfcvt z16.h, p2/M, z16.s\n" + "uzp1 z6.h, z6.h, z6.h\n" + "sub x22, x22, #0x8\n" + "uzp1 z16.h, z16.h, z16.h\n" + "trn1 z6.d, z6.d, z16.d\n" + ".inst 0xa040a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26]\n" + "cmp x22, #0x8\n" ".inst 0xa041a749 // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc150b098 // bfdot za.s[x9, 0], { z4.h-z7.h }, z0.h[0]\n" + ".inst 0xc156b198 // bfdot za.s[x9, 0], { z12.h-z15.h }, z6.h[0]\n" "add x23, x23, #0x20\n" - ".inst 0xa042a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc150b119 // bfdot za.s[x9, 1], { z8.h-z11.h }, z0.h[0]\n" - ".inst 0xa043a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc150b21a // bfdot za.s[x9, 2], { z16.h-z19.h }, z0.h[0]\n" - "addvl x26, x26, #16\n" - ".inst 0xc150b39b // bfdot za.s[x9, 3], { z28.h-z31.h }, z0.h[0]\n" - ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" - ".inst 0xa041a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc150b618 // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[1]\n" ".inst 0xa042a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc150b499 // bfdot za.s[x9, 1], { z4.h-z7.h }, z0.h[1]\n" - ".inst 0xa043a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc150b59a // bfdot za.s[x9, 2], { z12.h-z15.h }, z0.h[1]\n" - "addvl x26, x26, #16\n" - ".inst 0xc150b79b // bfdot za.s[x9, 3], { z28.h-z31.h }, z0.h[1]\n" - ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" - ".inst 0xa041a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc150ba18 // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[2]\n" - ".inst 0xa042a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc150ba99 // bfdot za.s[x9, 1], { z20.h-z23.h }, z0.h[2]\n" - ".inst 0xa043a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc150ba1a // bfdot za.s[x9, 2], { z16.h-z19.h }, z0.h[2]\n" - "addvl x26, x26, #16\n" - ".inst 0xc150b99b // bfdot za.s[x9, 3], { z12.h-z15.h }, z0.h[2]\n" + ".inst 0xc156b119 // bfdot za.s[x9, 1], { z8.h-z11.h }, z6.h[0]\n" + ".inst 0xa043a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc156b19a // bfdot za.s[x9, 2], { z12.h-z15.h }, z6.h[0]\n" + "addvl x26, x26, #16\n" + ".inst 0xc156b21b // bfdot za.s[x9, 3], { z16.h-z19.h }, z6.h[0]\n" + ".inst 0xa040a749 // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26]\n" + ".inst 0xa041a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc156b518 // bfdot za.s[x9, 0], { z8.h-z11.h }, z6.h[1]\n" + ".inst 0xa042a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc156b599 // bfdot za.s[x9, 1], { z12.h-z15.h }, z6.h[1]\n" + ".inst 0xa043a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc156b41a // bfdot za.s[x9, 2], { z0.h-z3.h }, z6.h[1]\n" + "addvl x26, x26, #16\n" + ".inst 0xc156b69b // bfdot za.s[x9, 3], { z20.h-z23.h }, z6.h[1]\n" + ".inst 0xa040a749 // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26]\n" + ".inst 0xa041a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc156b918 // bfdot za.s[x9, 0], { z8.h-z11.h }, z6.h[2]\n" + ".inst 0xa042a749 // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc156b999 // bfdot za.s[x9, 1], { z12.h-z15.h }, z6.h[2]\n" + ".inst 0xa043a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc156b91a // bfdot za.s[x9, 2], { z8.h-z11.h }, z6.h[2]\n" + "addvl x26, x26, #16\n" + ".inst 0xc156ba9b // bfdot za.s[x9, 3], { z20.h-z23.h }, z6.h[2]\n" ".inst 0xa040a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26]\n" - ".inst 0xa041a749 // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc150bf98 // bfdot za.s[x9, 0], { z28.h-z31.h }, z0.h[3]\n" - ".inst 0xa042a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc150bd19 // bfdot za.s[x9, 1], { z8.h-z11.h }, z0.h[3]\n" - ".inst 0xa043a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc150be1a // bfdot za.s[x9, 2], { z16.h-z19.h }, z0.h[3]\n" - "addvl x26, x26, #16\n" - ".inst 0xc150bf9b // bfdot za.s[x9, 3], { z28.h-z31.h }, z0.h[3]\n" + ".inst 0xa041a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc156bf98 // bfdot za.s[x9, 0], { z28.h-z31.h }, z6.h[3]\n" + ".inst 0xa042a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc156bd99 // bfdot za.s[x9, 1], { z12.h-z15.h }, z6.h[3]\n" + ".inst 0xa043a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc156bf1a // bfdot za.s[x9, 2], { z24.h-z27.h }, z6.h[3]\n" + "addvl x26, x26, #16\n" + ".inst 0xc156be1b // bfdot za.s[x9, 3], { z16.h-z19.h }, z6.h[3]\n" "bgt 31b\n" "32:" // Width 4: Multiply loop: Single iteration only - "whilelt p1.s, XZR, x21\n" - "whilelt p0.s, x27, x21\n" - "ld1rqw { z0.s }, p1/Z, [x23]\n" - ".inst 0x658aa800 // bfcvt z0.h, p2/M, z0.s\n" - "ld1rqw { z11.s }, p0/Z, [x23, #16]\n" - ".inst 0x658aa96b // bfcvt z11.h, p2/M, z11.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "subs x21, x21, #0x2\n" - "uzp1 z11.h, z11.h, z11.h\n" - "trn1 z0.d, z0.d, z11.d\n" - ".inst 0xa040a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26]\n" + "whilelt p1.s, XZR, x22\n" + "whilelt p0.s, x27, x22\n" + "ld1rqw { z15.s }, p1/Z, [x23]\n" + ".inst 0x658aa9ef // bfcvt z15.h, p2/M, z15.s\n" + "ld1rqw { z16.s }, p0/Z, [x23, #16]\n" + ".inst 0x658aaa10 // bfcvt z16.h, p2/M, z16.s\n" + "uzp1 z15.h, z15.h, z15.h\n" + "subs x22, x22, #0x2\n" + "uzp1 z16.h, z16.h, z16.h\n" + "trn1 z15.d, z15.d, z16.d\n" + ".inst 0xa040a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26]\n" "add x23, x23, #0x20\n" - ".inst 0xa041a749 // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc150b098 // bfdot za.s[x9, 0], { z4.h-z7.h }, z0.h[0]\n" - ".inst 0xa042a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc150b119 // bfdot za.s[x9, 1], { z8.h-z11.h }, z0.h[0]\n" - ".inst 0xa043a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc150b21a // bfdot za.s[x9, 2], { z16.h-z19.h }, z0.h[0]\n" + ".inst 0xa041a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15fb318 // bfdot za.s[x9, 0], { z24.h-z27.h }, z15.h[0]\n" + ".inst 0xa042a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc15fb099 // bfdot za.s[x9, 1], { z4.h-z7.h }, z15.h[0]\n" + ".inst 0xa043a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc15fb01a // bfdot za.s[x9, 2], { z0.h-z3.h }, z15.h[0]\n" "addvl x26, x26, #16\n" - ".inst 0xc150b39b // bfdot za.s[x9, 3], { z28.h-z31.h }, z0.h[0]\n" + ".inst 0xc15fb21b // bfdot za.s[x9, 3], { z16.h-z19.h }, z15.h[0]\n" "ble 33f\n" - ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" - "subs x21, x21, #0x2\n" - ".inst 0xc150b618 // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[1]\n" - ".inst 0xa041a745 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc150b499 // bfdot za.s[x9, 1], { z4.h-z7.h }, z0.h[1]\n" - ".inst 0xa042a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc150b59a // bfdot za.s[x9, 2], { z12.h-z15.h }, z0.h[1]\n" - ".inst 0xa043a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc150b79b // bfdot za.s[x9, 3], { z28.h-z31.h }, z0.h[1]\n" + ".inst 0xa040a759 // ldnt1h { z24.h-z27.h }, pn9.b/Z, [x26]\n" + "subs x22, x22, #0x2\n" + ".inst 0xc15fb718 // bfdot za.s[x9, 0], { z24.h-z27.h }, z15.h[1]\n" + ".inst 0xa041a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15fb619 // bfdot za.s[x9, 1], { z16.h-z19.h }, z15.h[1]\n" + ".inst 0xa042a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc15fb69a // bfdot za.s[x9, 2], { z20.h-z23.h }, z15.h[1]\n" + ".inst 0xa043a741 // ldnt1h { z0.h-z3.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc15fb41b // bfdot za.s[x9, 3], { z0.h-z3.h }, z15.h[1]\n" "addvl x26, x26, #16\n" "ble 33f\n" ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" - "subs x21, x21, #0x2\n" - ".inst 0xc150ba18 // bfdot za.s[x9, 0], { z16.h-z19.h }, z0.h[2]\n" - ".inst 0xa041a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc150ba99 // bfdot za.s[x9, 1], { z20.h-z23.h }, z0.h[2]\n" + "subs x22, x22, #0x2\n" + ".inst 0xc15fba18 // bfdot za.s[x9, 0], { z16.h-z19.h }, z15.h[2]\n" + ".inst 0xa041a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15fba19 // bfdot za.s[x9, 1], { z16.h-z19.h }, z15.h[2]\n" ".inst 0xa042a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc150ba1a // bfdot za.s[x9, 2], { z16.h-z19.h }, z0.h[2]\n" - ".inst 0xa043a74d // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc150b99b // bfdot za.s[x9, 3], { z12.h-z15.h }, z0.h[2]\n" + ".inst 0xc15fba1a // bfdot za.s[x9, 2], { z16.h-z19.h }, z15.h[2]\n" + ".inst 0xa043a755 // ldnt1h { z20.h-z23.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc15fba9b // bfdot za.s[x9, 3], { z20.h-z23.h }, z15.h[2]\n" "addvl x26, x26, #16\n" "ble 33f\n" - ".inst 0xa040a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26]\n" - ".inst 0xc150bf98 // bfdot za.s[x9, 0], { z28.h-z31.h }, z0.h[3]\n" - ".inst 0xa041a749 // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc150bd19 // bfdot za.s[x9, 1], { z8.h-z11.h }, z0.h[3]\n" + ".inst 0xa040a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26]\n" + ".inst 0xc15fbe18 // bfdot za.s[x9, 0], { z16.h-z19.h }, z15.h[3]\n" + ".inst 0xa041a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc15fbe19 // bfdot za.s[x9, 1], { z16.h-z19.h }, z15.h[3]\n" ".inst 0xa042a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc150be1a // bfdot za.s[x9, 2], { z16.h-z19.h }, z0.h[3]\n" - ".inst 0xa043a75d // ldnt1h { z28.h-z31.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc150bf9b // bfdot za.s[x9, 3], { z28.h-z31.h }, z0.h[3]\n" + ".inst 0xc15fbe1a // bfdot za.s[x9, 2], { z16.h-z19.h }, z15.h[3]\n" + ".inst 0xa043a751 // ldnt1h { z16.h-z19.h }, pn9.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc15fbe1b // bfdot za.s[x9, 3], { z16.h-z19.h }, z15.h[3]\n" "addvl x26, x26, #16\n" "33:" // Width 4: Multiply loop: multiply skip "tbz %x[flags], #1, 34f\n" "add x21, %x[args_ptr], %[offset_min]\n" "add x20, %x[args_ptr], %[offset_max]\n" - ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" - "ld1rw { z29.s }, p2/Z, [x21]\n" - ".inst 0xc0062c2c // mova { z12.d-z15.d }, za.d[x9, #1]\n" - "ld1rw { z18.s }, p2/Z, [x20]\n" - ".inst 0xc1b2cba8 // fclamp { z8.s-z11.s }, z29.s, z18.s\n" - ".inst 0xc0062c44 // mova { z4.d-z7.d }, za.d[x9, #2]\n" - ".inst 0xa060c728 // st1w { z8.s-z11.s }, pn9.b, [x25]\n" - ".inst 0xc1b2cbac // fclamp { z12.s-z15.s }, z29.s, z18.s\n" - ".inst 0xc0062c60 // mova { z0.d-z3.d }, za.d[x9, #3]\n" - ".inst 0xa061c72c // st1w { z12.s-z15.s }, pn9.b, [x25, #0x4, MUL VL]\n" - ".inst 0xc1b2cba4 // fclamp { z4.s-z7.s }, z29.s, z18.s\n" - ".inst 0xa062c724 // st1w { z4.s-z7.s }, pn9.b, [x25, #0x8, MUL VL]\n" - ".inst 0xc1b2cba0 // fclamp { z0.s-z3.s }, z29.s, z18.s\n" - ".inst 0xa063c320 // st1w { z0.s-z3.s }, p8, [x25, #0xc, MUL VL]\n" + ".inst 0xc0062c0c // mova { z12.d-z15.d }, za.d[x9, #0]\n" + "ld1rw { z21.s }, p2/Z, [x21]\n" + ".inst 0xc0062c38 // mova { z24.d-z27.d }, za.d[x9, #1]\n" + "ld1rw { z20.s }, p2/Z, [x20]\n" + ".inst 0xc1b4caac // fclamp { z12.s-z15.s }, z21.s, z20.s\n" + ".inst 0xc0062c40 // mova { z0.d-z3.d }, za.d[x9, #2]\n" + ".inst 0xa060c72c // st1w { z12.s-z15.s }, pn9.b, [x25]\n" + ".inst 0xc1b4cab8 // fclamp { z24.s-z27.s }, z21.s, z20.s\n" + ".inst 0xc0062c70 // mova { z16.d-z19.d }, za.d[x9, #3]\n" + ".inst 0xa061c738 // st1w { z24.s-z27.s }, pn9.b, [x25, #0x4, MUL VL]\n" + ".inst 0xc1b4caa0 // fclamp { z0.s-z3.s }, z21.s, z20.s\n" + ".inst 0xa062c720 // st1w { z0.s-z3.s }, pn9.b, [x25, #0x8, MUL VL]\n" + ".inst 0xc1b4cab0 // fclamp { z16.s-z19.s }, z21.s, z20.s\n" + ".inst 0xa063c330 // st1w { z16.s-z19.s }, p8, [x25, #0xc, MUL VL]\n" "addvl x25, x25, #16\n" "b 35f\n" "34:" // Width 4: No activation - ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" - ".inst 0xa060c728 // st1w { z8.s-z11.s }, pn9.b, [x25]\n" - ".inst 0xc0062c2c // mova { z12.d-z15.d }, za.d[x9, #1]\n" - ".inst 0xa061c72c // st1w { z12.s-z15.s }, pn9.b, [x25, #0x4, MUL VL]\n" - ".inst 0xc0062c44 // mova { z4.d-z7.d }, za.d[x9, #2]\n" - ".inst 0xa062c724 // st1w { z4.s-z7.s }, pn9.b, [x25, #0x8, MUL VL]\n" - ".inst 0xc0062c60 // mova { z0.d-z3.d }, za.d[x9, #3]\n" - ".inst 0xa063c320 // st1w { z0.s-z3.s }, p8, [x25, #0xc, MUL VL]\n" + ".inst 0xc0062c10 // mova { z16.d-z19.d }, za.d[x9, #0]\n" + ".inst 0xa060c730 // st1w { z16.s-z19.s }, pn9.b, [x25]\n" + ".inst 0xc0062c30 // mova { z16.d-z19.d }, za.d[x9, #1]\n" + ".inst 0xa061c730 // st1w { z16.s-z19.s }, pn9.b, [x25, #0x4, MUL VL]\n" + ".inst 0xc0062c54 // mova { z20.d-z23.d }, za.d[x9, #2]\n" + ".inst 0xa062c734 // st1w { z20.s-z23.s }, pn9.b, [x25, #0x8, MUL VL]\n" + ".inst 0xc0062c78 // mova { z24.d-z27.d }, za.d[x9, #3]\n" + ".inst 0xa063c338 // st1w { z24.s-z27.s }, p8, [x25, #0xc, MUL VL]\n" "addvl x25, x25, #16\n" "35:" // Width 4: Output done "subs x28, x28, #0x4\n" @@ -598,7 +598,7 @@ void sme2_gemv_fp32bf16fp32_dot_16VL ( "bgt 4b\n" "36:" // Exit ".inst 0xd503467f // SMSTOP\n" - "ptrue p2.b\n" + "ptrue p8.b\n" : [N] "+&r" (N) : [A_ptr] "r" (A_ptr), [B_ptr] "r" (B_ptr), [K] "r" (K), [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [output_ptr] "r" (output_ptr) : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -607,5 +607,4 @@ void sme2_gemv_fp32bf16fp32_dot_16VL ( } // namespace arm_gemm -#endif // ARM_COMPUTE_ENABLE_SME2 -#endif +#endif // defined(ARM_COMPUTE_ENABLE_SME2) diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_s8qa_dot_16VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_s8qa_dot_16VL.hpp index 4c9f9cff9a89acf6bd0ab81c725affc061f69c69..65e4667f884a5aaecb40ca8c633038822106714a 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_s8qa_dot_16VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_s8qa_dot_16VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,19 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - -#ifdef __aarch64__ +#if defined(ARM_COMPUTE_ENABLE_SME2) #include "../std_transforms_sme.hpp" #define ARGLIST \ @@ -83,4 +82,4 @@ public: #undef ARGLIST -#endif // __aarch64__ +#endif // defined(ARM_COMPUTE_ENABLE_SME2) diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_s8qa_dot_16VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_s8qa_dot_16VL/generic.cpp index 26dc0b9dd279f33de6327fcf26bd6134a8dea24c..86bd8aeb04a43f759cc57a0d6a5c25fed8f38bbf 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_s8qa_dot_16VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_s8qa_dot_16VL/generic.cpp @@ -21,8 +21,8 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifdef __ARM_FEATURE_SVE -#ifdef ARM_COMPUTE_ENABLE_SME2 + +#if defined(ARM_COMPUTE_ENABLE_SME2) #include "arm_gemm.hpp" #include "../../utils.hpp" @@ -35,11 +35,9 @@ namespace arm_gemm { void sme2_gemv_s8qa_dot_16VL ( const int8_t *A_ptr, const int8_t *B_ptr, int8_t *output_ptr, size_t N, size_t K, - const Requantize32 *qp, const int32_t *col_bias, unsigned int col_base + const Requantize32 *qp, const int32_t *col_bias, unsigned int ) { - ARM_COMPUTE_UNUSED(col_base); - struct KernelArgs { const int8_t *B_ptr = {}; size_t output_offset = {}; @@ -52,7 +50,7 @@ void sme2_gemv_s8qa_dot_16VL ( flags |= 0x20; } __asm__ __volatile__( - "ptrue p2.b\n" + "ptrue p8.b\n" ".inst 0xd503477f // SMSTART ZA\n" "cntw x28, ALL, MUL #4\n" "add x27, %x[N], x28\n" @@ -84,8 +82,8 @@ void sme2_gemv_s8qa_dot_16VL ( ".inst 0xf8b64b5a // rprfm pldonce, x22, [x26]\n" "3:" // RHS prefetch exit "mov x24, %x[col_bias]\n" - "mov z26.s, #0x0\n" - "mov z24.b, #0x1\n" + "mov z28.s, #0x0\n" + "mov z29.b, #0x1\n" "bic %x[flags], %x[flags], #0x80000000\n" "4:" // Column loop "cmp x27, #0x4\n" @@ -94,404 +92,404 @@ void sme2_gemv_s8qa_dot_16VL ( "bgt 24f\n" "beq 14f\n" "mov x23, %x[A_ptr]\n" - "mov x22, %x[K]\n" - "mov x20, %x[N]\n" "mov x21, %x[K]\n" - ".inst 0xf8b64af8 // rprfm pldmany, x22, [x23]\n" + "mov x20, %x[N]\n" + "mov x22, %x[K]\n" + ".inst 0xf8b54af8 // rprfm pldmany, x21, [x23]\n" "whilelt p1.b, XZR, x20\n" "cbz x24, 5f\n" - ".inst 0xa040c304 // ld1w { z4.s-z7.s }, pn8.b/Z, [x24]\n" - ".inst 0xc0042c80 // mova za.d[x9, #0], { z4.d-z7.d }\n" + ".inst 0xa040c300 // ld1w { z0.s-z3.s }, pn8.b/Z, [x24]\n" + ".inst 0xc0042c00 // mova za.d[x9, #0], { z0.d-z3.d }\n" "b 6f\n" "5:" // Width 1: no bias ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "6:" // Width 1: setup done - "cmp x21, #0x10\n" + "cmp x22, #0x10\n" "ble 9f\n" "7:" // Width 1: Multiply loop: Main loop head - "whilelt p0.b, XZR, x21\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" + "whilelt p0.b, XZR, x22\n" + "ld1rqb { z1.b }, p0/Z, [x23]\n" "add x23, x23, #0x10\n" - ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153b220 // sdot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n" - "addvl x26, x26, #16\n" ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153b6a0 // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n" + ".inst 0xc151b2a0 // sdot za.s[x9, 0], { z20.b-z23.b }, z1.b[0]\n" "addvl x26, x26, #16\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153baa0 // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n" + ".inst 0xa040834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151b5a0 // sdot za.s[x9, 0], { z12.b-z15.b }, z1.b[1]\n" "addvl x26, x26, #16\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153bea0 // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n" + ".inst 0xa040834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151b9a0 // sdot za.s[x9, 0], { z12.b-z15.b }, z1.b[2]\n" + "addvl x26, x26, #16\n" + ".inst 0xa040834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151bda0 // sdot za.s[x9, 0], { z12.b-z15.b }, z1.b[3]\n" "addvl x26, x26, #16\n" "tbnz %x[flags], #31, 8f\n" - "sdot z26.s, z3.b, z24.b\n" + "sdot z28.s, z1.b, z29.b\n" "8:" // Width 1: Multiply loop: unique 1: skip row sum - "sub x21, x21, #0x10\n" - "cmp x21, #0x10\n" + "sub x22, x22, #0x10\n" + "cmp x22, #0x10\n" "bgt 7b\n" "9:" // Width 1: Multiply loop: Single iteration only - "whilelt p0.b, XZR, x21\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" - "subs x21, x21, #0x4\n" - ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" + "whilelt p0.b, XZR, x22\n" + "ld1rqb { z1.b }, p0/Z, [x23]\n" + "subs x22, x22, #0x4\n" + ".inst 0xa040834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26]\n" "add x23, x23, #0x10\n" - ".inst 0xc153b220 // sdot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n" + ".inst 0xc151b1a0 // sdot za.s[x9, 0], { z12.b-z15.b }, z1.b[0]\n" "addvl x26, x26, #16\n" "ble 10f\n" ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - "subs x21, x21, #0x4\n" - ".inst 0xc153b6a0 // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n" + "subs x22, x22, #0x4\n" + ".inst 0xc151b6a0 // sdot za.s[x9, 0], { z20.b-z23.b }, z1.b[1]\n" "addvl x26, x26, #16\n" "ble 10f\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - "subs x21, x21, #0x4\n" - ".inst 0xc153baa0 // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n" + ".inst 0xa0408349 // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26]\n" + "subs x22, x22, #0x4\n" + ".inst 0xc151b920 // sdot za.s[x9, 0], { z8.b-z11.b }, z1.b[2]\n" "addvl x26, x26, #16\n" "ble 10f\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153bea0 // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n" + ".inst 0xa0408349 // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151bd20 // sdot za.s[x9, 0], { z8.b-z11.b }, z1.b[3]\n" "addvl x26, x26, #16\n" "10:" // Width 1: Multiply loop: multiply skip "tbnz %x[flags], #31, 11f\n" - "sdot z26.s, z3.b, z24.b\n" + "sdot z28.s, z1.b, z29.b\n" "11:" // Width 1: Multiply loop: unique 2: skip row sum "tbnz %x[flags], #31, 12f\n" "add x21, %x[qp], %[b_offset]\n" "mov x20, #0x4\n" - "ld1rw { z10.s }, p2/Z, [x21]\n" - "neg z10.s, p2/M, z10.s\n" + "ld1rw { z26.s }, p2/Z, [x21]\n" + "neg z26.s, p2/M, z26.s\n" "whilelt p0.s, XZR, x20\n" - "saddv d26, p0, z26.s\n" - "mov z26.s, z26.s[0]\n" - "mul z26.s, p2/M, z26.s, z10.s\n" + "saddv d28, p0, z28.s\n" + "mov z28.s, z28.s[0]\n" + "mul z28.s, p2/M, z28.s, z26.s\n" "orr %x[flags], %x[flags], #0x80000000\n" "12:" // Width 1: skip row sum fixup - ".inst 0xc0904b40 // addha za0.s, p2/M, p2/M, z26.s\n" + ".inst 0xc0904b80 // addha za0.s, p2/M, p2/M, z28.s\n" "add x20, %x[qp], %[per_layer_mul]\n" - "ld1rw { z5.s }, p2/Z, [x20]\n" + "ld1rw { z1.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[per_layer_right_shift]\n" - ".inst 0xc0904b41 // addha za1.s, p2/M, p2/M, z26.s\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + ".inst 0xc0904b81 // addha za1.s, p2/M, p2/M, z28.s\n" + "ld1rw { z7.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[c_offset]\n" "add x21, %x[qp], %[minval]\n" - ".inst 0xc0904b42 // addha za2.s, p2/M, p2/M, z26.s\n" - "ld1rw { z6.s }, p2/Z, [x20]\n" + ".inst 0xc0904b82 // addha za2.s, p2/M, p2/M, z28.s\n" + "ld1rw { z2.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[maxval]\n" - ".inst 0xc0904b43 // addha za3.s, p2/M, p2/M, z26.s\n" - "ld1rw { z21.s }, p2/Z, [x21]\n" - ".inst 0xc0062c1c // mova { z28.d-z31.d }, za.d[x9, #0]\n" - ".inst 0xc1a5ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z5.s\n" - ".inst 0xc1a4aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" - "ld1rw { z16.s }, p2/Z, [x20]\n" - ".inst 0xc1a6ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z6.s\n" - ".inst 0xc1b0cebc // sclamp { z28.s-z31.s }, z21.s, z16.s\n" - "uzp1 z28.h, z28.h, z29.h\n" - "uzp1 z29.h, z30.h, z31.h\n" - "uzp1 z28.b, z28.b, z29.b\n" - "st1b { z28.b }, p1, [x25]\n" + ".inst 0xc0904b83 // addha za3.s, p2/M, p2/M, z28.s\n" + "ld1rw { z16.s }, p2/Z, [x21]\n" + ".inst 0xc0062c0c // mova { z12.d-z15.d }, za.d[x9, #0]\n" + ".inst 0xc1a1ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n" + ".inst 0xc1a7aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z7.s\n" + "ld1rw { z30.s }, p2/Z, [x20]\n" + ".inst 0xc1a2ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z2.s\n" + ".inst 0xc1bece0c // sclamp { z12.s-z15.s }, z16.s, z30.s\n" + "uzp1 z12.h, z12.h, z13.h\n" + "uzp1 z19.h, z14.h, z15.h\n" + "uzp1 z12.b, z12.b, z19.b\n" + "st1b { z12.b }, p1, [x25]\n" "addvl x25, x25, #1\n" "13:" // Width 1: Output done "b 44f\n" "14:" // Width 2 "mov x23, %x[A_ptr]\n" - "mov x22, %x[K]\n" - "sub x20, %x[N], x28\n" "mov x21, %x[K]\n" - ".inst 0xf8b64af8 // rprfm pldmany, x22, [x23]\n" + "sub x20, %x[N], x28\n" + "mov x22, %x[K]\n" + ".inst 0xf8b54af8 // rprfm pldmany, x21, [x23]\n" "whilelt p1.b, XZR, x20\n" "cbz x24, 15f\n" - ".inst 0xa040c304 // ld1w { z4.s-z7.s }, pn8.b/Z, [x24]\n" - ".inst 0xc0042c80 // mova za.d[x9, #0], { z4.d-z7.d }\n" - ".inst 0xa041c314 // ld1w { z20.s-z23.s }, pn8.b/Z, [x24, #0x4, MUL VL]\n" - ".inst 0xc0042e81 // mova za.d[x9, #1], { z20.d-z23.d }\n" + ".inst 0xa040c310 // ld1w { z16.s-z19.s }, pn8.b/Z, [x24]\n" + ".inst 0xc0042e00 // mova za.d[x9, #0], { z16.d-z19.d }\n" + ".inst 0xa041c318 // ld1w { z24.s-z27.s }, pn8.b/Z, [x24, #0x4, MUL VL]\n" + ".inst 0xc0042f01 // mova za.d[x9, #1], { z24.d-z27.d }\n" "b 16f\n" "15:" // Width 2: no bias ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "16:" // Width 2: setup done - "cmp x21, #0x10\n" + "cmp x22, #0x10\n" "ble 19f\n" "17:" // Width 2: Multiply loop: Main loop head - "whilelt p0.b, XZR, x21\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" + "whilelt p0.b, XZR, x22\n" + "ld1rqb { z1.b }, p0/Z, [x23]\n" "add x23, x23, #0x10\n" - ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153b220 // sdot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n" - ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b0a1 // sdot za.s[x9, 1], { z4.b-z7.b }, z3.b[0]\n" + ".inst 0xa040834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151b1a0 // sdot za.s[x9, 0], { z12.b-z15.b }, z1.b[0]\n" + ".inst 0xa0418359 // ldnt1b { z24.b-z27.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b321 // sdot za.s[x9, 1], { z24.b-z27.b }, z1.b[0]\n" "addvl x26, x26, #16\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153b6a0 // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n" - ".inst 0xa0418349 // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b521 // sdot za.s[x9, 1], { z8.b-z11.b }, z3.b[1]\n" + ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151b620 // sdot za.s[x9, 0], { z16.b-z19.b }, z1.b[1]\n" + ".inst 0xa0418355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b6a1 // sdot za.s[x9, 1], { z20.b-z23.b }, z1.b[1]\n" "addvl x26, x26, #16\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153baa0 // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n" - ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b8a1 // sdot za.s[x9, 1], { z4.b-z7.b }, z3.b[2]\n" + ".inst 0xa040834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151b9a0 // sdot za.s[x9, 0], { z12.b-z15.b }, z1.b[2]\n" + ".inst 0xa041834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b9a1 // sdot za.s[x9, 1], { z12.b-z15.b }, z1.b[2]\n" "addvl x26, x26, #16\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153bea0 // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n" - ".inst 0xa0418355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153bea1 // sdot za.s[x9, 1], { z20.b-z23.b }, z3.b[3]\n" + ".inst 0xa0408345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151bca0 // sdot za.s[x9, 0], { z4.b-z7.b }, z1.b[3]\n" + ".inst 0xa0418349 // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151bd21 // sdot za.s[x9, 1], { z8.b-z11.b }, z1.b[3]\n" "addvl x26, x26, #16\n" "tbnz %x[flags], #31, 18f\n" - "sdot z26.s, z3.b, z24.b\n" + "sdot z28.s, z1.b, z29.b\n" "18:" // Width 2: Multiply loop: unique 3: skip row sum - "sub x21, x21, #0x10\n" - "cmp x21, #0x10\n" + "sub x22, x22, #0x10\n" + "cmp x22, #0x10\n" "bgt 17b\n" "19:" // Width 2: Multiply loop: Single iteration only - "whilelt p0.b, XZR, x21\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" - "subs x21, x21, #0x4\n" - ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" + "whilelt p0.b, XZR, x22\n" + "ld1rqb { z1.b }, p0/Z, [x23]\n" + "subs x22, x22, #0x4\n" + ".inst 0xa0408359 // ldnt1b { z24.b-z27.b }, pn8.b/Z, [x26]\n" "add x23, x23, #0x10\n" - ".inst 0xc153b220 // sdot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n" - ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b0a1 // sdot za.s[x9, 1], { z4.b-z7.b }, z3.b[0]\n" + ".inst 0xc151b320 // sdot za.s[x9, 0], { z24.b-z27.b }, z1.b[0]\n" + ".inst 0xa0418351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b221 // sdot za.s[x9, 1], { z16.b-z19.b }, z1.b[0]\n" "addvl x26, x26, #16\n" "ble 20f\n" ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - "subs x21, x21, #0x4\n" - ".inst 0xc153b6a0 // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n" - ".inst 0xa0418349 // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b521 // sdot za.s[x9, 1], { z8.b-z11.b }, z3.b[1]\n" + "subs x22, x22, #0x4\n" + ".inst 0xc151b6a0 // sdot za.s[x9, 0], { z20.b-z23.b }, z1.b[1]\n" + ".inst 0xa041834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b5a1 // sdot za.s[x9, 1], { z12.b-z15.b }, z1.b[1]\n" "addvl x26, x26, #16\n" "ble 20f\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - "subs x21, x21, #0x4\n" - ".inst 0xc153baa0 // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n" - ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b8a1 // sdot za.s[x9, 1], { z4.b-z7.b }, z3.b[2]\n" + ".inst 0xa040834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26]\n" + "subs x22, x22, #0x4\n" + ".inst 0xc151b9a0 // sdot za.s[x9, 0], { z12.b-z15.b }, z1.b[2]\n" + ".inst 0xa041834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b9a1 // sdot za.s[x9, 1], { z12.b-z15.b }, z1.b[2]\n" "addvl x26, x26, #16\n" "ble 20f\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153bea0 // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n" - ".inst 0xa0418355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153bea1 // sdot za.s[x9, 1], { z20.b-z23.b }, z3.b[3]\n" + ".inst 0xa0408359 // ldnt1b { z24.b-z27.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151bf20 // sdot za.s[x9, 0], { z24.b-z27.b }, z1.b[3]\n" + ".inst 0xa0418349 // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151bd21 // sdot za.s[x9, 1], { z8.b-z11.b }, z1.b[3]\n" "addvl x26, x26, #16\n" "20:" // Width 2: Multiply loop: multiply skip "tbnz %x[flags], #31, 21f\n" - "sdot z26.s, z3.b, z24.b\n" + "sdot z28.s, z1.b, z29.b\n" "21:" // Width 2: Multiply loop: unique 4: skip row sum "tbnz %x[flags], #31, 22f\n" "add x21, %x[qp], %[b_offset]\n" "mov x20, #0x4\n" - "ld1rw { z10.s }, p2/Z, [x21]\n" - "neg z10.s, p2/M, z10.s\n" + "ld1rw { z16.s }, p2/Z, [x21]\n" + "neg z16.s, p2/M, z16.s\n" "whilelt p0.s, XZR, x20\n" - "saddv d26, p0, z26.s\n" - "mov z26.s, z26.s[0]\n" - "mul z26.s, p2/M, z26.s, z10.s\n" + "saddv d28, p0, z28.s\n" + "mov z28.s, z28.s[0]\n" + "mul z28.s, p2/M, z28.s, z16.s\n" "orr %x[flags], %x[flags], #0x80000000\n" "22:" // Width 2: skip row sum fixup - ".inst 0xc0904b40 // addha za0.s, p2/M, p2/M, z26.s\n" + ".inst 0xc0904b80 // addha za0.s, p2/M, p2/M, z28.s\n" "add x20, %x[qp], %[per_layer_mul]\n" - "ld1rw { z5.s }, p2/Z, [x20]\n" + "ld1rw { z6.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[per_layer_right_shift]\n" - ".inst 0xc0904b41 // addha za1.s, p2/M, p2/M, z26.s\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + ".inst 0xc0904b81 // addha za1.s, p2/M, p2/M, z28.s\n" + "ld1rw { z5.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[c_offset]\n" "add x21, %x[qp], %[minval]\n" - ".inst 0xc0904b42 // addha za2.s, p2/M, p2/M, z26.s\n" - "ld1rw { z6.s }, p2/Z, [x20]\n" + ".inst 0xc0904b82 // addha za2.s, p2/M, p2/M, z28.s\n" + "ld1rw { z9.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[maxval]\n" - ".inst 0xc0904b43 // addha za3.s, p2/M, p2/M, z26.s\n" - "ld1rw { z21.s }, p2/Z, [x21]\n" - ".inst 0xc0062c1c // mova { z28.d-z31.d }, za.d[x9, #0]\n" - ".inst 0xc1a5ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z5.s\n" - ".inst 0xc0062c2c // mova { z12.d-z15.d }, za.d[x9, #1]\n" - ".inst 0xc1a5ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z5.s\n" - ".inst 0xc1a4aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" - "ld1rw { z16.s }, p2/Z, [x20]\n" - ".inst 0xc1a4aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z4.s\n" - ".inst 0xc1a6ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z6.s\n" - ".inst 0xc1a6ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z6.s\n" - ".inst 0xc1b0cebc // sclamp { z28.s-z31.s }, z21.s, z16.s\n" - ".inst 0xc1b0ceac // sclamp { z12.s-z15.s }, z21.s, z16.s\n" - "uzp1 z28.h, z28.h, z29.h\n" - "uzp1 z29.h, z30.h, z31.h\n" - "uzp1 z12.h, z12.h, z13.h\n" - "uzp1 z13.h, z14.h, z15.h\n" - "uzp1 z28.b, z28.b, z29.b\n" - "st1b { z28.b }, p2, [x25]\n" - "uzp1 z12.b, z12.b, z13.b\n" - "st1b { z12.b }, p1, [x25, #1, MUL VL]\n" + ".inst 0xc0904b83 // addha za3.s, p2/M, p2/M, z28.s\n" + "ld1rw { z16.s }, p2/Z, [x21]\n" + ".inst 0xc0062c18 // mova { z24.d-z27.d }, za.d[x9, #0]\n" + ".inst 0xc1a6ac18 // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z6.s\n" + ".inst 0xc0062c20 // mova { z0.d-z3.d }, za.d[x9, #1]\n" + ".inst 0xc1a6ac00 // sqdmulh { z0.s-z3.s }, { z0.s-z3.s }, z6.s\n" + ".inst 0xc1a5aa38 // srshl { z24.s-z27.s }, { z24.s-z27.s }, z5.s\n" + "ld1rw { z21.s }, p2/Z, [x20]\n" + ".inst 0xc1a5aa20 // srshl { z0.s-z3.s }, { z0.s-z3.s }, z5.s\n" + ".inst 0xc1a9ab18 // add { z24.s-z27.s }, { z24.s-z27.s }, z9.s\n" + ".inst 0xc1a9ab00 // add { z0.s-z3.s }, { z0.s-z3.s }, z9.s\n" + ".inst 0xc1b5ce18 // sclamp { z24.s-z27.s }, z16.s, z21.s\n" + ".inst 0xc1b5ce00 // sclamp { z0.s-z3.s }, z16.s, z21.s\n" + "uzp1 z24.h, z24.h, z25.h\n" + "uzp1 z9.h, z26.h, z27.h\n" + "uzp1 z0.h, z0.h, z1.h\n" + "uzp1 z26.h, z2.h, z3.h\n" + "uzp1 z24.b, z24.b, z9.b\n" + "st1b { z24.b }, p2, [x25]\n" + "uzp1 z0.b, z0.b, z26.b\n" + "st1b { z0.b }, p1, [x25, #1, MUL VL]\n" "addvl x25, x25, #2\n" "23:" // Width 2: Output done "b 44f\n" "24:" // Width 3 "mov x20, #0x2\n" "mov x23, %x[A_ptr]\n" - "mov x22, %x[K]\n" - "msub x20, x28, x20, %x[N]\n" "mov x21, %x[K]\n" - ".inst 0xf8b64af8 // rprfm pldmany, x22, [x23]\n" + "msub x20, x28, x20, %x[N]\n" + "mov x22, %x[K]\n" + ".inst 0xf8b54af8 // rprfm pldmany, x21, [x23]\n" "whilelt p1.b, XZR, x20\n" "cbz x24, 25f\n" - ".inst 0xa040c304 // ld1w { z4.s-z7.s }, pn8.b/Z, [x24]\n" - ".inst 0xc0042c80 // mova za.d[x9, #0], { z4.d-z7.d }\n" - ".inst 0xa041c314 // ld1w { z20.s-z23.s }, pn8.b/Z, [x24, #0x4, MUL VL]\n" - ".inst 0xc0042e81 // mova za.d[x9, #1], { z20.d-z23.d }\n" - ".inst 0xa042c314 // ld1w { z20.s-z23.s }, pn8.b/Z, [x24, #0x8, MUL VL]\n" - ".inst 0xc0042e82 // mova za.d[x9, #2], { z20.d-z23.d }\n" + ".inst 0xa040c310 // ld1w { z16.s-z19.s }, pn8.b/Z, [x24]\n" + ".inst 0xc0042e00 // mova za.d[x9, #0], { z16.d-z19.d }\n" + ".inst 0xa041c30c // ld1w { z12.s-z15.s }, pn8.b/Z, [x24, #0x4, MUL VL]\n" + ".inst 0xc0042d81 // mova za.d[x9, #1], { z12.d-z15.d }\n" + ".inst 0xa042c318 // ld1w { z24.s-z27.s }, pn8.b/Z, [x24, #0x8, MUL VL]\n" + ".inst 0xc0042f02 // mova za.d[x9, #2], { z24.d-z27.d }\n" "b 26f\n" "25:" // Width 3: no bias ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "26:" // Width 3: setup done - "cmp x21, #0x10\n" + "cmp x22, #0x10\n" "ble 29f\n" "27:" // Width 3: Multiply loop: Main loop head - "whilelt p0.b, XZR, x21\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" + "whilelt p0.b, XZR, x22\n" + "ld1rqb { z1.b }, p0/Z, [x23]\n" "add x23, x23, #0x10\n" ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153b220 // sdot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n" - ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b0a1 // sdot za.s[x9, 1], { z4.b-z7.b }, z3.b[0]\n" + ".inst 0xc151b220 // sdot za.s[x9, 0], { z16.b-z19.b }, z1.b[0]\n" + ".inst 0xa0418351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b221 // sdot za.s[x9, 1], { z16.b-z19.b }, z1.b[0]\n" ".inst 0xa042834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153b1a2 // sdot za.s[x9, 2], { z12.b-z15.b }, z3.b[0]\n" + ".inst 0xc151b1a2 // sdot za.s[x9, 2], { z12.b-z15.b }, z1.b[0]\n" "addvl x26, x26, #16\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153b6a0 // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n" - ".inst 0xa0418349 // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b521 // sdot za.s[x9, 1], { z8.b-z11.b }, z3.b[1]\n" - ".inst 0xa0428351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153b622 // sdot za.s[x9, 2], { z16.b-z19.b }, z3.b[1]\n" + ".inst 0xa040834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151b5a0 // sdot za.s[x9, 0], { z12.b-z15.b }, z1.b[1]\n" + ".inst 0xa041834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b5a1 // sdot za.s[x9, 1], { z12.b-z15.b }, z1.b[1]\n" + ".inst 0xa0428355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc151b6a2 // sdot za.s[x9, 2], { z20.b-z23.b }, z1.b[1]\n" "addvl x26, x26, #16\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153baa0 // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n" + ".inst 0xa0408349 // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151b920 // sdot za.s[x9, 0], { z8.b-z11.b }, z1.b[2]\n" ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b8a1 // sdot za.s[x9, 1], { z4.b-z7.b }, z3.b[2]\n" - ".inst 0xa042834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153b9a2 // sdot za.s[x9, 2], { z12.b-z15.b }, z3.b[2]\n" + ".inst 0xc151b8a1 // sdot za.s[x9, 1], { z4.b-z7.b }, z1.b[2]\n" + ".inst 0xa0428351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc151ba22 // sdot za.s[x9, 2], { z16.b-z19.b }, z1.b[2]\n" "addvl x26, x26, #16\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153bea0 // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n" - ".inst 0xa0418355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153bea1 // sdot za.s[x9, 1], { z20.b-z23.b }, z3.b[3]\n" + ".inst 0xa0408359 // ldnt1b { z24.b-z27.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151bf20 // sdot za.s[x9, 0], { z24.b-z27.b }, z1.b[3]\n" + ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151bca1 // sdot za.s[x9, 1], { z4.b-z7.b }, z1.b[3]\n" ".inst 0xa0428351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153be22 // sdot za.s[x9, 2], { z16.b-z19.b }, z3.b[3]\n" + ".inst 0xc151be22 // sdot za.s[x9, 2], { z16.b-z19.b }, z1.b[3]\n" "addvl x26, x26, #16\n" "tbnz %x[flags], #31, 28f\n" - "sdot z26.s, z3.b, z24.b\n" + "sdot z28.s, z1.b, z29.b\n" "28:" // Width 3: Multiply loop: unique 5: skip row sum - "sub x21, x21, #0x10\n" - "cmp x21, #0x10\n" + "sub x22, x22, #0x10\n" + "cmp x22, #0x10\n" "bgt 27b\n" "29:" // Width 3: Multiply loop: Single iteration only - "whilelt p0.b, XZR, x21\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" - "subs x21, x21, #0x4\n" - ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" + "whilelt p0.b, XZR, x22\n" + "ld1rqb { z1.b }, p0/Z, [x23]\n" + "subs x22, x22, #0x4\n" + ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" "add x23, x23, #0x10\n" - ".inst 0xc153b220 // sdot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n" - ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b0a1 // sdot za.s[x9, 1], { z4.b-z7.b }, z3.b[0]\n" - ".inst 0xa042834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153b1a2 // sdot za.s[x9, 2], { z12.b-z15.b }, z3.b[0]\n" + ".inst 0xc151b2a0 // sdot za.s[x9, 0], { z20.b-z23.b }, z1.b[0]\n" + ".inst 0xa0418351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b221 // sdot za.s[x9, 1], { z16.b-z19.b }, z1.b[0]\n" + ".inst 0xa0428351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc151b222 // sdot za.s[x9, 2], { z16.b-z19.b }, z1.b[0]\n" "addvl x26, x26, #16\n" "ble 30f\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - "subs x21, x21, #0x4\n" - ".inst 0xc153b6a0 // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n" - ".inst 0xa0418349 // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b521 // sdot za.s[x9, 1], { z8.b-z11.b }, z3.b[1]\n" + ".inst 0xa0408359 // ldnt1b { z24.b-z27.b }, pn8.b/Z, [x26]\n" + "subs x22, x22, #0x4\n" + ".inst 0xc151b720 // sdot za.s[x9, 0], { z24.b-z27.b }, z1.b[1]\n" + ".inst 0xa041834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b5a1 // sdot za.s[x9, 1], { z12.b-z15.b }, z1.b[1]\n" ".inst 0xa0428351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153b622 // sdot za.s[x9, 2], { z16.b-z19.b }, z3.b[1]\n" + ".inst 0xc151b622 // sdot za.s[x9, 2], { z16.b-z19.b }, z1.b[1]\n" "addvl x26, x26, #16\n" "ble 30f\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - "subs x21, x21, #0x4\n" - ".inst 0xc153baa0 // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n" - ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b8a1 // sdot za.s[x9, 1], { z4.b-z7.b }, z3.b[2]\n" - ".inst 0xa042834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153b9a2 // sdot za.s[x9, 2], { z12.b-z15.b }, z3.b[2]\n" + ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" + "subs x22, x22, #0x4\n" + ".inst 0xc151ba20 // sdot za.s[x9, 0], { z16.b-z19.b }, z1.b[2]\n" + ".inst 0xa0418351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151ba21 // sdot za.s[x9, 1], { z16.b-z19.b }, z1.b[2]\n" + ".inst 0xa0428355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc151baa2 // sdot za.s[x9, 2], { z20.b-z23.b }, z1.b[2]\n" "addvl x26, x26, #16\n" "ble 30f\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153bea0 // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n" - ".inst 0xa0418355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153bea1 // sdot za.s[x9, 1], { z20.b-z23.b }, z3.b[3]\n" - ".inst 0xa0428351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153be22 // sdot za.s[x9, 2], { z16.b-z19.b }, z3.b[3]\n" + ".inst 0xa040834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151bda0 // sdot za.s[x9, 0], { z12.b-z15.b }, z1.b[3]\n" + ".inst 0xa0418351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151be21 // sdot za.s[x9, 1], { z16.b-z19.b }, z1.b[3]\n" + ".inst 0xa042834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc151bda2 // sdot za.s[x9, 2], { z12.b-z15.b }, z1.b[3]\n" "addvl x26, x26, #16\n" "30:" // Width 3: Multiply loop: multiply skip "tbnz %x[flags], #31, 31f\n" - "sdot z26.s, z3.b, z24.b\n" + "sdot z28.s, z1.b, z29.b\n" "31:" // Width 3: Multiply loop: unique 6: skip row sum "tbnz %x[flags], #31, 32f\n" "add x21, %x[qp], %[b_offset]\n" "mov x20, #0x4\n" - "ld1rw { z10.s }, p2/Z, [x21]\n" - "neg z10.s, p2/M, z10.s\n" + "ld1rw { z16.s }, p2/Z, [x21]\n" + "neg z16.s, p2/M, z16.s\n" "whilelt p0.s, XZR, x20\n" - "saddv d26, p0, z26.s\n" - "mov z26.s, z26.s[0]\n" - "mul z26.s, p2/M, z26.s, z10.s\n" + "saddv d28, p0, z28.s\n" + "mov z28.s, z28.s[0]\n" + "mul z28.s, p2/M, z28.s, z16.s\n" "orr %x[flags], %x[flags], #0x80000000\n" "32:" // Width 3: skip row sum fixup - ".inst 0xc0904b40 // addha za0.s, p2/M, p2/M, z26.s\n" + ".inst 0xc0904b80 // addha za0.s, p2/M, p2/M, z28.s\n" "add x20, %x[qp], %[per_layer_mul]\n" - "ld1rw { z5.s }, p2/Z, [x20]\n" + "ld1rw { z2.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[per_layer_right_shift]\n" - ".inst 0xc0904b41 // addha za1.s, p2/M, p2/M, z26.s\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + ".inst 0xc0904b81 // addha za1.s, p2/M, p2/M, z28.s\n" + "ld1rw { z1.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[c_offset]\n" "add x21, %x[qp], %[minval]\n" - ".inst 0xc0904b42 // addha za2.s, p2/M, p2/M, z26.s\n" - "ld1rw { z6.s }, p2/Z, [x20]\n" + ".inst 0xc0904b82 // addha za2.s, p2/M, p2/M, z28.s\n" + "ld1rw { z3.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[maxval]\n" - ".inst 0xc0904b43 // addha za3.s, p2/M, p2/M, z26.s\n" - "ld1rw { z21.s }, p2/Z, [x21]\n" - ".inst 0xc0062c1c // mova { z28.d-z31.d }, za.d[x9, #0]\n" - ".inst 0xc1a5ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z5.s\n" - ".inst 0xc0062c2c // mova { z12.d-z15.d }, za.d[x9, #1]\n" - ".inst 0xc1a5ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z5.s\n" - ".inst 0xc0062c40 // mova { z0.d-z3.d }, za.d[x9, #2]\n" - ".inst 0xc1a5ac00 // sqdmulh { z0.s-z3.s }, { z0.s-z3.s }, z5.s\n" - ".inst 0xc1a4aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" - "ld1rw { z16.s }, p2/Z, [x20]\n" - ".inst 0xc1a4aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z4.s\n" - ".inst 0xc1a4aa20 // srshl { z0.s-z3.s }, { z0.s-z3.s }, z4.s\n" - ".inst 0xc1a6ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z6.s\n" - ".inst 0xc1a6ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z6.s\n" - ".inst 0xc1a6ab00 // add { z0.s-z3.s }, { z0.s-z3.s }, z6.s\n" - ".inst 0xc1b0cebc // sclamp { z28.s-z31.s }, z21.s, z16.s\n" - ".inst 0xc1b0ceac // sclamp { z12.s-z15.s }, z21.s, z16.s\n" - "uzp1 z28.h, z28.h, z29.h\n" - ".inst 0xc1b0cea0 // sclamp { z0.s-z3.s }, z21.s, z16.s\n" - "uzp1 z29.h, z30.h, z31.h\n" + ".inst 0xc0904b83 // addha za3.s, p2/M, p2/M, z28.s\n" + "ld1rw { z16.s }, p2/Z, [x21]\n" + ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" + ".inst 0xc1a2ac08 // sqdmulh { z8.s-z11.s }, { z8.s-z11.s }, z2.s\n" + ".inst 0xc0062c24 // mova { z4.d-z7.d }, za.d[x9, #1]\n" + ".inst 0xc1a2ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z2.s\n" + ".inst 0xc0062c4c // mova { z12.d-z15.d }, za.d[x9, #2]\n" + ".inst 0xc1a2ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z2.s\n" + ".inst 0xc1a1aa28 // srshl { z8.s-z11.s }, { z8.s-z11.s }, z1.s\n" + "ld1rw { z0.s }, p2/Z, [x20]\n" + ".inst 0xc1a1aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" + ".inst 0xc1a1aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n" + ".inst 0xc1a3ab08 // add { z8.s-z11.s }, { z8.s-z11.s }, z3.s\n" + ".inst 0xc1a3ab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z3.s\n" + ".inst 0xc1a3ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z3.s\n" + ".inst 0xc1a0ce08 // sclamp { z8.s-z11.s }, z16.s, z0.s\n" + ".inst 0xc1a0ce04 // sclamp { z4.s-z7.s }, z16.s, z0.s\n" + "uzp1 z8.h, z8.h, z9.h\n" + ".inst 0xc1a0ce0c // sclamp { z12.s-z15.s }, z16.s, z0.s\n" + "uzp1 z18.h, z10.h, z11.h\n" + "uzp1 z4.h, z4.h, z5.h\n" + "uzp1 z17.h, z6.h, z7.h\n" "uzp1 z12.h, z12.h, z13.h\n" - "uzp1 z13.h, z14.h, z15.h\n" - "uzp1 z0.h, z0.h, z1.h\n" - "uzp1 z1.h, z2.h, z3.h\n" - "uzp1 z28.b, z28.b, z29.b\n" - "st1b { z28.b }, p2, [x25]\n" - "uzp1 z12.b, z12.b, z13.b\n" - "st1b { z12.b }, p2, [x25, #1, MUL VL]\n" - "uzp1 z0.b, z0.b, z1.b\n" - "st1b { z0.b }, p1, [x25, #2, MUL VL]\n" + "uzp1 z16.h, z14.h, z15.h\n" + "uzp1 z8.b, z8.b, z18.b\n" + "st1b { z8.b }, p2, [x25]\n" + "uzp1 z4.b, z4.b, z17.b\n" + "st1b { z4.b }, p2, [x25, #1, MUL VL]\n" + "uzp1 z12.b, z12.b, z16.b\n" + "st1b { z12.b }, p1, [x25, #2, MUL VL]\n" "addvl x25, x25, #3\n" "33:" // Width 3: Output done "b 44f\n" "34:" // Width 4 "mov x20, #0x3\n" "mov x23, %x[A_ptr]\n" - "mov x22, %x[K]\n" - "msub x20, x28, x20, %x[N]\n" "mov x21, %x[K]\n" - ".inst 0xf8b64af8 // rprfm pldmany, x22, [x23]\n" + "msub x20, x28, x20, %x[N]\n" + "mov x22, %x[K]\n" + ".inst 0xf8b54af8 // rprfm pldmany, x21, [x23]\n" "whilelt p1.b, XZR, x20\n" "cbz x24, 35f\n" - ".inst 0xa040c304 // ld1w { z4.s-z7.s }, pn8.b/Z, [x24]\n" - ".inst 0xc0042c80 // mova za.d[x9, #0], { z4.d-z7.d }\n" - ".inst 0xa041c314 // ld1w { z20.s-z23.s }, pn8.b/Z, [x24, #0x4, MUL VL]\n" - ".inst 0xc0042e81 // mova za.d[x9, #1], { z20.d-z23.d }\n" - ".inst 0xa042c314 // ld1w { z20.s-z23.s }, pn8.b/Z, [x24, #0x8, MUL VL]\n" - ".inst 0xc0042e82 // mova za.d[x9, #2], { z20.d-z23.d }\n" + ".inst 0xa040c314 // ld1w { z20.s-z23.s }, pn8.b/Z, [x24]\n" + ".inst 0xc0042e80 // mova za.d[x9, #0], { z20.d-z23.d }\n" + ".inst 0xa041c310 // ld1w { z16.s-z19.s }, pn8.b/Z, [x24, #0x4, MUL VL]\n" + ".inst 0xc0042e01 // mova za.d[x9, #1], { z16.d-z19.d }\n" + ".inst 0xa042c310 // ld1w { z16.s-z19.s }, pn8.b/Z, [x24, #0x8, MUL VL]\n" + ".inst 0xc0042e02 // mova za.d[x9, #2], { z16.d-z19.d }\n" ".inst 0xa043c310 // ld1w { z16.s-z19.s }, pn8.b/Z, [x24, #0xc, MUL VL]\n" ".inst 0xc0042e03 // mova za.d[x9, #3], { z16.d-z19.d }\n" "addvl x24, x24, #16\n" @@ -499,165 +497,165 @@ void sme2_gemv_s8qa_dot_16VL ( "35:" // Width 4: no bias ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "36:" // Width 4: setup done - "cmp x21, #0x10\n" + "cmp x22, #0x10\n" "ble 39f\n" "37:" // Width 4: Multiply loop: Main loop head - "whilelt p0.b, XZR, x21\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" + "whilelt p0.b, XZR, x22\n" + "ld1rqb { z1.b }, p0/Z, [x23]\n" "add x23, x23, #0x10\n" ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153b220 // sdot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n" - ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b0a1 // sdot za.s[x9, 1], { z4.b-z7.b }, z3.b[0]\n" + ".inst 0xc151b220 // sdot za.s[x9, 0], { z16.b-z19.b }, z1.b[0]\n" + ".inst 0xa0418351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b221 // sdot za.s[x9, 1], { z16.b-z19.b }, z1.b[0]\n" ".inst 0xa042834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153b1a2 // sdot za.s[x9, 2], { z12.b-z15.b }, z3.b[0]\n" + ".inst 0xc151b1a2 // sdot za.s[x9, 2], { z12.b-z15.b }, z1.b[0]\n" ".inst 0xa043834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc153b1a3 // sdot za.s[x9, 3], { z12.b-z15.b }, z3.b[0]\n" + ".inst 0xc151b1a3 // sdot za.s[x9, 3], { z12.b-z15.b }, z1.b[0]\n" "addvl x26, x26, #16\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153b6a0 // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n" - ".inst 0xa0418349 // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b521 // sdot za.s[x9, 1], { z8.b-z11.b }, z3.b[1]\n" + ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151b620 // sdot za.s[x9, 0], { z16.b-z19.b }, z1.b[1]\n" + ".inst 0xa041834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b5a1 // sdot za.s[x9, 1], { z12.b-z15.b }, z1.b[1]\n" ".inst 0xa0428351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153b622 // sdot za.s[x9, 2], { z16.b-z19.b }, z3.b[1]\n" - ".inst 0xa043834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc153b5a3 // sdot za.s[x9, 3], { z12.b-z15.b }, z3.b[1]\n" - "addvl x26, x26, #16\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153baa0 // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n" - ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b8a1 // sdot za.s[x9, 1], { z4.b-z7.b }, z3.b[2]\n" - ".inst 0xa042834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153b9a2 // sdot za.s[x9, 2], { z12.b-z15.b }, z3.b[2]\n" - ".inst 0xa043835d // ldnt1b { z28.b-z31.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc153bba3 // sdot za.s[x9, 3], { z28.b-z31.b }, z3.b[2]\n" + ".inst 0xc151b622 // sdot za.s[x9, 2], { z16.b-z19.b }, z1.b[1]\n" + ".inst 0xa0438351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc151b623 // sdot za.s[x9, 3], { z16.b-z19.b }, z1.b[1]\n" "addvl x26, x26, #16\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153bea0 // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n" - ".inst 0xa0418355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153bea1 // sdot za.s[x9, 1], { z20.b-z23.b }, z3.b[3]\n" + ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151ba20 // sdot za.s[x9, 0], { z16.b-z19.b }, z1.b[2]\n" + ".inst 0xa041834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b9a1 // sdot za.s[x9, 1], { z12.b-z15.b }, z1.b[2]\n" ".inst 0xa0428351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153be22 // sdot za.s[x9, 2], { z16.b-z19.b }, z3.b[3]\n" + ".inst 0xc151ba22 // sdot za.s[x9, 2], { z16.b-z19.b }, z1.b[2]\n" ".inst 0xa0438351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc153be23 // sdot za.s[x9, 3], { z16.b-z19.b }, z3.b[3]\n" + ".inst 0xc151ba23 // sdot za.s[x9, 3], { z16.b-z19.b }, z1.b[2]\n" + "addvl x26, x26, #16\n" + ".inst 0xa040834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151bda0 // sdot za.s[x9, 0], { z12.b-z15.b }, z1.b[3]\n" + ".inst 0xa041834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151bda1 // sdot za.s[x9, 1], { z12.b-z15.b }, z1.b[3]\n" + ".inst 0xa0428359 // ldnt1b { z24.b-z27.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc151bf22 // sdot za.s[x9, 2], { z24.b-z27.b }, z1.b[3]\n" + ".inst 0xa0438345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc151bca3 // sdot za.s[x9, 3], { z4.b-z7.b }, z1.b[3]\n" "addvl x26, x26, #16\n" "tbnz %x[flags], #31, 38f\n" - "sdot z26.s, z3.b, z24.b\n" + "sdot z28.s, z1.b, z29.b\n" "38:" // Width 4: Multiply loop: unique 7: skip row sum - "sub x21, x21, #0x10\n" - "cmp x21, #0x10\n" + "sub x22, x22, #0x10\n" + "cmp x22, #0x10\n" "bgt 37b\n" "39:" // Width 4: Multiply loop: Single iteration only - "whilelt p0.b, XZR, x21\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" - "subs x21, x21, #0x4\n" - ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" + "whilelt p0.b, XZR, x22\n" + "ld1rqb { z1.b }, p0/Z, [x23]\n" + "subs x22, x22, #0x4\n" + ".inst 0xa040834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26]\n" "add x23, x23, #0x10\n" - ".inst 0xc153b220 // sdot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n" - ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b0a1 // sdot za.s[x9, 1], { z4.b-z7.b }, z3.b[0]\n" - ".inst 0xa042834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153b1a2 // sdot za.s[x9, 2], { z12.b-z15.b }, z3.b[0]\n" - ".inst 0xa043834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc153b1a3 // sdot za.s[x9, 3], { z12.b-z15.b }, z3.b[0]\n" + ".inst 0xc151b1a0 // sdot za.s[x9, 0], { z12.b-z15.b }, z1.b[0]\n" + ".inst 0xa0418359 // ldnt1b { z24.b-z27.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b321 // sdot za.s[x9, 1], { z24.b-z27.b }, z1.b[0]\n" + ".inst 0xa0428349 // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc151b122 // sdot za.s[x9, 2], { z8.b-z11.b }, z1.b[0]\n" + ".inst 0xa0438351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc151b223 // sdot za.s[x9, 3], { z16.b-z19.b }, z1.b[0]\n" "addvl x26, x26, #16\n" "ble 40f\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - "subs x21, x21, #0x4\n" - ".inst 0xc153b6a0 // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n" - ".inst 0xa0418349 // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b521 // sdot za.s[x9, 1], { z8.b-z11.b }, z3.b[1]\n" - ".inst 0xa0428351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153b622 // sdot za.s[x9, 2], { z16.b-z19.b }, z3.b[1]\n" - ".inst 0xa043834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc153b5a3 // sdot za.s[x9, 3], { z12.b-z15.b }, z3.b[1]\n" + ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" + "subs x22, x22, #0x4\n" + ".inst 0xc151b620 // sdot za.s[x9, 0], { z16.b-z19.b }, z1.b[1]\n" + ".inst 0xa0418351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b621 // sdot za.s[x9, 1], { z16.b-z19.b }, z1.b[1]\n" + ".inst 0xa042834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc151b5a2 // sdot za.s[x9, 2], { z12.b-z15.b }, z1.b[1]\n" + ".inst 0xa0438355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc151b6a3 // sdot za.s[x9, 3], { z20.b-z23.b }, z1.b[1]\n" "addvl x26, x26, #16\n" "ble 40f\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - "subs x21, x21, #0x4\n" - ".inst 0xc153baa0 // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n" - ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b8a1 // sdot za.s[x9, 1], { z4.b-z7.b }, z3.b[2]\n" - ".inst 0xa042834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153b9a2 // sdot za.s[x9, 2], { z12.b-z15.b }, z3.b[2]\n" - ".inst 0xa043835d // ldnt1b { z28.b-z31.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc153bba3 // sdot za.s[x9, 3], { z28.b-z31.b }, z3.b[2]\n" + ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" + "subs x22, x22, #0x4\n" + ".inst 0xc151ba20 // sdot za.s[x9, 0], { z16.b-z19.b }, z1.b[2]\n" + ".inst 0xa0418351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151ba21 // sdot za.s[x9, 1], { z16.b-z19.b }, z1.b[2]\n" + ".inst 0xa0428351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc151ba22 // sdot za.s[x9, 2], { z16.b-z19.b }, z1.b[2]\n" + ".inst 0xa0438351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc151ba23 // sdot za.s[x9, 3], { z16.b-z19.b }, z1.b[2]\n" "addvl x26, x26, #16\n" "ble 40f\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153bea0 // sdot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n" - ".inst 0xa0418355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153bea1 // sdot za.s[x9, 1], { z20.b-z23.b }, z3.b[3]\n" + ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151be20 // sdot za.s[x9, 0], { z16.b-z19.b }, z1.b[3]\n" + ".inst 0xa0418351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151be21 // sdot za.s[x9, 1], { z16.b-z19.b }, z1.b[3]\n" ".inst 0xa0428351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153be22 // sdot za.s[x9, 2], { z16.b-z19.b }, z3.b[3]\n" + ".inst 0xc151be22 // sdot za.s[x9, 2], { z16.b-z19.b }, z1.b[3]\n" ".inst 0xa0438351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc153be23 // sdot za.s[x9, 3], { z16.b-z19.b }, z3.b[3]\n" + ".inst 0xc151be23 // sdot za.s[x9, 3], { z16.b-z19.b }, z1.b[3]\n" "addvl x26, x26, #16\n" "40:" // Width 4: Multiply loop: multiply skip "tbnz %x[flags], #31, 41f\n" - "sdot z26.s, z3.b, z24.b\n" + "sdot z28.s, z1.b, z29.b\n" "41:" // Width 4: Multiply loop: unique 8: skip row sum "tbnz %x[flags], #31, 42f\n" "add x21, %x[qp], %[b_offset]\n" "mov x20, #0x4\n" - "ld1rw { z10.s }, p2/Z, [x21]\n" - "neg z10.s, p2/M, z10.s\n" + "ld1rw { z16.s }, p2/Z, [x21]\n" + "neg z16.s, p2/M, z16.s\n" "whilelt p0.s, XZR, x20\n" - "saddv d26, p0, z26.s\n" - "mov z26.s, z26.s[0]\n" - "mul z26.s, p2/M, z26.s, z10.s\n" + "saddv d28, p0, z28.s\n" + "mov z28.s, z28.s[0]\n" + "mul z28.s, p2/M, z28.s, z16.s\n" "orr %x[flags], %x[flags], #0x80000000\n" "42:" // Width 4: skip row sum fixup - ".inst 0xc0904b40 // addha za0.s, p2/M, p2/M, z26.s\n" + ".inst 0xc0904b80 // addha za0.s, p2/M, p2/M, z28.s\n" "add x20, %x[qp], %[per_layer_mul]\n" - "ld1rw { z5.s }, p2/Z, [x20]\n" + "ld1rw { z11.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[per_layer_right_shift]\n" - ".inst 0xc0904b41 // addha za1.s, p2/M, p2/M, z26.s\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + ".inst 0xc0904b81 // addha za1.s, p2/M, p2/M, z28.s\n" + "ld1rw { z7.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[c_offset]\n" "add x21, %x[qp], %[minval]\n" - ".inst 0xc0904b42 // addha za2.s, p2/M, p2/M, z26.s\n" + ".inst 0xc0904b82 // addha za2.s, p2/M, p2/M, z28.s\n" "ld1rw { z6.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[maxval]\n" - ".inst 0xc0904b43 // addha za3.s, p2/M, p2/M, z26.s\n" - "ld1rw { z21.s }, p2/Z, [x21]\n" - ".inst 0xc0062c1c // mova { z28.d-z31.d }, za.d[x9, #0]\n" - ".inst 0xc1a5ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z5.s\n" - ".inst 0xc0062c2c // mova { z12.d-z15.d }, za.d[x9, #1]\n" - ".inst 0xc1a5ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z5.s\n" - ".inst 0xc0062c40 // mova { z0.d-z3.d }, za.d[x9, #2]\n" - ".inst 0xc1a5ac00 // sqdmulh { z0.s-z3.s }, { z0.s-z3.s }, z5.s\n" - ".inst 0xc0062c68 // mova { z8.d-z11.d }, za.d[x9, #3]\n" - ".inst 0xc1a5ac08 // sqdmulh { z8.s-z11.s }, { z8.s-z11.s }, z5.s\n" - ".inst 0xc1a4aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" - "ld1rw { z16.s }, p2/Z, [x20]\n" - ".inst 0xc1a4aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z4.s\n" - ".inst 0xc1a4aa20 // srshl { z0.s-z3.s }, { z0.s-z3.s }, z4.s\n" - ".inst 0xc1a4aa28 // srshl { z8.s-z11.s }, { z8.s-z11.s }, z4.s\n" - ".inst 0xc1a6ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z6.s\n" + ".inst 0xc0904b83 // addha za3.s, p2/M, p2/M, z28.s\n" + "ld1rw { z3.s }, p2/Z, [x21]\n" + ".inst 0xc0062c18 // mova { z24.d-z27.d }, za.d[x9, #0]\n" + ".inst 0xc1abac18 // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z11.s\n" + ".inst 0xc0062c30 // mova { z16.d-z19.d }, za.d[x9, #1]\n" + ".inst 0xc1abac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z11.s\n" + ".inst 0xc0062c54 // mova { z20.d-z23.d }, za.d[x9, #2]\n" + ".inst 0xc1abac14 // sqdmulh { z20.s-z23.s }, { z20.s-z23.s }, z11.s\n" + ".inst 0xc0062c6c // mova { z12.d-z15.d }, za.d[x9, #3]\n" + ".inst 0xc1abac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z11.s\n" + ".inst 0xc1a7aa38 // srshl { z24.s-z27.s }, { z24.s-z27.s }, z7.s\n" + "ld1rw { z31.s }, p2/Z, [x20]\n" + ".inst 0xc1a7aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z7.s\n" + ".inst 0xc1a7aa34 // srshl { z20.s-z23.s }, { z20.s-z23.s }, z7.s\n" + ".inst 0xc1a7aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z7.s\n" + ".inst 0xc1a6ab18 // add { z24.s-z27.s }, { z24.s-z27.s }, z6.s\n" + ".inst 0xc1a6ab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n" + ".inst 0xc1a6ab14 // add { z20.s-z23.s }, { z20.s-z23.s }, z6.s\n" ".inst 0xc1a6ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z6.s\n" - ".inst 0xc1a6ab00 // add { z0.s-z3.s }, { z0.s-z3.s }, z6.s\n" - ".inst 0xc1a6ab08 // add { z8.s-z11.s }, { z8.s-z11.s }, z6.s\n" - ".inst 0xc1b0cebc // sclamp { z28.s-z31.s }, z21.s, z16.s\n" - ".inst 0xc1b0ceac // sclamp { z12.s-z15.s }, z21.s, z16.s\n" - "uzp1 z28.h, z28.h, z29.h\n" - ".inst 0xc1b0cea0 // sclamp { z0.s-z3.s }, z21.s, z16.s\n" - ".inst 0xc1b0cea8 // sclamp { z8.s-z11.s }, z21.s, z16.s\n" - "uzp1 z29.h, z30.h, z31.h\n" + ".inst 0xc1bfcc78 // sclamp { z24.s-z27.s }, z3.s, z31.s\n" + ".inst 0xc1bfcc70 // sclamp { z16.s-z19.s }, z3.s, z31.s\n" + "uzp1 z24.h, z24.h, z25.h\n" + ".inst 0xc1bfcc74 // sclamp { z20.s-z23.s }, z3.s, z31.s\n" + ".inst 0xc1bfcc6c // sclamp { z12.s-z15.s }, z3.s, z31.s\n" + "uzp1 z25.h, z26.h, z27.h\n" + "uzp1 z16.h, z16.h, z17.h\n" + "uzp1 z18.h, z18.h, z19.h\n" + "uzp1 z20.h, z20.h, z21.h\n" + "uzp1 z17.h, z22.h, z23.h\n" "uzp1 z12.h, z12.h, z13.h\n" - "uzp1 z13.h, z14.h, z15.h\n" - "uzp1 z0.h, z0.h, z1.h\n" - "uzp1 z1.h, z2.h, z3.h\n" - "uzp1 z8.h, z8.h, z9.h\n" - "uzp1 z9.h, z10.h, z11.h\n" - "uzp1 z28.b, z28.b, z29.b\n" - "st1b { z28.b }, p2, [x25]\n" - "uzp1 z12.b, z12.b, z13.b\n" - "st1b { z12.b }, p2, [x25, #1, MUL VL]\n" - "uzp1 z0.b, z0.b, z1.b\n" - "uzp1 z8.b, z8.b, z9.b\n" - "st1b { z0.b }, p2, [x25, #2, MUL VL]\n" - "st1b { z8.b }, p1, [x25, #3, MUL VL]\n" + "uzp1 z30.h, z14.h, z15.h\n" + "uzp1 z24.b, z24.b, z25.b\n" + "st1b { z24.b }, p2, [x25]\n" + "uzp1 z16.b, z16.b, z18.b\n" + "st1b { z16.b }, p2, [x25, #1, MUL VL]\n" + "uzp1 z20.b, z20.b, z17.b\n" + "uzp1 z12.b, z12.b, z30.b\n" + "st1b { z20.b }, p2, [x25, #2, MUL VL]\n" + "st1b { z12.b }, p1, [x25, #3, MUL VL]\n" "addvl x25, x25, #4\n" "43:" // Width 4: Output done "subs x27, x27, #0x4\n" @@ -665,7 +663,7 @@ void sme2_gemv_s8qa_dot_16VL ( "bgt 4b\n" "44:" // Exit ".inst 0xd503467f // SMSTOP\n" - "ptrue p2.b\n" + "ptrue p8.b\n" : [N] "+&r" (N), [flags] "+&r" (flags) : [A_ptr] "r" (A_ptr), [B_ptr] "r" (B_ptr), [K] "r" (K), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [output_ptr] "r" (output_ptr), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp) : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -674,5 +672,4 @@ void sme2_gemv_s8qa_dot_16VL ( } // namespace arm_gemm -#endif // ARM_COMPUTE_ENABLE_SME2 -#endif +#endif // defined(ARM_COMPUTE_ENABLE_SME2) diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_u8qa_dot_16VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_u8qa_dot_16VL.hpp index e15b95445ec26494f39950100655585b0a63895b..46d8c4439b3fe4e0b43197eb76cdc0295707300c 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_u8qa_dot_16VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_u8qa_dot_16VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,19 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - -#ifdef __aarch64__ +#if defined(ARM_COMPUTE_ENABLE_SME2) #include "../std_transforms_sme.hpp" #define ARGLIST \ @@ -83,4 +82,4 @@ public: #undef ARGLIST -#endif // __aarch64__ +#endif // defined(ARM_COMPUTE_ENABLE_SME2) diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_u8qa_dot_16VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_u8qa_dot_16VL/generic.cpp index dfdc4ea28965b0d90ebedb483fc8dd4807775989..093feee6ce5c1a446734b2dc9699c9753733d48a 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_u8qa_dot_16VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_gemv_u8qa_dot_16VL/generic.cpp @@ -21,8 +21,8 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifdef __ARM_FEATURE_SVE -#ifdef ARM_COMPUTE_ENABLE_SME2 + +#if defined(ARM_COMPUTE_ENABLE_SME2) #include "arm_gemm.hpp" #include "../../utils.hpp" @@ -35,11 +35,9 @@ namespace arm_gemm { void sme2_gemv_u8qa_dot_16VL ( const uint8_t *A_ptr, const uint8_t *B_ptr, uint8_t *output_ptr, size_t N, size_t K, - const Requantize32 *qp, const int32_t *col_bias, unsigned int col_base + const Requantize32 *qp, const int32_t *col_bias, unsigned int ) { - ARM_COMPUTE_UNUSED(col_base); - struct KernelArgs { const uint8_t *B_ptr = {}; size_t output_offset = {}; @@ -52,7 +50,7 @@ void sme2_gemv_u8qa_dot_16VL ( flags |= 0x20; } __asm__ __volatile__( - "ptrue p2.b\n" + "ptrue p8.b\n" ".inst 0xd503477f // SMSTART ZA\n" "cntw x28, ALL, MUL #4\n" "add x27, %x[N], x28\n" @@ -84,8 +82,8 @@ void sme2_gemv_u8qa_dot_16VL ( ".inst 0xf8b64b5a // rprfm pldonce, x22, [x26]\n" "3:" // RHS prefetch exit "mov x24, %x[col_bias]\n" - "mov z26.s, #0x0\n" - "mov z24.b, #0x1\n" + "mov z28.s, #0x0\n" + "mov z29.b, #0x1\n" "bic %x[flags], %x[flags], #0x80000000\n" "4:" // Column loop "cmp x27, #0x4\n" @@ -94,404 +92,404 @@ void sme2_gemv_u8qa_dot_16VL ( "bgt 24f\n" "beq 14f\n" "mov x23, %x[A_ptr]\n" - "mov x22, %x[K]\n" - "mov x20, %x[N]\n" "mov x21, %x[K]\n" - ".inst 0xf8b64af8 // rprfm pldmany, x22, [x23]\n" + "mov x20, %x[N]\n" + "mov x22, %x[K]\n" + ".inst 0xf8b54af8 // rprfm pldmany, x21, [x23]\n" "whilelt p1.b, XZR, x20\n" "cbz x24, 5f\n" - ".inst 0xa040c304 // ld1w { z4.s-z7.s }, pn8.b/Z, [x24]\n" - ".inst 0xc0042c80 // mova za.d[x9, #0], { z4.d-z7.d }\n" + ".inst 0xa040c300 // ld1w { z0.s-z3.s }, pn8.b/Z, [x24]\n" + ".inst 0xc0042c00 // mova za.d[x9, #0], { z0.d-z3.d }\n" "b 6f\n" "5:" // Width 1: no bias ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "6:" // Width 1: setup done - "cmp x21, #0x10\n" + "cmp x22, #0x10\n" "ble 9f\n" "7:" // Width 1: Multiply loop: Main loop head - "whilelt p0.b, XZR, x21\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" + "whilelt p0.b, XZR, x22\n" + "ld1rqb { z1.b }, p0/Z, [x23]\n" "add x23, x23, #0x10\n" - ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153b230 // udot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n" - "addvl x26, x26, #16\n" ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153b6b0 // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n" + ".inst 0xc151b2b0 // udot za.s[x9, 0], { z20.b-z23.b }, z1.b[0]\n" "addvl x26, x26, #16\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153bab0 // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n" + ".inst 0xa040834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151b5b0 // udot za.s[x9, 0], { z12.b-z15.b }, z1.b[1]\n" "addvl x26, x26, #16\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153beb0 // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n" + ".inst 0xa040834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151b9b0 // udot za.s[x9, 0], { z12.b-z15.b }, z1.b[2]\n" + "addvl x26, x26, #16\n" + ".inst 0xa040834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151bdb0 // udot za.s[x9, 0], { z12.b-z15.b }, z1.b[3]\n" "addvl x26, x26, #16\n" "tbnz %x[flags], #31, 8f\n" - "udot z26.s, z3.b, z24.b\n" + "udot z28.s, z1.b, z29.b\n" "8:" // Width 1: Multiply loop: unique 1: skip row sum - "sub x21, x21, #0x10\n" - "cmp x21, #0x10\n" + "sub x22, x22, #0x10\n" + "cmp x22, #0x10\n" "bgt 7b\n" "9:" // Width 1: Multiply loop: Single iteration only - "whilelt p0.b, XZR, x21\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" - "subs x21, x21, #0x4\n" - ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" + "whilelt p0.b, XZR, x22\n" + "ld1rqb { z1.b }, p0/Z, [x23]\n" + "subs x22, x22, #0x4\n" + ".inst 0xa040834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26]\n" "add x23, x23, #0x10\n" - ".inst 0xc153b230 // udot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n" + ".inst 0xc151b1b0 // udot za.s[x9, 0], { z12.b-z15.b }, z1.b[0]\n" "addvl x26, x26, #16\n" "ble 10f\n" ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - "subs x21, x21, #0x4\n" - ".inst 0xc153b6b0 // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n" + "subs x22, x22, #0x4\n" + ".inst 0xc151b6b0 // udot za.s[x9, 0], { z20.b-z23.b }, z1.b[1]\n" "addvl x26, x26, #16\n" "ble 10f\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - "subs x21, x21, #0x4\n" - ".inst 0xc153bab0 // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n" + ".inst 0xa0408349 // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26]\n" + "subs x22, x22, #0x4\n" + ".inst 0xc151b930 // udot za.s[x9, 0], { z8.b-z11.b }, z1.b[2]\n" "addvl x26, x26, #16\n" "ble 10f\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153beb0 // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n" + ".inst 0xa0408349 // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151bd30 // udot za.s[x9, 0], { z8.b-z11.b }, z1.b[3]\n" "addvl x26, x26, #16\n" "10:" // Width 1: Multiply loop: multiply skip "tbnz %x[flags], #31, 11f\n" - "udot z26.s, z3.b, z24.b\n" + "udot z28.s, z1.b, z29.b\n" "11:" // Width 1: Multiply loop: unique 2: skip row sum "tbnz %x[flags], #31, 12f\n" "add x21, %x[qp], %[b_offset]\n" "mov x20, #0x4\n" - "ld1rw { z10.s }, p2/Z, [x21]\n" - "neg z10.s, p2/M, z10.s\n" + "ld1rw { z26.s }, p2/Z, [x21]\n" + "neg z26.s, p2/M, z26.s\n" "whilelt p0.s, XZR, x20\n" - "uaddv d26, p0, z26.s\n" - "mov z26.s, z26.s[0]\n" - "mul z26.s, p2/M, z26.s, z10.s\n" + "uaddv d28, p0, z28.s\n" + "mov z28.s, z28.s[0]\n" + "mul z28.s, p2/M, z28.s, z26.s\n" "orr %x[flags], %x[flags], #0x80000000\n" "12:" // Width 1: skip row sum fixup - ".inst 0xc0904b40 // addha za0.s, p2/M, p2/M, z26.s\n" + ".inst 0xc0904b80 // addha za0.s, p2/M, p2/M, z28.s\n" "add x20, %x[qp], %[per_layer_mul]\n" - "ld1rw { z5.s }, p2/Z, [x20]\n" + "ld1rw { z1.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[per_layer_right_shift]\n" - ".inst 0xc0904b41 // addha za1.s, p2/M, p2/M, z26.s\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + ".inst 0xc0904b81 // addha za1.s, p2/M, p2/M, z28.s\n" + "ld1rw { z7.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[c_offset]\n" "add x21, %x[qp], %[minval]\n" - ".inst 0xc0904b42 // addha za2.s, p2/M, p2/M, z26.s\n" - "ld1rw { z6.s }, p2/Z, [x20]\n" + ".inst 0xc0904b82 // addha za2.s, p2/M, p2/M, z28.s\n" + "ld1rw { z2.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[maxval]\n" - ".inst 0xc0904b43 // addha za3.s, p2/M, p2/M, z26.s\n" - "ld1rw { z21.s }, p2/Z, [x21]\n" - ".inst 0xc0062c1c // mova { z28.d-z31.d }, za.d[x9, #0]\n" - ".inst 0xc1a5ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z5.s\n" - ".inst 0xc1a4aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" - "ld1rw { z16.s }, p2/Z, [x20]\n" - ".inst 0xc1a6ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z6.s\n" - ".inst 0xc1b0cebc // sclamp { z28.s-z31.s }, z21.s, z16.s\n" - "uzp1 z28.h, z28.h, z29.h\n" - "uzp1 z29.h, z30.h, z31.h\n" - "uzp1 z28.b, z28.b, z29.b\n" - "st1b { z28.b }, p1, [x25]\n" + ".inst 0xc0904b83 // addha za3.s, p2/M, p2/M, z28.s\n" + "ld1rw { z16.s }, p2/Z, [x21]\n" + ".inst 0xc0062c0c // mova { z12.d-z15.d }, za.d[x9, #0]\n" + ".inst 0xc1a1ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n" + ".inst 0xc1a7aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z7.s\n" + "ld1rw { z30.s }, p2/Z, [x20]\n" + ".inst 0xc1a2ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z2.s\n" + ".inst 0xc1bece0c // sclamp { z12.s-z15.s }, z16.s, z30.s\n" + "uzp1 z12.h, z12.h, z13.h\n" + "uzp1 z19.h, z14.h, z15.h\n" + "uzp1 z12.b, z12.b, z19.b\n" + "st1b { z12.b }, p1, [x25]\n" "addvl x25, x25, #1\n" "13:" // Width 1: Output done "b 44f\n" "14:" // Width 2 "mov x23, %x[A_ptr]\n" - "mov x22, %x[K]\n" - "sub x20, %x[N], x28\n" "mov x21, %x[K]\n" - ".inst 0xf8b64af8 // rprfm pldmany, x22, [x23]\n" + "sub x20, %x[N], x28\n" + "mov x22, %x[K]\n" + ".inst 0xf8b54af8 // rprfm pldmany, x21, [x23]\n" "whilelt p1.b, XZR, x20\n" "cbz x24, 15f\n" - ".inst 0xa040c304 // ld1w { z4.s-z7.s }, pn8.b/Z, [x24]\n" - ".inst 0xc0042c80 // mova za.d[x9, #0], { z4.d-z7.d }\n" - ".inst 0xa041c314 // ld1w { z20.s-z23.s }, pn8.b/Z, [x24, #0x4, MUL VL]\n" - ".inst 0xc0042e81 // mova za.d[x9, #1], { z20.d-z23.d }\n" + ".inst 0xa040c310 // ld1w { z16.s-z19.s }, pn8.b/Z, [x24]\n" + ".inst 0xc0042e00 // mova za.d[x9, #0], { z16.d-z19.d }\n" + ".inst 0xa041c318 // ld1w { z24.s-z27.s }, pn8.b/Z, [x24, #0x4, MUL VL]\n" + ".inst 0xc0042f01 // mova za.d[x9, #1], { z24.d-z27.d }\n" "b 16f\n" "15:" // Width 2: no bias ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "16:" // Width 2: setup done - "cmp x21, #0x10\n" + "cmp x22, #0x10\n" "ble 19f\n" "17:" // Width 2: Multiply loop: Main loop head - "whilelt p0.b, XZR, x21\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" + "whilelt p0.b, XZR, x22\n" + "ld1rqb { z1.b }, p0/Z, [x23]\n" "add x23, x23, #0x10\n" - ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153b230 // udot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n" - ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b0b1 // udot za.s[x9, 1], { z4.b-z7.b }, z3.b[0]\n" + ".inst 0xa040834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151b1b0 // udot za.s[x9, 0], { z12.b-z15.b }, z1.b[0]\n" + ".inst 0xa0418359 // ldnt1b { z24.b-z27.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b331 // udot za.s[x9, 1], { z24.b-z27.b }, z1.b[0]\n" "addvl x26, x26, #16\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153b6b0 // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n" - ".inst 0xa0418349 // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b531 // udot za.s[x9, 1], { z8.b-z11.b }, z3.b[1]\n" + ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151b630 // udot za.s[x9, 0], { z16.b-z19.b }, z1.b[1]\n" + ".inst 0xa0418355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b6b1 // udot za.s[x9, 1], { z20.b-z23.b }, z1.b[1]\n" "addvl x26, x26, #16\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153bab0 // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n" - ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b8b1 // udot za.s[x9, 1], { z4.b-z7.b }, z3.b[2]\n" + ".inst 0xa040834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151b9b0 // udot za.s[x9, 0], { z12.b-z15.b }, z1.b[2]\n" + ".inst 0xa041834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b9b1 // udot za.s[x9, 1], { z12.b-z15.b }, z1.b[2]\n" "addvl x26, x26, #16\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153beb0 // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n" - ".inst 0xa0418355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153beb1 // udot za.s[x9, 1], { z20.b-z23.b }, z3.b[3]\n" + ".inst 0xa0408345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151bcb0 // udot za.s[x9, 0], { z4.b-z7.b }, z1.b[3]\n" + ".inst 0xa0418349 // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151bd31 // udot za.s[x9, 1], { z8.b-z11.b }, z1.b[3]\n" "addvl x26, x26, #16\n" "tbnz %x[flags], #31, 18f\n" - "udot z26.s, z3.b, z24.b\n" + "udot z28.s, z1.b, z29.b\n" "18:" // Width 2: Multiply loop: unique 3: skip row sum - "sub x21, x21, #0x10\n" - "cmp x21, #0x10\n" + "sub x22, x22, #0x10\n" + "cmp x22, #0x10\n" "bgt 17b\n" "19:" // Width 2: Multiply loop: Single iteration only - "whilelt p0.b, XZR, x21\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" - "subs x21, x21, #0x4\n" - ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" + "whilelt p0.b, XZR, x22\n" + "ld1rqb { z1.b }, p0/Z, [x23]\n" + "subs x22, x22, #0x4\n" + ".inst 0xa0408359 // ldnt1b { z24.b-z27.b }, pn8.b/Z, [x26]\n" "add x23, x23, #0x10\n" - ".inst 0xc153b230 // udot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n" - ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b0b1 // udot za.s[x9, 1], { z4.b-z7.b }, z3.b[0]\n" + ".inst 0xc151b330 // udot za.s[x9, 0], { z24.b-z27.b }, z1.b[0]\n" + ".inst 0xa0418351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b231 // udot za.s[x9, 1], { z16.b-z19.b }, z1.b[0]\n" "addvl x26, x26, #16\n" "ble 20f\n" ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - "subs x21, x21, #0x4\n" - ".inst 0xc153b6b0 // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n" - ".inst 0xa0418349 // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b531 // udot za.s[x9, 1], { z8.b-z11.b }, z3.b[1]\n" + "subs x22, x22, #0x4\n" + ".inst 0xc151b6b0 // udot za.s[x9, 0], { z20.b-z23.b }, z1.b[1]\n" + ".inst 0xa041834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b5b1 // udot za.s[x9, 1], { z12.b-z15.b }, z1.b[1]\n" "addvl x26, x26, #16\n" "ble 20f\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - "subs x21, x21, #0x4\n" - ".inst 0xc153bab0 // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n" - ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b8b1 // udot za.s[x9, 1], { z4.b-z7.b }, z3.b[2]\n" + ".inst 0xa040834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26]\n" + "subs x22, x22, #0x4\n" + ".inst 0xc151b9b0 // udot za.s[x9, 0], { z12.b-z15.b }, z1.b[2]\n" + ".inst 0xa041834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b9b1 // udot za.s[x9, 1], { z12.b-z15.b }, z1.b[2]\n" "addvl x26, x26, #16\n" "ble 20f\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153beb0 // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n" - ".inst 0xa0418355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153beb1 // udot za.s[x9, 1], { z20.b-z23.b }, z3.b[3]\n" + ".inst 0xa0408359 // ldnt1b { z24.b-z27.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151bf30 // udot za.s[x9, 0], { z24.b-z27.b }, z1.b[3]\n" + ".inst 0xa0418349 // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151bd31 // udot za.s[x9, 1], { z8.b-z11.b }, z1.b[3]\n" "addvl x26, x26, #16\n" "20:" // Width 2: Multiply loop: multiply skip "tbnz %x[flags], #31, 21f\n" - "udot z26.s, z3.b, z24.b\n" + "udot z28.s, z1.b, z29.b\n" "21:" // Width 2: Multiply loop: unique 4: skip row sum "tbnz %x[flags], #31, 22f\n" "add x21, %x[qp], %[b_offset]\n" "mov x20, #0x4\n" - "ld1rw { z10.s }, p2/Z, [x21]\n" - "neg z10.s, p2/M, z10.s\n" + "ld1rw { z16.s }, p2/Z, [x21]\n" + "neg z16.s, p2/M, z16.s\n" "whilelt p0.s, XZR, x20\n" - "uaddv d26, p0, z26.s\n" - "mov z26.s, z26.s[0]\n" - "mul z26.s, p2/M, z26.s, z10.s\n" + "uaddv d28, p0, z28.s\n" + "mov z28.s, z28.s[0]\n" + "mul z28.s, p2/M, z28.s, z16.s\n" "orr %x[flags], %x[flags], #0x80000000\n" "22:" // Width 2: skip row sum fixup - ".inst 0xc0904b40 // addha za0.s, p2/M, p2/M, z26.s\n" + ".inst 0xc0904b80 // addha za0.s, p2/M, p2/M, z28.s\n" "add x20, %x[qp], %[per_layer_mul]\n" - "ld1rw { z5.s }, p2/Z, [x20]\n" + "ld1rw { z6.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[per_layer_right_shift]\n" - ".inst 0xc0904b41 // addha za1.s, p2/M, p2/M, z26.s\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + ".inst 0xc0904b81 // addha za1.s, p2/M, p2/M, z28.s\n" + "ld1rw { z5.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[c_offset]\n" "add x21, %x[qp], %[minval]\n" - ".inst 0xc0904b42 // addha za2.s, p2/M, p2/M, z26.s\n" - "ld1rw { z6.s }, p2/Z, [x20]\n" + ".inst 0xc0904b82 // addha za2.s, p2/M, p2/M, z28.s\n" + "ld1rw { z9.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[maxval]\n" - ".inst 0xc0904b43 // addha za3.s, p2/M, p2/M, z26.s\n" - "ld1rw { z21.s }, p2/Z, [x21]\n" - ".inst 0xc0062c1c // mova { z28.d-z31.d }, za.d[x9, #0]\n" - ".inst 0xc1a5ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z5.s\n" - ".inst 0xc0062c2c // mova { z12.d-z15.d }, za.d[x9, #1]\n" - ".inst 0xc1a5ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z5.s\n" - ".inst 0xc1a4aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" - "ld1rw { z16.s }, p2/Z, [x20]\n" - ".inst 0xc1a4aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z4.s\n" - ".inst 0xc1a6ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z6.s\n" - ".inst 0xc1a6ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z6.s\n" - ".inst 0xc1b0cebc // sclamp { z28.s-z31.s }, z21.s, z16.s\n" - ".inst 0xc1b0ceac // sclamp { z12.s-z15.s }, z21.s, z16.s\n" - "uzp1 z28.h, z28.h, z29.h\n" - "uzp1 z29.h, z30.h, z31.h\n" - "uzp1 z12.h, z12.h, z13.h\n" - "uzp1 z13.h, z14.h, z15.h\n" - "uzp1 z28.b, z28.b, z29.b\n" - "st1b { z28.b }, p2, [x25]\n" - "uzp1 z12.b, z12.b, z13.b\n" - "st1b { z12.b }, p1, [x25, #1, MUL VL]\n" + ".inst 0xc0904b83 // addha za3.s, p2/M, p2/M, z28.s\n" + "ld1rw { z16.s }, p2/Z, [x21]\n" + ".inst 0xc0062c18 // mova { z24.d-z27.d }, za.d[x9, #0]\n" + ".inst 0xc1a6ac18 // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z6.s\n" + ".inst 0xc0062c20 // mova { z0.d-z3.d }, za.d[x9, #1]\n" + ".inst 0xc1a6ac00 // sqdmulh { z0.s-z3.s }, { z0.s-z3.s }, z6.s\n" + ".inst 0xc1a5aa38 // srshl { z24.s-z27.s }, { z24.s-z27.s }, z5.s\n" + "ld1rw { z21.s }, p2/Z, [x20]\n" + ".inst 0xc1a5aa20 // srshl { z0.s-z3.s }, { z0.s-z3.s }, z5.s\n" + ".inst 0xc1a9ab18 // add { z24.s-z27.s }, { z24.s-z27.s }, z9.s\n" + ".inst 0xc1a9ab00 // add { z0.s-z3.s }, { z0.s-z3.s }, z9.s\n" + ".inst 0xc1b5ce18 // sclamp { z24.s-z27.s }, z16.s, z21.s\n" + ".inst 0xc1b5ce00 // sclamp { z0.s-z3.s }, z16.s, z21.s\n" + "uzp1 z24.h, z24.h, z25.h\n" + "uzp1 z9.h, z26.h, z27.h\n" + "uzp1 z0.h, z0.h, z1.h\n" + "uzp1 z26.h, z2.h, z3.h\n" + "uzp1 z24.b, z24.b, z9.b\n" + "st1b { z24.b }, p2, [x25]\n" + "uzp1 z0.b, z0.b, z26.b\n" + "st1b { z0.b }, p1, [x25, #1, MUL VL]\n" "addvl x25, x25, #2\n" "23:" // Width 2: Output done "b 44f\n" "24:" // Width 3 "mov x20, #0x2\n" "mov x23, %x[A_ptr]\n" - "mov x22, %x[K]\n" - "msub x20, x28, x20, %x[N]\n" "mov x21, %x[K]\n" - ".inst 0xf8b64af8 // rprfm pldmany, x22, [x23]\n" + "msub x20, x28, x20, %x[N]\n" + "mov x22, %x[K]\n" + ".inst 0xf8b54af8 // rprfm pldmany, x21, [x23]\n" "whilelt p1.b, XZR, x20\n" "cbz x24, 25f\n" - ".inst 0xa040c304 // ld1w { z4.s-z7.s }, pn8.b/Z, [x24]\n" - ".inst 0xc0042c80 // mova za.d[x9, #0], { z4.d-z7.d }\n" - ".inst 0xa041c314 // ld1w { z20.s-z23.s }, pn8.b/Z, [x24, #0x4, MUL VL]\n" - ".inst 0xc0042e81 // mova za.d[x9, #1], { z20.d-z23.d }\n" - ".inst 0xa042c314 // ld1w { z20.s-z23.s }, pn8.b/Z, [x24, #0x8, MUL VL]\n" - ".inst 0xc0042e82 // mova za.d[x9, #2], { z20.d-z23.d }\n" + ".inst 0xa040c310 // ld1w { z16.s-z19.s }, pn8.b/Z, [x24]\n" + ".inst 0xc0042e00 // mova za.d[x9, #0], { z16.d-z19.d }\n" + ".inst 0xa041c30c // ld1w { z12.s-z15.s }, pn8.b/Z, [x24, #0x4, MUL VL]\n" + ".inst 0xc0042d81 // mova za.d[x9, #1], { z12.d-z15.d }\n" + ".inst 0xa042c318 // ld1w { z24.s-z27.s }, pn8.b/Z, [x24, #0x8, MUL VL]\n" + ".inst 0xc0042f02 // mova za.d[x9, #2], { z24.d-z27.d }\n" "b 26f\n" "25:" // Width 3: no bias ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "26:" // Width 3: setup done - "cmp x21, #0x10\n" + "cmp x22, #0x10\n" "ble 29f\n" "27:" // Width 3: Multiply loop: Main loop head - "whilelt p0.b, XZR, x21\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" + "whilelt p0.b, XZR, x22\n" + "ld1rqb { z1.b }, p0/Z, [x23]\n" "add x23, x23, #0x10\n" ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153b230 // udot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n" - ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b0b1 // udot za.s[x9, 1], { z4.b-z7.b }, z3.b[0]\n" + ".inst 0xc151b230 // udot za.s[x9, 0], { z16.b-z19.b }, z1.b[0]\n" + ".inst 0xa0418351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b231 // udot za.s[x9, 1], { z16.b-z19.b }, z1.b[0]\n" ".inst 0xa042834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153b1b2 // udot za.s[x9, 2], { z12.b-z15.b }, z3.b[0]\n" + ".inst 0xc151b1b2 // udot za.s[x9, 2], { z12.b-z15.b }, z1.b[0]\n" "addvl x26, x26, #16\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153b6b0 // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n" - ".inst 0xa0418349 // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b531 // udot za.s[x9, 1], { z8.b-z11.b }, z3.b[1]\n" - ".inst 0xa0428351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153b632 // udot za.s[x9, 2], { z16.b-z19.b }, z3.b[1]\n" + ".inst 0xa040834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151b5b0 // udot za.s[x9, 0], { z12.b-z15.b }, z1.b[1]\n" + ".inst 0xa041834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b5b1 // udot za.s[x9, 1], { z12.b-z15.b }, z1.b[1]\n" + ".inst 0xa0428355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc151b6b2 // udot za.s[x9, 2], { z20.b-z23.b }, z1.b[1]\n" "addvl x26, x26, #16\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153bab0 // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n" + ".inst 0xa0408349 // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151b930 // udot za.s[x9, 0], { z8.b-z11.b }, z1.b[2]\n" ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b8b1 // udot za.s[x9, 1], { z4.b-z7.b }, z3.b[2]\n" - ".inst 0xa042834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153b9b2 // udot za.s[x9, 2], { z12.b-z15.b }, z3.b[2]\n" + ".inst 0xc151b8b1 // udot za.s[x9, 1], { z4.b-z7.b }, z1.b[2]\n" + ".inst 0xa0428351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc151ba32 // udot za.s[x9, 2], { z16.b-z19.b }, z1.b[2]\n" "addvl x26, x26, #16\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153beb0 // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n" - ".inst 0xa0418355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153beb1 // udot za.s[x9, 1], { z20.b-z23.b }, z3.b[3]\n" + ".inst 0xa0408359 // ldnt1b { z24.b-z27.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151bf30 // udot za.s[x9, 0], { z24.b-z27.b }, z1.b[3]\n" + ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151bcb1 // udot za.s[x9, 1], { z4.b-z7.b }, z1.b[3]\n" ".inst 0xa0428351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153be32 // udot za.s[x9, 2], { z16.b-z19.b }, z3.b[3]\n" + ".inst 0xc151be32 // udot za.s[x9, 2], { z16.b-z19.b }, z1.b[3]\n" "addvl x26, x26, #16\n" "tbnz %x[flags], #31, 28f\n" - "udot z26.s, z3.b, z24.b\n" + "udot z28.s, z1.b, z29.b\n" "28:" // Width 3: Multiply loop: unique 5: skip row sum - "sub x21, x21, #0x10\n" - "cmp x21, #0x10\n" + "sub x22, x22, #0x10\n" + "cmp x22, #0x10\n" "bgt 27b\n" "29:" // Width 3: Multiply loop: Single iteration only - "whilelt p0.b, XZR, x21\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" - "subs x21, x21, #0x4\n" - ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" + "whilelt p0.b, XZR, x22\n" + "ld1rqb { z1.b }, p0/Z, [x23]\n" + "subs x22, x22, #0x4\n" + ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" "add x23, x23, #0x10\n" - ".inst 0xc153b230 // udot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n" - ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b0b1 // udot za.s[x9, 1], { z4.b-z7.b }, z3.b[0]\n" - ".inst 0xa042834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153b1b2 // udot za.s[x9, 2], { z12.b-z15.b }, z3.b[0]\n" + ".inst 0xc151b2b0 // udot za.s[x9, 0], { z20.b-z23.b }, z1.b[0]\n" + ".inst 0xa0418351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b231 // udot za.s[x9, 1], { z16.b-z19.b }, z1.b[0]\n" + ".inst 0xa0428351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc151b232 // udot za.s[x9, 2], { z16.b-z19.b }, z1.b[0]\n" "addvl x26, x26, #16\n" "ble 30f\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - "subs x21, x21, #0x4\n" - ".inst 0xc153b6b0 // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n" - ".inst 0xa0418349 // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b531 // udot za.s[x9, 1], { z8.b-z11.b }, z3.b[1]\n" + ".inst 0xa0408359 // ldnt1b { z24.b-z27.b }, pn8.b/Z, [x26]\n" + "subs x22, x22, #0x4\n" + ".inst 0xc151b730 // udot za.s[x9, 0], { z24.b-z27.b }, z1.b[1]\n" + ".inst 0xa041834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b5b1 // udot za.s[x9, 1], { z12.b-z15.b }, z1.b[1]\n" ".inst 0xa0428351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153b632 // udot za.s[x9, 2], { z16.b-z19.b }, z3.b[1]\n" + ".inst 0xc151b632 // udot za.s[x9, 2], { z16.b-z19.b }, z1.b[1]\n" "addvl x26, x26, #16\n" "ble 30f\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - "subs x21, x21, #0x4\n" - ".inst 0xc153bab0 // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n" - ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b8b1 // udot za.s[x9, 1], { z4.b-z7.b }, z3.b[2]\n" - ".inst 0xa042834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153b9b2 // udot za.s[x9, 2], { z12.b-z15.b }, z3.b[2]\n" + ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" + "subs x22, x22, #0x4\n" + ".inst 0xc151ba30 // udot za.s[x9, 0], { z16.b-z19.b }, z1.b[2]\n" + ".inst 0xa0418351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151ba31 // udot za.s[x9, 1], { z16.b-z19.b }, z1.b[2]\n" + ".inst 0xa0428355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc151bab2 // udot za.s[x9, 2], { z20.b-z23.b }, z1.b[2]\n" "addvl x26, x26, #16\n" "ble 30f\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153beb0 // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n" - ".inst 0xa0418355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153beb1 // udot za.s[x9, 1], { z20.b-z23.b }, z3.b[3]\n" - ".inst 0xa0428351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153be32 // udot za.s[x9, 2], { z16.b-z19.b }, z3.b[3]\n" + ".inst 0xa040834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151bdb0 // udot za.s[x9, 0], { z12.b-z15.b }, z1.b[3]\n" + ".inst 0xa0418351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151be31 // udot za.s[x9, 1], { z16.b-z19.b }, z1.b[3]\n" + ".inst 0xa042834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc151bdb2 // udot za.s[x9, 2], { z12.b-z15.b }, z1.b[3]\n" "addvl x26, x26, #16\n" "30:" // Width 3: Multiply loop: multiply skip "tbnz %x[flags], #31, 31f\n" - "udot z26.s, z3.b, z24.b\n" + "udot z28.s, z1.b, z29.b\n" "31:" // Width 3: Multiply loop: unique 6: skip row sum "tbnz %x[flags], #31, 32f\n" "add x21, %x[qp], %[b_offset]\n" "mov x20, #0x4\n" - "ld1rw { z10.s }, p2/Z, [x21]\n" - "neg z10.s, p2/M, z10.s\n" + "ld1rw { z16.s }, p2/Z, [x21]\n" + "neg z16.s, p2/M, z16.s\n" "whilelt p0.s, XZR, x20\n" - "uaddv d26, p0, z26.s\n" - "mov z26.s, z26.s[0]\n" - "mul z26.s, p2/M, z26.s, z10.s\n" + "uaddv d28, p0, z28.s\n" + "mov z28.s, z28.s[0]\n" + "mul z28.s, p2/M, z28.s, z16.s\n" "orr %x[flags], %x[flags], #0x80000000\n" "32:" // Width 3: skip row sum fixup - ".inst 0xc0904b40 // addha za0.s, p2/M, p2/M, z26.s\n" + ".inst 0xc0904b80 // addha za0.s, p2/M, p2/M, z28.s\n" "add x20, %x[qp], %[per_layer_mul]\n" - "ld1rw { z5.s }, p2/Z, [x20]\n" + "ld1rw { z2.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[per_layer_right_shift]\n" - ".inst 0xc0904b41 // addha za1.s, p2/M, p2/M, z26.s\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + ".inst 0xc0904b81 // addha za1.s, p2/M, p2/M, z28.s\n" + "ld1rw { z1.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[c_offset]\n" "add x21, %x[qp], %[minval]\n" - ".inst 0xc0904b42 // addha za2.s, p2/M, p2/M, z26.s\n" - "ld1rw { z6.s }, p2/Z, [x20]\n" + ".inst 0xc0904b82 // addha za2.s, p2/M, p2/M, z28.s\n" + "ld1rw { z3.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[maxval]\n" - ".inst 0xc0904b43 // addha za3.s, p2/M, p2/M, z26.s\n" - "ld1rw { z21.s }, p2/Z, [x21]\n" - ".inst 0xc0062c1c // mova { z28.d-z31.d }, za.d[x9, #0]\n" - ".inst 0xc1a5ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z5.s\n" - ".inst 0xc0062c2c // mova { z12.d-z15.d }, za.d[x9, #1]\n" - ".inst 0xc1a5ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z5.s\n" - ".inst 0xc0062c40 // mova { z0.d-z3.d }, za.d[x9, #2]\n" - ".inst 0xc1a5ac00 // sqdmulh { z0.s-z3.s }, { z0.s-z3.s }, z5.s\n" - ".inst 0xc1a4aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" - "ld1rw { z16.s }, p2/Z, [x20]\n" - ".inst 0xc1a4aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z4.s\n" - ".inst 0xc1a4aa20 // srshl { z0.s-z3.s }, { z0.s-z3.s }, z4.s\n" - ".inst 0xc1a6ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z6.s\n" - ".inst 0xc1a6ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z6.s\n" - ".inst 0xc1a6ab00 // add { z0.s-z3.s }, { z0.s-z3.s }, z6.s\n" - ".inst 0xc1b0cebc // sclamp { z28.s-z31.s }, z21.s, z16.s\n" - ".inst 0xc1b0ceac // sclamp { z12.s-z15.s }, z21.s, z16.s\n" - "uzp1 z28.h, z28.h, z29.h\n" - ".inst 0xc1b0cea0 // sclamp { z0.s-z3.s }, z21.s, z16.s\n" - "uzp1 z29.h, z30.h, z31.h\n" + ".inst 0xc0904b83 // addha za3.s, p2/M, p2/M, z28.s\n" + "ld1rw { z16.s }, p2/Z, [x21]\n" + ".inst 0xc0062c08 // mova { z8.d-z11.d }, za.d[x9, #0]\n" + ".inst 0xc1a2ac08 // sqdmulh { z8.s-z11.s }, { z8.s-z11.s }, z2.s\n" + ".inst 0xc0062c24 // mova { z4.d-z7.d }, za.d[x9, #1]\n" + ".inst 0xc1a2ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z2.s\n" + ".inst 0xc0062c4c // mova { z12.d-z15.d }, za.d[x9, #2]\n" + ".inst 0xc1a2ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z2.s\n" + ".inst 0xc1a1aa28 // srshl { z8.s-z11.s }, { z8.s-z11.s }, z1.s\n" + "ld1rw { z0.s }, p2/Z, [x20]\n" + ".inst 0xc1a1aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" + ".inst 0xc1a1aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n" + ".inst 0xc1a3ab08 // add { z8.s-z11.s }, { z8.s-z11.s }, z3.s\n" + ".inst 0xc1a3ab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z3.s\n" + ".inst 0xc1a3ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z3.s\n" + ".inst 0xc1a0ce08 // sclamp { z8.s-z11.s }, z16.s, z0.s\n" + ".inst 0xc1a0ce04 // sclamp { z4.s-z7.s }, z16.s, z0.s\n" + "uzp1 z8.h, z8.h, z9.h\n" + ".inst 0xc1a0ce0c // sclamp { z12.s-z15.s }, z16.s, z0.s\n" + "uzp1 z18.h, z10.h, z11.h\n" + "uzp1 z4.h, z4.h, z5.h\n" + "uzp1 z17.h, z6.h, z7.h\n" "uzp1 z12.h, z12.h, z13.h\n" - "uzp1 z13.h, z14.h, z15.h\n" - "uzp1 z0.h, z0.h, z1.h\n" - "uzp1 z1.h, z2.h, z3.h\n" - "uzp1 z28.b, z28.b, z29.b\n" - "st1b { z28.b }, p2, [x25]\n" - "uzp1 z12.b, z12.b, z13.b\n" - "st1b { z12.b }, p2, [x25, #1, MUL VL]\n" - "uzp1 z0.b, z0.b, z1.b\n" - "st1b { z0.b }, p1, [x25, #2, MUL VL]\n" + "uzp1 z16.h, z14.h, z15.h\n" + "uzp1 z8.b, z8.b, z18.b\n" + "st1b { z8.b }, p2, [x25]\n" + "uzp1 z4.b, z4.b, z17.b\n" + "st1b { z4.b }, p2, [x25, #1, MUL VL]\n" + "uzp1 z12.b, z12.b, z16.b\n" + "st1b { z12.b }, p1, [x25, #2, MUL VL]\n" "addvl x25, x25, #3\n" "33:" // Width 3: Output done "b 44f\n" "34:" // Width 4 "mov x20, #0x3\n" "mov x23, %x[A_ptr]\n" - "mov x22, %x[K]\n" - "msub x20, x28, x20, %x[N]\n" "mov x21, %x[K]\n" - ".inst 0xf8b64af8 // rprfm pldmany, x22, [x23]\n" + "msub x20, x28, x20, %x[N]\n" + "mov x22, %x[K]\n" + ".inst 0xf8b54af8 // rprfm pldmany, x21, [x23]\n" "whilelt p1.b, XZR, x20\n" "cbz x24, 35f\n" - ".inst 0xa040c304 // ld1w { z4.s-z7.s }, pn8.b/Z, [x24]\n" - ".inst 0xc0042c80 // mova za.d[x9, #0], { z4.d-z7.d }\n" - ".inst 0xa041c314 // ld1w { z20.s-z23.s }, pn8.b/Z, [x24, #0x4, MUL VL]\n" - ".inst 0xc0042e81 // mova za.d[x9, #1], { z20.d-z23.d }\n" - ".inst 0xa042c314 // ld1w { z20.s-z23.s }, pn8.b/Z, [x24, #0x8, MUL VL]\n" - ".inst 0xc0042e82 // mova za.d[x9, #2], { z20.d-z23.d }\n" + ".inst 0xa040c314 // ld1w { z20.s-z23.s }, pn8.b/Z, [x24]\n" + ".inst 0xc0042e80 // mova za.d[x9, #0], { z20.d-z23.d }\n" + ".inst 0xa041c310 // ld1w { z16.s-z19.s }, pn8.b/Z, [x24, #0x4, MUL VL]\n" + ".inst 0xc0042e01 // mova za.d[x9, #1], { z16.d-z19.d }\n" + ".inst 0xa042c310 // ld1w { z16.s-z19.s }, pn8.b/Z, [x24, #0x8, MUL VL]\n" + ".inst 0xc0042e02 // mova za.d[x9, #2], { z16.d-z19.d }\n" ".inst 0xa043c310 // ld1w { z16.s-z19.s }, pn8.b/Z, [x24, #0xc, MUL VL]\n" ".inst 0xc0042e03 // mova za.d[x9, #3], { z16.d-z19.d }\n" "addvl x24, x24, #16\n" @@ -499,165 +497,165 @@ void sme2_gemv_u8qa_dot_16VL ( "35:" // Width 4: no bias ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "36:" // Width 4: setup done - "cmp x21, #0x10\n" + "cmp x22, #0x10\n" "ble 39f\n" "37:" // Width 4: Multiply loop: Main loop head - "whilelt p0.b, XZR, x21\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" + "whilelt p0.b, XZR, x22\n" + "ld1rqb { z1.b }, p0/Z, [x23]\n" "add x23, x23, #0x10\n" ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153b230 // udot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n" - ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b0b1 // udot za.s[x9, 1], { z4.b-z7.b }, z3.b[0]\n" + ".inst 0xc151b230 // udot za.s[x9, 0], { z16.b-z19.b }, z1.b[0]\n" + ".inst 0xa0418351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b231 // udot za.s[x9, 1], { z16.b-z19.b }, z1.b[0]\n" ".inst 0xa042834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153b1b2 // udot za.s[x9, 2], { z12.b-z15.b }, z3.b[0]\n" + ".inst 0xc151b1b2 // udot za.s[x9, 2], { z12.b-z15.b }, z1.b[0]\n" ".inst 0xa043834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc153b1b3 // udot za.s[x9, 3], { z12.b-z15.b }, z3.b[0]\n" + ".inst 0xc151b1b3 // udot za.s[x9, 3], { z12.b-z15.b }, z1.b[0]\n" "addvl x26, x26, #16\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153b6b0 // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n" - ".inst 0xa0418349 // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b531 // udot za.s[x9, 1], { z8.b-z11.b }, z3.b[1]\n" + ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151b630 // udot za.s[x9, 0], { z16.b-z19.b }, z1.b[1]\n" + ".inst 0xa041834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b5b1 // udot za.s[x9, 1], { z12.b-z15.b }, z1.b[1]\n" ".inst 0xa0428351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153b632 // udot za.s[x9, 2], { z16.b-z19.b }, z3.b[1]\n" - ".inst 0xa043834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc153b5b3 // udot za.s[x9, 3], { z12.b-z15.b }, z3.b[1]\n" - "addvl x26, x26, #16\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153bab0 // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n" - ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b8b1 // udot za.s[x9, 1], { z4.b-z7.b }, z3.b[2]\n" - ".inst 0xa042834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153b9b2 // udot za.s[x9, 2], { z12.b-z15.b }, z3.b[2]\n" - ".inst 0xa043835d // ldnt1b { z28.b-z31.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc153bbb3 // udot za.s[x9, 3], { z28.b-z31.b }, z3.b[2]\n" + ".inst 0xc151b632 // udot za.s[x9, 2], { z16.b-z19.b }, z1.b[1]\n" + ".inst 0xa0438351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc151b633 // udot za.s[x9, 3], { z16.b-z19.b }, z1.b[1]\n" "addvl x26, x26, #16\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153beb0 // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n" - ".inst 0xa0418355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153beb1 // udot za.s[x9, 1], { z20.b-z23.b }, z3.b[3]\n" + ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151ba30 // udot za.s[x9, 0], { z16.b-z19.b }, z1.b[2]\n" + ".inst 0xa041834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b9b1 // udot za.s[x9, 1], { z12.b-z15.b }, z1.b[2]\n" ".inst 0xa0428351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153be32 // udot za.s[x9, 2], { z16.b-z19.b }, z3.b[3]\n" + ".inst 0xc151ba32 // udot za.s[x9, 2], { z16.b-z19.b }, z1.b[2]\n" ".inst 0xa0438351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc153be33 // udot za.s[x9, 3], { z16.b-z19.b }, z3.b[3]\n" + ".inst 0xc151ba33 // udot za.s[x9, 3], { z16.b-z19.b }, z1.b[2]\n" + "addvl x26, x26, #16\n" + ".inst 0xa040834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151bdb0 // udot za.s[x9, 0], { z12.b-z15.b }, z1.b[3]\n" + ".inst 0xa041834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151bdb1 // udot za.s[x9, 1], { z12.b-z15.b }, z1.b[3]\n" + ".inst 0xa0428359 // ldnt1b { z24.b-z27.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc151bf32 // udot za.s[x9, 2], { z24.b-z27.b }, z1.b[3]\n" + ".inst 0xa0438345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc151bcb3 // udot za.s[x9, 3], { z4.b-z7.b }, z1.b[3]\n" "addvl x26, x26, #16\n" "tbnz %x[flags], #31, 38f\n" - "udot z26.s, z3.b, z24.b\n" + "udot z28.s, z1.b, z29.b\n" "38:" // Width 4: Multiply loop: unique 7: skip row sum - "sub x21, x21, #0x10\n" - "cmp x21, #0x10\n" + "sub x22, x22, #0x10\n" + "cmp x22, #0x10\n" "bgt 37b\n" "39:" // Width 4: Multiply loop: Single iteration only - "whilelt p0.b, XZR, x21\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" - "subs x21, x21, #0x4\n" - ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" + "whilelt p0.b, XZR, x22\n" + "ld1rqb { z1.b }, p0/Z, [x23]\n" + "subs x22, x22, #0x4\n" + ".inst 0xa040834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26]\n" "add x23, x23, #0x10\n" - ".inst 0xc153b230 // udot za.s[x9, 0], { z16.b-z19.b }, z3.b[0]\n" - ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b0b1 // udot za.s[x9, 1], { z4.b-z7.b }, z3.b[0]\n" - ".inst 0xa042834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153b1b2 // udot za.s[x9, 2], { z12.b-z15.b }, z3.b[0]\n" - ".inst 0xa043834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc153b1b3 // udot za.s[x9, 3], { z12.b-z15.b }, z3.b[0]\n" + ".inst 0xc151b1b0 // udot za.s[x9, 0], { z12.b-z15.b }, z1.b[0]\n" + ".inst 0xa0418359 // ldnt1b { z24.b-z27.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b331 // udot za.s[x9, 1], { z24.b-z27.b }, z1.b[0]\n" + ".inst 0xa0428349 // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc151b132 // udot za.s[x9, 2], { z8.b-z11.b }, z1.b[0]\n" + ".inst 0xa0438351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc151b233 // udot za.s[x9, 3], { z16.b-z19.b }, z1.b[0]\n" "addvl x26, x26, #16\n" "ble 40f\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - "subs x21, x21, #0x4\n" - ".inst 0xc153b6b0 // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[1]\n" - ".inst 0xa0418349 // ldnt1b { z8.b-z11.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b531 // udot za.s[x9, 1], { z8.b-z11.b }, z3.b[1]\n" - ".inst 0xa0428351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153b632 // udot za.s[x9, 2], { z16.b-z19.b }, z3.b[1]\n" - ".inst 0xa043834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc153b5b3 // udot za.s[x9, 3], { z12.b-z15.b }, z3.b[1]\n" + ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" + "subs x22, x22, #0x4\n" + ".inst 0xc151b630 // udot za.s[x9, 0], { z16.b-z19.b }, z1.b[1]\n" + ".inst 0xa0418351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151b631 // udot za.s[x9, 1], { z16.b-z19.b }, z1.b[1]\n" + ".inst 0xa042834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc151b5b2 // udot za.s[x9, 2], { z12.b-z15.b }, z1.b[1]\n" + ".inst 0xa0438355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc151b6b3 // udot za.s[x9, 3], { z20.b-z23.b }, z1.b[1]\n" "addvl x26, x26, #16\n" "ble 40f\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - "subs x21, x21, #0x4\n" - ".inst 0xc153bab0 // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[2]\n" - ".inst 0xa0418345 // ldnt1b { z4.b-z7.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153b8b1 // udot za.s[x9, 1], { z4.b-z7.b }, z3.b[2]\n" - ".inst 0xa042834d // ldnt1b { z12.b-z15.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153b9b2 // udot za.s[x9, 2], { z12.b-z15.b }, z3.b[2]\n" - ".inst 0xa043835d // ldnt1b { z28.b-z31.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc153bbb3 // udot za.s[x9, 3], { z28.b-z31.b }, z3.b[2]\n" + ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" + "subs x22, x22, #0x4\n" + ".inst 0xc151ba30 // udot za.s[x9, 0], { z16.b-z19.b }, z1.b[2]\n" + ".inst 0xa0418351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151ba31 // udot za.s[x9, 1], { z16.b-z19.b }, z1.b[2]\n" + ".inst 0xa0428351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" + ".inst 0xc151ba32 // udot za.s[x9, 2], { z16.b-z19.b }, z1.b[2]\n" + ".inst 0xa0438351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" + ".inst 0xc151ba33 // udot za.s[x9, 3], { z16.b-z19.b }, z1.b[2]\n" "addvl x26, x26, #16\n" "ble 40f\n" - ".inst 0xa0408355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26]\n" - ".inst 0xc153beb0 // udot za.s[x9, 0], { z20.b-z23.b }, z3.b[3]\n" - ".inst 0xa0418355 // ldnt1b { z20.b-z23.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" - ".inst 0xc153beb1 // udot za.s[x9, 1], { z20.b-z23.b }, z3.b[3]\n" + ".inst 0xa0408351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26]\n" + ".inst 0xc151be30 // udot za.s[x9, 0], { z16.b-z19.b }, z1.b[3]\n" + ".inst 0xa0418351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x4, MUL VL]\n" + ".inst 0xc151be31 // udot za.s[x9, 1], { z16.b-z19.b }, z1.b[3]\n" ".inst 0xa0428351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0x8, MUL VL]\n" - ".inst 0xc153be32 // udot za.s[x9, 2], { z16.b-z19.b }, z3.b[3]\n" + ".inst 0xc151be32 // udot za.s[x9, 2], { z16.b-z19.b }, z1.b[3]\n" ".inst 0xa0438351 // ldnt1b { z16.b-z19.b }, pn8.b/Z, [x26, #0xc, MUL VL]\n" - ".inst 0xc153be33 // udot za.s[x9, 3], { z16.b-z19.b }, z3.b[3]\n" + ".inst 0xc151be33 // udot za.s[x9, 3], { z16.b-z19.b }, z1.b[3]\n" "addvl x26, x26, #16\n" "40:" // Width 4: Multiply loop: multiply skip "tbnz %x[flags], #31, 41f\n" - "udot z26.s, z3.b, z24.b\n" + "udot z28.s, z1.b, z29.b\n" "41:" // Width 4: Multiply loop: unique 8: skip row sum "tbnz %x[flags], #31, 42f\n" "add x21, %x[qp], %[b_offset]\n" "mov x20, #0x4\n" - "ld1rw { z10.s }, p2/Z, [x21]\n" - "neg z10.s, p2/M, z10.s\n" + "ld1rw { z16.s }, p2/Z, [x21]\n" + "neg z16.s, p2/M, z16.s\n" "whilelt p0.s, XZR, x20\n" - "uaddv d26, p0, z26.s\n" - "mov z26.s, z26.s[0]\n" - "mul z26.s, p2/M, z26.s, z10.s\n" + "uaddv d28, p0, z28.s\n" + "mov z28.s, z28.s[0]\n" + "mul z28.s, p2/M, z28.s, z16.s\n" "orr %x[flags], %x[flags], #0x80000000\n" "42:" // Width 4: skip row sum fixup - ".inst 0xc0904b40 // addha za0.s, p2/M, p2/M, z26.s\n" + ".inst 0xc0904b80 // addha za0.s, p2/M, p2/M, z28.s\n" "add x20, %x[qp], %[per_layer_mul]\n" - "ld1rw { z5.s }, p2/Z, [x20]\n" + "ld1rw { z11.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[per_layer_right_shift]\n" - ".inst 0xc0904b41 // addha za1.s, p2/M, p2/M, z26.s\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + ".inst 0xc0904b81 // addha za1.s, p2/M, p2/M, z28.s\n" + "ld1rw { z7.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[c_offset]\n" "add x21, %x[qp], %[minval]\n" - ".inst 0xc0904b42 // addha za2.s, p2/M, p2/M, z26.s\n" + ".inst 0xc0904b82 // addha za2.s, p2/M, p2/M, z28.s\n" "ld1rw { z6.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[maxval]\n" - ".inst 0xc0904b43 // addha za3.s, p2/M, p2/M, z26.s\n" - "ld1rw { z21.s }, p2/Z, [x21]\n" - ".inst 0xc0062c1c // mova { z28.d-z31.d }, za.d[x9, #0]\n" - ".inst 0xc1a5ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z5.s\n" - ".inst 0xc0062c2c // mova { z12.d-z15.d }, za.d[x9, #1]\n" - ".inst 0xc1a5ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z5.s\n" - ".inst 0xc0062c40 // mova { z0.d-z3.d }, za.d[x9, #2]\n" - ".inst 0xc1a5ac00 // sqdmulh { z0.s-z3.s }, { z0.s-z3.s }, z5.s\n" - ".inst 0xc0062c68 // mova { z8.d-z11.d }, za.d[x9, #3]\n" - ".inst 0xc1a5ac08 // sqdmulh { z8.s-z11.s }, { z8.s-z11.s }, z5.s\n" - ".inst 0xc1a4aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z4.s\n" - "ld1rw { z16.s }, p2/Z, [x20]\n" - ".inst 0xc1a4aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z4.s\n" - ".inst 0xc1a4aa20 // srshl { z0.s-z3.s }, { z0.s-z3.s }, z4.s\n" - ".inst 0xc1a4aa28 // srshl { z8.s-z11.s }, { z8.s-z11.s }, z4.s\n" - ".inst 0xc1a6ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z6.s\n" + ".inst 0xc0904b83 // addha za3.s, p2/M, p2/M, z28.s\n" + "ld1rw { z3.s }, p2/Z, [x21]\n" + ".inst 0xc0062c18 // mova { z24.d-z27.d }, za.d[x9, #0]\n" + ".inst 0xc1abac18 // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z11.s\n" + ".inst 0xc0062c30 // mova { z16.d-z19.d }, za.d[x9, #1]\n" + ".inst 0xc1abac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z11.s\n" + ".inst 0xc0062c54 // mova { z20.d-z23.d }, za.d[x9, #2]\n" + ".inst 0xc1abac14 // sqdmulh { z20.s-z23.s }, { z20.s-z23.s }, z11.s\n" + ".inst 0xc0062c6c // mova { z12.d-z15.d }, za.d[x9, #3]\n" + ".inst 0xc1abac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z11.s\n" + ".inst 0xc1a7aa38 // srshl { z24.s-z27.s }, { z24.s-z27.s }, z7.s\n" + "ld1rw { z31.s }, p2/Z, [x20]\n" + ".inst 0xc1a7aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z7.s\n" + ".inst 0xc1a7aa34 // srshl { z20.s-z23.s }, { z20.s-z23.s }, z7.s\n" + ".inst 0xc1a7aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z7.s\n" + ".inst 0xc1a6ab18 // add { z24.s-z27.s }, { z24.s-z27.s }, z6.s\n" + ".inst 0xc1a6ab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n" + ".inst 0xc1a6ab14 // add { z20.s-z23.s }, { z20.s-z23.s }, z6.s\n" ".inst 0xc1a6ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z6.s\n" - ".inst 0xc1a6ab00 // add { z0.s-z3.s }, { z0.s-z3.s }, z6.s\n" - ".inst 0xc1a6ab08 // add { z8.s-z11.s }, { z8.s-z11.s }, z6.s\n" - ".inst 0xc1b0cebc // sclamp { z28.s-z31.s }, z21.s, z16.s\n" - ".inst 0xc1b0ceac // sclamp { z12.s-z15.s }, z21.s, z16.s\n" - "uzp1 z28.h, z28.h, z29.h\n" - ".inst 0xc1b0cea0 // sclamp { z0.s-z3.s }, z21.s, z16.s\n" - ".inst 0xc1b0cea8 // sclamp { z8.s-z11.s }, z21.s, z16.s\n" - "uzp1 z29.h, z30.h, z31.h\n" + ".inst 0xc1bfcc78 // sclamp { z24.s-z27.s }, z3.s, z31.s\n" + ".inst 0xc1bfcc70 // sclamp { z16.s-z19.s }, z3.s, z31.s\n" + "uzp1 z24.h, z24.h, z25.h\n" + ".inst 0xc1bfcc74 // sclamp { z20.s-z23.s }, z3.s, z31.s\n" + ".inst 0xc1bfcc6c // sclamp { z12.s-z15.s }, z3.s, z31.s\n" + "uzp1 z25.h, z26.h, z27.h\n" + "uzp1 z16.h, z16.h, z17.h\n" + "uzp1 z18.h, z18.h, z19.h\n" + "uzp1 z20.h, z20.h, z21.h\n" + "uzp1 z17.h, z22.h, z23.h\n" "uzp1 z12.h, z12.h, z13.h\n" - "uzp1 z13.h, z14.h, z15.h\n" - "uzp1 z0.h, z0.h, z1.h\n" - "uzp1 z1.h, z2.h, z3.h\n" - "uzp1 z8.h, z8.h, z9.h\n" - "uzp1 z9.h, z10.h, z11.h\n" - "uzp1 z28.b, z28.b, z29.b\n" - "st1b { z28.b }, p2, [x25]\n" - "uzp1 z12.b, z12.b, z13.b\n" - "st1b { z12.b }, p2, [x25, #1, MUL VL]\n" - "uzp1 z0.b, z0.b, z1.b\n" - "uzp1 z8.b, z8.b, z9.b\n" - "st1b { z0.b }, p2, [x25, #2, MUL VL]\n" - "st1b { z8.b }, p1, [x25, #3, MUL VL]\n" + "uzp1 z30.h, z14.h, z15.h\n" + "uzp1 z24.b, z24.b, z25.b\n" + "st1b { z24.b }, p2, [x25]\n" + "uzp1 z16.b, z16.b, z18.b\n" + "st1b { z16.b }, p2, [x25, #1, MUL VL]\n" + "uzp1 z20.b, z20.b, z17.b\n" + "uzp1 z12.b, z12.b, z30.b\n" + "st1b { z20.b }, p2, [x25, #2, MUL VL]\n" + "st1b { z12.b }, p1, [x25, #3, MUL VL]\n" "addvl x25, x25, #4\n" "43:" // Width 4: Output done "subs x27, x27, #0x4\n" @@ -665,7 +663,7 @@ void sme2_gemv_u8qa_dot_16VL ( "bgt 4b\n" "44:" // Exit ".inst 0xd503467f // SMSTOP\n" - "ptrue p2.b\n" + "ptrue p8.b\n" : [N] "+&r" (N), [flags] "+&r" (flags) : [A_ptr] "r" (A_ptr), [B_ptr] "r" (B_ptr), [K] "r" (K), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [output_ptr] "r" (output_ptr), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp) : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -674,5 +672,4 @@ void sme2_gemv_u8qa_dot_16VL ( } // namespace arm_gemm -#endif // ARM_COMPUTE_ENABLE_SME2 -#endif // __ARM_FEATURE_SVE +#endif // defined(ARM_COMPUTE_ENABLE_SME2) diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL.hpp index 37eb63d898e68fffea3ef139992dfec6ff7b444b..edfb362aab297b8fa4529b8763e2629456eba8c9 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,7 +23,7 @@ */ #pragma once -#ifdef __aarch64__ +#ifdef ARM_COMPUTE_ENABLE_SME2 #include "../bfloat.hpp" #include "../std_transforms_sme.hpp" @@ -83,12 +83,11 @@ public: StdTransformsSME transforms = {}; - cls_sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL(const CPUInfo *ci) + cls_sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL(const CPUInfo *) { - ARM_COMPUTE_UNUSED(ci); } }; } // namespace arm_gemm -#endif // __aarch64__ +#endif // ARM_COMPUTE_ENABLE_SME2 diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL/generic.cpp index c6eb858ade94460e21c998e5b0eafab8021a8393..8105300cb744eeba670d94eff288fd0dec092905 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL/generic.cpp @@ -21,7 +21,6 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifdef __ARM_FEATURE_SVE #ifdef ARM_COMPUTE_ENABLE_SME2 #include "arm_gemm.hpp" @@ -113,12 +112,12 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL(const bfloat16 *const A, con "mov x12, #0x0\n" "cntw x20\n" "1:" // Initial accumulator load from buffer: Loop - ".inst 0xa040c5cc // ld1w { z12.s-z15.s }, pn9.b/Z, [x14]\n" - ".inst 0xc0840580 // mova za0h.s[x12], { z12.s-z15.s }\n" - ".inst 0xa041c5d0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n" - ".inst 0xc0840601 // mova za1h.s[x12], { z16.s-z19.s }\n" - ".inst 0xa042c5c4 // ld1w { z4.s-z7.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n" - ".inst 0xc0840482 // mova za2h.s[x12], { z4.s-z7.s }\n" + ".inst 0xa040c5c4 // ld1w { z4.s-z7.s }, pn9.b/Z, [x14]\n" + ".inst 0xc0840480 // mova za0h.s[x12], { z4.s-z7.s }\n" + ".inst 0xa041c5d8 // ld1w { z24.s-z27.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n" + ".inst 0xc0840701 // mova za1h.s[x12], { z24.s-z27.s }\n" + ".inst 0xa042c5d4 // ld1w { z20.s-z23.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n" + ".inst 0xc0840682 // mova za2h.s[x12], { z20.s-z23.s }\n" ".inst 0xa043c5c4 // ld1w { z4.s-z7.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n" ".inst 0xc0840483 // mova za3h.s[x12], { z4.s-z7.s }\n" "add x12, x12, #0x4\n" @@ -138,12 +137,12 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL(const bfloat16 *const A, con "ldr x20, [%x[args], %[offsetof_bias]]\n" ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "cbz x20, 5f\n" - "fmov z21.s, #1.0\n" + "fmov z6.s, #1.0\n" ".inst 0xa009c29d // ldnt1w { z28.s-z31.s }, p8/Z, [x20, x9, LSL #2]\n" - ".inst 0x809c02a0 // fmopa za0.s, p0/M, p0/M, z21.s, z28.s\n" - ".inst 0x809d02a1 // fmopa za1.s, p0/M, p0/M, z21.s, z29.s\n" - ".inst 0x809e02a2 // fmopa za2.s, p0/M, p0/M, z21.s, z30.s\n" - ".inst 0x809f02a3 // fmopa za3.s, p0/M, p0/M, z21.s, z31.s\n" + ".inst 0x809c00c0 // fmopa za0.s, p0/M, p0/M, z6.s, z28.s\n" + ".inst 0x809d00c1 // fmopa za1.s, p0/M, p0/M, z6.s, z29.s\n" + ".inst 0x809e00c2 // fmopa za2.s, p0/M, p0/M, z6.s, z30.s\n" + ".inst 0x809f00c3 // fmopa za3.s, p0/M, p0/M, z6.s, z31.s\n" "4:" // Prepare accumulators: Test for last block "mov x20, x9\n" "mov x21, x10\n" @@ -166,75 +165,75 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL(const bfloat16 *const A, con "madd x23, x9, x20, x23\n" // bptr = B + n * kstride_bytes "cbz x22, 8f\n" "subs x22, x22, #0x1\n" - "ld1h { z0.h }, p0/Z, [x26]\n" - ".inst 0xa140a6fb // ldnt1h { z19.h, z23.h, z27.h, z31.h }, pn9.b/Z, [x23]\n" - "ld1h { z13.h }, p0/Z, [x26, #1, MUL VL]\n" - ".inst 0xa141a6ea // ldnt1h { z2.h, z6.h, z10.h, z14.h }, pn9.b/Z, [x23, #0x4, MUL VL]\n" - "ld1h { z12.h }, p0/Z, [x26, #2, MUL VL]\n" - ".inst 0xa142a6eb // ldnt1h { z3.h, z7.h, z11.h, z15.h }, pn9.b/Z, [x23, #0x8, MUL VL]\n" - "ld1h { z26.h }, p0/Z, [x26, #3, MUL VL]\n" + "ld1h { z28.h }, p0/Z, [x26]\n" + ".inst 0xa040a6e9 // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x23]\n" + "ld1h { z22.h }, p0/Z, [x26, #1, MUL VL]\n" + ".inst 0xa041a6ed // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x23, #0x4, MUL VL]\n" + "ld1h { z30.h }, p0/Z, [x26, #2, MUL VL]\n" + ".inst 0xa042a6e5 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x23, #0x8, MUL VL]\n" + "ld1h { z20.h }, p0/Z, [x26, #3, MUL VL]\n" "addvl x26, x26, #4\n" - ".inst 0xa143a6f8 // ldnt1h { z16.h, z20.h, z24.h, z28.h }, pn9.b/Z, [x23, #0xc, MUL VL]\n" + ".inst 0xa143a6fb // ldnt1h { z19.h, z23.h, z27.h, z31.h }, pn9.b/Z, [x23, #0xc, MUL VL]\n" "addvl x23, x23, #16\n" "ble 7f\n" "6:" // K loop - ".inst 0x81930000 // bfmopa za0.s, p0/M, p0/M, z0.h, z19.h\n" + ".inst 0x81880380 // bfmopa za0.s, p0/M, p0/M, z28.h, z8.h\n" "subs x22, x22, #0x1\n" - ".inst 0x81970001 // bfmopa za1.s, p0/M, p0/M, z0.h, z23.h\n" - ".inst 0x819b0002 // bfmopa za2.s, p0/M, p0/M, z0.h, z27.h\n" - ".inst 0x819f0003 // bfmopa za3.s, p0/M, p0/M, z0.h, z31.h\n" - "ld1h { z0.h }, p0/Z, [x26]\n" - ".inst 0x818201a0 // bfmopa za0.s, p0/M, p0/M, z13.h, z2.h\n" - ".inst 0xa140a6fb // ldnt1h { z19.h, z23.h, z27.h, z31.h }, pn9.b/Z, [x23]\n" - ".inst 0x818601a1 // bfmopa za1.s, p0/M, p0/M, z13.h, z6.h\n" - ".inst 0x818a01a2 // bfmopa za2.s, p0/M, p0/M, z13.h, z10.h\n" - ".inst 0x818e01a3 // bfmopa za3.s, p0/M, p0/M, z13.h, z14.h\n" - "ld1h { z13.h }, p0/Z, [x26, #1, MUL VL]\n" - ".inst 0x81830180 // bfmopa za0.s, p0/M, p0/M, z12.h, z3.h\n" - ".inst 0xa141a6ea // ldnt1h { z2.h, z6.h, z10.h, z14.h }, pn9.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0x81870181 // bfmopa za1.s, p0/M, p0/M, z12.h, z7.h\n" - ".inst 0x818b0182 // bfmopa za2.s, p0/M, p0/M, z12.h, z11.h\n" - ".inst 0x818f0183 // bfmopa za3.s, p0/M, p0/M, z12.h, z15.h\n" - "ld1h { z12.h }, p0/Z, [x26, #2, MUL VL]\n" - ".inst 0xa142a6eb // ldnt1h { z3.h, z7.h, z11.h, z15.h }, pn9.b/Z, [x23, #0x8, MUL VL]\n" - ".inst 0x81900340 // bfmopa za0.s, p0/M, p0/M, z26.h, z16.h\n" - ".inst 0x81940341 // bfmopa za1.s, p0/M, p0/M, z26.h, z20.h\n" - ".inst 0x81980342 // bfmopa za2.s, p0/M, p0/M, z26.h, z24.h\n" - ".inst 0x819c0343 // bfmopa za3.s, p0/M, p0/M, z26.h, z28.h\n" - "ld1h { z26.h }, p0/Z, [x26, #3, MUL VL]\n" + ".inst 0x81890381 // bfmopa za1.s, p0/M, p0/M, z28.h, z9.h\n" + ".inst 0x818a0382 // bfmopa za2.s, p0/M, p0/M, z28.h, z10.h\n" + ".inst 0x818b0383 // bfmopa za3.s, p0/M, p0/M, z28.h, z11.h\n" + "ld1h { z28.h }, p0/Z, [x26]\n" + ".inst 0x818c02c0 // bfmopa za0.s, p0/M, p0/M, z22.h, z12.h\n" + ".inst 0xa040a6e9 // ldnt1h { z8.h-z11.h }, pn9.b/Z, [x23]\n" + ".inst 0x818d02c1 // bfmopa za1.s, p0/M, p0/M, z22.h, z13.h\n" + ".inst 0x818e02c2 // bfmopa za2.s, p0/M, p0/M, z22.h, z14.h\n" + ".inst 0x818f02c3 // bfmopa za3.s, p0/M, p0/M, z22.h, z15.h\n" + "ld1h { z22.h }, p0/Z, [x26, #1, MUL VL]\n" + ".inst 0x818403c0 // bfmopa za0.s, p0/M, p0/M, z30.h, z4.h\n" + ".inst 0xa041a6ed // ldnt1h { z12.h-z15.h }, pn9.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0x818503c1 // bfmopa za1.s, p0/M, p0/M, z30.h, z5.h\n" + ".inst 0x818603c2 // bfmopa za2.s, p0/M, p0/M, z30.h, z6.h\n" + ".inst 0x818703c3 // bfmopa za3.s, p0/M, p0/M, z30.h, z7.h\n" + "ld1h { z30.h }, p0/Z, [x26, #2, MUL VL]\n" + ".inst 0xa042a6e5 // ldnt1h { z4.h-z7.h }, pn9.b/Z, [x23, #0x8, MUL VL]\n" + ".inst 0x81930280 // bfmopa za0.s, p0/M, p0/M, z20.h, z19.h\n" + ".inst 0x81970281 // bfmopa za1.s, p0/M, p0/M, z20.h, z23.h\n" + ".inst 0x819b0282 // bfmopa za2.s, p0/M, p0/M, z20.h, z27.h\n" + ".inst 0x819f0283 // bfmopa za3.s, p0/M, p0/M, z20.h, z31.h\n" + "ld1h { z20.h }, p0/Z, [x26, #3, MUL VL]\n" "addvl x26, x26, #4\n" - ".inst 0xa143a6f8 // ldnt1h { z16.h, z20.h, z24.h, z28.h }, pn9.b/Z, [x23, #0xc, MUL VL]\n" + ".inst 0xa143a6fb // ldnt1h { z19.h, z23.h, z27.h, z31.h }, pn9.b/Z, [x23, #0xc, MUL VL]\n" "addvl x23, x23, #16\n" "bgt 6b\n" "7:" // K loop tail - ".inst 0x81930000 // bfmopa za0.s, p0/M, p0/M, z0.h, z19.h\n" - ".inst 0x81970001 // bfmopa za1.s, p0/M, p0/M, z0.h, z23.h\n" - ".inst 0x819b0002 // bfmopa za2.s, p0/M, p0/M, z0.h, z27.h\n" - ".inst 0x819f0003 // bfmopa za3.s, p0/M, p0/M, z0.h, z31.h\n" - ".inst 0x818201a0 // bfmopa za0.s, p0/M, p0/M, z13.h, z2.h\n" - ".inst 0x818601a1 // bfmopa za1.s, p0/M, p0/M, z13.h, z6.h\n" - ".inst 0x818a01a2 // bfmopa za2.s, p0/M, p0/M, z13.h, z10.h\n" - ".inst 0x818e01a3 // bfmopa za3.s, p0/M, p0/M, z13.h, z14.h\n" - ".inst 0x81830180 // bfmopa za0.s, p0/M, p0/M, z12.h, z3.h\n" - ".inst 0x81870181 // bfmopa za1.s, p0/M, p0/M, z12.h, z7.h\n" - ".inst 0x818b0182 // bfmopa za2.s, p0/M, p0/M, z12.h, z11.h\n" - ".inst 0x818f0183 // bfmopa za3.s, p0/M, p0/M, z12.h, z15.h\n" - ".inst 0x81900340 // bfmopa za0.s, p0/M, p0/M, z26.h, z16.h\n" - ".inst 0x81940341 // bfmopa za1.s, p0/M, p0/M, z26.h, z20.h\n" - ".inst 0x81980342 // bfmopa za2.s, p0/M, p0/M, z26.h, z24.h\n" - ".inst 0x819c0343 // bfmopa za3.s, p0/M, p0/M, z26.h, z28.h\n" + ".inst 0x81880380 // bfmopa za0.s, p0/M, p0/M, z28.h, z8.h\n" + ".inst 0x81890381 // bfmopa za1.s, p0/M, p0/M, z28.h, z9.h\n" + ".inst 0x818a0382 // bfmopa za2.s, p0/M, p0/M, z28.h, z10.h\n" + ".inst 0x818b0383 // bfmopa za3.s, p0/M, p0/M, z28.h, z11.h\n" + ".inst 0x818c02c0 // bfmopa za0.s, p0/M, p0/M, z22.h, z12.h\n" + ".inst 0x818d02c1 // bfmopa za1.s, p0/M, p0/M, z22.h, z13.h\n" + ".inst 0x818e02c2 // bfmopa za2.s, p0/M, p0/M, z22.h, z14.h\n" + ".inst 0x818f02c3 // bfmopa za3.s, p0/M, p0/M, z22.h, z15.h\n" + ".inst 0x818403c0 // bfmopa za0.s, p0/M, p0/M, z30.h, z4.h\n" + ".inst 0x818503c1 // bfmopa za1.s, p0/M, p0/M, z30.h, z5.h\n" + ".inst 0x818603c2 // bfmopa za2.s, p0/M, p0/M, z30.h, z6.h\n" + ".inst 0x818703c3 // bfmopa za3.s, p0/M, p0/M, z30.h, z7.h\n" + ".inst 0x81930280 // bfmopa za0.s, p0/M, p0/M, z20.h, z19.h\n" + ".inst 0x81970281 // bfmopa za1.s, p0/M, p0/M, z20.h, z23.h\n" + ".inst 0x819b0282 // bfmopa za2.s, p0/M, p0/M, z20.h, z27.h\n" + ".inst 0x819f0283 // bfmopa za3.s, p0/M, p0/M, z20.h, z31.h\n" "8:" // K oddments "cbz x21, 10f\n" "9:" // K oddments: Loop - "ld1h { z0.h }, p0/Z, [x26]\n" + "ld1h { z8.h }, p0/Z, [x26]\n" "subs x21, x21, #0x1\n" "addvl x26, x26, #1\n" - ".inst 0xa140a6f3 // ld1h { z19.h, z23.h, z27.h, z31.h }, pn9.b/Z, [x23]\n" + ".inst 0xa140a6e3 // ld1h { z3.h, z7.h, z11.h, z15.h }, pn9.b/Z, [x23]\n" "addvl x23, x23, #4\n" - ".inst 0x81930000 // bfmopa za0.s, p0/M, p0/M, z0.h, z19.h\n" - ".inst 0x81970001 // bfmopa za1.s, p0/M, p0/M, z0.h, z23.h\n" - ".inst 0x819b0002 // bfmopa za2.s, p0/M, p0/M, z0.h, z27.h\n" - ".inst 0x819f0003 // bfmopa za3.s, p0/M, p0/M, z0.h, z31.h\n" + ".inst 0x81830100 // bfmopa za0.s, p0/M, p0/M, z8.h, z3.h\n" + ".inst 0x81870101 // bfmopa za1.s, p0/M, p0/M, z8.h, z7.h\n" + ".inst 0x818b0102 // bfmopa za2.s, p0/M, p0/M, z8.h, z11.h\n" + ".inst 0x818f0103 // bfmopa za3.s, p0/M, p0/M, z8.h, z15.h\n" "bgt 9b\n" "10:" // K oddments: End "tbz x15, #1, 14f\n" @@ -242,25 +241,25 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL(const bfloat16 *const A, con "mov x12, #0x0\n" "cntw x20\n" "11:" // Store to partial result buffer: Store and refill: Loop - ".inst 0xa040c5cc // ld1w { z12.s-z15.s }, pn9.b/Z, [x14]\n" - ".inst 0xc0860418 // mova { z24.s-z27.s }, za0h.s[x12]\n" - ".inst 0xc0840580 // mova za0h.s[x12], { z12.s-z15.s }\n" - ".inst 0xc0860434 // mova { z20.s-z23.s }, za1h.s[x12]\n" - ".inst 0xa041c5dc // ld1w { z28.s-z31.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n" - ".inst 0xc0840781 // mova za1h.s[x12], { z28.s-z31.s }\n" - ".inst 0xc086045c // mova { z28.s-z31.s }, za2h.s[x12]\n" - ".inst 0xc0860470 // mova { z16.s-z19.s }, za3h.s[x12]\n" - ".inst 0xa042c5cc // ld1w { z12.s-z15.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n" - ".inst 0xc0840582 // mova za2h.s[x12], { z12.s-z15.s }\n" - ".inst 0xa043c5cc // ld1w { z12.s-z15.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n" - ".inst 0xc0840583 // mova za3h.s[x12], { z12.s-z15.s }\n" + ".inst 0xa040c5d4 // ld1w { z20.s-z23.s }, pn9.b/Z, [x14]\n" + ".inst 0xc0860408 // mova { z8.s-z11.s }, za0h.s[x12]\n" + ".inst 0xc0840680 // mova za0h.s[x12], { z20.s-z23.s }\n" + ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" + ".inst 0xa041c5cc // ld1w { z12.s-z15.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n" + ".inst 0xc0840581 // mova za1h.s[x12], { z12.s-z15.s }\n" + ".inst 0xc086044c // mova { z12.s-z15.s }, za2h.s[x12]\n" + ".inst 0xc0860460 // mova { z0.s-z3.s }, za3h.s[x12]\n" + ".inst 0xa042c5d0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n" + ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa043c5d0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n" + ".inst 0xc0840603 // mova za3h.s[x12], { z16.s-z19.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" - ".inst 0xa060c5b8 // st1w { z24.s-z27.s }, pn9.b, [x13]\n" + ".inst 0xa060c5a8 // st1w { z8.s-z11.s }, pn9.b, [x13]\n" "addvl x14, x14, #16\n" - ".inst 0xa061c5b4 // st1w { z20.s-z23.s }, pn9.b, [x13, #0x4, MUL VL]\n" - ".inst 0xa062c5bc // st1w { z28.s-z31.s }, pn9.b, [x13, #0x8, MUL VL]\n" - ".inst 0xa063c5b0 // st1w { z16.s-z19.s }, pn9.b, [x13, #0xc, MUL VL]\n" + ".inst 0xa061c5a4 // st1w { z4.s-z7.s }, pn9.b, [x13, #0x4, MUL VL]\n" + ".inst 0xa062c5ac // st1w { z12.s-z15.s }, pn9.b, [x13, #0x8, MUL VL]\n" + ".inst 0xa063c5a0 // st1w { z0.s-z3.s }, pn9.b, [x13, #0xc, MUL VL]\n" "addvl x13, x13, #16\n" "blt 11b\n" "b 24f\n" @@ -268,15 +267,15 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL(const bfloat16 *const A, con "mov x12, #0x0\n" "cntw x20\n" "13:" // Store to partial result buffer: Store only: Loop - ".inst 0xc086040c // mova { z12.s-z15.s }, za0h.s[x12]\n" - ".inst 0xc086043c // mova { z28.s-z31.s }, za1h.s[x12]\n" - ".inst 0xa060c5ac // st1w { z12.s-z15.s }, pn9.b, [x13]\n" - ".inst 0xc0860450 // mova { z16.s-z19.s }, za2h.s[x12]\n" + ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" + ".inst 0xc0860438 // mova { z24.s-z27.s }, za1h.s[x12]\n" + ".inst 0xa060c5b0 // st1w { z16.s-z19.s }, pn9.b, [x13]\n" + ".inst 0xc0860448 // mova { z8.s-z11.s }, za2h.s[x12]\n" ".inst 0xc0860464 // mova { z4.s-z7.s }, za3h.s[x12]\n" - ".inst 0xa061c5bc // st1w { z28.s-z31.s }, pn9.b, [x13, #0x4, MUL VL]\n" + ".inst 0xa061c5b8 // st1w { z24.s-z27.s }, pn9.b, [x13, #0x4, MUL VL]\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" - ".inst 0xa062c5b0 // st1w { z16.s-z19.s }, pn9.b, [x13, #0x8, MUL VL]\n" + ".inst 0xa062c5a8 // st1w { z8.s-z11.s }, pn9.b, [x13, #0x8, MUL VL]\n" ".inst 0xa063c5a4 // st1w { z4.s-z7.s }, pn9.b, [x13, #0xc, MUL VL]\n" "addvl x13, x13, #16\n" "blt 13b\n" @@ -314,18 +313,18 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL(const bfloat16 *const A, con "16:" // Store to output array: Skip activation: Accumulator row 0 oddments "cbz x20, 17f\n" "subs x20, x20, #0x1\n" - ".inst 0xc0860400 // mova { z0.s-z3.s }, za0h.s[x12]\n" - ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" - ".inst 0xc0860448 // mova { z8.s-z11.s }, za2h.s[x12]\n" - ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" - ".inst 0xa160c320 // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x25]\n" + ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" + ".inst 0xc0860434 // mova { z20.s-z23.s }, za1h.s[x12]\n" + ".inst 0xc0860458 // mova { z24.s-z27.s }, za2h.s[x12]\n" + ".inst 0xc086047c // mova { z28.s-z31.s }, za3h.s[x12]\n" + ".inst 0xa160c330 // st1w { z16.s, z20.s, z24.s, z28.s }, p8, [x25]\n" "add x25, x25, x23\n" "beq 17f\n" "subs x20, x20, #0x1\n" - ".inst 0xa160c321 // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x25]\n" + ".inst 0xa160c331 // st1w { z17.s, z21.s, z25.s, z29.s }, p8, [x25]\n" "add x25, x25, x23\n" "beq 17f\n" - ".inst 0xa160c322 // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x25]\n" + ".inst 0xa160c332 // st1w { z18.s, z22.s, z26.s, z30.s }, p8, [x25]\n" "add x25, x25, x23\n" "17:" // Store to output array: Skip activation: Accumulator row 0 oddments: End "subs x24, x24, x22\n" @@ -334,66 +333,66 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL(const bfloat16 *const A, con "18:" // Store to output array: Skip activation: End "cntw x20\n" "cmp x24, x20\n" - "ld1rw { z23.s }, p0/Z, [%x[args], %[offsetof_KernelArgs_min]]\n" + "ld1rw { z1.s }, p0/Z, [%x[args], %[offsetof_KernelArgs_min]]\n" "csel x20, x24, x20, LT\n" "lsr x21, x20, #0x2\n" - "ld1rw { z16.s }, p0/Z, [%x[args], %[offsetof_KernelArgs_max]]\n" + "ld1rw { z0.s }, p0/Z, [%x[args], %[offsetof_KernelArgs_max]]\n" "mov x12, #0x0\n" "and x20, x20, #0x3\n" "cbz x21, 20f\n" "19:" // Store to output array: Accumulator row 0 loop - ".inst 0xc0860400 // mova { z0.s-z3.s }, za0h.s[x12]\n" - ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" - ".inst 0xc1b0cae0 // fclamp { z0.s-z3.s }, z23.s, z16.s\n" - ".inst 0xc1b0cae4 // fclamp { z4.s-z7.s }, z23.s, z16.s\n" - ".inst 0xc0860448 // mova { z8.s-z11.s }, za2h.s[x12]\n" - ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" - ".inst 0xc1b0cae8 // fclamp { z8.s-z11.s }, z23.s, z16.s\n" - ".inst 0xc1b0caec // fclamp { z12.s-z15.s }, z23.s, z16.s\n" - ".inst 0xa160c320 // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x25]\n" + ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" + ".inst 0xc0860434 // mova { z20.s-z23.s }, za1h.s[x12]\n" + ".inst 0xc1a0c830 // fclamp { z16.s-z19.s }, z1.s, z0.s\n" + ".inst 0xc1a0c834 // fclamp { z20.s-z23.s }, z1.s, z0.s\n" + ".inst 0xc0860458 // mova { z24.s-z27.s }, za2h.s[x12]\n" + ".inst 0xc086047c // mova { z28.s-z31.s }, za3h.s[x12]\n" + ".inst 0xc1a0c838 // fclamp { z24.s-z27.s }, z1.s, z0.s\n" + ".inst 0xc1a0c83c // fclamp { z28.s-z31.s }, z1.s, z0.s\n" + ".inst 0xa160c330 // st1w { z16.s, z20.s, z24.s, z28.s }, p8, [x25]\n" "add x25, x25, x23\n" "add x12, x12, #0x4\n" - ".inst 0xa160c321 // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x25]\n" + ".inst 0xa160c331 // st1w { z17.s, z21.s, z25.s, z29.s }, p8, [x25]\n" "add x25, x25, x23\n" "cmp x12, x21, LSL #2\n" - ".inst 0xa160c322 // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x25]\n" + ".inst 0xa160c332 // st1w { z18.s, z22.s, z26.s, z30.s }, p8, [x25]\n" "add x25, x25, x23\n" - ".inst 0xa160c323 // st1w { z3.s, z7.s, z11.s, z15.s }, p8, [x25]\n" + ".inst 0xa160c333 // st1w { z19.s, z23.s, z27.s, z31.s }, p8, [x25]\n" "add x25, x25, x23\n" "blt 19b\n" "20:" // Store to output array: Accumulator row 0 oddments "cbz x20, 21f\n" - ".inst 0xc0860400 // mova { z0.s-z3.s }, za0h.s[x12]\n" - ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" - ".inst 0xc1b0cae0 // fclamp { z0.s-z3.s }, z23.s, z16.s\n" - ".inst 0xc1b0cae4 // fclamp { z4.s-z7.s }, z23.s, z16.s\n" - ".inst 0xc0860448 // mova { z8.s-z11.s }, za2h.s[x12]\n" - ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" - ".inst 0xc1b0cae8 // fclamp { z8.s-z11.s }, z23.s, z16.s\n" - ".inst 0xc1b0caec // fclamp { z12.s-z15.s }, z23.s, z16.s\n" + ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" + ".inst 0xc0860434 // mova { z20.s-z23.s }, za1h.s[x12]\n" + ".inst 0xc1a0c830 // fclamp { z16.s-z19.s }, z1.s, z0.s\n" + ".inst 0xc1a0c834 // fclamp { z20.s-z23.s }, z1.s, z0.s\n" + ".inst 0xc0860458 // mova { z24.s-z27.s }, za2h.s[x12]\n" + ".inst 0xc086047c // mova { z28.s-z31.s }, za3h.s[x12]\n" + ".inst 0xc1a0c838 // fclamp { z24.s-z27.s }, z1.s, z0.s\n" + ".inst 0xc1a0c83c // fclamp { z28.s-z31.s }, z1.s, z0.s\n" "subs x20, x20, #0x1\n" - ".inst 0xa160c320 // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x25]\n" + ".inst 0xa160c330 // st1w { z16.s, z20.s, z24.s, z28.s }, p8, [x25]\n" "add x25, x25, x23\n" "beq 21f\n" "subs x20, x20, #0x1\n" - ".inst 0xa160c321 // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x25]\n" + ".inst 0xa160c331 // st1w { z17.s, z21.s, z25.s, z29.s }, p8, [x25]\n" "add x25, x25, x23\n" "beq 21f\n" - ".inst 0xa160c322 // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x25]\n" + ".inst 0xa160c332 // st1w { z18.s, z22.s, z26.s, z30.s }, p8, [x25]\n" "21:" // Store to output array: Accumulator row 0 oddments: End "22:" // Store to output array: End "tbz x15, #0, 24f\n" "mov x12, #0x0\n" "cntw x20\n" "23:" // Store to output array: Refill accumulators: Loop - ".inst 0xa040c5d0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x14]\n" - ".inst 0xc0840600 // mova za0h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa040c5cc // ld1w { z12.s-z15.s }, pn9.b/Z, [x14]\n" + ".inst 0xc0840580 // mova za0h.s[x12], { z12.s-z15.s }\n" ".inst 0xa041c5d0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n" ".inst 0xc0840601 // mova za1h.s[x12], { z16.s-z19.s }\n" ".inst 0xa042c5d0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n" ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" - ".inst 0xa043c5c8 // ld1w { z8.s-z11.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n" - ".inst 0xc0840503 // mova za3h.s[x12], { z8.s-z11.s }\n" + ".inst 0xa043c5c4 // ld1w { z4.s-z7.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n" + ".inst 0xc0840483 // mova za3h.s[x12], { z4.s-z7.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x14, x14, #16\n" @@ -417,4 +416,3 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_1VLx4VL(const bfloat16 *const A, con } // namespace arm_gemm #endif // ARM_COMPUTE_ENABLE_SME2 -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL.hpp index 89c79cfb0a6a4ed2cb3d4acb38cb6317dce5b0ee..ca7b0573fc2fb4dc3125557697b2076793222823 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,7 +23,7 @@ */ #pragma once -#ifdef __aarch64__ +#ifdef ARM_COMPUTE_ENABLE_SME2 #include "../bfloat.hpp" #include "../std_transforms_sme.hpp" @@ -83,12 +83,11 @@ public: StdTransformsSME transforms = {}; - cls_sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL(const CPUInfo *ci) + cls_sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL(const CPUInfo *) { - ARM_COMPUTE_UNUSED(ci); } }; } // namespace arm_gemm -#endif // __aarch64__ +#endif // ARM_COMPUTE_ENABLE_SME2 diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL/generic.cpp index b63f2110ff31273e742e0b6eb4a08958e7295410..20c1de941890071ed406950192dde4b35b5759a4 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL/generic.cpp @@ -21,7 +21,6 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifdef __ARM_FEATURE_SVE #ifdef ARM_COMPUTE_ENABLE_SME2 #include "arm_gemm.hpp" @@ -113,14 +112,14 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL(const bfloat16 *const A, con "mov x12, #0x0\n" "cntw x20\n" "1:" // Initial accumulator load from buffer: Loop - ".inst 0xa040c5e8 // ld1w { z8.s-z11.s }, pn9.b/Z, [x15]\n" - ".inst 0xc0840500 // mova za0h.s[x12], { z8.s-z11.s }\n" - ".inst 0xa041c5f0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n" - ".inst 0xc0840601 // mova za1h.s[x12], { z16.s-z19.s }\n" - ".inst 0xa042c5e0 // ld1w { z0.s-z3.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n" - ".inst 0xc0840402 // mova za2h.s[x12], { z0.s-z3.s }\n" - ".inst 0xa043c5fc // ld1w { z28.s-z31.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840783 // mova za3h.s[x12], { z28.s-z31.s }\n" + ".inst 0xa040c5e4 // ld1w { z4.s-z7.s }, pn9.b/Z, [x15]\n" + ".inst 0xc0840480 // mova za0h.s[x12], { z4.s-z7.s }\n" + ".inst 0xa041c5e8 // ld1w { z8.s-z11.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n" + ".inst 0xc0840501 // mova za1h.s[x12], { z8.s-z11.s }\n" + ".inst 0xa042c5f4 // ld1w { z20.s-z23.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n" + ".inst 0xc0840682 // mova za2h.s[x12], { z20.s-z23.s }\n" + ".inst 0xa043c5e8 // ld1w { z8.s-z11.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840503 // mova za3h.s[x12], { z8.s-z11.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x15, x15, #16\n" @@ -138,12 +137,12 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL(const bfloat16 *const A, con "ldr x20, [%x[args], %[offsetof_bias]]\n" ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "cbz x20, 5f\n" - "fmov z21.s, #1.0\n" - ".inst 0xa00a428f // ldnt1w { z14.s-z15.s }, p8/Z, [x20, x10, LSL #2]\n" - ".inst 0x808e02a0 // fmopa za0.s, p0/M, p0/M, z21.s, z14.s\n" - ".inst 0x808f02a1 // fmopa za1.s, p0/M, p0/M, z21.s, z15.s\n" - ".inst 0x808e02a2 // fmopa za2.s, p0/M, p0/M, z21.s, z14.s\n" - ".inst 0x808f02a3 // fmopa za3.s, p0/M, p0/M, z21.s, z15.s\n" + "fmov z12.s, #1.0\n" + ".inst 0xa10a4289 // ldnt1w { z1.s, z9.s }, p8/Z, [x20, x10, LSL #2]\n" + ".inst 0x80810180 // fmopa za0.s, p0/M, p0/M, z12.s, z1.s\n" + ".inst 0x80890181 // fmopa za1.s, p0/M, p0/M, z12.s, z9.s\n" + ".inst 0x80810182 // fmopa za2.s, p0/M, p0/M, z12.s, z1.s\n" + ".inst 0x80890183 // fmopa za3.s, p0/M, p0/M, z12.s, z9.s\n" "4:" // Prepare accumulators: Test for last block "mov x20, x10\n" "mov x21, x11\n" @@ -166,75 +165,75 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL(const bfloat16 *const A, con "madd x23, x10, x20, x23\n" // bptr = B + n * kstride_bytes "cbz x22, 8f\n" "subs x22, x22, #0x1\n" - ".inst 0xa1402767 // ld1h { z7.h, z15.h }, pn9.b/Z, [x27]\n" - ".inst 0xa14026ff // ldnt1h { z23.h, z31.h }, pn9.b/Z, [x23]\n" - ".inst 0xa0412768 // ld1h { z8.h-z9.h }, pn9.b/Z, [x27, #0x2, MUL VL]\n" - ".inst 0xa04126e3 // ldnt1h { z2.h-z3.h }, pn9.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0xa1422772 // ld1h { z18.h, z26.h }, pn9.b/Z, [x27, #0x4, MUL VL]\n" - ".inst 0xa04226f1 // ldnt1h { z16.h-z17.h }, pn9.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xa1432776 // ld1h { z22.h, z30.h }, pn9.b/Z, [x27, #0x6, MUL VL]\n" + ".inst 0xa0402772 // ld1h { z18.h-z19.h }, pn9.b/Z, [x27]\n" + ".inst 0xa04026e3 // ldnt1h { z2.h-z3.h }, pn9.b/Z, [x23]\n" + ".inst 0xa0412764 // ld1h { z4.h-z5.h }, pn9.b/Z, [x27, #0x2, MUL VL]\n" + ".inst 0xa04126fb // ldnt1h { z26.h-z27.h }, pn9.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xa042276a // ld1h { z10.h-z11.h }, pn9.b/Z, [x27, #0x4, MUL VL]\n" + ".inst 0xa04226f5 // ldnt1h { z20.h-z21.h }, pn9.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xa0432766 // ld1h { z6.h-z7.h }, pn9.b/Z, [x27, #0x6, MUL VL]\n" "addvl x27, x27, #8\n" - ".inst 0xa14326ec // ldnt1h { z4.h, z12.h }, pn9.b/Z, [x23, #0x6, MUL VL]\n" + ".inst 0xa04326e9 // ldnt1h { z8.h-z9.h }, pn9.b/Z, [x23, #0x6, MUL VL]\n" "addvl x23, x23, #8\n" "ble 7f\n" "6:" // K loop - ".inst 0x819700e0 // bfmopa za0.s, p0/M, p0/M, z7.h, z23.h\n" + ".inst 0x81820240 // bfmopa za0.s, p0/M, p0/M, z18.h, z2.h\n" "subs x22, x22, #0x1\n" - ".inst 0x819f00e1 // bfmopa za1.s, p0/M, p0/M, z7.h, z31.h\n" - ".inst 0x819701e2 // bfmopa za2.s, p0/M, p0/M, z15.h, z23.h\n" - ".inst 0x819f01e3 // bfmopa za3.s, p0/M, p0/M, z15.h, z31.h\n" - ".inst 0xa1402767 // ld1h { z7.h, z15.h }, pn9.b/Z, [x27]\n" - ".inst 0x81820100 // bfmopa za0.s, p0/M, p0/M, z8.h, z2.h\n" - ".inst 0xa14026ff // ldnt1h { z23.h, z31.h }, pn9.b/Z, [x23]\n" - ".inst 0x81830101 // bfmopa za1.s, p0/M, p0/M, z8.h, z3.h\n" - ".inst 0x81820122 // bfmopa za2.s, p0/M, p0/M, z9.h, z2.h\n" - ".inst 0x81830123 // bfmopa za3.s, p0/M, p0/M, z9.h, z3.h\n" - ".inst 0xa0412768 // ld1h { z8.h-z9.h }, pn9.b/Z, [x27, #0x2, MUL VL]\n" - ".inst 0x81900240 // bfmopa za0.s, p0/M, p0/M, z18.h, z16.h\n" - ".inst 0xa04126e3 // ldnt1h { z2.h-z3.h }, pn9.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0x81910241 // bfmopa za1.s, p0/M, p0/M, z18.h, z17.h\n" - ".inst 0x81900342 // bfmopa za2.s, p0/M, p0/M, z26.h, z16.h\n" - ".inst 0x81910343 // bfmopa za3.s, p0/M, p0/M, z26.h, z17.h\n" - ".inst 0xa1422772 // ld1h { z18.h, z26.h }, pn9.b/Z, [x27, #0x4, MUL VL]\n" - ".inst 0xa04226f1 // ldnt1h { z16.h-z17.h }, pn9.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0x818402c0 // bfmopa za0.s, p0/M, p0/M, z22.h, z4.h\n" - ".inst 0x818c02c1 // bfmopa za1.s, p0/M, p0/M, z22.h, z12.h\n" - ".inst 0x818403c2 // bfmopa za2.s, p0/M, p0/M, z30.h, z4.h\n" - ".inst 0x818c03c3 // bfmopa za3.s, p0/M, p0/M, z30.h, z12.h\n" - ".inst 0xa1432776 // ld1h { z22.h, z30.h }, pn9.b/Z, [x27, #0x6, MUL VL]\n" + ".inst 0x81830241 // bfmopa za1.s, p0/M, p0/M, z18.h, z3.h\n" + ".inst 0x81820262 // bfmopa za2.s, p0/M, p0/M, z19.h, z2.h\n" + ".inst 0x81830263 // bfmopa za3.s, p0/M, p0/M, z19.h, z3.h\n" + ".inst 0xa0402772 // ld1h { z18.h-z19.h }, pn9.b/Z, [x27]\n" + ".inst 0x819a0080 // bfmopa za0.s, p0/M, p0/M, z4.h, z26.h\n" + ".inst 0xa04026e3 // ldnt1h { z2.h-z3.h }, pn9.b/Z, [x23]\n" + ".inst 0x819b0081 // bfmopa za1.s, p0/M, p0/M, z4.h, z27.h\n" + ".inst 0x819a00a2 // bfmopa za2.s, p0/M, p0/M, z5.h, z26.h\n" + ".inst 0x819b00a3 // bfmopa za3.s, p0/M, p0/M, z5.h, z27.h\n" + ".inst 0xa0412764 // ld1h { z4.h-z5.h }, pn9.b/Z, [x27, #0x2, MUL VL]\n" + ".inst 0x81940140 // bfmopa za0.s, p0/M, p0/M, z10.h, z20.h\n" + ".inst 0xa04126fb // ldnt1h { z26.h-z27.h }, pn9.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0x81950141 // bfmopa za1.s, p0/M, p0/M, z10.h, z21.h\n" + ".inst 0x81940162 // bfmopa za2.s, p0/M, p0/M, z11.h, z20.h\n" + ".inst 0x81950163 // bfmopa za3.s, p0/M, p0/M, z11.h, z21.h\n" + ".inst 0xa042276a // ld1h { z10.h-z11.h }, pn9.b/Z, [x27, #0x4, MUL VL]\n" + ".inst 0xa04226f5 // ldnt1h { z20.h-z21.h }, pn9.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0x818800c0 // bfmopa za0.s, p0/M, p0/M, z6.h, z8.h\n" + ".inst 0x818900c1 // bfmopa za1.s, p0/M, p0/M, z6.h, z9.h\n" + ".inst 0x818800e2 // bfmopa za2.s, p0/M, p0/M, z7.h, z8.h\n" + ".inst 0x818900e3 // bfmopa za3.s, p0/M, p0/M, z7.h, z9.h\n" + ".inst 0xa0432766 // ld1h { z6.h-z7.h }, pn9.b/Z, [x27, #0x6, MUL VL]\n" "addvl x27, x27, #8\n" - ".inst 0xa14326ec // ldnt1h { z4.h, z12.h }, pn9.b/Z, [x23, #0x6, MUL VL]\n" + ".inst 0xa04326e9 // ldnt1h { z8.h-z9.h }, pn9.b/Z, [x23, #0x6, MUL VL]\n" "addvl x23, x23, #8\n" "bgt 6b\n" "7:" // K loop tail - ".inst 0x819700e0 // bfmopa za0.s, p0/M, p0/M, z7.h, z23.h\n" - ".inst 0x819f00e1 // bfmopa za1.s, p0/M, p0/M, z7.h, z31.h\n" - ".inst 0x819701e2 // bfmopa za2.s, p0/M, p0/M, z15.h, z23.h\n" - ".inst 0x819f01e3 // bfmopa za3.s, p0/M, p0/M, z15.h, z31.h\n" - ".inst 0x81820100 // bfmopa za0.s, p0/M, p0/M, z8.h, z2.h\n" - ".inst 0x81830101 // bfmopa za1.s, p0/M, p0/M, z8.h, z3.h\n" - ".inst 0x81820122 // bfmopa za2.s, p0/M, p0/M, z9.h, z2.h\n" - ".inst 0x81830123 // bfmopa za3.s, p0/M, p0/M, z9.h, z3.h\n" - ".inst 0x81900240 // bfmopa za0.s, p0/M, p0/M, z18.h, z16.h\n" - ".inst 0x81910241 // bfmopa za1.s, p0/M, p0/M, z18.h, z17.h\n" - ".inst 0x81900342 // bfmopa za2.s, p0/M, p0/M, z26.h, z16.h\n" - ".inst 0x81910343 // bfmopa za3.s, p0/M, p0/M, z26.h, z17.h\n" - ".inst 0x818402c0 // bfmopa za0.s, p0/M, p0/M, z22.h, z4.h\n" - ".inst 0x818c02c1 // bfmopa za1.s, p0/M, p0/M, z22.h, z12.h\n" - ".inst 0x818403c2 // bfmopa za2.s, p0/M, p0/M, z30.h, z4.h\n" - ".inst 0x818c03c3 // bfmopa za3.s, p0/M, p0/M, z30.h, z12.h\n" + ".inst 0x81820240 // bfmopa za0.s, p0/M, p0/M, z18.h, z2.h\n" + ".inst 0x81830241 // bfmopa za1.s, p0/M, p0/M, z18.h, z3.h\n" + ".inst 0x81820262 // bfmopa za2.s, p0/M, p0/M, z19.h, z2.h\n" + ".inst 0x81830263 // bfmopa za3.s, p0/M, p0/M, z19.h, z3.h\n" + ".inst 0x819a0080 // bfmopa za0.s, p0/M, p0/M, z4.h, z26.h\n" + ".inst 0x819b0081 // bfmopa za1.s, p0/M, p0/M, z4.h, z27.h\n" + ".inst 0x819a00a2 // bfmopa za2.s, p0/M, p0/M, z5.h, z26.h\n" + ".inst 0x819b00a3 // bfmopa za3.s, p0/M, p0/M, z5.h, z27.h\n" + ".inst 0x81940140 // bfmopa za0.s, p0/M, p0/M, z10.h, z20.h\n" + ".inst 0x81950141 // bfmopa za1.s, p0/M, p0/M, z10.h, z21.h\n" + ".inst 0x81940162 // bfmopa za2.s, p0/M, p0/M, z11.h, z20.h\n" + ".inst 0x81950163 // bfmopa za3.s, p0/M, p0/M, z11.h, z21.h\n" + ".inst 0x818800c0 // bfmopa za0.s, p0/M, p0/M, z6.h, z8.h\n" + ".inst 0x818900c1 // bfmopa za1.s, p0/M, p0/M, z6.h, z9.h\n" + ".inst 0x818800e2 // bfmopa za2.s, p0/M, p0/M, z7.h, z8.h\n" + ".inst 0x818900e3 // bfmopa za3.s, p0/M, p0/M, z7.h, z9.h\n" "8:" // K oddments "cbz x21, 10f\n" "9:" // K oddments: Loop - ".inst 0xa1402767 // ld1h { z7.h, z15.h }, pn9.b/Z, [x27]\n" + ".inst 0xa040277e // ld1h { z30.h-z31.h }, pn9.b/Z, [x27]\n" "subs x21, x21, #0x1\n" "addvl x27, x27, #2\n" - ".inst 0xa14026f7 // ld1h { z23.h, z31.h }, pn9.b/Z, [x23]\n" + ".inst 0xa14026e5 // ld1h { z5.h, z13.h }, pn9.b/Z, [x23]\n" "addvl x23, x23, #2\n" - ".inst 0x819700e0 // bfmopa za0.s, p0/M, p0/M, z7.h, z23.h\n" - ".inst 0x819f00e1 // bfmopa za1.s, p0/M, p0/M, z7.h, z31.h\n" - ".inst 0x819701e2 // bfmopa za2.s, p0/M, p0/M, z15.h, z23.h\n" - ".inst 0x819f01e3 // bfmopa za3.s, p0/M, p0/M, z15.h, z31.h\n" + ".inst 0x818503c0 // bfmopa za0.s, p0/M, p0/M, z30.h, z5.h\n" + ".inst 0x818d03c1 // bfmopa za1.s, p0/M, p0/M, z30.h, z13.h\n" + ".inst 0x818503e2 // bfmopa za2.s, p0/M, p0/M, z31.h, z5.h\n" + ".inst 0x818d03e3 // bfmopa za3.s, p0/M, p0/M, z31.h, z13.h\n" "bgt 9b\n" "10:" // K oddments: End "tbz x16, #1, 14f\n" @@ -243,24 +242,24 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL(const bfloat16 *const A, con "cntw x20\n" "11:" // Store to partial result buffer: Store and refill: Loop ".inst 0xa040c5ec // ld1w { z12.s-z15.s }, pn9.b/Z, [x15]\n" - ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" + ".inst 0xc0860400 // mova { z0.s-z3.s }, za0h.s[x12]\n" ".inst 0xc0840580 // mova za0h.s[x12], { z12.s-z15.s }\n" - ".inst 0xc086042c // mova { z12.s-z15.s }, za1h.s[x12]\n" - ".inst 0xa041c5e0 // ld1w { z0.s-z3.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n" - ".inst 0xc0840401 // mova za1h.s[x12], { z0.s-z3.s }\n" - ".inst 0xc0860454 // mova { z20.s-z23.s }, za2h.s[x12]\n" - ".inst 0xc0860468 // mova { z8.s-z11.s }, za3h.s[x12]\n" - ".inst 0xa042c5f8 // ld1w { z24.s-z27.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n" - ".inst 0xc0840702 // mova za2h.s[x12], { z24.s-z27.s }\n" - ".inst 0xa043c5f8 // ld1w { z24.s-z27.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840703 // mova za3h.s[x12], { z24.s-z27.s }\n" + ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" + ".inst 0xa041c5f0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n" + ".inst 0xc0840601 // mova za1h.s[x12], { z16.s-z19.s }\n" + ".inst 0xc086045c // mova { z28.s-z31.s }, za2h.s[x12]\n" + ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" + ".inst 0xa042c5f0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n" + ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa043c5e8 // ld1w { z8.s-z11.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840503 // mova za3h.s[x12], { z8.s-z11.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" - ".inst 0xa060c5d0 // st1w { z16.s-z19.s }, pn9.b, [x14]\n" + ".inst 0xa060c5c0 // st1w { z0.s-z3.s }, pn9.b, [x14]\n" "addvl x15, x15, #16\n" - ".inst 0xa061c5cc // st1w { z12.s-z15.s }, pn9.b, [x14, #0x4, MUL VL]\n" - ".inst 0xa062c5d4 // st1w { z20.s-z23.s }, pn9.b, [x14, #0x8, MUL VL]\n" - ".inst 0xa063c5c8 // st1w { z8.s-z11.s }, pn9.b, [x14, #0xc, MUL VL]\n" + ".inst 0xa061c5c4 // st1w { z4.s-z7.s }, pn9.b, [x14, #0x4, MUL VL]\n" + ".inst 0xa062c5dc // st1w { z28.s-z31.s }, pn9.b, [x14, #0x8, MUL VL]\n" + ".inst 0xa063c5cc // st1w { z12.s-z15.s }, pn9.b, [x14, #0xc, MUL VL]\n" "addvl x14, x14, #16\n" "blt 11b\n" "b 30f\n" @@ -268,16 +267,16 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL(const bfloat16 *const A, con "mov x12, #0x0\n" "cntw x20\n" "13:" // Store to partial result buffer: Store only: Loop - ".inst 0xc086040c // mova { z12.s-z15.s }, za0h.s[x12]\n" - ".inst 0xc0860430 // mova { z16.s-z19.s }, za1h.s[x12]\n" - ".inst 0xa060c5cc // st1w { z12.s-z15.s }, pn9.b, [x14]\n" - ".inst 0xc0860444 // mova { z4.s-z7.s }, za2h.s[x12]\n" - ".inst 0xc0860460 // mova { z0.s-z3.s }, za3h.s[x12]\n" - ".inst 0xa061c5d0 // st1w { z16.s-z19.s }, pn9.b, [x14, #0x4, MUL VL]\n" + ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" + ".inst 0xc0860420 // mova { z0.s-z3.s }, za1h.s[x12]\n" + ".inst 0xa060c5d0 // st1w { z16.s-z19.s }, pn9.b, [x14]\n" + ".inst 0xc0860458 // mova { z24.s-z27.s }, za2h.s[x12]\n" + ".inst 0xc0860464 // mova { z4.s-z7.s }, za3h.s[x12]\n" + ".inst 0xa061c5c0 // st1w { z0.s-z3.s }, pn9.b, [x14, #0x4, MUL VL]\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" - ".inst 0xa062c5c4 // st1w { z4.s-z7.s }, pn9.b, [x14, #0x8, MUL VL]\n" - ".inst 0xa063c5c0 // st1w { z0.s-z3.s }, pn9.b, [x14, #0xc, MUL VL]\n" + ".inst 0xa062c5d8 // st1w { z24.s-z27.s }, pn9.b, [x14, #0x8, MUL VL]\n" + ".inst 0xa063c5c4 // st1w { z4.s-z7.s }, pn9.b, [x14, #0xc, MUL VL]\n" "addvl x14, x14, #16\n" "blt 13b\n" "b 30f\n" @@ -312,16 +311,16 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL(const bfloat16 *const A, con "16:" // Store to output array: Skip activation: Accumulator row 0 oddments "cbz x20, 17f\n" "subs x20, x20, #0x1\n" - ".inst 0xc0860404 // mova { z4.s-z7.s }, za0h.s[x12]\n" - ".inst 0xc086042c // mova { z12.s-z15.s }, za1h.s[x12]\n" - ".inst 0xa1604344 // st1w { z4.s, z12.s }, p8, [x26]\n" + ".inst 0xc0860414 // mova { z20.s-z23.s }, za0h.s[x12]\n" + ".inst 0xc086043c // mova { z28.s-z31.s }, za1h.s[x12]\n" + ".inst 0xa1604354 // st1w { z20.s, z28.s }, p8, [x26]\n" "add x26, x26, x24\n" "beq 17f\n" "subs x20, x20, #0x1\n" - ".inst 0xa1604345 // st1w { z5.s, z13.s }, p8, [x26]\n" + ".inst 0xa1604355 // st1w { z21.s, z29.s }, p8, [x26]\n" "add x26, x26, x24\n" "beq 17f\n" - ".inst 0xa1604346 // st1w { z6.s, z14.s }, p8, [x26]\n" + ".inst 0xa1604356 // st1w { z22.s, z30.s }, p8, [x26]\n" "add x26, x26, x24\n" "17:" // Store to output array: Skip activation: Accumulator row 0 oddments: End "subs x25, x25, x22\n" @@ -349,16 +348,16 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL(const bfloat16 *const A, con "19:" // Store to output array: Skip activation: Accumulator row 1 oddments "cbz x20, 20f\n" "subs x20, x20, #0x1\n" - ".inst 0xc0860454 // mova { z20.s-z23.s }, za2h.s[x12]\n" - ".inst 0xc086047c // mova { z28.s-z31.s }, za3h.s[x12]\n" - ".inst 0xa1604354 // st1w { z20.s, z28.s }, p8, [x26]\n" + ".inst 0xc0860450 // mova { z16.s-z19.s }, za2h.s[x12]\n" + ".inst 0xc0860478 // mova { z24.s-z27.s }, za3h.s[x12]\n" + ".inst 0xa1604350 // st1w { z16.s, z24.s }, p8, [x26]\n" "add x26, x26, x24\n" "beq 20f\n" "subs x20, x20, #0x1\n" - ".inst 0xa1604355 // st1w { z21.s, z29.s }, p8, [x26]\n" + ".inst 0xa1604351 // st1w { z17.s, z25.s }, p8, [x26]\n" "add x26, x26, x24\n" "beq 20f\n" - ".inst 0xa1604356 // st1w { z22.s, z30.s }, p8, [x26]\n" + ".inst 0xa1604352 // st1w { z18.s, z26.s }, p8, [x26]\n" "add x26, x26, x24\n" "20:" // Store to output array: Skip activation: Accumulator row 1 oddments: End "subs x25, x25, x22\n" @@ -367,44 +366,44 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL(const bfloat16 *const A, con "21:" // Store to output array: Skip activation: End "cntw x23\n" "cmp x25, x23\n" - "ld1rw { z21.s }, p0/Z, [%x[args], %[offsetof_KernelArgs_min]]\n" + "ld1rw { z1.s }, p0/Z, [%x[args], %[offsetof_KernelArgs_min]]\n" "csel x22, x25, x23, LT\n" "lsr x21, x22, #0x2\n" - "ld1rw { z20.s }, p0/Z, [%x[args], %[offsetof_KernelArgs_max]]\n" + "ld1rw { z0.s }, p0/Z, [%x[args], %[offsetof_KernelArgs_max]]\n" "mov x12, #0x0\n" "and x20, x22, #0x3\n" "cbz x21, 23f\n" "22:" // Store to output array: Accumulator row 0 loop - ".inst 0xc0860404 // mova { z4.s-z7.s }, za0h.s[x12]\n" - ".inst 0xc086042c // mova { z12.s-z15.s }, za1h.s[x12]\n" - ".inst 0xc1b4caa4 // fclamp { z4.s-z7.s }, z21.s, z20.s\n" - ".inst 0xc1b4caac // fclamp { z12.s-z15.s }, z21.s, z20.s\n" - ".inst 0xa1604344 // st1w { z4.s, z12.s }, p8, [x26]\n" + ".inst 0xc0860414 // mova { z20.s-z23.s }, za0h.s[x12]\n" + ".inst 0xc086043c // mova { z28.s-z31.s }, za1h.s[x12]\n" + ".inst 0xc1a0c834 // fclamp { z20.s-z23.s }, z1.s, z0.s\n" + ".inst 0xc1a0c83c // fclamp { z28.s-z31.s }, z1.s, z0.s\n" + ".inst 0xa1604354 // st1w { z20.s, z28.s }, p8, [x26]\n" "add x26, x26, x24\n" "add x12, x12, #0x4\n" - ".inst 0xa1604345 // st1w { z5.s, z13.s }, p8, [x26]\n" + ".inst 0xa1604355 // st1w { z21.s, z29.s }, p8, [x26]\n" "add x26, x26, x24\n" "cmp x12, x21, LSL #2\n" - ".inst 0xa1604346 // st1w { z6.s, z14.s }, p8, [x26]\n" + ".inst 0xa1604356 // st1w { z22.s, z30.s }, p8, [x26]\n" "add x26, x26, x24\n" - ".inst 0xa1604347 // st1w { z7.s, z15.s }, p8, [x26]\n" + ".inst 0xa1604357 // st1w { z23.s, z31.s }, p8, [x26]\n" "add x26, x26, x24\n" "blt 22b\n" "23:" // Store to output array: Accumulator row 0 oddments "cbz x20, 24f\n" - ".inst 0xc0860400 // mova { z0.s-z3.s }, za0h.s[x12]\n" - ".inst 0xc0860428 // mova { z8.s-z11.s }, za1h.s[x12]\n" - ".inst 0xc1b4caa0 // fclamp { z0.s-z3.s }, z21.s, z20.s\n" - ".inst 0xc1b4caa8 // fclamp { z8.s-z11.s }, z21.s, z20.s\n" + ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" + ".inst 0xc0860438 // mova { z24.s-z27.s }, za1h.s[x12]\n" + ".inst 0xc1a0c830 // fclamp { z16.s-z19.s }, z1.s, z0.s\n" + ".inst 0xc1a0c838 // fclamp { z24.s-z27.s }, z1.s, z0.s\n" "subs x20, x20, #0x1\n" - ".inst 0xa1604340 // st1w { z0.s, z8.s }, p8, [x26]\n" + ".inst 0xa1604350 // st1w { z16.s, z24.s }, p8, [x26]\n" "add x26, x26, x24\n" "beq 24f\n" "subs x20, x20, #0x1\n" - ".inst 0xa1604341 // st1w { z1.s, z9.s }, p8, [x26]\n" + ".inst 0xa1604351 // st1w { z17.s, z25.s }, p8, [x26]\n" "add x26, x26, x24\n" "beq 24f\n" - ".inst 0xa1604342 // st1w { z2.s, z10.s }, p8, [x26]\n" + ".inst 0xa1604352 // st1w { z18.s, z26.s }, p8, [x26]\n" "add x26, x26, x24\n" "24:" // Store to output array: Accumulator row 0 oddments: End "subs x25, x25, x22\n" @@ -418,8 +417,8 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL(const bfloat16 *const A, con "25:" // Store to output array: Accumulator row 1 loop ".inst 0xc0860450 // mova { z16.s-z19.s }, za2h.s[x12]\n" ".inst 0xc0860478 // mova { z24.s-z27.s }, za3h.s[x12]\n" - ".inst 0xc1b4cab0 // fclamp { z16.s-z19.s }, z21.s, z20.s\n" - ".inst 0xc1b4cab8 // fclamp { z24.s-z27.s }, z21.s, z20.s\n" + ".inst 0xc1a0c830 // fclamp { z16.s-z19.s }, z1.s, z0.s\n" + ".inst 0xc1a0c838 // fclamp { z24.s-z27.s }, z1.s, z0.s\n" ".inst 0xa1604350 // st1w { z16.s, z24.s }, p8, [x26]\n" "add x26, x26, x24\n" "add x12, x12, #0x4\n" @@ -435,8 +434,8 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL(const bfloat16 *const A, con "cbz x20, 27f\n" ".inst 0xc0860450 // mova { z16.s-z19.s }, za2h.s[x12]\n" ".inst 0xc0860478 // mova { z24.s-z27.s }, za3h.s[x12]\n" - ".inst 0xc1b4cab0 // fclamp { z16.s-z19.s }, z21.s, z20.s\n" - ".inst 0xc1b4cab8 // fclamp { z24.s-z27.s }, z21.s, z20.s\n" + ".inst 0xc1a0c830 // fclamp { z16.s-z19.s }, z1.s, z0.s\n" + ".inst 0xc1a0c838 // fclamp { z24.s-z27.s }, z1.s, z0.s\n" "subs x20, x20, #0x1\n" ".inst 0xa1604350 // st1w { z16.s, z24.s }, p8, [x26]\n" "add x26, x26, x24\n" @@ -452,14 +451,14 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL(const bfloat16 *const A, con "mov x12, #0x0\n" "cntw x20\n" "29:" // Store to output array: Refill accumulators: Loop - ".inst 0xa040c5f0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x15]\n" - ".inst 0xc0840600 // mova za0h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa040c5ec // ld1w { z12.s-z15.s }, pn9.b/Z, [x15]\n" + ".inst 0xc0840580 // mova za0h.s[x12], { z12.s-z15.s }\n" ".inst 0xa041c5f0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n" ".inst 0xc0840601 // mova za1h.s[x12], { z16.s-z19.s }\n" ".inst 0xa042c5f0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n" ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" - ".inst 0xa043c5e8 // ld1w { z8.s-z11.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840503 // mova za3h.s[x12], { z8.s-z11.s }\n" + ".inst 0xa043c5ec // ld1w { z12.s-z15.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840583 // mova za3h.s[x12], { z12.s-z15.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x15, x15, #16\n" @@ -483,4 +482,3 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_2VLx2VL(const bfloat16 *const A, con } // namespace arm_gemm #endif // ARM_COMPUTE_ENABLE_SME2 -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL.hpp index 0d407e0cbad7904623efd01fcadf2cc6ae0bdb4f..7b31d6d2db297357208af7638f270c82dda60163 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,7 +23,7 @@ */ #pragma once -#ifdef __aarch64__ +#ifdef ARM_COMPUTE_ENABLE_SME2 #include "../bfloat.hpp" #include "../std_transforms_sme.hpp" @@ -83,12 +83,11 @@ public: StdTransformsSME transforms = {}; - cls_sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL(const CPUInfo *ci) + cls_sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL(const CPUInfo *) { - ARM_COMPUTE_UNUSED(ci); } }; } // namespace arm_gemm -#endif // __aarch64__ +#endif // ARM_COMPUTE_ENABLE_SME2 diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL/generic.cpp index a51b3db4b081dfc5db0f8d55ca0ee5d01cfeb612..70c94d32a3c0ee802704698dd3b121b71ed9560d 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL/generic.cpp @@ -21,7 +21,6 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifdef __ARM_FEATURE_SVE #ifdef ARM_COMPUTE_ENABLE_SME2 #include "arm_gemm.hpp" @@ -113,14 +112,14 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL(const bfloat16 *const A, con "mov x12, #0x0\n" "cntw x20\n" "1:" // Initial accumulator load from buffer: Loop - ".inst 0xa040c1e4 // ld1w { z4.s-z7.s }, pn8.b/Z, [x15]\n" - ".inst 0xc0840480 // mova za0h.s[x12], { z4.s-z7.s }\n" - ".inst 0xa041c1f8 // ld1w { z24.s-z27.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" - ".inst 0xc0840701 // mova za1h.s[x12], { z24.s-z27.s }\n" - ".inst 0xa042c1e0 // ld1w { z0.s-z3.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" - ".inst 0xc0840402 // mova za2h.s[x12], { z0.s-z3.s }\n" - ".inst 0xa043c1e4 // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840483 // mova za3h.s[x12], { z4.s-z7.s }\n" + ".inst 0xa040c1fc // ld1w { z28.s-z31.s }, pn8.b/Z, [x15]\n" + ".inst 0xc0840780 // mova za0h.s[x12], { z28.s-z31.s }\n" + ".inst 0xa041c1e0 // ld1w { z0.s-z3.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" + ".inst 0xc0840401 // mova za1h.s[x12], { z0.s-z3.s }\n" + ".inst 0xa042c1f4 // ld1w { z20.s-z23.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" + ".inst 0xc0840682 // mova za2h.s[x12], { z20.s-z23.s }\n" + ".inst 0xa043c1f4 // ld1w { z20.s-z23.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840683 // mova za3h.s[x12], { z20.s-z23.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x15, x15, #16\n" @@ -138,12 +137,12 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL(const bfloat16 *const A, con "ldr x20, [%x[args], %[offsetof_bias]]\n" ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "cbz x20, 5f\n" - "fmov z8.s, #1.0\n" - "ldnt1w { z27.s }, p0/Z, [x20, x10, LSL #2]\n" - ".inst 0x809b2500 // fmopa za0.s, p1/M, p1/M, z8.s, z27.s\n" - ".inst 0x809b2501 // fmopa za1.s, p1/M, p1/M, z8.s, z27.s\n" - ".inst 0x809b2502 // fmopa za2.s, p1/M, p1/M, z8.s, z27.s\n" - ".inst 0x809b2503 // fmopa za3.s, p1/M, p1/M, z8.s, z27.s\n" + "fmov z11.s, #1.0\n" + "ldnt1w { z13.s }, p0/Z, [x20, x10, LSL #2]\n" + ".inst 0x808d2560 // fmopa za0.s, p1/M, p1/M, z11.s, z13.s\n" + ".inst 0x808d2561 // fmopa za1.s, p1/M, p1/M, z11.s, z13.s\n" + ".inst 0x808d2562 // fmopa za2.s, p1/M, p1/M, z11.s, z13.s\n" + ".inst 0x808d2563 // fmopa za3.s, p1/M, p1/M, z11.s, z13.s\n" "4:" // Prepare accumulators: Test for last block "mov x20, x10\n" "mov x21, x11\n" @@ -166,75 +165,75 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL(const bfloat16 *const A, con "madd x23, x10, x20, x23\n" // bptr = B + n * kstride_bytes "cbz x22, 8f\n" "subs x22, x22, #0x1\n" - ".inst 0xa040a364 // ld1h { z4.h-z7.h }, pn8.b/Z, [x27]\n" - "ldnt1h { z29.h }, p1/Z, [x23]\n" - ".inst 0xa041a36c // ld1h { z12.h-z15.h }, pn8.b/Z, [x27, #0x4, MUL VL]\n" - "ldnt1h { z23.h }, p1/Z, [x23, #1, MUL VL]\n" - ".inst 0xa042a360 // ld1h { z0.h-z3.h }, pn8.b/Z, [x27, #0x8, MUL VL]\n" - "ldnt1h { z21.h }, p1/Z, [x23, #2, MUL VL]\n" - ".inst 0xa143a372 // ld1h { z18.h, z22.h, z26.h, z30.h }, pn8.b/Z, [x27, #0xc, MUL VL]\n" + ".inst 0xa140a360 // ld1h { z0.h, z4.h, z8.h, z12.h }, pn8.b/Z, [x27]\n" + "ldnt1h { z19.h }, p1/Z, [x23]\n" + ".inst 0xa141a371 // ld1h { z17.h, z21.h, z25.h, z29.h }, pn8.b/Z, [x27, #0x4, MUL VL]\n" + "ldnt1h { z22.h }, p1/Z, [x23, #1, MUL VL]\n" + ".inst 0xa142a370 // ld1h { z16.h, z20.h, z24.h, z28.h }, pn8.b/Z, [x27, #0x8, MUL VL]\n" + "ldnt1h { z23.h }, p1/Z, [x23, #2, MUL VL]\n" + ".inst 0xa143a363 // ld1h { z3.h, z7.h, z11.h, z15.h }, pn8.b/Z, [x27, #0xc, MUL VL]\n" "addvl x27, x27, #16\n" - "ldnt1h { z27.h }, p1/Z, [x23, #3, MUL VL]\n" + "ldnt1h { z2.h }, p1/Z, [x23, #3, MUL VL]\n" "addvl x23, x23, #4\n" "ble 7f\n" "6:" // K loop - ".inst 0x819d2480 // bfmopa za0.s, p1/M, p1/M, z4.h, z29.h\n" + ".inst 0x81932400 // bfmopa za0.s, p1/M, p1/M, z0.h, z19.h\n" "subs x22, x22, #0x1\n" - ".inst 0x819d24a1 // bfmopa za1.s, p1/M, p1/M, z5.h, z29.h\n" - ".inst 0x819d24c2 // bfmopa za2.s, p1/M, p1/M, z6.h, z29.h\n" - ".inst 0x819d24e3 // bfmopa za3.s, p1/M, p1/M, z7.h, z29.h\n" - ".inst 0xa040a364 // ld1h { z4.h-z7.h }, pn8.b/Z, [x27]\n" - ".inst 0x81972580 // bfmopa za0.s, p1/M, p1/M, z12.h, z23.h\n" - "ldnt1h { z29.h }, p1/Z, [x23]\n" - ".inst 0x819725a1 // bfmopa za1.s, p1/M, p1/M, z13.h, z23.h\n" - ".inst 0x819725c2 // bfmopa za2.s, p1/M, p1/M, z14.h, z23.h\n" - ".inst 0x819725e3 // bfmopa za3.s, p1/M, p1/M, z15.h, z23.h\n" - ".inst 0xa041a36c // ld1h { z12.h-z15.h }, pn8.b/Z, [x27, #0x4, MUL VL]\n" - ".inst 0x81952400 // bfmopa za0.s, p1/M, p1/M, z0.h, z21.h\n" - "ldnt1h { z23.h }, p1/Z, [x23, #1, MUL VL]\n" - ".inst 0x81952421 // bfmopa za1.s, p1/M, p1/M, z1.h, z21.h\n" - ".inst 0x81952442 // bfmopa za2.s, p1/M, p1/M, z2.h, z21.h\n" - ".inst 0x81952463 // bfmopa za3.s, p1/M, p1/M, z3.h, z21.h\n" - ".inst 0xa042a360 // ld1h { z0.h-z3.h }, pn8.b/Z, [x27, #0x8, MUL VL]\n" - "ldnt1h { z21.h }, p1/Z, [x23, #2, MUL VL]\n" - ".inst 0x819b2640 // bfmopa za0.s, p1/M, p1/M, z18.h, z27.h\n" - ".inst 0x819b26c1 // bfmopa za1.s, p1/M, p1/M, z22.h, z27.h\n" - ".inst 0x819b2742 // bfmopa za2.s, p1/M, p1/M, z26.h, z27.h\n" - ".inst 0x819b27c3 // bfmopa za3.s, p1/M, p1/M, z30.h, z27.h\n" - ".inst 0xa143a372 // ld1h { z18.h, z22.h, z26.h, z30.h }, pn8.b/Z, [x27, #0xc, MUL VL]\n" + ".inst 0x81932481 // bfmopa za1.s, p1/M, p1/M, z4.h, z19.h\n" + ".inst 0x81932502 // bfmopa za2.s, p1/M, p1/M, z8.h, z19.h\n" + ".inst 0x81932583 // bfmopa za3.s, p1/M, p1/M, z12.h, z19.h\n" + ".inst 0xa140a360 // ld1h { z0.h, z4.h, z8.h, z12.h }, pn8.b/Z, [x27]\n" + ".inst 0x81962620 // bfmopa za0.s, p1/M, p1/M, z17.h, z22.h\n" + "ldnt1h { z19.h }, p1/Z, [x23]\n" + ".inst 0x819626a1 // bfmopa za1.s, p1/M, p1/M, z21.h, z22.h\n" + ".inst 0x81962722 // bfmopa za2.s, p1/M, p1/M, z25.h, z22.h\n" + ".inst 0x819627a3 // bfmopa za3.s, p1/M, p1/M, z29.h, z22.h\n" + ".inst 0xa141a371 // ld1h { z17.h, z21.h, z25.h, z29.h }, pn8.b/Z, [x27, #0x4, MUL VL]\n" + ".inst 0x81972600 // bfmopa za0.s, p1/M, p1/M, z16.h, z23.h\n" + "ldnt1h { z22.h }, p1/Z, [x23, #1, MUL VL]\n" + ".inst 0x81972681 // bfmopa za1.s, p1/M, p1/M, z20.h, z23.h\n" + ".inst 0x81972702 // bfmopa za2.s, p1/M, p1/M, z24.h, z23.h\n" + ".inst 0x81972783 // bfmopa za3.s, p1/M, p1/M, z28.h, z23.h\n" + ".inst 0xa142a370 // ld1h { z16.h, z20.h, z24.h, z28.h }, pn8.b/Z, [x27, #0x8, MUL VL]\n" + "ldnt1h { z23.h }, p1/Z, [x23, #2, MUL VL]\n" + ".inst 0x81822460 // bfmopa za0.s, p1/M, p1/M, z3.h, z2.h\n" + ".inst 0x818224e1 // bfmopa za1.s, p1/M, p1/M, z7.h, z2.h\n" + ".inst 0x81822562 // bfmopa za2.s, p1/M, p1/M, z11.h, z2.h\n" + ".inst 0x818225e3 // bfmopa za3.s, p1/M, p1/M, z15.h, z2.h\n" + ".inst 0xa143a363 // ld1h { z3.h, z7.h, z11.h, z15.h }, pn8.b/Z, [x27, #0xc, MUL VL]\n" "addvl x27, x27, #16\n" - "ldnt1h { z27.h }, p1/Z, [x23, #3, MUL VL]\n" + "ldnt1h { z2.h }, p1/Z, [x23, #3, MUL VL]\n" "addvl x23, x23, #4\n" "bgt 6b\n" "7:" // K loop tail - ".inst 0x819d2480 // bfmopa za0.s, p1/M, p1/M, z4.h, z29.h\n" - ".inst 0x819d24a1 // bfmopa za1.s, p1/M, p1/M, z5.h, z29.h\n" - ".inst 0x819d24c2 // bfmopa za2.s, p1/M, p1/M, z6.h, z29.h\n" - ".inst 0x819d24e3 // bfmopa za3.s, p1/M, p1/M, z7.h, z29.h\n" - ".inst 0x81972580 // bfmopa za0.s, p1/M, p1/M, z12.h, z23.h\n" - ".inst 0x819725a1 // bfmopa za1.s, p1/M, p1/M, z13.h, z23.h\n" - ".inst 0x819725c2 // bfmopa za2.s, p1/M, p1/M, z14.h, z23.h\n" - ".inst 0x819725e3 // bfmopa za3.s, p1/M, p1/M, z15.h, z23.h\n" - ".inst 0x81952400 // bfmopa za0.s, p1/M, p1/M, z0.h, z21.h\n" - ".inst 0x81952421 // bfmopa za1.s, p1/M, p1/M, z1.h, z21.h\n" - ".inst 0x81952442 // bfmopa za2.s, p1/M, p1/M, z2.h, z21.h\n" - ".inst 0x81952463 // bfmopa za3.s, p1/M, p1/M, z3.h, z21.h\n" - ".inst 0x819b2640 // bfmopa za0.s, p1/M, p1/M, z18.h, z27.h\n" - ".inst 0x819b26c1 // bfmopa za1.s, p1/M, p1/M, z22.h, z27.h\n" - ".inst 0x819b2742 // bfmopa za2.s, p1/M, p1/M, z26.h, z27.h\n" - ".inst 0x819b27c3 // bfmopa za3.s, p1/M, p1/M, z30.h, z27.h\n" + ".inst 0x81932400 // bfmopa za0.s, p1/M, p1/M, z0.h, z19.h\n" + ".inst 0x81932481 // bfmopa za1.s, p1/M, p1/M, z4.h, z19.h\n" + ".inst 0x81932502 // bfmopa za2.s, p1/M, p1/M, z8.h, z19.h\n" + ".inst 0x81932583 // bfmopa za3.s, p1/M, p1/M, z12.h, z19.h\n" + ".inst 0x81962620 // bfmopa za0.s, p1/M, p1/M, z17.h, z22.h\n" + ".inst 0x819626a1 // bfmopa za1.s, p1/M, p1/M, z21.h, z22.h\n" + ".inst 0x81962722 // bfmopa za2.s, p1/M, p1/M, z25.h, z22.h\n" + ".inst 0x819627a3 // bfmopa za3.s, p1/M, p1/M, z29.h, z22.h\n" + ".inst 0x81972600 // bfmopa za0.s, p1/M, p1/M, z16.h, z23.h\n" + ".inst 0x81972681 // bfmopa za1.s, p1/M, p1/M, z20.h, z23.h\n" + ".inst 0x81972702 // bfmopa za2.s, p1/M, p1/M, z24.h, z23.h\n" + ".inst 0x81972783 // bfmopa za3.s, p1/M, p1/M, z28.h, z23.h\n" + ".inst 0x81822460 // bfmopa za0.s, p1/M, p1/M, z3.h, z2.h\n" + ".inst 0x818224e1 // bfmopa za1.s, p1/M, p1/M, z7.h, z2.h\n" + ".inst 0x81822562 // bfmopa za2.s, p1/M, p1/M, z11.h, z2.h\n" + ".inst 0x818225e3 // bfmopa za3.s, p1/M, p1/M, z15.h, z2.h\n" "8:" // K oddments "cbz x21, 10f\n" "9:" // K oddments: Loop - ".inst 0xa040a364 // ld1h { z4.h-z7.h }, pn8.b/Z, [x27]\n" + ".inst 0xa140a373 // ld1h { z19.h, z23.h, z27.h, z31.h }, pn8.b/Z, [x27]\n" "subs x21, x21, #0x1\n" "addvl x27, x27, #4\n" - "ld1h { z29.h }, p1/Z, [x23]\n" + "ld1h { z11.h }, p1/Z, [x23]\n" "addvl x23, x23, #1\n" - ".inst 0x819d2480 // bfmopa za0.s, p1/M, p1/M, z4.h, z29.h\n" - ".inst 0x819d24a1 // bfmopa za1.s, p1/M, p1/M, z5.h, z29.h\n" - ".inst 0x819d24c2 // bfmopa za2.s, p1/M, p1/M, z6.h, z29.h\n" - ".inst 0x819d24e3 // bfmopa za3.s, p1/M, p1/M, z7.h, z29.h\n" + ".inst 0x818b2660 // bfmopa za0.s, p1/M, p1/M, z19.h, z11.h\n" + ".inst 0x818b26e1 // bfmopa za1.s, p1/M, p1/M, z23.h, z11.h\n" + ".inst 0x818b2762 // bfmopa za2.s, p1/M, p1/M, z27.h, z11.h\n" + ".inst 0x818b27e3 // bfmopa za3.s, p1/M, p1/M, z31.h, z11.h\n" "bgt 9b\n" "10:" // K oddments: End "tbz x16, #1, 14f\n" @@ -242,25 +241,25 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL(const bfloat16 *const A, con "mov x12, #0x0\n" "cntw x20\n" "11:" // Store to partial result buffer: Store and refill: Loop - ".inst 0xa040c1e8 // ld1w { z8.s-z11.s }, pn8.b/Z, [x15]\n" + ".inst 0xa040c1f4 // ld1w { z20.s-z23.s }, pn8.b/Z, [x15]\n" ".inst 0xc0860418 // mova { z24.s-z27.s }, za0h.s[x12]\n" - ".inst 0xc0840500 // mova za0h.s[x12], { z8.s-z11.s }\n" + ".inst 0xc0840680 // mova za0h.s[x12], { z20.s-z23.s }\n" ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" - ".inst 0xa041c1ec // ld1w { z12.s-z15.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" - ".inst 0xc0840581 // mova za1h.s[x12], { z12.s-z15.s }\n" - ".inst 0xc086044c // mova { z12.s-z15.s }, za2h.s[x12]\n" - ".inst 0xc0860460 // mova { z0.s-z3.s }, za3h.s[x12]\n" - ".inst 0xa042c1e8 // ld1w { z8.s-z11.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" - ".inst 0xc0840502 // mova za2h.s[x12], { z8.s-z11.s }\n" - ".inst 0xa043c1fc // ld1w { z28.s-z31.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840783 // mova za3h.s[x12], { z28.s-z31.s }\n" + ".inst 0xa041c1e8 // ld1w { z8.s-z11.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" + ".inst 0xc0840501 // mova za1h.s[x12], { z8.s-z11.s }\n" + ".inst 0xc0860450 // mova { z16.s-z19.s }, za2h.s[x12]\n" + ".inst 0xc086047c // mova { z28.s-z31.s }, za3h.s[x12]\n" + ".inst 0xa042c1ec // ld1w { z12.s-z15.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" + ".inst 0xc0840582 // mova za2h.s[x12], { z12.s-z15.s }\n" + ".inst 0xa043c1ec // ld1w { z12.s-z15.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840583 // mova za3h.s[x12], { z12.s-z15.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" ".inst 0xa060c1d8 // st1w { z24.s-z27.s }, pn8.b, [x14]\n" "addvl x15, x15, #16\n" ".inst 0xa061c1c4 // st1w { z4.s-z7.s }, pn8.b, [x14, #0x4, MUL VL]\n" - ".inst 0xa062c1cc // st1w { z12.s-z15.s }, pn8.b, [x14, #0x8, MUL VL]\n" - ".inst 0xa063c1c0 // st1w { z0.s-z3.s }, pn8.b, [x14, #0xc, MUL VL]\n" + ".inst 0xa062c1d0 // st1w { z16.s-z19.s }, pn8.b, [x14, #0x8, MUL VL]\n" + ".inst 0xa063c1dc // st1w { z28.s-z31.s }, pn8.b, [x14, #0xc, MUL VL]\n" "addvl x14, x14, #16\n" "blt 11b\n" "b 42f\n" @@ -269,15 +268,15 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL(const bfloat16 *const A, con "cntw x20\n" "13:" // Store to partial result buffer: Store only: Loop ".inst 0xc086040c // mova { z12.s-z15.s }, za0h.s[x12]\n" - ".inst 0xc0860438 // mova { z24.s-z27.s }, za1h.s[x12]\n" + ".inst 0xc086043c // mova { z28.s-z31.s }, za1h.s[x12]\n" ".inst 0xa060c1cc // st1w { z12.s-z15.s }, pn8.b, [x14]\n" ".inst 0xc0860440 // mova { z0.s-z3.s }, za2h.s[x12]\n" - ".inst 0xc0860468 // mova { z8.s-z11.s }, za3h.s[x12]\n" - ".inst 0xa061c1d8 // st1w { z24.s-z27.s }, pn8.b, [x14, #0x4, MUL VL]\n" + ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" + ".inst 0xa061c1dc // st1w { z28.s-z31.s }, pn8.b, [x14, #0x4, MUL VL]\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" ".inst 0xa062c1c0 // st1w { z0.s-z3.s }, pn8.b, [x14, #0x8, MUL VL]\n" - ".inst 0xa063c1c8 // st1w { z8.s-z11.s }, pn8.b, [x14, #0xc, MUL VL]\n" + ".inst 0xa063c1cc // st1w { z12.s-z15.s }, pn8.b, [x14, #0xc, MUL VL]\n" "addvl x14, x14, #16\n" "blt 13b\n" "b 42f\n" @@ -296,16 +295,16 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL(const bfloat16 *const A, con "and x20, x22, #0x3\n" "cbz x21, 16f\n" "15:" // Store to output array: Skip activation: Accumulator row 0 loop - ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" - "st1w { z16.s }, p0, [x26]\n" + ".inst 0xc0860404 // mova { z4.s-z7.s }, za0h.s[x12]\n" + "st1w { z4.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1w { z17.s }, p0, [x26]\n" + "st1w { z5.s }, p0, [x26]\n" "add x26, x26, x24\n" "add x12, x12, #0x4\n" - "st1w { z18.s }, p0, [x26]\n" + "st1w { z6.s }, p0, [x26]\n" "add x26, x26, x24\n" "cmp x12, x21, LSL #2\n" - "st1w { z19.s }, p0, [x26]\n" + "st1w { z7.s }, p0, [x26]\n" "add x26, x26, x24\n" "blt 15b\n" "16:" // Store to output array: Skip activation: Accumulator row 0 oddments @@ -331,30 +330,30 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL(const bfloat16 *const A, con "and x20, x22, #0x3\n" "cbz x21, 19f\n" "18:" // Store to output array: Skip activation: Accumulator row 1 loop - ".inst 0xc0860430 // mova { z16.s-z19.s }, za1h.s[x12]\n" - "st1w { z16.s }, p0, [x26]\n" + ".inst 0xc0860428 // mova { z8.s-z11.s }, za1h.s[x12]\n" + "st1w { z8.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1w { z17.s }, p0, [x26]\n" + "st1w { z9.s }, p0, [x26]\n" "add x26, x26, x24\n" "add x12, x12, #0x4\n" - "st1w { z18.s }, p0, [x26]\n" + "st1w { z10.s }, p0, [x26]\n" "add x26, x26, x24\n" "cmp x12, x21, LSL #2\n" - "st1w { z19.s }, p0, [x26]\n" + "st1w { z11.s }, p0, [x26]\n" "add x26, x26, x24\n" "blt 18b\n" "19:" // Store to output array: Skip activation: Accumulator row 1 oddments "cbz x20, 20f\n" "subs x20, x20, #0x1\n" - ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" - "st1w { z4.s }, p0, [x26]\n" + ".inst 0xc0860438 // mova { z24.s-z27.s }, za1h.s[x12]\n" + "st1w { z24.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 20f\n" "subs x20, x20, #0x1\n" - "st1w { z5.s }, p0, [x26]\n" + "st1w { z25.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 20f\n" - "st1w { z6.s }, p0, [x26]\n" + "st1w { z26.s }, p0, [x26]\n" "add x26, x26, x24\n" "20:" // Store to output array: Skip activation: Accumulator row 1 oddments: End "subs x25, x25, x22\n" @@ -366,30 +365,30 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL(const bfloat16 *const A, con "and x20, x22, #0x3\n" "cbz x21, 22f\n" "21:" // Store to output array: Skip activation: Accumulator row 2 loop - ".inst 0xc0860450 // mova { z16.s-z19.s }, za2h.s[x12]\n" - "st1w { z16.s }, p0, [x26]\n" + ".inst 0xc0860444 // mova { z4.s-z7.s }, za2h.s[x12]\n" + "st1w { z4.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1w { z17.s }, p0, [x26]\n" + "st1w { z5.s }, p0, [x26]\n" "add x26, x26, x24\n" "add x12, x12, #0x4\n" - "st1w { z18.s }, p0, [x26]\n" + "st1w { z6.s }, p0, [x26]\n" "add x26, x26, x24\n" "cmp x12, x21, LSL #2\n" - "st1w { z19.s }, p0, [x26]\n" + "st1w { z7.s }, p0, [x26]\n" "add x26, x26, x24\n" "blt 21b\n" "22:" // Store to output array: Skip activation: Accumulator row 2 oddments "cbz x20, 23f\n" "subs x20, x20, #0x1\n" - ".inst 0xc0860454 // mova { z20.s-z23.s }, za2h.s[x12]\n" - "st1w { z20.s }, p0, [x26]\n" + ".inst 0xc086044c // mova { z12.s-z15.s }, za2h.s[x12]\n" + "st1w { z12.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 23f\n" "subs x20, x20, #0x1\n" - "st1w { z21.s }, p0, [x26]\n" + "st1w { z13.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 23f\n" - "st1w { z22.s }, p0, [x26]\n" + "st1w { z14.s }, p0, [x26]\n" "add x26, x26, x24\n" "23:" // Store to output array: Skip activation: Accumulator row 2 oddments: End "subs x25, x25, x22\n" @@ -401,30 +400,30 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL(const bfloat16 *const A, con "and x20, x22, #0x3\n" "cbz x21, 25f\n" "24:" // Store to output array: Skip activation: Accumulator row 3 loop - ".inst 0xc0860464 // mova { z4.s-z7.s }, za3h.s[x12]\n" - "st1w { z4.s }, p0, [x26]\n" + ".inst 0xc0860470 // mova { z16.s-z19.s }, za3h.s[x12]\n" + "st1w { z16.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1w { z5.s }, p0, [x26]\n" + "st1w { z17.s }, p0, [x26]\n" "add x26, x26, x24\n" "add x12, x12, #0x4\n" - "st1w { z6.s }, p0, [x26]\n" + "st1w { z18.s }, p0, [x26]\n" "add x26, x26, x24\n" "cmp x12, x21, LSL #2\n" - "st1w { z7.s }, p0, [x26]\n" + "st1w { z19.s }, p0, [x26]\n" "add x26, x26, x24\n" "blt 24b\n" "25:" // Store to output array: Skip activation: Accumulator row 3 oddments "cbz x20, 26f\n" "subs x20, x20, #0x1\n" - ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" - "st1w { z12.s }, p0, [x26]\n" + ".inst 0xc0860470 // mova { z16.s-z19.s }, za3h.s[x12]\n" + "st1w { z16.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 26f\n" "subs x20, x20, #0x1\n" - "st1w { z13.s }, p0, [x26]\n" + "st1w { z17.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 26f\n" - "st1w { z14.s }, p0, [x26]\n" + "st1w { z18.s }, p0, [x26]\n" "add x26, x26, x24\n" "26:" // Store to output array: Skip activation: Accumulator row 3 oddments: End "subs x25, x25, x22\n" @@ -433,40 +432,40 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL(const bfloat16 *const A, con "27:" // Store to output array: Skip activation: End "cntw x23\n" "cmp x25, x23\n" - "ld1rw { z25.s }, p1/Z, [%x[args], %[offsetof_KernelArgs_min]]\n" + "ld1rw { z21.s }, p1/Z, [%x[args], %[offsetof_KernelArgs_min]]\n" "csel x22, x25, x23, LT\n" "lsr x21, x22, #0x2\n" - "ld1rw { z24.s }, p1/Z, [%x[args], %[offsetof_KernelArgs_max]]\n" + "ld1rw { z20.s }, p1/Z, [%x[args], %[offsetof_KernelArgs_max]]\n" "mov x12, #0x0\n" "and x20, x22, #0x3\n" "cbz x21, 29f\n" "28:" // Store to output array: Accumulator row 0 loop - ".inst 0xc0860414 // mova { z20.s-z23.s }, za0h.s[x12]\n" - ".inst 0xc1b8cb34 // fclamp { z20.s-z23.s }, z25.s, z24.s\n" - "st1w { z20.s }, p0, [x26]\n" + ".inst 0xc086041c // mova { z28.s-z31.s }, za0h.s[x12]\n" + ".inst 0xc1b4cabc // fclamp { z28.s-z31.s }, z21.s, z20.s\n" + "st1w { z28.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1w { z21.s }, p0, [x26]\n" + "st1w { z29.s }, p0, [x26]\n" "add x26, x26, x24\n" "add x12, x12, #0x4\n" - "st1w { z22.s }, p0, [x26]\n" + "st1w { z30.s }, p0, [x26]\n" "add x26, x26, x24\n" "cmp x12, x21, LSL #2\n" - "st1w { z23.s }, p0, [x26]\n" + "st1w { z31.s }, p0, [x26]\n" "add x26, x26, x24\n" "blt 28b\n" "29:" // Store to output array: Accumulator row 0 oddments "cbz x20, 30f\n" - ".inst 0xc0860408 // mova { z8.s-z11.s }, za0h.s[x12]\n" + ".inst 0xc086041c // mova { z28.s-z31.s }, za0h.s[x12]\n" "subs x20, x20, #0x1\n" - ".inst 0xc1b8cb28 // fclamp { z8.s-z11.s }, z25.s, z24.s\n" - "st1w { z8.s }, p0, [x26]\n" + ".inst 0xc1b4cabc // fclamp { z28.s-z31.s }, z21.s, z20.s\n" + "st1w { z28.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 30f\n" "subs x20, x20, #0x1\n" - "st1w { z9.s }, p0, [x26]\n" + "st1w { z29.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 30f\n" - "st1w { z10.s }, p0, [x26]\n" + "st1w { z30.s }, p0, [x26]\n" "add x26, x26, x24\n" "30:" // Store to output array: Accumulator row 0 oddments: End "subs x25, x25, x22\n" @@ -478,24 +477,24 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL(const bfloat16 *const A, con "and x20, x22, #0x3\n" "cbz x21, 32f\n" "31:" // Store to output array: Accumulator row 1 loop - ".inst 0xc0860430 // mova { z16.s-z19.s }, za1h.s[x12]\n" - ".inst 0xc1b8cb30 // fclamp { z16.s-z19.s }, z25.s, z24.s\n" - "st1w { z16.s }, p0, [x26]\n" + ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" + ".inst 0xc1b4caa4 // fclamp { z4.s-z7.s }, z21.s, z20.s\n" + "st1w { z4.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1w { z17.s }, p0, [x26]\n" + "st1w { z5.s }, p0, [x26]\n" "add x26, x26, x24\n" "add x12, x12, #0x4\n" - "st1w { z18.s }, p0, [x26]\n" + "st1w { z6.s }, p0, [x26]\n" "add x26, x26, x24\n" "cmp x12, x21, LSL #2\n" - "st1w { z19.s }, p0, [x26]\n" + "st1w { z7.s }, p0, [x26]\n" "add x26, x26, x24\n" "blt 31b\n" "32:" // Store to output array: Accumulator row 1 oddments "cbz x20, 33f\n" ".inst 0xc0860430 // mova { z16.s-z19.s }, za1h.s[x12]\n" "subs x20, x20, #0x1\n" - ".inst 0xc1b8cb30 // fclamp { z16.s-z19.s }, z25.s, z24.s\n" + ".inst 0xc1b4cab0 // fclamp { z16.s-z19.s }, z21.s, z20.s\n" "st1w { z16.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 33f\n" @@ -516,7 +515,7 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL(const bfloat16 *const A, con "cbz x21, 35f\n" "34:" // Store to output array: Accumulator row 2 loop ".inst 0xc0860450 // mova { z16.s-z19.s }, za2h.s[x12]\n" - ".inst 0xc1b8cb30 // fclamp { z16.s-z19.s }, z25.s, z24.s\n" + ".inst 0xc1b4cab0 // fclamp { z16.s-z19.s }, z21.s, z20.s\n" "st1w { z16.s }, p0, [x26]\n" "add x26, x26, x24\n" "st1w { z17.s }, p0, [x26]\n" @@ -532,7 +531,7 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL(const bfloat16 *const A, con "cbz x20, 36f\n" ".inst 0xc0860450 // mova { z16.s-z19.s }, za2h.s[x12]\n" "subs x20, x20, #0x1\n" - ".inst 0xc1b8cb30 // fclamp { z16.s-z19.s }, z25.s, z24.s\n" + ".inst 0xc1b4cab0 // fclamp { z16.s-z19.s }, z21.s, z20.s\n" "st1w { z16.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 36f\n" @@ -552,24 +551,24 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL(const bfloat16 *const A, con "and x20, x20, #0x3\n" "cbz x21, 38f\n" "37:" // Store to output array: Accumulator row 3 loop - ".inst 0xc0860474 // mova { z20.s-z23.s }, za3h.s[x12]\n" - ".inst 0xc1b8cb34 // fclamp { z20.s-z23.s }, z25.s, z24.s\n" - "st1w { z20.s }, p0, [x26]\n" + ".inst 0xc0860470 // mova { z16.s-z19.s }, za3h.s[x12]\n" + ".inst 0xc1b4cab0 // fclamp { z16.s-z19.s }, z21.s, z20.s\n" + "st1w { z16.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1w { z21.s }, p0, [x26]\n" + "st1w { z17.s }, p0, [x26]\n" "add x26, x26, x24\n" "add x12, x12, #0x4\n" - "st1w { z22.s }, p0, [x26]\n" + "st1w { z18.s }, p0, [x26]\n" "add x26, x26, x24\n" "cmp x12, x21, LSL #2\n" - "st1w { z23.s }, p0, [x26]\n" + "st1w { z19.s }, p0, [x26]\n" "add x26, x26, x24\n" "blt 37b\n" "38:" // Store to output array: Accumulator row 3 oddments "cbz x20, 39f\n" ".inst 0xc0860470 // mova { z16.s-z19.s }, za3h.s[x12]\n" "subs x20, x20, #0x1\n" - ".inst 0xc1b8cb30 // fclamp { z16.s-z19.s }, z25.s, z24.s\n" + ".inst 0xc1b4cab0 // fclamp { z16.s-z19.s }, z21.s, z20.s\n" "st1w { z16.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 39f\n" @@ -588,10 +587,10 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL(const bfloat16 *const A, con ".inst 0xc0840600 // mova za0h.s[x12], { z16.s-z19.s }\n" ".inst 0xa041c1f0 // ld1w { z16.s-z19.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" ".inst 0xc0840601 // mova za1h.s[x12], { z16.s-z19.s }\n" - ".inst 0xa042c1f4 // ld1w { z20.s-z23.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" - ".inst 0xc0840682 // mova za2h.s[x12], { z20.s-z23.s }\n" - ".inst 0xa043c1e4 // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840483 // mova za3h.s[x12], { z4.s-z7.s }\n" + ".inst 0xa042c1f0 // ld1w { z16.s-z19.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" + ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa043c1e8 // ld1w { z8.s-z11.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840503 // mova za3h.s[x12], { z8.s-z11.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x15, x15, #16\n" @@ -615,4 +614,3 @@ void sme2_interleaved_nomerge_bf16fp32_mopa_4VLx1VL(const bfloat16 *const A, con } // namespace arm_gemm #endif // ARM_COMPUTE_ENABLE_SME2 -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_1VLx4VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_1VLx4VL.hpp index 7777349b42c4cf2fe25ed54226f7f41b104c7acb..bf3de2118eec80e124a17fd363fb8aa4fff60c52 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_1VLx4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_1VLx4VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,7 +23,7 @@ */ #pragma once -#ifdef __aarch64__ +#ifdef ARM_COMPUTE_ENABLE_SME2 #include "../std_transforms_sme.hpp" @@ -83,12 +83,11 @@ public: StdTransformsSME transforms = {}; - cls_sme2_interleaved_nomerge_fp32_mopa_1VLx4VL(const CPUInfo *ci) + cls_sme2_interleaved_nomerge_fp32_mopa_1VLx4VL(const CPUInfo *) { - ARM_COMPUTE_UNUSED(ci); } }; } // namespace arm_gemm -#endif // __aarch64__ +#endif // ARM_COMPUTE_ENABLE_SME2 diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_1VLx4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_1VLx4VL/generic.cpp index dd99387c5e00a5e7f328e52dcc164fe55fabbb39..97be758bd6f5ad37025495c5c0f6af780609ef07 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_1VLx4VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_1VLx4VL/generic.cpp @@ -21,7 +21,6 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifdef __ARM_FEATURE_SVE #ifdef ARM_COMPUTE_ENABLE_SME2 #include "arm_gemm.hpp" @@ -113,12 +112,12 @@ void sme2_interleaved_nomerge_fp32_mopa_1VLx4VL(const float *const A, const floa "mov x12, #0x0\n" "cntw x20\n" "1:" // Initial accumulator load from buffer: Loop - ".inst 0xa040c5cc // ld1w { z12.s-z15.s }, pn9.b/Z, [x14]\n" - ".inst 0xc0840580 // mova za0h.s[x12], { z12.s-z15.s }\n" - ".inst 0xa041c5d0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n" - ".inst 0xc0840601 // mova za1h.s[x12], { z16.s-z19.s }\n" - ".inst 0xa042c5c4 // ld1w { z4.s-z7.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n" - ".inst 0xc0840482 // mova za2h.s[x12], { z4.s-z7.s }\n" + ".inst 0xa040c5c4 // ld1w { z4.s-z7.s }, pn9.b/Z, [x14]\n" + ".inst 0xc0840480 // mova za0h.s[x12], { z4.s-z7.s }\n" + ".inst 0xa041c5d8 // ld1w { z24.s-z27.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n" + ".inst 0xc0840701 // mova za1h.s[x12], { z24.s-z27.s }\n" + ".inst 0xa042c5d4 // ld1w { z20.s-z23.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n" + ".inst 0xc0840682 // mova za2h.s[x12], { z20.s-z23.s }\n" ".inst 0xa043c5c4 // ld1w { z4.s-z7.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n" ".inst 0xc0840483 // mova za3h.s[x12], { z4.s-z7.s }\n" "add x12, x12, #0x4\n" @@ -138,12 +137,12 @@ void sme2_interleaved_nomerge_fp32_mopa_1VLx4VL(const float *const A, const floa "ldr x20, [%x[args], %[offsetof_bias]]\n" ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "cbz x20, 5f\n" - "fmov z21.s, #1.0\n" + "fmov z6.s, #1.0\n" ".inst 0xa009c29d // ldnt1w { z28.s-z31.s }, p8/Z, [x20, x9, LSL #2]\n" - ".inst 0x809c02a0 // fmopa za0.s, p0/M, p0/M, z21.s, z28.s\n" - ".inst 0x809d02a1 // fmopa za1.s, p0/M, p0/M, z21.s, z29.s\n" - ".inst 0x809e02a2 // fmopa za2.s, p0/M, p0/M, z21.s, z30.s\n" - ".inst 0x809f02a3 // fmopa za3.s, p0/M, p0/M, z21.s, z31.s\n" + ".inst 0x809c00c0 // fmopa za0.s, p0/M, p0/M, z6.s, z28.s\n" + ".inst 0x809d00c1 // fmopa za1.s, p0/M, p0/M, z6.s, z29.s\n" + ".inst 0x809e00c2 // fmopa za2.s, p0/M, p0/M, z6.s, z30.s\n" + ".inst 0x809f00c3 // fmopa za3.s, p0/M, p0/M, z6.s, z31.s\n" "4:" // Prepare accumulators: Test for last block "mov x20, x9\n" "mov x21, x10\n" @@ -164,75 +163,75 @@ void sme2_interleaved_nomerge_fp32_mopa_1VLx4VL(const float *const A, const floa "madd x21, x9, x20, x21\n" // bptr = B + n * kstride_bytes "cbz x23, 8f\n" "subs x23, x23, #0x1\n" - "ld1w { z0.s }, p0/Z, [x26]\n" - ".inst 0xa140c6bb // ldnt1w { z19.s, z23.s, z27.s, z31.s }, pn9.b/Z, [x21]\n" - "ld1w { z13.s }, p0/Z, [x26, #1, MUL VL]\n" - ".inst 0xa141c6aa // ldnt1w { z2.s, z6.s, z10.s, z14.s }, pn9.b/Z, [x21, #0x4, MUL VL]\n" - "ld1w { z12.s }, p0/Z, [x26, #2, MUL VL]\n" - ".inst 0xa142c6ab // ldnt1w { z3.s, z7.s, z11.s, z15.s }, pn9.b/Z, [x21, #0x8, MUL VL]\n" - "ld1w { z26.s }, p0/Z, [x26, #3, MUL VL]\n" + "ld1w { z28.s }, p0/Z, [x26]\n" + ".inst 0xa040c6a9 // ldnt1w { z8.s-z11.s }, pn9.b/Z, [x21]\n" + "ld1w { z22.s }, p0/Z, [x26, #1, MUL VL]\n" + ".inst 0xa041c6ad // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x21, #0x4, MUL VL]\n" + "ld1w { z30.s }, p0/Z, [x26, #2, MUL VL]\n" + ".inst 0xa042c6a5 // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x21, #0x8, MUL VL]\n" + "ld1w { z20.s }, p0/Z, [x26, #3, MUL VL]\n" "addvl x26, x26, #4\n" - ".inst 0xa143c6b8 // ldnt1w { z16.s, z20.s, z24.s, z28.s }, pn9.b/Z, [x21, #0xc, MUL VL]\n" + ".inst 0xa143c6bb // ldnt1w { z19.s, z23.s, z27.s, z31.s }, pn9.b/Z, [x21, #0xc, MUL VL]\n" "addvl x21, x21, #16\n" "ble 7f\n" "6:" // K loop - ".inst 0x80930000 // fmopa za0.s, p0/M, p0/M, z0.s, z19.s\n" + ".inst 0x80880380 // fmopa za0.s, p0/M, p0/M, z28.s, z8.s\n" "subs x23, x23, #0x1\n" - ".inst 0x80970001 // fmopa za1.s, p0/M, p0/M, z0.s, z23.s\n" - ".inst 0x809b0002 // fmopa za2.s, p0/M, p0/M, z0.s, z27.s\n" - ".inst 0x809f0003 // fmopa za3.s, p0/M, p0/M, z0.s, z31.s\n" - "ld1w { z0.s }, p0/Z, [x26]\n" - ".inst 0x808201a0 // fmopa za0.s, p0/M, p0/M, z13.s, z2.s\n" - ".inst 0xa140c6bb // ldnt1w { z19.s, z23.s, z27.s, z31.s }, pn9.b/Z, [x21]\n" - ".inst 0x808601a1 // fmopa za1.s, p0/M, p0/M, z13.s, z6.s\n" - ".inst 0x808a01a2 // fmopa za2.s, p0/M, p0/M, z13.s, z10.s\n" - ".inst 0x808e01a3 // fmopa za3.s, p0/M, p0/M, z13.s, z14.s\n" - "ld1w { z13.s }, p0/Z, [x26, #1, MUL VL]\n" - ".inst 0x80830180 // fmopa za0.s, p0/M, p0/M, z12.s, z3.s\n" - ".inst 0xa141c6aa // ldnt1w { z2.s, z6.s, z10.s, z14.s }, pn9.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0x80870181 // fmopa za1.s, p0/M, p0/M, z12.s, z7.s\n" - ".inst 0x808b0182 // fmopa za2.s, p0/M, p0/M, z12.s, z11.s\n" - ".inst 0x808f0183 // fmopa za3.s, p0/M, p0/M, z12.s, z15.s\n" - "ld1w { z12.s }, p0/Z, [x26, #2, MUL VL]\n" - ".inst 0xa142c6ab // ldnt1w { z3.s, z7.s, z11.s, z15.s }, pn9.b/Z, [x21, #0x8, MUL VL]\n" - ".inst 0x80900340 // fmopa za0.s, p0/M, p0/M, z26.s, z16.s\n" - ".inst 0x80940341 // fmopa za1.s, p0/M, p0/M, z26.s, z20.s\n" - ".inst 0x80980342 // fmopa za2.s, p0/M, p0/M, z26.s, z24.s\n" - ".inst 0x809c0343 // fmopa za3.s, p0/M, p0/M, z26.s, z28.s\n" - "ld1w { z26.s }, p0/Z, [x26, #3, MUL VL]\n" + ".inst 0x80890381 // fmopa za1.s, p0/M, p0/M, z28.s, z9.s\n" + ".inst 0x808a0382 // fmopa za2.s, p0/M, p0/M, z28.s, z10.s\n" + ".inst 0x808b0383 // fmopa za3.s, p0/M, p0/M, z28.s, z11.s\n" + "ld1w { z28.s }, p0/Z, [x26]\n" + ".inst 0x808c02c0 // fmopa za0.s, p0/M, p0/M, z22.s, z12.s\n" + ".inst 0xa040c6a9 // ldnt1w { z8.s-z11.s }, pn9.b/Z, [x21]\n" + ".inst 0x808d02c1 // fmopa za1.s, p0/M, p0/M, z22.s, z13.s\n" + ".inst 0x808e02c2 // fmopa za2.s, p0/M, p0/M, z22.s, z14.s\n" + ".inst 0x808f02c3 // fmopa za3.s, p0/M, p0/M, z22.s, z15.s\n" + "ld1w { z22.s }, p0/Z, [x26, #1, MUL VL]\n" + ".inst 0x808403c0 // fmopa za0.s, p0/M, p0/M, z30.s, z4.s\n" + ".inst 0xa041c6ad // ldnt1w { z12.s-z15.s }, pn9.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0x808503c1 // fmopa za1.s, p0/M, p0/M, z30.s, z5.s\n" + ".inst 0x808603c2 // fmopa za2.s, p0/M, p0/M, z30.s, z6.s\n" + ".inst 0x808703c3 // fmopa za3.s, p0/M, p0/M, z30.s, z7.s\n" + "ld1w { z30.s }, p0/Z, [x26, #2, MUL VL]\n" + ".inst 0xa042c6a5 // ldnt1w { z4.s-z7.s }, pn9.b/Z, [x21, #0x8, MUL VL]\n" + ".inst 0x80930280 // fmopa za0.s, p0/M, p0/M, z20.s, z19.s\n" + ".inst 0x80970281 // fmopa za1.s, p0/M, p0/M, z20.s, z23.s\n" + ".inst 0x809b0282 // fmopa za2.s, p0/M, p0/M, z20.s, z27.s\n" + ".inst 0x809f0283 // fmopa za3.s, p0/M, p0/M, z20.s, z31.s\n" + "ld1w { z20.s }, p0/Z, [x26, #3, MUL VL]\n" "addvl x26, x26, #4\n" - ".inst 0xa143c6b8 // ldnt1w { z16.s, z20.s, z24.s, z28.s }, pn9.b/Z, [x21, #0xc, MUL VL]\n" + ".inst 0xa143c6bb // ldnt1w { z19.s, z23.s, z27.s, z31.s }, pn9.b/Z, [x21, #0xc, MUL VL]\n" "addvl x21, x21, #16\n" "bgt 6b\n" "7:" // K loop tail - ".inst 0x80930000 // fmopa za0.s, p0/M, p0/M, z0.s, z19.s\n" - ".inst 0x80970001 // fmopa za1.s, p0/M, p0/M, z0.s, z23.s\n" - ".inst 0x809b0002 // fmopa za2.s, p0/M, p0/M, z0.s, z27.s\n" - ".inst 0x809f0003 // fmopa za3.s, p0/M, p0/M, z0.s, z31.s\n" - ".inst 0x808201a0 // fmopa za0.s, p0/M, p0/M, z13.s, z2.s\n" - ".inst 0x808601a1 // fmopa za1.s, p0/M, p0/M, z13.s, z6.s\n" - ".inst 0x808a01a2 // fmopa za2.s, p0/M, p0/M, z13.s, z10.s\n" - ".inst 0x808e01a3 // fmopa za3.s, p0/M, p0/M, z13.s, z14.s\n" - ".inst 0x80830180 // fmopa za0.s, p0/M, p0/M, z12.s, z3.s\n" - ".inst 0x80870181 // fmopa za1.s, p0/M, p0/M, z12.s, z7.s\n" - ".inst 0x808b0182 // fmopa za2.s, p0/M, p0/M, z12.s, z11.s\n" - ".inst 0x808f0183 // fmopa za3.s, p0/M, p0/M, z12.s, z15.s\n" - ".inst 0x80900340 // fmopa za0.s, p0/M, p0/M, z26.s, z16.s\n" - ".inst 0x80940341 // fmopa za1.s, p0/M, p0/M, z26.s, z20.s\n" - ".inst 0x80980342 // fmopa za2.s, p0/M, p0/M, z26.s, z24.s\n" - ".inst 0x809c0343 // fmopa za3.s, p0/M, p0/M, z26.s, z28.s\n" + ".inst 0x80880380 // fmopa za0.s, p0/M, p0/M, z28.s, z8.s\n" + ".inst 0x80890381 // fmopa za1.s, p0/M, p0/M, z28.s, z9.s\n" + ".inst 0x808a0382 // fmopa za2.s, p0/M, p0/M, z28.s, z10.s\n" + ".inst 0x808b0383 // fmopa za3.s, p0/M, p0/M, z28.s, z11.s\n" + ".inst 0x808c02c0 // fmopa za0.s, p0/M, p0/M, z22.s, z12.s\n" + ".inst 0x808d02c1 // fmopa za1.s, p0/M, p0/M, z22.s, z13.s\n" + ".inst 0x808e02c2 // fmopa za2.s, p0/M, p0/M, z22.s, z14.s\n" + ".inst 0x808f02c3 // fmopa za3.s, p0/M, p0/M, z22.s, z15.s\n" + ".inst 0x808403c0 // fmopa za0.s, p0/M, p0/M, z30.s, z4.s\n" + ".inst 0x808503c1 // fmopa za1.s, p0/M, p0/M, z30.s, z5.s\n" + ".inst 0x808603c2 // fmopa za2.s, p0/M, p0/M, z30.s, z6.s\n" + ".inst 0x808703c3 // fmopa za3.s, p0/M, p0/M, z30.s, z7.s\n" + ".inst 0x80930280 // fmopa za0.s, p0/M, p0/M, z20.s, z19.s\n" + ".inst 0x80970281 // fmopa za1.s, p0/M, p0/M, z20.s, z23.s\n" + ".inst 0x809b0282 // fmopa za2.s, p0/M, p0/M, z20.s, z27.s\n" + ".inst 0x809f0283 // fmopa za3.s, p0/M, p0/M, z20.s, z31.s\n" "8:" // K oddments "cbz x22, 10f\n" "9:" // K oddments: Loop - "ld1w { z0.s }, p0/Z, [x26]\n" + "ld1w { z8.s }, p0/Z, [x26]\n" "subs x22, x22, #0x1\n" "addvl x26, x26, #1\n" - ".inst 0xa140c6b3 // ld1w { z19.s, z23.s, z27.s, z31.s }, pn9.b/Z, [x21]\n" + ".inst 0xa140c6a3 // ld1w { z3.s, z7.s, z11.s, z15.s }, pn9.b/Z, [x21]\n" "addvl x21, x21, #4\n" - ".inst 0x80930000 // fmopa za0.s, p0/M, p0/M, z0.s, z19.s\n" - ".inst 0x80970001 // fmopa za1.s, p0/M, p0/M, z0.s, z23.s\n" - ".inst 0x809b0002 // fmopa za2.s, p0/M, p0/M, z0.s, z27.s\n" - ".inst 0x809f0003 // fmopa za3.s, p0/M, p0/M, z0.s, z31.s\n" + ".inst 0x80830100 // fmopa za0.s, p0/M, p0/M, z8.s, z3.s\n" + ".inst 0x80870101 // fmopa za1.s, p0/M, p0/M, z8.s, z7.s\n" + ".inst 0x808b0102 // fmopa za2.s, p0/M, p0/M, z8.s, z11.s\n" + ".inst 0x808f0103 // fmopa za3.s, p0/M, p0/M, z8.s, z15.s\n" "bgt 9b\n" "10:" // K oddments: End "tbz x15, #1, 14f\n" @@ -240,25 +239,25 @@ void sme2_interleaved_nomerge_fp32_mopa_1VLx4VL(const float *const A, const floa "mov x12, #0x0\n" "cntw x20\n" "11:" // Store to partial result buffer: Store and refill: Loop - ".inst 0xa040c5cc // ld1w { z12.s-z15.s }, pn9.b/Z, [x14]\n" - ".inst 0xc0860418 // mova { z24.s-z27.s }, za0h.s[x12]\n" - ".inst 0xc0840580 // mova za0h.s[x12], { z12.s-z15.s }\n" - ".inst 0xc0860434 // mova { z20.s-z23.s }, za1h.s[x12]\n" - ".inst 0xa041c5dc // ld1w { z28.s-z31.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n" - ".inst 0xc0840781 // mova za1h.s[x12], { z28.s-z31.s }\n" - ".inst 0xc086045c // mova { z28.s-z31.s }, za2h.s[x12]\n" - ".inst 0xc0860470 // mova { z16.s-z19.s }, za3h.s[x12]\n" - ".inst 0xa042c5cc // ld1w { z12.s-z15.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n" - ".inst 0xc0840582 // mova za2h.s[x12], { z12.s-z15.s }\n" - ".inst 0xa043c5cc // ld1w { z12.s-z15.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n" - ".inst 0xc0840583 // mova za3h.s[x12], { z12.s-z15.s }\n" + ".inst 0xa040c5d4 // ld1w { z20.s-z23.s }, pn9.b/Z, [x14]\n" + ".inst 0xc0860408 // mova { z8.s-z11.s }, za0h.s[x12]\n" + ".inst 0xc0840680 // mova za0h.s[x12], { z20.s-z23.s }\n" + ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" + ".inst 0xa041c5cc // ld1w { z12.s-z15.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n" + ".inst 0xc0840581 // mova za1h.s[x12], { z12.s-z15.s }\n" + ".inst 0xc086044c // mova { z12.s-z15.s }, za2h.s[x12]\n" + ".inst 0xc0860460 // mova { z0.s-z3.s }, za3h.s[x12]\n" + ".inst 0xa042c5d0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n" + ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa043c5d0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n" + ".inst 0xc0840603 // mova za3h.s[x12], { z16.s-z19.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" - ".inst 0xa060c5b8 // st1w { z24.s-z27.s }, pn9.b, [x13]\n" + ".inst 0xa060c5a8 // st1w { z8.s-z11.s }, pn9.b, [x13]\n" "addvl x14, x14, #16\n" - ".inst 0xa061c5b4 // st1w { z20.s-z23.s }, pn9.b, [x13, #0x4, MUL VL]\n" - ".inst 0xa062c5bc // st1w { z28.s-z31.s }, pn9.b, [x13, #0x8, MUL VL]\n" - ".inst 0xa063c5b0 // st1w { z16.s-z19.s }, pn9.b, [x13, #0xc, MUL VL]\n" + ".inst 0xa061c5a4 // st1w { z4.s-z7.s }, pn9.b, [x13, #0x4, MUL VL]\n" + ".inst 0xa062c5ac // st1w { z12.s-z15.s }, pn9.b, [x13, #0x8, MUL VL]\n" + ".inst 0xa063c5a0 // st1w { z0.s-z3.s }, pn9.b, [x13, #0xc, MUL VL]\n" "addvl x13, x13, #16\n" "blt 11b\n" "b 24f\n" @@ -266,15 +265,15 @@ void sme2_interleaved_nomerge_fp32_mopa_1VLx4VL(const float *const A, const floa "mov x12, #0x0\n" "cntw x20\n" "13:" // Store to partial result buffer: Store only: Loop - ".inst 0xc086040c // mova { z12.s-z15.s }, za0h.s[x12]\n" - ".inst 0xc086043c // mova { z28.s-z31.s }, za1h.s[x12]\n" - ".inst 0xa060c5ac // st1w { z12.s-z15.s }, pn9.b, [x13]\n" - ".inst 0xc0860450 // mova { z16.s-z19.s }, za2h.s[x12]\n" + ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" + ".inst 0xc0860438 // mova { z24.s-z27.s }, za1h.s[x12]\n" + ".inst 0xa060c5b0 // st1w { z16.s-z19.s }, pn9.b, [x13]\n" + ".inst 0xc0860448 // mova { z8.s-z11.s }, za2h.s[x12]\n" ".inst 0xc0860464 // mova { z4.s-z7.s }, za3h.s[x12]\n" - ".inst 0xa061c5bc // st1w { z28.s-z31.s }, pn9.b, [x13, #0x4, MUL VL]\n" + ".inst 0xa061c5b8 // st1w { z24.s-z27.s }, pn9.b, [x13, #0x4, MUL VL]\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" - ".inst 0xa062c5b0 // st1w { z16.s-z19.s }, pn9.b, [x13, #0x8, MUL VL]\n" + ".inst 0xa062c5a8 // st1w { z8.s-z11.s }, pn9.b, [x13, #0x8, MUL VL]\n" ".inst 0xa063c5a4 // st1w { z4.s-z7.s }, pn9.b, [x13, #0xc, MUL VL]\n" "addvl x13, x13, #16\n" "blt 13b\n" @@ -312,18 +311,18 @@ void sme2_interleaved_nomerge_fp32_mopa_1VLx4VL(const float *const A, const floa "16:" // Store to output array: Skip activation: Accumulator row 0 oddments "cbz x20, 17f\n" "subs x20, x20, #0x1\n" - ".inst 0xc0860400 // mova { z0.s-z3.s }, za0h.s[x12]\n" - ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" - ".inst 0xc0860448 // mova { z8.s-z11.s }, za2h.s[x12]\n" - ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" - ".inst 0xa160c320 // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x25]\n" + ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" + ".inst 0xc0860434 // mova { z20.s-z23.s }, za1h.s[x12]\n" + ".inst 0xc0860458 // mova { z24.s-z27.s }, za2h.s[x12]\n" + ".inst 0xc086047c // mova { z28.s-z31.s }, za3h.s[x12]\n" + ".inst 0xa160c330 // st1w { z16.s, z20.s, z24.s, z28.s }, p8, [x25]\n" "add x25, x25, x23\n" "beq 17f\n" "subs x20, x20, #0x1\n" - ".inst 0xa160c321 // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x25]\n" + ".inst 0xa160c331 // st1w { z17.s, z21.s, z25.s, z29.s }, p8, [x25]\n" "add x25, x25, x23\n" "beq 17f\n" - ".inst 0xa160c322 // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x25]\n" + ".inst 0xa160c332 // st1w { z18.s, z22.s, z26.s, z30.s }, p8, [x25]\n" "add x25, x25, x23\n" "17:" // Store to output array: Skip activation: Accumulator row 0 oddments: End "subs x24, x24, x22\n" @@ -332,66 +331,66 @@ void sme2_interleaved_nomerge_fp32_mopa_1VLx4VL(const float *const A, const floa "18:" // Store to output array: Skip activation: End "cntw x20\n" "cmp x24, x20\n" - "ld1rw { z23.s }, p0/Z, [%x[args], %[offsetof_KernelArgs_min]]\n" + "ld1rw { z1.s }, p0/Z, [%x[args], %[offsetof_KernelArgs_min]]\n" "csel x20, x24, x20, LT\n" "lsr x21, x20, #0x2\n" - "ld1rw { z16.s }, p0/Z, [%x[args], %[offsetof_KernelArgs_max]]\n" + "ld1rw { z0.s }, p0/Z, [%x[args], %[offsetof_KernelArgs_max]]\n" "mov x12, #0x0\n" "and x20, x20, #0x3\n" "cbz x21, 20f\n" "19:" // Store to output array: Accumulator row 0 loop - ".inst 0xc0860400 // mova { z0.s-z3.s }, za0h.s[x12]\n" - ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" - ".inst 0xc1b0cae0 // fclamp { z0.s-z3.s }, z23.s, z16.s\n" - ".inst 0xc1b0cae4 // fclamp { z4.s-z7.s }, z23.s, z16.s\n" - ".inst 0xc0860448 // mova { z8.s-z11.s }, za2h.s[x12]\n" - ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" - ".inst 0xc1b0cae8 // fclamp { z8.s-z11.s }, z23.s, z16.s\n" - ".inst 0xc1b0caec // fclamp { z12.s-z15.s }, z23.s, z16.s\n" - ".inst 0xa160c320 // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x25]\n" + ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" + ".inst 0xc0860434 // mova { z20.s-z23.s }, za1h.s[x12]\n" + ".inst 0xc1a0c830 // fclamp { z16.s-z19.s }, z1.s, z0.s\n" + ".inst 0xc1a0c834 // fclamp { z20.s-z23.s }, z1.s, z0.s\n" + ".inst 0xc0860458 // mova { z24.s-z27.s }, za2h.s[x12]\n" + ".inst 0xc086047c // mova { z28.s-z31.s }, za3h.s[x12]\n" + ".inst 0xc1a0c838 // fclamp { z24.s-z27.s }, z1.s, z0.s\n" + ".inst 0xc1a0c83c // fclamp { z28.s-z31.s }, z1.s, z0.s\n" + ".inst 0xa160c330 // st1w { z16.s, z20.s, z24.s, z28.s }, p8, [x25]\n" "add x25, x25, x23\n" "add x12, x12, #0x4\n" - ".inst 0xa160c321 // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x25]\n" + ".inst 0xa160c331 // st1w { z17.s, z21.s, z25.s, z29.s }, p8, [x25]\n" "add x25, x25, x23\n" "cmp x12, x21, LSL #2\n" - ".inst 0xa160c322 // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x25]\n" + ".inst 0xa160c332 // st1w { z18.s, z22.s, z26.s, z30.s }, p8, [x25]\n" "add x25, x25, x23\n" - ".inst 0xa160c323 // st1w { z3.s, z7.s, z11.s, z15.s }, p8, [x25]\n" + ".inst 0xa160c333 // st1w { z19.s, z23.s, z27.s, z31.s }, p8, [x25]\n" "add x25, x25, x23\n" "blt 19b\n" "20:" // Store to output array: Accumulator row 0 oddments "cbz x20, 21f\n" - ".inst 0xc0860400 // mova { z0.s-z3.s }, za0h.s[x12]\n" - ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" - ".inst 0xc1b0cae0 // fclamp { z0.s-z3.s }, z23.s, z16.s\n" - ".inst 0xc1b0cae4 // fclamp { z4.s-z7.s }, z23.s, z16.s\n" - ".inst 0xc0860448 // mova { z8.s-z11.s }, za2h.s[x12]\n" - ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" - ".inst 0xc1b0cae8 // fclamp { z8.s-z11.s }, z23.s, z16.s\n" - ".inst 0xc1b0caec // fclamp { z12.s-z15.s }, z23.s, z16.s\n" + ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" + ".inst 0xc0860434 // mova { z20.s-z23.s }, za1h.s[x12]\n" + ".inst 0xc1a0c830 // fclamp { z16.s-z19.s }, z1.s, z0.s\n" + ".inst 0xc1a0c834 // fclamp { z20.s-z23.s }, z1.s, z0.s\n" + ".inst 0xc0860458 // mova { z24.s-z27.s }, za2h.s[x12]\n" + ".inst 0xc086047c // mova { z28.s-z31.s }, za3h.s[x12]\n" + ".inst 0xc1a0c838 // fclamp { z24.s-z27.s }, z1.s, z0.s\n" + ".inst 0xc1a0c83c // fclamp { z28.s-z31.s }, z1.s, z0.s\n" "subs x20, x20, #0x1\n" - ".inst 0xa160c320 // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x25]\n" + ".inst 0xa160c330 // st1w { z16.s, z20.s, z24.s, z28.s }, p8, [x25]\n" "add x25, x25, x23\n" "beq 21f\n" "subs x20, x20, #0x1\n" - ".inst 0xa160c321 // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x25]\n" + ".inst 0xa160c331 // st1w { z17.s, z21.s, z25.s, z29.s }, p8, [x25]\n" "add x25, x25, x23\n" "beq 21f\n" - ".inst 0xa160c322 // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x25]\n" + ".inst 0xa160c332 // st1w { z18.s, z22.s, z26.s, z30.s }, p8, [x25]\n" "21:" // Store to output array: Accumulator row 0 oddments: End "22:" // Store to output array: End "tbz x15, #0, 24f\n" "mov x12, #0x0\n" "cntw x20\n" "23:" // Store to output array: Refill accumulators: Loop - ".inst 0xa040c5d0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x14]\n" - ".inst 0xc0840600 // mova za0h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa040c5cc // ld1w { z12.s-z15.s }, pn9.b/Z, [x14]\n" + ".inst 0xc0840580 // mova za0h.s[x12], { z12.s-z15.s }\n" ".inst 0xa041c5d0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x4, MUL VL]\n" ".inst 0xc0840601 // mova za1h.s[x12], { z16.s-z19.s }\n" ".inst 0xa042c5d0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x14, #0x8, MUL VL]\n" ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" - ".inst 0xa043c5c8 // ld1w { z8.s-z11.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n" - ".inst 0xc0840503 // mova za3h.s[x12], { z8.s-z11.s }\n" + ".inst 0xa043c5c4 // ld1w { z4.s-z7.s }, pn9.b/Z, [x14, #0xc, MUL VL]\n" + ".inst 0xc0840483 // mova za3h.s[x12], { z4.s-z7.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x14, x14, #16\n" @@ -415,4 +414,3 @@ void sme2_interleaved_nomerge_fp32_mopa_1VLx4VL(const float *const A, const floa } // namespace arm_gemm #endif // ARM_COMPUTE_ENABLE_SME2 -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_2VLx2VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_2VLx2VL.hpp index 51e8c43335e1761384c064dace06bf578b04246b..9bc1f831006e7ead8af95457129b0500cceaa543 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_2VLx2VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_2VLx2VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,7 +23,7 @@ */ #pragma once -#ifdef __aarch64__ +#ifdef ARM_COMPUTE_ENABLE_SME2 #include "../std_transforms_sme.hpp" @@ -83,12 +83,11 @@ public: StdTransformsSME transforms = {}; - cls_sme2_interleaved_nomerge_fp32_mopa_2VLx2VL(const CPUInfo *ci) + cls_sme2_interleaved_nomerge_fp32_mopa_2VLx2VL(const CPUInfo *) { - ARM_COMPUTE_UNUSED(ci); } }; } // namespace arm_gemm -#endif // __aarch64__ +#endif // ARM_COMPUTE_ENABLE_SME2 diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_2VLx2VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_2VLx2VL/generic.cpp index 87d7827c5bbcb625edc31b308c2846fe98b0147c..3c475044e29dc252b60475db6489b1fde1df29a6 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_2VLx2VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_2VLx2VL/generic.cpp @@ -21,7 +21,6 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifdef __ARM_FEATURE_SVE #ifdef ARM_COMPUTE_ENABLE_SME2 #include "arm_gemm.hpp" @@ -113,14 +112,14 @@ void sme2_interleaved_nomerge_fp32_mopa_2VLx2VL(const float *const A, const floa "mov x12, #0x0\n" "cntw x20\n" "1:" // Initial accumulator load from buffer: Loop - ".inst 0xa040c5e8 // ld1w { z8.s-z11.s }, pn9.b/Z, [x15]\n" - ".inst 0xc0840500 // mova za0h.s[x12], { z8.s-z11.s }\n" - ".inst 0xa041c5f0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n" - ".inst 0xc0840601 // mova za1h.s[x12], { z16.s-z19.s }\n" - ".inst 0xa042c5e0 // ld1w { z0.s-z3.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n" - ".inst 0xc0840402 // mova za2h.s[x12], { z0.s-z3.s }\n" - ".inst 0xa043c5fc // ld1w { z28.s-z31.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840783 // mova za3h.s[x12], { z28.s-z31.s }\n" + ".inst 0xa040c5e4 // ld1w { z4.s-z7.s }, pn9.b/Z, [x15]\n" + ".inst 0xc0840480 // mova za0h.s[x12], { z4.s-z7.s }\n" + ".inst 0xa041c5e8 // ld1w { z8.s-z11.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n" + ".inst 0xc0840501 // mova za1h.s[x12], { z8.s-z11.s }\n" + ".inst 0xa042c5f4 // ld1w { z20.s-z23.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n" + ".inst 0xc0840682 // mova za2h.s[x12], { z20.s-z23.s }\n" + ".inst 0xa043c5e8 // ld1w { z8.s-z11.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840503 // mova za3h.s[x12], { z8.s-z11.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x15, x15, #16\n" @@ -138,12 +137,12 @@ void sme2_interleaved_nomerge_fp32_mopa_2VLx2VL(const float *const A, const floa "ldr x20, [%x[args], %[offsetof_bias]]\n" ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "cbz x20, 5f\n" - "fmov z21.s, #1.0\n" - ".inst 0xa00a428f // ldnt1w { z14.s-z15.s }, p8/Z, [x20, x10, LSL #2]\n" - ".inst 0x808e02a0 // fmopa za0.s, p0/M, p0/M, z21.s, z14.s\n" - ".inst 0x808f02a1 // fmopa za1.s, p0/M, p0/M, z21.s, z15.s\n" - ".inst 0x808e02a2 // fmopa za2.s, p0/M, p0/M, z21.s, z14.s\n" - ".inst 0x808f02a3 // fmopa za3.s, p0/M, p0/M, z21.s, z15.s\n" + "fmov z12.s, #1.0\n" + ".inst 0xa10a4289 // ldnt1w { z1.s, z9.s }, p8/Z, [x20, x10, LSL #2]\n" + ".inst 0x80810180 // fmopa za0.s, p0/M, p0/M, z12.s, z1.s\n" + ".inst 0x80890181 // fmopa za1.s, p0/M, p0/M, z12.s, z9.s\n" + ".inst 0x80810182 // fmopa za2.s, p0/M, p0/M, z12.s, z1.s\n" + ".inst 0x80890183 // fmopa za3.s, p0/M, p0/M, z12.s, z9.s\n" "4:" // Prepare accumulators: Test for last block "mov x20, x10\n" "mov x21, x11\n" @@ -164,75 +163,75 @@ void sme2_interleaved_nomerge_fp32_mopa_2VLx2VL(const float *const A, const floa "madd x21, x10, x20, x21\n" // bptr = B + n * kstride_bytes "cbz x23, 8f\n" "subs x23, x23, #0x1\n" - ".inst 0xa1404767 // ld1w { z7.s, z15.s }, pn9.b/Z, [x27]\n" - ".inst 0xa14046bf // ldnt1w { z23.s, z31.s }, pn9.b/Z, [x21]\n" - ".inst 0xa0414768 // ld1w { z8.s-z9.s }, pn9.b/Z, [x27, #0x2, MUL VL]\n" - ".inst 0xa04146a3 // ldnt1w { z2.s-z3.s }, pn9.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0xa1424772 // ld1w { z18.s, z26.s }, pn9.b/Z, [x27, #0x4, MUL VL]\n" - ".inst 0xa04246b1 // ldnt1w { z16.s-z17.s }, pn9.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0xa1434776 // ld1w { z22.s, z30.s }, pn9.b/Z, [x27, #0x6, MUL VL]\n" + ".inst 0xa0404772 // ld1w { z18.s-z19.s }, pn9.b/Z, [x27]\n" + ".inst 0xa04046a3 // ldnt1w { z2.s-z3.s }, pn9.b/Z, [x21]\n" + ".inst 0xa0414764 // ld1w { z4.s-z5.s }, pn9.b/Z, [x27, #0x2, MUL VL]\n" + ".inst 0xa04146bb // ldnt1w { z26.s-z27.s }, pn9.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0xa042476a // ld1w { z10.s-z11.s }, pn9.b/Z, [x27, #0x4, MUL VL]\n" + ".inst 0xa04246b5 // ldnt1w { z20.s-z21.s }, pn9.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0xa0434766 // ld1w { z6.s-z7.s }, pn9.b/Z, [x27, #0x6, MUL VL]\n" "addvl x27, x27, #8\n" - ".inst 0xa14346ac // ldnt1w { z4.s, z12.s }, pn9.b/Z, [x21, #0x6, MUL VL]\n" + ".inst 0xa04346a9 // ldnt1w { z8.s-z9.s }, pn9.b/Z, [x21, #0x6, MUL VL]\n" "addvl x21, x21, #8\n" "ble 7f\n" "6:" // K loop - ".inst 0x809700e0 // fmopa za0.s, p0/M, p0/M, z7.s, z23.s\n" + ".inst 0x80820240 // fmopa za0.s, p0/M, p0/M, z18.s, z2.s\n" "subs x23, x23, #0x1\n" - ".inst 0x809f00e1 // fmopa za1.s, p0/M, p0/M, z7.s, z31.s\n" - ".inst 0x809701e2 // fmopa za2.s, p0/M, p0/M, z15.s, z23.s\n" - ".inst 0x809f01e3 // fmopa za3.s, p0/M, p0/M, z15.s, z31.s\n" - ".inst 0xa1404767 // ld1w { z7.s, z15.s }, pn9.b/Z, [x27]\n" - ".inst 0x80820100 // fmopa za0.s, p0/M, p0/M, z8.s, z2.s\n" - ".inst 0xa14046bf // ldnt1w { z23.s, z31.s }, pn9.b/Z, [x21]\n" - ".inst 0x80830101 // fmopa za1.s, p0/M, p0/M, z8.s, z3.s\n" - ".inst 0x80820122 // fmopa za2.s, p0/M, p0/M, z9.s, z2.s\n" - ".inst 0x80830123 // fmopa za3.s, p0/M, p0/M, z9.s, z3.s\n" - ".inst 0xa0414768 // ld1w { z8.s-z9.s }, pn9.b/Z, [x27, #0x2, MUL VL]\n" - ".inst 0x80900240 // fmopa za0.s, p0/M, p0/M, z18.s, z16.s\n" - ".inst 0xa04146a3 // ldnt1w { z2.s-z3.s }, pn9.b/Z, [x21, #0x2, MUL VL]\n" - ".inst 0x80910241 // fmopa za1.s, p0/M, p0/M, z18.s, z17.s\n" - ".inst 0x80900342 // fmopa za2.s, p0/M, p0/M, z26.s, z16.s\n" - ".inst 0x80910343 // fmopa za3.s, p0/M, p0/M, z26.s, z17.s\n" - ".inst 0xa1424772 // ld1w { z18.s, z26.s }, pn9.b/Z, [x27, #0x4, MUL VL]\n" - ".inst 0xa04246b1 // ldnt1w { z16.s-z17.s }, pn9.b/Z, [x21, #0x4, MUL VL]\n" - ".inst 0x808402c0 // fmopa za0.s, p0/M, p0/M, z22.s, z4.s\n" - ".inst 0x808c02c1 // fmopa za1.s, p0/M, p0/M, z22.s, z12.s\n" - ".inst 0x808403c2 // fmopa za2.s, p0/M, p0/M, z30.s, z4.s\n" - ".inst 0x808c03c3 // fmopa za3.s, p0/M, p0/M, z30.s, z12.s\n" - ".inst 0xa1434776 // ld1w { z22.s, z30.s }, pn9.b/Z, [x27, #0x6, MUL VL]\n" + ".inst 0x80830241 // fmopa za1.s, p0/M, p0/M, z18.s, z3.s\n" + ".inst 0x80820262 // fmopa za2.s, p0/M, p0/M, z19.s, z2.s\n" + ".inst 0x80830263 // fmopa za3.s, p0/M, p0/M, z19.s, z3.s\n" + ".inst 0xa0404772 // ld1w { z18.s-z19.s }, pn9.b/Z, [x27]\n" + ".inst 0x809a0080 // fmopa za0.s, p0/M, p0/M, z4.s, z26.s\n" + ".inst 0xa04046a3 // ldnt1w { z2.s-z3.s }, pn9.b/Z, [x21]\n" + ".inst 0x809b0081 // fmopa za1.s, p0/M, p0/M, z4.s, z27.s\n" + ".inst 0x809a00a2 // fmopa za2.s, p0/M, p0/M, z5.s, z26.s\n" + ".inst 0x809b00a3 // fmopa za3.s, p0/M, p0/M, z5.s, z27.s\n" + ".inst 0xa0414764 // ld1w { z4.s-z5.s }, pn9.b/Z, [x27, #0x2, MUL VL]\n" + ".inst 0x80940140 // fmopa za0.s, p0/M, p0/M, z10.s, z20.s\n" + ".inst 0xa04146bb // ldnt1w { z26.s-z27.s }, pn9.b/Z, [x21, #0x2, MUL VL]\n" + ".inst 0x80950141 // fmopa za1.s, p0/M, p0/M, z10.s, z21.s\n" + ".inst 0x80940162 // fmopa za2.s, p0/M, p0/M, z11.s, z20.s\n" + ".inst 0x80950163 // fmopa za3.s, p0/M, p0/M, z11.s, z21.s\n" + ".inst 0xa042476a // ld1w { z10.s-z11.s }, pn9.b/Z, [x27, #0x4, MUL VL]\n" + ".inst 0xa04246b5 // ldnt1w { z20.s-z21.s }, pn9.b/Z, [x21, #0x4, MUL VL]\n" + ".inst 0x808800c0 // fmopa za0.s, p0/M, p0/M, z6.s, z8.s\n" + ".inst 0x808900c1 // fmopa za1.s, p0/M, p0/M, z6.s, z9.s\n" + ".inst 0x808800e2 // fmopa za2.s, p0/M, p0/M, z7.s, z8.s\n" + ".inst 0x808900e3 // fmopa za3.s, p0/M, p0/M, z7.s, z9.s\n" + ".inst 0xa0434766 // ld1w { z6.s-z7.s }, pn9.b/Z, [x27, #0x6, MUL VL]\n" "addvl x27, x27, #8\n" - ".inst 0xa14346ac // ldnt1w { z4.s, z12.s }, pn9.b/Z, [x21, #0x6, MUL VL]\n" + ".inst 0xa04346a9 // ldnt1w { z8.s-z9.s }, pn9.b/Z, [x21, #0x6, MUL VL]\n" "addvl x21, x21, #8\n" "bgt 6b\n" "7:" // K loop tail - ".inst 0x809700e0 // fmopa za0.s, p0/M, p0/M, z7.s, z23.s\n" - ".inst 0x809f00e1 // fmopa za1.s, p0/M, p0/M, z7.s, z31.s\n" - ".inst 0x809701e2 // fmopa za2.s, p0/M, p0/M, z15.s, z23.s\n" - ".inst 0x809f01e3 // fmopa za3.s, p0/M, p0/M, z15.s, z31.s\n" - ".inst 0x80820100 // fmopa za0.s, p0/M, p0/M, z8.s, z2.s\n" - ".inst 0x80830101 // fmopa za1.s, p0/M, p0/M, z8.s, z3.s\n" - ".inst 0x80820122 // fmopa za2.s, p0/M, p0/M, z9.s, z2.s\n" - ".inst 0x80830123 // fmopa za3.s, p0/M, p0/M, z9.s, z3.s\n" - ".inst 0x80900240 // fmopa za0.s, p0/M, p0/M, z18.s, z16.s\n" - ".inst 0x80910241 // fmopa za1.s, p0/M, p0/M, z18.s, z17.s\n" - ".inst 0x80900342 // fmopa za2.s, p0/M, p0/M, z26.s, z16.s\n" - ".inst 0x80910343 // fmopa za3.s, p0/M, p0/M, z26.s, z17.s\n" - ".inst 0x808402c0 // fmopa za0.s, p0/M, p0/M, z22.s, z4.s\n" - ".inst 0x808c02c1 // fmopa za1.s, p0/M, p0/M, z22.s, z12.s\n" - ".inst 0x808403c2 // fmopa za2.s, p0/M, p0/M, z30.s, z4.s\n" - ".inst 0x808c03c3 // fmopa za3.s, p0/M, p0/M, z30.s, z12.s\n" + ".inst 0x80820240 // fmopa za0.s, p0/M, p0/M, z18.s, z2.s\n" + ".inst 0x80830241 // fmopa za1.s, p0/M, p0/M, z18.s, z3.s\n" + ".inst 0x80820262 // fmopa za2.s, p0/M, p0/M, z19.s, z2.s\n" + ".inst 0x80830263 // fmopa za3.s, p0/M, p0/M, z19.s, z3.s\n" + ".inst 0x809a0080 // fmopa za0.s, p0/M, p0/M, z4.s, z26.s\n" + ".inst 0x809b0081 // fmopa za1.s, p0/M, p0/M, z4.s, z27.s\n" + ".inst 0x809a00a2 // fmopa za2.s, p0/M, p0/M, z5.s, z26.s\n" + ".inst 0x809b00a3 // fmopa za3.s, p0/M, p0/M, z5.s, z27.s\n" + ".inst 0x80940140 // fmopa za0.s, p0/M, p0/M, z10.s, z20.s\n" + ".inst 0x80950141 // fmopa za1.s, p0/M, p0/M, z10.s, z21.s\n" + ".inst 0x80940162 // fmopa za2.s, p0/M, p0/M, z11.s, z20.s\n" + ".inst 0x80950163 // fmopa za3.s, p0/M, p0/M, z11.s, z21.s\n" + ".inst 0x808800c0 // fmopa za0.s, p0/M, p0/M, z6.s, z8.s\n" + ".inst 0x808900c1 // fmopa za1.s, p0/M, p0/M, z6.s, z9.s\n" + ".inst 0x808800e2 // fmopa za2.s, p0/M, p0/M, z7.s, z8.s\n" + ".inst 0x808900e3 // fmopa za3.s, p0/M, p0/M, z7.s, z9.s\n" "8:" // K oddments "cbz x22, 10f\n" "9:" // K oddments: Loop - ".inst 0xa1404767 // ld1w { z7.s, z15.s }, pn9.b/Z, [x27]\n" + ".inst 0xa040477e // ld1w { z30.s-z31.s }, pn9.b/Z, [x27]\n" "subs x22, x22, #0x1\n" "addvl x27, x27, #2\n" - ".inst 0xa14046b7 // ld1w { z23.s, z31.s }, pn9.b/Z, [x21]\n" + ".inst 0xa14046a5 // ld1w { z5.s, z13.s }, pn9.b/Z, [x21]\n" "addvl x21, x21, #2\n" - ".inst 0x809700e0 // fmopa za0.s, p0/M, p0/M, z7.s, z23.s\n" - ".inst 0x809f00e1 // fmopa za1.s, p0/M, p0/M, z7.s, z31.s\n" - ".inst 0x809701e2 // fmopa za2.s, p0/M, p0/M, z15.s, z23.s\n" - ".inst 0x809f01e3 // fmopa za3.s, p0/M, p0/M, z15.s, z31.s\n" + ".inst 0x808503c0 // fmopa za0.s, p0/M, p0/M, z30.s, z5.s\n" + ".inst 0x808d03c1 // fmopa za1.s, p0/M, p0/M, z30.s, z13.s\n" + ".inst 0x808503e2 // fmopa za2.s, p0/M, p0/M, z31.s, z5.s\n" + ".inst 0x808d03e3 // fmopa za3.s, p0/M, p0/M, z31.s, z13.s\n" "bgt 9b\n" "10:" // K oddments: End "tbz x16, #1, 14f\n" @@ -241,24 +240,24 @@ void sme2_interleaved_nomerge_fp32_mopa_2VLx2VL(const float *const A, const floa "cntw x20\n" "11:" // Store to partial result buffer: Store and refill: Loop ".inst 0xa040c5ec // ld1w { z12.s-z15.s }, pn9.b/Z, [x15]\n" - ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" + ".inst 0xc0860400 // mova { z0.s-z3.s }, za0h.s[x12]\n" ".inst 0xc0840580 // mova za0h.s[x12], { z12.s-z15.s }\n" - ".inst 0xc086042c // mova { z12.s-z15.s }, za1h.s[x12]\n" - ".inst 0xa041c5e0 // ld1w { z0.s-z3.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n" - ".inst 0xc0840401 // mova za1h.s[x12], { z0.s-z3.s }\n" - ".inst 0xc0860454 // mova { z20.s-z23.s }, za2h.s[x12]\n" - ".inst 0xc0860468 // mova { z8.s-z11.s }, za3h.s[x12]\n" - ".inst 0xa042c5f8 // ld1w { z24.s-z27.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n" - ".inst 0xc0840702 // mova za2h.s[x12], { z24.s-z27.s }\n" - ".inst 0xa043c5f8 // ld1w { z24.s-z27.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840703 // mova za3h.s[x12], { z24.s-z27.s }\n" + ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" + ".inst 0xa041c5f0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n" + ".inst 0xc0840601 // mova za1h.s[x12], { z16.s-z19.s }\n" + ".inst 0xc086045c // mova { z28.s-z31.s }, za2h.s[x12]\n" + ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" + ".inst 0xa042c5f0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n" + ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa043c5e8 // ld1w { z8.s-z11.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840503 // mova za3h.s[x12], { z8.s-z11.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" - ".inst 0xa060c5d0 // st1w { z16.s-z19.s }, pn9.b, [x14]\n" + ".inst 0xa060c5c0 // st1w { z0.s-z3.s }, pn9.b, [x14]\n" "addvl x15, x15, #16\n" - ".inst 0xa061c5cc // st1w { z12.s-z15.s }, pn9.b, [x14, #0x4, MUL VL]\n" - ".inst 0xa062c5d4 // st1w { z20.s-z23.s }, pn9.b, [x14, #0x8, MUL VL]\n" - ".inst 0xa063c5c8 // st1w { z8.s-z11.s }, pn9.b, [x14, #0xc, MUL VL]\n" + ".inst 0xa061c5c4 // st1w { z4.s-z7.s }, pn9.b, [x14, #0x4, MUL VL]\n" + ".inst 0xa062c5dc // st1w { z28.s-z31.s }, pn9.b, [x14, #0x8, MUL VL]\n" + ".inst 0xa063c5cc // st1w { z12.s-z15.s }, pn9.b, [x14, #0xc, MUL VL]\n" "addvl x14, x14, #16\n" "blt 11b\n" "b 30f\n" @@ -266,16 +265,16 @@ void sme2_interleaved_nomerge_fp32_mopa_2VLx2VL(const float *const A, const floa "mov x12, #0x0\n" "cntw x20\n" "13:" // Store to partial result buffer: Store only: Loop - ".inst 0xc086040c // mova { z12.s-z15.s }, za0h.s[x12]\n" - ".inst 0xc0860430 // mova { z16.s-z19.s }, za1h.s[x12]\n" - ".inst 0xa060c5cc // st1w { z12.s-z15.s }, pn9.b, [x14]\n" - ".inst 0xc0860444 // mova { z4.s-z7.s }, za2h.s[x12]\n" - ".inst 0xc0860460 // mova { z0.s-z3.s }, za3h.s[x12]\n" - ".inst 0xa061c5d0 // st1w { z16.s-z19.s }, pn9.b, [x14, #0x4, MUL VL]\n" + ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" + ".inst 0xc0860420 // mova { z0.s-z3.s }, za1h.s[x12]\n" + ".inst 0xa060c5d0 // st1w { z16.s-z19.s }, pn9.b, [x14]\n" + ".inst 0xc0860458 // mova { z24.s-z27.s }, za2h.s[x12]\n" + ".inst 0xc0860464 // mova { z4.s-z7.s }, za3h.s[x12]\n" + ".inst 0xa061c5c0 // st1w { z0.s-z3.s }, pn9.b, [x14, #0x4, MUL VL]\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" - ".inst 0xa062c5c4 // st1w { z4.s-z7.s }, pn9.b, [x14, #0x8, MUL VL]\n" - ".inst 0xa063c5c0 // st1w { z0.s-z3.s }, pn9.b, [x14, #0xc, MUL VL]\n" + ".inst 0xa062c5d8 // st1w { z24.s-z27.s }, pn9.b, [x14, #0x8, MUL VL]\n" + ".inst 0xa063c5c4 // st1w { z4.s-z7.s }, pn9.b, [x14, #0xc, MUL VL]\n" "addvl x14, x14, #16\n" "blt 13b\n" "b 30f\n" @@ -310,16 +309,16 @@ void sme2_interleaved_nomerge_fp32_mopa_2VLx2VL(const float *const A, const floa "16:" // Store to output array: Skip activation: Accumulator row 0 oddments "cbz x20, 17f\n" "subs x20, x20, #0x1\n" - ".inst 0xc0860404 // mova { z4.s-z7.s }, za0h.s[x12]\n" - ".inst 0xc086042c // mova { z12.s-z15.s }, za1h.s[x12]\n" - ".inst 0xa1604344 // st1w { z4.s, z12.s }, p8, [x26]\n" + ".inst 0xc0860414 // mova { z20.s-z23.s }, za0h.s[x12]\n" + ".inst 0xc086043c // mova { z28.s-z31.s }, za1h.s[x12]\n" + ".inst 0xa1604354 // st1w { z20.s, z28.s }, p8, [x26]\n" "add x26, x26, x24\n" "beq 17f\n" "subs x20, x20, #0x1\n" - ".inst 0xa1604345 // st1w { z5.s, z13.s }, p8, [x26]\n" + ".inst 0xa1604355 // st1w { z21.s, z29.s }, p8, [x26]\n" "add x26, x26, x24\n" "beq 17f\n" - ".inst 0xa1604346 // st1w { z6.s, z14.s }, p8, [x26]\n" + ".inst 0xa1604356 // st1w { z22.s, z30.s }, p8, [x26]\n" "add x26, x26, x24\n" "17:" // Store to output array: Skip activation: Accumulator row 0 oddments: End "subs x25, x25, x22\n" @@ -347,16 +346,16 @@ void sme2_interleaved_nomerge_fp32_mopa_2VLx2VL(const float *const A, const floa "19:" // Store to output array: Skip activation: Accumulator row 1 oddments "cbz x20, 20f\n" "subs x20, x20, #0x1\n" - ".inst 0xc0860454 // mova { z20.s-z23.s }, za2h.s[x12]\n" - ".inst 0xc086047c // mova { z28.s-z31.s }, za3h.s[x12]\n" - ".inst 0xa1604354 // st1w { z20.s, z28.s }, p8, [x26]\n" + ".inst 0xc0860450 // mova { z16.s-z19.s }, za2h.s[x12]\n" + ".inst 0xc0860478 // mova { z24.s-z27.s }, za3h.s[x12]\n" + ".inst 0xa1604350 // st1w { z16.s, z24.s }, p8, [x26]\n" "add x26, x26, x24\n" "beq 20f\n" "subs x20, x20, #0x1\n" - ".inst 0xa1604355 // st1w { z21.s, z29.s }, p8, [x26]\n" + ".inst 0xa1604351 // st1w { z17.s, z25.s }, p8, [x26]\n" "add x26, x26, x24\n" "beq 20f\n" - ".inst 0xa1604356 // st1w { z22.s, z30.s }, p8, [x26]\n" + ".inst 0xa1604352 // st1w { z18.s, z26.s }, p8, [x26]\n" "add x26, x26, x24\n" "20:" // Store to output array: Skip activation: Accumulator row 1 oddments: End "subs x25, x25, x22\n" @@ -365,44 +364,44 @@ void sme2_interleaved_nomerge_fp32_mopa_2VLx2VL(const float *const A, const floa "21:" // Store to output array: Skip activation: End "cntw x23\n" "cmp x25, x23\n" - "ld1rw { z21.s }, p0/Z, [%x[args], %[offsetof_KernelArgs_min]]\n" + "ld1rw { z1.s }, p0/Z, [%x[args], %[offsetof_KernelArgs_min]]\n" "csel x22, x25, x23, LT\n" "lsr x21, x22, #0x2\n" - "ld1rw { z20.s }, p0/Z, [%x[args], %[offsetof_KernelArgs_max]]\n" + "ld1rw { z0.s }, p0/Z, [%x[args], %[offsetof_KernelArgs_max]]\n" "mov x12, #0x0\n" "and x20, x22, #0x3\n" "cbz x21, 23f\n" "22:" // Store to output array: Accumulator row 0 loop - ".inst 0xc0860404 // mova { z4.s-z7.s }, za0h.s[x12]\n" - ".inst 0xc086042c // mova { z12.s-z15.s }, za1h.s[x12]\n" - ".inst 0xc1b4caa4 // fclamp { z4.s-z7.s }, z21.s, z20.s\n" - ".inst 0xc1b4caac // fclamp { z12.s-z15.s }, z21.s, z20.s\n" - ".inst 0xa1604344 // st1w { z4.s, z12.s }, p8, [x26]\n" + ".inst 0xc0860414 // mova { z20.s-z23.s }, za0h.s[x12]\n" + ".inst 0xc086043c // mova { z28.s-z31.s }, za1h.s[x12]\n" + ".inst 0xc1a0c834 // fclamp { z20.s-z23.s }, z1.s, z0.s\n" + ".inst 0xc1a0c83c // fclamp { z28.s-z31.s }, z1.s, z0.s\n" + ".inst 0xa1604354 // st1w { z20.s, z28.s }, p8, [x26]\n" "add x26, x26, x24\n" "add x12, x12, #0x4\n" - ".inst 0xa1604345 // st1w { z5.s, z13.s }, p8, [x26]\n" + ".inst 0xa1604355 // st1w { z21.s, z29.s }, p8, [x26]\n" "add x26, x26, x24\n" "cmp x12, x21, LSL #2\n" - ".inst 0xa1604346 // st1w { z6.s, z14.s }, p8, [x26]\n" + ".inst 0xa1604356 // st1w { z22.s, z30.s }, p8, [x26]\n" "add x26, x26, x24\n" - ".inst 0xa1604347 // st1w { z7.s, z15.s }, p8, [x26]\n" + ".inst 0xa1604357 // st1w { z23.s, z31.s }, p8, [x26]\n" "add x26, x26, x24\n" "blt 22b\n" "23:" // Store to output array: Accumulator row 0 oddments "cbz x20, 24f\n" - ".inst 0xc0860400 // mova { z0.s-z3.s }, za0h.s[x12]\n" - ".inst 0xc0860428 // mova { z8.s-z11.s }, za1h.s[x12]\n" - ".inst 0xc1b4caa0 // fclamp { z0.s-z3.s }, z21.s, z20.s\n" - ".inst 0xc1b4caa8 // fclamp { z8.s-z11.s }, z21.s, z20.s\n" + ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" + ".inst 0xc0860438 // mova { z24.s-z27.s }, za1h.s[x12]\n" + ".inst 0xc1a0c830 // fclamp { z16.s-z19.s }, z1.s, z0.s\n" + ".inst 0xc1a0c838 // fclamp { z24.s-z27.s }, z1.s, z0.s\n" "subs x20, x20, #0x1\n" - ".inst 0xa1604340 // st1w { z0.s, z8.s }, p8, [x26]\n" + ".inst 0xa1604350 // st1w { z16.s, z24.s }, p8, [x26]\n" "add x26, x26, x24\n" "beq 24f\n" "subs x20, x20, #0x1\n" - ".inst 0xa1604341 // st1w { z1.s, z9.s }, p8, [x26]\n" + ".inst 0xa1604351 // st1w { z17.s, z25.s }, p8, [x26]\n" "add x26, x26, x24\n" "beq 24f\n" - ".inst 0xa1604342 // st1w { z2.s, z10.s }, p8, [x26]\n" + ".inst 0xa1604352 // st1w { z18.s, z26.s }, p8, [x26]\n" "add x26, x26, x24\n" "24:" // Store to output array: Accumulator row 0 oddments: End "subs x25, x25, x22\n" @@ -416,8 +415,8 @@ void sme2_interleaved_nomerge_fp32_mopa_2VLx2VL(const float *const A, const floa "25:" // Store to output array: Accumulator row 1 loop ".inst 0xc0860450 // mova { z16.s-z19.s }, za2h.s[x12]\n" ".inst 0xc0860478 // mova { z24.s-z27.s }, za3h.s[x12]\n" - ".inst 0xc1b4cab0 // fclamp { z16.s-z19.s }, z21.s, z20.s\n" - ".inst 0xc1b4cab8 // fclamp { z24.s-z27.s }, z21.s, z20.s\n" + ".inst 0xc1a0c830 // fclamp { z16.s-z19.s }, z1.s, z0.s\n" + ".inst 0xc1a0c838 // fclamp { z24.s-z27.s }, z1.s, z0.s\n" ".inst 0xa1604350 // st1w { z16.s, z24.s }, p8, [x26]\n" "add x26, x26, x24\n" "add x12, x12, #0x4\n" @@ -433,8 +432,8 @@ void sme2_interleaved_nomerge_fp32_mopa_2VLx2VL(const float *const A, const floa "cbz x20, 27f\n" ".inst 0xc0860450 // mova { z16.s-z19.s }, za2h.s[x12]\n" ".inst 0xc0860478 // mova { z24.s-z27.s }, za3h.s[x12]\n" - ".inst 0xc1b4cab0 // fclamp { z16.s-z19.s }, z21.s, z20.s\n" - ".inst 0xc1b4cab8 // fclamp { z24.s-z27.s }, z21.s, z20.s\n" + ".inst 0xc1a0c830 // fclamp { z16.s-z19.s }, z1.s, z0.s\n" + ".inst 0xc1a0c838 // fclamp { z24.s-z27.s }, z1.s, z0.s\n" "subs x20, x20, #0x1\n" ".inst 0xa1604350 // st1w { z16.s, z24.s }, p8, [x26]\n" "add x26, x26, x24\n" @@ -450,14 +449,14 @@ void sme2_interleaved_nomerge_fp32_mopa_2VLx2VL(const float *const A, const floa "mov x12, #0x0\n" "cntw x20\n" "29:" // Store to output array: Refill accumulators: Loop - ".inst 0xa040c5f0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x15]\n" - ".inst 0xc0840600 // mova za0h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa040c5ec // ld1w { z12.s-z15.s }, pn9.b/Z, [x15]\n" + ".inst 0xc0840580 // mova za0h.s[x12], { z12.s-z15.s }\n" ".inst 0xa041c5f0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n" ".inst 0xc0840601 // mova za1h.s[x12], { z16.s-z19.s }\n" ".inst 0xa042c5f0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n" ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" - ".inst 0xa043c5e8 // ld1w { z8.s-z11.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840503 // mova za3h.s[x12], { z8.s-z11.s }\n" + ".inst 0xa043c5ec // ld1w { z12.s-z15.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840583 // mova za3h.s[x12], { z12.s-z15.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x15, x15, #16\n" @@ -481,4 +480,3 @@ void sme2_interleaved_nomerge_fp32_mopa_2VLx2VL(const float *const A, const floa } // namespace arm_gemm #endif // ARM_COMPUTE_ENABLE_SME2 -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_4VLx1VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_4VLx1VL.hpp index a315ebb3239712f486c8139d83e513436a37f739..165e25dd8faded9a9911996d6cf71bb047dfee37 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_4VLx1VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_4VLx1VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,7 +23,7 @@ */ #pragma once -#ifdef __aarch64__ +#ifdef ARM_COMPUTE_ENABLE_SME2 #include "../std_transforms_sme.hpp" @@ -83,12 +83,11 @@ public: StdTransformsSME transforms = {}; - cls_sme2_interleaved_nomerge_fp32_mopa_4VLx1VL(const CPUInfo *ci) + cls_sme2_interleaved_nomerge_fp32_mopa_4VLx1VL(const CPUInfo *) { - ARM_COMPUTE_UNUSED(ci); } }; } // namespace arm_gemm -#endif // __aarch64__ +#endif // ARM_COMPUTE_ENABLE_SME2 diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_4VLx1VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_4VLx1VL/generic.cpp index 291a7ced5ae9cf530e8f648f0cce967d93c32d6e..ae1f81244273945eb208f50c9ee77f2882f8079e 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_4VLx1VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_fp32_mopa_4VLx1VL/generic.cpp @@ -21,7 +21,6 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifdef __ARM_FEATURE_SVE #ifdef ARM_COMPUTE_ENABLE_SME2 #include "arm_gemm.hpp" @@ -113,14 +112,14 @@ void sme2_interleaved_nomerge_fp32_mopa_4VLx1VL(const float *const A, const floa "mov x12, #0x0\n" "cntw x20\n" "1:" // Initial accumulator load from buffer: Loop - ".inst 0xa040c1e4 // ld1w { z4.s-z7.s }, pn8.b/Z, [x15]\n" - ".inst 0xc0840480 // mova za0h.s[x12], { z4.s-z7.s }\n" - ".inst 0xa041c1f8 // ld1w { z24.s-z27.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" - ".inst 0xc0840701 // mova za1h.s[x12], { z24.s-z27.s }\n" - ".inst 0xa042c1e0 // ld1w { z0.s-z3.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" - ".inst 0xc0840402 // mova za2h.s[x12], { z0.s-z3.s }\n" - ".inst 0xa043c1e4 // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840483 // mova za3h.s[x12], { z4.s-z7.s }\n" + ".inst 0xa040c1fc // ld1w { z28.s-z31.s }, pn8.b/Z, [x15]\n" + ".inst 0xc0840780 // mova za0h.s[x12], { z28.s-z31.s }\n" + ".inst 0xa041c1e0 // ld1w { z0.s-z3.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" + ".inst 0xc0840401 // mova za1h.s[x12], { z0.s-z3.s }\n" + ".inst 0xa042c1f4 // ld1w { z20.s-z23.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" + ".inst 0xc0840682 // mova za2h.s[x12], { z20.s-z23.s }\n" + ".inst 0xa043c1f4 // ld1w { z20.s-z23.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840683 // mova za3h.s[x12], { z20.s-z23.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x15, x15, #16\n" @@ -138,12 +137,12 @@ void sme2_interleaved_nomerge_fp32_mopa_4VLx1VL(const float *const A, const floa "ldr x20, [%x[args], %[offsetof_bias]]\n" ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "cbz x20, 5f\n" - "fmov z8.s, #1.0\n" - "ldnt1w { z27.s }, p0/Z, [x20, x10, LSL #2]\n" - ".inst 0x809b2500 // fmopa za0.s, p1/M, p1/M, z8.s, z27.s\n" - ".inst 0x809b2501 // fmopa za1.s, p1/M, p1/M, z8.s, z27.s\n" - ".inst 0x809b2502 // fmopa za2.s, p1/M, p1/M, z8.s, z27.s\n" - ".inst 0x809b2503 // fmopa za3.s, p1/M, p1/M, z8.s, z27.s\n" + "fmov z11.s, #1.0\n" + "ldnt1w { z13.s }, p0/Z, [x20, x10, LSL #2]\n" + ".inst 0x808d2560 // fmopa za0.s, p1/M, p1/M, z11.s, z13.s\n" + ".inst 0x808d2561 // fmopa za1.s, p1/M, p1/M, z11.s, z13.s\n" + ".inst 0x808d2562 // fmopa za2.s, p1/M, p1/M, z11.s, z13.s\n" + ".inst 0x808d2563 // fmopa za3.s, p1/M, p1/M, z11.s, z13.s\n" "4:" // Prepare accumulators: Test for last block "mov x20, x10\n" "mov x21, x11\n" @@ -164,75 +163,75 @@ void sme2_interleaved_nomerge_fp32_mopa_4VLx1VL(const float *const A, const floa "madd x21, x10, x20, x21\n" // bptr = B + n * kstride_bytes "cbz x23, 8f\n" "subs x23, x23, #0x1\n" - ".inst 0xa040c364 // ld1w { z4.s-z7.s }, pn8.b/Z, [x27]\n" - "ldnt1w { z29.s }, p1/Z, [x21]\n" - ".inst 0xa041c36c // ld1w { z12.s-z15.s }, pn8.b/Z, [x27, #0x4, MUL VL]\n" - "ldnt1w { z23.s }, p1/Z, [x21, #1, MUL VL]\n" - ".inst 0xa042c360 // ld1w { z0.s-z3.s }, pn8.b/Z, [x27, #0x8, MUL VL]\n" - "ldnt1w { z21.s }, p1/Z, [x21, #2, MUL VL]\n" - ".inst 0xa143c372 // ld1w { z18.s, z22.s, z26.s, z30.s }, pn8.b/Z, [x27, #0xc, MUL VL]\n" + ".inst 0xa140c360 // ld1w { z0.s, z4.s, z8.s, z12.s }, pn8.b/Z, [x27]\n" + "ldnt1w { z19.s }, p1/Z, [x21]\n" + ".inst 0xa141c371 // ld1w { z17.s, z21.s, z25.s, z29.s }, pn8.b/Z, [x27, #0x4, MUL VL]\n" + "ldnt1w { z22.s }, p1/Z, [x21, #1, MUL VL]\n" + ".inst 0xa142c370 // ld1w { z16.s, z20.s, z24.s, z28.s }, pn8.b/Z, [x27, #0x8, MUL VL]\n" + "ldnt1w { z23.s }, p1/Z, [x21, #2, MUL VL]\n" + ".inst 0xa143c363 // ld1w { z3.s, z7.s, z11.s, z15.s }, pn8.b/Z, [x27, #0xc, MUL VL]\n" "addvl x27, x27, #16\n" - "ldnt1w { z27.s }, p1/Z, [x21, #3, MUL VL]\n" + "ldnt1w { z2.s }, p1/Z, [x21, #3, MUL VL]\n" "addvl x21, x21, #4\n" "ble 7f\n" "6:" // K loop - ".inst 0x809d2480 // fmopa za0.s, p1/M, p1/M, z4.s, z29.s\n" + ".inst 0x80932400 // fmopa za0.s, p1/M, p1/M, z0.s, z19.s\n" "subs x23, x23, #0x1\n" - ".inst 0x809d24a1 // fmopa za1.s, p1/M, p1/M, z5.s, z29.s\n" - ".inst 0x809d24c2 // fmopa za2.s, p1/M, p1/M, z6.s, z29.s\n" - ".inst 0x809d24e3 // fmopa za3.s, p1/M, p1/M, z7.s, z29.s\n" - ".inst 0xa040c364 // ld1w { z4.s-z7.s }, pn8.b/Z, [x27]\n" - ".inst 0x80972580 // fmopa za0.s, p1/M, p1/M, z12.s, z23.s\n" - "ldnt1w { z29.s }, p1/Z, [x21]\n" - ".inst 0x809725a1 // fmopa za1.s, p1/M, p1/M, z13.s, z23.s\n" - ".inst 0x809725c2 // fmopa za2.s, p1/M, p1/M, z14.s, z23.s\n" - ".inst 0x809725e3 // fmopa za3.s, p1/M, p1/M, z15.s, z23.s\n" - ".inst 0xa041c36c // ld1w { z12.s-z15.s }, pn8.b/Z, [x27, #0x4, MUL VL]\n" - ".inst 0x80952400 // fmopa za0.s, p1/M, p1/M, z0.s, z21.s\n" - "ldnt1w { z23.s }, p1/Z, [x21, #1, MUL VL]\n" - ".inst 0x80952421 // fmopa za1.s, p1/M, p1/M, z1.s, z21.s\n" - ".inst 0x80952442 // fmopa za2.s, p1/M, p1/M, z2.s, z21.s\n" - ".inst 0x80952463 // fmopa za3.s, p1/M, p1/M, z3.s, z21.s\n" - ".inst 0xa042c360 // ld1w { z0.s-z3.s }, pn8.b/Z, [x27, #0x8, MUL VL]\n" - "ldnt1w { z21.s }, p1/Z, [x21, #2, MUL VL]\n" - ".inst 0x809b2640 // fmopa za0.s, p1/M, p1/M, z18.s, z27.s\n" - ".inst 0x809b26c1 // fmopa za1.s, p1/M, p1/M, z22.s, z27.s\n" - ".inst 0x809b2742 // fmopa za2.s, p1/M, p1/M, z26.s, z27.s\n" - ".inst 0x809b27c3 // fmopa za3.s, p1/M, p1/M, z30.s, z27.s\n" - ".inst 0xa143c372 // ld1w { z18.s, z22.s, z26.s, z30.s }, pn8.b/Z, [x27, #0xc, MUL VL]\n" + ".inst 0x80932481 // fmopa za1.s, p1/M, p1/M, z4.s, z19.s\n" + ".inst 0x80932502 // fmopa za2.s, p1/M, p1/M, z8.s, z19.s\n" + ".inst 0x80932583 // fmopa za3.s, p1/M, p1/M, z12.s, z19.s\n" + ".inst 0xa140c360 // ld1w { z0.s, z4.s, z8.s, z12.s }, pn8.b/Z, [x27]\n" + ".inst 0x80962620 // fmopa za0.s, p1/M, p1/M, z17.s, z22.s\n" + "ldnt1w { z19.s }, p1/Z, [x21]\n" + ".inst 0x809626a1 // fmopa za1.s, p1/M, p1/M, z21.s, z22.s\n" + ".inst 0x80962722 // fmopa za2.s, p1/M, p1/M, z25.s, z22.s\n" + ".inst 0x809627a3 // fmopa za3.s, p1/M, p1/M, z29.s, z22.s\n" + ".inst 0xa141c371 // ld1w { z17.s, z21.s, z25.s, z29.s }, pn8.b/Z, [x27, #0x4, MUL VL]\n" + ".inst 0x80972600 // fmopa za0.s, p1/M, p1/M, z16.s, z23.s\n" + "ldnt1w { z22.s }, p1/Z, [x21, #1, MUL VL]\n" + ".inst 0x80972681 // fmopa za1.s, p1/M, p1/M, z20.s, z23.s\n" + ".inst 0x80972702 // fmopa za2.s, p1/M, p1/M, z24.s, z23.s\n" + ".inst 0x80972783 // fmopa za3.s, p1/M, p1/M, z28.s, z23.s\n" + ".inst 0xa142c370 // ld1w { z16.s, z20.s, z24.s, z28.s }, pn8.b/Z, [x27, #0x8, MUL VL]\n" + "ldnt1w { z23.s }, p1/Z, [x21, #2, MUL VL]\n" + ".inst 0x80822460 // fmopa za0.s, p1/M, p1/M, z3.s, z2.s\n" + ".inst 0x808224e1 // fmopa za1.s, p1/M, p1/M, z7.s, z2.s\n" + ".inst 0x80822562 // fmopa za2.s, p1/M, p1/M, z11.s, z2.s\n" + ".inst 0x808225e3 // fmopa za3.s, p1/M, p1/M, z15.s, z2.s\n" + ".inst 0xa143c363 // ld1w { z3.s, z7.s, z11.s, z15.s }, pn8.b/Z, [x27, #0xc, MUL VL]\n" "addvl x27, x27, #16\n" - "ldnt1w { z27.s }, p1/Z, [x21, #3, MUL VL]\n" + "ldnt1w { z2.s }, p1/Z, [x21, #3, MUL VL]\n" "addvl x21, x21, #4\n" "bgt 6b\n" "7:" // K loop tail - ".inst 0x809d2480 // fmopa za0.s, p1/M, p1/M, z4.s, z29.s\n" - ".inst 0x809d24a1 // fmopa za1.s, p1/M, p1/M, z5.s, z29.s\n" - ".inst 0x809d24c2 // fmopa za2.s, p1/M, p1/M, z6.s, z29.s\n" - ".inst 0x809d24e3 // fmopa za3.s, p1/M, p1/M, z7.s, z29.s\n" - ".inst 0x80972580 // fmopa za0.s, p1/M, p1/M, z12.s, z23.s\n" - ".inst 0x809725a1 // fmopa za1.s, p1/M, p1/M, z13.s, z23.s\n" - ".inst 0x809725c2 // fmopa za2.s, p1/M, p1/M, z14.s, z23.s\n" - ".inst 0x809725e3 // fmopa za3.s, p1/M, p1/M, z15.s, z23.s\n" - ".inst 0x80952400 // fmopa za0.s, p1/M, p1/M, z0.s, z21.s\n" - ".inst 0x80952421 // fmopa za1.s, p1/M, p1/M, z1.s, z21.s\n" - ".inst 0x80952442 // fmopa za2.s, p1/M, p1/M, z2.s, z21.s\n" - ".inst 0x80952463 // fmopa za3.s, p1/M, p1/M, z3.s, z21.s\n" - ".inst 0x809b2640 // fmopa za0.s, p1/M, p1/M, z18.s, z27.s\n" - ".inst 0x809b26c1 // fmopa za1.s, p1/M, p1/M, z22.s, z27.s\n" - ".inst 0x809b2742 // fmopa za2.s, p1/M, p1/M, z26.s, z27.s\n" - ".inst 0x809b27c3 // fmopa za3.s, p1/M, p1/M, z30.s, z27.s\n" + ".inst 0x80932400 // fmopa za0.s, p1/M, p1/M, z0.s, z19.s\n" + ".inst 0x80932481 // fmopa za1.s, p1/M, p1/M, z4.s, z19.s\n" + ".inst 0x80932502 // fmopa za2.s, p1/M, p1/M, z8.s, z19.s\n" + ".inst 0x80932583 // fmopa za3.s, p1/M, p1/M, z12.s, z19.s\n" + ".inst 0x80962620 // fmopa za0.s, p1/M, p1/M, z17.s, z22.s\n" + ".inst 0x809626a1 // fmopa za1.s, p1/M, p1/M, z21.s, z22.s\n" + ".inst 0x80962722 // fmopa za2.s, p1/M, p1/M, z25.s, z22.s\n" + ".inst 0x809627a3 // fmopa za3.s, p1/M, p1/M, z29.s, z22.s\n" + ".inst 0x80972600 // fmopa za0.s, p1/M, p1/M, z16.s, z23.s\n" + ".inst 0x80972681 // fmopa za1.s, p1/M, p1/M, z20.s, z23.s\n" + ".inst 0x80972702 // fmopa za2.s, p1/M, p1/M, z24.s, z23.s\n" + ".inst 0x80972783 // fmopa za3.s, p1/M, p1/M, z28.s, z23.s\n" + ".inst 0x80822460 // fmopa za0.s, p1/M, p1/M, z3.s, z2.s\n" + ".inst 0x808224e1 // fmopa za1.s, p1/M, p1/M, z7.s, z2.s\n" + ".inst 0x80822562 // fmopa za2.s, p1/M, p1/M, z11.s, z2.s\n" + ".inst 0x808225e3 // fmopa za3.s, p1/M, p1/M, z15.s, z2.s\n" "8:" // K oddments "cbz x22, 10f\n" "9:" // K oddments: Loop - ".inst 0xa040c364 // ld1w { z4.s-z7.s }, pn8.b/Z, [x27]\n" + ".inst 0xa140c373 // ld1w { z19.s, z23.s, z27.s, z31.s }, pn8.b/Z, [x27]\n" "subs x22, x22, #0x1\n" "addvl x27, x27, #4\n" - "ld1w { z29.s }, p1/Z, [x21]\n" + "ld1w { z11.s }, p1/Z, [x21]\n" "addvl x21, x21, #1\n" - ".inst 0x809d2480 // fmopa za0.s, p1/M, p1/M, z4.s, z29.s\n" - ".inst 0x809d24a1 // fmopa za1.s, p1/M, p1/M, z5.s, z29.s\n" - ".inst 0x809d24c2 // fmopa za2.s, p1/M, p1/M, z6.s, z29.s\n" - ".inst 0x809d24e3 // fmopa za3.s, p1/M, p1/M, z7.s, z29.s\n" + ".inst 0x808b2660 // fmopa za0.s, p1/M, p1/M, z19.s, z11.s\n" + ".inst 0x808b26e1 // fmopa za1.s, p1/M, p1/M, z23.s, z11.s\n" + ".inst 0x808b2762 // fmopa za2.s, p1/M, p1/M, z27.s, z11.s\n" + ".inst 0x808b27e3 // fmopa za3.s, p1/M, p1/M, z31.s, z11.s\n" "bgt 9b\n" "10:" // K oddments: End "tbz x16, #1, 14f\n" @@ -240,25 +239,25 @@ void sme2_interleaved_nomerge_fp32_mopa_4VLx1VL(const float *const A, const floa "mov x12, #0x0\n" "cntw x20\n" "11:" // Store to partial result buffer: Store and refill: Loop - ".inst 0xa040c1e8 // ld1w { z8.s-z11.s }, pn8.b/Z, [x15]\n" + ".inst 0xa040c1f4 // ld1w { z20.s-z23.s }, pn8.b/Z, [x15]\n" ".inst 0xc0860418 // mova { z24.s-z27.s }, za0h.s[x12]\n" - ".inst 0xc0840500 // mova za0h.s[x12], { z8.s-z11.s }\n" + ".inst 0xc0840680 // mova za0h.s[x12], { z20.s-z23.s }\n" ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" - ".inst 0xa041c1ec // ld1w { z12.s-z15.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" - ".inst 0xc0840581 // mova za1h.s[x12], { z12.s-z15.s }\n" - ".inst 0xc086044c // mova { z12.s-z15.s }, za2h.s[x12]\n" - ".inst 0xc0860460 // mova { z0.s-z3.s }, za3h.s[x12]\n" - ".inst 0xa042c1e8 // ld1w { z8.s-z11.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" - ".inst 0xc0840502 // mova za2h.s[x12], { z8.s-z11.s }\n" - ".inst 0xa043c1fc // ld1w { z28.s-z31.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840783 // mova za3h.s[x12], { z28.s-z31.s }\n" + ".inst 0xa041c1e8 // ld1w { z8.s-z11.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" + ".inst 0xc0840501 // mova za1h.s[x12], { z8.s-z11.s }\n" + ".inst 0xc0860450 // mova { z16.s-z19.s }, za2h.s[x12]\n" + ".inst 0xc086047c // mova { z28.s-z31.s }, za3h.s[x12]\n" + ".inst 0xa042c1ec // ld1w { z12.s-z15.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" + ".inst 0xc0840582 // mova za2h.s[x12], { z12.s-z15.s }\n" + ".inst 0xa043c1ec // ld1w { z12.s-z15.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840583 // mova za3h.s[x12], { z12.s-z15.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" ".inst 0xa060c1d8 // st1w { z24.s-z27.s }, pn8.b, [x14]\n" "addvl x15, x15, #16\n" ".inst 0xa061c1c4 // st1w { z4.s-z7.s }, pn8.b, [x14, #0x4, MUL VL]\n" - ".inst 0xa062c1cc // st1w { z12.s-z15.s }, pn8.b, [x14, #0x8, MUL VL]\n" - ".inst 0xa063c1c0 // st1w { z0.s-z3.s }, pn8.b, [x14, #0xc, MUL VL]\n" + ".inst 0xa062c1d0 // st1w { z16.s-z19.s }, pn8.b, [x14, #0x8, MUL VL]\n" + ".inst 0xa063c1dc // st1w { z28.s-z31.s }, pn8.b, [x14, #0xc, MUL VL]\n" "addvl x14, x14, #16\n" "blt 11b\n" "b 42f\n" @@ -267,15 +266,15 @@ void sme2_interleaved_nomerge_fp32_mopa_4VLx1VL(const float *const A, const floa "cntw x20\n" "13:" // Store to partial result buffer: Store only: Loop ".inst 0xc086040c // mova { z12.s-z15.s }, za0h.s[x12]\n" - ".inst 0xc0860438 // mova { z24.s-z27.s }, za1h.s[x12]\n" + ".inst 0xc086043c // mova { z28.s-z31.s }, za1h.s[x12]\n" ".inst 0xa060c1cc // st1w { z12.s-z15.s }, pn8.b, [x14]\n" ".inst 0xc0860440 // mova { z0.s-z3.s }, za2h.s[x12]\n" - ".inst 0xc0860468 // mova { z8.s-z11.s }, za3h.s[x12]\n" - ".inst 0xa061c1d8 // st1w { z24.s-z27.s }, pn8.b, [x14, #0x4, MUL VL]\n" + ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" + ".inst 0xa061c1dc // st1w { z28.s-z31.s }, pn8.b, [x14, #0x4, MUL VL]\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" ".inst 0xa062c1c0 // st1w { z0.s-z3.s }, pn8.b, [x14, #0x8, MUL VL]\n" - ".inst 0xa063c1c8 // st1w { z8.s-z11.s }, pn8.b, [x14, #0xc, MUL VL]\n" + ".inst 0xa063c1cc // st1w { z12.s-z15.s }, pn8.b, [x14, #0xc, MUL VL]\n" "addvl x14, x14, #16\n" "blt 13b\n" "b 42f\n" @@ -294,16 +293,16 @@ void sme2_interleaved_nomerge_fp32_mopa_4VLx1VL(const float *const A, const floa "and x20, x22, #0x3\n" "cbz x21, 16f\n" "15:" // Store to output array: Skip activation: Accumulator row 0 loop - ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" - "st1w { z16.s }, p0, [x26]\n" + ".inst 0xc0860404 // mova { z4.s-z7.s }, za0h.s[x12]\n" + "st1w { z4.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1w { z17.s }, p0, [x26]\n" + "st1w { z5.s }, p0, [x26]\n" "add x26, x26, x24\n" "add x12, x12, #0x4\n" - "st1w { z18.s }, p0, [x26]\n" + "st1w { z6.s }, p0, [x26]\n" "add x26, x26, x24\n" "cmp x12, x21, LSL #2\n" - "st1w { z19.s }, p0, [x26]\n" + "st1w { z7.s }, p0, [x26]\n" "add x26, x26, x24\n" "blt 15b\n" "16:" // Store to output array: Skip activation: Accumulator row 0 oddments @@ -329,30 +328,30 @@ void sme2_interleaved_nomerge_fp32_mopa_4VLx1VL(const float *const A, const floa "and x20, x22, #0x3\n" "cbz x21, 19f\n" "18:" // Store to output array: Skip activation: Accumulator row 1 loop - ".inst 0xc0860430 // mova { z16.s-z19.s }, za1h.s[x12]\n" - "st1w { z16.s }, p0, [x26]\n" + ".inst 0xc0860428 // mova { z8.s-z11.s }, za1h.s[x12]\n" + "st1w { z8.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1w { z17.s }, p0, [x26]\n" + "st1w { z9.s }, p0, [x26]\n" "add x26, x26, x24\n" "add x12, x12, #0x4\n" - "st1w { z18.s }, p0, [x26]\n" + "st1w { z10.s }, p0, [x26]\n" "add x26, x26, x24\n" "cmp x12, x21, LSL #2\n" - "st1w { z19.s }, p0, [x26]\n" + "st1w { z11.s }, p0, [x26]\n" "add x26, x26, x24\n" "blt 18b\n" "19:" // Store to output array: Skip activation: Accumulator row 1 oddments "cbz x20, 20f\n" "subs x20, x20, #0x1\n" - ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" - "st1w { z4.s }, p0, [x26]\n" + ".inst 0xc0860438 // mova { z24.s-z27.s }, za1h.s[x12]\n" + "st1w { z24.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 20f\n" "subs x20, x20, #0x1\n" - "st1w { z5.s }, p0, [x26]\n" + "st1w { z25.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 20f\n" - "st1w { z6.s }, p0, [x26]\n" + "st1w { z26.s }, p0, [x26]\n" "add x26, x26, x24\n" "20:" // Store to output array: Skip activation: Accumulator row 1 oddments: End "subs x25, x25, x22\n" @@ -364,30 +363,30 @@ void sme2_interleaved_nomerge_fp32_mopa_4VLx1VL(const float *const A, const floa "and x20, x22, #0x3\n" "cbz x21, 22f\n" "21:" // Store to output array: Skip activation: Accumulator row 2 loop - ".inst 0xc0860450 // mova { z16.s-z19.s }, za2h.s[x12]\n" - "st1w { z16.s }, p0, [x26]\n" + ".inst 0xc0860444 // mova { z4.s-z7.s }, za2h.s[x12]\n" + "st1w { z4.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1w { z17.s }, p0, [x26]\n" + "st1w { z5.s }, p0, [x26]\n" "add x26, x26, x24\n" "add x12, x12, #0x4\n" - "st1w { z18.s }, p0, [x26]\n" + "st1w { z6.s }, p0, [x26]\n" "add x26, x26, x24\n" "cmp x12, x21, LSL #2\n" - "st1w { z19.s }, p0, [x26]\n" + "st1w { z7.s }, p0, [x26]\n" "add x26, x26, x24\n" "blt 21b\n" "22:" // Store to output array: Skip activation: Accumulator row 2 oddments "cbz x20, 23f\n" "subs x20, x20, #0x1\n" - ".inst 0xc0860454 // mova { z20.s-z23.s }, za2h.s[x12]\n" - "st1w { z20.s }, p0, [x26]\n" + ".inst 0xc086044c // mova { z12.s-z15.s }, za2h.s[x12]\n" + "st1w { z12.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 23f\n" "subs x20, x20, #0x1\n" - "st1w { z21.s }, p0, [x26]\n" + "st1w { z13.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 23f\n" - "st1w { z22.s }, p0, [x26]\n" + "st1w { z14.s }, p0, [x26]\n" "add x26, x26, x24\n" "23:" // Store to output array: Skip activation: Accumulator row 2 oddments: End "subs x25, x25, x22\n" @@ -399,30 +398,30 @@ void sme2_interleaved_nomerge_fp32_mopa_4VLx1VL(const float *const A, const floa "and x20, x22, #0x3\n" "cbz x21, 25f\n" "24:" // Store to output array: Skip activation: Accumulator row 3 loop - ".inst 0xc0860464 // mova { z4.s-z7.s }, za3h.s[x12]\n" - "st1w { z4.s }, p0, [x26]\n" + ".inst 0xc0860470 // mova { z16.s-z19.s }, za3h.s[x12]\n" + "st1w { z16.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1w { z5.s }, p0, [x26]\n" + "st1w { z17.s }, p0, [x26]\n" "add x26, x26, x24\n" "add x12, x12, #0x4\n" - "st1w { z6.s }, p0, [x26]\n" + "st1w { z18.s }, p0, [x26]\n" "add x26, x26, x24\n" "cmp x12, x21, LSL #2\n" - "st1w { z7.s }, p0, [x26]\n" + "st1w { z19.s }, p0, [x26]\n" "add x26, x26, x24\n" "blt 24b\n" "25:" // Store to output array: Skip activation: Accumulator row 3 oddments "cbz x20, 26f\n" "subs x20, x20, #0x1\n" - ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" - "st1w { z12.s }, p0, [x26]\n" + ".inst 0xc0860470 // mova { z16.s-z19.s }, za3h.s[x12]\n" + "st1w { z16.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 26f\n" "subs x20, x20, #0x1\n" - "st1w { z13.s }, p0, [x26]\n" + "st1w { z17.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 26f\n" - "st1w { z14.s }, p0, [x26]\n" + "st1w { z18.s }, p0, [x26]\n" "add x26, x26, x24\n" "26:" // Store to output array: Skip activation: Accumulator row 3 oddments: End "subs x25, x25, x22\n" @@ -431,40 +430,40 @@ void sme2_interleaved_nomerge_fp32_mopa_4VLx1VL(const float *const A, const floa "27:" // Store to output array: Skip activation: End "cntw x23\n" "cmp x25, x23\n" - "ld1rw { z25.s }, p1/Z, [%x[args], %[offsetof_KernelArgs_min]]\n" + "ld1rw { z21.s }, p1/Z, [%x[args], %[offsetof_KernelArgs_min]]\n" "csel x22, x25, x23, LT\n" "lsr x21, x22, #0x2\n" - "ld1rw { z24.s }, p1/Z, [%x[args], %[offsetof_KernelArgs_max]]\n" + "ld1rw { z20.s }, p1/Z, [%x[args], %[offsetof_KernelArgs_max]]\n" "mov x12, #0x0\n" "and x20, x22, #0x3\n" "cbz x21, 29f\n" "28:" // Store to output array: Accumulator row 0 loop - ".inst 0xc0860414 // mova { z20.s-z23.s }, za0h.s[x12]\n" - ".inst 0xc1b8cb34 // fclamp { z20.s-z23.s }, z25.s, z24.s\n" - "st1w { z20.s }, p0, [x26]\n" + ".inst 0xc086041c // mova { z28.s-z31.s }, za0h.s[x12]\n" + ".inst 0xc1b4cabc // fclamp { z28.s-z31.s }, z21.s, z20.s\n" + "st1w { z28.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1w { z21.s }, p0, [x26]\n" + "st1w { z29.s }, p0, [x26]\n" "add x26, x26, x24\n" "add x12, x12, #0x4\n" - "st1w { z22.s }, p0, [x26]\n" + "st1w { z30.s }, p0, [x26]\n" "add x26, x26, x24\n" "cmp x12, x21, LSL #2\n" - "st1w { z23.s }, p0, [x26]\n" + "st1w { z31.s }, p0, [x26]\n" "add x26, x26, x24\n" "blt 28b\n" "29:" // Store to output array: Accumulator row 0 oddments "cbz x20, 30f\n" - ".inst 0xc0860408 // mova { z8.s-z11.s }, za0h.s[x12]\n" + ".inst 0xc086041c // mova { z28.s-z31.s }, za0h.s[x12]\n" "subs x20, x20, #0x1\n" - ".inst 0xc1b8cb28 // fclamp { z8.s-z11.s }, z25.s, z24.s\n" - "st1w { z8.s }, p0, [x26]\n" + ".inst 0xc1b4cabc // fclamp { z28.s-z31.s }, z21.s, z20.s\n" + "st1w { z28.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 30f\n" "subs x20, x20, #0x1\n" - "st1w { z9.s }, p0, [x26]\n" + "st1w { z29.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 30f\n" - "st1w { z10.s }, p0, [x26]\n" + "st1w { z30.s }, p0, [x26]\n" "add x26, x26, x24\n" "30:" // Store to output array: Accumulator row 0 oddments: End "subs x25, x25, x22\n" @@ -476,24 +475,24 @@ void sme2_interleaved_nomerge_fp32_mopa_4VLx1VL(const float *const A, const floa "and x20, x22, #0x3\n" "cbz x21, 32f\n" "31:" // Store to output array: Accumulator row 1 loop - ".inst 0xc0860430 // mova { z16.s-z19.s }, za1h.s[x12]\n" - ".inst 0xc1b8cb30 // fclamp { z16.s-z19.s }, z25.s, z24.s\n" - "st1w { z16.s }, p0, [x26]\n" + ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" + ".inst 0xc1b4caa4 // fclamp { z4.s-z7.s }, z21.s, z20.s\n" + "st1w { z4.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1w { z17.s }, p0, [x26]\n" + "st1w { z5.s }, p0, [x26]\n" "add x26, x26, x24\n" "add x12, x12, #0x4\n" - "st1w { z18.s }, p0, [x26]\n" + "st1w { z6.s }, p0, [x26]\n" "add x26, x26, x24\n" "cmp x12, x21, LSL #2\n" - "st1w { z19.s }, p0, [x26]\n" + "st1w { z7.s }, p0, [x26]\n" "add x26, x26, x24\n" "blt 31b\n" "32:" // Store to output array: Accumulator row 1 oddments "cbz x20, 33f\n" ".inst 0xc0860430 // mova { z16.s-z19.s }, za1h.s[x12]\n" "subs x20, x20, #0x1\n" - ".inst 0xc1b8cb30 // fclamp { z16.s-z19.s }, z25.s, z24.s\n" + ".inst 0xc1b4cab0 // fclamp { z16.s-z19.s }, z21.s, z20.s\n" "st1w { z16.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 33f\n" @@ -514,7 +513,7 @@ void sme2_interleaved_nomerge_fp32_mopa_4VLx1VL(const float *const A, const floa "cbz x21, 35f\n" "34:" // Store to output array: Accumulator row 2 loop ".inst 0xc0860450 // mova { z16.s-z19.s }, za2h.s[x12]\n" - ".inst 0xc1b8cb30 // fclamp { z16.s-z19.s }, z25.s, z24.s\n" + ".inst 0xc1b4cab0 // fclamp { z16.s-z19.s }, z21.s, z20.s\n" "st1w { z16.s }, p0, [x26]\n" "add x26, x26, x24\n" "st1w { z17.s }, p0, [x26]\n" @@ -530,7 +529,7 @@ void sme2_interleaved_nomerge_fp32_mopa_4VLx1VL(const float *const A, const floa "cbz x20, 36f\n" ".inst 0xc0860450 // mova { z16.s-z19.s }, za2h.s[x12]\n" "subs x20, x20, #0x1\n" - ".inst 0xc1b8cb30 // fclamp { z16.s-z19.s }, z25.s, z24.s\n" + ".inst 0xc1b4cab0 // fclamp { z16.s-z19.s }, z21.s, z20.s\n" "st1w { z16.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 36f\n" @@ -550,24 +549,24 @@ void sme2_interleaved_nomerge_fp32_mopa_4VLx1VL(const float *const A, const floa "and x20, x20, #0x3\n" "cbz x21, 38f\n" "37:" // Store to output array: Accumulator row 3 loop - ".inst 0xc0860474 // mova { z20.s-z23.s }, za3h.s[x12]\n" - ".inst 0xc1b8cb34 // fclamp { z20.s-z23.s }, z25.s, z24.s\n" - "st1w { z20.s }, p0, [x26]\n" + ".inst 0xc0860470 // mova { z16.s-z19.s }, za3h.s[x12]\n" + ".inst 0xc1b4cab0 // fclamp { z16.s-z19.s }, z21.s, z20.s\n" + "st1w { z16.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1w { z21.s }, p0, [x26]\n" + "st1w { z17.s }, p0, [x26]\n" "add x26, x26, x24\n" "add x12, x12, #0x4\n" - "st1w { z22.s }, p0, [x26]\n" + "st1w { z18.s }, p0, [x26]\n" "add x26, x26, x24\n" "cmp x12, x21, LSL #2\n" - "st1w { z23.s }, p0, [x26]\n" + "st1w { z19.s }, p0, [x26]\n" "add x26, x26, x24\n" "blt 37b\n" "38:" // Store to output array: Accumulator row 3 oddments "cbz x20, 39f\n" ".inst 0xc0860470 // mova { z16.s-z19.s }, za3h.s[x12]\n" "subs x20, x20, #0x1\n" - ".inst 0xc1b8cb30 // fclamp { z16.s-z19.s }, z25.s, z24.s\n" + ".inst 0xc1b4cab0 // fclamp { z16.s-z19.s }, z21.s, z20.s\n" "st1w { z16.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 39f\n" @@ -586,10 +585,10 @@ void sme2_interleaved_nomerge_fp32_mopa_4VLx1VL(const float *const A, const floa ".inst 0xc0840600 // mova za0h.s[x12], { z16.s-z19.s }\n" ".inst 0xa041c1f0 // ld1w { z16.s-z19.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" ".inst 0xc0840601 // mova za1h.s[x12], { z16.s-z19.s }\n" - ".inst 0xa042c1f4 // ld1w { z20.s-z23.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" - ".inst 0xc0840682 // mova za2h.s[x12], { z20.s-z23.s }\n" - ".inst 0xa043c1e4 // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840483 // mova za3h.s[x12], { z4.s-z7.s }\n" + ".inst 0xa042c1f0 // ld1w { z16.s-z19.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" + ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa043c1e8 // ld1w { z8.s-z11.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840503 // mova za3h.s[x12], { z8.s-z11.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x15, x15, #16\n" @@ -613,4 +612,3 @@ void sme2_interleaved_nomerge_fp32_mopa_4VLx1VL(const float *const A, const floa } // namespace arm_gemm #endif // ARM_COMPUTE_ENABLE_SME2 -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_1VLx4VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_1VLx4VL.hpp index b8bcd53c2185865c47ac2e3dada4bbf44ad1ffc1..7b3cc77867e5b1f0673841ee0ac1ba401874621c 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_1VLx4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_1VLx4VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,7 +23,7 @@ */ #pragma once -#ifdef __aarch64__ +#ifdef ARM_COMPUTE_ENABLE_SME2 #include #include "../std_transforms_sme.hpp" @@ -83,12 +83,11 @@ public: StdTransformsSME transforms = {}; - cls_sme2_interleaved_nomerge_s8q_mopa_1VLx4VL(const CPUInfo *ci) + cls_sme2_interleaved_nomerge_s8q_mopa_1VLx4VL(const CPUInfo *) { - ARM_COMPUTE_UNUSED(ci); } }; } // namespace arm_gemm -#endif // __aarch64__ +#endif // ARM_COMPUTE_ENABLE_SME2 diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_1VLx4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_1VLx4VL/generic.cpp index 929af0403247c13d16153a2cbcb40d4a7188b620..aba677b1587601b6042c8da3f78410c2526655ba 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_1VLx4VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_1VLx4VL/generic.cpp @@ -21,7 +21,6 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifdef __ARM_FEATURE_SVE #ifdef ARM_COMPUTE_ENABLE_SME2 #include "arm_gemm.hpp" @@ -100,14 +99,14 @@ void sme2_interleaved_nomerge_s8q_mopa_1VLx4VL(const int8_t *const A, const int8 "mov x12, #0x0\n" "cntw x20\n" "1:" // Initial accumulator load from buffer: Loop - ".inst 0xa040c5ac // ld1w { z12.s-z15.s }, pn9.b/Z, [x13]\n" - ".inst 0xc0840580 // mova za0h.s[x12], { z12.s-z15.s }\n" - ".inst 0xa041c5bc // ld1w { z28.s-z31.s }, pn9.b/Z, [x13, #0x4, MUL VL]\n" - ".inst 0xc0840781 // mova za1h.s[x12], { z28.s-z31.s }\n" - ".inst 0xa042c5bc // ld1w { z28.s-z31.s }, pn9.b/Z, [x13, #0x8, MUL VL]\n" - ".inst 0xc0840782 // mova za2h.s[x12], { z28.s-z31.s }\n" - ".inst 0xa043c5a4 // ld1w { z4.s-z7.s }, pn9.b/Z, [x13, #0xc, MUL VL]\n" - ".inst 0xc0840483 // mova za3h.s[x12], { z4.s-z7.s }\n" + ".inst 0xa040c5a4 // ld1w { z4.s-z7.s }, pn9.b/Z, [x13]\n" + ".inst 0xc0840480 // mova za0h.s[x12], { z4.s-z7.s }\n" + ".inst 0xa041c5a8 // ld1w { z8.s-z11.s }, pn9.b/Z, [x13, #0x4, MUL VL]\n" + ".inst 0xc0840501 // mova za1h.s[x12], { z8.s-z11.s }\n" + ".inst 0xa042c5a8 // ld1w { z8.s-z11.s }, pn9.b/Z, [x13, #0x8, MUL VL]\n" + ".inst 0xc0840502 // mova za2h.s[x12], { z8.s-z11.s }\n" + ".inst 0xa043c5b0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x13, #0xc, MUL VL]\n" + ".inst 0xc0840603 // mova za3h.s[x12], { z16.s-z19.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x13, x13, #16\n" @@ -125,11 +124,11 @@ void sme2_interleaved_nomerge_s8q_mopa_1VLx4VL(const int8_t *const A, const int8 "ldr x20, [%x[args], %[offsetof_bias]]\n" ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "cbz x20, 5f\n" - ".inst 0xa01cc299 // ldnt1w { z24.s-z27.s }, p8/Z, [x20, x28, LSL #2]\n" - ".inst 0xc0902700 // addha za0.s, p1/M, p1/M, z24.s\n" - ".inst 0xc0902721 // addha za1.s, p1/M, p1/M, z25.s\n" - ".inst 0xc0902742 // addha za2.s, p1/M, p1/M, z26.s\n" - ".inst 0xc0902763 // addha za3.s, p1/M, p1/M, z27.s\n" + ".inst 0xa11cc289 // ldnt1w { z1.s, z5.s, z9.s, z13.s }, p8/Z, [x20, x28, LSL #2]\n" + ".inst 0xc0902420 // addha za0.s, p1/M, p1/M, z1.s\n" + ".inst 0xc09024a1 // addha za1.s, p1/M, p1/M, z5.s\n" + ".inst 0xc0902522 // addha za2.s, p1/M, p1/M, z9.s\n" + ".inst 0xc09025a3 // addha za3.s, p1/M, p1/M, z13.s\n" "4:" // Prepare accumulators: Test for last block "mov x20, x28\n" "mov x21, x9\n" @@ -152,107 +151,107 @@ void sme2_interleaved_nomerge_s8q_mopa_1VLx4VL(const int8_t *const A, const int8 "madd x23, x28, x20, x23\n" // bptr = B + n * kstride_bytes "cbz x22, 8f\n" "subs x22, x22, #0x1\n" - "ld1b { z10.b }, p1/Z, [x25]\n" - ".inst 0xa04086fd // ldnt1b { z28.b-z31.b }, pn9.b/Z, [x23]\n" - "ld1b { z16.b }, p1/Z, [x25, #1, MUL VL]\n" - ".inst 0xa04186ed // ldnt1b { z12.b-z15.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n" - "ld1b { z21.b }, p1/Z, [x25, #2, MUL VL]\n" - ".inst 0xa04286f9 // ldnt1b { z24.b-z27.b }, pn9.b/Z, [x23, #0x8, MUL VL]\n" - "ld1b { z19.b }, p1/Z, [x25, #3, MUL VL]\n" + "ld1b { z20.b }, p1/Z, [x25]\n" + ".inst 0xa04086e5 // ldnt1b { z4.b-z7.b }, pn9.b/Z, [x23]\n" + "ld1b { z11.b }, p1/Z, [x25, #1, MUL VL]\n" + ".inst 0xa04186f9 // ldnt1b { z24.b-z27.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n" + "ld1b { z2.b }, p1/Z, [x25, #2, MUL VL]\n" + ".inst 0xa04286fd // ldnt1b { z28.b-z31.b }, pn9.b/Z, [x23, #0x8, MUL VL]\n" + "ld1b { z14.b }, p1/Z, [x25, #3, MUL VL]\n" "addvl x25, x25, #4\n" - ".inst 0xa04386e1 // ldnt1b { z0.b-z3.b }, pn9.b/Z, [x23, #0xc, MUL VL]\n" + ".inst 0xa04386f1 // ldnt1b { z16.b-z19.b }, pn9.b/Z, [x23, #0xc, MUL VL]\n" "addvl x23, x23, #16\n" "ble 7f\n" "6:" // K loop - ".inst 0xa09c2540 // smopa za0.s, p1/M, p1/M, z10.b, z28.b\n" + ".inst 0xa0842680 // smopa za0.s, p1/M, p1/M, z20.b, z4.b\n" "subs x22, x22, #0x1\n" - ".inst 0xa09d2541 // smopa za1.s, p1/M, p1/M, z10.b, z29.b\n" - ".inst 0xa09e2542 // smopa za2.s, p1/M, p1/M, z10.b, z30.b\n" - ".inst 0xa09f2543 // smopa za3.s, p1/M, p1/M, z10.b, z31.b\n" - "ld1b { z10.b }, p1/Z, [x25]\n" - ".inst 0xa08c2600 // smopa za0.s, p1/M, p1/M, z16.b, z12.b\n" - ".inst 0xa04086fd // ldnt1b { z28.b-z31.b }, pn9.b/Z, [x23]\n" - ".inst 0xa08d2601 // smopa za1.s, p1/M, p1/M, z16.b, z13.b\n" - ".inst 0xa08e2602 // smopa za2.s, p1/M, p1/M, z16.b, z14.b\n" - ".inst 0xa08f2603 // smopa za3.s, p1/M, p1/M, z16.b, z15.b\n" - "ld1b { z16.b }, p1/Z, [x25, #1, MUL VL]\n" - ".inst 0xa09826a0 // smopa za0.s, p1/M, p1/M, z21.b, z24.b\n" - ".inst 0xa04186ed // ldnt1b { z12.b-z15.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xa09926a1 // smopa za1.s, p1/M, p1/M, z21.b, z25.b\n" - ".inst 0xa09a26a2 // smopa za2.s, p1/M, p1/M, z21.b, z26.b\n" - ".inst 0xa09b26a3 // smopa za3.s, p1/M, p1/M, z21.b, z27.b\n" - "ld1b { z21.b }, p1/Z, [x25, #2, MUL VL]\n" - ".inst 0xa04286f9 // ldnt1b { z24.b-z27.b }, pn9.b/Z, [x23, #0x8, MUL VL]\n" - ".inst 0xa0802660 // smopa za0.s, p1/M, p1/M, z19.b, z0.b\n" - ".inst 0xa0812661 // smopa za1.s, p1/M, p1/M, z19.b, z1.b\n" - ".inst 0xa0822662 // smopa za2.s, p1/M, p1/M, z19.b, z2.b\n" - ".inst 0xa0832663 // smopa za3.s, p1/M, p1/M, z19.b, z3.b\n" - "ld1b { z19.b }, p1/Z, [x25, #3, MUL VL]\n" + ".inst 0xa0852681 // smopa za1.s, p1/M, p1/M, z20.b, z5.b\n" + ".inst 0xa0862682 // smopa za2.s, p1/M, p1/M, z20.b, z6.b\n" + ".inst 0xa0872683 // smopa za3.s, p1/M, p1/M, z20.b, z7.b\n" + "ld1b { z20.b }, p1/Z, [x25]\n" + ".inst 0xa0982560 // smopa za0.s, p1/M, p1/M, z11.b, z24.b\n" + ".inst 0xa04086e5 // ldnt1b { z4.b-z7.b }, pn9.b/Z, [x23]\n" + ".inst 0xa0992561 // smopa za1.s, p1/M, p1/M, z11.b, z25.b\n" + ".inst 0xa09a2562 // smopa za2.s, p1/M, p1/M, z11.b, z26.b\n" + ".inst 0xa09b2563 // smopa za3.s, p1/M, p1/M, z11.b, z27.b\n" + "ld1b { z11.b }, p1/Z, [x25, #1, MUL VL]\n" + ".inst 0xa09c2440 // smopa za0.s, p1/M, p1/M, z2.b, z28.b\n" + ".inst 0xa04186f9 // ldnt1b { z24.b-z27.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xa09d2441 // smopa za1.s, p1/M, p1/M, z2.b, z29.b\n" + ".inst 0xa09e2442 // smopa za2.s, p1/M, p1/M, z2.b, z30.b\n" + ".inst 0xa09f2443 // smopa za3.s, p1/M, p1/M, z2.b, z31.b\n" + "ld1b { z2.b }, p1/Z, [x25, #2, MUL VL]\n" + ".inst 0xa04286fd // ldnt1b { z28.b-z31.b }, pn9.b/Z, [x23, #0x8, MUL VL]\n" + ".inst 0xa09025c0 // smopa za0.s, p1/M, p1/M, z14.b, z16.b\n" + ".inst 0xa09125c1 // smopa za1.s, p1/M, p1/M, z14.b, z17.b\n" + ".inst 0xa09225c2 // smopa za2.s, p1/M, p1/M, z14.b, z18.b\n" + ".inst 0xa09325c3 // smopa za3.s, p1/M, p1/M, z14.b, z19.b\n" + "ld1b { z14.b }, p1/Z, [x25, #3, MUL VL]\n" "addvl x25, x25, #4\n" - ".inst 0xa04386e1 // ldnt1b { z0.b-z3.b }, pn9.b/Z, [x23, #0xc, MUL VL]\n" + ".inst 0xa04386f1 // ldnt1b { z16.b-z19.b }, pn9.b/Z, [x23, #0xc, MUL VL]\n" "addvl x23, x23, #16\n" "bgt 6b\n" "7:" // K loop tail - ".inst 0xa09c2540 // smopa za0.s, p1/M, p1/M, z10.b, z28.b\n" - ".inst 0xa09d2541 // smopa za1.s, p1/M, p1/M, z10.b, z29.b\n" - ".inst 0xa09e2542 // smopa za2.s, p1/M, p1/M, z10.b, z30.b\n" - ".inst 0xa09f2543 // smopa za3.s, p1/M, p1/M, z10.b, z31.b\n" - ".inst 0xa08c2600 // smopa za0.s, p1/M, p1/M, z16.b, z12.b\n" - ".inst 0xa08d2601 // smopa za1.s, p1/M, p1/M, z16.b, z13.b\n" - ".inst 0xa08e2602 // smopa za2.s, p1/M, p1/M, z16.b, z14.b\n" - ".inst 0xa08f2603 // smopa za3.s, p1/M, p1/M, z16.b, z15.b\n" - ".inst 0xa09826a0 // smopa za0.s, p1/M, p1/M, z21.b, z24.b\n" - ".inst 0xa09926a1 // smopa za1.s, p1/M, p1/M, z21.b, z25.b\n" - ".inst 0xa09a26a2 // smopa za2.s, p1/M, p1/M, z21.b, z26.b\n" - ".inst 0xa09b26a3 // smopa za3.s, p1/M, p1/M, z21.b, z27.b\n" - ".inst 0xa0802660 // smopa za0.s, p1/M, p1/M, z19.b, z0.b\n" - ".inst 0xa0812661 // smopa za1.s, p1/M, p1/M, z19.b, z1.b\n" - ".inst 0xa0822662 // smopa za2.s, p1/M, p1/M, z19.b, z2.b\n" - ".inst 0xa0832663 // smopa za3.s, p1/M, p1/M, z19.b, z3.b\n" + ".inst 0xa0842680 // smopa za0.s, p1/M, p1/M, z20.b, z4.b\n" + ".inst 0xa0852681 // smopa za1.s, p1/M, p1/M, z20.b, z5.b\n" + ".inst 0xa0862682 // smopa za2.s, p1/M, p1/M, z20.b, z6.b\n" + ".inst 0xa0872683 // smopa za3.s, p1/M, p1/M, z20.b, z7.b\n" + ".inst 0xa0982560 // smopa za0.s, p1/M, p1/M, z11.b, z24.b\n" + ".inst 0xa0992561 // smopa za1.s, p1/M, p1/M, z11.b, z25.b\n" + ".inst 0xa09a2562 // smopa za2.s, p1/M, p1/M, z11.b, z26.b\n" + ".inst 0xa09b2563 // smopa za3.s, p1/M, p1/M, z11.b, z27.b\n" + ".inst 0xa09c2440 // smopa za0.s, p1/M, p1/M, z2.b, z28.b\n" + ".inst 0xa09d2441 // smopa za1.s, p1/M, p1/M, z2.b, z29.b\n" + ".inst 0xa09e2442 // smopa za2.s, p1/M, p1/M, z2.b, z30.b\n" + ".inst 0xa09f2443 // smopa za3.s, p1/M, p1/M, z2.b, z31.b\n" + ".inst 0xa09025c0 // smopa za0.s, p1/M, p1/M, z14.b, z16.b\n" + ".inst 0xa09125c1 // smopa za1.s, p1/M, p1/M, z14.b, z17.b\n" + ".inst 0xa09225c2 // smopa za2.s, p1/M, p1/M, z14.b, z18.b\n" + ".inst 0xa09325c3 // smopa za3.s, p1/M, p1/M, z14.b, z19.b\n" "8:" // K oddments "cbz x21, 10f\n" "9:" // K oddments: Loop - "ld1b { z10.b }, p1/Z, [x25]\n" + "ld1b { z16.b }, p1/Z, [x25]\n" "subs x21, x21, #0x1\n" "addvl x25, x25, #1\n" - ".inst 0xa04086fc // ld1b { z28.b-z31.b }, pn9.b/Z, [x23]\n" + ".inst 0xa04086e4 // ld1b { z4.b-z7.b }, pn9.b/Z, [x23]\n" "addvl x23, x23, #4\n" - ".inst 0xa09c2540 // smopa za0.s, p1/M, p1/M, z10.b, z28.b\n" - ".inst 0xa09d2541 // smopa za1.s, p1/M, p1/M, z10.b, z29.b\n" - ".inst 0xa09e2542 // smopa za2.s, p1/M, p1/M, z10.b, z30.b\n" - ".inst 0xa09f2543 // smopa za3.s, p1/M, p1/M, z10.b, z31.b\n" + ".inst 0xa0842600 // smopa za0.s, p1/M, p1/M, z16.b, z4.b\n" + ".inst 0xa0852601 // smopa za1.s, p1/M, p1/M, z16.b, z5.b\n" + ".inst 0xa0862602 // smopa za2.s, p1/M, p1/M, z16.b, z6.b\n" + ".inst 0xa0872603 // smopa za3.s, p1/M, p1/M, z16.b, z7.b\n" "bgt 9b\n" "10:" // K oddments: End - "ld1w { z14.s }, p1/Z, [x25]\n" + "ld1w { z15.s }, p1/Z, [x25]\n" "addvl x25, x25, #1\n" - ".inst 0xc09125c0 // addva za0.s, p1/M, p1/M, z14.s\n" - ".inst 0xc09125c1 // addva za1.s, p1/M, p1/M, z14.s\n" - ".inst 0xc09125c2 // addva za2.s, p1/M, p1/M, z14.s\n" - ".inst 0xc09125c3 // addva za3.s, p1/M, p1/M, z14.s\n" + ".inst 0xc09125e0 // addva za0.s, p1/M, p1/M, z15.s\n" + ".inst 0xc09125e1 // addva za1.s, p1/M, p1/M, z15.s\n" + ".inst 0xc09125e2 // addva za2.s, p1/M, p1/M, z15.s\n" + ".inst 0xc09125e3 // addva za3.s, p1/M, p1/M, z15.s\n" "tbz x14, #1, 14f\n" "tbz x14, #0, 12f\n" "mov x12, #0x0\n" "cntw x20\n" "11:" // Store to partial result buffer: Store and refill: Loop - ".inst 0xa040c5b8 // ld1w { z24.s-z27.s }, pn9.b/Z, [x13]\n" - ".inst 0xc086041c // mova { z28.s-z31.s }, za0h.s[x12]\n" - ".inst 0xc0840700 // mova za0h.s[x12], { z24.s-z27.s }\n" - ".inst 0xc0860428 // mova { z8.s-z11.s }, za1h.s[x12]\n" - ".inst 0xa041c5b0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x13, #0x4, MUL VL]\n" - ".inst 0xc0840601 // mova za1h.s[x12], { z16.s-z19.s }\n" - ".inst 0xc0860458 // mova { z24.s-z27.s }, za2h.s[x12]\n" - ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" - ".inst 0xa042c5a4 // ld1w { z4.s-z7.s }, pn9.b/Z, [x13, #0x8, MUL VL]\n" - ".inst 0xc0840482 // mova za2h.s[x12], { z4.s-z7.s }\n" - ".inst 0xa043c5a4 // ld1w { z4.s-z7.s }, pn9.b/Z, [x13, #0xc, MUL VL]\n" - ".inst 0xc0840483 // mova za3h.s[x12], { z4.s-z7.s }\n" + ".inst 0xa040c5a0 // ld1w { z0.s-z3.s }, pn9.b/Z, [x13]\n" + ".inst 0xc0860418 // mova { z24.s-z27.s }, za0h.s[x12]\n" + ".inst 0xc0840400 // mova za0h.s[x12], { z0.s-z3.s }\n" + ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" + ".inst 0xa041c5a0 // ld1w { z0.s-z3.s }, pn9.b/Z, [x13, #0x4, MUL VL]\n" + ".inst 0xc0840401 // mova za1h.s[x12], { z0.s-z3.s }\n" + ".inst 0xc0860454 // mova { z20.s-z23.s }, za2h.s[x12]\n" + ".inst 0xc0860460 // mova { z0.s-z3.s }, za3h.s[x12]\n" + ".inst 0xa042c5b0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x13, #0x8, MUL VL]\n" + ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa043c5b0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x13, #0xc, MUL VL]\n" + ".inst 0xc0840603 // mova za3h.s[x12], { z16.s-z19.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" - ".inst 0xa060c57c // st1w { z28.s-z31.s }, pn9.b, [x11]\n" + ".inst 0xa060c578 // st1w { z24.s-z27.s }, pn9.b, [x11]\n" "addvl x13, x13, #16\n" - ".inst 0xa061c568 // st1w { z8.s-z11.s }, pn9.b, [x11, #0x4, MUL VL]\n" - ".inst 0xa062c578 // st1w { z24.s-z27.s }, pn9.b, [x11, #0x8, MUL VL]\n" - ".inst 0xa063c56c // st1w { z12.s-z15.s }, pn9.b, [x11, #0xc, MUL VL]\n" + ".inst 0xa061c564 // st1w { z4.s-z7.s }, pn9.b, [x11, #0x4, MUL VL]\n" + ".inst 0xa062c574 // st1w { z20.s-z23.s }, pn9.b, [x11, #0x8, MUL VL]\n" + ".inst 0xa063c560 // st1w { z0.s-z3.s }, pn9.b, [x11, #0xc, MUL VL]\n" "addvl x11, x11, #16\n" "blt 11b\n" "b 21f\n" @@ -260,16 +259,16 @@ void sme2_interleaved_nomerge_s8q_mopa_1VLx4VL(const int8_t *const A, const int8 "mov x12, #0x0\n" "cntw x20\n" "13:" // Store to partial result buffer: Store only: Loop - ".inst 0xc086041c // mova { z28.s-z31.s }, za0h.s[x12]\n" - ".inst 0xc0860420 // mova { z0.s-z3.s }, za1h.s[x12]\n" - ".inst 0xa060c57c // st1w { z28.s-z31.s }, pn9.b, [x11]\n" - ".inst 0xc0860448 // mova { z8.s-z11.s }, za2h.s[x12]\n" - ".inst 0xc0860470 // mova { z16.s-z19.s }, za3h.s[x12]\n" - ".inst 0xa061c560 // st1w { z0.s-z3.s }, pn9.b, [x11, #0x4, MUL VL]\n" + ".inst 0xc0860404 // mova { z4.s-z7.s }, za0h.s[x12]\n" + ".inst 0xc0860434 // mova { z20.s-z23.s }, za1h.s[x12]\n" + ".inst 0xa060c564 // st1w { z4.s-z7.s }, pn9.b, [x11]\n" + ".inst 0xc086044c // mova { z12.s-z15.s }, za2h.s[x12]\n" + ".inst 0xc0860468 // mova { z8.s-z11.s }, za3h.s[x12]\n" + ".inst 0xa061c574 // st1w { z20.s-z23.s }, pn9.b, [x11, #0x4, MUL VL]\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" - ".inst 0xa062c568 // st1w { z8.s-z11.s }, pn9.b, [x11, #0x8, MUL VL]\n" - ".inst 0xa063c570 // st1w { z16.s-z19.s }, pn9.b, [x11, #0xc, MUL VL]\n" + ".inst 0xa062c56c // st1w { z12.s-z15.s }, pn9.b, [x11, #0x8, MUL VL]\n" + ".inst 0xa063c568 // st1w { z8.s-z11.s }, pn9.b, [x11, #0xc, MUL VL]\n" "addvl x11, x11, #16\n" "blt 13b\n" "b 21f\n" @@ -277,17 +276,17 @@ void sme2_interleaved_nomerge_s8q_mopa_1VLx4VL(const int8_t *const A, const int8 "ldr x24, [%x[args], %[offsetof_C]]\n" "add x24, x24, x28\n" // C += n "sub x23, x10, x9\n" - "ld1rw { z12.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z4.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" "ldr x22, [%x[args], %[offsetof_ldcb]]\n" "madd x24, x9, x22, x24\n" // C += m * ldc - "ld1rw { z13.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" - "ld1rw { z14.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" - "ld1rw { z15.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" - "ld1rw { z4.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" - "ld1rw { z5.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" - "ld1rw { z6.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" - "ld1rw { z7.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" - "ld1rw { z1.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_c_offset]]\n" + "ld1rw { z5.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z6.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z7.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z12.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z13.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z14.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z15.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z0.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_c_offset]]\n" "ld1rw { z21.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_minval]]\n" "ld1rw { z20.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_maxval]]\n" "tbz x14, #2, 15f\n" @@ -295,10 +294,10 @@ void sme2_interleaved_nomerge_s8q_mopa_1VLx4VL(const int8_t *const A, const int8 "add x21, x21, x28\n" "ldr x20, [%x[rq], %[offsetof_Requantize32_per_channel_muls]]\n" "add x20, x20, x21, LSL #2\n" - ".inst 0xa040c28c // ld1w { z12.s-z15.s }, p8/Z, [x20]\n" + ".inst 0xa040c284 // ld1w { z4.s-z7.s }, p8/Z, [x20]\n" "ldr x20, [%x[rq], %[offsetof_Requantize32_per_channel_right_shifts]]\n" "add x20, x20, x21, LSL #2\n" - ".inst 0xa040c284 // ld1w { z4.s-z7.s }, p8/Z, [x20]\n" + ".inst 0xa040c28c // ld1w { z12.s-z15.s }, p8/Z, [x20]\n" "15:" // Store to output array: Load per-channel parameters: End "cntw x20\n" "whilelt p0.b, x28, x27\n" @@ -311,22 +310,22 @@ void sme2_interleaved_nomerge_s8q_mopa_1VLx4VL(const int8_t *const A, const int8 "16:" // Store to output array: Accumulator row 0 loop ".inst 0xc086001a // mova { z26.s-z27.s }, za0h.s[x12, 0:1]\n" ".inst 0xc086005c // mova { z28.s-z29.s }, za1h.s[x12, 0:1]\n" - ".inst 0xc1aca41a // sqdmulh { z26.s-z27.s }, { z26.s-z27.s }, z12.s\n" + ".inst 0xc1a4a41a // sqdmulh { z26.s-z27.s }, { z26.s-z27.s }, z4.s\n" ".inst 0xc0860096 // mova { z22.s-z23.s }, za2h.s[x12, 0:1]\n" ".inst 0xc08600d0 // mova { z16.s-z17.s }, za3h.s[x12, 0:1]\n" - ".inst 0xc1ada41c // sqdmulh { z28.s-z29.s }, { z28.s-z29.s }, z13.s\n" - ".inst 0xc1aea416 // sqdmulh { z22.s-z23.s }, { z22.s-z23.s }, z14.s\n" + ".inst 0xc1a5a41c // sqdmulh { z28.s-z29.s }, { z28.s-z29.s }, z5.s\n" + ".inst 0xc1a6a416 // sqdmulh { z22.s-z23.s }, { z22.s-z23.s }, z6.s\n" "add x12, x12, #0x2\n" "cmp x12, x21, LSL #1\n" - ".inst 0xc1afa410 // sqdmulh { z16.s-z17.s }, { z16.s-z17.s }, z15.s\n" - ".inst 0xc1a4a23a // srshl { z26.s-z27.s }, { z26.s-z27.s }, z4.s\n" - ".inst 0xc1a5a23c // srshl { z28.s-z29.s }, { z28.s-z29.s }, z5.s\n" - ".inst 0xc1a6a236 // srshl { z22.s-z23.s }, { z22.s-z23.s }, z6.s\n" - ".inst 0xc1a7a230 // srshl { z16.s-z17.s }, { z16.s-z17.s }, z7.s\n" - ".inst 0xc1a1a31a // add { z26.s-z27.s }, { z26.s-z27.s }, z1.s\n" - ".inst 0xc1a1a31c // add { z28.s-z29.s }, { z28.s-z29.s }, z1.s\n" - ".inst 0xc1a1a316 // add { z22.s-z23.s }, { z22.s-z23.s }, z1.s\n" - ".inst 0xc1a1a310 // add { z16.s-z17.s }, { z16.s-z17.s }, z1.s\n" + ".inst 0xc1a7a410 // sqdmulh { z16.s-z17.s }, { z16.s-z17.s }, z7.s\n" + ".inst 0xc1aca23a // srshl { z26.s-z27.s }, { z26.s-z27.s }, z12.s\n" + ".inst 0xc1ada23c // srshl { z28.s-z29.s }, { z28.s-z29.s }, z13.s\n" + ".inst 0xc1aea236 // srshl { z22.s-z23.s }, { z22.s-z23.s }, z14.s\n" + ".inst 0xc1afa230 // srshl { z16.s-z17.s }, { z16.s-z17.s }, z15.s\n" + ".inst 0xc1a0a31a // add { z26.s-z27.s }, { z26.s-z27.s }, z0.s\n" + ".inst 0xc1a0a31c // add { z28.s-z29.s }, { z28.s-z29.s }, z0.s\n" + ".inst 0xc1a0a316 // add { z22.s-z23.s }, { z22.s-z23.s }, z0.s\n" + ".inst 0xc1a0a310 // add { z16.s-z17.s }, { z16.s-z17.s }, z0.s\n" ".inst 0xc1b4c6ba // sclamp { z26.s-z27.s }, z21.s, z20.s\n" ".inst 0xc1b4c6bc // sclamp { z28.s-z29.s }, z21.s, z20.s\n" "uzp1 z19.b, z26.b, z28.b\n" @@ -344,29 +343,29 @@ void sme2_interleaved_nomerge_s8q_mopa_1VLx4VL(const int8_t *const A, const int8 "blt 16b\n" "17:" // Store to output array: Accumulator row 0 oddments "cbz x20, 18f\n" - ".inst 0xc0860002 // mova { z2.s-z3.s }, za0h.s[x12, 0:1]\n" + ".inst 0xc086000a // mova { z10.s-z11.s }, za0h.s[x12, 0:1]\n" ".inst 0xc0860058 // mova { z24.s-z25.s }, za1h.s[x12, 0:1]\n" - ".inst 0xc1aca402 // sqdmulh { z2.s-z3.s }, { z2.s-z3.s }, z12.s\n" - ".inst 0xc0860090 // mova { z16.s-z17.s }, za2h.s[x12, 0:1]\n" - ".inst 0xc08600ca // mova { z10.s-z11.s }, za3h.s[x12, 0:1]\n" - ".inst 0xc1ada418 // sqdmulh { z24.s-z25.s }, { z24.s-z25.s }, z13.s\n" - ".inst 0xc1aea410 // sqdmulh { z16.s-z17.s }, { z16.s-z17.s }, z14.s\n" - ".inst 0xc1afa40a // sqdmulh { z10.s-z11.s }, { z10.s-z11.s }, z15.s\n" - ".inst 0xc1a4a222 // srshl { z2.s-z3.s }, { z2.s-z3.s }, z4.s\n" - ".inst 0xc1a5a238 // srshl { z24.s-z25.s }, { z24.s-z25.s }, z5.s\n" - ".inst 0xc1a6a230 // srshl { z16.s-z17.s }, { z16.s-z17.s }, z6.s\n" - ".inst 0xc1a7a22a // srshl { z10.s-z11.s }, { z10.s-z11.s }, z7.s\n" - ".inst 0xc1a1a302 // add { z2.s-z3.s }, { z2.s-z3.s }, z1.s\n" - ".inst 0xc1a1a318 // add { z24.s-z25.s }, { z24.s-z25.s }, z1.s\n" - ".inst 0xc1a1a310 // add { z16.s-z17.s }, { z16.s-z17.s }, z1.s\n" - ".inst 0xc1a1a30a // add { z10.s-z11.s }, { z10.s-z11.s }, z1.s\n" - ".inst 0xc1b4c6a2 // sclamp { z2.s-z3.s }, z21.s, z20.s\n" - ".inst 0xc1b4c6b8 // sclamp { z24.s-z25.s }, z21.s, z20.s\n" - "uzp1 z23.b, z2.b, z24.b\n" - ".inst 0xc1b4c6b0 // sclamp { z16.s-z17.s }, z21.s, z20.s\n" + ".inst 0xc1a4a40a // sqdmulh { z10.s-z11.s }, { z10.s-z11.s }, z4.s\n" + ".inst 0xc086009a // mova { z26.s-z27.s }, za2h.s[x12, 0:1]\n" + ".inst 0xc08600de // mova { z30.s-z31.s }, za3h.s[x12, 0:1]\n" + ".inst 0xc1a5a418 // sqdmulh { z24.s-z25.s }, { z24.s-z25.s }, z5.s\n" + ".inst 0xc1a6a41a // sqdmulh { z26.s-z27.s }, { z26.s-z27.s }, z6.s\n" + ".inst 0xc1a7a41e // sqdmulh { z30.s-z31.s }, { z30.s-z31.s }, z7.s\n" + ".inst 0xc1aca22a // srshl { z10.s-z11.s }, { z10.s-z11.s }, z12.s\n" + ".inst 0xc1ada238 // srshl { z24.s-z25.s }, { z24.s-z25.s }, z13.s\n" + ".inst 0xc1aea23a // srshl { z26.s-z27.s }, { z26.s-z27.s }, z14.s\n" + ".inst 0xc1afa23e // srshl { z30.s-z31.s }, { z30.s-z31.s }, z15.s\n" + ".inst 0xc1a0a30a // add { z10.s-z11.s }, { z10.s-z11.s }, z0.s\n" + ".inst 0xc1a0a318 // add { z24.s-z25.s }, { z24.s-z25.s }, z0.s\n" + ".inst 0xc1a0a31a // add { z26.s-z27.s }, { z26.s-z27.s }, z0.s\n" + ".inst 0xc1a0a31e // add { z30.s-z31.s }, { z30.s-z31.s }, z0.s\n" ".inst 0xc1b4c6aa // sclamp { z10.s-z11.s }, z21.s, z20.s\n" - "uzp1 z16.b, z16.b, z10.b\n" - "uzp1 z16.b, z23.b, z16.b\n" + ".inst 0xc1b4c6b8 // sclamp { z24.s-z25.s }, z21.s, z20.s\n" + "uzp1 z17.b, z10.b, z24.b\n" + ".inst 0xc1b4c6ba // sclamp { z26.s-z27.s }, z21.s, z20.s\n" + ".inst 0xc1b4c6be // sclamp { z30.s-z31.s }, z21.s, z20.s\n" + "uzp1 z16.b, z26.b, z30.b\n" + "uzp1 z16.b, z17.b, z16.b\n" "st1b { z16.b }, p0, [x24]\n" "18:" // Store to output array: Accumulator row 0 oddments: End "19:" // Store to output array: End @@ -374,14 +373,14 @@ void sme2_interleaved_nomerge_s8q_mopa_1VLx4VL(const int8_t *const A, const int8 "mov x12, #0x0\n" "cntw x20\n" "20:" // Store to output array: Refill accumulators: Loop - ".inst 0xa040c5b0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x13]\n" - ".inst 0xc0840600 // mova za0h.s[x12], { z16.s-z19.s }\n" - ".inst 0xa041c5ac // ld1w { z12.s-z15.s }, pn9.b/Z, [x13, #0x4, MUL VL]\n" - ".inst 0xc0840581 // mova za1h.s[x12], { z12.s-z15.s }\n" - ".inst 0xa042c5b0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x13, #0x8, MUL VL]\n" - ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" - ".inst 0xa043c5ac // ld1w { z12.s-z15.s }, pn9.b/Z, [x13, #0xc, MUL VL]\n" - ".inst 0xc0840583 // mova za3h.s[x12], { z12.s-z15.s }\n" + ".inst 0xa040c5bc // ld1w { z28.s-z31.s }, pn9.b/Z, [x13]\n" + ".inst 0xc0840780 // mova za0h.s[x12], { z28.s-z31.s }\n" + ".inst 0xa041c5b0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x13, #0x4, MUL VL]\n" + ".inst 0xc0840601 // mova za1h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa042c5ac // ld1w { z12.s-z15.s }, pn9.b/Z, [x13, #0x8, MUL VL]\n" + ".inst 0xc0840582 // mova za2h.s[x12], { z12.s-z15.s }\n" + ".inst 0xa043c5a0 // ld1w { z0.s-z3.s }, pn9.b/Z, [x13, #0xc, MUL VL]\n" + ".inst 0xc0840403 // mova za3h.s[x12], { z0.s-z3.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x13, x13, #16\n" @@ -405,4 +404,3 @@ void sme2_interleaved_nomerge_s8q_mopa_1VLx4VL(const int8_t *const A, const int8 } // namespace arm_gemm #endif // ARM_COMPUTE_ENABLE_SME2 -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_2VLx2VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_2VLx2VL.hpp index 954b0da0e1b2c1108a862dc34a4c52c2c90ba778..79990f72e53c47273717ae93b0695ab72604e91d 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_2VLx2VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_2VLx2VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,7 +23,7 @@ */ #pragma once -#ifdef __aarch64__ +#ifdef ARM_COMPUTE_ENABLE_SME2 #include #include "../std_transforms_sme.hpp" @@ -83,12 +83,11 @@ public: StdTransformsSME transforms = {}; - cls_sme2_interleaved_nomerge_s8q_mopa_2VLx2VL(const CPUInfo *ci) + cls_sme2_interleaved_nomerge_s8q_mopa_2VLx2VL(const CPUInfo *) { - ARM_COMPUTE_UNUSED(ci); } }; } // namespace arm_gemm -#endif // __aarch64__ +#endif // ARM_COMPUTE_ENABLE_SME2 diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_2VLx2VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_2VLx2VL/generic.cpp index 0b642818e2a14b92c385f2211bdf5c67f23921af..7033de5fe3275d58a5314bb09024c807b21841f1 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_2VLx2VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_2VLx2VL/generic.cpp @@ -21,7 +21,6 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifdef __ARM_FEATURE_SVE #ifdef ARM_COMPUTE_ENABLE_SME2 #include "arm_gemm.hpp" @@ -100,14 +99,14 @@ void sme2_interleaved_nomerge_s8q_mopa_2VLx2VL(const int8_t *const A, const int8 "mov x12, #0x0\n" "cntw x20\n" "1:" // Initial accumulator load from buffer: Loop - ".inst 0xa040c5e0 // ld1w { z0.s-z3.s }, pn9.b/Z, [x15]\n" - ".inst 0xc0840400 // mova za0h.s[x12], { z0.s-z3.s }\n" - ".inst 0xa041c5ec // ld1w { z12.s-z15.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n" - ".inst 0xc0840581 // mova za1h.s[x12], { z12.s-z15.s }\n" - ".inst 0xa042c5e0 // ld1w { z0.s-z3.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n" - ".inst 0xc0840402 // mova za2h.s[x12], { z0.s-z3.s }\n" - ".inst 0xa043c5fc // ld1w { z28.s-z31.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840783 // mova za3h.s[x12], { z28.s-z31.s }\n" + ".inst 0xa040c5e8 // ld1w { z8.s-z11.s }, pn9.b/Z, [x15]\n" + ".inst 0xc0840500 // mova za0h.s[x12], { z8.s-z11.s }\n" + ".inst 0xa041c5e0 // ld1w { z0.s-z3.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n" + ".inst 0xc0840401 // mova za1h.s[x12], { z0.s-z3.s }\n" + ".inst 0xa042c5f0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n" + ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa043c5f0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840603 // mova za3h.s[x12], { z16.s-z19.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x15, x15, #16\n" @@ -125,11 +124,11 @@ void sme2_interleaved_nomerge_s8q_mopa_2VLx2VL(const int8_t *const A, const int8 "ldr x20, [%x[args], %[offsetof_bias]]\n" ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "cbz x20, 5f\n" - ".inst 0xa00a4295 // ldnt1w { z20.s-z21.s }, p8/Z, [x20, x10, LSL #2]\n" - ".inst 0xc0902680 // addha za0.s, p1/M, p1/M, z20.s\n" - ".inst 0xc09026a1 // addha za1.s, p1/M, p1/M, z21.s\n" - ".inst 0xc0902682 // addha za2.s, p1/M, p1/M, z20.s\n" - ".inst 0xc09026a3 // addha za3.s, p1/M, p1/M, z21.s\n" + ".inst 0xa00a4299 // ldnt1w { z24.s-z25.s }, p8/Z, [x20, x10, LSL #2]\n" + ".inst 0xc0902700 // addha za0.s, p1/M, p1/M, z24.s\n" + ".inst 0xc0902721 // addha za1.s, p1/M, p1/M, z25.s\n" + ".inst 0xc0902702 // addha za2.s, p1/M, p1/M, z24.s\n" + ".inst 0xc0902723 // addha za3.s, p1/M, p1/M, z25.s\n" "4:" // Prepare accumulators: Test for last block "mov x20, x10\n" "mov x21, x11\n" @@ -152,75 +151,75 @@ void sme2_interleaved_nomerge_s8q_mopa_2VLx2VL(const int8_t *const A, const int8 "madd x23, x10, x20, x23\n" // bptr = B + n * kstride_bytes "cbz x22, 8f\n" "subs x22, x22, #0x1\n" - ".inst 0xa040077e // ld1b { z30.b-z31.b }, pn9.b/Z, [x27]\n" - ".inst 0xa04006f1 // ldnt1b { z16.b-z17.b }, pn9.b/Z, [x23]\n" - ".inst 0xa041076e // ld1b { z14.b-z15.b }, pn9.b/Z, [x27, #0x2, MUL VL]\n" - ".inst 0xa04106e9 // ldnt1b { z8.b-z9.b }, pn9.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0xa0420760 // ld1b { z0.b-z1.b }, pn9.b/Z, [x27, #0x4, MUL VL]\n" - ".inst 0xa14206fc // ldnt1b { z20.b, z28.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xa0430764 // ld1b { z4.b-z5.b }, pn9.b/Z, [x27, #0x6, MUL VL]\n" + ".inst 0xa1400763 // ld1b { z3.b, z11.b }, pn9.b/Z, [x27]\n" + ".inst 0xa14006f9 // ldnt1b { z17.b, z25.b }, pn9.b/Z, [x23]\n" + ".inst 0xa1410774 // ld1b { z20.b, z28.b }, pn9.b/Z, [x27, #0x2, MUL VL]\n" + ".inst 0xa04106f7 // ldnt1b { z22.b-z23.b }, pn9.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xa1420775 // ld1b { z21.b, z29.b }, pn9.b/Z, [x27, #0x4, MUL VL]\n" + ".inst 0xa14206f8 // ldnt1b { z16.b, z24.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xa1430765 // ld1b { z5.b, z13.b }, pn9.b/Z, [x27, #0x6, MUL VL]\n" "addvl x27, x27, #8\n" - ".inst 0xa14306ea // ldnt1b { z2.b, z10.b }, pn9.b/Z, [x23, #0x6, MUL VL]\n" + ".inst 0xa14306ef // ldnt1b { z7.b, z15.b }, pn9.b/Z, [x23, #0x6, MUL VL]\n" "addvl x23, x23, #8\n" "ble 7f\n" "6:" // K loop - ".inst 0xa09027c0 // smopa za0.s, p1/M, p1/M, z30.b, z16.b\n" + ".inst 0xa0912460 // smopa za0.s, p1/M, p1/M, z3.b, z17.b\n" "subs x22, x22, #0x1\n" - ".inst 0xa09127c1 // smopa za1.s, p1/M, p1/M, z30.b, z17.b\n" - ".inst 0xa09027e2 // smopa za2.s, p1/M, p1/M, z31.b, z16.b\n" - ".inst 0xa09127e3 // smopa za3.s, p1/M, p1/M, z31.b, z17.b\n" - ".inst 0xa040077e // ld1b { z30.b-z31.b }, pn9.b/Z, [x27]\n" - ".inst 0xa08825c0 // smopa za0.s, p1/M, p1/M, z14.b, z8.b\n" - ".inst 0xa04006f1 // ldnt1b { z16.b-z17.b }, pn9.b/Z, [x23]\n" - ".inst 0xa08925c1 // smopa za1.s, p1/M, p1/M, z14.b, z9.b\n" - ".inst 0xa08825e2 // smopa za2.s, p1/M, p1/M, z15.b, z8.b\n" - ".inst 0xa08925e3 // smopa za3.s, p1/M, p1/M, z15.b, z9.b\n" - ".inst 0xa041076e // ld1b { z14.b-z15.b }, pn9.b/Z, [x27, #0x2, MUL VL]\n" - ".inst 0xa0942400 // smopa za0.s, p1/M, p1/M, z0.b, z20.b\n" - ".inst 0xa04106e9 // ldnt1b { z8.b-z9.b }, pn9.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0xa09c2401 // smopa za1.s, p1/M, p1/M, z0.b, z28.b\n" - ".inst 0xa0942422 // smopa za2.s, p1/M, p1/M, z1.b, z20.b\n" - ".inst 0xa09c2423 // smopa za3.s, p1/M, p1/M, z1.b, z28.b\n" - ".inst 0xa0420760 // ld1b { z0.b-z1.b }, pn9.b/Z, [x27, #0x4, MUL VL]\n" - ".inst 0xa14206fc // ldnt1b { z20.b, z28.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xa0822480 // smopa za0.s, p1/M, p1/M, z4.b, z2.b\n" - ".inst 0xa08a2481 // smopa za1.s, p1/M, p1/M, z4.b, z10.b\n" - ".inst 0xa08224a2 // smopa za2.s, p1/M, p1/M, z5.b, z2.b\n" - ".inst 0xa08a24a3 // smopa za3.s, p1/M, p1/M, z5.b, z10.b\n" - ".inst 0xa0430764 // ld1b { z4.b-z5.b }, pn9.b/Z, [x27, #0x6, MUL VL]\n" + ".inst 0xa0992461 // smopa za1.s, p1/M, p1/M, z3.b, z25.b\n" + ".inst 0xa0912562 // smopa za2.s, p1/M, p1/M, z11.b, z17.b\n" + ".inst 0xa0992563 // smopa za3.s, p1/M, p1/M, z11.b, z25.b\n" + ".inst 0xa1400763 // ld1b { z3.b, z11.b }, pn9.b/Z, [x27]\n" + ".inst 0xa0962680 // smopa za0.s, p1/M, p1/M, z20.b, z22.b\n" + ".inst 0xa14006f9 // ldnt1b { z17.b, z25.b }, pn9.b/Z, [x23]\n" + ".inst 0xa0972681 // smopa za1.s, p1/M, p1/M, z20.b, z23.b\n" + ".inst 0xa0962782 // smopa za2.s, p1/M, p1/M, z28.b, z22.b\n" + ".inst 0xa0972783 // smopa za3.s, p1/M, p1/M, z28.b, z23.b\n" + ".inst 0xa1410774 // ld1b { z20.b, z28.b }, pn9.b/Z, [x27, #0x2, MUL VL]\n" + ".inst 0xa09026a0 // smopa za0.s, p1/M, p1/M, z21.b, z16.b\n" + ".inst 0xa04106f7 // ldnt1b { z22.b-z23.b }, pn9.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xa09826a1 // smopa za1.s, p1/M, p1/M, z21.b, z24.b\n" + ".inst 0xa09027a2 // smopa za2.s, p1/M, p1/M, z29.b, z16.b\n" + ".inst 0xa09827a3 // smopa za3.s, p1/M, p1/M, z29.b, z24.b\n" + ".inst 0xa1420775 // ld1b { z21.b, z29.b }, pn9.b/Z, [x27, #0x4, MUL VL]\n" + ".inst 0xa14206f8 // ldnt1b { z16.b, z24.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xa08724a0 // smopa za0.s, p1/M, p1/M, z5.b, z7.b\n" + ".inst 0xa08f24a1 // smopa za1.s, p1/M, p1/M, z5.b, z15.b\n" + ".inst 0xa08725a2 // smopa za2.s, p1/M, p1/M, z13.b, z7.b\n" + ".inst 0xa08f25a3 // smopa za3.s, p1/M, p1/M, z13.b, z15.b\n" + ".inst 0xa1430765 // ld1b { z5.b, z13.b }, pn9.b/Z, [x27, #0x6, MUL VL]\n" "addvl x27, x27, #8\n" - ".inst 0xa14306ea // ldnt1b { z2.b, z10.b }, pn9.b/Z, [x23, #0x6, MUL VL]\n" + ".inst 0xa14306ef // ldnt1b { z7.b, z15.b }, pn9.b/Z, [x23, #0x6, MUL VL]\n" "addvl x23, x23, #8\n" "bgt 6b\n" "7:" // K loop tail - ".inst 0xa09027c0 // smopa za0.s, p1/M, p1/M, z30.b, z16.b\n" - ".inst 0xa09127c1 // smopa za1.s, p1/M, p1/M, z30.b, z17.b\n" - ".inst 0xa09027e2 // smopa za2.s, p1/M, p1/M, z31.b, z16.b\n" - ".inst 0xa09127e3 // smopa za3.s, p1/M, p1/M, z31.b, z17.b\n" - ".inst 0xa08825c0 // smopa za0.s, p1/M, p1/M, z14.b, z8.b\n" - ".inst 0xa08925c1 // smopa za1.s, p1/M, p1/M, z14.b, z9.b\n" - ".inst 0xa08825e2 // smopa za2.s, p1/M, p1/M, z15.b, z8.b\n" - ".inst 0xa08925e3 // smopa za3.s, p1/M, p1/M, z15.b, z9.b\n" - ".inst 0xa0942400 // smopa za0.s, p1/M, p1/M, z0.b, z20.b\n" - ".inst 0xa09c2401 // smopa za1.s, p1/M, p1/M, z0.b, z28.b\n" - ".inst 0xa0942422 // smopa za2.s, p1/M, p1/M, z1.b, z20.b\n" - ".inst 0xa09c2423 // smopa za3.s, p1/M, p1/M, z1.b, z28.b\n" - ".inst 0xa0822480 // smopa za0.s, p1/M, p1/M, z4.b, z2.b\n" - ".inst 0xa08a2481 // smopa za1.s, p1/M, p1/M, z4.b, z10.b\n" - ".inst 0xa08224a2 // smopa za2.s, p1/M, p1/M, z5.b, z2.b\n" - ".inst 0xa08a24a3 // smopa za3.s, p1/M, p1/M, z5.b, z10.b\n" + ".inst 0xa0912460 // smopa za0.s, p1/M, p1/M, z3.b, z17.b\n" + ".inst 0xa0992461 // smopa za1.s, p1/M, p1/M, z3.b, z25.b\n" + ".inst 0xa0912562 // smopa za2.s, p1/M, p1/M, z11.b, z17.b\n" + ".inst 0xa0992563 // smopa za3.s, p1/M, p1/M, z11.b, z25.b\n" + ".inst 0xa0962680 // smopa za0.s, p1/M, p1/M, z20.b, z22.b\n" + ".inst 0xa0972681 // smopa za1.s, p1/M, p1/M, z20.b, z23.b\n" + ".inst 0xa0962782 // smopa za2.s, p1/M, p1/M, z28.b, z22.b\n" + ".inst 0xa0972783 // smopa za3.s, p1/M, p1/M, z28.b, z23.b\n" + ".inst 0xa09026a0 // smopa za0.s, p1/M, p1/M, z21.b, z16.b\n" + ".inst 0xa09826a1 // smopa za1.s, p1/M, p1/M, z21.b, z24.b\n" + ".inst 0xa09027a2 // smopa za2.s, p1/M, p1/M, z29.b, z16.b\n" + ".inst 0xa09827a3 // smopa za3.s, p1/M, p1/M, z29.b, z24.b\n" + ".inst 0xa08724a0 // smopa za0.s, p1/M, p1/M, z5.b, z7.b\n" + ".inst 0xa08f24a1 // smopa za1.s, p1/M, p1/M, z5.b, z15.b\n" + ".inst 0xa08725a2 // smopa za2.s, p1/M, p1/M, z13.b, z7.b\n" + ".inst 0xa08f25a3 // smopa za3.s, p1/M, p1/M, z13.b, z15.b\n" "8:" // K oddments "cbz x21, 10f\n" "9:" // K oddments: Loop - ".inst 0xa040077e // ld1b { z30.b-z31.b }, pn9.b/Z, [x27]\n" + ".inst 0xa1400773 // ld1b { z19.b, z27.b }, pn9.b/Z, [x27]\n" "subs x21, x21, #0x1\n" "addvl x27, x27, #2\n" ".inst 0xa04006f0 // ld1b { z16.b-z17.b }, pn9.b/Z, [x23]\n" "addvl x23, x23, #2\n" - ".inst 0xa09027c0 // smopa za0.s, p1/M, p1/M, z30.b, z16.b\n" - ".inst 0xa09127c1 // smopa za1.s, p1/M, p1/M, z30.b, z17.b\n" - ".inst 0xa09027e2 // smopa za2.s, p1/M, p1/M, z31.b, z16.b\n" - ".inst 0xa09127e3 // smopa za3.s, p1/M, p1/M, z31.b, z17.b\n" + ".inst 0xa0902660 // smopa za0.s, p1/M, p1/M, z19.b, z16.b\n" + ".inst 0xa0912661 // smopa za1.s, p1/M, p1/M, z19.b, z17.b\n" + ".inst 0xa0902762 // smopa za2.s, p1/M, p1/M, z27.b, z16.b\n" + ".inst 0xa0912763 // smopa za3.s, p1/M, p1/M, z27.b, z17.b\n" "bgt 9b\n" "10:" // K oddments: End ".inst 0xa040476e // ld1w { z14.s-z15.s }, pn9.b/Z, [x27]\n" @@ -234,25 +233,25 @@ void sme2_interleaved_nomerge_s8q_mopa_2VLx2VL(const int8_t *const A, const int8 "mov x12, #0x0\n" "cntw x20\n" "11:" // Store to partial result buffer: Store and refill: Loop - ".inst 0xa040c5fc // ld1w { z28.s-z31.s }, pn9.b/Z, [x15]\n" - ".inst 0xc0860408 // mova { z8.s-z11.s }, za0h.s[x12]\n" - ".inst 0xc0840780 // mova za0h.s[x12], { z28.s-z31.s }\n" - ".inst 0xc0860434 // mova { z20.s-z23.s }, za1h.s[x12]\n" - ".inst 0xa041c5f8 // ld1w { z24.s-z27.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n" - ".inst 0xc0840701 // mova za1h.s[x12], { z24.s-z27.s }\n" - ".inst 0xc086045c // mova { z28.s-z31.s }, za2h.s[x12]\n" - ".inst 0xc0860470 // mova { z16.s-z19.s }, za3h.s[x12]\n" - ".inst 0xa042c5f8 // ld1w { z24.s-z27.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n" - ".inst 0xc0840702 // mova za2h.s[x12], { z24.s-z27.s }\n" - ".inst 0xa043c5ec // ld1w { z12.s-z15.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840583 // mova za3h.s[x12], { z12.s-z15.s }\n" + ".inst 0xa040c5f0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x15]\n" + ".inst 0xc0860400 // mova { z0.s-z3.s }, za0h.s[x12]\n" + ".inst 0xc0840600 // mova za0h.s[x12], { z16.s-z19.s }\n" + ".inst 0xc0860428 // mova { z8.s-z11.s }, za1h.s[x12]\n" + ".inst 0xa041c5fc // ld1w { z28.s-z31.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n" + ".inst 0xc0840781 // mova za1h.s[x12], { z28.s-z31.s }\n" + ".inst 0xc0860444 // mova { z4.s-z7.s }, za2h.s[x12]\n" + ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" + ".inst 0xa042c5f0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n" + ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa043c5fc // ld1w { z28.s-z31.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840783 // mova za3h.s[x12], { z28.s-z31.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" - ".inst 0xa060c5c8 // st1w { z8.s-z11.s }, pn9.b, [x14]\n" + ".inst 0xa060c5c0 // st1w { z0.s-z3.s }, pn9.b, [x14]\n" "addvl x15, x15, #16\n" - ".inst 0xa061c5d4 // st1w { z20.s-z23.s }, pn9.b, [x14, #0x4, MUL VL]\n" - ".inst 0xa062c5dc // st1w { z28.s-z31.s }, pn9.b, [x14, #0x8, MUL VL]\n" - ".inst 0xa063c5d0 // st1w { z16.s-z19.s }, pn9.b, [x14, #0xc, MUL VL]\n" + ".inst 0xa061c5c8 // st1w { z8.s-z11.s }, pn9.b, [x14, #0x4, MUL VL]\n" + ".inst 0xa062c5c4 // st1w { z4.s-z7.s }, pn9.b, [x14, #0x8, MUL VL]\n" + ".inst 0xa063c5cc // st1w { z12.s-z15.s }, pn9.b, [x14, #0xc, MUL VL]\n" "addvl x14, x14, #16\n" "blt 11b\n" "b 24f\n" @@ -260,16 +259,16 @@ void sme2_interleaved_nomerge_s8q_mopa_2VLx2VL(const int8_t *const A, const int8 "mov x12, #0x0\n" "cntw x20\n" "13:" // Store to partial result buffer: Store only: Loop - ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" - ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" - ".inst 0xa060c5d0 // st1w { z16.s-z19.s }, pn9.b, [x14]\n" - ".inst 0xc0860448 // mova { z8.s-z11.s }, za2h.s[x12]\n" - ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" - ".inst 0xa061c5c4 // st1w { z4.s-z7.s }, pn9.b, [x14, #0x4, MUL VL]\n" + ".inst 0xc086040c // mova { z12.s-z15.s }, za0h.s[x12]\n" + ".inst 0xc0860428 // mova { z8.s-z11.s }, za1h.s[x12]\n" + ".inst 0xa060c5cc // st1w { z12.s-z15.s }, pn9.b, [x14]\n" + ".inst 0xc0860450 // mova { z16.s-z19.s }, za2h.s[x12]\n" + ".inst 0xc0860474 // mova { z20.s-z23.s }, za3h.s[x12]\n" + ".inst 0xa061c5c8 // st1w { z8.s-z11.s }, pn9.b, [x14, #0x4, MUL VL]\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" - ".inst 0xa062c5c8 // st1w { z8.s-z11.s }, pn9.b, [x14, #0x8, MUL VL]\n" - ".inst 0xa063c5cc // st1w { z12.s-z15.s }, pn9.b, [x14, #0xc, MUL VL]\n" + ".inst 0xa062c5d0 // st1w { z16.s-z19.s }, pn9.b, [x14, #0x8, MUL VL]\n" + ".inst 0xa063c5d4 // st1w { z20.s-z23.s }, pn9.b, [x14, #0xc, MUL VL]\n" "addvl x14, x14, #16\n" "blt 13b\n" "b 24f\n" @@ -277,13 +276,13 @@ void sme2_interleaved_nomerge_s8q_mopa_2VLx2VL(const int8_t *const A, const int8 "ldr x26, [%x[args], %[offsetof_C]]\n" "add x26, x26, x10\n" // C += n "sub x25, x13, x11\n" - "ld1rw { z2.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z0.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" "ldr x24, [%x[args], %[offsetof_ldcb]]\n" "madd x26, x11, x24, x26\n" // C += m * ldc - "ld1rw { z3.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" - "ld1rw { z0.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" - "ld1rw { z1.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" - "ld1rw { z11.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_c_offset]]\n" + "ld1rw { z1.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z2.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z3.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z14.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_c_offset]]\n" "ld1rw { z25.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_minval]]\n" "ld1rw { z24.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_maxval]]\n" "tbz x16, #2, 15f\n" @@ -291,10 +290,10 @@ void sme2_interleaved_nomerge_s8q_mopa_2VLx2VL(const int8_t *const A, const int8 "add x21, x21, x10\n" "ldr x20, [%x[rq], %[offsetof_Requantize32_per_channel_muls]]\n" "add x20, x20, x21, LSL #2\n" - ".inst 0xa0404282 // ld1w { z2.s-z3.s }, p8/Z, [x20]\n" + ".inst 0xa0404280 // ld1w { z0.s-z1.s }, p8/Z, [x20]\n" "ldr x20, [%x[rq], %[offsetof_Requantize32_per_channel_right_shifts]]\n" "add x20, x20, x21, LSL #2\n" - ".inst 0xa0404280 // ld1w { z0.s-z1.s }, p8/Z, [x20]\n" + ".inst 0xa0404282 // ld1w { z2.s-z3.s }, p8/Z, [x20]\n" "15:" // Store to output array: Load per-channel parameters: End "cntw x23\n" "whilelt p0.h, x10, x9\n" @@ -305,26 +304,26 @@ void sme2_interleaved_nomerge_s8q_mopa_2VLx2VL(const int8_t *const A, const int8 "and x20, x22, #0x3\n" "cbz x21, 17f\n" "16:" // Store to output array: Accumulator row 0 loop - ".inst 0xc086040c // mova { z12.s-z15.s }, za0h.s[x12]\n" - ".inst 0xc086043c // mova { z28.s-z31.s }, za1h.s[x12]\n" - ".inst 0xc1a2ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z2.s\n" - ".inst 0xc1a3ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z3.s\n" + ".inst 0xc0860404 // mova { z4.s-z7.s }, za0h.s[x12]\n" + ".inst 0xc0860428 // mova { z8.s-z11.s }, za1h.s[x12]\n" + ".inst 0xc1a0ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z0.s\n" + ".inst 0xc1a1ac08 // sqdmulh { z8.s-z11.s }, { z8.s-z11.s }, z1.s\n" "add x12, x12, #0x4\n" "cmp x12, x21, LSL #2\n" - ".inst 0xc1a0aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z0.s\n" - ".inst 0xc1a1aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z1.s\n" - ".inst 0xc1abab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z11.s\n" - ".inst 0xc1abab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z11.s\n" - ".inst 0xc1b8cf2c // sclamp { z12.s-z15.s }, z25.s, z24.s\n" - ".inst 0xc1b8cf3c // sclamp { z28.s-z31.s }, z25.s, z24.s\n" - "uzp1 z16.h, z12.h, z28.h\n" + ".inst 0xc1a2aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z2.s\n" + ".inst 0xc1a3aa28 // srshl { z8.s-z11.s }, { z8.s-z11.s }, z3.s\n" + ".inst 0xc1aeab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z14.s\n" + ".inst 0xc1aeab08 // add { z8.s-z11.s }, { z8.s-z11.s }, z14.s\n" + ".inst 0xc1b8cf24 // sclamp { z4.s-z7.s }, z25.s, z24.s\n" + ".inst 0xc1b8cf28 // sclamp { z8.s-z11.s }, z25.s, z24.s\n" + "uzp1 z16.h, z4.h, z8.h\n" "st1b { z16.h }, p0, [x26]\n" "add x26, x26, x24\n" - "uzp1 z16.h, z13.h, z29.h\n" - "uzp1 z17.h, z14.h, z30.h\n" + "uzp1 z16.h, z5.h, z9.h\n" + "uzp1 z17.h, z6.h, z10.h\n" "st1b { z16.h }, p0, [x26]\n" "add x26, x26, x24\n" - "uzp1 z16.h, z15.h, z31.h\n" + "uzp1 z16.h, z7.h, z11.h\n" "st1b { z17.h }, p0, [x26]\n" "add x26, x26, x24\n" "st1b { z16.h }, p0, [x26]\n" @@ -332,27 +331,27 @@ void sme2_interleaved_nomerge_s8q_mopa_2VLx2VL(const int8_t *const A, const int8 "blt 16b\n" "17:" // Store to output array: Accumulator row 0 oddments "cbz x20, 18f\n" - ".inst 0xc086041c // mova { z28.s-z31.s }, za0h.s[x12]\n" - ".inst 0xc086042c // mova { z12.s-z15.s }, za1h.s[x12]\n" - ".inst 0xc1a2ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z2.s\n" - ".inst 0xc1a3ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z3.s\n" + ".inst 0xc0860408 // mova { z8.s-z11.s }, za0h.s[x12]\n" + ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" + ".inst 0xc1a0ac08 // sqdmulh { z8.s-z11.s }, { z8.s-z11.s }, z0.s\n" + ".inst 0xc1a1ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" "subs x20, x20, #0x1\n" - ".inst 0xc1a0aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z0.s\n" - ".inst 0xc1a1aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n" - ".inst 0xc1abab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z11.s\n" - ".inst 0xc1abab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z11.s\n" - ".inst 0xc1b8cf3c // sclamp { z28.s-z31.s }, z25.s, z24.s\n" - ".inst 0xc1b8cf2c // sclamp { z12.s-z15.s }, z25.s, z24.s\n" - "uzp1 z16.h, z28.h, z12.h\n" + ".inst 0xc1a2aa28 // srshl { z8.s-z11.s }, { z8.s-z11.s }, z2.s\n" + ".inst 0xc1a3aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z3.s\n" + ".inst 0xc1aeab08 // add { z8.s-z11.s }, { z8.s-z11.s }, z14.s\n" + ".inst 0xc1aeab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z14.s\n" + ".inst 0xc1b8cf28 // sclamp { z8.s-z11.s }, z25.s, z24.s\n" + ".inst 0xc1b8cf24 // sclamp { z4.s-z7.s }, z25.s, z24.s\n" + "uzp1 z16.h, z8.h, z4.h\n" "st1b { z16.h }, p0, [x26]\n" "add x26, x26, x24\n" "beq 18f\n" "subs x20, x20, #0x1\n" - "uzp1 z16.h, z29.h, z13.h\n" + "uzp1 z16.h, z9.h, z5.h\n" "st1b { z16.h }, p0, [x26]\n" "add x26, x26, x24\n" "beq 18f\n" - "uzp1 z16.h, z30.h, z14.h\n" + "uzp1 z16.h, z10.h, z6.h\n" "st1b { z16.h }, p0, [x26]\n" "add x26, x26, x24\n" "18:" // Store to output array: Accumulator row 0 oddments: End @@ -367,25 +366,25 @@ void sme2_interleaved_nomerge_s8q_mopa_2VLx2VL(const int8_t *const A, const int8 "cbz x21, 20f\n" "19:" // Store to output array: Accumulator row 1 loop ".inst 0xc0860444 // mova { z4.s-z7.s }, za2h.s[x12]\n" - ".inst 0xc0860470 // mova { z16.s-z19.s }, za3h.s[x12]\n" - ".inst 0xc1a2ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z2.s\n" - ".inst 0xc1a3ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z3.s\n" + ".inst 0xc0860474 // mova { z20.s-z23.s }, za3h.s[x12]\n" + ".inst 0xc1a0ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z0.s\n" + ".inst 0xc1a1ac14 // sqdmulh { z20.s-z23.s }, { z20.s-z23.s }, z1.s\n" "add x12, x12, #0x4\n" "cmp x12, x21, LSL #2\n" - ".inst 0xc1a0aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z0.s\n" - ".inst 0xc1a1aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z1.s\n" - ".inst 0xc1abab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z11.s\n" - ".inst 0xc1abab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z11.s\n" + ".inst 0xc1a2aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z2.s\n" + ".inst 0xc1a3aa34 // srshl { z20.s-z23.s }, { z20.s-z23.s }, z3.s\n" + ".inst 0xc1aeab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z14.s\n" + ".inst 0xc1aeab14 // add { z20.s-z23.s }, { z20.s-z23.s }, z14.s\n" ".inst 0xc1b8cf24 // sclamp { z4.s-z7.s }, z25.s, z24.s\n" - ".inst 0xc1b8cf30 // sclamp { z16.s-z19.s }, z25.s, z24.s\n" - "uzp1 z16.h, z4.h, z16.h\n" + ".inst 0xc1b8cf34 // sclamp { z20.s-z23.s }, z25.s, z24.s\n" + "uzp1 z16.h, z4.h, z20.h\n" "st1b { z16.h }, p0, [x26]\n" "add x26, x26, x24\n" - "uzp1 z16.h, z5.h, z17.h\n" - "uzp1 z17.h, z6.h, z18.h\n" + "uzp1 z16.h, z5.h, z21.h\n" + "uzp1 z17.h, z6.h, z22.h\n" "st1b { z16.h }, p0, [x26]\n" "add x26, x26, x24\n" - "uzp1 z16.h, z7.h, z19.h\n" + "uzp1 z16.h, z7.h, z23.h\n" "st1b { z17.h }, p0, [x26]\n" "add x26, x26, x24\n" "st1b { z16.h }, p0, [x26]\n" @@ -393,27 +392,27 @@ void sme2_interleaved_nomerge_s8q_mopa_2VLx2VL(const int8_t *const A, const int8 "blt 19b\n" "20:" // Store to output array: Accumulator row 1 oddments "cbz x20, 21f\n" - ".inst 0xc0860454 // mova { z20.s-z23.s }, za2h.s[x12]\n" + ".inst 0xc0860444 // mova { z4.s-z7.s }, za2h.s[x12]\n" ".inst 0xc0860470 // mova { z16.s-z19.s }, za3h.s[x12]\n" - ".inst 0xc1a2ac14 // sqdmulh { z20.s-z23.s }, { z20.s-z23.s }, z2.s\n" - ".inst 0xc1a3ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z3.s\n" + ".inst 0xc1a0ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z0.s\n" + ".inst 0xc1a1ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z1.s\n" "subs x20, x20, #0x1\n" - ".inst 0xc1a0aa34 // srshl { z20.s-z23.s }, { z20.s-z23.s }, z0.s\n" - ".inst 0xc1a1aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z1.s\n" - ".inst 0xc1abab14 // add { z20.s-z23.s }, { z20.s-z23.s }, z11.s\n" - ".inst 0xc1abab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z11.s\n" - ".inst 0xc1b8cf34 // sclamp { z20.s-z23.s }, z25.s, z24.s\n" + ".inst 0xc1a2aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z2.s\n" + ".inst 0xc1a3aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z3.s\n" + ".inst 0xc1aeab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z14.s\n" + ".inst 0xc1aeab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z14.s\n" + ".inst 0xc1b8cf24 // sclamp { z4.s-z7.s }, z25.s, z24.s\n" ".inst 0xc1b8cf30 // sclamp { z16.s-z19.s }, z25.s, z24.s\n" - "uzp1 z16.h, z20.h, z16.h\n" + "uzp1 z16.h, z4.h, z16.h\n" "st1b { z16.h }, p0, [x26]\n" "add x26, x26, x24\n" "beq 21f\n" "subs x20, x20, #0x1\n" - "uzp1 z16.h, z21.h, z17.h\n" + "uzp1 z16.h, z5.h, z17.h\n" "st1b { z16.h }, p0, [x26]\n" "add x26, x26, x24\n" "beq 21f\n" - "uzp1 z16.h, z22.h, z18.h\n" + "uzp1 z16.h, z6.h, z18.h\n" "st1b { z16.h }, p0, [x26]\n" "21:" // Store to output array: Accumulator row 1 oddments: End "22:" // Store to output array: End @@ -452,4 +451,3 @@ void sme2_interleaved_nomerge_s8q_mopa_2VLx2VL(const int8_t *const A, const int8 } // namespace arm_gemm #endif // ARM_COMPUTE_ENABLE_SME2 -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_4VLx1VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_4VLx1VL.hpp index 420c219af5f69d7adfe61f6be93cfb2f9f1098bc..ef39cbbb28a5998b47dc7d95da01c6533f9f4fa9 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_4VLx1VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_4VLx1VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,7 +23,7 @@ */ #pragma once -#ifdef __aarch64__ +#ifdef ARM_COMPUTE_ENABLE_SME2 #include #include "../std_transforms_sme.hpp" @@ -83,12 +83,11 @@ public: StdTransformsSME transforms = {}; - cls_sme2_interleaved_nomerge_s8q_mopa_4VLx1VL(const CPUInfo *ci) + cls_sme2_interleaved_nomerge_s8q_mopa_4VLx1VL(const CPUInfo *) { - ARM_COMPUTE_UNUSED(ci); } }; } // namespace arm_gemm -#endif // __aarch64__ +#endif // ARM_COMPUTE_ENABLE_SME2 diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_4VLx1VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_4VLx1VL/generic.cpp index 0d0e3da224fc78a8ff55f5a8de5cbe39e8bb947a..4601f05501df10e36fb6dab58eb1038dc43e974e 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_4VLx1VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8q_mopa_4VLx1VL/generic.cpp @@ -21,7 +21,6 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifdef __ARM_FEATURE_SVE #ifdef ARM_COMPUTE_ENABLE_SME2 #include "arm_gemm.hpp" @@ -100,14 +99,14 @@ void sme2_interleaved_nomerge_s8q_mopa_4VLx1VL(const int8_t *const A, const int8 "mov x12, #0x0\n" "cntw x20\n" "1:" // Initial accumulator load from buffer: Loop - ".inst 0xa040c1fc // ld1w { z28.s-z31.s }, pn8.b/Z, [x15]\n" - ".inst 0xc0840780 // mova za0h.s[x12], { z28.s-z31.s }\n" - ".inst 0xa041c1ec // ld1w { z12.s-z15.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" - ".inst 0xc0840581 // mova za1h.s[x12], { z12.s-z15.s }\n" - ".inst 0xa042c1f4 // ld1w { z20.s-z23.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" - ".inst 0xc0840682 // mova za2h.s[x12], { z20.s-z23.s }\n" - ".inst 0xa043c1f8 // ld1w { z24.s-z27.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840703 // mova za3h.s[x12], { z24.s-z27.s }\n" + ".inst 0xa040c1f0 // ld1w { z16.s-z19.s }, pn8.b/Z, [x15]\n" + ".inst 0xc0840600 // mova za0h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa041c1f8 // ld1w { z24.s-z27.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" + ".inst 0xc0840701 // mova za1h.s[x12], { z24.s-z27.s }\n" + ".inst 0xa042c1fc // ld1w { z28.s-z31.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" + ".inst 0xc0840782 // mova za2h.s[x12], { z28.s-z31.s }\n" + ".inst 0xa043c1f4 // ld1w { z20.s-z23.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840683 // mova za3h.s[x12], { z20.s-z23.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x15, x15, #16\n" @@ -125,11 +124,11 @@ void sme2_interleaved_nomerge_s8q_mopa_4VLx1VL(const int8_t *const A, const int8 "ldr x20, [%x[args], %[offsetof_bias]]\n" ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "cbz x20, 5f\n" - "ldnt1w { z15.s }, p0/Z, [x20, x10, LSL #2]\n" - ".inst 0xc09025e0 // addha za0.s, p1/M, p1/M, z15.s\n" - ".inst 0xc09025e1 // addha za1.s, p1/M, p1/M, z15.s\n" - ".inst 0xc09025e2 // addha za2.s, p1/M, p1/M, z15.s\n" - ".inst 0xc09025e3 // addha za3.s, p1/M, p1/M, z15.s\n" + "ldnt1w { z8.s }, p0/Z, [x20, x10, LSL #2]\n" + ".inst 0xc0902500 // addha za0.s, p1/M, p1/M, z8.s\n" + ".inst 0xc0902501 // addha za1.s, p1/M, p1/M, z8.s\n" + ".inst 0xc0902502 // addha za2.s, p1/M, p1/M, z8.s\n" + ".inst 0xc0902503 // addha za3.s, p1/M, p1/M, z8.s\n" "4:" // Prepare accumulators: Test for last block "mov x20, x10\n" "mov x21, x11\n" @@ -152,107 +151,107 @@ void sme2_interleaved_nomerge_s8q_mopa_4VLx1VL(const int8_t *const A, const int8 "madd x23, x10, x20, x23\n" // bptr = B + n * kstride_bytes "cbz x22, 8f\n" "subs x22, x22, #0x1\n" - ".inst 0xa1408372 // ld1b { z18.b, z22.b, z26.b, z30.b }, pn8.b/Z, [x27]\n" - "ldnt1b { z0.b }, p1/Z, [x23]\n" - ".inst 0xa1418373 // ld1b { z19.b, z23.b, z27.b, z31.b }, pn8.b/Z, [x27, #0x4, MUL VL]\n" - "ldnt1b { z9.b }, p1/Z, [x23, #1, MUL VL]\n" - ".inst 0xa1428370 // ld1b { z16.b, z20.b, z24.b, z28.b }, pn8.b/Z, [x27, #0x8, MUL VL]\n" - "ldnt1b { z21.b }, p1/Z, [x23, #2, MUL VL]\n" - ".inst 0xa1438362 // ld1b { z2.b, z6.b, z10.b, z14.b }, pn8.b/Z, [x27, #0xc, MUL VL]\n" + ".inst 0xa0408364 // ld1b { z4.b-z7.b }, pn8.b/Z, [x27]\n" + "ldnt1b { z14.b }, p1/Z, [x23]\n" + ".inst 0xa0418374 // ld1b { z20.b-z23.b }, pn8.b/Z, [x27, #0x4, MUL VL]\n" + "ldnt1b { z31.b }, p1/Z, [x23, #1, MUL VL]\n" + ".inst 0xa0428378 // ld1b { z24.b-z27.b }, pn8.b/Z, [x27, #0x8, MUL VL]\n" + "ldnt1b { z13.b }, p1/Z, [x23, #2, MUL VL]\n" + ".inst 0xa0438368 // ld1b { z8.b-z11.b }, pn8.b/Z, [x27, #0xc, MUL VL]\n" "addvl x27, x27, #16\n" - "ldnt1b { z12.b }, p1/Z, [x23, #3, MUL VL]\n" + "ldnt1b { z29.b }, p1/Z, [x23, #3, MUL VL]\n" "addvl x23, x23, #4\n" "ble 7f\n" "6:" // K loop - ".inst 0xa0802640 // smopa za0.s, p1/M, p1/M, z18.b, z0.b\n" + ".inst 0xa08e2480 // smopa za0.s, p1/M, p1/M, z4.b, z14.b\n" "subs x22, x22, #0x1\n" - ".inst 0xa08026c1 // smopa za1.s, p1/M, p1/M, z22.b, z0.b\n" - ".inst 0xa0802742 // smopa za2.s, p1/M, p1/M, z26.b, z0.b\n" - ".inst 0xa08027c3 // smopa za3.s, p1/M, p1/M, z30.b, z0.b\n" - ".inst 0xa1408372 // ld1b { z18.b, z22.b, z26.b, z30.b }, pn8.b/Z, [x27]\n" - ".inst 0xa0892660 // smopa za0.s, p1/M, p1/M, z19.b, z9.b\n" - "ldnt1b { z0.b }, p1/Z, [x23]\n" - ".inst 0xa08926e1 // smopa za1.s, p1/M, p1/M, z23.b, z9.b\n" - ".inst 0xa0892762 // smopa za2.s, p1/M, p1/M, z27.b, z9.b\n" - ".inst 0xa08927e3 // smopa za3.s, p1/M, p1/M, z31.b, z9.b\n" - ".inst 0xa1418373 // ld1b { z19.b, z23.b, z27.b, z31.b }, pn8.b/Z, [x27, #0x4, MUL VL]\n" - ".inst 0xa0952600 // smopa za0.s, p1/M, p1/M, z16.b, z21.b\n" - "ldnt1b { z9.b }, p1/Z, [x23, #1, MUL VL]\n" - ".inst 0xa0952681 // smopa za1.s, p1/M, p1/M, z20.b, z21.b\n" - ".inst 0xa0952702 // smopa za2.s, p1/M, p1/M, z24.b, z21.b\n" - ".inst 0xa0952783 // smopa za3.s, p1/M, p1/M, z28.b, z21.b\n" - ".inst 0xa1428370 // ld1b { z16.b, z20.b, z24.b, z28.b }, pn8.b/Z, [x27, #0x8, MUL VL]\n" - "ldnt1b { z21.b }, p1/Z, [x23, #2, MUL VL]\n" - ".inst 0xa08c2440 // smopa za0.s, p1/M, p1/M, z2.b, z12.b\n" - ".inst 0xa08c24c1 // smopa za1.s, p1/M, p1/M, z6.b, z12.b\n" - ".inst 0xa08c2542 // smopa za2.s, p1/M, p1/M, z10.b, z12.b\n" - ".inst 0xa08c25c3 // smopa za3.s, p1/M, p1/M, z14.b, z12.b\n" - ".inst 0xa1438362 // ld1b { z2.b, z6.b, z10.b, z14.b }, pn8.b/Z, [x27, #0xc, MUL VL]\n" + ".inst 0xa08e24a1 // smopa za1.s, p1/M, p1/M, z5.b, z14.b\n" + ".inst 0xa08e24c2 // smopa za2.s, p1/M, p1/M, z6.b, z14.b\n" + ".inst 0xa08e24e3 // smopa za3.s, p1/M, p1/M, z7.b, z14.b\n" + ".inst 0xa0408364 // ld1b { z4.b-z7.b }, pn8.b/Z, [x27]\n" + ".inst 0xa09f2680 // smopa za0.s, p1/M, p1/M, z20.b, z31.b\n" + "ldnt1b { z14.b }, p1/Z, [x23]\n" + ".inst 0xa09f26a1 // smopa za1.s, p1/M, p1/M, z21.b, z31.b\n" + ".inst 0xa09f26c2 // smopa za2.s, p1/M, p1/M, z22.b, z31.b\n" + ".inst 0xa09f26e3 // smopa za3.s, p1/M, p1/M, z23.b, z31.b\n" + ".inst 0xa0418374 // ld1b { z20.b-z23.b }, pn8.b/Z, [x27, #0x4, MUL VL]\n" + ".inst 0xa08d2700 // smopa za0.s, p1/M, p1/M, z24.b, z13.b\n" + "ldnt1b { z31.b }, p1/Z, [x23, #1, MUL VL]\n" + ".inst 0xa08d2721 // smopa za1.s, p1/M, p1/M, z25.b, z13.b\n" + ".inst 0xa08d2742 // smopa za2.s, p1/M, p1/M, z26.b, z13.b\n" + ".inst 0xa08d2763 // smopa za3.s, p1/M, p1/M, z27.b, z13.b\n" + ".inst 0xa0428378 // ld1b { z24.b-z27.b }, pn8.b/Z, [x27, #0x8, MUL VL]\n" + "ldnt1b { z13.b }, p1/Z, [x23, #2, MUL VL]\n" + ".inst 0xa09d2500 // smopa za0.s, p1/M, p1/M, z8.b, z29.b\n" + ".inst 0xa09d2521 // smopa za1.s, p1/M, p1/M, z9.b, z29.b\n" + ".inst 0xa09d2542 // smopa za2.s, p1/M, p1/M, z10.b, z29.b\n" + ".inst 0xa09d2563 // smopa za3.s, p1/M, p1/M, z11.b, z29.b\n" + ".inst 0xa0438368 // ld1b { z8.b-z11.b }, pn8.b/Z, [x27, #0xc, MUL VL]\n" "addvl x27, x27, #16\n" - "ldnt1b { z12.b }, p1/Z, [x23, #3, MUL VL]\n" + "ldnt1b { z29.b }, p1/Z, [x23, #3, MUL VL]\n" "addvl x23, x23, #4\n" "bgt 6b\n" "7:" // K loop tail - ".inst 0xa0802640 // smopa za0.s, p1/M, p1/M, z18.b, z0.b\n" - ".inst 0xa08026c1 // smopa za1.s, p1/M, p1/M, z22.b, z0.b\n" - ".inst 0xa0802742 // smopa za2.s, p1/M, p1/M, z26.b, z0.b\n" - ".inst 0xa08027c3 // smopa za3.s, p1/M, p1/M, z30.b, z0.b\n" - ".inst 0xa0892660 // smopa za0.s, p1/M, p1/M, z19.b, z9.b\n" - ".inst 0xa08926e1 // smopa za1.s, p1/M, p1/M, z23.b, z9.b\n" - ".inst 0xa0892762 // smopa za2.s, p1/M, p1/M, z27.b, z9.b\n" - ".inst 0xa08927e3 // smopa za3.s, p1/M, p1/M, z31.b, z9.b\n" - ".inst 0xa0952600 // smopa za0.s, p1/M, p1/M, z16.b, z21.b\n" - ".inst 0xa0952681 // smopa za1.s, p1/M, p1/M, z20.b, z21.b\n" - ".inst 0xa0952702 // smopa za2.s, p1/M, p1/M, z24.b, z21.b\n" - ".inst 0xa0952783 // smopa za3.s, p1/M, p1/M, z28.b, z21.b\n" - ".inst 0xa08c2440 // smopa za0.s, p1/M, p1/M, z2.b, z12.b\n" - ".inst 0xa08c24c1 // smopa za1.s, p1/M, p1/M, z6.b, z12.b\n" - ".inst 0xa08c2542 // smopa za2.s, p1/M, p1/M, z10.b, z12.b\n" - ".inst 0xa08c25c3 // smopa za3.s, p1/M, p1/M, z14.b, z12.b\n" + ".inst 0xa08e2480 // smopa za0.s, p1/M, p1/M, z4.b, z14.b\n" + ".inst 0xa08e24a1 // smopa za1.s, p1/M, p1/M, z5.b, z14.b\n" + ".inst 0xa08e24c2 // smopa za2.s, p1/M, p1/M, z6.b, z14.b\n" + ".inst 0xa08e24e3 // smopa za3.s, p1/M, p1/M, z7.b, z14.b\n" + ".inst 0xa09f2680 // smopa za0.s, p1/M, p1/M, z20.b, z31.b\n" + ".inst 0xa09f26a1 // smopa za1.s, p1/M, p1/M, z21.b, z31.b\n" + ".inst 0xa09f26c2 // smopa za2.s, p1/M, p1/M, z22.b, z31.b\n" + ".inst 0xa09f26e3 // smopa za3.s, p1/M, p1/M, z23.b, z31.b\n" + ".inst 0xa08d2700 // smopa za0.s, p1/M, p1/M, z24.b, z13.b\n" + ".inst 0xa08d2721 // smopa za1.s, p1/M, p1/M, z25.b, z13.b\n" + ".inst 0xa08d2742 // smopa za2.s, p1/M, p1/M, z26.b, z13.b\n" + ".inst 0xa08d2763 // smopa za3.s, p1/M, p1/M, z27.b, z13.b\n" + ".inst 0xa09d2500 // smopa za0.s, p1/M, p1/M, z8.b, z29.b\n" + ".inst 0xa09d2521 // smopa za1.s, p1/M, p1/M, z9.b, z29.b\n" + ".inst 0xa09d2542 // smopa za2.s, p1/M, p1/M, z10.b, z29.b\n" + ".inst 0xa09d2563 // smopa za3.s, p1/M, p1/M, z11.b, z29.b\n" "8:" // K oddments "cbz x21, 10f\n" "9:" // K oddments: Loop ".inst 0xa1408372 // ld1b { z18.b, z22.b, z26.b, z30.b }, pn8.b/Z, [x27]\n" "subs x21, x21, #0x1\n" "addvl x27, x27, #4\n" - "ld1b { z0.b }, p1/Z, [x23]\n" + "ld1b { z15.b }, p1/Z, [x23]\n" "addvl x23, x23, #1\n" - ".inst 0xa0802640 // smopa za0.s, p1/M, p1/M, z18.b, z0.b\n" - ".inst 0xa08026c1 // smopa za1.s, p1/M, p1/M, z22.b, z0.b\n" - ".inst 0xa0802742 // smopa za2.s, p1/M, p1/M, z26.b, z0.b\n" - ".inst 0xa08027c3 // smopa za3.s, p1/M, p1/M, z30.b, z0.b\n" + ".inst 0xa08f2640 // smopa za0.s, p1/M, p1/M, z18.b, z15.b\n" + ".inst 0xa08f26c1 // smopa za1.s, p1/M, p1/M, z22.b, z15.b\n" + ".inst 0xa08f2742 // smopa za2.s, p1/M, p1/M, z26.b, z15.b\n" + ".inst 0xa08f27c3 // smopa za3.s, p1/M, p1/M, z30.b, z15.b\n" "bgt 9b\n" "10:" // K oddments: End - ".inst 0xa040c360 // ld1w { z0.s-z3.s }, pn8.b/Z, [x27]\n" + ".inst 0xa140c363 // ld1w { z3.s, z7.s, z11.s, z15.s }, pn8.b/Z, [x27]\n" "addvl x27, x27, #4\n" - ".inst 0xc0912400 // addva za0.s, p1/M, p1/M, z0.s\n" - ".inst 0xc0912421 // addva za1.s, p1/M, p1/M, z1.s\n" - ".inst 0xc0912442 // addva za2.s, p1/M, p1/M, z2.s\n" - ".inst 0xc0912463 // addva za3.s, p1/M, p1/M, z3.s\n" + ".inst 0xc0912460 // addva za0.s, p1/M, p1/M, z3.s\n" + ".inst 0xc09124e1 // addva za1.s, p1/M, p1/M, z7.s\n" + ".inst 0xc0912562 // addva za2.s, p1/M, p1/M, z11.s\n" + ".inst 0xc09125e3 // addva za3.s, p1/M, p1/M, z15.s\n" "tbz x16, #1, 14f\n" "tbz x16, #0, 12f\n" "mov x12, #0x0\n" "cntw x20\n" "11:" // Store to partial result buffer: Store and refill: Loop - ".inst 0xa040c1f4 // ld1w { z20.s-z23.s }, pn8.b/Z, [x15]\n" - ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" - ".inst 0xc0840680 // mova za0h.s[x12], { z20.s-z23.s }\n" - ".inst 0xc0860428 // mova { z8.s-z11.s }, za1h.s[x12]\n" - ".inst 0xa041c1e4 // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" - ".inst 0xc0840481 // mova za1h.s[x12], { z4.s-z7.s }\n" - ".inst 0xc086044c // mova { z12.s-z15.s }, za2h.s[x12]\n" - ".inst 0xc086047c // mova { z28.s-z31.s }, za3h.s[x12]\n" + ".inst 0xa040c1ec // ld1w { z12.s-z15.s }, pn8.b/Z, [x15]\n" + ".inst 0xc0860400 // mova { z0.s-z3.s }, za0h.s[x12]\n" + ".inst 0xc0840580 // mova za0h.s[x12], { z12.s-z15.s }\n" + ".inst 0xc086043c // mova { z28.s-z31.s }, za1h.s[x12]\n" + ".inst 0xa041c1f0 // ld1w { z16.s-z19.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" + ".inst 0xc0840601 // mova za1h.s[x12], { z16.s-z19.s }\n" + ".inst 0xc0860454 // mova { z20.s-z23.s }, za2h.s[x12]\n" + ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" ".inst 0xa042c1e4 // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" ".inst 0xc0840482 // mova za2h.s[x12], { z4.s-z7.s }\n" - ".inst 0xa043c1f4 // ld1w { z20.s-z23.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840683 // mova za3h.s[x12], { z20.s-z23.s }\n" + ".inst 0xa043c1e8 // ld1w { z8.s-z11.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840503 // mova za3h.s[x12], { z8.s-z11.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" - ".inst 0xa060c1d0 // st1w { z16.s-z19.s }, pn8.b, [x14]\n" + ".inst 0xa060c1c0 // st1w { z0.s-z3.s }, pn8.b, [x14]\n" "addvl x15, x15, #16\n" - ".inst 0xa061c1c8 // st1w { z8.s-z11.s }, pn8.b, [x14, #0x4, MUL VL]\n" - ".inst 0xa062c1cc // st1w { z12.s-z15.s }, pn8.b, [x14, #0x8, MUL VL]\n" - ".inst 0xa063c1dc // st1w { z28.s-z31.s }, pn8.b, [x14, #0xc, MUL VL]\n" + ".inst 0xa061c1dc // st1w { z28.s-z31.s }, pn8.b, [x14, #0x4, MUL VL]\n" + ".inst 0xa062c1d4 // st1w { z20.s-z23.s }, pn8.b, [x14, #0x8, MUL VL]\n" + ".inst 0xa063c1cc // st1w { z12.s-z15.s }, pn8.b, [x14, #0xc, MUL VL]\n" "addvl x14, x14, #16\n" "blt 11b\n" "b 30f\n" @@ -260,16 +259,16 @@ void sme2_interleaved_nomerge_s8q_mopa_4VLx1VL(const int8_t *const A, const int8 "mov x12, #0x0\n" "cntw x20\n" "13:" // Store to partial result buffer: Store only: Loop - ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" + ".inst 0xc0860400 // mova { z0.s-z3.s }, za0h.s[x12]\n" ".inst 0xc086042c // mova { z12.s-z15.s }, za1h.s[x12]\n" - ".inst 0xa060c1d0 // st1w { z16.s-z19.s }, pn8.b, [x14]\n" - ".inst 0xc0860454 // mova { z20.s-z23.s }, za2h.s[x12]\n" - ".inst 0xc0860478 // mova { z24.s-z27.s }, za3h.s[x12]\n" + ".inst 0xa060c1c0 // st1w { z0.s-z3.s }, pn8.b, [x14]\n" + ".inst 0xc0860444 // mova { z4.s-z7.s }, za2h.s[x12]\n" + ".inst 0xc0860460 // mova { z0.s-z3.s }, za3h.s[x12]\n" ".inst 0xa061c1cc // st1w { z12.s-z15.s }, pn8.b, [x14, #0x4, MUL VL]\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" - ".inst 0xa062c1d4 // st1w { z20.s-z23.s }, pn8.b, [x14, #0x8, MUL VL]\n" - ".inst 0xa063c1d8 // st1w { z24.s-z27.s }, pn8.b, [x14, #0xc, MUL VL]\n" + ".inst 0xa062c1c4 // st1w { z4.s-z7.s }, pn8.b, [x14, #0x8, MUL VL]\n" + ".inst 0xa063c1c0 // st1w { z0.s-z3.s }, pn8.b, [x14, #0xc, MUL VL]\n" "addvl x14, x14, #16\n" "blt 13b\n" "b 30f\n" @@ -277,22 +276,22 @@ void sme2_interleaved_nomerge_s8q_mopa_4VLx1VL(const int8_t *const A, const int8 "ldr x26, [%x[args], %[offsetof_C]]\n" "add x26, x26, x10\n" // C += n "sub x25, x13, x11\n" - "ld1rw { z8.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z2.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" "ldr x24, [%x[args], %[offsetof_ldcb]]\n" "madd x26, x11, x24, x26\n" // C += m * ldc - "ld1rw { z7.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" - "ld1rw { z6.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_c_offset]]\n" - "ld1rw { z5.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_minval]]\n" - "ld1rw { z4.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_maxval]]\n" + "ld1rw { z1.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z0.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_c_offset]]\n" + "ld1rw { z21.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_minval]]\n" + "ld1rw { z20.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_maxval]]\n" "tbz x16, #2, 15f\n" "ldr w21, [%x[args], %[offsetof_n_0]]\n" "add x21, x21, x10\n" "ldr x20, [%x[rq], %[offsetof_Requantize32_per_channel_muls]]\n" "add x20, x20, x21, LSL #2\n" - "ld1w { z8.s }, p0/Z, [x20]\n" + "ld1w { z2.s }, p0/Z, [x20]\n" "ldr x20, [%x[rq], %[offsetof_Requantize32_per_channel_right_shifts]]\n" "add x20, x20, x21, LSL #2\n" - "ld1w { z7.s }, p0/Z, [x20]\n" + "ld1w { z1.s }, p0/Z, [x20]\n" "15:" // Store to output array: Load per-channel parameters: End "cntw x23\n" "whilelt p0.s, x10, x9\n" @@ -303,30 +302,30 @@ void sme2_interleaved_nomerge_s8q_mopa_4VLx1VL(const int8_t *const A, const int8 "and x20, x22, #0x3\n" "cbz x21, 17f\n" "16:" // Store to output array: Accumulator row 0 loop - ".inst 0xc086040c // mova { z12.s-z15.s }, za0h.s[x12]\n" - ".inst 0xc1a8ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z8.s\n" + ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" + ".inst 0xc1a2ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z2.s\n" "add x12, x12, #0x4\n" - ".inst 0xc1a7aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z7.s\n" + ".inst 0xc1a1aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z1.s\n" "cmp x12, x21, LSL #2\n" - ".inst 0xc1a6ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z6.s\n" - ".inst 0xc1a4ccac // sclamp { z12.s-z15.s }, z5.s, z4.s\n" - "st1b { z12.s }, p0, [x26]\n" + ".inst 0xc1a0ab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z0.s\n" + ".inst 0xc1b4ceb0 // sclamp { z16.s-z19.s }, z21.s, z20.s\n" + "st1b { z16.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1b { z13.s }, p0, [x26]\n" + "st1b { z17.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1b { z14.s }, p0, [x26]\n" + "st1b { z18.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1b { z15.s }, p0, [x26]\n" + "st1b { z19.s }, p0, [x26]\n" "add x26, x26, x24\n" "blt 16b\n" "17:" // Store to output array: Accumulator row 0 oddments "cbz x20, 18f\n" ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" - ".inst 0xc1a8ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z8.s\n" + ".inst 0xc1a2ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z2.s\n" "subs x20, x20, #0x1\n" - ".inst 0xc1a7aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z7.s\n" - ".inst 0xc1a6ab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n" - ".inst 0xc1a4ccb0 // sclamp { z16.s-z19.s }, z5.s, z4.s\n" + ".inst 0xc1a1aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z1.s\n" + ".inst 0xc1a0ab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z0.s\n" + ".inst 0xc1b4ceb0 // sclamp { z16.s-z19.s }, z21.s, z20.s\n" "st1b { z16.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 18f\n" @@ -347,38 +346,38 @@ void sme2_interleaved_nomerge_s8q_mopa_4VLx1VL(const int8_t *const A, const int8 "and x20, x22, #0x3\n" "cbz x21, 20f\n" "19:" // Store to output array: Accumulator row 1 loop - ".inst 0xc0860430 // mova { z16.s-z19.s }, za1h.s[x12]\n" - ".inst 0xc1a8ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z8.s\n" + ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" + ".inst 0xc1a2ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z2.s\n" "add x12, x12, #0x4\n" - ".inst 0xc1a7aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z7.s\n" + ".inst 0xc1a1aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" "cmp x12, x21, LSL #2\n" - ".inst 0xc1a6ab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n" - ".inst 0xc1a4ccb0 // sclamp { z16.s-z19.s }, z5.s, z4.s\n" - "st1b { z16.s }, p0, [x26]\n" + ".inst 0xc1a0ab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z0.s\n" + ".inst 0xc1b4cea4 // sclamp { z4.s-z7.s }, z21.s, z20.s\n" + "st1b { z4.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1b { z17.s }, p0, [x26]\n" + "st1b { z5.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1b { z18.s }, p0, [x26]\n" + "st1b { z6.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1b { z19.s }, p0, [x26]\n" + "st1b { z7.s }, p0, [x26]\n" "add x26, x26, x24\n" "blt 19b\n" "20:" // Store to output array: Accumulator row 1 oddments "cbz x20, 21f\n" - ".inst 0xc086043c // mova { z28.s-z31.s }, za1h.s[x12]\n" - ".inst 0xc1a8ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z8.s\n" + ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" + ".inst 0xc1a2ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z2.s\n" "subs x20, x20, #0x1\n" - ".inst 0xc1a7aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z7.s\n" - ".inst 0xc1a6ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z6.s\n" - ".inst 0xc1a4ccbc // sclamp { z28.s-z31.s }, z5.s, z4.s\n" - "st1b { z28.s }, p0, [x26]\n" + ".inst 0xc1a1aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" + ".inst 0xc1a0ab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z0.s\n" + ".inst 0xc1b4cea4 // sclamp { z4.s-z7.s }, z21.s, z20.s\n" + "st1b { z4.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 21f\n" "subs x20, x20, #0x1\n" - "st1b { z29.s }, p0, [x26]\n" + "st1b { z5.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 21f\n" - "st1b { z30.s }, p0, [x26]\n" + "st1b { z6.s }, p0, [x26]\n" "add x26, x26, x24\n" "21:" // Store to output array: Accumulator row 1 oddments: End "subs x25, x25, x22\n" @@ -391,30 +390,30 @@ void sme2_interleaved_nomerge_s8q_mopa_4VLx1VL(const int8_t *const A, const int8 "and x20, x22, #0x3\n" "cbz x21, 23f\n" "22:" // Store to output array: Accumulator row 2 loop - ".inst 0xc0860458 // mova { z24.s-z27.s }, za2h.s[x12]\n" - ".inst 0xc1a8ac18 // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z8.s\n" + ".inst 0xc0860448 // mova { z8.s-z11.s }, za2h.s[x12]\n" + ".inst 0xc1a2ac08 // sqdmulh { z8.s-z11.s }, { z8.s-z11.s }, z2.s\n" "add x12, x12, #0x4\n" - ".inst 0xc1a7aa38 // srshl { z24.s-z27.s }, { z24.s-z27.s }, z7.s\n" + ".inst 0xc1a1aa28 // srshl { z8.s-z11.s }, { z8.s-z11.s }, z1.s\n" "cmp x12, x21, LSL #2\n" - ".inst 0xc1a6ab18 // add { z24.s-z27.s }, { z24.s-z27.s }, z6.s\n" - ".inst 0xc1a4ccb8 // sclamp { z24.s-z27.s }, z5.s, z4.s\n" - "st1b { z24.s }, p0, [x26]\n" + ".inst 0xc1a0ab08 // add { z8.s-z11.s }, { z8.s-z11.s }, z0.s\n" + ".inst 0xc1b4cea8 // sclamp { z8.s-z11.s }, z21.s, z20.s\n" + "st1b { z8.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1b { z25.s }, p0, [x26]\n" + "st1b { z9.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1b { z26.s }, p0, [x26]\n" + "st1b { z10.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1b { z27.s }, p0, [x26]\n" + "st1b { z11.s }, p0, [x26]\n" "add x26, x26, x24\n" "blt 22b\n" "23:" // Store to output array: Accumulator row 2 oddments "cbz x20, 24f\n" ".inst 0xc086044c // mova { z12.s-z15.s }, za2h.s[x12]\n" - ".inst 0xc1a8ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z8.s\n" + ".inst 0xc1a2ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z2.s\n" "subs x20, x20, #0x1\n" - ".inst 0xc1a7aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z7.s\n" - ".inst 0xc1a6ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z6.s\n" - ".inst 0xc1a4ccac // sclamp { z12.s-z15.s }, z5.s, z4.s\n" + ".inst 0xc1a1aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n" + ".inst 0xc1a0ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z0.s\n" + ".inst 0xc1b4ceac // sclamp { z12.s-z15.s }, z21.s, z20.s\n" "st1b { z12.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 24f\n" @@ -435,52 +434,52 @@ void sme2_interleaved_nomerge_s8q_mopa_4VLx1VL(const int8_t *const A, const int8 "and x20, x20, #0x3\n" "cbz x21, 26f\n" "25:" // Store to output array: Accumulator row 3 loop - ".inst 0xc0860474 // mova { z20.s-z23.s }, za3h.s[x12]\n" - ".inst 0xc1a8ac14 // sqdmulh { z20.s-z23.s }, { z20.s-z23.s }, z8.s\n" + ".inst 0xc086047c // mova { z28.s-z31.s }, za3h.s[x12]\n" + ".inst 0xc1a2ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z2.s\n" "add x12, x12, #0x4\n" - ".inst 0xc1a7aa34 // srshl { z20.s-z23.s }, { z20.s-z23.s }, z7.s\n" + ".inst 0xc1a1aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z1.s\n" "cmp x12, x21, LSL #2\n" - ".inst 0xc1a6ab14 // add { z20.s-z23.s }, { z20.s-z23.s }, z6.s\n" - ".inst 0xc1a4ccb4 // sclamp { z20.s-z23.s }, z5.s, z4.s\n" - "st1b { z20.s }, p0, [x26]\n" + ".inst 0xc1a0ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z0.s\n" + ".inst 0xc1b4cebc // sclamp { z28.s-z31.s }, z21.s, z20.s\n" + "st1b { z28.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1b { z21.s }, p0, [x26]\n" + "st1b { z29.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1b { z22.s }, p0, [x26]\n" + "st1b { z30.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1b { z23.s }, p0, [x26]\n" + "st1b { z31.s }, p0, [x26]\n" "add x26, x26, x24\n" "blt 25b\n" "26:" // Store to output array: Accumulator row 3 oddments "cbz x20, 27f\n" - ".inst 0xc0860460 // mova { z0.s-z3.s }, za3h.s[x12]\n" - ".inst 0xc1a8ac00 // sqdmulh { z0.s-z3.s }, { z0.s-z3.s }, z8.s\n" + ".inst 0xc086047c // mova { z28.s-z31.s }, za3h.s[x12]\n" + ".inst 0xc1a2ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z2.s\n" "subs x20, x20, #0x1\n" - ".inst 0xc1a7aa20 // srshl { z0.s-z3.s }, { z0.s-z3.s }, z7.s\n" - ".inst 0xc1a6ab00 // add { z0.s-z3.s }, { z0.s-z3.s }, z6.s\n" - ".inst 0xc1a4cca0 // sclamp { z0.s-z3.s }, z5.s, z4.s\n" - "st1b { z0.s }, p0, [x26]\n" + ".inst 0xc1a1aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z1.s\n" + ".inst 0xc1a0ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z0.s\n" + ".inst 0xc1b4cebc // sclamp { z28.s-z31.s }, z21.s, z20.s\n" + "st1b { z28.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 27f\n" "subs x20, x20, #0x1\n" - "st1b { z1.s }, p0, [x26]\n" + "st1b { z29.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 27f\n" - "st1b { z2.s }, p0, [x26]\n" + "st1b { z30.s }, p0, [x26]\n" "27:" // Store to output array: Accumulator row 3 oddments: End "28:" // Store to output array: End "tbz x16, #0, 30f\n" "mov x12, #0x0\n" "cntw x20\n" "29:" // Store to output array: Refill accumulators: Loop - ".inst 0xa040c1e4 // ld1w { z4.s-z7.s }, pn8.b/Z, [x15]\n" - ".inst 0xc0840480 // mova za0h.s[x12], { z4.s-z7.s }\n" + ".inst 0xa040c1fc // ld1w { z28.s-z31.s }, pn8.b/Z, [x15]\n" + ".inst 0xc0840780 // mova za0h.s[x12], { z28.s-z31.s }\n" ".inst 0xa041c1f0 // ld1w { z16.s-z19.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" ".inst 0xc0840601 // mova za1h.s[x12], { z16.s-z19.s }\n" ".inst 0xa042c1f0 // ld1w { z16.s-z19.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" - ".inst 0xa043c1e4 // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840483 // mova za3h.s[x12], { z4.s-z7.s }\n" + ".inst 0xa043c1e0 // ld1w { z0.s-z3.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840403 // mova za3h.s[x12], { z0.s-z3.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x15, x15, #16\n" @@ -504,4 +503,3 @@ void sme2_interleaved_nomerge_s8q_mopa_4VLx1VL(const int8_t *const A, const int8 } // namespace arm_gemm #endif // ARM_COMPUTE_ENABLE_SME2 -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL.hpp index c969c7aaff6f8eeb32fa8e6dd6f493f055413f84..b9d8b60c8d85b054c4304ab5d9b1ce1f6443bca7 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,7 +23,7 @@ */ #pragma once -#ifdef __aarch64__ +#ifdef ARM_COMPUTE_ENABLE_SME2 #include #include "../std_transforms_sme.hpp" @@ -32,7 +32,7 @@ namespace arm_gemm { // Implementations -void sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL(const int8_t *const A, const int8_t *const B, int32_t *const C, int ldc, const int M, const int N, const int K, const int32_t *const bias, const Activation act, bool accumulate, int32_t *const accumulator_buffer); +void sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL(const int8_t *const A, const int8_t *const B, int32_t *const C, int ldc, const int M, const int N, const int K, const int32_t *const bias, const Activation, bool accumulate, int32_t *const accumulator_buffer); class cls_sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL { @@ -40,7 +40,7 @@ public: typedef int8_t operand_type; typedef int32_t result_type; - typedef void (*kern_type)(const int8_t *const A, const int8_t *const B, int32_t *const C, int ldc, const int M, const int N, const int K, const int32_t *const bias, const Activation act, bool accumulate, int32_t *const accumulator_buffer); + typedef void (*kern_type)(const int8_t *const A, const int8_t *const B, int32_t *const C, int ldc, const int M, const int N, const int K, const int32_t *const bias, const Activation, bool accumulate, int32_t *const accumulator_buffer); /* Kernel blocking parameters */ static unsigned int out_height() @@ -83,12 +83,11 @@ public: StdTransformsSME transforms = {}; - cls_sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL(const CPUInfo *ci) + cls_sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL(const CPUInfo *) { - ARM_COMPUTE_UNUSED(ci); } }; } // namespace arm_gemm -#endif // __aarch64__ +#endif // ARM_COMPUTE_ENABLE_SME2 diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL/generic.cpp index 12e714a4713da514340da4961c56dfef794815ff..d11faa634d1600e7147aeb2ef081541cdb2bdce2 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL/generic.cpp @@ -21,7 +21,6 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifdef __ARM_FEATURE_SVE #ifdef ARM_COMPUTE_ENABLE_SME2 #include "arm_gemm.hpp" @@ -32,10 +31,8 @@ namespace arm_gemm { -void sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL(const int8_t *const A, const int8_t *const B, int32_t *const C, int ldc, const int M, const int N, const int K, const int32_t *const bias, const Activation act, bool accumulate, int32_t *const accumulator_buffer) +void sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL(const int8_t *const A, const int8_t *const B, int32_t *const C, int ldc, const int M, const int N, const int K, const int32_t *const bias, const Activation, bool accumulate, int32_t *const accumulator_buffer) { - ARM_COMPUTE_UNUSED(act); - struct KernelArgs { KernelArgs( @@ -96,12 +93,12 @@ void sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL(const int8_t *const A, const in "1:" // Initial accumulator load from buffer: Loop ".inst 0xa040c564 // ld1w { z4.s-z7.s }, pn9.b/Z, [x11]\n" ".inst 0xc0840480 // mova za0h.s[x12], { z4.s-z7.s }\n" - ".inst 0xa041c57c // ld1w { z28.s-z31.s }, pn9.b/Z, [x11, #0x4, MUL VL]\n" - ".inst 0xc0840781 // mova za1h.s[x12], { z28.s-z31.s }\n" - ".inst 0xa042c570 // ld1w { z16.s-z19.s }, pn9.b/Z, [x11, #0x8, MUL VL]\n" - ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" - ".inst 0xa043c560 // ld1w { z0.s-z3.s }, pn9.b/Z, [x11, #0xc, MUL VL]\n" - ".inst 0xc0840403 // mova za3h.s[x12], { z0.s-z3.s }\n" + ".inst 0xa041c56c // ld1w { z12.s-z15.s }, pn9.b/Z, [x11, #0x4, MUL VL]\n" + ".inst 0xc0840581 // mova za1h.s[x12], { z12.s-z15.s }\n" + ".inst 0xa042c57c // ld1w { z28.s-z31.s }, pn9.b/Z, [x11, #0x8, MUL VL]\n" + ".inst 0xc0840782 // mova za2h.s[x12], { z28.s-z31.s }\n" + ".inst 0xa043c574 // ld1w { z20.s-z23.s }, pn9.b/Z, [x11, #0xc, MUL VL]\n" + ".inst 0xc0840683 // mova za3h.s[x12], { z20.s-z23.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x11, x11, #16\n" @@ -119,11 +116,11 @@ void sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL(const int8_t *const A, const in "ldr x20, [%x[args], %[offsetof_bias]]\n" ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "cbz x20, 5f\n" - ".inst 0xa11bc28a // ldnt1w { z2.s, z6.s, z10.s, z14.s }, p8/Z, [x20, x27, LSL #2]\n" - ".inst 0xc0900040 // addha za0.s, p0/M, p0/M, z2.s\n" - ".inst 0xc09000c1 // addha za1.s, p0/M, p0/M, z6.s\n" - ".inst 0xc0900142 // addha za2.s, p0/M, p0/M, z10.s\n" - ".inst 0xc09001c3 // addha za3.s, p0/M, p0/M, z14.s\n" + ".inst 0xa11bc29b // ldnt1w { z19.s, z23.s, z27.s, z31.s }, p8/Z, [x20, x27, LSL #2]\n" + ".inst 0xc0900260 // addha za0.s, p0/M, p0/M, z19.s\n" + ".inst 0xc09002e1 // addha za1.s, p0/M, p0/M, z23.s\n" + ".inst 0xc0900362 // addha za2.s, p0/M, p0/M, z27.s\n" + ".inst 0xc09003e3 // addha za3.s, p0/M, p0/M, z31.s\n" "4:" // Prepare accumulators: Test for last block "mov x20, x27\n" "mov x21, x28\n" @@ -146,75 +143,75 @@ void sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL(const int8_t *const A, const in "madd x23, x27, x20, x23\n" // bptr = B + n * kstride_bytes "cbz x22, 8f\n" "subs x22, x22, #0x1\n" - "ld1b { z20.b }, p0/Z, [x24]\n" - ".inst 0xa14086e9 // ldnt1b { z1.b, z5.b, z9.b, z13.b }, pn9.b/Z, [x23]\n" - "ld1b { z10.b }, p0/Z, [x24, #1, MUL VL]\n" - ".inst 0xa14186fa // ldnt1b { z18.b, z22.b, z26.b, z30.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n" - "ld1b { z16.b }, p0/Z, [x24, #2, MUL VL]\n" - ".inst 0xa14286eb // ldnt1b { z3.b, z7.b, z11.b, z15.b }, pn9.b/Z, [x23, #0x8, MUL VL]\n" - "ld1b { z25.b }, p0/Z, [x24, #3, MUL VL]\n" + "ld1b { z30.b }, p0/Z, [x24]\n" + ".inst 0xa04086e1 // ldnt1b { z0.b-z3.b }, pn9.b/Z, [x23]\n" + "ld1b { z21.b }, p0/Z, [x24, #1, MUL VL]\n" + ".inst 0xa04186f9 // ldnt1b { z24.b-z27.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n" + "ld1b { z28.b }, p0/Z, [x24, #2, MUL VL]\n" + ".inst 0xa04286e5 // ldnt1b { z4.b-z7.b }, pn9.b/Z, [x23, #0x8, MUL VL]\n" + "ld1b { z11.b }, p0/Z, [x24, #3, MUL VL]\n" "addvl x24, x24, #4\n" - ".inst 0xa14386e8 // ldnt1b { z0.b, z4.b, z8.b, z12.b }, pn9.b/Z, [x23, #0xc, MUL VL]\n" + ".inst 0xa04386f1 // ldnt1b { z16.b-z19.b }, pn9.b/Z, [x23, #0xc, MUL VL]\n" "addvl x23, x23, #16\n" "ble 7f\n" "6:" // K loop - ".inst 0xa0810280 // smopa za0.s, p0/M, p0/M, z20.b, z1.b\n" + ".inst 0xa08003c0 // smopa za0.s, p0/M, p0/M, z30.b, z0.b\n" "subs x22, x22, #0x1\n" - ".inst 0xa0850281 // smopa za1.s, p0/M, p0/M, z20.b, z5.b\n" - ".inst 0xa0890282 // smopa za2.s, p0/M, p0/M, z20.b, z9.b\n" - ".inst 0xa08d0283 // smopa za3.s, p0/M, p0/M, z20.b, z13.b\n" - "ld1b { z20.b }, p0/Z, [x24]\n" - ".inst 0xa0920140 // smopa za0.s, p0/M, p0/M, z10.b, z18.b\n" - ".inst 0xa14086e9 // ldnt1b { z1.b, z5.b, z9.b, z13.b }, pn9.b/Z, [x23]\n" - ".inst 0xa0960141 // smopa za1.s, p0/M, p0/M, z10.b, z22.b\n" - ".inst 0xa09a0142 // smopa za2.s, p0/M, p0/M, z10.b, z26.b\n" - ".inst 0xa09e0143 // smopa za3.s, p0/M, p0/M, z10.b, z30.b\n" - "ld1b { z10.b }, p0/Z, [x24, #1, MUL VL]\n" - ".inst 0xa0830200 // smopa za0.s, p0/M, p0/M, z16.b, z3.b\n" - ".inst 0xa14186fa // ldnt1b { z18.b, z22.b, z26.b, z30.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xa0870201 // smopa za1.s, p0/M, p0/M, z16.b, z7.b\n" - ".inst 0xa08b0202 // smopa za2.s, p0/M, p0/M, z16.b, z11.b\n" - ".inst 0xa08f0203 // smopa za3.s, p0/M, p0/M, z16.b, z15.b\n" - "ld1b { z16.b }, p0/Z, [x24, #2, MUL VL]\n" - ".inst 0xa14286eb // ldnt1b { z3.b, z7.b, z11.b, z15.b }, pn9.b/Z, [x23, #0x8, MUL VL]\n" - ".inst 0xa0800320 // smopa za0.s, p0/M, p0/M, z25.b, z0.b\n" - ".inst 0xa0840321 // smopa za1.s, p0/M, p0/M, z25.b, z4.b\n" - ".inst 0xa0880322 // smopa za2.s, p0/M, p0/M, z25.b, z8.b\n" - ".inst 0xa08c0323 // smopa za3.s, p0/M, p0/M, z25.b, z12.b\n" - "ld1b { z25.b }, p0/Z, [x24, #3, MUL VL]\n" + ".inst 0xa08103c1 // smopa za1.s, p0/M, p0/M, z30.b, z1.b\n" + ".inst 0xa08203c2 // smopa za2.s, p0/M, p0/M, z30.b, z2.b\n" + ".inst 0xa08303c3 // smopa za3.s, p0/M, p0/M, z30.b, z3.b\n" + "ld1b { z30.b }, p0/Z, [x24]\n" + ".inst 0xa09802a0 // smopa za0.s, p0/M, p0/M, z21.b, z24.b\n" + ".inst 0xa04086e1 // ldnt1b { z0.b-z3.b }, pn9.b/Z, [x23]\n" + ".inst 0xa09902a1 // smopa za1.s, p0/M, p0/M, z21.b, z25.b\n" + ".inst 0xa09a02a2 // smopa za2.s, p0/M, p0/M, z21.b, z26.b\n" + ".inst 0xa09b02a3 // smopa za3.s, p0/M, p0/M, z21.b, z27.b\n" + "ld1b { z21.b }, p0/Z, [x24, #1, MUL VL]\n" + ".inst 0xa0840380 // smopa za0.s, p0/M, p0/M, z28.b, z4.b\n" + ".inst 0xa04186f9 // ldnt1b { z24.b-z27.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xa0850381 // smopa za1.s, p0/M, p0/M, z28.b, z5.b\n" + ".inst 0xa0860382 // smopa za2.s, p0/M, p0/M, z28.b, z6.b\n" + ".inst 0xa0870383 // smopa za3.s, p0/M, p0/M, z28.b, z7.b\n" + "ld1b { z28.b }, p0/Z, [x24, #2, MUL VL]\n" + ".inst 0xa04286e5 // ldnt1b { z4.b-z7.b }, pn9.b/Z, [x23, #0x8, MUL VL]\n" + ".inst 0xa0900160 // smopa za0.s, p0/M, p0/M, z11.b, z16.b\n" + ".inst 0xa0910161 // smopa za1.s, p0/M, p0/M, z11.b, z17.b\n" + ".inst 0xa0920162 // smopa za2.s, p0/M, p0/M, z11.b, z18.b\n" + ".inst 0xa0930163 // smopa za3.s, p0/M, p0/M, z11.b, z19.b\n" + "ld1b { z11.b }, p0/Z, [x24, #3, MUL VL]\n" "addvl x24, x24, #4\n" - ".inst 0xa14386e8 // ldnt1b { z0.b, z4.b, z8.b, z12.b }, pn9.b/Z, [x23, #0xc, MUL VL]\n" + ".inst 0xa04386f1 // ldnt1b { z16.b-z19.b }, pn9.b/Z, [x23, #0xc, MUL VL]\n" "addvl x23, x23, #16\n" "bgt 6b\n" "7:" // K loop tail - ".inst 0xa0810280 // smopa za0.s, p0/M, p0/M, z20.b, z1.b\n" - ".inst 0xa0850281 // smopa za1.s, p0/M, p0/M, z20.b, z5.b\n" - ".inst 0xa0890282 // smopa za2.s, p0/M, p0/M, z20.b, z9.b\n" - ".inst 0xa08d0283 // smopa za3.s, p0/M, p0/M, z20.b, z13.b\n" - ".inst 0xa0920140 // smopa za0.s, p0/M, p0/M, z10.b, z18.b\n" - ".inst 0xa0960141 // smopa za1.s, p0/M, p0/M, z10.b, z22.b\n" - ".inst 0xa09a0142 // smopa za2.s, p0/M, p0/M, z10.b, z26.b\n" - ".inst 0xa09e0143 // smopa za3.s, p0/M, p0/M, z10.b, z30.b\n" - ".inst 0xa0830200 // smopa za0.s, p0/M, p0/M, z16.b, z3.b\n" - ".inst 0xa0870201 // smopa za1.s, p0/M, p0/M, z16.b, z7.b\n" - ".inst 0xa08b0202 // smopa za2.s, p0/M, p0/M, z16.b, z11.b\n" - ".inst 0xa08f0203 // smopa za3.s, p0/M, p0/M, z16.b, z15.b\n" - ".inst 0xa0800320 // smopa za0.s, p0/M, p0/M, z25.b, z0.b\n" - ".inst 0xa0840321 // smopa za1.s, p0/M, p0/M, z25.b, z4.b\n" - ".inst 0xa0880322 // smopa za2.s, p0/M, p0/M, z25.b, z8.b\n" - ".inst 0xa08c0323 // smopa za3.s, p0/M, p0/M, z25.b, z12.b\n" + ".inst 0xa08003c0 // smopa za0.s, p0/M, p0/M, z30.b, z0.b\n" + ".inst 0xa08103c1 // smopa za1.s, p0/M, p0/M, z30.b, z1.b\n" + ".inst 0xa08203c2 // smopa za2.s, p0/M, p0/M, z30.b, z2.b\n" + ".inst 0xa08303c3 // smopa za3.s, p0/M, p0/M, z30.b, z3.b\n" + ".inst 0xa09802a0 // smopa za0.s, p0/M, p0/M, z21.b, z24.b\n" + ".inst 0xa09902a1 // smopa za1.s, p0/M, p0/M, z21.b, z25.b\n" + ".inst 0xa09a02a2 // smopa za2.s, p0/M, p0/M, z21.b, z26.b\n" + ".inst 0xa09b02a3 // smopa za3.s, p0/M, p0/M, z21.b, z27.b\n" + ".inst 0xa0840380 // smopa za0.s, p0/M, p0/M, z28.b, z4.b\n" + ".inst 0xa0850381 // smopa za1.s, p0/M, p0/M, z28.b, z5.b\n" + ".inst 0xa0860382 // smopa za2.s, p0/M, p0/M, z28.b, z6.b\n" + ".inst 0xa0870383 // smopa za3.s, p0/M, p0/M, z28.b, z7.b\n" + ".inst 0xa0900160 // smopa za0.s, p0/M, p0/M, z11.b, z16.b\n" + ".inst 0xa0910161 // smopa za1.s, p0/M, p0/M, z11.b, z17.b\n" + ".inst 0xa0920162 // smopa za2.s, p0/M, p0/M, z11.b, z18.b\n" + ".inst 0xa0930163 // smopa za3.s, p0/M, p0/M, z11.b, z19.b\n" "8:" // K oddments "cbz x21, 10f\n" "9:" // K oddments: Loop - "ld1b { z20.b }, p0/Z, [x24]\n" + "ld1b { z22.b }, p0/Z, [x24]\n" "subs x21, x21, #0x1\n" "addvl x24, x24, #1\n" - ".inst 0xa14086e1 // ld1b { z1.b, z5.b, z9.b, z13.b }, pn9.b/Z, [x23]\n" + ".inst 0xa14086f1 // ld1b { z17.b, z21.b, z25.b, z29.b }, pn9.b/Z, [x23]\n" "addvl x23, x23, #4\n" - ".inst 0xa0810280 // smopa za0.s, p0/M, p0/M, z20.b, z1.b\n" - ".inst 0xa0850281 // smopa za1.s, p0/M, p0/M, z20.b, z5.b\n" - ".inst 0xa0890282 // smopa za2.s, p0/M, p0/M, z20.b, z9.b\n" - ".inst 0xa08d0283 // smopa za3.s, p0/M, p0/M, z20.b, z13.b\n" + ".inst 0xa09102c0 // smopa za0.s, p0/M, p0/M, z22.b, z17.b\n" + ".inst 0xa09502c1 // smopa za1.s, p0/M, p0/M, z22.b, z21.b\n" + ".inst 0xa09902c2 // smopa za2.s, p0/M, p0/M, z22.b, z25.b\n" + ".inst 0xa09d02c3 // smopa za3.s, p0/M, p0/M, z22.b, z29.b\n" "bgt 9b\n" "10:" // K oddments: End "tbz x13, #1, 14f\n" @@ -222,25 +219,25 @@ void sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL(const int8_t *const A, const in "mov x12, #0x0\n" "cntw x20\n" "11:" // Store to partial result buffer: Store and refill: Loop - ".inst 0xa040c57c // ld1w { z28.s-z31.s }, pn9.b/Z, [x11]\n" + ".inst 0xa040c570 // ld1w { z16.s-z19.s }, pn9.b/Z, [x11]\n" ".inst 0xc0860404 // mova { z4.s-z7.s }, za0h.s[x12]\n" - ".inst 0xc0840780 // mova za0h.s[x12], { z28.s-z31.s }\n" - ".inst 0xc0860434 // mova { z20.s-z23.s }, za1h.s[x12]\n" - ".inst 0xa041c560 // ld1w { z0.s-z3.s }, pn9.b/Z, [x11, #0x4, MUL VL]\n" - ".inst 0xc0840401 // mova za1h.s[x12], { z0.s-z3.s }\n" - ".inst 0xc0860458 // mova { z24.s-z27.s }, za2h.s[x12]\n" - ".inst 0xc086047c // mova { z28.s-z31.s }, za3h.s[x12]\n" - ".inst 0xa042c570 // ld1w { z16.s-z19.s }, pn9.b/Z, [x11, #0x8, MUL VL]\n" - ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" - ".inst 0xa043c56c // ld1w { z12.s-z15.s }, pn9.b/Z, [x11, #0xc, MUL VL]\n" - ".inst 0xc0840583 // mova za3h.s[x12], { z12.s-z15.s }\n" + ".inst 0xc0840600 // mova za0h.s[x12], { z16.s-z19.s }\n" + ".inst 0xc086042c // mova { z12.s-z15.s }, za1h.s[x12]\n" + ".inst 0xa041c570 // ld1w { z16.s-z19.s }, pn9.b/Z, [x11, #0x4, MUL VL]\n" + ".inst 0xc0840601 // mova za1h.s[x12], { z16.s-z19.s }\n" + ".inst 0xc086045c // mova { z28.s-z31.s }, za2h.s[x12]\n" + ".inst 0xc0860470 // mova { z16.s-z19.s }, za3h.s[x12]\n" + ".inst 0xa042c560 // ld1w { z0.s-z3.s }, pn9.b/Z, [x11, #0x8, MUL VL]\n" + ".inst 0xc0840402 // mova za2h.s[x12], { z0.s-z3.s }\n" + ".inst 0xa043c574 // ld1w { z20.s-z23.s }, pn9.b/Z, [x11, #0xc, MUL VL]\n" + ".inst 0xc0840683 // mova za3h.s[x12], { z20.s-z23.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" ".inst 0xa060c544 // st1w { z4.s-z7.s }, pn9.b, [x10]\n" "addvl x11, x11, #16\n" - ".inst 0xa061c554 // st1w { z20.s-z23.s }, pn9.b, [x10, #0x4, MUL VL]\n" - ".inst 0xa062c558 // st1w { z24.s-z27.s }, pn9.b, [x10, #0x8, MUL VL]\n" - ".inst 0xa063c55c // st1w { z28.s-z31.s }, pn9.b, [x10, #0xc, MUL VL]\n" + ".inst 0xa061c54c // st1w { z12.s-z15.s }, pn9.b, [x10, #0x4, MUL VL]\n" + ".inst 0xa062c55c // st1w { z28.s-z31.s }, pn9.b, [x10, #0x8, MUL VL]\n" + ".inst 0xa063c550 // st1w { z16.s-z19.s }, pn9.b, [x10, #0xc, MUL VL]\n" "addvl x10, x10, #16\n" "blt 11b\n" "b 20f\n" @@ -248,16 +245,16 @@ void sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL(const int8_t *const A, const in "mov x12, #0x0\n" "cntw x20\n" "13:" // Store to partial result buffer: Store only: Loop - ".inst 0xc0860414 // mova { z20.s-z23.s }, za0h.s[x12]\n" - ".inst 0xc0860420 // mova { z0.s-z3.s }, za1h.s[x12]\n" - ".inst 0xa060c554 // st1w { z20.s-z23.s }, pn9.b, [x10]\n" - ".inst 0xc0860448 // mova { z8.s-z11.s }, za2h.s[x12]\n" - ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" - ".inst 0xa061c540 // st1w { z0.s-z3.s }, pn9.b, [x10, #0x4, MUL VL]\n" + ".inst 0xc086040c // mova { z12.s-z15.s }, za0h.s[x12]\n" + ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" + ".inst 0xa060c54c // st1w { z12.s-z15.s }, pn9.b, [x10]\n" + ".inst 0xc0860440 // mova { z0.s-z3.s }, za2h.s[x12]\n" + ".inst 0xc0860478 // mova { z24.s-z27.s }, za3h.s[x12]\n" + ".inst 0xa061c544 // st1w { z4.s-z7.s }, pn9.b, [x10, #0x4, MUL VL]\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" - ".inst 0xa062c548 // st1w { z8.s-z11.s }, pn9.b, [x10, #0x8, MUL VL]\n" - ".inst 0xa063c54c // st1w { z12.s-z15.s }, pn9.b, [x10, #0xc, MUL VL]\n" + ".inst 0xa062c540 // st1w { z0.s-z3.s }, pn9.b, [x10, #0x8, MUL VL]\n" + ".inst 0xa063c558 // st1w { z24.s-z27.s }, pn9.b, [x10, #0xc, MUL VL]\n" "addvl x10, x10, #16\n" "blt 13b\n" "b 20f\n" @@ -293,32 +290,32 @@ void sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL(const int8_t *const A, const in "16:" // Store to output array: Accumulator row 0 oddments "cbz x20, 17f\n" "subs x20, x20, #0x1\n" - ".inst 0xc0860400 // mova { z0.s-z3.s }, za0h.s[x12]\n" - ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" - ".inst 0xc0860448 // mova { z8.s-z11.s }, za2h.s[x12]\n" - ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" - ".inst 0xa160c2e0 // st1w { z0.s, z4.s, z8.s, z12.s }, p8, [x23]\n" + ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" + ".inst 0xc0860434 // mova { z20.s-z23.s }, za1h.s[x12]\n" + ".inst 0xc0860458 // mova { z24.s-z27.s }, za2h.s[x12]\n" + ".inst 0xc086047c // mova { z28.s-z31.s }, za3h.s[x12]\n" + ".inst 0xa160c2f0 // st1w { z16.s, z20.s, z24.s, z28.s }, p8, [x23]\n" "add x23, x23, x22\n" "beq 17f\n" "subs x20, x20, #0x1\n" - ".inst 0xa160c2e1 // st1w { z1.s, z5.s, z9.s, z13.s }, p8, [x23]\n" + ".inst 0xa160c2f1 // st1w { z17.s, z21.s, z25.s, z29.s }, p8, [x23]\n" "add x23, x23, x22\n" "beq 17f\n" - ".inst 0xa160c2e2 // st1w { z2.s, z6.s, z10.s, z14.s }, p8, [x23]\n" + ".inst 0xa160c2f2 // st1w { z18.s, z22.s, z26.s, z30.s }, p8, [x23]\n" "17:" // Store to output array: Accumulator row 0 oddments: End "18:" // Store to output array: End "tbz x13, #0, 20f\n" "mov x12, #0x0\n" "cntw x20\n" "19:" // Store to output array: Refill accumulators: Loop - ".inst 0xa040c56c // ld1w { z12.s-z15.s }, pn9.b/Z, [x11]\n" - ".inst 0xc0840580 // mova za0h.s[x12], { z12.s-z15.s }\n" + ".inst 0xa040c568 // ld1w { z8.s-z11.s }, pn9.b/Z, [x11]\n" + ".inst 0xc0840500 // mova za0h.s[x12], { z8.s-z11.s }\n" ".inst 0xa041c570 // ld1w { z16.s-z19.s }, pn9.b/Z, [x11, #0x4, MUL VL]\n" ".inst 0xc0840601 // mova za1h.s[x12], { z16.s-z19.s }\n" ".inst 0xa042c570 // ld1w { z16.s-z19.s }, pn9.b/Z, [x11, #0x8, MUL VL]\n" ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" - ".inst 0xa043c564 // ld1w { z4.s-z7.s }, pn9.b/Z, [x11, #0xc, MUL VL]\n" - ".inst 0xc0840483 // mova za3h.s[x12], { z4.s-z7.s }\n" + ".inst 0xa043c568 // ld1w { z8.s-z11.s }, pn9.b/Z, [x11, #0xc, MUL VL]\n" + ".inst 0xc0840503 // mova za3h.s[x12], { z8.s-z11.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x11, x11, #16\n" @@ -342,4 +339,3 @@ void sme2_interleaved_nomerge_s8s32_mopa_1VLx4VL(const int8_t *const A, const in } // namespace arm_gemm #endif // ARM_COMPUTE_ENABLE_SME2 -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL.hpp index a0705e50cd6eebd01927deee7075a2b16773b78d..f05d2cf215d8926cabe1f4e0eeae2a9103a36138 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,7 +23,7 @@ */ #pragma once -#ifdef __aarch64__ +#ifdef ARM_COMPUTE_ENABLE_SME2 #include #include "../std_transforms_sme.hpp" @@ -32,7 +32,7 @@ namespace arm_gemm { // Implementations -void sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL(const int8_t *const A, const int8_t *const B, int32_t *const C, int ldc, const int M, const int N, const int K, const int32_t *const bias, const Activation act, bool accumulate, int32_t *const accumulator_buffer); +void sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL(const int8_t *const A, const int8_t *const B, int32_t *const C, int ldc, const int M, const int N, const int K, const int32_t *const bias, const Activation, bool accumulate, int32_t *const accumulator_buffer); class cls_sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL { @@ -40,7 +40,7 @@ public: typedef int8_t operand_type; typedef int32_t result_type; - typedef void (*kern_type)(const int8_t *const A, const int8_t *const B, int32_t *const C, int ldc, const int M, const int N, const int K, const int32_t *const bias, const Activation act, bool accumulate, int32_t *const accumulator_buffer); + typedef void (*kern_type)(const int8_t *const A, const int8_t *const B, int32_t *const C, int ldc, const int M, const int N, const int K, const int32_t *const bias, const Activation, bool accumulate, int32_t *const accumulator_buffer); /* Kernel blocking parameters */ static unsigned int out_height() @@ -83,12 +83,11 @@ public: StdTransformsSME transforms = {}; - cls_sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL(const CPUInfo *ci) + cls_sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL(const CPUInfo *) { - ARM_COMPUTE_UNUSED(ci); } }; } // namespace arm_gemm -#endif // __aarch64__ +#endif // ARM_COMPUTE_ENABLE_SME2 diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL/generic.cpp index d7a75282110f81f626a8eda35681a498c89a9a06..47de894306ed053f4c3896d528846358b825782e 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL/generic.cpp @@ -21,7 +21,6 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifdef __ARM_FEATURE_SVE #ifdef ARM_COMPUTE_ENABLE_SME2 #include "arm_gemm.hpp" @@ -32,10 +31,8 @@ namespace arm_gemm { -void sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL(const int8_t *const A, const int8_t *const B, int32_t *const C, int ldc, const int M, const int N, const int K, const int32_t *const bias, const Activation act, bool accumulate, int32_t *const accumulator_buffer) +void sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL(const int8_t *const A, const int8_t *const B, int32_t *const C, int ldc, const int M, const int N, const int K, const int32_t *const bias, const Activation, bool accumulate, int32_t *const accumulator_buffer) { - ARM_COMPUTE_UNUSED(act); - struct KernelArgs { KernelArgs( @@ -96,12 +93,12 @@ void sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL(const int8_t *const A, const in "1:" // Initial accumulator load from buffer: Loop ".inst 0xa040c5e8 // ld1w { z8.s-z11.s }, pn9.b/Z, [x15]\n" ".inst 0xc0840500 // mova za0h.s[x12], { z8.s-z11.s }\n" - ".inst 0xa041c5f8 // ld1w { z24.s-z27.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n" - ".inst 0xc0840701 // mova za1h.s[x12], { z24.s-z27.s }\n" - ".inst 0xa042c5fc // ld1w { z28.s-z31.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n" - ".inst 0xc0840782 // mova za2h.s[x12], { z28.s-z31.s }\n" - ".inst 0xa043c5f0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840603 // mova za3h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa041c5e0 // ld1w { z0.s-z3.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n" + ".inst 0xc0840401 // mova za1h.s[x12], { z0.s-z3.s }\n" + ".inst 0xa042c5ec // ld1w { z12.s-z15.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n" + ".inst 0xc0840582 // mova za2h.s[x12], { z12.s-z15.s }\n" + ".inst 0xa043c5e0 // ld1w { z0.s-z3.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840403 // mova za3h.s[x12], { z0.s-z3.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x15, x15, #16\n" @@ -119,11 +116,11 @@ void sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL(const int8_t *const A, const in "ldr x20, [%x[args], %[offsetof_bias]]\n" ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "cbz x20, 5f\n" - ".inst 0xa10a429c // ldnt1w { z20.s, z28.s }, p8/Z, [x20, x10, LSL #2]\n" + ".inst 0xa00a4295 // ldnt1w { z20.s-z21.s }, p8/Z, [x20, x10, LSL #2]\n" ".inst 0xc0900280 // addha za0.s, p0/M, p0/M, z20.s\n" - ".inst 0xc0900381 // addha za1.s, p0/M, p0/M, z28.s\n" + ".inst 0xc09002a1 // addha za1.s, p0/M, p0/M, z21.s\n" ".inst 0xc0900282 // addha za2.s, p0/M, p0/M, z20.s\n" - ".inst 0xc0900383 // addha za3.s, p0/M, p0/M, z28.s\n" + ".inst 0xc09002a3 // addha za3.s, p0/M, p0/M, z21.s\n" "4:" // Prepare accumulators: Test for last block "mov x20, x10\n" "mov x21, x11\n" @@ -146,75 +143,75 @@ void sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL(const int8_t *const A, const in "madd x23, x10, x20, x23\n" // bptr = B + n * kstride_bytes "cbz x22, 8f\n" "subs x22, x22, #0x1\n" - ".inst 0xa1400776 // ld1b { z22.b, z30.b }, pn9.b/Z, [x27]\n" - ".inst 0xa14006f9 // ldnt1b { z17.b, z25.b }, pn9.b/Z, [x23]\n" - ".inst 0xa1410770 // ld1b { z16.b, z24.b }, pn9.b/Z, [x27, #0x2, MUL VL]\n" - ".inst 0xa14106eb // ldnt1b { z3.b, z11.b }, pn9.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0xa0420768 // ld1b { z8.b-z9.b }, pn9.b/Z, [x27, #0x4, MUL VL]\n" - ".inst 0xa04206f3 // ldnt1b { z18.b-z19.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xa040077c // ld1b { z28.b-z29.b }, pn9.b/Z, [x27]\n" + ".inst 0xa14006e8 // ldnt1b { z0.b, z8.b }, pn9.b/Z, [x23]\n" + ".inst 0xa0410762 // ld1b { z2.b-z3.b }, pn9.b/Z, [x27, #0x2, MUL VL]\n" + ".inst 0xa14106ff // ldnt1b { z23.b, z31.b }, pn9.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xa042076e // ld1b { z14.b-z15.b }, pn9.b/Z, [x27, #0x4, MUL VL]\n" + ".inst 0xa14206f8 // ldnt1b { z16.b, z24.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n" ".inst 0xa0430764 // ld1b { z4.b-z5.b }, pn9.b/Z, [x27, #0x6, MUL VL]\n" "addvl x27, x27, #8\n" - ".inst 0xa14306fd // ldnt1b { z21.b, z29.b }, pn9.b/Z, [x23, #0x6, MUL VL]\n" + ".inst 0xa04306f5 // ldnt1b { z20.b-z21.b }, pn9.b/Z, [x23, #0x6, MUL VL]\n" "addvl x23, x23, #8\n" "ble 7f\n" "6:" // K loop - ".inst 0xa09102c0 // smopa za0.s, p0/M, p0/M, z22.b, z17.b\n" + ".inst 0xa0800380 // smopa za0.s, p0/M, p0/M, z28.b, z0.b\n" "subs x22, x22, #0x1\n" - ".inst 0xa09902c1 // smopa za1.s, p0/M, p0/M, z22.b, z25.b\n" - ".inst 0xa09103c2 // smopa za2.s, p0/M, p0/M, z30.b, z17.b\n" - ".inst 0xa09903c3 // smopa za3.s, p0/M, p0/M, z30.b, z25.b\n" - ".inst 0xa1400776 // ld1b { z22.b, z30.b }, pn9.b/Z, [x27]\n" - ".inst 0xa0830200 // smopa za0.s, p0/M, p0/M, z16.b, z3.b\n" - ".inst 0xa14006f9 // ldnt1b { z17.b, z25.b }, pn9.b/Z, [x23]\n" - ".inst 0xa08b0201 // smopa za1.s, p0/M, p0/M, z16.b, z11.b\n" - ".inst 0xa0830302 // smopa za2.s, p0/M, p0/M, z24.b, z3.b\n" - ".inst 0xa08b0303 // smopa za3.s, p0/M, p0/M, z24.b, z11.b\n" - ".inst 0xa1410770 // ld1b { z16.b, z24.b }, pn9.b/Z, [x27, #0x2, MUL VL]\n" - ".inst 0xa0920100 // smopa za0.s, p0/M, p0/M, z8.b, z18.b\n" - ".inst 0xa14106eb // ldnt1b { z3.b, z11.b }, pn9.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0xa0930101 // smopa za1.s, p0/M, p0/M, z8.b, z19.b\n" - ".inst 0xa0920122 // smopa za2.s, p0/M, p0/M, z9.b, z18.b\n" - ".inst 0xa0930123 // smopa za3.s, p0/M, p0/M, z9.b, z19.b\n" - ".inst 0xa0420768 // ld1b { z8.b-z9.b }, pn9.b/Z, [x27, #0x4, MUL VL]\n" - ".inst 0xa04206f3 // ldnt1b { z18.b-z19.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xa0950080 // smopa za0.s, p0/M, p0/M, z4.b, z21.b\n" - ".inst 0xa09d0081 // smopa za1.s, p0/M, p0/M, z4.b, z29.b\n" - ".inst 0xa09500a2 // smopa za2.s, p0/M, p0/M, z5.b, z21.b\n" - ".inst 0xa09d00a3 // smopa za3.s, p0/M, p0/M, z5.b, z29.b\n" + ".inst 0xa0880381 // smopa za1.s, p0/M, p0/M, z28.b, z8.b\n" + ".inst 0xa08003a2 // smopa za2.s, p0/M, p0/M, z29.b, z0.b\n" + ".inst 0xa08803a3 // smopa za3.s, p0/M, p0/M, z29.b, z8.b\n" + ".inst 0xa040077c // ld1b { z28.b-z29.b }, pn9.b/Z, [x27]\n" + ".inst 0xa0970040 // smopa za0.s, p0/M, p0/M, z2.b, z23.b\n" + ".inst 0xa14006e8 // ldnt1b { z0.b, z8.b }, pn9.b/Z, [x23]\n" + ".inst 0xa09f0041 // smopa za1.s, p0/M, p0/M, z2.b, z31.b\n" + ".inst 0xa0970062 // smopa za2.s, p0/M, p0/M, z3.b, z23.b\n" + ".inst 0xa09f0063 // smopa za3.s, p0/M, p0/M, z3.b, z31.b\n" + ".inst 0xa0410762 // ld1b { z2.b-z3.b }, pn9.b/Z, [x27, #0x2, MUL VL]\n" + ".inst 0xa09001c0 // smopa za0.s, p0/M, p0/M, z14.b, z16.b\n" + ".inst 0xa14106ff // ldnt1b { z23.b, z31.b }, pn9.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xa09801c1 // smopa za1.s, p0/M, p0/M, z14.b, z24.b\n" + ".inst 0xa09001e2 // smopa za2.s, p0/M, p0/M, z15.b, z16.b\n" + ".inst 0xa09801e3 // smopa za3.s, p0/M, p0/M, z15.b, z24.b\n" + ".inst 0xa042076e // ld1b { z14.b-z15.b }, pn9.b/Z, [x27, #0x4, MUL VL]\n" + ".inst 0xa14206f8 // ldnt1b { z16.b, z24.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xa0940080 // smopa za0.s, p0/M, p0/M, z4.b, z20.b\n" + ".inst 0xa0950081 // smopa za1.s, p0/M, p0/M, z4.b, z21.b\n" + ".inst 0xa09400a2 // smopa za2.s, p0/M, p0/M, z5.b, z20.b\n" + ".inst 0xa09500a3 // smopa za3.s, p0/M, p0/M, z5.b, z21.b\n" ".inst 0xa0430764 // ld1b { z4.b-z5.b }, pn9.b/Z, [x27, #0x6, MUL VL]\n" "addvl x27, x27, #8\n" - ".inst 0xa14306fd // ldnt1b { z21.b, z29.b }, pn9.b/Z, [x23, #0x6, MUL VL]\n" + ".inst 0xa04306f5 // ldnt1b { z20.b-z21.b }, pn9.b/Z, [x23, #0x6, MUL VL]\n" "addvl x23, x23, #8\n" "bgt 6b\n" "7:" // K loop tail - ".inst 0xa09102c0 // smopa za0.s, p0/M, p0/M, z22.b, z17.b\n" - ".inst 0xa09902c1 // smopa za1.s, p0/M, p0/M, z22.b, z25.b\n" - ".inst 0xa09103c2 // smopa za2.s, p0/M, p0/M, z30.b, z17.b\n" - ".inst 0xa09903c3 // smopa za3.s, p0/M, p0/M, z30.b, z25.b\n" - ".inst 0xa0830200 // smopa za0.s, p0/M, p0/M, z16.b, z3.b\n" - ".inst 0xa08b0201 // smopa za1.s, p0/M, p0/M, z16.b, z11.b\n" - ".inst 0xa0830302 // smopa za2.s, p0/M, p0/M, z24.b, z3.b\n" - ".inst 0xa08b0303 // smopa za3.s, p0/M, p0/M, z24.b, z11.b\n" - ".inst 0xa0920100 // smopa za0.s, p0/M, p0/M, z8.b, z18.b\n" - ".inst 0xa0930101 // smopa za1.s, p0/M, p0/M, z8.b, z19.b\n" - ".inst 0xa0920122 // smopa za2.s, p0/M, p0/M, z9.b, z18.b\n" - ".inst 0xa0930123 // smopa za3.s, p0/M, p0/M, z9.b, z19.b\n" - ".inst 0xa0950080 // smopa za0.s, p0/M, p0/M, z4.b, z21.b\n" - ".inst 0xa09d0081 // smopa za1.s, p0/M, p0/M, z4.b, z29.b\n" - ".inst 0xa09500a2 // smopa za2.s, p0/M, p0/M, z5.b, z21.b\n" - ".inst 0xa09d00a3 // smopa za3.s, p0/M, p0/M, z5.b, z29.b\n" + ".inst 0xa0800380 // smopa za0.s, p0/M, p0/M, z28.b, z0.b\n" + ".inst 0xa0880381 // smopa za1.s, p0/M, p0/M, z28.b, z8.b\n" + ".inst 0xa08003a2 // smopa za2.s, p0/M, p0/M, z29.b, z0.b\n" + ".inst 0xa08803a3 // smopa za3.s, p0/M, p0/M, z29.b, z8.b\n" + ".inst 0xa0970040 // smopa za0.s, p0/M, p0/M, z2.b, z23.b\n" + ".inst 0xa09f0041 // smopa za1.s, p0/M, p0/M, z2.b, z31.b\n" + ".inst 0xa0970062 // smopa za2.s, p0/M, p0/M, z3.b, z23.b\n" + ".inst 0xa09f0063 // smopa za3.s, p0/M, p0/M, z3.b, z31.b\n" + ".inst 0xa09001c0 // smopa za0.s, p0/M, p0/M, z14.b, z16.b\n" + ".inst 0xa09801c1 // smopa za1.s, p0/M, p0/M, z14.b, z24.b\n" + ".inst 0xa09001e2 // smopa za2.s, p0/M, p0/M, z15.b, z16.b\n" + ".inst 0xa09801e3 // smopa za3.s, p0/M, p0/M, z15.b, z24.b\n" + ".inst 0xa0940080 // smopa za0.s, p0/M, p0/M, z4.b, z20.b\n" + ".inst 0xa0950081 // smopa za1.s, p0/M, p0/M, z4.b, z21.b\n" + ".inst 0xa09400a2 // smopa za2.s, p0/M, p0/M, z5.b, z20.b\n" + ".inst 0xa09500a3 // smopa za3.s, p0/M, p0/M, z5.b, z21.b\n" "8:" // K oddments "cbz x21, 10f\n" "9:" // K oddments: Loop - ".inst 0xa1400776 // ld1b { z22.b, z30.b }, pn9.b/Z, [x27]\n" + ".inst 0xa1400774 // ld1b { z20.b, z28.b }, pn9.b/Z, [x27]\n" "subs x21, x21, #0x1\n" "addvl x27, x27, #2\n" - ".inst 0xa14006f1 // ld1b { z17.b, z25.b }, pn9.b/Z, [x23]\n" + ".inst 0xa14006e7 // ld1b { z7.b, z15.b }, pn9.b/Z, [x23]\n" "addvl x23, x23, #2\n" - ".inst 0xa09102c0 // smopa za0.s, p0/M, p0/M, z22.b, z17.b\n" - ".inst 0xa09902c1 // smopa za1.s, p0/M, p0/M, z22.b, z25.b\n" - ".inst 0xa09103c2 // smopa za2.s, p0/M, p0/M, z30.b, z17.b\n" - ".inst 0xa09903c3 // smopa za3.s, p0/M, p0/M, z30.b, z25.b\n" + ".inst 0xa0870280 // smopa za0.s, p0/M, p0/M, z20.b, z7.b\n" + ".inst 0xa08f0281 // smopa za1.s, p0/M, p0/M, z20.b, z15.b\n" + ".inst 0xa0870382 // smopa za2.s, p0/M, p0/M, z28.b, z7.b\n" + ".inst 0xa08f0383 // smopa za3.s, p0/M, p0/M, z28.b, z15.b\n" "bgt 9b\n" "10:" // K oddments: End "tbz x16, #1, 14f\n" @@ -223,24 +220,24 @@ void sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL(const int8_t *const A, const in "cntw x20\n" "11:" // Store to partial result buffer: Store and refill: Loop ".inst 0xa040c5ec // ld1w { z12.s-z15.s }, pn9.b/Z, [x15]\n" - ".inst 0xc0860418 // mova { z24.s-z27.s }, za0h.s[x12]\n" + ".inst 0xc0860408 // mova { z8.s-z11.s }, za0h.s[x12]\n" ".inst 0xc0840580 // mova za0h.s[x12], { z12.s-z15.s }\n" - ".inst 0xc0860420 // mova { z0.s-z3.s }, za1h.s[x12]\n" + ".inst 0xc086043c // mova { z28.s-z31.s }, za1h.s[x12]\n" ".inst 0xa041c5ec // ld1w { z12.s-z15.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n" ".inst 0xc0840581 // mova za1h.s[x12], { z12.s-z15.s }\n" - ".inst 0xc086045c // mova { z28.s-z31.s }, za2h.s[x12]\n" - ".inst 0xc0860468 // mova { z8.s-z11.s }, za3h.s[x12]\n" - ".inst 0xa042c5ec // ld1w { z12.s-z15.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n" - ".inst 0xc0840582 // mova za2h.s[x12], { z12.s-z15.s }\n" - ".inst 0xa043c5e4 // ld1w { z4.s-z7.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840483 // mova za3h.s[x12], { z4.s-z7.s }\n" + ".inst 0xc086044c // mova { z12.s-z15.s }, za2h.s[x12]\n" + ".inst 0xc0860464 // mova { z4.s-z7.s }, za3h.s[x12]\n" + ".inst 0xa042c5f8 // ld1w { z24.s-z27.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n" + ".inst 0xc0840702 // mova za2h.s[x12], { z24.s-z27.s }\n" + ".inst 0xa043c5f0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840603 // mova za3h.s[x12], { z16.s-z19.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" - ".inst 0xa060c5d8 // st1w { z24.s-z27.s }, pn9.b, [x14]\n" + ".inst 0xa060c5c8 // st1w { z8.s-z11.s }, pn9.b, [x14]\n" "addvl x15, x15, #16\n" - ".inst 0xa061c5c0 // st1w { z0.s-z3.s }, pn9.b, [x14, #0x4, MUL VL]\n" - ".inst 0xa062c5dc // st1w { z28.s-z31.s }, pn9.b, [x14, #0x8, MUL VL]\n" - ".inst 0xa063c5c8 // st1w { z8.s-z11.s }, pn9.b, [x14, #0xc, MUL VL]\n" + ".inst 0xa061c5dc // st1w { z28.s-z31.s }, pn9.b, [x14, #0x4, MUL VL]\n" + ".inst 0xa062c5cc // st1w { z12.s-z15.s }, pn9.b, [x14, #0x8, MUL VL]\n" + ".inst 0xa063c5c4 // st1w { z4.s-z7.s }, pn9.b, [x14, #0xc, MUL VL]\n" "addvl x14, x14, #16\n" "blt 11b\n" "b 23f\n" @@ -248,16 +245,16 @@ void sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL(const int8_t *const A, const in "mov x12, #0x0\n" "cntw x20\n" "13:" // Store to partial result buffer: Store only: Loop - ".inst 0xc0860400 // mova { z0.s-z3.s }, za0h.s[x12]\n" - ".inst 0xc086042c // mova { z12.s-z15.s }, za1h.s[x12]\n" - ".inst 0xa060c5c0 // st1w { z0.s-z3.s }, pn9.b, [x14]\n" - ".inst 0xc086045c // mova { z28.s-z31.s }, za2h.s[x12]\n" - ".inst 0xc0860464 // mova { z4.s-z7.s }, za3h.s[x12]\n" - ".inst 0xa061c5cc // st1w { z12.s-z15.s }, pn9.b, [x14, #0x4, MUL VL]\n" + ".inst 0xc086040c // mova { z12.s-z15.s }, za0h.s[x12]\n" + ".inst 0xc0860428 // mova { z8.s-z11.s }, za1h.s[x12]\n" + ".inst 0xa060c5cc // st1w { z12.s-z15.s }, pn9.b, [x14]\n" + ".inst 0xc086044c // mova { z12.s-z15.s }, za2h.s[x12]\n" + ".inst 0xc0860460 // mova { z0.s-z3.s }, za3h.s[x12]\n" + ".inst 0xa061c5c8 // st1w { z8.s-z11.s }, pn9.b, [x14, #0x4, MUL VL]\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" - ".inst 0xa062c5dc // st1w { z28.s-z31.s }, pn9.b, [x14, #0x8, MUL VL]\n" - ".inst 0xa063c5c4 // st1w { z4.s-z7.s }, pn9.b, [x14, #0xc, MUL VL]\n" + ".inst 0xa062c5cc // st1w { z12.s-z15.s }, pn9.b, [x14, #0x8, MUL VL]\n" + ".inst 0xa063c5c0 // st1w { z0.s-z3.s }, pn9.b, [x14, #0xc, MUL VL]\n" "addvl x14, x14, #16\n" "blt 13b\n" "b 23f\n" @@ -275,32 +272,32 @@ void sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL(const int8_t *const A, const in "and x20, x22, #0x3\n" "cbz x21, 16f\n" "15:" // Store to output array: Accumulator row 0 loop - ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" - ".inst 0xc0860438 // mova { z24.s-z27.s }, za1h.s[x12]\n" - ".inst 0xa1604350 // st1w { z16.s, z24.s }, p8, [x26]\n" + ".inst 0xc0860404 // mova { z4.s-z7.s }, za0h.s[x12]\n" + ".inst 0xc086042c // mova { z12.s-z15.s }, za1h.s[x12]\n" + ".inst 0xa1604344 // st1w { z4.s, z12.s }, p8, [x26]\n" "add x26, x26, x23\n" - ".inst 0xa1604351 // st1w { z17.s, z25.s }, p8, [x26]\n" + ".inst 0xa1604345 // st1w { z5.s, z13.s }, p8, [x26]\n" "add x26, x26, x23\n" "add x12, x12, #0x4\n" - ".inst 0xa1604352 // st1w { z18.s, z26.s }, p8, [x26]\n" + ".inst 0xa1604346 // st1w { z6.s, z14.s }, p8, [x26]\n" "add x26, x26, x23\n" "cmp x12, x21, LSL #2\n" - ".inst 0xa1604353 // st1w { z19.s, z27.s }, p8, [x26]\n" + ".inst 0xa1604347 // st1w { z7.s, z15.s }, p8, [x26]\n" "add x26, x26, x23\n" "blt 15b\n" "16:" // Store to output array: Accumulator row 0 oddments "cbz x20, 17f\n" "subs x20, x20, #0x1\n" - ".inst 0xc0860400 // mova { z0.s-z3.s }, za0h.s[x12]\n" - ".inst 0xc0860428 // mova { z8.s-z11.s }, za1h.s[x12]\n" - ".inst 0xa1604340 // st1w { z0.s, z8.s }, p8, [x26]\n" + ".inst 0xc0860404 // mova { z4.s-z7.s }, za0h.s[x12]\n" + ".inst 0xc086042c // mova { z12.s-z15.s }, za1h.s[x12]\n" + ".inst 0xa1604344 // st1w { z4.s, z12.s }, p8, [x26]\n" "add x26, x26, x23\n" "beq 17f\n" "subs x20, x20, #0x1\n" - ".inst 0xa1604341 // st1w { z1.s, z9.s }, p8, [x26]\n" + ".inst 0xa1604345 // st1w { z5.s, z13.s }, p8, [x26]\n" "add x26, x26, x23\n" "beq 17f\n" - ".inst 0xa1604342 // st1w { z2.s, z10.s }, p8, [x26]\n" + ".inst 0xa1604346 // st1w { z6.s, z14.s }, p8, [x26]\n" "add x26, x26, x23\n" "17:" // Store to output array: Accumulator row 0 oddments: End "subs x25, x25, x22\n" @@ -328,30 +325,30 @@ void sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL(const int8_t *const A, const in "19:" // Store to output array: Accumulator row 1 oddments "cbz x20, 20f\n" "subs x20, x20, #0x1\n" - ".inst 0xc0860444 // mova { z4.s-z7.s }, za2h.s[x12]\n" - ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" - ".inst 0xa1604344 // st1w { z4.s, z12.s }, p8, [x26]\n" + ".inst 0xc0860440 // mova { z0.s-z3.s }, za2h.s[x12]\n" + ".inst 0xc0860468 // mova { z8.s-z11.s }, za3h.s[x12]\n" + ".inst 0xa1604340 // st1w { z0.s, z8.s }, p8, [x26]\n" "add x26, x26, x23\n" "beq 20f\n" "subs x20, x20, #0x1\n" - ".inst 0xa1604345 // st1w { z5.s, z13.s }, p8, [x26]\n" + ".inst 0xa1604341 // st1w { z1.s, z9.s }, p8, [x26]\n" "add x26, x26, x23\n" "beq 20f\n" - ".inst 0xa1604346 // st1w { z6.s, z14.s }, p8, [x26]\n" + ".inst 0xa1604342 // st1w { z2.s, z10.s }, p8, [x26]\n" "20:" // Store to output array: Accumulator row 1 oddments: End "21:" // Store to output array: End "tbz x16, #0, 23f\n" "mov x12, #0x0\n" "cntw x20\n" "22:" // Store to output array: Refill accumulators: Loop - ".inst 0xa040c5f0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x15]\n" - ".inst 0xc0840600 // mova za0h.s[x12], { z16.s-z19.s }\n" - ".inst 0xa041c5f0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n" - ".inst 0xc0840601 // mova za1h.s[x12], { z16.s-z19.s }\n" - ".inst 0xa042c5e0 // ld1w { z0.s-z3.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n" - ".inst 0xc0840402 // mova za2h.s[x12], { z0.s-z3.s }\n" - ".inst 0xa043c5ec // ld1w { z12.s-z15.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840583 // mova za3h.s[x12], { z12.s-z15.s }\n" + ".inst 0xa040c5ec // ld1w { z12.s-z15.s }, pn9.b/Z, [x15]\n" + ".inst 0xc0840580 // mova za0h.s[x12], { z12.s-z15.s }\n" + ".inst 0xa041c5e4 // ld1w { z4.s-z7.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n" + ".inst 0xc0840481 // mova za1h.s[x12], { z4.s-z7.s }\n" + ".inst 0xa042c5ec // ld1w { z12.s-z15.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n" + ".inst 0xc0840582 // mova za2h.s[x12], { z12.s-z15.s }\n" + ".inst 0xa043c5e4 // ld1w { z4.s-z7.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840483 // mova za3h.s[x12], { z4.s-z7.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x15, x15, #16\n" @@ -375,4 +372,3 @@ void sme2_interleaved_nomerge_s8s32_mopa_2VLx2VL(const int8_t *const A, const in } // namespace arm_gemm #endif // ARM_COMPUTE_ENABLE_SME2 -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL.hpp index be1106da1331c95362c9a4ad1616af924d09c5de..ce10ab30e74ac79056add4daed2862c34e6423ed 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,7 +23,7 @@ */ #pragma once -#ifdef __aarch64__ +#ifdef ARM_COMPUTE_ENABLE_SME2 #include #include "../std_transforms_sme.hpp" @@ -32,7 +32,7 @@ namespace arm_gemm { // Implementations -void sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL(const int8_t *const A, const int8_t *const B, int32_t *const C, int ldc, const int M, const int N, const int K, const int32_t *const bias, const Activation act, bool accumulate, int32_t *const accumulator_buffer); +void sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL(const int8_t *const A, const int8_t *const B, int32_t *const C, int ldc, const int M, const int N, const int K, const int32_t *const bias, const Activation, bool accumulate, int32_t *const accumulator_buffer); class cls_sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL { @@ -40,7 +40,7 @@ public: typedef int8_t operand_type; typedef int32_t result_type; - typedef void (*kern_type)(const int8_t *const A, const int8_t *const B, int32_t *const C, int ldc, const int M, const int N, const int K, const int32_t *const bias, const Activation act, bool accumulate, int32_t *const accumulator_buffer); + typedef void (*kern_type)(const int8_t *const A, const int8_t *const B, int32_t *const C, int ldc, const int M, const int N, const int K, const int32_t *const bias, const Activation, bool accumulate, int32_t *const accumulator_buffer); /* Kernel blocking parameters */ static unsigned int out_height() @@ -83,12 +83,11 @@ public: StdTransformsSME transforms = {}; - cls_sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL(const CPUInfo *ci) + cls_sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL(const CPUInfo *) { - ARM_COMPUTE_UNUSED(ci); } }; } // namespace arm_gemm -#endif // __aarch64__ +#endif // ARM_COMPUTE_ENABLE_SME2 diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL/generic.cpp index d863b6c72a84498a2874bec91209673403951111..a23c44b7da73c39dfc2c3c38d7fea126ef6fc374 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL/generic.cpp @@ -21,7 +21,6 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifdef __ARM_FEATURE_SVE #ifdef ARM_COMPUTE_ENABLE_SME2 #include "arm_gemm.hpp" @@ -32,10 +31,8 @@ namespace arm_gemm { -void sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL(const int8_t *const A, const int8_t *const B, int32_t *const C, int ldc, const int M, const int N, const int K, const int32_t *const bias, const Activation act, bool accumulate, int32_t *const accumulator_buffer) +void sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL(const int8_t *const A, const int8_t *const B, int32_t *const C, int ldc, const int M, const int N, const int K, const int32_t *const bias, const Activation, bool accumulate, int32_t *const accumulator_buffer) { - ARM_COMPUTE_UNUSED(act); - struct KernelArgs { KernelArgs( @@ -94,14 +91,14 @@ void sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL(const int8_t *const A, const in "mov x12, #0x0\n" "cntw x20\n" "1:" // Initial accumulator load from buffer: Loop - ".inst 0xa040c1fc // ld1w { z28.s-z31.s }, pn8.b/Z, [x15]\n" - ".inst 0xc0840780 // mova za0h.s[x12], { z28.s-z31.s }\n" - ".inst 0xa041c1f8 // ld1w { z24.s-z27.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" - ".inst 0xc0840701 // mova za1h.s[x12], { z24.s-z27.s }\n" - ".inst 0xa042c1e4 // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" - ".inst 0xc0840482 // mova za2h.s[x12], { z4.s-z7.s }\n" - ".inst 0xa043c1e4 // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840483 // mova za3h.s[x12], { z4.s-z7.s }\n" + ".inst 0xa040c1f8 // ld1w { z24.s-z27.s }, pn8.b/Z, [x15]\n" + ".inst 0xc0840700 // mova za0h.s[x12], { z24.s-z27.s }\n" + ".inst 0xa041c1fc // ld1w { z28.s-z31.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" + ".inst 0xc0840781 // mova za1h.s[x12], { z28.s-z31.s }\n" + ".inst 0xa042c1f0 // ld1w { z16.s-z19.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" + ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa043c1f8 // ld1w { z24.s-z27.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840703 // mova za3h.s[x12], { z24.s-z27.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x15, x15, #16\n" @@ -119,11 +116,11 @@ void sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL(const int8_t *const A, const in "ldr x20, [%x[args], %[offsetof_bias]]\n" ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "cbz x20, 5f\n" - "ldnt1w { z15.s }, p0/Z, [x20, x10, LSL #2]\n" - ".inst 0xc09025e0 // addha za0.s, p1/M, p1/M, z15.s\n" - ".inst 0xc09025e1 // addha za1.s, p1/M, p1/M, z15.s\n" - ".inst 0xc09025e2 // addha za2.s, p1/M, p1/M, z15.s\n" - ".inst 0xc09025e3 // addha za3.s, p1/M, p1/M, z15.s\n" + "ldnt1w { z17.s }, p0/Z, [x20, x10, LSL #2]\n" + ".inst 0xc0902620 // addha za0.s, p1/M, p1/M, z17.s\n" + ".inst 0xc0902621 // addha za1.s, p1/M, p1/M, z17.s\n" + ".inst 0xc0902622 // addha za2.s, p1/M, p1/M, z17.s\n" + ".inst 0xc0902623 // addha za3.s, p1/M, p1/M, z17.s\n" "4:" // Prepare accumulators: Test for last block "mov x20, x10\n" "mov x21, x11\n" @@ -146,75 +143,75 @@ void sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL(const int8_t *const A, const in "madd x23, x10, x20, x23\n" // bptr = B + n * kstride_bytes "cbz x22, 8f\n" "subs x22, x22, #0x1\n" - ".inst 0xa0408370 // ld1b { z16.b-z19.b }, pn8.b/Z, [x27]\n" - "ldnt1b { z7.b }, p1/Z, [x23]\n" - ".inst 0xa041837c // ld1b { z28.b-z31.b }, pn8.b/Z, [x27, #0x4, MUL VL]\n" - "ldnt1b { z13.b }, p1/Z, [x23, #1, MUL VL]\n" - ".inst 0xa0428360 // ld1b { z0.b-z3.b }, pn8.b/Z, [x27, #0x8, MUL VL]\n" - "ldnt1b { z12.b }, p1/Z, [x23, #2, MUL VL]\n" - ".inst 0xa0438378 // ld1b { z24.b-z27.b }, pn8.b/Z, [x27, #0xc, MUL VL]\n" + ".inst 0xa1408372 // ld1b { z18.b, z22.b, z26.b, z30.b }, pn8.b/Z, [x27]\n" + "ldnt1b { z12.b }, p1/Z, [x23]\n" + ".inst 0xa1418370 // ld1b { z16.b, z20.b, z24.b, z28.b }, pn8.b/Z, [x27, #0x4, MUL VL]\n" + "ldnt1b { z5.b }, p1/Z, [x23, #1, MUL VL]\n" + ".inst 0xa1428363 // ld1b { z3.b, z7.b, z11.b, z15.b }, pn8.b/Z, [x27, #0x8, MUL VL]\n" + "ldnt1b { z4.b }, p1/Z, [x23, #2, MUL VL]\n" + ".inst 0xa1438362 // ld1b { z2.b, z6.b, z10.b, z14.b }, pn8.b/Z, [x27, #0xc, MUL VL]\n" "addvl x27, x27, #16\n" - "ldnt1b { z23.b }, p1/Z, [x23, #3, MUL VL]\n" + "ldnt1b { z19.b }, p1/Z, [x23, #3, MUL VL]\n" "addvl x23, x23, #4\n" "ble 7f\n" "6:" // K loop - ".inst 0xa0872600 // smopa za0.s, p1/M, p1/M, z16.b, z7.b\n" + ".inst 0xa08c2640 // smopa za0.s, p1/M, p1/M, z18.b, z12.b\n" "subs x22, x22, #0x1\n" - ".inst 0xa0872621 // smopa za1.s, p1/M, p1/M, z17.b, z7.b\n" - ".inst 0xa0872642 // smopa za2.s, p1/M, p1/M, z18.b, z7.b\n" - ".inst 0xa0872663 // smopa za3.s, p1/M, p1/M, z19.b, z7.b\n" - ".inst 0xa0408370 // ld1b { z16.b-z19.b }, pn8.b/Z, [x27]\n" - ".inst 0xa08d2780 // smopa za0.s, p1/M, p1/M, z28.b, z13.b\n" - "ldnt1b { z7.b }, p1/Z, [x23]\n" - ".inst 0xa08d27a1 // smopa za1.s, p1/M, p1/M, z29.b, z13.b\n" - ".inst 0xa08d27c2 // smopa za2.s, p1/M, p1/M, z30.b, z13.b\n" - ".inst 0xa08d27e3 // smopa za3.s, p1/M, p1/M, z31.b, z13.b\n" - ".inst 0xa041837c // ld1b { z28.b-z31.b }, pn8.b/Z, [x27, #0x4, MUL VL]\n" - ".inst 0xa08c2400 // smopa za0.s, p1/M, p1/M, z0.b, z12.b\n" - "ldnt1b { z13.b }, p1/Z, [x23, #1, MUL VL]\n" - ".inst 0xa08c2421 // smopa za1.s, p1/M, p1/M, z1.b, z12.b\n" - ".inst 0xa08c2442 // smopa za2.s, p1/M, p1/M, z2.b, z12.b\n" - ".inst 0xa08c2463 // smopa za3.s, p1/M, p1/M, z3.b, z12.b\n" - ".inst 0xa0428360 // ld1b { z0.b-z3.b }, pn8.b/Z, [x27, #0x8, MUL VL]\n" - "ldnt1b { z12.b }, p1/Z, [x23, #2, MUL VL]\n" - ".inst 0xa0972700 // smopa za0.s, p1/M, p1/M, z24.b, z23.b\n" - ".inst 0xa0972721 // smopa za1.s, p1/M, p1/M, z25.b, z23.b\n" - ".inst 0xa0972742 // smopa za2.s, p1/M, p1/M, z26.b, z23.b\n" - ".inst 0xa0972763 // smopa za3.s, p1/M, p1/M, z27.b, z23.b\n" - ".inst 0xa0438378 // ld1b { z24.b-z27.b }, pn8.b/Z, [x27, #0xc, MUL VL]\n" + ".inst 0xa08c26c1 // smopa za1.s, p1/M, p1/M, z22.b, z12.b\n" + ".inst 0xa08c2742 // smopa za2.s, p1/M, p1/M, z26.b, z12.b\n" + ".inst 0xa08c27c3 // smopa za3.s, p1/M, p1/M, z30.b, z12.b\n" + ".inst 0xa1408372 // ld1b { z18.b, z22.b, z26.b, z30.b }, pn8.b/Z, [x27]\n" + ".inst 0xa0852600 // smopa za0.s, p1/M, p1/M, z16.b, z5.b\n" + "ldnt1b { z12.b }, p1/Z, [x23]\n" + ".inst 0xa0852681 // smopa za1.s, p1/M, p1/M, z20.b, z5.b\n" + ".inst 0xa0852702 // smopa za2.s, p1/M, p1/M, z24.b, z5.b\n" + ".inst 0xa0852783 // smopa za3.s, p1/M, p1/M, z28.b, z5.b\n" + ".inst 0xa1418370 // ld1b { z16.b, z20.b, z24.b, z28.b }, pn8.b/Z, [x27, #0x4, MUL VL]\n" + ".inst 0xa0842460 // smopa za0.s, p1/M, p1/M, z3.b, z4.b\n" + "ldnt1b { z5.b }, p1/Z, [x23, #1, MUL VL]\n" + ".inst 0xa08424e1 // smopa za1.s, p1/M, p1/M, z7.b, z4.b\n" + ".inst 0xa0842562 // smopa za2.s, p1/M, p1/M, z11.b, z4.b\n" + ".inst 0xa08425e3 // smopa za3.s, p1/M, p1/M, z15.b, z4.b\n" + ".inst 0xa1428363 // ld1b { z3.b, z7.b, z11.b, z15.b }, pn8.b/Z, [x27, #0x8, MUL VL]\n" + "ldnt1b { z4.b }, p1/Z, [x23, #2, MUL VL]\n" + ".inst 0xa0932440 // smopa za0.s, p1/M, p1/M, z2.b, z19.b\n" + ".inst 0xa09324c1 // smopa za1.s, p1/M, p1/M, z6.b, z19.b\n" + ".inst 0xa0932542 // smopa za2.s, p1/M, p1/M, z10.b, z19.b\n" + ".inst 0xa09325c3 // smopa za3.s, p1/M, p1/M, z14.b, z19.b\n" + ".inst 0xa1438362 // ld1b { z2.b, z6.b, z10.b, z14.b }, pn8.b/Z, [x27, #0xc, MUL VL]\n" "addvl x27, x27, #16\n" - "ldnt1b { z23.b }, p1/Z, [x23, #3, MUL VL]\n" + "ldnt1b { z19.b }, p1/Z, [x23, #3, MUL VL]\n" "addvl x23, x23, #4\n" "bgt 6b\n" "7:" // K loop tail - ".inst 0xa0872600 // smopa za0.s, p1/M, p1/M, z16.b, z7.b\n" - ".inst 0xa0872621 // smopa za1.s, p1/M, p1/M, z17.b, z7.b\n" - ".inst 0xa0872642 // smopa za2.s, p1/M, p1/M, z18.b, z7.b\n" - ".inst 0xa0872663 // smopa za3.s, p1/M, p1/M, z19.b, z7.b\n" - ".inst 0xa08d2780 // smopa za0.s, p1/M, p1/M, z28.b, z13.b\n" - ".inst 0xa08d27a1 // smopa za1.s, p1/M, p1/M, z29.b, z13.b\n" - ".inst 0xa08d27c2 // smopa za2.s, p1/M, p1/M, z30.b, z13.b\n" - ".inst 0xa08d27e3 // smopa za3.s, p1/M, p1/M, z31.b, z13.b\n" - ".inst 0xa08c2400 // smopa za0.s, p1/M, p1/M, z0.b, z12.b\n" - ".inst 0xa08c2421 // smopa za1.s, p1/M, p1/M, z1.b, z12.b\n" - ".inst 0xa08c2442 // smopa za2.s, p1/M, p1/M, z2.b, z12.b\n" - ".inst 0xa08c2463 // smopa za3.s, p1/M, p1/M, z3.b, z12.b\n" - ".inst 0xa0972700 // smopa za0.s, p1/M, p1/M, z24.b, z23.b\n" - ".inst 0xa0972721 // smopa za1.s, p1/M, p1/M, z25.b, z23.b\n" - ".inst 0xa0972742 // smopa za2.s, p1/M, p1/M, z26.b, z23.b\n" - ".inst 0xa0972763 // smopa za3.s, p1/M, p1/M, z27.b, z23.b\n" + ".inst 0xa08c2640 // smopa za0.s, p1/M, p1/M, z18.b, z12.b\n" + ".inst 0xa08c26c1 // smopa za1.s, p1/M, p1/M, z22.b, z12.b\n" + ".inst 0xa08c2742 // smopa za2.s, p1/M, p1/M, z26.b, z12.b\n" + ".inst 0xa08c27c3 // smopa za3.s, p1/M, p1/M, z30.b, z12.b\n" + ".inst 0xa0852600 // smopa za0.s, p1/M, p1/M, z16.b, z5.b\n" + ".inst 0xa0852681 // smopa za1.s, p1/M, p1/M, z20.b, z5.b\n" + ".inst 0xa0852702 // smopa za2.s, p1/M, p1/M, z24.b, z5.b\n" + ".inst 0xa0852783 // smopa za3.s, p1/M, p1/M, z28.b, z5.b\n" + ".inst 0xa0842460 // smopa za0.s, p1/M, p1/M, z3.b, z4.b\n" + ".inst 0xa08424e1 // smopa za1.s, p1/M, p1/M, z7.b, z4.b\n" + ".inst 0xa0842562 // smopa za2.s, p1/M, p1/M, z11.b, z4.b\n" + ".inst 0xa08425e3 // smopa za3.s, p1/M, p1/M, z15.b, z4.b\n" + ".inst 0xa0932440 // smopa za0.s, p1/M, p1/M, z2.b, z19.b\n" + ".inst 0xa09324c1 // smopa za1.s, p1/M, p1/M, z6.b, z19.b\n" + ".inst 0xa0932542 // smopa za2.s, p1/M, p1/M, z10.b, z19.b\n" + ".inst 0xa09325c3 // smopa za3.s, p1/M, p1/M, z14.b, z19.b\n" "8:" // K oddments "cbz x21, 10f\n" "9:" // K oddments: Loop - ".inst 0xa0408370 // ld1b { z16.b-z19.b }, pn8.b/Z, [x27]\n" + ".inst 0xa0408368 // ld1b { z8.b-z11.b }, pn8.b/Z, [x27]\n" "subs x21, x21, #0x1\n" "addvl x27, x27, #4\n" - "ld1b { z7.b }, p1/Z, [x23]\n" + "ld1b { z15.b }, p1/Z, [x23]\n" "addvl x23, x23, #1\n" - ".inst 0xa0872600 // smopa za0.s, p1/M, p1/M, z16.b, z7.b\n" - ".inst 0xa0872621 // smopa za1.s, p1/M, p1/M, z17.b, z7.b\n" - ".inst 0xa0872642 // smopa za2.s, p1/M, p1/M, z18.b, z7.b\n" - ".inst 0xa0872663 // smopa za3.s, p1/M, p1/M, z19.b, z7.b\n" + ".inst 0xa08f2500 // smopa za0.s, p1/M, p1/M, z8.b, z15.b\n" + ".inst 0xa08f2521 // smopa za1.s, p1/M, p1/M, z9.b, z15.b\n" + ".inst 0xa08f2542 // smopa za2.s, p1/M, p1/M, z10.b, z15.b\n" + ".inst 0xa08f2563 // smopa za3.s, p1/M, p1/M, z11.b, z15.b\n" "bgt 9b\n" "10:" // K oddments: End "tbz x16, #1, 14f\n" @@ -222,25 +219,25 @@ void sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL(const int8_t *const A, const in "mov x12, #0x0\n" "cntw x20\n" "11:" // Store to partial result buffer: Store and refill: Loop - ".inst 0xa040c1f4 // ld1w { z20.s-z23.s }, pn8.b/Z, [x15]\n" - ".inst 0xc0860400 // mova { z0.s-z3.s }, za0h.s[x12]\n" - ".inst 0xc0840680 // mova za0h.s[x12], { z20.s-z23.s }\n" - ".inst 0xc0860428 // mova { z8.s-z11.s }, za1h.s[x12]\n" - ".inst 0xa041c1f8 // ld1w { z24.s-z27.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" - ".inst 0xc0840701 // mova za1h.s[x12], { z24.s-z27.s }\n" - ".inst 0xc086045c // mova { z28.s-z31.s }, za2h.s[x12]\n" - ".inst 0xc0860470 // mova { z16.s-z19.s }, za3h.s[x12]\n" - ".inst 0xa042c1e4 // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" - ".inst 0xc0840482 // mova za2h.s[x12], { z4.s-z7.s }\n" - ".inst 0xa043c1f4 // ld1w { z20.s-z23.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840683 // mova za3h.s[x12], { z20.s-z23.s }\n" + ".inst 0xa040c1ec // ld1w { z12.s-z15.s }, pn8.b/Z, [x15]\n" + ".inst 0xc0860404 // mova { z4.s-z7.s }, za0h.s[x12]\n" + ".inst 0xc0840580 // mova za0h.s[x12], { z12.s-z15.s }\n" + ".inst 0xc086043c // mova { z28.s-z31.s }, za1h.s[x12]\n" + ".inst 0xa041c1ec // ld1w { z12.s-z15.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" + ".inst 0xc0840581 // mova za1h.s[x12], { z12.s-z15.s }\n" + ".inst 0xc0860448 // mova { z8.s-z11.s }, za2h.s[x12]\n" + ".inst 0xc0860474 // mova { z20.s-z23.s }, za3h.s[x12]\n" + ".inst 0xa042c1f0 // ld1w { z16.s-z19.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" + ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa043c1ec // ld1w { z12.s-z15.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840583 // mova za3h.s[x12], { z12.s-z15.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" - ".inst 0xa060c1c0 // st1w { z0.s-z3.s }, pn8.b, [x14]\n" + ".inst 0xa060c1c4 // st1w { z4.s-z7.s }, pn8.b, [x14]\n" "addvl x15, x15, #16\n" - ".inst 0xa061c1c8 // st1w { z8.s-z11.s }, pn8.b, [x14, #0x4, MUL VL]\n" - ".inst 0xa062c1dc // st1w { z28.s-z31.s }, pn8.b, [x14, #0x8, MUL VL]\n" - ".inst 0xa063c1d0 // st1w { z16.s-z19.s }, pn8.b, [x14, #0xc, MUL VL]\n" + ".inst 0xa061c1dc // st1w { z28.s-z31.s }, pn8.b, [x14, #0x4, MUL VL]\n" + ".inst 0xa062c1c8 // st1w { z8.s-z11.s }, pn8.b, [x14, #0x8, MUL VL]\n" + ".inst 0xa063c1d4 // st1w { z20.s-z23.s }, pn8.b, [x14, #0xc, MUL VL]\n" "addvl x14, x14, #16\n" "blt 11b\n" "b 29f\n" @@ -248,12 +245,12 @@ void sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL(const int8_t *const A, const in "mov x12, #0x0\n" "cntw x20\n" "13:" // Store to partial result buffer: Store only: Loop - ".inst 0xc0860408 // mova { z8.s-z11.s }, za0h.s[x12]\n" - ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" - ".inst 0xa060c1c8 // st1w { z8.s-z11.s }, pn8.b, [x14]\n" + ".inst 0xc0860400 // mova { z0.s-z3.s }, za0h.s[x12]\n" + ".inst 0xc0860428 // mova { z8.s-z11.s }, za1h.s[x12]\n" + ".inst 0xa060c1c0 // st1w { z0.s-z3.s }, pn8.b, [x14]\n" ".inst 0xc086044c // mova { z12.s-z15.s }, za2h.s[x12]\n" ".inst 0xc0860460 // mova { z0.s-z3.s }, za3h.s[x12]\n" - ".inst 0xa061c1c4 // st1w { z4.s-z7.s }, pn8.b, [x14, #0x4, MUL VL]\n" + ".inst 0xa061c1c8 // st1w { z8.s-z11.s }, pn8.b, [x14, #0x4, MUL VL]\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" ".inst 0xa062c1cc // st1w { z12.s-z15.s }, pn8.b, [x14, #0x8, MUL VL]\n" @@ -275,30 +272,30 @@ void sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL(const int8_t *const A, const in "and x20, x22, #0x3\n" "cbz x21, 16f\n" "15:" // Store to output array: Accumulator row 0 loop - ".inst 0xc086041c // mova { z28.s-z31.s }, za0h.s[x12]\n" - "st1w { z28.s }, p0, [x26]\n" + ".inst 0xc0860408 // mova { z8.s-z11.s }, za0h.s[x12]\n" + "st1w { z8.s }, p0, [x26]\n" "add x26, x26, x23\n" - "st1w { z29.s }, p0, [x26]\n" + "st1w { z9.s }, p0, [x26]\n" "add x26, x26, x23\n" "add x12, x12, #0x4\n" - "st1w { z30.s }, p0, [x26]\n" + "st1w { z10.s }, p0, [x26]\n" "add x26, x26, x23\n" "cmp x12, x21, LSL #2\n" - "st1w { z31.s }, p0, [x26]\n" + "st1w { z11.s }, p0, [x26]\n" "add x26, x26, x23\n" "blt 15b\n" "16:" // Store to output array: Accumulator row 0 oddments "cbz x20, 17f\n" "subs x20, x20, #0x1\n" - ".inst 0xc0860408 // mova { z8.s-z11.s }, za0h.s[x12]\n" - "st1w { z8.s }, p0, [x26]\n" + ".inst 0xc0860404 // mova { z4.s-z7.s }, za0h.s[x12]\n" + "st1w { z4.s }, p0, [x26]\n" "add x26, x26, x23\n" "beq 17f\n" "subs x20, x20, #0x1\n" - "st1w { z9.s }, p0, [x26]\n" + "st1w { z5.s }, p0, [x26]\n" "add x26, x26, x23\n" "beq 17f\n" - "st1w { z10.s }, p0, [x26]\n" + "st1w { z6.s }, p0, [x26]\n" "add x26, x26, x23\n" "17:" // Store to output array: Accumulator row 0 oddments: End "subs x25, x25, x22\n" @@ -310,30 +307,30 @@ void sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL(const int8_t *const A, const in "and x20, x22, #0x3\n" "cbz x21, 19f\n" "18:" // Store to output array: Accumulator row 1 loop - ".inst 0xc0860420 // mova { z0.s-z3.s }, za1h.s[x12]\n" - "st1w { z0.s }, p0, [x26]\n" + ".inst 0xc0860430 // mova { z16.s-z19.s }, za1h.s[x12]\n" + "st1w { z16.s }, p0, [x26]\n" "add x26, x26, x23\n" - "st1w { z1.s }, p0, [x26]\n" + "st1w { z17.s }, p0, [x26]\n" "add x26, x26, x23\n" "add x12, x12, #0x4\n" - "st1w { z2.s }, p0, [x26]\n" + "st1w { z18.s }, p0, [x26]\n" "add x26, x26, x23\n" "cmp x12, x21, LSL #2\n" - "st1w { z3.s }, p0, [x26]\n" + "st1w { z19.s }, p0, [x26]\n" "add x26, x26, x23\n" "blt 18b\n" "19:" // Store to output array: Accumulator row 1 oddments "cbz x20, 20f\n" "subs x20, x20, #0x1\n" - ".inst 0xc0860430 // mova { z16.s-z19.s }, za1h.s[x12]\n" - "st1w { z16.s }, p0, [x26]\n" + ".inst 0xc0860434 // mova { z20.s-z23.s }, za1h.s[x12]\n" + "st1w { z20.s }, p0, [x26]\n" "add x26, x26, x23\n" "beq 20f\n" "subs x20, x20, #0x1\n" - "st1w { z17.s }, p0, [x26]\n" + "st1w { z21.s }, p0, [x26]\n" "add x26, x26, x23\n" "beq 20f\n" - "st1w { z18.s }, p0, [x26]\n" + "st1w { z22.s }, p0, [x26]\n" "add x26, x26, x23\n" "20:" // Store to output array: Accumulator row 1 oddments: End "subs x25, x25, x22\n" @@ -345,30 +342,30 @@ void sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL(const int8_t *const A, const in "and x20, x22, #0x3\n" "cbz x21, 22f\n" "21:" // Store to output array: Accumulator row 2 loop - ".inst 0xc0860450 // mova { z16.s-z19.s }, za2h.s[x12]\n" - "st1w { z16.s }, p0, [x26]\n" + ".inst 0xc0860458 // mova { z24.s-z27.s }, za2h.s[x12]\n" + "st1w { z24.s }, p0, [x26]\n" "add x26, x26, x23\n" - "st1w { z17.s }, p0, [x26]\n" + "st1w { z25.s }, p0, [x26]\n" "add x26, x26, x23\n" "add x12, x12, #0x4\n" - "st1w { z18.s }, p0, [x26]\n" + "st1w { z26.s }, p0, [x26]\n" "add x26, x26, x23\n" "cmp x12, x21, LSL #2\n" - "st1w { z19.s }, p0, [x26]\n" + "st1w { z27.s }, p0, [x26]\n" "add x26, x26, x23\n" "blt 21b\n" "22:" // Store to output array: Accumulator row 2 oddments "cbz x20, 23f\n" "subs x20, x20, #0x1\n" - ".inst 0xc0860440 // mova { z0.s-z3.s }, za2h.s[x12]\n" - "st1w { z0.s }, p0, [x26]\n" + ".inst 0xc0860450 // mova { z16.s-z19.s }, za2h.s[x12]\n" + "st1w { z16.s }, p0, [x26]\n" "add x26, x26, x23\n" "beq 23f\n" "subs x20, x20, #0x1\n" - "st1w { z1.s }, p0, [x26]\n" + "st1w { z17.s }, p0, [x26]\n" "add x26, x26, x23\n" "beq 23f\n" - "st1w { z2.s }, p0, [x26]\n" + "st1w { z18.s }, p0, [x26]\n" "add x26, x26, x23\n" "23:" // Store to output array: Accumulator row 2 oddments: End "subs x25, x25, x22\n" @@ -380,44 +377,44 @@ void sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL(const int8_t *const A, const in "and x20, x20, #0x3\n" "cbz x21, 25f\n" "24:" // Store to output array: Accumulator row 3 loop - ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" - "st1w { z12.s }, p0, [x26]\n" + ".inst 0xc0860470 // mova { z16.s-z19.s }, za3h.s[x12]\n" + "st1w { z16.s }, p0, [x26]\n" "add x26, x26, x23\n" - "st1w { z13.s }, p0, [x26]\n" + "st1w { z17.s }, p0, [x26]\n" "add x26, x26, x23\n" "add x12, x12, #0x4\n" - "st1w { z14.s }, p0, [x26]\n" + "st1w { z18.s }, p0, [x26]\n" "add x26, x26, x23\n" "cmp x12, x21, LSL #2\n" - "st1w { z15.s }, p0, [x26]\n" + "st1w { z19.s }, p0, [x26]\n" "add x26, x26, x23\n" "blt 24b\n" "25:" // Store to output array: Accumulator row 3 oddments "cbz x20, 26f\n" "subs x20, x20, #0x1\n" - ".inst 0xc0860470 // mova { z16.s-z19.s }, za3h.s[x12]\n" - "st1w { z16.s }, p0, [x26]\n" + ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" + "st1w { z12.s }, p0, [x26]\n" "add x26, x26, x23\n" "beq 26f\n" "subs x20, x20, #0x1\n" - "st1w { z17.s }, p0, [x26]\n" + "st1w { z13.s }, p0, [x26]\n" "add x26, x26, x23\n" "beq 26f\n" - "st1w { z18.s }, p0, [x26]\n" + "st1w { z14.s }, p0, [x26]\n" "26:" // Store to output array: Accumulator row 3 oddments: End "27:" // Store to output array: End "tbz x16, #0, 29f\n" "mov x12, #0x0\n" "cntw x20\n" "28:" // Store to output array: Refill accumulators: Loop - ".inst 0xa040c1f0 // ld1w { z16.s-z19.s }, pn8.b/Z, [x15]\n" - ".inst 0xc0840600 // mova za0h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa040c1ec // ld1w { z12.s-z15.s }, pn8.b/Z, [x15]\n" + ".inst 0xc0840580 // mova za0h.s[x12], { z12.s-z15.s }\n" ".inst 0xa041c1ec // ld1w { z12.s-z15.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" ".inst 0xc0840581 // mova za1h.s[x12], { z12.s-z15.s }\n" - ".inst 0xa042c1f8 // ld1w { z24.s-z27.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" - ".inst 0xc0840702 // mova za2h.s[x12], { z24.s-z27.s }\n" - ".inst 0xa043c1e8 // ld1w { z8.s-z11.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840503 // mova za3h.s[x12], { z8.s-z11.s }\n" + ".inst 0xa042c1f0 // ld1w { z16.s-z19.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" + ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa043c1e4 // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840483 // mova za3h.s[x12], { z4.s-z7.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x15, x15, #16\n" @@ -441,4 +438,3 @@ void sme2_interleaved_nomerge_s8s32_mopa_4VLx1VL(const int8_t *const A, const in } // namespace arm_gemm #endif // ARM_COMPUTE_ENABLE_SME2 -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_1VLx4VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_1VLx4VL.hpp index c7bd38d905cb4a141bd03dbc882182b6381b1aed..fb848839132d2f21b3b1d6c97324047267c03857 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_1VLx4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_1VLx4VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,7 +23,7 @@ */ #pragma once -#ifdef __aarch64__ +#ifdef ARM_COMPUTE_ENABLE_SME2 #include #include "../std_transforms_sme.hpp" @@ -83,12 +83,11 @@ public: StdTransformsSME transforms = {}; - cls_sme2_interleaved_nomerge_u8q_mopa_1VLx4VL(const CPUInfo *ci) + cls_sme2_interleaved_nomerge_u8q_mopa_1VLx4VL(const CPUInfo *) { - ARM_COMPUTE_UNUSED(ci); } }; } // namespace arm_gemm -#endif // __aarch64__ +#endif // ARM_COMPUTE_ENABLE_SME2 diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_1VLx4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_1VLx4VL/generic.cpp index d868ed2b6718f9ff7d673781ade73606d65f7803..96247d2db5988cf686fea948fb399b5f3eb727bd 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_1VLx4VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_1VLx4VL/generic.cpp @@ -21,7 +21,6 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifdef __ARM_FEATURE_SVE #ifdef ARM_COMPUTE_ENABLE_SME2 #include "arm_gemm.hpp" @@ -100,14 +99,14 @@ void sme2_interleaved_nomerge_u8q_mopa_1VLx4VL(const uint8_t *const A, const uin "mov x12, #0x0\n" "cntw x20\n" "1:" // Initial accumulator load from buffer: Loop - ".inst 0xa040c5ac // ld1w { z12.s-z15.s }, pn9.b/Z, [x13]\n" - ".inst 0xc0840580 // mova za0h.s[x12], { z12.s-z15.s }\n" - ".inst 0xa041c5bc // ld1w { z28.s-z31.s }, pn9.b/Z, [x13, #0x4, MUL VL]\n" - ".inst 0xc0840781 // mova za1h.s[x12], { z28.s-z31.s }\n" - ".inst 0xa042c5bc // ld1w { z28.s-z31.s }, pn9.b/Z, [x13, #0x8, MUL VL]\n" - ".inst 0xc0840782 // mova za2h.s[x12], { z28.s-z31.s }\n" - ".inst 0xa043c5a4 // ld1w { z4.s-z7.s }, pn9.b/Z, [x13, #0xc, MUL VL]\n" - ".inst 0xc0840483 // mova za3h.s[x12], { z4.s-z7.s }\n" + ".inst 0xa040c5a4 // ld1w { z4.s-z7.s }, pn9.b/Z, [x13]\n" + ".inst 0xc0840480 // mova za0h.s[x12], { z4.s-z7.s }\n" + ".inst 0xa041c5a8 // ld1w { z8.s-z11.s }, pn9.b/Z, [x13, #0x4, MUL VL]\n" + ".inst 0xc0840501 // mova za1h.s[x12], { z8.s-z11.s }\n" + ".inst 0xa042c5a8 // ld1w { z8.s-z11.s }, pn9.b/Z, [x13, #0x8, MUL VL]\n" + ".inst 0xc0840502 // mova za2h.s[x12], { z8.s-z11.s }\n" + ".inst 0xa043c5b0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x13, #0xc, MUL VL]\n" + ".inst 0xc0840603 // mova za3h.s[x12], { z16.s-z19.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x13, x13, #16\n" @@ -125,11 +124,11 @@ void sme2_interleaved_nomerge_u8q_mopa_1VLx4VL(const uint8_t *const A, const uin "ldr x20, [%x[args], %[offsetof_bias]]\n" ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "cbz x20, 5f\n" - ".inst 0xa01cc299 // ldnt1w { z24.s-z27.s }, p8/Z, [x20, x28, LSL #2]\n" - ".inst 0xc0902700 // addha za0.s, p1/M, p1/M, z24.s\n" - ".inst 0xc0902721 // addha za1.s, p1/M, p1/M, z25.s\n" - ".inst 0xc0902742 // addha za2.s, p1/M, p1/M, z26.s\n" - ".inst 0xc0902763 // addha za3.s, p1/M, p1/M, z27.s\n" + ".inst 0xa11cc289 // ldnt1w { z1.s, z5.s, z9.s, z13.s }, p8/Z, [x20, x28, LSL #2]\n" + ".inst 0xc0902420 // addha za0.s, p1/M, p1/M, z1.s\n" + ".inst 0xc09024a1 // addha za1.s, p1/M, p1/M, z5.s\n" + ".inst 0xc0902522 // addha za2.s, p1/M, p1/M, z9.s\n" + ".inst 0xc09025a3 // addha za3.s, p1/M, p1/M, z13.s\n" "4:" // Prepare accumulators: Test for last block "mov x20, x28\n" "mov x21, x9\n" @@ -152,107 +151,107 @@ void sme2_interleaved_nomerge_u8q_mopa_1VLx4VL(const uint8_t *const A, const uin "madd x23, x28, x20, x23\n" // bptr = B + n * kstride_bytes "cbz x22, 8f\n" "subs x22, x22, #0x1\n" - "ld1b { z10.b }, p1/Z, [x25]\n" - ".inst 0xa04086fd // ldnt1b { z28.b-z31.b }, pn9.b/Z, [x23]\n" - "ld1b { z16.b }, p1/Z, [x25, #1, MUL VL]\n" - ".inst 0xa04186ed // ldnt1b { z12.b-z15.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n" - "ld1b { z21.b }, p1/Z, [x25, #2, MUL VL]\n" - ".inst 0xa04286f9 // ldnt1b { z24.b-z27.b }, pn9.b/Z, [x23, #0x8, MUL VL]\n" - "ld1b { z19.b }, p1/Z, [x25, #3, MUL VL]\n" + "ld1b { z20.b }, p1/Z, [x25]\n" + ".inst 0xa04086e5 // ldnt1b { z4.b-z7.b }, pn9.b/Z, [x23]\n" + "ld1b { z11.b }, p1/Z, [x25, #1, MUL VL]\n" + ".inst 0xa04186f9 // ldnt1b { z24.b-z27.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n" + "ld1b { z2.b }, p1/Z, [x25, #2, MUL VL]\n" + ".inst 0xa04286fd // ldnt1b { z28.b-z31.b }, pn9.b/Z, [x23, #0x8, MUL VL]\n" + "ld1b { z14.b }, p1/Z, [x25, #3, MUL VL]\n" "addvl x25, x25, #4\n" - ".inst 0xa04386e1 // ldnt1b { z0.b-z3.b }, pn9.b/Z, [x23, #0xc, MUL VL]\n" + ".inst 0xa04386f1 // ldnt1b { z16.b-z19.b }, pn9.b/Z, [x23, #0xc, MUL VL]\n" "addvl x23, x23, #16\n" "ble 7f\n" "6:" // K loop - ".inst 0xa1bc2540 // umopa za0.s, p1/M, p1/M, z10.b, z28.b\n" + ".inst 0xa1a42680 // umopa za0.s, p1/M, p1/M, z20.b, z4.b\n" "subs x22, x22, #0x1\n" - ".inst 0xa1bd2541 // umopa za1.s, p1/M, p1/M, z10.b, z29.b\n" - ".inst 0xa1be2542 // umopa za2.s, p1/M, p1/M, z10.b, z30.b\n" - ".inst 0xa1bf2543 // umopa za3.s, p1/M, p1/M, z10.b, z31.b\n" - "ld1b { z10.b }, p1/Z, [x25]\n" - ".inst 0xa1ac2600 // umopa za0.s, p1/M, p1/M, z16.b, z12.b\n" - ".inst 0xa04086fd // ldnt1b { z28.b-z31.b }, pn9.b/Z, [x23]\n" - ".inst 0xa1ad2601 // umopa za1.s, p1/M, p1/M, z16.b, z13.b\n" - ".inst 0xa1ae2602 // umopa za2.s, p1/M, p1/M, z16.b, z14.b\n" - ".inst 0xa1af2603 // umopa za3.s, p1/M, p1/M, z16.b, z15.b\n" - "ld1b { z16.b }, p1/Z, [x25, #1, MUL VL]\n" - ".inst 0xa1b826a0 // umopa za0.s, p1/M, p1/M, z21.b, z24.b\n" - ".inst 0xa04186ed // ldnt1b { z12.b-z15.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xa1b926a1 // umopa za1.s, p1/M, p1/M, z21.b, z25.b\n" - ".inst 0xa1ba26a2 // umopa za2.s, p1/M, p1/M, z21.b, z26.b\n" - ".inst 0xa1bb26a3 // umopa za3.s, p1/M, p1/M, z21.b, z27.b\n" - "ld1b { z21.b }, p1/Z, [x25, #2, MUL VL]\n" - ".inst 0xa04286f9 // ldnt1b { z24.b-z27.b }, pn9.b/Z, [x23, #0x8, MUL VL]\n" - ".inst 0xa1a02660 // umopa za0.s, p1/M, p1/M, z19.b, z0.b\n" - ".inst 0xa1a12661 // umopa za1.s, p1/M, p1/M, z19.b, z1.b\n" - ".inst 0xa1a22662 // umopa za2.s, p1/M, p1/M, z19.b, z2.b\n" - ".inst 0xa1a32663 // umopa za3.s, p1/M, p1/M, z19.b, z3.b\n" - "ld1b { z19.b }, p1/Z, [x25, #3, MUL VL]\n" + ".inst 0xa1a52681 // umopa za1.s, p1/M, p1/M, z20.b, z5.b\n" + ".inst 0xa1a62682 // umopa za2.s, p1/M, p1/M, z20.b, z6.b\n" + ".inst 0xa1a72683 // umopa za3.s, p1/M, p1/M, z20.b, z7.b\n" + "ld1b { z20.b }, p1/Z, [x25]\n" + ".inst 0xa1b82560 // umopa za0.s, p1/M, p1/M, z11.b, z24.b\n" + ".inst 0xa04086e5 // ldnt1b { z4.b-z7.b }, pn9.b/Z, [x23]\n" + ".inst 0xa1b92561 // umopa za1.s, p1/M, p1/M, z11.b, z25.b\n" + ".inst 0xa1ba2562 // umopa za2.s, p1/M, p1/M, z11.b, z26.b\n" + ".inst 0xa1bb2563 // umopa za3.s, p1/M, p1/M, z11.b, z27.b\n" + "ld1b { z11.b }, p1/Z, [x25, #1, MUL VL]\n" + ".inst 0xa1bc2440 // umopa za0.s, p1/M, p1/M, z2.b, z28.b\n" + ".inst 0xa04186f9 // ldnt1b { z24.b-z27.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xa1bd2441 // umopa za1.s, p1/M, p1/M, z2.b, z29.b\n" + ".inst 0xa1be2442 // umopa za2.s, p1/M, p1/M, z2.b, z30.b\n" + ".inst 0xa1bf2443 // umopa za3.s, p1/M, p1/M, z2.b, z31.b\n" + "ld1b { z2.b }, p1/Z, [x25, #2, MUL VL]\n" + ".inst 0xa04286fd // ldnt1b { z28.b-z31.b }, pn9.b/Z, [x23, #0x8, MUL VL]\n" + ".inst 0xa1b025c0 // umopa za0.s, p1/M, p1/M, z14.b, z16.b\n" + ".inst 0xa1b125c1 // umopa za1.s, p1/M, p1/M, z14.b, z17.b\n" + ".inst 0xa1b225c2 // umopa za2.s, p1/M, p1/M, z14.b, z18.b\n" + ".inst 0xa1b325c3 // umopa za3.s, p1/M, p1/M, z14.b, z19.b\n" + "ld1b { z14.b }, p1/Z, [x25, #3, MUL VL]\n" "addvl x25, x25, #4\n" - ".inst 0xa04386e1 // ldnt1b { z0.b-z3.b }, pn9.b/Z, [x23, #0xc, MUL VL]\n" + ".inst 0xa04386f1 // ldnt1b { z16.b-z19.b }, pn9.b/Z, [x23, #0xc, MUL VL]\n" "addvl x23, x23, #16\n" "bgt 6b\n" "7:" // K loop tail - ".inst 0xa1bc2540 // umopa za0.s, p1/M, p1/M, z10.b, z28.b\n" - ".inst 0xa1bd2541 // umopa za1.s, p1/M, p1/M, z10.b, z29.b\n" - ".inst 0xa1be2542 // umopa za2.s, p1/M, p1/M, z10.b, z30.b\n" - ".inst 0xa1bf2543 // umopa za3.s, p1/M, p1/M, z10.b, z31.b\n" - ".inst 0xa1ac2600 // umopa za0.s, p1/M, p1/M, z16.b, z12.b\n" - ".inst 0xa1ad2601 // umopa za1.s, p1/M, p1/M, z16.b, z13.b\n" - ".inst 0xa1ae2602 // umopa za2.s, p1/M, p1/M, z16.b, z14.b\n" - ".inst 0xa1af2603 // umopa za3.s, p1/M, p1/M, z16.b, z15.b\n" - ".inst 0xa1b826a0 // umopa za0.s, p1/M, p1/M, z21.b, z24.b\n" - ".inst 0xa1b926a1 // umopa za1.s, p1/M, p1/M, z21.b, z25.b\n" - ".inst 0xa1ba26a2 // umopa za2.s, p1/M, p1/M, z21.b, z26.b\n" - ".inst 0xa1bb26a3 // umopa za3.s, p1/M, p1/M, z21.b, z27.b\n" - ".inst 0xa1a02660 // umopa za0.s, p1/M, p1/M, z19.b, z0.b\n" - ".inst 0xa1a12661 // umopa za1.s, p1/M, p1/M, z19.b, z1.b\n" - ".inst 0xa1a22662 // umopa za2.s, p1/M, p1/M, z19.b, z2.b\n" - ".inst 0xa1a32663 // umopa za3.s, p1/M, p1/M, z19.b, z3.b\n" + ".inst 0xa1a42680 // umopa za0.s, p1/M, p1/M, z20.b, z4.b\n" + ".inst 0xa1a52681 // umopa za1.s, p1/M, p1/M, z20.b, z5.b\n" + ".inst 0xa1a62682 // umopa za2.s, p1/M, p1/M, z20.b, z6.b\n" + ".inst 0xa1a72683 // umopa za3.s, p1/M, p1/M, z20.b, z7.b\n" + ".inst 0xa1b82560 // umopa za0.s, p1/M, p1/M, z11.b, z24.b\n" + ".inst 0xa1b92561 // umopa za1.s, p1/M, p1/M, z11.b, z25.b\n" + ".inst 0xa1ba2562 // umopa za2.s, p1/M, p1/M, z11.b, z26.b\n" + ".inst 0xa1bb2563 // umopa za3.s, p1/M, p1/M, z11.b, z27.b\n" + ".inst 0xa1bc2440 // umopa za0.s, p1/M, p1/M, z2.b, z28.b\n" + ".inst 0xa1bd2441 // umopa za1.s, p1/M, p1/M, z2.b, z29.b\n" + ".inst 0xa1be2442 // umopa za2.s, p1/M, p1/M, z2.b, z30.b\n" + ".inst 0xa1bf2443 // umopa za3.s, p1/M, p1/M, z2.b, z31.b\n" + ".inst 0xa1b025c0 // umopa za0.s, p1/M, p1/M, z14.b, z16.b\n" + ".inst 0xa1b125c1 // umopa za1.s, p1/M, p1/M, z14.b, z17.b\n" + ".inst 0xa1b225c2 // umopa za2.s, p1/M, p1/M, z14.b, z18.b\n" + ".inst 0xa1b325c3 // umopa za3.s, p1/M, p1/M, z14.b, z19.b\n" "8:" // K oddments "cbz x21, 10f\n" "9:" // K oddments: Loop - "ld1b { z10.b }, p1/Z, [x25]\n" + "ld1b { z16.b }, p1/Z, [x25]\n" "subs x21, x21, #0x1\n" "addvl x25, x25, #1\n" - ".inst 0xa04086fc // ld1b { z28.b-z31.b }, pn9.b/Z, [x23]\n" + ".inst 0xa04086e4 // ld1b { z4.b-z7.b }, pn9.b/Z, [x23]\n" "addvl x23, x23, #4\n" - ".inst 0xa1bc2540 // umopa za0.s, p1/M, p1/M, z10.b, z28.b\n" - ".inst 0xa1bd2541 // umopa za1.s, p1/M, p1/M, z10.b, z29.b\n" - ".inst 0xa1be2542 // umopa za2.s, p1/M, p1/M, z10.b, z30.b\n" - ".inst 0xa1bf2543 // umopa za3.s, p1/M, p1/M, z10.b, z31.b\n" + ".inst 0xa1a42600 // umopa za0.s, p1/M, p1/M, z16.b, z4.b\n" + ".inst 0xa1a52601 // umopa za1.s, p1/M, p1/M, z16.b, z5.b\n" + ".inst 0xa1a62602 // umopa za2.s, p1/M, p1/M, z16.b, z6.b\n" + ".inst 0xa1a72603 // umopa za3.s, p1/M, p1/M, z16.b, z7.b\n" "bgt 9b\n" "10:" // K oddments: End - "ld1w { z14.s }, p1/Z, [x25]\n" + "ld1w { z15.s }, p1/Z, [x25]\n" "addvl x25, x25, #1\n" - ".inst 0xc09125c0 // addva za0.s, p1/M, p1/M, z14.s\n" - ".inst 0xc09125c1 // addva za1.s, p1/M, p1/M, z14.s\n" - ".inst 0xc09125c2 // addva za2.s, p1/M, p1/M, z14.s\n" - ".inst 0xc09125c3 // addva za3.s, p1/M, p1/M, z14.s\n" + ".inst 0xc09125e0 // addva za0.s, p1/M, p1/M, z15.s\n" + ".inst 0xc09125e1 // addva za1.s, p1/M, p1/M, z15.s\n" + ".inst 0xc09125e2 // addva za2.s, p1/M, p1/M, z15.s\n" + ".inst 0xc09125e3 // addva za3.s, p1/M, p1/M, z15.s\n" "tbz x14, #1, 14f\n" "tbz x14, #0, 12f\n" "mov x12, #0x0\n" "cntw x20\n" "11:" // Store to partial result buffer: Store and refill: Loop - ".inst 0xa040c5b8 // ld1w { z24.s-z27.s }, pn9.b/Z, [x13]\n" - ".inst 0xc086041c // mova { z28.s-z31.s }, za0h.s[x12]\n" - ".inst 0xc0840700 // mova za0h.s[x12], { z24.s-z27.s }\n" - ".inst 0xc0860428 // mova { z8.s-z11.s }, za1h.s[x12]\n" - ".inst 0xa041c5b0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x13, #0x4, MUL VL]\n" - ".inst 0xc0840601 // mova za1h.s[x12], { z16.s-z19.s }\n" - ".inst 0xc0860458 // mova { z24.s-z27.s }, za2h.s[x12]\n" - ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" - ".inst 0xa042c5a4 // ld1w { z4.s-z7.s }, pn9.b/Z, [x13, #0x8, MUL VL]\n" - ".inst 0xc0840482 // mova za2h.s[x12], { z4.s-z7.s }\n" - ".inst 0xa043c5a4 // ld1w { z4.s-z7.s }, pn9.b/Z, [x13, #0xc, MUL VL]\n" - ".inst 0xc0840483 // mova za3h.s[x12], { z4.s-z7.s }\n" + ".inst 0xa040c5a0 // ld1w { z0.s-z3.s }, pn9.b/Z, [x13]\n" + ".inst 0xc0860418 // mova { z24.s-z27.s }, za0h.s[x12]\n" + ".inst 0xc0840400 // mova za0h.s[x12], { z0.s-z3.s }\n" + ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" + ".inst 0xa041c5a0 // ld1w { z0.s-z3.s }, pn9.b/Z, [x13, #0x4, MUL VL]\n" + ".inst 0xc0840401 // mova za1h.s[x12], { z0.s-z3.s }\n" + ".inst 0xc0860454 // mova { z20.s-z23.s }, za2h.s[x12]\n" + ".inst 0xc0860460 // mova { z0.s-z3.s }, za3h.s[x12]\n" + ".inst 0xa042c5b0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x13, #0x8, MUL VL]\n" + ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa043c5b0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x13, #0xc, MUL VL]\n" + ".inst 0xc0840603 // mova za3h.s[x12], { z16.s-z19.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" - ".inst 0xa060c57c // st1w { z28.s-z31.s }, pn9.b, [x11]\n" + ".inst 0xa060c578 // st1w { z24.s-z27.s }, pn9.b, [x11]\n" "addvl x13, x13, #16\n" - ".inst 0xa061c568 // st1w { z8.s-z11.s }, pn9.b, [x11, #0x4, MUL VL]\n" - ".inst 0xa062c578 // st1w { z24.s-z27.s }, pn9.b, [x11, #0x8, MUL VL]\n" - ".inst 0xa063c56c // st1w { z12.s-z15.s }, pn9.b, [x11, #0xc, MUL VL]\n" + ".inst 0xa061c564 // st1w { z4.s-z7.s }, pn9.b, [x11, #0x4, MUL VL]\n" + ".inst 0xa062c574 // st1w { z20.s-z23.s }, pn9.b, [x11, #0x8, MUL VL]\n" + ".inst 0xa063c560 // st1w { z0.s-z3.s }, pn9.b, [x11, #0xc, MUL VL]\n" "addvl x11, x11, #16\n" "blt 11b\n" "b 21f\n" @@ -260,16 +259,16 @@ void sme2_interleaved_nomerge_u8q_mopa_1VLx4VL(const uint8_t *const A, const uin "mov x12, #0x0\n" "cntw x20\n" "13:" // Store to partial result buffer: Store only: Loop - ".inst 0xc086041c // mova { z28.s-z31.s }, za0h.s[x12]\n" - ".inst 0xc0860420 // mova { z0.s-z3.s }, za1h.s[x12]\n" - ".inst 0xa060c57c // st1w { z28.s-z31.s }, pn9.b, [x11]\n" - ".inst 0xc0860448 // mova { z8.s-z11.s }, za2h.s[x12]\n" - ".inst 0xc0860470 // mova { z16.s-z19.s }, za3h.s[x12]\n" - ".inst 0xa061c560 // st1w { z0.s-z3.s }, pn9.b, [x11, #0x4, MUL VL]\n" + ".inst 0xc0860404 // mova { z4.s-z7.s }, za0h.s[x12]\n" + ".inst 0xc0860434 // mova { z20.s-z23.s }, za1h.s[x12]\n" + ".inst 0xa060c564 // st1w { z4.s-z7.s }, pn9.b, [x11]\n" + ".inst 0xc086044c // mova { z12.s-z15.s }, za2h.s[x12]\n" + ".inst 0xc0860468 // mova { z8.s-z11.s }, za3h.s[x12]\n" + ".inst 0xa061c574 // st1w { z20.s-z23.s }, pn9.b, [x11, #0x4, MUL VL]\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" - ".inst 0xa062c568 // st1w { z8.s-z11.s }, pn9.b, [x11, #0x8, MUL VL]\n" - ".inst 0xa063c570 // st1w { z16.s-z19.s }, pn9.b, [x11, #0xc, MUL VL]\n" + ".inst 0xa062c56c // st1w { z12.s-z15.s }, pn9.b, [x11, #0x8, MUL VL]\n" + ".inst 0xa063c568 // st1w { z8.s-z11.s }, pn9.b, [x11, #0xc, MUL VL]\n" "addvl x11, x11, #16\n" "blt 13b\n" "b 21f\n" @@ -277,17 +276,17 @@ void sme2_interleaved_nomerge_u8q_mopa_1VLx4VL(const uint8_t *const A, const uin "ldr x24, [%x[args], %[offsetof_C]]\n" "add x24, x24, x28\n" // C += n "sub x23, x10, x9\n" - "ld1rw { z12.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z4.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" "ldr x22, [%x[args], %[offsetof_ldcb]]\n" "madd x24, x9, x22, x24\n" // C += m * ldc - "ld1rw { z13.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" - "ld1rw { z14.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" - "ld1rw { z15.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" - "ld1rw { z4.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" - "ld1rw { z5.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" - "ld1rw { z6.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" - "ld1rw { z7.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" - "ld1rw { z1.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_c_offset]]\n" + "ld1rw { z5.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z6.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z7.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z12.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z13.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z14.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z15.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z0.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_c_offset]]\n" "ld1rw { z21.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_minval]]\n" "ld1rw { z20.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_maxval]]\n" "tbz x14, #2, 15f\n" @@ -295,10 +294,10 @@ void sme2_interleaved_nomerge_u8q_mopa_1VLx4VL(const uint8_t *const A, const uin "add x21, x21, x28\n" "ldr x20, [%x[rq], %[offsetof_Requantize32_per_channel_muls]]\n" "add x20, x20, x21, LSL #2\n" - ".inst 0xa040c28c // ld1w { z12.s-z15.s }, p8/Z, [x20]\n" + ".inst 0xa040c284 // ld1w { z4.s-z7.s }, p8/Z, [x20]\n" "ldr x20, [%x[rq], %[offsetof_Requantize32_per_channel_right_shifts]]\n" "add x20, x20, x21, LSL #2\n" - ".inst 0xa040c284 // ld1w { z4.s-z7.s }, p8/Z, [x20]\n" + ".inst 0xa040c28c // ld1w { z12.s-z15.s }, p8/Z, [x20]\n" "15:" // Store to output array: Load per-channel parameters: End "cntw x20\n" "whilelt p0.b, x28, x27\n" @@ -311,22 +310,22 @@ void sme2_interleaved_nomerge_u8q_mopa_1VLx4VL(const uint8_t *const A, const uin "16:" // Store to output array: Accumulator row 0 loop ".inst 0xc086001a // mova { z26.s-z27.s }, za0h.s[x12, 0:1]\n" ".inst 0xc086005c // mova { z28.s-z29.s }, za1h.s[x12, 0:1]\n" - ".inst 0xc1aca41a // sqdmulh { z26.s-z27.s }, { z26.s-z27.s }, z12.s\n" + ".inst 0xc1a4a41a // sqdmulh { z26.s-z27.s }, { z26.s-z27.s }, z4.s\n" ".inst 0xc0860096 // mova { z22.s-z23.s }, za2h.s[x12, 0:1]\n" ".inst 0xc08600d0 // mova { z16.s-z17.s }, za3h.s[x12, 0:1]\n" - ".inst 0xc1ada41c // sqdmulh { z28.s-z29.s }, { z28.s-z29.s }, z13.s\n" - ".inst 0xc1aea416 // sqdmulh { z22.s-z23.s }, { z22.s-z23.s }, z14.s\n" + ".inst 0xc1a5a41c // sqdmulh { z28.s-z29.s }, { z28.s-z29.s }, z5.s\n" + ".inst 0xc1a6a416 // sqdmulh { z22.s-z23.s }, { z22.s-z23.s }, z6.s\n" "add x12, x12, #0x2\n" "cmp x12, x21, LSL #1\n" - ".inst 0xc1afa410 // sqdmulh { z16.s-z17.s }, { z16.s-z17.s }, z15.s\n" - ".inst 0xc1a4a23a // srshl { z26.s-z27.s }, { z26.s-z27.s }, z4.s\n" - ".inst 0xc1a5a23c // srshl { z28.s-z29.s }, { z28.s-z29.s }, z5.s\n" - ".inst 0xc1a6a236 // srshl { z22.s-z23.s }, { z22.s-z23.s }, z6.s\n" - ".inst 0xc1a7a230 // srshl { z16.s-z17.s }, { z16.s-z17.s }, z7.s\n" - ".inst 0xc1a1a31a // add { z26.s-z27.s }, { z26.s-z27.s }, z1.s\n" - ".inst 0xc1a1a31c // add { z28.s-z29.s }, { z28.s-z29.s }, z1.s\n" - ".inst 0xc1a1a316 // add { z22.s-z23.s }, { z22.s-z23.s }, z1.s\n" - ".inst 0xc1a1a310 // add { z16.s-z17.s }, { z16.s-z17.s }, z1.s\n" + ".inst 0xc1a7a410 // sqdmulh { z16.s-z17.s }, { z16.s-z17.s }, z7.s\n" + ".inst 0xc1aca23a // srshl { z26.s-z27.s }, { z26.s-z27.s }, z12.s\n" + ".inst 0xc1ada23c // srshl { z28.s-z29.s }, { z28.s-z29.s }, z13.s\n" + ".inst 0xc1aea236 // srshl { z22.s-z23.s }, { z22.s-z23.s }, z14.s\n" + ".inst 0xc1afa230 // srshl { z16.s-z17.s }, { z16.s-z17.s }, z15.s\n" + ".inst 0xc1a0a31a // add { z26.s-z27.s }, { z26.s-z27.s }, z0.s\n" + ".inst 0xc1a0a31c // add { z28.s-z29.s }, { z28.s-z29.s }, z0.s\n" + ".inst 0xc1a0a316 // add { z22.s-z23.s }, { z22.s-z23.s }, z0.s\n" + ".inst 0xc1a0a310 // add { z16.s-z17.s }, { z16.s-z17.s }, z0.s\n" ".inst 0xc1b4c6ba // sclamp { z26.s-z27.s }, z21.s, z20.s\n" ".inst 0xc1b4c6bc // sclamp { z28.s-z29.s }, z21.s, z20.s\n" "uzp1 z19.b, z26.b, z28.b\n" @@ -344,29 +343,29 @@ void sme2_interleaved_nomerge_u8q_mopa_1VLx4VL(const uint8_t *const A, const uin "blt 16b\n" "17:" // Store to output array: Accumulator row 0 oddments "cbz x20, 18f\n" - ".inst 0xc0860002 // mova { z2.s-z3.s }, za0h.s[x12, 0:1]\n" + ".inst 0xc086000a // mova { z10.s-z11.s }, za0h.s[x12, 0:1]\n" ".inst 0xc0860058 // mova { z24.s-z25.s }, za1h.s[x12, 0:1]\n" - ".inst 0xc1aca402 // sqdmulh { z2.s-z3.s }, { z2.s-z3.s }, z12.s\n" - ".inst 0xc0860090 // mova { z16.s-z17.s }, za2h.s[x12, 0:1]\n" - ".inst 0xc08600ca // mova { z10.s-z11.s }, za3h.s[x12, 0:1]\n" - ".inst 0xc1ada418 // sqdmulh { z24.s-z25.s }, { z24.s-z25.s }, z13.s\n" - ".inst 0xc1aea410 // sqdmulh { z16.s-z17.s }, { z16.s-z17.s }, z14.s\n" - ".inst 0xc1afa40a // sqdmulh { z10.s-z11.s }, { z10.s-z11.s }, z15.s\n" - ".inst 0xc1a4a222 // srshl { z2.s-z3.s }, { z2.s-z3.s }, z4.s\n" - ".inst 0xc1a5a238 // srshl { z24.s-z25.s }, { z24.s-z25.s }, z5.s\n" - ".inst 0xc1a6a230 // srshl { z16.s-z17.s }, { z16.s-z17.s }, z6.s\n" - ".inst 0xc1a7a22a // srshl { z10.s-z11.s }, { z10.s-z11.s }, z7.s\n" - ".inst 0xc1a1a302 // add { z2.s-z3.s }, { z2.s-z3.s }, z1.s\n" - ".inst 0xc1a1a318 // add { z24.s-z25.s }, { z24.s-z25.s }, z1.s\n" - ".inst 0xc1a1a310 // add { z16.s-z17.s }, { z16.s-z17.s }, z1.s\n" - ".inst 0xc1a1a30a // add { z10.s-z11.s }, { z10.s-z11.s }, z1.s\n" - ".inst 0xc1b4c6a2 // sclamp { z2.s-z3.s }, z21.s, z20.s\n" - ".inst 0xc1b4c6b8 // sclamp { z24.s-z25.s }, z21.s, z20.s\n" - "uzp1 z23.b, z2.b, z24.b\n" - ".inst 0xc1b4c6b0 // sclamp { z16.s-z17.s }, z21.s, z20.s\n" + ".inst 0xc1a4a40a // sqdmulh { z10.s-z11.s }, { z10.s-z11.s }, z4.s\n" + ".inst 0xc086009a // mova { z26.s-z27.s }, za2h.s[x12, 0:1]\n" + ".inst 0xc08600de // mova { z30.s-z31.s }, za3h.s[x12, 0:1]\n" + ".inst 0xc1a5a418 // sqdmulh { z24.s-z25.s }, { z24.s-z25.s }, z5.s\n" + ".inst 0xc1a6a41a // sqdmulh { z26.s-z27.s }, { z26.s-z27.s }, z6.s\n" + ".inst 0xc1a7a41e // sqdmulh { z30.s-z31.s }, { z30.s-z31.s }, z7.s\n" + ".inst 0xc1aca22a // srshl { z10.s-z11.s }, { z10.s-z11.s }, z12.s\n" + ".inst 0xc1ada238 // srshl { z24.s-z25.s }, { z24.s-z25.s }, z13.s\n" + ".inst 0xc1aea23a // srshl { z26.s-z27.s }, { z26.s-z27.s }, z14.s\n" + ".inst 0xc1afa23e // srshl { z30.s-z31.s }, { z30.s-z31.s }, z15.s\n" + ".inst 0xc1a0a30a // add { z10.s-z11.s }, { z10.s-z11.s }, z0.s\n" + ".inst 0xc1a0a318 // add { z24.s-z25.s }, { z24.s-z25.s }, z0.s\n" + ".inst 0xc1a0a31a // add { z26.s-z27.s }, { z26.s-z27.s }, z0.s\n" + ".inst 0xc1a0a31e // add { z30.s-z31.s }, { z30.s-z31.s }, z0.s\n" ".inst 0xc1b4c6aa // sclamp { z10.s-z11.s }, z21.s, z20.s\n" - "uzp1 z16.b, z16.b, z10.b\n" - "uzp1 z16.b, z23.b, z16.b\n" + ".inst 0xc1b4c6b8 // sclamp { z24.s-z25.s }, z21.s, z20.s\n" + "uzp1 z17.b, z10.b, z24.b\n" + ".inst 0xc1b4c6ba // sclamp { z26.s-z27.s }, z21.s, z20.s\n" + ".inst 0xc1b4c6be // sclamp { z30.s-z31.s }, z21.s, z20.s\n" + "uzp1 z16.b, z26.b, z30.b\n" + "uzp1 z16.b, z17.b, z16.b\n" "st1b { z16.b }, p0, [x24]\n" "18:" // Store to output array: Accumulator row 0 oddments: End "19:" // Store to output array: End @@ -374,14 +373,14 @@ void sme2_interleaved_nomerge_u8q_mopa_1VLx4VL(const uint8_t *const A, const uin "mov x12, #0x0\n" "cntw x20\n" "20:" // Store to output array: Refill accumulators: Loop - ".inst 0xa040c5b0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x13]\n" - ".inst 0xc0840600 // mova za0h.s[x12], { z16.s-z19.s }\n" - ".inst 0xa041c5ac // ld1w { z12.s-z15.s }, pn9.b/Z, [x13, #0x4, MUL VL]\n" - ".inst 0xc0840581 // mova za1h.s[x12], { z12.s-z15.s }\n" - ".inst 0xa042c5b0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x13, #0x8, MUL VL]\n" - ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" - ".inst 0xa043c5ac // ld1w { z12.s-z15.s }, pn9.b/Z, [x13, #0xc, MUL VL]\n" - ".inst 0xc0840583 // mova za3h.s[x12], { z12.s-z15.s }\n" + ".inst 0xa040c5bc // ld1w { z28.s-z31.s }, pn9.b/Z, [x13]\n" + ".inst 0xc0840780 // mova za0h.s[x12], { z28.s-z31.s }\n" + ".inst 0xa041c5b0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x13, #0x4, MUL VL]\n" + ".inst 0xc0840601 // mova za1h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa042c5ac // ld1w { z12.s-z15.s }, pn9.b/Z, [x13, #0x8, MUL VL]\n" + ".inst 0xc0840582 // mova za2h.s[x12], { z12.s-z15.s }\n" + ".inst 0xa043c5a0 // ld1w { z0.s-z3.s }, pn9.b/Z, [x13, #0xc, MUL VL]\n" + ".inst 0xc0840403 // mova za3h.s[x12], { z0.s-z3.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x13, x13, #16\n" @@ -405,4 +404,3 @@ void sme2_interleaved_nomerge_u8q_mopa_1VLx4VL(const uint8_t *const A, const uin } // namespace arm_gemm #endif // ARM_COMPUTE_ENABLE_SME2 -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_2VLx2VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_2VLx2VL.hpp index 123405bd172fbd06b2938419bc54a915bf2dfa8f..f8c375f9f5b22f6399e7cdb302c9deca5969d07a 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_2VLx2VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_2VLx2VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,7 +23,7 @@ */ #pragma once -#ifdef __aarch64__ +#ifdef ARM_COMPUTE_ENABLE_SME2 #include #include "../std_transforms_sme.hpp" @@ -83,12 +83,11 @@ public: StdTransformsSME transforms = {}; - cls_sme2_interleaved_nomerge_u8q_mopa_2VLx2VL(const CPUInfo *ci) + cls_sme2_interleaved_nomerge_u8q_mopa_2VLx2VL(const CPUInfo *) { - ARM_COMPUTE_UNUSED(ci); } }; } // namespace arm_gemm -#endif // __aarch64__ +#endif // ARM_COMPUTE_ENABLE_SME2 diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_2VLx2VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_2VLx2VL/generic.cpp index cb0e9521e31e46588d8d78bbee32fe9eafd7e068..9a597995299e53fb4c9ba014ac94748b345c6f46 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_2VLx2VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_2VLx2VL/generic.cpp @@ -21,7 +21,6 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifdef __ARM_FEATURE_SVE #ifdef ARM_COMPUTE_ENABLE_SME2 #include "arm_gemm.hpp" @@ -100,14 +99,14 @@ void sme2_interleaved_nomerge_u8q_mopa_2VLx2VL(const uint8_t *const A, const uin "mov x12, #0x0\n" "cntw x20\n" "1:" // Initial accumulator load from buffer: Loop - ".inst 0xa040c5e0 // ld1w { z0.s-z3.s }, pn9.b/Z, [x15]\n" - ".inst 0xc0840400 // mova za0h.s[x12], { z0.s-z3.s }\n" - ".inst 0xa041c5ec // ld1w { z12.s-z15.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n" - ".inst 0xc0840581 // mova za1h.s[x12], { z12.s-z15.s }\n" - ".inst 0xa042c5e0 // ld1w { z0.s-z3.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n" - ".inst 0xc0840402 // mova za2h.s[x12], { z0.s-z3.s }\n" - ".inst 0xa043c5fc // ld1w { z28.s-z31.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840783 // mova za3h.s[x12], { z28.s-z31.s }\n" + ".inst 0xa040c5e8 // ld1w { z8.s-z11.s }, pn9.b/Z, [x15]\n" + ".inst 0xc0840500 // mova za0h.s[x12], { z8.s-z11.s }\n" + ".inst 0xa041c5e0 // ld1w { z0.s-z3.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n" + ".inst 0xc0840401 // mova za1h.s[x12], { z0.s-z3.s }\n" + ".inst 0xa042c5f0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n" + ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa043c5f0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840603 // mova za3h.s[x12], { z16.s-z19.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x15, x15, #16\n" @@ -125,11 +124,11 @@ void sme2_interleaved_nomerge_u8q_mopa_2VLx2VL(const uint8_t *const A, const uin "ldr x20, [%x[args], %[offsetof_bias]]\n" ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "cbz x20, 5f\n" - ".inst 0xa00a4295 // ldnt1w { z20.s-z21.s }, p8/Z, [x20, x10, LSL #2]\n" - ".inst 0xc0902680 // addha za0.s, p1/M, p1/M, z20.s\n" - ".inst 0xc09026a1 // addha za1.s, p1/M, p1/M, z21.s\n" - ".inst 0xc0902682 // addha za2.s, p1/M, p1/M, z20.s\n" - ".inst 0xc09026a3 // addha za3.s, p1/M, p1/M, z21.s\n" + ".inst 0xa00a4299 // ldnt1w { z24.s-z25.s }, p8/Z, [x20, x10, LSL #2]\n" + ".inst 0xc0902700 // addha za0.s, p1/M, p1/M, z24.s\n" + ".inst 0xc0902721 // addha za1.s, p1/M, p1/M, z25.s\n" + ".inst 0xc0902702 // addha za2.s, p1/M, p1/M, z24.s\n" + ".inst 0xc0902723 // addha za3.s, p1/M, p1/M, z25.s\n" "4:" // Prepare accumulators: Test for last block "mov x20, x10\n" "mov x21, x11\n" @@ -152,75 +151,75 @@ void sme2_interleaved_nomerge_u8q_mopa_2VLx2VL(const uint8_t *const A, const uin "madd x23, x10, x20, x23\n" // bptr = B + n * kstride_bytes "cbz x22, 8f\n" "subs x22, x22, #0x1\n" - ".inst 0xa040077e // ld1b { z30.b-z31.b }, pn9.b/Z, [x27]\n" - ".inst 0xa04006f1 // ldnt1b { z16.b-z17.b }, pn9.b/Z, [x23]\n" - ".inst 0xa041076e // ld1b { z14.b-z15.b }, pn9.b/Z, [x27, #0x2, MUL VL]\n" - ".inst 0xa04106e9 // ldnt1b { z8.b-z9.b }, pn9.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0xa0420760 // ld1b { z0.b-z1.b }, pn9.b/Z, [x27, #0x4, MUL VL]\n" - ".inst 0xa14206fc // ldnt1b { z20.b, z28.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xa0430764 // ld1b { z4.b-z5.b }, pn9.b/Z, [x27, #0x6, MUL VL]\n" + ".inst 0xa1400763 // ld1b { z3.b, z11.b }, pn9.b/Z, [x27]\n" + ".inst 0xa14006f9 // ldnt1b { z17.b, z25.b }, pn9.b/Z, [x23]\n" + ".inst 0xa1410774 // ld1b { z20.b, z28.b }, pn9.b/Z, [x27, #0x2, MUL VL]\n" + ".inst 0xa04106f7 // ldnt1b { z22.b-z23.b }, pn9.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xa1420775 // ld1b { z21.b, z29.b }, pn9.b/Z, [x27, #0x4, MUL VL]\n" + ".inst 0xa14206f8 // ldnt1b { z16.b, z24.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xa1430765 // ld1b { z5.b, z13.b }, pn9.b/Z, [x27, #0x6, MUL VL]\n" "addvl x27, x27, #8\n" - ".inst 0xa14306ea // ldnt1b { z2.b, z10.b }, pn9.b/Z, [x23, #0x6, MUL VL]\n" + ".inst 0xa14306ef // ldnt1b { z7.b, z15.b }, pn9.b/Z, [x23, #0x6, MUL VL]\n" "addvl x23, x23, #8\n" "ble 7f\n" "6:" // K loop - ".inst 0xa1b027c0 // umopa za0.s, p1/M, p1/M, z30.b, z16.b\n" + ".inst 0xa1b12460 // umopa za0.s, p1/M, p1/M, z3.b, z17.b\n" "subs x22, x22, #0x1\n" - ".inst 0xa1b127c1 // umopa za1.s, p1/M, p1/M, z30.b, z17.b\n" - ".inst 0xa1b027e2 // umopa za2.s, p1/M, p1/M, z31.b, z16.b\n" - ".inst 0xa1b127e3 // umopa za3.s, p1/M, p1/M, z31.b, z17.b\n" - ".inst 0xa040077e // ld1b { z30.b-z31.b }, pn9.b/Z, [x27]\n" - ".inst 0xa1a825c0 // umopa za0.s, p1/M, p1/M, z14.b, z8.b\n" - ".inst 0xa04006f1 // ldnt1b { z16.b-z17.b }, pn9.b/Z, [x23]\n" - ".inst 0xa1a925c1 // umopa za1.s, p1/M, p1/M, z14.b, z9.b\n" - ".inst 0xa1a825e2 // umopa za2.s, p1/M, p1/M, z15.b, z8.b\n" - ".inst 0xa1a925e3 // umopa za3.s, p1/M, p1/M, z15.b, z9.b\n" - ".inst 0xa041076e // ld1b { z14.b-z15.b }, pn9.b/Z, [x27, #0x2, MUL VL]\n" - ".inst 0xa1b42400 // umopa za0.s, p1/M, p1/M, z0.b, z20.b\n" - ".inst 0xa04106e9 // ldnt1b { z8.b-z9.b }, pn9.b/Z, [x23, #0x2, MUL VL]\n" - ".inst 0xa1bc2401 // umopa za1.s, p1/M, p1/M, z0.b, z28.b\n" - ".inst 0xa1b42422 // umopa za2.s, p1/M, p1/M, z1.b, z20.b\n" - ".inst 0xa1bc2423 // umopa za3.s, p1/M, p1/M, z1.b, z28.b\n" - ".inst 0xa0420760 // ld1b { z0.b-z1.b }, pn9.b/Z, [x27, #0x4, MUL VL]\n" - ".inst 0xa14206fc // ldnt1b { z20.b, z28.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n" - ".inst 0xa1a22480 // umopa za0.s, p1/M, p1/M, z4.b, z2.b\n" - ".inst 0xa1aa2481 // umopa za1.s, p1/M, p1/M, z4.b, z10.b\n" - ".inst 0xa1a224a2 // umopa za2.s, p1/M, p1/M, z5.b, z2.b\n" - ".inst 0xa1aa24a3 // umopa za3.s, p1/M, p1/M, z5.b, z10.b\n" - ".inst 0xa0430764 // ld1b { z4.b-z5.b }, pn9.b/Z, [x27, #0x6, MUL VL]\n" + ".inst 0xa1b92461 // umopa za1.s, p1/M, p1/M, z3.b, z25.b\n" + ".inst 0xa1b12562 // umopa za2.s, p1/M, p1/M, z11.b, z17.b\n" + ".inst 0xa1b92563 // umopa za3.s, p1/M, p1/M, z11.b, z25.b\n" + ".inst 0xa1400763 // ld1b { z3.b, z11.b }, pn9.b/Z, [x27]\n" + ".inst 0xa1b62680 // umopa za0.s, p1/M, p1/M, z20.b, z22.b\n" + ".inst 0xa14006f9 // ldnt1b { z17.b, z25.b }, pn9.b/Z, [x23]\n" + ".inst 0xa1b72681 // umopa za1.s, p1/M, p1/M, z20.b, z23.b\n" + ".inst 0xa1b62782 // umopa za2.s, p1/M, p1/M, z28.b, z22.b\n" + ".inst 0xa1b72783 // umopa za3.s, p1/M, p1/M, z28.b, z23.b\n" + ".inst 0xa1410774 // ld1b { z20.b, z28.b }, pn9.b/Z, [x27, #0x2, MUL VL]\n" + ".inst 0xa1b026a0 // umopa za0.s, p1/M, p1/M, z21.b, z16.b\n" + ".inst 0xa04106f7 // ldnt1b { z22.b-z23.b }, pn9.b/Z, [x23, #0x2, MUL VL]\n" + ".inst 0xa1b826a1 // umopa za1.s, p1/M, p1/M, z21.b, z24.b\n" + ".inst 0xa1b027a2 // umopa za2.s, p1/M, p1/M, z29.b, z16.b\n" + ".inst 0xa1b827a3 // umopa za3.s, p1/M, p1/M, z29.b, z24.b\n" + ".inst 0xa1420775 // ld1b { z21.b, z29.b }, pn9.b/Z, [x27, #0x4, MUL VL]\n" + ".inst 0xa14206f8 // ldnt1b { z16.b, z24.b }, pn9.b/Z, [x23, #0x4, MUL VL]\n" + ".inst 0xa1a724a0 // umopa za0.s, p1/M, p1/M, z5.b, z7.b\n" + ".inst 0xa1af24a1 // umopa za1.s, p1/M, p1/M, z5.b, z15.b\n" + ".inst 0xa1a725a2 // umopa za2.s, p1/M, p1/M, z13.b, z7.b\n" + ".inst 0xa1af25a3 // umopa za3.s, p1/M, p1/M, z13.b, z15.b\n" + ".inst 0xa1430765 // ld1b { z5.b, z13.b }, pn9.b/Z, [x27, #0x6, MUL VL]\n" "addvl x27, x27, #8\n" - ".inst 0xa14306ea // ldnt1b { z2.b, z10.b }, pn9.b/Z, [x23, #0x6, MUL VL]\n" + ".inst 0xa14306ef // ldnt1b { z7.b, z15.b }, pn9.b/Z, [x23, #0x6, MUL VL]\n" "addvl x23, x23, #8\n" "bgt 6b\n" "7:" // K loop tail - ".inst 0xa1b027c0 // umopa za0.s, p1/M, p1/M, z30.b, z16.b\n" - ".inst 0xa1b127c1 // umopa za1.s, p1/M, p1/M, z30.b, z17.b\n" - ".inst 0xa1b027e2 // umopa za2.s, p1/M, p1/M, z31.b, z16.b\n" - ".inst 0xa1b127e3 // umopa za3.s, p1/M, p1/M, z31.b, z17.b\n" - ".inst 0xa1a825c0 // umopa za0.s, p1/M, p1/M, z14.b, z8.b\n" - ".inst 0xa1a925c1 // umopa za1.s, p1/M, p1/M, z14.b, z9.b\n" - ".inst 0xa1a825e2 // umopa za2.s, p1/M, p1/M, z15.b, z8.b\n" - ".inst 0xa1a925e3 // umopa za3.s, p1/M, p1/M, z15.b, z9.b\n" - ".inst 0xa1b42400 // umopa za0.s, p1/M, p1/M, z0.b, z20.b\n" - ".inst 0xa1bc2401 // umopa za1.s, p1/M, p1/M, z0.b, z28.b\n" - ".inst 0xa1b42422 // umopa za2.s, p1/M, p1/M, z1.b, z20.b\n" - ".inst 0xa1bc2423 // umopa za3.s, p1/M, p1/M, z1.b, z28.b\n" - ".inst 0xa1a22480 // umopa za0.s, p1/M, p1/M, z4.b, z2.b\n" - ".inst 0xa1aa2481 // umopa za1.s, p1/M, p1/M, z4.b, z10.b\n" - ".inst 0xa1a224a2 // umopa za2.s, p1/M, p1/M, z5.b, z2.b\n" - ".inst 0xa1aa24a3 // umopa za3.s, p1/M, p1/M, z5.b, z10.b\n" + ".inst 0xa1b12460 // umopa za0.s, p1/M, p1/M, z3.b, z17.b\n" + ".inst 0xa1b92461 // umopa za1.s, p1/M, p1/M, z3.b, z25.b\n" + ".inst 0xa1b12562 // umopa za2.s, p1/M, p1/M, z11.b, z17.b\n" + ".inst 0xa1b92563 // umopa za3.s, p1/M, p1/M, z11.b, z25.b\n" + ".inst 0xa1b62680 // umopa za0.s, p1/M, p1/M, z20.b, z22.b\n" + ".inst 0xa1b72681 // umopa za1.s, p1/M, p1/M, z20.b, z23.b\n" + ".inst 0xa1b62782 // umopa za2.s, p1/M, p1/M, z28.b, z22.b\n" + ".inst 0xa1b72783 // umopa za3.s, p1/M, p1/M, z28.b, z23.b\n" + ".inst 0xa1b026a0 // umopa za0.s, p1/M, p1/M, z21.b, z16.b\n" + ".inst 0xa1b826a1 // umopa za1.s, p1/M, p1/M, z21.b, z24.b\n" + ".inst 0xa1b027a2 // umopa za2.s, p1/M, p1/M, z29.b, z16.b\n" + ".inst 0xa1b827a3 // umopa za3.s, p1/M, p1/M, z29.b, z24.b\n" + ".inst 0xa1a724a0 // umopa za0.s, p1/M, p1/M, z5.b, z7.b\n" + ".inst 0xa1af24a1 // umopa za1.s, p1/M, p1/M, z5.b, z15.b\n" + ".inst 0xa1a725a2 // umopa za2.s, p1/M, p1/M, z13.b, z7.b\n" + ".inst 0xa1af25a3 // umopa za3.s, p1/M, p1/M, z13.b, z15.b\n" "8:" // K oddments "cbz x21, 10f\n" "9:" // K oddments: Loop - ".inst 0xa040077e // ld1b { z30.b-z31.b }, pn9.b/Z, [x27]\n" + ".inst 0xa1400773 // ld1b { z19.b, z27.b }, pn9.b/Z, [x27]\n" "subs x21, x21, #0x1\n" "addvl x27, x27, #2\n" ".inst 0xa04006f0 // ld1b { z16.b-z17.b }, pn9.b/Z, [x23]\n" "addvl x23, x23, #2\n" - ".inst 0xa1b027c0 // umopa za0.s, p1/M, p1/M, z30.b, z16.b\n" - ".inst 0xa1b127c1 // umopa za1.s, p1/M, p1/M, z30.b, z17.b\n" - ".inst 0xa1b027e2 // umopa za2.s, p1/M, p1/M, z31.b, z16.b\n" - ".inst 0xa1b127e3 // umopa za3.s, p1/M, p1/M, z31.b, z17.b\n" + ".inst 0xa1b02660 // umopa za0.s, p1/M, p1/M, z19.b, z16.b\n" + ".inst 0xa1b12661 // umopa za1.s, p1/M, p1/M, z19.b, z17.b\n" + ".inst 0xa1b02762 // umopa za2.s, p1/M, p1/M, z27.b, z16.b\n" + ".inst 0xa1b12763 // umopa za3.s, p1/M, p1/M, z27.b, z17.b\n" "bgt 9b\n" "10:" // K oddments: End ".inst 0xa040476e // ld1w { z14.s-z15.s }, pn9.b/Z, [x27]\n" @@ -234,25 +233,25 @@ void sme2_interleaved_nomerge_u8q_mopa_2VLx2VL(const uint8_t *const A, const uin "mov x12, #0x0\n" "cntw x20\n" "11:" // Store to partial result buffer: Store and refill: Loop - ".inst 0xa040c5fc // ld1w { z28.s-z31.s }, pn9.b/Z, [x15]\n" - ".inst 0xc0860408 // mova { z8.s-z11.s }, za0h.s[x12]\n" - ".inst 0xc0840780 // mova za0h.s[x12], { z28.s-z31.s }\n" - ".inst 0xc0860434 // mova { z20.s-z23.s }, za1h.s[x12]\n" - ".inst 0xa041c5f8 // ld1w { z24.s-z27.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n" - ".inst 0xc0840701 // mova za1h.s[x12], { z24.s-z27.s }\n" - ".inst 0xc086045c // mova { z28.s-z31.s }, za2h.s[x12]\n" - ".inst 0xc0860470 // mova { z16.s-z19.s }, za3h.s[x12]\n" - ".inst 0xa042c5f8 // ld1w { z24.s-z27.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n" - ".inst 0xc0840702 // mova za2h.s[x12], { z24.s-z27.s }\n" - ".inst 0xa043c5ec // ld1w { z12.s-z15.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840583 // mova za3h.s[x12], { z12.s-z15.s }\n" + ".inst 0xa040c5f0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x15]\n" + ".inst 0xc0860400 // mova { z0.s-z3.s }, za0h.s[x12]\n" + ".inst 0xc0840600 // mova za0h.s[x12], { z16.s-z19.s }\n" + ".inst 0xc0860428 // mova { z8.s-z11.s }, za1h.s[x12]\n" + ".inst 0xa041c5fc // ld1w { z28.s-z31.s }, pn9.b/Z, [x15, #0x4, MUL VL]\n" + ".inst 0xc0840781 // mova za1h.s[x12], { z28.s-z31.s }\n" + ".inst 0xc0860444 // mova { z4.s-z7.s }, za2h.s[x12]\n" + ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" + ".inst 0xa042c5f0 // ld1w { z16.s-z19.s }, pn9.b/Z, [x15, #0x8, MUL VL]\n" + ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa043c5fc // ld1w { z28.s-z31.s }, pn9.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840783 // mova za3h.s[x12], { z28.s-z31.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" - ".inst 0xa060c5c8 // st1w { z8.s-z11.s }, pn9.b, [x14]\n" + ".inst 0xa060c5c0 // st1w { z0.s-z3.s }, pn9.b, [x14]\n" "addvl x15, x15, #16\n" - ".inst 0xa061c5d4 // st1w { z20.s-z23.s }, pn9.b, [x14, #0x4, MUL VL]\n" - ".inst 0xa062c5dc // st1w { z28.s-z31.s }, pn9.b, [x14, #0x8, MUL VL]\n" - ".inst 0xa063c5d0 // st1w { z16.s-z19.s }, pn9.b, [x14, #0xc, MUL VL]\n" + ".inst 0xa061c5c8 // st1w { z8.s-z11.s }, pn9.b, [x14, #0x4, MUL VL]\n" + ".inst 0xa062c5c4 // st1w { z4.s-z7.s }, pn9.b, [x14, #0x8, MUL VL]\n" + ".inst 0xa063c5cc // st1w { z12.s-z15.s }, pn9.b, [x14, #0xc, MUL VL]\n" "addvl x14, x14, #16\n" "blt 11b\n" "b 24f\n" @@ -260,16 +259,16 @@ void sme2_interleaved_nomerge_u8q_mopa_2VLx2VL(const uint8_t *const A, const uin "mov x12, #0x0\n" "cntw x20\n" "13:" // Store to partial result buffer: Store only: Loop - ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" - ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" - ".inst 0xa060c5d0 // st1w { z16.s-z19.s }, pn9.b, [x14]\n" - ".inst 0xc0860448 // mova { z8.s-z11.s }, za2h.s[x12]\n" - ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" - ".inst 0xa061c5c4 // st1w { z4.s-z7.s }, pn9.b, [x14, #0x4, MUL VL]\n" + ".inst 0xc086040c // mova { z12.s-z15.s }, za0h.s[x12]\n" + ".inst 0xc0860428 // mova { z8.s-z11.s }, za1h.s[x12]\n" + ".inst 0xa060c5cc // st1w { z12.s-z15.s }, pn9.b, [x14]\n" + ".inst 0xc0860450 // mova { z16.s-z19.s }, za2h.s[x12]\n" + ".inst 0xc0860474 // mova { z20.s-z23.s }, za3h.s[x12]\n" + ".inst 0xa061c5c8 // st1w { z8.s-z11.s }, pn9.b, [x14, #0x4, MUL VL]\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" - ".inst 0xa062c5c8 // st1w { z8.s-z11.s }, pn9.b, [x14, #0x8, MUL VL]\n" - ".inst 0xa063c5cc // st1w { z12.s-z15.s }, pn9.b, [x14, #0xc, MUL VL]\n" + ".inst 0xa062c5d0 // st1w { z16.s-z19.s }, pn9.b, [x14, #0x8, MUL VL]\n" + ".inst 0xa063c5d4 // st1w { z20.s-z23.s }, pn9.b, [x14, #0xc, MUL VL]\n" "addvl x14, x14, #16\n" "blt 13b\n" "b 24f\n" @@ -277,13 +276,13 @@ void sme2_interleaved_nomerge_u8q_mopa_2VLx2VL(const uint8_t *const A, const uin "ldr x26, [%x[args], %[offsetof_C]]\n" "add x26, x26, x10\n" // C += n "sub x25, x13, x11\n" - "ld1rw { z2.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z0.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" "ldr x24, [%x[args], %[offsetof_ldcb]]\n" "madd x26, x11, x24, x26\n" // C += m * ldc - "ld1rw { z3.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" - "ld1rw { z0.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" - "ld1rw { z1.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" - "ld1rw { z11.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_c_offset]]\n" + "ld1rw { z1.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z2.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z3.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z14.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_c_offset]]\n" "ld1rw { z25.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_minval]]\n" "ld1rw { z24.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_maxval]]\n" "tbz x16, #2, 15f\n" @@ -291,10 +290,10 @@ void sme2_interleaved_nomerge_u8q_mopa_2VLx2VL(const uint8_t *const A, const uin "add x21, x21, x10\n" "ldr x20, [%x[rq], %[offsetof_Requantize32_per_channel_muls]]\n" "add x20, x20, x21, LSL #2\n" - ".inst 0xa0404282 // ld1w { z2.s-z3.s }, p8/Z, [x20]\n" + ".inst 0xa0404280 // ld1w { z0.s-z1.s }, p8/Z, [x20]\n" "ldr x20, [%x[rq], %[offsetof_Requantize32_per_channel_right_shifts]]\n" "add x20, x20, x21, LSL #2\n" - ".inst 0xa0404280 // ld1w { z0.s-z1.s }, p8/Z, [x20]\n" + ".inst 0xa0404282 // ld1w { z2.s-z3.s }, p8/Z, [x20]\n" "15:" // Store to output array: Load per-channel parameters: End "cntw x23\n" "whilelt p0.h, x10, x9\n" @@ -305,26 +304,26 @@ void sme2_interleaved_nomerge_u8q_mopa_2VLx2VL(const uint8_t *const A, const uin "and x20, x22, #0x3\n" "cbz x21, 17f\n" "16:" // Store to output array: Accumulator row 0 loop - ".inst 0xc086040c // mova { z12.s-z15.s }, za0h.s[x12]\n" - ".inst 0xc086043c // mova { z28.s-z31.s }, za1h.s[x12]\n" - ".inst 0xc1a2ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z2.s\n" - ".inst 0xc1a3ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z3.s\n" + ".inst 0xc0860404 // mova { z4.s-z7.s }, za0h.s[x12]\n" + ".inst 0xc0860428 // mova { z8.s-z11.s }, za1h.s[x12]\n" + ".inst 0xc1a0ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z0.s\n" + ".inst 0xc1a1ac08 // sqdmulh { z8.s-z11.s }, { z8.s-z11.s }, z1.s\n" "add x12, x12, #0x4\n" "cmp x12, x21, LSL #2\n" - ".inst 0xc1a0aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z0.s\n" - ".inst 0xc1a1aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z1.s\n" - ".inst 0xc1abab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z11.s\n" - ".inst 0xc1abab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z11.s\n" - ".inst 0xc1b8cf2c // sclamp { z12.s-z15.s }, z25.s, z24.s\n" - ".inst 0xc1b8cf3c // sclamp { z28.s-z31.s }, z25.s, z24.s\n" - "uzp1 z16.h, z12.h, z28.h\n" + ".inst 0xc1a2aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z2.s\n" + ".inst 0xc1a3aa28 // srshl { z8.s-z11.s }, { z8.s-z11.s }, z3.s\n" + ".inst 0xc1aeab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z14.s\n" + ".inst 0xc1aeab08 // add { z8.s-z11.s }, { z8.s-z11.s }, z14.s\n" + ".inst 0xc1b8cf24 // sclamp { z4.s-z7.s }, z25.s, z24.s\n" + ".inst 0xc1b8cf28 // sclamp { z8.s-z11.s }, z25.s, z24.s\n" + "uzp1 z16.h, z4.h, z8.h\n" "st1b { z16.h }, p0, [x26]\n" "add x26, x26, x24\n" - "uzp1 z16.h, z13.h, z29.h\n" - "uzp1 z17.h, z14.h, z30.h\n" + "uzp1 z16.h, z5.h, z9.h\n" + "uzp1 z17.h, z6.h, z10.h\n" "st1b { z16.h }, p0, [x26]\n" "add x26, x26, x24\n" - "uzp1 z16.h, z15.h, z31.h\n" + "uzp1 z16.h, z7.h, z11.h\n" "st1b { z17.h }, p0, [x26]\n" "add x26, x26, x24\n" "st1b { z16.h }, p0, [x26]\n" @@ -332,27 +331,27 @@ void sme2_interleaved_nomerge_u8q_mopa_2VLx2VL(const uint8_t *const A, const uin "blt 16b\n" "17:" // Store to output array: Accumulator row 0 oddments "cbz x20, 18f\n" - ".inst 0xc086041c // mova { z28.s-z31.s }, za0h.s[x12]\n" - ".inst 0xc086042c // mova { z12.s-z15.s }, za1h.s[x12]\n" - ".inst 0xc1a2ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z2.s\n" - ".inst 0xc1a3ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z3.s\n" + ".inst 0xc0860408 // mova { z8.s-z11.s }, za0h.s[x12]\n" + ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" + ".inst 0xc1a0ac08 // sqdmulh { z8.s-z11.s }, { z8.s-z11.s }, z0.s\n" + ".inst 0xc1a1ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" "subs x20, x20, #0x1\n" - ".inst 0xc1a0aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z0.s\n" - ".inst 0xc1a1aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n" - ".inst 0xc1abab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z11.s\n" - ".inst 0xc1abab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z11.s\n" - ".inst 0xc1b8cf3c // sclamp { z28.s-z31.s }, z25.s, z24.s\n" - ".inst 0xc1b8cf2c // sclamp { z12.s-z15.s }, z25.s, z24.s\n" - "uzp1 z16.h, z28.h, z12.h\n" + ".inst 0xc1a2aa28 // srshl { z8.s-z11.s }, { z8.s-z11.s }, z2.s\n" + ".inst 0xc1a3aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z3.s\n" + ".inst 0xc1aeab08 // add { z8.s-z11.s }, { z8.s-z11.s }, z14.s\n" + ".inst 0xc1aeab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z14.s\n" + ".inst 0xc1b8cf28 // sclamp { z8.s-z11.s }, z25.s, z24.s\n" + ".inst 0xc1b8cf24 // sclamp { z4.s-z7.s }, z25.s, z24.s\n" + "uzp1 z16.h, z8.h, z4.h\n" "st1b { z16.h }, p0, [x26]\n" "add x26, x26, x24\n" "beq 18f\n" "subs x20, x20, #0x1\n" - "uzp1 z16.h, z29.h, z13.h\n" + "uzp1 z16.h, z9.h, z5.h\n" "st1b { z16.h }, p0, [x26]\n" "add x26, x26, x24\n" "beq 18f\n" - "uzp1 z16.h, z30.h, z14.h\n" + "uzp1 z16.h, z10.h, z6.h\n" "st1b { z16.h }, p0, [x26]\n" "add x26, x26, x24\n" "18:" // Store to output array: Accumulator row 0 oddments: End @@ -367,25 +366,25 @@ void sme2_interleaved_nomerge_u8q_mopa_2VLx2VL(const uint8_t *const A, const uin "cbz x21, 20f\n" "19:" // Store to output array: Accumulator row 1 loop ".inst 0xc0860444 // mova { z4.s-z7.s }, za2h.s[x12]\n" - ".inst 0xc0860470 // mova { z16.s-z19.s }, za3h.s[x12]\n" - ".inst 0xc1a2ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z2.s\n" - ".inst 0xc1a3ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z3.s\n" + ".inst 0xc0860474 // mova { z20.s-z23.s }, za3h.s[x12]\n" + ".inst 0xc1a0ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z0.s\n" + ".inst 0xc1a1ac14 // sqdmulh { z20.s-z23.s }, { z20.s-z23.s }, z1.s\n" "add x12, x12, #0x4\n" "cmp x12, x21, LSL #2\n" - ".inst 0xc1a0aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z0.s\n" - ".inst 0xc1a1aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z1.s\n" - ".inst 0xc1abab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z11.s\n" - ".inst 0xc1abab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z11.s\n" + ".inst 0xc1a2aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z2.s\n" + ".inst 0xc1a3aa34 // srshl { z20.s-z23.s }, { z20.s-z23.s }, z3.s\n" + ".inst 0xc1aeab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z14.s\n" + ".inst 0xc1aeab14 // add { z20.s-z23.s }, { z20.s-z23.s }, z14.s\n" ".inst 0xc1b8cf24 // sclamp { z4.s-z7.s }, z25.s, z24.s\n" - ".inst 0xc1b8cf30 // sclamp { z16.s-z19.s }, z25.s, z24.s\n" - "uzp1 z16.h, z4.h, z16.h\n" + ".inst 0xc1b8cf34 // sclamp { z20.s-z23.s }, z25.s, z24.s\n" + "uzp1 z16.h, z4.h, z20.h\n" "st1b { z16.h }, p0, [x26]\n" "add x26, x26, x24\n" - "uzp1 z16.h, z5.h, z17.h\n" - "uzp1 z17.h, z6.h, z18.h\n" + "uzp1 z16.h, z5.h, z21.h\n" + "uzp1 z17.h, z6.h, z22.h\n" "st1b { z16.h }, p0, [x26]\n" "add x26, x26, x24\n" - "uzp1 z16.h, z7.h, z19.h\n" + "uzp1 z16.h, z7.h, z23.h\n" "st1b { z17.h }, p0, [x26]\n" "add x26, x26, x24\n" "st1b { z16.h }, p0, [x26]\n" @@ -393,27 +392,27 @@ void sme2_interleaved_nomerge_u8q_mopa_2VLx2VL(const uint8_t *const A, const uin "blt 19b\n" "20:" // Store to output array: Accumulator row 1 oddments "cbz x20, 21f\n" - ".inst 0xc0860454 // mova { z20.s-z23.s }, za2h.s[x12]\n" + ".inst 0xc0860444 // mova { z4.s-z7.s }, za2h.s[x12]\n" ".inst 0xc0860470 // mova { z16.s-z19.s }, za3h.s[x12]\n" - ".inst 0xc1a2ac14 // sqdmulh { z20.s-z23.s }, { z20.s-z23.s }, z2.s\n" - ".inst 0xc1a3ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z3.s\n" + ".inst 0xc1a0ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z0.s\n" + ".inst 0xc1a1ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z1.s\n" "subs x20, x20, #0x1\n" - ".inst 0xc1a0aa34 // srshl { z20.s-z23.s }, { z20.s-z23.s }, z0.s\n" - ".inst 0xc1a1aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z1.s\n" - ".inst 0xc1abab14 // add { z20.s-z23.s }, { z20.s-z23.s }, z11.s\n" - ".inst 0xc1abab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z11.s\n" - ".inst 0xc1b8cf34 // sclamp { z20.s-z23.s }, z25.s, z24.s\n" + ".inst 0xc1a2aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z2.s\n" + ".inst 0xc1a3aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z3.s\n" + ".inst 0xc1aeab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z14.s\n" + ".inst 0xc1aeab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z14.s\n" + ".inst 0xc1b8cf24 // sclamp { z4.s-z7.s }, z25.s, z24.s\n" ".inst 0xc1b8cf30 // sclamp { z16.s-z19.s }, z25.s, z24.s\n" - "uzp1 z16.h, z20.h, z16.h\n" + "uzp1 z16.h, z4.h, z16.h\n" "st1b { z16.h }, p0, [x26]\n" "add x26, x26, x24\n" "beq 21f\n" "subs x20, x20, #0x1\n" - "uzp1 z16.h, z21.h, z17.h\n" + "uzp1 z16.h, z5.h, z17.h\n" "st1b { z16.h }, p0, [x26]\n" "add x26, x26, x24\n" "beq 21f\n" - "uzp1 z16.h, z22.h, z18.h\n" + "uzp1 z16.h, z6.h, z18.h\n" "st1b { z16.h }, p0, [x26]\n" "21:" // Store to output array: Accumulator row 1 oddments: End "22:" // Store to output array: End @@ -452,4 +451,3 @@ void sme2_interleaved_nomerge_u8q_mopa_2VLx2VL(const uint8_t *const A, const uin } // namespace arm_gemm #endif // ARM_COMPUTE_ENABLE_SME2 -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_4VLx1VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_4VLx1VL.hpp index 2e61cf49a8847fea29a02e9e8ceefc1b9564d263..04d19324c5649de49b2430ed2ed388e9cf6d360c 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_4VLx1VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_4VLx1VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,7 +23,7 @@ */ #pragma once -#ifdef __aarch64__ +#ifdef ARM_COMPUTE_ENABLE_SME2 #include #include "../std_transforms_sme.hpp" @@ -83,12 +83,11 @@ public: StdTransformsSME transforms = {}; - cls_sme2_interleaved_nomerge_u8q_mopa_4VLx1VL(const CPUInfo *ci) + cls_sme2_interleaved_nomerge_u8q_mopa_4VLx1VL(const CPUInfo *) { - ARM_COMPUTE_UNUSED(ci); } }; } // namespace arm_gemm -#endif // __aarch64__ +#endif // ARM_COMPUTE_ENABLE_SME2 diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_4VLx1VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_4VLx1VL/generic.cpp index 8f8886b8762cd5a17f07fddd44b434b029baf339..0f3346e65ee2d5b0021a87ab523aba537ae2aff3 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_4VLx1VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sme2_interleaved_nomerge_u8q_mopa_4VLx1VL/generic.cpp @@ -21,7 +21,6 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifdef __ARM_FEATURE_SVE #ifdef ARM_COMPUTE_ENABLE_SME2 #include "arm_gemm.hpp" @@ -100,14 +99,14 @@ void sme2_interleaved_nomerge_u8q_mopa_4VLx1VL(const uint8_t *const A, const uin "mov x12, #0x0\n" "cntw x20\n" "1:" // Initial accumulator load from buffer: Loop - ".inst 0xa040c1fc // ld1w { z28.s-z31.s }, pn8.b/Z, [x15]\n" - ".inst 0xc0840780 // mova za0h.s[x12], { z28.s-z31.s }\n" - ".inst 0xa041c1ec // ld1w { z12.s-z15.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" - ".inst 0xc0840581 // mova za1h.s[x12], { z12.s-z15.s }\n" - ".inst 0xa042c1f4 // ld1w { z20.s-z23.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" - ".inst 0xc0840682 // mova za2h.s[x12], { z20.s-z23.s }\n" - ".inst 0xa043c1f8 // ld1w { z24.s-z27.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840703 // mova za3h.s[x12], { z24.s-z27.s }\n" + ".inst 0xa040c1f0 // ld1w { z16.s-z19.s }, pn8.b/Z, [x15]\n" + ".inst 0xc0840600 // mova za0h.s[x12], { z16.s-z19.s }\n" + ".inst 0xa041c1f8 // ld1w { z24.s-z27.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" + ".inst 0xc0840701 // mova za1h.s[x12], { z24.s-z27.s }\n" + ".inst 0xa042c1fc // ld1w { z28.s-z31.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" + ".inst 0xc0840782 // mova za2h.s[x12], { z28.s-z31.s }\n" + ".inst 0xa043c1f4 // ld1w { z20.s-z23.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840683 // mova za3h.s[x12], { z20.s-z23.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x15, x15, #16\n" @@ -125,11 +124,11 @@ void sme2_interleaved_nomerge_u8q_mopa_4VLx1VL(const uint8_t *const A, const uin "ldr x20, [%x[args], %[offsetof_bias]]\n" ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "cbz x20, 5f\n" - "ldnt1w { z15.s }, p0/Z, [x20, x10, LSL #2]\n" - ".inst 0xc09025e0 // addha za0.s, p1/M, p1/M, z15.s\n" - ".inst 0xc09025e1 // addha za1.s, p1/M, p1/M, z15.s\n" - ".inst 0xc09025e2 // addha za2.s, p1/M, p1/M, z15.s\n" - ".inst 0xc09025e3 // addha za3.s, p1/M, p1/M, z15.s\n" + "ldnt1w { z8.s }, p0/Z, [x20, x10, LSL #2]\n" + ".inst 0xc0902500 // addha za0.s, p1/M, p1/M, z8.s\n" + ".inst 0xc0902501 // addha za1.s, p1/M, p1/M, z8.s\n" + ".inst 0xc0902502 // addha za2.s, p1/M, p1/M, z8.s\n" + ".inst 0xc0902503 // addha za3.s, p1/M, p1/M, z8.s\n" "4:" // Prepare accumulators: Test for last block "mov x20, x10\n" "mov x21, x11\n" @@ -152,107 +151,107 @@ void sme2_interleaved_nomerge_u8q_mopa_4VLx1VL(const uint8_t *const A, const uin "madd x23, x10, x20, x23\n" // bptr = B + n * kstride_bytes "cbz x22, 8f\n" "subs x22, x22, #0x1\n" - ".inst 0xa1408372 // ld1b { z18.b, z22.b, z26.b, z30.b }, pn8.b/Z, [x27]\n" - "ldnt1b { z0.b }, p1/Z, [x23]\n" - ".inst 0xa1418373 // ld1b { z19.b, z23.b, z27.b, z31.b }, pn8.b/Z, [x27, #0x4, MUL VL]\n" - "ldnt1b { z9.b }, p1/Z, [x23, #1, MUL VL]\n" - ".inst 0xa1428370 // ld1b { z16.b, z20.b, z24.b, z28.b }, pn8.b/Z, [x27, #0x8, MUL VL]\n" - "ldnt1b { z21.b }, p1/Z, [x23, #2, MUL VL]\n" - ".inst 0xa1438362 // ld1b { z2.b, z6.b, z10.b, z14.b }, pn8.b/Z, [x27, #0xc, MUL VL]\n" + ".inst 0xa0408364 // ld1b { z4.b-z7.b }, pn8.b/Z, [x27]\n" + "ldnt1b { z14.b }, p1/Z, [x23]\n" + ".inst 0xa0418374 // ld1b { z20.b-z23.b }, pn8.b/Z, [x27, #0x4, MUL VL]\n" + "ldnt1b { z31.b }, p1/Z, [x23, #1, MUL VL]\n" + ".inst 0xa0428378 // ld1b { z24.b-z27.b }, pn8.b/Z, [x27, #0x8, MUL VL]\n" + "ldnt1b { z13.b }, p1/Z, [x23, #2, MUL VL]\n" + ".inst 0xa0438368 // ld1b { z8.b-z11.b }, pn8.b/Z, [x27, #0xc, MUL VL]\n" "addvl x27, x27, #16\n" - "ldnt1b { z12.b }, p1/Z, [x23, #3, MUL VL]\n" + "ldnt1b { z29.b }, p1/Z, [x23, #3, MUL VL]\n" "addvl x23, x23, #4\n" "ble 7f\n" "6:" // K loop - ".inst 0xa1a02640 // umopa za0.s, p1/M, p1/M, z18.b, z0.b\n" + ".inst 0xa1ae2480 // umopa za0.s, p1/M, p1/M, z4.b, z14.b\n" "subs x22, x22, #0x1\n" - ".inst 0xa1a026c1 // umopa za1.s, p1/M, p1/M, z22.b, z0.b\n" - ".inst 0xa1a02742 // umopa za2.s, p1/M, p1/M, z26.b, z0.b\n" - ".inst 0xa1a027c3 // umopa za3.s, p1/M, p1/M, z30.b, z0.b\n" - ".inst 0xa1408372 // ld1b { z18.b, z22.b, z26.b, z30.b }, pn8.b/Z, [x27]\n" - ".inst 0xa1a92660 // umopa za0.s, p1/M, p1/M, z19.b, z9.b\n" - "ldnt1b { z0.b }, p1/Z, [x23]\n" - ".inst 0xa1a926e1 // umopa za1.s, p1/M, p1/M, z23.b, z9.b\n" - ".inst 0xa1a92762 // umopa za2.s, p1/M, p1/M, z27.b, z9.b\n" - ".inst 0xa1a927e3 // umopa za3.s, p1/M, p1/M, z31.b, z9.b\n" - ".inst 0xa1418373 // ld1b { z19.b, z23.b, z27.b, z31.b }, pn8.b/Z, [x27, #0x4, MUL VL]\n" - ".inst 0xa1b52600 // umopa za0.s, p1/M, p1/M, z16.b, z21.b\n" - "ldnt1b { z9.b }, p1/Z, [x23, #1, MUL VL]\n" - ".inst 0xa1b52681 // umopa za1.s, p1/M, p1/M, z20.b, z21.b\n" - ".inst 0xa1b52702 // umopa za2.s, p1/M, p1/M, z24.b, z21.b\n" - ".inst 0xa1b52783 // umopa za3.s, p1/M, p1/M, z28.b, z21.b\n" - ".inst 0xa1428370 // ld1b { z16.b, z20.b, z24.b, z28.b }, pn8.b/Z, [x27, #0x8, MUL VL]\n" - "ldnt1b { z21.b }, p1/Z, [x23, #2, MUL VL]\n" - ".inst 0xa1ac2440 // umopa za0.s, p1/M, p1/M, z2.b, z12.b\n" - ".inst 0xa1ac24c1 // umopa za1.s, p1/M, p1/M, z6.b, z12.b\n" - ".inst 0xa1ac2542 // umopa za2.s, p1/M, p1/M, z10.b, z12.b\n" - ".inst 0xa1ac25c3 // umopa za3.s, p1/M, p1/M, z14.b, z12.b\n" - ".inst 0xa1438362 // ld1b { z2.b, z6.b, z10.b, z14.b }, pn8.b/Z, [x27, #0xc, MUL VL]\n" + ".inst 0xa1ae24a1 // umopa za1.s, p1/M, p1/M, z5.b, z14.b\n" + ".inst 0xa1ae24c2 // umopa za2.s, p1/M, p1/M, z6.b, z14.b\n" + ".inst 0xa1ae24e3 // umopa za3.s, p1/M, p1/M, z7.b, z14.b\n" + ".inst 0xa0408364 // ld1b { z4.b-z7.b }, pn8.b/Z, [x27]\n" + ".inst 0xa1bf2680 // umopa za0.s, p1/M, p1/M, z20.b, z31.b\n" + "ldnt1b { z14.b }, p1/Z, [x23]\n" + ".inst 0xa1bf26a1 // umopa za1.s, p1/M, p1/M, z21.b, z31.b\n" + ".inst 0xa1bf26c2 // umopa za2.s, p1/M, p1/M, z22.b, z31.b\n" + ".inst 0xa1bf26e3 // umopa za3.s, p1/M, p1/M, z23.b, z31.b\n" + ".inst 0xa0418374 // ld1b { z20.b-z23.b }, pn8.b/Z, [x27, #0x4, MUL VL]\n" + ".inst 0xa1ad2700 // umopa za0.s, p1/M, p1/M, z24.b, z13.b\n" + "ldnt1b { z31.b }, p1/Z, [x23, #1, MUL VL]\n" + ".inst 0xa1ad2721 // umopa za1.s, p1/M, p1/M, z25.b, z13.b\n" + ".inst 0xa1ad2742 // umopa za2.s, p1/M, p1/M, z26.b, z13.b\n" + ".inst 0xa1ad2763 // umopa za3.s, p1/M, p1/M, z27.b, z13.b\n" + ".inst 0xa0428378 // ld1b { z24.b-z27.b }, pn8.b/Z, [x27, #0x8, MUL VL]\n" + "ldnt1b { z13.b }, p1/Z, [x23, #2, MUL VL]\n" + ".inst 0xa1bd2500 // umopa za0.s, p1/M, p1/M, z8.b, z29.b\n" + ".inst 0xa1bd2521 // umopa za1.s, p1/M, p1/M, z9.b, z29.b\n" + ".inst 0xa1bd2542 // umopa za2.s, p1/M, p1/M, z10.b, z29.b\n" + ".inst 0xa1bd2563 // umopa za3.s, p1/M, p1/M, z11.b, z29.b\n" + ".inst 0xa0438368 // ld1b { z8.b-z11.b }, pn8.b/Z, [x27, #0xc, MUL VL]\n" "addvl x27, x27, #16\n" - "ldnt1b { z12.b }, p1/Z, [x23, #3, MUL VL]\n" + "ldnt1b { z29.b }, p1/Z, [x23, #3, MUL VL]\n" "addvl x23, x23, #4\n" "bgt 6b\n" "7:" // K loop tail - ".inst 0xa1a02640 // umopa za0.s, p1/M, p1/M, z18.b, z0.b\n" - ".inst 0xa1a026c1 // umopa za1.s, p1/M, p1/M, z22.b, z0.b\n" - ".inst 0xa1a02742 // umopa za2.s, p1/M, p1/M, z26.b, z0.b\n" - ".inst 0xa1a027c3 // umopa za3.s, p1/M, p1/M, z30.b, z0.b\n" - ".inst 0xa1a92660 // umopa za0.s, p1/M, p1/M, z19.b, z9.b\n" - ".inst 0xa1a926e1 // umopa za1.s, p1/M, p1/M, z23.b, z9.b\n" - ".inst 0xa1a92762 // umopa za2.s, p1/M, p1/M, z27.b, z9.b\n" - ".inst 0xa1a927e3 // umopa za3.s, p1/M, p1/M, z31.b, z9.b\n" - ".inst 0xa1b52600 // umopa za0.s, p1/M, p1/M, z16.b, z21.b\n" - ".inst 0xa1b52681 // umopa za1.s, p1/M, p1/M, z20.b, z21.b\n" - ".inst 0xa1b52702 // umopa za2.s, p1/M, p1/M, z24.b, z21.b\n" - ".inst 0xa1b52783 // umopa za3.s, p1/M, p1/M, z28.b, z21.b\n" - ".inst 0xa1ac2440 // umopa za0.s, p1/M, p1/M, z2.b, z12.b\n" - ".inst 0xa1ac24c1 // umopa za1.s, p1/M, p1/M, z6.b, z12.b\n" - ".inst 0xa1ac2542 // umopa za2.s, p1/M, p1/M, z10.b, z12.b\n" - ".inst 0xa1ac25c3 // umopa za3.s, p1/M, p1/M, z14.b, z12.b\n" + ".inst 0xa1ae2480 // umopa za0.s, p1/M, p1/M, z4.b, z14.b\n" + ".inst 0xa1ae24a1 // umopa za1.s, p1/M, p1/M, z5.b, z14.b\n" + ".inst 0xa1ae24c2 // umopa za2.s, p1/M, p1/M, z6.b, z14.b\n" + ".inst 0xa1ae24e3 // umopa za3.s, p1/M, p1/M, z7.b, z14.b\n" + ".inst 0xa1bf2680 // umopa za0.s, p1/M, p1/M, z20.b, z31.b\n" + ".inst 0xa1bf26a1 // umopa za1.s, p1/M, p1/M, z21.b, z31.b\n" + ".inst 0xa1bf26c2 // umopa za2.s, p1/M, p1/M, z22.b, z31.b\n" + ".inst 0xa1bf26e3 // umopa za3.s, p1/M, p1/M, z23.b, z31.b\n" + ".inst 0xa1ad2700 // umopa za0.s, p1/M, p1/M, z24.b, z13.b\n" + ".inst 0xa1ad2721 // umopa za1.s, p1/M, p1/M, z25.b, z13.b\n" + ".inst 0xa1ad2742 // umopa za2.s, p1/M, p1/M, z26.b, z13.b\n" + ".inst 0xa1ad2763 // umopa za3.s, p1/M, p1/M, z27.b, z13.b\n" + ".inst 0xa1bd2500 // umopa za0.s, p1/M, p1/M, z8.b, z29.b\n" + ".inst 0xa1bd2521 // umopa za1.s, p1/M, p1/M, z9.b, z29.b\n" + ".inst 0xa1bd2542 // umopa za2.s, p1/M, p1/M, z10.b, z29.b\n" + ".inst 0xa1bd2563 // umopa za3.s, p1/M, p1/M, z11.b, z29.b\n" "8:" // K oddments "cbz x21, 10f\n" "9:" // K oddments: Loop ".inst 0xa1408372 // ld1b { z18.b, z22.b, z26.b, z30.b }, pn8.b/Z, [x27]\n" "subs x21, x21, #0x1\n" "addvl x27, x27, #4\n" - "ld1b { z0.b }, p1/Z, [x23]\n" + "ld1b { z15.b }, p1/Z, [x23]\n" "addvl x23, x23, #1\n" - ".inst 0xa1a02640 // umopa za0.s, p1/M, p1/M, z18.b, z0.b\n" - ".inst 0xa1a026c1 // umopa za1.s, p1/M, p1/M, z22.b, z0.b\n" - ".inst 0xa1a02742 // umopa za2.s, p1/M, p1/M, z26.b, z0.b\n" - ".inst 0xa1a027c3 // umopa za3.s, p1/M, p1/M, z30.b, z0.b\n" + ".inst 0xa1af2640 // umopa za0.s, p1/M, p1/M, z18.b, z15.b\n" + ".inst 0xa1af26c1 // umopa za1.s, p1/M, p1/M, z22.b, z15.b\n" + ".inst 0xa1af2742 // umopa za2.s, p1/M, p1/M, z26.b, z15.b\n" + ".inst 0xa1af27c3 // umopa za3.s, p1/M, p1/M, z30.b, z15.b\n" "bgt 9b\n" "10:" // K oddments: End - ".inst 0xa040c360 // ld1w { z0.s-z3.s }, pn8.b/Z, [x27]\n" + ".inst 0xa140c363 // ld1w { z3.s, z7.s, z11.s, z15.s }, pn8.b/Z, [x27]\n" "addvl x27, x27, #4\n" - ".inst 0xc0912400 // addva za0.s, p1/M, p1/M, z0.s\n" - ".inst 0xc0912421 // addva za1.s, p1/M, p1/M, z1.s\n" - ".inst 0xc0912442 // addva za2.s, p1/M, p1/M, z2.s\n" - ".inst 0xc0912463 // addva za3.s, p1/M, p1/M, z3.s\n" + ".inst 0xc0912460 // addva za0.s, p1/M, p1/M, z3.s\n" + ".inst 0xc09124e1 // addva za1.s, p1/M, p1/M, z7.s\n" + ".inst 0xc0912562 // addva za2.s, p1/M, p1/M, z11.s\n" + ".inst 0xc09125e3 // addva za3.s, p1/M, p1/M, z15.s\n" "tbz x16, #1, 14f\n" "tbz x16, #0, 12f\n" "mov x12, #0x0\n" "cntw x20\n" "11:" // Store to partial result buffer: Store and refill: Loop - ".inst 0xa040c1f4 // ld1w { z20.s-z23.s }, pn8.b/Z, [x15]\n" - ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" - ".inst 0xc0840680 // mova za0h.s[x12], { z20.s-z23.s }\n" - ".inst 0xc0860428 // mova { z8.s-z11.s }, za1h.s[x12]\n" - ".inst 0xa041c1e4 // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" - ".inst 0xc0840481 // mova za1h.s[x12], { z4.s-z7.s }\n" - ".inst 0xc086044c // mova { z12.s-z15.s }, za2h.s[x12]\n" - ".inst 0xc086047c // mova { z28.s-z31.s }, za3h.s[x12]\n" + ".inst 0xa040c1ec // ld1w { z12.s-z15.s }, pn8.b/Z, [x15]\n" + ".inst 0xc0860400 // mova { z0.s-z3.s }, za0h.s[x12]\n" + ".inst 0xc0840580 // mova za0h.s[x12], { z12.s-z15.s }\n" + ".inst 0xc086043c // mova { z28.s-z31.s }, za1h.s[x12]\n" + ".inst 0xa041c1f0 // ld1w { z16.s-z19.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" + ".inst 0xc0840601 // mova za1h.s[x12], { z16.s-z19.s }\n" + ".inst 0xc0860454 // mova { z20.s-z23.s }, za2h.s[x12]\n" + ".inst 0xc086046c // mova { z12.s-z15.s }, za3h.s[x12]\n" ".inst 0xa042c1e4 // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" ".inst 0xc0840482 // mova za2h.s[x12], { z4.s-z7.s }\n" - ".inst 0xa043c1f4 // ld1w { z20.s-z23.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840683 // mova za3h.s[x12], { z20.s-z23.s }\n" + ".inst 0xa043c1e8 // ld1w { z8.s-z11.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840503 // mova za3h.s[x12], { z8.s-z11.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" - ".inst 0xa060c1d0 // st1w { z16.s-z19.s }, pn8.b, [x14]\n" + ".inst 0xa060c1c0 // st1w { z0.s-z3.s }, pn8.b, [x14]\n" "addvl x15, x15, #16\n" - ".inst 0xa061c1c8 // st1w { z8.s-z11.s }, pn8.b, [x14, #0x4, MUL VL]\n" - ".inst 0xa062c1cc // st1w { z12.s-z15.s }, pn8.b, [x14, #0x8, MUL VL]\n" - ".inst 0xa063c1dc // st1w { z28.s-z31.s }, pn8.b, [x14, #0xc, MUL VL]\n" + ".inst 0xa061c1dc // st1w { z28.s-z31.s }, pn8.b, [x14, #0x4, MUL VL]\n" + ".inst 0xa062c1d4 // st1w { z20.s-z23.s }, pn8.b, [x14, #0x8, MUL VL]\n" + ".inst 0xa063c1cc // st1w { z12.s-z15.s }, pn8.b, [x14, #0xc, MUL VL]\n" "addvl x14, x14, #16\n" "blt 11b\n" "b 30f\n" @@ -260,16 +259,16 @@ void sme2_interleaved_nomerge_u8q_mopa_4VLx1VL(const uint8_t *const A, const uin "mov x12, #0x0\n" "cntw x20\n" "13:" // Store to partial result buffer: Store only: Loop - ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" + ".inst 0xc0860400 // mova { z0.s-z3.s }, za0h.s[x12]\n" ".inst 0xc086042c // mova { z12.s-z15.s }, za1h.s[x12]\n" - ".inst 0xa060c1d0 // st1w { z16.s-z19.s }, pn8.b, [x14]\n" - ".inst 0xc0860454 // mova { z20.s-z23.s }, za2h.s[x12]\n" - ".inst 0xc0860478 // mova { z24.s-z27.s }, za3h.s[x12]\n" + ".inst 0xa060c1c0 // st1w { z0.s-z3.s }, pn8.b, [x14]\n" + ".inst 0xc0860444 // mova { z4.s-z7.s }, za2h.s[x12]\n" + ".inst 0xc0860460 // mova { z0.s-z3.s }, za3h.s[x12]\n" ".inst 0xa061c1cc // st1w { z12.s-z15.s }, pn8.b, [x14, #0x4, MUL VL]\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" - ".inst 0xa062c1d4 // st1w { z20.s-z23.s }, pn8.b, [x14, #0x8, MUL VL]\n" - ".inst 0xa063c1d8 // st1w { z24.s-z27.s }, pn8.b, [x14, #0xc, MUL VL]\n" + ".inst 0xa062c1c4 // st1w { z4.s-z7.s }, pn8.b, [x14, #0x8, MUL VL]\n" + ".inst 0xa063c1c0 // st1w { z0.s-z3.s }, pn8.b, [x14, #0xc, MUL VL]\n" "addvl x14, x14, #16\n" "blt 13b\n" "b 30f\n" @@ -277,22 +276,22 @@ void sme2_interleaved_nomerge_u8q_mopa_4VLx1VL(const uint8_t *const A, const uin "ldr x26, [%x[args], %[offsetof_C]]\n" "add x26, x26, x10\n" // C += n "sub x25, x13, x11\n" - "ld1rw { z8.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" + "ld1rw { z2.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_mul]]\n" "ldr x24, [%x[args], %[offsetof_ldcb]]\n" "madd x26, x11, x24, x26\n" // C += m * ldc - "ld1rw { z7.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" - "ld1rw { z6.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_c_offset]]\n" - "ld1rw { z5.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_minval]]\n" - "ld1rw { z4.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_maxval]]\n" + "ld1rw { z1.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_per_layer_right_shift]]\n" + "ld1rw { z0.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_c_offset]]\n" + "ld1rw { z21.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_minval]]\n" + "ld1rw { z20.s }, p1/Z, [%x[rq], %[offsetof_Requantize32_maxval]]\n" "tbz x16, #2, 15f\n" "ldr w21, [%x[args], %[offsetof_n_0]]\n" "add x21, x21, x10\n" "ldr x20, [%x[rq], %[offsetof_Requantize32_per_channel_muls]]\n" "add x20, x20, x21, LSL #2\n" - "ld1w { z8.s }, p0/Z, [x20]\n" + "ld1w { z2.s }, p0/Z, [x20]\n" "ldr x20, [%x[rq], %[offsetof_Requantize32_per_channel_right_shifts]]\n" "add x20, x20, x21, LSL #2\n" - "ld1w { z7.s }, p0/Z, [x20]\n" + "ld1w { z1.s }, p0/Z, [x20]\n" "15:" // Store to output array: Load per-channel parameters: End "cntw x23\n" "whilelt p0.s, x10, x9\n" @@ -303,30 +302,30 @@ void sme2_interleaved_nomerge_u8q_mopa_4VLx1VL(const uint8_t *const A, const uin "and x20, x22, #0x3\n" "cbz x21, 17f\n" "16:" // Store to output array: Accumulator row 0 loop - ".inst 0xc086040c // mova { z12.s-z15.s }, za0h.s[x12]\n" - ".inst 0xc1a8ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z8.s\n" + ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" + ".inst 0xc1a2ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z2.s\n" "add x12, x12, #0x4\n" - ".inst 0xc1a7aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z7.s\n" + ".inst 0xc1a1aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z1.s\n" "cmp x12, x21, LSL #2\n" - ".inst 0xc1a6ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z6.s\n" - ".inst 0xc1a4ccac // sclamp { z12.s-z15.s }, z5.s, z4.s\n" - "st1b { z12.s }, p0, [x26]\n" + ".inst 0xc1a0ab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z0.s\n" + ".inst 0xc1b4ceb0 // sclamp { z16.s-z19.s }, z21.s, z20.s\n" + "st1b { z16.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1b { z13.s }, p0, [x26]\n" + "st1b { z17.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1b { z14.s }, p0, [x26]\n" + "st1b { z18.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1b { z15.s }, p0, [x26]\n" + "st1b { z19.s }, p0, [x26]\n" "add x26, x26, x24\n" "blt 16b\n" "17:" // Store to output array: Accumulator row 0 oddments "cbz x20, 18f\n" ".inst 0xc0860410 // mova { z16.s-z19.s }, za0h.s[x12]\n" - ".inst 0xc1a8ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z8.s\n" + ".inst 0xc1a2ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z2.s\n" "subs x20, x20, #0x1\n" - ".inst 0xc1a7aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z7.s\n" - ".inst 0xc1a6ab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n" - ".inst 0xc1a4ccb0 // sclamp { z16.s-z19.s }, z5.s, z4.s\n" + ".inst 0xc1a1aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z1.s\n" + ".inst 0xc1a0ab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z0.s\n" + ".inst 0xc1b4ceb0 // sclamp { z16.s-z19.s }, z21.s, z20.s\n" "st1b { z16.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 18f\n" @@ -347,38 +346,38 @@ void sme2_interleaved_nomerge_u8q_mopa_4VLx1VL(const uint8_t *const A, const uin "and x20, x22, #0x3\n" "cbz x21, 20f\n" "19:" // Store to output array: Accumulator row 1 loop - ".inst 0xc0860430 // mova { z16.s-z19.s }, za1h.s[x12]\n" - ".inst 0xc1a8ac10 // sqdmulh { z16.s-z19.s }, { z16.s-z19.s }, z8.s\n" + ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" + ".inst 0xc1a2ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z2.s\n" "add x12, x12, #0x4\n" - ".inst 0xc1a7aa30 // srshl { z16.s-z19.s }, { z16.s-z19.s }, z7.s\n" + ".inst 0xc1a1aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" "cmp x12, x21, LSL #2\n" - ".inst 0xc1a6ab10 // add { z16.s-z19.s }, { z16.s-z19.s }, z6.s\n" - ".inst 0xc1a4ccb0 // sclamp { z16.s-z19.s }, z5.s, z4.s\n" - "st1b { z16.s }, p0, [x26]\n" + ".inst 0xc1a0ab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z0.s\n" + ".inst 0xc1b4cea4 // sclamp { z4.s-z7.s }, z21.s, z20.s\n" + "st1b { z4.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1b { z17.s }, p0, [x26]\n" + "st1b { z5.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1b { z18.s }, p0, [x26]\n" + "st1b { z6.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1b { z19.s }, p0, [x26]\n" + "st1b { z7.s }, p0, [x26]\n" "add x26, x26, x24\n" "blt 19b\n" "20:" // Store to output array: Accumulator row 1 oddments "cbz x20, 21f\n" - ".inst 0xc086043c // mova { z28.s-z31.s }, za1h.s[x12]\n" - ".inst 0xc1a8ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z8.s\n" + ".inst 0xc0860424 // mova { z4.s-z7.s }, za1h.s[x12]\n" + ".inst 0xc1a2ac04 // sqdmulh { z4.s-z7.s }, { z4.s-z7.s }, z2.s\n" "subs x20, x20, #0x1\n" - ".inst 0xc1a7aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z7.s\n" - ".inst 0xc1a6ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z6.s\n" - ".inst 0xc1a4ccbc // sclamp { z28.s-z31.s }, z5.s, z4.s\n" - "st1b { z28.s }, p0, [x26]\n" + ".inst 0xc1a1aa24 // srshl { z4.s-z7.s }, { z4.s-z7.s }, z1.s\n" + ".inst 0xc1a0ab04 // add { z4.s-z7.s }, { z4.s-z7.s }, z0.s\n" + ".inst 0xc1b4cea4 // sclamp { z4.s-z7.s }, z21.s, z20.s\n" + "st1b { z4.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 21f\n" "subs x20, x20, #0x1\n" - "st1b { z29.s }, p0, [x26]\n" + "st1b { z5.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 21f\n" - "st1b { z30.s }, p0, [x26]\n" + "st1b { z6.s }, p0, [x26]\n" "add x26, x26, x24\n" "21:" // Store to output array: Accumulator row 1 oddments: End "subs x25, x25, x22\n" @@ -391,30 +390,30 @@ void sme2_interleaved_nomerge_u8q_mopa_4VLx1VL(const uint8_t *const A, const uin "and x20, x22, #0x3\n" "cbz x21, 23f\n" "22:" // Store to output array: Accumulator row 2 loop - ".inst 0xc0860458 // mova { z24.s-z27.s }, za2h.s[x12]\n" - ".inst 0xc1a8ac18 // sqdmulh { z24.s-z27.s }, { z24.s-z27.s }, z8.s\n" + ".inst 0xc0860448 // mova { z8.s-z11.s }, za2h.s[x12]\n" + ".inst 0xc1a2ac08 // sqdmulh { z8.s-z11.s }, { z8.s-z11.s }, z2.s\n" "add x12, x12, #0x4\n" - ".inst 0xc1a7aa38 // srshl { z24.s-z27.s }, { z24.s-z27.s }, z7.s\n" + ".inst 0xc1a1aa28 // srshl { z8.s-z11.s }, { z8.s-z11.s }, z1.s\n" "cmp x12, x21, LSL #2\n" - ".inst 0xc1a6ab18 // add { z24.s-z27.s }, { z24.s-z27.s }, z6.s\n" - ".inst 0xc1a4ccb8 // sclamp { z24.s-z27.s }, z5.s, z4.s\n" - "st1b { z24.s }, p0, [x26]\n" + ".inst 0xc1a0ab08 // add { z8.s-z11.s }, { z8.s-z11.s }, z0.s\n" + ".inst 0xc1b4cea8 // sclamp { z8.s-z11.s }, z21.s, z20.s\n" + "st1b { z8.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1b { z25.s }, p0, [x26]\n" + "st1b { z9.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1b { z26.s }, p0, [x26]\n" + "st1b { z10.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1b { z27.s }, p0, [x26]\n" + "st1b { z11.s }, p0, [x26]\n" "add x26, x26, x24\n" "blt 22b\n" "23:" // Store to output array: Accumulator row 2 oddments "cbz x20, 24f\n" ".inst 0xc086044c // mova { z12.s-z15.s }, za2h.s[x12]\n" - ".inst 0xc1a8ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z8.s\n" + ".inst 0xc1a2ac0c // sqdmulh { z12.s-z15.s }, { z12.s-z15.s }, z2.s\n" "subs x20, x20, #0x1\n" - ".inst 0xc1a7aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z7.s\n" - ".inst 0xc1a6ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z6.s\n" - ".inst 0xc1a4ccac // sclamp { z12.s-z15.s }, z5.s, z4.s\n" + ".inst 0xc1a1aa2c // srshl { z12.s-z15.s }, { z12.s-z15.s }, z1.s\n" + ".inst 0xc1a0ab0c // add { z12.s-z15.s }, { z12.s-z15.s }, z0.s\n" + ".inst 0xc1b4ceac // sclamp { z12.s-z15.s }, z21.s, z20.s\n" "st1b { z12.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 24f\n" @@ -435,52 +434,52 @@ void sme2_interleaved_nomerge_u8q_mopa_4VLx1VL(const uint8_t *const A, const uin "and x20, x20, #0x3\n" "cbz x21, 26f\n" "25:" // Store to output array: Accumulator row 3 loop - ".inst 0xc0860474 // mova { z20.s-z23.s }, za3h.s[x12]\n" - ".inst 0xc1a8ac14 // sqdmulh { z20.s-z23.s }, { z20.s-z23.s }, z8.s\n" + ".inst 0xc086047c // mova { z28.s-z31.s }, za3h.s[x12]\n" + ".inst 0xc1a2ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z2.s\n" "add x12, x12, #0x4\n" - ".inst 0xc1a7aa34 // srshl { z20.s-z23.s }, { z20.s-z23.s }, z7.s\n" + ".inst 0xc1a1aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z1.s\n" "cmp x12, x21, LSL #2\n" - ".inst 0xc1a6ab14 // add { z20.s-z23.s }, { z20.s-z23.s }, z6.s\n" - ".inst 0xc1a4ccb4 // sclamp { z20.s-z23.s }, z5.s, z4.s\n" - "st1b { z20.s }, p0, [x26]\n" + ".inst 0xc1a0ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z0.s\n" + ".inst 0xc1b4cebc // sclamp { z28.s-z31.s }, z21.s, z20.s\n" + "st1b { z28.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1b { z21.s }, p0, [x26]\n" + "st1b { z29.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1b { z22.s }, p0, [x26]\n" + "st1b { z30.s }, p0, [x26]\n" "add x26, x26, x24\n" - "st1b { z23.s }, p0, [x26]\n" + "st1b { z31.s }, p0, [x26]\n" "add x26, x26, x24\n" "blt 25b\n" "26:" // Store to output array: Accumulator row 3 oddments "cbz x20, 27f\n" - ".inst 0xc0860460 // mova { z0.s-z3.s }, za3h.s[x12]\n" - ".inst 0xc1a8ac00 // sqdmulh { z0.s-z3.s }, { z0.s-z3.s }, z8.s\n" + ".inst 0xc086047c // mova { z28.s-z31.s }, za3h.s[x12]\n" + ".inst 0xc1a2ac1c // sqdmulh { z28.s-z31.s }, { z28.s-z31.s }, z2.s\n" "subs x20, x20, #0x1\n" - ".inst 0xc1a7aa20 // srshl { z0.s-z3.s }, { z0.s-z3.s }, z7.s\n" - ".inst 0xc1a6ab00 // add { z0.s-z3.s }, { z0.s-z3.s }, z6.s\n" - ".inst 0xc1a4cca0 // sclamp { z0.s-z3.s }, z5.s, z4.s\n" - "st1b { z0.s }, p0, [x26]\n" + ".inst 0xc1a1aa3c // srshl { z28.s-z31.s }, { z28.s-z31.s }, z1.s\n" + ".inst 0xc1a0ab1c // add { z28.s-z31.s }, { z28.s-z31.s }, z0.s\n" + ".inst 0xc1b4cebc // sclamp { z28.s-z31.s }, z21.s, z20.s\n" + "st1b { z28.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 27f\n" "subs x20, x20, #0x1\n" - "st1b { z1.s }, p0, [x26]\n" + "st1b { z29.s }, p0, [x26]\n" "add x26, x26, x24\n" "beq 27f\n" - "st1b { z2.s }, p0, [x26]\n" + "st1b { z30.s }, p0, [x26]\n" "27:" // Store to output array: Accumulator row 3 oddments: End "28:" // Store to output array: End "tbz x16, #0, 30f\n" "mov x12, #0x0\n" "cntw x20\n" "29:" // Store to output array: Refill accumulators: Loop - ".inst 0xa040c1e4 // ld1w { z4.s-z7.s }, pn8.b/Z, [x15]\n" - ".inst 0xc0840480 // mova za0h.s[x12], { z4.s-z7.s }\n" + ".inst 0xa040c1fc // ld1w { z28.s-z31.s }, pn8.b/Z, [x15]\n" + ".inst 0xc0840780 // mova za0h.s[x12], { z28.s-z31.s }\n" ".inst 0xa041c1f0 // ld1w { z16.s-z19.s }, pn8.b/Z, [x15, #0x4, MUL VL]\n" ".inst 0xc0840601 // mova za1h.s[x12], { z16.s-z19.s }\n" ".inst 0xa042c1f0 // ld1w { z16.s-z19.s }, pn8.b/Z, [x15, #0x8, MUL VL]\n" ".inst 0xc0840602 // mova za2h.s[x12], { z16.s-z19.s }\n" - ".inst 0xa043c1e4 // ld1w { z4.s-z7.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" - ".inst 0xc0840483 // mova za3h.s[x12], { z4.s-z7.s }\n" + ".inst 0xa043c1e0 // ld1w { z0.s-z3.s }, pn8.b/Z, [x15, #0xc, MUL VL]\n" + ".inst 0xc0840403 // mova za3h.s[x12], { z0.s-z3.s }\n" "add x12, x12, #0x4\n" "cmp x12, x20\n" "addvl x15, x15, #16\n" @@ -504,4 +503,3 @@ void sme2_interleaved_nomerge_u8q_mopa_4VLx1VL(const uint8_t *const A, const uin } // namespace arm_gemm #endif // ARM_COMPUTE_ENABLE_SME2 -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_bf16fp32_mmla_6x4VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_bf16fp32_mmla_6x4VL.hpp index e07fa549f3059c65bc83dde370b2307507617ccd..1ce169d562adcfbf3befbcf7eae7ec01ea820f4e 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_bf16fp32_mmla_6x4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_bf16fp32_mmla_6x4VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,16 +10,16 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once #ifdef ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_bf16fp32_mmla_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_bf16fp32_mmla_6x4VL/generic.cpp index 13f2e488dd76281d818d48178f561ac57420a574..9136e325675ee482802fb7f0225a9484f9aaab3c 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_bf16fp32_mmla_6x4VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_bf16fp32_mmla_6x4VL/generic.cpp @@ -157,16 +157,16 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "b 6f\n" "4:" // Height 1: no bias "tbz %x[flags], #0, 5f\n" - "ld1w { z9.s }, p4/Z, [x13]\n" - "ld1w { z10.s }, p3/Z, [x13, #1, MUL VL]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z11.s }, p2/Z, [x13, #2, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x13]\n" + "ld1w { z18.s }, p3/Z, [x13, #1, MUL VL]\n" + "zip1 z8.d, z16.d, z12.d\n" + "zip2 z12.d, z16.d, z12.d\n" + "ld1w { z17.s }, p2/Z, [x13, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x13, #3, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" + "zip1 z9.d, z18.d, z13.d\n" + "zip2 z13.d, z18.d, z13.d\n" + "zip1 z10.d, z17.d, z14.d\n" + "zip2 z14.d, z17.d, z14.d\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" "b 6f\n" @@ -184,11 +184,11 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "7:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 8f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 9f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -200,43 +200,43 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "ble 11f\n" "10:" // Height 1: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z1.h }, p0/Z, [x26]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1h { z7.h }, p5/Z, [x12]\n" - "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n" - ".inst 0x6467e408 // bfmmla z8.s, z0.h, z7.h\n" - ".inst 0x6466e40c // bfmmla z12.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n" - ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e40a // bfmmla z10.s, z0.h, z7.h\n" - ".inst 0x6466e40e // bfmmla z14.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x9]\n" - "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n" - "trn2 z1.d, z1.d, z2.d\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" - ".inst 0x6466e40f // bfmmla z15.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x12, #2, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x12, #3, MUL VL]\n" - ".inst 0x6467e428 // bfmmla z8.s, z1.h, z7.h\n" - ".inst 0x6466e42c // bfmmla z12.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x11, #3, MUL VL]\n" - ".inst 0x6467e429 // bfmmla z9.s, z1.h, z7.h\n" - ".inst 0x6466e42d // bfmmla z13.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6467e42a // bfmmla z10.s, z1.h, z7.h\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x9, #3, MUL VL]\n" + "ld1rqh { z20.h }, p0/Z, [x26]\n" + "trn1 z18.d, z20.d, z19.d\n" + "ld1h { z17.h }, p5/Z, [x12]\n" + "ld1h { z16.h }, p5/Z, [x12, #1, MUL VL]\n" + ".inst 0x6471e648 // bfmmla z8.s, z18.h, z17.h\n" + ".inst 0x6470e64c // bfmmla z12.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x11]\n" + "ld1h { z16.h }, p5/Z, [x11, #1, MUL VL]\n" + ".inst 0x6471e649 // bfmmla z9.s, z18.h, z17.h\n" + ".inst 0x6470e64d // bfmmla z13.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6471e64a // bfmmla z10.s, z18.h, z17.h\n" + ".inst 0x6470e64e // bfmmla z14.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x9]\n" + "ld1h { z16.h }, p5/Z, [x9, #1, MUL VL]\n" + "trn2 z20.d, z20.d, z19.d\n" + ".inst 0x6471e64b // bfmmla z11.s, z18.h, z17.h\n" + ".inst 0x6470e64f // bfmmla z15.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x12, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x12, #3, MUL VL]\n" + ".inst 0x6471e688 // bfmmla z8.s, z20.h, z17.h\n" + ".inst 0x6470e68c // bfmmla z12.s, z20.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x11, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x11, #3, MUL VL]\n" + ".inst 0x6471e689 // bfmmla z9.s, z20.h, z17.h\n" + ".inst 0x6470e68d // bfmmla z13.s, z20.h, z16.h\n" + "ld1h { z16.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z17.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6470e68a // bfmmla z10.s, z20.h, z16.h\n" + ".inst 0x6471e68e // bfmmla z14.s, z20.h, z17.h\n" + "ld1h { z17.h }, p5/Z, [x9, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x9, #3, MUL VL]\n" "sub x27, x27, #0x8\n" "cmp x27, #0x8\n" - ".inst 0x6467e42b // bfmmla z11.s, z1.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" + ".inst 0x6471e68b // bfmmla z11.s, z20.h, z17.h\n" + ".inst 0x6470e68f // bfmmla z15.s, z20.h, z16.h\n" "add x26, x26, #0x10\n" "addvl x12, x12, #4\n" "addvl x11, x11, #4\n" @@ -246,46 +246,46 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "11:" // Height 1: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" "ld1rqh { z1.h }, p0/Z, [x26]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1h { z7.h }, p5/Z, [x12]\n" - "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n" - ".inst 0x6467e408 // bfmmla z8.s, z0.h, z7.h\n" - ".inst 0x6466e40c // bfmmla z12.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n" - ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e40a // bfmmla z10.s, z0.h, z7.h\n" - ".inst 0x6466e40e // bfmmla z14.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x9]\n" - "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n" + "trn1 z18.d, z1.d, z19.d\n" + "ld1h { z17.h }, p5/Z, [x12]\n" + "ld1h { z16.h }, p5/Z, [x12, #1, MUL VL]\n" + ".inst 0x6471e648 // bfmmla z8.s, z18.h, z17.h\n" + ".inst 0x6470e64c // bfmmla z12.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x11]\n" + "ld1h { z16.h }, p5/Z, [x11, #1, MUL VL]\n" + ".inst 0x6471e649 // bfmmla z9.s, z18.h, z17.h\n" + ".inst 0x6470e64d // bfmmla z13.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6471e64a // bfmmla z10.s, z18.h, z17.h\n" + ".inst 0x6470e64e // bfmmla z14.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x9]\n" + "ld1h { z16.h }, p5/Z, [x9, #1, MUL VL]\n" "subs x27, x27, #0x4\n" - "trn2 z1.d, z1.d, z2.d\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" - ".inst 0x6466e40f // bfmmla z15.s, z0.h, z6.h\n" + "trn2 z1.d, z1.d, z19.d\n" + ".inst 0x6471e64b // bfmmla z11.s, z18.h, z17.h\n" + ".inst 0x6470e64f // bfmmla z15.s, z18.h, z16.h\n" "addvl x12, x12, #2\n" "addvl x11, x11, #2\n" "addvl x10, x10, #2\n" "addvl x9, x9, #2\n" "ble 12f\n" - "ld1h { z7.h }, p5/Z, [x12]\n" - "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n" - ".inst 0x6467e428 // bfmmla z8.s, z1.h, z7.h\n" - ".inst 0x6466e42c // bfmmla z12.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n" - ".inst 0x6467e429 // bfmmla z9.s, z1.h, z7.h\n" - ".inst 0x6466e42d // bfmmla z13.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e42a // bfmmla z10.s, z1.h, z7.h\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x9]\n" - "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n" - ".inst 0x6467e42b // bfmmla z11.s, z1.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" + "ld1h { z17.h }, p5/Z, [x12]\n" + "ld1h { z16.h }, p5/Z, [x12, #1, MUL VL]\n" + ".inst 0x6471e428 // bfmmla z8.s, z1.h, z17.h\n" + ".inst 0x6470e42c // bfmmla z12.s, z1.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x11]\n" + "ld1h { z16.h }, p5/Z, [x11, #1, MUL VL]\n" + ".inst 0x6471e429 // bfmmla z9.s, z1.h, z17.h\n" + ".inst 0x6470e42d // bfmmla z13.s, z1.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6471e42a // bfmmla z10.s, z1.h, z17.h\n" + ".inst 0x6470e42e // bfmmla z14.s, z1.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x9]\n" + "ld1h { z16.h }, p5/Z, [x9, #1, MUL VL]\n" + ".inst 0x6471e42b // bfmmla z11.s, z1.h, z17.h\n" + ".inst 0x6470e42f // bfmmla z15.s, z1.h, z16.h\n" "addvl x12, x12, #2\n" "addvl x11, x11, #2\n" "addvl x10, x10, #2\n" @@ -301,17 +301,17 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "uzp1 z11.d, z11.d, z15.d\n" "tbz %x[flags], #1, 13f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z21.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" + "ld1rw { z16.s }, p5/Z, [x20]\n" + "fmin z8.s, p5/M, z8.s, z21.s\n" + "fmin z9.s, p5/M, z9.s, z21.s\n" + "fmin z10.s, p5/M, z10.s, z21.s\n" + "fmin z11.s, p5/M, z11.s, z21.s\n" + "fmax z8.s, p5/M, z8.s, z16.s\n" + "fmax z9.s, p5/M, z9.s, z16.s\n" + "fmax z10.s, p5/M, z10.s, z16.s\n" + "fmax z11.s, p5/M, z11.s, z16.s\n" "13:" // Height 1: No activation "st1w { z8.s }, p4, [x13]\n" "st1w { z9.s }, p3, [x13, #1, MUL VL]\n" @@ -376,21 +376,21 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "18:" // Height 2: no bias "tbz %x[flags], #0, 19f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x13]\n" - "ld1w { z10.s }, p3/Z, [x13, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x13, #2, MUL VL]\n" + "add x20, x13, x20, LSL #2\n" + "ld1w { z19.s }, p4/Z, [x13]\n" + "ld1w { z18.s }, p3/Z, [x13, #1, MUL VL]\n" + "ld1w { z17.s }, p2/Z, [x13, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x13, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" + "ld1w { z12.s }, p4/Z, [x20]\n" + "zip1 z8.d, z19.d, z12.d\n" + "zip2 z12.d, z19.d, z12.d\n" + "ld1w { z13.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x20, #2, MUL VL]\n" + "zip1 z9.d, z18.d, z13.d\n" + "zip2 z13.d, z18.d, z13.d\n" + "ld1w { z15.s }, p1/Z, [x20, #3, MUL VL]\n" + "zip1 z10.d, z17.d, z14.d\n" + "zip2 z14.d, z17.d, z14.d\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" "b 20f\n" @@ -408,12 +408,12 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "21:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 22f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 23f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -421,50 +421,50 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "b 23f\n" "22:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" "23:" // Height 2: input setup done "cmp x27, #0x8\n" "ble 25f\n" "24:" // Height 2: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z1.h }, p0/Z, [x26]\n" - "ld1rqh { z2.h }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1h { z7.h }, p5/Z, [x12]\n" - "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n" - ".inst 0x6467e408 // bfmmla z8.s, z0.h, z7.h\n" - ".inst 0x6466e40c // bfmmla z12.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n" - ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e40a // bfmmla z10.s, z0.h, z7.h\n" - ".inst 0x6466e40e // bfmmla z14.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x9]\n" - "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n" - "trn2 z1.d, z1.d, z2.d\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" - ".inst 0x6466e40f // bfmmla z15.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x12, #2, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x12, #3, MUL VL]\n" - ".inst 0x6467e428 // bfmmla z8.s, z1.h, z7.h\n" - ".inst 0x6466e42c // bfmmla z12.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x11, #3, MUL VL]\n" - ".inst 0x6467e429 // bfmmla z9.s, z1.h, z7.h\n" - ".inst 0x6466e42d // bfmmla z13.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6467e42a // bfmmla z10.s, z1.h, z7.h\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x9, #3, MUL VL]\n" + "ld1rqh { z20.h }, p0/Z, [x26]\n" + "ld1rqh { z19.h }, p0/Z, [x25]\n" + "trn1 z18.d, z20.d, z19.d\n" + "ld1h { z17.h }, p5/Z, [x12]\n" + "ld1h { z16.h }, p5/Z, [x12, #1, MUL VL]\n" + ".inst 0x6471e648 // bfmmla z8.s, z18.h, z17.h\n" + ".inst 0x6470e64c // bfmmla z12.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x11]\n" + "ld1h { z16.h }, p5/Z, [x11, #1, MUL VL]\n" + ".inst 0x6471e649 // bfmmla z9.s, z18.h, z17.h\n" + ".inst 0x6470e64d // bfmmla z13.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6471e64a // bfmmla z10.s, z18.h, z17.h\n" + ".inst 0x6470e64e // bfmmla z14.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x9]\n" + "ld1h { z16.h }, p5/Z, [x9, #1, MUL VL]\n" + "trn2 z20.d, z20.d, z19.d\n" + ".inst 0x6471e64b // bfmmla z11.s, z18.h, z17.h\n" + ".inst 0x6470e64f // bfmmla z15.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x12, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x12, #3, MUL VL]\n" + ".inst 0x6471e688 // bfmmla z8.s, z20.h, z17.h\n" + ".inst 0x6470e68c // bfmmla z12.s, z20.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x11, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x11, #3, MUL VL]\n" + ".inst 0x6471e689 // bfmmla z9.s, z20.h, z17.h\n" + ".inst 0x6470e68d // bfmmla z13.s, z20.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6471e68a // bfmmla z10.s, z20.h, z17.h\n" + ".inst 0x6470e68e // bfmmla z14.s, z20.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x9, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x9, #3, MUL VL]\n" "sub x27, x27, #0x8\n" "cmp x27, #0x8\n" - ".inst 0x6467e42b // bfmmla z11.s, z1.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" + ".inst 0x6471e68b // bfmmla z11.s, z20.h, z17.h\n" + ".inst 0x6470e68f // bfmmla z15.s, z20.h, z16.h\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" "addvl x12, x12, #4\n" @@ -475,47 +475,47 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "25:" // Height 2: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" "ld1rqh { z1.h }, p0/Z, [x26]\n" - "ld1rqh { z2.h }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1h { z7.h }, p5/Z, [x12]\n" - "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n" - ".inst 0x6467e408 // bfmmla z8.s, z0.h, z7.h\n" - ".inst 0x6466e40c // bfmmla z12.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n" - ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e40a // bfmmla z10.s, z0.h, z7.h\n" - ".inst 0x6466e40e // bfmmla z14.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x9]\n" - "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n" + "ld1rqh { z19.h }, p0/Z, [x25]\n" + "trn1 z18.d, z1.d, z19.d\n" + "ld1h { z17.h }, p5/Z, [x12]\n" + "ld1h { z16.h }, p5/Z, [x12, #1, MUL VL]\n" + ".inst 0x6471e648 // bfmmla z8.s, z18.h, z17.h\n" + ".inst 0x6470e64c // bfmmla z12.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x11]\n" + "ld1h { z16.h }, p5/Z, [x11, #1, MUL VL]\n" + ".inst 0x6471e649 // bfmmla z9.s, z18.h, z17.h\n" + ".inst 0x6470e64d // bfmmla z13.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6471e64a // bfmmla z10.s, z18.h, z17.h\n" + ".inst 0x6470e64e // bfmmla z14.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x9]\n" + "ld1h { z16.h }, p5/Z, [x9, #1, MUL VL]\n" "subs x27, x27, #0x4\n" - "trn2 z1.d, z1.d, z2.d\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" - ".inst 0x6466e40f // bfmmla z15.s, z0.h, z6.h\n" + "trn2 z1.d, z1.d, z19.d\n" + ".inst 0x6471e64b // bfmmla z11.s, z18.h, z17.h\n" + ".inst 0x6470e64f // bfmmla z15.s, z18.h, z16.h\n" "addvl x12, x12, #2\n" "addvl x11, x11, #2\n" "addvl x10, x10, #2\n" "addvl x9, x9, #2\n" "ble 26f\n" - "ld1h { z7.h }, p5/Z, [x12]\n" - "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n" - ".inst 0x6467e428 // bfmmla z8.s, z1.h, z7.h\n" - ".inst 0x6466e42c // bfmmla z12.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n" - ".inst 0x6467e429 // bfmmla z9.s, z1.h, z7.h\n" - ".inst 0x6466e42d // bfmmla z13.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e42a // bfmmla z10.s, z1.h, z7.h\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x9]\n" - "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n" - ".inst 0x6467e42b // bfmmla z11.s, z1.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" + "ld1h { z17.h }, p5/Z, [x12]\n" + "ld1h { z16.h }, p5/Z, [x12, #1, MUL VL]\n" + ".inst 0x6471e428 // bfmmla z8.s, z1.h, z17.h\n" + ".inst 0x6470e42c // bfmmla z12.s, z1.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x11]\n" + "ld1h { z16.h }, p5/Z, [x11, #1, MUL VL]\n" + ".inst 0x6471e429 // bfmmla z9.s, z1.h, z17.h\n" + ".inst 0x6470e42d // bfmmla z13.s, z1.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6471e42a // bfmmla z10.s, z1.h, z17.h\n" + ".inst 0x6470e42e // bfmmla z14.s, z1.h, z16.h\n" + "ld1h { z22.h }, p5/Z, [x9]\n" + "ld1h { z16.h }, p5/Z, [x9, #1, MUL VL]\n" + ".inst 0x6476e42b // bfmmla z11.s, z1.h, z22.h\n" + ".inst 0x6470e42f // bfmmla z15.s, z1.h, z16.h\n" "addvl x12, x12, #2\n" "addvl x11, x11, #2\n" "addvl x10, x10, #2\n" @@ -537,25 +537,25 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "uzp2 z11.d, z11.d, z15.d\n" "tbz %x[flags], #1, 27f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z17.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z7.s, p5/M, z7.s, z1.s\n" - "fmin z12.s, p5/M, z12.s, z1.s\n" - "fmin z13.s, p5/M, z13.s, z1.s\n" - "fmin z14.s, p5/M, z14.s, z1.s\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmax z7.s, p5/M, z7.s, z0.s\n" - "fmax z12.s, p5/M, z12.s, z0.s\n" - "fmax z13.s, p5/M, z13.s, z0.s\n" - "fmax z14.s, p5/M, z14.s, z0.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" + "ld1rw { z16.s }, p5/Z, [x20]\n" + "fmin z7.s, p5/M, z7.s, z17.s\n" + "fmin z12.s, p5/M, z12.s, z17.s\n" + "fmin z13.s, p5/M, z13.s, z17.s\n" + "fmin z14.s, p5/M, z14.s, z17.s\n" + "fmin z8.s, p5/M, z8.s, z17.s\n" + "fmin z9.s, p5/M, z9.s, z17.s\n" + "fmin z10.s, p5/M, z10.s, z17.s\n" + "fmin z11.s, p5/M, z11.s, z17.s\n" + "fmax z7.s, p5/M, z7.s, z16.s\n" + "fmax z12.s, p5/M, z12.s, z16.s\n" + "fmax z13.s, p5/M, z13.s, z16.s\n" + "fmax z14.s, p5/M, z14.s, z16.s\n" + "fmax z8.s, p5/M, z8.s, z16.s\n" + "fmax z9.s, p5/M, z9.s, z16.s\n" + "fmax z10.s, p5/M, z10.s, z16.s\n" + "fmax z11.s, p5/M, z11.s, z16.s\n" "27:" // Height 2: No activation "st1w { z7.s }, p4, [x13]\n" "st1w { z12.s }, p3, [x13, #1, MUL VL]\n" @@ -632,28 +632,28 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "32:" // Height 3: no bias "tbz %x[flags], #0, 33f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x13]\n" - "ld1w { z10.s }, p3/Z, [x13, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x13, #2, MUL VL]\n" + "add x21, x13, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" + "ld1w { z19.s }, p4/Z, [x13]\n" + "ld1w { z17.s }, p3/Z, [x13, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x13, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x13, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "ld1w { z17.s }, p4/Z, [x24]\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" - "ld1w { z18.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z19.s }, p2/Z, [x24, #2, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x21]\n" + "zip1 z8.d, z19.d, z12.d\n" + "zip2 z12.d, z19.d, z12.d\n" + "ld1w { z13.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x21, #2, MUL VL]\n" + "zip1 z9.d, z17.d, z13.d\n" + "zip2 z13.d, z17.d, z13.d\n" + "ld1w { z15.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z17.s }, p4/Z, [x20]\n" + "zip1 z10.d, z18.d, z14.d\n" + "zip2 z14.d, z18.d, z14.d\n" + "ld1w { z18.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z19.s }, p2/Z, [x20, #2, MUL VL]\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" - "ld1w { z24.s }, p1/Z, [x24, #3, MUL VL]\n" + "ld1w { z24.s }, p1/Z, [x20, #3, MUL VL]\n" "zip1 z16.d, z17.d, z20.d\n" "zip2 z20.d, z17.d, z20.d\n" "zip1 z17.d, z18.d, z21.d\n" @@ -685,13 +685,13 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "35:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 36f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 37f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -700,145 +700,145 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "b 37f\n" "36:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" "37:" // Height 3: input setup done "cmp x27, #0x8\n" "ble 39f\n" "38:" // Height 3: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z1.h }, p0/Z, [x26]\n" - "ld1rqh { z2.h }, p0/Z, [x25]\n" - "ld1rqh { z3.h }, p0/Z, [x24]\n" - "trn1 z0.d, z1.d, z2.d\n" - "trn2 z1.d, z1.d, z2.d\n" - "ld1h { z7.h }, p5/Z, [x12]\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n" - ".inst 0x6467e408 // bfmmla z8.s, z0.h, z7.h\n" - ".inst 0x6467e450 // bfmmla z16.s, z2.h, z7.h\n" - ".inst 0x6466e40c // bfmmla z12.s, z0.h, z6.h\n" - ".inst 0x6466e454 // bfmmla z20.s, z2.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n" - ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" - ".inst 0x6467e451 // bfmmla z17.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" + "ld1rqh { z30.h }, p0/Z, [x26]\n" + "ld1rqh { z24.h }, p0/Z, [x25]\n" + "ld1rqh { z28.h }, p0/Z, [x24]\n" + "trn1 z27.d, z30.d, z24.d\n" + "trn2 z30.d, z30.d, z24.d\n" + "ld1h { z25.h }, p5/Z, [x12]\n" + "trn1 z26.d, z28.d, z29.d\n" + "ld1h { z24.h }, p5/Z, [x12, #1, MUL VL]\n" + ".inst 0x6479e768 // bfmmla z8.s, z27.h, z25.h\n" + ".inst 0x6479e750 // bfmmla z16.s, z26.h, z25.h\n" + ".inst 0x6478e76c // bfmmla z12.s, z27.h, z24.h\n" + ".inst 0x6478e754 // bfmmla z20.s, z26.h, z24.h\n" + "ld1h { z25.h }, p5/Z, [x11]\n" + "ld1h { z24.h }, p5/Z, [x11, #1, MUL VL]\n" + ".inst 0x6479e769 // bfmmla z9.s, z27.h, z25.h\n" + ".inst 0x6479e751 // bfmmla z17.s, z26.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "trn2 z28.d, z28.d, z29.d\n" + ".inst 0x6478e76d // bfmmla z13.s, z27.h, z24.h\n" + ".inst 0x6478e755 // bfmmla z21.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" "sub x27, x27, #0x8\n" - ".inst 0x6467e40a // bfmmla z10.s, z0.h, z7.h\n" - ".inst 0x6467e452 // bfmmla z18.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + ".inst 0x6479e76a // bfmmla z10.s, z27.h, z25.h\n" + ".inst 0x6479e752 // bfmmla z18.s, z26.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x9]\n" "cmp x27, #0x8\n" - ".inst 0x6466e40e // bfmmla z14.s, z0.h, z6.h\n" - ".inst 0x6466e456 // bfmmla z22.s, z2.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n" + ".inst 0x6478e76e // bfmmla z14.s, z27.h, z24.h\n" + ".inst 0x6478e756 // bfmmla z22.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x9, #1, MUL VL]\n" "add x26, x26, #0x10\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" - ".inst 0x6467e453 // bfmmla z19.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x12, #2, MUL VL]\n" + ".inst 0x6479e76b // bfmmla z11.s, z27.h, z25.h\n" + ".inst 0x6479e753 // bfmmla z19.s, z26.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x12, #2, MUL VL]\n" "add x25, x25, #0x10\n" - ".inst 0x6466e40f // bfmmla z15.s, z0.h, z6.h\n" - ".inst 0x6466e457 // bfmmla z23.s, z2.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x12, #3, MUL VL]\n" + ".inst 0x6478e76f // bfmmla z15.s, z27.h, z24.h\n" + ".inst 0x6478e757 // bfmmla z23.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x12, #3, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x6467e428 // bfmmla z8.s, z1.h, z7.h\n" - ".inst 0x6467e470 // bfmmla z16.s, z3.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n" + ".inst 0x6479e7c8 // bfmmla z8.s, z30.h, z25.h\n" + ".inst 0x6479e790 // bfmmla z16.s, z28.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x11, #2, MUL VL]\n" "addvl x12, x12, #4\n" - ".inst 0x6466e42c // bfmmla z12.s, z1.h, z6.h\n" - ".inst 0x6466e474 // bfmmla z20.s, z3.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x11, #3, MUL VL]\n" + ".inst 0x6478e7cc // bfmmla z12.s, z30.h, z24.h\n" + ".inst 0x6478e794 // bfmmla z20.s, z28.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x11, #3, MUL VL]\n" "addvl x11, x11, #4\n" - ".inst 0x6467e429 // bfmmla z9.s, z1.h, z7.h\n" - ".inst 0x6467e471 // bfmmla z17.s, z3.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x6466e42d // bfmmla z13.s, z1.h, z6.h\n" - ".inst 0x6466e475 // bfmmla z21.s, z3.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6479e7c9 // bfmmla z9.s, z30.h, z25.h\n" + ".inst 0x6479e791 // bfmmla z17.s, z28.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x6478e7cd // bfmmla z13.s, z30.h, z24.h\n" + ".inst 0x6478e795 // bfmmla z21.s, z28.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - ".inst 0x6467e42a // bfmmla z10.s, z1.h, z7.h\n" - ".inst 0x6467e472 // bfmmla z18.s, z3.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - ".inst 0x6466e476 // bfmmla z22.s, z3.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x9, #3, MUL VL]\n" + ".inst 0x6479e7ca // bfmmla z10.s, z30.h, z25.h\n" + ".inst 0x6479e792 // bfmmla z18.s, z28.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x9, #2, MUL VL]\n" + ".inst 0x6478e7ce // bfmmla z14.s, z30.h, z24.h\n" + ".inst 0x6478e796 // bfmmla z22.s, z28.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - ".inst 0x6467e42b // bfmmla z11.s, z1.h, z7.h\n" - ".inst 0x6467e473 // bfmmla z19.s, z3.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" - ".inst 0x6466e477 // bfmmla z23.s, z3.h, z6.h\n" + ".inst 0x6479e7cb // bfmmla z11.s, z30.h, z25.h\n" + ".inst 0x6479e793 // bfmmla z19.s, z28.h, z25.h\n" + ".inst 0x6478e7cf // bfmmla z15.s, z30.h, z24.h\n" + ".inst 0x6478e797 // bfmmla z23.s, z28.h, z24.h\n" "bgt 38b\n" "39:" // Height 3: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" "ld1rqh { z1.h }, p0/Z, [x26]\n" - "ld1rqh { z2.h }, p0/Z, [x25]\n" + "ld1rqh { z24.h }, p0/Z, [x25]\n" "ld1rqh { z3.h }, p0/Z, [x24]\n" - "trn1 z0.d, z1.d, z2.d\n" - "trn2 z1.d, z1.d, z2.d\n" - "ld1h { z7.h }, p5/Z, [x12]\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n" - ".inst 0x6467e408 // bfmmla z8.s, z0.h, z7.h\n" - ".inst 0x6467e450 // bfmmla z16.s, z2.h, z7.h\n" - ".inst 0x6466e40c // bfmmla z12.s, z0.h, z6.h\n" - ".inst 0x6466e454 // bfmmla z20.s, z2.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n" - ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" - ".inst 0x6467e451 // bfmmla z17.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10]\n" + "trn1 z27.d, z1.d, z24.d\n" + "trn2 z1.d, z1.d, z24.d\n" + "ld1h { z25.h }, p5/Z, [x12]\n" + "trn1 z26.d, z3.d, z28.d\n" + "ld1h { z24.h }, p5/Z, [x12, #1, MUL VL]\n" + ".inst 0x6479e768 // bfmmla z8.s, z27.h, z25.h\n" + ".inst 0x6479e750 // bfmmla z16.s, z26.h, z25.h\n" + ".inst 0x6478e76c // bfmmla z12.s, z27.h, z24.h\n" + ".inst 0x6478e754 // bfmmla z20.s, z26.h, z24.h\n" + "ld1h { z25.h }, p5/Z, [x11]\n" + "ld1h { z24.h }, p5/Z, [x11, #1, MUL VL]\n" + ".inst 0x6479e769 // bfmmla z9.s, z27.h, z25.h\n" + ".inst 0x6479e751 // bfmmla z17.s, z26.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10]\n" "subs x27, x27, #0x4\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x6467e40a // bfmmla z10.s, z0.h, z7.h\n" - ".inst 0x6467e452 // bfmmla z18.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + ".inst 0x6478e76d // bfmmla z13.s, z27.h, z24.h\n" + ".inst 0x6478e755 // bfmmla z21.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + "trn2 z3.d, z3.d, z28.d\n" + ".inst 0x6479e76a // bfmmla z10.s, z27.h, z25.h\n" + ".inst 0x6479e752 // bfmmla z18.s, z26.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x9]\n" "addvl x12, x12, #2\n" - ".inst 0x6466e40e // bfmmla z14.s, z0.h, z6.h\n" - ".inst 0x6466e456 // bfmmla z22.s, z2.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n" + ".inst 0x6478e76e // bfmmla z14.s, z27.h, z24.h\n" + ".inst 0x6478e756 // bfmmla z22.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x9, #1, MUL VL]\n" "addvl x11, x11, #2\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" - ".inst 0x6467e453 // bfmmla z19.s, z2.h, z7.h\n" + ".inst 0x6479e76b // bfmmla z11.s, z27.h, z25.h\n" + ".inst 0x6479e753 // bfmmla z19.s, z26.h, z25.h\n" "addvl x10, x10, #2\n" "addvl x9, x9, #2\n" - ".inst 0x6466e40f // bfmmla z15.s, z0.h, z6.h\n" - ".inst 0x6466e457 // bfmmla z23.s, z2.h, z6.h\n" + ".inst 0x6478e76f // bfmmla z15.s, z27.h, z24.h\n" + ".inst 0x6478e757 // bfmmla z23.s, z26.h, z24.h\n" "ble 40f\n" - "ld1h { z7.h }, p5/Z, [x12]\n" - "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n" - ".inst 0x6467e428 // bfmmla z8.s, z1.h, z7.h\n" - ".inst 0x6467e470 // bfmmla z16.s, z3.h, z7.h\n" - ".inst 0x6466e42c // bfmmla z12.s, z1.h, z6.h\n" - ".inst 0x6466e474 // bfmmla z20.s, z3.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n" - ".inst 0x6467e429 // bfmmla z9.s, z1.h, z7.h\n" - ".inst 0x6467e471 // bfmmla z17.s, z3.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10]\n" + "ld1h { z25.h }, p5/Z, [x12]\n" + "ld1h { z24.h }, p5/Z, [x12, #1, MUL VL]\n" + ".inst 0x6479e428 // bfmmla z8.s, z1.h, z25.h\n" + ".inst 0x6479e470 // bfmmla z16.s, z3.h, z25.h\n" + ".inst 0x6478e42c // bfmmla z12.s, z1.h, z24.h\n" + ".inst 0x6478e474 // bfmmla z20.s, z3.h, z24.h\n" + "ld1h { z25.h }, p5/Z, [x11]\n" + "ld1h { z24.h }, p5/Z, [x11, #1, MUL VL]\n" + ".inst 0x6479e429 // bfmmla z9.s, z1.h, z25.h\n" + ".inst 0x6479e471 // bfmmla z17.s, z3.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10]\n" "addvl x12, x12, #2\n" - ".inst 0x6466e42d // bfmmla z13.s, z1.h, z6.h\n" - ".inst 0x6466e475 // bfmmla z21.s, z3.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6478e42d // bfmmla z13.s, z1.h, z24.h\n" + ".inst 0x6478e475 // bfmmla z21.s, z3.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" "addvl x11, x11, #2\n" - ".inst 0x6467e42a // bfmmla z10.s, z1.h, z7.h\n" - ".inst 0x6467e472 // bfmmla z18.s, z3.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + ".inst 0x6479e42a // bfmmla z10.s, z1.h, z25.h\n" + ".inst 0x6479e472 // bfmmla z18.s, z3.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x9]\n" "addvl x10, x10, #2\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - ".inst 0x6466e476 // bfmmla z22.s, z3.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n" + ".inst 0x6478e42e // bfmmla z14.s, z1.h, z24.h\n" + ".inst 0x6478e476 // bfmmla z22.s, z3.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x9, #1, MUL VL]\n" "addvl x9, x9, #2\n" - ".inst 0x6467e42b // bfmmla z11.s, z1.h, z7.h\n" - ".inst 0x6467e473 // bfmmla z19.s, z3.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" - ".inst 0x6466e477 // bfmmla z23.s, z3.h, z6.h\n" + ".inst 0x6479e42b // bfmmla z11.s, z1.h, z25.h\n" + ".inst 0x6479e473 // bfmmla z19.s, z3.h, z25.h\n" + ".inst 0x6478e42f // bfmmla z15.s, z1.h, z24.h\n" + ".inst 0x6478e477 // bfmmla z23.s, z3.h, z24.h\n" "40:" // Height 3: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -861,33 +861,33 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "uzp1 z19.d, z19.d, z23.d\n" "tbz %x[flags], #1, 41f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z25.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z7.s, p5/M, z7.s, z1.s\n" - "fmin z12.s, p5/M, z12.s, z1.s\n" - "fmin z13.s, p5/M, z13.s, z1.s\n" - "fmin z14.s, p5/M, z14.s, z1.s\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmin z16.s, p5/M, z16.s, z1.s\n" - "fmin z17.s, p5/M, z17.s, z1.s\n" - "fmin z18.s, p5/M, z18.s, z1.s\n" - "fmin z19.s, p5/M, z19.s, z1.s\n" - "fmax z7.s, p5/M, z7.s, z0.s\n" - "fmax z12.s, p5/M, z12.s, z0.s\n" - "fmax z13.s, p5/M, z13.s, z0.s\n" - "fmax z14.s, p5/M, z14.s, z0.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" - "fmax z16.s, p5/M, z16.s, z0.s\n" - "fmax z17.s, p5/M, z17.s, z0.s\n" - "fmax z18.s, p5/M, z18.s, z0.s\n" - "fmax z19.s, p5/M, z19.s, z0.s\n" + "ld1rw { z24.s }, p5/Z, [x20]\n" + "fmin z7.s, p5/M, z7.s, z25.s\n" + "fmin z12.s, p5/M, z12.s, z25.s\n" + "fmin z13.s, p5/M, z13.s, z25.s\n" + "fmin z14.s, p5/M, z14.s, z25.s\n" + "fmin z8.s, p5/M, z8.s, z25.s\n" + "fmin z9.s, p5/M, z9.s, z25.s\n" + "fmin z10.s, p5/M, z10.s, z25.s\n" + "fmin z11.s, p5/M, z11.s, z25.s\n" + "fmin z16.s, p5/M, z16.s, z25.s\n" + "fmin z17.s, p5/M, z17.s, z25.s\n" + "fmin z18.s, p5/M, z18.s, z25.s\n" + "fmin z19.s, p5/M, z19.s, z25.s\n" + "fmax z7.s, p5/M, z7.s, z24.s\n" + "fmax z12.s, p5/M, z12.s, z24.s\n" + "fmax z13.s, p5/M, z13.s, z24.s\n" + "fmax z14.s, p5/M, z14.s, z24.s\n" + "fmax z8.s, p5/M, z8.s, z24.s\n" + "fmax z9.s, p5/M, z9.s, z24.s\n" + "fmax z10.s, p5/M, z10.s, z24.s\n" + "fmax z11.s, p5/M, z11.s, z24.s\n" + "fmax z16.s, p5/M, z16.s, z24.s\n" + "fmax z17.s, p5/M, z17.s, z24.s\n" + "fmax z18.s, p5/M, z18.s, z24.s\n" + "fmax z19.s, p5/M, z19.s, z24.s\n" "41:" // Height 3: No activation "st1w { z7.s }, p4, [x13]\n" "st1w { z12.s }, p3, [x13, #1, MUL VL]\n" @@ -968,37 +968,37 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "46:" // Height 4: no bias "tbz %x[flags], #0, 47f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x13]\n" - "add x23, x24, x20, LSL #2\n" - "ld1w { z10.s }, p3/Z, [x13, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x13, #2, MUL VL]\n" + "add x22, x13, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" + "ld1w { z19.s }, p4/Z, [x13]\n" + "add x20, x21, x20, LSL #2\n" + "ld1w { z17.s }, p3/Z, [x13, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x13, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x13, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "ld1w { z17.s }, p4/Z, [x24]\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" - "ld1w { z18.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z19.s }, p2/Z, [x24, #2, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x22]\n" + "zip1 z8.d, z19.d, z12.d\n" + "zip2 z12.d, z19.d, z12.d\n" + "ld1w { z13.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x22, #2, MUL VL]\n" + "zip1 z9.d, z17.d, z13.d\n" + "zip2 z13.d, z17.d, z13.d\n" + "ld1w { z15.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z17.s }, p4/Z, [x21]\n" + "zip1 z10.d, z18.d, z14.d\n" + "zip2 z14.d, z18.d, z14.d\n" + "ld1w { z18.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z19.s }, p2/Z, [x21, #2, MUL VL]\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" - "ld1w { z24.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x23]\n" + "ld1w { z24.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x20]\n" "zip1 z16.d, z17.d, z20.d\n" "zip2 z20.d, z17.d, z20.d\n" - "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n" + "ld1w { z21.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x20, #2, MUL VL]\n" "zip1 z17.d, z18.d, z21.d\n" "zip2 z21.d, z18.d, z21.d\n" - "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z23.s }, p1/Z, [x20, #3, MUL VL]\n" "zip1 z18.d, z19.d, z22.d\n" "zip2 z22.d, z19.d, z22.d\n" "zip1 z19.d, z24.d, z23.d\n" @@ -1026,14 +1026,14 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "49:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 50f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 51f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -1043,149 +1043,149 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "b 51f\n" "50:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" "51:" // Height 4: input setup done "cmp x27, #0x8\n" "ble 53f\n" "52:" // Height 4: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z1.h }, p0/Z, [x26]\n" - "ld1rqh { z2.h }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1rqh { z3.h }, p0/Z, [x24]\n" - "ld1rqh { z4.h }, p0/Z, [x23]\n" - "trn2 z1.d, z1.d, z2.d\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1h { z7.h }, p5/Z, [x12]\n" - "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n" - ".inst 0x6467e408 // bfmmla z8.s, z0.h, z7.h\n" - ".inst 0x6467e450 // bfmmla z16.s, z2.h, z7.h\n" - ".inst 0x6466e40c // bfmmla z12.s, z0.h, z6.h\n" - ".inst 0x6466e454 // bfmmla z20.s, z2.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n" - ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" - ".inst 0x6467e451 // bfmmla z17.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" + "ld1rqh { z30.h }, p0/Z, [x26]\n" + "ld1rqh { z24.h }, p0/Z, [x25]\n" + "trn1 z29.d, z30.d, z24.d\n" + "ld1rqh { z28.h }, p0/Z, [x24]\n" + "ld1rqh { z27.h }, p0/Z, [x23]\n" + "trn2 z30.d, z30.d, z24.d\n" + "trn1 z26.d, z28.d, z27.d\n" + "ld1h { z25.h }, p5/Z, [x12]\n" + "ld1h { z24.h }, p5/Z, [x12, #1, MUL VL]\n" + ".inst 0x6479e7a8 // bfmmla z8.s, z29.h, z25.h\n" + ".inst 0x6479e750 // bfmmla z16.s, z26.h, z25.h\n" + ".inst 0x6478e7ac // bfmmla z12.s, z29.h, z24.h\n" + ".inst 0x6478e754 // bfmmla z20.s, z26.h, z24.h\n" + "ld1h { z25.h }, p5/Z, [x11]\n" + "ld1h { z24.h }, p5/Z, [x11, #1, MUL VL]\n" + ".inst 0x6479e7a9 // bfmmla z9.s, z29.h, z25.h\n" + ".inst 0x6479e751 // bfmmla z17.s, z26.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "trn2 z28.d, z28.d, z27.d\n" + ".inst 0x6478e7ad // bfmmla z13.s, z29.h, z24.h\n" + ".inst 0x6478e755 // bfmmla z21.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" "sub x27, x27, #0x8\n" - ".inst 0x6467e40a // bfmmla z10.s, z0.h, z7.h\n" - ".inst 0x6467e452 // bfmmla z18.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + ".inst 0x6479e7aa // bfmmla z10.s, z29.h, z25.h\n" + ".inst 0x6479e752 // bfmmla z18.s, z26.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x9]\n" "cmp x27, #0x8\n" - ".inst 0x6466e40e // bfmmla z14.s, z0.h, z6.h\n" - ".inst 0x6466e456 // bfmmla z22.s, z2.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n" + ".inst 0x6478e7ae // bfmmla z14.s, z29.h, z24.h\n" + ".inst 0x6478e756 // bfmmla z22.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x9, #1, MUL VL]\n" "add x26, x26, #0x10\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" - ".inst 0x6467e453 // bfmmla z19.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x12, #2, MUL VL]\n" + ".inst 0x6479e7ab // bfmmla z11.s, z29.h, z25.h\n" + ".inst 0x6479e753 // bfmmla z19.s, z26.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x12, #2, MUL VL]\n" "add x25, x25, #0x10\n" - ".inst 0x6466e40f // bfmmla z15.s, z0.h, z6.h\n" - ".inst 0x6466e457 // bfmmla z23.s, z2.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x12, #3, MUL VL]\n" + ".inst 0x6478e7af // bfmmla z15.s, z29.h, z24.h\n" + ".inst 0x6478e757 // bfmmla z23.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x12, #3, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x6467e428 // bfmmla z8.s, z1.h, z7.h\n" - ".inst 0x6467e470 // bfmmla z16.s, z3.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n" + ".inst 0x6479e7c8 // bfmmla z8.s, z30.h, z25.h\n" + ".inst 0x6479e790 // bfmmla z16.s, z28.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x11, #2, MUL VL]\n" "add x23, x23, #0x10\n" - ".inst 0x6466e42c // bfmmla z12.s, z1.h, z6.h\n" - ".inst 0x6466e474 // bfmmla z20.s, z3.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x11, #3, MUL VL]\n" + ".inst 0x6478e7cc // bfmmla z12.s, z30.h, z24.h\n" + ".inst 0x6478e794 // bfmmla z20.s, z28.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x11, #3, MUL VL]\n" "addvl x12, x12, #4\n" - ".inst 0x6467e429 // bfmmla z9.s, z1.h, z7.h\n" - ".inst 0x6467e471 // bfmmla z17.s, z3.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x6479e7c9 // bfmmla z9.s, z30.h, z25.h\n" + ".inst 0x6479e791 // bfmmla z17.s, z28.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" "addvl x11, x11, #4\n" - ".inst 0x6466e42d // bfmmla z13.s, z1.h, z6.h\n" - ".inst 0x6466e475 // bfmmla z21.s, z3.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6478e7cd // bfmmla z13.s, z30.h, z24.h\n" + ".inst 0x6478e795 // bfmmla z21.s, z28.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - ".inst 0x6467e42a // bfmmla z10.s, z1.h, z7.h\n" - ".inst 0x6467e472 // bfmmla z18.s, z3.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - ".inst 0x6466e476 // bfmmla z22.s, z3.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x9, #3, MUL VL]\n" + ".inst 0x6479e7ca // bfmmla z10.s, z30.h, z25.h\n" + ".inst 0x6479e792 // bfmmla z18.s, z28.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x9, #2, MUL VL]\n" + ".inst 0x6478e7ce // bfmmla z14.s, z30.h, z24.h\n" + ".inst 0x6478e796 // bfmmla z22.s, z28.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - ".inst 0x6467e42b // bfmmla z11.s, z1.h, z7.h\n" - ".inst 0x6467e473 // bfmmla z19.s, z3.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" - ".inst 0x6466e477 // bfmmla z23.s, z3.h, z6.h\n" + ".inst 0x6479e7cb // bfmmla z11.s, z30.h, z25.h\n" + ".inst 0x6479e793 // bfmmla z19.s, z28.h, z25.h\n" + ".inst 0x6478e7cf // bfmmla z15.s, z30.h, z24.h\n" + ".inst 0x6478e797 // bfmmla z23.s, z28.h, z24.h\n" "bgt 52b\n" "53:" // Height 4: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" "ld1rqh { z1.h }, p0/Z, [x26]\n" - "ld1rqh { z2.h }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" + "ld1rqh { z24.h }, p0/Z, [x25]\n" + "trn1 z28.d, z1.d, z24.d\n" "ld1rqh { z3.h }, p0/Z, [x24]\n" - "ld1rqh { z4.h }, p0/Z, [x23]\n" - "trn2 z1.d, z1.d, z2.d\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1h { z7.h }, p5/Z, [x12]\n" - "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n" - ".inst 0x6467e408 // bfmmla z8.s, z0.h, z7.h\n" - ".inst 0x6467e450 // bfmmla z16.s, z2.h, z7.h\n" - ".inst 0x6466e40c // bfmmla z12.s, z0.h, z6.h\n" - ".inst 0x6466e454 // bfmmla z20.s, z2.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n" - ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" - ".inst 0x6467e451 // bfmmla z17.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10]\n" + "ld1rqh { z27.h }, p0/Z, [x23]\n" + "trn2 z1.d, z1.d, z24.d\n" + "trn1 z26.d, z3.d, z27.d\n" + "ld1h { z25.h }, p5/Z, [x12]\n" + "ld1h { z24.h }, p5/Z, [x12, #1, MUL VL]\n" + ".inst 0x6479e788 // bfmmla z8.s, z28.h, z25.h\n" + ".inst 0x6479e750 // bfmmla z16.s, z26.h, z25.h\n" + ".inst 0x6478e78c // bfmmla z12.s, z28.h, z24.h\n" + ".inst 0x6478e754 // bfmmla z20.s, z26.h, z24.h\n" + "ld1h { z25.h }, p5/Z, [x11]\n" + "ld1h { z24.h }, p5/Z, [x11, #1, MUL VL]\n" + ".inst 0x6479e789 // bfmmla z9.s, z28.h, z25.h\n" + ".inst 0x6479e751 // bfmmla z17.s, z26.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10]\n" "subs x27, x27, #0x4\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x6467e40a // bfmmla z10.s, z0.h, z7.h\n" - ".inst 0x6467e452 // bfmmla z18.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + ".inst 0x6478e78d // bfmmla z13.s, z28.h, z24.h\n" + ".inst 0x6478e755 // bfmmla z21.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + "trn2 z3.d, z3.d, z27.d\n" + ".inst 0x6479e78a // bfmmla z10.s, z28.h, z25.h\n" + ".inst 0x6479e752 // bfmmla z18.s, z26.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x9]\n" "addvl x12, x12, #2\n" - ".inst 0x6466e40e // bfmmla z14.s, z0.h, z6.h\n" - ".inst 0x6466e456 // bfmmla z22.s, z2.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n" + ".inst 0x6478e78e // bfmmla z14.s, z28.h, z24.h\n" + ".inst 0x6478e756 // bfmmla z22.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x9, #1, MUL VL]\n" "addvl x11, x11, #2\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" - ".inst 0x6467e453 // bfmmla z19.s, z2.h, z7.h\n" + ".inst 0x6479e78b // bfmmla z11.s, z28.h, z25.h\n" + ".inst 0x6479e753 // bfmmla z19.s, z26.h, z25.h\n" "addvl x10, x10, #2\n" "addvl x9, x9, #2\n" - ".inst 0x6466e40f // bfmmla z15.s, z0.h, z6.h\n" - ".inst 0x6466e457 // bfmmla z23.s, z2.h, z6.h\n" + ".inst 0x6478e78f // bfmmla z15.s, z28.h, z24.h\n" + ".inst 0x6478e757 // bfmmla z23.s, z26.h, z24.h\n" "ble 54f\n" - "ld1h { z7.h }, p5/Z, [x12]\n" - "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n" - ".inst 0x6467e428 // bfmmla z8.s, z1.h, z7.h\n" - ".inst 0x6467e470 // bfmmla z16.s, z3.h, z7.h\n" - ".inst 0x6466e42c // bfmmla z12.s, z1.h, z6.h\n" - ".inst 0x6466e474 // bfmmla z20.s, z3.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n" - ".inst 0x6467e429 // bfmmla z9.s, z1.h, z7.h\n" - ".inst 0x6467e471 // bfmmla z17.s, z3.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10]\n" + "ld1h { z25.h }, p5/Z, [x12]\n" + "ld1h { z24.h }, p5/Z, [x12, #1, MUL VL]\n" + ".inst 0x6479e428 // bfmmla z8.s, z1.h, z25.h\n" + ".inst 0x6479e470 // bfmmla z16.s, z3.h, z25.h\n" + ".inst 0x6478e42c // bfmmla z12.s, z1.h, z24.h\n" + ".inst 0x6478e474 // bfmmla z20.s, z3.h, z24.h\n" + "ld1h { z25.h }, p5/Z, [x11]\n" + "ld1h { z24.h }, p5/Z, [x11, #1, MUL VL]\n" + ".inst 0x6479e429 // bfmmla z9.s, z1.h, z25.h\n" + ".inst 0x6479e471 // bfmmla z17.s, z3.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10]\n" "addvl x12, x12, #2\n" - ".inst 0x6466e42d // bfmmla z13.s, z1.h, z6.h\n" - ".inst 0x6466e475 // bfmmla z21.s, z3.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6478e42d // bfmmla z13.s, z1.h, z24.h\n" + ".inst 0x6478e475 // bfmmla z21.s, z3.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" "addvl x11, x11, #2\n" - ".inst 0x6467e42a // bfmmla z10.s, z1.h, z7.h\n" - ".inst 0x6467e472 // bfmmla z18.s, z3.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + ".inst 0x6479e42a // bfmmla z10.s, z1.h, z25.h\n" + ".inst 0x6479e472 // bfmmla z18.s, z3.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x9]\n" "addvl x10, x10, #2\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - ".inst 0x6466e476 // bfmmla z22.s, z3.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n" + ".inst 0x6478e42e // bfmmla z14.s, z1.h, z24.h\n" + ".inst 0x6478e476 // bfmmla z22.s, z3.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x9, #1, MUL VL]\n" "addvl x9, x9, #2\n" - ".inst 0x6467e42b // bfmmla z11.s, z1.h, z7.h\n" - ".inst 0x6467e473 // bfmmla z19.s, z3.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" - ".inst 0x6466e477 // bfmmla z23.s, z3.h, z6.h\n" + ".inst 0x6479e42b // bfmmla z11.s, z1.h, z25.h\n" + ".inst 0x6479e473 // bfmmla z19.s, z3.h, z25.h\n" + ".inst 0x6478e42f // bfmmla z15.s, z1.h, z24.h\n" + ".inst 0x6478e477 // bfmmla z23.s, z3.h, z24.h\n" "54:" // Height 4: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1213,41 +1213,41 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "uzp2 z19.d, z19.d, z23.d\n" "tbz %x[flags], #1, 55f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z24.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z7.s, p5/M, z7.s, z1.s\n" - "fmin z12.s, p5/M, z12.s, z1.s\n" - "fmin z13.s, p5/M, z13.s, z1.s\n" - "fmin z14.s, p5/M, z14.s, z1.s\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmin z15.s, p5/M, z15.s, z1.s\n" - "fmin z20.s, p5/M, z20.s, z1.s\n" - "fmin z21.s, p5/M, z21.s, z1.s\n" - "fmin z22.s, p5/M, z22.s, z1.s\n" - "fmin z16.s, p5/M, z16.s, z1.s\n" - "fmin z17.s, p5/M, z17.s, z1.s\n" - "fmin z18.s, p5/M, z18.s, z1.s\n" - "fmin z19.s, p5/M, z19.s, z1.s\n" - "fmax z7.s, p5/M, z7.s, z0.s\n" - "fmax z12.s, p5/M, z12.s, z0.s\n" - "fmax z13.s, p5/M, z13.s, z0.s\n" - "fmax z14.s, p5/M, z14.s, z0.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" - "fmax z15.s, p5/M, z15.s, z0.s\n" - "fmax z20.s, p5/M, z20.s, z0.s\n" - "fmax z21.s, p5/M, z21.s, z0.s\n" - "fmax z22.s, p5/M, z22.s, z0.s\n" - "fmax z16.s, p5/M, z16.s, z0.s\n" - "fmax z17.s, p5/M, z17.s, z0.s\n" - "fmax z18.s, p5/M, z18.s, z0.s\n" - "fmax z19.s, p5/M, z19.s, z0.s\n" + "ld1rw { z23.s }, p5/Z, [x20]\n" + "fmin z7.s, p5/M, z7.s, z24.s\n" + "fmin z12.s, p5/M, z12.s, z24.s\n" + "fmin z13.s, p5/M, z13.s, z24.s\n" + "fmin z14.s, p5/M, z14.s, z24.s\n" + "fmin z8.s, p5/M, z8.s, z24.s\n" + "fmin z9.s, p5/M, z9.s, z24.s\n" + "fmin z10.s, p5/M, z10.s, z24.s\n" + "fmin z11.s, p5/M, z11.s, z24.s\n" + "fmin z15.s, p5/M, z15.s, z24.s\n" + "fmin z20.s, p5/M, z20.s, z24.s\n" + "fmin z21.s, p5/M, z21.s, z24.s\n" + "fmin z22.s, p5/M, z22.s, z24.s\n" + "fmin z16.s, p5/M, z16.s, z24.s\n" + "fmin z17.s, p5/M, z17.s, z24.s\n" + "fmin z18.s, p5/M, z18.s, z24.s\n" + "fmin z19.s, p5/M, z19.s, z24.s\n" + "fmax z7.s, p5/M, z7.s, z23.s\n" + "fmax z12.s, p5/M, z12.s, z23.s\n" + "fmax z13.s, p5/M, z13.s, z23.s\n" + "fmax z14.s, p5/M, z14.s, z23.s\n" + "fmax z8.s, p5/M, z8.s, z23.s\n" + "fmax z9.s, p5/M, z9.s, z23.s\n" + "fmax z10.s, p5/M, z10.s, z23.s\n" + "fmax z11.s, p5/M, z11.s, z23.s\n" + "fmax z15.s, p5/M, z15.s, z23.s\n" + "fmax z20.s, p5/M, z20.s, z23.s\n" + "fmax z21.s, p5/M, z21.s, z23.s\n" + "fmax z22.s, p5/M, z22.s, z23.s\n" + "fmax z16.s, p5/M, z16.s, z23.s\n" + "fmax z17.s, p5/M, z17.s, z23.s\n" + "fmax z18.s, p5/M, z18.s, z23.s\n" + "fmax z19.s, p5/M, z19.s, z23.s\n" "55:" // Height 4: No activation "st1w { z7.s }, p4, [x13]\n" "st1w { z12.s }, p3, [x13, #1, MUL VL]\n" @@ -1340,54 +1340,54 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "60:" // Height 5: no bias "tbz %x[flags], #0, 61f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x13]\n" - "add x23, x24, x20, LSL #2\n" + "add x23, x13, x20, LSL #2\n" "add x22, x23, x20, LSL #2\n" - "ld1w { z10.s }, p3/Z, [x13, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x13, #2, MUL VL]\n" + "ld1w { z19.s }, p4/Z, [x13]\n" + "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" + "ld1w { z17.s }, p3/Z, [x13, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x13, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x13, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "ld1w { z17.s }, p4/Z, [x24]\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" - "ld1w { z18.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z19.s }, p2/Z, [x24, #2, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x23]\n" + "zip1 z8.d, z19.d, z12.d\n" + "zip2 z12.d, z19.d, z12.d\n" + "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n" + "zip1 z9.d, z17.d, z13.d\n" + "zip2 z13.d, z17.d, z13.d\n" + "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z17.s }, p4/Z, [x22]\n" + "zip1 z10.d, z18.d, z14.d\n" + "zip2 z14.d, z18.d, z14.d\n" + "ld1w { z18.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z19.s }, p2/Z, [x22, #2, MUL VL]\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" - "ld1w { z24.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x23]\n" + "ld1w { z24.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x21]\n" "zip1 z16.d, z17.d, z20.d\n" "zip2 z20.d, z17.d, z20.d\n" - "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n" + "ld1w { z21.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x21, #2, MUL VL]\n" "zip1 z17.d, z18.d, z21.d\n" "zip2 z21.d, z18.d, z21.d\n" - "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n" - "ld1w { z25.s }, p4/Z, [x22]\n" + "ld1w { z23.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z25.s }, p4/Z, [x20]\n" "zip1 z18.d, z19.d, z22.d\n" "zip2 z22.d, z19.d, z22.d\n" - "ld1w { z26.s }, p3/Z, [x22, #1, MUL VL]\n" - "ld1w { z27.s }, p2/Z, [x22, #2, MUL VL]\n" + "ld1w { z26.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z27.s }, p2/Z, [x20, #2, MUL VL]\n" "zip1 z19.d, z24.d, z23.d\n" "zip2 z23.d, z24.d, z23.d\n" - "ld1w { z6.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z0.s }, p1/Z, [x20, #3, MUL VL]\n" "zip1 z24.d, z25.d, z28.d\n" "zip2 z28.d, z25.d, z28.d\n" "zip1 z25.d, z26.d, z29.d\n" "zip2 z29.d, z26.d, z29.d\n" "zip1 z26.d, z27.d, z30.d\n" "zip2 z30.d, z27.d, z30.d\n" - "zip1 z27.d, z6.d, z31.d\n" - "zip2 z31.d, z6.d, z31.d\n" + "zip1 z27.d, z0.d, z31.d\n" + "zip2 z31.d, z0.d, z31.d\n" "b 62f\n" "61:" // Height 5: no accumulate "mov z8.b, #0x0\n" @@ -1419,15 +1419,15 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "63:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 64f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 65f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -1438,189 +1438,189 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "b 65f\n" "64:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" - "add x22, x23, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" + "add x22, x23, x21, LSL #1\n" "65:" // Height 5: input setup done "cmp x27, #0x8\n" "ble 67f\n" "66:" // Height 5: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z1.h }, p0/Z, [x26]\n" - "ld1rqh { z2.h }, p0/Z, [x25]\n" - "ld1rqh { z3.h }, p0/Z, [x24]\n" - "ld1rqh { z4.h }, p0/Z, [x23]\n" - "trn1 z0.d, z1.d, z2.d\n" - "trn2 z1.d, z1.d, z2.d\n" - "ld1rqh { z5.h }, p0/Z, [x22]\n" - "trn1 z2.d, z3.d, z4.d\n" - "trn2 z3.d, z3.d, z4.d\n" - "ld1h { z7.h }, p5/Z, [x12]\n" - "trn1 z4.d, z5.d, z6.d\n" - "trn2 z5.d, z5.d, z6.d\n" - "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n" - ".inst 0x6467e408 // bfmmla z8.s, z0.h, z7.h\n" - ".inst 0x6467e450 // bfmmla z16.s, z2.h, z7.h\n" - ".inst 0x6467e498 // bfmmla z24.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x11]\n" + "ld1rqh { z6.h }, p0/Z, [x26]\n" + "ld1rqh { z1.h }, p0/Z, [x25]\n" + "ld1rqh { z7.h }, p0/Z, [x24]\n" + "ld1rqh { z2.h }, p0/Z, [x23]\n" + "trn1 z5.d, z6.d, z1.d\n" + "trn2 z6.d, z6.d, z1.d\n" + "ld1rqh { z4.h }, p0/Z, [x22]\n" + "trn1 z3.d, z7.d, z2.d\n" + "trn2 z7.d, z7.d, z2.d\n" + "ld1h { z1.h }, p5/Z, [x12]\n" + "trn1 z2.d, z4.d, z0.d\n" + "trn2 z4.d, z4.d, z0.d\n" + "ld1h { z0.h }, p5/Z, [x12, #1, MUL VL]\n" + ".inst 0x6461e4a8 // bfmmla z8.s, z5.h, z1.h\n" + ".inst 0x6461e470 // bfmmla z16.s, z3.h, z1.h\n" + ".inst 0x6461e458 // bfmmla z24.s, z2.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x11]\n" "sub x27, x27, #0x8\n" - ".inst 0x6466e40c // bfmmla z12.s, z0.h, z6.h\n" - ".inst 0x6466e454 // bfmmla z20.s, z2.h, z6.h\n" + ".inst 0x6460e4ac // bfmmla z12.s, z5.h, z0.h\n" + ".inst 0x6460e474 // bfmmla z20.s, z3.h, z0.h\n" "cmp x27, #0x8\n" "add x26, x26, #0x10\n" - ".inst 0x6466e49c // bfmmla z28.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n" - ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" + ".inst 0x6460e45c // bfmmla z28.s, z2.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x11, #1, MUL VL]\n" + ".inst 0x6461e4a9 // bfmmla z9.s, z5.h, z1.h\n" "add x25, x25, #0x10\n" - ".inst 0x6467e451 // bfmmla z17.s, z2.h, z7.h\n" - ".inst 0x6467e499 // bfmmla z25.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10]\n" + ".inst 0x6461e471 // bfmmla z17.s, z3.h, z1.h\n" + ".inst 0x6461e459 // bfmmla z25.s, z2.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x10]\n" "add x24, x24, #0x10\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" + ".inst 0x6460e4ad // bfmmla z13.s, z5.h, z0.h\n" + ".inst 0x6460e475 // bfmmla z21.s, z3.h, z0.h\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - ".inst 0x6466e49d // bfmmla z29.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e40a // bfmmla z10.s, z0.h, z7.h\n" - ".inst 0x6467e452 // bfmmla z18.s, z2.h, z7.h\n" - ".inst 0x6467e49a // bfmmla z26.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x9]\n" - ".inst 0x6466e40e // bfmmla z14.s, z0.h, z6.h\n" - ".inst 0x6466e456 // bfmmla z22.s, z2.h, z6.h\n" - ".inst 0x6466e49e // bfmmla z30.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" - ".inst 0x6467e453 // bfmmla z19.s, z2.h, z7.h\n" - ".inst 0x6467e49b // bfmmla z27.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x12, #2, MUL VL]\n" - ".inst 0x6466e40f // bfmmla z15.s, z0.h, z6.h\n" - ".inst 0x6466e457 // bfmmla z23.s, z2.h, z6.h\n" - ".inst 0x6466e49f // bfmmla z31.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x12, #3, MUL VL]\n" - ".inst 0x6467e428 // bfmmla z8.s, z1.h, z7.h\n" + ".inst 0x6460e45d // bfmmla z29.s, z2.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6461e4aa // bfmmla z10.s, z5.h, z1.h\n" + ".inst 0x6461e472 // bfmmla z18.s, z3.h, z1.h\n" + ".inst 0x6461e45a // bfmmla z26.s, z2.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x9]\n" + ".inst 0x6460e4ae // bfmmla z14.s, z5.h, z0.h\n" + ".inst 0x6460e476 // bfmmla z22.s, z3.h, z0.h\n" + ".inst 0x6460e45e // bfmmla z30.s, z2.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x9, #1, MUL VL]\n" + ".inst 0x6461e4ab // bfmmla z11.s, z5.h, z1.h\n" + ".inst 0x6461e473 // bfmmla z19.s, z3.h, z1.h\n" + ".inst 0x6461e45b // bfmmla z27.s, z2.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x12, #2, MUL VL]\n" + ".inst 0x6460e4af // bfmmla z15.s, z5.h, z0.h\n" + ".inst 0x6460e477 // bfmmla z23.s, z3.h, z0.h\n" + ".inst 0x6460e45f // bfmmla z31.s, z2.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x12, #3, MUL VL]\n" + ".inst 0x6461e4c8 // bfmmla z8.s, z6.h, z1.h\n" "addvl x12, x12, #4\n" - ".inst 0x6467e470 // bfmmla z16.s, z3.h, z7.h\n" - ".inst 0x6467e4b8 // bfmmla z24.s, z5.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n" - ".inst 0x6466e42c // bfmmla z12.s, z1.h, z6.h\n" - ".inst 0x6466e474 // bfmmla z20.s, z3.h, z6.h\n" - ".inst 0x6466e4bc // bfmmla z28.s, z5.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x11, #3, MUL VL]\n" - ".inst 0x6467e429 // bfmmla z9.s, z1.h, z7.h\n" + ".inst 0x6461e4f0 // bfmmla z16.s, z7.h, z1.h\n" + ".inst 0x6461e498 // bfmmla z24.s, z4.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x11, #2, MUL VL]\n" + ".inst 0x6460e4cc // bfmmla z12.s, z6.h, z0.h\n" + ".inst 0x6460e4f4 // bfmmla z20.s, z7.h, z0.h\n" + ".inst 0x6460e49c // bfmmla z28.s, z4.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x11, #3, MUL VL]\n" + ".inst 0x6461e4c9 // bfmmla z9.s, z6.h, z1.h\n" "addvl x11, x11, #4\n" - ".inst 0x6467e471 // bfmmla z17.s, z3.h, z7.h\n" - ".inst 0x6467e4b9 // bfmmla z25.s, z5.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x6466e42d // bfmmla z13.s, z1.h, z6.h\n" - ".inst 0x6466e475 // bfmmla z21.s, z3.h, z6.h\n" - ".inst 0x6466e4bd // bfmmla z29.s, z5.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6467e42a // bfmmla z10.s, z1.h, z7.h\n" + ".inst 0x6461e4f1 // bfmmla z17.s, z7.h, z1.h\n" + ".inst 0x6461e499 // bfmmla z25.s, z4.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x6460e4cd // bfmmla z13.s, z6.h, z0.h\n" + ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n" + ".inst 0x6460e49d // bfmmla z29.s, z4.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6461e4ca // bfmmla z10.s, z6.h, z1.h\n" "addvl x10, x10, #4\n" - ".inst 0x6467e472 // bfmmla z18.s, z3.h, z7.h\n" - ".inst 0x6467e4ba // bfmmla z26.s, z5.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - ".inst 0x6466e476 // bfmmla z22.s, z3.h, z6.h\n" - ".inst 0x6466e4be // bfmmla z30.s, z5.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x9, #3, MUL VL]\n" - ".inst 0x6467e42b // bfmmla z11.s, z1.h, z7.h\n" + ".inst 0x6461e4f2 // bfmmla z18.s, z7.h, z1.h\n" + ".inst 0x6461e49a // bfmmla z26.s, z4.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x9, #2, MUL VL]\n" + ".inst 0x6460e4ce // bfmmla z14.s, z6.h, z0.h\n" + ".inst 0x6460e4f6 // bfmmla z22.s, z7.h, z0.h\n" + ".inst 0x6460e49e // bfmmla z30.s, z4.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x9, #3, MUL VL]\n" + ".inst 0x6461e4cb // bfmmla z11.s, z6.h, z1.h\n" "addvl x9, x9, #4\n" - ".inst 0x6467e473 // bfmmla z19.s, z3.h, z7.h\n" - ".inst 0x6467e4bb // bfmmla z27.s, z5.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" - ".inst 0x6466e477 // bfmmla z23.s, z3.h, z6.h\n" - ".inst 0x6466e4bf // bfmmla z31.s, z5.h, z6.h\n" + ".inst 0x6461e4f3 // bfmmla z19.s, z7.h, z1.h\n" + ".inst 0x6461e49b // bfmmla z27.s, z4.h, z1.h\n" + ".inst 0x6460e4cf // bfmmla z15.s, z6.h, z0.h\n" + ".inst 0x6460e4f7 // bfmmla z23.s, z7.h, z0.h\n" + ".inst 0x6460e49f // bfmmla z31.s, z4.h, z0.h\n" "bgt 66b\n" "67:" // Height 5: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" "ld1rqh { z1.h }, p0/Z, [x26]\n" - "ld1rqh { z2.h }, p0/Z, [x25]\n" + "ld1rqh { z4.h }, p0/Z, [x25]\n" "ld1rqh { z3.h }, p0/Z, [x24]\n" - "ld1rqh { z4.h }, p0/Z, [x23]\n" - "trn1 z0.d, z1.d, z2.d\n" - "trn2 z1.d, z1.d, z2.d\n" + "ld1rqh { z2.h }, p0/Z, [x23]\n" + "trn1 z7.d, z1.d, z4.d\n" + "trn2 z1.d, z1.d, z4.d\n" "ld1rqh { z5.h }, p0/Z, [x22]\n" - "trn1 z2.d, z3.d, z4.d\n" - "trn2 z3.d, z3.d, z4.d\n" - "ld1h { z7.h }, p5/Z, [x12]\n" - "trn1 z4.d, z5.d, z6.d\n" - "trn2 z5.d, z5.d, z6.d\n" - "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n" - ".inst 0x6467e408 // bfmmla z8.s, z0.h, z7.h\n" - ".inst 0x6467e450 // bfmmla z16.s, z2.h, z7.h\n" - ".inst 0x6467e498 // bfmmla z24.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x11]\n" + "trn1 z6.d, z3.d, z2.d\n" + "trn2 z3.d, z3.d, z2.d\n" + "ld1h { z2.h }, p5/Z, [x12]\n" + "trn1 z4.d, z5.d, z0.d\n" + "trn2 z5.d, z5.d, z0.d\n" + "ld1h { z0.h }, p5/Z, [x12, #1, MUL VL]\n" + ".inst 0x6462e4e8 // bfmmla z8.s, z7.h, z2.h\n" + ".inst 0x6462e4d0 // bfmmla z16.s, z6.h, z2.h\n" + ".inst 0x6462e498 // bfmmla z24.s, z4.h, z2.h\n" + "ld1h { z2.h }, p5/Z, [x11]\n" "subs x27, x27, #0x4\n" - ".inst 0x6466e40c // bfmmla z12.s, z0.h, z6.h\n" - ".inst 0x6466e454 // bfmmla z20.s, z2.h, z6.h\n" + ".inst 0x6460e4ec // bfmmla z12.s, z7.h, z0.h\n" + ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n" "addvl x12, x12, #2\n" - ".inst 0x6466e49c // bfmmla z28.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n" - ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" + ".inst 0x6460e49c // bfmmla z28.s, z4.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x11, #1, MUL VL]\n" + ".inst 0x6462e4e9 // bfmmla z9.s, z7.h, z2.h\n" "addvl x11, x11, #2\n" - ".inst 0x6467e451 // bfmmla z17.s, z2.h, z7.h\n" - ".inst 0x6467e499 // bfmmla z25.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" - ".inst 0x6466e49d // bfmmla z29.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e40a // bfmmla z10.s, z0.h, z7.h\n" + ".inst 0x6462e4d1 // bfmmla z17.s, z6.h, z2.h\n" + ".inst 0x6462e499 // bfmmla z25.s, z4.h, z2.h\n" + "ld1h { z2.h }, p5/Z, [x10]\n" + ".inst 0x6460e4ed // bfmmla z13.s, z7.h, z0.h\n" + ".inst 0x6460e4d5 // bfmmla z21.s, z6.h, z0.h\n" + ".inst 0x6460e49d // bfmmla z29.s, z4.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6462e4ea // bfmmla z10.s, z7.h, z2.h\n" "addvl x10, x10, #2\n" - ".inst 0x6467e452 // bfmmla z18.s, z2.h, z7.h\n" - ".inst 0x6467e49a // bfmmla z26.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x9]\n" - ".inst 0x6466e40e // bfmmla z14.s, z0.h, z6.h\n" - ".inst 0x6466e456 // bfmmla z22.s, z2.h, z6.h\n" - ".inst 0x6466e49e // bfmmla z30.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" + ".inst 0x6462e4d2 // bfmmla z18.s, z6.h, z2.h\n" + ".inst 0x6462e49a // bfmmla z26.s, z4.h, z2.h\n" + "ld1h { z2.h }, p5/Z, [x9]\n" + ".inst 0x6460e4ee // bfmmla z14.s, z7.h, z0.h\n" + ".inst 0x6460e4d6 // bfmmla z22.s, z6.h, z0.h\n" + ".inst 0x6460e49e // bfmmla z30.s, z4.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x9, #1, MUL VL]\n" + ".inst 0x6462e4eb // bfmmla z11.s, z7.h, z2.h\n" "addvl x9, x9, #2\n" - ".inst 0x6467e453 // bfmmla z19.s, z2.h, z7.h\n" - ".inst 0x6467e49b // bfmmla z27.s, z4.h, z7.h\n" - ".inst 0x6466e40f // bfmmla z15.s, z0.h, z6.h\n" - ".inst 0x6466e457 // bfmmla z23.s, z2.h, z6.h\n" - ".inst 0x6466e49f // bfmmla z31.s, z4.h, z6.h\n" + ".inst 0x6462e4d3 // bfmmla z19.s, z6.h, z2.h\n" + ".inst 0x6462e49b // bfmmla z27.s, z4.h, z2.h\n" + ".inst 0x6460e4ef // bfmmla z15.s, z7.h, z0.h\n" + ".inst 0x6460e4d7 // bfmmla z23.s, z6.h, z0.h\n" + ".inst 0x6460e49f // bfmmla z31.s, z4.h, z0.h\n" "ble 68f\n" - "ld1h { z7.h }, p5/Z, [x12]\n" - "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n" - ".inst 0x6467e428 // bfmmla z8.s, z1.h, z7.h\n" - ".inst 0x6467e470 // bfmmla z16.s, z3.h, z7.h\n" - ".inst 0x6467e4b8 // bfmmla z24.s, z5.h, z7.h\n" - ".inst 0x6466e42c // bfmmla z12.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x11]\n" + "ld1h { z2.h }, p5/Z, [x12]\n" + "ld1h { z0.h }, p5/Z, [x12, #1, MUL VL]\n" + ".inst 0x6462e428 // bfmmla z8.s, z1.h, z2.h\n" + ".inst 0x6462e470 // bfmmla z16.s, z3.h, z2.h\n" + ".inst 0x6462e4b8 // bfmmla z24.s, z5.h, z2.h\n" + ".inst 0x6460e42c // bfmmla z12.s, z1.h, z0.h\n" + "ld1h { z2.h }, p5/Z, [x11]\n" "addvl x12, x12, #2\n" - ".inst 0x6466e474 // bfmmla z20.s, z3.h, z6.h\n" - ".inst 0x6466e4bc // bfmmla z28.s, z5.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n" + ".inst 0x6460e474 // bfmmla z20.s, z3.h, z0.h\n" + ".inst 0x6460e4bc // bfmmla z28.s, z5.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x11, #1, MUL VL]\n" "addvl x11, x11, #2\n" - ".inst 0x6467e429 // bfmmla z9.s, z1.h, z7.h\n" - ".inst 0x6467e471 // bfmmla z17.s, z3.h, z7.h\n" - ".inst 0x6467e4b9 // bfmmla z25.s, z5.h, z7.h\n" - ".inst 0x6466e42d // bfmmla z13.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - ".inst 0x6466e475 // bfmmla z21.s, z3.h, z6.h\n" - ".inst 0x6466e4bd // bfmmla z29.s, z5.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6462e429 // bfmmla z9.s, z1.h, z2.h\n" + ".inst 0x6462e471 // bfmmla z17.s, z3.h, z2.h\n" + ".inst 0x6462e4b9 // bfmmla z25.s, z5.h, z2.h\n" + ".inst 0x6460e42d // bfmmla z13.s, z1.h, z0.h\n" + "ld1h { z2.h }, p5/Z, [x10]\n" + ".inst 0x6460e475 // bfmmla z21.s, z3.h, z0.h\n" + ".inst 0x6460e4bd // bfmmla z29.s, z5.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #1, MUL VL]\n" "addvl x10, x10, #2\n" - ".inst 0x6467e42a // bfmmla z10.s, z1.h, z7.h\n" - ".inst 0x6467e472 // bfmmla z18.s, z3.h, z7.h\n" - ".inst 0x6467e4ba // bfmmla z26.s, z5.h, z7.h\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x9]\n" - ".inst 0x6466e476 // bfmmla z22.s, z3.h, z6.h\n" - ".inst 0x6466e4be // bfmmla z30.s, z5.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n" + ".inst 0x6462e42a // bfmmla z10.s, z1.h, z2.h\n" + ".inst 0x6462e472 // bfmmla z18.s, z3.h, z2.h\n" + ".inst 0x6462e4ba // bfmmla z26.s, z5.h, z2.h\n" + ".inst 0x6460e42e // bfmmla z14.s, z1.h, z0.h\n" + "ld1h { z2.h }, p5/Z, [x9]\n" + ".inst 0x6460e476 // bfmmla z22.s, z3.h, z0.h\n" + ".inst 0x6460e4be // bfmmla z30.s, z5.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x9, #1, MUL VL]\n" "addvl x9, x9, #2\n" - ".inst 0x6467e42b // bfmmla z11.s, z1.h, z7.h\n" - ".inst 0x6467e473 // bfmmla z19.s, z3.h, z7.h\n" - ".inst 0x6467e4bb // bfmmla z27.s, z5.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" - ".inst 0x6466e477 // bfmmla z23.s, z3.h, z6.h\n" - ".inst 0x6466e4bf // bfmmla z31.s, z5.h, z6.h\n" + ".inst 0x6462e42b // bfmmla z11.s, z1.h, z2.h\n" + ".inst 0x6462e473 // bfmmla z19.s, z3.h, z2.h\n" + ".inst 0x6462e4bb // bfmmla z27.s, z5.h, z2.h\n" + ".inst 0x6460e42f // bfmmla z15.s, z1.h, z0.h\n" + ".inst 0x6460e477 // bfmmla z23.s, z3.h, z0.h\n" + ".inst 0x6460e4bf // bfmmla z31.s, z5.h, z0.h\n" "68:" // Height 5: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1653,49 +1653,49 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "uzp1 z27.d, z27.d, z31.d\n" "tbz %x[flags], #1, 69f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" - "add x20, %x[args_ptr], %[offset_min]\n" "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z7.s, p5/M, z7.s, z1.s\n" - "fmin z12.s, p5/M, z12.s, z1.s\n" - "fmin z13.s, p5/M, z13.s, z1.s\n" - "fmin z14.s, p5/M, z14.s, z1.s\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmin z15.s, p5/M, z15.s, z1.s\n" - "fmin z20.s, p5/M, z20.s, z1.s\n" - "fmin z21.s, p5/M, z21.s, z1.s\n" - "fmin z22.s, p5/M, z22.s, z1.s\n" - "fmin z16.s, p5/M, z16.s, z1.s\n" - "fmin z17.s, p5/M, z17.s, z1.s\n" - "fmin z18.s, p5/M, z18.s, z1.s\n" - "fmin z19.s, p5/M, z19.s, z1.s\n" - "fmin z24.s, p5/M, z24.s, z1.s\n" - "fmin z25.s, p5/M, z25.s, z1.s\n" - "fmin z26.s, p5/M, z26.s, z1.s\n" - "fmin z27.s, p5/M, z27.s, z1.s\n" - "fmax z7.s, p5/M, z7.s, z0.s\n" - "fmax z12.s, p5/M, z12.s, z0.s\n" - "fmax z13.s, p5/M, z13.s, z0.s\n" - "fmax z14.s, p5/M, z14.s, z0.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" - "fmax z15.s, p5/M, z15.s, z0.s\n" - "fmax z20.s, p5/M, z20.s, z0.s\n" - "fmax z21.s, p5/M, z21.s, z0.s\n" - "fmax z22.s, p5/M, z22.s, z0.s\n" - "fmax z16.s, p5/M, z16.s, z0.s\n" - "fmax z17.s, p5/M, z17.s, z0.s\n" - "fmax z18.s, p5/M, z18.s, z0.s\n" - "fmax z19.s, p5/M, z19.s, z0.s\n" - "fmax z24.s, p5/M, z24.s, z0.s\n" - "fmax z25.s, p5/M, z25.s, z0.s\n" - "fmax z26.s, p5/M, z26.s, z0.s\n" - "fmax z27.s, p5/M, z27.s, z0.s\n" + "add x20, %x[args_ptr], %[offset_min]\n" + "ld1rw { z23.s }, p5/Z, [x20]\n" + "fmin z7.s, p5/M, z7.s, z0.s\n" + "fmin z12.s, p5/M, z12.s, z0.s\n" + "fmin z13.s, p5/M, z13.s, z0.s\n" + "fmin z14.s, p5/M, z14.s, z0.s\n" + "fmin z8.s, p5/M, z8.s, z0.s\n" + "fmin z9.s, p5/M, z9.s, z0.s\n" + "fmin z10.s, p5/M, z10.s, z0.s\n" + "fmin z11.s, p5/M, z11.s, z0.s\n" + "fmin z15.s, p5/M, z15.s, z0.s\n" + "fmin z20.s, p5/M, z20.s, z0.s\n" + "fmin z21.s, p5/M, z21.s, z0.s\n" + "fmin z22.s, p5/M, z22.s, z0.s\n" + "fmin z16.s, p5/M, z16.s, z0.s\n" + "fmin z17.s, p5/M, z17.s, z0.s\n" + "fmin z18.s, p5/M, z18.s, z0.s\n" + "fmin z19.s, p5/M, z19.s, z0.s\n" + "fmin z24.s, p5/M, z24.s, z0.s\n" + "fmin z25.s, p5/M, z25.s, z0.s\n" + "fmin z26.s, p5/M, z26.s, z0.s\n" + "fmin z27.s, p5/M, z27.s, z0.s\n" + "fmax z7.s, p5/M, z7.s, z23.s\n" + "fmax z12.s, p5/M, z12.s, z23.s\n" + "fmax z13.s, p5/M, z13.s, z23.s\n" + "fmax z14.s, p5/M, z14.s, z23.s\n" + "fmax z8.s, p5/M, z8.s, z23.s\n" + "fmax z9.s, p5/M, z9.s, z23.s\n" + "fmax z10.s, p5/M, z10.s, z23.s\n" + "fmax z11.s, p5/M, z11.s, z23.s\n" + "fmax z15.s, p5/M, z15.s, z23.s\n" + "fmax z20.s, p5/M, z20.s, z23.s\n" + "fmax z21.s, p5/M, z21.s, z23.s\n" + "fmax z22.s, p5/M, z22.s, z23.s\n" + "fmax z16.s, p5/M, z16.s, z23.s\n" + "fmax z17.s, p5/M, z17.s, z23.s\n" + "fmax z18.s, p5/M, z18.s, z23.s\n" + "fmax z19.s, p5/M, z19.s, z23.s\n" + "fmax z24.s, p5/M, z24.s, z23.s\n" + "fmax z25.s, p5/M, z25.s, z23.s\n" + "fmax z26.s, p5/M, z26.s, z23.s\n" + "fmax z27.s, p5/M, z27.s, z23.s\n" "69:" // Height 5: No activation "st1w { z7.s }, p4, [x13]\n" "st1w { z12.s }, p3, [x13, #1, MUL VL]\n" @@ -1795,59 +1795,59 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "74:" // Height 6: no bias "tbz %x[flags], #0, 75f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x13]\n" + "add x24, x13, x20, LSL #2\n" "add x23, x24, x20, LSL #2\n" + "ld1w { z17.s }, p4/Z, [x13]\n" "add x22, x23, x20, LSL #2\n" - "ld1w { z10.s }, p3/Z, [x13, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x13, #2, MUL VL]\n" "add x21, x22, x20, LSL #2\n" + "ld1w { z18.s }, p3/Z, [x13, #1, MUL VL]\n" + "ld1w { z20.s }, p2/Z, [x13, #2, MUL VL]\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z16.s }, p1/Z, [x13, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "zip1 z8.d, z9.d, z12.d\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "zip2 z12.d, z9.d, z12.d\n" - "zip1 z9.d, z10.d, z13.d\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "ld1w { z17.s }, p4/Z, [x24]\n" - "zip2 z13.d, z10.d, z13.d\n" - "zip1 z10.d, z11.d, z14.d\n" - "ld1w { z18.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z19.s }, p2/Z, [x24, #2, MUL VL]\n" - "zip2 z14.d, z11.d, z14.d\n" + "ld1w { z12.s }, p4/Z, [x24]\n" + "zip1 z8.d, z17.d, z12.d\n" + "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n" + "zip2 z12.d, z17.d, z12.d\n" + "zip1 z9.d, z18.d, z13.d\n" + "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n" + "ld1w { z17.s }, p4/Z, [x23]\n" + "zip2 z13.d, z18.d, z13.d\n" + "zip1 z10.d, z20.d, z14.d\n" + "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n" + "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n" + "zip2 z14.d, z20.d, z14.d\n" "zip1 z11.d, z16.d, z15.d\n" - "ld1w { z24.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x23]\n" + "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x22]\n" "zip2 z15.d, z16.d, z15.d\n" "zip1 z16.d, z17.d, z20.d\n" - "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n" + "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n" "zip2 z20.d, z17.d, z20.d\n" "zip1 z17.d, z18.d, z21.d\n" - "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n" - "ld1w { z25.s }, p4/Z, [x22]\n" + "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z25.s }, p4/Z, [x21]\n" "zip2 z21.d, z18.d, z21.d\n" "zip1 z18.d, z19.d, z22.d\n" - "ld1w { z26.s }, p3/Z, [x22, #1, MUL VL]\n" - "ld1w { z27.s }, p2/Z, [x22, #2, MUL VL]\n" + "ld1w { z26.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z27.s }, p2/Z, [x21, #2, MUL VL]\n" "zip2 z22.d, z19.d, z22.d\n" "zip1 z19.d, z24.d, z23.d\n" - "ld1w { z6.s }, p1/Z, [x22, #3, MUL VL]\n" - "ld1w { z28.s }, p4/Z, [x21]\n" + "ld1w { z0.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z28.s }, p4/Z, [x20]\n" "zip2 z23.d, z24.d, z23.d\n" "zip1 z24.d, z25.d, z28.d\n" - "ld1w { z29.s }, p3/Z, [x21, #1, MUL VL]\n" - "ld1w { z30.s }, p2/Z, [x21, #2, MUL VL]\n" + "ld1w { z29.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z30.s }, p2/Z, [x20, #2, MUL VL]\n" "zip2 z28.d, z25.d, z28.d\n" "zip1 z25.d, z26.d, z29.d\n" - "ld1w { z31.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z31.s }, p1/Z, [x20, #3, MUL VL]\n" "zip2 z29.d, z26.d, z29.d\n" "zip1 z26.d, z27.d, z30.d\n" "zip2 z30.d, z27.d, z30.d\n" - "zip1 z27.d, z6.d, z31.d\n" - "zip2 z31.d, z6.d, z31.d\n" + "zip1 z27.d, z0.d, z31.d\n" + "zip2 z31.d, z0.d, z31.d\n" "b 76f\n" "75:" // Height 6: no accumulate "mov z8.b, #0x0\n" @@ -1879,16 +1879,16 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "77:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 78f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 79f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -1900,193 +1900,193 @@ void sve_ffhybrid_bf16fp32_mmla_6x4VL ( "b 79f\n" "78:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" - "add x22, x23, x20, LSL #1\n" - "add x21, x22, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" + "add x22, x23, x21, LSL #1\n" + "add x21, x22, x21, LSL #1\n" "79:" // Height 6: input setup done "cmp x27, #0x8\n" "ble 81f\n" "80:" // Height 6: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z1.h }, p0/Z, [x26]\n" - "ld1rqh { z2.h }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1rqh { z3.h }, p0/Z, [x24]\n" - "ld1rqh { z4.h }, p0/Z, [x23]\n" - "trn2 z1.d, z1.d, z2.d\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1rqh { z5.h }, p0/Z, [x22]\n" - "ld1rqh { z6.h }, p0/Z, [x21]\n" - "trn2 z3.d, z3.d, z4.d\n" - "trn1 z4.d, z5.d, z6.d\n" - "trn2 z5.d, z5.d, z6.d\n" - "ld1h { z7.h }, p5/Z, [x12]\n" - "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n" - ".inst 0x6467e408 // bfmmla z8.s, z0.h, z7.h\n" - ".inst 0x6467e450 // bfmmla z16.s, z2.h, z7.h\n" - ".inst 0x6467e498 // bfmmla z24.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x11]\n" + "ld1rqh { z7.h }, p0/Z, [x26]\n" + "ld1rqh { z0.h }, p0/Z, [x25]\n" + "trn1 z6.d, z7.d, z0.d\n" + "ld1rqh { z5.h }, p0/Z, [x24]\n" + "ld1rqh { z1.h }, p0/Z, [x23]\n" + "trn2 z7.d, z7.d, z0.d\n" + "trn1 z4.d, z5.d, z1.d\n" + "ld1rqh { z3.h }, p0/Z, [x22]\n" + "ld1rqh { z0.h }, p0/Z, [x21]\n" + "trn2 z5.d, z5.d, z1.d\n" + "trn1 z2.d, z3.d, z0.d\n" + "trn2 z3.d, z3.d, z0.d\n" + "ld1h { z1.h }, p5/Z, [x12]\n" + "ld1h { z0.h }, p5/Z, [x12, #1, MUL VL]\n" + ".inst 0x6461e4c8 // bfmmla z8.s, z6.h, z1.h\n" + ".inst 0x6461e490 // bfmmla z16.s, z4.h, z1.h\n" + ".inst 0x6461e458 // bfmmla z24.s, z2.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x11]\n" "sub x27, x27, #0x8\n" - ".inst 0x6466e40c // bfmmla z12.s, z0.h, z6.h\n" - ".inst 0x6466e454 // bfmmla z20.s, z2.h, z6.h\n" + ".inst 0x6460e4cc // bfmmla z12.s, z6.h, z0.h\n" + ".inst 0x6460e494 // bfmmla z20.s, z4.h, z0.h\n" "cmp x27, #0x8\n" "add x26, x26, #0x10\n" - ".inst 0x6466e49c // bfmmla z28.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n" - ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" + ".inst 0x6460e45c // bfmmla z28.s, z2.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x11, #1, MUL VL]\n" + ".inst 0x6461e4c9 // bfmmla z9.s, z6.h, z1.h\n" "add x25, x25, #0x10\n" - ".inst 0x6467e451 // bfmmla z17.s, z2.h, z7.h\n" - ".inst 0x6467e499 // bfmmla z25.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10]\n" + ".inst 0x6461e491 // bfmmla z17.s, z4.h, z1.h\n" + ".inst 0x6461e459 // bfmmla z25.s, z2.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x10]\n" "add x24, x24, #0x10\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" + ".inst 0x6460e4cd // bfmmla z13.s, z6.h, z0.h\n" + ".inst 0x6460e495 // bfmmla z21.s, z4.h, z0.h\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - ".inst 0x6466e49d // bfmmla z29.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e40a // bfmmla z10.s, z0.h, z7.h\n" + ".inst 0x6460e45d // bfmmla z29.s, z2.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6461e4ca // bfmmla z10.s, z6.h, z1.h\n" "add x21, x21, #0x10\n" - ".inst 0x6467e452 // bfmmla z18.s, z2.h, z7.h\n" - ".inst 0x6467e49a // bfmmla z26.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x9]\n" - ".inst 0x6466e40e // bfmmla z14.s, z0.h, z6.h\n" - ".inst 0x6466e456 // bfmmla z22.s, z2.h, z6.h\n" - ".inst 0x6466e49e // bfmmla z30.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" - ".inst 0x6467e453 // bfmmla z19.s, z2.h, z7.h\n" - ".inst 0x6467e49b // bfmmla z27.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x12, #2, MUL VL]\n" - ".inst 0x6466e40f // bfmmla z15.s, z0.h, z6.h\n" - ".inst 0x6466e457 // bfmmla z23.s, z2.h, z6.h\n" - ".inst 0x6466e49f // bfmmla z31.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x12, #3, MUL VL]\n" - ".inst 0x6467e428 // bfmmla z8.s, z1.h, z7.h\n" + ".inst 0x6461e492 // bfmmla z18.s, z4.h, z1.h\n" + ".inst 0x6461e45a // bfmmla z26.s, z2.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x9]\n" + ".inst 0x6460e4ce // bfmmla z14.s, z6.h, z0.h\n" + ".inst 0x6460e496 // bfmmla z22.s, z4.h, z0.h\n" + ".inst 0x6460e45e // bfmmla z30.s, z2.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x9, #1, MUL VL]\n" + ".inst 0x6461e4cb // bfmmla z11.s, z6.h, z1.h\n" + ".inst 0x6461e493 // bfmmla z19.s, z4.h, z1.h\n" + ".inst 0x6461e45b // bfmmla z27.s, z2.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x12, #2, MUL VL]\n" + ".inst 0x6460e4cf // bfmmla z15.s, z6.h, z0.h\n" + ".inst 0x6460e497 // bfmmla z23.s, z4.h, z0.h\n" + ".inst 0x6460e45f // bfmmla z31.s, z2.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x12, #3, MUL VL]\n" + ".inst 0x6461e4e8 // bfmmla z8.s, z7.h, z1.h\n" "addvl x12, x12, #4\n" - ".inst 0x6467e470 // bfmmla z16.s, z3.h, z7.h\n" - ".inst 0x6467e4b8 // bfmmla z24.s, z5.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n" - ".inst 0x6466e42c // bfmmla z12.s, z1.h, z6.h\n" - ".inst 0x6466e474 // bfmmla z20.s, z3.h, z6.h\n" - ".inst 0x6466e4bc // bfmmla z28.s, z5.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x11, #3, MUL VL]\n" - ".inst 0x6467e429 // bfmmla z9.s, z1.h, z7.h\n" + ".inst 0x6461e4b0 // bfmmla z16.s, z5.h, z1.h\n" + ".inst 0x6461e478 // bfmmla z24.s, z3.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x11, #2, MUL VL]\n" + ".inst 0x6460e4ec // bfmmla z12.s, z7.h, z0.h\n" + ".inst 0x6460e4b4 // bfmmla z20.s, z5.h, z0.h\n" + ".inst 0x6460e47c // bfmmla z28.s, z3.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x11, #3, MUL VL]\n" + ".inst 0x6461e4e9 // bfmmla z9.s, z7.h, z1.h\n" "addvl x11, x11, #4\n" - ".inst 0x6467e471 // bfmmla z17.s, z3.h, z7.h\n" - ".inst 0x6467e4b9 // bfmmla z25.s, z5.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x6466e42d // bfmmla z13.s, z1.h, z6.h\n" - ".inst 0x6466e475 // bfmmla z21.s, z3.h, z6.h\n" - ".inst 0x6466e4bd // bfmmla z29.s, z5.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6467e42a // bfmmla z10.s, z1.h, z7.h\n" + ".inst 0x6461e4b1 // bfmmla z17.s, z5.h, z1.h\n" + ".inst 0x6461e479 // bfmmla z25.s, z3.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x6460e4ed // bfmmla z13.s, z7.h, z0.h\n" + ".inst 0x6460e4b5 // bfmmla z21.s, z5.h, z0.h\n" + ".inst 0x6460e47d // bfmmla z29.s, z3.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6461e4ea // bfmmla z10.s, z7.h, z1.h\n" "addvl x10, x10, #4\n" - ".inst 0x6467e472 // bfmmla z18.s, z3.h, z7.h\n" - ".inst 0x6467e4ba // bfmmla z26.s, z5.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - ".inst 0x6466e476 // bfmmla z22.s, z3.h, z6.h\n" - ".inst 0x6466e4be // bfmmla z30.s, z5.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x9, #3, MUL VL]\n" - ".inst 0x6467e42b // bfmmla z11.s, z1.h, z7.h\n" + ".inst 0x6461e4b2 // bfmmla z18.s, z5.h, z1.h\n" + ".inst 0x6461e47a // bfmmla z26.s, z3.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x9, #2, MUL VL]\n" + ".inst 0x6460e4ee // bfmmla z14.s, z7.h, z0.h\n" + ".inst 0x6460e4b6 // bfmmla z22.s, z5.h, z0.h\n" + ".inst 0x6460e47e // bfmmla z30.s, z3.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x9, #3, MUL VL]\n" + ".inst 0x6461e4eb // bfmmla z11.s, z7.h, z1.h\n" "addvl x9, x9, #4\n" - ".inst 0x6467e473 // bfmmla z19.s, z3.h, z7.h\n" - ".inst 0x6467e4bb // bfmmla z27.s, z5.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" - ".inst 0x6466e477 // bfmmla z23.s, z3.h, z6.h\n" - ".inst 0x6466e4bf // bfmmla z31.s, z5.h, z6.h\n" + ".inst 0x6461e4b3 // bfmmla z19.s, z5.h, z1.h\n" + ".inst 0x6461e47b // bfmmla z27.s, z3.h, z1.h\n" + ".inst 0x6460e4ef // bfmmla z15.s, z7.h, z0.h\n" + ".inst 0x6460e4b7 // bfmmla z23.s, z5.h, z0.h\n" + ".inst 0x6460e47f // bfmmla z31.s, z3.h, z0.h\n" "bgt 80b\n" "81:" // Height 6: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" "ld1rqh { z1.h }, p0/Z, [x26]\n" - "ld1rqh { z2.h }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" + "ld1rqh { z0.h }, p0/Z, [x25]\n" + "trn1 z7.d, z1.d, z0.d\n" "ld1rqh { z3.h }, p0/Z, [x24]\n" - "ld1rqh { z4.h }, p0/Z, [x23]\n" - "trn2 z1.d, z1.d, z2.d\n" - "trn1 z2.d, z3.d, z4.d\n" + "ld1rqh { z2.h }, p0/Z, [x23]\n" + "trn2 z1.d, z1.d, z0.d\n" + "trn1 z6.d, z3.d, z2.d\n" "ld1rqh { z5.h }, p0/Z, [x22]\n" - "ld1rqh { z6.h }, p0/Z, [x21]\n" - "trn2 z3.d, z3.d, z4.d\n" - "trn1 z4.d, z5.d, z6.d\n" - "trn2 z5.d, z5.d, z6.d\n" - "ld1h { z7.h }, p5/Z, [x12]\n" - "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n" - ".inst 0x6467e408 // bfmmla z8.s, z0.h, z7.h\n" - ".inst 0x6467e450 // bfmmla z16.s, z2.h, z7.h\n" - ".inst 0x6467e498 // bfmmla z24.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x11]\n" + "ld1rqh { z0.h }, p0/Z, [x21]\n" + "trn2 z3.d, z3.d, z2.d\n" + "trn1 z4.d, z5.d, z0.d\n" + "trn2 z5.d, z5.d, z0.d\n" + "ld1h { z2.h }, p5/Z, [x12]\n" + "ld1h { z0.h }, p5/Z, [x12, #1, MUL VL]\n" + ".inst 0x6462e4e8 // bfmmla z8.s, z7.h, z2.h\n" + ".inst 0x6462e4d0 // bfmmla z16.s, z6.h, z2.h\n" + ".inst 0x6462e498 // bfmmla z24.s, z4.h, z2.h\n" + "ld1h { z2.h }, p5/Z, [x11]\n" "subs x27, x27, #0x4\n" - ".inst 0x6466e40c // bfmmla z12.s, z0.h, z6.h\n" - ".inst 0x6466e454 // bfmmla z20.s, z2.h, z6.h\n" + ".inst 0x6460e4ec // bfmmla z12.s, z7.h, z0.h\n" + ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n" "addvl x12, x12, #2\n" - ".inst 0x6466e49c // bfmmla z28.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n" - ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" + ".inst 0x6460e49c // bfmmla z28.s, z4.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x11, #1, MUL VL]\n" + ".inst 0x6462e4e9 // bfmmla z9.s, z7.h, z2.h\n" "addvl x11, x11, #2\n" - ".inst 0x6467e451 // bfmmla z17.s, z2.h, z7.h\n" - ".inst 0x6467e499 // bfmmla z25.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" - ".inst 0x6466e49d // bfmmla z29.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e40a // bfmmla z10.s, z0.h, z7.h\n" + ".inst 0x6462e4d1 // bfmmla z17.s, z6.h, z2.h\n" + ".inst 0x6462e499 // bfmmla z25.s, z4.h, z2.h\n" + "ld1h { z2.h }, p5/Z, [x10]\n" + ".inst 0x6460e4ed // bfmmla z13.s, z7.h, z0.h\n" + ".inst 0x6460e4d5 // bfmmla z21.s, z6.h, z0.h\n" + ".inst 0x6460e49d // bfmmla z29.s, z4.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6462e4ea // bfmmla z10.s, z7.h, z2.h\n" "addvl x10, x10, #2\n" - ".inst 0x6467e452 // bfmmla z18.s, z2.h, z7.h\n" - ".inst 0x6467e49a // bfmmla z26.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x9]\n" - ".inst 0x6466e40e // bfmmla z14.s, z0.h, z6.h\n" - ".inst 0x6466e456 // bfmmla z22.s, z2.h, z6.h\n" - ".inst 0x6466e49e // bfmmla z30.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" + ".inst 0x6462e4d2 // bfmmla z18.s, z6.h, z2.h\n" + ".inst 0x6462e49a // bfmmla z26.s, z4.h, z2.h\n" + "ld1h { z2.h }, p5/Z, [x9]\n" + ".inst 0x6460e4ee // bfmmla z14.s, z7.h, z0.h\n" + ".inst 0x6460e4d6 // bfmmla z22.s, z6.h, z0.h\n" + ".inst 0x6460e49e // bfmmla z30.s, z4.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x9, #1, MUL VL]\n" + ".inst 0x6462e4eb // bfmmla z11.s, z7.h, z2.h\n" "addvl x9, x9, #2\n" - ".inst 0x6467e453 // bfmmla z19.s, z2.h, z7.h\n" - ".inst 0x6467e49b // bfmmla z27.s, z4.h, z7.h\n" - ".inst 0x6466e40f // bfmmla z15.s, z0.h, z6.h\n" - ".inst 0x6466e457 // bfmmla z23.s, z2.h, z6.h\n" - ".inst 0x6466e49f // bfmmla z31.s, z4.h, z6.h\n" + ".inst 0x6462e4d3 // bfmmla z19.s, z6.h, z2.h\n" + ".inst 0x6462e49b // bfmmla z27.s, z4.h, z2.h\n" + ".inst 0x6460e4ef // bfmmla z15.s, z7.h, z0.h\n" + ".inst 0x6460e4d7 // bfmmla z23.s, z6.h, z0.h\n" + ".inst 0x6460e49f // bfmmla z31.s, z4.h, z0.h\n" "ble 82f\n" - "ld1h { z7.h }, p5/Z, [x12]\n" - "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n" - ".inst 0x6467e428 // bfmmla z8.s, z1.h, z7.h\n" - ".inst 0x6467e470 // bfmmla z16.s, z3.h, z7.h\n" - ".inst 0x6467e4b8 // bfmmla z24.s, z5.h, z7.h\n" - ".inst 0x6466e42c // bfmmla z12.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x11]\n" + "ld1h { z2.h }, p5/Z, [x12]\n" + "ld1h { z0.h }, p5/Z, [x12, #1, MUL VL]\n" + ".inst 0x6462e428 // bfmmla z8.s, z1.h, z2.h\n" + ".inst 0x6462e470 // bfmmla z16.s, z3.h, z2.h\n" + ".inst 0x6462e4b8 // bfmmla z24.s, z5.h, z2.h\n" + ".inst 0x6460e42c // bfmmla z12.s, z1.h, z0.h\n" + "ld1h { z2.h }, p5/Z, [x11]\n" "addvl x12, x12, #2\n" - ".inst 0x6466e474 // bfmmla z20.s, z3.h, z6.h\n" - ".inst 0x6466e4bc // bfmmla z28.s, z5.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x11, #1, MUL VL]\n" + ".inst 0x6460e474 // bfmmla z20.s, z3.h, z0.h\n" + ".inst 0x6460e4bc // bfmmla z28.s, z5.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x11, #1, MUL VL]\n" "addvl x11, x11, #2\n" - ".inst 0x6467e429 // bfmmla z9.s, z1.h, z7.h\n" - ".inst 0x6467e471 // bfmmla z17.s, z3.h, z7.h\n" - ".inst 0x6467e4b9 // bfmmla z25.s, z5.h, z7.h\n" - ".inst 0x6466e42d // bfmmla z13.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - ".inst 0x6466e475 // bfmmla z21.s, z3.h, z6.h\n" - ".inst 0x6466e4bd // bfmmla z29.s, z5.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6462e429 // bfmmla z9.s, z1.h, z2.h\n" + ".inst 0x6462e471 // bfmmla z17.s, z3.h, z2.h\n" + ".inst 0x6462e4b9 // bfmmla z25.s, z5.h, z2.h\n" + ".inst 0x6460e42d // bfmmla z13.s, z1.h, z0.h\n" + "ld1h { z2.h }, p5/Z, [x10]\n" + ".inst 0x6460e475 // bfmmla z21.s, z3.h, z0.h\n" + ".inst 0x6460e4bd // bfmmla z29.s, z5.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #1, MUL VL]\n" "addvl x10, x10, #2\n" - ".inst 0x6467e42a // bfmmla z10.s, z1.h, z7.h\n" - ".inst 0x6467e472 // bfmmla z18.s, z3.h, z7.h\n" - ".inst 0x6467e4ba // bfmmla z26.s, z5.h, z7.h\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x9]\n" - ".inst 0x6466e476 // bfmmla z22.s, z3.h, z6.h\n" - ".inst 0x6466e4be // bfmmla z30.s, z5.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x9, #1, MUL VL]\n" + ".inst 0x6462e42a // bfmmla z10.s, z1.h, z2.h\n" + ".inst 0x6462e472 // bfmmla z18.s, z3.h, z2.h\n" + ".inst 0x6462e4ba // bfmmla z26.s, z5.h, z2.h\n" + ".inst 0x6460e42e // bfmmla z14.s, z1.h, z0.h\n" + "ld1h { z2.h }, p5/Z, [x9]\n" + ".inst 0x6460e476 // bfmmla z22.s, z3.h, z0.h\n" + ".inst 0x6460e4be // bfmmla z30.s, z5.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x9, #1, MUL VL]\n" "addvl x9, x9, #2\n" - ".inst 0x6467e42b // bfmmla z11.s, z1.h, z7.h\n" - ".inst 0x6467e473 // bfmmla z19.s, z3.h, z7.h\n" - ".inst 0x6467e4bb // bfmmla z27.s, z5.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" - ".inst 0x6466e477 // bfmmla z23.s, z3.h, z6.h\n" - ".inst 0x6466e4bf // bfmmla z31.s, z5.h, z6.h\n" + ".inst 0x6462e42b // bfmmla z11.s, z1.h, z2.h\n" + ".inst 0x6462e473 // bfmmla z19.s, z3.h, z2.h\n" + ".inst 0x6462e4bb // bfmmla z27.s, z5.h, z2.h\n" + ".inst 0x6460e42f // bfmmla z15.s, z1.h, z0.h\n" + ".inst 0x6460e477 // bfmmla z23.s, z3.h, z0.h\n" + ".inst 0x6460e4bf // bfmmla z31.s, z5.h, z0.h\n" "82:" // Height 6: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp16_mla_6x4VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp16_mla_6x4VL.hpp index acbc619eedfd66da0c521b9d88e1e510a71f3db7..c42ad7e879ba94930bf5d15dac75229f03327eef 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp16_mla_6x4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp16_mla_6x4VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,16 +10,16 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once #ifdef ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp16_mla_6x4VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp16_mla_6x4VL/a64fx.cpp index 5f093bf08a5cc12b0bfd8b4504a3bcd676c582a3..66601bd31241d20a3c2ace27244f5ea07682de4a 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp16_mla_6x4VL/a64fx.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp16_mla_6x4VL/a64fx.cpp @@ -163,11 +163,11 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "7:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 8f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 9f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -183,12 +183,12 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "10:" // Height 1: Multiply loop: Main loop "fmla z8.h, p4/M, z6.h, z0.h\n" "fmla z9.h, p4/M, z7.h, z0.h\n" - "ld1h { z6.h }, p4/Z, [x10]\n" - "ld1h { z7.h }, p4/Z, [x9]\n" + "ld1h { z17.h }, p4/Z, [x10]\n" + "ld1h { z16.h }, p4/Z, [x9]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" - "fmla z10.h, p4/M, z6.h, z0.h\n" - "fmla z11.h, p4/M, z7.h, z0.h\n" + "fmla z10.h, p4/M, z17.h, z0.h\n" + "fmla z11.h, p4/M, z16.h, z0.h\n" "add x26, x26, #0x2\n" "subs x27, x27, #0x1\n" "ld1rh { z0.h }, p4/Z, [x26]\n" @@ -201,12 +201,12 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "fmla z8.h, p4/M, z6.h, z0.h\n" "fmla z9.h, p4/M, z7.h, z0.h\n" - "ld1h { z6.h }, p4/Z, [x10]\n" - "ld1h { z7.h }, p4/Z, [x9]\n" + "ld1h { z17.h }, p4/Z, [x10]\n" + "ld1h { z16.h }, p4/Z, [x9]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" - "fmla z10.h, p4/M, z6.h, z0.h\n" - "fmla z11.h, p4/M, z7.h, z0.h\n" + "fmla z10.h, p4/M, z17.h, z0.h\n" + "fmla z11.h, p4/M, z16.h, z0.h\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" @@ -214,17 +214,17 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "bne 7b\n" "tbz %x[flags], #1, 12f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rh { z1.h }, p4/Z, [x20]\n" + "ld1rh { z17.h }, p4/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rh { z0.h }, p4/Z, [x20]\n" - "fmin z8.h, p4/M, z8.h, z1.h\n" - "fmin z9.h, p4/M, z9.h, z1.h\n" - "fmin z10.h, p4/M, z10.h, z1.h\n" - "fmin z11.h, p4/M, z11.h, z1.h\n" - "fmax z8.h, p4/M, z8.h, z0.h\n" - "fmax z9.h, p4/M, z9.h, z0.h\n" - "fmax z10.h, p4/M, z10.h, z0.h\n" - "fmax z11.h, p4/M, z11.h, z0.h\n" + "ld1rh { z16.h }, p4/Z, [x20]\n" + "fmin z8.h, p4/M, z8.h, z17.h\n" + "fmin z9.h, p4/M, z9.h, z17.h\n" + "fmin z10.h, p4/M, z10.h, z17.h\n" + "fmin z11.h, p4/M, z11.h, z17.h\n" + "fmax z8.h, p4/M, z8.h, z16.h\n" + "fmax z9.h, p4/M, z9.h, z16.h\n" + "fmax z10.h, p4/M, z10.h, z16.h\n" + "fmax z11.h, p4/M, z11.h, z16.h\n" "12:" // Height 1: No activation "st1h { z8.h }, p3, [x13]\n" "st1h { z9.h }, p2, [x13, #1, MUL VL]\n" @@ -285,15 +285,15 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "17:" // Height 2: no bias "tbz %x[flags], #0, 18f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #1\n" + "add x20, x13, x20, LSL #1\n" "ld1h { z8.h }, p3/Z, [x13]\n" "ld1h { z9.h }, p2/Z, [x13, #1, MUL VL]\n" "ld1h { z10.h }, p1/Z, [x13, #2, MUL VL]\n" "ld1h { z11.h }, p0/Z, [x13, #3, MUL VL]\n" - "ld1h { z12.h }, p3/Z, [x25]\n" - "ld1h { z13.h }, p2/Z, [x25, #1, MUL VL]\n" - "ld1h { z14.h }, p1/Z, [x25, #2, MUL VL]\n" - "ld1h { z15.h }, p0/Z, [x25, #3, MUL VL]\n" + "ld1h { z12.h }, p3/Z, [x20]\n" + "ld1h { z13.h }, p2/Z, [x20, #1, MUL VL]\n" + "ld1h { z14.h }, p1/Z, [x20, #2, MUL VL]\n" + "ld1h { z15.h }, p0/Z, [x20, #3, MUL VL]\n" "b 19f\n" "18:" // Height 2: no accumulate "mov z8.b, #0x0\n" @@ -309,12 +309,12 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "20:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 21f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 22f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -322,7 +322,7 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "b 22f\n" "21:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" "22:" // Height 2: input setup done "subs x27, x27, #0x1\n" "ld1rh { z0.h }, p4/Z, [x26]\n" @@ -333,19 +333,19 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "23:" // Height 2: Multiply loop: Main loop "fmla z8.h, p4/M, z6.h, z0.h\n" "fmla z12.h, p4/M, z6.h, z1.h\n" - "ld1h { z6.h }, p4/Z, [x10]\n" + "ld1h { z17.h }, p4/Z, [x10]\n" "addvl x12, x12, #1\n" "fmla z9.h, p4/M, z7.h, z0.h\n" "fmla z13.h, p4/M, z7.h, z1.h\n" - "ld1h { z7.h }, p4/Z, [x9]\n" + "ld1h { z16.h }, p4/Z, [x9]\n" "addvl x11, x11, #1\n" "add x26, x26, #0x2\n" "subs x27, x27, #0x1\n" - "fmla z10.h, p4/M, z6.h, z0.h\n" - "fmla z14.h, p4/M, z6.h, z1.h\n" + "fmla z10.h, p4/M, z17.h, z0.h\n" + "fmla z14.h, p4/M, z17.h, z1.h\n" "add x25, x25, #0x2\n" - "fmla z11.h, p4/M, z7.h, z0.h\n" - "fmla z15.h, p4/M, z7.h, z1.h\n" + "fmla z11.h, p4/M, z16.h, z0.h\n" + "fmla z15.h, p4/M, z16.h, z1.h\n" "addvl x10, x10, #1\n" "ld1rh { z0.h }, p4/Z, [x26]\n" "ld1rh { z1.h }, p4/Z, [x25]\n" @@ -357,18 +357,18 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "fmla z8.h, p4/M, z6.h, z0.h\n" "fmla z12.h, p4/M, z6.h, z1.h\n" - "ld1h { z6.h }, p4/Z, [x10]\n" + "ld1h { z17.h }, p4/Z, [x10]\n" "fmla z9.h, p4/M, z7.h, z0.h\n" "fmla z13.h, p4/M, z7.h, z1.h\n" - "ld1h { z7.h }, p4/Z, [x9]\n" + "ld1h { z16.h }, p4/Z, [x9]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" - "fmla z10.h, p4/M, z6.h, z0.h\n" - "fmla z14.h, p4/M, z6.h, z1.h\n" + "fmla z10.h, p4/M, z17.h, z0.h\n" + "fmla z14.h, p4/M, z17.h, z1.h\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" - "fmla z11.h, p4/M, z7.h, z0.h\n" - "fmla z15.h, p4/M, z7.h, z1.h\n" + "fmla z11.h, p4/M, z16.h, z0.h\n" + "fmla z15.h, p4/M, z16.h, z1.h\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" "bne 20b\n" @@ -376,25 +376,25 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "add x25, x13, x20, LSL #1\n" "tbz %x[flags], #1, 25f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rh { z1.h }, p4/Z, [x20]\n" + "ld1rh { z17.h }, p4/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rh { z0.h }, p4/Z, [x20]\n" - "fmin z8.h, p4/M, z8.h, z1.h\n" - "fmin z9.h, p4/M, z9.h, z1.h\n" - "fmin z10.h, p4/M, z10.h, z1.h\n" - "fmin z11.h, p4/M, z11.h, z1.h\n" - "fmin z12.h, p4/M, z12.h, z1.h\n" - "fmin z13.h, p4/M, z13.h, z1.h\n" - "fmin z14.h, p4/M, z14.h, z1.h\n" - "fmin z15.h, p4/M, z15.h, z1.h\n" - "fmax z8.h, p4/M, z8.h, z0.h\n" - "fmax z9.h, p4/M, z9.h, z0.h\n" - "fmax z10.h, p4/M, z10.h, z0.h\n" - "fmax z11.h, p4/M, z11.h, z0.h\n" - "fmax z12.h, p4/M, z12.h, z0.h\n" - "fmax z13.h, p4/M, z13.h, z0.h\n" - "fmax z14.h, p4/M, z14.h, z0.h\n" - "fmax z15.h, p4/M, z15.h, z0.h\n" + "ld1rh { z16.h }, p4/Z, [x20]\n" + "fmin z8.h, p4/M, z8.h, z17.h\n" + "fmin z9.h, p4/M, z9.h, z17.h\n" + "fmin z10.h, p4/M, z10.h, z17.h\n" + "fmin z11.h, p4/M, z11.h, z17.h\n" + "fmin z12.h, p4/M, z12.h, z17.h\n" + "fmin z13.h, p4/M, z13.h, z17.h\n" + "fmin z14.h, p4/M, z14.h, z17.h\n" + "fmin z15.h, p4/M, z15.h, z17.h\n" + "fmax z8.h, p4/M, z8.h, z16.h\n" + "fmax z9.h, p4/M, z9.h, z16.h\n" + "fmax z10.h, p4/M, z10.h, z16.h\n" + "fmax z11.h, p4/M, z11.h, z16.h\n" + "fmax z12.h, p4/M, z12.h, z16.h\n" + "fmax z13.h, p4/M, z13.h, z16.h\n" + "fmax z14.h, p4/M, z14.h, z16.h\n" + "fmax z15.h, p4/M, z15.h, z16.h\n" "25:" // Height 2: No activation "st1h { z8.h }, p3, [x13]\n" "st1h { z9.h }, p2, [x13, #1, MUL VL]\n" @@ -463,20 +463,20 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "30:" // Height 3: no bias "tbz %x[flags], #0, 31f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" + "add x21, x13, x20, LSL #1\n" + "add x20, x21, x20, LSL #1\n" "ld1h { z8.h }, p3/Z, [x13]\n" "ld1h { z9.h }, p2/Z, [x13, #1, MUL VL]\n" "ld1h { z10.h }, p1/Z, [x13, #2, MUL VL]\n" "ld1h { z11.h }, p0/Z, [x13, #3, MUL VL]\n" - "ld1h { z12.h }, p3/Z, [x25]\n" - "ld1h { z13.h }, p2/Z, [x25, #1, MUL VL]\n" - "ld1h { z14.h }, p1/Z, [x25, #2, MUL VL]\n" - "ld1h { z15.h }, p0/Z, [x25, #3, MUL VL]\n" - "ld1h { z16.h }, p3/Z, [x24]\n" - "ld1h { z17.h }, p2/Z, [x24, #1, MUL VL]\n" - "ld1h { z18.h }, p1/Z, [x24, #2, MUL VL]\n" - "ld1h { z19.h }, p0/Z, [x24, #3, MUL VL]\n" + "ld1h { z12.h }, p3/Z, [x21]\n" + "ld1h { z13.h }, p2/Z, [x21, #1, MUL VL]\n" + "ld1h { z14.h }, p1/Z, [x21, #2, MUL VL]\n" + "ld1h { z15.h }, p0/Z, [x21, #3, MUL VL]\n" + "ld1h { z16.h }, p3/Z, [x20]\n" + "ld1h { z17.h }, p2/Z, [x20, #1, MUL VL]\n" + "ld1h { z18.h }, p1/Z, [x20, #2, MUL VL]\n" + "ld1h { z19.h }, p0/Z, [x20, #3, MUL VL]\n" "b 32f\n" "31:" // Height 3: no accumulate "mov z8.b, #0x0\n" @@ -496,13 +496,13 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "33:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 34f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 35f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -511,8 +511,8 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "b 35f\n" "34:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" "35:" // Height 3: input setup done "subs x27, x27, #0x1\n" "ld1rh { z0.h }, p4/Z, [x26]\n" @@ -528,22 +528,22 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "addvl x11, x11, #1\n" "fmla z16.h, p4/M, z6.h, z2.h\n" "fmla z9.h, p4/M, z7.h, z0.h\n" - "ld1h { z6.h }, p4/Z, [x10]\n" + "ld1h { z21.h }, p4/Z, [x10]\n" "add x26, x26, #0x2\n" "fmla z13.h, p4/M, z7.h, z1.h\n" "fmla z17.h, p4/M, z7.h, z2.h\n" - "ld1h { z7.h }, p4/Z, [x9]\n" + "ld1h { z20.h }, p4/Z, [x9]\n" "subs x27, x27, #0x1\n" "add x25, x25, #0x2\n" "add x24, x24, #0x2\n" - "fmla z10.h, p4/M, z6.h, z0.h\n" - "fmla z14.h, p4/M, z6.h, z1.h\n" - "fmla z18.h, p4/M, z6.h, z2.h\n" - "fmla z11.h, p4/M, z7.h, z0.h\n" + "fmla z10.h, p4/M, z21.h, z0.h\n" + "fmla z14.h, p4/M, z21.h, z1.h\n" + "fmla z18.h, p4/M, z21.h, z2.h\n" + "fmla z11.h, p4/M, z20.h, z0.h\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" - "fmla z15.h, p4/M, z7.h, z1.h\n" - "fmla z19.h, p4/M, z7.h, z2.h\n" + "fmla z15.h, p4/M, z20.h, z1.h\n" + "fmla z19.h, p4/M, z20.h, z2.h\n" "ld1rh { z0.h }, p4/Z, [x26]\n" "ld1rh { z1.h }, p4/Z, [x25]\n" "ld1rh { z2.h }, p4/Z, [x24]\n" @@ -557,54 +557,54 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "add x28, x28, #0x1\n" "fmla z16.h, p4/M, z6.h, z2.h\n" "fmla z9.h, p4/M, z7.h, z0.h\n" - "ld1h { z6.h }, p4/Z, [x10]\n" + "ld1h { z21.h }, p4/Z, [x10]\n" "cmp x28, x20\n" "fmla z13.h, p4/M, z7.h, z1.h\n" "fmla z17.h, p4/M, z7.h, z2.h\n" - "ld1h { z7.h }, p4/Z, [x9]\n" + "ld1h { z20.h }, p4/Z, [x9]\n" "addvl x12, x12, #1\n" - "fmla z10.h, p4/M, z6.h, z0.h\n" - "fmla z14.h, p4/M, z6.h, z1.h\n" + "fmla z10.h, p4/M, z21.h, z0.h\n" + "fmla z14.h, p4/M, z21.h, z1.h\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" - "fmla z18.h, p4/M, z6.h, z2.h\n" - "fmla z11.h, p4/M, z7.h, z0.h\n" + "fmla z18.h, p4/M, z21.h, z2.h\n" + "fmla z11.h, p4/M, z20.h, z0.h\n" "addvl x9, x9, #1\n" - "fmla z15.h, p4/M, z7.h, z1.h\n" - "fmla z19.h, p4/M, z7.h, z2.h\n" + "fmla z15.h, p4/M, z20.h, z1.h\n" + "fmla z19.h, p4/M, z20.h, z2.h\n" "bne 33b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "add x25, x13, x20, LSL #1\n" "add x24, x25, x20, LSL #1\n" "tbz %x[flags], #1, 38f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rh { z1.h }, p4/Z, [x20]\n" + "ld1rh { z21.h }, p4/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rh { z0.h }, p4/Z, [x20]\n" - "fmin z8.h, p4/M, z8.h, z1.h\n" - "fmin z9.h, p4/M, z9.h, z1.h\n" - "fmin z10.h, p4/M, z10.h, z1.h\n" - "fmin z11.h, p4/M, z11.h, z1.h\n" - "fmin z12.h, p4/M, z12.h, z1.h\n" - "fmin z13.h, p4/M, z13.h, z1.h\n" - "fmin z14.h, p4/M, z14.h, z1.h\n" - "fmin z15.h, p4/M, z15.h, z1.h\n" - "fmin z16.h, p4/M, z16.h, z1.h\n" - "fmin z17.h, p4/M, z17.h, z1.h\n" - "fmin z18.h, p4/M, z18.h, z1.h\n" - "fmin z19.h, p4/M, z19.h, z1.h\n" - "fmax z8.h, p4/M, z8.h, z0.h\n" - "fmax z9.h, p4/M, z9.h, z0.h\n" - "fmax z10.h, p4/M, z10.h, z0.h\n" - "fmax z11.h, p4/M, z11.h, z0.h\n" - "fmax z12.h, p4/M, z12.h, z0.h\n" - "fmax z13.h, p4/M, z13.h, z0.h\n" - "fmax z14.h, p4/M, z14.h, z0.h\n" - "fmax z15.h, p4/M, z15.h, z0.h\n" - "fmax z16.h, p4/M, z16.h, z0.h\n" - "fmax z17.h, p4/M, z17.h, z0.h\n" - "fmax z18.h, p4/M, z18.h, z0.h\n" - "fmax z19.h, p4/M, z19.h, z0.h\n" + "ld1rh { z20.h }, p4/Z, [x20]\n" + "fmin z8.h, p4/M, z8.h, z21.h\n" + "fmin z9.h, p4/M, z9.h, z21.h\n" + "fmin z10.h, p4/M, z10.h, z21.h\n" + "fmin z11.h, p4/M, z11.h, z21.h\n" + "fmin z12.h, p4/M, z12.h, z21.h\n" + "fmin z13.h, p4/M, z13.h, z21.h\n" + "fmin z14.h, p4/M, z14.h, z21.h\n" + "fmin z15.h, p4/M, z15.h, z21.h\n" + "fmin z16.h, p4/M, z16.h, z21.h\n" + "fmin z17.h, p4/M, z17.h, z21.h\n" + "fmin z18.h, p4/M, z18.h, z21.h\n" + "fmin z19.h, p4/M, z19.h, z21.h\n" + "fmax z8.h, p4/M, z8.h, z20.h\n" + "fmax z9.h, p4/M, z9.h, z20.h\n" + "fmax z10.h, p4/M, z10.h, z20.h\n" + "fmax z11.h, p4/M, z11.h, z20.h\n" + "fmax z12.h, p4/M, z12.h, z20.h\n" + "fmax z13.h, p4/M, z13.h, z20.h\n" + "fmax z14.h, p4/M, z14.h, z20.h\n" + "fmax z15.h, p4/M, z15.h, z20.h\n" + "fmax z16.h, p4/M, z16.h, z20.h\n" + "fmax z17.h, p4/M, z17.h, z20.h\n" + "fmax z18.h, p4/M, z18.h, z20.h\n" + "fmax z19.h, p4/M, z19.h, z20.h\n" "38:" // Height 3: No activation "st1h { z8.h }, p3, [x13]\n" "st1h { z9.h }, p2, [x13, #1, MUL VL]\n" @@ -681,25 +681,25 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "43:" // Height 4: no bias "tbz %x[flags], #0, 44f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" + "add x22, x13, x20, LSL #1\n" + "add x21, x22, x20, LSL #1\n" "ld1h { z8.h }, p3/Z, [x13]\n" - "add x23, x24, x20, LSL #1\n" + "add x20, x21, x20, LSL #1\n" "ld1h { z9.h }, p2/Z, [x13, #1, MUL VL]\n" "ld1h { z10.h }, p1/Z, [x13, #2, MUL VL]\n" "ld1h { z11.h }, p0/Z, [x13, #3, MUL VL]\n" - "ld1h { z12.h }, p3/Z, [x25]\n" - "ld1h { z13.h }, p2/Z, [x25, #1, MUL VL]\n" - "ld1h { z14.h }, p1/Z, [x25, #2, MUL VL]\n" - "ld1h { z15.h }, p0/Z, [x25, #3, MUL VL]\n" - "ld1h { z16.h }, p3/Z, [x24]\n" - "ld1h { z17.h }, p2/Z, [x24, #1, MUL VL]\n" - "ld1h { z18.h }, p1/Z, [x24, #2, MUL VL]\n" - "ld1h { z19.h }, p0/Z, [x24, #3, MUL VL]\n" - "ld1h { z20.h }, p3/Z, [x23]\n" - "ld1h { z21.h }, p2/Z, [x23, #1, MUL VL]\n" - "ld1h { z22.h }, p1/Z, [x23, #2, MUL VL]\n" - "ld1h { z23.h }, p0/Z, [x23, #3, MUL VL]\n" + "ld1h { z12.h }, p3/Z, [x22]\n" + "ld1h { z13.h }, p2/Z, [x22, #1, MUL VL]\n" + "ld1h { z14.h }, p1/Z, [x22, #2, MUL VL]\n" + "ld1h { z15.h }, p0/Z, [x22, #3, MUL VL]\n" + "ld1h { z16.h }, p3/Z, [x21]\n" + "ld1h { z17.h }, p2/Z, [x21, #1, MUL VL]\n" + "ld1h { z18.h }, p1/Z, [x21, #2, MUL VL]\n" + "ld1h { z19.h }, p0/Z, [x21, #3, MUL VL]\n" + "ld1h { z20.h }, p3/Z, [x20]\n" + "ld1h { z21.h }, p2/Z, [x20, #1, MUL VL]\n" + "ld1h { z22.h }, p1/Z, [x20, #2, MUL VL]\n" + "ld1h { z23.h }, p0/Z, [x20, #3, MUL VL]\n" "b 45f\n" "44:" // Height 4: no accumulate "mov z8.b, #0x0\n" @@ -723,14 +723,14 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "46:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 47f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 48f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -740,9 +740,9 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "b 48f\n" "47:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" "48:" // Height 4: input setup done "subs x27, x27, #0x1\n" "ld1rh { z0.h }, p4/Z, [x26]\n" @@ -759,7 +759,7 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "addvl x11, x11, #1\n" "fmla z16.h, p4/M, z6.h, z2.h\n" "fmla z20.h, p4/M, z6.h, z3.h\n" - "ld1h { z6.h }, p4/Z, [x10]\n" + "ld1h { z25.h }, p4/Z, [x10]\n" "add x26, x26, #0x2\n" "fmla z9.h, p4/M, z7.h, z0.h\n" "fmla z13.h, p4/M, z7.h, z1.h\n" @@ -767,22 +767,22 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "add x25, x25, #0x2\n" "fmla z17.h, p4/M, z7.h, z2.h\n" "fmla z21.h, p4/M, z7.h, z3.h\n" - "ld1h { z7.h }, p4/Z, [x9]\n" + "ld1h { z24.h }, p4/Z, [x9]\n" "add x24, x24, #0x2\n" "add x23, x23, #0x2\n" - "fmla z10.h, p4/M, z6.h, z0.h\n" - "fmla z14.h, p4/M, z6.h, z1.h\n" + "fmla z10.h, p4/M, z25.h, z0.h\n" + "fmla z14.h, p4/M, z25.h, z1.h\n" "addvl x10, x10, #1\n" - "fmla z18.h, p4/M, z6.h, z2.h\n" - "fmla z22.h, p4/M, z6.h, z3.h\n" + "fmla z18.h, p4/M, z25.h, z2.h\n" + "fmla z22.h, p4/M, z25.h, z3.h\n" "addvl x9, x9, #1\n" "ld1h { z6.h }, p4/Z, [x12]\n" - "fmla z11.h, p4/M, z7.h, z0.h\n" - "fmla z15.h, p4/M, z7.h, z1.h\n" + "fmla z11.h, p4/M, z24.h, z0.h\n" + "fmla z15.h, p4/M, z24.h, z1.h\n" "ld1rh { z0.h }, p4/Z, [x26]\n" "ld1rh { z1.h }, p4/Z, [x25]\n" - "fmla z19.h, p4/M, z7.h, z2.h\n" - "fmla z23.h, p4/M, z7.h, z3.h\n" + "fmla z19.h, p4/M, z24.h, z2.h\n" + "fmla z23.h, p4/M, z24.h, z3.h\n" "ld1rh { z2.h }, p4/Z, [x24]\n" "ld1rh { z3.h }, p4/Z, [x23]\n" "ld1h { z7.h }, p4/Z, [x11]\n" @@ -794,7 +794,7 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "add x28, x28, #0x1\n" "fmla z16.h, p4/M, z6.h, z2.h\n" "fmla z20.h, p4/M, z6.h, z3.h\n" - "ld1h { z6.h }, p4/Z, [x10]\n" + "ld1h { z25.h }, p4/Z, [x10]\n" "cmp x28, x20\n" "fmla z9.h, p4/M, z7.h, z0.h\n" "fmla z13.h, p4/M, z7.h, z1.h\n" @@ -802,17 +802,17 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "addvl x11, x11, #1\n" "fmla z17.h, p4/M, z7.h, z2.h\n" "fmla z21.h, p4/M, z7.h, z3.h\n" - "ld1h { z7.h }, p4/Z, [x9]\n" + "ld1h { z24.h }, p4/Z, [x9]\n" "addvl x10, x10, #1\n" - "fmla z10.h, p4/M, z6.h, z0.h\n" - "fmla z14.h, p4/M, z6.h, z1.h\n" + "fmla z10.h, p4/M, z25.h, z0.h\n" + "fmla z14.h, p4/M, z25.h, z1.h\n" "addvl x9, x9, #1\n" - "fmla z18.h, p4/M, z6.h, z2.h\n" - "fmla z22.h, p4/M, z6.h, z3.h\n" - "fmla z11.h, p4/M, z7.h, z0.h\n" - "fmla z15.h, p4/M, z7.h, z1.h\n" - "fmla z19.h, p4/M, z7.h, z2.h\n" - "fmla z23.h, p4/M, z7.h, z3.h\n" + "fmla z18.h, p4/M, z25.h, z2.h\n" + "fmla z22.h, p4/M, z25.h, z3.h\n" + "fmla z11.h, p4/M, z24.h, z0.h\n" + "fmla z15.h, p4/M, z24.h, z1.h\n" + "fmla z19.h, p4/M, z24.h, z2.h\n" + "fmla z23.h, p4/M, z24.h, z3.h\n" "bne 46b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "add x25, x13, x20, LSL #1\n" @@ -820,41 +820,41 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "add x23, x24, x20, LSL #1\n" "tbz %x[flags], #1, 51f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rh { z1.h }, p4/Z, [x20]\n" + "ld1rh { z25.h }, p4/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rh { z0.h }, p4/Z, [x20]\n" - "fmin z8.h, p4/M, z8.h, z1.h\n" - "fmin z9.h, p4/M, z9.h, z1.h\n" - "fmin z10.h, p4/M, z10.h, z1.h\n" - "fmin z11.h, p4/M, z11.h, z1.h\n" - "fmin z12.h, p4/M, z12.h, z1.h\n" - "fmin z13.h, p4/M, z13.h, z1.h\n" - "fmin z14.h, p4/M, z14.h, z1.h\n" - "fmin z15.h, p4/M, z15.h, z1.h\n" - "fmin z16.h, p4/M, z16.h, z1.h\n" - "fmin z17.h, p4/M, z17.h, z1.h\n" - "fmin z18.h, p4/M, z18.h, z1.h\n" - "fmin z19.h, p4/M, z19.h, z1.h\n" - "fmin z20.h, p4/M, z20.h, z1.h\n" - "fmin z21.h, p4/M, z21.h, z1.h\n" - "fmin z22.h, p4/M, z22.h, z1.h\n" - "fmin z23.h, p4/M, z23.h, z1.h\n" - "fmax z8.h, p4/M, z8.h, z0.h\n" - "fmax z9.h, p4/M, z9.h, z0.h\n" - "fmax z10.h, p4/M, z10.h, z0.h\n" - "fmax z11.h, p4/M, z11.h, z0.h\n" - "fmax z12.h, p4/M, z12.h, z0.h\n" - "fmax z13.h, p4/M, z13.h, z0.h\n" - "fmax z14.h, p4/M, z14.h, z0.h\n" - "fmax z15.h, p4/M, z15.h, z0.h\n" - "fmax z16.h, p4/M, z16.h, z0.h\n" - "fmax z17.h, p4/M, z17.h, z0.h\n" - "fmax z18.h, p4/M, z18.h, z0.h\n" - "fmax z19.h, p4/M, z19.h, z0.h\n" - "fmax z20.h, p4/M, z20.h, z0.h\n" - "fmax z21.h, p4/M, z21.h, z0.h\n" - "fmax z22.h, p4/M, z22.h, z0.h\n" - "fmax z23.h, p4/M, z23.h, z0.h\n" + "ld1rh { z24.h }, p4/Z, [x20]\n" + "fmin z8.h, p4/M, z8.h, z25.h\n" + "fmin z9.h, p4/M, z9.h, z25.h\n" + "fmin z10.h, p4/M, z10.h, z25.h\n" + "fmin z11.h, p4/M, z11.h, z25.h\n" + "fmin z12.h, p4/M, z12.h, z25.h\n" + "fmin z13.h, p4/M, z13.h, z25.h\n" + "fmin z14.h, p4/M, z14.h, z25.h\n" + "fmin z15.h, p4/M, z15.h, z25.h\n" + "fmin z16.h, p4/M, z16.h, z25.h\n" + "fmin z17.h, p4/M, z17.h, z25.h\n" + "fmin z18.h, p4/M, z18.h, z25.h\n" + "fmin z19.h, p4/M, z19.h, z25.h\n" + "fmin z20.h, p4/M, z20.h, z25.h\n" + "fmin z21.h, p4/M, z21.h, z25.h\n" + "fmin z22.h, p4/M, z22.h, z25.h\n" + "fmin z23.h, p4/M, z23.h, z25.h\n" + "fmax z8.h, p4/M, z8.h, z24.h\n" + "fmax z9.h, p4/M, z9.h, z24.h\n" + "fmax z10.h, p4/M, z10.h, z24.h\n" + "fmax z11.h, p4/M, z11.h, z24.h\n" + "fmax z12.h, p4/M, z12.h, z24.h\n" + "fmax z13.h, p4/M, z13.h, z24.h\n" + "fmax z14.h, p4/M, z14.h, z24.h\n" + "fmax z15.h, p4/M, z15.h, z24.h\n" + "fmax z16.h, p4/M, z16.h, z24.h\n" + "fmax z17.h, p4/M, z17.h, z24.h\n" + "fmax z18.h, p4/M, z18.h, z24.h\n" + "fmax z19.h, p4/M, z19.h, z24.h\n" + "fmax z20.h, p4/M, z20.h, z24.h\n" + "fmax z21.h, p4/M, z21.h, z24.h\n" + "fmax z22.h, p4/M, z22.h, z24.h\n" + "fmax z23.h, p4/M, z23.h, z24.h\n" "51:" // Height 4: No activation "st1h { z8.h }, p3, [x13]\n" "st1h { z9.h }, p2, [x13, #1, MUL VL]\n" @@ -939,30 +939,30 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "56:" // Height 5: no bias "tbz %x[flags], #0, 57f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "ld1h { z8.h }, p3/Z, [x13]\n" - "add x23, x24, x20, LSL #1\n" + "add x23, x13, x20, LSL #1\n" "add x22, x23, x20, LSL #1\n" + "ld1h { z8.h }, p3/Z, [x13]\n" + "add x21, x22, x20, LSL #1\n" + "add x20, x21, x20, LSL #1\n" "ld1h { z9.h }, p2/Z, [x13, #1, MUL VL]\n" "ld1h { z10.h }, p1/Z, [x13, #2, MUL VL]\n" "ld1h { z11.h }, p0/Z, [x13, #3, MUL VL]\n" - "ld1h { z12.h }, p3/Z, [x25]\n" - "ld1h { z13.h }, p2/Z, [x25, #1, MUL VL]\n" - "ld1h { z14.h }, p1/Z, [x25, #2, MUL VL]\n" - "ld1h { z15.h }, p0/Z, [x25, #3, MUL VL]\n" - "ld1h { z16.h }, p3/Z, [x24]\n" - "ld1h { z17.h }, p2/Z, [x24, #1, MUL VL]\n" - "ld1h { z18.h }, p1/Z, [x24, #2, MUL VL]\n" - "ld1h { z19.h }, p0/Z, [x24, #3, MUL VL]\n" - "ld1h { z20.h }, p3/Z, [x23]\n" - "ld1h { z21.h }, p2/Z, [x23, #1, MUL VL]\n" - "ld1h { z22.h }, p1/Z, [x23, #2, MUL VL]\n" - "ld1h { z23.h }, p0/Z, [x23, #3, MUL VL]\n" - "ld1h { z24.h }, p3/Z, [x22]\n" - "ld1h { z25.h }, p2/Z, [x22, #1, MUL VL]\n" - "ld1h { z26.h }, p1/Z, [x22, #2, MUL VL]\n" - "ld1h { z27.h }, p0/Z, [x22, #3, MUL VL]\n" + "ld1h { z12.h }, p3/Z, [x23]\n" + "ld1h { z13.h }, p2/Z, [x23, #1, MUL VL]\n" + "ld1h { z14.h }, p1/Z, [x23, #2, MUL VL]\n" + "ld1h { z15.h }, p0/Z, [x23, #3, MUL VL]\n" + "ld1h { z16.h }, p3/Z, [x22]\n" + "ld1h { z17.h }, p2/Z, [x22, #1, MUL VL]\n" + "ld1h { z18.h }, p1/Z, [x22, #2, MUL VL]\n" + "ld1h { z19.h }, p0/Z, [x22, #3, MUL VL]\n" + "ld1h { z20.h }, p3/Z, [x21]\n" + "ld1h { z21.h }, p2/Z, [x21, #1, MUL VL]\n" + "ld1h { z22.h }, p1/Z, [x21, #2, MUL VL]\n" + "ld1h { z23.h }, p0/Z, [x21, #3, MUL VL]\n" + "ld1h { z24.h }, p3/Z, [x20]\n" + "ld1h { z25.h }, p2/Z, [x20, #1, MUL VL]\n" + "ld1h { z26.h }, p1/Z, [x20, #2, MUL VL]\n" + "ld1h { z27.h }, p0/Z, [x20, #3, MUL VL]\n" "b 58f\n" "57:" // Height 5: no accumulate "mov z8.b, #0x0\n" @@ -990,15 +990,15 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "59:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 60f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 61f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -1009,10 +1009,10 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "b 61f\n" "60:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" - "add x22, x23, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" + "add x22, x23, x21, LSL #1\n" "61:" // Height 5: input setup done "subs x27, x27, #0x1\n" "ld1rh { z0.h }, p4/Z, [x26]\n" @@ -1034,7 +1034,7 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "subs x27, x27, #0x1\n" "fmla z24.h, p4/M, z6.h, z4.h\n" "fmla z9.h, p4/M, z7.h, z0.h\n" - "ld1h { z6.h }, p4/Z, [x10]\n" + "ld1h { z29.h }, p4/Z, [x10]\n" "add x25, x25, #0x2\n" "fmla z13.h, p4/M, z7.h, z1.h\n" "fmla z17.h, p4/M, z7.h, z2.h\n" @@ -1042,24 +1042,24 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "add x23, x23, #0x2\n" "fmla z21.h, p4/M, z7.h, z3.h\n" "fmla z25.h, p4/M, z7.h, z4.h\n" - "ld1h { z7.h }, p4/Z, [x9]\n" + "ld1h { z28.h }, p4/Z, [x9]\n" "add x22, x22, #0x2\n" - "fmla z10.h, p4/M, z6.h, z0.h\n" - "fmla z14.h, p4/M, z6.h, z1.h\n" + "fmla z10.h, p4/M, z29.h, z0.h\n" + "fmla z14.h, p4/M, z29.h, z1.h\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" - "fmla z18.h, p4/M, z6.h, z2.h\n" - "fmla z22.h, p4/M, z6.h, z3.h\n" - "fmla z26.h, p4/M, z6.h, z4.h\n" - "fmla z11.h, p4/M, z7.h, z0.h\n" + "fmla z18.h, p4/M, z29.h, z2.h\n" + "fmla z22.h, p4/M, z29.h, z3.h\n" + "fmla z26.h, p4/M, z29.h, z4.h\n" + "fmla z11.h, p4/M, z28.h, z0.h\n" "ld1rh { z0.h }, p4/Z, [x26]\n" "ld1h { z6.h }, p4/Z, [x12]\n" - "fmla z15.h, p4/M, z7.h, z1.h\n" - "fmla z19.h, p4/M, z7.h, z2.h\n" + "fmla z15.h, p4/M, z28.h, z1.h\n" + "fmla z19.h, p4/M, z28.h, z2.h\n" "ld1rh { z1.h }, p4/Z, [x25]\n" "ld1rh { z2.h }, p4/Z, [x24]\n" - "fmla z23.h, p4/M, z7.h, z3.h\n" - "fmla z27.h, p4/M, z7.h, z4.h\n" + "fmla z23.h, p4/M, z28.h, z3.h\n" + "fmla z27.h, p4/M, z28.h, z4.h\n" "ld1rh { z3.h }, p4/Z, [x23]\n" "ld1rh { z4.h }, p4/Z, [x22]\n" "ld1h { z7.h }, p4/Z, [x11]\n" @@ -1075,25 +1075,25 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "addvl x12, x12, #1\n" "fmla z24.h, p4/M, z6.h, z4.h\n" "fmla z9.h, p4/M, z7.h, z0.h\n" - "ld1h { z6.h }, p4/Z, [x10]\n" + "ld1h { z29.h }, p4/Z, [x10]\n" "addvl x11, x11, #1\n" "fmla z13.h, p4/M, z7.h, z1.h\n" "fmla z17.h, p4/M, z7.h, z2.h\n" "addvl x10, x10, #1\n" "fmla z21.h, p4/M, z7.h, z3.h\n" "fmla z25.h, p4/M, z7.h, z4.h\n" - "ld1h { z7.h }, p4/Z, [x9]\n" + "ld1h { z28.h }, p4/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.h, p4/M, z6.h, z0.h\n" - "fmla z14.h, p4/M, z6.h, z1.h\n" - "fmla z18.h, p4/M, z6.h, z2.h\n" - "fmla z22.h, p4/M, z6.h, z3.h\n" - "fmla z26.h, p4/M, z6.h, z4.h\n" - "fmla z11.h, p4/M, z7.h, z0.h\n" - "fmla z15.h, p4/M, z7.h, z1.h\n" - "fmla z19.h, p4/M, z7.h, z2.h\n" - "fmla z23.h, p4/M, z7.h, z3.h\n" - "fmla z27.h, p4/M, z7.h, z4.h\n" + "fmla z10.h, p4/M, z29.h, z0.h\n" + "fmla z14.h, p4/M, z29.h, z1.h\n" + "fmla z18.h, p4/M, z29.h, z2.h\n" + "fmla z22.h, p4/M, z29.h, z3.h\n" + "fmla z26.h, p4/M, z29.h, z4.h\n" + "fmla z11.h, p4/M, z28.h, z0.h\n" + "fmla z15.h, p4/M, z28.h, z1.h\n" + "fmla z19.h, p4/M, z28.h, z2.h\n" + "fmla z23.h, p4/M, z28.h, z3.h\n" + "fmla z27.h, p4/M, z28.h, z4.h\n" "bne 59b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "add x25, x13, x20, LSL #1\n" @@ -1102,49 +1102,49 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "add x22, x23, x20, LSL #1\n" "tbz %x[flags], #1, 64f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rh { z1.h }, p4/Z, [x20]\n" + "ld1rh { z29.h }, p4/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rh { z0.h }, p4/Z, [x20]\n" - "fmin z8.h, p4/M, z8.h, z1.h\n" - "fmin z9.h, p4/M, z9.h, z1.h\n" - "fmin z10.h, p4/M, z10.h, z1.h\n" - "fmin z11.h, p4/M, z11.h, z1.h\n" - "fmin z12.h, p4/M, z12.h, z1.h\n" - "fmin z13.h, p4/M, z13.h, z1.h\n" - "fmin z14.h, p4/M, z14.h, z1.h\n" - "fmin z15.h, p4/M, z15.h, z1.h\n" - "fmin z16.h, p4/M, z16.h, z1.h\n" - "fmin z17.h, p4/M, z17.h, z1.h\n" - "fmin z18.h, p4/M, z18.h, z1.h\n" - "fmin z19.h, p4/M, z19.h, z1.h\n" - "fmin z20.h, p4/M, z20.h, z1.h\n" - "fmin z21.h, p4/M, z21.h, z1.h\n" - "fmin z22.h, p4/M, z22.h, z1.h\n" - "fmin z23.h, p4/M, z23.h, z1.h\n" - "fmin z24.h, p4/M, z24.h, z1.h\n" - "fmin z25.h, p4/M, z25.h, z1.h\n" - "fmin z26.h, p4/M, z26.h, z1.h\n" - "fmin z27.h, p4/M, z27.h, z1.h\n" - "fmax z8.h, p4/M, z8.h, z0.h\n" - "fmax z9.h, p4/M, z9.h, z0.h\n" - "fmax z10.h, p4/M, z10.h, z0.h\n" - "fmax z11.h, p4/M, z11.h, z0.h\n" - "fmax z12.h, p4/M, z12.h, z0.h\n" - "fmax z13.h, p4/M, z13.h, z0.h\n" - "fmax z14.h, p4/M, z14.h, z0.h\n" - "fmax z15.h, p4/M, z15.h, z0.h\n" - "fmax z16.h, p4/M, z16.h, z0.h\n" - "fmax z17.h, p4/M, z17.h, z0.h\n" - "fmax z18.h, p4/M, z18.h, z0.h\n" - "fmax z19.h, p4/M, z19.h, z0.h\n" - "fmax z20.h, p4/M, z20.h, z0.h\n" - "fmax z21.h, p4/M, z21.h, z0.h\n" - "fmax z22.h, p4/M, z22.h, z0.h\n" - "fmax z23.h, p4/M, z23.h, z0.h\n" - "fmax z24.h, p4/M, z24.h, z0.h\n" - "fmax z25.h, p4/M, z25.h, z0.h\n" - "fmax z26.h, p4/M, z26.h, z0.h\n" - "fmax z27.h, p4/M, z27.h, z0.h\n" + "ld1rh { z28.h }, p4/Z, [x20]\n" + "fmin z8.h, p4/M, z8.h, z29.h\n" + "fmin z9.h, p4/M, z9.h, z29.h\n" + "fmin z10.h, p4/M, z10.h, z29.h\n" + "fmin z11.h, p4/M, z11.h, z29.h\n" + "fmin z12.h, p4/M, z12.h, z29.h\n" + "fmin z13.h, p4/M, z13.h, z29.h\n" + "fmin z14.h, p4/M, z14.h, z29.h\n" + "fmin z15.h, p4/M, z15.h, z29.h\n" + "fmin z16.h, p4/M, z16.h, z29.h\n" + "fmin z17.h, p4/M, z17.h, z29.h\n" + "fmin z18.h, p4/M, z18.h, z29.h\n" + "fmin z19.h, p4/M, z19.h, z29.h\n" + "fmin z20.h, p4/M, z20.h, z29.h\n" + "fmin z21.h, p4/M, z21.h, z29.h\n" + "fmin z22.h, p4/M, z22.h, z29.h\n" + "fmin z23.h, p4/M, z23.h, z29.h\n" + "fmin z24.h, p4/M, z24.h, z29.h\n" + "fmin z25.h, p4/M, z25.h, z29.h\n" + "fmin z26.h, p4/M, z26.h, z29.h\n" + "fmin z27.h, p4/M, z27.h, z29.h\n" + "fmax z8.h, p4/M, z8.h, z28.h\n" + "fmax z9.h, p4/M, z9.h, z28.h\n" + "fmax z10.h, p4/M, z10.h, z28.h\n" + "fmax z11.h, p4/M, z11.h, z28.h\n" + "fmax z12.h, p4/M, z12.h, z28.h\n" + "fmax z13.h, p4/M, z13.h, z28.h\n" + "fmax z14.h, p4/M, z14.h, z28.h\n" + "fmax z15.h, p4/M, z15.h, z28.h\n" + "fmax z16.h, p4/M, z16.h, z28.h\n" + "fmax z17.h, p4/M, z17.h, z28.h\n" + "fmax z18.h, p4/M, z18.h, z28.h\n" + "fmax z19.h, p4/M, z19.h, z28.h\n" + "fmax z20.h, p4/M, z20.h, z28.h\n" + "fmax z21.h, p4/M, z21.h, z28.h\n" + "fmax z22.h, p4/M, z22.h, z28.h\n" + "fmax z23.h, p4/M, z23.h, z28.h\n" + "fmax z24.h, p4/M, z24.h, z28.h\n" + "fmax z25.h, p4/M, z25.h, z28.h\n" + "fmax z26.h, p4/M, z26.h, z28.h\n" + "fmax z27.h, p4/M, z27.h, z28.h\n" "64:" // Height 5: No activation "st1h { z8.h }, p3, [x13]\n" "st1h { z9.h }, p2, [x13, #1, MUL VL]\n" @@ -1240,35 +1240,35 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "69:" // Height 6: no bias "tbz %x[flags], #0, 70f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "ld1h { z8.h }, p3/Z, [x13]\n" + "add x24, x13, x20, LSL #1\n" "add x23, x24, x20, LSL #1\n" + "ld1h { z8.h }, p3/Z, [x13]\n" "add x22, x23, x20, LSL #1\n" + "add x21, x22, x20, LSL #1\n" "ld1h { z9.h }, p2/Z, [x13, #1, MUL VL]\n" "ld1h { z10.h }, p1/Z, [x13, #2, MUL VL]\n" - "add x21, x22, x20, LSL #1\n" + "add x20, x21, x20, LSL #1\n" "ld1h { z11.h }, p0/Z, [x13, #3, MUL VL]\n" - "ld1h { z12.h }, p3/Z, [x25]\n" - "ld1h { z13.h }, p2/Z, [x25, #1, MUL VL]\n" - "ld1h { z14.h }, p1/Z, [x25, #2, MUL VL]\n" - "ld1h { z15.h }, p0/Z, [x25, #3, MUL VL]\n" - "ld1h { z16.h }, p3/Z, [x24]\n" - "ld1h { z17.h }, p2/Z, [x24, #1, MUL VL]\n" - "ld1h { z18.h }, p1/Z, [x24, #2, MUL VL]\n" - "ld1h { z19.h }, p0/Z, [x24, #3, MUL VL]\n" - "ld1h { z20.h }, p3/Z, [x23]\n" - "ld1h { z21.h }, p2/Z, [x23, #1, MUL VL]\n" - "ld1h { z22.h }, p1/Z, [x23, #2, MUL VL]\n" - "ld1h { z23.h }, p0/Z, [x23, #3, MUL VL]\n" - "ld1h { z24.h }, p3/Z, [x22]\n" - "ld1h { z25.h }, p2/Z, [x22, #1, MUL VL]\n" - "ld1h { z26.h }, p1/Z, [x22, #2, MUL VL]\n" - "ld1h { z27.h }, p0/Z, [x22, #3, MUL VL]\n" - "ld1h { z28.h }, p3/Z, [x21]\n" - "ld1h { z29.h }, p2/Z, [x21, #1, MUL VL]\n" - "ld1h { z30.h }, p1/Z, [x21, #2, MUL VL]\n" - "ld1h { z31.h }, p0/Z, [x21, #3, MUL VL]\n" + "ld1h { z12.h }, p3/Z, [x24]\n" + "ld1h { z13.h }, p2/Z, [x24, #1, MUL VL]\n" + "ld1h { z14.h }, p1/Z, [x24, #2, MUL VL]\n" + "ld1h { z15.h }, p0/Z, [x24, #3, MUL VL]\n" + "ld1h { z16.h }, p3/Z, [x23]\n" + "ld1h { z17.h }, p2/Z, [x23, #1, MUL VL]\n" + "ld1h { z18.h }, p1/Z, [x23, #2, MUL VL]\n" + "ld1h { z19.h }, p0/Z, [x23, #3, MUL VL]\n" + "ld1h { z20.h }, p3/Z, [x22]\n" + "ld1h { z21.h }, p2/Z, [x22, #1, MUL VL]\n" + "ld1h { z22.h }, p1/Z, [x22, #2, MUL VL]\n" + "ld1h { z23.h }, p0/Z, [x22, #3, MUL VL]\n" + "ld1h { z24.h }, p3/Z, [x21]\n" + "ld1h { z25.h }, p2/Z, [x21, #1, MUL VL]\n" + "ld1h { z26.h }, p1/Z, [x21, #2, MUL VL]\n" + "ld1h { z27.h }, p0/Z, [x21, #3, MUL VL]\n" + "ld1h { z28.h }, p3/Z, [x20]\n" + "ld1h { z29.h }, p2/Z, [x20, #1, MUL VL]\n" + "ld1h { z30.h }, p1/Z, [x20, #2, MUL VL]\n" + "ld1h { z31.h }, p0/Z, [x20, #3, MUL VL]\n" "b 71f\n" "70:" // Height 6: no accumulate "mov z8.b, #0x0\n" @@ -1300,16 +1300,16 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "72:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 73f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 74f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -1321,11 +1321,11 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( "b 74f\n" "73:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" - "add x22, x23, x20, LSL #1\n" - "add x21, x22, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" + "add x22, x23, x21, LSL #1\n" + "add x21, x22, x21, LSL #1\n" "74:" // Height 6: input setup done "subs x27, x27, #0x1\n" "ld1rh { z0.h }, p4/Z, [x26]\n" @@ -1527,4 +1527,4 @@ void sve_ffhybrid_fp16_mla_6x4VL_a64fx ( } } // namespace arm_gemm -#endif // ARM_COMPUTE_ENABLE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp16_mla_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp16_mla_6x4VL/generic.cpp index 0b543b667f748551284a037a597ce7d128b73fcb..842db1a4fcb910ca823c9759278268508b602ca1 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp16_mla_6x4VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp16_mla_6x4VL/generic.cpp @@ -163,11 +163,11 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "7:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 8f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 9f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -180,72 +180,72 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "10:" // Height 1: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" "ld1rqh { z0.h }, p0/Z, [x26]\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "fmla z8.h, z6.h, z0.h[0]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z9.h, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "fmla z10.h, z6.h, z0.h[0]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" - "fmla z11.h, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[1]\n" - "ld1h { z7.h }, p5/Z, [x11, #1, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[1]\n" - "ld1h { z7.h }, p5/Z, [x9, #1, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x12, #2, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[2]\n" - "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[2]\n" - "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x12, #3, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[3]\n" - "ld1h { z7.h }, p5/Z, [x11, #3, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[3]\n" - "ld1h { z7.h }, p5/Z, [x9, #3, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[3]\n" - "ld1h { z6.h }, p5/Z, [x12, #4, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[4]\n" - "ld1h { z7.h }, p5/Z, [x11, #4, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[4]\n" - "ld1h { z7.h }, p5/Z, [x9, #4, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[4]\n" - "ld1h { z6.h }, p5/Z, [x12, #5, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[5]\n" - "ld1h { z7.h }, p5/Z, [x11, #5, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[5]\n" - "ld1h { z7.h }, p5/Z, [x9, #5, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[5]\n" - "ld1h { z6.h }, p5/Z, [x12, #6, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[6]\n" - "ld1h { z7.h }, p5/Z, [x11, #6, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[6]\n" - "ld1h { z7.h }, p5/Z, [x9, #6, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[6]\n" - "ld1h { z6.h }, p5/Z, [x12, #7, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[7]\n" - "ld1h { z7.h }, p5/Z, [x11, #7, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[7]\n" - "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x12]\n" + "fmla z8.h, z16.h, z0.h[0]\n" + "ld1h { z16.h }, p5/Z, [x11]\n" + "fmla z9.h, z16.h, z0.h[0]\n" + "ld1h { z16.h }, p5/Z, [x10]\n" + "fmla z10.h, z16.h, z0.h[0]\n" + "ld1h { z16.h }, p5/Z, [x9]\n" + "fmla z11.h, z16.h, z0.h[0]\n" + "ld1h { z16.h }, p5/Z, [x12, #1, MUL VL]\n" + "fmla z8.h, z16.h, z0.h[1]\n" + "ld1h { z16.h }, p5/Z, [x11, #1, MUL VL]\n" + "fmla z9.h, z16.h, z0.h[1]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z10.h, z16.h, z0.h[1]\n" + "ld1h { z16.h }, p5/Z, [x9, #1, MUL VL]\n" + "fmla z11.h, z16.h, z0.h[1]\n" + "ld1h { z16.h }, p5/Z, [x12, #2, MUL VL]\n" + "fmla z8.h, z16.h, z0.h[2]\n" + "ld1h { z16.h }, p5/Z, [x11, #2, MUL VL]\n" + "fmla z9.h, z16.h, z0.h[2]\n" + "ld1h { z16.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z10.h, z16.h, z0.h[2]\n" + "ld1h { z16.h }, p5/Z, [x9, #2, MUL VL]\n" + "fmla z11.h, z16.h, z0.h[2]\n" + "ld1h { z16.h }, p5/Z, [x12, #3, MUL VL]\n" + "fmla z8.h, z16.h, z0.h[3]\n" + "ld1h { z16.h }, p5/Z, [x11, #3, MUL VL]\n" + "fmla z9.h, z16.h, z0.h[3]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z10.h, z16.h, z0.h[3]\n" + "ld1h { z16.h }, p5/Z, [x9, #3, MUL VL]\n" + "fmla z11.h, z16.h, z0.h[3]\n" + "ld1h { z16.h }, p5/Z, [x12, #4, MUL VL]\n" + "fmla z8.h, z16.h, z0.h[4]\n" + "ld1h { z16.h }, p5/Z, [x11, #4, MUL VL]\n" + "fmla z9.h, z16.h, z0.h[4]\n" + "ld1h { z16.h }, p5/Z, [x10, #4, MUL VL]\n" + "fmla z10.h, z16.h, z0.h[4]\n" + "ld1h { z16.h }, p5/Z, [x9, #4, MUL VL]\n" + "fmla z11.h, z16.h, z0.h[4]\n" + "ld1h { z16.h }, p5/Z, [x12, #5, MUL VL]\n" + "fmla z8.h, z16.h, z0.h[5]\n" + "ld1h { z16.h }, p5/Z, [x11, #5, MUL VL]\n" + "fmla z9.h, z16.h, z0.h[5]\n" + "ld1h { z16.h }, p5/Z, [x10, #5, MUL VL]\n" + "fmla z10.h, z16.h, z0.h[5]\n" + "ld1h { z16.h }, p5/Z, [x9, #5, MUL VL]\n" + "fmla z11.h, z16.h, z0.h[5]\n" + "ld1h { z16.h }, p5/Z, [x12, #6, MUL VL]\n" + "fmla z8.h, z16.h, z0.h[6]\n" + "ld1h { z16.h }, p5/Z, [x11, #6, MUL VL]\n" + "fmla z9.h, z16.h, z0.h[6]\n" + "ld1h { z16.h }, p5/Z, [x10, #6, MUL VL]\n" + "fmla z10.h, z16.h, z0.h[6]\n" + "ld1h { z16.h }, p5/Z, [x9, #6, MUL VL]\n" + "fmla z11.h, z16.h, z0.h[6]\n" + "ld1h { z16.h }, p5/Z, [x12, #7, MUL VL]\n" + "fmla z8.h, z16.h, z0.h[7]\n" + "ld1h { z16.h }, p5/Z, [x11, #7, MUL VL]\n" + "fmla z9.h, z16.h, z0.h[7]\n" + "ld1h { z17.h }, p5/Z, [x10, #7, MUL VL]\n" "sub x27, x27, #0x8\n" - "ld1h { z7.h }, p5/Z, [x9, #7, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x9, #7, MUL VL]\n" "cmp x27, #0x8\n" - "fmla z10.h, z6.h, z0.h[7]\n" - "fmla z11.h, z7.h, z0.h[7]\n" + "fmla z10.h, z17.h, z0.h[7]\n" + "fmla z11.h, z16.h, z0.h[7]\n" "add x26, x26, #0x10\n" "addvl x12, x12, #8\n" "addvl x11, x11, #8\n" @@ -255,112 +255,112 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "11:" // Height 1: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" "ld1rqh { z0.h }, p0/Z, [x26]\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "fmla z8.h, z6.h, z0.h[0]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z9.h, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x12]\n" + "fmla z8.h, z16.h, z0.h[0]\n" + "ld1h { z16.h }, p5/Z, [x11]\n" + "fmla z9.h, z16.h, z0.h[0]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" "subs x27, x27, #0x1\n" - "ld1h { z7.h }, p5/Z, [x9]\n" - "fmla z10.h, z6.h, z0.h[0]\n" - "fmla z11.h, z7.h, z0.h[0]\n" + "ld1h { z16.h }, p5/Z, [x9]\n" + "fmla z10.h, z17.h, z0.h[0]\n" + "fmla z11.h, z16.h, z0.h[0]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" "ble 12f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[1]\n" - "fmla z9.h, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "ld1h { z17.h }, p5/Z, [x12]\n" + "ld1h { z16.h }, p5/Z, [x11]\n" + "fmla z8.h, z17.h, z0.h[1]\n" + "fmla z9.h, z16.h, z0.h[1]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x9]\n" "subs x27, x27, #0x1\n" - "fmla z10.h, z6.h, z0.h[1]\n" - "fmla z11.h, z7.h, z0.h[1]\n" + "fmla z10.h, z17.h, z0.h[1]\n" + "fmla z11.h, z16.h, z0.h[1]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" "ble 12f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[2]\n" - "fmla z9.h, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "ld1h { z17.h }, p5/Z, [x12]\n" + "ld1h { z16.h }, p5/Z, [x11]\n" + "fmla z8.h, z17.h, z0.h[2]\n" + "fmla z9.h, z16.h, z0.h[2]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x9]\n" "subs x27, x27, #0x1\n" - "fmla z10.h, z6.h, z0.h[2]\n" - "fmla z11.h, z7.h, z0.h[2]\n" + "fmla z10.h, z17.h, z0.h[2]\n" + "fmla z11.h, z16.h, z0.h[2]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" "ble 12f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[3]\n" - "fmla z9.h, z7.h, z0.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "ld1h { z17.h }, p5/Z, [x12]\n" + "ld1h { z16.h }, p5/Z, [x11]\n" + "fmla z8.h, z17.h, z0.h[3]\n" + "fmla z9.h, z16.h, z0.h[3]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x9]\n" "subs x27, x27, #0x1\n" - "fmla z10.h, z6.h, z0.h[3]\n" - "fmla z11.h, z7.h, z0.h[3]\n" + "fmla z10.h, z17.h, z0.h[3]\n" + "fmla z11.h, z16.h, z0.h[3]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" "ble 12f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[4]\n" - "fmla z9.h, z7.h, z0.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "ld1h { z17.h }, p5/Z, [x12]\n" + "ld1h { z16.h }, p5/Z, [x11]\n" + "fmla z8.h, z17.h, z0.h[4]\n" + "fmla z9.h, z16.h, z0.h[4]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x9]\n" "subs x27, x27, #0x1\n" - "fmla z10.h, z6.h, z0.h[4]\n" - "fmla z11.h, z7.h, z0.h[4]\n" + "fmla z10.h, z17.h, z0.h[4]\n" + "fmla z11.h, z16.h, z0.h[4]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" "ble 12f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[5]\n" - "fmla z9.h, z7.h, z0.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "ld1h { z17.h }, p5/Z, [x12]\n" + "ld1h { z16.h }, p5/Z, [x11]\n" + "fmla z8.h, z17.h, z0.h[5]\n" + "fmla z9.h, z16.h, z0.h[5]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x9]\n" "subs x27, x27, #0x1\n" - "fmla z10.h, z6.h, z0.h[5]\n" - "fmla z11.h, z7.h, z0.h[5]\n" + "fmla z10.h, z17.h, z0.h[5]\n" + "fmla z11.h, z16.h, z0.h[5]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" "ble 12f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[6]\n" - "fmla z9.h, z7.h, z0.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "ld1h { z17.h }, p5/Z, [x12]\n" + "ld1h { z16.h }, p5/Z, [x11]\n" + "fmla z8.h, z17.h, z0.h[6]\n" + "fmla z9.h, z16.h, z0.h[6]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x9]\n" "subs x27, x27, #0x1\n" - "fmla z10.h, z6.h, z0.h[6]\n" - "fmla z11.h, z7.h, z0.h[6]\n" + "fmla z10.h, z17.h, z0.h[6]\n" + "fmla z11.h, z16.h, z0.h[6]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" "ble 12f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[7]\n" - "fmla z9.h, z7.h, z0.h[7]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" - "fmla z10.h, z6.h, z0.h[7]\n" - "fmla z11.h, z7.h, z0.h[7]\n" + "ld1h { z17.h }, p5/Z, [x12]\n" + "ld1h { z16.h }, p5/Z, [x11]\n" + "fmla z8.h, z17.h, z0.h[7]\n" + "fmla z9.h, z16.h, z0.h[7]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x9]\n" + "fmla z10.h, z17.h, z0.h[7]\n" + "fmla z11.h, z16.h, z0.h[7]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" @@ -372,17 +372,17 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "bne 7b\n" "tbz %x[flags], #1, 13f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rh { z1.h }, p5/Z, [x20]\n" + "ld1rh { z17.h }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rh { z0.h }, p5/Z, [x20]\n" - "fmin z8.h, p5/M, z8.h, z1.h\n" - "fmin z9.h, p5/M, z9.h, z1.h\n" - "fmin z10.h, p5/M, z10.h, z1.h\n" - "fmin z11.h, p5/M, z11.h, z1.h\n" - "fmax z8.h, p5/M, z8.h, z0.h\n" - "fmax z9.h, p5/M, z9.h, z0.h\n" - "fmax z10.h, p5/M, z10.h, z0.h\n" - "fmax z11.h, p5/M, z11.h, z0.h\n" + "ld1rh { z16.h }, p5/Z, [x20]\n" + "fmin z8.h, p5/M, z8.h, z17.h\n" + "fmin z9.h, p5/M, z9.h, z17.h\n" + "fmin z10.h, p5/M, z10.h, z17.h\n" + "fmin z11.h, p5/M, z11.h, z17.h\n" + "fmax z8.h, p5/M, z8.h, z16.h\n" + "fmax z9.h, p5/M, z9.h, z16.h\n" + "fmax z10.h, p5/M, z10.h, z16.h\n" + "fmax z11.h, p5/M, z11.h, z16.h\n" "13:" // Height 1: No activation "st1h { z8.h }, p4, [x13]\n" "st1h { z9.h }, p3, [x13, #1, MUL VL]\n" @@ -443,15 +443,15 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "18:" // Height 2: no bias "tbz %x[flags], #0, 19f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #1\n" + "add x20, x13, x20, LSL #1\n" "ld1h { z8.h }, p4/Z, [x13]\n" "ld1h { z9.h }, p3/Z, [x13, #1, MUL VL]\n" "ld1h { z10.h }, p2/Z, [x13, #2, MUL VL]\n" "ld1h { z11.h }, p1/Z, [x13, #3, MUL VL]\n" - "ld1h { z12.h }, p4/Z, [x25]\n" - "ld1h { z13.h }, p3/Z, [x25, #1, MUL VL]\n" - "ld1h { z14.h }, p2/Z, [x25, #2, MUL VL]\n" - "ld1h { z15.h }, p1/Z, [x25, #3, MUL VL]\n" + "ld1h { z12.h }, p4/Z, [x20]\n" + "ld1h { z13.h }, p3/Z, [x20, #1, MUL VL]\n" + "ld1h { z14.h }, p2/Z, [x20, #2, MUL VL]\n" + "ld1h { z15.h }, p1/Z, [x20, #3, MUL VL]\n" "b 20f\n" "19:" // Height 2: no accumulate "mov z8.b, #0x0\n" @@ -467,12 +467,12 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "21:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 22f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 23f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -480,263 +480,263 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "b 23f\n" "22:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" "23:" // Height 2: input setup done "cmp x27, #0x8\n" "ble 25f\n" "24:" // Height 2: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z0.h }, p0/Z, [x26]\n" - "ld1rqh { z1.h }, p0/Z, [x25]\n" + "ld1rqh { z1.h }, p0/Z, [x26]\n" + "ld1rqh { z0.h }, p0/Z, [x25]\n" "sub x27, x27, #0x8\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[0]\n" - "fmla z12.h, z6.h, z1.h[0]\n" - "fmla z9.h, z7.h, z0.h[0]\n" - "fmla z13.h, z7.h, z1.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" - "fmla z10.h, z6.h, z0.h[0]\n" - "fmla z14.h, z6.h, z1.h[0]\n" - "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n" + "ld1h { z17.h }, p5/Z, [x12]\n" + "ld1h { z16.h }, p5/Z, [x11]\n" + "fmla z8.h, z17.h, z1.h[0]\n" + "fmla z12.h, z17.h, z0.h[0]\n" + "fmla z9.h, z16.h, z1.h[0]\n" + "fmla z13.h, z16.h, z0.h[0]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x9]\n" + "fmla z10.h, z17.h, z1.h[0]\n" + "fmla z14.h, z17.h, z0.h[0]\n" + "ld1h { z17.h }, p5/Z, [x12, #1, MUL VL]\n" "cmp x27, #0x8\n" - "fmla z11.h, z7.h, z0.h[0]\n" - "fmla z15.h, z7.h, z1.h[0]\n" - "ld1h { z7.h }, p5/Z, [x11, #1, MUL VL]\n" + "fmla z11.h, z16.h, z1.h[0]\n" + "fmla z15.h, z16.h, z0.h[0]\n" + "ld1h { z16.h }, p5/Z, [x11, #1, MUL VL]\n" "add x26, x26, #0x10\n" - "fmla z8.h, z6.h, z0.h[1]\n" - "fmla z12.h, z6.h, z1.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z17.h, z1.h[1]\n" + "fmla z12.h, z17.h, z0.h[1]\n" + "ld1h { z17.h }, p5/Z, [x10, #1, MUL VL]\n" "add x25, x25, #0x10\n" - "fmla z9.h, z7.h, z0.h[1]\n" - "fmla z13.h, z7.h, z1.h[1]\n" - "ld1h { z7.h }, p5/Z, [x9, #1, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[1]\n" - "fmla z14.h, z6.h, z1.h[1]\n" - "ld1h { z6.h }, p5/Z, [x12, #2, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[1]\n" - "fmla z15.h, z7.h, z1.h[1]\n" - "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[2]\n" - "fmla z12.h, z6.h, z1.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[2]\n" - "fmla z13.h, z7.h, z1.h[2]\n" - "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[2]\n" - "fmla z14.h, z6.h, z1.h[2]\n" - "ld1h { z6.h }, p5/Z, [x12, #3, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[2]\n" - "fmla z15.h, z7.h, z1.h[2]\n" - "ld1h { z7.h }, p5/Z, [x11, #3, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[3]\n" - "fmla z12.h, z6.h, z1.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[3]\n" - "fmla z13.h, z7.h, z1.h[3]\n" - "ld1h { z7.h }, p5/Z, [x9, #3, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[3]\n" - "fmla z14.h, z6.h, z1.h[3]\n" - "ld1h { z6.h }, p5/Z, [x12, #4, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[3]\n" - "fmla z15.h, z7.h, z1.h[3]\n" - "ld1h { z7.h }, p5/Z, [x11, #4, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[4]\n" - "fmla z12.h, z6.h, z1.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[4]\n" - "fmla z13.h, z7.h, z1.h[4]\n" - "ld1h { z7.h }, p5/Z, [x9, #4, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[4]\n" - "fmla z14.h, z6.h, z1.h[4]\n" - "ld1h { z6.h }, p5/Z, [x12, #5, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[4]\n" - "fmla z15.h, z7.h, z1.h[4]\n" - "ld1h { z7.h }, p5/Z, [x11, #5, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[5]\n" - "fmla z12.h, z6.h, z1.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[5]\n" - "fmla z13.h, z7.h, z1.h[5]\n" - "ld1h { z7.h }, p5/Z, [x9, #5, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[5]\n" - "fmla z14.h, z6.h, z1.h[5]\n" - "ld1h { z6.h }, p5/Z, [x12, #6, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[5]\n" - "fmla z15.h, z7.h, z1.h[5]\n" - "ld1h { z7.h }, p5/Z, [x11, #6, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[6]\n" - "fmla z12.h, z6.h, z1.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[6]\n" - "fmla z13.h, z7.h, z1.h[6]\n" - "ld1h { z7.h }, p5/Z, [x9, #6, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[6]\n" - "fmla z14.h, z6.h, z1.h[6]\n" - "ld1h { z6.h }, p5/Z, [x12, #7, MUL VL]\n" + "fmla z9.h, z16.h, z1.h[1]\n" + "fmla z13.h, z16.h, z0.h[1]\n" + "ld1h { z16.h }, p5/Z, [x9, #1, MUL VL]\n" + "fmla z10.h, z17.h, z1.h[1]\n" + "fmla z14.h, z17.h, z0.h[1]\n" + "ld1h { z17.h }, p5/Z, [x12, #2, MUL VL]\n" + "fmla z11.h, z16.h, z1.h[1]\n" + "fmla z15.h, z16.h, z0.h[1]\n" + "ld1h { z16.h }, p5/Z, [x11, #2, MUL VL]\n" + "fmla z8.h, z17.h, z1.h[2]\n" + "fmla z12.h, z17.h, z0.h[2]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.h, z16.h, z1.h[2]\n" + "fmla z13.h, z16.h, z0.h[2]\n" + "ld1h { z16.h }, p5/Z, [x9, #2, MUL VL]\n" + "fmla z10.h, z17.h, z1.h[2]\n" + "fmla z14.h, z17.h, z0.h[2]\n" + "ld1h { z17.h }, p5/Z, [x12, #3, MUL VL]\n" + "fmla z11.h, z16.h, z1.h[2]\n" + "fmla z15.h, z16.h, z0.h[2]\n" + "ld1h { z16.h }, p5/Z, [x11, #3, MUL VL]\n" + "fmla z8.h, z17.h, z1.h[3]\n" + "fmla z12.h, z17.h, z0.h[3]\n" + "ld1h { z17.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z9.h, z16.h, z1.h[3]\n" + "fmla z13.h, z16.h, z0.h[3]\n" + "ld1h { z16.h }, p5/Z, [x9, #3, MUL VL]\n" + "fmla z10.h, z17.h, z1.h[3]\n" + "fmla z14.h, z17.h, z0.h[3]\n" + "ld1h { z17.h }, p5/Z, [x12, #4, MUL VL]\n" + "fmla z11.h, z16.h, z1.h[3]\n" + "fmla z15.h, z16.h, z0.h[3]\n" + "ld1h { z16.h }, p5/Z, [x11, #4, MUL VL]\n" + "fmla z8.h, z17.h, z1.h[4]\n" + "fmla z12.h, z17.h, z0.h[4]\n" + "ld1h { z17.h }, p5/Z, [x10, #4, MUL VL]\n" + "fmla z9.h, z16.h, z1.h[4]\n" + "fmla z13.h, z16.h, z0.h[4]\n" + "ld1h { z16.h }, p5/Z, [x9, #4, MUL VL]\n" + "fmla z10.h, z17.h, z1.h[4]\n" + "fmla z14.h, z17.h, z0.h[4]\n" + "ld1h { z17.h }, p5/Z, [x12, #5, MUL VL]\n" + "fmla z11.h, z16.h, z1.h[4]\n" + "fmla z15.h, z16.h, z0.h[4]\n" + "ld1h { z16.h }, p5/Z, [x11, #5, MUL VL]\n" + "fmla z8.h, z17.h, z1.h[5]\n" + "fmla z12.h, z17.h, z0.h[5]\n" + "ld1h { z17.h }, p5/Z, [x10, #5, MUL VL]\n" + "fmla z9.h, z16.h, z1.h[5]\n" + "fmla z13.h, z16.h, z0.h[5]\n" + "ld1h { z16.h }, p5/Z, [x9, #5, MUL VL]\n" + "fmla z10.h, z17.h, z1.h[5]\n" + "fmla z14.h, z17.h, z0.h[5]\n" + "ld1h { z17.h }, p5/Z, [x12, #6, MUL VL]\n" + "fmla z11.h, z16.h, z1.h[5]\n" + "fmla z15.h, z16.h, z0.h[5]\n" + "ld1h { z16.h }, p5/Z, [x11, #6, MUL VL]\n" + "fmla z8.h, z17.h, z1.h[6]\n" + "fmla z12.h, z17.h, z0.h[6]\n" + "ld1h { z17.h }, p5/Z, [x10, #6, MUL VL]\n" + "fmla z9.h, z16.h, z1.h[6]\n" + "fmla z13.h, z16.h, z0.h[6]\n" + "ld1h { z16.h }, p5/Z, [x9, #6, MUL VL]\n" + "fmla z10.h, z17.h, z1.h[6]\n" + "fmla z14.h, z17.h, z0.h[6]\n" + "ld1h { z17.h }, p5/Z, [x12, #7, MUL VL]\n" "addvl x12, x12, #8\n" - "fmla z11.h, z7.h, z0.h[6]\n" - "fmla z15.h, z7.h, z1.h[6]\n" - "ld1h { z7.h }, p5/Z, [x11, #7, MUL VL]\n" + "fmla z11.h, z16.h, z1.h[6]\n" + "fmla z15.h, z16.h, z0.h[6]\n" + "ld1h { z16.h }, p5/Z, [x11, #7, MUL VL]\n" "addvl x11, x11, #8\n" - "fmla z8.h, z6.h, z0.h[7]\n" - "fmla z12.h, z6.h, z1.h[7]\n" - "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n" + "fmla z8.h, z17.h, z1.h[7]\n" + "fmla z12.h, z17.h, z0.h[7]\n" + "ld1h { z17.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - "fmla z9.h, z7.h, z0.h[7]\n" - "fmla z13.h, z7.h, z1.h[7]\n" - "ld1h { z7.h }, p5/Z, [x9, #7, MUL VL]\n" + "fmla z9.h, z16.h, z1.h[7]\n" + "fmla z13.h, z16.h, z0.h[7]\n" + "ld1h { z16.h }, p5/Z, [x9, #7, MUL VL]\n" "addvl x9, x9, #8\n" - "fmla z10.h, z6.h, z0.h[7]\n" - "fmla z14.h, z6.h, z1.h[7]\n" - "fmla z11.h, z7.h, z0.h[7]\n" - "fmla z15.h, z7.h, z1.h[7]\n" + "fmla z10.h, z17.h, z1.h[7]\n" + "fmla z14.h, z17.h, z0.h[7]\n" + "fmla z11.h, z16.h, z1.h[7]\n" + "fmla z15.h, z16.h, z0.h[7]\n" "bgt 24b\n" "25:" // Height 2: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" "ld1rqh { z0.h }, p0/Z, [x26]\n" "ld1rqh { z1.h }, p0/Z, [x25]\n" "subs x27, x27, #0x1\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[0]\n" - "fmla z12.h, z6.h, z1.h[0]\n" - "fmla z9.h, z7.h, z0.h[0]\n" - "fmla z13.h, z7.h, z1.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" - "fmla z10.h, z6.h, z0.h[0]\n" - "fmla z14.h, z6.h, z1.h[0]\n" + "ld1h { z17.h }, p5/Z, [x12]\n" + "ld1h { z16.h }, p5/Z, [x11]\n" + "fmla z8.h, z17.h, z0.h[0]\n" + "fmla z12.h, z17.h, z1.h[0]\n" + "fmla z9.h, z16.h, z0.h[0]\n" + "fmla z13.h, z16.h, z1.h[0]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x9]\n" + "fmla z10.h, z17.h, z0.h[0]\n" + "fmla z14.h, z17.h, z1.h[0]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" - "fmla z11.h, z7.h, z0.h[0]\n" - "fmla z15.h, z7.h, z1.h[0]\n" + "fmla z11.h, z16.h, z0.h[0]\n" + "fmla z15.h, z16.h, z1.h[0]\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" "ble 26f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[1]\n" - "fmla z12.h, z6.h, z1.h[1]\n" - "fmla z9.h, z7.h, z0.h[1]\n" - "fmla z13.h, z7.h, z1.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "ld1h { z17.h }, p5/Z, [x12]\n" + "ld1h { z16.h }, p5/Z, [x11]\n" + "fmla z8.h, z17.h, z0.h[1]\n" + "fmla z12.h, z17.h, z1.h[1]\n" + "fmla z9.h, z16.h, z0.h[1]\n" + "fmla z13.h, z16.h, z1.h[1]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x9]\n" "subs x27, x27, #0x1\n" - "fmla z10.h, z6.h, z0.h[1]\n" - "fmla z14.h, z6.h, z1.h[1]\n" + "fmla z10.h, z17.h, z0.h[1]\n" + "fmla z14.h, z17.h, z1.h[1]\n" "addvl x12, x12, #1\n" - "fmla z11.h, z7.h, z0.h[1]\n" - "fmla z15.h, z7.h, z1.h[1]\n" + "fmla z11.h, z16.h, z0.h[1]\n" + "fmla z15.h, z16.h, z1.h[1]\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" "ble 26f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[2]\n" - "fmla z12.h, z6.h, z1.h[2]\n" - "fmla z9.h, z7.h, z0.h[2]\n" - "fmla z13.h, z7.h, z1.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "ld1h { z17.h }, p5/Z, [x12]\n" + "ld1h { z16.h }, p5/Z, [x11]\n" + "fmla z8.h, z17.h, z0.h[2]\n" + "fmla z12.h, z17.h, z1.h[2]\n" + "fmla z9.h, z16.h, z0.h[2]\n" + "fmla z13.h, z16.h, z1.h[2]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x9]\n" "subs x27, x27, #0x1\n" - "fmla z10.h, z6.h, z0.h[2]\n" - "fmla z14.h, z6.h, z1.h[2]\n" + "fmla z10.h, z17.h, z0.h[2]\n" + "fmla z14.h, z17.h, z1.h[2]\n" "addvl x12, x12, #1\n" - "fmla z11.h, z7.h, z0.h[2]\n" - "fmla z15.h, z7.h, z1.h[2]\n" + "fmla z11.h, z16.h, z0.h[2]\n" + "fmla z15.h, z16.h, z1.h[2]\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" "ble 26f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[3]\n" - "fmla z12.h, z6.h, z1.h[3]\n" - "fmla z9.h, z7.h, z0.h[3]\n" - "fmla z13.h, z7.h, z1.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "ld1h { z17.h }, p5/Z, [x12]\n" + "ld1h { z16.h }, p5/Z, [x11]\n" + "fmla z8.h, z17.h, z0.h[3]\n" + "fmla z12.h, z17.h, z1.h[3]\n" + "fmla z9.h, z16.h, z0.h[3]\n" + "fmla z13.h, z16.h, z1.h[3]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x9]\n" "subs x27, x27, #0x1\n" - "fmla z10.h, z6.h, z0.h[3]\n" - "fmla z14.h, z6.h, z1.h[3]\n" + "fmla z10.h, z17.h, z0.h[3]\n" + "fmla z14.h, z17.h, z1.h[3]\n" "addvl x12, x12, #1\n" - "fmla z11.h, z7.h, z0.h[3]\n" - "fmla z15.h, z7.h, z1.h[3]\n" + "fmla z11.h, z16.h, z0.h[3]\n" + "fmla z15.h, z16.h, z1.h[3]\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" "ble 26f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[4]\n" - "fmla z12.h, z6.h, z1.h[4]\n" - "fmla z9.h, z7.h, z0.h[4]\n" - "fmla z13.h, z7.h, z1.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "ld1h { z17.h }, p5/Z, [x12]\n" + "ld1h { z16.h }, p5/Z, [x11]\n" + "fmla z8.h, z17.h, z0.h[4]\n" + "fmla z12.h, z17.h, z1.h[4]\n" + "fmla z9.h, z16.h, z0.h[4]\n" + "fmla z13.h, z16.h, z1.h[4]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x9]\n" "subs x27, x27, #0x1\n" - "fmla z10.h, z6.h, z0.h[4]\n" - "fmla z14.h, z6.h, z1.h[4]\n" + "fmla z10.h, z17.h, z0.h[4]\n" + "fmla z14.h, z17.h, z1.h[4]\n" "addvl x12, x12, #1\n" - "fmla z11.h, z7.h, z0.h[4]\n" - "fmla z15.h, z7.h, z1.h[4]\n" + "fmla z11.h, z16.h, z0.h[4]\n" + "fmla z15.h, z16.h, z1.h[4]\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" "ble 26f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[5]\n" - "fmla z12.h, z6.h, z1.h[5]\n" - "fmla z9.h, z7.h, z0.h[5]\n" - "fmla z13.h, z7.h, z1.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "ld1h { z17.h }, p5/Z, [x12]\n" + "ld1h { z16.h }, p5/Z, [x11]\n" + "fmla z8.h, z17.h, z0.h[5]\n" + "fmla z12.h, z17.h, z1.h[5]\n" + "fmla z9.h, z16.h, z0.h[5]\n" + "fmla z13.h, z16.h, z1.h[5]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x9]\n" "subs x27, x27, #0x1\n" - "fmla z10.h, z6.h, z0.h[5]\n" - "fmla z14.h, z6.h, z1.h[5]\n" + "fmla z10.h, z17.h, z0.h[5]\n" + "fmla z14.h, z17.h, z1.h[5]\n" "addvl x12, x12, #1\n" - "fmla z11.h, z7.h, z0.h[5]\n" - "fmla z15.h, z7.h, z1.h[5]\n" + "fmla z11.h, z16.h, z0.h[5]\n" + "fmla z15.h, z16.h, z1.h[5]\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" "ble 26f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[6]\n" - "fmla z12.h, z6.h, z1.h[6]\n" - "fmla z9.h, z7.h, z0.h[6]\n" - "fmla z13.h, z7.h, z1.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "ld1h { z17.h }, p5/Z, [x12]\n" + "ld1h { z16.h }, p5/Z, [x11]\n" + "fmla z8.h, z17.h, z0.h[6]\n" + "fmla z12.h, z17.h, z1.h[6]\n" + "fmla z9.h, z16.h, z0.h[6]\n" + "fmla z13.h, z16.h, z1.h[6]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x9]\n" "subs x27, x27, #0x1\n" - "fmla z10.h, z6.h, z0.h[6]\n" - "fmla z14.h, z6.h, z1.h[6]\n" + "fmla z10.h, z17.h, z0.h[6]\n" + "fmla z14.h, z17.h, z1.h[6]\n" "addvl x12, x12, #1\n" - "fmla z11.h, z7.h, z0.h[6]\n" - "fmla z15.h, z7.h, z1.h[6]\n" + "fmla z11.h, z16.h, z0.h[6]\n" + "fmla z15.h, z16.h, z1.h[6]\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" "ble 26f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[7]\n" - "fmla z12.h, z6.h, z1.h[7]\n" - "fmla z9.h, z7.h, z0.h[7]\n" - "fmla z13.h, z7.h, z1.h[7]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" - "fmla z10.h, z6.h, z0.h[7]\n" - "fmla z14.h, z6.h, z1.h[7]\n" + "ld1h { z17.h }, p5/Z, [x12]\n" + "ld1h { z16.h }, p5/Z, [x11]\n" + "fmla z8.h, z17.h, z0.h[7]\n" + "fmla z12.h, z17.h, z1.h[7]\n" + "fmla z9.h, z16.h, z0.h[7]\n" + "fmla z13.h, z16.h, z1.h[7]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x9]\n" + "fmla z10.h, z17.h, z0.h[7]\n" + "fmla z14.h, z17.h, z1.h[7]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" - "fmla z11.h, z7.h, z0.h[7]\n" - "fmla z15.h, z7.h, z1.h[7]\n" + "fmla z11.h, z16.h, z0.h[7]\n" + "fmla z15.h, z16.h, z1.h[7]\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" "26:" // Height 2: Multiply loop: multiply skip @@ -748,25 +748,25 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "add x25, x13, x20, LSL #1\n" "tbz %x[flags], #1, 27f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rh { z1.h }, p5/Z, [x20]\n" + "ld1rh { z17.h }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rh { z0.h }, p5/Z, [x20]\n" - "fmin z8.h, p5/M, z8.h, z1.h\n" - "fmin z9.h, p5/M, z9.h, z1.h\n" - "fmin z10.h, p5/M, z10.h, z1.h\n" - "fmin z11.h, p5/M, z11.h, z1.h\n" - "fmin z12.h, p5/M, z12.h, z1.h\n" - "fmin z13.h, p5/M, z13.h, z1.h\n" - "fmin z14.h, p5/M, z14.h, z1.h\n" - "fmin z15.h, p5/M, z15.h, z1.h\n" - "fmax z8.h, p5/M, z8.h, z0.h\n" - "fmax z9.h, p5/M, z9.h, z0.h\n" - "fmax z10.h, p5/M, z10.h, z0.h\n" - "fmax z11.h, p5/M, z11.h, z0.h\n" - "fmax z12.h, p5/M, z12.h, z0.h\n" - "fmax z13.h, p5/M, z13.h, z0.h\n" - "fmax z14.h, p5/M, z14.h, z0.h\n" - "fmax z15.h, p5/M, z15.h, z0.h\n" + "ld1rh { z16.h }, p5/Z, [x20]\n" + "fmin z8.h, p5/M, z8.h, z17.h\n" + "fmin z9.h, p5/M, z9.h, z17.h\n" + "fmin z10.h, p5/M, z10.h, z17.h\n" + "fmin z11.h, p5/M, z11.h, z17.h\n" + "fmin z12.h, p5/M, z12.h, z17.h\n" + "fmin z13.h, p5/M, z13.h, z17.h\n" + "fmin z14.h, p5/M, z14.h, z17.h\n" + "fmin z15.h, p5/M, z15.h, z17.h\n" + "fmax z8.h, p5/M, z8.h, z16.h\n" + "fmax z9.h, p5/M, z9.h, z16.h\n" + "fmax z10.h, p5/M, z10.h, z16.h\n" + "fmax z11.h, p5/M, z11.h, z16.h\n" + "fmax z12.h, p5/M, z12.h, z16.h\n" + "fmax z13.h, p5/M, z13.h, z16.h\n" + "fmax z14.h, p5/M, z14.h, z16.h\n" + "fmax z15.h, p5/M, z15.h, z16.h\n" "27:" // Height 2: No activation "st1h { z8.h }, p4, [x13]\n" "st1h { z9.h }, p3, [x13, #1, MUL VL]\n" @@ -835,20 +835,20 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "32:" // Height 3: no bias "tbz %x[flags], #0, 33f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" + "add x21, x13, x20, LSL #1\n" + "add x20, x21, x20, LSL #1\n" "ld1h { z8.h }, p4/Z, [x13]\n" "ld1h { z9.h }, p3/Z, [x13, #1, MUL VL]\n" "ld1h { z10.h }, p2/Z, [x13, #2, MUL VL]\n" "ld1h { z11.h }, p1/Z, [x13, #3, MUL VL]\n" - "ld1h { z12.h }, p4/Z, [x25]\n" - "ld1h { z13.h }, p3/Z, [x25, #1, MUL VL]\n" - "ld1h { z14.h }, p2/Z, [x25, #2, MUL VL]\n" - "ld1h { z15.h }, p1/Z, [x25, #3, MUL VL]\n" - "ld1h { z16.h }, p4/Z, [x24]\n" - "ld1h { z17.h }, p3/Z, [x24, #1, MUL VL]\n" - "ld1h { z18.h }, p2/Z, [x24, #2, MUL VL]\n" - "ld1h { z19.h }, p1/Z, [x24, #3, MUL VL]\n" + "ld1h { z12.h }, p4/Z, [x21]\n" + "ld1h { z13.h }, p3/Z, [x21, #1, MUL VL]\n" + "ld1h { z14.h }, p2/Z, [x21, #2, MUL VL]\n" + "ld1h { z15.h }, p1/Z, [x21, #3, MUL VL]\n" + "ld1h { z16.h }, p4/Z, [x20]\n" + "ld1h { z17.h }, p3/Z, [x20, #1, MUL VL]\n" + "ld1h { z18.h }, p2/Z, [x20, #2, MUL VL]\n" + "ld1h { z19.h }, p1/Z, [x20, #3, MUL VL]\n" "b 34f\n" "33:" // Height 3: no accumulate "mov z8.b, #0x0\n" @@ -868,13 +868,13 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "35:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 36f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 37f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -883,153 +883,153 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "b 37f\n" "36:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" "37:" // Height 3: input setup done "cmp x27, #0x8\n" "ble 39f\n" "38:" // Height 3: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z0.h }, p0/Z, [x26]\n" + "ld1rqh { z2.h }, p0/Z, [x26]\n" "ld1rqh { z1.h }, p0/Z, [x25]\n" "sub x27, x27, #0x8\n" - "ld1rqh { z2.h }, p0/Z, [x24]\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "fmla z8.h, z6.h, z0.h[0]\n" - "fmla z12.h, z6.h, z1.h[0]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z16.h, z6.h, z2.h[0]\n" - "fmla z9.h, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "fmla z13.h, z7.h, z1.h[0]\n" - "fmla z17.h, z7.h, z2.h[0]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "ld1rqh { z0.h }, p0/Z, [x24]\n" + "ld1h { z21.h }, p5/Z, [x12]\n" + "fmla z8.h, z21.h, z2.h[0]\n" + "fmla z12.h, z21.h, z1.h[0]\n" + "ld1h { z20.h }, p5/Z, [x11]\n" + "fmla z16.h, z21.h, z0.h[0]\n" + "fmla z9.h, z20.h, z2.h[0]\n" + "ld1h { z21.h }, p5/Z, [x10]\n" + "fmla z13.h, z20.h, z1.h[0]\n" + "fmla z17.h, z20.h, z0.h[0]\n" + "ld1h { z20.h }, p5/Z, [x9]\n" "cmp x27, #0x8\n" - "fmla z10.h, z6.h, z0.h[0]\n" - "fmla z14.h, z6.h, z1.h[0]\n" + "fmla z10.h, z21.h, z2.h[0]\n" + "fmla z14.h, z21.h, z1.h[0]\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" - "fmla z18.h, z6.h, z2.h[0]\n" - "fmla z11.h, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n" + "fmla z18.h, z21.h, z0.h[0]\n" + "fmla z11.h, z20.h, z2.h[0]\n" + "ld1h { z21.h }, p5/Z, [x12, #1, MUL VL]\n" "add x24, x24, #0x10\n" - "fmla z15.h, z7.h, z1.h[0]\n" - "fmla z19.h, z7.h, z2.h[0]\n" - "ld1h { z7.h }, p5/Z, [x11, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[1]\n" - "fmla z12.h, z6.h, z1.h[1]\n" - "fmla z16.h, z6.h, z2.h[1]\n" - "fmla z9.h, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[1]\n" - "fmla z17.h, z7.h, z2.h[1]\n" - "ld1h { z7.h }, p5/Z, [x9, #1, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[1]\n" - "fmla z14.h, z6.h, z1.h[1]\n" - "fmla z18.h, z6.h, z2.h[1]\n" - "fmla z11.h, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x12, #2, MUL VL]\n" - "fmla z15.h, z7.h, z1.h[1]\n" - "fmla z19.h, z7.h, z2.h[1]\n" - "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[2]\n" - "fmla z12.h, z6.h, z1.h[2]\n" - "fmla z16.h, z6.h, z2.h[2]\n" - "fmla z9.h, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[2]\n" - "fmla z17.h, z7.h, z2.h[2]\n" - "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[2]\n" - "fmla z14.h, z6.h, z1.h[2]\n" - "fmla z18.h, z6.h, z2.h[2]\n" - "fmla z11.h, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x12, #3, MUL VL]\n" - "fmla z15.h, z7.h, z1.h[2]\n" - "fmla z19.h, z7.h, z2.h[2]\n" - "ld1h { z7.h }, p5/Z, [x11, #3, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[3]\n" - "fmla z12.h, z6.h, z1.h[3]\n" - "fmla z16.h, z6.h, z2.h[3]\n" - "fmla z9.h, z7.h, z0.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[3]\n" - "fmla z17.h, z7.h, z2.h[3]\n" - "ld1h { z7.h }, p5/Z, [x9, #3, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[3]\n" - "fmla z14.h, z6.h, z1.h[3]\n" - "fmla z18.h, z6.h, z2.h[3]\n" - "fmla z11.h, z7.h, z0.h[3]\n" - "ld1h { z6.h }, p5/Z, [x12, #4, MUL VL]\n" - "fmla z15.h, z7.h, z1.h[3]\n" - "fmla z19.h, z7.h, z2.h[3]\n" - "ld1h { z7.h }, p5/Z, [x11, #4, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[4]\n" - "fmla z12.h, z6.h, z1.h[4]\n" - "fmla z16.h, z6.h, z2.h[4]\n" - "fmla z9.h, z7.h, z0.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[4]\n" - "fmla z17.h, z7.h, z2.h[4]\n" - "ld1h { z7.h }, p5/Z, [x9, #4, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[4]\n" - "fmla z14.h, z6.h, z1.h[4]\n" - "fmla z18.h, z6.h, z2.h[4]\n" - "fmla z11.h, z7.h, z0.h[4]\n" - "ld1h { z6.h }, p5/Z, [x12, #5, MUL VL]\n" - "fmla z15.h, z7.h, z1.h[4]\n" - "fmla z19.h, z7.h, z2.h[4]\n" - "ld1h { z7.h }, p5/Z, [x11, #5, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[5]\n" - "fmla z12.h, z6.h, z1.h[5]\n" - "fmla z16.h, z6.h, z2.h[5]\n" - "fmla z9.h, z7.h, z0.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[5]\n" - "fmla z17.h, z7.h, z2.h[5]\n" - "ld1h { z7.h }, p5/Z, [x9, #5, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[5]\n" - "fmla z14.h, z6.h, z1.h[5]\n" - "fmla z18.h, z6.h, z2.h[5]\n" - "fmla z11.h, z7.h, z0.h[5]\n" - "ld1h { z6.h }, p5/Z, [x12, #6, MUL VL]\n" - "fmla z15.h, z7.h, z1.h[5]\n" - "fmla z19.h, z7.h, z2.h[5]\n" - "ld1h { z7.h }, p5/Z, [x11, #6, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[6]\n" - "fmla z12.h, z6.h, z1.h[6]\n" - "fmla z16.h, z6.h, z2.h[6]\n" - "fmla z9.h, z7.h, z0.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[6]\n" - "fmla z17.h, z7.h, z2.h[6]\n" - "ld1h { z7.h }, p5/Z, [x9, #6, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[6]\n" - "fmla z14.h, z6.h, z1.h[6]\n" - "fmla z18.h, z6.h, z2.h[6]\n" - "fmla z11.h, z7.h, z0.h[6]\n" - "ld1h { z6.h }, p5/Z, [x12, #7, MUL VL]\n" + "fmla z15.h, z20.h, z1.h[0]\n" + "fmla z19.h, z20.h, z0.h[0]\n" + "ld1h { z20.h }, p5/Z, [x11, #1, MUL VL]\n" + "fmla z8.h, z21.h, z2.h[1]\n" + "fmla z12.h, z21.h, z1.h[1]\n" + "fmla z16.h, z21.h, z0.h[1]\n" + "fmla z9.h, z20.h, z2.h[1]\n" + "ld1h { z21.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z13.h, z20.h, z1.h[1]\n" + "fmla z17.h, z20.h, z0.h[1]\n" + "ld1h { z20.h }, p5/Z, [x9, #1, MUL VL]\n" + "fmla z10.h, z21.h, z2.h[1]\n" + "fmla z14.h, z21.h, z1.h[1]\n" + "fmla z18.h, z21.h, z0.h[1]\n" + "fmla z11.h, z20.h, z2.h[1]\n" + "ld1h { z21.h }, p5/Z, [x12, #2, MUL VL]\n" + "fmla z15.h, z20.h, z1.h[1]\n" + "fmla z19.h, z20.h, z0.h[1]\n" + "ld1h { z20.h }, p5/Z, [x11, #2, MUL VL]\n" + "fmla z8.h, z21.h, z2.h[2]\n" + "fmla z12.h, z21.h, z1.h[2]\n" + "fmla z16.h, z21.h, z0.h[2]\n" + "fmla z9.h, z20.h, z2.h[2]\n" + "ld1h { z21.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z13.h, z20.h, z1.h[2]\n" + "fmla z17.h, z20.h, z0.h[2]\n" + "ld1h { z20.h }, p5/Z, [x9, #2, MUL VL]\n" + "fmla z10.h, z21.h, z2.h[2]\n" + "fmla z14.h, z21.h, z1.h[2]\n" + "fmla z18.h, z21.h, z0.h[2]\n" + "fmla z11.h, z20.h, z2.h[2]\n" + "ld1h { z21.h }, p5/Z, [x12, #3, MUL VL]\n" + "fmla z15.h, z20.h, z1.h[2]\n" + "fmla z19.h, z20.h, z0.h[2]\n" + "ld1h { z20.h }, p5/Z, [x11, #3, MUL VL]\n" + "fmla z8.h, z21.h, z2.h[3]\n" + "fmla z12.h, z21.h, z1.h[3]\n" + "fmla z16.h, z21.h, z0.h[3]\n" + "fmla z9.h, z20.h, z2.h[3]\n" + "ld1h { z21.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z13.h, z20.h, z1.h[3]\n" + "fmla z17.h, z20.h, z0.h[3]\n" + "ld1h { z20.h }, p5/Z, [x9, #3, MUL VL]\n" + "fmla z10.h, z21.h, z2.h[3]\n" + "fmla z14.h, z21.h, z1.h[3]\n" + "fmla z18.h, z21.h, z0.h[3]\n" + "fmla z11.h, z20.h, z2.h[3]\n" + "ld1h { z21.h }, p5/Z, [x12, #4, MUL VL]\n" + "fmla z15.h, z20.h, z1.h[3]\n" + "fmla z19.h, z20.h, z0.h[3]\n" + "ld1h { z20.h }, p5/Z, [x11, #4, MUL VL]\n" + "fmla z8.h, z21.h, z2.h[4]\n" + "fmla z12.h, z21.h, z1.h[4]\n" + "fmla z16.h, z21.h, z0.h[4]\n" + "fmla z9.h, z20.h, z2.h[4]\n" + "ld1h { z21.h }, p5/Z, [x10, #4, MUL VL]\n" + "fmla z13.h, z20.h, z1.h[4]\n" + "fmla z17.h, z20.h, z0.h[4]\n" + "ld1h { z20.h }, p5/Z, [x9, #4, MUL VL]\n" + "fmla z10.h, z21.h, z2.h[4]\n" + "fmla z14.h, z21.h, z1.h[4]\n" + "fmla z18.h, z21.h, z0.h[4]\n" + "fmla z11.h, z20.h, z2.h[4]\n" + "ld1h { z21.h }, p5/Z, [x12, #5, MUL VL]\n" + "fmla z15.h, z20.h, z1.h[4]\n" + "fmla z19.h, z20.h, z0.h[4]\n" + "ld1h { z20.h }, p5/Z, [x11, #5, MUL VL]\n" + "fmla z8.h, z21.h, z2.h[5]\n" + "fmla z12.h, z21.h, z1.h[5]\n" + "fmla z16.h, z21.h, z0.h[5]\n" + "fmla z9.h, z20.h, z2.h[5]\n" + "ld1h { z21.h }, p5/Z, [x10, #5, MUL VL]\n" + "fmla z13.h, z20.h, z1.h[5]\n" + "fmla z17.h, z20.h, z0.h[5]\n" + "ld1h { z20.h }, p5/Z, [x9, #5, MUL VL]\n" + "fmla z10.h, z21.h, z2.h[5]\n" + "fmla z14.h, z21.h, z1.h[5]\n" + "fmla z18.h, z21.h, z0.h[5]\n" + "fmla z11.h, z20.h, z2.h[5]\n" + "ld1h { z21.h }, p5/Z, [x12, #6, MUL VL]\n" + "fmla z15.h, z20.h, z1.h[5]\n" + "fmla z19.h, z20.h, z0.h[5]\n" + "ld1h { z20.h }, p5/Z, [x11, #6, MUL VL]\n" + "fmla z8.h, z21.h, z2.h[6]\n" + "fmla z12.h, z21.h, z1.h[6]\n" + "fmla z16.h, z21.h, z0.h[6]\n" + "fmla z9.h, z20.h, z2.h[6]\n" + "ld1h { z21.h }, p5/Z, [x10, #6, MUL VL]\n" + "fmla z13.h, z20.h, z1.h[6]\n" + "fmla z17.h, z20.h, z0.h[6]\n" + "ld1h { z20.h }, p5/Z, [x9, #6, MUL VL]\n" + "fmla z10.h, z21.h, z2.h[6]\n" + "fmla z14.h, z21.h, z1.h[6]\n" + "fmla z18.h, z21.h, z0.h[6]\n" + "fmla z11.h, z20.h, z2.h[6]\n" + "ld1h { z21.h }, p5/Z, [x12, #7, MUL VL]\n" "addvl x12, x12, #8\n" - "fmla z15.h, z7.h, z1.h[6]\n" - "fmla z19.h, z7.h, z2.h[6]\n" - "ld1h { z7.h }, p5/Z, [x11, #7, MUL VL]\n" + "fmla z15.h, z20.h, z1.h[6]\n" + "fmla z19.h, z20.h, z0.h[6]\n" + "ld1h { z20.h }, p5/Z, [x11, #7, MUL VL]\n" "addvl x11, x11, #8\n" - "fmla z8.h, z6.h, z0.h[7]\n" - "fmla z12.h, z6.h, z1.h[7]\n" - "fmla z16.h, z6.h, z2.h[7]\n" - "fmla z9.h, z7.h, z0.h[7]\n" - "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n" + "fmla z8.h, z21.h, z2.h[7]\n" + "fmla z12.h, z21.h, z1.h[7]\n" + "fmla z16.h, z21.h, z0.h[7]\n" + "fmla z9.h, z20.h, z2.h[7]\n" + "ld1h { z21.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - "fmla z13.h, z7.h, z1.h[7]\n" - "fmla z17.h, z7.h, z2.h[7]\n" - "ld1h { z7.h }, p5/Z, [x9, #7, MUL VL]\n" + "fmla z13.h, z20.h, z1.h[7]\n" + "fmla z17.h, z20.h, z0.h[7]\n" + "ld1h { z20.h }, p5/Z, [x9, #7, MUL VL]\n" "addvl x9, x9, #8\n" - "fmla z10.h, z6.h, z0.h[7]\n" - "fmla z14.h, z6.h, z1.h[7]\n" - "fmla z18.h, z6.h, z2.h[7]\n" - "fmla z11.h, z7.h, z0.h[7]\n" - "fmla z15.h, z7.h, z1.h[7]\n" - "fmla z19.h, z7.h, z2.h[7]\n" + "fmla z10.h, z21.h, z2.h[7]\n" + "fmla z14.h, z21.h, z1.h[7]\n" + "fmla z18.h, z21.h, z0.h[7]\n" + "fmla z11.h, z20.h, z2.h[7]\n" + "fmla z15.h, z20.h, z1.h[7]\n" + "fmla z19.h, z20.h, z0.h[7]\n" "bgt 38b\n" "39:" // Height 3: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" @@ -1037,179 +1037,179 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "ld1rqh { z1.h }, p0/Z, [x25]\n" "subs x27, x27, #0x1\n" "ld1rqh { z2.h }, p0/Z, [x24]\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "fmla z8.h, z6.h, z0.h[0]\n" - "fmla z12.h, z6.h, z1.h[0]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z16.h, z6.h, z2.h[0]\n" - "fmla z9.h, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "fmla z13.h, z7.h, z1.h[0]\n" - "fmla z17.h, z7.h, z2.h[0]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "ld1h { z21.h }, p5/Z, [x12]\n" + "fmla z8.h, z21.h, z0.h[0]\n" + "fmla z12.h, z21.h, z1.h[0]\n" + "ld1h { z20.h }, p5/Z, [x11]\n" + "fmla z16.h, z21.h, z2.h[0]\n" + "fmla z9.h, z20.h, z0.h[0]\n" + "ld1h { z21.h }, p5/Z, [x10]\n" + "fmla z13.h, z20.h, z1.h[0]\n" + "fmla z17.h, z20.h, z2.h[0]\n" + "ld1h { z20.h }, p5/Z, [x9]\n" "addvl x12, x12, #1\n" - "fmla z10.h, z6.h, z0.h[0]\n" - "fmla z14.h, z6.h, z1.h[0]\n" + "fmla z10.h, z21.h, z0.h[0]\n" + "fmla z14.h, z21.h, z1.h[0]\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" - "fmla z18.h, z6.h, z2.h[0]\n" - "fmla z11.h, z7.h, z0.h[0]\n" + "fmla z18.h, z21.h, z2.h[0]\n" + "fmla z11.h, z20.h, z0.h[0]\n" "addvl x9, x9, #1\n" - "fmla z15.h, z7.h, z1.h[0]\n" - "fmla z19.h, z7.h, z2.h[0]\n" + "fmla z15.h, z20.h, z1.h[0]\n" + "fmla z19.h, z20.h, z2.h[0]\n" "ble 40f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[1]\n" - "fmla z12.h, z6.h, z1.h[1]\n" - "fmla z16.h, z6.h, z2.h[1]\n" - "fmla z9.h, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "ld1h { z21.h }, p5/Z, [x12]\n" + "ld1h { z20.h }, p5/Z, [x11]\n" + "fmla z8.h, z21.h, z0.h[1]\n" + "fmla z12.h, z21.h, z1.h[1]\n" + "fmla z16.h, z21.h, z2.h[1]\n" + "fmla z9.h, z20.h, z0.h[1]\n" + "ld1h { z21.h }, p5/Z, [x10]\n" "subs x27, x27, #0x1\n" - "fmla z13.h, z7.h, z1.h[1]\n" - "fmla z17.h, z7.h, z2.h[1]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z13.h, z20.h, z1.h[1]\n" + "fmla z17.h, z20.h, z2.h[1]\n" + "ld1h { z20.h }, p5/Z, [x9]\n" "addvl x12, x12, #1\n" - "fmla z10.h, z6.h, z0.h[1]\n" - "fmla z14.h, z6.h, z1.h[1]\n" + "fmla z10.h, z21.h, z0.h[1]\n" + "fmla z14.h, z21.h, z1.h[1]\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" - "fmla z18.h, z6.h, z2.h[1]\n" - "fmla z11.h, z7.h, z0.h[1]\n" + "fmla z18.h, z21.h, z2.h[1]\n" + "fmla z11.h, z20.h, z0.h[1]\n" "addvl x9, x9, #1\n" - "fmla z15.h, z7.h, z1.h[1]\n" - "fmla z19.h, z7.h, z2.h[1]\n" + "fmla z15.h, z20.h, z1.h[1]\n" + "fmla z19.h, z20.h, z2.h[1]\n" "ble 40f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[2]\n" - "fmla z12.h, z6.h, z1.h[2]\n" - "fmla z16.h, z6.h, z2.h[2]\n" - "fmla z9.h, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "ld1h { z21.h }, p5/Z, [x12]\n" + "ld1h { z20.h }, p5/Z, [x11]\n" + "fmla z8.h, z21.h, z0.h[2]\n" + "fmla z12.h, z21.h, z1.h[2]\n" + "fmla z16.h, z21.h, z2.h[2]\n" + "fmla z9.h, z20.h, z0.h[2]\n" + "ld1h { z21.h }, p5/Z, [x10]\n" "subs x27, x27, #0x1\n" - "fmla z13.h, z7.h, z1.h[2]\n" - "fmla z17.h, z7.h, z2.h[2]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z13.h, z20.h, z1.h[2]\n" + "fmla z17.h, z20.h, z2.h[2]\n" + "ld1h { z20.h }, p5/Z, [x9]\n" "addvl x12, x12, #1\n" - "fmla z10.h, z6.h, z0.h[2]\n" - "fmla z14.h, z6.h, z1.h[2]\n" + "fmla z10.h, z21.h, z0.h[2]\n" + "fmla z14.h, z21.h, z1.h[2]\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" - "fmla z18.h, z6.h, z2.h[2]\n" - "fmla z11.h, z7.h, z0.h[2]\n" + "fmla z18.h, z21.h, z2.h[2]\n" + "fmla z11.h, z20.h, z0.h[2]\n" "addvl x9, x9, #1\n" - "fmla z15.h, z7.h, z1.h[2]\n" - "fmla z19.h, z7.h, z2.h[2]\n" + "fmla z15.h, z20.h, z1.h[2]\n" + "fmla z19.h, z20.h, z2.h[2]\n" "ble 40f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[3]\n" - "fmla z12.h, z6.h, z1.h[3]\n" - "fmla z16.h, z6.h, z2.h[3]\n" - "fmla z9.h, z7.h, z0.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "ld1h { z21.h }, p5/Z, [x12]\n" + "ld1h { z20.h }, p5/Z, [x11]\n" + "fmla z8.h, z21.h, z0.h[3]\n" + "fmla z12.h, z21.h, z1.h[3]\n" + "fmla z16.h, z21.h, z2.h[3]\n" + "fmla z9.h, z20.h, z0.h[3]\n" + "ld1h { z21.h }, p5/Z, [x10]\n" "subs x27, x27, #0x1\n" - "fmla z13.h, z7.h, z1.h[3]\n" - "fmla z17.h, z7.h, z2.h[3]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z13.h, z20.h, z1.h[3]\n" + "fmla z17.h, z20.h, z2.h[3]\n" + "ld1h { z20.h }, p5/Z, [x9]\n" "addvl x12, x12, #1\n" - "fmla z10.h, z6.h, z0.h[3]\n" - "fmla z14.h, z6.h, z1.h[3]\n" + "fmla z10.h, z21.h, z0.h[3]\n" + "fmla z14.h, z21.h, z1.h[3]\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" - "fmla z18.h, z6.h, z2.h[3]\n" - "fmla z11.h, z7.h, z0.h[3]\n" + "fmla z18.h, z21.h, z2.h[3]\n" + "fmla z11.h, z20.h, z0.h[3]\n" "addvl x9, x9, #1\n" - "fmla z15.h, z7.h, z1.h[3]\n" - "fmla z19.h, z7.h, z2.h[3]\n" + "fmla z15.h, z20.h, z1.h[3]\n" + "fmla z19.h, z20.h, z2.h[3]\n" "ble 40f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[4]\n" - "fmla z12.h, z6.h, z1.h[4]\n" - "fmla z16.h, z6.h, z2.h[4]\n" - "fmla z9.h, z7.h, z0.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "ld1h { z21.h }, p5/Z, [x12]\n" + "ld1h { z20.h }, p5/Z, [x11]\n" + "fmla z8.h, z21.h, z0.h[4]\n" + "fmla z12.h, z21.h, z1.h[4]\n" + "fmla z16.h, z21.h, z2.h[4]\n" + "fmla z9.h, z20.h, z0.h[4]\n" + "ld1h { z21.h }, p5/Z, [x10]\n" "subs x27, x27, #0x1\n" - "fmla z13.h, z7.h, z1.h[4]\n" - "fmla z17.h, z7.h, z2.h[4]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z13.h, z20.h, z1.h[4]\n" + "fmla z17.h, z20.h, z2.h[4]\n" + "ld1h { z20.h }, p5/Z, [x9]\n" "addvl x12, x12, #1\n" - "fmla z10.h, z6.h, z0.h[4]\n" - "fmla z14.h, z6.h, z1.h[4]\n" + "fmla z10.h, z21.h, z0.h[4]\n" + "fmla z14.h, z21.h, z1.h[4]\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" - "fmla z18.h, z6.h, z2.h[4]\n" - "fmla z11.h, z7.h, z0.h[4]\n" + "fmla z18.h, z21.h, z2.h[4]\n" + "fmla z11.h, z20.h, z0.h[4]\n" "addvl x9, x9, #1\n" - "fmla z15.h, z7.h, z1.h[4]\n" - "fmla z19.h, z7.h, z2.h[4]\n" + "fmla z15.h, z20.h, z1.h[4]\n" + "fmla z19.h, z20.h, z2.h[4]\n" "ble 40f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[5]\n" - "fmla z12.h, z6.h, z1.h[5]\n" - "fmla z16.h, z6.h, z2.h[5]\n" - "fmla z9.h, z7.h, z0.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "ld1h { z21.h }, p5/Z, [x12]\n" + "ld1h { z20.h }, p5/Z, [x11]\n" + "fmla z8.h, z21.h, z0.h[5]\n" + "fmla z12.h, z21.h, z1.h[5]\n" + "fmla z16.h, z21.h, z2.h[5]\n" + "fmla z9.h, z20.h, z0.h[5]\n" + "ld1h { z21.h }, p5/Z, [x10]\n" "subs x27, x27, #0x1\n" - "fmla z13.h, z7.h, z1.h[5]\n" - "fmla z17.h, z7.h, z2.h[5]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z13.h, z20.h, z1.h[5]\n" + "fmla z17.h, z20.h, z2.h[5]\n" + "ld1h { z20.h }, p5/Z, [x9]\n" "addvl x12, x12, #1\n" - "fmla z10.h, z6.h, z0.h[5]\n" - "fmla z14.h, z6.h, z1.h[5]\n" + "fmla z10.h, z21.h, z0.h[5]\n" + "fmla z14.h, z21.h, z1.h[5]\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" - "fmla z18.h, z6.h, z2.h[5]\n" - "fmla z11.h, z7.h, z0.h[5]\n" + "fmla z18.h, z21.h, z2.h[5]\n" + "fmla z11.h, z20.h, z0.h[5]\n" "addvl x9, x9, #1\n" - "fmla z15.h, z7.h, z1.h[5]\n" - "fmla z19.h, z7.h, z2.h[5]\n" + "fmla z15.h, z20.h, z1.h[5]\n" + "fmla z19.h, z20.h, z2.h[5]\n" "ble 40f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[6]\n" - "fmla z12.h, z6.h, z1.h[6]\n" - "fmla z16.h, z6.h, z2.h[6]\n" - "fmla z9.h, z7.h, z0.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "ld1h { z21.h }, p5/Z, [x12]\n" + "ld1h { z20.h }, p5/Z, [x11]\n" + "fmla z8.h, z21.h, z0.h[6]\n" + "fmla z12.h, z21.h, z1.h[6]\n" + "fmla z16.h, z21.h, z2.h[6]\n" + "fmla z9.h, z20.h, z0.h[6]\n" + "ld1h { z21.h }, p5/Z, [x10]\n" "subs x27, x27, #0x1\n" - "fmla z13.h, z7.h, z1.h[6]\n" - "fmla z17.h, z7.h, z2.h[6]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z13.h, z20.h, z1.h[6]\n" + "fmla z17.h, z20.h, z2.h[6]\n" + "ld1h { z20.h }, p5/Z, [x9]\n" "addvl x12, x12, #1\n" - "fmla z10.h, z6.h, z0.h[6]\n" - "fmla z14.h, z6.h, z1.h[6]\n" + "fmla z10.h, z21.h, z0.h[6]\n" + "fmla z14.h, z21.h, z1.h[6]\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" - "fmla z18.h, z6.h, z2.h[6]\n" - "fmla z11.h, z7.h, z0.h[6]\n" + "fmla z18.h, z21.h, z2.h[6]\n" + "fmla z11.h, z20.h, z0.h[6]\n" "addvl x9, x9, #1\n" - "fmla z15.h, z7.h, z1.h[6]\n" - "fmla z19.h, z7.h, z2.h[6]\n" + "fmla z15.h, z20.h, z1.h[6]\n" + "fmla z19.h, z20.h, z2.h[6]\n" "ble 40f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[7]\n" - "fmla z12.h, z6.h, z1.h[7]\n" - "fmla z16.h, z6.h, z2.h[7]\n" - "fmla z9.h, z7.h, z0.h[7]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "ld1h { z21.h }, p5/Z, [x12]\n" + "ld1h { z20.h }, p5/Z, [x11]\n" + "fmla z8.h, z21.h, z0.h[7]\n" + "fmla z12.h, z21.h, z1.h[7]\n" + "fmla z16.h, z21.h, z2.h[7]\n" + "fmla z9.h, z20.h, z0.h[7]\n" + "ld1h { z21.h }, p5/Z, [x10]\n" "addvl x12, x12, #1\n" - "fmla z13.h, z7.h, z1.h[7]\n" - "fmla z17.h, z7.h, z2.h[7]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z13.h, z20.h, z1.h[7]\n" + "fmla z17.h, z20.h, z2.h[7]\n" + "ld1h { z20.h }, p5/Z, [x9]\n" "addvl x11, x11, #1\n" - "fmla z10.h, z6.h, z0.h[7]\n" - "fmla z14.h, z6.h, z1.h[7]\n" + "fmla z10.h, z21.h, z0.h[7]\n" + "fmla z14.h, z21.h, z1.h[7]\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" - "fmla z18.h, z6.h, z2.h[7]\n" - "fmla z11.h, z7.h, z0.h[7]\n" - "fmla z15.h, z7.h, z1.h[7]\n" - "fmla z19.h, z7.h, z2.h[7]\n" + "fmla z18.h, z21.h, z2.h[7]\n" + "fmla z11.h, z20.h, z0.h[7]\n" + "fmla z15.h, z20.h, z1.h[7]\n" + "fmla z19.h, z20.h, z2.h[7]\n" "40:" // Height 3: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1220,33 +1220,33 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "add x24, x25, x20, LSL #1\n" "tbz %x[flags], #1, 41f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rh { z1.h }, p5/Z, [x20]\n" + "ld1rh { z21.h }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rh { z0.h }, p5/Z, [x20]\n" - "fmin z8.h, p5/M, z8.h, z1.h\n" - "fmin z9.h, p5/M, z9.h, z1.h\n" - "fmin z10.h, p5/M, z10.h, z1.h\n" - "fmin z11.h, p5/M, z11.h, z1.h\n" - "fmin z12.h, p5/M, z12.h, z1.h\n" - "fmin z13.h, p5/M, z13.h, z1.h\n" - "fmin z14.h, p5/M, z14.h, z1.h\n" - "fmin z15.h, p5/M, z15.h, z1.h\n" - "fmin z16.h, p5/M, z16.h, z1.h\n" - "fmin z17.h, p5/M, z17.h, z1.h\n" - "fmin z18.h, p5/M, z18.h, z1.h\n" - "fmin z19.h, p5/M, z19.h, z1.h\n" - "fmax z8.h, p5/M, z8.h, z0.h\n" - "fmax z9.h, p5/M, z9.h, z0.h\n" - "fmax z10.h, p5/M, z10.h, z0.h\n" - "fmax z11.h, p5/M, z11.h, z0.h\n" - "fmax z12.h, p5/M, z12.h, z0.h\n" - "fmax z13.h, p5/M, z13.h, z0.h\n" - "fmax z14.h, p5/M, z14.h, z0.h\n" - "fmax z15.h, p5/M, z15.h, z0.h\n" - "fmax z16.h, p5/M, z16.h, z0.h\n" - "fmax z17.h, p5/M, z17.h, z0.h\n" - "fmax z18.h, p5/M, z18.h, z0.h\n" - "fmax z19.h, p5/M, z19.h, z0.h\n" + "ld1rh { z20.h }, p5/Z, [x20]\n" + "fmin z8.h, p5/M, z8.h, z21.h\n" + "fmin z9.h, p5/M, z9.h, z21.h\n" + "fmin z10.h, p5/M, z10.h, z21.h\n" + "fmin z11.h, p5/M, z11.h, z21.h\n" + "fmin z12.h, p5/M, z12.h, z21.h\n" + "fmin z13.h, p5/M, z13.h, z21.h\n" + "fmin z14.h, p5/M, z14.h, z21.h\n" + "fmin z15.h, p5/M, z15.h, z21.h\n" + "fmin z16.h, p5/M, z16.h, z21.h\n" + "fmin z17.h, p5/M, z17.h, z21.h\n" + "fmin z18.h, p5/M, z18.h, z21.h\n" + "fmin z19.h, p5/M, z19.h, z21.h\n" + "fmax z8.h, p5/M, z8.h, z20.h\n" + "fmax z9.h, p5/M, z9.h, z20.h\n" + "fmax z10.h, p5/M, z10.h, z20.h\n" + "fmax z11.h, p5/M, z11.h, z20.h\n" + "fmax z12.h, p5/M, z12.h, z20.h\n" + "fmax z13.h, p5/M, z13.h, z20.h\n" + "fmax z14.h, p5/M, z14.h, z20.h\n" + "fmax z15.h, p5/M, z15.h, z20.h\n" + "fmax z16.h, p5/M, z16.h, z20.h\n" + "fmax z17.h, p5/M, z17.h, z20.h\n" + "fmax z18.h, p5/M, z18.h, z20.h\n" + "fmax z19.h, p5/M, z19.h, z20.h\n" "41:" // Height 3: No activation "st1h { z8.h }, p4, [x13]\n" "st1h { z9.h }, p3, [x13, #1, MUL VL]\n" @@ -1323,25 +1323,25 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "46:" // Height 4: no bias "tbz %x[flags], #0, 47f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" + "add x22, x13, x20, LSL #1\n" + "add x21, x22, x20, LSL #1\n" "ld1h { z8.h }, p4/Z, [x13]\n" - "add x23, x24, x20, LSL #1\n" + "add x20, x21, x20, LSL #1\n" "ld1h { z9.h }, p3/Z, [x13, #1, MUL VL]\n" "ld1h { z10.h }, p2/Z, [x13, #2, MUL VL]\n" "ld1h { z11.h }, p1/Z, [x13, #3, MUL VL]\n" - "ld1h { z12.h }, p4/Z, [x25]\n" - "ld1h { z13.h }, p3/Z, [x25, #1, MUL VL]\n" - "ld1h { z14.h }, p2/Z, [x25, #2, MUL VL]\n" - "ld1h { z15.h }, p1/Z, [x25, #3, MUL VL]\n" - "ld1h { z16.h }, p4/Z, [x24]\n" - "ld1h { z17.h }, p3/Z, [x24, #1, MUL VL]\n" - "ld1h { z18.h }, p2/Z, [x24, #2, MUL VL]\n" - "ld1h { z19.h }, p1/Z, [x24, #3, MUL VL]\n" - "ld1h { z20.h }, p4/Z, [x23]\n" - "ld1h { z21.h }, p3/Z, [x23, #1, MUL VL]\n" - "ld1h { z22.h }, p2/Z, [x23, #2, MUL VL]\n" - "ld1h { z23.h }, p1/Z, [x23, #3, MUL VL]\n" + "ld1h { z12.h }, p4/Z, [x22]\n" + "ld1h { z13.h }, p3/Z, [x22, #1, MUL VL]\n" + "ld1h { z14.h }, p2/Z, [x22, #2, MUL VL]\n" + "ld1h { z15.h }, p1/Z, [x22, #3, MUL VL]\n" + "ld1h { z16.h }, p4/Z, [x21]\n" + "ld1h { z17.h }, p3/Z, [x21, #1, MUL VL]\n" + "ld1h { z18.h }, p2/Z, [x21, #2, MUL VL]\n" + "ld1h { z19.h }, p1/Z, [x21, #3, MUL VL]\n" + "ld1h { z20.h }, p4/Z, [x20]\n" + "ld1h { z21.h }, p3/Z, [x20, #1, MUL VL]\n" + "ld1h { z22.h }, p2/Z, [x20, #2, MUL VL]\n" + "ld1h { z23.h }, p1/Z, [x20, #3, MUL VL]\n" "b 48f\n" "47:" // Height 4: no accumulate "mov z8.b, #0x0\n" @@ -1365,14 +1365,14 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "49:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 50f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 51f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -1382,188 +1382,188 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "b 51f\n" "50:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" "51:" // Height 4: input setup done "cmp x27, #0x8\n" "ble 53f\n" "52:" // Height 4: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z0.h }, p0/Z, [x26]\n" - "ld1rqh { z1.h }, p0/Z, [x25]\n" + "ld1rqh { z3.h }, p0/Z, [x26]\n" + "ld1rqh { z2.h }, p0/Z, [x25]\n" "sub x27, x27, #0x8\n" - "ld1rqh { z2.h }, p0/Z, [x24]\n" - "ld1rqh { z3.h }, p0/Z, [x23]\n" + "ld1rqh { z1.h }, p0/Z, [x24]\n" + "ld1rqh { z0.h }, p0/Z, [x23]\n" "cmp x27, #0x8\n" "add x26, x26, #0x10\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[0]\n" - "fmla z12.h, z6.h, z1.h[0]\n" - "fmla z16.h, z6.h, z2.h[0]\n" - "fmla z20.h, z6.h, z3.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "ld1h { z25.h }, p5/Z, [x12]\n" + "ld1h { z24.h }, p5/Z, [x11]\n" + "fmla z8.h, z25.h, z3.h[0]\n" + "fmla z12.h, z25.h, z2.h[0]\n" + "fmla z16.h, z25.h, z1.h[0]\n" + "fmla z20.h, z25.h, z0.h[0]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" "add x25, x25, #0x10\n" - "fmla z9.h, z7.h, z0.h[0]\n" - "fmla z13.h, z7.h, z1.h[0]\n" + "fmla z9.h, z24.h, z3.h[0]\n" + "fmla z13.h, z24.h, z2.h[0]\n" "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" - "fmla z17.h, z7.h, z2.h[0]\n" - "fmla z21.h, z7.h, z3.h[0]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" - "fmla z10.h, z6.h, z0.h[0]\n" - "fmla z14.h, z6.h, z1.h[0]\n" - "fmla z18.h, z6.h, z2.h[0]\n" - "fmla z22.h, z6.h, z3.h[0]\n" - "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[0]\n" - "fmla z15.h, z7.h, z1.h[0]\n" - "fmla z19.h, z7.h, z2.h[0]\n" - "fmla z23.h, z7.h, z3.h[0]\n" - "ld1h { z7.h }, p5/Z, [x11, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[1]\n" - "fmla z12.h, z6.h, z1.h[1]\n" - "fmla z16.h, z6.h, z2.h[1]\n" - "fmla z20.h, z6.h, z3.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[1]\n" - "fmla z13.h, z7.h, z1.h[1]\n" - "fmla z17.h, z7.h, z2.h[1]\n" - "fmla z21.h, z7.h, z3.h[1]\n" - "ld1h { z7.h }, p5/Z, [x9, #1, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[1]\n" - "fmla z14.h, z6.h, z1.h[1]\n" - "fmla z18.h, z6.h, z2.h[1]\n" - "fmla z22.h, z6.h, z3.h[1]\n" - "ld1h { z6.h }, p5/Z, [x12, #2, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[1]\n" - "fmla z15.h, z7.h, z1.h[1]\n" - "fmla z19.h, z7.h, z2.h[1]\n" - "fmla z23.h, z7.h, z3.h[1]\n" - "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[2]\n" - "fmla z12.h, z6.h, z1.h[2]\n" - "fmla z16.h, z6.h, z2.h[2]\n" - "fmla z20.h, z6.h, z3.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[2]\n" - "fmla z13.h, z7.h, z1.h[2]\n" - "fmla z17.h, z7.h, z2.h[2]\n" - "fmla z21.h, z7.h, z3.h[2]\n" - "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[2]\n" - "fmla z14.h, z6.h, z1.h[2]\n" - "fmla z18.h, z6.h, z2.h[2]\n" - "fmla z22.h, z6.h, z3.h[2]\n" - "ld1h { z6.h }, p5/Z, [x12, #3, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[2]\n" - "fmla z15.h, z7.h, z1.h[2]\n" - "fmla z19.h, z7.h, z2.h[2]\n" - "fmla z23.h, z7.h, z3.h[2]\n" - "ld1h { z7.h }, p5/Z, [x11, #3, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[3]\n" - "fmla z12.h, z6.h, z1.h[3]\n" - "fmla z16.h, z6.h, z2.h[3]\n" - "fmla z20.h, z6.h, z3.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[3]\n" - "fmla z13.h, z7.h, z1.h[3]\n" - "fmla z17.h, z7.h, z2.h[3]\n" - "fmla z21.h, z7.h, z3.h[3]\n" - "ld1h { z7.h }, p5/Z, [x9, #3, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[3]\n" - "fmla z14.h, z6.h, z1.h[3]\n" - "fmla z18.h, z6.h, z2.h[3]\n" - "fmla z22.h, z6.h, z3.h[3]\n" - "ld1h { z6.h }, p5/Z, [x12, #4, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[3]\n" - "fmla z15.h, z7.h, z1.h[3]\n" - "fmla z19.h, z7.h, z2.h[3]\n" - "fmla z23.h, z7.h, z3.h[3]\n" - "ld1h { z7.h }, p5/Z, [x11, #4, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[4]\n" - "fmla z12.h, z6.h, z1.h[4]\n" - "fmla z16.h, z6.h, z2.h[4]\n" - "fmla z20.h, z6.h, z3.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[4]\n" - "fmla z13.h, z7.h, z1.h[4]\n" - "fmla z17.h, z7.h, z2.h[4]\n" - "fmla z21.h, z7.h, z3.h[4]\n" - "ld1h { z7.h }, p5/Z, [x9, #4, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[4]\n" - "fmla z14.h, z6.h, z1.h[4]\n" - "fmla z18.h, z6.h, z2.h[4]\n" - "fmla z22.h, z6.h, z3.h[4]\n" - "ld1h { z6.h }, p5/Z, [x12, #5, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[4]\n" - "fmla z15.h, z7.h, z1.h[4]\n" - "fmla z19.h, z7.h, z2.h[4]\n" - "fmla z23.h, z7.h, z3.h[4]\n" - "ld1h { z7.h }, p5/Z, [x11, #5, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[5]\n" - "fmla z12.h, z6.h, z1.h[5]\n" - "fmla z16.h, z6.h, z2.h[5]\n" - "fmla z20.h, z6.h, z3.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[5]\n" - "fmla z13.h, z7.h, z1.h[5]\n" - "fmla z17.h, z7.h, z2.h[5]\n" - "fmla z21.h, z7.h, z3.h[5]\n" - "ld1h { z7.h }, p5/Z, [x9, #5, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[5]\n" - "fmla z14.h, z6.h, z1.h[5]\n" - "fmla z18.h, z6.h, z2.h[5]\n" - "fmla z22.h, z6.h, z3.h[5]\n" - "ld1h { z6.h }, p5/Z, [x12, #6, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[5]\n" - "fmla z15.h, z7.h, z1.h[5]\n" - "fmla z19.h, z7.h, z2.h[5]\n" - "fmla z23.h, z7.h, z3.h[5]\n" - "ld1h { z7.h }, p5/Z, [x11, #6, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[6]\n" - "fmla z12.h, z6.h, z1.h[6]\n" - "fmla z16.h, z6.h, z2.h[6]\n" - "fmla z20.h, z6.h, z3.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[6]\n" - "fmla z13.h, z7.h, z1.h[6]\n" - "fmla z17.h, z7.h, z2.h[6]\n" - "fmla z21.h, z7.h, z3.h[6]\n" - "ld1h { z7.h }, p5/Z, [x9, #6, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[6]\n" - "fmla z14.h, z6.h, z1.h[6]\n" - "fmla z18.h, z6.h, z2.h[6]\n" - "fmla z22.h, z6.h, z3.h[6]\n" - "ld1h { z6.h }, p5/Z, [x12, #7, MUL VL]\n" + "fmla z17.h, z24.h, z1.h[0]\n" + "fmla z21.h, z24.h, z0.h[0]\n" + "ld1h { z24.h }, p5/Z, [x9]\n" + "fmla z10.h, z25.h, z3.h[0]\n" + "fmla z14.h, z25.h, z2.h[0]\n" + "fmla z18.h, z25.h, z1.h[0]\n" + "fmla z22.h, z25.h, z0.h[0]\n" + "ld1h { z25.h }, p5/Z, [x12, #1, MUL VL]\n" + "fmla z11.h, z24.h, z3.h[0]\n" + "fmla z15.h, z24.h, z2.h[0]\n" + "fmla z19.h, z24.h, z1.h[0]\n" + "fmla z23.h, z24.h, z0.h[0]\n" + "ld1h { z24.h }, p5/Z, [x11, #1, MUL VL]\n" + "fmla z8.h, z25.h, z3.h[1]\n" + "fmla z12.h, z25.h, z2.h[1]\n" + "fmla z16.h, z25.h, z1.h[1]\n" + "fmla z20.h, z25.h, z0.h[1]\n" + "ld1h { z25.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z9.h, z24.h, z3.h[1]\n" + "fmla z13.h, z24.h, z2.h[1]\n" + "fmla z17.h, z24.h, z1.h[1]\n" + "fmla z21.h, z24.h, z0.h[1]\n" + "ld1h { z24.h }, p5/Z, [x9, #1, MUL VL]\n" + "fmla z10.h, z25.h, z3.h[1]\n" + "fmla z14.h, z25.h, z2.h[1]\n" + "fmla z18.h, z25.h, z1.h[1]\n" + "fmla z22.h, z25.h, z0.h[1]\n" + "ld1h { z25.h }, p5/Z, [x12, #2, MUL VL]\n" + "fmla z11.h, z24.h, z3.h[1]\n" + "fmla z15.h, z24.h, z2.h[1]\n" + "fmla z19.h, z24.h, z1.h[1]\n" + "fmla z23.h, z24.h, z0.h[1]\n" + "ld1h { z24.h }, p5/Z, [x11, #2, MUL VL]\n" + "fmla z8.h, z25.h, z3.h[2]\n" + "fmla z12.h, z25.h, z2.h[2]\n" + "fmla z16.h, z25.h, z1.h[2]\n" + "fmla z20.h, z25.h, z0.h[2]\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.h, z24.h, z3.h[2]\n" + "fmla z13.h, z24.h, z2.h[2]\n" + "fmla z17.h, z24.h, z1.h[2]\n" + "fmla z21.h, z24.h, z0.h[2]\n" + "ld1h { z24.h }, p5/Z, [x9, #2, MUL VL]\n" + "fmla z10.h, z25.h, z3.h[2]\n" + "fmla z14.h, z25.h, z2.h[2]\n" + "fmla z18.h, z25.h, z1.h[2]\n" + "fmla z22.h, z25.h, z0.h[2]\n" + "ld1h { z25.h }, p5/Z, [x12, #3, MUL VL]\n" + "fmla z11.h, z24.h, z3.h[2]\n" + "fmla z15.h, z24.h, z2.h[2]\n" + "fmla z19.h, z24.h, z1.h[2]\n" + "fmla z23.h, z24.h, z0.h[2]\n" + "ld1h { z24.h }, p5/Z, [x11, #3, MUL VL]\n" + "fmla z8.h, z25.h, z3.h[3]\n" + "fmla z12.h, z25.h, z2.h[3]\n" + "fmla z16.h, z25.h, z1.h[3]\n" + "fmla z20.h, z25.h, z0.h[3]\n" + "ld1h { z25.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z9.h, z24.h, z3.h[3]\n" + "fmla z13.h, z24.h, z2.h[3]\n" + "fmla z17.h, z24.h, z1.h[3]\n" + "fmla z21.h, z24.h, z0.h[3]\n" + "ld1h { z24.h }, p5/Z, [x9, #3, MUL VL]\n" + "fmla z10.h, z25.h, z3.h[3]\n" + "fmla z14.h, z25.h, z2.h[3]\n" + "fmla z18.h, z25.h, z1.h[3]\n" + "fmla z22.h, z25.h, z0.h[3]\n" + "ld1h { z25.h }, p5/Z, [x12, #4, MUL VL]\n" + "fmla z11.h, z24.h, z3.h[3]\n" + "fmla z15.h, z24.h, z2.h[3]\n" + "fmla z19.h, z24.h, z1.h[3]\n" + "fmla z23.h, z24.h, z0.h[3]\n" + "ld1h { z24.h }, p5/Z, [x11, #4, MUL VL]\n" + "fmla z8.h, z25.h, z3.h[4]\n" + "fmla z12.h, z25.h, z2.h[4]\n" + "fmla z16.h, z25.h, z1.h[4]\n" + "fmla z20.h, z25.h, z0.h[4]\n" + "ld1h { z25.h }, p5/Z, [x10, #4, MUL VL]\n" + "fmla z9.h, z24.h, z3.h[4]\n" + "fmla z13.h, z24.h, z2.h[4]\n" + "fmla z17.h, z24.h, z1.h[4]\n" + "fmla z21.h, z24.h, z0.h[4]\n" + "ld1h { z24.h }, p5/Z, [x9, #4, MUL VL]\n" + "fmla z10.h, z25.h, z3.h[4]\n" + "fmla z14.h, z25.h, z2.h[4]\n" + "fmla z18.h, z25.h, z1.h[4]\n" + "fmla z22.h, z25.h, z0.h[4]\n" + "ld1h { z25.h }, p5/Z, [x12, #5, MUL VL]\n" + "fmla z11.h, z24.h, z3.h[4]\n" + "fmla z15.h, z24.h, z2.h[4]\n" + "fmla z19.h, z24.h, z1.h[4]\n" + "fmla z23.h, z24.h, z0.h[4]\n" + "ld1h { z24.h }, p5/Z, [x11, #5, MUL VL]\n" + "fmla z8.h, z25.h, z3.h[5]\n" + "fmla z12.h, z25.h, z2.h[5]\n" + "fmla z16.h, z25.h, z1.h[5]\n" + "fmla z20.h, z25.h, z0.h[5]\n" + "ld1h { z25.h }, p5/Z, [x10, #5, MUL VL]\n" + "fmla z9.h, z24.h, z3.h[5]\n" + "fmla z13.h, z24.h, z2.h[5]\n" + "fmla z17.h, z24.h, z1.h[5]\n" + "fmla z21.h, z24.h, z0.h[5]\n" + "ld1h { z24.h }, p5/Z, [x9, #5, MUL VL]\n" + "fmla z10.h, z25.h, z3.h[5]\n" + "fmla z14.h, z25.h, z2.h[5]\n" + "fmla z18.h, z25.h, z1.h[5]\n" + "fmla z22.h, z25.h, z0.h[5]\n" + "ld1h { z25.h }, p5/Z, [x12, #6, MUL VL]\n" + "fmla z11.h, z24.h, z3.h[5]\n" + "fmla z15.h, z24.h, z2.h[5]\n" + "fmla z19.h, z24.h, z1.h[5]\n" + "fmla z23.h, z24.h, z0.h[5]\n" + "ld1h { z24.h }, p5/Z, [x11, #6, MUL VL]\n" + "fmla z8.h, z25.h, z3.h[6]\n" + "fmla z12.h, z25.h, z2.h[6]\n" + "fmla z16.h, z25.h, z1.h[6]\n" + "fmla z20.h, z25.h, z0.h[6]\n" + "ld1h { z25.h }, p5/Z, [x10, #6, MUL VL]\n" + "fmla z9.h, z24.h, z3.h[6]\n" + "fmla z13.h, z24.h, z2.h[6]\n" + "fmla z17.h, z24.h, z1.h[6]\n" + "fmla z21.h, z24.h, z0.h[6]\n" + "ld1h { z24.h }, p5/Z, [x9, #6, MUL VL]\n" + "fmla z10.h, z25.h, z3.h[6]\n" + "fmla z14.h, z25.h, z2.h[6]\n" + "fmla z18.h, z25.h, z1.h[6]\n" + "fmla z22.h, z25.h, z0.h[6]\n" + "ld1h { z25.h }, p5/Z, [x12, #7, MUL VL]\n" "addvl x12, x12, #8\n" - "fmla z11.h, z7.h, z0.h[6]\n" - "fmla z15.h, z7.h, z1.h[6]\n" - "fmla z19.h, z7.h, z2.h[6]\n" - "fmla z23.h, z7.h, z3.h[6]\n" - "ld1h { z7.h }, p5/Z, [x11, #7, MUL VL]\n" + "fmla z11.h, z24.h, z3.h[6]\n" + "fmla z15.h, z24.h, z2.h[6]\n" + "fmla z19.h, z24.h, z1.h[6]\n" + "fmla z23.h, z24.h, z0.h[6]\n" + "ld1h { z24.h }, p5/Z, [x11, #7, MUL VL]\n" "addvl x11, x11, #8\n" - "fmla z8.h, z6.h, z0.h[7]\n" - "fmla z12.h, z6.h, z1.h[7]\n" - "fmla z16.h, z6.h, z2.h[7]\n" - "fmla z20.h, z6.h, z3.h[7]\n" - "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n" + "fmla z8.h, z25.h, z3.h[7]\n" + "fmla z12.h, z25.h, z2.h[7]\n" + "fmla z16.h, z25.h, z1.h[7]\n" + "fmla z20.h, z25.h, z0.h[7]\n" + "ld1h { z25.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - "fmla z9.h, z7.h, z0.h[7]\n" - "fmla z13.h, z7.h, z1.h[7]\n" - "fmla z17.h, z7.h, z2.h[7]\n" - "fmla z21.h, z7.h, z3.h[7]\n" - "ld1h { z7.h }, p5/Z, [x9, #7, MUL VL]\n" + "fmla z9.h, z24.h, z3.h[7]\n" + "fmla z13.h, z24.h, z2.h[7]\n" + "fmla z17.h, z24.h, z1.h[7]\n" + "fmla z21.h, z24.h, z0.h[7]\n" + "ld1h { z24.h }, p5/Z, [x9, #7, MUL VL]\n" "addvl x9, x9, #8\n" - "fmla z10.h, z6.h, z0.h[7]\n" - "fmla z14.h, z6.h, z1.h[7]\n" - "fmla z18.h, z6.h, z2.h[7]\n" - "fmla z22.h, z6.h, z3.h[7]\n" - "fmla z11.h, z7.h, z0.h[7]\n" - "fmla z15.h, z7.h, z1.h[7]\n" - "fmla z19.h, z7.h, z2.h[7]\n" - "fmla z23.h, z7.h, z3.h[7]\n" + "fmla z10.h, z25.h, z3.h[7]\n" + "fmla z14.h, z25.h, z2.h[7]\n" + "fmla z18.h, z25.h, z1.h[7]\n" + "fmla z22.h, z25.h, z0.h[7]\n" + "fmla z11.h, z24.h, z3.h[7]\n" + "fmla z15.h, z24.h, z2.h[7]\n" + "fmla z19.h, z24.h, z1.h[7]\n" + "fmla z23.h, z24.h, z0.h[7]\n" "bgt 52b\n" "53:" // Height 4: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" @@ -1572,211 +1572,211 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "subs x27, x27, #0x1\n" "ld1rqh { z2.h }, p0/Z, [x24]\n" "ld1rqh { z3.h }, p0/Z, [x23]\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[0]\n" - "fmla z12.h, z6.h, z1.h[0]\n" - "fmla z16.h, z6.h, z2.h[0]\n" - "fmla z20.h, z6.h, z3.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "ld1h { z25.h }, p5/Z, [x12]\n" + "ld1h { z24.h }, p5/Z, [x11]\n" + "fmla z8.h, z25.h, z0.h[0]\n" + "fmla z12.h, z25.h, z1.h[0]\n" + "fmla z16.h, z25.h, z2.h[0]\n" + "fmla z20.h, z25.h, z3.h[0]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" "addvl x12, x12, #1\n" - "fmla z9.h, z7.h, z0.h[0]\n" - "fmla z13.h, z7.h, z1.h[0]\n" + "fmla z9.h, z24.h, z0.h[0]\n" + "fmla z13.h, z24.h, z1.h[0]\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" - "fmla z17.h, z7.h, z2.h[0]\n" - "fmla z21.h, z7.h, z3.h[0]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z17.h, z24.h, z2.h[0]\n" + "fmla z21.h, z24.h, z3.h[0]\n" + "ld1h { z24.h }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.h, z6.h, z0.h[0]\n" - "fmla z14.h, z6.h, z1.h[0]\n" - "fmla z18.h, z6.h, z2.h[0]\n" - "fmla z22.h, z6.h, z3.h[0]\n" - "fmla z11.h, z7.h, z0.h[0]\n" - "fmla z15.h, z7.h, z1.h[0]\n" - "fmla z19.h, z7.h, z2.h[0]\n" - "fmla z23.h, z7.h, z3.h[0]\n" + "fmla z10.h, z25.h, z0.h[0]\n" + "fmla z14.h, z25.h, z1.h[0]\n" + "fmla z18.h, z25.h, z2.h[0]\n" + "fmla z22.h, z25.h, z3.h[0]\n" + "fmla z11.h, z24.h, z0.h[0]\n" + "fmla z15.h, z24.h, z1.h[0]\n" + "fmla z19.h, z24.h, z2.h[0]\n" + "fmla z23.h, z24.h, z3.h[0]\n" "ble 54f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[1]\n" - "fmla z12.h, z6.h, z1.h[1]\n" - "fmla z16.h, z6.h, z2.h[1]\n" - "fmla z20.h, z6.h, z3.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "ld1h { z25.h }, p5/Z, [x12]\n" + "ld1h { z24.h }, p5/Z, [x11]\n" + "fmla z8.h, z25.h, z0.h[1]\n" + "fmla z12.h, z25.h, z1.h[1]\n" + "fmla z16.h, z25.h, z2.h[1]\n" + "fmla z20.h, z25.h, z3.h[1]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" "subs x27, x27, #0x1\n" - "fmla z9.h, z7.h, z0.h[1]\n" - "fmla z13.h, z7.h, z1.h[1]\n" + "fmla z9.h, z24.h, z0.h[1]\n" + "fmla z13.h, z24.h, z1.h[1]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" - "fmla z17.h, z7.h, z2.h[1]\n" - "fmla z21.h, z7.h, z3.h[1]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z17.h, z24.h, z2.h[1]\n" + "fmla z21.h, z24.h, z3.h[1]\n" + "ld1h { z24.h }, p5/Z, [x9]\n" "addvl x10, x10, #1\n" - "fmla z10.h, z6.h, z0.h[1]\n" - "fmla z14.h, z6.h, z1.h[1]\n" + "fmla z10.h, z25.h, z0.h[1]\n" + "fmla z14.h, z25.h, z1.h[1]\n" "addvl x9, x9, #1\n" - "fmla z18.h, z6.h, z2.h[1]\n" - "fmla z22.h, z6.h, z3.h[1]\n" - "fmla z11.h, z7.h, z0.h[1]\n" - "fmla z15.h, z7.h, z1.h[1]\n" - "fmla z19.h, z7.h, z2.h[1]\n" - "fmla z23.h, z7.h, z3.h[1]\n" + "fmla z18.h, z25.h, z2.h[1]\n" + "fmla z22.h, z25.h, z3.h[1]\n" + "fmla z11.h, z24.h, z0.h[1]\n" + "fmla z15.h, z24.h, z1.h[1]\n" + "fmla z19.h, z24.h, z2.h[1]\n" + "fmla z23.h, z24.h, z3.h[1]\n" "ble 54f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[2]\n" - "fmla z12.h, z6.h, z1.h[2]\n" - "fmla z16.h, z6.h, z2.h[2]\n" - "fmla z20.h, z6.h, z3.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "ld1h { z25.h }, p5/Z, [x12]\n" + "ld1h { z24.h }, p5/Z, [x11]\n" + "fmla z8.h, z25.h, z0.h[2]\n" + "fmla z12.h, z25.h, z1.h[2]\n" + "fmla z16.h, z25.h, z2.h[2]\n" + "fmla z20.h, z25.h, z3.h[2]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" "subs x27, x27, #0x1\n" - "fmla z9.h, z7.h, z0.h[2]\n" - "fmla z13.h, z7.h, z1.h[2]\n" + "fmla z9.h, z24.h, z0.h[2]\n" + "fmla z13.h, z24.h, z1.h[2]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" - "fmla z17.h, z7.h, z2.h[2]\n" - "fmla z21.h, z7.h, z3.h[2]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z17.h, z24.h, z2.h[2]\n" + "fmla z21.h, z24.h, z3.h[2]\n" + "ld1h { z24.h }, p5/Z, [x9]\n" "addvl x10, x10, #1\n" - "fmla z10.h, z6.h, z0.h[2]\n" - "fmla z14.h, z6.h, z1.h[2]\n" + "fmla z10.h, z25.h, z0.h[2]\n" + "fmla z14.h, z25.h, z1.h[2]\n" "addvl x9, x9, #1\n" - "fmla z18.h, z6.h, z2.h[2]\n" - "fmla z22.h, z6.h, z3.h[2]\n" - "fmla z11.h, z7.h, z0.h[2]\n" - "fmla z15.h, z7.h, z1.h[2]\n" - "fmla z19.h, z7.h, z2.h[2]\n" - "fmla z23.h, z7.h, z3.h[2]\n" + "fmla z18.h, z25.h, z2.h[2]\n" + "fmla z22.h, z25.h, z3.h[2]\n" + "fmla z11.h, z24.h, z0.h[2]\n" + "fmla z15.h, z24.h, z1.h[2]\n" + "fmla z19.h, z24.h, z2.h[2]\n" + "fmla z23.h, z24.h, z3.h[2]\n" "ble 54f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[3]\n" - "fmla z12.h, z6.h, z1.h[3]\n" - "fmla z16.h, z6.h, z2.h[3]\n" - "fmla z20.h, z6.h, z3.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "ld1h { z25.h }, p5/Z, [x12]\n" + "ld1h { z24.h }, p5/Z, [x11]\n" + "fmla z8.h, z25.h, z0.h[3]\n" + "fmla z12.h, z25.h, z1.h[3]\n" + "fmla z16.h, z25.h, z2.h[3]\n" + "fmla z20.h, z25.h, z3.h[3]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" "subs x27, x27, #0x1\n" - "fmla z9.h, z7.h, z0.h[3]\n" - "fmla z13.h, z7.h, z1.h[3]\n" + "fmla z9.h, z24.h, z0.h[3]\n" + "fmla z13.h, z24.h, z1.h[3]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" - "fmla z17.h, z7.h, z2.h[3]\n" - "fmla z21.h, z7.h, z3.h[3]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z17.h, z24.h, z2.h[3]\n" + "fmla z21.h, z24.h, z3.h[3]\n" + "ld1h { z24.h }, p5/Z, [x9]\n" "addvl x10, x10, #1\n" - "fmla z10.h, z6.h, z0.h[3]\n" - "fmla z14.h, z6.h, z1.h[3]\n" + "fmla z10.h, z25.h, z0.h[3]\n" + "fmla z14.h, z25.h, z1.h[3]\n" "addvl x9, x9, #1\n" - "fmla z18.h, z6.h, z2.h[3]\n" - "fmla z22.h, z6.h, z3.h[3]\n" - "fmla z11.h, z7.h, z0.h[3]\n" - "fmla z15.h, z7.h, z1.h[3]\n" - "fmla z19.h, z7.h, z2.h[3]\n" - "fmla z23.h, z7.h, z3.h[3]\n" + "fmla z18.h, z25.h, z2.h[3]\n" + "fmla z22.h, z25.h, z3.h[3]\n" + "fmla z11.h, z24.h, z0.h[3]\n" + "fmla z15.h, z24.h, z1.h[3]\n" + "fmla z19.h, z24.h, z2.h[3]\n" + "fmla z23.h, z24.h, z3.h[3]\n" "ble 54f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[4]\n" - "fmla z12.h, z6.h, z1.h[4]\n" - "fmla z16.h, z6.h, z2.h[4]\n" - "fmla z20.h, z6.h, z3.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "ld1h { z25.h }, p5/Z, [x12]\n" + "ld1h { z24.h }, p5/Z, [x11]\n" + "fmla z8.h, z25.h, z0.h[4]\n" + "fmla z12.h, z25.h, z1.h[4]\n" + "fmla z16.h, z25.h, z2.h[4]\n" + "fmla z20.h, z25.h, z3.h[4]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" "subs x27, x27, #0x1\n" - "fmla z9.h, z7.h, z0.h[4]\n" - "fmla z13.h, z7.h, z1.h[4]\n" + "fmla z9.h, z24.h, z0.h[4]\n" + "fmla z13.h, z24.h, z1.h[4]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" - "fmla z17.h, z7.h, z2.h[4]\n" - "fmla z21.h, z7.h, z3.h[4]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z17.h, z24.h, z2.h[4]\n" + "fmla z21.h, z24.h, z3.h[4]\n" + "ld1h { z24.h }, p5/Z, [x9]\n" "addvl x10, x10, #1\n" - "fmla z10.h, z6.h, z0.h[4]\n" - "fmla z14.h, z6.h, z1.h[4]\n" + "fmla z10.h, z25.h, z0.h[4]\n" + "fmla z14.h, z25.h, z1.h[4]\n" "addvl x9, x9, #1\n" - "fmla z18.h, z6.h, z2.h[4]\n" - "fmla z22.h, z6.h, z3.h[4]\n" - "fmla z11.h, z7.h, z0.h[4]\n" - "fmla z15.h, z7.h, z1.h[4]\n" - "fmla z19.h, z7.h, z2.h[4]\n" - "fmla z23.h, z7.h, z3.h[4]\n" + "fmla z18.h, z25.h, z2.h[4]\n" + "fmla z22.h, z25.h, z3.h[4]\n" + "fmla z11.h, z24.h, z0.h[4]\n" + "fmla z15.h, z24.h, z1.h[4]\n" + "fmla z19.h, z24.h, z2.h[4]\n" + "fmla z23.h, z24.h, z3.h[4]\n" "ble 54f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[5]\n" - "fmla z12.h, z6.h, z1.h[5]\n" - "fmla z16.h, z6.h, z2.h[5]\n" - "fmla z20.h, z6.h, z3.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "ld1h { z25.h }, p5/Z, [x12]\n" + "ld1h { z24.h }, p5/Z, [x11]\n" + "fmla z8.h, z25.h, z0.h[5]\n" + "fmla z12.h, z25.h, z1.h[5]\n" + "fmla z16.h, z25.h, z2.h[5]\n" + "fmla z20.h, z25.h, z3.h[5]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" "subs x27, x27, #0x1\n" - "fmla z9.h, z7.h, z0.h[5]\n" - "fmla z13.h, z7.h, z1.h[5]\n" + "fmla z9.h, z24.h, z0.h[5]\n" + "fmla z13.h, z24.h, z1.h[5]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" - "fmla z17.h, z7.h, z2.h[5]\n" - "fmla z21.h, z7.h, z3.h[5]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z17.h, z24.h, z2.h[5]\n" + "fmla z21.h, z24.h, z3.h[5]\n" + "ld1h { z24.h }, p5/Z, [x9]\n" "addvl x10, x10, #1\n" - "fmla z10.h, z6.h, z0.h[5]\n" - "fmla z14.h, z6.h, z1.h[5]\n" + "fmla z10.h, z25.h, z0.h[5]\n" + "fmla z14.h, z25.h, z1.h[5]\n" "addvl x9, x9, #1\n" - "fmla z18.h, z6.h, z2.h[5]\n" - "fmla z22.h, z6.h, z3.h[5]\n" - "fmla z11.h, z7.h, z0.h[5]\n" - "fmla z15.h, z7.h, z1.h[5]\n" - "fmla z19.h, z7.h, z2.h[5]\n" - "fmla z23.h, z7.h, z3.h[5]\n" + "fmla z18.h, z25.h, z2.h[5]\n" + "fmla z22.h, z25.h, z3.h[5]\n" + "fmla z11.h, z24.h, z0.h[5]\n" + "fmla z15.h, z24.h, z1.h[5]\n" + "fmla z19.h, z24.h, z2.h[5]\n" + "fmla z23.h, z24.h, z3.h[5]\n" "ble 54f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[6]\n" - "fmla z12.h, z6.h, z1.h[6]\n" - "fmla z16.h, z6.h, z2.h[6]\n" - "fmla z20.h, z6.h, z3.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "ld1h { z25.h }, p5/Z, [x12]\n" + "ld1h { z24.h }, p5/Z, [x11]\n" + "fmla z8.h, z25.h, z0.h[6]\n" + "fmla z12.h, z25.h, z1.h[6]\n" + "fmla z16.h, z25.h, z2.h[6]\n" + "fmla z20.h, z25.h, z3.h[6]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" "subs x27, x27, #0x1\n" - "fmla z9.h, z7.h, z0.h[6]\n" - "fmla z13.h, z7.h, z1.h[6]\n" + "fmla z9.h, z24.h, z0.h[6]\n" + "fmla z13.h, z24.h, z1.h[6]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" - "fmla z17.h, z7.h, z2.h[6]\n" - "fmla z21.h, z7.h, z3.h[6]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z17.h, z24.h, z2.h[6]\n" + "fmla z21.h, z24.h, z3.h[6]\n" + "ld1h { z24.h }, p5/Z, [x9]\n" "addvl x10, x10, #1\n" - "fmla z10.h, z6.h, z0.h[6]\n" - "fmla z14.h, z6.h, z1.h[6]\n" + "fmla z10.h, z25.h, z0.h[6]\n" + "fmla z14.h, z25.h, z1.h[6]\n" "addvl x9, x9, #1\n" - "fmla z18.h, z6.h, z2.h[6]\n" - "fmla z22.h, z6.h, z3.h[6]\n" - "fmla z11.h, z7.h, z0.h[6]\n" - "fmla z15.h, z7.h, z1.h[6]\n" - "fmla z19.h, z7.h, z2.h[6]\n" - "fmla z23.h, z7.h, z3.h[6]\n" + "fmla z18.h, z25.h, z2.h[6]\n" + "fmla z22.h, z25.h, z3.h[6]\n" + "fmla z11.h, z24.h, z0.h[6]\n" + "fmla z15.h, z24.h, z1.h[6]\n" + "fmla z19.h, z24.h, z2.h[6]\n" + "fmla z23.h, z24.h, z3.h[6]\n" "ble 54f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[7]\n" - "fmla z12.h, z6.h, z1.h[7]\n" - "fmla z16.h, z6.h, z2.h[7]\n" - "fmla z20.h, z6.h, z3.h[7]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "ld1h { z25.h }, p5/Z, [x12]\n" + "ld1h { z24.h }, p5/Z, [x11]\n" + "fmla z8.h, z25.h, z0.h[7]\n" + "fmla z12.h, z25.h, z1.h[7]\n" + "fmla z16.h, z25.h, z2.h[7]\n" + "fmla z20.h, z25.h, z3.h[7]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" "addvl x12, x12, #1\n" - "fmla z9.h, z7.h, z0.h[7]\n" - "fmla z13.h, z7.h, z1.h[7]\n" + "fmla z9.h, z24.h, z0.h[7]\n" + "fmla z13.h, z24.h, z1.h[7]\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" - "fmla z17.h, z7.h, z2.h[7]\n" - "fmla z21.h, z7.h, z3.h[7]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z17.h, z24.h, z2.h[7]\n" + "fmla z21.h, z24.h, z3.h[7]\n" + "ld1h { z24.h }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.h, z6.h, z0.h[7]\n" - "fmla z14.h, z6.h, z1.h[7]\n" - "fmla z18.h, z6.h, z2.h[7]\n" - "fmla z22.h, z6.h, z3.h[7]\n" - "fmla z11.h, z7.h, z0.h[7]\n" - "fmla z15.h, z7.h, z1.h[7]\n" - "fmla z19.h, z7.h, z2.h[7]\n" - "fmla z23.h, z7.h, z3.h[7]\n" + "fmla z10.h, z25.h, z0.h[7]\n" + "fmla z14.h, z25.h, z1.h[7]\n" + "fmla z18.h, z25.h, z2.h[7]\n" + "fmla z22.h, z25.h, z3.h[7]\n" + "fmla z11.h, z24.h, z0.h[7]\n" + "fmla z15.h, z24.h, z1.h[7]\n" + "fmla z19.h, z24.h, z2.h[7]\n" + "fmla z23.h, z24.h, z3.h[7]\n" "54:" // Height 4: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1788,41 +1788,41 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "add x23, x24, x20, LSL #1\n" "tbz %x[flags], #1, 55f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rh { z1.h }, p5/Z, [x20]\n" + "ld1rh { z25.h }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rh { z0.h }, p5/Z, [x20]\n" - "fmin z8.h, p5/M, z8.h, z1.h\n" - "fmin z9.h, p5/M, z9.h, z1.h\n" - "fmin z10.h, p5/M, z10.h, z1.h\n" - "fmin z11.h, p5/M, z11.h, z1.h\n" - "fmin z12.h, p5/M, z12.h, z1.h\n" - "fmin z13.h, p5/M, z13.h, z1.h\n" - "fmin z14.h, p5/M, z14.h, z1.h\n" - "fmin z15.h, p5/M, z15.h, z1.h\n" - "fmin z16.h, p5/M, z16.h, z1.h\n" - "fmin z17.h, p5/M, z17.h, z1.h\n" - "fmin z18.h, p5/M, z18.h, z1.h\n" - "fmin z19.h, p5/M, z19.h, z1.h\n" - "fmin z20.h, p5/M, z20.h, z1.h\n" - "fmin z21.h, p5/M, z21.h, z1.h\n" - "fmin z22.h, p5/M, z22.h, z1.h\n" - "fmin z23.h, p5/M, z23.h, z1.h\n" - "fmax z8.h, p5/M, z8.h, z0.h\n" - "fmax z9.h, p5/M, z9.h, z0.h\n" - "fmax z10.h, p5/M, z10.h, z0.h\n" - "fmax z11.h, p5/M, z11.h, z0.h\n" - "fmax z12.h, p5/M, z12.h, z0.h\n" - "fmax z13.h, p5/M, z13.h, z0.h\n" - "fmax z14.h, p5/M, z14.h, z0.h\n" - "fmax z15.h, p5/M, z15.h, z0.h\n" - "fmax z16.h, p5/M, z16.h, z0.h\n" - "fmax z17.h, p5/M, z17.h, z0.h\n" - "fmax z18.h, p5/M, z18.h, z0.h\n" - "fmax z19.h, p5/M, z19.h, z0.h\n" - "fmax z20.h, p5/M, z20.h, z0.h\n" - "fmax z21.h, p5/M, z21.h, z0.h\n" - "fmax z22.h, p5/M, z22.h, z0.h\n" - "fmax z23.h, p5/M, z23.h, z0.h\n" + "ld1rh { z24.h }, p5/Z, [x20]\n" + "fmin z8.h, p5/M, z8.h, z25.h\n" + "fmin z9.h, p5/M, z9.h, z25.h\n" + "fmin z10.h, p5/M, z10.h, z25.h\n" + "fmin z11.h, p5/M, z11.h, z25.h\n" + "fmin z12.h, p5/M, z12.h, z25.h\n" + "fmin z13.h, p5/M, z13.h, z25.h\n" + "fmin z14.h, p5/M, z14.h, z25.h\n" + "fmin z15.h, p5/M, z15.h, z25.h\n" + "fmin z16.h, p5/M, z16.h, z25.h\n" + "fmin z17.h, p5/M, z17.h, z25.h\n" + "fmin z18.h, p5/M, z18.h, z25.h\n" + "fmin z19.h, p5/M, z19.h, z25.h\n" + "fmin z20.h, p5/M, z20.h, z25.h\n" + "fmin z21.h, p5/M, z21.h, z25.h\n" + "fmin z22.h, p5/M, z22.h, z25.h\n" + "fmin z23.h, p5/M, z23.h, z25.h\n" + "fmax z8.h, p5/M, z8.h, z24.h\n" + "fmax z9.h, p5/M, z9.h, z24.h\n" + "fmax z10.h, p5/M, z10.h, z24.h\n" + "fmax z11.h, p5/M, z11.h, z24.h\n" + "fmax z12.h, p5/M, z12.h, z24.h\n" + "fmax z13.h, p5/M, z13.h, z24.h\n" + "fmax z14.h, p5/M, z14.h, z24.h\n" + "fmax z15.h, p5/M, z15.h, z24.h\n" + "fmax z16.h, p5/M, z16.h, z24.h\n" + "fmax z17.h, p5/M, z17.h, z24.h\n" + "fmax z18.h, p5/M, z18.h, z24.h\n" + "fmax z19.h, p5/M, z19.h, z24.h\n" + "fmax z20.h, p5/M, z20.h, z24.h\n" + "fmax z21.h, p5/M, z21.h, z24.h\n" + "fmax z22.h, p5/M, z22.h, z24.h\n" + "fmax z23.h, p5/M, z23.h, z24.h\n" "55:" // Height 4: No activation "st1h { z8.h }, p4, [x13]\n" "st1h { z9.h }, p3, [x13, #1, MUL VL]\n" @@ -1907,30 +1907,30 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "60:" // Height 5: no bias "tbz %x[flags], #0, 61f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "ld1h { z8.h }, p4/Z, [x13]\n" - "add x23, x24, x20, LSL #1\n" + "add x23, x13, x20, LSL #1\n" "add x22, x23, x20, LSL #1\n" + "ld1h { z8.h }, p4/Z, [x13]\n" + "add x21, x22, x20, LSL #1\n" + "add x20, x21, x20, LSL #1\n" "ld1h { z9.h }, p3/Z, [x13, #1, MUL VL]\n" "ld1h { z10.h }, p2/Z, [x13, #2, MUL VL]\n" "ld1h { z11.h }, p1/Z, [x13, #3, MUL VL]\n" - "ld1h { z12.h }, p4/Z, [x25]\n" - "ld1h { z13.h }, p3/Z, [x25, #1, MUL VL]\n" - "ld1h { z14.h }, p2/Z, [x25, #2, MUL VL]\n" - "ld1h { z15.h }, p1/Z, [x25, #3, MUL VL]\n" - "ld1h { z16.h }, p4/Z, [x24]\n" - "ld1h { z17.h }, p3/Z, [x24, #1, MUL VL]\n" - "ld1h { z18.h }, p2/Z, [x24, #2, MUL VL]\n" - "ld1h { z19.h }, p1/Z, [x24, #3, MUL VL]\n" - "ld1h { z20.h }, p4/Z, [x23]\n" - "ld1h { z21.h }, p3/Z, [x23, #1, MUL VL]\n" - "ld1h { z22.h }, p2/Z, [x23, #2, MUL VL]\n" - "ld1h { z23.h }, p1/Z, [x23, #3, MUL VL]\n" - "ld1h { z24.h }, p4/Z, [x22]\n" - "ld1h { z25.h }, p3/Z, [x22, #1, MUL VL]\n" - "ld1h { z26.h }, p2/Z, [x22, #2, MUL VL]\n" - "ld1h { z27.h }, p1/Z, [x22, #3, MUL VL]\n" + "ld1h { z12.h }, p4/Z, [x23]\n" + "ld1h { z13.h }, p3/Z, [x23, #1, MUL VL]\n" + "ld1h { z14.h }, p2/Z, [x23, #2, MUL VL]\n" + "ld1h { z15.h }, p1/Z, [x23, #3, MUL VL]\n" + "ld1h { z16.h }, p4/Z, [x22]\n" + "ld1h { z17.h }, p3/Z, [x22, #1, MUL VL]\n" + "ld1h { z18.h }, p2/Z, [x22, #2, MUL VL]\n" + "ld1h { z19.h }, p1/Z, [x22, #3, MUL VL]\n" + "ld1h { z20.h }, p4/Z, [x21]\n" + "ld1h { z21.h }, p3/Z, [x21, #1, MUL VL]\n" + "ld1h { z22.h }, p2/Z, [x21, #2, MUL VL]\n" + "ld1h { z23.h }, p1/Z, [x21, #3, MUL VL]\n" + "ld1h { z24.h }, p4/Z, [x20]\n" + "ld1h { z25.h }, p3/Z, [x20, #1, MUL VL]\n" + "ld1h { z26.h }, p2/Z, [x20, #2, MUL VL]\n" + "ld1h { z27.h }, p1/Z, [x20, #3, MUL VL]\n" "b 62f\n" "61:" // Height 5: no accumulate "mov z8.b, #0x0\n" @@ -1958,15 +1958,15 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "63:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 64f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 65f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -1977,223 +1977,223 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "b 65f\n" "64:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" - "add x22, x23, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" + "add x22, x23, x21, LSL #1\n" "65:" // Height 5: input setup done "cmp x27, #0x8\n" "ble 67f\n" "66:" // Height 5: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z0.h }, p0/Z, [x26]\n" - "ld1rqh { z1.h }, p0/Z, [x25]\n" + "ld1rqh { z4.h }, p0/Z, [x26]\n" + "ld1rqh { z3.h }, p0/Z, [x25]\n" "sub x27, x27, #0x8\n" "ld1rqh { z2.h }, p0/Z, [x24]\n" - "ld1rqh { z3.h }, p0/Z, [x23]\n" + "ld1rqh { z1.h }, p0/Z, [x23]\n" "cmp x27, #0x8\n" "add x26, x26, #0x10\n" - "ld1rqh { z4.h }, p0/Z, [x22]\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "fmla z8.h, z6.h, z0.h[0]\n" - "fmla z12.h, z6.h, z1.h[0]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z16.h, z6.h, z2.h[0]\n" - "fmla z20.h, z6.h, z3.h[0]\n" + "ld1rqh { z0.h }, p0/Z, [x22]\n" + "ld1h { z29.h }, p5/Z, [x12]\n" + "fmla z8.h, z29.h, z4.h[0]\n" + "fmla z12.h, z29.h, z3.h[0]\n" + "ld1h { z28.h }, p5/Z, [x11]\n" + "fmla z16.h, z29.h, z2.h[0]\n" + "fmla z20.h, z29.h, z1.h[0]\n" "add x25, x25, #0x10\n" - "fmla z24.h, z6.h, z4.h[0]\n" - "fmla z9.h, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "fmla z24.h, z29.h, z0.h[0]\n" + "fmla z9.h, z28.h, z4.h[0]\n" + "ld1h { z29.h }, p5/Z, [x10]\n" "add x24, x24, #0x10\n" - "fmla z13.h, z7.h, z1.h[0]\n" - "fmla z17.h, z7.h, z2.h[0]\n" + "fmla z13.h, z28.h, z3.h[0]\n" + "fmla z17.h, z28.h, z2.h[0]\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - "fmla z21.h, z7.h, z3.h[0]\n" - "fmla z25.h, z7.h, z4.h[0]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" - "fmla z10.h, z6.h, z0.h[0]\n" - "fmla z14.h, z6.h, z1.h[0]\n" - "fmla z18.h, z6.h, z2.h[0]\n" - "fmla z22.h, z6.h, z3.h[0]\n" - "fmla z26.h, z6.h, z4.h[0]\n" - "fmla z11.h, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n" - "fmla z15.h, z7.h, z1.h[0]\n" - "fmla z19.h, z7.h, z2.h[0]\n" - "fmla z23.h, z7.h, z3.h[0]\n" - "fmla z27.h, z7.h, z4.h[0]\n" - "ld1h { z7.h }, p5/Z, [x11, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[1]\n" - "fmla z12.h, z6.h, z1.h[1]\n" - "fmla z16.h, z6.h, z2.h[1]\n" - "fmla z20.h, z6.h, z3.h[1]\n" - "fmla z24.h, z6.h, z4.h[1]\n" - "fmla z9.h, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[1]\n" - "fmla z17.h, z7.h, z2.h[1]\n" - "fmla z21.h, z7.h, z3.h[1]\n" - "fmla z25.h, z7.h, z4.h[1]\n" - "ld1h { z7.h }, p5/Z, [x9, #1, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[1]\n" - "fmla z14.h, z6.h, z1.h[1]\n" - "fmla z18.h, z6.h, z2.h[1]\n" - "fmla z22.h, z6.h, z3.h[1]\n" - "fmla z26.h, z6.h, z4.h[1]\n" - "fmla z11.h, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x12, #2, MUL VL]\n" - "fmla z15.h, z7.h, z1.h[1]\n" - "fmla z19.h, z7.h, z2.h[1]\n" - "fmla z23.h, z7.h, z3.h[1]\n" - "fmla z27.h, z7.h, z4.h[1]\n" - "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[2]\n" - "fmla z12.h, z6.h, z1.h[2]\n" - "fmla z16.h, z6.h, z2.h[2]\n" - "fmla z20.h, z6.h, z3.h[2]\n" - "fmla z24.h, z6.h, z4.h[2]\n" - "fmla z9.h, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[2]\n" - "fmla z17.h, z7.h, z2.h[2]\n" - "fmla z21.h, z7.h, z3.h[2]\n" - "fmla z25.h, z7.h, z4.h[2]\n" - "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[2]\n" - "fmla z14.h, z6.h, z1.h[2]\n" - "fmla z18.h, z6.h, z2.h[2]\n" - "fmla z22.h, z6.h, z3.h[2]\n" - "fmla z26.h, z6.h, z4.h[2]\n" - "fmla z11.h, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x12, #3, MUL VL]\n" - "fmla z15.h, z7.h, z1.h[2]\n" - "fmla z19.h, z7.h, z2.h[2]\n" - "fmla z23.h, z7.h, z3.h[2]\n" - "fmla z27.h, z7.h, z4.h[2]\n" - "ld1h { z7.h }, p5/Z, [x11, #3, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[3]\n" - "fmla z12.h, z6.h, z1.h[3]\n" - "fmla z16.h, z6.h, z2.h[3]\n" - "fmla z20.h, z6.h, z3.h[3]\n" - "fmla z24.h, z6.h, z4.h[3]\n" - "fmla z9.h, z7.h, z0.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[3]\n" - "fmla z17.h, z7.h, z2.h[3]\n" - "fmla z21.h, z7.h, z3.h[3]\n" - "fmla z25.h, z7.h, z4.h[3]\n" - "ld1h { z7.h }, p5/Z, [x9, #3, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[3]\n" - "fmla z14.h, z6.h, z1.h[3]\n" - "fmla z18.h, z6.h, z2.h[3]\n" - "fmla z22.h, z6.h, z3.h[3]\n" - "fmla z26.h, z6.h, z4.h[3]\n" - "fmla z11.h, z7.h, z0.h[3]\n" - "ld1h { z6.h }, p5/Z, [x12, #4, MUL VL]\n" - "fmla z15.h, z7.h, z1.h[3]\n" - "fmla z19.h, z7.h, z2.h[3]\n" - "fmla z23.h, z7.h, z3.h[3]\n" - "fmla z27.h, z7.h, z4.h[3]\n" - "ld1h { z7.h }, p5/Z, [x11, #4, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[4]\n" - "fmla z12.h, z6.h, z1.h[4]\n" - "fmla z16.h, z6.h, z2.h[4]\n" - "fmla z20.h, z6.h, z3.h[4]\n" - "fmla z24.h, z6.h, z4.h[4]\n" - "fmla z9.h, z7.h, z0.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[4]\n" - "fmla z17.h, z7.h, z2.h[4]\n" - "fmla z21.h, z7.h, z3.h[4]\n" - "fmla z25.h, z7.h, z4.h[4]\n" - "ld1h { z7.h }, p5/Z, [x9, #4, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[4]\n" - "fmla z14.h, z6.h, z1.h[4]\n" - "fmla z18.h, z6.h, z2.h[4]\n" - "fmla z22.h, z6.h, z3.h[4]\n" - "fmla z26.h, z6.h, z4.h[4]\n" - "fmla z11.h, z7.h, z0.h[4]\n" - "ld1h { z6.h }, p5/Z, [x12, #5, MUL VL]\n" - "fmla z15.h, z7.h, z1.h[4]\n" - "fmla z19.h, z7.h, z2.h[4]\n" - "fmla z23.h, z7.h, z3.h[4]\n" - "fmla z27.h, z7.h, z4.h[4]\n" - "ld1h { z7.h }, p5/Z, [x11, #5, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[5]\n" - "fmla z12.h, z6.h, z1.h[5]\n" - "fmla z16.h, z6.h, z2.h[5]\n" - "fmla z20.h, z6.h, z3.h[5]\n" - "fmla z24.h, z6.h, z4.h[5]\n" - "fmla z9.h, z7.h, z0.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[5]\n" - "fmla z17.h, z7.h, z2.h[5]\n" - "fmla z21.h, z7.h, z3.h[5]\n" - "fmla z25.h, z7.h, z4.h[5]\n" - "ld1h { z7.h }, p5/Z, [x9, #5, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[5]\n" - "fmla z14.h, z6.h, z1.h[5]\n" - "fmla z18.h, z6.h, z2.h[5]\n" - "fmla z22.h, z6.h, z3.h[5]\n" - "fmla z26.h, z6.h, z4.h[5]\n" - "fmla z11.h, z7.h, z0.h[5]\n" - "ld1h { z6.h }, p5/Z, [x12, #6, MUL VL]\n" - "fmla z15.h, z7.h, z1.h[5]\n" - "fmla z19.h, z7.h, z2.h[5]\n" - "fmla z23.h, z7.h, z3.h[5]\n" - "fmla z27.h, z7.h, z4.h[5]\n" - "ld1h { z7.h }, p5/Z, [x11, #6, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[6]\n" - "fmla z12.h, z6.h, z1.h[6]\n" - "fmla z16.h, z6.h, z2.h[6]\n" - "fmla z20.h, z6.h, z3.h[6]\n" - "fmla z24.h, z6.h, z4.h[6]\n" - "fmla z9.h, z7.h, z0.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[6]\n" - "fmla z17.h, z7.h, z2.h[6]\n" - "fmla z21.h, z7.h, z3.h[6]\n" - "fmla z25.h, z7.h, z4.h[6]\n" - "ld1h { z7.h }, p5/Z, [x9, #6, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[6]\n" - "fmla z14.h, z6.h, z1.h[6]\n" - "fmla z18.h, z6.h, z2.h[6]\n" - "fmla z22.h, z6.h, z3.h[6]\n" - "fmla z26.h, z6.h, z4.h[6]\n" - "fmla z11.h, z7.h, z0.h[6]\n" - "ld1h { z6.h }, p5/Z, [x12, #7, MUL VL]\n" + "fmla z21.h, z28.h, z1.h[0]\n" + "fmla z25.h, z28.h, z0.h[0]\n" + "ld1h { z28.h }, p5/Z, [x9]\n" + "fmla z10.h, z29.h, z4.h[0]\n" + "fmla z14.h, z29.h, z3.h[0]\n" + "fmla z18.h, z29.h, z2.h[0]\n" + "fmla z22.h, z29.h, z1.h[0]\n" + "fmla z26.h, z29.h, z0.h[0]\n" + "fmla z11.h, z28.h, z4.h[0]\n" + "ld1h { z29.h }, p5/Z, [x12, #1, MUL VL]\n" + "fmla z15.h, z28.h, z3.h[0]\n" + "fmla z19.h, z28.h, z2.h[0]\n" + "fmla z23.h, z28.h, z1.h[0]\n" + "fmla z27.h, z28.h, z0.h[0]\n" + "ld1h { z28.h }, p5/Z, [x11, #1, MUL VL]\n" + "fmla z8.h, z29.h, z4.h[1]\n" + "fmla z12.h, z29.h, z3.h[1]\n" + "fmla z16.h, z29.h, z2.h[1]\n" + "fmla z20.h, z29.h, z1.h[1]\n" + "fmla z24.h, z29.h, z0.h[1]\n" + "fmla z9.h, z28.h, z4.h[1]\n" + "ld1h { z29.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z13.h, z28.h, z3.h[1]\n" + "fmla z17.h, z28.h, z2.h[1]\n" + "fmla z21.h, z28.h, z1.h[1]\n" + "fmla z25.h, z28.h, z0.h[1]\n" + "ld1h { z28.h }, p5/Z, [x9, #1, MUL VL]\n" + "fmla z10.h, z29.h, z4.h[1]\n" + "fmla z14.h, z29.h, z3.h[1]\n" + "fmla z18.h, z29.h, z2.h[1]\n" + "fmla z22.h, z29.h, z1.h[1]\n" + "fmla z26.h, z29.h, z0.h[1]\n" + "fmla z11.h, z28.h, z4.h[1]\n" + "ld1h { z29.h }, p5/Z, [x12, #2, MUL VL]\n" + "fmla z15.h, z28.h, z3.h[1]\n" + "fmla z19.h, z28.h, z2.h[1]\n" + "fmla z23.h, z28.h, z1.h[1]\n" + "fmla z27.h, z28.h, z0.h[1]\n" + "ld1h { z28.h }, p5/Z, [x11, #2, MUL VL]\n" + "fmla z8.h, z29.h, z4.h[2]\n" + "fmla z12.h, z29.h, z3.h[2]\n" + "fmla z16.h, z29.h, z2.h[2]\n" + "fmla z20.h, z29.h, z1.h[2]\n" + "fmla z24.h, z29.h, z0.h[2]\n" + "fmla z9.h, z28.h, z4.h[2]\n" + "ld1h { z29.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z13.h, z28.h, z3.h[2]\n" + "fmla z17.h, z28.h, z2.h[2]\n" + "fmla z21.h, z28.h, z1.h[2]\n" + "fmla z25.h, z28.h, z0.h[2]\n" + "ld1h { z28.h }, p5/Z, [x9, #2, MUL VL]\n" + "fmla z10.h, z29.h, z4.h[2]\n" + "fmla z14.h, z29.h, z3.h[2]\n" + "fmla z18.h, z29.h, z2.h[2]\n" + "fmla z22.h, z29.h, z1.h[2]\n" + "fmla z26.h, z29.h, z0.h[2]\n" + "fmla z11.h, z28.h, z4.h[2]\n" + "ld1h { z29.h }, p5/Z, [x12, #3, MUL VL]\n" + "fmla z15.h, z28.h, z3.h[2]\n" + "fmla z19.h, z28.h, z2.h[2]\n" + "fmla z23.h, z28.h, z1.h[2]\n" + "fmla z27.h, z28.h, z0.h[2]\n" + "ld1h { z28.h }, p5/Z, [x11, #3, MUL VL]\n" + "fmla z8.h, z29.h, z4.h[3]\n" + "fmla z12.h, z29.h, z3.h[3]\n" + "fmla z16.h, z29.h, z2.h[3]\n" + "fmla z20.h, z29.h, z1.h[3]\n" + "fmla z24.h, z29.h, z0.h[3]\n" + "fmla z9.h, z28.h, z4.h[3]\n" + "ld1h { z29.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z13.h, z28.h, z3.h[3]\n" + "fmla z17.h, z28.h, z2.h[3]\n" + "fmla z21.h, z28.h, z1.h[3]\n" + "fmla z25.h, z28.h, z0.h[3]\n" + "ld1h { z28.h }, p5/Z, [x9, #3, MUL VL]\n" + "fmla z10.h, z29.h, z4.h[3]\n" + "fmla z14.h, z29.h, z3.h[3]\n" + "fmla z18.h, z29.h, z2.h[3]\n" + "fmla z22.h, z29.h, z1.h[3]\n" + "fmla z26.h, z29.h, z0.h[3]\n" + "fmla z11.h, z28.h, z4.h[3]\n" + "ld1h { z29.h }, p5/Z, [x12, #4, MUL VL]\n" + "fmla z15.h, z28.h, z3.h[3]\n" + "fmla z19.h, z28.h, z2.h[3]\n" + "fmla z23.h, z28.h, z1.h[3]\n" + "fmla z27.h, z28.h, z0.h[3]\n" + "ld1h { z28.h }, p5/Z, [x11, #4, MUL VL]\n" + "fmla z8.h, z29.h, z4.h[4]\n" + "fmla z12.h, z29.h, z3.h[4]\n" + "fmla z16.h, z29.h, z2.h[4]\n" + "fmla z20.h, z29.h, z1.h[4]\n" + "fmla z24.h, z29.h, z0.h[4]\n" + "fmla z9.h, z28.h, z4.h[4]\n" + "ld1h { z29.h }, p5/Z, [x10, #4, MUL VL]\n" + "fmla z13.h, z28.h, z3.h[4]\n" + "fmla z17.h, z28.h, z2.h[4]\n" + "fmla z21.h, z28.h, z1.h[4]\n" + "fmla z25.h, z28.h, z0.h[4]\n" + "ld1h { z28.h }, p5/Z, [x9, #4, MUL VL]\n" + "fmla z10.h, z29.h, z4.h[4]\n" + "fmla z14.h, z29.h, z3.h[4]\n" + "fmla z18.h, z29.h, z2.h[4]\n" + "fmla z22.h, z29.h, z1.h[4]\n" + "fmla z26.h, z29.h, z0.h[4]\n" + "fmla z11.h, z28.h, z4.h[4]\n" + "ld1h { z29.h }, p5/Z, [x12, #5, MUL VL]\n" + "fmla z15.h, z28.h, z3.h[4]\n" + "fmla z19.h, z28.h, z2.h[4]\n" + "fmla z23.h, z28.h, z1.h[4]\n" + "fmla z27.h, z28.h, z0.h[4]\n" + "ld1h { z28.h }, p5/Z, [x11, #5, MUL VL]\n" + "fmla z8.h, z29.h, z4.h[5]\n" + "fmla z12.h, z29.h, z3.h[5]\n" + "fmla z16.h, z29.h, z2.h[5]\n" + "fmla z20.h, z29.h, z1.h[5]\n" + "fmla z24.h, z29.h, z0.h[5]\n" + "fmla z9.h, z28.h, z4.h[5]\n" + "ld1h { z29.h }, p5/Z, [x10, #5, MUL VL]\n" + "fmla z13.h, z28.h, z3.h[5]\n" + "fmla z17.h, z28.h, z2.h[5]\n" + "fmla z21.h, z28.h, z1.h[5]\n" + "fmla z25.h, z28.h, z0.h[5]\n" + "ld1h { z28.h }, p5/Z, [x9, #5, MUL VL]\n" + "fmla z10.h, z29.h, z4.h[5]\n" + "fmla z14.h, z29.h, z3.h[5]\n" + "fmla z18.h, z29.h, z2.h[5]\n" + "fmla z22.h, z29.h, z1.h[5]\n" + "fmla z26.h, z29.h, z0.h[5]\n" + "fmla z11.h, z28.h, z4.h[5]\n" + "ld1h { z29.h }, p5/Z, [x12, #6, MUL VL]\n" + "fmla z15.h, z28.h, z3.h[5]\n" + "fmla z19.h, z28.h, z2.h[5]\n" + "fmla z23.h, z28.h, z1.h[5]\n" + "fmla z27.h, z28.h, z0.h[5]\n" + "ld1h { z28.h }, p5/Z, [x11, #6, MUL VL]\n" + "fmla z8.h, z29.h, z4.h[6]\n" + "fmla z12.h, z29.h, z3.h[6]\n" + "fmla z16.h, z29.h, z2.h[6]\n" + "fmla z20.h, z29.h, z1.h[6]\n" + "fmla z24.h, z29.h, z0.h[6]\n" + "fmla z9.h, z28.h, z4.h[6]\n" + "ld1h { z29.h }, p5/Z, [x10, #6, MUL VL]\n" + "fmla z13.h, z28.h, z3.h[6]\n" + "fmla z17.h, z28.h, z2.h[6]\n" + "fmla z21.h, z28.h, z1.h[6]\n" + "fmla z25.h, z28.h, z0.h[6]\n" + "ld1h { z28.h }, p5/Z, [x9, #6, MUL VL]\n" + "fmla z10.h, z29.h, z4.h[6]\n" + "fmla z14.h, z29.h, z3.h[6]\n" + "fmla z18.h, z29.h, z2.h[6]\n" + "fmla z22.h, z29.h, z1.h[6]\n" + "fmla z26.h, z29.h, z0.h[6]\n" + "fmla z11.h, z28.h, z4.h[6]\n" + "ld1h { z29.h }, p5/Z, [x12, #7, MUL VL]\n" "addvl x12, x12, #8\n" - "fmla z15.h, z7.h, z1.h[6]\n" - "fmla z19.h, z7.h, z2.h[6]\n" - "fmla z23.h, z7.h, z3.h[6]\n" - "fmla z27.h, z7.h, z4.h[6]\n" - "ld1h { z7.h }, p5/Z, [x11, #7, MUL VL]\n" + "fmla z15.h, z28.h, z3.h[6]\n" + "fmla z19.h, z28.h, z2.h[6]\n" + "fmla z23.h, z28.h, z1.h[6]\n" + "fmla z27.h, z28.h, z0.h[6]\n" + "ld1h { z28.h }, p5/Z, [x11, #7, MUL VL]\n" "addvl x11, x11, #8\n" - "fmla z8.h, z6.h, z0.h[7]\n" - "fmla z12.h, z6.h, z1.h[7]\n" - "fmla z16.h, z6.h, z2.h[7]\n" - "fmla z20.h, z6.h, z3.h[7]\n" - "fmla z24.h, z6.h, z4.h[7]\n" - "fmla z9.h, z7.h, z0.h[7]\n" - "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n" + "fmla z8.h, z29.h, z4.h[7]\n" + "fmla z12.h, z29.h, z3.h[7]\n" + "fmla z16.h, z29.h, z2.h[7]\n" + "fmla z20.h, z29.h, z1.h[7]\n" + "fmla z24.h, z29.h, z0.h[7]\n" + "fmla z9.h, z28.h, z4.h[7]\n" + "ld1h { z29.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - "fmla z13.h, z7.h, z1.h[7]\n" - "fmla z17.h, z7.h, z2.h[7]\n" - "fmla z21.h, z7.h, z3.h[7]\n" - "fmla z25.h, z7.h, z4.h[7]\n" - "ld1h { z7.h }, p5/Z, [x9, #7, MUL VL]\n" + "fmla z13.h, z28.h, z3.h[7]\n" + "fmla z17.h, z28.h, z2.h[7]\n" + "fmla z21.h, z28.h, z1.h[7]\n" + "fmla z25.h, z28.h, z0.h[7]\n" + "ld1h { z28.h }, p5/Z, [x9, #7, MUL VL]\n" "addvl x9, x9, #8\n" - "fmla z10.h, z6.h, z0.h[7]\n" - "fmla z14.h, z6.h, z1.h[7]\n" - "fmla z18.h, z6.h, z2.h[7]\n" - "fmla z22.h, z6.h, z3.h[7]\n" - "fmla z26.h, z6.h, z4.h[7]\n" - "fmla z11.h, z7.h, z0.h[7]\n" - "fmla z15.h, z7.h, z1.h[7]\n" - "fmla z19.h, z7.h, z2.h[7]\n" - "fmla z23.h, z7.h, z3.h[7]\n" - "fmla z27.h, z7.h, z4.h[7]\n" + "fmla z10.h, z29.h, z4.h[7]\n" + "fmla z14.h, z29.h, z3.h[7]\n" + "fmla z18.h, z29.h, z2.h[7]\n" + "fmla z22.h, z29.h, z1.h[7]\n" + "fmla z26.h, z29.h, z0.h[7]\n" + "fmla z11.h, z28.h, z4.h[7]\n" + "fmla z15.h, z28.h, z3.h[7]\n" + "fmla z19.h, z28.h, z2.h[7]\n" + "fmla z23.h, z28.h, z1.h[7]\n" + "fmla z27.h, z28.h, z0.h[7]\n" "bgt 66b\n" "67:" // Height 5: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" @@ -2203,243 +2203,243 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "ld1rqh { z2.h }, p0/Z, [x24]\n" "ld1rqh { z3.h }, p0/Z, [x23]\n" "ld1rqh { z4.h }, p0/Z, [x22]\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "fmla z8.h, z6.h, z0.h[0]\n" - "fmla z12.h, z6.h, z1.h[0]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z16.h, z6.h, z2.h[0]\n" - "fmla z20.h, z6.h, z3.h[0]\n" + "ld1h { z29.h }, p5/Z, [x12]\n" + "fmla z8.h, z29.h, z0.h[0]\n" + "fmla z12.h, z29.h, z1.h[0]\n" + "ld1h { z28.h }, p5/Z, [x11]\n" + "fmla z16.h, z29.h, z2.h[0]\n" + "fmla z20.h, z29.h, z3.h[0]\n" "addvl x12, x12, #1\n" - "fmla z24.h, z6.h, z4.h[0]\n" - "fmla z9.h, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "fmla z24.h, z29.h, z4.h[0]\n" + "fmla z9.h, z28.h, z0.h[0]\n" + "ld1h { z29.h }, p5/Z, [x10]\n" "addvl x11, x11, #1\n" - "fmla z13.h, z7.h, z1.h[0]\n" - "fmla z17.h, z7.h, z2.h[0]\n" + "fmla z13.h, z28.h, z1.h[0]\n" + "fmla z17.h, z28.h, z2.h[0]\n" "addvl x10, x10, #1\n" - "fmla z21.h, z7.h, z3.h[0]\n" - "fmla z25.h, z7.h, z4.h[0]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z21.h, z28.h, z3.h[0]\n" + "fmla z25.h, z28.h, z4.h[0]\n" + "ld1h { z28.h }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.h, z6.h, z0.h[0]\n" - "fmla z14.h, z6.h, z1.h[0]\n" - "fmla z18.h, z6.h, z2.h[0]\n" - "fmla z22.h, z6.h, z3.h[0]\n" - "fmla z26.h, z6.h, z4.h[0]\n" - "fmla z11.h, z7.h, z0.h[0]\n" - "fmla z15.h, z7.h, z1.h[0]\n" - "fmla z19.h, z7.h, z2.h[0]\n" - "fmla z23.h, z7.h, z3.h[0]\n" - "fmla z27.h, z7.h, z4.h[0]\n" + "fmla z10.h, z29.h, z0.h[0]\n" + "fmla z14.h, z29.h, z1.h[0]\n" + "fmla z18.h, z29.h, z2.h[0]\n" + "fmla z22.h, z29.h, z3.h[0]\n" + "fmla z26.h, z29.h, z4.h[0]\n" + "fmla z11.h, z28.h, z0.h[0]\n" + "fmla z15.h, z28.h, z1.h[0]\n" + "fmla z19.h, z28.h, z2.h[0]\n" + "fmla z23.h, z28.h, z3.h[0]\n" + "fmla z27.h, z28.h, z4.h[0]\n" "ble 68f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[1]\n" - "fmla z12.h, z6.h, z1.h[1]\n" - "fmla z16.h, z6.h, z2.h[1]\n" - "fmla z20.h, z6.h, z3.h[1]\n" + "ld1h { z29.h }, p5/Z, [x12]\n" + "ld1h { z28.h }, p5/Z, [x11]\n" + "fmla z8.h, z29.h, z0.h[1]\n" + "fmla z12.h, z29.h, z1.h[1]\n" + "fmla z16.h, z29.h, z2.h[1]\n" + "fmla z20.h, z29.h, z3.h[1]\n" "subs x27, x27, #0x1\n" "addvl x12, x12, #1\n" - "fmla z24.h, z6.h, z4.h[1]\n" - "fmla z9.h, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "fmla z24.h, z29.h, z4.h[1]\n" + "fmla z9.h, z28.h, z0.h[1]\n" + "ld1h { z29.h }, p5/Z, [x10]\n" "addvl x11, x11, #1\n" - "fmla z13.h, z7.h, z1.h[1]\n" - "fmla z17.h, z7.h, z2.h[1]\n" + "fmla z13.h, z28.h, z1.h[1]\n" + "fmla z17.h, z28.h, z2.h[1]\n" "addvl x10, x10, #1\n" - "fmla z21.h, z7.h, z3.h[1]\n" - "fmla z25.h, z7.h, z4.h[1]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z21.h, z28.h, z3.h[1]\n" + "fmla z25.h, z28.h, z4.h[1]\n" + "ld1h { z28.h }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.h, z6.h, z0.h[1]\n" - "fmla z14.h, z6.h, z1.h[1]\n" - "fmla z18.h, z6.h, z2.h[1]\n" - "fmla z22.h, z6.h, z3.h[1]\n" - "fmla z26.h, z6.h, z4.h[1]\n" - "fmla z11.h, z7.h, z0.h[1]\n" - "fmla z15.h, z7.h, z1.h[1]\n" - "fmla z19.h, z7.h, z2.h[1]\n" - "fmla z23.h, z7.h, z3.h[1]\n" - "fmla z27.h, z7.h, z4.h[1]\n" + "fmla z10.h, z29.h, z0.h[1]\n" + "fmla z14.h, z29.h, z1.h[1]\n" + "fmla z18.h, z29.h, z2.h[1]\n" + "fmla z22.h, z29.h, z3.h[1]\n" + "fmla z26.h, z29.h, z4.h[1]\n" + "fmla z11.h, z28.h, z0.h[1]\n" + "fmla z15.h, z28.h, z1.h[1]\n" + "fmla z19.h, z28.h, z2.h[1]\n" + "fmla z23.h, z28.h, z3.h[1]\n" + "fmla z27.h, z28.h, z4.h[1]\n" "ble 68f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[2]\n" - "fmla z12.h, z6.h, z1.h[2]\n" - "fmla z16.h, z6.h, z2.h[2]\n" - "fmla z20.h, z6.h, z3.h[2]\n" + "ld1h { z29.h }, p5/Z, [x12]\n" + "ld1h { z28.h }, p5/Z, [x11]\n" + "fmla z8.h, z29.h, z0.h[2]\n" + "fmla z12.h, z29.h, z1.h[2]\n" + "fmla z16.h, z29.h, z2.h[2]\n" + "fmla z20.h, z29.h, z3.h[2]\n" "subs x27, x27, #0x1\n" "addvl x12, x12, #1\n" - "fmla z24.h, z6.h, z4.h[2]\n" - "fmla z9.h, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "fmla z24.h, z29.h, z4.h[2]\n" + "fmla z9.h, z28.h, z0.h[2]\n" + "ld1h { z29.h }, p5/Z, [x10]\n" "addvl x11, x11, #1\n" - "fmla z13.h, z7.h, z1.h[2]\n" - "fmla z17.h, z7.h, z2.h[2]\n" + "fmla z13.h, z28.h, z1.h[2]\n" + "fmla z17.h, z28.h, z2.h[2]\n" "addvl x10, x10, #1\n" - "fmla z21.h, z7.h, z3.h[2]\n" - "fmla z25.h, z7.h, z4.h[2]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z21.h, z28.h, z3.h[2]\n" + "fmla z25.h, z28.h, z4.h[2]\n" + "ld1h { z28.h }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.h, z6.h, z0.h[2]\n" - "fmla z14.h, z6.h, z1.h[2]\n" - "fmla z18.h, z6.h, z2.h[2]\n" - "fmla z22.h, z6.h, z3.h[2]\n" - "fmla z26.h, z6.h, z4.h[2]\n" - "fmla z11.h, z7.h, z0.h[2]\n" - "fmla z15.h, z7.h, z1.h[2]\n" - "fmla z19.h, z7.h, z2.h[2]\n" - "fmla z23.h, z7.h, z3.h[2]\n" - "fmla z27.h, z7.h, z4.h[2]\n" + "fmla z10.h, z29.h, z0.h[2]\n" + "fmla z14.h, z29.h, z1.h[2]\n" + "fmla z18.h, z29.h, z2.h[2]\n" + "fmla z22.h, z29.h, z3.h[2]\n" + "fmla z26.h, z29.h, z4.h[2]\n" + "fmla z11.h, z28.h, z0.h[2]\n" + "fmla z15.h, z28.h, z1.h[2]\n" + "fmla z19.h, z28.h, z2.h[2]\n" + "fmla z23.h, z28.h, z3.h[2]\n" + "fmla z27.h, z28.h, z4.h[2]\n" "ble 68f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[3]\n" - "fmla z12.h, z6.h, z1.h[3]\n" - "fmla z16.h, z6.h, z2.h[3]\n" - "fmla z20.h, z6.h, z3.h[3]\n" + "ld1h { z29.h }, p5/Z, [x12]\n" + "ld1h { z28.h }, p5/Z, [x11]\n" + "fmla z8.h, z29.h, z0.h[3]\n" + "fmla z12.h, z29.h, z1.h[3]\n" + "fmla z16.h, z29.h, z2.h[3]\n" + "fmla z20.h, z29.h, z3.h[3]\n" "subs x27, x27, #0x1\n" "addvl x12, x12, #1\n" - "fmla z24.h, z6.h, z4.h[3]\n" - "fmla z9.h, z7.h, z0.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "fmla z24.h, z29.h, z4.h[3]\n" + "fmla z9.h, z28.h, z0.h[3]\n" + "ld1h { z29.h }, p5/Z, [x10]\n" "addvl x11, x11, #1\n" - "fmla z13.h, z7.h, z1.h[3]\n" - "fmla z17.h, z7.h, z2.h[3]\n" + "fmla z13.h, z28.h, z1.h[3]\n" + "fmla z17.h, z28.h, z2.h[3]\n" "addvl x10, x10, #1\n" - "fmla z21.h, z7.h, z3.h[3]\n" - "fmla z25.h, z7.h, z4.h[3]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z21.h, z28.h, z3.h[3]\n" + "fmla z25.h, z28.h, z4.h[3]\n" + "ld1h { z28.h }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.h, z6.h, z0.h[3]\n" - "fmla z14.h, z6.h, z1.h[3]\n" - "fmla z18.h, z6.h, z2.h[3]\n" - "fmla z22.h, z6.h, z3.h[3]\n" - "fmla z26.h, z6.h, z4.h[3]\n" - "fmla z11.h, z7.h, z0.h[3]\n" - "fmla z15.h, z7.h, z1.h[3]\n" - "fmla z19.h, z7.h, z2.h[3]\n" - "fmla z23.h, z7.h, z3.h[3]\n" - "fmla z27.h, z7.h, z4.h[3]\n" + "fmla z10.h, z29.h, z0.h[3]\n" + "fmla z14.h, z29.h, z1.h[3]\n" + "fmla z18.h, z29.h, z2.h[3]\n" + "fmla z22.h, z29.h, z3.h[3]\n" + "fmla z26.h, z29.h, z4.h[3]\n" + "fmla z11.h, z28.h, z0.h[3]\n" + "fmla z15.h, z28.h, z1.h[3]\n" + "fmla z19.h, z28.h, z2.h[3]\n" + "fmla z23.h, z28.h, z3.h[3]\n" + "fmla z27.h, z28.h, z4.h[3]\n" "ble 68f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[4]\n" - "fmla z12.h, z6.h, z1.h[4]\n" - "fmla z16.h, z6.h, z2.h[4]\n" - "fmla z20.h, z6.h, z3.h[4]\n" + "ld1h { z29.h }, p5/Z, [x12]\n" + "ld1h { z28.h }, p5/Z, [x11]\n" + "fmla z8.h, z29.h, z0.h[4]\n" + "fmla z12.h, z29.h, z1.h[4]\n" + "fmla z16.h, z29.h, z2.h[4]\n" + "fmla z20.h, z29.h, z3.h[4]\n" "subs x27, x27, #0x1\n" "addvl x12, x12, #1\n" - "fmla z24.h, z6.h, z4.h[4]\n" - "fmla z9.h, z7.h, z0.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "fmla z24.h, z29.h, z4.h[4]\n" + "fmla z9.h, z28.h, z0.h[4]\n" + "ld1h { z29.h }, p5/Z, [x10]\n" "addvl x11, x11, #1\n" - "fmla z13.h, z7.h, z1.h[4]\n" - "fmla z17.h, z7.h, z2.h[4]\n" + "fmla z13.h, z28.h, z1.h[4]\n" + "fmla z17.h, z28.h, z2.h[4]\n" "addvl x10, x10, #1\n" - "fmla z21.h, z7.h, z3.h[4]\n" - "fmla z25.h, z7.h, z4.h[4]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z21.h, z28.h, z3.h[4]\n" + "fmla z25.h, z28.h, z4.h[4]\n" + "ld1h { z28.h }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.h, z6.h, z0.h[4]\n" - "fmla z14.h, z6.h, z1.h[4]\n" - "fmla z18.h, z6.h, z2.h[4]\n" - "fmla z22.h, z6.h, z3.h[4]\n" - "fmla z26.h, z6.h, z4.h[4]\n" - "fmla z11.h, z7.h, z0.h[4]\n" - "fmla z15.h, z7.h, z1.h[4]\n" - "fmla z19.h, z7.h, z2.h[4]\n" - "fmla z23.h, z7.h, z3.h[4]\n" - "fmla z27.h, z7.h, z4.h[4]\n" + "fmla z10.h, z29.h, z0.h[4]\n" + "fmla z14.h, z29.h, z1.h[4]\n" + "fmla z18.h, z29.h, z2.h[4]\n" + "fmla z22.h, z29.h, z3.h[4]\n" + "fmla z26.h, z29.h, z4.h[4]\n" + "fmla z11.h, z28.h, z0.h[4]\n" + "fmla z15.h, z28.h, z1.h[4]\n" + "fmla z19.h, z28.h, z2.h[4]\n" + "fmla z23.h, z28.h, z3.h[4]\n" + "fmla z27.h, z28.h, z4.h[4]\n" "ble 68f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[5]\n" - "fmla z12.h, z6.h, z1.h[5]\n" - "fmla z16.h, z6.h, z2.h[5]\n" - "fmla z20.h, z6.h, z3.h[5]\n" + "ld1h { z29.h }, p5/Z, [x12]\n" + "ld1h { z28.h }, p5/Z, [x11]\n" + "fmla z8.h, z29.h, z0.h[5]\n" + "fmla z12.h, z29.h, z1.h[5]\n" + "fmla z16.h, z29.h, z2.h[5]\n" + "fmla z20.h, z29.h, z3.h[5]\n" "subs x27, x27, #0x1\n" "addvl x12, x12, #1\n" - "fmla z24.h, z6.h, z4.h[5]\n" - "fmla z9.h, z7.h, z0.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "fmla z24.h, z29.h, z4.h[5]\n" + "fmla z9.h, z28.h, z0.h[5]\n" + "ld1h { z29.h }, p5/Z, [x10]\n" "addvl x11, x11, #1\n" - "fmla z13.h, z7.h, z1.h[5]\n" - "fmla z17.h, z7.h, z2.h[5]\n" + "fmla z13.h, z28.h, z1.h[5]\n" + "fmla z17.h, z28.h, z2.h[5]\n" "addvl x10, x10, #1\n" - "fmla z21.h, z7.h, z3.h[5]\n" - "fmla z25.h, z7.h, z4.h[5]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z21.h, z28.h, z3.h[5]\n" + "fmla z25.h, z28.h, z4.h[5]\n" + "ld1h { z28.h }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.h, z6.h, z0.h[5]\n" - "fmla z14.h, z6.h, z1.h[5]\n" - "fmla z18.h, z6.h, z2.h[5]\n" - "fmla z22.h, z6.h, z3.h[5]\n" - "fmla z26.h, z6.h, z4.h[5]\n" - "fmla z11.h, z7.h, z0.h[5]\n" - "fmla z15.h, z7.h, z1.h[5]\n" - "fmla z19.h, z7.h, z2.h[5]\n" - "fmla z23.h, z7.h, z3.h[5]\n" - "fmla z27.h, z7.h, z4.h[5]\n" + "fmla z10.h, z29.h, z0.h[5]\n" + "fmla z14.h, z29.h, z1.h[5]\n" + "fmla z18.h, z29.h, z2.h[5]\n" + "fmla z22.h, z29.h, z3.h[5]\n" + "fmla z26.h, z29.h, z4.h[5]\n" + "fmla z11.h, z28.h, z0.h[5]\n" + "fmla z15.h, z28.h, z1.h[5]\n" + "fmla z19.h, z28.h, z2.h[5]\n" + "fmla z23.h, z28.h, z3.h[5]\n" + "fmla z27.h, z28.h, z4.h[5]\n" "ble 68f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[6]\n" - "fmla z12.h, z6.h, z1.h[6]\n" - "fmla z16.h, z6.h, z2.h[6]\n" - "fmla z20.h, z6.h, z3.h[6]\n" + "ld1h { z29.h }, p5/Z, [x12]\n" + "ld1h { z28.h }, p5/Z, [x11]\n" + "fmla z8.h, z29.h, z0.h[6]\n" + "fmla z12.h, z29.h, z1.h[6]\n" + "fmla z16.h, z29.h, z2.h[6]\n" + "fmla z20.h, z29.h, z3.h[6]\n" "subs x27, x27, #0x1\n" "addvl x12, x12, #1\n" - "fmla z24.h, z6.h, z4.h[6]\n" - "fmla z9.h, z7.h, z0.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "fmla z24.h, z29.h, z4.h[6]\n" + "fmla z9.h, z28.h, z0.h[6]\n" + "ld1h { z29.h }, p5/Z, [x10]\n" "addvl x11, x11, #1\n" - "fmla z13.h, z7.h, z1.h[6]\n" - "fmla z17.h, z7.h, z2.h[6]\n" + "fmla z13.h, z28.h, z1.h[6]\n" + "fmla z17.h, z28.h, z2.h[6]\n" "addvl x10, x10, #1\n" - "fmla z21.h, z7.h, z3.h[6]\n" - "fmla z25.h, z7.h, z4.h[6]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z21.h, z28.h, z3.h[6]\n" + "fmla z25.h, z28.h, z4.h[6]\n" + "ld1h { z28.h }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.h, z6.h, z0.h[6]\n" - "fmla z14.h, z6.h, z1.h[6]\n" - "fmla z18.h, z6.h, z2.h[6]\n" - "fmla z22.h, z6.h, z3.h[6]\n" - "fmla z26.h, z6.h, z4.h[6]\n" - "fmla z11.h, z7.h, z0.h[6]\n" - "fmla z15.h, z7.h, z1.h[6]\n" - "fmla z19.h, z7.h, z2.h[6]\n" - "fmla z23.h, z7.h, z3.h[6]\n" - "fmla z27.h, z7.h, z4.h[6]\n" + "fmla z10.h, z29.h, z0.h[6]\n" + "fmla z14.h, z29.h, z1.h[6]\n" + "fmla z18.h, z29.h, z2.h[6]\n" + "fmla z22.h, z29.h, z3.h[6]\n" + "fmla z26.h, z29.h, z4.h[6]\n" + "fmla z11.h, z28.h, z0.h[6]\n" + "fmla z15.h, z28.h, z1.h[6]\n" + "fmla z19.h, z28.h, z2.h[6]\n" + "fmla z23.h, z28.h, z3.h[6]\n" + "fmla z27.h, z28.h, z4.h[6]\n" "ble 68f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[7]\n" - "fmla z12.h, z6.h, z1.h[7]\n" - "fmla z16.h, z6.h, z2.h[7]\n" - "fmla z20.h, z6.h, z3.h[7]\n" + "ld1h { z29.h }, p5/Z, [x12]\n" + "ld1h { z28.h }, p5/Z, [x11]\n" + "fmla z8.h, z29.h, z0.h[7]\n" + "fmla z12.h, z29.h, z1.h[7]\n" + "fmla z16.h, z29.h, z2.h[7]\n" + "fmla z20.h, z29.h, z3.h[7]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" - "fmla z24.h, z6.h, z4.h[7]\n" - "fmla z9.h, z7.h, z0.h[7]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "fmla z24.h, z29.h, z4.h[7]\n" + "fmla z9.h, z28.h, z0.h[7]\n" + "ld1h { z29.h }, p5/Z, [x10]\n" "addvl x10, x10, #1\n" - "fmla z13.h, z7.h, z1.h[7]\n" - "fmla z17.h, z7.h, z2.h[7]\n" - "fmla z21.h, z7.h, z3.h[7]\n" - "fmla z25.h, z7.h, z4.h[7]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z13.h, z28.h, z1.h[7]\n" + "fmla z17.h, z28.h, z2.h[7]\n" + "fmla z21.h, z28.h, z3.h[7]\n" + "fmla z25.h, z28.h, z4.h[7]\n" + "ld1h { z28.h }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.h, z6.h, z0.h[7]\n" - "fmla z14.h, z6.h, z1.h[7]\n" - "fmla z18.h, z6.h, z2.h[7]\n" - "fmla z22.h, z6.h, z3.h[7]\n" - "fmla z26.h, z6.h, z4.h[7]\n" - "fmla z11.h, z7.h, z0.h[7]\n" - "fmla z15.h, z7.h, z1.h[7]\n" - "fmla z19.h, z7.h, z2.h[7]\n" - "fmla z23.h, z7.h, z3.h[7]\n" - "fmla z27.h, z7.h, z4.h[7]\n" + "fmla z10.h, z29.h, z0.h[7]\n" + "fmla z14.h, z29.h, z1.h[7]\n" + "fmla z18.h, z29.h, z2.h[7]\n" + "fmla z22.h, z29.h, z3.h[7]\n" + "fmla z26.h, z29.h, z4.h[7]\n" + "fmla z11.h, z28.h, z0.h[7]\n" + "fmla z15.h, z28.h, z1.h[7]\n" + "fmla z19.h, z28.h, z2.h[7]\n" + "fmla z23.h, z28.h, z3.h[7]\n" + "fmla z27.h, z28.h, z4.h[7]\n" "68:" // Height 5: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -2452,49 +2452,49 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "add x22, x23, x20, LSL #1\n" "tbz %x[flags], #1, 69f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rh { z1.h }, p5/Z, [x20]\n" + "ld1rh { z29.h }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rh { z0.h }, p5/Z, [x20]\n" - "fmin z8.h, p5/M, z8.h, z1.h\n" - "fmin z9.h, p5/M, z9.h, z1.h\n" - "fmin z10.h, p5/M, z10.h, z1.h\n" - "fmin z11.h, p5/M, z11.h, z1.h\n" - "fmin z12.h, p5/M, z12.h, z1.h\n" - "fmin z13.h, p5/M, z13.h, z1.h\n" - "fmin z14.h, p5/M, z14.h, z1.h\n" - "fmin z15.h, p5/M, z15.h, z1.h\n" - "fmin z16.h, p5/M, z16.h, z1.h\n" - "fmin z17.h, p5/M, z17.h, z1.h\n" - "fmin z18.h, p5/M, z18.h, z1.h\n" - "fmin z19.h, p5/M, z19.h, z1.h\n" - "fmin z20.h, p5/M, z20.h, z1.h\n" - "fmin z21.h, p5/M, z21.h, z1.h\n" - "fmin z22.h, p5/M, z22.h, z1.h\n" - "fmin z23.h, p5/M, z23.h, z1.h\n" - "fmin z24.h, p5/M, z24.h, z1.h\n" - "fmin z25.h, p5/M, z25.h, z1.h\n" - "fmin z26.h, p5/M, z26.h, z1.h\n" - "fmin z27.h, p5/M, z27.h, z1.h\n" - "fmax z8.h, p5/M, z8.h, z0.h\n" - "fmax z9.h, p5/M, z9.h, z0.h\n" - "fmax z10.h, p5/M, z10.h, z0.h\n" - "fmax z11.h, p5/M, z11.h, z0.h\n" - "fmax z12.h, p5/M, z12.h, z0.h\n" - "fmax z13.h, p5/M, z13.h, z0.h\n" - "fmax z14.h, p5/M, z14.h, z0.h\n" - "fmax z15.h, p5/M, z15.h, z0.h\n" - "fmax z16.h, p5/M, z16.h, z0.h\n" - "fmax z17.h, p5/M, z17.h, z0.h\n" - "fmax z18.h, p5/M, z18.h, z0.h\n" - "fmax z19.h, p5/M, z19.h, z0.h\n" - "fmax z20.h, p5/M, z20.h, z0.h\n" - "fmax z21.h, p5/M, z21.h, z0.h\n" - "fmax z22.h, p5/M, z22.h, z0.h\n" - "fmax z23.h, p5/M, z23.h, z0.h\n" - "fmax z24.h, p5/M, z24.h, z0.h\n" - "fmax z25.h, p5/M, z25.h, z0.h\n" - "fmax z26.h, p5/M, z26.h, z0.h\n" - "fmax z27.h, p5/M, z27.h, z0.h\n" + "ld1rh { z28.h }, p5/Z, [x20]\n" + "fmin z8.h, p5/M, z8.h, z29.h\n" + "fmin z9.h, p5/M, z9.h, z29.h\n" + "fmin z10.h, p5/M, z10.h, z29.h\n" + "fmin z11.h, p5/M, z11.h, z29.h\n" + "fmin z12.h, p5/M, z12.h, z29.h\n" + "fmin z13.h, p5/M, z13.h, z29.h\n" + "fmin z14.h, p5/M, z14.h, z29.h\n" + "fmin z15.h, p5/M, z15.h, z29.h\n" + "fmin z16.h, p5/M, z16.h, z29.h\n" + "fmin z17.h, p5/M, z17.h, z29.h\n" + "fmin z18.h, p5/M, z18.h, z29.h\n" + "fmin z19.h, p5/M, z19.h, z29.h\n" + "fmin z20.h, p5/M, z20.h, z29.h\n" + "fmin z21.h, p5/M, z21.h, z29.h\n" + "fmin z22.h, p5/M, z22.h, z29.h\n" + "fmin z23.h, p5/M, z23.h, z29.h\n" + "fmin z24.h, p5/M, z24.h, z29.h\n" + "fmin z25.h, p5/M, z25.h, z29.h\n" + "fmin z26.h, p5/M, z26.h, z29.h\n" + "fmin z27.h, p5/M, z27.h, z29.h\n" + "fmax z8.h, p5/M, z8.h, z28.h\n" + "fmax z9.h, p5/M, z9.h, z28.h\n" + "fmax z10.h, p5/M, z10.h, z28.h\n" + "fmax z11.h, p5/M, z11.h, z28.h\n" + "fmax z12.h, p5/M, z12.h, z28.h\n" + "fmax z13.h, p5/M, z13.h, z28.h\n" + "fmax z14.h, p5/M, z14.h, z28.h\n" + "fmax z15.h, p5/M, z15.h, z28.h\n" + "fmax z16.h, p5/M, z16.h, z28.h\n" + "fmax z17.h, p5/M, z17.h, z28.h\n" + "fmax z18.h, p5/M, z18.h, z28.h\n" + "fmax z19.h, p5/M, z19.h, z28.h\n" + "fmax z20.h, p5/M, z20.h, z28.h\n" + "fmax z21.h, p5/M, z21.h, z28.h\n" + "fmax z22.h, p5/M, z22.h, z28.h\n" + "fmax z23.h, p5/M, z23.h, z28.h\n" + "fmax z24.h, p5/M, z24.h, z28.h\n" + "fmax z25.h, p5/M, z25.h, z28.h\n" + "fmax z26.h, p5/M, z26.h, z28.h\n" + "fmax z27.h, p5/M, z27.h, z28.h\n" "69:" // Height 5: No activation "st1h { z8.h }, p4, [x13]\n" "st1h { z9.h }, p3, [x13, #1, MUL VL]\n" @@ -2590,35 +2590,35 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "74:" // Height 6: no bias "tbz %x[flags], #0, 75f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "ld1h { z8.h }, p4/Z, [x13]\n" + "add x24, x13, x20, LSL #1\n" "add x23, x24, x20, LSL #1\n" + "ld1h { z8.h }, p4/Z, [x13]\n" "add x22, x23, x20, LSL #1\n" + "add x21, x22, x20, LSL #1\n" "ld1h { z9.h }, p3/Z, [x13, #1, MUL VL]\n" "ld1h { z10.h }, p2/Z, [x13, #2, MUL VL]\n" - "add x21, x22, x20, LSL #1\n" + "add x20, x21, x20, LSL #1\n" "ld1h { z11.h }, p1/Z, [x13, #3, MUL VL]\n" - "ld1h { z12.h }, p4/Z, [x25]\n" - "ld1h { z13.h }, p3/Z, [x25, #1, MUL VL]\n" - "ld1h { z14.h }, p2/Z, [x25, #2, MUL VL]\n" - "ld1h { z15.h }, p1/Z, [x25, #3, MUL VL]\n" - "ld1h { z16.h }, p4/Z, [x24]\n" - "ld1h { z17.h }, p3/Z, [x24, #1, MUL VL]\n" - "ld1h { z18.h }, p2/Z, [x24, #2, MUL VL]\n" - "ld1h { z19.h }, p1/Z, [x24, #3, MUL VL]\n" - "ld1h { z20.h }, p4/Z, [x23]\n" - "ld1h { z21.h }, p3/Z, [x23, #1, MUL VL]\n" - "ld1h { z22.h }, p2/Z, [x23, #2, MUL VL]\n" - "ld1h { z23.h }, p1/Z, [x23, #3, MUL VL]\n" - "ld1h { z24.h }, p4/Z, [x22]\n" - "ld1h { z25.h }, p3/Z, [x22, #1, MUL VL]\n" - "ld1h { z26.h }, p2/Z, [x22, #2, MUL VL]\n" - "ld1h { z27.h }, p1/Z, [x22, #3, MUL VL]\n" - "ld1h { z28.h }, p4/Z, [x21]\n" - "ld1h { z29.h }, p3/Z, [x21, #1, MUL VL]\n" - "ld1h { z30.h }, p2/Z, [x21, #2, MUL VL]\n" - "ld1h { z31.h }, p1/Z, [x21, #3, MUL VL]\n" + "ld1h { z12.h }, p4/Z, [x24]\n" + "ld1h { z13.h }, p3/Z, [x24, #1, MUL VL]\n" + "ld1h { z14.h }, p2/Z, [x24, #2, MUL VL]\n" + "ld1h { z15.h }, p1/Z, [x24, #3, MUL VL]\n" + "ld1h { z16.h }, p4/Z, [x23]\n" + "ld1h { z17.h }, p3/Z, [x23, #1, MUL VL]\n" + "ld1h { z18.h }, p2/Z, [x23, #2, MUL VL]\n" + "ld1h { z19.h }, p1/Z, [x23, #3, MUL VL]\n" + "ld1h { z20.h }, p4/Z, [x22]\n" + "ld1h { z21.h }, p3/Z, [x22, #1, MUL VL]\n" + "ld1h { z22.h }, p2/Z, [x22, #2, MUL VL]\n" + "ld1h { z23.h }, p1/Z, [x22, #3, MUL VL]\n" + "ld1h { z24.h }, p4/Z, [x21]\n" + "ld1h { z25.h }, p3/Z, [x21, #1, MUL VL]\n" + "ld1h { z26.h }, p2/Z, [x21, #2, MUL VL]\n" + "ld1h { z27.h }, p1/Z, [x21, #3, MUL VL]\n" + "ld1h { z28.h }, p4/Z, [x20]\n" + "ld1h { z29.h }, p3/Z, [x20, #1, MUL VL]\n" + "ld1h { z30.h }, p2/Z, [x20, #2, MUL VL]\n" + "ld1h { z31.h }, p1/Z, [x20, #3, MUL VL]\n" "b 76f\n" "75:" // Height 6: no accumulate "mov z8.b, #0x0\n" @@ -2650,16 +2650,16 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "77:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 78f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 79f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -2671,258 +2671,258 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "b 79f\n" "78:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" - "add x22, x23, x20, LSL #1\n" - "add x21, x22, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" + "add x22, x23, x21, LSL #1\n" + "add x21, x22, x21, LSL #1\n" "79:" // Height 6: input setup done "cmp x27, #0x8\n" "ble 81f\n" "80:" // Height 6: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z0.h }, p0/Z, [x26]\n" - "ld1rqh { z1.h }, p0/Z, [x25]\n" + "ld1rqh { z7.h }, p0/Z, [x26]\n" + "ld1rqh { z6.h }, p0/Z, [x25]\n" "sub x27, x27, #0x8\n" - "ld1rqh { z2.h }, p0/Z, [x24]\n" - "ld1rqh { z3.h }, p0/Z, [x23]\n" + "ld1rqh { z5.h }, p0/Z, [x24]\n" + "ld1rqh { z4.h }, p0/Z, [x23]\n" "cmp x27, #0x8\n" "add x26, x26, #0x10\n" - "ld1rqh { z4.h }, p0/Z, [x22]\n" - "ld1rqh { z5.h }, p0/Z, [x21]\n" + "ld1rqh { z3.h }, p0/Z, [x22]\n" + "ld1rqh { z2.h }, p0/Z, [x21]\n" "add x25, x25, #0x10\n" "add x24, x24, #0x10\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[0]\n" - "fmla z12.h, z6.h, z1.h[0]\n" - "fmla z16.h, z6.h, z2.h[0]\n" - "fmla z20.h, z6.h, z3.h[0]\n" + "ld1h { z1.h }, p5/Z, [x12]\n" + "ld1h { z0.h }, p5/Z, [x11]\n" + "fmla z8.h, z1.h, z7.h[0]\n" + "fmla z12.h, z1.h, z6.h[0]\n" + "fmla z16.h, z1.h, z5.h[0]\n" + "fmla z20.h, z1.h, z4.h[0]\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - "fmla z24.h, z6.h, z4.h[0]\n" - "fmla z28.h, z6.h, z5.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "fmla z24.h, z1.h, z3.h[0]\n" + "fmla z28.h, z1.h, z2.h[0]\n" + "ld1h { z1.h }, p5/Z, [x10]\n" "add x21, x21, #0x10\n" - "fmla z9.h, z7.h, z0.h[0]\n" - "fmla z13.h, z7.h, z1.h[0]\n" - "fmla z17.h, z7.h, z2.h[0]\n" - "fmla z21.h, z7.h, z3.h[0]\n" - "fmla z25.h, z7.h, z4.h[0]\n" - "fmla z29.h, z7.h, z5.h[0]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" - "fmla z10.h, z6.h, z0.h[0]\n" - "fmla z14.h, z6.h, z1.h[0]\n" - "fmla z18.h, z6.h, z2.h[0]\n" - "fmla z22.h, z6.h, z3.h[0]\n" - "fmla z26.h, z6.h, z4.h[0]\n" - "fmla z30.h, z6.h, z5.h[0]\n" - "ld1h { z6.h }, p5/Z, [x12, #1, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[0]\n" - "fmla z15.h, z7.h, z1.h[0]\n" - "fmla z19.h, z7.h, z2.h[0]\n" - "fmla z23.h, z7.h, z3.h[0]\n" - "fmla z27.h, z7.h, z4.h[0]\n" - "fmla z31.h, z7.h, z5.h[0]\n" - "ld1h { z7.h }, p5/Z, [x11, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[1]\n" - "fmla z12.h, z6.h, z1.h[1]\n" - "fmla z16.h, z6.h, z2.h[1]\n" - "fmla z20.h, z6.h, z3.h[1]\n" - "fmla z24.h, z6.h, z4.h[1]\n" - "fmla z28.h, z6.h, z5.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[1]\n" - "fmla z13.h, z7.h, z1.h[1]\n" - "fmla z17.h, z7.h, z2.h[1]\n" - "fmla z21.h, z7.h, z3.h[1]\n" - "fmla z25.h, z7.h, z4.h[1]\n" - "fmla z29.h, z7.h, z5.h[1]\n" - "ld1h { z7.h }, p5/Z, [x9, #1, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[1]\n" - "fmla z14.h, z6.h, z1.h[1]\n" - "fmla z18.h, z6.h, z2.h[1]\n" - "fmla z22.h, z6.h, z3.h[1]\n" - "fmla z26.h, z6.h, z4.h[1]\n" - "fmla z30.h, z6.h, z5.h[1]\n" - "ld1h { z6.h }, p5/Z, [x12, #2, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[1]\n" - "fmla z15.h, z7.h, z1.h[1]\n" - "fmla z19.h, z7.h, z2.h[1]\n" - "fmla z23.h, z7.h, z3.h[1]\n" - "fmla z27.h, z7.h, z4.h[1]\n" - "fmla z31.h, z7.h, z5.h[1]\n" - "ld1h { z7.h }, p5/Z, [x11, #2, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[2]\n" - "fmla z12.h, z6.h, z1.h[2]\n" - "fmla z16.h, z6.h, z2.h[2]\n" - "fmla z20.h, z6.h, z3.h[2]\n" - "fmla z24.h, z6.h, z4.h[2]\n" - "fmla z28.h, z6.h, z5.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[2]\n" - "fmla z13.h, z7.h, z1.h[2]\n" - "fmla z17.h, z7.h, z2.h[2]\n" - "fmla z21.h, z7.h, z3.h[2]\n" - "fmla z25.h, z7.h, z4.h[2]\n" - "fmla z29.h, z7.h, z5.h[2]\n" - "ld1h { z7.h }, p5/Z, [x9, #2, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[2]\n" - "fmla z14.h, z6.h, z1.h[2]\n" - "fmla z18.h, z6.h, z2.h[2]\n" - "fmla z22.h, z6.h, z3.h[2]\n" - "fmla z26.h, z6.h, z4.h[2]\n" - "fmla z30.h, z6.h, z5.h[2]\n" - "ld1h { z6.h }, p5/Z, [x12, #3, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[2]\n" - "fmla z15.h, z7.h, z1.h[2]\n" - "fmla z19.h, z7.h, z2.h[2]\n" - "fmla z23.h, z7.h, z3.h[2]\n" - "fmla z27.h, z7.h, z4.h[2]\n" - "fmla z31.h, z7.h, z5.h[2]\n" - "ld1h { z7.h }, p5/Z, [x11, #3, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[3]\n" - "fmla z12.h, z6.h, z1.h[3]\n" - "fmla z16.h, z6.h, z2.h[3]\n" - "fmla z20.h, z6.h, z3.h[3]\n" - "fmla z24.h, z6.h, z4.h[3]\n" - "fmla z28.h, z6.h, z5.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[3]\n" - "fmla z13.h, z7.h, z1.h[3]\n" - "fmla z17.h, z7.h, z2.h[3]\n" - "fmla z21.h, z7.h, z3.h[3]\n" - "fmla z25.h, z7.h, z4.h[3]\n" - "fmla z29.h, z7.h, z5.h[3]\n" - "ld1h { z7.h }, p5/Z, [x9, #3, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[3]\n" - "fmla z14.h, z6.h, z1.h[3]\n" - "fmla z18.h, z6.h, z2.h[3]\n" - "fmla z22.h, z6.h, z3.h[3]\n" - "fmla z26.h, z6.h, z4.h[3]\n" - "fmla z30.h, z6.h, z5.h[3]\n" - "ld1h { z6.h }, p5/Z, [x12, #4, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[3]\n" - "fmla z15.h, z7.h, z1.h[3]\n" - "fmla z19.h, z7.h, z2.h[3]\n" - "fmla z23.h, z7.h, z3.h[3]\n" - "fmla z27.h, z7.h, z4.h[3]\n" - "fmla z31.h, z7.h, z5.h[3]\n" - "ld1h { z7.h }, p5/Z, [x11, #4, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[4]\n" - "fmla z12.h, z6.h, z1.h[4]\n" - "fmla z16.h, z6.h, z2.h[4]\n" - "fmla z20.h, z6.h, z3.h[4]\n" - "fmla z24.h, z6.h, z4.h[4]\n" - "fmla z28.h, z6.h, z5.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[4]\n" - "fmla z13.h, z7.h, z1.h[4]\n" - "fmla z17.h, z7.h, z2.h[4]\n" - "fmla z21.h, z7.h, z3.h[4]\n" - "fmla z25.h, z7.h, z4.h[4]\n" - "fmla z29.h, z7.h, z5.h[4]\n" - "ld1h { z7.h }, p5/Z, [x9, #4, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[4]\n" - "fmla z14.h, z6.h, z1.h[4]\n" - "fmla z18.h, z6.h, z2.h[4]\n" - "fmla z22.h, z6.h, z3.h[4]\n" - "fmla z26.h, z6.h, z4.h[4]\n" - "fmla z30.h, z6.h, z5.h[4]\n" - "ld1h { z6.h }, p5/Z, [x12, #5, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[4]\n" - "fmla z15.h, z7.h, z1.h[4]\n" - "fmla z19.h, z7.h, z2.h[4]\n" - "fmla z23.h, z7.h, z3.h[4]\n" - "fmla z27.h, z7.h, z4.h[4]\n" - "fmla z31.h, z7.h, z5.h[4]\n" - "ld1h { z7.h }, p5/Z, [x11, #5, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[5]\n" - "fmla z12.h, z6.h, z1.h[5]\n" - "fmla z16.h, z6.h, z2.h[5]\n" - "fmla z20.h, z6.h, z3.h[5]\n" - "fmla z24.h, z6.h, z4.h[5]\n" - "fmla z28.h, z6.h, z5.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[5]\n" - "fmla z13.h, z7.h, z1.h[5]\n" - "fmla z17.h, z7.h, z2.h[5]\n" - "fmla z21.h, z7.h, z3.h[5]\n" - "fmla z25.h, z7.h, z4.h[5]\n" - "fmla z29.h, z7.h, z5.h[5]\n" - "ld1h { z7.h }, p5/Z, [x9, #5, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[5]\n" - "fmla z14.h, z6.h, z1.h[5]\n" - "fmla z18.h, z6.h, z2.h[5]\n" - "fmla z22.h, z6.h, z3.h[5]\n" - "fmla z26.h, z6.h, z4.h[5]\n" - "fmla z30.h, z6.h, z5.h[5]\n" - "ld1h { z6.h }, p5/Z, [x12, #6, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[5]\n" - "fmla z15.h, z7.h, z1.h[5]\n" - "fmla z19.h, z7.h, z2.h[5]\n" - "fmla z23.h, z7.h, z3.h[5]\n" - "fmla z27.h, z7.h, z4.h[5]\n" - "fmla z31.h, z7.h, z5.h[5]\n" - "ld1h { z7.h }, p5/Z, [x11, #6, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[6]\n" - "fmla z12.h, z6.h, z1.h[6]\n" - "fmla z16.h, z6.h, z2.h[6]\n" - "fmla z20.h, z6.h, z3.h[6]\n" - "fmla z24.h, z6.h, z4.h[6]\n" - "fmla z28.h, z6.h, z5.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[6]\n" - "fmla z13.h, z7.h, z1.h[6]\n" - "fmla z17.h, z7.h, z2.h[6]\n" - "fmla z21.h, z7.h, z3.h[6]\n" - "fmla z25.h, z7.h, z4.h[6]\n" - "fmla z29.h, z7.h, z5.h[6]\n" - "ld1h { z7.h }, p5/Z, [x9, #6, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[6]\n" - "fmla z14.h, z6.h, z1.h[6]\n" - "fmla z18.h, z6.h, z2.h[6]\n" - "fmla z22.h, z6.h, z3.h[6]\n" - "fmla z26.h, z6.h, z4.h[6]\n" - "fmla z30.h, z6.h, z5.h[6]\n" - "ld1h { z6.h }, p5/Z, [x12, #7, MUL VL]\n" + "fmla z9.h, z0.h, z7.h[0]\n" + "fmla z13.h, z0.h, z6.h[0]\n" + "fmla z17.h, z0.h, z5.h[0]\n" + "fmla z21.h, z0.h, z4.h[0]\n" + "fmla z25.h, z0.h, z3.h[0]\n" + "fmla z29.h, z0.h, z2.h[0]\n" + "ld1h { z0.h }, p5/Z, [x9]\n" + "fmla z10.h, z1.h, z7.h[0]\n" + "fmla z14.h, z1.h, z6.h[0]\n" + "fmla z18.h, z1.h, z5.h[0]\n" + "fmla z22.h, z1.h, z4.h[0]\n" + "fmla z26.h, z1.h, z3.h[0]\n" + "fmla z30.h, z1.h, z2.h[0]\n" + "ld1h { z1.h }, p5/Z, [x12, #1, MUL VL]\n" + "fmla z11.h, z0.h, z7.h[0]\n" + "fmla z15.h, z0.h, z6.h[0]\n" + "fmla z19.h, z0.h, z5.h[0]\n" + "fmla z23.h, z0.h, z4.h[0]\n" + "fmla z27.h, z0.h, z3.h[0]\n" + "fmla z31.h, z0.h, z2.h[0]\n" + "ld1h { z0.h }, p5/Z, [x11, #1, MUL VL]\n" + "fmla z8.h, z1.h, z7.h[1]\n" + "fmla z12.h, z1.h, z6.h[1]\n" + "fmla z16.h, z1.h, z5.h[1]\n" + "fmla z20.h, z1.h, z4.h[1]\n" + "fmla z24.h, z1.h, z3.h[1]\n" + "fmla z28.h, z1.h, z2.h[1]\n" + "ld1h { z1.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z9.h, z0.h, z7.h[1]\n" + "fmla z13.h, z0.h, z6.h[1]\n" + "fmla z17.h, z0.h, z5.h[1]\n" + "fmla z21.h, z0.h, z4.h[1]\n" + "fmla z25.h, z0.h, z3.h[1]\n" + "fmla z29.h, z0.h, z2.h[1]\n" + "ld1h { z0.h }, p5/Z, [x9, #1, MUL VL]\n" + "fmla z10.h, z1.h, z7.h[1]\n" + "fmla z14.h, z1.h, z6.h[1]\n" + "fmla z18.h, z1.h, z5.h[1]\n" + "fmla z22.h, z1.h, z4.h[1]\n" + "fmla z26.h, z1.h, z3.h[1]\n" + "fmla z30.h, z1.h, z2.h[1]\n" + "ld1h { z1.h }, p5/Z, [x12, #2, MUL VL]\n" + "fmla z11.h, z0.h, z7.h[1]\n" + "fmla z15.h, z0.h, z6.h[1]\n" + "fmla z19.h, z0.h, z5.h[1]\n" + "fmla z23.h, z0.h, z4.h[1]\n" + "fmla z27.h, z0.h, z3.h[1]\n" + "fmla z31.h, z0.h, z2.h[1]\n" + "ld1h { z0.h }, p5/Z, [x11, #2, MUL VL]\n" + "fmla z8.h, z1.h, z7.h[2]\n" + "fmla z12.h, z1.h, z6.h[2]\n" + "fmla z16.h, z1.h, z5.h[2]\n" + "fmla z20.h, z1.h, z4.h[2]\n" + "fmla z24.h, z1.h, z3.h[2]\n" + "fmla z28.h, z1.h, z2.h[2]\n" + "ld1h { z1.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.h, z0.h, z7.h[2]\n" + "fmla z13.h, z0.h, z6.h[2]\n" + "fmla z17.h, z0.h, z5.h[2]\n" + "fmla z21.h, z0.h, z4.h[2]\n" + "fmla z25.h, z0.h, z3.h[2]\n" + "fmla z29.h, z0.h, z2.h[2]\n" + "ld1h { z0.h }, p5/Z, [x9, #2, MUL VL]\n" + "fmla z10.h, z1.h, z7.h[2]\n" + "fmla z14.h, z1.h, z6.h[2]\n" + "fmla z18.h, z1.h, z5.h[2]\n" + "fmla z22.h, z1.h, z4.h[2]\n" + "fmla z26.h, z1.h, z3.h[2]\n" + "fmla z30.h, z1.h, z2.h[2]\n" + "ld1h { z1.h }, p5/Z, [x12, #3, MUL VL]\n" + "fmla z11.h, z0.h, z7.h[2]\n" + "fmla z15.h, z0.h, z6.h[2]\n" + "fmla z19.h, z0.h, z5.h[2]\n" + "fmla z23.h, z0.h, z4.h[2]\n" + "fmla z27.h, z0.h, z3.h[2]\n" + "fmla z31.h, z0.h, z2.h[2]\n" + "ld1h { z0.h }, p5/Z, [x11, #3, MUL VL]\n" + "fmla z8.h, z1.h, z7.h[3]\n" + "fmla z12.h, z1.h, z6.h[3]\n" + "fmla z16.h, z1.h, z5.h[3]\n" + "fmla z20.h, z1.h, z4.h[3]\n" + "fmla z24.h, z1.h, z3.h[3]\n" + "fmla z28.h, z1.h, z2.h[3]\n" + "ld1h { z1.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z9.h, z0.h, z7.h[3]\n" + "fmla z13.h, z0.h, z6.h[3]\n" + "fmla z17.h, z0.h, z5.h[3]\n" + "fmla z21.h, z0.h, z4.h[3]\n" + "fmla z25.h, z0.h, z3.h[3]\n" + "fmla z29.h, z0.h, z2.h[3]\n" + "ld1h { z0.h }, p5/Z, [x9, #3, MUL VL]\n" + "fmla z10.h, z1.h, z7.h[3]\n" + "fmla z14.h, z1.h, z6.h[3]\n" + "fmla z18.h, z1.h, z5.h[3]\n" + "fmla z22.h, z1.h, z4.h[3]\n" + "fmla z26.h, z1.h, z3.h[3]\n" + "fmla z30.h, z1.h, z2.h[3]\n" + "ld1h { z1.h }, p5/Z, [x12, #4, MUL VL]\n" + "fmla z11.h, z0.h, z7.h[3]\n" + "fmla z15.h, z0.h, z6.h[3]\n" + "fmla z19.h, z0.h, z5.h[3]\n" + "fmla z23.h, z0.h, z4.h[3]\n" + "fmla z27.h, z0.h, z3.h[3]\n" + "fmla z31.h, z0.h, z2.h[3]\n" + "ld1h { z0.h }, p5/Z, [x11, #4, MUL VL]\n" + "fmla z8.h, z1.h, z7.h[4]\n" + "fmla z12.h, z1.h, z6.h[4]\n" + "fmla z16.h, z1.h, z5.h[4]\n" + "fmla z20.h, z1.h, z4.h[4]\n" + "fmla z24.h, z1.h, z3.h[4]\n" + "fmla z28.h, z1.h, z2.h[4]\n" + "ld1h { z1.h }, p5/Z, [x10, #4, MUL VL]\n" + "fmla z9.h, z0.h, z7.h[4]\n" + "fmla z13.h, z0.h, z6.h[4]\n" + "fmla z17.h, z0.h, z5.h[4]\n" + "fmla z21.h, z0.h, z4.h[4]\n" + "fmla z25.h, z0.h, z3.h[4]\n" + "fmla z29.h, z0.h, z2.h[4]\n" + "ld1h { z0.h }, p5/Z, [x9, #4, MUL VL]\n" + "fmla z10.h, z1.h, z7.h[4]\n" + "fmla z14.h, z1.h, z6.h[4]\n" + "fmla z18.h, z1.h, z5.h[4]\n" + "fmla z22.h, z1.h, z4.h[4]\n" + "fmla z26.h, z1.h, z3.h[4]\n" + "fmla z30.h, z1.h, z2.h[4]\n" + "ld1h { z1.h }, p5/Z, [x12, #5, MUL VL]\n" + "fmla z11.h, z0.h, z7.h[4]\n" + "fmla z15.h, z0.h, z6.h[4]\n" + "fmla z19.h, z0.h, z5.h[4]\n" + "fmla z23.h, z0.h, z4.h[4]\n" + "fmla z27.h, z0.h, z3.h[4]\n" + "fmla z31.h, z0.h, z2.h[4]\n" + "ld1h { z0.h }, p5/Z, [x11, #5, MUL VL]\n" + "fmla z8.h, z1.h, z7.h[5]\n" + "fmla z12.h, z1.h, z6.h[5]\n" + "fmla z16.h, z1.h, z5.h[5]\n" + "fmla z20.h, z1.h, z4.h[5]\n" + "fmla z24.h, z1.h, z3.h[5]\n" + "fmla z28.h, z1.h, z2.h[5]\n" + "ld1h { z1.h }, p5/Z, [x10, #5, MUL VL]\n" + "fmla z9.h, z0.h, z7.h[5]\n" + "fmla z13.h, z0.h, z6.h[5]\n" + "fmla z17.h, z0.h, z5.h[5]\n" + "fmla z21.h, z0.h, z4.h[5]\n" + "fmla z25.h, z0.h, z3.h[5]\n" + "fmla z29.h, z0.h, z2.h[5]\n" + "ld1h { z0.h }, p5/Z, [x9, #5, MUL VL]\n" + "fmla z10.h, z1.h, z7.h[5]\n" + "fmla z14.h, z1.h, z6.h[5]\n" + "fmla z18.h, z1.h, z5.h[5]\n" + "fmla z22.h, z1.h, z4.h[5]\n" + "fmla z26.h, z1.h, z3.h[5]\n" + "fmla z30.h, z1.h, z2.h[5]\n" + "ld1h { z1.h }, p5/Z, [x12, #6, MUL VL]\n" + "fmla z11.h, z0.h, z7.h[5]\n" + "fmla z15.h, z0.h, z6.h[5]\n" + "fmla z19.h, z0.h, z5.h[5]\n" + "fmla z23.h, z0.h, z4.h[5]\n" + "fmla z27.h, z0.h, z3.h[5]\n" + "fmla z31.h, z0.h, z2.h[5]\n" + "ld1h { z0.h }, p5/Z, [x11, #6, MUL VL]\n" + "fmla z8.h, z1.h, z7.h[6]\n" + "fmla z12.h, z1.h, z6.h[6]\n" + "fmla z16.h, z1.h, z5.h[6]\n" + "fmla z20.h, z1.h, z4.h[6]\n" + "fmla z24.h, z1.h, z3.h[6]\n" + "fmla z28.h, z1.h, z2.h[6]\n" + "ld1h { z1.h }, p5/Z, [x10, #6, MUL VL]\n" + "fmla z9.h, z0.h, z7.h[6]\n" + "fmla z13.h, z0.h, z6.h[6]\n" + "fmla z17.h, z0.h, z5.h[6]\n" + "fmla z21.h, z0.h, z4.h[6]\n" + "fmla z25.h, z0.h, z3.h[6]\n" + "fmla z29.h, z0.h, z2.h[6]\n" + "ld1h { z0.h }, p5/Z, [x9, #6, MUL VL]\n" + "fmla z10.h, z1.h, z7.h[6]\n" + "fmla z14.h, z1.h, z6.h[6]\n" + "fmla z18.h, z1.h, z5.h[6]\n" + "fmla z22.h, z1.h, z4.h[6]\n" + "fmla z26.h, z1.h, z3.h[6]\n" + "fmla z30.h, z1.h, z2.h[6]\n" + "ld1h { z1.h }, p5/Z, [x12, #7, MUL VL]\n" "addvl x12, x12, #8\n" - "fmla z11.h, z7.h, z0.h[6]\n" - "fmla z15.h, z7.h, z1.h[6]\n" - "fmla z19.h, z7.h, z2.h[6]\n" - "fmla z23.h, z7.h, z3.h[6]\n" - "fmla z27.h, z7.h, z4.h[6]\n" - "fmla z31.h, z7.h, z5.h[6]\n" - "ld1h { z7.h }, p5/Z, [x11, #7, MUL VL]\n" + "fmla z11.h, z0.h, z7.h[6]\n" + "fmla z15.h, z0.h, z6.h[6]\n" + "fmla z19.h, z0.h, z5.h[6]\n" + "fmla z23.h, z0.h, z4.h[6]\n" + "fmla z27.h, z0.h, z3.h[6]\n" + "fmla z31.h, z0.h, z2.h[6]\n" + "ld1h { z0.h }, p5/Z, [x11, #7, MUL VL]\n" "addvl x11, x11, #8\n" - "fmla z8.h, z6.h, z0.h[7]\n" - "fmla z12.h, z6.h, z1.h[7]\n" - "fmla z16.h, z6.h, z2.h[7]\n" - "fmla z20.h, z6.h, z3.h[7]\n" - "fmla z24.h, z6.h, z4.h[7]\n" - "fmla z28.h, z6.h, z5.h[7]\n" - "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n" + "fmla z8.h, z1.h, z7.h[7]\n" + "fmla z12.h, z1.h, z6.h[7]\n" + "fmla z16.h, z1.h, z5.h[7]\n" + "fmla z20.h, z1.h, z4.h[7]\n" + "fmla z24.h, z1.h, z3.h[7]\n" + "fmla z28.h, z1.h, z2.h[7]\n" + "ld1h { z1.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - "fmla z9.h, z7.h, z0.h[7]\n" - "fmla z13.h, z7.h, z1.h[7]\n" - "fmla z17.h, z7.h, z2.h[7]\n" - "fmla z21.h, z7.h, z3.h[7]\n" - "fmla z25.h, z7.h, z4.h[7]\n" - "fmla z29.h, z7.h, z5.h[7]\n" - "ld1h { z7.h }, p5/Z, [x9, #7, MUL VL]\n" + "fmla z9.h, z0.h, z7.h[7]\n" + "fmla z13.h, z0.h, z6.h[7]\n" + "fmla z17.h, z0.h, z5.h[7]\n" + "fmla z21.h, z0.h, z4.h[7]\n" + "fmla z25.h, z0.h, z3.h[7]\n" + "fmla z29.h, z0.h, z2.h[7]\n" + "ld1h { z0.h }, p5/Z, [x9, #7, MUL VL]\n" "addvl x9, x9, #8\n" - "fmla z10.h, z6.h, z0.h[7]\n" - "fmla z14.h, z6.h, z1.h[7]\n" - "fmla z18.h, z6.h, z2.h[7]\n" - "fmla z22.h, z6.h, z3.h[7]\n" - "fmla z26.h, z6.h, z4.h[7]\n" - "fmla z30.h, z6.h, z5.h[7]\n" - "fmla z11.h, z7.h, z0.h[7]\n" - "fmla z15.h, z7.h, z1.h[7]\n" - "fmla z19.h, z7.h, z2.h[7]\n" - "fmla z23.h, z7.h, z3.h[7]\n" - "fmla z27.h, z7.h, z4.h[7]\n" - "fmla z31.h, z7.h, z5.h[7]\n" + "fmla z10.h, z1.h, z7.h[7]\n" + "fmla z14.h, z1.h, z6.h[7]\n" + "fmla z18.h, z1.h, z5.h[7]\n" + "fmla z22.h, z1.h, z4.h[7]\n" + "fmla z26.h, z1.h, z3.h[7]\n" + "fmla z30.h, z1.h, z2.h[7]\n" + "fmla z11.h, z0.h, z7.h[7]\n" + "fmla z15.h, z0.h, z6.h[7]\n" + "fmla z19.h, z0.h, z5.h[7]\n" + "fmla z23.h, z0.h, z4.h[7]\n" + "fmla z27.h, z0.h, z3.h[7]\n" + "fmla z31.h, z0.h, z2.h[7]\n" "bgt 80b\n" "81:" // Height 6: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" @@ -2933,275 +2933,275 @@ void sve_ffhybrid_fp16_mla_6x4VL ( "ld1rqh { z3.h }, p0/Z, [x23]\n" "ld1rqh { z4.h }, p0/Z, [x22]\n" "ld1rqh { z5.h }, p0/Z, [x21]\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[0]\n" - "fmla z12.h, z6.h, z1.h[0]\n" - "fmla z16.h, z6.h, z2.h[0]\n" - "fmla z20.h, z6.h, z3.h[0]\n" + "ld1h { z7.h }, p5/Z, [x12]\n" + "ld1h { z6.h }, p5/Z, [x11]\n" + "fmla z8.h, z7.h, z0.h[0]\n" + "fmla z12.h, z7.h, z1.h[0]\n" + "fmla z16.h, z7.h, z2.h[0]\n" + "fmla z20.h, z7.h, z3.h[0]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" - "fmla z24.h, z6.h, z4.h[0]\n" - "fmla z28.h, z6.h, z5.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "fmla z24.h, z7.h, z4.h[0]\n" + "fmla z28.h, z7.h, z5.h[0]\n" + "ld1h { z7.h }, p5/Z, [x10]\n" "addvl x10, x10, #1\n" - "fmla z9.h, z7.h, z0.h[0]\n" - "fmla z13.h, z7.h, z1.h[0]\n" - "fmla z17.h, z7.h, z2.h[0]\n" - "fmla z21.h, z7.h, z3.h[0]\n" - "fmla z25.h, z7.h, z4.h[0]\n" - "fmla z29.h, z7.h, z5.h[0]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z9.h, z6.h, z0.h[0]\n" + "fmla z13.h, z6.h, z1.h[0]\n" + "fmla z17.h, z6.h, z2.h[0]\n" + "fmla z21.h, z6.h, z3.h[0]\n" + "fmla z25.h, z6.h, z4.h[0]\n" + "fmla z29.h, z6.h, z5.h[0]\n" + "ld1h { z6.h }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.h, z6.h, z0.h[0]\n" - "fmla z14.h, z6.h, z1.h[0]\n" - "fmla z18.h, z6.h, z2.h[0]\n" - "fmla z22.h, z6.h, z3.h[0]\n" - "fmla z26.h, z6.h, z4.h[0]\n" - "fmla z30.h, z6.h, z5.h[0]\n" - "fmla z11.h, z7.h, z0.h[0]\n" - "fmla z15.h, z7.h, z1.h[0]\n" - "fmla z19.h, z7.h, z2.h[0]\n" - "fmla z23.h, z7.h, z3.h[0]\n" - "fmla z27.h, z7.h, z4.h[0]\n" - "fmla z31.h, z7.h, z5.h[0]\n" + "fmla z10.h, z7.h, z0.h[0]\n" + "fmla z14.h, z7.h, z1.h[0]\n" + "fmla z18.h, z7.h, z2.h[0]\n" + "fmla z22.h, z7.h, z3.h[0]\n" + "fmla z26.h, z7.h, z4.h[0]\n" + "fmla z30.h, z7.h, z5.h[0]\n" + "fmla z11.h, z6.h, z0.h[0]\n" + "fmla z15.h, z6.h, z1.h[0]\n" + "fmla z19.h, z6.h, z2.h[0]\n" + "fmla z23.h, z6.h, z3.h[0]\n" + "fmla z27.h, z6.h, z4.h[0]\n" + "fmla z31.h, z6.h, z5.h[0]\n" "ble 82f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[1]\n" - "fmla z12.h, z6.h, z1.h[1]\n" - "fmla z16.h, z6.h, z2.h[1]\n" - "fmla z20.h, z6.h, z3.h[1]\n" + "ld1h { z7.h }, p5/Z, [x12]\n" + "ld1h { z6.h }, p5/Z, [x11]\n" + "fmla z8.h, z7.h, z0.h[1]\n" + "fmla z12.h, z7.h, z1.h[1]\n" + "fmla z16.h, z7.h, z2.h[1]\n" + "fmla z20.h, z7.h, z3.h[1]\n" "subs x27, x27, #0x1\n" "addvl x12, x12, #1\n" - "fmla z24.h, z6.h, z4.h[1]\n" - "fmla z28.h, z6.h, z5.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "fmla z24.h, z7.h, z4.h[1]\n" + "fmla z28.h, z7.h, z5.h[1]\n" + "ld1h { z7.h }, p5/Z, [x10]\n" "addvl x11, x11, #1\n" - "fmla z9.h, z7.h, z0.h[1]\n" - "fmla z13.h, z7.h, z1.h[1]\n" + "fmla z9.h, z6.h, z0.h[1]\n" + "fmla z13.h, z6.h, z1.h[1]\n" "addvl x10, x10, #1\n" - "fmla z17.h, z7.h, z2.h[1]\n" - "fmla z21.h, z7.h, z3.h[1]\n" - "fmla z25.h, z7.h, z4.h[1]\n" - "fmla z29.h, z7.h, z5.h[1]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z17.h, z6.h, z2.h[1]\n" + "fmla z21.h, z6.h, z3.h[1]\n" + "fmla z25.h, z6.h, z4.h[1]\n" + "fmla z29.h, z6.h, z5.h[1]\n" + "ld1h { z6.h }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.h, z6.h, z0.h[1]\n" - "fmla z14.h, z6.h, z1.h[1]\n" - "fmla z18.h, z6.h, z2.h[1]\n" - "fmla z22.h, z6.h, z3.h[1]\n" - "fmla z26.h, z6.h, z4.h[1]\n" - "fmla z30.h, z6.h, z5.h[1]\n" - "fmla z11.h, z7.h, z0.h[1]\n" - "fmla z15.h, z7.h, z1.h[1]\n" - "fmla z19.h, z7.h, z2.h[1]\n" - "fmla z23.h, z7.h, z3.h[1]\n" - "fmla z27.h, z7.h, z4.h[1]\n" - "fmla z31.h, z7.h, z5.h[1]\n" + "fmla z10.h, z7.h, z0.h[1]\n" + "fmla z14.h, z7.h, z1.h[1]\n" + "fmla z18.h, z7.h, z2.h[1]\n" + "fmla z22.h, z7.h, z3.h[1]\n" + "fmla z26.h, z7.h, z4.h[1]\n" + "fmla z30.h, z7.h, z5.h[1]\n" + "fmla z11.h, z6.h, z0.h[1]\n" + "fmla z15.h, z6.h, z1.h[1]\n" + "fmla z19.h, z6.h, z2.h[1]\n" + "fmla z23.h, z6.h, z3.h[1]\n" + "fmla z27.h, z6.h, z4.h[1]\n" + "fmla z31.h, z6.h, z5.h[1]\n" "ble 82f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[2]\n" - "fmla z12.h, z6.h, z1.h[2]\n" - "fmla z16.h, z6.h, z2.h[2]\n" - "fmla z20.h, z6.h, z3.h[2]\n" + "ld1h { z7.h }, p5/Z, [x12]\n" + "ld1h { z6.h }, p5/Z, [x11]\n" + "fmla z8.h, z7.h, z0.h[2]\n" + "fmla z12.h, z7.h, z1.h[2]\n" + "fmla z16.h, z7.h, z2.h[2]\n" + "fmla z20.h, z7.h, z3.h[2]\n" "subs x27, x27, #0x1\n" "addvl x12, x12, #1\n" - "fmla z24.h, z6.h, z4.h[2]\n" - "fmla z28.h, z6.h, z5.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "fmla z24.h, z7.h, z4.h[2]\n" + "fmla z28.h, z7.h, z5.h[2]\n" + "ld1h { z7.h }, p5/Z, [x10]\n" "addvl x11, x11, #1\n" - "fmla z9.h, z7.h, z0.h[2]\n" - "fmla z13.h, z7.h, z1.h[2]\n" + "fmla z9.h, z6.h, z0.h[2]\n" + "fmla z13.h, z6.h, z1.h[2]\n" "addvl x10, x10, #1\n" - "fmla z17.h, z7.h, z2.h[2]\n" - "fmla z21.h, z7.h, z3.h[2]\n" - "fmla z25.h, z7.h, z4.h[2]\n" - "fmla z29.h, z7.h, z5.h[2]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z17.h, z6.h, z2.h[2]\n" + "fmla z21.h, z6.h, z3.h[2]\n" + "fmla z25.h, z6.h, z4.h[2]\n" + "fmla z29.h, z6.h, z5.h[2]\n" + "ld1h { z6.h }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.h, z6.h, z0.h[2]\n" - "fmla z14.h, z6.h, z1.h[2]\n" - "fmla z18.h, z6.h, z2.h[2]\n" - "fmla z22.h, z6.h, z3.h[2]\n" - "fmla z26.h, z6.h, z4.h[2]\n" - "fmla z30.h, z6.h, z5.h[2]\n" - "fmla z11.h, z7.h, z0.h[2]\n" - "fmla z15.h, z7.h, z1.h[2]\n" - "fmla z19.h, z7.h, z2.h[2]\n" - "fmla z23.h, z7.h, z3.h[2]\n" - "fmla z27.h, z7.h, z4.h[2]\n" - "fmla z31.h, z7.h, z5.h[2]\n" + "fmla z10.h, z7.h, z0.h[2]\n" + "fmla z14.h, z7.h, z1.h[2]\n" + "fmla z18.h, z7.h, z2.h[2]\n" + "fmla z22.h, z7.h, z3.h[2]\n" + "fmla z26.h, z7.h, z4.h[2]\n" + "fmla z30.h, z7.h, z5.h[2]\n" + "fmla z11.h, z6.h, z0.h[2]\n" + "fmla z15.h, z6.h, z1.h[2]\n" + "fmla z19.h, z6.h, z2.h[2]\n" + "fmla z23.h, z6.h, z3.h[2]\n" + "fmla z27.h, z6.h, z4.h[2]\n" + "fmla z31.h, z6.h, z5.h[2]\n" "ble 82f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[3]\n" - "fmla z12.h, z6.h, z1.h[3]\n" - "fmla z16.h, z6.h, z2.h[3]\n" - "fmla z20.h, z6.h, z3.h[3]\n" + "ld1h { z7.h }, p5/Z, [x12]\n" + "ld1h { z6.h }, p5/Z, [x11]\n" + "fmla z8.h, z7.h, z0.h[3]\n" + "fmla z12.h, z7.h, z1.h[3]\n" + "fmla z16.h, z7.h, z2.h[3]\n" + "fmla z20.h, z7.h, z3.h[3]\n" "subs x27, x27, #0x1\n" "addvl x12, x12, #1\n" - "fmla z24.h, z6.h, z4.h[3]\n" - "fmla z28.h, z6.h, z5.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "fmla z24.h, z7.h, z4.h[3]\n" + "fmla z28.h, z7.h, z5.h[3]\n" + "ld1h { z7.h }, p5/Z, [x10]\n" "addvl x11, x11, #1\n" - "fmla z9.h, z7.h, z0.h[3]\n" - "fmla z13.h, z7.h, z1.h[3]\n" + "fmla z9.h, z6.h, z0.h[3]\n" + "fmla z13.h, z6.h, z1.h[3]\n" "addvl x10, x10, #1\n" - "fmla z17.h, z7.h, z2.h[3]\n" - "fmla z21.h, z7.h, z3.h[3]\n" - "fmla z25.h, z7.h, z4.h[3]\n" - "fmla z29.h, z7.h, z5.h[3]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z17.h, z6.h, z2.h[3]\n" + "fmla z21.h, z6.h, z3.h[3]\n" + "fmla z25.h, z6.h, z4.h[3]\n" + "fmla z29.h, z6.h, z5.h[3]\n" + "ld1h { z6.h }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.h, z6.h, z0.h[3]\n" - "fmla z14.h, z6.h, z1.h[3]\n" - "fmla z18.h, z6.h, z2.h[3]\n" - "fmla z22.h, z6.h, z3.h[3]\n" - "fmla z26.h, z6.h, z4.h[3]\n" - "fmla z30.h, z6.h, z5.h[3]\n" - "fmla z11.h, z7.h, z0.h[3]\n" - "fmla z15.h, z7.h, z1.h[3]\n" - "fmla z19.h, z7.h, z2.h[3]\n" - "fmla z23.h, z7.h, z3.h[3]\n" - "fmla z27.h, z7.h, z4.h[3]\n" - "fmla z31.h, z7.h, z5.h[3]\n" + "fmla z10.h, z7.h, z0.h[3]\n" + "fmla z14.h, z7.h, z1.h[3]\n" + "fmla z18.h, z7.h, z2.h[3]\n" + "fmla z22.h, z7.h, z3.h[3]\n" + "fmla z26.h, z7.h, z4.h[3]\n" + "fmla z30.h, z7.h, z5.h[3]\n" + "fmla z11.h, z6.h, z0.h[3]\n" + "fmla z15.h, z6.h, z1.h[3]\n" + "fmla z19.h, z6.h, z2.h[3]\n" + "fmla z23.h, z6.h, z3.h[3]\n" + "fmla z27.h, z6.h, z4.h[3]\n" + "fmla z31.h, z6.h, z5.h[3]\n" "ble 82f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[4]\n" - "fmla z12.h, z6.h, z1.h[4]\n" - "fmla z16.h, z6.h, z2.h[4]\n" - "fmla z20.h, z6.h, z3.h[4]\n" + "ld1h { z7.h }, p5/Z, [x12]\n" + "ld1h { z6.h }, p5/Z, [x11]\n" + "fmla z8.h, z7.h, z0.h[4]\n" + "fmla z12.h, z7.h, z1.h[4]\n" + "fmla z16.h, z7.h, z2.h[4]\n" + "fmla z20.h, z7.h, z3.h[4]\n" "subs x27, x27, #0x1\n" "addvl x12, x12, #1\n" - "fmla z24.h, z6.h, z4.h[4]\n" - "fmla z28.h, z6.h, z5.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "fmla z24.h, z7.h, z4.h[4]\n" + "fmla z28.h, z7.h, z5.h[4]\n" + "ld1h { z7.h }, p5/Z, [x10]\n" "addvl x11, x11, #1\n" - "fmla z9.h, z7.h, z0.h[4]\n" - "fmla z13.h, z7.h, z1.h[4]\n" + "fmla z9.h, z6.h, z0.h[4]\n" + "fmla z13.h, z6.h, z1.h[4]\n" "addvl x10, x10, #1\n" - "fmla z17.h, z7.h, z2.h[4]\n" - "fmla z21.h, z7.h, z3.h[4]\n" - "fmla z25.h, z7.h, z4.h[4]\n" - "fmla z29.h, z7.h, z5.h[4]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z17.h, z6.h, z2.h[4]\n" + "fmla z21.h, z6.h, z3.h[4]\n" + "fmla z25.h, z6.h, z4.h[4]\n" + "fmla z29.h, z6.h, z5.h[4]\n" + "ld1h { z6.h }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.h, z6.h, z0.h[4]\n" - "fmla z14.h, z6.h, z1.h[4]\n" - "fmla z18.h, z6.h, z2.h[4]\n" - "fmla z22.h, z6.h, z3.h[4]\n" - "fmla z26.h, z6.h, z4.h[4]\n" - "fmla z30.h, z6.h, z5.h[4]\n" - "fmla z11.h, z7.h, z0.h[4]\n" - "fmla z15.h, z7.h, z1.h[4]\n" - "fmla z19.h, z7.h, z2.h[4]\n" - "fmla z23.h, z7.h, z3.h[4]\n" - "fmla z27.h, z7.h, z4.h[4]\n" - "fmla z31.h, z7.h, z5.h[4]\n" + "fmla z10.h, z7.h, z0.h[4]\n" + "fmla z14.h, z7.h, z1.h[4]\n" + "fmla z18.h, z7.h, z2.h[4]\n" + "fmla z22.h, z7.h, z3.h[4]\n" + "fmla z26.h, z7.h, z4.h[4]\n" + "fmla z30.h, z7.h, z5.h[4]\n" + "fmla z11.h, z6.h, z0.h[4]\n" + "fmla z15.h, z6.h, z1.h[4]\n" + "fmla z19.h, z6.h, z2.h[4]\n" + "fmla z23.h, z6.h, z3.h[4]\n" + "fmla z27.h, z6.h, z4.h[4]\n" + "fmla z31.h, z6.h, z5.h[4]\n" "ble 82f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[5]\n" - "fmla z12.h, z6.h, z1.h[5]\n" - "fmla z16.h, z6.h, z2.h[5]\n" - "fmla z20.h, z6.h, z3.h[5]\n" + "ld1h { z7.h }, p5/Z, [x12]\n" + "ld1h { z6.h }, p5/Z, [x11]\n" + "fmla z8.h, z7.h, z0.h[5]\n" + "fmla z12.h, z7.h, z1.h[5]\n" + "fmla z16.h, z7.h, z2.h[5]\n" + "fmla z20.h, z7.h, z3.h[5]\n" "subs x27, x27, #0x1\n" "addvl x12, x12, #1\n" - "fmla z24.h, z6.h, z4.h[5]\n" - "fmla z28.h, z6.h, z5.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "fmla z24.h, z7.h, z4.h[5]\n" + "fmla z28.h, z7.h, z5.h[5]\n" + "ld1h { z7.h }, p5/Z, [x10]\n" "addvl x11, x11, #1\n" - "fmla z9.h, z7.h, z0.h[5]\n" - "fmla z13.h, z7.h, z1.h[5]\n" + "fmla z9.h, z6.h, z0.h[5]\n" + "fmla z13.h, z6.h, z1.h[5]\n" "addvl x10, x10, #1\n" - "fmla z17.h, z7.h, z2.h[5]\n" - "fmla z21.h, z7.h, z3.h[5]\n" - "fmla z25.h, z7.h, z4.h[5]\n" - "fmla z29.h, z7.h, z5.h[5]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z17.h, z6.h, z2.h[5]\n" + "fmla z21.h, z6.h, z3.h[5]\n" + "fmla z25.h, z6.h, z4.h[5]\n" + "fmla z29.h, z6.h, z5.h[5]\n" + "ld1h { z6.h }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.h, z6.h, z0.h[5]\n" - "fmla z14.h, z6.h, z1.h[5]\n" - "fmla z18.h, z6.h, z2.h[5]\n" - "fmla z22.h, z6.h, z3.h[5]\n" - "fmla z26.h, z6.h, z4.h[5]\n" - "fmla z30.h, z6.h, z5.h[5]\n" - "fmla z11.h, z7.h, z0.h[5]\n" - "fmla z15.h, z7.h, z1.h[5]\n" - "fmla z19.h, z7.h, z2.h[5]\n" - "fmla z23.h, z7.h, z3.h[5]\n" - "fmla z27.h, z7.h, z4.h[5]\n" - "fmla z31.h, z7.h, z5.h[5]\n" + "fmla z10.h, z7.h, z0.h[5]\n" + "fmla z14.h, z7.h, z1.h[5]\n" + "fmla z18.h, z7.h, z2.h[5]\n" + "fmla z22.h, z7.h, z3.h[5]\n" + "fmla z26.h, z7.h, z4.h[5]\n" + "fmla z30.h, z7.h, z5.h[5]\n" + "fmla z11.h, z6.h, z0.h[5]\n" + "fmla z15.h, z6.h, z1.h[5]\n" + "fmla z19.h, z6.h, z2.h[5]\n" + "fmla z23.h, z6.h, z3.h[5]\n" + "fmla z27.h, z6.h, z4.h[5]\n" + "fmla z31.h, z6.h, z5.h[5]\n" "ble 82f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[6]\n" - "fmla z12.h, z6.h, z1.h[6]\n" - "fmla z16.h, z6.h, z2.h[6]\n" - "fmla z20.h, z6.h, z3.h[6]\n" + "ld1h { z7.h }, p5/Z, [x12]\n" + "ld1h { z6.h }, p5/Z, [x11]\n" + "fmla z8.h, z7.h, z0.h[6]\n" + "fmla z12.h, z7.h, z1.h[6]\n" + "fmla z16.h, z7.h, z2.h[6]\n" + "fmla z20.h, z7.h, z3.h[6]\n" "subs x27, x27, #0x1\n" "addvl x12, x12, #1\n" - "fmla z24.h, z6.h, z4.h[6]\n" - "fmla z28.h, z6.h, z5.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "fmla z24.h, z7.h, z4.h[6]\n" + "fmla z28.h, z7.h, z5.h[6]\n" + "ld1h { z7.h }, p5/Z, [x10]\n" "addvl x11, x11, #1\n" - "fmla z9.h, z7.h, z0.h[6]\n" - "fmla z13.h, z7.h, z1.h[6]\n" + "fmla z9.h, z6.h, z0.h[6]\n" + "fmla z13.h, z6.h, z1.h[6]\n" "addvl x10, x10, #1\n" - "fmla z17.h, z7.h, z2.h[6]\n" - "fmla z21.h, z7.h, z3.h[6]\n" - "fmla z25.h, z7.h, z4.h[6]\n" - "fmla z29.h, z7.h, z5.h[6]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z17.h, z6.h, z2.h[6]\n" + "fmla z21.h, z6.h, z3.h[6]\n" + "fmla z25.h, z6.h, z4.h[6]\n" + "fmla z29.h, z6.h, z5.h[6]\n" + "ld1h { z6.h }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.h, z6.h, z0.h[6]\n" - "fmla z14.h, z6.h, z1.h[6]\n" - "fmla z18.h, z6.h, z2.h[6]\n" - "fmla z22.h, z6.h, z3.h[6]\n" - "fmla z26.h, z6.h, z4.h[6]\n" - "fmla z30.h, z6.h, z5.h[6]\n" - "fmla z11.h, z7.h, z0.h[6]\n" - "fmla z15.h, z7.h, z1.h[6]\n" - "fmla z19.h, z7.h, z2.h[6]\n" - "fmla z23.h, z7.h, z3.h[6]\n" - "fmla z27.h, z7.h, z4.h[6]\n" - "fmla z31.h, z7.h, z5.h[6]\n" + "fmla z10.h, z7.h, z0.h[6]\n" + "fmla z14.h, z7.h, z1.h[6]\n" + "fmla z18.h, z7.h, z2.h[6]\n" + "fmla z22.h, z7.h, z3.h[6]\n" + "fmla z26.h, z7.h, z4.h[6]\n" + "fmla z30.h, z7.h, z5.h[6]\n" + "fmla z11.h, z6.h, z0.h[6]\n" + "fmla z15.h, z6.h, z1.h[6]\n" + "fmla z19.h, z6.h, z2.h[6]\n" + "fmla z23.h, z6.h, z3.h[6]\n" + "fmla z27.h, z6.h, z4.h[6]\n" + "fmla z31.h, z6.h, z5.h[6]\n" "ble 82f\n" - "ld1h { z6.h }, p5/Z, [x12]\n" - "ld1h { z7.h }, p5/Z, [x11]\n" - "fmla z8.h, z6.h, z0.h[7]\n" - "fmla z12.h, z6.h, z1.h[7]\n" - "fmla z16.h, z6.h, z2.h[7]\n" - "fmla z20.h, z6.h, z3.h[7]\n" + "ld1h { z7.h }, p5/Z, [x12]\n" + "ld1h { z6.h }, p5/Z, [x11]\n" + "fmla z8.h, z7.h, z0.h[7]\n" + "fmla z12.h, z7.h, z1.h[7]\n" + "fmla z16.h, z7.h, z2.h[7]\n" + "fmla z20.h, z7.h, z3.h[7]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" - "fmla z24.h, z6.h, z4.h[7]\n" - "fmla z28.h, z6.h, z5.h[7]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" + "fmla z24.h, z7.h, z4.h[7]\n" + "fmla z28.h, z7.h, z5.h[7]\n" + "ld1h { z7.h }, p5/Z, [x10]\n" "addvl x10, x10, #1\n" - "fmla z9.h, z7.h, z0.h[7]\n" - "fmla z13.h, z7.h, z1.h[7]\n" - "fmla z17.h, z7.h, z2.h[7]\n" - "fmla z21.h, z7.h, z3.h[7]\n" - "fmla z25.h, z7.h, z4.h[7]\n" - "fmla z29.h, z7.h, z5.h[7]\n" - "ld1h { z7.h }, p5/Z, [x9]\n" + "fmla z9.h, z6.h, z0.h[7]\n" + "fmla z13.h, z6.h, z1.h[7]\n" + "fmla z17.h, z6.h, z2.h[7]\n" + "fmla z21.h, z6.h, z3.h[7]\n" + "fmla z25.h, z6.h, z4.h[7]\n" + "fmla z29.h, z6.h, z5.h[7]\n" + "ld1h { z6.h }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.h, z6.h, z0.h[7]\n" - "fmla z14.h, z6.h, z1.h[7]\n" - "fmla z18.h, z6.h, z2.h[7]\n" - "fmla z22.h, z6.h, z3.h[7]\n" - "fmla z26.h, z6.h, z4.h[7]\n" - "fmla z30.h, z6.h, z5.h[7]\n" - "fmla z11.h, z7.h, z0.h[7]\n" - "fmla z15.h, z7.h, z1.h[7]\n" - "fmla z19.h, z7.h, z2.h[7]\n" - "fmla z23.h, z7.h, z3.h[7]\n" - "fmla z27.h, z7.h, z4.h[7]\n" - "fmla z31.h, z7.h, z5.h[7]\n" + "fmla z10.h, z7.h, z0.h[7]\n" + "fmla z14.h, z7.h, z1.h[7]\n" + "fmla z18.h, z7.h, z2.h[7]\n" + "fmla z22.h, z7.h, z3.h[7]\n" + "fmla z26.h, z7.h, z4.h[7]\n" + "fmla z30.h, z7.h, z5.h[7]\n" + "fmla z11.h, z6.h, z0.h[7]\n" + "fmla z15.h, z6.h, z1.h[7]\n" + "fmla z19.h, z6.h, z2.h[7]\n" + "fmla z23.h, z6.h, z3.h[7]\n" + "fmla z27.h, z6.h, z4.h[7]\n" + "fmla z31.h, z6.h, z5.h[7]\n" "82:" // Height 6: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -3315,4 +3315,4 @@ void sve_ffhybrid_fp16_mla_6x4VL ( } } // namespace arm_gemm -#endif // ARM_COMPUTE_ENABLE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32_mla_6x4VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32_mla_6x4VL.hpp index b4c124c1e316646c9c71a61be830dd8e5a35f3ec..3a93a2f7c8fff8131a8510eebdc3a15f6e858d7d 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32_mla_6x4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32_mla_6x4VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,16 +10,16 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once #ifdef ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32_mla_6x4VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32_mla_6x4VL/a64fx.cpp index 32fcac3a456e227128b1cb5cdbf5c08d8cb67203..8e4fd4388e880110d09865905a4500cb92f3c887 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32_mla_6x4VL/a64fx.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32_mla_6x4VL/a64fx.cpp @@ -163,11 +163,11 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "7:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 8f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 9f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -183,12 +183,12 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "10:" // Height 1: Multiply loop: Main loop "fmla z8.s, p4/M, z6.s, z0.s\n" "fmla z9.s, p4/M, z7.s, z0.s\n" - "ld1w { z6.s }, p4/Z, [x10]\n" - "ld1w { z7.s }, p4/Z, [x9]\n" + "ld1w { z17.s }, p4/Z, [x10]\n" + "ld1w { z16.s }, p4/Z, [x9]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" - "fmla z10.s, p4/M, z6.s, z0.s\n" - "fmla z11.s, p4/M, z7.s, z0.s\n" + "fmla z10.s, p4/M, z17.s, z0.s\n" + "fmla z11.s, p4/M, z16.s, z0.s\n" "add x26, x26, #0x4\n" "subs x27, x27, #0x1\n" "ld1rw { z0.s }, p4/Z, [x26]\n" @@ -201,12 +201,12 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "fmla z8.s, p4/M, z6.s, z0.s\n" "fmla z9.s, p4/M, z7.s, z0.s\n" - "ld1w { z6.s }, p4/Z, [x10]\n" - "ld1w { z7.s }, p4/Z, [x9]\n" + "ld1w { z17.s }, p4/Z, [x10]\n" + "ld1w { z16.s }, p4/Z, [x9]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" - "fmla z10.s, p4/M, z6.s, z0.s\n" - "fmla z11.s, p4/M, z7.s, z0.s\n" + "fmla z10.s, p4/M, z17.s, z0.s\n" + "fmla z11.s, p4/M, z16.s, z0.s\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" @@ -214,17 +214,17 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "bne 7b\n" "tbz %x[flags], #1, 12f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p4/Z, [x20]\n" + "ld1rw { z17.s }, p4/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p4/Z, [x20]\n" - "fmin z8.s, p4/M, z8.s, z1.s\n" - "fmin z9.s, p4/M, z9.s, z1.s\n" - "fmin z10.s, p4/M, z10.s, z1.s\n" - "fmin z11.s, p4/M, z11.s, z1.s\n" - "fmax z8.s, p4/M, z8.s, z0.s\n" - "fmax z9.s, p4/M, z9.s, z0.s\n" - "fmax z10.s, p4/M, z10.s, z0.s\n" - "fmax z11.s, p4/M, z11.s, z0.s\n" + "ld1rw { z16.s }, p4/Z, [x20]\n" + "fmin z8.s, p4/M, z8.s, z17.s\n" + "fmin z9.s, p4/M, z9.s, z17.s\n" + "fmin z10.s, p4/M, z10.s, z17.s\n" + "fmin z11.s, p4/M, z11.s, z17.s\n" + "fmax z8.s, p4/M, z8.s, z16.s\n" + "fmax z9.s, p4/M, z9.s, z16.s\n" + "fmax z10.s, p4/M, z10.s, z16.s\n" + "fmax z11.s, p4/M, z11.s, z16.s\n" "12:" // Height 1: No activation "st1w { z8.s }, p3, [x13]\n" "st1w { z9.s }, p2, [x13, #1, MUL VL]\n" @@ -285,15 +285,15 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "17:" // Height 2: no bias "tbz %x[flags], #0, 18f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #2\n" + "add x20, x13, x20, LSL #2\n" "ld1w { z8.s }, p3/Z, [x13]\n" "ld1w { z9.s }, p2/Z, [x13, #1, MUL VL]\n" "ld1w { z10.s }, p1/Z, [x13, #2, MUL VL]\n" "ld1w { z11.s }, p0/Z, [x13, #3, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x25]\n" - "ld1w { z13.s }, p2/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p1/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p0/Z, [x25, #3, MUL VL]\n" + "ld1w { z12.s }, p3/Z, [x20]\n" + "ld1w { z13.s }, p2/Z, [x20, #1, MUL VL]\n" + "ld1w { z14.s }, p1/Z, [x20, #2, MUL VL]\n" + "ld1w { z15.s }, p0/Z, [x20, #3, MUL VL]\n" "b 19f\n" "18:" // Height 2: no accumulate "mov z8.b, #0x0\n" @@ -309,12 +309,12 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "20:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 21f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 22f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -322,7 +322,7 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "b 22f\n" "21:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" "22:" // Height 2: input setup done "subs x27, x27, #0x1\n" "ld1rw { z0.s }, p4/Z, [x26]\n" @@ -333,19 +333,19 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "23:" // Height 2: Multiply loop: Main loop "fmla z8.s, p4/M, z6.s, z0.s\n" "fmla z12.s, p4/M, z6.s, z1.s\n" - "ld1w { z6.s }, p4/Z, [x10]\n" + "ld1w { z17.s }, p4/Z, [x10]\n" "addvl x12, x12, #1\n" "fmla z9.s, p4/M, z7.s, z0.s\n" "fmla z13.s, p4/M, z7.s, z1.s\n" - "ld1w { z7.s }, p4/Z, [x9]\n" + "ld1w { z16.s }, p4/Z, [x9]\n" "addvl x11, x11, #1\n" "add x26, x26, #0x4\n" "subs x27, x27, #0x1\n" - "fmla z10.s, p4/M, z6.s, z0.s\n" - "fmla z14.s, p4/M, z6.s, z1.s\n" + "fmla z10.s, p4/M, z17.s, z0.s\n" + "fmla z14.s, p4/M, z17.s, z1.s\n" "add x25, x25, #0x4\n" - "fmla z11.s, p4/M, z7.s, z0.s\n" - "fmla z15.s, p4/M, z7.s, z1.s\n" + "fmla z11.s, p4/M, z16.s, z0.s\n" + "fmla z15.s, p4/M, z16.s, z1.s\n" "addvl x10, x10, #1\n" "ld1rw { z0.s }, p4/Z, [x26]\n" "ld1rw { z1.s }, p4/Z, [x25]\n" @@ -357,18 +357,18 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "fmla z8.s, p4/M, z6.s, z0.s\n" "fmla z12.s, p4/M, z6.s, z1.s\n" - "ld1w { z6.s }, p4/Z, [x10]\n" + "ld1w { z17.s }, p4/Z, [x10]\n" "fmla z9.s, p4/M, z7.s, z0.s\n" "fmla z13.s, p4/M, z7.s, z1.s\n" - "ld1w { z7.s }, p4/Z, [x9]\n" + "ld1w { z16.s }, p4/Z, [x9]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" - "fmla z10.s, p4/M, z6.s, z0.s\n" - "fmla z14.s, p4/M, z6.s, z1.s\n" + "fmla z10.s, p4/M, z17.s, z0.s\n" + "fmla z14.s, p4/M, z17.s, z1.s\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" - "fmla z11.s, p4/M, z7.s, z0.s\n" - "fmla z15.s, p4/M, z7.s, z1.s\n" + "fmla z11.s, p4/M, z16.s, z0.s\n" + "fmla z15.s, p4/M, z16.s, z1.s\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" "bne 20b\n" @@ -376,25 +376,25 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "add x25, x13, x20, LSL #2\n" "tbz %x[flags], #1, 25f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p4/Z, [x20]\n" + "ld1rw { z17.s }, p4/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p4/Z, [x20]\n" - "fmin z8.s, p4/M, z8.s, z1.s\n" - "fmin z9.s, p4/M, z9.s, z1.s\n" - "fmin z10.s, p4/M, z10.s, z1.s\n" - "fmin z11.s, p4/M, z11.s, z1.s\n" - "fmin z12.s, p4/M, z12.s, z1.s\n" - "fmin z13.s, p4/M, z13.s, z1.s\n" - "fmin z14.s, p4/M, z14.s, z1.s\n" - "fmin z15.s, p4/M, z15.s, z1.s\n" - "fmax z8.s, p4/M, z8.s, z0.s\n" - "fmax z9.s, p4/M, z9.s, z0.s\n" - "fmax z10.s, p4/M, z10.s, z0.s\n" - "fmax z11.s, p4/M, z11.s, z0.s\n" - "fmax z12.s, p4/M, z12.s, z0.s\n" - "fmax z13.s, p4/M, z13.s, z0.s\n" - "fmax z14.s, p4/M, z14.s, z0.s\n" - "fmax z15.s, p4/M, z15.s, z0.s\n" + "ld1rw { z16.s }, p4/Z, [x20]\n" + "fmin z8.s, p4/M, z8.s, z17.s\n" + "fmin z9.s, p4/M, z9.s, z17.s\n" + "fmin z10.s, p4/M, z10.s, z17.s\n" + "fmin z11.s, p4/M, z11.s, z17.s\n" + "fmin z12.s, p4/M, z12.s, z17.s\n" + "fmin z13.s, p4/M, z13.s, z17.s\n" + "fmin z14.s, p4/M, z14.s, z17.s\n" + "fmin z15.s, p4/M, z15.s, z17.s\n" + "fmax z8.s, p4/M, z8.s, z16.s\n" + "fmax z9.s, p4/M, z9.s, z16.s\n" + "fmax z10.s, p4/M, z10.s, z16.s\n" + "fmax z11.s, p4/M, z11.s, z16.s\n" + "fmax z12.s, p4/M, z12.s, z16.s\n" + "fmax z13.s, p4/M, z13.s, z16.s\n" + "fmax z14.s, p4/M, z14.s, z16.s\n" + "fmax z15.s, p4/M, z15.s, z16.s\n" "25:" // Height 2: No activation "st1w { z8.s }, p3, [x13]\n" "st1w { z9.s }, p2, [x13, #1, MUL VL]\n" @@ -463,20 +463,20 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "30:" // Height 3: no bias "tbz %x[flags], #0, 31f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" + "add x21, x13, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z8.s }, p3/Z, [x13]\n" "ld1w { z9.s }, p2/Z, [x13, #1, MUL VL]\n" "ld1w { z10.s }, p1/Z, [x13, #2, MUL VL]\n" "ld1w { z11.s }, p0/Z, [x13, #3, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x25]\n" - "ld1w { z13.s }, p2/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p1/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p0/Z, [x25, #3, MUL VL]\n" - "ld1w { z16.s }, p3/Z, [x24]\n" - "ld1w { z17.s }, p2/Z, [x24, #1, MUL VL]\n" - "ld1w { z18.s }, p1/Z, [x24, #2, MUL VL]\n" - "ld1w { z19.s }, p0/Z, [x24, #3, MUL VL]\n" + "ld1w { z12.s }, p3/Z, [x21]\n" + "ld1w { z13.s }, p2/Z, [x21, #1, MUL VL]\n" + "ld1w { z14.s }, p1/Z, [x21, #2, MUL VL]\n" + "ld1w { z15.s }, p0/Z, [x21, #3, MUL VL]\n" + "ld1w { z16.s }, p3/Z, [x20]\n" + "ld1w { z17.s }, p2/Z, [x20, #1, MUL VL]\n" + "ld1w { z18.s }, p1/Z, [x20, #2, MUL VL]\n" + "ld1w { z19.s }, p0/Z, [x20, #3, MUL VL]\n" "b 32f\n" "31:" // Height 3: no accumulate "mov z8.b, #0x0\n" @@ -496,13 +496,13 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "33:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 34f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 35f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -511,8 +511,8 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "b 35f\n" "34:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" "35:" // Height 3: input setup done "subs x27, x27, #0x1\n" "ld1rw { z0.s }, p4/Z, [x26]\n" @@ -528,22 +528,22 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "addvl x11, x11, #1\n" "fmla z16.s, p4/M, z6.s, z2.s\n" "fmla z9.s, p4/M, z7.s, z0.s\n" - "ld1w { z6.s }, p4/Z, [x10]\n" + "ld1w { z21.s }, p4/Z, [x10]\n" "add x26, x26, #0x4\n" "fmla z13.s, p4/M, z7.s, z1.s\n" "fmla z17.s, p4/M, z7.s, z2.s\n" - "ld1w { z7.s }, p4/Z, [x9]\n" + "ld1w { z20.s }, p4/Z, [x9]\n" "subs x27, x27, #0x1\n" "add x25, x25, #0x4\n" "add x24, x24, #0x4\n" - "fmla z10.s, p4/M, z6.s, z0.s\n" - "fmla z14.s, p4/M, z6.s, z1.s\n" - "fmla z18.s, p4/M, z6.s, z2.s\n" - "fmla z11.s, p4/M, z7.s, z0.s\n" + "fmla z10.s, p4/M, z21.s, z0.s\n" + "fmla z14.s, p4/M, z21.s, z1.s\n" + "fmla z18.s, p4/M, z21.s, z2.s\n" + "fmla z11.s, p4/M, z20.s, z0.s\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" - "fmla z15.s, p4/M, z7.s, z1.s\n" - "fmla z19.s, p4/M, z7.s, z2.s\n" + "fmla z15.s, p4/M, z20.s, z1.s\n" + "fmla z19.s, p4/M, z20.s, z2.s\n" "ld1rw { z0.s }, p4/Z, [x26]\n" "ld1rw { z1.s }, p4/Z, [x25]\n" "ld1rw { z2.s }, p4/Z, [x24]\n" @@ -557,54 +557,54 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "add x28, x28, #0x1\n" "fmla z16.s, p4/M, z6.s, z2.s\n" "fmla z9.s, p4/M, z7.s, z0.s\n" - "ld1w { z6.s }, p4/Z, [x10]\n" + "ld1w { z21.s }, p4/Z, [x10]\n" "cmp x28, x20\n" "fmla z13.s, p4/M, z7.s, z1.s\n" "fmla z17.s, p4/M, z7.s, z2.s\n" - "ld1w { z7.s }, p4/Z, [x9]\n" + "ld1w { z20.s }, p4/Z, [x9]\n" "addvl x12, x12, #1\n" - "fmla z10.s, p4/M, z6.s, z0.s\n" - "fmla z14.s, p4/M, z6.s, z1.s\n" + "fmla z10.s, p4/M, z21.s, z0.s\n" + "fmla z14.s, p4/M, z21.s, z1.s\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" - "fmla z18.s, p4/M, z6.s, z2.s\n" - "fmla z11.s, p4/M, z7.s, z0.s\n" + "fmla z18.s, p4/M, z21.s, z2.s\n" + "fmla z11.s, p4/M, z20.s, z0.s\n" "addvl x9, x9, #1\n" - "fmla z15.s, p4/M, z7.s, z1.s\n" - "fmla z19.s, p4/M, z7.s, z2.s\n" + "fmla z15.s, p4/M, z20.s, z1.s\n" + "fmla z19.s, p4/M, z20.s, z2.s\n" "bne 33b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "add x25, x13, x20, LSL #2\n" "add x24, x25, x20, LSL #2\n" "tbz %x[flags], #1, 38f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p4/Z, [x20]\n" + "ld1rw { z21.s }, p4/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p4/Z, [x20]\n" - "fmin z8.s, p4/M, z8.s, z1.s\n" - "fmin z9.s, p4/M, z9.s, z1.s\n" - "fmin z10.s, p4/M, z10.s, z1.s\n" - "fmin z11.s, p4/M, z11.s, z1.s\n" - "fmin z12.s, p4/M, z12.s, z1.s\n" - "fmin z13.s, p4/M, z13.s, z1.s\n" - "fmin z14.s, p4/M, z14.s, z1.s\n" - "fmin z15.s, p4/M, z15.s, z1.s\n" - "fmin z16.s, p4/M, z16.s, z1.s\n" - "fmin z17.s, p4/M, z17.s, z1.s\n" - "fmin z18.s, p4/M, z18.s, z1.s\n" - "fmin z19.s, p4/M, z19.s, z1.s\n" - "fmax z8.s, p4/M, z8.s, z0.s\n" - "fmax z9.s, p4/M, z9.s, z0.s\n" - "fmax z10.s, p4/M, z10.s, z0.s\n" - "fmax z11.s, p4/M, z11.s, z0.s\n" - "fmax z12.s, p4/M, z12.s, z0.s\n" - "fmax z13.s, p4/M, z13.s, z0.s\n" - "fmax z14.s, p4/M, z14.s, z0.s\n" - "fmax z15.s, p4/M, z15.s, z0.s\n" - "fmax z16.s, p4/M, z16.s, z0.s\n" - "fmax z17.s, p4/M, z17.s, z0.s\n" - "fmax z18.s, p4/M, z18.s, z0.s\n" - "fmax z19.s, p4/M, z19.s, z0.s\n" + "ld1rw { z20.s }, p4/Z, [x20]\n" + "fmin z8.s, p4/M, z8.s, z21.s\n" + "fmin z9.s, p4/M, z9.s, z21.s\n" + "fmin z10.s, p4/M, z10.s, z21.s\n" + "fmin z11.s, p4/M, z11.s, z21.s\n" + "fmin z12.s, p4/M, z12.s, z21.s\n" + "fmin z13.s, p4/M, z13.s, z21.s\n" + "fmin z14.s, p4/M, z14.s, z21.s\n" + "fmin z15.s, p4/M, z15.s, z21.s\n" + "fmin z16.s, p4/M, z16.s, z21.s\n" + "fmin z17.s, p4/M, z17.s, z21.s\n" + "fmin z18.s, p4/M, z18.s, z21.s\n" + "fmin z19.s, p4/M, z19.s, z21.s\n" + "fmax z8.s, p4/M, z8.s, z20.s\n" + "fmax z9.s, p4/M, z9.s, z20.s\n" + "fmax z10.s, p4/M, z10.s, z20.s\n" + "fmax z11.s, p4/M, z11.s, z20.s\n" + "fmax z12.s, p4/M, z12.s, z20.s\n" + "fmax z13.s, p4/M, z13.s, z20.s\n" + "fmax z14.s, p4/M, z14.s, z20.s\n" + "fmax z15.s, p4/M, z15.s, z20.s\n" + "fmax z16.s, p4/M, z16.s, z20.s\n" + "fmax z17.s, p4/M, z17.s, z20.s\n" + "fmax z18.s, p4/M, z18.s, z20.s\n" + "fmax z19.s, p4/M, z19.s, z20.s\n" "38:" // Height 3: No activation "st1w { z8.s }, p3, [x13]\n" "st1w { z9.s }, p2, [x13, #1, MUL VL]\n" @@ -681,25 +681,25 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "43:" // Height 4: no bias "tbz %x[flags], #0, 44f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" + "add x22, x13, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" "ld1w { z8.s }, p3/Z, [x13]\n" - "add x23, x24, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z9.s }, p2/Z, [x13, #1, MUL VL]\n" "ld1w { z10.s }, p1/Z, [x13, #2, MUL VL]\n" "ld1w { z11.s }, p0/Z, [x13, #3, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x25]\n" - "ld1w { z13.s }, p2/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p1/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p0/Z, [x25, #3, MUL VL]\n" - "ld1w { z16.s }, p3/Z, [x24]\n" - "ld1w { z17.s }, p2/Z, [x24, #1, MUL VL]\n" - "ld1w { z18.s }, p1/Z, [x24, #2, MUL VL]\n" - "ld1w { z19.s }, p0/Z, [x24, #3, MUL VL]\n" - "ld1w { z20.s }, p3/Z, [x23]\n" - "ld1w { z21.s }, p2/Z, [x23, #1, MUL VL]\n" - "ld1w { z22.s }, p1/Z, [x23, #2, MUL VL]\n" - "ld1w { z23.s }, p0/Z, [x23, #3, MUL VL]\n" + "ld1w { z12.s }, p3/Z, [x22]\n" + "ld1w { z13.s }, p2/Z, [x22, #1, MUL VL]\n" + "ld1w { z14.s }, p1/Z, [x22, #2, MUL VL]\n" + "ld1w { z15.s }, p0/Z, [x22, #3, MUL VL]\n" + "ld1w { z16.s }, p3/Z, [x21]\n" + "ld1w { z17.s }, p2/Z, [x21, #1, MUL VL]\n" + "ld1w { z18.s }, p1/Z, [x21, #2, MUL VL]\n" + "ld1w { z19.s }, p0/Z, [x21, #3, MUL VL]\n" + "ld1w { z20.s }, p3/Z, [x20]\n" + "ld1w { z21.s }, p2/Z, [x20, #1, MUL VL]\n" + "ld1w { z22.s }, p1/Z, [x20, #2, MUL VL]\n" + "ld1w { z23.s }, p0/Z, [x20, #3, MUL VL]\n" "b 45f\n" "44:" // Height 4: no accumulate "mov z8.b, #0x0\n" @@ -723,14 +723,14 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "46:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 47f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 48f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -740,9 +740,9 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "b 48f\n" "47:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" "48:" // Height 4: input setup done "subs x27, x27, #0x1\n" "ld1rw { z0.s }, p4/Z, [x26]\n" @@ -759,7 +759,7 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "addvl x11, x11, #1\n" "fmla z16.s, p4/M, z6.s, z2.s\n" "fmla z20.s, p4/M, z6.s, z3.s\n" - "ld1w { z6.s }, p4/Z, [x10]\n" + "ld1w { z25.s }, p4/Z, [x10]\n" "add x26, x26, #0x4\n" "fmla z9.s, p4/M, z7.s, z0.s\n" "fmla z13.s, p4/M, z7.s, z1.s\n" @@ -767,22 +767,22 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "add x25, x25, #0x4\n" "fmla z17.s, p4/M, z7.s, z2.s\n" "fmla z21.s, p4/M, z7.s, z3.s\n" - "ld1w { z7.s }, p4/Z, [x9]\n" + "ld1w { z24.s }, p4/Z, [x9]\n" "add x24, x24, #0x4\n" "add x23, x23, #0x4\n" - "fmla z10.s, p4/M, z6.s, z0.s\n" - "fmla z14.s, p4/M, z6.s, z1.s\n" + "fmla z10.s, p4/M, z25.s, z0.s\n" + "fmla z14.s, p4/M, z25.s, z1.s\n" "addvl x10, x10, #1\n" - "fmla z18.s, p4/M, z6.s, z2.s\n" - "fmla z22.s, p4/M, z6.s, z3.s\n" + "fmla z18.s, p4/M, z25.s, z2.s\n" + "fmla z22.s, p4/M, z25.s, z3.s\n" "addvl x9, x9, #1\n" "ld1w { z6.s }, p4/Z, [x12]\n" - "fmla z11.s, p4/M, z7.s, z0.s\n" - "fmla z15.s, p4/M, z7.s, z1.s\n" + "fmla z11.s, p4/M, z24.s, z0.s\n" + "fmla z15.s, p4/M, z24.s, z1.s\n" "ld1rw { z0.s }, p4/Z, [x26]\n" "ld1rw { z1.s }, p4/Z, [x25]\n" - "fmla z19.s, p4/M, z7.s, z2.s\n" - "fmla z23.s, p4/M, z7.s, z3.s\n" + "fmla z19.s, p4/M, z24.s, z2.s\n" + "fmla z23.s, p4/M, z24.s, z3.s\n" "ld1rw { z2.s }, p4/Z, [x24]\n" "ld1rw { z3.s }, p4/Z, [x23]\n" "ld1w { z7.s }, p4/Z, [x11]\n" @@ -794,7 +794,7 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "add x28, x28, #0x1\n" "fmla z16.s, p4/M, z6.s, z2.s\n" "fmla z20.s, p4/M, z6.s, z3.s\n" - "ld1w { z6.s }, p4/Z, [x10]\n" + "ld1w { z25.s }, p4/Z, [x10]\n" "cmp x28, x20\n" "fmla z9.s, p4/M, z7.s, z0.s\n" "fmla z13.s, p4/M, z7.s, z1.s\n" @@ -802,17 +802,17 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "addvl x11, x11, #1\n" "fmla z17.s, p4/M, z7.s, z2.s\n" "fmla z21.s, p4/M, z7.s, z3.s\n" - "ld1w { z7.s }, p4/Z, [x9]\n" + "ld1w { z24.s }, p4/Z, [x9]\n" "addvl x10, x10, #1\n" - "fmla z10.s, p4/M, z6.s, z0.s\n" - "fmla z14.s, p4/M, z6.s, z1.s\n" + "fmla z10.s, p4/M, z25.s, z0.s\n" + "fmla z14.s, p4/M, z25.s, z1.s\n" "addvl x9, x9, #1\n" - "fmla z18.s, p4/M, z6.s, z2.s\n" - "fmla z22.s, p4/M, z6.s, z3.s\n" - "fmla z11.s, p4/M, z7.s, z0.s\n" - "fmla z15.s, p4/M, z7.s, z1.s\n" - "fmla z19.s, p4/M, z7.s, z2.s\n" - "fmla z23.s, p4/M, z7.s, z3.s\n" + "fmla z18.s, p4/M, z25.s, z2.s\n" + "fmla z22.s, p4/M, z25.s, z3.s\n" + "fmla z11.s, p4/M, z24.s, z0.s\n" + "fmla z15.s, p4/M, z24.s, z1.s\n" + "fmla z19.s, p4/M, z24.s, z2.s\n" + "fmla z23.s, p4/M, z24.s, z3.s\n" "bne 46b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "add x25, x13, x20, LSL #2\n" @@ -820,41 +820,41 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "add x23, x24, x20, LSL #2\n" "tbz %x[flags], #1, 51f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p4/Z, [x20]\n" + "ld1rw { z25.s }, p4/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p4/Z, [x20]\n" - "fmin z8.s, p4/M, z8.s, z1.s\n" - "fmin z9.s, p4/M, z9.s, z1.s\n" - "fmin z10.s, p4/M, z10.s, z1.s\n" - "fmin z11.s, p4/M, z11.s, z1.s\n" - "fmin z12.s, p4/M, z12.s, z1.s\n" - "fmin z13.s, p4/M, z13.s, z1.s\n" - "fmin z14.s, p4/M, z14.s, z1.s\n" - "fmin z15.s, p4/M, z15.s, z1.s\n" - "fmin z16.s, p4/M, z16.s, z1.s\n" - "fmin z17.s, p4/M, z17.s, z1.s\n" - "fmin z18.s, p4/M, z18.s, z1.s\n" - "fmin z19.s, p4/M, z19.s, z1.s\n" - "fmin z20.s, p4/M, z20.s, z1.s\n" - "fmin z21.s, p4/M, z21.s, z1.s\n" - "fmin z22.s, p4/M, z22.s, z1.s\n" - "fmin z23.s, p4/M, z23.s, z1.s\n" - "fmax z8.s, p4/M, z8.s, z0.s\n" - "fmax z9.s, p4/M, z9.s, z0.s\n" - "fmax z10.s, p4/M, z10.s, z0.s\n" - "fmax z11.s, p4/M, z11.s, z0.s\n" - "fmax z12.s, p4/M, z12.s, z0.s\n" - "fmax z13.s, p4/M, z13.s, z0.s\n" - "fmax z14.s, p4/M, z14.s, z0.s\n" - "fmax z15.s, p4/M, z15.s, z0.s\n" - "fmax z16.s, p4/M, z16.s, z0.s\n" - "fmax z17.s, p4/M, z17.s, z0.s\n" - "fmax z18.s, p4/M, z18.s, z0.s\n" - "fmax z19.s, p4/M, z19.s, z0.s\n" - "fmax z20.s, p4/M, z20.s, z0.s\n" - "fmax z21.s, p4/M, z21.s, z0.s\n" - "fmax z22.s, p4/M, z22.s, z0.s\n" - "fmax z23.s, p4/M, z23.s, z0.s\n" + "ld1rw { z24.s }, p4/Z, [x20]\n" + "fmin z8.s, p4/M, z8.s, z25.s\n" + "fmin z9.s, p4/M, z9.s, z25.s\n" + "fmin z10.s, p4/M, z10.s, z25.s\n" + "fmin z11.s, p4/M, z11.s, z25.s\n" + "fmin z12.s, p4/M, z12.s, z25.s\n" + "fmin z13.s, p4/M, z13.s, z25.s\n" + "fmin z14.s, p4/M, z14.s, z25.s\n" + "fmin z15.s, p4/M, z15.s, z25.s\n" + "fmin z16.s, p4/M, z16.s, z25.s\n" + "fmin z17.s, p4/M, z17.s, z25.s\n" + "fmin z18.s, p4/M, z18.s, z25.s\n" + "fmin z19.s, p4/M, z19.s, z25.s\n" + "fmin z20.s, p4/M, z20.s, z25.s\n" + "fmin z21.s, p4/M, z21.s, z25.s\n" + "fmin z22.s, p4/M, z22.s, z25.s\n" + "fmin z23.s, p4/M, z23.s, z25.s\n" + "fmax z8.s, p4/M, z8.s, z24.s\n" + "fmax z9.s, p4/M, z9.s, z24.s\n" + "fmax z10.s, p4/M, z10.s, z24.s\n" + "fmax z11.s, p4/M, z11.s, z24.s\n" + "fmax z12.s, p4/M, z12.s, z24.s\n" + "fmax z13.s, p4/M, z13.s, z24.s\n" + "fmax z14.s, p4/M, z14.s, z24.s\n" + "fmax z15.s, p4/M, z15.s, z24.s\n" + "fmax z16.s, p4/M, z16.s, z24.s\n" + "fmax z17.s, p4/M, z17.s, z24.s\n" + "fmax z18.s, p4/M, z18.s, z24.s\n" + "fmax z19.s, p4/M, z19.s, z24.s\n" + "fmax z20.s, p4/M, z20.s, z24.s\n" + "fmax z21.s, p4/M, z21.s, z24.s\n" + "fmax z22.s, p4/M, z22.s, z24.s\n" + "fmax z23.s, p4/M, z23.s, z24.s\n" "51:" // Height 4: No activation "st1w { z8.s }, p3, [x13]\n" "st1w { z9.s }, p2, [x13, #1, MUL VL]\n" @@ -939,30 +939,30 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "56:" // Height 5: no bias "tbz %x[flags], #0, 57f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z8.s }, p3/Z, [x13]\n" - "add x23, x24, x20, LSL #2\n" + "add x23, x13, x20, LSL #2\n" "add x22, x23, x20, LSL #2\n" + "ld1w { z8.s }, p3/Z, [x13]\n" + "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z9.s }, p2/Z, [x13, #1, MUL VL]\n" "ld1w { z10.s }, p1/Z, [x13, #2, MUL VL]\n" "ld1w { z11.s }, p0/Z, [x13, #3, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x25]\n" - "ld1w { z13.s }, p2/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p1/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p0/Z, [x25, #3, MUL VL]\n" - "ld1w { z16.s }, p3/Z, [x24]\n" - "ld1w { z17.s }, p2/Z, [x24, #1, MUL VL]\n" - "ld1w { z18.s }, p1/Z, [x24, #2, MUL VL]\n" - "ld1w { z19.s }, p0/Z, [x24, #3, MUL VL]\n" - "ld1w { z20.s }, p3/Z, [x23]\n" - "ld1w { z21.s }, p2/Z, [x23, #1, MUL VL]\n" - "ld1w { z22.s }, p1/Z, [x23, #2, MUL VL]\n" - "ld1w { z23.s }, p0/Z, [x23, #3, MUL VL]\n" - "ld1w { z24.s }, p3/Z, [x22]\n" - "ld1w { z25.s }, p2/Z, [x22, #1, MUL VL]\n" - "ld1w { z26.s }, p1/Z, [x22, #2, MUL VL]\n" - "ld1w { z27.s }, p0/Z, [x22, #3, MUL VL]\n" + "ld1w { z12.s }, p3/Z, [x23]\n" + "ld1w { z13.s }, p2/Z, [x23, #1, MUL VL]\n" + "ld1w { z14.s }, p1/Z, [x23, #2, MUL VL]\n" + "ld1w { z15.s }, p0/Z, [x23, #3, MUL VL]\n" + "ld1w { z16.s }, p3/Z, [x22]\n" + "ld1w { z17.s }, p2/Z, [x22, #1, MUL VL]\n" + "ld1w { z18.s }, p1/Z, [x22, #2, MUL VL]\n" + "ld1w { z19.s }, p0/Z, [x22, #3, MUL VL]\n" + "ld1w { z20.s }, p3/Z, [x21]\n" + "ld1w { z21.s }, p2/Z, [x21, #1, MUL VL]\n" + "ld1w { z22.s }, p1/Z, [x21, #2, MUL VL]\n" + "ld1w { z23.s }, p0/Z, [x21, #3, MUL VL]\n" + "ld1w { z24.s }, p3/Z, [x20]\n" + "ld1w { z25.s }, p2/Z, [x20, #1, MUL VL]\n" + "ld1w { z26.s }, p1/Z, [x20, #2, MUL VL]\n" + "ld1w { z27.s }, p0/Z, [x20, #3, MUL VL]\n" "b 58f\n" "57:" // Height 5: no accumulate "mov z8.b, #0x0\n" @@ -990,15 +990,15 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "59:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 60f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 61f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -1009,10 +1009,10 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "b 61f\n" "60:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" "61:" // Height 5: input setup done "subs x27, x27, #0x1\n" "ld1rw { z0.s }, p4/Z, [x26]\n" @@ -1034,7 +1034,7 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "subs x27, x27, #0x1\n" "fmla z24.s, p4/M, z6.s, z4.s\n" "fmla z9.s, p4/M, z7.s, z0.s\n" - "ld1w { z6.s }, p4/Z, [x10]\n" + "ld1w { z29.s }, p4/Z, [x10]\n" "add x25, x25, #0x4\n" "fmla z13.s, p4/M, z7.s, z1.s\n" "fmla z17.s, p4/M, z7.s, z2.s\n" @@ -1042,24 +1042,24 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "add x23, x23, #0x4\n" "fmla z21.s, p4/M, z7.s, z3.s\n" "fmla z25.s, p4/M, z7.s, z4.s\n" - "ld1w { z7.s }, p4/Z, [x9]\n" + "ld1w { z28.s }, p4/Z, [x9]\n" "add x22, x22, #0x4\n" - "fmla z10.s, p4/M, z6.s, z0.s\n" - "fmla z14.s, p4/M, z6.s, z1.s\n" + "fmla z10.s, p4/M, z29.s, z0.s\n" + "fmla z14.s, p4/M, z29.s, z1.s\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" - "fmla z18.s, p4/M, z6.s, z2.s\n" - "fmla z22.s, p4/M, z6.s, z3.s\n" - "fmla z26.s, p4/M, z6.s, z4.s\n" - "fmla z11.s, p4/M, z7.s, z0.s\n" + "fmla z18.s, p4/M, z29.s, z2.s\n" + "fmla z22.s, p4/M, z29.s, z3.s\n" + "fmla z26.s, p4/M, z29.s, z4.s\n" + "fmla z11.s, p4/M, z28.s, z0.s\n" "ld1rw { z0.s }, p4/Z, [x26]\n" "ld1w { z6.s }, p4/Z, [x12]\n" - "fmla z15.s, p4/M, z7.s, z1.s\n" - "fmla z19.s, p4/M, z7.s, z2.s\n" + "fmla z15.s, p4/M, z28.s, z1.s\n" + "fmla z19.s, p4/M, z28.s, z2.s\n" "ld1rw { z1.s }, p4/Z, [x25]\n" "ld1rw { z2.s }, p4/Z, [x24]\n" - "fmla z23.s, p4/M, z7.s, z3.s\n" - "fmla z27.s, p4/M, z7.s, z4.s\n" + "fmla z23.s, p4/M, z28.s, z3.s\n" + "fmla z27.s, p4/M, z28.s, z4.s\n" "ld1rw { z3.s }, p4/Z, [x23]\n" "ld1rw { z4.s }, p4/Z, [x22]\n" "ld1w { z7.s }, p4/Z, [x11]\n" @@ -1075,25 +1075,25 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "addvl x12, x12, #1\n" "fmla z24.s, p4/M, z6.s, z4.s\n" "fmla z9.s, p4/M, z7.s, z0.s\n" - "ld1w { z6.s }, p4/Z, [x10]\n" + "ld1w { z29.s }, p4/Z, [x10]\n" "addvl x11, x11, #1\n" "fmla z13.s, p4/M, z7.s, z1.s\n" "fmla z17.s, p4/M, z7.s, z2.s\n" "addvl x10, x10, #1\n" "fmla z21.s, p4/M, z7.s, z3.s\n" "fmla z25.s, p4/M, z7.s, z4.s\n" - "ld1w { z7.s }, p4/Z, [x9]\n" + "ld1w { z28.s }, p4/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.s, p4/M, z6.s, z0.s\n" - "fmla z14.s, p4/M, z6.s, z1.s\n" - "fmla z18.s, p4/M, z6.s, z2.s\n" - "fmla z22.s, p4/M, z6.s, z3.s\n" - "fmla z26.s, p4/M, z6.s, z4.s\n" - "fmla z11.s, p4/M, z7.s, z0.s\n" - "fmla z15.s, p4/M, z7.s, z1.s\n" - "fmla z19.s, p4/M, z7.s, z2.s\n" - "fmla z23.s, p4/M, z7.s, z3.s\n" - "fmla z27.s, p4/M, z7.s, z4.s\n" + "fmla z10.s, p4/M, z29.s, z0.s\n" + "fmla z14.s, p4/M, z29.s, z1.s\n" + "fmla z18.s, p4/M, z29.s, z2.s\n" + "fmla z22.s, p4/M, z29.s, z3.s\n" + "fmla z26.s, p4/M, z29.s, z4.s\n" + "fmla z11.s, p4/M, z28.s, z0.s\n" + "fmla z15.s, p4/M, z28.s, z1.s\n" + "fmla z19.s, p4/M, z28.s, z2.s\n" + "fmla z23.s, p4/M, z28.s, z3.s\n" + "fmla z27.s, p4/M, z28.s, z4.s\n" "bne 59b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "add x25, x13, x20, LSL #2\n" @@ -1102,49 +1102,49 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "add x22, x23, x20, LSL #2\n" "tbz %x[flags], #1, 64f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p4/Z, [x20]\n" + "ld1rw { z29.s }, p4/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p4/Z, [x20]\n" - "fmin z8.s, p4/M, z8.s, z1.s\n" - "fmin z9.s, p4/M, z9.s, z1.s\n" - "fmin z10.s, p4/M, z10.s, z1.s\n" - "fmin z11.s, p4/M, z11.s, z1.s\n" - "fmin z12.s, p4/M, z12.s, z1.s\n" - "fmin z13.s, p4/M, z13.s, z1.s\n" - "fmin z14.s, p4/M, z14.s, z1.s\n" - "fmin z15.s, p4/M, z15.s, z1.s\n" - "fmin z16.s, p4/M, z16.s, z1.s\n" - "fmin z17.s, p4/M, z17.s, z1.s\n" - "fmin z18.s, p4/M, z18.s, z1.s\n" - "fmin z19.s, p4/M, z19.s, z1.s\n" - "fmin z20.s, p4/M, z20.s, z1.s\n" - "fmin z21.s, p4/M, z21.s, z1.s\n" - "fmin z22.s, p4/M, z22.s, z1.s\n" - "fmin z23.s, p4/M, z23.s, z1.s\n" - "fmin z24.s, p4/M, z24.s, z1.s\n" - "fmin z25.s, p4/M, z25.s, z1.s\n" - "fmin z26.s, p4/M, z26.s, z1.s\n" - "fmin z27.s, p4/M, z27.s, z1.s\n" - "fmax z8.s, p4/M, z8.s, z0.s\n" - "fmax z9.s, p4/M, z9.s, z0.s\n" - "fmax z10.s, p4/M, z10.s, z0.s\n" - "fmax z11.s, p4/M, z11.s, z0.s\n" - "fmax z12.s, p4/M, z12.s, z0.s\n" - "fmax z13.s, p4/M, z13.s, z0.s\n" - "fmax z14.s, p4/M, z14.s, z0.s\n" - "fmax z15.s, p4/M, z15.s, z0.s\n" - "fmax z16.s, p4/M, z16.s, z0.s\n" - "fmax z17.s, p4/M, z17.s, z0.s\n" - "fmax z18.s, p4/M, z18.s, z0.s\n" - "fmax z19.s, p4/M, z19.s, z0.s\n" - "fmax z20.s, p4/M, z20.s, z0.s\n" - "fmax z21.s, p4/M, z21.s, z0.s\n" - "fmax z22.s, p4/M, z22.s, z0.s\n" - "fmax z23.s, p4/M, z23.s, z0.s\n" - "fmax z24.s, p4/M, z24.s, z0.s\n" - "fmax z25.s, p4/M, z25.s, z0.s\n" - "fmax z26.s, p4/M, z26.s, z0.s\n" - "fmax z27.s, p4/M, z27.s, z0.s\n" + "ld1rw { z28.s }, p4/Z, [x20]\n" + "fmin z8.s, p4/M, z8.s, z29.s\n" + "fmin z9.s, p4/M, z9.s, z29.s\n" + "fmin z10.s, p4/M, z10.s, z29.s\n" + "fmin z11.s, p4/M, z11.s, z29.s\n" + "fmin z12.s, p4/M, z12.s, z29.s\n" + "fmin z13.s, p4/M, z13.s, z29.s\n" + "fmin z14.s, p4/M, z14.s, z29.s\n" + "fmin z15.s, p4/M, z15.s, z29.s\n" + "fmin z16.s, p4/M, z16.s, z29.s\n" + "fmin z17.s, p4/M, z17.s, z29.s\n" + "fmin z18.s, p4/M, z18.s, z29.s\n" + "fmin z19.s, p4/M, z19.s, z29.s\n" + "fmin z20.s, p4/M, z20.s, z29.s\n" + "fmin z21.s, p4/M, z21.s, z29.s\n" + "fmin z22.s, p4/M, z22.s, z29.s\n" + "fmin z23.s, p4/M, z23.s, z29.s\n" + "fmin z24.s, p4/M, z24.s, z29.s\n" + "fmin z25.s, p4/M, z25.s, z29.s\n" + "fmin z26.s, p4/M, z26.s, z29.s\n" + "fmin z27.s, p4/M, z27.s, z29.s\n" + "fmax z8.s, p4/M, z8.s, z28.s\n" + "fmax z9.s, p4/M, z9.s, z28.s\n" + "fmax z10.s, p4/M, z10.s, z28.s\n" + "fmax z11.s, p4/M, z11.s, z28.s\n" + "fmax z12.s, p4/M, z12.s, z28.s\n" + "fmax z13.s, p4/M, z13.s, z28.s\n" + "fmax z14.s, p4/M, z14.s, z28.s\n" + "fmax z15.s, p4/M, z15.s, z28.s\n" + "fmax z16.s, p4/M, z16.s, z28.s\n" + "fmax z17.s, p4/M, z17.s, z28.s\n" + "fmax z18.s, p4/M, z18.s, z28.s\n" + "fmax z19.s, p4/M, z19.s, z28.s\n" + "fmax z20.s, p4/M, z20.s, z28.s\n" + "fmax z21.s, p4/M, z21.s, z28.s\n" + "fmax z22.s, p4/M, z22.s, z28.s\n" + "fmax z23.s, p4/M, z23.s, z28.s\n" + "fmax z24.s, p4/M, z24.s, z28.s\n" + "fmax z25.s, p4/M, z25.s, z28.s\n" + "fmax z26.s, p4/M, z26.s, z28.s\n" + "fmax z27.s, p4/M, z27.s, z28.s\n" "64:" // Height 5: No activation "st1w { z8.s }, p3, [x13]\n" "st1w { z9.s }, p2, [x13, #1, MUL VL]\n" @@ -1240,35 +1240,35 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "69:" // Height 6: no bias "tbz %x[flags], #0, 70f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z8.s }, p3/Z, [x13]\n" + "add x24, x13, x20, LSL #2\n" "add x23, x24, x20, LSL #2\n" + "ld1w { z8.s }, p3/Z, [x13]\n" "add x22, x23, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" "ld1w { z9.s }, p2/Z, [x13, #1, MUL VL]\n" "ld1w { z10.s }, p1/Z, [x13, #2, MUL VL]\n" - "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z11.s }, p0/Z, [x13, #3, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x25]\n" - "ld1w { z13.s }, p2/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p1/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p0/Z, [x25, #3, MUL VL]\n" - "ld1w { z16.s }, p3/Z, [x24]\n" - "ld1w { z17.s }, p2/Z, [x24, #1, MUL VL]\n" - "ld1w { z18.s }, p1/Z, [x24, #2, MUL VL]\n" - "ld1w { z19.s }, p0/Z, [x24, #3, MUL VL]\n" - "ld1w { z20.s }, p3/Z, [x23]\n" - "ld1w { z21.s }, p2/Z, [x23, #1, MUL VL]\n" - "ld1w { z22.s }, p1/Z, [x23, #2, MUL VL]\n" - "ld1w { z23.s }, p0/Z, [x23, #3, MUL VL]\n" - "ld1w { z24.s }, p3/Z, [x22]\n" - "ld1w { z25.s }, p2/Z, [x22, #1, MUL VL]\n" - "ld1w { z26.s }, p1/Z, [x22, #2, MUL VL]\n" - "ld1w { z27.s }, p0/Z, [x22, #3, MUL VL]\n" - "ld1w { z28.s }, p3/Z, [x21]\n" - "ld1w { z29.s }, p2/Z, [x21, #1, MUL VL]\n" - "ld1w { z30.s }, p1/Z, [x21, #2, MUL VL]\n" - "ld1w { z31.s }, p0/Z, [x21, #3, MUL VL]\n" + "ld1w { z12.s }, p3/Z, [x24]\n" + "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n" + "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n" + "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n" + "ld1w { z16.s }, p3/Z, [x23]\n" + "ld1w { z17.s }, p2/Z, [x23, #1, MUL VL]\n" + "ld1w { z18.s }, p1/Z, [x23, #2, MUL VL]\n" + "ld1w { z19.s }, p0/Z, [x23, #3, MUL VL]\n" + "ld1w { z20.s }, p3/Z, [x22]\n" + "ld1w { z21.s }, p2/Z, [x22, #1, MUL VL]\n" + "ld1w { z22.s }, p1/Z, [x22, #2, MUL VL]\n" + "ld1w { z23.s }, p0/Z, [x22, #3, MUL VL]\n" + "ld1w { z24.s }, p3/Z, [x21]\n" + "ld1w { z25.s }, p2/Z, [x21, #1, MUL VL]\n" + "ld1w { z26.s }, p1/Z, [x21, #2, MUL VL]\n" + "ld1w { z27.s }, p0/Z, [x21, #3, MUL VL]\n" + "ld1w { z28.s }, p3/Z, [x20]\n" + "ld1w { z29.s }, p2/Z, [x20, #1, MUL VL]\n" + "ld1w { z30.s }, p1/Z, [x20, #2, MUL VL]\n" + "ld1w { z31.s }, p0/Z, [x20, #3, MUL VL]\n" "b 71f\n" "70:" // Height 6: no accumulate "mov z8.b, #0x0\n" @@ -1300,16 +1300,16 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "72:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 73f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 74f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -1321,11 +1321,11 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( "b 74f\n" "73:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "add x21, x22, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" + "add x21, x22, x21, LSL #2\n" "74:" // Height 6: input setup done "subs x27, x27, #0x1\n" "ld1rw { z0.s }, p4/Z, [x26]\n" @@ -1527,4 +1527,4 @@ void sve_ffhybrid_fp32_mla_6x4VL_a64fx ( } } // namespace arm_gemm -#endif // ARM_COMPUTE_ENABLE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32_mla_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32_mla_6x4VL/generic.cpp index eb057e7734b7abbd8755ea7528b1b5bf07598319..b1ab31e618a09eb45e7cddf16620ec5097f4d91e 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32_mla_6x4VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32_mla_6x4VL/generic.cpp @@ -163,11 +163,11 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "7:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 8f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 9f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -180,40 +180,40 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "10:" // Height 1: Multiply loop: Main loop head "whilelt p0.s, XZR, x27\n" "ld1rqw { z0.s }, p0/Z, [x26]\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "fmla z8.s, z6.s, z0.s[0]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z9.s, z7.s, z0.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "fmla z10.s, z6.s, z0.s[0]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" - "fmla z11.s, z7.s, z0.s[0]\n" - "ld1w { z6.s }, p5/Z, [x12, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[1]\n" - "ld1w { z7.s }, p5/Z, [x11, #1, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[1]\n" - "ld1w { z7.s }, p5/Z, [x9, #1, MUL VL]\n" - "fmla z11.s, z7.s, z0.s[1]\n" - "ld1w { z6.s }, p5/Z, [x12, #2, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[2]\n" - "ld1w { z7.s }, p5/Z, [x11, #2, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[2]\n" - "ld1w { z7.s }, p5/Z, [x9, #2, MUL VL]\n" - "fmla z11.s, z7.s, z0.s[2]\n" - "ld1w { z6.s }, p5/Z, [x12, #3, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[3]\n" - "ld1w { z7.s }, p5/Z, [x11, #3, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[3]\n" - "ld1w { z6.s }, p5/Z, [x10, #3, MUL VL]\n" + "ld1w { z16.s }, p5/Z, [x12]\n" + "fmla z8.s, z16.s, z0.s[0]\n" + "ld1w { z16.s }, p5/Z, [x11]\n" + "fmla z9.s, z16.s, z0.s[0]\n" + "ld1w { z16.s }, p5/Z, [x10]\n" + "fmla z10.s, z16.s, z0.s[0]\n" + "ld1w { z16.s }, p5/Z, [x9]\n" + "fmla z11.s, z16.s, z0.s[0]\n" + "ld1w { z16.s }, p5/Z, [x12, #1, MUL VL]\n" + "fmla z8.s, z16.s, z0.s[1]\n" + "ld1w { z16.s }, p5/Z, [x11, #1, MUL VL]\n" + "fmla z9.s, z16.s, z0.s[1]\n" + "ld1w { z16.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z10.s, z16.s, z0.s[1]\n" + "ld1w { z16.s }, p5/Z, [x9, #1, MUL VL]\n" + "fmla z11.s, z16.s, z0.s[1]\n" + "ld1w { z16.s }, p5/Z, [x12, #2, MUL VL]\n" + "fmla z8.s, z16.s, z0.s[2]\n" + "ld1w { z16.s }, p5/Z, [x11, #2, MUL VL]\n" + "fmla z9.s, z16.s, z0.s[2]\n" + "ld1w { z16.s }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z10.s, z16.s, z0.s[2]\n" + "ld1w { z16.s }, p5/Z, [x9, #2, MUL VL]\n" + "fmla z11.s, z16.s, z0.s[2]\n" + "ld1w { z16.s }, p5/Z, [x12, #3, MUL VL]\n" + "fmla z8.s, z16.s, z0.s[3]\n" + "ld1w { z16.s }, p5/Z, [x11, #3, MUL VL]\n" + "fmla z9.s, z16.s, z0.s[3]\n" + "ld1w { z17.s }, p5/Z, [x10, #3, MUL VL]\n" "sub x27, x27, #0x4\n" - "ld1w { z7.s }, p5/Z, [x9, #3, MUL VL]\n" + "ld1w { z16.s }, p5/Z, [x9, #3, MUL VL]\n" "cmp x27, #0x4\n" - "fmla z10.s, z6.s, z0.s[3]\n" - "fmla z11.s, z7.s, z0.s[3]\n" + "fmla z10.s, z17.s, z0.s[3]\n" + "fmla z11.s, z16.s, z0.s[3]\n" "add x26, x26, #0x10\n" "addvl x12, x12, #4\n" "addvl x11, x11, #4\n" @@ -223,56 +223,56 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "11:" // Height 1: Multiply loop: Single iteration only "whilelt p0.s, XZR, x27\n" "ld1rqw { z0.s }, p0/Z, [x26]\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "fmla z8.s, z6.s, z0.s[0]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z9.s, z7.s, z0.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" + "ld1w { z16.s }, p5/Z, [x12]\n" + "fmla z8.s, z16.s, z0.s[0]\n" + "ld1w { z16.s }, p5/Z, [x11]\n" + "fmla z9.s, z16.s, z0.s[0]\n" + "ld1w { z17.s }, p5/Z, [x10]\n" "subs x27, x27, #0x1\n" - "ld1w { z7.s }, p5/Z, [x9]\n" - "fmla z10.s, z6.s, z0.s[0]\n" - "fmla z11.s, z7.s, z0.s[0]\n" + "ld1w { z16.s }, p5/Z, [x9]\n" + "fmla z10.s, z17.s, z0.s[0]\n" + "fmla z11.s, z16.s, z0.s[0]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" "ble 12f\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z8.s, z6.s, z0.s[1]\n" - "fmla z9.s, z7.s, z0.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" + "ld1w { z17.s }, p5/Z, [x12]\n" + "ld1w { z16.s }, p5/Z, [x11]\n" + "fmla z8.s, z17.s, z0.s[1]\n" + "fmla z9.s, z16.s, z0.s[1]\n" + "ld1w { z17.s }, p5/Z, [x10]\n" + "ld1w { z16.s }, p5/Z, [x9]\n" "subs x27, x27, #0x1\n" - "fmla z10.s, z6.s, z0.s[1]\n" - "fmla z11.s, z7.s, z0.s[1]\n" + "fmla z10.s, z17.s, z0.s[1]\n" + "fmla z11.s, z16.s, z0.s[1]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" "ble 12f\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z8.s, z6.s, z0.s[2]\n" - "fmla z9.s, z7.s, z0.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" + "ld1w { z17.s }, p5/Z, [x12]\n" + "ld1w { z16.s }, p5/Z, [x11]\n" + "fmla z8.s, z17.s, z0.s[2]\n" + "fmla z9.s, z16.s, z0.s[2]\n" + "ld1w { z17.s }, p5/Z, [x10]\n" + "ld1w { z16.s }, p5/Z, [x9]\n" "subs x27, x27, #0x1\n" - "fmla z10.s, z6.s, z0.s[2]\n" - "fmla z11.s, z7.s, z0.s[2]\n" + "fmla z10.s, z17.s, z0.s[2]\n" + "fmla z11.s, z16.s, z0.s[2]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" "ble 12f\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z8.s, z6.s, z0.s[3]\n" - "fmla z9.s, z7.s, z0.s[3]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" - "fmla z10.s, z6.s, z0.s[3]\n" - "fmla z11.s, z7.s, z0.s[3]\n" + "ld1w { z17.s }, p5/Z, [x12]\n" + "ld1w { z16.s }, p5/Z, [x11]\n" + "fmla z8.s, z17.s, z0.s[3]\n" + "fmla z9.s, z16.s, z0.s[3]\n" + "ld1w { z17.s }, p5/Z, [x10]\n" + "ld1w { z16.s }, p5/Z, [x9]\n" + "fmla z10.s, z17.s, z0.s[3]\n" + "fmla z11.s, z16.s, z0.s[3]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" @@ -284,17 +284,17 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "bne 7b\n" "tbz %x[flags], #1, 13f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z17.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" + "ld1rw { z16.s }, p5/Z, [x20]\n" + "fmin z8.s, p5/M, z8.s, z17.s\n" + "fmin z9.s, p5/M, z9.s, z17.s\n" + "fmin z10.s, p5/M, z10.s, z17.s\n" + "fmin z11.s, p5/M, z11.s, z17.s\n" + "fmax z8.s, p5/M, z8.s, z16.s\n" + "fmax z9.s, p5/M, z9.s, z16.s\n" + "fmax z10.s, p5/M, z10.s, z16.s\n" + "fmax z11.s, p5/M, z11.s, z16.s\n" "13:" // Height 1: No activation "st1w { z8.s }, p4, [x13]\n" "st1w { z9.s }, p3, [x13, #1, MUL VL]\n" @@ -355,15 +355,15 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "18:" // Height 2: no bias "tbz %x[flags], #0, 19f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #2\n" + "add x20, x13, x20, LSL #2\n" "ld1w { z8.s }, p4/Z, [x13]\n" "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n" "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n" "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x20]\n" + "ld1w { z13.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z15.s }, p1/Z, [x20, #3, MUL VL]\n" "b 20f\n" "19:" // Height 2: no accumulate "mov z8.b, #0x0\n" @@ -379,12 +379,12 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "21:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 22f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 23f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -392,143 +392,143 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "b 23f\n" "22:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" "23:" // Height 2: input setup done "cmp x27, #0x4\n" "ble 25f\n" "24:" // Height 2: Multiply loop: Main loop head "whilelt p0.s, XZR, x27\n" - "ld1rqw { z0.s }, p0/Z, [x26]\n" - "ld1rqw { z1.s }, p0/Z, [x25]\n" + "ld1rqw { z1.s }, p0/Z, [x26]\n" + "ld1rqw { z0.s }, p0/Z, [x25]\n" "sub x27, x27, #0x4\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z8.s, z6.s, z0.s[0]\n" - "fmla z12.s, z6.s, z1.s[0]\n" - "fmla z9.s, z7.s, z0.s[0]\n" - "fmla z13.s, z7.s, z1.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" - "fmla z10.s, z6.s, z0.s[0]\n" - "fmla z14.s, z6.s, z1.s[0]\n" - "ld1w { z6.s }, p5/Z, [x12, #1, MUL VL]\n" + "ld1w { z17.s }, p5/Z, [x12]\n" + "ld1w { z16.s }, p5/Z, [x11]\n" + "fmla z8.s, z17.s, z1.s[0]\n" + "fmla z12.s, z17.s, z0.s[0]\n" + "fmla z9.s, z16.s, z1.s[0]\n" + "fmla z13.s, z16.s, z0.s[0]\n" + "ld1w { z17.s }, p5/Z, [x10]\n" + "ld1w { z16.s }, p5/Z, [x9]\n" + "fmla z10.s, z17.s, z1.s[0]\n" + "fmla z14.s, z17.s, z0.s[0]\n" + "ld1w { z17.s }, p5/Z, [x12, #1, MUL VL]\n" "cmp x27, #0x4\n" - "fmla z11.s, z7.s, z0.s[0]\n" - "fmla z15.s, z7.s, z1.s[0]\n" - "ld1w { z7.s }, p5/Z, [x11, #1, MUL VL]\n" + "fmla z11.s, z16.s, z1.s[0]\n" + "fmla z15.s, z16.s, z0.s[0]\n" + "ld1w { z16.s }, p5/Z, [x11, #1, MUL VL]\n" "add x26, x26, #0x10\n" - "fmla z8.s, z6.s, z0.s[1]\n" - "fmla z12.s, z6.s, z1.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z17.s, z1.s[1]\n" + "fmla z12.s, z17.s, z0.s[1]\n" + "ld1w { z17.s }, p5/Z, [x10, #1, MUL VL]\n" "add x25, x25, #0x10\n" - "fmla z9.s, z7.s, z0.s[1]\n" - "fmla z13.s, z7.s, z1.s[1]\n" - "ld1w { z7.s }, p5/Z, [x9, #1, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[1]\n" - "fmla z14.s, z6.s, z1.s[1]\n" - "ld1w { z6.s }, p5/Z, [x12, #2, MUL VL]\n" - "fmla z11.s, z7.s, z0.s[1]\n" - "fmla z15.s, z7.s, z1.s[1]\n" - "ld1w { z7.s }, p5/Z, [x11, #2, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[2]\n" - "fmla z12.s, z6.s, z1.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[2]\n" - "fmla z13.s, z7.s, z1.s[2]\n" - "ld1w { z7.s }, p5/Z, [x9, #2, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[2]\n" - "fmla z14.s, z6.s, z1.s[2]\n" - "ld1w { z6.s }, p5/Z, [x12, #3, MUL VL]\n" + "fmla z9.s, z16.s, z1.s[1]\n" + "fmla z13.s, z16.s, z0.s[1]\n" + "ld1w { z16.s }, p5/Z, [x9, #1, MUL VL]\n" + "fmla z10.s, z17.s, z1.s[1]\n" + "fmla z14.s, z17.s, z0.s[1]\n" + "ld1w { z17.s }, p5/Z, [x12, #2, MUL VL]\n" + "fmla z11.s, z16.s, z1.s[1]\n" + "fmla z15.s, z16.s, z0.s[1]\n" + "ld1w { z16.s }, p5/Z, [x11, #2, MUL VL]\n" + "fmla z8.s, z17.s, z1.s[2]\n" + "fmla z12.s, z17.s, z0.s[2]\n" + "ld1w { z17.s }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.s, z16.s, z1.s[2]\n" + "fmla z13.s, z16.s, z0.s[2]\n" + "ld1w { z16.s }, p5/Z, [x9, #2, MUL VL]\n" + "fmla z10.s, z17.s, z1.s[2]\n" + "fmla z14.s, z17.s, z0.s[2]\n" + "ld1w { z17.s }, p5/Z, [x12, #3, MUL VL]\n" "addvl x12, x12, #4\n" - "fmla z11.s, z7.s, z0.s[2]\n" - "fmla z15.s, z7.s, z1.s[2]\n" - "ld1w { z7.s }, p5/Z, [x11, #3, MUL VL]\n" + "fmla z11.s, z16.s, z1.s[2]\n" + "fmla z15.s, z16.s, z0.s[2]\n" + "ld1w { z16.s }, p5/Z, [x11, #3, MUL VL]\n" "addvl x11, x11, #4\n" - "fmla z8.s, z6.s, z0.s[3]\n" - "fmla z12.s, z6.s, z1.s[3]\n" - "ld1w { z6.s }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z8.s, z17.s, z1.s[3]\n" + "fmla z12.s, z17.s, z0.s[3]\n" + "ld1w { z17.s }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z9.s, z7.s, z0.s[3]\n" - "fmla z13.s, z7.s, z1.s[3]\n" - "ld1w { z7.s }, p5/Z, [x9, #3, MUL VL]\n" + "fmla z9.s, z16.s, z1.s[3]\n" + "fmla z13.s, z16.s, z0.s[3]\n" + "ld1w { z16.s }, p5/Z, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "fmla z10.s, z6.s, z0.s[3]\n" - "fmla z14.s, z6.s, z1.s[3]\n" - "fmla z11.s, z7.s, z0.s[3]\n" - "fmla z15.s, z7.s, z1.s[3]\n" + "fmla z10.s, z17.s, z1.s[3]\n" + "fmla z14.s, z17.s, z0.s[3]\n" + "fmla z11.s, z16.s, z1.s[3]\n" + "fmla z15.s, z16.s, z0.s[3]\n" "bgt 24b\n" "25:" // Height 2: Multiply loop: Single iteration only "whilelt p0.s, XZR, x27\n" "ld1rqw { z0.s }, p0/Z, [x26]\n" "ld1rqw { z1.s }, p0/Z, [x25]\n" "subs x27, x27, #0x1\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z8.s, z6.s, z0.s[0]\n" - "fmla z12.s, z6.s, z1.s[0]\n" - "fmla z9.s, z7.s, z0.s[0]\n" - "fmla z13.s, z7.s, z1.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" - "fmla z10.s, z6.s, z0.s[0]\n" - "fmla z14.s, z6.s, z1.s[0]\n" + "ld1w { z17.s }, p5/Z, [x12]\n" + "ld1w { z16.s }, p5/Z, [x11]\n" + "fmla z8.s, z17.s, z0.s[0]\n" + "fmla z12.s, z17.s, z1.s[0]\n" + "fmla z9.s, z16.s, z0.s[0]\n" + "fmla z13.s, z16.s, z1.s[0]\n" + "ld1w { z17.s }, p5/Z, [x10]\n" + "ld1w { z16.s }, p5/Z, [x9]\n" + "fmla z10.s, z17.s, z0.s[0]\n" + "fmla z14.s, z17.s, z1.s[0]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" - "fmla z11.s, z7.s, z0.s[0]\n" - "fmla z15.s, z7.s, z1.s[0]\n" + "fmla z11.s, z16.s, z0.s[0]\n" + "fmla z15.s, z16.s, z1.s[0]\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" "ble 26f\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z8.s, z6.s, z0.s[1]\n" - "fmla z12.s, z6.s, z1.s[1]\n" - "fmla z9.s, z7.s, z0.s[1]\n" - "fmla z13.s, z7.s, z1.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" + "ld1w { z17.s }, p5/Z, [x12]\n" + "ld1w { z16.s }, p5/Z, [x11]\n" + "fmla z8.s, z17.s, z0.s[1]\n" + "fmla z12.s, z17.s, z1.s[1]\n" + "fmla z9.s, z16.s, z0.s[1]\n" + "fmla z13.s, z16.s, z1.s[1]\n" + "ld1w { z17.s }, p5/Z, [x10]\n" + "ld1w { z16.s }, p5/Z, [x9]\n" "subs x27, x27, #0x1\n" - "fmla z10.s, z6.s, z0.s[1]\n" - "fmla z14.s, z6.s, z1.s[1]\n" + "fmla z10.s, z17.s, z0.s[1]\n" + "fmla z14.s, z17.s, z1.s[1]\n" "addvl x12, x12, #1\n" - "fmla z11.s, z7.s, z0.s[1]\n" - "fmla z15.s, z7.s, z1.s[1]\n" + "fmla z11.s, z16.s, z0.s[1]\n" + "fmla z15.s, z16.s, z1.s[1]\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" "ble 26f\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z8.s, z6.s, z0.s[2]\n" - "fmla z12.s, z6.s, z1.s[2]\n" - "fmla z9.s, z7.s, z0.s[2]\n" - "fmla z13.s, z7.s, z1.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" + "ld1w { z17.s }, p5/Z, [x12]\n" + "ld1w { z16.s }, p5/Z, [x11]\n" + "fmla z8.s, z17.s, z0.s[2]\n" + "fmla z12.s, z17.s, z1.s[2]\n" + "fmla z9.s, z16.s, z0.s[2]\n" + "fmla z13.s, z16.s, z1.s[2]\n" + "ld1w { z17.s }, p5/Z, [x10]\n" + "ld1w { z16.s }, p5/Z, [x9]\n" "subs x27, x27, #0x1\n" - "fmla z10.s, z6.s, z0.s[2]\n" - "fmla z14.s, z6.s, z1.s[2]\n" + "fmla z10.s, z17.s, z0.s[2]\n" + "fmla z14.s, z17.s, z1.s[2]\n" "addvl x12, x12, #1\n" - "fmla z11.s, z7.s, z0.s[2]\n" - "fmla z15.s, z7.s, z1.s[2]\n" + "fmla z11.s, z16.s, z0.s[2]\n" + "fmla z15.s, z16.s, z1.s[2]\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" "ble 26f\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z8.s, z6.s, z0.s[3]\n" - "fmla z12.s, z6.s, z1.s[3]\n" - "fmla z9.s, z7.s, z0.s[3]\n" - "fmla z13.s, z7.s, z1.s[3]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" - "fmla z10.s, z6.s, z0.s[3]\n" - "fmla z14.s, z6.s, z1.s[3]\n" + "ld1w { z17.s }, p5/Z, [x12]\n" + "ld1w { z16.s }, p5/Z, [x11]\n" + "fmla z8.s, z17.s, z0.s[3]\n" + "fmla z12.s, z17.s, z1.s[3]\n" + "fmla z9.s, z16.s, z0.s[3]\n" + "fmla z13.s, z16.s, z1.s[3]\n" + "ld1w { z17.s }, p5/Z, [x10]\n" + "ld1w { z16.s }, p5/Z, [x9]\n" + "fmla z10.s, z17.s, z0.s[3]\n" + "fmla z14.s, z17.s, z1.s[3]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" - "fmla z11.s, z7.s, z0.s[3]\n" - "fmla z15.s, z7.s, z1.s[3]\n" + "fmla z11.s, z16.s, z0.s[3]\n" + "fmla z15.s, z16.s, z1.s[3]\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" "26:" // Height 2: Multiply loop: multiply skip @@ -540,25 +540,25 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "add x25, x13, x20, LSL #2\n" "tbz %x[flags], #1, 27f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z17.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmin z12.s, p5/M, z12.s, z1.s\n" - "fmin z13.s, p5/M, z13.s, z1.s\n" - "fmin z14.s, p5/M, z14.s, z1.s\n" - "fmin z15.s, p5/M, z15.s, z1.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" - "fmax z12.s, p5/M, z12.s, z0.s\n" - "fmax z13.s, p5/M, z13.s, z0.s\n" - "fmax z14.s, p5/M, z14.s, z0.s\n" - "fmax z15.s, p5/M, z15.s, z0.s\n" + "ld1rw { z16.s }, p5/Z, [x20]\n" + "fmin z8.s, p5/M, z8.s, z17.s\n" + "fmin z9.s, p5/M, z9.s, z17.s\n" + "fmin z10.s, p5/M, z10.s, z17.s\n" + "fmin z11.s, p5/M, z11.s, z17.s\n" + "fmin z12.s, p5/M, z12.s, z17.s\n" + "fmin z13.s, p5/M, z13.s, z17.s\n" + "fmin z14.s, p5/M, z14.s, z17.s\n" + "fmin z15.s, p5/M, z15.s, z17.s\n" + "fmax z8.s, p5/M, z8.s, z16.s\n" + "fmax z9.s, p5/M, z9.s, z16.s\n" + "fmax z10.s, p5/M, z10.s, z16.s\n" + "fmax z11.s, p5/M, z11.s, z16.s\n" + "fmax z12.s, p5/M, z12.s, z16.s\n" + "fmax z13.s, p5/M, z13.s, z16.s\n" + "fmax z14.s, p5/M, z14.s, z16.s\n" + "fmax z15.s, p5/M, z15.s, z16.s\n" "27:" // Height 2: No activation "st1w { z8.s }, p4, [x13]\n" "st1w { z9.s }, p3, [x13, #1, MUL VL]\n" @@ -627,20 +627,20 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "32:" // Height 3: no bias "tbz %x[flags], #0, 33f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" + "add x21, x13, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z8.s }, p4/Z, [x13]\n" "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n" "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n" "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "ld1w { z16.s }, p4/Z, [x24]\n" - "ld1w { z17.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x24, #2, MUL VL]\n" - "ld1w { z19.s }, p1/Z, [x24, #3, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x21]\n" + "ld1w { z13.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x21, #2, MUL VL]\n" + "ld1w { z15.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x20]\n" + "ld1w { z17.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z19.s }, p1/Z, [x20, #3, MUL VL]\n" "b 34f\n" "33:" // Height 3: no accumulate "mov z8.b, #0x0\n" @@ -660,13 +660,13 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "35:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 36f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 37f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -675,89 +675,89 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "b 37f\n" "36:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" "37:" // Height 3: input setup done "cmp x27, #0x4\n" "ble 39f\n" "38:" // Height 3: Multiply loop: Main loop head "whilelt p0.s, XZR, x27\n" - "ld1rqw { z0.s }, p0/Z, [x26]\n" + "ld1rqw { z2.s }, p0/Z, [x26]\n" "ld1rqw { z1.s }, p0/Z, [x25]\n" "sub x27, x27, #0x4\n" - "ld1rqw { z2.s }, p0/Z, [x24]\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "fmla z8.s, z6.s, z0.s[0]\n" - "fmla z12.s, z6.s, z1.s[0]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z16.s, z6.s, z2.s[0]\n" - "fmla z9.s, z7.s, z0.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "fmla z13.s, z7.s, z1.s[0]\n" - "fmla z17.s, z7.s, z2.s[0]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" + "ld1rqw { z0.s }, p0/Z, [x24]\n" + "ld1w { z21.s }, p5/Z, [x12]\n" + "fmla z8.s, z21.s, z2.s[0]\n" + "fmla z12.s, z21.s, z1.s[0]\n" + "ld1w { z20.s }, p5/Z, [x11]\n" + "fmla z16.s, z21.s, z0.s[0]\n" + "fmla z9.s, z20.s, z2.s[0]\n" + "ld1w { z21.s }, p5/Z, [x10]\n" + "fmla z13.s, z20.s, z1.s[0]\n" + "fmla z17.s, z20.s, z0.s[0]\n" + "ld1w { z20.s }, p5/Z, [x9]\n" "cmp x27, #0x4\n" - "fmla z10.s, z6.s, z0.s[0]\n" - "fmla z14.s, z6.s, z1.s[0]\n" + "fmla z10.s, z21.s, z2.s[0]\n" + "fmla z14.s, z21.s, z1.s[0]\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" - "fmla z18.s, z6.s, z2.s[0]\n" - "fmla z11.s, z7.s, z0.s[0]\n" - "ld1w { z6.s }, p5/Z, [x12, #1, MUL VL]\n" + "fmla z18.s, z21.s, z0.s[0]\n" + "fmla z11.s, z20.s, z2.s[0]\n" + "ld1w { z21.s }, p5/Z, [x12, #1, MUL VL]\n" "add x24, x24, #0x10\n" - "fmla z15.s, z7.s, z1.s[0]\n" - "fmla z19.s, z7.s, z2.s[0]\n" - "ld1w { z7.s }, p5/Z, [x11, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[1]\n" - "fmla z12.s, z6.s, z1.s[1]\n" - "fmla z16.s, z6.s, z2.s[1]\n" - "fmla z9.s, z7.s, z0.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z13.s, z7.s, z1.s[1]\n" - "fmla z17.s, z7.s, z2.s[1]\n" - "ld1w { z7.s }, p5/Z, [x9, #1, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[1]\n" - "fmla z14.s, z6.s, z1.s[1]\n" - "fmla z18.s, z6.s, z2.s[1]\n" - "fmla z11.s, z7.s, z0.s[1]\n" - "ld1w { z6.s }, p5/Z, [x12, #2, MUL VL]\n" - "fmla z15.s, z7.s, z1.s[1]\n" - "fmla z19.s, z7.s, z2.s[1]\n" - "ld1w { z7.s }, p5/Z, [x11, #2, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[2]\n" - "fmla z12.s, z6.s, z1.s[2]\n" - "fmla z16.s, z6.s, z2.s[2]\n" - "fmla z9.s, z7.s, z0.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z13.s, z7.s, z1.s[2]\n" - "fmla z17.s, z7.s, z2.s[2]\n" - "ld1w { z7.s }, p5/Z, [x9, #2, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[2]\n" - "fmla z14.s, z6.s, z1.s[2]\n" - "fmla z18.s, z6.s, z2.s[2]\n" - "fmla z11.s, z7.s, z0.s[2]\n" - "ld1w { z6.s }, p5/Z, [x12, #3, MUL VL]\n" + "fmla z15.s, z20.s, z1.s[0]\n" + "fmla z19.s, z20.s, z0.s[0]\n" + "ld1w { z20.s }, p5/Z, [x11, #1, MUL VL]\n" + "fmla z8.s, z21.s, z2.s[1]\n" + "fmla z12.s, z21.s, z1.s[1]\n" + "fmla z16.s, z21.s, z0.s[1]\n" + "fmla z9.s, z20.s, z2.s[1]\n" + "ld1w { z21.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z13.s, z20.s, z1.s[1]\n" + "fmla z17.s, z20.s, z0.s[1]\n" + "ld1w { z20.s }, p5/Z, [x9, #1, MUL VL]\n" + "fmla z10.s, z21.s, z2.s[1]\n" + "fmla z14.s, z21.s, z1.s[1]\n" + "fmla z18.s, z21.s, z0.s[1]\n" + "fmla z11.s, z20.s, z2.s[1]\n" + "ld1w { z21.s }, p5/Z, [x12, #2, MUL VL]\n" + "fmla z15.s, z20.s, z1.s[1]\n" + "fmla z19.s, z20.s, z0.s[1]\n" + "ld1w { z20.s }, p5/Z, [x11, #2, MUL VL]\n" + "fmla z8.s, z21.s, z2.s[2]\n" + "fmla z12.s, z21.s, z1.s[2]\n" + "fmla z16.s, z21.s, z0.s[2]\n" + "fmla z9.s, z20.s, z2.s[2]\n" + "ld1w { z21.s }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z13.s, z20.s, z1.s[2]\n" + "fmla z17.s, z20.s, z0.s[2]\n" + "ld1w { z20.s }, p5/Z, [x9, #2, MUL VL]\n" + "fmla z10.s, z21.s, z2.s[2]\n" + "fmla z14.s, z21.s, z1.s[2]\n" + "fmla z18.s, z21.s, z0.s[2]\n" + "fmla z11.s, z20.s, z2.s[2]\n" + "ld1w { z21.s }, p5/Z, [x12, #3, MUL VL]\n" "addvl x12, x12, #4\n" - "fmla z15.s, z7.s, z1.s[2]\n" - "fmla z19.s, z7.s, z2.s[2]\n" - "ld1w { z7.s }, p5/Z, [x11, #3, MUL VL]\n" + "fmla z15.s, z20.s, z1.s[2]\n" + "fmla z19.s, z20.s, z0.s[2]\n" + "ld1w { z20.s }, p5/Z, [x11, #3, MUL VL]\n" "addvl x11, x11, #4\n" - "fmla z8.s, z6.s, z0.s[3]\n" - "fmla z12.s, z6.s, z1.s[3]\n" - "fmla z16.s, z6.s, z2.s[3]\n" - "fmla z9.s, z7.s, z0.s[3]\n" - "ld1w { z6.s }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z8.s, z21.s, z2.s[3]\n" + "fmla z12.s, z21.s, z1.s[3]\n" + "fmla z16.s, z21.s, z0.s[3]\n" + "fmla z9.s, z20.s, z2.s[3]\n" + "ld1w { z21.s }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z13.s, z7.s, z1.s[3]\n" - "fmla z17.s, z7.s, z2.s[3]\n" - "ld1w { z7.s }, p5/Z, [x9, #3, MUL VL]\n" + "fmla z13.s, z20.s, z1.s[3]\n" + "fmla z17.s, z20.s, z0.s[3]\n" + "ld1w { z20.s }, p5/Z, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "fmla z10.s, z6.s, z0.s[3]\n" - "fmla z14.s, z6.s, z1.s[3]\n" - "fmla z18.s, z6.s, z2.s[3]\n" - "fmla z11.s, z7.s, z0.s[3]\n" - "fmla z15.s, z7.s, z1.s[3]\n" - "fmla z19.s, z7.s, z2.s[3]\n" + "fmla z10.s, z21.s, z2.s[3]\n" + "fmla z14.s, z21.s, z1.s[3]\n" + "fmla z18.s, z21.s, z0.s[3]\n" + "fmla z11.s, z20.s, z2.s[3]\n" + "fmla z15.s, z20.s, z1.s[3]\n" + "fmla z19.s, z20.s, z0.s[3]\n" "bgt 38b\n" "39:" // Height 3: Multiply loop: Single iteration only "whilelt p0.s, XZR, x27\n" @@ -765,91 +765,91 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "ld1rqw { z1.s }, p0/Z, [x25]\n" "subs x27, x27, #0x1\n" "ld1rqw { z2.s }, p0/Z, [x24]\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "fmla z8.s, z6.s, z0.s[0]\n" - "fmla z12.s, z6.s, z1.s[0]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z16.s, z6.s, z2.s[0]\n" - "fmla z9.s, z7.s, z0.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "fmla z13.s, z7.s, z1.s[0]\n" - "fmla z17.s, z7.s, z2.s[0]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" + "ld1w { z21.s }, p5/Z, [x12]\n" + "fmla z8.s, z21.s, z0.s[0]\n" + "fmla z12.s, z21.s, z1.s[0]\n" + "ld1w { z20.s }, p5/Z, [x11]\n" + "fmla z16.s, z21.s, z2.s[0]\n" + "fmla z9.s, z20.s, z0.s[0]\n" + "ld1w { z21.s }, p5/Z, [x10]\n" + "fmla z13.s, z20.s, z1.s[0]\n" + "fmla z17.s, z20.s, z2.s[0]\n" + "ld1w { z20.s }, p5/Z, [x9]\n" "addvl x12, x12, #1\n" - "fmla z10.s, z6.s, z0.s[0]\n" - "fmla z14.s, z6.s, z1.s[0]\n" + "fmla z10.s, z21.s, z0.s[0]\n" + "fmla z14.s, z21.s, z1.s[0]\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" - "fmla z18.s, z6.s, z2.s[0]\n" - "fmla z11.s, z7.s, z0.s[0]\n" + "fmla z18.s, z21.s, z2.s[0]\n" + "fmla z11.s, z20.s, z0.s[0]\n" "addvl x9, x9, #1\n" - "fmla z15.s, z7.s, z1.s[0]\n" - "fmla z19.s, z7.s, z2.s[0]\n" + "fmla z15.s, z20.s, z1.s[0]\n" + "fmla z19.s, z20.s, z2.s[0]\n" "ble 40f\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z8.s, z6.s, z0.s[1]\n" - "fmla z12.s, z6.s, z1.s[1]\n" - "fmla z16.s, z6.s, z2.s[1]\n" - "fmla z9.s, z7.s, z0.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" + "ld1w { z21.s }, p5/Z, [x12]\n" + "ld1w { z20.s }, p5/Z, [x11]\n" + "fmla z8.s, z21.s, z0.s[1]\n" + "fmla z12.s, z21.s, z1.s[1]\n" + "fmla z16.s, z21.s, z2.s[1]\n" + "fmla z9.s, z20.s, z0.s[1]\n" + "ld1w { z21.s }, p5/Z, [x10]\n" "subs x27, x27, #0x1\n" - "fmla z13.s, z7.s, z1.s[1]\n" - "fmla z17.s, z7.s, z2.s[1]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" + "fmla z13.s, z20.s, z1.s[1]\n" + "fmla z17.s, z20.s, z2.s[1]\n" + "ld1w { z20.s }, p5/Z, [x9]\n" "addvl x12, x12, #1\n" - "fmla z10.s, z6.s, z0.s[1]\n" - "fmla z14.s, z6.s, z1.s[1]\n" + "fmla z10.s, z21.s, z0.s[1]\n" + "fmla z14.s, z21.s, z1.s[1]\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" - "fmla z18.s, z6.s, z2.s[1]\n" - "fmla z11.s, z7.s, z0.s[1]\n" + "fmla z18.s, z21.s, z2.s[1]\n" + "fmla z11.s, z20.s, z0.s[1]\n" "addvl x9, x9, #1\n" - "fmla z15.s, z7.s, z1.s[1]\n" - "fmla z19.s, z7.s, z2.s[1]\n" + "fmla z15.s, z20.s, z1.s[1]\n" + "fmla z19.s, z20.s, z2.s[1]\n" "ble 40f\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z8.s, z6.s, z0.s[2]\n" - "fmla z12.s, z6.s, z1.s[2]\n" - "fmla z16.s, z6.s, z2.s[2]\n" - "fmla z9.s, z7.s, z0.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" + "ld1w { z21.s }, p5/Z, [x12]\n" + "ld1w { z20.s }, p5/Z, [x11]\n" + "fmla z8.s, z21.s, z0.s[2]\n" + "fmla z12.s, z21.s, z1.s[2]\n" + "fmla z16.s, z21.s, z2.s[2]\n" + "fmla z9.s, z20.s, z0.s[2]\n" + "ld1w { z21.s }, p5/Z, [x10]\n" "subs x27, x27, #0x1\n" - "fmla z13.s, z7.s, z1.s[2]\n" - "fmla z17.s, z7.s, z2.s[2]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" + "fmla z13.s, z20.s, z1.s[2]\n" + "fmla z17.s, z20.s, z2.s[2]\n" + "ld1w { z20.s }, p5/Z, [x9]\n" "addvl x12, x12, #1\n" - "fmla z10.s, z6.s, z0.s[2]\n" - "fmla z14.s, z6.s, z1.s[2]\n" + "fmla z10.s, z21.s, z0.s[2]\n" + "fmla z14.s, z21.s, z1.s[2]\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" - "fmla z18.s, z6.s, z2.s[2]\n" - "fmla z11.s, z7.s, z0.s[2]\n" + "fmla z18.s, z21.s, z2.s[2]\n" + "fmla z11.s, z20.s, z0.s[2]\n" "addvl x9, x9, #1\n" - "fmla z15.s, z7.s, z1.s[2]\n" - "fmla z19.s, z7.s, z2.s[2]\n" + "fmla z15.s, z20.s, z1.s[2]\n" + "fmla z19.s, z20.s, z2.s[2]\n" "ble 40f\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z8.s, z6.s, z0.s[3]\n" - "fmla z12.s, z6.s, z1.s[3]\n" - "fmla z16.s, z6.s, z2.s[3]\n" - "fmla z9.s, z7.s, z0.s[3]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" + "ld1w { z21.s }, p5/Z, [x12]\n" + "ld1w { z20.s }, p5/Z, [x11]\n" + "fmla z8.s, z21.s, z0.s[3]\n" + "fmla z12.s, z21.s, z1.s[3]\n" + "fmla z16.s, z21.s, z2.s[3]\n" + "fmla z9.s, z20.s, z0.s[3]\n" + "ld1w { z21.s }, p5/Z, [x10]\n" "addvl x12, x12, #1\n" - "fmla z13.s, z7.s, z1.s[3]\n" - "fmla z17.s, z7.s, z2.s[3]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" + "fmla z13.s, z20.s, z1.s[3]\n" + "fmla z17.s, z20.s, z2.s[3]\n" + "ld1w { z20.s }, p5/Z, [x9]\n" "addvl x11, x11, #1\n" - "fmla z10.s, z6.s, z0.s[3]\n" - "fmla z14.s, z6.s, z1.s[3]\n" + "fmla z10.s, z21.s, z0.s[3]\n" + "fmla z14.s, z21.s, z1.s[3]\n" "addvl x10, x10, #1\n" "addvl x9, x9, #1\n" - "fmla z18.s, z6.s, z2.s[3]\n" - "fmla z11.s, z7.s, z0.s[3]\n" - "fmla z15.s, z7.s, z1.s[3]\n" - "fmla z19.s, z7.s, z2.s[3]\n" + "fmla z18.s, z21.s, z2.s[3]\n" + "fmla z11.s, z20.s, z0.s[3]\n" + "fmla z15.s, z20.s, z1.s[3]\n" + "fmla z19.s, z20.s, z2.s[3]\n" "40:" // Height 3: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -860,33 +860,33 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "add x24, x25, x20, LSL #2\n" "tbz %x[flags], #1, 41f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z21.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmin z12.s, p5/M, z12.s, z1.s\n" - "fmin z13.s, p5/M, z13.s, z1.s\n" - "fmin z14.s, p5/M, z14.s, z1.s\n" - "fmin z15.s, p5/M, z15.s, z1.s\n" - "fmin z16.s, p5/M, z16.s, z1.s\n" - "fmin z17.s, p5/M, z17.s, z1.s\n" - "fmin z18.s, p5/M, z18.s, z1.s\n" - "fmin z19.s, p5/M, z19.s, z1.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" - "fmax z12.s, p5/M, z12.s, z0.s\n" - "fmax z13.s, p5/M, z13.s, z0.s\n" - "fmax z14.s, p5/M, z14.s, z0.s\n" - "fmax z15.s, p5/M, z15.s, z0.s\n" - "fmax z16.s, p5/M, z16.s, z0.s\n" - "fmax z17.s, p5/M, z17.s, z0.s\n" - "fmax z18.s, p5/M, z18.s, z0.s\n" - "fmax z19.s, p5/M, z19.s, z0.s\n" + "ld1rw { z20.s }, p5/Z, [x20]\n" + "fmin z8.s, p5/M, z8.s, z21.s\n" + "fmin z9.s, p5/M, z9.s, z21.s\n" + "fmin z10.s, p5/M, z10.s, z21.s\n" + "fmin z11.s, p5/M, z11.s, z21.s\n" + "fmin z12.s, p5/M, z12.s, z21.s\n" + "fmin z13.s, p5/M, z13.s, z21.s\n" + "fmin z14.s, p5/M, z14.s, z21.s\n" + "fmin z15.s, p5/M, z15.s, z21.s\n" + "fmin z16.s, p5/M, z16.s, z21.s\n" + "fmin z17.s, p5/M, z17.s, z21.s\n" + "fmin z18.s, p5/M, z18.s, z21.s\n" + "fmin z19.s, p5/M, z19.s, z21.s\n" + "fmax z8.s, p5/M, z8.s, z20.s\n" + "fmax z9.s, p5/M, z9.s, z20.s\n" + "fmax z10.s, p5/M, z10.s, z20.s\n" + "fmax z11.s, p5/M, z11.s, z20.s\n" + "fmax z12.s, p5/M, z12.s, z20.s\n" + "fmax z13.s, p5/M, z13.s, z20.s\n" + "fmax z14.s, p5/M, z14.s, z20.s\n" + "fmax z15.s, p5/M, z15.s, z20.s\n" + "fmax z16.s, p5/M, z16.s, z20.s\n" + "fmax z17.s, p5/M, z17.s, z20.s\n" + "fmax z18.s, p5/M, z18.s, z20.s\n" + "fmax z19.s, p5/M, z19.s, z20.s\n" "41:" // Height 3: No activation "st1w { z8.s }, p4, [x13]\n" "st1w { z9.s }, p3, [x13, #1, MUL VL]\n" @@ -963,25 +963,25 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "46:" // Height 4: no bias "tbz %x[flags], #0, 47f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" + "add x22, x13, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" "ld1w { z8.s }, p4/Z, [x13]\n" - "add x23, x24, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n" "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n" "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "ld1w { z16.s }, p4/Z, [x24]\n" - "ld1w { z17.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x24, #2, MUL VL]\n" - "ld1w { z19.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x23]\n" - "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n" - "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x22]\n" + "ld1w { z13.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x22, #2, MUL VL]\n" + "ld1w { z15.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x21]\n" + "ld1w { z17.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x21, #2, MUL VL]\n" + "ld1w { z19.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x20]\n" + "ld1w { z21.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z23.s }, p1/Z, [x20, #3, MUL VL]\n" "b 48f\n" "47:" // Height 4: no accumulate "mov z8.b, #0x0\n" @@ -1005,14 +1005,14 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "49:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 50f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 51f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -1022,108 +1022,108 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "b 51f\n" "50:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" "51:" // Height 4: input setup done "cmp x27, #0x4\n" "ble 53f\n" "52:" // Height 4: Multiply loop: Main loop head "whilelt p0.s, XZR, x27\n" - "ld1rqw { z0.s }, p0/Z, [x26]\n" - "ld1rqw { z1.s }, p0/Z, [x25]\n" + "ld1rqw { z3.s }, p0/Z, [x26]\n" + "ld1rqw { z2.s }, p0/Z, [x25]\n" "sub x27, x27, #0x4\n" - "ld1rqw { z2.s }, p0/Z, [x24]\n" - "ld1rqw { z3.s }, p0/Z, [x23]\n" + "ld1rqw { z1.s }, p0/Z, [x24]\n" + "ld1rqw { z0.s }, p0/Z, [x23]\n" "cmp x27, #0x4\n" "add x26, x26, #0x10\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z8.s, z6.s, z0.s[0]\n" - "fmla z12.s, z6.s, z1.s[0]\n" - "fmla z16.s, z6.s, z2.s[0]\n" - "fmla z20.s, z6.s, z3.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" + "ld1w { z25.s }, p5/Z, [x12]\n" + "ld1w { z24.s }, p5/Z, [x11]\n" + "fmla z8.s, z25.s, z3.s[0]\n" + "fmla z12.s, z25.s, z2.s[0]\n" + "fmla z16.s, z25.s, z1.s[0]\n" + "fmla z20.s, z25.s, z0.s[0]\n" + "ld1w { z25.s }, p5/Z, [x10]\n" "add x25, x25, #0x10\n" - "fmla z9.s, z7.s, z0.s[0]\n" - "fmla z13.s, z7.s, z1.s[0]\n" + "fmla z9.s, z24.s, z3.s[0]\n" + "fmla z13.s, z24.s, z2.s[0]\n" "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" - "fmla z17.s, z7.s, z2.s[0]\n" - "fmla z21.s, z7.s, z3.s[0]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" - "fmla z10.s, z6.s, z0.s[0]\n" - "fmla z14.s, z6.s, z1.s[0]\n" - "fmla z18.s, z6.s, z2.s[0]\n" - "fmla z22.s, z6.s, z3.s[0]\n" - "ld1w { z6.s }, p5/Z, [x12, #1, MUL VL]\n" - "fmla z11.s, z7.s, z0.s[0]\n" - "fmla z15.s, z7.s, z1.s[0]\n" - "fmla z19.s, z7.s, z2.s[0]\n" - "fmla z23.s, z7.s, z3.s[0]\n" - "ld1w { z7.s }, p5/Z, [x11, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[1]\n" - "fmla z12.s, z6.s, z1.s[1]\n" - "fmla z16.s, z6.s, z2.s[1]\n" - "fmla z20.s, z6.s, z3.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[1]\n" - "fmla z13.s, z7.s, z1.s[1]\n" - "fmla z17.s, z7.s, z2.s[1]\n" - "fmla z21.s, z7.s, z3.s[1]\n" - "ld1w { z7.s }, p5/Z, [x9, #1, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[1]\n" - "fmla z14.s, z6.s, z1.s[1]\n" - "fmla z18.s, z6.s, z2.s[1]\n" - "fmla z22.s, z6.s, z3.s[1]\n" - "ld1w { z6.s }, p5/Z, [x12, #2, MUL VL]\n" - "fmla z11.s, z7.s, z0.s[1]\n" - "fmla z15.s, z7.s, z1.s[1]\n" - "fmla z19.s, z7.s, z2.s[1]\n" - "fmla z23.s, z7.s, z3.s[1]\n" - "ld1w { z7.s }, p5/Z, [x11, #2, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[2]\n" - "fmla z12.s, z6.s, z1.s[2]\n" - "fmla z16.s, z6.s, z2.s[2]\n" - "fmla z20.s, z6.s, z3.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[2]\n" - "fmla z13.s, z7.s, z1.s[2]\n" - "fmla z17.s, z7.s, z2.s[2]\n" - "fmla z21.s, z7.s, z3.s[2]\n" - "ld1w { z7.s }, p5/Z, [x9, #2, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[2]\n" - "fmla z14.s, z6.s, z1.s[2]\n" - "fmla z18.s, z6.s, z2.s[2]\n" - "fmla z22.s, z6.s, z3.s[2]\n" - "ld1w { z6.s }, p5/Z, [x12, #3, MUL VL]\n" + "fmla z17.s, z24.s, z1.s[0]\n" + "fmla z21.s, z24.s, z0.s[0]\n" + "ld1w { z24.s }, p5/Z, [x9]\n" + "fmla z10.s, z25.s, z3.s[0]\n" + "fmla z14.s, z25.s, z2.s[0]\n" + "fmla z18.s, z25.s, z1.s[0]\n" + "fmla z22.s, z25.s, z0.s[0]\n" + "ld1w { z25.s }, p5/Z, [x12, #1, MUL VL]\n" + "fmla z11.s, z24.s, z3.s[0]\n" + "fmla z15.s, z24.s, z2.s[0]\n" + "fmla z19.s, z24.s, z1.s[0]\n" + "fmla z23.s, z24.s, z0.s[0]\n" + "ld1w { z24.s }, p5/Z, [x11, #1, MUL VL]\n" + "fmla z8.s, z25.s, z3.s[1]\n" + "fmla z12.s, z25.s, z2.s[1]\n" + "fmla z16.s, z25.s, z1.s[1]\n" + "fmla z20.s, z25.s, z0.s[1]\n" + "ld1w { z25.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z9.s, z24.s, z3.s[1]\n" + "fmla z13.s, z24.s, z2.s[1]\n" + "fmla z17.s, z24.s, z1.s[1]\n" + "fmla z21.s, z24.s, z0.s[1]\n" + "ld1w { z24.s }, p5/Z, [x9, #1, MUL VL]\n" + "fmla z10.s, z25.s, z3.s[1]\n" + "fmla z14.s, z25.s, z2.s[1]\n" + "fmla z18.s, z25.s, z1.s[1]\n" + "fmla z22.s, z25.s, z0.s[1]\n" + "ld1w { z25.s }, p5/Z, [x12, #2, MUL VL]\n" + "fmla z11.s, z24.s, z3.s[1]\n" + "fmla z15.s, z24.s, z2.s[1]\n" + "fmla z19.s, z24.s, z1.s[1]\n" + "fmla z23.s, z24.s, z0.s[1]\n" + "ld1w { z24.s }, p5/Z, [x11, #2, MUL VL]\n" + "fmla z8.s, z25.s, z3.s[2]\n" + "fmla z12.s, z25.s, z2.s[2]\n" + "fmla z16.s, z25.s, z1.s[2]\n" + "fmla z20.s, z25.s, z0.s[2]\n" + "ld1w { z25.s }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.s, z24.s, z3.s[2]\n" + "fmla z13.s, z24.s, z2.s[2]\n" + "fmla z17.s, z24.s, z1.s[2]\n" + "fmla z21.s, z24.s, z0.s[2]\n" + "ld1w { z24.s }, p5/Z, [x9, #2, MUL VL]\n" + "fmla z10.s, z25.s, z3.s[2]\n" + "fmla z14.s, z25.s, z2.s[2]\n" + "fmla z18.s, z25.s, z1.s[2]\n" + "fmla z22.s, z25.s, z0.s[2]\n" + "ld1w { z25.s }, p5/Z, [x12, #3, MUL VL]\n" "addvl x12, x12, #4\n" - "fmla z11.s, z7.s, z0.s[2]\n" - "fmla z15.s, z7.s, z1.s[2]\n" - "fmla z19.s, z7.s, z2.s[2]\n" - "fmla z23.s, z7.s, z3.s[2]\n" - "ld1w { z7.s }, p5/Z, [x11, #3, MUL VL]\n" + "fmla z11.s, z24.s, z3.s[2]\n" + "fmla z15.s, z24.s, z2.s[2]\n" + "fmla z19.s, z24.s, z1.s[2]\n" + "fmla z23.s, z24.s, z0.s[2]\n" + "ld1w { z24.s }, p5/Z, [x11, #3, MUL VL]\n" "addvl x11, x11, #4\n" - "fmla z8.s, z6.s, z0.s[3]\n" - "fmla z12.s, z6.s, z1.s[3]\n" - "fmla z16.s, z6.s, z2.s[3]\n" - "fmla z20.s, z6.s, z3.s[3]\n" - "ld1w { z6.s }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z8.s, z25.s, z3.s[3]\n" + "fmla z12.s, z25.s, z2.s[3]\n" + "fmla z16.s, z25.s, z1.s[3]\n" + "fmla z20.s, z25.s, z0.s[3]\n" + "ld1w { z25.s }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z9.s, z7.s, z0.s[3]\n" - "fmla z13.s, z7.s, z1.s[3]\n" - "fmla z17.s, z7.s, z2.s[3]\n" - "fmla z21.s, z7.s, z3.s[3]\n" - "ld1w { z7.s }, p5/Z, [x9, #3, MUL VL]\n" + "fmla z9.s, z24.s, z3.s[3]\n" + "fmla z13.s, z24.s, z2.s[3]\n" + "fmla z17.s, z24.s, z1.s[3]\n" + "fmla z21.s, z24.s, z0.s[3]\n" + "ld1w { z24.s }, p5/Z, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "fmla z10.s, z6.s, z0.s[3]\n" - "fmla z14.s, z6.s, z1.s[3]\n" - "fmla z18.s, z6.s, z2.s[3]\n" - "fmla z22.s, z6.s, z3.s[3]\n" - "fmla z11.s, z7.s, z0.s[3]\n" - "fmla z15.s, z7.s, z1.s[3]\n" - "fmla z19.s, z7.s, z2.s[3]\n" - "fmla z23.s, z7.s, z3.s[3]\n" + "fmla z10.s, z25.s, z3.s[3]\n" + "fmla z14.s, z25.s, z2.s[3]\n" + "fmla z18.s, z25.s, z1.s[3]\n" + "fmla z22.s, z25.s, z0.s[3]\n" + "fmla z11.s, z24.s, z3.s[3]\n" + "fmla z15.s, z24.s, z2.s[3]\n" + "fmla z19.s, z24.s, z1.s[3]\n" + "fmla z23.s, z24.s, z0.s[3]\n" "bgt 52b\n" "53:" // Height 4: Multiply loop: Single iteration only "whilelt p0.s, XZR, x27\n" @@ -1132,107 +1132,107 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "subs x27, x27, #0x1\n" "ld1rqw { z2.s }, p0/Z, [x24]\n" "ld1rqw { z3.s }, p0/Z, [x23]\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z8.s, z6.s, z0.s[0]\n" - "fmla z12.s, z6.s, z1.s[0]\n" - "fmla z16.s, z6.s, z2.s[0]\n" - "fmla z20.s, z6.s, z3.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" + "ld1w { z25.s }, p5/Z, [x12]\n" + "ld1w { z24.s }, p5/Z, [x11]\n" + "fmla z8.s, z25.s, z0.s[0]\n" + "fmla z12.s, z25.s, z1.s[0]\n" + "fmla z16.s, z25.s, z2.s[0]\n" + "fmla z20.s, z25.s, z3.s[0]\n" + "ld1w { z25.s }, p5/Z, [x10]\n" "addvl x12, x12, #1\n" - "fmla z9.s, z7.s, z0.s[0]\n" - "fmla z13.s, z7.s, z1.s[0]\n" + "fmla z9.s, z24.s, z0.s[0]\n" + "fmla z13.s, z24.s, z1.s[0]\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" - "fmla z17.s, z7.s, z2.s[0]\n" - "fmla z21.s, z7.s, z3.s[0]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" + "fmla z17.s, z24.s, z2.s[0]\n" + "fmla z21.s, z24.s, z3.s[0]\n" + "ld1w { z24.s }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.s, z6.s, z0.s[0]\n" - "fmla z14.s, z6.s, z1.s[0]\n" - "fmla z18.s, z6.s, z2.s[0]\n" - "fmla z22.s, z6.s, z3.s[0]\n" - "fmla z11.s, z7.s, z0.s[0]\n" - "fmla z15.s, z7.s, z1.s[0]\n" - "fmla z19.s, z7.s, z2.s[0]\n" - "fmla z23.s, z7.s, z3.s[0]\n" + "fmla z10.s, z25.s, z0.s[0]\n" + "fmla z14.s, z25.s, z1.s[0]\n" + "fmla z18.s, z25.s, z2.s[0]\n" + "fmla z22.s, z25.s, z3.s[0]\n" + "fmla z11.s, z24.s, z0.s[0]\n" + "fmla z15.s, z24.s, z1.s[0]\n" + "fmla z19.s, z24.s, z2.s[0]\n" + "fmla z23.s, z24.s, z3.s[0]\n" "ble 54f\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z8.s, z6.s, z0.s[1]\n" - "fmla z12.s, z6.s, z1.s[1]\n" - "fmla z16.s, z6.s, z2.s[1]\n" - "fmla z20.s, z6.s, z3.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" + "ld1w { z25.s }, p5/Z, [x12]\n" + "ld1w { z24.s }, p5/Z, [x11]\n" + "fmla z8.s, z25.s, z0.s[1]\n" + "fmla z12.s, z25.s, z1.s[1]\n" + "fmla z16.s, z25.s, z2.s[1]\n" + "fmla z20.s, z25.s, z3.s[1]\n" + "ld1w { z25.s }, p5/Z, [x10]\n" "subs x27, x27, #0x1\n" - "fmla z9.s, z7.s, z0.s[1]\n" - "fmla z13.s, z7.s, z1.s[1]\n" + "fmla z9.s, z24.s, z0.s[1]\n" + "fmla z13.s, z24.s, z1.s[1]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" - "fmla z17.s, z7.s, z2.s[1]\n" - "fmla z21.s, z7.s, z3.s[1]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" + "fmla z17.s, z24.s, z2.s[1]\n" + "fmla z21.s, z24.s, z3.s[1]\n" + "ld1w { z24.s }, p5/Z, [x9]\n" "addvl x10, x10, #1\n" - "fmla z10.s, z6.s, z0.s[1]\n" - "fmla z14.s, z6.s, z1.s[1]\n" + "fmla z10.s, z25.s, z0.s[1]\n" + "fmla z14.s, z25.s, z1.s[1]\n" "addvl x9, x9, #1\n" - "fmla z18.s, z6.s, z2.s[1]\n" - "fmla z22.s, z6.s, z3.s[1]\n" - "fmla z11.s, z7.s, z0.s[1]\n" - "fmla z15.s, z7.s, z1.s[1]\n" - "fmla z19.s, z7.s, z2.s[1]\n" - "fmla z23.s, z7.s, z3.s[1]\n" + "fmla z18.s, z25.s, z2.s[1]\n" + "fmla z22.s, z25.s, z3.s[1]\n" + "fmla z11.s, z24.s, z0.s[1]\n" + "fmla z15.s, z24.s, z1.s[1]\n" + "fmla z19.s, z24.s, z2.s[1]\n" + "fmla z23.s, z24.s, z3.s[1]\n" "ble 54f\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z8.s, z6.s, z0.s[2]\n" - "fmla z12.s, z6.s, z1.s[2]\n" - "fmla z16.s, z6.s, z2.s[2]\n" - "fmla z20.s, z6.s, z3.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" + "ld1w { z25.s }, p5/Z, [x12]\n" + "ld1w { z24.s }, p5/Z, [x11]\n" + "fmla z8.s, z25.s, z0.s[2]\n" + "fmla z12.s, z25.s, z1.s[2]\n" + "fmla z16.s, z25.s, z2.s[2]\n" + "fmla z20.s, z25.s, z3.s[2]\n" + "ld1w { z25.s }, p5/Z, [x10]\n" "subs x27, x27, #0x1\n" - "fmla z9.s, z7.s, z0.s[2]\n" - "fmla z13.s, z7.s, z1.s[2]\n" + "fmla z9.s, z24.s, z0.s[2]\n" + "fmla z13.s, z24.s, z1.s[2]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" - "fmla z17.s, z7.s, z2.s[2]\n" - "fmla z21.s, z7.s, z3.s[2]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" + "fmla z17.s, z24.s, z2.s[2]\n" + "fmla z21.s, z24.s, z3.s[2]\n" + "ld1w { z24.s }, p5/Z, [x9]\n" "addvl x10, x10, #1\n" - "fmla z10.s, z6.s, z0.s[2]\n" - "fmla z14.s, z6.s, z1.s[2]\n" + "fmla z10.s, z25.s, z0.s[2]\n" + "fmla z14.s, z25.s, z1.s[2]\n" "addvl x9, x9, #1\n" - "fmla z18.s, z6.s, z2.s[2]\n" - "fmla z22.s, z6.s, z3.s[2]\n" - "fmla z11.s, z7.s, z0.s[2]\n" - "fmla z15.s, z7.s, z1.s[2]\n" - "fmla z19.s, z7.s, z2.s[2]\n" - "fmla z23.s, z7.s, z3.s[2]\n" + "fmla z18.s, z25.s, z2.s[2]\n" + "fmla z22.s, z25.s, z3.s[2]\n" + "fmla z11.s, z24.s, z0.s[2]\n" + "fmla z15.s, z24.s, z1.s[2]\n" + "fmla z19.s, z24.s, z2.s[2]\n" + "fmla z23.s, z24.s, z3.s[2]\n" "ble 54f\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z8.s, z6.s, z0.s[3]\n" - "fmla z12.s, z6.s, z1.s[3]\n" - "fmla z16.s, z6.s, z2.s[3]\n" - "fmla z20.s, z6.s, z3.s[3]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" + "ld1w { z25.s }, p5/Z, [x12]\n" + "ld1w { z24.s }, p5/Z, [x11]\n" + "fmla z8.s, z25.s, z0.s[3]\n" + "fmla z12.s, z25.s, z1.s[3]\n" + "fmla z16.s, z25.s, z2.s[3]\n" + "fmla z20.s, z25.s, z3.s[3]\n" + "ld1w { z25.s }, p5/Z, [x10]\n" "addvl x12, x12, #1\n" - "fmla z9.s, z7.s, z0.s[3]\n" - "fmla z13.s, z7.s, z1.s[3]\n" + "fmla z9.s, z24.s, z0.s[3]\n" + "fmla z13.s, z24.s, z1.s[3]\n" "addvl x11, x11, #1\n" "addvl x10, x10, #1\n" - "fmla z17.s, z7.s, z2.s[3]\n" - "fmla z21.s, z7.s, z3.s[3]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" + "fmla z17.s, z24.s, z2.s[3]\n" + "fmla z21.s, z24.s, z3.s[3]\n" + "ld1w { z24.s }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.s, z6.s, z0.s[3]\n" - "fmla z14.s, z6.s, z1.s[3]\n" - "fmla z18.s, z6.s, z2.s[3]\n" - "fmla z22.s, z6.s, z3.s[3]\n" - "fmla z11.s, z7.s, z0.s[3]\n" - "fmla z15.s, z7.s, z1.s[3]\n" - "fmla z19.s, z7.s, z2.s[3]\n" - "fmla z23.s, z7.s, z3.s[3]\n" + "fmla z10.s, z25.s, z0.s[3]\n" + "fmla z14.s, z25.s, z1.s[3]\n" + "fmla z18.s, z25.s, z2.s[3]\n" + "fmla z22.s, z25.s, z3.s[3]\n" + "fmla z11.s, z24.s, z0.s[3]\n" + "fmla z15.s, z24.s, z1.s[3]\n" + "fmla z19.s, z24.s, z2.s[3]\n" + "fmla z23.s, z24.s, z3.s[3]\n" "54:" // Height 4: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1244,41 +1244,41 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "add x23, x24, x20, LSL #2\n" "tbz %x[flags], #1, 55f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z25.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmin z12.s, p5/M, z12.s, z1.s\n" - "fmin z13.s, p5/M, z13.s, z1.s\n" - "fmin z14.s, p5/M, z14.s, z1.s\n" - "fmin z15.s, p5/M, z15.s, z1.s\n" - "fmin z16.s, p5/M, z16.s, z1.s\n" - "fmin z17.s, p5/M, z17.s, z1.s\n" - "fmin z18.s, p5/M, z18.s, z1.s\n" - "fmin z19.s, p5/M, z19.s, z1.s\n" - "fmin z20.s, p5/M, z20.s, z1.s\n" - "fmin z21.s, p5/M, z21.s, z1.s\n" - "fmin z22.s, p5/M, z22.s, z1.s\n" - "fmin z23.s, p5/M, z23.s, z1.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" - "fmax z12.s, p5/M, z12.s, z0.s\n" - "fmax z13.s, p5/M, z13.s, z0.s\n" - "fmax z14.s, p5/M, z14.s, z0.s\n" - "fmax z15.s, p5/M, z15.s, z0.s\n" - "fmax z16.s, p5/M, z16.s, z0.s\n" - "fmax z17.s, p5/M, z17.s, z0.s\n" - "fmax z18.s, p5/M, z18.s, z0.s\n" - "fmax z19.s, p5/M, z19.s, z0.s\n" - "fmax z20.s, p5/M, z20.s, z0.s\n" - "fmax z21.s, p5/M, z21.s, z0.s\n" - "fmax z22.s, p5/M, z22.s, z0.s\n" - "fmax z23.s, p5/M, z23.s, z0.s\n" + "ld1rw { z24.s }, p5/Z, [x20]\n" + "fmin z8.s, p5/M, z8.s, z25.s\n" + "fmin z9.s, p5/M, z9.s, z25.s\n" + "fmin z10.s, p5/M, z10.s, z25.s\n" + "fmin z11.s, p5/M, z11.s, z25.s\n" + "fmin z12.s, p5/M, z12.s, z25.s\n" + "fmin z13.s, p5/M, z13.s, z25.s\n" + "fmin z14.s, p5/M, z14.s, z25.s\n" + "fmin z15.s, p5/M, z15.s, z25.s\n" + "fmin z16.s, p5/M, z16.s, z25.s\n" + "fmin z17.s, p5/M, z17.s, z25.s\n" + "fmin z18.s, p5/M, z18.s, z25.s\n" + "fmin z19.s, p5/M, z19.s, z25.s\n" + "fmin z20.s, p5/M, z20.s, z25.s\n" + "fmin z21.s, p5/M, z21.s, z25.s\n" + "fmin z22.s, p5/M, z22.s, z25.s\n" + "fmin z23.s, p5/M, z23.s, z25.s\n" + "fmax z8.s, p5/M, z8.s, z24.s\n" + "fmax z9.s, p5/M, z9.s, z24.s\n" + "fmax z10.s, p5/M, z10.s, z24.s\n" + "fmax z11.s, p5/M, z11.s, z24.s\n" + "fmax z12.s, p5/M, z12.s, z24.s\n" + "fmax z13.s, p5/M, z13.s, z24.s\n" + "fmax z14.s, p5/M, z14.s, z24.s\n" + "fmax z15.s, p5/M, z15.s, z24.s\n" + "fmax z16.s, p5/M, z16.s, z24.s\n" + "fmax z17.s, p5/M, z17.s, z24.s\n" + "fmax z18.s, p5/M, z18.s, z24.s\n" + "fmax z19.s, p5/M, z19.s, z24.s\n" + "fmax z20.s, p5/M, z20.s, z24.s\n" + "fmax z21.s, p5/M, z21.s, z24.s\n" + "fmax z22.s, p5/M, z22.s, z24.s\n" + "fmax z23.s, p5/M, z23.s, z24.s\n" "55:" // Height 4: No activation "st1w { z8.s }, p4, [x13]\n" "st1w { z9.s }, p3, [x13, #1, MUL VL]\n" @@ -1363,30 +1363,30 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "60:" // Height 5: no bias "tbz %x[flags], #0, 61f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z8.s }, p4/Z, [x13]\n" - "add x23, x24, x20, LSL #2\n" + "add x23, x13, x20, LSL #2\n" "add x22, x23, x20, LSL #2\n" + "ld1w { z8.s }, p4/Z, [x13]\n" + "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n" "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n" "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "ld1w { z16.s }, p4/Z, [x24]\n" - "ld1w { z17.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x24, #2, MUL VL]\n" - "ld1w { z19.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x23]\n" - "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n" - "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n" - "ld1w { z24.s }, p4/Z, [x22]\n" - "ld1w { z25.s }, p3/Z, [x22, #1, MUL VL]\n" - "ld1w { z26.s }, p2/Z, [x22, #2, MUL VL]\n" - "ld1w { z27.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x23]\n" + "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n" + "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x22]\n" + "ld1w { z17.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x22, #2, MUL VL]\n" + "ld1w { z19.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x21]\n" + "ld1w { z21.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x21, #2, MUL VL]\n" + "ld1w { z23.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z24.s }, p4/Z, [x20]\n" + "ld1w { z25.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z26.s }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z27.s }, p1/Z, [x20, #3, MUL VL]\n" "b 62f\n" "61:" // Height 5: no accumulate "mov z8.b, #0x0\n" @@ -1414,15 +1414,15 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "63:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 64f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 65f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -1433,127 +1433,127 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "b 65f\n" "64:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" "65:" // Height 5: input setup done "cmp x27, #0x4\n" "ble 67f\n" "66:" // Height 5: Multiply loop: Main loop head "whilelt p0.s, XZR, x27\n" - "ld1rqw { z0.s }, p0/Z, [x26]\n" - "ld1rqw { z1.s }, p0/Z, [x25]\n" + "ld1rqw { z4.s }, p0/Z, [x26]\n" + "ld1rqw { z3.s }, p0/Z, [x25]\n" "sub x27, x27, #0x4\n" "ld1rqw { z2.s }, p0/Z, [x24]\n" - "ld1rqw { z3.s }, p0/Z, [x23]\n" + "ld1rqw { z1.s }, p0/Z, [x23]\n" "cmp x27, #0x4\n" "add x26, x26, #0x10\n" - "ld1rqw { z4.s }, p0/Z, [x22]\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "fmla z8.s, z6.s, z0.s[0]\n" - "fmla z12.s, z6.s, z1.s[0]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z16.s, z6.s, z2.s[0]\n" - "fmla z20.s, z6.s, z3.s[0]\n" + "ld1rqw { z0.s }, p0/Z, [x22]\n" + "ld1w { z29.s }, p5/Z, [x12]\n" + "fmla z8.s, z29.s, z4.s[0]\n" + "fmla z12.s, z29.s, z3.s[0]\n" + "ld1w { z28.s }, p5/Z, [x11]\n" + "fmla z16.s, z29.s, z2.s[0]\n" + "fmla z20.s, z29.s, z1.s[0]\n" "add x25, x25, #0x10\n" - "fmla z24.s, z6.s, z4.s[0]\n" - "fmla z9.s, z7.s, z0.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" + "fmla z24.s, z29.s, z0.s[0]\n" + "fmla z9.s, z28.s, z4.s[0]\n" + "ld1w { z29.s }, p5/Z, [x10]\n" "add x24, x24, #0x10\n" - "fmla z13.s, z7.s, z1.s[0]\n" - "fmla z17.s, z7.s, z2.s[0]\n" + "fmla z13.s, z28.s, z3.s[0]\n" + "fmla z17.s, z28.s, z2.s[0]\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - "fmla z21.s, z7.s, z3.s[0]\n" - "fmla z25.s, z7.s, z4.s[0]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" - "fmla z10.s, z6.s, z0.s[0]\n" - "fmla z14.s, z6.s, z1.s[0]\n" - "fmla z18.s, z6.s, z2.s[0]\n" - "fmla z22.s, z6.s, z3.s[0]\n" - "fmla z26.s, z6.s, z4.s[0]\n" - "fmla z11.s, z7.s, z0.s[0]\n" - "ld1w { z6.s }, p5/Z, [x12, #1, MUL VL]\n" - "fmla z15.s, z7.s, z1.s[0]\n" - "fmla z19.s, z7.s, z2.s[0]\n" - "fmla z23.s, z7.s, z3.s[0]\n" - "fmla z27.s, z7.s, z4.s[0]\n" - "ld1w { z7.s }, p5/Z, [x11, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[1]\n" - "fmla z12.s, z6.s, z1.s[1]\n" - "fmla z16.s, z6.s, z2.s[1]\n" - "fmla z20.s, z6.s, z3.s[1]\n" - "fmla z24.s, z6.s, z4.s[1]\n" - "fmla z9.s, z7.s, z0.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z13.s, z7.s, z1.s[1]\n" - "fmla z17.s, z7.s, z2.s[1]\n" - "fmla z21.s, z7.s, z3.s[1]\n" - "fmla z25.s, z7.s, z4.s[1]\n" - "ld1w { z7.s }, p5/Z, [x9, #1, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[1]\n" - "fmla z14.s, z6.s, z1.s[1]\n" - "fmla z18.s, z6.s, z2.s[1]\n" - "fmla z22.s, z6.s, z3.s[1]\n" - "fmla z26.s, z6.s, z4.s[1]\n" - "fmla z11.s, z7.s, z0.s[1]\n" - "ld1w { z6.s }, p5/Z, [x12, #2, MUL VL]\n" - "fmla z15.s, z7.s, z1.s[1]\n" - "fmla z19.s, z7.s, z2.s[1]\n" - "fmla z23.s, z7.s, z3.s[1]\n" - "fmla z27.s, z7.s, z4.s[1]\n" - "ld1w { z7.s }, p5/Z, [x11, #2, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[2]\n" - "fmla z12.s, z6.s, z1.s[2]\n" - "fmla z16.s, z6.s, z2.s[2]\n" - "fmla z20.s, z6.s, z3.s[2]\n" - "fmla z24.s, z6.s, z4.s[2]\n" - "fmla z9.s, z7.s, z0.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z13.s, z7.s, z1.s[2]\n" - "fmla z17.s, z7.s, z2.s[2]\n" - "fmla z21.s, z7.s, z3.s[2]\n" - "fmla z25.s, z7.s, z4.s[2]\n" - "ld1w { z7.s }, p5/Z, [x9, #2, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[2]\n" - "fmla z14.s, z6.s, z1.s[2]\n" - "fmla z18.s, z6.s, z2.s[2]\n" - "fmla z22.s, z6.s, z3.s[2]\n" - "fmla z26.s, z6.s, z4.s[2]\n" - "fmla z11.s, z7.s, z0.s[2]\n" - "ld1w { z6.s }, p5/Z, [x12, #3, MUL VL]\n" + "fmla z21.s, z28.s, z1.s[0]\n" + "fmla z25.s, z28.s, z0.s[0]\n" + "ld1w { z28.s }, p5/Z, [x9]\n" + "fmla z10.s, z29.s, z4.s[0]\n" + "fmla z14.s, z29.s, z3.s[0]\n" + "fmla z18.s, z29.s, z2.s[0]\n" + "fmla z22.s, z29.s, z1.s[0]\n" + "fmla z26.s, z29.s, z0.s[0]\n" + "fmla z11.s, z28.s, z4.s[0]\n" + "ld1w { z29.s }, p5/Z, [x12, #1, MUL VL]\n" + "fmla z15.s, z28.s, z3.s[0]\n" + "fmla z19.s, z28.s, z2.s[0]\n" + "fmla z23.s, z28.s, z1.s[0]\n" + "fmla z27.s, z28.s, z0.s[0]\n" + "ld1w { z28.s }, p5/Z, [x11, #1, MUL VL]\n" + "fmla z8.s, z29.s, z4.s[1]\n" + "fmla z12.s, z29.s, z3.s[1]\n" + "fmla z16.s, z29.s, z2.s[1]\n" + "fmla z20.s, z29.s, z1.s[1]\n" + "fmla z24.s, z29.s, z0.s[1]\n" + "fmla z9.s, z28.s, z4.s[1]\n" + "ld1w { z29.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z13.s, z28.s, z3.s[1]\n" + "fmla z17.s, z28.s, z2.s[1]\n" + "fmla z21.s, z28.s, z1.s[1]\n" + "fmla z25.s, z28.s, z0.s[1]\n" + "ld1w { z28.s }, p5/Z, [x9, #1, MUL VL]\n" + "fmla z10.s, z29.s, z4.s[1]\n" + "fmla z14.s, z29.s, z3.s[1]\n" + "fmla z18.s, z29.s, z2.s[1]\n" + "fmla z22.s, z29.s, z1.s[1]\n" + "fmla z26.s, z29.s, z0.s[1]\n" + "fmla z11.s, z28.s, z4.s[1]\n" + "ld1w { z29.s }, p5/Z, [x12, #2, MUL VL]\n" + "fmla z15.s, z28.s, z3.s[1]\n" + "fmla z19.s, z28.s, z2.s[1]\n" + "fmla z23.s, z28.s, z1.s[1]\n" + "fmla z27.s, z28.s, z0.s[1]\n" + "ld1w { z28.s }, p5/Z, [x11, #2, MUL VL]\n" + "fmla z8.s, z29.s, z4.s[2]\n" + "fmla z12.s, z29.s, z3.s[2]\n" + "fmla z16.s, z29.s, z2.s[2]\n" + "fmla z20.s, z29.s, z1.s[2]\n" + "fmla z24.s, z29.s, z0.s[2]\n" + "fmla z9.s, z28.s, z4.s[2]\n" + "ld1w { z29.s }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z13.s, z28.s, z3.s[2]\n" + "fmla z17.s, z28.s, z2.s[2]\n" + "fmla z21.s, z28.s, z1.s[2]\n" + "fmla z25.s, z28.s, z0.s[2]\n" + "ld1w { z28.s }, p5/Z, [x9, #2, MUL VL]\n" + "fmla z10.s, z29.s, z4.s[2]\n" + "fmla z14.s, z29.s, z3.s[2]\n" + "fmla z18.s, z29.s, z2.s[2]\n" + "fmla z22.s, z29.s, z1.s[2]\n" + "fmla z26.s, z29.s, z0.s[2]\n" + "fmla z11.s, z28.s, z4.s[2]\n" + "ld1w { z29.s }, p5/Z, [x12, #3, MUL VL]\n" "addvl x12, x12, #4\n" - "fmla z15.s, z7.s, z1.s[2]\n" - "fmla z19.s, z7.s, z2.s[2]\n" - "fmla z23.s, z7.s, z3.s[2]\n" - "fmla z27.s, z7.s, z4.s[2]\n" - "ld1w { z7.s }, p5/Z, [x11, #3, MUL VL]\n" + "fmla z15.s, z28.s, z3.s[2]\n" + "fmla z19.s, z28.s, z2.s[2]\n" + "fmla z23.s, z28.s, z1.s[2]\n" + "fmla z27.s, z28.s, z0.s[2]\n" + "ld1w { z28.s }, p5/Z, [x11, #3, MUL VL]\n" "addvl x11, x11, #4\n" - "fmla z8.s, z6.s, z0.s[3]\n" - "fmla z12.s, z6.s, z1.s[3]\n" - "fmla z16.s, z6.s, z2.s[3]\n" - "fmla z20.s, z6.s, z3.s[3]\n" - "fmla z24.s, z6.s, z4.s[3]\n" - "fmla z9.s, z7.s, z0.s[3]\n" - "ld1w { z6.s }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z8.s, z29.s, z4.s[3]\n" + "fmla z12.s, z29.s, z3.s[3]\n" + "fmla z16.s, z29.s, z2.s[3]\n" + "fmla z20.s, z29.s, z1.s[3]\n" + "fmla z24.s, z29.s, z0.s[3]\n" + "fmla z9.s, z28.s, z4.s[3]\n" + "ld1w { z29.s }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z13.s, z7.s, z1.s[3]\n" - "fmla z17.s, z7.s, z2.s[3]\n" - "fmla z21.s, z7.s, z3.s[3]\n" - "fmla z25.s, z7.s, z4.s[3]\n" - "ld1w { z7.s }, p5/Z, [x9, #3, MUL VL]\n" + "fmla z13.s, z28.s, z3.s[3]\n" + "fmla z17.s, z28.s, z2.s[3]\n" + "fmla z21.s, z28.s, z1.s[3]\n" + "fmla z25.s, z28.s, z0.s[3]\n" + "ld1w { z28.s }, p5/Z, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "fmla z10.s, z6.s, z0.s[3]\n" - "fmla z14.s, z6.s, z1.s[3]\n" - "fmla z18.s, z6.s, z2.s[3]\n" - "fmla z22.s, z6.s, z3.s[3]\n" - "fmla z26.s, z6.s, z4.s[3]\n" - "fmla z11.s, z7.s, z0.s[3]\n" - "fmla z15.s, z7.s, z1.s[3]\n" - "fmla z19.s, z7.s, z2.s[3]\n" - "fmla z23.s, z7.s, z3.s[3]\n" - "fmla z27.s, z7.s, z4.s[3]\n" + "fmla z10.s, z29.s, z4.s[3]\n" + "fmla z14.s, z29.s, z3.s[3]\n" + "fmla z18.s, z29.s, z2.s[3]\n" + "fmla z22.s, z29.s, z1.s[3]\n" + "fmla z26.s, z29.s, z0.s[3]\n" + "fmla z11.s, z28.s, z4.s[3]\n" + "fmla z15.s, z28.s, z3.s[3]\n" + "fmla z19.s, z28.s, z2.s[3]\n" + "fmla z23.s, z28.s, z1.s[3]\n" + "fmla z27.s, z28.s, z0.s[3]\n" "bgt 66b\n" "67:" // Height 5: Multiply loop: Single iteration only "whilelt p0.s, XZR, x27\n" @@ -1563,123 +1563,123 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "ld1rqw { z2.s }, p0/Z, [x24]\n" "ld1rqw { z3.s }, p0/Z, [x23]\n" "ld1rqw { z4.s }, p0/Z, [x22]\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "fmla z8.s, z6.s, z0.s[0]\n" - "fmla z12.s, z6.s, z1.s[0]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z16.s, z6.s, z2.s[0]\n" - "fmla z20.s, z6.s, z3.s[0]\n" + "ld1w { z29.s }, p5/Z, [x12]\n" + "fmla z8.s, z29.s, z0.s[0]\n" + "fmla z12.s, z29.s, z1.s[0]\n" + "ld1w { z28.s }, p5/Z, [x11]\n" + "fmla z16.s, z29.s, z2.s[0]\n" + "fmla z20.s, z29.s, z3.s[0]\n" "addvl x12, x12, #1\n" - "fmla z24.s, z6.s, z4.s[0]\n" - "fmla z9.s, z7.s, z0.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" + "fmla z24.s, z29.s, z4.s[0]\n" + "fmla z9.s, z28.s, z0.s[0]\n" + "ld1w { z29.s }, p5/Z, [x10]\n" "addvl x11, x11, #1\n" - "fmla z13.s, z7.s, z1.s[0]\n" - "fmla z17.s, z7.s, z2.s[0]\n" + "fmla z13.s, z28.s, z1.s[0]\n" + "fmla z17.s, z28.s, z2.s[0]\n" "addvl x10, x10, #1\n" - "fmla z21.s, z7.s, z3.s[0]\n" - "fmla z25.s, z7.s, z4.s[0]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" + "fmla z21.s, z28.s, z3.s[0]\n" + "fmla z25.s, z28.s, z4.s[0]\n" + "ld1w { z28.s }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.s, z6.s, z0.s[0]\n" - "fmla z14.s, z6.s, z1.s[0]\n" - "fmla z18.s, z6.s, z2.s[0]\n" - "fmla z22.s, z6.s, z3.s[0]\n" - "fmla z26.s, z6.s, z4.s[0]\n" - "fmla z11.s, z7.s, z0.s[0]\n" - "fmla z15.s, z7.s, z1.s[0]\n" - "fmla z19.s, z7.s, z2.s[0]\n" - "fmla z23.s, z7.s, z3.s[0]\n" - "fmla z27.s, z7.s, z4.s[0]\n" + "fmla z10.s, z29.s, z0.s[0]\n" + "fmla z14.s, z29.s, z1.s[0]\n" + "fmla z18.s, z29.s, z2.s[0]\n" + "fmla z22.s, z29.s, z3.s[0]\n" + "fmla z26.s, z29.s, z4.s[0]\n" + "fmla z11.s, z28.s, z0.s[0]\n" + "fmla z15.s, z28.s, z1.s[0]\n" + "fmla z19.s, z28.s, z2.s[0]\n" + "fmla z23.s, z28.s, z3.s[0]\n" + "fmla z27.s, z28.s, z4.s[0]\n" "ble 68f\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z8.s, z6.s, z0.s[1]\n" - "fmla z12.s, z6.s, z1.s[1]\n" - "fmla z16.s, z6.s, z2.s[1]\n" - "fmla z20.s, z6.s, z3.s[1]\n" + "ld1w { z29.s }, p5/Z, [x12]\n" + "ld1w { z28.s }, p5/Z, [x11]\n" + "fmla z8.s, z29.s, z0.s[1]\n" + "fmla z12.s, z29.s, z1.s[1]\n" + "fmla z16.s, z29.s, z2.s[1]\n" + "fmla z20.s, z29.s, z3.s[1]\n" "subs x27, x27, #0x1\n" "addvl x12, x12, #1\n" - "fmla z24.s, z6.s, z4.s[1]\n" - "fmla z9.s, z7.s, z0.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" + "fmla z24.s, z29.s, z4.s[1]\n" + "fmla z9.s, z28.s, z0.s[1]\n" + "ld1w { z29.s }, p5/Z, [x10]\n" "addvl x11, x11, #1\n" - "fmla z13.s, z7.s, z1.s[1]\n" - "fmla z17.s, z7.s, z2.s[1]\n" + "fmla z13.s, z28.s, z1.s[1]\n" + "fmla z17.s, z28.s, z2.s[1]\n" "addvl x10, x10, #1\n" - "fmla z21.s, z7.s, z3.s[1]\n" - "fmla z25.s, z7.s, z4.s[1]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" + "fmla z21.s, z28.s, z3.s[1]\n" + "fmla z25.s, z28.s, z4.s[1]\n" + "ld1w { z28.s }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.s, z6.s, z0.s[1]\n" - "fmla z14.s, z6.s, z1.s[1]\n" - "fmla z18.s, z6.s, z2.s[1]\n" - "fmla z22.s, z6.s, z3.s[1]\n" - "fmla z26.s, z6.s, z4.s[1]\n" - "fmla z11.s, z7.s, z0.s[1]\n" - "fmla z15.s, z7.s, z1.s[1]\n" - "fmla z19.s, z7.s, z2.s[1]\n" - "fmla z23.s, z7.s, z3.s[1]\n" - "fmla z27.s, z7.s, z4.s[1]\n" + "fmla z10.s, z29.s, z0.s[1]\n" + "fmla z14.s, z29.s, z1.s[1]\n" + "fmla z18.s, z29.s, z2.s[1]\n" + "fmla z22.s, z29.s, z3.s[1]\n" + "fmla z26.s, z29.s, z4.s[1]\n" + "fmla z11.s, z28.s, z0.s[1]\n" + "fmla z15.s, z28.s, z1.s[1]\n" + "fmla z19.s, z28.s, z2.s[1]\n" + "fmla z23.s, z28.s, z3.s[1]\n" + "fmla z27.s, z28.s, z4.s[1]\n" "ble 68f\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z8.s, z6.s, z0.s[2]\n" - "fmla z12.s, z6.s, z1.s[2]\n" - "fmla z16.s, z6.s, z2.s[2]\n" - "fmla z20.s, z6.s, z3.s[2]\n" + "ld1w { z29.s }, p5/Z, [x12]\n" + "ld1w { z28.s }, p5/Z, [x11]\n" + "fmla z8.s, z29.s, z0.s[2]\n" + "fmla z12.s, z29.s, z1.s[2]\n" + "fmla z16.s, z29.s, z2.s[2]\n" + "fmla z20.s, z29.s, z3.s[2]\n" "subs x27, x27, #0x1\n" "addvl x12, x12, #1\n" - "fmla z24.s, z6.s, z4.s[2]\n" - "fmla z9.s, z7.s, z0.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" + "fmla z24.s, z29.s, z4.s[2]\n" + "fmla z9.s, z28.s, z0.s[2]\n" + "ld1w { z29.s }, p5/Z, [x10]\n" "addvl x11, x11, #1\n" - "fmla z13.s, z7.s, z1.s[2]\n" - "fmla z17.s, z7.s, z2.s[2]\n" + "fmla z13.s, z28.s, z1.s[2]\n" + "fmla z17.s, z28.s, z2.s[2]\n" "addvl x10, x10, #1\n" - "fmla z21.s, z7.s, z3.s[2]\n" - "fmla z25.s, z7.s, z4.s[2]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" + "fmla z21.s, z28.s, z3.s[2]\n" + "fmla z25.s, z28.s, z4.s[2]\n" + "ld1w { z28.s }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.s, z6.s, z0.s[2]\n" - "fmla z14.s, z6.s, z1.s[2]\n" - "fmla z18.s, z6.s, z2.s[2]\n" - "fmla z22.s, z6.s, z3.s[2]\n" - "fmla z26.s, z6.s, z4.s[2]\n" - "fmla z11.s, z7.s, z0.s[2]\n" - "fmla z15.s, z7.s, z1.s[2]\n" - "fmla z19.s, z7.s, z2.s[2]\n" - "fmla z23.s, z7.s, z3.s[2]\n" - "fmla z27.s, z7.s, z4.s[2]\n" + "fmla z10.s, z29.s, z0.s[2]\n" + "fmla z14.s, z29.s, z1.s[2]\n" + "fmla z18.s, z29.s, z2.s[2]\n" + "fmla z22.s, z29.s, z3.s[2]\n" + "fmla z26.s, z29.s, z4.s[2]\n" + "fmla z11.s, z28.s, z0.s[2]\n" + "fmla z15.s, z28.s, z1.s[2]\n" + "fmla z19.s, z28.s, z2.s[2]\n" + "fmla z23.s, z28.s, z3.s[2]\n" + "fmla z27.s, z28.s, z4.s[2]\n" "ble 68f\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z8.s, z6.s, z0.s[3]\n" - "fmla z12.s, z6.s, z1.s[3]\n" - "fmla z16.s, z6.s, z2.s[3]\n" - "fmla z20.s, z6.s, z3.s[3]\n" + "ld1w { z29.s }, p5/Z, [x12]\n" + "ld1w { z28.s }, p5/Z, [x11]\n" + "fmla z8.s, z29.s, z0.s[3]\n" + "fmla z12.s, z29.s, z1.s[3]\n" + "fmla z16.s, z29.s, z2.s[3]\n" + "fmla z20.s, z29.s, z3.s[3]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" - "fmla z24.s, z6.s, z4.s[3]\n" - "fmla z9.s, z7.s, z0.s[3]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" + "fmla z24.s, z29.s, z4.s[3]\n" + "fmla z9.s, z28.s, z0.s[3]\n" + "ld1w { z29.s }, p5/Z, [x10]\n" "addvl x10, x10, #1\n" - "fmla z13.s, z7.s, z1.s[3]\n" - "fmla z17.s, z7.s, z2.s[3]\n" - "fmla z21.s, z7.s, z3.s[3]\n" - "fmla z25.s, z7.s, z4.s[3]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" + "fmla z13.s, z28.s, z1.s[3]\n" + "fmla z17.s, z28.s, z2.s[3]\n" + "fmla z21.s, z28.s, z3.s[3]\n" + "fmla z25.s, z28.s, z4.s[3]\n" + "ld1w { z28.s }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.s, z6.s, z0.s[3]\n" - "fmla z14.s, z6.s, z1.s[3]\n" - "fmla z18.s, z6.s, z2.s[3]\n" - "fmla z22.s, z6.s, z3.s[3]\n" - "fmla z26.s, z6.s, z4.s[3]\n" - "fmla z11.s, z7.s, z0.s[3]\n" - "fmla z15.s, z7.s, z1.s[3]\n" - "fmla z19.s, z7.s, z2.s[3]\n" - "fmla z23.s, z7.s, z3.s[3]\n" - "fmla z27.s, z7.s, z4.s[3]\n" + "fmla z10.s, z29.s, z0.s[3]\n" + "fmla z14.s, z29.s, z1.s[3]\n" + "fmla z18.s, z29.s, z2.s[3]\n" + "fmla z22.s, z29.s, z3.s[3]\n" + "fmla z26.s, z29.s, z4.s[3]\n" + "fmla z11.s, z28.s, z0.s[3]\n" + "fmla z15.s, z28.s, z1.s[3]\n" + "fmla z19.s, z28.s, z2.s[3]\n" + "fmla z23.s, z28.s, z3.s[3]\n" + "fmla z27.s, z28.s, z4.s[3]\n" "68:" // Height 5: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1692,49 +1692,49 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "add x22, x23, x20, LSL #2\n" "tbz %x[flags], #1, 69f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z29.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmin z12.s, p5/M, z12.s, z1.s\n" - "fmin z13.s, p5/M, z13.s, z1.s\n" - "fmin z14.s, p5/M, z14.s, z1.s\n" - "fmin z15.s, p5/M, z15.s, z1.s\n" - "fmin z16.s, p5/M, z16.s, z1.s\n" - "fmin z17.s, p5/M, z17.s, z1.s\n" - "fmin z18.s, p5/M, z18.s, z1.s\n" - "fmin z19.s, p5/M, z19.s, z1.s\n" - "fmin z20.s, p5/M, z20.s, z1.s\n" - "fmin z21.s, p5/M, z21.s, z1.s\n" - "fmin z22.s, p5/M, z22.s, z1.s\n" - "fmin z23.s, p5/M, z23.s, z1.s\n" - "fmin z24.s, p5/M, z24.s, z1.s\n" - "fmin z25.s, p5/M, z25.s, z1.s\n" - "fmin z26.s, p5/M, z26.s, z1.s\n" - "fmin z27.s, p5/M, z27.s, z1.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" - "fmax z12.s, p5/M, z12.s, z0.s\n" - "fmax z13.s, p5/M, z13.s, z0.s\n" - "fmax z14.s, p5/M, z14.s, z0.s\n" - "fmax z15.s, p5/M, z15.s, z0.s\n" - "fmax z16.s, p5/M, z16.s, z0.s\n" - "fmax z17.s, p5/M, z17.s, z0.s\n" - "fmax z18.s, p5/M, z18.s, z0.s\n" - "fmax z19.s, p5/M, z19.s, z0.s\n" - "fmax z20.s, p5/M, z20.s, z0.s\n" - "fmax z21.s, p5/M, z21.s, z0.s\n" - "fmax z22.s, p5/M, z22.s, z0.s\n" - "fmax z23.s, p5/M, z23.s, z0.s\n" - "fmax z24.s, p5/M, z24.s, z0.s\n" - "fmax z25.s, p5/M, z25.s, z0.s\n" - "fmax z26.s, p5/M, z26.s, z0.s\n" - "fmax z27.s, p5/M, z27.s, z0.s\n" + "ld1rw { z28.s }, p5/Z, [x20]\n" + "fmin z8.s, p5/M, z8.s, z29.s\n" + "fmin z9.s, p5/M, z9.s, z29.s\n" + "fmin z10.s, p5/M, z10.s, z29.s\n" + "fmin z11.s, p5/M, z11.s, z29.s\n" + "fmin z12.s, p5/M, z12.s, z29.s\n" + "fmin z13.s, p5/M, z13.s, z29.s\n" + "fmin z14.s, p5/M, z14.s, z29.s\n" + "fmin z15.s, p5/M, z15.s, z29.s\n" + "fmin z16.s, p5/M, z16.s, z29.s\n" + "fmin z17.s, p5/M, z17.s, z29.s\n" + "fmin z18.s, p5/M, z18.s, z29.s\n" + "fmin z19.s, p5/M, z19.s, z29.s\n" + "fmin z20.s, p5/M, z20.s, z29.s\n" + "fmin z21.s, p5/M, z21.s, z29.s\n" + "fmin z22.s, p5/M, z22.s, z29.s\n" + "fmin z23.s, p5/M, z23.s, z29.s\n" + "fmin z24.s, p5/M, z24.s, z29.s\n" + "fmin z25.s, p5/M, z25.s, z29.s\n" + "fmin z26.s, p5/M, z26.s, z29.s\n" + "fmin z27.s, p5/M, z27.s, z29.s\n" + "fmax z8.s, p5/M, z8.s, z28.s\n" + "fmax z9.s, p5/M, z9.s, z28.s\n" + "fmax z10.s, p5/M, z10.s, z28.s\n" + "fmax z11.s, p5/M, z11.s, z28.s\n" + "fmax z12.s, p5/M, z12.s, z28.s\n" + "fmax z13.s, p5/M, z13.s, z28.s\n" + "fmax z14.s, p5/M, z14.s, z28.s\n" + "fmax z15.s, p5/M, z15.s, z28.s\n" + "fmax z16.s, p5/M, z16.s, z28.s\n" + "fmax z17.s, p5/M, z17.s, z28.s\n" + "fmax z18.s, p5/M, z18.s, z28.s\n" + "fmax z19.s, p5/M, z19.s, z28.s\n" + "fmax z20.s, p5/M, z20.s, z28.s\n" + "fmax z21.s, p5/M, z21.s, z28.s\n" + "fmax z22.s, p5/M, z22.s, z28.s\n" + "fmax z23.s, p5/M, z23.s, z28.s\n" + "fmax z24.s, p5/M, z24.s, z28.s\n" + "fmax z25.s, p5/M, z25.s, z28.s\n" + "fmax z26.s, p5/M, z26.s, z28.s\n" + "fmax z27.s, p5/M, z27.s, z28.s\n" "69:" // Height 5: No activation "st1w { z8.s }, p4, [x13]\n" "st1w { z9.s }, p3, [x13, #1, MUL VL]\n" @@ -1830,35 +1830,35 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "74:" // Height 6: no bias "tbz %x[flags], #0, 75f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x13, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z8.s }, p4/Z, [x13]\n" + "add x24, x13, x20, LSL #2\n" "add x23, x24, x20, LSL #2\n" + "ld1w { z8.s }, p4/Z, [x13]\n" "add x22, x23, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" "ld1w { z9.s }, p3/Z, [x13, #1, MUL VL]\n" "ld1w { z10.s }, p2/Z, [x13, #2, MUL VL]\n" - "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z11.s }, p1/Z, [x13, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "ld1w { z16.s }, p4/Z, [x24]\n" - "ld1w { z17.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x24, #2, MUL VL]\n" - "ld1w { z19.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x23]\n" - "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n" - "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n" - "ld1w { z24.s }, p4/Z, [x22]\n" - "ld1w { z25.s }, p3/Z, [x22, #1, MUL VL]\n" - "ld1w { z26.s }, p2/Z, [x22, #2, MUL VL]\n" - "ld1w { z27.s }, p1/Z, [x22, #3, MUL VL]\n" - "ld1w { z28.s }, p4/Z, [x21]\n" - "ld1w { z29.s }, p3/Z, [x21, #1, MUL VL]\n" - "ld1w { z30.s }, p2/Z, [x21, #2, MUL VL]\n" - "ld1w { z31.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x24]\n" + "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n" + "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x23]\n" + "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n" + "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x22]\n" + "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n" + "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z24.s }, p4/Z, [x21]\n" + "ld1w { z25.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z26.s }, p2/Z, [x21, #2, MUL VL]\n" + "ld1w { z27.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z28.s }, p4/Z, [x20]\n" + "ld1w { z29.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z30.s }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z31.s }, p1/Z, [x20, #3, MUL VL]\n" "b 76f\n" "75:" // Height 6: no accumulate "mov z8.b, #0x0\n" @@ -1890,16 +1890,16 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "77:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 78f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 79f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -1911,146 +1911,146 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "b 79f\n" "78:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "add x21, x22, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" + "add x21, x22, x21, LSL #2\n" "79:" // Height 6: input setup done "cmp x27, #0x4\n" "ble 81f\n" "80:" // Height 6: Multiply loop: Main loop head "whilelt p0.s, XZR, x27\n" - "ld1rqw { z0.s }, p0/Z, [x26]\n" - "ld1rqw { z1.s }, p0/Z, [x25]\n" + "ld1rqw { z7.s }, p0/Z, [x26]\n" + "ld1rqw { z6.s }, p0/Z, [x25]\n" "sub x27, x27, #0x4\n" - "ld1rqw { z2.s }, p0/Z, [x24]\n" - "ld1rqw { z3.s }, p0/Z, [x23]\n" + "ld1rqw { z5.s }, p0/Z, [x24]\n" + "ld1rqw { z4.s }, p0/Z, [x23]\n" "cmp x27, #0x4\n" "add x26, x26, #0x10\n" - "ld1rqw { z4.s }, p0/Z, [x22]\n" - "ld1rqw { z5.s }, p0/Z, [x21]\n" + "ld1rqw { z3.s }, p0/Z, [x22]\n" + "ld1rqw { z2.s }, p0/Z, [x21]\n" "add x25, x25, #0x10\n" "add x24, x24, #0x10\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z8.s, z6.s, z0.s[0]\n" - "fmla z12.s, z6.s, z1.s[0]\n" - "fmla z16.s, z6.s, z2.s[0]\n" - "fmla z20.s, z6.s, z3.s[0]\n" + "ld1w { z1.s }, p5/Z, [x12]\n" + "ld1w { z0.s }, p5/Z, [x11]\n" + "fmla z8.s, z1.s, z7.s[0]\n" + "fmla z12.s, z1.s, z6.s[0]\n" + "fmla z16.s, z1.s, z5.s[0]\n" + "fmla z20.s, z1.s, z4.s[0]\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - "fmla z24.s, z6.s, z4.s[0]\n" - "fmla z28.s, z6.s, z5.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" + "fmla z24.s, z1.s, z3.s[0]\n" + "fmla z28.s, z1.s, z2.s[0]\n" + "ld1w { z1.s }, p5/Z, [x10]\n" "add x21, x21, #0x10\n" - "fmla z9.s, z7.s, z0.s[0]\n" - "fmla z13.s, z7.s, z1.s[0]\n" - "fmla z17.s, z7.s, z2.s[0]\n" - "fmla z21.s, z7.s, z3.s[0]\n" - "fmla z25.s, z7.s, z4.s[0]\n" - "fmla z29.s, z7.s, z5.s[0]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" - "fmla z10.s, z6.s, z0.s[0]\n" - "fmla z14.s, z6.s, z1.s[0]\n" - "fmla z18.s, z6.s, z2.s[0]\n" - "fmla z22.s, z6.s, z3.s[0]\n" - "fmla z26.s, z6.s, z4.s[0]\n" - "fmla z30.s, z6.s, z5.s[0]\n" - "ld1w { z6.s }, p5/Z, [x12, #1, MUL VL]\n" - "fmla z11.s, z7.s, z0.s[0]\n" - "fmla z15.s, z7.s, z1.s[0]\n" - "fmla z19.s, z7.s, z2.s[0]\n" - "fmla z23.s, z7.s, z3.s[0]\n" - "fmla z27.s, z7.s, z4.s[0]\n" - "fmla z31.s, z7.s, z5.s[0]\n" - "ld1w { z7.s }, p5/Z, [x11, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[1]\n" - "fmla z12.s, z6.s, z1.s[1]\n" - "fmla z16.s, z6.s, z2.s[1]\n" - "fmla z20.s, z6.s, z3.s[1]\n" - "fmla z24.s, z6.s, z4.s[1]\n" - "fmla z28.s, z6.s, z5.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[1]\n" - "fmla z13.s, z7.s, z1.s[1]\n" - "fmla z17.s, z7.s, z2.s[1]\n" - "fmla z21.s, z7.s, z3.s[1]\n" - "fmla z25.s, z7.s, z4.s[1]\n" - "fmla z29.s, z7.s, z5.s[1]\n" - "ld1w { z7.s }, p5/Z, [x9, #1, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[1]\n" - "fmla z14.s, z6.s, z1.s[1]\n" - "fmla z18.s, z6.s, z2.s[1]\n" - "fmla z22.s, z6.s, z3.s[1]\n" - "fmla z26.s, z6.s, z4.s[1]\n" - "fmla z30.s, z6.s, z5.s[1]\n" - "ld1w { z6.s }, p5/Z, [x12, #2, MUL VL]\n" - "fmla z11.s, z7.s, z0.s[1]\n" - "fmla z15.s, z7.s, z1.s[1]\n" - "fmla z19.s, z7.s, z2.s[1]\n" - "fmla z23.s, z7.s, z3.s[1]\n" - "fmla z27.s, z7.s, z4.s[1]\n" - "fmla z31.s, z7.s, z5.s[1]\n" - "ld1w { z7.s }, p5/Z, [x11, #2, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[2]\n" - "fmla z12.s, z6.s, z1.s[2]\n" - "fmla z16.s, z6.s, z2.s[2]\n" - "fmla z20.s, z6.s, z3.s[2]\n" - "fmla z24.s, z6.s, z4.s[2]\n" - "fmla z28.s, z6.s, z5.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[2]\n" - "fmla z13.s, z7.s, z1.s[2]\n" - "fmla z17.s, z7.s, z2.s[2]\n" - "fmla z21.s, z7.s, z3.s[2]\n" - "fmla z25.s, z7.s, z4.s[2]\n" - "fmla z29.s, z7.s, z5.s[2]\n" - "ld1w { z7.s }, p5/Z, [x9, #2, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[2]\n" - "fmla z14.s, z6.s, z1.s[2]\n" - "fmla z18.s, z6.s, z2.s[2]\n" - "fmla z22.s, z6.s, z3.s[2]\n" - "fmla z26.s, z6.s, z4.s[2]\n" - "fmla z30.s, z6.s, z5.s[2]\n" - "ld1w { z6.s }, p5/Z, [x12, #3, MUL VL]\n" + "fmla z9.s, z0.s, z7.s[0]\n" + "fmla z13.s, z0.s, z6.s[0]\n" + "fmla z17.s, z0.s, z5.s[0]\n" + "fmla z21.s, z0.s, z4.s[0]\n" + "fmla z25.s, z0.s, z3.s[0]\n" + "fmla z29.s, z0.s, z2.s[0]\n" + "ld1w { z0.s }, p5/Z, [x9]\n" + "fmla z10.s, z1.s, z7.s[0]\n" + "fmla z14.s, z1.s, z6.s[0]\n" + "fmla z18.s, z1.s, z5.s[0]\n" + "fmla z22.s, z1.s, z4.s[0]\n" + "fmla z26.s, z1.s, z3.s[0]\n" + "fmla z30.s, z1.s, z2.s[0]\n" + "ld1w { z1.s }, p5/Z, [x12, #1, MUL VL]\n" + "fmla z11.s, z0.s, z7.s[0]\n" + "fmla z15.s, z0.s, z6.s[0]\n" + "fmla z19.s, z0.s, z5.s[0]\n" + "fmla z23.s, z0.s, z4.s[0]\n" + "fmla z27.s, z0.s, z3.s[0]\n" + "fmla z31.s, z0.s, z2.s[0]\n" + "ld1w { z0.s }, p5/Z, [x11, #1, MUL VL]\n" + "fmla z8.s, z1.s, z7.s[1]\n" + "fmla z12.s, z1.s, z6.s[1]\n" + "fmla z16.s, z1.s, z5.s[1]\n" + "fmla z20.s, z1.s, z4.s[1]\n" + "fmla z24.s, z1.s, z3.s[1]\n" + "fmla z28.s, z1.s, z2.s[1]\n" + "ld1w { z1.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z9.s, z0.s, z7.s[1]\n" + "fmla z13.s, z0.s, z6.s[1]\n" + "fmla z17.s, z0.s, z5.s[1]\n" + "fmla z21.s, z0.s, z4.s[1]\n" + "fmla z25.s, z0.s, z3.s[1]\n" + "fmla z29.s, z0.s, z2.s[1]\n" + "ld1w { z0.s }, p5/Z, [x9, #1, MUL VL]\n" + "fmla z10.s, z1.s, z7.s[1]\n" + "fmla z14.s, z1.s, z6.s[1]\n" + "fmla z18.s, z1.s, z5.s[1]\n" + "fmla z22.s, z1.s, z4.s[1]\n" + "fmla z26.s, z1.s, z3.s[1]\n" + "fmla z30.s, z1.s, z2.s[1]\n" + "ld1w { z1.s }, p5/Z, [x12, #2, MUL VL]\n" + "fmla z11.s, z0.s, z7.s[1]\n" + "fmla z15.s, z0.s, z6.s[1]\n" + "fmla z19.s, z0.s, z5.s[1]\n" + "fmla z23.s, z0.s, z4.s[1]\n" + "fmla z27.s, z0.s, z3.s[1]\n" + "fmla z31.s, z0.s, z2.s[1]\n" + "ld1w { z0.s }, p5/Z, [x11, #2, MUL VL]\n" + "fmla z8.s, z1.s, z7.s[2]\n" + "fmla z12.s, z1.s, z6.s[2]\n" + "fmla z16.s, z1.s, z5.s[2]\n" + "fmla z20.s, z1.s, z4.s[2]\n" + "fmla z24.s, z1.s, z3.s[2]\n" + "fmla z28.s, z1.s, z2.s[2]\n" + "ld1w { z1.s }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.s, z0.s, z7.s[2]\n" + "fmla z13.s, z0.s, z6.s[2]\n" + "fmla z17.s, z0.s, z5.s[2]\n" + "fmla z21.s, z0.s, z4.s[2]\n" + "fmla z25.s, z0.s, z3.s[2]\n" + "fmla z29.s, z0.s, z2.s[2]\n" + "ld1w { z0.s }, p5/Z, [x9, #2, MUL VL]\n" + "fmla z10.s, z1.s, z7.s[2]\n" + "fmla z14.s, z1.s, z6.s[2]\n" + "fmla z18.s, z1.s, z5.s[2]\n" + "fmla z22.s, z1.s, z4.s[2]\n" + "fmla z26.s, z1.s, z3.s[2]\n" + "fmla z30.s, z1.s, z2.s[2]\n" + "ld1w { z1.s }, p5/Z, [x12, #3, MUL VL]\n" "addvl x12, x12, #4\n" - "fmla z11.s, z7.s, z0.s[2]\n" - "fmla z15.s, z7.s, z1.s[2]\n" - "fmla z19.s, z7.s, z2.s[2]\n" - "fmla z23.s, z7.s, z3.s[2]\n" - "fmla z27.s, z7.s, z4.s[2]\n" - "fmla z31.s, z7.s, z5.s[2]\n" - "ld1w { z7.s }, p5/Z, [x11, #3, MUL VL]\n" + "fmla z11.s, z0.s, z7.s[2]\n" + "fmla z15.s, z0.s, z6.s[2]\n" + "fmla z19.s, z0.s, z5.s[2]\n" + "fmla z23.s, z0.s, z4.s[2]\n" + "fmla z27.s, z0.s, z3.s[2]\n" + "fmla z31.s, z0.s, z2.s[2]\n" + "ld1w { z0.s }, p5/Z, [x11, #3, MUL VL]\n" "addvl x11, x11, #4\n" - "fmla z8.s, z6.s, z0.s[3]\n" - "fmla z12.s, z6.s, z1.s[3]\n" - "fmla z16.s, z6.s, z2.s[3]\n" - "fmla z20.s, z6.s, z3.s[3]\n" - "fmla z24.s, z6.s, z4.s[3]\n" - "fmla z28.s, z6.s, z5.s[3]\n" - "ld1w { z6.s }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z8.s, z1.s, z7.s[3]\n" + "fmla z12.s, z1.s, z6.s[3]\n" + "fmla z16.s, z1.s, z5.s[3]\n" + "fmla z20.s, z1.s, z4.s[3]\n" + "fmla z24.s, z1.s, z3.s[3]\n" + "fmla z28.s, z1.s, z2.s[3]\n" + "ld1w { z1.s }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z9.s, z7.s, z0.s[3]\n" - "fmla z13.s, z7.s, z1.s[3]\n" - "fmla z17.s, z7.s, z2.s[3]\n" - "fmla z21.s, z7.s, z3.s[3]\n" - "fmla z25.s, z7.s, z4.s[3]\n" - "fmla z29.s, z7.s, z5.s[3]\n" - "ld1w { z7.s }, p5/Z, [x9, #3, MUL VL]\n" + "fmla z9.s, z0.s, z7.s[3]\n" + "fmla z13.s, z0.s, z6.s[3]\n" + "fmla z17.s, z0.s, z5.s[3]\n" + "fmla z21.s, z0.s, z4.s[3]\n" + "fmla z25.s, z0.s, z3.s[3]\n" + "fmla z29.s, z0.s, z2.s[3]\n" + "ld1w { z0.s }, p5/Z, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "fmla z10.s, z6.s, z0.s[3]\n" - "fmla z14.s, z6.s, z1.s[3]\n" - "fmla z18.s, z6.s, z2.s[3]\n" - "fmla z22.s, z6.s, z3.s[3]\n" - "fmla z26.s, z6.s, z4.s[3]\n" - "fmla z30.s, z6.s, z5.s[3]\n" - "fmla z11.s, z7.s, z0.s[3]\n" - "fmla z15.s, z7.s, z1.s[3]\n" - "fmla z19.s, z7.s, z2.s[3]\n" - "fmla z23.s, z7.s, z3.s[3]\n" - "fmla z27.s, z7.s, z4.s[3]\n" - "fmla z31.s, z7.s, z5.s[3]\n" + "fmla z10.s, z1.s, z7.s[3]\n" + "fmla z14.s, z1.s, z6.s[3]\n" + "fmla z18.s, z1.s, z5.s[3]\n" + "fmla z22.s, z1.s, z4.s[3]\n" + "fmla z26.s, z1.s, z3.s[3]\n" + "fmla z30.s, z1.s, z2.s[3]\n" + "fmla z11.s, z0.s, z7.s[3]\n" + "fmla z15.s, z0.s, z6.s[3]\n" + "fmla z19.s, z0.s, z5.s[3]\n" + "fmla z23.s, z0.s, z4.s[3]\n" + "fmla z27.s, z0.s, z3.s[3]\n" + "fmla z31.s, z0.s, z2.s[3]\n" "bgt 80b\n" "81:" // Height 6: Multiply loop: Single iteration only "whilelt p0.s, XZR, x27\n" @@ -2061,139 +2061,139 @@ void sve_ffhybrid_fp32_mla_6x4VL ( "ld1rqw { z3.s }, p0/Z, [x23]\n" "ld1rqw { z4.s }, p0/Z, [x22]\n" "ld1rqw { z5.s }, p0/Z, [x21]\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z8.s, z6.s, z0.s[0]\n" - "fmla z12.s, z6.s, z1.s[0]\n" - "fmla z16.s, z6.s, z2.s[0]\n" - "fmla z20.s, z6.s, z3.s[0]\n" + "ld1w { z7.s }, p5/Z, [x12]\n" + "ld1w { z6.s }, p5/Z, [x11]\n" + "fmla z8.s, z7.s, z0.s[0]\n" + "fmla z12.s, z7.s, z1.s[0]\n" + "fmla z16.s, z7.s, z2.s[0]\n" + "fmla z20.s, z7.s, z3.s[0]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" - "fmla z24.s, z6.s, z4.s[0]\n" - "fmla z28.s, z6.s, z5.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" + "fmla z24.s, z7.s, z4.s[0]\n" + "fmla z28.s, z7.s, z5.s[0]\n" + "ld1w { z7.s }, p5/Z, [x10]\n" "addvl x10, x10, #1\n" - "fmla z9.s, z7.s, z0.s[0]\n" - "fmla z13.s, z7.s, z1.s[0]\n" - "fmla z17.s, z7.s, z2.s[0]\n" - "fmla z21.s, z7.s, z3.s[0]\n" - "fmla z25.s, z7.s, z4.s[0]\n" - "fmla z29.s, z7.s, z5.s[0]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" + "fmla z9.s, z6.s, z0.s[0]\n" + "fmla z13.s, z6.s, z1.s[0]\n" + "fmla z17.s, z6.s, z2.s[0]\n" + "fmla z21.s, z6.s, z3.s[0]\n" + "fmla z25.s, z6.s, z4.s[0]\n" + "fmla z29.s, z6.s, z5.s[0]\n" + "ld1w { z6.s }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.s, z6.s, z0.s[0]\n" - "fmla z14.s, z6.s, z1.s[0]\n" - "fmla z18.s, z6.s, z2.s[0]\n" - "fmla z22.s, z6.s, z3.s[0]\n" - "fmla z26.s, z6.s, z4.s[0]\n" - "fmla z30.s, z6.s, z5.s[0]\n" - "fmla z11.s, z7.s, z0.s[0]\n" - "fmla z15.s, z7.s, z1.s[0]\n" - "fmla z19.s, z7.s, z2.s[0]\n" - "fmla z23.s, z7.s, z3.s[0]\n" - "fmla z27.s, z7.s, z4.s[0]\n" - "fmla z31.s, z7.s, z5.s[0]\n" + "fmla z10.s, z7.s, z0.s[0]\n" + "fmla z14.s, z7.s, z1.s[0]\n" + "fmla z18.s, z7.s, z2.s[0]\n" + "fmla z22.s, z7.s, z3.s[0]\n" + "fmla z26.s, z7.s, z4.s[0]\n" + "fmla z30.s, z7.s, z5.s[0]\n" + "fmla z11.s, z6.s, z0.s[0]\n" + "fmla z15.s, z6.s, z1.s[0]\n" + "fmla z19.s, z6.s, z2.s[0]\n" + "fmla z23.s, z6.s, z3.s[0]\n" + "fmla z27.s, z6.s, z4.s[0]\n" + "fmla z31.s, z6.s, z5.s[0]\n" "ble 82f\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z8.s, z6.s, z0.s[1]\n" - "fmla z12.s, z6.s, z1.s[1]\n" - "fmla z16.s, z6.s, z2.s[1]\n" - "fmla z20.s, z6.s, z3.s[1]\n" + "ld1w { z7.s }, p5/Z, [x12]\n" + "ld1w { z6.s }, p5/Z, [x11]\n" + "fmla z8.s, z7.s, z0.s[1]\n" + "fmla z12.s, z7.s, z1.s[1]\n" + "fmla z16.s, z7.s, z2.s[1]\n" + "fmla z20.s, z7.s, z3.s[1]\n" "subs x27, x27, #0x1\n" "addvl x12, x12, #1\n" - "fmla z24.s, z6.s, z4.s[1]\n" - "fmla z28.s, z6.s, z5.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" + "fmla z24.s, z7.s, z4.s[1]\n" + "fmla z28.s, z7.s, z5.s[1]\n" + "ld1w { z7.s }, p5/Z, [x10]\n" "addvl x11, x11, #1\n" - "fmla z9.s, z7.s, z0.s[1]\n" - "fmla z13.s, z7.s, z1.s[1]\n" + "fmla z9.s, z6.s, z0.s[1]\n" + "fmla z13.s, z6.s, z1.s[1]\n" "addvl x10, x10, #1\n" - "fmla z17.s, z7.s, z2.s[1]\n" - "fmla z21.s, z7.s, z3.s[1]\n" - "fmla z25.s, z7.s, z4.s[1]\n" - "fmla z29.s, z7.s, z5.s[1]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" + "fmla z17.s, z6.s, z2.s[1]\n" + "fmla z21.s, z6.s, z3.s[1]\n" + "fmla z25.s, z6.s, z4.s[1]\n" + "fmla z29.s, z6.s, z5.s[1]\n" + "ld1w { z6.s }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.s, z6.s, z0.s[1]\n" - "fmla z14.s, z6.s, z1.s[1]\n" - "fmla z18.s, z6.s, z2.s[1]\n" - "fmla z22.s, z6.s, z3.s[1]\n" - "fmla z26.s, z6.s, z4.s[1]\n" - "fmla z30.s, z6.s, z5.s[1]\n" - "fmla z11.s, z7.s, z0.s[1]\n" - "fmla z15.s, z7.s, z1.s[1]\n" - "fmla z19.s, z7.s, z2.s[1]\n" - "fmla z23.s, z7.s, z3.s[1]\n" - "fmla z27.s, z7.s, z4.s[1]\n" - "fmla z31.s, z7.s, z5.s[1]\n" + "fmla z10.s, z7.s, z0.s[1]\n" + "fmla z14.s, z7.s, z1.s[1]\n" + "fmla z18.s, z7.s, z2.s[1]\n" + "fmla z22.s, z7.s, z3.s[1]\n" + "fmla z26.s, z7.s, z4.s[1]\n" + "fmla z30.s, z7.s, z5.s[1]\n" + "fmla z11.s, z6.s, z0.s[1]\n" + "fmla z15.s, z6.s, z1.s[1]\n" + "fmla z19.s, z6.s, z2.s[1]\n" + "fmla z23.s, z6.s, z3.s[1]\n" + "fmla z27.s, z6.s, z4.s[1]\n" + "fmla z31.s, z6.s, z5.s[1]\n" "ble 82f\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z8.s, z6.s, z0.s[2]\n" - "fmla z12.s, z6.s, z1.s[2]\n" - "fmla z16.s, z6.s, z2.s[2]\n" - "fmla z20.s, z6.s, z3.s[2]\n" + "ld1w { z7.s }, p5/Z, [x12]\n" + "ld1w { z6.s }, p5/Z, [x11]\n" + "fmla z8.s, z7.s, z0.s[2]\n" + "fmla z12.s, z7.s, z1.s[2]\n" + "fmla z16.s, z7.s, z2.s[2]\n" + "fmla z20.s, z7.s, z3.s[2]\n" "subs x27, x27, #0x1\n" "addvl x12, x12, #1\n" - "fmla z24.s, z6.s, z4.s[2]\n" - "fmla z28.s, z6.s, z5.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" + "fmla z24.s, z7.s, z4.s[2]\n" + "fmla z28.s, z7.s, z5.s[2]\n" + "ld1w { z7.s }, p5/Z, [x10]\n" "addvl x11, x11, #1\n" - "fmla z9.s, z7.s, z0.s[2]\n" - "fmla z13.s, z7.s, z1.s[2]\n" + "fmla z9.s, z6.s, z0.s[2]\n" + "fmla z13.s, z6.s, z1.s[2]\n" "addvl x10, x10, #1\n" - "fmla z17.s, z7.s, z2.s[2]\n" - "fmla z21.s, z7.s, z3.s[2]\n" - "fmla z25.s, z7.s, z4.s[2]\n" - "fmla z29.s, z7.s, z5.s[2]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" + "fmla z17.s, z6.s, z2.s[2]\n" + "fmla z21.s, z6.s, z3.s[2]\n" + "fmla z25.s, z6.s, z4.s[2]\n" + "fmla z29.s, z6.s, z5.s[2]\n" + "ld1w { z6.s }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.s, z6.s, z0.s[2]\n" - "fmla z14.s, z6.s, z1.s[2]\n" - "fmla z18.s, z6.s, z2.s[2]\n" - "fmla z22.s, z6.s, z3.s[2]\n" - "fmla z26.s, z6.s, z4.s[2]\n" - "fmla z30.s, z6.s, z5.s[2]\n" - "fmla z11.s, z7.s, z0.s[2]\n" - "fmla z15.s, z7.s, z1.s[2]\n" - "fmla z19.s, z7.s, z2.s[2]\n" - "fmla z23.s, z7.s, z3.s[2]\n" - "fmla z27.s, z7.s, z4.s[2]\n" - "fmla z31.s, z7.s, z5.s[2]\n" + "fmla z10.s, z7.s, z0.s[2]\n" + "fmla z14.s, z7.s, z1.s[2]\n" + "fmla z18.s, z7.s, z2.s[2]\n" + "fmla z22.s, z7.s, z3.s[2]\n" + "fmla z26.s, z7.s, z4.s[2]\n" + "fmla z30.s, z7.s, z5.s[2]\n" + "fmla z11.s, z6.s, z0.s[2]\n" + "fmla z15.s, z6.s, z1.s[2]\n" + "fmla z19.s, z6.s, z2.s[2]\n" + "fmla z23.s, z6.s, z3.s[2]\n" + "fmla z27.s, z6.s, z4.s[2]\n" + "fmla z31.s, z6.s, z5.s[2]\n" "ble 82f\n" - "ld1w { z6.s }, p5/Z, [x12]\n" - "ld1w { z7.s }, p5/Z, [x11]\n" - "fmla z8.s, z6.s, z0.s[3]\n" - "fmla z12.s, z6.s, z1.s[3]\n" - "fmla z16.s, z6.s, z2.s[3]\n" - "fmla z20.s, z6.s, z3.s[3]\n" + "ld1w { z7.s }, p5/Z, [x12]\n" + "ld1w { z6.s }, p5/Z, [x11]\n" + "fmla z8.s, z7.s, z0.s[3]\n" + "fmla z12.s, z7.s, z1.s[3]\n" + "fmla z16.s, z7.s, z2.s[3]\n" + "fmla z20.s, z7.s, z3.s[3]\n" "addvl x12, x12, #1\n" "addvl x11, x11, #1\n" - "fmla z24.s, z6.s, z4.s[3]\n" - "fmla z28.s, z6.s, z5.s[3]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" + "fmla z24.s, z7.s, z4.s[3]\n" + "fmla z28.s, z7.s, z5.s[3]\n" + "ld1w { z7.s }, p5/Z, [x10]\n" "addvl x10, x10, #1\n" - "fmla z9.s, z7.s, z0.s[3]\n" - "fmla z13.s, z7.s, z1.s[3]\n" - "fmla z17.s, z7.s, z2.s[3]\n" - "fmla z21.s, z7.s, z3.s[3]\n" - "fmla z25.s, z7.s, z4.s[3]\n" - "fmla z29.s, z7.s, z5.s[3]\n" - "ld1w { z7.s }, p5/Z, [x9]\n" + "fmla z9.s, z6.s, z0.s[3]\n" + "fmla z13.s, z6.s, z1.s[3]\n" + "fmla z17.s, z6.s, z2.s[3]\n" + "fmla z21.s, z6.s, z3.s[3]\n" + "fmla z25.s, z6.s, z4.s[3]\n" + "fmla z29.s, z6.s, z5.s[3]\n" + "ld1w { z6.s }, p5/Z, [x9]\n" "addvl x9, x9, #1\n" - "fmla z10.s, z6.s, z0.s[3]\n" - "fmla z14.s, z6.s, z1.s[3]\n" - "fmla z18.s, z6.s, z2.s[3]\n" - "fmla z22.s, z6.s, z3.s[3]\n" - "fmla z26.s, z6.s, z4.s[3]\n" - "fmla z30.s, z6.s, z5.s[3]\n" - "fmla z11.s, z7.s, z0.s[3]\n" - "fmla z15.s, z7.s, z1.s[3]\n" - "fmla z19.s, z7.s, z2.s[3]\n" - "fmla z23.s, z7.s, z3.s[3]\n" - "fmla z27.s, z7.s, z4.s[3]\n" - "fmla z31.s, z7.s, z5.s[3]\n" + "fmla z10.s, z7.s, z0.s[3]\n" + "fmla z14.s, z7.s, z1.s[3]\n" + "fmla z18.s, z7.s, z2.s[3]\n" + "fmla z22.s, z7.s, z3.s[3]\n" + "fmla z26.s, z7.s, z4.s[3]\n" + "fmla z30.s, z7.s, z5.s[3]\n" + "fmla z11.s, z6.s, z0.s[3]\n" + "fmla z15.s, z6.s, z1.s[3]\n" + "fmla z19.s, z6.s, z2.s[3]\n" + "fmla z23.s, z6.s, z3.s[3]\n" + "fmla z27.s, z6.s, z4.s[3]\n" + "fmla z31.s, z6.s, z5.s[3]\n" "82:" // Height 6: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -2307,4 +2307,4 @@ void sve_ffhybrid_fp32_mla_6x4VL ( } } // namespace arm_gemm -#endif // ARM_COMPUTE_ENABLE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32bf16fp32_mmla_4x6VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32bf16fp32_mmla_4x6VL.hpp index 3ee3e31206466d9adf6c1147653b18b6a49857d6..887d78e1de767f96d65815856c018b79be52ae80 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32bf16fp32_mmla_4x6VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32bf16fp32_mmla_4x6VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,16 +10,16 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once #ifdef ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32bf16fp32_mmla_4x6VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32bf16fp32_mmla_4x6VL/generic.cpp index 36fc9d75ca08ad88962d051bdcc6dd9b2e9a1d3b..57f42cce776aa0544777a7b3a9e6ff68a0332838 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32bf16fp32_mmla_4x6VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffhybrid_fp32bf16fp32_mmla_4x6VL/generic.cpp @@ -174,22 +174,22 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL ( "b 6f\n" "4:" // Height 1: no bias "tbz %x[flags], #0, 5f\n" - "ld1w { z9.s }, p6/Z, [x13]\n" - "ld1w { z10.s }, p5/Z, [x13, #1, MUL VL]\n" - "zip1 z8.d, z9.d, z14.d\n" - "zip2 z14.d, z9.d, z14.d\n" - "ld1w { z11.s }, p4/Z, [x13, #2, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x13, #3, MUL VL]\n" - "zip1 z9.d, z10.d, z15.d\n" - "zip2 z15.d, z10.d, z15.d\n" - "ld1w { z13.s }, p2/Z, [x13, #4, MUL VL]\n" + "ld1w { z21.s }, p6/Z, [x13]\n" + "ld1w { z20.s }, p5/Z, [x13, #1, MUL VL]\n" + "zip1 z8.d, z21.d, z14.d\n" + "zip2 z14.d, z21.d, z14.d\n" + "ld1w { z23.s }, p4/Z, [x13, #2, MUL VL]\n" + "ld1w { z22.s }, p3/Z, [x13, #3, MUL VL]\n" + "zip1 z9.d, z20.d, z15.d\n" + "zip2 z15.d, z20.d, z15.d\n" + "ld1w { z21.s }, p2/Z, [x13, #4, MUL VL]\n" "ld1w { z20.s }, p1/Z, [x13, #5, MUL VL]\n" - "zip1 z10.d, z11.d, z16.d\n" - "zip2 z16.d, z11.d, z16.d\n" - "zip1 z11.d, z12.d, z17.d\n" - "zip2 z17.d, z12.d, z17.d\n" - "zip1 z12.d, z13.d, z18.d\n" - "zip2 z18.d, z13.d, z18.d\n" + "zip1 z10.d, z23.d, z16.d\n" + "zip2 z16.d, z23.d, z16.d\n" + "zip1 z11.d, z22.d, z17.d\n" + "zip2 z17.d, z22.d, z17.d\n" + "zip1 z12.d, z21.d, z18.d\n" + "zip2 z18.d, z21.d, z18.d\n" "zip1 z13.d, z20.d, z19.d\n" "zip2 z19.d, z20.d, z19.d\n" "b 6f\n" @@ -211,11 +211,11 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL ( "7:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 8f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" "cbnz x26, 9f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20, LSL #2\n" @@ -227,35 +227,35 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL ( "ble 11f\n" "10:" // Height 1: Multiply loop: Main loop head "whilelt p0.s, XZR, x25\n" - "ld1rqw { z0.s }, p0/Z, [x24]\n" - ".inst 0x658abc00 // bfcvt z0.h, p7/M, z0.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z4.h }, p7/Z, [x12]\n" - "ld1h { z5.h }, p7/Z, [x12, #1, MUL VL]\n" - ".inst 0x6464e408 // bfmmla z8.s, z0.h, z4.h\n" - ".inst 0x6465e40e // bfmmla z14.s, z0.h, z5.h\n" - "ld1h { z6.h }, p7/Z, [x11]\n" - "ld1h { z7.h }, p7/Z, [x11, #1, MUL VL]\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" - "ld1h { z4.h }, p7/Z, [x10]\n" - "ld1h { z5.h }, p7/Z, [x10, #1, MUL VL]\n" - ".inst 0x6464e40a // bfmmla z10.s, z0.h, z4.h\n" - ".inst 0x6465e410 // bfmmla z16.s, z0.h, z5.h\n" - "ld1h { z6.h }, p7/Z, [x9]\n" - "ld1h { z7.h }, p7/Z, [x9, #1, MUL VL]\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" - ".inst 0x6467e411 // bfmmla z17.s, z0.h, z7.h\n" - "ld1h { z4.h }, p7/Z, [x28]\n" - "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n" + "ld1rqw { z24.s }, p0/Z, [x24]\n" + ".inst 0x658abf18 // bfcvt z24.h, p7/M, z24.s\n" + "uzp1 z24.h, z24.h, z24.h\n" + "ld1h { z21.h }, p7/Z, [x12]\n" + "ld1h { z20.h }, p7/Z, [x12, #1, MUL VL]\n" + ".inst 0x6475e708 // bfmmla z8.s, z24.h, z21.h\n" + ".inst 0x6474e70e // bfmmla z14.s, z24.h, z20.h\n" + "ld1h { z21.h }, p7/Z, [x11]\n" + "ld1h { z20.h }, p7/Z, [x11, #1, MUL VL]\n" + ".inst 0x6475e709 // bfmmla z9.s, z24.h, z21.h\n" + ".inst 0x6474e70f // bfmmla z15.s, z24.h, z20.h\n" + "ld1h { z21.h }, p7/Z, [x10]\n" + "ld1h { z20.h }, p7/Z, [x10, #1, MUL VL]\n" + ".inst 0x6475e70a // bfmmla z10.s, z24.h, z21.h\n" + ".inst 0x6474e710 // bfmmla z16.s, z24.h, z20.h\n" + "ld1h { z21.h }, p7/Z, [x9]\n" + "ld1h { z20.h }, p7/Z, [x9, #1, MUL VL]\n" + ".inst 0x6475e70b // bfmmla z11.s, z24.h, z21.h\n" + ".inst 0x6474e711 // bfmmla z17.s, z24.h, z20.h\n" + "ld1h { z23.h }, p7/Z, [x28]\n" + "ld1h { z22.h }, p7/Z, [x28, #1, MUL VL]\n" "sub x25, x25, #0x4\n" "cmp x25, #0x4\n" - "ld1h { z6.h }, p7/Z, [x27]\n" - "ld1h { z7.h }, p7/Z, [x27, #1, MUL VL]\n" - ".inst 0x6464e40c // bfmmla z12.s, z0.h, z4.h\n" - ".inst 0x6465e412 // bfmmla z18.s, z0.h, z5.h\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6467e413 // bfmmla z19.s, z0.h, z7.h\n" + "ld1h { z21.h }, p7/Z, [x27]\n" + "ld1h { z20.h }, p7/Z, [x27, #1, MUL VL]\n" + ".inst 0x6477e70c // bfmmla z12.s, z24.h, z23.h\n" + ".inst 0x6476e712 // bfmmla z18.s, z24.h, z22.h\n" + ".inst 0x6475e70d // bfmmla z13.s, z24.h, z21.h\n" + ".inst 0x6474e713 // bfmmla z19.s, z24.h, z20.h\n" "add x24, x24, #0x10\n" "addvl x12, x12, #2\n" "addvl x11, x11, #2\n" @@ -266,33 +266,33 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL ( "bgt 10b\n" "11:" // Height 1: Multiply loop: Single iteration only "whilelt p0.s, XZR, x25\n" - "ld1rqw { z0.s }, p0/Z, [x24]\n" - ".inst 0x658abc00 // bfcvt z0.h, p7/M, z0.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z4.h }, p7/Z, [x12]\n" - "ld1h { z5.h }, p7/Z, [x12, #1, MUL VL]\n" - ".inst 0x6464e408 // bfmmla z8.s, z0.h, z4.h\n" - ".inst 0x6465e40e // bfmmla z14.s, z0.h, z5.h\n" - "ld1h { z6.h }, p7/Z, [x11]\n" - "ld1h { z7.h }, p7/Z, [x11, #1, MUL VL]\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" - "ld1h { z4.h }, p7/Z, [x10]\n" - "ld1h { z5.h }, p7/Z, [x10, #1, MUL VL]\n" - ".inst 0x6464e40a // bfmmla z10.s, z0.h, z4.h\n" - ".inst 0x6465e410 // bfmmla z16.s, z0.h, z5.h\n" - "ld1h { z6.h }, p7/Z, [x9]\n" - "ld1h { z7.h }, p7/Z, [x9, #1, MUL VL]\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" - ".inst 0x6467e411 // bfmmla z17.s, z0.h, z7.h\n" - "ld1h { z4.h }, p7/Z, [x28]\n" - "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n" - ".inst 0x6464e40c // bfmmla z12.s, z0.h, z4.h\n" - ".inst 0x6465e412 // bfmmla z18.s, z0.h, z5.h\n" - "ld1h { z6.h }, p7/Z, [x27]\n" - "ld1h { z7.h }, p7/Z, [x27, #1, MUL VL]\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6467e413 // bfmmla z19.s, z0.h, z7.h\n" + "ld1rqw { z22.s }, p0/Z, [x24]\n" + ".inst 0x658abed6 // bfcvt z22.h, p7/M, z22.s\n" + "uzp1 z22.h, z22.h, z22.h\n" + "ld1h { z21.h }, p7/Z, [x12]\n" + "ld1h { z20.h }, p7/Z, [x12, #1, MUL VL]\n" + ".inst 0x6475e6c8 // bfmmla z8.s, z22.h, z21.h\n" + ".inst 0x6474e6ce // bfmmla z14.s, z22.h, z20.h\n" + "ld1h { z21.h }, p7/Z, [x11]\n" + "ld1h { z20.h }, p7/Z, [x11, #1, MUL VL]\n" + ".inst 0x6475e6c9 // bfmmla z9.s, z22.h, z21.h\n" + ".inst 0x6474e6cf // bfmmla z15.s, z22.h, z20.h\n" + "ld1h { z21.h }, p7/Z, [x10]\n" + "ld1h { z20.h }, p7/Z, [x10, #1, MUL VL]\n" + ".inst 0x6475e6ca // bfmmla z10.s, z22.h, z21.h\n" + ".inst 0x6474e6d0 // bfmmla z16.s, z22.h, z20.h\n" + "ld1h { z21.h }, p7/Z, [x9]\n" + "ld1h { z20.h }, p7/Z, [x9, #1, MUL VL]\n" + ".inst 0x6475e6cb // bfmmla z11.s, z22.h, z21.h\n" + ".inst 0x6474e6d1 // bfmmla z17.s, z22.h, z20.h\n" + "ld1h { z21.h }, p7/Z, [x28]\n" + "ld1h { z20.h }, p7/Z, [x28, #1, MUL VL]\n" + ".inst 0x6475e6cc // bfmmla z12.s, z22.h, z21.h\n" + ".inst 0x6474e6d2 // bfmmla z18.s, z22.h, z20.h\n" + "ld1h { z21.h }, p7/Z, [x27]\n" + "ld1h { z20.h }, p7/Z, [x27, #1, MUL VL]\n" + ".inst 0x6475e6cd // bfmmla z13.s, z22.h, z21.h\n" + ".inst 0x6474e6d3 // bfmmla z19.s, z22.h, z20.h\n" "addvl x12, x12, #2\n" "addvl x11, x11, #2\n" "addvl x10, x10, #2\n" @@ -312,21 +312,21 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL ( "uzp1 z13.d, z13.d, z19.d\n" "tbz %x[flags], #1, 13f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p7/Z, [x20]\n" + "ld1rw { z21.s }, p7/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p7/Z, [x20]\n" - "fmin z8.s, p7/M, z8.s, z1.s\n" - "fmin z9.s, p7/M, z9.s, z1.s\n" - "fmin z10.s, p7/M, z10.s, z1.s\n" - "fmin z11.s, p7/M, z11.s, z1.s\n" - "fmin z12.s, p7/M, z12.s, z1.s\n" - "fmin z13.s, p7/M, z13.s, z1.s\n" - "fmax z8.s, p7/M, z8.s, z0.s\n" - "fmax z9.s, p7/M, z9.s, z0.s\n" - "fmax z10.s, p7/M, z10.s, z0.s\n" - "fmax z11.s, p7/M, z11.s, z0.s\n" - "fmax z12.s, p7/M, z12.s, z0.s\n" - "fmax z13.s, p7/M, z13.s, z0.s\n" + "ld1rw { z20.s }, p7/Z, [x20]\n" + "fmin z8.s, p7/M, z8.s, z21.s\n" + "fmin z9.s, p7/M, z9.s, z21.s\n" + "fmin z10.s, p7/M, z10.s, z21.s\n" + "fmin z11.s, p7/M, z11.s, z21.s\n" + "fmin z12.s, p7/M, z12.s, z21.s\n" + "fmin z13.s, p7/M, z13.s, z21.s\n" + "fmax z8.s, p7/M, z8.s, z20.s\n" + "fmax z9.s, p7/M, z9.s, z20.s\n" + "fmax z10.s, p7/M, z10.s, z20.s\n" + "fmax z11.s, p7/M, z11.s, z20.s\n" + "fmax z12.s, p7/M, z12.s, z20.s\n" + "fmax z13.s, p7/M, z13.s, z20.s\n" "13:" // Height 1: No activation "st1w { z8.s }, p6, [x13]\n" "st1w { z9.s }, p5, [x13, #1, MUL VL]\n" @@ -413,29 +413,29 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL ( "18:" // Height 2: no bias "tbz %x[flags], #0, 19f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x23, x13, x20, LSL #2\n" - "ld1w { z9.s }, p6/Z, [x13]\n" - "ld1w { z10.s }, p5/Z, [x13, #1, MUL VL]\n" - "ld1w { z11.s }, p4/Z, [x13, #2, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x13, #3, MUL VL]\n" - "ld1w { z13.s }, p2/Z, [x13, #4, MUL VL]\n" + "add x20, x13, x20, LSL #2\n" + "ld1w { z16.s }, p6/Z, [x13]\n" + "ld1w { z17.s }, p5/Z, [x13, #1, MUL VL]\n" + "ld1w { z19.s }, p4/Z, [x13, #2, MUL VL]\n" + "ld1w { z22.s }, p3/Z, [x13, #3, MUL VL]\n" + "ld1w { z21.s }, p2/Z, [x13, #4, MUL VL]\n" "ld1w { z20.s }, p1/Z, [x13, #5, MUL VL]\n" - "ld1w { z14.s }, p6/Z, [x23]\n" - "zip1 z8.d, z9.d, z14.d\n" - "zip2 z14.d, z9.d, z14.d\n" - "ld1w { z15.s }, p5/Z, [x23, #1, MUL VL]\n" - "ld1w { z16.s }, p4/Z, [x23, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z15.d\n" - "zip2 z15.d, z10.d, z15.d\n" - "ld1w { z17.s }, p3/Z, [x23, #3, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x23, #4, MUL VL]\n" - "zip1 z10.d, z11.d, z16.d\n" - "zip2 z16.d, z11.d, z16.d\n" - "ld1w { z19.s }, p1/Z, [x23, #5, MUL VL]\n" - "zip1 z11.d, z12.d, z17.d\n" - "zip2 z17.d, z12.d, z17.d\n" - "zip1 z12.d, z13.d, z18.d\n" - "zip2 z18.d, z13.d, z18.d\n" + "ld1w { z14.s }, p6/Z, [x20]\n" + "zip1 z8.d, z16.d, z14.d\n" + "zip2 z14.d, z16.d, z14.d\n" + "ld1w { z15.s }, p5/Z, [x20, #1, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x20, #2, MUL VL]\n" + "zip1 z9.d, z17.d, z15.d\n" + "zip2 z15.d, z17.d, z15.d\n" + "ld1w { z17.s }, p3/Z, [x20, #3, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x20, #4, MUL VL]\n" + "zip1 z10.d, z19.d, z16.d\n" + "zip2 z16.d, z19.d, z16.d\n" + "ld1w { z19.s }, p1/Z, [x20, #5, MUL VL]\n" + "zip1 z11.d, z22.d, z17.d\n" + "zip2 z17.d, z22.d, z17.d\n" + "zip1 z12.d, z21.d, z18.d\n" + "zip2 z18.d, z21.d, z18.d\n" "zip1 z13.d, z20.d, z19.d\n" "zip2 z19.d, z20.d, z19.d\n" "b 20f\n" @@ -457,12 +457,12 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL ( "21:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 22f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" "cbnz x26, 23f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20, LSL #2\n" @@ -470,45 +470,45 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL ( "b 23f\n" "22:" // Height 2: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20, LSL #2\n" + "add x23, x24, x21, LSL #2\n" "23:" // Height 2: input setup done "cmp x25, #0x4\n" "ble 25f\n" "24:" // Height 2: Multiply loop: Main loop head "whilelt p0.s, XZR, x25\n" - "ld1rqw { z0.s }, p0/Z, [x24]\n" - "ld1rqw { z1.s }, p0/Z, [x23]\n" - ".inst 0x658abc00 // bfcvt z0.h, p7/M, z0.s\n" - ".inst 0x658abc21 // bfcvt z1.h, p7/M, z1.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z4.h }, p7/Z, [x12]\n" - "ld1h { z5.h }, p7/Z, [x12, #1, MUL VL]\n" - "uzp1 z1.h, z1.h, z1.h\n" - "trn1 z0.d, z0.d, z1.d\n" - "ld1h { z6.h }, p7/Z, [x11]\n" - "ld1h { z7.h }, p7/Z, [x11, #1, MUL VL]\n" - ".inst 0x6464e408 // bfmmla z8.s, z0.h, z4.h\n" - ".inst 0x6465e40e // bfmmla z14.s, z0.h, z5.h\n" - "ld1h { z4.h }, p7/Z, [x10]\n" - "ld1h { z5.h }, p7/Z, [x10, #1, MUL VL]\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" - "ld1h { z6.h }, p7/Z, [x9]\n" - "ld1h { z7.h }, p7/Z, [x9, #1, MUL VL]\n" - ".inst 0x6464e40a // bfmmla z10.s, z0.h, z4.h\n" - ".inst 0x6465e410 // bfmmla z16.s, z0.h, z5.h\n" - "ld1h { z4.h }, p7/Z, [x28]\n" - "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" - ".inst 0x6467e411 // bfmmla z17.s, z0.h, z7.h\n" - "ld1h { z6.h }, p7/Z, [x27]\n" - "ld1h { z7.h }, p7/Z, [x27, #1, MUL VL]\n" + "ld1rqw { z24.s }, p0/Z, [x24]\n" + "ld1rqw { z20.s }, p0/Z, [x23]\n" + ".inst 0x658abf18 // bfcvt z24.h, p7/M, z24.s\n" + ".inst 0x658abe94 // bfcvt z20.h, p7/M, z20.s\n" + "uzp1 z24.h, z24.h, z24.h\n" + "ld1h { z23.h }, p7/Z, [x12]\n" + "ld1h { z22.h }, p7/Z, [x12, #1, MUL VL]\n" + "uzp1 z20.h, z20.h, z20.h\n" + "trn1 z24.d, z24.d, z20.d\n" + "ld1h { z21.h }, p7/Z, [x11]\n" + "ld1h { z20.h }, p7/Z, [x11, #1, MUL VL]\n" + ".inst 0x6477e708 // bfmmla z8.s, z24.h, z23.h\n" + ".inst 0x6476e70e // bfmmla z14.s, z24.h, z22.h\n" + "ld1h { z23.h }, p7/Z, [x10]\n" + "ld1h { z22.h }, p7/Z, [x10, #1, MUL VL]\n" + ".inst 0x6475e709 // bfmmla z9.s, z24.h, z21.h\n" + ".inst 0x6474e70f // bfmmla z15.s, z24.h, z20.h\n" + "ld1h { z21.h }, p7/Z, [x9]\n" + "ld1h { z20.h }, p7/Z, [x9, #1, MUL VL]\n" + ".inst 0x6477e70a // bfmmla z10.s, z24.h, z23.h\n" + ".inst 0x6476e710 // bfmmla z16.s, z24.h, z22.h\n" + "ld1h { z23.h }, p7/Z, [x28]\n" + "ld1h { z22.h }, p7/Z, [x28, #1, MUL VL]\n" + ".inst 0x6475e70b // bfmmla z11.s, z24.h, z21.h\n" + ".inst 0x6474e711 // bfmmla z17.s, z24.h, z20.h\n" + "ld1h { z21.h }, p7/Z, [x27]\n" + "ld1h { z20.h }, p7/Z, [x27, #1, MUL VL]\n" "sub x25, x25, #0x4\n" "cmp x25, #0x4\n" - ".inst 0x6464e40c // bfmmla z12.s, z0.h, z4.h\n" - ".inst 0x6465e412 // bfmmla z18.s, z0.h, z5.h\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6467e413 // bfmmla z19.s, z0.h, z7.h\n" + ".inst 0x6477e70c // bfmmla z12.s, z24.h, z23.h\n" + ".inst 0x6476e712 // bfmmla z18.s, z24.h, z22.h\n" + ".inst 0x6475e70d // bfmmla z13.s, z24.h, z21.h\n" + ".inst 0x6474e713 // bfmmla z19.s, z24.h, z20.h\n" "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" "addvl x12, x12, #2\n" @@ -520,39 +520,39 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL ( "bgt 24b\n" "25:" // Height 2: Multiply loop: Single iteration only "whilelt p0.s, XZR, x25\n" - "ld1rqw { z0.s }, p0/Z, [x24]\n" - "ld1rqw { z1.s }, p0/Z, [x23]\n" - ".inst 0x658abc00 // bfcvt z0.h, p7/M, z0.s\n" - ".inst 0x658abc21 // bfcvt z1.h, p7/M, z1.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z4.h }, p7/Z, [x12]\n" - "ld1h { z5.h }, p7/Z, [x12, #1, MUL VL]\n" - "uzp1 z1.h, z1.h, z1.h\n" - "trn1 z0.d, z0.d, z1.d\n" - "ld1h { z6.h }, p7/Z, [x11]\n" - "ld1h { z7.h }, p7/Z, [x11, #1, MUL VL]\n" - ".inst 0x6464e408 // bfmmla z8.s, z0.h, z4.h\n" - ".inst 0x6465e40e // bfmmla z14.s, z0.h, z5.h\n" - "ld1h { z4.h }, p7/Z, [x10]\n" - "ld1h { z5.h }, p7/Z, [x10, #1, MUL VL]\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" - "ld1h { z6.h }, p7/Z, [x9]\n" - "ld1h { z7.h }, p7/Z, [x9, #1, MUL VL]\n" - ".inst 0x6464e40a // bfmmla z10.s, z0.h, z4.h\n" - ".inst 0x6465e410 // bfmmla z16.s, z0.h, z5.h\n" - "ld1h { z4.h }, p7/Z, [x28]\n" - "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" - ".inst 0x6467e411 // bfmmla z17.s, z0.h, z7.h\n" - "ld1h { z6.h }, p7/Z, [x27]\n" - "ld1h { z7.h }, p7/Z, [x27, #1, MUL VL]\n" - ".inst 0x6464e40c // bfmmla z12.s, z0.h, z4.h\n" - ".inst 0x6465e412 // bfmmla z18.s, z0.h, z5.h\n" + "ld1rqw { z24.s }, p0/Z, [x24]\n" + "ld1rqw { z20.s }, p0/Z, [x23]\n" + ".inst 0x658abf18 // bfcvt z24.h, p7/M, z24.s\n" + ".inst 0x658abe94 // bfcvt z20.h, p7/M, z20.s\n" + "uzp1 z24.h, z24.h, z24.h\n" + "ld1h { z23.h }, p7/Z, [x12]\n" + "ld1h { z22.h }, p7/Z, [x12, #1, MUL VL]\n" + "uzp1 z20.h, z20.h, z20.h\n" + "trn1 z24.d, z24.d, z20.d\n" + "ld1h { z21.h }, p7/Z, [x11]\n" + "ld1h { z20.h }, p7/Z, [x11, #1, MUL VL]\n" + ".inst 0x6477e708 // bfmmla z8.s, z24.h, z23.h\n" + ".inst 0x6476e70e // bfmmla z14.s, z24.h, z22.h\n" + "ld1h { z23.h }, p7/Z, [x10]\n" + "ld1h { z22.h }, p7/Z, [x10, #1, MUL VL]\n" + ".inst 0x6475e709 // bfmmla z9.s, z24.h, z21.h\n" + ".inst 0x6474e70f // bfmmla z15.s, z24.h, z20.h\n" + "ld1h { z21.h }, p7/Z, [x9]\n" + "ld1h { z20.h }, p7/Z, [x9, #1, MUL VL]\n" + ".inst 0x6477e70a // bfmmla z10.s, z24.h, z23.h\n" + ".inst 0x6476e710 // bfmmla z16.s, z24.h, z22.h\n" + "ld1h { z23.h }, p7/Z, [x28]\n" + "ld1h { z22.h }, p7/Z, [x28, #1, MUL VL]\n" + ".inst 0x6475e70b // bfmmla z11.s, z24.h, z21.h\n" + ".inst 0x6474e711 // bfmmla z17.s, z24.h, z20.h\n" + "ld1h { z21.h }, p7/Z, [x27]\n" + "ld1h { z20.h }, p7/Z, [x27, #1, MUL VL]\n" + ".inst 0x6477e70c // bfmmla z12.s, z24.h, z23.h\n" + ".inst 0x6476e712 // bfmmla z18.s, z24.h, z22.h\n" "addvl x12, x12, #2\n" "addvl x11, x11, #2\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6467e413 // bfmmla z19.s, z0.h, z7.h\n" + ".inst 0x6475e70d // bfmmla z13.s, z24.h, z21.h\n" + ".inst 0x6474e713 // bfmmla z19.s, z24.h, z20.h\n" "addvl x10, x10, #2\n" "addvl x9, x9, #2\n" "addvl x28, x28, #2\n" @@ -578,33 +578,33 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL ( "uzp2 z13.d, z13.d, z19.d\n" "tbz %x[flags], #1, 27f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p7/Z, [x20]\n" + "ld1rw { z20.s }, p7/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p7/Z, [x20]\n" - "fmin z4.s, p7/M, z4.s, z1.s\n" - "fmin z14.s, p7/M, z14.s, z1.s\n" - "fmin z15.s, p7/M, z15.s, z1.s\n" - "fmin z16.s, p7/M, z16.s, z1.s\n" - "fmin z17.s, p7/M, z17.s, z1.s\n" - "fmin z18.s, p7/M, z18.s, z1.s\n" - "fmin z8.s, p7/M, z8.s, z1.s\n" - "fmin z9.s, p7/M, z9.s, z1.s\n" - "fmin z10.s, p7/M, z10.s, z1.s\n" - "fmin z11.s, p7/M, z11.s, z1.s\n" - "fmin z12.s, p7/M, z12.s, z1.s\n" - "fmin z13.s, p7/M, z13.s, z1.s\n" - "fmax z4.s, p7/M, z4.s, z0.s\n" - "fmax z14.s, p7/M, z14.s, z0.s\n" - "fmax z15.s, p7/M, z15.s, z0.s\n" - "fmax z16.s, p7/M, z16.s, z0.s\n" - "fmax z17.s, p7/M, z17.s, z0.s\n" - "fmax z18.s, p7/M, z18.s, z0.s\n" - "fmax z8.s, p7/M, z8.s, z0.s\n" - "fmax z9.s, p7/M, z9.s, z0.s\n" - "fmax z10.s, p7/M, z10.s, z0.s\n" - "fmax z11.s, p7/M, z11.s, z0.s\n" - "fmax z12.s, p7/M, z12.s, z0.s\n" - "fmax z13.s, p7/M, z13.s, z0.s\n" + "ld1rw { z19.s }, p7/Z, [x20]\n" + "fmin z4.s, p7/M, z4.s, z20.s\n" + "fmin z14.s, p7/M, z14.s, z20.s\n" + "fmin z15.s, p7/M, z15.s, z20.s\n" + "fmin z16.s, p7/M, z16.s, z20.s\n" + "fmin z17.s, p7/M, z17.s, z20.s\n" + "fmin z18.s, p7/M, z18.s, z20.s\n" + "fmin z8.s, p7/M, z8.s, z20.s\n" + "fmin z9.s, p7/M, z9.s, z20.s\n" + "fmin z10.s, p7/M, z10.s, z20.s\n" + "fmin z11.s, p7/M, z11.s, z20.s\n" + "fmin z12.s, p7/M, z12.s, z20.s\n" + "fmin z13.s, p7/M, z13.s, z20.s\n" + "fmax z4.s, p7/M, z4.s, z19.s\n" + "fmax z14.s, p7/M, z14.s, z19.s\n" + "fmax z15.s, p7/M, z15.s, z19.s\n" + "fmax z16.s, p7/M, z16.s, z19.s\n" + "fmax z17.s, p7/M, z17.s, z19.s\n" + "fmax z18.s, p7/M, z18.s, z19.s\n" + "fmax z8.s, p7/M, z8.s, z19.s\n" + "fmax z9.s, p7/M, z9.s, z19.s\n" + "fmax z10.s, p7/M, z10.s, z19.s\n" + "fmax z11.s, p7/M, z11.s, z19.s\n" + "fmax z12.s, p7/M, z12.s, z19.s\n" + "fmax z13.s, p7/M, z13.s, z19.s\n" "27:" // Height 2: No activation "st1w { z4.s }, p6, [x13]\n" "st1w { z14.s }, p5, [x13, #1, MUL VL]\n" @@ -709,38 +709,38 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL ( "32:" // Height 3: no bias "tbz %x[flags], #0, 33f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x23, x13, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "ld1w { z9.s }, p6/Z, [x13]\n" - "ld1w { z10.s }, p5/Z, [x13, #1, MUL VL]\n" - "ld1w { z11.s }, p4/Z, [x13, #2, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x13, #3, MUL VL]\n" - "ld1w { z13.s }, p2/Z, [x13, #4, MUL VL]\n" + "add x21, x13, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" + "ld1w { z16.s }, p6/Z, [x13]\n" + "ld1w { z17.s }, p5/Z, [x13, #1, MUL VL]\n" + "ld1w { z19.s }, p4/Z, [x13, #2, MUL VL]\n" + "ld1w { z22.s }, p3/Z, [x13, #3, MUL VL]\n" + "ld1w { z24.s }, p2/Z, [x13, #4, MUL VL]\n" "ld1w { z20.s }, p1/Z, [x13, #5, MUL VL]\n" - "ld1w { z14.s }, p6/Z, [x23]\n" - "zip1 z8.d, z9.d, z14.d\n" - "zip2 z14.d, z9.d, z14.d\n" - "ld1w { z15.s }, p5/Z, [x23, #1, MUL VL]\n" - "ld1w { z16.s }, p4/Z, [x23, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z15.d\n" - "zip2 z15.d, z10.d, z15.d\n" - "ld1w { z17.s }, p3/Z, [x23, #3, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x23, #4, MUL VL]\n" - "zip1 z10.d, z11.d, z16.d\n" - "zip2 z16.d, z11.d, z16.d\n" - "ld1w { z19.s }, p1/Z, [x23, #5, MUL VL]\n" - "ld1w { z21.s }, p6/Z, [x22]\n" - "zip1 z11.d, z12.d, z17.d\n" - "zip2 z17.d, z12.d, z17.d\n" - "ld1w { z22.s }, p5/Z, [x22, #1, MUL VL]\n" - "ld1w { z23.s }, p4/Z, [x22, #2, MUL VL]\n" - "zip1 z12.d, z13.d, z18.d\n" - "zip2 z18.d, z13.d, z18.d\n" - "ld1w { z24.s }, p3/Z, [x22, #3, MUL VL]\n" - "ld1w { z25.s }, p2/Z, [x22, #4, MUL VL]\n" + "ld1w { z14.s }, p6/Z, [x21]\n" + "zip1 z8.d, z16.d, z14.d\n" + "zip2 z14.d, z16.d, z14.d\n" + "ld1w { z15.s }, p5/Z, [x21, #1, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x21, #2, MUL VL]\n" + "zip1 z9.d, z17.d, z15.d\n" + "zip2 z15.d, z17.d, z15.d\n" + "ld1w { z17.s }, p3/Z, [x21, #3, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x21, #4, MUL VL]\n" + "zip1 z10.d, z19.d, z16.d\n" + "zip2 z16.d, z19.d, z16.d\n" + "ld1w { z19.s }, p1/Z, [x21, #5, MUL VL]\n" + "ld1w { z21.s }, p6/Z, [x20]\n" + "zip1 z11.d, z22.d, z17.d\n" + "zip2 z17.d, z22.d, z17.d\n" + "ld1w { z22.s }, p5/Z, [x20, #1, MUL VL]\n" + "ld1w { z23.s }, p4/Z, [x20, #2, MUL VL]\n" + "zip1 z12.d, z24.d, z18.d\n" + "zip2 z18.d, z24.d, z18.d\n" + "ld1w { z24.s }, p3/Z, [x20, #3, MUL VL]\n" + "ld1w { z25.s }, p2/Z, [x20, #4, MUL VL]\n" "zip1 z13.d, z20.d, z19.d\n" "zip2 z19.d, z20.d, z19.d\n" - "ld1w { z4.s }, p1/Z, [x22, #5, MUL VL]\n" + "ld1w { z0.s }, p1/Z, [x20, #5, MUL VL]\n" "zip1 z20.d, z21.d, z26.d\n" "zip2 z26.d, z21.d, z26.d\n" "zip1 z21.d, z22.d, z27.d\n" @@ -751,8 +751,8 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL ( "zip2 z29.d, z24.d, z29.d\n" "zip1 z24.d, z25.d, z30.d\n" "zip2 z30.d, z25.d, z30.d\n" - "zip1 z25.d, z4.d, z31.d\n" - "zip2 z31.d, z4.d, z31.d\n" + "zip1 z25.d, z0.d, z31.d\n" + "zip2 z31.d, z0.d, z31.d\n" "b 34f\n" "33:" // Height 3: no accumulate "mov z8.b, #0x0\n" @@ -784,13 +784,13 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL ( "35:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 36f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" "cbnz x26, 37f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20, LSL #2\n" @@ -799,125 +799,125 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL ( "b 37f\n" "36:" // Height 3: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" "37:" // Height 3: input setup done "cmp x25, #0x4\n" "ble 39f\n" "38:" // Height 3: Multiply loop: Main loop head "whilelt p0.s, XZR, x25\n" - "ld1rqw { z0.s }, p0/Z, [x24]\n" - "ld1rqw { z1.s }, p0/Z, [x23]\n" + "ld1rqw { z5.s }, p0/Z, [x24]\n" + "ld1rqw { z0.s }, p0/Z, [x23]\n" + ".inst 0x658abca5 // bfcvt z5.h, p7/M, z5.s\n" + "ld1rqw { z4.s }, p0/Z, [x22]\n" ".inst 0x658abc00 // bfcvt z0.h, p7/M, z0.s\n" - "ld1rqw { z2.s }, p0/Z, [x22]\n" - ".inst 0x658abc21 // bfcvt z1.h, p7/M, z1.s\n" + "uzp1 z5.h, z5.h, z5.h\n" + "ld1h { z3.h }, p7/Z, [x12]\n" "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z4.h }, p7/Z, [x12]\n" - "uzp1 z1.h, z1.h, z1.h\n" - ".inst 0x658abc42 // bfcvt z2.h, p7/M, z2.s\n" - "ld1h { z5.h }, p7/Z, [x12, #1, MUL VL]\n" - "ld1h { z6.h }, p7/Z, [x11]\n" - "trn1 z0.d, z0.d, z1.d\n" - "uzp1 z2.h, z2.h, z2.h\n" - "ld1h { z7.h }, p7/Z, [x11, #1, MUL VL]\n" - ".inst 0x6464e408 // bfmmla z8.s, z0.h, z4.h\n" - ".inst 0x6464e454 // bfmmla z20.s, z2.h, z4.h\n" - ".inst 0x6465e40e // bfmmla z14.s, z0.h, z5.h\n" - "ld1h { z4.h }, p7/Z, [x10]\n" + ".inst 0x658abc84 // bfcvt z4.h, p7/M, z4.s\n" + "ld1h { z2.h }, p7/Z, [x12, #1, MUL VL]\n" + "ld1h { z1.h }, p7/Z, [x11]\n" + "trn1 z5.d, z5.d, z0.d\n" + "uzp1 z4.h, z4.h, z4.h\n" + "ld1h { z0.h }, p7/Z, [x11, #1, MUL VL]\n" + ".inst 0x6463e4a8 // bfmmla z8.s, z5.h, z3.h\n" + ".inst 0x6463e494 // bfmmla z20.s, z4.h, z3.h\n" + ".inst 0x6462e4ae // bfmmla z14.s, z5.h, z2.h\n" + "ld1h { z3.h }, p7/Z, [x10]\n" "sub x25, x25, #0x4\n" - ".inst 0x6465e45a // bfmmla z26.s, z2.h, z5.h\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - "ld1h { z5.h }, p7/Z, [x10, #1, MUL VL]\n" + ".inst 0x6462e49a // bfmmla z26.s, z4.h, z2.h\n" + ".inst 0x6461e4a9 // bfmmla z9.s, z5.h, z1.h\n" + "ld1h { z2.h }, p7/Z, [x10, #1, MUL VL]\n" "cmp x25, #0x4\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" - "ld1h { z6.h }, p7/Z, [x9]\n" + ".inst 0x6461e495 // bfmmla z21.s, z4.h, z1.h\n" + ".inst 0x6460e4af // bfmmla z15.s, z5.h, z0.h\n" + "ld1h { z1.h }, p7/Z, [x9]\n" "add x24, x24, #0x10\n" - ".inst 0x6467e45b // bfmmla z27.s, z2.h, z7.h\n" - "ld1h { z7.h }, p7/Z, [x9, #1, MUL VL]\n" - ".inst 0x6464e40a // bfmmla z10.s, z0.h, z4.h\n" + ".inst 0x6460e49b // bfmmla z27.s, z4.h, z0.h\n" + "ld1h { z0.h }, p7/Z, [x9, #1, MUL VL]\n" + ".inst 0x6463e4aa // bfmmla z10.s, z5.h, z3.h\n" "add x23, x23, #0x10\n" - ".inst 0x6464e456 // bfmmla z22.s, z2.h, z4.h\n" - ".inst 0x6465e410 // bfmmla z16.s, z0.h, z5.h\n" - "ld1h { z4.h }, p7/Z, [x28]\n" + ".inst 0x6463e496 // bfmmla z22.s, z4.h, z3.h\n" + ".inst 0x6462e4b0 // bfmmla z16.s, z5.h, z2.h\n" + "ld1h { z3.h }, p7/Z, [x28]\n" "add x22, x22, #0x10\n" - ".inst 0x6465e45c // bfmmla z28.s, z2.h, z5.h\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" - "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n" + ".inst 0x6462e49c // bfmmla z28.s, z4.h, z2.h\n" + ".inst 0x6461e4ab // bfmmla z11.s, z5.h, z1.h\n" + "ld1h { z2.h }, p7/Z, [x28, #1, MUL VL]\n" "addvl x12, x12, #2\n" - ".inst 0x6466e457 // bfmmla z23.s, z2.h, z6.h\n" - ".inst 0x6467e411 // bfmmla z17.s, z0.h, z7.h\n" - "ld1h { z6.h }, p7/Z, [x27]\n" + ".inst 0x6461e497 // bfmmla z23.s, z4.h, z1.h\n" + ".inst 0x6460e4b1 // bfmmla z17.s, z5.h, z0.h\n" + "ld1h { z1.h }, p7/Z, [x27]\n" "addvl x11, x11, #2\n" - ".inst 0x6467e45d // bfmmla z29.s, z2.h, z7.h\n" - "ld1h { z7.h }, p7/Z, [x27, #1, MUL VL]\n" - ".inst 0x6464e40c // bfmmla z12.s, z0.h, z4.h\n" + ".inst 0x6460e49d // bfmmla z29.s, z4.h, z0.h\n" + "ld1h { z0.h }, p7/Z, [x27, #1, MUL VL]\n" + ".inst 0x6463e4ac // bfmmla z12.s, z5.h, z3.h\n" "addvl x10, x10, #2\n" - ".inst 0x6464e458 // bfmmla z24.s, z2.h, z4.h\n" - ".inst 0x6465e412 // bfmmla z18.s, z0.h, z5.h\n" + ".inst 0x6463e498 // bfmmla z24.s, z4.h, z3.h\n" + ".inst 0x6462e4b2 // bfmmla z18.s, z5.h, z2.h\n" "addvl x9, x9, #2\n" "addvl x28, x28, #2\n" - ".inst 0x6465e45e // bfmmla z30.s, z2.h, z5.h\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" + ".inst 0x6462e49e // bfmmla z30.s, z4.h, z2.h\n" + ".inst 0x6461e4ad // bfmmla z13.s, z5.h, z1.h\n" "addvl x27, x27, #2\n" - ".inst 0x6466e459 // bfmmla z25.s, z2.h, z6.h\n" - ".inst 0x6467e413 // bfmmla z19.s, z0.h, z7.h\n" - ".inst 0x6467e45f // bfmmla z31.s, z2.h, z7.h\n" + ".inst 0x6461e499 // bfmmla z25.s, z4.h, z1.h\n" + ".inst 0x6460e4b3 // bfmmla z19.s, z5.h, z0.h\n" + ".inst 0x6460e49f // bfmmla z31.s, z4.h, z0.h\n" "bgt 38b\n" "39:" // Height 3: Multiply loop: Single iteration only "whilelt p0.s, XZR, x25\n" - "ld1rqw { z0.s }, p0/Z, [x24]\n" - "ld1rqw { z1.s }, p0/Z, [x23]\n" + "ld1rqw { z5.s }, p0/Z, [x24]\n" + "ld1rqw { z0.s }, p0/Z, [x23]\n" + ".inst 0x658abca5 // bfcvt z5.h, p7/M, z5.s\n" + "ld1rqw { z4.s }, p0/Z, [x22]\n" ".inst 0x658abc00 // bfcvt z0.h, p7/M, z0.s\n" - "ld1rqw { z2.s }, p0/Z, [x22]\n" - ".inst 0x658abc21 // bfcvt z1.h, p7/M, z1.s\n" + "uzp1 z5.h, z5.h, z5.h\n" + "ld1h { z3.h }, p7/Z, [x12]\n" "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z4.h }, p7/Z, [x12]\n" - "uzp1 z1.h, z1.h, z1.h\n" - ".inst 0x658abc42 // bfcvt z2.h, p7/M, z2.s\n" - "ld1h { z5.h }, p7/Z, [x12, #1, MUL VL]\n" - "ld1h { z6.h }, p7/Z, [x11]\n" - "trn1 z0.d, z0.d, z1.d\n" - "uzp1 z2.h, z2.h, z2.h\n" - "ld1h { z7.h }, p7/Z, [x11, #1, MUL VL]\n" - ".inst 0x6464e408 // bfmmla z8.s, z0.h, z4.h\n" - ".inst 0x6464e454 // bfmmla z20.s, z2.h, z4.h\n" - ".inst 0x6465e40e // bfmmla z14.s, z0.h, z5.h\n" - "ld1h { z4.h }, p7/Z, [x10]\n" + ".inst 0x658abc84 // bfcvt z4.h, p7/M, z4.s\n" + "ld1h { z2.h }, p7/Z, [x12, #1, MUL VL]\n" + "ld1h { z1.h }, p7/Z, [x11]\n" + "trn1 z5.d, z5.d, z0.d\n" + "uzp1 z4.h, z4.h, z4.h\n" + "ld1h { z0.h }, p7/Z, [x11, #1, MUL VL]\n" + ".inst 0x6463e4a8 // bfmmla z8.s, z5.h, z3.h\n" + ".inst 0x6463e494 // bfmmla z20.s, z4.h, z3.h\n" + ".inst 0x6462e4ae // bfmmla z14.s, z5.h, z2.h\n" + "ld1h { z3.h }, p7/Z, [x10]\n" "addvl x12, x12, #2\n" - ".inst 0x6465e45a // bfmmla z26.s, z2.h, z5.h\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - "ld1h { z5.h }, p7/Z, [x10, #1, MUL VL]\n" + ".inst 0x6462e49a // bfmmla z26.s, z4.h, z2.h\n" + ".inst 0x6461e4a9 // bfmmla z9.s, z5.h, z1.h\n" + "ld1h { z2.h }, p7/Z, [x10, #1, MUL VL]\n" "addvl x11, x11, #2\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" - "ld1h { z6.h }, p7/Z, [x9]\n" + ".inst 0x6461e495 // bfmmla z21.s, z4.h, z1.h\n" + ".inst 0x6460e4af // bfmmla z15.s, z5.h, z0.h\n" + "ld1h { z1.h }, p7/Z, [x9]\n" "addvl x10, x10, #2\n" - ".inst 0x6467e45b // bfmmla z27.s, z2.h, z7.h\n" - "ld1h { z7.h }, p7/Z, [x9, #1, MUL VL]\n" - ".inst 0x6464e40a // bfmmla z10.s, z0.h, z4.h\n" + ".inst 0x6460e49b // bfmmla z27.s, z4.h, z0.h\n" + "ld1h { z0.h }, p7/Z, [x9, #1, MUL VL]\n" + ".inst 0x6463e4aa // bfmmla z10.s, z5.h, z3.h\n" "addvl x9, x9, #2\n" - ".inst 0x6464e456 // bfmmla z22.s, z2.h, z4.h\n" - ".inst 0x6465e410 // bfmmla z16.s, z0.h, z5.h\n" - "ld1h { z4.h }, p7/Z, [x28]\n" - ".inst 0x6465e45c // bfmmla z28.s, z2.h, z5.h\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" - "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n" + ".inst 0x6463e496 // bfmmla z22.s, z4.h, z3.h\n" + ".inst 0x6462e4b0 // bfmmla z16.s, z5.h, z2.h\n" + "ld1h { z3.h }, p7/Z, [x28]\n" + ".inst 0x6462e49c // bfmmla z28.s, z4.h, z2.h\n" + ".inst 0x6461e4ab // bfmmla z11.s, z5.h, z1.h\n" + "ld1h { z2.h }, p7/Z, [x28, #1, MUL VL]\n" "addvl x28, x28, #2\n" - ".inst 0x6466e457 // bfmmla z23.s, z2.h, z6.h\n" - ".inst 0x6467e411 // bfmmla z17.s, z0.h, z7.h\n" - "ld1h { z6.h }, p7/Z, [x27]\n" - ".inst 0x6467e45d // bfmmla z29.s, z2.h, z7.h\n" - "ld1h { z7.h }, p7/Z, [x27, #1, MUL VL]\n" - ".inst 0x6464e40c // bfmmla z12.s, z0.h, z4.h\n" + ".inst 0x6461e497 // bfmmla z23.s, z4.h, z1.h\n" + ".inst 0x6460e4b1 // bfmmla z17.s, z5.h, z0.h\n" + "ld1h { z1.h }, p7/Z, [x27]\n" + ".inst 0x6460e49d // bfmmla z29.s, z4.h, z0.h\n" + "ld1h { z0.h }, p7/Z, [x27, #1, MUL VL]\n" + ".inst 0x6463e4ac // bfmmla z12.s, z5.h, z3.h\n" "addvl x27, x27, #2\n" - ".inst 0x6464e458 // bfmmla z24.s, z2.h, z4.h\n" - ".inst 0x6465e412 // bfmmla z18.s, z0.h, z5.h\n" - ".inst 0x6465e45e // bfmmla z30.s, z2.h, z5.h\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6466e459 // bfmmla z25.s, z2.h, z6.h\n" - ".inst 0x6467e413 // bfmmla z19.s, z0.h, z7.h\n" - ".inst 0x6467e45f // bfmmla z31.s, z2.h, z7.h\n" + ".inst 0x6463e498 // bfmmla z24.s, z4.h, z3.h\n" + ".inst 0x6462e4b2 // bfmmla z18.s, z5.h, z2.h\n" + ".inst 0x6462e49e // bfmmla z30.s, z4.h, z2.h\n" + ".inst 0x6461e4ad // bfmmla z13.s, z5.h, z1.h\n" + ".inst 0x6461e499 // bfmmla z25.s, z4.h, z1.h\n" + ".inst 0x6460e4b3 // bfmmla z19.s, z5.h, z0.h\n" + ".inst 0x6460e49f // bfmmla z31.s, z4.h, z0.h\n" "40:" // Height 3: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x26, x26, #0x1\n" @@ -946,45 +946,45 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL ( "uzp1 z25.d, z25.d, z31.d\n" "tbz %x[flags], #1, 41f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p7/Z, [x20]\n" - "add x20, %x[args_ptr], %[offset_min]\n" "ld1rw { z0.s }, p7/Z, [x20]\n" - "fmin z4.s, p7/M, z4.s, z1.s\n" - "fmin z14.s, p7/M, z14.s, z1.s\n" - "fmin z15.s, p7/M, z15.s, z1.s\n" - "fmin z16.s, p7/M, z16.s, z1.s\n" - "fmin z17.s, p7/M, z17.s, z1.s\n" - "fmin z18.s, p7/M, z18.s, z1.s\n" - "fmin z8.s, p7/M, z8.s, z1.s\n" - "fmin z9.s, p7/M, z9.s, z1.s\n" - "fmin z10.s, p7/M, z10.s, z1.s\n" - "fmin z11.s, p7/M, z11.s, z1.s\n" - "fmin z12.s, p7/M, z12.s, z1.s\n" - "fmin z13.s, p7/M, z13.s, z1.s\n" - "fmin z20.s, p7/M, z20.s, z1.s\n" - "fmin z21.s, p7/M, z21.s, z1.s\n" - "fmin z22.s, p7/M, z22.s, z1.s\n" - "fmin z23.s, p7/M, z23.s, z1.s\n" - "fmin z24.s, p7/M, z24.s, z1.s\n" - "fmin z25.s, p7/M, z25.s, z1.s\n" - "fmax z4.s, p7/M, z4.s, z0.s\n" - "fmax z14.s, p7/M, z14.s, z0.s\n" - "fmax z15.s, p7/M, z15.s, z0.s\n" - "fmax z16.s, p7/M, z16.s, z0.s\n" - "fmax z17.s, p7/M, z17.s, z0.s\n" - "fmax z18.s, p7/M, z18.s, z0.s\n" - "fmax z8.s, p7/M, z8.s, z0.s\n" - "fmax z9.s, p7/M, z9.s, z0.s\n" - "fmax z10.s, p7/M, z10.s, z0.s\n" - "fmax z11.s, p7/M, z11.s, z0.s\n" - "fmax z12.s, p7/M, z12.s, z0.s\n" - "fmax z13.s, p7/M, z13.s, z0.s\n" - "fmax z20.s, p7/M, z20.s, z0.s\n" - "fmax z21.s, p7/M, z21.s, z0.s\n" - "fmax z22.s, p7/M, z22.s, z0.s\n" - "fmax z23.s, p7/M, z23.s, z0.s\n" - "fmax z24.s, p7/M, z24.s, z0.s\n" - "fmax z25.s, p7/M, z25.s, z0.s\n" + "add x20, %x[args_ptr], %[offset_min]\n" + "ld1rw { z19.s }, p7/Z, [x20]\n" + "fmin z4.s, p7/M, z4.s, z0.s\n" + "fmin z14.s, p7/M, z14.s, z0.s\n" + "fmin z15.s, p7/M, z15.s, z0.s\n" + "fmin z16.s, p7/M, z16.s, z0.s\n" + "fmin z17.s, p7/M, z17.s, z0.s\n" + "fmin z18.s, p7/M, z18.s, z0.s\n" + "fmin z8.s, p7/M, z8.s, z0.s\n" + "fmin z9.s, p7/M, z9.s, z0.s\n" + "fmin z10.s, p7/M, z10.s, z0.s\n" + "fmin z11.s, p7/M, z11.s, z0.s\n" + "fmin z12.s, p7/M, z12.s, z0.s\n" + "fmin z13.s, p7/M, z13.s, z0.s\n" + "fmin z20.s, p7/M, z20.s, z0.s\n" + "fmin z21.s, p7/M, z21.s, z0.s\n" + "fmin z22.s, p7/M, z22.s, z0.s\n" + "fmin z23.s, p7/M, z23.s, z0.s\n" + "fmin z24.s, p7/M, z24.s, z0.s\n" + "fmin z25.s, p7/M, z25.s, z0.s\n" + "fmax z4.s, p7/M, z4.s, z19.s\n" + "fmax z14.s, p7/M, z14.s, z19.s\n" + "fmax z15.s, p7/M, z15.s, z19.s\n" + "fmax z16.s, p7/M, z16.s, z19.s\n" + "fmax z17.s, p7/M, z17.s, z19.s\n" + "fmax z18.s, p7/M, z18.s, z19.s\n" + "fmax z8.s, p7/M, z8.s, z19.s\n" + "fmax z9.s, p7/M, z9.s, z19.s\n" + "fmax z10.s, p7/M, z10.s, z19.s\n" + "fmax z11.s, p7/M, z11.s, z19.s\n" + "fmax z12.s, p7/M, z12.s, z19.s\n" + "fmax z13.s, p7/M, z13.s, z19.s\n" + "fmax z20.s, p7/M, z20.s, z19.s\n" + "fmax z21.s, p7/M, z21.s, z19.s\n" + "fmax z22.s, p7/M, z22.s, z19.s\n" + "fmax z23.s, p7/M, z23.s, z19.s\n" + "fmax z24.s, p7/M, z24.s, z19.s\n" + "fmax z25.s, p7/M, z25.s, z19.s\n" "41:" // Height 3: No activation "st1w { z4.s }, p6, [x13]\n" "st1w { z14.s }, p5, [x13, #1, MUL VL]\n" @@ -1098,57 +1098,57 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL ( "46:" // Height 4: no bias "tbz %x[flags], #0, 47f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x23, x13, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "ld1w { z9.s }, p6/Z, [x13]\n" + "add x22, x13, x20, LSL #2\n" "add x21, x22, x20, LSL #2\n" - "ld1w { z10.s }, p5/Z, [x13, #1, MUL VL]\n" - "ld1w { z11.s }, p4/Z, [x13, #2, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x13, #3, MUL VL]\n" - "ld1w { z13.s }, p2/Z, [x13, #4, MUL VL]\n" + "ld1w { z16.s }, p6/Z, [x13]\n" + "add x20, x21, x20, LSL #2\n" + "ld1w { z17.s }, p5/Z, [x13, #1, MUL VL]\n" + "ld1w { z19.s }, p4/Z, [x13, #2, MUL VL]\n" + "ld1w { z22.s }, p3/Z, [x13, #3, MUL VL]\n" + "ld1w { z24.s }, p2/Z, [x13, #4, MUL VL]\n" "ld1w { z20.s }, p1/Z, [x13, #5, MUL VL]\n" - "ld1w { z14.s }, p6/Z, [x23]\n" - "zip1 z8.d, z9.d, z14.d\n" - "zip2 z14.d, z9.d, z14.d\n" - "ld1w { z15.s }, p5/Z, [x23, #1, MUL VL]\n" - "ld1w { z16.s }, p4/Z, [x23, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z15.d\n" - "zip2 z15.d, z10.d, z15.d\n" - "ld1w { z17.s }, p3/Z, [x23, #3, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x23, #4, MUL VL]\n" - "zip1 z10.d, z11.d, z16.d\n" - "zip2 z16.d, z11.d, z16.d\n" - "ld1w { z19.s }, p1/Z, [x23, #5, MUL VL]\n" - "ld1w { z21.s }, p6/Z, [x22]\n" - "zip1 z11.d, z12.d, z17.d\n" - "zip2 z17.d, z12.d, z17.d\n" - "ld1w { z22.s }, p5/Z, [x22, #1, MUL VL]\n" - "ld1w { z23.s }, p4/Z, [x22, #2, MUL VL]\n" - "zip1 z12.d, z13.d, z18.d\n" - "zip2 z18.d, z13.d, z18.d\n" - "ld1w { z24.s }, p3/Z, [x22, #3, MUL VL]\n" - "ld1w { z25.s }, p2/Z, [x22, #4, MUL VL]\n" + "ld1w { z14.s }, p6/Z, [x22]\n" + "zip1 z8.d, z16.d, z14.d\n" + "zip2 z14.d, z16.d, z14.d\n" + "ld1w { z15.s }, p5/Z, [x22, #1, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x22, #2, MUL VL]\n" + "zip1 z9.d, z17.d, z15.d\n" + "zip2 z15.d, z17.d, z15.d\n" + "ld1w { z17.s }, p3/Z, [x22, #3, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x22, #4, MUL VL]\n" + "zip1 z10.d, z19.d, z16.d\n" + "zip2 z16.d, z19.d, z16.d\n" + "ld1w { z19.s }, p1/Z, [x22, #5, MUL VL]\n" + "ld1w { z21.s }, p6/Z, [x21]\n" + "zip1 z11.d, z22.d, z17.d\n" + "zip2 z17.d, z22.d, z17.d\n" + "ld1w { z22.s }, p5/Z, [x21, #1, MUL VL]\n" + "ld1w { z23.s }, p4/Z, [x21, #2, MUL VL]\n" + "zip1 z12.d, z24.d, z18.d\n" + "zip2 z18.d, z24.d, z18.d\n" + "ld1w { z24.s }, p3/Z, [x21, #3, MUL VL]\n" + "ld1w { z25.s }, p2/Z, [x21, #4, MUL VL]\n" "zip1 z13.d, z20.d, z19.d\n" "zip2 z19.d, z20.d, z19.d\n" - "ld1w { z4.s }, p1/Z, [x22, #5, MUL VL]\n" - "ld1w { z26.s }, p6/Z, [x21]\n" + "ld1w { z0.s }, p1/Z, [x21, #5, MUL VL]\n" + "ld1w { z26.s }, p6/Z, [x20]\n" "zip1 z20.d, z21.d, z26.d\n" "zip2 z26.d, z21.d, z26.d\n" - "ld1w { z27.s }, p5/Z, [x21, #1, MUL VL]\n" - "ld1w { z28.s }, p4/Z, [x21, #2, MUL VL]\n" + "ld1w { z27.s }, p5/Z, [x20, #1, MUL VL]\n" + "ld1w { z28.s }, p4/Z, [x20, #2, MUL VL]\n" "zip1 z21.d, z22.d, z27.d\n" "zip2 z27.d, z22.d, z27.d\n" - "ld1w { z29.s }, p3/Z, [x21, #3, MUL VL]\n" - "ld1w { z30.s }, p2/Z, [x21, #4, MUL VL]\n" + "ld1w { z29.s }, p3/Z, [x20, #3, MUL VL]\n" + "ld1w { z30.s }, p2/Z, [x20, #4, MUL VL]\n" "zip1 z22.d, z23.d, z28.d\n" "zip2 z28.d, z23.d, z28.d\n" - "ld1w { z31.s }, p1/Z, [x21, #5, MUL VL]\n" + "ld1w { z31.s }, p1/Z, [x20, #5, MUL VL]\n" "zip1 z23.d, z24.d, z29.d\n" "zip2 z29.d, z24.d, z29.d\n" "zip1 z24.d, z25.d, z30.d\n" "zip2 z30.d, z25.d, z30.d\n" - "zip1 z25.d, z4.d, z31.d\n" - "zip2 z31.d, z4.d, z31.d\n" + "zip1 z25.d, z0.d, z31.d\n" + "zip2 z31.d, z0.d, z31.d\n" "b 48f\n" "47:" // Height 4: no accumulate "mov z8.b, #0x0\n" @@ -1180,14 +1180,14 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL ( "49:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 50f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" - "ldr x21, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" + "ldr x21, [x20, #0x18]\n" "cbnz x26, 51f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20, LSL #2\n" @@ -1197,135 +1197,135 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL ( "b 51f\n" "50:" // Height 4: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "add x21, x22, x20, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" + "add x21, x22, x21, LSL #2\n" "51:" // Height 4: input setup done "cmp x25, #0x4\n" "ble 53f\n" "52:" // Height 4: Multiply loop: Main loop head "whilelt p0.s, XZR, x25\n" - "ld1rqw { z0.s }, p0/Z, [x24]\n" - "ld1rqw { z1.s }, p0/Z, [x23]\n" - ".inst 0x658abc00 // bfcvt z0.h, p7/M, z0.s\n" - "ld1rqw { z2.s }, p0/Z, [x22]\n" - "ld1rqw { z3.s }, p0/Z, [x21]\n" - ".inst 0x658abc21 // bfcvt z1.h, p7/M, z1.s\n" - ".inst 0x658abc42 // bfcvt z2.h, p7/M, z2.s\n" - ".inst 0x658abc63 // bfcvt z3.h, p7/M, z3.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z4.h }, p7/Z, [x12]\n" - "ld1h { z5.h }, p7/Z, [x12, #1, MUL VL]\n" - "uzp1 z1.h, z1.h, z1.h\n" - "uzp1 z2.h, z2.h, z2.h\n" - "ld1h { z6.h }, p7/Z, [x11]\n" - "ld1h { z7.h }, p7/Z, [x11, #1, MUL VL]\n" - "uzp1 z3.h, z3.h, z3.h\n" - "trn1 z0.d, z0.d, z1.d\n" - ".inst 0x6464e408 // bfmmla z8.s, z0.h, z4.h\n" + "ld1rqw { z7.s }, p0/Z, [x24]\n" + "ld1rqw { z6.s }, p0/Z, [x23]\n" + ".inst 0x658abce7 // bfcvt z7.h, p7/M, z7.s\n" + "ld1rqw { z5.s }, p0/Z, [x22]\n" + "ld1rqw { z4.s }, p0/Z, [x21]\n" + ".inst 0x658abcc6 // bfcvt z6.h, p7/M, z6.s\n" + ".inst 0x658abca5 // bfcvt z5.h, p7/M, z5.s\n" + ".inst 0x658abc84 // bfcvt z4.h, p7/M, z4.s\n" + "uzp1 z7.h, z7.h, z7.h\n" + "ld1h { z3.h }, p7/Z, [x12]\n" + "ld1h { z2.h }, p7/Z, [x12, #1, MUL VL]\n" + "uzp1 z6.h, z6.h, z6.h\n" + "uzp1 z5.h, z5.h, z5.h\n" + "ld1h { z1.h }, p7/Z, [x11]\n" + "ld1h { z0.h }, p7/Z, [x11, #1, MUL VL]\n" + "uzp1 z4.h, z4.h, z4.h\n" + "trn1 z7.d, z7.d, z6.d\n" + ".inst 0x6463e4e8 // bfmmla z8.s, z7.h, z3.h\n" "sub x25, x25, #0x4\n" - "trn1 z2.d, z2.d, z3.d\n" - ".inst 0x6464e454 // bfmmla z20.s, z2.h, z4.h\n" - ".inst 0x6465e40e // bfmmla z14.s, z0.h, z5.h\n" - "ld1h { z4.h }, p7/Z, [x10]\n" - ".inst 0x6465e45a // bfmmla z26.s, z2.h, z5.h\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - "ld1h { z5.h }, p7/Z, [x10, #1, MUL VL]\n" + "trn1 z5.d, z5.d, z4.d\n" + ".inst 0x6463e4b4 // bfmmla z20.s, z5.h, z3.h\n" + ".inst 0x6462e4ee // bfmmla z14.s, z7.h, z2.h\n" + "ld1h { z3.h }, p7/Z, [x10]\n" + ".inst 0x6462e4ba // bfmmla z26.s, z5.h, z2.h\n" + ".inst 0x6461e4e9 // bfmmla z9.s, z7.h, z1.h\n" + "ld1h { z2.h }, p7/Z, [x10, #1, MUL VL]\n" "cmp x25, #0x4\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" - "ld1h { z6.h }, p7/Z, [x9]\n" + ".inst 0x6461e4b5 // bfmmla z21.s, z5.h, z1.h\n" + ".inst 0x6460e4ef // bfmmla z15.s, z7.h, z0.h\n" + "ld1h { z1.h }, p7/Z, [x9]\n" "add x24, x24, #0x10\n" - ".inst 0x6467e45b // bfmmla z27.s, z2.h, z7.h\n" - "ld1h { z7.h }, p7/Z, [x9, #1, MUL VL]\n" - ".inst 0x6464e40a // bfmmla z10.s, z0.h, z4.h\n" + ".inst 0x6460e4bb // bfmmla z27.s, z5.h, z0.h\n" + "ld1h { z0.h }, p7/Z, [x9, #1, MUL VL]\n" + ".inst 0x6463e4ea // bfmmla z10.s, z7.h, z3.h\n" "add x23, x23, #0x10\n" - ".inst 0x6464e456 // bfmmla z22.s, z2.h, z4.h\n" - ".inst 0x6465e410 // bfmmla z16.s, z0.h, z5.h\n" - "ld1h { z4.h }, p7/Z, [x28]\n" + ".inst 0x6463e4b6 // bfmmla z22.s, z5.h, z3.h\n" + ".inst 0x6462e4f0 // bfmmla z16.s, z7.h, z2.h\n" + "ld1h { z3.h }, p7/Z, [x28]\n" "add x22, x22, #0x10\n" - ".inst 0x6465e45c // bfmmla z28.s, z2.h, z5.h\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" - "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n" + ".inst 0x6462e4bc // bfmmla z28.s, z5.h, z2.h\n" + ".inst 0x6461e4eb // bfmmla z11.s, z7.h, z1.h\n" + "ld1h { z2.h }, p7/Z, [x28, #1, MUL VL]\n" "add x21, x21, #0x10\n" - ".inst 0x6466e457 // bfmmla z23.s, z2.h, z6.h\n" - ".inst 0x6467e411 // bfmmla z17.s, z0.h, z7.h\n" - "ld1h { z6.h }, p7/Z, [x27]\n" + ".inst 0x6461e4b7 // bfmmla z23.s, z5.h, z1.h\n" + ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n" + "ld1h { z1.h }, p7/Z, [x27]\n" "addvl x12, x12, #2\n" - ".inst 0x6467e45d // bfmmla z29.s, z2.h, z7.h\n" - "ld1h { z7.h }, p7/Z, [x27, #1, MUL VL]\n" - ".inst 0x6464e40c // bfmmla z12.s, z0.h, z4.h\n" + ".inst 0x6460e4bd // bfmmla z29.s, z5.h, z0.h\n" + "ld1h { z0.h }, p7/Z, [x27, #1, MUL VL]\n" + ".inst 0x6463e4ec // bfmmla z12.s, z7.h, z3.h\n" "addvl x11, x11, #2\n" - ".inst 0x6464e458 // bfmmla z24.s, z2.h, z4.h\n" - ".inst 0x6465e412 // bfmmla z18.s, z0.h, z5.h\n" + ".inst 0x6463e4b8 // bfmmla z24.s, z5.h, z3.h\n" + ".inst 0x6462e4f2 // bfmmla z18.s, z7.h, z2.h\n" "addvl x10, x10, #2\n" "addvl x9, x9, #2\n" - ".inst 0x6465e45e // bfmmla z30.s, z2.h, z5.h\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" + ".inst 0x6462e4be // bfmmla z30.s, z5.h, z2.h\n" + ".inst 0x6461e4ed // bfmmla z13.s, z7.h, z1.h\n" "addvl x28, x28, #2\n" "addvl x27, x27, #2\n" - ".inst 0x6466e459 // bfmmla z25.s, z2.h, z6.h\n" - ".inst 0x6467e413 // bfmmla z19.s, z0.h, z7.h\n" - ".inst 0x6467e45f // bfmmla z31.s, z2.h, z7.h\n" + ".inst 0x6461e4b9 // bfmmla z25.s, z5.h, z1.h\n" + ".inst 0x6460e4f3 // bfmmla z19.s, z7.h, z0.h\n" + ".inst 0x6460e4bf // bfmmla z31.s, z5.h, z0.h\n" "bgt 52b\n" "53:" // Height 4: Multiply loop: Single iteration only "whilelt p0.s, XZR, x25\n" - "ld1rqw { z0.s }, p0/Z, [x24]\n" - "ld1rqw { z1.s }, p0/Z, [x23]\n" - ".inst 0x658abc00 // bfcvt z0.h, p7/M, z0.s\n" - "ld1rqw { z2.s }, p0/Z, [x22]\n" - "ld1rqw { z3.s }, p0/Z, [x21]\n" - ".inst 0x658abc21 // bfcvt z1.h, p7/M, z1.s\n" - ".inst 0x658abc42 // bfcvt z2.h, p7/M, z2.s\n" - ".inst 0x658abc63 // bfcvt z3.h, p7/M, z3.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z4.h }, p7/Z, [x12]\n" - "ld1h { z5.h }, p7/Z, [x12, #1, MUL VL]\n" - "uzp1 z1.h, z1.h, z1.h\n" - "uzp1 z2.h, z2.h, z2.h\n" - "ld1h { z6.h }, p7/Z, [x11]\n" - "ld1h { z7.h }, p7/Z, [x11, #1, MUL VL]\n" - "uzp1 z3.h, z3.h, z3.h\n" - "trn1 z0.d, z0.d, z1.d\n" - ".inst 0x6464e408 // bfmmla z8.s, z0.h, z4.h\n" + "ld1rqw { z7.s }, p0/Z, [x24]\n" + "ld1rqw { z6.s }, p0/Z, [x23]\n" + ".inst 0x658abce7 // bfcvt z7.h, p7/M, z7.s\n" + "ld1rqw { z5.s }, p0/Z, [x22]\n" + "ld1rqw { z4.s }, p0/Z, [x21]\n" + ".inst 0x658abcc6 // bfcvt z6.h, p7/M, z6.s\n" + ".inst 0x658abca5 // bfcvt z5.h, p7/M, z5.s\n" + ".inst 0x658abc84 // bfcvt z4.h, p7/M, z4.s\n" + "uzp1 z7.h, z7.h, z7.h\n" + "ld1h { z3.h }, p7/Z, [x12]\n" + "ld1h { z2.h }, p7/Z, [x12, #1, MUL VL]\n" + "uzp1 z6.h, z6.h, z6.h\n" + "uzp1 z5.h, z5.h, z5.h\n" + "ld1h { z1.h }, p7/Z, [x11]\n" + "ld1h { z0.h }, p7/Z, [x11, #1, MUL VL]\n" + "uzp1 z4.h, z4.h, z4.h\n" + "trn1 z7.d, z7.d, z6.d\n" + ".inst 0x6463e4e8 // bfmmla z8.s, z7.h, z3.h\n" "addvl x12, x12, #2\n" - "trn1 z2.d, z2.d, z3.d\n" - ".inst 0x6464e454 // bfmmla z20.s, z2.h, z4.h\n" - ".inst 0x6465e40e // bfmmla z14.s, z0.h, z5.h\n" - "ld1h { z4.h }, p7/Z, [x10]\n" - ".inst 0x6465e45a // bfmmla z26.s, z2.h, z5.h\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - "ld1h { z5.h }, p7/Z, [x10, #1, MUL VL]\n" + "trn1 z5.d, z5.d, z4.d\n" + ".inst 0x6463e4b4 // bfmmla z20.s, z5.h, z3.h\n" + ".inst 0x6462e4ee // bfmmla z14.s, z7.h, z2.h\n" + "ld1h { z3.h }, p7/Z, [x10]\n" + ".inst 0x6462e4ba // bfmmla z26.s, z5.h, z2.h\n" + ".inst 0x6461e4e9 // bfmmla z9.s, z7.h, z1.h\n" + "ld1h { z2.h }, p7/Z, [x10, #1, MUL VL]\n" "addvl x11, x11, #2\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" - "ld1h { z6.h }, p7/Z, [x9]\n" + ".inst 0x6461e4b5 // bfmmla z21.s, z5.h, z1.h\n" + ".inst 0x6460e4ef // bfmmla z15.s, z7.h, z0.h\n" + "ld1h { z1.h }, p7/Z, [x9]\n" "addvl x10, x10, #2\n" - ".inst 0x6467e45b // bfmmla z27.s, z2.h, z7.h\n" - "ld1h { z7.h }, p7/Z, [x9, #1, MUL VL]\n" - ".inst 0x6464e40a // bfmmla z10.s, z0.h, z4.h\n" + ".inst 0x6460e4bb // bfmmla z27.s, z5.h, z0.h\n" + "ld1h { z0.h }, p7/Z, [x9, #1, MUL VL]\n" + ".inst 0x6463e4ea // bfmmla z10.s, z7.h, z3.h\n" "addvl x9, x9, #2\n" - ".inst 0x6464e456 // bfmmla z22.s, z2.h, z4.h\n" - ".inst 0x6465e410 // bfmmla z16.s, z0.h, z5.h\n" - "ld1h { z4.h }, p7/Z, [x28]\n" - ".inst 0x6465e45c // bfmmla z28.s, z2.h, z5.h\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" - "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n" + ".inst 0x6463e4b6 // bfmmla z22.s, z5.h, z3.h\n" + ".inst 0x6462e4f0 // bfmmla z16.s, z7.h, z2.h\n" + "ld1h { z3.h }, p7/Z, [x28]\n" + ".inst 0x6462e4bc // bfmmla z28.s, z5.h, z2.h\n" + ".inst 0x6461e4eb // bfmmla z11.s, z7.h, z1.h\n" + "ld1h { z2.h }, p7/Z, [x28, #1, MUL VL]\n" "addvl x28, x28, #2\n" - ".inst 0x6466e457 // bfmmla z23.s, z2.h, z6.h\n" - ".inst 0x6467e411 // bfmmla z17.s, z0.h, z7.h\n" - "ld1h { z6.h }, p7/Z, [x27]\n" - ".inst 0x6467e45d // bfmmla z29.s, z2.h, z7.h\n" - "ld1h { z7.h }, p7/Z, [x27, #1, MUL VL]\n" - ".inst 0x6464e40c // bfmmla z12.s, z0.h, z4.h\n" + ".inst 0x6461e4b7 // bfmmla z23.s, z5.h, z1.h\n" + ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n" + "ld1h { z1.h }, p7/Z, [x27]\n" + ".inst 0x6460e4bd // bfmmla z29.s, z5.h, z0.h\n" + "ld1h { z0.h }, p7/Z, [x27, #1, MUL VL]\n" + ".inst 0x6463e4ec // bfmmla z12.s, z7.h, z3.h\n" "addvl x27, x27, #2\n" - ".inst 0x6464e458 // bfmmla z24.s, z2.h, z4.h\n" - ".inst 0x6465e412 // bfmmla z18.s, z0.h, z5.h\n" - ".inst 0x6465e45e // bfmmla z30.s, z2.h, z5.h\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6466e459 // bfmmla z25.s, z2.h, z6.h\n" - ".inst 0x6467e413 // bfmmla z19.s, z0.h, z7.h\n" - ".inst 0x6467e45f // bfmmla z31.s, z2.h, z7.h\n" + ".inst 0x6463e4b8 // bfmmla z24.s, z5.h, z3.h\n" + ".inst 0x6462e4f2 // bfmmla z18.s, z7.h, z2.h\n" + ".inst 0x6462e4be // bfmmla z30.s, z5.h, z2.h\n" + ".inst 0x6461e4ed // bfmmla z13.s, z7.h, z1.h\n" + ".inst 0x6461e4b9 // bfmmla z25.s, z5.h, z1.h\n" + ".inst 0x6460e4f3 // bfmmla z19.s, z7.h, z0.h\n" + ".inst 0x6460e4bf // bfmmla z31.s, z5.h, z0.h\n" "54:" // Height 4: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x26, x26, #0x1\n" @@ -1461,4 +1461,4 @@ void sve_ffhybrid_fp32bf16fp32_mmla_4x6VL ( } } // namespace arm_gemm -#endif // ARM_COMPUTE_ENABLE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_bf16fp32_mmla_8x3VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_bf16fp32_mmla_8x3VL.hpp index 5792a7152d8b8e81b7eb1d752170ab16ff5e6a09..d0ef531c33c2a521b1e3c04d4ee3580cfee7d2f4 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_bf16fp32_mmla_8x3VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_bf16fp32_mmla_8x3VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,16 +10,16 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once #ifdef ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_bf16fp32_mmla_8x3VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_bf16fp32_mmla_8x3VL/generic.cpp index 7649336c36f57ee3fcbda1bc7d2763eb9645ab8f..576bd47039447569fa0718941240fb337c13a561 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_bf16fp32_mmla_8x3VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_bf16fp32_mmla_8x3VL/generic.cpp @@ -53,33 +53,33 @@ void sve_ffinterleaved_bf16fp32_mmla_8x3VL( __asm__ __volatile__( "ptrue p0.b\n" "1:" // Height loop - "ldr x26, [%x[args_ptr], %[offsetof_Bpanel]]\n" - "ldr x25, [%x[args_ptr], %[offsetof_N]]\n" - "str x26, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" - "mov x24, %x[Apanel]\n" + "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n" + "ldr x26, [%x[args_ptr], %[offsetof_N]]\n" + "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" + "mov x25, %x[Apanel]\n" "2:" // Width loop - "ldr x26, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" + "ldr x24, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n" "cntw x23, ALL, MUL #2\n" - "add x22, x26, x20, LSL #1\n" + "add x22, x24, x20, LSL #1\n" "add x21, x22, x20, LSL #1\n" "add x20, x21, x20, LSL #1\n" - "cmp x25, x23\n" + "cmp x26, x23\n" "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" - "mov %x[Apanel], x24\n" + "mov %x[Apanel], x25\n" "bgt 3f\n" "decw x23\n" - "cmp x25, x23\n" - "mov x21, x26\n" + "cmp x26, x23\n" + "mov x21, x24\n" "bgt 3f\n" - "mov x22, x26\n" + "mov x22, x24\n" "3:" // B setup done "ldr x20, [%x[args_ptr], %[offsetof_K]]\n" "cmp x20, #0x2\n" "mov z8.b, #0x0\n" "mov z9.b, #0x0\n" "mov z10.b, #0x0\n" - "ld1h { z4.h }, p0/Z, [x26]\n" + "ld1h { z4.h }, p0/Z, [x24]\n" "mov z11.b, #0x0\n" "mov z12.b, #0x0\n" "ld1rqh { z0.h }, p0/Z, [%x[Apanel]]\n" @@ -88,13 +88,13 @@ void sve_ffinterleaved_bf16fp32_mmla_8x3VL( "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #16]\n" "mov z15.b, #0x0\n" "mov z16.b, #0x0\n" - "ld1h { z5.h }, p0/Z, [x26, #1, MUL VL]\n" + "ld1h { z5.h }, p0/Z, [x24, #1, MUL VL]\n" "mov z17.b, #0x0\n" "mov z18.b, #0x0\n" "ld1rqh { z2.h }, p0/Z, [%x[Apanel], #32]\n" "mov z19.b, #0x0\n" "mov z20.b, #0x0\n" - "addvl x26, x26, #2\n" + "addvl x24, x24, #2\n" "mov z21.b, #0x0\n" "mov z22.b, #0x0\n" "add %x[Apanel], %x[Apanel], #0x30\n" @@ -109,83 +109,83 @@ void sve_ffinterleaved_bf16fp32_mmla_8x3VL( "mov z31.b, #0x0\n" "blt 5f\n" "4:" // main loop head - "ld1rqh { z3.h }, p0/Z, [%x[Apanel]]\n" + "ld1rqh { z6.h }, p0/Z, [%x[Apanel]]\n" ".inst 0x6464e408 // bfmmla z8.s, z0.h, z4.h\n" ".inst 0x6465e40b // bfmmla z11.s, z0.h, z5.h\n" ".inst 0x6464e42e // bfmmla z14.s, z1.h, z4.h\n" ".inst 0x6465e431 // bfmmla z17.s, z1.h, z5.h\n" - "ld1h { z6.h }, p0/Z, [x22]\n" + "ld1h { z7.h }, p0/Z, [x22]\n" ".inst 0x6464e454 // bfmmla z20.s, z2.h, z4.h\n" ".inst 0x6465e457 // bfmmla z23.s, z2.h, z5.h\n" - "ld1h { z7.h }, p0/Z, [x22, #1, MUL VL]\n" - ".inst 0x6464e47a // bfmmla z26.s, z3.h, z4.h\n" - ".inst 0x6465e47d // bfmmla z29.s, z3.h, z5.h\n" - "ld1h { z4.h }, p0/Z, [x21]\n" - "ld1h { z5.h }, p0/Z, [x21, #1, MUL VL]\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - ".inst 0x6467e40c // bfmmla z12.s, z0.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" - ".inst 0x6467e432 // bfmmla z18.s, z1.h, z7.h\n" + "ld1h { z3.h }, p0/Z, [x22, #1, MUL VL]\n" + ".inst 0x6464e4da // bfmmla z26.s, z6.h, z4.h\n" + ".inst 0x6465e4dd // bfmmla z29.s, z6.h, z5.h\n" + "ld1h { z5.h }, p0/Z, [x21]\n" + "ld1h { z4.h }, p0/Z, [x21, #1, MUL VL]\n" + ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" + ".inst 0x6463e40c // bfmmla z12.s, z0.h, z3.h\n" + ".inst 0x6467e42f // bfmmla z15.s, z1.h, z7.h\n" + ".inst 0x6463e432 // bfmmla z18.s, z1.h, z3.h\n" "sub x20, x20, #0x2\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" - ".inst 0x6467e458 // bfmmla z24.s, z2.h, z7.h\n" + ".inst 0x6467e455 // bfmmla z21.s, z2.h, z7.h\n" + ".inst 0x6463e458 // bfmmla z24.s, z2.h, z3.h\n" "cmp x20, #0x2\n" - ".inst 0x6466e47b // bfmmla z27.s, z3.h, z6.h\n" - ".inst 0x6467e47e // bfmmla z30.s, z3.h, z7.h\n" - "ld1h { z6.h }, p0/Z, [x26]\n" - ".inst 0x6464e40a // bfmmla z10.s, z0.h, z4.h\n" - ".inst 0x6465e40d // bfmmla z13.s, z0.h, z5.h\n" + ".inst 0x6467e4db // bfmmla z27.s, z6.h, z7.h\n" + ".inst 0x6463e4de // bfmmla z30.s, z6.h, z3.h\n" + "ld1h { z3.h }, p0/Z, [x24]\n" + ".inst 0x6465e40a // bfmmla z10.s, z0.h, z5.h\n" + ".inst 0x6464e40d // bfmmla z13.s, z0.h, z4.h\n" "ld1rqh { z0.h }, p0/Z, [%x[Apanel], #16]\n" - ".inst 0x6464e430 // bfmmla z16.s, z1.h, z4.h\n" - ".inst 0x6465e433 // bfmmla z19.s, z1.h, z5.h\n" + ".inst 0x6465e430 // bfmmla z16.s, z1.h, z5.h\n" + ".inst 0x6464e433 // bfmmla z19.s, z1.h, z4.h\n" "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #32]\n" - ".inst 0x6464e456 // bfmmla z22.s, z2.h, z4.h\n" - ".inst 0x6465e459 // bfmmla z25.s, z2.h, z5.h\n" - "ld1h { z7.h }, p0/Z, [x26, #1, MUL VL]\n" - ".inst 0x6464e47c // bfmmla z28.s, z3.h, z4.h\n" - ".inst 0x6465e47f // bfmmla z31.s, z3.h, z5.h\n" - "ld1rqh { z2.h }, p0/Z, [%x[Apanel], #48]\n" - "ld1rqh { z3.h }, p0/Z, [%x[Apanel], #64]\n" - ".inst 0x6466e408 // bfmmla z8.s, z0.h, z6.h\n" + ".inst 0x6465e456 // bfmmla z22.s, z2.h, z5.h\n" + ".inst 0x6464e459 // bfmmla z25.s, z2.h, z4.h\n" + "ld1h { z7.h }, p0/Z, [x24, #1, MUL VL]\n" + ".inst 0x6465e4dc // bfmmla z28.s, z6.h, z5.h\n" + ".inst 0x6464e4df // bfmmla z31.s, z6.h, z4.h\n" + "ld1rqh { z5.h }, p0/Z, [%x[Apanel], #48]\n" + "ld1rqh { z6.h }, p0/Z, [%x[Apanel], #64]\n" + ".inst 0x6463e408 // bfmmla z8.s, z0.h, z3.h\n" ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" + ".inst 0x6463e42e // bfmmla z14.s, z1.h, z3.h\n" ".inst 0x6467e431 // bfmmla z17.s, z1.h, z7.h\n" - "ld1h { z4.h }, p0/Z, [x22, #2, MUL VL]\n" - ".inst 0x6466e454 // bfmmla z20.s, z2.h, z6.h\n" - ".inst 0x6467e457 // bfmmla z23.s, z2.h, z7.h\n" - "ld1h { z5.h }, p0/Z, [x22, #3, MUL VL]\n" - ".inst 0x6466e47a // bfmmla z26.s, z3.h, z6.h\n" - ".inst 0x6467e47d // bfmmla z29.s, z3.h, z7.h\n" - "ld1h { z6.h }, p0/Z, [x21, #2, MUL VL]\n" + "ld1h { z2.h }, p0/Z, [x22, #2, MUL VL]\n" + ".inst 0x6463e4b4 // bfmmla z20.s, z5.h, z3.h\n" + ".inst 0x6467e4b7 // bfmmla z23.s, z5.h, z7.h\n" + "ld1h { z4.h }, p0/Z, [x22, #3, MUL VL]\n" + ".inst 0x6463e4da // bfmmla z26.s, z6.h, z3.h\n" + ".inst 0x6467e4dd // bfmmla z29.s, z6.h, z7.h\n" + "ld1h { z3.h }, p0/Z, [x21, #2, MUL VL]\n" "ld1h { z7.h }, p0/Z, [x21, #3, MUL VL]\n" - ".inst 0x6464e409 // bfmmla z9.s, z0.h, z4.h\n" - ".inst 0x6465e40c // bfmmla z12.s, z0.h, z5.h\n" - ".inst 0x6464e42f // bfmmla z15.s, z1.h, z4.h\n" - ".inst 0x6465e432 // bfmmla z18.s, z1.h, z5.h\n" + ".inst 0x6462e409 // bfmmla z9.s, z0.h, z2.h\n" + ".inst 0x6464e40c // bfmmla z12.s, z0.h, z4.h\n" + ".inst 0x6462e42f // bfmmla z15.s, z1.h, z2.h\n" + ".inst 0x6464e432 // bfmmla z18.s, z1.h, z4.h\n" "addvl x22, x22, #4\n" - ".inst 0x6464e455 // bfmmla z21.s, z2.h, z4.h\n" - ".inst 0x6465e458 // bfmmla z24.s, z2.h, z5.h\n" + ".inst 0x6462e4b5 // bfmmla z21.s, z5.h, z2.h\n" + ".inst 0x6464e4b8 // bfmmla z24.s, z5.h, z4.h\n" "addvl x21, x21, #4\n" - ".inst 0x6464e47b // bfmmla z27.s, z3.h, z4.h\n" - ".inst 0x6465e47e // bfmmla z30.s, z3.h, z5.h\n" - "ld1h { z4.h }, p0/Z, [x26, #2, MUL VL]\n" - ".inst 0x6466e40a // bfmmla z10.s, z0.h, z6.h\n" + ".inst 0x6462e4db // bfmmla z27.s, z6.h, z2.h\n" + ".inst 0x6464e4de // bfmmla z30.s, z6.h, z4.h\n" + "ld1h { z4.h }, p0/Z, [x24, #2, MUL VL]\n" + ".inst 0x6463e40a // bfmmla z10.s, z0.h, z3.h\n" ".inst 0x6467e40d // bfmmla z13.s, z0.h, z7.h\n" "ld1rqh { z0.h }, p0/Z, [%x[Apanel], #80]\n" - ".inst 0x6466e430 // bfmmla z16.s, z1.h, z6.h\n" + ".inst 0x6463e430 // bfmmla z16.s, z1.h, z3.h\n" ".inst 0x6467e433 // bfmmla z19.s, z1.h, z7.h\n" "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #96]\n" - ".inst 0x6466e456 // bfmmla z22.s, z2.h, z6.h\n" - ".inst 0x6467e459 // bfmmla z25.s, z2.h, z7.h\n" - "ld1h { z5.h }, p0/Z, [x26, #3, MUL VL]\n" - ".inst 0x6466e47c // bfmmla z28.s, z3.h, z6.h\n" - ".inst 0x6467e47f // bfmmla z31.s, z3.h, z7.h\n" + ".inst 0x6463e4b6 // bfmmla z22.s, z5.h, z3.h\n" + ".inst 0x6467e4b9 // bfmmla z25.s, z5.h, z7.h\n" + "ld1h { z5.h }, p0/Z, [x24, #3, MUL VL]\n" + ".inst 0x6463e4dc // bfmmla z28.s, z6.h, z3.h\n" + ".inst 0x6467e4df // bfmmla z31.s, z6.h, z7.h\n" "ld1rqh { z2.h }, p0/Z, [%x[Apanel], #112]\n" "add %x[Apanel], %x[Apanel], #0x80\n" - "addvl x26, x26, #4\n" + "addvl x24, x24, #4\n" "bge 4b\n" "5:" // main loop skip - "ld1rqh { z3.h }, p0/Z, [%x[Apanel]]\n" + "ld1rqh { z7.h }, p0/Z, [%x[Apanel]]\n" ".inst 0x6464e408 // bfmmla z8.s, z0.h, z4.h\n" ".inst 0x6465e40b // bfmmla z11.s, z0.h, z5.h\n" ".inst 0x6464e42e // bfmmla z14.s, z1.h, z4.h\n" @@ -193,115 +193,115 @@ void sve_ffinterleaved_bf16fp32_mmla_8x3VL( "ld1h { z6.h }, p0/Z, [x22]\n" ".inst 0x6464e454 // bfmmla z20.s, z2.h, z4.h\n" ".inst 0x6465e457 // bfmmla z23.s, z2.h, z5.h\n" - "ld1h { z7.h }, p0/Z, [x22, #1, MUL VL]\n" - ".inst 0x6464e47a // bfmmla z26.s, z3.h, z4.h\n" - ".inst 0x6465e47d // bfmmla z29.s, z3.h, z5.h\n" - "ld1h { z4.h }, p0/Z, [x21]\n" - "ld1h { z5.h }, p0/Z, [x21, #1, MUL VL]\n" + "ld1h { z3.h }, p0/Z, [x22, #1, MUL VL]\n" + ".inst 0x6464e4fa // bfmmla z26.s, z7.h, z4.h\n" + ".inst 0x6465e4fd // bfmmla z29.s, z7.h, z5.h\n" + "ld1h { z5.h }, p0/Z, [x21]\n" + "ld1h { z4.h }, p0/Z, [x21, #1, MUL VL]\n" ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - ".inst 0x6467e40c // bfmmla z12.s, z0.h, z7.h\n" + ".inst 0x6463e40c // bfmmla z12.s, z0.h, z3.h\n" ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" - ".inst 0x6467e432 // bfmmla z18.s, z1.h, z7.h\n" + ".inst 0x6463e432 // bfmmla z18.s, z1.h, z3.h\n" "add %x[Apanel], %x[Apanel], #0x10\n" ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" - ".inst 0x6467e458 // bfmmla z24.s, z2.h, z7.h\n" + ".inst 0x6463e458 // bfmmla z24.s, z2.h, z3.h\n" "addvl x22, x22, #2\n" - ".inst 0x6466e47b // bfmmla z27.s, z3.h, z6.h\n" - ".inst 0x6467e47e // bfmmla z30.s, z3.h, z7.h\n" + ".inst 0x6466e4fb // bfmmla z27.s, z7.h, z6.h\n" + ".inst 0x6463e4fe // bfmmla z30.s, z7.h, z3.h\n" "addvl x21, x21, #2\n" - ".inst 0x6464e40a // bfmmla z10.s, z0.h, z4.h\n" - ".inst 0x6465e40d // bfmmla z13.s, z0.h, z5.h\n" - ".inst 0x6464e430 // bfmmla z16.s, z1.h, z4.h\n" - ".inst 0x6465e433 // bfmmla z19.s, z1.h, z5.h\n" - ".inst 0x6464e456 // bfmmla z22.s, z2.h, z4.h\n" - ".inst 0x6465e459 // bfmmla z25.s, z2.h, z5.h\n" - ".inst 0x6464e47c // bfmmla z28.s, z3.h, z4.h\n" - ".inst 0x6465e47f // bfmmla z31.s, z3.h, z5.h\n" + ".inst 0x6465e40a // bfmmla z10.s, z0.h, z5.h\n" + ".inst 0x6464e40d // bfmmla z13.s, z0.h, z4.h\n" + ".inst 0x6465e430 // bfmmla z16.s, z1.h, z5.h\n" + ".inst 0x6464e433 // bfmmla z19.s, z1.h, z4.h\n" + ".inst 0x6465e456 // bfmmla z22.s, z2.h, z5.h\n" + ".inst 0x6464e459 // bfmmla z25.s, z2.h, z4.h\n" + ".inst 0x6465e4fc // bfmmla z28.s, z7.h, z5.h\n" + ".inst 0x6464e4ff // bfmmla z31.s, z7.h, z4.h\n" "cbz x20, 6f\n" - "ld1h { z6.h }, p0/Z, [x26]\n" - "ld1rqh { z0.h }, p0/Z, [%x[Apanel]]\n" - ".inst 0x6466e408 // bfmmla z8.s, z0.h, z6.h\n" - "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #16]\n" - "ld1h { z7.h }, p0/Z, [x26, #1, MUL VL]\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" - "ld1rqh { z2.h }, p0/Z, [%x[Apanel], #32]\n" - "ld1rqh { z3.h }, p0/Z, [%x[Apanel], #48]\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - ".inst 0x6467e431 // bfmmla z17.s, z1.h, z7.h\n" - ".inst 0x6466e454 // bfmmla z20.s, z2.h, z6.h\n" - "ld1h { z4.h }, p0/Z, [x22]\n" - ".inst 0x6467e457 // bfmmla z23.s, z2.h, z7.h\n" - ".inst 0x6466e47a // bfmmla z26.s, z3.h, z6.h\n" - "ld1h { z5.h }, p0/Z, [x22, #1, MUL VL]\n" - ".inst 0x6467e47d // bfmmla z29.s, z3.h, z7.h\n" - "ld1h { z6.h }, p0/Z, [x21]\n" - "ld1h { z7.h }, p0/Z, [x21, #1, MUL VL]\n" - ".inst 0x6464e409 // bfmmla z9.s, z0.h, z4.h\n" - ".inst 0x6465e40c // bfmmla z12.s, z0.h, z5.h\n" + "ld1h { z1.h }, p0/Z, [x24]\n" + "ld1rqh { z7.h }, p0/Z, [%x[Apanel]]\n" + ".inst 0x6461e4e8 // bfmmla z8.s, z7.h, z1.h\n" + "ld1rqh { z6.h }, p0/Z, [%x[Apanel], #16]\n" + "ld1h { z0.h }, p0/Z, [x24, #1, MUL VL]\n" + ".inst 0x6460e4eb // bfmmla z11.s, z7.h, z0.h\n" + "ld1rqh { z5.h }, p0/Z, [%x[Apanel], #32]\n" + "ld1rqh { z4.h }, p0/Z, [%x[Apanel], #48]\n" + ".inst 0x6461e4ce // bfmmla z14.s, z6.h, z1.h\n" + ".inst 0x6460e4d1 // bfmmla z17.s, z6.h, z0.h\n" + ".inst 0x6461e4b4 // bfmmla z20.s, z5.h, z1.h\n" + "ld1h { z3.h }, p0/Z, [x22]\n" + ".inst 0x6460e4b7 // bfmmla z23.s, z5.h, z0.h\n" + ".inst 0x6461e49a // bfmmla z26.s, z4.h, z1.h\n" + "ld1h { z2.h }, p0/Z, [x22, #1, MUL VL]\n" + ".inst 0x6460e49d // bfmmla z29.s, z4.h, z0.h\n" + "ld1h { z1.h }, p0/Z, [x21]\n" + "ld1h { z0.h }, p0/Z, [x21, #1, MUL VL]\n" + ".inst 0x6463e4e9 // bfmmla z9.s, z7.h, z3.h\n" + ".inst 0x6462e4ec // bfmmla z12.s, z7.h, z2.h\n" "add %x[Apanel], %x[Apanel], #0x40\n" - ".inst 0x6464e42f // bfmmla z15.s, z1.h, z4.h\n" - ".inst 0x6465e432 // bfmmla z18.s, z1.h, z5.h\n" - ".inst 0x6464e455 // bfmmla z21.s, z2.h, z4.h\n" - ".inst 0x6465e458 // bfmmla z24.s, z2.h, z5.h\n" - ".inst 0x6464e47b // bfmmla z27.s, z3.h, z4.h\n" - ".inst 0x6465e47e // bfmmla z30.s, z3.h, z5.h\n" - ".inst 0x6466e40a // bfmmla z10.s, z0.h, z6.h\n" - ".inst 0x6467e40d // bfmmla z13.s, z0.h, z7.h\n" - ".inst 0x6466e430 // bfmmla z16.s, z1.h, z6.h\n" - ".inst 0x6467e433 // bfmmla z19.s, z1.h, z7.h\n" - ".inst 0x6466e456 // bfmmla z22.s, z2.h, z6.h\n" - ".inst 0x6467e459 // bfmmla z25.s, z2.h, z7.h\n" - ".inst 0x6466e47c // bfmmla z28.s, z3.h, z6.h\n" - ".inst 0x6467e47f // bfmmla z31.s, z3.h, z7.h\n" + ".inst 0x6463e4cf // bfmmla z15.s, z6.h, z3.h\n" + ".inst 0x6462e4d2 // bfmmla z18.s, z6.h, z2.h\n" + ".inst 0x6463e4b5 // bfmmla z21.s, z5.h, z3.h\n" + ".inst 0x6462e4b8 // bfmmla z24.s, z5.h, z2.h\n" + ".inst 0x6463e49b // bfmmla z27.s, z4.h, z3.h\n" + ".inst 0x6462e49e // bfmmla z30.s, z4.h, z2.h\n" + ".inst 0x6461e4ea // bfmmla z10.s, z7.h, z1.h\n" + ".inst 0x6460e4ed // bfmmla z13.s, z7.h, z0.h\n" + ".inst 0x6461e4d0 // bfmmla z16.s, z6.h, z1.h\n" + ".inst 0x6460e4d3 // bfmmla z19.s, z6.h, z0.h\n" + ".inst 0x6461e4b6 // bfmmla z22.s, z5.h, z1.h\n" + ".inst 0x6460e4b9 // bfmmla z25.s, z5.h, z0.h\n" + ".inst 0x6461e49c // bfmmla z28.s, z4.h, z1.h\n" + ".inst 0x6460e49f // bfmmla z31.s, z4.h, z0.h\n" "6:" // multiply loop done - "decw x25, ALL, MUL #3\n" - "uzp1 z4.d, z8.d, z11.d\n" + "decw x26, ALL, MUL #3\n" + "uzp1 z0.d, z8.d, z11.d\n" "uzp2 z8.d, z8.d, z11.d\n" - "uzp1 z11.d, z9.d, z12.d\n" + "uzp1 z1.d, z9.d, z12.d\n" "uzp2 z9.d, z9.d, z12.d\n" - "st1w { z4.s }, p0, [%x[Cpanel]]\n" - "uzp1 z12.d, z10.d, z13.d\n" + "st1w { z0.s }, p0, [%x[Cpanel]]\n" + "uzp1 z0.d, z10.d, z13.d\n" "uzp2 z10.d, z10.d, z13.d\n" - "st1w { z11.s }, p0, [%x[Cpanel], #1, MUL VL]\n" - "st1w { z12.s }, p0, [%x[Cpanel], #2, MUL VL]\n" - "uzp1 z13.d, z14.d, z17.d\n" + "st1w { z1.s }, p0, [%x[Cpanel], #1, MUL VL]\n" + "st1w { z0.s }, p0, [%x[Cpanel], #2, MUL VL]\n" + "uzp1 z2.d, z14.d, z17.d\n" "uzp2 z14.d, z14.d, z17.d\n" "st1w { z8.s }, p0, [%x[Cpanel], #3, MUL VL]\n" - "uzp1 z17.d, z15.d, z18.d\n" - "cmp x25, XZR\n" + "uzp1 z1.d, z15.d, z18.d\n" + "cmp x26, XZR\n" "st1w { z9.s }, p0, [%x[Cpanel], #4, MUL VL]\n" "uzp2 z15.d, z15.d, z18.d\n" - "uzp1 z18.d, z16.d, z19.d\n" + "uzp1 z17.d, z16.d, z19.d\n" "st1w { z10.s }, p0, [%x[Cpanel], #5, MUL VL]\n" "uzp2 z16.d, z16.d, z19.d\n" - "uzp1 z19.d, z20.d, z23.d\n" - "st1w { z13.s }, p0, [%x[Cpanel], #6, MUL VL]\n" + "uzp1 z0.d, z20.d, z23.d\n" + "st1w { z2.s }, p0, [%x[Cpanel], #6, MUL VL]\n" "uzp2 z20.d, z20.d, z23.d\n" "uzp1 z23.d, z21.d, z24.d\n" - "st1w { z17.s }, p0, [%x[Cpanel], #7, MUL VL]\n" + "st1w { z1.s }, p0, [%x[Cpanel], #7, MUL VL]\n" "addvl %x[Cpanel], %x[Cpanel], #16\n" "uzp2 z21.d, z21.d, z24.d\n" - "st1w { z18.s }, p0, [%x[Cpanel], #-8, MUL VL]\n" - "uzp1 z24.d, z22.d, z25.d\n" + "st1w { z17.s }, p0, [%x[Cpanel], #-8, MUL VL]\n" + "uzp1 z19.d, z22.d, z25.d\n" "uzp2 z22.d, z22.d, z25.d\n" "st1w { z14.s }, p0, [%x[Cpanel], #-7, MUL VL]\n" - "uzp1 z25.d, z26.d, z29.d\n" + "uzp1 z18.d, z26.d, z29.d\n" "uzp2 z26.d, z26.d, z29.d\n" "st1w { z15.s }, p0, [%x[Cpanel], #-6, MUL VL]\n" - "uzp1 z29.d, z27.d, z30.d\n" + "uzp1 z17.d, z27.d, z30.d\n" "uzp2 z27.d, z27.d, z30.d\n" "st1w { z16.s }, p0, [%x[Cpanel], #-5, MUL VL]\n" - "uzp1 z30.d, z28.d, z31.d\n" + "uzp1 z16.d, z28.d, z31.d\n" "uzp2 z28.d, z28.d, z31.d\n" - "st1w { z19.s }, p0, [%x[Cpanel], #-4, MUL VL]\n" + "st1w { z0.s }, p0, [%x[Cpanel], #-4, MUL VL]\n" "st1w { z23.s }, p0, [%x[Cpanel], #-3, MUL VL]\n" - "st1w { z24.s }, p0, [%x[Cpanel], #-2, MUL VL]\n" + "st1w { z19.s }, p0, [%x[Cpanel], #-2, MUL VL]\n" "st1w { z20.s }, p0, [%x[Cpanel], #-1, MUL VL]\n" "st1w { z21.s }, p0, [%x[Cpanel]]\n" "st1w { z22.s }, p0, [%x[Cpanel], #1, MUL VL]\n" - "st1w { z25.s }, p0, [%x[Cpanel], #2, MUL VL]\n" - "st1w { z29.s }, p0, [%x[Cpanel], #3, MUL VL]\n" - "st1w { z30.s }, p0, [%x[Cpanel], #4, MUL VL]\n" + "st1w { z18.s }, p0, [%x[Cpanel], #2, MUL VL]\n" + "st1w { z17.s }, p0, [%x[Cpanel], #3, MUL VL]\n" + "st1w { z16.s }, p0, [%x[Cpanel], #4, MUL VL]\n" "st1w { z26.s }, p0, [%x[Cpanel], #5, MUL VL]\n" "st1w { z27.s }, p0, [%x[Cpanel], #6, MUL VL]\n" "st1w { z28.s }, p0, [%x[Cpanel], #7, MUL VL]\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp16_mla_8x3VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp16_mla_8x3VL.hpp index 6d36bf8bbf0074ba26a2d13a444cab321379864c..60f1b699c3cc486d1204bba3ad5f1b5eff9b9630 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp16_mla_8x3VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp16_mla_8x3VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,16 +10,16 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once #ifdef ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp16_mla_8x3VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp16_mla_8x3VL/a64fx.cpp index 1d502f53543e7e00456ea38d3bc98db17f9e06eb..69ddb21c31e3bb47ab18ca5ea315d96c238cabca 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp16_mla_8x3VL/a64fx.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp16_mla_8x3VL/a64fx.cpp @@ -52,33 +52,33 @@ void sve_ffinterleaved_fp16_mla_8x3VL_a64fx( __asm__ __volatile__( "ptrue p0.b\n" "1:" // Height loop - "ldr x26, [%x[args_ptr], %[offsetof_Bpanel]]\n" - "ldr x25, [%x[args_ptr], %[offsetof_N]]\n" - "str x26, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" - "mov x24, %x[Apanel]\n" + "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n" + "ldr x26, [%x[args_ptr], %[offsetof_N]]\n" + "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" + "mov x25, %x[Apanel]\n" "2:" // Width loop - "ldr x26, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" + "ldr x24, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n" "cnth x23, ALL, MUL #2\n" - "add x22, x26, x20, LSL #1\n" + "add x22, x24, x20, LSL #1\n" "add x21, x22, x20, LSL #1\n" "add x20, x21, x20, LSL #1\n" - "cmp x25, x23\n" + "cmp x26, x23\n" "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" - "mov %x[Apanel], x24\n" + "mov %x[Apanel], x25\n" "bgt 3f\n" "dech x23\n" - "cmp x25, x23\n" - "mov x21, x26\n" + "cmp x26, x23\n" + "mov x21, x24\n" "bgt 3f\n" - "mov x22, x26\n" + "mov x22, x24\n" "3:" // B setup done "ldr x20, [%x[args_ptr], %[offsetof_K]]\n" "cmp x20, #0x2\n" "mov z8.b, #0x0\n" "mov z9.b, #0x0\n" "mov z10.b, #0x0\n" - "ld1h { z0.h }, p0/Z, [x26]\n" + "ld1h { z0.h }, p0/Z, [x24]\n" "mov z11.b, #0x0\n" "mov z12.b, #0x0\n" "ld1h { z1.h }, p0/Z, [x22]\n" @@ -116,12 +116,12 @@ void sve_ffinterleaved_fp16_mla_8x3VL_a64fx( "fmla z11.h, p0/M, z0.h, z4.h\n" "fmla z12.h, p0/M, z1.h, z4.h\n" "fmla z13.h, p0/M, z2.h, z4.h\n" - "ld1rh { z4.h }, p0/Z, [%x[Apanel], #10]\n" + "ld1rh { z7.h }, p0/Z, [%x[Apanel], #10]\n" "fmla z14.h, p0/M, z0.h, z5.h\n" "fmla z15.h, p0/M, z1.h, z5.h\n" "cmp x20, #0x2\n" "fmla z16.h, p0/M, z2.h, z5.h\n" - "ld1rh { z5.h }, p0/Z, [%x[Apanel], #12]\n" + "ld1rh { z4.h }, p0/Z, [%x[Apanel], #12]\n" "fmla z17.h, p0/M, z0.h, z6.h\n" "fmla z18.h, p0/M, z1.h, z6.h\n" "fmla z19.h, p0/M, z2.h, z6.h\n" @@ -130,57 +130,57 @@ void sve_ffinterleaved_fp16_mla_8x3VL_a64fx( "fmla z21.h, p0/M, z1.h, z3.h\n" "fmla z22.h, p0/M, z2.h, z3.h\n" "ld1rh { z3.h }, p0/Z, [%x[Apanel], #16]\n" - "fmla z23.h, p0/M, z0.h, z4.h\n" - "fmla z24.h, p0/M, z1.h, z4.h\n" - "fmla z25.h, p0/M, z2.h, z4.h\n" - "ld1rh { z4.h }, p0/Z, [%x[Apanel], #18]\n" - "fmla z26.h, p0/M, z0.h, z5.h\n" - "fmla z27.h, p0/M, z1.h, z5.h\n" - "fmla z28.h, p0/M, z2.h, z5.h\n" - "ld1rh { z5.h }, p0/Z, [%x[Apanel], #20]\n" + "fmla z23.h, p0/M, z0.h, z7.h\n" + "fmla z24.h, p0/M, z1.h, z7.h\n" + "fmla z25.h, p0/M, z2.h, z7.h\n" + "ld1rh { z5.h }, p0/Z, [%x[Apanel], #18]\n" + "fmla z26.h, p0/M, z0.h, z4.h\n" + "fmla z27.h, p0/M, z1.h, z4.h\n" + "fmla z28.h, p0/M, z2.h, z4.h\n" + "ld1rh { z4.h }, p0/Z, [%x[Apanel], #20]\n" "fmla z29.h, p0/M, z0.h, z6.h\n" - "ld1h { z0.h }, p0/Z, [x26, #1, MUL VL]\n" + "ld1h { z7.h }, p0/Z, [x24, #1, MUL VL]\n" "fmla z30.h, p0/M, z1.h, z6.h\n" "fmla z31.h, p0/M, z2.h, z6.h\n" - "ld1h { z1.h }, p0/Z, [x22, #1, MUL VL]\n" + "ld1h { z6.h }, p0/Z, [x22, #1, MUL VL]\n" "ld1h { z2.h }, p0/Z, [x21, #1, MUL VL]\n" - "fmla z8.h, p0/M, z0.h, z3.h\n" - "ld1rh { z6.h }, p0/Z, [%x[Apanel], #22]\n" - "fmla z9.h, p0/M, z1.h, z3.h\n" + "fmla z8.h, p0/M, z7.h, z3.h\n" + "ld1rh { z1.h }, p0/Z, [%x[Apanel], #22]\n" + "fmla z9.h, p0/M, z6.h, z3.h\n" "fmla z10.h, p0/M, z2.h, z3.h\n" - "fmla z11.h, p0/M, z0.h, z4.h\n" + "fmla z11.h, p0/M, z7.h, z5.h\n" "ld1rh { z3.h }, p0/Z, [%x[Apanel], #24]\n" - "fmla z12.h, p0/M, z1.h, z4.h\n" - "fmla z13.h, p0/M, z2.h, z4.h\n" - "ld1rh { z4.h }, p0/Z, [%x[Apanel], #26]\n" - "fmla z14.h, p0/M, z0.h, z5.h\n" - "fmla z15.h, p0/M, z1.h, z5.h\n" - "addvl x26, x26, #2\n" - "fmla z16.h, p0/M, z2.h, z5.h\n" - "ld1rh { z5.h }, p0/Z, [%x[Apanel], #28]\n" - "fmla z17.h, p0/M, z0.h, z6.h\n" - "fmla z18.h, p0/M, z1.h, z6.h\n" - "fmla z19.h, p0/M, z2.h, z6.h\n" - "ld1rh { z6.h }, p0/Z, [%x[Apanel], #30]\n" + "fmla z12.h, p0/M, z6.h, z5.h\n" + "fmla z13.h, p0/M, z2.h, z5.h\n" + "ld1rh { z5.h }, p0/Z, [%x[Apanel], #26]\n" + "fmla z14.h, p0/M, z7.h, z4.h\n" + "fmla z15.h, p0/M, z6.h, z4.h\n" + "addvl x24, x24, #2\n" + "fmla z16.h, p0/M, z2.h, z4.h\n" + "ld1rh { z0.h }, p0/Z, [%x[Apanel], #28]\n" + "fmla z17.h, p0/M, z7.h, z1.h\n" + "fmla z18.h, p0/M, z6.h, z1.h\n" + "fmla z19.h, p0/M, z2.h, z1.h\n" + "ld1rh { z1.h }, p0/Z, [%x[Apanel], #30]\n" "addvl x22, x22, #2\n" "addvl x21, x21, #2\n" "add %x[Apanel], %x[Apanel], #0x20\n" - "fmla z20.h, p0/M, z0.h, z3.h\n" - "fmla z21.h, p0/M, z1.h, z3.h\n" + "fmla z20.h, p0/M, z7.h, z3.h\n" + "fmla z21.h, p0/M, z6.h, z3.h\n" "fmla z22.h, p0/M, z2.h, z3.h\n" "ld1rh { z3.h }, p0/Z, [%x[Apanel]]\n" - "fmla z23.h, p0/M, z0.h, z4.h\n" - "fmla z24.h, p0/M, z1.h, z4.h\n" - "fmla z25.h, p0/M, z2.h, z4.h\n" - "fmla z26.h, p0/M, z0.h, z5.h\n" + "fmla z23.h, p0/M, z7.h, z5.h\n" + "fmla z24.h, p0/M, z6.h, z5.h\n" + "fmla z25.h, p0/M, z2.h, z5.h\n" + "fmla z26.h, p0/M, z7.h, z0.h\n" "ld1rh { z4.h }, p0/Z, [%x[Apanel], #2]\n" - "fmla z27.h, p0/M, z1.h, z5.h\n" - "fmla z28.h, p0/M, z2.h, z5.h\n" + "fmla z27.h, p0/M, z6.h, z0.h\n" + "fmla z28.h, p0/M, z2.h, z0.h\n" "ld1rh { z5.h }, p0/Z, [%x[Apanel], #4]\n" - "fmla z29.h, p0/M, z0.h, z6.h\n" - "ld1h { z0.h }, p0/Z, [x26]\n" - "fmla z30.h, p0/M, z1.h, z6.h\n" - "fmla z31.h, p0/M, z2.h, z6.h\n" + "fmla z29.h, p0/M, z7.h, z1.h\n" + "ld1h { z0.h }, p0/Z, [x24]\n" + "fmla z30.h, p0/M, z6.h, z1.h\n" + "fmla z31.h, p0/M, z2.h, z1.h\n" "ld1h { z1.h }, p0/Z, [x22]\n" "ld1h { z2.h }, p0/Z, [x21]\n" "ld1rh { z6.h }, p0/Z, [%x[Apanel], #6]\n" @@ -188,9 +188,9 @@ void sve_ffinterleaved_fp16_mla_8x3VL_a64fx( "5:" // main loop skip "fmla z8.h, p0/M, z0.h, z3.h\n" "fmla z9.h, p0/M, z1.h, z3.h\n" - "addvl x26, x26, #1\n" + "addvl x24, x24, #1\n" "fmla z10.h, p0/M, z2.h, z3.h\n" - "ld1rh { z3.h }, p0/Z, [%x[Apanel], #8]\n" + "ld1rh { z7.h }, p0/Z, [%x[Apanel], #8]\n" "fmla z11.h, p0/M, z0.h, z4.h\n" "fmla z12.h, p0/M, z1.h, z4.h\n" "fmla z13.h, p0/M, z2.h, z4.h\n" @@ -203,11 +203,11 @@ void sve_ffinterleaved_fp16_mla_8x3VL_a64fx( "fmla z17.h, p0/M, z0.h, z6.h\n" "fmla z18.h, p0/M, z1.h, z6.h\n" "fmla z19.h, p0/M, z2.h, z6.h\n" - "ld1rh { z6.h }, p0/Z, [%x[Apanel], #14]\n" - "fmla z20.h, p0/M, z0.h, z3.h\n" - "fmla z21.h, p0/M, z1.h, z3.h\n" + "ld1rh { z3.h }, p0/Z, [%x[Apanel], #14]\n" + "fmla z20.h, p0/M, z0.h, z7.h\n" + "fmla z21.h, p0/M, z1.h, z7.h\n" "addvl x21, x21, #1\n" - "fmla z22.h, p0/M, z2.h, z3.h\n" + "fmla z22.h, p0/M, z2.h, z7.h\n" "fmla z23.h, p0/M, z0.h, z4.h\n" "add %x[Apanel], %x[Apanel], #0x10\n" "fmla z24.h, p0/M, z1.h, z4.h\n" @@ -215,50 +215,50 @@ void sve_ffinterleaved_fp16_mla_8x3VL_a64fx( "fmla z26.h, p0/M, z0.h, z5.h\n" "fmla z27.h, p0/M, z1.h, z5.h\n" "fmla z28.h, p0/M, z2.h, z5.h\n" - "fmla z29.h, p0/M, z0.h, z6.h\n" - "fmla z30.h, p0/M, z1.h, z6.h\n" - "fmla z31.h, p0/M, z2.h, z6.h\n" + "fmla z29.h, p0/M, z0.h, z3.h\n" + "fmla z30.h, p0/M, z1.h, z3.h\n" + "fmla z31.h, p0/M, z2.h, z3.h\n" "cbz x20, 6f\n" - "ld1h { z0.h }, p0/Z, [x26]\n" - "ld1h { z1.h }, p0/Z, [x22]\n" - "ld1h { z2.h }, p0/Z, [x21]\n" + "ld1h { z6.h }, p0/Z, [x24]\n" + "ld1h { z5.h }, p0/Z, [x22]\n" + "ld1h { z4.h }, p0/Z, [x21]\n" "ld1rh { z3.h }, p0/Z, [%x[Apanel]]\n" - "fmla z8.h, p0/M, z0.h, z3.h\n" - "ld1rh { z4.h }, p0/Z, [%x[Apanel], #2]\n" - "ld1rh { z5.h }, p0/Z, [%x[Apanel], #4]\n" - "fmla z9.h, p0/M, z1.h, z3.h\n" - "ld1rh { z6.h }, p0/Z, [%x[Apanel], #6]\n" - "fmla z10.h, p0/M, z2.h, z3.h\n" - "fmla z11.h, p0/M, z0.h, z4.h\n" - "fmla z12.h, p0/M, z1.h, z4.h\n" - "fmla z13.h, p0/M, z2.h, z4.h\n" + "fmla z8.h, p0/M, z6.h, z3.h\n" + "ld1rh { z2.h }, p0/Z, [%x[Apanel], #2]\n" + "ld1rh { z1.h }, p0/Z, [%x[Apanel], #4]\n" + "fmla z9.h, p0/M, z5.h, z3.h\n" + "ld1rh { z0.h }, p0/Z, [%x[Apanel], #6]\n" + "fmla z10.h, p0/M, z4.h, z3.h\n" + "fmla z11.h, p0/M, z6.h, z2.h\n" + "fmla z12.h, p0/M, z5.h, z2.h\n" + "fmla z13.h, p0/M, z4.h, z2.h\n" "ld1rh { z3.h }, p0/Z, [%x[Apanel], #8]\n" - "fmla z14.h, p0/M, z0.h, z5.h\n" - "fmla z15.h, p0/M, z1.h, z5.h\n" - "ld1rh { z4.h }, p0/Z, [%x[Apanel], #10]\n" - "fmla z16.h, p0/M, z2.h, z5.h\n" - "fmla z17.h, p0/M, z0.h, z6.h\n" - "ld1rh { z5.h }, p0/Z, [%x[Apanel], #12]\n" - "fmla z18.h, p0/M, z1.h, z6.h\n" - "fmla z19.h, p0/M, z2.h, z6.h\n" - "ld1rh { z6.h }, p0/Z, [%x[Apanel], #14]\n" - "fmla z20.h, p0/M, z0.h, z3.h\n" - "fmla z21.h, p0/M, z1.h, z3.h\n" + "fmla z14.h, p0/M, z6.h, z1.h\n" + "fmla z15.h, p0/M, z5.h, z1.h\n" + "ld1rh { z2.h }, p0/Z, [%x[Apanel], #10]\n" + "fmla z16.h, p0/M, z4.h, z1.h\n" + "fmla z17.h, p0/M, z6.h, z0.h\n" + "ld1rh { z1.h }, p0/Z, [%x[Apanel], #12]\n" + "fmla z18.h, p0/M, z5.h, z0.h\n" + "fmla z19.h, p0/M, z4.h, z0.h\n" + "ld1rh { z0.h }, p0/Z, [%x[Apanel], #14]\n" + "fmla z20.h, p0/M, z6.h, z3.h\n" + "fmla z21.h, p0/M, z5.h, z3.h\n" "add %x[Apanel], %x[Apanel], #0x10\n" - "fmla z22.h, p0/M, z2.h, z3.h\n" - "fmla z23.h, p0/M, z0.h, z4.h\n" - "fmla z24.h, p0/M, z1.h, z4.h\n" - "fmla z25.h, p0/M, z2.h, z4.h\n" - "fmla z26.h, p0/M, z0.h, z5.h\n" - "fmla z27.h, p0/M, z1.h, z5.h\n" - "fmla z28.h, p0/M, z2.h, z5.h\n" - "fmla z29.h, p0/M, z0.h, z6.h\n" - "fmla z30.h, p0/M, z1.h, z6.h\n" - "fmla z31.h, p0/M, z2.h, z6.h\n" + "fmla z22.h, p0/M, z4.h, z3.h\n" + "fmla z23.h, p0/M, z6.h, z2.h\n" + "fmla z24.h, p0/M, z5.h, z2.h\n" + "fmla z25.h, p0/M, z4.h, z2.h\n" + "fmla z26.h, p0/M, z6.h, z1.h\n" + "fmla z27.h, p0/M, z5.h, z1.h\n" + "fmla z28.h, p0/M, z4.h, z1.h\n" + "fmla z29.h, p0/M, z6.h, z0.h\n" + "fmla z30.h, p0/M, z5.h, z0.h\n" + "fmla z31.h, p0/M, z4.h, z0.h\n" "6:" // multiply loop done - "dech x25, ALL, MUL #3\n" + "dech x26, ALL, MUL #3\n" "st1h { z8.h }, p0, [%x[Cpanel]]\n" - "cmp x25, XZR\n" + "cmp x26, XZR\n" "st1h { z9.h }, p0, [%x[Cpanel], #1, MUL VL]\n" "st1h { z10.h }, p0, [%x[Cpanel], #2, MUL VL]\n" "st1h { z11.h }, p0, [%x[Cpanel], #3, MUL VL]\n" @@ -289,7 +289,7 @@ void sve_ffinterleaved_fp16_mla_8x3VL_a64fx( "bne 1b\n" : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks) : [args_ptr] "r" (&ka), [offsetof_B_stride] "I" (offsetof(KernelArgs, B_stride)), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_cur_B_ptr] "I" (offsetof(KernelArgs, cur_B_ptr)) - : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp16_mla_8x3VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp16_mla_8x3VL/generic.cpp index de219aa2bf827975fac37ed8e6b1871b9529d87b..23503fa108d988846463824df589f3c27c86ce54 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp16_mla_8x3VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp16_mla_8x3VL/generic.cpp @@ -52,26 +52,26 @@ void sve_ffinterleaved_fp16_mla_8x3VL( __asm__ __volatile__( "ptrue p0.b\n" "1:" // Height loop - "ldr x26, [%x[args_ptr], %[offsetof_Bpanel]]\n" - "ldr x25, [%x[args_ptr], %[offsetof_N]]\n" - "str x26, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" - "mov x24, %x[Apanel]\n" + "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n" + "ldr x26, [%x[args_ptr], %[offsetof_N]]\n" + "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" + "mov x25, %x[Apanel]\n" "2:" // Width loop - "ldr x26, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" + "ldr x24, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n" "cnth x23, ALL, MUL #2\n" - "add x22, x26, x20, LSL #1\n" + "add x22, x24, x20, LSL #1\n" "add x21, x22, x20, LSL #1\n" "add x20, x21, x20, LSL #1\n" - "cmp x25, x23\n" + "cmp x26, x23\n" "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" - "mov %x[Apanel], x24\n" + "mov %x[Apanel], x25\n" "bgt 3f\n" "dech x23\n" - "cmp x25, x23\n" - "mov x21, x26\n" + "cmp x26, x23\n" + "mov x21, x24\n" "bgt 3f\n" - "mov x22, x26\n" + "mov x22, x24\n" "3:" // B setup done "ldr x20, [%x[args_ptr], %[offsetof_K]]\n" "cmp x20, #0x2\n" @@ -81,7 +81,7 @@ void sve_ffinterleaved_fp16_mla_8x3VL( "ld1rqh { z0.h }, p0/Z, [%x[Apanel]]\n" "mov z11.b, #0x0\n" "mov z12.b, #0x0\n" - "ld1h { z2.h }, p0/Z, [x26]\n" + "ld1h { z2.h }, p0/Z, [x24]\n" "mov z13.b, #0x0\n" "mov z14.b, #0x0\n" "ld1h { z3.h }, p0/Z, [x22]\n" @@ -107,19 +107,19 @@ void sve_ffinterleaved_fp16_mla_8x3VL( "4:" // main loop head "fmla z8.h, z2.h, z0.h[0]\n" "fmla z11.h, z2.h, z0.h[1]\n" - "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #16]\n" + "ld1rqh { z7.h }, p0/Z, [%x[Apanel], #16]\n" "fmla z14.h, z2.h, z0.h[2]\n" "fmla z17.h, z2.h, z0.h[3]\n" - "ld1h { z5.h }, p0/Z, [x26, #1, MUL VL]\n" + "ld1h { z6.h }, p0/Z, [x24, #1, MUL VL]\n" "fmla z20.h, z2.h, z0.h[4]\n" "fmla z23.h, z2.h, z0.h[5]\n" - "ld1h { z6.h }, p0/Z, [x22, #1, MUL VL]\n" + "ld1h { z5.h }, p0/Z, [x22, #1, MUL VL]\n" "fmla z26.h, z2.h, z0.h[6]\n" "fmla z29.h, z2.h, z0.h[7]\n" - "ld1h { z7.h }, p0/Z, [x21, #1, MUL VL]\n" + "ld1h { z1.h }, p0/Z, [x21, #1, MUL VL]\n" "fmla z9.h, z3.h, z0.h[0]\n" "fmla z12.h, z3.h, z0.h[1]\n" - "addvl x26, x26, #2\n" + "addvl x24, x24, #2\n" "fmla z15.h, z3.h, z0.h[2]\n" "fmla z18.h, z3.h, z0.h[3]\n" "addvl x22, x22, #2\n" @@ -137,36 +137,36 @@ void sve_ffinterleaved_fp16_mla_8x3VL( "add %x[Apanel], %x[Apanel], #0x20\n" "fmla z22.h, z4.h, z0.h[4]\n" "fmla z25.h, z4.h, z0.h[5]\n" - "ld1h { z2.h }, p0/Z, [x26]\n" + "ld1h { z2.h }, p0/Z, [x24]\n" "fmla z28.h, z4.h, z0.h[6]\n" "fmla z31.h, z4.h, z0.h[7]\n" "ld1rqh { z0.h }, p0/Z, [%x[Apanel]]\n" - "fmla z8.h, z5.h, z1.h[0]\n" - "fmla z11.h, z5.h, z1.h[1]\n" + "fmla z8.h, z6.h, z7.h[0]\n" + "fmla z11.h, z6.h, z7.h[1]\n" "ld1h { z3.h }, p0/Z, [x22]\n" - "fmla z14.h, z5.h, z1.h[2]\n" - "fmla z17.h, z5.h, z1.h[3]\n" + "fmla z14.h, z6.h, z7.h[2]\n" + "fmla z17.h, z6.h, z7.h[3]\n" "ld1h { z4.h }, p0/Z, [x21]\n" - "fmla z20.h, z5.h, z1.h[4]\n" - "fmla z23.h, z5.h, z1.h[5]\n" - "fmla z26.h, z5.h, z1.h[6]\n" - "fmla z29.h, z5.h, z1.h[7]\n" - "fmla z9.h, z6.h, z1.h[0]\n" - "fmla z12.h, z6.h, z1.h[1]\n" - "fmla z15.h, z6.h, z1.h[2]\n" - "fmla z18.h, z6.h, z1.h[3]\n" - "fmla z21.h, z6.h, z1.h[4]\n" - "fmla z24.h, z6.h, z1.h[5]\n" - "fmla z27.h, z6.h, z1.h[6]\n" - "fmla z30.h, z6.h, z1.h[7]\n" - "fmla z10.h, z7.h, z1.h[0]\n" - "fmla z13.h, z7.h, z1.h[1]\n" - "fmla z16.h, z7.h, z1.h[2]\n" - "fmla z19.h, z7.h, z1.h[3]\n" - "fmla z22.h, z7.h, z1.h[4]\n" - "fmla z25.h, z7.h, z1.h[5]\n" - "fmla z28.h, z7.h, z1.h[6]\n" - "fmla z31.h, z7.h, z1.h[7]\n" + "fmla z20.h, z6.h, z7.h[4]\n" + "fmla z23.h, z6.h, z7.h[5]\n" + "fmla z26.h, z6.h, z7.h[6]\n" + "fmla z29.h, z6.h, z7.h[7]\n" + "fmla z9.h, z5.h, z7.h[0]\n" + "fmla z12.h, z5.h, z7.h[1]\n" + "fmla z15.h, z5.h, z7.h[2]\n" + "fmla z18.h, z5.h, z7.h[3]\n" + "fmla z21.h, z5.h, z7.h[4]\n" + "fmla z24.h, z5.h, z7.h[5]\n" + "fmla z27.h, z5.h, z7.h[6]\n" + "fmla z30.h, z5.h, z7.h[7]\n" + "fmla z10.h, z1.h, z7.h[0]\n" + "fmla z13.h, z1.h, z7.h[1]\n" + "fmla z16.h, z1.h, z7.h[2]\n" + "fmla z19.h, z1.h, z7.h[3]\n" + "fmla z22.h, z1.h, z7.h[4]\n" + "fmla z25.h, z1.h, z7.h[5]\n" + "fmla z28.h, z1.h, z7.h[6]\n" + "fmla z31.h, z1.h, z7.h[7]\n" "bge 4b\n" "5:" // main loop skip "fmla z8.h, z2.h, z0.h[0]\n" @@ -174,7 +174,7 @@ void sve_ffinterleaved_fp16_mla_8x3VL( "add %x[Apanel], %x[Apanel], #0x10\n" "fmla z14.h, z2.h, z0.h[2]\n" "fmla z17.h, z2.h, z0.h[3]\n" - "addvl x26, x26, #1\n" + "addvl x24, x24, #1\n" "fmla z20.h, z2.h, z0.h[4]\n" "fmla z23.h, z2.h, z0.h[5]\n" "addvl x22, x22, #1\n" @@ -198,39 +198,39 @@ void sve_ffinterleaved_fp16_mla_8x3VL( "fmla z28.h, z4.h, z0.h[6]\n" "fmla z31.h, z4.h, z0.h[7]\n" "cbz x20, 6f\n" - "ld1rqh { z0.h }, p0/Z, [%x[Apanel]]\n" - "ld1h { z5.h }, p0/Z, [x26]\n" - "fmla z8.h, z5.h, z0.h[0]\n" - "ld1h { z6.h }, p0/Z, [x22]\n" - "ld1h { z7.h }, p0/Z, [x21]\n" - "fmla z11.h, z5.h, z0.h[1]\n" - "fmla z14.h, z5.h, z0.h[2]\n" - "fmla z17.h, z5.h, z0.h[3]\n" + "ld1rqh { z3.h }, p0/Z, [%x[Apanel]]\n" + "ld1h { z2.h }, p0/Z, [x24]\n" + "fmla z8.h, z2.h, z3.h[0]\n" + "ld1h { z1.h }, p0/Z, [x22]\n" + "ld1h { z0.h }, p0/Z, [x21]\n" + "fmla z11.h, z2.h, z3.h[1]\n" + "fmla z14.h, z2.h, z3.h[2]\n" + "fmla z17.h, z2.h, z3.h[3]\n" "add %x[Apanel], %x[Apanel], #0x10\n" - "fmla z20.h, z5.h, z0.h[4]\n" - "fmla z23.h, z5.h, z0.h[5]\n" - "fmla z26.h, z5.h, z0.h[6]\n" - "fmla z29.h, z5.h, z0.h[7]\n" - "fmla z9.h, z6.h, z0.h[0]\n" - "fmla z12.h, z6.h, z0.h[1]\n" - "fmla z15.h, z6.h, z0.h[2]\n" - "fmla z18.h, z6.h, z0.h[3]\n" - "fmla z21.h, z6.h, z0.h[4]\n" - "fmla z24.h, z6.h, z0.h[5]\n" - "fmla z27.h, z6.h, z0.h[6]\n" - "fmla z30.h, z6.h, z0.h[7]\n" - "fmla z10.h, z7.h, z0.h[0]\n" - "fmla z13.h, z7.h, z0.h[1]\n" - "fmla z16.h, z7.h, z0.h[2]\n" - "fmla z19.h, z7.h, z0.h[3]\n" - "fmla z22.h, z7.h, z0.h[4]\n" - "fmla z25.h, z7.h, z0.h[5]\n" - "fmla z28.h, z7.h, z0.h[6]\n" - "fmla z31.h, z7.h, z0.h[7]\n" + "fmla z20.h, z2.h, z3.h[4]\n" + "fmla z23.h, z2.h, z3.h[5]\n" + "fmla z26.h, z2.h, z3.h[6]\n" + "fmla z29.h, z2.h, z3.h[7]\n" + "fmla z9.h, z1.h, z3.h[0]\n" + "fmla z12.h, z1.h, z3.h[1]\n" + "fmla z15.h, z1.h, z3.h[2]\n" + "fmla z18.h, z1.h, z3.h[3]\n" + "fmla z21.h, z1.h, z3.h[4]\n" + "fmla z24.h, z1.h, z3.h[5]\n" + "fmla z27.h, z1.h, z3.h[6]\n" + "fmla z30.h, z1.h, z3.h[7]\n" + "fmla z10.h, z0.h, z3.h[0]\n" + "fmla z13.h, z0.h, z3.h[1]\n" + "fmla z16.h, z0.h, z3.h[2]\n" + "fmla z19.h, z0.h, z3.h[3]\n" + "fmla z22.h, z0.h, z3.h[4]\n" + "fmla z25.h, z0.h, z3.h[5]\n" + "fmla z28.h, z0.h, z3.h[6]\n" + "fmla z31.h, z0.h, z3.h[7]\n" "6:" // multiply loop done - "dech x25, ALL, MUL #3\n" + "dech x26, ALL, MUL #3\n" "st1h { z8.h }, p0, [%x[Cpanel]]\n" - "cmp x25, XZR\n" + "cmp x26, XZR\n" "st1h { z9.h }, p0, [%x[Cpanel], #1, MUL VL]\n" "st1h { z10.h }, p0, [%x[Cpanel], #2, MUL VL]\n" "st1h { z11.h }, p0, [%x[Cpanel], #3, MUL VL]\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp32_mla_8x3VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp32_mla_8x3VL.hpp index aa3507ee73030c035583ab415b4f593373fbb76f..ac6986913d33810fddedc6b8b175cfeab731b4d3 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp32_mla_8x3VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp32_mla_8x3VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,16 +10,16 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once #ifdef ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp32_mla_8x3VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp32_mla_8x3VL/a64fx.cpp index 8c8b6b067576e75c79c2caba9ceba3736601d773..c65c3a3ce403c999d24749685461fd10edac5da9 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp32_mla_8x3VL/a64fx.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp32_mla_8x3VL/a64fx.cpp @@ -52,33 +52,33 @@ void sve_ffinterleaved_fp32_mla_8x3VL_a64fx( __asm__ __volatile__( "ptrue p0.b\n" "1:" // Height loop - "ldr x26, [%x[args_ptr], %[offsetof_Bpanel]]\n" - "ldr x25, [%x[args_ptr], %[offsetof_N]]\n" - "str x26, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" - "mov x24, %x[Apanel]\n" + "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n" + "ldr x26, [%x[args_ptr], %[offsetof_N]]\n" + "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" + "mov x25, %x[Apanel]\n" "2:" // Width loop - "ldr x26, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" + "ldr x24, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n" "cntw x23, ALL, MUL #2\n" - "add x22, x26, x20, LSL #2\n" + "add x22, x24, x20, LSL #2\n" "add x21, x22, x20, LSL #2\n" "add x20, x21, x20, LSL #2\n" - "cmp x25, x23\n" + "cmp x26, x23\n" "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" - "mov %x[Apanel], x24\n" + "mov %x[Apanel], x25\n" "bgt 3f\n" "decw x23\n" - "cmp x25, x23\n" - "mov x21, x26\n" + "cmp x26, x23\n" + "mov x21, x24\n" "bgt 3f\n" - "mov x22, x26\n" + "mov x22, x24\n" "3:" // B setup done "ldr x20, [%x[args_ptr], %[offsetof_K]]\n" "cmp x20, #0x2\n" "mov z8.b, #0x0\n" "mov z9.b, #0x0\n" "mov z10.b, #0x0\n" - "ld1w { z0.s }, p0/Z, [x26]\n" + "ld1w { z0.s }, p0/Z, [x24]\n" "mov z11.b, #0x0\n" "mov z12.b, #0x0\n" "ld1w { z1.s }, p0/Z, [x22]\n" @@ -116,12 +116,12 @@ void sve_ffinterleaved_fp32_mla_8x3VL_a64fx( "fmla z11.s, p0/M, z0.s, z4.s\n" "fmla z12.s, p0/M, z1.s, z4.s\n" "fmla z13.s, p0/M, z2.s, z4.s\n" - "ld1rw { z4.s }, p0/Z, [%x[Apanel], #20]\n" + "ld1rw { z7.s }, p0/Z, [%x[Apanel], #20]\n" "fmla z14.s, p0/M, z0.s, z5.s\n" "fmla z15.s, p0/M, z1.s, z5.s\n" "cmp x20, #0x2\n" "fmla z16.s, p0/M, z2.s, z5.s\n" - "ld1rw { z5.s }, p0/Z, [%x[Apanel], #24]\n" + "ld1rw { z4.s }, p0/Z, [%x[Apanel], #24]\n" "fmla z17.s, p0/M, z0.s, z6.s\n" "fmla z18.s, p0/M, z1.s, z6.s\n" "fmla z19.s, p0/M, z2.s, z6.s\n" @@ -130,57 +130,57 @@ void sve_ffinterleaved_fp32_mla_8x3VL_a64fx( "fmla z21.s, p0/M, z1.s, z3.s\n" "fmla z22.s, p0/M, z2.s, z3.s\n" "ld1rw { z3.s }, p0/Z, [%x[Apanel], #32]\n" - "fmla z23.s, p0/M, z0.s, z4.s\n" - "fmla z24.s, p0/M, z1.s, z4.s\n" - "fmla z25.s, p0/M, z2.s, z4.s\n" - "ld1rw { z4.s }, p0/Z, [%x[Apanel], #36]\n" - "fmla z26.s, p0/M, z0.s, z5.s\n" - "fmla z27.s, p0/M, z1.s, z5.s\n" - "fmla z28.s, p0/M, z2.s, z5.s\n" - "ld1rw { z5.s }, p0/Z, [%x[Apanel], #40]\n" + "fmla z23.s, p0/M, z0.s, z7.s\n" + "fmla z24.s, p0/M, z1.s, z7.s\n" + "fmla z25.s, p0/M, z2.s, z7.s\n" + "ld1rw { z5.s }, p0/Z, [%x[Apanel], #36]\n" + "fmla z26.s, p0/M, z0.s, z4.s\n" + "fmla z27.s, p0/M, z1.s, z4.s\n" + "fmla z28.s, p0/M, z2.s, z4.s\n" + "ld1rw { z4.s }, p0/Z, [%x[Apanel], #40]\n" "fmla z29.s, p0/M, z0.s, z6.s\n" - "ld1w { z0.s }, p0/Z, [x26, #1, MUL VL]\n" + "ld1w { z7.s }, p0/Z, [x24, #1, MUL VL]\n" "fmla z30.s, p0/M, z1.s, z6.s\n" "fmla z31.s, p0/M, z2.s, z6.s\n" - "ld1w { z1.s }, p0/Z, [x22, #1, MUL VL]\n" + "ld1w { z6.s }, p0/Z, [x22, #1, MUL VL]\n" "ld1w { z2.s }, p0/Z, [x21, #1, MUL VL]\n" - "fmla z8.s, p0/M, z0.s, z3.s\n" - "ld1rw { z6.s }, p0/Z, [%x[Apanel], #44]\n" - "fmla z9.s, p0/M, z1.s, z3.s\n" + "fmla z8.s, p0/M, z7.s, z3.s\n" + "ld1rw { z1.s }, p0/Z, [%x[Apanel], #44]\n" + "fmla z9.s, p0/M, z6.s, z3.s\n" "fmla z10.s, p0/M, z2.s, z3.s\n" - "fmla z11.s, p0/M, z0.s, z4.s\n" + "fmla z11.s, p0/M, z7.s, z5.s\n" "ld1rw { z3.s }, p0/Z, [%x[Apanel], #48]\n" - "fmla z12.s, p0/M, z1.s, z4.s\n" - "fmla z13.s, p0/M, z2.s, z4.s\n" - "ld1rw { z4.s }, p0/Z, [%x[Apanel], #52]\n" - "fmla z14.s, p0/M, z0.s, z5.s\n" - "fmla z15.s, p0/M, z1.s, z5.s\n" - "addvl x26, x26, #2\n" - "fmla z16.s, p0/M, z2.s, z5.s\n" - "ld1rw { z5.s }, p0/Z, [%x[Apanel], #56]\n" - "fmla z17.s, p0/M, z0.s, z6.s\n" - "fmla z18.s, p0/M, z1.s, z6.s\n" - "fmla z19.s, p0/M, z2.s, z6.s\n" - "ld1rw { z6.s }, p0/Z, [%x[Apanel], #60]\n" + "fmla z12.s, p0/M, z6.s, z5.s\n" + "fmla z13.s, p0/M, z2.s, z5.s\n" + "ld1rw { z5.s }, p0/Z, [%x[Apanel], #52]\n" + "fmla z14.s, p0/M, z7.s, z4.s\n" + "fmla z15.s, p0/M, z6.s, z4.s\n" + "addvl x24, x24, #2\n" + "fmla z16.s, p0/M, z2.s, z4.s\n" + "ld1rw { z0.s }, p0/Z, [%x[Apanel], #56]\n" + "fmla z17.s, p0/M, z7.s, z1.s\n" + "fmla z18.s, p0/M, z6.s, z1.s\n" + "fmla z19.s, p0/M, z2.s, z1.s\n" + "ld1rw { z1.s }, p0/Z, [%x[Apanel], #60]\n" "addvl x22, x22, #2\n" "addvl x21, x21, #2\n" "add %x[Apanel], %x[Apanel], #0x40\n" - "fmla z20.s, p0/M, z0.s, z3.s\n" - "fmla z21.s, p0/M, z1.s, z3.s\n" + "fmla z20.s, p0/M, z7.s, z3.s\n" + "fmla z21.s, p0/M, z6.s, z3.s\n" "fmla z22.s, p0/M, z2.s, z3.s\n" "ld1rw { z3.s }, p0/Z, [%x[Apanel]]\n" - "fmla z23.s, p0/M, z0.s, z4.s\n" - "fmla z24.s, p0/M, z1.s, z4.s\n" - "fmla z25.s, p0/M, z2.s, z4.s\n" - "fmla z26.s, p0/M, z0.s, z5.s\n" + "fmla z23.s, p0/M, z7.s, z5.s\n" + "fmla z24.s, p0/M, z6.s, z5.s\n" + "fmla z25.s, p0/M, z2.s, z5.s\n" + "fmla z26.s, p0/M, z7.s, z0.s\n" "ld1rw { z4.s }, p0/Z, [%x[Apanel], #4]\n" - "fmla z27.s, p0/M, z1.s, z5.s\n" - "fmla z28.s, p0/M, z2.s, z5.s\n" + "fmla z27.s, p0/M, z6.s, z0.s\n" + "fmla z28.s, p0/M, z2.s, z0.s\n" "ld1rw { z5.s }, p0/Z, [%x[Apanel], #8]\n" - "fmla z29.s, p0/M, z0.s, z6.s\n" - "ld1w { z0.s }, p0/Z, [x26]\n" - "fmla z30.s, p0/M, z1.s, z6.s\n" - "fmla z31.s, p0/M, z2.s, z6.s\n" + "fmla z29.s, p0/M, z7.s, z1.s\n" + "ld1w { z0.s }, p0/Z, [x24]\n" + "fmla z30.s, p0/M, z6.s, z1.s\n" + "fmla z31.s, p0/M, z2.s, z1.s\n" "ld1w { z1.s }, p0/Z, [x22]\n" "ld1w { z2.s }, p0/Z, [x21]\n" "ld1rw { z6.s }, p0/Z, [%x[Apanel], #12]\n" @@ -188,9 +188,9 @@ void sve_ffinterleaved_fp32_mla_8x3VL_a64fx( "5:" // main loop skip "fmla z8.s, p0/M, z0.s, z3.s\n" "fmla z9.s, p0/M, z1.s, z3.s\n" - "addvl x26, x26, #1\n" + "addvl x24, x24, #1\n" "fmla z10.s, p0/M, z2.s, z3.s\n" - "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n" + "ld1rw { z7.s }, p0/Z, [%x[Apanel], #16]\n" "fmla z11.s, p0/M, z0.s, z4.s\n" "fmla z12.s, p0/M, z1.s, z4.s\n" "fmla z13.s, p0/M, z2.s, z4.s\n" @@ -203,11 +203,11 @@ void sve_ffinterleaved_fp32_mla_8x3VL_a64fx( "fmla z17.s, p0/M, z0.s, z6.s\n" "fmla z18.s, p0/M, z1.s, z6.s\n" "fmla z19.s, p0/M, z2.s, z6.s\n" - "ld1rw { z6.s }, p0/Z, [%x[Apanel], #28]\n" - "fmla z20.s, p0/M, z0.s, z3.s\n" - "fmla z21.s, p0/M, z1.s, z3.s\n" + "ld1rw { z3.s }, p0/Z, [%x[Apanel], #28]\n" + "fmla z20.s, p0/M, z0.s, z7.s\n" + "fmla z21.s, p0/M, z1.s, z7.s\n" "addvl x21, x21, #1\n" - "fmla z22.s, p0/M, z2.s, z3.s\n" + "fmla z22.s, p0/M, z2.s, z7.s\n" "fmla z23.s, p0/M, z0.s, z4.s\n" "add %x[Apanel], %x[Apanel], #0x20\n" "fmla z24.s, p0/M, z1.s, z4.s\n" @@ -215,50 +215,50 @@ void sve_ffinterleaved_fp32_mla_8x3VL_a64fx( "fmla z26.s, p0/M, z0.s, z5.s\n" "fmla z27.s, p0/M, z1.s, z5.s\n" "fmla z28.s, p0/M, z2.s, z5.s\n" - "fmla z29.s, p0/M, z0.s, z6.s\n" - "fmla z30.s, p0/M, z1.s, z6.s\n" - "fmla z31.s, p0/M, z2.s, z6.s\n" + "fmla z29.s, p0/M, z0.s, z3.s\n" + "fmla z30.s, p0/M, z1.s, z3.s\n" + "fmla z31.s, p0/M, z2.s, z3.s\n" "cbz x20, 6f\n" - "ld1w { z0.s }, p0/Z, [x26]\n" - "ld1w { z1.s }, p0/Z, [x22]\n" - "ld1w { z2.s }, p0/Z, [x21]\n" + "ld1w { z6.s }, p0/Z, [x24]\n" + "ld1w { z5.s }, p0/Z, [x22]\n" + "ld1w { z4.s }, p0/Z, [x21]\n" "ld1rw { z3.s }, p0/Z, [%x[Apanel]]\n" - "fmla z8.s, p0/M, z0.s, z3.s\n" - "ld1rw { z4.s }, p0/Z, [%x[Apanel], #4]\n" - "ld1rw { z5.s }, p0/Z, [%x[Apanel], #8]\n" - "fmla z9.s, p0/M, z1.s, z3.s\n" - "ld1rw { z6.s }, p0/Z, [%x[Apanel], #12]\n" - "fmla z10.s, p0/M, z2.s, z3.s\n" - "fmla z11.s, p0/M, z0.s, z4.s\n" - "fmla z12.s, p0/M, z1.s, z4.s\n" - "fmla z13.s, p0/M, z2.s, z4.s\n" + "fmla z8.s, p0/M, z6.s, z3.s\n" + "ld1rw { z2.s }, p0/Z, [%x[Apanel], #4]\n" + "ld1rw { z1.s }, p0/Z, [%x[Apanel], #8]\n" + "fmla z9.s, p0/M, z5.s, z3.s\n" + "ld1rw { z0.s }, p0/Z, [%x[Apanel], #12]\n" + "fmla z10.s, p0/M, z4.s, z3.s\n" + "fmla z11.s, p0/M, z6.s, z2.s\n" + "fmla z12.s, p0/M, z5.s, z2.s\n" + "fmla z13.s, p0/M, z4.s, z2.s\n" "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n" - "fmla z14.s, p0/M, z0.s, z5.s\n" - "fmla z15.s, p0/M, z1.s, z5.s\n" - "ld1rw { z4.s }, p0/Z, [%x[Apanel], #20]\n" - "fmla z16.s, p0/M, z2.s, z5.s\n" - "fmla z17.s, p0/M, z0.s, z6.s\n" - "ld1rw { z5.s }, p0/Z, [%x[Apanel], #24]\n" - "fmla z18.s, p0/M, z1.s, z6.s\n" - "fmla z19.s, p0/M, z2.s, z6.s\n" - "ld1rw { z6.s }, p0/Z, [%x[Apanel], #28]\n" - "fmla z20.s, p0/M, z0.s, z3.s\n" - "fmla z21.s, p0/M, z1.s, z3.s\n" + "fmla z14.s, p0/M, z6.s, z1.s\n" + "fmla z15.s, p0/M, z5.s, z1.s\n" + "ld1rw { z2.s }, p0/Z, [%x[Apanel], #20]\n" + "fmla z16.s, p0/M, z4.s, z1.s\n" + "fmla z17.s, p0/M, z6.s, z0.s\n" + "ld1rw { z1.s }, p0/Z, [%x[Apanel], #24]\n" + "fmla z18.s, p0/M, z5.s, z0.s\n" + "fmla z19.s, p0/M, z4.s, z0.s\n" + "ld1rw { z0.s }, p0/Z, [%x[Apanel], #28]\n" + "fmla z20.s, p0/M, z6.s, z3.s\n" + "fmla z21.s, p0/M, z5.s, z3.s\n" "add %x[Apanel], %x[Apanel], #0x20\n" - "fmla z22.s, p0/M, z2.s, z3.s\n" - "fmla z23.s, p0/M, z0.s, z4.s\n" - "fmla z24.s, p0/M, z1.s, z4.s\n" - "fmla z25.s, p0/M, z2.s, z4.s\n" - "fmla z26.s, p0/M, z0.s, z5.s\n" - "fmla z27.s, p0/M, z1.s, z5.s\n" - "fmla z28.s, p0/M, z2.s, z5.s\n" - "fmla z29.s, p0/M, z0.s, z6.s\n" - "fmla z30.s, p0/M, z1.s, z6.s\n" - "fmla z31.s, p0/M, z2.s, z6.s\n" + "fmla z22.s, p0/M, z4.s, z3.s\n" + "fmla z23.s, p0/M, z6.s, z2.s\n" + "fmla z24.s, p0/M, z5.s, z2.s\n" + "fmla z25.s, p0/M, z4.s, z2.s\n" + "fmla z26.s, p0/M, z6.s, z1.s\n" + "fmla z27.s, p0/M, z5.s, z1.s\n" + "fmla z28.s, p0/M, z4.s, z1.s\n" + "fmla z29.s, p0/M, z6.s, z0.s\n" + "fmla z30.s, p0/M, z5.s, z0.s\n" + "fmla z31.s, p0/M, z4.s, z0.s\n" "6:" // multiply loop done - "decw x25, ALL, MUL #3\n" + "decw x26, ALL, MUL #3\n" "st1w { z8.s }, p0, [%x[Cpanel]]\n" - "cmp x25, XZR\n" + "cmp x26, XZR\n" "st1w { z9.s }, p0, [%x[Cpanel], #1, MUL VL]\n" "st1w { z10.s }, p0, [%x[Cpanel], #2, MUL VL]\n" "st1w { z11.s }, p0, [%x[Cpanel], #3, MUL VL]\n" @@ -289,7 +289,7 @@ void sve_ffinterleaved_fp32_mla_8x3VL_a64fx( "bne 1b\n" : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks) : [args_ptr] "r" (&ka), [offsetof_B_stride] "I" (offsetof(KernelArgs, B_stride)), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_cur_B_ptr] "I" (offsetof(KernelArgs, cur_B_ptr)) - : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp32_mla_8x3VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp32_mla_8x3VL/generic.cpp index 4a0b31daffa0898ec78ddf37bc4a30bb7c5d424e..4b20be6f012b98d267bcd67c987f92aab480df75 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp32_mla_8x3VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_ffinterleaved_fp32_mla_8x3VL/generic.cpp @@ -52,26 +52,26 @@ void sve_ffinterleaved_fp32_mla_8x3VL( __asm__ __volatile__( "ptrue p0.b\n" "1:" // Height loop - "ldr x26, [%x[args_ptr], %[offsetof_Bpanel]]\n" - "ldr x25, [%x[args_ptr], %[offsetof_N]]\n" - "str x26, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" - "mov x24, %x[Apanel]\n" + "ldr x20, [%x[args_ptr], %[offsetof_Bpanel]]\n" + "ldr x26, [%x[args_ptr], %[offsetof_N]]\n" + "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" + "mov x25, %x[Apanel]\n" "2:" // Width loop - "ldr x26, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" + "ldr x24, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" "ldr x20, [%x[args_ptr], %[offsetof_B_stride]]\n" "cntw x23, ALL, MUL #2\n" - "add x22, x26, x20, LSL #2\n" + "add x22, x24, x20, LSL #2\n" "add x21, x22, x20, LSL #2\n" "add x20, x21, x20, LSL #2\n" - "cmp x25, x23\n" + "cmp x26, x23\n" "str x20, [%x[args_ptr], %[offsetof_cur_B_ptr]]\n" - "mov %x[Apanel], x24\n" + "mov %x[Apanel], x25\n" "bgt 3f\n" "decw x23\n" - "cmp x25, x23\n" - "mov x21, x26\n" + "cmp x26, x23\n" + "mov x21, x24\n" "bgt 3f\n" - "mov x22, x26\n" + "mov x22, x24\n" "3:" // B setup done "ldr x20, [%x[args_ptr], %[offsetof_K]]\n" "cmp x20, #0x2\n" @@ -84,7 +84,7 @@ void sve_ffinterleaved_fp32_mla_8x3VL( "ld1rqw { z1.s }, p0/Z, [%x[Apanel], #16]\n" "mov z13.b, #0x0\n" "mov z14.b, #0x0\n" - "ld1w { z4.s }, p0/Z, [x26]\n" + "ld1w { z4.s }, p0/Z, [x24]\n" "mov z15.b, #0x0\n" "mov z16.b, #0x0\n" "ld1w { z5.s }, p0/Z, [x22]\n" @@ -108,19 +108,19 @@ void sve_ffinterleaved_fp32_mla_8x3VL( "4:" // main loop head "fmla z8.s, z4.s, z0.s[0]\n" "fmla z11.s, z4.s, z0.s[1]\n" - "ld1rqw { z2.s }, p0/Z, [%x[Apanel], #32]\n" + "ld1rqw { z3.s }, p0/Z, [%x[Apanel], #32]\n" "fmla z14.s, z4.s, z0.s[2]\n" "fmla z17.s, z4.s, z0.s[3]\n" - "ld1rqw { z3.s }, p0/Z, [%x[Apanel], #48]\n" + "ld1rqw { z7.s }, p0/Z, [%x[Apanel], #48]\n" "fmla z20.s, z4.s, z1.s[0]\n" "fmla z23.s, z4.s, z1.s[1]\n" "sub x20, x20, #0x2\n" "fmla z26.s, z4.s, z1.s[2]\n" "fmla z29.s, z4.s, z1.s[3]\n" - "ld1w { z4.s }, p0/Z, [x26, #1, MUL VL]\n" + "ld1w { z4.s }, p0/Z, [x24, #1, MUL VL]\n" "fmla z9.s, z5.s, z0.s[0]\n" "fmla z12.s, z5.s, z0.s[1]\n" - "addvl x26, x26, #2\n" + "addvl x24, x24, #2\n" "fmla z15.s, z5.s, z0.s[2]\n" "fmla z18.s, z5.s, z0.s[3]\n" "cmp x20, #0x2\n" @@ -140,35 +140,35 @@ void sve_ffinterleaved_fp32_mla_8x3VL( "fmla z25.s, z6.s, z1.s[1]\n" "fmla z28.s, z6.s, z1.s[2]\n" "fmla z31.s, z6.s, z1.s[3]\n" - "ld1w { z6.s }, p0/Z, [x21, #1, MUL VL]\n" + "ld1w { z2.s }, p0/Z, [x21, #1, MUL VL]\n" "addvl x21, x21, #2\n" - "fmla z8.s, z4.s, z2.s[0]\n" - "fmla z11.s, z4.s, z2.s[1]\n" - "fmla z14.s, z4.s, z2.s[2]\n" - "fmla z17.s, z4.s, z2.s[3]\n" + "fmla z8.s, z4.s, z3.s[0]\n" + "fmla z11.s, z4.s, z3.s[1]\n" + "fmla z14.s, z4.s, z3.s[2]\n" + "fmla z17.s, z4.s, z3.s[3]\n" "ld1rqw { z1.s }, p0/Z, [%x[Apanel], #16]\n" - "fmla z20.s, z4.s, z3.s[0]\n" - "fmla z23.s, z4.s, z3.s[1]\n" - "fmla z26.s, z4.s, z3.s[2]\n" - "fmla z29.s, z4.s, z3.s[3]\n" - "ld1w { z4.s }, p0/Z, [x26]\n" - "fmla z9.s, z5.s, z2.s[0]\n" - "fmla z12.s, z5.s, z2.s[1]\n" - "fmla z15.s, z5.s, z2.s[2]\n" - "fmla z18.s, z5.s, z2.s[3]\n" - "fmla z21.s, z5.s, z3.s[0]\n" - "fmla z24.s, z5.s, z3.s[1]\n" - "fmla z27.s, z5.s, z3.s[2]\n" - "fmla z30.s, z5.s, z3.s[3]\n" + "fmla z20.s, z4.s, z7.s[0]\n" + "fmla z23.s, z4.s, z7.s[1]\n" + "fmla z26.s, z4.s, z7.s[2]\n" + "fmla z29.s, z4.s, z7.s[3]\n" + "ld1w { z4.s }, p0/Z, [x24]\n" + "fmla z9.s, z5.s, z3.s[0]\n" + "fmla z12.s, z5.s, z3.s[1]\n" + "fmla z15.s, z5.s, z3.s[2]\n" + "fmla z18.s, z5.s, z3.s[3]\n" + "fmla z21.s, z5.s, z7.s[0]\n" + "fmla z24.s, z5.s, z7.s[1]\n" + "fmla z27.s, z5.s, z7.s[2]\n" + "fmla z30.s, z5.s, z7.s[3]\n" "ld1w { z5.s }, p0/Z, [x22]\n" - "fmla z10.s, z6.s, z2.s[0]\n" - "fmla z13.s, z6.s, z2.s[1]\n" - "fmla z16.s, z6.s, z2.s[2]\n" - "fmla z19.s, z6.s, z2.s[3]\n" - "fmla z22.s, z6.s, z3.s[0]\n" - "fmla z25.s, z6.s, z3.s[1]\n" - "fmla z28.s, z6.s, z3.s[2]\n" - "fmla z31.s, z6.s, z3.s[3]\n" + "fmla z10.s, z2.s, z3.s[0]\n" + "fmla z13.s, z2.s, z3.s[1]\n" + "fmla z16.s, z2.s, z3.s[2]\n" + "fmla z19.s, z2.s, z3.s[3]\n" + "fmla z22.s, z2.s, z7.s[0]\n" + "fmla z25.s, z2.s, z7.s[1]\n" + "fmla z28.s, z2.s, z7.s[2]\n" + "fmla z31.s, z2.s, z7.s[3]\n" "ld1w { z6.s }, p0/Z, [x21]\n" "bge 4b\n" "5:" // main loop skip @@ -177,7 +177,7 @@ void sve_ffinterleaved_fp32_mla_8x3VL( "add %x[Apanel], %x[Apanel], #0x20\n" "fmla z14.s, z4.s, z0.s[2]\n" "fmla z17.s, z4.s, z0.s[3]\n" - "addvl x26, x26, #1\n" + "addvl x24, x24, #1\n" "fmla z20.s, z4.s, z1.s[0]\n" "fmla z23.s, z4.s, z1.s[1]\n" "addvl x22, x22, #1\n" @@ -201,40 +201,40 @@ void sve_ffinterleaved_fp32_mla_8x3VL( "fmla z28.s, z6.s, z1.s[2]\n" "fmla z31.s, z6.s, z1.s[3]\n" "cbz x20, 6f\n" - "ld1rqw { z0.s }, p0/Z, [%x[Apanel]]\n" - "ld1rqw { z1.s }, p0/Z, [%x[Apanel], #16]\n" + "ld1rqw { z4.s }, p0/Z, [%x[Apanel]]\n" + "ld1rqw { z3.s }, p0/Z, [%x[Apanel], #16]\n" "add %x[Apanel], %x[Apanel], #0x20\n" - "ld1w { z7.s }, p0/Z, [x26]\n" - "ld1w { z4.s }, p0/Z, [x22]\n" - "fmla z8.s, z7.s, z0.s[0]\n" - "ld1w { z5.s }, p0/Z, [x21]\n" - "fmla z11.s, z7.s, z0.s[1]\n" - "fmla z14.s, z7.s, z0.s[2]\n" - "fmla z17.s, z7.s, z0.s[3]\n" - "fmla z20.s, z7.s, z1.s[0]\n" - "fmla z23.s, z7.s, z1.s[1]\n" - "fmla z26.s, z7.s, z1.s[2]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "fmla z9.s, z4.s, z0.s[0]\n" - "fmla z12.s, z4.s, z0.s[1]\n" - "fmla z15.s, z4.s, z0.s[2]\n" - "fmla z18.s, z4.s, z0.s[3]\n" - "fmla z21.s, z4.s, z1.s[0]\n" - "fmla z24.s, z4.s, z1.s[1]\n" - "fmla z27.s, z4.s, z1.s[2]\n" - "fmla z30.s, z4.s, z1.s[3]\n" - "fmla z10.s, z5.s, z0.s[0]\n" - "fmla z13.s, z5.s, z0.s[1]\n" - "fmla z16.s, z5.s, z0.s[2]\n" - "fmla z19.s, z5.s, z0.s[3]\n" - "fmla z22.s, z5.s, z1.s[0]\n" - "fmla z25.s, z5.s, z1.s[1]\n" - "fmla z28.s, z5.s, z1.s[2]\n" - "fmla z31.s, z5.s, z1.s[3]\n" + "ld1w { z2.s }, p0/Z, [x24]\n" + "ld1w { z1.s }, p0/Z, [x22]\n" + "fmla z8.s, z2.s, z4.s[0]\n" + "ld1w { z0.s }, p0/Z, [x21]\n" + "fmla z11.s, z2.s, z4.s[1]\n" + "fmla z14.s, z2.s, z4.s[2]\n" + "fmla z17.s, z2.s, z4.s[3]\n" + "fmla z20.s, z2.s, z3.s[0]\n" + "fmla z23.s, z2.s, z3.s[1]\n" + "fmla z26.s, z2.s, z3.s[2]\n" + "fmla z29.s, z2.s, z3.s[3]\n" + "fmla z9.s, z1.s, z4.s[0]\n" + "fmla z12.s, z1.s, z4.s[1]\n" + "fmla z15.s, z1.s, z4.s[2]\n" + "fmla z18.s, z1.s, z4.s[3]\n" + "fmla z21.s, z1.s, z3.s[0]\n" + "fmla z24.s, z1.s, z3.s[1]\n" + "fmla z27.s, z1.s, z3.s[2]\n" + "fmla z30.s, z1.s, z3.s[3]\n" + "fmla z10.s, z0.s, z4.s[0]\n" + "fmla z13.s, z0.s, z4.s[1]\n" + "fmla z16.s, z0.s, z4.s[2]\n" + "fmla z19.s, z0.s, z4.s[3]\n" + "fmla z22.s, z0.s, z3.s[0]\n" + "fmla z25.s, z0.s, z3.s[1]\n" + "fmla z28.s, z0.s, z3.s[2]\n" + "fmla z31.s, z0.s, z3.s[3]\n" "6:" // multiply loop done - "decw x25, ALL, MUL #3\n" + "decw x26, ALL, MUL #3\n" "st1w { z8.s }, p0, [%x[Cpanel]]\n" - "cmp x25, XZR\n" + "cmp x26, XZR\n" "st1w { z9.s }, p0, [%x[Cpanel], #1, MUL VL]\n" "st1w { z10.s }, p0, [%x[Cpanel], #2, MUL VL]\n" "st1w { z11.s }, p0, [%x[Cpanel], #3, MUL VL]\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_dot_6x4VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_dot_6x4VL.hpp index 6677c23216924b5c65752d69bab0a81528571fd9..49ccce342e65c9d36759f4aac8305a60786a4d6f 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_dot_6x4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_dot_6x4VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef ARM_COMPUTE_ENABLE_SVE + #include "../std_transforms_sve.hpp" #include "../bfloat.hpp" #include "../performance_parameters.hpp" @@ -75,7 +75,6 @@ public: template static inline PerformanceParameters get_performance_parameters(const CPUInfo *ci) { - if (std::is_same::value) { switch (ci->get_cpu_model()) { default: @@ -100,5 +99,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_dot_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_dot_6x4VL/generic.cpp index f0b00e6251092f307686beba43142e3117329dfb..176f6e0d3a514e818408a5baca812b8557687485 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_dot_6x4VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_dot_6x4VL/generic.cpp @@ -140,11 +140,11 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "6:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 7f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 8f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -157,87 +157,87 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "9:" // Height 1: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" "ld1rqh { z0.h }, p0/Z, [x26]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - ".inst 0x646040c8 // bfdot z8.s, z6.h, z0.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x646040e9 // bfdot z9.s, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x646040ca // bfdot z10.s, z6.h, z0.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x646040eb // bfdot z11.s, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x646840c8 // bfdot z8.s, z6.h, z0.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x646840e9 // bfdot z9.s, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x646840ca // bfdot z10.s, z6.h, z0.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10]\n" + ".inst 0x64604208 // bfdot z8.s, z16.h, z0.h[0]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x64604209 // bfdot z9.s, z16.h, z0.h[0]\n" + "ld1h { z16.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x6460420a // bfdot z10.s, z16.h, z0.h[0]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6460420b // bfdot z11.s, z16.h, z0.h[0]\n" + "ld1h { z16.h }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x64684208 // bfdot z8.s, z16.h, z0.h[1]\n" + "ld1h { z16.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x64684209 // bfdot z9.s, z16.h, z0.h[1]\n" + "ld1h { z16.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x6468420a // bfdot z10.s, z16.h, z0.h[1]\n" + "ld1h { z16.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #-7, MUL VL]\n" - ".inst 0x647040c8 // bfdot z8.s, z6.h, z0.h[2]\n" - ".inst 0x647040e9 // bfdot z9.s, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #-5, MUL VL]\n" - ".inst 0x647040ca // bfdot z10.s, z6.h, z0.h[2]\n" - ".inst 0x647040eb // bfdot z11.s, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #-3, MUL VL]\n" - ".inst 0x647840c8 // bfdot z8.s, z6.h, z0.h[3]\n" - ".inst 0x647840e9 // bfdot z9.s, z7.h, z0.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #-1, MUL VL]\n" + ".inst 0x6468420b // bfdot z11.s, z16.h, z0.h[1]\n" + "ld1h { z17.h }, p5/Z, [x10, #-8, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #-7, MUL VL]\n" + ".inst 0x64704228 // bfdot z8.s, z17.h, z0.h[2]\n" + ".inst 0x64704209 // bfdot z9.s, z16.h, z0.h[2]\n" + "ld1h { z17.h }, p5/Z, [x10, #-6, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #-5, MUL VL]\n" + ".inst 0x6470422a // bfdot z10.s, z17.h, z0.h[2]\n" + ".inst 0x6470420b // bfdot z11.s, z16.h, z0.h[2]\n" + "ld1h { z17.h }, p5/Z, [x10, #-4, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #-3, MUL VL]\n" + ".inst 0x64784228 // bfdot z8.s, z17.h, z0.h[3]\n" + ".inst 0x64784209 // bfdot z9.s, z16.h, z0.h[3]\n" + "ld1h { z17.h }, p5/Z, [x10, #-2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #-1, MUL VL]\n" "sub x27, x27, #0x8\n" "cmp x27, #0x8\n" - ".inst 0x647840ca // bfdot z10.s, z6.h, z0.h[3]\n" - ".inst 0x647840eb // bfdot z11.s, z7.h, z0.h[3]\n" + ".inst 0x6478422a // bfdot z10.s, z17.h, z0.h[3]\n" + ".inst 0x6478420b // bfdot z11.s, z16.h, z0.h[3]\n" "add x26, x26, #0x10\n" "bgt 9b\n" "10:" // Height 1: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" "ld1rqh { z0.h }, p0/Z, [x26]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - ".inst 0x646040c8 // bfdot z8.s, z6.h, z0.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x646040e9 // bfdot z9.s, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10]\n" + ".inst 0x64604208 // bfdot z8.s, z16.h, z0.h[0]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x64604209 // bfdot z9.s, z16.h, z0.h[0]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x2\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x646040ca // bfdot z10.s, z6.h, z0.h[0]\n" - ".inst 0x646040eb // bfdot z11.s, z7.h, z0.h[0]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6460422a // bfdot z10.s, z17.h, z0.h[0]\n" + ".inst 0x6460420b // bfdot z11.s, z16.h, z0.h[0]\n" "addvl x10, x10, #4\n" "ble 11f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x646840c8 // bfdot z8.s, z6.h, z0.h[1]\n" - ".inst 0x646840e9 // bfdot z9.s, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x64684228 // bfdot z8.s, z17.h, z0.h[1]\n" + ".inst 0x64684209 // bfdot z9.s, z16.h, z0.h[1]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x2\n" - ".inst 0x646840ca // bfdot z10.s, z6.h, z0.h[1]\n" - ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n" + ".inst 0x6468422a // bfdot z10.s, z17.h, z0.h[1]\n" + ".inst 0x6468420b // bfdot z11.s, z16.h, z0.h[1]\n" "addvl x10, x10, #4\n" "ble 11f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x647040c8 // bfdot z8.s, z6.h, z0.h[2]\n" - ".inst 0x647040e9 // bfdot z9.s, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x64704228 // bfdot z8.s, z17.h, z0.h[2]\n" + ".inst 0x64704209 // bfdot z9.s, z16.h, z0.h[2]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x2\n" - ".inst 0x647040ca // bfdot z10.s, z6.h, z0.h[2]\n" - ".inst 0x647040eb // bfdot z11.s, z7.h, z0.h[2]\n" + ".inst 0x6470422a // bfdot z10.s, z17.h, z0.h[2]\n" + ".inst 0x6470420b // bfdot z11.s, z16.h, z0.h[2]\n" "addvl x10, x10, #4\n" "ble 11f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x647840c8 // bfdot z8.s, z6.h, z0.h[3]\n" - ".inst 0x647840e9 // bfdot z9.s, z7.h, z0.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x647840ca // bfdot z10.s, z6.h, z0.h[3]\n" - ".inst 0x647840eb // bfdot z11.s, z7.h, z0.h[3]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x64784228 // bfdot z8.s, z17.h, z0.h[3]\n" + ".inst 0x64784209 // bfdot z9.s, z16.h, z0.h[3]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6478422a // bfdot z10.s, z17.h, z0.h[3]\n" + ".inst 0x6478420b // bfdot z11.s, z16.h, z0.h[3]\n" "addvl x10, x10, #4\n" "11:" // Height 1: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -246,17 +246,17 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "bne 6b\n" "tbz %x[flags], #1, 12f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z17.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" + "ld1rw { z16.s }, p5/Z, [x20]\n" + "fmin z8.s, p5/M, z8.s, z17.s\n" + "fmin z9.s, p5/M, z9.s, z17.s\n" + "fmin z10.s, p5/M, z10.s, z17.s\n" + "fmin z11.s, p5/M, z11.s, z17.s\n" + "fmax z8.s, p5/M, z8.s, z16.s\n" + "fmax z9.s, p5/M, z9.s, z16.s\n" + "fmax z10.s, p5/M, z10.s, z16.s\n" + "fmax z11.s, p5/M, z11.s, z16.s\n" "12:" // Height 1: No activation "st1w { z8.s }, p4, [x9]\n" "st1w { z9.s }, p3, [x9, #1, MUL VL]\n" @@ -296,15 +296,15 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "16:" // Height 2: no bias "tbz %x[flags], #0, 17f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" + "add x20, x9, x20, LSL #2\n" "ld1w { z8.s }, p4/Z, [x9]\n" "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x20]\n" + "ld1w { z13.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z15.s }, p1/Z, [x20, #3, MUL VL]\n" "b 18f\n" "17:" // Height 2: no accumulate "mov z8.b, #0x0\n" @@ -320,12 +320,12 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "19:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 20f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 21f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -333,130 +333,130 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "b 21f\n" "20:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" "21:" // Height 2: input setup done "cmp x27, #0x8\n" "ble 23f\n" "22:" // Height 2: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z0.h }, p0/Z, [x26]\n" - "ld1rqh { z1.h }, p0/Z, [x25]\n" + "ld1rqh { z1.h }, p0/Z, [x26]\n" + "ld1rqh { z0.h }, p0/Z, [x25]\n" "sub x27, x27, #0x8\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x646040c8 // bfdot z8.s, z6.h, z0.h[0]\n" - ".inst 0x646140cc // bfdot z12.s, z6.h, z1.h[0]\n" - ".inst 0x646040e9 // bfdot z9.s, z7.h, z0.h[0]\n" - ".inst 0x646140ed // bfdot z13.s, z7.h, z1.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x646040ca // bfdot z10.s, z6.h, z0.h[0]\n" - ".inst 0x646140ce // bfdot z14.s, z6.h, z1.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x64614228 // bfdot z8.s, z17.h, z1.h[0]\n" + ".inst 0x6460422c // bfdot z12.s, z17.h, z0.h[0]\n" + ".inst 0x64614209 // bfdot z9.s, z16.h, z1.h[0]\n" + ".inst 0x6460420d // bfdot z13.s, z16.h, z0.h[0]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6461422a // bfdot z10.s, z17.h, z1.h[0]\n" + ".inst 0x6460422e // bfdot z14.s, z17.h, z0.h[0]\n" + "ld1h { z17.h }, p5/Z, [x10, #4, MUL VL]\n" "cmp x27, #0x8\n" - ".inst 0x646040eb // bfdot z11.s, z7.h, z0.h[0]\n" - ".inst 0x646140ef // bfdot z15.s, z7.h, z1.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6461420b // bfdot z11.s, z16.h, z1.h[0]\n" + ".inst 0x6460420f // bfdot z15.s, z16.h, z0.h[0]\n" + "ld1h { z16.h }, p5/Z, [x10, #5, MUL VL]\n" "add x26, x26, #0x10\n" - ".inst 0x646840c8 // bfdot z8.s, z6.h, z0.h[1]\n" - ".inst 0x646940cc // bfdot z12.s, z6.h, z1.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x64694228 // bfdot z8.s, z17.h, z1.h[1]\n" + ".inst 0x6468422c // bfdot z12.s, z17.h, z0.h[1]\n" + "ld1h { z17.h }, p5/Z, [x10, #6, MUL VL]\n" "add x25, x25, #0x10\n" - ".inst 0x646840e9 // bfdot z9.s, z7.h, z0.h[1]\n" - ".inst 0x646940ed // bfdot z13.s, z7.h, z1.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x64694209 // bfdot z9.s, z16.h, z1.h[1]\n" + ".inst 0x6468420d // bfdot z13.s, z16.h, z0.h[1]\n" + "ld1h { z16.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - ".inst 0x646840ca // bfdot z10.s, z6.h, z0.h[1]\n" - ".inst 0x646940ce // bfdot z14.s, z6.h, z1.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n" - ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n" - ".inst 0x646940ef // bfdot z15.s, z7.h, z1.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #-7, MUL VL]\n" - ".inst 0x647040c8 // bfdot z8.s, z6.h, z0.h[2]\n" - ".inst 0x647140cc // bfdot z12.s, z6.h, z1.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n" - ".inst 0x647040e9 // bfdot z9.s, z7.h, z0.h[2]\n" - ".inst 0x647140ed // bfdot z13.s, z7.h, z1.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #-5, MUL VL]\n" - ".inst 0x647040ca // bfdot z10.s, z6.h, z0.h[2]\n" - ".inst 0x647140ce // bfdot z14.s, z6.h, z1.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n" - ".inst 0x647040eb // bfdot z11.s, z7.h, z0.h[2]\n" - ".inst 0x647140ef // bfdot z15.s, z7.h, z1.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #-3, MUL VL]\n" - ".inst 0x647840c8 // bfdot z8.s, z6.h, z0.h[3]\n" - ".inst 0x647940cc // bfdot z12.s, z6.h, z1.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n" - ".inst 0x647840e9 // bfdot z9.s, z7.h, z0.h[3]\n" - ".inst 0x647940ed // bfdot z13.s, z7.h, z1.h[3]\n" - "ld1h { z7.h }, p5/Z, [x10, #-1, MUL VL]\n" - ".inst 0x647840ca // bfdot z10.s, z6.h, z0.h[3]\n" - ".inst 0x647940ce // bfdot z14.s, z6.h, z1.h[3]\n" - ".inst 0x647840eb // bfdot z11.s, z7.h, z0.h[3]\n" - ".inst 0x647940ef // bfdot z15.s, z7.h, z1.h[3]\n" + ".inst 0x6469422a // bfdot z10.s, z17.h, z1.h[1]\n" + ".inst 0x6468422e // bfdot z14.s, z17.h, z0.h[1]\n" + "ld1h { z17.h }, p5/Z, [x10, #-8, MUL VL]\n" + ".inst 0x6469420b // bfdot z11.s, z16.h, z1.h[1]\n" + ".inst 0x6468420f // bfdot z15.s, z16.h, z0.h[1]\n" + "ld1h { z16.h }, p5/Z, [x10, #-7, MUL VL]\n" + ".inst 0x64714228 // bfdot z8.s, z17.h, z1.h[2]\n" + ".inst 0x6470422c // bfdot z12.s, z17.h, z0.h[2]\n" + "ld1h { z17.h }, p5/Z, [x10, #-6, MUL VL]\n" + ".inst 0x64714209 // bfdot z9.s, z16.h, z1.h[2]\n" + ".inst 0x6470420d // bfdot z13.s, z16.h, z0.h[2]\n" + "ld1h { z16.h }, p5/Z, [x10, #-5, MUL VL]\n" + ".inst 0x6471422a // bfdot z10.s, z17.h, z1.h[2]\n" + ".inst 0x6470422e // bfdot z14.s, z17.h, z0.h[2]\n" + "ld1h { z17.h }, p5/Z, [x10, #-4, MUL VL]\n" + ".inst 0x6471420b // bfdot z11.s, z16.h, z1.h[2]\n" + ".inst 0x6470420f // bfdot z15.s, z16.h, z0.h[2]\n" + "ld1h { z16.h }, p5/Z, [x10, #-3, MUL VL]\n" + ".inst 0x64794228 // bfdot z8.s, z17.h, z1.h[3]\n" + ".inst 0x6478422c // bfdot z12.s, z17.h, z0.h[3]\n" + "ld1h { z17.h }, p5/Z, [x10, #-2, MUL VL]\n" + ".inst 0x64794209 // bfdot z9.s, z16.h, z1.h[3]\n" + ".inst 0x6478420d // bfdot z13.s, z16.h, z0.h[3]\n" + "ld1h { z16.h }, p5/Z, [x10, #-1, MUL VL]\n" + ".inst 0x6479422a // bfdot z10.s, z17.h, z1.h[3]\n" + ".inst 0x6478422e // bfdot z14.s, z17.h, z0.h[3]\n" + ".inst 0x6479420b // bfdot z11.s, z16.h, z1.h[3]\n" + ".inst 0x6478420f // bfdot z15.s, z16.h, z0.h[3]\n" "bgt 22b\n" "23:" // Height 2: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" "ld1rqh { z0.h }, p0/Z, [x26]\n" "ld1rqh { z1.h }, p0/Z, [x25]\n" "subs x27, x27, #0x2\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x646040c8 // bfdot z8.s, z6.h, z0.h[0]\n" - ".inst 0x646140cc // bfdot z12.s, z6.h, z1.h[0]\n" - ".inst 0x646040e9 // bfdot z9.s, z7.h, z0.h[0]\n" - ".inst 0x646140ed // bfdot z13.s, z7.h, z1.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x646040ca // bfdot z10.s, z6.h, z0.h[0]\n" - ".inst 0x646140ce // bfdot z14.s, z6.h, z1.h[0]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x64604228 // bfdot z8.s, z17.h, z0.h[0]\n" + ".inst 0x6461422c // bfdot z12.s, z17.h, z1.h[0]\n" + ".inst 0x64604209 // bfdot z9.s, z16.h, z0.h[0]\n" + ".inst 0x6461420d // bfdot z13.s, z16.h, z1.h[0]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6460422a // bfdot z10.s, z17.h, z0.h[0]\n" + ".inst 0x6461422e // bfdot z14.s, z17.h, z1.h[0]\n" "addvl x10, x10, #4\n" - ".inst 0x646040eb // bfdot z11.s, z7.h, z0.h[0]\n" - ".inst 0x646140ef // bfdot z15.s, z7.h, z1.h[0]\n" + ".inst 0x6460420b // bfdot z11.s, z16.h, z0.h[0]\n" + ".inst 0x6461420f // bfdot z15.s, z16.h, z1.h[0]\n" "ble 24f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x646840c8 // bfdot z8.s, z6.h, z0.h[1]\n" - ".inst 0x646940cc // bfdot z12.s, z6.h, z1.h[1]\n" - ".inst 0x646840e9 // bfdot z9.s, z7.h, z0.h[1]\n" - ".inst 0x646940ed // bfdot z13.s, z7.h, z1.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x64684228 // bfdot z8.s, z17.h, z0.h[1]\n" + ".inst 0x6469422c // bfdot z12.s, z17.h, z1.h[1]\n" + ".inst 0x64684209 // bfdot z9.s, z16.h, z0.h[1]\n" + ".inst 0x6469420d // bfdot z13.s, z16.h, z1.h[1]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x2\n" - ".inst 0x646840ca // bfdot z10.s, z6.h, z0.h[1]\n" - ".inst 0x646940ce // bfdot z14.s, z6.h, z1.h[1]\n" + ".inst 0x6468422a // bfdot z10.s, z17.h, z0.h[1]\n" + ".inst 0x6469422e // bfdot z14.s, z17.h, z1.h[1]\n" "addvl x10, x10, #4\n" - ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n" - ".inst 0x646940ef // bfdot z15.s, z7.h, z1.h[1]\n" + ".inst 0x6468420b // bfdot z11.s, z16.h, z0.h[1]\n" + ".inst 0x6469420f // bfdot z15.s, z16.h, z1.h[1]\n" "ble 24f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x647040c8 // bfdot z8.s, z6.h, z0.h[2]\n" - ".inst 0x647140cc // bfdot z12.s, z6.h, z1.h[2]\n" - ".inst 0x647040e9 // bfdot z9.s, z7.h, z0.h[2]\n" - ".inst 0x647140ed // bfdot z13.s, z7.h, z1.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x64704228 // bfdot z8.s, z17.h, z0.h[2]\n" + ".inst 0x6471422c // bfdot z12.s, z17.h, z1.h[2]\n" + ".inst 0x64704209 // bfdot z9.s, z16.h, z0.h[2]\n" + ".inst 0x6471420d // bfdot z13.s, z16.h, z1.h[2]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x2\n" - ".inst 0x647040ca // bfdot z10.s, z6.h, z0.h[2]\n" - ".inst 0x647140ce // bfdot z14.s, z6.h, z1.h[2]\n" + ".inst 0x6470422a // bfdot z10.s, z17.h, z0.h[2]\n" + ".inst 0x6471422e // bfdot z14.s, z17.h, z1.h[2]\n" "addvl x10, x10, #4\n" - ".inst 0x647040eb // bfdot z11.s, z7.h, z0.h[2]\n" - ".inst 0x647140ef // bfdot z15.s, z7.h, z1.h[2]\n" + ".inst 0x6470420b // bfdot z11.s, z16.h, z0.h[2]\n" + ".inst 0x6471420f // bfdot z15.s, z16.h, z1.h[2]\n" "ble 24f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x647840c8 // bfdot z8.s, z6.h, z0.h[3]\n" - ".inst 0x647940cc // bfdot z12.s, z6.h, z1.h[3]\n" - ".inst 0x647840e9 // bfdot z9.s, z7.h, z0.h[3]\n" - ".inst 0x647940ed // bfdot z13.s, z7.h, z1.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x647840ca // bfdot z10.s, z6.h, z0.h[3]\n" - ".inst 0x647940ce // bfdot z14.s, z6.h, z1.h[3]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x64784228 // bfdot z8.s, z17.h, z0.h[3]\n" + ".inst 0x6479422c // bfdot z12.s, z17.h, z1.h[3]\n" + ".inst 0x64784209 // bfdot z9.s, z16.h, z0.h[3]\n" + ".inst 0x6479420d // bfdot z13.s, z16.h, z1.h[3]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6478422a // bfdot z10.s, z17.h, z0.h[3]\n" + ".inst 0x6479422e // bfdot z14.s, z17.h, z1.h[3]\n" "addvl x10, x10, #4\n" - ".inst 0x647840eb // bfdot z11.s, z7.h, z0.h[3]\n" - ".inst 0x647940ef // bfdot z15.s, z7.h, z1.h[3]\n" + ".inst 0x6478420b // bfdot z11.s, z16.h, z0.h[3]\n" + ".inst 0x6479420f // bfdot z15.s, z16.h, z1.h[3]\n" "24:" // Height 2: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -466,25 +466,25 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "add x25, x9, x20, LSL #2\n" "tbz %x[flags], #1, 25f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z17.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmin z12.s, p5/M, z12.s, z1.s\n" - "fmin z13.s, p5/M, z13.s, z1.s\n" - "fmin z14.s, p5/M, z14.s, z1.s\n" - "fmin z15.s, p5/M, z15.s, z1.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" - "fmax z12.s, p5/M, z12.s, z0.s\n" - "fmax z13.s, p5/M, z13.s, z0.s\n" - "fmax z14.s, p5/M, z14.s, z0.s\n" - "fmax z15.s, p5/M, z15.s, z0.s\n" + "ld1rw { z16.s }, p5/Z, [x20]\n" + "fmin z8.s, p5/M, z8.s, z17.s\n" + "fmin z9.s, p5/M, z9.s, z17.s\n" + "fmin z10.s, p5/M, z10.s, z17.s\n" + "fmin z11.s, p5/M, z11.s, z17.s\n" + "fmin z12.s, p5/M, z12.s, z17.s\n" + "fmin z13.s, p5/M, z13.s, z17.s\n" + "fmin z14.s, p5/M, z14.s, z17.s\n" + "fmin z15.s, p5/M, z15.s, z17.s\n" + "fmax z8.s, p5/M, z8.s, z16.s\n" + "fmax z9.s, p5/M, z9.s, z16.s\n" + "fmax z10.s, p5/M, z10.s, z16.s\n" + "fmax z11.s, p5/M, z11.s, z16.s\n" + "fmax z12.s, p5/M, z12.s, z16.s\n" + "fmax z13.s, p5/M, z13.s, z16.s\n" + "fmax z14.s, p5/M, z14.s, z16.s\n" + "fmax z15.s, p5/M, z15.s, z16.s\n" "25:" // Height 2: No activation "st1w { z8.s }, p4, [x9]\n" "st1w { z9.s }, p3, [x9, #1, MUL VL]\n" @@ -532,20 +532,20 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "29:" // Height 3: no bias "tbz %x[flags], #0, 30f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" + "add x21, x9, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z8.s }, p4/Z, [x9]\n" "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "ld1w { z16.s }, p4/Z, [x24]\n" - "ld1w { z17.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x24, #2, MUL VL]\n" - "ld1w { z19.s }, p1/Z, [x24, #3, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x21]\n" + "ld1w { z13.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x21, #2, MUL VL]\n" + "ld1w { z15.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x20]\n" + "ld1w { z17.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z19.s }, p1/Z, [x20, #3, MUL VL]\n" "b 31f\n" "30:" // Height 3: no accumulate "mov z8.b, #0x0\n" @@ -565,13 +565,13 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "32:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 33f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 34f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -580,86 +580,86 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "b 34f\n" "33:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" "34:" // Height 3: input setup done "cmp x27, #0x8\n" "ble 36f\n" "35:" // Height 3: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z0.h }, p0/Z, [x26]\n" + "ld1rqh { z2.h }, p0/Z, [x26]\n" "ld1rqh { z1.h }, p0/Z, [x25]\n" "sub x27, x27, #0x8\n" - "ld1rqh { z2.h }, p0/Z, [x24]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - ".inst 0x646040c8 // bfdot z8.s, z6.h, z0.h[0]\n" - ".inst 0x646140cc // bfdot z12.s, z6.h, z1.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x646240d0 // bfdot z16.s, z6.h, z2.h[0]\n" - ".inst 0x646040e9 // bfdot z9.s, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x646140ed // bfdot z13.s, z7.h, z1.h[0]\n" - ".inst 0x646240f1 // bfdot z17.s, z7.h, z2.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1rqh { z0.h }, p0/Z, [x24]\n" + "ld1h { z21.h }, p5/Z, [x10]\n" + ".inst 0x646242a8 // bfdot z8.s, z21.h, z2.h[0]\n" + ".inst 0x646142ac // bfdot z12.s, z21.h, z1.h[0]\n" + "ld1h { z20.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x646042b0 // bfdot z16.s, z21.h, z0.h[0]\n" + ".inst 0x64624289 // bfdot z9.s, z20.h, z2.h[0]\n" + "ld1h { z21.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x6461428d // bfdot z13.s, z20.h, z1.h[0]\n" + ".inst 0x64604291 // bfdot z17.s, z20.h, z0.h[0]\n" + "ld1h { z20.h }, p5/Z, [x10, #3, MUL VL]\n" "cmp x27, #0x8\n" - ".inst 0x646040ca // bfdot z10.s, z6.h, z0.h[0]\n" - ".inst 0x646140ce // bfdot z14.s, z6.h, z1.h[0]\n" + ".inst 0x646242aa // bfdot z10.s, z21.h, z2.h[0]\n" + ".inst 0x646142ae // bfdot z14.s, z21.h, z1.h[0]\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" - ".inst 0x646240d2 // bfdot z18.s, z6.h, z2.h[0]\n" - ".inst 0x646040eb // bfdot z11.s, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x646042b2 // bfdot z18.s, z21.h, z0.h[0]\n" + ".inst 0x6462428b // bfdot z11.s, z20.h, z2.h[0]\n" + "ld1h { z21.h }, p5/Z, [x10, #4, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x646140ef // bfdot z15.s, z7.h, z1.h[0]\n" - ".inst 0x646240f3 // bfdot z19.s, z7.h, z2.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x646840c8 // bfdot z8.s, z6.h, z0.h[1]\n" - ".inst 0x646940cc // bfdot z12.s, z6.h, z1.h[1]\n" - ".inst 0x646a40d0 // bfdot z16.s, z6.h, z2.h[1]\n" - ".inst 0x646840e9 // bfdot z9.s, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x646940ed // bfdot z13.s, z7.h, z1.h[1]\n" - ".inst 0x646a40f1 // bfdot z17.s, z7.h, z2.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x6461428f // bfdot z15.s, z20.h, z1.h[0]\n" + ".inst 0x64604293 // bfdot z19.s, z20.h, z0.h[0]\n" + "ld1h { z20.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x646a42a8 // bfdot z8.s, z21.h, z2.h[1]\n" + ".inst 0x646942ac // bfdot z12.s, z21.h, z1.h[1]\n" + ".inst 0x646842b0 // bfdot z16.s, z21.h, z0.h[1]\n" + ".inst 0x646a4289 // bfdot z9.s, z20.h, z2.h[1]\n" + "ld1h { z21.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x6469428d // bfdot z13.s, z20.h, z1.h[1]\n" + ".inst 0x64684291 // bfdot z17.s, z20.h, z0.h[1]\n" + "ld1h { z20.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - ".inst 0x646840ca // bfdot z10.s, z6.h, z0.h[1]\n" - ".inst 0x646940ce // bfdot z14.s, z6.h, z1.h[1]\n" - ".inst 0x646a40d2 // bfdot z18.s, z6.h, z2.h[1]\n" - ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n" - ".inst 0x646940ef // bfdot z15.s, z7.h, z1.h[1]\n" - ".inst 0x646a40f3 // bfdot z19.s, z7.h, z2.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #-7, MUL VL]\n" - ".inst 0x647040c8 // bfdot z8.s, z6.h, z0.h[2]\n" - ".inst 0x647140cc // bfdot z12.s, z6.h, z1.h[2]\n" - ".inst 0x647240d0 // bfdot z16.s, z6.h, z2.h[2]\n" - ".inst 0x647040e9 // bfdot z9.s, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n" - ".inst 0x647140ed // bfdot z13.s, z7.h, z1.h[2]\n" - ".inst 0x647240f1 // bfdot z17.s, z7.h, z2.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #-5, MUL VL]\n" - ".inst 0x647040ca // bfdot z10.s, z6.h, z0.h[2]\n" - ".inst 0x647140ce // bfdot z14.s, z6.h, z1.h[2]\n" - ".inst 0x647240d2 // bfdot z18.s, z6.h, z2.h[2]\n" - ".inst 0x647040eb // bfdot z11.s, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n" - ".inst 0x647140ef // bfdot z15.s, z7.h, z1.h[2]\n" - ".inst 0x647240f3 // bfdot z19.s, z7.h, z2.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #-3, MUL VL]\n" - ".inst 0x647840c8 // bfdot z8.s, z6.h, z0.h[3]\n" - ".inst 0x647940cc // bfdot z12.s, z6.h, z1.h[3]\n" - ".inst 0x647a40d0 // bfdot z16.s, z6.h, z2.h[3]\n" - ".inst 0x647840e9 // bfdot z9.s, z7.h, z0.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n" - ".inst 0x647940ed // bfdot z13.s, z7.h, z1.h[3]\n" - ".inst 0x647a40f1 // bfdot z17.s, z7.h, z2.h[3]\n" - "ld1h { z7.h }, p5/Z, [x10, #-1, MUL VL]\n" - ".inst 0x647840ca // bfdot z10.s, z6.h, z0.h[3]\n" - ".inst 0x647940ce // bfdot z14.s, z6.h, z1.h[3]\n" - ".inst 0x647a40d2 // bfdot z18.s, z6.h, z2.h[3]\n" - ".inst 0x647840eb // bfdot z11.s, z7.h, z0.h[3]\n" - ".inst 0x647940ef // bfdot z15.s, z7.h, z1.h[3]\n" - ".inst 0x647a40f3 // bfdot z19.s, z7.h, z2.h[3]\n" + ".inst 0x646a42aa // bfdot z10.s, z21.h, z2.h[1]\n" + ".inst 0x646942ae // bfdot z14.s, z21.h, z1.h[1]\n" + ".inst 0x646842b2 // bfdot z18.s, z21.h, z0.h[1]\n" + ".inst 0x646a428b // bfdot z11.s, z20.h, z2.h[1]\n" + "ld1h { z21.h }, p5/Z, [x10, #-8, MUL VL]\n" + ".inst 0x6469428f // bfdot z15.s, z20.h, z1.h[1]\n" + ".inst 0x64684293 // bfdot z19.s, z20.h, z0.h[1]\n" + "ld1h { z20.h }, p5/Z, [x10, #-7, MUL VL]\n" + ".inst 0x647242a8 // bfdot z8.s, z21.h, z2.h[2]\n" + ".inst 0x647142ac // bfdot z12.s, z21.h, z1.h[2]\n" + ".inst 0x647042b0 // bfdot z16.s, z21.h, z0.h[2]\n" + ".inst 0x64724289 // bfdot z9.s, z20.h, z2.h[2]\n" + "ld1h { z21.h }, p5/Z, [x10, #-6, MUL VL]\n" + ".inst 0x6471428d // bfdot z13.s, z20.h, z1.h[2]\n" + ".inst 0x64704291 // bfdot z17.s, z20.h, z0.h[2]\n" + "ld1h { z20.h }, p5/Z, [x10, #-5, MUL VL]\n" + ".inst 0x647242aa // bfdot z10.s, z21.h, z2.h[2]\n" + ".inst 0x647142ae // bfdot z14.s, z21.h, z1.h[2]\n" + ".inst 0x647042b2 // bfdot z18.s, z21.h, z0.h[2]\n" + ".inst 0x6472428b // bfdot z11.s, z20.h, z2.h[2]\n" + "ld1h { z21.h }, p5/Z, [x10, #-4, MUL VL]\n" + ".inst 0x6471428f // bfdot z15.s, z20.h, z1.h[2]\n" + ".inst 0x64704293 // bfdot z19.s, z20.h, z0.h[2]\n" + "ld1h { z20.h }, p5/Z, [x10, #-3, MUL VL]\n" + ".inst 0x647a42a8 // bfdot z8.s, z21.h, z2.h[3]\n" + ".inst 0x647942ac // bfdot z12.s, z21.h, z1.h[3]\n" + ".inst 0x647842b0 // bfdot z16.s, z21.h, z0.h[3]\n" + ".inst 0x647a4289 // bfdot z9.s, z20.h, z2.h[3]\n" + "ld1h { z21.h }, p5/Z, [x10, #-2, MUL VL]\n" + ".inst 0x6479428d // bfdot z13.s, z20.h, z1.h[3]\n" + ".inst 0x64784291 // bfdot z17.s, z20.h, z0.h[3]\n" + "ld1h { z20.h }, p5/Z, [x10, #-1, MUL VL]\n" + ".inst 0x647a42aa // bfdot z10.s, z21.h, z2.h[3]\n" + ".inst 0x647942ae // bfdot z14.s, z21.h, z1.h[3]\n" + ".inst 0x647842b2 // bfdot z18.s, z21.h, z0.h[3]\n" + ".inst 0x647a428b // bfdot z11.s, z20.h, z2.h[3]\n" + ".inst 0x6479428f // bfdot z15.s, z20.h, z1.h[3]\n" + ".inst 0x64784293 // bfdot z19.s, z20.h, z0.h[3]\n" "bgt 35b\n" "36:" // Height 3: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" @@ -667,79 +667,79 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "ld1rqh { z1.h }, p0/Z, [x25]\n" "subs x27, x27, #0x2\n" "ld1rqh { z2.h }, p0/Z, [x24]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - ".inst 0x646040c8 // bfdot z8.s, z6.h, z0.h[0]\n" - ".inst 0x646140cc // bfdot z12.s, z6.h, z1.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x646240d0 // bfdot z16.s, z6.h, z2.h[0]\n" - ".inst 0x646040e9 // bfdot z9.s, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x646140ed // bfdot z13.s, z7.h, z1.h[0]\n" - ".inst 0x646240f1 // bfdot z17.s, z7.h, z2.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z21.h }, p5/Z, [x10]\n" + ".inst 0x646042a8 // bfdot z8.s, z21.h, z0.h[0]\n" + ".inst 0x646142ac // bfdot z12.s, z21.h, z1.h[0]\n" + "ld1h { z20.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x646242b0 // bfdot z16.s, z21.h, z2.h[0]\n" + ".inst 0x64604289 // bfdot z9.s, z20.h, z0.h[0]\n" + "ld1h { z21.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x6461428d // bfdot z13.s, z20.h, z1.h[0]\n" + ".inst 0x64624291 // bfdot z17.s, z20.h, z2.h[0]\n" + "ld1h { z20.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - ".inst 0x646040ca // bfdot z10.s, z6.h, z0.h[0]\n" - ".inst 0x646140ce // bfdot z14.s, z6.h, z1.h[0]\n" - ".inst 0x646240d2 // bfdot z18.s, z6.h, z2.h[0]\n" - ".inst 0x646040eb // bfdot z11.s, z7.h, z0.h[0]\n" - ".inst 0x646140ef // bfdot z15.s, z7.h, z1.h[0]\n" - ".inst 0x646240f3 // bfdot z19.s, z7.h, z2.h[0]\n" + ".inst 0x646042aa // bfdot z10.s, z21.h, z0.h[0]\n" + ".inst 0x646142ae // bfdot z14.s, z21.h, z1.h[0]\n" + ".inst 0x646242b2 // bfdot z18.s, z21.h, z2.h[0]\n" + ".inst 0x6460428b // bfdot z11.s, z20.h, z0.h[0]\n" + ".inst 0x6461428f // bfdot z15.s, z20.h, z1.h[0]\n" + ".inst 0x64624293 // bfdot z19.s, z20.h, z2.h[0]\n" "ble 37f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x646840c8 // bfdot z8.s, z6.h, z0.h[1]\n" - ".inst 0x646940cc // bfdot z12.s, z6.h, z1.h[1]\n" - ".inst 0x646a40d0 // bfdot z16.s, z6.h, z2.h[1]\n" - ".inst 0x646840e9 // bfdot z9.s, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z21.h }, p5/Z, [x10]\n" + "ld1h { z20.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x646842a8 // bfdot z8.s, z21.h, z0.h[1]\n" + ".inst 0x646942ac // bfdot z12.s, z21.h, z1.h[1]\n" + ".inst 0x646a42b0 // bfdot z16.s, z21.h, z2.h[1]\n" + ".inst 0x64684289 // bfdot z9.s, z20.h, z0.h[1]\n" + "ld1h { z21.h }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x2\n" - ".inst 0x646940ed // bfdot z13.s, z7.h, z1.h[1]\n" - ".inst 0x646a40f1 // bfdot z17.s, z7.h, z2.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6469428d // bfdot z13.s, z20.h, z1.h[1]\n" + ".inst 0x646a4291 // bfdot z17.s, z20.h, z2.h[1]\n" + "ld1h { z20.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - ".inst 0x646840ca // bfdot z10.s, z6.h, z0.h[1]\n" - ".inst 0x646940ce // bfdot z14.s, z6.h, z1.h[1]\n" - ".inst 0x646a40d2 // bfdot z18.s, z6.h, z2.h[1]\n" - ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n" - ".inst 0x646940ef // bfdot z15.s, z7.h, z1.h[1]\n" - ".inst 0x646a40f3 // bfdot z19.s, z7.h, z2.h[1]\n" + ".inst 0x646842aa // bfdot z10.s, z21.h, z0.h[1]\n" + ".inst 0x646942ae // bfdot z14.s, z21.h, z1.h[1]\n" + ".inst 0x646a42b2 // bfdot z18.s, z21.h, z2.h[1]\n" + ".inst 0x6468428b // bfdot z11.s, z20.h, z0.h[1]\n" + ".inst 0x6469428f // bfdot z15.s, z20.h, z1.h[1]\n" + ".inst 0x646a4293 // bfdot z19.s, z20.h, z2.h[1]\n" "ble 37f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x647040c8 // bfdot z8.s, z6.h, z0.h[2]\n" - ".inst 0x647140cc // bfdot z12.s, z6.h, z1.h[2]\n" - ".inst 0x647240d0 // bfdot z16.s, z6.h, z2.h[2]\n" - ".inst 0x647040e9 // bfdot z9.s, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z21.h }, p5/Z, [x10]\n" + "ld1h { z20.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x647042a8 // bfdot z8.s, z21.h, z0.h[2]\n" + ".inst 0x647142ac // bfdot z12.s, z21.h, z1.h[2]\n" + ".inst 0x647242b0 // bfdot z16.s, z21.h, z2.h[2]\n" + ".inst 0x64704289 // bfdot z9.s, z20.h, z0.h[2]\n" + "ld1h { z21.h }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x2\n" - ".inst 0x647140ed // bfdot z13.s, z7.h, z1.h[2]\n" - ".inst 0x647240f1 // bfdot z17.s, z7.h, z2.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6471428d // bfdot z13.s, z20.h, z1.h[2]\n" + ".inst 0x64724291 // bfdot z17.s, z20.h, z2.h[2]\n" + "ld1h { z20.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - ".inst 0x647040ca // bfdot z10.s, z6.h, z0.h[2]\n" - ".inst 0x647140ce // bfdot z14.s, z6.h, z1.h[2]\n" - ".inst 0x647240d2 // bfdot z18.s, z6.h, z2.h[2]\n" - ".inst 0x647040eb // bfdot z11.s, z7.h, z0.h[2]\n" - ".inst 0x647140ef // bfdot z15.s, z7.h, z1.h[2]\n" - ".inst 0x647240f3 // bfdot z19.s, z7.h, z2.h[2]\n" + ".inst 0x647042aa // bfdot z10.s, z21.h, z0.h[2]\n" + ".inst 0x647142ae // bfdot z14.s, z21.h, z1.h[2]\n" + ".inst 0x647242b2 // bfdot z18.s, z21.h, z2.h[2]\n" + ".inst 0x6470428b // bfdot z11.s, z20.h, z0.h[2]\n" + ".inst 0x6471428f // bfdot z15.s, z20.h, z1.h[2]\n" + ".inst 0x64724293 // bfdot z19.s, z20.h, z2.h[2]\n" "ble 37f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x647840c8 // bfdot z8.s, z6.h, z0.h[3]\n" - ".inst 0x647940cc // bfdot z12.s, z6.h, z1.h[3]\n" - ".inst 0x647a40d0 // bfdot z16.s, z6.h, z2.h[3]\n" - ".inst 0x647840e9 // bfdot z9.s, z7.h, z0.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x647940ed // bfdot z13.s, z7.h, z1.h[3]\n" - ".inst 0x647a40f1 // bfdot z17.s, z7.h, z2.h[3]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z21.h }, p5/Z, [x10]\n" + "ld1h { z20.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x647842a8 // bfdot z8.s, z21.h, z0.h[3]\n" + ".inst 0x647942ac // bfdot z12.s, z21.h, z1.h[3]\n" + ".inst 0x647a42b0 // bfdot z16.s, z21.h, z2.h[3]\n" + ".inst 0x64784289 // bfdot z9.s, z20.h, z0.h[3]\n" + "ld1h { z21.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x6479428d // bfdot z13.s, z20.h, z1.h[3]\n" + ".inst 0x647a4291 // bfdot z17.s, z20.h, z2.h[3]\n" + "ld1h { z20.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - ".inst 0x647840ca // bfdot z10.s, z6.h, z0.h[3]\n" - ".inst 0x647940ce // bfdot z14.s, z6.h, z1.h[3]\n" - ".inst 0x647a40d2 // bfdot z18.s, z6.h, z2.h[3]\n" - ".inst 0x647840eb // bfdot z11.s, z7.h, z0.h[3]\n" - ".inst 0x647940ef // bfdot z15.s, z7.h, z1.h[3]\n" - ".inst 0x647a40f3 // bfdot z19.s, z7.h, z2.h[3]\n" + ".inst 0x647842aa // bfdot z10.s, z21.h, z0.h[3]\n" + ".inst 0x647942ae // bfdot z14.s, z21.h, z1.h[3]\n" + ".inst 0x647a42b2 // bfdot z18.s, z21.h, z2.h[3]\n" + ".inst 0x6478428b // bfdot z11.s, z20.h, z0.h[3]\n" + ".inst 0x6479428f // bfdot z15.s, z20.h, z1.h[3]\n" + ".inst 0x647a4293 // bfdot z19.s, z20.h, z2.h[3]\n" "37:" // Height 3: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -750,33 +750,33 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "add x24, x25, x20, LSL #2\n" "tbz %x[flags], #1, 38f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z21.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmin z12.s, p5/M, z12.s, z1.s\n" - "fmin z13.s, p5/M, z13.s, z1.s\n" - "fmin z14.s, p5/M, z14.s, z1.s\n" - "fmin z15.s, p5/M, z15.s, z1.s\n" - "fmin z16.s, p5/M, z16.s, z1.s\n" - "fmin z17.s, p5/M, z17.s, z1.s\n" - "fmin z18.s, p5/M, z18.s, z1.s\n" - "fmin z19.s, p5/M, z19.s, z1.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" - "fmax z12.s, p5/M, z12.s, z0.s\n" - "fmax z13.s, p5/M, z13.s, z0.s\n" - "fmax z14.s, p5/M, z14.s, z0.s\n" - "fmax z15.s, p5/M, z15.s, z0.s\n" - "fmax z16.s, p5/M, z16.s, z0.s\n" - "fmax z17.s, p5/M, z17.s, z0.s\n" - "fmax z18.s, p5/M, z18.s, z0.s\n" - "fmax z19.s, p5/M, z19.s, z0.s\n" + "ld1rw { z20.s }, p5/Z, [x20]\n" + "fmin z8.s, p5/M, z8.s, z21.s\n" + "fmin z9.s, p5/M, z9.s, z21.s\n" + "fmin z10.s, p5/M, z10.s, z21.s\n" + "fmin z11.s, p5/M, z11.s, z21.s\n" + "fmin z12.s, p5/M, z12.s, z21.s\n" + "fmin z13.s, p5/M, z13.s, z21.s\n" + "fmin z14.s, p5/M, z14.s, z21.s\n" + "fmin z15.s, p5/M, z15.s, z21.s\n" + "fmin z16.s, p5/M, z16.s, z21.s\n" + "fmin z17.s, p5/M, z17.s, z21.s\n" + "fmin z18.s, p5/M, z18.s, z21.s\n" + "fmin z19.s, p5/M, z19.s, z21.s\n" + "fmax z8.s, p5/M, z8.s, z20.s\n" + "fmax z9.s, p5/M, z9.s, z20.s\n" + "fmax z10.s, p5/M, z10.s, z20.s\n" + "fmax z11.s, p5/M, z11.s, z20.s\n" + "fmax z12.s, p5/M, z12.s, z20.s\n" + "fmax z13.s, p5/M, z13.s, z20.s\n" + "fmax z14.s, p5/M, z14.s, z20.s\n" + "fmax z15.s, p5/M, z15.s, z20.s\n" + "fmax z16.s, p5/M, z16.s, z20.s\n" + "fmax z17.s, p5/M, z17.s, z20.s\n" + "fmax z18.s, p5/M, z18.s, z20.s\n" + "fmax z19.s, p5/M, z19.s, z20.s\n" "38:" // Height 3: No activation "st1w { z8.s }, p4, [x9]\n" "st1w { z9.s }, p3, [x9, #1, MUL VL]\n" @@ -832,25 +832,25 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "42:" // Height 4: no bias "tbz %x[flags], #0, 43f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" + "add x22, x9, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" "ld1w { z8.s }, p4/Z, [x9]\n" - "add x23, x24, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "ld1w { z16.s }, p4/Z, [x24]\n" - "ld1w { z17.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x24, #2, MUL VL]\n" - "ld1w { z19.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x23]\n" - "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n" - "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x22]\n" + "ld1w { z13.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x22, #2, MUL VL]\n" + "ld1w { z15.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x21]\n" + "ld1w { z17.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x21, #2, MUL VL]\n" + "ld1w { z19.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x20]\n" + "ld1w { z21.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z23.s }, p1/Z, [x20, #3, MUL VL]\n" "b 44f\n" "43:" // Height 4: no accumulate "mov z8.b, #0x0\n" @@ -874,14 +874,14 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "45:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 46f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 47f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -891,105 +891,105 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "b 47f\n" "46:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" "47:" // Height 4: input setup done "cmp x27, #0x8\n" "ble 49f\n" "48:" // Height 4: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z0.h }, p0/Z, [x26]\n" - "ld1rqh { z1.h }, p0/Z, [x25]\n" + "ld1rqh { z3.h }, p0/Z, [x26]\n" + "ld1rqh { z2.h }, p0/Z, [x25]\n" "sub x27, x27, #0x8\n" - "ld1rqh { z2.h }, p0/Z, [x24]\n" - "ld1rqh { z3.h }, p0/Z, [x23]\n" + "ld1rqh { z1.h }, p0/Z, [x24]\n" + "ld1rqh { z0.h }, p0/Z, [x23]\n" "cmp x27, #0x8\n" "add x26, x26, #0x10\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x646040c8 // bfdot z8.s, z6.h, z0.h[0]\n" - ".inst 0x646140cc // bfdot z12.s, z6.h, z1.h[0]\n" - ".inst 0x646240d0 // bfdot z16.s, z6.h, z2.h[0]\n" - ".inst 0x646340d4 // bfdot z20.s, z6.h, z3.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x64634328 // bfdot z8.s, z25.h, z3.h[0]\n" + ".inst 0x6462432c // bfdot z12.s, z25.h, z2.h[0]\n" + ".inst 0x64614330 // bfdot z16.s, z25.h, z1.h[0]\n" + ".inst 0x64604334 // bfdot z20.s, z25.h, z0.h[0]\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" "add x25, x25, #0x10\n" - ".inst 0x646040e9 // bfdot z9.s, z7.h, z0.h[0]\n" - ".inst 0x646140ed // bfdot z13.s, z7.h, z1.h[0]\n" + ".inst 0x64634309 // bfdot z9.s, z24.h, z3.h[0]\n" + ".inst 0x6462430d // bfdot z13.s, z24.h, z2.h[0]\n" "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" - ".inst 0x646240f1 // bfdot z17.s, z7.h, z2.h[0]\n" - ".inst 0x646340f5 // bfdot z21.s, z7.h, z3.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x646040ca // bfdot z10.s, z6.h, z0.h[0]\n" - ".inst 0x646140ce // bfdot z14.s, z6.h, z1.h[0]\n" - ".inst 0x646240d2 // bfdot z18.s, z6.h, z2.h[0]\n" - ".inst 0x646340d6 // bfdot z22.s, z6.h, z3.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x646040eb // bfdot z11.s, z7.h, z0.h[0]\n" - ".inst 0x646140ef // bfdot z15.s, z7.h, z1.h[0]\n" - ".inst 0x646240f3 // bfdot z19.s, z7.h, z2.h[0]\n" - ".inst 0x646340f7 // bfdot z23.s, z7.h, z3.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x646840c8 // bfdot z8.s, z6.h, z0.h[1]\n" - ".inst 0x646940cc // bfdot z12.s, z6.h, z1.h[1]\n" - ".inst 0x646a40d0 // bfdot z16.s, z6.h, z2.h[1]\n" - ".inst 0x646b40d4 // bfdot z20.s, z6.h, z3.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x646840e9 // bfdot z9.s, z7.h, z0.h[1]\n" - ".inst 0x646940ed // bfdot z13.s, z7.h, z1.h[1]\n" - ".inst 0x646a40f1 // bfdot z17.s, z7.h, z2.h[1]\n" - ".inst 0x646b40f5 // bfdot z21.s, z7.h, z3.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x64614311 // bfdot z17.s, z24.h, z1.h[0]\n" + ".inst 0x64604315 // bfdot z21.s, z24.h, z0.h[0]\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6463432a // bfdot z10.s, z25.h, z3.h[0]\n" + ".inst 0x6462432e // bfdot z14.s, z25.h, z2.h[0]\n" + ".inst 0x64614332 // bfdot z18.s, z25.h, z1.h[0]\n" + ".inst 0x64604336 // bfdot z22.s, z25.h, z0.h[0]\n" + "ld1h { z25.h }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x6463430b // bfdot z11.s, z24.h, z3.h[0]\n" + ".inst 0x6462430f // bfdot z15.s, z24.h, z2.h[0]\n" + ".inst 0x64614313 // bfdot z19.s, z24.h, z1.h[0]\n" + ".inst 0x64604317 // bfdot z23.s, z24.h, z0.h[0]\n" + "ld1h { z24.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x646b4328 // bfdot z8.s, z25.h, z3.h[1]\n" + ".inst 0x646a432c // bfdot z12.s, z25.h, z2.h[1]\n" + ".inst 0x64694330 // bfdot z16.s, z25.h, z1.h[1]\n" + ".inst 0x64684334 // bfdot z20.s, z25.h, z0.h[1]\n" + "ld1h { z25.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x646b4309 // bfdot z9.s, z24.h, z3.h[1]\n" + ".inst 0x646a430d // bfdot z13.s, z24.h, z2.h[1]\n" + ".inst 0x64694311 // bfdot z17.s, z24.h, z1.h[1]\n" + ".inst 0x64684315 // bfdot z21.s, z24.h, z0.h[1]\n" + "ld1h { z24.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - ".inst 0x646840ca // bfdot z10.s, z6.h, z0.h[1]\n" - ".inst 0x646940ce // bfdot z14.s, z6.h, z1.h[1]\n" - ".inst 0x646a40d2 // bfdot z18.s, z6.h, z2.h[1]\n" - ".inst 0x646b40d6 // bfdot z22.s, z6.h, z3.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n" - ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n" - ".inst 0x646940ef // bfdot z15.s, z7.h, z1.h[1]\n" - ".inst 0x646a40f3 // bfdot z19.s, z7.h, z2.h[1]\n" - ".inst 0x646b40f7 // bfdot z23.s, z7.h, z3.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #-7, MUL VL]\n" - ".inst 0x647040c8 // bfdot z8.s, z6.h, z0.h[2]\n" - ".inst 0x647140cc // bfdot z12.s, z6.h, z1.h[2]\n" - ".inst 0x647240d0 // bfdot z16.s, z6.h, z2.h[2]\n" - ".inst 0x647340d4 // bfdot z20.s, z6.h, z3.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n" - ".inst 0x647040e9 // bfdot z9.s, z7.h, z0.h[2]\n" - ".inst 0x647140ed // bfdot z13.s, z7.h, z1.h[2]\n" - ".inst 0x647240f1 // bfdot z17.s, z7.h, z2.h[2]\n" - ".inst 0x647340f5 // bfdot z21.s, z7.h, z3.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #-5, MUL VL]\n" - ".inst 0x647040ca // bfdot z10.s, z6.h, z0.h[2]\n" - ".inst 0x647140ce // bfdot z14.s, z6.h, z1.h[2]\n" - ".inst 0x647240d2 // bfdot z18.s, z6.h, z2.h[2]\n" - ".inst 0x647340d6 // bfdot z22.s, z6.h, z3.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n" - ".inst 0x647040eb // bfdot z11.s, z7.h, z0.h[2]\n" - ".inst 0x647140ef // bfdot z15.s, z7.h, z1.h[2]\n" - ".inst 0x647240f3 // bfdot z19.s, z7.h, z2.h[2]\n" - ".inst 0x647340f7 // bfdot z23.s, z7.h, z3.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #-3, MUL VL]\n" - ".inst 0x647840c8 // bfdot z8.s, z6.h, z0.h[3]\n" - ".inst 0x647940cc // bfdot z12.s, z6.h, z1.h[3]\n" - ".inst 0x647a40d0 // bfdot z16.s, z6.h, z2.h[3]\n" - ".inst 0x647b40d4 // bfdot z20.s, z6.h, z3.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n" - ".inst 0x647840e9 // bfdot z9.s, z7.h, z0.h[3]\n" - ".inst 0x647940ed // bfdot z13.s, z7.h, z1.h[3]\n" - ".inst 0x647a40f1 // bfdot z17.s, z7.h, z2.h[3]\n" - ".inst 0x647b40f5 // bfdot z21.s, z7.h, z3.h[3]\n" - "ld1h { z7.h }, p5/Z, [x10, #-1, MUL VL]\n" - ".inst 0x647840ca // bfdot z10.s, z6.h, z0.h[3]\n" - ".inst 0x647940ce // bfdot z14.s, z6.h, z1.h[3]\n" - ".inst 0x647a40d2 // bfdot z18.s, z6.h, z2.h[3]\n" - ".inst 0x647b40d6 // bfdot z22.s, z6.h, z3.h[3]\n" - ".inst 0x647840eb // bfdot z11.s, z7.h, z0.h[3]\n" - ".inst 0x647940ef // bfdot z15.s, z7.h, z1.h[3]\n" - ".inst 0x647a40f3 // bfdot z19.s, z7.h, z2.h[3]\n" - ".inst 0x647b40f7 // bfdot z23.s, z7.h, z3.h[3]\n" + ".inst 0x646b432a // bfdot z10.s, z25.h, z3.h[1]\n" + ".inst 0x646a432e // bfdot z14.s, z25.h, z2.h[1]\n" + ".inst 0x64694332 // bfdot z18.s, z25.h, z1.h[1]\n" + ".inst 0x64684336 // bfdot z22.s, z25.h, z0.h[1]\n" + "ld1h { z25.h }, p5/Z, [x10, #-8, MUL VL]\n" + ".inst 0x646b430b // bfdot z11.s, z24.h, z3.h[1]\n" + ".inst 0x646a430f // bfdot z15.s, z24.h, z2.h[1]\n" + ".inst 0x64694313 // bfdot z19.s, z24.h, z1.h[1]\n" + ".inst 0x64684317 // bfdot z23.s, z24.h, z0.h[1]\n" + "ld1h { z24.h }, p5/Z, [x10, #-7, MUL VL]\n" + ".inst 0x64734328 // bfdot z8.s, z25.h, z3.h[2]\n" + ".inst 0x6472432c // bfdot z12.s, z25.h, z2.h[2]\n" + ".inst 0x64714330 // bfdot z16.s, z25.h, z1.h[2]\n" + ".inst 0x64704334 // bfdot z20.s, z25.h, z0.h[2]\n" + "ld1h { z25.h }, p5/Z, [x10, #-6, MUL VL]\n" + ".inst 0x64734309 // bfdot z9.s, z24.h, z3.h[2]\n" + ".inst 0x6472430d // bfdot z13.s, z24.h, z2.h[2]\n" + ".inst 0x64714311 // bfdot z17.s, z24.h, z1.h[2]\n" + ".inst 0x64704315 // bfdot z21.s, z24.h, z0.h[2]\n" + "ld1h { z24.h }, p5/Z, [x10, #-5, MUL VL]\n" + ".inst 0x6473432a // bfdot z10.s, z25.h, z3.h[2]\n" + ".inst 0x6472432e // bfdot z14.s, z25.h, z2.h[2]\n" + ".inst 0x64714332 // bfdot z18.s, z25.h, z1.h[2]\n" + ".inst 0x64704336 // bfdot z22.s, z25.h, z0.h[2]\n" + "ld1h { z25.h }, p5/Z, [x10, #-4, MUL VL]\n" + ".inst 0x6473430b // bfdot z11.s, z24.h, z3.h[2]\n" + ".inst 0x6472430f // bfdot z15.s, z24.h, z2.h[2]\n" + ".inst 0x64714313 // bfdot z19.s, z24.h, z1.h[2]\n" + ".inst 0x64704317 // bfdot z23.s, z24.h, z0.h[2]\n" + "ld1h { z24.h }, p5/Z, [x10, #-3, MUL VL]\n" + ".inst 0x647b4328 // bfdot z8.s, z25.h, z3.h[3]\n" + ".inst 0x647a432c // bfdot z12.s, z25.h, z2.h[3]\n" + ".inst 0x64794330 // bfdot z16.s, z25.h, z1.h[3]\n" + ".inst 0x64784334 // bfdot z20.s, z25.h, z0.h[3]\n" + "ld1h { z25.h }, p5/Z, [x10, #-2, MUL VL]\n" + ".inst 0x647b4309 // bfdot z9.s, z24.h, z3.h[3]\n" + ".inst 0x647a430d // bfdot z13.s, z24.h, z2.h[3]\n" + ".inst 0x64794311 // bfdot z17.s, z24.h, z1.h[3]\n" + ".inst 0x64784315 // bfdot z21.s, z24.h, z0.h[3]\n" + "ld1h { z24.h }, p5/Z, [x10, #-1, MUL VL]\n" + ".inst 0x647b432a // bfdot z10.s, z25.h, z3.h[3]\n" + ".inst 0x647a432e // bfdot z14.s, z25.h, z2.h[3]\n" + ".inst 0x64794332 // bfdot z18.s, z25.h, z1.h[3]\n" + ".inst 0x64784336 // bfdot z22.s, z25.h, z0.h[3]\n" + ".inst 0x647b430b // bfdot z11.s, z24.h, z3.h[3]\n" + ".inst 0x647a430f // bfdot z15.s, z24.h, z2.h[3]\n" + ".inst 0x64794313 // bfdot z19.s, z24.h, z1.h[3]\n" + ".inst 0x64784317 // bfdot z23.s, z24.h, z0.h[3]\n" "bgt 48b\n" "49:" // Height 4: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" @@ -998,95 +998,95 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "subs x27, x27, #0x2\n" "ld1rqh { z2.h }, p0/Z, [x24]\n" "ld1rqh { z3.h }, p0/Z, [x23]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x646040c8 // bfdot z8.s, z6.h, z0.h[0]\n" - ".inst 0x646140cc // bfdot z12.s, z6.h, z1.h[0]\n" - ".inst 0x646240d0 // bfdot z16.s, z6.h, z2.h[0]\n" - ".inst 0x646340d4 // bfdot z20.s, z6.h, z3.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x646040e9 // bfdot z9.s, z7.h, z0.h[0]\n" - ".inst 0x646140ed // bfdot z13.s, z7.h, z1.h[0]\n" - ".inst 0x646240f1 // bfdot z17.s, z7.h, z2.h[0]\n" - ".inst 0x646340f5 // bfdot z21.s, z7.h, z3.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x64604328 // bfdot z8.s, z25.h, z0.h[0]\n" + ".inst 0x6461432c // bfdot z12.s, z25.h, z1.h[0]\n" + ".inst 0x64624330 // bfdot z16.s, z25.h, z2.h[0]\n" + ".inst 0x64634334 // bfdot z20.s, z25.h, z3.h[0]\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x64604309 // bfdot z9.s, z24.h, z0.h[0]\n" + ".inst 0x6461430d // bfdot z13.s, z24.h, z1.h[0]\n" + ".inst 0x64624311 // bfdot z17.s, z24.h, z2.h[0]\n" + ".inst 0x64634315 // bfdot z21.s, z24.h, z3.h[0]\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - ".inst 0x646040ca // bfdot z10.s, z6.h, z0.h[0]\n" - ".inst 0x646140ce // bfdot z14.s, z6.h, z1.h[0]\n" - ".inst 0x646240d2 // bfdot z18.s, z6.h, z2.h[0]\n" - ".inst 0x646340d6 // bfdot z22.s, z6.h, z3.h[0]\n" - ".inst 0x646040eb // bfdot z11.s, z7.h, z0.h[0]\n" - ".inst 0x646140ef // bfdot z15.s, z7.h, z1.h[0]\n" - ".inst 0x646240f3 // bfdot z19.s, z7.h, z2.h[0]\n" - ".inst 0x646340f7 // bfdot z23.s, z7.h, z3.h[0]\n" + ".inst 0x6460432a // bfdot z10.s, z25.h, z0.h[0]\n" + ".inst 0x6461432e // bfdot z14.s, z25.h, z1.h[0]\n" + ".inst 0x64624332 // bfdot z18.s, z25.h, z2.h[0]\n" + ".inst 0x64634336 // bfdot z22.s, z25.h, z3.h[0]\n" + ".inst 0x6460430b // bfdot z11.s, z24.h, z0.h[0]\n" + ".inst 0x6461430f // bfdot z15.s, z24.h, z1.h[0]\n" + ".inst 0x64624313 // bfdot z19.s, z24.h, z2.h[0]\n" + ".inst 0x64634317 // bfdot z23.s, z24.h, z3.h[0]\n" "ble 50f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x646840c8 // bfdot z8.s, z6.h, z0.h[1]\n" - ".inst 0x646940cc // bfdot z12.s, z6.h, z1.h[1]\n" - ".inst 0x646a40d0 // bfdot z16.s, z6.h, z2.h[1]\n" - ".inst 0x646b40d4 // bfdot z20.s, z6.h, z3.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x64684328 // bfdot z8.s, z25.h, z0.h[1]\n" + ".inst 0x6469432c // bfdot z12.s, z25.h, z1.h[1]\n" + ".inst 0x646a4330 // bfdot z16.s, z25.h, z2.h[1]\n" + ".inst 0x646b4334 // bfdot z20.s, z25.h, z3.h[1]\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x2\n" - ".inst 0x646840e9 // bfdot z9.s, z7.h, z0.h[1]\n" - ".inst 0x646940ed // bfdot z13.s, z7.h, z1.h[1]\n" - ".inst 0x646a40f1 // bfdot z17.s, z7.h, z2.h[1]\n" - ".inst 0x646b40f5 // bfdot z21.s, z7.h, z3.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x64684309 // bfdot z9.s, z24.h, z0.h[1]\n" + ".inst 0x6469430d // bfdot z13.s, z24.h, z1.h[1]\n" + ".inst 0x646a4311 // bfdot z17.s, z24.h, z2.h[1]\n" + ".inst 0x646b4315 // bfdot z21.s, z24.h, z3.h[1]\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - ".inst 0x646840ca // bfdot z10.s, z6.h, z0.h[1]\n" - ".inst 0x646940ce // bfdot z14.s, z6.h, z1.h[1]\n" - ".inst 0x646a40d2 // bfdot z18.s, z6.h, z2.h[1]\n" - ".inst 0x646b40d6 // bfdot z22.s, z6.h, z3.h[1]\n" - ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n" - ".inst 0x646940ef // bfdot z15.s, z7.h, z1.h[1]\n" - ".inst 0x646a40f3 // bfdot z19.s, z7.h, z2.h[1]\n" - ".inst 0x646b40f7 // bfdot z23.s, z7.h, z3.h[1]\n" + ".inst 0x6468432a // bfdot z10.s, z25.h, z0.h[1]\n" + ".inst 0x6469432e // bfdot z14.s, z25.h, z1.h[1]\n" + ".inst 0x646a4332 // bfdot z18.s, z25.h, z2.h[1]\n" + ".inst 0x646b4336 // bfdot z22.s, z25.h, z3.h[1]\n" + ".inst 0x6468430b // bfdot z11.s, z24.h, z0.h[1]\n" + ".inst 0x6469430f // bfdot z15.s, z24.h, z1.h[1]\n" + ".inst 0x646a4313 // bfdot z19.s, z24.h, z2.h[1]\n" + ".inst 0x646b4317 // bfdot z23.s, z24.h, z3.h[1]\n" "ble 50f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x647040c8 // bfdot z8.s, z6.h, z0.h[2]\n" - ".inst 0x647140cc // bfdot z12.s, z6.h, z1.h[2]\n" - ".inst 0x647240d0 // bfdot z16.s, z6.h, z2.h[2]\n" - ".inst 0x647340d4 // bfdot z20.s, z6.h, z3.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x64704328 // bfdot z8.s, z25.h, z0.h[2]\n" + ".inst 0x6471432c // bfdot z12.s, z25.h, z1.h[2]\n" + ".inst 0x64724330 // bfdot z16.s, z25.h, z2.h[2]\n" + ".inst 0x64734334 // bfdot z20.s, z25.h, z3.h[2]\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x2\n" - ".inst 0x647040e9 // bfdot z9.s, z7.h, z0.h[2]\n" - ".inst 0x647140ed // bfdot z13.s, z7.h, z1.h[2]\n" - ".inst 0x647240f1 // bfdot z17.s, z7.h, z2.h[2]\n" - ".inst 0x647340f5 // bfdot z21.s, z7.h, z3.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x64704309 // bfdot z9.s, z24.h, z0.h[2]\n" + ".inst 0x6471430d // bfdot z13.s, z24.h, z1.h[2]\n" + ".inst 0x64724311 // bfdot z17.s, z24.h, z2.h[2]\n" + ".inst 0x64734315 // bfdot z21.s, z24.h, z3.h[2]\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - ".inst 0x647040ca // bfdot z10.s, z6.h, z0.h[2]\n" - ".inst 0x647140ce // bfdot z14.s, z6.h, z1.h[2]\n" - ".inst 0x647240d2 // bfdot z18.s, z6.h, z2.h[2]\n" - ".inst 0x647340d6 // bfdot z22.s, z6.h, z3.h[2]\n" - ".inst 0x647040eb // bfdot z11.s, z7.h, z0.h[2]\n" - ".inst 0x647140ef // bfdot z15.s, z7.h, z1.h[2]\n" - ".inst 0x647240f3 // bfdot z19.s, z7.h, z2.h[2]\n" - ".inst 0x647340f7 // bfdot z23.s, z7.h, z3.h[2]\n" + ".inst 0x6470432a // bfdot z10.s, z25.h, z0.h[2]\n" + ".inst 0x6471432e // bfdot z14.s, z25.h, z1.h[2]\n" + ".inst 0x64724332 // bfdot z18.s, z25.h, z2.h[2]\n" + ".inst 0x64734336 // bfdot z22.s, z25.h, z3.h[2]\n" + ".inst 0x6470430b // bfdot z11.s, z24.h, z0.h[2]\n" + ".inst 0x6471430f // bfdot z15.s, z24.h, z1.h[2]\n" + ".inst 0x64724313 // bfdot z19.s, z24.h, z2.h[2]\n" + ".inst 0x64734317 // bfdot z23.s, z24.h, z3.h[2]\n" "ble 50f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x647840c8 // bfdot z8.s, z6.h, z0.h[3]\n" - ".inst 0x647940cc // bfdot z12.s, z6.h, z1.h[3]\n" - ".inst 0x647a40d0 // bfdot z16.s, z6.h, z2.h[3]\n" - ".inst 0x647b40d4 // bfdot z20.s, z6.h, z3.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x647840e9 // bfdot z9.s, z7.h, z0.h[3]\n" - ".inst 0x647940ed // bfdot z13.s, z7.h, z1.h[3]\n" - ".inst 0x647a40f1 // bfdot z17.s, z7.h, z2.h[3]\n" - ".inst 0x647b40f5 // bfdot z21.s, z7.h, z3.h[3]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x64784328 // bfdot z8.s, z25.h, z0.h[3]\n" + ".inst 0x6479432c // bfdot z12.s, z25.h, z1.h[3]\n" + ".inst 0x647a4330 // bfdot z16.s, z25.h, z2.h[3]\n" + ".inst 0x647b4334 // bfdot z20.s, z25.h, z3.h[3]\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x64784309 // bfdot z9.s, z24.h, z0.h[3]\n" + ".inst 0x6479430d // bfdot z13.s, z24.h, z1.h[3]\n" + ".inst 0x647a4311 // bfdot z17.s, z24.h, z2.h[3]\n" + ".inst 0x647b4315 // bfdot z21.s, z24.h, z3.h[3]\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - ".inst 0x647840ca // bfdot z10.s, z6.h, z0.h[3]\n" - ".inst 0x647940ce // bfdot z14.s, z6.h, z1.h[3]\n" - ".inst 0x647a40d2 // bfdot z18.s, z6.h, z2.h[3]\n" - ".inst 0x647b40d6 // bfdot z22.s, z6.h, z3.h[3]\n" - ".inst 0x647840eb // bfdot z11.s, z7.h, z0.h[3]\n" - ".inst 0x647940ef // bfdot z15.s, z7.h, z1.h[3]\n" - ".inst 0x647a40f3 // bfdot z19.s, z7.h, z2.h[3]\n" - ".inst 0x647b40f7 // bfdot z23.s, z7.h, z3.h[3]\n" + ".inst 0x6478432a // bfdot z10.s, z25.h, z0.h[3]\n" + ".inst 0x6479432e // bfdot z14.s, z25.h, z1.h[3]\n" + ".inst 0x647a4332 // bfdot z18.s, z25.h, z2.h[3]\n" + ".inst 0x647b4336 // bfdot z22.s, z25.h, z3.h[3]\n" + ".inst 0x6478430b // bfdot z11.s, z24.h, z0.h[3]\n" + ".inst 0x6479430f // bfdot z15.s, z24.h, z1.h[3]\n" + ".inst 0x647a4313 // bfdot z19.s, z24.h, z2.h[3]\n" + ".inst 0x647b4317 // bfdot z23.s, z24.h, z3.h[3]\n" "50:" // Height 4: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1098,41 +1098,41 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "add x23, x24, x20, LSL #2\n" "tbz %x[flags], #1, 51f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z25.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmin z12.s, p5/M, z12.s, z1.s\n" - "fmin z13.s, p5/M, z13.s, z1.s\n" - "fmin z14.s, p5/M, z14.s, z1.s\n" - "fmin z15.s, p5/M, z15.s, z1.s\n" - "fmin z16.s, p5/M, z16.s, z1.s\n" - "fmin z17.s, p5/M, z17.s, z1.s\n" - "fmin z18.s, p5/M, z18.s, z1.s\n" - "fmin z19.s, p5/M, z19.s, z1.s\n" - "fmin z20.s, p5/M, z20.s, z1.s\n" - "fmin z21.s, p5/M, z21.s, z1.s\n" - "fmin z22.s, p5/M, z22.s, z1.s\n" - "fmin z23.s, p5/M, z23.s, z1.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" - "fmax z12.s, p5/M, z12.s, z0.s\n" - "fmax z13.s, p5/M, z13.s, z0.s\n" - "fmax z14.s, p5/M, z14.s, z0.s\n" - "fmax z15.s, p5/M, z15.s, z0.s\n" - "fmax z16.s, p5/M, z16.s, z0.s\n" - "fmax z17.s, p5/M, z17.s, z0.s\n" - "fmax z18.s, p5/M, z18.s, z0.s\n" - "fmax z19.s, p5/M, z19.s, z0.s\n" - "fmax z20.s, p5/M, z20.s, z0.s\n" - "fmax z21.s, p5/M, z21.s, z0.s\n" - "fmax z22.s, p5/M, z22.s, z0.s\n" - "fmax z23.s, p5/M, z23.s, z0.s\n" + "ld1rw { z24.s }, p5/Z, [x20]\n" + "fmin z8.s, p5/M, z8.s, z25.s\n" + "fmin z9.s, p5/M, z9.s, z25.s\n" + "fmin z10.s, p5/M, z10.s, z25.s\n" + "fmin z11.s, p5/M, z11.s, z25.s\n" + "fmin z12.s, p5/M, z12.s, z25.s\n" + "fmin z13.s, p5/M, z13.s, z25.s\n" + "fmin z14.s, p5/M, z14.s, z25.s\n" + "fmin z15.s, p5/M, z15.s, z25.s\n" + "fmin z16.s, p5/M, z16.s, z25.s\n" + "fmin z17.s, p5/M, z17.s, z25.s\n" + "fmin z18.s, p5/M, z18.s, z25.s\n" + "fmin z19.s, p5/M, z19.s, z25.s\n" + "fmin z20.s, p5/M, z20.s, z25.s\n" + "fmin z21.s, p5/M, z21.s, z25.s\n" + "fmin z22.s, p5/M, z22.s, z25.s\n" + "fmin z23.s, p5/M, z23.s, z25.s\n" + "fmax z8.s, p5/M, z8.s, z24.s\n" + "fmax z9.s, p5/M, z9.s, z24.s\n" + "fmax z10.s, p5/M, z10.s, z24.s\n" + "fmax z11.s, p5/M, z11.s, z24.s\n" + "fmax z12.s, p5/M, z12.s, z24.s\n" + "fmax z13.s, p5/M, z13.s, z24.s\n" + "fmax z14.s, p5/M, z14.s, z24.s\n" + "fmax z15.s, p5/M, z15.s, z24.s\n" + "fmax z16.s, p5/M, z16.s, z24.s\n" + "fmax z17.s, p5/M, z17.s, z24.s\n" + "fmax z18.s, p5/M, z18.s, z24.s\n" + "fmax z19.s, p5/M, z19.s, z24.s\n" + "fmax z20.s, p5/M, z20.s, z24.s\n" + "fmax z21.s, p5/M, z21.s, z24.s\n" + "fmax z22.s, p5/M, z22.s, z24.s\n" + "fmax z23.s, p5/M, z23.s, z24.s\n" "51:" // Height 4: No activation "st1w { z8.s }, p4, [x9]\n" "st1w { z9.s }, p3, [x9, #1, MUL VL]\n" @@ -1196,30 +1196,30 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "55:" // Height 5: no bias "tbz %x[flags], #0, 56f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z8.s }, p4/Z, [x9]\n" - "add x23, x24, x20, LSL #2\n" + "add x23, x9, x20, LSL #2\n" "add x22, x23, x20, LSL #2\n" + "ld1w { z8.s }, p4/Z, [x9]\n" + "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "ld1w { z16.s }, p4/Z, [x24]\n" - "ld1w { z17.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x24, #2, MUL VL]\n" - "ld1w { z19.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x23]\n" - "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n" - "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n" - "ld1w { z24.s }, p4/Z, [x22]\n" - "ld1w { z25.s }, p3/Z, [x22, #1, MUL VL]\n" - "ld1w { z26.s }, p2/Z, [x22, #2, MUL VL]\n" - "ld1w { z27.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x23]\n" + "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n" + "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x22]\n" + "ld1w { z17.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x22, #2, MUL VL]\n" + "ld1w { z19.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x21]\n" + "ld1w { z21.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x21, #2, MUL VL]\n" + "ld1w { z23.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z24.s }, p4/Z, [x20]\n" + "ld1w { z25.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z26.s }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z27.s }, p1/Z, [x20, #3, MUL VL]\n" "b 57f\n" "56:" // Height 5: no accumulate "mov z8.b, #0x0\n" @@ -1247,15 +1247,15 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "58:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 59f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 60f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -1266,124 +1266,124 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "b 60f\n" "59:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" - "add x22, x23, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" + "add x22, x23, x21, LSL #1\n" "60:" // Height 5: input setup done "cmp x27, #0x8\n" "ble 62f\n" "61:" // Height 5: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z0.h }, p0/Z, [x26]\n" - "ld1rqh { z1.h }, p0/Z, [x25]\n" + "ld1rqh { z4.h }, p0/Z, [x26]\n" + "ld1rqh { z3.h }, p0/Z, [x25]\n" "sub x27, x27, #0x8\n" "ld1rqh { z2.h }, p0/Z, [x24]\n" - "ld1rqh { z3.h }, p0/Z, [x23]\n" + "ld1rqh { z1.h }, p0/Z, [x23]\n" "cmp x27, #0x8\n" "add x26, x26, #0x10\n" - "ld1rqh { z4.h }, p0/Z, [x22]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - ".inst 0x646040c8 // bfdot z8.s, z6.h, z0.h[0]\n" - ".inst 0x646140cc // bfdot z12.s, z6.h, z1.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x646240d0 // bfdot z16.s, z6.h, z2.h[0]\n" - ".inst 0x646340d4 // bfdot z20.s, z6.h, z3.h[0]\n" + "ld1rqh { z0.h }, p0/Z, [x22]\n" + "ld1h { z29.h }, p5/Z, [x10]\n" + ".inst 0x646443a8 // bfdot z8.s, z29.h, z4.h[0]\n" + ".inst 0x646343ac // bfdot z12.s, z29.h, z3.h[0]\n" + "ld1h { z28.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x646243b0 // bfdot z16.s, z29.h, z2.h[0]\n" + ".inst 0x646143b4 // bfdot z20.s, z29.h, z1.h[0]\n" "add x25, x25, #0x10\n" - ".inst 0x646440d8 // bfdot z24.s, z6.h, z4.h[0]\n" - ".inst 0x646040e9 // bfdot z9.s, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x646043b8 // bfdot z24.s, z29.h, z0.h[0]\n" + ".inst 0x64644389 // bfdot z9.s, z28.h, z4.h[0]\n" + "ld1h { z29.h }, p5/Z, [x10, #2, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x646140ed // bfdot z13.s, z7.h, z1.h[0]\n" - ".inst 0x646240f1 // bfdot z17.s, z7.h, z2.h[0]\n" + ".inst 0x6463438d // bfdot z13.s, z28.h, z3.h[0]\n" + ".inst 0x64624391 // bfdot z17.s, z28.h, z2.h[0]\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - ".inst 0x646340f5 // bfdot z21.s, z7.h, z3.h[0]\n" - ".inst 0x646440f9 // bfdot z25.s, z7.h, z4.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x646040ca // bfdot z10.s, z6.h, z0.h[0]\n" - ".inst 0x646140ce // bfdot z14.s, z6.h, z1.h[0]\n" - ".inst 0x646240d2 // bfdot z18.s, z6.h, z2.h[0]\n" - ".inst 0x646340d6 // bfdot z22.s, z6.h, z3.h[0]\n" - ".inst 0x646440da // bfdot z26.s, z6.h, z4.h[0]\n" - ".inst 0x646040eb // bfdot z11.s, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x646140ef // bfdot z15.s, z7.h, z1.h[0]\n" - ".inst 0x646240f3 // bfdot z19.s, z7.h, z2.h[0]\n" - ".inst 0x646340f7 // bfdot z23.s, z7.h, z3.h[0]\n" - ".inst 0x646440fb // bfdot z27.s, z7.h, z4.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x646840c8 // bfdot z8.s, z6.h, z0.h[1]\n" - ".inst 0x646940cc // bfdot z12.s, z6.h, z1.h[1]\n" - ".inst 0x646a40d0 // bfdot z16.s, z6.h, z2.h[1]\n" - ".inst 0x646b40d4 // bfdot z20.s, z6.h, z3.h[1]\n" - ".inst 0x646c40d8 // bfdot z24.s, z6.h, z4.h[1]\n" - ".inst 0x646840e9 // bfdot z9.s, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x646940ed // bfdot z13.s, z7.h, z1.h[1]\n" - ".inst 0x646a40f1 // bfdot z17.s, z7.h, z2.h[1]\n" - ".inst 0x646b40f5 // bfdot z21.s, z7.h, z3.h[1]\n" - ".inst 0x646c40f9 // bfdot z25.s, z7.h, z4.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x64614395 // bfdot z21.s, z28.h, z1.h[0]\n" + ".inst 0x64604399 // bfdot z25.s, z28.h, z0.h[0]\n" + "ld1h { z28.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x646443aa // bfdot z10.s, z29.h, z4.h[0]\n" + ".inst 0x646343ae // bfdot z14.s, z29.h, z3.h[0]\n" + ".inst 0x646243b2 // bfdot z18.s, z29.h, z2.h[0]\n" + ".inst 0x646143b6 // bfdot z22.s, z29.h, z1.h[0]\n" + ".inst 0x646043ba // bfdot z26.s, z29.h, z0.h[0]\n" + ".inst 0x6464438b // bfdot z11.s, z28.h, z4.h[0]\n" + "ld1h { z29.h }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x6463438f // bfdot z15.s, z28.h, z3.h[0]\n" + ".inst 0x64624393 // bfdot z19.s, z28.h, z2.h[0]\n" + ".inst 0x64614397 // bfdot z23.s, z28.h, z1.h[0]\n" + ".inst 0x6460439b // bfdot z27.s, z28.h, z0.h[0]\n" + "ld1h { z28.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x646c43a8 // bfdot z8.s, z29.h, z4.h[1]\n" + ".inst 0x646b43ac // bfdot z12.s, z29.h, z3.h[1]\n" + ".inst 0x646a43b0 // bfdot z16.s, z29.h, z2.h[1]\n" + ".inst 0x646943b4 // bfdot z20.s, z29.h, z1.h[1]\n" + ".inst 0x646843b8 // bfdot z24.s, z29.h, z0.h[1]\n" + ".inst 0x646c4389 // bfdot z9.s, z28.h, z4.h[1]\n" + "ld1h { z29.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x646b438d // bfdot z13.s, z28.h, z3.h[1]\n" + ".inst 0x646a4391 // bfdot z17.s, z28.h, z2.h[1]\n" + ".inst 0x64694395 // bfdot z21.s, z28.h, z1.h[1]\n" + ".inst 0x64684399 // bfdot z25.s, z28.h, z0.h[1]\n" + "ld1h { z28.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - ".inst 0x646840ca // bfdot z10.s, z6.h, z0.h[1]\n" - ".inst 0x646940ce // bfdot z14.s, z6.h, z1.h[1]\n" - ".inst 0x646a40d2 // bfdot z18.s, z6.h, z2.h[1]\n" - ".inst 0x646b40d6 // bfdot z22.s, z6.h, z3.h[1]\n" - ".inst 0x646c40da // bfdot z26.s, z6.h, z4.h[1]\n" - ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n" - ".inst 0x646940ef // bfdot z15.s, z7.h, z1.h[1]\n" - ".inst 0x646a40f3 // bfdot z19.s, z7.h, z2.h[1]\n" - ".inst 0x646b40f7 // bfdot z23.s, z7.h, z3.h[1]\n" - ".inst 0x646c40fb // bfdot z27.s, z7.h, z4.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #-7, MUL VL]\n" - ".inst 0x647040c8 // bfdot z8.s, z6.h, z0.h[2]\n" - ".inst 0x647140cc // bfdot z12.s, z6.h, z1.h[2]\n" - ".inst 0x647240d0 // bfdot z16.s, z6.h, z2.h[2]\n" - ".inst 0x647340d4 // bfdot z20.s, z6.h, z3.h[2]\n" - ".inst 0x647440d8 // bfdot z24.s, z6.h, z4.h[2]\n" - ".inst 0x647040e9 // bfdot z9.s, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n" - ".inst 0x647140ed // bfdot z13.s, z7.h, z1.h[2]\n" - ".inst 0x647240f1 // bfdot z17.s, z7.h, z2.h[2]\n" - ".inst 0x647340f5 // bfdot z21.s, z7.h, z3.h[2]\n" - ".inst 0x647440f9 // bfdot z25.s, z7.h, z4.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #-5, MUL VL]\n" - ".inst 0x647040ca // bfdot z10.s, z6.h, z0.h[2]\n" - ".inst 0x647140ce // bfdot z14.s, z6.h, z1.h[2]\n" - ".inst 0x647240d2 // bfdot z18.s, z6.h, z2.h[2]\n" - ".inst 0x647340d6 // bfdot z22.s, z6.h, z3.h[2]\n" - ".inst 0x647440da // bfdot z26.s, z6.h, z4.h[2]\n" - ".inst 0x647040eb // bfdot z11.s, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n" - ".inst 0x647140ef // bfdot z15.s, z7.h, z1.h[2]\n" - ".inst 0x647240f3 // bfdot z19.s, z7.h, z2.h[2]\n" - ".inst 0x647340f7 // bfdot z23.s, z7.h, z3.h[2]\n" - ".inst 0x647440fb // bfdot z27.s, z7.h, z4.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #-3, MUL VL]\n" - ".inst 0x647840c8 // bfdot z8.s, z6.h, z0.h[3]\n" - ".inst 0x647940cc // bfdot z12.s, z6.h, z1.h[3]\n" - ".inst 0x647a40d0 // bfdot z16.s, z6.h, z2.h[3]\n" - ".inst 0x647b40d4 // bfdot z20.s, z6.h, z3.h[3]\n" - ".inst 0x647c40d8 // bfdot z24.s, z6.h, z4.h[3]\n" - ".inst 0x647840e9 // bfdot z9.s, z7.h, z0.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n" - ".inst 0x647940ed // bfdot z13.s, z7.h, z1.h[3]\n" - ".inst 0x647a40f1 // bfdot z17.s, z7.h, z2.h[3]\n" - ".inst 0x647b40f5 // bfdot z21.s, z7.h, z3.h[3]\n" - ".inst 0x647c40f9 // bfdot z25.s, z7.h, z4.h[3]\n" - "ld1h { z7.h }, p5/Z, [x10, #-1, MUL VL]\n" - ".inst 0x647840ca // bfdot z10.s, z6.h, z0.h[3]\n" - ".inst 0x647940ce // bfdot z14.s, z6.h, z1.h[3]\n" - ".inst 0x647a40d2 // bfdot z18.s, z6.h, z2.h[3]\n" - ".inst 0x647b40d6 // bfdot z22.s, z6.h, z3.h[3]\n" - ".inst 0x647c40da // bfdot z26.s, z6.h, z4.h[3]\n" - ".inst 0x647840eb // bfdot z11.s, z7.h, z0.h[3]\n" - ".inst 0x647940ef // bfdot z15.s, z7.h, z1.h[3]\n" - ".inst 0x647a40f3 // bfdot z19.s, z7.h, z2.h[3]\n" - ".inst 0x647b40f7 // bfdot z23.s, z7.h, z3.h[3]\n" - ".inst 0x647c40fb // bfdot z27.s, z7.h, z4.h[3]\n" + ".inst 0x646c43aa // bfdot z10.s, z29.h, z4.h[1]\n" + ".inst 0x646b43ae // bfdot z14.s, z29.h, z3.h[1]\n" + ".inst 0x646a43b2 // bfdot z18.s, z29.h, z2.h[1]\n" + ".inst 0x646943b6 // bfdot z22.s, z29.h, z1.h[1]\n" + ".inst 0x646843ba // bfdot z26.s, z29.h, z0.h[1]\n" + ".inst 0x646c438b // bfdot z11.s, z28.h, z4.h[1]\n" + "ld1h { z29.h }, p5/Z, [x10, #-8, MUL VL]\n" + ".inst 0x646b438f // bfdot z15.s, z28.h, z3.h[1]\n" + ".inst 0x646a4393 // bfdot z19.s, z28.h, z2.h[1]\n" + ".inst 0x64694397 // bfdot z23.s, z28.h, z1.h[1]\n" + ".inst 0x6468439b // bfdot z27.s, z28.h, z0.h[1]\n" + "ld1h { z28.h }, p5/Z, [x10, #-7, MUL VL]\n" + ".inst 0x647443a8 // bfdot z8.s, z29.h, z4.h[2]\n" + ".inst 0x647343ac // bfdot z12.s, z29.h, z3.h[2]\n" + ".inst 0x647243b0 // bfdot z16.s, z29.h, z2.h[2]\n" + ".inst 0x647143b4 // bfdot z20.s, z29.h, z1.h[2]\n" + ".inst 0x647043b8 // bfdot z24.s, z29.h, z0.h[2]\n" + ".inst 0x64744389 // bfdot z9.s, z28.h, z4.h[2]\n" + "ld1h { z29.h }, p5/Z, [x10, #-6, MUL VL]\n" + ".inst 0x6473438d // bfdot z13.s, z28.h, z3.h[2]\n" + ".inst 0x64724391 // bfdot z17.s, z28.h, z2.h[2]\n" + ".inst 0x64714395 // bfdot z21.s, z28.h, z1.h[2]\n" + ".inst 0x64704399 // bfdot z25.s, z28.h, z0.h[2]\n" + "ld1h { z28.h }, p5/Z, [x10, #-5, MUL VL]\n" + ".inst 0x647443aa // bfdot z10.s, z29.h, z4.h[2]\n" + ".inst 0x647343ae // bfdot z14.s, z29.h, z3.h[2]\n" + ".inst 0x647243b2 // bfdot z18.s, z29.h, z2.h[2]\n" + ".inst 0x647143b6 // bfdot z22.s, z29.h, z1.h[2]\n" + ".inst 0x647043ba // bfdot z26.s, z29.h, z0.h[2]\n" + ".inst 0x6474438b // bfdot z11.s, z28.h, z4.h[2]\n" + "ld1h { z29.h }, p5/Z, [x10, #-4, MUL VL]\n" + ".inst 0x6473438f // bfdot z15.s, z28.h, z3.h[2]\n" + ".inst 0x64724393 // bfdot z19.s, z28.h, z2.h[2]\n" + ".inst 0x64714397 // bfdot z23.s, z28.h, z1.h[2]\n" + ".inst 0x6470439b // bfdot z27.s, z28.h, z0.h[2]\n" + "ld1h { z28.h }, p5/Z, [x10, #-3, MUL VL]\n" + ".inst 0x647c43a8 // bfdot z8.s, z29.h, z4.h[3]\n" + ".inst 0x647b43ac // bfdot z12.s, z29.h, z3.h[3]\n" + ".inst 0x647a43b0 // bfdot z16.s, z29.h, z2.h[3]\n" + ".inst 0x647943b4 // bfdot z20.s, z29.h, z1.h[3]\n" + ".inst 0x647843b8 // bfdot z24.s, z29.h, z0.h[3]\n" + ".inst 0x647c4389 // bfdot z9.s, z28.h, z4.h[3]\n" + "ld1h { z29.h }, p5/Z, [x10, #-2, MUL VL]\n" + ".inst 0x647b438d // bfdot z13.s, z28.h, z3.h[3]\n" + ".inst 0x647a4391 // bfdot z17.s, z28.h, z2.h[3]\n" + ".inst 0x64794395 // bfdot z21.s, z28.h, z1.h[3]\n" + ".inst 0x64784399 // bfdot z25.s, z28.h, z0.h[3]\n" + "ld1h { z28.h }, p5/Z, [x10, #-1, MUL VL]\n" + ".inst 0x647c43aa // bfdot z10.s, z29.h, z4.h[3]\n" + ".inst 0x647b43ae // bfdot z14.s, z29.h, z3.h[3]\n" + ".inst 0x647a43b2 // bfdot z18.s, z29.h, z2.h[3]\n" + ".inst 0x647943b6 // bfdot z22.s, z29.h, z1.h[3]\n" + ".inst 0x647843ba // bfdot z26.s, z29.h, z0.h[3]\n" + ".inst 0x647c438b // bfdot z11.s, z28.h, z4.h[3]\n" + ".inst 0x647b438f // bfdot z15.s, z28.h, z3.h[3]\n" + ".inst 0x647a4393 // bfdot z19.s, z28.h, z2.h[3]\n" + ".inst 0x64794397 // bfdot z23.s, z28.h, z1.h[3]\n" + ".inst 0x6478439b // bfdot z27.s, z28.h, z0.h[3]\n" "bgt 61b\n" "62:" // Height 5: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" @@ -1393,111 +1393,111 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "ld1rqh { z2.h }, p0/Z, [x24]\n" "ld1rqh { z3.h }, p0/Z, [x23]\n" "ld1rqh { z4.h }, p0/Z, [x22]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - ".inst 0x646040c8 // bfdot z8.s, z6.h, z0.h[0]\n" - ".inst 0x646140cc // bfdot z12.s, z6.h, z1.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x646240d0 // bfdot z16.s, z6.h, z2.h[0]\n" - ".inst 0x646340d4 // bfdot z20.s, z6.h, z3.h[0]\n" - ".inst 0x646440d8 // bfdot z24.s, z6.h, z4.h[0]\n" - ".inst 0x646040e9 // bfdot z9.s, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x646140ed // bfdot z13.s, z7.h, z1.h[0]\n" - ".inst 0x646240f1 // bfdot z17.s, z7.h, z2.h[0]\n" - ".inst 0x646340f5 // bfdot z21.s, z7.h, z3.h[0]\n" - ".inst 0x646440f9 // bfdot z25.s, z7.h, z4.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z29.h }, p5/Z, [x10]\n" + ".inst 0x646043a8 // bfdot z8.s, z29.h, z0.h[0]\n" + ".inst 0x646143ac // bfdot z12.s, z29.h, z1.h[0]\n" + "ld1h { z28.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x646243b0 // bfdot z16.s, z29.h, z2.h[0]\n" + ".inst 0x646343b4 // bfdot z20.s, z29.h, z3.h[0]\n" + ".inst 0x646443b8 // bfdot z24.s, z29.h, z4.h[0]\n" + ".inst 0x64604389 // bfdot z9.s, z28.h, z0.h[0]\n" + "ld1h { z29.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x6461438d // bfdot z13.s, z28.h, z1.h[0]\n" + ".inst 0x64624391 // bfdot z17.s, z28.h, z2.h[0]\n" + ".inst 0x64634395 // bfdot z21.s, z28.h, z3.h[0]\n" + ".inst 0x64644399 // bfdot z25.s, z28.h, z4.h[0]\n" + "ld1h { z28.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - ".inst 0x646040ca // bfdot z10.s, z6.h, z0.h[0]\n" - ".inst 0x646140ce // bfdot z14.s, z6.h, z1.h[0]\n" - ".inst 0x646240d2 // bfdot z18.s, z6.h, z2.h[0]\n" - ".inst 0x646340d6 // bfdot z22.s, z6.h, z3.h[0]\n" - ".inst 0x646440da // bfdot z26.s, z6.h, z4.h[0]\n" - ".inst 0x646040eb // bfdot z11.s, z7.h, z0.h[0]\n" - ".inst 0x646140ef // bfdot z15.s, z7.h, z1.h[0]\n" - ".inst 0x646240f3 // bfdot z19.s, z7.h, z2.h[0]\n" - ".inst 0x646340f7 // bfdot z23.s, z7.h, z3.h[0]\n" - ".inst 0x646440fb // bfdot z27.s, z7.h, z4.h[0]\n" + ".inst 0x646043aa // bfdot z10.s, z29.h, z0.h[0]\n" + ".inst 0x646143ae // bfdot z14.s, z29.h, z1.h[0]\n" + ".inst 0x646243b2 // bfdot z18.s, z29.h, z2.h[0]\n" + ".inst 0x646343b6 // bfdot z22.s, z29.h, z3.h[0]\n" + ".inst 0x646443ba // bfdot z26.s, z29.h, z4.h[0]\n" + ".inst 0x6460438b // bfdot z11.s, z28.h, z0.h[0]\n" + ".inst 0x6461438f // bfdot z15.s, z28.h, z1.h[0]\n" + ".inst 0x64624393 // bfdot z19.s, z28.h, z2.h[0]\n" + ".inst 0x64634397 // bfdot z23.s, z28.h, z3.h[0]\n" + ".inst 0x6464439b // bfdot z27.s, z28.h, z4.h[0]\n" "ble 63f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x646840c8 // bfdot z8.s, z6.h, z0.h[1]\n" - ".inst 0x646940cc // bfdot z12.s, z6.h, z1.h[1]\n" - ".inst 0x646a40d0 // bfdot z16.s, z6.h, z2.h[1]\n" - ".inst 0x646b40d4 // bfdot z20.s, z6.h, z3.h[1]\n" + "ld1h { z29.h }, p5/Z, [x10]\n" + "ld1h { z28.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x646843a8 // bfdot z8.s, z29.h, z0.h[1]\n" + ".inst 0x646943ac // bfdot z12.s, z29.h, z1.h[1]\n" + ".inst 0x646a43b0 // bfdot z16.s, z29.h, z2.h[1]\n" + ".inst 0x646b43b4 // bfdot z20.s, z29.h, z3.h[1]\n" "subs x27, x27, #0x2\n" - ".inst 0x646c40d8 // bfdot z24.s, z6.h, z4.h[1]\n" - ".inst 0x646840e9 // bfdot z9.s, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x646940ed // bfdot z13.s, z7.h, z1.h[1]\n" - ".inst 0x646a40f1 // bfdot z17.s, z7.h, z2.h[1]\n" - ".inst 0x646b40f5 // bfdot z21.s, z7.h, z3.h[1]\n" - ".inst 0x646c40f9 // bfdot z25.s, z7.h, z4.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x646c43b8 // bfdot z24.s, z29.h, z4.h[1]\n" + ".inst 0x64684389 // bfdot z9.s, z28.h, z0.h[1]\n" + "ld1h { z29.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x6469438d // bfdot z13.s, z28.h, z1.h[1]\n" + ".inst 0x646a4391 // bfdot z17.s, z28.h, z2.h[1]\n" + ".inst 0x646b4395 // bfdot z21.s, z28.h, z3.h[1]\n" + ".inst 0x646c4399 // bfdot z25.s, z28.h, z4.h[1]\n" + "ld1h { z28.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - ".inst 0x646840ca // bfdot z10.s, z6.h, z0.h[1]\n" - ".inst 0x646940ce // bfdot z14.s, z6.h, z1.h[1]\n" - ".inst 0x646a40d2 // bfdot z18.s, z6.h, z2.h[1]\n" - ".inst 0x646b40d6 // bfdot z22.s, z6.h, z3.h[1]\n" - ".inst 0x646c40da // bfdot z26.s, z6.h, z4.h[1]\n" - ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n" - ".inst 0x646940ef // bfdot z15.s, z7.h, z1.h[1]\n" - ".inst 0x646a40f3 // bfdot z19.s, z7.h, z2.h[1]\n" - ".inst 0x646b40f7 // bfdot z23.s, z7.h, z3.h[1]\n" - ".inst 0x646c40fb // bfdot z27.s, z7.h, z4.h[1]\n" + ".inst 0x646843aa // bfdot z10.s, z29.h, z0.h[1]\n" + ".inst 0x646943ae // bfdot z14.s, z29.h, z1.h[1]\n" + ".inst 0x646a43b2 // bfdot z18.s, z29.h, z2.h[1]\n" + ".inst 0x646b43b6 // bfdot z22.s, z29.h, z3.h[1]\n" + ".inst 0x646c43ba // bfdot z26.s, z29.h, z4.h[1]\n" + ".inst 0x6468438b // bfdot z11.s, z28.h, z0.h[1]\n" + ".inst 0x6469438f // bfdot z15.s, z28.h, z1.h[1]\n" + ".inst 0x646a4393 // bfdot z19.s, z28.h, z2.h[1]\n" + ".inst 0x646b4397 // bfdot z23.s, z28.h, z3.h[1]\n" + ".inst 0x646c439b // bfdot z27.s, z28.h, z4.h[1]\n" "ble 63f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x647040c8 // bfdot z8.s, z6.h, z0.h[2]\n" - ".inst 0x647140cc // bfdot z12.s, z6.h, z1.h[2]\n" - ".inst 0x647240d0 // bfdot z16.s, z6.h, z2.h[2]\n" - ".inst 0x647340d4 // bfdot z20.s, z6.h, z3.h[2]\n" + "ld1h { z29.h }, p5/Z, [x10]\n" + "ld1h { z28.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x647043a8 // bfdot z8.s, z29.h, z0.h[2]\n" + ".inst 0x647143ac // bfdot z12.s, z29.h, z1.h[2]\n" + ".inst 0x647243b0 // bfdot z16.s, z29.h, z2.h[2]\n" + ".inst 0x647343b4 // bfdot z20.s, z29.h, z3.h[2]\n" "subs x27, x27, #0x2\n" - ".inst 0x647440d8 // bfdot z24.s, z6.h, z4.h[2]\n" - ".inst 0x647040e9 // bfdot z9.s, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x647140ed // bfdot z13.s, z7.h, z1.h[2]\n" - ".inst 0x647240f1 // bfdot z17.s, z7.h, z2.h[2]\n" - ".inst 0x647340f5 // bfdot z21.s, z7.h, z3.h[2]\n" - ".inst 0x647440f9 // bfdot z25.s, z7.h, z4.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x647443b8 // bfdot z24.s, z29.h, z4.h[2]\n" + ".inst 0x64704389 // bfdot z9.s, z28.h, z0.h[2]\n" + "ld1h { z29.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x6471438d // bfdot z13.s, z28.h, z1.h[2]\n" + ".inst 0x64724391 // bfdot z17.s, z28.h, z2.h[2]\n" + ".inst 0x64734395 // bfdot z21.s, z28.h, z3.h[2]\n" + ".inst 0x64744399 // bfdot z25.s, z28.h, z4.h[2]\n" + "ld1h { z28.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - ".inst 0x647040ca // bfdot z10.s, z6.h, z0.h[2]\n" - ".inst 0x647140ce // bfdot z14.s, z6.h, z1.h[2]\n" - ".inst 0x647240d2 // bfdot z18.s, z6.h, z2.h[2]\n" - ".inst 0x647340d6 // bfdot z22.s, z6.h, z3.h[2]\n" - ".inst 0x647440da // bfdot z26.s, z6.h, z4.h[2]\n" - ".inst 0x647040eb // bfdot z11.s, z7.h, z0.h[2]\n" - ".inst 0x647140ef // bfdot z15.s, z7.h, z1.h[2]\n" - ".inst 0x647240f3 // bfdot z19.s, z7.h, z2.h[2]\n" - ".inst 0x647340f7 // bfdot z23.s, z7.h, z3.h[2]\n" - ".inst 0x647440fb // bfdot z27.s, z7.h, z4.h[2]\n" + ".inst 0x647043aa // bfdot z10.s, z29.h, z0.h[2]\n" + ".inst 0x647143ae // bfdot z14.s, z29.h, z1.h[2]\n" + ".inst 0x647243b2 // bfdot z18.s, z29.h, z2.h[2]\n" + ".inst 0x647343b6 // bfdot z22.s, z29.h, z3.h[2]\n" + ".inst 0x647443ba // bfdot z26.s, z29.h, z4.h[2]\n" + ".inst 0x6470438b // bfdot z11.s, z28.h, z0.h[2]\n" + ".inst 0x6471438f // bfdot z15.s, z28.h, z1.h[2]\n" + ".inst 0x64724393 // bfdot z19.s, z28.h, z2.h[2]\n" + ".inst 0x64734397 // bfdot z23.s, z28.h, z3.h[2]\n" + ".inst 0x6474439b // bfdot z27.s, z28.h, z4.h[2]\n" "ble 63f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x647840c8 // bfdot z8.s, z6.h, z0.h[3]\n" - ".inst 0x647940cc // bfdot z12.s, z6.h, z1.h[3]\n" - ".inst 0x647a40d0 // bfdot z16.s, z6.h, z2.h[3]\n" - ".inst 0x647b40d4 // bfdot z20.s, z6.h, z3.h[3]\n" - ".inst 0x647c40d8 // bfdot z24.s, z6.h, z4.h[3]\n" - ".inst 0x647840e9 // bfdot z9.s, z7.h, z0.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x647940ed // bfdot z13.s, z7.h, z1.h[3]\n" - ".inst 0x647a40f1 // bfdot z17.s, z7.h, z2.h[3]\n" - ".inst 0x647b40f5 // bfdot z21.s, z7.h, z3.h[3]\n" - ".inst 0x647c40f9 // bfdot z25.s, z7.h, z4.h[3]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z29.h }, p5/Z, [x10]\n" + "ld1h { z28.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x647843a8 // bfdot z8.s, z29.h, z0.h[3]\n" + ".inst 0x647943ac // bfdot z12.s, z29.h, z1.h[3]\n" + ".inst 0x647a43b0 // bfdot z16.s, z29.h, z2.h[3]\n" + ".inst 0x647b43b4 // bfdot z20.s, z29.h, z3.h[3]\n" + ".inst 0x647c43b8 // bfdot z24.s, z29.h, z4.h[3]\n" + ".inst 0x64784389 // bfdot z9.s, z28.h, z0.h[3]\n" + "ld1h { z29.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x6479438d // bfdot z13.s, z28.h, z1.h[3]\n" + ".inst 0x647a4391 // bfdot z17.s, z28.h, z2.h[3]\n" + ".inst 0x647b4395 // bfdot z21.s, z28.h, z3.h[3]\n" + ".inst 0x647c4399 // bfdot z25.s, z28.h, z4.h[3]\n" + "ld1h { z28.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - ".inst 0x647840ca // bfdot z10.s, z6.h, z0.h[3]\n" - ".inst 0x647940ce // bfdot z14.s, z6.h, z1.h[3]\n" - ".inst 0x647a40d2 // bfdot z18.s, z6.h, z2.h[3]\n" - ".inst 0x647b40d6 // bfdot z22.s, z6.h, z3.h[3]\n" - ".inst 0x647c40da // bfdot z26.s, z6.h, z4.h[3]\n" - ".inst 0x647840eb // bfdot z11.s, z7.h, z0.h[3]\n" - ".inst 0x647940ef // bfdot z15.s, z7.h, z1.h[3]\n" - ".inst 0x647a40f3 // bfdot z19.s, z7.h, z2.h[3]\n" - ".inst 0x647b40f7 // bfdot z23.s, z7.h, z3.h[3]\n" - ".inst 0x647c40fb // bfdot z27.s, z7.h, z4.h[3]\n" + ".inst 0x647843aa // bfdot z10.s, z29.h, z0.h[3]\n" + ".inst 0x647943ae // bfdot z14.s, z29.h, z1.h[3]\n" + ".inst 0x647a43b2 // bfdot z18.s, z29.h, z2.h[3]\n" + ".inst 0x647b43b6 // bfdot z22.s, z29.h, z3.h[3]\n" + ".inst 0x647c43ba // bfdot z26.s, z29.h, z4.h[3]\n" + ".inst 0x6478438b // bfdot z11.s, z28.h, z0.h[3]\n" + ".inst 0x6479438f // bfdot z15.s, z28.h, z1.h[3]\n" + ".inst 0x647a4393 // bfdot z19.s, z28.h, z2.h[3]\n" + ".inst 0x647b4397 // bfdot z23.s, z28.h, z3.h[3]\n" + ".inst 0x647c439b // bfdot z27.s, z28.h, z4.h[3]\n" "63:" // Height 5: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1510,49 +1510,49 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "add x22, x23, x20, LSL #2\n" "tbz %x[flags], #1, 64f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z29.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmin z12.s, p5/M, z12.s, z1.s\n" - "fmin z13.s, p5/M, z13.s, z1.s\n" - "fmin z14.s, p5/M, z14.s, z1.s\n" - "fmin z15.s, p5/M, z15.s, z1.s\n" - "fmin z16.s, p5/M, z16.s, z1.s\n" - "fmin z17.s, p5/M, z17.s, z1.s\n" - "fmin z18.s, p5/M, z18.s, z1.s\n" - "fmin z19.s, p5/M, z19.s, z1.s\n" - "fmin z20.s, p5/M, z20.s, z1.s\n" - "fmin z21.s, p5/M, z21.s, z1.s\n" - "fmin z22.s, p5/M, z22.s, z1.s\n" - "fmin z23.s, p5/M, z23.s, z1.s\n" - "fmin z24.s, p5/M, z24.s, z1.s\n" - "fmin z25.s, p5/M, z25.s, z1.s\n" - "fmin z26.s, p5/M, z26.s, z1.s\n" - "fmin z27.s, p5/M, z27.s, z1.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" - "fmax z12.s, p5/M, z12.s, z0.s\n" - "fmax z13.s, p5/M, z13.s, z0.s\n" - "fmax z14.s, p5/M, z14.s, z0.s\n" - "fmax z15.s, p5/M, z15.s, z0.s\n" - "fmax z16.s, p5/M, z16.s, z0.s\n" - "fmax z17.s, p5/M, z17.s, z0.s\n" - "fmax z18.s, p5/M, z18.s, z0.s\n" - "fmax z19.s, p5/M, z19.s, z0.s\n" - "fmax z20.s, p5/M, z20.s, z0.s\n" - "fmax z21.s, p5/M, z21.s, z0.s\n" - "fmax z22.s, p5/M, z22.s, z0.s\n" - "fmax z23.s, p5/M, z23.s, z0.s\n" - "fmax z24.s, p5/M, z24.s, z0.s\n" - "fmax z25.s, p5/M, z25.s, z0.s\n" - "fmax z26.s, p5/M, z26.s, z0.s\n" - "fmax z27.s, p5/M, z27.s, z0.s\n" + "ld1rw { z28.s }, p5/Z, [x20]\n" + "fmin z8.s, p5/M, z8.s, z29.s\n" + "fmin z9.s, p5/M, z9.s, z29.s\n" + "fmin z10.s, p5/M, z10.s, z29.s\n" + "fmin z11.s, p5/M, z11.s, z29.s\n" + "fmin z12.s, p5/M, z12.s, z29.s\n" + "fmin z13.s, p5/M, z13.s, z29.s\n" + "fmin z14.s, p5/M, z14.s, z29.s\n" + "fmin z15.s, p5/M, z15.s, z29.s\n" + "fmin z16.s, p5/M, z16.s, z29.s\n" + "fmin z17.s, p5/M, z17.s, z29.s\n" + "fmin z18.s, p5/M, z18.s, z29.s\n" + "fmin z19.s, p5/M, z19.s, z29.s\n" + "fmin z20.s, p5/M, z20.s, z29.s\n" + "fmin z21.s, p5/M, z21.s, z29.s\n" + "fmin z22.s, p5/M, z22.s, z29.s\n" + "fmin z23.s, p5/M, z23.s, z29.s\n" + "fmin z24.s, p5/M, z24.s, z29.s\n" + "fmin z25.s, p5/M, z25.s, z29.s\n" + "fmin z26.s, p5/M, z26.s, z29.s\n" + "fmin z27.s, p5/M, z27.s, z29.s\n" + "fmax z8.s, p5/M, z8.s, z28.s\n" + "fmax z9.s, p5/M, z9.s, z28.s\n" + "fmax z10.s, p5/M, z10.s, z28.s\n" + "fmax z11.s, p5/M, z11.s, z28.s\n" + "fmax z12.s, p5/M, z12.s, z28.s\n" + "fmax z13.s, p5/M, z13.s, z28.s\n" + "fmax z14.s, p5/M, z14.s, z28.s\n" + "fmax z15.s, p5/M, z15.s, z28.s\n" + "fmax z16.s, p5/M, z16.s, z28.s\n" + "fmax z17.s, p5/M, z17.s, z28.s\n" + "fmax z18.s, p5/M, z18.s, z28.s\n" + "fmax z19.s, p5/M, z19.s, z28.s\n" + "fmax z20.s, p5/M, z20.s, z28.s\n" + "fmax z21.s, p5/M, z21.s, z28.s\n" + "fmax z22.s, p5/M, z22.s, z28.s\n" + "fmax z23.s, p5/M, z23.s, z28.s\n" + "fmax z24.s, p5/M, z24.s, z28.s\n" + "fmax z25.s, p5/M, z25.s, z28.s\n" + "fmax z26.s, p5/M, z26.s, z28.s\n" + "fmax z27.s, p5/M, z27.s, z28.s\n" "64:" // Height 5: No activation "st1w { z8.s }, p4, [x9]\n" "st1w { z9.s }, p3, [x9, #1, MUL VL]\n" @@ -1627,35 +1627,35 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "68:" // Height 6: no bias "tbz %x[flags], #0, 69f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z8.s }, p4/Z, [x9]\n" + "add x24, x9, x20, LSL #2\n" "add x23, x24, x20, LSL #2\n" + "ld1w { z8.s }, p4/Z, [x9]\n" "add x22, x23, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n" - "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "ld1w { z16.s }, p4/Z, [x24]\n" - "ld1w { z17.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x24, #2, MUL VL]\n" - "ld1w { z19.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x23]\n" - "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n" - "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n" - "ld1w { z24.s }, p4/Z, [x22]\n" - "ld1w { z25.s }, p3/Z, [x22, #1, MUL VL]\n" - "ld1w { z26.s }, p2/Z, [x22, #2, MUL VL]\n" - "ld1w { z27.s }, p1/Z, [x22, #3, MUL VL]\n" - "ld1w { z28.s }, p4/Z, [x21]\n" - "ld1w { z29.s }, p3/Z, [x21, #1, MUL VL]\n" - "ld1w { z30.s }, p2/Z, [x21, #2, MUL VL]\n" - "ld1w { z31.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x24]\n" + "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n" + "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x23]\n" + "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n" + "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x22]\n" + "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n" + "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z24.s }, p4/Z, [x21]\n" + "ld1w { z25.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z26.s }, p2/Z, [x21, #2, MUL VL]\n" + "ld1w { z27.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z28.s }, p4/Z, [x20]\n" + "ld1w { z29.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z30.s }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z31.s }, p1/Z, [x20, #3, MUL VL]\n" "b 70f\n" "69:" // Height 6: no accumulate "mov z8.b, #0x0\n" @@ -1687,16 +1687,16 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "71:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 72f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 73f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -1708,143 +1708,143 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "b 73f\n" "72:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" - "add x22, x23, x20, LSL #1\n" - "add x21, x22, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" + "add x22, x23, x21, LSL #1\n" + "add x21, x22, x21, LSL #1\n" "73:" // Height 6: input setup done "cmp x27, #0x8\n" "ble 75f\n" "74:" // Height 6: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z0.h }, p0/Z, [x26]\n" - "ld1rqh { z1.h }, p0/Z, [x25]\n" + "ld1rqh { z7.h }, p0/Z, [x26]\n" + "ld1rqh { z6.h }, p0/Z, [x25]\n" "sub x27, x27, #0x8\n" - "ld1rqh { z2.h }, p0/Z, [x24]\n" - "ld1rqh { z3.h }, p0/Z, [x23]\n" + "ld1rqh { z5.h }, p0/Z, [x24]\n" + "ld1rqh { z4.h }, p0/Z, [x23]\n" "cmp x27, #0x8\n" "add x26, x26, #0x10\n" - "ld1rqh { z4.h }, p0/Z, [x22]\n" - "ld1rqh { z5.h }, p0/Z, [x21]\n" + "ld1rqh { z3.h }, p0/Z, [x22]\n" + "ld1rqh { z2.h }, p0/Z, [x21]\n" "add x25, x25, #0x10\n" "add x24, x24, #0x10\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x646040c8 // bfdot z8.s, z6.h, z0.h[0]\n" - ".inst 0x646140cc // bfdot z12.s, z6.h, z1.h[0]\n" - ".inst 0x646240d0 // bfdot z16.s, z6.h, z2.h[0]\n" - ".inst 0x646340d4 // bfdot z20.s, z6.h, z3.h[0]\n" + "ld1h { z1.h }, p5/Z, [x10]\n" + "ld1h { z0.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x64674028 // bfdot z8.s, z1.h, z7.h[0]\n" + ".inst 0x6466402c // bfdot z12.s, z1.h, z6.h[0]\n" + ".inst 0x64654030 // bfdot z16.s, z1.h, z5.h[0]\n" + ".inst 0x64644034 // bfdot z20.s, z1.h, z4.h[0]\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - ".inst 0x646440d8 // bfdot z24.s, z6.h, z4.h[0]\n" - ".inst 0x646540dc // bfdot z28.s, z6.h, z5.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x64634038 // bfdot z24.s, z1.h, z3.h[0]\n" + ".inst 0x6462403c // bfdot z28.s, z1.h, z2.h[0]\n" + "ld1h { z1.h }, p5/Z, [x10, #2, MUL VL]\n" "add x21, x21, #0x10\n" - ".inst 0x646040e9 // bfdot z9.s, z7.h, z0.h[0]\n" - ".inst 0x646140ed // bfdot z13.s, z7.h, z1.h[0]\n" - ".inst 0x646240f1 // bfdot z17.s, z7.h, z2.h[0]\n" - ".inst 0x646340f5 // bfdot z21.s, z7.h, z3.h[0]\n" - ".inst 0x646440f9 // bfdot z25.s, z7.h, z4.h[0]\n" - ".inst 0x646540fd // bfdot z29.s, z7.h, z5.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x646040ca // bfdot z10.s, z6.h, z0.h[0]\n" - ".inst 0x646140ce // bfdot z14.s, z6.h, z1.h[0]\n" - ".inst 0x646240d2 // bfdot z18.s, z6.h, z2.h[0]\n" - ".inst 0x646340d6 // bfdot z22.s, z6.h, z3.h[0]\n" - ".inst 0x646440da // bfdot z26.s, z6.h, z4.h[0]\n" - ".inst 0x646540de // bfdot z30.s, z6.h, z5.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x646040eb // bfdot z11.s, z7.h, z0.h[0]\n" - ".inst 0x646140ef // bfdot z15.s, z7.h, z1.h[0]\n" - ".inst 0x646240f3 // bfdot z19.s, z7.h, z2.h[0]\n" - ".inst 0x646340f7 // bfdot z23.s, z7.h, z3.h[0]\n" - ".inst 0x646440fb // bfdot z27.s, z7.h, z4.h[0]\n" - ".inst 0x646540ff // bfdot z31.s, z7.h, z5.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x646840c8 // bfdot z8.s, z6.h, z0.h[1]\n" - ".inst 0x646940cc // bfdot z12.s, z6.h, z1.h[1]\n" - ".inst 0x646a40d0 // bfdot z16.s, z6.h, z2.h[1]\n" - ".inst 0x646b40d4 // bfdot z20.s, z6.h, z3.h[1]\n" - ".inst 0x646c40d8 // bfdot z24.s, z6.h, z4.h[1]\n" - ".inst 0x646d40dc // bfdot z28.s, z6.h, z5.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x646840e9 // bfdot z9.s, z7.h, z0.h[1]\n" - ".inst 0x646940ed // bfdot z13.s, z7.h, z1.h[1]\n" - ".inst 0x646a40f1 // bfdot z17.s, z7.h, z2.h[1]\n" - ".inst 0x646b40f5 // bfdot z21.s, z7.h, z3.h[1]\n" - ".inst 0x646c40f9 // bfdot z25.s, z7.h, z4.h[1]\n" - ".inst 0x646d40fd // bfdot z29.s, z7.h, z5.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x64674009 // bfdot z9.s, z0.h, z7.h[0]\n" + ".inst 0x6466400d // bfdot z13.s, z0.h, z6.h[0]\n" + ".inst 0x64654011 // bfdot z17.s, z0.h, z5.h[0]\n" + ".inst 0x64644015 // bfdot z21.s, z0.h, z4.h[0]\n" + ".inst 0x64634019 // bfdot z25.s, z0.h, z3.h[0]\n" + ".inst 0x6462401d // bfdot z29.s, z0.h, z2.h[0]\n" + "ld1h { z0.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6467402a // bfdot z10.s, z1.h, z7.h[0]\n" + ".inst 0x6466402e // bfdot z14.s, z1.h, z6.h[0]\n" + ".inst 0x64654032 // bfdot z18.s, z1.h, z5.h[0]\n" + ".inst 0x64644036 // bfdot z22.s, z1.h, z4.h[0]\n" + ".inst 0x6463403a // bfdot z26.s, z1.h, z3.h[0]\n" + ".inst 0x6462403e // bfdot z30.s, z1.h, z2.h[0]\n" + "ld1h { z1.h }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x6467400b // bfdot z11.s, z0.h, z7.h[0]\n" + ".inst 0x6466400f // bfdot z15.s, z0.h, z6.h[0]\n" + ".inst 0x64654013 // bfdot z19.s, z0.h, z5.h[0]\n" + ".inst 0x64644017 // bfdot z23.s, z0.h, z4.h[0]\n" + ".inst 0x6463401b // bfdot z27.s, z0.h, z3.h[0]\n" + ".inst 0x6462401f // bfdot z31.s, z0.h, z2.h[0]\n" + "ld1h { z0.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x646f4028 // bfdot z8.s, z1.h, z7.h[1]\n" + ".inst 0x646e402c // bfdot z12.s, z1.h, z6.h[1]\n" + ".inst 0x646d4030 // bfdot z16.s, z1.h, z5.h[1]\n" + ".inst 0x646c4034 // bfdot z20.s, z1.h, z4.h[1]\n" + ".inst 0x646b4038 // bfdot z24.s, z1.h, z3.h[1]\n" + ".inst 0x646a403c // bfdot z28.s, z1.h, z2.h[1]\n" + "ld1h { z1.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x646f4009 // bfdot z9.s, z0.h, z7.h[1]\n" + ".inst 0x646e400d // bfdot z13.s, z0.h, z6.h[1]\n" + ".inst 0x646d4011 // bfdot z17.s, z0.h, z5.h[1]\n" + ".inst 0x646c4015 // bfdot z21.s, z0.h, z4.h[1]\n" + ".inst 0x646b4019 // bfdot z25.s, z0.h, z3.h[1]\n" + ".inst 0x646a401d // bfdot z29.s, z0.h, z2.h[1]\n" + "ld1h { z0.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - ".inst 0x646840ca // bfdot z10.s, z6.h, z0.h[1]\n" - ".inst 0x646940ce // bfdot z14.s, z6.h, z1.h[1]\n" - ".inst 0x646a40d2 // bfdot z18.s, z6.h, z2.h[1]\n" - ".inst 0x646b40d6 // bfdot z22.s, z6.h, z3.h[1]\n" - ".inst 0x646c40da // bfdot z26.s, z6.h, z4.h[1]\n" - ".inst 0x646d40de // bfdot z30.s, z6.h, z5.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n" - ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n" - ".inst 0x646940ef // bfdot z15.s, z7.h, z1.h[1]\n" - ".inst 0x646a40f3 // bfdot z19.s, z7.h, z2.h[1]\n" - ".inst 0x646b40f7 // bfdot z23.s, z7.h, z3.h[1]\n" - ".inst 0x646c40fb // bfdot z27.s, z7.h, z4.h[1]\n" - ".inst 0x646d40ff // bfdot z31.s, z7.h, z5.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #-7, MUL VL]\n" - ".inst 0x647040c8 // bfdot z8.s, z6.h, z0.h[2]\n" - ".inst 0x647140cc // bfdot z12.s, z6.h, z1.h[2]\n" - ".inst 0x647240d0 // bfdot z16.s, z6.h, z2.h[2]\n" - ".inst 0x647340d4 // bfdot z20.s, z6.h, z3.h[2]\n" - ".inst 0x647440d8 // bfdot z24.s, z6.h, z4.h[2]\n" - ".inst 0x647540dc // bfdot z28.s, z6.h, z5.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n" - ".inst 0x647040e9 // bfdot z9.s, z7.h, z0.h[2]\n" - ".inst 0x647140ed // bfdot z13.s, z7.h, z1.h[2]\n" - ".inst 0x647240f1 // bfdot z17.s, z7.h, z2.h[2]\n" - ".inst 0x647340f5 // bfdot z21.s, z7.h, z3.h[2]\n" - ".inst 0x647440f9 // bfdot z25.s, z7.h, z4.h[2]\n" - ".inst 0x647540fd // bfdot z29.s, z7.h, z5.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #-5, MUL VL]\n" - ".inst 0x647040ca // bfdot z10.s, z6.h, z0.h[2]\n" - ".inst 0x647140ce // bfdot z14.s, z6.h, z1.h[2]\n" - ".inst 0x647240d2 // bfdot z18.s, z6.h, z2.h[2]\n" - ".inst 0x647340d6 // bfdot z22.s, z6.h, z3.h[2]\n" - ".inst 0x647440da // bfdot z26.s, z6.h, z4.h[2]\n" - ".inst 0x647540de // bfdot z30.s, z6.h, z5.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n" - ".inst 0x647040eb // bfdot z11.s, z7.h, z0.h[2]\n" - ".inst 0x647140ef // bfdot z15.s, z7.h, z1.h[2]\n" - ".inst 0x647240f3 // bfdot z19.s, z7.h, z2.h[2]\n" - ".inst 0x647340f7 // bfdot z23.s, z7.h, z3.h[2]\n" - ".inst 0x647440fb // bfdot z27.s, z7.h, z4.h[2]\n" - ".inst 0x647540ff // bfdot z31.s, z7.h, z5.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #-3, MUL VL]\n" - ".inst 0x647840c8 // bfdot z8.s, z6.h, z0.h[3]\n" - ".inst 0x647940cc // bfdot z12.s, z6.h, z1.h[3]\n" - ".inst 0x647a40d0 // bfdot z16.s, z6.h, z2.h[3]\n" - ".inst 0x647b40d4 // bfdot z20.s, z6.h, z3.h[3]\n" - ".inst 0x647c40d8 // bfdot z24.s, z6.h, z4.h[3]\n" - ".inst 0x647d40dc // bfdot z28.s, z6.h, z5.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n" - ".inst 0x647840e9 // bfdot z9.s, z7.h, z0.h[3]\n" - ".inst 0x647940ed // bfdot z13.s, z7.h, z1.h[3]\n" - ".inst 0x647a40f1 // bfdot z17.s, z7.h, z2.h[3]\n" - ".inst 0x647b40f5 // bfdot z21.s, z7.h, z3.h[3]\n" - ".inst 0x647c40f9 // bfdot z25.s, z7.h, z4.h[3]\n" - ".inst 0x647d40fd // bfdot z29.s, z7.h, z5.h[3]\n" - "ld1h { z7.h }, p5/Z, [x10, #-1, MUL VL]\n" - ".inst 0x647840ca // bfdot z10.s, z6.h, z0.h[3]\n" - ".inst 0x647940ce // bfdot z14.s, z6.h, z1.h[3]\n" - ".inst 0x647a40d2 // bfdot z18.s, z6.h, z2.h[3]\n" - ".inst 0x647b40d6 // bfdot z22.s, z6.h, z3.h[3]\n" - ".inst 0x647c40da // bfdot z26.s, z6.h, z4.h[3]\n" - ".inst 0x647d40de // bfdot z30.s, z6.h, z5.h[3]\n" - ".inst 0x647840eb // bfdot z11.s, z7.h, z0.h[3]\n" - ".inst 0x647940ef // bfdot z15.s, z7.h, z1.h[3]\n" - ".inst 0x647a40f3 // bfdot z19.s, z7.h, z2.h[3]\n" - ".inst 0x647b40f7 // bfdot z23.s, z7.h, z3.h[3]\n" - ".inst 0x647c40fb // bfdot z27.s, z7.h, z4.h[3]\n" - ".inst 0x647d40ff // bfdot z31.s, z7.h, z5.h[3]\n" + ".inst 0x646f402a // bfdot z10.s, z1.h, z7.h[1]\n" + ".inst 0x646e402e // bfdot z14.s, z1.h, z6.h[1]\n" + ".inst 0x646d4032 // bfdot z18.s, z1.h, z5.h[1]\n" + ".inst 0x646c4036 // bfdot z22.s, z1.h, z4.h[1]\n" + ".inst 0x646b403a // bfdot z26.s, z1.h, z3.h[1]\n" + ".inst 0x646a403e // bfdot z30.s, z1.h, z2.h[1]\n" + "ld1h { z1.h }, p5/Z, [x10, #-8, MUL VL]\n" + ".inst 0x646f400b // bfdot z11.s, z0.h, z7.h[1]\n" + ".inst 0x646e400f // bfdot z15.s, z0.h, z6.h[1]\n" + ".inst 0x646d4013 // bfdot z19.s, z0.h, z5.h[1]\n" + ".inst 0x646c4017 // bfdot z23.s, z0.h, z4.h[1]\n" + ".inst 0x646b401b // bfdot z27.s, z0.h, z3.h[1]\n" + ".inst 0x646a401f // bfdot z31.s, z0.h, z2.h[1]\n" + "ld1h { z0.h }, p5/Z, [x10, #-7, MUL VL]\n" + ".inst 0x64774028 // bfdot z8.s, z1.h, z7.h[2]\n" + ".inst 0x6476402c // bfdot z12.s, z1.h, z6.h[2]\n" + ".inst 0x64754030 // bfdot z16.s, z1.h, z5.h[2]\n" + ".inst 0x64744034 // bfdot z20.s, z1.h, z4.h[2]\n" + ".inst 0x64734038 // bfdot z24.s, z1.h, z3.h[2]\n" + ".inst 0x6472403c // bfdot z28.s, z1.h, z2.h[2]\n" + "ld1h { z1.h }, p5/Z, [x10, #-6, MUL VL]\n" + ".inst 0x64774009 // bfdot z9.s, z0.h, z7.h[2]\n" + ".inst 0x6476400d // bfdot z13.s, z0.h, z6.h[2]\n" + ".inst 0x64754011 // bfdot z17.s, z0.h, z5.h[2]\n" + ".inst 0x64744015 // bfdot z21.s, z0.h, z4.h[2]\n" + ".inst 0x64734019 // bfdot z25.s, z0.h, z3.h[2]\n" + ".inst 0x6472401d // bfdot z29.s, z0.h, z2.h[2]\n" + "ld1h { z0.h }, p5/Z, [x10, #-5, MUL VL]\n" + ".inst 0x6477402a // bfdot z10.s, z1.h, z7.h[2]\n" + ".inst 0x6476402e // bfdot z14.s, z1.h, z6.h[2]\n" + ".inst 0x64754032 // bfdot z18.s, z1.h, z5.h[2]\n" + ".inst 0x64744036 // bfdot z22.s, z1.h, z4.h[2]\n" + ".inst 0x6473403a // bfdot z26.s, z1.h, z3.h[2]\n" + ".inst 0x6472403e // bfdot z30.s, z1.h, z2.h[2]\n" + "ld1h { z1.h }, p5/Z, [x10, #-4, MUL VL]\n" + ".inst 0x6477400b // bfdot z11.s, z0.h, z7.h[2]\n" + ".inst 0x6476400f // bfdot z15.s, z0.h, z6.h[2]\n" + ".inst 0x64754013 // bfdot z19.s, z0.h, z5.h[2]\n" + ".inst 0x64744017 // bfdot z23.s, z0.h, z4.h[2]\n" + ".inst 0x6473401b // bfdot z27.s, z0.h, z3.h[2]\n" + ".inst 0x6472401f // bfdot z31.s, z0.h, z2.h[2]\n" + "ld1h { z0.h }, p5/Z, [x10, #-3, MUL VL]\n" + ".inst 0x647f4028 // bfdot z8.s, z1.h, z7.h[3]\n" + ".inst 0x647e402c // bfdot z12.s, z1.h, z6.h[3]\n" + ".inst 0x647d4030 // bfdot z16.s, z1.h, z5.h[3]\n" + ".inst 0x647c4034 // bfdot z20.s, z1.h, z4.h[3]\n" + ".inst 0x647b4038 // bfdot z24.s, z1.h, z3.h[3]\n" + ".inst 0x647a403c // bfdot z28.s, z1.h, z2.h[3]\n" + "ld1h { z1.h }, p5/Z, [x10, #-2, MUL VL]\n" + ".inst 0x647f4009 // bfdot z9.s, z0.h, z7.h[3]\n" + ".inst 0x647e400d // bfdot z13.s, z0.h, z6.h[3]\n" + ".inst 0x647d4011 // bfdot z17.s, z0.h, z5.h[3]\n" + ".inst 0x647c4015 // bfdot z21.s, z0.h, z4.h[3]\n" + ".inst 0x647b4019 // bfdot z25.s, z0.h, z3.h[3]\n" + ".inst 0x647a401d // bfdot z29.s, z0.h, z2.h[3]\n" + "ld1h { z0.h }, p5/Z, [x10, #-1, MUL VL]\n" + ".inst 0x647f402a // bfdot z10.s, z1.h, z7.h[3]\n" + ".inst 0x647e402e // bfdot z14.s, z1.h, z6.h[3]\n" + ".inst 0x647d4032 // bfdot z18.s, z1.h, z5.h[3]\n" + ".inst 0x647c4036 // bfdot z22.s, z1.h, z4.h[3]\n" + ".inst 0x647b403a // bfdot z26.s, z1.h, z3.h[3]\n" + ".inst 0x647a403e // bfdot z30.s, z1.h, z2.h[3]\n" + ".inst 0x647f400b // bfdot z11.s, z0.h, z7.h[3]\n" + ".inst 0x647e400f // bfdot z15.s, z0.h, z6.h[3]\n" + ".inst 0x647d4013 // bfdot z19.s, z0.h, z5.h[3]\n" + ".inst 0x647c4017 // bfdot z23.s, z0.h, z4.h[3]\n" + ".inst 0x647b401b // bfdot z27.s, z0.h, z3.h[3]\n" + ".inst 0x647a401f // bfdot z31.s, z0.h, z2.h[3]\n" "bgt 74b\n" "75:" // Height 6: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" @@ -1855,127 +1855,127 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "ld1rqh { z3.h }, p0/Z, [x23]\n" "ld1rqh { z4.h }, p0/Z, [x22]\n" "ld1rqh { z5.h }, p0/Z, [x21]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x646040c8 // bfdot z8.s, z6.h, z0.h[0]\n" - ".inst 0x646140cc // bfdot z12.s, z6.h, z1.h[0]\n" - ".inst 0x646240d0 // bfdot z16.s, z6.h, z2.h[0]\n" - ".inst 0x646340d4 // bfdot z20.s, z6.h, z3.h[0]\n" - ".inst 0x646440d8 // bfdot z24.s, z6.h, z4.h[0]\n" - ".inst 0x646540dc // bfdot z28.s, z6.h, z5.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x646040e9 // bfdot z9.s, z7.h, z0.h[0]\n" - ".inst 0x646140ed // bfdot z13.s, z7.h, z1.h[0]\n" - ".inst 0x646240f1 // bfdot z17.s, z7.h, z2.h[0]\n" - ".inst 0x646340f5 // bfdot z21.s, z7.h, z3.h[0]\n" - ".inst 0x646440f9 // bfdot z25.s, z7.h, z4.h[0]\n" - ".inst 0x646540fd // bfdot z29.s, z7.h, z5.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z7.h }, p5/Z, [x10]\n" + "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x646040e8 // bfdot z8.s, z7.h, z0.h[0]\n" + ".inst 0x646140ec // bfdot z12.s, z7.h, z1.h[0]\n" + ".inst 0x646240f0 // bfdot z16.s, z7.h, z2.h[0]\n" + ".inst 0x646340f4 // bfdot z20.s, z7.h, z3.h[0]\n" + ".inst 0x646440f8 // bfdot z24.s, z7.h, z4.h[0]\n" + ".inst 0x646540fc // bfdot z28.s, z7.h, z5.h[0]\n" + "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x646040c9 // bfdot z9.s, z6.h, z0.h[0]\n" + ".inst 0x646140cd // bfdot z13.s, z6.h, z1.h[0]\n" + ".inst 0x646240d1 // bfdot z17.s, z6.h, z2.h[0]\n" + ".inst 0x646340d5 // bfdot z21.s, z6.h, z3.h[0]\n" + ".inst 0x646440d9 // bfdot z25.s, z6.h, z4.h[0]\n" + ".inst 0x646540dd // bfdot z29.s, z6.h, z5.h[0]\n" + "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - ".inst 0x646040ca // bfdot z10.s, z6.h, z0.h[0]\n" - ".inst 0x646140ce // bfdot z14.s, z6.h, z1.h[0]\n" - ".inst 0x646240d2 // bfdot z18.s, z6.h, z2.h[0]\n" - ".inst 0x646340d6 // bfdot z22.s, z6.h, z3.h[0]\n" - ".inst 0x646440da // bfdot z26.s, z6.h, z4.h[0]\n" - ".inst 0x646540de // bfdot z30.s, z6.h, z5.h[0]\n" - ".inst 0x646040eb // bfdot z11.s, z7.h, z0.h[0]\n" - ".inst 0x646140ef // bfdot z15.s, z7.h, z1.h[0]\n" - ".inst 0x646240f3 // bfdot z19.s, z7.h, z2.h[0]\n" - ".inst 0x646340f7 // bfdot z23.s, z7.h, z3.h[0]\n" - ".inst 0x646440fb // bfdot z27.s, z7.h, z4.h[0]\n" - ".inst 0x646540ff // bfdot z31.s, z7.h, z5.h[0]\n" + ".inst 0x646040ea // bfdot z10.s, z7.h, z0.h[0]\n" + ".inst 0x646140ee // bfdot z14.s, z7.h, z1.h[0]\n" + ".inst 0x646240f2 // bfdot z18.s, z7.h, z2.h[0]\n" + ".inst 0x646340f6 // bfdot z22.s, z7.h, z3.h[0]\n" + ".inst 0x646440fa // bfdot z26.s, z7.h, z4.h[0]\n" + ".inst 0x646540fe // bfdot z30.s, z7.h, z5.h[0]\n" + ".inst 0x646040cb // bfdot z11.s, z6.h, z0.h[0]\n" + ".inst 0x646140cf // bfdot z15.s, z6.h, z1.h[0]\n" + ".inst 0x646240d3 // bfdot z19.s, z6.h, z2.h[0]\n" + ".inst 0x646340d7 // bfdot z23.s, z6.h, z3.h[0]\n" + ".inst 0x646440db // bfdot z27.s, z6.h, z4.h[0]\n" + ".inst 0x646540df // bfdot z31.s, z6.h, z5.h[0]\n" "ble 76f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x646840c8 // bfdot z8.s, z6.h, z0.h[1]\n" - ".inst 0x646940cc // bfdot z12.s, z6.h, z1.h[1]\n" - ".inst 0x646a40d0 // bfdot z16.s, z6.h, z2.h[1]\n" - ".inst 0x646b40d4 // bfdot z20.s, z6.h, z3.h[1]\n" + "ld1h { z7.h }, p5/Z, [x10]\n" + "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x646840e8 // bfdot z8.s, z7.h, z0.h[1]\n" + ".inst 0x646940ec // bfdot z12.s, z7.h, z1.h[1]\n" + ".inst 0x646a40f0 // bfdot z16.s, z7.h, z2.h[1]\n" + ".inst 0x646b40f4 // bfdot z20.s, z7.h, z3.h[1]\n" "subs x27, x27, #0x2\n" - ".inst 0x646c40d8 // bfdot z24.s, z6.h, z4.h[1]\n" - ".inst 0x646d40dc // bfdot z28.s, z6.h, z5.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x646840e9 // bfdot z9.s, z7.h, z0.h[1]\n" - ".inst 0x646940ed // bfdot z13.s, z7.h, z1.h[1]\n" - ".inst 0x646a40f1 // bfdot z17.s, z7.h, z2.h[1]\n" - ".inst 0x646b40f5 // bfdot z21.s, z7.h, z3.h[1]\n" - ".inst 0x646c40f9 // bfdot z25.s, z7.h, z4.h[1]\n" - ".inst 0x646d40fd // bfdot z29.s, z7.h, z5.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x646c40f8 // bfdot z24.s, z7.h, z4.h[1]\n" + ".inst 0x646d40fc // bfdot z28.s, z7.h, z5.h[1]\n" + "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x646840c9 // bfdot z9.s, z6.h, z0.h[1]\n" + ".inst 0x646940cd // bfdot z13.s, z6.h, z1.h[1]\n" + ".inst 0x646a40d1 // bfdot z17.s, z6.h, z2.h[1]\n" + ".inst 0x646b40d5 // bfdot z21.s, z6.h, z3.h[1]\n" + ".inst 0x646c40d9 // bfdot z25.s, z6.h, z4.h[1]\n" + ".inst 0x646d40dd // bfdot z29.s, z6.h, z5.h[1]\n" + "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - ".inst 0x646840ca // bfdot z10.s, z6.h, z0.h[1]\n" - ".inst 0x646940ce // bfdot z14.s, z6.h, z1.h[1]\n" - ".inst 0x646a40d2 // bfdot z18.s, z6.h, z2.h[1]\n" - ".inst 0x646b40d6 // bfdot z22.s, z6.h, z3.h[1]\n" - ".inst 0x646c40da // bfdot z26.s, z6.h, z4.h[1]\n" - ".inst 0x646d40de // bfdot z30.s, z6.h, z5.h[1]\n" - ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n" - ".inst 0x646940ef // bfdot z15.s, z7.h, z1.h[1]\n" - ".inst 0x646a40f3 // bfdot z19.s, z7.h, z2.h[1]\n" - ".inst 0x646b40f7 // bfdot z23.s, z7.h, z3.h[1]\n" - ".inst 0x646c40fb // bfdot z27.s, z7.h, z4.h[1]\n" - ".inst 0x646d40ff // bfdot z31.s, z7.h, z5.h[1]\n" + ".inst 0x646840ea // bfdot z10.s, z7.h, z0.h[1]\n" + ".inst 0x646940ee // bfdot z14.s, z7.h, z1.h[1]\n" + ".inst 0x646a40f2 // bfdot z18.s, z7.h, z2.h[1]\n" + ".inst 0x646b40f6 // bfdot z22.s, z7.h, z3.h[1]\n" + ".inst 0x646c40fa // bfdot z26.s, z7.h, z4.h[1]\n" + ".inst 0x646d40fe // bfdot z30.s, z7.h, z5.h[1]\n" + ".inst 0x646840cb // bfdot z11.s, z6.h, z0.h[1]\n" + ".inst 0x646940cf // bfdot z15.s, z6.h, z1.h[1]\n" + ".inst 0x646a40d3 // bfdot z19.s, z6.h, z2.h[1]\n" + ".inst 0x646b40d7 // bfdot z23.s, z6.h, z3.h[1]\n" + ".inst 0x646c40db // bfdot z27.s, z6.h, z4.h[1]\n" + ".inst 0x646d40df // bfdot z31.s, z6.h, z5.h[1]\n" "ble 76f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x647040c8 // bfdot z8.s, z6.h, z0.h[2]\n" - ".inst 0x647140cc // bfdot z12.s, z6.h, z1.h[2]\n" - ".inst 0x647240d0 // bfdot z16.s, z6.h, z2.h[2]\n" - ".inst 0x647340d4 // bfdot z20.s, z6.h, z3.h[2]\n" + "ld1h { z7.h }, p5/Z, [x10]\n" + "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x647040e8 // bfdot z8.s, z7.h, z0.h[2]\n" + ".inst 0x647140ec // bfdot z12.s, z7.h, z1.h[2]\n" + ".inst 0x647240f0 // bfdot z16.s, z7.h, z2.h[2]\n" + ".inst 0x647340f4 // bfdot z20.s, z7.h, z3.h[2]\n" "subs x27, x27, #0x2\n" - ".inst 0x647440d8 // bfdot z24.s, z6.h, z4.h[2]\n" - ".inst 0x647540dc // bfdot z28.s, z6.h, z5.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x647040e9 // bfdot z9.s, z7.h, z0.h[2]\n" - ".inst 0x647140ed // bfdot z13.s, z7.h, z1.h[2]\n" - ".inst 0x647240f1 // bfdot z17.s, z7.h, z2.h[2]\n" - ".inst 0x647340f5 // bfdot z21.s, z7.h, z3.h[2]\n" - ".inst 0x647440f9 // bfdot z25.s, z7.h, z4.h[2]\n" - ".inst 0x647540fd // bfdot z29.s, z7.h, z5.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x647440f8 // bfdot z24.s, z7.h, z4.h[2]\n" + ".inst 0x647540fc // bfdot z28.s, z7.h, z5.h[2]\n" + "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x647040c9 // bfdot z9.s, z6.h, z0.h[2]\n" + ".inst 0x647140cd // bfdot z13.s, z6.h, z1.h[2]\n" + ".inst 0x647240d1 // bfdot z17.s, z6.h, z2.h[2]\n" + ".inst 0x647340d5 // bfdot z21.s, z6.h, z3.h[2]\n" + ".inst 0x647440d9 // bfdot z25.s, z6.h, z4.h[2]\n" + ".inst 0x647540dd // bfdot z29.s, z6.h, z5.h[2]\n" + "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - ".inst 0x647040ca // bfdot z10.s, z6.h, z0.h[2]\n" - ".inst 0x647140ce // bfdot z14.s, z6.h, z1.h[2]\n" - ".inst 0x647240d2 // bfdot z18.s, z6.h, z2.h[2]\n" - ".inst 0x647340d6 // bfdot z22.s, z6.h, z3.h[2]\n" - ".inst 0x647440da // bfdot z26.s, z6.h, z4.h[2]\n" - ".inst 0x647540de // bfdot z30.s, z6.h, z5.h[2]\n" - ".inst 0x647040eb // bfdot z11.s, z7.h, z0.h[2]\n" - ".inst 0x647140ef // bfdot z15.s, z7.h, z1.h[2]\n" - ".inst 0x647240f3 // bfdot z19.s, z7.h, z2.h[2]\n" - ".inst 0x647340f7 // bfdot z23.s, z7.h, z3.h[2]\n" - ".inst 0x647440fb // bfdot z27.s, z7.h, z4.h[2]\n" - ".inst 0x647540ff // bfdot z31.s, z7.h, z5.h[2]\n" + ".inst 0x647040ea // bfdot z10.s, z7.h, z0.h[2]\n" + ".inst 0x647140ee // bfdot z14.s, z7.h, z1.h[2]\n" + ".inst 0x647240f2 // bfdot z18.s, z7.h, z2.h[2]\n" + ".inst 0x647340f6 // bfdot z22.s, z7.h, z3.h[2]\n" + ".inst 0x647440fa // bfdot z26.s, z7.h, z4.h[2]\n" + ".inst 0x647540fe // bfdot z30.s, z7.h, z5.h[2]\n" + ".inst 0x647040cb // bfdot z11.s, z6.h, z0.h[2]\n" + ".inst 0x647140cf // bfdot z15.s, z6.h, z1.h[2]\n" + ".inst 0x647240d3 // bfdot z19.s, z6.h, z2.h[2]\n" + ".inst 0x647340d7 // bfdot z23.s, z6.h, z3.h[2]\n" + ".inst 0x647440db // bfdot z27.s, z6.h, z4.h[2]\n" + ".inst 0x647540df // bfdot z31.s, z6.h, z5.h[2]\n" "ble 76f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x647840c8 // bfdot z8.s, z6.h, z0.h[3]\n" - ".inst 0x647940cc // bfdot z12.s, z6.h, z1.h[3]\n" - ".inst 0x647a40d0 // bfdot z16.s, z6.h, z2.h[3]\n" - ".inst 0x647b40d4 // bfdot z20.s, z6.h, z3.h[3]\n" - ".inst 0x647c40d8 // bfdot z24.s, z6.h, z4.h[3]\n" - ".inst 0x647d40dc // bfdot z28.s, z6.h, z5.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x647840e9 // bfdot z9.s, z7.h, z0.h[3]\n" - ".inst 0x647940ed // bfdot z13.s, z7.h, z1.h[3]\n" - ".inst 0x647a40f1 // bfdot z17.s, z7.h, z2.h[3]\n" - ".inst 0x647b40f5 // bfdot z21.s, z7.h, z3.h[3]\n" - ".inst 0x647c40f9 // bfdot z25.s, z7.h, z4.h[3]\n" - ".inst 0x647d40fd // bfdot z29.s, z7.h, z5.h[3]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z7.h }, p5/Z, [x10]\n" + "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x647840e8 // bfdot z8.s, z7.h, z0.h[3]\n" + ".inst 0x647940ec // bfdot z12.s, z7.h, z1.h[3]\n" + ".inst 0x647a40f0 // bfdot z16.s, z7.h, z2.h[3]\n" + ".inst 0x647b40f4 // bfdot z20.s, z7.h, z3.h[3]\n" + ".inst 0x647c40f8 // bfdot z24.s, z7.h, z4.h[3]\n" + ".inst 0x647d40fc // bfdot z28.s, z7.h, z5.h[3]\n" + "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x647840c9 // bfdot z9.s, z6.h, z0.h[3]\n" + ".inst 0x647940cd // bfdot z13.s, z6.h, z1.h[3]\n" + ".inst 0x647a40d1 // bfdot z17.s, z6.h, z2.h[3]\n" + ".inst 0x647b40d5 // bfdot z21.s, z6.h, z3.h[3]\n" + ".inst 0x647c40d9 // bfdot z25.s, z6.h, z4.h[3]\n" + ".inst 0x647d40dd // bfdot z29.s, z6.h, z5.h[3]\n" + "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - ".inst 0x647840ca // bfdot z10.s, z6.h, z0.h[3]\n" - ".inst 0x647940ce // bfdot z14.s, z6.h, z1.h[3]\n" - ".inst 0x647a40d2 // bfdot z18.s, z6.h, z2.h[3]\n" - ".inst 0x647b40d6 // bfdot z22.s, z6.h, z3.h[3]\n" - ".inst 0x647c40da // bfdot z26.s, z6.h, z4.h[3]\n" - ".inst 0x647d40de // bfdot z30.s, z6.h, z5.h[3]\n" - ".inst 0x647840eb // bfdot z11.s, z7.h, z0.h[3]\n" - ".inst 0x647940ef // bfdot z15.s, z7.h, z1.h[3]\n" - ".inst 0x647a40f3 // bfdot z19.s, z7.h, z2.h[3]\n" - ".inst 0x647b40f7 // bfdot z23.s, z7.h, z3.h[3]\n" - ".inst 0x647c40fb // bfdot z27.s, z7.h, z4.h[3]\n" - ".inst 0x647d40ff // bfdot z31.s, z7.h, z5.h[3]\n" + ".inst 0x647840ea // bfdot z10.s, z7.h, z0.h[3]\n" + ".inst 0x647940ee // bfdot z14.s, z7.h, z1.h[3]\n" + ".inst 0x647a40f2 // bfdot z18.s, z7.h, z2.h[3]\n" + ".inst 0x647b40f6 // bfdot z22.s, z7.h, z3.h[3]\n" + ".inst 0x647c40fa // bfdot z26.s, z7.h, z4.h[3]\n" + ".inst 0x647d40fe // bfdot z30.s, z7.h, z5.h[3]\n" + ".inst 0x647840cb // bfdot z11.s, z6.h, z0.h[3]\n" + ".inst 0x647940cf // bfdot z15.s, z6.h, z1.h[3]\n" + ".inst 0x647a40d3 // bfdot z19.s, z6.h, z2.h[3]\n" + ".inst 0x647b40d7 // bfdot z23.s, z6.h, z3.h[3]\n" + ".inst 0x647c40db // bfdot z27.s, z6.h, z4.h[3]\n" + ".inst 0x647d40df // bfdot z31.s, z6.h, z5.h[3]\n" "76:" // Height 6: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -2082,7 +2082,6 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "80:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -2090,4 +2089,4 @@ void sve_hybrid_bf16fp32_dot_6x4VL ( } } // namespace arm_gemm -#endif // __ARM_FEATURE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_6x4VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_6x4VL.hpp index b8d237ff23300500b312deecd75a0a822fcb5ca0..223d8a78de0af4518f40f2dffabef7656f1fe44d 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_6x4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_6x4VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef ARM_COMPUTE_ENABLE_SVE + #include "../std_transforms_sve.hpp" #include "../bfloat.hpp" #include "../performance_parameters.hpp" @@ -75,7 +75,6 @@ public: template static inline PerformanceParameters get_performance_parameters(const CPUInfo *ci) { - if (std::is_same::value) { switch (ci->get_cpu_model()) { default: @@ -100,5 +99,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_6x4VL/generic.cpp index 9bb67f18d2f1243eeb0d2e9411114b18b53994b2..74e2d267bcb4493bef7156b9c19c50d1af9c76ea 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_6x4VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_bf16fp32_mmla_6x4VL/generic.cpp @@ -133,16 +133,16 @@ void sve_hybrid_bf16fp32_mmla_6x4VL ( "b 5f\n" "3:" // Height 1: no bias "tbz %x[flags], #0, 4f\n" - "ld1w { z9.s }, p4/Z, [x9]\n" - "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x9]\n" + "ld1w { z18.s }, p3/Z, [x9, #1, MUL VL]\n" + "zip1 z8.d, z16.d, z12.d\n" + "zip2 z12.d, z16.d, z12.d\n" + "ld1w { z17.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" + "zip1 z9.d, z18.d, z13.d\n" + "zip2 z13.d, z18.d, z13.d\n" + "zip1 z10.d, z17.d, z14.d\n" + "zip2 z14.d, z17.d, z14.d\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" "b 5f\n" @@ -160,11 +160,11 @@ void sve_hybrid_bf16fp32_mmla_6x4VL ( "6:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 7f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 8f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -176,86 +176,86 @@ void sve_hybrid_bf16fp32_mmla_6x4VL ( "ble 10f\n" "9:" // Height 1: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z1.h }, p0/Z, [x26]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e408 // bfmmla z8.s, z0.h, z7.h\n" - ".inst 0x6466e40c // bfmmla z12.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6467e40a // bfmmla z10.s, z0.h, z7.h\n" - ".inst 0x6466e40e // bfmmla z14.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n" + "ld1rqh { z20.h }, p0/Z, [x26]\n" + "trn1 z18.d, z20.d, z19.d\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6471e648 // bfmmla z8.s, z18.h, z17.h\n" + ".inst 0x6470e64c // bfmmla z12.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6471e649 // bfmmla z9.s, z18.h, z17.h\n" + ".inst 0x6470e64d // bfmmla z13.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #4, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6471e64a // bfmmla z10.s, z18.h, z17.h\n" + ".inst 0x6470e64e // bfmmla z14.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #6, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "trn2 z1.d, z1.d, z2.d\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" - ".inst 0x6466e40f // bfmmla z15.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #-8, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #-7, MUL VL]\n" - ".inst 0x6467e428 // bfmmla z8.s, z1.h, z7.h\n" - ".inst 0x6466e42c // bfmmla z12.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #-6, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #-5, MUL VL]\n" - ".inst 0x6467e429 // bfmmla z9.s, z1.h, z7.h\n" - ".inst 0x6466e42d // bfmmla z13.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #-4, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #-3, MUL VL]\n" - ".inst 0x6467e42a // bfmmla z10.s, z1.h, z7.h\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #-2, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #-1, MUL VL]\n" + "trn2 z20.d, z20.d, z19.d\n" + ".inst 0x6471e64b // bfmmla z11.s, z18.h, z17.h\n" + ".inst 0x6470e64f // bfmmla z15.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #-8, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #-7, MUL VL]\n" + ".inst 0x6471e688 // bfmmla z8.s, z20.h, z17.h\n" + ".inst 0x6470e68c // bfmmla z12.s, z20.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #-6, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #-5, MUL VL]\n" + ".inst 0x6471e689 // bfmmla z9.s, z20.h, z17.h\n" + ".inst 0x6470e68d // bfmmla z13.s, z20.h, z16.h\n" + "ld1h { z16.h }, p5/Z, [x10, #-4, MUL VL]\n" + "ld1h { z17.h }, p5/Z, [x10, #-3, MUL VL]\n" + ".inst 0x6470e68a // bfmmla z10.s, z20.h, z16.h\n" + ".inst 0x6471e68e // bfmmla z14.s, z20.h, z17.h\n" + "ld1h { z17.h }, p5/Z, [x10, #-2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #-1, MUL VL]\n" "sub x27, x27, #0x8\n" "cmp x27, #0x8\n" - ".inst 0x6467e42b // bfmmla z11.s, z1.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" + ".inst 0x6471e68b // bfmmla z11.s, z20.h, z17.h\n" + ".inst 0x6470e68f // bfmmla z15.s, z20.h, z16.h\n" "add x26, x26, #0x10\n" "bgt 9b\n" "10:" // Height 1: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" "ld1rqh { z1.h }, p0/Z, [x26]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e408 // bfmmla z8.s, z0.h, z7.h\n" - ".inst 0x6466e40c // bfmmla z12.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6467e40a // bfmmla z10.s, z0.h, z7.h\n" - ".inst 0x6466e40e // bfmmla z14.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n" + "trn1 z18.d, z1.d, z19.d\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6471e648 // bfmmla z8.s, z18.h, z17.h\n" + ".inst 0x6470e64c // bfmmla z12.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6471e649 // bfmmla z9.s, z18.h, z17.h\n" + ".inst 0x6470e64d // bfmmla z13.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #4, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6471e64a // bfmmla z10.s, z18.h, z17.h\n" + ".inst 0x6470e64e // bfmmla z14.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #6, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #7, MUL VL]\n" "subs x27, x27, #0x4\n" - "trn2 z1.d, z1.d, z2.d\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" - ".inst 0x6466e40f // bfmmla z15.s, z0.h, z6.h\n" + "trn2 z1.d, z1.d, z19.d\n" + ".inst 0x6471e64b // bfmmla z11.s, z18.h, z17.h\n" + ".inst 0x6470e64f // bfmmla z15.s, z18.h, z16.h\n" "addvl x10, x10, #8\n" "ble 11f\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e428 // bfmmla z8.s, z1.h, z7.h\n" - ".inst 0x6466e42c // bfmmla z12.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6467e429 // bfmmla z9.s, z1.h, z7.h\n" - ".inst 0x6466e42d // bfmmla z13.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6467e42a // bfmmla z10.s, z1.h, z7.h\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n" - ".inst 0x6467e42b // bfmmla z11.s, z1.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6471e428 // bfmmla z8.s, z1.h, z17.h\n" + ".inst 0x6470e42c // bfmmla z12.s, z1.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6471e429 // bfmmla z9.s, z1.h, z17.h\n" + ".inst 0x6470e42d // bfmmla z13.s, z1.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #4, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6471e42a // bfmmla z10.s, z1.h, z17.h\n" + ".inst 0x6470e42e // bfmmla z14.s, z1.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #6, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x6471e42b // bfmmla z11.s, z1.h, z17.h\n" + ".inst 0x6470e42f // bfmmla z15.s, z1.h, z16.h\n" "addvl x10, x10, #8\n" "11:" // Height 1: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -268,17 +268,17 @@ void sve_hybrid_bf16fp32_mmla_6x4VL ( "uzp1 z11.d, z11.d, z15.d\n" "tbz %x[flags], #1, 12f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z21.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" + "ld1rw { z16.s }, p5/Z, [x20]\n" + "fmin z8.s, p5/M, z8.s, z21.s\n" + "fmin z9.s, p5/M, z9.s, z21.s\n" + "fmin z10.s, p5/M, z10.s, z21.s\n" + "fmin z11.s, p5/M, z11.s, z21.s\n" + "fmax z8.s, p5/M, z8.s, z16.s\n" + "fmax z9.s, p5/M, z9.s, z16.s\n" + "fmax z10.s, p5/M, z10.s, z16.s\n" + "fmax z11.s, p5/M, z11.s, z16.s\n" "12:" // Height 1: No activation "st1w { z8.s }, p4, [x9]\n" "st1w { z9.s }, p3, [x9, #1, MUL VL]\n" @@ -322,21 +322,21 @@ void sve_hybrid_bf16fp32_mmla_6x4VL ( "16:" // Height 2: no bias "tbz %x[flags], #0, 17f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x9]\n" - "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n" + "add x20, x9, x20, LSL #2\n" + "ld1w { z19.s }, p4/Z, [x9]\n" + "ld1w { z18.s }, p3/Z, [x9, #1, MUL VL]\n" + "ld1w { z17.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" + "ld1w { z12.s }, p4/Z, [x20]\n" + "zip1 z8.d, z19.d, z12.d\n" + "zip2 z12.d, z19.d, z12.d\n" + "ld1w { z13.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x20, #2, MUL VL]\n" + "zip1 z9.d, z18.d, z13.d\n" + "zip2 z13.d, z18.d, z13.d\n" + "ld1w { z15.s }, p1/Z, [x20, #3, MUL VL]\n" + "zip1 z10.d, z17.d, z14.d\n" + "zip2 z14.d, z17.d, z14.d\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" "b 18f\n" @@ -354,12 +354,12 @@ void sve_hybrid_bf16fp32_mmla_6x4VL ( "19:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 20f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 21f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -367,95 +367,95 @@ void sve_hybrid_bf16fp32_mmla_6x4VL ( "b 21f\n" "20:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" "21:" // Height 2: input setup done "cmp x27, #0x8\n" "ble 23f\n" "22:" // Height 2: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z1.h }, p0/Z, [x26]\n" - "ld1rqh { z2.h }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e408 // bfmmla z8.s, z0.h, z7.h\n" - ".inst 0x6466e40c // bfmmla z12.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6467e40a // bfmmla z10.s, z0.h, z7.h\n" - ".inst 0x6466e40e // bfmmla z14.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n" + "ld1rqh { z20.h }, p0/Z, [x26]\n" + "ld1rqh { z19.h }, p0/Z, [x25]\n" + "trn1 z18.d, z20.d, z19.d\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6471e648 // bfmmla z8.s, z18.h, z17.h\n" + ".inst 0x6470e64c // bfmmla z12.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6471e649 // bfmmla z9.s, z18.h, z17.h\n" + ".inst 0x6470e64d // bfmmla z13.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #4, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6471e64a // bfmmla z10.s, z18.h, z17.h\n" + ".inst 0x6470e64e // bfmmla z14.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #6, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "trn2 z1.d, z1.d, z2.d\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" - ".inst 0x6466e40f // bfmmla z15.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #-8, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #-7, MUL VL]\n" - ".inst 0x6467e428 // bfmmla z8.s, z1.h, z7.h\n" - ".inst 0x6466e42c // bfmmla z12.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #-6, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #-5, MUL VL]\n" - ".inst 0x6467e429 // bfmmla z9.s, z1.h, z7.h\n" - ".inst 0x6466e42d // bfmmla z13.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #-4, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #-3, MUL VL]\n" - ".inst 0x6467e42a // bfmmla z10.s, z1.h, z7.h\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #-2, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #-1, MUL VL]\n" + "trn2 z20.d, z20.d, z19.d\n" + ".inst 0x6471e64b // bfmmla z11.s, z18.h, z17.h\n" + ".inst 0x6470e64f // bfmmla z15.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #-8, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #-7, MUL VL]\n" + ".inst 0x6471e688 // bfmmla z8.s, z20.h, z17.h\n" + ".inst 0x6470e68c // bfmmla z12.s, z20.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #-6, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #-5, MUL VL]\n" + ".inst 0x6471e689 // bfmmla z9.s, z20.h, z17.h\n" + ".inst 0x6470e68d // bfmmla z13.s, z20.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #-4, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #-3, MUL VL]\n" + ".inst 0x6471e68a // bfmmla z10.s, z20.h, z17.h\n" + ".inst 0x6470e68e // bfmmla z14.s, z20.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #-2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #-1, MUL VL]\n" "sub x27, x27, #0x8\n" "cmp x27, #0x8\n" - ".inst 0x6467e42b // bfmmla z11.s, z1.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" + ".inst 0x6471e68b // bfmmla z11.s, z20.h, z17.h\n" + ".inst 0x6470e68f // bfmmla z15.s, z20.h, z16.h\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" "bgt 22b\n" "23:" // Height 2: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" "ld1rqh { z1.h }, p0/Z, [x26]\n" - "ld1rqh { z2.h }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e408 // bfmmla z8.s, z0.h, z7.h\n" - ".inst 0x6466e40c // bfmmla z12.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6467e40a // bfmmla z10.s, z0.h, z7.h\n" - ".inst 0x6466e40e // bfmmla z14.s, z0.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n" + "ld1rqh { z19.h }, p0/Z, [x25]\n" + "trn1 z18.d, z1.d, z19.d\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6471e648 // bfmmla z8.s, z18.h, z17.h\n" + ".inst 0x6470e64c // bfmmla z12.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6471e649 // bfmmla z9.s, z18.h, z17.h\n" + ".inst 0x6470e64d // bfmmla z13.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #4, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6471e64a // bfmmla z10.s, z18.h, z17.h\n" + ".inst 0x6470e64e // bfmmla z14.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #6, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #7, MUL VL]\n" "subs x27, x27, #0x4\n" - "trn2 z1.d, z1.d, z2.d\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" - ".inst 0x6466e40f // bfmmla z15.s, z0.h, z6.h\n" + "trn2 z1.d, z1.d, z19.d\n" + ".inst 0x6471e64b // bfmmla z11.s, z18.h, z17.h\n" + ".inst 0x6470e64f // bfmmla z15.s, z18.h, z16.h\n" "addvl x10, x10, #8\n" "ble 24f\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e428 // bfmmla z8.s, z1.h, z7.h\n" - ".inst 0x6466e42c // bfmmla z12.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6467e429 // bfmmla z9.s, z1.h, z7.h\n" - ".inst 0x6466e42d // bfmmla z13.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6467e42a // bfmmla z10.s, z1.h, z7.h\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n" - ".inst 0x6467e42b // bfmmla z11.s, z1.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6471e428 // bfmmla z8.s, z1.h, z17.h\n" + ".inst 0x6470e42c // bfmmla z12.s, z1.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6471e429 // bfmmla z9.s, z1.h, z17.h\n" + ".inst 0x6470e42d // bfmmla z13.s, z1.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #4, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6471e42a // bfmmla z10.s, z1.h, z17.h\n" + ".inst 0x6470e42e // bfmmla z14.s, z1.h, z16.h\n" + "ld1h { z22.h }, p5/Z, [x10, #6, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x6476e42b // bfmmla z11.s, z1.h, z22.h\n" + ".inst 0x6470e42f // bfmmla z15.s, z1.h, z16.h\n" "addvl x10, x10, #8\n" "24:" // Height 2: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -474,25 +474,25 @@ void sve_hybrid_bf16fp32_mmla_6x4VL ( "uzp2 z11.d, z11.d, z15.d\n" "tbz %x[flags], #1, 25f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z17.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z7.s, p5/M, z7.s, z1.s\n" - "fmin z12.s, p5/M, z12.s, z1.s\n" - "fmin z13.s, p5/M, z13.s, z1.s\n" - "fmin z14.s, p5/M, z14.s, z1.s\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmax z7.s, p5/M, z7.s, z0.s\n" - "fmax z12.s, p5/M, z12.s, z0.s\n" - "fmax z13.s, p5/M, z13.s, z0.s\n" - "fmax z14.s, p5/M, z14.s, z0.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" + "ld1rw { z16.s }, p5/Z, [x20]\n" + "fmin z7.s, p5/M, z7.s, z17.s\n" + "fmin z12.s, p5/M, z12.s, z17.s\n" + "fmin z13.s, p5/M, z13.s, z17.s\n" + "fmin z14.s, p5/M, z14.s, z17.s\n" + "fmin z8.s, p5/M, z8.s, z17.s\n" + "fmin z9.s, p5/M, z9.s, z17.s\n" + "fmin z10.s, p5/M, z10.s, z17.s\n" + "fmin z11.s, p5/M, z11.s, z17.s\n" + "fmax z7.s, p5/M, z7.s, z16.s\n" + "fmax z12.s, p5/M, z12.s, z16.s\n" + "fmax z13.s, p5/M, z13.s, z16.s\n" + "fmax z14.s, p5/M, z14.s, z16.s\n" + "fmax z8.s, p5/M, z8.s, z16.s\n" + "fmax z9.s, p5/M, z9.s, z16.s\n" + "fmax z10.s, p5/M, z10.s, z16.s\n" + "fmax z11.s, p5/M, z11.s, z16.s\n" "25:" // Height 2: No activation "st1w { z7.s }, p4, [x9]\n" "st1w { z12.s }, p3, [x9, #1, MUL VL]\n" @@ -548,28 +548,28 @@ void sve_hybrid_bf16fp32_mmla_6x4VL ( "29:" // Height 3: no bias "tbz %x[flags], #0, 30f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x9]\n" - "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n" + "add x21, x9, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" + "ld1w { z19.s }, p4/Z, [x9]\n" + "ld1w { z17.s }, p3/Z, [x9, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "ld1w { z17.s }, p4/Z, [x24]\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" - "ld1w { z18.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z19.s }, p2/Z, [x24, #2, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x21]\n" + "zip1 z8.d, z19.d, z12.d\n" + "zip2 z12.d, z19.d, z12.d\n" + "ld1w { z13.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x21, #2, MUL VL]\n" + "zip1 z9.d, z17.d, z13.d\n" + "zip2 z13.d, z17.d, z13.d\n" + "ld1w { z15.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z17.s }, p4/Z, [x20]\n" + "zip1 z10.d, z18.d, z14.d\n" + "zip2 z14.d, z18.d, z14.d\n" + "ld1w { z18.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z19.s }, p2/Z, [x20, #2, MUL VL]\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" - "ld1w { z24.s }, p1/Z, [x24, #3, MUL VL]\n" + "ld1w { z24.s }, p1/Z, [x20, #3, MUL VL]\n" "zip1 z16.d, z17.d, z20.d\n" "zip2 z20.d, z17.d, z20.d\n" "zip1 z17.d, z18.d, z21.d\n" @@ -601,13 +601,13 @@ void sve_hybrid_bf16fp32_mmla_6x4VL ( "32:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 33f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 34f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -616,136 +616,136 @@ void sve_hybrid_bf16fp32_mmla_6x4VL ( "b 34f\n" "33:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" "34:" // Height 3: input setup done "cmp x27, #0x8\n" "ble 36f\n" "35:" // Height 3: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z1.h }, p0/Z, [x26]\n" - "ld1rqh { z2.h }, p0/Z, [x25]\n" - "ld1rqh { z3.h }, p0/Z, [x24]\n" - "trn1 z0.d, z1.d, z2.d\n" - "trn2 z1.d, z1.d, z2.d\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e408 // bfmmla z8.s, z0.h, z7.h\n" - ".inst 0x6467e450 // bfmmla z16.s, z2.h, z7.h\n" - ".inst 0x6466e40c // bfmmla z12.s, z0.h, z6.h\n" - ".inst 0x6466e454 // bfmmla z20.s, z2.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" - ".inst 0x6467e451 // bfmmla z17.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n" + "ld1rqh { z30.h }, p0/Z, [x26]\n" + "ld1rqh { z24.h }, p0/Z, [x25]\n" + "ld1rqh { z28.h }, p0/Z, [x24]\n" + "trn1 z27.d, z30.d, z24.d\n" + "trn2 z30.d, z30.d, z24.d\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "trn1 z26.d, z28.d, z29.d\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6479e768 // bfmmla z8.s, z27.h, z25.h\n" + ".inst 0x6479e750 // bfmmla z16.s, z26.h, z25.h\n" + ".inst 0x6478e76c // bfmmla z12.s, z27.h, z24.h\n" + ".inst 0x6478e754 // bfmmla z20.s, z26.h, z24.h\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6479e769 // bfmmla z9.s, z27.h, z25.h\n" + ".inst 0x6479e751 // bfmmla z17.s, z26.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10, #4, MUL VL]\n" + "trn2 z28.d, z28.d, z29.d\n" + ".inst 0x6478e76d // bfmmla z13.s, z27.h, z24.h\n" + ".inst 0x6478e755 // bfmmla z21.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #5, MUL VL]\n" "sub x27, x27, #0x8\n" - ".inst 0x6467e40a // bfmmla z10.s, z0.h, z7.h\n" - ".inst 0x6467e452 // bfmmla z18.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x6479e76a // bfmmla z10.s, z27.h, z25.h\n" + ".inst 0x6479e752 // bfmmla z18.s, z26.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10, #6, MUL VL]\n" "cmp x27, #0x8\n" - ".inst 0x6466e40e // bfmmla z14.s, z0.h, z6.h\n" - ".inst 0x6466e456 // bfmmla z22.s, z2.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x6478e76e // bfmmla z14.s, z27.h, z24.h\n" + ".inst 0x6478e756 // bfmmla z22.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" - ".inst 0x6467e453 // bfmmla z19.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #-8, MUL VL]\n" + ".inst 0x6479e76b // bfmmla z11.s, z27.h, z25.h\n" + ".inst 0x6479e753 // bfmmla z19.s, z26.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10, #-8, MUL VL]\n" "add x26, x26, #0x10\n" - ".inst 0x6466e40f // bfmmla z15.s, z0.h, z6.h\n" - ".inst 0x6466e457 // bfmmla z23.s, z2.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #-7, MUL VL]\n" + ".inst 0x6478e76f // bfmmla z15.s, z27.h, z24.h\n" + ".inst 0x6478e757 // bfmmla z23.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #-7, MUL VL]\n" "add x25, x25, #0x10\n" - ".inst 0x6467e428 // bfmmla z8.s, z1.h, z7.h\n" - ".inst 0x6467e470 // bfmmla z16.s, z3.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #-6, MUL VL]\n" + ".inst 0x6479e7c8 // bfmmla z8.s, z30.h, z25.h\n" + ".inst 0x6479e790 // bfmmla z16.s, z28.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10, #-6, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x6466e42c // bfmmla z12.s, z1.h, z6.h\n" - ".inst 0x6466e474 // bfmmla z20.s, z3.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #-5, MUL VL]\n" - ".inst 0x6467e429 // bfmmla z9.s, z1.h, z7.h\n" - ".inst 0x6467e471 // bfmmla z17.s, z3.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #-4, MUL VL]\n" - ".inst 0x6466e42d // bfmmla z13.s, z1.h, z6.h\n" - ".inst 0x6466e475 // bfmmla z21.s, z3.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #-3, MUL VL]\n" - ".inst 0x6467e42a // bfmmla z10.s, z1.h, z7.h\n" - ".inst 0x6467e472 // bfmmla z18.s, z3.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #-2, MUL VL]\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - ".inst 0x6466e476 // bfmmla z22.s, z3.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #-1, MUL VL]\n" - ".inst 0x6467e42b // bfmmla z11.s, z1.h, z7.h\n" - ".inst 0x6467e473 // bfmmla z19.s, z3.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" - ".inst 0x6466e477 // bfmmla z23.s, z3.h, z6.h\n" + ".inst 0x6478e7cc // bfmmla z12.s, z30.h, z24.h\n" + ".inst 0x6478e794 // bfmmla z20.s, z28.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #-5, MUL VL]\n" + ".inst 0x6479e7c9 // bfmmla z9.s, z30.h, z25.h\n" + ".inst 0x6479e791 // bfmmla z17.s, z28.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10, #-4, MUL VL]\n" + ".inst 0x6478e7cd // bfmmla z13.s, z30.h, z24.h\n" + ".inst 0x6478e795 // bfmmla z21.s, z28.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #-3, MUL VL]\n" + ".inst 0x6479e7ca // bfmmla z10.s, z30.h, z25.h\n" + ".inst 0x6479e792 // bfmmla z18.s, z28.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10, #-2, MUL VL]\n" + ".inst 0x6478e7ce // bfmmla z14.s, z30.h, z24.h\n" + ".inst 0x6478e796 // bfmmla z22.s, z28.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #-1, MUL VL]\n" + ".inst 0x6479e7cb // bfmmla z11.s, z30.h, z25.h\n" + ".inst 0x6479e793 // bfmmla z19.s, z28.h, z25.h\n" + ".inst 0x6478e7cf // bfmmla z15.s, z30.h, z24.h\n" + ".inst 0x6478e797 // bfmmla z23.s, z28.h, z24.h\n" "bgt 35b\n" "36:" // Height 3: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" "ld1rqh { z1.h }, p0/Z, [x26]\n" - "ld1rqh { z2.h }, p0/Z, [x25]\n" + "ld1rqh { z24.h }, p0/Z, [x25]\n" "ld1rqh { z3.h }, p0/Z, [x24]\n" - "trn1 z0.d, z1.d, z2.d\n" - "trn2 z1.d, z1.d, z2.d\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e408 // bfmmla z8.s, z0.h, z7.h\n" - ".inst 0x6467e450 // bfmmla z16.s, z2.h, z7.h\n" - ".inst 0x6466e40c // bfmmla z12.s, z0.h, z6.h\n" - ".inst 0x6466e454 // bfmmla z20.s, z2.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" - ".inst 0x6467e451 // bfmmla z17.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n" + "trn1 z27.d, z1.d, z24.d\n" + "trn2 z1.d, z1.d, z24.d\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "trn1 z26.d, z3.d, z28.d\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6479e768 // bfmmla z8.s, z27.h, z25.h\n" + ".inst 0x6479e750 // bfmmla z16.s, z26.h, z25.h\n" + ".inst 0x6478e76c // bfmmla z12.s, z27.h, z24.h\n" + ".inst 0x6478e754 // bfmmla z20.s, z26.h, z24.h\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6479e769 // bfmmla z9.s, z27.h, z25.h\n" + ".inst 0x6479e751 // bfmmla z17.s, z26.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10, #4, MUL VL]\n" "subs x27, x27, #0x4\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x6467e40a // bfmmla z10.s, z0.h, z7.h\n" - ".inst 0x6467e452 // bfmmla z18.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x6466e40e // bfmmla z14.s, z0.h, z6.h\n" - ".inst 0x6466e456 // bfmmla z22.s, z2.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x6478e76d // bfmmla z13.s, z27.h, z24.h\n" + ".inst 0x6478e755 // bfmmla z21.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #5, MUL VL]\n" + "trn2 z3.d, z3.d, z28.d\n" + ".inst 0x6479e76a // bfmmla z10.s, z27.h, z25.h\n" + ".inst 0x6479e752 // bfmmla z18.s, z26.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x6478e76e // bfmmla z14.s, z27.h, z24.h\n" + ".inst 0x6478e756 // bfmmla z22.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" - ".inst 0x6467e453 // bfmmla z19.s, z2.h, z7.h\n" - ".inst 0x6466e40f // bfmmla z15.s, z0.h, z6.h\n" - ".inst 0x6466e457 // bfmmla z23.s, z2.h, z6.h\n" + ".inst 0x6479e76b // bfmmla z11.s, z27.h, z25.h\n" + ".inst 0x6479e753 // bfmmla z19.s, z26.h, z25.h\n" + ".inst 0x6478e76f // bfmmla z15.s, z27.h, z24.h\n" + ".inst 0x6478e757 // bfmmla z23.s, z26.h, z24.h\n" "ble 37f\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e428 // bfmmla z8.s, z1.h, z7.h\n" - ".inst 0x6467e470 // bfmmla z16.s, z3.h, z7.h\n" - ".inst 0x6466e42c // bfmmla z12.s, z1.h, z6.h\n" - ".inst 0x6466e474 // bfmmla z20.s, z3.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6467e429 // bfmmla z9.s, z1.h, z7.h\n" - ".inst 0x6467e471 // bfmmla z17.s, z3.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x6466e42d // bfmmla z13.s, z1.h, z6.h\n" - ".inst 0x6466e475 // bfmmla z21.s, z3.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6467e42a // bfmmla z10.s, z1.h, z7.h\n" - ".inst 0x6467e472 // bfmmla z18.s, z3.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - ".inst 0x6466e476 // bfmmla z22.s, z3.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6479e428 // bfmmla z8.s, z1.h, z25.h\n" + ".inst 0x6479e470 // bfmmla z16.s, z3.h, z25.h\n" + ".inst 0x6478e42c // bfmmla z12.s, z1.h, z24.h\n" + ".inst 0x6478e474 // bfmmla z20.s, z3.h, z24.h\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6479e429 // bfmmla z9.s, z1.h, z25.h\n" + ".inst 0x6479e471 // bfmmla z17.s, z3.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x6478e42d // bfmmla z13.s, z1.h, z24.h\n" + ".inst 0x6478e475 // bfmmla z21.s, z3.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6479e42a // bfmmla z10.s, z1.h, z25.h\n" + ".inst 0x6479e472 // bfmmla z18.s, z3.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x6478e42e // bfmmla z14.s, z1.h, z24.h\n" + ".inst 0x6478e476 // bfmmla z22.s, z3.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - ".inst 0x6467e42b // bfmmla z11.s, z1.h, z7.h\n" - ".inst 0x6467e473 // bfmmla z19.s, z3.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" - ".inst 0x6466e477 // bfmmla z23.s, z3.h, z6.h\n" + ".inst 0x6479e42b // bfmmla z11.s, z1.h, z25.h\n" + ".inst 0x6479e473 // bfmmla z19.s, z3.h, z25.h\n" + ".inst 0x6478e42f // bfmmla z15.s, z1.h, z24.h\n" + ".inst 0x6478e477 // bfmmla z23.s, z3.h, z24.h\n" "37:" // Height 3: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -768,33 +768,33 @@ void sve_hybrid_bf16fp32_mmla_6x4VL ( "uzp1 z19.d, z19.d, z23.d\n" "tbz %x[flags], #1, 38f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z25.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z7.s, p5/M, z7.s, z1.s\n" - "fmin z12.s, p5/M, z12.s, z1.s\n" - "fmin z13.s, p5/M, z13.s, z1.s\n" - "fmin z14.s, p5/M, z14.s, z1.s\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmin z16.s, p5/M, z16.s, z1.s\n" - "fmin z17.s, p5/M, z17.s, z1.s\n" - "fmin z18.s, p5/M, z18.s, z1.s\n" - "fmin z19.s, p5/M, z19.s, z1.s\n" - "fmax z7.s, p5/M, z7.s, z0.s\n" - "fmax z12.s, p5/M, z12.s, z0.s\n" - "fmax z13.s, p5/M, z13.s, z0.s\n" - "fmax z14.s, p5/M, z14.s, z0.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" - "fmax z16.s, p5/M, z16.s, z0.s\n" - "fmax z17.s, p5/M, z17.s, z0.s\n" - "fmax z18.s, p5/M, z18.s, z0.s\n" - "fmax z19.s, p5/M, z19.s, z0.s\n" + "ld1rw { z24.s }, p5/Z, [x20]\n" + "fmin z7.s, p5/M, z7.s, z25.s\n" + "fmin z12.s, p5/M, z12.s, z25.s\n" + "fmin z13.s, p5/M, z13.s, z25.s\n" + "fmin z14.s, p5/M, z14.s, z25.s\n" + "fmin z8.s, p5/M, z8.s, z25.s\n" + "fmin z9.s, p5/M, z9.s, z25.s\n" + "fmin z10.s, p5/M, z10.s, z25.s\n" + "fmin z11.s, p5/M, z11.s, z25.s\n" + "fmin z16.s, p5/M, z16.s, z25.s\n" + "fmin z17.s, p5/M, z17.s, z25.s\n" + "fmin z18.s, p5/M, z18.s, z25.s\n" + "fmin z19.s, p5/M, z19.s, z25.s\n" + "fmax z7.s, p5/M, z7.s, z24.s\n" + "fmax z12.s, p5/M, z12.s, z24.s\n" + "fmax z13.s, p5/M, z13.s, z24.s\n" + "fmax z14.s, p5/M, z14.s, z24.s\n" + "fmax z8.s, p5/M, z8.s, z24.s\n" + "fmax z9.s, p5/M, z9.s, z24.s\n" + "fmax z10.s, p5/M, z10.s, z24.s\n" + "fmax z11.s, p5/M, z11.s, z24.s\n" + "fmax z16.s, p5/M, z16.s, z24.s\n" + "fmax z17.s, p5/M, z17.s, z24.s\n" + "fmax z18.s, p5/M, z18.s, z24.s\n" + "fmax z19.s, p5/M, z19.s, z24.s\n" "38:" // Height 3: No activation "st1w { z7.s }, p4, [x9]\n" "st1w { z12.s }, p3, [x9, #1, MUL VL]\n" @@ -854,37 +854,37 @@ void sve_hybrid_bf16fp32_mmla_6x4VL ( "42:" // Height 4: no bias "tbz %x[flags], #0, 43f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x9]\n" - "add x23, x24, x20, LSL #2\n" - "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n" + "add x22, x9, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" + "ld1w { z19.s }, p4/Z, [x9]\n" + "add x20, x21, x20, LSL #2\n" + "ld1w { z17.s }, p3/Z, [x9, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "ld1w { z17.s }, p4/Z, [x24]\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" - "ld1w { z18.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z19.s }, p2/Z, [x24, #2, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x22]\n" + "zip1 z8.d, z19.d, z12.d\n" + "zip2 z12.d, z19.d, z12.d\n" + "ld1w { z13.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x22, #2, MUL VL]\n" + "zip1 z9.d, z17.d, z13.d\n" + "zip2 z13.d, z17.d, z13.d\n" + "ld1w { z15.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z17.s }, p4/Z, [x21]\n" + "zip1 z10.d, z18.d, z14.d\n" + "zip2 z14.d, z18.d, z14.d\n" + "ld1w { z18.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z19.s }, p2/Z, [x21, #2, MUL VL]\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" - "ld1w { z24.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x23]\n" + "ld1w { z24.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x20]\n" "zip1 z16.d, z17.d, z20.d\n" "zip2 z20.d, z17.d, z20.d\n" - "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n" + "ld1w { z21.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x20, #2, MUL VL]\n" "zip1 z17.d, z18.d, z21.d\n" "zip2 z21.d, z18.d, z21.d\n" - "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z23.s }, p1/Z, [x20, #3, MUL VL]\n" "zip1 z18.d, z19.d, z22.d\n" "zip2 z22.d, z19.d, z22.d\n" "zip1 z19.d, z24.d, z23.d\n" @@ -912,14 +912,14 @@ void sve_hybrid_bf16fp32_mmla_6x4VL ( "45:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 46f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 47f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -929,140 +929,140 @@ void sve_hybrid_bf16fp32_mmla_6x4VL ( "b 47f\n" "46:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" "47:" // Height 4: input setup done "cmp x27, #0x8\n" "ble 49f\n" "48:" // Height 4: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z1.h }, p0/Z, [x26]\n" - "ld1rqh { z2.h }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1rqh { z3.h }, p0/Z, [x24]\n" - "ld1rqh { z4.h }, p0/Z, [x23]\n" - "trn2 z1.d, z1.d, z2.d\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e408 // bfmmla z8.s, z0.h, z7.h\n" - ".inst 0x6467e450 // bfmmla z16.s, z2.h, z7.h\n" - ".inst 0x6466e40c // bfmmla z12.s, z0.h, z6.h\n" - ".inst 0x6466e454 // bfmmla z20.s, z2.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" - ".inst 0x6467e451 // bfmmla z17.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n" + "ld1rqh { z30.h }, p0/Z, [x26]\n" + "ld1rqh { z24.h }, p0/Z, [x25]\n" + "trn1 z29.d, z30.d, z24.d\n" + "ld1rqh { z28.h }, p0/Z, [x24]\n" + "ld1rqh { z27.h }, p0/Z, [x23]\n" + "trn2 z30.d, z30.d, z24.d\n" + "trn1 z26.d, z28.d, z27.d\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6479e7a8 // bfmmla z8.s, z29.h, z25.h\n" + ".inst 0x6479e750 // bfmmla z16.s, z26.h, z25.h\n" + ".inst 0x6478e7ac // bfmmla z12.s, z29.h, z24.h\n" + ".inst 0x6478e754 // bfmmla z20.s, z26.h, z24.h\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6479e7a9 // bfmmla z9.s, z29.h, z25.h\n" + ".inst 0x6479e751 // bfmmla z17.s, z26.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10, #4, MUL VL]\n" + "trn2 z28.d, z28.d, z27.d\n" + ".inst 0x6478e7ad // bfmmla z13.s, z29.h, z24.h\n" + ".inst 0x6478e755 // bfmmla z21.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #5, MUL VL]\n" "sub x27, x27, #0x8\n" - ".inst 0x6467e40a // bfmmla z10.s, z0.h, z7.h\n" - ".inst 0x6467e452 // bfmmla z18.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x6479e7aa // bfmmla z10.s, z29.h, z25.h\n" + ".inst 0x6479e752 // bfmmla z18.s, z26.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10, #6, MUL VL]\n" "cmp x27, #0x8\n" - ".inst 0x6466e40e // bfmmla z14.s, z0.h, z6.h\n" - ".inst 0x6466e456 // bfmmla z22.s, z2.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x6478e7ae // bfmmla z14.s, z29.h, z24.h\n" + ".inst 0x6478e756 // bfmmla z22.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" - ".inst 0x6467e453 // bfmmla z19.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #-8, MUL VL]\n" + ".inst 0x6479e7ab // bfmmla z11.s, z29.h, z25.h\n" + ".inst 0x6479e753 // bfmmla z19.s, z26.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10, #-8, MUL VL]\n" "add x26, x26, #0x10\n" - ".inst 0x6466e40f // bfmmla z15.s, z0.h, z6.h\n" - ".inst 0x6466e457 // bfmmla z23.s, z2.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #-7, MUL VL]\n" + ".inst 0x6478e7af // bfmmla z15.s, z29.h, z24.h\n" + ".inst 0x6478e757 // bfmmla z23.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #-7, MUL VL]\n" "add x25, x25, #0x10\n" - ".inst 0x6467e428 // bfmmla z8.s, z1.h, z7.h\n" - ".inst 0x6467e470 // bfmmla z16.s, z3.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #-6, MUL VL]\n" + ".inst 0x6479e7c8 // bfmmla z8.s, z30.h, z25.h\n" + ".inst 0x6479e790 // bfmmla z16.s, z28.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10, #-6, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x6466e42c // bfmmla z12.s, z1.h, z6.h\n" - ".inst 0x6466e474 // bfmmla z20.s, z3.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #-5, MUL VL]\n" + ".inst 0x6478e7cc // bfmmla z12.s, z30.h, z24.h\n" + ".inst 0x6478e794 // bfmmla z20.s, z28.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #-5, MUL VL]\n" "add x23, x23, #0x10\n" - ".inst 0x6467e429 // bfmmla z9.s, z1.h, z7.h\n" - ".inst 0x6467e471 // bfmmla z17.s, z3.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #-4, MUL VL]\n" - ".inst 0x6466e42d // bfmmla z13.s, z1.h, z6.h\n" - ".inst 0x6466e475 // bfmmla z21.s, z3.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #-3, MUL VL]\n" - ".inst 0x6467e42a // bfmmla z10.s, z1.h, z7.h\n" - ".inst 0x6467e472 // bfmmla z18.s, z3.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #-2, MUL VL]\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - ".inst 0x6466e476 // bfmmla z22.s, z3.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #-1, MUL VL]\n" - ".inst 0x6467e42b // bfmmla z11.s, z1.h, z7.h\n" - ".inst 0x6467e473 // bfmmla z19.s, z3.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" - ".inst 0x6466e477 // bfmmla z23.s, z3.h, z6.h\n" + ".inst 0x6479e7c9 // bfmmla z9.s, z30.h, z25.h\n" + ".inst 0x6479e791 // bfmmla z17.s, z28.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10, #-4, MUL VL]\n" + ".inst 0x6478e7cd // bfmmla z13.s, z30.h, z24.h\n" + ".inst 0x6478e795 // bfmmla z21.s, z28.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #-3, MUL VL]\n" + ".inst 0x6479e7ca // bfmmla z10.s, z30.h, z25.h\n" + ".inst 0x6479e792 // bfmmla z18.s, z28.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10, #-2, MUL VL]\n" + ".inst 0x6478e7ce // bfmmla z14.s, z30.h, z24.h\n" + ".inst 0x6478e796 // bfmmla z22.s, z28.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #-1, MUL VL]\n" + ".inst 0x6479e7cb // bfmmla z11.s, z30.h, z25.h\n" + ".inst 0x6479e793 // bfmmla z19.s, z28.h, z25.h\n" + ".inst 0x6478e7cf // bfmmla z15.s, z30.h, z24.h\n" + ".inst 0x6478e797 // bfmmla z23.s, z28.h, z24.h\n" "bgt 48b\n" "49:" // Height 4: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" "ld1rqh { z1.h }, p0/Z, [x26]\n" - "ld1rqh { z2.h }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" + "ld1rqh { z24.h }, p0/Z, [x25]\n" + "trn1 z28.d, z1.d, z24.d\n" "ld1rqh { z3.h }, p0/Z, [x24]\n" - "ld1rqh { z4.h }, p0/Z, [x23]\n" - "trn2 z1.d, z1.d, z2.d\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e408 // bfmmla z8.s, z0.h, z7.h\n" - ".inst 0x6467e450 // bfmmla z16.s, z2.h, z7.h\n" - ".inst 0x6466e40c // bfmmla z12.s, z0.h, z6.h\n" - ".inst 0x6466e454 // bfmmla z20.s, z2.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" - ".inst 0x6467e451 // bfmmla z17.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n" + "ld1rqh { z27.h }, p0/Z, [x23]\n" + "trn2 z1.d, z1.d, z24.d\n" + "trn1 z26.d, z3.d, z27.d\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6479e788 // bfmmla z8.s, z28.h, z25.h\n" + ".inst 0x6479e750 // bfmmla z16.s, z26.h, z25.h\n" + ".inst 0x6478e78c // bfmmla z12.s, z28.h, z24.h\n" + ".inst 0x6478e754 // bfmmla z20.s, z26.h, z24.h\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6479e789 // bfmmla z9.s, z28.h, z25.h\n" + ".inst 0x6479e751 // bfmmla z17.s, z26.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10, #4, MUL VL]\n" "subs x27, x27, #0x4\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x6467e40a // bfmmla z10.s, z0.h, z7.h\n" - ".inst 0x6467e452 // bfmmla z18.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x6466e40e // bfmmla z14.s, z0.h, z6.h\n" - ".inst 0x6466e456 // bfmmla z22.s, z2.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x6478e78d // bfmmla z13.s, z28.h, z24.h\n" + ".inst 0x6478e755 // bfmmla z21.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #5, MUL VL]\n" + "trn2 z3.d, z3.d, z27.d\n" + ".inst 0x6479e78a // bfmmla z10.s, z28.h, z25.h\n" + ".inst 0x6479e752 // bfmmla z18.s, z26.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x6478e78e // bfmmla z14.s, z28.h, z24.h\n" + ".inst 0x6478e756 // bfmmla z22.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" - ".inst 0x6467e453 // bfmmla z19.s, z2.h, z7.h\n" - ".inst 0x6466e40f // bfmmla z15.s, z0.h, z6.h\n" - ".inst 0x6466e457 // bfmmla z23.s, z2.h, z6.h\n" + ".inst 0x6479e78b // bfmmla z11.s, z28.h, z25.h\n" + ".inst 0x6479e753 // bfmmla z19.s, z26.h, z25.h\n" + ".inst 0x6478e78f // bfmmla z15.s, z28.h, z24.h\n" + ".inst 0x6478e757 // bfmmla z23.s, z26.h, z24.h\n" "ble 50f\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e428 // bfmmla z8.s, z1.h, z7.h\n" - ".inst 0x6467e470 // bfmmla z16.s, z3.h, z7.h\n" - ".inst 0x6466e42c // bfmmla z12.s, z1.h, z6.h\n" - ".inst 0x6466e474 // bfmmla z20.s, z3.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6467e429 // bfmmla z9.s, z1.h, z7.h\n" - ".inst 0x6467e471 // bfmmla z17.s, z3.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x6466e42d // bfmmla z13.s, z1.h, z6.h\n" - ".inst 0x6466e475 // bfmmla z21.s, z3.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6467e42a // bfmmla z10.s, z1.h, z7.h\n" - ".inst 0x6467e472 // bfmmla z18.s, z3.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - ".inst 0x6466e476 // bfmmla z22.s, z3.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6479e428 // bfmmla z8.s, z1.h, z25.h\n" + ".inst 0x6479e470 // bfmmla z16.s, z3.h, z25.h\n" + ".inst 0x6478e42c // bfmmla z12.s, z1.h, z24.h\n" + ".inst 0x6478e474 // bfmmla z20.s, z3.h, z24.h\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6479e429 // bfmmla z9.s, z1.h, z25.h\n" + ".inst 0x6479e471 // bfmmla z17.s, z3.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x6478e42d // bfmmla z13.s, z1.h, z24.h\n" + ".inst 0x6478e475 // bfmmla z21.s, z3.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6479e42a // bfmmla z10.s, z1.h, z25.h\n" + ".inst 0x6479e472 // bfmmla z18.s, z3.h, z25.h\n" + "ld1h { z25.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x6478e42e // bfmmla z14.s, z1.h, z24.h\n" + ".inst 0x6478e476 // bfmmla z22.s, z3.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - ".inst 0x6467e42b // bfmmla z11.s, z1.h, z7.h\n" - ".inst 0x6467e473 // bfmmla z19.s, z3.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" - ".inst 0x6466e477 // bfmmla z23.s, z3.h, z6.h\n" + ".inst 0x6479e42b // bfmmla z11.s, z1.h, z25.h\n" + ".inst 0x6479e473 // bfmmla z19.s, z3.h, z25.h\n" + ".inst 0x6478e42f // bfmmla z15.s, z1.h, z24.h\n" + ".inst 0x6478e477 // bfmmla z23.s, z3.h, z24.h\n" "50:" // Height 4: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1090,41 +1090,41 @@ void sve_hybrid_bf16fp32_mmla_6x4VL ( "uzp2 z19.d, z19.d, z23.d\n" "tbz %x[flags], #1, 51f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z24.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z7.s, p5/M, z7.s, z1.s\n" - "fmin z12.s, p5/M, z12.s, z1.s\n" - "fmin z13.s, p5/M, z13.s, z1.s\n" - "fmin z14.s, p5/M, z14.s, z1.s\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmin z15.s, p5/M, z15.s, z1.s\n" - "fmin z20.s, p5/M, z20.s, z1.s\n" - "fmin z21.s, p5/M, z21.s, z1.s\n" - "fmin z22.s, p5/M, z22.s, z1.s\n" - "fmin z16.s, p5/M, z16.s, z1.s\n" - "fmin z17.s, p5/M, z17.s, z1.s\n" - "fmin z18.s, p5/M, z18.s, z1.s\n" - "fmin z19.s, p5/M, z19.s, z1.s\n" - "fmax z7.s, p5/M, z7.s, z0.s\n" - "fmax z12.s, p5/M, z12.s, z0.s\n" - "fmax z13.s, p5/M, z13.s, z0.s\n" - "fmax z14.s, p5/M, z14.s, z0.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" - "fmax z15.s, p5/M, z15.s, z0.s\n" - "fmax z20.s, p5/M, z20.s, z0.s\n" - "fmax z21.s, p5/M, z21.s, z0.s\n" - "fmax z22.s, p5/M, z22.s, z0.s\n" - "fmax z16.s, p5/M, z16.s, z0.s\n" - "fmax z17.s, p5/M, z17.s, z0.s\n" - "fmax z18.s, p5/M, z18.s, z0.s\n" - "fmax z19.s, p5/M, z19.s, z0.s\n" + "ld1rw { z23.s }, p5/Z, [x20]\n" + "fmin z7.s, p5/M, z7.s, z24.s\n" + "fmin z12.s, p5/M, z12.s, z24.s\n" + "fmin z13.s, p5/M, z13.s, z24.s\n" + "fmin z14.s, p5/M, z14.s, z24.s\n" + "fmin z8.s, p5/M, z8.s, z24.s\n" + "fmin z9.s, p5/M, z9.s, z24.s\n" + "fmin z10.s, p5/M, z10.s, z24.s\n" + "fmin z11.s, p5/M, z11.s, z24.s\n" + "fmin z15.s, p5/M, z15.s, z24.s\n" + "fmin z20.s, p5/M, z20.s, z24.s\n" + "fmin z21.s, p5/M, z21.s, z24.s\n" + "fmin z22.s, p5/M, z22.s, z24.s\n" + "fmin z16.s, p5/M, z16.s, z24.s\n" + "fmin z17.s, p5/M, z17.s, z24.s\n" + "fmin z18.s, p5/M, z18.s, z24.s\n" + "fmin z19.s, p5/M, z19.s, z24.s\n" + "fmax z7.s, p5/M, z7.s, z23.s\n" + "fmax z12.s, p5/M, z12.s, z23.s\n" + "fmax z13.s, p5/M, z13.s, z23.s\n" + "fmax z14.s, p5/M, z14.s, z23.s\n" + "fmax z8.s, p5/M, z8.s, z23.s\n" + "fmax z9.s, p5/M, z9.s, z23.s\n" + "fmax z10.s, p5/M, z10.s, z23.s\n" + "fmax z11.s, p5/M, z11.s, z23.s\n" + "fmax z15.s, p5/M, z15.s, z23.s\n" + "fmax z20.s, p5/M, z20.s, z23.s\n" + "fmax z21.s, p5/M, z21.s, z23.s\n" + "fmax z22.s, p5/M, z22.s, z23.s\n" + "fmax z16.s, p5/M, z16.s, z23.s\n" + "fmax z17.s, p5/M, z17.s, z23.s\n" + "fmax z18.s, p5/M, z18.s, z23.s\n" + "fmax z19.s, p5/M, z19.s, z23.s\n" "51:" // Height 4: No activation "st1w { z7.s }, p4, [x9]\n" "st1w { z12.s }, p3, [x9, #1, MUL VL]\n" @@ -1196,54 +1196,54 @@ void sve_hybrid_bf16fp32_mmla_6x4VL ( "55:" // Height 5: no bias "tbz %x[flags], #0, 56f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x9]\n" - "add x23, x24, x20, LSL #2\n" + "add x23, x9, x20, LSL #2\n" "add x22, x23, x20, LSL #2\n" - "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n" + "ld1w { z19.s }, p4/Z, [x9]\n" + "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" + "ld1w { z17.s }, p3/Z, [x9, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "ld1w { z17.s }, p4/Z, [x24]\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" - "ld1w { z18.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z19.s }, p2/Z, [x24, #2, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x23]\n" + "zip1 z8.d, z19.d, z12.d\n" + "zip2 z12.d, z19.d, z12.d\n" + "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n" + "zip1 z9.d, z17.d, z13.d\n" + "zip2 z13.d, z17.d, z13.d\n" + "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z17.s }, p4/Z, [x22]\n" + "zip1 z10.d, z18.d, z14.d\n" + "zip2 z14.d, z18.d, z14.d\n" + "ld1w { z18.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z19.s }, p2/Z, [x22, #2, MUL VL]\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" - "ld1w { z24.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x23]\n" + "ld1w { z24.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x21]\n" "zip1 z16.d, z17.d, z20.d\n" "zip2 z20.d, z17.d, z20.d\n" - "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n" + "ld1w { z21.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x21, #2, MUL VL]\n" "zip1 z17.d, z18.d, z21.d\n" "zip2 z21.d, z18.d, z21.d\n" - "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n" - "ld1w { z25.s }, p4/Z, [x22]\n" + "ld1w { z23.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z25.s }, p4/Z, [x20]\n" "zip1 z18.d, z19.d, z22.d\n" "zip2 z22.d, z19.d, z22.d\n" - "ld1w { z26.s }, p3/Z, [x22, #1, MUL VL]\n" - "ld1w { z27.s }, p2/Z, [x22, #2, MUL VL]\n" + "ld1w { z26.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z27.s }, p2/Z, [x20, #2, MUL VL]\n" "zip1 z19.d, z24.d, z23.d\n" "zip2 z23.d, z24.d, z23.d\n" - "ld1w { z6.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z0.s }, p1/Z, [x20, #3, MUL VL]\n" "zip1 z24.d, z25.d, z28.d\n" "zip2 z28.d, z25.d, z28.d\n" "zip1 z25.d, z26.d, z29.d\n" "zip2 z29.d, z26.d, z29.d\n" "zip1 z26.d, z27.d, z30.d\n" "zip2 z30.d, z27.d, z30.d\n" - "zip1 z27.d, z6.d, z31.d\n" - "zip2 z31.d, z6.d, z31.d\n" + "zip1 z27.d, z0.d, z31.d\n" + "zip2 z31.d, z0.d, z31.d\n" "b 57f\n" "56:" // Height 5: no accumulate "mov z8.b, #0x0\n" @@ -1275,15 +1275,15 @@ void sve_hybrid_bf16fp32_mmla_6x4VL ( "58:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 59f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 60f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -1294,180 +1294,180 @@ void sve_hybrid_bf16fp32_mmla_6x4VL ( "b 60f\n" "59:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" - "add x22, x23, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" + "add x22, x23, x21, LSL #1\n" "60:" // Height 5: input setup done "cmp x27, #0x8\n" "ble 62f\n" "61:" // Height 5: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z1.h }, p0/Z, [x26]\n" - "ld1rqh { z2.h }, p0/Z, [x25]\n" - "ld1rqh { z3.h }, p0/Z, [x24]\n" - "ld1rqh { z4.h }, p0/Z, [x23]\n" - "trn1 z0.d, z1.d, z2.d\n" - "trn2 z1.d, z1.d, z2.d\n" - "ld1rqh { z5.h }, p0/Z, [x22]\n" - "trn1 z2.d, z3.d, z4.d\n" - "trn2 z3.d, z3.d, z4.d\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "trn1 z4.d, z5.d, z6.d\n" - "trn2 z5.d, z5.d, z6.d\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e408 // bfmmla z8.s, z0.h, z7.h\n" - ".inst 0x6467e450 // bfmmla z16.s, z2.h, z7.h\n" - ".inst 0x6467e498 // bfmmla z24.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1rqh { z6.h }, p0/Z, [x26]\n" + "ld1rqh { z1.h }, p0/Z, [x25]\n" + "ld1rqh { z7.h }, p0/Z, [x24]\n" + "ld1rqh { z2.h }, p0/Z, [x23]\n" + "trn1 z5.d, z6.d, z1.d\n" + "trn2 z6.d, z6.d, z1.d\n" + "ld1rqh { z4.h }, p0/Z, [x22]\n" + "trn1 z3.d, z7.d, z2.d\n" + "trn2 z7.d, z7.d, z2.d\n" + "ld1h { z1.h }, p5/Z, [x10]\n" + "trn1 z2.d, z4.d, z0.d\n" + "trn2 z4.d, z4.d, z0.d\n" + "ld1h { z0.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6461e4a8 // bfmmla z8.s, z5.h, z1.h\n" + ".inst 0x6461e470 // bfmmla z16.s, z3.h, z1.h\n" + ".inst 0x6461e458 // bfmmla z24.s, z2.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x10, #2, MUL VL]\n" "sub x27, x27, #0x8\n" - ".inst 0x6466e40c // bfmmla z12.s, z0.h, z6.h\n" - ".inst 0x6466e454 // bfmmla z20.s, z2.h, z6.h\n" + ".inst 0x6460e4ac // bfmmla z12.s, z5.h, z0.h\n" + ".inst 0x6460e474 // bfmmla z20.s, z3.h, z0.h\n" "cmp x27, #0x8\n" "add x26, x26, #0x10\n" - ".inst 0x6466e49c // bfmmla z28.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" + ".inst 0x6460e45c // bfmmla z28.s, z2.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6461e4a9 // bfmmla z9.s, z5.h, z1.h\n" "add x25, x25, #0x10\n" - ".inst 0x6467e451 // bfmmla z17.s, z2.h, z7.h\n" - ".inst 0x6467e499 // bfmmla z25.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x6461e471 // bfmmla z17.s, z3.h, z1.h\n" + ".inst 0x6461e459 // bfmmla z25.s, z2.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x10, #4, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" + ".inst 0x6460e4ad // bfmmla z13.s, z5.h, z0.h\n" + ".inst 0x6460e475 // bfmmla z21.s, z3.h, z0.h\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - ".inst 0x6466e49d // bfmmla z29.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6467e40a // bfmmla z10.s, z0.h, z7.h\n" - ".inst 0x6467e452 // bfmmla z18.s, z2.h, z7.h\n" - ".inst 0x6467e49a // bfmmla z26.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x6466e40e // bfmmla z14.s, z0.h, z6.h\n" - ".inst 0x6466e456 // bfmmla z22.s, z2.h, z6.h\n" - ".inst 0x6466e49e // bfmmla z30.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x6460e45d // bfmmla z29.s, z2.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6461e4aa // bfmmla z10.s, z5.h, z1.h\n" + ".inst 0x6461e472 // bfmmla z18.s, z3.h, z1.h\n" + ".inst 0x6461e45a // bfmmla z26.s, z2.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x6460e4ae // bfmmla z14.s, z5.h, z0.h\n" + ".inst 0x6460e476 // bfmmla z22.s, z3.h, z0.h\n" + ".inst 0x6460e45e // bfmmla z30.s, z2.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" - ".inst 0x6467e453 // bfmmla z19.s, z2.h, z7.h\n" - ".inst 0x6467e49b // bfmmla z27.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #-8, MUL VL]\n" - ".inst 0x6466e40f // bfmmla z15.s, z0.h, z6.h\n" - ".inst 0x6466e457 // bfmmla z23.s, z2.h, z6.h\n" - ".inst 0x6466e49f // bfmmla z31.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #-7, MUL VL]\n" - ".inst 0x6467e428 // bfmmla z8.s, z1.h, z7.h\n" - ".inst 0x6467e470 // bfmmla z16.s, z3.h, z7.h\n" - ".inst 0x6467e4b8 // bfmmla z24.s, z5.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #-6, MUL VL]\n" - ".inst 0x6466e42c // bfmmla z12.s, z1.h, z6.h\n" - ".inst 0x6466e474 // bfmmla z20.s, z3.h, z6.h\n" - ".inst 0x6466e4bc // bfmmla z28.s, z5.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #-5, MUL VL]\n" - ".inst 0x6467e429 // bfmmla z9.s, z1.h, z7.h\n" - ".inst 0x6467e471 // bfmmla z17.s, z3.h, z7.h\n" - ".inst 0x6467e4b9 // bfmmla z25.s, z5.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #-4, MUL VL]\n" - ".inst 0x6466e42d // bfmmla z13.s, z1.h, z6.h\n" - ".inst 0x6466e475 // bfmmla z21.s, z3.h, z6.h\n" - ".inst 0x6466e4bd // bfmmla z29.s, z5.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #-3, MUL VL]\n" - ".inst 0x6467e42a // bfmmla z10.s, z1.h, z7.h\n" - ".inst 0x6467e472 // bfmmla z18.s, z3.h, z7.h\n" - ".inst 0x6467e4ba // bfmmla z26.s, z5.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #-2, MUL VL]\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - ".inst 0x6466e476 // bfmmla z22.s, z3.h, z6.h\n" - ".inst 0x6466e4be // bfmmla z30.s, z5.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #-1, MUL VL]\n" - ".inst 0x6467e42b // bfmmla z11.s, z1.h, z7.h\n" - ".inst 0x6467e473 // bfmmla z19.s, z3.h, z7.h\n" - ".inst 0x6467e4bb // bfmmla z27.s, z5.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" - ".inst 0x6466e477 // bfmmla z23.s, z3.h, z6.h\n" - ".inst 0x6466e4bf // bfmmla z31.s, z5.h, z6.h\n" + ".inst 0x6461e4ab // bfmmla z11.s, z5.h, z1.h\n" + ".inst 0x6461e473 // bfmmla z19.s, z3.h, z1.h\n" + ".inst 0x6461e45b // bfmmla z27.s, z2.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x10, #-8, MUL VL]\n" + ".inst 0x6460e4af // bfmmla z15.s, z5.h, z0.h\n" + ".inst 0x6460e477 // bfmmla z23.s, z3.h, z0.h\n" + ".inst 0x6460e45f // bfmmla z31.s, z2.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #-7, MUL VL]\n" + ".inst 0x6461e4c8 // bfmmla z8.s, z6.h, z1.h\n" + ".inst 0x6461e4f0 // bfmmla z16.s, z7.h, z1.h\n" + ".inst 0x6461e498 // bfmmla z24.s, z4.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x10, #-6, MUL VL]\n" + ".inst 0x6460e4cc // bfmmla z12.s, z6.h, z0.h\n" + ".inst 0x6460e4f4 // bfmmla z20.s, z7.h, z0.h\n" + ".inst 0x6460e49c // bfmmla z28.s, z4.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #-5, MUL VL]\n" + ".inst 0x6461e4c9 // bfmmla z9.s, z6.h, z1.h\n" + ".inst 0x6461e4f1 // bfmmla z17.s, z7.h, z1.h\n" + ".inst 0x6461e499 // bfmmla z25.s, z4.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x10, #-4, MUL VL]\n" + ".inst 0x6460e4cd // bfmmla z13.s, z6.h, z0.h\n" + ".inst 0x6460e4f5 // bfmmla z21.s, z7.h, z0.h\n" + ".inst 0x6460e49d // bfmmla z29.s, z4.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #-3, MUL VL]\n" + ".inst 0x6461e4ca // bfmmla z10.s, z6.h, z1.h\n" + ".inst 0x6461e4f2 // bfmmla z18.s, z7.h, z1.h\n" + ".inst 0x6461e49a // bfmmla z26.s, z4.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x10, #-2, MUL VL]\n" + ".inst 0x6460e4ce // bfmmla z14.s, z6.h, z0.h\n" + ".inst 0x6460e4f6 // bfmmla z22.s, z7.h, z0.h\n" + ".inst 0x6460e49e // bfmmla z30.s, z4.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #-1, MUL VL]\n" + ".inst 0x6461e4cb // bfmmla z11.s, z6.h, z1.h\n" + ".inst 0x6461e4f3 // bfmmla z19.s, z7.h, z1.h\n" + ".inst 0x6461e49b // bfmmla z27.s, z4.h, z1.h\n" + ".inst 0x6460e4cf // bfmmla z15.s, z6.h, z0.h\n" + ".inst 0x6460e4f7 // bfmmla z23.s, z7.h, z0.h\n" + ".inst 0x6460e49f // bfmmla z31.s, z4.h, z0.h\n" "bgt 61b\n" "62:" // Height 5: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" "ld1rqh { z1.h }, p0/Z, [x26]\n" - "ld1rqh { z2.h }, p0/Z, [x25]\n" + "ld1rqh { z4.h }, p0/Z, [x25]\n" "ld1rqh { z3.h }, p0/Z, [x24]\n" - "ld1rqh { z4.h }, p0/Z, [x23]\n" - "trn1 z0.d, z1.d, z2.d\n" - "trn2 z1.d, z1.d, z2.d\n" + "ld1rqh { z2.h }, p0/Z, [x23]\n" + "trn1 z7.d, z1.d, z4.d\n" + "trn2 z1.d, z1.d, z4.d\n" "ld1rqh { z5.h }, p0/Z, [x22]\n" - "trn1 z2.d, z3.d, z4.d\n" - "trn2 z3.d, z3.d, z4.d\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "trn1 z4.d, z5.d, z6.d\n" - "trn2 z5.d, z5.d, z6.d\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e408 // bfmmla z8.s, z0.h, z7.h\n" - ".inst 0x6467e450 // bfmmla z16.s, z2.h, z7.h\n" - ".inst 0x6467e498 // bfmmla z24.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" + "trn1 z6.d, z3.d, z2.d\n" + "trn2 z3.d, z3.d, z2.d\n" + "ld1h { z2.h }, p5/Z, [x10]\n" + "trn1 z4.d, z5.d, z0.d\n" + "trn2 z5.d, z5.d, z0.d\n" + "ld1h { z0.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6462e4e8 // bfmmla z8.s, z7.h, z2.h\n" + ".inst 0x6462e4d0 // bfmmla z16.s, z6.h, z2.h\n" + ".inst 0x6462e498 // bfmmla z24.s, z4.h, z2.h\n" + "ld1h { z2.h }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x4\n" - ".inst 0x6466e40c // bfmmla z12.s, z0.h, z6.h\n" - ".inst 0x6466e454 // bfmmla z20.s, z2.h, z6.h\n" - ".inst 0x6466e49c // bfmmla z28.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" - ".inst 0x6467e451 // bfmmla z17.s, z2.h, z7.h\n" - ".inst 0x6467e499 // bfmmla z25.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" - ".inst 0x6466e49d // bfmmla z29.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6467e40a // bfmmla z10.s, z0.h, z7.h\n" - ".inst 0x6467e452 // bfmmla z18.s, z2.h, z7.h\n" - ".inst 0x6467e49a // bfmmla z26.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x6466e40e // bfmmla z14.s, z0.h, z6.h\n" - ".inst 0x6466e456 // bfmmla z22.s, z2.h, z6.h\n" - ".inst 0x6466e49e // bfmmla z30.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" + ".inst 0x6460e4ec // bfmmla z12.s, z7.h, z0.h\n" + ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n" + ".inst 0x6460e49c // bfmmla z28.s, z4.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6462e4e9 // bfmmla z9.s, z7.h, z2.h\n" + ".inst 0x6462e4d1 // bfmmla z17.s, z6.h, z2.h\n" + ".inst 0x6462e499 // bfmmla z25.s, z4.h, z2.h\n" + "ld1h { z2.h }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x6460e4ed // bfmmla z13.s, z7.h, z0.h\n" + ".inst 0x6460e4d5 // bfmmla z21.s, z6.h, z0.h\n" + ".inst 0x6460e49d // bfmmla z29.s, z4.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6462e4ea // bfmmla z10.s, z7.h, z2.h\n" + ".inst 0x6462e4d2 // bfmmla z18.s, z6.h, z2.h\n" + ".inst 0x6462e49a // bfmmla z26.s, z4.h, z2.h\n" + "ld1h { z2.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x6460e4ee // bfmmla z14.s, z7.h, z0.h\n" + ".inst 0x6460e4d6 // bfmmla z22.s, z6.h, z0.h\n" + ".inst 0x6460e49e // bfmmla z30.s, z4.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x6462e4eb // bfmmla z11.s, z7.h, z2.h\n" "addvl x10, x10, #8\n" - ".inst 0x6467e453 // bfmmla z19.s, z2.h, z7.h\n" - ".inst 0x6467e49b // bfmmla z27.s, z4.h, z7.h\n" - ".inst 0x6466e40f // bfmmla z15.s, z0.h, z6.h\n" - ".inst 0x6466e457 // bfmmla z23.s, z2.h, z6.h\n" - ".inst 0x6466e49f // bfmmla z31.s, z4.h, z6.h\n" + ".inst 0x6462e4d3 // bfmmla z19.s, z6.h, z2.h\n" + ".inst 0x6462e49b // bfmmla z27.s, z4.h, z2.h\n" + ".inst 0x6460e4ef // bfmmla z15.s, z7.h, z0.h\n" + ".inst 0x6460e4d7 // bfmmla z23.s, z6.h, z0.h\n" + ".inst 0x6460e49f // bfmmla z31.s, z4.h, z0.h\n" "ble 63f\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e428 // bfmmla z8.s, z1.h, z7.h\n" - ".inst 0x6467e470 // bfmmla z16.s, z3.h, z7.h\n" - ".inst 0x6467e4b8 // bfmmla z24.s, z5.h, z7.h\n" - ".inst 0x6466e42c // bfmmla z12.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x6466e474 // bfmmla z20.s, z3.h, z6.h\n" - ".inst 0x6466e4bc // bfmmla z28.s, z5.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6467e429 // bfmmla z9.s, z1.h, z7.h\n" - ".inst 0x6467e471 // bfmmla z17.s, z3.h, z7.h\n" - ".inst 0x6467e4b9 // bfmmla z25.s, z5.h, z7.h\n" - ".inst 0x6466e42d // bfmmla z13.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x6466e475 // bfmmla z21.s, z3.h, z6.h\n" - ".inst 0x6466e4bd // bfmmla z29.s, z5.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6467e42a // bfmmla z10.s, z1.h, z7.h\n" - ".inst 0x6467e472 // bfmmla z18.s, z3.h, z7.h\n" - ".inst 0x6467e4ba // bfmmla z26.s, z5.h, z7.h\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x6466e476 // bfmmla z22.s, z3.h, z6.h\n" - ".inst 0x6466e4be // bfmmla z30.s, z5.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n" + "ld1h { z2.h }, p5/Z, [x10]\n" + "ld1h { z0.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6462e428 // bfmmla z8.s, z1.h, z2.h\n" + ".inst 0x6462e470 // bfmmla z16.s, z3.h, z2.h\n" + ".inst 0x6462e4b8 // bfmmla z24.s, z5.h, z2.h\n" + ".inst 0x6460e42c // bfmmla z12.s, z1.h, z0.h\n" + "ld1h { z2.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x6460e474 // bfmmla z20.s, z3.h, z0.h\n" + ".inst 0x6460e4bc // bfmmla z28.s, z5.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6462e429 // bfmmla z9.s, z1.h, z2.h\n" + ".inst 0x6462e471 // bfmmla z17.s, z3.h, z2.h\n" + ".inst 0x6462e4b9 // bfmmla z25.s, z5.h, z2.h\n" + ".inst 0x6460e42d // bfmmla z13.s, z1.h, z0.h\n" + "ld1h { z2.h }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x6460e475 // bfmmla z21.s, z3.h, z0.h\n" + ".inst 0x6460e4bd // bfmmla z29.s, z5.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6462e42a // bfmmla z10.s, z1.h, z2.h\n" + ".inst 0x6462e472 // bfmmla z18.s, z3.h, z2.h\n" + ".inst 0x6462e4ba // bfmmla z26.s, z5.h, z2.h\n" + ".inst 0x6460e42e // bfmmla z14.s, z1.h, z0.h\n" + "ld1h { z2.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x6460e476 // bfmmla z22.s, z3.h, z0.h\n" + ".inst 0x6460e4be // bfmmla z30.s, z5.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - ".inst 0x6467e42b // bfmmla z11.s, z1.h, z7.h\n" - ".inst 0x6467e473 // bfmmla z19.s, z3.h, z7.h\n" - ".inst 0x6467e4bb // bfmmla z27.s, z5.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" - ".inst 0x6466e477 // bfmmla z23.s, z3.h, z6.h\n" - ".inst 0x6466e4bf // bfmmla z31.s, z5.h, z6.h\n" + ".inst 0x6462e42b // bfmmla z11.s, z1.h, z2.h\n" + ".inst 0x6462e473 // bfmmla z19.s, z3.h, z2.h\n" + ".inst 0x6462e4bb // bfmmla z27.s, z5.h, z2.h\n" + ".inst 0x6460e42f // bfmmla z15.s, z1.h, z0.h\n" + ".inst 0x6460e477 // bfmmla z23.s, z3.h, z0.h\n" + ".inst 0x6460e4bf // bfmmla z31.s, z5.h, z0.h\n" "63:" // Height 5: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1500,49 +1500,49 @@ void sve_hybrid_bf16fp32_mmla_6x4VL ( "uzp1 z27.d, z27.d, z31.d\n" "tbz %x[flags], #1, 64f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" - "add x20, %x[args_ptr], %[offset_min]\n" "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z7.s, p5/M, z7.s, z1.s\n" - "fmin z12.s, p5/M, z12.s, z1.s\n" - "fmin z13.s, p5/M, z13.s, z1.s\n" - "fmin z14.s, p5/M, z14.s, z1.s\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmin z15.s, p5/M, z15.s, z1.s\n" - "fmin z20.s, p5/M, z20.s, z1.s\n" - "fmin z21.s, p5/M, z21.s, z1.s\n" - "fmin z22.s, p5/M, z22.s, z1.s\n" - "fmin z16.s, p5/M, z16.s, z1.s\n" - "fmin z17.s, p5/M, z17.s, z1.s\n" - "fmin z18.s, p5/M, z18.s, z1.s\n" - "fmin z19.s, p5/M, z19.s, z1.s\n" - "fmin z24.s, p5/M, z24.s, z1.s\n" - "fmin z25.s, p5/M, z25.s, z1.s\n" - "fmin z26.s, p5/M, z26.s, z1.s\n" - "fmin z27.s, p5/M, z27.s, z1.s\n" - "fmax z7.s, p5/M, z7.s, z0.s\n" - "fmax z12.s, p5/M, z12.s, z0.s\n" - "fmax z13.s, p5/M, z13.s, z0.s\n" - "fmax z14.s, p5/M, z14.s, z0.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" - "fmax z15.s, p5/M, z15.s, z0.s\n" - "fmax z20.s, p5/M, z20.s, z0.s\n" - "fmax z21.s, p5/M, z21.s, z0.s\n" - "fmax z22.s, p5/M, z22.s, z0.s\n" - "fmax z16.s, p5/M, z16.s, z0.s\n" - "fmax z17.s, p5/M, z17.s, z0.s\n" - "fmax z18.s, p5/M, z18.s, z0.s\n" - "fmax z19.s, p5/M, z19.s, z0.s\n" - "fmax z24.s, p5/M, z24.s, z0.s\n" - "fmax z25.s, p5/M, z25.s, z0.s\n" - "fmax z26.s, p5/M, z26.s, z0.s\n" - "fmax z27.s, p5/M, z27.s, z0.s\n" + "add x20, %x[args_ptr], %[offset_min]\n" + "ld1rw { z23.s }, p5/Z, [x20]\n" + "fmin z7.s, p5/M, z7.s, z0.s\n" + "fmin z12.s, p5/M, z12.s, z0.s\n" + "fmin z13.s, p5/M, z13.s, z0.s\n" + "fmin z14.s, p5/M, z14.s, z0.s\n" + "fmin z8.s, p5/M, z8.s, z0.s\n" + "fmin z9.s, p5/M, z9.s, z0.s\n" + "fmin z10.s, p5/M, z10.s, z0.s\n" + "fmin z11.s, p5/M, z11.s, z0.s\n" + "fmin z15.s, p5/M, z15.s, z0.s\n" + "fmin z20.s, p5/M, z20.s, z0.s\n" + "fmin z21.s, p5/M, z21.s, z0.s\n" + "fmin z22.s, p5/M, z22.s, z0.s\n" + "fmin z16.s, p5/M, z16.s, z0.s\n" + "fmin z17.s, p5/M, z17.s, z0.s\n" + "fmin z18.s, p5/M, z18.s, z0.s\n" + "fmin z19.s, p5/M, z19.s, z0.s\n" + "fmin z24.s, p5/M, z24.s, z0.s\n" + "fmin z25.s, p5/M, z25.s, z0.s\n" + "fmin z26.s, p5/M, z26.s, z0.s\n" + "fmin z27.s, p5/M, z27.s, z0.s\n" + "fmax z7.s, p5/M, z7.s, z23.s\n" + "fmax z12.s, p5/M, z12.s, z23.s\n" + "fmax z13.s, p5/M, z13.s, z23.s\n" + "fmax z14.s, p5/M, z14.s, z23.s\n" + "fmax z8.s, p5/M, z8.s, z23.s\n" + "fmax z9.s, p5/M, z9.s, z23.s\n" + "fmax z10.s, p5/M, z10.s, z23.s\n" + "fmax z11.s, p5/M, z11.s, z23.s\n" + "fmax z15.s, p5/M, z15.s, z23.s\n" + "fmax z20.s, p5/M, z20.s, z23.s\n" + "fmax z21.s, p5/M, z21.s, z23.s\n" + "fmax z22.s, p5/M, z22.s, z23.s\n" + "fmax z16.s, p5/M, z16.s, z23.s\n" + "fmax z17.s, p5/M, z17.s, z23.s\n" + "fmax z18.s, p5/M, z18.s, z23.s\n" + "fmax z19.s, p5/M, z19.s, z23.s\n" + "fmax z24.s, p5/M, z24.s, z23.s\n" + "fmax z25.s, p5/M, z25.s, z23.s\n" + "fmax z26.s, p5/M, z26.s, z23.s\n" + "fmax z27.s, p5/M, z27.s, z23.s\n" "64:" // Height 5: No activation "st1w { z7.s }, p4, [x9]\n" "st1w { z12.s }, p3, [x9, #1, MUL VL]\n" @@ -1621,59 +1621,59 @@ void sve_hybrid_bf16fp32_mmla_6x4VL ( "68:" // Height 6: no bias "tbz %x[flags], #0, 69f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x9]\n" + "add x24, x9, x20, LSL #2\n" "add x23, x24, x20, LSL #2\n" + "ld1w { z17.s }, p4/Z, [x9]\n" "add x22, x23, x20, LSL #2\n" - "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n" "add x21, x22, x20, LSL #2\n" + "ld1w { z18.s }, p3/Z, [x9, #1, MUL VL]\n" + "ld1w { z20.s }, p2/Z, [x9, #2, MUL VL]\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "zip1 z8.d, z9.d, z12.d\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "zip2 z12.d, z9.d, z12.d\n" - "zip1 z9.d, z10.d, z13.d\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "ld1w { z17.s }, p4/Z, [x24]\n" - "zip2 z13.d, z10.d, z13.d\n" - "zip1 z10.d, z11.d, z14.d\n" - "ld1w { z18.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z19.s }, p2/Z, [x24, #2, MUL VL]\n" - "zip2 z14.d, z11.d, z14.d\n" + "ld1w { z12.s }, p4/Z, [x24]\n" + "zip1 z8.d, z17.d, z12.d\n" + "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n" + "zip2 z12.d, z17.d, z12.d\n" + "zip1 z9.d, z18.d, z13.d\n" + "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n" + "ld1w { z17.s }, p4/Z, [x23]\n" + "zip2 z13.d, z18.d, z13.d\n" + "zip1 z10.d, z20.d, z14.d\n" + "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n" + "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n" + "zip2 z14.d, z20.d, z14.d\n" "zip1 z11.d, z16.d, z15.d\n" - "ld1w { z24.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x23]\n" + "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x22]\n" "zip2 z15.d, z16.d, z15.d\n" "zip1 z16.d, z17.d, z20.d\n" - "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n" + "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n" "zip2 z20.d, z17.d, z20.d\n" "zip1 z17.d, z18.d, z21.d\n" - "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n" - "ld1w { z25.s }, p4/Z, [x22]\n" + "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z25.s }, p4/Z, [x21]\n" "zip2 z21.d, z18.d, z21.d\n" "zip1 z18.d, z19.d, z22.d\n" - "ld1w { z26.s }, p3/Z, [x22, #1, MUL VL]\n" - "ld1w { z27.s }, p2/Z, [x22, #2, MUL VL]\n" + "ld1w { z26.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z27.s }, p2/Z, [x21, #2, MUL VL]\n" "zip2 z22.d, z19.d, z22.d\n" "zip1 z19.d, z24.d, z23.d\n" - "ld1w { z6.s }, p1/Z, [x22, #3, MUL VL]\n" - "ld1w { z28.s }, p4/Z, [x21]\n" + "ld1w { z0.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z28.s }, p4/Z, [x20]\n" "zip2 z23.d, z24.d, z23.d\n" "zip1 z24.d, z25.d, z28.d\n" - "ld1w { z29.s }, p3/Z, [x21, #1, MUL VL]\n" - "ld1w { z30.s }, p2/Z, [x21, #2, MUL VL]\n" + "ld1w { z29.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z30.s }, p2/Z, [x20, #2, MUL VL]\n" "zip2 z28.d, z25.d, z28.d\n" "zip1 z25.d, z26.d, z29.d\n" - "ld1w { z31.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z31.s }, p1/Z, [x20, #3, MUL VL]\n" "zip2 z29.d, z26.d, z29.d\n" "zip1 z26.d, z27.d, z30.d\n" "zip2 z30.d, z27.d, z30.d\n" - "zip1 z27.d, z6.d, z31.d\n" - "zip2 z31.d, z6.d, z31.d\n" + "zip1 z27.d, z0.d, z31.d\n" + "zip2 z31.d, z0.d, z31.d\n" "b 70f\n" "69:" // Height 6: no accumulate "mov z8.b, #0x0\n" @@ -1705,16 +1705,16 @@ void sve_hybrid_bf16fp32_mmla_6x4VL ( "71:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 72f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 73f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -1726,184 +1726,184 @@ void sve_hybrid_bf16fp32_mmla_6x4VL ( "b 73f\n" "72:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" - "add x22, x23, x20, LSL #1\n" - "add x21, x22, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" + "add x22, x23, x21, LSL #1\n" + "add x21, x22, x21, LSL #1\n" "73:" // Height 6: input setup done "cmp x27, #0x8\n" "ble 75f\n" "74:" // Height 6: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z1.h }, p0/Z, [x26]\n" - "ld1rqh { z2.h }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1rqh { z3.h }, p0/Z, [x24]\n" - "ld1rqh { z4.h }, p0/Z, [x23]\n" - "trn2 z1.d, z1.d, z2.d\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1rqh { z5.h }, p0/Z, [x22]\n" - "ld1rqh { z6.h }, p0/Z, [x21]\n" - "trn2 z3.d, z3.d, z4.d\n" - "trn1 z4.d, z5.d, z6.d\n" - "trn2 z5.d, z5.d, z6.d\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e408 // bfmmla z8.s, z0.h, z7.h\n" - ".inst 0x6467e450 // bfmmla z16.s, z2.h, z7.h\n" - ".inst 0x6467e498 // bfmmla z24.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1rqh { z7.h }, p0/Z, [x26]\n" + "ld1rqh { z0.h }, p0/Z, [x25]\n" + "trn1 z6.d, z7.d, z0.d\n" + "ld1rqh { z5.h }, p0/Z, [x24]\n" + "ld1rqh { z1.h }, p0/Z, [x23]\n" + "trn2 z7.d, z7.d, z0.d\n" + "trn1 z4.d, z5.d, z1.d\n" + "ld1rqh { z3.h }, p0/Z, [x22]\n" + "ld1rqh { z0.h }, p0/Z, [x21]\n" + "trn2 z5.d, z5.d, z1.d\n" + "trn1 z2.d, z3.d, z0.d\n" + "trn2 z3.d, z3.d, z0.d\n" + "ld1h { z1.h }, p5/Z, [x10]\n" + "ld1h { z0.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6461e4c8 // bfmmla z8.s, z6.h, z1.h\n" + ".inst 0x6461e490 // bfmmla z16.s, z4.h, z1.h\n" + ".inst 0x6461e458 // bfmmla z24.s, z2.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x10, #2, MUL VL]\n" "sub x27, x27, #0x8\n" - ".inst 0x6466e40c // bfmmla z12.s, z0.h, z6.h\n" - ".inst 0x6466e454 // bfmmla z20.s, z2.h, z6.h\n" + ".inst 0x6460e4cc // bfmmla z12.s, z6.h, z0.h\n" + ".inst 0x6460e494 // bfmmla z20.s, z4.h, z0.h\n" "cmp x27, #0x8\n" "add x26, x26, #0x10\n" - ".inst 0x6466e49c // bfmmla z28.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" + ".inst 0x6460e45c // bfmmla z28.s, z2.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6461e4c9 // bfmmla z9.s, z6.h, z1.h\n" "add x25, x25, #0x10\n" - ".inst 0x6467e451 // bfmmla z17.s, z2.h, z7.h\n" - ".inst 0x6467e499 // bfmmla z25.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x6461e491 // bfmmla z17.s, z4.h, z1.h\n" + ".inst 0x6461e459 // bfmmla z25.s, z2.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x10, #4, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" + ".inst 0x6460e4cd // bfmmla z13.s, z6.h, z0.h\n" + ".inst 0x6460e495 // bfmmla z21.s, z4.h, z0.h\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - ".inst 0x6466e49d // bfmmla z29.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6467e40a // bfmmla z10.s, z0.h, z7.h\n" + ".inst 0x6460e45d // bfmmla z29.s, z2.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6461e4ca // bfmmla z10.s, z6.h, z1.h\n" "add x21, x21, #0x10\n" - ".inst 0x6467e452 // bfmmla z18.s, z2.h, z7.h\n" - ".inst 0x6467e49a // bfmmla z26.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x6466e40e // bfmmla z14.s, z0.h, z6.h\n" - ".inst 0x6466e456 // bfmmla z22.s, z2.h, z6.h\n" - ".inst 0x6466e49e // bfmmla z30.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x6461e492 // bfmmla z18.s, z4.h, z1.h\n" + ".inst 0x6461e45a // bfmmla z26.s, z2.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x6460e4ce // bfmmla z14.s, z6.h, z0.h\n" + ".inst 0x6460e496 // bfmmla z22.s, z4.h, z0.h\n" + ".inst 0x6460e45e // bfmmla z30.s, z2.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" - ".inst 0x6467e453 // bfmmla z19.s, z2.h, z7.h\n" - ".inst 0x6467e49b // bfmmla z27.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #-8, MUL VL]\n" - ".inst 0x6466e40f // bfmmla z15.s, z0.h, z6.h\n" - ".inst 0x6466e457 // bfmmla z23.s, z2.h, z6.h\n" - ".inst 0x6466e49f // bfmmla z31.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #-7, MUL VL]\n" - ".inst 0x6467e428 // bfmmla z8.s, z1.h, z7.h\n" - ".inst 0x6467e470 // bfmmla z16.s, z3.h, z7.h\n" - ".inst 0x6467e4b8 // bfmmla z24.s, z5.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #-6, MUL VL]\n" - ".inst 0x6466e42c // bfmmla z12.s, z1.h, z6.h\n" - ".inst 0x6466e474 // bfmmla z20.s, z3.h, z6.h\n" - ".inst 0x6466e4bc // bfmmla z28.s, z5.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #-5, MUL VL]\n" - ".inst 0x6467e429 // bfmmla z9.s, z1.h, z7.h\n" - ".inst 0x6467e471 // bfmmla z17.s, z3.h, z7.h\n" - ".inst 0x6467e4b9 // bfmmla z25.s, z5.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #-4, MUL VL]\n" - ".inst 0x6466e42d // bfmmla z13.s, z1.h, z6.h\n" - ".inst 0x6466e475 // bfmmla z21.s, z3.h, z6.h\n" - ".inst 0x6466e4bd // bfmmla z29.s, z5.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #-3, MUL VL]\n" - ".inst 0x6467e42a // bfmmla z10.s, z1.h, z7.h\n" - ".inst 0x6467e472 // bfmmla z18.s, z3.h, z7.h\n" - ".inst 0x6467e4ba // bfmmla z26.s, z5.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #-2, MUL VL]\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - ".inst 0x6466e476 // bfmmla z22.s, z3.h, z6.h\n" - ".inst 0x6466e4be // bfmmla z30.s, z5.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #-1, MUL VL]\n" - ".inst 0x6467e42b // bfmmla z11.s, z1.h, z7.h\n" - ".inst 0x6467e473 // bfmmla z19.s, z3.h, z7.h\n" - ".inst 0x6467e4bb // bfmmla z27.s, z5.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" - ".inst 0x6466e477 // bfmmla z23.s, z3.h, z6.h\n" - ".inst 0x6466e4bf // bfmmla z31.s, z5.h, z6.h\n" + ".inst 0x6461e4cb // bfmmla z11.s, z6.h, z1.h\n" + ".inst 0x6461e493 // bfmmla z19.s, z4.h, z1.h\n" + ".inst 0x6461e45b // bfmmla z27.s, z2.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x10, #-8, MUL VL]\n" + ".inst 0x6460e4cf // bfmmla z15.s, z6.h, z0.h\n" + ".inst 0x6460e497 // bfmmla z23.s, z4.h, z0.h\n" + ".inst 0x6460e45f // bfmmla z31.s, z2.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #-7, MUL VL]\n" + ".inst 0x6461e4e8 // bfmmla z8.s, z7.h, z1.h\n" + ".inst 0x6461e4b0 // bfmmla z16.s, z5.h, z1.h\n" + ".inst 0x6461e478 // bfmmla z24.s, z3.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x10, #-6, MUL VL]\n" + ".inst 0x6460e4ec // bfmmla z12.s, z7.h, z0.h\n" + ".inst 0x6460e4b4 // bfmmla z20.s, z5.h, z0.h\n" + ".inst 0x6460e47c // bfmmla z28.s, z3.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #-5, MUL VL]\n" + ".inst 0x6461e4e9 // bfmmla z9.s, z7.h, z1.h\n" + ".inst 0x6461e4b1 // bfmmla z17.s, z5.h, z1.h\n" + ".inst 0x6461e479 // bfmmla z25.s, z3.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x10, #-4, MUL VL]\n" + ".inst 0x6460e4ed // bfmmla z13.s, z7.h, z0.h\n" + ".inst 0x6460e4b5 // bfmmla z21.s, z5.h, z0.h\n" + ".inst 0x6460e47d // bfmmla z29.s, z3.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #-3, MUL VL]\n" + ".inst 0x6461e4ea // bfmmla z10.s, z7.h, z1.h\n" + ".inst 0x6461e4b2 // bfmmla z18.s, z5.h, z1.h\n" + ".inst 0x6461e47a // bfmmla z26.s, z3.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x10, #-2, MUL VL]\n" + ".inst 0x6460e4ee // bfmmla z14.s, z7.h, z0.h\n" + ".inst 0x6460e4b6 // bfmmla z22.s, z5.h, z0.h\n" + ".inst 0x6460e47e // bfmmla z30.s, z3.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #-1, MUL VL]\n" + ".inst 0x6461e4eb // bfmmla z11.s, z7.h, z1.h\n" + ".inst 0x6461e4b3 // bfmmla z19.s, z5.h, z1.h\n" + ".inst 0x6461e47b // bfmmla z27.s, z3.h, z1.h\n" + ".inst 0x6460e4ef // bfmmla z15.s, z7.h, z0.h\n" + ".inst 0x6460e4b7 // bfmmla z23.s, z5.h, z0.h\n" + ".inst 0x6460e47f // bfmmla z31.s, z3.h, z0.h\n" "bgt 74b\n" "75:" // Height 6: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" "ld1rqh { z1.h }, p0/Z, [x26]\n" - "ld1rqh { z2.h }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" + "ld1rqh { z0.h }, p0/Z, [x25]\n" + "trn1 z7.d, z1.d, z0.d\n" "ld1rqh { z3.h }, p0/Z, [x24]\n" - "ld1rqh { z4.h }, p0/Z, [x23]\n" - "trn2 z1.d, z1.d, z2.d\n" - "trn1 z2.d, z3.d, z4.d\n" + "ld1rqh { z2.h }, p0/Z, [x23]\n" + "trn2 z1.d, z1.d, z0.d\n" + "trn1 z6.d, z3.d, z2.d\n" "ld1rqh { z5.h }, p0/Z, [x22]\n" - "ld1rqh { z6.h }, p0/Z, [x21]\n" - "trn2 z3.d, z3.d, z4.d\n" - "trn1 z4.d, z5.d, z6.d\n" - "trn2 z5.d, z5.d, z6.d\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e408 // bfmmla z8.s, z0.h, z7.h\n" - ".inst 0x6467e450 // bfmmla z16.s, z2.h, z7.h\n" - ".inst 0x6467e498 // bfmmla z24.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1rqh { z0.h }, p0/Z, [x21]\n" + "trn2 z3.d, z3.d, z2.d\n" + "trn1 z4.d, z5.d, z0.d\n" + "trn2 z5.d, z5.d, z0.d\n" + "ld1h { z2.h }, p5/Z, [x10]\n" + "ld1h { z0.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6462e4e8 // bfmmla z8.s, z7.h, z2.h\n" + ".inst 0x6462e4d0 // bfmmla z16.s, z6.h, z2.h\n" + ".inst 0x6462e498 // bfmmla z24.s, z4.h, z2.h\n" + "ld1h { z2.h }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x4\n" - ".inst 0x6466e40c // bfmmla z12.s, z0.h, z6.h\n" - ".inst 0x6466e454 // bfmmla z20.s, z2.h, z6.h\n" - ".inst 0x6466e49c // bfmmla z28.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" - ".inst 0x6467e451 // bfmmla z17.s, z2.h, z7.h\n" - ".inst 0x6467e499 // bfmmla z25.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" - ".inst 0x6466e49d // bfmmla z29.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6467e40a // bfmmla z10.s, z0.h, z7.h\n" - ".inst 0x6467e452 // bfmmla z18.s, z2.h, z7.h\n" - ".inst 0x6467e49a // bfmmla z26.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x6466e40e // bfmmla z14.s, z0.h, z6.h\n" - ".inst 0x6466e456 // bfmmla z22.s, z2.h, z6.h\n" - ".inst 0x6466e49e // bfmmla z30.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" + ".inst 0x6460e4ec // bfmmla z12.s, z7.h, z0.h\n" + ".inst 0x6460e4d4 // bfmmla z20.s, z6.h, z0.h\n" + ".inst 0x6460e49c // bfmmla z28.s, z4.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6462e4e9 // bfmmla z9.s, z7.h, z2.h\n" + ".inst 0x6462e4d1 // bfmmla z17.s, z6.h, z2.h\n" + ".inst 0x6462e499 // bfmmla z25.s, z4.h, z2.h\n" + "ld1h { z2.h }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x6460e4ed // bfmmla z13.s, z7.h, z0.h\n" + ".inst 0x6460e4d5 // bfmmla z21.s, z6.h, z0.h\n" + ".inst 0x6460e49d // bfmmla z29.s, z4.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6462e4ea // bfmmla z10.s, z7.h, z2.h\n" + ".inst 0x6462e4d2 // bfmmla z18.s, z6.h, z2.h\n" + ".inst 0x6462e49a // bfmmla z26.s, z4.h, z2.h\n" + "ld1h { z2.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x6460e4ee // bfmmla z14.s, z7.h, z0.h\n" + ".inst 0x6460e4d6 // bfmmla z22.s, z6.h, z0.h\n" + ".inst 0x6460e49e // bfmmla z30.s, z4.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x6462e4eb // bfmmla z11.s, z7.h, z2.h\n" "addvl x10, x10, #8\n" - ".inst 0x6467e453 // bfmmla z19.s, z2.h, z7.h\n" - ".inst 0x6467e49b // bfmmla z27.s, z4.h, z7.h\n" - ".inst 0x6466e40f // bfmmla z15.s, z0.h, z6.h\n" - ".inst 0x6466e457 // bfmmla z23.s, z2.h, z6.h\n" - ".inst 0x6466e49f // bfmmla z31.s, z4.h, z6.h\n" + ".inst 0x6462e4d3 // bfmmla z19.s, z6.h, z2.h\n" + ".inst 0x6462e49b // bfmmla z27.s, z4.h, z2.h\n" + ".inst 0x6460e4ef // bfmmla z15.s, z7.h, z0.h\n" + ".inst 0x6460e4d7 // bfmmla z23.s, z6.h, z0.h\n" + ".inst 0x6460e49f // bfmmla z31.s, z4.h, z0.h\n" "ble 76f\n" - "ld1h { z7.h }, p5/Z, [x10]\n" - "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6467e428 // bfmmla z8.s, z1.h, z7.h\n" - ".inst 0x6467e470 // bfmmla z16.s, z3.h, z7.h\n" - ".inst 0x6467e4b8 // bfmmla z24.s, z5.h, z7.h\n" - ".inst 0x6466e42c // bfmmla z12.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x6466e474 // bfmmla z20.s, z3.h, z6.h\n" - ".inst 0x6466e4bc // bfmmla z28.s, z5.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6467e429 // bfmmla z9.s, z1.h, z7.h\n" - ".inst 0x6467e471 // bfmmla z17.s, z3.h, z7.h\n" - ".inst 0x6467e4b9 // bfmmla z25.s, z5.h, z7.h\n" - ".inst 0x6466e42d // bfmmla z13.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x6466e475 // bfmmla z21.s, z3.h, z6.h\n" - ".inst 0x6466e4bd // bfmmla z29.s, z5.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6467e42a // bfmmla z10.s, z1.h, z7.h\n" - ".inst 0x6467e472 // bfmmla z18.s, z3.h, z7.h\n" - ".inst 0x6467e4ba // bfmmla z26.s, z5.h, z7.h\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - "ld1h { z7.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x6466e476 // bfmmla z22.s, z3.h, z6.h\n" - ".inst 0x6466e4be // bfmmla z30.s, z5.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #7, MUL VL]\n" + "ld1h { z2.h }, p5/Z, [x10]\n" + "ld1h { z0.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6462e428 // bfmmla z8.s, z1.h, z2.h\n" + ".inst 0x6462e470 // bfmmla z16.s, z3.h, z2.h\n" + ".inst 0x6462e4b8 // bfmmla z24.s, z5.h, z2.h\n" + ".inst 0x6460e42c // bfmmla z12.s, z1.h, z0.h\n" + "ld1h { z2.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x6460e474 // bfmmla z20.s, z3.h, z0.h\n" + ".inst 0x6460e4bc // bfmmla z28.s, z5.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6462e429 // bfmmla z9.s, z1.h, z2.h\n" + ".inst 0x6462e471 // bfmmla z17.s, z3.h, z2.h\n" + ".inst 0x6462e4b9 // bfmmla z25.s, z5.h, z2.h\n" + ".inst 0x6460e42d // bfmmla z13.s, z1.h, z0.h\n" + "ld1h { z2.h }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x6460e475 // bfmmla z21.s, z3.h, z0.h\n" + ".inst 0x6460e4bd // bfmmla z29.s, z5.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6462e42a // bfmmla z10.s, z1.h, z2.h\n" + ".inst 0x6462e472 // bfmmla z18.s, z3.h, z2.h\n" + ".inst 0x6462e4ba // bfmmla z26.s, z5.h, z2.h\n" + ".inst 0x6460e42e // bfmmla z14.s, z1.h, z0.h\n" + "ld1h { z2.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x6460e476 // bfmmla z22.s, z3.h, z0.h\n" + ".inst 0x6460e4be // bfmmla z30.s, z5.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - ".inst 0x6467e42b // bfmmla z11.s, z1.h, z7.h\n" - ".inst 0x6467e473 // bfmmla z19.s, z3.h, z7.h\n" - ".inst 0x6467e4bb // bfmmla z27.s, z5.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" - ".inst 0x6466e477 // bfmmla z23.s, z3.h, z6.h\n" - ".inst 0x6466e4bf // bfmmla z31.s, z5.h, z6.h\n" + ".inst 0x6462e42b // bfmmla z11.s, z1.h, z2.h\n" + ".inst 0x6462e473 // bfmmla z19.s, z3.h, z2.h\n" + ".inst 0x6462e4bb // bfmmla z27.s, z5.h, z2.h\n" + ".inst 0x6460e42f // bfmmla z15.s, z1.h, z0.h\n" + ".inst 0x6460e477 // bfmmla z23.s, z3.h, z0.h\n" + ".inst 0x6460e4bf // bfmmla z31.s, z5.h, z0.h\n" "76:" // Height 6: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -2041,4 +2041,4 @@ void sve_hybrid_bf16fp32_mmla_6x4VL ( } } // namespace arm_gemm -#endif // __ARM_FEATURE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL.hpp index 6db9c0cdf354f63ac01ad129b0cf4a374fee3674..b930e4c0d53413ec580882dbd915b5eb994b4d70 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,16 +10,16 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once #ifdef ARM_COMPUTE_ENABLE_SVE @@ -75,7 +75,6 @@ public: template static inline PerformanceParameters get_performance_parameters(const CPUInfo *ci) { - if (std::is_same::value) { switch (ci->get_cpu_model()) { default: @@ -84,6 +83,8 @@ public: return { 12.44 }; case CPUModel::V1: return { 31.51 }; + case CPUModel::A64FX: + return { 49.14 }; } } @@ -107,5 +108,4 @@ public: } // namespace arm_gemm #undef ARGLIST - -#endif // __aarch64__ +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL/a64fx.cpp index a70e66cbe4c87a6c7858003e9b2b9d356eed4190..d1a9bb4a2648727f645cd431fd5448f08c2c32c0 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL/a64fx.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL/a64fx.cpp @@ -139,11 +139,11 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "6:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 7f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 8f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -159,12 +159,12 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "9:" // Height 1: Multiply loop: Main loop "fmla z8.h, p4/M, z6.h, z0.h\n" "fmla z9.h, p4/M, z7.h, z0.h\n" - "ld1h { z6.h }, p4/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p4/Z, [x10, #3, MUL VL]\n" + "ld1h { z17.h }, p4/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" "add x26, x26, #0x2\n" - "fmla z10.h, p4/M, z6.h, z0.h\n" - "fmla z11.h, p4/M, z7.h, z0.h\n" + "fmla z10.h, p4/M, z17.h, z0.h\n" + "fmla z11.h, p4/M, z16.h, z0.h\n" "subs x27, x27, #0x1\n" "ld1rh { z0.h }, p4/Z, [x26]\n" "ld1h { z6.h }, p4/Z, [x10]\n" @@ -174,27 +174,27 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "fmla z8.h, p4/M, z6.h, z0.h\n" "fmla z9.h, p4/M, z7.h, z0.h\n" - "ld1h { z6.h }, p4/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p4/Z, [x10, #3, MUL VL]\n" + "ld1h { z17.h }, p4/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p4/Z, [x10, #3, MUL VL]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" - "fmla z10.h, p4/M, z6.h, z0.h\n" - "fmla z11.h, p4/M, z7.h, z0.h\n" + "fmla z10.h, p4/M, z17.h, z0.h\n" + "fmla z11.h, p4/M, z16.h, z0.h\n" "addvl x10, x10, #4\n" "bne 6b\n" "tbz %x[flags], #1, 11f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rh { z1.h }, p4/Z, [x20]\n" + "ld1rh { z17.h }, p4/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rh { z0.h }, p4/Z, [x20]\n" - "fmin z8.h, p4/M, z8.h, z1.h\n" - "fmin z9.h, p4/M, z9.h, z1.h\n" - "fmin z10.h, p4/M, z10.h, z1.h\n" - "fmin z11.h, p4/M, z11.h, z1.h\n" - "fmax z8.h, p4/M, z8.h, z0.h\n" - "fmax z9.h, p4/M, z9.h, z0.h\n" - "fmax z10.h, p4/M, z10.h, z0.h\n" - "fmax z11.h, p4/M, z11.h, z0.h\n" + "ld1rh { z16.h }, p4/Z, [x20]\n" + "fmin z8.h, p4/M, z8.h, z17.h\n" + "fmin z9.h, p4/M, z9.h, z17.h\n" + "fmin z10.h, p4/M, z10.h, z17.h\n" + "fmin z11.h, p4/M, z11.h, z17.h\n" + "fmax z8.h, p4/M, z8.h, z16.h\n" + "fmax z9.h, p4/M, z9.h, z16.h\n" + "fmax z10.h, p4/M, z10.h, z16.h\n" + "fmax z11.h, p4/M, z11.h, z16.h\n" "11:" // Height 1: No activation "st1h { z8.h }, p3, [x9]\n" "st1h { z9.h }, p2, [x9, #1, MUL VL]\n" @@ -234,15 +234,15 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "15:" // Height 2: no bias "tbz %x[flags], #0, 16f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #1\n" + "add x20, x9, x20, LSL #1\n" "ld1h { z8.h }, p3/Z, [x9]\n" "ld1h { z9.h }, p2/Z, [x9, #1, MUL VL]\n" "ld1h { z10.h }, p1/Z, [x9, #2, MUL VL]\n" "ld1h { z11.h }, p0/Z, [x9, #3, MUL VL]\n" - "ld1h { z12.h }, p3/Z, [x25]\n" - "ld1h { z13.h }, p2/Z, [x25, #1, MUL VL]\n" - "ld1h { z14.h }, p1/Z, [x25, #2, MUL VL]\n" - "ld1h { z15.h }, p0/Z, [x25, #3, MUL VL]\n" + "ld1h { z12.h }, p3/Z, [x20]\n" + "ld1h { z13.h }, p2/Z, [x20, #1, MUL VL]\n" + "ld1h { z14.h }, p1/Z, [x20, #2, MUL VL]\n" + "ld1h { z15.h }, p0/Z, [x20, #3, MUL VL]\n" "b 17f\n" "16:" // Height 2: no accumulate "mov z8.b, #0x0\n" @@ -258,12 +258,12 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "18:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 19f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 20f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -271,7 +271,7 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "b 20f\n" "19:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" "20:" // Height 2: input setup done "subs x27, x27, #0x1\n" "ld1rh { z0.h }, p4/Z, [x26]\n" @@ -282,18 +282,18 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "21:" // Height 2: Multiply loop: Main loop "fmla z8.h, p4/M, z6.h, z0.h\n" "fmla z12.h, p4/M, z6.h, z1.h\n" - "ld1h { z6.h }, p4/Z, [x10, #2, MUL VL]\n" + "ld1h { z17.h }, p4/Z, [x10, #2, MUL VL]\n" "add x26, x26, #0x2\n" "fmla z9.h, p4/M, z7.h, z0.h\n" "fmla z13.h, p4/M, z7.h, z1.h\n" - "ld1h { z7.h }, p4/Z, [x10, #3, MUL VL]\n" + "ld1h { z16.h }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" "subs x27, x27, #0x1\n" "add x25, x25, #0x2\n" - "fmla z10.h, p4/M, z6.h, z0.h\n" - "fmla z14.h, p4/M, z6.h, z1.h\n" - "fmla z11.h, p4/M, z7.h, z0.h\n" - "fmla z15.h, p4/M, z7.h, z1.h\n" + "fmla z10.h, p4/M, z17.h, z0.h\n" + "fmla z14.h, p4/M, z17.h, z1.h\n" + "fmla z11.h, p4/M, z16.h, z0.h\n" + "fmla z15.h, p4/M, z16.h, z1.h\n" "ld1rh { z0.h }, p4/Z, [x26]\n" "ld1rh { z1.h }, p4/Z, [x25]\n" "ld1h { z6.h }, p4/Z, [x10]\n" @@ -303,41 +303,41 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "fmla z8.h, p4/M, z6.h, z0.h\n" "fmla z12.h, p4/M, z6.h, z1.h\n" - "ld1h { z6.h }, p4/Z, [x10, #2, MUL VL]\n" + "ld1h { z17.h }, p4/Z, [x10, #2, MUL VL]\n" "fmla z9.h, p4/M, z7.h, z0.h\n" "fmla z13.h, p4/M, z7.h, z1.h\n" - "ld1h { z7.h }, p4/Z, [x10, #3, MUL VL]\n" + "ld1h { z16.h }, p4/Z, [x10, #3, MUL VL]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" - "fmla z10.h, p4/M, z6.h, z0.h\n" - "fmla z14.h, p4/M, z6.h, z1.h\n" + "fmla z10.h, p4/M, z17.h, z0.h\n" + "fmla z14.h, p4/M, z17.h, z1.h\n" "addvl x10, x10, #4\n" - "fmla z11.h, p4/M, z7.h, z0.h\n" - "fmla z15.h, p4/M, z7.h, z1.h\n" + "fmla z11.h, p4/M, z16.h, z0.h\n" + "fmla z15.h, p4/M, z16.h, z1.h\n" "bne 18b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "add x25, x9, x20, LSL #1\n" "tbz %x[flags], #1, 23f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rh { z1.h }, p4/Z, [x20]\n" + "ld1rh { z17.h }, p4/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rh { z0.h }, p4/Z, [x20]\n" - "fmin z8.h, p4/M, z8.h, z1.h\n" - "fmin z9.h, p4/M, z9.h, z1.h\n" - "fmin z10.h, p4/M, z10.h, z1.h\n" - "fmin z11.h, p4/M, z11.h, z1.h\n" - "fmin z12.h, p4/M, z12.h, z1.h\n" - "fmin z13.h, p4/M, z13.h, z1.h\n" - "fmin z14.h, p4/M, z14.h, z1.h\n" - "fmin z15.h, p4/M, z15.h, z1.h\n" - "fmax z8.h, p4/M, z8.h, z0.h\n" - "fmax z9.h, p4/M, z9.h, z0.h\n" - "fmax z10.h, p4/M, z10.h, z0.h\n" - "fmax z11.h, p4/M, z11.h, z0.h\n" - "fmax z12.h, p4/M, z12.h, z0.h\n" - "fmax z13.h, p4/M, z13.h, z0.h\n" - "fmax z14.h, p4/M, z14.h, z0.h\n" - "fmax z15.h, p4/M, z15.h, z0.h\n" + "ld1rh { z16.h }, p4/Z, [x20]\n" + "fmin z8.h, p4/M, z8.h, z17.h\n" + "fmin z9.h, p4/M, z9.h, z17.h\n" + "fmin z10.h, p4/M, z10.h, z17.h\n" + "fmin z11.h, p4/M, z11.h, z17.h\n" + "fmin z12.h, p4/M, z12.h, z17.h\n" + "fmin z13.h, p4/M, z13.h, z17.h\n" + "fmin z14.h, p4/M, z14.h, z17.h\n" + "fmin z15.h, p4/M, z15.h, z17.h\n" + "fmax z8.h, p4/M, z8.h, z16.h\n" + "fmax z9.h, p4/M, z9.h, z16.h\n" + "fmax z10.h, p4/M, z10.h, z16.h\n" + "fmax z11.h, p4/M, z11.h, z16.h\n" + "fmax z12.h, p4/M, z12.h, z16.h\n" + "fmax z13.h, p4/M, z13.h, z16.h\n" + "fmax z14.h, p4/M, z14.h, z16.h\n" + "fmax z15.h, p4/M, z15.h, z16.h\n" "23:" // Height 2: No activation "st1h { z8.h }, p3, [x9]\n" "st1h { z9.h }, p2, [x9, #1, MUL VL]\n" @@ -385,20 +385,20 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "27:" // Height 3: no bias "tbz %x[flags], #0, 28f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" + "add x21, x9, x20, LSL #1\n" + "add x20, x21, x20, LSL #1\n" "ld1h { z8.h }, p3/Z, [x9]\n" "ld1h { z9.h }, p2/Z, [x9, #1, MUL VL]\n" "ld1h { z10.h }, p1/Z, [x9, #2, MUL VL]\n" "ld1h { z11.h }, p0/Z, [x9, #3, MUL VL]\n" - "ld1h { z12.h }, p3/Z, [x25]\n" - "ld1h { z13.h }, p2/Z, [x25, #1, MUL VL]\n" - "ld1h { z14.h }, p1/Z, [x25, #2, MUL VL]\n" - "ld1h { z15.h }, p0/Z, [x25, #3, MUL VL]\n" - "ld1h { z16.h }, p3/Z, [x24]\n" - "ld1h { z17.h }, p2/Z, [x24, #1, MUL VL]\n" - "ld1h { z18.h }, p1/Z, [x24, #2, MUL VL]\n" - "ld1h { z19.h }, p0/Z, [x24, #3, MUL VL]\n" + "ld1h { z12.h }, p3/Z, [x21]\n" + "ld1h { z13.h }, p2/Z, [x21, #1, MUL VL]\n" + "ld1h { z14.h }, p1/Z, [x21, #2, MUL VL]\n" + "ld1h { z15.h }, p0/Z, [x21, #3, MUL VL]\n" + "ld1h { z16.h }, p3/Z, [x20]\n" + "ld1h { z17.h }, p2/Z, [x20, #1, MUL VL]\n" + "ld1h { z18.h }, p1/Z, [x20, #2, MUL VL]\n" + "ld1h { z19.h }, p0/Z, [x20, #3, MUL VL]\n" "b 29f\n" "28:" // Height 3: no accumulate "mov z8.b, #0x0\n" @@ -418,13 +418,13 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "30:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 31f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 32f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -433,8 +433,8 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "b 32f\n" "31:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" "32:" // Height 3: input setup done "subs x27, x27, #0x1\n" "ld1rh { z0.h }, p4/Z, [x26]\n" @@ -450,21 +450,21 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "subs x27, x27, #0x1\n" "fmla z16.h, p4/M, z6.h, z2.h\n" "fmla z9.h, p4/M, z7.h, z0.h\n" - "ld1h { z6.h }, p4/Z, [x10, #2, MUL VL]\n" + "ld1h { z21.h }, p4/Z, [x10, #2, MUL VL]\n" "add x25, x25, #0x2\n" "fmla z13.h, p4/M, z7.h, z1.h\n" "fmla z17.h, p4/M, z7.h, z2.h\n" - "ld1h { z7.h }, p4/Z, [x10, #3, MUL VL]\n" + "ld1h { z20.h }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" "add x24, x24, #0x2\n" - "fmla z10.h, p4/M, z6.h, z0.h\n" - "fmla z14.h, p4/M, z6.h, z1.h\n" - "fmla z18.h, p4/M, z6.h, z2.h\n" - "fmla z11.h, p4/M, z7.h, z0.h\n" + "fmla z10.h, p4/M, z21.h, z0.h\n" + "fmla z14.h, p4/M, z21.h, z1.h\n" + "fmla z18.h, p4/M, z21.h, z2.h\n" + "fmla z11.h, p4/M, z20.h, z0.h\n" "ld1rh { z0.h }, p4/Z, [x26]\n" "ld1h { z6.h }, p4/Z, [x10]\n" - "fmla z15.h, p4/M, z7.h, z1.h\n" - "fmla z19.h, p4/M, z7.h, z2.h\n" + "fmla z15.h, p4/M, z20.h, z1.h\n" + "fmla z19.h, p4/M, z20.h, z2.h\n" "ld1rh { z1.h }, p4/Z, [x25]\n" "ld1rh { z2.h }, p4/Z, [x24]\n" "ld1h { z7.h }, p4/Z, [x10, #1, MUL VL]\n" @@ -476,51 +476,51 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "add x28, x28, #0x1\n" "fmla z16.h, p4/M, z6.h, z2.h\n" "fmla z9.h, p4/M, z7.h, z0.h\n" - "ld1h { z6.h }, p4/Z, [x10, #2, MUL VL]\n" + "ld1h { z21.h }, p4/Z, [x10, #2, MUL VL]\n" "cmp x28, x20\n" "fmla z13.h, p4/M, z7.h, z1.h\n" "fmla z17.h, p4/M, z7.h, z2.h\n" - "ld1h { z7.h }, p4/Z, [x10, #3, MUL VL]\n" + "ld1h { z20.h }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, p4/M, z6.h, z0.h\n" - "fmla z14.h, p4/M, z6.h, z1.h\n" - "fmla z18.h, p4/M, z6.h, z2.h\n" - "fmla z11.h, p4/M, z7.h, z0.h\n" - "fmla z15.h, p4/M, z7.h, z1.h\n" - "fmla z19.h, p4/M, z7.h, z2.h\n" + "fmla z10.h, p4/M, z21.h, z0.h\n" + "fmla z14.h, p4/M, z21.h, z1.h\n" + "fmla z18.h, p4/M, z21.h, z2.h\n" + "fmla z11.h, p4/M, z20.h, z0.h\n" + "fmla z15.h, p4/M, z20.h, z1.h\n" + "fmla z19.h, p4/M, z20.h, z2.h\n" "bne 30b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "add x25, x9, x20, LSL #1\n" "add x24, x25, x20, LSL #1\n" "tbz %x[flags], #1, 35f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rh { z1.h }, p4/Z, [x20]\n" + "ld1rh { z21.h }, p4/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rh { z0.h }, p4/Z, [x20]\n" - "fmin z8.h, p4/M, z8.h, z1.h\n" - "fmin z9.h, p4/M, z9.h, z1.h\n" - "fmin z10.h, p4/M, z10.h, z1.h\n" - "fmin z11.h, p4/M, z11.h, z1.h\n" - "fmin z12.h, p4/M, z12.h, z1.h\n" - "fmin z13.h, p4/M, z13.h, z1.h\n" - "fmin z14.h, p4/M, z14.h, z1.h\n" - "fmin z15.h, p4/M, z15.h, z1.h\n" - "fmin z16.h, p4/M, z16.h, z1.h\n" - "fmin z17.h, p4/M, z17.h, z1.h\n" - "fmin z18.h, p4/M, z18.h, z1.h\n" - "fmin z19.h, p4/M, z19.h, z1.h\n" - "fmax z8.h, p4/M, z8.h, z0.h\n" - "fmax z9.h, p4/M, z9.h, z0.h\n" - "fmax z10.h, p4/M, z10.h, z0.h\n" - "fmax z11.h, p4/M, z11.h, z0.h\n" - "fmax z12.h, p4/M, z12.h, z0.h\n" - "fmax z13.h, p4/M, z13.h, z0.h\n" - "fmax z14.h, p4/M, z14.h, z0.h\n" - "fmax z15.h, p4/M, z15.h, z0.h\n" - "fmax z16.h, p4/M, z16.h, z0.h\n" - "fmax z17.h, p4/M, z17.h, z0.h\n" - "fmax z18.h, p4/M, z18.h, z0.h\n" - "fmax z19.h, p4/M, z19.h, z0.h\n" + "ld1rh { z20.h }, p4/Z, [x20]\n" + "fmin z8.h, p4/M, z8.h, z21.h\n" + "fmin z9.h, p4/M, z9.h, z21.h\n" + "fmin z10.h, p4/M, z10.h, z21.h\n" + "fmin z11.h, p4/M, z11.h, z21.h\n" + "fmin z12.h, p4/M, z12.h, z21.h\n" + "fmin z13.h, p4/M, z13.h, z21.h\n" + "fmin z14.h, p4/M, z14.h, z21.h\n" + "fmin z15.h, p4/M, z15.h, z21.h\n" + "fmin z16.h, p4/M, z16.h, z21.h\n" + "fmin z17.h, p4/M, z17.h, z21.h\n" + "fmin z18.h, p4/M, z18.h, z21.h\n" + "fmin z19.h, p4/M, z19.h, z21.h\n" + "fmax z8.h, p4/M, z8.h, z20.h\n" + "fmax z9.h, p4/M, z9.h, z20.h\n" + "fmax z10.h, p4/M, z10.h, z20.h\n" + "fmax z11.h, p4/M, z11.h, z20.h\n" + "fmax z12.h, p4/M, z12.h, z20.h\n" + "fmax z13.h, p4/M, z13.h, z20.h\n" + "fmax z14.h, p4/M, z14.h, z20.h\n" + "fmax z15.h, p4/M, z15.h, z20.h\n" + "fmax z16.h, p4/M, z16.h, z20.h\n" + "fmax z17.h, p4/M, z17.h, z20.h\n" + "fmax z18.h, p4/M, z18.h, z20.h\n" + "fmax z19.h, p4/M, z19.h, z20.h\n" "35:" // Height 3: No activation "st1h { z8.h }, p3, [x9]\n" "st1h { z9.h }, p2, [x9, #1, MUL VL]\n" @@ -576,25 +576,25 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "39:" // Height 4: no bias "tbz %x[flags], #0, 40f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" + "add x22, x9, x20, LSL #1\n" + "add x21, x22, x20, LSL #1\n" "ld1h { z8.h }, p3/Z, [x9]\n" - "add x23, x24, x20, LSL #1\n" + "add x20, x21, x20, LSL #1\n" "ld1h { z9.h }, p2/Z, [x9, #1, MUL VL]\n" "ld1h { z10.h }, p1/Z, [x9, #2, MUL VL]\n" "ld1h { z11.h }, p0/Z, [x9, #3, MUL VL]\n" - "ld1h { z12.h }, p3/Z, [x25]\n" - "ld1h { z13.h }, p2/Z, [x25, #1, MUL VL]\n" - "ld1h { z14.h }, p1/Z, [x25, #2, MUL VL]\n" - "ld1h { z15.h }, p0/Z, [x25, #3, MUL VL]\n" - "ld1h { z16.h }, p3/Z, [x24]\n" - "ld1h { z17.h }, p2/Z, [x24, #1, MUL VL]\n" - "ld1h { z18.h }, p1/Z, [x24, #2, MUL VL]\n" - "ld1h { z19.h }, p0/Z, [x24, #3, MUL VL]\n" - "ld1h { z20.h }, p3/Z, [x23]\n" - "ld1h { z21.h }, p2/Z, [x23, #1, MUL VL]\n" - "ld1h { z22.h }, p1/Z, [x23, #2, MUL VL]\n" - "ld1h { z23.h }, p0/Z, [x23, #3, MUL VL]\n" + "ld1h { z12.h }, p3/Z, [x22]\n" + "ld1h { z13.h }, p2/Z, [x22, #1, MUL VL]\n" + "ld1h { z14.h }, p1/Z, [x22, #2, MUL VL]\n" + "ld1h { z15.h }, p0/Z, [x22, #3, MUL VL]\n" + "ld1h { z16.h }, p3/Z, [x21]\n" + "ld1h { z17.h }, p2/Z, [x21, #1, MUL VL]\n" + "ld1h { z18.h }, p1/Z, [x21, #2, MUL VL]\n" + "ld1h { z19.h }, p0/Z, [x21, #3, MUL VL]\n" + "ld1h { z20.h }, p3/Z, [x20]\n" + "ld1h { z21.h }, p2/Z, [x20, #1, MUL VL]\n" + "ld1h { z22.h }, p1/Z, [x20, #2, MUL VL]\n" + "ld1h { z23.h }, p0/Z, [x20, #3, MUL VL]\n" "b 41f\n" "40:" // Height 4: no accumulate "mov z8.b, #0x0\n" @@ -618,14 +618,14 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "42:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 43f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 44f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -635,9 +635,9 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "b 44f\n" "43:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" "44:" // Height 4: input setup done "subs x27, x27, #0x1\n" "ld1rh { z0.h }, p4/Z, [x26]\n" @@ -654,7 +654,7 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "subs x27, x27, #0x1\n" "fmla z16.h, p4/M, z6.h, z2.h\n" "fmla z20.h, p4/M, z6.h, z3.h\n" - "ld1h { z6.h }, p4/Z, [x10, #2, MUL VL]\n" + "ld1h { z25.h }, p4/Z, [x10, #2, MUL VL]\n" "add x25, x25, #0x2\n" "fmla z9.h, p4/M, z7.h, z0.h\n" "fmla z13.h, p4/M, z7.h, z1.h\n" @@ -662,19 +662,19 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "add x23, x23, #0x2\n" "fmla z17.h, p4/M, z7.h, z2.h\n" "fmla z21.h, p4/M, z7.h, z3.h\n" - "ld1h { z7.h }, p4/Z, [x10, #3, MUL VL]\n" + "ld1h { z24.h }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, p4/M, z6.h, z0.h\n" - "fmla z14.h, p4/M, z6.h, z1.h\n" - "fmla z18.h, p4/M, z6.h, z2.h\n" - "fmla z22.h, p4/M, z6.h, z3.h\n" + "fmla z10.h, p4/M, z25.h, z0.h\n" + "fmla z14.h, p4/M, z25.h, z1.h\n" + "fmla z18.h, p4/M, z25.h, z2.h\n" + "fmla z22.h, p4/M, z25.h, z3.h\n" "ld1h { z6.h }, p4/Z, [x10]\n" - "fmla z11.h, p4/M, z7.h, z0.h\n" - "fmla z15.h, p4/M, z7.h, z1.h\n" + "fmla z11.h, p4/M, z24.h, z0.h\n" + "fmla z15.h, p4/M, z24.h, z1.h\n" "ld1rh { z0.h }, p4/Z, [x26]\n" "ld1rh { z1.h }, p4/Z, [x25]\n" - "fmla z19.h, p4/M, z7.h, z2.h\n" - "fmla z23.h, p4/M, z7.h, z3.h\n" + "fmla z19.h, p4/M, z24.h, z2.h\n" + "fmla z23.h, p4/M, z24.h, z3.h\n" "ld1rh { z2.h }, p4/Z, [x24]\n" "ld1rh { z3.h }, p4/Z, [x23]\n" "ld1h { z7.h }, p4/Z, [x10, #1, MUL VL]\n" @@ -686,22 +686,22 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "add x28, x28, #0x1\n" "fmla z16.h, p4/M, z6.h, z2.h\n" "fmla z20.h, p4/M, z6.h, z3.h\n" - "ld1h { z6.h }, p4/Z, [x10, #2, MUL VL]\n" + "ld1h { z25.h }, p4/Z, [x10, #2, MUL VL]\n" "cmp x28, x20\n" "fmla z9.h, p4/M, z7.h, z0.h\n" "fmla z13.h, p4/M, z7.h, z1.h\n" "fmla z17.h, p4/M, z7.h, z2.h\n" "fmla z21.h, p4/M, z7.h, z3.h\n" - "ld1h { z7.h }, p4/Z, [x10, #3, MUL VL]\n" + "ld1h { z24.h }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, p4/M, z6.h, z0.h\n" - "fmla z14.h, p4/M, z6.h, z1.h\n" - "fmla z18.h, p4/M, z6.h, z2.h\n" - "fmla z22.h, p4/M, z6.h, z3.h\n" - "fmla z11.h, p4/M, z7.h, z0.h\n" - "fmla z15.h, p4/M, z7.h, z1.h\n" - "fmla z19.h, p4/M, z7.h, z2.h\n" - "fmla z23.h, p4/M, z7.h, z3.h\n" + "fmla z10.h, p4/M, z25.h, z0.h\n" + "fmla z14.h, p4/M, z25.h, z1.h\n" + "fmla z18.h, p4/M, z25.h, z2.h\n" + "fmla z22.h, p4/M, z25.h, z3.h\n" + "fmla z11.h, p4/M, z24.h, z0.h\n" + "fmla z15.h, p4/M, z24.h, z1.h\n" + "fmla z19.h, p4/M, z24.h, z2.h\n" + "fmla z23.h, p4/M, z24.h, z3.h\n" "bne 42b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "add x25, x9, x20, LSL #1\n" @@ -709,41 +709,41 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "add x23, x24, x20, LSL #1\n" "tbz %x[flags], #1, 47f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rh { z1.h }, p4/Z, [x20]\n" + "ld1rh { z25.h }, p4/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rh { z0.h }, p4/Z, [x20]\n" - "fmin z8.h, p4/M, z8.h, z1.h\n" - "fmin z9.h, p4/M, z9.h, z1.h\n" - "fmin z10.h, p4/M, z10.h, z1.h\n" - "fmin z11.h, p4/M, z11.h, z1.h\n" - "fmin z12.h, p4/M, z12.h, z1.h\n" - "fmin z13.h, p4/M, z13.h, z1.h\n" - "fmin z14.h, p4/M, z14.h, z1.h\n" - "fmin z15.h, p4/M, z15.h, z1.h\n" - "fmin z16.h, p4/M, z16.h, z1.h\n" - "fmin z17.h, p4/M, z17.h, z1.h\n" - "fmin z18.h, p4/M, z18.h, z1.h\n" - "fmin z19.h, p4/M, z19.h, z1.h\n" - "fmin z20.h, p4/M, z20.h, z1.h\n" - "fmin z21.h, p4/M, z21.h, z1.h\n" - "fmin z22.h, p4/M, z22.h, z1.h\n" - "fmin z23.h, p4/M, z23.h, z1.h\n" - "fmax z8.h, p4/M, z8.h, z0.h\n" - "fmax z9.h, p4/M, z9.h, z0.h\n" - "fmax z10.h, p4/M, z10.h, z0.h\n" - "fmax z11.h, p4/M, z11.h, z0.h\n" - "fmax z12.h, p4/M, z12.h, z0.h\n" - "fmax z13.h, p4/M, z13.h, z0.h\n" - "fmax z14.h, p4/M, z14.h, z0.h\n" - "fmax z15.h, p4/M, z15.h, z0.h\n" - "fmax z16.h, p4/M, z16.h, z0.h\n" - "fmax z17.h, p4/M, z17.h, z0.h\n" - "fmax z18.h, p4/M, z18.h, z0.h\n" - "fmax z19.h, p4/M, z19.h, z0.h\n" - "fmax z20.h, p4/M, z20.h, z0.h\n" - "fmax z21.h, p4/M, z21.h, z0.h\n" - "fmax z22.h, p4/M, z22.h, z0.h\n" - "fmax z23.h, p4/M, z23.h, z0.h\n" + "ld1rh { z24.h }, p4/Z, [x20]\n" + "fmin z8.h, p4/M, z8.h, z25.h\n" + "fmin z9.h, p4/M, z9.h, z25.h\n" + "fmin z10.h, p4/M, z10.h, z25.h\n" + "fmin z11.h, p4/M, z11.h, z25.h\n" + "fmin z12.h, p4/M, z12.h, z25.h\n" + "fmin z13.h, p4/M, z13.h, z25.h\n" + "fmin z14.h, p4/M, z14.h, z25.h\n" + "fmin z15.h, p4/M, z15.h, z25.h\n" + "fmin z16.h, p4/M, z16.h, z25.h\n" + "fmin z17.h, p4/M, z17.h, z25.h\n" + "fmin z18.h, p4/M, z18.h, z25.h\n" + "fmin z19.h, p4/M, z19.h, z25.h\n" + "fmin z20.h, p4/M, z20.h, z25.h\n" + "fmin z21.h, p4/M, z21.h, z25.h\n" + "fmin z22.h, p4/M, z22.h, z25.h\n" + "fmin z23.h, p4/M, z23.h, z25.h\n" + "fmax z8.h, p4/M, z8.h, z24.h\n" + "fmax z9.h, p4/M, z9.h, z24.h\n" + "fmax z10.h, p4/M, z10.h, z24.h\n" + "fmax z11.h, p4/M, z11.h, z24.h\n" + "fmax z12.h, p4/M, z12.h, z24.h\n" + "fmax z13.h, p4/M, z13.h, z24.h\n" + "fmax z14.h, p4/M, z14.h, z24.h\n" + "fmax z15.h, p4/M, z15.h, z24.h\n" + "fmax z16.h, p4/M, z16.h, z24.h\n" + "fmax z17.h, p4/M, z17.h, z24.h\n" + "fmax z18.h, p4/M, z18.h, z24.h\n" + "fmax z19.h, p4/M, z19.h, z24.h\n" + "fmax z20.h, p4/M, z20.h, z24.h\n" + "fmax z21.h, p4/M, z21.h, z24.h\n" + "fmax z22.h, p4/M, z22.h, z24.h\n" + "fmax z23.h, p4/M, z23.h, z24.h\n" "47:" // Height 4: No activation "st1h { z8.h }, p3, [x9]\n" "st1h { z9.h }, p2, [x9, #1, MUL VL]\n" @@ -807,30 +807,30 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "51:" // Height 5: no bias "tbz %x[flags], #0, 52f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "ld1h { z8.h }, p3/Z, [x9]\n" - "add x23, x24, x20, LSL #1\n" + "add x23, x9, x20, LSL #1\n" "add x22, x23, x20, LSL #1\n" + "ld1h { z8.h }, p3/Z, [x9]\n" + "add x21, x22, x20, LSL #1\n" + "add x20, x21, x20, LSL #1\n" "ld1h { z9.h }, p2/Z, [x9, #1, MUL VL]\n" "ld1h { z10.h }, p1/Z, [x9, #2, MUL VL]\n" "ld1h { z11.h }, p0/Z, [x9, #3, MUL VL]\n" - "ld1h { z12.h }, p3/Z, [x25]\n" - "ld1h { z13.h }, p2/Z, [x25, #1, MUL VL]\n" - "ld1h { z14.h }, p1/Z, [x25, #2, MUL VL]\n" - "ld1h { z15.h }, p0/Z, [x25, #3, MUL VL]\n" - "ld1h { z16.h }, p3/Z, [x24]\n" - "ld1h { z17.h }, p2/Z, [x24, #1, MUL VL]\n" - "ld1h { z18.h }, p1/Z, [x24, #2, MUL VL]\n" - "ld1h { z19.h }, p0/Z, [x24, #3, MUL VL]\n" - "ld1h { z20.h }, p3/Z, [x23]\n" - "ld1h { z21.h }, p2/Z, [x23, #1, MUL VL]\n" - "ld1h { z22.h }, p1/Z, [x23, #2, MUL VL]\n" - "ld1h { z23.h }, p0/Z, [x23, #3, MUL VL]\n" - "ld1h { z24.h }, p3/Z, [x22]\n" - "ld1h { z25.h }, p2/Z, [x22, #1, MUL VL]\n" - "ld1h { z26.h }, p1/Z, [x22, #2, MUL VL]\n" - "ld1h { z27.h }, p0/Z, [x22, #3, MUL VL]\n" + "ld1h { z12.h }, p3/Z, [x23]\n" + "ld1h { z13.h }, p2/Z, [x23, #1, MUL VL]\n" + "ld1h { z14.h }, p1/Z, [x23, #2, MUL VL]\n" + "ld1h { z15.h }, p0/Z, [x23, #3, MUL VL]\n" + "ld1h { z16.h }, p3/Z, [x22]\n" + "ld1h { z17.h }, p2/Z, [x22, #1, MUL VL]\n" + "ld1h { z18.h }, p1/Z, [x22, #2, MUL VL]\n" + "ld1h { z19.h }, p0/Z, [x22, #3, MUL VL]\n" + "ld1h { z20.h }, p3/Z, [x21]\n" + "ld1h { z21.h }, p2/Z, [x21, #1, MUL VL]\n" + "ld1h { z22.h }, p1/Z, [x21, #2, MUL VL]\n" + "ld1h { z23.h }, p0/Z, [x21, #3, MUL VL]\n" + "ld1h { z24.h }, p3/Z, [x20]\n" + "ld1h { z25.h }, p2/Z, [x20, #1, MUL VL]\n" + "ld1h { z26.h }, p1/Z, [x20, #2, MUL VL]\n" + "ld1h { z27.h }, p0/Z, [x20, #3, MUL VL]\n" "b 53f\n" "52:" // Height 5: no accumulate "mov z8.b, #0x0\n" @@ -858,15 +858,15 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "54:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 55f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 56f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -877,10 +877,10 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "b 56f\n" "55:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" - "add x22, x23, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" + "add x22, x23, x21, LSL #1\n" "56:" // Height 5: input setup done "subs x27, x27, #0x1\n" "ld1rh { z0.h }, p4/Z, [x26]\n" @@ -902,29 +902,29 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "add x24, x24, #0x2\n" "fmla z24.h, p4/M, z6.h, z4.h\n" "fmla z9.h, p4/M, z7.h, z0.h\n" - "ld1h { z6.h }, p4/Z, [x10, #2, MUL VL]\n" + "ld1h { z29.h }, p4/Z, [x10, #2, MUL VL]\n" "add x23, x23, #0x2\n" "fmla z13.h, p4/M, z7.h, z1.h\n" "fmla z17.h, p4/M, z7.h, z2.h\n" "add x22, x22, #0x2\n" "fmla z21.h, p4/M, z7.h, z3.h\n" "fmla z25.h, p4/M, z7.h, z4.h\n" - "ld1h { z7.h }, p4/Z, [x10, #3, MUL VL]\n" + "ld1h { z28.h }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, p4/M, z6.h, z0.h\n" - "fmla z14.h, p4/M, z6.h, z1.h\n" - "fmla z18.h, p4/M, z6.h, z2.h\n" - "fmla z22.h, p4/M, z6.h, z3.h\n" - "fmla z26.h, p4/M, z6.h, z4.h\n" - "fmla z11.h, p4/M, z7.h, z0.h\n" + "fmla z10.h, p4/M, z29.h, z0.h\n" + "fmla z14.h, p4/M, z29.h, z1.h\n" + "fmla z18.h, p4/M, z29.h, z2.h\n" + "fmla z22.h, p4/M, z29.h, z3.h\n" + "fmla z26.h, p4/M, z29.h, z4.h\n" + "fmla z11.h, p4/M, z28.h, z0.h\n" "ld1rh { z0.h }, p4/Z, [x26]\n" "ld1h { z6.h }, p4/Z, [x10]\n" - "fmla z15.h, p4/M, z7.h, z1.h\n" - "fmla z19.h, p4/M, z7.h, z2.h\n" + "fmla z15.h, p4/M, z28.h, z1.h\n" + "fmla z19.h, p4/M, z28.h, z2.h\n" "ld1rh { z1.h }, p4/Z, [x25]\n" "ld1rh { z2.h }, p4/Z, [x24]\n" - "fmla z23.h, p4/M, z7.h, z3.h\n" - "fmla z27.h, p4/M, z7.h, z4.h\n" + "fmla z23.h, p4/M, z28.h, z3.h\n" + "fmla z27.h, p4/M, z28.h, z4.h\n" "ld1rh { z3.h }, p4/Z, [x23]\n" "ld1rh { z4.h }, p4/Z, [x22]\n" "ld1h { z7.h }, p4/Z, [x10, #1, MUL VL]\n" @@ -939,23 +939,23 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "cmp x28, x20\n" "fmla z24.h, p4/M, z6.h, z4.h\n" "fmla z9.h, p4/M, z7.h, z0.h\n" - "ld1h { z6.h }, p4/Z, [x10, #2, MUL VL]\n" + "ld1h { z29.h }, p4/Z, [x10, #2, MUL VL]\n" "fmla z13.h, p4/M, z7.h, z1.h\n" "fmla z17.h, p4/M, z7.h, z2.h\n" "fmla z21.h, p4/M, z7.h, z3.h\n" "fmla z25.h, p4/M, z7.h, z4.h\n" - "ld1h { z7.h }, p4/Z, [x10, #3, MUL VL]\n" + "ld1h { z28.h }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, p4/M, z6.h, z0.h\n" - "fmla z14.h, p4/M, z6.h, z1.h\n" - "fmla z18.h, p4/M, z6.h, z2.h\n" - "fmla z22.h, p4/M, z6.h, z3.h\n" - "fmla z26.h, p4/M, z6.h, z4.h\n" - "fmla z11.h, p4/M, z7.h, z0.h\n" - "fmla z15.h, p4/M, z7.h, z1.h\n" - "fmla z19.h, p4/M, z7.h, z2.h\n" - "fmla z23.h, p4/M, z7.h, z3.h\n" - "fmla z27.h, p4/M, z7.h, z4.h\n" + "fmla z10.h, p4/M, z29.h, z0.h\n" + "fmla z14.h, p4/M, z29.h, z1.h\n" + "fmla z18.h, p4/M, z29.h, z2.h\n" + "fmla z22.h, p4/M, z29.h, z3.h\n" + "fmla z26.h, p4/M, z29.h, z4.h\n" + "fmla z11.h, p4/M, z28.h, z0.h\n" + "fmla z15.h, p4/M, z28.h, z1.h\n" + "fmla z19.h, p4/M, z28.h, z2.h\n" + "fmla z23.h, p4/M, z28.h, z3.h\n" + "fmla z27.h, p4/M, z28.h, z4.h\n" "bne 54b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "add x25, x9, x20, LSL #1\n" @@ -964,49 +964,49 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "add x22, x23, x20, LSL #1\n" "tbz %x[flags], #1, 59f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rh { z1.h }, p4/Z, [x20]\n" + "ld1rh { z29.h }, p4/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rh { z0.h }, p4/Z, [x20]\n" - "fmin z8.h, p4/M, z8.h, z1.h\n" - "fmin z9.h, p4/M, z9.h, z1.h\n" - "fmin z10.h, p4/M, z10.h, z1.h\n" - "fmin z11.h, p4/M, z11.h, z1.h\n" - "fmin z12.h, p4/M, z12.h, z1.h\n" - "fmin z13.h, p4/M, z13.h, z1.h\n" - "fmin z14.h, p4/M, z14.h, z1.h\n" - "fmin z15.h, p4/M, z15.h, z1.h\n" - "fmin z16.h, p4/M, z16.h, z1.h\n" - "fmin z17.h, p4/M, z17.h, z1.h\n" - "fmin z18.h, p4/M, z18.h, z1.h\n" - "fmin z19.h, p4/M, z19.h, z1.h\n" - "fmin z20.h, p4/M, z20.h, z1.h\n" - "fmin z21.h, p4/M, z21.h, z1.h\n" - "fmin z22.h, p4/M, z22.h, z1.h\n" - "fmin z23.h, p4/M, z23.h, z1.h\n" - "fmin z24.h, p4/M, z24.h, z1.h\n" - "fmin z25.h, p4/M, z25.h, z1.h\n" - "fmin z26.h, p4/M, z26.h, z1.h\n" - "fmin z27.h, p4/M, z27.h, z1.h\n" - "fmax z8.h, p4/M, z8.h, z0.h\n" - "fmax z9.h, p4/M, z9.h, z0.h\n" - "fmax z10.h, p4/M, z10.h, z0.h\n" - "fmax z11.h, p4/M, z11.h, z0.h\n" - "fmax z12.h, p4/M, z12.h, z0.h\n" - "fmax z13.h, p4/M, z13.h, z0.h\n" - "fmax z14.h, p4/M, z14.h, z0.h\n" - "fmax z15.h, p4/M, z15.h, z0.h\n" - "fmax z16.h, p4/M, z16.h, z0.h\n" - "fmax z17.h, p4/M, z17.h, z0.h\n" - "fmax z18.h, p4/M, z18.h, z0.h\n" - "fmax z19.h, p4/M, z19.h, z0.h\n" - "fmax z20.h, p4/M, z20.h, z0.h\n" - "fmax z21.h, p4/M, z21.h, z0.h\n" - "fmax z22.h, p4/M, z22.h, z0.h\n" - "fmax z23.h, p4/M, z23.h, z0.h\n" - "fmax z24.h, p4/M, z24.h, z0.h\n" - "fmax z25.h, p4/M, z25.h, z0.h\n" - "fmax z26.h, p4/M, z26.h, z0.h\n" - "fmax z27.h, p4/M, z27.h, z0.h\n" + "ld1rh { z28.h }, p4/Z, [x20]\n" + "fmin z8.h, p4/M, z8.h, z29.h\n" + "fmin z9.h, p4/M, z9.h, z29.h\n" + "fmin z10.h, p4/M, z10.h, z29.h\n" + "fmin z11.h, p4/M, z11.h, z29.h\n" + "fmin z12.h, p4/M, z12.h, z29.h\n" + "fmin z13.h, p4/M, z13.h, z29.h\n" + "fmin z14.h, p4/M, z14.h, z29.h\n" + "fmin z15.h, p4/M, z15.h, z29.h\n" + "fmin z16.h, p4/M, z16.h, z29.h\n" + "fmin z17.h, p4/M, z17.h, z29.h\n" + "fmin z18.h, p4/M, z18.h, z29.h\n" + "fmin z19.h, p4/M, z19.h, z29.h\n" + "fmin z20.h, p4/M, z20.h, z29.h\n" + "fmin z21.h, p4/M, z21.h, z29.h\n" + "fmin z22.h, p4/M, z22.h, z29.h\n" + "fmin z23.h, p4/M, z23.h, z29.h\n" + "fmin z24.h, p4/M, z24.h, z29.h\n" + "fmin z25.h, p4/M, z25.h, z29.h\n" + "fmin z26.h, p4/M, z26.h, z29.h\n" + "fmin z27.h, p4/M, z27.h, z29.h\n" + "fmax z8.h, p4/M, z8.h, z28.h\n" + "fmax z9.h, p4/M, z9.h, z28.h\n" + "fmax z10.h, p4/M, z10.h, z28.h\n" + "fmax z11.h, p4/M, z11.h, z28.h\n" + "fmax z12.h, p4/M, z12.h, z28.h\n" + "fmax z13.h, p4/M, z13.h, z28.h\n" + "fmax z14.h, p4/M, z14.h, z28.h\n" + "fmax z15.h, p4/M, z15.h, z28.h\n" + "fmax z16.h, p4/M, z16.h, z28.h\n" + "fmax z17.h, p4/M, z17.h, z28.h\n" + "fmax z18.h, p4/M, z18.h, z28.h\n" + "fmax z19.h, p4/M, z19.h, z28.h\n" + "fmax z20.h, p4/M, z20.h, z28.h\n" + "fmax z21.h, p4/M, z21.h, z28.h\n" + "fmax z22.h, p4/M, z22.h, z28.h\n" + "fmax z23.h, p4/M, z23.h, z28.h\n" + "fmax z24.h, p4/M, z24.h, z28.h\n" + "fmax z25.h, p4/M, z25.h, z28.h\n" + "fmax z26.h, p4/M, z26.h, z28.h\n" + "fmax z27.h, p4/M, z27.h, z28.h\n" "59:" // Height 5: No activation "st1h { z8.h }, p3, [x9]\n" "st1h { z9.h }, p2, [x9, #1, MUL VL]\n" @@ -1081,35 +1081,35 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "63:" // Height 6: no bias "tbz %x[flags], #0, 64f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "ld1h { z8.h }, p3/Z, [x9]\n" + "add x24, x9, x20, LSL #1\n" "add x23, x24, x20, LSL #1\n" + "ld1h { z8.h }, p3/Z, [x9]\n" "add x22, x23, x20, LSL #1\n" + "add x21, x22, x20, LSL #1\n" "ld1h { z9.h }, p2/Z, [x9, #1, MUL VL]\n" "ld1h { z10.h }, p1/Z, [x9, #2, MUL VL]\n" - "add x21, x22, x20, LSL #1\n" + "add x20, x21, x20, LSL #1\n" "ld1h { z11.h }, p0/Z, [x9, #3, MUL VL]\n" - "ld1h { z12.h }, p3/Z, [x25]\n" - "ld1h { z13.h }, p2/Z, [x25, #1, MUL VL]\n" - "ld1h { z14.h }, p1/Z, [x25, #2, MUL VL]\n" - "ld1h { z15.h }, p0/Z, [x25, #3, MUL VL]\n" - "ld1h { z16.h }, p3/Z, [x24]\n" - "ld1h { z17.h }, p2/Z, [x24, #1, MUL VL]\n" - "ld1h { z18.h }, p1/Z, [x24, #2, MUL VL]\n" - "ld1h { z19.h }, p0/Z, [x24, #3, MUL VL]\n" - "ld1h { z20.h }, p3/Z, [x23]\n" - "ld1h { z21.h }, p2/Z, [x23, #1, MUL VL]\n" - "ld1h { z22.h }, p1/Z, [x23, #2, MUL VL]\n" - "ld1h { z23.h }, p0/Z, [x23, #3, MUL VL]\n" - "ld1h { z24.h }, p3/Z, [x22]\n" - "ld1h { z25.h }, p2/Z, [x22, #1, MUL VL]\n" - "ld1h { z26.h }, p1/Z, [x22, #2, MUL VL]\n" - "ld1h { z27.h }, p0/Z, [x22, #3, MUL VL]\n" - "ld1h { z28.h }, p3/Z, [x21]\n" - "ld1h { z29.h }, p2/Z, [x21, #1, MUL VL]\n" - "ld1h { z30.h }, p1/Z, [x21, #2, MUL VL]\n" - "ld1h { z31.h }, p0/Z, [x21, #3, MUL VL]\n" + "ld1h { z12.h }, p3/Z, [x24]\n" + "ld1h { z13.h }, p2/Z, [x24, #1, MUL VL]\n" + "ld1h { z14.h }, p1/Z, [x24, #2, MUL VL]\n" + "ld1h { z15.h }, p0/Z, [x24, #3, MUL VL]\n" + "ld1h { z16.h }, p3/Z, [x23]\n" + "ld1h { z17.h }, p2/Z, [x23, #1, MUL VL]\n" + "ld1h { z18.h }, p1/Z, [x23, #2, MUL VL]\n" + "ld1h { z19.h }, p0/Z, [x23, #3, MUL VL]\n" + "ld1h { z20.h }, p3/Z, [x22]\n" + "ld1h { z21.h }, p2/Z, [x22, #1, MUL VL]\n" + "ld1h { z22.h }, p1/Z, [x22, #2, MUL VL]\n" + "ld1h { z23.h }, p0/Z, [x22, #3, MUL VL]\n" + "ld1h { z24.h }, p3/Z, [x21]\n" + "ld1h { z25.h }, p2/Z, [x21, #1, MUL VL]\n" + "ld1h { z26.h }, p1/Z, [x21, #2, MUL VL]\n" + "ld1h { z27.h }, p0/Z, [x21, #3, MUL VL]\n" + "ld1h { z28.h }, p3/Z, [x20]\n" + "ld1h { z29.h }, p2/Z, [x20, #1, MUL VL]\n" + "ld1h { z30.h }, p1/Z, [x20, #2, MUL VL]\n" + "ld1h { z31.h }, p0/Z, [x20, #3, MUL VL]\n" "b 65f\n" "64:" // Height 6: no accumulate "mov z8.b, #0x0\n" @@ -1141,16 +1141,16 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "66:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 67f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 68f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -1162,11 +1162,11 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "b 68f\n" "67:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" - "add x22, x23, x20, LSL #1\n" - "add x21, x22, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" + "add x22, x23, x21, LSL #1\n" + "add x21, x22, x21, LSL #1\n" "68:" // Height 6: input setup done "subs x27, x27, #0x1\n" "ld1rh { z0.h }, p4/Z, [x26]\n" @@ -1355,7 +1355,6 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "74:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -1363,4 +1362,4 @@ void sve_hybrid_fp16_mla_6x4VL_a64fx ( } } // namespace arm_gemm -#endif // ARM_COMPUTE_ENABLE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL/generic.cpp index 6f0b3e0008ffb673e2f0554a44f8951c2dbded0c..041825df6b6c7322e8f821925ada55773fcd44d6 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp16_mla_6x4VL/generic.cpp @@ -139,11 +139,11 @@ void sve_hybrid_fp16_mla_6x4VL ( "6:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 7f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 8f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -156,164 +156,164 @@ void sve_hybrid_fp16_mla_6x4VL ( "9:" // Height 1: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" "ld1rqh { z0.h }, p0/Z, [x26]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "fmla z8.h, z6.h, z0.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10]\n" + "fmla z8.h, z16.h, z0.h[0]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z9.h, z16.h, z0.h[0]\n" + "ld1h { z16.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z10.h, z16.h, z0.h[0]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z11.h, z16.h, z0.h[0]\n" + "ld1h { z16.h }, p5/Z, [x10, #4, MUL VL]\n" + "fmla z8.h, z16.h, z0.h[1]\n" + "ld1h { z16.h }, p5/Z, [x10, #5, MUL VL]\n" + "fmla z9.h, z16.h, z0.h[1]\n" + "ld1h { z16.h }, p5/Z, [x10, #6, MUL VL]\n" + "fmla z10.h, z16.h, z0.h[1]\n" + "ld1h { z16.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "fmla z11.h, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #-7, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[2]\n" - "fmla z9.h, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #-5, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[2]\n" - "fmla z11.h, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #-3, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[3]\n" - "fmla z9.h, z7.h, z0.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #-1, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[3]\n" - "fmla z11.h, z7.h, z0.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[4]\n" - "fmla z9.h, z7.h, z0.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[4]\n" - "fmla z11.h, z7.h, z0.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[5]\n" - "fmla z9.h, z7.h, z0.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" + "fmla z11.h, z16.h, z0.h[1]\n" + "ld1h { z17.h }, p5/Z, [x10, #-8, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #-7, MUL VL]\n" + "fmla z8.h, z17.h, z0.h[2]\n" + "fmla z9.h, z16.h, z0.h[2]\n" + "ld1h { z17.h }, p5/Z, [x10, #-6, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #-5, MUL VL]\n" + "fmla z10.h, z17.h, z0.h[2]\n" + "fmla z11.h, z16.h, z0.h[2]\n" + "ld1h { z17.h }, p5/Z, [x10, #-4, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #-3, MUL VL]\n" + "fmla z8.h, z17.h, z0.h[3]\n" + "fmla z9.h, z16.h, z0.h[3]\n" + "ld1h { z17.h }, p5/Z, [x10, #-2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #-1, MUL VL]\n" + "fmla z10.h, z17.h, z0.h[3]\n" + "fmla z11.h, z16.h, z0.h[3]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z17.h, z0.h[4]\n" + "fmla z9.h, z16.h, z0.h[4]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z10.h, z17.h, z0.h[4]\n" + "fmla z11.h, z16.h, z0.h[4]\n" + "ld1h { z17.h }, p5/Z, [x10, #4, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #5, MUL VL]\n" + "fmla z8.h, z17.h, z0.h[5]\n" + "fmla z9.h, z16.h, z0.h[5]\n" + "ld1h { z17.h }, p5/Z, [x10, #6, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "fmla z10.h, z6.h, z0.h[5]\n" - "fmla z11.h, z7.h, z0.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #-7, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[6]\n" - "fmla z9.h, z7.h, z0.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #-5, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[6]\n" - "fmla z11.h, z7.h, z0.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #-3, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[7]\n" - "fmla z9.h, z7.h, z0.h[7]\n" - "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #-1, MUL VL]\n" + "fmla z10.h, z17.h, z0.h[5]\n" + "fmla z11.h, z16.h, z0.h[5]\n" + "ld1h { z17.h }, p5/Z, [x10, #-8, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #-7, MUL VL]\n" + "fmla z8.h, z17.h, z0.h[6]\n" + "fmla z9.h, z16.h, z0.h[6]\n" + "ld1h { z17.h }, p5/Z, [x10, #-6, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #-5, MUL VL]\n" + "fmla z10.h, z17.h, z0.h[6]\n" + "fmla z11.h, z16.h, z0.h[6]\n" + "ld1h { z17.h }, p5/Z, [x10, #-4, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #-3, MUL VL]\n" + "fmla z8.h, z17.h, z0.h[7]\n" + "fmla z9.h, z16.h, z0.h[7]\n" + "ld1h { z17.h }, p5/Z, [x10, #-2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #-1, MUL VL]\n" "sub x27, x27, #0x8\n" "cmp x27, #0x8\n" - "fmla z10.h, z6.h, z0.h[7]\n" - "fmla z11.h, z7.h, z0.h[7]\n" + "fmla z10.h, z17.h, z0.h[7]\n" + "fmla z11.h, z16.h, z0.h[7]\n" "add x26, x26, #0x10\n" "bgt 9b\n" "10:" // Height 1: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" "ld1rqh { z0.h }, p0/Z, [x26]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "fmla z8.h, z6.h, z0.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10]\n" + "fmla z8.h, z16.h, z0.h[0]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z9.h, z16.h, z0.h[0]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x1\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[0]\n" - "fmla z11.h, z7.h, z0.h[0]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z10.h, z17.h, z0.h[0]\n" + "fmla z11.h, z16.h, z0.h[0]\n" "addvl x10, x10, #4\n" "ble 11f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[1]\n" - "fmla z9.h, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z17.h, z0.h[1]\n" + "fmla z9.h, z16.h, z0.h[1]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z10.h, z6.h, z0.h[1]\n" - "fmla z11.h, z7.h, z0.h[1]\n" + "fmla z10.h, z17.h, z0.h[1]\n" + "fmla z11.h, z16.h, z0.h[1]\n" "addvl x10, x10, #4\n" "ble 11f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[2]\n" - "fmla z9.h, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z17.h, z0.h[2]\n" + "fmla z9.h, z16.h, z0.h[2]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z10.h, z6.h, z0.h[2]\n" - "fmla z11.h, z7.h, z0.h[2]\n" + "fmla z10.h, z17.h, z0.h[2]\n" + "fmla z11.h, z16.h, z0.h[2]\n" "addvl x10, x10, #4\n" "ble 11f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[3]\n" - "fmla z9.h, z7.h, z0.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z17.h, z0.h[3]\n" + "fmla z9.h, z16.h, z0.h[3]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z10.h, z6.h, z0.h[3]\n" - "fmla z11.h, z7.h, z0.h[3]\n" + "fmla z10.h, z17.h, z0.h[3]\n" + "fmla z11.h, z16.h, z0.h[3]\n" "addvl x10, x10, #4\n" "ble 11f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[4]\n" - "fmla z9.h, z7.h, z0.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z17.h, z0.h[4]\n" + "fmla z9.h, z16.h, z0.h[4]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z10.h, z6.h, z0.h[4]\n" - "fmla z11.h, z7.h, z0.h[4]\n" + "fmla z10.h, z17.h, z0.h[4]\n" + "fmla z11.h, z16.h, z0.h[4]\n" "addvl x10, x10, #4\n" "ble 11f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[5]\n" - "fmla z9.h, z7.h, z0.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z17.h, z0.h[5]\n" + "fmla z9.h, z16.h, z0.h[5]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z10.h, z6.h, z0.h[5]\n" - "fmla z11.h, z7.h, z0.h[5]\n" + "fmla z10.h, z17.h, z0.h[5]\n" + "fmla z11.h, z16.h, z0.h[5]\n" "addvl x10, x10, #4\n" "ble 11f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[6]\n" - "fmla z9.h, z7.h, z0.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z17.h, z0.h[6]\n" + "fmla z9.h, z16.h, z0.h[6]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z10.h, z6.h, z0.h[6]\n" - "fmla z11.h, z7.h, z0.h[6]\n" + "fmla z10.h, z17.h, z0.h[6]\n" + "fmla z11.h, z16.h, z0.h[6]\n" "addvl x10, x10, #4\n" "ble 11f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[7]\n" - "fmla z9.h, z7.h, z0.h[7]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[7]\n" - "fmla z11.h, z7.h, z0.h[7]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z17.h, z0.h[7]\n" + "fmla z9.h, z16.h, z0.h[7]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z10.h, z17.h, z0.h[7]\n" + "fmla z11.h, z16.h, z0.h[7]\n" "addvl x10, x10, #4\n" "11:" // Height 1: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -322,17 +322,17 @@ void sve_hybrid_fp16_mla_6x4VL ( "bne 6b\n" "tbz %x[flags], #1, 12f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rh { z1.h }, p5/Z, [x20]\n" + "ld1rh { z17.h }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rh { z0.h }, p5/Z, [x20]\n" - "fmin z8.h, p5/M, z8.h, z1.h\n" - "fmin z9.h, p5/M, z9.h, z1.h\n" - "fmin z10.h, p5/M, z10.h, z1.h\n" - "fmin z11.h, p5/M, z11.h, z1.h\n" - "fmax z8.h, p5/M, z8.h, z0.h\n" - "fmax z9.h, p5/M, z9.h, z0.h\n" - "fmax z10.h, p5/M, z10.h, z0.h\n" - "fmax z11.h, p5/M, z11.h, z0.h\n" + "ld1rh { z16.h }, p5/Z, [x20]\n" + "fmin z8.h, p5/M, z8.h, z17.h\n" + "fmin z9.h, p5/M, z9.h, z17.h\n" + "fmin z10.h, p5/M, z10.h, z17.h\n" + "fmin z11.h, p5/M, z11.h, z17.h\n" + "fmax z8.h, p5/M, z8.h, z16.h\n" + "fmax z9.h, p5/M, z9.h, z16.h\n" + "fmax z10.h, p5/M, z10.h, z16.h\n" + "fmax z11.h, p5/M, z11.h, z16.h\n" "12:" // Height 1: No activation "st1h { z8.h }, p4, [x9]\n" "st1h { z9.h }, p3, [x9, #1, MUL VL]\n" @@ -372,15 +372,15 @@ void sve_hybrid_fp16_mla_6x4VL ( "16:" // Height 2: no bias "tbz %x[flags], #0, 17f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #1\n" + "add x20, x9, x20, LSL #1\n" "ld1h { z8.h }, p4/Z, [x9]\n" "ld1h { z9.h }, p3/Z, [x9, #1, MUL VL]\n" "ld1h { z10.h }, p2/Z, [x9, #2, MUL VL]\n" "ld1h { z11.h }, p1/Z, [x9, #3, MUL VL]\n" - "ld1h { z12.h }, p4/Z, [x25]\n" - "ld1h { z13.h }, p3/Z, [x25, #1, MUL VL]\n" - "ld1h { z14.h }, p2/Z, [x25, #2, MUL VL]\n" - "ld1h { z15.h }, p1/Z, [x25, #3, MUL VL]\n" + "ld1h { z12.h }, p4/Z, [x20]\n" + "ld1h { z13.h }, p3/Z, [x20, #1, MUL VL]\n" + "ld1h { z14.h }, p2/Z, [x20, #2, MUL VL]\n" + "ld1h { z15.h }, p1/Z, [x20, #3, MUL VL]\n" "b 18f\n" "17:" // Height 2: no accumulate "mov z8.b, #0x0\n" @@ -396,12 +396,12 @@ void sve_hybrid_fp16_mla_6x4VL ( "19:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 20f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 21f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -409,239 +409,239 @@ void sve_hybrid_fp16_mla_6x4VL ( "b 21f\n" "20:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" "21:" // Height 2: input setup done "cmp x27, #0x8\n" "ble 23f\n" "22:" // Height 2: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z0.h }, p0/Z, [x26]\n" - "ld1rqh { z1.h }, p0/Z, [x25]\n" + "ld1rqh { z1.h }, p0/Z, [x26]\n" + "ld1rqh { z0.h }, p0/Z, [x25]\n" "sub x27, x27, #0x8\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[0]\n" - "fmla z12.h, z6.h, z1.h[0]\n" - "fmla z9.h, z7.h, z0.h[0]\n" - "fmla z13.h, z7.h, z1.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[0]\n" - "fmla z14.h, z6.h, z1.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z17.h, z1.h[0]\n" + "fmla z12.h, z17.h, z0.h[0]\n" + "fmla z9.h, z16.h, z1.h[0]\n" + "fmla z13.h, z16.h, z0.h[0]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z10.h, z17.h, z1.h[0]\n" + "fmla z14.h, z17.h, z0.h[0]\n" + "ld1h { z17.h }, p5/Z, [x10, #4, MUL VL]\n" "cmp x27, #0x8\n" - "fmla z11.h, z7.h, z0.h[0]\n" - "fmla z15.h, z7.h, z1.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" + "fmla z11.h, z16.h, z1.h[0]\n" + "fmla z15.h, z16.h, z0.h[0]\n" + "ld1h { z16.h }, p5/Z, [x10, #5, MUL VL]\n" "add x26, x26, #0x10\n" - "fmla z8.h, z6.h, z0.h[1]\n" - "fmla z12.h, z6.h, z1.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" + "fmla z8.h, z17.h, z1.h[1]\n" + "fmla z12.h, z17.h, z0.h[1]\n" + "ld1h { z17.h }, p5/Z, [x10, #6, MUL VL]\n" "add x25, x25, #0x10\n" - "fmla z9.h, z7.h, z0.h[1]\n" - "fmla z13.h, z7.h, z1.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" + "fmla z9.h, z16.h, z1.h[1]\n" + "fmla z13.h, z16.h, z0.h[1]\n" + "ld1h { z16.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "fmla z10.h, z6.h, z0.h[1]\n" - "fmla z14.h, z6.h, z1.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[1]\n" - "fmla z15.h, z7.h, z1.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #-7, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[2]\n" - "fmla z12.h, z6.h, z1.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[2]\n" - "fmla z13.h, z7.h, z1.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #-5, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[2]\n" - "fmla z14.h, z6.h, z1.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[2]\n" - "fmla z15.h, z7.h, z1.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #-3, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[3]\n" - "fmla z12.h, z6.h, z1.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[3]\n" - "fmla z13.h, z7.h, z1.h[3]\n" - "ld1h { z7.h }, p5/Z, [x10, #-1, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[3]\n" - "fmla z14.h, z6.h, z1.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "fmla z11.h, z7.h, z0.h[3]\n" - "fmla z15.h, z7.h, z1.h[3]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[4]\n" - "fmla z12.h, z6.h, z1.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[4]\n" - "fmla z13.h, z7.h, z1.h[4]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[4]\n" - "fmla z14.h, z6.h, z1.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[4]\n" - "fmla z15.h, z7.h, z1.h[4]\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[5]\n" - "fmla z12.h, z6.h, z1.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[5]\n" - "fmla z13.h, z7.h, z1.h[5]\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" + "fmla z10.h, z17.h, z1.h[1]\n" + "fmla z14.h, z17.h, z0.h[1]\n" + "ld1h { z17.h }, p5/Z, [x10, #-8, MUL VL]\n" + "fmla z11.h, z16.h, z1.h[1]\n" + "fmla z15.h, z16.h, z0.h[1]\n" + "ld1h { z16.h }, p5/Z, [x10, #-7, MUL VL]\n" + "fmla z8.h, z17.h, z1.h[2]\n" + "fmla z12.h, z17.h, z0.h[2]\n" + "ld1h { z17.h }, p5/Z, [x10, #-6, MUL VL]\n" + "fmla z9.h, z16.h, z1.h[2]\n" + "fmla z13.h, z16.h, z0.h[2]\n" + "ld1h { z16.h }, p5/Z, [x10, #-5, MUL VL]\n" + "fmla z10.h, z17.h, z1.h[2]\n" + "fmla z14.h, z17.h, z0.h[2]\n" + "ld1h { z17.h }, p5/Z, [x10, #-4, MUL VL]\n" + "fmla z11.h, z16.h, z1.h[2]\n" + "fmla z15.h, z16.h, z0.h[2]\n" + "ld1h { z16.h }, p5/Z, [x10, #-3, MUL VL]\n" + "fmla z8.h, z17.h, z1.h[3]\n" + "fmla z12.h, z17.h, z0.h[3]\n" + "ld1h { z17.h }, p5/Z, [x10, #-2, MUL VL]\n" + "fmla z9.h, z16.h, z1.h[3]\n" + "fmla z13.h, z16.h, z0.h[3]\n" + "ld1h { z16.h }, p5/Z, [x10, #-1, MUL VL]\n" + "fmla z10.h, z17.h, z1.h[3]\n" + "fmla z14.h, z17.h, z0.h[3]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "fmla z11.h, z16.h, z1.h[3]\n" + "fmla z15.h, z16.h, z0.h[3]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z17.h, z1.h[4]\n" + "fmla z12.h, z17.h, z0.h[4]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.h, z16.h, z1.h[4]\n" + "fmla z13.h, z16.h, z0.h[4]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z10.h, z17.h, z1.h[4]\n" + "fmla z14.h, z17.h, z0.h[4]\n" + "ld1h { z17.h }, p5/Z, [x10, #4, MUL VL]\n" + "fmla z11.h, z16.h, z1.h[4]\n" + "fmla z15.h, z16.h, z0.h[4]\n" + "ld1h { z16.h }, p5/Z, [x10, #5, MUL VL]\n" + "fmla z8.h, z17.h, z1.h[5]\n" + "fmla z12.h, z17.h, z0.h[5]\n" + "ld1h { z17.h }, p5/Z, [x10, #6, MUL VL]\n" + "fmla z9.h, z16.h, z1.h[5]\n" + "fmla z13.h, z16.h, z0.h[5]\n" + "ld1h { z16.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "fmla z10.h, z6.h, z0.h[5]\n" - "fmla z14.h, z6.h, z1.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[5]\n" - "fmla z15.h, z7.h, z1.h[5]\n" - "ld1h { z7.h }, p5/Z, [x10, #-7, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[6]\n" - "fmla z12.h, z6.h, z1.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[6]\n" - "fmla z13.h, z7.h, z1.h[6]\n" - "ld1h { z7.h }, p5/Z, [x10, #-5, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[6]\n" - "fmla z14.h, z6.h, z1.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[6]\n" - "fmla z15.h, z7.h, z1.h[6]\n" - "ld1h { z7.h }, p5/Z, [x10, #-3, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[7]\n" - "fmla z12.h, z6.h, z1.h[7]\n" - "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[7]\n" - "fmla z13.h, z7.h, z1.h[7]\n" - "ld1h { z7.h }, p5/Z, [x10, #-1, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[7]\n" - "fmla z14.h, z6.h, z1.h[7]\n" - "fmla z11.h, z7.h, z0.h[7]\n" - "fmla z15.h, z7.h, z1.h[7]\n" + "fmla z10.h, z17.h, z1.h[5]\n" + "fmla z14.h, z17.h, z0.h[5]\n" + "ld1h { z17.h }, p5/Z, [x10, #-8, MUL VL]\n" + "fmla z11.h, z16.h, z1.h[5]\n" + "fmla z15.h, z16.h, z0.h[5]\n" + "ld1h { z16.h }, p5/Z, [x10, #-7, MUL VL]\n" + "fmla z8.h, z17.h, z1.h[6]\n" + "fmla z12.h, z17.h, z0.h[6]\n" + "ld1h { z17.h }, p5/Z, [x10, #-6, MUL VL]\n" + "fmla z9.h, z16.h, z1.h[6]\n" + "fmla z13.h, z16.h, z0.h[6]\n" + "ld1h { z16.h }, p5/Z, [x10, #-5, MUL VL]\n" + "fmla z10.h, z17.h, z1.h[6]\n" + "fmla z14.h, z17.h, z0.h[6]\n" + "ld1h { z17.h }, p5/Z, [x10, #-4, MUL VL]\n" + "fmla z11.h, z16.h, z1.h[6]\n" + "fmla z15.h, z16.h, z0.h[6]\n" + "ld1h { z16.h }, p5/Z, [x10, #-3, MUL VL]\n" + "fmla z8.h, z17.h, z1.h[7]\n" + "fmla z12.h, z17.h, z0.h[7]\n" + "ld1h { z17.h }, p5/Z, [x10, #-2, MUL VL]\n" + "fmla z9.h, z16.h, z1.h[7]\n" + "fmla z13.h, z16.h, z0.h[7]\n" + "ld1h { z16.h }, p5/Z, [x10, #-1, MUL VL]\n" + "fmla z10.h, z17.h, z1.h[7]\n" + "fmla z14.h, z17.h, z0.h[7]\n" + "fmla z11.h, z16.h, z1.h[7]\n" + "fmla z15.h, z16.h, z0.h[7]\n" "bgt 22b\n" "23:" // Height 2: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" "ld1rqh { z0.h }, p0/Z, [x26]\n" "ld1rqh { z1.h }, p0/Z, [x25]\n" "subs x27, x27, #0x1\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[0]\n" - "fmla z12.h, z6.h, z1.h[0]\n" - "fmla z9.h, z7.h, z0.h[0]\n" - "fmla z13.h, z7.h, z1.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[0]\n" - "fmla z14.h, z6.h, z1.h[0]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z17.h, z0.h[0]\n" + "fmla z12.h, z17.h, z1.h[0]\n" + "fmla z9.h, z16.h, z0.h[0]\n" + "fmla z13.h, z16.h, z1.h[0]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z10.h, z17.h, z0.h[0]\n" + "fmla z14.h, z17.h, z1.h[0]\n" "addvl x10, x10, #4\n" - "fmla z11.h, z7.h, z0.h[0]\n" - "fmla z15.h, z7.h, z1.h[0]\n" + "fmla z11.h, z16.h, z0.h[0]\n" + "fmla z15.h, z16.h, z1.h[0]\n" "ble 24f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[1]\n" - "fmla z12.h, z6.h, z1.h[1]\n" - "fmla z9.h, z7.h, z0.h[1]\n" - "fmla z13.h, z7.h, z1.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z17.h, z0.h[1]\n" + "fmla z12.h, z17.h, z1.h[1]\n" + "fmla z9.h, z16.h, z0.h[1]\n" + "fmla z13.h, z16.h, z1.h[1]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z10.h, z6.h, z0.h[1]\n" - "fmla z14.h, z6.h, z1.h[1]\n" + "fmla z10.h, z17.h, z0.h[1]\n" + "fmla z14.h, z17.h, z1.h[1]\n" "addvl x10, x10, #4\n" - "fmla z11.h, z7.h, z0.h[1]\n" - "fmla z15.h, z7.h, z1.h[1]\n" + "fmla z11.h, z16.h, z0.h[1]\n" + "fmla z15.h, z16.h, z1.h[1]\n" "ble 24f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[2]\n" - "fmla z12.h, z6.h, z1.h[2]\n" - "fmla z9.h, z7.h, z0.h[2]\n" - "fmla z13.h, z7.h, z1.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z17.h, z0.h[2]\n" + "fmla z12.h, z17.h, z1.h[2]\n" + "fmla z9.h, z16.h, z0.h[2]\n" + "fmla z13.h, z16.h, z1.h[2]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z10.h, z6.h, z0.h[2]\n" - "fmla z14.h, z6.h, z1.h[2]\n" + "fmla z10.h, z17.h, z0.h[2]\n" + "fmla z14.h, z17.h, z1.h[2]\n" "addvl x10, x10, #4\n" - "fmla z11.h, z7.h, z0.h[2]\n" - "fmla z15.h, z7.h, z1.h[2]\n" + "fmla z11.h, z16.h, z0.h[2]\n" + "fmla z15.h, z16.h, z1.h[2]\n" "ble 24f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[3]\n" - "fmla z12.h, z6.h, z1.h[3]\n" - "fmla z9.h, z7.h, z0.h[3]\n" - "fmla z13.h, z7.h, z1.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z17.h, z0.h[3]\n" + "fmla z12.h, z17.h, z1.h[3]\n" + "fmla z9.h, z16.h, z0.h[3]\n" + "fmla z13.h, z16.h, z1.h[3]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z10.h, z6.h, z0.h[3]\n" - "fmla z14.h, z6.h, z1.h[3]\n" + "fmla z10.h, z17.h, z0.h[3]\n" + "fmla z14.h, z17.h, z1.h[3]\n" "addvl x10, x10, #4\n" - "fmla z11.h, z7.h, z0.h[3]\n" - "fmla z15.h, z7.h, z1.h[3]\n" + "fmla z11.h, z16.h, z0.h[3]\n" + "fmla z15.h, z16.h, z1.h[3]\n" "ble 24f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[4]\n" - "fmla z12.h, z6.h, z1.h[4]\n" - "fmla z9.h, z7.h, z0.h[4]\n" - "fmla z13.h, z7.h, z1.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z17.h, z0.h[4]\n" + "fmla z12.h, z17.h, z1.h[4]\n" + "fmla z9.h, z16.h, z0.h[4]\n" + "fmla z13.h, z16.h, z1.h[4]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z10.h, z6.h, z0.h[4]\n" - "fmla z14.h, z6.h, z1.h[4]\n" + "fmla z10.h, z17.h, z0.h[4]\n" + "fmla z14.h, z17.h, z1.h[4]\n" "addvl x10, x10, #4\n" - "fmla z11.h, z7.h, z0.h[4]\n" - "fmla z15.h, z7.h, z1.h[4]\n" + "fmla z11.h, z16.h, z0.h[4]\n" + "fmla z15.h, z16.h, z1.h[4]\n" "ble 24f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[5]\n" - "fmla z12.h, z6.h, z1.h[5]\n" - "fmla z9.h, z7.h, z0.h[5]\n" - "fmla z13.h, z7.h, z1.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z17.h, z0.h[5]\n" + "fmla z12.h, z17.h, z1.h[5]\n" + "fmla z9.h, z16.h, z0.h[5]\n" + "fmla z13.h, z16.h, z1.h[5]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z10.h, z6.h, z0.h[5]\n" - "fmla z14.h, z6.h, z1.h[5]\n" + "fmla z10.h, z17.h, z0.h[5]\n" + "fmla z14.h, z17.h, z1.h[5]\n" "addvl x10, x10, #4\n" - "fmla z11.h, z7.h, z0.h[5]\n" - "fmla z15.h, z7.h, z1.h[5]\n" + "fmla z11.h, z16.h, z0.h[5]\n" + "fmla z15.h, z16.h, z1.h[5]\n" "ble 24f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[6]\n" - "fmla z12.h, z6.h, z1.h[6]\n" - "fmla z9.h, z7.h, z0.h[6]\n" - "fmla z13.h, z7.h, z1.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z17.h, z0.h[6]\n" + "fmla z12.h, z17.h, z1.h[6]\n" + "fmla z9.h, z16.h, z0.h[6]\n" + "fmla z13.h, z16.h, z1.h[6]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z10.h, z6.h, z0.h[6]\n" - "fmla z14.h, z6.h, z1.h[6]\n" + "fmla z10.h, z17.h, z0.h[6]\n" + "fmla z14.h, z17.h, z1.h[6]\n" "addvl x10, x10, #4\n" - "fmla z11.h, z7.h, z0.h[6]\n" - "fmla z15.h, z7.h, z1.h[6]\n" + "fmla z11.h, z16.h, z0.h[6]\n" + "fmla z15.h, z16.h, z1.h[6]\n" "ble 24f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[7]\n" - "fmla z12.h, z6.h, z1.h[7]\n" - "fmla z9.h, z7.h, z0.h[7]\n" - "fmla z13.h, z7.h, z1.h[7]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[7]\n" - "fmla z14.h, z6.h, z1.h[7]\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z17.h, z0.h[7]\n" + "fmla z12.h, z17.h, z1.h[7]\n" + "fmla z9.h, z16.h, z0.h[7]\n" + "fmla z13.h, z16.h, z1.h[7]\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z10.h, z17.h, z0.h[7]\n" + "fmla z14.h, z17.h, z1.h[7]\n" "addvl x10, x10, #4\n" - "fmla z11.h, z7.h, z0.h[7]\n" - "fmla z15.h, z7.h, z1.h[7]\n" + "fmla z11.h, z16.h, z0.h[7]\n" + "fmla z15.h, z16.h, z1.h[7]\n" "24:" // Height 2: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -651,25 +651,25 @@ void sve_hybrid_fp16_mla_6x4VL ( "add x25, x9, x20, LSL #1\n" "tbz %x[flags], #1, 25f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rh { z1.h }, p5/Z, [x20]\n" + "ld1rh { z17.h }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rh { z0.h }, p5/Z, [x20]\n" - "fmin z8.h, p5/M, z8.h, z1.h\n" - "fmin z9.h, p5/M, z9.h, z1.h\n" - "fmin z10.h, p5/M, z10.h, z1.h\n" - "fmin z11.h, p5/M, z11.h, z1.h\n" - "fmin z12.h, p5/M, z12.h, z1.h\n" - "fmin z13.h, p5/M, z13.h, z1.h\n" - "fmin z14.h, p5/M, z14.h, z1.h\n" - "fmin z15.h, p5/M, z15.h, z1.h\n" - "fmax z8.h, p5/M, z8.h, z0.h\n" - "fmax z9.h, p5/M, z9.h, z0.h\n" - "fmax z10.h, p5/M, z10.h, z0.h\n" - "fmax z11.h, p5/M, z11.h, z0.h\n" - "fmax z12.h, p5/M, z12.h, z0.h\n" - "fmax z13.h, p5/M, z13.h, z0.h\n" - "fmax z14.h, p5/M, z14.h, z0.h\n" - "fmax z15.h, p5/M, z15.h, z0.h\n" + "ld1rh { z16.h }, p5/Z, [x20]\n" + "fmin z8.h, p5/M, z8.h, z17.h\n" + "fmin z9.h, p5/M, z9.h, z17.h\n" + "fmin z10.h, p5/M, z10.h, z17.h\n" + "fmin z11.h, p5/M, z11.h, z17.h\n" + "fmin z12.h, p5/M, z12.h, z17.h\n" + "fmin z13.h, p5/M, z13.h, z17.h\n" + "fmin z14.h, p5/M, z14.h, z17.h\n" + "fmin z15.h, p5/M, z15.h, z17.h\n" + "fmax z8.h, p5/M, z8.h, z16.h\n" + "fmax z9.h, p5/M, z9.h, z16.h\n" + "fmax z10.h, p5/M, z10.h, z16.h\n" + "fmax z11.h, p5/M, z11.h, z16.h\n" + "fmax z12.h, p5/M, z12.h, z16.h\n" + "fmax z13.h, p5/M, z13.h, z16.h\n" + "fmax z14.h, p5/M, z14.h, z16.h\n" + "fmax z15.h, p5/M, z15.h, z16.h\n" "25:" // Height 2: No activation "st1h { z8.h }, p4, [x9]\n" "st1h { z9.h }, p3, [x9, #1, MUL VL]\n" @@ -717,20 +717,20 @@ void sve_hybrid_fp16_mla_6x4VL ( "29:" // Height 3: no bias "tbz %x[flags], #0, 30f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" + "add x21, x9, x20, LSL #1\n" + "add x20, x21, x20, LSL #1\n" "ld1h { z8.h }, p4/Z, [x9]\n" "ld1h { z9.h }, p3/Z, [x9, #1, MUL VL]\n" "ld1h { z10.h }, p2/Z, [x9, #2, MUL VL]\n" "ld1h { z11.h }, p1/Z, [x9, #3, MUL VL]\n" - "ld1h { z12.h }, p4/Z, [x25]\n" - "ld1h { z13.h }, p3/Z, [x25, #1, MUL VL]\n" - "ld1h { z14.h }, p2/Z, [x25, #2, MUL VL]\n" - "ld1h { z15.h }, p1/Z, [x25, #3, MUL VL]\n" - "ld1h { z16.h }, p4/Z, [x24]\n" - "ld1h { z17.h }, p3/Z, [x24, #1, MUL VL]\n" - "ld1h { z18.h }, p2/Z, [x24, #2, MUL VL]\n" - "ld1h { z19.h }, p1/Z, [x24, #3, MUL VL]\n" + "ld1h { z12.h }, p4/Z, [x21]\n" + "ld1h { z13.h }, p3/Z, [x21, #1, MUL VL]\n" + "ld1h { z14.h }, p2/Z, [x21, #2, MUL VL]\n" + "ld1h { z15.h }, p1/Z, [x21, #3, MUL VL]\n" + "ld1h { z16.h }, p4/Z, [x20]\n" + "ld1h { z17.h }, p3/Z, [x20, #1, MUL VL]\n" + "ld1h { z18.h }, p2/Z, [x20, #2, MUL VL]\n" + "ld1h { z19.h }, p1/Z, [x20, #3, MUL VL]\n" "b 31f\n" "30:" // Height 3: no accumulate "mov z8.b, #0x0\n" @@ -750,13 +750,13 @@ void sve_hybrid_fp16_mla_6x4VL ( "32:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 33f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 34f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -765,151 +765,151 @@ void sve_hybrid_fp16_mla_6x4VL ( "b 34f\n" "33:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" "34:" // Height 3: input setup done "cmp x27, #0x8\n" "ble 36f\n" "35:" // Height 3: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z0.h }, p0/Z, [x26]\n" + "ld1rqh { z2.h }, p0/Z, [x26]\n" "ld1rqh { z1.h }, p0/Z, [x25]\n" "sub x27, x27, #0x8\n" - "ld1rqh { z2.h }, p0/Z, [x24]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "fmla z8.h, z6.h, z0.h[0]\n" - "fmla z12.h, z6.h, z1.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z16.h, z6.h, z2.h[0]\n" - "fmla z9.h, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[0]\n" - "fmla z17.h, z7.h, z2.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1rqh { z0.h }, p0/Z, [x24]\n" + "ld1h { z21.h }, p5/Z, [x10]\n" + "fmla z8.h, z21.h, z2.h[0]\n" + "fmla z12.h, z21.h, z1.h[0]\n" + "ld1h { z20.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z16.h, z21.h, z0.h[0]\n" + "fmla z9.h, z20.h, z2.h[0]\n" + "ld1h { z21.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z13.h, z20.h, z1.h[0]\n" + "fmla z17.h, z20.h, z0.h[0]\n" + "ld1h { z20.h }, p5/Z, [x10, #3, MUL VL]\n" "cmp x27, #0x8\n" - "fmla z10.h, z6.h, z0.h[0]\n" - "fmla z14.h, z6.h, z1.h[0]\n" + "fmla z10.h, z21.h, z2.h[0]\n" + "fmla z14.h, z21.h, z1.h[0]\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" - "fmla z18.h, z6.h, z2.h[0]\n" - "fmla z11.h, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" + "fmla z18.h, z21.h, z0.h[0]\n" + "fmla z11.h, z20.h, z2.h[0]\n" + "ld1h { z21.h }, p5/Z, [x10, #4, MUL VL]\n" "add x24, x24, #0x10\n" - "fmla z15.h, z7.h, z1.h[0]\n" - "fmla z19.h, z7.h, z2.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[1]\n" - "fmla z12.h, z6.h, z1.h[1]\n" - "fmla z16.h, z6.h, z2.h[1]\n" - "fmla z9.h, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[1]\n" - "fmla z17.h, z7.h, z2.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" + "fmla z15.h, z20.h, z1.h[0]\n" + "fmla z19.h, z20.h, z0.h[0]\n" + "ld1h { z20.h }, p5/Z, [x10, #5, MUL VL]\n" + "fmla z8.h, z21.h, z2.h[1]\n" + "fmla z12.h, z21.h, z1.h[1]\n" + "fmla z16.h, z21.h, z0.h[1]\n" + "fmla z9.h, z20.h, z2.h[1]\n" + "ld1h { z21.h }, p5/Z, [x10, #6, MUL VL]\n" + "fmla z13.h, z20.h, z1.h[1]\n" + "fmla z17.h, z20.h, z0.h[1]\n" + "ld1h { z20.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "fmla z10.h, z6.h, z0.h[1]\n" - "fmla z14.h, z6.h, z1.h[1]\n" - "fmla z18.h, z6.h, z2.h[1]\n" - "fmla z11.h, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n" - "fmla z15.h, z7.h, z1.h[1]\n" - "fmla z19.h, z7.h, z2.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #-7, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[2]\n" - "fmla z12.h, z6.h, z1.h[2]\n" - "fmla z16.h, z6.h, z2.h[2]\n" - "fmla z9.h, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[2]\n" - "fmla z17.h, z7.h, z2.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #-5, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[2]\n" - "fmla z14.h, z6.h, z1.h[2]\n" - "fmla z18.h, z6.h, z2.h[2]\n" - "fmla z11.h, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n" - "fmla z15.h, z7.h, z1.h[2]\n" - "fmla z19.h, z7.h, z2.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #-3, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[3]\n" - "fmla z12.h, z6.h, z1.h[3]\n" - "fmla z16.h, z6.h, z2.h[3]\n" - "fmla z9.h, z7.h, z0.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[3]\n" - "fmla z17.h, z7.h, z2.h[3]\n" - "ld1h { z7.h }, p5/Z, [x10, #-1, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[3]\n" - "fmla z14.h, z6.h, z1.h[3]\n" - "fmla z18.h, z6.h, z2.h[3]\n" - "fmla z11.h, z7.h, z0.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "fmla z15.h, z7.h, z1.h[3]\n" - "fmla z19.h, z7.h, z2.h[3]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[4]\n" - "fmla z12.h, z6.h, z1.h[4]\n" - "fmla z16.h, z6.h, z2.h[4]\n" - "fmla z9.h, z7.h, z0.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[4]\n" - "fmla z17.h, z7.h, z2.h[4]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[4]\n" - "fmla z14.h, z6.h, z1.h[4]\n" - "fmla z18.h, z6.h, z2.h[4]\n" - "fmla z11.h, z7.h, z0.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - "fmla z15.h, z7.h, z1.h[4]\n" - "fmla z19.h, z7.h, z2.h[4]\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[5]\n" - "fmla z12.h, z6.h, z1.h[5]\n" - "fmla z16.h, z6.h, z2.h[5]\n" - "fmla z9.h, z7.h, z0.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[5]\n" - "fmla z17.h, z7.h, z2.h[5]\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" + "fmla z10.h, z21.h, z2.h[1]\n" + "fmla z14.h, z21.h, z1.h[1]\n" + "fmla z18.h, z21.h, z0.h[1]\n" + "fmla z11.h, z20.h, z2.h[1]\n" + "ld1h { z21.h }, p5/Z, [x10, #-8, MUL VL]\n" + "fmla z15.h, z20.h, z1.h[1]\n" + "fmla z19.h, z20.h, z0.h[1]\n" + "ld1h { z20.h }, p5/Z, [x10, #-7, MUL VL]\n" + "fmla z8.h, z21.h, z2.h[2]\n" + "fmla z12.h, z21.h, z1.h[2]\n" + "fmla z16.h, z21.h, z0.h[2]\n" + "fmla z9.h, z20.h, z2.h[2]\n" + "ld1h { z21.h }, p5/Z, [x10, #-6, MUL VL]\n" + "fmla z13.h, z20.h, z1.h[2]\n" + "fmla z17.h, z20.h, z0.h[2]\n" + "ld1h { z20.h }, p5/Z, [x10, #-5, MUL VL]\n" + "fmla z10.h, z21.h, z2.h[2]\n" + "fmla z14.h, z21.h, z1.h[2]\n" + "fmla z18.h, z21.h, z0.h[2]\n" + "fmla z11.h, z20.h, z2.h[2]\n" + "ld1h { z21.h }, p5/Z, [x10, #-4, MUL VL]\n" + "fmla z15.h, z20.h, z1.h[2]\n" + "fmla z19.h, z20.h, z0.h[2]\n" + "ld1h { z20.h }, p5/Z, [x10, #-3, MUL VL]\n" + "fmla z8.h, z21.h, z2.h[3]\n" + "fmla z12.h, z21.h, z1.h[3]\n" + "fmla z16.h, z21.h, z0.h[3]\n" + "fmla z9.h, z20.h, z2.h[3]\n" + "ld1h { z21.h }, p5/Z, [x10, #-2, MUL VL]\n" + "fmla z13.h, z20.h, z1.h[3]\n" + "fmla z17.h, z20.h, z0.h[3]\n" + "ld1h { z20.h }, p5/Z, [x10, #-1, MUL VL]\n" + "fmla z10.h, z21.h, z2.h[3]\n" + "fmla z14.h, z21.h, z1.h[3]\n" + "fmla z18.h, z21.h, z0.h[3]\n" + "fmla z11.h, z20.h, z2.h[3]\n" + "ld1h { z21.h }, p5/Z, [x10]\n" + "fmla z15.h, z20.h, z1.h[3]\n" + "fmla z19.h, z20.h, z0.h[3]\n" + "ld1h { z20.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z21.h, z2.h[4]\n" + "fmla z12.h, z21.h, z1.h[4]\n" + "fmla z16.h, z21.h, z0.h[4]\n" + "fmla z9.h, z20.h, z2.h[4]\n" + "ld1h { z21.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z13.h, z20.h, z1.h[4]\n" + "fmla z17.h, z20.h, z0.h[4]\n" + "ld1h { z20.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z10.h, z21.h, z2.h[4]\n" + "fmla z14.h, z21.h, z1.h[4]\n" + "fmla z18.h, z21.h, z0.h[4]\n" + "fmla z11.h, z20.h, z2.h[4]\n" + "ld1h { z21.h }, p5/Z, [x10, #4, MUL VL]\n" + "fmla z15.h, z20.h, z1.h[4]\n" + "fmla z19.h, z20.h, z0.h[4]\n" + "ld1h { z20.h }, p5/Z, [x10, #5, MUL VL]\n" + "fmla z8.h, z21.h, z2.h[5]\n" + "fmla z12.h, z21.h, z1.h[5]\n" + "fmla z16.h, z21.h, z0.h[5]\n" + "fmla z9.h, z20.h, z2.h[5]\n" + "ld1h { z21.h }, p5/Z, [x10, #6, MUL VL]\n" + "fmla z13.h, z20.h, z1.h[5]\n" + "fmla z17.h, z20.h, z0.h[5]\n" + "ld1h { z20.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "fmla z10.h, z6.h, z0.h[5]\n" - "fmla z14.h, z6.h, z1.h[5]\n" - "fmla z18.h, z6.h, z2.h[5]\n" - "fmla z11.h, z7.h, z0.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n" - "fmla z15.h, z7.h, z1.h[5]\n" - "fmla z19.h, z7.h, z2.h[5]\n" - "ld1h { z7.h }, p5/Z, [x10, #-7, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[6]\n" - "fmla z12.h, z6.h, z1.h[6]\n" - "fmla z16.h, z6.h, z2.h[6]\n" - "fmla z9.h, z7.h, z0.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[6]\n" - "fmla z17.h, z7.h, z2.h[6]\n" - "ld1h { z7.h }, p5/Z, [x10, #-5, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[6]\n" - "fmla z14.h, z6.h, z1.h[6]\n" - "fmla z18.h, z6.h, z2.h[6]\n" - "fmla z11.h, z7.h, z0.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n" - "fmla z15.h, z7.h, z1.h[6]\n" - "fmla z19.h, z7.h, z2.h[6]\n" - "ld1h { z7.h }, p5/Z, [x10, #-3, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[7]\n" - "fmla z12.h, z6.h, z1.h[7]\n" - "fmla z16.h, z6.h, z2.h[7]\n" - "fmla z9.h, z7.h, z0.h[7]\n" - "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[7]\n" - "fmla z17.h, z7.h, z2.h[7]\n" - "ld1h { z7.h }, p5/Z, [x10, #-1, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[7]\n" - "fmla z14.h, z6.h, z1.h[7]\n" - "fmla z18.h, z6.h, z2.h[7]\n" - "fmla z11.h, z7.h, z0.h[7]\n" - "fmla z15.h, z7.h, z1.h[7]\n" - "fmla z19.h, z7.h, z2.h[7]\n" + "fmla z10.h, z21.h, z2.h[5]\n" + "fmla z14.h, z21.h, z1.h[5]\n" + "fmla z18.h, z21.h, z0.h[5]\n" + "fmla z11.h, z20.h, z2.h[5]\n" + "ld1h { z21.h }, p5/Z, [x10, #-8, MUL VL]\n" + "fmla z15.h, z20.h, z1.h[5]\n" + "fmla z19.h, z20.h, z0.h[5]\n" + "ld1h { z20.h }, p5/Z, [x10, #-7, MUL VL]\n" + "fmla z8.h, z21.h, z2.h[6]\n" + "fmla z12.h, z21.h, z1.h[6]\n" + "fmla z16.h, z21.h, z0.h[6]\n" + "fmla z9.h, z20.h, z2.h[6]\n" + "ld1h { z21.h }, p5/Z, [x10, #-6, MUL VL]\n" + "fmla z13.h, z20.h, z1.h[6]\n" + "fmla z17.h, z20.h, z0.h[6]\n" + "ld1h { z20.h }, p5/Z, [x10, #-5, MUL VL]\n" + "fmla z10.h, z21.h, z2.h[6]\n" + "fmla z14.h, z21.h, z1.h[6]\n" + "fmla z18.h, z21.h, z0.h[6]\n" + "fmla z11.h, z20.h, z2.h[6]\n" + "ld1h { z21.h }, p5/Z, [x10, #-4, MUL VL]\n" + "fmla z15.h, z20.h, z1.h[6]\n" + "fmla z19.h, z20.h, z0.h[6]\n" + "ld1h { z20.h }, p5/Z, [x10, #-3, MUL VL]\n" + "fmla z8.h, z21.h, z2.h[7]\n" + "fmla z12.h, z21.h, z1.h[7]\n" + "fmla z16.h, z21.h, z0.h[7]\n" + "fmla z9.h, z20.h, z2.h[7]\n" + "ld1h { z21.h }, p5/Z, [x10, #-2, MUL VL]\n" + "fmla z13.h, z20.h, z1.h[7]\n" + "fmla z17.h, z20.h, z0.h[7]\n" + "ld1h { z20.h }, p5/Z, [x10, #-1, MUL VL]\n" + "fmla z10.h, z21.h, z2.h[7]\n" + "fmla z14.h, z21.h, z1.h[7]\n" + "fmla z18.h, z21.h, z0.h[7]\n" + "fmla z11.h, z20.h, z2.h[7]\n" + "fmla z15.h, z20.h, z1.h[7]\n" + "fmla z19.h, z20.h, z0.h[7]\n" "bgt 35b\n" "36:" // Height 3: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" @@ -917,155 +917,155 @@ void sve_hybrid_fp16_mla_6x4VL ( "ld1rqh { z1.h }, p0/Z, [x25]\n" "subs x27, x27, #0x1\n" "ld1rqh { z2.h }, p0/Z, [x24]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "fmla z8.h, z6.h, z0.h[0]\n" - "fmla z12.h, z6.h, z1.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z16.h, z6.h, z2.h[0]\n" - "fmla z9.h, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[0]\n" - "fmla z17.h, z7.h, z2.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z21.h }, p5/Z, [x10]\n" + "fmla z8.h, z21.h, z0.h[0]\n" + "fmla z12.h, z21.h, z1.h[0]\n" + "ld1h { z20.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z16.h, z21.h, z2.h[0]\n" + "fmla z9.h, z20.h, z0.h[0]\n" + "ld1h { z21.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z13.h, z20.h, z1.h[0]\n" + "fmla z17.h, z20.h, z2.h[0]\n" + "ld1h { z20.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[0]\n" - "fmla z14.h, z6.h, z1.h[0]\n" - "fmla z18.h, z6.h, z2.h[0]\n" - "fmla z11.h, z7.h, z0.h[0]\n" - "fmla z15.h, z7.h, z1.h[0]\n" - "fmla z19.h, z7.h, z2.h[0]\n" + "fmla z10.h, z21.h, z0.h[0]\n" + "fmla z14.h, z21.h, z1.h[0]\n" + "fmla z18.h, z21.h, z2.h[0]\n" + "fmla z11.h, z20.h, z0.h[0]\n" + "fmla z15.h, z20.h, z1.h[0]\n" + "fmla z19.h, z20.h, z2.h[0]\n" "ble 37f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[1]\n" - "fmla z12.h, z6.h, z1.h[1]\n" - "fmla z16.h, z6.h, z2.h[1]\n" - "fmla z9.h, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z21.h }, p5/Z, [x10]\n" + "ld1h { z20.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z21.h, z0.h[1]\n" + "fmla z12.h, z21.h, z1.h[1]\n" + "fmla z16.h, z21.h, z2.h[1]\n" + "fmla z9.h, z20.h, z0.h[1]\n" + "ld1h { z21.h }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z13.h, z7.h, z1.h[1]\n" - "fmla z17.h, z7.h, z2.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z13.h, z20.h, z1.h[1]\n" + "fmla z17.h, z20.h, z2.h[1]\n" + "ld1h { z20.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[1]\n" - "fmla z14.h, z6.h, z1.h[1]\n" - "fmla z18.h, z6.h, z2.h[1]\n" - "fmla z11.h, z7.h, z0.h[1]\n" - "fmla z15.h, z7.h, z1.h[1]\n" - "fmla z19.h, z7.h, z2.h[1]\n" + "fmla z10.h, z21.h, z0.h[1]\n" + "fmla z14.h, z21.h, z1.h[1]\n" + "fmla z18.h, z21.h, z2.h[1]\n" + "fmla z11.h, z20.h, z0.h[1]\n" + "fmla z15.h, z20.h, z1.h[1]\n" + "fmla z19.h, z20.h, z2.h[1]\n" "ble 37f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[2]\n" - "fmla z12.h, z6.h, z1.h[2]\n" - "fmla z16.h, z6.h, z2.h[2]\n" - "fmla z9.h, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z21.h }, p5/Z, [x10]\n" + "ld1h { z20.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z21.h, z0.h[2]\n" + "fmla z12.h, z21.h, z1.h[2]\n" + "fmla z16.h, z21.h, z2.h[2]\n" + "fmla z9.h, z20.h, z0.h[2]\n" + "ld1h { z21.h }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z13.h, z7.h, z1.h[2]\n" - "fmla z17.h, z7.h, z2.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z13.h, z20.h, z1.h[2]\n" + "fmla z17.h, z20.h, z2.h[2]\n" + "ld1h { z20.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[2]\n" - "fmla z14.h, z6.h, z1.h[2]\n" - "fmla z18.h, z6.h, z2.h[2]\n" - "fmla z11.h, z7.h, z0.h[2]\n" - "fmla z15.h, z7.h, z1.h[2]\n" - "fmla z19.h, z7.h, z2.h[2]\n" + "fmla z10.h, z21.h, z0.h[2]\n" + "fmla z14.h, z21.h, z1.h[2]\n" + "fmla z18.h, z21.h, z2.h[2]\n" + "fmla z11.h, z20.h, z0.h[2]\n" + "fmla z15.h, z20.h, z1.h[2]\n" + "fmla z19.h, z20.h, z2.h[2]\n" "ble 37f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[3]\n" - "fmla z12.h, z6.h, z1.h[3]\n" - "fmla z16.h, z6.h, z2.h[3]\n" - "fmla z9.h, z7.h, z0.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z21.h }, p5/Z, [x10]\n" + "ld1h { z20.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z21.h, z0.h[3]\n" + "fmla z12.h, z21.h, z1.h[3]\n" + "fmla z16.h, z21.h, z2.h[3]\n" + "fmla z9.h, z20.h, z0.h[3]\n" + "ld1h { z21.h }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z13.h, z7.h, z1.h[3]\n" - "fmla z17.h, z7.h, z2.h[3]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z13.h, z20.h, z1.h[3]\n" + "fmla z17.h, z20.h, z2.h[3]\n" + "ld1h { z20.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[3]\n" - "fmla z14.h, z6.h, z1.h[3]\n" - "fmla z18.h, z6.h, z2.h[3]\n" - "fmla z11.h, z7.h, z0.h[3]\n" - "fmla z15.h, z7.h, z1.h[3]\n" - "fmla z19.h, z7.h, z2.h[3]\n" + "fmla z10.h, z21.h, z0.h[3]\n" + "fmla z14.h, z21.h, z1.h[3]\n" + "fmla z18.h, z21.h, z2.h[3]\n" + "fmla z11.h, z20.h, z0.h[3]\n" + "fmla z15.h, z20.h, z1.h[3]\n" + "fmla z19.h, z20.h, z2.h[3]\n" "ble 37f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[4]\n" - "fmla z12.h, z6.h, z1.h[4]\n" - "fmla z16.h, z6.h, z2.h[4]\n" - "fmla z9.h, z7.h, z0.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z21.h }, p5/Z, [x10]\n" + "ld1h { z20.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z21.h, z0.h[4]\n" + "fmla z12.h, z21.h, z1.h[4]\n" + "fmla z16.h, z21.h, z2.h[4]\n" + "fmla z9.h, z20.h, z0.h[4]\n" + "ld1h { z21.h }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z13.h, z7.h, z1.h[4]\n" - "fmla z17.h, z7.h, z2.h[4]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z13.h, z20.h, z1.h[4]\n" + "fmla z17.h, z20.h, z2.h[4]\n" + "ld1h { z20.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[4]\n" - "fmla z14.h, z6.h, z1.h[4]\n" - "fmla z18.h, z6.h, z2.h[4]\n" - "fmla z11.h, z7.h, z0.h[4]\n" - "fmla z15.h, z7.h, z1.h[4]\n" - "fmla z19.h, z7.h, z2.h[4]\n" + "fmla z10.h, z21.h, z0.h[4]\n" + "fmla z14.h, z21.h, z1.h[4]\n" + "fmla z18.h, z21.h, z2.h[4]\n" + "fmla z11.h, z20.h, z0.h[4]\n" + "fmla z15.h, z20.h, z1.h[4]\n" + "fmla z19.h, z20.h, z2.h[4]\n" "ble 37f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[5]\n" - "fmla z12.h, z6.h, z1.h[5]\n" - "fmla z16.h, z6.h, z2.h[5]\n" - "fmla z9.h, z7.h, z0.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z21.h }, p5/Z, [x10]\n" + "ld1h { z20.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z21.h, z0.h[5]\n" + "fmla z12.h, z21.h, z1.h[5]\n" + "fmla z16.h, z21.h, z2.h[5]\n" + "fmla z9.h, z20.h, z0.h[5]\n" + "ld1h { z21.h }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z13.h, z7.h, z1.h[5]\n" - "fmla z17.h, z7.h, z2.h[5]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z13.h, z20.h, z1.h[5]\n" + "fmla z17.h, z20.h, z2.h[5]\n" + "ld1h { z20.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[5]\n" - "fmla z14.h, z6.h, z1.h[5]\n" - "fmla z18.h, z6.h, z2.h[5]\n" - "fmla z11.h, z7.h, z0.h[5]\n" - "fmla z15.h, z7.h, z1.h[5]\n" - "fmla z19.h, z7.h, z2.h[5]\n" + "fmla z10.h, z21.h, z0.h[5]\n" + "fmla z14.h, z21.h, z1.h[5]\n" + "fmla z18.h, z21.h, z2.h[5]\n" + "fmla z11.h, z20.h, z0.h[5]\n" + "fmla z15.h, z20.h, z1.h[5]\n" + "fmla z19.h, z20.h, z2.h[5]\n" "ble 37f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[6]\n" - "fmla z12.h, z6.h, z1.h[6]\n" - "fmla z16.h, z6.h, z2.h[6]\n" - "fmla z9.h, z7.h, z0.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z21.h }, p5/Z, [x10]\n" + "ld1h { z20.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z21.h, z0.h[6]\n" + "fmla z12.h, z21.h, z1.h[6]\n" + "fmla z16.h, z21.h, z2.h[6]\n" + "fmla z9.h, z20.h, z0.h[6]\n" + "ld1h { z21.h }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z13.h, z7.h, z1.h[6]\n" - "fmla z17.h, z7.h, z2.h[6]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z13.h, z20.h, z1.h[6]\n" + "fmla z17.h, z20.h, z2.h[6]\n" + "ld1h { z20.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[6]\n" - "fmla z14.h, z6.h, z1.h[6]\n" - "fmla z18.h, z6.h, z2.h[6]\n" - "fmla z11.h, z7.h, z0.h[6]\n" - "fmla z15.h, z7.h, z1.h[6]\n" - "fmla z19.h, z7.h, z2.h[6]\n" + "fmla z10.h, z21.h, z0.h[6]\n" + "fmla z14.h, z21.h, z1.h[6]\n" + "fmla z18.h, z21.h, z2.h[6]\n" + "fmla z11.h, z20.h, z0.h[6]\n" + "fmla z15.h, z20.h, z1.h[6]\n" + "fmla z19.h, z20.h, z2.h[6]\n" "ble 37f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[7]\n" - "fmla z12.h, z6.h, z1.h[7]\n" - "fmla z16.h, z6.h, z2.h[7]\n" - "fmla z9.h, z7.h, z0.h[7]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[7]\n" - "fmla z17.h, z7.h, z2.h[7]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z21.h }, p5/Z, [x10]\n" + "ld1h { z20.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z21.h, z0.h[7]\n" + "fmla z12.h, z21.h, z1.h[7]\n" + "fmla z16.h, z21.h, z2.h[7]\n" + "fmla z9.h, z20.h, z0.h[7]\n" + "ld1h { z21.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z13.h, z20.h, z1.h[7]\n" + "fmla z17.h, z20.h, z2.h[7]\n" + "ld1h { z20.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[7]\n" - "fmla z14.h, z6.h, z1.h[7]\n" - "fmla z18.h, z6.h, z2.h[7]\n" - "fmla z11.h, z7.h, z0.h[7]\n" - "fmla z15.h, z7.h, z1.h[7]\n" - "fmla z19.h, z7.h, z2.h[7]\n" + "fmla z10.h, z21.h, z0.h[7]\n" + "fmla z14.h, z21.h, z1.h[7]\n" + "fmla z18.h, z21.h, z2.h[7]\n" + "fmla z11.h, z20.h, z0.h[7]\n" + "fmla z15.h, z20.h, z1.h[7]\n" + "fmla z19.h, z20.h, z2.h[7]\n" "37:" // Height 3: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1076,33 +1076,33 @@ void sve_hybrid_fp16_mla_6x4VL ( "add x24, x25, x20, LSL #1\n" "tbz %x[flags], #1, 38f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rh { z1.h }, p5/Z, [x20]\n" + "ld1rh { z21.h }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rh { z0.h }, p5/Z, [x20]\n" - "fmin z8.h, p5/M, z8.h, z1.h\n" - "fmin z9.h, p5/M, z9.h, z1.h\n" - "fmin z10.h, p5/M, z10.h, z1.h\n" - "fmin z11.h, p5/M, z11.h, z1.h\n" - "fmin z12.h, p5/M, z12.h, z1.h\n" - "fmin z13.h, p5/M, z13.h, z1.h\n" - "fmin z14.h, p5/M, z14.h, z1.h\n" - "fmin z15.h, p5/M, z15.h, z1.h\n" - "fmin z16.h, p5/M, z16.h, z1.h\n" - "fmin z17.h, p5/M, z17.h, z1.h\n" - "fmin z18.h, p5/M, z18.h, z1.h\n" - "fmin z19.h, p5/M, z19.h, z1.h\n" - "fmax z8.h, p5/M, z8.h, z0.h\n" - "fmax z9.h, p5/M, z9.h, z0.h\n" - "fmax z10.h, p5/M, z10.h, z0.h\n" - "fmax z11.h, p5/M, z11.h, z0.h\n" - "fmax z12.h, p5/M, z12.h, z0.h\n" - "fmax z13.h, p5/M, z13.h, z0.h\n" - "fmax z14.h, p5/M, z14.h, z0.h\n" - "fmax z15.h, p5/M, z15.h, z0.h\n" - "fmax z16.h, p5/M, z16.h, z0.h\n" - "fmax z17.h, p5/M, z17.h, z0.h\n" - "fmax z18.h, p5/M, z18.h, z0.h\n" - "fmax z19.h, p5/M, z19.h, z0.h\n" + "ld1rh { z20.h }, p5/Z, [x20]\n" + "fmin z8.h, p5/M, z8.h, z21.h\n" + "fmin z9.h, p5/M, z9.h, z21.h\n" + "fmin z10.h, p5/M, z10.h, z21.h\n" + "fmin z11.h, p5/M, z11.h, z21.h\n" + "fmin z12.h, p5/M, z12.h, z21.h\n" + "fmin z13.h, p5/M, z13.h, z21.h\n" + "fmin z14.h, p5/M, z14.h, z21.h\n" + "fmin z15.h, p5/M, z15.h, z21.h\n" + "fmin z16.h, p5/M, z16.h, z21.h\n" + "fmin z17.h, p5/M, z17.h, z21.h\n" + "fmin z18.h, p5/M, z18.h, z21.h\n" + "fmin z19.h, p5/M, z19.h, z21.h\n" + "fmax z8.h, p5/M, z8.h, z20.h\n" + "fmax z9.h, p5/M, z9.h, z20.h\n" + "fmax z10.h, p5/M, z10.h, z20.h\n" + "fmax z11.h, p5/M, z11.h, z20.h\n" + "fmax z12.h, p5/M, z12.h, z20.h\n" + "fmax z13.h, p5/M, z13.h, z20.h\n" + "fmax z14.h, p5/M, z14.h, z20.h\n" + "fmax z15.h, p5/M, z15.h, z20.h\n" + "fmax z16.h, p5/M, z16.h, z20.h\n" + "fmax z17.h, p5/M, z17.h, z20.h\n" + "fmax z18.h, p5/M, z18.h, z20.h\n" + "fmax z19.h, p5/M, z19.h, z20.h\n" "38:" // Height 3: No activation "st1h { z8.h }, p4, [x9]\n" "st1h { z9.h }, p3, [x9, #1, MUL VL]\n" @@ -1158,25 +1158,25 @@ void sve_hybrid_fp16_mla_6x4VL ( "42:" // Height 4: no bias "tbz %x[flags], #0, 43f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" + "add x22, x9, x20, LSL #1\n" + "add x21, x22, x20, LSL #1\n" "ld1h { z8.h }, p4/Z, [x9]\n" - "add x23, x24, x20, LSL #1\n" + "add x20, x21, x20, LSL #1\n" "ld1h { z9.h }, p3/Z, [x9, #1, MUL VL]\n" "ld1h { z10.h }, p2/Z, [x9, #2, MUL VL]\n" "ld1h { z11.h }, p1/Z, [x9, #3, MUL VL]\n" - "ld1h { z12.h }, p4/Z, [x25]\n" - "ld1h { z13.h }, p3/Z, [x25, #1, MUL VL]\n" - "ld1h { z14.h }, p2/Z, [x25, #2, MUL VL]\n" - "ld1h { z15.h }, p1/Z, [x25, #3, MUL VL]\n" - "ld1h { z16.h }, p4/Z, [x24]\n" - "ld1h { z17.h }, p3/Z, [x24, #1, MUL VL]\n" - "ld1h { z18.h }, p2/Z, [x24, #2, MUL VL]\n" - "ld1h { z19.h }, p1/Z, [x24, #3, MUL VL]\n" - "ld1h { z20.h }, p4/Z, [x23]\n" - "ld1h { z21.h }, p3/Z, [x23, #1, MUL VL]\n" - "ld1h { z22.h }, p2/Z, [x23, #2, MUL VL]\n" - "ld1h { z23.h }, p1/Z, [x23, #3, MUL VL]\n" + "ld1h { z12.h }, p4/Z, [x22]\n" + "ld1h { z13.h }, p3/Z, [x22, #1, MUL VL]\n" + "ld1h { z14.h }, p2/Z, [x22, #2, MUL VL]\n" + "ld1h { z15.h }, p1/Z, [x22, #3, MUL VL]\n" + "ld1h { z16.h }, p4/Z, [x21]\n" + "ld1h { z17.h }, p3/Z, [x21, #1, MUL VL]\n" + "ld1h { z18.h }, p2/Z, [x21, #2, MUL VL]\n" + "ld1h { z19.h }, p1/Z, [x21, #3, MUL VL]\n" + "ld1h { z20.h }, p4/Z, [x20]\n" + "ld1h { z21.h }, p3/Z, [x20, #1, MUL VL]\n" + "ld1h { z22.h }, p2/Z, [x20, #2, MUL VL]\n" + "ld1h { z23.h }, p1/Z, [x20, #3, MUL VL]\n" "b 44f\n" "43:" // Height 4: no accumulate "mov z8.b, #0x0\n" @@ -1200,14 +1200,14 @@ void sve_hybrid_fp16_mla_6x4VL ( "45:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 46f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 47f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -1217,186 +1217,186 @@ void sve_hybrid_fp16_mla_6x4VL ( "b 47f\n" "46:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" "47:" // Height 4: input setup done "cmp x27, #0x8\n" "ble 49f\n" "48:" // Height 4: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z0.h }, p0/Z, [x26]\n" - "ld1rqh { z1.h }, p0/Z, [x25]\n" + "ld1rqh { z3.h }, p0/Z, [x26]\n" + "ld1rqh { z2.h }, p0/Z, [x25]\n" "sub x27, x27, #0x8\n" - "ld1rqh { z2.h }, p0/Z, [x24]\n" - "ld1rqh { z3.h }, p0/Z, [x23]\n" + "ld1rqh { z1.h }, p0/Z, [x24]\n" + "ld1rqh { z0.h }, p0/Z, [x23]\n" "cmp x27, #0x8\n" "add x26, x26, #0x10\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[0]\n" - "fmla z12.h, z6.h, z1.h[0]\n" - "fmla z16.h, z6.h, z2.h[0]\n" - "fmla z20.h, z6.h, z3.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z25.h, z3.h[0]\n" + "fmla z12.h, z25.h, z2.h[0]\n" + "fmla z16.h, z25.h, z1.h[0]\n" + "fmla z20.h, z25.h, z0.h[0]\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" "add x25, x25, #0x10\n" - "fmla z9.h, z7.h, z0.h[0]\n" - "fmla z13.h, z7.h, z1.h[0]\n" + "fmla z9.h, z24.h, z3.h[0]\n" + "fmla z13.h, z24.h, z2.h[0]\n" "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" - "fmla z17.h, z7.h, z2.h[0]\n" - "fmla z21.h, z7.h, z3.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[0]\n" - "fmla z14.h, z6.h, z1.h[0]\n" - "fmla z18.h, z6.h, z2.h[0]\n" - "fmla z22.h, z6.h, z3.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[0]\n" - "fmla z15.h, z7.h, z1.h[0]\n" - "fmla z19.h, z7.h, z2.h[0]\n" - "fmla z23.h, z7.h, z3.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[1]\n" - "fmla z12.h, z6.h, z1.h[1]\n" - "fmla z16.h, z6.h, z2.h[1]\n" - "fmla z20.h, z6.h, z3.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[1]\n" - "fmla z13.h, z7.h, z1.h[1]\n" - "fmla z17.h, z7.h, z2.h[1]\n" - "fmla z21.h, z7.h, z3.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" + "fmla z17.h, z24.h, z1.h[0]\n" + "fmla z21.h, z24.h, z0.h[0]\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z10.h, z25.h, z3.h[0]\n" + "fmla z14.h, z25.h, z2.h[0]\n" + "fmla z18.h, z25.h, z1.h[0]\n" + "fmla z22.h, z25.h, z0.h[0]\n" + "ld1h { z25.h }, p5/Z, [x10, #4, MUL VL]\n" + "fmla z11.h, z24.h, z3.h[0]\n" + "fmla z15.h, z24.h, z2.h[0]\n" + "fmla z19.h, z24.h, z1.h[0]\n" + "fmla z23.h, z24.h, z0.h[0]\n" + "ld1h { z24.h }, p5/Z, [x10, #5, MUL VL]\n" + "fmla z8.h, z25.h, z3.h[1]\n" + "fmla z12.h, z25.h, z2.h[1]\n" + "fmla z16.h, z25.h, z1.h[1]\n" + "fmla z20.h, z25.h, z0.h[1]\n" + "ld1h { z25.h }, p5/Z, [x10, #6, MUL VL]\n" + "fmla z9.h, z24.h, z3.h[1]\n" + "fmla z13.h, z24.h, z2.h[1]\n" + "fmla z17.h, z24.h, z1.h[1]\n" + "fmla z21.h, z24.h, z0.h[1]\n" + "ld1h { z24.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "fmla z10.h, z6.h, z0.h[1]\n" - "fmla z14.h, z6.h, z1.h[1]\n" - "fmla z18.h, z6.h, z2.h[1]\n" - "fmla z22.h, z6.h, z3.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[1]\n" - "fmla z15.h, z7.h, z1.h[1]\n" - "fmla z19.h, z7.h, z2.h[1]\n" - "fmla z23.h, z7.h, z3.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #-7, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[2]\n" - "fmla z12.h, z6.h, z1.h[2]\n" - "fmla z16.h, z6.h, z2.h[2]\n" - "fmla z20.h, z6.h, z3.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[2]\n" - "fmla z13.h, z7.h, z1.h[2]\n" - "fmla z17.h, z7.h, z2.h[2]\n" - "fmla z21.h, z7.h, z3.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #-5, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[2]\n" - "fmla z14.h, z6.h, z1.h[2]\n" - "fmla z18.h, z6.h, z2.h[2]\n" - "fmla z22.h, z6.h, z3.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[2]\n" - "fmla z15.h, z7.h, z1.h[2]\n" - "fmla z19.h, z7.h, z2.h[2]\n" - "fmla z23.h, z7.h, z3.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #-3, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[3]\n" - "fmla z12.h, z6.h, z1.h[3]\n" - "fmla z16.h, z6.h, z2.h[3]\n" - "fmla z20.h, z6.h, z3.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[3]\n" - "fmla z13.h, z7.h, z1.h[3]\n" - "fmla z17.h, z7.h, z2.h[3]\n" - "fmla z21.h, z7.h, z3.h[3]\n" - "ld1h { z7.h }, p5/Z, [x10, #-1, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[3]\n" - "fmla z14.h, z6.h, z1.h[3]\n" - "fmla z18.h, z6.h, z2.h[3]\n" - "fmla z22.h, z6.h, z3.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "fmla z11.h, z7.h, z0.h[3]\n" - "fmla z15.h, z7.h, z1.h[3]\n" - "fmla z19.h, z7.h, z2.h[3]\n" - "fmla z23.h, z7.h, z3.h[3]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[4]\n" - "fmla z12.h, z6.h, z1.h[4]\n" - "fmla z16.h, z6.h, z2.h[4]\n" - "fmla z20.h, z6.h, z3.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[4]\n" - "fmla z13.h, z7.h, z1.h[4]\n" - "fmla z17.h, z7.h, z2.h[4]\n" - "fmla z21.h, z7.h, z3.h[4]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[4]\n" - "fmla z14.h, z6.h, z1.h[4]\n" - "fmla z18.h, z6.h, z2.h[4]\n" - "fmla z22.h, z6.h, z3.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[4]\n" - "fmla z15.h, z7.h, z1.h[4]\n" - "fmla z19.h, z7.h, z2.h[4]\n" - "fmla z23.h, z7.h, z3.h[4]\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[5]\n" - "fmla z12.h, z6.h, z1.h[5]\n" - "fmla z16.h, z6.h, z2.h[5]\n" - "fmla z20.h, z6.h, z3.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[5]\n" - "fmla z13.h, z7.h, z1.h[5]\n" - "fmla z17.h, z7.h, z2.h[5]\n" - "fmla z21.h, z7.h, z3.h[5]\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" + "fmla z10.h, z25.h, z3.h[1]\n" + "fmla z14.h, z25.h, z2.h[1]\n" + "fmla z18.h, z25.h, z1.h[1]\n" + "fmla z22.h, z25.h, z0.h[1]\n" + "ld1h { z25.h }, p5/Z, [x10, #-8, MUL VL]\n" + "fmla z11.h, z24.h, z3.h[1]\n" + "fmla z15.h, z24.h, z2.h[1]\n" + "fmla z19.h, z24.h, z1.h[1]\n" + "fmla z23.h, z24.h, z0.h[1]\n" + "ld1h { z24.h }, p5/Z, [x10, #-7, MUL VL]\n" + "fmla z8.h, z25.h, z3.h[2]\n" + "fmla z12.h, z25.h, z2.h[2]\n" + "fmla z16.h, z25.h, z1.h[2]\n" + "fmla z20.h, z25.h, z0.h[2]\n" + "ld1h { z25.h }, p5/Z, [x10, #-6, MUL VL]\n" + "fmla z9.h, z24.h, z3.h[2]\n" + "fmla z13.h, z24.h, z2.h[2]\n" + "fmla z17.h, z24.h, z1.h[2]\n" + "fmla z21.h, z24.h, z0.h[2]\n" + "ld1h { z24.h }, p5/Z, [x10, #-5, MUL VL]\n" + "fmla z10.h, z25.h, z3.h[2]\n" + "fmla z14.h, z25.h, z2.h[2]\n" + "fmla z18.h, z25.h, z1.h[2]\n" + "fmla z22.h, z25.h, z0.h[2]\n" + "ld1h { z25.h }, p5/Z, [x10, #-4, MUL VL]\n" + "fmla z11.h, z24.h, z3.h[2]\n" + "fmla z15.h, z24.h, z2.h[2]\n" + "fmla z19.h, z24.h, z1.h[2]\n" + "fmla z23.h, z24.h, z0.h[2]\n" + "ld1h { z24.h }, p5/Z, [x10, #-3, MUL VL]\n" + "fmla z8.h, z25.h, z3.h[3]\n" + "fmla z12.h, z25.h, z2.h[3]\n" + "fmla z16.h, z25.h, z1.h[3]\n" + "fmla z20.h, z25.h, z0.h[3]\n" + "ld1h { z25.h }, p5/Z, [x10, #-2, MUL VL]\n" + "fmla z9.h, z24.h, z3.h[3]\n" + "fmla z13.h, z24.h, z2.h[3]\n" + "fmla z17.h, z24.h, z1.h[3]\n" + "fmla z21.h, z24.h, z0.h[3]\n" + "ld1h { z24.h }, p5/Z, [x10, #-1, MUL VL]\n" + "fmla z10.h, z25.h, z3.h[3]\n" + "fmla z14.h, z25.h, z2.h[3]\n" + "fmla z18.h, z25.h, z1.h[3]\n" + "fmla z22.h, z25.h, z0.h[3]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "fmla z11.h, z24.h, z3.h[3]\n" + "fmla z15.h, z24.h, z2.h[3]\n" + "fmla z19.h, z24.h, z1.h[3]\n" + "fmla z23.h, z24.h, z0.h[3]\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z25.h, z3.h[4]\n" + "fmla z12.h, z25.h, z2.h[4]\n" + "fmla z16.h, z25.h, z1.h[4]\n" + "fmla z20.h, z25.h, z0.h[4]\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.h, z24.h, z3.h[4]\n" + "fmla z13.h, z24.h, z2.h[4]\n" + "fmla z17.h, z24.h, z1.h[4]\n" + "fmla z21.h, z24.h, z0.h[4]\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z10.h, z25.h, z3.h[4]\n" + "fmla z14.h, z25.h, z2.h[4]\n" + "fmla z18.h, z25.h, z1.h[4]\n" + "fmla z22.h, z25.h, z0.h[4]\n" + "ld1h { z25.h }, p5/Z, [x10, #4, MUL VL]\n" + "fmla z11.h, z24.h, z3.h[4]\n" + "fmla z15.h, z24.h, z2.h[4]\n" + "fmla z19.h, z24.h, z1.h[4]\n" + "fmla z23.h, z24.h, z0.h[4]\n" + "ld1h { z24.h }, p5/Z, [x10, #5, MUL VL]\n" + "fmla z8.h, z25.h, z3.h[5]\n" + "fmla z12.h, z25.h, z2.h[5]\n" + "fmla z16.h, z25.h, z1.h[5]\n" + "fmla z20.h, z25.h, z0.h[5]\n" + "ld1h { z25.h }, p5/Z, [x10, #6, MUL VL]\n" + "fmla z9.h, z24.h, z3.h[5]\n" + "fmla z13.h, z24.h, z2.h[5]\n" + "fmla z17.h, z24.h, z1.h[5]\n" + "fmla z21.h, z24.h, z0.h[5]\n" + "ld1h { z24.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "fmla z10.h, z6.h, z0.h[5]\n" - "fmla z14.h, z6.h, z1.h[5]\n" - "fmla z18.h, z6.h, z2.h[5]\n" - "fmla z22.h, z6.h, z3.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[5]\n" - "fmla z15.h, z7.h, z1.h[5]\n" - "fmla z19.h, z7.h, z2.h[5]\n" - "fmla z23.h, z7.h, z3.h[5]\n" - "ld1h { z7.h }, p5/Z, [x10, #-7, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[6]\n" - "fmla z12.h, z6.h, z1.h[6]\n" - "fmla z16.h, z6.h, z2.h[6]\n" - "fmla z20.h, z6.h, z3.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[6]\n" - "fmla z13.h, z7.h, z1.h[6]\n" - "fmla z17.h, z7.h, z2.h[6]\n" - "fmla z21.h, z7.h, z3.h[6]\n" - "ld1h { z7.h }, p5/Z, [x10, #-5, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[6]\n" - "fmla z14.h, z6.h, z1.h[6]\n" - "fmla z18.h, z6.h, z2.h[6]\n" - "fmla z22.h, z6.h, z3.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[6]\n" - "fmla z15.h, z7.h, z1.h[6]\n" - "fmla z19.h, z7.h, z2.h[6]\n" - "fmla z23.h, z7.h, z3.h[6]\n" - "ld1h { z7.h }, p5/Z, [x10, #-3, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[7]\n" - "fmla z12.h, z6.h, z1.h[7]\n" - "fmla z16.h, z6.h, z2.h[7]\n" - "fmla z20.h, z6.h, z3.h[7]\n" - "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[7]\n" - "fmla z13.h, z7.h, z1.h[7]\n" - "fmla z17.h, z7.h, z2.h[7]\n" - "fmla z21.h, z7.h, z3.h[7]\n" - "ld1h { z7.h }, p5/Z, [x10, #-1, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[7]\n" - "fmla z14.h, z6.h, z1.h[7]\n" - "fmla z18.h, z6.h, z2.h[7]\n" - "fmla z22.h, z6.h, z3.h[7]\n" - "fmla z11.h, z7.h, z0.h[7]\n" - "fmla z15.h, z7.h, z1.h[7]\n" - "fmla z19.h, z7.h, z2.h[7]\n" - "fmla z23.h, z7.h, z3.h[7]\n" + "fmla z10.h, z25.h, z3.h[5]\n" + "fmla z14.h, z25.h, z2.h[5]\n" + "fmla z18.h, z25.h, z1.h[5]\n" + "fmla z22.h, z25.h, z0.h[5]\n" + "ld1h { z25.h }, p5/Z, [x10, #-8, MUL VL]\n" + "fmla z11.h, z24.h, z3.h[5]\n" + "fmla z15.h, z24.h, z2.h[5]\n" + "fmla z19.h, z24.h, z1.h[5]\n" + "fmla z23.h, z24.h, z0.h[5]\n" + "ld1h { z24.h }, p5/Z, [x10, #-7, MUL VL]\n" + "fmla z8.h, z25.h, z3.h[6]\n" + "fmla z12.h, z25.h, z2.h[6]\n" + "fmla z16.h, z25.h, z1.h[6]\n" + "fmla z20.h, z25.h, z0.h[6]\n" + "ld1h { z25.h }, p5/Z, [x10, #-6, MUL VL]\n" + "fmla z9.h, z24.h, z3.h[6]\n" + "fmla z13.h, z24.h, z2.h[6]\n" + "fmla z17.h, z24.h, z1.h[6]\n" + "fmla z21.h, z24.h, z0.h[6]\n" + "ld1h { z24.h }, p5/Z, [x10, #-5, MUL VL]\n" + "fmla z10.h, z25.h, z3.h[6]\n" + "fmla z14.h, z25.h, z2.h[6]\n" + "fmla z18.h, z25.h, z1.h[6]\n" + "fmla z22.h, z25.h, z0.h[6]\n" + "ld1h { z25.h }, p5/Z, [x10, #-4, MUL VL]\n" + "fmla z11.h, z24.h, z3.h[6]\n" + "fmla z15.h, z24.h, z2.h[6]\n" + "fmla z19.h, z24.h, z1.h[6]\n" + "fmla z23.h, z24.h, z0.h[6]\n" + "ld1h { z24.h }, p5/Z, [x10, #-3, MUL VL]\n" + "fmla z8.h, z25.h, z3.h[7]\n" + "fmla z12.h, z25.h, z2.h[7]\n" + "fmla z16.h, z25.h, z1.h[7]\n" + "fmla z20.h, z25.h, z0.h[7]\n" + "ld1h { z25.h }, p5/Z, [x10, #-2, MUL VL]\n" + "fmla z9.h, z24.h, z3.h[7]\n" + "fmla z13.h, z24.h, z2.h[7]\n" + "fmla z17.h, z24.h, z1.h[7]\n" + "fmla z21.h, z24.h, z0.h[7]\n" + "ld1h { z24.h }, p5/Z, [x10, #-1, MUL VL]\n" + "fmla z10.h, z25.h, z3.h[7]\n" + "fmla z14.h, z25.h, z2.h[7]\n" + "fmla z18.h, z25.h, z1.h[7]\n" + "fmla z22.h, z25.h, z0.h[7]\n" + "fmla z11.h, z24.h, z3.h[7]\n" + "fmla z15.h, z24.h, z2.h[7]\n" + "fmla z19.h, z24.h, z1.h[7]\n" + "fmla z23.h, z24.h, z0.h[7]\n" "bgt 48b\n" "49:" // Height 4: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" @@ -1405,187 +1405,187 @@ void sve_hybrid_fp16_mla_6x4VL ( "subs x27, x27, #0x1\n" "ld1rqh { z2.h }, p0/Z, [x24]\n" "ld1rqh { z3.h }, p0/Z, [x23]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[0]\n" - "fmla z12.h, z6.h, z1.h[0]\n" - "fmla z16.h, z6.h, z2.h[0]\n" - "fmla z20.h, z6.h, z3.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[0]\n" - "fmla z13.h, z7.h, z1.h[0]\n" - "fmla z17.h, z7.h, z2.h[0]\n" - "fmla z21.h, z7.h, z3.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z25.h, z0.h[0]\n" + "fmla z12.h, z25.h, z1.h[0]\n" + "fmla z16.h, z25.h, z2.h[0]\n" + "fmla z20.h, z25.h, z3.h[0]\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.h, z24.h, z0.h[0]\n" + "fmla z13.h, z24.h, z1.h[0]\n" + "fmla z17.h, z24.h, z2.h[0]\n" + "fmla z21.h, z24.h, z3.h[0]\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[0]\n" - "fmla z14.h, z6.h, z1.h[0]\n" - "fmla z18.h, z6.h, z2.h[0]\n" - "fmla z22.h, z6.h, z3.h[0]\n" - "fmla z11.h, z7.h, z0.h[0]\n" - "fmla z15.h, z7.h, z1.h[0]\n" - "fmla z19.h, z7.h, z2.h[0]\n" - "fmla z23.h, z7.h, z3.h[0]\n" + "fmla z10.h, z25.h, z0.h[0]\n" + "fmla z14.h, z25.h, z1.h[0]\n" + "fmla z18.h, z25.h, z2.h[0]\n" + "fmla z22.h, z25.h, z3.h[0]\n" + "fmla z11.h, z24.h, z0.h[0]\n" + "fmla z15.h, z24.h, z1.h[0]\n" + "fmla z19.h, z24.h, z2.h[0]\n" + "fmla z23.h, z24.h, z3.h[0]\n" "ble 50f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[1]\n" - "fmla z12.h, z6.h, z1.h[1]\n" - "fmla z16.h, z6.h, z2.h[1]\n" - "fmla z20.h, z6.h, z3.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z25.h, z0.h[1]\n" + "fmla z12.h, z25.h, z1.h[1]\n" + "fmla z16.h, z25.h, z2.h[1]\n" + "fmla z20.h, z25.h, z3.h[1]\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z9.h, z7.h, z0.h[1]\n" - "fmla z13.h, z7.h, z1.h[1]\n" - "fmla z17.h, z7.h, z2.h[1]\n" - "fmla z21.h, z7.h, z3.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z9.h, z24.h, z0.h[1]\n" + "fmla z13.h, z24.h, z1.h[1]\n" + "fmla z17.h, z24.h, z2.h[1]\n" + "fmla z21.h, z24.h, z3.h[1]\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[1]\n" - "fmla z14.h, z6.h, z1.h[1]\n" - "fmla z18.h, z6.h, z2.h[1]\n" - "fmla z22.h, z6.h, z3.h[1]\n" - "fmla z11.h, z7.h, z0.h[1]\n" - "fmla z15.h, z7.h, z1.h[1]\n" - "fmla z19.h, z7.h, z2.h[1]\n" - "fmla z23.h, z7.h, z3.h[1]\n" + "fmla z10.h, z25.h, z0.h[1]\n" + "fmla z14.h, z25.h, z1.h[1]\n" + "fmla z18.h, z25.h, z2.h[1]\n" + "fmla z22.h, z25.h, z3.h[1]\n" + "fmla z11.h, z24.h, z0.h[1]\n" + "fmla z15.h, z24.h, z1.h[1]\n" + "fmla z19.h, z24.h, z2.h[1]\n" + "fmla z23.h, z24.h, z3.h[1]\n" "ble 50f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[2]\n" - "fmla z12.h, z6.h, z1.h[2]\n" - "fmla z16.h, z6.h, z2.h[2]\n" - "fmla z20.h, z6.h, z3.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z25.h, z0.h[2]\n" + "fmla z12.h, z25.h, z1.h[2]\n" + "fmla z16.h, z25.h, z2.h[2]\n" + "fmla z20.h, z25.h, z3.h[2]\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z9.h, z7.h, z0.h[2]\n" - "fmla z13.h, z7.h, z1.h[2]\n" - "fmla z17.h, z7.h, z2.h[2]\n" - "fmla z21.h, z7.h, z3.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z9.h, z24.h, z0.h[2]\n" + "fmla z13.h, z24.h, z1.h[2]\n" + "fmla z17.h, z24.h, z2.h[2]\n" + "fmla z21.h, z24.h, z3.h[2]\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[2]\n" - "fmla z14.h, z6.h, z1.h[2]\n" - "fmla z18.h, z6.h, z2.h[2]\n" - "fmla z22.h, z6.h, z3.h[2]\n" - "fmla z11.h, z7.h, z0.h[2]\n" - "fmla z15.h, z7.h, z1.h[2]\n" - "fmla z19.h, z7.h, z2.h[2]\n" - "fmla z23.h, z7.h, z3.h[2]\n" + "fmla z10.h, z25.h, z0.h[2]\n" + "fmla z14.h, z25.h, z1.h[2]\n" + "fmla z18.h, z25.h, z2.h[2]\n" + "fmla z22.h, z25.h, z3.h[2]\n" + "fmla z11.h, z24.h, z0.h[2]\n" + "fmla z15.h, z24.h, z1.h[2]\n" + "fmla z19.h, z24.h, z2.h[2]\n" + "fmla z23.h, z24.h, z3.h[2]\n" "ble 50f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[3]\n" - "fmla z12.h, z6.h, z1.h[3]\n" - "fmla z16.h, z6.h, z2.h[3]\n" - "fmla z20.h, z6.h, z3.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z25.h, z0.h[3]\n" + "fmla z12.h, z25.h, z1.h[3]\n" + "fmla z16.h, z25.h, z2.h[3]\n" + "fmla z20.h, z25.h, z3.h[3]\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z9.h, z7.h, z0.h[3]\n" - "fmla z13.h, z7.h, z1.h[3]\n" - "fmla z17.h, z7.h, z2.h[3]\n" - "fmla z21.h, z7.h, z3.h[3]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z9.h, z24.h, z0.h[3]\n" + "fmla z13.h, z24.h, z1.h[3]\n" + "fmla z17.h, z24.h, z2.h[3]\n" + "fmla z21.h, z24.h, z3.h[3]\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[3]\n" - "fmla z14.h, z6.h, z1.h[3]\n" - "fmla z18.h, z6.h, z2.h[3]\n" - "fmla z22.h, z6.h, z3.h[3]\n" - "fmla z11.h, z7.h, z0.h[3]\n" - "fmla z15.h, z7.h, z1.h[3]\n" - "fmla z19.h, z7.h, z2.h[3]\n" - "fmla z23.h, z7.h, z3.h[3]\n" + "fmla z10.h, z25.h, z0.h[3]\n" + "fmla z14.h, z25.h, z1.h[3]\n" + "fmla z18.h, z25.h, z2.h[3]\n" + "fmla z22.h, z25.h, z3.h[3]\n" + "fmla z11.h, z24.h, z0.h[3]\n" + "fmla z15.h, z24.h, z1.h[3]\n" + "fmla z19.h, z24.h, z2.h[3]\n" + "fmla z23.h, z24.h, z3.h[3]\n" "ble 50f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[4]\n" - "fmla z12.h, z6.h, z1.h[4]\n" - "fmla z16.h, z6.h, z2.h[4]\n" - "fmla z20.h, z6.h, z3.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z25.h, z0.h[4]\n" + "fmla z12.h, z25.h, z1.h[4]\n" + "fmla z16.h, z25.h, z2.h[4]\n" + "fmla z20.h, z25.h, z3.h[4]\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z9.h, z7.h, z0.h[4]\n" - "fmla z13.h, z7.h, z1.h[4]\n" - "fmla z17.h, z7.h, z2.h[4]\n" - "fmla z21.h, z7.h, z3.h[4]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z9.h, z24.h, z0.h[4]\n" + "fmla z13.h, z24.h, z1.h[4]\n" + "fmla z17.h, z24.h, z2.h[4]\n" + "fmla z21.h, z24.h, z3.h[4]\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[4]\n" - "fmla z14.h, z6.h, z1.h[4]\n" - "fmla z18.h, z6.h, z2.h[4]\n" - "fmla z22.h, z6.h, z3.h[4]\n" - "fmla z11.h, z7.h, z0.h[4]\n" - "fmla z15.h, z7.h, z1.h[4]\n" - "fmla z19.h, z7.h, z2.h[4]\n" - "fmla z23.h, z7.h, z3.h[4]\n" + "fmla z10.h, z25.h, z0.h[4]\n" + "fmla z14.h, z25.h, z1.h[4]\n" + "fmla z18.h, z25.h, z2.h[4]\n" + "fmla z22.h, z25.h, z3.h[4]\n" + "fmla z11.h, z24.h, z0.h[4]\n" + "fmla z15.h, z24.h, z1.h[4]\n" + "fmla z19.h, z24.h, z2.h[4]\n" + "fmla z23.h, z24.h, z3.h[4]\n" "ble 50f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[5]\n" - "fmla z12.h, z6.h, z1.h[5]\n" - "fmla z16.h, z6.h, z2.h[5]\n" - "fmla z20.h, z6.h, z3.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z25.h, z0.h[5]\n" + "fmla z12.h, z25.h, z1.h[5]\n" + "fmla z16.h, z25.h, z2.h[5]\n" + "fmla z20.h, z25.h, z3.h[5]\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z9.h, z7.h, z0.h[5]\n" - "fmla z13.h, z7.h, z1.h[5]\n" - "fmla z17.h, z7.h, z2.h[5]\n" - "fmla z21.h, z7.h, z3.h[5]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z9.h, z24.h, z0.h[5]\n" + "fmla z13.h, z24.h, z1.h[5]\n" + "fmla z17.h, z24.h, z2.h[5]\n" + "fmla z21.h, z24.h, z3.h[5]\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[5]\n" - "fmla z14.h, z6.h, z1.h[5]\n" - "fmla z18.h, z6.h, z2.h[5]\n" - "fmla z22.h, z6.h, z3.h[5]\n" - "fmla z11.h, z7.h, z0.h[5]\n" - "fmla z15.h, z7.h, z1.h[5]\n" - "fmla z19.h, z7.h, z2.h[5]\n" - "fmla z23.h, z7.h, z3.h[5]\n" + "fmla z10.h, z25.h, z0.h[5]\n" + "fmla z14.h, z25.h, z1.h[5]\n" + "fmla z18.h, z25.h, z2.h[5]\n" + "fmla z22.h, z25.h, z3.h[5]\n" + "fmla z11.h, z24.h, z0.h[5]\n" + "fmla z15.h, z24.h, z1.h[5]\n" + "fmla z19.h, z24.h, z2.h[5]\n" + "fmla z23.h, z24.h, z3.h[5]\n" "ble 50f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[6]\n" - "fmla z12.h, z6.h, z1.h[6]\n" - "fmla z16.h, z6.h, z2.h[6]\n" - "fmla z20.h, z6.h, z3.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z25.h, z0.h[6]\n" + "fmla z12.h, z25.h, z1.h[6]\n" + "fmla z16.h, z25.h, z2.h[6]\n" + "fmla z20.h, z25.h, z3.h[6]\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z9.h, z7.h, z0.h[6]\n" - "fmla z13.h, z7.h, z1.h[6]\n" - "fmla z17.h, z7.h, z2.h[6]\n" - "fmla z21.h, z7.h, z3.h[6]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z9.h, z24.h, z0.h[6]\n" + "fmla z13.h, z24.h, z1.h[6]\n" + "fmla z17.h, z24.h, z2.h[6]\n" + "fmla z21.h, z24.h, z3.h[6]\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[6]\n" - "fmla z14.h, z6.h, z1.h[6]\n" - "fmla z18.h, z6.h, z2.h[6]\n" - "fmla z22.h, z6.h, z3.h[6]\n" - "fmla z11.h, z7.h, z0.h[6]\n" - "fmla z15.h, z7.h, z1.h[6]\n" - "fmla z19.h, z7.h, z2.h[6]\n" - "fmla z23.h, z7.h, z3.h[6]\n" + "fmla z10.h, z25.h, z0.h[6]\n" + "fmla z14.h, z25.h, z1.h[6]\n" + "fmla z18.h, z25.h, z2.h[6]\n" + "fmla z22.h, z25.h, z3.h[6]\n" + "fmla z11.h, z24.h, z0.h[6]\n" + "fmla z15.h, z24.h, z1.h[6]\n" + "fmla z19.h, z24.h, z2.h[6]\n" + "fmla z23.h, z24.h, z3.h[6]\n" "ble 50f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[7]\n" - "fmla z12.h, z6.h, z1.h[7]\n" - "fmla z16.h, z6.h, z2.h[7]\n" - "fmla z20.h, z6.h, z3.h[7]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[7]\n" - "fmla z13.h, z7.h, z1.h[7]\n" - "fmla z17.h, z7.h, z2.h[7]\n" - "fmla z21.h, z7.h, z3.h[7]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z25.h, z0.h[7]\n" + "fmla z12.h, z25.h, z1.h[7]\n" + "fmla z16.h, z25.h, z2.h[7]\n" + "fmla z20.h, z25.h, z3.h[7]\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.h, z24.h, z0.h[7]\n" + "fmla z13.h, z24.h, z1.h[7]\n" + "fmla z17.h, z24.h, z2.h[7]\n" + "fmla z21.h, z24.h, z3.h[7]\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[7]\n" - "fmla z14.h, z6.h, z1.h[7]\n" - "fmla z18.h, z6.h, z2.h[7]\n" - "fmla z22.h, z6.h, z3.h[7]\n" - "fmla z11.h, z7.h, z0.h[7]\n" - "fmla z15.h, z7.h, z1.h[7]\n" - "fmla z19.h, z7.h, z2.h[7]\n" - "fmla z23.h, z7.h, z3.h[7]\n" + "fmla z10.h, z25.h, z0.h[7]\n" + "fmla z14.h, z25.h, z1.h[7]\n" + "fmla z18.h, z25.h, z2.h[7]\n" + "fmla z22.h, z25.h, z3.h[7]\n" + "fmla z11.h, z24.h, z0.h[7]\n" + "fmla z15.h, z24.h, z1.h[7]\n" + "fmla z19.h, z24.h, z2.h[7]\n" + "fmla z23.h, z24.h, z3.h[7]\n" "50:" // Height 4: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1597,41 +1597,41 @@ void sve_hybrid_fp16_mla_6x4VL ( "add x23, x24, x20, LSL #1\n" "tbz %x[flags], #1, 51f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rh { z1.h }, p5/Z, [x20]\n" + "ld1rh { z25.h }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rh { z0.h }, p5/Z, [x20]\n" - "fmin z8.h, p5/M, z8.h, z1.h\n" - "fmin z9.h, p5/M, z9.h, z1.h\n" - "fmin z10.h, p5/M, z10.h, z1.h\n" - "fmin z11.h, p5/M, z11.h, z1.h\n" - "fmin z12.h, p5/M, z12.h, z1.h\n" - "fmin z13.h, p5/M, z13.h, z1.h\n" - "fmin z14.h, p5/M, z14.h, z1.h\n" - "fmin z15.h, p5/M, z15.h, z1.h\n" - "fmin z16.h, p5/M, z16.h, z1.h\n" - "fmin z17.h, p5/M, z17.h, z1.h\n" - "fmin z18.h, p5/M, z18.h, z1.h\n" - "fmin z19.h, p5/M, z19.h, z1.h\n" - "fmin z20.h, p5/M, z20.h, z1.h\n" - "fmin z21.h, p5/M, z21.h, z1.h\n" - "fmin z22.h, p5/M, z22.h, z1.h\n" - "fmin z23.h, p5/M, z23.h, z1.h\n" - "fmax z8.h, p5/M, z8.h, z0.h\n" - "fmax z9.h, p5/M, z9.h, z0.h\n" - "fmax z10.h, p5/M, z10.h, z0.h\n" - "fmax z11.h, p5/M, z11.h, z0.h\n" - "fmax z12.h, p5/M, z12.h, z0.h\n" - "fmax z13.h, p5/M, z13.h, z0.h\n" - "fmax z14.h, p5/M, z14.h, z0.h\n" - "fmax z15.h, p5/M, z15.h, z0.h\n" - "fmax z16.h, p5/M, z16.h, z0.h\n" - "fmax z17.h, p5/M, z17.h, z0.h\n" - "fmax z18.h, p5/M, z18.h, z0.h\n" - "fmax z19.h, p5/M, z19.h, z0.h\n" - "fmax z20.h, p5/M, z20.h, z0.h\n" - "fmax z21.h, p5/M, z21.h, z0.h\n" - "fmax z22.h, p5/M, z22.h, z0.h\n" - "fmax z23.h, p5/M, z23.h, z0.h\n" + "ld1rh { z24.h }, p5/Z, [x20]\n" + "fmin z8.h, p5/M, z8.h, z25.h\n" + "fmin z9.h, p5/M, z9.h, z25.h\n" + "fmin z10.h, p5/M, z10.h, z25.h\n" + "fmin z11.h, p5/M, z11.h, z25.h\n" + "fmin z12.h, p5/M, z12.h, z25.h\n" + "fmin z13.h, p5/M, z13.h, z25.h\n" + "fmin z14.h, p5/M, z14.h, z25.h\n" + "fmin z15.h, p5/M, z15.h, z25.h\n" + "fmin z16.h, p5/M, z16.h, z25.h\n" + "fmin z17.h, p5/M, z17.h, z25.h\n" + "fmin z18.h, p5/M, z18.h, z25.h\n" + "fmin z19.h, p5/M, z19.h, z25.h\n" + "fmin z20.h, p5/M, z20.h, z25.h\n" + "fmin z21.h, p5/M, z21.h, z25.h\n" + "fmin z22.h, p5/M, z22.h, z25.h\n" + "fmin z23.h, p5/M, z23.h, z25.h\n" + "fmax z8.h, p5/M, z8.h, z24.h\n" + "fmax z9.h, p5/M, z9.h, z24.h\n" + "fmax z10.h, p5/M, z10.h, z24.h\n" + "fmax z11.h, p5/M, z11.h, z24.h\n" + "fmax z12.h, p5/M, z12.h, z24.h\n" + "fmax z13.h, p5/M, z13.h, z24.h\n" + "fmax z14.h, p5/M, z14.h, z24.h\n" + "fmax z15.h, p5/M, z15.h, z24.h\n" + "fmax z16.h, p5/M, z16.h, z24.h\n" + "fmax z17.h, p5/M, z17.h, z24.h\n" + "fmax z18.h, p5/M, z18.h, z24.h\n" + "fmax z19.h, p5/M, z19.h, z24.h\n" + "fmax z20.h, p5/M, z20.h, z24.h\n" + "fmax z21.h, p5/M, z21.h, z24.h\n" + "fmax z22.h, p5/M, z22.h, z24.h\n" + "fmax z23.h, p5/M, z23.h, z24.h\n" "51:" // Height 4: No activation "st1h { z8.h }, p4, [x9]\n" "st1h { z9.h }, p3, [x9, #1, MUL VL]\n" @@ -1695,30 +1695,30 @@ void sve_hybrid_fp16_mla_6x4VL ( "55:" // Height 5: no bias "tbz %x[flags], #0, 56f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "ld1h { z8.h }, p4/Z, [x9]\n" - "add x23, x24, x20, LSL #1\n" + "add x23, x9, x20, LSL #1\n" "add x22, x23, x20, LSL #1\n" + "ld1h { z8.h }, p4/Z, [x9]\n" + "add x21, x22, x20, LSL #1\n" + "add x20, x21, x20, LSL #1\n" "ld1h { z9.h }, p3/Z, [x9, #1, MUL VL]\n" "ld1h { z10.h }, p2/Z, [x9, #2, MUL VL]\n" "ld1h { z11.h }, p1/Z, [x9, #3, MUL VL]\n" - "ld1h { z12.h }, p4/Z, [x25]\n" - "ld1h { z13.h }, p3/Z, [x25, #1, MUL VL]\n" - "ld1h { z14.h }, p2/Z, [x25, #2, MUL VL]\n" - "ld1h { z15.h }, p1/Z, [x25, #3, MUL VL]\n" - "ld1h { z16.h }, p4/Z, [x24]\n" - "ld1h { z17.h }, p3/Z, [x24, #1, MUL VL]\n" - "ld1h { z18.h }, p2/Z, [x24, #2, MUL VL]\n" - "ld1h { z19.h }, p1/Z, [x24, #3, MUL VL]\n" - "ld1h { z20.h }, p4/Z, [x23]\n" - "ld1h { z21.h }, p3/Z, [x23, #1, MUL VL]\n" - "ld1h { z22.h }, p2/Z, [x23, #2, MUL VL]\n" - "ld1h { z23.h }, p1/Z, [x23, #3, MUL VL]\n" - "ld1h { z24.h }, p4/Z, [x22]\n" - "ld1h { z25.h }, p3/Z, [x22, #1, MUL VL]\n" - "ld1h { z26.h }, p2/Z, [x22, #2, MUL VL]\n" - "ld1h { z27.h }, p1/Z, [x22, #3, MUL VL]\n" + "ld1h { z12.h }, p4/Z, [x23]\n" + "ld1h { z13.h }, p3/Z, [x23, #1, MUL VL]\n" + "ld1h { z14.h }, p2/Z, [x23, #2, MUL VL]\n" + "ld1h { z15.h }, p1/Z, [x23, #3, MUL VL]\n" + "ld1h { z16.h }, p4/Z, [x22]\n" + "ld1h { z17.h }, p3/Z, [x22, #1, MUL VL]\n" + "ld1h { z18.h }, p2/Z, [x22, #2, MUL VL]\n" + "ld1h { z19.h }, p1/Z, [x22, #3, MUL VL]\n" + "ld1h { z20.h }, p4/Z, [x21]\n" + "ld1h { z21.h }, p3/Z, [x21, #1, MUL VL]\n" + "ld1h { z22.h }, p2/Z, [x21, #2, MUL VL]\n" + "ld1h { z23.h }, p1/Z, [x21, #3, MUL VL]\n" + "ld1h { z24.h }, p4/Z, [x20]\n" + "ld1h { z25.h }, p3/Z, [x20, #1, MUL VL]\n" + "ld1h { z26.h }, p2/Z, [x20, #2, MUL VL]\n" + "ld1h { z27.h }, p1/Z, [x20, #3, MUL VL]\n" "b 57f\n" "56:" // Height 5: no accumulate "mov z8.b, #0x0\n" @@ -1746,15 +1746,15 @@ void sve_hybrid_fp16_mla_6x4VL ( "58:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 59f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 60f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -1765,221 +1765,221 @@ void sve_hybrid_fp16_mla_6x4VL ( "b 60f\n" "59:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" - "add x22, x23, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" + "add x22, x23, x21, LSL #1\n" "60:" // Height 5: input setup done "cmp x27, #0x8\n" "ble 62f\n" "61:" // Height 5: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z0.h }, p0/Z, [x26]\n" - "ld1rqh { z1.h }, p0/Z, [x25]\n" + "ld1rqh { z4.h }, p0/Z, [x26]\n" + "ld1rqh { z3.h }, p0/Z, [x25]\n" "sub x27, x27, #0x8\n" "ld1rqh { z2.h }, p0/Z, [x24]\n" - "ld1rqh { z3.h }, p0/Z, [x23]\n" + "ld1rqh { z1.h }, p0/Z, [x23]\n" "cmp x27, #0x8\n" "add x26, x26, #0x10\n" - "ld1rqh { z4.h }, p0/Z, [x22]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "fmla z8.h, z6.h, z0.h[0]\n" - "fmla z12.h, z6.h, z1.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z16.h, z6.h, z2.h[0]\n" - "fmla z20.h, z6.h, z3.h[0]\n" + "ld1rqh { z0.h }, p0/Z, [x22]\n" + "ld1h { z29.h }, p5/Z, [x10]\n" + "fmla z8.h, z29.h, z4.h[0]\n" + "fmla z12.h, z29.h, z3.h[0]\n" + "ld1h { z28.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z16.h, z29.h, z2.h[0]\n" + "fmla z20.h, z29.h, z1.h[0]\n" "add x25, x25, #0x10\n" - "fmla z24.h, z6.h, z4.h[0]\n" - "fmla z9.h, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z24.h, z29.h, z0.h[0]\n" + "fmla z9.h, z28.h, z4.h[0]\n" + "ld1h { z29.h }, p5/Z, [x10, #2, MUL VL]\n" "add x24, x24, #0x10\n" - "fmla z13.h, z7.h, z1.h[0]\n" - "fmla z17.h, z7.h, z2.h[0]\n" + "fmla z13.h, z28.h, z3.h[0]\n" + "fmla z17.h, z28.h, z2.h[0]\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - "fmla z21.h, z7.h, z3.h[0]\n" - "fmla z25.h, z7.h, z4.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[0]\n" - "fmla z14.h, z6.h, z1.h[0]\n" - "fmla z18.h, z6.h, z2.h[0]\n" - "fmla z22.h, z6.h, z3.h[0]\n" - "fmla z26.h, z6.h, z4.h[0]\n" - "fmla z11.h, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - "fmla z15.h, z7.h, z1.h[0]\n" - "fmla z19.h, z7.h, z2.h[0]\n" - "fmla z23.h, z7.h, z3.h[0]\n" - "fmla z27.h, z7.h, z4.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[1]\n" - "fmla z12.h, z6.h, z1.h[1]\n" - "fmla z16.h, z6.h, z2.h[1]\n" - "fmla z20.h, z6.h, z3.h[1]\n" - "fmla z24.h, z6.h, z4.h[1]\n" - "fmla z9.h, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[1]\n" - "fmla z17.h, z7.h, z2.h[1]\n" - "fmla z21.h, z7.h, z3.h[1]\n" - "fmla z25.h, z7.h, z4.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" + "fmla z21.h, z28.h, z1.h[0]\n" + "fmla z25.h, z28.h, z0.h[0]\n" + "ld1h { z28.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z10.h, z29.h, z4.h[0]\n" + "fmla z14.h, z29.h, z3.h[0]\n" + "fmla z18.h, z29.h, z2.h[0]\n" + "fmla z22.h, z29.h, z1.h[0]\n" + "fmla z26.h, z29.h, z0.h[0]\n" + "fmla z11.h, z28.h, z4.h[0]\n" + "ld1h { z29.h }, p5/Z, [x10, #4, MUL VL]\n" + "fmla z15.h, z28.h, z3.h[0]\n" + "fmla z19.h, z28.h, z2.h[0]\n" + "fmla z23.h, z28.h, z1.h[0]\n" + "fmla z27.h, z28.h, z0.h[0]\n" + "ld1h { z28.h }, p5/Z, [x10, #5, MUL VL]\n" + "fmla z8.h, z29.h, z4.h[1]\n" + "fmla z12.h, z29.h, z3.h[1]\n" + "fmla z16.h, z29.h, z2.h[1]\n" + "fmla z20.h, z29.h, z1.h[1]\n" + "fmla z24.h, z29.h, z0.h[1]\n" + "fmla z9.h, z28.h, z4.h[1]\n" + "ld1h { z29.h }, p5/Z, [x10, #6, MUL VL]\n" + "fmla z13.h, z28.h, z3.h[1]\n" + "fmla z17.h, z28.h, z2.h[1]\n" + "fmla z21.h, z28.h, z1.h[1]\n" + "fmla z25.h, z28.h, z0.h[1]\n" + "ld1h { z28.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "fmla z10.h, z6.h, z0.h[1]\n" - "fmla z14.h, z6.h, z1.h[1]\n" - "fmla z18.h, z6.h, z2.h[1]\n" - "fmla z22.h, z6.h, z3.h[1]\n" - "fmla z26.h, z6.h, z4.h[1]\n" - "fmla z11.h, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n" - "fmla z15.h, z7.h, z1.h[1]\n" - "fmla z19.h, z7.h, z2.h[1]\n" - "fmla z23.h, z7.h, z3.h[1]\n" - "fmla z27.h, z7.h, z4.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #-7, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[2]\n" - "fmla z12.h, z6.h, z1.h[2]\n" - "fmla z16.h, z6.h, z2.h[2]\n" - "fmla z20.h, z6.h, z3.h[2]\n" - "fmla z24.h, z6.h, z4.h[2]\n" - "fmla z9.h, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[2]\n" - "fmla z17.h, z7.h, z2.h[2]\n" - "fmla z21.h, z7.h, z3.h[2]\n" - "fmla z25.h, z7.h, z4.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #-5, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[2]\n" - "fmla z14.h, z6.h, z1.h[2]\n" - "fmla z18.h, z6.h, z2.h[2]\n" - "fmla z22.h, z6.h, z3.h[2]\n" - "fmla z26.h, z6.h, z4.h[2]\n" - "fmla z11.h, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n" - "fmla z15.h, z7.h, z1.h[2]\n" - "fmla z19.h, z7.h, z2.h[2]\n" - "fmla z23.h, z7.h, z3.h[2]\n" - "fmla z27.h, z7.h, z4.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #-3, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[3]\n" - "fmla z12.h, z6.h, z1.h[3]\n" - "fmla z16.h, z6.h, z2.h[3]\n" - "fmla z20.h, z6.h, z3.h[3]\n" - "fmla z24.h, z6.h, z4.h[3]\n" - "fmla z9.h, z7.h, z0.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[3]\n" - "fmla z17.h, z7.h, z2.h[3]\n" - "fmla z21.h, z7.h, z3.h[3]\n" - "fmla z25.h, z7.h, z4.h[3]\n" - "ld1h { z7.h }, p5/Z, [x10, #-1, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[3]\n" - "fmla z14.h, z6.h, z1.h[3]\n" - "fmla z18.h, z6.h, z2.h[3]\n" - "fmla z22.h, z6.h, z3.h[3]\n" - "fmla z26.h, z6.h, z4.h[3]\n" - "fmla z11.h, z7.h, z0.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "fmla z15.h, z7.h, z1.h[3]\n" - "fmla z19.h, z7.h, z2.h[3]\n" - "fmla z23.h, z7.h, z3.h[3]\n" - "fmla z27.h, z7.h, z4.h[3]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[4]\n" - "fmla z12.h, z6.h, z1.h[4]\n" - "fmla z16.h, z6.h, z2.h[4]\n" - "fmla z20.h, z6.h, z3.h[4]\n" - "fmla z24.h, z6.h, z4.h[4]\n" - "fmla z9.h, z7.h, z0.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[4]\n" - "fmla z17.h, z7.h, z2.h[4]\n" - "fmla z21.h, z7.h, z3.h[4]\n" - "fmla z25.h, z7.h, z4.h[4]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[4]\n" - "fmla z14.h, z6.h, z1.h[4]\n" - "fmla z18.h, z6.h, z2.h[4]\n" - "fmla z22.h, z6.h, z3.h[4]\n" - "fmla z26.h, z6.h, z4.h[4]\n" - "fmla z11.h, z7.h, z0.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - "fmla z15.h, z7.h, z1.h[4]\n" - "fmla z19.h, z7.h, z2.h[4]\n" - "fmla z23.h, z7.h, z3.h[4]\n" - "fmla z27.h, z7.h, z4.h[4]\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[5]\n" - "fmla z12.h, z6.h, z1.h[5]\n" - "fmla z16.h, z6.h, z2.h[5]\n" - "fmla z20.h, z6.h, z3.h[5]\n" - "fmla z24.h, z6.h, z4.h[5]\n" - "fmla z9.h, z7.h, z0.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[5]\n" - "fmla z17.h, z7.h, z2.h[5]\n" - "fmla z21.h, z7.h, z3.h[5]\n" - "fmla z25.h, z7.h, z4.h[5]\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" + "fmla z10.h, z29.h, z4.h[1]\n" + "fmla z14.h, z29.h, z3.h[1]\n" + "fmla z18.h, z29.h, z2.h[1]\n" + "fmla z22.h, z29.h, z1.h[1]\n" + "fmla z26.h, z29.h, z0.h[1]\n" + "fmla z11.h, z28.h, z4.h[1]\n" + "ld1h { z29.h }, p5/Z, [x10, #-8, MUL VL]\n" + "fmla z15.h, z28.h, z3.h[1]\n" + "fmla z19.h, z28.h, z2.h[1]\n" + "fmla z23.h, z28.h, z1.h[1]\n" + "fmla z27.h, z28.h, z0.h[1]\n" + "ld1h { z28.h }, p5/Z, [x10, #-7, MUL VL]\n" + "fmla z8.h, z29.h, z4.h[2]\n" + "fmla z12.h, z29.h, z3.h[2]\n" + "fmla z16.h, z29.h, z2.h[2]\n" + "fmla z20.h, z29.h, z1.h[2]\n" + "fmla z24.h, z29.h, z0.h[2]\n" + "fmla z9.h, z28.h, z4.h[2]\n" + "ld1h { z29.h }, p5/Z, [x10, #-6, MUL VL]\n" + "fmla z13.h, z28.h, z3.h[2]\n" + "fmla z17.h, z28.h, z2.h[2]\n" + "fmla z21.h, z28.h, z1.h[2]\n" + "fmla z25.h, z28.h, z0.h[2]\n" + "ld1h { z28.h }, p5/Z, [x10, #-5, MUL VL]\n" + "fmla z10.h, z29.h, z4.h[2]\n" + "fmla z14.h, z29.h, z3.h[2]\n" + "fmla z18.h, z29.h, z2.h[2]\n" + "fmla z22.h, z29.h, z1.h[2]\n" + "fmla z26.h, z29.h, z0.h[2]\n" + "fmla z11.h, z28.h, z4.h[2]\n" + "ld1h { z29.h }, p5/Z, [x10, #-4, MUL VL]\n" + "fmla z15.h, z28.h, z3.h[2]\n" + "fmla z19.h, z28.h, z2.h[2]\n" + "fmla z23.h, z28.h, z1.h[2]\n" + "fmla z27.h, z28.h, z0.h[2]\n" + "ld1h { z28.h }, p5/Z, [x10, #-3, MUL VL]\n" + "fmla z8.h, z29.h, z4.h[3]\n" + "fmla z12.h, z29.h, z3.h[3]\n" + "fmla z16.h, z29.h, z2.h[3]\n" + "fmla z20.h, z29.h, z1.h[3]\n" + "fmla z24.h, z29.h, z0.h[3]\n" + "fmla z9.h, z28.h, z4.h[3]\n" + "ld1h { z29.h }, p5/Z, [x10, #-2, MUL VL]\n" + "fmla z13.h, z28.h, z3.h[3]\n" + "fmla z17.h, z28.h, z2.h[3]\n" + "fmla z21.h, z28.h, z1.h[3]\n" + "fmla z25.h, z28.h, z0.h[3]\n" + "ld1h { z28.h }, p5/Z, [x10, #-1, MUL VL]\n" + "fmla z10.h, z29.h, z4.h[3]\n" + "fmla z14.h, z29.h, z3.h[3]\n" + "fmla z18.h, z29.h, z2.h[3]\n" + "fmla z22.h, z29.h, z1.h[3]\n" + "fmla z26.h, z29.h, z0.h[3]\n" + "fmla z11.h, z28.h, z4.h[3]\n" + "ld1h { z29.h }, p5/Z, [x10]\n" + "fmla z15.h, z28.h, z3.h[3]\n" + "fmla z19.h, z28.h, z2.h[3]\n" + "fmla z23.h, z28.h, z1.h[3]\n" + "fmla z27.h, z28.h, z0.h[3]\n" + "ld1h { z28.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z29.h, z4.h[4]\n" + "fmla z12.h, z29.h, z3.h[4]\n" + "fmla z16.h, z29.h, z2.h[4]\n" + "fmla z20.h, z29.h, z1.h[4]\n" + "fmla z24.h, z29.h, z0.h[4]\n" + "fmla z9.h, z28.h, z4.h[4]\n" + "ld1h { z29.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z13.h, z28.h, z3.h[4]\n" + "fmla z17.h, z28.h, z2.h[4]\n" + "fmla z21.h, z28.h, z1.h[4]\n" + "fmla z25.h, z28.h, z0.h[4]\n" + "ld1h { z28.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z10.h, z29.h, z4.h[4]\n" + "fmla z14.h, z29.h, z3.h[4]\n" + "fmla z18.h, z29.h, z2.h[4]\n" + "fmla z22.h, z29.h, z1.h[4]\n" + "fmla z26.h, z29.h, z0.h[4]\n" + "fmla z11.h, z28.h, z4.h[4]\n" + "ld1h { z29.h }, p5/Z, [x10, #4, MUL VL]\n" + "fmla z15.h, z28.h, z3.h[4]\n" + "fmla z19.h, z28.h, z2.h[4]\n" + "fmla z23.h, z28.h, z1.h[4]\n" + "fmla z27.h, z28.h, z0.h[4]\n" + "ld1h { z28.h }, p5/Z, [x10, #5, MUL VL]\n" + "fmla z8.h, z29.h, z4.h[5]\n" + "fmla z12.h, z29.h, z3.h[5]\n" + "fmla z16.h, z29.h, z2.h[5]\n" + "fmla z20.h, z29.h, z1.h[5]\n" + "fmla z24.h, z29.h, z0.h[5]\n" + "fmla z9.h, z28.h, z4.h[5]\n" + "ld1h { z29.h }, p5/Z, [x10, #6, MUL VL]\n" + "fmla z13.h, z28.h, z3.h[5]\n" + "fmla z17.h, z28.h, z2.h[5]\n" + "fmla z21.h, z28.h, z1.h[5]\n" + "fmla z25.h, z28.h, z0.h[5]\n" + "ld1h { z28.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "fmla z10.h, z6.h, z0.h[5]\n" - "fmla z14.h, z6.h, z1.h[5]\n" - "fmla z18.h, z6.h, z2.h[5]\n" - "fmla z22.h, z6.h, z3.h[5]\n" - "fmla z26.h, z6.h, z4.h[5]\n" - "fmla z11.h, z7.h, z0.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n" - "fmla z15.h, z7.h, z1.h[5]\n" - "fmla z19.h, z7.h, z2.h[5]\n" - "fmla z23.h, z7.h, z3.h[5]\n" - "fmla z27.h, z7.h, z4.h[5]\n" - "ld1h { z7.h }, p5/Z, [x10, #-7, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[6]\n" - "fmla z12.h, z6.h, z1.h[6]\n" - "fmla z16.h, z6.h, z2.h[6]\n" - "fmla z20.h, z6.h, z3.h[6]\n" - "fmla z24.h, z6.h, z4.h[6]\n" - "fmla z9.h, z7.h, z0.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[6]\n" - "fmla z17.h, z7.h, z2.h[6]\n" - "fmla z21.h, z7.h, z3.h[6]\n" - "fmla z25.h, z7.h, z4.h[6]\n" - "ld1h { z7.h }, p5/Z, [x10, #-5, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[6]\n" - "fmla z14.h, z6.h, z1.h[6]\n" - "fmla z18.h, z6.h, z2.h[6]\n" - "fmla z22.h, z6.h, z3.h[6]\n" - "fmla z26.h, z6.h, z4.h[6]\n" - "fmla z11.h, z7.h, z0.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n" - "fmla z15.h, z7.h, z1.h[6]\n" - "fmla z19.h, z7.h, z2.h[6]\n" - "fmla z23.h, z7.h, z3.h[6]\n" - "fmla z27.h, z7.h, z4.h[6]\n" - "ld1h { z7.h }, p5/Z, [x10, #-3, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[7]\n" - "fmla z12.h, z6.h, z1.h[7]\n" - "fmla z16.h, z6.h, z2.h[7]\n" - "fmla z20.h, z6.h, z3.h[7]\n" - "fmla z24.h, z6.h, z4.h[7]\n" - "fmla z9.h, z7.h, z0.h[7]\n" - "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[7]\n" - "fmla z17.h, z7.h, z2.h[7]\n" - "fmla z21.h, z7.h, z3.h[7]\n" - "fmla z25.h, z7.h, z4.h[7]\n" - "ld1h { z7.h }, p5/Z, [x10, #-1, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[7]\n" - "fmla z14.h, z6.h, z1.h[7]\n" - "fmla z18.h, z6.h, z2.h[7]\n" - "fmla z22.h, z6.h, z3.h[7]\n" - "fmla z26.h, z6.h, z4.h[7]\n" - "fmla z11.h, z7.h, z0.h[7]\n" - "fmla z15.h, z7.h, z1.h[7]\n" - "fmla z19.h, z7.h, z2.h[7]\n" - "fmla z23.h, z7.h, z3.h[7]\n" - "fmla z27.h, z7.h, z4.h[7]\n" + "fmla z10.h, z29.h, z4.h[5]\n" + "fmla z14.h, z29.h, z3.h[5]\n" + "fmla z18.h, z29.h, z2.h[5]\n" + "fmla z22.h, z29.h, z1.h[5]\n" + "fmla z26.h, z29.h, z0.h[5]\n" + "fmla z11.h, z28.h, z4.h[5]\n" + "ld1h { z29.h }, p5/Z, [x10, #-8, MUL VL]\n" + "fmla z15.h, z28.h, z3.h[5]\n" + "fmla z19.h, z28.h, z2.h[5]\n" + "fmla z23.h, z28.h, z1.h[5]\n" + "fmla z27.h, z28.h, z0.h[5]\n" + "ld1h { z28.h }, p5/Z, [x10, #-7, MUL VL]\n" + "fmla z8.h, z29.h, z4.h[6]\n" + "fmla z12.h, z29.h, z3.h[6]\n" + "fmla z16.h, z29.h, z2.h[6]\n" + "fmla z20.h, z29.h, z1.h[6]\n" + "fmla z24.h, z29.h, z0.h[6]\n" + "fmla z9.h, z28.h, z4.h[6]\n" + "ld1h { z29.h }, p5/Z, [x10, #-6, MUL VL]\n" + "fmla z13.h, z28.h, z3.h[6]\n" + "fmla z17.h, z28.h, z2.h[6]\n" + "fmla z21.h, z28.h, z1.h[6]\n" + "fmla z25.h, z28.h, z0.h[6]\n" + "ld1h { z28.h }, p5/Z, [x10, #-5, MUL VL]\n" + "fmla z10.h, z29.h, z4.h[6]\n" + "fmla z14.h, z29.h, z3.h[6]\n" + "fmla z18.h, z29.h, z2.h[6]\n" + "fmla z22.h, z29.h, z1.h[6]\n" + "fmla z26.h, z29.h, z0.h[6]\n" + "fmla z11.h, z28.h, z4.h[6]\n" + "ld1h { z29.h }, p5/Z, [x10, #-4, MUL VL]\n" + "fmla z15.h, z28.h, z3.h[6]\n" + "fmla z19.h, z28.h, z2.h[6]\n" + "fmla z23.h, z28.h, z1.h[6]\n" + "fmla z27.h, z28.h, z0.h[6]\n" + "ld1h { z28.h }, p5/Z, [x10, #-3, MUL VL]\n" + "fmla z8.h, z29.h, z4.h[7]\n" + "fmla z12.h, z29.h, z3.h[7]\n" + "fmla z16.h, z29.h, z2.h[7]\n" + "fmla z20.h, z29.h, z1.h[7]\n" + "fmla z24.h, z29.h, z0.h[7]\n" + "fmla z9.h, z28.h, z4.h[7]\n" + "ld1h { z29.h }, p5/Z, [x10, #-2, MUL VL]\n" + "fmla z13.h, z28.h, z3.h[7]\n" + "fmla z17.h, z28.h, z2.h[7]\n" + "fmla z21.h, z28.h, z1.h[7]\n" + "fmla z25.h, z28.h, z0.h[7]\n" + "ld1h { z28.h }, p5/Z, [x10, #-1, MUL VL]\n" + "fmla z10.h, z29.h, z4.h[7]\n" + "fmla z14.h, z29.h, z3.h[7]\n" + "fmla z18.h, z29.h, z2.h[7]\n" + "fmla z22.h, z29.h, z1.h[7]\n" + "fmla z26.h, z29.h, z0.h[7]\n" + "fmla z11.h, z28.h, z4.h[7]\n" + "fmla z15.h, z28.h, z3.h[7]\n" + "fmla z19.h, z28.h, z2.h[7]\n" + "fmla z23.h, z28.h, z1.h[7]\n" + "fmla z27.h, z28.h, z0.h[7]\n" "bgt 61b\n" "62:" // Height 5: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" @@ -1989,219 +1989,219 @@ void sve_hybrid_fp16_mla_6x4VL ( "ld1rqh { z2.h }, p0/Z, [x24]\n" "ld1rqh { z3.h }, p0/Z, [x23]\n" "ld1rqh { z4.h }, p0/Z, [x22]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "fmla z8.h, z6.h, z0.h[0]\n" - "fmla z12.h, z6.h, z1.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z16.h, z6.h, z2.h[0]\n" - "fmla z20.h, z6.h, z3.h[0]\n" - "fmla z24.h, z6.h, z4.h[0]\n" - "fmla z9.h, z7.h, z0.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[0]\n" - "fmla z17.h, z7.h, z2.h[0]\n" - "fmla z21.h, z7.h, z3.h[0]\n" - "fmla z25.h, z7.h, z4.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z29.h }, p5/Z, [x10]\n" + "fmla z8.h, z29.h, z0.h[0]\n" + "fmla z12.h, z29.h, z1.h[0]\n" + "ld1h { z28.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z16.h, z29.h, z2.h[0]\n" + "fmla z20.h, z29.h, z3.h[0]\n" + "fmla z24.h, z29.h, z4.h[0]\n" + "fmla z9.h, z28.h, z0.h[0]\n" + "ld1h { z29.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z13.h, z28.h, z1.h[0]\n" + "fmla z17.h, z28.h, z2.h[0]\n" + "fmla z21.h, z28.h, z3.h[0]\n" + "fmla z25.h, z28.h, z4.h[0]\n" + "ld1h { z28.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[0]\n" - "fmla z14.h, z6.h, z1.h[0]\n" - "fmla z18.h, z6.h, z2.h[0]\n" - "fmla z22.h, z6.h, z3.h[0]\n" - "fmla z26.h, z6.h, z4.h[0]\n" - "fmla z11.h, z7.h, z0.h[0]\n" - "fmla z15.h, z7.h, z1.h[0]\n" - "fmla z19.h, z7.h, z2.h[0]\n" - "fmla z23.h, z7.h, z3.h[0]\n" - "fmla z27.h, z7.h, z4.h[0]\n" + "fmla z10.h, z29.h, z0.h[0]\n" + "fmla z14.h, z29.h, z1.h[0]\n" + "fmla z18.h, z29.h, z2.h[0]\n" + "fmla z22.h, z29.h, z3.h[0]\n" + "fmla z26.h, z29.h, z4.h[0]\n" + "fmla z11.h, z28.h, z0.h[0]\n" + "fmla z15.h, z28.h, z1.h[0]\n" + "fmla z19.h, z28.h, z2.h[0]\n" + "fmla z23.h, z28.h, z3.h[0]\n" + "fmla z27.h, z28.h, z4.h[0]\n" "ble 63f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[1]\n" - "fmla z12.h, z6.h, z1.h[1]\n" - "fmla z16.h, z6.h, z2.h[1]\n" - "fmla z20.h, z6.h, z3.h[1]\n" + "ld1h { z29.h }, p5/Z, [x10]\n" + "ld1h { z28.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z29.h, z0.h[1]\n" + "fmla z12.h, z29.h, z1.h[1]\n" + "fmla z16.h, z29.h, z2.h[1]\n" + "fmla z20.h, z29.h, z3.h[1]\n" "subs x27, x27, #0x1\n" - "fmla z24.h, z6.h, z4.h[1]\n" - "fmla z9.h, z7.h, z0.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[1]\n" - "fmla z17.h, z7.h, z2.h[1]\n" - "fmla z21.h, z7.h, z3.h[1]\n" - "fmla z25.h, z7.h, z4.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z24.h, z29.h, z4.h[1]\n" + "fmla z9.h, z28.h, z0.h[1]\n" + "ld1h { z29.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z13.h, z28.h, z1.h[1]\n" + "fmla z17.h, z28.h, z2.h[1]\n" + "fmla z21.h, z28.h, z3.h[1]\n" + "fmla z25.h, z28.h, z4.h[1]\n" + "ld1h { z28.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[1]\n" - "fmla z14.h, z6.h, z1.h[1]\n" - "fmla z18.h, z6.h, z2.h[1]\n" - "fmla z22.h, z6.h, z3.h[1]\n" - "fmla z26.h, z6.h, z4.h[1]\n" - "fmla z11.h, z7.h, z0.h[1]\n" - "fmla z15.h, z7.h, z1.h[1]\n" - "fmla z19.h, z7.h, z2.h[1]\n" - "fmla z23.h, z7.h, z3.h[1]\n" - "fmla z27.h, z7.h, z4.h[1]\n" + "fmla z10.h, z29.h, z0.h[1]\n" + "fmla z14.h, z29.h, z1.h[1]\n" + "fmla z18.h, z29.h, z2.h[1]\n" + "fmla z22.h, z29.h, z3.h[1]\n" + "fmla z26.h, z29.h, z4.h[1]\n" + "fmla z11.h, z28.h, z0.h[1]\n" + "fmla z15.h, z28.h, z1.h[1]\n" + "fmla z19.h, z28.h, z2.h[1]\n" + "fmla z23.h, z28.h, z3.h[1]\n" + "fmla z27.h, z28.h, z4.h[1]\n" "ble 63f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[2]\n" - "fmla z12.h, z6.h, z1.h[2]\n" - "fmla z16.h, z6.h, z2.h[2]\n" - "fmla z20.h, z6.h, z3.h[2]\n" + "ld1h { z29.h }, p5/Z, [x10]\n" + "ld1h { z28.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z29.h, z0.h[2]\n" + "fmla z12.h, z29.h, z1.h[2]\n" + "fmla z16.h, z29.h, z2.h[2]\n" + "fmla z20.h, z29.h, z3.h[2]\n" "subs x27, x27, #0x1\n" - "fmla z24.h, z6.h, z4.h[2]\n" - "fmla z9.h, z7.h, z0.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[2]\n" - "fmla z17.h, z7.h, z2.h[2]\n" - "fmla z21.h, z7.h, z3.h[2]\n" - "fmla z25.h, z7.h, z4.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z24.h, z29.h, z4.h[2]\n" + "fmla z9.h, z28.h, z0.h[2]\n" + "ld1h { z29.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z13.h, z28.h, z1.h[2]\n" + "fmla z17.h, z28.h, z2.h[2]\n" + "fmla z21.h, z28.h, z3.h[2]\n" + "fmla z25.h, z28.h, z4.h[2]\n" + "ld1h { z28.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[2]\n" - "fmla z14.h, z6.h, z1.h[2]\n" - "fmla z18.h, z6.h, z2.h[2]\n" - "fmla z22.h, z6.h, z3.h[2]\n" - "fmla z26.h, z6.h, z4.h[2]\n" - "fmla z11.h, z7.h, z0.h[2]\n" - "fmla z15.h, z7.h, z1.h[2]\n" - "fmla z19.h, z7.h, z2.h[2]\n" - "fmla z23.h, z7.h, z3.h[2]\n" - "fmla z27.h, z7.h, z4.h[2]\n" + "fmla z10.h, z29.h, z0.h[2]\n" + "fmla z14.h, z29.h, z1.h[2]\n" + "fmla z18.h, z29.h, z2.h[2]\n" + "fmla z22.h, z29.h, z3.h[2]\n" + "fmla z26.h, z29.h, z4.h[2]\n" + "fmla z11.h, z28.h, z0.h[2]\n" + "fmla z15.h, z28.h, z1.h[2]\n" + "fmla z19.h, z28.h, z2.h[2]\n" + "fmla z23.h, z28.h, z3.h[2]\n" + "fmla z27.h, z28.h, z4.h[2]\n" "ble 63f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[3]\n" - "fmla z12.h, z6.h, z1.h[3]\n" - "fmla z16.h, z6.h, z2.h[3]\n" - "fmla z20.h, z6.h, z3.h[3]\n" + "ld1h { z29.h }, p5/Z, [x10]\n" + "ld1h { z28.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z29.h, z0.h[3]\n" + "fmla z12.h, z29.h, z1.h[3]\n" + "fmla z16.h, z29.h, z2.h[3]\n" + "fmla z20.h, z29.h, z3.h[3]\n" "subs x27, x27, #0x1\n" - "fmla z24.h, z6.h, z4.h[3]\n" - "fmla z9.h, z7.h, z0.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[3]\n" - "fmla z17.h, z7.h, z2.h[3]\n" - "fmla z21.h, z7.h, z3.h[3]\n" - "fmla z25.h, z7.h, z4.h[3]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z24.h, z29.h, z4.h[3]\n" + "fmla z9.h, z28.h, z0.h[3]\n" + "ld1h { z29.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z13.h, z28.h, z1.h[3]\n" + "fmla z17.h, z28.h, z2.h[3]\n" + "fmla z21.h, z28.h, z3.h[3]\n" + "fmla z25.h, z28.h, z4.h[3]\n" + "ld1h { z28.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[3]\n" - "fmla z14.h, z6.h, z1.h[3]\n" - "fmla z18.h, z6.h, z2.h[3]\n" - "fmla z22.h, z6.h, z3.h[3]\n" - "fmla z26.h, z6.h, z4.h[3]\n" - "fmla z11.h, z7.h, z0.h[3]\n" - "fmla z15.h, z7.h, z1.h[3]\n" - "fmla z19.h, z7.h, z2.h[3]\n" - "fmla z23.h, z7.h, z3.h[3]\n" - "fmla z27.h, z7.h, z4.h[3]\n" + "fmla z10.h, z29.h, z0.h[3]\n" + "fmla z14.h, z29.h, z1.h[3]\n" + "fmla z18.h, z29.h, z2.h[3]\n" + "fmla z22.h, z29.h, z3.h[3]\n" + "fmla z26.h, z29.h, z4.h[3]\n" + "fmla z11.h, z28.h, z0.h[3]\n" + "fmla z15.h, z28.h, z1.h[3]\n" + "fmla z19.h, z28.h, z2.h[3]\n" + "fmla z23.h, z28.h, z3.h[3]\n" + "fmla z27.h, z28.h, z4.h[3]\n" "ble 63f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[4]\n" - "fmla z12.h, z6.h, z1.h[4]\n" - "fmla z16.h, z6.h, z2.h[4]\n" - "fmla z20.h, z6.h, z3.h[4]\n" + "ld1h { z29.h }, p5/Z, [x10]\n" + "ld1h { z28.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z29.h, z0.h[4]\n" + "fmla z12.h, z29.h, z1.h[4]\n" + "fmla z16.h, z29.h, z2.h[4]\n" + "fmla z20.h, z29.h, z3.h[4]\n" "subs x27, x27, #0x1\n" - "fmla z24.h, z6.h, z4.h[4]\n" - "fmla z9.h, z7.h, z0.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[4]\n" - "fmla z17.h, z7.h, z2.h[4]\n" - "fmla z21.h, z7.h, z3.h[4]\n" - "fmla z25.h, z7.h, z4.h[4]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z24.h, z29.h, z4.h[4]\n" + "fmla z9.h, z28.h, z0.h[4]\n" + "ld1h { z29.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z13.h, z28.h, z1.h[4]\n" + "fmla z17.h, z28.h, z2.h[4]\n" + "fmla z21.h, z28.h, z3.h[4]\n" + "fmla z25.h, z28.h, z4.h[4]\n" + "ld1h { z28.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[4]\n" - "fmla z14.h, z6.h, z1.h[4]\n" - "fmla z18.h, z6.h, z2.h[4]\n" - "fmla z22.h, z6.h, z3.h[4]\n" - "fmla z26.h, z6.h, z4.h[4]\n" - "fmla z11.h, z7.h, z0.h[4]\n" - "fmla z15.h, z7.h, z1.h[4]\n" - "fmla z19.h, z7.h, z2.h[4]\n" - "fmla z23.h, z7.h, z3.h[4]\n" - "fmla z27.h, z7.h, z4.h[4]\n" + "fmla z10.h, z29.h, z0.h[4]\n" + "fmla z14.h, z29.h, z1.h[4]\n" + "fmla z18.h, z29.h, z2.h[4]\n" + "fmla z22.h, z29.h, z3.h[4]\n" + "fmla z26.h, z29.h, z4.h[4]\n" + "fmla z11.h, z28.h, z0.h[4]\n" + "fmla z15.h, z28.h, z1.h[4]\n" + "fmla z19.h, z28.h, z2.h[4]\n" + "fmla z23.h, z28.h, z3.h[4]\n" + "fmla z27.h, z28.h, z4.h[4]\n" "ble 63f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[5]\n" - "fmla z12.h, z6.h, z1.h[5]\n" - "fmla z16.h, z6.h, z2.h[5]\n" - "fmla z20.h, z6.h, z3.h[5]\n" + "ld1h { z29.h }, p5/Z, [x10]\n" + "ld1h { z28.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z29.h, z0.h[5]\n" + "fmla z12.h, z29.h, z1.h[5]\n" + "fmla z16.h, z29.h, z2.h[5]\n" + "fmla z20.h, z29.h, z3.h[5]\n" "subs x27, x27, #0x1\n" - "fmla z24.h, z6.h, z4.h[5]\n" - "fmla z9.h, z7.h, z0.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[5]\n" - "fmla z17.h, z7.h, z2.h[5]\n" - "fmla z21.h, z7.h, z3.h[5]\n" - "fmla z25.h, z7.h, z4.h[5]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z24.h, z29.h, z4.h[5]\n" + "fmla z9.h, z28.h, z0.h[5]\n" + "ld1h { z29.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z13.h, z28.h, z1.h[5]\n" + "fmla z17.h, z28.h, z2.h[5]\n" + "fmla z21.h, z28.h, z3.h[5]\n" + "fmla z25.h, z28.h, z4.h[5]\n" + "ld1h { z28.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[5]\n" - "fmla z14.h, z6.h, z1.h[5]\n" - "fmla z18.h, z6.h, z2.h[5]\n" - "fmla z22.h, z6.h, z3.h[5]\n" - "fmla z26.h, z6.h, z4.h[5]\n" - "fmla z11.h, z7.h, z0.h[5]\n" - "fmla z15.h, z7.h, z1.h[5]\n" - "fmla z19.h, z7.h, z2.h[5]\n" - "fmla z23.h, z7.h, z3.h[5]\n" - "fmla z27.h, z7.h, z4.h[5]\n" + "fmla z10.h, z29.h, z0.h[5]\n" + "fmla z14.h, z29.h, z1.h[5]\n" + "fmla z18.h, z29.h, z2.h[5]\n" + "fmla z22.h, z29.h, z3.h[5]\n" + "fmla z26.h, z29.h, z4.h[5]\n" + "fmla z11.h, z28.h, z0.h[5]\n" + "fmla z15.h, z28.h, z1.h[5]\n" + "fmla z19.h, z28.h, z2.h[5]\n" + "fmla z23.h, z28.h, z3.h[5]\n" + "fmla z27.h, z28.h, z4.h[5]\n" "ble 63f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[6]\n" - "fmla z12.h, z6.h, z1.h[6]\n" - "fmla z16.h, z6.h, z2.h[6]\n" - "fmla z20.h, z6.h, z3.h[6]\n" + "ld1h { z29.h }, p5/Z, [x10]\n" + "ld1h { z28.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z29.h, z0.h[6]\n" + "fmla z12.h, z29.h, z1.h[6]\n" + "fmla z16.h, z29.h, z2.h[6]\n" + "fmla z20.h, z29.h, z3.h[6]\n" "subs x27, x27, #0x1\n" - "fmla z24.h, z6.h, z4.h[6]\n" - "fmla z9.h, z7.h, z0.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[6]\n" - "fmla z17.h, z7.h, z2.h[6]\n" - "fmla z21.h, z7.h, z3.h[6]\n" - "fmla z25.h, z7.h, z4.h[6]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z24.h, z29.h, z4.h[6]\n" + "fmla z9.h, z28.h, z0.h[6]\n" + "ld1h { z29.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z13.h, z28.h, z1.h[6]\n" + "fmla z17.h, z28.h, z2.h[6]\n" + "fmla z21.h, z28.h, z3.h[6]\n" + "fmla z25.h, z28.h, z4.h[6]\n" + "ld1h { z28.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[6]\n" - "fmla z14.h, z6.h, z1.h[6]\n" - "fmla z18.h, z6.h, z2.h[6]\n" - "fmla z22.h, z6.h, z3.h[6]\n" - "fmla z26.h, z6.h, z4.h[6]\n" - "fmla z11.h, z7.h, z0.h[6]\n" - "fmla z15.h, z7.h, z1.h[6]\n" - "fmla z19.h, z7.h, z2.h[6]\n" - "fmla z23.h, z7.h, z3.h[6]\n" - "fmla z27.h, z7.h, z4.h[6]\n" + "fmla z10.h, z29.h, z0.h[6]\n" + "fmla z14.h, z29.h, z1.h[6]\n" + "fmla z18.h, z29.h, z2.h[6]\n" + "fmla z22.h, z29.h, z3.h[6]\n" + "fmla z26.h, z29.h, z4.h[6]\n" + "fmla z11.h, z28.h, z0.h[6]\n" + "fmla z15.h, z28.h, z1.h[6]\n" + "fmla z19.h, z28.h, z2.h[6]\n" + "fmla z23.h, z28.h, z3.h[6]\n" + "fmla z27.h, z28.h, z4.h[6]\n" "ble 63f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[7]\n" - "fmla z12.h, z6.h, z1.h[7]\n" - "fmla z16.h, z6.h, z2.h[7]\n" - "fmla z20.h, z6.h, z3.h[7]\n" - "fmla z24.h, z6.h, z4.h[7]\n" - "fmla z9.h, z7.h, z0.h[7]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z13.h, z7.h, z1.h[7]\n" - "fmla z17.h, z7.h, z2.h[7]\n" - "fmla z21.h, z7.h, z3.h[7]\n" - "fmla z25.h, z7.h, z4.h[7]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z29.h }, p5/Z, [x10]\n" + "ld1h { z28.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z29.h, z0.h[7]\n" + "fmla z12.h, z29.h, z1.h[7]\n" + "fmla z16.h, z29.h, z2.h[7]\n" + "fmla z20.h, z29.h, z3.h[7]\n" + "fmla z24.h, z29.h, z4.h[7]\n" + "fmla z9.h, z28.h, z0.h[7]\n" + "ld1h { z29.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z13.h, z28.h, z1.h[7]\n" + "fmla z17.h, z28.h, z2.h[7]\n" + "fmla z21.h, z28.h, z3.h[7]\n" + "fmla z25.h, z28.h, z4.h[7]\n" + "ld1h { z28.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[7]\n" - "fmla z14.h, z6.h, z1.h[7]\n" - "fmla z18.h, z6.h, z2.h[7]\n" - "fmla z22.h, z6.h, z3.h[7]\n" - "fmla z26.h, z6.h, z4.h[7]\n" - "fmla z11.h, z7.h, z0.h[7]\n" - "fmla z15.h, z7.h, z1.h[7]\n" - "fmla z19.h, z7.h, z2.h[7]\n" - "fmla z23.h, z7.h, z3.h[7]\n" - "fmla z27.h, z7.h, z4.h[7]\n" + "fmla z10.h, z29.h, z0.h[7]\n" + "fmla z14.h, z29.h, z1.h[7]\n" + "fmla z18.h, z29.h, z2.h[7]\n" + "fmla z22.h, z29.h, z3.h[7]\n" + "fmla z26.h, z29.h, z4.h[7]\n" + "fmla z11.h, z28.h, z0.h[7]\n" + "fmla z15.h, z28.h, z1.h[7]\n" + "fmla z19.h, z28.h, z2.h[7]\n" + "fmla z23.h, z28.h, z3.h[7]\n" + "fmla z27.h, z28.h, z4.h[7]\n" "63:" // Height 5: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -2214,49 +2214,49 @@ void sve_hybrid_fp16_mla_6x4VL ( "add x22, x23, x20, LSL #1\n" "tbz %x[flags], #1, 64f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rh { z1.h }, p5/Z, [x20]\n" + "ld1rh { z29.h }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rh { z0.h }, p5/Z, [x20]\n" - "fmin z8.h, p5/M, z8.h, z1.h\n" - "fmin z9.h, p5/M, z9.h, z1.h\n" - "fmin z10.h, p5/M, z10.h, z1.h\n" - "fmin z11.h, p5/M, z11.h, z1.h\n" - "fmin z12.h, p5/M, z12.h, z1.h\n" - "fmin z13.h, p5/M, z13.h, z1.h\n" - "fmin z14.h, p5/M, z14.h, z1.h\n" - "fmin z15.h, p5/M, z15.h, z1.h\n" - "fmin z16.h, p5/M, z16.h, z1.h\n" - "fmin z17.h, p5/M, z17.h, z1.h\n" - "fmin z18.h, p5/M, z18.h, z1.h\n" - "fmin z19.h, p5/M, z19.h, z1.h\n" - "fmin z20.h, p5/M, z20.h, z1.h\n" - "fmin z21.h, p5/M, z21.h, z1.h\n" - "fmin z22.h, p5/M, z22.h, z1.h\n" - "fmin z23.h, p5/M, z23.h, z1.h\n" - "fmin z24.h, p5/M, z24.h, z1.h\n" - "fmin z25.h, p5/M, z25.h, z1.h\n" - "fmin z26.h, p5/M, z26.h, z1.h\n" - "fmin z27.h, p5/M, z27.h, z1.h\n" - "fmax z8.h, p5/M, z8.h, z0.h\n" - "fmax z9.h, p5/M, z9.h, z0.h\n" - "fmax z10.h, p5/M, z10.h, z0.h\n" - "fmax z11.h, p5/M, z11.h, z0.h\n" - "fmax z12.h, p5/M, z12.h, z0.h\n" - "fmax z13.h, p5/M, z13.h, z0.h\n" - "fmax z14.h, p5/M, z14.h, z0.h\n" - "fmax z15.h, p5/M, z15.h, z0.h\n" - "fmax z16.h, p5/M, z16.h, z0.h\n" - "fmax z17.h, p5/M, z17.h, z0.h\n" - "fmax z18.h, p5/M, z18.h, z0.h\n" - "fmax z19.h, p5/M, z19.h, z0.h\n" - "fmax z20.h, p5/M, z20.h, z0.h\n" - "fmax z21.h, p5/M, z21.h, z0.h\n" - "fmax z22.h, p5/M, z22.h, z0.h\n" - "fmax z23.h, p5/M, z23.h, z0.h\n" - "fmax z24.h, p5/M, z24.h, z0.h\n" - "fmax z25.h, p5/M, z25.h, z0.h\n" - "fmax z26.h, p5/M, z26.h, z0.h\n" - "fmax z27.h, p5/M, z27.h, z0.h\n" + "ld1rh { z28.h }, p5/Z, [x20]\n" + "fmin z8.h, p5/M, z8.h, z29.h\n" + "fmin z9.h, p5/M, z9.h, z29.h\n" + "fmin z10.h, p5/M, z10.h, z29.h\n" + "fmin z11.h, p5/M, z11.h, z29.h\n" + "fmin z12.h, p5/M, z12.h, z29.h\n" + "fmin z13.h, p5/M, z13.h, z29.h\n" + "fmin z14.h, p5/M, z14.h, z29.h\n" + "fmin z15.h, p5/M, z15.h, z29.h\n" + "fmin z16.h, p5/M, z16.h, z29.h\n" + "fmin z17.h, p5/M, z17.h, z29.h\n" + "fmin z18.h, p5/M, z18.h, z29.h\n" + "fmin z19.h, p5/M, z19.h, z29.h\n" + "fmin z20.h, p5/M, z20.h, z29.h\n" + "fmin z21.h, p5/M, z21.h, z29.h\n" + "fmin z22.h, p5/M, z22.h, z29.h\n" + "fmin z23.h, p5/M, z23.h, z29.h\n" + "fmin z24.h, p5/M, z24.h, z29.h\n" + "fmin z25.h, p5/M, z25.h, z29.h\n" + "fmin z26.h, p5/M, z26.h, z29.h\n" + "fmin z27.h, p5/M, z27.h, z29.h\n" + "fmax z8.h, p5/M, z8.h, z28.h\n" + "fmax z9.h, p5/M, z9.h, z28.h\n" + "fmax z10.h, p5/M, z10.h, z28.h\n" + "fmax z11.h, p5/M, z11.h, z28.h\n" + "fmax z12.h, p5/M, z12.h, z28.h\n" + "fmax z13.h, p5/M, z13.h, z28.h\n" + "fmax z14.h, p5/M, z14.h, z28.h\n" + "fmax z15.h, p5/M, z15.h, z28.h\n" + "fmax z16.h, p5/M, z16.h, z28.h\n" + "fmax z17.h, p5/M, z17.h, z28.h\n" + "fmax z18.h, p5/M, z18.h, z28.h\n" + "fmax z19.h, p5/M, z19.h, z28.h\n" + "fmax z20.h, p5/M, z20.h, z28.h\n" + "fmax z21.h, p5/M, z21.h, z28.h\n" + "fmax z22.h, p5/M, z22.h, z28.h\n" + "fmax z23.h, p5/M, z23.h, z28.h\n" + "fmax z24.h, p5/M, z24.h, z28.h\n" + "fmax z25.h, p5/M, z25.h, z28.h\n" + "fmax z26.h, p5/M, z26.h, z28.h\n" + "fmax z27.h, p5/M, z27.h, z28.h\n" "64:" // Height 5: No activation "st1h { z8.h }, p4, [x9]\n" "st1h { z9.h }, p3, [x9, #1, MUL VL]\n" @@ -2331,35 +2331,35 @@ void sve_hybrid_fp16_mla_6x4VL ( "68:" // Height 6: no bias "tbz %x[flags], #0, 69f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "ld1h { z8.h }, p4/Z, [x9]\n" + "add x24, x9, x20, LSL #1\n" "add x23, x24, x20, LSL #1\n" + "ld1h { z8.h }, p4/Z, [x9]\n" "add x22, x23, x20, LSL #1\n" + "add x21, x22, x20, LSL #1\n" "ld1h { z9.h }, p3/Z, [x9, #1, MUL VL]\n" "ld1h { z10.h }, p2/Z, [x9, #2, MUL VL]\n" - "add x21, x22, x20, LSL #1\n" + "add x20, x21, x20, LSL #1\n" "ld1h { z11.h }, p1/Z, [x9, #3, MUL VL]\n" - "ld1h { z12.h }, p4/Z, [x25]\n" - "ld1h { z13.h }, p3/Z, [x25, #1, MUL VL]\n" - "ld1h { z14.h }, p2/Z, [x25, #2, MUL VL]\n" - "ld1h { z15.h }, p1/Z, [x25, #3, MUL VL]\n" - "ld1h { z16.h }, p4/Z, [x24]\n" - "ld1h { z17.h }, p3/Z, [x24, #1, MUL VL]\n" - "ld1h { z18.h }, p2/Z, [x24, #2, MUL VL]\n" - "ld1h { z19.h }, p1/Z, [x24, #3, MUL VL]\n" - "ld1h { z20.h }, p4/Z, [x23]\n" - "ld1h { z21.h }, p3/Z, [x23, #1, MUL VL]\n" - "ld1h { z22.h }, p2/Z, [x23, #2, MUL VL]\n" - "ld1h { z23.h }, p1/Z, [x23, #3, MUL VL]\n" - "ld1h { z24.h }, p4/Z, [x22]\n" - "ld1h { z25.h }, p3/Z, [x22, #1, MUL VL]\n" - "ld1h { z26.h }, p2/Z, [x22, #2, MUL VL]\n" - "ld1h { z27.h }, p1/Z, [x22, #3, MUL VL]\n" - "ld1h { z28.h }, p4/Z, [x21]\n" - "ld1h { z29.h }, p3/Z, [x21, #1, MUL VL]\n" - "ld1h { z30.h }, p2/Z, [x21, #2, MUL VL]\n" - "ld1h { z31.h }, p1/Z, [x21, #3, MUL VL]\n" + "ld1h { z12.h }, p4/Z, [x24]\n" + "ld1h { z13.h }, p3/Z, [x24, #1, MUL VL]\n" + "ld1h { z14.h }, p2/Z, [x24, #2, MUL VL]\n" + "ld1h { z15.h }, p1/Z, [x24, #3, MUL VL]\n" + "ld1h { z16.h }, p4/Z, [x23]\n" + "ld1h { z17.h }, p3/Z, [x23, #1, MUL VL]\n" + "ld1h { z18.h }, p2/Z, [x23, #2, MUL VL]\n" + "ld1h { z19.h }, p1/Z, [x23, #3, MUL VL]\n" + "ld1h { z20.h }, p4/Z, [x22]\n" + "ld1h { z21.h }, p3/Z, [x22, #1, MUL VL]\n" + "ld1h { z22.h }, p2/Z, [x22, #2, MUL VL]\n" + "ld1h { z23.h }, p1/Z, [x22, #3, MUL VL]\n" + "ld1h { z24.h }, p4/Z, [x21]\n" + "ld1h { z25.h }, p3/Z, [x21, #1, MUL VL]\n" + "ld1h { z26.h }, p2/Z, [x21, #2, MUL VL]\n" + "ld1h { z27.h }, p1/Z, [x21, #3, MUL VL]\n" + "ld1h { z28.h }, p4/Z, [x20]\n" + "ld1h { z29.h }, p3/Z, [x20, #1, MUL VL]\n" + "ld1h { z30.h }, p2/Z, [x20, #2, MUL VL]\n" + "ld1h { z31.h }, p1/Z, [x20, #3, MUL VL]\n" "b 70f\n" "69:" // Height 6: no accumulate "mov z8.b, #0x0\n" @@ -2391,16 +2391,16 @@ void sve_hybrid_fp16_mla_6x4VL ( "71:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 72f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 73f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #1\n" @@ -2412,256 +2412,256 @@ void sve_hybrid_fp16_mla_6x4VL ( "b 73f\n" "72:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #1\n" - "add x24, x25, x20, LSL #1\n" - "add x23, x24, x20, LSL #1\n" - "add x22, x23, x20, LSL #1\n" - "add x21, x22, x20, LSL #1\n" + "add x25, x26, x21, LSL #1\n" + "add x24, x25, x21, LSL #1\n" + "add x23, x24, x21, LSL #1\n" + "add x22, x23, x21, LSL #1\n" + "add x21, x22, x21, LSL #1\n" "73:" // Height 6: input setup done "cmp x27, #0x8\n" "ble 75f\n" "74:" // Height 6: Multiply loop: Main loop head "whilelt p0.h, XZR, x27\n" - "ld1rqh { z0.h }, p0/Z, [x26]\n" - "ld1rqh { z1.h }, p0/Z, [x25]\n" + "ld1rqh { z7.h }, p0/Z, [x26]\n" + "ld1rqh { z6.h }, p0/Z, [x25]\n" "sub x27, x27, #0x8\n" - "ld1rqh { z2.h }, p0/Z, [x24]\n" - "ld1rqh { z3.h }, p0/Z, [x23]\n" + "ld1rqh { z5.h }, p0/Z, [x24]\n" + "ld1rqh { z4.h }, p0/Z, [x23]\n" "cmp x27, #0x8\n" "add x26, x26, #0x10\n" - "ld1rqh { z4.h }, p0/Z, [x22]\n" - "ld1rqh { z5.h }, p0/Z, [x21]\n" + "ld1rqh { z3.h }, p0/Z, [x22]\n" + "ld1rqh { z2.h }, p0/Z, [x21]\n" "add x25, x25, #0x10\n" "add x24, x24, #0x10\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[0]\n" - "fmla z12.h, z6.h, z1.h[0]\n" - "fmla z16.h, z6.h, z2.h[0]\n" - "fmla z20.h, z6.h, z3.h[0]\n" + "ld1h { z1.h }, p5/Z, [x10]\n" + "ld1h { z0.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z1.h, z7.h[0]\n" + "fmla z12.h, z1.h, z6.h[0]\n" + "fmla z16.h, z1.h, z5.h[0]\n" + "fmla z20.h, z1.h, z4.h[0]\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - "fmla z24.h, z6.h, z4.h[0]\n" - "fmla z28.h, z6.h, z5.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z24.h, z1.h, z3.h[0]\n" + "fmla z28.h, z1.h, z2.h[0]\n" + "ld1h { z1.h }, p5/Z, [x10, #2, MUL VL]\n" "add x21, x21, #0x10\n" - "fmla z9.h, z7.h, z0.h[0]\n" - "fmla z13.h, z7.h, z1.h[0]\n" - "fmla z17.h, z7.h, z2.h[0]\n" - "fmla z21.h, z7.h, z3.h[0]\n" - "fmla z25.h, z7.h, z4.h[0]\n" - "fmla z29.h, z7.h, z5.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[0]\n" - "fmla z14.h, z6.h, z1.h[0]\n" - "fmla z18.h, z6.h, z2.h[0]\n" - "fmla z22.h, z6.h, z3.h[0]\n" - "fmla z26.h, z6.h, z4.h[0]\n" - "fmla z30.h, z6.h, z5.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[0]\n" - "fmla z15.h, z7.h, z1.h[0]\n" - "fmla z19.h, z7.h, z2.h[0]\n" - "fmla z23.h, z7.h, z3.h[0]\n" - "fmla z27.h, z7.h, z4.h[0]\n" - "fmla z31.h, z7.h, z5.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[1]\n" - "fmla z12.h, z6.h, z1.h[1]\n" - "fmla z16.h, z6.h, z2.h[1]\n" - "fmla z20.h, z6.h, z3.h[1]\n" - "fmla z24.h, z6.h, z4.h[1]\n" - "fmla z28.h, z6.h, z5.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[1]\n" - "fmla z13.h, z7.h, z1.h[1]\n" - "fmla z17.h, z7.h, z2.h[1]\n" - "fmla z21.h, z7.h, z3.h[1]\n" - "fmla z25.h, z7.h, z4.h[1]\n" - "fmla z29.h, z7.h, z5.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" + "fmla z9.h, z0.h, z7.h[0]\n" + "fmla z13.h, z0.h, z6.h[0]\n" + "fmla z17.h, z0.h, z5.h[0]\n" + "fmla z21.h, z0.h, z4.h[0]\n" + "fmla z25.h, z0.h, z3.h[0]\n" + "fmla z29.h, z0.h, z2.h[0]\n" + "ld1h { z0.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z10.h, z1.h, z7.h[0]\n" + "fmla z14.h, z1.h, z6.h[0]\n" + "fmla z18.h, z1.h, z5.h[0]\n" + "fmla z22.h, z1.h, z4.h[0]\n" + "fmla z26.h, z1.h, z3.h[0]\n" + "fmla z30.h, z1.h, z2.h[0]\n" + "ld1h { z1.h }, p5/Z, [x10, #4, MUL VL]\n" + "fmla z11.h, z0.h, z7.h[0]\n" + "fmla z15.h, z0.h, z6.h[0]\n" + "fmla z19.h, z0.h, z5.h[0]\n" + "fmla z23.h, z0.h, z4.h[0]\n" + "fmla z27.h, z0.h, z3.h[0]\n" + "fmla z31.h, z0.h, z2.h[0]\n" + "ld1h { z0.h }, p5/Z, [x10, #5, MUL VL]\n" + "fmla z8.h, z1.h, z7.h[1]\n" + "fmla z12.h, z1.h, z6.h[1]\n" + "fmla z16.h, z1.h, z5.h[1]\n" + "fmla z20.h, z1.h, z4.h[1]\n" + "fmla z24.h, z1.h, z3.h[1]\n" + "fmla z28.h, z1.h, z2.h[1]\n" + "ld1h { z1.h }, p5/Z, [x10, #6, MUL VL]\n" + "fmla z9.h, z0.h, z7.h[1]\n" + "fmla z13.h, z0.h, z6.h[1]\n" + "fmla z17.h, z0.h, z5.h[1]\n" + "fmla z21.h, z0.h, z4.h[1]\n" + "fmla z25.h, z0.h, z3.h[1]\n" + "fmla z29.h, z0.h, z2.h[1]\n" + "ld1h { z0.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "fmla z10.h, z6.h, z0.h[1]\n" - "fmla z14.h, z6.h, z1.h[1]\n" - "fmla z18.h, z6.h, z2.h[1]\n" - "fmla z22.h, z6.h, z3.h[1]\n" - "fmla z26.h, z6.h, z4.h[1]\n" - "fmla z30.h, z6.h, z5.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[1]\n" - "fmla z15.h, z7.h, z1.h[1]\n" - "fmla z19.h, z7.h, z2.h[1]\n" - "fmla z23.h, z7.h, z3.h[1]\n" - "fmla z27.h, z7.h, z4.h[1]\n" - "fmla z31.h, z7.h, z5.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #-7, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[2]\n" - "fmla z12.h, z6.h, z1.h[2]\n" - "fmla z16.h, z6.h, z2.h[2]\n" - "fmla z20.h, z6.h, z3.h[2]\n" - "fmla z24.h, z6.h, z4.h[2]\n" - "fmla z28.h, z6.h, z5.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[2]\n" - "fmla z13.h, z7.h, z1.h[2]\n" - "fmla z17.h, z7.h, z2.h[2]\n" - "fmla z21.h, z7.h, z3.h[2]\n" - "fmla z25.h, z7.h, z4.h[2]\n" - "fmla z29.h, z7.h, z5.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #-5, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[2]\n" - "fmla z14.h, z6.h, z1.h[2]\n" - "fmla z18.h, z6.h, z2.h[2]\n" - "fmla z22.h, z6.h, z3.h[2]\n" - "fmla z26.h, z6.h, z4.h[2]\n" - "fmla z30.h, z6.h, z5.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[2]\n" - "fmla z15.h, z7.h, z1.h[2]\n" - "fmla z19.h, z7.h, z2.h[2]\n" - "fmla z23.h, z7.h, z3.h[2]\n" - "fmla z27.h, z7.h, z4.h[2]\n" - "fmla z31.h, z7.h, z5.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #-3, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[3]\n" - "fmla z12.h, z6.h, z1.h[3]\n" - "fmla z16.h, z6.h, z2.h[3]\n" - "fmla z20.h, z6.h, z3.h[3]\n" - "fmla z24.h, z6.h, z4.h[3]\n" - "fmla z28.h, z6.h, z5.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[3]\n" - "fmla z13.h, z7.h, z1.h[3]\n" - "fmla z17.h, z7.h, z2.h[3]\n" - "fmla z21.h, z7.h, z3.h[3]\n" - "fmla z25.h, z7.h, z4.h[3]\n" - "fmla z29.h, z7.h, z5.h[3]\n" - "ld1h { z7.h }, p5/Z, [x10, #-1, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[3]\n" - "fmla z14.h, z6.h, z1.h[3]\n" - "fmla z18.h, z6.h, z2.h[3]\n" - "fmla z22.h, z6.h, z3.h[3]\n" - "fmla z26.h, z6.h, z4.h[3]\n" - "fmla z30.h, z6.h, z5.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "fmla z11.h, z7.h, z0.h[3]\n" - "fmla z15.h, z7.h, z1.h[3]\n" - "fmla z19.h, z7.h, z2.h[3]\n" - "fmla z23.h, z7.h, z3.h[3]\n" - "fmla z27.h, z7.h, z4.h[3]\n" - "fmla z31.h, z7.h, z5.h[3]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[4]\n" - "fmla z12.h, z6.h, z1.h[4]\n" - "fmla z16.h, z6.h, z2.h[4]\n" - "fmla z20.h, z6.h, z3.h[4]\n" - "fmla z24.h, z6.h, z4.h[4]\n" - "fmla z28.h, z6.h, z5.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[4]\n" - "fmla z13.h, z7.h, z1.h[4]\n" - "fmla z17.h, z7.h, z2.h[4]\n" - "fmla z21.h, z7.h, z3.h[4]\n" - "fmla z25.h, z7.h, z4.h[4]\n" - "fmla z29.h, z7.h, z5.h[4]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[4]\n" - "fmla z14.h, z6.h, z1.h[4]\n" - "fmla z18.h, z6.h, z2.h[4]\n" - "fmla z22.h, z6.h, z3.h[4]\n" - "fmla z26.h, z6.h, z4.h[4]\n" - "fmla z30.h, z6.h, z5.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[4]\n" - "fmla z15.h, z7.h, z1.h[4]\n" - "fmla z19.h, z7.h, z2.h[4]\n" - "fmla z23.h, z7.h, z3.h[4]\n" - "fmla z27.h, z7.h, z4.h[4]\n" - "fmla z31.h, z7.h, z5.h[4]\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[5]\n" - "fmla z12.h, z6.h, z1.h[5]\n" - "fmla z16.h, z6.h, z2.h[5]\n" - "fmla z20.h, z6.h, z3.h[5]\n" - "fmla z24.h, z6.h, z4.h[5]\n" - "fmla z28.h, z6.h, z5.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[5]\n" - "fmla z13.h, z7.h, z1.h[5]\n" - "fmla z17.h, z7.h, z2.h[5]\n" - "fmla z21.h, z7.h, z3.h[5]\n" - "fmla z25.h, z7.h, z4.h[5]\n" - "fmla z29.h, z7.h, z5.h[5]\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" + "fmla z10.h, z1.h, z7.h[1]\n" + "fmla z14.h, z1.h, z6.h[1]\n" + "fmla z18.h, z1.h, z5.h[1]\n" + "fmla z22.h, z1.h, z4.h[1]\n" + "fmla z26.h, z1.h, z3.h[1]\n" + "fmla z30.h, z1.h, z2.h[1]\n" + "ld1h { z1.h }, p5/Z, [x10, #-8, MUL VL]\n" + "fmla z11.h, z0.h, z7.h[1]\n" + "fmla z15.h, z0.h, z6.h[1]\n" + "fmla z19.h, z0.h, z5.h[1]\n" + "fmla z23.h, z0.h, z4.h[1]\n" + "fmla z27.h, z0.h, z3.h[1]\n" + "fmla z31.h, z0.h, z2.h[1]\n" + "ld1h { z0.h }, p5/Z, [x10, #-7, MUL VL]\n" + "fmla z8.h, z1.h, z7.h[2]\n" + "fmla z12.h, z1.h, z6.h[2]\n" + "fmla z16.h, z1.h, z5.h[2]\n" + "fmla z20.h, z1.h, z4.h[2]\n" + "fmla z24.h, z1.h, z3.h[2]\n" + "fmla z28.h, z1.h, z2.h[2]\n" + "ld1h { z1.h }, p5/Z, [x10, #-6, MUL VL]\n" + "fmla z9.h, z0.h, z7.h[2]\n" + "fmla z13.h, z0.h, z6.h[2]\n" + "fmla z17.h, z0.h, z5.h[2]\n" + "fmla z21.h, z0.h, z4.h[2]\n" + "fmla z25.h, z0.h, z3.h[2]\n" + "fmla z29.h, z0.h, z2.h[2]\n" + "ld1h { z0.h }, p5/Z, [x10, #-5, MUL VL]\n" + "fmla z10.h, z1.h, z7.h[2]\n" + "fmla z14.h, z1.h, z6.h[2]\n" + "fmla z18.h, z1.h, z5.h[2]\n" + "fmla z22.h, z1.h, z4.h[2]\n" + "fmla z26.h, z1.h, z3.h[2]\n" + "fmla z30.h, z1.h, z2.h[2]\n" + "ld1h { z1.h }, p5/Z, [x10, #-4, MUL VL]\n" + "fmla z11.h, z0.h, z7.h[2]\n" + "fmla z15.h, z0.h, z6.h[2]\n" + "fmla z19.h, z0.h, z5.h[2]\n" + "fmla z23.h, z0.h, z4.h[2]\n" + "fmla z27.h, z0.h, z3.h[2]\n" + "fmla z31.h, z0.h, z2.h[2]\n" + "ld1h { z0.h }, p5/Z, [x10, #-3, MUL VL]\n" + "fmla z8.h, z1.h, z7.h[3]\n" + "fmla z12.h, z1.h, z6.h[3]\n" + "fmla z16.h, z1.h, z5.h[3]\n" + "fmla z20.h, z1.h, z4.h[3]\n" + "fmla z24.h, z1.h, z3.h[3]\n" + "fmla z28.h, z1.h, z2.h[3]\n" + "ld1h { z1.h }, p5/Z, [x10, #-2, MUL VL]\n" + "fmla z9.h, z0.h, z7.h[3]\n" + "fmla z13.h, z0.h, z6.h[3]\n" + "fmla z17.h, z0.h, z5.h[3]\n" + "fmla z21.h, z0.h, z4.h[3]\n" + "fmla z25.h, z0.h, z3.h[3]\n" + "fmla z29.h, z0.h, z2.h[3]\n" + "ld1h { z0.h }, p5/Z, [x10, #-1, MUL VL]\n" + "fmla z10.h, z1.h, z7.h[3]\n" + "fmla z14.h, z1.h, z6.h[3]\n" + "fmla z18.h, z1.h, z5.h[3]\n" + "fmla z22.h, z1.h, z4.h[3]\n" + "fmla z26.h, z1.h, z3.h[3]\n" + "fmla z30.h, z1.h, z2.h[3]\n" + "ld1h { z1.h }, p5/Z, [x10]\n" + "fmla z11.h, z0.h, z7.h[3]\n" + "fmla z15.h, z0.h, z6.h[3]\n" + "fmla z19.h, z0.h, z5.h[3]\n" + "fmla z23.h, z0.h, z4.h[3]\n" + "fmla z27.h, z0.h, z3.h[3]\n" + "fmla z31.h, z0.h, z2.h[3]\n" + "ld1h { z0.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z1.h, z7.h[4]\n" + "fmla z12.h, z1.h, z6.h[4]\n" + "fmla z16.h, z1.h, z5.h[4]\n" + "fmla z20.h, z1.h, z4.h[4]\n" + "fmla z24.h, z1.h, z3.h[4]\n" + "fmla z28.h, z1.h, z2.h[4]\n" + "ld1h { z1.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.h, z0.h, z7.h[4]\n" + "fmla z13.h, z0.h, z6.h[4]\n" + "fmla z17.h, z0.h, z5.h[4]\n" + "fmla z21.h, z0.h, z4.h[4]\n" + "fmla z25.h, z0.h, z3.h[4]\n" + "fmla z29.h, z0.h, z2.h[4]\n" + "ld1h { z0.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z10.h, z1.h, z7.h[4]\n" + "fmla z14.h, z1.h, z6.h[4]\n" + "fmla z18.h, z1.h, z5.h[4]\n" + "fmla z22.h, z1.h, z4.h[4]\n" + "fmla z26.h, z1.h, z3.h[4]\n" + "fmla z30.h, z1.h, z2.h[4]\n" + "ld1h { z1.h }, p5/Z, [x10, #4, MUL VL]\n" + "fmla z11.h, z0.h, z7.h[4]\n" + "fmla z15.h, z0.h, z6.h[4]\n" + "fmla z19.h, z0.h, z5.h[4]\n" + "fmla z23.h, z0.h, z4.h[4]\n" + "fmla z27.h, z0.h, z3.h[4]\n" + "fmla z31.h, z0.h, z2.h[4]\n" + "ld1h { z0.h }, p5/Z, [x10, #5, MUL VL]\n" + "fmla z8.h, z1.h, z7.h[5]\n" + "fmla z12.h, z1.h, z6.h[5]\n" + "fmla z16.h, z1.h, z5.h[5]\n" + "fmla z20.h, z1.h, z4.h[5]\n" + "fmla z24.h, z1.h, z3.h[5]\n" + "fmla z28.h, z1.h, z2.h[5]\n" + "ld1h { z1.h }, p5/Z, [x10, #6, MUL VL]\n" + "fmla z9.h, z0.h, z7.h[5]\n" + "fmla z13.h, z0.h, z6.h[5]\n" + "fmla z17.h, z0.h, z5.h[5]\n" + "fmla z21.h, z0.h, z4.h[5]\n" + "fmla z25.h, z0.h, z3.h[5]\n" + "fmla z29.h, z0.h, z2.h[5]\n" + "ld1h { z0.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "fmla z10.h, z6.h, z0.h[5]\n" - "fmla z14.h, z6.h, z1.h[5]\n" - "fmla z18.h, z6.h, z2.h[5]\n" - "fmla z22.h, z6.h, z3.h[5]\n" - "fmla z26.h, z6.h, z4.h[5]\n" - "fmla z30.h, z6.h, z5.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10, #-8, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[5]\n" - "fmla z15.h, z7.h, z1.h[5]\n" - "fmla z19.h, z7.h, z2.h[5]\n" - "fmla z23.h, z7.h, z3.h[5]\n" - "fmla z27.h, z7.h, z4.h[5]\n" - "fmla z31.h, z7.h, z5.h[5]\n" - "ld1h { z7.h }, p5/Z, [x10, #-7, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[6]\n" - "fmla z12.h, z6.h, z1.h[6]\n" - "fmla z16.h, z6.h, z2.h[6]\n" - "fmla z20.h, z6.h, z3.h[6]\n" - "fmla z24.h, z6.h, z4.h[6]\n" - "fmla z28.h, z6.h, z5.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10, #-6, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[6]\n" - "fmla z13.h, z7.h, z1.h[6]\n" - "fmla z17.h, z7.h, z2.h[6]\n" - "fmla z21.h, z7.h, z3.h[6]\n" - "fmla z25.h, z7.h, z4.h[6]\n" - "fmla z29.h, z7.h, z5.h[6]\n" - "ld1h { z7.h }, p5/Z, [x10, #-5, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[6]\n" - "fmla z14.h, z6.h, z1.h[6]\n" - "fmla z18.h, z6.h, z2.h[6]\n" - "fmla z22.h, z6.h, z3.h[6]\n" - "fmla z26.h, z6.h, z4.h[6]\n" - "fmla z30.h, z6.h, z5.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10, #-4, MUL VL]\n" - "fmla z11.h, z7.h, z0.h[6]\n" - "fmla z15.h, z7.h, z1.h[6]\n" - "fmla z19.h, z7.h, z2.h[6]\n" - "fmla z23.h, z7.h, z3.h[6]\n" - "fmla z27.h, z7.h, z4.h[6]\n" - "fmla z31.h, z7.h, z5.h[6]\n" - "ld1h { z7.h }, p5/Z, [x10, #-3, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[7]\n" - "fmla z12.h, z6.h, z1.h[7]\n" - "fmla z16.h, z6.h, z2.h[7]\n" - "fmla z20.h, z6.h, z3.h[7]\n" - "fmla z24.h, z6.h, z4.h[7]\n" - "fmla z28.h, z6.h, z5.h[7]\n" - "ld1h { z6.h }, p5/Z, [x10, #-2, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[7]\n" - "fmla z13.h, z7.h, z1.h[7]\n" - "fmla z17.h, z7.h, z2.h[7]\n" - "fmla z21.h, z7.h, z3.h[7]\n" - "fmla z25.h, z7.h, z4.h[7]\n" - "fmla z29.h, z7.h, z5.h[7]\n" - "ld1h { z7.h }, p5/Z, [x10, #-1, MUL VL]\n" - "fmla z10.h, z6.h, z0.h[7]\n" - "fmla z14.h, z6.h, z1.h[7]\n" - "fmla z18.h, z6.h, z2.h[7]\n" - "fmla z22.h, z6.h, z3.h[7]\n" - "fmla z26.h, z6.h, z4.h[7]\n" - "fmla z30.h, z6.h, z5.h[7]\n" - "fmla z11.h, z7.h, z0.h[7]\n" - "fmla z15.h, z7.h, z1.h[7]\n" - "fmla z19.h, z7.h, z2.h[7]\n" - "fmla z23.h, z7.h, z3.h[7]\n" - "fmla z27.h, z7.h, z4.h[7]\n" - "fmla z31.h, z7.h, z5.h[7]\n" + "fmla z10.h, z1.h, z7.h[5]\n" + "fmla z14.h, z1.h, z6.h[5]\n" + "fmla z18.h, z1.h, z5.h[5]\n" + "fmla z22.h, z1.h, z4.h[5]\n" + "fmla z26.h, z1.h, z3.h[5]\n" + "fmla z30.h, z1.h, z2.h[5]\n" + "ld1h { z1.h }, p5/Z, [x10, #-8, MUL VL]\n" + "fmla z11.h, z0.h, z7.h[5]\n" + "fmla z15.h, z0.h, z6.h[5]\n" + "fmla z19.h, z0.h, z5.h[5]\n" + "fmla z23.h, z0.h, z4.h[5]\n" + "fmla z27.h, z0.h, z3.h[5]\n" + "fmla z31.h, z0.h, z2.h[5]\n" + "ld1h { z0.h }, p5/Z, [x10, #-7, MUL VL]\n" + "fmla z8.h, z1.h, z7.h[6]\n" + "fmla z12.h, z1.h, z6.h[6]\n" + "fmla z16.h, z1.h, z5.h[6]\n" + "fmla z20.h, z1.h, z4.h[6]\n" + "fmla z24.h, z1.h, z3.h[6]\n" + "fmla z28.h, z1.h, z2.h[6]\n" + "ld1h { z1.h }, p5/Z, [x10, #-6, MUL VL]\n" + "fmla z9.h, z0.h, z7.h[6]\n" + "fmla z13.h, z0.h, z6.h[6]\n" + "fmla z17.h, z0.h, z5.h[6]\n" + "fmla z21.h, z0.h, z4.h[6]\n" + "fmla z25.h, z0.h, z3.h[6]\n" + "fmla z29.h, z0.h, z2.h[6]\n" + "ld1h { z0.h }, p5/Z, [x10, #-5, MUL VL]\n" + "fmla z10.h, z1.h, z7.h[6]\n" + "fmla z14.h, z1.h, z6.h[6]\n" + "fmla z18.h, z1.h, z5.h[6]\n" + "fmla z22.h, z1.h, z4.h[6]\n" + "fmla z26.h, z1.h, z3.h[6]\n" + "fmla z30.h, z1.h, z2.h[6]\n" + "ld1h { z1.h }, p5/Z, [x10, #-4, MUL VL]\n" + "fmla z11.h, z0.h, z7.h[6]\n" + "fmla z15.h, z0.h, z6.h[6]\n" + "fmla z19.h, z0.h, z5.h[6]\n" + "fmla z23.h, z0.h, z4.h[6]\n" + "fmla z27.h, z0.h, z3.h[6]\n" + "fmla z31.h, z0.h, z2.h[6]\n" + "ld1h { z0.h }, p5/Z, [x10, #-3, MUL VL]\n" + "fmla z8.h, z1.h, z7.h[7]\n" + "fmla z12.h, z1.h, z6.h[7]\n" + "fmla z16.h, z1.h, z5.h[7]\n" + "fmla z20.h, z1.h, z4.h[7]\n" + "fmla z24.h, z1.h, z3.h[7]\n" + "fmla z28.h, z1.h, z2.h[7]\n" + "ld1h { z1.h }, p5/Z, [x10, #-2, MUL VL]\n" + "fmla z9.h, z0.h, z7.h[7]\n" + "fmla z13.h, z0.h, z6.h[7]\n" + "fmla z17.h, z0.h, z5.h[7]\n" + "fmla z21.h, z0.h, z4.h[7]\n" + "fmla z25.h, z0.h, z3.h[7]\n" + "fmla z29.h, z0.h, z2.h[7]\n" + "ld1h { z0.h }, p5/Z, [x10, #-1, MUL VL]\n" + "fmla z10.h, z1.h, z7.h[7]\n" + "fmla z14.h, z1.h, z6.h[7]\n" + "fmla z18.h, z1.h, z5.h[7]\n" + "fmla z22.h, z1.h, z4.h[7]\n" + "fmla z26.h, z1.h, z3.h[7]\n" + "fmla z30.h, z1.h, z2.h[7]\n" + "fmla z11.h, z0.h, z7.h[7]\n" + "fmla z15.h, z0.h, z6.h[7]\n" + "fmla z19.h, z0.h, z5.h[7]\n" + "fmla z23.h, z0.h, z4.h[7]\n" + "fmla z27.h, z0.h, z3.h[7]\n" + "fmla z31.h, z0.h, z2.h[7]\n" "bgt 74b\n" "75:" // Height 6: Multiply loop: Single iteration only "whilelt p0.h, XZR, x27\n" @@ -2672,251 +2672,251 @@ void sve_hybrid_fp16_mla_6x4VL ( "ld1rqh { z3.h }, p0/Z, [x23]\n" "ld1rqh { z4.h }, p0/Z, [x22]\n" "ld1rqh { z5.h }, p0/Z, [x21]\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[0]\n" - "fmla z12.h, z6.h, z1.h[0]\n" - "fmla z16.h, z6.h, z2.h[0]\n" - "fmla z20.h, z6.h, z3.h[0]\n" - "fmla z24.h, z6.h, z4.h[0]\n" - "fmla z28.h, z6.h, z5.h[0]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[0]\n" - "fmla z13.h, z7.h, z1.h[0]\n" - "fmla z17.h, z7.h, z2.h[0]\n" - "fmla z21.h, z7.h, z3.h[0]\n" - "fmla z25.h, z7.h, z4.h[0]\n" - "fmla z29.h, z7.h, z5.h[0]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z7.h }, p5/Z, [x10]\n" + "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z7.h, z0.h[0]\n" + "fmla z12.h, z7.h, z1.h[0]\n" + "fmla z16.h, z7.h, z2.h[0]\n" + "fmla z20.h, z7.h, z3.h[0]\n" + "fmla z24.h, z7.h, z4.h[0]\n" + "fmla z28.h, z7.h, z5.h[0]\n" + "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.h, z6.h, z0.h[0]\n" + "fmla z13.h, z6.h, z1.h[0]\n" + "fmla z17.h, z6.h, z2.h[0]\n" + "fmla z21.h, z6.h, z3.h[0]\n" + "fmla z25.h, z6.h, z4.h[0]\n" + "fmla z29.h, z6.h, z5.h[0]\n" + "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[0]\n" - "fmla z14.h, z6.h, z1.h[0]\n" - "fmla z18.h, z6.h, z2.h[0]\n" - "fmla z22.h, z6.h, z3.h[0]\n" - "fmla z26.h, z6.h, z4.h[0]\n" - "fmla z30.h, z6.h, z5.h[0]\n" - "fmla z11.h, z7.h, z0.h[0]\n" - "fmla z15.h, z7.h, z1.h[0]\n" - "fmla z19.h, z7.h, z2.h[0]\n" - "fmla z23.h, z7.h, z3.h[0]\n" - "fmla z27.h, z7.h, z4.h[0]\n" - "fmla z31.h, z7.h, z5.h[0]\n" + "fmla z10.h, z7.h, z0.h[0]\n" + "fmla z14.h, z7.h, z1.h[0]\n" + "fmla z18.h, z7.h, z2.h[0]\n" + "fmla z22.h, z7.h, z3.h[0]\n" + "fmla z26.h, z7.h, z4.h[0]\n" + "fmla z30.h, z7.h, z5.h[0]\n" + "fmla z11.h, z6.h, z0.h[0]\n" + "fmla z15.h, z6.h, z1.h[0]\n" + "fmla z19.h, z6.h, z2.h[0]\n" + "fmla z23.h, z6.h, z3.h[0]\n" + "fmla z27.h, z6.h, z4.h[0]\n" + "fmla z31.h, z6.h, z5.h[0]\n" "ble 76f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[1]\n" - "fmla z12.h, z6.h, z1.h[1]\n" - "fmla z16.h, z6.h, z2.h[1]\n" - "fmla z20.h, z6.h, z3.h[1]\n" + "ld1h { z7.h }, p5/Z, [x10]\n" + "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z7.h, z0.h[1]\n" + "fmla z12.h, z7.h, z1.h[1]\n" + "fmla z16.h, z7.h, z2.h[1]\n" + "fmla z20.h, z7.h, z3.h[1]\n" "subs x27, x27, #0x1\n" - "fmla z24.h, z6.h, z4.h[1]\n" - "fmla z28.h, z6.h, z5.h[1]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[1]\n" - "fmla z13.h, z7.h, z1.h[1]\n" - "fmla z17.h, z7.h, z2.h[1]\n" - "fmla z21.h, z7.h, z3.h[1]\n" - "fmla z25.h, z7.h, z4.h[1]\n" - "fmla z29.h, z7.h, z5.h[1]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z24.h, z7.h, z4.h[1]\n" + "fmla z28.h, z7.h, z5.h[1]\n" + "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.h, z6.h, z0.h[1]\n" + "fmla z13.h, z6.h, z1.h[1]\n" + "fmla z17.h, z6.h, z2.h[1]\n" + "fmla z21.h, z6.h, z3.h[1]\n" + "fmla z25.h, z6.h, z4.h[1]\n" + "fmla z29.h, z6.h, z5.h[1]\n" + "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[1]\n" - "fmla z14.h, z6.h, z1.h[1]\n" - "fmla z18.h, z6.h, z2.h[1]\n" - "fmla z22.h, z6.h, z3.h[1]\n" - "fmla z26.h, z6.h, z4.h[1]\n" - "fmla z30.h, z6.h, z5.h[1]\n" - "fmla z11.h, z7.h, z0.h[1]\n" - "fmla z15.h, z7.h, z1.h[1]\n" - "fmla z19.h, z7.h, z2.h[1]\n" - "fmla z23.h, z7.h, z3.h[1]\n" - "fmla z27.h, z7.h, z4.h[1]\n" - "fmla z31.h, z7.h, z5.h[1]\n" + "fmla z10.h, z7.h, z0.h[1]\n" + "fmla z14.h, z7.h, z1.h[1]\n" + "fmla z18.h, z7.h, z2.h[1]\n" + "fmla z22.h, z7.h, z3.h[1]\n" + "fmla z26.h, z7.h, z4.h[1]\n" + "fmla z30.h, z7.h, z5.h[1]\n" + "fmla z11.h, z6.h, z0.h[1]\n" + "fmla z15.h, z6.h, z1.h[1]\n" + "fmla z19.h, z6.h, z2.h[1]\n" + "fmla z23.h, z6.h, z3.h[1]\n" + "fmla z27.h, z6.h, z4.h[1]\n" + "fmla z31.h, z6.h, z5.h[1]\n" "ble 76f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[2]\n" - "fmla z12.h, z6.h, z1.h[2]\n" - "fmla z16.h, z6.h, z2.h[2]\n" - "fmla z20.h, z6.h, z3.h[2]\n" + "ld1h { z7.h }, p5/Z, [x10]\n" + "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z7.h, z0.h[2]\n" + "fmla z12.h, z7.h, z1.h[2]\n" + "fmla z16.h, z7.h, z2.h[2]\n" + "fmla z20.h, z7.h, z3.h[2]\n" "subs x27, x27, #0x1\n" - "fmla z24.h, z6.h, z4.h[2]\n" - "fmla z28.h, z6.h, z5.h[2]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[2]\n" - "fmla z13.h, z7.h, z1.h[2]\n" - "fmla z17.h, z7.h, z2.h[2]\n" - "fmla z21.h, z7.h, z3.h[2]\n" - "fmla z25.h, z7.h, z4.h[2]\n" - "fmla z29.h, z7.h, z5.h[2]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z24.h, z7.h, z4.h[2]\n" + "fmla z28.h, z7.h, z5.h[2]\n" + "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.h, z6.h, z0.h[2]\n" + "fmla z13.h, z6.h, z1.h[2]\n" + "fmla z17.h, z6.h, z2.h[2]\n" + "fmla z21.h, z6.h, z3.h[2]\n" + "fmla z25.h, z6.h, z4.h[2]\n" + "fmla z29.h, z6.h, z5.h[2]\n" + "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[2]\n" - "fmla z14.h, z6.h, z1.h[2]\n" - "fmla z18.h, z6.h, z2.h[2]\n" - "fmla z22.h, z6.h, z3.h[2]\n" - "fmla z26.h, z6.h, z4.h[2]\n" - "fmla z30.h, z6.h, z5.h[2]\n" - "fmla z11.h, z7.h, z0.h[2]\n" - "fmla z15.h, z7.h, z1.h[2]\n" - "fmla z19.h, z7.h, z2.h[2]\n" - "fmla z23.h, z7.h, z3.h[2]\n" - "fmla z27.h, z7.h, z4.h[2]\n" - "fmla z31.h, z7.h, z5.h[2]\n" + "fmla z10.h, z7.h, z0.h[2]\n" + "fmla z14.h, z7.h, z1.h[2]\n" + "fmla z18.h, z7.h, z2.h[2]\n" + "fmla z22.h, z7.h, z3.h[2]\n" + "fmla z26.h, z7.h, z4.h[2]\n" + "fmla z30.h, z7.h, z5.h[2]\n" + "fmla z11.h, z6.h, z0.h[2]\n" + "fmla z15.h, z6.h, z1.h[2]\n" + "fmla z19.h, z6.h, z2.h[2]\n" + "fmla z23.h, z6.h, z3.h[2]\n" + "fmla z27.h, z6.h, z4.h[2]\n" + "fmla z31.h, z6.h, z5.h[2]\n" "ble 76f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[3]\n" - "fmla z12.h, z6.h, z1.h[3]\n" - "fmla z16.h, z6.h, z2.h[3]\n" - "fmla z20.h, z6.h, z3.h[3]\n" + "ld1h { z7.h }, p5/Z, [x10]\n" + "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z7.h, z0.h[3]\n" + "fmla z12.h, z7.h, z1.h[3]\n" + "fmla z16.h, z7.h, z2.h[3]\n" + "fmla z20.h, z7.h, z3.h[3]\n" "subs x27, x27, #0x1\n" - "fmla z24.h, z6.h, z4.h[3]\n" - "fmla z28.h, z6.h, z5.h[3]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[3]\n" - "fmla z13.h, z7.h, z1.h[3]\n" - "fmla z17.h, z7.h, z2.h[3]\n" - "fmla z21.h, z7.h, z3.h[3]\n" - "fmla z25.h, z7.h, z4.h[3]\n" - "fmla z29.h, z7.h, z5.h[3]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z24.h, z7.h, z4.h[3]\n" + "fmla z28.h, z7.h, z5.h[3]\n" + "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.h, z6.h, z0.h[3]\n" + "fmla z13.h, z6.h, z1.h[3]\n" + "fmla z17.h, z6.h, z2.h[3]\n" + "fmla z21.h, z6.h, z3.h[3]\n" + "fmla z25.h, z6.h, z4.h[3]\n" + "fmla z29.h, z6.h, z5.h[3]\n" + "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[3]\n" - "fmla z14.h, z6.h, z1.h[3]\n" - "fmla z18.h, z6.h, z2.h[3]\n" - "fmla z22.h, z6.h, z3.h[3]\n" - "fmla z26.h, z6.h, z4.h[3]\n" - "fmla z30.h, z6.h, z5.h[3]\n" - "fmla z11.h, z7.h, z0.h[3]\n" - "fmla z15.h, z7.h, z1.h[3]\n" - "fmla z19.h, z7.h, z2.h[3]\n" - "fmla z23.h, z7.h, z3.h[3]\n" - "fmla z27.h, z7.h, z4.h[3]\n" - "fmla z31.h, z7.h, z5.h[3]\n" + "fmla z10.h, z7.h, z0.h[3]\n" + "fmla z14.h, z7.h, z1.h[3]\n" + "fmla z18.h, z7.h, z2.h[3]\n" + "fmla z22.h, z7.h, z3.h[3]\n" + "fmla z26.h, z7.h, z4.h[3]\n" + "fmla z30.h, z7.h, z5.h[3]\n" + "fmla z11.h, z6.h, z0.h[3]\n" + "fmla z15.h, z6.h, z1.h[3]\n" + "fmla z19.h, z6.h, z2.h[3]\n" + "fmla z23.h, z6.h, z3.h[3]\n" + "fmla z27.h, z6.h, z4.h[3]\n" + "fmla z31.h, z6.h, z5.h[3]\n" "ble 76f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[4]\n" - "fmla z12.h, z6.h, z1.h[4]\n" - "fmla z16.h, z6.h, z2.h[4]\n" - "fmla z20.h, z6.h, z3.h[4]\n" + "ld1h { z7.h }, p5/Z, [x10]\n" + "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z7.h, z0.h[4]\n" + "fmla z12.h, z7.h, z1.h[4]\n" + "fmla z16.h, z7.h, z2.h[4]\n" + "fmla z20.h, z7.h, z3.h[4]\n" "subs x27, x27, #0x1\n" - "fmla z24.h, z6.h, z4.h[4]\n" - "fmla z28.h, z6.h, z5.h[4]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[4]\n" - "fmla z13.h, z7.h, z1.h[4]\n" - "fmla z17.h, z7.h, z2.h[4]\n" - "fmla z21.h, z7.h, z3.h[4]\n" - "fmla z25.h, z7.h, z4.h[4]\n" - "fmla z29.h, z7.h, z5.h[4]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z24.h, z7.h, z4.h[4]\n" + "fmla z28.h, z7.h, z5.h[4]\n" + "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.h, z6.h, z0.h[4]\n" + "fmla z13.h, z6.h, z1.h[4]\n" + "fmla z17.h, z6.h, z2.h[4]\n" + "fmla z21.h, z6.h, z3.h[4]\n" + "fmla z25.h, z6.h, z4.h[4]\n" + "fmla z29.h, z6.h, z5.h[4]\n" + "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[4]\n" - "fmla z14.h, z6.h, z1.h[4]\n" - "fmla z18.h, z6.h, z2.h[4]\n" - "fmla z22.h, z6.h, z3.h[4]\n" - "fmla z26.h, z6.h, z4.h[4]\n" - "fmla z30.h, z6.h, z5.h[4]\n" - "fmla z11.h, z7.h, z0.h[4]\n" - "fmla z15.h, z7.h, z1.h[4]\n" - "fmla z19.h, z7.h, z2.h[4]\n" - "fmla z23.h, z7.h, z3.h[4]\n" - "fmla z27.h, z7.h, z4.h[4]\n" - "fmla z31.h, z7.h, z5.h[4]\n" + "fmla z10.h, z7.h, z0.h[4]\n" + "fmla z14.h, z7.h, z1.h[4]\n" + "fmla z18.h, z7.h, z2.h[4]\n" + "fmla z22.h, z7.h, z3.h[4]\n" + "fmla z26.h, z7.h, z4.h[4]\n" + "fmla z30.h, z7.h, z5.h[4]\n" + "fmla z11.h, z6.h, z0.h[4]\n" + "fmla z15.h, z6.h, z1.h[4]\n" + "fmla z19.h, z6.h, z2.h[4]\n" + "fmla z23.h, z6.h, z3.h[4]\n" + "fmla z27.h, z6.h, z4.h[4]\n" + "fmla z31.h, z6.h, z5.h[4]\n" "ble 76f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[5]\n" - "fmla z12.h, z6.h, z1.h[5]\n" - "fmla z16.h, z6.h, z2.h[5]\n" - "fmla z20.h, z6.h, z3.h[5]\n" + "ld1h { z7.h }, p5/Z, [x10]\n" + "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z7.h, z0.h[5]\n" + "fmla z12.h, z7.h, z1.h[5]\n" + "fmla z16.h, z7.h, z2.h[5]\n" + "fmla z20.h, z7.h, z3.h[5]\n" "subs x27, x27, #0x1\n" - "fmla z24.h, z6.h, z4.h[5]\n" - "fmla z28.h, z6.h, z5.h[5]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[5]\n" - "fmla z13.h, z7.h, z1.h[5]\n" - "fmla z17.h, z7.h, z2.h[5]\n" - "fmla z21.h, z7.h, z3.h[5]\n" - "fmla z25.h, z7.h, z4.h[5]\n" - "fmla z29.h, z7.h, z5.h[5]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z24.h, z7.h, z4.h[5]\n" + "fmla z28.h, z7.h, z5.h[5]\n" + "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.h, z6.h, z0.h[5]\n" + "fmla z13.h, z6.h, z1.h[5]\n" + "fmla z17.h, z6.h, z2.h[5]\n" + "fmla z21.h, z6.h, z3.h[5]\n" + "fmla z25.h, z6.h, z4.h[5]\n" + "fmla z29.h, z6.h, z5.h[5]\n" + "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[5]\n" - "fmla z14.h, z6.h, z1.h[5]\n" - "fmla z18.h, z6.h, z2.h[5]\n" - "fmla z22.h, z6.h, z3.h[5]\n" - "fmla z26.h, z6.h, z4.h[5]\n" - "fmla z30.h, z6.h, z5.h[5]\n" - "fmla z11.h, z7.h, z0.h[5]\n" - "fmla z15.h, z7.h, z1.h[5]\n" - "fmla z19.h, z7.h, z2.h[5]\n" - "fmla z23.h, z7.h, z3.h[5]\n" - "fmla z27.h, z7.h, z4.h[5]\n" - "fmla z31.h, z7.h, z5.h[5]\n" + "fmla z10.h, z7.h, z0.h[5]\n" + "fmla z14.h, z7.h, z1.h[5]\n" + "fmla z18.h, z7.h, z2.h[5]\n" + "fmla z22.h, z7.h, z3.h[5]\n" + "fmla z26.h, z7.h, z4.h[5]\n" + "fmla z30.h, z7.h, z5.h[5]\n" + "fmla z11.h, z6.h, z0.h[5]\n" + "fmla z15.h, z6.h, z1.h[5]\n" + "fmla z19.h, z6.h, z2.h[5]\n" + "fmla z23.h, z6.h, z3.h[5]\n" + "fmla z27.h, z6.h, z4.h[5]\n" + "fmla z31.h, z6.h, z5.h[5]\n" "ble 76f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[6]\n" - "fmla z12.h, z6.h, z1.h[6]\n" - "fmla z16.h, z6.h, z2.h[6]\n" - "fmla z20.h, z6.h, z3.h[6]\n" + "ld1h { z7.h }, p5/Z, [x10]\n" + "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z7.h, z0.h[6]\n" + "fmla z12.h, z7.h, z1.h[6]\n" + "fmla z16.h, z7.h, z2.h[6]\n" + "fmla z20.h, z7.h, z3.h[6]\n" "subs x27, x27, #0x1\n" - "fmla z24.h, z6.h, z4.h[6]\n" - "fmla z28.h, z6.h, z5.h[6]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[6]\n" - "fmla z13.h, z7.h, z1.h[6]\n" - "fmla z17.h, z7.h, z2.h[6]\n" - "fmla z21.h, z7.h, z3.h[6]\n" - "fmla z25.h, z7.h, z4.h[6]\n" - "fmla z29.h, z7.h, z5.h[6]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z24.h, z7.h, z4.h[6]\n" + "fmla z28.h, z7.h, z5.h[6]\n" + "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.h, z6.h, z0.h[6]\n" + "fmla z13.h, z6.h, z1.h[6]\n" + "fmla z17.h, z6.h, z2.h[6]\n" + "fmla z21.h, z6.h, z3.h[6]\n" + "fmla z25.h, z6.h, z4.h[6]\n" + "fmla z29.h, z6.h, z5.h[6]\n" + "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[6]\n" - "fmla z14.h, z6.h, z1.h[6]\n" - "fmla z18.h, z6.h, z2.h[6]\n" - "fmla z22.h, z6.h, z3.h[6]\n" - "fmla z26.h, z6.h, z4.h[6]\n" - "fmla z30.h, z6.h, z5.h[6]\n" - "fmla z11.h, z7.h, z0.h[6]\n" - "fmla z15.h, z7.h, z1.h[6]\n" - "fmla z19.h, z7.h, z2.h[6]\n" - "fmla z23.h, z7.h, z3.h[6]\n" - "fmla z27.h, z7.h, z4.h[6]\n" - "fmla z31.h, z7.h, z5.h[6]\n" + "fmla z10.h, z7.h, z0.h[6]\n" + "fmla z14.h, z7.h, z1.h[6]\n" + "fmla z18.h, z7.h, z2.h[6]\n" + "fmla z22.h, z7.h, z3.h[6]\n" + "fmla z26.h, z7.h, z4.h[6]\n" + "fmla z30.h, z7.h, z5.h[6]\n" + "fmla z11.h, z6.h, z0.h[6]\n" + "fmla z15.h, z6.h, z1.h[6]\n" + "fmla z19.h, z6.h, z2.h[6]\n" + "fmla z23.h, z6.h, z3.h[6]\n" + "fmla z27.h, z6.h, z4.h[6]\n" + "fmla z31.h, z6.h, z5.h[6]\n" "ble 76f\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.h, z6.h, z0.h[7]\n" - "fmla z12.h, z6.h, z1.h[7]\n" - "fmla z16.h, z6.h, z2.h[7]\n" - "fmla z20.h, z6.h, z3.h[7]\n" - "fmla z24.h, z6.h, z4.h[7]\n" - "fmla z28.h, z6.h, z5.h[7]\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.h, z7.h, z0.h[7]\n" - "fmla z13.h, z7.h, z1.h[7]\n" - "fmla z17.h, z7.h, z2.h[7]\n" - "fmla z21.h, z7.h, z3.h[7]\n" - "fmla z25.h, z7.h, z4.h[7]\n" - "fmla z29.h, z7.h, z5.h[7]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + "ld1h { z7.h }, p5/Z, [x10]\n" + "ld1h { z6.h }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.h, z7.h, z0.h[7]\n" + "fmla z12.h, z7.h, z1.h[7]\n" + "fmla z16.h, z7.h, z2.h[7]\n" + "fmla z20.h, z7.h, z3.h[7]\n" + "fmla z24.h, z7.h, z4.h[7]\n" + "fmla z28.h, z7.h, z5.h[7]\n" + "ld1h { z7.h }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.h, z6.h, z0.h[7]\n" + "fmla z13.h, z6.h, z1.h[7]\n" + "fmla z17.h, z6.h, z2.h[7]\n" + "fmla z21.h, z6.h, z3.h[7]\n" + "fmla z25.h, z6.h, z4.h[7]\n" + "fmla z29.h, z6.h, z5.h[7]\n" + "ld1h { z6.h }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.h, z6.h, z0.h[7]\n" - "fmla z14.h, z6.h, z1.h[7]\n" - "fmla z18.h, z6.h, z2.h[7]\n" - "fmla z22.h, z6.h, z3.h[7]\n" - "fmla z26.h, z6.h, z4.h[7]\n" - "fmla z30.h, z6.h, z5.h[7]\n" - "fmla z11.h, z7.h, z0.h[7]\n" - "fmla z15.h, z7.h, z1.h[7]\n" - "fmla z19.h, z7.h, z2.h[7]\n" - "fmla z23.h, z7.h, z3.h[7]\n" - "fmla z27.h, z7.h, z4.h[7]\n" - "fmla z31.h, z7.h, z5.h[7]\n" + "fmla z10.h, z7.h, z0.h[7]\n" + "fmla z14.h, z7.h, z1.h[7]\n" + "fmla z18.h, z7.h, z2.h[7]\n" + "fmla z22.h, z7.h, z3.h[7]\n" + "fmla z26.h, z7.h, z4.h[7]\n" + "fmla z30.h, z7.h, z5.h[7]\n" + "fmla z11.h, z6.h, z0.h[7]\n" + "fmla z15.h, z6.h, z1.h[7]\n" + "fmla z19.h, z6.h, z2.h[7]\n" + "fmla z23.h, z6.h, z3.h[7]\n" + "fmla z27.h, z6.h, z4.h[7]\n" + "fmla z31.h, z6.h, z5.h[7]\n" "76:" // Height 6: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -3023,7 +3023,6 @@ void sve_hybrid_fp16_mla_6x4VL ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "80:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -3031,4 +3030,4 @@ void sve_hybrid_fp16_mla_6x4VL ( } } // namespace arm_gemm -#endif // ARM_COMPUTE_ENABLE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL.hpp index b63b143d4c481aa09203a23f56ed75548afc79ec..880f9d1a27e92b97216f1b2d519abfb895e3a7f9 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef ARM_COMPUTE_ENABLE_SVE + #include "../std_transforms_sve.hpp" #include "../performance_parameters.hpp" @@ -75,13 +75,16 @@ public: template static inline PerformanceParameters get_performance_parameters(const CPUInfo *ci) { - if (std::is_same::value) { switch (ci->get_cpu_model()) { - case CPUModel::V1: - return { 15.65 }; default: return { 6.667 }; + case CPUModel::A510: + return { 5.41 }; + case CPUModel::V1: + return { 15.65 }; + case CPUModel::A64FX: + return { 25.55 }; } } @@ -105,5 +108,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL/a64fx.cpp index 9ae51af59b356c0e80995110ded8391899c546f0..66481f04f91942e116110ca566bd03142c84b80c 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL/a64fx.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL/a64fx.cpp @@ -139,11 +139,11 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "6:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 7f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 8f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -159,12 +159,12 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "9:" // Height 1: Multiply loop: Main loop "fmla z8.s, p4/M, z6.s, z0.s\n" "fmla z9.s, p4/M, z7.s, z0.s\n" - "ld1w { z6.s }, p4/Z, [x10, #2, MUL VL]\n" - "ld1w { z7.s }, p4/Z, [x10, #3, MUL VL]\n" + "ld1w { z17.s }, p4/Z, [x10, #2, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" "add x26, x26, #0x4\n" - "fmla z10.s, p4/M, z6.s, z0.s\n" - "fmla z11.s, p4/M, z7.s, z0.s\n" + "fmla z10.s, p4/M, z17.s, z0.s\n" + "fmla z11.s, p4/M, z16.s, z0.s\n" "subs x27, x27, #0x1\n" "ld1rw { z0.s }, p4/Z, [x26]\n" "ld1w { z6.s }, p4/Z, [x10]\n" @@ -174,27 +174,27 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "fmla z8.s, p4/M, z6.s, z0.s\n" "fmla z9.s, p4/M, z7.s, z0.s\n" - "ld1w { z6.s }, p4/Z, [x10, #2, MUL VL]\n" - "ld1w { z7.s }, p4/Z, [x10, #3, MUL VL]\n" + "ld1w { z17.s }, p4/Z, [x10, #2, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x10, #3, MUL VL]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" - "fmla z10.s, p4/M, z6.s, z0.s\n" - "fmla z11.s, p4/M, z7.s, z0.s\n" + "fmla z10.s, p4/M, z17.s, z0.s\n" + "fmla z11.s, p4/M, z16.s, z0.s\n" "addvl x10, x10, #4\n" "bne 6b\n" "tbz %x[flags], #1, 11f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p4/Z, [x20]\n" + "ld1rw { z17.s }, p4/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p4/Z, [x20]\n" - "fmin z8.s, p4/M, z8.s, z1.s\n" - "fmin z9.s, p4/M, z9.s, z1.s\n" - "fmin z10.s, p4/M, z10.s, z1.s\n" - "fmin z11.s, p4/M, z11.s, z1.s\n" - "fmax z8.s, p4/M, z8.s, z0.s\n" - "fmax z9.s, p4/M, z9.s, z0.s\n" - "fmax z10.s, p4/M, z10.s, z0.s\n" - "fmax z11.s, p4/M, z11.s, z0.s\n" + "ld1rw { z16.s }, p4/Z, [x20]\n" + "fmin z8.s, p4/M, z8.s, z17.s\n" + "fmin z9.s, p4/M, z9.s, z17.s\n" + "fmin z10.s, p4/M, z10.s, z17.s\n" + "fmin z11.s, p4/M, z11.s, z17.s\n" + "fmax z8.s, p4/M, z8.s, z16.s\n" + "fmax z9.s, p4/M, z9.s, z16.s\n" + "fmax z10.s, p4/M, z10.s, z16.s\n" + "fmax z11.s, p4/M, z11.s, z16.s\n" "11:" // Height 1: No activation "st1w { z8.s }, p3, [x9]\n" "st1w { z9.s }, p2, [x9, #1, MUL VL]\n" @@ -234,15 +234,15 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "15:" // Height 2: no bias "tbz %x[flags], #0, 16f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" + "add x20, x9, x20, LSL #2\n" "ld1w { z8.s }, p3/Z, [x9]\n" "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x25]\n" - "ld1w { z13.s }, p2/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p1/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p0/Z, [x25, #3, MUL VL]\n" + "ld1w { z12.s }, p3/Z, [x20]\n" + "ld1w { z13.s }, p2/Z, [x20, #1, MUL VL]\n" + "ld1w { z14.s }, p1/Z, [x20, #2, MUL VL]\n" + "ld1w { z15.s }, p0/Z, [x20, #3, MUL VL]\n" "b 17f\n" "16:" // Height 2: no accumulate "mov z8.b, #0x0\n" @@ -258,12 +258,12 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "18:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 19f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 20f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -271,7 +271,7 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "b 20f\n" "19:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" "20:" // Height 2: input setup done "subs x27, x27, #0x1\n" "ld1rw { z0.s }, p4/Z, [x26]\n" @@ -282,18 +282,18 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "21:" // Height 2: Multiply loop: Main loop "fmla z8.s, p4/M, z6.s, z0.s\n" "fmla z12.s, p4/M, z6.s, z1.s\n" - "ld1w { z6.s }, p4/Z, [x10, #2, MUL VL]\n" + "ld1w { z17.s }, p4/Z, [x10, #2, MUL VL]\n" "add x26, x26, #0x4\n" "fmla z9.s, p4/M, z7.s, z0.s\n" "fmla z13.s, p4/M, z7.s, z1.s\n" - "ld1w { z7.s }, p4/Z, [x10, #3, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" "subs x27, x27, #0x1\n" "add x25, x25, #0x4\n" - "fmla z10.s, p4/M, z6.s, z0.s\n" - "fmla z14.s, p4/M, z6.s, z1.s\n" - "fmla z11.s, p4/M, z7.s, z0.s\n" - "fmla z15.s, p4/M, z7.s, z1.s\n" + "fmla z10.s, p4/M, z17.s, z0.s\n" + "fmla z14.s, p4/M, z17.s, z1.s\n" + "fmla z11.s, p4/M, z16.s, z0.s\n" + "fmla z15.s, p4/M, z16.s, z1.s\n" "ld1rw { z0.s }, p4/Z, [x26]\n" "ld1rw { z1.s }, p4/Z, [x25]\n" "ld1w { z6.s }, p4/Z, [x10]\n" @@ -303,41 +303,41 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "fmla z8.s, p4/M, z6.s, z0.s\n" "fmla z12.s, p4/M, z6.s, z1.s\n" - "ld1w { z6.s }, p4/Z, [x10, #2, MUL VL]\n" + "ld1w { z17.s }, p4/Z, [x10, #2, MUL VL]\n" "fmla z9.s, p4/M, z7.s, z0.s\n" "fmla z13.s, p4/M, z7.s, z1.s\n" - "ld1w { z7.s }, p4/Z, [x10, #3, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x10, #3, MUL VL]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" - "fmla z10.s, p4/M, z6.s, z0.s\n" - "fmla z14.s, p4/M, z6.s, z1.s\n" + "fmla z10.s, p4/M, z17.s, z0.s\n" + "fmla z14.s, p4/M, z17.s, z1.s\n" "addvl x10, x10, #4\n" - "fmla z11.s, p4/M, z7.s, z0.s\n" - "fmla z15.s, p4/M, z7.s, z1.s\n" + "fmla z11.s, p4/M, z16.s, z0.s\n" + "fmla z15.s, p4/M, z16.s, z1.s\n" "bne 18b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "add x25, x9, x20, LSL #2\n" "tbz %x[flags], #1, 23f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p4/Z, [x20]\n" + "ld1rw { z17.s }, p4/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p4/Z, [x20]\n" - "fmin z8.s, p4/M, z8.s, z1.s\n" - "fmin z9.s, p4/M, z9.s, z1.s\n" - "fmin z10.s, p4/M, z10.s, z1.s\n" - "fmin z11.s, p4/M, z11.s, z1.s\n" - "fmin z12.s, p4/M, z12.s, z1.s\n" - "fmin z13.s, p4/M, z13.s, z1.s\n" - "fmin z14.s, p4/M, z14.s, z1.s\n" - "fmin z15.s, p4/M, z15.s, z1.s\n" - "fmax z8.s, p4/M, z8.s, z0.s\n" - "fmax z9.s, p4/M, z9.s, z0.s\n" - "fmax z10.s, p4/M, z10.s, z0.s\n" - "fmax z11.s, p4/M, z11.s, z0.s\n" - "fmax z12.s, p4/M, z12.s, z0.s\n" - "fmax z13.s, p4/M, z13.s, z0.s\n" - "fmax z14.s, p4/M, z14.s, z0.s\n" - "fmax z15.s, p4/M, z15.s, z0.s\n" + "ld1rw { z16.s }, p4/Z, [x20]\n" + "fmin z8.s, p4/M, z8.s, z17.s\n" + "fmin z9.s, p4/M, z9.s, z17.s\n" + "fmin z10.s, p4/M, z10.s, z17.s\n" + "fmin z11.s, p4/M, z11.s, z17.s\n" + "fmin z12.s, p4/M, z12.s, z17.s\n" + "fmin z13.s, p4/M, z13.s, z17.s\n" + "fmin z14.s, p4/M, z14.s, z17.s\n" + "fmin z15.s, p4/M, z15.s, z17.s\n" + "fmax z8.s, p4/M, z8.s, z16.s\n" + "fmax z9.s, p4/M, z9.s, z16.s\n" + "fmax z10.s, p4/M, z10.s, z16.s\n" + "fmax z11.s, p4/M, z11.s, z16.s\n" + "fmax z12.s, p4/M, z12.s, z16.s\n" + "fmax z13.s, p4/M, z13.s, z16.s\n" + "fmax z14.s, p4/M, z14.s, z16.s\n" + "fmax z15.s, p4/M, z15.s, z16.s\n" "23:" // Height 2: No activation "st1w { z8.s }, p3, [x9]\n" "st1w { z9.s }, p2, [x9, #1, MUL VL]\n" @@ -385,20 +385,20 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "27:" // Height 3: no bias "tbz %x[flags], #0, 28f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" + "add x21, x9, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z8.s }, p3/Z, [x9]\n" "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x25]\n" - "ld1w { z13.s }, p2/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p1/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p0/Z, [x25, #3, MUL VL]\n" - "ld1w { z16.s }, p3/Z, [x24]\n" - "ld1w { z17.s }, p2/Z, [x24, #1, MUL VL]\n" - "ld1w { z18.s }, p1/Z, [x24, #2, MUL VL]\n" - "ld1w { z19.s }, p0/Z, [x24, #3, MUL VL]\n" + "ld1w { z12.s }, p3/Z, [x21]\n" + "ld1w { z13.s }, p2/Z, [x21, #1, MUL VL]\n" + "ld1w { z14.s }, p1/Z, [x21, #2, MUL VL]\n" + "ld1w { z15.s }, p0/Z, [x21, #3, MUL VL]\n" + "ld1w { z16.s }, p3/Z, [x20]\n" + "ld1w { z17.s }, p2/Z, [x20, #1, MUL VL]\n" + "ld1w { z18.s }, p1/Z, [x20, #2, MUL VL]\n" + "ld1w { z19.s }, p0/Z, [x20, #3, MUL VL]\n" "b 29f\n" "28:" // Height 3: no accumulate "mov z8.b, #0x0\n" @@ -418,13 +418,13 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "30:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 31f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 32f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -433,8 +433,8 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "b 32f\n" "31:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" "32:" // Height 3: input setup done "subs x27, x27, #0x1\n" "ld1rw { z0.s }, p4/Z, [x26]\n" @@ -450,21 +450,21 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "subs x27, x27, #0x1\n" "fmla z16.s, p4/M, z6.s, z2.s\n" "fmla z9.s, p4/M, z7.s, z0.s\n" - "ld1w { z6.s }, p4/Z, [x10, #2, MUL VL]\n" + "ld1w { z21.s }, p4/Z, [x10, #2, MUL VL]\n" "add x25, x25, #0x4\n" "fmla z13.s, p4/M, z7.s, z1.s\n" "fmla z17.s, p4/M, z7.s, z2.s\n" - "ld1w { z7.s }, p4/Z, [x10, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" "add x24, x24, #0x4\n" - "fmla z10.s, p4/M, z6.s, z0.s\n" - "fmla z14.s, p4/M, z6.s, z1.s\n" - "fmla z18.s, p4/M, z6.s, z2.s\n" - "fmla z11.s, p4/M, z7.s, z0.s\n" + "fmla z10.s, p4/M, z21.s, z0.s\n" + "fmla z14.s, p4/M, z21.s, z1.s\n" + "fmla z18.s, p4/M, z21.s, z2.s\n" + "fmla z11.s, p4/M, z20.s, z0.s\n" "ld1rw { z0.s }, p4/Z, [x26]\n" "ld1w { z6.s }, p4/Z, [x10]\n" - "fmla z15.s, p4/M, z7.s, z1.s\n" - "fmla z19.s, p4/M, z7.s, z2.s\n" + "fmla z15.s, p4/M, z20.s, z1.s\n" + "fmla z19.s, p4/M, z20.s, z2.s\n" "ld1rw { z1.s }, p4/Z, [x25]\n" "ld1rw { z2.s }, p4/Z, [x24]\n" "ld1w { z7.s }, p4/Z, [x10, #1, MUL VL]\n" @@ -476,51 +476,51 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "add x28, x28, #0x1\n" "fmla z16.s, p4/M, z6.s, z2.s\n" "fmla z9.s, p4/M, z7.s, z0.s\n" - "ld1w { z6.s }, p4/Z, [x10, #2, MUL VL]\n" + "ld1w { z21.s }, p4/Z, [x10, #2, MUL VL]\n" "cmp x28, x20\n" "fmla z13.s, p4/M, z7.s, z1.s\n" "fmla z17.s, p4/M, z7.s, z2.s\n" - "ld1w { z7.s }, p4/Z, [x10, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.s, p4/M, z6.s, z0.s\n" - "fmla z14.s, p4/M, z6.s, z1.s\n" - "fmla z18.s, p4/M, z6.s, z2.s\n" - "fmla z11.s, p4/M, z7.s, z0.s\n" - "fmla z15.s, p4/M, z7.s, z1.s\n" - "fmla z19.s, p4/M, z7.s, z2.s\n" + "fmla z10.s, p4/M, z21.s, z0.s\n" + "fmla z14.s, p4/M, z21.s, z1.s\n" + "fmla z18.s, p4/M, z21.s, z2.s\n" + "fmla z11.s, p4/M, z20.s, z0.s\n" + "fmla z15.s, p4/M, z20.s, z1.s\n" + "fmla z19.s, p4/M, z20.s, z2.s\n" "bne 30b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "add x25, x9, x20, LSL #2\n" "add x24, x25, x20, LSL #2\n" "tbz %x[flags], #1, 35f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p4/Z, [x20]\n" + "ld1rw { z21.s }, p4/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p4/Z, [x20]\n" - "fmin z8.s, p4/M, z8.s, z1.s\n" - "fmin z9.s, p4/M, z9.s, z1.s\n" - "fmin z10.s, p4/M, z10.s, z1.s\n" - "fmin z11.s, p4/M, z11.s, z1.s\n" - "fmin z12.s, p4/M, z12.s, z1.s\n" - "fmin z13.s, p4/M, z13.s, z1.s\n" - "fmin z14.s, p4/M, z14.s, z1.s\n" - "fmin z15.s, p4/M, z15.s, z1.s\n" - "fmin z16.s, p4/M, z16.s, z1.s\n" - "fmin z17.s, p4/M, z17.s, z1.s\n" - "fmin z18.s, p4/M, z18.s, z1.s\n" - "fmin z19.s, p4/M, z19.s, z1.s\n" - "fmax z8.s, p4/M, z8.s, z0.s\n" - "fmax z9.s, p4/M, z9.s, z0.s\n" - "fmax z10.s, p4/M, z10.s, z0.s\n" - "fmax z11.s, p4/M, z11.s, z0.s\n" - "fmax z12.s, p4/M, z12.s, z0.s\n" - "fmax z13.s, p4/M, z13.s, z0.s\n" - "fmax z14.s, p4/M, z14.s, z0.s\n" - "fmax z15.s, p4/M, z15.s, z0.s\n" - "fmax z16.s, p4/M, z16.s, z0.s\n" - "fmax z17.s, p4/M, z17.s, z0.s\n" - "fmax z18.s, p4/M, z18.s, z0.s\n" - "fmax z19.s, p4/M, z19.s, z0.s\n" + "ld1rw { z20.s }, p4/Z, [x20]\n" + "fmin z8.s, p4/M, z8.s, z21.s\n" + "fmin z9.s, p4/M, z9.s, z21.s\n" + "fmin z10.s, p4/M, z10.s, z21.s\n" + "fmin z11.s, p4/M, z11.s, z21.s\n" + "fmin z12.s, p4/M, z12.s, z21.s\n" + "fmin z13.s, p4/M, z13.s, z21.s\n" + "fmin z14.s, p4/M, z14.s, z21.s\n" + "fmin z15.s, p4/M, z15.s, z21.s\n" + "fmin z16.s, p4/M, z16.s, z21.s\n" + "fmin z17.s, p4/M, z17.s, z21.s\n" + "fmin z18.s, p4/M, z18.s, z21.s\n" + "fmin z19.s, p4/M, z19.s, z21.s\n" + "fmax z8.s, p4/M, z8.s, z20.s\n" + "fmax z9.s, p4/M, z9.s, z20.s\n" + "fmax z10.s, p4/M, z10.s, z20.s\n" + "fmax z11.s, p4/M, z11.s, z20.s\n" + "fmax z12.s, p4/M, z12.s, z20.s\n" + "fmax z13.s, p4/M, z13.s, z20.s\n" + "fmax z14.s, p4/M, z14.s, z20.s\n" + "fmax z15.s, p4/M, z15.s, z20.s\n" + "fmax z16.s, p4/M, z16.s, z20.s\n" + "fmax z17.s, p4/M, z17.s, z20.s\n" + "fmax z18.s, p4/M, z18.s, z20.s\n" + "fmax z19.s, p4/M, z19.s, z20.s\n" "35:" // Height 3: No activation "st1w { z8.s }, p3, [x9]\n" "st1w { z9.s }, p2, [x9, #1, MUL VL]\n" @@ -576,25 +576,25 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "39:" // Height 4: no bias "tbz %x[flags], #0, 40f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" + "add x22, x9, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" "ld1w { z8.s }, p3/Z, [x9]\n" - "add x23, x24, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x25]\n" - "ld1w { z13.s }, p2/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p1/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p0/Z, [x25, #3, MUL VL]\n" - "ld1w { z16.s }, p3/Z, [x24]\n" - "ld1w { z17.s }, p2/Z, [x24, #1, MUL VL]\n" - "ld1w { z18.s }, p1/Z, [x24, #2, MUL VL]\n" - "ld1w { z19.s }, p0/Z, [x24, #3, MUL VL]\n" - "ld1w { z20.s }, p3/Z, [x23]\n" - "ld1w { z21.s }, p2/Z, [x23, #1, MUL VL]\n" - "ld1w { z22.s }, p1/Z, [x23, #2, MUL VL]\n" - "ld1w { z23.s }, p0/Z, [x23, #3, MUL VL]\n" + "ld1w { z12.s }, p3/Z, [x22]\n" + "ld1w { z13.s }, p2/Z, [x22, #1, MUL VL]\n" + "ld1w { z14.s }, p1/Z, [x22, #2, MUL VL]\n" + "ld1w { z15.s }, p0/Z, [x22, #3, MUL VL]\n" + "ld1w { z16.s }, p3/Z, [x21]\n" + "ld1w { z17.s }, p2/Z, [x21, #1, MUL VL]\n" + "ld1w { z18.s }, p1/Z, [x21, #2, MUL VL]\n" + "ld1w { z19.s }, p0/Z, [x21, #3, MUL VL]\n" + "ld1w { z20.s }, p3/Z, [x20]\n" + "ld1w { z21.s }, p2/Z, [x20, #1, MUL VL]\n" + "ld1w { z22.s }, p1/Z, [x20, #2, MUL VL]\n" + "ld1w { z23.s }, p0/Z, [x20, #3, MUL VL]\n" "b 41f\n" "40:" // Height 4: no accumulate "mov z8.b, #0x0\n" @@ -618,14 +618,14 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "42:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 43f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 44f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -635,9 +635,9 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "b 44f\n" "43:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" "44:" // Height 4: input setup done "subs x27, x27, #0x1\n" "ld1rw { z0.s }, p4/Z, [x26]\n" @@ -654,7 +654,7 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "subs x27, x27, #0x1\n" "fmla z16.s, p4/M, z6.s, z2.s\n" "fmla z20.s, p4/M, z6.s, z3.s\n" - "ld1w { z6.s }, p4/Z, [x10, #2, MUL VL]\n" + "ld1w { z25.s }, p4/Z, [x10, #2, MUL VL]\n" "add x25, x25, #0x4\n" "fmla z9.s, p4/M, z7.s, z0.s\n" "fmla z13.s, p4/M, z7.s, z1.s\n" @@ -662,19 +662,19 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "add x23, x23, #0x4\n" "fmla z17.s, p4/M, z7.s, z2.s\n" "fmla z21.s, p4/M, z7.s, z3.s\n" - "ld1w { z7.s }, p4/Z, [x10, #3, MUL VL]\n" + "ld1w { z24.s }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.s, p4/M, z6.s, z0.s\n" - "fmla z14.s, p4/M, z6.s, z1.s\n" - "fmla z18.s, p4/M, z6.s, z2.s\n" - "fmla z22.s, p4/M, z6.s, z3.s\n" + "fmla z10.s, p4/M, z25.s, z0.s\n" + "fmla z14.s, p4/M, z25.s, z1.s\n" + "fmla z18.s, p4/M, z25.s, z2.s\n" + "fmla z22.s, p4/M, z25.s, z3.s\n" "ld1w { z6.s }, p4/Z, [x10]\n" - "fmla z11.s, p4/M, z7.s, z0.s\n" - "fmla z15.s, p4/M, z7.s, z1.s\n" + "fmla z11.s, p4/M, z24.s, z0.s\n" + "fmla z15.s, p4/M, z24.s, z1.s\n" "ld1rw { z0.s }, p4/Z, [x26]\n" "ld1rw { z1.s }, p4/Z, [x25]\n" - "fmla z19.s, p4/M, z7.s, z2.s\n" - "fmla z23.s, p4/M, z7.s, z3.s\n" + "fmla z19.s, p4/M, z24.s, z2.s\n" + "fmla z23.s, p4/M, z24.s, z3.s\n" "ld1rw { z2.s }, p4/Z, [x24]\n" "ld1rw { z3.s }, p4/Z, [x23]\n" "ld1w { z7.s }, p4/Z, [x10, #1, MUL VL]\n" @@ -686,22 +686,22 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "add x28, x28, #0x1\n" "fmla z16.s, p4/M, z6.s, z2.s\n" "fmla z20.s, p4/M, z6.s, z3.s\n" - "ld1w { z6.s }, p4/Z, [x10, #2, MUL VL]\n" + "ld1w { z25.s }, p4/Z, [x10, #2, MUL VL]\n" "cmp x28, x20\n" "fmla z9.s, p4/M, z7.s, z0.s\n" "fmla z13.s, p4/M, z7.s, z1.s\n" "fmla z17.s, p4/M, z7.s, z2.s\n" "fmla z21.s, p4/M, z7.s, z3.s\n" - "ld1w { z7.s }, p4/Z, [x10, #3, MUL VL]\n" + "ld1w { z24.s }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.s, p4/M, z6.s, z0.s\n" - "fmla z14.s, p4/M, z6.s, z1.s\n" - "fmla z18.s, p4/M, z6.s, z2.s\n" - "fmla z22.s, p4/M, z6.s, z3.s\n" - "fmla z11.s, p4/M, z7.s, z0.s\n" - "fmla z15.s, p4/M, z7.s, z1.s\n" - "fmla z19.s, p4/M, z7.s, z2.s\n" - "fmla z23.s, p4/M, z7.s, z3.s\n" + "fmla z10.s, p4/M, z25.s, z0.s\n" + "fmla z14.s, p4/M, z25.s, z1.s\n" + "fmla z18.s, p4/M, z25.s, z2.s\n" + "fmla z22.s, p4/M, z25.s, z3.s\n" + "fmla z11.s, p4/M, z24.s, z0.s\n" + "fmla z15.s, p4/M, z24.s, z1.s\n" + "fmla z19.s, p4/M, z24.s, z2.s\n" + "fmla z23.s, p4/M, z24.s, z3.s\n" "bne 42b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "add x25, x9, x20, LSL #2\n" @@ -709,41 +709,41 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "add x23, x24, x20, LSL #2\n" "tbz %x[flags], #1, 47f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p4/Z, [x20]\n" + "ld1rw { z25.s }, p4/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p4/Z, [x20]\n" - "fmin z8.s, p4/M, z8.s, z1.s\n" - "fmin z9.s, p4/M, z9.s, z1.s\n" - "fmin z10.s, p4/M, z10.s, z1.s\n" - "fmin z11.s, p4/M, z11.s, z1.s\n" - "fmin z12.s, p4/M, z12.s, z1.s\n" - "fmin z13.s, p4/M, z13.s, z1.s\n" - "fmin z14.s, p4/M, z14.s, z1.s\n" - "fmin z15.s, p4/M, z15.s, z1.s\n" - "fmin z16.s, p4/M, z16.s, z1.s\n" - "fmin z17.s, p4/M, z17.s, z1.s\n" - "fmin z18.s, p4/M, z18.s, z1.s\n" - "fmin z19.s, p4/M, z19.s, z1.s\n" - "fmin z20.s, p4/M, z20.s, z1.s\n" - "fmin z21.s, p4/M, z21.s, z1.s\n" - "fmin z22.s, p4/M, z22.s, z1.s\n" - "fmin z23.s, p4/M, z23.s, z1.s\n" - "fmax z8.s, p4/M, z8.s, z0.s\n" - "fmax z9.s, p4/M, z9.s, z0.s\n" - "fmax z10.s, p4/M, z10.s, z0.s\n" - "fmax z11.s, p4/M, z11.s, z0.s\n" - "fmax z12.s, p4/M, z12.s, z0.s\n" - "fmax z13.s, p4/M, z13.s, z0.s\n" - "fmax z14.s, p4/M, z14.s, z0.s\n" - "fmax z15.s, p4/M, z15.s, z0.s\n" - "fmax z16.s, p4/M, z16.s, z0.s\n" - "fmax z17.s, p4/M, z17.s, z0.s\n" - "fmax z18.s, p4/M, z18.s, z0.s\n" - "fmax z19.s, p4/M, z19.s, z0.s\n" - "fmax z20.s, p4/M, z20.s, z0.s\n" - "fmax z21.s, p4/M, z21.s, z0.s\n" - "fmax z22.s, p4/M, z22.s, z0.s\n" - "fmax z23.s, p4/M, z23.s, z0.s\n" + "ld1rw { z24.s }, p4/Z, [x20]\n" + "fmin z8.s, p4/M, z8.s, z25.s\n" + "fmin z9.s, p4/M, z9.s, z25.s\n" + "fmin z10.s, p4/M, z10.s, z25.s\n" + "fmin z11.s, p4/M, z11.s, z25.s\n" + "fmin z12.s, p4/M, z12.s, z25.s\n" + "fmin z13.s, p4/M, z13.s, z25.s\n" + "fmin z14.s, p4/M, z14.s, z25.s\n" + "fmin z15.s, p4/M, z15.s, z25.s\n" + "fmin z16.s, p4/M, z16.s, z25.s\n" + "fmin z17.s, p4/M, z17.s, z25.s\n" + "fmin z18.s, p4/M, z18.s, z25.s\n" + "fmin z19.s, p4/M, z19.s, z25.s\n" + "fmin z20.s, p4/M, z20.s, z25.s\n" + "fmin z21.s, p4/M, z21.s, z25.s\n" + "fmin z22.s, p4/M, z22.s, z25.s\n" + "fmin z23.s, p4/M, z23.s, z25.s\n" + "fmax z8.s, p4/M, z8.s, z24.s\n" + "fmax z9.s, p4/M, z9.s, z24.s\n" + "fmax z10.s, p4/M, z10.s, z24.s\n" + "fmax z11.s, p4/M, z11.s, z24.s\n" + "fmax z12.s, p4/M, z12.s, z24.s\n" + "fmax z13.s, p4/M, z13.s, z24.s\n" + "fmax z14.s, p4/M, z14.s, z24.s\n" + "fmax z15.s, p4/M, z15.s, z24.s\n" + "fmax z16.s, p4/M, z16.s, z24.s\n" + "fmax z17.s, p4/M, z17.s, z24.s\n" + "fmax z18.s, p4/M, z18.s, z24.s\n" + "fmax z19.s, p4/M, z19.s, z24.s\n" + "fmax z20.s, p4/M, z20.s, z24.s\n" + "fmax z21.s, p4/M, z21.s, z24.s\n" + "fmax z22.s, p4/M, z22.s, z24.s\n" + "fmax z23.s, p4/M, z23.s, z24.s\n" "47:" // Height 4: No activation "st1w { z8.s }, p3, [x9]\n" "st1w { z9.s }, p2, [x9, #1, MUL VL]\n" @@ -807,30 +807,30 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "51:" // Height 5: no bias "tbz %x[flags], #0, 52f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z8.s }, p3/Z, [x9]\n" - "add x23, x24, x20, LSL #2\n" + "add x23, x9, x20, LSL #2\n" "add x22, x23, x20, LSL #2\n" + "ld1w { z8.s }, p3/Z, [x9]\n" + "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x25]\n" - "ld1w { z13.s }, p2/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p1/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p0/Z, [x25, #3, MUL VL]\n" - "ld1w { z16.s }, p3/Z, [x24]\n" - "ld1w { z17.s }, p2/Z, [x24, #1, MUL VL]\n" - "ld1w { z18.s }, p1/Z, [x24, #2, MUL VL]\n" - "ld1w { z19.s }, p0/Z, [x24, #3, MUL VL]\n" - "ld1w { z20.s }, p3/Z, [x23]\n" - "ld1w { z21.s }, p2/Z, [x23, #1, MUL VL]\n" - "ld1w { z22.s }, p1/Z, [x23, #2, MUL VL]\n" - "ld1w { z23.s }, p0/Z, [x23, #3, MUL VL]\n" - "ld1w { z24.s }, p3/Z, [x22]\n" - "ld1w { z25.s }, p2/Z, [x22, #1, MUL VL]\n" - "ld1w { z26.s }, p1/Z, [x22, #2, MUL VL]\n" - "ld1w { z27.s }, p0/Z, [x22, #3, MUL VL]\n" + "ld1w { z12.s }, p3/Z, [x23]\n" + "ld1w { z13.s }, p2/Z, [x23, #1, MUL VL]\n" + "ld1w { z14.s }, p1/Z, [x23, #2, MUL VL]\n" + "ld1w { z15.s }, p0/Z, [x23, #3, MUL VL]\n" + "ld1w { z16.s }, p3/Z, [x22]\n" + "ld1w { z17.s }, p2/Z, [x22, #1, MUL VL]\n" + "ld1w { z18.s }, p1/Z, [x22, #2, MUL VL]\n" + "ld1w { z19.s }, p0/Z, [x22, #3, MUL VL]\n" + "ld1w { z20.s }, p3/Z, [x21]\n" + "ld1w { z21.s }, p2/Z, [x21, #1, MUL VL]\n" + "ld1w { z22.s }, p1/Z, [x21, #2, MUL VL]\n" + "ld1w { z23.s }, p0/Z, [x21, #3, MUL VL]\n" + "ld1w { z24.s }, p3/Z, [x20]\n" + "ld1w { z25.s }, p2/Z, [x20, #1, MUL VL]\n" + "ld1w { z26.s }, p1/Z, [x20, #2, MUL VL]\n" + "ld1w { z27.s }, p0/Z, [x20, #3, MUL VL]\n" "b 53f\n" "52:" // Height 5: no accumulate "mov z8.b, #0x0\n" @@ -858,15 +858,15 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "54:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 55f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 56f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -877,10 +877,10 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "b 56f\n" "55:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" "56:" // Height 5: input setup done "subs x27, x27, #0x1\n" "ld1rw { z0.s }, p4/Z, [x26]\n" @@ -902,29 +902,29 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "add x24, x24, #0x4\n" "fmla z24.s, p4/M, z6.s, z4.s\n" "fmla z9.s, p4/M, z7.s, z0.s\n" - "ld1w { z6.s }, p4/Z, [x10, #2, MUL VL]\n" + "ld1w { z29.s }, p4/Z, [x10, #2, MUL VL]\n" "add x23, x23, #0x4\n" "fmla z13.s, p4/M, z7.s, z1.s\n" "fmla z17.s, p4/M, z7.s, z2.s\n" "add x22, x22, #0x4\n" "fmla z21.s, p4/M, z7.s, z3.s\n" "fmla z25.s, p4/M, z7.s, z4.s\n" - "ld1w { z7.s }, p4/Z, [x10, #3, MUL VL]\n" + "ld1w { z28.s }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.s, p4/M, z6.s, z0.s\n" - "fmla z14.s, p4/M, z6.s, z1.s\n" - "fmla z18.s, p4/M, z6.s, z2.s\n" - "fmla z22.s, p4/M, z6.s, z3.s\n" - "fmla z26.s, p4/M, z6.s, z4.s\n" - "fmla z11.s, p4/M, z7.s, z0.s\n" + "fmla z10.s, p4/M, z29.s, z0.s\n" + "fmla z14.s, p4/M, z29.s, z1.s\n" + "fmla z18.s, p4/M, z29.s, z2.s\n" + "fmla z22.s, p4/M, z29.s, z3.s\n" + "fmla z26.s, p4/M, z29.s, z4.s\n" + "fmla z11.s, p4/M, z28.s, z0.s\n" "ld1rw { z0.s }, p4/Z, [x26]\n" "ld1w { z6.s }, p4/Z, [x10]\n" - "fmla z15.s, p4/M, z7.s, z1.s\n" - "fmla z19.s, p4/M, z7.s, z2.s\n" + "fmla z15.s, p4/M, z28.s, z1.s\n" + "fmla z19.s, p4/M, z28.s, z2.s\n" "ld1rw { z1.s }, p4/Z, [x25]\n" "ld1rw { z2.s }, p4/Z, [x24]\n" - "fmla z23.s, p4/M, z7.s, z3.s\n" - "fmla z27.s, p4/M, z7.s, z4.s\n" + "fmla z23.s, p4/M, z28.s, z3.s\n" + "fmla z27.s, p4/M, z28.s, z4.s\n" "ld1rw { z3.s }, p4/Z, [x23]\n" "ld1rw { z4.s }, p4/Z, [x22]\n" "ld1w { z7.s }, p4/Z, [x10, #1, MUL VL]\n" @@ -939,23 +939,23 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "cmp x28, x20\n" "fmla z24.s, p4/M, z6.s, z4.s\n" "fmla z9.s, p4/M, z7.s, z0.s\n" - "ld1w { z6.s }, p4/Z, [x10, #2, MUL VL]\n" + "ld1w { z29.s }, p4/Z, [x10, #2, MUL VL]\n" "fmla z13.s, p4/M, z7.s, z1.s\n" "fmla z17.s, p4/M, z7.s, z2.s\n" "fmla z21.s, p4/M, z7.s, z3.s\n" "fmla z25.s, p4/M, z7.s, z4.s\n" - "ld1w { z7.s }, p4/Z, [x10, #3, MUL VL]\n" + "ld1w { z28.s }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.s, p4/M, z6.s, z0.s\n" - "fmla z14.s, p4/M, z6.s, z1.s\n" - "fmla z18.s, p4/M, z6.s, z2.s\n" - "fmla z22.s, p4/M, z6.s, z3.s\n" - "fmla z26.s, p4/M, z6.s, z4.s\n" - "fmla z11.s, p4/M, z7.s, z0.s\n" - "fmla z15.s, p4/M, z7.s, z1.s\n" - "fmla z19.s, p4/M, z7.s, z2.s\n" - "fmla z23.s, p4/M, z7.s, z3.s\n" - "fmla z27.s, p4/M, z7.s, z4.s\n" + "fmla z10.s, p4/M, z29.s, z0.s\n" + "fmla z14.s, p4/M, z29.s, z1.s\n" + "fmla z18.s, p4/M, z29.s, z2.s\n" + "fmla z22.s, p4/M, z29.s, z3.s\n" + "fmla z26.s, p4/M, z29.s, z4.s\n" + "fmla z11.s, p4/M, z28.s, z0.s\n" + "fmla z15.s, p4/M, z28.s, z1.s\n" + "fmla z19.s, p4/M, z28.s, z2.s\n" + "fmla z23.s, p4/M, z28.s, z3.s\n" + "fmla z27.s, p4/M, z28.s, z4.s\n" "bne 54b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "add x25, x9, x20, LSL #2\n" @@ -964,49 +964,49 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "add x22, x23, x20, LSL #2\n" "tbz %x[flags], #1, 59f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p4/Z, [x20]\n" + "ld1rw { z29.s }, p4/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p4/Z, [x20]\n" - "fmin z8.s, p4/M, z8.s, z1.s\n" - "fmin z9.s, p4/M, z9.s, z1.s\n" - "fmin z10.s, p4/M, z10.s, z1.s\n" - "fmin z11.s, p4/M, z11.s, z1.s\n" - "fmin z12.s, p4/M, z12.s, z1.s\n" - "fmin z13.s, p4/M, z13.s, z1.s\n" - "fmin z14.s, p4/M, z14.s, z1.s\n" - "fmin z15.s, p4/M, z15.s, z1.s\n" - "fmin z16.s, p4/M, z16.s, z1.s\n" - "fmin z17.s, p4/M, z17.s, z1.s\n" - "fmin z18.s, p4/M, z18.s, z1.s\n" - "fmin z19.s, p4/M, z19.s, z1.s\n" - "fmin z20.s, p4/M, z20.s, z1.s\n" - "fmin z21.s, p4/M, z21.s, z1.s\n" - "fmin z22.s, p4/M, z22.s, z1.s\n" - "fmin z23.s, p4/M, z23.s, z1.s\n" - "fmin z24.s, p4/M, z24.s, z1.s\n" - "fmin z25.s, p4/M, z25.s, z1.s\n" - "fmin z26.s, p4/M, z26.s, z1.s\n" - "fmin z27.s, p4/M, z27.s, z1.s\n" - "fmax z8.s, p4/M, z8.s, z0.s\n" - "fmax z9.s, p4/M, z9.s, z0.s\n" - "fmax z10.s, p4/M, z10.s, z0.s\n" - "fmax z11.s, p4/M, z11.s, z0.s\n" - "fmax z12.s, p4/M, z12.s, z0.s\n" - "fmax z13.s, p4/M, z13.s, z0.s\n" - "fmax z14.s, p4/M, z14.s, z0.s\n" - "fmax z15.s, p4/M, z15.s, z0.s\n" - "fmax z16.s, p4/M, z16.s, z0.s\n" - "fmax z17.s, p4/M, z17.s, z0.s\n" - "fmax z18.s, p4/M, z18.s, z0.s\n" - "fmax z19.s, p4/M, z19.s, z0.s\n" - "fmax z20.s, p4/M, z20.s, z0.s\n" - "fmax z21.s, p4/M, z21.s, z0.s\n" - "fmax z22.s, p4/M, z22.s, z0.s\n" - "fmax z23.s, p4/M, z23.s, z0.s\n" - "fmax z24.s, p4/M, z24.s, z0.s\n" - "fmax z25.s, p4/M, z25.s, z0.s\n" - "fmax z26.s, p4/M, z26.s, z0.s\n" - "fmax z27.s, p4/M, z27.s, z0.s\n" + "ld1rw { z28.s }, p4/Z, [x20]\n" + "fmin z8.s, p4/M, z8.s, z29.s\n" + "fmin z9.s, p4/M, z9.s, z29.s\n" + "fmin z10.s, p4/M, z10.s, z29.s\n" + "fmin z11.s, p4/M, z11.s, z29.s\n" + "fmin z12.s, p4/M, z12.s, z29.s\n" + "fmin z13.s, p4/M, z13.s, z29.s\n" + "fmin z14.s, p4/M, z14.s, z29.s\n" + "fmin z15.s, p4/M, z15.s, z29.s\n" + "fmin z16.s, p4/M, z16.s, z29.s\n" + "fmin z17.s, p4/M, z17.s, z29.s\n" + "fmin z18.s, p4/M, z18.s, z29.s\n" + "fmin z19.s, p4/M, z19.s, z29.s\n" + "fmin z20.s, p4/M, z20.s, z29.s\n" + "fmin z21.s, p4/M, z21.s, z29.s\n" + "fmin z22.s, p4/M, z22.s, z29.s\n" + "fmin z23.s, p4/M, z23.s, z29.s\n" + "fmin z24.s, p4/M, z24.s, z29.s\n" + "fmin z25.s, p4/M, z25.s, z29.s\n" + "fmin z26.s, p4/M, z26.s, z29.s\n" + "fmin z27.s, p4/M, z27.s, z29.s\n" + "fmax z8.s, p4/M, z8.s, z28.s\n" + "fmax z9.s, p4/M, z9.s, z28.s\n" + "fmax z10.s, p4/M, z10.s, z28.s\n" + "fmax z11.s, p4/M, z11.s, z28.s\n" + "fmax z12.s, p4/M, z12.s, z28.s\n" + "fmax z13.s, p4/M, z13.s, z28.s\n" + "fmax z14.s, p4/M, z14.s, z28.s\n" + "fmax z15.s, p4/M, z15.s, z28.s\n" + "fmax z16.s, p4/M, z16.s, z28.s\n" + "fmax z17.s, p4/M, z17.s, z28.s\n" + "fmax z18.s, p4/M, z18.s, z28.s\n" + "fmax z19.s, p4/M, z19.s, z28.s\n" + "fmax z20.s, p4/M, z20.s, z28.s\n" + "fmax z21.s, p4/M, z21.s, z28.s\n" + "fmax z22.s, p4/M, z22.s, z28.s\n" + "fmax z23.s, p4/M, z23.s, z28.s\n" + "fmax z24.s, p4/M, z24.s, z28.s\n" + "fmax z25.s, p4/M, z25.s, z28.s\n" + "fmax z26.s, p4/M, z26.s, z28.s\n" + "fmax z27.s, p4/M, z27.s, z28.s\n" "59:" // Height 5: No activation "st1w { z8.s }, p3, [x9]\n" "st1w { z9.s }, p2, [x9, #1, MUL VL]\n" @@ -1081,35 +1081,35 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "63:" // Height 6: no bias "tbz %x[flags], #0, 64f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z8.s }, p3/Z, [x9]\n" + "add x24, x9, x20, LSL #2\n" "add x23, x24, x20, LSL #2\n" + "ld1w { z8.s }, p3/Z, [x9]\n" "add x22, x23, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n" - "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x25]\n" - "ld1w { z13.s }, p2/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p1/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p0/Z, [x25, #3, MUL VL]\n" - "ld1w { z16.s }, p3/Z, [x24]\n" - "ld1w { z17.s }, p2/Z, [x24, #1, MUL VL]\n" - "ld1w { z18.s }, p1/Z, [x24, #2, MUL VL]\n" - "ld1w { z19.s }, p0/Z, [x24, #3, MUL VL]\n" - "ld1w { z20.s }, p3/Z, [x23]\n" - "ld1w { z21.s }, p2/Z, [x23, #1, MUL VL]\n" - "ld1w { z22.s }, p1/Z, [x23, #2, MUL VL]\n" - "ld1w { z23.s }, p0/Z, [x23, #3, MUL VL]\n" - "ld1w { z24.s }, p3/Z, [x22]\n" - "ld1w { z25.s }, p2/Z, [x22, #1, MUL VL]\n" - "ld1w { z26.s }, p1/Z, [x22, #2, MUL VL]\n" - "ld1w { z27.s }, p0/Z, [x22, #3, MUL VL]\n" - "ld1w { z28.s }, p3/Z, [x21]\n" - "ld1w { z29.s }, p2/Z, [x21, #1, MUL VL]\n" - "ld1w { z30.s }, p1/Z, [x21, #2, MUL VL]\n" - "ld1w { z31.s }, p0/Z, [x21, #3, MUL VL]\n" + "ld1w { z12.s }, p3/Z, [x24]\n" + "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n" + "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n" + "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n" + "ld1w { z16.s }, p3/Z, [x23]\n" + "ld1w { z17.s }, p2/Z, [x23, #1, MUL VL]\n" + "ld1w { z18.s }, p1/Z, [x23, #2, MUL VL]\n" + "ld1w { z19.s }, p0/Z, [x23, #3, MUL VL]\n" + "ld1w { z20.s }, p3/Z, [x22]\n" + "ld1w { z21.s }, p2/Z, [x22, #1, MUL VL]\n" + "ld1w { z22.s }, p1/Z, [x22, #2, MUL VL]\n" + "ld1w { z23.s }, p0/Z, [x22, #3, MUL VL]\n" + "ld1w { z24.s }, p3/Z, [x21]\n" + "ld1w { z25.s }, p2/Z, [x21, #1, MUL VL]\n" + "ld1w { z26.s }, p1/Z, [x21, #2, MUL VL]\n" + "ld1w { z27.s }, p0/Z, [x21, #3, MUL VL]\n" + "ld1w { z28.s }, p3/Z, [x20]\n" + "ld1w { z29.s }, p2/Z, [x20, #1, MUL VL]\n" + "ld1w { z30.s }, p1/Z, [x20, #2, MUL VL]\n" + "ld1w { z31.s }, p0/Z, [x20, #3, MUL VL]\n" "b 65f\n" "64:" // Height 6: no accumulate "mov z8.b, #0x0\n" @@ -1141,16 +1141,16 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "66:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 67f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 68f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -1162,11 +1162,11 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "b 68f\n" "67:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "add x21, x22, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" + "add x21, x22, x21, LSL #2\n" "68:" // Height 6: input setup done "subs x27, x27, #0x1\n" "ld1rw { z0.s }, p4/Z, [x26]\n" @@ -1355,7 +1355,6 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "74:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -1363,4 +1362,4 @@ void sve_hybrid_fp32_mla_6x4VL_a64fx ( } } // namespace arm_gemm -#endif // ARM_COMPUTE_ENABLE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL/generic.cpp index 71c6afba42e33845f865449fa2f87902cbd0ee0c..e1581f20262d1ccfde84efc7416f6efc442597a9 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_6x4VL/generic.cpp @@ -139,11 +139,11 @@ void sve_hybrid_fp32_mla_6x4VL ( "6:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 7f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 8f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -156,87 +156,87 @@ void sve_hybrid_fp32_mla_6x4VL ( "9:" // Height 1: Multiply loop: Main loop head "whilelt p0.s, XZR, x27\n" "ld1rqw { z0.s }, p0/Z, [x26]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "fmla z8.s, z6.s, z0.s[0]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[0]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z11.s, z7.s, z0.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10, #4, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[1]\n" - "ld1w { z7.s }, p5/Z, [x10, #5, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10, #6, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[1]\n" - "ld1w { z7.s }, p5/Z, [x10, #7, MUL VL]\n" + "ld1w { z16.s }, p5/Z, [x10]\n" + "fmla z8.s, z16.s, z0.s[0]\n" + "ld1w { z16.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z9.s, z16.s, z0.s[0]\n" + "ld1w { z16.s }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z10.s, z16.s, z0.s[0]\n" + "ld1w { z16.s }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z11.s, z16.s, z0.s[0]\n" + "ld1w { z16.s }, p5/Z, [x10, #4, MUL VL]\n" + "fmla z8.s, z16.s, z0.s[1]\n" + "ld1w { z16.s }, p5/Z, [x10, #5, MUL VL]\n" + "fmla z9.s, z16.s, z0.s[1]\n" + "ld1w { z16.s }, p5/Z, [x10, #6, MUL VL]\n" + "fmla z10.s, z16.s, z0.s[1]\n" + "ld1w { z16.s }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "fmla z11.s, z7.s, z0.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10, #-8, MUL VL]\n" - "ld1w { z7.s }, p5/Z, [x10, #-7, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[2]\n" - "fmla z9.s, z7.s, z0.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10, #-6, MUL VL]\n" - "ld1w { z7.s }, p5/Z, [x10, #-5, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[2]\n" - "fmla z11.s, z7.s, z0.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10, #-4, MUL VL]\n" - "ld1w { z7.s }, p5/Z, [x10, #-3, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[3]\n" - "fmla z9.s, z7.s, z0.s[3]\n" - "ld1w { z6.s }, p5/Z, [x10, #-2, MUL VL]\n" - "ld1w { z7.s }, p5/Z, [x10, #-1, MUL VL]\n" + "fmla z11.s, z16.s, z0.s[1]\n" + "ld1w { z17.s }, p5/Z, [x10, #-8, MUL VL]\n" + "ld1w { z16.s }, p5/Z, [x10, #-7, MUL VL]\n" + "fmla z8.s, z17.s, z0.s[2]\n" + "fmla z9.s, z16.s, z0.s[2]\n" + "ld1w { z17.s }, p5/Z, [x10, #-6, MUL VL]\n" + "ld1w { z16.s }, p5/Z, [x10, #-5, MUL VL]\n" + "fmla z10.s, z17.s, z0.s[2]\n" + "fmla z11.s, z16.s, z0.s[2]\n" + "ld1w { z17.s }, p5/Z, [x10, #-4, MUL VL]\n" + "ld1w { z16.s }, p5/Z, [x10, #-3, MUL VL]\n" + "fmla z8.s, z17.s, z0.s[3]\n" + "fmla z9.s, z16.s, z0.s[3]\n" + "ld1w { z17.s }, p5/Z, [x10, #-2, MUL VL]\n" + "ld1w { z16.s }, p5/Z, [x10, #-1, MUL VL]\n" "sub x27, x27, #0x4\n" "cmp x27, #0x4\n" - "fmla z10.s, z6.s, z0.s[3]\n" - "fmla z11.s, z7.s, z0.s[3]\n" + "fmla z10.s, z17.s, z0.s[3]\n" + "fmla z11.s, z16.s, z0.s[3]\n" "add x26, x26, #0x10\n" "bgt 9b\n" "10:" // Height 1: Multiply loop: Single iteration only "whilelt p0.s, XZR, x27\n" "ld1rqw { z0.s }, p0/Z, [x26]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "fmla z8.s, z6.s, z0.s[0]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" + "ld1w { z16.s }, p5/Z, [x10]\n" + "fmla z8.s, z16.s, z0.s[0]\n" + "ld1w { z16.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z9.s, z16.s, z0.s[0]\n" + "ld1w { z17.s }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x1\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[0]\n" - "fmla z11.s, z7.s, z0.s[0]\n" + "ld1w { z16.s }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z10.s, z17.s, z0.s[0]\n" + "fmla z11.s, z16.s, z0.s[0]\n" "addvl x10, x10, #4\n" "ble 11f\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[1]\n" - "fmla z9.s, z7.s, z0.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" + "ld1w { z17.s }, p5/Z, [x10]\n" + "ld1w { z16.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z17.s, z0.s[1]\n" + "fmla z9.s, z16.s, z0.s[1]\n" + "ld1w { z17.s }, p5/Z, [x10, #2, MUL VL]\n" + "ld1w { z16.s }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z10.s, z6.s, z0.s[1]\n" - "fmla z11.s, z7.s, z0.s[1]\n" + "fmla z10.s, z17.s, z0.s[1]\n" + "fmla z11.s, z16.s, z0.s[1]\n" "addvl x10, x10, #4\n" "ble 11f\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[2]\n" - "fmla z9.s, z7.s, z0.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" + "ld1w { z17.s }, p5/Z, [x10]\n" + "ld1w { z16.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z17.s, z0.s[2]\n" + "fmla z9.s, z16.s, z0.s[2]\n" + "ld1w { z17.s }, p5/Z, [x10, #2, MUL VL]\n" + "ld1w { z16.s }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z10.s, z6.s, z0.s[2]\n" - "fmla z11.s, z7.s, z0.s[2]\n" + "fmla z10.s, z17.s, z0.s[2]\n" + "fmla z11.s, z16.s, z0.s[2]\n" "addvl x10, x10, #4\n" "ble 11f\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[3]\n" - "fmla z9.s, z7.s, z0.s[3]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[3]\n" - "fmla z11.s, z7.s, z0.s[3]\n" + "ld1w { z17.s }, p5/Z, [x10]\n" + "ld1w { z16.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z17.s, z0.s[3]\n" + "fmla z9.s, z16.s, z0.s[3]\n" + "ld1w { z17.s }, p5/Z, [x10, #2, MUL VL]\n" + "ld1w { z16.s }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z10.s, z17.s, z0.s[3]\n" + "fmla z11.s, z16.s, z0.s[3]\n" "addvl x10, x10, #4\n" "11:" // Height 1: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -245,17 +245,17 @@ void sve_hybrid_fp32_mla_6x4VL ( "bne 6b\n" "tbz %x[flags], #1, 12f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z17.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" + "ld1rw { z16.s }, p5/Z, [x20]\n" + "fmin z8.s, p5/M, z8.s, z17.s\n" + "fmin z9.s, p5/M, z9.s, z17.s\n" + "fmin z10.s, p5/M, z10.s, z17.s\n" + "fmin z11.s, p5/M, z11.s, z17.s\n" + "fmax z8.s, p5/M, z8.s, z16.s\n" + "fmax z9.s, p5/M, z9.s, z16.s\n" + "fmax z10.s, p5/M, z10.s, z16.s\n" + "fmax z11.s, p5/M, z11.s, z16.s\n" "12:" // Height 1: No activation "st1w { z8.s }, p4, [x9]\n" "st1w { z9.s }, p3, [x9, #1, MUL VL]\n" @@ -295,15 +295,15 @@ void sve_hybrid_fp32_mla_6x4VL ( "16:" // Height 2: no bias "tbz %x[flags], #0, 17f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" + "add x20, x9, x20, LSL #2\n" "ld1w { z8.s }, p4/Z, [x9]\n" "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x20]\n" + "ld1w { z13.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z15.s }, p1/Z, [x20, #3, MUL VL]\n" "b 18f\n" "17:" // Height 2: no accumulate "mov z8.b, #0x0\n" @@ -319,12 +319,12 @@ void sve_hybrid_fp32_mla_6x4VL ( "19:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 20f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 21f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -332,130 +332,130 @@ void sve_hybrid_fp32_mla_6x4VL ( "b 21f\n" "20:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" "21:" // Height 2: input setup done "cmp x27, #0x4\n" "ble 23f\n" "22:" // Height 2: Multiply loop: Main loop head "whilelt p0.s, XZR, x27\n" - "ld1rqw { z0.s }, p0/Z, [x26]\n" - "ld1rqw { z1.s }, p0/Z, [x25]\n" + "ld1rqw { z1.s }, p0/Z, [x26]\n" + "ld1rqw { z0.s }, p0/Z, [x25]\n" "sub x27, x27, #0x4\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[0]\n" - "fmla z12.s, z6.s, z1.s[0]\n" - "fmla z9.s, z7.s, z0.s[0]\n" - "fmla z13.s, z7.s, z1.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[0]\n" - "fmla z14.s, z6.s, z1.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10, #4, MUL VL]\n" + "ld1w { z17.s }, p5/Z, [x10]\n" + "ld1w { z16.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z17.s, z1.s[0]\n" + "fmla z12.s, z17.s, z0.s[0]\n" + "fmla z9.s, z16.s, z1.s[0]\n" + "fmla z13.s, z16.s, z0.s[0]\n" + "ld1w { z17.s }, p5/Z, [x10, #2, MUL VL]\n" + "ld1w { z16.s }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z10.s, z17.s, z1.s[0]\n" + "fmla z14.s, z17.s, z0.s[0]\n" + "ld1w { z17.s }, p5/Z, [x10, #4, MUL VL]\n" "cmp x27, #0x4\n" - "fmla z11.s, z7.s, z0.s[0]\n" - "fmla z15.s, z7.s, z1.s[0]\n" - "ld1w { z7.s }, p5/Z, [x10, #5, MUL VL]\n" + "fmla z11.s, z16.s, z1.s[0]\n" + "fmla z15.s, z16.s, z0.s[0]\n" + "ld1w { z16.s }, p5/Z, [x10, #5, MUL VL]\n" "add x26, x26, #0x10\n" - "fmla z8.s, z6.s, z0.s[1]\n" - "fmla z12.s, z6.s, z1.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10, #6, MUL VL]\n" + "fmla z8.s, z17.s, z1.s[1]\n" + "fmla z12.s, z17.s, z0.s[1]\n" + "ld1w { z17.s }, p5/Z, [x10, #6, MUL VL]\n" "add x25, x25, #0x10\n" - "fmla z9.s, z7.s, z0.s[1]\n" - "fmla z13.s, z7.s, z1.s[1]\n" - "ld1w { z7.s }, p5/Z, [x10, #7, MUL VL]\n" + "fmla z9.s, z16.s, z1.s[1]\n" + "fmla z13.s, z16.s, z0.s[1]\n" + "ld1w { z16.s }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "fmla z10.s, z6.s, z0.s[1]\n" - "fmla z14.s, z6.s, z1.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10, #-8, MUL VL]\n" - "fmla z11.s, z7.s, z0.s[1]\n" - "fmla z15.s, z7.s, z1.s[1]\n" - "ld1w { z7.s }, p5/Z, [x10, #-7, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[2]\n" - "fmla z12.s, z6.s, z1.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10, #-6, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[2]\n" - "fmla z13.s, z7.s, z1.s[2]\n" - "ld1w { z7.s }, p5/Z, [x10, #-5, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[2]\n" - "fmla z14.s, z6.s, z1.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10, #-4, MUL VL]\n" - "fmla z11.s, z7.s, z0.s[2]\n" - "fmla z15.s, z7.s, z1.s[2]\n" - "ld1w { z7.s }, p5/Z, [x10, #-3, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[3]\n" - "fmla z12.s, z6.s, z1.s[3]\n" - "ld1w { z6.s }, p5/Z, [x10, #-2, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[3]\n" - "fmla z13.s, z7.s, z1.s[3]\n" - "ld1w { z7.s }, p5/Z, [x10, #-1, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[3]\n" - "fmla z14.s, z6.s, z1.s[3]\n" - "fmla z11.s, z7.s, z0.s[3]\n" - "fmla z15.s, z7.s, z1.s[3]\n" + "fmla z10.s, z17.s, z1.s[1]\n" + "fmla z14.s, z17.s, z0.s[1]\n" + "ld1w { z17.s }, p5/Z, [x10, #-8, MUL VL]\n" + "fmla z11.s, z16.s, z1.s[1]\n" + "fmla z15.s, z16.s, z0.s[1]\n" + "ld1w { z16.s }, p5/Z, [x10, #-7, MUL VL]\n" + "fmla z8.s, z17.s, z1.s[2]\n" + "fmla z12.s, z17.s, z0.s[2]\n" + "ld1w { z17.s }, p5/Z, [x10, #-6, MUL VL]\n" + "fmla z9.s, z16.s, z1.s[2]\n" + "fmla z13.s, z16.s, z0.s[2]\n" + "ld1w { z16.s }, p5/Z, [x10, #-5, MUL VL]\n" + "fmla z10.s, z17.s, z1.s[2]\n" + "fmla z14.s, z17.s, z0.s[2]\n" + "ld1w { z17.s }, p5/Z, [x10, #-4, MUL VL]\n" + "fmla z11.s, z16.s, z1.s[2]\n" + "fmla z15.s, z16.s, z0.s[2]\n" + "ld1w { z16.s }, p5/Z, [x10, #-3, MUL VL]\n" + "fmla z8.s, z17.s, z1.s[3]\n" + "fmla z12.s, z17.s, z0.s[3]\n" + "ld1w { z17.s }, p5/Z, [x10, #-2, MUL VL]\n" + "fmla z9.s, z16.s, z1.s[3]\n" + "fmla z13.s, z16.s, z0.s[3]\n" + "ld1w { z16.s }, p5/Z, [x10, #-1, MUL VL]\n" + "fmla z10.s, z17.s, z1.s[3]\n" + "fmla z14.s, z17.s, z0.s[3]\n" + "fmla z11.s, z16.s, z1.s[3]\n" + "fmla z15.s, z16.s, z0.s[3]\n" "bgt 22b\n" "23:" // Height 2: Multiply loop: Single iteration only "whilelt p0.s, XZR, x27\n" "ld1rqw { z0.s }, p0/Z, [x26]\n" "ld1rqw { z1.s }, p0/Z, [x25]\n" "subs x27, x27, #0x1\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[0]\n" - "fmla z12.s, z6.s, z1.s[0]\n" - "fmla z9.s, z7.s, z0.s[0]\n" - "fmla z13.s, z7.s, z1.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[0]\n" - "fmla z14.s, z6.s, z1.s[0]\n" + "ld1w { z17.s }, p5/Z, [x10]\n" + "ld1w { z16.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z17.s, z0.s[0]\n" + "fmla z12.s, z17.s, z1.s[0]\n" + "fmla z9.s, z16.s, z0.s[0]\n" + "fmla z13.s, z16.s, z1.s[0]\n" + "ld1w { z17.s }, p5/Z, [x10, #2, MUL VL]\n" + "ld1w { z16.s }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z10.s, z17.s, z0.s[0]\n" + "fmla z14.s, z17.s, z1.s[0]\n" "addvl x10, x10, #4\n" - "fmla z11.s, z7.s, z0.s[0]\n" - "fmla z15.s, z7.s, z1.s[0]\n" + "fmla z11.s, z16.s, z0.s[0]\n" + "fmla z15.s, z16.s, z1.s[0]\n" "ble 24f\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[1]\n" - "fmla z12.s, z6.s, z1.s[1]\n" - "fmla z9.s, z7.s, z0.s[1]\n" - "fmla z13.s, z7.s, z1.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" + "ld1w { z17.s }, p5/Z, [x10]\n" + "ld1w { z16.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z17.s, z0.s[1]\n" + "fmla z12.s, z17.s, z1.s[1]\n" + "fmla z9.s, z16.s, z0.s[1]\n" + "fmla z13.s, z16.s, z1.s[1]\n" + "ld1w { z17.s }, p5/Z, [x10, #2, MUL VL]\n" + "ld1w { z16.s }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z10.s, z6.s, z0.s[1]\n" - "fmla z14.s, z6.s, z1.s[1]\n" + "fmla z10.s, z17.s, z0.s[1]\n" + "fmla z14.s, z17.s, z1.s[1]\n" "addvl x10, x10, #4\n" - "fmla z11.s, z7.s, z0.s[1]\n" - "fmla z15.s, z7.s, z1.s[1]\n" + "fmla z11.s, z16.s, z0.s[1]\n" + "fmla z15.s, z16.s, z1.s[1]\n" "ble 24f\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[2]\n" - "fmla z12.s, z6.s, z1.s[2]\n" - "fmla z9.s, z7.s, z0.s[2]\n" - "fmla z13.s, z7.s, z1.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" + "ld1w { z17.s }, p5/Z, [x10]\n" + "ld1w { z16.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z17.s, z0.s[2]\n" + "fmla z12.s, z17.s, z1.s[2]\n" + "fmla z9.s, z16.s, z0.s[2]\n" + "fmla z13.s, z16.s, z1.s[2]\n" + "ld1w { z17.s }, p5/Z, [x10, #2, MUL VL]\n" + "ld1w { z16.s }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z10.s, z6.s, z0.s[2]\n" - "fmla z14.s, z6.s, z1.s[2]\n" + "fmla z10.s, z17.s, z0.s[2]\n" + "fmla z14.s, z17.s, z1.s[2]\n" "addvl x10, x10, #4\n" - "fmla z11.s, z7.s, z0.s[2]\n" - "fmla z15.s, z7.s, z1.s[2]\n" + "fmla z11.s, z16.s, z0.s[2]\n" + "fmla z15.s, z16.s, z1.s[2]\n" "ble 24f\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[3]\n" - "fmla z12.s, z6.s, z1.s[3]\n" - "fmla z9.s, z7.s, z0.s[3]\n" - "fmla z13.s, z7.s, z1.s[3]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[3]\n" - "fmla z14.s, z6.s, z1.s[3]\n" + "ld1w { z17.s }, p5/Z, [x10]\n" + "ld1w { z16.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z17.s, z0.s[3]\n" + "fmla z12.s, z17.s, z1.s[3]\n" + "fmla z9.s, z16.s, z0.s[3]\n" + "fmla z13.s, z16.s, z1.s[3]\n" + "ld1w { z17.s }, p5/Z, [x10, #2, MUL VL]\n" + "ld1w { z16.s }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z10.s, z17.s, z0.s[3]\n" + "fmla z14.s, z17.s, z1.s[3]\n" "addvl x10, x10, #4\n" - "fmla z11.s, z7.s, z0.s[3]\n" - "fmla z15.s, z7.s, z1.s[3]\n" + "fmla z11.s, z16.s, z0.s[3]\n" + "fmla z15.s, z16.s, z1.s[3]\n" "24:" // Height 2: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -465,25 +465,25 @@ void sve_hybrid_fp32_mla_6x4VL ( "add x25, x9, x20, LSL #2\n" "tbz %x[flags], #1, 25f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z17.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmin z12.s, p5/M, z12.s, z1.s\n" - "fmin z13.s, p5/M, z13.s, z1.s\n" - "fmin z14.s, p5/M, z14.s, z1.s\n" - "fmin z15.s, p5/M, z15.s, z1.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" - "fmax z12.s, p5/M, z12.s, z0.s\n" - "fmax z13.s, p5/M, z13.s, z0.s\n" - "fmax z14.s, p5/M, z14.s, z0.s\n" - "fmax z15.s, p5/M, z15.s, z0.s\n" + "ld1rw { z16.s }, p5/Z, [x20]\n" + "fmin z8.s, p5/M, z8.s, z17.s\n" + "fmin z9.s, p5/M, z9.s, z17.s\n" + "fmin z10.s, p5/M, z10.s, z17.s\n" + "fmin z11.s, p5/M, z11.s, z17.s\n" + "fmin z12.s, p5/M, z12.s, z17.s\n" + "fmin z13.s, p5/M, z13.s, z17.s\n" + "fmin z14.s, p5/M, z14.s, z17.s\n" + "fmin z15.s, p5/M, z15.s, z17.s\n" + "fmax z8.s, p5/M, z8.s, z16.s\n" + "fmax z9.s, p5/M, z9.s, z16.s\n" + "fmax z10.s, p5/M, z10.s, z16.s\n" + "fmax z11.s, p5/M, z11.s, z16.s\n" + "fmax z12.s, p5/M, z12.s, z16.s\n" + "fmax z13.s, p5/M, z13.s, z16.s\n" + "fmax z14.s, p5/M, z14.s, z16.s\n" + "fmax z15.s, p5/M, z15.s, z16.s\n" "25:" // Height 2: No activation "st1w { z8.s }, p4, [x9]\n" "st1w { z9.s }, p3, [x9, #1, MUL VL]\n" @@ -531,20 +531,20 @@ void sve_hybrid_fp32_mla_6x4VL ( "29:" // Height 3: no bias "tbz %x[flags], #0, 30f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" + "add x21, x9, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z8.s }, p4/Z, [x9]\n" "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "ld1w { z16.s }, p4/Z, [x24]\n" - "ld1w { z17.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x24, #2, MUL VL]\n" - "ld1w { z19.s }, p1/Z, [x24, #3, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x21]\n" + "ld1w { z13.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x21, #2, MUL VL]\n" + "ld1w { z15.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x20]\n" + "ld1w { z17.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z19.s }, p1/Z, [x20, #3, MUL VL]\n" "b 31f\n" "30:" // Height 3: no accumulate "mov z8.b, #0x0\n" @@ -564,13 +564,13 @@ void sve_hybrid_fp32_mla_6x4VL ( "32:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 33f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 34f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -579,86 +579,86 @@ void sve_hybrid_fp32_mla_6x4VL ( "b 34f\n" "33:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" "34:" // Height 3: input setup done "cmp x27, #0x4\n" "ble 36f\n" "35:" // Height 3: Multiply loop: Main loop head "whilelt p0.s, XZR, x27\n" - "ld1rqw { z0.s }, p0/Z, [x26]\n" + "ld1rqw { z2.s }, p0/Z, [x26]\n" "ld1rqw { z1.s }, p0/Z, [x25]\n" "sub x27, x27, #0x4\n" - "ld1rqw { z2.s }, p0/Z, [x24]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "fmla z8.s, z6.s, z0.s[0]\n" - "fmla z12.s, z6.s, z1.s[0]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z16.s, z6.s, z2.s[0]\n" - "fmla z9.s, z7.s, z0.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z13.s, z7.s, z1.s[0]\n" - "fmla z17.s, z7.s, z2.s[0]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" + "ld1rqw { z0.s }, p0/Z, [x24]\n" + "ld1w { z21.s }, p5/Z, [x10]\n" + "fmla z8.s, z21.s, z2.s[0]\n" + "fmla z12.s, z21.s, z1.s[0]\n" + "ld1w { z20.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z16.s, z21.s, z0.s[0]\n" + "fmla z9.s, z20.s, z2.s[0]\n" + "ld1w { z21.s }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z13.s, z20.s, z1.s[0]\n" + "fmla z17.s, z20.s, z0.s[0]\n" + "ld1w { z20.s }, p5/Z, [x10, #3, MUL VL]\n" "cmp x27, #0x4\n" - "fmla z10.s, z6.s, z0.s[0]\n" - "fmla z14.s, z6.s, z1.s[0]\n" + "fmla z10.s, z21.s, z2.s[0]\n" + "fmla z14.s, z21.s, z1.s[0]\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" - "fmla z18.s, z6.s, z2.s[0]\n" - "fmla z11.s, z7.s, z0.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10, #4, MUL VL]\n" + "fmla z18.s, z21.s, z0.s[0]\n" + "fmla z11.s, z20.s, z2.s[0]\n" + "ld1w { z21.s }, p5/Z, [x10, #4, MUL VL]\n" "add x24, x24, #0x10\n" - "fmla z15.s, z7.s, z1.s[0]\n" - "fmla z19.s, z7.s, z2.s[0]\n" - "ld1w { z7.s }, p5/Z, [x10, #5, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[1]\n" - "fmla z12.s, z6.s, z1.s[1]\n" - "fmla z16.s, z6.s, z2.s[1]\n" - "fmla z9.s, z7.s, z0.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10, #6, MUL VL]\n" - "fmla z13.s, z7.s, z1.s[1]\n" - "fmla z17.s, z7.s, z2.s[1]\n" - "ld1w { z7.s }, p5/Z, [x10, #7, MUL VL]\n" + "fmla z15.s, z20.s, z1.s[0]\n" + "fmla z19.s, z20.s, z0.s[0]\n" + "ld1w { z20.s }, p5/Z, [x10, #5, MUL VL]\n" + "fmla z8.s, z21.s, z2.s[1]\n" + "fmla z12.s, z21.s, z1.s[1]\n" + "fmla z16.s, z21.s, z0.s[1]\n" + "fmla z9.s, z20.s, z2.s[1]\n" + "ld1w { z21.s }, p5/Z, [x10, #6, MUL VL]\n" + "fmla z13.s, z20.s, z1.s[1]\n" + "fmla z17.s, z20.s, z0.s[1]\n" + "ld1w { z20.s }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "fmla z10.s, z6.s, z0.s[1]\n" - "fmla z14.s, z6.s, z1.s[1]\n" - "fmla z18.s, z6.s, z2.s[1]\n" - "fmla z11.s, z7.s, z0.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10, #-8, MUL VL]\n" - "fmla z15.s, z7.s, z1.s[1]\n" - "fmla z19.s, z7.s, z2.s[1]\n" - "ld1w { z7.s }, p5/Z, [x10, #-7, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[2]\n" - "fmla z12.s, z6.s, z1.s[2]\n" - "fmla z16.s, z6.s, z2.s[2]\n" - "fmla z9.s, z7.s, z0.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10, #-6, MUL VL]\n" - "fmla z13.s, z7.s, z1.s[2]\n" - "fmla z17.s, z7.s, z2.s[2]\n" - "ld1w { z7.s }, p5/Z, [x10, #-5, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[2]\n" - "fmla z14.s, z6.s, z1.s[2]\n" - "fmla z18.s, z6.s, z2.s[2]\n" - "fmla z11.s, z7.s, z0.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10, #-4, MUL VL]\n" - "fmla z15.s, z7.s, z1.s[2]\n" - "fmla z19.s, z7.s, z2.s[2]\n" - "ld1w { z7.s }, p5/Z, [x10, #-3, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[3]\n" - "fmla z12.s, z6.s, z1.s[3]\n" - "fmla z16.s, z6.s, z2.s[3]\n" - "fmla z9.s, z7.s, z0.s[3]\n" - "ld1w { z6.s }, p5/Z, [x10, #-2, MUL VL]\n" - "fmla z13.s, z7.s, z1.s[3]\n" - "fmla z17.s, z7.s, z2.s[3]\n" - "ld1w { z7.s }, p5/Z, [x10, #-1, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[3]\n" - "fmla z14.s, z6.s, z1.s[3]\n" - "fmla z18.s, z6.s, z2.s[3]\n" - "fmla z11.s, z7.s, z0.s[3]\n" - "fmla z15.s, z7.s, z1.s[3]\n" - "fmla z19.s, z7.s, z2.s[3]\n" + "fmla z10.s, z21.s, z2.s[1]\n" + "fmla z14.s, z21.s, z1.s[1]\n" + "fmla z18.s, z21.s, z0.s[1]\n" + "fmla z11.s, z20.s, z2.s[1]\n" + "ld1w { z21.s }, p5/Z, [x10, #-8, MUL VL]\n" + "fmla z15.s, z20.s, z1.s[1]\n" + "fmla z19.s, z20.s, z0.s[1]\n" + "ld1w { z20.s }, p5/Z, [x10, #-7, MUL VL]\n" + "fmla z8.s, z21.s, z2.s[2]\n" + "fmla z12.s, z21.s, z1.s[2]\n" + "fmla z16.s, z21.s, z0.s[2]\n" + "fmla z9.s, z20.s, z2.s[2]\n" + "ld1w { z21.s }, p5/Z, [x10, #-6, MUL VL]\n" + "fmla z13.s, z20.s, z1.s[2]\n" + "fmla z17.s, z20.s, z0.s[2]\n" + "ld1w { z20.s }, p5/Z, [x10, #-5, MUL VL]\n" + "fmla z10.s, z21.s, z2.s[2]\n" + "fmla z14.s, z21.s, z1.s[2]\n" + "fmla z18.s, z21.s, z0.s[2]\n" + "fmla z11.s, z20.s, z2.s[2]\n" + "ld1w { z21.s }, p5/Z, [x10, #-4, MUL VL]\n" + "fmla z15.s, z20.s, z1.s[2]\n" + "fmla z19.s, z20.s, z0.s[2]\n" + "ld1w { z20.s }, p5/Z, [x10, #-3, MUL VL]\n" + "fmla z8.s, z21.s, z2.s[3]\n" + "fmla z12.s, z21.s, z1.s[3]\n" + "fmla z16.s, z21.s, z0.s[3]\n" + "fmla z9.s, z20.s, z2.s[3]\n" + "ld1w { z21.s }, p5/Z, [x10, #-2, MUL VL]\n" + "fmla z13.s, z20.s, z1.s[3]\n" + "fmla z17.s, z20.s, z0.s[3]\n" + "ld1w { z20.s }, p5/Z, [x10, #-1, MUL VL]\n" + "fmla z10.s, z21.s, z2.s[3]\n" + "fmla z14.s, z21.s, z1.s[3]\n" + "fmla z18.s, z21.s, z0.s[3]\n" + "fmla z11.s, z20.s, z2.s[3]\n" + "fmla z15.s, z20.s, z1.s[3]\n" + "fmla z19.s, z20.s, z0.s[3]\n" "bgt 35b\n" "36:" // Height 3: Multiply loop: Single iteration only "whilelt p0.s, XZR, x27\n" @@ -666,79 +666,79 @@ void sve_hybrid_fp32_mla_6x4VL ( "ld1rqw { z1.s }, p0/Z, [x25]\n" "subs x27, x27, #0x1\n" "ld1rqw { z2.s }, p0/Z, [x24]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "fmla z8.s, z6.s, z0.s[0]\n" - "fmla z12.s, z6.s, z1.s[0]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z16.s, z6.s, z2.s[0]\n" - "fmla z9.s, z7.s, z0.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z13.s, z7.s, z1.s[0]\n" - "fmla z17.s, z7.s, z2.s[0]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" + "ld1w { z21.s }, p5/Z, [x10]\n" + "fmla z8.s, z21.s, z0.s[0]\n" + "fmla z12.s, z21.s, z1.s[0]\n" + "ld1w { z20.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z16.s, z21.s, z2.s[0]\n" + "fmla z9.s, z20.s, z0.s[0]\n" + "ld1w { z21.s }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z13.s, z20.s, z1.s[0]\n" + "fmla z17.s, z20.s, z2.s[0]\n" + "ld1w { z20.s }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.s, z6.s, z0.s[0]\n" - "fmla z14.s, z6.s, z1.s[0]\n" - "fmla z18.s, z6.s, z2.s[0]\n" - "fmla z11.s, z7.s, z0.s[0]\n" - "fmla z15.s, z7.s, z1.s[0]\n" - "fmla z19.s, z7.s, z2.s[0]\n" + "fmla z10.s, z21.s, z0.s[0]\n" + "fmla z14.s, z21.s, z1.s[0]\n" + "fmla z18.s, z21.s, z2.s[0]\n" + "fmla z11.s, z20.s, z0.s[0]\n" + "fmla z15.s, z20.s, z1.s[0]\n" + "fmla z19.s, z20.s, z2.s[0]\n" "ble 37f\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[1]\n" - "fmla z12.s, z6.s, z1.s[1]\n" - "fmla z16.s, z6.s, z2.s[1]\n" - "fmla z9.s, z7.s, z0.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" + "ld1w { z21.s }, p5/Z, [x10]\n" + "ld1w { z20.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z21.s, z0.s[1]\n" + "fmla z12.s, z21.s, z1.s[1]\n" + "fmla z16.s, z21.s, z2.s[1]\n" + "fmla z9.s, z20.s, z0.s[1]\n" + "ld1w { z21.s }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z13.s, z7.s, z1.s[1]\n" - "fmla z17.s, z7.s, z2.s[1]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z13.s, z20.s, z1.s[1]\n" + "fmla z17.s, z20.s, z2.s[1]\n" + "ld1w { z20.s }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.s, z6.s, z0.s[1]\n" - "fmla z14.s, z6.s, z1.s[1]\n" - "fmla z18.s, z6.s, z2.s[1]\n" - "fmla z11.s, z7.s, z0.s[1]\n" - "fmla z15.s, z7.s, z1.s[1]\n" - "fmla z19.s, z7.s, z2.s[1]\n" + "fmla z10.s, z21.s, z0.s[1]\n" + "fmla z14.s, z21.s, z1.s[1]\n" + "fmla z18.s, z21.s, z2.s[1]\n" + "fmla z11.s, z20.s, z0.s[1]\n" + "fmla z15.s, z20.s, z1.s[1]\n" + "fmla z19.s, z20.s, z2.s[1]\n" "ble 37f\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[2]\n" - "fmla z12.s, z6.s, z1.s[2]\n" - "fmla z16.s, z6.s, z2.s[2]\n" - "fmla z9.s, z7.s, z0.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" + "ld1w { z21.s }, p5/Z, [x10]\n" + "ld1w { z20.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z21.s, z0.s[2]\n" + "fmla z12.s, z21.s, z1.s[2]\n" + "fmla z16.s, z21.s, z2.s[2]\n" + "fmla z9.s, z20.s, z0.s[2]\n" + "ld1w { z21.s }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z13.s, z7.s, z1.s[2]\n" - "fmla z17.s, z7.s, z2.s[2]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z13.s, z20.s, z1.s[2]\n" + "fmla z17.s, z20.s, z2.s[2]\n" + "ld1w { z20.s }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.s, z6.s, z0.s[2]\n" - "fmla z14.s, z6.s, z1.s[2]\n" - "fmla z18.s, z6.s, z2.s[2]\n" - "fmla z11.s, z7.s, z0.s[2]\n" - "fmla z15.s, z7.s, z1.s[2]\n" - "fmla z19.s, z7.s, z2.s[2]\n" + "fmla z10.s, z21.s, z0.s[2]\n" + "fmla z14.s, z21.s, z1.s[2]\n" + "fmla z18.s, z21.s, z2.s[2]\n" + "fmla z11.s, z20.s, z0.s[2]\n" + "fmla z15.s, z20.s, z1.s[2]\n" + "fmla z19.s, z20.s, z2.s[2]\n" "ble 37f\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[3]\n" - "fmla z12.s, z6.s, z1.s[3]\n" - "fmla z16.s, z6.s, z2.s[3]\n" - "fmla z9.s, z7.s, z0.s[3]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z13.s, z7.s, z1.s[3]\n" - "fmla z17.s, z7.s, z2.s[3]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" + "ld1w { z21.s }, p5/Z, [x10]\n" + "ld1w { z20.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z21.s, z0.s[3]\n" + "fmla z12.s, z21.s, z1.s[3]\n" + "fmla z16.s, z21.s, z2.s[3]\n" + "fmla z9.s, z20.s, z0.s[3]\n" + "ld1w { z21.s }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z13.s, z20.s, z1.s[3]\n" + "fmla z17.s, z20.s, z2.s[3]\n" + "ld1w { z20.s }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.s, z6.s, z0.s[3]\n" - "fmla z14.s, z6.s, z1.s[3]\n" - "fmla z18.s, z6.s, z2.s[3]\n" - "fmla z11.s, z7.s, z0.s[3]\n" - "fmla z15.s, z7.s, z1.s[3]\n" - "fmla z19.s, z7.s, z2.s[3]\n" + "fmla z10.s, z21.s, z0.s[3]\n" + "fmla z14.s, z21.s, z1.s[3]\n" + "fmla z18.s, z21.s, z2.s[3]\n" + "fmla z11.s, z20.s, z0.s[3]\n" + "fmla z15.s, z20.s, z1.s[3]\n" + "fmla z19.s, z20.s, z2.s[3]\n" "37:" // Height 3: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -749,33 +749,33 @@ void sve_hybrid_fp32_mla_6x4VL ( "add x24, x25, x20, LSL #2\n" "tbz %x[flags], #1, 38f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z21.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmin z12.s, p5/M, z12.s, z1.s\n" - "fmin z13.s, p5/M, z13.s, z1.s\n" - "fmin z14.s, p5/M, z14.s, z1.s\n" - "fmin z15.s, p5/M, z15.s, z1.s\n" - "fmin z16.s, p5/M, z16.s, z1.s\n" - "fmin z17.s, p5/M, z17.s, z1.s\n" - "fmin z18.s, p5/M, z18.s, z1.s\n" - "fmin z19.s, p5/M, z19.s, z1.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" - "fmax z12.s, p5/M, z12.s, z0.s\n" - "fmax z13.s, p5/M, z13.s, z0.s\n" - "fmax z14.s, p5/M, z14.s, z0.s\n" - "fmax z15.s, p5/M, z15.s, z0.s\n" - "fmax z16.s, p5/M, z16.s, z0.s\n" - "fmax z17.s, p5/M, z17.s, z0.s\n" - "fmax z18.s, p5/M, z18.s, z0.s\n" - "fmax z19.s, p5/M, z19.s, z0.s\n" + "ld1rw { z20.s }, p5/Z, [x20]\n" + "fmin z8.s, p5/M, z8.s, z21.s\n" + "fmin z9.s, p5/M, z9.s, z21.s\n" + "fmin z10.s, p5/M, z10.s, z21.s\n" + "fmin z11.s, p5/M, z11.s, z21.s\n" + "fmin z12.s, p5/M, z12.s, z21.s\n" + "fmin z13.s, p5/M, z13.s, z21.s\n" + "fmin z14.s, p5/M, z14.s, z21.s\n" + "fmin z15.s, p5/M, z15.s, z21.s\n" + "fmin z16.s, p5/M, z16.s, z21.s\n" + "fmin z17.s, p5/M, z17.s, z21.s\n" + "fmin z18.s, p5/M, z18.s, z21.s\n" + "fmin z19.s, p5/M, z19.s, z21.s\n" + "fmax z8.s, p5/M, z8.s, z20.s\n" + "fmax z9.s, p5/M, z9.s, z20.s\n" + "fmax z10.s, p5/M, z10.s, z20.s\n" + "fmax z11.s, p5/M, z11.s, z20.s\n" + "fmax z12.s, p5/M, z12.s, z20.s\n" + "fmax z13.s, p5/M, z13.s, z20.s\n" + "fmax z14.s, p5/M, z14.s, z20.s\n" + "fmax z15.s, p5/M, z15.s, z20.s\n" + "fmax z16.s, p5/M, z16.s, z20.s\n" + "fmax z17.s, p5/M, z17.s, z20.s\n" + "fmax z18.s, p5/M, z18.s, z20.s\n" + "fmax z19.s, p5/M, z19.s, z20.s\n" "38:" // Height 3: No activation "st1w { z8.s }, p4, [x9]\n" "st1w { z9.s }, p3, [x9, #1, MUL VL]\n" @@ -831,25 +831,25 @@ void sve_hybrid_fp32_mla_6x4VL ( "42:" // Height 4: no bias "tbz %x[flags], #0, 43f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" + "add x22, x9, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" "ld1w { z8.s }, p4/Z, [x9]\n" - "add x23, x24, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "ld1w { z16.s }, p4/Z, [x24]\n" - "ld1w { z17.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x24, #2, MUL VL]\n" - "ld1w { z19.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x23]\n" - "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n" - "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x22]\n" + "ld1w { z13.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x22, #2, MUL VL]\n" + "ld1w { z15.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x21]\n" + "ld1w { z17.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x21, #2, MUL VL]\n" + "ld1w { z19.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x20]\n" + "ld1w { z21.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z23.s }, p1/Z, [x20, #3, MUL VL]\n" "b 44f\n" "43:" // Height 4: no accumulate "mov z8.b, #0x0\n" @@ -873,14 +873,14 @@ void sve_hybrid_fp32_mla_6x4VL ( "45:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 46f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 47f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -890,105 +890,105 @@ void sve_hybrid_fp32_mla_6x4VL ( "b 47f\n" "46:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" "47:" // Height 4: input setup done "cmp x27, #0x4\n" "ble 49f\n" "48:" // Height 4: Multiply loop: Main loop head "whilelt p0.s, XZR, x27\n" - "ld1rqw { z0.s }, p0/Z, [x26]\n" - "ld1rqw { z1.s }, p0/Z, [x25]\n" + "ld1rqw { z3.s }, p0/Z, [x26]\n" + "ld1rqw { z2.s }, p0/Z, [x25]\n" "sub x27, x27, #0x4\n" - "ld1rqw { z2.s }, p0/Z, [x24]\n" - "ld1rqw { z3.s }, p0/Z, [x23]\n" + "ld1rqw { z1.s }, p0/Z, [x24]\n" + "ld1rqw { z0.s }, p0/Z, [x23]\n" "cmp x27, #0x4\n" "add x26, x26, #0x10\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[0]\n" - "fmla z12.s, z6.s, z1.s[0]\n" - "fmla z16.s, z6.s, z2.s[0]\n" - "fmla z20.s, z6.s, z3.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" + "ld1w { z25.s }, p5/Z, [x10]\n" + "ld1w { z24.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z25.s, z3.s[0]\n" + "fmla z12.s, z25.s, z2.s[0]\n" + "fmla z16.s, z25.s, z1.s[0]\n" + "fmla z20.s, z25.s, z0.s[0]\n" + "ld1w { z25.s }, p5/Z, [x10, #2, MUL VL]\n" "add x25, x25, #0x10\n" - "fmla z9.s, z7.s, z0.s[0]\n" - "fmla z13.s, z7.s, z1.s[0]\n" + "fmla z9.s, z24.s, z3.s[0]\n" + "fmla z13.s, z24.s, z2.s[0]\n" "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" - "fmla z17.s, z7.s, z2.s[0]\n" - "fmla z21.s, z7.s, z3.s[0]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[0]\n" - "fmla z14.s, z6.s, z1.s[0]\n" - "fmla z18.s, z6.s, z2.s[0]\n" - "fmla z22.s, z6.s, z3.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10, #4, MUL VL]\n" - "fmla z11.s, z7.s, z0.s[0]\n" - "fmla z15.s, z7.s, z1.s[0]\n" - "fmla z19.s, z7.s, z2.s[0]\n" - "fmla z23.s, z7.s, z3.s[0]\n" - "ld1w { z7.s }, p5/Z, [x10, #5, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[1]\n" - "fmla z12.s, z6.s, z1.s[1]\n" - "fmla z16.s, z6.s, z2.s[1]\n" - "fmla z20.s, z6.s, z3.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10, #6, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[1]\n" - "fmla z13.s, z7.s, z1.s[1]\n" - "fmla z17.s, z7.s, z2.s[1]\n" - "fmla z21.s, z7.s, z3.s[1]\n" - "ld1w { z7.s }, p5/Z, [x10, #7, MUL VL]\n" + "fmla z17.s, z24.s, z1.s[0]\n" + "fmla z21.s, z24.s, z0.s[0]\n" + "ld1w { z24.s }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z10.s, z25.s, z3.s[0]\n" + "fmla z14.s, z25.s, z2.s[0]\n" + "fmla z18.s, z25.s, z1.s[0]\n" + "fmla z22.s, z25.s, z0.s[0]\n" + "ld1w { z25.s }, p5/Z, [x10, #4, MUL VL]\n" + "fmla z11.s, z24.s, z3.s[0]\n" + "fmla z15.s, z24.s, z2.s[0]\n" + "fmla z19.s, z24.s, z1.s[0]\n" + "fmla z23.s, z24.s, z0.s[0]\n" + "ld1w { z24.s }, p5/Z, [x10, #5, MUL VL]\n" + "fmla z8.s, z25.s, z3.s[1]\n" + "fmla z12.s, z25.s, z2.s[1]\n" + "fmla z16.s, z25.s, z1.s[1]\n" + "fmla z20.s, z25.s, z0.s[1]\n" + "ld1w { z25.s }, p5/Z, [x10, #6, MUL VL]\n" + "fmla z9.s, z24.s, z3.s[1]\n" + "fmla z13.s, z24.s, z2.s[1]\n" + "fmla z17.s, z24.s, z1.s[1]\n" + "fmla z21.s, z24.s, z0.s[1]\n" + "ld1w { z24.s }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "fmla z10.s, z6.s, z0.s[1]\n" - "fmla z14.s, z6.s, z1.s[1]\n" - "fmla z18.s, z6.s, z2.s[1]\n" - "fmla z22.s, z6.s, z3.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10, #-8, MUL VL]\n" - "fmla z11.s, z7.s, z0.s[1]\n" - "fmla z15.s, z7.s, z1.s[1]\n" - "fmla z19.s, z7.s, z2.s[1]\n" - "fmla z23.s, z7.s, z3.s[1]\n" - "ld1w { z7.s }, p5/Z, [x10, #-7, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[2]\n" - "fmla z12.s, z6.s, z1.s[2]\n" - "fmla z16.s, z6.s, z2.s[2]\n" - "fmla z20.s, z6.s, z3.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10, #-6, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[2]\n" - "fmla z13.s, z7.s, z1.s[2]\n" - "fmla z17.s, z7.s, z2.s[2]\n" - "fmla z21.s, z7.s, z3.s[2]\n" - "ld1w { z7.s }, p5/Z, [x10, #-5, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[2]\n" - "fmla z14.s, z6.s, z1.s[2]\n" - "fmla z18.s, z6.s, z2.s[2]\n" - "fmla z22.s, z6.s, z3.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10, #-4, MUL VL]\n" - "fmla z11.s, z7.s, z0.s[2]\n" - "fmla z15.s, z7.s, z1.s[2]\n" - "fmla z19.s, z7.s, z2.s[2]\n" - "fmla z23.s, z7.s, z3.s[2]\n" - "ld1w { z7.s }, p5/Z, [x10, #-3, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[3]\n" - "fmla z12.s, z6.s, z1.s[3]\n" - "fmla z16.s, z6.s, z2.s[3]\n" - "fmla z20.s, z6.s, z3.s[3]\n" - "ld1w { z6.s }, p5/Z, [x10, #-2, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[3]\n" - "fmla z13.s, z7.s, z1.s[3]\n" - "fmla z17.s, z7.s, z2.s[3]\n" - "fmla z21.s, z7.s, z3.s[3]\n" - "ld1w { z7.s }, p5/Z, [x10, #-1, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[3]\n" - "fmla z14.s, z6.s, z1.s[3]\n" - "fmla z18.s, z6.s, z2.s[3]\n" - "fmla z22.s, z6.s, z3.s[3]\n" - "fmla z11.s, z7.s, z0.s[3]\n" - "fmla z15.s, z7.s, z1.s[3]\n" - "fmla z19.s, z7.s, z2.s[3]\n" - "fmla z23.s, z7.s, z3.s[3]\n" + "fmla z10.s, z25.s, z3.s[1]\n" + "fmla z14.s, z25.s, z2.s[1]\n" + "fmla z18.s, z25.s, z1.s[1]\n" + "fmla z22.s, z25.s, z0.s[1]\n" + "ld1w { z25.s }, p5/Z, [x10, #-8, MUL VL]\n" + "fmla z11.s, z24.s, z3.s[1]\n" + "fmla z15.s, z24.s, z2.s[1]\n" + "fmla z19.s, z24.s, z1.s[1]\n" + "fmla z23.s, z24.s, z0.s[1]\n" + "ld1w { z24.s }, p5/Z, [x10, #-7, MUL VL]\n" + "fmla z8.s, z25.s, z3.s[2]\n" + "fmla z12.s, z25.s, z2.s[2]\n" + "fmla z16.s, z25.s, z1.s[2]\n" + "fmla z20.s, z25.s, z0.s[2]\n" + "ld1w { z25.s }, p5/Z, [x10, #-6, MUL VL]\n" + "fmla z9.s, z24.s, z3.s[2]\n" + "fmla z13.s, z24.s, z2.s[2]\n" + "fmla z17.s, z24.s, z1.s[2]\n" + "fmla z21.s, z24.s, z0.s[2]\n" + "ld1w { z24.s }, p5/Z, [x10, #-5, MUL VL]\n" + "fmla z10.s, z25.s, z3.s[2]\n" + "fmla z14.s, z25.s, z2.s[2]\n" + "fmla z18.s, z25.s, z1.s[2]\n" + "fmla z22.s, z25.s, z0.s[2]\n" + "ld1w { z25.s }, p5/Z, [x10, #-4, MUL VL]\n" + "fmla z11.s, z24.s, z3.s[2]\n" + "fmla z15.s, z24.s, z2.s[2]\n" + "fmla z19.s, z24.s, z1.s[2]\n" + "fmla z23.s, z24.s, z0.s[2]\n" + "ld1w { z24.s }, p5/Z, [x10, #-3, MUL VL]\n" + "fmla z8.s, z25.s, z3.s[3]\n" + "fmla z12.s, z25.s, z2.s[3]\n" + "fmla z16.s, z25.s, z1.s[3]\n" + "fmla z20.s, z25.s, z0.s[3]\n" + "ld1w { z25.s }, p5/Z, [x10, #-2, MUL VL]\n" + "fmla z9.s, z24.s, z3.s[3]\n" + "fmla z13.s, z24.s, z2.s[3]\n" + "fmla z17.s, z24.s, z1.s[3]\n" + "fmla z21.s, z24.s, z0.s[3]\n" + "ld1w { z24.s }, p5/Z, [x10, #-1, MUL VL]\n" + "fmla z10.s, z25.s, z3.s[3]\n" + "fmla z14.s, z25.s, z2.s[3]\n" + "fmla z18.s, z25.s, z1.s[3]\n" + "fmla z22.s, z25.s, z0.s[3]\n" + "fmla z11.s, z24.s, z3.s[3]\n" + "fmla z15.s, z24.s, z2.s[3]\n" + "fmla z19.s, z24.s, z1.s[3]\n" + "fmla z23.s, z24.s, z0.s[3]\n" "bgt 48b\n" "49:" // Height 4: Multiply loop: Single iteration only "whilelt p0.s, XZR, x27\n" @@ -997,95 +997,95 @@ void sve_hybrid_fp32_mla_6x4VL ( "subs x27, x27, #0x1\n" "ld1rqw { z2.s }, p0/Z, [x24]\n" "ld1rqw { z3.s }, p0/Z, [x23]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[0]\n" - "fmla z12.s, z6.s, z1.s[0]\n" - "fmla z16.s, z6.s, z2.s[0]\n" - "fmla z20.s, z6.s, z3.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[0]\n" - "fmla z13.s, z7.s, z1.s[0]\n" - "fmla z17.s, z7.s, z2.s[0]\n" - "fmla z21.s, z7.s, z3.s[0]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" + "ld1w { z25.s }, p5/Z, [x10]\n" + "ld1w { z24.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z25.s, z0.s[0]\n" + "fmla z12.s, z25.s, z1.s[0]\n" + "fmla z16.s, z25.s, z2.s[0]\n" + "fmla z20.s, z25.s, z3.s[0]\n" + "ld1w { z25.s }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.s, z24.s, z0.s[0]\n" + "fmla z13.s, z24.s, z1.s[0]\n" + "fmla z17.s, z24.s, z2.s[0]\n" + "fmla z21.s, z24.s, z3.s[0]\n" + "ld1w { z24.s }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.s, z6.s, z0.s[0]\n" - "fmla z14.s, z6.s, z1.s[0]\n" - "fmla z18.s, z6.s, z2.s[0]\n" - "fmla z22.s, z6.s, z3.s[0]\n" - "fmla z11.s, z7.s, z0.s[0]\n" - "fmla z15.s, z7.s, z1.s[0]\n" - "fmla z19.s, z7.s, z2.s[0]\n" - "fmla z23.s, z7.s, z3.s[0]\n" + "fmla z10.s, z25.s, z0.s[0]\n" + "fmla z14.s, z25.s, z1.s[0]\n" + "fmla z18.s, z25.s, z2.s[0]\n" + "fmla z22.s, z25.s, z3.s[0]\n" + "fmla z11.s, z24.s, z0.s[0]\n" + "fmla z15.s, z24.s, z1.s[0]\n" + "fmla z19.s, z24.s, z2.s[0]\n" + "fmla z23.s, z24.s, z3.s[0]\n" "ble 50f\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[1]\n" - "fmla z12.s, z6.s, z1.s[1]\n" - "fmla z16.s, z6.s, z2.s[1]\n" - "fmla z20.s, z6.s, z3.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" + "ld1w { z25.s }, p5/Z, [x10]\n" + "ld1w { z24.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z25.s, z0.s[1]\n" + "fmla z12.s, z25.s, z1.s[1]\n" + "fmla z16.s, z25.s, z2.s[1]\n" + "fmla z20.s, z25.s, z3.s[1]\n" + "ld1w { z25.s }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z9.s, z7.s, z0.s[1]\n" - "fmla z13.s, z7.s, z1.s[1]\n" - "fmla z17.s, z7.s, z2.s[1]\n" - "fmla z21.s, z7.s, z3.s[1]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z9.s, z24.s, z0.s[1]\n" + "fmla z13.s, z24.s, z1.s[1]\n" + "fmla z17.s, z24.s, z2.s[1]\n" + "fmla z21.s, z24.s, z3.s[1]\n" + "ld1w { z24.s }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.s, z6.s, z0.s[1]\n" - "fmla z14.s, z6.s, z1.s[1]\n" - "fmla z18.s, z6.s, z2.s[1]\n" - "fmla z22.s, z6.s, z3.s[1]\n" - "fmla z11.s, z7.s, z0.s[1]\n" - "fmla z15.s, z7.s, z1.s[1]\n" - "fmla z19.s, z7.s, z2.s[1]\n" - "fmla z23.s, z7.s, z3.s[1]\n" + "fmla z10.s, z25.s, z0.s[1]\n" + "fmla z14.s, z25.s, z1.s[1]\n" + "fmla z18.s, z25.s, z2.s[1]\n" + "fmla z22.s, z25.s, z3.s[1]\n" + "fmla z11.s, z24.s, z0.s[1]\n" + "fmla z15.s, z24.s, z1.s[1]\n" + "fmla z19.s, z24.s, z2.s[1]\n" + "fmla z23.s, z24.s, z3.s[1]\n" "ble 50f\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[2]\n" - "fmla z12.s, z6.s, z1.s[2]\n" - "fmla z16.s, z6.s, z2.s[2]\n" - "fmla z20.s, z6.s, z3.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" + "ld1w { z25.s }, p5/Z, [x10]\n" + "ld1w { z24.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z25.s, z0.s[2]\n" + "fmla z12.s, z25.s, z1.s[2]\n" + "fmla z16.s, z25.s, z2.s[2]\n" + "fmla z20.s, z25.s, z3.s[2]\n" + "ld1w { z25.s }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x1\n" - "fmla z9.s, z7.s, z0.s[2]\n" - "fmla z13.s, z7.s, z1.s[2]\n" - "fmla z17.s, z7.s, z2.s[2]\n" - "fmla z21.s, z7.s, z3.s[2]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z9.s, z24.s, z0.s[2]\n" + "fmla z13.s, z24.s, z1.s[2]\n" + "fmla z17.s, z24.s, z2.s[2]\n" + "fmla z21.s, z24.s, z3.s[2]\n" + "ld1w { z24.s }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.s, z6.s, z0.s[2]\n" - "fmla z14.s, z6.s, z1.s[2]\n" - "fmla z18.s, z6.s, z2.s[2]\n" - "fmla z22.s, z6.s, z3.s[2]\n" - "fmla z11.s, z7.s, z0.s[2]\n" - "fmla z15.s, z7.s, z1.s[2]\n" - "fmla z19.s, z7.s, z2.s[2]\n" - "fmla z23.s, z7.s, z3.s[2]\n" + "fmla z10.s, z25.s, z0.s[2]\n" + "fmla z14.s, z25.s, z1.s[2]\n" + "fmla z18.s, z25.s, z2.s[2]\n" + "fmla z22.s, z25.s, z3.s[2]\n" + "fmla z11.s, z24.s, z0.s[2]\n" + "fmla z15.s, z24.s, z1.s[2]\n" + "fmla z19.s, z24.s, z2.s[2]\n" + "fmla z23.s, z24.s, z3.s[2]\n" "ble 50f\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[3]\n" - "fmla z12.s, z6.s, z1.s[3]\n" - "fmla z16.s, z6.s, z2.s[3]\n" - "fmla z20.s, z6.s, z3.s[3]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[3]\n" - "fmla z13.s, z7.s, z1.s[3]\n" - "fmla z17.s, z7.s, z2.s[3]\n" - "fmla z21.s, z7.s, z3.s[3]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" + "ld1w { z25.s }, p5/Z, [x10]\n" + "ld1w { z24.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z25.s, z0.s[3]\n" + "fmla z12.s, z25.s, z1.s[3]\n" + "fmla z16.s, z25.s, z2.s[3]\n" + "fmla z20.s, z25.s, z3.s[3]\n" + "ld1w { z25.s }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.s, z24.s, z0.s[3]\n" + "fmla z13.s, z24.s, z1.s[3]\n" + "fmla z17.s, z24.s, z2.s[3]\n" + "fmla z21.s, z24.s, z3.s[3]\n" + "ld1w { z24.s }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.s, z6.s, z0.s[3]\n" - "fmla z14.s, z6.s, z1.s[3]\n" - "fmla z18.s, z6.s, z2.s[3]\n" - "fmla z22.s, z6.s, z3.s[3]\n" - "fmla z11.s, z7.s, z0.s[3]\n" - "fmla z15.s, z7.s, z1.s[3]\n" - "fmla z19.s, z7.s, z2.s[3]\n" - "fmla z23.s, z7.s, z3.s[3]\n" + "fmla z10.s, z25.s, z0.s[3]\n" + "fmla z14.s, z25.s, z1.s[3]\n" + "fmla z18.s, z25.s, z2.s[3]\n" + "fmla z22.s, z25.s, z3.s[3]\n" + "fmla z11.s, z24.s, z0.s[3]\n" + "fmla z15.s, z24.s, z1.s[3]\n" + "fmla z19.s, z24.s, z2.s[3]\n" + "fmla z23.s, z24.s, z3.s[3]\n" "50:" // Height 4: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1097,41 +1097,41 @@ void sve_hybrid_fp32_mla_6x4VL ( "add x23, x24, x20, LSL #2\n" "tbz %x[flags], #1, 51f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z25.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmin z12.s, p5/M, z12.s, z1.s\n" - "fmin z13.s, p5/M, z13.s, z1.s\n" - "fmin z14.s, p5/M, z14.s, z1.s\n" - "fmin z15.s, p5/M, z15.s, z1.s\n" - "fmin z16.s, p5/M, z16.s, z1.s\n" - "fmin z17.s, p5/M, z17.s, z1.s\n" - "fmin z18.s, p5/M, z18.s, z1.s\n" - "fmin z19.s, p5/M, z19.s, z1.s\n" - "fmin z20.s, p5/M, z20.s, z1.s\n" - "fmin z21.s, p5/M, z21.s, z1.s\n" - "fmin z22.s, p5/M, z22.s, z1.s\n" - "fmin z23.s, p5/M, z23.s, z1.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" - "fmax z12.s, p5/M, z12.s, z0.s\n" - "fmax z13.s, p5/M, z13.s, z0.s\n" - "fmax z14.s, p5/M, z14.s, z0.s\n" - "fmax z15.s, p5/M, z15.s, z0.s\n" - "fmax z16.s, p5/M, z16.s, z0.s\n" - "fmax z17.s, p5/M, z17.s, z0.s\n" - "fmax z18.s, p5/M, z18.s, z0.s\n" - "fmax z19.s, p5/M, z19.s, z0.s\n" - "fmax z20.s, p5/M, z20.s, z0.s\n" - "fmax z21.s, p5/M, z21.s, z0.s\n" - "fmax z22.s, p5/M, z22.s, z0.s\n" - "fmax z23.s, p5/M, z23.s, z0.s\n" + "ld1rw { z24.s }, p5/Z, [x20]\n" + "fmin z8.s, p5/M, z8.s, z25.s\n" + "fmin z9.s, p5/M, z9.s, z25.s\n" + "fmin z10.s, p5/M, z10.s, z25.s\n" + "fmin z11.s, p5/M, z11.s, z25.s\n" + "fmin z12.s, p5/M, z12.s, z25.s\n" + "fmin z13.s, p5/M, z13.s, z25.s\n" + "fmin z14.s, p5/M, z14.s, z25.s\n" + "fmin z15.s, p5/M, z15.s, z25.s\n" + "fmin z16.s, p5/M, z16.s, z25.s\n" + "fmin z17.s, p5/M, z17.s, z25.s\n" + "fmin z18.s, p5/M, z18.s, z25.s\n" + "fmin z19.s, p5/M, z19.s, z25.s\n" + "fmin z20.s, p5/M, z20.s, z25.s\n" + "fmin z21.s, p5/M, z21.s, z25.s\n" + "fmin z22.s, p5/M, z22.s, z25.s\n" + "fmin z23.s, p5/M, z23.s, z25.s\n" + "fmax z8.s, p5/M, z8.s, z24.s\n" + "fmax z9.s, p5/M, z9.s, z24.s\n" + "fmax z10.s, p5/M, z10.s, z24.s\n" + "fmax z11.s, p5/M, z11.s, z24.s\n" + "fmax z12.s, p5/M, z12.s, z24.s\n" + "fmax z13.s, p5/M, z13.s, z24.s\n" + "fmax z14.s, p5/M, z14.s, z24.s\n" + "fmax z15.s, p5/M, z15.s, z24.s\n" + "fmax z16.s, p5/M, z16.s, z24.s\n" + "fmax z17.s, p5/M, z17.s, z24.s\n" + "fmax z18.s, p5/M, z18.s, z24.s\n" + "fmax z19.s, p5/M, z19.s, z24.s\n" + "fmax z20.s, p5/M, z20.s, z24.s\n" + "fmax z21.s, p5/M, z21.s, z24.s\n" + "fmax z22.s, p5/M, z22.s, z24.s\n" + "fmax z23.s, p5/M, z23.s, z24.s\n" "51:" // Height 4: No activation "st1w { z8.s }, p4, [x9]\n" "st1w { z9.s }, p3, [x9, #1, MUL VL]\n" @@ -1195,30 +1195,30 @@ void sve_hybrid_fp32_mla_6x4VL ( "55:" // Height 5: no bias "tbz %x[flags], #0, 56f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z8.s }, p4/Z, [x9]\n" - "add x23, x24, x20, LSL #2\n" + "add x23, x9, x20, LSL #2\n" "add x22, x23, x20, LSL #2\n" + "ld1w { z8.s }, p4/Z, [x9]\n" + "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "ld1w { z16.s }, p4/Z, [x24]\n" - "ld1w { z17.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x24, #2, MUL VL]\n" - "ld1w { z19.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x23]\n" - "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n" - "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n" - "ld1w { z24.s }, p4/Z, [x22]\n" - "ld1w { z25.s }, p3/Z, [x22, #1, MUL VL]\n" - "ld1w { z26.s }, p2/Z, [x22, #2, MUL VL]\n" - "ld1w { z27.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x23]\n" + "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n" + "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x22]\n" + "ld1w { z17.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x22, #2, MUL VL]\n" + "ld1w { z19.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x21]\n" + "ld1w { z21.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x21, #2, MUL VL]\n" + "ld1w { z23.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z24.s }, p4/Z, [x20]\n" + "ld1w { z25.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z26.s }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z27.s }, p1/Z, [x20, #3, MUL VL]\n" "b 57f\n" "56:" // Height 5: no accumulate "mov z8.b, #0x0\n" @@ -1246,15 +1246,15 @@ void sve_hybrid_fp32_mla_6x4VL ( "58:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 59f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 60f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -1265,124 +1265,124 @@ void sve_hybrid_fp32_mla_6x4VL ( "b 60f\n" "59:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" "60:" // Height 5: input setup done "cmp x27, #0x4\n" "ble 62f\n" "61:" // Height 5: Multiply loop: Main loop head "whilelt p0.s, XZR, x27\n" - "ld1rqw { z0.s }, p0/Z, [x26]\n" - "ld1rqw { z1.s }, p0/Z, [x25]\n" + "ld1rqw { z4.s }, p0/Z, [x26]\n" + "ld1rqw { z3.s }, p0/Z, [x25]\n" "sub x27, x27, #0x4\n" "ld1rqw { z2.s }, p0/Z, [x24]\n" - "ld1rqw { z3.s }, p0/Z, [x23]\n" + "ld1rqw { z1.s }, p0/Z, [x23]\n" "cmp x27, #0x4\n" "add x26, x26, #0x10\n" - "ld1rqw { z4.s }, p0/Z, [x22]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "fmla z8.s, z6.s, z0.s[0]\n" - "fmla z12.s, z6.s, z1.s[0]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z16.s, z6.s, z2.s[0]\n" - "fmla z20.s, z6.s, z3.s[0]\n" + "ld1rqw { z0.s }, p0/Z, [x22]\n" + "ld1w { z29.s }, p5/Z, [x10]\n" + "fmla z8.s, z29.s, z4.s[0]\n" + "fmla z12.s, z29.s, z3.s[0]\n" + "ld1w { z28.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z16.s, z29.s, z2.s[0]\n" + "fmla z20.s, z29.s, z1.s[0]\n" "add x25, x25, #0x10\n" - "fmla z24.s, z6.s, z4.s[0]\n" - "fmla z9.s, z7.s, z0.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z24.s, z29.s, z0.s[0]\n" + "fmla z9.s, z28.s, z4.s[0]\n" + "ld1w { z29.s }, p5/Z, [x10, #2, MUL VL]\n" "add x24, x24, #0x10\n" - "fmla z13.s, z7.s, z1.s[0]\n" - "fmla z17.s, z7.s, z2.s[0]\n" + "fmla z13.s, z28.s, z3.s[0]\n" + "fmla z17.s, z28.s, z2.s[0]\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - "fmla z21.s, z7.s, z3.s[0]\n" - "fmla z25.s, z7.s, z4.s[0]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[0]\n" - "fmla z14.s, z6.s, z1.s[0]\n" - "fmla z18.s, z6.s, z2.s[0]\n" - "fmla z22.s, z6.s, z3.s[0]\n" - "fmla z26.s, z6.s, z4.s[0]\n" - "fmla z11.s, z7.s, z0.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10, #4, MUL VL]\n" - "fmla z15.s, z7.s, z1.s[0]\n" - "fmla z19.s, z7.s, z2.s[0]\n" - "fmla z23.s, z7.s, z3.s[0]\n" - "fmla z27.s, z7.s, z4.s[0]\n" - "ld1w { z7.s }, p5/Z, [x10, #5, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[1]\n" - "fmla z12.s, z6.s, z1.s[1]\n" - "fmla z16.s, z6.s, z2.s[1]\n" - "fmla z20.s, z6.s, z3.s[1]\n" - "fmla z24.s, z6.s, z4.s[1]\n" - "fmla z9.s, z7.s, z0.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10, #6, MUL VL]\n" - "fmla z13.s, z7.s, z1.s[1]\n" - "fmla z17.s, z7.s, z2.s[1]\n" - "fmla z21.s, z7.s, z3.s[1]\n" - "fmla z25.s, z7.s, z4.s[1]\n" - "ld1w { z7.s }, p5/Z, [x10, #7, MUL VL]\n" + "fmla z21.s, z28.s, z1.s[0]\n" + "fmla z25.s, z28.s, z0.s[0]\n" + "ld1w { z28.s }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z10.s, z29.s, z4.s[0]\n" + "fmla z14.s, z29.s, z3.s[0]\n" + "fmla z18.s, z29.s, z2.s[0]\n" + "fmla z22.s, z29.s, z1.s[0]\n" + "fmla z26.s, z29.s, z0.s[0]\n" + "fmla z11.s, z28.s, z4.s[0]\n" + "ld1w { z29.s }, p5/Z, [x10, #4, MUL VL]\n" + "fmla z15.s, z28.s, z3.s[0]\n" + "fmla z19.s, z28.s, z2.s[0]\n" + "fmla z23.s, z28.s, z1.s[0]\n" + "fmla z27.s, z28.s, z0.s[0]\n" + "ld1w { z28.s }, p5/Z, [x10, #5, MUL VL]\n" + "fmla z8.s, z29.s, z4.s[1]\n" + "fmla z12.s, z29.s, z3.s[1]\n" + "fmla z16.s, z29.s, z2.s[1]\n" + "fmla z20.s, z29.s, z1.s[1]\n" + "fmla z24.s, z29.s, z0.s[1]\n" + "fmla z9.s, z28.s, z4.s[1]\n" + "ld1w { z29.s }, p5/Z, [x10, #6, MUL VL]\n" + "fmla z13.s, z28.s, z3.s[1]\n" + "fmla z17.s, z28.s, z2.s[1]\n" + "fmla z21.s, z28.s, z1.s[1]\n" + "fmla z25.s, z28.s, z0.s[1]\n" + "ld1w { z28.s }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "fmla z10.s, z6.s, z0.s[1]\n" - "fmla z14.s, z6.s, z1.s[1]\n" - "fmla z18.s, z6.s, z2.s[1]\n" - "fmla z22.s, z6.s, z3.s[1]\n" - "fmla z26.s, z6.s, z4.s[1]\n" - "fmla z11.s, z7.s, z0.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10, #-8, MUL VL]\n" - "fmla z15.s, z7.s, z1.s[1]\n" - "fmla z19.s, z7.s, z2.s[1]\n" - "fmla z23.s, z7.s, z3.s[1]\n" - "fmla z27.s, z7.s, z4.s[1]\n" - "ld1w { z7.s }, p5/Z, [x10, #-7, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[2]\n" - "fmla z12.s, z6.s, z1.s[2]\n" - "fmla z16.s, z6.s, z2.s[2]\n" - "fmla z20.s, z6.s, z3.s[2]\n" - "fmla z24.s, z6.s, z4.s[2]\n" - "fmla z9.s, z7.s, z0.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10, #-6, MUL VL]\n" - "fmla z13.s, z7.s, z1.s[2]\n" - "fmla z17.s, z7.s, z2.s[2]\n" - "fmla z21.s, z7.s, z3.s[2]\n" - "fmla z25.s, z7.s, z4.s[2]\n" - "ld1w { z7.s }, p5/Z, [x10, #-5, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[2]\n" - "fmla z14.s, z6.s, z1.s[2]\n" - "fmla z18.s, z6.s, z2.s[2]\n" - "fmla z22.s, z6.s, z3.s[2]\n" - "fmla z26.s, z6.s, z4.s[2]\n" - "fmla z11.s, z7.s, z0.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10, #-4, MUL VL]\n" - "fmla z15.s, z7.s, z1.s[2]\n" - "fmla z19.s, z7.s, z2.s[2]\n" - "fmla z23.s, z7.s, z3.s[2]\n" - "fmla z27.s, z7.s, z4.s[2]\n" - "ld1w { z7.s }, p5/Z, [x10, #-3, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[3]\n" - "fmla z12.s, z6.s, z1.s[3]\n" - "fmla z16.s, z6.s, z2.s[3]\n" - "fmla z20.s, z6.s, z3.s[3]\n" - "fmla z24.s, z6.s, z4.s[3]\n" - "fmla z9.s, z7.s, z0.s[3]\n" - "ld1w { z6.s }, p5/Z, [x10, #-2, MUL VL]\n" - "fmla z13.s, z7.s, z1.s[3]\n" - "fmla z17.s, z7.s, z2.s[3]\n" - "fmla z21.s, z7.s, z3.s[3]\n" - "fmla z25.s, z7.s, z4.s[3]\n" - "ld1w { z7.s }, p5/Z, [x10, #-1, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[3]\n" - "fmla z14.s, z6.s, z1.s[3]\n" - "fmla z18.s, z6.s, z2.s[3]\n" - "fmla z22.s, z6.s, z3.s[3]\n" - "fmla z26.s, z6.s, z4.s[3]\n" - "fmla z11.s, z7.s, z0.s[3]\n" - "fmla z15.s, z7.s, z1.s[3]\n" - "fmla z19.s, z7.s, z2.s[3]\n" - "fmla z23.s, z7.s, z3.s[3]\n" - "fmla z27.s, z7.s, z4.s[3]\n" + "fmla z10.s, z29.s, z4.s[1]\n" + "fmla z14.s, z29.s, z3.s[1]\n" + "fmla z18.s, z29.s, z2.s[1]\n" + "fmla z22.s, z29.s, z1.s[1]\n" + "fmla z26.s, z29.s, z0.s[1]\n" + "fmla z11.s, z28.s, z4.s[1]\n" + "ld1w { z29.s }, p5/Z, [x10, #-8, MUL VL]\n" + "fmla z15.s, z28.s, z3.s[1]\n" + "fmla z19.s, z28.s, z2.s[1]\n" + "fmla z23.s, z28.s, z1.s[1]\n" + "fmla z27.s, z28.s, z0.s[1]\n" + "ld1w { z28.s }, p5/Z, [x10, #-7, MUL VL]\n" + "fmla z8.s, z29.s, z4.s[2]\n" + "fmla z12.s, z29.s, z3.s[2]\n" + "fmla z16.s, z29.s, z2.s[2]\n" + "fmla z20.s, z29.s, z1.s[2]\n" + "fmla z24.s, z29.s, z0.s[2]\n" + "fmla z9.s, z28.s, z4.s[2]\n" + "ld1w { z29.s }, p5/Z, [x10, #-6, MUL VL]\n" + "fmla z13.s, z28.s, z3.s[2]\n" + "fmla z17.s, z28.s, z2.s[2]\n" + "fmla z21.s, z28.s, z1.s[2]\n" + "fmla z25.s, z28.s, z0.s[2]\n" + "ld1w { z28.s }, p5/Z, [x10, #-5, MUL VL]\n" + "fmla z10.s, z29.s, z4.s[2]\n" + "fmla z14.s, z29.s, z3.s[2]\n" + "fmla z18.s, z29.s, z2.s[2]\n" + "fmla z22.s, z29.s, z1.s[2]\n" + "fmla z26.s, z29.s, z0.s[2]\n" + "fmla z11.s, z28.s, z4.s[2]\n" + "ld1w { z29.s }, p5/Z, [x10, #-4, MUL VL]\n" + "fmla z15.s, z28.s, z3.s[2]\n" + "fmla z19.s, z28.s, z2.s[2]\n" + "fmla z23.s, z28.s, z1.s[2]\n" + "fmla z27.s, z28.s, z0.s[2]\n" + "ld1w { z28.s }, p5/Z, [x10, #-3, MUL VL]\n" + "fmla z8.s, z29.s, z4.s[3]\n" + "fmla z12.s, z29.s, z3.s[3]\n" + "fmla z16.s, z29.s, z2.s[3]\n" + "fmla z20.s, z29.s, z1.s[3]\n" + "fmla z24.s, z29.s, z0.s[3]\n" + "fmla z9.s, z28.s, z4.s[3]\n" + "ld1w { z29.s }, p5/Z, [x10, #-2, MUL VL]\n" + "fmla z13.s, z28.s, z3.s[3]\n" + "fmla z17.s, z28.s, z2.s[3]\n" + "fmla z21.s, z28.s, z1.s[3]\n" + "fmla z25.s, z28.s, z0.s[3]\n" + "ld1w { z28.s }, p5/Z, [x10, #-1, MUL VL]\n" + "fmla z10.s, z29.s, z4.s[3]\n" + "fmla z14.s, z29.s, z3.s[3]\n" + "fmla z18.s, z29.s, z2.s[3]\n" + "fmla z22.s, z29.s, z1.s[3]\n" + "fmla z26.s, z29.s, z0.s[3]\n" + "fmla z11.s, z28.s, z4.s[3]\n" + "fmla z15.s, z28.s, z3.s[3]\n" + "fmla z19.s, z28.s, z2.s[3]\n" + "fmla z23.s, z28.s, z1.s[3]\n" + "fmla z27.s, z28.s, z0.s[3]\n" "bgt 61b\n" "62:" // Height 5: Multiply loop: Single iteration only "whilelt p0.s, XZR, x27\n" @@ -1392,111 +1392,111 @@ void sve_hybrid_fp32_mla_6x4VL ( "ld1rqw { z2.s }, p0/Z, [x24]\n" "ld1rqw { z3.s }, p0/Z, [x23]\n" "ld1rqw { z4.s }, p0/Z, [x22]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "fmla z8.s, z6.s, z0.s[0]\n" - "fmla z12.s, z6.s, z1.s[0]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z16.s, z6.s, z2.s[0]\n" - "fmla z20.s, z6.s, z3.s[0]\n" - "fmla z24.s, z6.s, z4.s[0]\n" - "fmla z9.s, z7.s, z0.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z13.s, z7.s, z1.s[0]\n" - "fmla z17.s, z7.s, z2.s[0]\n" - "fmla z21.s, z7.s, z3.s[0]\n" - "fmla z25.s, z7.s, z4.s[0]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" + "ld1w { z29.s }, p5/Z, [x10]\n" + "fmla z8.s, z29.s, z0.s[0]\n" + "fmla z12.s, z29.s, z1.s[0]\n" + "ld1w { z28.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z16.s, z29.s, z2.s[0]\n" + "fmla z20.s, z29.s, z3.s[0]\n" + "fmla z24.s, z29.s, z4.s[0]\n" + "fmla z9.s, z28.s, z0.s[0]\n" + "ld1w { z29.s }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z13.s, z28.s, z1.s[0]\n" + "fmla z17.s, z28.s, z2.s[0]\n" + "fmla z21.s, z28.s, z3.s[0]\n" + "fmla z25.s, z28.s, z4.s[0]\n" + "ld1w { z28.s }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.s, z6.s, z0.s[0]\n" - "fmla z14.s, z6.s, z1.s[0]\n" - "fmla z18.s, z6.s, z2.s[0]\n" - "fmla z22.s, z6.s, z3.s[0]\n" - "fmla z26.s, z6.s, z4.s[0]\n" - "fmla z11.s, z7.s, z0.s[0]\n" - "fmla z15.s, z7.s, z1.s[0]\n" - "fmla z19.s, z7.s, z2.s[0]\n" - "fmla z23.s, z7.s, z3.s[0]\n" - "fmla z27.s, z7.s, z4.s[0]\n" + "fmla z10.s, z29.s, z0.s[0]\n" + "fmla z14.s, z29.s, z1.s[0]\n" + "fmla z18.s, z29.s, z2.s[0]\n" + "fmla z22.s, z29.s, z3.s[0]\n" + "fmla z26.s, z29.s, z4.s[0]\n" + "fmla z11.s, z28.s, z0.s[0]\n" + "fmla z15.s, z28.s, z1.s[0]\n" + "fmla z19.s, z28.s, z2.s[0]\n" + "fmla z23.s, z28.s, z3.s[0]\n" + "fmla z27.s, z28.s, z4.s[0]\n" "ble 63f\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[1]\n" - "fmla z12.s, z6.s, z1.s[1]\n" - "fmla z16.s, z6.s, z2.s[1]\n" - "fmla z20.s, z6.s, z3.s[1]\n" + "ld1w { z29.s }, p5/Z, [x10]\n" + "ld1w { z28.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z29.s, z0.s[1]\n" + "fmla z12.s, z29.s, z1.s[1]\n" + "fmla z16.s, z29.s, z2.s[1]\n" + "fmla z20.s, z29.s, z3.s[1]\n" "subs x27, x27, #0x1\n" - "fmla z24.s, z6.s, z4.s[1]\n" - "fmla z9.s, z7.s, z0.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z13.s, z7.s, z1.s[1]\n" - "fmla z17.s, z7.s, z2.s[1]\n" - "fmla z21.s, z7.s, z3.s[1]\n" - "fmla z25.s, z7.s, z4.s[1]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z24.s, z29.s, z4.s[1]\n" + "fmla z9.s, z28.s, z0.s[1]\n" + "ld1w { z29.s }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z13.s, z28.s, z1.s[1]\n" + "fmla z17.s, z28.s, z2.s[1]\n" + "fmla z21.s, z28.s, z3.s[1]\n" + "fmla z25.s, z28.s, z4.s[1]\n" + "ld1w { z28.s }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.s, z6.s, z0.s[1]\n" - "fmla z14.s, z6.s, z1.s[1]\n" - "fmla z18.s, z6.s, z2.s[1]\n" - "fmla z22.s, z6.s, z3.s[1]\n" - "fmla z26.s, z6.s, z4.s[1]\n" - "fmla z11.s, z7.s, z0.s[1]\n" - "fmla z15.s, z7.s, z1.s[1]\n" - "fmla z19.s, z7.s, z2.s[1]\n" - "fmla z23.s, z7.s, z3.s[1]\n" - "fmla z27.s, z7.s, z4.s[1]\n" + "fmla z10.s, z29.s, z0.s[1]\n" + "fmla z14.s, z29.s, z1.s[1]\n" + "fmla z18.s, z29.s, z2.s[1]\n" + "fmla z22.s, z29.s, z3.s[1]\n" + "fmla z26.s, z29.s, z4.s[1]\n" + "fmla z11.s, z28.s, z0.s[1]\n" + "fmla z15.s, z28.s, z1.s[1]\n" + "fmla z19.s, z28.s, z2.s[1]\n" + "fmla z23.s, z28.s, z3.s[1]\n" + "fmla z27.s, z28.s, z4.s[1]\n" "ble 63f\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[2]\n" - "fmla z12.s, z6.s, z1.s[2]\n" - "fmla z16.s, z6.s, z2.s[2]\n" - "fmla z20.s, z6.s, z3.s[2]\n" + "ld1w { z29.s }, p5/Z, [x10]\n" + "ld1w { z28.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z29.s, z0.s[2]\n" + "fmla z12.s, z29.s, z1.s[2]\n" + "fmla z16.s, z29.s, z2.s[2]\n" + "fmla z20.s, z29.s, z3.s[2]\n" "subs x27, x27, #0x1\n" - "fmla z24.s, z6.s, z4.s[2]\n" - "fmla z9.s, z7.s, z0.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z13.s, z7.s, z1.s[2]\n" - "fmla z17.s, z7.s, z2.s[2]\n" - "fmla z21.s, z7.s, z3.s[2]\n" - "fmla z25.s, z7.s, z4.s[2]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z24.s, z29.s, z4.s[2]\n" + "fmla z9.s, z28.s, z0.s[2]\n" + "ld1w { z29.s }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z13.s, z28.s, z1.s[2]\n" + "fmla z17.s, z28.s, z2.s[2]\n" + "fmla z21.s, z28.s, z3.s[2]\n" + "fmla z25.s, z28.s, z4.s[2]\n" + "ld1w { z28.s }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.s, z6.s, z0.s[2]\n" - "fmla z14.s, z6.s, z1.s[2]\n" - "fmla z18.s, z6.s, z2.s[2]\n" - "fmla z22.s, z6.s, z3.s[2]\n" - "fmla z26.s, z6.s, z4.s[2]\n" - "fmla z11.s, z7.s, z0.s[2]\n" - "fmla z15.s, z7.s, z1.s[2]\n" - "fmla z19.s, z7.s, z2.s[2]\n" - "fmla z23.s, z7.s, z3.s[2]\n" - "fmla z27.s, z7.s, z4.s[2]\n" + "fmla z10.s, z29.s, z0.s[2]\n" + "fmla z14.s, z29.s, z1.s[2]\n" + "fmla z18.s, z29.s, z2.s[2]\n" + "fmla z22.s, z29.s, z3.s[2]\n" + "fmla z26.s, z29.s, z4.s[2]\n" + "fmla z11.s, z28.s, z0.s[2]\n" + "fmla z15.s, z28.s, z1.s[2]\n" + "fmla z19.s, z28.s, z2.s[2]\n" + "fmla z23.s, z28.s, z3.s[2]\n" + "fmla z27.s, z28.s, z4.s[2]\n" "ble 63f\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[3]\n" - "fmla z12.s, z6.s, z1.s[3]\n" - "fmla z16.s, z6.s, z2.s[3]\n" - "fmla z20.s, z6.s, z3.s[3]\n" - "fmla z24.s, z6.s, z4.s[3]\n" - "fmla z9.s, z7.s, z0.s[3]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z13.s, z7.s, z1.s[3]\n" - "fmla z17.s, z7.s, z2.s[3]\n" - "fmla z21.s, z7.s, z3.s[3]\n" - "fmla z25.s, z7.s, z4.s[3]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" + "ld1w { z29.s }, p5/Z, [x10]\n" + "ld1w { z28.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z29.s, z0.s[3]\n" + "fmla z12.s, z29.s, z1.s[3]\n" + "fmla z16.s, z29.s, z2.s[3]\n" + "fmla z20.s, z29.s, z3.s[3]\n" + "fmla z24.s, z29.s, z4.s[3]\n" + "fmla z9.s, z28.s, z0.s[3]\n" + "ld1w { z29.s }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z13.s, z28.s, z1.s[3]\n" + "fmla z17.s, z28.s, z2.s[3]\n" + "fmla z21.s, z28.s, z3.s[3]\n" + "fmla z25.s, z28.s, z4.s[3]\n" + "ld1w { z28.s }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.s, z6.s, z0.s[3]\n" - "fmla z14.s, z6.s, z1.s[3]\n" - "fmla z18.s, z6.s, z2.s[3]\n" - "fmla z22.s, z6.s, z3.s[3]\n" - "fmla z26.s, z6.s, z4.s[3]\n" - "fmla z11.s, z7.s, z0.s[3]\n" - "fmla z15.s, z7.s, z1.s[3]\n" - "fmla z19.s, z7.s, z2.s[3]\n" - "fmla z23.s, z7.s, z3.s[3]\n" - "fmla z27.s, z7.s, z4.s[3]\n" + "fmla z10.s, z29.s, z0.s[3]\n" + "fmla z14.s, z29.s, z1.s[3]\n" + "fmla z18.s, z29.s, z2.s[3]\n" + "fmla z22.s, z29.s, z3.s[3]\n" + "fmla z26.s, z29.s, z4.s[3]\n" + "fmla z11.s, z28.s, z0.s[3]\n" + "fmla z15.s, z28.s, z1.s[3]\n" + "fmla z19.s, z28.s, z2.s[3]\n" + "fmla z23.s, z28.s, z3.s[3]\n" + "fmla z27.s, z28.s, z4.s[3]\n" "63:" // Height 5: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1509,49 +1509,49 @@ void sve_hybrid_fp32_mla_6x4VL ( "add x22, x23, x20, LSL #2\n" "tbz %x[flags], #1, 64f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z29.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmin z12.s, p5/M, z12.s, z1.s\n" - "fmin z13.s, p5/M, z13.s, z1.s\n" - "fmin z14.s, p5/M, z14.s, z1.s\n" - "fmin z15.s, p5/M, z15.s, z1.s\n" - "fmin z16.s, p5/M, z16.s, z1.s\n" - "fmin z17.s, p5/M, z17.s, z1.s\n" - "fmin z18.s, p5/M, z18.s, z1.s\n" - "fmin z19.s, p5/M, z19.s, z1.s\n" - "fmin z20.s, p5/M, z20.s, z1.s\n" - "fmin z21.s, p5/M, z21.s, z1.s\n" - "fmin z22.s, p5/M, z22.s, z1.s\n" - "fmin z23.s, p5/M, z23.s, z1.s\n" - "fmin z24.s, p5/M, z24.s, z1.s\n" - "fmin z25.s, p5/M, z25.s, z1.s\n" - "fmin z26.s, p5/M, z26.s, z1.s\n" - "fmin z27.s, p5/M, z27.s, z1.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" - "fmax z12.s, p5/M, z12.s, z0.s\n" - "fmax z13.s, p5/M, z13.s, z0.s\n" - "fmax z14.s, p5/M, z14.s, z0.s\n" - "fmax z15.s, p5/M, z15.s, z0.s\n" - "fmax z16.s, p5/M, z16.s, z0.s\n" - "fmax z17.s, p5/M, z17.s, z0.s\n" - "fmax z18.s, p5/M, z18.s, z0.s\n" - "fmax z19.s, p5/M, z19.s, z0.s\n" - "fmax z20.s, p5/M, z20.s, z0.s\n" - "fmax z21.s, p5/M, z21.s, z0.s\n" - "fmax z22.s, p5/M, z22.s, z0.s\n" - "fmax z23.s, p5/M, z23.s, z0.s\n" - "fmax z24.s, p5/M, z24.s, z0.s\n" - "fmax z25.s, p5/M, z25.s, z0.s\n" - "fmax z26.s, p5/M, z26.s, z0.s\n" - "fmax z27.s, p5/M, z27.s, z0.s\n" + "ld1rw { z28.s }, p5/Z, [x20]\n" + "fmin z8.s, p5/M, z8.s, z29.s\n" + "fmin z9.s, p5/M, z9.s, z29.s\n" + "fmin z10.s, p5/M, z10.s, z29.s\n" + "fmin z11.s, p5/M, z11.s, z29.s\n" + "fmin z12.s, p5/M, z12.s, z29.s\n" + "fmin z13.s, p5/M, z13.s, z29.s\n" + "fmin z14.s, p5/M, z14.s, z29.s\n" + "fmin z15.s, p5/M, z15.s, z29.s\n" + "fmin z16.s, p5/M, z16.s, z29.s\n" + "fmin z17.s, p5/M, z17.s, z29.s\n" + "fmin z18.s, p5/M, z18.s, z29.s\n" + "fmin z19.s, p5/M, z19.s, z29.s\n" + "fmin z20.s, p5/M, z20.s, z29.s\n" + "fmin z21.s, p5/M, z21.s, z29.s\n" + "fmin z22.s, p5/M, z22.s, z29.s\n" + "fmin z23.s, p5/M, z23.s, z29.s\n" + "fmin z24.s, p5/M, z24.s, z29.s\n" + "fmin z25.s, p5/M, z25.s, z29.s\n" + "fmin z26.s, p5/M, z26.s, z29.s\n" + "fmin z27.s, p5/M, z27.s, z29.s\n" + "fmax z8.s, p5/M, z8.s, z28.s\n" + "fmax z9.s, p5/M, z9.s, z28.s\n" + "fmax z10.s, p5/M, z10.s, z28.s\n" + "fmax z11.s, p5/M, z11.s, z28.s\n" + "fmax z12.s, p5/M, z12.s, z28.s\n" + "fmax z13.s, p5/M, z13.s, z28.s\n" + "fmax z14.s, p5/M, z14.s, z28.s\n" + "fmax z15.s, p5/M, z15.s, z28.s\n" + "fmax z16.s, p5/M, z16.s, z28.s\n" + "fmax z17.s, p5/M, z17.s, z28.s\n" + "fmax z18.s, p5/M, z18.s, z28.s\n" + "fmax z19.s, p5/M, z19.s, z28.s\n" + "fmax z20.s, p5/M, z20.s, z28.s\n" + "fmax z21.s, p5/M, z21.s, z28.s\n" + "fmax z22.s, p5/M, z22.s, z28.s\n" + "fmax z23.s, p5/M, z23.s, z28.s\n" + "fmax z24.s, p5/M, z24.s, z28.s\n" + "fmax z25.s, p5/M, z25.s, z28.s\n" + "fmax z26.s, p5/M, z26.s, z28.s\n" + "fmax z27.s, p5/M, z27.s, z28.s\n" "64:" // Height 5: No activation "st1w { z8.s }, p4, [x9]\n" "st1w { z9.s }, p3, [x9, #1, MUL VL]\n" @@ -1626,35 +1626,35 @@ void sve_hybrid_fp32_mla_6x4VL ( "68:" // Height 6: no bias "tbz %x[flags], #0, 69f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z8.s }, p4/Z, [x9]\n" + "add x24, x9, x20, LSL #2\n" "add x23, x24, x20, LSL #2\n" + "ld1w { z8.s }, p4/Z, [x9]\n" "add x22, x23, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n" - "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "ld1w { z16.s }, p4/Z, [x24]\n" - "ld1w { z17.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x24, #2, MUL VL]\n" - "ld1w { z19.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x23]\n" - "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n" - "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n" - "ld1w { z24.s }, p4/Z, [x22]\n" - "ld1w { z25.s }, p3/Z, [x22, #1, MUL VL]\n" - "ld1w { z26.s }, p2/Z, [x22, #2, MUL VL]\n" - "ld1w { z27.s }, p1/Z, [x22, #3, MUL VL]\n" - "ld1w { z28.s }, p4/Z, [x21]\n" - "ld1w { z29.s }, p3/Z, [x21, #1, MUL VL]\n" - "ld1w { z30.s }, p2/Z, [x21, #2, MUL VL]\n" - "ld1w { z31.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x24]\n" + "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n" + "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x23]\n" + "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n" + "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x22]\n" + "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n" + "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z24.s }, p4/Z, [x21]\n" + "ld1w { z25.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z26.s }, p2/Z, [x21, #2, MUL VL]\n" + "ld1w { z27.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z28.s }, p4/Z, [x20]\n" + "ld1w { z29.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z30.s }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z31.s }, p1/Z, [x20, #3, MUL VL]\n" "b 70f\n" "69:" // Height 6: no accumulate "mov z8.b, #0x0\n" @@ -1686,16 +1686,16 @@ void sve_hybrid_fp32_mla_6x4VL ( "71:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 72f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 73f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -1707,143 +1707,143 @@ void sve_hybrid_fp32_mla_6x4VL ( "b 73f\n" "72:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "add x21, x22, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" + "add x21, x22, x21, LSL #2\n" "73:" // Height 6: input setup done "cmp x27, #0x4\n" "ble 75f\n" "74:" // Height 6: Multiply loop: Main loop head "whilelt p0.s, XZR, x27\n" - "ld1rqw { z0.s }, p0/Z, [x26]\n" - "ld1rqw { z1.s }, p0/Z, [x25]\n" + "ld1rqw { z7.s }, p0/Z, [x26]\n" + "ld1rqw { z6.s }, p0/Z, [x25]\n" "sub x27, x27, #0x4\n" - "ld1rqw { z2.s }, p0/Z, [x24]\n" - "ld1rqw { z3.s }, p0/Z, [x23]\n" + "ld1rqw { z5.s }, p0/Z, [x24]\n" + "ld1rqw { z4.s }, p0/Z, [x23]\n" "cmp x27, #0x4\n" "add x26, x26, #0x10\n" - "ld1rqw { z4.s }, p0/Z, [x22]\n" - "ld1rqw { z5.s }, p0/Z, [x21]\n" + "ld1rqw { z3.s }, p0/Z, [x22]\n" + "ld1rqw { z2.s }, p0/Z, [x21]\n" "add x25, x25, #0x10\n" "add x24, x24, #0x10\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[0]\n" - "fmla z12.s, z6.s, z1.s[0]\n" - "fmla z16.s, z6.s, z2.s[0]\n" - "fmla z20.s, z6.s, z3.s[0]\n" + "ld1w { z1.s }, p5/Z, [x10]\n" + "ld1w { z0.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z1.s, z7.s[0]\n" + "fmla z12.s, z1.s, z6.s[0]\n" + "fmla z16.s, z1.s, z5.s[0]\n" + "fmla z20.s, z1.s, z4.s[0]\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - "fmla z24.s, z6.s, z4.s[0]\n" - "fmla z28.s, z6.s, z5.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z24.s, z1.s, z3.s[0]\n" + "fmla z28.s, z1.s, z2.s[0]\n" + "ld1w { z1.s }, p5/Z, [x10, #2, MUL VL]\n" "add x21, x21, #0x10\n" - "fmla z9.s, z7.s, z0.s[0]\n" - "fmla z13.s, z7.s, z1.s[0]\n" - "fmla z17.s, z7.s, z2.s[0]\n" - "fmla z21.s, z7.s, z3.s[0]\n" - "fmla z25.s, z7.s, z4.s[0]\n" - "fmla z29.s, z7.s, z5.s[0]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[0]\n" - "fmla z14.s, z6.s, z1.s[0]\n" - "fmla z18.s, z6.s, z2.s[0]\n" - "fmla z22.s, z6.s, z3.s[0]\n" - "fmla z26.s, z6.s, z4.s[0]\n" - "fmla z30.s, z6.s, z5.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10, #4, MUL VL]\n" - "fmla z11.s, z7.s, z0.s[0]\n" - "fmla z15.s, z7.s, z1.s[0]\n" - "fmla z19.s, z7.s, z2.s[0]\n" - "fmla z23.s, z7.s, z3.s[0]\n" - "fmla z27.s, z7.s, z4.s[0]\n" - "fmla z31.s, z7.s, z5.s[0]\n" - "ld1w { z7.s }, p5/Z, [x10, #5, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[1]\n" - "fmla z12.s, z6.s, z1.s[1]\n" - "fmla z16.s, z6.s, z2.s[1]\n" - "fmla z20.s, z6.s, z3.s[1]\n" - "fmla z24.s, z6.s, z4.s[1]\n" - "fmla z28.s, z6.s, z5.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10, #6, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[1]\n" - "fmla z13.s, z7.s, z1.s[1]\n" - "fmla z17.s, z7.s, z2.s[1]\n" - "fmla z21.s, z7.s, z3.s[1]\n" - "fmla z25.s, z7.s, z4.s[1]\n" - "fmla z29.s, z7.s, z5.s[1]\n" - "ld1w { z7.s }, p5/Z, [x10, #7, MUL VL]\n" + "fmla z9.s, z0.s, z7.s[0]\n" + "fmla z13.s, z0.s, z6.s[0]\n" + "fmla z17.s, z0.s, z5.s[0]\n" + "fmla z21.s, z0.s, z4.s[0]\n" + "fmla z25.s, z0.s, z3.s[0]\n" + "fmla z29.s, z0.s, z2.s[0]\n" + "ld1w { z0.s }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z10.s, z1.s, z7.s[0]\n" + "fmla z14.s, z1.s, z6.s[0]\n" + "fmla z18.s, z1.s, z5.s[0]\n" + "fmla z22.s, z1.s, z4.s[0]\n" + "fmla z26.s, z1.s, z3.s[0]\n" + "fmla z30.s, z1.s, z2.s[0]\n" + "ld1w { z1.s }, p5/Z, [x10, #4, MUL VL]\n" + "fmla z11.s, z0.s, z7.s[0]\n" + "fmla z15.s, z0.s, z6.s[0]\n" + "fmla z19.s, z0.s, z5.s[0]\n" + "fmla z23.s, z0.s, z4.s[0]\n" + "fmla z27.s, z0.s, z3.s[0]\n" + "fmla z31.s, z0.s, z2.s[0]\n" + "ld1w { z0.s }, p5/Z, [x10, #5, MUL VL]\n" + "fmla z8.s, z1.s, z7.s[1]\n" + "fmla z12.s, z1.s, z6.s[1]\n" + "fmla z16.s, z1.s, z5.s[1]\n" + "fmla z20.s, z1.s, z4.s[1]\n" + "fmla z24.s, z1.s, z3.s[1]\n" + "fmla z28.s, z1.s, z2.s[1]\n" + "ld1w { z1.s }, p5/Z, [x10, #6, MUL VL]\n" + "fmla z9.s, z0.s, z7.s[1]\n" + "fmla z13.s, z0.s, z6.s[1]\n" + "fmla z17.s, z0.s, z5.s[1]\n" + "fmla z21.s, z0.s, z4.s[1]\n" + "fmla z25.s, z0.s, z3.s[1]\n" + "fmla z29.s, z0.s, z2.s[1]\n" + "ld1w { z0.s }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "fmla z10.s, z6.s, z0.s[1]\n" - "fmla z14.s, z6.s, z1.s[1]\n" - "fmla z18.s, z6.s, z2.s[1]\n" - "fmla z22.s, z6.s, z3.s[1]\n" - "fmla z26.s, z6.s, z4.s[1]\n" - "fmla z30.s, z6.s, z5.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10, #-8, MUL VL]\n" - "fmla z11.s, z7.s, z0.s[1]\n" - "fmla z15.s, z7.s, z1.s[1]\n" - "fmla z19.s, z7.s, z2.s[1]\n" - "fmla z23.s, z7.s, z3.s[1]\n" - "fmla z27.s, z7.s, z4.s[1]\n" - "fmla z31.s, z7.s, z5.s[1]\n" - "ld1w { z7.s }, p5/Z, [x10, #-7, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[2]\n" - "fmla z12.s, z6.s, z1.s[2]\n" - "fmla z16.s, z6.s, z2.s[2]\n" - "fmla z20.s, z6.s, z3.s[2]\n" - "fmla z24.s, z6.s, z4.s[2]\n" - "fmla z28.s, z6.s, z5.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10, #-6, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[2]\n" - "fmla z13.s, z7.s, z1.s[2]\n" - "fmla z17.s, z7.s, z2.s[2]\n" - "fmla z21.s, z7.s, z3.s[2]\n" - "fmla z25.s, z7.s, z4.s[2]\n" - "fmla z29.s, z7.s, z5.s[2]\n" - "ld1w { z7.s }, p5/Z, [x10, #-5, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[2]\n" - "fmla z14.s, z6.s, z1.s[2]\n" - "fmla z18.s, z6.s, z2.s[2]\n" - "fmla z22.s, z6.s, z3.s[2]\n" - "fmla z26.s, z6.s, z4.s[2]\n" - "fmla z30.s, z6.s, z5.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10, #-4, MUL VL]\n" - "fmla z11.s, z7.s, z0.s[2]\n" - "fmla z15.s, z7.s, z1.s[2]\n" - "fmla z19.s, z7.s, z2.s[2]\n" - "fmla z23.s, z7.s, z3.s[2]\n" - "fmla z27.s, z7.s, z4.s[2]\n" - "fmla z31.s, z7.s, z5.s[2]\n" - "ld1w { z7.s }, p5/Z, [x10, #-3, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[3]\n" - "fmla z12.s, z6.s, z1.s[3]\n" - "fmla z16.s, z6.s, z2.s[3]\n" - "fmla z20.s, z6.s, z3.s[3]\n" - "fmla z24.s, z6.s, z4.s[3]\n" - "fmla z28.s, z6.s, z5.s[3]\n" - "ld1w { z6.s }, p5/Z, [x10, #-2, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[3]\n" - "fmla z13.s, z7.s, z1.s[3]\n" - "fmla z17.s, z7.s, z2.s[3]\n" - "fmla z21.s, z7.s, z3.s[3]\n" - "fmla z25.s, z7.s, z4.s[3]\n" - "fmla z29.s, z7.s, z5.s[3]\n" - "ld1w { z7.s }, p5/Z, [x10, #-1, MUL VL]\n" - "fmla z10.s, z6.s, z0.s[3]\n" - "fmla z14.s, z6.s, z1.s[3]\n" - "fmla z18.s, z6.s, z2.s[3]\n" - "fmla z22.s, z6.s, z3.s[3]\n" - "fmla z26.s, z6.s, z4.s[3]\n" - "fmla z30.s, z6.s, z5.s[3]\n" - "fmla z11.s, z7.s, z0.s[3]\n" - "fmla z15.s, z7.s, z1.s[3]\n" - "fmla z19.s, z7.s, z2.s[3]\n" - "fmla z23.s, z7.s, z3.s[3]\n" - "fmla z27.s, z7.s, z4.s[3]\n" - "fmla z31.s, z7.s, z5.s[3]\n" + "fmla z10.s, z1.s, z7.s[1]\n" + "fmla z14.s, z1.s, z6.s[1]\n" + "fmla z18.s, z1.s, z5.s[1]\n" + "fmla z22.s, z1.s, z4.s[1]\n" + "fmla z26.s, z1.s, z3.s[1]\n" + "fmla z30.s, z1.s, z2.s[1]\n" + "ld1w { z1.s }, p5/Z, [x10, #-8, MUL VL]\n" + "fmla z11.s, z0.s, z7.s[1]\n" + "fmla z15.s, z0.s, z6.s[1]\n" + "fmla z19.s, z0.s, z5.s[1]\n" + "fmla z23.s, z0.s, z4.s[1]\n" + "fmla z27.s, z0.s, z3.s[1]\n" + "fmla z31.s, z0.s, z2.s[1]\n" + "ld1w { z0.s }, p5/Z, [x10, #-7, MUL VL]\n" + "fmla z8.s, z1.s, z7.s[2]\n" + "fmla z12.s, z1.s, z6.s[2]\n" + "fmla z16.s, z1.s, z5.s[2]\n" + "fmla z20.s, z1.s, z4.s[2]\n" + "fmla z24.s, z1.s, z3.s[2]\n" + "fmla z28.s, z1.s, z2.s[2]\n" + "ld1w { z1.s }, p5/Z, [x10, #-6, MUL VL]\n" + "fmla z9.s, z0.s, z7.s[2]\n" + "fmla z13.s, z0.s, z6.s[2]\n" + "fmla z17.s, z0.s, z5.s[2]\n" + "fmla z21.s, z0.s, z4.s[2]\n" + "fmla z25.s, z0.s, z3.s[2]\n" + "fmla z29.s, z0.s, z2.s[2]\n" + "ld1w { z0.s }, p5/Z, [x10, #-5, MUL VL]\n" + "fmla z10.s, z1.s, z7.s[2]\n" + "fmla z14.s, z1.s, z6.s[2]\n" + "fmla z18.s, z1.s, z5.s[2]\n" + "fmla z22.s, z1.s, z4.s[2]\n" + "fmla z26.s, z1.s, z3.s[2]\n" + "fmla z30.s, z1.s, z2.s[2]\n" + "ld1w { z1.s }, p5/Z, [x10, #-4, MUL VL]\n" + "fmla z11.s, z0.s, z7.s[2]\n" + "fmla z15.s, z0.s, z6.s[2]\n" + "fmla z19.s, z0.s, z5.s[2]\n" + "fmla z23.s, z0.s, z4.s[2]\n" + "fmla z27.s, z0.s, z3.s[2]\n" + "fmla z31.s, z0.s, z2.s[2]\n" + "ld1w { z0.s }, p5/Z, [x10, #-3, MUL VL]\n" + "fmla z8.s, z1.s, z7.s[3]\n" + "fmla z12.s, z1.s, z6.s[3]\n" + "fmla z16.s, z1.s, z5.s[3]\n" + "fmla z20.s, z1.s, z4.s[3]\n" + "fmla z24.s, z1.s, z3.s[3]\n" + "fmla z28.s, z1.s, z2.s[3]\n" + "ld1w { z1.s }, p5/Z, [x10, #-2, MUL VL]\n" + "fmla z9.s, z0.s, z7.s[3]\n" + "fmla z13.s, z0.s, z6.s[3]\n" + "fmla z17.s, z0.s, z5.s[3]\n" + "fmla z21.s, z0.s, z4.s[3]\n" + "fmla z25.s, z0.s, z3.s[3]\n" + "fmla z29.s, z0.s, z2.s[3]\n" + "ld1w { z0.s }, p5/Z, [x10, #-1, MUL VL]\n" + "fmla z10.s, z1.s, z7.s[3]\n" + "fmla z14.s, z1.s, z6.s[3]\n" + "fmla z18.s, z1.s, z5.s[3]\n" + "fmla z22.s, z1.s, z4.s[3]\n" + "fmla z26.s, z1.s, z3.s[3]\n" + "fmla z30.s, z1.s, z2.s[3]\n" + "fmla z11.s, z0.s, z7.s[3]\n" + "fmla z15.s, z0.s, z6.s[3]\n" + "fmla z19.s, z0.s, z5.s[3]\n" + "fmla z23.s, z0.s, z4.s[3]\n" + "fmla z27.s, z0.s, z3.s[3]\n" + "fmla z31.s, z0.s, z2.s[3]\n" "bgt 74b\n" "75:" // Height 6: Multiply loop: Single iteration only "whilelt p0.s, XZR, x27\n" @@ -1854,127 +1854,127 @@ void sve_hybrid_fp32_mla_6x4VL ( "ld1rqw { z3.s }, p0/Z, [x23]\n" "ld1rqw { z4.s }, p0/Z, [x22]\n" "ld1rqw { z5.s }, p0/Z, [x21]\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[0]\n" - "fmla z12.s, z6.s, z1.s[0]\n" - "fmla z16.s, z6.s, z2.s[0]\n" - "fmla z20.s, z6.s, z3.s[0]\n" - "fmla z24.s, z6.s, z4.s[0]\n" - "fmla z28.s, z6.s, z5.s[0]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[0]\n" - "fmla z13.s, z7.s, z1.s[0]\n" - "fmla z17.s, z7.s, z2.s[0]\n" - "fmla z21.s, z7.s, z3.s[0]\n" - "fmla z25.s, z7.s, z4.s[0]\n" - "fmla z29.s, z7.s, z5.s[0]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" + "ld1w { z7.s }, p5/Z, [x10]\n" + "ld1w { z6.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z7.s, z0.s[0]\n" + "fmla z12.s, z7.s, z1.s[0]\n" + "fmla z16.s, z7.s, z2.s[0]\n" + "fmla z20.s, z7.s, z3.s[0]\n" + "fmla z24.s, z7.s, z4.s[0]\n" + "fmla z28.s, z7.s, z5.s[0]\n" + "ld1w { z7.s }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.s, z6.s, z0.s[0]\n" + "fmla z13.s, z6.s, z1.s[0]\n" + "fmla z17.s, z6.s, z2.s[0]\n" + "fmla z21.s, z6.s, z3.s[0]\n" + "fmla z25.s, z6.s, z4.s[0]\n" + "fmla z29.s, z6.s, z5.s[0]\n" + "ld1w { z6.s }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.s, z6.s, z0.s[0]\n" - "fmla z14.s, z6.s, z1.s[0]\n" - "fmla z18.s, z6.s, z2.s[0]\n" - "fmla z22.s, z6.s, z3.s[0]\n" - "fmla z26.s, z6.s, z4.s[0]\n" - "fmla z30.s, z6.s, z5.s[0]\n" - "fmla z11.s, z7.s, z0.s[0]\n" - "fmla z15.s, z7.s, z1.s[0]\n" - "fmla z19.s, z7.s, z2.s[0]\n" - "fmla z23.s, z7.s, z3.s[0]\n" - "fmla z27.s, z7.s, z4.s[0]\n" - "fmla z31.s, z7.s, z5.s[0]\n" + "fmla z10.s, z7.s, z0.s[0]\n" + "fmla z14.s, z7.s, z1.s[0]\n" + "fmla z18.s, z7.s, z2.s[0]\n" + "fmla z22.s, z7.s, z3.s[0]\n" + "fmla z26.s, z7.s, z4.s[0]\n" + "fmla z30.s, z7.s, z5.s[0]\n" + "fmla z11.s, z6.s, z0.s[0]\n" + "fmla z15.s, z6.s, z1.s[0]\n" + "fmla z19.s, z6.s, z2.s[0]\n" + "fmla z23.s, z6.s, z3.s[0]\n" + "fmla z27.s, z6.s, z4.s[0]\n" + "fmla z31.s, z6.s, z5.s[0]\n" "ble 76f\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[1]\n" - "fmla z12.s, z6.s, z1.s[1]\n" - "fmla z16.s, z6.s, z2.s[1]\n" - "fmla z20.s, z6.s, z3.s[1]\n" + "ld1w { z7.s }, p5/Z, [x10]\n" + "ld1w { z6.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z7.s, z0.s[1]\n" + "fmla z12.s, z7.s, z1.s[1]\n" + "fmla z16.s, z7.s, z2.s[1]\n" + "fmla z20.s, z7.s, z3.s[1]\n" "subs x27, x27, #0x1\n" - "fmla z24.s, z6.s, z4.s[1]\n" - "fmla z28.s, z6.s, z5.s[1]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[1]\n" - "fmla z13.s, z7.s, z1.s[1]\n" - "fmla z17.s, z7.s, z2.s[1]\n" - "fmla z21.s, z7.s, z3.s[1]\n" - "fmla z25.s, z7.s, z4.s[1]\n" - "fmla z29.s, z7.s, z5.s[1]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z24.s, z7.s, z4.s[1]\n" + "fmla z28.s, z7.s, z5.s[1]\n" + "ld1w { z7.s }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.s, z6.s, z0.s[1]\n" + "fmla z13.s, z6.s, z1.s[1]\n" + "fmla z17.s, z6.s, z2.s[1]\n" + "fmla z21.s, z6.s, z3.s[1]\n" + "fmla z25.s, z6.s, z4.s[1]\n" + "fmla z29.s, z6.s, z5.s[1]\n" + "ld1w { z6.s }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.s, z6.s, z0.s[1]\n" - "fmla z14.s, z6.s, z1.s[1]\n" - "fmla z18.s, z6.s, z2.s[1]\n" - "fmla z22.s, z6.s, z3.s[1]\n" - "fmla z26.s, z6.s, z4.s[1]\n" - "fmla z30.s, z6.s, z5.s[1]\n" - "fmla z11.s, z7.s, z0.s[1]\n" - "fmla z15.s, z7.s, z1.s[1]\n" - "fmla z19.s, z7.s, z2.s[1]\n" - "fmla z23.s, z7.s, z3.s[1]\n" - "fmla z27.s, z7.s, z4.s[1]\n" - "fmla z31.s, z7.s, z5.s[1]\n" + "fmla z10.s, z7.s, z0.s[1]\n" + "fmla z14.s, z7.s, z1.s[1]\n" + "fmla z18.s, z7.s, z2.s[1]\n" + "fmla z22.s, z7.s, z3.s[1]\n" + "fmla z26.s, z7.s, z4.s[1]\n" + "fmla z30.s, z7.s, z5.s[1]\n" + "fmla z11.s, z6.s, z0.s[1]\n" + "fmla z15.s, z6.s, z1.s[1]\n" + "fmla z19.s, z6.s, z2.s[1]\n" + "fmla z23.s, z6.s, z3.s[1]\n" + "fmla z27.s, z6.s, z4.s[1]\n" + "fmla z31.s, z6.s, z5.s[1]\n" "ble 76f\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[2]\n" - "fmla z12.s, z6.s, z1.s[2]\n" - "fmla z16.s, z6.s, z2.s[2]\n" - "fmla z20.s, z6.s, z3.s[2]\n" + "ld1w { z7.s }, p5/Z, [x10]\n" + "ld1w { z6.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z7.s, z0.s[2]\n" + "fmla z12.s, z7.s, z1.s[2]\n" + "fmla z16.s, z7.s, z2.s[2]\n" + "fmla z20.s, z7.s, z3.s[2]\n" "subs x27, x27, #0x1\n" - "fmla z24.s, z6.s, z4.s[2]\n" - "fmla z28.s, z6.s, z5.s[2]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[2]\n" - "fmla z13.s, z7.s, z1.s[2]\n" - "fmla z17.s, z7.s, z2.s[2]\n" - "fmla z21.s, z7.s, z3.s[2]\n" - "fmla z25.s, z7.s, z4.s[2]\n" - "fmla z29.s, z7.s, z5.s[2]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" + "fmla z24.s, z7.s, z4.s[2]\n" + "fmla z28.s, z7.s, z5.s[2]\n" + "ld1w { z7.s }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.s, z6.s, z0.s[2]\n" + "fmla z13.s, z6.s, z1.s[2]\n" + "fmla z17.s, z6.s, z2.s[2]\n" + "fmla z21.s, z6.s, z3.s[2]\n" + "fmla z25.s, z6.s, z4.s[2]\n" + "fmla z29.s, z6.s, z5.s[2]\n" + "ld1w { z6.s }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.s, z6.s, z0.s[2]\n" - "fmla z14.s, z6.s, z1.s[2]\n" - "fmla z18.s, z6.s, z2.s[2]\n" - "fmla z22.s, z6.s, z3.s[2]\n" - "fmla z26.s, z6.s, z4.s[2]\n" - "fmla z30.s, z6.s, z5.s[2]\n" - "fmla z11.s, z7.s, z0.s[2]\n" - "fmla z15.s, z7.s, z1.s[2]\n" - "fmla z19.s, z7.s, z2.s[2]\n" - "fmla z23.s, z7.s, z3.s[2]\n" - "fmla z27.s, z7.s, z4.s[2]\n" - "fmla z31.s, z7.s, z5.s[2]\n" + "fmla z10.s, z7.s, z0.s[2]\n" + "fmla z14.s, z7.s, z1.s[2]\n" + "fmla z18.s, z7.s, z2.s[2]\n" + "fmla z22.s, z7.s, z3.s[2]\n" + "fmla z26.s, z7.s, z4.s[2]\n" + "fmla z30.s, z7.s, z5.s[2]\n" + "fmla z11.s, z6.s, z0.s[2]\n" + "fmla z15.s, z6.s, z1.s[2]\n" + "fmla z19.s, z6.s, z2.s[2]\n" + "fmla z23.s, z6.s, z3.s[2]\n" + "fmla z27.s, z6.s, z4.s[2]\n" + "fmla z31.s, z6.s, z5.s[2]\n" "ble 76f\n" - "ld1w { z6.s }, p5/Z, [x10]\n" - "ld1w { z7.s }, p5/Z, [x10, #1, MUL VL]\n" - "fmla z8.s, z6.s, z0.s[3]\n" - "fmla z12.s, z6.s, z1.s[3]\n" - "fmla z16.s, z6.s, z2.s[3]\n" - "fmla z20.s, z6.s, z3.s[3]\n" - "fmla z24.s, z6.s, z4.s[3]\n" - "fmla z28.s, z6.s, z5.s[3]\n" - "ld1w { z6.s }, p5/Z, [x10, #2, MUL VL]\n" - "fmla z9.s, z7.s, z0.s[3]\n" - "fmla z13.s, z7.s, z1.s[3]\n" - "fmla z17.s, z7.s, z2.s[3]\n" - "fmla z21.s, z7.s, z3.s[3]\n" - "fmla z25.s, z7.s, z4.s[3]\n" - "fmla z29.s, z7.s, z5.s[3]\n" - "ld1w { z7.s }, p5/Z, [x10, #3, MUL VL]\n" + "ld1w { z7.s }, p5/Z, [x10]\n" + "ld1w { z6.s }, p5/Z, [x10, #1, MUL VL]\n" + "fmla z8.s, z7.s, z0.s[3]\n" + "fmla z12.s, z7.s, z1.s[3]\n" + "fmla z16.s, z7.s, z2.s[3]\n" + "fmla z20.s, z7.s, z3.s[3]\n" + "fmla z24.s, z7.s, z4.s[3]\n" + "fmla z28.s, z7.s, z5.s[3]\n" + "ld1w { z7.s }, p5/Z, [x10, #2, MUL VL]\n" + "fmla z9.s, z6.s, z0.s[3]\n" + "fmla z13.s, z6.s, z1.s[3]\n" + "fmla z17.s, z6.s, z2.s[3]\n" + "fmla z21.s, z6.s, z3.s[3]\n" + "fmla z25.s, z6.s, z4.s[3]\n" + "fmla z29.s, z6.s, z5.s[3]\n" + "ld1w { z6.s }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "fmla z10.s, z6.s, z0.s[3]\n" - "fmla z14.s, z6.s, z1.s[3]\n" - "fmla z18.s, z6.s, z2.s[3]\n" - "fmla z22.s, z6.s, z3.s[3]\n" - "fmla z26.s, z6.s, z4.s[3]\n" - "fmla z30.s, z6.s, z5.s[3]\n" - "fmla z11.s, z7.s, z0.s[3]\n" - "fmla z15.s, z7.s, z1.s[3]\n" - "fmla z19.s, z7.s, z2.s[3]\n" - "fmla z23.s, z7.s, z3.s[3]\n" - "fmla z27.s, z7.s, z4.s[3]\n" - "fmla z31.s, z7.s, z5.s[3]\n" + "fmla z10.s, z7.s, z0.s[3]\n" + "fmla z14.s, z7.s, z1.s[3]\n" + "fmla z18.s, z7.s, z2.s[3]\n" + "fmla z22.s, z7.s, z3.s[3]\n" + "fmla z26.s, z7.s, z4.s[3]\n" + "fmla z30.s, z7.s, z5.s[3]\n" + "fmla z11.s, z6.s, z0.s[3]\n" + "fmla z15.s, z6.s, z1.s[3]\n" + "fmla z19.s, z6.s, z2.s[3]\n" + "fmla z23.s, z6.s, z3.s[3]\n" + "fmla z27.s, z6.s, z4.s[3]\n" + "fmla z31.s, z6.s, z5.s[3]\n" "76:" // Height 6: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -2081,7 +2081,6 @@ void sve_hybrid_fp32_mla_6x4VL ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "80:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -2089,4 +2088,4 @@ void sve_hybrid_fp32_mla_6x4VL ( } } // namespace arm_gemm -#endif // ARM_COMPUTE_ENABLE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL.hpp index c0718b1e7507486267961ab09ab916af24807dc6..a353c9d660515e42d42a847022766bd830a12094 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef ARM_COMPUTE_ENABLE_SVE + #include "../std_transforms_sve.hpp" #define ARGLIST \ @@ -89,5 +89,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL/a64fx.cpp index 2ccd050f1885d14c8a7f033033b282c13cc5269a..344341205bc180d40532a4e5f0df011e4eec12da 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL/a64fx.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL/a64fx.cpp @@ -127,11 +127,11 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "6:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w9, [x20, x10, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 7f\n" - "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x28, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x10, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x28, [x20, #0x0]\n" "cbnz x10, 8f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x28, x28, x20, LSL #2\n" @@ -143,19 +143,19 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "ld1rw { z0.s }, p1/Z, [x28]\n" "ble 10f\n" "9:" // Height 1: Multiply loop: Main loop - "ld1w { z8.s }, p1/Z, [x12]\n" + "ld1w { z16.s }, p1/Z, [x12]\n" "add x28, x28, #0x4\n" "subs x9, x9, #0x1\n" - "fmla z24.s, p1/M, z8.s, z0.s\n" + "fmla z24.s, p1/M, z16.s, z0.s\n" "addvl x12, x12, #1\n" "ld1rw { z0.s }, p1/Z, [x28]\n" "bgt 9b\n" "10:" // Height 1: Multiply loop: Main loop skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" - "ld1w { z9.s }, p1/Z, [x12]\n" + "ld1w { z16.s }, p1/Z, [x12]\n" "add x10, x10, #0x1\n" "cmp x10, x20\n" - "fmla z24.s, p1/M, z9.s, z0.s\n" + "fmla z24.s, p1/M, z16.s, z0.s\n" "addvl x12, x12, #1\n" "bne 6b\n" "tbz %x[flags], #1, 11f\n" @@ -189,9 +189,9 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "15:" // Height 2: no bias "tbz %x[flags], #0, 16f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x27, x11, x20, LSL #2\n" + "add x20, x11, x20, LSL #2\n" "ld1w { z24.s }, p0/Z, [x11]\n" - "ld1w { z25.s }, p0/Z, [x27]\n" + "ld1w { z25.s }, p0/Z, [x20]\n" "b 17f\n" "16:" // Height 2: no accumulate "mov z24.b, #0x0\n" @@ -201,12 +201,12 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "18:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w9, [x20, x10, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 19f\n" - "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x28, [x21, #0x0]\n" - "ldr x27, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x10, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x28, [x20, #0x0]\n" + "ldr x27, [x20, #0x8]\n" "cbnz x10, 20f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x28, x28, x20, LSL #2\n" @@ -214,30 +214,30 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "b 20f\n" "19:" // Height 2: setup direct input "mov x28, %x[input_ptr]\n" - "add x27, x28, x20, LSL #2\n" + "add x27, x28, x21, LSL #2\n" "20:" // Height 2: input setup done "subs x9, x9, #0x1\n" "ld1rw { z0.s }, p1/Z, [x28]\n" "ld1rw { z1.s }, p1/Z, [x27]\n" "ble 22f\n" "21:" // Height 2: Multiply loop: Main loop - "ld1w { z8.s }, p1/Z, [x12]\n" + "ld1w { z16.s }, p1/Z, [x12]\n" "add x28, x28, #0x4\n" "subs x9, x9, #0x1\n" - "fmla z24.s, p1/M, z8.s, z0.s\n" + "fmla z24.s, p1/M, z16.s, z0.s\n" "add x27, x27, #0x4\n" - "fmla z25.s, p1/M, z8.s, z1.s\n" + "fmla z25.s, p1/M, z16.s, z1.s\n" "addvl x12, x12, #1\n" "ld1rw { z0.s }, p1/Z, [x28]\n" "ld1rw { z1.s }, p1/Z, [x27]\n" "bgt 21b\n" "22:" // Height 2: Multiply loop: Main loop skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" - "ld1w { z9.s }, p1/Z, [x12]\n" + "ld1w { z16.s }, p1/Z, [x12]\n" "add x10, x10, #0x1\n" "cmp x10, x20\n" - "fmla z24.s, p1/M, z9.s, z0.s\n" - "fmla z25.s, p1/M, z9.s, z1.s\n" + "fmla z24.s, p1/M, z16.s, z0.s\n" + "fmla z25.s, p1/M, z16.s, z1.s\n" "addvl x12, x12, #1\n" "bne 18b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" @@ -277,11 +277,11 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "27:" // Height 3: no bias "tbz %x[flags], #0, 28f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x27, x11, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" + "add x21, x11, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z24.s }, p0/Z, [x11]\n" - "ld1w { z25.s }, p0/Z, [x27]\n" - "ld1w { z26.s }, p0/Z, [x26]\n" + "ld1w { z25.s }, p0/Z, [x21]\n" + "ld1w { z26.s }, p0/Z, [x20]\n" "b 29f\n" "28:" // Height 3: no accumulate "mov z24.b, #0x0\n" @@ -292,13 +292,13 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "30:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w9, [x20, x10, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 31f\n" - "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x28, [x21, #0x0]\n" - "ldr x27, [x21, #0x8]\n" - "ldr x26, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x10, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x28, [x20, #0x0]\n" + "ldr x27, [x20, #0x8]\n" + "ldr x26, [x20, #0x10]\n" "cbnz x10, 32f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x28, x28, x20, LSL #2\n" @@ -307,8 +307,8 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "b 32f\n" "31:" // Height 3: setup direct input "mov x28, %x[input_ptr]\n" - "add x27, x28, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" + "add x27, x28, x21, LSL #2\n" + "add x26, x27, x21, LSL #2\n" "32:" // Height 3: input setup done "subs x9, x9, #0x1\n" "ld1rw { z0.s }, p1/Z, [x28]\n" @@ -316,14 +316,14 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "ld1rw { z2.s }, p1/Z, [x26]\n" "ble 34f\n" "33:" // Height 3: Multiply loop: Main loop - "ld1w { z8.s }, p1/Z, [x12]\n" + "ld1w { z16.s }, p1/Z, [x12]\n" "add x28, x28, #0x4\n" "subs x9, x9, #0x1\n" - "fmla z24.s, p1/M, z8.s, z0.s\n" + "fmla z24.s, p1/M, z16.s, z0.s\n" "add x27, x27, #0x4\n" "add x26, x26, #0x4\n" - "fmla z25.s, p1/M, z8.s, z1.s\n" - "fmla z26.s, p1/M, z8.s, z2.s\n" + "fmla z25.s, p1/M, z16.s, z1.s\n" + "fmla z26.s, p1/M, z16.s, z2.s\n" "addvl x12, x12, #1\n" "ld1rw { z0.s }, p1/Z, [x28]\n" "ld1rw { z1.s }, p1/Z, [x27]\n" @@ -331,13 +331,13 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "bgt 33b\n" "34:" // Height 3: Multiply loop: Main loop skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" - "ld1w { z9.s }, p1/Z, [x12]\n" + "ld1w { z16.s }, p1/Z, [x12]\n" "add x10, x10, #0x1\n" "cmp x10, x20\n" - "fmla z24.s, p1/M, z9.s, z0.s\n" - "fmla z25.s, p1/M, z9.s, z1.s\n" + "fmla z24.s, p1/M, z16.s, z0.s\n" + "fmla z25.s, p1/M, z16.s, z1.s\n" "addvl x12, x12, #1\n" - "fmla z26.s, p1/M, z9.s, z2.s\n" + "fmla z26.s, p1/M, z16.s, z2.s\n" "bne 30b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "add x27, x11, x20, LSL #2\n" @@ -381,13 +381,13 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "39:" // Height 4: no bias "tbz %x[flags], #0, 40f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x27, x11, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" + "add x22, x11, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" "ld1w { z24.s }, p0/Z, [x11]\n" - "add x25, x26, x20, LSL #2\n" - "ld1w { z25.s }, p0/Z, [x27]\n" - "ld1w { z26.s }, p0/Z, [x26]\n" - "ld1w { z27.s }, p0/Z, [x25]\n" + "add x20, x21, x20, LSL #2\n" + "ld1w { z25.s }, p0/Z, [x22]\n" + "ld1w { z26.s }, p0/Z, [x21]\n" + "ld1w { z27.s }, p0/Z, [x20]\n" "b 41f\n" "40:" // Height 4: no accumulate "mov z24.b, #0x0\n" @@ -399,14 +399,14 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "42:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w9, [x20, x10, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 43f\n" - "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x28, [x21, #0x0]\n" - "ldr x27, [x21, #0x8]\n" - "ldr x26, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x10, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x28, [x20, #0x0]\n" + "ldr x27, [x20, #0x8]\n" + "ldr x26, [x20, #0x10]\n" + "ldr x25, [x20, #0x18]\n" "cbnz x10, 44f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x28, x28, x20, LSL #2\n" @@ -416,9 +416,9 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "b 44f\n" "43:" // Height 4: setup direct input "mov x28, %x[input_ptr]\n" - "add x27, x28, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" - "add x25, x26, x20, LSL #2\n" + "add x27, x28, x21, LSL #2\n" + "add x26, x27, x21, LSL #2\n" + "add x25, x26, x21, LSL #2\n" "44:" // Height 4: input setup done "subs x9, x9, #0x1\n" "ld1rw { z0.s }, p1/Z, [x28]\n" @@ -427,16 +427,16 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "ld1rw { z3.s }, p1/Z, [x25]\n" "ble 46f\n" "45:" // Height 4: Multiply loop: Main loop - "ld1w { z8.s }, p1/Z, [x12]\n" + "ld1w { z16.s }, p1/Z, [x12]\n" "add x28, x28, #0x4\n" "subs x9, x9, #0x1\n" - "fmla z24.s, p1/M, z8.s, z0.s\n" + "fmla z24.s, p1/M, z16.s, z0.s\n" "add x27, x27, #0x4\n" "add x26, x26, #0x4\n" - "fmla z25.s, p1/M, z8.s, z1.s\n" - "fmla z26.s, p1/M, z8.s, z2.s\n" + "fmla z25.s, p1/M, z16.s, z1.s\n" + "fmla z26.s, p1/M, z16.s, z2.s\n" "add x25, x25, #0x4\n" - "fmla z27.s, p1/M, z8.s, z3.s\n" + "fmla z27.s, p1/M, z16.s, z3.s\n" "addvl x12, x12, #1\n" "ld1rw { z0.s }, p1/Z, [x28]\n" "ld1rw { z1.s }, p1/Z, [x27]\n" @@ -445,14 +445,14 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "bgt 45b\n" "46:" // Height 4: Multiply loop: Main loop skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" - "ld1w { z9.s }, p1/Z, [x12]\n" + "ld1w { z16.s }, p1/Z, [x12]\n" "add x10, x10, #0x1\n" "cmp x10, x20\n" - "fmla z24.s, p1/M, z9.s, z0.s\n" - "fmla z25.s, p1/M, z9.s, z1.s\n" + "fmla z24.s, p1/M, z16.s, z0.s\n" + "fmla z25.s, p1/M, z16.s, z1.s\n" "addvl x12, x12, #1\n" - "fmla z26.s, p1/M, z9.s, z2.s\n" - "fmla z27.s, p1/M, z9.s, z3.s\n" + "fmla z26.s, p1/M, z16.s, z2.s\n" + "fmla z27.s, p1/M, z16.s, z3.s\n" "bne 42b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "add x27, x11, x20, LSL #2\n" @@ -501,15 +501,15 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "51:" // Height 5: no bias "tbz %x[flags], #0, 52f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x27, x11, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" + "add x23, x11, x20, LSL #2\n" + "add x22, x23, x20, LSL #2\n" "ld1w { z24.s }, p0/Z, [x11]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z25.s }, p0/Z, [x27]\n" - "ld1w { z26.s }, p0/Z, [x26]\n" - "ld1w { z27.s }, p0/Z, [x25]\n" - "ld1w { z28.s }, p0/Z, [x24]\n" + "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" + "ld1w { z25.s }, p0/Z, [x23]\n" + "ld1w { z26.s }, p0/Z, [x22]\n" + "ld1w { z27.s }, p0/Z, [x21]\n" + "ld1w { z28.s }, p0/Z, [x20]\n" "b 53f\n" "52:" // Height 5: no accumulate "mov z24.b, #0x0\n" @@ -522,15 +522,15 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "54:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w9, [x20, x10, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 55f\n" - "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x28, [x21, #0x0]\n" - "ldr x27, [x21, #0x8]\n" - "ldr x26, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" - "ldr x24, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x10, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x28, [x20, #0x0]\n" + "ldr x27, [x20, #0x8]\n" + "ldr x26, [x20, #0x10]\n" + "ldr x25, [x20, #0x18]\n" + "ldr x24, [x20, #0x20]\n" "cbnz x10, 56f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x28, x28, x20, LSL #2\n" @@ -541,10 +541,10 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "b 56f\n" "55:" // Height 5: setup direct input "mov x28, %x[input_ptr]\n" - "add x27, x28, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" + "add x27, x28, x21, LSL #2\n" + "add x26, x27, x21, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" "56:" // Height 5: input setup done "subs x9, x9, #0x1\n" "ld1rw { z0.s }, p1/Z, [x28]\n" @@ -554,20 +554,20 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "ld1rw { z4.s }, p1/Z, [x24]\n" "ble 58f\n" "57:" // Height 5: Multiply loop: Main loop - "ld1w { z8.s }, p1/Z, [x12]\n" + "ld1w { z16.s }, p1/Z, [x12]\n" "add x28, x28, #0x4\n" "subs x9, x9, #0x1\n" - "fmla z24.s, p1/M, z8.s, z0.s\n" + "fmla z24.s, p1/M, z16.s, z0.s\n" "add x27, x27, #0x4\n" "add x26, x26, #0x4\n" - "fmla z25.s, p1/M, z8.s, z1.s\n" - "fmla z26.s, p1/M, z8.s, z2.s\n" + "fmla z25.s, p1/M, z16.s, z1.s\n" + "fmla z26.s, p1/M, z16.s, z2.s\n" "add x25, x25, #0x4\n" "add x24, x24, #0x4\n" - "fmla z27.s, p1/M, z8.s, z3.s\n" + "fmla z27.s, p1/M, z16.s, z3.s\n" "ld1rw { z0.s }, p1/Z, [x28]\n" "addvl x12, x12, #1\n" - "fmla z28.s, p1/M, z8.s, z4.s\n" + "fmla z28.s, p1/M, z16.s, z4.s\n" "ld1rw { z1.s }, p1/Z, [x27]\n" "ld1rw { z2.s }, p1/Z, [x26]\n" "ld1rw { z3.s }, p1/Z, [x25]\n" @@ -575,15 +575,15 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "bgt 57b\n" "58:" // Height 5: Multiply loop: Main loop skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" - "ld1w { z9.s }, p1/Z, [x12]\n" + "ld1w { z16.s }, p1/Z, [x12]\n" "add x10, x10, #0x1\n" "cmp x10, x20\n" - "fmla z24.s, p1/M, z9.s, z0.s\n" - "fmla z25.s, p1/M, z9.s, z1.s\n" + "fmla z24.s, p1/M, z16.s, z0.s\n" + "fmla z25.s, p1/M, z16.s, z1.s\n" "addvl x12, x12, #1\n" - "fmla z26.s, p1/M, z9.s, z2.s\n" - "fmla z27.s, p1/M, z9.s, z3.s\n" - "fmla z28.s, p1/M, z9.s, z4.s\n" + "fmla z26.s, p1/M, z16.s, z2.s\n" + "fmla z27.s, p1/M, z16.s, z3.s\n" + "fmla z28.s, p1/M, z16.s, z4.s\n" "bne 54b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "add x27, x11, x20, LSL #2\n" @@ -636,18 +636,18 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "b 65f\n" "63:" // Height 6: no bias "tbz %x[flags], #0, 64f\n" - "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x27, x11, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" + "ldr x24, [%x[args_ptr], %[offsetof_output_offset]]\n" + "add x23, x11, x24, LSL #2\n" + "add x20, x23, x24, LSL #2\n" "ld1w { z24.s }, p0/Z, [x11]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z25.s }, p0/Z, [x27]\n" - "ld1w { z26.s }, p0/Z, [x26]\n" - "add x23, x24, x20, LSL #2\n" - "ld1w { z27.s }, p0/Z, [x25]\n" - "ld1w { z28.s }, p0/Z, [x24]\n" - "ld1w { z29.s }, p0/Z, [x23]\n" + "add x22, x20, x24, LSL #2\n" + "add x21, x22, x24, LSL #2\n" + "ld1w { z25.s }, p0/Z, [x23]\n" + "ld1w { z26.s }, p0/Z, [x20]\n" + "add x20, x21, x24, LSL #2\n" + "ld1w { z27.s }, p0/Z, [x22]\n" + "ld1w { z28.s }, p0/Z, [x21]\n" + "ld1w { z29.s }, p0/Z, [x20]\n" "b 65f\n" "64:" // Height 6: no accumulate "mov z24.b, #0x0\n" @@ -661,16 +661,16 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "66:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w9, [x20, x10, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 67f\n" - "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x28, [x21, #0x0]\n" - "ldr x27, [x21, #0x8]\n" - "ldr x26, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" - "ldr x24, [x21, #0x20]\n" - "ldr x23, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x10, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x28, [x20, #0x0]\n" + "ldr x27, [x20, #0x8]\n" + "ldr x26, [x20, #0x10]\n" + "ldr x25, [x20, #0x18]\n" + "ldr x24, [x20, #0x20]\n" + "ldr x23, [x20, #0x28]\n" "cbnz x10, 68f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x28, x28, x20, LSL #2\n" @@ -682,11 +682,11 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "b 68f\n" "67:" // Height 6: setup direct input "mov x28, %x[input_ptr]\n" - "add x27, x28, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x27, x28, x21, LSL #2\n" + "add x26, x27, x21, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" "68:" // Height 6: input setup done "subs x9, x9, #0x1\n" "ld1rw { z0.s }, p1/Z, [x28]\n" @@ -697,21 +697,21 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "ld1rw { z5.s }, p1/Z, [x23]\n" "ble 70f\n" "69:" // Height 6: Multiply loop: Main loop - "ld1w { z8.s }, p1/Z, [x12]\n" + "ld1w { z16.s }, p1/Z, [x12]\n" "add x28, x28, #0x4\n" "subs x9, x9, #0x1\n" - "fmla z24.s, p1/M, z8.s, z0.s\n" + "fmla z24.s, p1/M, z16.s, z0.s\n" "add x27, x27, #0x4\n" "add x26, x26, #0x4\n" - "fmla z25.s, p1/M, z8.s, z1.s\n" - "fmla z26.s, p1/M, z8.s, z2.s\n" + "fmla z25.s, p1/M, z16.s, z1.s\n" + "fmla z26.s, p1/M, z16.s, z2.s\n" "add x25, x25, #0x4\n" "add x24, x24, #0x4\n" - "fmla z27.s, p1/M, z8.s, z3.s\n" - "fmla z28.s, p1/M, z8.s, z4.s\n" + "fmla z27.s, p1/M, z16.s, z3.s\n" + "fmla z28.s, p1/M, z16.s, z4.s\n" "add x23, x23, #0x4\n" "addvl x12, x12, #1\n" - "fmla z29.s, p1/M, z8.s, z5.s\n" + "fmla z29.s, p1/M, z16.s, z5.s\n" "ld1rw { z0.s }, p1/Z, [x28]\n" "ld1rw { z1.s }, p1/Z, [x27]\n" "ld1rw { z2.s }, p1/Z, [x26]\n" @@ -721,16 +721,16 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "bgt 69b\n" "70:" // Height 6: Multiply loop: Main loop skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" - "ld1w { z9.s }, p1/Z, [x12]\n" + "ld1w { z16.s }, p1/Z, [x12]\n" "add x10, x10, #0x1\n" "cmp x10, x20\n" - "fmla z24.s, p1/M, z9.s, z0.s\n" - "fmla z25.s, p1/M, z9.s, z1.s\n" + "fmla z24.s, p1/M, z16.s, z0.s\n" + "fmla z25.s, p1/M, z16.s, z1.s\n" "addvl x12, x12, #1\n" - "fmla z26.s, p1/M, z9.s, z2.s\n" - "fmla z27.s, p1/M, z9.s, z3.s\n" - "fmla z28.s, p1/M, z9.s, z4.s\n" - "fmla z29.s, p1/M, z9.s, z5.s\n" + "fmla z26.s, p1/M, z16.s, z2.s\n" + "fmla z27.s, p1/M, z16.s, z3.s\n" + "fmla z28.s, p1/M, z16.s, z4.s\n" + "fmla z29.s, p1/M, z16.s, z5.s\n" "bne 66b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "add x27, x11, x20, LSL #2\n" @@ -788,20 +788,20 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "b 77f\n" "75:" // Height 7: no bias "tbz %x[flags], #0, 76f\n" - "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x27, x11, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" + "ldr x24, [%x[args_ptr], %[offsetof_output_offset]]\n" + "add x21, x11, x24, LSL #2\n" + "add x20, x21, x24, LSL #2\n" "ld1w { z24.s }, p0/Z, [x11]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z25.s }, p0/Z, [x27]\n" - "ld1w { z26.s }, p0/Z, [x26]\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "ld1w { z27.s }, p0/Z, [x25]\n" - "ld1w { z28.s }, p0/Z, [x24]\n" - "ld1w { z29.s }, p0/Z, [x23]\n" - "ld1w { z30.s }, p0/Z, [x22]\n" + "add x23, x20, x24, LSL #2\n" + "add x22, x23, x24, LSL #2\n" + "ld1w { z25.s }, p0/Z, [x21]\n" + "ld1w { z26.s }, p0/Z, [x20]\n" + "add x21, x22, x24, LSL #2\n" + "add x20, x21, x24, LSL #2\n" + "ld1w { z27.s }, p0/Z, [x23]\n" + "ld1w { z28.s }, p0/Z, [x22]\n" + "ld1w { z29.s }, p0/Z, [x21]\n" + "ld1w { z30.s }, p0/Z, [x20]\n" "b 77f\n" "76:" // Height 7: no accumulate "mov z24.b, #0x0\n" @@ -816,17 +816,17 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "78:" // Height 7: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w9, [x20, x10, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 79f\n" - "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x28, [x21, #0x0]\n" - "ldr x27, [x21, #0x8]\n" - "ldr x26, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" - "ldr x24, [x21, #0x20]\n" - "ldr x23, [x21, #0x28]\n" - "ldr x22, [x21, #0x30]\n" + "ldr x20, [%x[input_ptr], x10, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x28, [x20, #0x0]\n" + "ldr x27, [x20, #0x8]\n" + "ldr x26, [x20, #0x10]\n" + "ldr x25, [x20, #0x18]\n" + "ldr x24, [x20, #0x20]\n" + "ldr x23, [x20, #0x28]\n" + "ldr x22, [x20, #0x30]\n" "cbnz x10, 80f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x28, x28, x20, LSL #2\n" @@ -839,12 +839,12 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "b 80f\n" "79:" // Height 7: setup direct input "mov x28, %x[input_ptr]\n" - "add x27, x28, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" + "add x27, x28, x21, LSL #2\n" + "add x26, x27, x21, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" "80:" // Height 7: input setup done "subs x9, x9, #0x1\n" "ld1rw { z0.s }, p1/Z, [x28]\n" @@ -856,25 +856,25 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "ld1rw { z6.s }, p1/Z, [x22]\n" "ble 82f\n" "81:" // Height 7: Multiply loop: Main loop - "ld1w { z8.s }, p1/Z, [x12]\n" + "ld1w { z16.s }, p1/Z, [x12]\n" "add x28, x28, #0x4\n" "subs x9, x9, #0x1\n" - "fmla z24.s, p1/M, z8.s, z0.s\n" + "fmla z24.s, p1/M, z16.s, z0.s\n" "add x27, x27, #0x4\n" "add x26, x26, #0x4\n" - "fmla z25.s, p1/M, z8.s, z1.s\n" - "fmla z26.s, p1/M, z8.s, z2.s\n" + "fmla z25.s, p1/M, z16.s, z1.s\n" + "fmla z26.s, p1/M, z16.s, z2.s\n" "add x25, x25, #0x4\n" "add x24, x24, #0x4\n" - "fmla z27.s, p1/M, z8.s, z3.s\n" + "fmla z27.s, p1/M, z16.s, z3.s\n" "ld1rw { z0.s }, p1/Z, [x28]\n" "add x23, x23, #0x4\n" "add x22, x22, #0x4\n" - "fmla z28.s, p1/M, z8.s, z4.s\n" - "fmla z29.s, p1/M, z8.s, z5.s\n" + "fmla z28.s, p1/M, z16.s, z4.s\n" + "fmla z29.s, p1/M, z16.s, z5.s\n" "addvl x12, x12, #1\n" "ld1rw { z1.s }, p1/Z, [x27]\n" - "fmla z30.s, p1/M, z8.s, z6.s\n" + "fmla z30.s, p1/M, z16.s, z6.s\n" "ld1rw { z2.s }, p1/Z, [x26]\n" "ld1rw { z3.s }, p1/Z, [x25]\n" "ld1rw { z4.s }, p1/Z, [x24]\n" @@ -883,17 +883,17 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "bgt 81b\n" "82:" // Height 7: Multiply loop: Main loop skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" - "ld1w { z9.s }, p1/Z, [x12]\n" + "ld1w { z16.s }, p1/Z, [x12]\n" "add x10, x10, #0x1\n" "cmp x10, x20\n" - "fmla z24.s, p1/M, z9.s, z0.s\n" - "fmla z25.s, p1/M, z9.s, z1.s\n" + "fmla z24.s, p1/M, z16.s, z0.s\n" + "fmla z25.s, p1/M, z16.s, z1.s\n" "addvl x12, x12, #1\n" - "fmla z26.s, p1/M, z9.s, z2.s\n" - "fmla z27.s, p1/M, z9.s, z3.s\n" - "fmla z28.s, p1/M, z9.s, z4.s\n" - "fmla z29.s, p1/M, z9.s, z5.s\n" - "fmla z30.s, p1/M, z9.s, z6.s\n" + "fmla z26.s, p1/M, z16.s, z2.s\n" + "fmla z27.s, p1/M, z16.s, z3.s\n" + "fmla z28.s, p1/M, z16.s, z4.s\n" + "fmla z29.s, p1/M, z16.s, z5.s\n" + "fmla z30.s, p1/M, z16.s, z6.s\n" "bne 78b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "add x27, x11, x20, LSL #2\n" @@ -959,22 +959,22 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "b 89f\n" "87:" // Height 8: no bias "tbz %x[flags], #0, 88f\n" - "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x27, x11, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" + "ldr x24, [%x[args_ptr], %[offsetof_output_offset]]\n" + "add x22, x11, x24, LSL #2\n" + "add x21, x22, x24, LSL #2\n" "ld1w { z24.s }, p0/Z, [x11]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z25.s }, p0/Z, [x27]\n" - "ld1w { z26.s }, p0/Z, [x26]\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "ld1w { z27.s }, p0/Z, [x25]\n" - "ld1w { z28.s }, p0/Z, [x24]\n" - "add x21, x22, x20, LSL #2\n" - "ld1w { z29.s }, p0/Z, [x23]\n" - "ld1w { z30.s }, p0/Z, [x22]\n" - "ld1w { z31.s }, p0/Z, [x21]\n" + "add x23, x21, x24, LSL #2\n" + "add x20, x23, x24, LSL #2\n" + "ld1w { z25.s }, p0/Z, [x22]\n" + "ld1w { z26.s }, p0/Z, [x21]\n" + "add x22, x20, x24, LSL #2\n" + "add x21, x22, x24, LSL #2\n" + "ld1w { z27.s }, p0/Z, [x23]\n" + "ld1w { z28.s }, p0/Z, [x20]\n" + "add x20, x21, x24, LSL #2\n" + "ld1w { z29.s }, p0/Z, [x22]\n" + "ld1w { z30.s }, p0/Z, [x21]\n" + "ld1w { z31.s }, p0/Z, [x20]\n" "b 89f\n" "88:" // Height 8: no accumulate "mov z24.b, #0x0\n" @@ -990,18 +990,18 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "90:" // Height 8: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w9, [x20, x10, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 91f\n" - "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x28, [x21, #0x0]\n" - "ldr x27, [x21, #0x8]\n" - "ldr x26, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" - "ldr x24, [x21, #0x20]\n" - "ldr x23, [x21, #0x28]\n" - "ldr x22, [x21, #0x30]\n" - "ldr x21, [x21, #0x38]\n" + "ldr x20, [%x[input_ptr], x10, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x28, [x20, #0x0]\n" + "ldr x27, [x20, #0x8]\n" + "ldr x26, [x20, #0x10]\n" + "ldr x25, [x20, #0x18]\n" + "ldr x24, [x20, #0x20]\n" + "ldr x23, [x20, #0x28]\n" + "ldr x22, [x20, #0x30]\n" + "ldr x21, [x20, #0x38]\n" "cbnz x10, 92f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x28, x28, x20, LSL #2\n" @@ -1015,13 +1015,13 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "b 92f\n" "91:" // Height 8: setup direct input "mov x28, %x[input_ptr]\n" - "add x27, x28, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "add x21, x22, x20, LSL #2\n" + "add x27, x28, x21, LSL #2\n" + "add x26, x27, x21, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" + "add x21, x22, x21, LSL #2\n" "92:" // Height 8: input setup done "subs x9, x9, #0x1\n" "ld1rw { z0.s }, p1/Z, [x28]\n" @@ -1034,27 +1034,27 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "ld1rw { z7.s }, p1/Z, [x21]\n" "ble 94f\n" "93:" // Height 8: Multiply loop: Main loop - "ld1w { z8.s }, p1/Z, [x12]\n" + "ld1w { z16.s }, p1/Z, [x12]\n" "add x28, x28, #0x4\n" "subs x9, x9, #0x1\n" - "fmla z24.s, p1/M, z8.s, z0.s\n" + "fmla z24.s, p1/M, z16.s, z0.s\n" "add x27, x27, #0x4\n" "add x26, x26, #0x4\n" - "fmla z25.s, p1/M, z8.s, z1.s\n" - "fmla z26.s, p1/M, z8.s, z2.s\n" + "fmla z25.s, p1/M, z16.s, z1.s\n" + "fmla z26.s, p1/M, z16.s, z2.s\n" "add x25, x25, #0x4\n" "add x24, x24, #0x4\n" - "fmla z27.s, p1/M, z8.s, z3.s\n" - "fmla z28.s, p1/M, z8.s, z4.s\n" + "fmla z27.s, p1/M, z16.s, z3.s\n" + "fmla z28.s, p1/M, z16.s, z4.s\n" "add x23, x23, #0x4\n" "add x22, x22, #0x4\n" - "fmla z29.s, p1/M, z8.s, z5.s\n" + "fmla z29.s, p1/M, z16.s, z5.s\n" "ld1rw { z0.s }, p1/Z, [x28]\n" "add x21, x21, #0x4\n" "addvl x12, x12, #1\n" "ld1rw { z1.s }, p1/Z, [x27]\n" - "fmla z30.s, p1/M, z8.s, z6.s\n" - "fmla z31.s, p1/M, z8.s, z7.s\n" + "fmla z30.s, p1/M, z16.s, z6.s\n" + "fmla z31.s, p1/M, z16.s, z7.s\n" "ld1rw { z2.s }, p1/Z, [x26]\n" "ld1rw { z3.s }, p1/Z, [x25]\n" "ld1rw { z4.s }, p1/Z, [x24]\n" @@ -1064,18 +1064,18 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "bgt 93b\n" "94:" // Height 8: Multiply loop: Main loop skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" - "ld1w { z9.s }, p1/Z, [x12]\n" + "ld1w { z16.s }, p1/Z, [x12]\n" "add x10, x10, #0x1\n" "cmp x10, x20\n" - "fmla z24.s, p1/M, z9.s, z0.s\n" - "fmla z25.s, p1/M, z9.s, z1.s\n" + "fmla z24.s, p1/M, z16.s, z0.s\n" + "fmla z25.s, p1/M, z16.s, z1.s\n" "addvl x12, x12, #1\n" - "fmla z26.s, p1/M, z9.s, z2.s\n" - "fmla z27.s, p1/M, z9.s, z3.s\n" - "fmla z28.s, p1/M, z9.s, z4.s\n" - "fmla z29.s, p1/M, z9.s, z5.s\n" - "fmla z30.s, p1/M, z9.s, z6.s\n" - "fmla z31.s, p1/M, z9.s, z7.s\n" + "fmla z26.s, p1/M, z16.s, z2.s\n" + "fmla z27.s, p1/M, z16.s, z3.s\n" + "fmla z28.s, p1/M, z16.s, z4.s\n" + "fmla z29.s, p1/M, z16.s, z5.s\n" + "fmla z30.s, p1/M, z16.s, z6.s\n" + "fmla z31.s, p1/M, z16.s, z7.s\n" "bne 90b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "add x27, x11, x20, LSL #2\n" @@ -1132,12 +1132,11 @@ void sve_hybrid_fp32_mla_8x1VL_a64fx ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "98:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) - : "cc", "memory", "p0", "p1", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z16", "z17", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16", "z17", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace arm_gemm -#endif // ARM_COMPUTE_ENABLE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL/generic.cpp index 9679d49506716b7c42f33627983036703fc9ec8b..161c85e5f34db39eacf7733111dccabdf8b59727 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32_mla_8x1VL/generic.cpp @@ -127,11 +127,11 @@ void sve_hybrid_fp32_mla_8x1VL ( "6:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w9, [x20, x10, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 7f\n" - "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x28, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x10, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x28, [x20, #0x0]\n" "cbnz x10, 8f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x28, x28, x20, LSL #2\n" @@ -144,39 +144,39 @@ void sve_hybrid_fp32_mla_8x1VL ( "9:" // Height 1: Multiply loop: Main loop head "whilelt p0.s, XZR, x9\n" "ld1rqw { z0.s }, p0/Z, [x28]\n" - "ld1w { z8.s }, p2/Z, [x12]\n" - "fmla z24.s, z8.s, z0.s[0]\n" - "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n" - "fmla z24.s, z9.s, z0.s[1]\n" - "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n" - "fmla z24.s, z10.s, z0.s[2]\n" - "ld1w { z11.s }, p2/Z, [x12, #3, MUL VL]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" + "fmla z24.s, z16.s, z0.s[0]\n" + "ld1w { z16.s }, p2/Z, [x12, #1, MUL VL]\n" + "fmla z24.s, z16.s, z0.s[1]\n" + "ld1w { z16.s }, p2/Z, [x12, #2, MUL VL]\n" + "fmla z24.s, z16.s, z0.s[2]\n" + "ld1w { z16.s }, p2/Z, [x12, #3, MUL VL]\n" "sub x9, x9, #0x4\n" "cmp x9, #0x4\n" - "fmla z24.s, z11.s, z0.s[3]\n" + "fmla z24.s, z16.s, z0.s[3]\n" "add x28, x28, #0x10\n" "addvl x12, x12, #4\n" "bgt 9b\n" "10:" // Height 1: Multiply loop: Single iteration only "whilelt p0.s, XZR, x9\n" "ld1rqw { z0.s }, p0/Z, [x28]\n" - "ld1w { z8.s }, p2/Z, [x12]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" "subs x9, x9, #0x1\n" - "fmla z24.s, z8.s, z0.s[0]\n" + "fmla z24.s, z16.s, z0.s[0]\n" "addvl x12, x12, #1\n" "ble 11f\n" - "ld1w { z9.s }, p2/Z, [x12]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" "subs x9, x9, #0x1\n" - "fmla z24.s, z9.s, z0.s[1]\n" + "fmla z24.s, z16.s, z0.s[1]\n" "addvl x12, x12, #1\n" "ble 11f\n" - "ld1w { z10.s }, p2/Z, [x12]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" "subs x9, x9, #0x1\n" - "fmla z24.s, z10.s, z0.s[2]\n" + "fmla z24.s, z16.s, z0.s[2]\n" "addvl x12, x12, #1\n" "ble 11f\n" - "ld1w { z11.s }, p2/Z, [x12]\n" - "fmla z24.s, z11.s, z0.s[3]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" + "fmla z24.s, z16.s, z0.s[3]\n" "addvl x12, x12, #1\n" "11:" // Height 1: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -214,9 +214,9 @@ void sve_hybrid_fp32_mla_8x1VL ( "16:" // Height 2: no bias "tbz %x[flags], #0, 17f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x27, x11, x20, LSL #2\n" + "add x20, x11, x20, LSL #2\n" "ld1w { z24.s }, p1/Z, [x11]\n" - "ld1w { z25.s }, p1/Z, [x27]\n" + "ld1w { z25.s }, p1/Z, [x20]\n" "b 18f\n" "17:" // Height 2: no accumulate "mov z24.b, #0x0\n" @@ -226,12 +226,12 @@ void sve_hybrid_fp32_mla_8x1VL ( "19:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w9, [x20, x10, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 20f\n" - "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x28, [x21, #0x0]\n" - "ldr x27, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x10, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x28, [x20, #0x0]\n" + "ldr x27, [x20, #0x8]\n" "cbnz x10, 21f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x28, x28, x20, LSL #2\n" @@ -239,29 +239,29 @@ void sve_hybrid_fp32_mla_8x1VL ( "b 21f\n" "20:" // Height 2: setup direct input "mov x28, %x[input_ptr]\n" - "add x27, x28, x20, LSL #2\n" + "add x27, x28, x21, LSL #2\n" "21:" // Height 2: input setup done "cmp x9, #0x4\n" "ble 23f\n" "22:" // Height 2: Multiply loop: Main loop head "whilelt p0.s, XZR, x9\n" - "ld1rqw { z0.s }, p0/Z, [x28]\n" - "ld1rqw { z1.s }, p0/Z, [x27]\n" + "ld1rqw { z1.s }, p0/Z, [x28]\n" + "ld1rqw { z0.s }, p0/Z, [x27]\n" "sub x9, x9, #0x4\n" - "ld1w { z8.s }, p2/Z, [x12]\n" - "fmla z24.s, z8.s, z0.s[0]\n" - "fmla z25.s, z8.s, z1.s[0]\n" - "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n" - "fmla z24.s, z9.s, z0.s[1]\n" - "fmla z25.s, z9.s, z1.s[1]\n" - "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x12, #3, MUL VL]\n" - "fmla z24.s, z10.s, z0.s[2]\n" - "fmla z25.s, z10.s, z1.s[2]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" + "fmla z24.s, z16.s, z1.s[0]\n" + "fmla z25.s, z16.s, z0.s[0]\n" + "ld1w { z16.s }, p2/Z, [x12, #1, MUL VL]\n" + "fmla z24.s, z16.s, z1.s[1]\n" + "fmla z25.s, z16.s, z0.s[1]\n" + "ld1w { z17.s }, p2/Z, [x12, #2, MUL VL]\n" + "ld1w { z16.s }, p2/Z, [x12, #3, MUL VL]\n" + "fmla z24.s, z17.s, z1.s[2]\n" + "fmla z25.s, z17.s, z0.s[2]\n" "cmp x9, #0x4\n" "add x28, x28, #0x10\n" - "fmla z24.s, z11.s, z0.s[3]\n" - "fmla z25.s, z11.s, z1.s[3]\n" + "fmla z24.s, z16.s, z1.s[3]\n" + "fmla z25.s, z16.s, z0.s[3]\n" "add x27, x27, #0x10\n" "addvl x12, x12, #4\n" "bgt 22b\n" @@ -270,26 +270,26 @@ void sve_hybrid_fp32_mla_8x1VL ( "ld1rqw { z0.s }, p0/Z, [x28]\n" "ld1rqw { z1.s }, p0/Z, [x27]\n" "subs x9, x9, #0x1\n" - "ld1w { z8.s }, p2/Z, [x12]\n" - "fmla z24.s, z8.s, z0.s[0]\n" - "fmla z25.s, z8.s, z1.s[0]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" + "fmla z24.s, z16.s, z0.s[0]\n" + "fmla z25.s, z16.s, z1.s[0]\n" "addvl x12, x12, #1\n" "ble 24f\n" - "ld1w { z9.s }, p2/Z, [x12]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" "subs x9, x9, #0x1\n" - "fmla z24.s, z9.s, z0.s[1]\n" - "fmla z25.s, z9.s, z1.s[1]\n" + "fmla z24.s, z16.s, z0.s[1]\n" + "fmla z25.s, z16.s, z1.s[1]\n" "addvl x12, x12, #1\n" "ble 24f\n" - "ld1w { z10.s }, p2/Z, [x12]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" "subs x9, x9, #0x1\n" - "fmla z24.s, z10.s, z0.s[2]\n" - "fmla z25.s, z10.s, z1.s[2]\n" + "fmla z24.s, z16.s, z0.s[2]\n" + "fmla z25.s, z16.s, z1.s[2]\n" "addvl x12, x12, #1\n" "ble 24f\n" - "ld1w { z11.s }, p2/Z, [x12]\n" - "fmla z24.s, z11.s, z0.s[3]\n" - "fmla z25.s, z11.s, z1.s[3]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" + "fmla z24.s, z16.s, z0.s[3]\n" + "fmla z25.s, z16.s, z1.s[3]\n" "addvl x12, x12, #1\n" "24:" // Height 2: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -333,11 +333,11 @@ void sve_hybrid_fp32_mla_8x1VL ( "29:" // Height 3: no bias "tbz %x[flags], #0, 30f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x27, x11, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" + "add x21, x11, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z24.s }, p1/Z, [x11]\n" - "ld1w { z25.s }, p1/Z, [x27]\n" - "ld1w { z26.s }, p1/Z, [x26]\n" + "ld1w { z25.s }, p1/Z, [x21]\n" + "ld1w { z26.s }, p1/Z, [x20]\n" "b 31f\n" "30:" // Height 3: no accumulate "mov z24.b, #0x0\n" @@ -348,13 +348,13 @@ void sve_hybrid_fp32_mla_8x1VL ( "32:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w9, [x20, x10, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 33f\n" - "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x28, [x21, #0x0]\n" - "ldr x27, [x21, #0x8]\n" - "ldr x26, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x10, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x28, [x20, #0x0]\n" + "ldr x27, [x20, #0x8]\n" + "ldr x26, [x20, #0x10]\n" "cbnz x10, 34f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x28, x28, x20, LSL #2\n" @@ -363,38 +363,38 @@ void sve_hybrid_fp32_mla_8x1VL ( "b 34f\n" "33:" // Height 3: setup direct input "mov x28, %x[input_ptr]\n" - "add x27, x28, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" + "add x27, x28, x21, LSL #2\n" + "add x26, x27, x21, LSL #2\n" "34:" // Height 3: input setup done "cmp x9, #0x4\n" "ble 36f\n" "35:" // Height 3: Multiply loop: Main loop head "whilelt p0.s, XZR, x9\n" - "ld1rqw { z0.s }, p0/Z, [x28]\n" + "ld1rqw { z2.s }, p0/Z, [x28]\n" "ld1rqw { z1.s }, p0/Z, [x27]\n" "sub x9, x9, #0x4\n" - "ld1rqw { z2.s }, p0/Z, [x26]\n" - "ld1w { z8.s }, p2/Z, [x12]\n" - "fmla z24.s, z8.s, z0.s[0]\n" - "fmla z25.s, z8.s, z1.s[0]\n" - "fmla z26.s, z8.s, z2.s[0]\n" - "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n" - "fmla z24.s, z9.s, z0.s[1]\n" - "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n" - "fmla z25.s, z9.s, z1.s[1]\n" - "fmla z26.s, z9.s, z2.s[1]\n" - "ld1w { z11.s }, p2/Z, [x12, #3, MUL VL]\n" + "ld1rqw { z0.s }, p0/Z, [x26]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" + "fmla z24.s, z16.s, z2.s[0]\n" + "fmla z25.s, z16.s, z1.s[0]\n" + "fmla z26.s, z16.s, z0.s[0]\n" + "ld1w { z16.s }, p2/Z, [x12, #1, MUL VL]\n" + "fmla z24.s, z16.s, z2.s[1]\n" + "ld1w { z17.s }, p2/Z, [x12, #2, MUL VL]\n" + "fmla z25.s, z16.s, z1.s[1]\n" + "fmla z26.s, z16.s, z0.s[1]\n" + "ld1w { z16.s }, p2/Z, [x12, #3, MUL VL]\n" "cmp x9, #0x4\n" - "fmla z24.s, z10.s, z0.s[2]\n" - "fmla z25.s, z10.s, z1.s[2]\n" + "fmla z24.s, z17.s, z2.s[2]\n" + "fmla z25.s, z17.s, z1.s[2]\n" "add x28, x28, #0x10\n" "add x27, x27, #0x10\n" - "fmla z26.s, z10.s, z2.s[2]\n" - "fmla z24.s, z11.s, z0.s[3]\n" + "fmla z26.s, z17.s, z0.s[2]\n" + "fmla z24.s, z16.s, z2.s[3]\n" "add x26, x26, #0x10\n" "addvl x12, x12, #4\n" - "fmla z25.s, z11.s, z1.s[3]\n" - "fmla z26.s, z11.s, z2.s[3]\n" + "fmla z25.s, z16.s, z1.s[3]\n" + "fmla z26.s, z16.s, z0.s[3]\n" "bgt 35b\n" "36:" // Height 3: Multiply loop: Single iteration only "whilelt p0.s, XZR, x9\n" @@ -402,31 +402,31 @@ void sve_hybrid_fp32_mla_8x1VL ( "ld1rqw { z1.s }, p0/Z, [x27]\n" "subs x9, x9, #0x1\n" "ld1rqw { z2.s }, p0/Z, [x26]\n" - "ld1w { z8.s }, p2/Z, [x12]\n" - "fmla z24.s, z8.s, z0.s[0]\n" - "fmla z25.s, z8.s, z1.s[0]\n" - "fmla z26.s, z8.s, z2.s[0]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" + "fmla z24.s, z16.s, z0.s[0]\n" + "fmla z25.s, z16.s, z1.s[0]\n" + "fmla z26.s, z16.s, z2.s[0]\n" "addvl x12, x12, #1\n" "ble 37f\n" - "ld1w { z9.s }, p2/Z, [x12]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" "subs x9, x9, #0x1\n" - "fmla z24.s, z9.s, z0.s[1]\n" - "fmla z25.s, z9.s, z1.s[1]\n" - "fmla z26.s, z9.s, z2.s[1]\n" + "fmla z24.s, z16.s, z0.s[1]\n" + "fmla z25.s, z16.s, z1.s[1]\n" + "fmla z26.s, z16.s, z2.s[1]\n" "addvl x12, x12, #1\n" "ble 37f\n" - "ld1w { z10.s }, p2/Z, [x12]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" "subs x9, x9, #0x1\n" - "fmla z24.s, z10.s, z0.s[2]\n" - "fmla z25.s, z10.s, z1.s[2]\n" - "fmla z26.s, z10.s, z2.s[2]\n" + "fmla z24.s, z16.s, z0.s[2]\n" + "fmla z25.s, z16.s, z1.s[2]\n" + "fmla z26.s, z16.s, z2.s[2]\n" "addvl x12, x12, #1\n" "ble 37f\n" - "ld1w { z11.s }, p2/Z, [x12]\n" - "fmla z24.s, z11.s, z0.s[3]\n" - "fmla z25.s, z11.s, z1.s[3]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" + "fmla z24.s, z16.s, z0.s[3]\n" + "fmla z25.s, z16.s, z1.s[3]\n" "addvl x12, x12, #1\n" - "fmla z26.s, z11.s, z2.s[3]\n" + "fmla z26.s, z16.s, z2.s[3]\n" "37:" // Height 3: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x10, x10, #0x1\n" @@ -474,13 +474,13 @@ void sve_hybrid_fp32_mla_8x1VL ( "42:" // Height 4: no bias "tbz %x[flags], #0, 43f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x27, x11, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" + "add x22, x11, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" "ld1w { z24.s }, p1/Z, [x11]\n" - "add x25, x26, x20, LSL #2\n" - "ld1w { z25.s }, p1/Z, [x27]\n" - "ld1w { z26.s }, p1/Z, [x26]\n" - "ld1w { z27.s }, p1/Z, [x25]\n" + "add x20, x21, x20, LSL #2\n" + "ld1w { z25.s }, p1/Z, [x22]\n" + "ld1w { z26.s }, p1/Z, [x21]\n" + "ld1w { z27.s }, p1/Z, [x20]\n" "b 44f\n" "43:" // Height 4: no accumulate "mov z24.b, #0x0\n" @@ -492,14 +492,14 @@ void sve_hybrid_fp32_mla_8x1VL ( "45:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w9, [x20, x10, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 46f\n" - "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x28, [x21, #0x0]\n" - "ldr x27, [x21, #0x8]\n" - "ldr x26, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x10, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x28, [x20, #0x0]\n" + "ldr x27, [x20, #0x8]\n" + "ldr x26, [x20, #0x10]\n" + "ldr x25, [x20, #0x18]\n" "cbnz x10, 47f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x28, x28, x20, LSL #2\n" @@ -509,45 +509,45 @@ void sve_hybrid_fp32_mla_8x1VL ( "b 47f\n" "46:" // Height 4: setup direct input "mov x28, %x[input_ptr]\n" - "add x27, x28, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" - "add x25, x26, x20, LSL #2\n" + "add x27, x28, x21, LSL #2\n" + "add x26, x27, x21, LSL #2\n" + "add x25, x26, x21, LSL #2\n" "47:" // Height 4: input setup done "cmp x9, #0x4\n" "ble 49f\n" "48:" // Height 4: Multiply loop: Main loop head "whilelt p0.s, XZR, x9\n" - "ld1rqw { z0.s }, p0/Z, [x28]\n" - "ld1rqw { z1.s }, p0/Z, [x27]\n" + "ld1rqw { z3.s }, p0/Z, [x28]\n" + "ld1rqw { z2.s }, p0/Z, [x27]\n" "sub x9, x9, #0x4\n" - "ld1rqw { z2.s }, p0/Z, [x26]\n" - "ld1rqw { z3.s }, p0/Z, [x25]\n" + "ld1rqw { z1.s }, p0/Z, [x26]\n" + "ld1rqw { z0.s }, p0/Z, [x25]\n" "cmp x9, #0x4\n" "add x28, x28, #0x10\n" - "ld1w { z8.s }, p2/Z, [x12]\n" - "fmla z24.s, z8.s, z0.s[0]\n" - "fmla z25.s, z8.s, z1.s[0]\n" - "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n" - "fmla z26.s, z8.s, z2.s[0]\n" - "fmla z27.s, z8.s, z3.s[0]\n" - "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x12, #3, MUL VL]\n" - "fmla z24.s, z9.s, z0.s[1]\n" - "fmla z25.s, z9.s, z1.s[1]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" + "fmla z24.s, z16.s, z3.s[0]\n" + "fmla z25.s, z16.s, z2.s[0]\n" + "ld1w { z18.s }, p2/Z, [x12, #1, MUL VL]\n" + "fmla z26.s, z16.s, z1.s[0]\n" + "fmla z27.s, z16.s, z0.s[0]\n" + "ld1w { z17.s }, p2/Z, [x12, #2, MUL VL]\n" + "ld1w { z16.s }, p2/Z, [x12, #3, MUL VL]\n" + "fmla z24.s, z18.s, z3.s[1]\n" + "fmla z25.s, z18.s, z2.s[1]\n" "add x27, x27, #0x10\n" "add x26, x26, #0x10\n" - "fmla z26.s, z9.s, z2.s[1]\n" - "fmla z27.s, z9.s, z3.s[1]\n" + "fmla z26.s, z18.s, z1.s[1]\n" + "fmla z27.s, z18.s, z0.s[1]\n" "add x25, x25, #0x10\n" "addvl x12, x12, #4\n" - "fmla z24.s, z10.s, z0.s[2]\n" - "fmla z25.s, z10.s, z1.s[2]\n" - "fmla z26.s, z10.s, z2.s[2]\n" - "fmla z27.s, z10.s, z3.s[2]\n" - "fmla z24.s, z11.s, z0.s[3]\n" - "fmla z25.s, z11.s, z1.s[3]\n" - "fmla z26.s, z11.s, z2.s[3]\n" - "fmla z27.s, z11.s, z3.s[3]\n" + "fmla z24.s, z17.s, z3.s[2]\n" + "fmla z25.s, z17.s, z2.s[2]\n" + "fmla z26.s, z17.s, z1.s[2]\n" + "fmla z27.s, z17.s, z0.s[2]\n" + "fmla z24.s, z16.s, z3.s[3]\n" + "fmla z25.s, z16.s, z2.s[3]\n" + "fmla z26.s, z16.s, z1.s[3]\n" + "fmla z27.s, z16.s, z0.s[3]\n" "bgt 48b\n" "49:" // Height 4: Multiply loop: Single iteration only "whilelt p0.s, XZR, x9\n" @@ -556,35 +556,35 @@ void sve_hybrid_fp32_mla_8x1VL ( "subs x9, x9, #0x1\n" "ld1rqw { z2.s }, p0/Z, [x26]\n" "ld1rqw { z3.s }, p0/Z, [x25]\n" - "ld1w { z8.s }, p2/Z, [x12]\n" - "fmla z24.s, z8.s, z0.s[0]\n" - "fmla z25.s, z8.s, z1.s[0]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" + "fmla z24.s, z16.s, z0.s[0]\n" + "fmla z25.s, z16.s, z1.s[0]\n" "addvl x12, x12, #1\n" - "fmla z26.s, z8.s, z2.s[0]\n" - "fmla z27.s, z8.s, z3.s[0]\n" + "fmla z26.s, z16.s, z2.s[0]\n" + "fmla z27.s, z16.s, z3.s[0]\n" "ble 50f\n" - "ld1w { z9.s }, p2/Z, [x12]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" "subs x9, x9, #0x1\n" - "fmla z24.s, z9.s, z0.s[1]\n" - "fmla z25.s, z9.s, z1.s[1]\n" - "fmla z26.s, z9.s, z2.s[1]\n" - "fmla z27.s, z9.s, z3.s[1]\n" + "fmla z24.s, z16.s, z0.s[1]\n" + "fmla z25.s, z16.s, z1.s[1]\n" + "fmla z26.s, z16.s, z2.s[1]\n" + "fmla z27.s, z16.s, z3.s[1]\n" "addvl x12, x12, #1\n" "ble 50f\n" - "ld1w { z10.s }, p2/Z, [x12]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" "subs x9, x9, #0x1\n" - "fmla z24.s, z10.s, z0.s[2]\n" - "fmla z25.s, z10.s, z1.s[2]\n" - "fmla z26.s, z10.s, z2.s[2]\n" - "fmla z27.s, z10.s, z3.s[2]\n" + "fmla z24.s, z16.s, z0.s[2]\n" + "fmla z25.s, z16.s, z1.s[2]\n" + "fmla z26.s, z16.s, z2.s[2]\n" + "fmla z27.s, z16.s, z3.s[2]\n" "addvl x12, x12, #1\n" "ble 50f\n" - "ld1w { z11.s }, p2/Z, [x12]\n" - "fmla z24.s, z11.s, z0.s[3]\n" - "fmla z25.s, z11.s, z1.s[3]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" + "fmla z24.s, z16.s, z0.s[3]\n" + "fmla z25.s, z16.s, z1.s[3]\n" "addvl x12, x12, #1\n" - "fmla z26.s, z11.s, z2.s[3]\n" - "fmla z27.s, z11.s, z3.s[3]\n" + "fmla z26.s, z16.s, z2.s[3]\n" + "fmla z27.s, z16.s, z3.s[3]\n" "50:" // Height 4: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x10, x10, #0x1\n" @@ -637,15 +637,15 @@ void sve_hybrid_fp32_mla_8x1VL ( "55:" // Height 5: no bias "tbz %x[flags], #0, 56f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x27, x11, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" + "add x23, x11, x20, LSL #2\n" + "add x22, x23, x20, LSL #2\n" "ld1w { z24.s }, p1/Z, [x11]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z25.s }, p1/Z, [x27]\n" - "ld1w { z26.s }, p1/Z, [x26]\n" - "ld1w { z27.s }, p1/Z, [x25]\n" - "ld1w { z28.s }, p1/Z, [x24]\n" + "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" + "ld1w { z25.s }, p1/Z, [x23]\n" + "ld1w { z26.s }, p1/Z, [x22]\n" + "ld1w { z27.s }, p1/Z, [x21]\n" + "ld1w { z28.s }, p1/Z, [x20]\n" "b 57f\n" "56:" // Height 5: no accumulate "mov z24.b, #0x0\n" @@ -658,15 +658,15 @@ void sve_hybrid_fp32_mla_8x1VL ( "58:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w9, [x20, x10, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 59f\n" - "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x28, [x21, #0x0]\n" - "ldr x27, [x21, #0x8]\n" - "ldr x26, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" - "ldr x24, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x10, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x28, [x20, #0x0]\n" + "ldr x27, [x20, #0x8]\n" + "ldr x26, [x20, #0x10]\n" + "ldr x25, [x20, #0x18]\n" + "ldr x24, [x20, #0x20]\n" "cbnz x10, 60f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x28, x28, x20, LSL #2\n" @@ -677,52 +677,52 @@ void sve_hybrid_fp32_mla_8x1VL ( "b 60f\n" "59:" // Height 5: setup direct input "mov x28, %x[input_ptr]\n" - "add x27, x28, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" + "add x27, x28, x21, LSL #2\n" + "add x26, x27, x21, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" "60:" // Height 5: input setup done "cmp x9, #0x4\n" "ble 62f\n" "61:" // Height 5: Multiply loop: Main loop head "whilelt p0.s, XZR, x9\n" - "ld1rqw { z0.s }, p0/Z, [x28]\n" - "ld1rqw { z1.s }, p0/Z, [x27]\n" + "ld1rqw { z4.s }, p0/Z, [x28]\n" + "ld1rqw { z3.s }, p0/Z, [x27]\n" "sub x9, x9, #0x4\n" "ld1rqw { z2.s }, p0/Z, [x26]\n" - "ld1rqw { z3.s }, p0/Z, [x25]\n" + "ld1rqw { z1.s }, p0/Z, [x25]\n" "cmp x9, #0x4\n" "add x28, x28, #0x10\n" - "ld1rqw { z4.s }, p0/Z, [x24]\n" - "ld1w { z8.s }, p2/Z, [x12]\n" - "fmla z24.s, z8.s, z0.s[0]\n" - "fmla z25.s, z8.s, z1.s[0]\n" - "fmla z26.s, z8.s, z2.s[0]\n" - "fmla z27.s, z8.s, z3.s[0]\n" - "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n" - "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n" - "fmla z28.s, z8.s, z4.s[0]\n" - "fmla z24.s, z9.s, z0.s[1]\n" - "ld1w { z11.s }, p2/Z, [x12, #3, MUL VL]\n" + "ld1rqw { z0.s }, p0/Z, [x24]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" + "fmla z24.s, z16.s, z4.s[0]\n" + "fmla z25.s, z16.s, z3.s[0]\n" + "fmla z26.s, z16.s, z2.s[0]\n" + "fmla z27.s, z16.s, z1.s[0]\n" + "ld1w { z18.s }, p2/Z, [x12, #1, MUL VL]\n" + "ld1w { z17.s }, p2/Z, [x12, #2, MUL VL]\n" + "fmla z28.s, z16.s, z0.s[0]\n" + "fmla z24.s, z18.s, z4.s[1]\n" + "ld1w { z16.s }, p2/Z, [x12, #3, MUL VL]\n" "add x27, x27, #0x10\n" - "fmla z25.s, z9.s, z1.s[1]\n" - "fmla z26.s, z9.s, z2.s[1]\n" + "fmla z25.s, z18.s, z3.s[1]\n" + "fmla z26.s, z18.s, z2.s[1]\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" - "fmla z27.s, z9.s, z3.s[1]\n" - "fmla z28.s, z9.s, z4.s[1]\n" + "fmla z27.s, z18.s, z1.s[1]\n" + "fmla z28.s, z18.s, z0.s[1]\n" "add x24, x24, #0x10\n" "addvl x12, x12, #4\n" - "fmla z24.s, z10.s, z0.s[2]\n" - "fmla z25.s, z10.s, z1.s[2]\n" - "fmla z26.s, z10.s, z2.s[2]\n" - "fmla z27.s, z10.s, z3.s[2]\n" - "fmla z28.s, z10.s, z4.s[2]\n" - "fmla z24.s, z11.s, z0.s[3]\n" - "fmla z25.s, z11.s, z1.s[3]\n" - "fmla z26.s, z11.s, z2.s[3]\n" - "fmla z27.s, z11.s, z3.s[3]\n" - "fmla z28.s, z11.s, z4.s[3]\n" + "fmla z24.s, z17.s, z4.s[2]\n" + "fmla z25.s, z17.s, z3.s[2]\n" + "fmla z26.s, z17.s, z2.s[2]\n" + "fmla z27.s, z17.s, z1.s[2]\n" + "fmla z28.s, z17.s, z0.s[2]\n" + "fmla z24.s, z16.s, z4.s[3]\n" + "fmla z25.s, z16.s, z3.s[3]\n" + "fmla z26.s, z16.s, z2.s[3]\n" + "fmla z27.s, z16.s, z1.s[3]\n" + "fmla z28.s, z16.s, z0.s[3]\n" "bgt 61b\n" "62:" // Height 5: Multiply loop: Single iteration only "whilelt p0.s, XZR, x9\n" @@ -732,39 +732,39 @@ void sve_hybrid_fp32_mla_8x1VL ( "ld1rqw { z2.s }, p0/Z, [x26]\n" "ld1rqw { z3.s }, p0/Z, [x25]\n" "ld1rqw { z4.s }, p0/Z, [x24]\n" - "ld1w { z8.s }, p2/Z, [x12]\n" - "fmla z24.s, z8.s, z0.s[0]\n" - "fmla z25.s, z8.s, z1.s[0]\n" - "fmla z26.s, z8.s, z2.s[0]\n" - "fmla z27.s, z8.s, z3.s[0]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" + "fmla z24.s, z16.s, z0.s[0]\n" + "fmla z25.s, z16.s, z1.s[0]\n" + "fmla z26.s, z16.s, z2.s[0]\n" + "fmla z27.s, z16.s, z3.s[0]\n" "addvl x12, x12, #1\n" - "fmla z28.s, z8.s, z4.s[0]\n" + "fmla z28.s, z16.s, z4.s[0]\n" "ble 63f\n" - "ld1w { z9.s }, p2/Z, [x12]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" "subs x9, x9, #0x1\n" - "fmla z24.s, z9.s, z0.s[1]\n" - "fmla z25.s, z9.s, z1.s[1]\n" - "fmla z26.s, z9.s, z2.s[1]\n" - "fmla z27.s, z9.s, z3.s[1]\n" + "fmla z24.s, z16.s, z0.s[1]\n" + "fmla z25.s, z16.s, z1.s[1]\n" + "fmla z26.s, z16.s, z2.s[1]\n" + "fmla z27.s, z16.s, z3.s[1]\n" "addvl x12, x12, #1\n" - "fmla z28.s, z9.s, z4.s[1]\n" + "fmla z28.s, z16.s, z4.s[1]\n" "ble 63f\n" - "ld1w { z10.s }, p2/Z, [x12]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" "subs x9, x9, #0x1\n" - "fmla z24.s, z10.s, z0.s[2]\n" - "fmla z25.s, z10.s, z1.s[2]\n" - "fmla z26.s, z10.s, z2.s[2]\n" - "fmla z27.s, z10.s, z3.s[2]\n" + "fmla z24.s, z16.s, z0.s[2]\n" + "fmla z25.s, z16.s, z1.s[2]\n" + "fmla z26.s, z16.s, z2.s[2]\n" + "fmla z27.s, z16.s, z3.s[2]\n" "addvl x12, x12, #1\n" - "fmla z28.s, z10.s, z4.s[2]\n" + "fmla z28.s, z16.s, z4.s[2]\n" "ble 63f\n" - "ld1w { z11.s }, p2/Z, [x12]\n" - "fmla z24.s, z11.s, z0.s[3]\n" - "fmla z25.s, z11.s, z1.s[3]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" + "fmla z24.s, z16.s, z0.s[3]\n" + "fmla z25.s, z16.s, z1.s[3]\n" "addvl x12, x12, #1\n" - "fmla z26.s, z11.s, z2.s[3]\n" - "fmla z27.s, z11.s, z3.s[3]\n" - "fmla z28.s, z11.s, z4.s[3]\n" + "fmla z26.s, z16.s, z2.s[3]\n" + "fmla z27.s, z16.s, z3.s[3]\n" + "fmla z28.s, z16.s, z4.s[3]\n" "63:" // Height 5: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x10, x10, #0x1\n" @@ -821,18 +821,18 @@ void sve_hybrid_fp32_mla_8x1VL ( "b 70f\n" "68:" // Height 6: no bias "tbz %x[flags], #0, 69f\n" - "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x27, x11, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" + "ldr x24, [%x[args_ptr], %[offsetof_output_offset]]\n" + "add x23, x11, x24, LSL #2\n" + "add x20, x23, x24, LSL #2\n" "ld1w { z24.s }, p1/Z, [x11]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z25.s }, p1/Z, [x27]\n" - "ld1w { z26.s }, p1/Z, [x26]\n" - "add x23, x24, x20, LSL #2\n" - "ld1w { z27.s }, p1/Z, [x25]\n" - "ld1w { z28.s }, p1/Z, [x24]\n" - "ld1w { z29.s }, p1/Z, [x23]\n" + "add x22, x20, x24, LSL #2\n" + "add x21, x22, x24, LSL #2\n" + "ld1w { z25.s }, p1/Z, [x23]\n" + "ld1w { z26.s }, p1/Z, [x20]\n" + "add x20, x21, x24, LSL #2\n" + "ld1w { z27.s }, p1/Z, [x22]\n" + "ld1w { z28.s }, p1/Z, [x21]\n" + "ld1w { z29.s }, p1/Z, [x20]\n" "b 70f\n" "69:" // Height 6: no accumulate "mov z24.b, #0x0\n" @@ -846,16 +846,16 @@ void sve_hybrid_fp32_mla_8x1VL ( "71:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w9, [x20, x10, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 72f\n" - "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x28, [x21, #0x0]\n" - "ldr x27, [x21, #0x8]\n" - "ldr x26, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" - "ldr x24, [x21, #0x20]\n" - "ldr x23, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x10, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x28, [x20, #0x0]\n" + "ldr x27, [x20, #0x8]\n" + "ldr x26, [x20, #0x10]\n" + "ldr x25, [x20, #0x18]\n" + "ldr x24, [x20, #0x20]\n" + "ldr x23, [x20, #0x28]\n" "cbnz x10, 73f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x28, x28, x20, LSL #2\n" @@ -867,59 +867,59 @@ void sve_hybrid_fp32_mla_8x1VL ( "b 73f\n" "72:" // Height 6: setup direct input "mov x28, %x[input_ptr]\n" - "add x27, x28, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x27, x28, x21, LSL #2\n" + "add x26, x27, x21, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" "73:" // Height 6: input setup done "cmp x9, #0x4\n" "ble 75f\n" "74:" // Height 6: Multiply loop: Main loop head "whilelt p0.s, XZR, x9\n" - "ld1rqw { z0.s }, p0/Z, [x28]\n" - "ld1rqw { z1.s }, p0/Z, [x27]\n" + "ld1rqw { z5.s }, p0/Z, [x28]\n" + "ld1rqw { z4.s }, p0/Z, [x27]\n" "sub x9, x9, #0x4\n" - "ld1rqw { z2.s }, p0/Z, [x26]\n" - "ld1rqw { z3.s }, p0/Z, [x25]\n" + "ld1rqw { z3.s }, p0/Z, [x26]\n" + "ld1rqw { z2.s }, p0/Z, [x25]\n" "cmp x9, #0x4\n" "add x28, x28, #0x10\n" - "ld1rqw { z4.s }, p0/Z, [x24]\n" - "ld1rqw { z5.s }, p0/Z, [x23]\n" + "ld1rqw { z1.s }, p0/Z, [x24]\n" + "ld1rqw { z0.s }, p0/Z, [x23]\n" "add x27, x27, #0x10\n" "add x26, x26, #0x10\n" - "ld1w { z8.s }, p2/Z, [x12]\n" - "fmla z24.s, z8.s, z0.s[0]\n" - "fmla z25.s, z8.s, z1.s[0]\n" - "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n" - "fmla z26.s, z8.s, z2.s[0]\n" - "fmla z27.s, z8.s, z3.s[0]\n" - "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x12, #3, MUL VL]\n" - "fmla z28.s, z8.s, z4.s[0]\n" - "fmla z29.s, z8.s, z5.s[0]\n" + "ld1w { z19.s }, p2/Z, [x12]\n" + "fmla z24.s, z19.s, z5.s[0]\n" + "fmla z25.s, z19.s, z4.s[0]\n" + "ld1w { z18.s }, p2/Z, [x12, #1, MUL VL]\n" + "fmla z26.s, z19.s, z3.s[0]\n" + "fmla z27.s, z19.s, z2.s[0]\n" + "ld1w { z17.s }, p2/Z, [x12, #2, MUL VL]\n" + "ld1w { z16.s }, p2/Z, [x12, #3, MUL VL]\n" + "fmla z28.s, z19.s, z1.s[0]\n" + "fmla z29.s, z19.s, z0.s[0]\n" "add x25, x25, #0x10\n" "add x24, x24, #0x10\n" - "fmla z24.s, z9.s, z0.s[1]\n" - "fmla z25.s, z9.s, z1.s[1]\n" + "fmla z24.s, z18.s, z5.s[1]\n" + "fmla z25.s, z18.s, z4.s[1]\n" "add x23, x23, #0x10\n" "addvl x12, x12, #4\n" - "fmla z26.s, z9.s, z2.s[1]\n" - "fmla z27.s, z9.s, z3.s[1]\n" - "fmla z28.s, z9.s, z4.s[1]\n" - "fmla z29.s, z9.s, z5.s[1]\n" - "fmla z24.s, z10.s, z0.s[2]\n" - "fmla z25.s, z10.s, z1.s[2]\n" - "fmla z26.s, z10.s, z2.s[2]\n" - "fmla z27.s, z10.s, z3.s[2]\n" - "fmla z28.s, z10.s, z4.s[2]\n" - "fmla z29.s, z10.s, z5.s[2]\n" - "fmla z24.s, z11.s, z0.s[3]\n" - "fmla z25.s, z11.s, z1.s[3]\n" - "fmla z26.s, z11.s, z2.s[3]\n" - "fmla z27.s, z11.s, z3.s[3]\n" - "fmla z28.s, z11.s, z4.s[3]\n" - "fmla z29.s, z11.s, z5.s[3]\n" + "fmla z26.s, z18.s, z3.s[1]\n" + "fmla z27.s, z18.s, z2.s[1]\n" + "fmla z28.s, z18.s, z1.s[1]\n" + "fmla z29.s, z18.s, z0.s[1]\n" + "fmla z24.s, z17.s, z5.s[2]\n" + "fmla z25.s, z17.s, z4.s[2]\n" + "fmla z26.s, z17.s, z3.s[2]\n" + "fmla z27.s, z17.s, z2.s[2]\n" + "fmla z28.s, z17.s, z1.s[2]\n" + "fmla z29.s, z17.s, z0.s[2]\n" + "fmla z24.s, z16.s, z5.s[3]\n" + "fmla z25.s, z16.s, z4.s[3]\n" + "fmla z26.s, z16.s, z3.s[3]\n" + "fmla z27.s, z16.s, z2.s[3]\n" + "fmla z28.s, z16.s, z1.s[3]\n" + "fmla z29.s, z16.s, z0.s[3]\n" "bgt 74b\n" "75:" // Height 6: Multiply loop: Single iteration only "whilelt p0.s, XZR, x9\n" @@ -930,43 +930,43 @@ void sve_hybrid_fp32_mla_8x1VL ( "ld1rqw { z3.s }, p0/Z, [x25]\n" "ld1rqw { z4.s }, p0/Z, [x24]\n" "ld1rqw { z5.s }, p0/Z, [x23]\n" - "ld1w { z8.s }, p2/Z, [x12]\n" - "fmla z24.s, z8.s, z0.s[0]\n" - "fmla z25.s, z8.s, z1.s[0]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" + "fmla z24.s, z16.s, z0.s[0]\n" + "fmla z25.s, z16.s, z1.s[0]\n" "addvl x12, x12, #1\n" - "fmla z26.s, z8.s, z2.s[0]\n" - "fmla z27.s, z8.s, z3.s[0]\n" - "fmla z28.s, z8.s, z4.s[0]\n" - "fmla z29.s, z8.s, z5.s[0]\n" + "fmla z26.s, z16.s, z2.s[0]\n" + "fmla z27.s, z16.s, z3.s[0]\n" + "fmla z28.s, z16.s, z4.s[0]\n" + "fmla z29.s, z16.s, z5.s[0]\n" "ble 76f\n" - "ld1w { z9.s }, p2/Z, [x12]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" "subs x9, x9, #0x1\n" - "fmla z24.s, z9.s, z0.s[1]\n" - "fmla z25.s, z9.s, z1.s[1]\n" - "fmla z26.s, z9.s, z2.s[1]\n" - "fmla z27.s, z9.s, z3.s[1]\n" + "fmla z24.s, z16.s, z0.s[1]\n" + "fmla z25.s, z16.s, z1.s[1]\n" + "fmla z26.s, z16.s, z2.s[1]\n" + "fmla z27.s, z16.s, z3.s[1]\n" "addvl x12, x12, #1\n" - "fmla z28.s, z9.s, z4.s[1]\n" - "fmla z29.s, z9.s, z5.s[1]\n" + "fmla z28.s, z16.s, z4.s[1]\n" + "fmla z29.s, z16.s, z5.s[1]\n" "ble 76f\n" - "ld1w { z10.s }, p2/Z, [x12]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" "subs x9, x9, #0x1\n" - "fmla z24.s, z10.s, z0.s[2]\n" - "fmla z25.s, z10.s, z1.s[2]\n" - "fmla z26.s, z10.s, z2.s[2]\n" - "fmla z27.s, z10.s, z3.s[2]\n" + "fmla z24.s, z16.s, z0.s[2]\n" + "fmla z25.s, z16.s, z1.s[2]\n" + "fmla z26.s, z16.s, z2.s[2]\n" + "fmla z27.s, z16.s, z3.s[2]\n" "addvl x12, x12, #1\n" - "fmla z28.s, z10.s, z4.s[2]\n" - "fmla z29.s, z10.s, z5.s[2]\n" + "fmla z28.s, z16.s, z4.s[2]\n" + "fmla z29.s, z16.s, z5.s[2]\n" "ble 76f\n" - "ld1w { z11.s }, p2/Z, [x12]\n" - "fmla z24.s, z11.s, z0.s[3]\n" - "fmla z25.s, z11.s, z1.s[3]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" + "fmla z24.s, z16.s, z0.s[3]\n" + "fmla z25.s, z16.s, z1.s[3]\n" "addvl x12, x12, #1\n" - "fmla z26.s, z11.s, z2.s[3]\n" - "fmla z27.s, z11.s, z3.s[3]\n" - "fmla z28.s, z11.s, z4.s[3]\n" - "fmla z29.s, z11.s, z5.s[3]\n" + "fmla z26.s, z16.s, z2.s[3]\n" + "fmla z27.s, z16.s, z3.s[3]\n" + "fmla z28.s, z16.s, z4.s[3]\n" + "fmla z29.s, z16.s, z5.s[3]\n" "76:" // Height 6: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x10, x10, #0x1\n" @@ -1028,20 +1028,20 @@ void sve_hybrid_fp32_mla_8x1VL ( "b 83f\n" "81:" // Height 7: no bias "tbz %x[flags], #0, 82f\n" - "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x27, x11, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" + "ldr x24, [%x[args_ptr], %[offsetof_output_offset]]\n" + "add x21, x11, x24, LSL #2\n" + "add x20, x21, x24, LSL #2\n" "ld1w { z24.s }, p1/Z, [x11]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z25.s }, p1/Z, [x27]\n" - "ld1w { z26.s }, p1/Z, [x26]\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "ld1w { z27.s }, p1/Z, [x25]\n" - "ld1w { z28.s }, p1/Z, [x24]\n" - "ld1w { z29.s }, p1/Z, [x23]\n" - "ld1w { z30.s }, p1/Z, [x22]\n" + "add x23, x20, x24, LSL #2\n" + "add x22, x23, x24, LSL #2\n" + "ld1w { z25.s }, p1/Z, [x21]\n" + "ld1w { z26.s }, p1/Z, [x20]\n" + "add x21, x22, x24, LSL #2\n" + "add x20, x21, x24, LSL #2\n" + "ld1w { z27.s }, p1/Z, [x23]\n" + "ld1w { z28.s }, p1/Z, [x22]\n" + "ld1w { z29.s }, p1/Z, [x21]\n" + "ld1w { z30.s }, p1/Z, [x20]\n" "b 83f\n" "82:" // Height 7: no accumulate "mov z24.b, #0x0\n" @@ -1056,17 +1056,17 @@ void sve_hybrid_fp32_mla_8x1VL ( "84:" // Height 7: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w9, [x20, x10, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 85f\n" - "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x28, [x21, #0x0]\n" - "ldr x27, [x21, #0x8]\n" - "ldr x26, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" - "ldr x24, [x21, #0x20]\n" - "ldr x23, [x21, #0x28]\n" - "ldr x22, [x21, #0x30]\n" + "ldr x20, [%x[input_ptr], x10, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x28, [x20, #0x0]\n" + "ldr x27, [x20, #0x8]\n" + "ldr x26, [x20, #0x10]\n" + "ldr x25, [x20, #0x18]\n" + "ldr x24, [x20, #0x20]\n" + "ldr x23, [x20, #0x28]\n" + "ldr x22, [x20, #0x30]\n" "cbnz x10, 86f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x28, x28, x20, LSL #2\n" @@ -1079,66 +1079,66 @@ void sve_hybrid_fp32_mla_8x1VL ( "b 86f\n" "85:" // Height 7: setup direct input "mov x28, %x[input_ptr]\n" - "add x27, x28, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" + "add x27, x28, x21, LSL #2\n" + "add x26, x27, x21, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" "86:" // Height 7: input setup done "cmp x9, #0x4\n" "ble 88f\n" "87:" // Height 7: Multiply loop: Main loop head "whilelt p0.s, XZR, x9\n" - "ld1rqw { z0.s }, p0/Z, [x28]\n" - "ld1rqw { z1.s }, p0/Z, [x27]\n" + "ld1rqw { z6.s }, p0/Z, [x28]\n" + "ld1rqw { z5.s }, p0/Z, [x27]\n" "sub x9, x9, #0x4\n" - "ld1rqw { z2.s }, p0/Z, [x26]\n" + "ld1rqw { z4.s }, p0/Z, [x26]\n" "ld1rqw { z3.s }, p0/Z, [x25]\n" "cmp x9, #0x4\n" "add x28, x28, #0x10\n" - "ld1rqw { z4.s }, p0/Z, [x24]\n" - "ld1rqw { z5.s }, p0/Z, [x23]\n" + "ld1rqw { z2.s }, p0/Z, [x24]\n" + "ld1rqw { z1.s }, p0/Z, [x23]\n" "add x27, x27, #0x10\n" "add x26, x26, #0x10\n" - "ld1rqw { z6.s }, p0/Z, [x22]\n" - "ld1w { z8.s }, p2/Z, [x12]\n" - "fmla z24.s, z8.s, z0.s[0]\n" - "fmla z25.s, z8.s, z1.s[0]\n" - "fmla z26.s, z8.s, z2.s[0]\n" - "fmla z27.s, z8.s, z3.s[0]\n" - "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n" - "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n" - "fmla z28.s, z8.s, z4.s[0]\n" - "fmla z29.s, z8.s, z5.s[0]\n" - "ld1w { z11.s }, p2/Z, [x12, #3, MUL VL]\n" + "ld1rqw { z0.s }, p0/Z, [x22]\n" + "ld1w { z19.s }, p2/Z, [x12]\n" + "fmla z24.s, z19.s, z6.s[0]\n" + "fmla z25.s, z19.s, z5.s[0]\n" + "fmla z26.s, z19.s, z4.s[0]\n" + "fmla z27.s, z19.s, z3.s[0]\n" + "ld1w { z18.s }, p2/Z, [x12, #1, MUL VL]\n" + "ld1w { z17.s }, p2/Z, [x12, #2, MUL VL]\n" + "fmla z28.s, z19.s, z2.s[0]\n" + "fmla z29.s, z19.s, z1.s[0]\n" + "ld1w { z16.s }, p2/Z, [x12, #3, MUL VL]\n" "add x25, x25, #0x10\n" - "fmla z30.s, z8.s, z6.s[0]\n" - "fmla z24.s, z9.s, z0.s[1]\n" + "fmla z30.s, z19.s, z0.s[0]\n" + "fmla z24.s, z18.s, z6.s[1]\n" "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" - "fmla z25.s, z9.s, z1.s[1]\n" - "fmla z26.s, z9.s, z2.s[1]\n" + "fmla z25.s, z18.s, z5.s[1]\n" + "fmla z26.s, z18.s, z4.s[1]\n" "add x22, x22, #0x10\n" "addvl x12, x12, #4\n" - "fmla z27.s, z9.s, z3.s[1]\n" - "fmla z28.s, z9.s, z4.s[1]\n" - "fmla z29.s, z9.s, z5.s[1]\n" - "fmla z30.s, z9.s, z6.s[1]\n" - "fmla z24.s, z10.s, z0.s[2]\n" - "fmla z25.s, z10.s, z1.s[2]\n" - "fmla z26.s, z10.s, z2.s[2]\n" - "fmla z27.s, z10.s, z3.s[2]\n" - "fmla z28.s, z10.s, z4.s[2]\n" - "fmla z29.s, z10.s, z5.s[2]\n" - "fmla z30.s, z10.s, z6.s[2]\n" - "fmla z24.s, z11.s, z0.s[3]\n" - "fmla z25.s, z11.s, z1.s[3]\n" - "fmla z26.s, z11.s, z2.s[3]\n" - "fmla z27.s, z11.s, z3.s[3]\n" - "fmla z28.s, z11.s, z4.s[3]\n" - "fmla z29.s, z11.s, z5.s[3]\n" - "fmla z30.s, z11.s, z6.s[3]\n" + "fmla z27.s, z18.s, z3.s[1]\n" + "fmla z28.s, z18.s, z2.s[1]\n" + "fmla z29.s, z18.s, z1.s[1]\n" + "fmla z30.s, z18.s, z0.s[1]\n" + "fmla z24.s, z17.s, z6.s[2]\n" + "fmla z25.s, z17.s, z5.s[2]\n" + "fmla z26.s, z17.s, z4.s[2]\n" + "fmla z27.s, z17.s, z3.s[2]\n" + "fmla z28.s, z17.s, z2.s[2]\n" + "fmla z29.s, z17.s, z1.s[2]\n" + "fmla z30.s, z17.s, z0.s[2]\n" + "fmla z24.s, z16.s, z6.s[3]\n" + "fmla z25.s, z16.s, z5.s[3]\n" + "fmla z26.s, z16.s, z4.s[3]\n" + "fmla z27.s, z16.s, z3.s[3]\n" + "fmla z28.s, z16.s, z2.s[3]\n" + "fmla z29.s, z16.s, z1.s[3]\n" + "fmla z30.s, z16.s, z0.s[3]\n" "bgt 87b\n" "88:" // Height 7: Multiply loop: Single iteration only "whilelt p0.s, XZR, x9\n" @@ -1150,47 +1150,47 @@ void sve_hybrid_fp32_mla_8x1VL ( "ld1rqw { z4.s }, p0/Z, [x24]\n" "ld1rqw { z5.s }, p0/Z, [x23]\n" "ld1rqw { z6.s }, p0/Z, [x22]\n" - "ld1w { z8.s }, p2/Z, [x12]\n" - "fmla z24.s, z8.s, z0.s[0]\n" - "fmla z25.s, z8.s, z1.s[0]\n" - "fmla z26.s, z8.s, z2.s[0]\n" - "fmla z27.s, z8.s, z3.s[0]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" + "fmla z24.s, z16.s, z0.s[0]\n" + "fmla z25.s, z16.s, z1.s[0]\n" + "fmla z26.s, z16.s, z2.s[0]\n" + "fmla z27.s, z16.s, z3.s[0]\n" "addvl x12, x12, #1\n" - "fmla z28.s, z8.s, z4.s[0]\n" - "fmla z29.s, z8.s, z5.s[0]\n" - "fmla z30.s, z8.s, z6.s[0]\n" + "fmla z28.s, z16.s, z4.s[0]\n" + "fmla z29.s, z16.s, z5.s[0]\n" + "fmla z30.s, z16.s, z6.s[0]\n" "ble 89f\n" - "ld1w { z9.s }, p2/Z, [x12]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" "subs x9, x9, #0x1\n" - "fmla z24.s, z9.s, z0.s[1]\n" - "fmla z25.s, z9.s, z1.s[1]\n" - "fmla z26.s, z9.s, z2.s[1]\n" - "fmla z27.s, z9.s, z3.s[1]\n" + "fmla z24.s, z16.s, z0.s[1]\n" + "fmla z25.s, z16.s, z1.s[1]\n" + "fmla z26.s, z16.s, z2.s[1]\n" + "fmla z27.s, z16.s, z3.s[1]\n" "addvl x12, x12, #1\n" - "fmla z28.s, z9.s, z4.s[1]\n" - "fmla z29.s, z9.s, z5.s[1]\n" - "fmla z30.s, z9.s, z6.s[1]\n" + "fmla z28.s, z16.s, z4.s[1]\n" + "fmla z29.s, z16.s, z5.s[1]\n" + "fmla z30.s, z16.s, z6.s[1]\n" "ble 89f\n" - "ld1w { z10.s }, p2/Z, [x12]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" "subs x9, x9, #0x1\n" - "fmla z24.s, z10.s, z0.s[2]\n" - "fmla z25.s, z10.s, z1.s[2]\n" - "fmla z26.s, z10.s, z2.s[2]\n" - "fmla z27.s, z10.s, z3.s[2]\n" + "fmla z24.s, z16.s, z0.s[2]\n" + "fmla z25.s, z16.s, z1.s[2]\n" + "fmla z26.s, z16.s, z2.s[2]\n" + "fmla z27.s, z16.s, z3.s[2]\n" "addvl x12, x12, #1\n" - "fmla z28.s, z10.s, z4.s[2]\n" - "fmla z29.s, z10.s, z5.s[2]\n" - "fmla z30.s, z10.s, z6.s[2]\n" + "fmla z28.s, z16.s, z4.s[2]\n" + "fmla z29.s, z16.s, z5.s[2]\n" + "fmla z30.s, z16.s, z6.s[2]\n" "ble 89f\n" - "ld1w { z11.s }, p2/Z, [x12]\n" - "fmla z24.s, z11.s, z0.s[3]\n" - "fmla z25.s, z11.s, z1.s[3]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" + "fmla z24.s, z16.s, z0.s[3]\n" + "fmla z25.s, z16.s, z1.s[3]\n" "addvl x12, x12, #1\n" - "fmla z26.s, z11.s, z2.s[3]\n" - "fmla z27.s, z11.s, z3.s[3]\n" - "fmla z28.s, z11.s, z4.s[3]\n" - "fmla z29.s, z11.s, z5.s[3]\n" - "fmla z30.s, z11.s, z6.s[3]\n" + "fmla z26.s, z16.s, z2.s[3]\n" + "fmla z27.s, z16.s, z3.s[3]\n" + "fmla z28.s, z16.s, z4.s[3]\n" + "fmla z29.s, z16.s, z5.s[3]\n" + "fmla z30.s, z16.s, z6.s[3]\n" "89:" // Height 7: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x10, x10, #0x1\n" @@ -1260,22 +1260,22 @@ void sve_hybrid_fp32_mla_8x1VL ( "b 96f\n" "94:" // Height 8: no bias "tbz %x[flags], #0, 95f\n" - "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x27, x11, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" + "ldr x24, [%x[args_ptr], %[offsetof_output_offset]]\n" + "add x22, x11, x24, LSL #2\n" + "add x21, x22, x24, LSL #2\n" "ld1w { z24.s }, p1/Z, [x11]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z25.s }, p1/Z, [x27]\n" - "ld1w { z26.s }, p1/Z, [x26]\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "ld1w { z27.s }, p1/Z, [x25]\n" - "ld1w { z28.s }, p1/Z, [x24]\n" - "add x21, x22, x20, LSL #2\n" - "ld1w { z29.s }, p1/Z, [x23]\n" - "ld1w { z30.s }, p1/Z, [x22]\n" - "ld1w { z31.s }, p1/Z, [x21]\n" + "add x23, x21, x24, LSL #2\n" + "add x20, x23, x24, LSL #2\n" + "ld1w { z25.s }, p1/Z, [x22]\n" + "ld1w { z26.s }, p1/Z, [x21]\n" + "add x22, x20, x24, LSL #2\n" + "add x21, x22, x24, LSL #2\n" + "ld1w { z27.s }, p1/Z, [x23]\n" + "ld1w { z28.s }, p1/Z, [x20]\n" + "add x20, x21, x24, LSL #2\n" + "ld1w { z29.s }, p1/Z, [x22]\n" + "ld1w { z30.s }, p1/Z, [x21]\n" + "ld1w { z31.s }, p1/Z, [x20]\n" "b 96f\n" "95:" // Height 8: no accumulate "mov z24.b, #0x0\n" @@ -1291,18 +1291,18 @@ void sve_hybrid_fp32_mla_8x1VL ( "97:" // Height 8: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w9, [x20, x10, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 98f\n" - "ldr x21, [%x[input_ptr], x10, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x28, [x21, #0x0]\n" - "ldr x27, [x21, #0x8]\n" - "ldr x26, [x21, #0x10]\n" - "ldr x25, [x21, #0x18]\n" - "ldr x24, [x21, #0x20]\n" - "ldr x23, [x21, #0x28]\n" - "ldr x22, [x21, #0x30]\n" - "ldr x21, [x21, #0x38]\n" + "ldr x20, [%x[input_ptr], x10, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x28, [x20, #0x0]\n" + "ldr x27, [x20, #0x8]\n" + "ldr x26, [x20, #0x10]\n" + "ldr x25, [x20, #0x18]\n" + "ldr x24, [x20, #0x20]\n" + "ldr x23, [x20, #0x28]\n" + "ldr x22, [x20, #0x30]\n" + "ldr x21, [x20, #0x38]\n" "cbnz x10, 99f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x28, x28, x20, LSL #2\n" @@ -1316,73 +1316,73 @@ void sve_hybrid_fp32_mla_8x1VL ( "b 99f\n" "98:" // Height 8: setup direct input "mov x28, %x[input_ptr]\n" - "add x27, x28, x20, LSL #2\n" - "add x26, x27, x20, LSL #2\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "add x21, x22, x20, LSL #2\n" + "add x27, x28, x21, LSL #2\n" + "add x26, x27, x21, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" + "add x21, x22, x21, LSL #2\n" "99:" // Height 8: input setup done "cmp x9, #0x4\n" "ble 101f\n" "100:" // Height 8: Multiply loop: Main loop head "whilelt p0.s, XZR, x9\n" - "ld1rqw { z0.s }, p0/Z, [x28]\n" - "ld1rqw { z1.s }, p0/Z, [x27]\n" + "ld1rqw { z7.s }, p0/Z, [x28]\n" + "ld1rqw { z6.s }, p0/Z, [x27]\n" "sub x9, x9, #0x4\n" - "ld1rqw { z2.s }, p0/Z, [x26]\n" - "ld1rqw { z3.s }, p0/Z, [x25]\n" + "ld1rqw { z5.s }, p0/Z, [x26]\n" + "ld1rqw { z4.s }, p0/Z, [x25]\n" "cmp x9, #0x4\n" "add x28, x28, #0x10\n" - "ld1rqw { z4.s }, p0/Z, [x24]\n" - "ld1rqw { z5.s }, p0/Z, [x23]\n" + "ld1rqw { z3.s }, p0/Z, [x24]\n" + "ld1rqw { z2.s }, p0/Z, [x23]\n" "add x27, x27, #0x10\n" "add x26, x26, #0x10\n" - "ld1rqw { z6.s }, p0/Z, [x22]\n" - "ld1rqw { z7.s }, p0/Z, [x21]\n" + "ld1rqw { z1.s }, p0/Z, [x22]\n" + "ld1rqw { z0.s }, p0/Z, [x21]\n" "add x25, x25, #0x10\n" "add x24, x24, #0x10\n" - "ld1w { z8.s }, p2/Z, [x12]\n" - "fmla z24.s, z8.s, z0.s[0]\n" - "fmla z25.s, z8.s, z1.s[0]\n" - "ld1w { z9.s }, p2/Z, [x12, #1, MUL VL]\n" - "fmla z26.s, z8.s, z2.s[0]\n" - "fmla z27.s, z8.s, z3.s[0]\n" - "ld1w { z10.s }, p2/Z, [x12, #2, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x12, #3, MUL VL]\n" - "fmla z28.s, z8.s, z4.s[0]\n" - "fmla z29.s, z8.s, z5.s[0]\n" + "ld1w { z19.s }, p2/Z, [x12]\n" + "fmla z24.s, z19.s, z7.s[0]\n" + "fmla z25.s, z19.s, z6.s[0]\n" + "ld1w { z18.s }, p2/Z, [x12, #1, MUL VL]\n" + "fmla z26.s, z19.s, z5.s[0]\n" + "fmla z27.s, z19.s, z4.s[0]\n" + "ld1w { z17.s }, p2/Z, [x12, #2, MUL VL]\n" + "ld1w { z16.s }, p2/Z, [x12, #3, MUL VL]\n" + "fmla z28.s, z19.s, z3.s[0]\n" + "fmla z29.s, z19.s, z2.s[0]\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - "fmla z30.s, z8.s, z6.s[0]\n" - "fmla z31.s, z8.s, z7.s[0]\n" + "fmla z30.s, z19.s, z1.s[0]\n" + "fmla z31.s, z19.s, z0.s[0]\n" "add x21, x21, #0x10\n" "addvl x12, x12, #4\n" - "fmla z24.s, z9.s, z0.s[1]\n" - "fmla z25.s, z9.s, z1.s[1]\n" - "fmla z26.s, z9.s, z2.s[1]\n" - "fmla z27.s, z9.s, z3.s[1]\n" - "fmla z28.s, z9.s, z4.s[1]\n" - "fmla z29.s, z9.s, z5.s[1]\n" - "fmla z30.s, z9.s, z6.s[1]\n" - "fmla z31.s, z9.s, z7.s[1]\n" - "fmla z24.s, z10.s, z0.s[2]\n" - "fmla z25.s, z10.s, z1.s[2]\n" - "fmla z26.s, z10.s, z2.s[2]\n" - "fmla z27.s, z10.s, z3.s[2]\n" - "fmla z28.s, z10.s, z4.s[2]\n" - "fmla z29.s, z10.s, z5.s[2]\n" - "fmla z30.s, z10.s, z6.s[2]\n" - "fmla z31.s, z10.s, z7.s[2]\n" - "fmla z24.s, z11.s, z0.s[3]\n" - "fmla z25.s, z11.s, z1.s[3]\n" - "fmla z26.s, z11.s, z2.s[3]\n" - "fmla z27.s, z11.s, z3.s[3]\n" - "fmla z28.s, z11.s, z4.s[3]\n" - "fmla z29.s, z11.s, z5.s[3]\n" - "fmla z30.s, z11.s, z6.s[3]\n" - "fmla z31.s, z11.s, z7.s[3]\n" + "fmla z24.s, z18.s, z7.s[1]\n" + "fmla z25.s, z18.s, z6.s[1]\n" + "fmla z26.s, z18.s, z5.s[1]\n" + "fmla z27.s, z18.s, z4.s[1]\n" + "fmla z28.s, z18.s, z3.s[1]\n" + "fmla z29.s, z18.s, z2.s[1]\n" + "fmla z30.s, z18.s, z1.s[1]\n" + "fmla z31.s, z18.s, z0.s[1]\n" + "fmla z24.s, z17.s, z7.s[2]\n" + "fmla z25.s, z17.s, z6.s[2]\n" + "fmla z26.s, z17.s, z5.s[2]\n" + "fmla z27.s, z17.s, z4.s[2]\n" + "fmla z28.s, z17.s, z3.s[2]\n" + "fmla z29.s, z17.s, z2.s[2]\n" + "fmla z30.s, z17.s, z1.s[2]\n" + "fmla z31.s, z17.s, z0.s[2]\n" + "fmla z24.s, z16.s, z7.s[3]\n" + "fmla z25.s, z16.s, z6.s[3]\n" + "fmla z26.s, z16.s, z5.s[3]\n" + "fmla z27.s, z16.s, z4.s[3]\n" + "fmla z28.s, z16.s, z3.s[3]\n" + "fmla z29.s, z16.s, z2.s[3]\n" + "fmla z30.s, z16.s, z1.s[3]\n" + "fmla z31.s, z16.s, z0.s[3]\n" "bgt 100b\n" "101:" // Height 8: Multiply loop: Single iteration only "whilelt p0.s, XZR, x9\n" @@ -1395,51 +1395,51 @@ void sve_hybrid_fp32_mla_8x1VL ( "ld1rqw { z5.s }, p0/Z, [x23]\n" "ld1rqw { z6.s }, p0/Z, [x22]\n" "ld1rqw { z7.s }, p0/Z, [x21]\n" - "ld1w { z8.s }, p2/Z, [x12]\n" - "fmla z24.s, z8.s, z0.s[0]\n" - "fmla z25.s, z8.s, z1.s[0]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" + "fmla z24.s, z16.s, z0.s[0]\n" + "fmla z25.s, z16.s, z1.s[0]\n" "addvl x12, x12, #1\n" - "fmla z26.s, z8.s, z2.s[0]\n" - "fmla z27.s, z8.s, z3.s[0]\n" - "fmla z28.s, z8.s, z4.s[0]\n" - "fmla z29.s, z8.s, z5.s[0]\n" - "fmla z30.s, z8.s, z6.s[0]\n" - "fmla z31.s, z8.s, z7.s[0]\n" + "fmla z26.s, z16.s, z2.s[0]\n" + "fmla z27.s, z16.s, z3.s[0]\n" + "fmla z28.s, z16.s, z4.s[0]\n" + "fmla z29.s, z16.s, z5.s[0]\n" + "fmla z30.s, z16.s, z6.s[0]\n" + "fmla z31.s, z16.s, z7.s[0]\n" "ble 102f\n" - "ld1w { z9.s }, p2/Z, [x12]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" "subs x9, x9, #0x1\n" - "fmla z24.s, z9.s, z0.s[1]\n" - "fmla z25.s, z9.s, z1.s[1]\n" - "fmla z26.s, z9.s, z2.s[1]\n" - "fmla z27.s, z9.s, z3.s[1]\n" + "fmla z24.s, z16.s, z0.s[1]\n" + "fmla z25.s, z16.s, z1.s[1]\n" + "fmla z26.s, z16.s, z2.s[1]\n" + "fmla z27.s, z16.s, z3.s[1]\n" "addvl x12, x12, #1\n" - "fmla z28.s, z9.s, z4.s[1]\n" - "fmla z29.s, z9.s, z5.s[1]\n" - "fmla z30.s, z9.s, z6.s[1]\n" - "fmla z31.s, z9.s, z7.s[1]\n" + "fmla z28.s, z16.s, z4.s[1]\n" + "fmla z29.s, z16.s, z5.s[1]\n" + "fmla z30.s, z16.s, z6.s[1]\n" + "fmla z31.s, z16.s, z7.s[1]\n" "ble 102f\n" - "ld1w { z10.s }, p2/Z, [x12]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" "subs x9, x9, #0x1\n" - "fmla z24.s, z10.s, z0.s[2]\n" - "fmla z25.s, z10.s, z1.s[2]\n" - "fmla z26.s, z10.s, z2.s[2]\n" - "fmla z27.s, z10.s, z3.s[2]\n" + "fmla z24.s, z16.s, z0.s[2]\n" + "fmla z25.s, z16.s, z1.s[2]\n" + "fmla z26.s, z16.s, z2.s[2]\n" + "fmla z27.s, z16.s, z3.s[2]\n" "addvl x12, x12, #1\n" - "fmla z28.s, z10.s, z4.s[2]\n" - "fmla z29.s, z10.s, z5.s[2]\n" - "fmla z30.s, z10.s, z6.s[2]\n" - "fmla z31.s, z10.s, z7.s[2]\n" + "fmla z28.s, z16.s, z4.s[2]\n" + "fmla z29.s, z16.s, z5.s[2]\n" + "fmla z30.s, z16.s, z6.s[2]\n" + "fmla z31.s, z16.s, z7.s[2]\n" "ble 102f\n" - "ld1w { z11.s }, p2/Z, [x12]\n" - "fmla z24.s, z11.s, z0.s[3]\n" - "fmla z25.s, z11.s, z1.s[3]\n" + "ld1w { z16.s }, p2/Z, [x12]\n" + "fmla z24.s, z16.s, z0.s[3]\n" + "fmla z25.s, z16.s, z1.s[3]\n" "addvl x12, x12, #1\n" - "fmla z26.s, z11.s, z2.s[3]\n" - "fmla z27.s, z11.s, z3.s[3]\n" - "fmla z28.s, z11.s, z4.s[3]\n" - "fmla z29.s, z11.s, z5.s[3]\n" - "fmla z30.s, z11.s, z6.s[3]\n" - "fmla z31.s, z11.s, z7.s[3]\n" + "fmla z26.s, z16.s, z2.s[3]\n" + "fmla z27.s, z16.s, z3.s[3]\n" + "fmla z28.s, z16.s, z4.s[3]\n" + "fmla z29.s, z16.s, z5.s[3]\n" + "fmla z30.s, z16.s, z6.s[3]\n" + "fmla z31.s, z16.s, z7.s[3]\n" "102:" // Height 8: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x10, x10, #0x1\n" @@ -1500,12 +1500,11 @@ void sve_hybrid_fp32_mla_8x1VL ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "106:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) - : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z16", "z17", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16", "z17", "z18", "z19", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace arm_gemm -#endif // ARM_COMPUTE_ENABLE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32bf16fp32_mmla_4x6VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32bf16fp32_mmla_4x6VL.hpp index ab175a37587a88be875d2bf0c18223088d095ee3..66c106d2eb527b5154b18b70a2b411ebf66e86f3 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32bf16fp32_mmla_4x6VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32bf16fp32_mmla_4x6VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef ARM_COMPUTE_ENABLE_SVE + #include "../std_transforms_sve.hpp" #include "../bfloat.hpp" #include "../performance_parameters.hpp" @@ -75,7 +75,6 @@ public: template static inline PerformanceParameters get_performance_parameters(const CPUInfo *ci) { - if (std::is_same::value) { switch (ci->get_cpu_model()) { default: @@ -100,5 +99,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32bf16fp32_mmla_4x6VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32bf16fp32_mmla_4x6VL/generic.cpp index 8d05c1ffb38b2d743499a6692d3bb6b533880f10..2b2a0684f9d446f7f7f7e974c2990be5350f4c04 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32bf16fp32_mmla_4x6VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32bf16fp32_mmla_4x6VL/generic.cpp @@ -140,22 +140,22 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL ( "b 5f\n" "3:" // Height 1: no bias "tbz %x[flags], #0, 4f\n" - "ld1w { z9.s }, p6/Z, [x27]\n" - "ld1w { z10.s }, p5/Z, [x27, #1, MUL VL]\n" - "zip1 z8.d, z9.d, z14.d\n" - "zip2 z14.d, z9.d, z14.d\n" - "ld1w { z11.s }, p4/Z, [x27, #2, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x27, #3, MUL VL]\n" - "zip1 z9.d, z10.d, z15.d\n" - "zip2 z15.d, z10.d, z15.d\n" - "ld1w { z13.s }, p2/Z, [x27, #4, MUL VL]\n" + "ld1w { z21.s }, p6/Z, [x27]\n" + "ld1w { z20.s }, p5/Z, [x27, #1, MUL VL]\n" + "zip1 z8.d, z21.d, z14.d\n" + "zip2 z14.d, z21.d, z14.d\n" + "ld1w { z23.s }, p4/Z, [x27, #2, MUL VL]\n" + "ld1w { z22.s }, p3/Z, [x27, #3, MUL VL]\n" + "zip1 z9.d, z20.d, z15.d\n" + "zip2 z15.d, z20.d, z15.d\n" + "ld1w { z21.s }, p2/Z, [x27, #4, MUL VL]\n" "ld1w { z20.s }, p1/Z, [x27, #5, MUL VL]\n" - "zip1 z10.d, z11.d, z16.d\n" - "zip2 z16.d, z11.d, z16.d\n" - "zip1 z11.d, z12.d, z17.d\n" - "zip2 z17.d, z12.d, z17.d\n" - "zip1 z12.d, z13.d, z18.d\n" - "zip2 z18.d, z13.d, z18.d\n" + "zip1 z10.d, z23.d, z16.d\n" + "zip2 z16.d, z23.d, z16.d\n" + "zip1 z11.d, z22.d, z17.d\n" + "zip2 z17.d, z22.d, z17.d\n" + "zip1 z12.d, z21.d, z18.d\n" + "zip2 z18.d, z21.d, z18.d\n" "zip1 z13.d, z20.d, z19.d\n" "zip2 z19.d, z20.d, z19.d\n" "b 5f\n" @@ -177,11 +177,11 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL ( "6:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 7f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" "cbnz x26, 8f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20, LSL #2\n" @@ -193,69 +193,69 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL ( "ble 10f\n" "9:" // Height 1: Multiply loop: Main loop head "whilelt p0.s, XZR, x25\n" - "ld1rqw { z0.s }, p0/Z, [x24]\n" - ".inst 0x658abc00 // bfcvt z0.h, p7/M, z0.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z4.h }, p7/Z, [x28]\n" - "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n" - ".inst 0x6464e408 // bfmmla z8.s, z0.h, z4.h\n" - ".inst 0x6465e40e // bfmmla z14.s, z0.h, z5.h\n" - "ld1h { z6.h }, p7/Z, [x28, #2, MUL VL]\n" - "ld1h { z7.h }, p7/Z, [x28, #3, MUL VL]\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" - "ld1h { z4.h }, p7/Z, [x28, #4, MUL VL]\n" - "ld1h { z5.h }, p7/Z, [x28, #5, MUL VL]\n" - ".inst 0x6464e40a // bfmmla z10.s, z0.h, z4.h\n" - ".inst 0x6465e410 // bfmmla z16.s, z0.h, z5.h\n" - "ld1h { z6.h }, p7/Z, [x28, #6, MUL VL]\n" - "ld1h { z7.h }, p7/Z, [x28, #7, MUL VL]\n" + "ld1rqw { z24.s }, p0/Z, [x24]\n" + ".inst 0x658abf18 // bfcvt z24.h, p7/M, z24.s\n" + "uzp1 z24.h, z24.h, z24.h\n" + "ld1h { z21.h }, p7/Z, [x28]\n" + "ld1h { z20.h }, p7/Z, [x28, #1, MUL VL]\n" + ".inst 0x6475e708 // bfmmla z8.s, z24.h, z21.h\n" + ".inst 0x6474e70e // bfmmla z14.s, z24.h, z20.h\n" + "ld1h { z21.h }, p7/Z, [x28, #2, MUL VL]\n" + "ld1h { z20.h }, p7/Z, [x28, #3, MUL VL]\n" + ".inst 0x6475e709 // bfmmla z9.s, z24.h, z21.h\n" + ".inst 0x6474e70f // bfmmla z15.s, z24.h, z20.h\n" + "ld1h { z21.h }, p7/Z, [x28, #4, MUL VL]\n" + "ld1h { z20.h }, p7/Z, [x28, #5, MUL VL]\n" + ".inst 0x6475e70a // bfmmla z10.s, z24.h, z21.h\n" + ".inst 0x6474e710 // bfmmla z16.s, z24.h, z20.h\n" + "ld1h { z21.h }, p7/Z, [x28, #6, MUL VL]\n" + "ld1h { z20.h }, p7/Z, [x28, #7, MUL VL]\n" "addvl x28, x28, #16\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" - ".inst 0x6467e411 // bfmmla z17.s, z0.h, z7.h\n" - "ld1h { z4.h }, p7/Z, [x28, #-8, MUL VL]\n" - "ld1h { z5.h }, p7/Z, [x28, #-7, MUL VL]\n" + ".inst 0x6475e70b // bfmmla z11.s, z24.h, z21.h\n" + ".inst 0x6474e711 // bfmmla z17.s, z24.h, z20.h\n" + "ld1h { z23.h }, p7/Z, [x28, #-8, MUL VL]\n" + "ld1h { z22.h }, p7/Z, [x28, #-7, MUL VL]\n" "sub x25, x25, #0x4\n" - "ld1h { z6.h }, p7/Z, [x28, #-6, MUL VL]\n" - "ld1h { z7.h }, p7/Z, [x28, #-5, MUL VL]\n" + "ld1h { z21.h }, p7/Z, [x28, #-6, MUL VL]\n" + "ld1h { z20.h }, p7/Z, [x28, #-5, MUL VL]\n" "cmp x25, #0x4\n" - ".inst 0x6464e40c // bfmmla z12.s, z0.h, z4.h\n" - ".inst 0x6465e412 // bfmmla z18.s, z0.h, z5.h\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" + ".inst 0x6477e70c // bfmmla z12.s, z24.h, z23.h\n" + ".inst 0x6476e712 // bfmmla z18.s, z24.h, z22.h\n" + ".inst 0x6475e70d // bfmmla z13.s, z24.h, z21.h\n" "add x24, x24, #0x10\n" "addvl x28, x28, #-4\n" - ".inst 0x6467e413 // bfmmla z19.s, z0.h, z7.h\n" + ".inst 0x6474e713 // bfmmla z19.s, z24.h, z20.h\n" "bgt 9b\n" "10:" // Height 1: Multiply loop: Single iteration only "whilelt p0.s, XZR, x25\n" - "ld1rqw { z0.s }, p0/Z, [x24]\n" - ".inst 0x658abc00 // bfcvt z0.h, p7/M, z0.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z4.h }, p7/Z, [x28]\n" - "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n" - ".inst 0x6464e408 // bfmmla z8.s, z0.h, z4.h\n" - ".inst 0x6465e40e // bfmmla z14.s, z0.h, z5.h\n" - "ld1h { z6.h }, p7/Z, [x28, #2, MUL VL]\n" - "ld1h { z7.h }, p7/Z, [x28, #3, MUL VL]\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" - "ld1h { z4.h }, p7/Z, [x28, #4, MUL VL]\n" - "ld1h { z5.h }, p7/Z, [x28, #5, MUL VL]\n" - ".inst 0x6464e40a // bfmmla z10.s, z0.h, z4.h\n" - ".inst 0x6465e410 // bfmmla z16.s, z0.h, z5.h\n" - "ld1h { z6.h }, p7/Z, [x28, #6, MUL VL]\n" - "ld1h { z7.h }, p7/Z, [x28, #7, MUL VL]\n" + "ld1rqw { z23.s }, p0/Z, [x24]\n" + ".inst 0x658abef7 // bfcvt z23.h, p7/M, z23.s\n" + "uzp1 z23.h, z23.h, z23.h\n" + "ld1h { z21.h }, p7/Z, [x28]\n" + "ld1h { z20.h }, p7/Z, [x28, #1, MUL VL]\n" + ".inst 0x6475e6e8 // bfmmla z8.s, z23.h, z21.h\n" + ".inst 0x6474e6ee // bfmmla z14.s, z23.h, z20.h\n" + "ld1h { z21.h }, p7/Z, [x28, #2, MUL VL]\n" + "ld1h { z20.h }, p7/Z, [x28, #3, MUL VL]\n" + ".inst 0x6475e6e9 // bfmmla z9.s, z23.h, z21.h\n" + ".inst 0x6474e6ef // bfmmla z15.s, z23.h, z20.h\n" + "ld1h { z21.h }, p7/Z, [x28, #4, MUL VL]\n" + "ld1h { z20.h }, p7/Z, [x28, #5, MUL VL]\n" + ".inst 0x6475e6ea // bfmmla z10.s, z23.h, z21.h\n" + ".inst 0x6474e6f0 // bfmmla z16.s, z23.h, z20.h\n" + "ld1h { z21.h }, p7/Z, [x28, #6, MUL VL]\n" + "ld1h { z20.h }, p7/Z, [x28, #7, MUL VL]\n" "addvl x28, x28, #16\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" - ".inst 0x6467e411 // bfmmla z17.s, z0.h, z7.h\n" - "ld1h { z4.h }, p7/Z, [x28, #-8, MUL VL]\n" - "ld1h { z5.h }, p7/Z, [x28, #-7, MUL VL]\n" - ".inst 0x6464e40c // bfmmla z12.s, z0.h, z4.h\n" - "ld1h { z6.h }, p7/Z, [x28, #-6, MUL VL]\n" - "ld1h { z7.h }, p7/Z, [x28, #-5, MUL VL]\n" - ".inst 0x6465e412 // bfmmla z18.s, z0.h, z5.h\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6467e413 // bfmmla z19.s, z0.h, z7.h\n" + ".inst 0x6475e6eb // bfmmla z11.s, z23.h, z21.h\n" + ".inst 0x6474e6f1 // bfmmla z17.s, z23.h, z20.h\n" + "ld1h { z20.h }, p7/Z, [x28, #-8, MUL VL]\n" + "ld1h { z22.h }, p7/Z, [x28, #-7, MUL VL]\n" + ".inst 0x6474e6ec // bfmmla z12.s, z23.h, z20.h\n" + "ld1h { z21.h }, p7/Z, [x28, #-6, MUL VL]\n" + "ld1h { z20.h }, p7/Z, [x28, #-5, MUL VL]\n" + ".inst 0x6476e6f2 // bfmmla z18.s, z23.h, z22.h\n" + ".inst 0x6475e6ed // bfmmla z13.s, z23.h, z21.h\n" + ".inst 0x6474e6f3 // bfmmla z19.s, z23.h, z20.h\n" "addvl x28, x28, #-4\n" "11:" // Height 1: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -270,21 +270,21 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL ( "uzp1 z13.d, z13.d, z19.d\n" "tbz %x[flags], #1, 12f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p7/Z, [x20]\n" + "ld1rw { z21.s }, p7/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p7/Z, [x20]\n" - "fmin z8.s, p7/M, z8.s, z1.s\n" - "fmin z9.s, p7/M, z9.s, z1.s\n" - "fmin z10.s, p7/M, z10.s, z1.s\n" - "fmin z11.s, p7/M, z11.s, z1.s\n" - "fmin z12.s, p7/M, z12.s, z1.s\n" - "fmin z13.s, p7/M, z13.s, z1.s\n" - "fmax z8.s, p7/M, z8.s, z0.s\n" - "fmax z9.s, p7/M, z9.s, z0.s\n" - "fmax z10.s, p7/M, z10.s, z0.s\n" - "fmax z11.s, p7/M, z11.s, z0.s\n" - "fmax z12.s, p7/M, z12.s, z0.s\n" - "fmax z13.s, p7/M, z13.s, z0.s\n" + "ld1rw { z20.s }, p7/Z, [x20]\n" + "fmin z8.s, p7/M, z8.s, z21.s\n" + "fmin z9.s, p7/M, z9.s, z21.s\n" + "fmin z10.s, p7/M, z10.s, z21.s\n" + "fmin z11.s, p7/M, z11.s, z21.s\n" + "fmin z12.s, p7/M, z12.s, z21.s\n" + "fmin z13.s, p7/M, z13.s, z21.s\n" + "fmax z8.s, p7/M, z8.s, z20.s\n" + "fmax z9.s, p7/M, z9.s, z20.s\n" + "fmax z10.s, p7/M, z10.s, z20.s\n" + "fmax z11.s, p7/M, z11.s, z20.s\n" + "fmax z12.s, p7/M, z12.s, z20.s\n" + "fmax z13.s, p7/M, z13.s, z20.s\n" "12:" // Height 1: No activation "st1w { z8.s }, p6, [x27]\n" "st1w { z9.s }, p5, [x27, #1, MUL VL]\n" @@ -340,29 +340,29 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL ( "16:" // Height 2: no bias "tbz %x[flags], #0, 17f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x23, x27, x20, LSL #2\n" - "ld1w { z9.s }, p6/Z, [x27]\n" - "ld1w { z10.s }, p5/Z, [x27, #1, MUL VL]\n" - "ld1w { z11.s }, p4/Z, [x27, #2, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x27, #3, MUL VL]\n" - "ld1w { z13.s }, p2/Z, [x27, #4, MUL VL]\n" + "add x20, x27, x20, LSL #2\n" + "ld1w { z16.s }, p6/Z, [x27]\n" + "ld1w { z17.s }, p5/Z, [x27, #1, MUL VL]\n" + "ld1w { z19.s }, p4/Z, [x27, #2, MUL VL]\n" + "ld1w { z22.s }, p3/Z, [x27, #3, MUL VL]\n" + "ld1w { z21.s }, p2/Z, [x27, #4, MUL VL]\n" "ld1w { z20.s }, p1/Z, [x27, #5, MUL VL]\n" - "ld1w { z14.s }, p6/Z, [x23]\n" - "zip1 z8.d, z9.d, z14.d\n" - "zip2 z14.d, z9.d, z14.d\n" - "ld1w { z15.s }, p5/Z, [x23, #1, MUL VL]\n" - "ld1w { z16.s }, p4/Z, [x23, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z15.d\n" - "zip2 z15.d, z10.d, z15.d\n" - "ld1w { z17.s }, p3/Z, [x23, #3, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x23, #4, MUL VL]\n" - "zip1 z10.d, z11.d, z16.d\n" - "zip2 z16.d, z11.d, z16.d\n" - "ld1w { z19.s }, p1/Z, [x23, #5, MUL VL]\n" - "zip1 z11.d, z12.d, z17.d\n" - "zip2 z17.d, z12.d, z17.d\n" - "zip1 z12.d, z13.d, z18.d\n" - "zip2 z18.d, z13.d, z18.d\n" + "ld1w { z14.s }, p6/Z, [x20]\n" + "zip1 z8.d, z16.d, z14.d\n" + "zip2 z14.d, z16.d, z14.d\n" + "ld1w { z15.s }, p5/Z, [x20, #1, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x20, #2, MUL VL]\n" + "zip1 z9.d, z17.d, z15.d\n" + "zip2 z15.d, z17.d, z15.d\n" + "ld1w { z17.s }, p3/Z, [x20, #3, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x20, #4, MUL VL]\n" + "zip1 z10.d, z19.d, z16.d\n" + "zip2 z16.d, z19.d, z16.d\n" + "ld1w { z19.s }, p1/Z, [x20, #5, MUL VL]\n" + "zip1 z11.d, z22.d, z17.d\n" + "zip2 z17.d, z22.d, z17.d\n" + "zip1 z12.d, z21.d, z18.d\n" + "zip2 z18.d, z21.d, z18.d\n" "zip1 z13.d, z20.d, z19.d\n" "zip2 z19.d, z20.d, z19.d\n" "b 18f\n" @@ -384,12 +384,12 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL ( "19:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 20f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" "cbnz x26, 21f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20, LSL #2\n" @@ -397,85 +397,85 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL ( "b 21f\n" "20:" // Height 2: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20, LSL #2\n" + "add x23, x24, x21, LSL #2\n" "21:" // Height 2: input setup done "cmp x25, #0x4\n" "ble 23f\n" "22:" // Height 2: Multiply loop: Main loop head "whilelt p0.s, XZR, x25\n" - "ld1rqw { z0.s }, p0/Z, [x24]\n" - "ld1rqw { z1.s }, p0/Z, [x23]\n" - ".inst 0x658abc00 // bfcvt z0.h, p7/M, z0.s\n" - ".inst 0x658abc21 // bfcvt z1.h, p7/M, z1.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z4.h }, p7/Z, [x28]\n" - "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n" - "uzp1 z1.h, z1.h, z1.h\n" - "trn1 z0.d, z0.d, z1.d\n" - "ld1h { z6.h }, p7/Z, [x28, #2, MUL VL]\n" - "ld1h { z7.h }, p7/Z, [x28, #3, MUL VL]\n" - ".inst 0x6464e408 // bfmmla z8.s, z0.h, z4.h\n" - ".inst 0x6465e40e // bfmmla z14.s, z0.h, z5.h\n" - "ld1h { z4.h }, p7/Z, [x28, #4, MUL VL]\n" - "ld1h { z5.h }, p7/Z, [x28, #5, MUL VL]\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" - "ld1h { z6.h }, p7/Z, [x28, #6, MUL VL]\n" - "ld1h { z7.h }, p7/Z, [x28, #7, MUL VL]\n" + "ld1rqw { z24.s }, p0/Z, [x24]\n" + "ld1rqw { z20.s }, p0/Z, [x23]\n" + ".inst 0x658abf18 // bfcvt z24.h, p7/M, z24.s\n" + ".inst 0x658abe94 // bfcvt z20.h, p7/M, z20.s\n" + "uzp1 z24.h, z24.h, z24.h\n" + "ld1h { z23.h }, p7/Z, [x28]\n" + "ld1h { z22.h }, p7/Z, [x28, #1, MUL VL]\n" + "uzp1 z20.h, z20.h, z20.h\n" + "trn1 z24.d, z24.d, z20.d\n" + "ld1h { z21.h }, p7/Z, [x28, #2, MUL VL]\n" + "ld1h { z20.h }, p7/Z, [x28, #3, MUL VL]\n" + ".inst 0x6477e708 // bfmmla z8.s, z24.h, z23.h\n" + ".inst 0x6476e70e // bfmmla z14.s, z24.h, z22.h\n" + "ld1h { z23.h }, p7/Z, [x28, #4, MUL VL]\n" + "ld1h { z22.h }, p7/Z, [x28, #5, MUL VL]\n" + ".inst 0x6475e709 // bfmmla z9.s, z24.h, z21.h\n" + ".inst 0x6474e70f // bfmmla z15.s, z24.h, z20.h\n" + "ld1h { z21.h }, p7/Z, [x28, #6, MUL VL]\n" + "ld1h { z20.h }, p7/Z, [x28, #7, MUL VL]\n" "addvl x28, x28, #16\n" - ".inst 0x6464e40a // bfmmla z10.s, z0.h, z4.h\n" - ".inst 0x6465e410 // bfmmla z16.s, z0.h, z5.h\n" - "ld1h { z4.h }, p7/Z, [x28, #-8, MUL VL]\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" - ".inst 0x6467e411 // bfmmla z17.s, z0.h, z7.h\n" - "ld1h { z5.h }, p7/Z, [x28, #-7, MUL VL]\n" - "ld1h { z6.h }, p7/Z, [x28, #-6, MUL VL]\n" - "ld1h { z7.h }, p7/Z, [x28, #-5, MUL VL]\n" + ".inst 0x6477e70a // bfmmla z10.s, z24.h, z23.h\n" + ".inst 0x6476e710 // bfmmla z16.s, z24.h, z22.h\n" + "ld1h { z23.h }, p7/Z, [x28, #-8, MUL VL]\n" + ".inst 0x6475e70b // bfmmla z11.s, z24.h, z21.h\n" + ".inst 0x6474e711 // bfmmla z17.s, z24.h, z20.h\n" + "ld1h { z22.h }, p7/Z, [x28, #-7, MUL VL]\n" + "ld1h { z21.h }, p7/Z, [x28, #-6, MUL VL]\n" + "ld1h { z20.h }, p7/Z, [x28, #-5, MUL VL]\n" "sub x25, x25, #0x4\n" "cmp x25, #0x4\n" - ".inst 0x6464e40c // bfmmla z12.s, z0.h, z4.h\n" - ".inst 0x6465e412 // bfmmla z18.s, z0.h, z5.h\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" + ".inst 0x6477e70c // bfmmla z12.s, z24.h, z23.h\n" + ".inst 0x6476e712 // bfmmla z18.s, z24.h, z22.h\n" + ".inst 0x6475e70d // bfmmla z13.s, z24.h, z21.h\n" "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" - ".inst 0x6467e413 // bfmmla z19.s, z0.h, z7.h\n" + ".inst 0x6474e713 // bfmmla z19.s, z24.h, z20.h\n" "addvl x28, x28, #-4\n" "bgt 22b\n" "23:" // Height 2: Multiply loop: Single iteration only "whilelt p0.s, XZR, x25\n" - "ld1rqw { z0.s }, p0/Z, [x24]\n" - "ld1rqw { z1.s }, p0/Z, [x23]\n" - ".inst 0x658abc00 // bfcvt z0.h, p7/M, z0.s\n" - ".inst 0x658abc21 // bfcvt z1.h, p7/M, z1.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z4.h }, p7/Z, [x28]\n" - "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n" - "uzp1 z1.h, z1.h, z1.h\n" - "trn1 z0.d, z0.d, z1.d\n" - "ld1h { z6.h }, p7/Z, [x28, #2, MUL VL]\n" - "ld1h { z7.h }, p7/Z, [x28, #3, MUL VL]\n" - ".inst 0x6464e408 // bfmmla z8.s, z0.h, z4.h\n" - ".inst 0x6465e40e // bfmmla z14.s, z0.h, z5.h\n" - "ld1h { z4.h }, p7/Z, [x28, #4, MUL VL]\n" - "ld1h { z5.h }, p7/Z, [x28, #5, MUL VL]\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" - "ld1h { z6.h }, p7/Z, [x28, #6, MUL VL]\n" - "ld1h { z7.h }, p7/Z, [x28, #7, MUL VL]\n" + "ld1rqw { z24.s }, p0/Z, [x24]\n" + "ld1rqw { z20.s }, p0/Z, [x23]\n" + ".inst 0x658abf18 // bfcvt z24.h, p7/M, z24.s\n" + ".inst 0x658abe94 // bfcvt z20.h, p7/M, z20.s\n" + "uzp1 z24.h, z24.h, z24.h\n" + "ld1h { z23.h }, p7/Z, [x28]\n" + "ld1h { z22.h }, p7/Z, [x28, #1, MUL VL]\n" + "uzp1 z20.h, z20.h, z20.h\n" + "trn1 z24.d, z24.d, z20.d\n" + "ld1h { z21.h }, p7/Z, [x28, #2, MUL VL]\n" + "ld1h { z20.h }, p7/Z, [x28, #3, MUL VL]\n" + ".inst 0x6477e708 // bfmmla z8.s, z24.h, z23.h\n" + ".inst 0x6476e70e // bfmmla z14.s, z24.h, z22.h\n" + "ld1h { z23.h }, p7/Z, [x28, #4, MUL VL]\n" + "ld1h { z22.h }, p7/Z, [x28, #5, MUL VL]\n" + ".inst 0x6475e709 // bfmmla z9.s, z24.h, z21.h\n" + ".inst 0x6474e70f // bfmmla z15.s, z24.h, z20.h\n" + "ld1h { z21.h }, p7/Z, [x28, #6, MUL VL]\n" + "ld1h { z20.h }, p7/Z, [x28, #7, MUL VL]\n" "addvl x28, x28, #16\n" - ".inst 0x6464e40a // bfmmla z10.s, z0.h, z4.h\n" - ".inst 0x6465e410 // bfmmla z16.s, z0.h, z5.h\n" - "ld1h { z4.h }, p7/Z, [x28, #-8, MUL VL]\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" - ".inst 0x6467e411 // bfmmla z17.s, z0.h, z7.h\n" - "ld1h { z5.h }, p7/Z, [x28, #-7, MUL VL]\n" - "ld1h { z6.h }, p7/Z, [x28, #-6, MUL VL]\n" - "ld1h { z7.h }, p7/Z, [x28, #-5, MUL VL]\n" - ".inst 0x6464e40c // bfmmla z12.s, z0.h, z4.h\n" - ".inst 0x6465e412 // bfmmla z18.s, z0.h, z5.h\n" + ".inst 0x6477e70a // bfmmla z10.s, z24.h, z23.h\n" + ".inst 0x6476e710 // bfmmla z16.s, z24.h, z22.h\n" + "ld1h { z23.h }, p7/Z, [x28, #-8, MUL VL]\n" + ".inst 0x6475e70b // bfmmla z11.s, z24.h, z21.h\n" + ".inst 0x6474e711 // bfmmla z17.s, z24.h, z20.h\n" + "ld1h { z22.h }, p7/Z, [x28, #-7, MUL VL]\n" + "ld1h { z21.h }, p7/Z, [x28, #-6, MUL VL]\n" + "ld1h { z20.h }, p7/Z, [x28, #-5, MUL VL]\n" + ".inst 0x6477e70c // bfmmla z12.s, z24.h, z23.h\n" + ".inst 0x6476e712 // bfmmla z18.s, z24.h, z22.h\n" "addvl x28, x28, #-4\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6467e413 // bfmmla z19.s, z0.h, z7.h\n" + ".inst 0x6475e70d // bfmmla z13.s, z24.h, z21.h\n" + ".inst 0x6474e713 // bfmmla z19.s, z24.h, z20.h\n" "24:" // Height 2: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x26, x26, #0x1\n" @@ -497,33 +497,33 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL ( "uzp2 z13.d, z13.d, z19.d\n" "tbz %x[flags], #1, 25f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p7/Z, [x20]\n" + "ld1rw { z20.s }, p7/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p7/Z, [x20]\n" - "fmin z4.s, p7/M, z4.s, z1.s\n" - "fmin z14.s, p7/M, z14.s, z1.s\n" - "fmin z15.s, p7/M, z15.s, z1.s\n" - "fmin z16.s, p7/M, z16.s, z1.s\n" - "fmin z17.s, p7/M, z17.s, z1.s\n" - "fmin z18.s, p7/M, z18.s, z1.s\n" - "fmin z8.s, p7/M, z8.s, z1.s\n" - "fmin z9.s, p7/M, z9.s, z1.s\n" - "fmin z10.s, p7/M, z10.s, z1.s\n" - "fmin z11.s, p7/M, z11.s, z1.s\n" - "fmin z12.s, p7/M, z12.s, z1.s\n" - "fmin z13.s, p7/M, z13.s, z1.s\n" - "fmax z4.s, p7/M, z4.s, z0.s\n" - "fmax z14.s, p7/M, z14.s, z0.s\n" - "fmax z15.s, p7/M, z15.s, z0.s\n" - "fmax z16.s, p7/M, z16.s, z0.s\n" - "fmax z17.s, p7/M, z17.s, z0.s\n" - "fmax z18.s, p7/M, z18.s, z0.s\n" - "fmax z8.s, p7/M, z8.s, z0.s\n" - "fmax z9.s, p7/M, z9.s, z0.s\n" - "fmax z10.s, p7/M, z10.s, z0.s\n" - "fmax z11.s, p7/M, z11.s, z0.s\n" - "fmax z12.s, p7/M, z12.s, z0.s\n" - "fmax z13.s, p7/M, z13.s, z0.s\n" + "ld1rw { z19.s }, p7/Z, [x20]\n" + "fmin z4.s, p7/M, z4.s, z20.s\n" + "fmin z14.s, p7/M, z14.s, z20.s\n" + "fmin z15.s, p7/M, z15.s, z20.s\n" + "fmin z16.s, p7/M, z16.s, z20.s\n" + "fmin z17.s, p7/M, z17.s, z20.s\n" + "fmin z18.s, p7/M, z18.s, z20.s\n" + "fmin z8.s, p7/M, z8.s, z20.s\n" + "fmin z9.s, p7/M, z9.s, z20.s\n" + "fmin z10.s, p7/M, z10.s, z20.s\n" + "fmin z11.s, p7/M, z11.s, z20.s\n" + "fmin z12.s, p7/M, z12.s, z20.s\n" + "fmin z13.s, p7/M, z13.s, z20.s\n" + "fmax z4.s, p7/M, z4.s, z19.s\n" + "fmax z14.s, p7/M, z14.s, z19.s\n" + "fmax z15.s, p7/M, z15.s, z19.s\n" + "fmax z16.s, p7/M, z16.s, z19.s\n" + "fmax z17.s, p7/M, z17.s, z19.s\n" + "fmax z18.s, p7/M, z18.s, z19.s\n" + "fmax z8.s, p7/M, z8.s, z19.s\n" + "fmax z9.s, p7/M, z9.s, z19.s\n" + "fmax z10.s, p7/M, z10.s, z19.s\n" + "fmax z11.s, p7/M, z11.s, z19.s\n" + "fmax z12.s, p7/M, z12.s, z19.s\n" + "fmax z13.s, p7/M, z13.s, z19.s\n" "25:" // Height 2: No activation "st1w { z4.s }, p6, [x27]\n" "st1w { z14.s }, p5, [x27, #1, MUL VL]\n" @@ -597,38 +597,38 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL ( "29:" // Height 3: no bias "tbz %x[flags], #0, 30f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x23, x27, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "ld1w { z9.s }, p6/Z, [x27]\n" - "ld1w { z10.s }, p5/Z, [x27, #1, MUL VL]\n" - "ld1w { z11.s }, p4/Z, [x27, #2, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x27, #3, MUL VL]\n" - "ld1w { z13.s }, p2/Z, [x27, #4, MUL VL]\n" + "add x21, x27, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" + "ld1w { z16.s }, p6/Z, [x27]\n" + "ld1w { z17.s }, p5/Z, [x27, #1, MUL VL]\n" + "ld1w { z19.s }, p4/Z, [x27, #2, MUL VL]\n" + "ld1w { z22.s }, p3/Z, [x27, #3, MUL VL]\n" + "ld1w { z24.s }, p2/Z, [x27, #4, MUL VL]\n" "ld1w { z20.s }, p1/Z, [x27, #5, MUL VL]\n" - "ld1w { z14.s }, p6/Z, [x23]\n" - "zip1 z8.d, z9.d, z14.d\n" - "zip2 z14.d, z9.d, z14.d\n" - "ld1w { z15.s }, p5/Z, [x23, #1, MUL VL]\n" - "ld1w { z16.s }, p4/Z, [x23, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z15.d\n" - "zip2 z15.d, z10.d, z15.d\n" - "ld1w { z17.s }, p3/Z, [x23, #3, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x23, #4, MUL VL]\n" - "zip1 z10.d, z11.d, z16.d\n" - "zip2 z16.d, z11.d, z16.d\n" - "ld1w { z19.s }, p1/Z, [x23, #5, MUL VL]\n" - "ld1w { z21.s }, p6/Z, [x22]\n" - "zip1 z11.d, z12.d, z17.d\n" - "zip2 z17.d, z12.d, z17.d\n" - "ld1w { z22.s }, p5/Z, [x22, #1, MUL VL]\n" - "ld1w { z23.s }, p4/Z, [x22, #2, MUL VL]\n" - "zip1 z12.d, z13.d, z18.d\n" - "zip2 z18.d, z13.d, z18.d\n" - "ld1w { z24.s }, p3/Z, [x22, #3, MUL VL]\n" - "ld1w { z25.s }, p2/Z, [x22, #4, MUL VL]\n" + "ld1w { z14.s }, p6/Z, [x21]\n" + "zip1 z8.d, z16.d, z14.d\n" + "zip2 z14.d, z16.d, z14.d\n" + "ld1w { z15.s }, p5/Z, [x21, #1, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x21, #2, MUL VL]\n" + "zip1 z9.d, z17.d, z15.d\n" + "zip2 z15.d, z17.d, z15.d\n" + "ld1w { z17.s }, p3/Z, [x21, #3, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x21, #4, MUL VL]\n" + "zip1 z10.d, z19.d, z16.d\n" + "zip2 z16.d, z19.d, z16.d\n" + "ld1w { z19.s }, p1/Z, [x21, #5, MUL VL]\n" + "ld1w { z21.s }, p6/Z, [x20]\n" + "zip1 z11.d, z22.d, z17.d\n" + "zip2 z17.d, z22.d, z17.d\n" + "ld1w { z22.s }, p5/Z, [x20, #1, MUL VL]\n" + "ld1w { z23.s }, p4/Z, [x20, #2, MUL VL]\n" + "zip1 z12.d, z24.d, z18.d\n" + "zip2 z18.d, z24.d, z18.d\n" + "ld1w { z24.s }, p3/Z, [x20, #3, MUL VL]\n" + "ld1w { z25.s }, p2/Z, [x20, #4, MUL VL]\n" "zip1 z13.d, z20.d, z19.d\n" "zip2 z19.d, z20.d, z19.d\n" - "ld1w { z4.s }, p1/Z, [x22, #5, MUL VL]\n" + "ld1w { z0.s }, p1/Z, [x20, #5, MUL VL]\n" "zip1 z20.d, z21.d, z26.d\n" "zip2 z26.d, z21.d, z26.d\n" "zip1 z21.d, z22.d, z27.d\n" @@ -639,8 +639,8 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL ( "zip2 z29.d, z24.d, z29.d\n" "zip1 z24.d, z25.d, z30.d\n" "zip2 z30.d, z25.d, z30.d\n" - "zip1 z25.d, z4.d, z31.d\n" - "zip2 z31.d, z4.d, z31.d\n" + "zip1 z25.d, z0.d, z31.d\n" + "zip2 z31.d, z0.d, z31.d\n" "b 31f\n" "30:" // Height 3: no accumulate "mov z8.b, #0x0\n" @@ -672,13 +672,13 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL ( "32:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 33f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" "cbnz x26, 34f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20, LSL #2\n" @@ -687,117 +687,117 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL ( "b 34f\n" "33:" // Height 3: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" "34:" // Height 3: input setup done "cmp x25, #0x4\n" "ble 36f\n" "35:" // Height 3: Multiply loop: Main loop head "whilelt p0.s, XZR, x25\n" - "ld1rqw { z0.s }, p0/Z, [x24]\n" - "ld1rqw { z1.s }, p0/Z, [x23]\n" + "ld1rqw { z5.s }, p0/Z, [x24]\n" + "ld1rqw { z0.s }, p0/Z, [x23]\n" + ".inst 0x658abca5 // bfcvt z5.h, p7/M, z5.s\n" + "ld1rqw { z4.s }, p0/Z, [x22]\n" ".inst 0x658abc00 // bfcvt z0.h, p7/M, z0.s\n" - "ld1rqw { z2.s }, p0/Z, [x22]\n" - ".inst 0x658abc21 // bfcvt z1.h, p7/M, z1.s\n" + "uzp1 z5.h, z5.h, z5.h\n" + "ld1h { z3.h }, p7/Z, [x28]\n" "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z4.h }, p7/Z, [x28]\n" - "uzp1 z1.h, z1.h, z1.h\n" - ".inst 0x658abc42 // bfcvt z2.h, p7/M, z2.s\n" - "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n" - "ld1h { z6.h }, p7/Z, [x28, #2, MUL VL]\n" - "trn1 z0.d, z0.d, z1.d\n" - "uzp1 z2.h, z2.h, z2.h\n" - "ld1h { z7.h }, p7/Z, [x28, #3, MUL VL]\n" - ".inst 0x6464e408 // bfmmla z8.s, z0.h, z4.h\n" - ".inst 0x6464e454 // bfmmla z20.s, z2.h, z4.h\n" - ".inst 0x6465e40e // bfmmla z14.s, z0.h, z5.h\n" - "ld1h { z4.h }, p7/Z, [x28, #4, MUL VL]\n" + ".inst 0x658abc84 // bfcvt z4.h, p7/M, z4.s\n" + "ld1h { z2.h }, p7/Z, [x28, #1, MUL VL]\n" + "ld1h { z1.h }, p7/Z, [x28, #2, MUL VL]\n" + "trn1 z5.d, z5.d, z0.d\n" + "uzp1 z4.h, z4.h, z4.h\n" + "ld1h { z0.h }, p7/Z, [x28, #3, MUL VL]\n" + ".inst 0x6463e4a8 // bfmmla z8.s, z5.h, z3.h\n" + ".inst 0x6463e494 // bfmmla z20.s, z4.h, z3.h\n" + ".inst 0x6462e4ae // bfmmla z14.s, z5.h, z2.h\n" + "ld1h { z3.h }, p7/Z, [x28, #4, MUL VL]\n" "sub x25, x25, #0x4\n" - ".inst 0x6465e45a // bfmmla z26.s, z2.h, z5.h\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - "ld1h { z5.h }, p7/Z, [x28, #5, MUL VL]\n" + ".inst 0x6462e49a // bfmmla z26.s, z4.h, z2.h\n" + ".inst 0x6461e4a9 // bfmmla z9.s, z5.h, z1.h\n" + "ld1h { z2.h }, p7/Z, [x28, #5, MUL VL]\n" "cmp x25, #0x4\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" - "ld1h { z6.h }, p7/Z, [x28, #6, MUL VL]\n" + ".inst 0x6461e495 // bfmmla z21.s, z4.h, z1.h\n" + ".inst 0x6460e4af // bfmmla z15.s, z5.h, z0.h\n" + "ld1h { z1.h }, p7/Z, [x28, #6, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x6467e45b // bfmmla z27.s, z2.h, z7.h\n" - "ld1h { z7.h }, p7/Z, [x28, #7, MUL VL]\n" + ".inst 0x6460e49b // bfmmla z27.s, z4.h, z0.h\n" + "ld1h { z0.h }, p7/Z, [x28, #7, MUL VL]\n" "addvl x28, x28, #16\n" - ".inst 0x6464e40a // bfmmla z10.s, z0.h, z4.h\n" - ".inst 0x6464e456 // bfmmla z22.s, z2.h, z4.h\n" - ".inst 0x6465e410 // bfmmla z16.s, z0.h, z5.h\n" - "ld1h { z4.h }, p7/Z, [x28, #-8, MUL VL]\n" + ".inst 0x6463e4aa // bfmmla z10.s, z5.h, z3.h\n" + ".inst 0x6463e496 // bfmmla z22.s, z4.h, z3.h\n" + ".inst 0x6462e4b0 // bfmmla z16.s, z5.h, z2.h\n" + "ld1h { z3.h }, p7/Z, [x28, #-8, MUL VL]\n" "add x23, x23, #0x10\n" - ".inst 0x6465e45c // bfmmla z28.s, z2.h, z5.h\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" - "ld1h { z5.h }, p7/Z, [x28, #-7, MUL VL]\n" + ".inst 0x6462e49c // bfmmla z28.s, z4.h, z2.h\n" + ".inst 0x6461e4ab // bfmmla z11.s, z5.h, z1.h\n" + "ld1h { z2.h }, p7/Z, [x28, #-7, MUL VL]\n" "add x22, x22, #0x10\n" - ".inst 0x6466e457 // bfmmla z23.s, z2.h, z6.h\n" - ".inst 0x6467e411 // bfmmla z17.s, z0.h, z7.h\n" - "ld1h { z6.h }, p7/Z, [x28, #-6, MUL VL]\n" - ".inst 0x6467e45d // bfmmla z29.s, z2.h, z7.h\n" - "ld1h { z7.h }, p7/Z, [x28, #-5, MUL VL]\n" - ".inst 0x6464e40c // bfmmla z12.s, z0.h, z4.h\n" + ".inst 0x6461e497 // bfmmla z23.s, z4.h, z1.h\n" + ".inst 0x6460e4b1 // bfmmla z17.s, z5.h, z0.h\n" + "ld1h { z1.h }, p7/Z, [x28, #-6, MUL VL]\n" + ".inst 0x6460e49d // bfmmla z29.s, z4.h, z0.h\n" + "ld1h { z0.h }, p7/Z, [x28, #-5, MUL VL]\n" + ".inst 0x6463e4ac // bfmmla z12.s, z5.h, z3.h\n" "addvl x28, x28, #-4\n" - ".inst 0x6464e458 // bfmmla z24.s, z2.h, z4.h\n" - ".inst 0x6465e412 // bfmmla z18.s, z0.h, z5.h\n" - ".inst 0x6465e45e // bfmmla z30.s, z2.h, z5.h\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6466e459 // bfmmla z25.s, z2.h, z6.h\n" - ".inst 0x6467e413 // bfmmla z19.s, z0.h, z7.h\n" - ".inst 0x6467e45f // bfmmla z31.s, z2.h, z7.h\n" + ".inst 0x6463e498 // bfmmla z24.s, z4.h, z3.h\n" + ".inst 0x6462e4b2 // bfmmla z18.s, z5.h, z2.h\n" + ".inst 0x6462e49e // bfmmla z30.s, z4.h, z2.h\n" + ".inst 0x6461e4ad // bfmmla z13.s, z5.h, z1.h\n" + ".inst 0x6461e499 // bfmmla z25.s, z4.h, z1.h\n" + ".inst 0x6460e4b3 // bfmmla z19.s, z5.h, z0.h\n" + ".inst 0x6460e49f // bfmmla z31.s, z4.h, z0.h\n" "bgt 35b\n" "36:" // Height 3: Multiply loop: Single iteration only "whilelt p0.s, XZR, x25\n" - "ld1rqw { z0.s }, p0/Z, [x24]\n" - "ld1rqw { z1.s }, p0/Z, [x23]\n" + "ld1rqw { z5.s }, p0/Z, [x24]\n" + "ld1rqw { z0.s }, p0/Z, [x23]\n" + ".inst 0x658abca5 // bfcvt z5.h, p7/M, z5.s\n" + "ld1rqw { z4.s }, p0/Z, [x22]\n" ".inst 0x658abc00 // bfcvt z0.h, p7/M, z0.s\n" - "ld1rqw { z2.s }, p0/Z, [x22]\n" - ".inst 0x658abc21 // bfcvt z1.h, p7/M, z1.s\n" + "uzp1 z5.h, z5.h, z5.h\n" + "ld1h { z3.h }, p7/Z, [x28]\n" "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z4.h }, p7/Z, [x28]\n" - "uzp1 z1.h, z1.h, z1.h\n" - ".inst 0x658abc42 // bfcvt z2.h, p7/M, z2.s\n" - "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n" - "ld1h { z6.h }, p7/Z, [x28, #2, MUL VL]\n" - "trn1 z0.d, z0.d, z1.d\n" - "uzp1 z2.h, z2.h, z2.h\n" - "ld1h { z7.h }, p7/Z, [x28, #3, MUL VL]\n" - ".inst 0x6464e408 // bfmmla z8.s, z0.h, z4.h\n" - ".inst 0x6464e454 // bfmmla z20.s, z2.h, z4.h\n" - ".inst 0x6465e40e // bfmmla z14.s, z0.h, z5.h\n" - "ld1h { z4.h }, p7/Z, [x28, #4, MUL VL]\n" - ".inst 0x6465e45a // bfmmla z26.s, z2.h, z5.h\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - "ld1h { z5.h }, p7/Z, [x28, #5, MUL VL]\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" - "ld1h { z6.h }, p7/Z, [x28, #6, MUL VL]\n" - ".inst 0x6467e45b // bfmmla z27.s, z2.h, z7.h\n" - "ld1h { z7.h }, p7/Z, [x28, #7, MUL VL]\n" + ".inst 0x658abc84 // bfcvt z4.h, p7/M, z4.s\n" + "ld1h { z2.h }, p7/Z, [x28, #1, MUL VL]\n" + "ld1h { z1.h }, p7/Z, [x28, #2, MUL VL]\n" + "trn1 z5.d, z5.d, z0.d\n" + "uzp1 z4.h, z4.h, z4.h\n" + "ld1h { z0.h }, p7/Z, [x28, #3, MUL VL]\n" + ".inst 0x6463e4a8 // bfmmla z8.s, z5.h, z3.h\n" + ".inst 0x6463e494 // bfmmla z20.s, z4.h, z3.h\n" + ".inst 0x6462e4ae // bfmmla z14.s, z5.h, z2.h\n" + "ld1h { z3.h }, p7/Z, [x28, #4, MUL VL]\n" + ".inst 0x6462e49a // bfmmla z26.s, z4.h, z2.h\n" + ".inst 0x6461e4a9 // bfmmla z9.s, z5.h, z1.h\n" + "ld1h { z2.h }, p7/Z, [x28, #5, MUL VL]\n" + ".inst 0x6461e495 // bfmmla z21.s, z4.h, z1.h\n" + ".inst 0x6460e4af // bfmmla z15.s, z5.h, z0.h\n" + "ld1h { z1.h }, p7/Z, [x28, #6, MUL VL]\n" + ".inst 0x6460e49b // bfmmla z27.s, z4.h, z0.h\n" + "ld1h { z0.h }, p7/Z, [x28, #7, MUL VL]\n" "addvl x28, x28, #16\n" - ".inst 0x6464e40a // bfmmla z10.s, z0.h, z4.h\n" - ".inst 0x6464e456 // bfmmla z22.s, z2.h, z4.h\n" - ".inst 0x6465e410 // bfmmla z16.s, z0.h, z5.h\n" - "ld1h { z4.h }, p7/Z, [x28, #-8, MUL VL]\n" - ".inst 0x6465e45c // bfmmla z28.s, z2.h, z5.h\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" - "ld1h { z5.h }, p7/Z, [x28, #-7, MUL VL]\n" - ".inst 0x6466e457 // bfmmla z23.s, z2.h, z6.h\n" - ".inst 0x6467e411 // bfmmla z17.s, z0.h, z7.h\n" - "ld1h { z6.h }, p7/Z, [x28, #-6, MUL VL]\n" - ".inst 0x6467e45d // bfmmla z29.s, z2.h, z7.h\n" - "ld1h { z7.h }, p7/Z, [x28, #-5, MUL VL]\n" - ".inst 0x6464e40c // bfmmla z12.s, z0.h, z4.h\n" + ".inst 0x6463e4aa // bfmmla z10.s, z5.h, z3.h\n" + ".inst 0x6463e496 // bfmmla z22.s, z4.h, z3.h\n" + ".inst 0x6462e4b0 // bfmmla z16.s, z5.h, z2.h\n" + "ld1h { z3.h }, p7/Z, [x28, #-8, MUL VL]\n" + ".inst 0x6462e49c // bfmmla z28.s, z4.h, z2.h\n" + ".inst 0x6461e4ab // bfmmla z11.s, z5.h, z1.h\n" + "ld1h { z2.h }, p7/Z, [x28, #-7, MUL VL]\n" + ".inst 0x6461e497 // bfmmla z23.s, z4.h, z1.h\n" + ".inst 0x6460e4b1 // bfmmla z17.s, z5.h, z0.h\n" + "ld1h { z1.h }, p7/Z, [x28, #-6, MUL VL]\n" + ".inst 0x6460e49d // bfmmla z29.s, z4.h, z0.h\n" + "ld1h { z0.h }, p7/Z, [x28, #-5, MUL VL]\n" + ".inst 0x6463e4ac // bfmmla z12.s, z5.h, z3.h\n" "addvl x28, x28, #-4\n" - ".inst 0x6464e458 // bfmmla z24.s, z2.h, z4.h\n" - ".inst 0x6465e412 // bfmmla z18.s, z0.h, z5.h\n" - ".inst 0x6465e45e // bfmmla z30.s, z2.h, z5.h\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6466e459 // bfmmla z25.s, z2.h, z6.h\n" - ".inst 0x6467e413 // bfmmla z19.s, z0.h, z7.h\n" - ".inst 0x6467e45f // bfmmla z31.s, z2.h, z7.h\n" + ".inst 0x6463e498 // bfmmla z24.s, z4.h, z3.h\n" + ".inst 0x6462e4b2 // bfmmla z18.s, z5.h, z2.h\n" + ".inst 0x6462e49e // bfmmla z30.s, z4.h, z2.h\n" + ".inst 0x6461e4ad // bfmmla z13.s, z5.h, z1.h\n" + ".inst 0x6461e499 // bfmmla z25.s, z4.h, z1.h\n" + ".inst 0x6460e4b3 // bfmmla z19.s, z5.h, z0.h\n" + ".inst 0x6460e49f // bfmmla z31.s, z4.h, z0.h\n" "37:" // Height 3: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x26, x26, #0x1\n" @@ -826,45 +826,45 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL ( "uzp1 z25.d, z25.d, z31.d\n" "tbz %x[flags], #1, 38f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p7/Z, [x20]\n" - "add x20, %x[args_ptr], %[offset_min]\n" "ld1rw { z0.s }, p7/Z, [x20]\n" - "fmin z4.s, p7/M, z4.s, z1.s\n" - "fmin z14.s, p7/M, z14.s, z1.s\n" - "fmin z15.s, p7/M, z15.s, z1.s\n" - "fmin z16.s, p7/M, z16.s, z1.s\n" - "fmin z17.s, p7/M, z17.s, z1.s\n" - "fmin z18.s, p7/M, z18.s, z1.s\n" - "fmin z8.s, p7/M, z8.s, z1.s\n" - "fmin z9.s, p7/M, z9.s, z1.s\n" - "fmin z10.s, p7/M, z10.s, z1.s\n" - "fmin z11.s, p7/M, z11.s, z1.s\n" - "fmin z12.s, p7/M, z12.s, z1.s\n" - "fmin z13.s, p7/M, z13.s, z1.s\n" - "fmin z20.s, p7/M, z20.s, z1.s\n" - "fmin z21.s, p7/M, z21.s, z1.s\n" - "fmin z22.s, p7/M, z22.s, z1.s\n" - "fmin z23.s, p7/M, z23.s, z1.s\n" - "fmin z24.s, p7/M, z24.s, z1.s\n" - "fmin z25.s, p7/M, z25.s, z1.s\n" - "fmax z4.s, p7/M, z4.s, z0.s\n" - "fmax z14.s, p7/M, z14.s, z0.s\n" - "fmax z15.s, p7/M, z15.s, z0.s\n" - "fmax z16.s, p7/M, z16.s, z0.s\n" - "fmax z17.s, p7/M, z17.s, z0.s\n" - "fmax z18.s, p7/M, z18.s, z0.s\n" - "fmax z8.s, p7/M, z8.s, z0.s\n" - "fmax z9.s, p7/M, z9.s, z0.s\n" - "fmax z10.s, p7/M, z10.s, z0.s\n" - "fmax z11.s, p7/M, z11.s, z0.s\n" - "fmax z12.s, p7/M, z12.s, z0.s\n" - "fmax z13.s, p7/M, z13.s, z0.s\n" - "fmax z20.s, p7/M, z20.s, z0.s\n" - "fmax z21.s, p7/M, z21.s, z0.s\n" - "fmax z22.s, p7/M, z22.s, z0.s\n" - "fmax z23.s, p7/M, z23.s, z0.s\n" - "fmax z24.s, p7/M, z24.s, z0.s\n" - "fmax z25.s, p7/M, z25.s, z0.s\n" + "add x20, %x[args_ptr], %[offset_min]\n" + "ld1rw { z19.s }, p7/Z, [x20]\n" + "fmin z4.s, p7/M, z4.s, z0.s\n" + "fmin z14.s, p7/M, z14.s, z0.s\n" + "fmin z15.s, p7/M, z15.s, z0.s\n" + "fmin z16.s, p7/M, z16.s, z0.s\n" + "fmin z17.s, p7/M, z17.s, z0.s\n" + "fmin z18.s, p7/M, z18.s, z0.s\n" + "fmin z8.s, p7/M, z8.s, z0.s\n" + "fmin z9.s, p7/M, z9.s, z0.s\n" + "fmin z10.s, p7/M, z10.s, z0.s\n" + "fmin z11.s, p7/M, z11.s, z0.s\n" + "fmin z12.s, p7/M, z12.s, z0.s\n" + "fmin z13.s, p7/M, z13.s, z0.s\n" + "fmin z20.s, p7/M, z20.s, z0.s\n" + "fmin z21.s, p7/M, z21.s, z0.s\n" + "fmin z22.s, p7/M, z22.s, z0.s\n" + "fmin z23.s, p7/M, z23.s, z0.s\n" + "fmin z24.s, p7/M, z24.s, z0.s\n" + "fmin z25.s, p7/M, z25.s, z0.s\n" + "fmax z4.s, p7/M, z4.s, z19.s\n" + "fmax z14.s, p7/M, z14.s, z19.s\n" + "fmax z15.s, p7/M, z15.s, z19.s\n" + "fmax z16.s, p7/M, z16.s, z19.s\n" + "fmax z17.s, p7/M, z17.s, z19.s\n" + "fmax z18.s, p7/M, z18.s, z19.s\n" + "fmax z8.s, p7/M, z8.s, z19.s\n" + "fmax z9.s, p7/M, z9.s, z19.s\n" + "fmax z10.s, p7/M, z10.s, z19.s\n" + "fmax z11.s, p7/M, z11.s, z19.s\n" + "fmax z12.s, p7/M, z12.s, z19.s\n" + "fmax z13.s, p7/M, z13.s, z19.s\n" + "fmax z20.s, p7/M, z20.s, z19.s\n" + "fmax z21.s, p7/M, z21.s, z19.s\n" + "fmax z22.s, p7/M, z22.s, z19.s\n" + "fmax z23.s, p7/M, z23.s, z19.s\n" + "fmax z24.s, p7/M, z24.s, z19.s\n" + "fmax z25.s, p7/M, z25.s, z19.s\n" "38:" // Height 3: No activation "st1w { z4.s }, p6, [x27]\n" "st1w { z14.s }, p5, [x27, #1, MUL VL]\n" @@ -947,57 +947,57 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL ( "42:" // Height 4: no bias "tbz %x[flags], #0, 43f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x23, x27, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "ld1w { z9.s }, p6/Z, [x27]\n" + "add x22, x27, x20, LSL #2\n" "add x21, x22, x20, LSL #2\n" - "ld1w { z10.s }, p5/Z, [x27, #1, MUL VL]\n" - "ld1w { z11.s }, p4/Z, [x27, #2, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x27, #3, MUL VL]\n" - "ld1w { z13.s }, p2/Z, [x27, #4, MUL VL]\n" + "ld1w { z16.s }, p6/Z, [x27]\n" + "add x20, x21, x20, LSL #2\n" + "ld1w { z17.s }, p5/Z, [x27, #1, MUL VL]\n" + "ld1w { z19.s }, p4/Z, [x27, #2, MUL VL]\n" + "ld1w { z22.s }, p3/Z, [x27, #3, MUL VL]\n" + "ld1w { z24.s }, p2/Z, [x27, #4, MUL VL]\n" "ld1w { z20.s }, p1/Z, [x27, #5, MUL VL]\n" - "ld1w { z14.s }, p6/Z, [x23]\n" - "zip1 z8.d, z9.d, z14.d\n" - "zip2 z14.d, z9.d, z14.d\n" - "ld1w { z15.s }, p5/Z, [x23, #1, MUL VL]\n" - "ld1w { z16.s }, p4/Z, [x23, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z15.d\n" - "zip2 z15.d, z10.d, z15.d\n" - "ld1w { z17.s }, p3/Z, [x23, #3, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x23, #4, MUL VL]\n" - "zip1 z10.d, z11.d, z16.d\n" - "zip2 z16.d, z11.d, z16.d\n" - "ld1w { z19.s }, p1/Z, [x23, #5, MUL VL]\n" - "ld1w { z21.s }, p6/Z, [x22]\n" - "zip1 z11.d, z12.d, z17.d\n" - "zip2 z17.d, z12.d, z17.d\n" - "ld1w { z22.s }, p5/Z, [x22, #1, MUL VL]\n" - "ld1w { z23.s }, p4/Z, [x22, #2, MUL VL]\n" - "zip1 z12.d, z13.d, z18.d\n" - "zip2 z18.d, z13.d, z18.d\n" - "ld1w { z24.s }, p3/Z, [x22, #3, MUL VL]\n" - "ld1w { z25.s }, p2/Z, [x22, #4, MUL VL]\n" + "ld1w { z14.s }, p6/Z, [x22]\n" + "zip1 z8.d, z16.d, z14.d\n" + "zip2 z14.d, z16.d, z14.d\n" + "ld1w { z15.s }, p5/Z, [x22, #1, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x22, #2, MUL VL]\n" + "zip1 z9.d, z17.d, z15.d\n" + "zip2 z15.d, z17.d, z15.d\n" + "ld1w { z17.s }, p3/Z, [x22, #3, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x22, #4, MUL VL]\n" + "zip1 z10.d, z19.d, z16.d\n" + "zip2 z16.d, z19.d, z16.d\n" + "ld1w { z19.s }, p1/Z, [x22, #5, MUL VL]\n" + "ld1w { z21.s }, p6/Z, [x21]\n" + "zip1 z11.d, z22.d, z17.d\n" + "zip2 z17.d, z22.d, z17.d\n" + "ld1w { z22.s }, p5/Z, [x21, #1, MUL VL]\n" + "ld1w { z23.s }, p4/Z, [x21, #2, MUL VL]\n" + "zip1 z12.d, z24.d, z18.d\n" + "zip2 z18.d, z24.d, z18.d\n" + "ld1w { z24.s }, p3/Z, [x21, #3, MUL VL]\n" + "ld1w { z25.s }, p2/Z, [x21, #4, MUL VL]\n" "zip1 z13.d, z20.d, z19.d\n" "zip2 z19.d, z20.d, z19.d\n" - "ld1w { z4.s }, p1/Z, [x22, #5, MUL VL]\n" - "ld1w { z26.s }, p6/Z, [x21]\n" + "ld1w { z0.s }, p1/Z, [x21, #5, MUL VL]\n" + "ld1w { z26.s }, p6/Z, [x20]\n" "zip1 z20.d, z21.d, z26.d\n" "zip2 z26.d, z21.d, z26.d\n" - "ld1w { z27.s }, p5/Z, [x21, #1, MUL VL]\n" - "ld1w { z28.s }, p4/Z, [x21, #2, MUL VL]\n" + "ld1w { z27.s }, p5/Z, [x20, #1, MUL VL]\n" + "ld1w { z28.s }, p4/Z, [x20, #2, MUL VL]\n" "zip1 z21.d, z22.d, z27.d\n" "zip2 z27.d, z22.d, z27.d\n" - "ld1w { z29.s }, p3/Z, [x21, #3, MUL VL]\n" - "ld1w { z30.s }, p2/Z, [x21, #4, MUL VL]\n" + "ld1w { z29.s }, p3/Z, [x20, #3, MUL VL]\n" + "ld1w { z30.s }, p2/Z, [x20, #4, MUL VL]\n" "zip1 z22.d, z23.d, z28.d\n" "zip2 z28.d, z23.d, z28.d\n" - "ld1w { z31.s }, p1/Z, [x21, #5, MUL VL]\n" + "ld1w { z31.s }, p1/Z, [x20, #5, MUL VL]\n" "zip1 z23.d, z24.d, z29.d\n" "zip2 z29.d, z24.d, z29.d\n" "zip1 z24.d, z25.d, z30.d\n" "zip2 z30.d, z25.d, z30.d\n" - "zip1 z25.d, z4.d, z31.d\n" - "zip2 z31.d, z4.d, z31.d\n" + "zip1 z25.d, z0.d, z31.d\n" + "zip2 z31.d, z0.d, z31.d\n" "b 44f\n" "43:" // Height 4: no accumulate "mov z8.b, #0x0\n" @@ -1029,14 +1029,14 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL ( "45:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 46f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" - "ldr x21, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" + "ldr x21, [x20, #0x18]\n" "cbnz x26, 47f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20, LSL #2\n" @@ -1046,127 +1046,127 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL ( "b 47f\n" "46:" // Height 4: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "add x21, x22, x20, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" + "add x21, x22, x21, LSL #2\n" "47:" // Height 4: input setup done "cmp x25, #0x4\n" "ble 49f\n" "48:" // Height 4: Multiply loop: Main loop head "whilelt p0.s, XZR, x25\n" - "ld1rqw { z0.s }, p0/Z, [x24]\n" - "ld1rqw { z1.s }, p0/Z, [x23]\n" - ".inst 0x658abc00 // bfcvt z0.h, p7/M, z0.s\n" - "ld1rqw { z2.s }, p0/Z, [x22]\n" - "ld1rqw { z3.s }, p0/Z, [x21]\n" - ".inst 0x658abc21 // bfcvt z1.h, p7/M, z1.s\n" - ".inst 0x658abc42 // bfcvt z2.h, p7/M, z2.s\n" - ".inst 0x658abc63 // bfcvt z3.h, p7/M, z3.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z4.h }, p7/Z, [x28]\n" - "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n" - "uzp1 z1.h, z1.h, z1.h\n" - "uzp1 z2.h, z2.h, z2.h\n" - "ld1h { z6.h }, p7/Z, [x28, #2, MUL VL]\n" - "ld1h { z7.h }, p7/Z, [x28, #3, MUL VL]\n" - "uzp1 z3.h, z3.h, z3.h\n" - "trn1 z0.d, z0.d, z1.d\n" - ".inst 0x6464e408 // bfmmla z8.s, z0.h, z4.h\n" + "ld1rqw { z7.s }, p0/Z, [x24]\n" + "ld1rqw { z6.s }, p0/Z, [x23]\n" + ".inst 0x658abce7 // bfcvt z7.h, p7/M, z7.s\n" + "ld1rqw { z5.s }, p0/Z, [x22]\n" + "ld1rqw { z4.s }, p0/Z, [x21]\n" + ".inst 0x658abcc6 // bfcvt z6.h, p7/M, z6.s\n" + ".inst 0x658abca5 // bfcvt z5.h, p7/M, z5.s\n" + ".inst 0x658abc84 // bfcvt z4.h, p7/M, z4.s\n" + "uzp1 z7.h, z7.h, z7.h\n" + "ld1h { z3.h }, p7/Z, [x28]\n" + "ld1h { z2.h }, p7/Z, [x28, #1, MUL VL]\n" + "uzp1 z6.h, z6.h, z6.h\n" + "uzp1 z5.h, z5.h, z5.h\n" + "ld1h { z1.h }, p7/Z, [x28, #2, MUL VL]\n" + "ld1h { z0.h }, p7/Z, [x28, #3, MUL VL]\n" + "uzp1 z4.h, z4.h, z4.h\n" + "trn1 z7.d, z7.d, z6.d\n" + ".inst 0x6463e4e8 // bfmmla z8.s, z7.h, z3.h\n" "sub x25, x25, #0x4\n" - "trn1 z2.d, z2.d, z3.d\n" - ".inst 0x6464e454 // bfmmla z20.s, z2.h, z4.h\n" - ".inst 0x6465e40e // bfmmla z14.s, z0.h, z5.h\n" - "ld1h { z4.h }, p7/Z, [x28, #4, MUL VL]\n" - ".inst 0x6465e45a // bfmmla z26.s, z2.h, z5.h\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - "ld1h { z5.h }, p7/Z, [x28, #5, MUL VL]\n" + "trn1 z5.d, z5.d, z4.d\n" + ".inst 0x6463e4b4 // bfmmla z20.s, z5.h, z3.h\n" + ".inst 0x6462e4ee // bfmmla z14.s, z7.h, z2.h\n" + "ld1h { z3.h }, p7/Z, [x28, #4, MUL VL]\n" + ".inst 0x6462e4ba // bfmmla z26.s, z5.h, z2.h\n" + ".inst 0x6461e4e9 // bfmmla z9.s, z7.h, z1.h\n" + "ld1h { z2.h }, p7/Z, [x28, #5, MUL VL]\n" "cmp x25, #0x4\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" - "ld1h { z6.h }, p7/Z, [x28, #6, MUL VL]\n" + ".inst 0x6461e4b5 // bfmmla z21.s, z5.h, z1.h\n" + ".inst 0x6460e4ef // bfmmla z15.s, z7.h, z0.h\n" + "ld1h { z1.h }, p7/Z, [x28, #6, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x6467e45b // bfmmla z27.s, z2.h, z7.h\n" - "ld1h { z7.h }, p7/Z, [x28, #7, MUL VL]\n" + ".inst 0x6460e4bb // bfmmla z27.s, z5.h, z0.h\n" + "ld1h { z0.h }, p7/Z, [x28, #7, MUL VL]\n" "addvl x28, x28, #16\n" - ".inst 0x6464e40a // bfmmla z10.s, z0.h, z4.h\n" - ".inst 0x6464e456 // bfmmla z22.s, z2.h, z4.h\n" - ".inst 0x6465e410 // bfmmla z16.s, z0.h, z5.h\n" - "ld1h { z4.h }, p7/Z, [x28, #-8, MUL VL]\n" + ".inst 0x6463e4ea // bfmmla z10.s, z7.h, z3.h\n" + ".inst 0x6463e4b6 // bfmmla z22.s, z5.h, z3.h\n" + ".inst 0x6462e4f0 // bfmmla z16.s, z7.h, z2.h\n" + "ld1h { z3.h }, p7/Z, [x28, #-8, MUL VL]\n" "add x23, x23, #0x10\n" - ".inst 0x6465e45c // bfmmla z28.s, z2.h, z5.h\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" - "ld1h { z5.h }, p7/Z, [x28, #-7, MUL VL]\n" + ".inst 0x6462e4bc // bfmmla z28.s, z5.h, z2.h\n" + ".inst 0x6461e4eb // bfmmla z11.s, z7.h, z1.h\n" + "ld1h { z2.h }, p7/Z, [x28, #-7, MUL VL]\n" "add x22, x22, #0x10\n" - ".inst 0x6466e457 // bfmmla z23.s, z2.h, z6.h\n" - ".inst 0x6467e411 // bfmmla z17.s, z0.h, z7.h\n" - "ld1h { z6.h }, p7/Z, [x28, #-6, MUL VL]\n" + ".inst 0x6461e4b7 // bfmmla z23.s, z5.h, z1.h\n" + ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n" + "ld1h { z1.h }, p7/Z, [x28, #-6, MUL VL]\n" "add x21, x21, #0x10\n" - ".inst 0x6467e45d // bfmmla z29.s, z2.h, z7.h\n" - "ld1h { z7.h }, p7/Z, [x28, #-5, MUL VL]\n" - ".inst 0x6464e40c // bfmmla z12.s, z0.h, z4.h\n" + ".inst 0x6460e4bd // bfmmla z29.s, z5.h, z0.h\n" + "ld1h { z0.h }, p7/Z, [x28, #-5, MUL VL]\n" + ".inst 0x6463e4ec // bfmmla z12.s, z7.h, z3.h\n" "addvl x28, x28, #-4\n" - ".inst 0x6464e458 // bfmmla z24.s, z2.h, z4.h\n" - ".inst 0x6465e412 // bfmmla z18.s, z0.h, z5.h\n" - ".inst 0x6465e45e // bfmmla z30.s, z2.h, z5.h\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6466e459 // bfmmla z25.s, z2.h, z6.h\n" - ".inst 0x6467e413 // bfmmla z19.s, z0.h, z7.h\n" - ".inst 0x6467e45f // bfmmla z31.s, z2.h, z7.h\n" + ".inst 0x6463e4b8 // bfmmla z24.s, z5.h, z3.h\n" + ".inst 0x6462e4f2 // bfmmla z18.s, z7.h, z2.h\n" + ".inst 0x6462e4be // bfmmla z30.s, z5.h, z2.h\n" + ".inst 0x6461e4ed // bfmmla z13.s, z7.h, z1.h\n" + ".inst 0x6461e4b9 // bfmmla z25.s, z5.h, z1.h\n" + ".inst 0x6460e4f3 // bfmmla z19.s, z7.h, z0.h\n" + ".inst 0x6460e4bf // bfmmla z31.s, z5.h, z0.h\n" "bgt 48b\n" "49:" // Height 4: Multiply loop: Single iteration only "whilelt p0.s, XZR, x25\n" - "ld1rqw { z0.s }, p0/Z, [x24]\n" - "ld1rqw { z1.s }, p0/Z, [x23]\n" - ".inst 0x658abc00 // bfcvt z0.h, p7/M, z0.s\n" - "ld1rqw { z2.s }, p0/Z, [x22]\n" - "ld1rqw { z3.s }, p0/Z, [x21]\n" - ".inst 0x658abc21 // bfcvt z1.h, p7/M, z1.s\n" - ".inst 0x658abc42 // bfcvt z2.h, p7/M, z2.s\n" - ".inst 0x658abc63 // bfcvt z3.h, p7/M, z3.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z4.h }, p7/Z, [x28]\n" - "ld1h { z5.h }, p7/Z, [x28, #1, MUL VL]\n" - "uzp1 z1.h, z1.h, z1.h\n" - "uzp1 z2.h, z2.h, z2.h\n" - "ld1h { z6.h }, p7/Z, [x28, #2, MUL VL]\n" - "ld1h { z7.h }, p7/Z, [x28, #3, MUL VL]\n" - "uzp1 z3.h, z3.h, z3.h\n" - "trn1 z0.d, z0.d, z1.d\n" - ".inst 0x6464e408 // bfmmla z8.s, z0.h, z4.h\n" - "trn1 z2.d, z2.d, z3.d\n" - ".inst 0x6464e454 // bfmmla z20.s, z2.h, z4.h\n" - ".inst 0x6465e40e // bfmmla z14.s, z0.h, z5.h\n" - "ld1h { z4.h }, p7/Z, [x28, #4, MUL VL]\n" - ".inst 0x6465e45a // bfmmla z26.s, z2.h, z5.h\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - "ld1h { z5.h }, p7/Z, [x28, #5, MUL VL]\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" - "ld1h { z6.h }, p7/Z, [x28, #6, MUL VL]\n" - ".inst 0x6467e45b // bfmmla z27.s, z2.h, z7.h\n" - "ld1h { z7.h }, p7/Z, [x28, #7, MUL VL]\n" + "ld1rqw { z7.s }, p0/Z, [x24]\n" + "ld1rqw { z6.s }, p0/Z, [x23]\n" + ".inst 0x658abce7 // bfcvt z7.h, p7/M, z7.s\n" + "ld1rqw { z5.s }, p0/Z, [x22]\n" + "ld1rqw { z4.s }, p0/Z, [x21]\n" + ".inst 0x658abcc6 // bfcvt z6.h, p7/M, z6.s\n" + ".inst 0x658abca5 // bfcvt z5.h, p7/M, z5.s\n" + ".inst 0x658abc84 // bfcvt z4.h, p7/M, z4.s\n" + "uzp1 z7.h, z7.h, z7.h\n" + "ld1h { z3.h }, p7/Z, [x28]\n" + "ld1h { z2.h }, p7/Z, [x28, #1, MUL VL]\n" + "uzp1 z6.h, z6.h, z6.h\n" + "uzp1 z5.h, z5.h, z5.h\n" + "ld1h { z1.h }, p7/Z, [x28, #2, MUL VL]\n" + "ld1h { z0.h }, p7/Z, [x28, #3, MUL VL]\n" + "uzp1 z4.h, z4.h, z4.h\n" + "trn1 z7.d, z7.d, z6.d\n" + ".inst 0x6463e4e8 // bfmmla z8.s, z7.h, z3.h\n" + "trn1 z5.d, z5.d, z4.d\n" + ".inst 0x6463e4b4 // bfmmla z20.s, z5.h, z3.h\n" + ".inst 0x6462e4ee // bfmmla z14.s, z7.h, z2.h\n" + "ld1h { z3.h }, p7/Z, [x28, #4, MUL VL]\n" + ".inst 0x6462e4ba // bfmmla z26.s, z5.h, z2.h\n" + ".inst 0x6461e4e9 // bfmmla z9.s, z7.h, z1.h\n" + "ld1h { z2.h }, p7/Z, [x28, #5, MUL VL]\n" + ".inst 0x6461e4b5 // bfmmla z21.s, z5.h, z1.h\n" + ".inst 0x6460e4ef // bfmmla z15.s, z7.h, z0.h\n" + "ld1h { z1.h }, p7/Z, [x28, #6, MUL VL]\n" + ".inst 0x6460e4bb // bfmmla z27.s, z5.h, z0.h\n" + "ld1h { z0.h }, p7/Z, [x28, #7, MUL VL]\n" "addvl x28, x28, #16\n" - ".inst 0x6464e40a // bfmmla z10.s, z0.h, z4.h\n" - ".inst 0x6464e456 // bfmmla z22.s, z2.h, z4.h\n" - ".inst 0x6465e410 // bfmmla z16.s, z0.h, z5.h\n" - "ld1h { z4.h }, p7/Z, [x28, #-8, MUL VL]\n" - ".inst 0x6465e45c // bfmmla z28.s, z2.h, z5.h\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" - "ld1h { z5.h }, p7/Z, [x28, #-7, MUL VL]\n" - ".inst 0x6466e457 // bfmmla z23.s, z2.h, z6.h\n" - ".inst 0x6467e411 // bfmmla z17.s, z0.h, z7.h\n" - "ld1h { z6.h }, p7/Z, [x28, #-6, MUL VL]\n" - ".inst 0x6467e45d // bfmmla z29.s, z2.h, z7.h\n" - "ld1h { z7.h }, p7/Z, [x28, #-5, MUL VL]\n" - ".inst 0x6464e40c // bfmmla z12.s, z0.h, z4.h\n" + ".inst 0x6463e4ea // bfmmla z10.s, z7.h, z3.h\n" + ".inst 0x6463e4b6 // bfmmla z22.s, z5.h, z3.h\n" + ".inst 0x6462e4f0 // bfmmla z16.s, z7.h, z2.h\n" + "ld1h { z3.h }, p7/Z, [x28, #-8, MUL VL]\n" + ".inst 0x6462e4bc // bfmmla z28.s, z5.h, z2.h\n" + ".inst 0x6461e4eb // bfmmla z11.s, z7.h, z1.h\n" + "ld1h { z2.h }, p7/Z, [x28, #-7, MUL VL]\n" + ".inst 0x6461e4b7 // bfmmla z23.s, z5.h, z1.h\n" + ".inst 0x6460e4f1 // bfmmla z17.s, z7.h, z0.h\n" + "ld1h { z1.h }, p7/Z, [x28, #-6, MUL VL]\n" + ".inst 0x6460e4bd // bfmmla z29.s, z5.h, z0.h\n" + "ld1h { z0.h }, p7/Z, [x28, #-5, MUL VL]\n" + ".inst 0x6463e4ec // bfmmla z12.s, z7.h, z3.h\n" "addvl x28, x28, #-4\n" - ".inst 0x6464e458 // bfmmla z24.s, z2.h, z4.h\n" - ".inst 0x6465e412 // bfmmla z18.s, z0.h, z5.h\n" - ".inst 0x6465e45e // bfmmla z30.s, z2.h, z5.h\n" - ".inst 0x6466e40d // bfmmla z13.s, z0.h, z6.h\n" - ".inst 0x6466e459 // bfmmla z25.s, z2.h, z6.h\n" - ".inst 0x6467e413 // bfmmla z19.s, z0.h, z7.h\n" - ".inst 0x6467e45f // bfmmla z31.s, z2.h, z7.h\n" + ".inst 0x6463e4b8 // bfmmla z24.s, z5.h, z3.h\n" + ".inst 0x6462e4f2 // bfmmla z18.s, z7.h, z2.h\n" + ".inst 0x6462e4be // bfmmla z30.s, z5.h, z2.h\n" + ".inst 0x6461e4ed // bfmmla z13.s, z7.h, z1.h\n" + ".inst 0x6461e4b9 // bfmmla z25.s, z5.h, z1.h\n" + ".inst 0x6460e4f3 // bfmmla z19.s, z7.h, z0.h\n" + ".inst 0x6460e4bf // bfmmla z31.s, z5.h, z0.h\n" "50:" // Height 4: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x26, x26, #0x1\n" @@ -1295,7 +1295,6 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "54:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -1303,4 +1302,4 @@ void sve_hybrid_fp32bf16fp32_mmla_4x6VL ( } } // namespace arm_gemm -#endif // __ARM_FEATURE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32bf16fp32_mmla_6x4VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32bf16fp32_mmla_6x4VL.hpp index b7c9aca9dd96d0ee67a1f9917f7cf210437da6cf..15b7dd721c61f230a6834188c90e13aaf31050da 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32bf16fp32_mmla_6x4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32bf16fp32_mmla_6x4VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef ARM_COMPUTE_ENABLE_SVE + #include "../std_transforms_sve.hpp" #include "../bfloat.hpp" #include "../performance_parameters.hpp" @@ -75,7 +75,6 @@ public: template static inline PerformanceParameters get_performance_parameters(const CPUInfo *ci) { - if (std::is_same::value) { switch (ci->get_cpu_model()) { default: @@ -100,5 +99,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32bf16fp32_mmla_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32bf16fp32_mmla_6x4VL/generic.cpp index 23d7ff9c3bc806cdf6c6fc9f2a25afdbac719179..0d2b47ec39cbd041fdbecbf2369efae32593616b 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32bf16fp32_mmla_6x4VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_fp32bf16fp32_mmla_6x4VL/generic.cpp @@ -133,16 +133,16 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( "b 5f\n" "3:" // Height 1: no bias "tbz %x[flags], #0, 4f\n" - "ld1w { z9.s }, p4/Z, [x9]\n" - "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x9]\n" + "ld1w { z18.s }, p3/Z, [x9, #1, MUL VL]\n" + "zip1 z8.d, z16.d, z12.d\n" + "zip2 z12.d, z16.d, z12.d\n" + "ld1w { z17.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" + "zip1 z9.d, z18.d, z13.d\n" + "zip2 z13.d, z18.d, z13.d\n" + "zip1 z10.d, z17.d, z14.d\n" + "zip2 z14.d, z17.d, z14.d\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" "b 5f\n" @@ -160,11 +160,11 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( "6:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 7f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 8f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -176,51 +176,51 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( "ble 10f\n" "9:" // Height 1: Multiply loop: Main loop head "whilelt p0.s, XZR, x27\n" - "ld1rqw { z0.s }, p0/Z, [x26]\n" - ".inst 0x658ab400 // bfcvt z0.h, p5/M, z0.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6466e408 // bfmmla z8.s, z0.h, z6.h\n" - ".inst 0x6467e40c // bfmmla z12.s, z0.h, z7.h\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - ".inst 0x6467e40d // bfmmla z13.s, z0.h, z7.h\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6466e40a // bfmmla z10.s, z0.h, z6.h\n" - ".inst 0x6467e40e // bfmmla z14.s, z0.h, z7.h\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" + "ld1rqw { z18.s }, p0/Z, [x26]\n" + ".inst 0x658ab652 // bfcvt z18.h, p5/M, z18.s\n" + "uzp1 z18.h, z18.h, z18.h\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6471e648 // bfmmla z8.s, z18.h, z17.h\n" + ".inst 0x6470e64c // bfmmla z12.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6471e649 // bfmmla z9.s, z18.h, z17.h\n" + ".inst 0x6470e64d // bfmmla z13.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #4, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6471e64a // bfmmla z10.s, z18.h, z17.h\n" + ".inst 0x6470e64e // bfmmla z14.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #6, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #7, MUL VL]\n" "sub x27, x27, #0x4\n" "cmp x27, #0x4\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" + ".inst 0x6471e64b // bfmmla z11.s, z18.h, z17.h\n" + ".inst 0x6470e64f // bfmmla z15.s, z18.h, z16.h\n" "add x26, x26, #0x10\n" "addvl x10, x10, #8\n" "bgt 9b\n" "10:" // Height 1: Multiply loop: Single iteration only "whilelt p0.s, XZR, x27\n" - "ld1rqw { z0.s }, p0/Z, [x26]\n" - ".inst 0x658ab400 // bfcvt z0.h, p5/M, z0.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x6466e408 // bfmmla z8.s, z0.h, z6.h\n" - ".inst 0x6467e40c // bfmmla z12.s, z0.h, z7.h\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - ".inst 0x6467e40d // bfmmla z13.s, z0.h, z7.h\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6466e40a // bfmmla z10.s, z0.h, z6.h\n" - ".inst 0x6467e40e // bfmmla z14.s, z0.h, z7.h\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" + "ld1rqw { z18.s }, p0/Z, [x26]\n" + ".inst 0x658ab652 // bfcvt z18.h, p5/M, z18.s\n" + "uzp1 z18.h, z18.h, z18.h\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x6471e648 // bfmmla z8.s, z18.h, z17.h\n" + ".inst 0x6470e64c // bfmmla z12.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6471e649 // bfmmla z9.s, z18.h, z17.h\n" + ".inst 0x6470e64d // bfmmla z13.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #4, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6471e64a // bfmmla z10.s, z18.h, z17.h\n" + ".inst 0x6470e64e // bfmmla z14.s, z18.h, z16.h\n" + "ld1h { z17.h }, p5/Z, [x10, #6, MUL VL]\n" + "ld1h { z16.h }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x6471e64b // bfmmla z11.s, z18.h, z17.h\n" + ".inst 0x6470e64f // bfmmla z15.s, z18.h, z16.h\n" "addvl x10, x10, #8\n" "11:" // Height 1: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -233,17 +233,17 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( "uzp1 z11.d, z11.d, z15.d\n" "tbz %x[flags], #1, 12f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z17.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" + "ld1rw { z16.s }, p5/Z, [x20]\n" + "fmin z8.s, p5/M, z8.s, z17.s\n" + "fmin z9.s, p5/M, z9.s, z17.s\n" + "fmin z10.s, p5/M, z10.s, z17.s\n" + "fmin z11.s, p5/M, z11.s, z17.s\n" + "fmax z8.s, p5/M, z8.s, z16.s\n" + "fmax z9.s, p5/M, z9.s, z16.s\n" + "fmax z10.s, p5/M, z10.s, z16.s\n" + "fmax z11.s, p5/M, z11.s, z16.s\n" "12:" // Height 1: No activation "st1w { z8.s }, p4, [x9]\n" "st1w { z9.s }, p3, [x9, #1, MUL VL]\n" @@ -287,21 +287,21 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( "16:" // Height 2: no bias "tbz %x[flags], #0, 17f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x9]\n" - "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n" + "add x20, x9, x20, LSL #2\n" + "ld1w { z19.s }, p4/Z, [x9]\n" + "ld1w { z18.s }, p3/Z, [x9, #1, MUL VL]\n" + "ld1w { z17.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" + "ld1w { z12.s }, p4/Z, [x20]\n" + "zip1 z8.d, z19.d, z12.d\n" + "zip2 z12.d, z19.d, z12.d\n" + "ld1w { z13.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x20, #2, MUL VL]\n" + "zip1 z9.d, z18.d, z13.d\n" + "zip2 z13.d, z18.d, z13.d\n" + "ld1w { z15.s }, p1/Z, [x20, #3, MUL VL]\n" + "zip1 z10.d, z17.d, z14.d\n" + "zip2 z14.d, z17.d, z14.d\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" "b 18f\n" @@ -319,12 +319,12 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( "19:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 20f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 21f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -332,67 +332,67 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( "b 21f\n" "20:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" "21:" // Height 2: input setup done "cmp x27, #0x4\n" "ble 23f\n" "22:" // Height 2: Multiply loop: Main loop head "whilelt p0.s, XZR, x27\n" - "ld1rqw { z0.s }, p0/Z, [x26]\n" - "ld1rqw { z1.s }, p0/Z, [x25]\n" - ".inst 0x658ab400 // bfcvt z0.h, p5/M, z0.s\n" - ".inst 0x658ab421 // bfcvt z1.h, p5/M, z1.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "uzp1 z1.h, z1.h, z1.h\n" - "trn1 z0.d, z0.d, z1.d\n" - ".inst 0x6466e408 // bfmmla z8.s, z0.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x6467e40c // bfmmla z12.s, z0.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x6467e40d // bfmmla z13.s, z0.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6466e40a // bfmmla z10.s, z0.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x6467e40e // bfmmla z14.s, z0.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" + "ld1rqw { z19.s }, p0/Z, [x26]\n" + "ld1rqw { z18.s }, p0/Z, [x25]\n" + ".inst 0x658ab673 // bfcvt z19.h, p5/M, z19.s\n" + ".inst 0x658ab652 // bfcvt z18.h, p5/M, z18.s\n" + "uzp1 z19.h, z19.h, z19.h\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + "uzp1 z18.h, z18.h, z18.h\n" + "trn1 z19.d, z19.d, z18.d\n" + ".inst 0x6471e668 // bfmmla z8.s, z19.h, z17.h\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x6470e66c // bfmmla z12.s, z19.h, z16.h\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6471e669 // bfmmla z9.s, z19.h, z17.h\n" + "ld1h { z17.h }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x6470e66d // bfmmla z13.s, z19.h, z16.h\n" + "ld1h { z16.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6471e66a // bfmmla z10.s, z19.h, z17.h\n" + "ld1h { z17.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x6470e66e // bfmmla z14.s, z19.h, z16.h\n" + "ld1h { z16.h }, p5/Z, [x10, #7, MUL VL]\n" "sub x27, x27, #0x4\n" "cmp x27, #0x4\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" + ".inst 0x6471e66b // bfmmla z11.s, z19.h, z17.h\n" + ".inst 0x6470e66f // bfmmla z15.s, z19.h, z16.h\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" "addvl x10, x10, #8\n" "bgt 22b\n" "23:" // Height 2: Multiply loop: Single iteration only "whilelt p0.s, XZR, x27\n" - "ld1rqw { z0.s }, p0/Z, [x26]\n" - "ld1rqw { z1.s }, p0/Z, [x25]\n" - ".inst 0x658ab400 // bfcvt z0.h, p5/M, z0.s\n" - ".inst 0x658ab421 // bfcvt z1.h, p5/M, z1.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "uzp1 z1.h, z1.h, z1.h\n" - "trn1 z0.d, z0.d, z1.d\n" - ".inst 0x6466e408 // bfmmla z8.s, z0.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x6467e40c // bfmmla z12.s, z0.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x6467e40d // bfmmla z13.s, z0.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6466e40a // bfmmla z10.s, z0.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x6467e40e // bfmmla z14.s, z0.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" + "ld1rqw { z19.s }, p0/Z, [x26]\n" + "ld1rqw { z18.s }, p0/Z, [x25]\n" + ".inst 0x658ab673 // bfcvt z19.h, p5/M, z19.s\n" + ".inst 0x658ab652 // bfcvt z18.h, p5/M, z18.s\n" + "uzp1 z19.h, z19.h, z19.h\n" + "ld1h { z17.h }, p5/Z, [x10]\n" + "ld1h { z16.h }, p5/Z, [x10, #1, MUL VL]\n" + "uzp1 z18.h, z18.h, z18.h\n" + "trn1 z19.d, z19.d, z18.d\n" + ".inst 0x6471e668 // bfmmla z8.s, z19.h, z17.h\n" + "ld1h { z17.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x6470e66c // bfmmla z12.s, z19.h, z16.h\n" + "ld1h { z16.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6471e669 // bfmmla z9.s, z19.h, z17.h\n" + "ld1h { z17.h }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x6470e66d // bfmmla z13.s, z19.h, z16.h\n" + "ld1h { z16.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6471e66a // bfmmla z10.s, z19.h, z17.h\n" + "ld1h { z17.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x6470e66e // bfmmla z14.s, z19.h, z16.h\n" + "ld1h { z16.h }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x6471e66b // bfmmla z11.s, z19.h, z17.h\n" "addvl x10, x10, #8\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" + ".inst 0x6470e66f // bfmmla z15.s, z19.h, z16.h\n" "24:" // Height 2: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -410,25 +410,25 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( "uzp2 z11.d, z11.d, z15.d\n" "tbz %x[flags], #1, 25f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z17.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z6.s, p5/M, z6.s, z1.s\n" - "fmin z12.s, p5/M, z12.s, z1.s\n" - "fmin z13.s, p5/M, z13.s, z1.s\n" - "fmin z14.s, p5/M, z14.s, z1.s\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmax z6.s, p5/M, z6.s, z0.s\n" - "fmax z12.s, p5/M, z12.s, z0.s\n" - "fmax z13.s, p5/M, z13.s, z0.s\n" - "fmax z14.s, p5/M, z14.s, z0.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" + "ld1rw { z16.s }, p5/Z, [x20]\n" + "fmin z6.s, p5/M, z6.s, z17.s\n" + "fmin z12.s, p5/M, z12.s, z17.s\n" + "fmin z13.s, p5/M, z13.s, z17.s\n" + "fmin z14.s, p5/M, z14.s, z17.s\n" + "fmin z8.s, p5/M, z8.s, z17.s\n" + "fmin z9.s, p5/M, z9.s, z17.s\n" + "fmin z10.s, p5/M, z10.s, z17.s\n" + "fmin z11.s, p5/M, z11.s, z17.s\n" + "fmax z6.s, p5/M, z6.s, z16.s\n" + "fmax z12.s, p5/M, z12.s, z16.s\n" + "fmax z13.s, p5/M, z13.s, z16.s\n" + "fmax z14.s, p5/M, z14.s, z16.s\n" + "fmax z8.s, p5/M, z8.s, z16.s\n" + "fmax z9.s, p5/M, z9.s, z16.s\n" + "fmax z10.s, p5/M, z10.s, z16.s\n" + "fmax z11.s, p5/M, z11.s, z16.s\n" "25:" // Height 2: No activation "st1w { z6.s }, p4, [x9]\n" "st1w { z12.s }, p3, [x9, #1, MUL VL]\n" @@ -484,28 +484,28 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( "29:" // Height 3: no bias "tbz %x[flags], #0, 30f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x9]\n" - "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n" + "add x21, x9, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" + "ld1w { z19.s }, p4/Z, [x9]\n" + "ld1w { z17.s }, p3/Z, [x9, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "ld1w { z17.s }, p4/Z, [x24]\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" - "ld1w { z18.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z19.s }, p2/Z, [x24, #2, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x21]\n" + "zip1 z8.d, z19.d, z12.d\n" + "zip2 z12.d, z19.d, z12.d\n" + "ld1w { z13.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x21, #2, MUL VL]\n" + "zip1 z9.d, z17.d, z13.d\n" + "zip2 z13.d, z17.d, z13.d\n" + "ld1w { z15.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z17.s }, p4/Z, [x20]\n" + "zip1 z10.d, z18.d, z14.d\n" + "zip2 z14.d, z18.d, z14.d\n" + "ld1w { z18.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z19.s }, p2/Z, [x20, #2, MUL VL]\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" - "ld1w { z24.s }, p1/Z, [x24, #3, MUL VL]\n" + "ld1w { z24.s }, p1/Z, [x20, #3, MUL VL]\n" "zip1 z16.d, z17.d, z20.d\n" "zip2 z20.d, z17.d, z20.d\n" "zip1 z17.d, z18.d, z21.d\n" @@ -537,13 +537,13 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( "32:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 33f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 34f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -552,91 +552,91 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( "b 34f\n" "33:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" "34:" // Height 3: input setup done "cmp x27, #0x4\n" "ble 36f\n" "35:" // Height 3: Multiply loop: Main loop head "whilelt p0.s, XZR, x27\n" - "ld1rqw { z0.s }, p0/Z, [x26]\n" - "ld1rqw { z1.s }, p0/Z, [x25]\n" - ".inst 0x658ab400 // bfcvt z0.h, p5/M, z0.s\n" - "ld1rqw { z2.s }, p0/Z, [x24]\n" - ".inst 0x658ab421 // bfcvt z1.h, p5/M, z1.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "uzp1 z1.h, z1.h, z1.h\n" - ".inst 0x658ab442 // bfcvt z2.h, p5/M, z2.s\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" + "ld1rqw { z28.s }, p0/Z, [x26]\n" + "ld1rqw { z27.s }, p0/Z, [x25]\n" + ".inst 0x658ab79c // bfcvt z28.h, p5/M, z28.s\n" + "ld1rqw { z26.s }, p0/Z, [x24]\n" + ".inst 0x658ab77b // bfcvt z27.h, p5/M, z27.s\n" + "uzp1 z28.h, z28.h, z28.h\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "uzp1 z27.h, z27.h, z27.h\n" + ".inst 0x658ab75a // bfcvt z26.h, p5/M, z26.s\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" "sub x27, x27, #0x4\n" - "trn1 z0.d, z0.d, z1.d\n" - "uzp1 z2.h, z2.h, z2.h\n" - ".inst 0x6466e408 // bfmmla z8.s, z0.h, z6.h\n" + "trn1 z28.d, z28.d, z27.d\n" + "uzp1 z26.h, z26.h, z26.h\n" + ".inst 0x6479e788 // bfmmla z8.s, z28.h, z25.h\n" "cmp x27, #0x4\n" - ".inst 0x6466e450 // bfmmla z16.s, z2.h, z6.h\n" - ".inst 0x6467e40c // bfmmla z12.s, z0.h, z7.h\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x6479e750 // bfmmla z16.s, z26.h, z25.h\n" + ".inst 0x6478e78c // bfmmla z12.s, z28.h, z24.h\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" "add x26, x26, #0x10\n" - ".inst 0x6467e454 // bfmmla z20.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" + ".inst 0x6478e754 // bfmmla z20.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6479e789 // bfmmla z9.s, z28.h, z25.h\n" "add x25, x25, #0x10\n" - ".inst 0x6466e451 // bfmmla z17.s, z2.h, z6.h\n" - ".inst 0x6467e40d // bfmmla z13.s, z0.h, z7.h\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x6479e751 // bfmmla z17.s, z26.h, z25.h\n" + ".inst 0x6478e78d // bfmmla z13.s, z28.h, z24.h\n" + "ld1h { z25.h }, p5/Z, [x10, #4, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x6467e455 // bfmmla z21.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6466e40a // bfmmla z10.s, z0.h, z6.h\n" - ".inst 0x6466e452 // bfmmla z18.s, z2.h, z6.h\n" - ".inst 0x6467e40e // bfmmla z14.s, z0.h, z7.h\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x6467e456 // bfmmla z22.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" + ".inst 0x6478e755 // bfmmla z21.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6479e78a // bfmmla z10.s, z28.h, z25.h\n" + ".inst 0x6479e752 // bfmmla z18.s, z26.h, z25.h\n" + ".inst 0x6478e78e // bfmmla z14.s, z28.h, z24.h\n" + "ld1h { z25.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x6478e756 // bfmmla z22.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x6479e78b // bfmmla z11.s, z28.h, z25.h\n" "addvl x10, x10, #8\n" - ".inst 0x6466e453 // bfmmla z19.s, z2.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" - ".inst 0x6467e457 // bfmmla z23.s, z2.h, z7.h\n" + ".inst 0x6479e753 // bfmmla z19.s, z26.h, z25.h\n" + ".inst 0x6478e78f // bfmmla z15.s, z28.h, z24.h\n" + ".inst 0x6478e757 // bfmmla z23.s, z26.h, z24.h\n" "bgt 35b\n" "36:" // Height 3: Multiply loop: Single iteration only "whilelt p0.s, XZR, x27\n" - "ld1rqw { z0.s }, p0/Z, [x26]\n" - "ld1rqw { z1.s }, p0/Z, [x25]\n" - ".inst 0x658ab400 // bfcvt z0.h, p5/M, z0.s\n" - "ld1rqw { z2.s }, p0/Z, [x24]\n" - ".inst 0x658ab421 // bfcvt z1.h, p5/M, z1.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "uzp1 z1.h, z1.h, z1.h\n" - ".inst 0x658ab442 // bfcvt z2.h, p5/M, z2.s\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "trn1 z0.d, z0.d, z1.d\n" - "uzp1 z2.h, z2.h, z2.h\n" - ".inst 0x6466e408 // bfmmla z8.s, z0.h, z6.h\n" - ".inst 0x6466e450 // bfmmla z16.s, z2.h, z6.h\n" - ".inst 0x6467e40c // bfmmla z12.s, z0.h, z7.h\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x6467e454 // bfmmla z20.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - ".inst 0x6466e451 // bfmmla z17.s, z2.h, z6.h\n" - ".inst 0x6467e40d // bfmmla z13.s, z0.h, z7.h\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x6467e455 // bfmmla z21.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6466e40a // bfmmla z10.s, z0.h, z6.h\n" - ".inst 0x6466e452 // bfmmla z18.s, z2.h, z6.h\n" - ".inst 0x6467e40e // bfmmla z14.s, z0.h, z7.h\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x6467e456 // bfmmla z22.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" + "ld1rqw { z28.s }, p0/Z, [x26]\n" + "ld1rqw { z27.s }, p0/Z, [x25]\n" + ".inst 0x658ab79c // bfcvt z28.h, p5/M, z28.s\n" + "ld1rqw { z26.s }, p0/Z, [x24]\n" + ".inst 0x658ab77b // bfcvt z27.h, p5/M, z27.s\n" + "uzp1 z28.h, z28.h, z28.h\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "uzp1 z27.h, z27.h, z27.h\n" + ".inst 0x658ab75a // bfcvt z26.h, p5/M, z26.s\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + "trn1 z28.d, z28.d, z27.d\n" + "uzp1 z26.h, z26.h, z26.h\n" + ".inst 0x6479e788 // bfmmla z8.s, z28.h, z25.h\n" + ".inst 0x6479e750 // bfmmla z16.s, z26.h, z25.h\n" + ".inst 0x6478e78c // bfmmla z12.s, z28.h, z24.h\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x6478e754 // bfmmla z20.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6479e789 // bfmmla z9.s, z28.h, z25.h\n" + ".inst 0x6479e751 // bfmmla z17.s, z26.h, z25.h\n" + ".inst 0x6478e78d // bfmmla z13.s, z28.h, z24.h\n" + "ld1h { z25.h }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x6478e755 // bfmmla z21.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6479e78a // bfmmla z10.s, z28.h, z25.h\n" + ".inst 0x6479e752 // bfmmla z18.s, z26.h, z25.h\n" + ".inst 0x6478e78e // bfmmla z14.s, z28.h, z24.h\n" + "ld1h { z25.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x6478e756 // bfmmla z22.s, z26.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x6479e78b // bfmmla z11.s, z28.h, z25.h\n" "addvl x10, x10, #8\n" - ".inst 0x6466e453 // bfmmla z19.s, z2.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" - ".inst 0x6467e457 // bfmmla z23.s, z2.h, z7.h\n" + ".inst 0x6479e753 // bfmmla z19.s, z26.h, z25.h\n" + ".inst 0x6478e78f // bfmmla z15.s, z28.h, z24.h\n" + ".inst 0x6478e757 // bfmmla z23.s, z26.h, z24.h\n" "37:" // Height 3: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -659,33 +659,33 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( "uzp1 z19.d, z19.d, z23.d\n" "tbz %x[flags], #1, 38f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z25.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z6.s, p5/M, z6.s, z1.s\n" - "fmin z12.s, p5/M, z12.s, z1.s\n" - "fmin z13.s, p5/M, z13.s, z1.s\n" - "fmin z14.s, p5/M, z14.s, z1.s\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmin z16.s, p5/M, z16.s, z1.s\n" - "fmin z17.s, p5/M, z17.s, z1.s\n" - "fmin z18.s, p5/M, z18.s, z1.s\n" - "fmin z19.s, p5/M, z19.s, z1.s\n" - "fmax z6.s, p5/M, z6.s, z0.s\n" - "fmax z12.s, p5/M, z12.s, z0.s\n" - "fmax z13.s, p5/M, z13.s, z0.s\n" - "fmax z14.s, p5/M, z14.s, z0.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" - "fmax z16.s, p5/M, z16.s, z0.s\n" - "fmax z17.s, p5/M, z17.s, z0.s\n" - "fmax z18.s, p5/M, z18.s, z0.s\n" - "fmax z19.s, p5/M, z19.s, z0.s\n" + "ld1rw { z24.s }, p5/Z, [x20]\n" + "fmin z6.s, p5/M, z6.s, z25.s\n" + "fmin z12.s, p5/M, z12.s, z25.s\n" + "fmin z13.s, p5/M, z13.s, z25.s\n" + "fmin z14.s, p5/M, z14.s, z25.s\n" + "fmin z8.s, p5/M, z8.s, z25.s\n" + "fmin z9.s, p5/M, z9.s, z25.s\n" + "fmin z10.s, p5/M, z10.s, z25.s\n" + "fmin z11.s, p5/M, z11.s, z25.s\n" + "fmin z16.s, p5/M, z16.s, z25.s\n" + "fmin z17.s, p5/M, z17.s, z25.s\n" + "fmin z18.s, p5/M, z18.s, z25.s\n" + "fmin z19.s, p5/M, z19.s, z25.s\n" + "fmax z6.s, p5/M, z6.s, z24.s\n" + "fmax z12.s, p5/M, z12.s, z24.s\n" + "fmax z13.s, p5/M, z13.s, z24.s\n" + "fmax z14.s, p5/M, z14.s, z24.s\n" + "fmax z8.s, p5/M, z8.s, z24.s\n" + "fmax z9.s, p5/M, z9.s, z24.s\n" + "fmax z10.s, p5/M, z10.s, z24.s\n" + "fmax z11.s, p5/M, z11.s, z24.s\n" + "fmax z16.s, p5/M, z16.s, z24.s\n" + "fmax z17.s, p5/M, z17.s, z24.s\n" + "fmax z18.s, p5/M, z18.s, z24.s\n" + "fmax z19.s, p5/M, z19.s, z24.s\n" "38:" // Height 3: No activation "st1w { z6.s }, p4, [x9]\n" "st1w { z12.s }, p3, [x9, #1, MUL VL]\n" @@ -745,37 +745,37 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( "42:" // Height 4: no bias "tbz %x[flags], #0, 43f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x9]\n" - "add x23, x24, x20, LSL #2\n" - "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n" + "add x22, x9, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" + "ld1w { z19.s }, p4/Z, [x9]\n" + "add x20, x21, x20, LSL #2\n" + "ld1w { z17.s }, p3/Z, [x9, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "ld1w { z17.s }, p4/Z, [x24]\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" - "ld1w { z18.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z19.s }, p2/Z, [x24, #2, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x22]\n" + "zip1 z8.d, z19.d, z12.d\n" + "zip2 z12.d, z19.d, z12.d\n" + "ld1w { z13.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x22, #2, MUL VL]\n" + "zip1 z9.d, z17.d, z13.d\n" + "zip2 z13.d, z17.d, z13.d\n" + "ld1w { z15.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z17.s }, p4/Z, [x21]\n" + "zip1 z10.d, z18.d, z14.d\n" + "zip2 z14.d, z18.d, z14.d\n" + "ld1w { z18.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z19.s }, p2/Z, [x21, #2, MUL VL]\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" - "ld1w { z24.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x23]\n" + "ld1w { z24.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x20]\n" "zip1 z16.d, z17.d, z20.d\n" "zip2 z20.d, z17.d, z20.d\n" - "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n" + "ld1w { z21.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x20, #2, MUL VL]\n" "zip1 z17.d, z18.d, z21.d\n" "zip2 z21.d, z18.d, z21.d\n" - "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z23.s }, p1/Z, [x20, #3, MUL VL]\n" "zip1 z18.d, z19.d, z22.d\n" "zip2 z22.d, z19.d, z22.d\n" "zip1 z19.d, z24.d, z23.d\n" @@ -803,14 +803,14 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( "45:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 46f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 47f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -820,101 +820,101 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( "b 47f\n" "46:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" "47:" // Height 4: input setup done "cmp x27, #0x4\n" "ble 49f\n" "48:" // Height 4: Multiply loop: Main loop head "whilelt p0.s, XZR, x27\n" - "ld1rqw { z0.s }, p0/Z, [x26]\n" - "ld1rqw { z1.s }, p0/Z, [x25]\n" - ".inst 0x658ab400 // bfcvt z0.h, p5/M, z0.s\n" - "ld1rqw { z2.s }, p0/Z, [x24]\n" - "ld1rqw { z3.s }, p0/Z, [x23]\n" - ".inst 0x658ab421 // bfcvt z1.h, p5/M, z1.s\n" - ".inst 0x658ab442 // bfcvt z2.h, p5/M, z2.s\n" - ".inst 0x658ab463 // bfcvt z3.h, p5/M, z3.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "uzp1 z1.h, z1.h, z1.h\n" - "uzp1 z2.h, z2.h, z2.h\n" + "ld1rqw { z29.s }, p0/Z, [x26]\n" + "ld1rqw { z28.s }, p0/Z, [x25]\n" + ".inst 0x658ab7bd // bfcvt z29.h, p5/M, z29.s\n" + "ld1rqw { z27.s }, p0/Z, [x24]\n" + "ld1rqw { z26.s }, p0/Z, [x23]\n" + ".inst 0x658ab79c // bfcvt z28.h, p5/M, z28.s\n" + ".inst 0x658ab77b // bfcvt z27.h, p5/M, z27.s\n" + ".inst 0x658ab75a // bfcvt z26.h, p5/M, z26.s\n" + "uzp1 z29.h, z29.h, z29.h\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + "uzp1 z28.h, z28.h, z28.h\n" + "uzp1 z27.h, z27.h, z27.h\n" "sub x27, x27, #0x4\n" "cmp x27, #0x4\n" - "uzp1 z3.h, z3.h, z3.h\n" - "trn1 z0.d, z0.d, z1.d\n" - ".inst 0x6466e408 // bfmmla z8.s, z0.h, z6.h\n" + "uzp1 z26.h, z26.h, z26.h\n" + "trn1 z29.d, z29.d, z28.d\n" + ".inst 0x6479e7a8 // bfmmla z8.s, z29.h, z25.h\n" "add x26, x26, #0x10\n" - "trn1 z2.d, z2.d, z3.d\n" - ".inst 0x6466e450 // bfmmla z16.s, z2.h, z6.h\n" - ".inst 0x6467e40c // bfmmla z12.s, z0.h, z7.h\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x6467e454 // bfmmla z20.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" + "trn1 z27.d, z27.d, z26.d\n" + ".inst 0x6479e770 // bfmmla z16.s, z27.h, z25.h\n" + ".inst 0x6478e7ac // bfmmla z12.s, z29.h, z24.h\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x6478e774 // bfmmla z20.s, z27.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6479e7a9 // bfmmla z9.s, z29.h, z25.h\n" "add x25, x25, #0x10\n" - ".inst 0x6466e451 // bfmmla z17.s, z2.h, z6.h\n" - ".inst 0x6467e40d // bfmmla z13.s, z0.h, z7.h\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x6479e771 // bfmmla z17.s, z27.h, z25.h\n" + ".inst 0x6478e7ad // bfmmla z13.s, z29.h, z24.h\n" + "ld1h { z25.h }, p5/Z, [x10, #4, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x6467e455 // bfmmla z21.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6466e40a // bfmmla z10.s, z0.h, z6.h\n" + ".inst 0x6478e775 // bfmmla z21.s, z27.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6479e7aa // bfmmla z10.s, z29.h, z25.h\n" "add x23, x23, #0x10\n" - ".inst 0x6466e452 // bfmmla z18.s, z2.h, z6.h\n" - ".inst 0x6467e40e // bfmmla z14.s, z0.h, z7.h\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x6467e456 // bfmmla z22.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" + ".inst 0x6479e772 // bfmmla z18.s, z27.h, z25.h\n" + ".inst 0x6478e7ae // bfmmla z14.s, z29.h, z24.h\n" + "ld1h { z25.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x6478e776 // bfmmla z22.s, z27.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x6479e7ab // bfmmla z11.s, z29.h, z25.h\n" "addvl x10, x10, #8\n" - ".inst 0x6466e453 // bfmmla z19.s, z2.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" - ".inst 0x6467e457 // bfmmla z23.s, z2.h, z7.h\n" + ".inst 0x6479e773 // bfmmla z19.s, z27.h, z25.h\n" + ".inst 0x6478e7af // bfmmla z15.s, z29.h, z24.h\n" + ".inst 0x6478e777 // bfmmla z23.s, z27.h, z24.h\n" "bgt 48b\n" "49:" // Height 4: Multiply loop: Single iteration only "whilelt p0.s, XZR, x27\n" - "ld1rqw { z0.s }, p0/Z, [x26]\n" - "ld1rqw { z1.s }, p0/Z, [x25]\n" - ".inst 0x658ab400 // bfcvt z0.h, p5/M, z0.s\n" - "ld1rqw { z2.s }, p0/Z, [x24]\n" - "ld1rqw { z3.s }, p0/Z, [x23]\n" - ".inst 0x658ab421 // bfcvt z1.h, p5/M, z1.s\n" - ".inst 0x658ab442 // bfcvt z2.h, p5/M, z2.s\n" - ".inst 0x658ab463 // bfcvt z3.h, p5/M, z3.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "uzp1 z1.h, z1.h, z1.h\n" - "uzp1 z2.h, z2.h, z2.h\n" - "uzp1 z3.h, z3.h, z3.h\n" - "trn1 z0.d, z0.d, z1.d\n" - ".inst 0x6466e408 // bfmmla z8.s, z0.h, z6.h\n" - "trn1 z2.d, z2.d, z3.d\n" - ".inst 0x6466e450 // bfmmla z16.s, z2.h, z6.h\n" - ".inst 0x6467e40c // bfmmla z12.s, z0.h, z7.h\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x6467e454 // bfmmla z20.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - ".inst 0x6466e451 // bfmmla z17.s, z2.h, z6.h\n" - ".inst 0x6467e40d // bfmmla z13.s, z0.h, z7.h\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x6467e455 // bfmmla z21.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6466e40a // bfmmla z10.s, z0.h, z6.h\n" - ".inst 0x6466e452 // bfmmla z18.s, z2.h, z6.h\n" - ".inst 0x6467e40e // bfmmla z14.s, z0.h, z7.h\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x6467e456 // bfmmla z22.s, z2.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" + "ld1rqw { z29.s }, p0/Z, [x26]\n" + "ld1rqw { z28.s }, p0/Z, [x25]\n" + ".inst 0x658ab7bd // bfcvt z29.h, p5/M, z29.s\n" + "ld1rqw { z27.s }, p0/Z, [x24]\n" + "ld1rqw { z26.s }, p0/Z, [x23]\n" + ".inst 0x658ab79c // bfcvt z28.h, p5/M, z28.s\n" + ".inst 0x658ab77b // bfcvt z27.h, p5/M, z27.s\n" + ".inst 0x658ab75a // bfcvt z26.h, p5/M, z26.s\n" + "uzp1 z29.h, z29.h, z29.h\n" + "ld1h { z25.h }, p5/Z, [x10]\n" + "ld1h { z24.h }, p5/Z, [x10, #1, MUL VL]\n" + "uzp1 z28.h, z28.h, z28.h\n" + "uzp1 z27.h, z27.h, z27.h\n" + "uzp1 z26.h, z26.h, z26.h\n" + "trn1 z29.d, z29.d, z28.d\n" + ".inst 0x6479e7a8 // bfmmla z8.s, z29.h, z25.h\n" + "trn1 z27.d, z27.d, z26.d\n" + ".inst 0x6479e770 // bfmmla z16.s, z27.h, z25.h\n" + ".inst 0x6478e7ac // bfmmla z12.s, z29.h, z24.h\n" + "ld1h { z25.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x6478e774 // bfmmla z20.s, z27.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6479e7a9 // bfmmla z9.s, z29.h, z25.h\n" + ".inst 0x6479e771 // bfmmla z17.s, z27.h, z25.h\n" + ".inst 0x6478e7ad // bfmmla z13.s, z29.h, z24.h\n" + "ld1h { z25.h }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x6478e775 // bfmmla z21.s, z27.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6479e7aa // bfmmla z10.s, z29.h, z25.h\n" + ".inst 0x6479e772 // bfmmla z18.s, z27.h, z25.h\n" + ".inst 0x6478e7ae // bfmmla z14.s, z29.h, z24.h\n" + "ld1h { z25.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x6478e776 // bfmmla z22.s, z27.h, z24.h\n" + "ld1h { z24.h }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x6479e7ab // bfmmla z11.s, z29.h, z25.h\n" "addvl x10, x10, #8\n" - ".inst 0x6466e453 // bfmmla z19.s, z2.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" - ".inst 0x6467e457 // bfmmla z23.s, z2.h, z7.h\n" + ".inst 0x6479e773 // bfmmla z19.s, z27.h, z25.h\n" + ".inst 0x6478e7af // bfmmla z15.s, z29.h, z24.h\n" + ".inst 0x6478e777 // bfmmla z23.s, z27.h, z24.h\n" "50:" // Height 4: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -942,41 +942,41 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( "uzp2 z19.d, z19.d, z23.d\n" "tbz %x[flags], #1, 51f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" + "ld1rw { z24.s }, p5/Z, [x20]\n" "add x20, %x[args_ptr], %[offset_min]\n" - "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z6.s, p5/M, z6.s, z1.s\n" - "fmin z12.s, p5/M, z12.s, z1.s\n" - "fmin z13.s, p5/M, z13.s, z1.s\n" - "fmin z14.s, p5/M, z14.s, z1.s\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmin z15.s, p5/M, z15.s, z1.s\n" - "fmin z20.s, p5/M, z20.s, z1.s\n" - "fmin z21.s, p5/M, z21.s, z1.s\n" - "fmin z22.s, p5/M, z22.s, z1.s\n" - "fmin z16.s, p5/M, z16.s, z1.s\n" - "fmin z17.s, p5/M, z17.s, z1.s\n" - "fmin z18.s, p5/M, z18.s, z1.s\n" - "fmin z19.s, p5/M, z19.s, z1.s\n" - "fmax z6.s, p5/M, z6.s, z0.s\n" - "fmax z12.s, p5/M, z12.s, z0.s\n" - "fmax z13.s, p5/M, z13.s, z0.s\n" - "fmax z14.s, p5/M, z14.s, z0.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" - "fmax z15.s, p5/M, z15.s, z0.s\n" - "fmax z20.s, p5/M, z20.s, z0.s\n" - "fmax z21.s, p5/M, z21.s, z0.s\n" - "fmax z22.s, p5/M, z22.s, z0.s\n" - "fmax z16.s, p5/M, z16.s, z0.s\n" - "fmax z17.s, p5/M, z17.s, z0.s\n" - "fmax z18.s, p5/M, z18.s, z0.s\n" - "fmax z19.s, p5/M, z19.s, z0.s\n" + "ld1rw { z23.s }, p5/Z, [x20]\n" + "fmin z6.s, p5/M, z6.s, z24.s\n" + "fmin z12.s, p5/M, z12.s, z24.s\n" + "fmin z13.s, p5/M, z13.s, z24.s\n" + "fmin z14.s, p5/M, z14.s, z24.s\n" + "fmin z8.s, p5/M, z8.s, z24.s\n" + "fmin z9.s, p5/M, z9.s, z24.s\n" + "fmin z10.s, p5/M, z10.s, z24.s\n" + "fmin z11.s, p5/M, z11.s, z24.s\n" + "fmin z15.s, p5/M, z15.s, z24.s\n" + "fmin z20.s, p5/M, z20.s, z24.s\n" + "fmin z21.s, p5/M, z21.s, z24.s\n" + "fmin z22.s, p5/M, z22.s, z24.s\n" + "fmin z16.s, p5/M, z16.s, z24.s\n" + "fmin z17.s, p5/M, z17.s, z24.s\n" + "fmin z18.s, p5/M, z18.s, z24.s\n" + "fmin z19.s, p5/M, z19.s, z24.s\n" + "fmax z6.s, p5/M, z6.s, z23.s\n" + "fmax z12.s, p5/M, z12.s, z23.s\n" + "fmax z13.s, p5/M, z13.s, z23.s\n" + "fmax z14.s, p5/M, z14.s, z23.s\n" + "fmax z8.s, p5/M, z8.s, z23.s\n" + "fmax z9.s, p5/M, z9.s, z23.s\n" + "fmax z10.s, p5/M, z10.s, z23.s\n" + "fmax z11.s, p5/M, z11.s, z23.s\n" + "fmax z15.s, p5/M, z15.s, z23.s\n" + "fmax z20.s, p5/M, z20.s, z23.s\n" + "fmax z21.s, p5/M, z21.s, z23.s\n" + "fmax z22.s, p5/M, z22.s, z23.s\n" + "fmax z16.s, p5/M, z16.s, z23.s\n" + "fmax z17.s, p5/M, z17.s, z23.s\n" + "fmax z18.s, p5/M, z18.s, z23.s\n" + "fmax z19.s, p5/M, z19.s, z23.s\n" "51:" // Height 4: No activation "st1w { z6.s }, p4, [x9]\n" "st1w { z12.s }, p3, [x9, #1, MUL VL]\n" @@ -1048,54 +1048,54 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( "55:" // Height 5: no bias "tbz %x[flags], #0, 56f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x9]\n" - "add x23, x24, x20, LSL #2\n" + "add x23, x9, x20, LSL #2\n" "add x22, x23, x20, LSL #2\n" - "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n" + "ld1w { z19.s }, p4/Z, [x9]\n" + "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" + "ld1w { z17.s }, p3/Z, [x9, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "ld1w { z17.s }, p4/Z, [x24]\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" - "ld1w { z18.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z19.s }, p2/Z, [x24, #2, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x23]\n" + "zip1 z8.d, z19.d, z12.d\n" + "zip2 z12.d, z19.d, z12.d\n" + "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n" + "zip1 z9.d, z17.d, z13.d\n" + "zip2 z13.d, z17.d, z13.d\n" + "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z17.s }, p4/Z, [x22]\n" + "zip1 z10.d, z18.d, z14.d\n" + "zip2 z14.d, z18.d, z14.d\n" + "ld1w { z18.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z19.s }, p2/Z, [x22, #2, MUL VL]\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" - "ld1w { z24.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x23]\n" + "ld1w { z24.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x21]\n" "zip1 z16.d, z17.d, z20.d\n" "zip2 z20.d, z17.d, z20.d\n" - "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n" + "ld1w { z21.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x21, #2, MUL VL]\n" "zip1 z17.d, z18.d, z21.d\n" "zip2 z21.d, z18.d, z21.d\n" - "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n" - "ld1w { z25.s }, p4/Z, [x22]\n" + "ld1w { z23.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z25.s }, p4/Z, [x20]\n" "zip1 z18.d, z19.d, z22.d\n" "zip2 z22.d, z19.d, z22.d\n" - "ld1w { z26.s }, p3/Z, [x22, #1, MUL VL]\n" - "ld1w { z27.s }, p2/Z, [x22, #2, MUL VL]\n" + "ld1w { z26.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z27.s }, p2/Z, [x20, #2, MUL VL]\n" "zip1 z19.d, z24.d, z23.d\n" "zip2 z23.d, z24.d, z23.d\n" - "ld1w { z6.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z0.s }, p1/Z, [x20, #3, MUL VL]\n" "zip1 z24.d, z25.d, z28.d\n" "zip2 z28.d, z25.d, z28.d\n" "zip1 z25.d, z26.d, z29.d\n" "zip2 z29.d, z26.d, z29.d\n" "zip1 z26.d, z27.d, z30.d\n" "zip2 z30.d, z27.d, z30.d\n" - "zip1 z27.d, z6.d, z31.d\n" - "zip2 z31.d, z6.d, z31.d\n" + "zip1 z27.d, z0.d, z31.d\n" + "zip2 z31.d, z0.d, z31.d\n" "b 57f\n" "56:" // Height 5: no accumulate "mov z8.b, #0x0\n" @@ -1127,15 +1127,15 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( "58:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 59f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 60f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -1146,125 +1146,125 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( "b 60f\n" "59:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" "60:" // Height 5: input setup done "cmp x27, #0x4\n" "ble 62f\n" "61:" // Height 5: Multiply loop: Main loop head "whilelt p0.s, XZR, x27\n" - "ld1rqw { z0.s }, p0/Z, [x26]\n" - "ld1rqw { z1.s }, p0/Z, [x25]\n" - ".inst 0x658ab400 // bfcvt z0.h, p5/M, z0.s\n" - "ld1rqw { z2.s }, p0/Z, [x24]\n" + "ld1rqw { z6.s }, p0/Z, [x26]\n" + "ld1rqw { z5.s }, p0/Z, [x25]\n" + ".inst 0x658ab4c6 // bfcvt z6.h, p5/M, z6.s\n" + "ld1rqw { z4.s }, p0/Z, [x24]\n" "ld1rqw { z3.s }, p0/Z, [x23]\n" - ".inst 0x658ab421 // bfcvt z1.h, p5/M, z1.s\n" - ".inst 0x658ab442 // bfcvt z2.h, p5/M, z2.s\n" - "ld1rqw { z4.s }, p0/Z, [x22]\n" + ".inst 0x658ab4a5 // bfcvt z5.h, p5/M, z5.s\n" + ".inst 0x658ab484 // bfcvt z4.h, p5/M, z4.s\n" + "ld1rqw { z2.s }, p0/Z, [x22]\n" ".inst 0x658ab463 // bfcvt z3.h, p5/M, z3.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "uzp1 z1.h, z1.h, z1.h\n" - "uzp1 z2.h, z2.h, z2.h\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" + "uzp1 z6.h, z6.h, z6.h\n" + "ld1h { z1.h }, p5/Z, [x10]\n" + "uzp1 z5.h, z5.h, z5.h\n" + "uzp1 z4.h, z4.h, z4.h\n" + "ld1h { z0.h }, p5/Z, [x10, #1, MUL VL]\n" "sub x27, x27, #0x4\n" "uzp1 z3.h, z3.h, z3.h\n" - ".inst 0x658ab484 // bfcvt z4.h, p5/M, z4.s\n" + ".inst 0x658ab442 // bfcvt z2.h, p5/M, z2.s\n" "cmp x27, #0x4\n" "add x26, x26, #0x10\n" - "trn1 z0.d, z0.d, z1.d\n" - "trn1 z2.d, z2.d, z3.d\n" - ".inst 0x6466e408 // bfmmla z8.s, z0.h, z6.h\n" + "trn1 z6.d, z6.d, z5.d\n" + "trn1 z4.d, z4.d, z3.d\n" + ".inst 0x6461e4c8 // bfmmla z8.s, z6.h, z1.h\n" "add x25, x25, #0x10\n" - "uzp1 z4.h, z4.h, z4.h\n" - ".inst 0x6466e450 // bfmmla z16.s, z2.h, z6.h\n" - ".inst 0x6466e498 // bfmmla z24.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x6467e40c // bfmmla z12.s, z0.h, z7.h\n" - ".inst 0x6467e454 // bfmmla z20.s, z2.h, z7.h\n" + "uzp1 z2.h, z2.h, z2.h\n" + ".inst 0x6461e490 // bfmmla z16.s, z4.h, z1.h\n" + ".inst 0x6461e458 // bfmmla z24.s, z2.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x6460e4cc // bfmmla z12.s, z6.h, z0.h\n" + ".inst 0x6460e494 // bfmmla z20.s, z4.h, z0.h\n" "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" - ".inst 0x6467e49c // bfmmla z28.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" + ".inst 0x6460e45c // bfmmla z28.s, z2.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6461e4c9 // bfmmla z9.s, z6.h, z1.h\n" "add x22, x22, #0x10\n" - ".inst 0x6466e451 // bfmmla z17.s, z2.h, z6.h\n" - ".inst 0x6466e499 // bfmmla z25.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x6467e40d // bfmmla z13.s, z0.h, z7.h\n" - ".inst 0x6467e455 // bfmmla z21.s, z2.h, z7.h\n" - ".inst 0x6467e49d // bfmmla z29.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6466e40a // bfmmla z10.s, z0.h, z6.h\n" - ".inst 0x6466e452 // bfmmla z18.s, z2.h, z6.h\n" - ".inst 0x6466e49a // bfmmla z26.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x6467e40e // bfmmla z14.s, z0.h, z7.h\n" - ".inst 0x6467e456 // bfmmla z22.s, z2.h, z7.h\n" - ".inst 0x6467e49e // bfmmla z30.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" + ".inst 0x6461e491 // bfmmla z17.s, z4.h, z1.h\n" + ".inst 0x6461e459 // bfmmla z25.s, z2.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x6460e4cd // bfmmla z13.s, z6.h, z0.h\n" + ".inst 0x6460e495 // bfmmla z21.s, z4.h, z0.h\n" + ".inst 0x6460e45d // bfmmla z29.s, z2.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6461e4ca // bfmmla z10.s, z6.h, z1.h\n" + ".inst 0x6461e492 // bfmmla z18.s, z4.h, z1.h\n" + ".inst 0x6461e45a // bfmmla z26.s, z2.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x6460e4ce // bfmmla z14.s, z6.h, z0.h\n" + ".inst 0x6460e496 // bfmmla z22.s, z4.h, z0.h\n" + ".inst 0x6460e45e // bfmmla z30.s, z2.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x6461e4cb // bfmmla z11.s, z6.h, z1.h\n" "addvl x10, x10, #8\n" - ".inst 0x6466e453 // bfmmla z19.s, z2.h, z6.h\n" - ".inst 0x6466e49b // bfmmla z27.s, z4.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" - ".inst 0x6467e457 // bfmmla z23.s, z2.h, z7.h\n" - ".inst 0x6467e49f // bfmmla z31.s, z4.h, z7.h\n" + ".inst 0x6461e493 // bfmmla z19.s, z4.h, z1.h\n" + ".inst 0x6461e45b // bfmmla z27.s, z2.h, z1.h\n" + ".inst 0x6460e4cf // bfmmla z15.s, z6.h, z0.h\n" + ".inst 0x6460e497 // bfmmla z23.s, z4.h, z0.h\n" + ".inst 0x6460e45f // bfmmla z31.s, z2.h, z0.h\n" "bgt 61b\n" "62:" // Height 5: Multiply loop: Single iteration only "whilelt p0.s, XZR, x27\n" - "ld1rqw { z0.s }, p0/Z, [x26]\n" - "ld1rqw { z1.s }, p0/Z, [x25]\n" - ".inst 0x658ab400 // bfcvt z0.h, p5/M, z0.s\n" - "ld1rqw { z2.s }, p0/Z, [x24]\n" + "ld1rqw { z6.s }, p0/Z, [x26]\n" + "ld1rqw { z5.s }, p0/Z, [x25]\n" + ".inst 0x658ab4c6 // bfcvt z6.h, p5/M, z6.s\n" + "ld1rqw { z4.s }, p0/Z, [x24]\n" "ld1rqw { z3.s }, p0/Z, [x23]\n" - ".inst 0x658ab421 // bfcvt z1.h, p5/M, z1.s\n" - ".inst 0x658ab442 // bfcvt z2.h, p5/M, z2.s\n" - "ld1rqw { z4.s }, p0/Z, [x22]\n" - ".inst 0x658ab463 // bfcvt z3.h, p5/M, z3.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "uzp1 z1.h, z1.h, z1.h\n" - "uzp1 z2.h, z2.h, z2.h\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "uzp1 z3.h, z3.h, z3.h\n" + ".inst 0x658ab4a5 // bfcvt z5.h, p5/M, z5.s\n" ".inst 0x658ab484 // bfcvt z4.h, p5/M, z4.s\n" - "trn1 z0.d, z0.d, z1.d\n" - "trn1 z2.d, z2.d, z3.d\n" - ".inst 0x6466e408 // bfmmla z8.s, z0.h, z6.h\n" + "ld1rqw { z2.s }, p0/Z, [x22]\n" + ".inst 0x658ab463 // bfcvt z3.h, p5/M, z3.s\n" + "uzp1 z6.h, z6.h, z6.h\n" + "ld1h { z1.h }, p5/Z, [x10]\n" + "uzp1 z5.h, z5.h, z5.h\n" "uzp1 z4.h, z4.h, z4.h\n" - ".inst 0x6466e450 // bfmmla z16.s, z2.h, z6.h\n" - ".inst 0x6466e498 // bfmmla z24.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x6467e40c // bfmmla z12.s, z0.h, z7.h\n" - ".inst 0x6467e454 // bfmmla z20.s, z2.h, z7.h\n" - ".inst 0x6467e49c // bfmmla z28.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - ".inst 0x6466e451 // bfmmla z17.s, z2.h, z6.h\n" - ".inst 0x6466e499 // bfmmla z25.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x6467e40d // bfmmla z13.s, z0.h, z7.h\n" - ".inst 0x6467e455 // bfmmla z21.s, z2.h, z7.h\n" - ".inst 0x6467e49d // bfmmla z29.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6466e40a // bfmmla z10.s, z0.h, z6.h\n" - ".inst 0x6466e452 // bfmmla z18.s, z2.h, z6.h\n" - ".inst 0x6466e49a // bfmmla z26.s, z4.h, z6.h\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x6467e40e // bfmmla z14.s, z0.h, z7.h\n" - ".inst 0x6467e456 // bfmmla z22.s, z2.h, z7.h\n" - ".inst 0x6467e49e // bfmmla z30.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" + "ld1h { z0.h }, p5/Z, [x10, #1, MUL VL]\n" + "uzp1 z3.h, z3.h, z3.h\n" + ".inst 0x658ab442 // bfcvt z2.h, p5/M, z2.s\n" + "trn1 z6.d, z6.d, z5.d\n" + "trn1 z4.d, z4.d, z3.d\n" + ".inst 0x6461e4c8 // bfmmla z8.s, z6.h, z1.h\n" + "uzp1 z2.h, z2.h, z2.h\n" + ".inst 0x6461e490 // bfmmla z16.s, z4.h, z1.h\n" + ".inst 0x6461e458 // bfmmla z24.s, z2.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x6460e4cc // bfmmla z12.s, z6.h, z0.h\n" + ".inst 0x6460e494 // bfmmla z20.s, z4.h, z0.h\n" + ".inst 0x6460e45c // bfmmla z28.s, z2.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6461e4c9 // bfmmla z9.s, z6.h, z1.h\n" + ".inst 0x6461e491 // bfmmla z17.s, z4.h, z1.h\n" + ".inst 0x6461e459 // bfmmla z25.s, z2.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x6460e4cd // bfmmla z13.s, z6.h, z0.h\n" + ".inst 0x6460e495 // bfmmla z21.s, z4.h, z0.h\n" + ".inst 0x6460e45d // bfmmla z29.s, z2.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6461e4ca // bfmmla z10.s, z6.h, z1.h\n" + ".inst 0x6461e492 // bfmmla z18.s, z4.h, z1.h\n" + ".inst 0x6461e45a // bfmmla z26.s, z2.h, z1.h\n" + "ld1h { z1.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x6460e4ce // bfmmla z14.s, z6.h, z0.h\n" + ".inst 0x6460e496 // bfmmla z22.s, z4.h, z0.h\n" + ".inst 0x6460e45e // bfmmla z30.s, z2.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x6461e4cb // bfmmla z11.s, z6.h, z1.h\n" "addvl x10, x10, #8\n" - ".inst 0x6466e453 // bfmmla z19.s, z2.h, z6.h\n" - ".inst 0x6466e49b // bfmmla z27.s, z4.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" - ".inst 0x6467e457 // bfmmla z23.s, z2.h, z7.h\n" - ".inst 0x6467e49f // bfmmla z31.s, z4.h, z7.h\n" + ".inst 0x6461e493 // bfmmla z19.s, z4.h, z1.h\n" + ".inst 0x6461e45b // bfmmla z27.s, z2.h, z1.h\n" + ".inst 0x6460e4cf // bfmmla z15.s, z6.h, z0.h\n" + ".inst 0x6460e497 // bfmmla z23.s, z4.h, z0.h\n" + ".inst 0x6460e45f // bfmmla z31.s, z2.h, z0.h\n" "63:" // Height 5: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1297,49 +1297,49 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( "uzp1 z27.d, z27.d, z31.d\n" "tbz %x[flags], #1, 64f\n" "add x20, %x[args_ptr], %[offset_max]\n" - "ld1rw { z1.s }, p5/Z, [x20]\n" - "add x20, %x[args_ptr], %[offset_min]\n" "ld1rw { z0.s }, p5/Z, [x20]\n" - "fmin z6.s, p5/M, z6.s, z1.s\n" - "fmin z12.s, p5/M, z12.s, z1.s\n" - "fmin z13.s, p5/M, z13.s, z1.s\n" - "fmin z14.s, p5/M, z14.s, z1.s\n" - "fmin z8.s, p5/M, z8.s, z1.s\n" - "fmin z9.s, p5/M, z9.s, z1.s\n" - "fmin z10.s, p5/M, z10.s, z1.s\n" - "fmin z11.s, p5/M, z11.s, z1.s\n" - "fmin z15.s, p5/M, z15.s, z1.s\n" - "fmin z20.s, p5/M, z20.s, z1.s\n" - "fmin z21.s, p5/M, z21.s, z1.s\n" - "fmin z22.s, p5/M, z22.s, z1.s\n" - "fmin z16.s, p5/M, z16.s, z1.s\n" - "fmin z17.s, p5/M, z17.s, z1.s\n" - "fmin z18.s, p5/M, z18.s, z1.s\n" - "fmin z19.s, p5/M, z19.s, z1.s\n" - "fmin z24.s, p5/M, z24.s, z1.s\n" - "fmin z25.s, p5/M, z25.s, z1.s\n" - "fmin z26.s, p5/M, z26.s, z1.s\n" - "fmin z27.s, p5/M, z27.s, z1.s\n" - "fmax z6.s, p5/M, z6.s, z0.s\n" - "fmax z12.s, p5/M, z12.s, z0.s\n" - "fmax z13.s, p5/M, z13.s, z0.s\n" - "fmax z14.s, p5/M, z14.s, z0.s\n" - "fmax z8.s, p5/M, z8.s, z0.s\n" - "fmax z9.s, p5/M, z9.s, z0.s\n" - "fmax z10.s, p5/M, z10.s, z0.s\n" - "fmax z11.s, p5/M, z11.s, z0.s\n" - "fmax z15.s, p5/M, z15.s, z0.s\n" - "fmax z20.s, p5/M, z20.s, z0.s\n" - "fmax z21.s, p5/M, z21.s, z0.s\n" - "fmax z22.s, p5/M, z22.s, z0.s\n" - "fmax z16.s, p5/M, z16.s, z0.s\n" - "fmax z17.s, p5/M, z17.s, z0.s\n" - "fmax z18.s, p5/M, z18.s, z0.s\n" - "fmax z19.s, p5/M, z19.s, z0.s\n" - "fmax z24.s, p5/M, z24.s, z0.s\n" - "fmax z25.s, p5/M, z25.s, z0.s\n" - "fmax z26.s, p5/M, z26.s, z0.s\n" - "fmax z27.s, p5/M, z27.s, z0.s\n" + "add x20, %x[args_ptr], %[offset_min]\n" + "ld1rw { z23.s }, p5/Z, [x20]\n" + "fmin z6.s, p5/M, z6.s, z0.s\n" + "fmin z12.s, p5/M, z12.s, z0.s\n" + "fmin z13.s, p5/M, z13.s, z0.s\n" + "fmin z14.s, p5/M, z14.s, z0.s\n" + "fmin z8.s, p5/M, z8.s, z0.s\n" + "fmin z9.s, p5/M, z9.s, z0.s\n" + "fmin z10.s, p5/M, z10.s, z0.s\n" + "fmin z11.s, p5/M, z11.s, z0.s\n" + "fmin z15.s, p5/M, z15.s, z0.s\n" + "fmin z20.s, p5/M, z20.s, z0.s\n" + "fmin z21.s, p5/M, z21.s, z0.s\n" + "fmin z22.s, p5/M, z22.s, z0.s\n" + "fmin z16.s, p5/M, z16.s, z0.s\n" + "fmin z17.s, p5/M, z17.s, z0.s\n" + "fmin z18.s, p5/M, z18.s, z0.s\n" + "fmin z19.s, p5/M, z19.s, z0.s\n" + "fmin z24.s, p5/M, z24.s, z0.s\n" + "fmin z25.s, p5/M, z25.s, z0.s\n" + "fmin z26.s, p5/M, z26.s, z0.s\n" + "fmin z27.s, p5/M, z27.s, z0.s\n" + "fmax z6.s, p5/M, z6.s, z23.s\n" + "fmax z12.s, p5/M, z12.s, z23.s\n" + "fmax z13.s, p5/M, z13.s, z23.s\n" + "fmax z14.s, p5/M, z14.s, z23.s\n" + "fmax z8.s, p5/M, z8.s, z23.s\n" + "fmax z9.s, p5/M, z9.s, z23.s\n" + "fmax z10.s, p5/M, z10.s, z23.s\n" + "fmax z11.s, p5/M, z11.s, z23.s\n" + "fmax z15.s, p5/M, z15.s, z23.s\n" + "fmax z20.s, p5/M, z20.s, z23.s\n" + "fmax z21.s, p5/M, z21.s, z23.s\n" + "fmax z22.s, p5/M, z22.s, z23.s\n" + "fmax z16.s, p5/M, z16.s, z23.s\n" + "fmax z17.s, p5/M, z17.s, z23.s\n" + "fmax z18.s, p5/M, z18.s, z23.s\n" + "fmax z19.s, p5/M, z19.s, z23.s\n" + "fmax z24.s, p5/M, z24.s, z23.s\n" + "fmax z25.s, p5/M, z25.s, z23.s\n" + "fmax z26.s, p5/M, z26.s, z23.s\n" + "fmax z27.s, p5/M, z27.s, z23.s\n" "64:" // Height 5: No activation "st1w { z6.s }, p4, [x9]\n" "st1w { z12.s }, p3, [x9, #1, MUL VL]\n" @@ -1418,59 +1418,59 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( "68:" // Height 6: no bias "tbz %x[flags], #0, 69f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x25, x9, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x9]\n" + "add x24, x9, x20, LSL #2\n" "add x23, x24, x20, LSL #2\n" + "ld1w { z17.s }, p4/Z, [x9]\n" "add x22, x23, x20, LSL #2\n" - "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n" "add x21, x22, x20, LSL #2\n" + "ld1w { z18.s }, p3/Z, [x9, #1, MUL VL]\n" + "ld1w { z20.s }, p2/Z, [x9, #2, MUL VL]\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x25]\n" - "zip1 z8.d, z9.d, z12.d\n" - "ld1w { z13.s }, p3/Z, [x25, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x25, #2, MUL VL]\n" - "zip2 z12.d, z9.d, z12.d\n" - "zip1 z9.d, z10.d, z13.d\n" - "ld1w { z15.s }, p1/Z, [x25, #3, MUL VL]\n" - "ld1w { z17.s }, p4/Z, [x24]\n" - "zip2 z13.d, z10.d, z13.d\n" - "zip1 z10.d, z11.d, z14.d\n" - "ld1w { z18.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z19.s }, p2/Z, [x24, #2, MUL VL]\n" - "zip2 z14.d, z11.d, z14.d\n" + "ld1w { z12.s }, p4/Z, [x24]\n" + "zip1 z8.d, z17.d, z12.d\n" + "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n" + "zip2 z12.d, z17.d, z12.d\n" + "zip1 z9.d, z18.d, z13.d\n" + "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n" + "ld1w { z17.s }, p4/Z, [x23]\n" + "zip2 z13.d, z18.d, z13.d\n" + "zip1 z10.d, z20.d, z14.d\n" + "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n" + "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n" + "zip2 z14.d, z20.d, z14.d\n" "zip1 z11.d, z16.d, z15.d\n" - "ld1w { z24.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x23]\n" + "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x22]\n" "zip2 z15.d, z16.d, z15.d\n" "zip1 z16.d, z17.d, z20.d\n" - "ld1w { z21.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x23, #2, MUL VL]\n" + "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n" "zip2 z20.d, z17.d, z20.d\n" "zip1 z17.d, z18.d, z21.d\n" - "ld1w { z23.s }, p1/Z, [x23, #3, MUL VL]\n" - "ld1w { z25.s }, p4/Z, [x22]\n" + "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z25.s }, p4/Z, [x21]\n" "zip2 z21.d, z18.d, z21.d\n" "zip1 z18.d, z19.d, z22.d\n" - "ld1w { z26.s }, p3/Z, [x22, #1, MUL VL]\n" - "ld1w { z27.s }, p2/Z, [x22, #2, MUL VL]\n" + "ld1w { z26.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z27.s }, p2/Z, [x21, #2, MUL VL]\n" "zip2 z22.d, z19.d, z22.d\n" "zip1 z19.d, z24.d, z23.d\n" - "ld1w { z6.s }, p1/Z, [x22, #3, MUL VL]\n" - "ld1w { z28.s }, p4/Z, [x21]\n" + "ld1w { z0.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z28.s }, p4/Z, [x20]\n" "zip2 z23.d, z24.d, z23.d\n" "zip1 z24.d, z25.d, z28.d\n" - "ld1w { z29.s }, p3/Z, [x21, #1, MUL VL]\n" - "ld1w { z30.s }, p2/Z, [x21, #2, MUL VL]\n" + "ld1w { z29.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z30.s }, p2/Z, [x20, #2, MUL VL]\n" "zip2 z28.d, z25.d, z28.d\n" "zip1 z25.d, z26.d, z29.d\n" - "ld1w { z31.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z31.s }, p1/Z, [x20, #3, MUL VL]\n" "zip2 z29.d, z26.d, z29.d\n" "zip1 z26.d, z27.d, z30.d\n" "zip2 z30.d, z27.d, z30.d\n" - "zip1 z27.d, z6.d, z31.d\n" - "zip2 z31.d, z6.d, z31.d\n" + "zip1 z27.d, z0.d, z31.d\n" + "zip2 z31.d, z0.d, z31.d\n" "b 70f\n" "69:" // Height 6: no accumulate "mov z8.b, #0x0\n" @@ -1502,16 +1502,16 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( "71:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 72f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 73f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20, LSL #2\n" @@ -1523,135 +1523,135 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( "b 73f\n" "72:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20, LSL #2\n" - "add x24, x25, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "add x22, x23, x20, LSL #2\n" - "add x21, x22, x20, LSL #2\n" + "add x25, x26, x21, LSL #2\n" + "add x24, x25, x21, LSL #2\n" + "add x23, x24, x21, LSL #2\n" + "add x22, x23, x21, LSL #2\n" + "add x21, x22, x21, LSL #2\n" "73:" // Height 6: input setup done "cmp x27, #0x4\n" "ble 75f\n" "74:" // Height 6: Multiply loop: Main loop head "whilelt p0.s, XZR, x27\n" - "ld1rqw { z0.s }, p0/Z, [x26]\n" - "ld1rqw { z1.s }, p0/Z, [x25]\n" - ".inst 0x658ab400 // bfcvt z0.h, p5/M, z0.s\n" - "ld1rqw { z2.s }, p0/Z, [x24]\n" - "ld1rqw { z3.s }, p0/Z, [x23]\n" - ".inst 0x658ab421 // bfcvt z1.h, p5/M, z1.s\n" - ".inst 0x658ab442 // bfcvt z2.h, p5/M, z2.s\n" - "ld1rqw { z4.s }, p0/Z, [x22]\n" - "ld1rqw { z5.s }, p0/Z, [x21]\n" - ".inst 0x658ab463 // bfcvt z3.h, p5/M, z3.s\n" - ".inst 0x658ab484 // bfcvt z4.h, p5/M, z4.s\n" + "ld1rqw { z7.s }, p0/Z, [x26]\n" + "ld1rqw { z6.s }, p0/Z, [x25]\n" + ".inst 0x658ab4e7 // bfcvt z7.h, p5/M, z7.s\n" + "ld1rqw { z5.s }, p0/Z, [x24]\n" + "ld1rqw { z4.s }, p0/Z, [x23]\n" + ".inst 0x658ab4c6 // bfcvt z6.h, p5/M, z6.s\n" ".inst 0x658ab4a5 // bfcvt z5.h, p5/M, z5.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "uzp1 z1.h, z1.h, z1.h\n" - "uzp1 z2.h, z2.h, z2.h\n" + "ld1rqw { z3.s }, p0/Z, [x22]\n" + "ld1rqw { z2.s }, p0/Z, [x21]\n" + ".inst 0x658ab484 // bfcvt z4.h, p5/M, z4.s\n" + ".inst 0x658ab463 // bfcvt z3.h, p5/M, z3.s\n" + ".inst 0x658ab442 // bfcvt z2.h, p5/M, z2.s\n" + "uzp1 z7.h, z7.h, z7.h\n" + "ld1h { z1.h }, p5/Z, [x10]\n" + "ld1h { z0.h }, p5/Z, [x10, #1, MUL VL]\n" + "uzp1 z6.h, z6.h, z6.h\n" + "uzp1 z5.h, z5.h, z5.h\n" "sub x27, x27, #0x4\n" "cmp x27, #0x4\n" - "uzp1 z3.h, z3.h, z3.h\n" "uzp1 z4.h, z4.h, z4.h\n" + "uzp1 z3.h, z3.h, z3.h\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" - "uzp1 z5.h, z5.h, z5.h\n" - "trn1 z0.d, z0.d, z1.d\n" - ".inst 0x6466e408 // bfmmla z8.s, z0.h, z6.h\n" + "uzp1 z2.h, z2.h, z2.h\n" + "trn1 z7.d, z7.d, z6.d\n" + ".inst 0x6461e4e8 // bfmmla z8.s, z7.h, z1.h\n" "add x24, x24, #0x10\n" - "trn1 z2.d, z2.d, z3.d\n" - "trn1 z4.d, z4.d, z5.d\n" - ".inst 0x6466e450 // bfmmla z16.s, z2.h, z6.h\n" + "trn1 z5.d, z5.d, z4.d\n" + "trn1 z3.d, z3.d, z2.d\n" + ".inst 0x6461e4b0 // bfmmla z16.s, z5.h, z1.h\n" "add x23, x23, #0x10\n" - ".inst 0x6466e498 // bfmmla z24.s, z4.h, z6.h\n" - ".inst 0x6467e40c // bfmmla z12.s, z0.h, z7.h\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x6461e478 // bfmmla z24.s, z3.h, z1.h\n" + ".inst 0x6460e4ec // bfmmla z12.s, z7.h, z0.h\n" + "ld1h { z1.h }, p5/Z, [x10, #2, MUL VL]\n" "add x22, x22, #0x10\n" - ".inst 0x6467e454 // bfmmla z20.s, z2.h, z7.h\n" - ".inst 0x6467e49c // bfmmla z28.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6460e4b4 // bfmmla z20.s, z5.h, z0.h\n" + ".inst 0x6460e47c // bfmmla z28.s, z3.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #3, MUL VL]\n" "add x21, x21, #0x10\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - ".inst 0x6466e451 // bfmmla z17.s, z2.h, z6.h\n" - ".inst 0x6466e499 // bfmmla z25.s, z4.h, z6.h\n" - ".inst 0x6467e40d // bfmmla z13.s, z0.h, z7.h\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x6467e455 // bfmmla z21.s, z2.h, z7.h\n" - ".inst 0x6467e49d // bfmmla z29.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6466e40a // bfmmla z10.s, z0.h, z6.h\n" - ".inst 0x6466e452 // bfmmla z18.s, z2.h, z6.h\n" - ".inst 0x6466e49a // bfmmla z26.s, z4.h, z6.h\n" - ".inst 0x6467e40e // bfmmla z14.s, z0.h, z7.h\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x6467e456 // bfmmla z22.s, z2.h, z7.h\n" - ".inst 0x6467e49e // bfmmla z30.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x6461e4e9 // bfmmla z9.s, z7.h, z1.h\n" + ".inst 0x6461e4b1 // bfmmla z17.s, z5.h, z1.h\n" + ".inst 0x6461e479 // bfmmla z25.s, z3.h, z1.h\n" + ".inst 0x6460e4ed // bfmmla z13.s, z7.h, z0.h\n" + "ld1h { z1.h }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x6460e4b5 // bfmmla z21.s, z5.h, z0.h\n" + ".inst 0x6460e47d // bfmmla z29.s, z3.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6461e4ea // bfmmla z10.s, z7.h, z1.h\n" + ".inst 0x6461e4b2 // bfmmla z18.s, z5.h, z1.h\n" + ".inst 0x6461e47a // bfmmla z26.s, z3.h, z1.h\n" + ".inst 0x6460e4ee // bfmmla z14.s, z7.h, z0.h\n" + "ld1h { z1.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x6460e4b6 // bfmmla z22.s, z5.h, z0.h\n" + ".inst 0x6460e47e // bfmmla z30.s, z3.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" - ".inst 0x6466e453 // bfmmla z19.s, z2.h, z6.h\n" - ".inst 0x6466e49b // bfmmla z27.s, z4.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" - ".inst 0x6467e457 // bfmmla z23.s, z2.h, z7.h\n" - ".inst 0x6467e49f // bfmmla z31.s, z4.h, z7.h\n" + ".inst 0x6461e4eb // bfmmla z11.s, z7.h, z1.h\n" + ".inst 0x6461e4b3 // bfmmla z19.s, z5.h, z1.h\n" + ".inst 0x6461e47b // bfmmla z27.s, z3.h, z1.h\n" + ".inst 0x6460e4ef // bfmmla z15.s, z7.h, z0.h\n" + ".inst 0x6460e4b7 // bfmmla z23.s, z5.h, z0.h\n" + ".inst 0x6460e47f // bfmmla z31.s, z3.h, z0.h\n" "bgt 74b\n" "75:" // Height 6: Multiply loop: Single iteration only "whilelt p0.s, XZR, x27\n" - "ld1rqw { z0.s }, p0/Z, [x26]\n" - "ld1rqw { z1.s }, p0/Z, [x25]\n" - ".inst 0x658ab400 // bfcvt z0.h, p5/M, z0.s\n" - "ld1rqw { z2.s }, p0/Z, [x24]\n" - "ld1rqw { z3.s }, p0/Z, [x23]\n" - ".inst 0x658ab421 // bfcvt z1.h, p5/M, z1.s\n" - ".inst 0x658ab442 // bfcvt z2.h, p5/M, z2.s\n" - "ld1rqw { z4.s }, p0/Z, [x22]\n" - "ld1rqw { z5.s }, p0/Z, [x21]\n" - ".inst 0x658ab463 // bfcvt z3.h, p5/M, z3.s\n" - ".inst 0x658ab484 // bfcvt z4.h, p5/M, z4.s\n" + "ld1rqw { z7.s }, p0/Z, [x26]\n" + "ld1rqw { z6.s }, p0/Z, [x25]\n" + ".inst 0x658ab4e7 // bfcvt z7.h, p5/M, z7.s\n" + "ld1rqw { z5.s }, p0/Z, [x24]\n" + "ld1rqw { z4.s }, p0/Z, [x23]\n" + ".inst 0x658ab4c6 // bfcvt z6.h, p5/M, z6.s\n" ".inst 0x658ab4a5 // bfcvt z5.h, p5/M, z5.s\n" - "uzp1 z0.h, z0.h, z0.h\n" - "ld1h { z6.h }, p5/Z, [x10]\n" - "ld1h { z7.h }, p5/Z, [x10, #1, MUL VL]\n" - "uzp1 z1.h, z1.h, z1.h\n" - "uzp1 z2.h, z2.h, z2.h\n" - "uzp1 z3.h, z3.h, z3.h\n" - "uzp1 z4.h, z4.h, z4.h\n" + "ld1rqw { z3.s }, p0/Z, [x22]\n" + "ld1rqw { z2.s }, p0/Z, [x21]\n" + ".inst 0x658ab484 // bfcvt z4.h, p5/M, z4.s\n" + ".inst 0x658ab463 // bfcvt z3.h, p5/M, z3.s\n" + ".inst 0x658ab442 // bfcvt z2.h, p5/M, z2.s\n" + "uzp1 z7.h, z7.h, z7.h\n" + "ld1h { z1.h }, p5/Z, [x10]\n" + "ld1h { z0.h }, p5/Z, [x10, #1, MUL VL]\n" + "uzp1 z6.h, z6.h, z6.h\n" "uzp1 z5.h, z5.h, z5.h\n" - "trn1 z0.d, z0.d, z1.d\n" - ".inst 0x6466e408 // bfmmla z8.s, z0.h, z6.h\n" - "trn1 z2.d, z2.d, z3.d\n" - "trn1 z4.d, z4.d, z5.d\n" - ".inst 0x6466e450 // bfmmla z16.s, z2.h, z6.h\n" - ".inst 0x6466e498 // bfmmla z24.s, z4.h, z6.h\n" - ".inst 0x6467e40c // bfmmla z12.s, z0.h, z7.h\n" - "ld1h { z6.h }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x6467e454 // bfmmla z20.s, z2.h, z7.h\n" - ".inst 0x6467e49c // bfmmla z28.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - ".inst 0x6466e451 // bfmmla z17.s, z2.h, z6.h\n" - ".inst 0x6466e499 // bfmmla z25.s, z4.h, z6.h\n" - ".inst 0x6467e40d // bfmmla z13.s, z0.h, z7.h\n" - "ld1h { z6.h }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x6467e455 // bfmmla z21.s, z2.h, z7.h\n" - ".inst 0x6467e49d // bfmmla z29.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x6466e40a // bfmmla z10.s, z0.h, z6.h\n" - ".inst 0x6466e452 // bfmmla z18.s, z2.h, z6.h\n" - ".inst 0x6466e49a // bfmmla z26.s, z4.h, z6.h\n" - ".inst 0x6467e40e // bfmmla z14.s, z0.h, z7.h\n" - "ld1h { z6.h }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x6467e456 // bfmmla z22.s, z2.h, z7.h\n" - ".inst 0x6467e49e // bfmmla z30.s, z4.h, z7.h\n" - "ld1h { z7.h }, p5/Z, [x10, #7, MUL VL]\n" + "uzp1 z4.h, z4.h, z4.h\n" + "uzp1 z3.h, z3.h, z3.h\n" + "uzp1 z2.h, z2.h, z2.h\n" + "trn1 z7.d, z7.d, z6.d\n" + ".inst 0x6461e4e8 // bfmmla z8.s, z7.h, z1.h\n" + "trn1 z5.d, z5.d, z4.d\n" + "trn1 z3.d, z3.d, z2.d\n" + ".inst 0x6461e4b0 // bfmmla z16.s, z5.h, z1.h\n" + ".inst 0x6461e478 // bfmmla z24.s, z3.h, z1.h\n" + ".inst 0x6460e4ec // bfmmla z12.s, z7.h, z0.h\n" + "ld1h { z1.h }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x6460e4b4 // bfmmla z20.s, z5.h, z0.h\n" + ".inst 0x6460e47c // bfmmla z28.s, z3.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x6461e4e9 // bfmmla z9.s, z7.h, z1.h\n" + ".inst 0x6461e4b1 // bfmmla z17.s, z5.h, z1.h\n" + ".inst 0x6461e479 // bfmmla z25.s, z3.h, z1.h\n" + ".inst 0x6460e4ed // bfmmla z13.s, z7.h, z0.h\n" + "ld1h { z1.h }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x6460e4b5 // bfmmla z21.s, z5.h, z0.h\n" + ".inst 0x6460e47d // bfmmla z29.s, z3.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x6461e4ea // bfmmla z10.s, z7.h, z1.h\n" + ".inst 0x6461e4b2 // bfmmla z18.s, z5.h, z1.h\n" + ".inst 0x6461e47a // bfmmla z26.s, z3.h, z1.h\n" + ".inst 0x6460e4ee // bfmmla z14.s, z7.h, z0.h\n" + "ld1h { z1.h }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x6460e4b6 // bfmmla z22.s, z5.h, z0.h\n" + ".inst 0x6460e47e // bfmmla z30.s, z3.h, z0.h\n" + "ld1h { z0.h }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - ".inst 0x6466e40b // bfmmla z11.s, z0.h, z6.h\n" - ".inst 0x6466e453 // bfmmla z19.s, z2.h, z6.h\n" - ".inst 0x6466e49b // bfmmla z27.s, z4.h, z6.h\n" - ".inst 0x6467e40f // bfmmla z15.s, z0.h, z7.h\n" - ".inst 0x6467e457 // bfmmla z23.s, z2.h, z7.h\n" - ".inst 0x6467e49f // bfmmla z31.s, z4.h, z7.h\n" + ".inst 0x6461e4eb // bfmmla z11.s, z7.h, z1.h\n" + ".inst 0x6461e4b3 // bfmmla z19.s, z5.h, z1.h\n" + ".inst 0x6461e47b // bfmmla z27.s, z3.h, z1.h\n" + ".inst 0x6460e4ef // bfmmla z15.s, z7.h, z0.h\n" + ".inst 0x6460e4b7 // bfmmla z23.s, z5.h, z0.h\n" + ".inst 0x6460e47f // bfmmla z31.s, z3.h, z0.h\n" "76:" // Height 6: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1782,7 +1782,6 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "80:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [bias] "r" (bias), [flags] "r" (flags), [offset_max] "I" (offsetof(KernelArgs, maxval)), [offset_min] "I" (offsetof(KernelArgs, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -1790,4 +1789,4 @@ void sve_hybrid_fp32bf16fp32_mmla_6x4VL ( } } // namespace arm_gemm -#endif // __ARM_FEATURE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_dot_4x4VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_dot_4x4VL.hpp index c8a7d66f288bdca3cb292820afbd11f7b4269860..ffc1606b3fb7d455b4ddeece9f24c84c21639c99 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_dot_4x4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_dot_4x4VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef ARM_COMPUTE_ENABLE_SVE + #include "../std_transforms_sve.hpp" #include "../performance_parameters.hpp" @@ -74,7 +74,6 @@ public: template static inline PerformanceParameters get_performance_parameters(const CPUInfo *ci) { - if (std::is_same::value) { switch (ci->get_cpu_model()) { default: @@ -97,5 +96,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_dot_4x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_dot_4x4VL/generic.cpp index 562b2759aa6b75c00211e03b4b3b0971631e40f2..b7c523466e7ec4b9bfc6a8688359dbfbfc6b5c3b 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_dot_4x4VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_dot_4x4VL/generic.cpp @@ -104,11 +104,11 @@ void sve_hybrid_s8qa_dot_4x4VL ( "4:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 5f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" "cbnz x26, 6f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -121,39 +121,39 @@ void sve_hybrid_s8qa_dot_4x4VL ( "7:" // Height 1: Multiply loop: Main loop head "whilelt p0.b, XZR, x25\n" "ld1rqb { z0.b }, p0/Z, [x24]\n" - "ld1b { z4.b }, p2/Z, [x28]\n" - "sdot z16.s, z4.b, z0.b[0]\n" - "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n" - "sdot z17.s, z5.b, z0.b[0]\n" - "sdot z18.s, z6.b, z0.b[0]\n" - "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n" - "sdot z19.s, z7.b, z0.b[0]\n" - "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n" - "sdot z16.s, z8.b, z0.b[1]\n" - "ld1b { z9.b }, p2/Z, [x28, #5, MUL VL]\n" - "ld1b { z10.b }, p2/Z, [x28, #6, MUL VL]\n" - "sdot z17.s, z9.b, z0.b[1]\n" - "sdot z18.s, z10.b, z0.b[1]\n" - "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n" + "ld1b { z20.b }, p2/Z, [x28]\n" + "sdot z16.s, z20.b, z0.b[0]\n" + "ld1b { z21.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z20.b }, p2/Z, [x28, #2, MUL VL]\n" + "sdot z17.s, z21.b, z0.b[0]\n" + "sdot z18.s, z20.b, z0.b[0]\n" + "ld1b { z20.b }, p2/Z, [x28, #3, MUL VL]\n" + "sdot z19.s, z20.b, z0.b[0]\n" + "ld1b { z20.b }, p2/Z, [x28, #4, MUL VL]\n" + "sdot z16.s, z20.b, z0.b[1]\n" + "ld1b { z21.b }, p2/Z, [x28, #5, MUL VL]\n" + "ld1b { z20.b }, p2/Z, [x28, #6, MUL VL]\n" + "sdot z17.s, z21.b, z0.b[1]\n" + "sdot z18.s, z20.b, z0.b[1]\n" + "ld1b { z20.b }, p2/Z, [x28, #7, MUL VL]\n" "addvl x28, x28, #16\n" - "sdot z19.s, z4.b, z0.b[1]\n" - "ld1b { z5.b }, p2/Z, [x28, #-8, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x28, #-6, MUL VL]\n" - "sdot z16.s, z5.b, z0.b[2]\n" - "sdot z17.s, z6.b, z0.b[2]\n" - "ld1b { z8.b }, p2/Z, [x28, #-5, MUL VL]\n" - "sdot z18.s, z7.b, z0.b[2]\n" - "sdot z19.s, z8.b, z0.b[2]\n" - "ld1b { z9.b }, p2/Z, [x28, #-4, MUL VL]\n" - "ld1b { z10.b }, p2/Z, [x28, #-3, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #-2, MUL VL]\n" - "sdot z16.s, z9.b, z0.b[3]\n" - "sdot z17.s, z10.b, z0.b[3]\n" - "ld1b { z5.b }, p2/Z, [x28, #-1, MUL VL]\n" - "sdot z18.s, z4.b, z0.b[3]\n" - "sdot z19.s, z5.b, z0.b[3]\n" + "sdot z19.s, z20.b, z0.b[1]\n" + "ld1b { z22.b }, p2/Z, [x28, #-8, MUL VL]\n" + "ld1b { z20.b }, p2/Z, [x28, #-7, MUL VL]\n" + "ld1b { z21.b }, p2/Z, [x28, #-6, MUL VL]\n" + "sdot z16.s, z22.b, z0.b[2]\n" + "sdot z17.s, z20.b, z0.b[2]\n" + "ld1b { z20.b }, p2/Z, [x28, #-5, MUL VL]\n" + "sdot z18.s, z21.b, z0.b[2]\n" + "sdot z19.s, z20.b, z0.b[2]\n" + "ld1b { z22.b }, p2/Z, [x28, #-4, MUL VL]\n" + "ld1b { z20.b }, p2/Z, [x28, #-3, MUL VL]\n" + "ld1b { z21.b }, p2/Z, [x28, #-2, MUL VL]\n" + "sdot z16.s, z22.b, z0.b[3]\n" + "sdot z17.s, z20.b, z0.b[3]\n" + "ld1b { z20.b }, p2/Z, [x28, #-1, MUL VL]\n" + "sdot z18.s, z21.b, z0.b[3]\n" + "sdot z19.s, z20.b, z0.b[3]\n" "add x24, x24, #0x10\n" "tbnz %x[flags], #31, 8f\n" "sdot z11.s, z0.b, z15.b\n" @@ -164,47 +164,47 @@ void sve_hybrid_s8qa_dot_4x4VL ( "9:" // Height 1: Multiply loop: Single iteration only "whilelt p0.b, XZR, x25\n" "ld1rqb { z0.b }, p0/Z, [x24]\n" - "ld1b { z4.b }, p2/Z, [x28]\n" + "ld1b { z22.b }, p2/Z, [x28]\n" "subs x25, x25, #0x4\n" - "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n" - "sdot z16.s, z4.b, z0.b[0]\n" - "sdot z17.s, z5.b, z0.b[0]\n" - "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n" - "sdot z18.s, z6.b, z0.b[0]\n" - "sdot z19.s, z7.b, z0.b[0]\n" + "ld1b { z20.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z21.b }, p2/Z, [x28, #2, MUL VL]\n" + "sdot z16.s, z22.b, z0.b[0]\n" + "sdot z17.s, z20.b, z0.b[0]\n" + "ld1b { z20.b }, p2/Z, [x28, #3, MUL VL]\n" + "sdot z18.s, z21.b, z0.b[0]\n" + "sdot z19.s, z20.b, z0.b[0]\n" "addvl x28, x28, #4\n" "ble 10f\n" - "ld1b { z8.b }, p2/Z, [x28]\n" - "ld1b { z9.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z20.b }, p2/Z, [x28]\n" + "ld1b { z22.b }, p2/Z, [x28, #1, MUL VL]\n" "subs x25, x25, #0x4\n" - "sdot z16.s, z8.b, z0.b[1]\n" - "ld1b { z10.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #3, MUL VL]\n" - "sdot z17.s, z9.b, z0.b[1]\n" - "sdot z18.s, z10.b, z0.b[1]\n" - "sdot z19.s, z4.b, z0.b[1]\n" + "sdot z16.s, z20.b, z0.b[1]\n" + "ld1b { z21.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z20.b }, p2/Z, [x28, #3, MUL VL]\n" + "sdot z17.s, z22.b, z0.b[1]\n" + "sdot z18.s, z21.b, z0.b[1]\n" + "sdot z19.s, z20.b, z0.b[1]\n" "addvl x28, x28, #4\n" "ble 10f\n" - "ld1b { z5.b }, p2/Z, [x28]\n" - "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z20.b }, p2/Z, [x28]\n" + "ld1b { z22.b }, p2/Z, [x28, #1, MUL VL]\n" "subs x25, x25, #0x4\n" - "sdot z16.s, z5.b, z0.b[2]\n" - "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" - "sdot z17.s, z6.b, z0.b[2]\n" - "sdot z18.s, z7.b, z0.b[2]\n" - "sdot z19.s, z8.b, z0.b[2]\n" + "sdot z16.s, z20.b, z0.b[2]\n" + "ld1b { z21.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z20.b }, p2/Z, [x28, #3, MUL VL]\n" + "sdot z17.s, z22.b, z0.b[2]\n" + "sdot z18.s, z21.b, z0.b[2]\n" + "sdot z19.s, z20.b, z0.b[2]\n" "addvl x28, x28, #4\n" "ble 10f\n" - "ld1b { z9.b }, p2/Z, [x28]\n" - "ld1b { z10.b }, p2/Z, [x28, #1, MUL VL]\n" - "sdot z16.s, z9.b, z0.b[3]\n" - "sdot z17.s, z10.b, z0.b[3]\n" - "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z5.b }, p2/Z, [x28, #3, MUL VL]\n" - "sdot z18.s, z4.b, z0.b[3]\n" - "sdot z19.s, z5.b, z0.b[3]\n" + "ld1b { z21.b }, p2/Z, [x28]\n" + "ld1b { z20.b }, p2/Z, [x28, #1, MUL VL]\n" + "sdot z16.s, z21.b, z0.b[3]\n" + "sdot z17.s, z20.b, z0.b[3]\n" + "ld1b { z21.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z20.b }, p2/Z, [x28, #3, MUL VL]\n" + "sdot z18.s, z21.b, z0.b[3]\n" + "sdot z19.s, z20.b, z0.b[3]\n" "addvl x28, x28, #4\n" "10:" // Height 1: Multiply loop: multiply skip "tbnz %x[flags], #31, 11f\n" @@ -218,71 +218,71 @@ void sve_hybrid_s8qa_dot_4x4VL ( "mov x20, #0x4\n" "whilelt p0.s, XZR, x20\n" "add x20, %x[qp], %[b_offset]\n" - "ld1rw { z1.s }, p2/Z, [x20]\n" + "ld1rw { z20.s }, p2/Z, [x20]\n" "saddv d11, p0, z11.s\n" "mov z11.s, z11.s[0]\n" - "neg z1.s, p2/M, z1.s\n" - "mul z11.s, p2/M, z11.s, z1.s\n" + "neg z20.s, p2/M, z20.s\n" + "mul z11.s, p2/M, z11.s, z20.s\n" "12:" // Height 1: skip row sum fixup "add z16.s, z16.s, z11.s\n" "add z17.s, z17.s, z11.s\n" - "ld1w { z0.s }, p2/Z, [x10]\n" - "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n" + "ld1w { z23.s }, p2/Z, [x10]\n" + "ld1w { z22.s }, p2/Z, [x10, #1, MUL VL]\n" "add z18.s, z18.s, z11.s\n" "add z19.s, z19.s, z11.s\n" - "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n" + "ld1w { z21.s }, p2/Z, [x10, #2, MUL VL]\n" + "ld1w { z20.s }, p2/Z, [x10, #3, MUL VL]\n" "add x20, %x[qp], %[per_layer_mul]\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add z16.s, z16.s, z0.s\n" - "add z17.s, z17.s, z1.s\n" - "add z18.s, z18.s, z2.s\n" - "add z19.s, z19.s, z3.s\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + "add z16.s, z16.s, z23.s\n" + "add z17.s, z17.s, z22.s\n" + "add z18.s, z18.s, z21.s\n" + "add z19.s, z19.s, z20.s\n" + "ld1rw { z20.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[per_layer_right_shift]\n" "ld1rw { z0.s }, p2/Z, [x20]\n" - ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n" - ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n" + ".inst 0x04b47610 // sqrdmulh z16.s, z16.s, z20.s\n" + ".inst 0x04b47631 // sqrdmulh z17.s, z17.s, z20.s\n" "addvl x10, x10, #4\n" - ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n" - ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n" + ".inst 0x04b47652 // sqrdmulh z18.s, z18.s, z20.s\n" + ".inst 0x04b47673 // sqrdmulh z19.s, z19.s, z20.s\n" "tbz %x[flags], #5, 13f\n" - "and z4.d, z16.d, z0.d\n" - "and z5.d, z17.d, z0.d\n" - "and z6.d, z18.d, z0.d\n" - "and z7.d, z19.d, z0.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z16.s, z16.s, z4.s\n" - "sqadd z17.s, z17.s, z5.s\n" - "sqadd z18.s, z18.s, z6.s\n" - "sqadd z19.s, z19.s, z7.s\n" + "and z23.d, z16.d, z0.d\n" + "and z22.d, z17.d, z0.d\n" + "and z21.d, z18.d, z0.d\n" + "and z20.d, z19.d, z0.d\n" + "asr z23.s, z23.s, #0x1f\n" + "asr z22.s, z22.s, #0x1f\n" + "asr z21.s, z21.s, #0x1f\n" + "asr z20.s, z20.s, #0x1f\n" + "sqadd z16.s, z16.s, z23.s\n" + "sqadd z17.s, z17.s, z22.s\n" + "sqadd z18.s, z18.s, z21.s\n" + "sqadd z19.s, z19.s, z20.s\n" "13:" // Height 1: no shift correction "add x20, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + "ld1rw { z20.s }, p2/Z, [x20]\n" ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n" - "add z16.s, z16.s, z4.s\n" + "add z16.s, z16.s, z20.s\n" ".inst 0x44828811 // srshl z17.s, p2/M, z17.s, z0.s\n" ".inst 0x44828812 // srshl z18.s, p2/M, z18.s, z0.s\n" - "add z17.s, z17.s, z4.s\n" - "add z18.s, z18.s, z4.s\n" + "add z17.s, z17.s, z20.s\n" + "add z18.s, z18.s, z20.s\n" ".inst 0x44828813 // srshl z19.s, p2/M, z19.s, z0.s\n" "add x20, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x20]\n" - "add z19.s, z19.s, z4.s\n" + "ld1rw { z21.s }, p2/Z, [x20]\n" + "add z19.s, z19.s, z20.s\n" "add x20, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x20]\n" - "smin z16.s, p2/M, z16.s, z6.s\n" - "smin z17.s, p2/M, z17.s, z6.s\n" - "smin z18.s, p2/M, z18.s, z6.s\n" - "smin z19.s, p2/M, z19.s, z6.s\n" - "smax z16.s, p2/M, z16.s, z5.s\n" - "smax z17.s, p2/M, z17.s, z5.s\n" - "smax z18.s, p2/M, z18.s, z5.s\n" + "ld1rw { z20.s }, p2/Z, [x20]\n" + "smin z16.s, p2/M, z16.s, z21.s\n" + "smin z17.s, p2/M, z17.s, z21.s\n" + "smin z18.s, p2/M, z18.s, z21.s\n" + "smin z19.s, p2/M, z19.s, z21.s\n" + "smax z16.s, p2/M, z16.s, z20.s\n" + "smax z17.s, p2/M, z17.s, z20.s\n" + "smax z18.s, p2/M, z18.s, z20.s\n" "uzp1 z16.h, z16.h, z17.h\n" - "smax z19.s, p2/M, z19.s, z5.s\n" + "smax z19.s, p2/M, z19.s, z20.s\n" "uzp1 z17.h, z18.h, z19.h\n" "uzp1 z16.b, z16.b, z17.b\n" "st1b { z16.b }, p1, [x27]\n" @@ -317,12 +317,12 @@ void sve_hybrid_s8qa_dot_4x4VL ( "18:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 19f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" "cbnz x26, 20f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -330,7 +330,7 @@ void sve_hybrid_s8qa_dot_4x4VL ( "b 20f\n" "19:" // Height 2: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20\n" + "add x23, x24, x21\n" "20:" // Height 2: input setup done "cmp x25, #0x10\n" "ble 23f\n" @@ -339,56 +339,56 @@ void sve_hybrid_s8qa_dot_4x4VL ( "ld1rqb { z0.b }, p0/Z, [x24]\n" "ld1rqb { z1.b }, p0/Z, [x23]\n" "add x24, x24, #0x10\n" - "ld1b { z4.b }, p2/Z, [x28]\n" - "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n" - "sdot z16.s, z4.b, z0.b[0]\n" - "sdot z20.s, z4.b, z1.b[0]\n" - "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n" - "sdot z17.s, z5.b, z0.b[0]\n" - "sdot z21.s, z5.b, z1.b[0]\n" - "sdot z18.s, z6.b, z0.b[0]\n" - "sdot z22.s, z6.b, z1.b[0]\n" - "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n" - "ld1b { z9.b }, p2/Z, [x28, #5, MUL VL]\n" - "sdot z19.s, z7.b, z0.b[0]\n" - "sdot z23.s, z7.b, z1.b[0]\n" - "ld1b { z10.b }, p2/Z, [x28, #6, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28]\n" + "ld1b { z26.b }, p2/Z, [x28, #1, MUL VL]\n" + "sdot z16.s, z24.b, z0.b[0]\n" + "sdot z20.s, z24.b, z1.b[0]\n" + "ld1b { z24.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z25.b }, p2/Z, [x28, #3, MUL VL]\n" + "sdot z17.s, z26.b, z0.b[0]\n" + "sdot z21.s, z26.b, z1.b[0]\n" + "sdot z18.s, z24.b, z0.b[0]\n" + "sdot z22.s, z24.b, z1.b[0]\n" + "ld1b { z24.b }, p2/Z, [x28, #4, MUL VL]\n" + "ld1b { z27.b }, p2/Z, [x28, #5, MUL VL]\n" + "sdot z19.s, z25.b, z0.b[0]\n" + "sdot z23.s, z25.b, z1.b[0]\n" + "ld1b { z26.b }, p2/Z, [x28, #6, MUL VL]\n" + "ld1b { z25.b }, p2/Z, [x28, #7, MUL VL]\n" "addvl x28, x28, #16\n" - "sdot z16.s, z8.b, z0.b[1]\n" - "sdot z20.s, z8.b, z1.b[1]\n" - "ld1b { z5.b }, p2/Z, [x28, #-8, MUL VL]\n" - "sdot z17.s, z9.b, z0.b[1]\n" - "sdot z21.s, z9.b, z1.b[1]\n" - "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x28, #-6, MUL VL]\n" - "sdot z18.s, z10.b, z0.b[1]\n" - "sdot z22.s, z10.b, z1.b[1]\n" - "ld1b { z8.b }, p2/Z, [x28, #-5, MUL VL]\n" - "ld1b { z9.b }, p2/Z, [x28, #-4, MUL VL]\n" - "sdot z19.s, z4.b, z0.b[1]\n" - "sdot z23.s, z4.b, z1.b[1]\n" - "ld1b { z10.b }, p2/Z, [x28, #-3, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #-2, MUL VL]\n" - "sdot z16.s, z5.b, z0.b[2]\n" - "sdot z20.s, z5.b, z1.b[2]\n" - "ld1b { z5.b }, p2/Z, [x28, #-1, MUL VL]\n" + "sdot z16.s, z24.b, z0.b[1]\n" + "sdot z20.s, z24.b, z1.b[1]\n" + "ld1b { z24.b }, p2/Z, [x28, #-8, MUL VL]\n" + "sdot z17.s, z27.b, z0.b[1]\n" + "sdot z21.s, z27.b, z1.b[1]\n" + "ld1b { z30.b }, p2/Z, [x28, #-7, MUL VL]\n" + "ld1b { z29.b }, p2/Z, [x28, #-6, MUL VL]\n" + "sdot z18.s, z26.b, z0.b[1]\n" + "sdot z22.s, z26.b, z1.b[1]\n" + "ld1b { z28.b }, p2/Z, [x28, #-5, MUL VL]\n" + "ld1b { z27.b }, p2/Z, [x28, #-4, MUL VL]\n" + "sdot z19.s, z25.b, z0.b[1]\n" + "sdot z23.s, z25.b, z1.b[1]\n" + "ld1b { z26.b }, p2/Z, [x28, #-3, MUL VL]\n" + "ld1b { z25.b }, p2/Z, [x28, #-2, MUL VL]\n" + "sdot z16.s, z24.b, z0.b[2]\n" + "sdot z20.s, z24.b, z1.b[2]\n" + "ld1b { z24.b }, p2/Z, [x28, #-1, MUL VL]\n" "add x23, x23, #0x10\n" - "sdot z17.s, z6.b, z0.b[2]\n" - "sdot z21.s, z6.b, z1.b[2]\n" - "sdot z18.s, z7.b, z0.b[2]\n" - "sdot z22.s, z7.b, z1.b[2]\n" - "sdot z19.s, z8.b, z0.b[2]\n" - "sdot z23.s, z8.b, z1.b[2]\n" - "sdot z16.s, z9.b, z0.b[3]\n" - "sdot z20.s, z9.b, z1.b[3]\n" - "sdot z17.s, z10.b, z0.b[3]\n" - "sdot z21.s, z10.b, z1.b[3]\n" - "sdot z18.s, z4.b, z0.b[3]\n" - "sdot z22.s, z4.b, z1.b[3]\n" - "sdot z19.s, z5.b, z0.b[3]\n" - "sdot z23.s, z5.b, z1.b[3]\n" + "sdot z17.s, z30.b, z0.b[2]\n" + "sdot z21.s, z30.b, z1.b[2]\n" + "sdot z18.s, z29.b, z0.b[2]\n" + "sdot z22.s, z29.b, z1.b[2]\n" + "sdot z19.s, z28.b, z0.b[2]\n" + "sdot z23.s, z28.b, z1.b[2]\n" + "sdot z16.s, z27.b, z0.b[3]\n" + "sdot z20.s, z27.b, z1.b[3]\n" + "sdot z17.s, z26.b, z0.b[3]\n" + "sdot z21.s, z26.b, z1.b[3]\n" + "sdot z18.s, z25.b, z0.b[3]\n" + "sdot z22.s, z25.b, z1.b[3]\n" + "sdot z19.s, z24.b, z0.b[3]\n" + "sdot z23.s, z24.b, z1.b[3]\n" "tbnz %x[flags], #31, 22f\n" "sdot z11.s, z0.b, z15.b\n" "sdot z12.s, z1.b, z15.b\n" @@ -401,63 +401,63 @@ void sve_hybrid_s8qa_dot_4x4VL ( "ld1rqb { z0.b }, p0/Z, [x24]\n" "ld1rqb { z1.b }, p0/Z, [x23]\n" "subs x25, x25, #0x4\n" - "ld1b { z4.b }, p2/Z, [x28]\n" - "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n" - "sdot z16.s, z4.b, z0.b[0]\n" - "sdot z20.s, z4.b, z1.b[0]\n" - "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n" - "sdot z17.s, z5.b, z0.b[0]\n" - "sdot z21.s, z5.b, z1.b[0]\n" - "sdot z18.s, z6.b, z0.b[0]\n" - "sdot z22.s, z6.b, z1.b[0]\n" + "ld1b { z24.b }, p2/Z, [x28]\n" + "ld1b { z26.b }, p2/Z, [x28, #1, MUL VL]\n" + "sdot z16.s, z24.b, z0.b[0]\n" + "sdot z20.s, z24.b, z1.b[0]\n" + "ld1b { z25.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #3, MUL VL]\n" + "sdot z17.s, z26.b, z0.b[0]\n" + "sdot z21.s, z26.b, z1.b[0]\n" + "sdot z18.s, z25.b, z0.b[0]\n" + "sdot z22.s, z25.b, z1.b[0]\n" "addvl x28, x28, #4\n" - "sdot z19.s, z7.b, z0.b[0]\n" - "sdot z23.s, z7.b, z1.b[0]\n" + "sdot z19.s, z24.b, z0.b[0]\n" + "sdot z23.s, z24.b, z1.b[0]\n" "ble 24f\n" - "ld1b { z8.b }, p2/Z, [x28]\n" - "ld1b { z9.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z27.b }, p2/Z, [x28]\n" + "ld1b { z26.b }, p2/Z, [x28, #1, MUL VL]\n" "subs x25, x25, #0x4\n" - "sdot z16.s, z8.b, z0.b[1]\n" - "ld1b { z10.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #3, MUL VL]\n" - "sdot z20.s, z8.b, z1.b[1]\n" - "sdot z17.s, z9.b, z0.b[1]\n" - "sdot z21.s, z9.b, z1.b[1]\n" - "sdot z18.s, z10.b, z0.b[1]\n" + "sdot z16.s, z27.b, z0.b[1]\n" + "ld1b { z25.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #3, MUL VL]\n" + "sdot z20.s, z27.b, z1.b[1]\n" + "sdot z17.s, z26.b, z0.b[1]\n" + "sdot z21.s, z26.b, z1.b[1]\n" + "sdot z18.s, z25.b, z0.b[1]\n" "addvl x28, x28, #4\n" - "sdot z22.s, z10.b, z1.b[1]\n" - "sdot z19.s, z4.b, z0.b[1]\n" - "sdot z23.s, z4.b, z1.b[1]\n" + "sdot z22.s, z25.b, z1.b[1]\n" + "sdot z19.s, z24.b, z0.b[1]\n" + "sdot z23.s, z24.b, z1.b[1]\n" "ble 24f\n" - "ld1b { z5.b }, p2/Z, [x28]\n" - "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z27.b }, p2/Z, [x28]\n" + "ld1b { z26.b }, p2/Z, [x28, #1, MUL VL]\n" "subs x25, x25, #0x4\n" - "sdot z16.s, z5.b, z0.b[2]\n" - "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" - "sdot z20.s, z5.b, z1.b[2]\n" - "sdot z17.s, z6.b, z0.b[2]\n" - "sdot z21.s, z6.b, z1.b[2]\n" - "sdot z18.s, z7.b, z0.b[2]\n" + "sdot z16.s, z27.b, z0.b[2]\n" + "ld1b { z25.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #3, MUL VL]\n" + "sdot z20.s, z27.b, z1.b[2]\n" + "sdot z17.s, z26.b, z0.b[2]\n" + "sdot z21.s, z26.b, z1.b[2]\n" + "sdot z18.s, z25.b, z0.b[2]\n" "addvl x28, x28, #4\n" - "sdot z22.s, z7.b, z1.b[2]\n" - "sdot z19.s, z8.b, z0.b[2]\n" - "sdot z23.s, z8.b, z1.b[2]\n" + "sdot z22.s, z25.b, z1.b[2]\n" + "sdot z19.s, z24.b, z0.b[2]\n" + "sdot z23.s, z24.b, z1.b[2]\n" "ble 24f\n" - "ld1b { z9.b }, p2/Z, [x28]\n" - "ld1b { z10.b }, p2/Z, [x28, #1, MUL VL]\n" - "sdot z16.s, z9.b, z0.b[3]\n" - "sdot z20.s, z9.b, z1.b[3]\n" - "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z5.b }, p2/Z, [x28, #3, MUL VL]\n" - "sdot z17.s, z10.b, z0.b[3]\n" - "sdot z21.s, z10.b, z1.b[3]\n" - "sdot z18.s, z4.b, z0.b[3]\n" - "sdot z22.s, z4.b, z1.b[3]\n" + "ld1b { z24.b }, p2/Z, [x28]\n" + "ld1b { z26.b }, p2/Z, [x28, #1, MUL VL]\n" + "sdot z16.s, z24.b, z0.b[3]\n" + "sdot z20.s, z24.b, z1.b[3]\n" + "ld1b { z25.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #3, MUL VL]\n" + "sdot z17.s, z26.b, z0.b[3]\n" + "sdot z21.s, z26.b, z1.b[3]\n" + "sdot z18.s, z25.b, z0.b[3]\n" + "sdot z22.s, z25.b, z1.b[3]\n" "addvl x28, x28, #4\n" - "sdot z19.s, z5.b, z0.b[3]\n" - "sdot z23.s, z5.b, z1.b[3]\n" + "sdot z19.s, z24.b, z0.b[3]\n" + "sdot z23.s, z24.b, z1.b[3]\n" "24:" // Height 2: Multiply loop: multiply skip "tbnz %x[flags], #31, 25f\n" "sdot z11.s, z0.b, z15.b\n" @@ -473,120 +473,120 @@ void sve_hybrid_s8qa_dot_4x4VL ( "mov x20, #0x4\n" "whilelt p0.s, XZR, x20\n" "add x20, %x[qp], %[b_offset]\n" - "ld1rw { z2.s }, p2/Z, [x20]\n" + "ld1rw { z24.s }, p2/Z, [x20]\n" "saddv d11, p0, z11.s\n" "mov z11.s, z11.s[0]\n" "saddv d12, p0, z12.s\n" - "neg z2.s, p2/M, z2.s\n" + "neg z24.s, p2/M, z24.s\n" "mov z12.s, z12.s[0]\n" - "mul z11.s, p2/M, z11.s, z2.s\n" - "mul z12.s, p2/M, z12.s, z2.s\n" + "mul z11.s, p2/M, z11.s, z24.s\n" + "mul z12.s, p2/M, z12.s, z24.s\n" "26:" // Height 2: skip row sum fixup "add z16.s, z16.s, z11.s\n" "add z17.s, z17.s, z11.s\n" - "ld1w { z0.s }, p2/Z, [x10]\n" - "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n" + "ld1w { z28.s }, p2/Z, [x10]\n" + "ld1w { z27.s }, p2/Z, [x10, #1, MUL VL]\n" "add z18.s, z18.s, z11.s\n" "add z19.s, z19.s, z11.s\n" - "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n" + "ld1w { z26.s }, p2/Z, [x10, #2, MUL VL]\n" + "ld1w { z25.s }, p2/Z, [x10, #3, MUL VL]\n" "add z20.s, z20.s, z12.s\n" "add z21.s, z21.s, z12.s\n" "add x20, %x[qp], %[per_layer_mul]\n" "orr %x[flags], %x[flags], #0x80000000\n" "add z22.s, z22.s, z12.s\n" "add z23.s, z23.s, z12.s\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + "ld1rw { z24.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[per_layer_right_shift]\n" - "add z16.s, z16.s, z0.s\n" - "add z17.s, z17.s, z1.s\n" + "add z16.s, z16.s, z28.s\n" + "add z17.s, z17.s, z27.s\n" "addvl x10, x10, #4\n" - "add z18.s, z18.s, z2.s\n" - "add z19.s, z19.s, z3.s\n" - "add z20.s, z20.s, z0.s\n" - "add z21.s, z21.s, z1.s\n" + "add z18.s, z18.s, z26.s\n" + "add z19.s, z19.s, z25.s\n" + "add z20.s, z20.s, z28.s\n" + "add z21.s, z21.s, z27.s\n" "ld1rw { z0.s }, p2/Z, [x20]\n" - "add z22.s, z22.s, z2.s\n" - "add z23.s, z23.s, z3.s\n" - ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n" - ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n" - ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n" - ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n" - ".inst 0x04a47694 // sqrdmulh z20.s, z20.s, z4.s\n" - ".inst 0x04a476b5 // sqrdmulh z21.s, z21.s, z4.s\n" - ".inst 0x04a476d6 // sqrdmulh z22.s, z22.s, z4.s\n" - ".inst 0x04a476f7 // sqrdmulh z23.s, z23.s, z4.s\n" + "add z22.s, z22.s, z26.s\n" + "add z23.s, z23.s, z25.s\n" + ".inst 0x04b87610 // sqrdmulh z16.s, z16.s, z24.s\n" + ".inst 0x04b87631 // sqrdmulh z17.s, z17.s, z24.s\n" + ".inst 0x04b87652 // sqrdmulh z18.s, z18.s, z24.s\n" + ".inst 0x04b87673 // sqrdmulh z19.s, z19.s, z24.s\n" + ".inst 0x04b87694 // sqrdmulh z20.s, z20.s, z24.s\n" + ".inst 0x04b876b5 // sqrdmulh z21.s, z21.s, z24.s\n" + ".inst 0x04b876d6 // sqrdmulh z22.s, z22.s, z24.s\n" + ".inst 0x04b876f7 // sqrdmulh z23.s, z23.s, z24.s\n" "tbz %x[flags], #5, 27f\n" - "and z4.d, z16.d, z0.d\n" - "asr z4.s, z4.s, #0x1f\n" - "sqadd z16.s, z16.s, z4.s\n" - "and z5.d, z17.d, z0.d\n" - "and z6.d, z18.d, z0.d\n" - "and z7.d, z19.d, z0.d\n" - "and z8.d, z20.d, z0.d\n" - "and z9.d, z21.d, z0.d\n" - "and z10.d, z22.d, z0.d\n" - "and z4.d, z23.d, z0.d\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "asr z8.s, z8.s, #0x1f\n" - "asr z9.s, z9.s, #0x1f\n" - "asr z10.s, z10.s, #0x1f\n" - "asr z4.s, z4.s, #0x1f\n" - "sqadd z17.s, z17.s, z5.s\n" - "sqadd z18.s, z18.s, z6.s\n" - "sqadd z19.s, z19.s, z7.s\n" - "sqadd z20.s, z20.s, z8.s\n" - "sqadd z21.s, z21.s, z9.s\n" - "sqadd z22.s, z22.s, z10.s\n" - "sqadd z23.s, z23.s, z4.s\n" + "and z24.d, z16.d, z0.d\n" + "asr z24.s, z24.s, #0x1f\n" + "sqadd z16.s, z16.s, z24.s\n" + "and z30.d, z17.d, z0.d\n" + "and z29.d, z18.d, z0.d\n" + "and z28.d, z19.d, z0.d\n" + "and z27.d, z20.d, z0.d\n" + "and z26.d, z21.d, z0.d\n" + "and z25.d, z22.d, z0.d\n" + "and z24.d, z23.d, z0.d\n" + "asr z30.s, z30.s, #0x1f\n" + "asr z29.s, z29.s, #0x1f\n" + "asr z28.s, z28.s, #0x1f\n" + "asr z27.s, z27.s, #0x1f\n" + "asr z26.s, z26.s, #0x1f\n" + "asr z25.s, z25.s, #0x1f\n" + "asr z24.s, z24.s, #0x1f\n" + "sqadd z17.s, z17.s, z30.s\n" + "sqadd z18.s, z18.s, z29.s\n" + "sqadd z19.s, z19.s, z28.s\n" + "sqadd z20.s, z20.s, z27.s\n" + "sqadd z21.s, z21.s, z26.s\n" + "sqadd z22.s, z22.s, z25.s\n" + "sqadd z23.s, z23.s, z24.s\n" "27:" // Height 2: no shift correction "add x20, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + "ld1rw { z24.s }, p2/Z, [x20]\n" ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n" - "add z16.s, z16.s, z4.s\n" + "add z16.s, z16.s, z24.s\n" ".inst 0x44828811 // srshl z17.s, p2/M, z17.s, z0.s\n" ".inst 0x44828812 // srshl z18.s, p2/M, z18.s, z0.s\n" - "add z17.s, z17.s, z4.s\n" - "add z18.s, z18.s, z4.s\n" + "add z17.s, z17.s, z24.s\n" + "add z18.s, z18.s, z24.s\n" ".inst 0x44828813 // srshl z19.s, p2/M, z19.s, z0.s\n" ".inst 0x44828814 // srshl z20.s, p2/M, z20.s, z0.s\n" - "add z19.s, z19.s, z4.s\n" - "add z20.s, z20.s, z4.s\n" + "add z19.s, z19.s, z24.s\n" + "add z20.s, z20.s, z24.s\n" ".inst 0x44828815 // srshl z21.s, p2/M, z21.s, z0.s\n" ".inst 0x44828816 // srshl z22.s, p2/M, z22.s, z0.s\n" - "add z21.s, z21.s, z4.s\n" - "add z22.s, z22.s, z4.s\n" + "add z21.s, z21.s, z24.s\n" + "add z22.s, z22.s, z24.s\n" ".inst 0x44828817 // srshl z23.s, p2/M, z23.s, z0.s\n" "add x20, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x20]\n" - "add z23.s, z23.s, z4.s\n" + "ld1rw { z25.s }, p2/Z, [x20]\n" + "add z23.s, z23.s, z24.s\n" "add x20, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x20]\n" - "smin z16.s, p2/M, z16.s, z6.s\n" - "smin z17.s, p2/M, z17.s, z6.s\n" - "smin z18.s, p2/M, z18.s, z6.s\n" - "smin z19.s, p2/M, z19.s, z6.s\n" - "smin z20.s, p2/M, z20.s, z6.s\n" - "smin z21.s, p2/M, z21.s, z6.s\n" - "smin z22.s, p2/M, z22.s, z6.s\n" - "smin z23.s, p2/M, z23.s, z6.s\n" - "smax z16.s, p2/M, z16.s, z5.s\n" - "smax z17.s, p2/M, z17.s, z5.s\n" - "smax z18.s, p2/M, z18.s, z5.s\n" + "ld1rw { z24.s }, p2/Z, [x20]\n" + "smin z16.s, p2/M, z16.s, z25.s\n" + "smin z17.s, p2/M, z17.s, z25.s\n" + "smin z18.s, p2/M, z18.s, z25.s\n" + "smin z19.s, p2/M, z19.s, z25.s\n" + "smin z20.s, p2/M, z20.s, z25.s\n" + "smin z21.s, p2/M, z21.s, z25.s\n" + "smin z22.s, p2/M, z22.s, z25.s\n" + "smin z23.s, p2/M, z23.s, z25.s\n" + "smax z16.s, p2/M, z16.s, z24.s\n" + "smax z17.s, p2/M, z17.s, z24.s\n" + "smax z18.s, p2/M, z18.s, z24.s\n" "uzp1 z16.h, z16.h, z17.h\n" - "smax z19.s, p2/M, z19.s, z5.s\n" - "smax z20.s, p2/M, z20.s, z5.s\n" + "smax z19.s, p2/M, z19.s, z24.s\n" + "smax z20.s, p2/M, z20.s, z24.s\n" "uzp1 z17.h, z18.h, z19.h\n" "uzp1 z16.b, z16.b, z17.b\n" - "smax z21.s, p2/M, z21.s, z5.s\n" - "smax z22.s, p2/M, z22.s, z5.s\n" + "smax z21.s, p2/M, z21.s, z24.s\n" + "smax z22.s, p2/M, z22.s, z24.s\n" "uzp1 z20.h, z20.h, z21.h\n" "st1b { z16.b }, p1, [x27]\n" - "smax z23.s, p2/M, z23.s, z5.s\n" - "uzp1 z21.h, z22.h, z23.h\n" - "uzp1 z20.b, z20.b, z21.b\n" + "smax z23.s, p2/M, z23.s, z24.s\n" + "uzp1 z16.h, z22.h, z23.h\n" + "uzp1 z20.b, z20.b, z16.b\n" "st1b { z20.b }, p1, [x23]\n" "addvl x27, x27, #1\n" "28:" // Height 2: Writeback done @@ -624,13 +624,13 @@ void sve_hybrid_s8qa_dot_4x4VL ( "32:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 33f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" "cbnz x26, 34f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -639,8 +639,8 @@ void sve_hybrid_s8qa_dot_4x4VL ( "b 34f\n" "33:" // Height 3: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" "34:" // Height 3: input setup done "cmp x25, #0x10\n" "ble 37f\n" @@ -650,73 +650,73 @@ void sve_hybrid_s8qa_dot_4x4VL ( "ld1rqb { z1.b }, p0/Z, [x23]\n" "add x24, x24, #0x10\n" "ld1rqb { z2.b }, p0/Z, [x22]\n" - "ld1b { z4.b }, p2/Z, [x28]\n" - "sdot z16.s, z4.b, z0.b[0]\n" - "sdot z20.s, z4.b, z1.b[0]\n" - "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n" - "sdot z24.s, z4.b, z2.b[0]\n" - "sdot z17.s, z5.b, z0.b[0]\n" - "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n" - "sdot z21.s, z5.b, z1.b[0]\n" - "sdot z25.s, z5.b, z2.b[0]\n" - "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n" - "sdot z18.s, z6.b, z0.b[0]\n" - "sdot z22.s, z6.b, z1.b[0]\n" - "ld1b { z9.b }, p2/Z, [x28, #5, MUL VL]\n" - "ld1b { z10.b }, p2/Z, [x28, #6, MUL VL]\n" - "sdot z26.s, z6.b, z2.b[0]\n" - "sdot z19.s, z7.b, z0.b[0]\n" - "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n" + "ld1b { z28.b }, p2/Z, [x28]\n" + "sdot z16.s, z28.b, z0.b[0]\n" + "sdot z20.s, z28.b, z1.b[0]\n" + "ld1b { z30.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z29.b }, p2/Z, [x28, #2, MUL VL]\n" + "sdot z24.s, z28.b, z2.b[0]\n" + "sdot z17.s, z30.b, z0.b[0]\n" + "ld1b { z28.b }, p2/Z, [x28, #3, MUL VL]\n" + "sdot z21.s, z30.b, z1.b[0]\n" + "sdot z25.s, z30.b, z2.b[0]\n" + "ld1b { z3.b }, p2/Z, [x28, #4, MUL VL]\n" + "sdot z18.s, z29.b, z0.b[0]\n" + "sdot z22.s, z29.b, z1.b[0]\n" + "ld1b { z31.b }, p2/Z, [x28, #5, MUL VL]\n" + "ld1b { z30.b }, p2/Z, [x28, #6, MUL VL]\n" + "sdot z26.s, z29.b, z2.b[0]\n" + "sdot z19.s, z28.b, z0.b[0]\n" + "ld1b { z29.b }, p2/Z, [x28, #7, MUL VL]\n" "addvl x28, x28, #16\n" - "sdot z23.s, z7.b, z1.b[0]\n" - "sdot z27.s, z7.b, z2.b[0]\n" - "ld1b { z5.b }, p2/Z, [x28, #-8, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n" - "sdot z16.s, z8.b, z0.b[1]\n" - "sdot z20.s, z8.b, z1.b[1]\n" - "ld1b { z7.b }, p2/Z, [x28, #-6, MUL VL]\n" + "sdot z23.s, z28.b, z1.b[0]\n" + "sdot z27.s, z28.b, z2.b[0]\n" + "ld1b { z28.b }, p2/Z, [x28, #-8, MUL VL]\n" + "ld1b { z5.b }, p2/Z, [x28, #-7, MUL VL]\n" + "sdot z16.s, z3.b, z0.b[1]\n" + "sdot z20.s, z3.b, z1.b[1]\n" + "ld1b { z4.b }, p2/Z, [x28, #-6, MUL VL]\n" "add x23, x23, #0x10\n" - "sdot z24.s, z8.b, z2.b[1]\n" - "sdot z17.s, z9.b, z0.b[1]\n" - "ld1b { z8.b }, p2/Z, [x28, #-5, MUL VL]\n" + "sdot z24.s, z3.b, z2.b[1]\n" + "sdot z17.s, z31.b, z0.b[1]\n" + "ld1b { z3.b }, p2/Z, [x28, #-5, MUL VL]\n" "add x22, x22, #0x10\n" - "sdot z21.s, z9.b, z1.b[1]\n" - "sdot z25.s, z9.b, z2.b[1]\n" - "ld1b { z9.b }, p2/Z, [x28, #-4, MUL VL]\n" - "sdot z18.s, z10.b, z0.b[1]\n" - "sdot z22.s, z10.b, z1.b[1]\n" - "sdot z26.s, z10.b, z2.b[1]\n" - "sdot z19.s, z4.b, z0.b[1]\n" - "ld1b { z10.b }, p2/Z, [x28, #-3, MUL VL]\n" - "sdot z23.s, z4.b, z1.b[1]\n" - "sdot z27.s, z4.b, z2.b[1]\n" - "ld1b { z4.b }, p2/Z, [x28, #-2, MUL VL]\n" - "sdot z16.s, z5.b, z0.b[2]\n" - "sdot z20.s, z5.b, z1.b[2]\n" - "sdot z24.s, z5.b, z2.b[2]\n" - "sdot z17.s, z6.b, z0.b[2]\n" - "ld1b { z5.b }, p2/Z, [x28, #-1, MUL VL]\n" - "sdot z21.s, z6.b, z1.b[2]\n" - "sdot z25.s, z6.b, z2.b[2]\n" - "sdot z18.s, z7.b, z0.b[2]\n" - "sdot z22.s, z7.b, z1.b[2]\n" - "sdot z26.s, z7.b, z2.b[2]\n" - "sdot z19.s, z8.b, z0.b[2]\n" - "sdot z23.s, z8.b, z1.b[2]\n" - "sdot z27.s, z8.b, z2.b[2]\n" - "sdot z16.s, z9.b, z0.b[3]\n" - "sdot z20.s, z9.b, z1.b[3]\n" - "sdot z24.s, z9.b, z2.b[3]\n" - "sdot z17.s, z10.b, z0.b[3]\n" - "sdot z21.s, z10.b, z1.b[3]\n" - "sdot z25.s, z10.b, z2.b[3]\n" - "sdot z18.s, z4.b, z0.b[3]\n" - "sdot z22.s, z4.b, z1.b[3]\n" - "sdot z26.s, z4.b, z2.b[3]\n" - "sdot z19.s, z5.b, z0.b[3]\n" - "sdot z23.s, z5.b, z1.b[3]\n" - "sdot z27.s, z5.b, z2.b[3]\n" + "sdot z21.s, z31.b, z1.b[1]\n" + "sdot z25.s, z31.b, z2.b[1]\n" + "ld1b { z31.b }, p2/Z, [x28, #-4, MUL VL]\n" + "sdot z18.s, z30.b, z0.b[1]\n" + "sdot z22.s, z30.b, z1.b[1]\n" + "sdot z26.s, z30.b, z2.b[1]\n" + "sdot z19.s, z29.b, z0.b[1]\n" + "ld1b { z30.b }, p2/Z, [x28, #-3, MUL VL]\n" + "sdot z23.s, z29.b, z1.b[1]\n" + "sdot z27.s, z29.b, z2.b[1]\n" + "ld1b { z29.b }, p2/Z, [x28, #-2, MUL VL]\n" + "sdot z16.s, z28.b, z0.b[2]\n" + "sdot z20.s, z28.b, z1.b[2]\n" + "sdot z24.s, z28.b, z2.b[2]\n" + "sdot z17.s, z5.b, z0.b[2]\n" + "ld1b { z28.b }, p2/Z, [x28, #-1, MUL VL]\n" + "sdot z21.s, z5.b, z1.b[2]\n" + "sdot z25.s, z5.b, z2.b[2]\n" + "sdot z18.s, z4.b, z0.b[2]\n" + "sdot z22.s, z4.b, z1.b[2]\n" + "sdot z26.s, z4.b, z2.b[2]\n" + "sdot z19.s, z3.b, z0.b[2]\n" + "sdot z23.s, z3.b, z1.b[2]\n" + "sdot z27.s, z3.b, z2.b[2]\n" + "sdot z16.s, z31.b, z0.b[3]\n" + "sdot z20.s, z31.b, z1.b[3]\n" + "sdot z24.s, z31.b, z2.b[3]\n" + "sdot z17.s, z30.b, z0.b[3]\n" + "sdot z21.s, z30.b, z1.b[3]\n" + "sdot z25.s, z30.b, z2.b[3]\n" + "sdot z18.s, z29.b, z0.b[3]\n" + "sdot z22.s, z29.b, z1.b[3]\n" + "sdot z26.s, z29.b, z2.b[3]\n" + "sdot z19.s, z28.b, z0.b[3]\n" + "sdot z23.s, z28.b, z1.b[3]\n" + "sdot z27.s, z28.b, z2.b[3]\n" "tbnz %x[flags], #31, 36f\n" "sdot z11.s, z0.b, z15.b\n" "sdot z12.s, z1.b, z15.b\n" @@ -731,79 +731,79 @@ void sve_hybrid_s8qa_dot_4x4VL ( "ld1rqb { z1.b }, p0/Z, [x23]\n" "subs x25, x25, #0x4\n" "ld1rqb { z2.b }, p0/Z, [x22]\n" - "ld1b { z4.b }, p2/Z, [x28]\n" - "sdot z16.s, z4.b, z0.b[0]\n" - "sdot z20.s, z4.b, z1.b[0]\n" - "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n" - "sdot z24.s, z4.b, z2.b[0]\n" - "sdot z17.s, z5.b, z0.b[0]\n" - "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n" - "sdot z21.s, z5.b, z1.b[0]\n" - "sdot z25.s, z5.b, z2.b[0]\n" + "ld1b { z28.b }, p2/Z, [x28]\n" + "sdot z16.s, z28.b, z0.b[0]\n" + "sdot z20.s, z28.b, z1.b[0]\n" + "ld1b { z30.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z29.b }, p2/Z, [x28, #2, MUL VL]\n" + "sdot z24.s, z28.b, z2.b[0]\n" + "sdot z17.s, z30.b, z0.b[0]\n" + "ld1b { z28.b }, p2/Z, [x28, #3, MUL VL]\n" + "sdot z21.s, z30.b, z1.b[0]\n" + "sdot z25.s, z30.b, z2.b[0]\n" "addvl x28, x28, #4\n" - "sdot z18.s, z6.b, z0.b[0]\n" - "sdot z22.s, z6.b, z1.b[0]\n" - "sdot z26.s, z6.b, z2.b[0]\n" - "sdot z19.s, z7.b, z0.b[0]\n" - "sdot z23.s, z7.b, z1.b[0]\n" - "sdot z27.s, z7.b, z2.b[0]\n" + "sdot z18.s, z29.b, z0.b[0]\n" + "sdot z22.s, z29.b, z1.b[0]\n" + "sdot z26.s, z29.b, z2.b[0]\n" + "sdot z19.s, z28.b, z0.b[0]\n" + "sdot z23.s, z28.b, z1.b[0]\n" + "sdot z27.s, z28.b, z2.b[0]\n" "ble 38f\n" - "ld1b { z8.b }, p2/Z, [x28]\n" - "ld1b { z9.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z31.b }, p2/Z, [x28]\n" + "ld1b { z30.b }, p2/Z, [x28, #1, MUL VL]\n" "subs x25, x25, #0x4\n" - "sdot z16.s, z8.b, z0.b[1]\n" - "ld1b { z10.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #3, MUL VL]\n" - "sdot z20.s, z8.b, z1.b[1]\n" - "sdot z24.s, z8.b, z2.b[1]\n" - "sdot z17.s, z9.b, z0.b[1]\n" - "sdot z21.s, z9.b, z1.b[1]\n" + "sdot z16.s, z31.b, z0.b[1]\n" + "ld1b { z29.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z28.b }, p2/Z, [x28, #3, MUL VL]\n" + "sdot z20.s, z31.b, z1.b[1]\n" + "sdot z24.s, z31.b, z2.b[1]\n" + "sdot z17.s, z30.b, z0.b[1]\n" + "sdot z21.s, z30.b, z1.b[1]\n" "addvl x28, x28, #4\n" - "sdot z25.s, z9.b, z2.b[1]\n" - "sdot z18.s, z10.b, z0.b[1]\n" - "sdot z22.s, z10.b, z1.b[1]\n" - "sdot z26.s, z10.b, z2.b[1]\n" - "sdot z19.s, z4.b, z0.b[1]\n" - "sdot z23.s, z4.b, z1.b[1]\n" - "sdot z27.s, z4.b, z2.b[1]\n" + "sdot z25.s, z30.b, z2.b[1]\n" + "sdot z18.s, z29.b, z0.b[1]\n" + "sdot z22.s, z29.b, z1.b[1]\n" + "sdot z26.s, z29.b, z2.b[1]\n" + "sdot z19.s, z28.b, z0.b[1]\n" + "sdot z23.s, z28.b, z1.b[1]\n" + "sdot z27.s, z28.b, z2.b[1]\n" "ble 38f\n" - "ld1b { z5.b }, p2/Z, [x28]\n" - "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z31.b }, p2/Z, [x28]\n" + "ld1b { z30.b }, p2/Z, [x28, #1, MUL VL]\n" "subs x25, x25, #0x4\n" - "sdot z16.s, z5.b, z0.b[2]\n" - "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" - "sdot z20.s, z5.b, z1.b[2]\n" - "sdot z24.s, z5.b, z2.b[2]\n" - "sdot z17.s, z6.b, z0.b[2]\n" - "sdot z21.s, z6.b, z1.b[2]\n" + "sdot z16.s, z31.b, z0.b[2]\n" + "ld1b { z29.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z28.b }, p2/Z, [x28, #3, MUL VL]\n" + "sdot z20.s, z31.b, z1.b[2]\n" + "sdot z24.s, z31.b, z2.b[2]\n" + "sdot z17.s, z30.b, z0.b[2]\n" + "sdot z21.s, z30.b, z1.b[2]\n" "addvl x28, x28, #4\n" - "sdot z25.s, z6.b, z2.b[2]\n" - "sdot z18.s, z7.b, z0.b[2]\n" - "sdot z22.s, z7.b, z1.b[2]\n" - "sdot z26.s, z7.b, z2.b[2]\n" - "sdot z19.s, z8.b, z0.b[2]\n" - "sdot z23.s, z8.b, z1.b[2]\n" - "sdot z27.s, z8.b, z2.b[2]\n" + "sdot z25.s, z30.b, z2.b[2]\n" + "sdot z18.s, z29.b, z0.b[2]\n" + "sdot z22.s, z29.b, z1.b[2]\n" + "sdot z26.s, z29.b, z2.b[2]\n" + "sdot z19.s, z28.b, z0.b[2]\n" + "sdot z23.s, z28.b, z1.b[2]\n" + "sdot z27.s, z28.b, z2.b[2]\n" "ble 38f\n" - "ld1b { z9.b }, p2/Z, [x28]\n" - "ld1b { z10.b }, p2/Z, [x28, #1, MUL VL]\n" - "sdot z16.s, z9.b, z0.b[3]\n" - "sdot z20.s, z9.b, z1.b[3]\n" - "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z5.b }, p2/Z, [x28, #3, MUL VL]\n" - "sdot z24.s, z9.b, z2.b[3]\n" - "sdot z17.s, z10.b, z0.b[3]\n" - "sdot z21.s, z10.b, z1.b[3]\n" - "sdot z25.s, z10.b, z2.b[3]\n" + "ld1b { z31.b }, p2/Z, [x28]\n" + "ld1b { z30.b }, p2/Z, [x28, #1, MUL VL]\n" + "sdot z16.s, z31.b, z0.b[3]\n" + "sdot z20.s, z31.b, z1.b[3]\n" + "ld1b { z29.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z28.b }, p2/Z, [x28, #3, MUL VL]\n" + "sdot z24.s, z31.b, z2.b[3]\n" + "sdot z17.s, z30.b, z0.b[3]\n" + "sdot z21.s, z30.b, z1.b[3]\n" + "sdot z25.s, z30.b, z2.b[3]\n" "addvl x28, x28, #4\n" - "sdot z18.s, z4.b, z0.b[3]\n" - "sdot z22.s, z4.b, z1.b[3]\n" - "sdot z26.s, z4.b, z2.b[3]\n" - "sdot z19.s, z5.b, z0.b[3]\n" - "sdot z23.s, z5.b, z1.b[3]\n" - "sdot z27.s, z5.b, z2.b[3]\n" + "sdot z18.s, z29.b, z0.b[3]\n" + "sdot z22.s, z29.b, z1.b[3]\n" + "sdot z26.s, z29.b, z2.b[3]\n" + "sdot z19.s, z28.b, z0.b[3]\n" + "sdot z23.s, z28.b, z1.b[3]\n" + "sdot z27.s, z28.b, z2.b[3]\n" "38:" // Height 3: Multiply loop: multiply skip "tbnz %x[flags], #31, 39f\n" "sdot z11.s, z0.b, z15.b\n" @@ -821,33 +821,33 @@ void sve_hybrid_s8qa_dot_4x4VL ( "mov x20, #0x4\n" "whilelt p0.s, XZR, x20\n" "add x20, %x[qp], %[b_offset]\n" - "ld1rw { z3.s }, p2/Z, [x20]\n" + "ld1rw { z28.s }, p2/Z, [x20]\n" "saddv d11, p0, z11.s\n" "mov z11.s, z11.s[0]\n" "saddv d12, p0, z12.s\n" "saddv d13, p0, z13.s\n" "mov z12.s, z12.s[0]\n" "mov z13.s, z13.s[0]\n" - "neg z3.s, p2/M, z3.s\n" - "mul z11.s, p2/M, z11.s, z3.s\n" - "mul z12.s, p2/M, z12.s, z3.s\n" - "mul z13.s, p2/M, z13.s, z3.s\n" + "neg z28.s, p2/M, z28.s\n" + "mul z11.s, p2/M, z11.s, z28.s\n" + "mul z12.s, p2/M, z12.s, z28.s\n" + "mul z13.s, p2/M, z13.s, z28.s\n" "40:" // Height 3: skip row sum fixup "add z16.s, z16.s, z11.s\n" "add z17.s, z17.s, z11.s\n" "ld1w { z0.s }, p2/Z, [x10]\n" - "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n" + "ld1w { z31.s }, p2/Z, [x10, #1, MUL VL]\n" "add z18.s, z18.s, z11.s\n" "add z19.s, z19.s, z11.s\n" - "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n" + "ld1w { z30.s }, p2/Z, [x10, #2, MUL VL]\n" + "ld1w { z29.s }, p2/Z, [x10, #3, MUL VL]\n" "add z20.s, z20.s, z12.s\n" "add z21.s, z21.s, z12.s\n" "add x20, %x[qp], %[per_layer_mul]\n" "orr %x[flags], %x[flags], #0x80000000\n" "add z22.s, z22.s, z12.s\n" "add z23.s, z23.s, z12.s\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + "ld1rw { z28.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[per_layer_right_shift]\n" "add z24.s, z24.s, z13.s\n" "add z25.s, z25.s, z13.s\n" @@ -855,133 +855,133 @@ void sve_hybrid_s8qa_dot_4x4VL ( "add z26.s, z26.s, z13.s\n" "add z27.s, z27.s, z13.s\n" "add z16.s, z16.s, z0.s\n" - "add z17.s, z17.s, z1.s\n" - "add z18.s, z18.s, z2.s\n" - "add z19.s, z19.s, z3.s\n" + "add z17.s, z17.s, z31.s\n" + "add z18.s, z18.s, z30.s\n" + "add z19.s, z19.s, z29.s\n" "add z20.s, z20.s, z0.s\n" - "add z21.s, z21.s, z1.s\n" - "add z22.s, z22.s, z2.s\n" - "add z23.s, z23.s, z3.s\n" + "add z21.s, z21.s, z31.s\n" + "add z22.s, z22.s, z30.s\n" + "add z23.s, z23.s, z29.s\n" "add z24.s, z24.s, z0.s\n" - "add z25.s, z25.s, z1.s\n" + "add z25.s, z25.s, z31.s\n" "ld1rw { z0.s }, p2/Z, [x20]\n" - "add z26.s, z26.s, z2.s\n" - "add z27.s, z27.s, z3.s\n" - ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n" - ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n" - ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n" - ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n" - ".inst 0x04a47694 // sqrdmulh z20.s, z20.s, z4.s\n" - ".inst 0x04a476b5 // sqrdmulh z21.s, z21.s, z4.s\n" - ".inst 0x04a476d6 // sqrdmulh z22.s, z22.s, z4.s\n" - ".inst 0x04a476f7 // sqrdmulh z23.s, z23.s, z4.s\n" - ".inst 0x04a47718 // sqrdmulh z24.s, z24.s, z4.s\n" - ".inst 0x04a47739 // sqrdmulh z25.s, z25.s, z4.s\n" - ".inst 0x04a4775a // sqrdmulh z26.s, z26.s, z4.s\n" - ".inst 0x04a4777b // sqrdmulh z27.s, z27.s, z4.s\n" + "add z26.s, z26.s, z30.s\n" + "add z27.s, z27.s, z29.s\n" + ".inst 0x04bc7610 // sqrdmulh z16.s, z16.s, z28.s\n" + ".inst 0x04bc7631 // sqrdmulh z17.s, z17.s, z28.s\n" + ".inst 0x04bc7652 // sqrdmulh z18.s, z18.s, z28.s\n" + ".inst 0x04bc7673 // sqrdmulh z19.s, z19.s, z28.s\n" + ".inst 0x04bc7694 // sqrdmulh z20.s, z20.s, z28.s\n" + ".inst 0x04bc76b5 // sqrdmulh z21.s, z21.s, z28.s\n" + ".inst 0x04bc76d6 // sqrdmulh z22.s, z22.s, z28.s\n" + ".inst 0x04bc76f7 // sqrdmulh z23.s, z23.s, z28.s\n" + ".inst 0x04bc7718 // sqrdmulh z24.s, z24.s, z28.s\n" + ".inst 0x04bc7739 // sqrdmulh z25.s, z25.s, z28.s\n" + ".inst 0x04bc775a // sqrdmulh z26.s, z26.s, z28.s\n" + ".inst 0x04bc777b // sqrdmulh z27.s, z27.s, z28.s\n" "tbz %x[flags], #5, 41f\n" - "and z4.d, z16.d, z0.d\n" - "and z5.d, z17.d, z0.d\n" - "and z6.d, z18.d, z0.d\n" - "and z7.d, z19.d, z0.d\n" - "and z8.d, z20.d, z0.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "asr z8.s, z8.s, #0x1f\n" - "sqadd z16.s, z16.s, z4.s\n" - "sqadd z17.s, z17.s, z5.s\n" - "sqadd z18.s, z18.s, z6.s\n" - "sqadd z19.s, z19.s, z7.s\n" - "sqadd z20.s, z20.s, z8.s\n" - "and z9.d, z21.d, z0.d\n" - "and z10.d, z22.d, z0.d\n" - "and z4.d, z23.d, z0.d\n" - "and z5.d, z24.d, z0.d\n" - "and z6.d, z25.d, z0.d\n" - "and z7.d, z26.d, z0.d\n" - "and z8.d, z27.d, z0.d\n" - "asr z9.s, z9.s, #0x1f\n" - "asr z10.s, z10.s, #0x1f\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "asr z8.s, z8.s, #0x1f\n" - "sqadd z21.s, z21.s, z9.s\n" - "sqadd z22.s, z22.s, z10.s\n" - "sqadd z23.s, z23.s, z4.s\n" - "sqadd z24.s, z24.s, z5.s\n" - "sqadd z25.s, z25.s, z6.s\n" - "sqadd z26.s, z26.s, z7.s\n" - "sqadd z27.s, z27.s, z8.s\n" + "and z1.d, z16.d, z0.d\n" + "and z31.d, z17.d, z0.d\n" + "and z30.d, z18.d, z0.d\n" + "and z29.d, z19.d, z0.d\n" + "and z28.d, z20.d, z0.d\n" + "asr z1.s, z1.s, #0x1f\n" + "asr z31.s, z31.s, #0x1f\n" + "asr z30.s, z30.s, #0x1f\n" + "asr z29.s, z29.s, #0x1f\n" + "asr z28.s, z28.s, #0x1f\n" + "sqadd z16.s, z16.s, z1.s\n" + "sqadd z17.s, z17.s, z31.s\n" + "sqadd z18.s, z18.s, z30.s\n" + "sqadd z19.s, z19.s, z29.s\n" + "sqadd z20.s, z20.s, z28.s\n" + "and z3.d, z21.d, z0.d\n" + "and z2.d, z22.d, z0.d\n" + "and z1.d, z23.d, z0.d\n" + "and z31.d, z24.d, z0.d\n" + "and z30.d, z25.d, z0.d\n" + "and z29.d, z26.d, z0.d\n" + "and z28.d, z27.d, z0.d\n" + "asr z3.s, z3.s, #0x1f\n" + "asr z2.s, z2.s, #0x1f\n" + "asr z1.s, z1.s, #0x1f\n" + "asr z31.s, z31.s, #0x1f\n" + "asr z30.s, z30.s, #0x1f\n" + "asr z29.s, z29.s, #0x1f\n" + "asr z28.s, z28.s, #0x1f\n" + "sqadd z21.s, z21.s, z3.s\n" + "sqadd z22.s, z22.s, z2.s\n" + "sqadd z23.s, z23.s, z1.s\n" + "sqadd z24.s, z24.s, z31.s\n" + "sqadd z25.s, z25.s, z30.s\n" + "sqadd z26.s, z26.s, z29.s\n" + "sqadd z27.s, z27.s, z28.s\n" "41:" // Height 3: no shift correction "add x20, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + "ld1rw { z28.s }, p2/Z, [x20]\n" ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n" - "add z16.s, z16.s, z4.s\n" + "add z16.s, z16.s, z28.s\n" ".inst 0x44828811 // srshl z17.s, p2/M, z17.s, z0.s\n" ".inst 0x44828812 // srshl z18.s, p2/M, z18.s, z0.s\n" - "add z17.s, z17.s, z4.s\n" - "add z18.s, z18.s, z4.s\n" + "add z17.s, z17.s, z28.s\n" + "add z18.s, z18.s, z28.s\n" ".inst 0x44828813 // srshl z19.s, p2/M, z19.s, z0.s\n" ".inst 0x44828814 // srshl z20.s, p2/M, z20.s, z0.s\n" - "add z19.s, z19.s, z4.s\n" - "add z20.s, z20.s, z4.s\n" + "add z19.s, z19.s, z28.s\n" + "add z20.s, z20.s, z28.s\n" ".inst 0x44828815 // srshl z21.s, p2/M, z21.s, z0.s\n" ".inst 0x44828816 // srshl z22.s, p2/M, z22.s, z0.s\n" - "add z21.s, z21.s, z4.s\n" - "add z22.s, z22.s, z4.s\n" + "add z21.s, z21.s, z28.s\n" + "add z22.s, z22.s, z28.s\n" ".inst 0x44828817 // srshl z23.s, p2/M, z23.s, z0.s\n" ".inst 0x44828818 // srshl z24.s, p2/M, z24.s, z0.s\n" - "add z23.s, z23.s, z4.s\n" - "add z24.s, z24.s, z4.s\n" + "add z23.s, z23.s, z28.s\n" + "add z24.s, z24.s, z28.s\n" ".inst 0x44828819 // srshl z25.s, p2/M, z25.s, z0.s\n" ".inst 0x4482881a // srshl z26.s, p2/M, z26.s, z0.s\n" - "add z25.s, z25.s, z4.s\n" - "add z26.s, z26.s, z4.s\n" + "add z25.s, z25.s, z28.s\n" + "add z26.s, z26.s, z28.s\n" ".inst 0x4482881b // srshl z27.s, p2/M, z27.s, z0.s\n" "add x20, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x20]\n" - "add z27.s, z27.s, z4.s\n" + "ld1rw { z29.s }, p2/Z, [x20]\n" + "add z27.s, z27.s, z28.s\n" "add x20, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x20]\n" - "smin z16.s, p2/M, z16.s, z6.s\n" - "smin z17.s, p2/M, z17.s, z6.s\n" - "smin z18.s, p2/M, z18.s, z6.s\n" - "smin z19.s, p2/M, z19.s, z6.s\n" - "smin z20.s, p2/M, z20.s, z6.s\n" - "smin z21.s, p2/M, z21.s, z6.s\n" - "smin z22.s, p2/M, z22.s, z6.s\n" - "smin z23.s, p2/M, z23.s, z6.s\n" - "smin z24.s, p2/M, z24.s, z6.s\n" - "smin z25.s, p2/M, z25.s, z6.s\n" - "smin z26.s, p2/M, z26.s, z6.s\n" - "smin z27.s, p2/M, z27.s, z6.s\n" - "smax z16.s, p2/M, z16.s, z5.s\n" - "smax z17.s, p2/M, z17.s, z5.s\n" - "smax z18.s, p2/M, z18.s, z5.s\n" + "ld1rw { z28.s }, p2/Z, [x20]\n" + "smin z16.s, p2/M, z16.s, z29.s\n" + "smin z17.s, p2/M, z17.s, z29.s\n" + "smin z18.s, p2/M, z18.s, z29.s\n" + "smin z19.s, p2/M, z19.s, z29.s\n" + "smin z20.s, p2/M, z20.s, z29.s\n" + "smin z21.s, p2/M, z21.s, z29.s\n" + "smin z22.s, p2/M, z22.s, z29.s\n" + "smin z23.s, p2/M, z23.s, z29.s\n" + "smin z24.s, p2/M, z24.s, z29.s\n" + "smin z25.s, p2/M, z25.s, z29.s\n" + "smin z26.s, p2/M, z26.s, z29.s\n" + "smin z27.s, p2/M, z27.s, z29.s\n" + "smax z16.s, p2/M, z16.s, z28.s\n" + "smax z17.s, p2/M, z17.s, z28.s\n" + "smax z18.s, p2/M, z18.s, z28.s\n" "uzp1 z16.h, z16.h, z17.h\n" - "smax z19.s, p2/M, z19.s, z5.s\n" - "smax z20.s, p2/M, z20.s, z5.s\n" + "smax z19.s, p2/M, z19.s, z28.s\n" + "smax z20.s, p2/M, z20.s, z28.s\n" "uzp1 z17.h, z18.h, z19.h\n" "uzp1 z16.b, z16.b, z17.b\n" - "smax z21.s, p2/M, z21.s, z5.s\n" - "smax z22.s, p2/M, z22.s, z5.s\n" + "smax z21.s, p2/M, z21.s, z28.s\n" + "smax z22.s, p2/M, z22.s, z28.s\n" "uzp1 z20.h, z20.h, z21.h\n" "st1b { z16.b }, p1, [x27]\n" - "smax z23.s, p2/M, z23.s, z5.s\n" - "smax z24.s, p2/M, z24.s, z5.s\n" - "uzp1 z21.h, z22.h, z23.h\n" - "uzp1 z20.b, z20.b, z21.b\n" - "smax z25.s, p2/M, z25.s, z5.s\n" - "smax z26.s, p2/M, z26.s, z5.s\n" + "smax z23.s, p2/M, z23.s, z28.s\n" + "smax z24.s, p2/M, z24.s, z28.s\n" + "uzp1 z16.h, z22.h, z23.h\n" + "uzp1 z20.b, z20.b, z16.b\n" + "smax z25.s, p2/M, z25.s, z28.s\n" + "smax z26.s, p2/M, z26.s, z28.s\n" "uzp1 z24.h, z24.h, z25.h\n" "st1b { z20.b }, p1, [x23]\n" - "smax z27.s, p2/M, z27.s, z5.s\n" - "uzp1 z25.h, z26.h, z27.h\n" - "uzp1 z24.b, z24.b, z25.b\n" + "smax z27.s, p2/M, z27.s, z28.s\n" + "uzp1 z16.h, z26.h, z27.h\n" + "uzp1 z24.b, z24.b, z16.b\n" "st1b { z24.b }, p1, [x22]\n" "addvl x27, x27, #1\n" "42:" // Height 3: Writeback done @@ -1027,14 +1027,14 @@ void sve_hybrid_s8qa_dot_4x4VL ( "46:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 47f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" - "ldr x21, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" + "ldr x21, [x20, #0x18]\n" "cbnz x26, 48f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -1044,9 +1044,9 @@ void sve_hybrid_s8qa_dot_4x4VL ( "b 48f\n" "47:" // Height 4: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" + "add x21, x22, x21\n" "48:" // Height 4: input setup done "cmp x25, #0x10\n" "ble 51f\n" @@ -1059,88 +1059,88 @@ void sve_hybrid_s8qa_dot_4x4VL ( "ld1rqb { z3.b }, p0/Z, [x21]\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - "ld1b { z4.b }, p2/Z, [x28]\n" - "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n" - "sdot z16.s, z4.b, z0.b[0]\n" - "sdot z20.s, z4.b, z1.b[0]\n" - "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n" - "sdot z24.s, z4.b, z2.b[0]\n" - "sdot z28.s, z4.b, z3.b[0]\n" - "sdot z17.s, z5.b, z0.b[0]\n" - "sdot z21.s, z5.b, z1.b[0]\n" + "ld1b { z5.b }, p2/Z, [x28]\n" + "ld1b { z4.b }, p2/Z, [x28, #1, MUL VL]\n" + "sdot z16.s, z5.b, z0.b[0]\n" + "sdot z20.s, z5.b, z1.b[0]\n" + "ld1b { z10.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z9.b }, p2/Z, [x28, #3, MUL VL]\n" + "sdot z24.s, z5.b, z2.b[0]\n" + "sdot z28.s, z5.b, z3.b[0]\n" + "sdot z17.s, z4.b, z0.b[0]\n" + "sdot z21.s, z4.b, z1.b[0]\n" "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n" - "ld1b { z9.b }, p2/Z, [x28, #5, MUL VL]\n" - "sdot z25.s, z5.b, z2.b[0]\n" - "sdot z29.s, z5.b, z3.b[0]\n" - "ld1b { z10.b }, p2/Z, [x28, #6, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n" - "sdot z18.s, z6.b, z0.b[0]\n" - "sdot z22.s, z6.b, z1.b[0]\n" + "ld1b { z7.b }, p2/Z, [x28, #5, MUL VL]\n" + "sdot z25.s, z4.b, z2.b[0]\n" + "sdot z29.s, z4.b, z3.b[0]\n" + "ld1b { z6.b }, p2/Z, [x28, #6, MUL VL]\n" + "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n" + "sdot z18.s, z10.b, z0.b[0]\n" + "sdot z22.s, z10.b, z1.b[0]\n" "addvl x28, x28, #16\n" - "ld1b { z5.b }, p2/Z, [x28, #-8, MUL VL]\n" - "sdot z26.s, z6.b, z2.b[0]\n" - "sdot z30.s, z6.b, z3.b[0]\n" - "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n" + "ld1b { z4.b }, p2/Z, [x28, #-8, MUL VL]\n" + "sdot z26.s, z10.b, z2.b[0]\n" + "sdot z30.s, z10.b, z3.b[0]\n" + "ld1b { z10.b }, p2/Z, [x28, #-7, MUL VL]\n" "add x21, x21, #0x10\n" - "sdot z19.s, z7.b, z0.b[0]\n" - "sdot z23.s, z7.b, z1.b[0]\n" - "sdot z27.s, z7.b, z2.b[0]\n" - "sdot z31.s, z7.b, z3.b[0]\n" - "ld1b { z7.b }, p2/Z, [x28, #-6, MUL VL]\n" + "sdot z19.s, z9.b, z0.b[0]\n" + "sdot z23.s, z9.b, z1.b[0]\n" + "sdot z27.s, z9.b, z2.b[0]\n" + "sdot z31.s, z9.b, z3.b[0]\n" + "ld1b { z9.b }, p2/Z, [x28, #-6, MUL VL]\n" "sdot z16.s, z8.b, z0.b[1]\n" "sdot z20.s, z8.b, z1.b[1]\n" "sdot z24.s, z8.b, z2.b[1]\n" "sdot z28.s, z8.b, z3.b[1]\n" "ld1b { z8.b }, p2/Z, [x28, #-5, MUL VL]\n" - "sdot z17.s, z9.b, z0.b[1]\n" - "sdot z21.s, z9.b, z1.b[1]\n" - "sdot z25.s, z9.b, z2.b[1]\n" - "sdot z29.s, z9.b, z3.b[1]\n" - "ld1b { z9.b }, p2/Z, [x28, #-4, MUL VL]\n" - "sdot z18.s, z10.b, z0.b[1]\n" - "sdot z22.s, z10.b, z1.b[1]\n" - "sdot z26.s, z10.b, z2.b[1]\n" - "sdot z30.s, z10.b, z3.b[1]\n" - "ld1b { z10.b }, p2/Z, [x28, #-3, MUL VL]\n" - "sdot z19.s, z4.b, z0.b[1]\n" - "sdot z23.s, z4.b, z1.b[1]\n" - "sdot z27.s, z4.b, z2.b[1]\n" - "sdot z31.s, z4.b, z3.b[1]\n" - "ld1b { z4.b }, p2/Z, [x28, #-2, MUL VL]\n" - "sdot z16.s, z5.b, z0.b[2]\n" - "sdot z20.s, z5.b, z1.b[2]\n" - "sdot z24.s, z5.b, z2.b[2]\n" - "sdot z28.s, z5.b, z3.b[2]\n" - "ld1b { z5.b }, p2/Z, [x28, #-1, MUL VL]\n" - "sdot z17.s, z6.b, z0.b[2]\n" - "sdot z21.s, z6.b, z1.b[2]\n" - "sdot z25.s, z6.b, z2.b[2]\n" - "sdot z29.s, z6.b, z3.b[2]\n" - "sdot z18.s, z7.b, z0.b[2]\n" - "sdot z22.s, z7.b, z1.b[2]\n" - "sdot z26.s, z7.b, z2.b[2]\n" - "sdot z30.s, z7.b, z3.b[2]\n" + "sdot z17.s, z7.b, z0.b[1]\n" + "sdot z21.s, z7.b, z1.b[1]\n" + "sdot z25.s, z7.b, z2.b[1]\n" + "sdot z29.s, z7.b, z3.b[1]\n" + "ld1b { z7.b }, p2/Z, [x28, #-4, MUL VL]\n" + "sdot z18.s, z6.b, z0.b[1]\n" + "sdot z22.s, z6.b, z1.b[1]\n" + "sdot z26.s, z6.b, z2.b[1]\n" + "sdot z30.s, z6.b, z3.b[1]\n" + "ld1b { z6.b }, p2/Z, [x28, #-3, MUL VL]\n" + "sdot z19.s, z5.b, z0.b[1]\n" + "sdot z23.s, z5.b, z1.b[1]\n" + "sdot z27.s, z5.b, z2.b[1]\n" + "sdot z31.s, z5.b, z3.b[1]\n" + "ld1b { z5.b }, p2/Z, [x28, #-2, MUL VL]\n" + "sdot z16.s, z4.b, z0.b[2]\n" + "sdot z20.s, z4.b, z1.b[2]\n" + "sdot z24.s, z4.b, z2.b[2]\n" + "sdot z28.s, z4.b, z3.b[2]\n" + "ld1b { z4.b }, p2/Z, [x28, #-1, MUL VL]\n" + "sdot z17.s, z10.b, z0.b[2]\n" + "sdot z21.s, z10.b, z1.b[2]\n" + "sdot z25.s, z10.b, z2.b[2]\n" + "sdot z29.s, z10.b, z3.b[2]\n" + "sdot z18.s, z9.b, z0.b[2]\n" + "sdot z22.s, z9.b, z1.b[2]\n" + "sdot z26.s, z9.b, z2.b[2]\n" + "sdot z30.s, z9.b, z3.b[2]\n" "sdot z19.s, z8.b, z0.b[2]\n" "sdot z23.s, z8.b, z1.b[2]\n" "sdot z27.s, z8.b, z2.b[2]\n" "sdot z31.s, z8.b, z3.b[2]\n" - "sdot z16.s, z9.b, z0.b[3]\n" - "sdot z20.s, z9.b, z1.b[3]\n" - "sdot z24.s, z9.b, z2.b[3]\n" - "sdot z28.s, z9.b, z3.b[3]\n" - "sdot z17.s, z10.b, z0.b[3]\n" - "sdot z21.s, z10.b, z1.b[3]\n" - "sdot z25.s, z10.b, z2.b[3]\n" - "sdot z29.s, z10.b, z3.b[3]\n" - "sdot z18.s, z4.b, z0.b[3]\n" - "sdot z22.s, z4.b, z1.b[3]\n" - "sdot z26.s, z4.b, z2.b[3]\n" - "sdot z30.s, z4.b, z3.b[3]\n" - "sdot z19.s, z5.b, z0.b[3]\n" - "sdot z23.s, z5.b, z1.b[3]\n" - "sdot z27.s, z5.b, z2.b[3]\n" - "sdot z31.s, z5.b, z3.b[3]\n" + "sdot z16.s, z7.b, z0.b[3]\n" + "sdot z20.s, z7.b, z1.b[3]\n" + "sdot z24.s, z7.b, z2.b[3]\n" + "sdot z28.s, z7.b, z3.b[3]\n" + "sdot z17.s, z6.b, z0.b[3]\n" + "sdot z21.s, z6.b, z1.b[3]\n" + "sdot z25.s, z6.b, z2.b[3]\n" + "sdot z29.s, z6.b, z3.b[3]\n" + "sdot z18.s, z5.b, z0.b[3]\n" + "sdot z22.s, z5.b, z1.b[3]\n" + "sdot z26.s, z5.b, z2.b[3]\n" + "sdot z30.s, z5.b, z3.b[3]\n" + "sdot z19.s, z4.b, z0.b[3]\n" + "sdot z23.s, z4.b, z1.b[3]\n" + "sdot z27.s, z4.b, z2.b[3]\n" + "sdot z31.s, z4.b, z3.b[3]\n" "tbnz %x[flags], #31, 50f\n" "sdot z11.s, z0.b, z15.b\n" "sdot z12.s, z1.b, z15.b\n" @@ -1157,95 +1157,95 @@ void sve_hybrid_s8qa_dot_4x4VL ( "subs x25, x25, #0x4\n" "ld1rqb { z2.b }, p0/Z, [x22]\n" "ld1rqb { z3.b }, p0/Z, [x21]\n" - "ld1b { z4.b }, p2/Z, [x28]\n" - "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n" - "sdot z16.s, z4.b, z0.b[0]\n" - "sdot z20.s, z4.b, z1.b[0]\n" - "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n" - "sdot z24.s, z4.b, z2.b[0]\n" - "sdot z28.s, z4.b, z3.b[0]\n" - "sdot z17.s, z5.b, z0.b[0]\n" - "sdot z21.s, z5.b, z1.b[0]\n" + "ld1b { z7.b }, p2/Z, [x28]\n" + "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" + "sdot z16.s, z7.b, z0.b[0]\n" + "sdot z20.s, z7.b, z1.b[0]\n" + "ld1b { z5.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z4.b }, p2/Z, [x28, #3, MUL VL]\n" + "sdot z24.s, z7.b, z2.b[0]\n" + "sdot z28.s, z7.b, z3.b[0]\n" + "sdot z17.s, z6.b, z0.b[0]\n" + "sdot z21.s, z6.b, z1.b[0]\n" "addvl x28, x28, #4\n" - "sdot z25.s, z5.b, z2.b[0]\n" - "sdot z29.s, z5.b, z3.b[0]\n" - "sdot z18.s, z6.b, z0.b[0]\n" - "sdot z22.s, z6.b, z1.b[0]\n" - "sdot z26.s, z6.b, z2.b[0]\n" - "sdot z30.s, z6.b, z3.b[0]\n" - "sdot z19.s, z7.b, z0.b[0]\n" - "sdot z23.s, z7.b, z1.b[0]\n" - "sdot z27.s, z7.b, z2.b[0]\n" - "sdot z31.s, z7.b, z3.b[0]\n" + "sdot z25.s, z6.b, z2.b[0]\n" + "sdot z29.s, z6.b, z3.b[0]\n" + "sdot z18.s, z5.b, z0.b[0]\n" + "sdot z22.s, z5.b, z1.b[0]\n" + "sdot z26.s, z5.b, z2.b[0]\n" + "sdot z30.s, z5.b, z3.b[0]\n" + "sdot z19.s, z4.b, z0.b[0]\n" + "sdot z23.s, z4.b, z1.b[0]\n" + "sdot z27.s, z4.b, z2.b[0]\n" + "sdot z31.s, z4.b, z3.b[0]\n" "ble 52f\n" - "ld1b { z8.b }, p2/Z, [x28]\n" - "ld1b { z9.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z7.b }, p2/Z, [x28]\n" + "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" "subs x25, x25, #0x4\n" - "sdot z16.s, z8.b, z0.b[1]\n" - "ld1b { z10.b }, p2/Z, [x28, #2, MUL VL]\n" + "sdot z16.s, z7.b, z0.b[1]\n" + "ld1b { z5.b }, p2/Z, [x28, #2, MUL VL]\n" "ld1b { z4.b }, p2/Z, [x28, #3, MUL VL]\n" - "sdot z20.s, z8.b, z1.b[1]\n" - "sdot z24.s, z8.b, z2.b[1]\n" - "sdot z28.s, z8.b, z3.b[1]\n" - "sdot z17.s, z9.b, z0.b[1]\n" + "sdot z20.s, z7.b, z1.b[1]\n" + "sdot z24.s, z7.b, z2.b[1]\n" + "sdot z28.s, z7.b, z3.b[1]\n" + "sdot z17.s, z6.b, z0.b[1]\n" "addvl x28, x28, #4\n" - "sdot z21.s, z9.b, z1.b[1]\n" - "sdot z25.s, z9.b, z2.b[1]\n" - "sdot z29.s, z9.b, z3.b[1]\n" - "sdot z18.s, z10.b, z0.b[1]\n" - "sdot z22.s, z10.b, z1.b[1]\n" - "sdot z26.s, z10.b, z2.b[1]\n" - "sdot z30.s, z10.b, z3.b[1]\n" + "sdot z21.s, z6.b, z1.b[1]\n" + "sdot z25.s, z6.b, z2.b[1]\n" + "sdot z29.s, z6.b, z3.b[1]\n" + "sdot z18.s, z5.b, z0.b[1]\n" + "sdot z22.s, z5.b, z1.b[1]\n" + "sdot z26.s, z5.b, z2.b[1]\n" + "sdot z30.s, z5.b, z3.b[1]\n" "sdot z19.s, z4.b, z0.b[1]\n" "sdot z23.s, z4.b, z1.b[1]\n" "sdot z27.s, z4.b, z2.b[1]\n" "sdot z31.s, z4.b, z3.b[1]\n" "ble 52f\n" - "ld1b { z5.b }, p2/Z, [x28]\n" + "ld1b { z7.b }, p2/Z, [x28]\n" "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" "subs x25, x25, #0x4\n" - "sdot z16.s, z5.b, z0.b[2]\n" - "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" - "sdot z20.s, z5.b, z1.b[2]\n" - "sdot z24.s, z5.b, z2.b[2]\n" - "sdot z28.s, z5.b, z3.b[2]\n" + "sdot z16.s, z7.b, z0.b[2]\n" + "ld1b { z5.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z4.b }, p2/Z, [x28, #3, MUL VL]\n" + "sdot z20.s, z7.b, z1.b[2]\n" + "sdot z24.s, z7.b, z2.b[2]\n" + "sdot z28.s, z7.b, z3.b[2]\n" "sdot z17.s, z6.b, z0.b[2]\n" "addvl x28, x28, #4\n" "sdot z21.s, z6.b, z1.b[2]\n" "sdot z25.s, z6.b, z2.b[2]\n" "sdot z29.s, z6.b, z3.b[2]\n" - "sdot z18.s, z7.b, z0.b[2]\n" - "sdot z22.s, z7.b, z1.b[2]\n" - "sdot z26.s, z7.b, z2.b[2]\n" - "sdot z30.s, z7.b, z3.b[2]\n" - "sdot z19.s, z8.b, z0.b[2]\n" - "sdot z23.s, z8.b, z1.b[2]\n" - "sdot z27.s, z8.b, z2.b[2]\n" - "sdot z31.s, z8.b, z3.b[2]\n" + "sdot z18.s, z5.b, z0.b[2]\n" + "sdot z22.s, z5.b, z1.b[2]\n" + "sdot z26.s, z5.b, z2.b[2]\n" + "sdot z30.s, z5.b, z3.b[2]\n" + "sdot z19.s, z4.b, z0.b[2]\n" + "sdot z23.s, z4.b, z1.b[2]\n" + "sdot z27.s, z4.b, z2.b[2]\n" + "sdot z31.s, z4.b, z3.b[2]\n" "ble 52f\n" - "ld1b { z9.b }, p2/Z, [x28]\n" - "ld1b { z10.b }, p2/Z, [x28, #1, MUL VL]\n" - "sdot z16.s, z9.b, z0.b[3]\n" - "sdot z20.s, z9.b, z1.b[3]\n" - "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z5.b }, p2/Z, [x28, #3, MUL VL]\n" - "sdot z24.s, z9.b, z2.b[3]\n" - "sdot z28.s, z9.b, z3.b[3]\n" - "sdot z17.s, z10.b, z0.b[3]\n" - "sdot z21.s, z10.b, z1.b[3]\n" + "ld1b { z7.b }, p2/Z, [x28]\n" + "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" + "sdot z16.s, z7.b, z0.b[3]\n" + "sdot z20.s, z7.b, z1.b[3]\n" + "ld1b { z5.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z4.b }, p2/Z, [x28, #3, MUL VL]\n" + "sdot z24.s, z7.b, z2.b[3]\n" + "sdot z28.s, z7.b, z3.b[3]\n" + "sdot z17.s, z6.b, z0.b[3]\n" + "sdot z21.s, z6.b, z1.b[3]\n" "addvl x28, x28, #4\n" - "sdot z25.s, z10.b, z2.b[3]\n" - "sdot z29.s, z10.b, z3.b[3]\n" - "sdot z18.s, z4.b, z0.b[3]\n" - "sdot z22.s, z4.b, z1.b[3]\n" - "sdot z26.s, z4.b, z2.b[3]\n" - "sdot z30.s, z4.b, z3.b[3]\n" - "sdot z19.s, z5.b, z0.b[3]\n" - "sdot z23.s, z5.b, z1.b[3]\n" - "sdot z27.s, z5.b, z2.b[3]\n" - "sdot z31.s, z5.b, z3.b[3]\n" + "sdot z25.s, z6.b, z2.b[3]\n" + "sdot z29.s, z6.b, z3.b[3]\n" + "sdot z18.s, z5.b, z0.b[3]\n" + "sdot z22.s, z5.b, z1.b[3]\n" + "sdot z26.s, z5.b, z2.b[3]\n" + "sdot z30.s, z5.b, z3.b[3]\n" + "sdot z19.s, z4.b, z0.b[3]\n" + "sdot z23.s, z4.b, z1.b[3]\n" + "sdot z27.s, z4.b, z2.b[3]\n" + "sdot z31.s, z4.b, z3.b[3]\n" "52:" // Height 4: Multiply loop: multiply skip "tbnz %x[flags], #31, 53f\n" "sdot z11.s, z0.b, z15.b\n" @@ -1265,7 +1265,7 @@ void sve_hybrid_s8qa_dot_4x4VL ( "mov x20, #0x4\n" "whilelt p0.s, XZR, x20\n" "add x20, %x[qp], %[b_offset]\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + "ld1rw { z0.s }, p2/Z, [x20]\n" "saddv d11, p0, z11.s\n" "mov z11.s, z11.s[0]\n" "saddv d12, p0, z12.s\n" @@ -1273,28 +1273,28 @@ void sve_hybrid_s8qa_dot_4x4VL ( "mov z12.s, z12.s[0]\n" "mov z13.s, z13.s[0]\n" "saddv d14, p0, z14.s\n" - "neg z4.s, p2/M, z4.s\n" + "neg z0.s, p2/M, z0.s\n" "mov z14.s, z14.s[0]\n" - "mul z11.s, p2/M, z11.s, z4.s\n" - "mul z12.s, p2/M, z12.s, z4.s\n" - "mul z13.s, p2/M, z13.s, z4.s\n" - "mul z14.s, p2/M, z14.s, z4.s\n" + "mul z11.s, p2/M, z11.s, z0.s\n" + "mul z12.s, p2/M, z12.s, z0.s\n" + "mul z13.s, p2/M, z13.s, z0.s\n" + "mul z14.s, p2/M, z14.s, z0.s\n" "54:" // Height 4: skip row sum fixup "add z16.s, z16.s, z11.s\n" "add z17.s, z17.s, z11.s\n" - "ld1w { z0.s }, p2/Z, [x10]\n" - "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n" + "ld1w { z4.s }, p2/Z, [x10]\n" + "ld1w { z0.s }, p2/Z, [x10, #1, MUL VL]\n" "add z18.s, z18.s, z11.s\n" "add z19.s, z19.s, z11.s\n" - "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n" + "ld1w { z3.s }, p2/Z, [x10, #2, MUL VL]\n" + "ld1w { z2.s }, p2/Z, [x10, #3, MUL VL]\n" "add z20.s, z20.s, z12.s\n" "add z21.s, z21.s, z12.s\n" "add x20, %x[qp], %[per_layer_mul]\n" "orr %x[flags], %x[flags], #0x80000000\n" "add z22.s, z22.s, z12.s\n" "add z23.s, z23.s, z12.s\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + "ld1rw { z1.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[per_layer_right_shift]\n" "add z24.s, z24.s, z13.s\n" "add z25.s, z25.s, z13.s\n" @@ -1305,174 +1305,174 @@ void sve_hybrid_s8qa_dot_4x4VL ( "add z29.s, z29.s, z14.s\n" "add z30.s, z30.s, z14.s\n" "add z31.s, z31.s, z14.s\n" - "add z16.s, z16.s, z0.s\n" - "add z17.s, z17.s, z1.s\n" - "add z18.s, z18.s, z2.s\n" - "add z19.s, z19.s, z3.s\n" - "add z20.s, z20.s, z0.s\n" - "add z21.s, z21.s, z1.s\n" - "add z22.s, z22.s, z2.s\n" - "add z23.s, z23.s, z3.s\n" - "add z24.s, z24.s, z0.s\n" - "add z25.s, z25.s, z1.s\n" - "add z26.s, z26.s, z2.s\n" - "add z27.s, z27.s, z3.s\n" - "add z28.s, z28.s, z0.s\n" - "add z29.s, z29.s, z1.s\n" + "add z16.s, z16.s, z4.s\n" + "add z17.s, z17.s, z0.s\n" + "add z18.s, z18.s, z3.s\n" + "add z19.s, z19.s, z2.s\n" + "add z20.s, z20.s, z4.s\n" + "add z21.s, z21.s, z0.s\n" + "add z22.s, z22.s, z3.s\n" + "add z23.s, z23.s, z2.s\n" + "add z24.s, z24.s, z4.s\n" + "add z25.s, z25.s, z0.s\n" + "add z26.s, z26.s, z3.s\n" + "add z27.s, z27.s, z2.s\n" + "add z28.s, z28.s, z4.s\n" + "add z29.s, z29.s, z0.s\n" "ld1rw { z0.s }, p2/Z, [x20]\n" - "add z30.s, z30.s, z2.s\n" - "add z31.s, z31.s, z3.s\n" - ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n" - ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n" - ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n" - ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n" - ".inst 0x04a47694 // sqrdmulh z20.s, z20.s, z4.s\n" - ".inst 0x04a476b5 // sqrdmulh z21.s, z21.s, z4.s\n" - ".inst 0x04a476d6 // sqrdmulh z22.s, z22.s, z4.s\n" - ".inst 0x04a476f7 // sqrdmulh z23.s, z23.s, z4.s\n" - ".inst 0x04a47718 // sqrdmulh z24.s, z24.s, z4.s\n" - ".inst 0x04a47739 // sqrdmulh z25.s, z25.s, z4.s\n" - ".inst 0x04a4775a // sqrdmulh z26.s, z26.s, z4.s\n" - ".inst 0x04a4777b // sqrdmulh z27.s, z27.s, z4.s\n" - ".inst 0x04a4779c // sqrdmulh z28.s, z28.s, z4.s\n" - ".inst 0x04a477bd // sqrdmulh z29.s, z29.s, z4.s\n" - ".inst 0x04a477de // sqrdmulh z30.s, z30.s, z4.s\n" - ".inst 0x04a477ff // sqrdmulh z31.s, z31.s, z4.s\n" + "add z30.s, z30.s, z3.s\n" + "add z31.s, z31.s, z2.s\n" + ".inst 0x04a17610 // sqrdmulh z16.s, z16.s, z1.s\n" + ".inst 0x04a17631 // sqrdmulh z17.s, z17.s, z1.s\n" + ".inst 0x04a17652 // sqrdmulh z18.s, z18.s, z1.s\n" + ".inst 0x04a17673 // sqrdmulh z19.s, z19.s, z1.s\n" + ".inst 0x04a17694 // sqrdmulh z20.s, z20.s, z1.s\n" + ".inst 0x04a176b5 // sqrdmulh z21.s, z21.s, z1.s\n" + ".inst 0x04a176d6 // sqrdmulh z22.s, z22.s, z1.s\n" + ".inst 0x04a176f7 // sqrdmulh z23.s, z23.s, z1.s\n" + ".inst 0x04a17718 // sqrdmulh z24.s, z24.s, z1.s\n" + ".inst 0x04a17739 // sqrdmulh z25.s, z25.s, z1.s\n" + ".inst 0x04a1775a // sqrdmulh z26.s, z26.s, z1.s\n" + ".inst 0x04a1777b // sqrdmulh z27.s, z27.s, z1.s\n" + ".inst 0x04a1779c // sqrdmulh z28.s, z28.s, z1.s\n" + ".inst 0x04a177bd // sqrdmulh z29.s, z29.s, z1.s\n" + ".inst 0x04a177de // sqrdmulh z30.s, z30.s, z1.s\n" + ".inst 0x04a177ff // sqrdmulh z31.s, z31.s, z1.s\n" "tbz %x[flags], #5, 55f\n" - "and z4.d, z16.d, z0.d\n" - "and z5.d, z17.d, z0.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "sqadd z16.s, z16.s, z4.s\n" - "sqadd z17.s, z17.s, z5.s\n" - "and z6.d, z18.d, z0.d\n" - "and z7.d, z19.d, z0.d\n" - "and z8.d, z20.d, z0.d\n" - "and z9.d, z21.d, z0.d\n" - "and z10.d, z22.d, z0.d\n" - "and z4.d, z23.d, z0.d\n" - "and z5.d, z24.d, z0.d\n" - "asr z6.s, z6.s, #0x1f\n" + "and z2.d, z16.d, z0.d\n" + "and z1.d, z17.d, z0.d\n" + "asr z2.s, z2.s, #0x1f\n" + "asr z1.s, z1.s, #0x1f\n" + "sqadd z16.s, z16.s, z2.s\n" + "sqadd z17.s, z17.s, z1.s\n" + "and z7.d, z18.d, z0.d\n" + "and z6.d, z19.d, z0.d\n" + "and z5.d, z20.d, z0.d\n" + "and z4.d, z21.d, z0.d\n" + "and z3.d, z22.d, z0.d\n" + "and z2.d, z23.d, z0.d\n" + "and z1.d, z24.d, z0.d\n" "asr z7.s, z7.s, #0x1f\n" - "asr z8.s, z8.s, #0x1f\n" - "asr z9.s, z9.s, #0x1f\n" - "asr z10.s, z10.s, #0x1f\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "sqadd z18.s, z18.s, z6.s\n" - "sqadd z19.s, z19.s, z7.s\n" - "sqadd z20.s, z20.s, z8.s\n" - "sqadd z21.s, z21.s, z9.s\n" - "sqadd z22.s, z22.s, z10.s\n" - "sqadd z23.s, z23.s, z4.s\n" - "sqadd z24.s, z24.s, z5.s\n" - "and z6.d, z25.d, z0.d\n" - "and z7.d, z26.d, z0.d\n" - "and z8.d, z27.d, z0.d\n" - "and z9.d, z28.d, z0.d\n" - "and z10.d, z29.d, z0.d\n" - "and z4.d, z30.d, z0.d\n" - "and z5.d, z31.d, z0.d\n" "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "asr z8.s, z8.s, #0x1f\n" - "asr z9.s, z9.s, #0x1f\n" - "asr z10.s, z10.s, #0x1f\n" + "asr z5.s, z5.s, #0x1f\n" "asr z4.s, z4.s, #0x1f\n" + "asr z3.s, z3.s, #0x1f\n" + "asr z2.s, z2.s, #0x1f\n" + "asr z1.s, z1.s, #0x1f\n" + "sqadd z18.s, z18.s, z7.s\n" + "sqadd z19.s, z19.s, z6.s\n" + "sqadd z20.s, z20.s, z5.s\n" + "sqadd z21.s, z21.s, z4.s\n" + "sqadd z22.s, z22.s, z3.s\n" + "sqadd z23.s, z23.s, z2.s\n" + "sqadd z24.s, z24.s, z1.s\n" + "and z7.d, z25.d, z0.d\n" + "and z6.d, z26.d, z0.d\n" + "and z5.d, z27.d, z0.d\n" + "and z4.d, z28.d, z0.d\n" + "and z3.d, z29.d, z0.d\n" + "and z2.d, z30.d, z0.d\n" + "and z1.d, z31.d, z0.d\n" + "asr z7.s, z7.s, #0x1f\n" + "asr z6.s, z6.s, #0x1f\n" "asr z5.s, z5.s, #0x1f\n" - "sqadd z25.s, z25.s, z6.s\n" - "sqadd z26.s, z26.s, z7.s\n" - "sqadd z27.s, z27.s, z8.s\n" - "sqadd z28.s, z28.s, z9.s\n" - "sqadd z29.s, z29.s, z10.s\n" - "sqadd z30.s, z30.s, z4.s\n" - "sqadd z31.s, z31.s, z5.s\n" + "asr z4.s, z4.s, #0x1f\n" + "asr z3.s, z3.s, #0x1f\n" + "asr z2.s, z2.s, #0x1f\n" + "asr z1.s, z1.s, #0x1f\n" + "sqadd z25.s, z25.s, z7.s\n" + "sqadd z26.s, z26.s, z6.s\n" + "sqadd z27.s, z27.s, z5.s\n" + "sqadd z28.s, z28.s, z4.s\n" + "sqadd z29.s, z29.s, z3.s\n" + "sqadd z30.s, z30.s, z2.s\n" + "sqadd z31.s, z31.s, z1.s\n" "55:" // Height 4: no shift correction "add x20, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + "ld1rw { z2.s }, p2/Z, [x20]\n" ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n" - "add z16.s, z16.s, z4.s\n" + "add z16.s, z16.s, z2.s\n" ".inst 0x44828811 // srshl z17.s, p2/M, z17.s, z0.s\n" ".inst 0x44828812 // srshl z18.s, p2/M, z18.s, z0.s\n" - "add z17.s, z17.s, z4.s\n" - "add z18.s, z18.s, z4.s\n" + "add z17.s, z17.s, z2.s\n" + "add z18.s, z18.s, z2.s\n" ".inst 0x44828813 // srshl z19.s, p2/M, z19.s, z0.s\n" ".inst 0x44828814 // srshl z20.s, p2/M, z20.s, z0.s\n" - "add z19.s, z19.s, z4.s\n" - "add z20.s, z20.s, z4.s\n" + "add z19.s, z19.s, z2.s\n" + "add z20.s, z20.s, z2.s\n" ".inst 0x44828815 // srshl z21.s, p2/M, z21.s, z0.s\n" ".inst 0x44828816 // srshl z22.s, p2/M, z22.s, z0.s\n" - "add z21.s, z21.s, z4.s\n" - "add z22.s, z22.s, z4.s\n" + "add z21.s, z21.s, z2.s\n" + "add z22.s, z22.s, z2.s\n" ".inst 0x44828817 // srshl z23.s, p2/M, z23.s, z0.s\n" ".inst 0x44828818 // srshl z24.s, p2/M, z24.s, z0.s\n" - "add z23.s, z23.s, z4.s\n" - "add z24.s, z24.s, z4.s\n" + "add z23.s, z23.s, z2.s\n" + "add z24.s, z24.s, z2.s\n" ".inst 0x44828819 // srshl z25.s, p2/M, z25.s, z0.s\n" ".inst 0x4482881a // srshl z26.s, p2/M, z26.s, z0.s\n" - "add z25.s, z25.s, z4.s\n" - "add z26.s, z26.s, z4.s\n" + "add z25.s, z25.s, z2.s\n" + "add z26.s, z26.s, z2.s\n" ".inst 0x4482881b // srshl z27.s, p2/M, z27.s, z0.s\n" ".inst 0x4482881c // srshl z28.s, p2/M, z28.s, z0.s\n" - "add z27.s, z27.s, z4.s\n" - "add z28.s, z28.s, z4.s\n" + "add z27.s, z27.s, z2.s\n" + "add z28.s, z28.s, z2.s\n" ".inst 0x4482881d // srshl z29.s, p2/M, z29.s, z0.s\n" ".inst 0x4482881e // srshl z30.s, p2/M, z30.s, z0.s\n" - "add z29.s, z29.s, z4.s\n" - "add z30.s, z30.s, z4.s\n" + "add z29.s, z29.s, z2.s\n" + "add z30.s, z30.s, z2.s\n" ".inst 0x4482881f // srshl z31.s, p2/M, z31.s, z0.s\n" "add x20, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x20]\n" - "add z31.s, z31.s, z4.s\n" + "ld1rw { z1.s }, p2/Z, [x20]\n" + "add z31.s, z31.s, z2.s\n" "add x20, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x20]\n" - "smin z16.s, p2/M, z16.s, z6.s\n" - "smin z17.s, p2/M, z17.s, z6.s\n" - "smin z18.s, p2/M, z18.s, z6.s\n" - "smin z19.s, p2/M, z19.s, z6.s\n" - "smin z20.s, p2/M, z20.s, z6.s\n" - "smin z21.s, p2/M, z21.s, z6.s\n" - "smin z22.s, p2/M, z22.s, z6.s\n" - "smin z23.s, p2/M, z23.s, z6.s\n" - "smin z24.s, p2/M, z24.s, z6.s\n" - "smin z25.s, p2/M, z25.s, z6.s\n" - "smin z26.s, p2/M, z26.s, z6.s\n" - "smin z27.s, p2/M, z27.s, z6.s\n" - "smin z28.s, p2/M, z28.s, z6.s\n" - "smin z29.s, p2/M, z29.s, z6.s\n" - "smin z30.s, p2/M, z30.s, z6.s\n" - "smin z31.s, p2/M, z31.s, z6.s\n" - "smax z16.s, p2/M, z16.s, z5.s\n" - "smax z17.s, p2/M, z17.s, z5.s\n" - "smax z18.s, p2/M, z18.s, z5.s\n" + "ld1rw { z0.s }, p2/Z, [x20]\n" + "smin z16.s, p2/M, z16.s, z1.s\n" + "smin z17.s, p2/M, z17.s, z1.s\n" + "smin z18.s, p2/M, z18.s, z1.s\n" + "smin z19.s, p2/M, z19.s, z1.s\n" + "smin z20.s, p2/M, z20.s, z1.s\n" + "smin z21.s, p2/M, z21.s, z1.s\n" + "smin z22.s, p2/M, z22.s, z1.s\n" + "smin z23.s, p2/M, z23.s, z1.s\n" + "smin z24.s, p2/M, z24.s, z1.s\n" + "smin z25.s, p2/M, z25.s, z1.s\n" + "smin z26.s, p2/M, z26.s, z1.s\n" + "smin z27.s, p2/M, z27.s, z1.s\n" + "smin z28.s, p2/M, z28.s, z1.s\n" + "smin z29.s, p2/M, z29.s, z1.s\n" + "smin z30.s, p2/M, z30.s, z1.s\n" + "smin z31.s, p2/M, z31.s, z1.s\n" + "smax z16.s, p2/M, z16.s, z0.s\n" + "smax z17.s, p2/M, z17.s, z0.s\n" + "smax z18.s, p2/M, z18.s, z0.s\n" "uzp1 z16.h, z16.h, z17.h\n" - "smax z19.s, p2/M, z19.s, z5.s\n" - "smax z20.s, p2/M, z20.s, z5.s\n" + "smax z19.s, p2/M, z19.s, z0.s\n" + "smax z20.s, p2/M, z20.s, z0.s\n" "uzp1 z17.h, z18.h, z19.h\n" "uzp1 z16.b, z16.b, z17.b\n" - "smax z21.s, p2/M, z21.s, z5.s\n" - "smax z22.s, p2/M, z22.s, z5.s\n" + "smax z21.s, p2/M, z21.s, z0.s\n" + "smax z22.s, p2/M, z22.s, z0.s\n" "uzp1 z20.h, z20.h, z21.h\n" "st1b { z16.b }, p1, [x27]\n" - "smax z23.s, p2/M, z23.s, z5.s\n" - "smax z24.s, p2/M, z24.s, z5.s\n" - "uzp1 z21.h, z22.h, z23.h\n" - "uzp1 z20.b, z20.b, z21.b\n" - "smax z25.s, p2/M, z25.s, z5.s\n" - "smax z26.s, p2/M, z26.s, z5.s\n" + "smax z23.s, p2/M, z23.s, z0.s\n" + "smax z24.s, p2/M, z24.s, z0.s\n" + "uzp1 z16.h, z22.h, z23.h\n" + "uzp1 z20.b, z20.b, z16.b\n" + "smax z25.s, p2/M, z25.s, z0.s\n" + "smax z26.s, p2/M, z26.s, z0.s\n" "uzp1 z24.h, z24.h, z25.h\n" "st1b { z20.b }, p1, [x23]\n" - "smax z27.s, p2/M, z27.s, z5.s\n" - "smax z28.s, p2/M, z28.s, z5.s\n" - "uzp1 z25.h, z26.h, z27.h\n" - "uzp1 z24.b, z24.b, z25.b\n" - "smax z29.s, p2/M, z29.s, z5.s\n" - "smax z30.s, p2/M, z30.s, z5.s\n" + "smax z27.s, p2/M, z27.s, z0.s\n" + "smax z28.s, p2/M, z28.s, z0.s\n" + "uzp1 z16.h, z26.h, z27.h\n" + "uzp1 z24.b, z24.b, z16.b\n" + "smax z29.s, p2/M, z29.s, z0.s\n" + "smax z30.s, p2/M, z30.s, z0.s\n" "uzp1 z28.h, z28.h, z29.h\n" "st1b { z24.b }, p1, [x22]\n" - "smax z31.s, p2/M, z31.s, z5.s\n" - "uzp1 z29.h, z30.h, z31.h\n" - "uzp1 z28.b, z28.b, z29.b\n" + "smax z31.s, p2/M, z31.s, z0.s\n" + "uzp1 z16.h, z30.h, z31.h\n" + "uzp1 z28.b, z28.b, z16.b\n" "st1b { z28.b }, p1, [x21]\n" "addvl x27, x27, #1\n" "56:" // Height 4: Writeback done @@ -1491,7 +1491,6 @@ void sve_hybrid_s8qa_dot_4x4VL ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "58:" // Exit - : [M] "+&r" (M), [flags] "+&r" (flags), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp) : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -1499,4 +1498,4 @@ void sve_hybrid_s8qa_dot_4x4VL ( } } // namespace arm_gemm -#endif // __ARM_FEATURE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_mmla_4x4VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_mmla_4x4VL.hpp index 9681505e8c24d024a044bb3531476bcf42b5932e..ae922e9743bdc7c8bf446ef257ca5a59923a80fe 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_mmla_4x4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_mmla_4x4VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef ARM_COMPUTE_ENABLE_SVE + #include "../std_transforms_sve.hpp" #include "../performance_parameters.hpp" @@ -74,7 +74,6 @@ public: template static inline PerformanceParameters get_performance_parameters(const CPUInfo *ci) { - if (std::is_same::value) { switch (ci->get_cpu_model()) { default: @@ -97,5 +96,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_mmla_4x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_mmla_4x4VL/generic.cpp index 626a06b26b361cfd068946a8b9f44e4391d67bc6..e0628364f483f0823f5326bf97a11662cca693c0 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_mmla_4x4VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qa_mmla_4x4VL/generic.cpp @@ -108,11 +108,11 @@ void sve_hybrid_s8qa_mmla_4x4VL ( "4:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 5f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" "cbnz x26, 6f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -125,41 +125,41 @@ void sve_hybrid_s8qa_mmla_4x4VL ( "7:" // Height 1: Multiply loop: Main loop head "whilelt p0.b, XZR, x25\n" "ld1rqb { z1.b }, p0/Z, [x24]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1b { z5.b }, p2/Z, [x28]\n" - ".inst 0x45059810 // smmla z16.s, z0.b, z5.b\n" - "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" - "trn2 z1.d, z1.d, z2.d\n" - "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" - ".inst 0x45069814 // smmla z20.s, z0.b, z6.b\n" - ".inst 0x45079811 // smmla z17.s, z0.b, z7.b\n" - "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n" - "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n" - ".inst 0x45089815 // smmla z21.s, z0.b, z8.b\n" - ".inst 0x45099812 // smmla z18.s, z0.b, z9.b\n" - "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n" - "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n" + "trn1 z0.d, z1.d, z26.d\n" + "ld1b { z24.b }, p2/Z, [x28]\n" + ".inst 0x45189810 // smmla z16.s, z0.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #1, MUL VL]\n" + "trn2 z1.d, z1.d, z26.d\n" + "ld1b { z24.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z26.b }, p2/Z, [x28, #3, MUL VL]\n" + ".inst 0x45199814 // smmla z20.s, z0.b, z25.b\n" + ".inst 0x45189811 // smmla z17.s, z0.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #4, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #5, MUL VL]\n" + ".inst 0x451a9815 // smmla z21.s, z0.b, z26.b\n" + ".inst 0x45199812 // smmla z18.s, z0.b, z25.b\n" + "ld1b { z26.b }, p2/Z, [x28, #6, MUL VL]\n" + "ld1b { z25.b }, p2/Z, [x28, #7, MUL VL]\n" "addvl x28, x28, #16\n" - ".inst 0x450a9816 // smmla z22.s, z0.b, z10.b\n" - "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n" - ".inst 0x45049813 // smmla z19.s, z0.b, z4.b\n" - ".inst 0x45059817 // smmla z23.s, z0.b, z5.b\n" - "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n" - ".inst 0x45069830 // smmla z16.s, z1.b, z6.b\n" - "ld1b { z8.b }, p2/Z, [x28, #-6, MUL VL]\n" - "ld1b { z9.b }, p2/Z, [x28, #-5, MUL VL]\n" - ".inst 0x45079834 // smmla z20.s, z1.b, z7.b\n" - "ld1b { z10.b }, p2/Z, [x28, #-4, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #-3, MUL VL]\n" - ".inst 0x45089831 // smmla z17.s, z1.b, z8.b\n" - ".inst 0x45099835 // smmla z21.s, z1.b, z9.b\n" - "ld1b { z5.b }, p2/Z, [x28, #-2, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x28, #-1, MUL VL]\n" - ".inst 0x450a9832 // smmla z18.s, z1.b, z10.b\n" - ".inst 0x45049836 // smmla z22.s, z1.b, z4.b\n" - ".inst 0x45059833 // smmla z19.s, z1.b, z5.b\n" - ".inst 0x45069837 // smmla z23.s, z1.b, z6.b\n" + ".inst 0x45189816 // smmla z22.s, z0.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x28, #-8, MUL VL]\n" + ".inst 0x451a9813 // smmla z19.s, z0.b, z26.b\n" + ".inst 0x45199817 // smmla z23.s, z0.b, z25.b\n" + "ld1b { z26.b }, p2/Z, [x28, #-7, MUL VL]\n" + ".inst 0x45189830 // smmla z16.s, z1.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #-6, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #-5, MUL VL]\n" + ".inst 0x451a9834 // smmla z20.s, z1.b, z26.b\n" + "ld1b { z27.b }, p2/Z, [x28, #-4, MUL VL]\n" + "ld1b { z26.b }, p2/Z, [x28, #-3, MUL VL]\n" + ".inst 0x45199831 // smmla z17.s, z1.b, z25.b\n" + ".inst 0x45189835 // smmla z21.s, z1.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #-2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #-1, MUL VL]\n" + ".inst 0x451b9832 // smmla z18.s, z1.b, z27.b\n" + ".inst 0x451a9836 // smmla z22.s, z1.b, z26.b\n" + ".inst 0x45199833 // smmla z19.s, z1.b, z25.b\n" + ".inst 0x45189837 // smmla z23.s, z1.b, z24.b\n" "add x24, x24, #0x10\n" "tbnz %x[flags], #31, 8f\n" "sdot z11.s, z0.b, z15.b\n" @@ -171,43 +171,43 @@ void sve_hybrid_s8qa_mmla_4x4VL ( "9:" // Height 1: Multiply loop: Single iteration only "whilelt p0.b, XZR, x25\n" "ld1rqb { z1.b }, p0/Z, [x24]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1b { z5.b }, p2/Z, [x28]\n" - ".inst 0x45059810 // smmla z16.s, z0.b, z5.b\n" - "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" + "trn1 z0.d, z1.d, z27.d\n" + "ld1b { z24.b }, p2/Z, [x28]\n" + ".inst 0x45189810 // smmla z16.s, z0.b, z24.b\n" + "ld1b { z26.b }, p2/Z, [x28, #1, MUL VL]\n" "subs x25, x25, #0x8\n" - "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" - "trn2 z1.d, z1.d, z2.d\n" - ".inst 0x45069814 // smmla z20.s, z0.b, z6.b\n" - "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n" - "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n" - ".inst 0x45079811 // smmla z17.s, z0.b, z7.b\n" - ".inst 0x45089815 // smmla z21.s, z0.b, z8.b\n" - "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n" - "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n" - ".inst 0x45099812 // smmla z18.s, z0.b, z9.b\n" - ".inst 0x450a9816 // smmla z22.s, z0.b, z10.b\n" - ".inst 0x45049813 // smmla z19.s, z0.b, z4.b\n" - ".inst 0x45059817 // smmla z23.s, z0.b, z5.b\n" + "ld1b { z25.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #3, MUL VL]\n" + "trn2 z1.d, z1.d, z27.d\n" + ".inst 0x451a9814 // smmla z20.s, z0.b, z26.b\n" + "ld1b { z27.b }, p2/Z, [x28, #4, MUL VL]\n" + "ld1b { z26.b }, p2/Z, [x28, #5, MUL VL]\n" + ".inst 0x45199811 // smmla z17.s, z0.b, z25.b\n" + ".inst 0x45189815 // smmla z21.s, z0.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #6, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #7, MUL VL]\n" + ".inst 0x451b9812 // smmla z18.s, z0.b, z27.b\n" + ".inst 0x451a9816 // smmla z22.s, z0.b, z26.b\n" + ".inst 0x45199813 // smmla z19.s, z0.b, z25.b\n" + ".inst 0x45189817 // smmla z23.s, z0.b, z24.b\n" "addvl x28, x28, #8\n" "ble 10f\n" - "ld1b { z6.b }, p2/Z, [x28]\n" - ".inst 0x45069830 // smmla z16.s, z1.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n" - ".inst 0x45079834 // smmla z20.s, z1.b, z7.b\n" - "ld1b { z8.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z9.b }, p2/Z, [x28, #3, MUL VL]\n" - ".inst 0x45089831 // smmla z17.s, z1.b, z8.b\n" - ".inst 0x45099835 // smmla z21.s, z1.b, z9.b\n" - "ld1b { z10.b }, p2/Z, [x28, #4, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #5, MUL VL]\n" - ".inst 0x450a9832 // smmla z18.s, z1.b, z10.b\n" - ".inst 0x45049836 // smmla z22.s, z1.b, z4.b\n" - "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n" - ".inst 0x45059833 // smmla z19.s, z1.b, z5.b\n" - ".inst 0x45069837 // smmla z23.s, z1.b, z6.b\n" + "ld1b { z24.b }, p2/Z, [x28]\n" + ".inst 0x45189830 // smmla z16.s, z1.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x28, #1, MUL VL]\n" + ".inst 0x45189834 // smmla z20.s, z1.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #3, MUL VL]\n" + ".inst 0x45199831 // smmla z17.s, z1.b, z25.b\n" + ".inst 0x45189835 // smmla z21.s, z1.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #4, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #5, MUL VL]\n" + ".inst 0x45199832 // smmla z18.s, z1.b, z25.b\n" + ".inst 0x45189836 // smmla z22.s, z1.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #6, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #7, MUL VL]\n" + ".inst 0x45199833 // smmla z19.s, z1.b, z25.b\n" + ".inst 0x45189837 // smmla z23.s, z1.b, z24.b\n" "addvl x28, x28, #8\n" "10:" // Height 1: Multiply loop: multiply skip "tbnz %x[flags], #31, 11f\n" @@ -224,74 +224,74 @@ void sve_hybrid_s8qa_mmla_4x4VL ( "uzp1 z19.d, z19.d, z23.d\n" "mov z23.d, z16.d\n" "tbnz %x[flags], #31, 12f\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1rw { z1.s }, p2/Z, [x23]\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1rw { z16.s }, p2/Z, [x20]\n" ".inst 0x4491a96b // addp z11.s, p2/m, z11.s, z11.s\n" - "neg z1.s, p2/M, z1.s\n" + "neg z16.s, p2/M, z16.s\n" "mov z11.s, z11.s[0]\n" - "mul z11.s, p2/M, z11.s, z1.s\n" + "mul z11.s, p2/M, z11.s, z16.s\n" "12:" // Height 1: skip row sum fixup "add z23.s, z23.s, z11.s\n" "add z17.s, z17.s, z11.s\n" - "ld1w { z0.s }, p2/Z, [x10]\n" - "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x10]\n" + "ld1w { z21.s }, p2/Z, [x10, #1, MUL VL]\n" "add z18.s, z18.s, z11.s\n" "add z19.s, z19.s, z11.s\n" - "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n" - "add x23, %x[qp], %[per_layer_mul]\n" + "ld1w { z20.s }, p2/Z, [x10, #2, MUL VL]\n" + "ld1w { z16.s }, p2/Z, [x10, #3, MUL VL]\n" + "add x20, %x[qp], %[per_layer_mul]\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add z23.s, z23.s, z0.s\n" - "add z17.s, z17.s, z1.s\n" - "add z18.s, z18.s, z2.s\n" - "add z19.s, z19.s, z3.s\n" - "ld1rw { z4.s }, p2/Z, [x23]\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" - "ld1rw { z0.s }, p2/Z, [x23]\n" - ".inst 0x04a476f7 // sqrdmulh z23.s, z23.s, z4.s\n" - ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n" + "add z23.s, z23.s, z22.s\n" + "add z17.s, z17.s, z21.s\n" + "add z18.s, z18.s, z20.s\n" + "add z19.s, z19.s, z16.s\n" + "ld1rw { z16.s }, p2/Z, [x20]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1rw { z0.s }, p2/Z, [x20]\n" + ".inst 0x04b076f7 // sqrdmulh z23.s, z23.s, z16.s\n" + ".inst 0x04b07631 // sqrdmulh z17.s, z17.s, z16.s\n" "addvl x10, x10, #4\n" - ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n" - ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n" + ".inst 0x04b07652 // sqrdmulh z18.s, z18.s, z16.s\n" + ".inst 0x04b07673 // sqrdmulh z19.s, z19.s, z16.s\n" "tbz %x[flags], #5, 13f\n" - "and z4.d, z23.d, z0.d\n" - "and z5.d, z17.d, z0.d\n" - "and z6.d, z18.d, z0.d\n" - "and z7.d, z19.d, z0.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z23.s, z23.s, z4.s\n" - "sqadd z17.s, z17.s, z5.s\n" - "sqadd z18.s, z18.s, z6.s\n" - "sqadd z19.s, z19.s, z7.s\n" + "and z22.d, z23.d, z0.d\n" + "and z21.d, z17.d, z0.d\n" + "and z20.d, z18.d, z0.d\n" + "and z16.d, z19.d, z0.d\n" + "asr z22.s, z22.s, #0x1f\n" + "asr z21.s, z21.s, #0x1f\n" + "asr z20.s, z20.s, #0x1f\n" + "asr z16.s, z16.s, #0x1f\n" + "sqadd z23.s, z23.s, z22.s\n" + "sqadd z17.s, z17.s, z21.s\n" + "sqadd z18.s, z18.s, z20.s\n" + "sqadd z19.s, z19.s, z16.s\n" "13:" // Height 1: no shift correction - "add x23, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x23]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1rw { z16.s }, p2/Z, [x20]\n" ".inst 0x44828817 // srshl z23.s, p2/M, z23.s, z0.s\n" - "add z23.s, z23.s, z4.s\n" + "add z23.s, z23.s, z16.s\n" ".inst 0x44828811 // srshl z17.s, p2/M, z17.s, z0.s\n" ".inst 0x44828812 // srshl z18.s, p2/M, z18.s, z0.s\n" - "add z17.s, z17.s, z4.s\n" - "add z18.s, z18.s, z4.s\n" + "add z17.s, z17.s, z16.s\n" + "add z18.s, z18.s, z16.s\n" ".inst 0x44828813 // srshl z19.s, p2/M, z19.s, z0.s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x23]\n" - "add z19.s, z19.s, z4.s\n" - "add x23, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x23]\n" - "smin z23.s, p2/M, z23.s, z6.s\n" - "smin z17.s, p2/M, z17.s, z6.s\n" - "smin z18.s, p2/M, z18.s, z6.s\n" - "smin z19.s, p2/M, z19.s, z6.s\n" - "smax z23.s, p2/M, z23.s, z5.s\n" - "smax z17.s, p2/M, z17.s, z5.s\n" - "smax z18.s, p2/M, z18.s, z5.s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1rw { z20.s }, p2/Z, [x20]\n" + "add z19.s, z19.s, z16.s\n" + "add x20, %x[qp], %[minval]\n" + "ld1rw { z16.s }, p2/Z, [x20]\n" + "smin z23.s, p2/M, z23.s, z20.s\n" + "smin z17.s, p2/M, z17.s, z20.s\n" + "smin z18.s, p2/M, z18.s, z20.s\n" + "smin z19.s, p2/M, z19.s, z20.s\n" + "smax z23.s, p2/M, z23.s, z16.s\n" + "smax z17.s, p2/M, z17.s, z16.s\n" + "smax z18.s, p2/M, z18.s, z16.s\n" "uzp1 z23.h, z23.h, z17.h\n" - "smax z19.s, p2/M, z19.s, z5.s\n" - "uzp1 z17.h, z18.h, z19.h\n" - "uzp1 z23.b, z23.b, z17.b\n" + "smax z19.s, p2/M, z19.s, z16.s\n" + "uzp1 z16.h, z18.h, z19.h\n" + "uzp1 z23.b, z23.b, z16.b\n" "st1b { z23.b }, p1, [x27]\n" "addvl x27, x27, #1\n" "14:" // Height 1: Writeback done @@ -324,12 +324,12 @@ void sve_hybrid_s8qa_mmla_4x4VL ( "18:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 19f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" "cbnz x26, 20f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -337,49 +337,49 @@ void sve_hybrid_s8qa_mmla_4x4VL ( "b 20f\n" "19:" // Height 2: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20\n" + "add x23, x24, x21\n" "20:" // Height 2: input setup done "cmp x25, #0x10\n" "ble 23f\n" "21:" // Height 2: Multiply loop: Main loop head "whilelt p0.b, XZR, x25\n" "ld1rqb { z1.b }, p0/Z, [x24]\n" - "ld1rqb { z2.b }, p0/Z, [x23]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1b { z5.b }, p2/Z, [x28]\n" - ".inst 0x45059810 // smmla z16.s, z0.b, z5.b\n" - "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" - "trn2 z1.d, z1.d, z2.d\n" - "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" - ".inst 0x45069814 // smmla z20.s, z0.b, z6.b\n" - ".inst 0x45079811 // smmla z17.s, z0.b, z7.b\n" - "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n" - "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n" - ".inst 0x45089815 // smmla z21.s, z0.b, z8.b\n" - ".inst 0x45099812 // smmla z18.s, z0.b, z9.b\n" - "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n" - "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n" + "ld1rqb { z26.b }, p0/Z, [x23]\n" + "trn1 z0.d, z1.d, z26.d\n" + "ld1b { z24.b }, p2/Z, [x28]\n" + ".inst 0x45189810 // smmla z16.s, z0.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #1, MUL VL]\n" + "trn2 z1.d, z1.d, z26.d\n" + "ld1b { z24.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z26.b }, p2/Z, [x28, #3, MUL VL]\n" + ".inst 0x45199814 // smmla z20.s, z0.b, z25.b\n" + ".inst 0x45189811 // smmla z17.s, z0.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #4, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #5, MUL VL]\n" + ".inst 0x451a9815 // smmla z21.s, z0.b, z26.b\n" + ".inst 0x45199812 // smmla z18.s, z0.b, z25.b\n" + "ld1b { z26.b }, p2/Z, [x28, #6, MUL VL]\n" + "ld1b { z25.b }, p2/Z, [x28, #7, MUL VL]\n" "addvl x28, x28, #16\n" - ".inst 0x450a9816 // smmla z22.s, z0.b, z10.b\n" - "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n" - ".inst 0x45049813 // smmla z19.s, z0.b, z4.b\n" - ".inst 0x45059817 // smmla z23.s, z0.b, z5.b\n" - "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n" - ".inst 0x45069830 // smmla z16.s, z1.b, z6.b\n" - "ld1b { z8.b }, p2/Z, [x28, #-6, MUL VL]\n" - "ld1b { z9.b }, p2/Z, [x28, #-5, MUL VL]\n" - ".inst 0x45079834 // smmla z20.s, z1.b, z7.b\n" - "ld1b { z10.b }, p2/Z, [x28, #-4, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #-3, MUL VL]\n" - ".inst 0x45089831 // smmla z17.s, z1.b, z8.b\n" - ".inst 0x45099835 // smmla z21.s, z1.b, z9.b\n" - "ld1b { z5.b }, p2/Z, [x28, #-2, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x28, #-1, MUL VL]\n" - ".inst 0x450a9832 // smmla z18.s, z1.b, z10.b\n" - ".inst 0x45049836 // smmla z22.s, z1.b, z4.b\n" - ".inst 0x45059833 // smmla z19.s, z1.b, z5.b\n" - ".inst 0x45069837 // smmla z23.s, z1.b, z6.b\n" + ".inst 0x45189816 // smmla z22.s, z0.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x28, #-8, MUL VL]\n" + ".inst 0x451a9813 // smmla z19.s, z0.b, z26.b\n" + ".inst 0x45199817 // smmla z23.s, z0.b, z25.b\n" + "ld1b { z26.b }, p2/Z, [x28, #-7, MUL VL]\n" + ".inst 0x45189830 // smmla z16.s, z1.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #-6, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #-5, MUL VL]\n" + ".inst 0x451a9834 // smmla z20.s, z1.b, z26.b\n" + "ld1b { z27.b }, p2/Z, [x28, #-4, MUL VL]\n" + "ld1b { z26.b }, p2/Z, [x28, #-3, MUL VL]\n" + ".inst 0x45199831 // smmla z17.s, z1.b, z25.b\n" + ".inst 0x45189835 // smmla z21.s, z1.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #-2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #-1, MUL VL]\n" + ".inst 0x451b9832 // smmla z18.s, z1.b, z27.b\n" + ".inst 0x451a9836 // smmla z22.s, z1.b, z26.b\n" + ".inst 0x45199833 // smmla z19.s, z1.b, z25.b\n" + ".inst 0x45189837 // smmla z23.s, z1.b, z24.b\n" "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" "tbnz %x[flags], #31, 22f\n" @@ -392,44 +392,44 @@ void sve_hybrid_s8qa_mmla_4x4VL ( "23:" // Height 2: Multiply loop: Single iteration only "whilelt p0.b, XZR, x25\n" "ld1rqb { z1.b }, p0/Z, [x24]\n" - "ld1rqb { z2.b }, p0/Z, [x23]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1b { z5.b }, p2/Z, [x28]\n" - ".inst 0x45059810 // smmla z16.s, z0.b, z5.b\n" - "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1rqb { z27.b }, p0/Z, [x23]\n" + "trn1 z0.d, z1.d, z27.d\n" + "ld1b { z24.b }, p2/Z, [x28]\n" + ".inst 0x45189810 // smmla z16.s, z0.b, z24.b\n" + "ld1b { z26.b }, p2/Z, [x28, #1, MUL VL]\n" "subs x25, x25, #0x8\n" - "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" - "trn2 z1.d, z1.d, z2.d\n" - ".inst 0x45069814 // smmla z20.s, z0.b, z6.b\n" - "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n" - "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n" - ".inst 0x45079811 // smmla z17.s, z0.b, z7.b\n" - ".inst 0x45089815 // smmla z21.s, z0.b, z8.b\n" - "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n" - "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n" - ".inst 0x45099812 // smmla z18.s, z0.b, z9.b\n" - ".inst 0x450a9816 // smmla z22.s, z0.b, z10.b\n" - ".inst 0x45049813 // smmla z19.s, z0.b, z4.b\n" - ".inst 0x45059817 // smmla z23.s, z0.b, z5.b\n" + "ld1b { z25.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #3, MUL VL]\n" + "trn2 z1.d, z1.d, z27.d\n" + ".inst 0x451a9814 // smmla z20.s, z0.b, z26.b\n" + "ld1b { z27.b }, p2/Z, [x28, #4, MUL VL]\n" + "ld1b { z26.b }, p2/Z, [x28, #5, MUL VL]\n" + ".inst 0x45199811 // smmla z17.s, z0.b, z25.b\n" + ".inst 0x45189815 // smmla z21.s, z0.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #6, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #7, MUL VL]\n" + ".inst 0x451b9812 // smmla z18.s, z0.b, z27.b\n" + ".inst 0x451a9816 // smmla z22.s, z0.b, z26.b\n" + ".inst 0x45199813 // smmla z19.s, z0.b, z25.b\n" + ".inst 0x45189817 // smmla z23.s, z0.b, z24.b\n" "addvl x28, x28, #8\n" "ble 24f\n" - "ld1b { z6.b }, p2/Z, [x28]\n" - ".inst 0x45069830 // smmla z16.s, z1.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n" - ".inst 0x45079834 // smmla z20.s, z1.b, z7.b\n" - "ld1b { z8.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z9.b }, p2/Z, [x28, #3, MUL VL]\n" - ".inst 0x45089831 // smmla z17.s, z1.b, z8.b\n" - ".inst 0x45099835 // smmla z21.s, z1.b, z9.b\n" - "ld1b { z10.b }, p2/Z, [x28, #4, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #5, MUL VL]\n" - ".inst 0x450a9832 // smmla z18.s, z1.b, z10.b\n" - ".inst 0x45049836 // smmla z22.s, z1.b, z4.b\n" - "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n" - ".inst 0x45059833 // smmla z19.s, z1.b, z5.b\n" - ".inst 0x45069837 // smmla z23.s, z1.b, z6.b\n" + "ld1b { z24.b }, p2/Z, [x28]\n" + ".inst 0x45189830 // smmla z16.s, z1.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x28, #1, MUL VL]\n" + ".inst 0x45189834 // smmla z20.s, z1.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #3, MUL VL]\n" + ".inst 0x45199831 // smmla z17.s, z1.b, z25.b\n" + ".inst 0x45189835 // smmla z21.s, z1.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #4, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #5, MUL VL]\n" + ".inst 0x45199832 // smmla z18.s, z1.b, z25.b\n" + ".inst 0x45189836 // smmla z22.s, z1.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #6, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #7, MUL VL]\n" + ".inst 0x45199833 // smmla z19.s, z1.b, z25.b\n" + ".inst 0x45189837 // smmla z23.s, z1.b, z24.b\n" "addvl x28, x28, #8\n" "24:" // Height 2: Multiply loop: multiply skip "tbnz %x[flags], #31, 25f\n" @@ -440,133 +440,133 @@ void sve_hybrid_s8qa_mmla_4x4VL ( "add x26, x26, #0x1\n" "cmp x26, x20\n" "bne 18b\n" - "uzp1 z7.d, z16.d, z20.d\n" + "uzp1 z24.d, z16.d, z20.d\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "uzp2 z16.d, z16.d, z20.d\n" - "add x22, x27, x20\n" + "add x23, x27, x20\n" "uzp1 z20.d, z17.d, z21.d\n" "uzp2 z17.d, z17.d, z21.d\n" "uzp1 z21.d, z18.d, z22.d\n" "uzp2 z18.d, z18.d, z22.d\n" "uzp1 z22.d, z19.d, z23.d\n" "uzp2 z19.d, z19.d, z23.d\n" - "mov z23.d, z7.d\n" + "mov z23.d, z24.d\n" "tbnz %x[flags], #31, 26f\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1rw { z2.s }, p2/Z, [x23]\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1rw { z24.s }, p2/Z, [x20]\n" ".inst 0x4491a96b // addp z11.s, p2/m, z11.s, z11.s\n" - "neg z2.s, p2/M, z2.s\n" + "neg z24.s, p2/M, z24.s\n" "mov z12.s, z11.s[3]\n" "mov z11.s, z11.s[0]\n" - "mul z11.s, p2/M, z11.s, z2.s\n" - "mul z12.s, p2/M, z12.s, z2.s\n" + "mul z11.s, p2/M, z11.s, z24.s\n" + "mul z12.s, p2/M, z12.s, z24.s\n" "26:" // Height 2: skip row sum fixup "add z23.s, z23.s, z11.s\n" "add z20.s, z20.s, z11.s\n" - "ld1w { z0.s }, p2/Z, [x10]\n" - "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n" + "ld1w { z28.s }, p2/Z, [x10]\n" + "ld1w { z27.s }, p2/Z, [x10, #1, MUL VL]\n" "add z21.s, z21.s, z11.s\n" "add z22.s, z22.s, z11.s\n" - "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n" + "ld1w { z26.s }, p2/Z, [x10, #2, MUL VL]\n" + "ld1w { z25.s }, p2/Z, [x10, #3, MUL VL]\n" "add z16.s, z16.s, z12.s\n" "add z17.s, z17.s, z12.s\n" - "add x23, %x[qp], %[per_layer_mul]\n" + "add x20, %x[qp], %[per_layer_mul]\n" "orr %x[flags], %x[flags], #0x80000000\n" "add z18.s, z18.s, z12.s\n" "add z19.s, z19.s, z12.s\n" - "ld1rw { z4.s }, p2/Z, [x23]\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" - "add z23.s, z23.s, z0.s\n" - "add z20.s, z20.s, z1.s\n" + "ld1rw { z24.s }, p2/Z, [x20]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "add z23.s, z23.s, z28.s\n" + "add z20.s, z20.s, z27.s\n" "addvl x10, x10, #4\n" - "add z21.s, z21.s, z2.s\n" - "add z22.s, z22.s, z3.s\n" - "add z16.s, z16.s, z0.s\n" - "add z17.s, z17.s, z1.s\n" - "ld1rw { z0.s }, p2/Z, [x23]\n" - "add z18.s, z18.s, z2.s\n" - "add z19.s, z19.s, z3.s\n" - ".inst 0x04a476f7 // sqrdmulh z23.s, z23.s, z4.s\n" - ".inst 0x04a47694 // sqrdmulh z20.s, z20.s, z4.s\n" - ".inst 0x04a476b5 // sqrdmulh z21.s, z21.s, z4.s\n" - ".inst 0x04a476d6 // sqrdmulh z22.s, z22.s, z4.s\n" - ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n" - ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n" - ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n" - ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n" + "add z21.s, z21.s, z26.s\n" + "add z22.s, z22.s, z25.s\n" + "add z16.s, z16.s, z28.s\n" + "add z17.s, z17.s, z27.s\n" + "ld1rw { z0.s }, p2/Z, [x20]\n" + "add z18.s, z18.s, z26.s\n" + "add z19.s, z19.s, z25.s\n" + ".inst 0x04b876f7 // sqrdmulh z23.s, z23.s, z24.s\n" + ".inst 0x04b87694 // sqrdmulh z20.s, z20.s, z24.s\n" + ".inst 0x04b876b5 // sqrdmulh z21.s, z21.s, z24.s\n" + ".inst 0x04b876d6 // sqrdmulh z22.s, z22.s, z24.s\n" + ".inst 0x04b87610 // sqrdmulh z16.s, z16.s, z24.s\n" + ".inst 0x04b87631 // sqrdmulh z17.s, z17.s, z24.s\n" + ".inst 0x04b87652 // sqrdmulh z18.s, z18.s, z24.s\n" + ".inst 0x04b87673 // sqrdmulh z19.s, z19.s, z24.s\n" "tbz %x[flags], #5, 27f\n" - "and z4.d, z23.d, z0.d\n" - "asr z4.s, z4.s, #0x1f\n" - "sqadd z23.s, z23.s, z4.s\n" - "and z5.d, z20.d, z0.d\n" - "and z6.d, z21.d, z0.d\n" - "and z7.d, z22.d, z0.d\n" - "and z8.d, z16.d, z0.d\n" - "and z9.d, z17.d, z0.d\n" - "and z10.d, z18.d, z0.d\n" - "and z4.d, z19.d, z0.d\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "asr z8.s, z8.s, #0x1f\n" - "asr z9.s, z9.s, #0x1f\n" - "asr z10.s, z10.s, #0x1f\n" - "asr z4.s, z4.s, #0x1f\n" - "sqadd z20.s, z20.s, z5.s\n" - "sqadd z21.s, z21.s, z6.s\n" - "sqadd z22.s, z22.s, z7.s\n" - "sqadd z16.s, z16.s, z8.s\n" - "sqadd z17.s, z17.s, z9.s\n" - "sqadd z18.s, z18.s, z10.s\n" - "sqadd z19.s, z19.s, z4.s\n" + "and z24.d, z23.d, z0.d\n" + "asr z24.s, z24.s, #0x1f\n" + "sqadd z23.s, z23.s, z24.s\n" + "and z30.d, z20.d, z0.d\n" + "and z29.d, z21.d, z0.d\n" + "and z28.d, z22.d, z0.d\n" + "and z27.d, z16.d, z0.d\n" + "and z26.d, z17.d, z0.d\n" + "and z25.d, z18.d, z0.d\n" + "and z24.d, z19.d, z0.d\n" + "asr z30.s, z30.s, #0x1f\n" + "asr z29.s, z29.s, #0x1f\n" + "asr z28.s, z28.s, #0x1f\n" + "asr z27.s, z27.s, #0x1f\n" + "asr z26.s, z26.s, #0x1f\n" + "asr z25.s, z25.s, #0x1f\n" + "asr z24.s, z24.s, #0x1f\n" + "sqadd z20.s, z20.s, z30.s\n" + "sqadd z21.s, z21.s, z29.s\n" + "sqadd z22.s, z22.s, z28.s\n" + "sqadd z16.s, z16.s, z27.s\n" + "sqadd z17.s, z17.s, z26.s\n" + "sqadd z18.s, z18.s, z25.s\n" + "sqadd z19.s, z19.s, z24.s\n" "27:" // Height 2: no shift correction - "add x23, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x23]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1rw { z24.s }, p2/Z, [x20]\n" ".inst 0x44828817 // srshl z23.s, p2/M, z23.s, z0.s\n" - "add z23.s, z23.s, z4.s\n" + "add z23.s, z23.s, z24.s\n" ".inst 0x44828814 // srshl z20.s, p2/M, z20.s, z0.s\n" ".inst 0x44828815 // srshl z21.s, p2/M, z21.s, z0.s\n" - "add z20.s, z20.s, z4.s\n" - "add z21.s, z21.s, z4.s\n" + "add z20.s, z20.s, z24.s\n" + "add z21.s, z21.s, z24.s\n" ".inst 0x44828816 // srshl z22.s, p2/M, z22.s, z0.s\n" ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n" - "add z22.s, z22.s, z4.s\n" - "add z16.s, z16.s, z4.s\n" + "add z22.s, z22.s, z24.s\n" + "add z16.s, z16.s, z24.s\n" ".inst 0x44828811 // srshl z17.s, p2/M, z17.s, z0.s\n" ".inst 0x44828812 // srshl z18.s, p2/M, z18.s, z0.s\n" - "add z17.s, z17.s, z4.s\n" - "add z18.s, z18.s, z4.s\n" + "add z17.s, z17.s, z24.s\n" + "add z18.s, z18.s, z24.s\n" ".inst 0x44828813 // srshl z19.s, p2/M, z19.s, z0.s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x23]\n" - "add z19.s, z19.s, z4.s\n" - "add x23, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x23]\n" - "smin z23.s, p2/M, z23.s, z6.s\n" - "smin z20.s, p2/M, z20.s, z6.s\n" - "smin z21.s, p2/M, z21.s, z6.s\n" - "smin z22.s, p2/M, z22.s, z6.s\n" - "smin z16.s, p2/M, z16.s, z6.s\n" - "smin z17.s, p2/M, z17.s, z6.s\n" - "smin z18.s, p2/M, z18.s, z6.s\n" - "smin z19.s, p2/M, z19.s, z6.s\n" - "smax z23.s, p2/M, z23.s, z5.s\n" - "smax z20.s, p2/M, z20.s, z5.s\n" - "smax z21.s, p2/M, z21.s, z5.s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1rw { z25.s }, p2/Z, [x20]\n" + "add z19.s, z19.s, z24.s\n" + "add x20, %x[qp], %[minval]\n" + "ld1rw { z24.s }, p2/Z, [x20]\n" + "smin z23.s, p2/M, z23.s, z25.s\n" + "smin z20.s, p2/M, z20.s, z25.s\n" + "smin z21.s, p2/M, z21.s, z25.s\n" + "smin z22.s, p2/M, z22.s, z25.s\n" + "smin z16.s, p2/M, z16.s, z25.s\n" + "smin z17.s, p2/M, z17.s, z25.s\n" + "smin z18.s, p2/M, z18.s, z25.s\n" + "smin z19.s, p2/M, z19.s, z25.s\n" + "smax z23.s, p2/M, z23.s, z24.s\n" + "smax z20.s, p2/M, z20.s, z24.s\n" + "smax z21.s, p2/M, z21.s, z24.s\n" "uzp1 z23.h, z23.h, z20.h\n" - "smax z22.s, p2/M, z22.s, z5.s\n" - "smax z16.s, p2/M, z16.s, z5.s\n" + "smax z22.s, p2/M, z22.s, z24.s\n" + "smax z16.s, p2/M, z16.s, z24.s\n" "uzp1 z20.h, z21.h, z22.h\n" "uzp1 z23.b, z23.b, z20.b\n" - "smax z17.s, p2/M, z17.s, z5.s\n" - "smax z18.s, p2/M, z18.s, z5.s\n" + "smax z17.s, p2/M, z17.s, z24.s\n" + "smax z18.s, p2/M, z18.s, z24.s\n" "uzp1 z16.h, z16.h, z17.h\n" "st1b { z23.b }, p1, [x27]\n" - "smax z19.s, p2/M, z19.s, z5.s\n" + "smax z19.s, p2/M, z19.s, z24.s\n" "uzp1 z17.h, z18.h, z19.h\n" "uzp1 z16.b, z16.b, z17.b\n" - "st1b { z16.b }, p1, [x22]\n" + "st1b { z16.b }, p1, [x23]\n" "addvl x27, x27, #1\n" "28:" // Height 2: Writeback done "decw x9, ALL, MUL #4\n" @@ -607,13 +607,13 @@ void sve_hybrid_s8qa_mmla_4x4VL ( "32:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 33f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" "cbnz x26, 34f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -622,8 +622,8 @@ void sve_hybrid_s8qa_mmla_4x4VL ( "b 34f\n" "33:" // Height 3: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" "34:" // Height 3: input setup done "cmp x25, #0x10\n" "ble 37f\n" @@ -634,60 +634,60 @@ void sve_hybrid_s8qa_mmla_4x4VL ( "ld1rqb { z3.b }, p0/Z, [x22]\n" "trn1 z0.d, z1.d, z2.d\n" "trn2 z1.d, z1.d, z2.d\n" - "ld1b { z5.b }, p2/Z, [x28]\n" - "trn1 z2.d, z3.d, z4.d\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x45059810 // smmla z16.s, z0.b, z5.b\n" - "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" - ".inst 0x45059858 // smmla z24.s, z2.b, z5.b\n" - "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" - ".inst 0x45069814 // smmla z20.s, z0.b, z6.b\n" - "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n" - "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n" - ".inst 0x4506985c // smmla z28.s, z2.b, z6.b\n" - ".inst 0x45079811 // smmla z17.s, z0.b, z7.b\n" - "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n" + "ld1b { z4.b }, p2/Z, [x28]\n" + "trn1 z2.d, z3.d, z5.d\n" + "trn2 z3.d, z3.d, z5.d\n" + ".inst 0x45049810 // smmla z16.s, z0.b, z4.b\n" + "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n" + ".inst 0x45049858 // smmla z24.s, z2.b, z4.b\n" + "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z9.b }, p2/Z, [x28, #3, MUL VL]\n" + ".inst 0x45059814 // smmla z20.s, z0.b, z5.b\n" + "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n" + "ld1b { z7.b }, p2/Z, [x28, #5, MUL VL]\n" + ".inst 0x4505985c // smmla z28.s, z2.b, z5.b\n" + ".inst 0x45049811 // smmla z17.s, z0.b, z4.b\n" + "ld1b { z6.b }, p2/Z, [x28, #6, MUL VL]\n" "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n" "addvl x28, x28, #16\n" - ".inst 0x45079859 // smmla z25.s, z2.b, z7.b\n" - "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n" - ".inst 0x45089815 // smmla z21.s, z0.b, z8.b\n" - ".inst 0x4508985d // smmla z29.s, z2.b, z8.b\n" - "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n" - ".inst 0x45099812 // smmla z18.s, z0.b, z9.b\n" - ".inst 0x4509985a // smmla z26.s, z2.b, z9.b\n" - "ld1b { z8.b }, p2/Z, [x28, #-6, MUL VL]\n" - "ld1b { z9.b }, p2/Z, [x28, #-5, MUL VL]\n" - ".inst 0x450a9816 // smmla z22.s, z0.b, z10.b\n" - ".inst 0x450a985e // smmla z30.s, z2.b, z10.b\n" - "ld1b { z10.b }, p2/Z, [x28, #-4, MUL VL]\n" + ".inst 0x45049859 // smmla z25.s, z2.b, z4.b\n" + "ld1b { z4.b }, p2/Z, [x28, #-8, MUL VL]\n" + ".inst 0x45099815 // smmla z21.s, z0.b, z9.b\n" + ".inst 0x4509985d // smmla z29.s, z2.b, z9.b\n" + "ld1b { z10.b }, p2/Z, [x28, #-7, MUL VL]\n" + ".inst 0x45089812 // smmla z18.s, z0.b, z8.b\n" + ".inst 0x4508985a // smmla z26.s, z2.b, z8.b\n" + "ld1b { z9.b }, p2/Z, [x28, #-6, MUL VL]\n" + "ld1b { z8.b }, p2/Z, [x28, #-5, MUL VL]\n" + ".inst 0x45079816 // smmla z22.s, z0.b, z7.b\n" + ".inst 0x4507985e // smmla z30.s, z2.b, z7.b\n" + "ld1b { z7.b }, p2/Z, [x28, #-4, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x45049813 // smmla z19.s, z0.b, z4.b\n" - ".inst 0x4504985b // smmla z27.s, z2.b, z4.b\n" - "ld1b { z4.b }, p2/Z, [x28, #-3, MUL VL]\n" + ".inst 0x45069813 // smmla z19.s, z0.b, z6.b\n" + ".inst 0x4506985b // smmla z27.s, z2.b, z6.b\n" + "ld1b { z6.b }, p2/Z, [x28, #-3, MUL VL]\n" "add x23, x23, #0x10\n" ".inst 0x45059817 // smmla z23.s, z0.b, z5.b\n" ".inst 0x4505985f // smmla z31.s, z2.b, z5.b\n" "ld1b { z5.b }, p2/Z, [x28, #-2, MUL VL]\n" "add x22, x22, #0x10\n" - ".inst 0x45069830 // smmla z16.s, z1.b, z6.b\n" - ".inst 0x45069878 // smmla z24.s, z3.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x28, #-1, MUL VL]\n" - ".inst 0x45079834 // smmla z20.s, z1.b, z7.b\n" - ".inst 0x4507987c // smmla z28.s, z3.b, z7.b\n" - ".inst 0x45089831 // smmla z17.s, z1.b, z8.b\n" - ".inst 0x45089879 // smmla z25.s, z3.b, z8.b\n" - ".inst 0x45099835 // smmla z21.s, z1.b, z9.b\n" - ".inst 0x4509987d // smmla z29.s, z3.b, z9.b\n" - ".inst 0x450a9832 // smmla z18.s, z1.b, z10.b\n" - ".inst 0x450a987a // smmla z26.s, z3.b, z10.b\n" - ".inst 0x45049836 // smmla z22.s, z1.b, z4.b\n" - ".inst 0x4504987e // smmla z30.s, z3.b, z4.b\n" + ".inst 0x45049830 // smmla z16.s, z1.b, z4.b\n" + ".inst 0x45049878 // smmla z24.s, z3.b, z4.b\n" + "ld1b { z4.b }, p2/Z, [x28, #-1, MUL VL]\n" + ".inst 0x450a9834 // smmla z20.s, z1.b, z10.b\n" + ".inst 0x450a987c // smmla z28.s, z3.b, z10.b\n" + ".inst 0x45099831 // smmla z17.s, z1.b, z9.b\n" + ".inst 0x45099879 // smmla z25.s, z3.b, z9.b\n" + ".inst 0x45089835 // smmla z21.s, z1.b, z8.b\n" + ".inst 0x4508987d // smmla z29.s, z3.b, z8.b\n" + ".inst 0x45079832 // smmla z18.s, z1.b, z7.b\n" + ".inst 0x4507987a // smmla z26.s, z3.b, z7.b\n" + ".inst 0x45069836 // smmla z22.s, z1.b, z6.b\n" + ".inst 0x4506987e // smmla z30.s, z3.b, z6.b\n" ".inst 0x45059833 // smmla z19.s, z1.b, z5.b\n" ".inst 0x4505987b // smmla z27.s, z3.b, z5.b\n" - ".inst 0x45069837 // smmla z23.s, z1.b, z6.b\n" - ".inst 0x4506987f // smmla z31.s, z3.b, z6.b\n" + ".inst 0x45049837 // smmla z23.s, z1.b, z4.b\n" + ".inst 0x4504987f // smmla z31.s, z3.b, z4.b\n" "tbnz %x[flags], #31, 36f\n" "sdot z11.s, z0.b, z15.b\n" "sdot z13.s, z2.b, z15.b\n" @@ -708,56 +708,56 @@ void sve_hybrid_s8qa_mmla_4x4VL ( "trn1 z2.d, z3.d, z4.d\n" "trn2 z3.d, z3.d, z4.d\n" ".inst 0x45059810 // smmla z16.s, z0.b, z5.b\n" - "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z4.b }, p2/Z, [x28, #1, MUL VL]\n" ".inst 0x45059858 // smmla z24.s, z2.b, z5.b\n" - "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z9.b }, p2/Z, [x28, #2, MUL VL]\n" "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" "subs x25, x25, #0x8\n" - "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n" - "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n" - ".inst 0x45069814 // smmla z20.s, z0.b, z6.b\n" - ".inst 0x4506985c // smmla z28.s, z2.b, z6.b\n" - "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n" - "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n" - ".inst 0x45079811 // smmla z17.s, z0.b, z7.b\n" - ".inst 0x45079859 // smmla z25.s, z2.b, z7.b\n" + "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n" + "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n" + ".inst 0x45049814 // smmla z20.s, z0.b, z4.b\n" + ".inst 0x4504985c // smmla z28.s, z2.b, z4.b\n" + "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n" + "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n" + ".inst 0x45099811 // smmla z17.s, z0.b, z9.b\n" + ".inst 0x45099859 // smmla z25.s, z2.b, z9.b\n" ".inst 0x45089815 // smmla z21.s, z0.b, z8.b\n" ".inst 0x4508985d // smmla z29.s, z2.b, z8.b\n" "addvl x28, x28, #8\n" - ".inst 0x45099812 // smmla z18.s, z0.b, z9.b\n" - ".inst 0x4509985a // smmla z26.s, z2.b, z9.b\n" - ".inst 0x450a9816 // smmla z22.s, z0.b, z10.b\n" - ".inst 0x450a985e // smmla z30.s, z2.b, z10.b\n" - ".inst 0x45049813 // smmla z19.s, z0.b, z4.b\n" - ".inst 0x4504985b // smmla z27.s, z2.b, z4.b\n" - ".inst 0x45059817 // smmla z23.s, z0.b, z5.b\n" - ".inst 0x4505985f // smmla z31.s, z2.b, z5.b\n" + ".inst 0x45079812 // smmla z18.s, z0.b, z7.b\n" + ".inst 0x4507985a // smmla z26.s, z2.b, z7.b\n" + ".inst 0x45069816 // smmla z22.s, z0.b, z6.b\n" + ".inst 0x4506985e // smmla z30.s, z2.b, z6.b\n" + ".inst 0x45059813 // smmla z19.s, z0.b, z5.b\n" + ".inst 0x4505985b // smmla z27.s, z2.b, z5.b\n" + ".inst 0x45049817 // smmla z23.s, z0.b, z4.b\n" + ".inst 0x4504985f // smmla z31.s, z2.b, z4.b\n" "ble 38f\n" - "ld1b { z6.b }, p2/Z, [x28]\n" - ".inst 0x45069830 // smmla z16.s, z1.b, z6.b\n" - ".inst 0x45069878 // smmla z24.s, z3.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n" - "ld1b { z8.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z9.b }, p2/Z, [x28, #3, MUL VL]\n" - ".inst 0x45079834 // smmla z20.s, z1.b, z7.b\n" - ".inst 0x4507987c // smmla z28.s, z3.b, z7.b\n" - "ld1b { z10.b }, p2/Z, [x28, #4, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #5, MUL VL]\n" - ".inst 0x45089831 // smmla z17.s, z1.b, z8.b\n" - ".inst 0x45089879 // smmla z25.s, z3.b, z8.b\n" + "ld1b { z4.b }, p2/Z, [x28]\n" + ".inst 0x45049830 // smmla z16.s, z1.b, z4.b\n" + ".inst 0x45049878 // smmla z24.s, z3.b, z4.b\n" + "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" + ".inst 0x45059834 // smmla z20.s, z1.b, z5.b\n" + ".inst 0x4505987c // smmla z28.s, z3.b, z5.b\n" + "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n" + "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n" + ".inst 0x45049831 // smmla z17.s, z1.b, z4.b\n" + ".inst 0x45049879 // smmla z25.s, z3.b, z4.b\n" "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n" - ".inst 0x45099835 // smmla z21.s, z1.b, z9.b\n" - ".inst 0x4509987d // smmla z29.s, z3.b, z9.b\n" - ".inst 0x450a9832 // smmla z18.s, z1.b, z10.b\n" - ".inst 0x450a987a // smmla z26.s, z3.b, z10.b\n" + "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n" + ".inst 0x45089835 // smmla z21.s, z1.b, z8.b\n" + ".inst 0x4508987d // smmla z29.s, z3.b, z8.b\n" + ".inst 0x45079832 // smmla z18.s, z1.b, z7.b\n" + ".inst 0x4507987a // smmla z26.s, z3.b, z7.b\n" "addvl x28, x28, #8\n" - ".inst 0x45049836 // smmla z22.s, z1.b, z4.b\n" - ".inst 0x4504987e // smmla z30.s, z3.b, z4.b\n" + ".inst 0x45069836 // smmla z22.s, z1.b, z6.b\n" + ".inst 0x4506987e // smmla z30.s, z3.b, z6.b\n" ".inst 0x45059833 // smmla z19.s, z1.b, z5.b\n" ".inst 0x4505987b // smmla z27.s, z3.b, z5.b\n" - ".inst 0x45069837 // smmla z23.s, z1.b, z6.b\n" - ".inst 0x4506987f // smmla z31.s, z3.b, z6.b\n" + ".inst 0x45049837 // smmla z23.s, z1.b, z4.b\n" + ".inst 0x4504987f // smmla z31.s, z3.b, z4.b\n" "38:" // Height 3: Multiply loop: multiply skip "tbnz %x[flags], #31, 39f\n" "sdot z11.s, z0.b, z15.b\n" @@ -770,12 +770,12 @@ void sve_hybrid_s8qa_mmla_4x4VL ( "cmp x26, x20\n" "bne 32b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "uzp1 z7.d, z16.d, z20.d\n" - "add x22, x27, x20\n" + "uzp1 z0.d, z16.d, z20.d\n" + "add x23, x27, x20\n" "uzp2 z16.d, z16.d, z20.d\n" "uzp1 z20.d, z17.d, z21.d\n" "uzp2 z17.d, z17.d, z21.d\n" - "add x21, x22, x20\n" + "add x22, x23, x20\n" "uzp1 z21.d, z18.d, z22.d\n" "uzp2 z18.d, z18.d, z22.d\n" "uzp1 z22.d, z19.d, z23.d\n" @@ -784,170 +784,170 @@ void sve_hybrid_s8qa_mmla_4x4VL ( "uzp1 z25.d, z25.d, z29.d\n" "uzp1 z26.d, z26.d, z30.d\n" "uzp1 z27.d, z27.d, z31.d\n" - "mov z31.d, z7.d\n" + "mov z31.d, z0.d\n" "tbnz %x[flags], #31, 40f\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1rw { z3.s }, p2/Z, [x23]\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1rw { z23.s }, p2/Z, [x20]\n" ".inst 0x4491a96b // addp z11.s, p2/m, z11.s, z11.s\n" ".inst 0x4491a9ad // addp z13.s, p2/m, z13.s, z13.s\n" - "neg z3.s, p2/M, z3.s\n" + "neg z23.s, p2/M, z23.s\n" "mov z12.s, z11.s[3]\n" "mov z11.s, z11.s[0]\n" - "mul z11.s, p2/M, z11.s, z3.s\n" + "mul z11.s, p2/M, z11.s, z23.s\n" "mov z13.s, z13.s[0]\n" - "mul z12.s, p2/M, z12.s, z3.s\n" - "mul z13.s, p2/M, z13.s, z3.s\n" + "mul z12.s, p2/M, z12.s, z23.s\n" + "mul z13.s, p2/M, z13.s, z23.s\n" "40:" // Height 3: skip row sum fixup "add z31.s, z31.s, z11.s\n" "add z20.s, z20.s, z11.s\n" "ld1w { z0.s }, p2/Z, [x10]\n" - "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n" + "ld1w { z30.s }, p2/Z, [x10, #1, MUL VL]\n" "add z21.s, z21.s, z11.s\n" "add z22.s, z22.s, z11.s\n" - "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n" + "ld1w { z29.s }, p2/Z, [x10, #2, MUL VL]\n" + "ld1w { z28.s }, p2/Z, [x10, #3, MUL VL]\n" "add z16.s, z16.s, z12.s\n" "add z17.s, z17.s, z12.s\n" - "add x23, %x[qp], %[per_layer_mul]\n" + "add x20, %x[qp], %[per_layer_mul]\n" "orr %x[flags], %x[flags], #0x80000000\n" "add z18.s, z18.s, z12.s\n" "add z19.s, z19.s, z12.s\n" - "ld1rw { z4.s }, p2/Z, [x23]\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" + "ld1rw { z23.s }, p2/Z, [x20]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" "add z24.s, z24.s, z13.s\n" "add z25.s, z25.s, z13.s\n" "addvl x10, x10, #4\n" "add z26.s, z26.s, z13.s\n" "add z27.s, z27.s, z13.s\n" "add z31.s, z31.s, z0.s\n" - "add z20.s, z20.s, z1.s\n" - "add z21.s, z21.s, z2.s\n" - "add z22.s, z22.s, z3.s\n" + "add z20.s, z20.s, z30.s\n" + "add z21.s, z21.s, z29.s\n" + "add z22.s, z22.s, z28.s\n" "add z16.s, z16.s, z0.s\n" - "add z17.s, z17.s, z1.s\n" - "add z18.s, z18.s, z2.s\n" - "add z19.s, z19.s, z3.s\n" + "add z17.s, z17.s, z30.s\n" + "add z18.s, z18.s, z29.s\n" + "add z19.s, z19.s, z28.s\n" "add z24.s, z24.s, z0.s\n" - "add z25.s, z25.s, z1.s\n" - "ld1rw { z0.s }, p2/Z, [x23]\n" - "add z26.s, z26.s, z2.s\n" - "add z27.s, z27.s, z3.s\n" - ".inst 0x04a477ff // sqrdmulh z31.s, z31.s, z4.s\n" - ".inst 0x04a47694 // sqrdmulh z20.s, z20.s, z4.s\n" - ".inst 0x04a476b5 // sqrdmulh z21.s, z21.s, z4.s\n" - ".inst 0x04a476d6 // sqrdmulh z22.s, z22.s, z4.s\n" - ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n" - ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n" - ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n" - ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n" - ".inst 0x04a47718 // sqrdmulh z24.s, z24.s, z4.s\n" - ".inst 0x04a47739 // sqrdmulh z25.s, z25.s, z4.s\n" - ".inst 0x04a4775a // sqrdmulh z26.s, z26.s, z4.s\n" - ".inst 0x04a4777b // sqrdmulh z27.s, z27.s, z4.s\n" + "add z25.s, z25.s, z30.s\n" + "ld1rw { z0.s }, p2/Z, [x20]\n" + "add z26.s, z26.s, z29.s\n" + "add z27.s, z27.s, z28.s\n" + ".inst 0x04b777ff // sqrdmulh z31.s, z31.s, z23.s\n" + ".inst 0x04b77694 // sqrdmulh z20.s, z20.s, z23.s\n" + ".inst 0x04b776b5 // sqrdmulh z21.s, z21.s, z23.s\n" + ".inst 0x04b776d6 // sqrdmulh z22.s, z22.s, z23.s\n" + ".inst 0x04b77610 // sqrdmulh z16.s, z16.s, z23.s\n" + ".inst 0x04b77631 // sqrdmulh z17.s, z17.s, z23.s\n" + ".inst 0x04b77652 // sqrdmulh z18.s, z18.s, z23.s\n" + ".inst 0x04b77673 // sqrdmulh z19.s, z19.s, z23.s\n" + ".inst 0x04b77718 // sqrdmulh z24.s, z24.s, z23.s\n" + ".inst 0x04b77739 // sqrdmulh z25.s, z25.s, z23.s\n" + ".inst 0x04b7775a // sqrdmulh z26.s, z26.s, z23.s\n" + ".inst 0x04b7777b // sqrdmulh z27.s, z27.s, z23.s\n" "tbz %x[flags], #5, 41f\n" - "and z4.d, z31.d, z0.d\n" - "and z5.d, z20.d, z0.d\n" - "and z6.d, z21.d, z0.d\n" - "and z7.d, z22.d, z0.d\n" - "and z8.d, z16.d, z0.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "asr z8.s, z8.s, #0x1f\n" - "sqadd z31.s, z31.s, z4.s\n" - "sqadd z20.s, z20.s, z5.s\n" - "sqadd z21.s, z21.s, z6.s\n" - "sqadd z22.s, z22.s, z7.s\n" - "sqadd z16.s, z16.s, z8.s\n" - "and z9.d, z17.d, z0.d\n" - "and z10.d, z18.d, z0.d\n" - "and z4.d, z19.d, z0.d\n" - "and z5.d, z24.d, z0.d\n" - "and z6.d, z25.d, z0.d\n" - "and z7.d, z26.d, z0.d\n" - "and z8.d, z27.d, z0.d\n" - "asr z9.s, z9.s, #0x1f\n" - "asr z10.s, z10.s, #0x1f\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "asr z8.s, z8.s, #0x1f\n" - "sqadd z17.s, z17.s, z9.s\n" - "sqadd z18.s, z18.s, z10.s\n" - "sqadd z19.s, z19.s, z4.s\n" - "sqadd z24.s, z24.s, z5.s\n" - "sqadd z25.s, z25.s, z6.s\n" - "sqadd z26.s, z26.s, z7.s\n" - "sqadd z27.s, z27.s, z8.s\n" + "and z1.d, z31.d, z0.d\n" + "and z30.d, z20.d, z0.d\n" + "and z29.d, z21.d, z0.d\n" + "and z28.d, z22.d, z0.d\n" + "and z23.d, z16.d, z0.d\n" + "asr z1.s, z1.s, #0x1f\n" + "asr z30.s, z30.s, #0x1f\n" + "asr z29.s, z29.s, #0x1f\n" + "asr z28.s, z28.s, #0x1f\n" + "asr z23.s, z23.s, #0x1f\n" + "sqadd z31.s, z31.s, z1.s\n" + "sqadd z20.s, z20.s, z30.s\n" + "sqadd z21.s, z21.s, z29.s\n" + "sqadd z22.s, z22.s, z28.s\n" + "sqadd z16.s, z16.s, z23.s\n" + "and z3.d, z17.d, z0.d\n" + "and z2.d, z18.d, z0.d\n" + "and z1.d, z19.d, z0.d\n" + "and z30.d, z24.d, z0.d\n" + "and z29.d, z25.d, z0.d\n" + "and z28.d, z26.d, z0.d\n" + "and z23.d, z27.d, z0.d\n" + "asr z3.s, z3.s, #0x1f\n" + "asr z2.s, z2.s, #0x1f\n" + "asr z1.s, z1.s, #0x1f\n" + "asr z30.s, z30.s, #0x1f\n" + "asr z29.s, z29.s, #0x1f\n" + "asr z28.s, z28.s, #0x1f\n" + "asr z23.s, z23.s, #0x1f\n" + "sqadd z17.s, z17.s, z3.s\n" + "sqadd z18.s, z18.s, z2.s\n" + "sqadd z19.s, z19.s, z1.s\n" + "sqadd z24.s, z24.s, z30.s\n" + "sqadd z25.s, z25.s, z29.s\n" + "sqadd z26.s, z26.s, z28.s\n" + "sqadd z27.s, z27.s, z23.s\n" "41:" // Height 3: no shift correction - "add x23, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x23]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1rw { z23.s }, p2/Z, [x20]\n" ".inst 0x4482881f // srshl z31.s, p2/M, z31.s, z0.s\n" - "add z31.s, z31.s, z4.s\n" + "add z31.s, z31.s, z23.s\n" ".inst 0x44828814 // srshl z20.s, p2/M, z20.s, z0.s\n" ".inst 0x44828815 // srshl z21.s, p2/M, z21.s, z0.s\n" - "add z20.s, z20.s, z4.s\n" - "add z21.s, z21.s, z4.s\n" + "add z20.s, z20.s, z23.s\n" + "add z21.s, z21.s, z23.s\n" ".inst 0x44828816 // srshl z22.s, p2/M, z22.s, z0.s\n" ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n" - "add z22.s, z22.s, z4.s\n" - "add z16.s, z16.s, z4.s\n" + "add z22.s, z22.s, z23.s\n" + "add z16.s, z16.s, z23.s\n" ".inst 0x44828811 // srshl z17.s, p2/M, z17.s, z0.s\n" ".inst 0x44828812 // srshl z18.s, p2/M, z18.s, z0.s\n" - "add z17.s, z17.s, z4.s\n" - "add z18.s, z18.s, z4.s\n" + "add z17.s, z17.s, z23.s\n" + "add z18.s, z18.s, z23.s\n" ".inst 0x44828813 // srshl z19.s, p2/M, z19.s, z0.s\n" ".inst 0x44828818 // srshl z24.s, p2/M, z24.s, z0.s\n" - "add z19.s, z19.s, z4.s\n" - "add z24.s, z24.s, z4.s\n" + "add z19.s, z19.s, z23.s\n" + "add z24.s, z24.s, z23.s\n" ".inst 0x44828819 // srshl z25.s, p2/M, z25.s, z0.s\n" ".inst 0x4482881a // srshl z26.s, p2/M, z26.s, z0.s\n" - "add z25.s, z25.s, z4.s\n" - "add z26.s, z26.s, z4.s\n" + "add z25.s, z25.s, z23.s\n" + "add z26.s, z26.s, z23.s\n" ".inst 0x4482881b // srshl z27.s, p2/M, z27.s, z0.s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x23]\n" - "add z27.s, z27.s, z4.s\n" - "add x23, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x23]\n" - "smin z31.s, p2/M, z31.s, z6.s\n" - "smin z20.s, p2/M, z20.s, z6.s\n" - "smin z21.s, p2/M, z21.s, z6.s\n" - "smin z22.s, p2/M, z22.s, z6.s\n" - "smin z16.s, p2/M, z16.s, z6.s\n" - "smin z17.s, p2/M, z17.s, z6.s\n" - "smin z18.s, p2/M, z18.s, z6.s\n" - "smin z19.s, p2/M, z19.s, z6.s\n" - "smin z24.s, p2/M, z24.s, z6.s\n" - "smin z25.s, p2/M, z25.s, z6.s\n" - "smin z26.s, p2/M, z26.s, z6.s\n" - "smin z27.s, p2/M, z27.s, z6.s\n" - "smax z31.s, p2/M, z31.s, z5.s\n" - "smax z20.s, p2/M, z20.s, z5.s\n" - "smax z21.s, p2/M, z21.s, z5.s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1rw { z28.s }, p2/Z, [x20]\n" + "add z27.s, z27.s, z23.s\n" + "add x20, %x[qp], %[minval]\n" + "ld1rw { z23.s }, p2/Z, [x20]\n" + "smin z31.s, p2/M, z31.s, z28.s\n" + "smin z20.s, p2/M, z20.s, z28.s\n" + "smin z21.s, p2/M, z21.s, z28.s\n" + "smin z22.s, p2/M, z22.s, z28.s\n" + "smin z16.s, p2/M, z16.s, z28.s\n" + "smin z17.s, p2/M, z17.s, z28.s\n" + "smin z18.s, p2/M, z18.s, z28.s\n" + "smin z19.s, p2/M, z19.s, z28.s\n" + "smin z24.s, p2/M, z24.s, z28.s\n" + "smin z25.s, p2/M, z25.s, z28.s\n" + "smin z26.s, p2/M, z26.s, z28.s\n" + "smin z27.s, p2/M, z27.s, z28.s\n" + "smax z31.s, p2/M, z31.s, z23.s\n" + "smax z20.s, p2/M, z20.s, z23.s\n" + "smax z21.s, p2/M, z21.s, z23.s\n" "uzp1 z31.h, z31.h, z20.h\n" - "smax z22.s, p2/M, z22.s, z5.s\n" - "smax z16.s, p2/M, z16.s, z5.s\n" + "smax z22.s, p2/M, z22.s, z23.s\n" + "smax z16.s, p2/M, z16.s, z23.s\n" "uzp1 z20.h, z21.h, z22.h\n" "uzp1 z31.b, z31.b, z20.b\n" - "smax z17.s, p2/M, z17.s, z5.s\n" - "smax z18.s, p2/M, z18.s, z5.s\n" + "smax z17.s, p2/M, z17.s, z23.s\n" + "smax z18.s, p2/M, z18.s, z23.s\n" "uzp1 z16.h, z16.h, z17.h\n" "st1b { z31.b }, p1, [x27]\n" - "smax z19.s, p2/M, z19.s, z5.s\n" - "smax z24.s, p2/M, z24.s, z5.s\n" + "smax z19.s, p2/M, z19.s, z23.s\n" + "smax z24.s, p2/M, z24.s, z23.s\n" "uzp1 z17.h, z18.h, z19.h\n" "uzp1 z16.b, z16.b, z17.b\n" - "smax z25.s, p2/M, z25.s, z5.s\n" - "smax z26.s, p2/M, z26.s, z5.s\n" + "smax z25.s, p2/M, z25.s, z23.s\n" + "smax z26.s, p2/M, z26.s, z23.s\n" "uzp1 z24.h, z24.h, z25.h\n" - "st1b { z16.b }, p1, [x22]\n" - "smax z27.s, p2/M, z27.s, z5.s\n" - "uzp1 z25.h, z26.h, z27.h\n" - "uzp1 z24.b, z24.b, z25.b\n" - "st1b { z24.b }, p1, [x21]\n" + "st1b { z16.b }, p1, [x23]\n" + "smax z27.s, p2/M, z27.s, z23.s\n" + "uzp1 z16.h, z26.h, z27.h\n" + "uzp1 z24.b, z24.b, z16.b\n" + "st1b { z24.b }, p1, [x22]\n" "addvl x27, x27, #1\n" "42:" // Height 3: Writeback done "decw x9, ALL, MUL #4\n" @@ -992,14 +992,14 @@ void sve_hybrid_s8qa_mmla_4x4VL ( "46:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 47f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" - "ldr x21, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" + "ldr x21, [x20, #0x18]\n" "cbnz x26, 48f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -1009,9 +1009,9 @@ void sve_hybrid_s8qa_mmla_4x4VL ( "b 48f\n" "47:" // Height 4: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" + "add x21, x22, x21\n" "48:" // Height 4: input setup done "cmp x25, #0x10\n" "ble 51f\n" @@ -1021,63 +1021,63 @@ void sve_hybrid_s8qa_mmla_4x4VL ( "ld1rqb { z2.b }, p0/Z, [x23]\n" "trn1 z0.d, z1.d, z2.d\n" "ld1rqb { z3.b }, p0/Z, [x22]\n" - "ld1rqb { z4.b }, p0/Z, [x21]\n" + "ld1rqb { z5.b }, p0/Z, [x21]\n" "trn2 z1.d, z1.d, z2.d\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1b { z5.b }, p2/Z, [x28]\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x45059810 // smmla z16.s, z0.b, z5.b\n" - ".inst 0x45059858 // smmla z24.s, z2.b, z5.b\n" - "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n" - ".inst 0x45069814 // smmla z20.s, z0.b, z6.b\n" - ".inst 0x4506985c // smmla z28.s, z2.b, z6.b\n" - "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" - "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n" - ".inst 0x45079811 // smmla z17.s, z0.b, z7.b\n" - ".inst 0x45079859 // smmla z25.s, z2.b, z7.b\n" - "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n" - ".inst 0x45089815 // smmla z21.s, z0.b, z8.b\n" - ".inst 0x4508985d // smmla z29.s, z2.b, z8.b\n" + "trn1 z2.d, z3.d, z5.d\n" + "ld1b { z4.b }, p2/Z, [x28]\n" + "trn2 z3.d, z3.d, z5.d\n" + ".inst 0x45049810 // smmla z16.s, z0.b, z4.b\n" + ".inst 0x45049858 // smmla z24.s, z2.b, z4.b\n" + "ld1b { z4.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z5.b }, p2/Z, [x28, #2, MUL VL]\n" + ".inst 0x45049814 // smmla z20.s, z0.b, z4.b\n" + ".inst 0x4504985c // smmla z28.s, z2.b, z4.b\n" + "ld1b { z4.b }, p2/Z, [x28, #3, MUL VL]\n" + "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n" + ".inst 0x45059811 // smmla z17.s, z0.b, z5.b\n" + ".inst 0x45059859 // smmla z25.s, z2.b, z5.b\n" + "ld1b { z7.b }, p2/Z, [x28, #5, MUL VL]\n" + "ld1b { z6.b }, p2/Z, [x28, #6, MUL VL]\n" + ".inst 0x45049815 // smmla z21.s, z0.b, z4.b\n" + ".inst 0x4504985d // smmla z29.s, z2.b, z4.b\n" "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n" "addvl x28, x28, #16\n" - "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n" - ".inst 0x45099812 // smmla z18.s, z0.b, z9.b\n" - ".inst 0x4509985a // smmla z26.s, z2.b, z9.b\n" - ".inst 0x450a9816 // smmla z22.s, z0.b, z10.b\n" - "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n" - "ld1b { z8.b }, p2/Z, [x28, #-6, MUL VL]\n" - ".inst 0x450a985e // smmla z30.s, z2.b, z10.b\n" - ".inst 0x45049813 // smmla z19.s, z0.b, z4.b\n" - "ld1b { z9.b }, p2/Z, [x28, #-5, MUL VL]\n" - "ld1b { z10.b }, p2/Z, [x28, #-4, MUL VL]\n" - ".inst 0x4504985b // smmla z27.s, z2.b, z4.b\n" + "ld1b { z4.b }, p2/Z, [x28, #-8, MUL VL]\n" + ".inst 0x45089812 // smmla z18.s, z0.b, z8.b\n" + ".inst 0x4508985a // smmla z26.s, z2.b, z8.b\n" + ".inst 0x45079816 // smmla z22.s, z0.b, z7.b\n" + "ld1b { z10.b }, p2/Z, [x28, #-7, MUL VL]\n" + "ld1b { z9.b }, p2/Z, [x28, #-6, MUL VL]\n" + ".inst 0x4507985e // smmla z30.s, z2.b, z7.b\n" + ".inst 0x45069813 // smmla z19.s, z0.b, z6.b\n" + "ld1b { z8.b }, p2/Z, [x28, #-5, MUL VL]\n" + "ld1b { z7.b }, p2/Z, [x28, #-4, MUL VL]\n" + ".inst 0x4506985b // smmla z27.s, z2.b, z6.b\n" ".inst 0x45059817 // smmla z23.s, z0.b, z5.b\n" - "ld1b { z4.b }, p2/Z, [x28, #-3, MUL VL]\n" + "ld1b { z6.b }, p2/Z, [x28, #-3, MUL VL]\n" "add x24, x24, #0x10\n" ".inst 0x4505985f // smmla z31.s, z2.b, z5.b\n" - ".inst 0x45069830 // smmla z16.s, z1.b, z6.b\n" + ".inst 0x45049830 // smmla z16.s, z1.b, z4.b\n" "ld1b { z5.b }, p2/Z, [x28, #-2, MUL VL]\n" "add x23, x23, #0x10\n" - ".inst 0x45069878 // smmla z24.s, z3.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x28, #-1, MUL VL]\n" - ".inst 0x45079834 // smmla z20.s, z1.b, z7.b\n" + ".inst 0x45049878 // smmla z24.s, z3.b, z4.b\n" + "ld1b { z4.b }, p2/Z, [x28, #-1, MUL VL]\n" + ".inst 0x450a9834 // smmla z20.s, z1.b, z10.b\n" "add x22, x22, #0x10\n" - ".inst 0x4507987c // smmla z28.s, z3.b, z7.b\n" - ".inst 0x45089831 // smmla z17.s, z1.b, z8.b\n" + ".inst 0x450a987c // smmla z28.s, z3.b, z10.b\n" + ".inst 0x45099831 // smmla z17.s, z1.b, z9.b\n" "add x21, x21, #0x10\n" - ".inst 0x45089879 // smmla z25.s, z3.b, z8.b\n" - ".inst 0x45099835 // smmla z21.s, z1.b, z9.b\n" - ".inst 0x4509987d // smmla z29.s, z3.b, z9.b\n" - ".inst 0x450a9832 // smmla z18.s, z1.b, z10.b\n" - ".inst 0x450a987a // smmla z26.s, z3.b, z10.b\n" - ".inst 0x45049836 // smmla z22.s, z1.b, z4.b\n" - ".inst 0x4504987e // smmla z30.s, z3.b, z4.b\n" + ".inst 0x45099879 // smmla z25.s, z3.b, z9.b\n" + ".inst 0x45089835 // smmla z21.s, z1.b, z8.b\n" + ".inst 0x4508987d // smmla z29.s, z3.b, z8.b\n" + ".inst 0x45079832 // smmla z18.s, z1.b, z7.b\n" + ".inst 0x4507987a // smmla z26.s, z3.b, z7.b\n" + ".inst 0x45069836 // smmla z22.s, z1.b, z6.b\n" + ".inst 0x4506987e // smmla z30.s, z3.b, z6.b\n" ".inst 0x45059833 // smmla z19.s, z1.b, z5.b\n" ".inst 0x4505987b // smmla z27.s, z3.b, z5.b\n" - ".inst 0x45069837 // smmla z23.s, z1.b, z6.b\n" - ".inst 0x4506987f // smmla z31.s, z3.b, z6.b\n" + ".inst 0x45049837 // smmla z23.s, z1.b, z4.b\n" + ".inst 0x4504987f // smmla z31.s, z3.b, z4.b\n" "tbnz %x[flags], #31, 50f\n" "sdot z11.s, z0.b, z15.b\n" "sdot z13.s, z2.b, z15.b\n" @@ -1093,62 +1093,62 @@ void sve_hybrid_s8qa_mmla_4x4VL ( "ld1rqb { z2.b }, p0/Z, [x23]\n" "trn1 z0.d, z1.d, z2.d\n" "ld1rqb { z3.b }, p0/Z, [x22]\n" - "ld1rqb { z4.b }, p0/Z, [x21]\n" + "ld1rqb { z5.b }, p0/Z, [x21]\n" "trn2 z1.d, z1.d, z2.d\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1b { z5.b }, p2/Z, [x28]\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x45059810 // smmla z16.s, z0.b, z5.b\n" - ".inst 0x45059858 // smmla z24.s, z2.b, z5.b\n" - "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n" + "trn1 z2.d, z3.d, z5.d\n" + "ld1b { z4.b }, p2/Z, [x28]\n" + "trn2 z3.d, z3.d, z5.d\n" + ".inst 0x45049810 // smmla z16.s, z0.b, z4.b\n" + ".inst 0x45049858 // smmla z24.s, z2.b, z4.b\n" + "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n" "subs x25, x25, #0x8\n" - ".inst 0x45069814 // smmla z20.s, z0.b, z6.b\n" + ".inst 0x45059814 // smmla z20.s, z0.b, z5.b\n" "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" - "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n" - ".inst 0x4506985c // smmla z28.s, z2.b, z6.b\n" - ".inst 0x45079811 // smmla z17.s, z0.b, z7.b\n" - "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n" - ".inst 0x45079859 // smmla z25.s, z2.b, z7.b\n" + "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n" + ".inst 0x4505985c // smmla z28.s, z2.b, z5.b\n" + ".inst 0x45049811 // smmla z17.s, z0.b, z4.b\n" + "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n" + "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n" + ".inst 0x45049859 // smmla z25.s, z2.b, z4.b\n" ".inst 0x45089815 // smmla z21.s, z0.b, z8.b\n" - "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n" + "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n" ".inst 0x4508985d // smmla z29.s, z2.b, z8.b\n" - ".inst 0x45099812 // smmla z18.s, z0.b, z9.b\n" + ".inst 0x45079812 // smmla z18.s, z0.b, z7.b\n" "addvl x28, x28, #8\n" - ".inst 0x4509985a // smmla z26.s, z2.b, z9.b\n" - ".inst 0x450a9816 // smmla z22.s, z0.b, z10.b\n" - ".inst 0x450a985e // smmla z30.s, z2.b, z10.b\n" - ".inst 0x45049813 // smmla z19.s, z0.b, z4.b\n" - ".inst 0x4504985b // smmla z27.s, z2.b, z4.b\n" - ".inst 0x45059817 // smmla z23.s, z0.b, z5.b\n" - ".inst 0x4505985f // smmla z31.s, z2.b, z5.b\n" + ".inst 0x4507985a // smmla z26.s, z2.b, z7.b\n" + ".inst 0x45069816 // smmla z22.s, z0.b, z6.b\n" + ".inst 0x4506985e // smmla z30.s, z2.b, z6.b\n" + ".inst 0x45059813 // smmla z19.s, z0.b, z5.b\n" + ".inst 0x4505985b // smmla z27.s, z2.b, z5.b\n" + ".inst 0x45049817 // smmla z23.s, z0.b, z4.b\n" + ".inst 0x4504985f // smmla z31.s, z2.b, z4.b\n" "ble 52f\n" - "ld1b { z6.b }, p2/Z, [x28]\n" - ".inst 0x45069830 // smmla z16.s, z1.b, z6.b\n" - ".inst 0x45069878 // smmla z24.s, z3.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n" - "ld1b { z8.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z9.b }, p2/Z, [x28, #3, MUL VL]\n" - ".inst 0x45079834 // smmla z20.s, z1.b, z7.b\n" - ".inst 0x4507987c // smmla z28.s, z3.b, z7.b\n" - "ld1b { z10.b }, p2/Z, [x28, #4, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #5, MUL VL]\n" - ".inst 0x45089831 // smmla z17.s, z1.b, z8.b\n" - ".inst 0x45089879 // smmla z25.s, z3.b, z8.b\n" + "ld1b { z4.b }, p2/Z, [x28]\n" + ".inst 0x45049830 // smmla z16.s, z1.b, z4.b\n" + ".inst 0x45049878 // smmla z24.s, z3.b, z4.b\n" + "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" + ".inst 0x45059834 // smmla z20.s, z1.b, z5.b\n" + ".inst 0x4505987c // smmla z28.s, z3.b, z5.b\n" + "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n" + "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n" + ".inst 0x45049831 // smmla z17.s, z1.b, z4.b\n" + ".inst 0x45049879 // smmla z25.s, z3.b, z4.b\n" "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n" - ".inst 0x45099835 // smmla z21.s, z1.b, z9.b\n" - ".inst 0x4509987d // smmla z29.s, z3.b, z9.b\n" - ".inst 0x450a9832 // smmla z18.s, z1.b, z10.b\n" - ".inst 0x450a987a // smmla z26.s, z3.b, z10.b\n" + "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n" + ".inst 0x45089835 // smmla z21.s, z1.b, z8.b\n" + ".inst 0x4508987d // smmla z29.s, z3.b, z8.b\n" + ".inst 0x45079832 // smmla z18.s, z1.b, z7.b\n" + ".inst 0x4507987a // smmla z26.s, z3.b, z7.b\n" "addvl x28, x28, #8\n" - ".inst 0x45049836 // smmla z22.s, z1.b, z4.b\n" - ".inst 0x4504987e // smmla z30.s, z3.b, z4.b\n" + ".inst 0x45069836 // smmla z22.s, z1.b, z6.b\n" + ".inst 0x4506987e // smmla z30.s, z3.b, z6.b\n" ".inst 0x45059833 // smmla z19.s, z1.b, z5.b\n" ".inst 0x4505987b // smmla z27.s, z3.b, z5.b\n" - ".inst 0x45069837 // smmla z23.s, z1.b, z6.b\n" - ".inst 0x4506987f // smmla z31.s, z3.b, z6.b\n" + ".inst 0x45049837 // smmla z23.s, z1.b, z4.b\n" + ".inst 0x4504987f // smmla z31.s, z3.b, z4.b\n" "52:" // Height 4: Multiply loop: multiply skip "tbnz %x[flags], #31, 53f\n" "sdot z11.s, z0.b, z15.b\n" @@ -1161,12 +1161,12 @@ void sve_hybrid_s8qa_mmla_4x4VL ( "cmp x26, x20\n" "bne 46b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "uzp1 z7.d, z16.d, z20.d\n" - "add x22, x27, x20\n" - "add x21, x22, x20\n" + "uzp1 z0.d, z16.d, z20.d\n" + "add x23, x27, x20\n" + "add x22, x23, x20\n" "uzp2 z16.d, z16.d, z20.d\n" "uzp1 z20.d, z17.d, z21.d\n" - "add x20, x21, x20\n" + "add x21, x22, x20\n" "uzp2 z17.d, z17.d, z21.d\n" "uzp1 z21.d, z18.d, z22.d\n" "uzp2 z18.d, z18.d, z22.d\n" @@ -1180,38 +1180,38 @@ void sve_hybrid_s8qa_mmla_4x4VL ( "uzp2 z26.d, z26.d, z30.d\n" "uzp1 z30.d, z27.d, z31.d\n" "uzp2 z27.d, z27.d, z31.d\n" - "mov z31.d, z7.d\n" + "mov z31.d, z0.d\n" "tbnz %x[flags], #31, 54f\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1rw { z4.s }, p2/Z, [x23]\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1rw { z0.s }, p2/Z, [x20]\n" ".inst 0x4491a96b // addp z11.s, p2/m, z11.s, z11.s\n" ".inst 0x4491a9ad // addp z13.s, p2/m, z13.s, z13.s\n" - "neg z4.s, p2/M, z4.s\n" + "neg z0.s, p2/M, z0.s\n" "mov z12.s, z11.s[3]\n" "mov z11.s, z11.s[0]\n" - "mul z11.s, p2/M, z11.s, z4.s\n" + "mul z11.s, p2/M, z11.s, z0.s\n" "mov z14.s, z13.s[3]\n" "mov z13.s, z13.s[0]\n" - "mul z12.s, p2/M, z12.s, z4.s\n" - "mul z13.s, p2/M, z13.s, z4.s\n" - "mul z14.s, p2/M, z14.s, z4.s\n" + "mul z12.s, p2/M, z12.s, z0.s\n" + "mul z13.s, p2/M, z13.s, z0.s\n" + "mul z14.s, p2/M, z14.s, z0.s\n" "54:" // Height 4: skip row sum fixup "add z31.s, z31.s, z11.s\n" "add z20.s, z20.s, z11.s\n" - "ld1w { z0.s }, p2/Z, [x10]\n" - "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n" + "ld1w { z4.s }, p2/Z, [x10]\n" + "ld1w { z0.s }, p2/Z, [x10, #1, MUL VL]\n" "add z21.s, z21.s, z11.s\n" "add z22.s, z22.s, z11.s\n" - "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n" + "ld1w { z3.s }, p2/Z, [x10, #2, MUL VL]\n" + "ld1w { z2.s }, p2/Z, [x10, #3, MUL VL]\n" "add z16.s, z16.s, z12.s\n" "add z17.s, z17.s, z12.s\n" - "add x23, %x[qp], %[per_layer_mul]\n" + "add x20, %x[qp], %[per_layer_mul]\n" "orr %x[flags], %x[flags], #0x80000000\n" "add z18.s, z18.s, z12.s\n" "add z19.s, z19.s, z12.s\n" - "ld1rw { z4.s }, p2/Z, [x23]\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" + "ld1rw { z1.s }, p2/Z, [x20]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" "add z23.s, z23.s, z13.s\n" "add z28.s, z28.s, z13.s\n" "addvl x10, x10, #4\n" @@ -1221,175 +1221,175 @@ void sve_hybrid_s8qa_mmla_4x4VL ( "add z25.s, z25.s, z14.s\n" "add z26.s, z26.s, z14.s\n" "add z27.s, z27.s, z14.s\n" - "add z31.s, z31.s, z0.s\n" - "add z20.s, z20.s, z1.s\n" - "add z21.s, z21.s, z2.s\n" - "add z22.s, z22.s, z3.s\n" - "add z16.s, z16.s, z0.s\n" - "add z17.s, z17.s, z1.s\n" - "add z18.s, z18.s, z2.s\n" - "add z19.s, z19.s, z3.s\n" - "add z23.s, z23.s, z0.s\n" - "add z28.s, z28.s, z1.s\n" - "add z29.s, z29.s, z2.s\n" - "add z30.s, z30.s, z3.s\n" - "add z24.s, z24.s, z0.s\n" - "add z25.s, z25.s, z1.s\n" - "ld1rw { z0.s }, p2/Z, [x23]\n" - "add z26.s, z26.s, z2.s\n" - "add z27.s, z27.s, z3.s\n" - ".inst 0x04a477ff // sqrdmulh z31.s, z31.s, z4.s\n" - ".inst 0x04a47694 // sqrdmulh z20.s, z20.s, z4.s\n" - ".inst 0x04a476b5 // sqrdmulh z21.s, z21.s, z4.s\n" - ".inst 0x04a476d6 // sqrdmulh z22.s, z22.s, z4.s\n" - ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n" - ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n" - ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n" - ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n" - ".inst 0x04a476f7 // sqrdmulh z23.s, z23.s, z4.s\n" - ".inst 0x04a4779c // sqrdmulh z28.s, z28.s, z4.s\n" - ".inst 0x04a477bd // sqrdmulh z29.s, z29.s, z4.s\n" - ".inst 0x04a477de // sqrdmulh z30.s, z30.s, z4.s\n" - ".inst 0x04a47718 // sqrdmulh z24.s, z24.s, z4.s\n" - ".inst 0x04a47739 // sqrdmulh z25.s, z25.s, z4.s\n" - ".inst 0x04a4775a // sqrdmulh z26.s, z26.s, z4.s\n" - ".inst 0x04a4777b // sqrdmulh z27.s, z27.s, z4.s\n" + "add z31.s, z31.s, z4.s\n" + "add z20.s, z20.s, z0.s\n" + "add z21.s, z21.s, z3.s\n" + "add z22.s, z22.s, z2.s\n" + "add z16.s, z16.s, z4.s\n" + "add z17.s, z17.s, z0.s\n" + "add z18.s, z18.s, z3.s\n" + "add z19.s, z19.s, z2.s\n" + "add z23.s, z23.s, z4.s\n" + "add z28.s, z28.s, z0.s\n" + "add z29.s, z29.s, z3.s\n" + "add z30.s, z30.s, z2.s\n" + "add z24.s, z24.s, z4.s\n" + "add z25.s, z25.s, z0.s\n" + "ld1rw { z0.s }, p2/Z, [x20]\n" + "add z26.s, z26.s, z3.s\n" + "add z27.s, z27.s, z2.s\n" + ".inst 0x04a177ff // sqrdmulh z31.s, z31.s, z1.s\n" + ".inst 0x04a17694 // sqrdmulh z20.s, z20.s, z1.s\n" + ".inst 0x04a176b5 // sqrdmulh z21.s, z21.s, z1.s\n" + ".inst 0x04a176d6 // sqrdmulh z22.s, z22.s, z1.s\n" + ".inst 0x04a17610 // sqrdmulh z16.s, z16.s, z1.s\n" + ".inst 0x04a17631 // sqrdmulh z17.s, z17.s, z1.s\n" + ".inst 0x04a17652 // sqrdmulh z18.s, z18.s, z1.s\n" + ".inst 0x04a17673 // sqrdmulh z19.s, z19.s, z1.s\n" + ".inst 0x04a176f7 // sqrdmulh z23.s, z23.s, z1.s\n" + ".inst 0x04a1779c // sqrdmulh z28.s, z28.s, z1.s\n" + ".inst 0x04a177bd // sqrdmulh z29.s, z29.s, z1.s\n" + ".inst 0x04a177de // sqrdmulh z30.s, z30.s, z1.s\n" + ".inst 0x04a17718 // sqrdmulh z24.s, z24.s, z1.s\n" + ".inst 0x04a17739 // sqrdmulh z25.s, z25.s, z1.s\n" + ".inst 0x04a1775a // sqrdmulh z26.s, z26.s, z1.s\n" + ".inst 0x04a1777b // sqrdmulh z27.s, z27.s, z1.s\n" "tbz %x[flags], #5, 55f\n" - "and z4.d, z31.d, z0.d\n" - "and z5.d, z20.d, z0.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "sqadd z31.s, z31.s, z4.s\n" - "sqadd z20.s, z20.s, z5.s\n" - "and z6.d, z21.d, z0.d\n" - "and z7.d, z22.d, z0.d\n" - "and z8.d, z16.d, z0.d\n" - "and z9.d, z17.d, z0.d\n" - "and z10.d, z18.d, z0.d\n" - "and z4.d, z19.d, z0.d\n" - "and z5.d, z23.d, z0.d\n" - "asr z6.s, z6.s, #0x1f\n" + "and z2.d, z31.d, z0.d\n" + "and z1.d, z20.d, z0.d\n" + "asr z2.s, z2.s, #0x1f\n" + "asr z1.s, z1.s, #0x1f\n" + "sqadd z31.s, z31.s, z2.s\n" + "sqadd z20.s, z20.s, z1.s\n" + "and z7.d, z21.d, z0.d\n" + "and z6.d, z22.d, z0.d\n" + "and z5.d, z16.d, z0.d\n" + "and z4.d, z17.d, z0.d\n" + "and z3.d, z18.d, z0.d\n" + "and z2.d, z19.d, z0.d\n" + "and z1.d, z23.d, z0.d\n" "asr z7.s, z7.s, #0x1f\n" - "asr z8.s, z8.s, #0x1f\n" - "asr z9.s, z9.s, #0x1f\n" - "asr z10.s, z10.s, #0x1f\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "sqadd z21.s, z21.s, z6.s\n" - "sqadd z22.s, z22.s, z7.s\n" - "sqadd z16.s, z16.s, z8.s\n" - "sqadd z17.s, z17.s, z9.s\n" - "sqadd z18.s, z18.s, z10.s\n" - "sqadd z19.s, z19.s, z4.s\n" - "sqadd z23.s, z23.s, z5.s\n" - "and z6.d, z28.d, z0.d\n" - "and z7.d, z29.d, z0.d\n" - "and z8.d, z30.d, z0.d\n" - "and z9.d, z24.d, z0.d\n" - "and z10.d, z25.d, z0.d\n" - "and z4.d, z26.d, z0.d\n" - "and z5.d, z27.d, z0.d\n" "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "asr z8.s, z8.s, #0x1f\n" - "asr z9.s, z9.s, #0x1f\n" - "asr z10.s, z10.s, #0x1f\n" + "asr z5.s, z5.s, #0x1f\n" "asr z4.s, z4.s, #0x1f\n" + "asr z3.s, z3.s, #0x1f\n" + "asr z2.s, z2.s, #0x1f\n" + "asr z1.s, z1.s, #0x1f\n" + "sqadd z21.s, z21.s, z7.s\n" + "sqadd z22.s, z22.s, z6.s\n" + "sqadd z16.s, z16.s, z5.s\n" + "sqadd z17.s, z17.s, z4.s\n" + "sqadd z18.s, z18.s, z3.s\n" + "sqadd z19.s, z19.s, z2.s\n" + "sqadd z23.s, z23.s, z1.s\n" + "and z7.d, z28.d, z0.d\n" + "and z6.d, z29.d, z0.d\n" + "and z5.d, z30.d, z0.d\n" + "and z4.d, z24.d, z0.d\n" + "and z3.d, z25.d, z0.d\n" + "and z2.d, z26.d, z0.d\n" + "and z1.d, z27.d, z0.d\n" + "asr z7.s, z7.s, #0x1f\n" + "asr z6.s, z6.s, #0x1f\n" "asr z5.s, z5.s, #0x1f\n" - "sqadd z28.s, z28.s, z6.s\n" - "sqadd z29.s, z29.s, z7.s\n" - "sqadd z30.s, z30.s, z8.s\n" - "sqadd z24.s, z24.s, z9.s\n" - "sqadd z25.s, z25.s, z10.s\n" - "sqadd z26.s, z26.s, z4.s\n" - "sqadd z27.s, z27.s, z5.s\n" + "asr z4.s, z4.s, #0x1f\n" + "asr z3.s, z3.s, #0x1f\n" + "asr z2.s, z2.s, #0x1f\n" + "asr z1.s, z1.s, #0x1f\n" + "sqadd z28.s, z28.s, z7.s\n" + "sqadd z29.s, z29.s, z6.s\n" + "sqadd z30.s, z30.s, z5.s\n" + "sqadd z24.s, z24.s, z4.s\n" + "sqadd z25.s, z25.s, z3.s\n" + "sqadd z26.s, z26.s, z2.s\n" + "sqadd z27.s, z27.s, z1.s\n" "55:" // Height 4: no shift correction - "add x23, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x23]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1rw { z2.s }, p2/Z, [x20]\n" ".inst 0x4482881f // srshl z31.s, p2/M, z31.s, z0.s\n" - "add z31.s, z31.s, z4.s\n" + "add z31.s, z31.s, z2.s\n" ".inst 0x44828814 // srshl z20.s, p2/M, z20.s, z0.s\n" ".inst 0x44828815 // srshl z21.s, p2/M, z21.s, z0.s\n" - "add z20.s, z20.s, z4.s\n" - "add z21.s, z21.s, z4.s\n" + "add z20.s, z20.s, z2.s\n" + "add z21.s, z21.s, z2.s\n" ".inst 0x44828816 // srshl z22.s, p2/M, z22.s, z0.s\n" ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n" - "add z22.s, z22.s, z4.s\n" - "add z16.s, z16.s, z4.s\n" + "add z22.s, z22.s, z2.s\n" + "add z16.s, z16.s, z2.s\n" ".inst 0x44828811 // srshl z17.s, p2/M, z17.s, z0.s\n" ".inst 0x44828812 // srshl z18.s, p2/M, z18.s, z0.s\n" - "add z17.s, z17.s, z4.s\n" - "add z18.s, z18.s, z4.s\n" + "add z17.s, z17.s, z2.s\n" + "add z18.s, z18.s, z2.s\n" ".inst 0x44828813 // srshl z19.s, p2/M, z19.s, z0.s\n" ".inst 0x44828817 // srshl z23.s, p2/M, z23.s, z0.s\n" - "add z19.s, z19.s, z4.s\n" - "add z23.s, z23.s, z4.s\n" + "add z19.s, z19.s, z2.s\n" + "add z23.s, z23.s, z2.s\n" ".inst 0x4482881c // srshl z28.s, p2/M, z28.s, z0.s\n" ".inst 0x4482881d // srshl z29.s, p2/M, z29.s, z0.s\n" - "add z28.s, z28.s, z4.s\n" - "add z29.s, z29.s, z4.s\n" + "add z28.s, z28.s, z2.s\n" + "add z29.s, z29.s, z2.s\n" ".inst 0x4482881e // srshl z30.s, p2/M, z30.s, z0.s\n" ".inst 0x44828818 // srshl z24.s, p2/M, z24.s, z0.s\n" - "add z30.s, z30.s, z4.s\n" - "add z24.s, z24.s, z4.s\n" + "add z30.s, z30.s, z2.s\n" + "add z24.s, z24.s, z2.s\n" ".inst 0x44828819 // srshl z25.s, p2/M, z25.s, z0.s\n" ".inst 0x4482881a // srshl z26.s, p2/M, z26.s, z0.s\n" - "add z25.s, z25.s, z4.s\n" - "add z26.s, z26.s, z4.s\n" + "add z25.s, z25.s, z2.s\n" + "add z26.s, z26.s, z2.s\n" ".inst 0x4482881b // srshl z27.s, p2/M, z27.s, z0.s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x23]\n" - "add z27.s, z27.s, z4.s\n" - "add x23, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x23]\n" - "smin z31.s, p2/M, z31.s, z6.s\n" - "smin z20.s, p2/M, z20.s, z6.s\n" - "smin z21.s, p2/M, z21.s, z6.s\n" - "smin z22.s, p2/M, z22.s, z6.s\n" - "smin z16.s, p2/M, z16.s, z6.s\n" - "smin z17.s, p2/M, z17.s, z6.s\n" - "smin z18.s, p2/M, z18.s, z6.s\n" - "smin z19.s, p2/M, z19.s, z6.s\n" - "smin z23.s, p2/M, z23.s, z6.s\n" - "smin z28.s, p2/M, z28.s, z6.s\n" - "smin z29.s, p2/M, z29.s, z6.s\n" - "smin z30.s, p2/M, z30.s, z6.s\n" - "smin z24.s, p2/M, z24.s, z6.s\n" - "smin z25.s, p2/M, z25.s, z6.s\n" - "smin z26.s, p2/M, z26.s, z6.s\n" - "smin z27.s, p2/M, z27.s, z6.s\n" - "smax z31.s, p2/M, z31.s, z5.s\n" - "smax z20.s, p2/M, z20.s, z5.s\n" - "smax z21.s, p2/M, z21.s, z5.s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1rw { z1.s }, p2/Z, [x20]\n" + "add z27.s, z27.s, z2.s\n" + "add x20, %x[qp], %[minval]\n" + "ld1rw { z0.s }, p2/Z, [x20]\n" + "smin z31.s, p2/M, z31.s, z1.s\n" + "smin z20.s, p2/M, z20.s, z1.s\n" + "smin z21.s, p2/M, z21.s, z1.s\n" + "smin z22.s, p2/M, z22.s, z1.s\n" + "smin z16.s, p2/M, z16.s, z1.s\n" + "smin z17.s, p2/M, z17.s, z1.s\n" + "smin z18.s, p2/M, z18.s, z1.s\n" + "smin z19.s, p2/M, z19.s, z1.s\n" + "smin z23.s, p2/M, z23.s, z1.s\n" + "smin z28.s, p2/M, z28.s, z1.s\n" + "smin z29.s, p2/M, z29.s, z1.s\n" + "smin z30.s, p2/M, z30.s, z1.s\n" + "smin z24.s, p2/M, z24.s, z1.s\n" + "smin z25.s, p2/M, z25.s, z1.s\n" + "smin z26.s, p2/M, z26.s, z1.s\n" + "smin z27.s, p2/M, z27.s, z1.s\n" + "smax z31.s, p2/M, z31.s, z0.s\n" + "smax z20.s, p2/M, z20.s, z0.s\n" + "smax z21.s, p2/M, z21.s, z0.s\n" "uzp1 z31.h, z31.h, z20.h\n" - "smax z22.s, p2/M, z22.s, z5.s\n" - "smax z16.s, p2/M, z16.s, z5.s\n" + "smax z22.s, p2/M, z22.s, z0.s\n" + "smax z16.s, p2/M, z16.s, z0.s\n" "uzp1 z20.h, z21.h, z22.h\n" "uzp1 z31.b, z31.b, z20.b\n" - "smax z17.s, p2/M, z17.s, z5.s\n" - "smax z18.s, p2/M, z18.s, z5.s\n" + "smax z17.s, p2/M, z17.s, z0.s\n" + "smax z18.s, p2/M, z18.s, z0.s\n" "uzp1 z16.h, z16.h, z17.h\n" "st1b { z31.b }, p1, [x27]\n" - "smax z19.s, p2/M, z19.s, z5.s\n" - "smax z23.s, p2/M, z23.s, z5.s\n" + "smax z19.s, p2/M, z19.s, z0.s\n" + "smax z23.s, p2/M, z23.s, z0.s\n" "uzp1 z17.h, z18.h, z19.h\n" "uzp1 z16.b, z16.b, z17.b\n" - "smax z28.s, p2/M, z28.s, z5.s\n" - "smax z29.s, p2/M, z29.s, z5.s\n" + "smax z28.s, p2/M, z28.s, z0.s\n" + "smax z29.s, p2/M, z29.s, z0.s\n" "uzp1 z23.h, z23.h, z28.h\n" - "st1b { z16.b }, p1, [x22]\n" - "smax z30.s, p2/M, z30.s, z5.s\n" - "smax z24.s, p2/M, z24.s, z5.s\n" - "uzp1 z28.h, z29.h, z30.h\n" - "uzp1 z23.b, z23.b, z28.b\n" - "smax z25.s, p2/M, z25.s, z5.s\n" - "smax z26.s, p2/M, z26.s, z5.s\n" + "st1b { z16.b }, p1, [x23]\n" + "smax z30.s, p2/M, z30.s, z0.s\n" + "smax z24.s, p2/M, z24.s, z0.s\n" + "uzp1 z16.h, z29.h, z30.h\n" + "uzp1 z23.b, z23.b, z16.b\n" + "smax z25.s, p2/M, z25.s, z0.s\n" + "smax z26.s, p2/M, z26.s, z0.s\n" "uzp1 z24.h, z24.h, z25.h\n" - "st1b { z23.b }, p1, [x21]\n" - "smax z27.s, p2/M, z27.s, z5.s\n" - "uzp1 z25.h, z26.h, z27.h\n" - "uzp1 z24.b, z24.b, z25.b\n" - "st1b { z24.b }, p1, [x20]\n" + "st1b { z23.b }, p1, [x22]\n" + "smax z27.s, p2/M, z27.s, z0.s\n" + "uzp1 z16.h, z26.h, z27.h\n" + "uzp1 z24.b, z24.b, z16.b\n" + "st1b { z24.b }, p1, [x21]\n" "addvl x27, x27, #1\n" "56:" // Height 4: Writeback done "decw x9, ALL, MUL #4\n" @@ -1407,7 +1407,6 @@ void sve_hybrid_s8qa_mmla_4x4VL ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "58:" // Exit - : [M] "+&r" (M), [flags] "+&r" (flags), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp) : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -1415,4 +1414,4 @@ void sve_hybrid_s8qa_mmla_4x4VL ( } } // namespace arm_gemm -#endif // __ARM_FEATURE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_dot_6x4VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_dot_6x4VL.hpp index dad04c81e8a142a38de9bc184f1099ea38d7cc4d..056ae7a616225873f24e9b1f374975cd627213b7 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_dot_6x4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_dot_6x4VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef ARM_COMPUTE_ENABLE_SVE + #include "../std_transforms_sve.hpp" #include "../performance_parameters.hpp" @@ -74,7 +74,6 @@ public: template static inline PerformanceParameters get_performance_parameters(const CPUInfo *ci) { - if (std::is_same::value) { switch (ci->get_cpu_model()) { default: @@ -97,5 +96,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_dot_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_dot_6x4VL/generic.cpp index 1e71806838f0051cb129495bb5edb58775482782..c28717a37ee168f5a92b76097d4c248029cb9a56 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_dot_6x4VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_dot_6x4VL/generic.cpp @@ -113,11 +113,11 @@ void sve_hybrid_s8qs_dot_6x4VL ( "4:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 5f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 6f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -130,101 +130,101 @@ void sve_hybrid_s8qs_dot_6x4VL ( "7:" // Height 1: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" "ld1rqb { z0.b }, p0/Z, [x26]\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "sdot z8.s, z6.b, z0.b[0]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[0]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" - "sdot z11.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p2/Z, [x9, #4, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[1]\n" - "ld1b { z7.b }, p2/Z, [x9, #5, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p2/Z, [x9, #6, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[1]\n" - "ld1b { z7.b }, p2/Z, [x9, #7, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9]\n" + "sdot z8.s, z16.b, z0.b[0]\n" + "ld1b { z16.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z9.s, z16.b, z0.b[0]\n" + "ld1b { z16.b }, p2/Z, [x9, #2, MUL VL]\n" + "sdot z10.s, z16.b, z0.b[0]\n" + "ld1b { z16.b }, p2/Z, [x9, #3, MUL VL]\n" + "sdot z11.s, z16.b, z0.b[0]\n" + "ld1b { z16.b }, p2/Z, [x9, #4, MUL VL]\n" + "sdot z8.s, z16.b, z0.b[1]\n" + "ld1b { z16.b }, p2/Z, [x9, #5, MUL VL]\n" + "sdot z9.s, z16.b, z0.b[1]\n" + "ld1b { z16.b }, p2/Z, [x9, #6, MUL VL]\n" + "sdot z10.s, z16.b, z0.b[1]\n" + "ld1b { z16.b }, p2/Z, [x9, #7, MUL VL]\n" "addvl x9, x9, #16\n" - "sdot z11.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p2/Z, [x9, #-8, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x9, #-7, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[2]\n" - "sdot z9.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p2/Z, [x9, #-6, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x9, #-5, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[2]\n" - "sdot z11.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p2/Z, [x9, #-4, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x9, #-3, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[3]\n" - "sdot z9.s, z7.b, z0.b[3]\n" - "ld1b { z6.b }, p2/Z, [x9, #-2, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x9, #-1, MUL VL]\n" + "sdot z11.s, z16.b, z0.b[1]\n" + "ld1b { z17.b }, p2/Z, [x9, #-8, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #-7, MUL VL]\n" + "sdot z8.s, z17.b, z0.b[2]\n" + "sdot z9.s, z16.b, z0.b[2]\n" + "ld1b { z17.b }, p2/Z, [x9, #-6, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #-5, MUL VL]\n" + "sdot z10.s, z17.b, z0.b[2]\n" + "sdot z11.s, z16.b, z0.b[2]\n" + "ld1b { z17.b }, p2/Z, [x9, #-4, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #-3, MUL VL]\n" + "sdot z8.s, z17.b, z0.b[3]\n" + "sdot z9.s, z16.b, z0.b[3]\n" + "ld1b { z17.b }, p2/Z, [x9, #-2, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #-1, MUL VL]\n" "sub x27, x27, #0x10\n" "cmp x27, #0x10\n" - "sdot z10.s, z6.b, z0.b[3]\n" - "sdot z11.s, z7.b, z0.b[3]\n" + "sdot z10.s, z17.b, z0.b[3]\n" + "sdot z11.s, z16.b, z0.b[3]\n" "add x26, x26, #0x10\n" "bgt 7b\n" "8:" // Height 1: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" "ld1rqb { z0.b }, p0/Z, [x26]\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "sdot z8.s, z6.b, z0.b[0]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9]\n" + "sdot z8.s, z16.b, z0.b[0]\n" + "ld1b { z16.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z9.s, z16.b, z0.b[0]\n" + "ld1b { z17.b }, p2/Z, [x9, #2, MUL VL]\n" "subs x27, x27, #0x4\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[0]\n" - "sdot z11.s, z7.b, z0.b[0]\n" + "ld1b { z16.b }, p2/Z, [x9, #3, MUL VL]\n" + "sdot z10.s, z17.b, z0.b[0]\n" + "sdot z11.s, z16.b, z0.b[0]\n" "addvl x9, x9, #4\n" "ble 9f\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[1]\n" - "sdot z9.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" + "ld1b { z17.b }, p2/Z, [x9]\n" + "ld1b { z16.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z8.s, z17.b, z0.b[1]\n" + "sdot z9.s, z16.b, z0.b[1]\n" + "ld1b { z17.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #3, MUL VL]\n" "subs x27, x27, #0x4\n" - "sdot z10.s, z6.b, z0.b[1]\n" - "sdot z11.s, z7.b, z0.b[1]\n" + "sdot z10.s, z17.b, z0.b[1]\n" + "sdot z11.s, z16.b, z0.b[1]\n" "addvl x9, x9, #4\n" "ble 9f\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[2]\n" - "sdot z9.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" + "ld1b { z17.b }, p2/Z, [x9]\n" + "ld1b { z16.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z8.s, z17.b, z0.b[2]\n" + "sdot z9.s, z16.b, z0.b[2]\n" + "ld1b { z17.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #3, MUL VL]\n" "subs x27, x27, #0x4\n" - "sdot z10.s, z6.b, z0.b[2]\n" - "sdot z11.s, z7.b, z0.b[2]\n" + "sdot z10.s, z17.b, z0.b[2]\n" + "sdot z11.s, z16.b, z0.b[2]\n" "addvl x9, x9, #4\n" "ble 9f\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[3]\n" - "sdot z9.s, z7.b, z0.b[3]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[3]\n" - "sdot z11.s, z7.b, z0.b[3]\n" + "ld1b { z17.b }, p2/Z, [x9]\n" + "ld1b { z16.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z8.s, z17.b, z0.b[3]\n" + "sdot z9.s, z16.b, z0.b[3]\n" + "ld1b { z17.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #3, MUL VL]\n" + "sdot z10.s, z17.b, z0.b[3]\n" + "sdot z11.s, z16.b, z0.b[3]\n" "addvl x9, x9, #4\n" "9:" // Height 1: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 4b\n" - "ld1w { z0.s }, p2/Z, [x14]\n" - "ld1w { z1.s }, p2/Z, [x14, #1, MUL VL]\n" - "add z8.s, z8.s, z0.s\n" - "add z9.s, z9.s, z1.s\n" - "ld1w { z2.s }, p2/Z, [x14, #2, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x14, #3, MUL VL]\n" - "add z10.s, z10.s, z2.s\n" - "add z11.s, z11.s, z3.s\n" + "ld1w { z17.s }, p2/Z, [x14]\n" + "ld1w { z16.s }, p2/Z, [x14, #1, MUL VL]\n" + "add z8.s, z8.s, z17.s\n" + "add z9.s, z9.s, z16.s\n" + "ld1w { z17.s }, p2/Z, [x14, #2, MUL VL]\n" + "ld1w { z16.s }, p2/Z, [x14, #3, MUL VL]\n" + "add z10.s, z10.s, z17.s\n" + "add z11.s, z11.s, z16.s\n" "addvl x14, x14, #4\n" "tbz %x[flags], #4, 10f\n" "ld1w { z0.s }, p2/Z, [x12]\n" @@ -239,10 +239,10 @@ void sve_hybrid_s8qs_dot_6x4VL ( "addvl x13, x13, #4\n" "b 11f\n" "10:" // Height 1: per layer parameters - "add x26, %x[qp], %[per_layer_right_shift]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1rw { z0.s }, p2/Z, [x26]\n" - "ld1rw { z4.s }, p2/Z, [x25]\n" + "add x21, %x[qp], %[per_layer_right_shift]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1rw { z0.s }, p2/Z, [x21]\n" + "ld1rw { z4.s }, p2/Z, [x20]\n" "mov z1.d, z0.d\n" "mov z5.d, z4.d\n" "mov z2.d, z0.d\n" @@ -255,44 +255,44 @@ void sve_hybrid_s8qs_dot_6x4VL ( ".inst 0x04a6754a // sqrdmulh z10.s, z10.s, z6.s\n" ".inst 0x04a7756b // sqrdmulh z11.s, z11.s, z7.s\n" "tbz %x[flags], #5, 12f\n" - "and z4.d, z8.d, z0.d\n" - "and z5.d, z9.d, z1.d\n" - "and z6.d, z10.d, z2.d\n" - "and z7.d, z11.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z8.s, z8.s, z4.s\n" - "sqadd z9.s, z9.s, z5.s\n" - "sqadd z10.s, z10.s, z6.s\n" - "sqadd z11.s, z11.s, z7.s\n" + "and z19.d, z8.d, z0.d\n" + "and z18.d, z9.d, z1.d\n" + "and z17.d, z10.d, z2.d\n" + "and z16.d, z11.d, z3.d\n" + "asr z19.s, z19.s, #0x1f\n" + "asr z18.s, z18.s, #0x1f\n" + "asr z17.s, z17.s, #0x1f\n" + "asr z16.s, z16.s, #0x1f\n" + "sqadd z8.s, z8.s, z19.s\n" + "sqadd z9.s, z9.s, z18.s\n" + "sqadd z10.s, z10.s, z17.s\n" + "sqadd z11.s, z11.s, z16.s\n" "12:" // Height 1: no shift correction - "add x25, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x25]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1rw { z16.s }, p2/Z, [x20]\n" ".inst 0x44828808 // srshl z8.s, p2/M, z8.s, z0.s\n" - "add z8.s, z8.s, z4.s\n" + "add z8.s, z8.s, z16.s\n" ".inst 0x44828829 // srshl z9.s, p2/M, z9.s, z1.s\n" ".inst 0x4482884a // srshl z10.s, p2/M, z10.s, z2.s\n" - "add z9.s, z9.s, z4.s\n" - "add z10.s, z10.s, z4.s\n" + "add z9.s, z9.s, z16.s\n" + "add z10.s, z10.s, z16.s\n" ".inst 0x4482886b // srshl z11.s, p2/M, z11.s, z3.s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x25]\n" - "add z11.s, z11.s, z4.s\n" - "add x25, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x25]\n" - "smin z8.s, p2/M, z8.s, z6.s\n" - "smin z9.s, p2/M, z9.s, z6.s\n" - "smin z10.s, p2/M, z10.s, z6.s\n" - "smin z11.s, p2/M, z11.s, z6.s\n" - "smax z8.s, p2/M, z8.s, z5.s\n" - "smax z9.s, p2/M, z9.s, z5.s\n" - "smax z10.s, p2/M, z10.s, z5.s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1rw { z17.s }, p2/Z, [x20]\n" + "add z11.s, z11.s, z16.s\n" + "add x20, %x[qp], %[minval]\n" + "ld1rw { z16.s }, p2/Z, [x20]\n" + "smin z8.s, p2/M, z8.s, z17.s\n" + "smin z9.s, p2/M, z9.s, z17.s\n" + "smin z10.s, p2/M, z10.s, z17.s\n" + "smin z11.s, p2/M, z11.s, z17.s\n" + "smax z8.s, p2/M, z8.s, z16.s\n" + "smax z9.s, p2/M, z9.s, z16.s\n" + "smax z10.s, p2/M, z10.s, z16.s\n" "uzp1 z8.h, z8.h, z9.h\n" - "smax z11.s, p2/M, z11.s, z5.s\n" - "uzp1 z9.h, z10.h, z11.h\n" - "uzp1 z8.b, z8.b, z9.b\n" + "smax z11.s, p2/M, z11.s, z16.s\n" + "uzp1 z16.h, z10.h, z11.h\n" + "uzp1 z8.b, z8.b, z16.b\n" "st1b { z8.b }, p1, [x11]\n" "addvl x11, x11, #1\n" "13:" // Height 1: Writeback done @@ -323,12 +323,12 @@ void sve_hybrid_s8qs_dot_6x4VL ( "17:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 18f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 19f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -336,150 +336,150 @@ void sve_hybrid_s8qs_dot_6x4VL ( "b 19f\n" "18:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" + "add x25, x26, x21\n" "19:" // Height 2: input setup done "cmp x27, #0x10\n" "ble 21f\n" "20:" // Height 2: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z0.b }, p0/Z, [x26]\n" - "ld1rqb { z1.b }, p0/Z, [x25]\n" + "ld1rqb { z1.b }, p0/Z, [x26]\n" + "ld1rqb { z0.b }, p0/Z, [x25]\n" "sub x27, x27, #0x10\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[0]\n" - "sdot z12.s, z6.b, z1.b[0]\n" - "sdot z9.s, z7.b, z0.b[0]\n" - "sdot z13.s, z7.b, z1.b[0]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[0]\n" - "sdot z14.s, z6.b, z1.b[0]\n" - "ld1b { z6.b }, p2/Z, [x9, #4, MUL VL]\n" + "ld1b { z17.b }, p2/Z, [x9]\n" + "ld1b { z16.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z8.s, z17.b, z1.b[0]\n" + "sdot z12.s, z17.b, z0.b[0]\n" + "sdot z9.s, z16.b, z1.b[0]\n" + "sdot z13.s, z16.b, z0.b[0]\n" + "ld1b { z17.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #3, MUL VL]\n" + "sdot z10.s, z17.b, z1.b[0]\n" + "sdot z14.s, z17.b, z0.b[0]\n" + "ld1b { z17.b }, p2/Z, [x9, #4, MUL VL]\n" "cmp x27, #0x10\n" - "sdot z11.s, z7.b, z0.b[0]\n" - "sdot z15.s, z7.b, z1.b[0]\n" - "ld1b { z7.b }, p2/Z, [x9, #5, MUL VL]\n" + "sdot z11.s, z16.b, z1.b[0]\n" + "sdot z15.s, z16.b, z0.b[0]\n" + "ld1b { z16.b }, p2/Z, [x9, #5, MUL VL]\n" "add x26, x26, #0x10\n" - "sdot z8.s, z6.b, z0.b[1]\n" - "sdot z12.s, z6.b, z1.b[1]\n" - "ld1b { z6.b }, p2/Z, [x9, #6, MUL VL]\n" + "sdot z8.s, z17.b, z1.b[1]\n" + "sdot z12.s, z17.b, z0.b[1]\n" + "ld1b { z17.b }, p2/Z, [x9, #6, MUL VL]\n" "add x25, x25, #0x10\n" - "sdot z9.s, z7.b, z0.b[1]\n" - "sdot z13.s, z7.b, z1.b[1]\n" - "ld1b { z7.b }, p2/Z, [x9, #7, MUL VL]\n" + "sdot z9.s, z16.b, z1.b[1]\n" + "sdot z13.s, z16.b, z0.b[1]\n" + "ld1b { z16.b }, p2/Z, [x9, #7, MUL VL]\n" "addvl x9, x9, #16\n" - "sdot z10.s, z6.b, z0.b[1]\n" - "sdot z14.s, z6.b, z1.b[1]\n" - "ld1b { z6.b }, p2/Z, [x9, #-8, MUL VL]\n" - "sdot z11.s, z7.b, z0.b[1]\n" - "sdot z15.s, z7.b, z1.b[1]\n" - "ld1b { z7.b }, p2/Z, [x9, #-7, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[2]\n" - "sdot z12.s, z6.b, z1.b[2]\n" - "ld1b { z6.b }, p2/Z, [x9, #-6, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[2]\n" - "sdot z13.s, z7.b, z1.b[2]\n" - "ld1b { z7.b }, p2/Z, [x9, #-5, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[2]\n" - "sdot z14.s, z6.b, z1.b[2]\n" - "ld1b { z6.b }, p2/Z, [x9, #-4, MUL VL]\n" - "sdot z11.s, z7.b, z0.b[2]\n" - "sdot z15.s, z7.b, z1.b[2]\n" - "ld1b { z7.b }, p2/Z, [x9, #-3, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[3]\n" - "sdot z12.s, z6.b, z1.b[3]\n" - "ld1b { z6.b }, p2/Z, [x9, #-2, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[3]\n" - "sdot z13.s, z7.b, z1.b[3]\n" - "ld1b { z7.b }, p2/Z, [x9, #-1, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[3]\n" - "sdot z14.s, z6.b, z1.b[3]\n" - "sdot z11.s, z7.b, z0.b[3]\n" - "sdot z15.s, z7.b, z1.b[3]\n" + "sdot z10.s, z17.b, z1.b[1]\n" + "sdot z14.s, z17.b, z0.b[1]\n" + "ld1b { z17.b }, p2/Z, [x9, #-8, MUL VL]\n" + "sdot z11.s, z16.b, z1.b[1]\n" + "sdot z15.s, z16.b, z0.b[1]\n" + "ld1b { z16.b }, p2/Z, [x9, #-7, MUL VL]\n" + "sdot z8.s, z17.b, z1.b[2]\n" + "sdot z12.s, z17.b, z0.b[2]\n" + "ld1b { z17.b }, p2/Z, [x9, #-6, MUL VL]\n" + "sdot z9.s, z16.b, z1.b[2]\n" + "sdot z13.s, z16.b, z0.b[2]\n" + "ld1b { z16.b }, p2/Z, [x9, #-5, MUL VL]\n" + "sdot z10.s, z17.b, z1.b[2]\n" + "sdot z14.s, z17.b, z0.b[2]\n" + "ld1b { z17.b }, p2/Z, [x9, #-4, MUL VL]\n" + "sdot z11.s, z16.b, z1.b[2]\n" + "sdot z15.s, z16.b, z0.b[2]\n" + "ld1b { z16.b }, p2/Z, [x9, #-3, MUL VL]\n" + "sdot z8.s, z17.b, z1.b[3]\n" + "sdot z12.s, z17.b, z0.b[3]\n" + "ld1b { z17.b }, p2/Z, [x9, #-2, MUL VL]\n" + "sdot z9.s, z16.b, z1.b[3]\n" + "sdot z13.s, z16.b, z0.b[3]\n" + "ld1b { z16.b }, p2/Z, [x9, #-1, MUL VL]\n" + "sdot z10.s, z17.b, z1.b[3]\n" + "sdot z14.s, z17.b, z0.b[3]\n" + "sdot z11.s, z16.b, z1.b[3]\n" + "sdot z15.s, z16.b, z0.b[3]\n" "bgt 20b\n" "21:" // Height 2: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" "ld1rqb { z0.b }, p0/Z, [x26]\n" "ld1rqb { z1.b }, p0/Z, [x25]\n" "subs x27, x27, #0x4\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[0]\n" - "sdot z12.s, z6.b, z1.b[0]\n" - "sdot z9.s, z7.b, z0.b[0]\n" - "sdot z13.s, z7.b, z1.b[0]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[0]\n" - "sdot z14.s, z6.b, z1.b[0]\n" + "ld1b { z17.b }, p2/Z, [x9]\n" + "ld1b { z16.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z8.s, z17.b, z0.b[0]\n" + "sdot z12.s, z17.b, z1.b[0]\n" + "sdot z9.s, z16.b, z0.b[0]\n" + "sdot z13.s, z16.b, z1.b[0]\n" + "ld1b { z17.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #3, MUL VL]\n" + "sdot z10.s, z17.b, z0.b[0]\n" + "sdot z14.s, z17.b, z1.b[0]\n" "addvl x9, x9, #4\n" - "sdot z11.s, z7.b, z0.b[0]\n" - "sdot z15.s, z7.b, z1.b[0]\n" + "sdot z11.s, z16.b, z0.b[0]\n" + "sdot z15.s, z16.b, z1.b[0]\n" "ble 22f\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[1]\n" - "sdot z12.s, z6.b, z1.b[1]\n" - "sdot z9.s, z7.b, z0.b[1]\n" - "sdot z13.s, z7.b, z1.b[1]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" + "ld1b { z17.b }, p2/Z, [x9]\n" + "ld1b { z16.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z8.s, z17.b, z0.b[1]\n" + "sdot z12.s, z17.b, z1.b[1]\n" + "sdot z9.s, z16.b, z0.b[1]\n" + "sdot z13.s, z16.b, z1.b[1]\n" + "ld1b { z17.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #3, MUL VL]\n" "subs x27, x27, #0x4\n" - "sdot z10.s, z6.b, z0.b[1]\n" - "sdot z14.s, z6.b, z1.b[1]\n" + "sdot z10.s, z17.b, z0.b[1]\n" + "sdot z14.s, z17.b, z1.b[1]\n" "addvl x9, x9, #4\n" - "sdot z11.s, z7.b, z0.b[1]\n" - "sdot z15.s, z7.b, z1.b[1]\n" + "sdot z11.s, z16.b, z0.b[1]\n" + "sdot z15.s, z16.b, z1.b[1]\n" "ble 22f\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[2]\n" - "sdot z12.s, z6.b, z1.b[2]\n" - "sdot z9.s, z7.b, z0.b[2]\n" - "sdot z13.s, z7.b, z1.b[2]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" + "ld1b { z17.b }, p2/Z, [x9]\n" + "ld1b { z16.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z8.s, z17.b, z0.b[2]\n" + "sdot z12.s, z17.b, z1.b[2]\n" + "sdot z9.s, z16.b, z0.b[2]\n" + "sdot z13.s, z16.b, z1.b[2]\n" + "ld1b { z17.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #3, MUL VL]\n" "subs x27, x27, #0x4\n" - "sdot z10.s, z6.b, z0.b[2]\n" - "sdot z14.s, z6.b, z1.b[2]\n" + "sdot z10.s, z17.b, z0.b[2]\n" + "sdot z14.s, z17.b, z1.b[2]\n" "addvl x9, x9, #4\n" - "sdot z11.s, z7.b, z0.b[2]\n" - "sdot z15.s, z7.b, z1.b[2]\n" + "sdot z11.s, z16.b, z0.b[2]\n" + "sdot z15.s, z16.b, z1.b[2]\n" "ble 22f\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[3]\n" - "sdot z12.s, z6.b, z1.b[3]\n" - "sdot z9.s, z7.b, z0.b[3]\n" - "sdot z13.s, z7.b, z1.b[3]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[3]\n" - "sdot z14.s, z6.b, z1.b[3]\n" + "ld1b { z17.b }, p2/Z, [x9]\n" + "ld1b { z16.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z8.s, z17.b, z0.b[3]\n" + "sdot z12.s, z17.b, z1.b[3]\n" + "sdot z9.s, z16.b, z0.b[3]\n" + "sdot z13.s, z16.b, z1.b[3]\n" + "ld1b { z17.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #3, MUL VL]\n" + "sdot z10.s, z17.b, z0.b[3]\n" + "sdot z14.s, z17.b, z1.b[3]\n" "addvl x9, x9, #4\n" - "sdot z11.s, z7.b, z0.b[3]\n" - "sdot z15.s, z7.b, z1.b[3]\n" + "sdot z11.s, z16.b, z0.b[3]\n" + "sdot z15.s, z16.b, z1.b[3]\n" "22:" // Height 2: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 17b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "ld1w { z0.s }, p2/Z, [x14]\n" - "add x24, x11, x20\n" - "add z8.s, z8.s, z0.s\n" - "ld1w { z1.s }, p2/Z, [x14, #1, MUL VL]\n" - "ld1w { z2.s }, p2/Z, [x14, #2, MUL VL]\n" - "add z9.s, z9.s, z1.s\n" - "add z10.s, z10.s, z2.s\n" - "ld1w { z3.s }, p2/Z, [x14, #3, MUL VL]\n" - "add z11.s, z11.s, z3.s\n" - "add z12.s, z12.s, z0.s\n" + "ld1w { z19.s }, p2/Z, [x14]\n" + "add x26, x11, x20\n" + "add z8.s, z8.s, z19.s\n" + "ld1w { z18.s }, p2/Z, [x14, #1, MUL VL]\n" + "ld1w { z17.s }, p2/Z, [x14, #2, MUL VL]\n" + "add z9.s, z9.s, z18.s\n" + "add z10.s, z10.s, z17.s\n" + "ld1w { z16.s }, p2/Z, [x14, #3, MUL VL]\n" + "add z11.s, z11.s, z16.s\n" + "add z12.s, z12.s, z19.s\n" "addvl x14, x14, #4\n" - "add z13.s, z13.s, z1.s\n" - "add z14.s, z14.s, z2.s\n" - "add z15.s, z15.s, z3.s\n" + "add z13.s, z13.s, z18.s\n" + "add z14.s, z14.s, z17.s\n" + "add z15.s, z15.s, z16.s\n" "tbz %x[flags], #4, 23f\n" "ld1w { z0.s }, p2/Z, [x12]\n" "ld1w { z4.s }, p2/Z, [x13]\n" @@ -493,10 +493,10 @@ void sve_hybrid_s8qs_dot_6x4VL ( "addvl x13, x13, #4\n" "b 24f\n" "23:" // Height 2: per layer parameters - "add x26, %x[qp], %[per_layer_right_shift]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1rw { z0.s }, p2/Z, [x26]\n" - "ld1rw { z4.s }, p2/Z, [x25]\n" + "add x21, %x[qp], %[per_layer_right_shift]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1rw { z0.s }, p2/Z, [x21]\n" + "ld1rw { z4.s }, p2/Z, [x20]\n" "mov z1.d, z0.d\n" "mov z5.d, z4.d\n" "mov z2.d, z0.d\n" @@ -513,77 +513,77 @@ void sve_hybrid_s8qs_dot_6x4VL ( ".inst 0x04a675ce // sqrdmulh z14.s, z14.s, z6.s\n" ".inst 0x04a775ef // sqrdmulh z15.s, z15.s, z7.s\n" "tbz %x[flags], #5, 25f\n" - "and z4.d, z8.d, z0.d\n" - "and z5.d, z9.d, z1.d\n" - "and z6.d, z10.d, z2.d\n" - "and z7.d, z11.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z8.s, z8.s, z4.s\n" - "sqadd z9.s, z9.s, z5.s\n" - "sqadd z10.s, z10.s, z6.s\n" - "sqadd z11.s, z11.s, z7.s\n" - "and z4.d, z12.d, z0.d\n" - "and z5.d, z13.d, z1.d\n" - "and z6.d, z14.d, z2.d\n" - "and z7.d, z15.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z12.s, z12.s, z4.s\n" - "sqadd z13.s, z13.s, z5.s\n" - "sqadd z14.s, z14.s, z6.s\n" - "sqadd z15.s, z15.s, z7.s\n" + "and z19.d, z8.d, z0.d\n" + "and z18.d, z9.d, z1.d\n" + "and z17.d, z10.d, z2.d\n" + "and z16.d, z11.d, z3.d\n" + "asr z19.s, z19.s, #0x1f\n" + "asr z18.s, z18.s, #0x1f\n" + "asr z17.s, z17.s, #0x1f\n" + "asr z16.s, z16.s, #0x1f\n" + "sqadd z8.s, z8.s, z19.s\n" + "sqadd z9.s, z9.s, z18.s\n" + "sqadd z10.s, z10.s, z17.s\n" + "sqadd z11.s, z11.s, z16.s\n" + "and z19.d, z12.d, z0.d\n" + "and z18.d, z13.d, z1.d\n" + "and z17.d, z14.d, z2.d\n" + "and z16.d, z15.d, z3.d\n" + "asr z19.s, z19.s, #0x1f\n" + "asr z18.s, z18.s, #0x1f\n" + "asr z17.s, z17.s, #0x1f\n" + "asr z16.s, z16.s, #0x1f\n" + "sqadd z12.s, z12.s, z19.s\n" + "sqadd z13.s, z13.s, z18.s\n" + "sqadd z14.s, z14.s, z17.s\n" + "sqadd z15.s, z15.s, z16.s\n" "25:" // Height 2: no shift correction - "add x25, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x25]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1rw { z17.s }, p2/Z, [x20]\n" ".inst 0x44828808 // srshl z8.s, p2/M, z8.s, z0.s\n" - "add z8.s, z8.s, z4.s\n" + "add z8.s, z8.s, z17.s\n" ".inst 0x44828829 // srshl z9.s, p2/M, z9.s, z1.s\n" ".inst 0x4482884a // srshl z10.s, p2/M, z10.s, z2.s\n" - "add z9.s, z9.s, z4.s\n" - "add z10.s, z10.s, z4.s\n" + "add z9.s, z9.s, z17.s\n" + "add z10.s, z10.s, z17.s\n" ".inst 0x4482886b // srshl z11.s, p2/M, z11.s, z3.s\n" ".inst 0x4482880c // srshl z12.s, p2/M, z12.s, z0.s\n" - "add z11.s, z11.s, z4.s\n" - "add z12.s, z12.s, z4.s\n" + "add z11.s, z11.s, z17.s\n" + "add z12.s, z12.s, z17.s\n" ".inst 0x4482882d // srshl z13.s, p2/M, z13.s, z1.s\n" ".inst 0x4482884e // srshl z14.s, p2/M, z14.s, z2.s\n" - "add z13.s, z13.s, z4.s\n" - "add z14.s, z14.s, z4.s\n" + "add z13.s, z13.s, z17.s\n" + "add z14.s, z14.s, z17.s\n" ".inst 0x4482886f // srshl z15.s, p2/M, z15.s, z3.s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x25]\n" - "add z15.s, z15.s, z4.s\n" - "add x25, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x25]\n" - "smin z8.s, p2/M, z8.s, z6.s\n" - "smin z9.s, p2/M, z9.s, z6.s\n" - "smin z10.s, p2/M, z10.s, z6.s\n" - "smin z11.s, p2/M, z11.s, z6.s\n" - "smin z12.s, p2/M, z12.s, z6.s\n" - "smin z13.s, p2/M, z13.s, z6.s\n" - "smin z14.s, p2/M, z14.s, z6.s\n" - "smin z15.s, p2/M, z15.s, z6.s\n" - "smax z8.s, p2/M, z8.s, z5.s\n" - "smax z9.s, p2/M, z9.s, z5.s\n" - "smax z10.s, p2/M, z10.s, z5.s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1rw { z16.s }, p2/Z, [x20]\n" + "add z15.s, z15.s, z17.s\n" + "add x20, %x[qp], %[minval]\n" + "ld1rw { z17.s }, p2/Z, [x20]\n" + "smin z8.s, p2/M, z8.s, z16.s\n" + "smin z9.s, p2/M, z9.s, z16.s\n" + "smin z10.s, p2/M, z10.s, z16.s\n" + "smin z11.s, p2/M, z11.s, z16.s\n" + "smin z12.s, p2/M, z12.s, z16.s\n" + "smin z13.s, p2/M, z13.s, z16.s\n" + "smin z14.s, p2/M, z14.s, z16.s\n" + "smin z15.s, p2/M, z15.s, z16.s\n" + "smax z8.s, p2/M, z8.s, z17.s\n" + "smax z9.s, p2/M, z9.s, z17.s\n" + "smax z10.s, p2/M, z10.s, z17.s\n" "uzp1 z8.h, z8.h, z9.h\n" - "smax z11.s, p2/M, z11.s, z5.s\n" - "smax z12.s, p2/M, z12.s, z5.s\n" - "uzp1 z9.h, z10.h, z11.h\n" - "uzp1 z8.b, z8.b, z9.b\n" - "smax z13.s, p2/M, z13.s, z5.s\n" - "smax z14.s, p2/M, z14.s, z5.s\n" + "smax z11.s, p2/M, z11.s, z17.s\n" + "smax z12.s, p2/M, z12.s, z17.s\n" + "uzp1 z16.h, z10.h, z11.h\n" + "uzp1 z8.b, z8.b, z16.b\n" + "smax z13.s, p2/M, z13.s, z17.s\n" + "smax z14.s, p2/M, z14.s, z17.s\n" "uzp1 z12.h, z12.h, z13.h\n" "st1b { z8.b }, p1, [x11]\n" - "smax z15.s, p2/M, z15.s, z5.s\n" - "uzp1 z13.h, z14.h, z15.h\n" - "uzp1 z12.b, z12.b, z13.b\n" - "st1b { z12.b }, p1, [x24]\n" + "smax z15.s, p2/M, z15.s, z17.s\n" + "uzp1 z16.h, z14.h, z15.h\n" + "uzp1 z12.b, z12.b, z16.b\n" + "st1b { z12.b }, p1, [x26]\n" "addvl x11, x11, #1\n" "26:" // Height 2: Writeback done "decw x10, ALL, MUL #4\n" @@ -617,13 +617,13 @@ void sve_hybrid_s8qs_dot_6x4VL ( "30:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 31f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 32f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -632,86 +632,86 @@ void sve_hybrid_s8qs_dot_6x4VL ( "b 32f\n" "31:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" "32:" // Height 3: input setup done "cmp x27, #0x10\n" "ble 34f\n" "33:" // Height 3: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z0.b }, p0/Z, [x26]\n" + "ld1rqb { z2.b }, p0/Z, [x26]\n" "ld1rqb { z1.b }, p0/Z, [x25]\n" "sub x27, x27, #0x10\n" - "ld1rqb { z2.b }, p0/Z, [x24]\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "sdot z8.s, z6.b, z0.b[0]\n" - "sdot z12.s, z6.b, z1.b[0]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z16.s, z6.b, z2.b[0]\n" - "sdot z9.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[0]\n" - "sdot z17.s, z7.b, z2.b[0]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" + "ld1rqb { z0.b }, p0/Z, [x24]\n" + "ld1b { z21.b }, p2/Z, [x9]\n" + "sdot z8.s, z21.b, z2.b[0]\n" + "sdot z12.s, z21.b, z1.b[0]\n" + "ld1b { z20.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z16.s, z21.b, z0.b[0]\n" + "sdot z9.s, z20.b, z2.b[0]\n" + "ld1b { z21.b }, p2/Z, [x9, #2, MUL VL]\n" + "sdot z13.s, z20.b, z1.b[0]\n" + "sdot z17.s, z20.b, z0.b[0]\n" + "ld1b { z20.b }, p2/Z, [x9, #3, MUL VL]\n" "cmp x27, #0x10\n" - "sdot z10.s, z6.b, z0.b[0]\n" - "sdot z14.s, z6.b, z1.b[0]\n" + "sdot z10.s, z21.b, z2.b[0]\n" + "sdot z14.s, z21.b, z1.b[0]\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" - "sdot z18.s, z6.b, z2.b[0]\n" - "sdot z11.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p2/Z, [x9, #4, MUL VL]\n" + "sdot z18.s, z21.b, z0.b[0]\n" + "sdot z11.s, z20.b, z2.b[0]\n" + "ld1b { z21.b }, p2/Z, [x9, #4, MUL VL]\n" "add x24, x24, #0x10\n" - "sdot z15.s, z7.b, z1.b[0]\n" - "sdot z19.s, z7.b, z2.b[0]\n" - "ld1b { z7.b }, p2/Z, [x9, #5, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[1]\n" - "sdot z12.s, z6.b, z1.b[1]\n" - "sdot z16.s, z6.b, z2.b[1]\n" - "sdot z9.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p2/Z, [x9, #6, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[1]\n" - "sdot z17.s, z7.b, z2.b[1]\n" - "ld1b { z7.b }, p2/Z, [x9, #7, MUL VL]\n" + "sdot z15.s, z20.b, z1.b[0]\n" + "sdot z19.s, z20.b, z0.b[0]\n" + "ld1b { z20.b }, p2/Z, [x9, #5, MUL VL]\n" + "sdot z8.s, z21.b, z2.b[1]\n" + "sdot z12.s, z21.b, z1.b[1]\n" + "sdot z16.s, z21.b, z0.b[1]\n" + "sdot z9.s, z20.b, z2.b[1]\n" + "ld1b { z21.b }, p2/Z, [x9, #6, MUL VL]\n" + "sdot z13.s, z20.b, z1.b[1]\n" + "sdot z17.s, z20.b, z0.b[1]\n" + "ld1b { z20.b }, p2/Z, [x9, #7, MUL VL]\n" "addvl x9, x9, #16\n" - "sdot z10.s, z6.b, z0.b[1]\n" - "sdot z14.s, z6.b, z1.b[1]\n" - "sdot z18.s, z6.b, z2.b[1]\n" - "sdot z11.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p2/Z, [x9, #-8, MUL VL]\n" - "sdot z15.s, z7.b, z1.b[1]\n" - "sdot z19.s, z7.b, z2.b[1]\n" - "ld1b { z7.b }, p2/Z, [x9, #-7, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[2]\n" - "sdot z12.s, z6.b, z1.b[2]\n" - "sdot z16.s, z6.b, z2.b[2]\n" - "sdot z9.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p2/Z, [x9, #-6, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[2]\n" - "sdot z17.s, z7.b, z2.b[2]\n" - "ld1b { z7.b }, p2/Z, [x9, #-5, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[2]\n" - "sdot z14.s, z6.b, z1.b[2]\n" - "sdot z18.s, z6.b, z2.b[2]\n" - "sdot z11.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p2/Z, [x9, #-4, MUL VL]\n" - "sdot z15.s, z7.b, z1.b[2]\n" - "sdot z19.s, z7.b, z2.b[2]\n" - "ld1b { z7.b }, p2/Z, [x9, #-3, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[3]\n" - "sdot z12.s, z6.b, z1.b[3]\n" - "sdot z16.s, z6.b, z2.b[3]\n" - "sdot z9.s, z7.b, z0.b[3]\n" - "ld1b { z6.b }, p2/Z, [x9, #-2, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[3]\n" - "sdot z17.s, z7.b, z2.b[3]\n" - "ld1b { z7.b }, p2/Z, [x9, #-1, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[3]\n" - "sdot z14.s, z6.b, z1.b[3]\n" - "sdot z18.s, z6.b, z2.b[3]\n" - "sdot z11.s, z7.b, z0.b[3]\n" - "sdot z15.s, z7.b, z1.b[3]\n" - "sdot z19.s, z7.b, z2.b[3]\n" + "sdot z10.s, z21.b, z2.b[1]\n" + "sdot z14.s, z21.b, z1.b[1]\n" + "sdot z18.s, z21.b, z0.b[1]\n" + "sdot z11.s, z20.b, z2.b[1]\n" + "ld1b { z21.b }, p2/Z, [x9, #-8, MUL VL]\n" + "sdot z15.s, z20.b, z1.b[1]\n" + "sdot z19.s, z20.b, z0.b[1]\n" + "ld1b { z20.b }, p2/Z, [x9, #-7, MUL VL]\n" + "sdot z8.s, z21.b, z2.b[2]\n" + "sdot z12.s, z21.b, z1.b[2]\n" + "sdot z16.s, z21.b, z0.b[2]\n" + "sdot z9.s, z20.b, z2.b[2]\n" + "ld1b { z21.b }, p2/Z, [x9, #-6, MUL VL]\n" + "sdot z13.s, z20.b, z1.b[2]\n" + "sdot z17.s, z20.b, z0.b[2]\n" + "ld1b { z20.b }, p2/Z, [x9, #-5, MUL VL]\n" + "sdot z10.s, z21.b, z2.b[2]\n" + "sdot z14.s, z21.b, z1.b[2]\n" + "sdot z18.s, z21.b, z0.b[2]\n" + "sdot z11.s, z20.b, z2.b[2]\n" + "ld1b { z21.b }, p2/Z, [x9, #-4, MUL VL]\n" + "sdot z15.s, z20.b, z1.b[2]\n" + "sdot z19.s, z20.b, z0.b[2]\n" + "ld1b { z20.b }, p2/Z, [x9, #-3, MUL VL]\n" + "sdot z8.s, z21.b, z2.b[3]\n" + "sdot z12.s, z21.b, z1.b[3]\n" + "sdot z16.s, z21.b, z0.b[3]\n" + "sdot z9.s, z20.b, z2.b[3]\n" + "ld1b { z21.b }, p2/Z, [x9, #-2, MUL VL]\n" + "sdot z13.s, z20.b, z1.b[3]\n" + "sdot z17.s, z20.b, z0.b[3]\n" + "ld1b { z20.b }, p2/Z, [x9, #-1, MUL VL]\n" + "sdot z10.s, z21.b, z2.b[3]\n" + "sdot z14.s, z21.b, z1.b[3]\n" + "sdot z18.s, z21.b, z0.b[3]\n" + "sdot z11.s, z20.b, z2.b[3]\n" + "sdot z15.s, z20.b, z1.b[3]\n" + "sdot z19.s, z20.b, z0.b[3]\n" "bgt 33b\n" "34:" // Height 3: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" @@ -719,104 +719,104 @@ void sve_hybrid_s8qs_dot_6x4VL ( "ld1rqb { z1.b }, p0/Z, [x25]\n" "subs x27, x27, #0x4\n" "ld1rqb { z2.b }, p0/Z, [x24]\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "sdot z8.s, z6.b, z0.b[0]\n" - "sdot z12.s, z6.b, z1.b[0]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z16.s, z6.b, z2.b[0]\n" - "sdot z9.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[0]\n" - "sdot z17.s, z7.b, z2.b[0]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" + "ld1b { z21.b }, p2/Z, [x9]\n" + "sdot z8.s, z21.b, z0.b[0]\n" + "sdot z12.s, z21.b, z1.b[0]\n" + "ld1b { z20.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z16.s, z21.b, z2.b[0]\n" + "sdot z9.s, z20.b, z0.b[0]\n" + "ld1b { z21.b }, p2/Z, [x9, #2, MUL VL]\n" + "sdot z13.s, z20.b, z1.b[0]\n" + "sdot z17.s, z20.b, z2.b[0]\n" + "ld1b { z20.b }, p2/Z, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "sdot z10.s, z6.b, z0.b[0]\n" - "sdot z14.s, z6.b, z1.b[0]\n" - "sdot z18.s, z6.b, z2.b[0]\n" - "sdot z11.s, z7.b, z0.b[0]\n" - "sdot z15.s, z7.b, z1.b[0]\n" - "sdot z19.s, z7.b, z2.b[0]\n" + "sdot z10.s, z21.b, z0.b[0]\n" + "sdot z14.s, z21.b, z1.b[0]\n" + "sdot z18.s, z21.b, z2.b[0]\n" + "sdot z11.s, z20.b, z0.b[0]\n" + "sdot z15.s, z20.b, z1.b[0]\n" + "sdot z19.s, z20.b, z2.b[0]\n" "ble 35f\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[1]\n" - "sdot z12.s, z6.b, z1.b[1]\n" - "sdot z16.s, z6.b, z2.b[1]\n" - "sdot z9.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z21.b }, p2/Z, [x9]\n" + "ld1b { z20.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z8.s, z21.b, z0.b[1]\n" + "sdot z12.s, z21.b, z1.b[1]\n" + "sdot z16.s, z21.b, z2.b[1]\n" + "sdot z9.s, z20.b, z0.b[1]\n" + "ld1b { z21.b }, p2/Z, [x9, #2, MUL VL]\n" "subs x27, x27, #0x4\n" - "sdot z13.s, z7.b, z1.b[1]\n" - "sdot z17.s, z7.b, z2.b[1]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" + "sdot z13.s, z20.b, z1.b[1]\n" + "sdot z17.s, z20.b, z2.b[1]\n" + "ld1b { z20.b }, p2/Z, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "sdot z10.s, z6.b, z0.b[1]\n" - "sdot z14.s, z6.b, z1.b[1]\n" - "sdot z18.s, z6.b, z2.b[1]\n" - "sdot z11.s, z7.b, z0.b[1]\n" - "sdot z15.s, z7.b, z1.b[1]\n" - "sdot z19.s, z7.b, z2.b[1]\n" + "sdot z10.s, z21.b, z0.b[1]\n" + "sdot z14.s, z21.b, z1.b[1]\n" + "sdot z18.s, z21.b, z2.b[1]\n" + "sdot z11.s, z20.b, z0.b[1]\n" + "sdot z15.s, z20.b, z1.b[1]\n" + "sdot z19.s, z20.b, z2.b[1]\n" "ble 35f\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[2]\n" - "sdot z12.s, z6.b, z1.b[2]\n" - "sdot z16.s, z6.b, z2.b[2]\n" - "sdot z9.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z21.b }, p2/Z, [x9]\n" + "ld1b { z20.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z8.s, z21.b, z0.b[2]\n" + "sdot z12.s, z21.b, z1.b[2]\n" + "sdot z16.s, z21.b, z2.b[2]\n" + "sdot z9.s, z20.b, z0.b[2]\n" + "ld1b { z21.b }, p2/Z, [x9, #2, MUL VL]\n" "subs x27, x27, #0x4\n" - "sdot z13.s, z7.b, z1.b[2]\n" - "sdot z17.s, z7.b, z2.b[2]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" + "sdot z13.s, z20.b, z1.b[2]\n" + "sdot z17.s, z20.b, z2.b[2]\n" + "ld1b { z20.b }, p2/Z, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "sdot z10.s, z6.b, z0.b[2]\n" - "sdot z14.s, z6.b, z1.b[2]\n" - "sdot z18.s, z6.b, z2.b[2]\n" - "sdot z11.s, z7.b, z0.b[2]\n" - "sdot z15.s, z7.b, z1.b[2]\n" - "sdot z19.s, z7.b, z2.b[2]\n" + "sdot z10.s, z21.b, z0.b[2]\n" + "sdot z14.s, z21.b, z1.b[2]\n" + "sdot z18.s, z21.b, z2.b[2]\n" + "sdot z11.s, z20.b, z0.b[2]\n" + "sdot z15.s, z20.b, z1.b[2]\n" + "sdot z19.s, z20.b, z2.b[2]\n" "ble 35f\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[3]\n" - "sdot z12.s, z6.b, z1.b[3]\n" - "sdot z16.s, z6.b, z2.b[3]\n" - "sdot z9.s, z7.b, z0.b[3]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[3]\n" - "sdot z17.s, z7.b, z2.b[3]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" + "ld1b { z21.b }, p2/Z, [x9]\n" + "ld1b { z20.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z8.s, z21.b, z0.b[3]\n" + "sdot z12.s, z21.b, z1.b[3]\n" + "sdot z16.s, z21.b, z2.b[3]\n" + "sdot z9.s, z20.b, z0.b[3]\n" + "ld1b { z21.b }, p2/Z, [x9, #2, MUL VL]\n" + "sdot z13.s, z20.b, z1.b[3]\n" + "sdot z17.s, z20.b, z2.b[3]\n" + "ld1b { z20.b }, p2/Z, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "sdot z10.s, z6.b, z0.b[3]\n" - "sdot z14.s, z6.b, z1.b[3]\n" - "sdot z18.s, z6.b, z2.b[3]\n" - "sdot z11.s, z7.b, z0.b[3]\n" - "sdot z15.s, z7.b, z1.b[3]\n" - "sdot z19.s, z7.b, z2.b[3]\n" + "sdot z10.s, z21.b, z0.b[3]\n" + "sdot z14.s, z21.b, z1.b[3]\n" + "sdot z18.s, z21.b, z2.b[3]\n" + "sdot z11.s, z20.b, z0.b[3]\n" + "sdot z15.s, z20.b, z1.b[3]\n" + "sdot z19.s, z20.b, z2.b[3]\n" "35:" // Height 3: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 30b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "ld1w { z0.s }, p2/Z, [x14]\n" - "add x24, x11, x20\n" - "add x23, x24, x20\n" - "ld1w { z1.s }, p2/Z, [x14, #1, MUL VL]\n" - "ld1w { z2.s }, p2/Z, [x14, #2, MUL VL]\n" - "add z8.s, z8.s, z0.s\n" - "add z9.s, z9.s, z1.s\n" - "ld1w { z3.s }, p2/Z, [x14, #3, MUL VL]\n" - "add z10.s, z10.s, z2.s\n" - "add z11.s, z11.s, z3.s\n" + "ld1w { z23.s }, p2/Z, [x14]\n" + "add x26, x11, x20\n" + "add x25, x26, x20\n" + "ld1w { z22.s }, p2/Z, [x14, #1, MUL VL]\n" + "ld1w { z21.s }, p2/Z, [x14, #2, MUL VL]\n" + "add z8.s, z8.s, z23.s\n" + "add z9.s, z9.s, z22.s\n" + "ld1w { z20.s }, p2/Z, [x14, #3, MUL VL]\n" + "add z10.s, z10.s, z21.s\n" + "add z11.s, z11.s, z20.s\n" "addvl x14, x14, #4\n" - "add z12.s, z12.s, z0.s\n" - "add z13.s, z13.s, z1.s\n" - "add z14.s, z14.s, z2.s\n" - "add z15.s, z15.s, z3.s\n" - "add z16.s, z16.s, z0.s\n" - "add z17.s, z17.s, z1.s\n" - "add z18.s, z18.s, z2.s\n" - "add z19.s, z19.s, z3.s\n" + "add z12.s, z12.s, z23.s\n" + "add z13.s, z13.s, z22.s\n" + "add z14.s, z14.s, z21.s\n" + "add z15.s, z15.s, z20.s\n" + "add z16.s, z16.s, z23.s\n" + "add z17.s, z17.s, z22.s\n" + "add z18.s, z18.s, z21.s\n" + "add z19.s, z19.s, z20.s\n" "tbz %x[flags], #4, 36f\n" "ld1w { z0.s }, p2/Z, [x12]\n" "ld1w { z4.s }, p2/Z, [x13]\n" @@ -830,10 +830,10 @@ void sve_hybrid_s8qs_dot_6x4VL ( "addvl x13, x13, #4\n" "b 37f\n" "36:" // Height 3: per layer parameters - "add x26, %x[qp], %[per_layer_right_shift]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1rw { z0.s }, p2/Z, [x26]\n" - "ld1rw { z4.s }, p2/Z, [x25]\n" + "add x21, %x[qp], %[per_layer_right_shift]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1rw { z0.s }, p2/Z, [x21]\n" + "ld1rw { z4.s }, p2/Z, [x20]\n" "mov z1.d, z0.d\n" "mov z5.d, z4.d\n" "mov z2.d, z0.d\n" @@ -854,109 +854,109 @@ void sve_hybrid_s8qs_dot_6x4VL ( ".inst 0x04a67652 // sqrdmulh z18.s, z18.s, z6.s\n" ".inst 0x04a77673 // sqrdmulh z19.s, z19.s, z7.s\n" "tbz %x[flags], #5, 38f\n" - "and z4.d, z8.d, z0.d\n" - "and z5.d, z9.d, z1.d\n" - "and z6.d, z10.d, z2.d\n" - "and z7.d, z11.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z8.s, z8.s, z4.s\n" - "sqadd z9.s, z9.s, z5.s\n" - "sqadd z10.s, z10.s, z6.s\n" - "sqadd z11.s, z11.s, z7.s\n" - "and z4.d, z12.d, z0.d\n" - "and z5.d, z13.d, z1.d\n" - "and z6.d, z14.d, z2.d\n" - "and z7.d, z15.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z12.s, z12.s, z4.s\n" - "sqadd z13.s, z13.s, z5.s\n" - "sqadd z14.s, z14.s, z6.s\n" - "sqadd z15.s, z15.s, z7.s\n" - "and z4.d, z16.d, z0.d\n" - "and z5.d, z17.d, z1.d\n" - "and z6.d, z18.d, z2.d\n" - "and z7.d, z19.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z16.s, z16.s, z4.s\n" - "sqadd z17.s, z17.s, z5.s\n" - "sqadd z18.s, z18.s, z6.s\n" - "sqadd z19.s, z19.s, z7.s\n" + "and z23.d, z8.d, z0.d\n" + "and z22.d, z9.d, z1.d\n" + "and z21.d, z10.d, z2.d\n" + "and z20.d, z11.d, z3.d\n" + "asr z23.s, z23.s, #0x1f\n" + "asr z22.s, z22.s, #0x1f\n" + "asr z21.s, z21.s, #0x1f\n" + "asr z20.s, z20.s, #0x1f\n" + "sqadd z8.s, z8.s, z23.s\n" + "sqadd z9.s, z9.s, z22.s\n" + "sqadd z10.s, z10.s, z21.s\n" + "sqadd z11.s, z11.s, z20.s\n" + "and z23.d, z12.d, z0.d\n" + "and z22.d, z13.d, z1.d\n" + "and z21.d, z14.d, z2.d\n" + "and z20.d, z15.d, z3.d\n" + "asr z23.s, z23.s, #0x1f\n" + "asr z22.s, z22.s, #0x1f\n" + "asr z21.s, z21.s, #0x1f\n" + "asr z20.s, z20.s, #0x1f\n" + "sqadd z12.s, z12.s, z23.s\n" + "sqadd z13.s, z13.s, z22.s\n" + "sqadd z14.s, z14.s, z21.s\n" + "sqadd z15.s, z15.s, z20.s\n" + "and z23.d, z16.d, z0.d\n" + "and z22.d, z17.d, z1.d\n" + "and z21.d, z18.d, z2.d\n" + "and z20.d, z19.d, z3.d\n" + "asr z23.s, z23.s, #0x1f\n" + "asr z22.s, z22.s, #0x1f\n" + "asr z21.s, z21.s, #0x1f\n" + "asr z20.s, z20.s, #0x1f\n" + "sqadd z16.s, z16.s, z23.s\n" + "sqadd z17.s, z17.s, z22.s\n" + "sqadd z18.s, z18.s, z21.s\n" + "sqadd z19.s, z19.s, z20.s\n" "38:" // Height 3: no shift correction - "add x25, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x25]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1rw { z21.s }, p2/Z, [x20]\n" ".inst 0x44828808 // srshl z8.s, p2/M, z8.s, z0.s\n" - "add z8.s, z8.s, z4.s\n" + "add z8.s, z8.s, z21.s\n" ".inst 0x44828829 // srshl z9.s, p2/M, z9.s, z1.s\n" ".inst 0x4482884a // srshl z10.s, p2/M, z10.s, z2.s\n" - "add z9.s, z9.s, z4.s\n" - "add z10.s, z10.s, z4.s\n" + "add z9.s, z9.s, z21.s\n" + "add z10.s, z10.s, z21.s\n" ".inst 0x4482886b // srshl z11.s, p2/M, z11.s, z3.s\n" ".inst 0x4482880c // srshl z12.s, p2/M, z12.s, z0.s\n" - "add z11.s, z11.s, z4.s\n" - "add z12.s, z12.s, z4.s\n" + "add z11.s, z11.s, z21.s\n" + "add z12.s, z12.s, z21.s\n" ".inst 0x4482882d // srshl z13.s, p2/M, z13.s, z1.s\n" ".inst 0x4482884e // srshl z14.s, p2/M, z14.s, z2.s\n" - "add z13.s, z13.s, z4.s\n" - "add z14.s, z14.s, z4.s\n" + "add z13.s, z13.s, z21.s\n" + "add z14.s, z14.s, z21.s\n" ".inst 0x4482886f // srshl z15.s, p2/M, z15.s, z3.s\n" ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n" - "add z15.s, z15.s, z4.s\n" - "add z16.s, z16.s, z4.s\n" + "add z15.s, z15.s, z21.s\n" + "add z16.s, z16.s, z21.s\n" ".inst 0x44828831 // srshl z17.s, p2/M, z17.s, z1.s\n" ".inst 0x44828852 // srshl z18.s, p2/M, z18.s, z2.s\n" - "add z17.s, z17.s, z4.s\n" - "add z18.s, z18.s, z4.s\n" + "add z17.s, z17.s, z21.s\n" + "add z18.s, z18.s, z21.s\n" ".inst 0x44828873 // srshl z19.s, p2/M, z19.s, z3.s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x25]\n" - "add z19.s, z19.s, z4.s\n" - "add x25, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x25]\n" - "smin z8.s, p2/M, z8.s, z6.s\n" - "smin z9.s, p2/M, z9.s, z6.s\n" - "smin z10.s, p2/M, z10.s, z6.s\n" - "smin z11.s, p2/M, z11.s, z6.s\n" - "smin z12.s, p2/M, z12.s, z6.s\n" - "smin z13.s, p2/M, z13.s, z6.s\n" - "smin z14.s, p2/M, z14.s, z6.s\n" - "smin z15.s, p2/M, z15.s, z6.s\n" - "smin z16.s, p2/M, z16.s, z6.s\n" - "smin z17.s, p2/M, z17.s, z6.s\n" - "smin z18.s, p2/M, z18.s, z6.s\n" - "smin z19.s, p2/M, z19.s, z6.s\n" - "smax z8.s, p2/M, z8.s, z5.s\n" - "smax z9.s, p2/M, z9.s, z5.s\n" - "smax z10.s, p2/M, z10.s, z5.s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1rw { z20.s }, p2/Z, [x20]\n" + "add z19.s, z19.s, z21.s\n" + "add x20, %x[qp], %[minval]\n" + "ld1rw { z21.s }, p2/Z, [x20]\n" + "smin z8.s, p2/M, z8.s, z20.s\n" + "smin z9.s, p2/M, z9.s, z20.s\n" + "smin z10.s, p2/M, z10.s, z20.s\n" + "smin z11.s, p2/M, z11.s, z20.s\n" + "smin z12.s, p2/M, z12.s, z20.s\n" + "smin z13.s, p2/M, z13.s, z20.s\n" + "smin z14.s, p2/M, z14.s, z20.s\n" + "smin z15.s, p2/M, z15.s, z20.s\n" + "smin z16.s, p2/M, z16.s, z20.s\n" + "smin z17.s, p2/M, z17.s, z20.s\n" + "smin z18.s, p2/M, z18.s, z20.s\n" + "smin z19.s, p2/M, z19.s, z20.s\n" + "smax z8.s, p2/M, z8.s, z21.s\n" + "smax z9.s, p2/M, z9.s, z21.s\n" + "smax z10.s, p2/M, z10.s, z21.s\n" "uzp1 z8.h, z8.h, z9.h\n" - "smax z11.s, p2/M, z11.s, z5.s\n" - "smax z12.s, p2/M, z12.s, z5.s\n" - "uzp1 z9.h, z10.h, z11.h\n" - "uzp1 z8.b, z8.b, z9.b\n" - "smax z13.s, p2/M, z13.s, z5.s\n" - "smax z14.s, p2/M, z14.s, z5.s\n" + "smax z11.s, p2/M, z11.s, z21.s\n" + "smax z12.s, p2/M, z12.s, z21.s\n" + "uzp1 z20.h, z10.h, z11.h\n" + "uzp1 z8.b, z8.b, z20.b\n" + "smax z13.s, p2/M, z13.s, z21.s\n" + "smax z14.s, p2/M, z14.s, z21.s\n" "uzp1 z12.h, z12.h, z13.h\n" "st1b { z8.b }, p1, [x11]\n" - "smax z15.s, p2/M, z15.s, z5.s\n" - "smax z16.s, p2/M, z16.s, z5.s\n" - "uzp1 z13.h, z14.h, z15.h\n" - "uzp1 z12.b, z12.b, z13.b\n" - "smax z17.s, p2/M, z17.s, z5.s\n" - "smax z18.s, p2/M, z18.s, z5.s\n" + "smax z15.s, p2/M, z15.s, z21.s\n" + "smax z16.s, p2/M, z16.s, z21.s\n" + "uzp1 z20.h, z14.h, z15.h\n" + "uzp1 z12.b, z12.b, z20.b\n" + "smax z17.s, p2/M, z17.s, z21.s\n" + "smax z18.s, p2/M, z18.s, z21.s\n" "uzp1 z16.h, z16.h, z17.h\n" - "st1b { z12.b }, p1, [x24]\n" - "smax z19.s, p2/M, z19.s, z5.s\n" + "st1b { z12.b }, p1, [x26]\n" + "smax z19.s, p2/M, z19.s, z21.s\n" "uzp1 z17.h, z18.h, z19.h\n" "uzp1 z16.b, z16.b, z17.b\n" - "st1b { z16.b }, p1, [x23]\n" + "st1b { z16.b }, p1, [x25]\n" "addvl x11, x11, #1\n" "39:" // Height 3: Writeback done "decw x10, ALL, MUL #4\n" @@ -994,14 +994,14 @@ void sve_hybrid_s8qs_dot_6x4VL ( "43:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 44f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 45f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -1011,105 +1011,105 @@ void sve_hybrid_s8qs_dot_6x4VL ( "b 45f\n" "44:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" "45:" // Height 4: input setup done "cmp x27, #0x10\n" "ble 47f\n" "46:" // Height 4: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z0.b }, p0/Z, [x26]\n" - "ld1rqb { z1.b }, p0/Z, [x25]\n" + "ld1rqb { z3.b }, p0/Z, [x26]\n" + "ld1rqb { z2.b }, p0/Z, [x25]\n" "sub x27, x27, #0x10\n" - "ld1rqb { z2.b }, p0/Z, [x24]\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" + "ld1rqb { z1.b }, p0/Z, [x24]\n" + "ld1rqb { z0.b }, p0/Z, [x23]\n" "cmp x27, #0x10\n" "add x26, x26, #0x10\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[0]\n" - "sdot z12.s, z6.b, z1.b[0]\n" - "sdot z16.s, z6.b, z2.b[0]\n" - "sdot z20.s, z6.b, z3.b[0]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z25.b }, p2/Z, [x9]\n" + "ld1b { z24.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z8.s, z25.b, z3.b[0]\n" + "sdot z12.s, z25.b, z2.b[0]\n" + "sdot z16.s, z25.b, z1.b[0]\n" + "sdot z20.s, z25.b, z0.b[0]\n" + "ld1b { z25.b }, p2/Z, [x9, #2, MUL VL]\n" "add x25, x25, #0x10\n" - "sdot z9.s, z7.b, z0.b[0]\n" - "sdot z13.s, z7.b, z1.b[0]\n" + "sdot z9.s, z24.b, z3.b[0]\n" + "sdot z13.s, z24.b, z2.b[0]\n" "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" - "sdot z17.s, z7.b, z2.b[0]\n" - "sdot z21.s, z7.b, z3.b[0]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[0]\n" - "sdot z14.s, z6.b, z1.b[0]\n" - "sdot z18.s, z6.b, z2.b[0]\n" - "sdot z22.s, z6.b, z3.b[0]\n" - "ld1b { z6.b }, p2/Z, [x9, #4, MUL VL]\n" - "sdot z11.s, z7.b, z0.b[0]\n" - "sdot z15.s, z7.b, z1.b[0]\n" - "sdot z19.s, z7.b, z2.b[0]\n" - "sdot z23.s, z7.b, z3.b[0]\n" - "ld1b { z7.b }, p2/Z, [x9, #5, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[1]\n" - "sdot z12.s, z6.b, z1.b[1]\n" - "sdot z16.s, z6.b, z2.b[1]\n" - "sdot z20.s, z6.b, z3.b[1]\n" - "ld1b { z6.b }, p2/Z, [x9, #6, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[1]\n" - "sdot z13.s, z7.b, z1.b[1]\n" - "sdot z17.s, z7.b, z2.b[1]\n" - "sdot z21.s, z7.b, z3.b[1]\n" - "ld1b { z7.b }, p2/Z, [x9, #7, MUL VL]\n" + "sdot z17.s, z24.b, z1.b[0]\n" + "sdot z21.s, z24.b, z0.b[0]\n" + "ld1b { z24.b }, p2/Z, [x9, #3, MUL VL]\n" + "sdot z10.s, z25.b, z3.b[0]\n" + "sdot z14.s, z25.b, z2.b[0]\n" + "sdot z18.s, z25.b, z1.b[0]\n" + "sdot z22.s, z25.b, z0.b[0]\n" + "ld1b { z25.b }, p2/Z, [x9, #4, MUL VL]\n" + "sdot z11.s, z24.b, z3.b[0]\n" + "sdot z15.s, z24.b, z2.b[0]\n" + "sdot z19.s, z24.b, z1.b[0]\n" + "sdot z23.s, z24.b, z0.b[0]\n" + "ld1b { z24.b }, p2/Z, [x9, #5, MUL VL]\n" + "sdot z8.s, z25.b, z3.b[1]\n" + "sdot z12.s, z25.b, z2.b[1]\n" + "sdot z16.s, z25.b, z1.b[1]\n" + "sdot z20.s, z25.b, z0.b[1]\n" + "ld1b { z25.b }, p2/Z, [x9, #6, MUL VL]\n" + "sdot z9.s, z24.b, z3.b[1]\n" + "sdot z13.s, z24.b, z2.b[1]\n" + "sdot z17.s, z24.b, z1.b[1]\n" + "sdot z21.s, z24.b, z0.b[1]\n" + "ld1b { z24.b }, p2/Z, [x9, #7, MUL VL]\n" "addvl x9, x9, #16\n" - "sdot z10.s, z6.b, z0.b[1]\n" - "sdot z14.s, z6.b, z1.b[1]\n" - "sdot z18.s, z6.b, z2.b[1]\n" - "sdot z22.s, z6.b, z3.b[1]\n" - "ld1b { z6.b }, p2/Z, [x9, #-8, MUL VL]\n" - "sdot z11.s, z7.b, z0.b[1]\n" - "sdot z15.s, z7.b, z1.b[1]\n" - "sdot z19.s, z7.b, z2.b[1]\n" - "sdot z23.s, z7.b, z3.b[1]\n" - "ld1b { z7.b }, p2/Z, [x9, #-7, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[2]\n" - "sdot z12.s, z6.b, z1.b[2]\n" - "sdot z16.s, z6.b, z2.b[2]\n" - "sdot z20.s, z6.b, z3.b[2]\n" - "ld1b { z6.b }, p2/Z, [x9, #-6, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[2]\n" - "sdot z13.s, z7.b, z1.b[2]\n" - "sdot z17.s, z7.b, z2.b[2]\n" - "sdot z21.s, z7.b, z3.b[2]\n" - "ld1b { z7.b }, p2/Z, [x9, #-5, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[2]\n" - "sdot z14.s, z6.b, z1.b[2]\n" - "sdot z18.s, z6.b, z2.b[2]\n" - "sdot z22.s, z6.b, z3.b[2]\n" - "ld1b { z6.b }, p2/Z, [x9, #-4, MUL VL]\n" - "sdot z11.s, z7.b, z0.b[2]\n" - "sdot z15.s, z7.b, z1.b[2]\n" - "sdot z19.s, z7.b, z2.b[2]\n" - "sdot z23.s, z7.b, z3.b[2]\n" - "ld1b { z7.b }, p2/Z, [x9, #-3, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[3]\n" - "sdot z12.s, z6.b, z1.b[3]\n" - "sdot z16.s, z6.b, z2.b[3]\n" - "sdot z20.s, z6.b, z3.b[3]\n" - "ld1b { z6.b }, p2/Z, [x9, #-2, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[3]\n" - "sdot z13.s, z7.b, z1.b[3]\n" - "sdot z17.s, z7.b, z2.b[3]\n" - "sdot z21.s, z7.b, z3.b[3]\n" - "ld1b { z7.b }, p2/Z, [x9, #-1, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[3]\n" - "sdot z14.s, z6.b, z1.b[3]\n" - "sdot z18.s, z6.b, z2.b[3]\n" - "sdot z22.s, z6.b, z3.b[3]\n" - "sdot z11.s, z7.b, z0.b[3]\n" - "sdot z15.s, z7.b, z1.b[3]\n" - "sdot z19.s, z7.b, z2.b[3]\n" - "sdot z23.s, z7.b, z3.b[3]\n" + "sdot z10.s, z25.b, z3.b[1]\n" + "sdot z14.s, z25.b, z2.b[1]\n" + "sdot z18.s, z25.b, z1.b[1]\n" + "sdot z22.s, z25.b, z0.b[1]\n" + "ld1b { z25.b }, p2/Z, [x9, #-8, MUL VL]\n" + "sdot z11.s, z24.b, z3.b[1]\n" + "sdot z15.s, z24.b, z2.b[1]\n" + "sdot z19.s, z24.b, z1.b[1]\n" + "sdot z23.s, z24.b, z0.b[1]\n" + "ld1b { z24.b }, p2/Z, [x9, #-7, MUL VL]\n" + "sdot z8.s, z25.b, z3.b[2]\n" + "sdot z12.s, z25.b, z2.b[2]\n" + "sdot z16.s, z25.b, z1.b[2]\n" + "sdot z20.s, z25.b, z0.b[2]\n" + "ld1b { z25.b }, p2/Z, [x9, #-6, MUL VL]\n" + "sdot z9.s, z24.b, z3.b[2]\n" + "sdot z13.s, z24.b, z2.b[2]\n" + "sdot z17.s, z24.b, z1.b[2]\n" + "sdot z21.s, z24.b, z0.b[2]\n" + "ld1b { z24.b }, p2/Z, [x9, #-5, MUL VL]\n" + "sdot z10.s, z25.b, z3.b[2]\n" + "sdot z14.s, z25.b, z2.b[2]\n" + "sdot z18.s, z25.b, z1.b[2]\n" + "sdot z22.s, z25.b, z0.b[2]\n" + "ld1b { z25.b }, p2/Z, [x9, #-4, MUL VL]\n" + "sdot z11.s, z24.b, z3.b[2]\n" + "sdot z15.s, z24.b, z2.b[2]\n" + "sdot z19.s, z24.b, z1.b[2]\n" + "sdot z23.s, z24.b, z0.b[2]\n" + "ld1b { z24.b }, p2/Z, [x9, #-3, MUL VL]\n" + "sdot z8.s, z25.b, z3.b[3]\n" + "sdot z12.s, z25.b, z2.b[3]\n" + "sdot z16.s, z25.b, z1.b[3]\n" + "sdot z20.s, z25.b, z0.b[3]\n" + "ld1b { z25.b }, p2/Z, [x9, #-2, MUL VL]\n" + "sdot z9.s, z24.b, z3.b[3]\n" + "sdot z13.s, z24.b, z2.b[3]\n" + "sdot z17.s, z24.b, z1.b[3]\n" + "sdot z21.s, z24.b, z0.b[3]\n" + "ld1b { z24.b }, p2/Z, [x9, #-1, MUL VL]\n" + "sdot z10.s, z25.b, z3.b[3]\n" + "sdot z14.s, z25.b, z2.b[3]\n" + "sdot z18.s, z25.b, z1.b[3]\n" + "sdot z22.s, z25.b, z0.b[3]\n" + "sdot z11.s, z24.b, z3.b[3]\n" + "sdot z15.s, z24.b, z2.b[3]\n" + "sdot z19.s, z24.b, z1.b[3]\n" + "sdot z23.s, z24.b, z0.b[3]\n" "bgt 46b\n" "47:" // Height 4: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" @@ -1118,125 +1118,125 @@ void sve_hybrid_s8qs_dot_6x4VL ( "subs x27, x27, #0x4\n" "ld1rqb { z2.b }, p0/Z, [x24]\n" "ld1rqb { z3.b }, p0/Z, [x23]\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[0]\n" - "sdot z12.s, z6.b, z1.b[0]\n" - "sdot z16.s, z6.b, z2.b[0]\n" - "sdot z20.s, z6.b, z3.b[0]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[0]\n" - "sdot z13.s, z7.b, z1.b[0]\n" - "sdot z17.s, z7.b, z2.b[0]\n" - "sdot z21.s, z7.b, z3.b[0]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" + "ld1b { z25.b }, p2/Z, [x9]\n" + "ld1b { z24.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z8.s, z25.b, z0.b[0]\n" + "sdot z12.s, z25.b, z1.b[0]\n" + "sdot z16.s, z25.b, z2.b[0]\n" + "sdot z20.s, z25.b, z3.b[0]\n" + "ld1b { z25.b }, p2/Z, [x9, #2, MUL VL]\n" + "sdot z9.s, z24.b, z0.b[0]\n" + "sdot z13.s, z24.b, z1.b[0]\n" + "sdot z17.s, z24.b, z2.b[0]\n" + "sdot z21.s, z24.b, z3.b[0]\n" + "ld1b { z24.b }, p2/Z, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "sdot z10.s, z6.b, z0.b[0]\n" - "sdot z14.s, z6.b, z1.b[0]\n" - "sdot z18.s, z6.b, z2.b[0]\n" - "sdot z22.s, z6.b, z3.b[0]\n" - "sdot z11.s, z7.b, z0.b[0]\n" - "sdot z15.s, z7.b, z1.b[0]\n" - "sdot z19.s, z7.b, z2.b[0]\n" - "sdot z23.s, z7.b, z3.b[0]\n" + "sdot z10.s, z25.b, z0.b[0]\n" + "sdot z14.s, z25.b, z1.b[0]\n" + "sdot z18.s, z25.b, z2.b[0]\n" + "sdot z22.s, z25.b, z3.b[0]\n" + "sdot z11.s, z24.b, z0.b[0]\n" + "sdot z15.s, z24.b, z1.b[0]\n" + "sdot z19.s, z24.b, z2.b[0]\n" + "sdot z23.s, z24.b, z3.b[0]\n" "ble 48f\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[1]\n" - "sdot z12.s, z6.b, z1.b[1]\n" - "sdot z16.s, z6.b, z2.b[1]\n" - "sdot z20.s, z6.b, z3.b[1]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z25.b }, p2/Z, [x9]\n" + "ld1b { z24.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z8.s, z25.b, z0.b[1]\n" + "sdot z12.s, z25.b, z1.b[1]\n" + "sdot z16.s, z25.b, z2.b[1]\n" + "sdot z20.s, z25.b, z3.b[1]\n" + "ld1b { z25.b }, p2/Z, [x9, #2, MUL VL]\n" "subs x27, x27, #0x4\n" - "sdot z9.s, z7.b, z0.b[1]\n" - "sdot z13.s, z7.b, z1.b[1]\n" - "sdot z17.s, z7.b, z2.b[1]\n" - "sdot z21.s, z7.b, z3.b[1]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" + "sdot z9.s, z24.b, z0.b[1]\n" + "sdot z13.s, z24.b, z1.b[1]\n" + "sdot z17.s, z24.b, z2.b[1]\n" + "sdot z21.s, z24.b, z3.b[1]\n" + "ld1b { z24.b }, p2/Z, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "sdot z10.s, z6.b, z0.b[1]\n" - "sdot z14.s, z6.b, z1.b[1]\n" - "sdot z18.s, z6.b, z2.b[1]\n" - "sdot z22.s, z6.b, z3.b[1]\n" - "sdot z11.s, z7.b, z0.b[1]\n" - "sdot z15.s, z7.b, z1.b[1]\n" - "sdot z19.s, z7.b, z2.b[1]\n" - "sdot z23.s, z7.b, z3.b[1]\n" + "sdot z10.s, z25.b, z0.b[1]\n" + "sdot z14.s, z25.b, z1.b[1]\n" + "sdot z18.s, z25.b, z2.b[1]\n" + "sdot z22.s, z25.b, z3.b[1]\n" + "sdot z11.s, z24.b, z0.b[1]\n" + "sdot z15.s, z24.b, z1.b[1]\n" + "sdot z19.s, z24.b, z2.b[1]\n" + "sdot z23.s, z24.b, z3.b[1]\n" "ble 48f\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[2]\n" - "sdot z12.s, z6.b, z1.b[2]\n" - "sdot z16.s, z6.b, z2.b[2]\n" - "sdot z20.s, z6.b, z3.b[2]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z25.b }, p2/Z, [x9]\n" + "ld1b { z24.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z8.s, z25.b, z0.b[2]\n" + "sdot z12.s, z25.b, z1.b[2]\n" + "sdot z16.s, z25.b, z2.b[2]\n" + "sdot z20.s, z25.b, z3.b[2]\n" + "ld1b { z25.b }, p2/Z, [x9, #2, MUL VL]\n" "subs x27, x27, #0x4\n" - "sdot z9.s, z7.b, z0.b[2]\n" - "sdot z13.s, z7.b, z1.b[2]\n" - "sdot z17.s, z7.b, z2.b[2]\n" - "sdot z21.s, z7.b, z3.b[2]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" + "sdot z9.s, z24.b, z0.b[2]\n" + "sdot z13.s, z24.b, z1.b[2]\n" + "sdot z17.s, z24.b, z2.b[2]\n" + "sdot z21.s, z24.b, z3.b[2]\n" + "ld1b { z24.b }, p2/Z, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "sdot z10.s, z6.b, z0.b[2]\n" - "sdot z14.s, z6.b, z1.b[2]\n" - "sdot z18.s, z6.b, z2.b[2]\n" - "sdot z22.s, z6.b, z3.b[2]\n" - "sdot z11.s, z7.b, z0.b[2]\n" - "sdot z15.s, z7.b, z1.b[2]\n" - "sdot z19.s, z7.b, z2.b[2]\n" - "sdot z23.s, z7.b, z3.b[2]\n" + "sdot z10.s, z25.b, z0.b[2]\n" + "sdot z14.s, z25.b, z1.b[2]\n" + "sdot z18.s, z25.b, z2.b[2]\n" + "sdot z22.s, z25.b, z3.b[2]\n" + "sdot z11.s, z24.b, z0.b[2]\n" + "sdot z15.s, z24.b, z1.b[2]\n" + "sdot z19.s, z24.b, z2.b[2]\n" + "sdot z23.s, z24.b, z3.b[2]\n" "ble 48f\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[3]\n" - "sdot z12.s, z6.b, z1.b[3]\n" - "sdot z16.s, z6.b, z2.b[3]\n" - "sdot z20.s, z6.b, z3.b[3]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[3]\n" - "sdot z13.s, z7.b, z1.b[3]\n" - "sdot z17.s, z7.b, z2.b[3]\n" - "sdot z21.s, z7.b, z3.b[3]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" + "ld1b { z25.b }, p2/Z, [x9]\n" + "ld1b { z24.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z8.s, z25.b, z0.b[3]\n" + "sdot z12.s, z25.b, z1.b[3]\n" + "sdot z16.s, z25.b, z2.b[3]\n" + "sdot z20.s, z25.b, z3.b[3]\n" + "ld1b { z25.b }, p2/Z, [x9, #2, MUL VL]\n" + "sdot z9.s, z24.b, z0.b[3]\n" + "sdot z13.s, z24.b, z1.b[3]\n" + "sdot z17.s, z24.b, z2.b[3]\n" + "sdot z21.s, z24.b, z3.b[3]\n" + "ld1b { z24.b }, p2/Z, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "sdot z10.s, z6.b, z0.b[3]\n" - "sdot z14.s, z6.b, z1.b[3]\n" - "sdot z18.s, z6.b, z2.b[3]\n" - "sdot z22.s, z6.b, z3.b[3]\n" - "sdot z11.s, z7.b, z0.b[3]\n" - "sdot z15.s, z7.b, z1.b[3]\n" - "sdot z19.s, z7.b, z2.b[3]\n" - "sdot z23.s, z7.b, z3.b[3]\n" + "sdot z10.s, z25.b, z0.b[3]\n" + "sdot z14.s, z25.b, z1.b[3]\n" + "sdot z18.s, z25.b, z2.b[3]\n" + "sdot z22.s, z25.b, z3.b[3]\n" + "sdot z11.s, z24.b, z0.b[3]\n" + "sdot z15.s, z24.b, z1.b[3]\n" + "sdot z19.s, z24.b, z2.b[3]\n" + "sdot z23.s, z24.b, z3.b[3]\n" "48:" // Height 4: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 43b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "ld1w { z0.s }, p2/Z, [x14]\n" - "add x24, x11, x20\n" - "add x23, x24, x20\n" - "ld1w { z1.s }, p2/Z, [x14, #1, MUL VL]\n" - "ld1w { z2.s }, p2/Z, [x14, #2, MUL VL]\n" - "add x22, x23, x20\n" - "add z8.s, z8.s, z0.s\n" - "ld1w { z3.s }, p2/Z, [x14, #3, MUL VL]\n" - "add z9.s, z9.s, z1.s\n" - "add z10.s, z10.s, z2.s\n" + "ld1w { z27.s }, p2/Z, [x14]\n" + "add x26, x11, x20\n" + "add x25, x26, x20\n" + "ld1w { z26.s }, p2/Z, [x14, #1, MUL VL]\n" + "ld1w { z25.s }, p2/Z, [x14, #2, MUL VL]\n" + "add x24, x25, x20\n" + "add z8.s, z8.s, z27.s\n" + "ld1w { z24.s }, p2/Z, [x14, #3, MUL VL]\n" + "add z9.s, z9.s, z26.s\n" + "add z10.s, z10.s, z25.s\n" "addvl x14, x14, #4\n" - "add z11.s, z11.s, z3.s\n" - "add z12.s, z12.s, z0.s\n" - "add z13.s, z13.s, z1.s\n" - "add z14.s, z14.s, z2.s\n" - "add z15.s, z15.s, z3.s\n" - "add z16.s, z16.s, z0.s\n" - "add z17.s, z17.s, z1.s\n" - "add z18.s, z18.s, z2.s\n" - "add z19.s, z19.s, z3.s\n" - "add z20.s, z20.s, z0.s\n" - "add z21.s, z21.s, z1.s\n" - "add z22.s, z22.s, z2.s\n" - "add z23.s, z23.s, z3.s\n" + "add z11.s, z11.s, z24.s\n" + "add z12.s, z12.s, z27.s\n" + "add z13.s, z13.s, z26.s\n" + "add z14.s, z14.s, z25.s\n" + "add z15.s, z15.s, z24.s\n" + "add z16.s, z16.s, z27.s\n" + "add z17.s, z17.s, z26.s\n" + "add z18.s, z18.s, z25.s\n" + "add z19.s, z19.s, z24.s\n" + "add z20.s, z20.s, z27.s\n" + "add z21.s, z21.s, z26.s\n" + "add z22.s, z22.s, z25.s\n" + "add z23.s, z23.s, z24.s\n" "tbz %x[flags], #4, 49f\n" "ld1w { z0.s }, p2/Z, [x12]\n" "ld1w { z4.s }, p2/Z, [x13]\n" @@ -1250,10 +1250,10 @@ void sve_hybrid_s8qs_dot_6x4VL ( "addvl x13, x13, #4\n" "b 50f\n" "49:" // Height 4: per layer parameters - "add x26, %x[qp], %[per_layer_right_shift]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1rw { z0.s }, p2/Z, [x26]\n" - "ld1rw { z4.s }, p2/Z, [x25]\n" + "add x21, %x[qp], %[per_layer_right_shift]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1rw { z0.s }, p2/Z, [x21]\n" + "ld1rw { z4.s }, p2/Z, [x20]\n" "mov z1.d, z0.d\n" "mov z5.d, z4.d\n" "mov z2.d, z0.d\n" @@ -1278,141 +1278,141 @@ void sve_hybrid_s8qs_dot_6x4VL ( ".inst 0x04a676d6 // sqrdmulh z22.s, z22.s, z6.s\n" ".inst 0x04a776f7 // sqrdmulh z23.s, z23.s, z7.s\n" "tbz %x[flags], #5, 51f\n" - "and z4.d, z8.d, z0.d\n" - "and z5.d, z9.d, z1.d\n" - "and z6.d, z10.d, z2.d\n" - "and z7.d, z11.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z8.s, z8.s, z4.s\n" - "sqadd z9.s, z9.s, z5.s\n" - "sqadd z10.s, z10.s, z6.s\n" - "sqadd z11.s, z11.s, z7.s\n" - "and z4.d, z12.d, z0.d\n" - "and z5.d, z13.d, z1.d\n" - "and z6.d, z14.d, z2.d\n" - "and z7.d, z15.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z12.s, z12.s, z4.s\n" - "sqadd z13.s, z13.s, z5.s\n" - "sqadd z14.s, z14.s, z6.s\n" - "sqadd z15.s, z15.s, z7.s\n" - "and z4.d, z16.d, z0.d\n" - "and z5.d, z17.d, z1.d\n" - "and z6.d, z18.d, z2.d\n" - "and z7.d, z19.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z16.s, z16.s, z4.s\n" - "sqadd z17.s, z17.s, z5.s\n" - "sqadd z18.s, z18.s, z6.s\n" - "sqadd z19.s, z19.s, z7.s\n" - "and z4.d, z20.d, z0.d\n" - "and z5.d, z21.d, z1.d\n" - "and z6.d, z22.d, z2.d\n" - "and z7.d, z23.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z20.s, z20.s, z4.s\n" - "sqadd z21.s, z21.s, z5.s\n" - "sqadd z22.s, z22.s, z6.s\n" - "sqadd z23.s, z23.s, z7.s\n" + "and z27.d, z8.d, z0.d\n" + "and z26.d, z9.d, z1.d\n" + "and z25.d, z10.d, z2.d\n" + "and z24.d, z11.d, z3.d\n" + "asr z27.s, z27.s, #0x1f\n" + "asr z26.s, z26.s, #0x1f\n" + "asr z25.s, z25.s, #0x1f\n" + "asr z24.s, z24.s, #0x1f\n" + "sqadd z8.s, z8.s, z27.s\n" + "sqadd z9.s, z9.s, z26.s\n" + "sqadd z10.s, z10.s, z25.s\n" + "sqadd z11.s, z11.s, z24.s\n" + "and z27.d, z12.d, z0.d\n" + "and z26.d, z13.d, z1.d\n" + "and z25.d, z14.d, z2.d\n" + "and z24.d, z15.d, z3.d\n" + "asr z27.s, z27.s, #0x1f\n" + "asr z26.s, z26.s, #0x1f\n" + "asr z25.s, z25.s, #0x1f\n" + "asr z24.s, z24.s, #0x1f\n" + "sqadd z12.s, z12.s, z27.s\n" + "sqadd z13.s, z13.s, z26.s\n" + "sqadd z14.s, z14.s, z25.s\n" + "sqadd z15.s, z15.s, z24.s\n" + "and z27.d, z16.d, z0.d\n" + "and z26.d, z17.d, z1.d\n" + "and z25.d, z18.d, z2.d\n" + "and z24.d, z19.d, z3.d\n" + "asr z27.s, z27.s, #0x1f\n" + "asr z26.s, z26.s, #0x1f\n" + "asr z25.s, z25.s, #0x1f\n" + "asr z24.s, z24.s, #0x1f\n" + "sqadd z16.s, z16.s, z27.s\n" + "sqadd z17.s, z17.s, z26.s\n" + "sqadd z18.s, z18.s, z25.s\n" + "sqadd z19.s, z19.s, z24.s\n" + "and z27.d, z20.d, z0.d\n" + "and z26.d, z21.d, z1.d\n" + "and z25.d, z22.d, z2.d\n" + "and z24.d, z23.d, z3.d\n" + "asr z27.s, z27.s, #0x1f\n" + "asr z26.s, z26.s, #0x1f\n" + "asr z25.s, z25.s, #0x1f\n" + "asr z24.s, z24.s, #0x1f\n" + "sqadd z20.s, z20.s, z27.s\n" + "sqadd z21.s, z21.s, z26.s\n" + "sqadd z22.s, z22.s, z25.s\n" + "sqadd z23.s, z23.s, z24.s\n" "51:" // Height 4: no shift correction - "add x25, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x25]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1rw { z25.s }, p2/Z, [x20]\n" ".inst 0x44828808 // srshl z8.s, p2/M, z8.s, z0.s\n" - "add z8.s, z8.s, z4.s\n" + "add z8.s, z8.s, z25.s\n" ".inst 0x44828829 // srshl z9.s, p2/M, z9.s, z1.s\n" ".inst 0x4482884a // srshl z10.s, p2/M, z10.s, z2.s\n" - "add z9.s, z9.s, z4.s\n" - "add z10.s, z10.s, z4.s\n" + "add z9.s, z9.s, z25.s\n" + "add z10.s, z10.s, z25.s\n" ".inst 0x4482886b // srshl z11.s, p2/M, z11.s, z3.s\n" ".inst 0x4482880c // srshl z12.s, p2/M, z12.s, z0.s\n" - "add z11.s, z11.s, z4.s\n" - "add z12.s, z12.s, z4.s\n" + "add z11.s, z11.s, z25.s\n" + "add z12.s, z12.s, z25.s\n" ".inst 0x4482882d // srshl z13.s, p2/M, z13.s, z1.s\n" ".inst 0x4482884e // srshl z14.s, p2/M, z14.s, z2.s\n" - "add z13.s, z13.s, z4.s\n" - "add z14.s, z14.s, z4.s\n" + "add z13.s, z13.s, z25.s\n" + "add z14.s, z14.s, z25.s\n" ".inst 0x4482886f // srshl z15.s, p2/M, z15.s, z3.s\n" ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n" - "add z15.s, z15.s, z4.s\n" - "add z16.s, z16.s, z4.s\n" + "add z15.s, z15.s, z25.s\n" + "add z16.s, z16.s, z25.s\n" ".inst 0x44828831 // srshl z17.s, p2/M, z17.s, z1.s\n" ".inst 0x44828852 // srshl z18.s, p2/M, z18.s, z2.s\n" - "add z17.s, z17.s, z4.s\n" - "add z18.s, z18.s, z4.s\n" + "add z17.s, z17.s, z25.s\n" + "add z18.s, z18.s, z25.s\n" ".inst 0x44828873 // srshl z19.s, p2/M, z19.s, z3.s\n" ".inst 0x44828814 // srshl z20.s, p2/M, z20.s, z0.s\n" - "add z19.s, z19.s, z4.s\n" - "add z20.s, z20.s, z4.s\n" + "add z19.s, z19.s, z25.s\n" + "add z20.s, z20.s, z25.s\n" ".inst 0x44828835 // srshl z21.s, p2/M, z21.s, z1.s\n" ".inst 0x44828856 // srshl z22.s, p2/M, z22.s, z2.s\n" - "add z21.s, z21.s, z4.s\n" - "add z22.s, z22.s, z4.s\n" + "add z21.s, z21.s, z25.s\n" + "add z22.s, z22.s, z25.s\n" ".inst 0x44828877 // srshl z23.s, p2/M, z23.s, z3.s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x25]\n" - "add z23.s, z23.s, z4.s\n" - "add x25, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x25]\n" - "smin z8.s, p2/M, z8.s, z6.s\n" - "smin z9.s, p2/M, z9.s, z6.s\n" - "smin z10.s, p2/M, z10.s, z6.s\n" - "smin z11.s, p2/M, z11.s, z6.s\n" - "smin z12.s, p2/M, z12.s, z6.s\n" - "smin z13.s, p2/M, z13.s, z6.s\n" - "smin z14.s, p2/M, z14.s, z6.s\n" - "smin z15.s, p2/M, z15.s, z6.s\n" - "smin z16.s, p2/M, z16.s, z6.s\n" - "smin z17.s, p2/M, z17.s, z6.s\n" - "smin z18.s, p2/M, z18.s, z6.s\n" - "smin z19.s, p2/M, z19.s, z6.s\n" - "smin z20.s, p2/M, z20.s, z6.s\n" - "smin z21.s, p2/M, z21.s, z6.s\n" - "smin z22.s, p2/M, z22.s, z6.s\n" - "smin z23.s, p2/M, z23.s, z6.s\n" - "smax z8.s, p2/M, z8.s, z5.s\n" - "smax z9.s, p2/M, z9.s, z5.s\n" - "smax z10.s, p2/M, z10.s, z5.s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1rw { z24.s }, p2/Z, [x20]\n" + "add z23.s, z23.s, z25.s\n" + "add x20, %x[qp], %[minval]\n" + "ld1rw { z25.s }, p2/Z, [x20]\n" + "smin z8.s, p2/M, z8.s, z24.s\n" + "smin z9.s, p2/M, z9.s, z24.s\n" + "smin z10.s, p2/M, z10.s, z24.s\n" + "smin z11.s, p2/M, z11.s, z24.s\n" + "smin z12.s, p2/M, z12.s, z24.s\n" + "smin z13.s, p2/M, z13.s, z24.s\n" + "smin z14.s, p2/M, z14.s, z24.s\n" + "smin z15.s, p2/M, z15.s, z24.s\n" + "smin z16.s, p2/M, z16.s, z24.s\n" + "smin z17.s, p2/M, z17.s, z24.s\n" + "smin z18.s, p2/M, z18.s, z24.s\n" + "smin z19.s, p2/M, z19.s, z24.s\n" + "smin z20.s, p2/M, z20.s, z24.s\n" + "smin z21.s, p2/M, z21.s, z24.s\n" + "smin z22.s, p2/M, z22.s, z24.s\n" + "smin z23.s, p2/M, z23.s, z24.s\n" + "smax z8.s, p2/M, z8.s, z25.s\n" + "smax z9.s, p2/M, z9.s, z25.s\n" + "smax z10.s, p2/M, z10.s, z25.s\n" "uzp1 z8.h, z8.h, z9.h\n" - "smax z11.s, p2/M, z11.s, z5.s\n" - "smax z12.s, p2/M, z12.s, z5.s\n" - "uzp1 z9.h, z10.h, z11.h\n" - "uzp1 z8.b, z8.b, z9.b\n" - "smax z13.s, p2/M, z13.s, z5.s\n" - "smax z14.s, p2/M, z14.s, z5.s\n" + "smax z11.s, p2/M, z11.s, z25.s\n" + "smax z12.s, p2/M, z12.s, z25.s\n" + "uzp1 z24.h, z10.h, z11.h\n" + "uzp1 z8.b, z8.b, z24.b\n" + "smax z13.s, p2/M, z13.s, z25.s\n" + "smax z14.s, p2/M, z14.s, z25.s\n" "uzp1 z12.h, z12.h, z13.h\n" "st1b { z8.b }, p1, [x11]\n" - "smax z15.s, p2/M, z15.s, z5.s\n" - "smax z16.s, p2/M, z16.s, z5.s\n" - "uzp1 z13.h, z14.h, z15.h\n" - "uzp1 z12.b, z12.b, z13.b\n" - "smax z17.s, p2/M, z17.s, z5.s\n" - "smax z18.s, p2/M, z18.s, z5.s\n" + "smax z15.s, p2/M, z15.s, z25.s\n" + "smax z16.s, p2/M, z16.s, z25.s\n" + "uzp1 z24.h, z14.h, z15.h\n" + "uzp1 z12.b, z12.b, z24.b\n" + "smax z17.s, p2/M, z17.s, z25.s\n" + "smax z18.s, p2/M, z18.s, z25.s\n" "uzp1 z16.h, z16.h, z17.h\n" - "st1b { z12.b }, p1, [x24]\n" - "smax z19.s, p2/M, z19.s, z5.s\n" - "smax z20.s, p2/M, z20.s, z5.s\n" + "st1b { z12.b }, p1, [x26]\n" + "smax z19.s, p2/M, z19.s, z25.s\n" + "smax z20.s, p2/M, z20.s, z25.s\n" "uzp1 z17.h, z18.h, z19.h\n" "uzp1 z16.b, z16.b, z17.b\n" - "smax z21.s, p2/M, z21.s, z5.s\n" - "smax z22.s, p2/M, z22.s, z5.s\n" + "smax z21.s, p2/M, z21.s, z25.s\n" + "smax z22.s, p2/M, z22.s, z25.s\n" "uzp1 z20.h, z20.h, z21.h\n" - "st1b { z16.b }, p1, [x23]\n" - "smax z23.s, p2/M, z23.s, z5.s\n" - "uzp1 z21.h, z22.h, z23.h\n" - "uzp1 z20.b, z20.b, z21.b\n" - "st1b { z20.b }, p1, [x22]\n" + "st1b { z16.b }, p1, [x25]\n" + "smax z23.s, p2/M, z23.s, z25.s\n" + "uzp1 z16.h, z22.h, z23.h\n" + "uzp1 z20.b, z20.b, z16.b\n" + "st1b { z20.b }, p1, [x24]\n" "addvl x11, x11, #1\n" "52:" // Height 4: Writeback done "decw x10, ALL, MUL #4\n" @@ -1454,15 +1454,15 @@ void sve_hybrid_s8qs_dot_6x4VL ( "56:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 57f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 58f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -1473,124 +1473,124 @@ void sve_hybrid_s8qs_dot_6x4VL ( "b 58f\n" "57:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" "58:" // Height 5: input setup done "cmp x27, #0x10\n" "ble 60f\n" "59:" // Height 5: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z0.b }, p0/Z, [x26]\n" - "ld1rqb { z1.b }, p0/Z, [x25]\n" + "ld1rqb { z4.b }, p0/Z, [x26]\n" + "ld1rqb { z3.b }, p0/Z, [x25]\n" "sub x27, x27, #0x10\n" "ld1rqb { z2.b }, p0/Z, [x24]\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" + "ld1rqb { z1.b }, p0/Z, [x23]\n" "cmp x27, #0x10\n" "add x26, x26, #0x10\n" - "ld1rqb { z4.b }, p0/Z, [x22]\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "sdot z8.s, z6.b, z0.b[0]\n" - "sdot z12.s, z6.b, z1.b[0]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z16.s, z6.b, z2.b[0]\n" - "sdot z20.s, z6.b, z3.b[0]\n" + "ld1rqb { z0.b }, p0/Z, [x22]\n" + "ld1b { z29.b }, p2/Z, [x9]\n" + "sdot z8.s, z29.b, z4.b[0]\n" + "sdot z12.s, z29.b, z3.b[0]\n" + "ld1b { z28.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z16.s, z29.b, z2.b[0]\n" + "sdot z20.s, z29.b, z1.b[0]\n" "add x25, x25, #0x10\n" - "sdot z24.s, z6.b, z4.b[0]\n" - "sdot z9.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" + "sdot z24.s, z29.b, z0.b[0]\n" + "sdot z9.s, z28.b, z4.b[0]\n" + "ld1b { z29.b }, p2/Z, [x9, #2, MUL VL]\n" "add x24, x24, #0x10\n" - "sdot z13.s, z7.b, z1.b[0]\n" - "sdot z17.s, z7.b, z2.b[0]\n" + "sdot z13.s, z28.b, z3.b[0]\n" + "sdot z17.s, z28.b, z2.b[0]\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - "sdot z21.s, z7.b, z3.b[0]\n" - "sdot z25.s, z7.b, z4.b[0]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[0]\n" - "sdot z14.s, z6.b, z1.b[0]\n" - "sdot z18.s, z6.b, z2.b[0]\n" - "sdot z22.s, z6.b, z3.b[0]\n" - "sdot z26.s, z6.b, z4.b[0]\n" - "sdot z11.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p2/Z, [x9, #4, MUL VL]\n" - "sdot z15.s, z7.b, z1.b[0]\n" - "sdot z19.s, z7.b, z2.b[0]\n" - "sdot z23.s, z7.b, z3.b[0]\n" - "sdot z27.s, z7.b, z4.b[0]\n" - "ld1b { z7.b }, p2/Z, [x9, #5, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[1]\n" - "sdot z12.s, z6.b, z1.b[1]\n" - "sdot z16.s, z6.b, z2.b[1]\n" - "sdot z20.s, z6.b, z3.b[1]\n" - "sdot z24.s, z6.b, z4.b[1]\n" - "sdot z9.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p2/Z, [x9, #6, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[1]\n" - "sdot z17.s, z7.b, z2.b[1]\n" - "sdot z21.s, z7.b, z3.b[1]\n" - "sdot z25.s, z7.b, z4.b[1]\n" - "ld1b { z7.b }, p2/Z, [x9, #7, MUL VL]\n" + "sdot z21.s, z28.b, z1.b[0]\n" + "sdot z25.s, z28.b, z0.b[0]\n" + "ld1b { z28.b }, p2/Z, [x9, #3, MUL VL]\n" + "sdot z10.s, z29.b, z4.b[0]\n" + "sdot z14.s, z29.b, z3.b[0]\n" + "sdot z18.s, z29.b, z2.b[0]\n" + "sdot z22.s, z29.b, z1.b[0]\n" + "sdot z26.s, z29.b, z0.b[0]\n" + "sdot z11.s, z28.b, z4.b[0]\n" + "ld1b { z29.b }, p2/Z, [x9, #4, MUL VL]\n" + "sdot z15.s, z28.b, z3.b[0]\n" + "sdot z19.s, z28.b, z2.b[0]\n" + "sdot z23.s, z28.b, z1.b[0]\n" + "sdot z27.s, z28.b, z0.b[0]\n" + "ld1b { z28.b }, p2/Z, [x9, #5, MUL VL]\n" + "sdot z8.s, z29.b, z4.b[1]\n" + "sdot z12.s, z29.b, z3.b[1]\n" + "sdot z16.s, z29.b, z2.b[1]\n" + "sdot z20.s, z29.b, z1.b[1]\n" + "sdot z24.s, z29.b, z0.b[1]\n" + "sdot z9.s, z28.b, z4.b[1]\n" + "ld1b { z29.b }, p2/Z, [x9, #6, MUL VL]\n" + "sdot z13.s, z28.b, z3.b[1]\n" + "sdot z17.s, z28.b, z2.b[1]\n" + "sdot z21.s, z28.b, z1.b[1]\n" + "sdot z25.s, z28.b, z0.b[1]\n" + "ld1b { z28.b }, p2/Z, [x9, #7, MUL VL]\n" "addvl x9, x9, #16\n" - "sdot z10.s, z6.b, z0.b[1]\n" - "sdot z14.s, z6.b, z1.b[1]\n" - "sdot z18.s, z6.b, z2.b[1]\n" - "sdot z22.s, z6.b, z3.b[1]\n" - "sdot z26.s, z6.b, z4.b[1]\n" - "sdot z11.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p2/Z, [x9, #-8, MUL VL]\n" - "sdot z15.s, z7.b, z1.b[1]\n" - "sdot z19.s, z7.b, z2.b[1]\n" - "sdot z23.s, z7.b, z3.b[1]\n" - "sdot z27.s, z7.b, z4.b[1]\n" - "ld1b { z7.b }, p2/Z, [x9, #-7, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[2]\n" - "sdot z12.s, z6.b, z1.b[2]\n" - "sdot z16.s, z6.b, z2.b[2]\n" - "sdot z20.s, z6.b, z3.b[2]\n" - "sdot z24.s, z6.b, z4.b[2]\n" - "sdot z9.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p2/Z, [x9, #-6, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[2]\n" - "sdot z17.s, z7.b, z2.b[2]\n" - "sdot z21.s, z7.b, z3.b[2]\n" - "sdot z25.s, z7.b, z4.b[2]\n" - "ld1b { z7.b }, p2/Z, [x9, #-5, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[2]\n" - "sdot z14.s, z6.b, z1.b[2]\n" - "sdot z18.s, z6.b, z2.b[2]\n" - "sdot z22.s, z6.b, z3.b[2]\n" - "sdot z26.s, z6.b, z4.b[2]\n" - "sdot z11.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p2/Z, [x9, #-4, MUL VL]\n" - "sdot z15.s, z7.b, z1.b[2]\n" - "sdot z19.s, z7.b, z2.b[2]\n" - "sdot z23.s, z7.b, z3.b[2]\n" - "sdot z27.s, z7.b, z4.b[2]\n" - "ld1b { z7.b }, p2/Z, [x9, #-3, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[3]\n" - "sdot z12.s, z6.b, z1.b[3]\n" - "sdot z16.s, z6.b, z2.b[3]\n" - "sdot z20.s, z6.b, z3.b[3]\n" - "sdot z24.s, z6.b, z4.b[3]\n" - "sdot z9.s, z7.b, z0.b[3]\n" - "ld1b { z6.b }, p2/Z, [x9, #-2, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[3]\n" - "sdot z17.s, z7.b, z2.b[3]\n" - "sdot z21.s, z7.b, z3.b[3]\n" - "sdot z25.s, z7.b, z4.b[3]\n" - "ld1b { z7.b }, p2/Z, [x9, #-1, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[3]\n" - "sdot z14.s, z6.b, z1.b[3]\n" - "sdot z18.s, z6.b, z2.b[3]\n" - "sdot z22.s, z6.b, z3.b[3]\n" - "sdot z26.s, z6.b, z4.b[3]\n" - "sdot z11.s, z7.b, z0.b[3]\n" - "sdot z15.s, z7.b, z1.b[3]\n" - "sdot z19.s, z7.b, z2.b[3]\n" - "sdot z23.s, z7.b, z3.b[3]\n" - "sdot z27.s, z7.b, z4.b[3]\n" + "sdot z10.s, z29.b, z4.b[1]\n" + "sdot z14.s, z29.b, z3.b[1]\n" + "sdot z18.s, z29.b, z2.b[1]\n" + "sdot z22.s, z29.b, z1.b[1]\n" + "sdot z26.s, z29.b, z0.b[1]\n" + "sdot z11.s, z28.b, z4.b[1]\n" + "ld1b { z29.b }, p2/Z, [x9, #-8, MUL VL]\n" + "sdot z15.s, z28.b, z3.b[1]\n" + "sdot z19.s, z28.b, z2.b[1]\n" + "sdot z23.s, z28.b, z1.b[1]\n" + "sdot z27.s, z28.b, z0.b[1]\n" + "ld1b { z28.b }, p2/Z, [x9, #-7, MUL VL]\n" + "sdot z8.s, z29.b, z4.b[2]\n" + "sdot z12.s, z29.b, z3.b[2]\n" + "sdot z16.s, z29.b, z2.b[2]\n" + "sdot z20.s, z29.b, z1.b[2]\n" + "sdot z24.s, z29.b, z0.b[2]\n" + "sdot z9.s, z28.b, z4.b[2]\n" + "ld1b { z29.b }, p2/Z, [x9, #-6, MUL VL]\n" + "sdot z13.s, z28.b, z3.b[2]\n" + "sdot z17.s, z28.b, z2.b[2]\n" + "sdot z21.s, z28.b, z1.b[2]\n" + "sdot z25.s, z28.b, z0.b[2]\n" + "ld1b { z28.b }, p2/Z, [x9, #-5, MUL VL]\n" + "sdot z10.s, z29.b, z4.b[2]\n" + "sdot z14.s, z29.b, z3.b[2]\n" + "sdot z18.s, z29.b, z2.b[2]\n" + "sdot z22.s, z29.b, z1.b[2]\n" + "sdot z26.s, z29.b, z0.b[2]\n" + "sdot z11.s, z28.b, z4.b[2]\n" + "ld1b { z29.b }, p2/Z, [x9, #-4, MUL VL]\n" + "sdot z15.s, z28.b, z3.b[2]\n" + "sdot z19.s, z28.b, z2.b[2]\n" + "sdot z23.s, z28.b, z1.b[2]\n" + "sdot z27.s, z28.b, z0.b[2]\n" + "ld1b { z28.b }, p2/Z, [x9, #-3, MUL VL]\n" + "sdot z8.s, z29.b, z4.b[3]\n" + "sdot z12.s, z29.b, z3.b[3]\n" + "sdot z16.s, z29.b, z2.b[3]\n" + "sdot z20.s, z29.b, z1.b[3]\n" + "sdot z24.s, z29.b, z0.b[3]\n" + "sdot z9.s, z28.b, z4.b[3]\n" + "ld1b { z29.b }, p2/Z, [x9, #-2, MUL VL]\n" + "sdot z13.s, z28.b, z3.b[3]\n" + "sdot z17.s, z28.b, z2.b[3]\n" + "sdot z21.s, z28.b, z1.b[3]\n" + "sdot z25.s, z28.b, z0.b[3]\n" + "ld1b { z28.b }, p2/Z, [x9, #-1, MUL VL]\n" + "sdot z10.s, z29.b, z4.b[3]\n" + "sdot z14.s, z29.b, z3.b[3]\n" + "sdot z18.s, z29.b, z2.b[3]\n" + "sdot z22.s, z29.b, z1.b[3]\n" + "sdot z26.s, z29.b, z0.b[3]\n" + "sdot z11.s, z28.b, z4.b[3]\n" + "sdot z15.s, z28.b, z3.b[3]\n" + "sdot z19.s, z28.b, z2.b[3]\n" + "sdot z23.s, z28.b, z1.b[3]\n" + "sdot z27.s, z28.b, z0.b[3]\n" "bgt 59b\n" "60:" // Height 5: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" @@ -1600,146 +1600,146 @@ void sve_hybrid_s8qs_dot_6x4VL ( "ld1rqb { z2.b }, p0/Z, [x24]\n" "ld1rqb { z3.b }, p0/Z, [x23]\n" "ld1rqb { z4.b }, p0/Z, [x22]\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "sdot z8.s, z6.b, z0.b[0]\n" - "sdot z12.s, z6.b, z1.b[0]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z16.s, z6.b, z2.b[0]\n" - "sdot z20.s, z6.b, z3.b[0]\n" - "sdot z24.s, z6.b, z4.b[0]\n" - "sdot z9.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[0]\n" - "sdot z17.s, z7.b, z2.b[0]\n" - "sdot z21.s, z7.b, z3.b[0]\n" - "sdot z25.s, z7.b, z4.b[0]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" + "ld1b { z29.b }, p2/Z, [x9]\n" + "sdot z8.s, z29.b, z0.b[0]\n" + "sdot z12.s, z29.b, z1.b[0]\n" + "ld1b { z28.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z16.s, z29.b, z2.b[0]\n" + "sdot z20.s, z29.b, z3.b[0]\n" + "sdot z24.s, z29.b, z4.b[0]\n" + "sdot z9.s, z28.b, z0.b[0]\n" + "ld1b { z29.b }, p2/Z, [x9, #2, MUL VL]\n" + "sdot z13.s, z28.b, z1.b[0]\n" + "sdot z17.s, z28.b, z2.b[0]\n" + "sdot z21.s, z28.b, z3.b[0]\n" + "sdot z25.s, z28.b, z4.b[0]\n" + "ld1b { z28.b }, p2/Z, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "sdot z10.s, z6.b, z0.b[0]\n" - "sdot z14.s, z6.b, z1.b[0]\n" - "sdot z18.s, z6.b, z2.b[0]\n" - "sdot z22.s, z6.b, z3.b[0]\n" - "sdot z26.s, z6.b, z4.b[0]\n" - "sdot z11.s, z7.b, z0.b[0]\n" - "sdot z15.s, z7.b, z1.b[0]\n" - "sdot z19.s, z7.b, z2.b[0]\n" - "sdot z23.s, z7.b, z3.b[0]\n" - "sdot z27.s, z7.b, z4.b[0]\n" + "sdot z10.s, z29.b, z0.b[0]\n" + "sdot z14.s, z29.b, z1.b[0]\n" + "sdot z18.s, z29.b, z2.b[0]\n" + "sdot z22.s, z29.b, z3.b[0]\n" + "sdot z26.s, z29.b, z4.b[0]\n" + "sdot z11.s, z28.b, z0.b[0]\n" + "sdot z15.s, z28.b, z1.b[0]\n" + "sdot z19.s, z28.b, z2.b[0]\n" + "sdot z23.s, z28.b, z3.b[0]\n" + "sdot z27.s, z28.b, z4.b[0]\n" "ble 61f\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[1]\n" - "sdot z12.s, z6.b, z1.b[1]\n" - "sdot z16.s, z6.b, z2.b[1]\n" - "sdot z20.s, z6.b, z3.b[1]\n" + "ld1b { z29.b }, p2/Z, [x9]\n" + "ld1b { z28.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z8.s, z29.b, z0.b[1]\n" + "sdot z12.s, z29.b, z1.b[1]\n" + "sdot z16.s, z29.b, z2.b[1]\n" + "sdot z20.s, z29.b, z3.b[1]\n" "subs x27, x27, #0x4\n" - "sdot z24.s, z6.b, z4.b[1]\n" - "sdot z9.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[1]\n" - "sdot z17.s, z7.b, z2.b[1]\n" - "sdot z21.s, z7.b, z3.b[1]\n" - "sdot z25.s, z7.b, z4.b[1]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" + "sdot z24.s, z29.b, z4.b[1]\n" + "sdot z9.s, z28.b, z0.b[1]\n" + "ld1b { z29.b }, p2/Z, [x9, #2, MUL VL]\n" + "sdot z13.s, z28.b, z1.b[1]\n" + "sdot z17.s, z28.b, z2.b[1]\n" + "sdot z21.s, z28.b, z3.b[1]\n" + "sdot z25.s, z28.b, z4.b[1]\n" + "ld1b { z28.b }, p2/Z, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "sdot z10.s, z6.b, z0.b[1]\n" - "sdot z14.s, z6.b, z1.b[1]\n" - "sdot z18.s, z6.b, z2.b[1]\n" - "sdot z22.s, z6.b, z3.b[1]\n" - "sdot z26.s, z6.b, z4.b[1]\n" - "sdot z11.s, z7.b, z0.b[1]\n" - "sdot z15.s, z7.b, z1.b[1]\n" - "sdot z19.s, z7.b, z2.b[1]\n" - "sdot z23.s, z7.b, z3.b[1]\n" - "sdot z27.s, z7.b, z4.b[1]\n" + "sdot z10.s, z29.b, z0.b[1]\n" + "sdot z14.s, z29.b, z1.b[1]\n" + "sdot z18.s, z29.b, z2.b[1]\n" + "sdot z22.s, z29.b, z3.b[1]\n" + "sdot z26.s, z29.b, z4.b[1]\n" + "sdot z11.s, z28.b, z0.b[1]\n" + "sdot z15.s, z28.b, z1.b[1]\n" + "sdot z19.s, z28.b, z2.b[1]\n" + "sdot z23.s, z28.b, z3.b[1]\n" + "sdot z27.s, z28.b, z4.b[1]\n" "ble 61f\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[2]\n" - "sdot z12.s, z6.b, z1.b[2]\n" - "sdot z16.s, z6.b, z2.b[2]\n" - "sdot z20.s, z6.b, z3.b[2]\n" + "ld1b { z29.b }, p2/Z, [x9]\n" + "ld1b { z28.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z8.s, z29.b, z0.b[2]\n" + "sdot z12.s, z29.b, z1.b[2]\n" + "sdot z16.s, z29.b, z2.b[2]\n" + "sdot z20.s, z29.b, z3.b[2]\n" "subs x27, x27, #0x4\n" - "sdot z24.s, z6.b, z4.b[2]\n" - "sdot z9.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[2]\n" - "sdot z17.s, z7.b, z2.b[2]\n" - "sdot z21.s, z7.b, z3.b[2]\n" - "sdot z25.s, z7.b, z4.b[2]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" + "sdot z24.s, z29.b, z4.b[2]\n" + "sdot z9.s, z28.b, z0.b[2]\n" + "ld1b { z29.b }, p2/Z, [x9, #2, MUL VL]\n" + "sdot z13.s, z28.b, z1.b[2]\n" + "sdot z17.s, z28.b, z2.b[2]\n" + "sdot z21.s, z28.b, z3.b[2]\n" + "sdot z25.s, z28.b, z4.b[2]\n" + "ld1b { z28.b }, p2/Z, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "sdot z10.s, z6.b, z0.b[2]\n" - "sdot z14.s, z6.b, z1.b[2]\n" - "sdot z18.s, z6.b, z2.b[2]\n" - "sdot z22.s, z6.b, z3.b[2]\n" - "sdot z26.s, z6.b, z4.b[2]\n" - "sdot z11.s, z7.b, z0.b[2]\n" - "sdot z15.s, z7.b, z1.b[2]\n" - "sdot z19.s, z7.b, z2.b[2]\n" - "sdot z23.s, z7.b, z3.b[2]\n" - "sdot z27.s, z7.b, z4.b[2]\n" + "sdot z10.s, z29.b, z0.b[2]\n" + "sdot z14.s, z29.b, z1.b[2]\n" + "sdot z18.s, z29.b, z2.b[2]\n" + "sdot z22.s, z29.b, z3.b[2]\n" + "sdot z26.s, z29.b, z4.b[2]\n" + "sdot z11.s, z28.b, z0.b[2]\n" + "sdot z15.s, z28.b, z1.b[2]\n" + "sdot z19.s, z28.b, z2.b[2]\n" + "sdot z23.s, z28.b, z3.b[2]\n" + "sdot z27.s, z28.b, z4.b[2]\n" "ble 61f\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[3]\n" - "sdot z12.s, z6.b, z1.b[3]\n" - "sdot z16.s, z6.b, z2.b[3]\n" - "sdot z20.s, z6.b, z3.b[3]\n" - "sdot z24.s, z6.b, z4.b[3]\n" - "sdot z9.s, z7.b, z0.b[3]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[3]\n" - "sdot z17.s, z7.b, z2.b[3]\n" - "sdot z21.s, z7.b, z3.b[3]\n" - "sdot z25.s, z7.b, z4.b[3]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" + "ld1b { z29.b }, p2/Z, [x9]\n" + "ld1b { z28.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z8.s, z29.b, z0.b[3]\n" + "sdot z12.s, z29.b, z1.b[3]\n" + "sdot z16.s, z29.b, z2.b[3]\n" + "sdot z20.s, z29.b, z3.b[3]\n" + "sdot z24.s, z29.b, z4.b[3]\n" + "sdot z9.s, z28.b, z0.b[3]\n" + "ld1b { z29.b }, p2/Z, [x9, #2, MUL VL]\n" + "sdot z13.s, z28.b, z1.b[3]\n" + "sdot z17.s, z28.b, z2.b[3]\n" + "sdot z21.s, z28.b, z3.b[3]\n" + "sdot z25.s, z28.b, z4.b[3]\n" + "ld1b { z28.b }, p2/Z, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "sdot z10.s, z6.b, z0.b[3]\n" - "sdot z14.s, z6.b, z1.b[3]\n" - "sdot z18.s, z6.b, z2.b[3]\n" - "sdot z22.s, z6.b, z3.b[3]\n" - "sdot z26.s, z6.b, z4.b[3]\n" - "sdot z11.s, z7.b, z0.b[3]\n" - "sdot z15.s, z7.b, z1.b[3]\n" - "sdot z19.s, z7.b, z2.b[3]\n" - "sdot z23.s, z7.b, z3.b[3]\n" - "sdot z27.s, z7.b, z4.b[3]\n" + "sdot z10.s, z29.b, z0.b[3]\n" + "sdot z14.s, z29.b, z1.b[3]\n" + "sdot z18.s, z29.b, z2.b[3]\n" + "sdot z22.s, z29.b, z3.b[3]\n" + "sdot z26.s, z29.b, z4.b[3]\n" + "sdot z11.s, z28.b, z0.b[3]\n" + "sdot z15.s, z28.b, z1.b[3]\n" + "sdot z19.s, z28.b, z2.b[3]\n" + "sdot z23.s, z28.b, z3.b[3]\n" + "sdot z27.s, z28.b, z4.b[3]\n" "61:" // Height 5: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 56b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x11, x20\n" - "ld1w { z0.s }, p2/Z, [x14]\n" + "add x26, x11, x20\n" + "ld1w { z31.s }, p2/Z, [x14]\n" + "add x25, x26, x20\n" + "ld1w { z30.s }, p2/Z, [x14, #1, MUL VL]\n" + "ld1w { z29.s }, p2/Z, [x14, #2, MUL VL]\n" + "add x24, x25, x20\n" "add x23, x24, x20\n" - "ld1w { z1.s }, p2/Z, [x14, #1, MUL VL]\n" - "ld1w { z2.s }, p2/Z, [x14, #2, MUL VL]\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" - "ld1w { z3.s }, p2/Z, [x14, #3, MUL VL]\n" - "add z8.s, z8.s, z0.s\n" - "add z9.s, z9.s, z1.s\n" + "ld1w { z28.s }, p2/Z, [x14, #3, MUL VL]\n" + "add z8.s, z8.s, z31.s\n" + "add z9.s, z9.s, z30.s\n" "addvl x14, x14, #4\n" - "add z10.s, z10.s, z2.s\n" - "add z11.s, z11.s, z3.s\n" - "add z12.s, z12.s, z0.s\n" - "add z13.s, z13.s, z1.s\n" - "add z14.s, z14.s, z2.s\n" - "add z15.s, z15.s, z3.s\n" - "add z16.s, z16.s, z0.s\n" - "add z17.s, z17.s, z1.s\n" - "add z18.s, z18.s, z2.s\n" - "add z19.s, z19.s, z3.s\n" - "add z20.s, z20.s, z0.s\n" - "add z21.s, z21.s, z1.s\n" - "add z22.s, z22.s, z2.s\n" - "add z23.s, z23.s, z3.s\n" - "add z24.s, z24.s, z0.s\n" - "add z25.s, z25.s, z1.s\n" - "add z26.s, z26.s, z2.s\n" - "add z27.s, z27.s, z3.s\n" + "add z10.s, z10.s, z29.s\n" + "add z11.s, z11.s, z28.s\n" + "add z12.s, z12.s, z31.s\n" + "add z13.s, z13.s, z30.s\n" + "add z14.s, z14.s, z29.s\n" + "add z15.s, z15.s, z28.s\n" + "add z16.s, z16.s, z31.s\n" + "add z17.s, z17.s, z30.s\n" + "add z18.s, z18.s, z29.s\n" + "add z19.s, z19.s, z28.s\n" + "add z20.s, z20.s, z31.s\n" + "add z21.s, z21.s, z30.s\n" + "add z22.s, z22.s, z29.s\n" + "add z23.s, z23.s, z28.s\n" + "add z24.s, z24.s, z31.s\n" + "add z25.s, z25.s, z30.s\n" + "add z26.s, z26.s, z29.s\n" + "add z27.s, z27.s, z28.s\n" "tbz %x[flags], #4, 62f\n" "ld1w { z0.s }, p2/Z, [x12]\n" "ld1w { z4.s }, p2/Z, [x13]\n" @@ -1753,10 +1753,10 @@ void sve_hybrid_s8qs_dot_6x4VL ( "addvl x13, x13, #4\n" "b 63f\n" "62:" // Height 5: per layer parameters - "add x26, %x[qp], %[per_layer_right_shift]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1rw { z0.s }, p2/Z, [x26]\n" - "ld1rw { z4.s }, p2/Z, [x25]\n" + "add x21, %x[qp], %[per_layer_right_shift]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1rw { z0.s }, p2/Z, [x21]\n" + "ld1rw { z4.s }, p2/Z, [x20]\n" "mov z1.d, z0.d\n" "mov z5.d, z4.d\n" "mov z2.d, z0.d\n" @@ -1785,173 +1785,173 @@ void sve_hybrid_s8qs_dot_6x4VL ( ".inst 0x04a6775a // sqrdmulh z26.s, z26.s, z6.s\n" ".inst 0x04a7777b // sqrdmulh z27.s, z27.s, z7.s\n" "tbz %x[flags], #5, 64f\n" - "and z4.d, z8.d, z0.d\n" - "and z5.d, z9.d, z1.d\n" - "and z6.d, z10.d, z2.d\n" - "and z7.d, z11.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z8.s, z8.s, z4.s\n" - "sqadd z9.s, z9.s, z5.s\n" - "sqadd z10.s, z10.s, z6.s\n" - "sqadd z11.s, z11.s, z7.s\n" - "and z4.d, z12.d, z0.d\n" - "and z5.d, z13.d, z1.d\n" - "and z6.d, z14.d, z2.d\n" - "and z7.d, z15.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z12.s, z12.s, z4.s\n" - "sqadd z13.s, z13.s, z5.s\n" - "sqadd z14.s, z14.s, z6.s\n" - "sqadd z15.s, z15.s, z7.s\n" - "and z4.d, z16.d, z0.d\n" - "and z5.d, z17.d, z1.d\n" - "and z6.d, z18.d, z2.d\n" - "and z7.d, z19.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z16.s, z16.s, z4.s\n" - "sqadd z17.s, z17.s, z5.s\n" - "sqadd z18.s, z18.s, z6.s\n" - "sqadd z19.s, z19.s, z7.s\n" - "and z4.d, z20.d, z0.d\n" - "and z5.d, z21.d, z1.d\n" - "and z6.d, z22.d, z2.d\n" - "and z7.d, z23.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z20.s, z20.s, z4.s\n" - "sqadd z21.s, z21.s, z5.s\n" - "sqadd z22.s, z22.s, z6.s\n" - "sqadd z23.s, z23.s, z7.s\n" - "and z4.d, z24.d, z0.d\n" - "and z5.d, z25.d, z1.d\n" - "and z6.d, z26.d, z2.d\n" - "and z7.d, z27.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z24.s, z24.s, z4.s\n" - "sqadd z25.s, z25.s, z5.s\n" - "sqadd z26.s, z26.s, z6.s\n" - "sqadd z27.s, z27.s, z7.s\n" + "and z31.d, z8.d, z0.d\n" + "and z30.d, z9.d, z1.d\n" + "and z29.d, z10.d, z2.d\n" + "and z28.d, z11.d, z3.d\n" + "asr z31.s, z31.s, #0x1f\n" + "asr z30.s, z30.s, #0x1f\n" + "asr z29.s, z29.s, #0x1f\n" + "asr z28.s, z28.s, #0x1f\n" + "sqadd z8.s, z8.s, z31.s\n" + "sqadd z9.s, z9.s, z30.s\n" + "sqadd z10.s, z10.s, z29.s\n" + "sqadd z11.s, z11.s, z28.s\n" + "and z31.d, z12.d, z0.d\n" + "and z30.d, z13.d, z1.d\n" + "and z29.d, z14.d, z2.d\n" + "and z28.d, z15.d, z3.d\n" + "asr z31.s, z31.s, #0x1f\n" + "asr z30.s, z30.s, #0x1f\n" + "asr z29.s, z29.s, #0x1f\n" + "asr z28.s, z28.s, #0x1f\n" + "sqadd z12.s, z12.s, z31.s\n" + "sqadd z13.s, z13.s, z30.s\n" + "sqadd z14.s, z14.s, z29.s\n" + "sqadd z15.s, z15.s, z28.s\n" + "and z31.d, z16.d, z0.d\n" + "and z30.d, z17.d, z1.d\n" + "and z29.d, z18.d, z2.d\n" + "and z28.d, z19.d, z3.d\n" + "asr z31.s, z31.s, #0x1f\n" + "asr z30.s, z30.s, #0x1f\n" + "asr z29.s, z29.s, #0x1f\n" + "asr z28.s, z28.s, #0x1f\n" + "sqadd z16.s, z16.s, z31.s\n" + "sqadd z17.s, z17.s, z30.s\n" + "sqadd z18.s, z18.s, z29.s\n" + "sqadd z19.s, z19.s, z28.s\n" + "and z31.d, z20.d, z0.d\n" + "and z30.d, z21.d, z1.d\n" + "and z29.d, z22.d, z2.d\n" + "and z28.d, z23.d, z3.d\n" + "asr z31.s, z31.s, #0x1f\n" + "asr z30.s, z30.s, #0x1f\n" + "asr z29.s, z29.s, #0x1f\n" + "asr z28.s, z28.s, #0x1f\n" + "sqadd z20.s, z20.s, z31.s\n" + "sqadd z21.s, z21.s, z30.s\n" + "sqadd z22.s, z22.s, z29.s\n" + "sqadd z23.s, z23.s, z28.s\n" + "and z31.d, z24.d, z0.d\n" + "and z30.d, z25.d, z1.d\n" + "and z29.d, z26.d, z2.d\n" + "and z28.d, z27.d, z3.d\n" + "asr z31.s, z31.s, #0x1f\n" + "asr z30.s, z30.s, #0x1f\n" + "asr z29.s, z29.s, #0x1f\n" + "asr z28.s, z28.s, #0x1f\n" + "sqadd z24.s, z24.s, z31.s\n" + "sqadd z25.s, z25.s, z30.s\n" + "sqadd z26.s, z26.s, z29.s\n" + "sqadd z27.s, z27.s, z28.s\n" "64:" // Height 5: no shift correction - "add x25, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x25]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1rw { z29.s }, p2/Z, [x20]\n" ".inst 0x44828808 // srshl z8.s, p2/M, z8.s, z0.s\n" - "add z8.s, z8.s, z4.s\n" + "add z8.s, z8.s, z29.s\n" ".inst 0x44828829 // srshl z9.s, p2/M, z9.s, z1.s\n" ".inst 0x4482884a // srshl z10.s, p2/M, z10.s, z2.s\n" - "add z9.s, z9.s, z4.s\n" - "add z10.s, z10.s, z4.s\n" + "add z9.s, z9.s, z29.s\n" + "add z10.s, z10.s, z29.s\n" ".inst 0x4482886b // srshl z11.s, p2/M, z11.s, z3.s\n" ".inst 0x4482880c // srshl z12.s, p2/M, z12.s, z0.s\n" - "add z11.s, z11.s, z4.s\n" - "add z12.s, z12.s, z4.s\n" + "add z11.s, z11.s, z29.s\n" + "add z12.s, z12.s, z29.s\n" ".inst 0x4482882d // srshl z13.s, p2/M, z13.s, z1.s\n" ".inst 0x4482884e // srshl z14.s, p2/M, z14.s, z2.s\n" - "add z13.s, z13.s, z4.s\n" - "add z14.s, z14.s, z4.s\n" + "add z13.s, z13.s, z29.s\n" + "add z14.s, z14.s, z29.s\n" ".inst 0x4482886f // srshl z15.s, p2/M, z15.s, z3.s\n" ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n" - "add z15.s, z15.s, z4.s\n" - "add z16.s, z16.s, z4.s\n" + "add z15.s, z15.s, z29.s\n" + "add z16.s, z16.s, z29.s\n" ".inst 0x44828831 // srshl z17.s, p2/M, z17.s, z1.s\n" ".inst 0x44828852 // srshl z18.s, p2/M, z18.s, z2.s\n" - "add z17.s, z17.s, z4.s\n" - "add z18.s, z18.s, z4.s\n" + "add z17.s, z17.s, z29.s\n" + "add z18.s, z18.s, z29.s\n" ".inst 0x44828873 // srshl z19.s, p2/M, z19.s, z3.s\n" ".inst 0x44828814 // srshl z20.s, p2/M, z20.s, z0.s\n" - "add z19.s, z19.s, z4.s\n" - "add z20.s, z20.s, z4.s\n" + "add z19.s, z19.s, z29.s\n" + "add z20.s, z20.s, z29.s\n" ".inst 0x44828835 // srshl z21.s, p2/M, z21.s, z1.s\n" ".inst 0x44828856 // srshl z22.s, p2/M, z22.s, z2.s\n" - "add z21.s, z21.s, z4.s\n" - "add z22.s, z22.s, z4.s\n" + "add z21.s, z21.s, z29.s\n" + "add z22.s, z22.s, z29.s\n" ".inst 0x44828877 // srshl z23.s, p2/M, z23.s, z3.s\n" ".inst 0x44828818 // srshl z24.s, p2/M, z24.s, z0.s\n" - "add z23.s, z23.s, z4.s\n" - "add z24.s, z24.s, z4.s\n" + "add z23.s, z23.s, z29.s\n" + "add z24.s, z24.s, z29.s\n" ".inst 0x44828839 // srshl z25.s, p2/M, z25.s, z1.s\n" ".inst 0x4482885a // srshl z26.s, p2/M, z26.s, z2.s\n" - "add z25.s, z25.s, z4.s\n" - "add z26.s, z26.s, z4.s\n" + "add z25.s, z25.s, z29.s\n" + "add z26.s, z26.s, z29.s\n" ".inst 0x4482887b // srshl z27.s, p2/M, z27.s, z3.s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x25]\n" - "add z27.s, z27.s, z4.s\n" - "add x25, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x25]\n" - "smin z8.s, p2/M, z8.s, z6.s\n" - "smin z9.s, p2/M, z9.s, z6.s\n" - "smin z10.s, p2/M, z10.s, z6.s\n" - "smin z11.s, p2/M, z11.s, z6.s\n" - "smin z12.s, p2/M, z12.s, z6.s\n" - "smin z13.s, p2/M, z13.s, z6.s\n" - "smin z14.s, p2/M, z14.s, z6.s\n" - "smin z15.s, p2/M, z15.s, z6.s\n" - "smin z16.s, p2/M, z16.s, z6.s\n" - "smin z17.s, p2/M, z17.s, z6.s\n" - "smin z18.s, p2/M, z18.s, z6.s\n" - "smin z19.s, p2/M, z19.s, z6.s\n" - "smin z20.s, p2/M, z20.s, z6.s\n" - "smin z21.s, p2/M, z21.s, z6.s\n" - "smin z22.s, p2/M, z22.s, z6.s\n" - "smin z23.s, p2/M, z23.s, z6.s\n" - "smin z24.s, p2/M, z24.s, z6.s\n" - "smin z25.s, p2/M, z25.s, z6.s\n" - "smin z26.s, p2/M, z26.s, z6.s\n" - "smin z27.s, p2/M, z27.s, z6.s\n" - "smax z8.s, p2/M, z8.s, z5.s\n" - "smax z9.s, p2/M, z9.s, z5.s\n" - "smax z10.s, p2/M, z10.s, z5.s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1rw { z28.s }, p2/Z, [x20]\n" + "add z27.s, z27.s, z29.s\n" + "add x20, %x[qp], %[minval]\n" + "ld1rw { z29.s }, p2/Z, [x20]\n" + "smin z8.s, p2/M, z8.s, z28.s\n" + "smin z9.s, p2/M, z9.s, z28.s\n" + "smin z10.s, p2/M, z10.s, z28.s\n" + "smin z11.s, p2/M, z11.s, z28.s\n" + "smin z12.s, p2/M, z12.s, z28.s\n" + "smin z13.s, p2/M, z13.s, z28.s\n" + "smin z14.s, p2/M, z14.s, z28.s\n" + "smin z15.s, p2/M, z15.s, z28.s\n" + "smin z16.s, p2/M, z16.s, z28.s\n" + "smin z17.s, p2/M, z17.s, z28.s\n" + "smin z18.s, p2/M, z18.s, z28.s\n" + "smin z19.s, p2/M, z19.s, z28.s\n" + "smin z20.s, p2/M, z20.s, z28.s\n" + "smin z21.s, p2/M, z21.s, z28.s\n" + "smin z22.s, p2/M, z22.s, z28.s\n" + "smin z23.s, p2/M, z23.s, z28.s\n" + "smin z24.s, p2/M, z24.s, z28.s\n" + "smin z25.s, p2/M, z25.s, z28.s\n" + "smin z26.s, p2/M, z26.s, z28.s\n" + "smin z27.s, p2/M, z27.s, z28.s\n" + "smax z8.s, p2/M, z8.s, z29.s\n" + "smax z9.s, p2/M, z9.s, z29.s\n" + "smax z10.s, p2/M, z10.s, z29.s\n" "uzp1 z8.h, z8.h, z9.h\n" - "smax z11.s, p2/M, z11.s, z5.s\n" - "smax z12.s, p2/M, z12.s, z5.s\n" - "uzp1 z9.h, z10.h, z11.h\n" - "uzp1 z8.b, z8.b, z9.b\n" - "smax z13.s, p2/M, z13.s, z5.s\n" - "smax z14.s, p2/M, z14.s, z5.s\n" + "smax z11.s, p2/M, z11.s, z29.s\n" + "smax z12.s, p2/M, z12.s, z29.s\n" + "uzp1 z28.h, z10.h, z11.h\n" + "uzp1 z8.b, z8.b, z28.b\n" + "smax z13.s, p2/M, z13.s, z29.s\n" + "smax z14.s, p2/M, z14.s, z29.s\n" "uzp1 z12.h, z12.h, z13.h\n" "st1b { z8.b }, p1, [x11]\n" - "smax z15.s, p2/M, z15.s, z5.s\n" - "smax z16.s, p2/M, z16.s, z5.s\n" - "uzp1 z13.h, z14.h, z15.h\n" - "uzp1 z12.b, z12.b, z13.b\n" - "smax z17.s, p2/M, z17.s, z5.s\n" - "smax z18.s, p2/M, z18.s, z5.s\n" + "smax z15.s, p2/M, z15.s, z29.s\n" + "smax z16.s, p2/M, z16.s, z29.s\n" + "uzp1 z28.h, z14.h, z15.h\n" + "uzp1 z12.b, z12.b, z28.b\n" + "smax z17.s, p2/M, z17.s, z29.s\n" + "smax z18.s, p2/M, z18.s, z29.s\n" "uzp1 z16.h, z16.h, z17.h\n" - "st1b { z12.b }, p1, [x24]\n" - "smax z19.s, p2/M, z19.s, z5.s\n" - "smax z20.s, p2/M, z20.s, z5.s\n" + "st1b { z12.b }, p1, [x26]\n" + "smax z19.s, p2/M, z19.s, z29.s\n" + "smax z20.s, p2/M, z20.s, z29.s\n" "uzp1 z17.h, z18.h, z19.h\n" "uzp1 z16.b, z16.b, z17.b\n" - "smax z21.s, p2/M, z21.s, z5.s\n" - "smax z22.s, p2/M, z22.s, z5.s\n" + "smax z21.s, p2/M, z21.s, z29.s\n" + "smax z22.s, p2/M, z22.s, z29.s\n" "uzp1 z20.h, z20.h, z21.h\n" - "st1b { z16.b }, p1, [x23]\n" - "smax z23.s, p2/M, z23.s, z5.s\n" - "smax z24.s, p2/M, z24.s, z5.s\n" - "uzp1 z21.h, z22.h, z23.h\n" - "uzp1 z20.b, z20.b, z21.b\n" - "smax z25.s, p2/M, z25.s, z5.s\n" - "smax z26.s, p2/M, z26.s, z5.s\n" + "st1b { z16.b }, p1, [x25]\n" + "smax z23.s, p2/M, z23.s, z29.s\n" + "smax z24.s, p2/M, z24.s, z29.s\n" + "uzp1 z16.h, z22.h, z23.h\n" + "uzp1 z20.b, z20.b, z16.b\n" + "smax z25.s, p2/M, z25.s, z29.s\n" + "smax z26.s, p2/M, z26.s, z29.s\n" "uzp1 z24.h, z24.h, z25.h\n" - "st1b { z20.b }, p1, [x22]\n" - "smax z27.s, p2/M, z27.s, z5.s\n" - "uzp1 z25.h, z26.h, z27.h\n" - "uzp1 z24.b, z24.b, z25.b\n" - "st1b { z24.b }, p1, [x21]\n" + "st1b { z20.b }, p1, [x24]\n" + "smax z27.s, p2/M, z27.s, z29.s\n" + "uzp1 z16.h, z26.h, z27.h\n" + "uzp1 z24.b, z24.b, z16.b\n" + "st1b { z24.b }, p1, [x23]\n" "addvl x11, x11, #1\n" "65:" // Height 5: Writeback done "decw x10, ALL, MUL #4\n" @@ -2000,16 +2000,16 @@ void sve_hybrid_s8qs_dot_6x4VL ( "69:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 70f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 71f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -2021,143 +2021,143 @@ void sve_hybrid_s8qs_dot_6x4VL ( "b 71f\n" "70:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" + "add x21, x22, x21\n" "71:" // Height 6: input setup done "cmp x27, #0x10\n" "ble 73f\n" "72:" // Height 6: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z0.b }, p0/Z, [x26]\n" - "ld1rqb { z1.b }, p0/Z, [x25]\n" + "ld1rqb { z7.b }, p0/Z, [x26]\n" + "ld1rqb { z6.b }, p0/Z, [x25]\n" "sub x27, x27, #0x10\n" - "ld1rqb { z2.b }, p0/Z, [x24]\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" + "ld1rqb { z5.b }, p0/Z, [x24]\n" + "ld1rqb { z4.b }, p0/Z, [x23]\n" "cmp x27, #0x10\n" "add x26, x26, #0x10\n" - "ld1rqb { z4.b }, p0/Z, [x22]\n" - "ld1rqb { z5.b }, p0/Z, [x21]\n" + "ld1rqb { z3.b }, p0/Z, [x22]\n" + "ld1rqb { z2.b }, p0/Z, [x21]\n" "add x25, x25, #0x10\n" "add x24, x24, #0x10\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[0]\n" - "sdot z12.s, z6.b, z1.b[0]\n" - "sdot z16.s, z6.b, z2.b[0]\n" - "sdot z20.s, z6.b, z3.b[0]\n" + "ld1b { z1.b }, p2/Z, [x9]\n" + "ld1b { z0.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z8.s, z1.b, z7.b[0]\n" + "sdot z12.s, z1.b, z6.b[0]\n" + "sdot z16.s, z1.b, z5.b[0]\n" + "sdot z20.s, z1.b, z4.b[0]\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - "sdot z24.s, z6.b, z4.b[0]\n" - "sdot z28.s, z6.b, z5.b[0]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" + "sdot z24.s, z1.b, z3.b[0]\n" + "sdot z28.s, z1.b, z2.b[0]\n" + "ld1b { z1.b }, p2/Z, [x9, #2, MUL VL]\n" "add x21, x21, #0x10\n" - "sdot z9.s, z7.b, z0.b[0]\n" - "sdot z13.s, z7.b, z1.b[0]\n" - "sdot z17.s, z7.b, z2.b[0]\n" - "sdot z21.s, z7.b, z3.b[0]\n" - "sdot z25.s, z7.b, z4.b[0]\n" - "sdot z29.s, z7.b, z5.b[0]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[0]\n" - "sdot z14.s, z6.b, z1.b[0]\n" - "sdot z18.s, z6.b, z2.b[0]\n" - "sdot z22.s, z6.b, z3.b[0]\n" - "sdot z26.s, z6.b, z4.b[0]\n" - "sdot z30.s, z6.b, z5.b[0]\n" - "ld1b { z6.b }, p2/Z, [x9, #4, MUL VL]\n" - "sdot z11.s, z7.b, z0.b[0]\n" - "sdot z15.s, z7.b, z1.b[0]\n" - "sdot z19.s, z7.b, z2.b[0]\n" - "sdot z23.s, z7.b, z3.b[0]\n" - "sdot z27.s, z7.b, z4.b[0]\n" - "sdot z31.s, z7.b, z5.b[0]\n" - "ld1b { z7.b }, p2/Z, [x9, #5, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[1]\n" - "sdot z12.s, z6.b, z1.b[1]\n" - "sdot z16.s, z6.b, z2.b[1]\n" - "sdot z20.s, z6.b, z3.b[1]\n" - "sdot z24.s, z6.b, z4.b[1]\n" - "sdot z28.s, z6.b, z5.b[1]\n" - "ld1b { z6.b }, p2/Z, [x9, #6, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[1]\n" - "sdot z13.s, z7.b, z1.b[1]\n" - "sdot z17.s, z7.b, z2.b[1]\n" - "sdot z21.s, z7.b, z3.b[1]\n" - "sdot z25.s, z7.b, z4.b[1]\n" - "sdot z29.s, z7.b, z5.b[1]\n" - "ld1b { z7.b }, p2/Z, [x9, #7, MUL VL]\n" + "sdot z9.s, z0.b, z7.b[0]\n" + "sdot z13.s, z0.b, z6.b[0]\n" + "sdot z17.s, z0.b, z5.b[0]\n" + "sdot z21.s, z0.b, z4.b[0]\n" + "sdot z25.s, z0.b, z3.b[0]\n" + "sdot z29.s, z0.b, z2.b[0]\n" + "ld1b { z0.b }, p2/Z, [x9, #3, MUL VL]\n" + "sdot z10.s, z1.b, z7.b[0]\n" + "sdot z14.s, z1.b, z6.b[0]\n" + "sdot z18.s, z1.b, z5.b[0]\n" + "sdot z22.s, z1.b, z4.b[0]\n" + "sdot z26.s, z1.b, z3.b[0]\n" + "sdot z30.s, z1.b, z2.b[0]\n" + "ld1b { z1.b }, p2/Z, [x9, #4, MUL VL]\n" + "sdot z11.s, z0.b, z7.b[0]\n" + "sdot z15.s, z0.b, z6.b[0]\n" + "sdot z19.s, z0.b, z5.b[0]\n" + "sdot z23.s, z0.b, z4.b[0]\n" + "sdot z27.s, z0.b, z3.b[0]\n" + "sdot z31.s, z0.b, z2.b[0]\n" + "ld1b { z0.b }, p2/Z, [x9, #5, MUL VL]\n" + "sdot z8.s, z1.b, z7.b[1]\n" + "sdot z12.s, z1.b, z6.b[1]\n" + "sdot z16.s, z1.b, z5.b[1]\n" + "sdot z20.s, z1.b, z4.b[1]\n" + "sdot z24.s, z1.b, z3.b[1]\n" + "sdot z28.s, z1.b, z2.b[1]\n" + "ld1b { z1.b }, p2/Z, [x9, #6, MUL VL]\n" + "sdot z9.s, z0.b, z7.b[1]\n" + "sdot z13.s, z0.b, z6.b[1]\n" + "sdot z17.s, z0.b, z5.b[1]\n" + "sdot z21.s, z0.b, z4.b[1]\n" + "sdot z25.s, z0.b, z3.b[1]\n" + "sdot z29.s, z0.b, z2.b[1]\n" + "ld1b { z0.b }, p2/Z, [x9, #7, MUL VL]\n" "addvl x9, x9, #16\n" - "sdot z10.s, z6.b, z0.b[1]\n" - "sdot z14.s, z6.b, z1.b[1]\n" - "sdot z18.s, z6.b, z2.b[1]\n" - "sdot z22.s, z6.b, z3.b[1]\n" - "sdot z26.s, z6.b, z4.b[1]\n" - "sdot z30.s, z6.b, z5.b[1]\n" - "ld1b { z6.b }, p2/Z, [x9, #-8, MUL VL]\n" - "sdot z11.s, z7.b, z0.b[1]\n" - "sdot z15.s, z7.b, z1.b[1]\n" - "sdot z19.s, z7.b, z2.b[1]\n" - "sdot z23.s, z7.b, z3.b[1]\n" - "sdot z27.s, z7.b, z4.b[1]\n" - "sdot z31.s, z7.b, z5.b[1]\n" - "ld1b { z7.b }, p2/Z, [x9, #-7, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[2]\n" - "sdot z12.s, z6.b, z1.b[2]\n" - "sdot z16.s, z6.b, z2.b[2]\n" - "sdot z20.s, z6.b, z3.b[2]\n" - "sdot z24.s, z6.b, z4.b[2]\n" - "sdot z28.s, z6.b, z5.b[2]\n" - "ld1b { z6.b }, p2/Z, [x9, #-6, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[2]\n" - "sdot z13.s, z7.b, z1.b[2]\n" - "sdot z17.s, z7.b, z2.b[2]\n" - "sdot z21.s, z7.b, z3.b[2]\n" - "sdot z25.s, z7.b, z4.b[2]\n" - "sdot z29.s, z7.b, z5.b[2]\n" - "ld1b { z7.b }, p2/Z, [x9, #-5, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[2]\n" - "sdot z14.s, z6.b, z1.b[2]\n" - "sdot z18.s, z6.b, z2.b[2]\n" - "sdot z22.s, z6.b, z3.b[2]\n" - "sdot z26.s, z6.b, z4.b[2]\n" - "sdot z30.s, z6.b, z5.b[2]\n" - "ld1b { z6.b }, p2/Z, [x9, #-4, MUL VL]\n" - "sdot z11.s, z7.b, z0.b[2]\n" - "sdot z15.s, z7.b, z1.b[2]\n" - "sdot z19.s, z7.b, z2.b[2]\n" - "sdot z23.s, z7.b, z3.b[2]\n" - "sdot z27.s, z7.b, z4.b[2]\n" - "sdot z31.s, z7.b, z5.b[2]\n" - "ld1b { z7.b }, p2/Z, [x9, #-3, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[3]\n" - "sdot z12.s, z6.b, z1.b[3]\n" - "sdot z16.s, z6.b, z2.b[3]\n" - "sdot z20.s, z6.b, z3.b[3]\n" - "sdot z24.s, z6.b, z4.b[3]\n" - "sdot z28.s, z6.b, z5.b[3]\n" - "ld1b { z6.b }, p2/Z, [x9, #-2, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[3]\n" - "sdot z13.s, z7.b, z1.b[3]\n" - "sdot z17.s, z7.b, z2.b[3]\n" - "sdot z21.s, z7.b, z3.b[3]\n" - "sdot z25.s, z7.b, z4.b[3]\n" - "sdot z29.s, z7.b, z5.b[3]\n" - "ld1b { z7.b }, p2/Z, [x9, #-1, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[3]\n" - "sdot z14.s, z6.b, z1.b[3]\n" - "sdot z18.s, z6.b, z2.b[3]\n" - "sdot z22.s, z6.b, z3.b[3]\n" - "sdot z26.s, z6.b, z4.b[3]\n" - "sdot z30.s, z6.b, z5.b[3]\n" - "sdot z11.s, z7.b, z0.b[3]\n" - "sdot z15.s, z7.b, z1.b[3]\n" - "sdot z19.s, z7.b, z2.b[3]\n" - "sdot z23.s, z7.b, z3.b[3]\n" - "sdot z27.s, z7.b, z4.b[3]\n" - "sdot z31.s, z7.b, z5.b[3]\n" + "sdot z10.s, z1.b, z7.b[1]\n" + "sdot z14.s, z1.b, z6.b[1]\n" + "sdot z18.s, z1.b, z5.b[1]\n" + "sdot z22.s, z1.b, z4.b[1]\n" + "sdot z26.s, z1.b, z3.b[1]\n" + "sdot z30.s, z1.b, z2.b[1]\n" + "ld1b { z1.b }, p2/Z, [x9, #-8, MUL VL]\n" + "sdot z11.s, z0.b, z7.b[1]\n" + "sdot z15.s, z0.b, z6.b[1]\n" + "sdot z19.s, z0.b, z5.b[1]\n" + "sdot z23.s, z0.b, z4.b[1]\n" + "sdot z27.s, z0.b, z3.b[1]\n" + "sdot z31.s, z0.b, z2.b[1]\n" + "ld1b { z0.b }, p2/Z, [x9, #-7, MUL VL]\n" + "sdot z8.s, z1.b, z7.b[2]\n" + "sdot z12.s, z1.b, z6.b[2]\n" + "sdot z16.s, z1.b, z5.b[2]\n" + "sdot z20.s, z1.b, z4.b[2]\n" + "sdot z24.s, z1.b, z3.b[2]\n" + "sdot z28.s, z1.b, z2.b[2]\n" + "ld1b { z1.b }, p2/Z, [x9, #-6, MUL VL]\n" + "sdot z9.s, z0.b, z7.b[2]\n" + "sdot z13.s, z0.b, z6.b[2]\n" + "sdot z17.s, z0.b, z5.b[2]\n" + "sdot z21.s, z0.b, z4.b[2]\n" + "sdot z25.s, z0.b, z3.b[2]\n" + "sdot z29.s, z0.b, z2.b[2]\n" + "ld1b { z0.b }, p2/Z, [x9, #-5, MUL VL]\n" + "sdot z10.s, z1.b, z7.b[2]\n" + "sdot z14.s, z1.b, z6.b[2]\n" + "sdot z18.s, z1.b, z5.b[2]\n" + "sdot z22.s, z1.b, z4.b[2]\n" + "sdot z26.s, z1.b, z3.b[2]\n" + "sdot z30.s, z1.b, z2.b[2]\n" + "ld1b { z1.b }, p2/Z, [x9, #-4, MUL VL]\n" + "sdot z11.s, z0.b, z7.b[2]\n" + "sdot z15.s, z0.b, z6.b[2]\n" + "sdot z19.s, z0.b, z5.b[2]\n" + "sdot z23.s, z0.b, z4.b[2]\n" + "sdot z27.s, z0.b, z3.b[2]\n" + "sdot z31.s, z0.b, z2.b[2]\n" + "ld1b { z0.b }, p2/Z, [x9, #-3, MUL VL]\n" + "sdot z8.s, z1.b, z7.b[3]\n" + "sdot z12.s, z1.b, z6.b[3]\n" + "sdot z16.s, z1.b, z5.b[3]\n" + "sdot z20.s, z1.b, z4.b[3]\n" + "sdot z24.s, z1.b, z3.b[3]\n" + "sdot z28.s, z1.b, z2.b[3]\n" + "ld1b { z1.b }, p2/Z, [x9, #-2, MUL VL]\n" + "sdot z9.s, z0.b, z7.b[3]\n" + "sdot z13.s, z0.b, z6.b[3]\n" + "sdot z17.s, z0.b, z5.b[3]\n" + "sdot z21.s, z0.b, z4.b[3]\n" + "sdot z25.s, z0.b, z3.b[3]\n" + "sdot z29.s, z0.b, z2.b[3]\n" + "ld1b { z0.b }, p2/Z, [x9, #-1, MUL VL]\n" + "sdot z10.s, z1.b, z7.b[3]\n" + "sdot z14.s, z1.b, z6.b[3]\n" + "sdot z18.s, z1.b, z5.b[3]\n" + "sdot z22.s, z1.b, z4.b[3]\n" + "sdot z26.s, z1.b, z3.b[3]\n" + "sdot z30.s, z1.b, z2.b[3]\n" + "sdot z11.s, z0.b, z7.b[3]\n" + "sdot z15.s, z0.b, z6.b[3]\n" + "sdot z19.s, z0.b, z5.b[3]\n" + "sdot z23.s, z0.b, z4.b[3]\n" + "sdot z27.s, z0.b, z3.b[3]\n" + "sdot z31.s, z0.b, z2.b[3]\n" "bgt 72b\n" "73:" // Height 6: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" @@ -2168,167 +2168,167 @@ void sve_hybrid_s8qs_dot_6x4VL ( "ld1rqb { z3.b }, p0/Z, [x23]\n" "ld1rqb { z4.b }, p0/Z, [x22]\n" "ld1rqb { z5.b }, p0/Z, [x21]\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[0]\n" - "sdot z12.s, z6.b, z1.b[0]\n" - "sdot z16.s, z6.b, z2.b[0]\n" - "sdot z20.s, z6.b, z3.b[0]\n" - "sdot z24.s, z6.b, z4.b[0]\n" - "sdot z28.s, z6.b, z5.b[0]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[0]\n" - "sdot z13.s, z7.b, z1.b[0]\n" - "sdot z17.s, z7.b, z2.b[0]\n" - "sdot z21.s, z7.b, z3.b[0]\n" - "sdot z25.s, z7.b, z4.b[0]\n" - "sdot z29.s, z7.b, z5.b[0]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" + "ld1b { z7.b }, p2/Z, [x9]\n" + "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z8.s, z7.b, z0.b[0]\n" + "sdot z12.s, z7.b, z1.b[0]\n" + "sdot z16.s, z7.b, z2.b[0]\n" + "sdot z20.s, z7.b, z3.b[0]\n" + "sdot z24.s, z7.b, z4.b[0]\n" + "sdot z28.s, z7.b, z5.b[0]\n" + "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n" + "sdot z9.s, z6.b, z0.b[0]\n" + "sdot z13.s, z6.b, z1.b[0]\n" + "sdot z17.s, z6.b, z2.b[0]\n" + "sdot z21.s, z6.b, z3.b[0]\n" + "sdot z25.s, z6.b, z4.b[0]\n" + "sdot z29.s, z6.b, z5.b[0]\n" + "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "sdot z10.s, z6.b, z0.b[0]\n" - "sdot z14.s, z6.b, z1.b[0]\n" - "sdot z18.s, z6.b, z2.b[0]\n" - "sdot z22.s, z6.b, z3.b[0]\n" - "sdot z26.s, z6.b, z4.b[0]\n" - "sdot z30.s, z6.b, z5.b[0]\n" - "sdot z11.s, z7.b, z0.b[0]\n" - "sdot z15.s, z7.b, z1.b[0]\n" - "sdot z19.s, z7.b, z2.b[0]\n" - "sdot z23.s, z7.b, z3.b[0]\n" - "sdot z27.s, z7.b, z4.b[0]\n" - "sdot z31.s, z7.b, z5.b[0]\n" + "sdot z10.s, z7.b, z0.b[0]\n" + "sdot z14.s, z7.b, z1.b[0]\n" + "sdot z18.s, z7.b, z2.b[0]\n" + "sdot z22.s, z7.b, z3.b[0]\n" + "sdot z26.s, z7.b, z4.b[0]\n" + "sdot z30.s, z7.b, z5.b[0]\n" + "sdot z11.s, z6.b, z0.b[0]\n" + "sdot z15.s, z6.b, z1.b[0]\n" + "sdot z19.s, z6.b, z2.b[0]\n" + "sdot z23.s, z6.b, z3.b[0]\n" + "sdot z27.s, z6.b, z4.b[0]\n" + "sdot z31.s, z6.b, z5.b[0]\n" "ble 74f\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[1]\n" - "sdot z12.s, z6.b, z1.b[1]\n" - "sdot z16.s, z6.b, z2.b[1]\n" - "sdot z20.s, z6.b, z3.b[1]\n" + "ld1b { z7.b }, p2/Z, [x9]\n" + "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z8.s, z7.b, z0.b[1]\n" + "sdot z12.s, z7.b, z1.b[1]\n" + "sdot z16.s, z7.b, z2.b[1]\n" + "sdot z20.s, z7.b, z3.b[1]\n" "subs x27, x27, #0x4\n" - "sdot z24.s, z6.b, z4.b[1]\n" - "sdot z28.s, z6.b, z5.b[1]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[1]\n" - "sdot z13.s, z7.b, z1.b[1]\n" - "sdot z17.s, z7.b, z2.b[1]\n" - "sdot z21.s, z7.b, z3.b[1]\n" - "sdot z25.s, z7.b, z4.b[1]\n" - "sdot z29.s, z7.b, z5.b[1]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" + "sdot z24.s, z7.b, z4.b[1]\n" + "sdot z28.s, z7.b, z5.b[1]\n" + "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n" + "sdot z9.s, z6.b, z0.b[1]\n" + "sdot z13.s, z6.b, z1.b[1]\n" + "sdot z17.s, z6.b, z2.b[1]\n" + "sdot z21.s, z6.b, z3.b[1]\n" + "sdot z25.s, z6.b, z4.b[1]\n" + "sdot z29.s, z6.b, z5.b[1]\n" + "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "sdot z10.s, z6.b, z0.b[1]\n" - "sdot z14.s, z6.b, z1.b[1]\n" - "sdot z18.s, z6.b, z2.b[1]\n" - "sdot z22.s, z6.b, z3.b[1]\n" - "sdot z26.s, z6.b, z4.b[1]\n" - "sdot z30.s, z6.b, z5.b[1]\n" - "sdot z11.s, z7.b, z0.b[1]\n" - "sdot z15.s, z7.b, z1.b[1]\n" - "sdot z19.s, z7.b, z2.b[1]\n" - "sdot z23.s, z7.b, z3.b[1]\n" - "sdot z27.s, z7.b, z4.b[1]\n" - "sdot z31.s, z7.b, z5.b[1]\n" + "sdot z10.s, z7.b, z0.b[1]\n" + "sdot z14.s, z7.b, z1.b[1]\n" + "sdot z18.s, z7.b, z2.b[1]\n" + "sdot z22.s, z7.b, z3.b[1]\n" + "sdot z26.s, z7.b, z4.b[1]\n" + "sdot z30.s, z7.b, z5.b[1]\n" + "sdot z11.s, z6.b, z0.b[1]\n" + "sdot z15.s, z6.b, z1.b[1]\n" + "sdot z19.s, z6.b, z2.b[1]\n" + "sdot z23.s, z6.b, z3.b[1]\n" + "sdot z27.s, z6.b, z4.b[1]\n" + "sdot z31.s, z6.b, z5.b[1]\n" "ble 74f\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[2]\n" - "sdot z12.s, z6.b, z1.b[2]\n" - "sdot z16.s, z6.b, z2.b[2]\n" - "sdot z20.s, z6.b, z3.b[2]\n" + "ld1b { z7.b }, p2/Z, [x9]\n" + "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z8.s, z7.b, z0.b[2]\n" + "sdot z12.s, z7.b, z1.b[2]\n" + "sdot z16.s, z7.b, z2.b[2]\n" + "sdot z20.s, z7.b, z3.b[2]\n" "subs x27, x27, #0x4\n" - "sdot z24.s, z6.b, z4.b[2]\n" - "sdot z28.s, z6.b, z5.b[2]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[2]\n" - "sdot z13.s, z7.b, z1.b[2]\n" - "sdot z17.s, z7.b, z2.b[2]\n" - "sdot z21.s, z7.b, z3.b[2]\n" - "sdot z25.s, z7.b, z4.b[2]\n" - "sdot z29.s, z7.b, z5.b[2]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" + "sdot z24.s, z7.b, z4.b[2]\n" + "sdot z28.s, z7.b, z5.b[2]\n" + "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n" + "sdot z9.s, z6.b, z0.b[2]\n" + "sdot z13.s, z6.b, z1.b[2]\n" + "sdot z17.s, z6.b, z2.b[2]\n" + "sdot z21.s, z6.b, z3.b[2]\n" + "sdot z25.s, z6.b, z4.b[2]\n" + "sdot z29.s, z6.b, z5.b[2]\n" + "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "sdot z10.s, z6.b, z0.b[2]\n" - "sdot z14.s, z6.b, z1.b[2]\n" - "sdot z18.s, z6.b, z2.b[2]\n" - "sdot z22.s, z6.b, z3.b[2]\n" - "sdot z26.s, z6.b, z4.b[2]\n" - "sdot z30.s, z6.b, z5.b[2]\n" - "sdot z11.s, z7.b, z0.b[2]\n" - "sdot z15.s, z7.b, z1.b[2]\n" - "sdot z19.s, z7.b, z2.b[2]\n" - "sdot z23.s, z7.b, z3.b[2]\n" - "sdot z27.s, z7.b, z4.b[2]\n" - "sdot z31.s, z7.b, z5.b[2]\n" + "sdot z10.s, z7.b, z0.b[2]\n" + "sdot z14.s, z7.b, z1.b[2]\n" + "sdot z18.s, z7.b, z2.b[2]\n" + "sdot z22.s, z7.b, z3.b[2]\n" + "sdot z26.s, z7.b, z4.b[2]\n" + "sdot z30.s, z7.b, z5.b[2]\n" + "sdot z11.s, z6.b, z0.b[2]\n" + "sdot z15.s, z6.b, z1.b[2]\n" + "sdot z19.s, z6.b, z2.b[2]\n" + "sdot z23.s, z6.b, z3.b[2]\n" + "sdot z27.s, z6.b, z4.b[2]\n" + "sdot z31.s, z6.b, z5.b[2]\n" "ble 74f\n" - "ld1b { z6.b }, p2/Z, [x9]\n" - "ld1b { z7.b }, p2/Z, [x9, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[3]\n" - "sdot z12.s, z6.b, z1.b[3]\n" - "sdot z16.s, z6.b, z2.b[3]\n" - "sdot z20.s, z6.b, z3.b[3]\n" - "sdot z24.s, z6.b, z4.b[3]\n" - "sdot z28.s, z6.b, z5.b[3]\n" - "ld1b { z6.b }, p2/Z, [x9, #2, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[3]\n" - "sdot z13.s, z7.b, z1.b[3]\n" - "sdot z17.s, z7.b, z2.b[3]\n" - "sdot z21.s, z7.b, z3.b[3]\n" - "sdot z25.s, z7.b, z4.b[3]\n" - "sdot z29.s, z7.b, z5.b[3]\n" - "ld1b { z7.b }, p2/Z, [x9, #3, MUL VL]\n" + "ld1b { z7.b }, p2/Z, [x9]\n" + "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n" + "sdot z8.s, z7.b, z0.b[3]\n" + "sdot z12.s, z7.b, z1.b[3]\n" + "sdot z16.s, z7.b, z2.b[3]\n" + "sdot z20.s, z7.b, z3.b[3]\n" + "sdot z24.s, z7.b, z4.b[3]\n" + "sdot z28.s, z7.b, z5.b[3]\n" + "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n" + "sdot z9.s, z6.b, z0.b[3]\n" + "sdot z13.s, z6.b, z1.b[3]\n" + "sdot z17.s, z6.b, z2.b[3]\n" + "sdot z21.s, z6.b, z3.b[3]\n" + "sdot z25.s, z6.b, z4.b[3]\n" + "sdot z29.s, z6.b, z5.b[3]\n" + "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "sdot z10.s, z6.b, z0.b[3]\n" - "sdot z14.s, z6.b, z1.b[3]\n" - "sdot z18.s, z6.b, z2.b[3]\n" - "sdot z22.s, z6.b, z3.b[3]\n" - "sdot z26.s, z6.b, z4.b[3]\n" - "sdot z30.s, z6.b, z5.b[3]\n" - "sdot z11.s, z7.b, z0.b[3]\n" - "sdot z15.s, z7.b, z1.b[3]\n" - "sdot z19.s, z7.b, z2.b[3]\n" - "sdot z23.s, z7.b, z3.b[3]\n" - "sdot z27.s, z7.b, z4.b[3]\n" - "sdot z31.s, z7.b, z5.b[3]\n" + "sdot z10.s, z7.b, z0.b[3]\n" + "sdot z14.s, z7.b, z1.b[3]\n" + "sdot z18.s, z7.b, z2.b[3]\n" + "sdot z22.s, z7.b, z3.b[3]\n" + "sdot z26.s, z7.b, z4.b[3]\n" + "sdot z30.s, z7.b, z5.b[3]\n" + "sdot z11.s, z6.b, z0.b[3]\n" + "sdot z15.s, z6.b, z1.b[3]\n" + "sdot z19.s, z6.b, z2.b[3]\n" + "sdot z23.s, z6.b, z3.b[3]\n" + "sdot z27.s, z6.b, z4.b[3]\n" + "sdot z31.s, z6.b, z5.b[3]\n" "74:" // Height 6: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 69b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x11, x20\n" + "add x26, x11, x20\n" + "add x25, x26, x20\n" + "ld1w { z3.s }, p2/Z, [x14]\n" + "ld1w { z2.s }, p2/Z, [x14, #1, MUL VL]\n" + "ld1w { z1.s }, p2/Z, [x14, #2, MUL VL]\n" + "add x24, x25, x20\n" "add x23, x24, x20\n" - "ld1w { z0.s }, p2/Z, [x14]\n" - "ld1w { z1.s }, p2/Z, [x14, #1, MUL VL]\n" - "ld1w { z2.s }, p2/Z, [x14, #2, MUL VL]\n" + "ld1w { z0.s }, p2/Z, [x14, #3, MUL VL]\n" "add x22, x23, x20\n" - "add x21, x22, x20\n" - "ld1w { z3.s }, p2/Z, [x14, #3, MUL VL]\n" - "add x20, x21, x20\n" - "add z8.s, z8.s, z0.s\n" - "add z9.s, z9.s, z1.s\n" - "add z10.s, z10.s, z2.s\n" - "add z11.s, z11.s, z3.s\n" + "add z8.s, z8.s, z3.s\n" + "add z9.s, z9.s, z2.s\n" + "add z10.s, z10.s, z1.s\n" + "add z11.s, z11.s, z0.s\n" "addvl x14, x14, #4\n" - "add z12.s, z12.s, z0.s\n" - "add z13.s, z13.s, z1.s\n" - "add z14.s, z14.s, z2.s\n" - "add z15.s, z15.s, z3.s\n" - "add z16.s, z16.s, z0.s\n" - "add z17.s, z17.s, z1.s\n" - "add z18.s, z18.s, z2.s\n" - "add z19.s, z19.s, z3.s\n" - "add z20.s, z20.s, z0.s\n" - "add z21.s, z21.s, z1.s\n" - "add z22.s, z22.s, z2.s\n" - "add z23.s, z23.s, z3.s\n" - "add z24.s, z24.s, z0.s\n" - "add z25.s, z25.s, z1.s\n" - "add z26.s, z26.s, z2.s\n" - "add z27.s, z27.s, z3.s\n" - "add z28.s, z28.s, z0.s\n" - "add z29.s, z29.s, z1.s\n" - "add z30.s, z30.s, z2.s\n" - "add z31.s, z31.s, z3.s\n" + "add z12.s, z12.s, z3.s\n" + "add z13.s, z13.s, z2.s\n" + "add z14.s, z14.s, z1.s\n" + "add z15.s, z15.s, z0.s\n" + "add z16.s, z16.s, z3.s\n" + "add z17.s, z17.s, z2.s\n" + "add z18.s, z18.s, z1.s\n" + "add z19.s, z19.s, z0.s\n" + "add z20.s, z20.s, z3.s\n" + "add z21.s, z21.s, z2.s\n" + "add z22.s, z22.s, z1.s\n" + "add z23.s, z23.s, z0.s\n" + "add z24.s, z24.s, z3.s\n" + "add z25.s, z25.s, z2.s\n" + "add z26.s, z26.s, z1.s\n" + "add z27.s, z27.s, z0.s\n" + "add z28.s, z28.s, z3.s\n" + "add z29.s, z29.s, z2.s\n" + "add z30.s, z30.s, z1.s\n" + "add z31.s, z31.s, z0.s\n" "tbz %x[flags], #4, 75f\n" "ld1w { z0.s }, p2/Z, [x12]\n" "ld1w { z4.s }, p2/Z, [x13]\n" @@ -2342,10 +2342,10 @@ void sve_hybrid_s8qs_dot_6x4VL ( "addvl x13, x13, #4\n" "b 76f\n" "75:" // Height 6: per layer parameters - "add x26, %x[qp], %[per_layer_right_shift]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1rw { z0.s }, p2/Z, [x26]\n" - "ld1rw { z4.s }, p2/Z, [x25]\n" + "add x21, %x[qp], %[per_layer_right_shift]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1rw { z0.s }, p2/Z, [x21]\n" + "ld1rw { z4.s }, p2/Z, [x20]\n" "mov z1.d, z0.d\n" "mov z5.d, z4.d\n" "mov z2.d, z0.d\n" @@ -2378,81 +2378,81 @@ void sve_hybrid_s8qs_dot_6x4VL ( ".inst 0x04a677de // sqrdmulh z30.s, z30.s, z6.s\n" ".inst 0x04a777ff // sqrdmulh z31.s, z31.s, z7.s\n" "tbz %x[flags], #5, 77f\n" - "and z4.d, z8.d, z0.d\n" - "and z5.d, z9.d, z1.d\n" - "and z6.d, z10.d, z2.d\n" - "and z7.d, z11.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" + "and z7.d, z8.d, z0.d\n" + "and z6.d, z9.d, z1.d\n" + "and z5.d, z10.d, z2.d\n" + "and z4.d, z11.d, z3.d\n" "asr z7.s, z7.s, #0x1f\n" - "sqadd z8.s, z8.s, z4.s\n" - "sqadd z9.s, z9.s, z5.s\n" - "sqadd z10.s, z10.s, z6.s\n" - "sqadd z11.s, z11.s, z7.s\n" - "and z4.d, z12.d, z0.d\n" - "and z5.d, z13.d, z1.d\n" - "and z6.d, z14.d, z2.d\n" - "and z7.d, z15.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z12.s, z12.s, z4.s\n" - "sqadd z13.s, z13.s, z5.s\n" - "sqadd z14.s, z14.s, z6.s\n" - "sqadd z15.s, z15.s, z7.s\n" - "and z4.d, z16.d, z0.d\n" - "and z5.d, z17.d, z1.d\n" - "and z6.d, z18.d, z2.d\n" - "and z7.d, z19.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z16.s, z16.s, z4.s\n" - "sqadd z17.s, z17.s, z5.s\n" - "sqadd z18.s, z18.s, z6.s\n" - "sqadd z19.s, z19.s, z7.s\n" - "and z4.d, z20.d, z0.d\n" - "and z5.d, z21.d, z1.d\n" - "and z6.d, z22.d, z2.d\n" - "and z7.d, z23.d, z3.d\n" "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" + "sqadd z8.s, z8.s, z7.s\n" + "sqadd z9.s, z9.s, z6.s\n" + "sqadd z10.s, z10.s, z5.s\n" + "sqadd z11.s, z11.s, z4.s\n" + "and z7.d, z12.d, z0.d\n" + "and z6.d, z13.d, z1.d\n" + "and z5.d, z14.d, z2.d\n" + "and z4.d, z15.d, z3.d\n" "asr z7.s, z7.s, #0x1f\n" - "sqadd z20.s, z20.s, z4.s\n" - "sqadd z21.s, z21.s, z5.s\n" - "sqadd z22.s, z22.s, z6.s\n" - "sqadd z23.s, z23.s, z7.s\n" - "and z4.d, z24.d, z0.d\n" - "and z5.d, z25.d, z1.d\n" - "and z6.d, z26.d, z2.d\n" - "and z7.d, z27.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" "asr z6.s, z6.s, #0x1f\n" + "asr z5.s, z5.s, #0x1f\n" + "asr z4.s, z4.s, #0x1f\n" + "sqadd z12.s, z12.s, z7.s\n" + "sqadd z13.s, z13.s, z6.s\n" + "sqadd z14.s, z14.s, z5.s\n" + "sqadd z15.s, z15.s, z4.s\n" + "and z7.d, z16.d, z0.d\n" + "and z6.d, z17.d, z1.d\n" + "and z5.d, z18.d, z2.d\n" + "and z4.d, z19.d, z3.d\n" "asr z7.s, z7.s, #0x1f\n" - "sqadd z24.s, z24.s, z4.s\n" - "sqadd z25.s, z25.s, z5.s\n" - "sqadd z26.s, z26.s, z6.s\n" - "sqadd z27.s, z27.s, z7.s\n" - "and z4.d, z28.d, z0.d\n" - "and z5.d, z29.d, z1.d\n" - "and z6.d, z30.d, z2.d\n" - "and z7.d, z31.d, z3.d\n" + "asr z6.s, z6.s, #0x1f\n" + "asr z5.s, z5.s, #0x1f\n" "asr z4.s, z4.s, #0x1f\n" + "sqadd z16.s, z16.s, z7.s\n" + "sqadd z17.s, z17.s, z6.s\n" + "sqadd z18.s, z18.s, z5.s\n" + "sqadd z19.s, z19.s, z4.s\n" + "and z7.d, z20.d, z0.d\n" + "and z6.d, z21.d, z1.d\n" + "and z5.d, z22.d, z2.d\n" + "and z4.d, z23.d, z3.d\n" + "asr z7.s, z7.s, #0x1f\n" + "asr z6.s, z6.s, #0x1f\n" "asr z5.s, z5.s, #0x1f\n" + "asr z4.s, z4.s, #0x1f\n" + "sqadd z20.s, z20.s, z7.s\n" + "sqadd z21.s, z21.s, z6.s\n" + "sqadd z22.s, z22.s, z5.s\n" + "sqadd z23.s, z23.s, z4.s\n" + "and z7.d, z24.d, z0.d\n" + "and z6.d, z25.d, z1.d\n" + "and z5.d, z26.d, z2.d\n" + "and z4.d, z27.d, z3.d\n" + "asr z7.s, z7.s, #0x1f\n" "asr z6.s, z6.s, #0x1f\n" + "asr z5.s, z5.s, #0x1f\n" + "asr z4.s, z4.s, #0x1f\n" + "sqadd z24.s, z24.s, z7.s\n" + "sqadd z25.s, z25.s, z6.s\n" + "sqadd z26.s, z26.s, z5.s\n" + "sqadd z27.s, z27.s, z4.s\n" + "and z7.d, z28.d, z0.d\n" + "and z6.d, z29.d, z1.d\n" + "and z5.d, z30.d, z2.d\n" + "and z4.d, z31.d, z3.d\n" "asr z7.s, z7.s, #0x1f\n" - "sqadd z28.s, z28.s, z4.s\n" - "sqadd z29.s, z29.s, z5.s\n" - "sqadd z30.s, z30.s, z6.s\n" - "sqadd z31.s, z31.s, z7.s\n" + "asr z6.s, z6.s, #0x1f\n" + "asr z5.s, z5.s, #0x1f\n" + "asr z4.s, z4.s, #0x1f\n" + "sqadd z28.s, z28.s, z7.s\n" + "sqadd z29.s, z29.s, z6.s\n" + "sqadd z30.s, z30.s, z5.s\n" + "sqadd z31.s, z31.s, z4.s\n" "77:" // Height 6: no shift correction - "add x25, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x25]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1rw { z4.s }, p2/Z, [x20]\n" ".inst 0x44828808 // srshl z8.s, p2/M, z8.s, z0.s\n" "add z8.s, z8.s, z4.s\n" ".inst 0x44828829 // srshl z9.s, p2/M, z9.s, z1.s\n" @@ -2500,83 +2500,83 @@ void sve_hybrid_s8qs_dot_6x4VL ( "add z29.s, z29.s, z4.s\n" "add z30.s, z30.s, z4.s\n" ".inst 0x4482887f // srshl z31.s, p2/M, z31.s, z3.s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x25]\n" + "add x20, %x[qp], %[maxval]\n" + "ld1rw { z0.s }, p2/Z, [x20]\n" "add z31.s, z31.s, z4.s\n" - "add x25, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x25]\n" - "smin z8.s, p2/M, z8.s, z6.s\n" - "smin z9.s, p2/M, z9.s, z6.s\n" - "smin z10.s, p2/M, z10.s, z6.s\n" - "smin z11.s, p2/M, z11.s, z6.s\n" - "smin z12.s, p2/M, z12.s, z6.s\n" - "smin z13.s, p2/M, z13.s, z6.s\n" - "smin z14.s, p2/M, z14.s, z6.s\n" - "smin z15.s, p2/M, z15.s, z6.s\n" - "smin z16.s, p2/M, z16.s, z6.s\n" - "smin z17.s, p2/M, z17.s, z6.s\n" - "smin z18.s, p2/M, z18.s, z6.s\n" - "smin z19.s, p2/M, z19.s, z6.s\n" - "smin z20.s, p2/M, z20.s, z6.s\n" - "smin z21.s, p2/M, z21.s, z6.s\n" - "smin z22.s, p2/M, z22.s, z6.s\n" - "smin z23.s, p2/M, z23.s, z6.s\n" - "smin z24.s, p2/M, z24.s, z6.s\n" - "smin z25.s, p2/M, z25.s, z6.s\n" - "smin z26.s, p2/M, z26.s, z6.s\n" - "smin z27.s, p2/M, z27.s, z6.s\n" - "smin z28.s, p2/M, z28.s, z6.s\n" - "smin z29.s, p2/M, z29.s, z6.s\n" - "smin z30.s, p2/M, z30.s, z6.s\n" - "smin z31.s, p2/M, z31.s, z6.s\n" - "smax z8.s, p2/M, z8.s, z5.s\n" - "smax z9.s, p2/M, z9.s, z5.s\n" - "smax z10.s, p2/M, z10.s, z5.s\n" + "add x20, %x[qp], %[minval]\n" + "ld1rw { z1.s }, p2/Z, [x20]\n" + "smin z8.s, p2/M, z8.s, z0.s\n" + "smin z9.s, p2/M, z9.s, z0.s\n" + "smin z10.s, p2/M, z10.s, z0.s\n" + "smin z11.s, p2/M, z11.s, z0.s\n" + "smin z12.s, p2/M, z12.s, z0.s\n" + "smin z13.s, p2/M, z13.s, z0.s\n" + "smin z14.s, p2/M, z14.s, z0.s\n" + "smin z15.s, p2/M, z15.s, z0.s\n" + "smin z16.s, p2/M, z16.s, z0.s\n" + "smin z17.s, p2/M, z17.s, z0.s\n" + "smin z18.s, p2/M, z18.s, z0.s\n" + "smin z19.s, p2/M, z19.s, z0.s\n" + "smin z20.s, p2/M, z20.s, z0.s\n" + "smin z21.s, p2/M, z21.s, z0.s\n" + "smin z22.s, p2/M, z22.s, z0.s\n" + "smin z23.s, p2/M, z23.s, z0.s\n" + "smin z24.s, p2/M, z24.s, z0.s\n" + "smin z25.s, p2/M, z25.s, z0.s\n" + "smin z26.s, p2/M, z26.s, z0.s\n" + "smin z27.s, p2/M, z27.s, z0.s\n" + "smin z28.s, p2/M, z28.s, z0.s\n" + "smin z29.s, p2/M, z29.s, z0.s\n" + "smin z30.s, p2/M, z30.s, z0.s\n" + "smin z31.s, p2/M, z31.s, z0.s\n" + "smax z8.s, p2/M, z8.s, z1.s\n" + "smax z9.s, p2/M, z9.s, z1.s\n" + "smax z10.s, p2/M, z10.s, z1.s\n" "uzp1 z8.h, z8.h, z9.h\n" - "smax z11.s, p2/M, z11.s, z5.s\n" - "smax z12.s, p2/M, z12.s, z5.s\n" - "uzp1 z9.h, z10.h, z11.h\n" - "uzp1 z8.b, z8.b, z9.b\n" - "smax z13.s, p2/M, z13.s, z5.s\n" - "smax z14.s, p2/M, z14.s, z5.s\n" + "smax z11.s, p2/M, z11.s, z1.s\n" + "smax z12.s, p2/M, z12.s, z1.s\n" + "uzp1 z0.h, z10.h, z11.h\n" + "uzp1 z8.b, z8.b, z0.b\n" + "smax z13.s, p2/M, z13.s, z1.s\n" + "smax z14.s, p2/M, z14.s, z1.s\n" "uzp1 z12.h, z12.h, z13.h\n" "st1b { z8.b }, p1, [x11]\n" - "smax z15.s, p2/M, z15.s, z5.s\n" - "smax z16.s, p2/M, z16.s, z5.s\n" - "uzp1 z13.h, z14.h, z15.h\n" - "uzp1 z12.b, z12.b, z13.b\n" - "smax z17.s, p2/M, z17.s, z5.s\n" - "smax z18.s, p2/M, z18.s, z5.s\n" + "smax z15.s, p2/M, z15.s, z1.s\n" + "smax z16.s, p2/M, z16.s, z1.s\n" + "uzp1 z0.h, z14.h, z15.h\n" + "uzp1 z12.b, z12.b, z0.b\n" + "smax z17.s, p2/M, z17.s, z1.s\n" + "smax z18.s, p2/M, z18.s, z1.s\n" "uzp1 z16.h, z16.h, z17.h\n" - "st1b { z12.b }, p1, [x24]\n" - "smax z19.s, p2/M, z19.s, z5.s\n" - "smax z20.s, p2/M, z20.s, z5.s\n" + "st1b { z12.b }, p1, [x26]\n" + "smax z19.s, p2/M, z19.s, z1.s\n" + "smax z20.s, p2/M, z20.s, z1.s\n" "uzp1 z17.h, z18.h, z19.h\n" "uzp1 z16.b, z16.b, z17.b\n" - "smax z21.s, p2/M, z21.s, z5.s\n" - "smax z22.s, p2/M, z22.s, z5.s\n" + "smax z21.s, p2/M, z21.s, z1.s\n" + "smax z22.s, p2/M, z22.s, z1.s\n" "uzp1 z20.h, z20.h, z21.h\n" - "st1b { z16.b }, p1, [x23]\n" - "smax z23.s, p2/M, z23.s, z5.s\n" - "smax z24.s, p2/M, z24.s, z5.s\n" - "uzp1 z21.h, z22.h, z23.h\n" - "uzp1 z20.b, z20.b, z21.b\n" - "smax z25.s, p2/M, z25.s, z5.s\n" - "smax z26.s, p2/M, z26.s, z5.s\n" + "st1b { z16.b }, p1, [x25]\n" + "smax z23.s, p2/M, z23.s, z1.s\n" + "smax z24.s, p2/M, z24.s, z1.s\n" + "uzp1 z16.h, z22.h, z23.h\n" + "uzp1 z20.b, z20.b, z16.b\n" + "smax z25.s, p2/M, z25.s, z1.s\n" + "smax z26.s, p2/M, z26.s, z1.s\n" "uzp1 z24.h, z24.h, z25.h\n" - "st1b { z20.b }, p1, [x22]\n" - "smax z27.s, p2/M, z27.s, z5.s\n" - "smax z28.s, p2/M, z28.s, z5.s\n" - "uzp1 z25.h, z26.h, z27.h\n" - "uzp1 z24.b, z24.b, z25.b\n" - "smax z29.s, p2/M, z29.s, z5.s\n" - "smax z30.s, p2/M, z30.s, z5.s\n" + "st1b { z20.b }, p1, [x24]\n" + "smax z27.s, p2/M, z27.s, z1.s\n" + "smax z28.s, p2/M, z28.s, z1.s\n" + "uzp1 z16.h, z26.h, z27.h\n" + "uzp1 z24.b, z24.b, z16.b\n" + "smax z29.s, p2/M, z29.s, z1.s\n" + "smax z30.s, p2/M, z30.s, z1.s\n" "uzp1 z28.h, z28.h, z29.h\n" - "st1b { z24.b }, p1, [x21]\n" - "smax z31.s, p2/M, z31.s, z5.s\n" - "uzp1 z29.h, z30.h, z31.h\n" - "uzp1 z28.b, z28.b, z29.b\n" - "st1b { z28.b }, p1, [x20]\n" + "st1b { z24.b }, p1, [x23]\n" + "smax z31.s, p2/M, z31.s, z1.s\n" + "uzp1 z16.h, z30.h, z31.h\n" + "uzp1 z28.b, z28.b, z16.b\n" + "st1b { z28.b }, p1, [x22]\n" "addvl x11, x11, #1\n" "78:" // Height 6: Writeback done "decw x10, ALL, MUL #4\n" @@ -2594,7 +2594,6 @@ void sve_hybrid_s8qs_dot_6x4VL ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "80:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [flags] "r" (flags), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_multiplier_ptr] "I" (offsetof(KernelArgs, multiplier_ptr)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_shift_ptr] "I" (offsetof(KernelArgs, shift_ptr)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp) : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -2602,4 +2601,4 @@ void sve_hybrid_s8qs_dot_6x4VL ( } } // namespace arm_gemm -#endif // __ARM_FEATURE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_mmla_6x4VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_mmla_6x4VL.hpp index 2b7ad8bf4bd0033a44e36472b726bc19c84314d5..b1b1135c735d878a706033809481478b362770ef 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_mmla_6x4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_mmla_6x4VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef ARM_COMPUTE_ENABLE_SVE + #include "../std_transforms_sve.hpp" #include "../performance_parameters.hpp" @@ -74,7 +74,6 @@ public: template static inline PerformanceParameters get_performance_parameters(const CPUInfo *ci) { - if (std::is_same::value) { switch (ci->get_cpu_model()) { default: @@ -97,5 +96,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_mmla_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_mmla_6x4VL/generic.cpp index 6041794bdb5271716400a9bcae4a9016ff7f241a..cd5f85411c8459b0d3860172d51a9efd211f4e3d 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_mmla_6x4VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8qs_mmla_6x4VL/generic.cpp @@ -117,11 +117,11 @@ void sve_hybrid_s8qs_mmla_6x4VL ( "4:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 5f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 6f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -133,86 +133,86 @@ void sve_hybrid_s8qs_mmla_6x4VL ( "ble 8f\n" "7:" // Height 1: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z1.b }, p0/Z, [x26]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1b { z7.b }, p2/Z, [x9]\n" - "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n" - ".inst 0x45079808 // smmla z8.s, z0.b, z7.b\n" - ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n" - ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" - ".inst 0x4506980d // smmla z13.s, z0.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n" - ".inst 0x4507980a // smmla z10.s, z0.b, z7.b\n" - ".inst 0x4506980e // smmla z14.s, z0.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n" + "ld1rqb { z20.b }, p0/Z, [x26]\n" + "trn1 z18.d, z20.d, z19.d\n" + "ld1b { z17.b }, p2/Z, [x9]\n" + "ld1b { z16.b }, p2/Z, [x9, #1, MUL VL]\n" + ".inst 0x45119a48 // smmla z8.s, z18.b, z17.b\n" + ".inst 0x45109a4c // smmla z12.s, z18.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #3, MUL VL]\n" + ".inst 0x45119a49 // smmla z9.s, z18.b, z17.b\n" + ".inst 0x45109a4d // smmla z13.s, z18.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #4, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #5, MUL VL]\n" + ".inst 0x45119a4a // smmla z10.s, z18.b, z17.b\n" + ".inst 0x45109a4e // smmla z14.s, z18.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #6, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #7, MUL VL]\n" "addvl x9, x9, #16\n" - "trn2 z1.d, z1.d, z2.d\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" - ".inst 0x4506980f // smmla z15.s, z0.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #-8, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #-7, MUL VL]\n" - ".inst 0x45079828 // smmla z8.s, z1.b, z7.b\n" - ".inst 0x4506982c // smmla z12.s, z1.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #-6, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #-5, MUL VL]\n" - ".inst 0x45079829 // smmla z9.s, z1.b, z7.b\n" - ".inst 0x4506982d // smmla z13.s, z1.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #-4, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #-3, MUL VL]\n" - ".inst 0x4507982a // smmla z10.s, z1.b, z7.b\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #-2, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #-1, MUL VL]\n" + "trn2 z20.d, z20.d, z19.d\n" + ".inst 0x45119a4b // smmla z11.s, z18.b, z17.b\n" + ".inst 0x45109a4f // smmla z15.s, z18.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #-8, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #-7, MUL VL]\n" + ".inst 0x45119a88 // smmla z8.s, z20.b, z17.b\n" + ".inst 0x45109a8c // smmla z12.s, z20.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #-6, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #-5, MUL VL]\n" + ".inst 0x45119a89 // smmla z9.s, z20.b, z17.b\n" + ".inst 0x45109a8d // smmla z13.s, z20.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #-4, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #-3, MUL VL]\n" + ".inst 0x45119a8a // smmla z10.s, z20.b, z17.b\n" + ".inst 0x45109a8e // smmla z14.s, z20.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #-2, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #-1, MUL VL]\n" "sub x27, x27, #0x10\n" "cmp x27, #0x10\n" - ".inst 0x4507982b // smmla z11.s, z1.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" + ".inst 0x45119a8b // smmla z11.s, z20.b, z17.b\n" + ".inst 0x45109a8f // smmla z15.s, z20.b, z16.b\n" "add x26, x26, #0x10\n" "bgt 7b\n" "8:" // Height 1: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" "ld1rqb { z1.b }, p0/Z, [x26]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1b { z7.b }, p2/Z, [x9]\n" - "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n" - ".inst 0x45079808 // smmla z8.s, z0.b, z7.b\n" - ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n" - ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" - ".inst 0x4506980d // smmla z13.s, z0.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n" - ".inst 0x4507980a // smmla z10.s, z0.b, z7.b\n" - ".inst 0x4506980e // smmla z14.s, z0.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n" + "trn1 z18.d, z1.d, z19.d\n" + "ld1b { z17.b }, p2/Z, [x9]\n" + "ld1b { z16.b }, p2/Z, [x9, #1, MUL VL]\n" + ".inst 0x45119a48 // smmla z8.s, z18.b, z17.b\n" + ".inst 0x45109a4c // smmla z12.s, z18.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #3, MUL VL]\n" + ".inst 0x45119a49 // smmla z9.s, z18.b, z17.b\n" + ".inst 0x45109a4d // smmla z13.s, z18.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #4, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #5, MUL VL]\n" + ".inst 0x45119a4a // smmla z10.s, z18.b, z17.b\n" + ".inst 0x45109a4e // smmla z14.s, z18.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #6, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #7, MUL VL]\n" "subs x27, x27, #0x8\n" - "trn2 z1.d, z1.d, z2.d\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" - ".inst 0x4506980f // smmla z15.s, z0.b, z6.b\n" + "trn2 z1.d, z1.d, z19.d\n" + ".inst 0x45119a4b // smmla z11.s, z18.b, z17.b\n" + ".inst 0x45109a4f // smmla z15.s, z18.b, z16.b\n" "addvl x9, x9, #8\n" "ble 9f\n" - "ld1b { z7.b }, p2/Z, [x9]\n" - "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n" - ".inst 0x45079828 // smmla z8.s, z1.b, z7.b\n" - ".inst 0x4506982c // smmla z12.s, z1.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n" - ".inst 0x45079829 // smmla z9.s, z1.b, z7.b\n" - ".inst 0x4506982d // smmla z13.s, z1.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n" - ".inst 0x4507982a // smmla z10.s, z1.b, z7.b\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n" - ".inst 0x4507982b // smmla z11.s, z1.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" + "ld1b { z17.b }, p2/Z, [x9]\n" + "ld1b { z16.b }, p2/Z, [x9, #1, MUL VL]\n" + ".inst 0x45119828 // smmla z8.s, z1.b, z17.b\n" + ".inst 0x4510982c // smmla z12.s, z1.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #3, MUL VL]\n" + ".inst 0x45119829 // smmla z9.s, z1.b, z17.b\n" + ".inst 0x4510982d // smmla z13.s, z1.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #4, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #5, MUL VL]\n" + ".inst 0x4511982a // smmla z10.s, z1.b, z17.b\n" + ".inst 0x4510982e // smmla z14.s, z1.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #6, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #7, MUL VL]\n" + ".inst 0x4511982b // smmla z11.s, z1.b, z17.b\n" + ".inst 0x4510982f // smmla z15.s, z1.b, z16.b\n" "addvl x9, x9, #8\n" "9:" // Height 1: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -221,18 +221,18 @@ void sve_hybrid_s8qs_mmla_6x4VL ( "bne 4b\n" "uzp1 z8.d, z8.d, z12.d\n" "uzp1 z9.d, z9.d, z13.d\n" - "ld1w { z0.s }, p2/Z, [x14]\n" - "ld1w { z1.s }, p2/Z, [x14, #1, MUL VL]\n" + "ld1w { z19.s }, p2/Z, [x14]\n" + "ld1w { z18.s }, p2/Z, [x14, #1, MUL VL]\n" "uzp1 z10.d, z10.d, z14.d\n" "uzp1 z11.d, z11.d, z15.d\n" - "ld1w { z2.s }, p2/Z, [x14, #2, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x14, #3, MUL VL]\n" + "ld1w { z17.s }, p2/Z, [x14, #2, MUL VL]\n" + "ld1w { z16.s }, p2/Z, [x14, #3, MUL VL]\n" "mov z15.d, z8.d\n" - "add z15.s, z15.s, z0.s\n" + "add z15.s, z15.s, z19.s\n" "addvl x14, x14, #4\n" - "add z9.s, z9.s, z1.s\n" - "add z10.s, z10.s, z2.s\n" - "add z11.s, z11.s, z3.s\n" + "add z9.s, z9.s, z18.s\n" + "add z10.s, z10.s, z17.s\n" + "add z11.s, z11.s, z16.s\n" "tbz %x[flags], #4, 10f\n" "ld1w { z0.s }, p2/Z, [x12]\n" "ld1w { z4.s }, p2/Z, [x13]\n" @@ -246,10 +246,10 @@ void sve_hybrid_s8qs_mmla_6x4VL ( "addvl x13, x13, #4\n" "b 11f\n" "10:" // Height 1: per layer parameters - "add x26, %x[qp], %[per_layer_right_shift]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1rw { z0.s }, p2/Z, [x26]\n" - "ld1rw { z4.s }, p2/Z, [x25]\n" + "add x21, %x[qp], %[per_layer_right_shift]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1rw { z0.s }, p2/Z, [x21]\n" + "ld1rw { z4.s }, p2/Z, [x20]\n" "mov z1.d, z0.d\n" "mov z5.d, z4.d\n" "mov z2.d, z0.d\n" @@ -262,44 +262,44 @@ void sve_hybrid_s8qs_mmla_6x4VL ( ".inst 0x04a6754a // sqrdmulh z10.s, z10.s, z6.s\n" ".inst 0x04a7756b // sqrdmulh z11.s, z11.s, z7.s\n" "tbz %x[flags], #5, 12f\n" - "and z4.d, z15.d, z0.d\n" - "and z5.d, z9.d, z1.d\n" - "and z6.d, z10.d, z2.d\n" - "and z7.d, z11.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z15.s, z15.s, z4.s\n" - "sqadd z9.s, z9.s, z5.s\n" - "sqadd z10.s, z10.s, z6.s\n" - "sqadd z11.s, z11.s, z7.s\n" + "and z19.d, z15.d, z0.d\n" + "and z18.d, z9.d, z1.d\n" + "and z17.d, z10.d, z2.d\n" + "and z16.d, z11.d, z3.d\n" + "asr z19.s, z19.s, #0x1f\n" + "asr z18.s, z18.s, #0x1f\n" + "asr z17.s, z17.s, #0x1f\n" + "asr z16.s, z16.s, #0x1f\n" + "sqadd z15.s, z15.s, z19.s\n" + "sqadd z9.s, z9.s, z18.s\n" + "sqadd z10.s, z10.s, z17.s\n" + "sqadd z11.s, z11.s, z16.s\n" "12:" // Height 1: no shift correction - "add x25, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x25]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1rw { z17.s }, p2/Z, [x20]\n" ".inst 0x4482880f // srshl z15.s, p2/M, z15.s, z0.s\n" - "add z15.s, z15.s, z4.s\n" + "add z15.s, z15.s, z17.s\n" ".inst 0x44828829 // srshl z9.s, p2/M, z9.s, z1.s\n" ".inst 0x4482884a // srshl z10.s, p2/M, z10.s, z2.s\n" - "add z9.s, z9.s, z4.s\n" - "add z10.s, z10.s, z4.s\n" + "add z9.s, z9.s, z17.s\n" + "add z10.s, z10.s, z17.s\n" ".inst 0x4482886b // srshl z11.s, p2/M, z11.s, z3.s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x25]\n" - "add z11.s, z11.s, z4.s\n" - "add x25, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x25]\n" - "smin z15.s, p2/M, z15.s, z6.s\n" - "smin z9.s, p2/M, z9.s, z6.s\n" - "smin z10.s, p2/M, z10.s, z6.s\n" - "smin z11.s, p2/M, z11.s, z6.s\n" - "smax z15.s, p2/M, z15.s, z5.s\n" - "smax z9.s, p2/M, z9.s, z5.s\n" - "smax z10.s, p2/M, z10.s, z5.s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1rw { z16.s }, p2/Z, [x20]\n" + "add z11.s, z11.s, z17.s\n" + "add x20, %x[qp], %[minval]\n" + "ld1rw { z31.s }, p2/Z, [x20]\n" + "smin z15.s, p2/M, z15.s, z16.s\n" + "smin z9.s, p2/M, z9.s, z16.s\n" + "smin z10.s, p2/M, z10.s, z16.s\n" + "smin z11.s, p2/M, z11.s, z16.s\n" + "smax z15.s, p2/M, z15.s, z31.s\n" + "smax z9.s, p2/M, z9.s, z31.s\n" + "smax z10.s, p2/M, z10.s, z31.s\n" "uzp1 z15.h, z15.h, z9.h\n" - "smax z11.s, p2/M, z11.s, z5.s\n" - "uzp1 z9.h, z10.h, z11.h\n" - "uzp1 z15.b, z15.b, z9.b\n" + "smax z11.s, p2/M, z11.s, z31.s\n" + "uzp1 z16.h, z10.h, z11.h\n" + "uzp1 z15.b, z15.b, z16.b\n" "st1b { z15.b }, p1, [x11]\n" "addvl x11, x11, #1\n" "13:" // Height 1: Writeback done @@ -330,12 +330,12 @@ void sve_hybrid_s8qs_mmla_6x4VL ( "17:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 18f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 19f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -343,125 +343,125 @@ void sve_hybrid_s8qs_mmla_6x4VL ( "b 19f\n" "18:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" + "add x25, x26, x21\n" "19:" // Height 2: input setup done "cmp x27, #0x10\n" "ble 21f\n" "20:" // Height 2: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1b { z7.b }, p2/Z, [x9]\n" - "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n" - ".inst 0x45079808 // smmla z8.s, z0.b, z7.b\n" - ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n" - ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" - ".inst 0x4506980d // smmla z13.s, z0.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n" - ".inst 0x4507980a // smmla z10.s, z0.b, z7.b\n" - ".inst 0x4506980e // smmla z14.s, z0.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n" + "ld1rqb { z20.b }, p0/Z, [x26]\n" + "ld1rqb { z19.b }, p0/Z, [x25]\n" + "trn1 z18.d, z20.d, z19.d\n" + "ld1b { z17.b }, p2/Z, [x9]\n" + "ld1b { z16.b }, p2/Z, [x9, #1, MUL VL]\n" + ".inst 0x45119a48 // smmla z8.s, z18.b, z17.b\n" + ".inst 0x45109a4c // smmla z12.s, z18.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #3, MUL VL]\n" + ".inst 0x45119a49 // smmla z9.s, z18.b, z17.b\n" + ".inst 0x45109a4d // smmla z13.s, z18.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #4, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #5, MUL VL]\n" + ".inst 0x45119a4a // smmla z10.s, z18.b, z17.b\n" + ".inst 0x45109a4e // smmla z14.s, z18.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #6, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #7, MUL VL]\n" "addvl x9, x9, #16\n" - "trn2 z1.d, z1.d, z2.d\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" - ".inst 0x4506980f // smmla z15.s, z0.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #-8, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #-7, MUL VL]\n" - ".inst 0x45079828 // smmla z8.s, z1.b, z7.b\n" - ".inst 0x4506982c // smmla z12.s, z1.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #-6, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #-5, MUL VL]\n" - ".inst 0x45079829 // smmla z9.s, z1.b, z7.b\n" - ".inst 0x4506982d // smmla z13.s, z1.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #-4, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #-3, MUL VL]\n" - ".inst 0x4507982a // smmla z10.s, z1.b, z7.b\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #-2, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #-1, MUL VL]\n" + "trn2 z20.d, z20.d, z19.d\n" + ".inst 0x45119a4b // smmla z11.s, z18.b, z17.b\n" + ".inst 0x45109a4f // smmla z15.s, z18.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #-8, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #-7, MUL VL]\n" + ".inst 0x45119a88 // smmla z8.s, z20.b, z17.b\n" + ".inst 0x45109a8c // smmla z12.s, z20.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #-6, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #-5, MUL VL]\n" + ".inst 0x45119a89 // smmla z9.s, z20.b, z17.b\n" + ".inst 0x45109a8d // smmla z13.s, z20.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #-4, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #-3, MUL VL]\n" + ".inst 0x45119a8a // smmla z10.s, z20.b, z17.b\n" + ".inst 0x45109a8e // smmla z14.s, z20.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #-2, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #-1, MUL VL]\n" "sub x27, x27, #0x10\n" "cmp x27, #0x10\n" - ".inst 0x4507982b // smmla z11.s, z1.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" + ".inst 0x45119a8b // smmla z11.s, z20.b, z17.b\n" + ".inst 0x45109a8f // smmla z15.s, z20.b, z16.b\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" "bgt 20b\n" "21:" // Height 2: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1b { z7.b }, p2/Z, [x9]\n" - "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n" - ".inst 0x45079808 // smmla z8.s, z0.b, z7.b\n" - ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n" - ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" - ".inst 0x4506980d // smmla z13.s, z0.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n" - ".inst 0x4507980a // smmla z10.s, z0.b, z7.b\n" - ".inst 0x4506980e // smmla z14.s, z0.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n" + "ld1rqb { z19.b }, p0/Z, [x25]\n" + "trn1 z18.d, z1.d, z19.d\n" + "ld1b { z17.b }, p2/Z, [x9]\n" + "ld1b { z16.b }, p2/Z, [x9, #1, MUL VL]\n" + ".inst 0x45119a48 // smmla z8.s, z18.b, z17.b\n" + ".inst 0x45109a4c // smmla z12.s, z18.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #3, MUL VL]\n" + ".inst 0x45119a49 // smmla z9.s, z18.b, z17.b\n" + ".inst 0x45109a4d // smmla z13.s, z18.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #4, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #5, MUL VL]\n" + ".inst 0x45119a4a // smmla z10.s, z18.b, z17.b\n" + ".inst 0x45109a4e // smmla z14.s, z18.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #6, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #7, MUL VL]\n" "subs x27, x27, #0x8\n" - "trn2 z1.d, z1.d, z2.d\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" - ".inst 0x4506980f // smmla z15.s, z0.b, z6.b\n" + "trn2 z1.d, z1.d, z19.d\n" + ".inst 0x45119a4b // smmla z11.s, z18.b, z17.b\n" + ".inst 0x45109a4f // smmla z15.s, z18.b, z16.b\n" "addvl x9, x9, #8\n" "ble 22f\n" - "ld1b { z7.b }, p2/Z, [x9]\n" - "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n" - ".inst 0x45079828 // smmla z8.s, z1.b, z7.b\n" - ".inst 0x4506982c // smmla z12.s, z1.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n" - ".inst 0x45079829 // smmla z9.s, z1.b, z7.b\n" - ".inst 0x4506982d // smmla z13.s, z1.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n" - ".inst 0x4507982a // smmla z10.s, z1.b, z7.b\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n" - ".inst 0x4507982b // smmla z11.s, z1.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" + "ld1b { z17.b }, p2/Z, [x9]\n" + "ld1b { z16.b }, p2/Z, [x9, #1, MUL VL]\n" + ".inst 0x45119828 // smmla z8.s, z1.b, z17.b\n" + ".inst 0x4510982c // smmla z12.s, z1.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #3, MUL VL]\n" + ".inst 0x45119829 // smmla z9.s, z1.b, z17.b\n" + ".inst 0x4510982d // smmla z13.s, z1.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #4, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #5, MUL VL]\n" + ".inst 0x4511982a // smmla z10.s, z1.b, z17.b\n" + ".inst 0x4510982e // smmla z14.s, z1.b, z16.b\n" + "ld1b { z17.b }, p2/Z, [x9, #6, MUL VL]\n" + "ld1b { z16.b }, p2/Z, [x9, #7, MUL VL]\n" + ".inst 0x4511982b // smmla z11.s, z1.b, z17.b\n" + ".inst 0x4510982f // smmla z15.s, z1.b, z16.b\n" "addvl x9, x9, #8\n" "22:" // Height 2: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 17b\n" - "uzp1 z7.d, z8.d, z12.d\n" + "uzp1 z20.d, z8.d, z12.d\n" "uzp2 z8.d, z8.d, z12.d\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "ld1w { z0.s }, p2/Z, [x14]\n" + "ld1w { z19.s }, p2/Z, [x14]\n" "uzp1 z12.d, z9.d, z13.d\n" "uzp2 z9.d, z9.d, z13.d\n" - "ld1w { z1.s }, p2/Z, [x14, #1, MUL VL]\n" - "ld1w { z2.s }, p2/Z, [x14, #2, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x14, #1, MUL VL]\n" + "ld1w { z17.s }, p2/Z, [x14, #2, MUL VL]\n" "uzp1 z13.d, z10.d, z14.d\n" "uzp2 z10.d, z10.d, z14.d\n" - "ld1w { z3.s }, p2/Z, [x14, #3, MUL VL]\n" - "add x24, x11, x20\n" + "ld1w { z16.s }, p2/Z, [x14, #3, MUL VL]\n" + "add x26, x11, x20\n" "uzp1 z14.d, z11.d, z15.d\n" "uzp2 z11.d, z11.d, z15.d\n" "addvl x14, x14, #4\n" - "mov z15.d, z7.d\n" - "add z15.s, z15.s, z0.s\n" - "add z12.s, z12.s, z1.s\n" - "add z13.s, z13.s, z2.s\n" - "add z14.s, z14.s, z3.s\n" - "add z8.s, z8.s, z0.s\n" - "add z9.s, z9.s, z1.s\n" - "add z10.s, z10.s, z2.s\n" - "add z11.s, z11.s, z3.s\n" + "mov z15.d, z20.d\n" + "add z15.s, z15.s, z19.s\n" + "add z12.s, z12.s, z18.s\n" + "add z13.s, z13.s, z17.s\n" + "add z14.s, z14.s, z16.s\n" + "add z8.s, z8.s, z19.s\n" + "add z9.s, z9.s, z18.s\n" + "add z10.s, z10.s, z17.s\n" + "add z11.s, z11.s, z16.s\n" "tbz %x[flags], #4, 23f\n" "ld1w { z0.s }, p2/Z, [x12]\n" "ld1w { z4.s }, p2/Z, [x13]\n" @@ -475,10 +475,10 @@ void sve_hybrid_s8qs_mmla_6x4VL ( "addvl x13, x13, #4\n" "b 24f\n" "23:" // Height 2: per layer parameters - "add x26, %x[qp], %[per_layer_right_shift]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1rw { z0.s }, p2/Z, [x26]\n" - "ld1rw { z4.s }, p2/Z, [x25]\n" + "add x21, %x[qp], %[per_layer_right_shift]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1rw { z0.s }, p2/Z, [x21]\n" + "ld1rw { z4.s }, p2/Z, [x20]\n" "mov z1.d, z0.d\n" "mov z5.d, z4.d\n" "mov z2.d, z0.d\n" @@ -495,77 +495,77 @@ void sve_hybrid_s8qs_mmla_6x4VL ( ".inst 0x04a6754a // sqrdmulh z10.s, z10.s, z6.s\n" ".inst 0x04a7756b // sqrdmulh z11.s, z11.s, z7.s\n" "tbz %x[flags], #5, 25f\n" - "and z4.d, z15.d, z0.d\n" - "and z5.d, z12.d, z1.d\n" - "and z6.d, z13.d, z2.d\n" - "and z7.d, z14.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z15.s, z15.s, z4.s\n" - "sqadd z12.s, z12.s, z5.s\n" - "sqadd z13.s, z13.s, z6.s\n" - "sqadd z14.s, z14.s, z7.s\n" - "and z4.d, z8.d, z0.d\n" - "and z5.d, z9.d, z1.d\n" - "and z6.d, z10.d, z2.d\n" - "and z7.d, z11.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z8.s, z8.s, z4.s\n" - "sqadd z9.s, z9.s, z5.s\n" - "sqadd z10.s, z10.s, z6.s\n" - "sqadd z11.s, z11.s, z7.s\n" + "and z19.d, z15.d, z0.d\n" + "and z18.d, z12.d, z1.d\n" + "and z17.d, z13.d, z2.d\n" + "and z16.d, z14.d, z3.d\n" + "asr z19.s, z19.s, #0x1f\n" + "asr z18.s, z18.s, #0x1f\n" + "asr z17.s, z17.s, #0x1f\n" + "asr z16.s, z16.s, #0x1f\n" + "sqadd z15.s, z15.s, z19.s\n" + "sqadd z12.s, z12.s, z18.s\n" + "sqadd z13.s, z13.s, z17.s\n" + "sqadd z14.s, z14.s, z16.s\n" + "and z18.d, z8.d, z0.d\n" + "and z24.d, z9.d, z1.d\n" + "and z17.d, z10.d, z2.d\n" + "and z16.d, z11.d, z3.d\n" + "asr z18.s, z18.s, #0x1f\n" + "asr z24.s, z24.s, #0x1f\n" + "asr z17.s, z17.s, #0x1f\n" + "asr z16.s, z16.s, #0x1f\n" + "sqadd z8.s, z8.s, z18.s\n" + "sqadd z9.s, z9.s, z24.s\n" + "sqadd z10.s, z10.s, z17.s\n" + "sqadd z11.s, z11.s, z16.s\n" "25:" // Height 2: no shift correction - "add x25, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x25]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1rw { z17.s }, p2/Z, [x20]\n" ".inst 0x4482880f // srshl z15.s, p2/M, z15.s, z0.s\n" - "add z15.s, z15.s, z4.s\n" + "add z15.s, z15.s, z17.s\n" ".inst 0x4482882c // srshl z12.s, p2/M, z12.s, z1.s\n" ".inst 0x4482884d // srshl z13.s, p2/M, z13.s, z2.s\n" - "add z12.s, z12.s, z4.s\n" - "add z13.s, z13.s, z4.s\n" + "add z12.s, z12.s, z17.s\n" + "add z13.s, z13.s, z17.s\n" ".inst 0x4482886e // srshl z14.s, p2/M, z14.s, z3.s\n" ".inst 0x44828808 // srshl z8.s, p2/M, z8.s, z0.s\n" - "add z14.s, z14.s, z4.s\n" - "add z8.s, z8.s, z4.s\n" + "add z14.s, z14.s, z17.s\n" + "add z8.s, z8.s, z17.s\n" ".inst 0x44828829 // srshl z9.s, p2/M, z9.s, z1.s\n" ".inst 0x4482884a // srshl z10.s, p2/M, z10.s, z2.s\n" - "add z9.s, z9.s, z4.s\n" - "add z10.s, z10.s, z4.s\n" + "add z9.s, z9.s, z17.s\n" + "add z10.s, z10.s, z17.s\n" ".inst 0x4482886b // srshl z11.s, p2/M, z11.s, z3.s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x25]\n" - "add z11.s, z11.s, z4.s\n" - "add x25, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x25]\n" - "smin z15.s, p2/M, z15.s, z6.s\n" - "smin z12.s, p2/M, z12.s, z6.s\n" - "smin z13.s, p2/M, z13.s, z6.s\n" - "smin z14.s, p2/M, z14.s, z6.s\n" - "smin z8.s, p2/M, z8.s, z6.s\n" - "smin z9.s, p2/M, z9.s, z6.s\n" - "smin z10.s, p2/M, z10.s, z6.s\n" - "smin z11.s, p2/M, z11.s, z6.s\n" - "smax z15.s, p2/M, z15.s, z5.s\n" - "smax z12.s, p2/M, z12.s, z5.s\n" - "smax z13.s, p2/M, z13.s, z5.s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1rw { z16.s }, p2/Z, [x20]\n" + "add z11.s, z11.s, z17.s\n" + "add x20, %x[qp], %[minval]\n" + "ld1rw { z17.s }, p2/Z, [x20]\n" + "smin z15.s, p2/M, z15.s, z16.s\n" + "smin z12.s, p2/M, z12.s, z16.s\n" + "smin z13.s, p2/M, z13.s, z16.s\n" + "smin z14.s, p2/M, z14.s, z16.s\n" + "smin z8.s, p2/M, z8.s, z16.s\n" + "smin z9.s, p2/M, z9.s, z16.s\n" + "smin z10.s, p2/M, z10.s, z16.s\n" + "smin z11.s, p2/M, z11.s, z16.s\n" + "smax z15.s, p2/M, z15.s, z17.s\n" + "smax z12.s, p2/M, z12.s, z17.s\n" + "smax z13.s, p2/M, z13.s, z17.s\n" "uzp1 z15.h, z15.h, z12.h\n" - "smax z14.s, p2/M, z14.s, z5.s\n" - "smax z8.s, p2/M, z8.s, z5.s\n" - "uzp1 z12.h, z13.h, z14.h\n" - "uzp1 z15.b, z15.b, z12.b\n" - "smax z9.s, p2/M, z9.s, z5.s\n" - "smax z10.s, p2/M, z10.s, z5.s\n" + "smax z14.s, p2/M, z14.s, z17.s\n" + "smax z8.s, p2/M, z8.s, z17.s\n" + "uzp1 z16.h, z13.h, z14.h\n" + "uzp1 z15.b, z15.b, z16.b\n" + "smax z9.s, p2/M, z9.s, z17.s\n" + "smax z10.s, p2/M, z10.s, z17.s\n" "uzp1 z8.h, z8.h, z9.h\n" "st1b { z15.b }, p1, [x11]\n" - "smax z11.s, p2/M, z11.s, z5.s\n" - "uzp1 z9.h, z10.h, z11.h\n" - "uzp1 z8.b, z8.b, z9.b\n" - "st1b { z8.b }, p1, [x24]\n" + "smax z11.s, p2/M, z11.s, z17.s\n" + "uzp1 z16.h, z10.h, z11.h\n" + "uzp1 z8.b, z8.b, z16.b\n" + "st1b { z8.b }, p1, [x26]\n" "addvl x11, x11, #1\n" "26:" // Height 2: Writeback done "decw x10, ALL, MUL #4\n" @@ -603,13 +603,13 @@ void sve_hybrid_s8qs_mmla_6x4VL ( "30:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 31f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 32f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -618,174 +618,174 @@ void sve_hybrid_s8qs_mmla_6x4VL ( "b 32f\n" "31:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" "32:" // Height 3: input setup done "cmp x27, #0x10\n" "ble 34f\n" "33:" // Height 3: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" - "ld1rqb { z3.b }, p0/Z, [x24]\n" - "trn1 z0.d, z1.d, z2.d\n" - "trn2 z1.d, z1.d, z2.d\n" - "ld1b { z7.b }, p2/Z, [x9]\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n" - ".inst 0x45079808 // smmla z8.s, z0.b, z7.b\n" - ".inst 0x45079850 // smmla z16.s, z2.b, z7.b\n" - ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n" - ".inst 0x45069854 // smmla z20.s, z2.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n" - ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" - ".inst 0x45079851 // smmla z17.s, z2.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x4506980d // smmla z13.s, z0.b, z6.b\n" - ".inst 0x45069855 // smmla z21.s, z2.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n" + "ld1rqb { z30.b }, p0/Z, [x26]\n" + "ld1rqb { z24.b }, p0/Z, [x25]\n" + "ld1rqb { z28.b }, p0/Z, [x24]\n" + "trn1 z27.d, z30.d, z24.d\n" + "trn2 z30.d, z30.d, z24.d\n" + "ld1b { z25.b }, p2/Z, [x9]\n" + "trn1 z26.d, z28.d, z29.d\n" + "ld1b { z24.b }, p2/Z, [x9, #1, MUL VL]\n" + ".inst 0x45199b68 // smmla z8.s, z27.b, z25.b\n" + ".inst 0x45199b50 // smmla z16.s, z26.b, z25.b\n" + ".inst 0x45189b6c // smmla z12.s, z27.b, z24.b\n" + ".inst 0x45189b54 // smmla z20.s, z26.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x9, #3, MUL VL]\n" + ".inst 0x45199b69 // smmla z9.s, z27.b, z25.b\n" + ".inst 0x45199b51 // smmla z17.s, z26.b, z25.b\n" + "ld1b { z25.b }, p2/Z, [x9, #4, MUL VL]\n" + "trn2 z28.d, z28.d, z29.d\n" + ".inst 0x45189b6d // smmla z13.s, z27.b, z24.b\n" + ".inst 0x45189b55 // smmla z21.s, z26.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x9, #5, MUL VL]\n" "sub x27, x27, #0x10\n" - ".inst 0x4507980a // smmla z10.s, z0.b, z7.b\n" - ".inst 0x45079852 // smmla z18.s, z2.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n" + ".inst 0x45199b6a // smmla z10.s, z27.b, z25.b\n" + ".inst 0x45199b52 // smmla z18.s, z26.b, z25.b\n" + "ld1b { z25.b }, p2/Z, [x9, #6, MUL VL]\n" "cmp x27, #0x10\n" - ".inst 0x4506980e // smmla z14.s, z0.b, z6.b\n" - ".inst 0x45069856 // smmla z22.s, z2.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n" + ".inst 0x45189b6e // smmla z14.s, z27.b, z24.b\n" + ".inst 0x45189b56 // smmla z22.s, z26.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x9, #7, MUL VL]\n" "addvl x9, x9, #16\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" - ".inst 0x45079853 // smmla z19.s, z2.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #-8, MUL VL]\n" + ".inst 0x45199b6b // smmla z11.s, z27.b, z25.b\n" + ".inst 0x45199b53 // smmla z19.s, z26.b, z25.b\n" + "ld1b { z25.b }, p2/Z, [x9, #-8, MUL VL]\n" "add x26, x26, #0x10\n" - ".inst 0x4506980f // smmla z15.s, z0.b, z6.b\n" - ".inst 0x45069857 // smmla z23.s, z2.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #-7, MUL VL]\n" + ".inst 0x45189b6f // smmla z15.s, z27.b, z24.b\n" + ".inst 0x45189b57 // smmla z23.s, z26.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x9, #-7, MUL VL]\n" "add x25, x25, #0x10\n" - ".inst 0x45079828 // smmla z8.s, z1.b, z7.b\n" - ".inst 0x45079870 // smmla z16.s, z3.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #-6, MUL VL]\n" + ".inst 0x45199bc8 // smmla z8.s, z30.b, z25.b\n" + ".inst 0x45199b90 // smmla z16.s, z28.b, z25.b\n" + "ld1b { z25.b }, p2/Z, [x9, #-6, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x4506982c // smmla z12.s, z1.b, z6.b\n" - ".inst 0x45069874 // smmla z20.s, z3.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #-5, MUL VL]\n" - ".inst 0x45079829 // smmla z9.s, z1.b, z7.b\n" - ".inst 0x45079871 // smmla z17.s, z3.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #-4, MUL VL]\n" - ".inst 0x4506982d // smmla z13.s, z1.b, z6.b\n" - ".inst 0x45069875 // smmla z21.s, z3.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #-3, MUL VL]\n" - ".inst 0x4507982a // smmla z10.s, z1.b, z7.b\n" - ".inst 0x45079872 // smmla z18.s, z3.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #-2, MUL VL]\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - ".inst 0x45069876 // smmla z22.s, z3.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #-1, MUL VL]\n" - ".inst 0x4507982b // smmla z11.s, z1.b, z7.b\n" - ".inst 0x45079873 // smmla z19.s, z3.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" - ".inst 0x45069877 // smmla z23.s, z3.b, z6.b\n" + ".inst 0x45189bcc // smmla z12.s, z30.b, z24.b\n" + ".inst 0x45189b94 // smmla z20.s, z28.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x9, #-5, MUL VL]\n" + ".inst 0x45199bc9 // smmla z9.s, z30.b, z25.b\n" + ".inst 0x45199b91 // smmla z17.s, z28.b, z25.b\n" + "ld1b { z25.b }, p2/Z, [x9, #-4, MUL VL]\n" + ".inst 0x45189bcd // smmla z13.s, z30.b, z24.b\n" + ".inst 0x45189b95 // smmla z21.s, z28.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x9, #-3, MUL VL]\n" + ".inst 0x45199bca // smmla z10.s, z30.b, z25.b\n" + ".inst 0x45199b92 // smmla z18.s, z28.b, z25.b\n" + "ld1b { z25.b }, p2/Z, [x9, #-2, MUL VL]\n" + ".inst 0x45189bce // smmla z14.s, z30.b, z24.b\n" + ".inst 0x45189b96 // smmla z22.s, z28.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x9, #-1, MUL VL]\n" + ".inst 0x45199bcb // smmla z11.s, z30.b, z25.b\n" + ".inst 0x45199b93 // smmla z19.s, z28.b, z25.b\n" + ".inst 0x45189bcf // smmla z15.s, z30.b, z24.b\n" + ".inst 0x45189b97 // smmla z23.s, z28.b, z24.b\n" "bgt 33b\n" "34:" // Height 3: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" + "ld1rqb { z24.b }, p0/Z, [x25]\n" "ld1rqb { z3.b }, p0/Z, [x24]\n" - "trn1 z0.d, z1.d, z2.d\n" - "trn2 z1.d, z1.d, z2.d\n" - "ld1b { z7.b }, p2/Z, [x9]\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n" - ".inst 0x45079808 // smmla z8.s, z0.b, z7.b\n" - ".inst 0x45079850 // smmla z16.s, z2.b, z7.b\n" - ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n" - ".inst 0x45069854 // smmla z20.s, z2.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n" - ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" - ".inst 0x45079851 // smmla z17.s, z2.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n" + "trn1 z27.d, z1.d, z24.d\n" + "trn2 z1.d, z1.d, z24.d\n" + "ld1b { z25.b }, p2/Z, [x9]\n" + "trn1 z26.d, z3.d, z28.d\n" + "ld1b { z24.b }, p2/Z, [x9, #1, MUL VL]\n" + ".inst 0x45199b68 // smmla z8.s, z27.b, z25.b\n" + ".inst 0x45199b50 // smmla z16.s, z26.b, z25.b\n" + ".inst 0x45189b6c // smmla z12.s, z27.b, z24.b\n" + ".inst 0x45189b54 // smmla z20.s, z26.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x9, #3, MUL VL]\n" + ".inst 0x45199b69 // smmla z9.s, z27.b, z25.b\n" + ".inst 0x45199b51 // smmla z17.s, z26.b, z25.b\n" + "ld1b { z25.b }, p2/Z, [x9, #4, MUL VL]\n" "subs x27, x27, #0x8\n" - ".inst 0x4506980d // smmla z13.s, z0.b, z6.b\n" - ".inst 0x45069855 // smmla z21.s, z2.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x4507980a // smmla z10.s, z0.b, z7.b\n" - ".inst 0x45079852 // smmla z18.s, z2.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n" - ".inst 0x4506980e // smmla z14.s, z0.b, z6.b\n" - ".inst 0x45069856 // smmla z22.s, z2.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n" + ".inst 0x45189b6d // smmla z13.s, z27.b, z24.b\n" + ".inst 0x45189b55 // smmla z21.s, z26.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x9, #5, MUL VL]\n" + "trn2 z3.d, z3.d, z28.d\n" + ".inst 0x45199b6a // smmla z10.s, z27.b, z25.b\n" + ".inst 0x45199b52 // smmla z18.s, z26.b, z25.b\n" + "ld1b { z25.b }, p2/Z, [x9, #6, MUL VL]\n" + ".inst 0x45189b6e // smmla z14.s, z27.b, z24.b\n" + ".inst 0x45189b56 // smmla z22.s, z26.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x9, #7, MUL VL]\n" "addvl x9, x9, #8\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" - ".inst 0x45079853 // smmla z19.s, z2.b, z7.b\n" - ".inst 0x4506980f // smmla z15.s, z0.b, z6.b\n" - ".inst 0x45069857 // smmla z23.s, z2.b, z6.b\n" + ".inst 0x45199b6b // smmla z11.s, z27.b, z25.b\n" + ".inst 0x45199b53 // smmla z19.s, z26.b, z25.b\n" + ".inst 0x45189b6f // smmla z15.s, z27.b, z24.b\n" + ".inst 0x45189b57 // smmla z23.s, z26.b, z24.b\n" "ble 35f\n" - "ld1b { z7.b }, p2/Z, [x9]\n" - "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n" - ".inst 0x45079828 // smmla z8.s, z1.b, z7.b\n" - ".inst 0x45079870 // smmla z16.s, z3.b, z7.b\n" - ".inst 0x4506982c // smmla z12.s, z1.b, z6.b\n" - ".inst 0x45069874 // smmla z20.s, z3.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n" - ".inst 0x45079829 // smmla z9.s, z1.b, z7.b\n" - ".inst 0x45079871 // smmla z17.s, z3.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n" - ".inst 0x4506982d // smmla z13.s, z1.b, z6.b\n" - ".inst 0x45069875 // smmla z21.s, z3.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n" - ".inst 0x4507982a // smmla z10.s, z1.b, z7.b\n" - ".inst 0x45079872 // smmla z18.s, z3.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - ".inst 0x45069876 // smmla z22.s, z3.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n" + "ld1b { z25.b }, p2/Z, [x9]\n" + "ld1b { z24.b }, p2/Z, [x9, #1, MUL VL]\n" + ".inst 0x45199828 // smmla z8.s, z1.b, z25.b\n" + ".inst 0x45199870 // smmla z16.s, z3.b, z25.b\n" + ".inst 0x4518982c // smmla z12.s, z1.b, z24.b\n" + ".inst 0x45189874 // smmla z20.s, z3.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x9, #3, MUL VL]\n" + ".inst 0x45199829 // smmla z9.s, z1.b, z25.b\n" + ".inst 0x45199871 // smmla z17.s, z3.b, z25.b\n" + "ld1b { z25.b }, p2/Z, [x9, #4, MUL VL]\n" + ".inst 0x4518982d // smmla z13.s, z1.b, z24.b\n" + ".inst 0x45189875 // smmla z21.s, z3.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x9, #5, MUL VL]\n" + ".inst 0x4519982a // smmla z10.s, z1.b, z25.b\n" + ".inst 0x45199872 // smmla z18.s, z3.b, z25.b\n" + "ld1b { z25.b }, p2/Z, [x9, #6, MUL VL]\n" + ".inst 0x4518982e // smmla z14.s, z1.b, z24.b\n" + ".inst 0x45189876 // smmla z22.s, z3.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x9, #7, MUL VL]\n" "addvl x9, x9, #8\n" - ".inst 0x4507982b // smmla z11.s, z1.b, z7.b\n" - ".inst 0x45079873 // smmla z19.s, z3.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" - ".inst 0x45069877 // smmla z23.s, z3.b, z6.b\n" + ".inst 0x4519982b // smmla z11.s, z1.b, z25.b\n" + ".inst 0x45199873 // smmla z19.s, z3.b, z25.b\n" + ".inst 0x4518982f // smmla z15.s, z1.b, z24.b\n" + ".inst 0x45189877 // smmla z23.s, z3.b, z24.b\n" "35:" // Height 3: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 30b\n" - "uzp1 z7.d, z8.d, z12.d\n" + "uzp1 z28.d, z8.d, z12.d\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "uzp2 z8.d, z8.d, z12.d\n" - "ld1w { z0.s }, p2/Z, [x14]\n" + "ld1w { z27.s }, p2/Z, [x14]\n" "uzp1 z12.d, z9.d, z13.d\n" "uzp2 z9.d, z9.d, z13.d\n" - "ld1w { z1.s }, p2/Z, [x14, #1, MUL VL]\n" - "ld1w { z2.s }, p2/Z, [x14, #2, MUL VL]\n" + "ld1w { z26.s }, p2/Z, [x14, #1, MUL VL]\n" + "ld1w { z25.s }, p2/Z, [x14, #2, MUL VL]\n" "uzp1 z13.d, z10.d, z14.d\n" "uzp2 z10.d, z10.d, z14.d\n" - "ld1w { z3.s }, p2/Z, [x14, #3, MUL VL]\n" - "add x24, x11, x20\n" + "ld1w { z24.s }, p2/Z, [x14, #3, MUL VL]\n" + "add x26, x11, x20\n" "uzp1 z14.d, z11.d, z15.d\n" "uzp2 z11.d, z11.d, z15.d\n" - "add x23, x24, x20\n" + "add x25, x26, x20\n" "addvl x14, x14, #4\n" "uzp1 z16.d, z16.d, z20.d\n" "uzp1 z17.d, z17.d, z21.d\n" "uzp1 z18.d, z18.d, z22.d\n" "uzp1 z19.d, z19.d, z23.d\n" - "mov z23.d, z7.d\n" - "add z23.s, z23.s, z0.s\n" - "add z12.s, z12.s, z1.s\n" - "add z13.s, z13.s, z2.s\n" - "add z14.s, z14.s, z3.s\n" - "add z8.s, z8.s, z0.s\n" - "add z9.s, z9.s, z1.s\n" - "add z10.s, z10.s, z2.s\n" - "add z11.s, z11.s, z3.s\n" - "add z16.s, z16.s, z0.s\n" - "add z17.s, z17.s, z1.s\n" - "add z18.s, z18.s, z2.s\n" - "add z19.s, z19.s, z3.s\n" + "mov z23.d, z28.d\n" + "add z23.s, z23.s, z27.s\n" + "add z12.s, z12.s, z26.s\n" + "add z13.s, z13.s, z25.s\n" + "add z14.s, z14.s, z24.s\n" + "add z8.s, z8.s, z27.s\n" + "add z9.s, z9.s, z26.s\n" + "add z10.s, z10.s, z25.s\n" + "add z11.s, z11.s, z24.s\n" + "add z16.s, z16.s, z27.s\n" + "add z17.s, z17.s, z26.s\n" + "add z18.s, z18.s, z25.s\n" + "add z19.s, z19.s, z24.s\n" "tbz %x[flags], #4, 36f\n" "ld1w { z0.s }, p2/Z, [x12]\n" "ld1w { z4.s }, p2/Z, [x13]\n" @@ -799,10 +799,10 @@ void sve_hybrid_s8qs_mmla_6x4VL ( "addvl x13, x13, #4\n" "b 37f\n" "36:" // Height 3: per layer parameters - "add x26, %x[qp], %[per_layer_right_shift]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1rw { z0.s }, p2/Z, [x26]\n" - "ld1rw { z4.s }, p2/Z, [x25]\n" + "add x21, %x[qp], %[per_layer_right_shift]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1rw { z0.s }, p2/Z, [x21]\n" + "ld1rw { z4.s }, p2/Z, [x20]\n" "mov z1.d, z0.d\n" "mov z5.d, z4.d\n" "mov z2.d, z0.d\n" @@ -823,109 +823,109 @@ void sve_hybrid_s8qs_mmla_6x4VL ( ".inst 0x04a67652 // sqrdmulh z18.s, z18.s, z6.s\n" ".inst 0x04a77673 // sqrdmulh z19.s, z19.s, z7.s\n" "tbz %x[flags], #5, 38f\n" - "and z4.d, z23.d, z0.d\n" - "and z5.d, z12.d, z1.d\n" - "and z6.d, z13.d, z2.d\n" - "and z7.d, z14.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z23.s, z23.s, z4.s\n" - "sqadd z12.s, z12.s, z5.s\n" - "sqadd z13.s, z13.s, z6.s\n" - "sqadd z14.s, z14.s, z7.s\n" - "and z4.d, z8.d, z0.d\n" - "and z5.d, z9.d, z1.d\n" - "and z6.d, z10.d, z2.d\n" - "and z7.d, z11.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z8.s, z8.s, z4.s\n" - "sqadd z9.s, z9.s, z5.s\n" - "sqadd z10.s, z10.s, z6.s\n" - "sqadd z11.s, z11.s, z7.s\n" - "and z4.d, z16.d, z0.d\n" - "and z5.d, z17.d, z1.d\n" - "and z6.d, z18.d, z2.d\n" - "and z7.d, z19.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z16.s, z16.s, z4.s\n" - "sqadd z17.s, z17.s, z5.s\n" - "sqadd z18.s, z18.s, z6.s\n" - "sqadd z19.s, z19.s, z7.s\n" + "and z24.d, z23.d, z0.d\n" + "and z22.d, z12.d, z1.d\n" + "and z21.d, z13.d, z2.d\n" + "and z20.d, z14.d, z3.d\n" + "asr z24.s, z24.s, #0x1f\n" + "asr z22.s, z22.s, #0x1f\n" + "asr z21.s, z21.s, #0x1f\n" + "asr z20.s, z20.s, #0x1f\n" + "sqadd z23.s, z23.s, z24.s\n" + "sqadd z12.s, z12.s, z22.s\n" + "sqadd z13.s, z13.s, z21.s\n" + "sqadd z14.s, z14.s, z20.s\n" + "and z24.d, z8.d, z0.d\n" + "and z22.d, z9.d, z1.d\n" + "and z21.d, z10.d, z2.d\n" + "and z20.d, z11.d, z3.d\n" + "asr z24.s, z24.s, #0x1f\n" + "asr z22.s, z22.s, #0x1f\n" + "asr z21.s, z21.s, #0x1f\n" + "asr z20.s, z20.s, #0x1f\n" + "sqadd z8.s, z8.s, z24.s\n" + "sqadd z9.s, z9.s, z22.s\n" + "sqadd z10.s, z10.s, z21.s\n" + "sqadd z11.s, z11.s, z20.s\n" + "and z24.d, z16.d, z0.d\n" + "and z22.d, z17.d, z1.d\n" + "and z21.d, z18.d, z2.d\n" + "and z20.d, z19.d, z3.d\n" + "asr z24.s, z24.s, #0x1f\n" + "asr z22.s, z22.s, #0x1f\n" + "asr z21.s, z21.s, #0x1f\n" + "asr z20.s, z20.s, #0x1f\n" + "sqadd z16.s, z16.s, z24.s\n" + "sqadd z17.s, z17.s, z22.s\n" + "sqadd z18.s, z18.s, z21.s\n" + "sqadd z19.s, z19.s, z20.s\n" "38:" // Height 3: no shift correction - "add x25, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x25]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1rw { z21.s }, p2/Z, [x20]\n" ".inst 0x44828817 // srshl z23.s, p2/M, z23.s, z0.s\n" - "add z23.s, z23.s, z4.s\n" + "add z23.s, z23.s, z21.s\n" ".inst 0x4482882c // srshl z12.s, p2/M, z12.s, z1.s\n" ".inst 0x4482884d // srshl z13.s, p2/M, z13.s, z2.s\n" - "add z12.s, z12.s, z4.s\n" - "add z13.s, z13.s, z4.s\n" + "add z12.s, z12.s, z21.s\n" + "add z13.s, z13.s, z21.s\n" ".inst 0x4482886e // srshl z14.s, p2/M, z14.s, z3.s\n" ".inst 0x44828808 // srshl z8.s, p2/M, z8.s, z0.s\n" - "add z14.s, z14.s, z4.s\n" - "add z8.s, z8.s, z4.s\n" + "add z14.s, z14.s, z21.s\n" + "add z8.s, z8.s, z21.s\n" ".inst 0x44828829 // srshl z9.s, p2/M, z9.s, z1.s\n" ".inst 0x4482884a // srshl z10.s, p2/M, z10.s, z2.s\n" - "add z9.s, z9.s, z4.s\n" - "add z10.s, z10.s, z4.s\n" + "add z9.s, z9.s, z21.s\n" + "add z10.s, z10.s, z21.s\n" ".inst 0x4482886b // srshl z11.s, p2/M, z11.s, z3.s\n" ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n" - "add z11.s, z11.s, z4.s\n" - "add z16.s, z16.s, z4.s\n" + "add z11.s, z11.s, z21.s\n" + "add z16.s, z16.s, z21.s\n" ".inst 0x44828831 // srshl z17.s, p2/M, z17.s, z1.s\n" ".inst 0x44828852 // srshl z18.s, p2/M, z18.s, z2.s\n" - "add z17.s, z17.s, z4.s\n" - "add z18.s, z18.s, z4.s\n" + "add z17.s, z17.s, z21.s\n" + "add z18.s, z18.s, z21.s\n" ".inst 0x44828873 // srshl z19.s, p2/M, z19.s, z3.s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x25]\n" - "add z19.s, z19.s, z4.s\n" - "add x25, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x25]\n" - "smin z23.s, p2/M, z23.s, z6.s\n" - "smin z12.s, p2/M, z12.s, z6.s\n" - "smin z13.s, p2/M, z13.s, z6.s\n" - "smin z14.s, p2/M, z14.s, z6.s\n" - "smin z8.s, p2/M, z8.s, z6.s\n" - "smin z9.s, p2/M, z9.s, z6.s\n" - "smin z10.s, p2/M, z10.s, z6.s\n" - "smin z11.s, p2/M, z11.s, z6.s\n" - "smin z16.s, p2/M, z16.s, z6.s\n" - "smin z17.s, p2/M, z17.s, z6.s\n" - "smin z18.s, p2/M, z18.s, z6.s\n" - "smin z19.s, p2/M, z19.s, z6.s\n" - "smax z23.s, p2/M, z23.s, z5.s\n" - "smax z12.s, p2/M, z12.s, z5.s\n" - "smax z13.s, p2/M, z13.s, z5.s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1rw { z20.s }, p2/Z, [x20]\n" + "add z19.s, z19.s, z21.s\n" + "add x20, %x[qp], %[minval]\n" + "ld1rw { z21.s }, p2/Z, [x20]\n" + "smin z23.s, p2/M, z23.s, z20.s\n" + "smin z12.s, p2/M, z12.s, z20.s\n" + "smin z13.s, p2/M, z13.s, z20.s\n" + "smin z14.s, p2/M, z14.s, z20.s\n" + "smin z8.s, p2/M, z8.s, z20.s\n" + "smin z9.s, p2/M, z9.s, z20.s\n" + "smin z10.s, p2/M, z10.s, z20.s\n" + "smin z11.s, p2/M, z11.s, z20.s\n" + "smin z16.s, p2/M, z16.s, z20.s\n" + "smin z17.s, p2/M, z17.s, z20.s\n" + "smin z18.s, p2/M, z18.s, z20.s\n" + "smin z19.s, p2/M, z19.s, z20.s\n" + "smax z23.s, p2/M, z23.s, z21.s\n" + "smax z12.s, p2/M, z12.s, z21.s\n" + "smax z13.s, p2/M, z13.s, z21.s\n" "uzp1 z23.h, z23.h, z12.h\n" - "smax z14.s, p2/M, z14.s, z5.s\n" - "smax z8.s, p2/M, z8.s, z5.s\n" - "uzp1 z12.h, z13.h, z14.h\n" - "uzp1 z23.b, z23.b, z12.b\n" - "smax z9.s, p2/M, z9.s, z5.s\n" - "smax z10.s, p2/M, z10.s, z5.s\n" + "smax z14.s, p2/M, z14.s, z21.s\n" + "smax z8.s, p2/M, z8.s, z21.s\n" + "uzp1 z20.h, z13.h, z14.h\n" + "uzp1 z23.b, z23.b, z20.b\n" + "smax z9.s, p2/M, z9.s, z21.s\n" + "smax z10.s, p2/M, z10.s, z21.s\n" "uzp1 z8.h, z8.h, z9.h\n" "st1b { z23.b }, p1, [x11]\n" - "smax z11.s, p2/M, z11.s, z5.s\n" - "smax z16.s, p2/M, z16.s, z5.s\n" - "uzp1 z9.h, z10.h, z11.h\n" - "uzp1 z8.b, z8.b, z9.b\n" - "smax z17.s, p2/M, z17.s, z5.s\n" - "smax z18.s, p2/M, z18.s, z5.s\n" + "smax z11.s, p2/M, z11.s, z21.s\n" + "smax z16.s, p2/M, z16.s, z21.s\n" + "uzp1 z20.h, z10.h, z11.h\n" + "uzp1 z8.b, z8.b, z20.b\n" + "smax z17.s, p2/M, z17.s, z21.s\n" + "smax z18.s, p2/M, z18.s, z21.s\n" "uzp1 z16.h, z16.h, z17.h\n" - "st1b { z8.b }, p1, [x24]\n" - "smax z19.s, p2/M, z19.s, z5.s\n" + "st1b { z8.b }, p1, [x26]\n" + "smax z19.s, p2/M, z19.s, z21.s\n" "uzp1 z17.h, z18.h, z19.h\n" "uzp1 z16.b, z16.b, z17.b\n" - "st1b { z16.b }, p1, [x23]\n" + "st1b { z16.b }, p1, [x25]\n" "addvl x11, x11, #1\n" "39:" // Height 3: Writeback done "decw x10, ALL, MUL #4\n" @@ -963,14 +963,14 @@ void sve_hybrid_s8qs_mmla_6x4VL ( "43:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 44f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 45f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -980,161 +980,161 @@ void sve_hybrid_s8qs_mmla_6x4VL ( "b 45f\n" "44:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" "45:" // Height 4: input setup done "cmp x27, #0x10\n" "ble 47f\n" "46:" // Height 4: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1rqb { z3.b }, p0/Z, [x24]\n" - "ld1rqb { z4.b }, p0/Z, [x23]\n" - "trn2 z1.d, z1.d, z2.d\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1b { z7.b }, p2/Z, [x9]\n" - "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n" - ".inst 0x45079808 // smmla z8.s, z0.b, z7.b\n" - ".inst 0x45079850 // smmla z16.s, z2.b, z7.b\n" - ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n" - ".inst 0x45069854 // smmla z20.s, z2.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n" - ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" - ".inst 0x45079851 // smmla z17.s, z2.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x4506980d // smmla z13.s, z0.b, z6.b\n" - ".inst 0x45069855 // smmla z21.s, z2.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n" + "ld1rqb { z30.b }, p0/Z, [x26]\n" + "ld1rqb { z24.b }, p0/Z, [x25]\n" + "trn1 z29.d, z30.d, z24.d\n" + "ld1rqb { z28.b }, p0/Z, [x24]\n" + "ld1rqb { z27.b }, p0/Z, [x23]\n" + "trn2 z30.d, z30.d, z24.d\n" + "trn1 z26.d, z28.d, z27.d\n" + "ld1b { z25.b }, p2/Z, [x9]\n" + "ld1b { z24.b }, p2/Z, [x9, #1, MUL VL]\n" + ".inst 0x45199ba8 // smmla z8.s, z29.b, z25.b\n" + ".inst 0x45199b50 // smmla z16.s, z26.b, z25.b\n" + ".inst 0x45189bac // smmla z12.s, z29.b, z24.b\n" + ".inst 0x45189b54 // smmla z20.s, z26.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x9, #3, MUL VL]\n" + ".inst 0x45199ba9 // smmla z9.s, z29.b, z25.b\n" + ".inst 0x45199b51 // smmla z17.s, z26.b, z25.b\n" + "ld1b { z25.b }, p2/Z, [x9, #4, MUL VL]\n" + "trn2 z28.d, z28.d, z27.d\n" + ".inst 0x45189bad // smmla z13.s, z29.b, z24.b\n" + ".inst 0x45189b55 // smmla z21.s, z26.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x9, #5, MUL VL]\n" "sub x27, x27, #0x10\n" - ".inst 0x4507980a // smmla z10.s, z0.b, z7.b\n" - ".inst 0x45079852 // smmla z18.s, z2.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n" + ".inst 0x45199baa // smmla z10.s, z29.b, z25.b\n" + ".inst 0x45199b52 // smmla z18.s, z26.b, z25.b\n" + "ld1b { z25.b }, p2/Z, [x9, #6, MUL VL]\n" "cmp x27, #0x10\n" - ".inst 0x4506980e // smmla z14.s, z0.b, z6.b\n" - ".inst 0x45069856 // smmla z22.s, z2.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n" + ".inst 0x45189bae // smmla z14.s, z29.b, z24.b\n" + ".inst 0x45189b56 // smmla z22.s, z26.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x9, #7, MUL VL]\n" "addvl x9, x9, #16\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" - ".inst 0x45079853 // smmla z19.s, z2.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #-8, MUL VL]\n" + ".inst 0x45199bab // smmla z11.s, z29.b, z25.b\n" + ".inst 0x45199b53 // smmla z19.s, z26.b, z25.b\n" + "ld1b { z25.b }, p2/Z, [x9, #-8, MUL VL]\n" "add x26, x26, #0x10\n" - ".inst 0x4506980f // smmla z15.s, z0.b, z6.b\n" - ".inst 0x45069857 // smmla z23.s, z2.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #-7, MUL VL]\n" + ".inst 0x45189baf // smmla z15.s, z29.b, z24.b\n" + ".inst 0x45189b57 // smmla z23.s, z26.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x9, #-7, MUL VL]\n" "add x25, x25, #0x10\n" - ".inst 0x45079828 // smmla z8.s, z1.b, z7.b\n" - ".inst 0x45079870 // smmla z16.s, z3.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #-6, MUL VL]\n" + ".inst 0x45199bc8 // smmla z8.s, z30.b, z25.b\n" + ".inst 0x45199b90 // smmla z16.s, z28.b, z25.b\n" + "ld1b { z25.b }, p2/Z, [x9, #-6, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x4506982c // smmla z12.s, z1.b, z6.b\n" - ".inst 0x45069874 // smmla z20.s, z3.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #-5, MUL VL]\n" + ".inst 0x45189bcc // smmla z12.s, z30.b, z24.b\n" + ".inst 0x45189b94 // smmla z20.s, z28.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x9, #-5, MUL VL]\n" "add x23, x23, #0x10\n" - ".inst 0x45079829 // smmla z9.s, z1.b, z7.b\n" - ".inst 0x45079871 // smmla z17.s, z3.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #-4, MUL VL]\n" - ".inst 0x4506982d // smmla z13.s, z1.b, z6.b\n" - ".inst 0x45069875 // smmla z21.s, z3.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #-3, MUL VL]\n" - ".inst 0x4507982a // smmla z10.s, z1.b, z7.b\n" - ".inst 0x45079872 // smmla z18.s, z3.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #-2, MUL VL]\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - ".inst 0x45069876 // smmla z22.s, z3.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #-1, MUL VL]\n" - ".inst 0x4507982b // smmla z11.s, z1.b, z7.b\n" - ".inst 0x45079873 // smmla z19.s, z3.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" - ".inst 0x45069877 // smmla z23.s, z3.b, z6.b\n" + ".inst 0x45199bc9 // smmla z9.s, z30.b, z25.b\n" + ".inst 0x45199b91 // smmla z17.s, z28.b, z25.b\n" + "ld1b { z25.b }, p2/Z, [x9, #-4, MUL VL]\n" + ".inst 0x45189bcd // smmla z13.s, z30.b, z24.b\n" + ".inst 0x45189b95 // smmla z21.s, z28.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x9, #-3, MUL VL]\n" + ".inst 0x45199bca // smmla z10.s, z30.b, z25.b\n" + ".inst 0x45199b92 // smmla z18.s, z28.b, z25.b\n" + "ld1b { z25.b }, p2/Z, [x9, #-2, MUL VL]\n" + ".inst 0x45189bce // smmla z14.s, z30.b, z24.b\n" + ".inst 0x45189b96 // smmla z22.s, z28.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x9, #-1, MUL VL]\n" + ".inst 0x45199bcb // smmla z11.s, z30.b, z25.b\n" + ".inst 0x45199b93 // smmla z19.s, z28.b, z25.b\n" + ".inst 0x45189bcf // smmla z15.s, z30.b, z24.b\n" + ".inst 0x45189b97 // smmla z23.s, z28.b, z24.b\n" "bgt 46b\n" "47:" // Height 4: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" + "ld1rqb { z24.b }, p0/Z, [x25]\n" + "trn1 z28.d, z1.d, z24.d\n" "ld1rqb { z3.b }, p0/Z, [x24]\n" - "ld1rqb { z4.b }, p0/Z, [x23]\n" - "trn2 z1.d, z1.d, z2.d\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1b { z7.b }, p2/Z, [x9]\n" - "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n" - ".inst 0x45079808 // smmla z8.s, z0.b, z7.b\n" - ".inst 0x45079850 // smmla z16.s, z2.b, z7.b\n" - ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n" - ".inst 0x45069854 // smmla z20.s, z2.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n" - ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" - ".inst 0x45079851 // smmla z17.s, z2.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n" + "ld1rqb { z27.b }, p0/Z, [x23]\n" + "trn2 z1.d, z1.d, z24.d\n" + "trn1 z26.d, z3.d, z27.d\n" + "ld1b { z25.b }, p2/Z, [x9]\n" + "ld1b { z24.b }, p2/Z, [x9, #1, MUL VL]\n" + ".inst 0x45199b88 // smmla z8.s, z28.b, z25.b\n" + ".inst 0x45199b50 // smmla z16.s, z26.b, z25.b\n" + ".inst 0x45189b8c // smmla z12.s, z28.b, z24.b\n" + ".inst 0x45189b54 // smmla z20.s, z26.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x9, #3, MUL VL]\n" + ".inst 0x45199b89 // smmla z9.s, z28.b, z25.b\n" + ".inst 0x45199b51 // smmla z17.s, z26.b, z25.b\n" + "ld1b { z25.b }, p2/Z, [x9, #4, MUL VL]\n" "subs x27, x27, #0x8\n" - ".inst 0x4506980d // smmla z13.s, z0.b, z6.b\n" - ".inst 0x45069855 // smmla z21.s, z2.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x4507980a // smmla z10.s, z0.b, z7.b\n" - ".inst 0x45079852 // smmla z18.s, z2.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n" - ".inst 0x4506980e // smmla z14.s, z0.b, z6.b\n" - ".inst 0x45069856 // smmla z22.s, z2.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n" + ".inst 0x45189b8d // smmla z13.s, z28.b, z24.b\n" + ".inst 0x45189b55 // smmla z21.s, z26.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x9, #5, MUL VL]\n" + "trn2 z3.d, z3.d, z27.d\n" + ".inst 0x45199b8a // smmla z10.s, z28.b, z25.b\n" + ".inst 0x45199b52 // smmla z18.s, z26.b, z25.b\n" + "ld1b { z25.b }, p2/Z, [x9, #6, MUL VL]\n" + ".inst 0x45189b8e // smmla z14.s, z28.b, z24.b\n" + ".inst 0x45189b56 // smmla z22.s, z26.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x9, #7, MUL VL]\n" "addvl x9, x9, #8\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" - ".inst 0x45079853 // smmla z19.s, z2.b, z7.b\n" - ".inst 0x4506980f // smmla z15.s, z0.b, z6.b\n" - ".inst 0x45069857 // smmla z23.s, z2.b, z6.b\n" + ".inst 0x45199b8b // smmla z11.s, z28.b, z25.b\n" + ".inst 0x45199b53 // smmla z19.s, z26.b, z25.b\n" + ".inst 0x45189b8f // smmla z15.s, z28.b, z24.b\n" + ".inst 0x45189b57 // smmla z23.s, z26.b, z24.b\n" "ble 48f\n" - "ld1b { z7.b }, p2/Z, [x9]\n" - "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n" - ".inst 0x45079828 // smmla z8.s, z1.b, z7.b\n" - ".inst 0x45079870 // smmla z16.s, z3.b, z7.b\n" - ".inst 0x4506982c // smmla z12.s, z1.b, z6.b\n" - ".inst 0x45069874 // smmla z20.s, z3.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n" - ".inst 0x45079829 // smmla z9.s, z1.b, z7.b\n" - ".inst 0x45079871 // smmla z17.s, z3.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n" - ".inst 0x4506982d // smmla z13.s, z1.b, z6.b\n" - ".inst 0x45069875 // smmla z21.s, z3.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n" - ".inst 0x4507982a // smmla z10.s, z1.b, z7.b\n" - ".inst 0x45079872 // smmla z18.s, z3.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - ".inst 0x45069876 // smmla z22.s, z3.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n" + "ld1b { z25.b }, p2/Z, [x9]\n" + "ld1b { z24.b }, p2/Z, [x9, #1, MUL VL]\n" + ".inst 0x45199828 // smmla z8.s, z1.b, z25.b\n" + ".inst 0x45199870 // smmla z16.s, z3.b, z25.b\n" + ".inst 0x4518982c // smmla z12.s, z1.b, z24.b\n" + ".inst 0x45189874 // smmla z20.s, z3.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x9, #3, MUL VL]\n" + ".inst 0x45199829 // smmla z9.s, z1.b, z25.b\n" + ".inst 0x45199871 // smmla z17.s, z3.b, z25.b\n" + "ld1b { z25.b }, p2/Z, [x9, #4, MUL VL]\n" + ".inst 0x4518982d // smmla z13.s, z1.b, z24.b\n" + ".inst 0x45189875 // smmla z21.s, z3.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x9, #5, MUL VL]\n" + ".inst 0x4519982a // smmla z10.s, z1.b, z25.b\n" + ".inst 0x45199872 // smmla z18.s, z3.b, z25.b\n" + "ld1b { z25.b }, p2/Z, [x9, #6, MUL VL]\n" + ".inst 0x4518982e // smmla z14.s, z1.b, z24.b\n" + ".inst 0x45189876 // smmla z22.s, z3.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x9, #7, MUL VL]\n" "addvl x9, x9, #8\n" - ".inst 0x4507982b // smmla z11.s, z1.b, z7.b\n" - ".inst 0x45079873 // smmla z19.s, z3.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" - ".inst 0x45069877 // smmla z23.s, z3.b, z6.b\n" + ".inst 0x4519982b // smmla z11.s, z1.b, z25.b\n" + ".inst 0x45199873 // smmla z19.s, z3.b, z25.b\n" + ".inst 0x4518982f // smmla z15.s, z1.b, z24.b\n" + ".inst 0x45189877 // smmla z23.s, z3.b, z24.b\n" "48:" // Height 4: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 43b\n" - "uzp1 z7.d, z8.d, z12.d\n" + "uzp1 z28.d, z8.d, z12.d\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "uzp2 z8.d, z8.d, z12.d\n" - "ld1w { z0.s }, p2/Z, [x14]\n" + "ld1w { z27.s }, p2/Z, [x14]\n" "uzp1 z12.d, z9.d, z13.d\n" "uzp2 z9.d, z9.d, z13.d\n" - "ld1w { z1.s }, p2/Z, [x14, #1, MUL VL]\n" - "ld1w { z2.s }, p2/Z, [x14, #2, MUL VL]\n" + "ld1w { z26.s }, p2/Z, [x14, #1, MUL VL]\n" + "ld1w { z25.s }, p2/Z, [x14, #2, MUL VL]\n" "uzp1 z13.d, z10.d, z14.d\n" "uzp2 z10.d, z10.d, z14.d\n" - "ld1w { z3.s }, p2/Z, [x14, #3, MUL VL]\n" - "add x24, x11, x20\n" + "ld1w { z24.s }, p2/Z, [x14, #3, MUL VL]\n" + "add x26, x11, x20\n" "uzp1 z14.d, z11.d, z15.d\n" "uzp2 z11.d, z11.d, z15.d\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" + "add x25, x26, x20\n" + "add x24, x25, x20\n" "uzp1 z15.d, z16.d, z20.d\n" "uzp2 z16.d, z16.d, z20.d\n" "addvl x14, x14, #4\n" @@ -1144,23 +1144,23 @@ void sve_hybrid_s8qs_mmla_6x4VL ( "uzp2 z18.d, z18.d, z22.d\n" "uzp1 z22.d, z19.d, z23.d\n" "uzp2 z19.d, z19.d, z23.d\n" - "mov z23.d, z7.d\n" - "add z23.s, z23.s, z0.s\n" - "add z12.s, z12.s, z1.s\n" - "add z13.s, z13.s, z2.s\n" - "add z14.s, z14.s, z3.s\n" - "add z8.s, z8.s, z0.s\n" - "add z9.s, z9.s, z1.s\n" - "add z10.s, z10.s, z2.s\n" - "add z11.s, z11.s, z3.s\n" - "add z15.s, z15.s, z0.s\n" - "add z20.s, z20.s, z1.s\n" - "add z21.s, z21.s, z2.s\n" - "add z22.s, z22.s, z3.s\n" - "add z16.s, z16.s, z0.s\n" - "add z17.s, z17.s, z1.s\n" - "add z18.s, z18.s, z2.s\n" - "add z19.s, z19.s, z3.s\n" + "mov z23.d, z28.d\n" + "add z23.s, z23.s, z27.s\n" + "add z12.s, z12.s, z26.s\n" + "add z13.s, z13.s, z25.s\n" + "add z14.s, z14.s, z24.s\n" + "add z8.s, z8.s, z27.s\n" + "add z9.s, z9.s, z26.s\n" + "add z10.s, z10.s, z25.s\n" + "add z11.s, z11.s, z24.s\n" + "add z15.s, z15.s, z27.s\n" + "add z20.s, z20.s, z26.s\n" + "add z21.s, z21.s, z25.s\n" + "add z22.s, z22.s, z24.s\n" + "add z16.s, z16.s, z27.s\n" + "add z17.s, z17.s, z26.s\n" + "add z18.s, z18.s, z25.s\n" + "add z19.s, z19.s, z24.s\n" "tbz %x[flags], #4, 49f\n" "ld1w { z0.s }, p2/Z, [x12]\n" "ld1w { z4.s }, p2/Z, [x13]\n" @@ -1174,10 +1174,10 @@ void sve_hybrid_s8qs_mmla_6x4VL ( "addvl x13, x13, #4\n" "b 50f\n" "49:" // Height 4: per layer parameters - "add x26, %x[qp], %[per_layer_right_shift]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1rw { z0.s }, p2/Z, [x26]\n" - "ld1rw { z4.s }, p2/Z, [x25]\n" + "add x21, %x[qp], %[per_layer_right_shift]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1rw { z0.s }, p2/Z, [x21]\n" + "ld1rw { z4.s }, p2/Z, [x20]\n" "mov z1.d, z0.d\n" "mov z5.d, z4.d\n" "mov z2.d, z0.d\n" @@ -1202,141 +1202,141 @@ void sve_hybrid_s8qs_mmla_6x4VL ( ".inst 0x04a67652 // sqrdmulh z18.s, z18.s, z6.s\n" ".inst 0x04a77673 // sqrdmulh z19.s, z19.s, z7.s\n" "tbz %x[flags], #5, 51f\n" - "and z4.d, z23.d, z0.d\n" - "and z5.d, z12.d, z1.d\n" - "and z6.d, z13.d, z2.d\n" - "and z7.d, z14.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z23.s, z23.s, z4.s\n" - "sqadd z12.s, z12.s, z5.s\n" - "sqadd z13.s, z13.s, z6.s\n" - "sqadd z14.s, z14.s, z7.s\n" - "and z4.d, z8.d, z0.d\n" - "and z5.d, z9.d, z1.d\n" - "and z6.d, z10.d, z2.d\n" - "and z7.d, z11.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z8.s, z8.s, z4.s\n" - "sqadd z9.s, z9.s, z5.s\n" - "sqadd z10.s, z10.s, z6.s\n" - "sqadd z11.s, z11.s, z7.s\n" - "and z4.d, z15.d, z0.d\n" - "and z5.d, z20.d, z1.d\n" - "and z6.d, z21.d, z2.d\n" - "and z7.d, z22.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z15.s, z15.s, z4.s\n" - "sqadd z20.s, z20.s, z5.s\n" - "sqadd z21.s, z21.s, z6.s\n" - "sqadd z22.s, z22.s, z7.s\n" - "and z4.d, z16.d, z0.d\n" - "and z5.d, z17.d, z1.d\n" - "and z6.d, z18.d, z2.d\n" - "and z7.d, z19.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z16.s, z16.s, z4.s\n" - "sqadd z17.s, z17.s, z5.s\n" - "sqadd z18.s, z18.s, z6.s\n" - "sqadd z19.s, z19.s, z7.s\n" + "and z27.d, z23.d, z0.d\n" + "and z26.d, z12.d, z1.d\n" + "and z25.d, z13.d, z2.d\n" + "and z24.d, z14.d, z3.d\n" + "asr z27.s, z27.s, #0x1f\n" + "asr z26.s, z26.s, #0x1f\n" + "asr z25.s, z25.s, #0x1f\n" + "asr z24.s, z24.s, #0x1f\n" + "sqadd z23.s, z23.s, z27.s\n" + "sqadd z12.s, z12.s, z26.s\n" + "sqadd z13.s, z13.s, z25.s\n" + "sqadd z14.s, z14.s, z24.s\n" + "and z27.d, z8.d, z0.d\n" + "and z26.d, z9.d, z1.d\n" + "and z25.d, z10.d, z2.d\n" + "and z24.d, z11.d, z3.d\n" + "asr z27.s, z27.s, #0x1f\n" + "asr z26.s, z26.s, #0x1f\n" + "asr z25.s, z25.s, #0x1f\n" + "asr z24.s, z24.s, #0x1f\n" + "sqadd z8.s, z8.s, z27.s\n" + "sqadd z9.s, z9.s, z26.s\n" + "sqadd z10.s, z10.s, z25.s\n" + "sqadd z11.s, z11.s, z24.s\n" + "and z27.d, z15.d, z0.d\n" + "and z26.d, z20.d, z1.d\n" + "and z25.d, z21.d, z2.d\n" + "and z24.d, z22.d, z3.d\n" + "asr z27.s, z27.s, #0x1f\n" + "asr z26.s, z26.s, #0x1f\n" + "asr z25.s, z25.s, #0x1f\n" + "asr z24.s, z24.s, #0x1f\n" + "sqadd z15.s, z15.s, z27.s\n" + "sqadd z20.s, z20.s, z26.s\n" + "sqadd z21.s, z21.s, z25.s\n" + "sqadd z22.s, z22.s, z24.s\n" + "and z27.d, z16.d, z0.d\n" + "and z26.d, z17.d, z1.d\n" + "and z25.d, z18.d, z2.d\n" + "and z24.d, z19.d, z3.d\n" + "asr z27.s, z27.s, #0x1f\n" + "asr z26.s, z26.s, #0x1f\n" + "asr z25.s, z25.s, #0x1f\n" + "asr z24.s, z24.s, #0x1f\n" + "sqadd z16.s, z16.s, z27.s\n" + "sqadd z17.s, z17.s, z26.s\n" + "sqadd z18.s, z18.s, z25.s\n" + "sqadd z19.s, z19.s, z24.s\n" "51:" // Height 4: no shift correction - "add x25, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x25]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1rw { z25.s }, p2/Z, [x20]\n" ".inst 0x44828817 // srshl z23.s, p2/M, z23.s, z0.s\n" - "add z23.s, z23.s, z4.s\n" + "add z23.s, z23.s, z25.s\n" ".inst 0x4482882c // srshl z12.s, p2/M, z12.s, z1.s\n" ".inst 0x4482884d // srshl z13.s, p2/M, z13.s, z2.s\n" - "add z12.s, z12.s, z4.s\n" - "add z13.s, z13.s, z4.s\n" + "add z12.s, z12.s, z25.s\n" + "add z13.s, z13.s, z25.s\n" ".inst 0x4482886e // srshl z14.s, p2/M, z14.s, z3.s\n" ".inst 0x44828808 // srshl z8.s, p2/M, z8.s, z0.s\n" - "add z14.s, z14.s, z4.s\n" - "add z8.s, z8.s, z4.s\n" + "add z14.s, z14.s, z25.s\n" + "add z8.s, z8.s, z25.s\n" ".inst 0x44828829 // srshl z9.s, p2/M, z9.s, z1.s\n" ".inst 0x4482884a // srshl z10.s, p2/M, z10.s, z2.s\n" - "add z9.s, z9.s, z4.s\n" - "add z10.s, z10.s, z4.s\n" + "add z9.s, z9.s, z25.s\n" + "add z10.s, z10.s, z25.s\n" ".inst 0x4482886b // srshl z11.s, p2/M, z11.s, z3.s\n" ".inst 0x4482880f // srshl z15.s, p2/M, z15.s, z0.s\n" - "add z11.s, z11.s, z4.s\n" - "add z15.s, z15.s, z4.s\n" + "add z11.s, z11.s, z25.s\n" + "add z15.s, z15.s, z25.s\n" ".inst 0x44828834 // srshl z20.s, p2/M, z20.s, z1.s\n" ".inst 0x44828855 // srshl z21.s, p2/M, z21.s, z2.s\n" - "add z20.s, z20.s, z4.s\n" - "add z21.s, z21.s, z4.s\n" + "add z20.s, z20.s, z25.s\n" + "add z21.s, z21.s, z25.s\n" ".inst 0x44828876 // srshl z22.s, p2/M, z22.s, z3.s\n" ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n" - "add z22.s, z22.s, z4.s\n" - "add z16.s, z16.s, z4.s\n" + "add z22.s, z22.s, z25.s\n" + "add z16.s, z16.s, z25.s\n" ".inst 0x44828831 // srshl z17.s, p2/M, z17.s, z1.s\n" ".inst 0x44828852 // srshl z18.s, p2/M, z18.s, z2.s\n" - "add z17.s, z17.s, z4.s\n" - "add z18.s, z18.s, z4.s\n" + "add z17.s, z17.s, z25.s\n" + "add z18.s, z18.s, z25.s\n" ".inst 0x44828873 // srshl z19.s, p2/M, z19.s, z3.s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x25]\n" - "add z19.s, z19.s, z4.s\n" - "add x25, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x25]\n" - "smin z23.s, p2/M, z23.s, z6.s\n" - "smin z12.s, p2/M, z12.s, z6.s\n" - "smin z13.s, p2/M, z13.s, z6.s\n" - "smin z14.s, p2/M, z14.s, z6.s\n" - "smin z8.s, p2/M, z8.s, z6.s\n" - "smin z9.s, p2/M, z9.s, z6.s\n" - "smin z10.s, p2/M, z10.s, z6.s\n" - "smin z11.s, p2/M, z11.s, z6.s\n" - "smin z15.s, p2/M, z15.s, z6.s\n" - "smin z20.s, p2/M, z20.s, z6.s\n" - "smin z21.s, p2/M, z21.s, z6.s\n" - "smin z22.s, p2/M, z22.s, z6.s\n" - "smin z16.s, p2/M, z16.s, z6.s\n" - "smin z17.s, p2/M, z17.s, z6.s\n" - "smin z18.s, p2/M, z18.s, z6.s\n" - "smin z19.s, p2/M, z19.s, z6.s\n" - "smax z23.s, p2/M, z23.s, z5.s\n" - "smax z12.s, p2/M, z12.s, z5.s\n" - "smax z13.s, p2/M, z13.s, z5.s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1rw { z24.s }, p2/Z, [x20]\n" + "add z19.s, z19.s, z25.s\n" + "add x20, %x[qp], %[minval]\n" + "ld1rw { z25.s }, p2/Z, [x20]\n" + "smin z23.s, p2/M, z23.s, z24.s\n" + "smin z12.s, p2/M, z12.s, z24.s\n" + "smin z13.s, p2/M, z13.s, z24.s\n" + "smin z14.s, p2/M, z14.s, z24.s\n" + "smin z8.s, p2/M, z8.s, z24.s\n" + "smin z9.s, p2/M, z9.s, z24.s\n" + "smin z10.s, p2/M, z10.s, z24.s\n" + "smin z11.s, p2/M, z11.s, z24.s\n" + "smin z15.s, p2/M, z15.s, z24.s\n" + "smin z20.s, p2/M, z20.s, z24.s\n" + "smin z21.s, p2/M, z21.s, z24.s\n" + "smin z22.s, p2/M, z22.s, z24.s\n" + "smin z16.s, p2/M, z16.s, z24.s\n" + "smin z17.s, p2/M, z17.s, z24.s\n" + "smin z18.s, p2/M, z18.s, z24.s\n" + "smin z19.s, p2/M, z19.s, z24.s\n" + "smax z23.s, p2/M, z23.s, z25.s\n" + "smax z12.s, p2/M, z12.s, z25.s\n" + "smax z13.s, p2/M, z13.s, z25.s\n" "uzp1 z23.h, z23.h, z12.h\n" - "smax z14.s, p2/M, z14.s, z5.s\n" - "smax z8.s, p2/M, z8.s, z5.s\n" - "uzp1 z12.h, z13.h, z14.h\n" - "uzp1 z23.b, z23.b, z12.b\n" - "smax z9.s, p2/M, z9.s, z5.s\n" - "smax z10.s, p2/M, z10.s, z5.s\n" + "smax z14.s, p2/M, z14.s, z25.s\n" + "smax z8.s, p2/M, z8.s, z25.s\n" + "uzp1 z24.h, z13.h, z14.h\n" + "uzp1 z23.b, z23.b, z24.b\n" + "smax z9.s, p2/M, z9.s, z25.s\n" + "smax z10.s, p2/M, z10.s, z25.s\n" "uzp1 z8.h, z8.h, z9.h\n" "st1b { z23.b }, p1, [x11]\n" - "smax z11.s, p2/M, z11.s, z5.s\n" - "smax z15.s, p2/M, z15.s, z5.s\n" - "uzp1 z9.h, z10.h, z11.h\n" - "uzp1 z8.b, z8.b, z9.b\n" - "smax z20.s, p2/M, z20.s, z5.s\n" - "smax z21.s, p2/M, z21.s, z5.s\n" + "smax z11.s, p2/M, z11.s, z25.s\n" + "smax z15.s, p2/M, z15.s, z25.s\n" + "uzp1 z23.h, z10.h, z11.h\n" + "uzp1 z8.b, z8.b, z23.b\n" + "smax z20.s, p2/M, z20.s, z25.s\n" + "smax z21.s, p2/M, z21.s, z25.s\n" "uzp1 z15.h, z15.h, z20.h\n" - "st1b { z8.b }, p1, [x24]\n" - "smax z22.s, p2/M, z22.s, z5.s\n" - "smax z16.s, p2/M, z16.s, z5.s\n" + "st1b { z8.b }, p1, [x26]\n" + "smax z22.s, p2/M, z22.s, z25.s\n" + "smax z16.s, p2/M, z16.s, z25.s\n" "uzp1 z20.h, z21.h, z22.h\n" "uzp1 z15.b, z15.b, z20.b\n" - "smax z17.s, p2/M, z17.s, z5.s\n" - "smax z18.s, p2/M, z18.s, z5.s\n" + "smax z17.s, p2/M, z17.s, z25.s\n" + "smax z18.s, p2/M, z18.s, z25.s\n" "uzp1 z16.h, z16.h, z17.h\n" - "st1b { z15.b }, p1, [x23]\n" - "smax z19.s, p2/M, z19.s, z5.s\n" + "st1b { z15.b }, p1, [x25]\n" + "smax z19.s, p2/M, z19.s, z25.s\n" "uzp1 z17.h, z18.h, z19.h\n" "uzp1 z16.b, z16.b, z17.b\n" - "st1b { z16.b }, p1, [x22]\n" + "st1b { z16.b }, p1, [x24]\n" "addvl x11, x11, #1\n" "52:" // Height 4: Writeback done "decw x10, ALL, MUL #4\n" @@ -1382,15 +1382,15 @@ void sve_hybrid_s8qs_mmla_6x4VL ( "56:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 57f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 58f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -1401,204 +1401,204 @@ void sve_hybrid_s8qs_mmla_6x4VL ( "b 58f\n" "57:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" "58:" // Height 5: input setup done "cmp x27, #0x10\n" "ble 60f\n" "59:" // Height 5: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" - "ld1rqb { z3.b }, p0/Z, [x24]\n" - "ld1rqb { z4.b }, p0/Z, [x23]\n" - "trn1 z0.d, z1.d, z2.d\n" - "trn2 z1.d, z1.d, z2.d\n" - "ld1rqb { z5.b }, p0/Z, [x22]\n" - "trn1 z2.d, z3.d, z4.d\n" - "trn2 z3.d, z3.d, z4.d\n" - "ld1b { z7.b }, p2/Z, [x9]\n" - "trn1 z4.d, z5.d, z6.d\n" - "trn2 z5.d, z5.d, z6.d\n" - "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n" - ".inst 0x45079808 // smmla z8.s, z0.b, z7.b\n" - ".inst 0x45079850 // smmla z16.s, z2.b, z7.b\n" - ".inst 0x45079898 // smmla z24.s, z4.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1rqb { z6.b }, p0/Z, [x26]\n" + "ld1rqb { z1.b }, p0/Z, [x25]\n" + "ld1rqb { z7.b }, p0/Z, [x24]\n" + "ld1rqb { z2.b }, p0/Z, [x23]\n" + "trn1 z5.d, z6.d, z1.d\n" + "trn2 z6.d, z6.d, z1.d\n" + "ld1rqb { z4.b }, p0/Z, [x22]\n" + "trn1 z3.d, z7.d, z2.d\n" + "trn2 z7.d, z7.d, z2.d\n" + "ld1b { z1.b }, p2/Z, [x9]\n" + "trn1 z2.d, z4.d, z0.d\n" + "trn2 z4.d, z4.d, z0.d\n" + "ld1b { z0.b }, p2/Z, [x9, #1, MUL VL]\n" + ".inst 0x450198a8 // smmla z8.s, z5.b, z1.b\n" + ".inst 0x45019870 // smmla z16.s, z3.b, z1.b\n" + ".inst 0x45019858 // smmla z24.s, z2.b, z1.b\n" + "ld1b { z1.b }, p2/Z, [x9, #2, MUL VL]\n" "sub x27, x27, #0x10\n" - ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n" - ".inst 0x45069854 // smmla z20.s, z2.b, z6.b\n" + ".inst 0x450098ac // smmla z12.s, z5.b, z0.b\n" + ".inst 0x45009874 // smmla z20.s, z3.b, z0.b\n" "cmp x27, #0x10\n" "add x26, x26, #0x10\n" - ".inst 0x4506989c // smmla z28.s, z4.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n" - ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" + ".inst 0x4500985c // smmla z28.s, z2.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #3, MUL VL]\n" + ".inst 0x450198a9 // smmla z9.s, z5.b, z1.b\n" "add x25, x25, #0x10\n" - ".inst 0x45079851 // smmla z17.s, z2.b, z7.b\n" - ".inst 0x45079899 // smmla z25.s, z4.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n" + ".inst 0x45019871 // smmla z17.s, z3.b, z1.b\n" + ".inst 0x45019859 // smmla z25.s, z2.b, z1.b\n" + "ld1b { z1.b }, p2/Z, [x9, #4, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x4506980d // smmla z13.s, z0.b, z6.b\n" - ".inst 0x45069855 // smmla z21.s, z2.b, z6.b\n" + ".inst 0x450098ad // smmla z13.s, z5.b, z0.b\n" + ".inst 0x45009875 // smmla z21.s, z3.b, z0.b\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - ".inst 0x4506989d // smmla z29.s, z4.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n" - ".inst 0x4507980a // smmla z10.s, z0.b, z7.b\n" - ".inst 0x45079852 // smmla z18.s, z2.b, z7.b\n" - ".inst 0x4507989a // smmla z26.s, z4.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n" - ".inst 0x4506980e // smmla z14.s, z0.b, z6.b\n" - ".inst 0x45069856 // smmla z22.s, z2.b, z6.b\n" - ".inst 0x4506989e // smmla z30.s, z4.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n" + ".inst 0x4500985d // smmla z29.s, z2.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #5, MUL VL]\n" + ".inst 0x450198aa // smmla z10.s, z5.b, z1.b\n" + ".inst 0x45019872 // smmla z18.s, z3.b, z1.b\n" + ".inst 0x4501985a // smmla z26.s, z2.b, z1.b\n" + "ld1b { z1.b }, p2/Z, [x9, #6, MUL VL]\n" + ".inst 0x450098ae // smmla z14.s, z5.b, z0.b\n" + ".inst 0x45009876 // smmla z22.s, z3.b, z0.b\n" + ".inst 0x4500985e // smmla z30.s, z2.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #7, MUL VL]\n" "addvl x9, x9, #16\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" - ".inst 0x45079853 // smmla z19.s, z2.b, z7.b\n" - ".inst 0x4507989b // smmla z27.s, z4.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #-8, MUL VL]\n" - ".inst 0x4506980f // smmla z15.s, z0.b, z6.b\n" - ".inst 0x45069857 // smmla z23.s, z2.b, z6.b\n" - ".inst 0x4506989f // smmla z31.s, z4.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #-7, MUL VL]\n" - ".inst 0x45079828 // smmla z8.s, z1.b, z7.b\n" - ".inst 0x45079870 // smmla z16.s, z3.b, z7.b\n" - ".inst 0x450798b8 // smmla z24.s, z5.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #-6, MUL VL]\n" - ".inst 0x4506982c // smmla z12.s, z1.b, z6.b\n" - ".inst 0x45069874 // smmla z20.s, z3.b, z6.b\n" - ".inst 0x450698bc // smmla z28.s, z5.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #-5, MUL VL]\n" - ".inst 0x45079829 // smmla z9.s, z1.b, z7.b\n" - ".inst 0x45079871 // smmla z17.s, z3.b, z7.b\n" - ".inst 0x450798b9 // smmla z25.s, z5.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #-4, MUL VL]\n" - ".inst 0x4506982d // smmla z13.s, z1.b, z6.b\n" - ".inst 0x45069875 // smmla z21.s, z3.b, z6.b\n" - ".inst 0x450698bd // smmla z29.s, z5.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #-3, MUL VL]\n" - ".inst 0x4507982a // smmla z10.s, z1.b, z7.b\n" - ".inst 0x45079872 // smmla z18.s, z3.b, z7.b\n" - ".inst 0x450798ba // smmla z26.s, z5.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #-2, MUL VL]\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - ".inst 0x45069876 // smmla z22.s, z3.b, z6.b\n" - ".inst 0x450698be // smmla z30.s, z5.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #-1, MUL VL]\n" - ".inst 0x4507982b // smmla z11.s, z1.b, z7.b\n" - ".inst 0x45079873 // smmla z19.s, z3.b, z7.b\n" - ".inst 0x450798bb // smmla z27.s, z5.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" - ".inst 0x45069877 // smmla z23.s, z3.b, z6.b\n" - ".inst 0x450698bf // smmla z31.s, z5.b, z6.b\n" + ".inst 0x450198ab // smmla z11.s, z5.b, z1.b\n" + ".inst 0x45019873 // smmla z19.s, z3.b, z1.b\n" + ".inst 0x4501985b // smmla z27.s, z2.b, z1.b\n" + "ld1b { z1.b }, p2/Z, [x9, #-8, MUL VL]\n" + ".inst 0x450098af // smmla z15.s, z5.b, z0.b\n" + ".inst 0x45009877 // smmla z23.s, z3.b, z0.b\n" + ".inst 0x4500985f // smmla z31.s, z2.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #-7, MUL VL]\n" + ".inst 0x450198c8 // smmla z8.s, z6.b, z1.b\n" + ".inst 0x450198f0 // smmla z16.s, z7.b, z1.b\n" + ".inst 0x45019898 // smmla z24.s, z4.b, z1.b\n" + "ld1b { z1.b }, p2/Z, [x9, #-6, MUL VL]\n" + ".inst 0x450098cc // smmla z12.s, z6.b, z0.b\n" + ".inst 0x450098f4 // smmla z20.s, z7.b, z0.b\n" + ".inst 0x4500989c // smmla z28.s, z4.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #-5, MUL VL]\n" + ".inst 0x450198c9 // smmla z9.s, z6.b, z1.b\n" + ".inst 0x450198f1 // smmla z17.s, z7.b, z1.b\n" + ".inst 0x45019899 // smmla z25.s, z4.b, z1.b\n" + "ld1b { z1.b }, p2/Z, [x9, #-4, MUL VL]\n" + ".inst 0x450098cd // smmla z13.s, z6.b, z0.b\n" + ".inst 0x450098f5 // smmla z21.s, z7.b, z0.b\n" + ".inst 0x4500989d // smmla z29.s, z4.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #-3, MUL VL]\n" + ".inst 0x450198ca // smmla z10.s, z6.b, z1.b\n" + ".inst 0x450198f2 // smmla z18.s, z7.b, z1.b\n" + ".inst 0x4501989a // smmla z26.s, z4.b, z1.b\n" + "ld1b { z1.b }, p2/Z, [x9, #-2, MUL VL]\n" + ".inst 0x450098ce // smmla z14.s, z6.b, z0.b\n" + ".inst 0x450098f6 // smmla z22.s, z7.b, z0.b\n" + ".inst 0x4500989e // smmla z30.s, z4.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #-1, MUL VL]\n" + ".inst 0x450198cb // smmla z11.s, z6.b, z1.b\n" + ".inst 0x450198f3 // smmla z19.s, z7.b, z1.b\n" + ".inst 0x4501989b // smmla z27.s, z4.b, z1.b\n" + ".inst 0x450098cf // smmla z15.s, z6.b, z0.b\n" + ".inst 0x450098f7 // smmla z23.s, z7.b, z0.b\n" + ".inst 0x4500989f // smmla z31.s, z4.b, z0.b\n" "bgt 59b\n" "60:" // Height 5: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" + "ld1rqb { z4.b }, p0/Z, [x25]\n" "ld1rqb { z3.b }, p0/Z, [x24]\n" - "ld1rqb { z4.b }, p0/Z, [x23]\n" - "trn1 z0.d, z1.d, z2.d\n" - "trn2 z1.d, z1.d, z2.d\n" + "ld1rqb { z2.b }, p0/Z, [x23]\n" + "trn1 z7.d, z1.d, z4.d\n" + "trn2 z1.d, z1.d, z4.d\n" "ld1rqb { z5.b }, p0/Z, [x22]\n" - "trn1 z2.d, z3.d, z4.d\n" - "trn2 z3.d, z3.d, z4.d\n" - "ld1b { z7.b }, p2/Z, [x9]\n" - "trn1 z4.d, z5.d, z6.d\n" - "trn2 z5.d, z5.d, z6.d\n" - "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n" - ".inst 0x45079808 // smmla z8.s, z0.b, z7.b\n" - ".inst 0x45079850 // smmla z16.s, z2.b, z7.b\n" - ".inst 0x45079898 // smmla z24.s, z4.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n" + "trn1 z6.d, z3.d, z2.d\n" + "trn2 z3.d, z3.d, z2.d\n" + "ld1b { z2.b }, p2/Z, [x9]\n" + "trn1 z4.d, z5.d, z0.d\n" + "trn2 z5.d, z5.d, z0.d\n" + "ld1b { z0.b }, p2/Z, [x9, #1, MUL VL]\n" + ".inst 0x450298e8 // smmla z8.s, z7.b, z2.b\n" + ".inst 0x450298d0 // smmla z16.s, z6.b, z2.b\n" + ".inst 0x45029898 // smmla z24.s, z4.b, z2.b\n" + "ld1b { z2.b }, p2/Z, [x9, #2, MUL VL]\n" "subs x27, x27, #0x8\n" - ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n" - ".inst 0x45069854 // smmla z20.s, z2.b, z6.b\n" - ".inst 0x4506989c // smmla z28.s, z4.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n" - ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" - ".inst 0x45079851 // smmla z17.s, z2.b, z7.b\n" - ".inst 0x45079899 // smmla z25.s, z4.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n" - ".inst 0x4506980d // smmla z13.s, z0.b, z6.b\n" - ".inst 0x45069855 // smmla z21.s, z2.b, z6.b\n" - ".inst 0x4506989d // smmla z29.s, z4.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n" - ".inst 0x4507980a // smmla z10.s, z0.b, z7.b\n" - ".inst 0x45079852 // smmla z18.s, z2.b, z7.b\n" - ".inst 0x4507989a // smmla z26.s, z4.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n" - ".inst 0x4506980e // smmla z14.s, z0.b, z6.b\n" - ".inst 0x45069856 // smmla z22.s, z2.b, z6.b\n" - ".inst 0x4506989e // smmla z30.s, z4.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" + ".inst 0x450098ec // smmla z12.s, z7.b, z0.b\n" + ".inst 0x450098d4 // smmla z20.s, z6.b, z0.b\n" + ".inst 0x4500989c // smmla z28.s, z4.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #3, MUL VL]\n" + ".inst 0x450298e9 // smmla z9.s, z7.b, z2.b\n" + ".inst 0x450298d1 // smmla z17.s, z6.b, z2.b\n" + ".inst 0x45029899 // smmla z25.s, z4.b, z2.b\n" + "ld1b { z2.b }, p2/Z, [x9, #4, MUL VL]\n" + ".inst 0x450098ed // smmla z13.s, z7.b, z0.b\n" + ".inst 0x450098d5 // smmla z21.s, z6.b, z0.b\n" + ".inst 0x4500989d // smmla z29.s, z4.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #5, MUL VL]\n" + ".inst 0x450298ea // smmla z10.s, z7.b, z2.b\n" + ".inst 0x450298d2 // smmla z18.s, z6.b, z2.b\n" + ".inst 0x4502989a // smmla z26.s, z4.b, z2.b\n" + "ld1b { z2.b }, p2/Z, [x9, #6, MUL VL]\n" + ".inst 0x450098ee // smmla z14.s, z7.b, z0.b\n" + ".inst 0x450098d6 // smmla z22.s, z6.b, z0.b\n" + ".inst 0x4500989e // smmla z30.s, z4.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #7, MUL VL]\n" + ".inst 0x450298eb // smmla z11.s, z7.b, z2.b\n" "addvl x9, x9, #8\n" - ".inst 0x45079853 // smmla z19.s, z2.b, z7.b\n" - ".inst 0x4507989b // smmla z27.s, z4.b, z7.b\n" - ".inst 0x4506980f // smmla z15.s, z0.b, z6.b\n" - ".inst 0x45069857 // smmla z23.s, z2.b, z6.b\n" - ".inst 0x4506989f // smmla z31.s, z4.b, z6.b\n" + ".inst 0x450298d3 // smmla z19.s, z6.b, z2.b\n" + ".inst 0x4502989b // smmla z27.s, z4.b, z2.b\n" + ".inst 0x450098ef // smmla z15.s, z7.b, z0.b\n" + ".inst 0x450098d7 // smmla z23.s, z6.b, z0.b\n" + ".inst 0x4500989f // smmla z31.s, z4.b, z0.b\n" "ble 61f\n" - "ld1b { z7.b }, p2/Z, [x9]\n" - "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n" - ".inst 0x45079828 // smmla z8.s, z1.b, z7.b\n" - ".inst 0x45079870 // smmla z16.s, z3.b, z7.b\n" - ".inst 0x450798b8 // smmla z24.s, z5.b, z7.b\n" - ".inst 0x4506982c // smmla z12.s, z1.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n" - ".inst 0x45069874 // smmla z20.s, z3.b, z6.b\n" - ".inst 0x450698bc // smmla z28.s, z5.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n" - ".inst 0x45079829 // smmla z9.s, z1.b, z7.b\n" - ".inst 0x45079871 // smmla z17.s, z3.b, z7.b\n" - ".inst 0x450798b9 // smmla z25.s, z5.b, z7.b\n" - ".inst 0x4506982d // smmla z13.s, z1.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n" - ".inst 0x45069875 // smmla z21.s, z3.b, z6.b\n" - ".inst 0x450698bd // smmla z29.s, z5.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n" - ".inst 0x4507982a // smmla z10.s, z1.b, z7.b\n" - ".inst 0x45079872 // smmla z18.s, z3.b, z7.b\n" - ".inst 0x450798ba // smmla z26.s, z5.b, z7.b\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n" - ".inst 0x45069876 // smmla z22.s, z3.b, z6.b\n" - ".inst 0x450698be // smmla z30.s, z5.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n" + "ld1b { z2.b }, p2/Z, [x9]\n" + "ld1b { z0.b }, p2/Z, [x9, #1, MUL VL]\n" + ".inst 0x45029828 // smmla z8.s, z1.b, z2.b\n" + ".inst 0x45029870 // smmla z16.s, z3.b, z2.b\n" + ".inst 0x450298b8 // smmla z24.s, z5.b, z2.b\n" + ".inst 0x4500982c // smmla z12.s, z1.b, z0.b\n" + "ld1b { z2.b }, p2/Z, [x9, #2, MUL VL]\n" + ".inst 0x45009874 // smmla z20.s, z3.b, z0.b\n" + ".inst 0x450098bc // smmla z28.s, z5.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #3, MUL VL]\n" + ".inst 0x45029829 // smmla z9.s, z1.b, z2.b\n" + ".inst 0x45029871 // smmla z17.s, z3.b, z2.b\n" + ".inst 0x450298b9 // smmla z25.s, z5.b, z2.b\n" + ".inst 0x4500982d // smmla z13.s, z1.b, z0.b\n" + "ld1b { z2.b }, p2/Z, [x9, #4, MUL VL]\n" + ".inst 0x45009875 // smmla z21.s, z3.b, z0.b\n" + ".inst 0x450098bd // smmla z29.s, z5.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #5, MUL VL]\n" + ".inst 0x4502982a // smmla z10.s, z1.b, z2.b\n" + ".inst 0x45029872 // smmla z18.s, z3.b, z2.b\n" + ".inst 0x450298ba // smmla z26.s, z5.b, z2.b\n" + ".inst 0x4500982e // smmla z14.s, z1.b, z0.b\n" + "ld1b { z2.b }, p2/Z, [x9, #6, MUL VL]\n" + ".inst 0x45009876 // smmla z22.s, z3.b, z0.b\n" + ".inst 0x450098be // smmla z30.s, z5.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #7, MUL VL]\n" "addvl x9, x9, #8\n" - ".inst 0x4507982b // smmla z11.s, z1.b, z7.b\n" - ".inst 0x45079873 // smmla z19.s, z3.b, z7.b\n" - ".inst 0x450798bb // smmla z27.s, z5.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" - ".inst 0x45069877 // smmla z23.s, z3.b, z6.b\n" - ".inst 0x450698bf // smmla z31.s, z5.b, z6.b\n" + ".inst 0x4502982b // smmla z11.s, z1.b, z2.b\n" + ".inst 0x45029873 // smmla z19.s, z3.b, z2.b\n" + ".inst 0x450298bb // smmla z27.s, z5.b, z2.b\n" + ".inst 0x4500982f // smmla z15.s, z1.b, z0.b\n" + ".inst 0x45009877 // smmla z23.s, z3.b, z0.b\n" + ".inst 0x450098bf // smmla z31.s, z5.b, z0.b\n" "61:" // Height 5: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 56b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "uzp1 z7.d, z8.d, z12.d\n" + "uzp1 z4.d, z8.d, z12.d\n" "uzp2 z8.d, z8.d, z12.d\n" - "add x24, x11, x20\n" + "add x26, x11, x20\n" "uzp1 z12.d, z9.d, z13.d\n" "uzp2 z9.d, z9.d, z13.d\n" - "ld1w { z0.s }, p2/Z, [x14]\n" - "ld1w { z1.s }, p2/Z, [x14, #1, MUL VL]\n" + "ld1w { z3.s }, p2/Z, [x14]\n" + "ld1w { z2.s }, p2/Z, [x14, #1, MUL VL]\n" "uzp1 z13.d, z10.d, z14.d\n" "uzp2 z10.d, z10.d, z14.d\n" - "ld1w { z2.s }, p2/Z, [x14, #2, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x14, #3, MUL VL]\n" + "ld1w { z1.s }, p2/Z, [x14, #2, MUL VL]\n" + "ld1w { z0.s }, p2/Z, [x14, #3, MUL VL]\n" "uzp1 z14.d, z11.d, z15.d\n" "uzp2 z11.d, z11.d, z15.d\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" + "add x25, x26, x20\n" + "add x24, x25, x20\n" "uzp1 z15.d, z16.d, z20.d\n" "uzp2 z16.d, z16.d, z20.d\n" - "add x21, x22, x20\n" + "add x23, x24, x20\n" "addvl x14, x14, #4\n" "uzp1 z20.d, z17.d, z21.d\n" "uzp2 z17.d, z17.d, z21.d\n" @@ -1610,27 +1610,27 @@ void sve_hybrid_s8qs_mmla_6x4VL ( "uzp1 z25.d, z25.d, z29.d\n" "uzp1 z26.d, z26.d, z30.d\n" "uzp1 z27.d, z27.d, z31.d\n" - "mov z31.d, z7.d\n" - "add z31.s, z31.s, z0.s\n" - "add z12.s, z12.s, z1.s\n" - "add z13.s, z13.s, z2.s\n" - "add z14.s, z14.s, z3.s\n" - "add z8.s, z8.s, z0.s\n" - "add z9.s, z9.s, z1.s\n" - "add z10.s, z10.s, z2.s\n" - "add z11.s, z11.s, z3.s\n" - "add z15.s, z15.s, z0.s\n" - "add z20.s, z20.s, z1.s\n" - "add z21.s, z21.s, z2.s\n" - "add z22.s, z22.s, z3.s\n" - "add z16.s, z16.s, z0.s\n" - "add z17.s, z17.s, z1.s\n" - "add z18.s, z18.s, z2.s\n" - "add z19.s, z19.s, z3.s\n" - "add z24.s, z24.s, z0.s\n" - "add z25.s, z25.s, z1.s\n" - "add z26.s, z26.s, z2.s\n" - "add z27.s, z27.s, z3.s\n" + "mov z31.d, z4.d\n" + "add z31.s, z31.s, z3.s\n" + "add z12.s, z12.s, z2.s\n" + "add z13.s, z13.s, z1.s\n" + "add z14.s, z14.s, z0.s\n" + "add z8.s, z8.s, z3.s\n" + "add z9.s, z9.s, z2.s\n" + "add z10.s, z10.s, z1.s\n" + "add z11.s, z11.s, z0.s\n" + "add z15.s, z15.s, z3.s\n" + "add z20.s, z20.s, z2.s\n" + "add z21.s, z21.s, z1.s\n" + "add z22.s, z22.s, z0.s\n" + "add z16.s, z16.s, z3.s\n" + "add z17.s, z17.s, z2.s\n" + "add z18.s, z18.s, z1.s\n" + "add z19.s, z19.s, z0.s\n" + "add z24.s, z24.s, z3.s\n" + "add z25.s, z25.s, z2.s\n" + "add z26.s, z26.s, z1.s\n" + "add z27.s, z27.s, z0.s\n" "tbz %x[flags], #4, 62f\n" "ld1w { z0.s }, p2/Z, [x12]\n" "ld1w { z4.s }, p2/Z, [x13]\n" @@ -1644,10 +1644,10 @@ void sve_hybrid_s8qs_mmla_6x4VL ( "addvl x13, x13, #4\n" "b 63f\n" "62:" // Height 5: per layer parameters - "add x26, %x[qp], %[per_layer_right_shift]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1rw { z0.s }, p2/Z, [x26]\n" - "ld1rw { z4.s }, p2/Z, [x25]\n" + "add x21, %x[qp], %[per_layer_right_shift]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1rw { z0.s }, p2/Z, [x21]\n" + "ld1rw { z4.s }, p2/Z, [x20]\n" "mov z1.d, z0.d\n" "mov z5.d, z4.d\n" "mov z2.d, z0.d\n" @@ -1676,173 +1676,173 @@ void sve_hybrid_s8qs_mmla_6x4VL ( ".inst 0x04a6775a // sqrdmulh z26.s, z26.s, z6.s\n" ".inst 0x04a7777b // sqrdmulh z27.s, z27.s, z7.s\n" "tbz %x[flags], #5, 64f\n" - "and z4.d, z31.d, z0.d\n" - "and z5.d, z12.d, z1.d\n" - "and z6.d, z13.d, z2.d\n" - "and z7.d, z14.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z31.s, z31.s, z4.s\n" - "sqadd z12.s, z12.s, z5.s\n" - "sqadd z13.s, z13.s, z6.s\n" - "sqadd z14.s, z14.s, z7.s\n" - "and z4.d, z8.d, z0.d\n" - "and z5.d, z9.d, z1.d\n" - "and z6.d, z10.d, z2.d\n" - "and z7.d, z11.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z8.s, z8.s, z4.s\n" - "sqadd z9.s, z9.s, z5.s\n" - "sqadd z10.s, z10.s, z6.s\n" - "sqadd z11.s, z11.s, z7.s\n" - "and z4.d, z15.d, z0.d\n" - "and z5.d, z20.d, z1.d\n" - "and z6.d, z21.d, z2.d\n" - "and z7.d, z22.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z15.s, z15.s, z4.s\n" - "sqadd z20.s, z20.s, z5.s\n" - "sqadd z21.s, z21.s, z6.s\n" - "sqadd z22.s, z22.s, z7.s\n" - "and z4.d, z16.d, z0.d\n" - "and z5.d, z17.d, z1.d\n" - "and z6.d, z18.d, z2.d\n" - "and z7.d, z19.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z16.s, z16.s, z4.s\n" - "sqadd z17.s, z17.s, z5.s\n" - "sqadd z18.s, z18.s, z6.s\n" - "sqadd z19.s, z19.s, z7.s\n" - "and z4.d, z24.d, z0.d\n" - "and z5.d, z25.d, z1.d\n" - "and z6.d, z26.d, z2.d\n" - "and z7.d, z27.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z24.s, z24.s, z4.s\n" - "sqadd z25.s, z25.s, z5.s\n" - "sqadd z26.s, z26.s, z6.s\n" - "sqadd z27.s, z27.s, z7.s\n" + "and z30.d, z31.d, z0.d\n" + "and z29.d, z12.d, z1.d\n" + "and z28.d, z13.d, z2.d\n" + "and z23.d, z14.d, z3.d\n" + "asr z30.s, z30.s, #0x1f\n" + "asr z29.s, z29.s, #0x1f\n" + "asr z28.s, z28.s, #0x1f\n" + "asr z23.s, z23.s, #0x1f\n" + "sqadd z31.s, z31.s, z30.s\n" + "sqadd z12.s, z12.s, z29.s\n" + "sqadd z13.s, z13.s, z28.s\n" + "sqadd z14.s, z14.s, z23.s\n" + "and z30.d, z8.d, z0.d\n" + "and z29.d, z9.d, z1.d\n" + "and z28.d, z10.d, z2.d\n" + "and z23.d, z11.d, z3.d\n" + "asr z30.s, z30.s, #0x1f\n" + "asr z29.s, z29.s, #0x1f\n" + "asr z28.s, z28.s, #0x1f\n" + "asr z23.s, z23.s, #0x1f\n" + "sqadd z8.s, z8.s, z30.s\n" + "sqadd z9.s, z9.s, z29.s\n" + "sqadd z10.s, z10.s, z28.s\n" + "sqadd z11.s, z11.s, z23.s\n" + "and z30.d, z15.d, z0.d\n" + "and z29.d, z20.d, z1.d\n" + "and z28.d, z21.d, z2.d\n" + "and z23.d, z22.d, z3.d\n" + "asr z30.s, z30.s, #0x1f\n" + "asr z29.s, z29.s, #0x1f\n" + "asr z28.s, z28.s, #0x1f\n" + "asr z23.s, z23.s, #0x1f\n" + "sqadd z15.s, z15.s, z30.s\n" + "sqadd z20.s, z20.s, z29.s\n" + "sqadd z21.s, z21.s, z28.s\n" + "sqadd z22.s, z22.s, z23.s\n" + "and z30.d, z16.d, z0.d\n" + "and z29.d, z17.d, z1.d\n" + "and z28.d, z18.d, z2.d\n" + "and z23.d, z19.d, z3.d\n" + "asr z30.s, z30.s, #0x1f\n" + "asr z29.s, z29.s, #0x1f\n" + "asr z28.s, z28.s, #0x1f\n" + "asr z23.s, z23.s, #0x1f\n" + "sqadd z16.s, z16.s, z30.s\n" + "sqadd z17.s, z17.s, z29.s\n" + "sqadd z18.s, z18.s, z28.s\n" + "sqadd z19.s, z19.s, z23.s\n" + "and z30.d, z24.d, z0.d\n" + "and z29.d, z25.d, z1.d\n" + "and z28.d, z26.d, z2.d\n" + "and z23.d, z27.d, z3.d\n" + "asr z30.s, z30.s, #0x1f\n" + "asr z29.s, z29.s, #0x1f\n" + "asr z28.s, z28.s, #0x1f\n" + "asr z23.s, z23.s, #0x1f\n" + "sqadd z24.s, z24.s, z30.s\n" + "sqadd z25.s, z25.s, z29.s\n" + "sqadd z26.s, z26.s, z28.s\n" + "sqadd z27.s, z27.s, z23.s\n" "64:" // Height 5: no shift correction - "add x25, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x25]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1rw { z28.s }, p2/Z, [x20]\n" ".inst 0x4482881f // srshl z31.s, p2/M, z31.s, z0.s\n" - "add z31.s, z31.s, z4.s\n" + "add z31.s, z31.s, z28.s\n" ".inst 0x4482882c // srshl z12.s, p2/M, z12.s, z1.s\n" ".inst 0x4482884d // srshl z13.s, p2/M, z13.s, z2.s\n" - "add z12.s, z12.s, z4.s\n" - "add z13.s, z13.s, z4.s\n" + "add z12.s, z12.s, z28.s\n" + "add z13.s, z13.s, z28.s\n" ".inst 0x4482886e // srshl z14.s, p2/M, z14.s, z3.s\n" ".inst 0x44828808 // srshl z8.s, p2/M, z8.s, z0.s\n" - "add z14.s, z14.s, z4.s\n" - "add z8.s, z8.s, z4.s\n" + "add z14.s, z14.s, z28.s\n" + "add z8.s, z8.s, z28.s\n" ".inst 0x44828829 // srshl z9.s, p2/M, z9.s, z1.s\n" ".inst 0x4482884a // srshl z10.s, p2/M, z10.s, z2.s\n" - "add z9.s, z9.s, z4.s\n" - "add z10.s, z10.s, z4.s\n" + "add z9.s, z9.s, z28.s\n" + "add z10.s, z10.s, z28.s\n" ".inst 0x4482886b // srshl z11.s, p2/M, z11.s, z3.s\n" ".inst 0x4482880f // srshl z15.s, p2/M, z15.s, z0.s\n" - "add z11.s, z11.s, z4.s\n" - "add z15.s, z15.s, z4.s\n" + "add z11.s, z11.s, z28.s\n" + "add z15.s, z15.s, z28.s\n" ".inst 0x44828834 // srshl z20.s, p2/M, z20.s, z1.s\n" ".inst 0x44828855 // srshl z21.s, p2/M, z21.s, z2.s\n" - "add z20.s, z20.s, z4.s\n" - "add z21.s, z21.s, z4.s\n" + "add z20.s, z20.s, z28.s\n" + "add z21.s, z21.s, z28.s\n" ".inst 0x44828876 // srshl z22.s, p2/M, z22.s, z3.s\n" ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n" - "add z22.s, z22.s, z4.s\n" - "add z16.s, z16.s, z4.s\n" + "add z22.s, z22.s, z28.s\n" + "add z16.s, z16.s, z28.s\n" ".inst 0x44828831 // srshl z17.s, p2/M, z17.s, z1.s\n" ".inst 0x44828852 // srshl z18.s, p2/M, z18.s, z2.s\n" - "add z17.s, z17.s, z4.s\n" - "add z18.s, z18.s, z4.s\n" + "add z17.s, z17.s, z28.s\n" + "add z18.s, z18.s, z28.s\n" ".inst 0x44828873 // srshl z19.s, p2/M, z19.s, z3.s\n" ".inst 0x44828818 // srshl z24.s, p2/M, z24.s, z0.s\n" - "add z19.s, z19.s, z4.s\n" - "add z24.s, z24.s, z4.s\n" + "add z19.s, z19.s, z28.s\n" + "add z24.s, z24.s, z28.s\n" ".inst 0x44828839 // srshl z25.s, p2/M, z25.s, z1.s\n" ".inst 0x4482885a // srshl z26.s, p2/M, z26.s, z2.s\n" - "add z25.s, z25.s, z4.s\n" - "add z26.s, z26.s, z4.s\n" + "add z25.s, z25.s, z28.s\n" + "add z26.s, z26.s, z28.s\n" ".inst 0x4482887b // srshl z27.s, p2/M, z27.s, z3.s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x25]\n" - "add z27.s, z27.s, z4.s\n" - "add x25, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x25]\n" - "smin z31.s, p2/M, z31.s, z6.s\n" - "smin z12.s, p2/M, z12.s, z6.s\n" - "smin z13.s, p2/M, z13.s, z6.s\n" - "smin z14.s, p2/M, z14.s, z6.s\n" - "smin z8.s, p2/M, z8.s, z6.s\n" - "smin z9.s, p2/M, z9.s, z6.s\n" - "smin z10.s, p2/M, z10.s, z6.s\n" - "smin z11.s, p2/M, z11.s, z6.s\n" - "smin z15.s, p2/M, z15.s, z6.s\n" - "smin z20.s, p2/M, z20.s, z6.s\n" - "smin z21.s, p2/M, z21.s, z6.s\n" - "smin z22.s, p2/M, z22.s, z6.s\n" - "smin z16.s, p2/M, z16.s, z6.s\n" - "smin z17.s, p2/M, z17.s, z6.s\n" - "smin z18.s, p2/M, z18.s, z6.s\n" - "smin z19.s, p2/M, z19.s, z6.s\n" - "smin z24.s, p2/M, z24.s, z6.s\n" - "smin z25.s, p2/M, z25.s, z6.s\n" - "smin z26.s, p2/M, z26.s, z6.s\n" - "smin z27.s, p2/M, z27.s, z6.s\n" - "smax z31.s, p2/M, z31.s, z5.s\n" - "smax z12.s, p2/M, z12.s, z5.s\n" - "smax z13.s, p2/M, z13.s, z5.s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1rw { z23.s }, p2/Z, [x20]\n" + "add z27.s, z27.s, z28.s\n" + "add x20, %x[qp], %[minval]\n" + "ld1rw { z28.s }, p2/Z, [x20]\n" + "smin z31.s, p2/M, z31.s, z23.s\n" + "smin z12.s, p2/M, z12.s, z23.s\n" + "smin z13.s, p2/M, z13.s, z23.s\n" + "smin z14.s, p2/M, z14.s, z23.s\n" + "smin z8.s, p2/M, z8.s, z23.s\n" + "smin z9.s, p2/M, z9.s, z23.s\n" + "smin z10.s, p2/M, z10.s, z23.s\n" + "smin z11.s, p2/M, z11.s, z23.s\n" + "smin z15.s, p2/M, z15.s, z23.s\n" + "smin z20.s, p2/M, z20.s, z23.s\n" + "smin z21.s, p2/M, z21.s, z23.s\n" + "smin z22.s, p2/M, z22.s, z23.s\n" + "smin z16.s, p2/M, z16.s, z23.s\n" + "smin z17.s, p2/M, z17.s, z23.s\n" + "smin z18.s, p2/M, z18.s, z23.s\n" + "smin z19.s, p2/M, z19.s, z23.s\n" + "smin z24.s, p2/M, z24.s, z23.s\n" + "smin z25.s, p2/M, z25.s, z23.s\n" + "smin z26.s, p2/M, z26.s, z23.s\n" + "smin z27.s, p2/M, z27.s, z23.s\n" + "smax z31.s, p2/M, z31.s, z28.s\n" + "smax z12.s, p2/M, z12.s, z28.s\n" + "smax z13.s, p2/M, z13.s, z28.s\n" "uzp1 z31.h, z31.h, z12.h\n" - "smax z14.s, p2/M, z14.s, z5.s\n" - "smax z8.s, p2/M, z8.s, z5.s\n" - "uzp1 z12.h, z13.h, z14.h\n" - "uzp1 z31.b, z31.b, z12.b\n" - "smax z9.s, p2/M, z9.s, z5.s\n" - "smax z10.s, p2/M, z10.s, z5.s\n" + "smax z14.s, p2/M, z14.s, z28.s\n" + "smax z8.s, p2/M, z8.s, z28.s\n" + "uzp1 z23.h, z13.h, z14.h\n" + "uzp1 z31.b, z31.b, z23.b\n" + "smax z9.s, p2/M, z9.s, z28.s\n" + "smax z10.s, p2/M, z10.s, z28.s\n" "uzp1 z8.h, z8.h, z9.h\n" "st1b { z31.b }, p1, [x11]\n" - "smax z11.s, p2/M, z11.s, z5.s\n" - "smax z15.s, p2/M, z15.s, z5.s\n" - "uzp1 z9.h, z10.h, z11.h\n" - "uzp1 z8.b, z8.b, z9.b\n" - "smax z20.s, p2/M, z20.s, z5.s\n" - "smax z21.s, p2/M, z21.s, z5.s\n" + "smax z11.s, p2/M, z11.s, z28.s\n" + "smax z15.s, p2/M, z15.s, z28.s\n" + "uzp1 z23.h, z10.h, z11.h\n" + "uzp1 z8.b, z8.b, z23.b\n" + "smax z20.s, p2/M, z20.s, z28.s\n" + "smax z21.s, p2/M, z21.s, z28.s\n" "uzp1 z15.h, z15.h, z20.h\n" - "st1b { z8.b }, p1, [x24]\n" - "smax z22.s, p2/M, z22.s, z5.s\n" - "smax z16.s, p2/M, z16.s, z5.s\n" + "st1b { z8.b }, p1, [x26]\n" + "smax z22.s, p2/M, z22.s, z28.s\n" + "smax z16.s, p2/M, z16.s, z28.s\n" "uzp1 z20.h, z21.h, z22.h\n" "uzp1 z15.b, z15.b, z20.b\n" - "smax z17.s, p2/M, z17.s, z5.s\n" - "smax z18.s, p2/M, z18.s, z5.s\n" + "smax z17.s, p2/M, z17.s, z28.s\n" + "smax z18.s, p2/M, z18.s, z28.s\n" "uzp1 z16.h, z16.h, z17.h\n" - "st1b { z15.b }, p1, [x23]\n" - "smax z19.s, p2/M, z19.s, z5.s\n" - "smax z24.s, p2/M, z24.s, z5.s\n" + "st1b { z15.b }, p1, [x25]\n" + "smax z19.s, p2/M, z19.s, z28.s\n" + "smax z24.s, p2/M, z24.s, z28.s\n" "uzp1 z17.h, z18.h, z19.h\n" "uzp1 z16.b, z16.b, z17.b\n" - "smax z25.s, p2/M, z25.s, z5.s\n" - "smax z26.s, p2/M, z26.s, z5.s\n" + "smax z25.s, p2/M, z25.s, z28.s\n" + "smax z26.s, p2/M, z26.s, z28.s\n" "uzp1 z24.h, z24.h, z25.h\n" - "st1b { z16.b }, p1, [x22]\n" - "smax z27.s, p2/M, z27.s, z5.s\n" - "uzp1 z25.h, z26.h, z27.h\n" - "uzp1 z24.b, z24.b, z25.b\n" - "st1b { z24.b }, p1, [x21]\n" + "st1b { z16.b }, p1, [x24]\n" + "smax z27.s, p2/M, z27.s, z28.s\n" + "uzp1 z16.h, z26.h, z27.h\n" + "uzp1 z24.b, z24.b, z16.b\n" + "st1b { z24.b }, p1, [x23]\n" "addvl x11, x11, #1\n" "65:" // Height 5: Writeback done "decw x10, ALL, MUL #4\n" @@ -1891,16 +1891,16 @@ void sve_hybrid_s8qs_mmla_6x4VL ( "69:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 70f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 71f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -1912,209 +1912,209 @@ void sve_hybrid_s8qs_mmla_6x4VL ( "b 71f\n" "70:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" + "add x21, x22, x21\n" "71:" // Height 6: input setup done "cmp x27, #0x10\n" "ble 73f\n" "72:" // Height 6: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1rqb { z3.b }, p0/Z, [x24]\n" - "ld1rqb { z4.b }, p0/Z, [x23]\n" - "trn2 z1.d, z1.d, z2.d\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1rqb { z5.b }, p0/Z, [x22]\n" - "ld1rqb { z6.b }, p0/Z, [x21]\n" - "trn2 z3.d, z3.d, z4.d\n" - "trn1 z4.d, z5.d, z6.d\n" - "trn2 z5.d, z5.d, z6.d\n" - "ld1b { z7.b }, p2/Z, [x9]\n" - "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n" - ".inst 0x45079808 // smmla z8.s, z0.b, z7.b\n" - ".inst 0x45079850 // smmla z16.s, z2.b, z7.b\n" - ".inst 0x45079898 // smmla z24.s, z4.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1rqb { z7.b }, p0/Z, [x26]\n" + "ld1rqb { z0.b }, p0/Z, [x25]\n" + "trn1 z6.d, z7.d, z0.d\n" + "ld1rqb { z5.b }, p0/Z, [x24]\n" + "ld1rqb { z1.b }, p0/Z, [x23]\n" + "trn2 z7.d, z7.d, z0.d\n" + "trn1 z4.d, z5.d, z1.d\n" + "ld1rqb { z3.b }, p0/Z, [x22]\n" + "ld1rqb { z0.b }, p0/Z, [x21]\n" + "trn2 z5.d, z5.d, z1.d\n" + "trn1 z2.d, z3.d, z0.d\n" + "trn2 z3.d, z3.d, z0.d\n" + "ld1b { z1.b }, p2/Z, [x9]\n" + "ld1b { z0.b }, p2/Z, [x9, #1, MUL VL]\n" + ".inst 0x450198c8 // smmla z8.s, z6.b, z1.b\n" + ".inst 0x45019890 // smmla z16.s, z4.b, z1.b\n" + ".inst 0x45019858 // smmla z24.s, z2.b, z1.b\n" + "ld1b { z1.b }, p2/Z, [x9, #2, MUL VL]\n" "sub x27, x27, #0x10\n" - ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n" - ".inst 0x45069854 // smmla z20.s, z2.b, z6.b\n" + ".inst 0x450098cc // smmla z12.s, z6.b, z0.b\n" + ".inst 0x45009894 // smmla z20.s, z4.b, z0.b\n" "cmp x27, #0x10\n" "add x26, x26, #0x10\n" - ".inst 0x4506989c // smmla z28.s, z4.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n" - ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" + ".inst 0x4500985c // smmla z28.s, z2.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #3, MUL VL]\n" + ".inst 0x450198c9 // smmla z9.s, z6.b, z1.b\n" "add x25, x25, #0x10\n" - ".inst 0x45079851 // smmla z17.s, z2.b, z7.b\n" - ".inst 0x45079899 // smmla z25.s, z4.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n" + ".inst 0x45019891 // smmla z17.s, z4.b, z1.b\n" + ".inst 0x45019859 // smmla z25.s, z2.b, z1.b\n" + "ld1b { z1.b }, p2/Z, [x9, #4, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x4506980d // smmla z13.s, z0.b, z6.b\n" - ".inst 0x45069855 // smmla z21.s, z2.b, z6.b\n" + ".inst 0x450098cd // smmla z13.s, z6.b, z0.b\n" + ".inst 0x45009895 // smmla z21.s, z4.b, z0.b\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - ".inst 0x4506989d // smmla z29.s, z4.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n" - ".inst 0x4507980a // smmla z10.s, z0.b, z7.b\n" + ".inst 0x4500985d // smmla z29.s, z2.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #5, MUL VL]\n" + ".inst 0x450198ca // smmla z10.s, z6.b, z1.b\n" "add x21, x21, #0x10\n" - ".inst 0x45079852 // smmla z18.s, z2.b, z7.b\n" - ".inst 0x4507989a // smmla z26.s, z4.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n" - ".inst 0x4506980e // smmla z14.s, z0.b, z6.b\n" - ".inst 0x45069856 // smmla z22.s, z2.b, z6.b\n" - ".inst 0x4506989e // smmla z30.s, z4.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n" + ".inst 0x45019892 // smmla z18.s, z4.b, z1.b\n" + ".inst 0x4501985a // smmla z26.s, z2.b, z1.b\n" + "ld1b { z1.b }, p2/Z, [x9, #6, MUL VL]\n" + ".inst 0x450098ce // smmla z14.s, z6.b, z0.b\n" + ".inst 0x45009896 // smmla z22.s, z4.b, z0.b\n" + ".inst 0x4500985e // smmla z30.s, z2.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #7, MUL VL]\n" "addvl x9, x9, #16\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" - ".inst 0x45079853 // smmla z19.s, z2.b, z7.b\n" - ".inst 0x4507989b // smmla z27.s, z4.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #-8, MUL VL]\n" - ".inst 0x4506980f // smmla z15.s, z0.b, z6.b\n" - ".inst 0x45069857 // smmla z23.s, z2.b, z6.b\n" - ".inst 0x4506989f // smmla z31.s, z4.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #-7, MUL VL]\n" - ".inst 0x45079828 // smmla z8.s, z1.b, z7.b\n" - ".inst 0x45079870 // smmla z16.s, z3.b, z7.b\n" - ".inst 0x450798b8 // smmla z24.s, z5.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #-6, MUL VL]\n" - ".inst 0x4506982c // smmla z12.s, z1.b, z6.b\n" - ".inst 0x45069874 // smmla z20.s, z3.b, z6.b\n" - ".inst 0x450698bc // smmla z28.s, z5.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #-5, MUL VL]\n" - ".inst 0x45079829 // smmla z9.s, z1.b, z7.b\n" - ".inst 0x45079871 // smmla z17.s, z3.b, z7.b\n" - ".inst 0x450798b9 // smmla z25.s, z5.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #-4, MUL VL]\n" - ".inst 0x4506982d // smmla z13.s, z1.b, z6.b\n" - ".inst 0x45069875 // smmla z21.s, z3.b, z6.b\n" - ".inst 0x450698bd // smmla z29.s, z5.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #-3, MUL VL]\n" - ".inst 0x4507982a // smmla z10.s, z1.b, z7.b\n" - ".inst 0x45079872 // smmla z18.s, z3.b, z7.b\n" - ".inst 0x450798ba // smmla z26.s, z5.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #-2, MUL VL]\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - ".inst 0x45069876 // smmla z22.s, z3.b, z6.b\n" - ".inst 0x450698be // smmla z30.s, z5.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #-1, MUL VL]\n" - ".inst 0x4507982b // smmla z11.s, z1.b, z7.b\n" - ".inst 0x45079873 // smmla z19.s, z3.b, z7.b\n" - ".inst 0x450798bb // smmla z27.s, z5.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" - ".inst 0x45069877 // smmla z23.s, z3.b, z6.b\n" - ".inst 0x450698bf // smmla z31.s, z5.b, z6.b\n" + ".inst 0x450198cb // smmla z11.s, z6.b, z1.b\n" + ".inst 0x45019893 // smmla z19.s, z4.b, z1.b\n" + ".inst 0x4501985b // smmla z27.s, z2.b, z1.b\n" + "ld1b { z1.b }, p2/Z, [x9, #-8, MUL VL]\n" + ".inst 0x450098cf // smmla z15.s, z6.b, z0.b\n" + ".inst 0x45009897 // smmla z23.s, z4.b, z0.b\n" + ".inst 0x4500985f // smmla z31.s, z2.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #-7, MUL VL]\n" + ".inst 0x450198e8 // smmla z8.s, z7.b, z1.b\n" + ".inst 0x450198b0 // smmla z16.s, z5.b, z1.b\n" + ".inst 0x45019878 // smmla z24.s, z3.b, z1.b\n" + "ld1b { z1.b }, p2/Z, [x9, #-6, MUL VL]\n" + ".inst 0x450098ec // smmla z12.s, z7.b, z0.b\n" + ".inst 0x450098b4 // smmla z20.s, z5.b, z0.b\n" + ".inst 0x4500987c // smmla z28.s, z3.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #-5, MUL VL]\n" + ".inst 0x450198e9 // smmla z9.s, z7.b, z1.b\n" + ".inst 0x450198b1 // smmla z17.s, z5.b, z1.b\n" + ".inst 0x45019879 // smmla z25.s, z3.b, z1.b\n" + "ld1b { z1.b }, p2/Z, [x9, #-4, MUL VL]\n" + ".inst 0x450098ed // smmla z13.s, z7.b, z0.b\n" + ".inst 0x450098b5 // smmla z21.s, z5.b, z0.b\n" + ".inst 0x4500987d // smmla z29.s, z3.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #-3, MUL VL]\n" + ".inst 0x450198ea // smmla z10.s, z7.b, z1.b\n" + ".inst 0x450198b2 // smmla z18.s, z5.b, z1.b\n" + ".inst 0x4501987a // smmla z26.s, z3.b, z1.b\n" + "ld1b { z1.b }, p2/Z, [x9, #-2, MUL VL]\n" + ".inst 0x450098ee // smmla z14.s, z7.b, z0.b\n" + ".inst 0x450098b6 // smmla z22.s, z5.b, z0.b\n" + ".inst 0x4500987e // smmla z30.s, z3.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #-1, MUL VL]\n" + ".inst 0x450198eb // smmla z11.s, z7.b, z1.b\n" + ".inst 0x450198b3 // smmla z19.s, z5.b, z1.b\n" + ".inst 0x4501987b // smmla z27.s, z3.b, z1.b\n" + ".inst 0x450098ef // smmla z15.s, z7.b, z0.b\n" + ".inst 0x450098b7 // smmla z23.s, z5.b, z0.b\n" + ".inst 0x4500987f // smmla z31.s, z3.b, z0.b\n" "bgt 72b\n" "73:" // Height 6: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" + "ld1rqb { z0.b }, p0/Z, [x25]\n" + "trn1 z7.d, z1.d, z0.d\n" "ld1rqb { z3.b }, p0/Z, [x24]\n" - "ld1rqb { z4.b }, p0/Z, [x23]\n" - "trn2 z1.d, z1.d, z2.d\n" - "trn1 z2.d, z3.d, z4.d\n" + "ld1rqb { z2.b }, p0/Z, [x23]\n" + "trn2 z1.d, z1.d, z0.d\n" + "trn1 z6.d, z3.d, z2.d\n" "ld1rqb { z5.b }, p0/Z, [x22]\n" - "ld1rqb { z6.b }, p0/Z, [x21]\n" - "trn2 z3.d, z3.d, z4.d\n" - "trn1 z4.d, z5.d, z6.d\n" - "trn2 z5.d, z5.d, z6.d\n" - "ld1b { z7.b }, p2/Z, [x9]\n" - "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n" - ".inst 0x45079808 // smmla z8.s, z0.b, z7.b\n" - ".inst 0x45079850 // smmla z16.s, z2.b, z7.b\n" - ".inst 0x45079898 // smmla z24.s, z4.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n" + "ld1rqb { z0.b }, p0/Z, [x21]\n" + "trn2 z3.d, z3.d, z2.d\n" + "trn1 z4.d, z5.d, z0.d\n" + "trn2 z5.d, z5.d, z0.d\n" + "ld1b { z2.b }, p2/Z, [x9]\n" + "ld1b { z0.b }, p2/Z, [x9, #1, MUL VL]\n" + ".inst 0x450298e8 // smmla z8.s, z7.b, z2.b\n" + ".inst 0x450298d0 // smmla z16.s, z6.b, z2.b\n" + ".inst 0x45029898 // smmla z24.s, z4.b, z2.b\n" + "ld1b { z2.b }, p2/Z, [x9, #2, MUL VL]\n" "subs x27, x27, #0x8\n" - ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n" - ".inst 0x45069854 // smmla z20.s, z2.b, z6.b\n" - ".inst 0x4506989c // smmla z28.s, z4.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n" - ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" - ".inst 0x45079851 // smmla z17.s, z2.b, z7.b\n" - ".inst 0x45079899 // smmla z25.s, z4.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n" - ".inst 0x4506980d // smmla z13.s, z0.b, z6.b\n" - ".inst 0x45069855 // smmla z21.s, z2.b, z6.b\n" - ".inst 0x4506989d // smmla z29.s, z4.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n" - ".inst 0x4507980a // smmla z10.s, z0.b, z7.b\n" - ".inst 0x45079852 // smmla z18.s, z2.b, z7.b\n" - ".inst 0x4507989a // smmla z26.s, z4.b, z7.b\n" - "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n" - ".inst 0x4506980e // smmla z14.s, z0.b, z6.b\n" - ".inst 0x45069856 // smmla z22.s, z2.b, z6.b\n" - ".inst 0x4506989e // smmla z30.s, z4.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" + ".inst 0x450098ec // smmla z12.s, z7.b, z0.b\n" + ".inst 0x450098d4 // smmla z20.s, z6.b, z0.b\n" + ".inst 0x4500989c // smmla z28.s, z4.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #3, MUL VL]\n" + ".inst 0x450298e9 // smmla z9.s, z7.b, z2.b\n" + ".inst 0x450298d1 // smmla z17.s, z6.b, z2.b\n" + ".inst 0x45029899 // smmla z25.s, z4.b, z2.b\n" + "ld1b { z2.b }, p2/Z, [x9, #4, MUL VL]\n" + ".inst 0x450098ed // smmla z13.s, z7.b, z0.b\n" + ".inst 0x450098d5 // smmla z21.s, z6.b, z0.b\n" + ".inst 0x4500989d // smmla z29.s, z4.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #5, MUL VL]\n" + ".inst 0x450298ea // smmla z10.s, z7.b, z2.b\n" + ".inst 0x450298d2 // smmla z18.s, z6.b, z2.b\n" + ".inst 0x4502989a // smmla z26.s, z4.b, z2.b\n" + "ld1b { z2.b }, p2/Z, [x9, #6, MUL VL]\n" + ".inst 0x450098ee // smmla z14.s, z7.b, z0.b\n" + ".inst 0x450098d6 // smmla z22.s, z6.b, z0.b\n" + ".inst 0x4500989e // smmla z30.s, z4.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #7, MUL VL]\n" + ".inst 0x450298eb // smmla z11.s, z7.b, z2.b\n" "addvl x9, x9, #8\n" - ".inst 0x45079853 // smmla z19.s, z2.b, z7.b\n" - ".inst 0x4507989b // smmla z27.s, z4.b, z7.b\n" - ".inst 0x4506980f // smmla z15.s, z0.b, z6.b\n" - ".inst 0x45069857 // smmla z23.s, z2.b, z6.b\n" - ".inst 0x4506989f // smmla z31.s, z4.b, z6.b\n" + ".inst 0x450298d3 // smmla z19.s, z6.b, z2.b\n" + ".inst 0x4502989b // smmla z27.s, z4.b, z2.b\n" + ".inst 0x450098ef // smmla z15.s, z7.b, z0.b\n" + ".inst 0x450098d7 // smmla z23.s, z6.b, z0.b\n" + ".inst 0x4500989f // smmla z31.s, z4.b, z0.b\n" "ble 74f\n" - "ld1b { z7.b }, p2/Z, [x9]\n" - "ld1b { z6.b }, p2/Z, [x9, #1, MUL VL]\n" - ".inst 0x45079828 // smmla z8.s, z1.b, z7.b\n" - ".inst 0x45079870 // smmla z16.s, z3.b, z7.b\n" - ".inst 0x450798b8 // smmla z24.s, z5.b, z7.b\n" - ".inst 0x4506982c // smmla z12.s, z1.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #2, MUL VL]\n" - ".inst 0x45069874 // smmla z20.s, z3.b, z6.b\n" - ".inst 0x450698bc // smmla z28.s, z5.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #3, MUL VL]\n" - ".inst 0x45079829 // smmla z9.s, z1.b, z7.b\n" - ".inst 0x45079871 // smmla z17.s, z3.b, z7.b\n" - ".inst 0x450798b9 // smmla z25.s, z5.b, z7.b\n" - ".inst 0x4506982d // smmla z13.s, z1.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #4, MUL VL]\n" - ".inst 0x45069875 // smmla z21.s, z3.b, z6.b\n" - ".inst 0x450698bd // smmla z29.s, z5.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #5, MUL VL]\n" - ".inst 0x4507982a // smmla z10.s, z1.b, z7.b\n" - ".inst 0x45079872 // smmla z18.s, z3.b, z7.b\n" - ".inst 0x450798ba // smmla z26.s, z5.b, z7.b\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x9, #6, MUL VL]\n" - ".inst 0x45069876 // smmla z22.s, z3.b, z6.b\n" - ".inst 0x450698be // smmla z30.s, z5.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x9, #7, MUL VL]\n" + "ld1b { z2.b }, p2/Z, [x9]\n" + "ld1b { z0.b }, p2/Z, [x9, #1, MUL VL]\n" + ".inst 0x45029828 // smmla z8.s, z1.b, z2.b\n" + ".inst 0x45029870 // smmla z16.s, z3.b, z2.b\n" + ".inst 0x450298b8 // smmla z24.s, z5.b, z2.b\n" + ".inst 0x4500982c // smmla z12.s, z1.b, z0.b\n" + "ld1b { z2.b }, p2/Z, [x9, #2, MUL VL]\n" + ".inst 0x45009874 // smmla z20.s, z3.b, z0.b\n" + ".inst 0x450098bc // smmla z28.s, z5.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #3, MUL VL]\n" + ".inst 0x45029829 // smmla z9.s, z1.b, z2.b\n" + ".inst 0x45029871 // smmla z17.s, z3.b, z2.b\n" + ".inst 0x450298b9 // smmla z25.s, z5.b, z2.b\n" + ".inst 0x4500982d // smmla z13.s, z1.b, z0.b\n" + "ld1b { z2.b }, p2/Z, [x9, #4, MUL VL]\n" + ".inst 0x45009875 // smmla z21.s, z3.b, z0.b\n" + ".inst 0x450098bd // smmla z29.s, z5.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #5, MUL VL]\n" + ".inst 0x4502982a // smmla z10.s, z1.b, z2.b\n" + ".inst 0x45029872 // smmla z18.s, z3.b, z2.b\n" + ".inst 0x450298ba // smmla z26.s, z5.b, z2.b\n" + ".inst 0x4500982e // smmla z14.s, z1.b, z0.b\n" + "ld1b { z2.b }, p2/Z, [x9, #6, MUL VL]\n" + ".inst 0x45009876 // smmla z22.s, z3.b, z0.b\n" + ".inst 0x450098be // smmla z30.s, z5.b, z0.b\n" + "ld1b { z0.b }, p2/Z, [x9, #7, MUL VL]\n" "addvl x9, x9, #8\n" - ".inst 0x4507982b // smmla z11.s, z1.b, z7.b\n" - ".inst 0x45079873 // smmla z19.s, z3.b, z7.b\n" - ".inst 0x450798bb // smmla z27.s, z5.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" - ".inst 0x45069877 // smmla z23.s, z3.b, z6.b\n" - ".inst 0x450698bf // smmla z31.s, z5.b, z6.b\n" + ".inst 0x4502982b // smmla z11.s, z1.b, z2.b\n" + ".inst 0x45029873 // smmla z19.s, z3.b, z2.b\n" + ".inst 0x450298bb // smmla z27.s, z5.b, z2.b\n" + ".inst 0x4500982f // smmla z15.s, z1.b, z0.b\n" + ".inst 0x45009877 // smmla z23.s, z3.b, z0.b\n" + ".inst 0x450098bf // smmla z31.s, z5.b, z0.b\n" "74:" // Height 6: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 69b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "uzp1 z7.d, z8.d, z12.d\n" - "add x24, x11, x20\n" + "uzp1 z4.d, z8.d, z12.d\n" + "add x26, x11, x20\n" "uzp2 z8.d, z8.d, z12.d\n" "uzp1 z12.d, z9.d, z13.d\n" "uzp2 z9.d, z9.d, z13.d\n" - "add x23, x24, x20\n" - "ld1w { z0.s }, p2/Z, [x14]\n" + "add x25, x26, x20\n" + "ld1w { z3.s }, p2/Z, [x14]\n" "uzp1 z13.d, z10.d, z14.d\n" "uzp2 z10.d, z10.d, z14.d\n" - "ld1w { z1.s }, p2/Z, [x14, #1, MUL VL]\n" - "ld1w { z2.s }, p2/Z, [x14, #2, MUL VL]\n" + "ld1w { z2.s }, p2/Z, [x14, #1, MUL VL]\n" + "ld1w { z1.s }, p2/Z, [x14, #2, MUL VL]\n" "uzp1 z14.d, z11.d, z15.d\n" "uzp2 z11.d, z11.d, z15.d\n" - "ld1w { z3.s }, p2/Z, [x14, #3, MUL VL]\n" - "add x22, x23, x20\n" + "ld1w { z0.s }, p2/Z, [x14, #3, MUL VL]\n" + "add x24, x25, x20\n" "uzp1 z15.d, z16.d, z20.d\n" "uzp2 z16.d, z16.d, z20.d\n" - "add x21, x22, x20\n" - "add x20, x21, x20\n" + "add x23, x24, x20\n" + "add x22, x23, x20\n" "uzp1 z20.d, z17.d, z21.d\n" "uzp2 z17.d, z17.d, z21.d\n" "addvl x14, x14, #4\n" @@ -2130,31 +2130,31 @@ void sve_hybrid_s8qs_mmla_6x4VL ( "uzp2 z26.d, z26.d, z30.d\n" "uzp1 z30.d, z27.d, z31.d\n" "uzp2 z27.d, z27.d, z31.d\n" - "mov z31.d, z7.d\n" - "add z31.s, z31.s, z0.s\n" - "add z12.s, z12.s, z1.s\n" - "add z13.s, z13.s, z2.s\n" - "add z14.s, z14.s, z3.s\n" - "add z8.s, z8.s, z0.s\n" - "add z9.s, z9.s, z1.s\n" - "add z10.s, z10.s, z2.s\n" - "add z11.s, z11.s, z3.s\n" - "add z15.s, z15.s, z0.s\n" - "add z20.s, z20.s, z1.s\n" - "add z21.s, z21.s, z2.s\n" - "add z22.s, z22.s, z3.s\n" - "add z16.s, z16.s, z0.s\n" - "add z17.s, z17.s, z1.s\n" - "add z18.s, z18.s, z2.s\n" - "add z19.s, z19.s, z3.s\n" - "add z23.s, z23.s, z0.s\n" - "add z28.s, z28.s, z1.s\n" - "add z29.s, z29.s, z2.s\n" - "add z30.s, z30.s, z3.s\n" - "add z24.s, z24.s, z0.s\n" - "add z25.s, z25.s, z1.s\n" - "add z26.s, z26.s, z2.s\n" - "add z27.s, z27.s, z3.s\n" + "mov z31.d, z4.d\n" + "add z31.s, z31.s, z3.s\n" + "add z12.s, z12.s, z2.s\n" + "add z13.s, z13.s, z1.s\n" + "add z14.s, z14.s, z0.s\n" + "add z8.s, z8.s, z3.s\n" + "add z9.s, z9.s, z2.s\n" + "add z10.s, z10.s, z1.s\n" + "add z11.s, z11.s, z0.s\n" + "add z15.s, z15.s, z3.s\n" + "add z20.s, z20.s, z2.s\n" + "add z21.s, z21.s, z1.s\n" + "add z22.s, z22.s, z0.s\n" + "add z16.s, z16.s, z3.s\n" + "add z17.s, z17.s, z2.s\n" + "add z18.s, z18.s, z1.s\n" + "add z19.s, z19.s, z0.s\n" + "add z23.s, z23.s, z3.s\n" + "add z28.s, z28.s, z2.s\n" + "add z29.s, z29.s, z1.s\n" + "add z30.s, z30.s, z0.s\n" + "add z24.s, z24.s, z3.s\n" + "add z25.s, z25.s, z2.s\n" + "add z26.s, z26.s, z1.s\n" + "add z27.s, z27.s, z0.s\n" "tbz %x[flags], #4, 75f\n" "ld1w { z0.s }, p2/Z, [x12]\n" "ld1w { z4.s }, p2/Z, [x13]\n" @@ -2168,10 +2168,10 @@ void sve_hybrid_s8qs_mmla_6x4VL ( "addvl x13, x13, #4\n" "b 76f\n" "75:" // Height 6: per layer parameters - "add x26, %x[qp], %[per_layer_right_shift]\n" - "add x25, %x[qp], %[per_layer_mul]\n" - "ld1rw { z0.s }, p2/Z, [x26]\n" - "ld1rw { z4.s }, p2/Z, [x25]\n" + "add x21, %x[qp], %[per_layer_right_shift]\n" + "add x20, %x[qp], %[per_layer_mul]\n" + "ld1rw { z0.s }, p2/Z, [x21]\n" + "ld1rw { z4.s }, p2/Z, [x20]\n" "mov z1.d, z0.d\n" "mov z5.d, z4.d\n" "mov z2.d, z0.d\n" @@ -2204,81 +2204,81 @@ void sve_hybrid_s8qs_mmla_6x4VL ( ".inst 0x04a6775a // sqrdmulh z26.s, z26.s, z6.s\n" ".inst 0x04a7777b // sqrdmulh z27.s, z27.s, z7.s\n" "tbz %x[flags], #5, 77f\n" - "and z4.d, z31.d, z0.d\n" - "and z5.d, z12.d, z1.d\n" - "and z6.d, z13.d, z2.d\n" - "and z7.d, z14.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" + "and z7.d, z31.d, z0.d\n" + "and z6.d, z12.d, z1.d\n" + "and z5.d, z13.d, z2.d\n" + "and z4.d, z14.d, z3.d\n" "asr z7.s, z7.s, #0x1f\n" - "sqadd z31.s, z31.s, z4.s\n" - "sqadd z12.s, z12.s, z5.s\n" - "sqadd z13.s, z13.s, z6.s\n" - "sqadd z14.s, z14.s, z7.s\n" - "and z4.d, z8.d, z0.d\n" - "and z5.d, z9.d, z1.d\n" - "and z6.d, z10.d, z2.d\n" - "and z7.d, z11.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z8.s, z8.s, z4.s\n" - "sqadd z9.s, z9.s, z5.s\n" - "sqadd z10.s, z10.s, z6.s\n" - "sqadd z11.s, z11.s, z7.s\n" - "and z4.d, z15.d, z0.d\n" - "and z5.d, z20.d, z1.d\n" - "and z6.d, z21.d, z2.d\n" - "and z7.d, z22.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z15.s, z15.s, z4.s\n" - "sqadd z20.s, z20.s, z5.s\n" - "sqadd z21.s, z21.s, z6.s\n" - "sqadd z22.s, z22.s, z7.s\n" - "and z4.d, z16.d, z0.d\n" - "and z5.d, z17.d, z1.d\n" - "and z6.d, z18.d, z2.d\n" - "and z7.d, z19.d, z3.d\n" "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" + "sqadd z31.s, z31.s, z7.s\n" + "sqadd z12.s, z12.s, z6.s\n" + "sqadd z13.s, z13.s, z5.s\n" + "sqadd z14.s, z14.s, z4.s\n" + "and z7.d, z8.d, z0.d\n" + "and z6.d, z9.d, z1.d\n" + "and z5.d, z10.d, z2.d\n" + "and z4.d, z11.d, z3.d\n" "asr z7.s, z7.s, #0x1f\n" - "sqadd z16.s, z16.s, z4.s\n" - "sqadd z17.s, z17.s, z5.s\n" - "sqadd z18.s, z18.s, z6.s\n" - "sqadd z19.s, z19.s, z7.s\n" - "and z4.d, z23.d, z0.d\n" - "and z5.d, z28.d, z1.d\n" - "and z6.d, z29.d, z2.d\n" - "and z7.d, z30.d, z3.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" "asr z6.s, z6.s, #0x1f\n" + "asr z5.s, z5.s, #0x1f\n" + "asr z4.s, z4.s, #0x1f\n" + "sqadd z8.s, z8.s, z7.s\n" + "sqadd z9.s, z9.s, z6.s\n" + "sqadd z10.s, z10.s, z5.s\n" + "sqadd z11.s, z11.s, z4.s\n" + "and z7.d, z15.d, z0.d\n" + "and z6.d, z20.d, z1.d\n" + "and z5.d, z21.d, z2.d\n" + "and z4.d, z22.d, z3.d\n" "asr z7.s, z7.s, #0x1f\n" - "sqadd z23.s, z23.s, z4.s\n" - "sqadd z28.s, z28.s, z5.s\n" - "sqadd z29.s, z29.s, z6.s\n" - "sqadd z30.s, z30.s, z7.s\n" - "and z4.d, z24.d, z0.d\n" - "and z5.d, z25.d, z1.d\n" - "and z6.d, z26.d, z2.d\n" - "and z7.d, z27.d, z3.d\n" + "asr z6.s, z6.s, #0x1f\n" + "asr z5.s, z5.s, #0x1f\n" "asr z4.s, z4.s, #0x1f\n" + "sqadd z15.s, z15.s, z7.s\n" + "sqadd z20.s, z20.s, z6.s\n" + "sqadd z21.s, z21.s, z5.s\n" + "sqadd z22.s, z22.s, z4.s\n" + "and z7.d, z16.d, z0.d\n" + "and z6.d, z17.d, z1.d\n" + "and z5.d, z18.d, z2.d\n" + "and z4.d, z19.d, z3.d\n" + "asr z7.s, z7.s, #0x1f\n" + "asr z6.s, z6.s, #0x1f\n" "asr z5.s, z5.s, #0x1f\n" + "asr z4.s, z4.s, #0x1f\n" + "sqadd z16.s, z16.s, z7.s\n" + "sqadd z17.s, z17.s, z6.s\n" + "sqadd z18.s, z18.s, z5.s\n" + "sqadd z19.s, z19.s, z4.s\n" + "and z7.d, z23.d, z0.d\n" + "and z6.d, z28.d, z1.d\n" + "and z5.d, z29.d, z2.d\n" + "and z4.d, z30.d, z3.d\n" + "asr z7.s, z7.s, #0x1f\n" "asr z6.s, z6.s, #0x1f\n" + "asr z5.s, z5.s, #0x1f\n" + "asr z4.s, z4.s, #0x1f\n" + "sqadd z23.s, z23.s, z7.s\n" + "sqadd z28.s, z28.s, z6.s\n" + "sqadd z29.s, z29.s, z5.s\n" + "sqadd z30.s, z30.s, z4.s\n" + "and z7.d, z24.d, z0.d\n" + "and z6.d, z25.d, z1.d\n" + "and z5.d, z26.d, z2.d\n" + "and z4.d, z27.d, z3.d\n" "asr z7.s, z7.s, #0x1f\n" - "sqadd z24.s, z24.s, z4.s\n" - "sqadd z25.s, z25.s, z5.s\n" - "sqadd z26.s, z26.s, z6.s\n" - "sqadd z27.s, z27.s, z7.s\n" + "asr z6.s, z6.s, #0x1f\n" + "asr z5.s, z5.s, #0x1f\n" + "asr z4.s, z4.s, #0x1f\n" + "sqadd z24.s, z24.s, z7.s\n" + "sqadd z25.s, z25.s, z6.s\n" + "sqadd z26.s, z26.s, z5.s\n" + "sqadd z27.s, z27.s, z4.s\n" "77:" // Height 6: no shift correction - "add x25, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x25]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1rw { z4.s }, p2/Z, [x20]\n" ".inst 0x4482881f // srshl z31.s, p2/M, z31.s, z0.s\n" "add z31.s, z31.s, z4.s\n" ".inst 0x4482882c // srshl z12.s, p2/M, z12.s, z1.s\n" @@ -2326,83 +2326,83 @@ void sve_hybrid_s8qs_mmla_6x4VL ( "add z25.s, z25.s, z4.s\n" "add z26.s, z26.s, z4.s\n" ".inst 0x4482887b // srshl z27.s, p2/M, z27.s, z3.s\n" - "add x25, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x25]\n" + "add x20, %x[qp], %[maxval]\n" + "ld1rw { z0.s }, p2/Z, [x20]\n" "add z27.s, z27.s, z4.s\n" - "add x25, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x25]\n" - "smin z31.s, p2/M, z31.s, z6.s\n" - "smin z12.s, p2/M, z12.s, z6.s\n" - "smin z13.s, p2/M, z13.s, z6.s\n" - "smin z14.s, p2/M, z14.s, z6.s\n" - "smin z8.s, p2/M, z8.s, z6.s\n" - "smin z9.s, p2/M, z9.s, z6.s\n" - "smin z10.s, p2/M, z10.s, z6.s\n" - "smin z11.s, p2/M, z11.s, z6.s\n" - "smin z15.s, p2/M, z15.s, z6.s\n" - "smin z20.s, p2/M, z20.s, z6.s\n" - "smin z21.s, p2/M, z21.s, z6.s\n" - "smin z22.s, p2/M, z22.s, z6.s\n" - "smin z16.s, p2/M, z16.s, z6.s\n" - "smin z17.s, p2/M, z17.s, z6.s\n" - "smin z18.s, p2/M, z18.s, z6.s\n" - "smin z19.s, p2/M, z19.s, z6.s\n" - "smin z23.s, p2/M, z23.s, z6.s\n" - "smin z28.s, p2/M, z28.s, z6.s\n" - "smin z29.s, p2/M, z29.s, z6.s\n" - "smin z30.s, p2/M, z30.s, z6.s\n" - "smin z24.s, p2/M, z24.s, z6.s\n" - "smin z25.s, p2/M, z25.s, z6.s\n" - "smin z26.s, p2/M, z26.s, z6.s\n" - "smin z27.s, p2/M, z27.s, z6.s\n" - "smax z31.s, p2/M, z31.s, z5.s\n" - "smax z12.s, p2/M, z12.s, z5.s\n" - "smax z13.s, p2/M, z13.s, z5.s\n" + "add x20, %x[qp], %[minval]\n" + "ld1rw { z1.s }, p2/Z, [x20]\n" + "smin z31.s, p2/M, z31.s, z0.s\n" + "smin z12.s, p2/M, z12.s, z0.s\n" + "smin z13.s, p2/M, z13.s, z0.s\n" + "smin z14.s, p2/M, z14.s, z0.s\n" + "smin z8.s, p2/M, z8.s, z0.s\n" + "smin z9.s, p2/M, z9.s, z0.s\n" + "smin z10.s, p2/M, z10.s, z0.s\n" + "smin z11.s, p2/M, z11.s, z0.s\n" + "smin z15.s, p2/M, z15.s, z0.s\n" + "smin z20.s, p2/M, z20.s, z0.s\n" + "smin z21.s, p2/M, z21.s, z0.s\n" + "smin z22.s, p2/M, z22.s, z0.s\n" + "smin z16.s, p2/M, z16.s, z0.s\n" + "smin z17.s, p2/M, z17.s, z0.s\n" + "smin z18.s, p2/M, z18.s, z0.s\n" + "smin z19.s, p2/M, z19.s, z0.s\n" + "smin z23.s, p2/M, z23.s, z0.s\n" + "smin z28.s, p2/M, z28.s, z0.s\n" + "smin z29.s, p2/M, z29.s, z0.s\n" + "smin z30.s, p2/M, z30.s, z0.s\n" + "smin z24.s, p2/M, z24.s, z0.s\n" + "smin z25.s, p2/M, z25.s, z0.s\n" + "smin z26.s, p2/M, z26.s, z0.s\n" + "smin z27.s, p2/M, z27.s, z0.s\n" + "smax z31.s, p2/M, z31.s, z1.s\n" + "smax z12.s, p2/M, z12.s, z1.s\n" + "smax z13.s, p2/M, z13.s, z1.s\n" "uzp1 z31.h, z31.h, z12.h\n" - "smax z14.s, p2/M, z14.s, z5.s\n" - "smax z8.s, p2/M, z8.s, z5.s\n" - "uzp1 z12.h, z13.h, z14.h\n" - "uzp1 z31.b, z31.b, z12.b\n" - "smax z9.s, p2/M, z9.s, z5.s\n" - "smax z10.s, p2/M, z10.s, z5.s\n" + "smax z14.s, p2/M, z14.s, z1.s\n" + "smax z8.s, p2/M, z8.s, z1.s\n" + "uzp1 z0.h, z13.h, z14.h\n" + "uzp1 z31.b, z31.b, z0.b\n" + "smax z9.s, p2/M, z9.s, z1.s\n" + "smax z10.s, p2/M, z10.s, z1.s\n" "uzp1 z8.h, z8.h, z9.h\n" "st1b { z31.b }, p1, [x11]\n" - "smax z11.s, p2/M, z11.s, z5.s\n" - "smax z15.s, p2/M, z15.s, z5.s\n" - "uzp1 z9.h, z10.h, z11.h\n" - "uzp1 z8.b, z8.b, z9.b\n" - "smax z20.s, p2/M, z20.s, z5.s\n" - "smax z21.s, p2/M, z21.s, z5.s\n" + "smax z11.s, p2/M, z11.s, z1.s\n" + "smax z15.s, p2/M, z15.s, z1.s\n" + "uzp1 z31.h, z10.h, z11.h\n" + "uzp1 z8.b, z8.b, z31.b\n" + "smax z20.s, p2/M, z20.s, z1.s\n" + "smax z21.s, p2/M, z21.s, z1.s\n" "uzp1 z15.h, z15.h, z20.h\n" - "st1b { z8.b }, p1, [x24]\n" - "smax z22.s, p2/M, z22.s, z5.s\n" - "smax z16.s, p2/M, z16.s, z5.s\n" + "st1b { z8.b }, p1, [x26]\n" + "smax z22.s, p2/M, z22.s, z1.s\n" + "smax z16.s, p2/M, z16.s, z1.s\n" "uzp1 z20.h, z21.h, z22.h\n" "uzp1 z15.b, z15.b, z20.b\n" - "smax z17.s, p2/M, z17.s, z5.s\n" - "smax z18.s, p2/M, z18.s, z5.s\n" + "smax z17.s, p2/M, z17.s, z1.s\n" + "smax z18.s, p2/M, z18.s, z1.s\n" "uzp1 z16.h, z16.h, z17.h\n" - "st1b { z15.b }, p1, [x23]\n" - "smax z19.s, p2/M, z19.s, z5.s\n" - "smax z23.s, p2/M, z23.s, z5.s\n" + "st1b { z15.b }, p1, [x25]\n" + "smax z19.s, p2/M, z19.s, z1.s\n" + "smax z23.s, p2/M, z23.s, z1.s\n" "uzp1 z17.h, z18.h, z19.h\n" "uzp1 z16.b, z16.b, z17.b\n" - "smax z28.s, p2/M, z28.s, z5.s\n" - "smax z29.s, p2/M, z29.s, z5.s\n" + "smax z28.s, p2/M, z28.s, z1.s\n" + "smax z29.s, p2/M, z29.s, z1.s\n" "uzp1 z23.h, z23.h, z28.h\n" - "st1b { z16.b }, p1, [x22]\n" - "smax z30.s, p2/M, z30.s, z5.s\n" - "smax z24.s, p2/M, z24.s, z5.s\n" - "uzp1 z28.h, z29.h, z30.h\n" - "uzp1 z23.b, z23.b, z28.b\n" - "smax z25.s, p2/M, z25.s, z5.s\n" - "smax z26.s, p2/M, z26.s, z5.s\n" + "st1b { z16.b }, p1, [x24]\n" + "smax z30.s, p2/M, z30.s, z1.s\n" + "smax z24.s, p2/M, z24.s, z1.s\n" + "uzp1 z16.h, z29.h, z30.h\n" + "uzp1 z23.b, z23.b, z16.b\n" + "smax z25.s, p2/M, z25.s, z1.s\n" + "smax z26.s, p2/M, z26.s, z1.s\n" "uzp1 z24.h, z24.h, z25.h\n" - "st1b { z23.b }, p1, [x21]\n" - "smax z27.s, p2/M, z27.s, z5.s\n" - "uzp1 z25.h, z26.h, z27.h\n" - "uzp1 z24.b, z24.b, z25.b\n" - "st1b { z24.b }, p1, [x20]\n" + "st1b { z23.b }, p1, [x23]\n" + "smax z27.s, p2/M, z27.s, z1.s\n" + "uzp1 z16.h, z26.h, z27.h\n" + "uzp1 z24.b, z24.b, z16.b\n" + "st1b { z24.b }, p1, [x22]\n" "addvl x11, x11, #1\n" "78:" // Height 6: Writeback done "decw x10, ALL, MUL #4\n" @@ -2420,7 +2420,6 @@ void sve_hybrid_s8qs_mmla_6x4VL ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "80:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [flags] "r" (flags), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_multiplier_ptr] "I" (offsetof(KernelArgs, multiplier_ptr)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_shift_ptr] "I" (offsetof(KernelArgs, shift_ptr)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp) : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x13", "x14", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -2428,4 +2427,4 @@ void sve_hybrid_s8qs_mmla_6x4VL ( } } // namespace arm_gemm -#endif // __ARM_FEATURE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL.hpp index 28057aa96142b6fae4cfa8bd72a8a58f16a0011e..cfa349f3aa1313813efb92ad64547448dfd468a2 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef ARM_COMPUTE_ENABLE_SVE + #include "../std_transforms_sve.hpp" #include "../performance_parameters.hpp" @@ -39,6 +39,7 @@ namespace arm_gemm { // Actual kernel implementations void sve_hybrid_s8s32_dot_6x4VL( ARGLIST ); +void sve_hybrid_s8s32_dot_6x4VL_a64fx( ARGLIST ); class cls_sve_hybrid_s8s32_dot_6x4VL { @@ -74,7 +75,6 @@ public: template static inline PerformanceParameters get_performance_parameters(const CPUInfo *ci) { - if (std::is_same::value) { switch (ci->get_cpu_model()) { default: @@ -83,10 +83,11 @@ public: return { 20.92 }; case CPUModel::V1: return { 62.24 }; + case CPUModel::A64FX: + return { 94.32 }; } } - if (std::is_same::value) { switch (ci->get_cpu_model()) { default: @@ -95,6 +96,8 @@ public: return { 22.77, 3.90, 0.47 }; case CPUModel::V1: return { 48.09, 16.24, 0.83 }; + case CPUModel::A64FX: + return { 100.19, 3.13, 0.43 }; } } @@ -103,13 +106,19 @@ public: // Default to the generic kernel kern_type kernel=sve_hybrid_s8s32_dot_6x4VL; - cls_sve_hybrid_s8s32_dot_6x4VL(const CPUInfo *) + cls_sve_hybrid_s8s32_dot_6x4VL(const CPUInfo *ci) { + switch(ci->get_cpu_model()) { + default: + break; + case CPUModel::A64FX: + kernel=sve_hybrid_s8s32_dot_6x4VL_a64fx; + break; + } } }; } // namespace arm_gemm #undef ARGLIST - #endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL/a64fx.cpp index 51e9aa1b40a08667d771ac961247addce961b71c..1a483210f34454482ad23b6a0c9b11734b4a65e2 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL/a64fx.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL/a64fx.cpp @@ -115,11 +115,11 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "5:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 6f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 7f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -135,12 +135,12 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "8:" // Height 1: Multiply loop: Main loop "sdot z8.s, z6.b, z0.b\n" "sdot z9.s, z7.b, z0.b\n" - "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n" - "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n" + "ld1b { z17.b }, p4/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" "add x26, x26, #0x4\n" - "sdot z10.s, z6.b, z0.b\n" - "sdot z11.s, z7.b, z0.b\n" + "sdot z10.s, z17.b, z0.b\n" + "sdot z11.s, z16.b, z0.b\n" "subs x27, x27, #0x4\n" "ld1rw { z0.s }, p4/Z, [x26]\n" "ld1b { z6.b }, p4/Z, [x10]\n" @@ -150,12 +150,12 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "sdot z8.s, z6.b, z0.b\n" "sdot z9.s, z7.b, z0.b\n" - "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n" - "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n" + "ld1b { z17.b }, p4/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p4/Z, [x10, #3, MUL VL]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" - "sdot z10.s, z6.b, z0.b\n" - "sdot z11.s, z7.b, z0.b\n" + "sdot z10.s, z17.b, z0.b\n" + "sdot z11.s, z16.b, z0.b\n" "addvl x10, x10, #4\n" "bne 5b\n" "st1w { z8.s }, p3, [x9]\n" @@ -183,15 +183,15 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "whilelt p0.s, x20, x11\n" "tbz %x[flags], #0, 13f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" + "add x20, x9, x20, LSL #2\n" "ld1w { z8.s }, p3/Z, [x9]\n" "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x24]\n" - "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n" - "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n" - "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n" + "ld1w { z12.s }, p3/Z, [x20]\n" + "ld1w { z13.s }, p2/Z, [x20, #1, MUL VL]\n" + "ld1w { z14.s }, p1/Z, [x20, #2, MUL VL]\n" + "ld1w { z15.s }, p0/Z, [x20, #3, MUL VL]\n" "b 14f\n" "13:" // Height 2: no accumulate "mov z8.s, #0x0\n" @@ -207,12 +207,12 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "15:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 16f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 17f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -220,7 +220,7 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "b 17f\n" "16:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" + "add x25, x26, x21\n" "17:" // Height 2: input setup done "subs x27, x27, #0x4\n" "ld1rw { z0.s }, p4/Z, [x26]\n" @@ -231,18 +231,18 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "18:" // Height 2: Multiply loop: Main loop "sdot z8.s, z6.b, z0.b\n" "sdot z12.s, z6.b, z1.b\n" - "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n" + "ld1b { z17.b }, p4/Z, [x10, #2, MUL VL]\n" "add x26, x26, #0x4\n" "sdot z9.s, z7.b, z0.b\n" "sdot z13.s, z7.b, z1.b\n" - "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n" + "ld1b { z16.b }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" "subs x27, x27, #0x4\n" "add x25, x25, #0x4\n" - "sdot z10.s, z6.b, z0.b\n" - "sdot z14.s, z6.b, z1.b\n" - "sdot z11.s, z7.b, z0.b\n" - "sdot z15.s, z7.b, z1.b\n" + "sdot z10.s, z17.b, z0.b\n" + "sdot z14.s, z17.b, z1.b\n" + "sdot z11.s, z16.b, z0.b\n" + "sdot z15.s, z16.b, z1.b\n" "ld1rw { z0.s }, p4/Z, [x26]\n" "ld1rw { z1.s }, p4/Z, [x25]\n" "ld1b { z6.b }, p4/Z, [x10]\n" @@ -252,29 +252,29 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "sdot z8.s, z6.b, z0.b\n" "sdot z12.s, z6.b, z1.b\n" - "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n" + "ld1b { z17.b }, p4/Z, [x10, #2, MUL VL]\n" "sdot z9.s, z7.b, z0.b\n" "sdot z13.s, z7.b, z1.b\n" - "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n" + "ld1b { z16.b }, p4/Z, [x10, #3, MUL VL]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" - "sdot z10.s, z6.b, z0.b\n" - "sdot z14.s, z6.b, z1.b\n" + "sdot z10.s, z17.b, z0.b\n" + "sdot z14.s, z17.b, z1.b\n" "addvl x10, x10, #4\n" - "sdot z11.s, z7.b, z0.b\n" - "sdot z15.s, z7.b, z1.b\n" + "sdot z11.s, z16.b, z0.b\n" + "sdot z15.s, z16.b, z1.b\n" "bne 15b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" + "add x20, x9, x20, LSL #2\n" "st1w { z8.s }, p3, [x9]\n" "st1w { z9.s }, p2, [x9, #1, MUL VL]\n" "st1w { z10.s }, p1, [x9, #2, MUL VL]\n" "st1w { z11.s }, p0, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "st1w { z12.s }, p3, [x24]\n" - "st1w { z13.s }, p2, [x24, #1, MUL VL]\n" - "st1w { z14.s }, p1, [x24, #2, MUL VL]\n" - "st1w { z15.s }, p0, [x24, #3, MUL VL]\n" + "st1w { z12.s }, p3, [x20]\n" + "st1w { z13.s }, p2, [x20, #1, MUL VL]\n" + "st1w { z14.s }, p1, [x20, #2, MUL VL]\n" + "st1w { z15.s }, p0, [x20, #3, MUL VL]\n" "20:" // Height 2: Writeback done "decw x11, ALL, MUL #4\n" "cmp x11, XZR\n" @@ -295,20 +295,20 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "whilelt p0.s, x20, x11\n" "tbz %x[flags], #0, 23f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x21, x9, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z8.s }, p3/Z, [x9]\n" "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x24]\n" - "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n" - "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n" - "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n" - "ld1w { z16.s }, p3/Z, [x23]\n" - "ld1w { z17.s }, p2/Z, [x23, #1, MUL VL]\n" - "ld1w { z18.s }, p1/Z, [x23, #2, MUL VL]\n" - "ld1w { z19.s }, p0/Z, [x23, #3, MUL VL]\n" + "ld1w { z12.s }, p3/Z, [x21]\n" + "ld1w { z13.s }, p2/Z, [x21, #1, MUL VL]\n" + "ld1w { z14.s }, p1/Z, [x21, #2, MUL VL]\n" + "ld1w { z15.s }, p0/Z, [x21, #3, MUL VL]\n" + "ld1w { z16.s }, p3/Z, [x20]\n" + "ld1w { z17.s }, p2/Z, [x20, #1, MUL VL]\n" + "ld1w { z18.s }, p1/Z, [x20, #2, MUL VL]\n" + "ld1w { z19.s }, p0/Z, [x20, #3, MUL VL]\n" "b 24f\n" "23:" // Height 3: no accumulate "mov z8.s, #0x0\n" @@ -328,13 +328,13 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "25:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 26f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 27f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -343,8 +343,8 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "b 27f\n" "26:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" "27:" // Height 3: input setup done "subs x27, x27, #0x4\n" "ld1rw { z0.s }, p4/Z, [x26]\n" @@ -360,21 +360,21 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "subs x27, x27, #0x4\n" "sdot z16.s, z6.b, z2.b\n" "sdot z9.s, z7.b, z0.b\n" - "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n" + "ld1b { z21.b }, p4/Z, [x10, #2, MUL VL]\n" "add x25, x25, #0x4\n" "sdot z13.s, z7.b, z1.b\n" "sdot z17.s, z7.b, z2.b\n" - "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n" + "ld1b { z20.b }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" "add x24, x24, #0x4\n" - "sdot z10.s, z6.b, z0.b\n" - "sdot z14.s, z6.b, z1.b\n" - "sdot z18.s, z6.b, z2.b\n" - "sdot z11.s, z7.b, z0.b\n" + "sdot z10.s, z21.b, z0.b\n" + "sdot z14.s, z21.b, z1.b\n" + "sdot z18.s, z21.b, z2.b\n" + "sdot z11.s, z20.b, z0.b\n" "ld1rw { z0.s }, p4/Z, [x26]\n" "ld1b { z6.b }, p4/Z, [x10]\n" - "sdot z15.s, z7.b, z1.b\n" - "sdot z19.s, z7.b, z2.b\n" + "sdot z15.s, z20.b, z1.b\n" + "sdot z19.s, z20.b, z2.b\n" "ld1rw { z1.s }, p4/Z, [x25]\n" "ld1rw { z2.s }, p4/Z, [x24]\n" "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n" @@ -386,35 +386,35 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "add x28, x28, #0x1\n" "sdot z16.s, z6.b, z2.b\n" "sdot z9.s, z7.b, z0.b\n" - "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n" + "ld1b { z21.b }, p4/Z, [x10, #2, MUL VL]\n" "cmp x28, x20\n" "sdot z13.s, z7.b, z1.b\n" "sdot z17.s, z7.b, z2.b\n" - "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n" + "ld1b { z20.b }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "sdot z10.s, z6.b, z0.b\n" - "sdot z14.s, z6.b, z1.b\n" - "sdot z18.s, z6.b, z2.b\n" - "sdot z11.s, z7.b, z0.b\n" - "sdot z15.s, z7.b, z1.b\n" - "sdot z19.s, z7.b, z2.b\n" + "sdot z10.s, z21.b, z0.b\n" + "sdot z14.s, z21.b, z1.b\n" + "sdot z18.s, z21.b, z2.b\n" + "sdot z11.s, z20.b, z0.b\n" + "sdot z15.s, z20.b, z1.b\n" + "sdot z19.s, z20.b, z2.b\n" "bne 25b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x21, x9, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "st1w { z8.s }, p3, [x9]\n" "st1w { z9.s }, p2, [x9, #1, MUL VL]\n" "st1w { z10.s }, p1, [x9, #2, MUL VL]\n" "st1w { z11.s }, p0, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "st1w { z12.s }, p3, [x24]\n" - "st1w { z13.s }, p2, [x24, #1, MUL VL]\n" - "st1w { z14.s }, p1, [x24, #2, MUL VL]\n" - "st1w { z15.s }, p0, [x24, #3, MUL VL]\n" - "st1w { z16.s }, p3, [x23]\n" - "st1w { z17.s }, p2, [x23, #1, MUL VL]\n" - "st1w { z18.s }, p1, [x23, #2, MUL VL]\n" - "st1w { z19.s }, p0, [x23, #3, MUL VL]\n" + "st1w { z12.s }, p3, [x21]\n" + "st1w { z13.s }, p2, [x21, #1, MUL VL]\n" + "st1w { z14.s }, p1, [x21, #2, MUL VL]\n" + "st1w { z15.s }, p0, [x21, #3, MUL VL]\n" + "st1w { z16.s }, p3, [x20]\n" + "st1w { z17.s }, p2, [x20, #1, MUL VL]\n" + "st1w { z18.s }, p1, [x20, #2, MUL VL]\n" + "st1w { z19.s }, p0, [x20, #3, MUL VL]\n" "30:" // Height 3: Writeback done "decw x11, ALL, MUL #4\n" "cmp x11, XZR\n" @@ -435,25 +435,25 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "whilelt p0.s, x20, x11\n" "tbz %x[flags], #0, 33f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x22, x9, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" "ld1w { z8.s }, p3/Z, [x9]\n" - "add x22, x23, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x24]\n" - "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n" - "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n" - "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n" - "ld1w { z16.s }, p3/Z, [x23]\n" - "ld1w { z17.s }, p2/Z, [x23, #1, MUL VL]\n" - "ld1w { z18.s }, p1/Z, [x23, #2, MUL VL]\n" - "ld1w { z19.s }, p0/Z, [x23, #3, MUL VL]\n" - "ld1w { z20.s }, p3/Z, [x22]\n" - "ld1w { z21.s }, p2/Z, [x22, #1, MUL VL]\n" - "ld1w { z22.s }, p1/Z, [x22, #2, MUL VL]\n" - "ld1w { z23.s }, p0/Z, [x22, #3, MUL VL]\n" + "ld1w { z12.s }, p3/Z, [x22]\n" + "ld1w { z13.s }, p2/Z, [x22, #1, MUL VL]\n" + "ld1w { z14.s }, p1/Z, [x22, #2, MUL VL]\n" + "ld1w { z15.s }, p0/Z, [x22, #3, MUL VL]\n" + "ld1w { z16.s }, p3/Z, [x21]\n" + "ld1w { z17.s }, p2/Z, [x21, #1, MUL VL]\n" + "ld1w { z18.s }, p1/Z, [x21, #2, MUL VL]\n" + "ld1w { z19.s }, p0/Z, [x21, #3, MUL VL]\n" + "ld1w { z20.s }, p3/Z, [x20]\n" + "ld1w { z21.s }, p2/Z, [x20, #1, MUL VL]\n" + "ld1w { z22.s }, p1/Z, [x20, #2, MUL VL]\n" + "ld1w { z23.s }, p0/Z, [x20, #3, MUL VL]\n" "b 34f\n" "33:" // Height 4: no accumulate "mov z8.s, #0x0\n" @@ -477,14 +477,14 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "35:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 36f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 37f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -494,9 +494,9 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "b 37f\n" "36:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" "37:" // Height 4: input setup done "subs x27, x27, #0x4\n" "ld1rw { z0.s }, p4/Z, [x26]\n" @@ -513,7 +513,7 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "subs x27, x27, #0x4\n" "sdot z16.s, z6.b, z2.b\n" "sdot z20.s, z6.b, z3.b\n" - "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n" + "ld1b { z25.b }, p4/Z, [x10, #2, MUL VL]\n" "add x25, x25, #0x4\n" "sdot z9.s, z7.b, z0.b\n" "sdot z13.s, z7.b, z1.b\n" @@ -521,19 +521,19 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "add x23, x23, #0x4\n" "sdot z17.s, z7.b, z2.b\n" "sdot z21.s, z7.b, z3.b\n" - "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n" + "ld1b { z24.b }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "sdot z10.s, z6.b, z0.b\n" - "sdot z14.s, z6.b, z1.b\n" - "sdot z18.s, z6.b, z2.b\n" - "sdot z22.s, z6.b, z3.b\n" + "sdot z10.s, z25.b, z0.b\n" + "sdot z14.s, z25.b, z1.b\n" + "sdot z18.s, z25.b, z2.b\n" + "sdot z22.s, z25.b, z3.b\n" "ld1b { z6.b }, p4/Z, [x10]\n" - "sdot z11.s, z7.b, z0.b\n" - "sdot z15.s, z7.b, z1.b\n" + "sdot z11.s, z24.b, z0.b\n" + "sdot z15.s, z24.b, z1.b\n" "ld1rw { z0.s }, p4/Z, [x26]\n" "ld1rw { z1.s }, p4/Z, [x25]\n" - "sdot z19.s, z7.b, z2.b\n" - "sdot z23.s, z7.b, z3.b\n" + "sdot z19.s, z24.b, z2.b\n" + "sdot z23.s, z24.b, z3.b\n" "ld1rw { z2.s }, p4/Z, [x24]\n" "ld1rw { z3.s }, p4/Z, [x23]\n" "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n" @@ -545,44 +545,44 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "add x28, x28, #0x1\n" "sdot z16.s, z6.b, z2.b\n" "sdot z20.s, z6.b, z3.b\n" - "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n" + "ld1b { z25.b }, p4/Z, [x10, #2, MUL VL]\n" "cmp x28, x20\n" "sdot z9.s, z7.b, z0.b\n" "sdot z13.s, z7.b, z1.b\n" "sdot z17.s, z7.b, z2.b\n" "sdot z21.s, z7.b, z3.b\n" - "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n" + "ld1b { z24.b }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "sdot z10.s, z6.b, z0.b\n" - "sdot z14.s, z6.b, z1.b\n" - "sdot z18.s, z6.b, z2.b\n" - "sdot z22.s, z6.b, z3.b\n" - "sdot z11.s, z7.b, z0.b\n" - "sdot z15.s, z7.b, z1.b\n" - "sdot z19.s, z7.b, z2.b\n" - "sdot z23.s, z7.b, z3.b\n" + "sdot z10.s, z25.b, z0.b\n" + "sdot z14.s, z25.b, z1.b\n" + "sdot z18.s, z25.b, z2.b\n" + "sdot z22.s, z25.b, z3.b\n" + "sdot z11.s, z24.b, z0.b\n" + "sdot z15.s, z24.b, z1.b\n" + "sdot z19.s, z24.b, z2.b\n" + "sdot z23.s, z24.b, z3.b\n" "bne 35b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x22, x9, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" "st1w { z8.s }, p3, [x9]\n" - "add x22, x23, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "st1w { z9.s }, p2, [x9, #1, MUL VL]\n" "st1w { z10.s }, p1, [x9, #2, MUL VL]\n" "st1w { z11.s }, p0, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "st1w { z12.s }, p3, [x24]\n" - "st1w { z13.s }, p2, [x24, #1, MUL VL]\n" - "st1w { z14.s }, p1, [x24, #2, MUL VL]\n" - "st1w { z15.s }, p0, [x24, #3, MUL VL]\n" - "st1w { z16.s }, p3, [x23]\n" - "st1w { z17.s }, p2, [x23, #1, MUL VL]\n" - "st1w { z18.s }, p1, [x23, #2, MUL VL]\n" - "st1w { z19.s }, p0, [x23, #3, MUL VL]\n" - "st1w { z20.s }, p3, [x22]\n" - "st1w { z21.s }, p2, [x22, #1, MUL VL]\n" - "st1w { z22.s }, p1, [x22, #2, MUL VL]\n" - "st1w { z23.s }, p0, [x22, #3, MUL VL]\n" + "st1w { z12.s }, p3, [x22]\n" + "st1w { z13.s }, p2, [x22, #1, MUL VL]\n" + "st1w { z14.s }, p1, [x22, #2, MUL VL]\n" + "st1w { z15.s }, p0, [x22, #3, MUL VL]\n" + "st1w { z16.s }, p3, [x21]\n" + "st1w { z17.s }, p2, [x21, #1, MUL VL]\n" + "st1w { z18.s }, p1, [x21, #2, MUL VL]\n" + "st1w { z19.s }, p0, [x21, #3, MUL VL]\n" + "st1w { z20.s }, p3, [x20]\n" + "st1w { z21.s }, p2, [x20, #1, MUL VL]\n" + "st1w { z22.s }, p1, [x20, #2, MUL VL]\n" + "st1w { z23.s }, p0, [x20, #3, MUL VL]\n" "40:" // Height 4: Writeback done "decw x11, ALL, MUL #4\n" "cmp x11, XZR\n" @@ -603,30 +603,30 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "whilelt p0.s, x20, x11\n" "tbz %x[flags], #0, 43f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "ld1w { z8.s }, p3/Z, [x9]\n" + "add x23, x9, x20, LSL #2\n" "add x22, x23, x20, LSL #2\n" + "ld1w { z8.s }, p3/Z, [x9]\n" "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x24]\n" - "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n" - "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n" - "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n" - "ld1w { z16.s }, p3/Z, [x23]\n" - "ld1w { z17.s }, p2/Z, [x23, #1, MUL VL]\n" - "ld1w { z18.s }, p1/Z, [x23, #2, MUL VL]\n" - "ld1w { z19.s }, p0/Z, [x23, #3, MUL VL]\n" - "ld1w { z20.s }, p3/Z, [x22]\n" - "ld1w { z21.s }, p2/Z, [x22, #1, MUL VL]\n" - "ld1w { z22.s }, p1/Z, [x22, #2, MUL VL]\n" - "ld1w { z23.s }, p0/Z, [x22, #3, MUL VL]\n" - "ld1w { z24.s }, p3/Z, [x21]\n" - "ld1w { z25.s }, p2/Z, [x21, #1, MUL VL]\n" - "ld1w { z26.s }, p1/Z, [x21, #2, MUL VL]\n" - "ld1w { z27.s }, p0/Z, [x21, #3, MUL VL]\n" + "ld1w { z12.s }, p3/Z, [x23]\n" + "ld1w { z13.s }, p2/Z, [x23, #1, MUL VL]\n" + "ld1w { z14.s }, p1/Z, [x23, #2, MUL VL]\n" + "ld1w { z15.s }, p0/Z, [x23, #3, MUL VL]\n" + "ld1w { z16.s }, p3/Z, [x22]\n" + "ld1w { z17.s }, p2/Z, [x22, #1, MUL VL]\n" + "ld1w { z18.s }, p1/Z, [x22, #2, MUL VL]\n" + "ld1w { z19.s }, p0/Z, [x22, #3, MUL VL]\n" + "ld1w { z20.s }, p3/Z, [x21]\n" + "ld1w { z21.s }, p2/Z, [x21, #1, MUL VL]\n" + "ld1w { z22.s }, p1/Z, [x21, #2, MUL VL]\n" + "ld1w { z23.s }, p0/Z, [x21, #3, MUL VL]\n" + "ld1w { z24.s }, p3/Z, [x20]\n" + "ld1w { z25.s }, p2/Z, [x20, #1, MUL VL]\n" + "ld1w { z26.s }, p1/Z, [x20, #2, MUL VL]\n" + "ld1w { z27.s }, p0/Z, [x20, #3, MUL VL]\n" "b 44f\n" "43:" // Height 5: no accumulate "mov z8.s, #0x0\n" @@ -654,15 +654,15 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "45:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 46f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 47f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -673,10 +673,10 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "b 47f\n" "46:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" "47:" // Height 5: input setup done "subs x27, x27, #0x4\n" "ld1rw { z0.s }, p4/Z, [x26]\n" @@ -698,29 +698,29 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "add x24, x24, #0x4\n" "sdot z24.s, z6.b, z4.b\n" "sdot z9.s, z7.b, z0.b\n" - "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n" + "ld1b { z29.b }, p4/Z, [x10, #2, MUL VL]\n" "add x23, x23, #0x4\n" "sdot z13.s, z7.b, z1.b\n" "sdot z17.s, z7.b, z2.b\n" "add x22, x22, #0x4\n" "sdot z21.s, z7.b, z3.b\n" "sdot z25.s, z7.b, z4.b\n" - "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n" + "ld1b { z28.b }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "sdot z10.s, z6.b, z0.b\n" - "sdot z14.s, z6.b, z1.b\n" - "sdot z18.s, z6.b, z2.b\n" - "sdot z22.s, z6.b, z3.b\n" - "sdot z26.s, z6.b, z4.b\n" - "sdot z11.s, z7.b, z0.b\n" + "sdot z10.s, z29.b, z0.b\n" + "sdot z14.s, z29.b, z1.b\n" + "sdot z18.s, z29.b, z2.b\n" + "sdot z22.s, z29.b, z3.b\n" + "sdot z26.s, z29.b, z4.b\n" + "sdot z11.s, z28.b, z0.b\n" "ld1rw { z0.s }, p4/Z, [x26]\n" "ld1b { z6.b }, p4/Z, [x10]\n" - "sdot z15.s, z7.b, z1.b\n" - "sdot z19.s, z7.b, z2.b\n" + "sdot z15.s, z28.b, z1.b\n" + "sdot z19.s, z28.b, z2.b\n" "ld1rw { z1.s }, p4/Z, [x25]\n" "ld1rw { z2.s }, p4/Z, [x24]\n" - "sdot z23.s, z7.b, z3.b\n" - "sdot z27.s, z7.b, z4.b\n" + "sdot z23.s, z28.b, z3.b\n" + "sdot z27.s, z28.b, z4.b\n" "ld1rw { z3.s }, p4/Z, [x23]\n" "ld1rw { z4.s }, p4/Z, [x22]\n" "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n" @@ -735,50 +735,50 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "cmp x28, x20\n" "sdot z24.s, z6.b, z4.b\n" "sdot z9.s, z7.b, z0.b\n" - "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n" + "ld1b { z29.b }, p4/Z, [x10, #2, MUL VL]\n" "sdot z13.s, z7.b, z1.b\n" "sdot z17.s, z7.b, z2.b\n" "sdot z21.s, z7.b, z3.b\n" "sdot z25.s, z7.b, z4.b\n" - "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n" + "ld1b { z28.b }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "sdot z10.s, z6.b, z0.b\n" - "sdot z14.s, z6.b, z1.b\n" - "sdot z18.s, z6.b, z2.b\n" - "sdot z22.s, z6.b, z3.b\n" - "sdot z26.s, z6.b, z4.b\n" - "sdot z11.s, z7.b, z0.b\n" - "sdot z15.s, z7.b, z1.b\n" - "sdot z19.s, z7.b, z2.b\n" - "sdot z23.s, z7.b, z3.b\n" - "sdot z27.s, z7.b, z4.b\n" + "sdot z10.s, z29.b, z0.b\n" + "sdot z14.s, z29.b, z1.b\n" + "sdot z18.s, z29.b, z2.b\n" + "sdot z22.s, z29.b, z3.b\n" + "sdot z26.s, z29.b, z4.b\n" + "sdot z11.s, z28.b, z0.b\n" + "sdot z15.s, z28.b, z1.b\n" + "sdot z19.s, z28.b, z2.b\n" + "sdot z23.s, z28.b, z3.b\n" + "sdot z27.s, z28.b, z4.b\n" "bne 45b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "st1w { z8.s }, p3, [x9]\n" + "add x23, x9, x20, LSL #2\n" "add x22, x23, x20, LSL #2\n" + "st1w { z8.s }, p3, [x9]\n" "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "st1w { z9.s }, p2, [x9, #1, MUL VL]\n" "st1w { z10.s }, p1, [x9, #2, MUL VL]\n" "st1w { z11.s }, p0, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "st1w { z12.s }, p3, [x24]\n" - "st1w { z13.s }, p2, [x24, #1, MUL VL]\n" - "st1w { z14.s }, p1, [x24, #2, MUL VL]\n" - "st1w { z15.s }, p0, [x24, #3, MUL VL]\n" - "st1w { z16.s }, p3, [x23]\n" - "st1w { z17.s }, p2, [x23, #1, MUL VL]\n" - "st1w { z18.s }, p1, [x23, #2, MUL VL]\n" - "st1w { z19.s }, p0, [x23, #3, MUL VL]\n" - "st1w { z20.s }, p3, [x22]\n" - "st1w { z21.s }, p2, [x22, #1, MUL VL]\n" - "st1w { z22.s }, p1, [x22, #2, MUL VL]\n" - "st1w { z23.s }, p0, [x22, #3, MUL VL]\n" - "st1w { z24.s }, p3, [x21]\n" - "st1w { z25.s }, p2, [x21, #1, MUL VL]\n" - "st1w { z26.s }, p1, [x21, #2, MUL VL]\n" - "st1w { z27.s }, p0, [x21, #3, MUL VL]\n" + "st1w { z12.s }, p3, [x23]\n" + "st1w { z13.s }, p2, [x23, #1, MUL VL]\n" + "st1w { z14.s }, p1, [x23, #2, MUL VL]\n" + "st1w { z15.s }, p0, [x23, #3, MUL VL]\n" + "st1w { z16.s }, p3, [x22]\n" + "st1w { z17.s }, p2, [x22, #1, MUL VL]\n" + "st1w { z18.s }, p1, [x22, #2, MUL VL]\n" + "st1w { z19.s }, p0, [x22, #3, MUL VL]\n" + "st1w { z20.s }, p3, [x21]\n" + "st1w { z21.s }, p2, [x21, #1, MUL VL]\n" + "st1w { z22.s }, p1, [x21, #2, MUL VL]\n" + "st1w { z23.s }, p0, [x21, #3, MUL VL]\n" + "st1w { z24.s }, p3, [x20]\n" + "st1w { z25.s }, p2, [x20, #1, MUL VL]\n" + "st1w { z26.s }, p1, [x20, #2, MUL VL]\n" + "st1w { z27.s }, p0, [x20, #3, MUL VL]\n" "50:" // Height 5: Writeback done "decw x11, ALL, MUL #4\n" "cmp x11, XZR\n" @@ -862,16 +862,16 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "55:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 56f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 57f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -883,11 +883,11 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "b 57f\n" "56:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" + "add x21, x22, x21\n" "57:" // Height 6: input setup done "subs x27, x27, #0x4\n" "ld1rw { z0.s }, p4/Z, [x26]\n" @@ -1022,7 +1022,6 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "62:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -1030,4 +1029,4 @@ void sve_hybrid_s8s32_dot_6x4VL_a64fx ( } } // namespace arm_gemm -#endif // ARM_COMPUTE_ENABLE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL/generic.cpp index b3d2e6b271194dccd912e77ac598edb1ceb7dd1a..eeef192b6611e231214bda316b4fbb66902f4c3b 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_dot_6x4VL/generic.cpp @@ -115,11 +115,11 @@ void sve_hybrid_s8s32_dot_6x4VL ( "5:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 6f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 7f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -132,87 +132,87 @@ void sve_hybrid_s8s32_dot_6x4VL ( "8:" // Height 1: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" "ld1rqb { z0.b }, p0/Z, [x26]\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "sdot z8.s, z6.b, z0.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" - "sdot z11.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #4, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #5, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #6, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #7, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10]\n" + "sdot z8.s, z16.b, z0.b[0]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z9.s, z16.b, z0.b[0]\n" + "ld1b { z16.b }, p5/Z, [x10, #2, MUL VL]\n" + "sdot z10.s, z16.b, z0.b[0]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" + "sdot z11.s, z16.b, z0.b[0]\n" + "ld1b { z16.b }, p5/Z, [x10, #4, MUL VL]\n" + "sdot z8.s, z16.b, z0.b[1]\n" + "ld1b { z16.b }, p5/Z, [x10, #5, MUL VL]\n" + "sdot z9.s, z16.b, z0.b[1]\n" + "ld1b { z16.b }, p5/Z, [x10, #6, MUL VL]\n" + "sdot z10.s, z16.b, z0.b[1]\n" + "ld1b { z16.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "sdot z11.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #-8, MUL VL]\n" - "ld1b { z7.b }, p5/Z, [x10, #-7, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[2]\n" - "sdot z9.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #-6, MUL VL]\n" - "ld1b { z7.b }, p5/Z, [x10, #-5, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[2]\n" - "sdot z11.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #-4, MUL VL]\n" - "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[3]\n" - "sdot z9.s, z7.b, z0.b[3]\n" - "ld1b { z6.b }, p5/Z, [x10, #-2, MUL VL]\n" - "ld1b { z7.b }, p5/Z, [x10, #-1, MUL VL]\n" + "sdot z11.s, z16.b, z0.b[1]\n" + "ld1b { z17.b }, p5/Z, [x10, #-8, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #-7, MUL VL]\n" + "sdot z8.s, z17.b, z0.b[2]\n" + "sdot z9.s, z16.b, z0.b[2]\n" + "ld1b { z17.b }, p5/Z, [x10, #-6, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #-5, MUL VL]\n" + "sdot z10.s, z17.b, z0.b[2]\n" + "sdot z11.s, z16.b, z0.b[2]\n" + "ld1b { z17.b }, p5/Z, [x10, #-4, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #-3, MUL VL]\n" + "sdot z8.s, z17.b, z0.b[3]\n" + "sdot z9.s, z16.b, z0.b[3]\n" + "ld1b { z17.b }, p5/Z, [x10, #-2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #-1, MUL VL]\n" "sub x27, x27, #0x10\n" "cmp x27, #0x10\n" - "sdot z10.s, z6.b, z0.b[3]\n" - "sdot z11.s, z7.b, z0.b[3]\n" + "sdot z10.s, z17.b, z0.b[3]\n" + "sdot z11.s, z16.b, z0.b[3]\n" "add x26, x26, #0x10\n" "bgt 8b\n" "9:" // Height 1: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" "ld1rqb { z0.b }, p0/Z, [x26]\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "sdot z8.s, z6.b, z0.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10]\n" + "sdot z8.s, z16.b, z0.b[0]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z9.s, z16.b, z0.b[0]\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x4\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[0]\n" - "sdot z11.s, z7.b, z0.b[0]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" + "sdot z10.s, z17.b, z0.b[0]\n" + "sdot z11.s, z16.b, z0.b[0]\n" "addvl x10, x10, #4\n" "ble 10f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[1]\n" - "sdot z9.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z8.s, z17.b, z0.b[1]\n" + "sdot z9.s, z16.b, z0.b[1]\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x4\n" - "sdot z10.s, z6.b, z0.b[1]\n" - "sdot z11.s, z7.b, z0.b[1]\n" + "sdot z10.s, z17.b, z0.b[1]\n" + "sdot z11.s, z16.b, z0.b[1]\n" "addvl x10, x10, #4\n" "ble 10f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[2]\n" - "sdot z9.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z8.s, z17.b, z0.b[2]\n" + "sdot z9.s, z16.b, z0.b[2]\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x4\n" - "sdot z10.s, z6.b, z0.b[2]\n" - "sdot z11.s, z7.b, z0.b[2]\n" + "sdot z10.s, z17.b, z0.b[2]\n" + "sdot z11.s, z16.b, z0.b[2]\n" "addvl x10, x10, #4\n" "ble 10f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[3]\n" - "sdot z9.s, z7.b, z0.b[3]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[3]\n" - "sdot z11.s, z7.b, z0.b[3]\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z8.s, z17.b, z0.b[3]\n" + "sdot z9.s, z16.b, z0.b[3]\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" + "sdot z10.s, z17.b, z0.b[3]\n" + "sdot z11.s, z16.b, z0.b[3]\n" "addvl x10, x10, #4\n" "10:" // Height 1: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -244,15 +244,15 @@ void sve_hybrid_s8s32_dot_6x4VL ( "whilelt p1.s, x20, x11\n" "tbz %x[flags], #0, 14f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" + "add x20, x9, x20, LSL #2\n" "ld1w { z8.s }, p4/Z, [x9]\n" "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x24]\n" - "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n" - "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x20]\n" + "ld1w { z13.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z15.s }, p1/Z, [x20, #3, MUL VL]\n" "b 15f\n" "14:" // Height 2: no accumulate "mov z8.s, #0x0\n" @@ -268,12 +268,12 @@ void sve_hybrid_s8s32_dot_6x4VL ( "16:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 17f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 18f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -281,146 +281,146 @@ void sve_hybrid_s8s32_dot_6x4VL ( "b 18f\n" "17:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" + "add x25, x26, x21\n" "18:" // Height 2: input setup done "cmp x27, #0x10\n" "ble 20f\n" "19:" // Height 2: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z0.b }, p0/Z, [x26]\n" - "ld1rqb { z1.b }, p0/Z, [x25]\n" + "ld1rqb { z1.b }, p0/Z, [x26]\n" + "ld1rqb { z0.b }, p0/Z, [x25]\n" "sub x27, x27, #0x10\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[0]\n" - "sdot z12.s, z6.b, z1.b[0]\n" - "sdot z9.s, z7.b, z0.b[0]\n" - "sdot z13.s, z7.b, z1.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[0]\n" - "sdot z14.s, z6.b, z1.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #4, MUL VL]\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z8.s, z17.b, z1.b[0]\n" + "sdot z12.s, z17.b, z0.b[0]\n" + "sdot z9.s, z16.b, z1.b[0]\n" + "sdot z13.s, z16.b, z0.b[0]\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" + "sdot z10.s, z17.b, z1.b[0]\n" + "sdot z14.s, z17.b, z0.b[0]\n" + "ld1b { z17.b }, p5/Z, [x10, #4, MUL VL]\n" "cmp x27, #0x10\n" - "sdot z11.s, z7.b, z0.b[0]\n" - "sdot z15.s, z7.b, z1.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #5, MUL VL]\n" + "sdot z11.s, z16.b, z1.b[0]\n" + "sdot z15.s, z16.b, z0.b[0]\n" + "ld1b { z16.b }, p5/Z, [x10, #5, MUL VL]\n" "add x26, x26, #0x10\n" - "sdot z8.s, z6.b, z0.b[1]\n" - "sdot z12.s, z6.b, z1.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #6, MUL VL]\n" + "sdot z8.s, z17.b, z1.b[1]\n" + "sdot z12.s, z17.b, z0.b[1]\n" + "ld1b { z17.b }, p5/Z, [x10, #6, MUL VL]\n" "add x25, x25, #0x10\n" - "sdot z9.s, z7.b, z0.b[1]\n" - "sdot z13.s, z7.b, z1.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #7, MUL VL]\n" + "sdot z9.s, z16.b, z1.b[1]\n" + "sdot z13.s, z16.b, z0.b[1]\n" + "ld1b { z16.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "sdot z10.s, z6.b, z0.b[1]\n" - "sdot z14.s, z6.b, z1.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #-8, MUL VL]\n" - "sdot z11.s, z7.b, z0.b[1]\n" - "sdot z15.s, z7.b, z1.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #-7, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[2]\n" - "sdot z12.s, z6.b, z1.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #-6, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[2]\n" - "sdot z13.s, z7.b, z1.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #-5, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[2]\n" - "sdot z14.s, z6.b, z1.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #-4, MUL VL]\n" - "sdot z11.s, z7.b, z0.b[2]\n" - "sdot z15.s, z7.b, z1.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[3]\n" - "sdot z12.s, z6.b, z1.b[3]\n" - "ld1b { z6.b }, p5/Z, [x10, #-2, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[3]\n" - "sdot z13.s, z7.b, z1.b[3]\n" - "ld1b { z7.b }, p5/Z, [x10, #-1, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[3]\n" - "sdot z14.s, z6.b, z1.b[3]\n" - "sdot z11.s, z7.b, z0.b[3]\n" - "sdot z15.s, z7.b, z1.b[3]\n" + "sdot z10.s, z17.b, z1.b[1]\n" + "sdot z14.s, z17.b, z0.b[1]\n" + "ld1b { z17.b }, p5/Z, [x10, #-8, MUL VL]\n" + "sdot z11.s, z16.b, z1.b[1]\n" + "sdot z15.s, z16.b, z0.b[1]\n" + "ld1b { z16.b }, p5/Z, [x10, #-7, MUL VL]\n" + "sdot z8.s, z17.b, z1.b[2]\n" + "sdot z12.s, z17.b, z0.b[2]\n" + "ld1b { z17.b }, p5/Z, [x10, #-6, MUL VL]\n" + "sdot z9.s, z16.b, z1.b[2]\n" + "sdot z13.s, z16.b, z0.b[2]\n" + "ld1b { z16.b }, p5/Z, [x10, #-5, MUL VL]\n" + "sdot z10.s, z17.b, z1.b[2]\n" + "sdot z14.s, z17.b, z0.b[2]\n" + "ld1b { z17.b }, p5/Z, [x10, #-4, MUL VL]\n" + "sdot z11.s, z16.b, z1.b[2]\n" + "sdot z15.s, z16.b, z0.b[2]\n" + "ld1b { z16.b }, p5/Z, [x10, #-3, MUL VL]\n" + "sdot z8.s, z17.b, z1.b[3]\n" + "sdot z12.s, z17.b, z0.b[3]\n" + "ld1b { z17.b }, p5/Z, [x10, #-2, MUL VL]\n" + "sdot z9.s, z16.b, z1.b[3]\n" + "sdot z13.s, z16.b, z0.b[3]\n" + "ld1b { z16.b }, p5/Z, [x10, #-1, MUL VL]\n" + "sdot z10.s, z17.b, z1.b[3]\n" + "sdot z14.s, z17.b, z0.b[3]\n" + "sdot z11.s, z16.b, z1.b[3]\n" + "sdot z15.s, z16.b, z0.b[3]\n" "bgt 19b\n" "20:" // Height 2: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" "ld1rqb { z0.b }, p0/Z, [x26]\n" "ld1rqb { z1.b }, p0/Z, [x25]\n" "subs x27, x27, #0x4\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[0]\n" - "sdot z12.s, z6.b, z1.b[0]\n" - "sdot z9.s, z7.b, z0.b[0]\n" - "sdot z13.s, z7.b, z1.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[0]\n" - "sdot z14.s, z6.b, z1.b[0]\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z8.s, z17.b, z0.b[0]\n" + "sdot z12.s, z17.b, z1.b[0]\n" + "sdot z9.s, z16.b, z0.b[0]\n" + "sdot z13.s, z16.b, z1.b[0]\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" + "sdot z10.s, z17.b, z0.b[0]\n" + "sdot z14.s, z17.b, z1.b[0]\n" "addvl x10, x10, #4\n" - "sdot z11.s, z7.b, z0.b[0]\n" - "sdot z15.s, z7.b, z1.b[0]\n" + "sdot z11.s, z16.b, z0.b[0]\n" + "sdot z15.s, z16.b, z1.b[0]\n" "ble 21f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[1]\n" - "sdot z12.s, z6.b, z1.b[1]\n" - "sdot z9.s, z7.b, z0.b[1]\n" - "sdot z13.s, z7.b, z1.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z8.s, z17.b, z0.b[1]\n" + "sdot z12.s, z17.b, z1.b[1]\n" + "sdot z9.s, z16.b, z0.b[1]\n" + "sdot z13.s, z16.b, z1.b[1]\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x4\n" - "sdot z10.s, z6.b, z0.b[1]\n" - "sdot z14.s, z6.b, z1.b[1]\n" + "sdot z10.s, z17.b, z0.b[1]\n" + "sdot z14.s, z17.b, z1.b[1]\n" "addvl x10, x10, #4\n" - "sdot z11.s, z7.b, z0.b[1]\n" - "sdot z15.s, z7.b, z1.b[1]\n" + "sdot z11.s, z16.b, z0.b[1]\n" + "sdot z15.s, z16.b, z1.b[1]\n" "ble 21f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[2]\n" - "sdot z12.s, z6.b, z1.b[2]\n" - "sdot z9.s, z7.b, z0.b[2]\n" - "sdot z13.s, z7.b, z1.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z8.s, z17.b, z0.b[2]\n" + "sdot z12.s, z17.b, z1.b[2]\n" + "sdot z9.s, z16.b, z0.b[2]\n" + "sdot z13.s, z16.b, z1.b[2]\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x4\n" - "sdot z10.s, z6.b, z0.b[2]\n" - "sdot z14.s, z6.b, z1.b[2]\n" + "sdot z10.s, z17.b, z0.b[2]\n" + "sdot z14.s, z17.b, z1.b[2]\n" "addvl x10, x10, #4\n" - "sdot z11.s, z7.b, z0.b[2]\n" - "sdot z15.s, z7.b, z1.b[2]\n" + "sdot z11.s, z16.b, z0.b[2]\n" + "sdot z15.s, z16.b, z1.b[2]\n" "ble 21f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[3]\n" - "sdot z12.s, z6.b, z1.b[3]\n" - "sdot z9.s, z7.b, z0.b[3]\n" - "sdot z13.s, z7.b, z1.b[3]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[3]\n" - "sdot z14.s, z6.b, z1.b[3]\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z8.s, z17.b, z0.b[3]\n" + "sdot z12.s, z17.b, z1.b[3]\n" + "sdot z9.s, z16.b, z0.b[3]\n" + "sdot z13.s, z16.b, z1.b[3]\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" + "sdot z10.s, z17.b, z0.b[3]\n" + "sdot z14.s, z17.b, z1.b[3]\n" "addvl x10, x10, #4\n" - "sdot z11.s, z7.b, z0.b[3]\n" - "sdot z15.s, z7.b, z1.b[3]\n" + "sdot z11.s, z16.b, z0.b[3]\n" + "sdot z15.s, z16.b, z1.b[3]\n" "21:" // Height 2: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 16b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" + "add x20, x9, x20, LSL #2\n" "st1w { z8.s }, p4, [x9]\n" "st1w { z9.s }, p3, [x9, #1, MUL VL]\n" "st1w { z10.s }, p2, [x9, #2, MUL VL]\n" "st1w { z11.s }, p1, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "st1w { z12.s }, p4, [x24]\n" - "st1w { z13.s }, p3, [x24, #1, MUL VL]\n" - "st1w { z14.s }, p2, [x24, #2, MUL VL]\n" - "st1w { z15.s }, p1, [x24, #3, MUL VL]\n" + "st1w { z12.s }, p4, [x20]\n" + "st1w { z13.s }, p3, [x20, #1, MUL VL]\n" + "st1w { z14.s }, p2, [x20, #2, MUL VL]\n" + "st1w { z15.s }, p1, [x20, #3, MUL VL]\n" "22:" // Height 2: Writeback done "decw x11, ALL, MUL #4\n" "cmp x11, XZR\n" @@ -441,20 +441,20 @@ void sve_hybrid_s8s32_dot_6x4VL ( "whilelt p1.s, x20, x11\n" "tbz %x[flags], #0, 25f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x21, x9, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z8.s }, p4/Z, [x9]\n" "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x24]\n" - "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n" - "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z16.s }, p4/Z, [x23]\n" - "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n" - "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x21]\n" + "ld1w { z13.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x21, #2, MUL VL]\n" + "ld1w { z15.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x20]\n" + "ld1w { z17.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z19.s }, p1/Z, [x20, #3, MUL VL]\n" "b 26f\n" "25:" // Height 3: no accumulate "mov z8.s, #0x0\n" @@ -474,13 +474,13 @@ void sve_hybrid_s8s32_dot_6x4VL ( "27:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 28f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 29f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -489,86 +489,86 @@ void sve_hybrid_s8s32_dot_6x4VL ( "b 29f\n" "28:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" "29:" // Height 3: input setup done "cmp x27, #0x10\n" "ble 31f\n" "30:" // Height 3: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z0.b }, p0/Z, [x26]\n" + "ld1rqb { z2.b }, p0/Z, [x26]\n" "ld1rqb { z1.b }, p0/Z, [x25]\n" "sub x27, x27, #0x10\n" - "ld1rqb { z2.b }, p0/Z, [x24]\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "sdot z8.s, z6.b, z0.b[0]\n" - "sdot z12.s, z6.b, z1.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z16.s, z6.b, z2.b[0]\n" - "sdot z9.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[0]\n" - "sdot z17.s, z7.b, z2.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1rqb { z0.b }, p0/Z, [x24]\n" + "ld1b { z21.b }, p5/Z, [x10]\n" + "sdot z8.s, z21.b, z2.b[0]\n" + "sdot z12.s, z21.b, z1.b[0]\n" + "ld1b { z20.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z16.s, z21.b, z0.b[0]\n" + "sdot z9.s, z20.b, z2.b[0]\n" + "ld1b { z21.b }, p5/Z, [x10, #2, MUL VL]\n" + "sdot z13.s, z20.b, z1.b[0]\n" + "sdot z17.s, z20.b, z0.b[0]\n" + "ld1b { z20.b }, p5/Z, [x10, #3, MUL VL]\n" "cmp x27, #0x10\n" - "sdot z10.s, z6.b, z0.b[0]\n" - "sdot z14.s, z6.b, z1.b[0]\n" + "sdot z10.s, z21.b, z2.b[0]\n" + "sdot z14.s, z21.b, z1.b[0]\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" - "sdot z18.s, z6.b, z2.b[0]\n" - "sdot z11.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #4, MUL VL]\n" + "sdot z18.s, z21.b, z0.b[0]\n" + "sdot z11.s, z20.b, z2.b[0]\n" + "ld1b { z21.b }, p5/Z, [x10, #4, MUL VL]\n" "add x24, x24, #0x10\n" - "sdot z15.s, z7.b, z1.b[0]\n" - "sdot z19.s, z7.b, z2.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #5, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[1]\n" - "sdot z12.s, z6.b, z1.b[1]\n" - "sdot z16.s, z6.b, z2.b[1]\n" - "sdot z9.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #6, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[1]\n" - "sdot z17.s, z7.b, z2.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #7, MUL VL]\n" + "sdot z15.s, z20.b, z1.b[0]\n" + "sdot z19.s, z20.b, z0.b[0]\n" + "ld1b { z20.b }, p5/Z, [x10, #5, MUL VL]\n" + "sdot z8.s, z21.b, z2.b[1]\n" + "sdot z12.s, z21.b, z1.b[1]\n" + "sdot z16.s, z21.b, z0.b[1]\n" + "sdot z9.s, z20.b, z2.b[1]\n" + "ld1b { z21.b }, p5/Z, [x10, #6, MUL VL]\n" + "sdot z13.s, z20.b, z1.b[1]\n" + "sdot z17.s, z20.b, z0.b[1]\n" + "ld1b { z20.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "sdot z10.s, z6.b, z0.b[1]\n" - "sdot z14.s, z6.b, z1.b[1]\n" - "sdot z18.s, z6.b, z2.b[1]\n" - "sdot z11.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #-8, MUL VL]\n" - "sdot z15.s, z7.b, z1.b[1]\n" - "sdot z19.s, z7.b, z2.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #-7, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[2]\n" - "sdot z12.s, z6.b, z1.b[2]\n" - "sdot z16.s, z6.b, z2.b[2]\n" - "sdot z9.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #-6, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[2]\n" - "sdot z17.s, z7.b, z2.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #-5, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[2]\n" - "sdot z14.s, z6.b, z1.b[2]\n" - "sdot z18.s, z6.b, z2.b[2]\n" - "sdot z11.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #-4, MUL VL]\n" - "sdot z15.s, z7.b, z1.b[2]\n" - "sdot z19.s, z7.b, z2.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[3]\n" - "sdot z12.s, z6.b, z1.b[3]\n" - "sdot z16.s, z6.b, z2.b[3]\n" - "sdot z9.s, z7.b, z0.b[3]\n" - "ld1b { z6.b }, p5/Z, [x10, #-2, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[3]\n" - "sdot z17.s, z7.b, z2.b[3]\n" - "ld1b { z7.b }, p5/Z, [x10, #-1, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[3]\n" - "sdot z14.s, z6.b, z1.b[3]\n" - "sdot z18.s, z6.b, z2.b[3]\n" - "sdot z11.s, z7.b, z0.b[3]\n" - "sdot z15.s, z7.b, z1.b[3]\n" - "sdot z19.s, z7.b, z2.b[3]\n" + "sdot z10.s, z21.b, z2.b[1]\n" + "sdot z14.s, z21.b, z1.b[1]\n" + "sdot z18.s, z21.b, z0.b[1]\n" + "sdot z11.s, z20.b, z2.b[1]\n" + "ld1b { z21.b }, p5/Z, [x10, #-8, MUL VL]\n" + "sdot z15.s, z20.b, z1.b[1]\n" + "sdot z19.s, z20.b, z0.b[1]\n" + "ld1b { z20.b }, p5/Z, [x10, #-7, MUL VL]\n" + "sdot z8.s, z21.b, z2.b[2]\n" + "sdot z12.s, z21.b, z1.b[2]\n" + "sdot z16.s, z21.b, z0.b[2]\n" + "sdot z9.s, z20.b, z2.b[2]\n" + "ld1b { z21.b }, p5/Z, [x10, #-6, MUL VL]\n" + "sdot z13.s, z20.b, z1.b[2]\n" + "sdot z17.s, z20.b, z0.b[2]\n" + "ld1b { z20.b }, p5/Z, [x10, #-5, MUL VL]\n" + "sdot z10.s, z21.b, z2.b[2]\n" + "sdot z14.s, z21.b, z1.b[2]\n" + "sdot z18.s, z21.b, z0.b[2]\n" + "sdot z11.s, z20.b, z2.b[2]\n" + "ld1b { z21.b }, p5/Z, [x10, #-4, MUL VL]\n" + "sdot z15.s, z20.b, z1.b[2]\n" + "sdot z19.s, z20.b, z0.b[2]\n" + "ld1b { z20.b }, p5/Z, [x10, #-3, MUL VL]\n" + "sdot z8.s, z21.b, z2.b[3]\n" + "sdot z12.s, z21.b, z1.b[3]\n" + "sdot z16.s, z21.b, z0.b[3]\n" + "sdot z9.s, z20.b, z2.b[3]\n" + "ld1b { z21.b }, p5/Z, [x10, #-2, MUL VL]\n" + "sdot z13.s, z20.b, z1.b[3]\n" + "sdot z17.s, z20.b, z0.b[3]\n" + "ld1b { z20.b }, p5/Z, [x10, #-1, MUL VL]\n" + "sdot z10.s, z21.b, z2.b[3]\n" + "sdot z14.s, z21.b, z1.b[3]\n" + "sdot z18.s, z21.b, z0.b[3]\n" + "sdot z11.s, z20.b, z2.b[3]\n" + "sdot z15.s, z20.b, z1.b[3]\n" + "sdot z19.s, z20.b, z0.b[3]\n" "bgt 30b\n" "31:" // Height 3: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" @@ -576,100 +576,100 @@ void sve_hybrid_s8s32_dot_6x4VL ( "ld1rqb { z1.b }, p0/Z, [x25]\n" "subs x27, x27, #0x4\n" "ld1rqb { z2.b }, p0/Z, [x24]\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "sdot z8.s, z6.b, z0.b[0]\n" - "sdot z12.s, z6.b, z1.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z16.s, z6.b, z2.b[0]\n" - "sdot z9.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[0]\n" - "sdot z17.s, z7.b, z2.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1b { z21.b }, p5/Z, [x10]\n" + "sdot z8.s, z21.b, z0.b[0]\n" + "sdot z12.s, z21.b, z1.b[0]\n" + "ld1b { z20.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z16.s, z21.b, z2.b[0]\n" + "sdot z9.s, z20.b, z0.b[0]\n" + "ld1b { z21.b }, p5/Z, [x10, #2, MUL VL]\n" + "sdot z13.s, z20.b, z1.b[0]\n" + "sdot z17.s, z20.b, z2.b[0]\n" + "ld1b { z20.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "sdot z10.s, z6.b, z0.b[0]\n" - "sdot z14.s, z6.b, z1.b[0]\n" - "sdot z18.s, z6.b, z2.b[0]\n" - "sdot z11.s, z7.b, z0.b[0]\n" - "sdot z15.s, z7.b, z1.b[0]\n" - "sdot z19.s, z7.b, z2.b[0]\n" + "sdot z10.s, z21.b, z0.b[0]\n" + "sdot z14.s, z21.b, z1.b[0]\n" + "sdot z18.s, z21.b, z2.b[0]\n" + "sdot z11.s, z20.b, z0.b[0]\n" + "sdot z15.s, z20.b, z1.b[0]\n" + "sdot z19.s, z20.b, z2.b[0]\n" "ble 32f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[1]\n" - "sdot z12.s, z6.b, z1.b[1]\n" - "sdot z16.s, z6.b, z2.b[1]\n" - "sdot z9.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z21.b }, p5/Z, [x10]\n" + "ld1b { z20.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z8.s, z21.b, z0.b[1]\n" + "sdot z12.s, z21.b, z1.b[1]\n" + "sdot z16.s, z21.b, z2.b[1]\n" + "sdot z9.s, z20.b, z0.b[1]\n" + "ld1b { z21.b }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x4\n" - "sdot z13.s, z7.b, z1.b[1]\n" - "sdot z17.s, z7.b, z2.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "sdot z13.s, z20.b, z1.b[1]\n" + "sdot z17.s, z20.b, z2.b[1]\n" + "ld1b { z20.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "sdot z10.s, z6.b, z0.b[1]\n" - "sdot z14.s, z6.b, z1.b[1]\n" - "sdot z18.s, z6.b, z2.b[1]\n" - "sdot z11.s, z7.b, z0.b[1]\n" - "sdot z15.s, z7.b, z1.b[1]\n" - "sdot z19.s, z7.b, z2.b[1]\n" + "sdot z10.s, z21.b, z0.b[1]\n" + "sdot z14.s, z21.b, z1.b[1]\n" + "sdot z18.s, z21.b, z2.b[1]\n" + "sdot z11.s, z20.b, z0.b[1]\n" + "sdot z15.s, z20.b, z1.b[1]\n" + "sdot z19.s, z20.b, z2.b[1]\n" "ble 32f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[2]\n" - "sdot z12.s, z6.b, z1.b[2]\n" - "sdot z16.s, z6.b, z2.b[2]\n" - "sdot z9.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z21.b }, p5/Z, [x10]\n" + "ld1b { z20.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z8.s, z21.b, z0.b[2]\n" + "sdot z12.s, z21.b, z1.b[2]\n" + "sdot z16.s, z21.b, z2.b[2]\n" + "sdot z9.s, z20.b, z0.b[2]\n" + "ld1b { z21.b }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x4\n" - "sdot z13.s, z7.b, z1.b[2]\n" - "sdot z17.s, z7.b, z2.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "sdot z13.s, z20.b, z1.b[2]\n" + "sdot z17.s, z20.b, z2.b[2]\n" + "ld1b { z20.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "sdot z10.s, z6.b, z0.b[2]\n" - "sdot z14.s, z6.b, z1.b[2]\n" - "sdot z18.s, z6.b, z2.b[2]\n" - "sdot z11.s, z7.b, z0.b[2]\n" - "sdot z15.s, z7.b, z1.b[2]\n" - "sdot z19.s, z7.b, z2.b[2]\n" + "sdot z10.s, z21.b, z0.b[2]\n" + "sdot z14.s, z21.b, z1.b[2]\n" + "sdot z18.s, z21.b, z2.b[2]\n" + "sdot z11.s, z20.b, z0.b[2]\n" + "sdot z15.s, z20.b, z1.b[2]\n" + "sdot z19.s, z20.b, z2.b[2]\n" "ble 32f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[3]\n" - "sdot z12.s, z6.b, z1.b[3]\n" - "sdot z16.s, z6.b, z2.b[3]\n" - "sdot z9.s, z7.b, z0.b[3]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[3]\n" - "sdot z17.s, z7.b, z2.b[3]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1b { z21.b }, p5/Z, [x10]\n" + "ld1b { z20.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z8.s, z21.b, z0.b[3]\n" + "sdot z12.s, z21.b, z1.b[3]\n" + "sdot z16.s, z21.b, z2.b[3]\n" + "sdot z9.s, z20.b, z0.b[3]\n" + "ld1b { z21.b }, p5/Z, [x10, #2, MUL VL]\n" + "sdot z13.s, z20.b, z1.b[3]\n" + "sdot z17.s, z20.b, z2.b[3]\n" + "ld1b { z20.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "sdot z10.s, z6.b, z0.b[3]\n" - "sdot z14.s, z6.b, z1.b[3]\n" - "sdot z18.s, z6.b, z2.b[3]\n" - "sdot z11.s, z7.b, z0.b[3]\n" - "sdot z15.s, z7.b, z1.b[3]\n" - "sdot z19.s, z7.b, z2.b[3]\n" + "sdot z10.s, z21.b, z0.b[3]\n" + "sdot z14.s, z21.b, z1.b[3]\n" + "sdot z18.s, z21.b, z2.b[3]\n" + "sdot z11.s, z20.b, z0.b[3]\n" + "sdot z15.s, z20.b, z1.b[3]\n" + "sdot z19.s, z20.b, z2.b[3]\n" "32:" // Height 3: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 27b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x21, x9, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "st1w { z8.s }, p4, [x9]\n" "st1w { z9.s }, p3, [x9, #1, MUL VL]\n" "st1w { z10.s }, p2, [x9, #2, MUL VL]\n" "st1w { z11.s }, p1, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "st1w { z12.s }, p4, [x24]\n" - "st1w { z13.s }, p3, [x24, #1, MUL VL]\n" - "st1w { z14.s }, p2, [x24, #2, MUL VL]\n" - "st1w { z15.s }, p1, [x24, #3, MUL VL]\n" - "st1w { z16.s }, p4, [x23]\n" - "st1w { z17.s }, p3, [x23, #1, MUL VL]\n" - "st1w { z18.s }, p2, [x23, #2, MUL VL]\n" - "st1w { z19.s }, p1, [x23, #3, MUL VL]\n" + "st1w { z12.s }, p4, [x21]\n" + "st1w { z13.s }, p3, [x21, #1, MUL VL]\n" + "st1w { z14.s }, p2, [x21, #2, MUL VL]\n" + "st1w { z15.s }, p1, [x21, #3, MUL VL]\n" + "st1w { z16.s }, p4, [x20]\n" + "st1w { z17.s }, p3, [x20, #1, MUL VL]\n" + "st1w { z18.s }, p2, [x20, #2, MUL VL]\n" + "st1w { z19.s }, p1, [x20, #3, MUL VL]\n" "33:" // Height 3: Writeback done "decw x11, ALL, MUL #4\n" "cmp x11, XZR\n" @@ -690,25 +690,25 @@ void sve_hybrid_s8s32_dot_6x4VL ( "whilelt p1.s, x20, x11\n" "tbz %x[flags], #0, 36f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x22, x9, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" "ld1w { z8.s }, p4/Z, [x9]\n" - "add x22, x23, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x24]\n" - "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n" - "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z16.s }, p4/Z, [x23]\n" - "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n" - "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x22]\n" - "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n" - "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x22]\n" + "ld1w { z13.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x22, #2, MUL VL]\n" + "ld1w { z15.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x21]\n" + "ld1w { z17.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x21, #2, MUL VL]\n" + "ld1w { z19.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x20]\n" + "ld1w { z21.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z23.s }, p1/Z, [x20, #3, MUL VL]\n" "b 37f\n" "36:" // Height 4: no accumulate "mov z8.s, #0x0\n" @@ -732,14 +732,14 @@ void sve_hybrid_s8s32_dot_6x4VL ( "38:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 39f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 40f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -749,105 +749,105 @@ void sve_hybrid_s8s32_dot_6x4VL ( "b 40f\n" "39:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" "40:" // Height 4: input setup done "cmp x27, #0x10\n" "ble 42f\n" "41:" // Height 4: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z0.b }, p0/Z, [x26]\n" - "ld1rqb { z1.b }, p0/Z, [x25]\n" + "ld1rqb { z3.b }, p0/Z, [x26]\n" + "ld1rqb { z2.b }, p0/Z, [x25]\n" "sub x27, x27, #0x10\n" - "ld1rqb { z2.b }, p0/Z, [x24]\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" + "ld1rqb { z1.b }, p0/Z, [x24]\n" + "ld1rqb { z0.b }, p0/Z, [x23]\n" "cmp x27, #0x10\n" "add x26, x26, #0x10\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[0]\n" - "sdot z12.s, z6.b, z1.b[0]\n" - "sdot z16.s, z6.b, z2.b[0]\n" - "sdot z20.s, z6.b, z3.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z25.b }, p5/Z, [x10]\n" + "ld1b { z24.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z8.s, z25.b, z3.b[0]\n" + "sdot z12.s, z25.b, z2.b[0]\n" + "sdot z16.s, z25.b, z1.b[0]\n" + "sdot z20.s, z25.b, z0.b[0]\n" + "ld1b { z25.b }, p5/Z, [x10, #2, MUL VL]\n" "add x25, x25, #0x10\n" - "sdot z9.s, z7.b, z0.b[0]\n" - "sdot z13.s, z7.b, z1.b[0]\n" + "sdot z9.s, z24.b, z3.b[0]\n" + "sdot z13.s, z24.b, z2.b[0]\n" "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" - "sdot z17.s, z7.b, z2.b[0]\n" - "sdot z21.s, z7.b, z3.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[0]\n" - "sdot z14.s, z6.b, z1.b[0]\n" - "sdot z18.s, z6.b, z2.b[0]\n" - "sdot z22.s, z6.b, z3.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #4, MUL VL]\n" - "sdot z11.s, z7.b, z0.b[0]\n" - "sdot z15.s, z7.b, z1.b[0]\n" - "sdot z19.s, z7.b, z2.b[0]\n" - "sdot z23.s, z7.b, z3.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #5, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[1]\n" - "sdot z12.s, z6.b, z1.b[1]\n" - "sdot z16.s, z6.b, z2.b[1]\n" - "sdot z20.s, z6.b, z3.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #6, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[1]\n" - "sdot z13.s, z7.b, z1.b[1]\n" - "sdot z17.s, z7.b, z2.b[1]\n" - "sdot z21.s, z7.b, z3.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #7, MUL VL]\n" + "sdot z17.s, z24.b, z1.b[0]\n" + "sdot z21.s, z24.b, z0.b[0]\n" + "ld1b { z24.b }, p5/Z, [x10, #3, MUL VL]\n" + "sdot z10.s, z25.b, z3.b[0]\n" + "sdot z14.s, z25.b, z2.b[0]\n" + "sdot z18.s, z25.b, z1.b[0]\n" + "sdot z22.s, z25.b, z0.b[0]\n" + "ld1b { z25.b }, p5/Z, [x10, #4, MUL VL]\n" + "sdot z11.s, z24.b, z3.b[0]\n" + "sdot z15.s, z24.b, z2.b[0]\n" + "sdot z19.s, z24.b, z1.b[0]\n" + "sdot z23.s, z24.b, z0.b[0]\n" + "ld1b { z24.b }, p5/Z, [x10, #5, MUL VL]\n" + "sdot z8.s, z25.b, z3.b[1]\n" + "sdot z12.s, z25.b, z2.b[1]\n" + "sdot z16.s, z25.b, z1.b[1]\n" + "sdot z20.s, z25.b, z0.b[1]\n" + "ld1b { z25.b }, p5/Z, [x10, #6, MUL VL]\n" + "sdot z9.s, z24.b, z3.b[1]\n" + "sdot z13.s, z24.b, z2.b[1]\n" + "sdot z17.s, z24.b, z1.b[1]\n" + "sdot z21.s, z24.b, z0.b[1]\n" + "ld1b { z24.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "sdot z10.s, z6.b, z0.b[1]\n" - "sdot z14.s, z6.b, z1.b[1]\n" - "sdot z18.s, z6.b, z2.b[1]\n" - "sdot z22.s, z6.b, z3.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #-8, MUL VL]\n" - "sdot z11.s, z7.b, z0.b[1]\n" - "sdot z15.s, z7.b, z1.b[1]\n" - "sdot z19.s, z7.b, z2.b[1]\n" - "sdot z23.s, z7.b, z3.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #-7, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[2]\n" - "sdot z12.s, z6.b, z1.b[2]\n" - "sdot z16.s, z6.b, z2.b[2]\n" - "sdot z20.s, z6.b, z3.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #-6, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[2]\n" - "sdot z13.s, z7.b, z1.b[2]\n" - "sdot z17.s, z7.b, z2.b[2]\n" - "sdot z21.s, z7.b, z3.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #-5, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[2]\n" - "sdot z14.s, z6.b, z1.b[2]\n" - "sdot z18.s, z6.b, z2.b[2]\n" - "sdot z22.s, z6.b, z3.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #-4, MUL VL]\n" - "sdot z11.s, z7.b, z0.b[2]\n" - "sdot z15.s, z7.b, z1.b[2]\n" - "sdot z19.s, z7.b, z2.b[2]\n" - "sdot z23.s, z7.b, z3.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[3]\n" - "sdot z12.s, z6.b, z1.b[3]\n" - "sdot z16.s, z6.b, z2.b[3]\n" - "sdot z20.s, z6.b, z3.b[3]\n" - "ld1b { z6.b }, p5/Z, [x10, #-2, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[3]\n" - "sdot z13.s, z7.b, z1.b[3]\n" - "sdot z17.s, z7.b, z2.b[3]\n" - "sdot z21.s, z7.b, z3.b[3]\n" - "ld1b { z7.b }, p5/Z, [x10, #-1, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[3]\n" - "sdot z14.s, z6.b, z1.b[3]\n" - "sdot z18.s, z6.b, z2.b[3]\n" - "sdot z22.s, z6.b, z3.b[3]\n" - "sdot z11.s, z7.b, z0.b[3]\n" - "sdot z15.s, z7.b, z1.b[3]\n" - "sdot z19.s, z7.b, z2.b[3]\n" - "sdot z23.s, z7.b, z3.b[3]\n" + "sdot z10.s, z25.b, z3.b[1]\n" + "sdot z14.s, z25.b, z2.b[1]\n" + "sdot z18.s, z25.b, z1.b[1]\n" + "sdot z22.s, z25.b, z0.b[1]\n" + "ld1b { z25.b }, p5/Z, [x10, #-8, MUL VL]\n" + "sdot z11.s, z24.b, z3.b[1]\n" + "sdot z15.s, z24.b, z2.b[1]\n" + "sdot z19.s, z24.b, z1.b[1]\n" + "sdot z23.s, z24.b, z0.b[1]\n" + "ld1b { z24.b }, p5/Z, [x10, #-7, MUL VL]\n" + "sdot z8.s, z25.b, z3.b[2]\n" + "sdot z12.s, z25.b, z2.b[2]\n" + "sdot z16.s, z25.b, z1.b[2]\n" + "sdot z20.s, z25.b, z0.b[2]\n" + "ld1b { z25.b }, p5/Z, [x10, #-6, MUL VL]\n" + "sdot z9.s, z24.b, z3.b[2]\n" + "sdot z13.s, z24.b, z2.b[2]\n" + "sdot z17.s, z24.b, z1.b[2]\n" + "sdot z21.s, z24.b, z0.b[2]\n" + "ld1b { z24.b }, p5/Z, [x10, #-5, MUL VL]\n" + "sdot z10.s, z25.b, z3.b[2]\n" + "sdot z14.s, z25.b, z2.b[2]\n" + "sdot z18.s, z25.b, z1.b[2]\n" + "sdot z22.s, z25.b, z0.b[2]\n" + "ld1b { z25.b }, p5/Z, [x10, #-4, MUL VL]\n" + "sdot z11.s, z24.b, z3.b[2]\n" + "sdot z15.s, z24.b, z2.b[2]\n" + "sdot z19.s, z24.b, z1.b[2]\n" + "sdot z23.s, z24.b, z0.b[2]\n" + "ld1b { z24.b }, p5/Z, [x10, #-3, MUL VL]\n" + "sdot z8.s, z25.b, z3.b[3]\n" + "sdot z12.s, z25.b, z2.b[3]\n" + "sdot z16.s, z25.b, z1.b[3]\n" + "sdot z20.s, z25.b, z0.b[3]\n" + "ld1b { z25.b }, p5/Z, [x10, #-2, MUL VL]\n" + "sdot z9.s, z24.b, z3.b[3]\n" + "sdot z13.s, z24.b, z2.b[3]\n" + "sdot z17.s, z24.b, z1.b[3]\n" + "sdot z21.s, z24.b, z0.b[3]\n" + "ld1b { z24.b }, p5/Z, [x10, #-1, MUL VL]\n" + "sdot z10.s, z25.b, z3.b[3]\n" + "sdot z14.s, z25.b, z2.b[3]\n" + "sdot z18.s, z25.b, z1.b[3]\n" + "sdot z22.s, z25.b, z0.b[3]\n" + "sdot z11.s, z24.b, z3.b[3]\n" + "sdot z15.s, z24.b, z2.b[3]\n" + "sdot z19.s, z24.b, z1.b[3]\n" + "sdot z23.s, z24.b, z0.b[3]\n" "bgt 41b\n" "42:" // Height 4: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" @@ -856,121 +856,121 @@ void sve_hybrid_s8s32_dot_6x4VL ( "subs x27, x27, #0x4\n" "ld1rqb { z2.b }, p0/Z, [x24]\n" "ld1rqb { z3.b }, p0/Z, [x23]\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[0]\n" - "sdot z12.s, z6.b, z1.b[0]\n" - "sdot z16.s, z6.b, z2.b[0]\n" - "sdot z20.s, z6.b, z3.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[0]\n" - "sdot z13.s, z7.b, z1.b[0]\n" - "sdot z17.s, z7.b, z2.b[0]\n" - "sdot z21.s, z7.b, z3.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1b { z25.b }, p5/Z, [x10]\n" + "ld1b { z24.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z8.s, z25.b, z0.b[0]\n" + "sdot z12.s, z25.b, z1.b[0]\n" + "sdot z16.s, z25.b, z2.b[0]\n" + "sdot z20.s, z25.b, z3.b[0]\n" + "ld1b { z25.b }, p5/Z, [x10, #2, MUL VL]\n" + "sdot z9.s, z24.b, z0.b[0]\n" + "sdot z13.s, z24.b, z1.b[0]\n" + "sdot z17.s, z24.b, z2.b[0]\n" + "sdot z21.s, z24.b, z3.b[0]\n" + "ld1b { z24.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "sdot z10.s, z6.b, z0.b[0]\n" - "sdot z14.s, z6.b, z1.b[0]\n" - "sdot z18.s, z6.b, z2.b[0]\n" - "sdot z22.s, z6.b, z3.b[0]\n" - "sdot z11.s, z7.b, z0.b[0]\n" - "sdot z15.s, z7.b, z1.b[0]\n" - "sdot z19.s, z7.b, z2.b[0]\n" - "sdot z23.s, z7.b, z3.b[0]\n" + "sdot z10.s, z25.b, z0.b[0]\n" + "sdot z14.s, z25.b, z1.b[0]\n" + "sdot z18.s, z25.b, z2.b[0]\n" + "sdot z22.s, z25.b, z3.b[0]\n" + "sdot z11.s, z24.b, z0.b[0]\n" + "sdot z15.s, z24.b, z1.b[0]\n" + "sdot z19.s, z24.b, z2.b[0]\n" + "sdot z23.s, z24.b, z3.b[0]\n" "ble 43f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[1]\n" - "sdot z12.s, z6.b, z1.b[1]\n" - "sdot z16.s, z6.b, z2.b[1]\n" - "sdot z20.s, z6.b, z3.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z25.b }, p5/Z, [x10]\n" + "ld1b { z24.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z8.s, z25.b, z0.b[1]\n" + "sdot z12.s, z25.b, z1.b[1]\n" + "sdot z16.s, z25.b, z2.b[1]\n" + "sdot z20.s, z25.b, z3.b[1]\n" + "ld1b { z25.b }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x4\n" - "sdot z9.s, z7.b, z0.b[1]\n" - "sdot z13.s, z7.b, z1.b[1]\n" - "sdot z17.s, z7.b, z2.b[1]\n" - "sdot z21.s, z7.b, z3.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "sdot z9.s, z24.b, z0.b[1]\n" + "sdot z13.s, z24.b, z1.b[1]\n" + "sdot z17.s, z24.b, z2.b[1]\n" + "sdot z21.s, z24.b, z3.b[1]\n" + "ld1b { z24.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "sdot z10.s, z6.b, z0.b[1]\n" - "sdot z14.s, z6.b, z1.b[1]\n" - "sdot z18.s, z6.b, z2.b[1]\n" - "sdot z22.s, z6.b, z3.b[1]\n" - "sdot z11.s, z7.b, z0.b[1]\n" - "sdot z15.s, z7.b, z1.b[1]\n" - "sdot z19.s, z7.b, z2.b[1]\n" - "sdot z23.s, z7.b, z3.b[1]\n" + "sdot z10.s, z25.b, z0.b[1]\n" + "sdot z14.s, z25.b, z1.b[1]\n" + "sdot z18.s, z25.b, z2.b[1]\n" + "sdot z22.s, z25.b, z3.b[1]\n" + "sdot z11.s, z24.b, z0.b[1]\n" + "sdot z15.s, z24.b, z1.b[1]\n" + "sdot z19.s, z24.b, z2.b[1]\n" + "sdot z23.s, z24.b, z3.b[1]\n" "ble 43f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[2]\n" - "sdot z12.s, z6.b, z1.b[2]\n" - "sdot z16.s, z6.b, z2.b[2]\n" - "sdot z20.s, z6.b, z3.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z25.b }, p5/Z, [x10]\n" + "ld1b { z24.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z8.s, z25.b, z0.b[2]\n" + "sdot z12.s, z25.b, z1.b[2]\n" + "sdot z16.s, z25.b, z2.b[2]\n" + "sdot z20.s, z25.b, z3.b[2]\n" + "ld1b { z25.b }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x4\n" - "sdot z9.s, z7.b, z0.b[2]\n" - "sdot z13.s, z7.b, z1.b[2]\n" - "sdot z17.s, z7.b, z2.b[2]\n" - "sdot z21.s, z7.b, z3.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "sdot z9.s, z24.b, z0.b[2]\n" + "sdot z13.s, z24.b, z1.b[2]\n" + "sdot z17.s, z24.b, z2.b[2]\n" + "sdot z21.s, z24.b, z3.b[2]\n" + "ld1b { z24.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "sdot z10.s, z6.b, z0.b[2]\n" - "sdot z14.s, z6.b, z1.b[2]\n" - "sdot z18.s, z6.b, z2.b[2]\n" - "sdot z22.s, z6.b, z3.b[2]\n" - "sdot z11.s, z7.b, z0.b[2]\n" - "sdot z15.s, z7.b, z1.b[2]\n" - "sdot z19.s, z7.b, z2.b[2]\n" - "sdot z23.s, z7.b, z3.b[2]\n" + "sdot z10.s, z25.b, z0.b[2]\n" + "sdot z14.s, z25.b, z1.b[2]\n" + "sdot z18.s, z25.b, z2.b[2]\n" + "sdot z22.s, z25.b, z3.b[2]\n" + "sdot z11.s, z24.b, z0.b[2]\n" + "sdot z15.s, z24.b, z1.b[2]\n" + "sdot z19.s, z24.b, z2.b[2]\n" + "sdot z23.s, z24.b, z3.b[2]\n" "ble 43f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[3]\n" - "sdot z12.s, z6.b, z1.b[3]\n" - "sdot z16.s, z6.b, z2.b[3]\n" - "sdot z20.s, z6.b, z3.b[3]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[3]\n" - "sdot z13.s, z7.b, z1.b[3]\n" - "sdot z17.s, z7.b, z2.b[3]\n" - "sdot z21.s, z7.b, z3.b[3]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1b { z25.b }, p5/Z, [x10]\n" + "ld1b { z24.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z8.s, z25.b, z0.b[3]\n" + "sdot z12.s, z25.b, z1.b[3]\n" + "sdot z16.s, z25.b, z2.b[3]\n" + "sdot z20.s, z25.b, z3.b[3]\n" + "ld1b { z25.b }, p5/Z, [x10, #2, MUL VL]\n" + "sdot z9.s, z24.b, z0.b[3]\n" + "sdot z13.s, z24.b, z1.b[3]\n" + "sdot z17.s, z24.b, z2.b[3]\n" + "sdot z21.s, z24.b, z3.b[3]\n" + "ld1b { z24.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "sdot z10.s, z6.b, z0.b[3]\n" - "sdot z14.s, z6.b, z1.b[3]\n" - "sdot z18.s, z6.b, z2.b[3]\n" - "sdot z22.s, z6.b, z3.b[3]\n" - "sdot z11.s, z7.b, z0.b[3]\n" - "sdot z15.s, z7.b, z1.b[3]\n" - "sdot z19.s, z7.b, z2.b[3]\n" - "sdot z23.s, z7.b, z3.b[3]\n" + "sdot z10.s, z25.b, z0.b[3]\n" + "sdot z14.s, z25.b, z1.b[3]\n" + "sdot z18.s, z25.b, z2.b[3]\n" + "sdot z22.s, z25.b, z3.b[3]\n" + "sdot z11.s, z24.b, z0.b[3]\n" + "sdot z15.s, z24.b, z1.b[3]\n" + "sdot z19.s, z24.b, z2.b[3]\n" + "sdot z23.s, z24.b, z3.b[3]\n" "43:" // Height 4: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 38b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x22, x9, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" "st1w { z8.s }, p4, [x9]\n" - "add x22, x23, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "st1w { z9.s }, p3, [x9, #1, MUL VL]\n" "st1w { z10.s }, p2, [x9, #2, MUL VL]\n" "st1w { z11.s }, p1, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "st1w { z12.s }, p4, [x24]\n" - "st1w { z13.s }, p3, [x24, #1, MUL VL]\n" - "st1w { z14.s }, p2, [x24, #2, MUL VL]\n" - "st1w { z15.s }, p1, [x24, #3, MUL VL]\n" - "st1w { z16.s }, p4, [x23]\n" - "st1w { z17.s }, p3, [x23, #1, MUL VL]\n" - "st1w { z18.s }, p2, [x23, #2, MUL VL]\n" - "st1w { z19.s }, p1, [x23, #3, MUL VL]\n" - "st1w { z20.s }, p4, [x22]\n" - "st1w { z21.s }, p3, [x22, #1, MUL VL]\n" - "st1w { z22.s }, p2, [x22, #2, MUL VL]\n" - "st1w { z23.s }, p1, [x22, #3, MUL VL]\n" + "st1w { z12.s }, p4, [x22]\n" + "st1w { z13.s }, p3, [x22, #1, MUL VL]\n" + "st1w { z14.s }, p2, [x22, #2, MUL VL]\n" + "st1w { z15.s }, p1, [x22, #3, MUL VL]\n" + "st1w { z16.s }, p4, [x21]\n" + "st1w { z17.s }, p3, [x21, #1, MUL VL]\n" + "st1w { z18.s }, p2, [x21, #2, MUL VL]\n" + "st1w { z19.s }, p1, [x21, #3, MUL VL]\n" + "st1w { z20.s }, p4, [x20]\n" + "st1w { z21.s }, p3, [x20, #1, MUL VL]\n" + "st1w { z22.s }, p2, [x20, #2, MUL VL]\n" + "st1w { z23.s }, p1, [x20, #3, MUL VL]\n" "44:" // Height 4: Writeback done "decw x11, ALL, MUL #4\n" "cmp x11, XZR\n" @@ -991,30 +991,30 @@ void sve_hybrid_s8s32_dot_6x4VL ( "whilelt p1.s, x20, x11\n" "tbz %x[flags], #0, 47f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "ld1w { z8.s }, p4/Z, [x9]\n" + "add x23, x9, x20, LSL #2\n" "add x22, x23, x20, LSL #2\n" + "ld1w { z8.s }, p4/Z, [x9]\n" "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x24]\n" - "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n" - "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z16.s }, p4/Z, [x23]\n" - "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n" - "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x22]\n" - "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n" - "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n" - "ld1w { z24.s }, p4/Z, [x21]\n" - "ld1w { z25.s }, p3/Z, [x21, #1, MUL VL]\n" - "ld1w { z26.s }, p2/Z, [x21, #2, MUL VL]\n" - "ld1w { z27.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x23]\n" + "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n" + "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x22]\n" + "ld1w { z17.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x22, #2, MUL VL]\n" + "ld1w { z19.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x21]\n" + "ld1w { z21.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x21, #2, MUL VL]\n" + "ld1w { z23.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z24.s }, p4/Z, [x20]\n" + "ld1w { z25.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z26.s }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z27.s }, p1/Z, [x20, #3, MUL VL]\n" "b 48f\n" "47:" // Height 5: no accumulate "mov z8.s, #0x0\n" @@ -1042,15 +1042,15 @@ void sve_hybrid_s8s32_dot_6x4VL ( "49:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 50f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 51f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -1061,124 +1061,124 @@ void sve_hybrid_s8s32_dot_6x4VL ( "b 51f\n" "50:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" "51:" // Height 5: input setup done "cmp x27, #0x10\n" "ble 53f\n" "52:" // Height 5: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z0.b }, p0/Z, [x26]\n" - "ld1rqb { z1.b }, p0/Z, [x25]\n" + "ld1rqb { z4.b }, p0/Z, [x26]\n" + "ld1rqb { z3.b }, p0/Z, [x25]\n" "sub x27, x27, #0x10\n" "ld1rqb { z2.b }, p0/Z, [x24]\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" + "ld1rqb { z1.b }, p0/Z, [x23]\n" "cmp x27, #0x10\n" "add x26, x26, #0x10\n" - "ld1rqb { z4.b }, p0/Z, [x22]\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "sdot z8.s, z6.b, z0.b[0]\n" - "sdot z12.s, z6.b, z1.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z16.s, z6.b, z2.b[0]\n" - "sdot z20.s, z6.b, z3.b[0]\n" + "ld1rqb { z0.b }, p0/Z, [x22]\n" + "ld1b { z29.b }, p5/Z, [x10]\n" + "sdot z8.s, z29.b, z4.b[0]\n" + "sdot z12.s, z29.b, z3.b[0]\n" + "ld1b { z28.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z16.s, z29.b, z2.b[0]\n" + "sdot z20.s, z29.b, z1.b[0]\n" "add x25, x25, #0x10\n" - "sdot z24.s, z6.b, z4.b[0]\n" - "sdot z9.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" + "sdot z24.s, z29.b, z0.b[0]\n" + "sdot z9.s, z28.b, z4.b[0]\n" + "ld1b { z29.b }, p5/Z, [x10, #2, MUL VL]\n" "add x24, x24, #0x10\n" - "sdot z13.s, z7.b, z1.b[0]\n" - "sdot z17.s, z7.b, z2.b[0]\n" + "sdot z13.s, z28.b, z3.b[0]\n" + "sdot z17.s, z28.b, z2.b[0]\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - "sdot z21.s, z7.b, z3.b[0]\n" - "sdot z25.s, z7.b, z4.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[0]\n" - "sdot z14.s, z6.b, z1.b[0]\n" - "sdot z18.s, z6.b, z2.b[0]\n" - "sdot z22.s, z6.b, z3.b[0]\n" - "sdot z26.s, z6.b, z4.b[0]\n" - "sdot z11.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #4, MUL VL]\n" - "sdot z15.s, z7.b, z1.b[0]\n" - "sdot z19.s, z7.b, z2.b[0]\n" - "sdot z23.s, z7.b, z3.b[0]\n" - "sdot z27.s, z7.b, z4.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #5, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[1]\n" - "sdot z12.s, z6.b, z1.b[1]\n" - "sdot z16.s, z6.b, z2.b[1]\n" - "sdot z20.s, z6.b, z3.b[1]\n" - "sdot z24.s, z6.b, z4.b[1]\n" - "sdot z9.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #6, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[1]\n" - "sdot z17.s, z7.b, z2.b[1]\n" - "sdot z21.s, z7.b, z3.b[1]\n" - "sdot z25.s, z7.b, z4.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #7, MUL VL]\n" + "sdot z21.s, z28.b, z1.b[0]\n" + "sdot z25.s, z28.b, z0.b[0]\n" + "ld1b { z28.b }, p5/Z, [x10, #3, MUL VL]\n" + "sdot z10.s, z29.b, z4.b[0]\n" + "sdot z14.s, z29.b, z3.b[0]\n" + "sdot z18.s, z29.b, z2.b[0]\n" + "sdot z22.s, z29.b, z1.b[0]\n" + "sdot z26.s, z29.b, z0.b[0]\n" + "sdot z11.s, z28.b, z4.b[0]\n" + "ld1b { z29.b }, p5/Z, [x10, #4, MUL VL]\n" + "sdot z15.s, z28.b, z3.b[0]\n" + "sdot z19.s, z28.b, z2.b[0]\n" + "sdot z23.s, z28.b, z1.b[0]\n" + "sdot z27.s, z28.b, z0.b[0]\n" + "ld1b { z28.b }, p5/Z, [x10, #5, MUL VL]\n" + "sdot z8.s, z29.b, z4.b[1]\n" + "sdot z12.s, z29.b, z3.b[1]\n" + "sdot z16.s, z29.b, z2.b[1]\n" + "sdot z20.s, z29.b, z1.b[1]\n" + "sdot z24.s, z29.b, z0.b[1]\n" + "sdot z9.s, z28.b, z4.b[1]\n" + "ld1b { z29.b }, p5/Z, [x10, #6, MUL VL]\n" + "sdot z13.s, z28.b, z3.b[1]\n" + "sdot z17.s, z28.b, z2.b[1]\n" + "sdot z21.s, z28.b, z1.b[1]\n" + "sdot z25.s, z28.b, z0.b[1]\n" + "ld1b { z28.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "sdot z10.s, z6.b, z0.b[1]\n" - "sdot z14.s, z6.b, z1.b[1]\n" - "sdot z18.s, z6.b, z2.b[1]\n" - "sdot z22.s, z6.b, z3.b[1]\n" - "sdot z26.s, z6.b, z4.b[1]\n" - "sdot z11.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #-8, MUL VL]\n" - "sdot z15.s, z7.b, z1.b[1]\n" - "sdot z19.s, z7.b, z2.b[1]\n" - "sdot z23.s, z7.b, z3.b[1]\n" - "sdot z27.s, z7.b, z4.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #-7, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[2]\n" - "sdot z12.s, z6.b, z1.b[2]\n" - "sdot z16.s, z6.b, z2.b[2]\n" - "sdot z20.s, z6.b, z3.b[2]\n" - "sdot z24.s, z6.b, z4.b[2]\n" - "sdot z9.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #-6, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[2]\n" - "sdot z17.s, z7.b, z2.b[2]\n" - "sdot z21.s, z7.b, z3.b[2]\n" - "sdot z25.s, z7.b, z4.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #-5, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[2]\n" - "sdot z14.s, z6.b, z1.b[2]\n" - "sdot z18.s, z6.b, z2.b[2]\n" - "sdot z22.s, z6.b, z3.b[2]\n" - "sdot z26.s, z6.b, z4.b[2]\n" - "sdot z11.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #-4, MUL VL]\n" - "sdot z15.s, z7.b, z1.b[2]\n" - "sdot z19.s, z7.b, z2.b[2]\n" - "sdot z23.s, z7.b, z3.b[2]\n" - "sdot z27.s, z7.b, z4.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[3]\n" - "sdot z12.s, z6.b, z1.b[3]\n" - "sdot z16.s, z6.b, z2.b[3]\n" - "sdot z20.s, z6.b, z3.b[3]\n" - "sdot z24.s, z6.b, z4.b[3]\n" - "sdot z9.s, z7.b, z0.b[3]\n" - "ld1b { z6.b }, p5/Z, [x10, #-2, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[3]\n" - "sdot z17.s, z7.b, z2.b[3]\n" - "sdot z21.s, z7.b, z3.b[3]\n" - "sdot z25.s, z7.b, z4.b[3]\n" - "ld1b { z7.b }, p5/Z, [x10, #-1, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[3]\n" - "sdot z14.s, z6.b, z1.b[3]\n" - "sdot z18.s, z6.b, z2.b[3]\n" - "sdot z22.s, z6.b, z3.b[3]\n" - "sdot z26.s, z6.b, z4.b[3]\n" - "sdot z11.s, z7.b, z0.b[3]\n" - "sdot z15.s, z7.b, z1.b[3]\n" - "sdot z19.s, z7.b, z2.b[3]\n" - "sdot z23.s, z7.b, z3.b[3]\n" - "sdot z27.s, z7.b, z4.b[3]\n" + "sdot z10.s, z29.b, z4.b[1]\n" + "sdot z14.s, z29.b, z3.b[1]\n" + "sdot z18.s, z29.b, z2.b[1]\n" + "sdot z22.s, z29.b, z1.b[1]\n" + "sdot z26.s, z29.b, z0.b[1]\n" + "sdot z11.s, z28.b, z4.b[1]\n" + "ld1b { z29.b }, p5/Z, [x10, #-8, MUL VL]\n" + "sdot z15.s, z28.b, z3.b[1]\n" + "sdot z19.s, z28.b, z2.b[1]\n" + "sdot z23.s, z28.b, z1.b[1]\n" + "sdot z27.s, z28.b, z0.b[1]\n" + "ld1b { z28.b }, p5/Z, [x10, #-7, MUL VL]\n" + "sdot z8.s, z29.b, z4.b[2]\n" + "sdot z12.s, z29.b, z3.b[2]\n" + "sdot z16.s, z29.b, z2.b[2]\n" + "sdot z20.s, z29.b, z1.b[2]\n" + "sdot z24.s, z29.b, z0.b[2]\n" + "sdot z9.s, z28.b, z4.b[2]\n" + "ld1b { z29.b }, p5/Z, [x10, #-6, MUL VL]\n" + "sdot z13.s, z28.b, z3.b[2]\n" + "sdot z17.s, z28.b, z2.b[2]\n" + "sdot z21.s, z28.b, z1.b[2]\n" + "sdot z25.s, z28.b, z0.b[2]\n" + "ld1b { z28.b }, p5/Z, [x10, #-5, MUL VL]\n" + "sdot z10.s, z29.b, z4.b[2]\n" + "sdot z14.s, z29.b, z3.b[2]\n" + "sdot z18.s, z29.b, z2.b[2]\n" + "sdot z22.s, z29.b, z1.b[2]\n" + "sdot z26.s, z29.b, z0.b[2]\n" + "sdot z11.s, z28.b, z4.b[2]\n" + "ld1b { z29.b }, p5/Z, [x10, #-4, MUL VL]\n" + "sdot z15.s, z28.b, z3.b[2]\n" + "sdot z19.s, z28.b, z2.b[2]\n" + "sdot z23.s, z28.b, z1.b[2]\n" + "sdot z27.s, z28.b, z0.b[2]\n" + "ld1b { z28.b }, p5/Z, [x10, #-3, MUL VL]\n" + "sdot z8.s, z29.b, z4.b[3]\n" + "sdot z12.s, z29.b, z3.b[3]\n" + "sdot z16.s, z29.b, z2.b[3]\n" + "sdot z20.s, z29.b, z1.b[3]\n" + "sdot z24.s, z29.b, z0.b[3]\n" + "sdot z9.s, z28.b, z4.b[3]\n" + "ld1b { z29.b }, p5/Z, [x10, #-2, MUL VL]\n" + "sdot z13.s, z28.b, z3.b[3]\n" + "sdot z17.s, z28.b, z2.b[3]\n" + "sdot z21.s, z28.b, z1.b[3]\n" + "sdot z25.s, z28.b, z0.b[3]\n" + "ld1b { z28.b }, p5/Z, [x10, #-1, MUL VL]\n" + "sdot z10.s, z29.b, z4.b[3]\n" + "sdot z14.s, z29.b, z3.b[3]\n" + "sdot z18.s, z29.b, z2.b[3]\n" + "sdot z22.s, z29.b, z1.b[3]\n" + "sdot z26.s, z29.b, z0.b[3]\n" + "sdot z11.s, z28.b, z4.b[3]\n" + "sdot z15.s, z28.b, z3.b[3]\n" + "sdot z19.s, z28.b, z2.b[3]\n" + "sdot z23.s, z28.b, z1.b[3]\n" + "sdot z27.s, z28.b, z0.b[3]\n" "bgt 52b\n" "53:" // Height 5: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" @@ -1188,142 +1188,142 @@ void sve_hybrid_s8s32_dot_6x4VL ( "ld1rqb { z2.b }, p0/Z, [x24]\n" "ld1rqb { z3.b }, p0/Z, [x23]\n" "ld1rqb { z4.b }, p0/Z, [x22]\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "sdot z8.s, z6.b, z0.b[0]\n" - "sdot z12.s, z6.b, z1.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z16.s, z6.b, z2.b[0]\n" - "sdot z20.s, z6.b, z3.b[0]\n" - "sdot z24.s, z6.b, z4.b[0]\n" - "sdot z9.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[0]\n" - "sdot z17.s, z7.b, z2.b[0]\n" - "sdot z21.s, z7.b, z3.b[0]\n" - "sdot z25.s, z7.b, z4.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1b { z29.b }, p5/Z, [x10]\n" + "sdot z8.s, z29.b, z0.b[0]\n" + "sdot z12.s, z29.b, z1.b[0]\n" + "ld1b { z28.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z16.s, z29.b, z2.b[0]\n" + "sdot z20.s, z29.b, z3.b[0]\n" + "sdot z24.s, z29.b, z4.b[0]\n" + "sdot z9.s, z28.b, z0.b[0]\n" + "ld1b { z29.b }, p5/Z, [x10, #2, MUL VL]\n" + "sdot z13.s, z28.b, z1.b[0]\n" + "sdot z17.s, z28.b, z2.b[0]\n" + "sdot z21.s, z28.b, z3.b[0]\n" + "sdot z25.s, z28.b, z4.b[0]\n" + "ld1b { z28.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "sdot z10.s, z6.b, z0.b[0]\n" - "sdot z14.s, z6.b, z1.b[0]\n" - "sdot z18.s, z6.b, z2.b[0]\n" - "sdot z22.s, z6.b, z3.b[0]\n" - "sdot z26.s, z6.b, z4.b[0]\n" - "sdot z11.s, z7.b, z0.b[0]\n" - "sdot z15.s, z7.b, z1.b[0]\n" - "sdot z19.s, z7.b, z2.b[0]\n" - "sdot z23.s, z7.b, z3.b[0]\n" - "sdot z27.s, z7.b, z4.b[0]\n" + "sdot z10.s, z29.b, z0.b[0]\n" + "sdot z14.s, z29.b, z1.b[0]\n" + "sdot z18.s, z29.b, z2.b[0]\n" + "sdot z22.s, z29.b, z3.b[0]\n" + "sdot z26.s, z29.b, z4.b[0]\n" + "sdot z11.s, z28.b, z0.b[0]\n" + "sdot z15.s, z28.b, z1.b[0]\n" + "sdot z19.s, z28.b, z2.b[0]\n" + "sdot z23.s, z28.b, z3.b[0]\n" + "sdot z27.s, z28.b, z4.b[0]\n" "ble 54f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[1]\n" - "sdot z12.s, z6.b, z1.b[1]\n" - "sdot z16.s, z6.b, z2.b[1]\n" - "sdot z20.s, z6.b, z3.b[1]\n" + "ld1b { z29.b }, p5/Z, [x10]\n" + "ld1b { z28.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z8.s, z29.b, z0.b[1]\n" + "sdot z12.s, z29.b, z1.b[1]\n" + "sdot z16.s, z29.b, z2.b[1]\n" + "sdot z20.s, z29.b, z3.b[1]\n" "subs x27, x27, #0x4\n" - "sdot z24.s, z6.b, z4.b[1]\n" - "sdot z9.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[1]\n" - "sdot z17.s, z7.b, z2.b[1]\n" - "sdot z21.s, z7.b, z3.b[1]\n" - "sdot z25.s, z7.b, z4.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "sdot z24.s, z29.b, z4.b[1]\n" + "sdot z9.s, z28.b, z0.b[1]\n" + "ld1b { z29.b }, p5/Z, [x10, #2, MUL VL]\n" + "sdot z13.s, z28.b, z1.b[1]\n" + "sdot z17.s, z28.b, z2.b[1]\n" + "sdot z21.s, z28.b, z3.b[1]\n" + "sdot z25.s, z28.b, z4.b[1]\n" + "ld1b { z28.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "sdot z10.s, z6.b, z0.b[1]\n" - "sdot z14.s, z6.b, z1.b[1]\n" - "sdot z18.s, z6.b, z2.b[1]\n" - "sdot z22.s, z6.b, z3.b[1]\n" - "sdot z26.s, z6.b, z4.b[1]\n" - "sdot z11.s, z7.b, z0.b[1]\n" - "sdot z15.s, z7.b, z1.b[1]\n" - "sdot z19.s, z7.b, z2.b[1]\n" - "sdot z23.s, z7.b, z3.b[1]\n" - "sdot z27.s, z7.b, z4.b[1]\n" + "sdot z10.s, z29.b, z0.b[1]\n" + "sdot z14.s, z29.b, z1.b[1]\n" + "sdot z18.s, z29.b, z2.b[1]\n" + "sdot z22.s, z29.b, z3.b[1]\n" + "sdot z26.s, z29.b, z4.b[1]\n" + "sdot z11.s, z28.b, z0.b[1]\n" + "sdot z15.s, z28.b, z1.b[1]\n" + "sdot z19.s, z28.b, z2.b[1]\n" + "sdot z23.s, z28.b, z3.b[1]\n" + "sdot z27.s, z28.b, z4.b[1]\n" "ble 54f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[2]\n" - "sdot z12.s, z6.b, z1.b[2]\n" - "sdot z16.s, z6.b, z2.b[2]\n" - "sdot z20.s, z6.b, z3.b[2]\n" + "ld1b { z29.b }, p5/Z, [x10]\n" + "ld1b { z28.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z8.s, z29.b, z0.b[2]\n" + "sdot z12.s, z29.b, z1.b[2]\n" + "sdot z16.s, z29.b, z2.b[2]\n" + "sdot z20.s, z29.b, z3.b[2]\n" "subs x27, x27, #0x4\n" - "sdot z24.s, z6.b, z4.b[2]\n" - "sdot z9.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[2]\n" - "sdot z17.s, z7.b, z2.b[2]\n" - "sdot z21.s, z7.b, z3.b[2]\n" - "sdot z25.s, z7.b, z4.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "sdot z24.s, z29.b, z4.b[2]\n" + "sdot z9.s, z28.b, z0.b[2]\n" + "ld1b { z29.b }, p5/Z, [x10, #2, MUL VL]\n" + "sdot z13.s, z28.b, z1.b[2]\n" + "sdot z17.s, z28.b, z2.b[2]\n" + "sdot z21.s, z28.b, z3.b[2]\n" + "sdot z25.s, z28.b, z4.b[2]\n" + "ld1b { z28.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "sdot z10.s, z6.b, z0.b[2]\n" - "sdot z14.s, z6.b, z1.b[2]\n" - "sdot z18.s, z6.b, z2.b[2]\n" - "sdot z22.s, z6.b, z3.b[2]\n" - "sdot z26.s, z6.b, z4.b[2]\n" - "sdot z11.s, z7.b, z0.b[2]\n" - "sdot z15.s, z7.b, z1.b[2]\n" - "sdot z19.s, z7.b, z2.b[2]\n" - "sdot z23.s, z7.b, z3.b[2]\n" - "sdot z27.s, z7.b, z4.b[2]\n" + "sdot z10.s, z29.b, z0.b[2]\n" + "sdot z14.s, z29.b, z1.b[2]\n" + "sdot z18.s, z29.b, z2.b[2]\n" + "sdot z22.s, z29.b, z3.b[2]\n" + "sdot z26.s, z29.b, z4.b[2]\n" + "sdot z11.s, z28.b, z0.b[2]\n" + "sdot z15.s, z28.b, z1.b[2]\n" + "sdot z19.s, z28.b, z2.b[2]\n" + "sdot z23.s, z28.b, z3.b[2]\n" + "sdot z27.s, z28.b, z4.b[2]\n" "ble 54f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[3]\n" - "sdot z12.s, z6.b, z1.b[3]\n" - "sdot z16.s, z6.b, z2.b[3]\n" - "sdot z20.s, z6.b, z3.b[3]\n" - "sdot z24.s, z6.b, z4.b[3]\n" - "sdot z9.s, z7.b, z0.b[3]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "sdot z13.s, z7.b, z1.b[3]\n" - "sdot z17.s, z7.b, z2.b[3]\n" - "sdot z21.s, z7.b, z3.b[3]\n" - "sdot z25.s, z7.b, z4.b[3]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1b { z29.b }, p5/Z, [x10]\n" + "ld1b { z28.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z8.s, z29.b, z0.b[3]\n" + "sdot z12.s, z29.b, z1.b[3]\n" + "sdot z16.s, z29.b, z2.b[3]\n" + "sdot z20.s, z29.b, z3.b[3]\n" + "sdot z24.s, z29.b, z4.b[3]\n" + "sdot z9.s, z28.b, z0.b[3]\n" + "ld1b { z29.b }, p5/Z, [x10, #2, MUL VL]\n" + "sdot z13.s, z28.b, z1.b[3]\n" + "sdot z17.s, z28.b, z2.b[3]\n" + "sdot z21.s, z28.b, z3.b[3]\n" + "sdot z25.s, z28.b, z4.b[3]\n" + "ld1b { z28.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "sdot z10.s, z6.b, z0.b[3]\n" - "sdot z14.s, z6.b, z1.b[3]\n" - "sdot z18.s, z6.b, z2.b[3]\n" - "sdot z22.s, z6.b, z3.b[3]\n" - "sdot z26.s, z6.b, z4.b[3]\n" - "sdot z11.s, z7.b, z0.b[3]\n" - "sdot z15.s, z7.b, z1.b[3]\n" - "sdot z19.s, z7.b, z2.b[3]\n" - "sdot z23.s, z7.b, z3.b[3]\n" - "sdot z27.s, z7.b, z4.b[3]\n" + "sdot z10.s, z29.b, z0.b[3]\n" + "sdot z14.s, z29.b, z1.b[3]\n" + "sdot z18.s, z29.b, z2.b[3]\n" + "sdot z22.s, z29.b, z3.b[3]\n" + "sdot z26.s, z29.b, z4.b[3]\n" + "sdot z11.s, z28.b, z0.b[3]\n" + "sdot z15.s, z28.b, z1.b[3]\n" + "sdot z19.s, z28.b, z2.b[3]\n" + "sdot z23.s, z28.b, z3.b[3]\n" + "sdot z27.s, z28.b, z4.b[3]\n" "54:" // Height 5: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 49b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "st1w { z8.s }, p4, [x9]\n" + "add x23, x9, x20, LSL #2\n" "add x22, x23, x20, LSL #2\n" + "st1w { z8.s }, p4, [x9]\n" "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "st1w { z9.s }, p3, [x9, #1, MUL VL]\n" "st1w { z10.s }, p2, [x9, #2, MUL VL]\n" "st1w { z11.s }, p1, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "st1w { z12.s }, p4, [x24]\n" - "st1w { z13.s }, p3, [x24, #1, MUL VL]\n" - "st1w { z14.s }, p2, [x24, #2, MUL VL]\n" - "st1w { z15.s }, p1, [x24, #3, MUL VL]\n" - "st1w { z16.s }, p4, [x23]\n" - "st1w { z17.s }, p3, [x23, #1, MUL VL]\n" - "st1w { z18.s }, p2, [x23, #2, MUL VL]\n" - "st1w { z19.s }, p1, [x23, #3, MUL VL]\n" - "st1w { z20.s }, p4, [x22]\n" - "st1w { z21.s }, p3, [x22, #1, MUL VL]\n" - "st1w { z22.s }, p2, [x22, #2, MUL VL]\n" - "st1w { z23.s }, p1, [x22, #3, MUL VL]\n" - "st1w { z24.s }, p4, [x21]\n" - "st1w { z25.s }, p3, [x21, #1, MUL VL]\n" - "st1w { z26.s }, p2, [x21, #2, MUL VL]\n" - "st1w { z27.s }, p1, [x21, #3, MUL VL]\n" + "st1w { z12.s }, p4, [x23]\n" + "st1w { z13.s }, p3, [x23, #1, MUL VL]\n" + "st1w { z14.s }, p2, [x23, #2, MUL VL]\n" + "st1w { z15.s }, p1, [x23, #3, MUL VL]\n" + "st1w { z16.s }, p4, [x22]\n" + "st1w { z17.s }, p3, [x22, #1, MUL VL]\n" + "st1w { z18.s }, p2, [x22, #2, MUL VL]\n" + "st1w { z19.s }, p1, [x22, #3, MUL VL]\n" + "st1w { z20.s }, p4, [x21]\n" + "st1w { z21.s }, p3, [x21, #1, MUL VL]\n" + "st1w { z22.s }, p2, [x21, #2, MUL VL]\n" + "st1w { z23.s }, p1, [x21, #3, MUL VL]\n" + "st1w { z24.s }, p4, [x20]\n" + "st1w { z25.s }, p3, [x20, #1, MUL VL]\n" + "st1w { z26.s }, p2, [x20, #2, MUL VL]\n" + "st1w { z27.s }, p1, [x20, #3, MUL VL]\n" "55:" // Height 5: Writeback done "decw x11, ALL, MUL #4\n" "cmp x11, XZR\n" @@ -1407,16 +1407,16 @@ void sve_hybrid_s8s32_dot_6x4VL ( "60:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 61f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 62f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -1428,143 +1428,143 @@ void sve_hybrid_s8s32_dot_6x4VL ( "b 62f\n" "61:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" + "add x21, x22, x21\n" "62:" // Height 6: input setup done "cmp x27, #0x10\n" "ble 64f\n" "63:" // Height 6: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z0.b }, p0/Z, [x26]\n" - "ld1rqb { z1.b }, p0/Z, [x25]\n" + "ld1rqb { z7.b }, p0/Z, [x26]\n" + "ld1rqb { z6.b }, p0/Z, [x25]\n" "sub x27, x27, #0x10\n" - "ld1rqb { z2.b }, p0/Z, [x24]\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" + "ld1rqb { z5.b }, p0/Z, [x24]\n" + "ld1rqb { z4.b }, p0/Z, [x23]\n" "cmp x27, #0x10\n" "add x26, x26, #0x10\n" - "ld1rqb { z4.b }, p0/Z, [x22]\n" - "ld1rqb { z5.b }, p0/Z, [x21]\n" + "ld1rqb { z3.b }, p0/Z, [x22]\n" + "ld1rqb { z2.b }, p0/Z, [x21]\n" "add x25, x25, #0x10\n" "add x24, x24, #0x10\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[0]\n" - "sdot z12.s, z6.b, z1.b[0]\n" - "sdot z16.s, z6.b, z2.b[0]\n" - "sdot z20.s, z6.b, z3.b[0]\n" + "ld1b { z1.b }, p5/Z, [x10]\n" + "ld1b { z0.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z8.s, z1.b, z7.b[0]\n" + "sdot z12.s, z1.b, z6.b[0]\n" + "sdot z16.s, z1.b, z5.b[0]\n" + "sdot z20.s, z1.b, z4.b[0]\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - "sdot z24.s, z6.b, z4.b[0]\n" - "sdot z28.s, z6.b, z5.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" + "sdot z24.s, z1.b, z3.b[0]\n" + "sdot z28.s, z1.b, z2.b[0]\n" + "ld1b { z1.b }, p5/Z, [x10, #2, MUL VL]\n" "add x21, x21, #0x10\n" - "sdot z9.s, z7.b, z0.b[0]\n" - "sdot z13.s, z7.b, z1.b[0]\n" - "sdot z17.s, z7.b, z2.b[0]\n" - "sdot z21.s, z7.b, z3.b[0]\n" - "sdot z25.s, z7.b, z4.b[0]\n" - "sdot z29.s, z7.b, z5.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[0]\n" - "sdot z14.s, z6.b, z1.b[0]\n" - "sdot z18.s, z6.b, z2.b[0]\n" - "sdot z22.s, z6.b, z3.b[0]\n" - "sdot z26.s, z6.b, z4.b[0]\n" - "sdot z30.s, z6.b, z5.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #4, MUL VL]\n" - "sdot z11.s, z7.b, z0.b[0]\n" - "sdot z15.s, z7.b, z1.b[0]\n" - "sdot z19.s, z7.b, z2.b[0]\n" - "sdot z23.s, z7.b, z3.b[0]\n" - "sdot z27.s, z7.b, z4.b[0]\n" - "sdot z31.s, z7.b, z5.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #5, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[1]\n" - "sdot z12.s, z6.b, z1.b[1]\n" - "sdot z16.s, z6.b, z2.b[1]\n" - "sdot z20.s, z6.b, z3.b[1]\n" - "sdot z24.s, z6.b, z4.b[1]\n" - "sdot z28.s, z6.b, z5.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #6, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[1]\n" - "sdot z13.s, z7.b, z1.b[1]\n" - "sdot z17.s, z7.b, z2.b[1]\n" - "sdot z21.s, z7.b, z3.b[1]\n" - "sdot z25.s, z7.b, z4.b[1]\n" - "sdot z29.s, z7.b, z5.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #7, MUL VL]\n" + "sdot z9.s, z0.b, z7.b[0]\n" + "sdot z13.s, z0.b, z6.b[0]\n" + "sdot z17.s, z0.b, z5.b[0]\n" + "sdot z21.s, z0.b, z4.b[0]\n" + "sdot z25.s, z0.b, z3.b[0]\n" + "sdot z29.s, z0.b, z2.b[0]\n" + "ld1b { z0.b }, p5/Z, [x10, #3, MUL VL]\n" + "sdot z10.s, z1.b, z7.b[0]\n" + "sdot z14.s, z1.b, z6.b[0]\n" + "sdot z18.s, z1.b, z5.b[0]\n" + "sdot z22.s, z1.b, z4.b[0]\n" + "sdot z26.s, z1.b, z3.b[0]\n" + "sdot z30.s, z1.b, z2.b[0]\n" + "ld1b { z1.b }, p5/Z, [x10, #4, MUL VL]\n" + "sdot z11.s, z0.b, z7.b[0]\n" + "sdot z15.s, z0.b, z6.b[0]\n" + "sdot z19.s, z0.b, z5.b[0]\n" + "sdot z23.s, z0.b, z4.b[0]\n" + "sdot z27.s, z0.b, z3.b[0]\n" + "sdot z31.s, z0.b, z2.b[0]\n" + "ld1b { z0.b }, p5/Z, [x10, #5, MUL VL]\n" + "sdot z8.s, z1.b, z7.b[1]\n" + "sdot z12.s, z1.b, z6.b[1]\n" + "sdot z16.s, z1.b, z5.b[1]\n" + "sdot z20.s, z1.b, z4.b[1]\n" + "sdot z24.s, z1.b, z3.b[1]\n" + "sdot z28.s, z1.b, z2.b[1]\n" + "ld1b { z1.b }, p5/Z, [x10, #6, MUL VL]\n" + "sdot z9.s, z0.b, z7.b[1]\n" + "sdot z13.s, z0.b, z6.b[1]\n" + "sdot z17.s, z0.b, z5.b[1]\n" + "sdot z21.s, z0.b, z4.b[1]\n" + "sdot z25.s, z0.b, z3.b[1]\n" + "sdot z29.s, z0.b, z2.b[1]\n" + "ld1b { z0.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "sdot z10.s, z6.b, z0.b[1]\n" - "sdot z14.s, z6.b, z1.b[1]\n" - "sdot z18.s, z6.b, z2.b[1]\n" - "sdot z22.s, z6.b, z3.b[1]\n" - "sdot z26.s, z6.b, z4.b[1]\n" - "sdot z30.s, z6.b, z5.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #-8, MUL VL]\n" - "sdot z11.s, z7.b, z0.b[1]\n" - "sdot z15.s, z7.b, z1.b[1]\n" - "sdot z19.s, z7.b, z2.b[1]\n" - "sdot z23.s, z7.b, z3.b[1]\n" - "sdot z27.s, z7.b, z4.b[1]\n" - "sdot z31.s, z7.b, z5.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #-7, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[2]\n" - "sdot z12.s, z6.b, z1.b[2]\n" - "sdot z16.s, z6.b, z2.b[2]\n" - "sdot z20.s, z6.b, z3.b[2]\n" - "sdot z24.s, z6.b, z4.b[2]\n" - "sdot z28.s, z6.b, z5.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #-6, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[2]\n" - "sdot z13.s, z7.b, z1.b[2]\n" - "sdot z17.s, z7.b, z2.b[2]\n" - "sdot z21.s, z7.b, z3.b[2]\n" - "sdot z25.s, z7.b, z4.b[2]\n" - "sdot z29.s, z7.b, z5.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #-5, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[2]\n" - "sdot z14.s, z6.b, z1.b[2]\n" - "sdot z18.s, z6.b, z2.b[2]\n" - "sdot z22.s, z6.b, z3.b[2]\n" - "sdot z26.s, z6.b, z4.b[2]\n" - "sdot z30.s, z6.b, z5.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #-4, MUL VL]\n" - "sdot z11.s, z7.b, z0.b[2]\n" - "sdot z15.s, z7.b, z1.b[2]\n" - "sdot z19.s, z7.b, z2.b[2]\n" - "sdot z23.s, z7.b, z3.b[2]\n" - "sdot z27.s, z7.b, z4.b[2]\n" - "sdot z31.s, z7.b, z5.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[3]\n" - "sdot z12.s, z6.b, z1.b[3]\n" - "sdot z16.s, z6.b, z2.b[3]\n" - "sdot z20.s, z6.b, z3.b[3]\n" - "sdot z24.s, z6.b, z4.b[3]\n" - "sdot z28.s, z6.b, z5.b[3]\n" - "ld1b { z6.b }, p5/Z, [x10, #-2, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[3]\n" - "sdot z13.s, z7.b, z1.b[3]\n" - "sdot z17.s, z7.b, z2.b[3]\n" - "sdot z21.s, z7.b, z3.b[3]\n" - "sdot z25.s, z7.b, z4.b[3]\n" - "sdot z29.s, z7.b, z5.b[3]\n" - "ld1b { z7.b }, p5/Z, [x10, #-1, MUL VL]\n" - "sdot z10.s, z6.b, z0.b[3]\n" - "sdot z14.s, z6.b, z1.b[3]\n" - "sdot z18.s, z6.b, z2.b[3]\n" - "sdot z22.s, z6.b, z3.b[3]\n" - "sdot z26.s, z6.b, z4.b[3]\n" - "sdot z30.s, z6.b, z5.b[3]\n" - "sdot z11.s, z7.b, z0.b[3]\n" - "sdot z15.s, z7.b, z1.b[3]\n" - "sdot z19.s, z7.b, z2.b[3]\n" - "sdot z23.s, z7.b, z3.b[3]\n" - "sdot z27.s, z7.b, z4.b[3]\n" - "sdot z31.s, z7.b, z5.b[3]\n" + "sdot z10.s, z1.b, z7.b[1]\n" + "sdot z14.s, z1.b, z6.b[1]\n" + "sdot z18.s, z1.b, z5.b[1]\n" + "sdot z22.s, z1.b, z4.b[1]\n" + "sdot z26.s, z1.b, z3.b[1]\n" + "sdot z30.s, z1.b, z2.b[1]\n" + "ld1b { z1.b }, p5/Z, [x10, #-8, MUL VL]\n" + "sdot z11.s, z0.b, z7.b[1]\n" + "sdot z15.s, z0.b, z6.b[1]\n" + "sdot z19.s, z0.b, z5.b[1]\n" + "sdot z23.s, z0.b, z4.b[1]\n" + "sdot z27.s, z0.b, z3.b[1]\n" + "sdot z31.s, z0.b, z2.b[1]\n" + "ld1b { z0.b }, p5/Z, [x10, #-7, MUL VL]\n" + "sdot z8.s, z1.b, z7.b[2]\n" + "sdot z12.s, z1.b, z6.b[2]\n" + "sdot z16.s, z1.b, z5.b[2]\n" + "sdot z20.s, z1.b, z4.b[2]\n" + "sdot z24.s, z1.b, z3.b[2]\n" + "sdot z28.s, z1.b, z2.b[2]\n" + "ld1b { z1.b }, p5/Z, [x10, #-6, MUL VL]\n" + "sdot z9.s, z0.b, z7.b[2]\n" + "sdot z13.s, z0.b, z6.b[2]\n" + "sdot z17.s, z0.b, z5.b[2]\n" + "sdot z21.s, z0.b, z4.b[2]\n" + "sdot z25.s, z0.b, z3.b[2]\n" + "sdot z29.s, z0.b, z2.b[2]\n" + "ld1b { z0.b }, p5/Z, [x10, #-5, MUL VL]\n" + "sdot z10.s, z1.b, z7.b[2]\n" + "sdot z14.s, z1.b, z6.b[2]\n" + "sdot z18.s, z1.b, z5.b[2]\n" + "sdot z22.s, z1.b, z4.b[2]\n" + "sdot z26.s, z1.b, z3.b[2]\n" + "sdot z30.s, z1.b, z2.b[2]\n" + "ld1b { z1.b }, p5/Z, [x10, #-4, MUL VL]\n" + "sdot z11.s, z0.b, z7.b[2]\n" + "sdot z15.s, z0.b, z6.b[2]\n" + "sdot z19.s, z0.b, z5.b[2]\n" + "sdot z23.s, z0.b, z4.b[2]\n" + "sdot z27.s, z0.b, z3.b[2]\n" + "sdot z31.s, z0.b, z2.b[2]\n" + "ld1b { z0.b }, p5/Z, [x10, #-3, MUL VL]\n" + "sdot z8.s, z1.b, z7.b[3]\n" + "sdot z12.s, z1.b, z6.b[3]\n" + "sdot z16.s, z1.b, z5.b[3]\n" + "sdot z20.s, z1.b, z4.b[3]\n" + "sdot z24.s, z1.b, z3.b[3]\n" + "sdot z28.s, z1.b, z2.b[3]\n" + "ld1b { z1.b }, p5/Z, [x10, #-2, MUL VL]\n" + "sdot z9.s, z0.b, z7.b[3]\n" + "sdot z13.s, z0.b, z6.b[3]\n" + "sdot z17.s, z0.b, z5.b[3]\n" + "sdot z21.s, z0.b, z4.b[3]\n" + "sdot z25.s, z0.b, z3.b[3]\n" + "sdot z29.s, z0.b, z2.b[3]\n" + "ld1b { z0.b }, p5/Z, [x10, #-1, MUL VL]\n" + "sdot z10.s, z1.b, z7.b[3]\n" + "sdot z14.s, z1.b, z6.b[3]\n" + "sdot z18.s, z1.b, z5.b[3]\n" + "sdot z22.s, z1.b, z4.b[3]\n" + "sdot z26.s, z1.b, z3.b[3]\n" + "sdot z30.s, z1.b, z2.b[3]\n" + "sdot z11.s, z0.b, z7.b[3]\n" + "sdot z15.s, z0.b, z6.b[3]\n" + "sdot z19.s, z0.b, z5.b[3]\n" + "sdot z23.s, z0.b, z4.b[3]\n" + "sdot z27.s, z0.b, z3.b[3]\n" + "sdot z31.s, z0.b, z2.b[3]\n" "bgt 63b\n" "64:" // Height 6: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" @@ -1575,127 +1575,127 @@ void sve_hybrid_s8s32_dot_6x4VL ( "ld1rqb { z3.b }, p0/Z, [x23]\n" "ld1rqb { z4.b }, p0/Z, [x22]\n" "ld1rqb { z5.b }, p0/Z, [x21]\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[0]\n" - "sdot z12.s, z6.b, z1.b[0]\n" - "sdot z16.s, z6.b, z2.b[0]\n" - "sdot z20.s, z6.b, z3.b[0]\n" - "sdot z24.s, z6.b, z4.b[0]\n" - "sdot z28.s, z6.b, z5.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[0]\n" - "sdot z13.s, z7.b, z1.b[0]\n" - "sdot z17.s, z7.b, z2.b[0]\n" - "sdot z21.s, z7.b, z3.b[0]\n" - "sdot z25.s, z7.b, z4.b[0]\n" - "sdot z29.s, z7.b, z5.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1b { z7.b }, p5/Z, [x10]\n" + "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z8.s, z7.b, z0.b[0]\n" + "sdot z12.s, z7.b, z1.b[0]\n" + "sdot z16.s, z7.b, z2.b[0]\n" + "sdot z20.s, z7.b, z3.b[0]\n" + "sdot z24.s, z7.b, z4.b[0]\n" + "sdot z28.s, z7.b, z5.b[0]\n" + "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" + "sdot z9.s, z6.b, z0.b[0]\n" + "sdot z13.s, z6.b, z1.b[0]\n" + "sdot z17.s, z6.b, z2.b[0]\n" + "sdot z21.s, z6.b, z3.b[0]\n" + "sdot z25.s, z6.b, z4.b[0]\n" + "sdot z29.s, z6.b, z5.b[0]\n" + "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "sdot z10.s, z6.b, z0.b[0]\n" - "sdot z14.s, z6.b, z1.b[0]\n" - "sdot z18.s, z6.b, z2.b[0]\n" - "sdot z22.s, z6.b, z3.b[0]\n" - "sdot z26.s, z6.b, z4.b[0]\n" - "sdot z30.s, z6.b, z5.b[0]\n" - "sdot z11.s, z7.b, z0.b[0]\n" - "sdot z15.s, z7.b, z1.b[0]\n" - "sdot z19.s, z7.b, z2.b[0]\n" - "sdot z23.s, z7.b, z3.b[0]\n" - "sdot z27.s, z7.b, z4.b[0]\n" - "sdot z31.s, z7.b, z5.b[0]\n" + "sdot z10.s, z7.b, z0.b[0]\n" + "sdot z14.s, z7.b, z1.b[0]\n" + "sdot z18.s, z7.b, z2.b[0]\n" + "sdot z22.s, z7.b, z3.b[0]\n" + "sdot z26.s, z7.b, z4.b[0]\n" + "sdot z30.s, z7.b, z5.b[0]\n" + "sdot z11.s, z6.b, z0.b[0]\n" + "sdot z15.s, z6.b, z1.b[0]\n" + "sdot z19.s, z6.b, z2.b[0]\n" + "sdot z23.s, z6.b, z3.b[0]\n" + "sdot z27.s, z6.b, z4.b[0]\n" + "sdot z31.s, z6.b, z5.b[0]\n" "ble 65f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[1]\n" - "sdot z12.s, z6.b, z1.b[1]\n" - "sdot z16.s, z6.b, z2.b[1]\n" - "sdot z20.s, z6.b, z3.b[1]\n" + "ld1b { z7.b }, p5/Z, [x10]\n" + "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z8.s, z7.b, z0.b[1]\n" + "sdot z12.s, z7.b, z1.b[1]\n" + "sdot z16.s, z7.b, z2.b[1]\n" + "sdot z20.s, z7.b, z3.b[1]\n" "subs x27, x27, #0x4\n" - "sdot z24.s, z6.b, z4.b[1]\n" - "sdot z28.s, z6.b, z5.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[1]\n" - "sdot z13.s, z7.b, z1.b[1]\n" - "sdot z17.s, z7.b, z2.b[1]\n" - "sdot z21.s, z7.b, z3.b[1]\n" - "sdot z25.s, z7.b, z4.b[1]\n" - "sdot z29.s, z7.b, z5.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "sdot z24.s, z7.b, z4.b[1]\n" + "sdot z28.s, z7.b, z5.b[1]\n" + "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" + "sdot z9.s, z6.b, z0.b[1]\n" + "sdot z13.s, z6.b, z1.b[1]\n" + "sdot z17.s, z6.b, z2.b[1]\n" + "sdot z21.s, z6.b, z3.b[1]\n" + "sdot z25.s, z6.b, z4.b[1]\n" + "sdot z29.s, z6.b, z5.b[1]\n" + "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "sdot z10.s, z6.b, z0.b[1]\n" - "sdot z14.s, z6.b, z1.b[1]\n" - "sdot z18.s, z6.b, z2.b[1]\n" - "sdot z22.s, z6.b, z3.b[1]\n" - "sdot z26.s, z6.b, z4.b[1]\n" - "sdot z30.s, z6.b, z5.b[1]\n" - "sdot z11.s, z7.b, z0.b[1]\n" - "sdot z15.s, z7.b, z1.b[1]\n" - "sdot z19.s, z7.b, z2.b[1]\n" - "sdot z23.s, z7.b, z3.b[1]\n" - "sdot z27.s, z7.b, z4.b[1]\n" - "sdot z31.s, z7.b, z5.b[1]\n" + "sdot z10.s, z7.b, z0.b[1]\n" + "sdot z14.s, z7.b, z1.b[1]\n" + "sdot z18.s, z7.b, z2.b[1]\n" + "sdot z22.s, z7.b, z3.b[1]\n" + "sdot z26.s, z7.b, z4.b[1]\n" + "sdot z30.s, z7.b, z5.b[1]\n" + "sdot z11.s, z6.b, z0.b[1]\n" + "sdot z15.s, z6.b, z1.b[1]\n" + "sdot z19.s, z6.b, z2.b[1]\n" + "sdot z23.s, z6.b, z3.b[1]\n" + "sdot z27.s, z6.b, z4.b[1]\n" + "sdot z31.s, z6.b, z5.b[1]\n" "ble 65f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[2]\n" - "sdot z12.s, z6.b, z1.b[2]\n" - "sdot z16.s, z6.b, z2.b[2]\n" - "sdot z20.s, z6.b, z3.b[2]\n" + "ld1b { z7.b }, p5/Z, [x10]\n" + "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z8.s, z7.b, z0.b[2]\n" + "sdot z12.s, z7.b, z1.b[2]\n" + "sdot z16.s, z7.b, z2.b[2]\n" + "sdot z20.s, z7.b, z3.b[2]\n" "subs x27, x27, #0x4\n" - "sdot z24.s, z6.b, z4.b[2]\n" - "sdot z28.s, z6.b, z5.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[2]\n" - "sdot z13.s, z7.b, z1.b[2]\n" - "sdot z17.s, z7.b, z2.b[2]\n" - "sdot z21.s, z7.b, z3.b[2]\n" - "sdot z25.s, z7.b, z4.b[2]\n" - "sdot z29.s, z7.b, z5.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "sdot z24.s, z7.b, z4.b[2]\n" + "sdot z28.s, z7.b, z5.b[2]\n" + "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" + "sdot z9.s, z6.b, z0.b[2]\n" + "sdot z13.s, z6.b, z1.b[2]\n" + "sdot z17.s, z6.b, z2.b[2]\n" + "sdot z21.s, z6.b, z3.b[2]\n" + "sdot z25.s, z6.b, z4.b[2]\n" + "sdot z29.s, z6.b, z5.b[2]\n" + "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "sdot z10.s, z6.b, z0.b[2]\n" - "sdot z14.s, z6.b, z1.b[2]\n" - "sdot z18.s, z6.b, z2.b[2]\n" - "sdot z22.s, z6.b, z3.b[2]\n" - "sdot z26.s, z6.b, z4.b[2]\n" - "sdot z30.s, z6.b, z5.b[2]\n" - "sdot z11.s, z7.b, z0.b[2]\n" - "sdot z15.s, z7.b, z1.b[2]\n" - "sdot z19.s, z7.b, z2.b[2]\n" - "sdot z23.s, z7.b, z3.b[2]\n" - "sdot z27.s, z7.b, z4.b[2]\n" - "sdot z31.s, z7.b, z5.b[2]\n" + "sdot z10.s, z7.b, z0.b[2]\n" + "sdot z14.s, z7.b, z1.b[2]\n" + "sdot z18.s, z7.b, z2.b[2]\n" + "sdot z22.s, z7.b, z3.b[2]\n" + "sdot z26.s, z7.b, z4.b[2]\n" + "sdot z30.s, z7.b, z5.b[2]\n" + "sdot z11.s, z6.b, z0.b[2]\n" + "sdot z15.s, z6.b, z1.b[2]\n" + "sdot z19.s, z6.b, z2.b[2]\n" + "sdot z23.s, z6.b, z3.b[2]\n" + "sdot z27.s, z6.b, z4.b[2]\n" + "sdot z31.s, z6.b, z5.b[2]\n" "ble 65f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "sdot z8.s, z6.b, z0.b[3]\n" - "sdot z12.s, z6.b, z1.b[3]\n" - "sdot z16.s, z6.b, z2.b[3]\n" - "sdot z20.s, z6.b, z3.b[3]\n" - "sdot z24.s, z6.b, z4.b[3]\n" - "sdot z28.s, z6.b, z5.b[3]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "sdot z9.s, z7.b, z0.b[3]\n" - "sdot z13.s, z7.b, z1.b[3]\n" - "sdot z17.s, z7.b, z2.b[3]\n" - "sdot z21.s, z7.b, z3.b[3]\n" - "sdot z25.s, z7.b, z4.b[3]\n" - "sdot z29.s, z7.b, z5.b[3]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1b { z7.b }, p5/Z, [x10]\n" + "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" + "sdot z8.s, z7.b, z0.b[3]\n" + "sdot z12.s, z7.b, z1.b[3]\n" + "sdot z16.s, z7.b, z2.b[3]\n" + "sdot z20.s, z7.b, z3.b[3]\n" + "sdot z24.s, z7.b, z4.b[3]\n" + "sdot z28.s, z7.b, z5.b[3]\n" + "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" + "sdot z9.s, z6.b, z0.b[3]\n" + "sdot z13.s, z6.b, z1.b[3]\n" + "sdot z17.s, z6.b, z2.b[3]\n" + "sdot z21.s, z6.b, z3.b[3]\n" + "sdot z25.s, z6.b, z4.b[3]\n" + "sdot z29.s, z6.b, z5.b[3]\n" + "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "sdot z10.s, z6.b, z0.b[3]\n" - "sdot z14.s, z6.b, z1.b[3]\n" - "sdot z18.s, z6.b, z2.b[3]\n" - "sdot z22.s, z6.b, z3.b[3]\n" - "sdot z26.s, z6.b, z4.b[3]\n" - "sdot z30.s, z6.b, z5.b[3]\n" - "sdot z11.s, z7.b, z0.b[3]\n" - "sdot z15.s, z7.b, z1.b[3]\n" - "sdot z19.s, z7.b, z2.b[3]\n" - "sdot z23.s, z7.b, z3.b[3]\n" - "sdot z27.s, z7.b, z4.b[3]\n" - "sdot z31.s, z7.b, z5.b[3]\n" + "sdot z10.s, z7.b, z0.b[3]\n" + "sdot z14.s, z7.b, z1.b[3]\n" + "sdot z18.s, z7.b, z2.b[3]\n" + "sdot z22.s, z7.b, z3.b[3]\n" + "sdot z26.s, z7.b, z4.b[3]\n" + "sdot z30.s, z7.b, z5.b[3]\n" + "sdot z11.s, z6.b, z0.b[3]\n" + "sdot z15.s, z6.b, z1.b[3]\n" + "sdot z19.s, z6.b, z2.b[3]\n" + "sdot z23.s, z6.b, z3.b[3]\n" + "sdot z27.s, z6.b, z4.b[3]\n" + "sdot z31.s, z6.b, z5.b[3]\n" "65:" // Height 6: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1748,7 +1748,6 @@ void sve_hybrid_s8s32_dot_6x4VL ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "68:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -1756,4 +1755,4 @@ void sve_hybrid_s8s32_dot_6x4VL ( } } // namespace arm_gemm -#endif // ARM_COMPUTE_ENABLE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_mmla_6x4VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_mmla_6x4VL.hpp index c08977570e4f2cd4812f0f920d38120396971b49..686295496e654f2a4a5874eb65433c04385c309f 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_mmla_6x4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_mmla_6x4VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef ARM_COMPUTE_ENABLE_SVE + #include "../std_transforms_sve.hpp" #include "../performance_parameters.hpp" @@ -74,7 +74,6 @@ public: template static inline PerformanceParameters get_performance_parameters(const CPUInfo *ci) { - if (std::is_same::value) { switch (ci->get_cpu_model()) { default: @@ -86,7 +85,6 @@ public: } } - if (std::is_same::value) { switch (ci->get_cpu_model()) { default: @@ -111,5 +109,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_mmla_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_mmla_6x4VL/generic.cpp index 350425647a68412aa78265cf2c93798da1911657..f66b6345ea59619d384643556171f15930ce60e3 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_mmla_6x4VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_s8s32_mmla_6x4VL/generic.cpp @@ -100,16 +100,16 @@ void sve_hybrid_s8s32_mmla_6x4VL ( "incw x20\n" "whilelt p1.s, x20, x11\n" "tbz %x[flags], #0, 3f\n" - "ld1w { z9.s }, p4/Z, [x9]\n" - "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n" + "ld1w { z19.s }, p4/Z, [x9]\n" + "ld1w { z18.s }, p3/Z, [x9, #1, MUL VL]\n" + "zip1 z8.d, z19.d, z12.d\n" + "zip2 z12.d, z19.d, z12.d\n" + "ld1w { z17.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" + "zip1 z9.d, z18.d, z13.d\n" + "zip2 z13.d, z18.d, z13.d\n" + "zip1 z10.d, z17.d, z14.d\n" + "zip2 z14.d, z17.d, z14.d\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" "b 4f\n" @@ -127,11 +127,11 @@ void sve_hybrid_s8s32_mmla_6x4VL ( "5:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 6f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 7f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -143,86 +143,86 @@ void sve_hybrid_s8s32_mmla_6x4VL ( "ble 9f\n" "8:" // Height 1: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z1.b }, p0/Z, [x26]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45079808 // smmla z8.s, z0.b, z7.b\n" - ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" - ".inst 0x4506980d // smmla z13.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x4507980a // smmla z10.s, z0.b, z7.b\n" - ".inst 0x4506980e // smmla z14.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + "ld1rqb { z20.b }, p0/Z, [x26]\n" + "trn1 z18.d, z20.d, z19.d\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45119a48 // smmla z8.s, z18.b, z17.b\n" + ".inst 0x45109a4c // smmla z12.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45119a49 // smmla z9.s, z18.b, z17.b\n" + ".inst 0x45109a4d // smmla z13.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #4, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x45119a4a // smmla z10.s, z18.b, z17.b\n" + ".inst 0x45109a4e // smmla z14.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #6, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "trn2 z1.d, z1.d, z2.d\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" - ".inst 0x4506980f // smmla z15.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-8, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #-7, MUL VL]\n" - ".inst 0x45079828 // smmla z8.s, z1.b, z7.b\n" - ".inst 0x4506982c // smmla z12.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-6, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #-5, MUL VL]\n" - ".inst 0x45079829 // smmla z9.s, z1.b, z7.b\n" - ".inst 0x4506982d // smmla z13.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-4, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #-3, MUL VL]\n" - ".inst 0x4507982a // smmla z10.s, z1.b, z7.b\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #-1, MUL VL]\n" + "trn2 z20.d, z20.d, z19.d\n" + ".inst 0x45119a4b // smmla z11.s, z18.b, z17.b\n" + ".inst 0x45109a4f // smmla z15.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #-8, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #-7, MUL VL]\n" + ".inst 0x45119a88 // smmla z8.s, z20.b, z17.b\n" + ".inst 0x45109a8c // smmla z12.s, z20.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #-6, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #-5, MUL VL]\n" + ".inst 0x45119a89 // smmla z9.s, z20.b, z17.b\n" + ".inst 0x45109a8d // smmla z13.s, z20.b, z16.b\n" + "ld1b { z16.b }, p5/Z, [x10, #-4, MUL VL]\n" + "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n" + ".inst 0x45109a8a // smmla z10.s, z20.b, z16.b\n" + ".inst 0x45079a8e // smmla z14.s, z20.b, z7.b\n" + "ld1b { z17.b }, p5/Z, [x10, #-2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #-1, MUL VL]\n" "sub x27, x27, #0x10\n" "cmp x27, #0x10\n" - ".inst 0x4507982b // smmla z11.s, z1.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" + ".inst 0x45119a8b // smmla z11.s, z20.b, z17.b\n" + ".inst 0x45109a8f // smmla z15.s, z20.b, z16.b\n" "add x26, x26, #0x10\n" "bgt 8b\n" "9:" // Height 1: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" "ld1rqb { z1.b }, p0/Z, [x26]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45079808 // smmla z8.s, z0.b, z7.b\n" - ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" - ".inst 0x4506980d // smmla z13.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x4507980a // smmla z10.s, z0.b, z7.b\n" - ".inst 0x4506980e // smmla z14.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + "trn1 z18.d, z1.d, z19.d\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45119a48 // smmla z8.s, z18.b, z17.b\n" + ".inst 0x45109a4c // smmla z12.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45119a49 // smmla z9.s, z18.b, z17.b\n" + ".inst 0x45109a4d // smmla z13.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #4, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x45119a4a // smmla z10.s, z18.b, z17.b\n" + ".inst 0x45109a4e // smmla z14.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #6, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #7, MUL VL]\n" "subs x27, x27, #0x8\n" - "trn2 z1.d, z1.d, z2.d\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" - ".inst 0x4506980f // smmla z15.s, z0.b, z6.b\n" + "trn2 z1.d, z1.d, z19.d\n" + ".inst 0x45119a4b // smmla z11.s, z18.b, z17.b\n" + ".inst 0x45109a4f // smmla z15.s, z18.b, z16.b\n" "addvl x10, x10, #8\n" "ble 10f\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45079828 // smmla z8.s, z1.b, z7.b\n" - ".inst 0x4506982c // smmla z12.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45079829 // smmla z9.s, z1.b, z7.b\n" - ".inst 0x4506982d // smmla z13.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x4507982a // smmla z10.s, z1.b, z7.b\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" - ".inst 0x4507982b // smmla z11.s, z1.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45119828 // smmla z8.s, z1.b, z17.b\n" + ".inst 0x4510982c // smmla z12.s, z1.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45119829 // smmla z9.s, z1.b, z17.b\n" + ".inst 0x4510982d // smmla z13.s, z1.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #4, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x4511982a // smmla z10.s, z1.b, z17.b\n" + ".inst 0x4510982e // smmla z14.s, z1.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #6, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x4511982b // smmla z11.s, z1.b, z17.b\n" + ".inst 0x4510982f // smmla z15.s, z1.b, z16.b\n" "addvl x10, x10, #8\n" "10:" // Height 1: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -258,21 +258,21 @@ void sve_hybrid_s8s32_mmla_6x4VL ( "whilelt p1.s, x20, x11\n" "tbz %x[flags], #0, 14f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x9]\n" - "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n" + "add x20, x9, x20, LSL #2\n" + "ld1w { z18.s }, p4/Z, [x9]\n" + "ld1w { z2.s }, p3/Z, [x9, #1, MUL VL]\n" + "ld1w { z17.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x24]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" + "ld1w { z12.s }, p4/Z, [x20]\n" + "zip1 z8.d, z18.d, z12.d\n" + "zip2 z12.d, z18.d, z12.d\n" + "ld1w { z13.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x20, #2, MUL VL]\n" + "zip1 z9.d, z2.d, z13.d\n" + "zip2 z13.d, z2.d, z13.d\n" + "ld1w { z15.s }, p1/Z, [x20, #3, MUL VL]\n" + "zip1 z10.d, z17.d, z14.d\n" + "zip2 z14.d, z17.d, z14.d\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" "b 15f\n" @@ -290,12 +290,12 @@ void sve_hybrid_s8s32_mmla_6x4VL ( "16:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 17f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 18f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -303,95 +303,95 @@ void sve_hybrid_s8s32_mmla_6x4VL ( "b 18f\n" "17:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" + "add x25, x26, x21\n" "18:" // Height 2: input setup done "cmp x27, #0x10\n" "ble 20f\n" "19:" // Height 2: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45079808 // smmla z8.s, z0.b, z7.b\n" - ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" - ".inst 0x4506980d // smmla z13.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x4507980a // smmla z10.s, z0.b, z7.b\n" - ".inst 0x4506980e // smmla z14.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + "ld1rqb { z20.b }, p0/Z, [x26]\n" + "ld1rqb { z19.b }, p0/Z, [x25]\n" + "trn1 z18.d, z20.d, z19.d\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45119a48 // smmla z8.s, z18.b, z17.b\n" + ".inst 0x45109a4c // smmla z12.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45119a49 // smmla z9.s, z18.b, z17.b\n" + ".inst 0x45109a4d // smmla z13.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #4, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x45119a4a // smmla z10.s, z18.b, z17.b\n" + ".inst 0x45109a4e // smmla z14.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #6, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "trn2 z1.d, z1.d, z2.d\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" - ".inst 0x4506980f // smmla z15.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-8, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #-7, MUL VL]\n" - ".inst 0x45079828 // smmla z8.s, z1.b, z7.b\n" - ".inst 0x4506982c // smmla z12.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-6, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #-5, MUL VL]\n" - ".inst 0x45079829 // smmla z9.s, z1.b, z7.b\n" - ".inst 0x4506982d // smmla z13.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-4, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #-3, MUL VL]\n" - ".inst 0x4507982a // smmla z10.s, z1.b, z7.b\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #-1, MUL VL]\n" + "trn2 z20.d, z20.d, z19.d\n" + ".inst 0x45119a4b // smmla z11.s, z18.b, z17.b\n" + ".inst 0x45109a4f // smmla z15.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #-8, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #-7, MUL VL]\n" + ".inst 0x45119a88 // smmla z8.s, z20.b, z17.b\n" + ".inst 0x45109a8c // smmla z12.s, z20.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #-6, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #-5, MUL VL]\n" + ".inst 0x45119a89 // smmla z9.s, z20.b, z17.b\n" + ".inst 0x45109a8d // smmla z13.s, z20.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #-4, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #-3, MUL VL]\n" + ".inst 0x45119a8a // smmla z10.s, z20.b, z17.b\n" + ".inst 0x45109a8e // smmla z14.s, z20.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #-2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #-1, MUL VL]\n" "sub x27, x27, #0x10\n" "cmp x27, #0x10\n" - ".inst 0x4507982b // smmla z11.s, z1.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" + ".inst 0x45119a8b // smmla z11.s, z20.b, z17.b\n" + ".inst 0x45109a8f // smmla z15.s, z20.b, z16.b\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" "bgt 19b\n" "20:" // Height 2: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45079808 // smmla z8.s, z0.b, z7.b\n" - ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" - ".inst 0x4506980d // smmla z13.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x4507980a // smmla z10.s, z0.b, z7.b\n" - ".inst 0x4506980e // smmla z14.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + "ld1rqb { z19.b }, p0/Z, [x25]\n" + "trn1 z18.d, z1.d, z19.d\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45119a48 // smmla z8.s, z18.b, z17.b\n" + ".inst 0x45109a4c // smmla z12.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45119a49 // smmla z9.s, z18.b, z17.b\n" + ".inst 0x45109a4d // smmla z13.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #4, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x45119a4a // smmla z10.s, z18.b, z17.b\n" + ".inst 0x45109a4e // smmla z14.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #6, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #7, MUL VL]\n" "subs x27, x27, #0x8\n" - "trn2 z1.d, z1.d, z2.d\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" - ".inst 0x4506980f // smmla z15.s, z0.b, z6.b\n" + "trn2 z1.d, z1.d, z19.d\n" + ".inst 0x45119a4b // smmla z11.s, z18.b, z17.b\n" + ".inst 0x45109a4f // smmla z15.s, z18.b, z16.b\n" "addvl x10, x10, #8\n" "ble 21f\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45079828 // smmla z8.s, z1.b, z7.b\n" - ".inst 0x4506982c // smmla z12.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45079829 // smmla z9.s, z1.b, z7.b\n" - ".inst 0x4506982d // smmla z13.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x4507982a // smmla z10.s, z1.b, z7.b\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" - ".inst 0x4507982b // smmla z11.s, z1.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45119828 // smmla z8.s, z1.b, z17.b\n" + ".inst 0x4510982c // smmla z12.s, z1.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45119829 // smmla z9.s, z1.b, z17.b\n" + ".inst 0x4510982d // smmla z13.s, z1.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #4, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x4511982a // smmla z10.s, z1.b, z17.b\n" + ".inst 0x4510982e // smmla z14.s, z1.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #6, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x4511982b // smmla z11.s, z1.b, z17.b\n" + ".inst 0x4510982f // smmla z15.s, z1.b, z16.b\n" "addvl x10, x10, #8\n" "21:" // Height 2: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -399,24 +399,24 @@ void sve_hybrid_s8s32_mmla_6x4VL ( "cmp x28, x20\n" "bne 16b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "uzp1 z7.d, z8.d, z12.d\n" + "add x20, x9, x20, LSL #2\n" + "uzp1 z16.d, z8.d, z12.d\n" "uzp2 z8.d, z8.d, z12.d\n" - "uzp1 z12.d, z9.d, z13.d\n" + "uzp1 z17.d, z9.d, z13.d\n" "uzp2 z9.d, z9.d, z13.d\n" - "st1w { z7.s }, p4, [x9]\n" - "uzp1 z13.d, z10.d, z14.d\n" + "st1w { z16.s }, p4, [x9]\n" + "uzp1 z16.d, z10.d, z14.d\n" "uzp2 z10.d, z10.d, z14.d\n" - "st1w { z12.s }, p3, [x9, #1, MUL VL]\n" - "uzp1 z14.d, z11.d, z15.d\n" + "st1w { z17.s }, p3, [x9, #1, MUL VL]\n" + "uzp1 z2.d, z11.d, z15.d\n" "uzp2 z11.d, z11.d, z15.d\n" - "st1w { z13.s }, p2, [x9, #2, MUL VL]\n" - "st1w { z14.s }, p1, [x9, #3, MUL VL]\n" + "st1w { z16.s }, p2, [x9, #2, MUL VL]\n" + "st1w { z2.s }, p1, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "st1w { z8.s }, p4, [x24]\n" - "st1w { z9.s }, p3, [x24, #1, MUL VL]\n" - "st1w { z10.s }, p2, [x24, #2, MUL VL]\n" - "st1w { z11.s }, p1, [x24, #3, MUL VL]\n" + "st1w { z8.s }, p4, [x20]\n" + "st1w { z9.s }, p3, [x20, #1, MUL VL]\n" + "st1w { z10.s }, p2, [x20, #2, MUL VL]\n" + "st1w { z11.s }, p1, [x20, #3, MUL VL]\n" "22:" // Height 2: Writeback done "decw x11, ALL, MUL #4\n" "cmp x11, XZR\n" @@ -437,28 +437,28 @@ void sve_hybrid_s8s32_mmla_6x4VL ( "whilelt p1.s, x20, x11\n" "tbz %x[flags], #0, 25f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x9]\n" - "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n" + "add x21, x9, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" + "ld1w { z19.s }, p4/Z, [x9]\n" + "ld1w { z17.s }, p3/Z, [x9, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x24]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z17.s }, p4/Z, [x23]\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" - "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x21]\n" + "zip1 z8.d, z19.d, z12.d\n" + "zip2 z12.d, z19.d, z12.d\n" + "ld1w { z13.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x21, #2, MUL VL]\n" + "zip1 z9.d, z17.d, z13.d\n" + "zip2 z13.d, z17.d, z13.d\n" + "ld1w { z15.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z17.s }, p4/Z, [x20]\n" + "zip1 z10.d, z18.d, z14.d\n" + "zip2 z14.d, z18.d, z14.d\n" + "ld1w { z18.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z19.s }, p2/Z, [x20, #2, MUL VL]\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" - "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z24.s }, p1/Z, [x20, #3, MUL VL]\n" "zip1 z16.d, z17.d, z20.d\n" "zip2 z20.d, z17.d, z20.d\n" "zip1 z17.d, z18.d, z21.d\n" @@ -490,13 +490,13 @@ void sve_hybrid_s8s32_mmla_6x4VL ( "27:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 28f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 29f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -505,169 +505,169 @@ void sve_hybrid_s8s32_mmla_6x4VL ( "b 29f\n" "28:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" "29:" // Height 3: input setup done "cmp x27, #0x10\n" "ble 31f\n" "30:" // Height 3: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" - "ld1rqb { z3.b }, p0/Z, [x24]\n" - "trn1 z0.d, z1.d, z2.d\n" - "trn2 z1.d, z1.d, z2.d\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45079808 // smmla z8.s, z0.b, z7.b\n" - ".inst 0x45079850 // smmla z16.s, z2.b, z7.b\n" - ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n" - ".inst 0x45069854 // smmla z20.s, z2.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" - ".inst 0x45079851 // smmla z17.s, z2.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x4506980d // smmla z13.s, z0.b, z6.b\n" - ".inst 0x45069855 // smmla z21.s, z2.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" + "ld1rqb { z30.b }, p0/Z, [x26]\n" + "ld1rqb { z24.b }, p0/Z, [x25]\n" + "ld1rqb { z28.b }, p0/Z, [x24]\n" + "trn1 z27.d, z30.d, z24.d\n" + "trn2 z30.d, z30.d, z24.d\n" + "ld1b { z25.b }, p5/Z, [x10]\n" + "trn1 z26.d, z28.d, z29.d\n" + "ld1b { z24.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45199b68 // smmla z8.s, z27.b, z25.b\n" + ".inst 0x45199b50 // smmla z16.s, z26.b, z25.b\n" + ".inst 0x45189b6c // smmla z12.s, z27.b, z24.b\n" + ".inst 0x45189b54 // smmla z20.s, z26.b, z24.b\n" + "ld1b { z25.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z24.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45199b69 // smmla z9.s, z27.b, z25.b\n" + ".inst 0x45199b51 // smmla z17.s, z26.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #4, MUL VL]\n" + "trn2 z28.d, z28.d, z29.d\n" + ".inst 0x45189b6d // smmla z13.s, z27.b, z24.b\n" + ".inst 0x45189b55 // smmla z21.s, z26.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #5, MUL VL]\n" "sub x27, x27, #0x10\n" - ".inst 0x4507980a // smmla z10.s, z0.b, z7.b\n" - ".inst 0x45079852 // smmla z18.s, z2.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x45199b6a // smmla z10.s, z27.b, z25.b\n" + ".inst 0x45199b52 // smmla z18.s, z26.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #6, MUL VL]\n" "cmp x27, #0x10\n" - ".inst 0x4506980e // smmla z14.s, z0.b, z6.b\n" - ".inst 0x45069856 // smmla z22.s, z2.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x45189b6e // smmla z14.s, z27.b, z24.b\n" + ".inst 0x45189b56 // smmla z22.s, z26.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" - ".inst 0x45079853 // smmla z19.s, z2.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-8, MUL VL]\n" + ".inst 0x45199b6b // smmla z11.s, z27.b, z25.b\n" + ".inst 0x45199b53 // smmla z19.s, z26.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #-8, MUL VL]\n" "add x26, x26, #0x10\n" - ".inst 0x4506980f // smmla z15.s, z0.b, z6.b\n" - ".inst 0x45069857 // smmla z23.s, z2.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-7, MUL VL]\n" + ".inst 0x45189b6f // smmla z15.s, z27.b, z24.b\n" + ".inst 0x45189b57 // smmla z23.s, z26.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #-7, MUL VL]\n" "add x25, x25, #0x10\n" - ".inst 0x45079828 // smmla z8.s, z1.b, z7.b\n" - ".inst 0x45079870 // smmla z16.s, z3.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-6, MUL VL]\n" + ".inst 0x45199bc8 // smmla z8.s, z30.b, z25.b\n" + ".inst 0x45199b90 // smmla z16.s, z28.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #-6, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x4506982c // smmla z12.s, z1.b, z6.b\n" - ".inst 0x45069874 // smmla z20.s, z3.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-5, MUL VL]\n" - ".inst 0x45079829 // smmla z9.s, z1.b, z7.b\n" - ".inst 0x45079871 // smmla z17.s, z3.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-4, MUL VL]\n" - ".inst 0x4506982d // smmla z13.s, z1.b, z6.b\n" - ".inst 0x45069875 // smmla z21.s, z3.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-3, MUL VL]\n" - ".inst 0x4507982a // smmla z10.s, z1.b, z7.b\n" - ".inst 0x45079872 // smmla z18.s, z3.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-2, MUL VL]\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - ".inst 0x45069876 // smmla z22.s, z3.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-1, MUL VL]\n" - ".inst 0x4507982b // smmla z11.s, z1.b, z7.b\n" - ".inst 0x45079873 // smmla z19.s, z3.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" - ".inst 0x45069877 // smmla z23.s, z3.b, z6.b\n" + ".inst 0x45189bcc // smmla z12.s, z30.b, z24.b\n" + ".inst 0x45189b94 // smmla z20.s, z28.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #-5, MUL VL]\n" + ".inst 0x45199bc9 // smmla z9.s, z30.b, z25.b\n" + ".inst 0x45199b91 // smmla z17.s, z28.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #-4, MUL VL]\n" + ".inst 0x45189bcd // smmla z13.s, z30.b, z24.b\n" + ".inst 0x45189b95 // smmla z21.s, z28.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #-3, MUL VL]\n" + ".inst 0x45199bca // smmla z10.s, z30.b, z25.b\n" + ".inst 0x45199b92 // smmla z18.s, z28.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #-2, MUL VL]\n" + ".inst 0x45189bce // smmla z14.s, z30.b, z24.b\n" + ".inst 0x45189b96 // smmla z22.s, z28.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #-1, MUL VL]\n" + ".inst 0x45199bcb // smmla z11.s, z30.b, z25.b\n" + ".inst 0x45199b93 // smmla z19.s, z28.b, z25.b\n" + ".inst 0x45189bcf // smmla z15.s, z30.b, z24.b\n" + ".inst 0x45189b97 // smmla z23.s, z28.b, z24.b\n" "bgt 30b\n" "31:" // Height 3: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" + "ld1rqb { z24.b }, p0/Z, [x25]\n" "ld1rqb { z3.b }, p0/Z, [x24]\n" - "trn1 z0.d, z1.d, z2.d\n" - "trn2 z1.d, z1.d, z2.d\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45079808 // smmla z8.s, z0.b, z7.b\n" - ".inst 0x45079850 // smmla z16.s, z2.b, z7.b\n" - ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n" - ".inst 0x45069854 // smmla z20.s, z2.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" - ".inst 0x45079851 // smmla z17.s, z2.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" + "trn1 z27.d, z1.d, z24.d\n" + "trn2 z1.d, z1.d, z24.d\n" + "ld1b { z25.b }, p5/Z, [x10]\n" + "trn1 z26.d, z3.d, z28.d\n" + "ld1b { z24.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45199b68 // smmla z8.s, z27.b, z25.b\n" + ".inst 0x45199b50 // smmla z16.s, z26.b, z25.b\n" + ".inst 0x45189b6c // smmla z12.s, z27.b, z24.b\n" + ".inst 0x45189b54 // smmla z20.s, z26.b, z24.b\n" + "ld1b { z25.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z24.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45199b69 // smmla z9.s, z27.b, z25.b\n" + ".inst 0x45199b51 // smmla z17.s, z26.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #4, MUL VL]\n" "subs x27, x27, #0x8\n" - ".inst 0x4506980d // smmla z13.s, z0.b, z6.b\n" - ".inst 0x45069855 // smmla z21.s, z2.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x4507980a // smmla z10.s, z0.b, z7.b\n" - ".inst 0x45079852 // smmla z18.s, z2.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x4506980e // smmla z14.s, z0.b, z6.b\n" - ".inst 0x45069856 // smmla z22.s, z2.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x45189b6d // smmla z13.s, z27.b, z24.b\n" + ".inst 0x45189b55 // smmla z21.s, z26.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #5, MUL VL]\n" + "trn2 z3.d, z3.d, z28.d\n" + ".inst 0x45199b6a // smmla z10.s, z27.b, z25.b\n" + ".inst 0x45199b52 // smmla z18.s, z26.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x45189b6e // smmla z14.s, z27.b, z24.b\n" + ".inst 0x45189b56 // smmla z22.s, z26.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" - ".inst 0x45079853 // smmla z19.s, z2.b, z7.b\n" - ".inst 0x4506980f // smmla z15.s, z0.b, z6.b\n" - ".inst 0x45069857 // smmla z23.s, z2.b, z6.b\n" + ".inst 0x45199b6b // smmla z11.s, z27.b, z25.b\n" + ".inst 0x45199b53 // smmla z19.s, z26.b, z25.b\n" + ".inst 0x45189b6f // smmla z15.s, z27.b, z24.b\n" + ".inst 0x45189b57 // smmla z23.s, z26.b, z24.b\n" "ble 32f\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45079828 // smmla z8.s, z1.b, z7.b\n" - ".inst 0x45079870 // smmla z16.s, z3.b, z7.b\n" - ".inst 0x4506982c // smmla z12.s, z1.b, z6.b\n" - ".inst 0x45069874 // smmla z20.s, z3.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45079829 // smmla z9.s, z1.b, z7.b\n" - ".inst 0x45079871 // smmla z17.s, z3.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x4506982d // smmla z13.s, z1.b, z6.b\n" - ".inst 0x45069875 // smmla z21.s, z3.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x4507982a // smmla z10.s, z1.b, z7.b\n" - ".inst 0x45079872 // smmla z18.s, z3.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - ".inst 0x45069876 // smmla z22.s, z3.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + "ld1b { z25.b }, p5/Z, [x10]\n" + "ld1b { z24.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45199828 // smmla z8.s, z1.b, z25.b\n" + ".inst 0x45199870 // smmla z16.s, z3.b, z25.b\n" + ".inst 0x4518982c // smmla z12.s, z1.b, z24.b\n" + ".inst 0x45189874 // smmla z20.s, z3.b, z24.b\n" + "ld1b { z25.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z24.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45199829 // smmla z9.s, z1.b, z25.b\n" + ".inst 0x45199871 // smmla z17.s, z3.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x4518982d // smmla z13.s, z1.b, z24.b\n" + ".inst 0x45189875 // smmla z21.s, z3.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x4519982a // smmla z10.s, z1.b, z25.b\n" + ".inst 0x45199872 // smmla z18.s, z3.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x4518982e // smmla z14.s, z1.b, z24.b\n" + ".inst 0x45189876 // smmla z22.s, z3.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - ".inst 0x4507982b // smmla z11.s, z1.b, z7.b\n" - ".inst 0x45079873 // smmla z19.s, z3.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" - ".inst 0x45069877 // smmla z23.s, z3.b, z6.b\n" + ".inst 0x4519982b // smmla z11.s, z1.b, z25.b\n" + ".inst 0x45199873 // smmla z19.s, z3.b, z25.b\n" + ".inst 0x4518982f // smmla z15.s, z1.b, z24.b\n" + ".inst 0x45189877 // smmla z23.s, z3.b, z24.b\n" "32:" // Height 3: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 27b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "uzp1 z7.d, z8.d, z12.d\n" + "add x21, x9, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" + "uzp1 z25.d, z8.d, z12.d\n" "uzp2 z8.d, z8.d, z12.d\n" - "uzp1 z12.d, z9.d, z13.d\n" - "st1w { z7.s }, p4, [x9]\n" + "uzp1 z24.d, z9.d, z13.d\n" + "st1w { z25.s }, p4, [x9]\n" "uzp2 z9.d, z9.d, z13.d\n" - "uzp1 z13.d, z10.d, z14.d\n" - "st1w { z12.s }, p3, [x9, #1, MUL VL]\n" + "uzp1 z25.d, z10.d, z14.d\n" + "st1w { z24.s }, p3, [x9, #1, MUL VL]\n" "uzp2 z10.d, z10.d, z14.d\n" - "uzp1 z14.d, z11.d, z15.d\n" - "st1w { z13.s }, p2, [x9, #2, MUL VL]\n" + "uzp1 z24.d, z11.d, z15.d\n" + "st1w { z25.s }, p2, [x9, #2, MUL VL]\n" "uzp2 z11.d, z11.d, z15.d\n" "uzp1 z16.d, z16.d, z20.d\n" - "st1w { z14.s }, p1, [x9, #3, MUL VL]\n" + "st1w { z24.s }, p1, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" "uzp1 z17.d, z17.d, z21.d\n" "uzp1 z18.d, z18.d, z22.d\n" - "st1w { z8.s }, p4, [x24]\n" + "st1w { z8.s }, p4, [x21]\n" "uzp1 z19.d, z19.d, z23.d\n" - "st1w { z9.s }, p3, [x24, #1, MUL VL]\n" - "st1w { z10.s }, p2, [x24, #2, MUL VL]\n" - "st1w { z11.s }, p1, [x24, #3, MUL VL]\n" - "st1w { z16.s }, p4, [x23]\n" - "st1w { z17.s }, p3, [x23, #1, MUL VL]\n" - "st1w { z18.s }, p2, [x23, #2, MUL VL]\n" - "st1w { z19.s }, p1, [x23, #3, MUL VL]\n" + "st1w { z9.s }, p3, [x21, #1, MUL VL]\n" + "st1w { z10.s }, p2, [x21, #2, MUL VL]\n" + "st1w { z11.s }, p1, [x21, #3, MUL VL]\n" + "st1w { z16.s }, p4, [x20]\n" + "st1w { z17.s }, p3, [x20, #1, MUL VL]\n" + "st1w { z18.s }, p2, [x20, #2, MUL VL]\n" + "st1w { z19.s }, p1, [x20, #3, MUL VL]\n" "33:" // Height 3: Writeback done "decw x11, ALL, MUL #4\n" "cmp x11, XZR\n" @@ -688,37 +688,37 @@ void sve_hybrid_s8s32_mmla_6x4VL ( "whilelt p1.s, x20, x11\n" "tbz %x[flags], #0, 36f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x9]\n" - "add x22, x23, x20, LSL #2\n" - "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n" + "add x22, x9, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" + "ld1w { z19.s }, p4/Z, [x9]\n" + "add x20, x21, x20, LSL #2\n" + "ld1w { z17.s }, p3/Z, [x9, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x24]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z17.s }, p4/Z, [x23]\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" - "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x22]\n" + "zip1 z8.d, z19.d, z12.d\n" + "zip2 z12.d, z19.d, z12.d\n" + "ld1w { z13.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x22, #2, MUL VL]\n" + "zip1 z9.d, z17.d, z13.d\n" + "zip2 z13.d, z17.d, z13.d\n" + "ld1w { z15.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z17.s }, p4/Z, [x21]\n" + "zip1 z10.d, z18.d, z14.d\n" + "zip2 z14.d, z18.d, z14.d\n" + "ld1w { z18.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z19.s }, p2/Z, [x21, #2, MUL VL]\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" - "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x22]\n" + "ld1w { z24.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x20]\n" "zip1 z16.d, z17.d, z20.d\n" "zip2 z20.d, z17.d, z20.d\n" - "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n" + "ld1w { z21.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x20, #2, MUL VL]\n" "zip1 z17.d, z18.d, z21.d\n" "zip2 z21.d, z18.d, z21.d\n" - "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z23.s }, p1/Z, [x20, #3, MUL VL]\n" "zip1 z18.d, z19.d, z22.d\n" "zip2 z22.d, z19.d, z22.d\n" "zip1 z19.d, z24.d, z23.d\n" @@ -746,14 +746,14 @@ void sve_hybrid_s8s32_mmla_6x4VL ( "38:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 39f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 40f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -763,182 +763,182 @@ void sve_hybrid_s8s32_mmla_6x4VL ( "b 40f\n" "39:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" "40:" // Height 4: input setup done "cmp x27, #0x10\n" "ble 42f\n" "41:" // Height 4: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1rqb { z3.b }, p0/Z, [x24]\n" - "ld1rqb { z4.b }, p0/Z, [x23]\n" - "trn2 z1.d, z1.d, z2.d\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45079808 // smmla z8.s, z0.b, z7.b\n" - ".inst 0x45079850 // smmla z16.s, z2.b, z7.b\n" - ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n" - ".inst 0x45069854 // smmla z20.s, z2.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" - ".inst 0x45079851 // smmla z17.s, z2.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x4506980d // smmla z13.s, z0.b, z6.b\n" - ".inst 0x45069855 // smmla z21.s, z2.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" + "ld1rqb { z30.b }, p0/Z, [x26]\n" + "ld1rqb { z24.b }, p0/Z, [x25]\n" + "trn1 z29.d, z30.d, z24.d\n" + "ld1rqb { z28.b }, p0/Z, [x24]\n" + "ld1rqb { z27.b }, p0/Z, [x23]\n" + "trn2 z30.d, z30.d, z24.d\n" + "trn1 z26.d, z28.d, z27.d\n" + "ld1b { z25.b }, p5/Z, [x10]\n" + "ld1b { z24.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45199ba8 // smmla z8.s, z29.b, z25.b\n" + ".inst 0x45199b50 // smmla z16.s, z26.b, z25.b\n" + ".inst 0x45189bac // smmla z12.s, z29.b, z24.b\n" + ".inst 0x45189b54 // smmla z20.s, z26.b, z24.b\n" + "ld1b { z25.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z24.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45199ba9 // smmla z9.s, z29.b, z25.b\n" + ".inst 0x45199b51 // smmla z17.s, z26.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #4, MUL VL]\n" + "trn2 z28.d, z28.d, z27.d\n" + ".inst 0x45189bad // smmla z13.s, z29.b, z24.b\n" + ".inst 0x45189b55 // smmla z21.s, z26.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #5, MUL VL]\n" "sub x27, x27, #0x10\n" - ".inst 0x4507980a // smmla z10.s, z0.b, z7.b\n" - ".inst 0x45079852 // smmla z18.s, z2.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x45199baa // smmla z10.s, z29.b, z25.b\n" + ".inst 0x45199b52 // smmla z18.s, z26.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #6, MUL VL]\n" "cmp x27, #0x10\n" - ".inst 0x4506980e // smmla z14.s, z0.b, z6.b\n" - ".inst 0x45069856 // smmla z22.s, z2.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x45189bae // smmla z14.s, z29.b, z24.b\n" + ".inst 0x45189b56 // smmla z22.s, z26.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" - ".inst 0x45079853 // smmla z19.s, z2.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-8, MUL VL]\n" + ".inst 0x45199bab // smmla z11.s, z29.b, z25.b\n" + ".inst 0x45199b53 // smmla z19.s, z26.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #-8, MUL VL]\n" "add x26, x26, #0x10\n" - ".inst 0x4506980f // smmla z15.s, z0.b, z6.b\n" - ".inst 0x45069857 // smmla z23.s, z2.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-7, MUL VL]\n" + ".inst 0x45189baf // smmla z15.s, z29.b, z24.b\n" + ".inst 0x45189b57 // smmla z23.s, z26.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #-7, MUL VL]\n" "add x25, x25, #0x10\n" - ".inst 0x45079828 // smmla z8.s, z1.b, z7.b\n" - ".inst 0x45079870 // smmla z16.s, z3.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-6, MUL VL]\n" + ".inst 0x45199bc8 // smmla z8.s, z30.b, z25.b\n" + ".inst 0x45199b90 // smmla z16.s, z28.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #-6, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x4506982c // smmla z12.s, z1.b, z6.b\n" - ".inst 0x45069874 // smmla z20.s, z3.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-5, MUL VL]\n" + ".inst 0x45189bcc // smmla z12.s, z30.b, z24.b\n" + ".inst 0x45189b94 // smmla z20.s, z28.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #-5, MUL VL]\n" "add x23, x23, #0x10\n" - ".inst 0x45079829 // smmla z9.s, z1.b, z7.b\n" - ".inst 0x45079871 // smmla z17.s, z3.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-4, MUL VL]\n" - ".inst 0x4506982d // smmla z13.s, z1.b, z6.b\n" - ".inst 0x45069875 // smmla z21.s, z3.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-3, MUL VL]\n" - ".inst 0x4507982a // smmla z10.s, z1.b, z7.b\n" - ".inst 0x45079872 // smmla z18.s, z3.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-2, MUL VL]\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - ".inst 0x45069876 // smmla z22.s, z3.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-1, MUL VL]\n" - ".inst 0x4507982b // smmla z11.s, z1.b, z7.b\n" - ".inst 0x45079873 // smmla z19.s, z3.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" - ".inst 0x45069877 // smmla z23.s, z3.b, z6.b\n" + ".inst 0x45199bc9 // smmla z9.s, z30.b, z25.b\n" + ".inst 0x45199b91 // smmla z17.s, z28.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #-4, MUL VL]\n" + ".inst 0x45189bcd // smmla z13.s, z30.b, z24.b\n" + ".inst 0x45189b95 // smmla z21.s, z28.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #-3, MUL VL]\n" + ".inst 0x45199bca // smmla z10.s, z30.b, z25.b\n" + ".inst 0x45199b92 // smmla z18.s, z28.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #-2, MUL VL]\n" + ".inst 0x45189bce // smmla z14.s, z30.b, z24.b\n" + ".inst 0x45189b96 // smmla z22.s, z28.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #-1, MUL VL]\n" + ".inst 0x45199bcb // smmla z11.s, z30.b, z25.b\n" + ".inst 0x45199b93 // smmla z19.s, z28.b, z25.b\n" + ".inst 0x45189bcf // smmla z15.s, z30.b, z24.b\n" + ".inst 0x45189b97 // smmla z23.s, z28.b, z24.b\n" "bgt 41b\n" "42:" // Height 4: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" + "ld1rqb { z24.b }, p0/Z, [x25]\n" + "trn1 z28.d, z1.d, z24.d\n" "ld1rqb { z3.b }, p0/Z, [x24]\n" - "ld1rqb { z4.b }, p0/Z, [x23]\n" - "trn2 z1.d, z1.d, z2.d\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45079808 // smmla z8.s, z0.b, z7.b\n" - ".inst 0x45079850 // smmla z16.s, z2.b, z7.b\n" - ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n" - ".inst 0x45069854 // smmla z20.s, z2.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" - ".inst 0x45079851 // smmla z17.s, z2.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" + "ld1rqb { z27.b }, p0/Z, [x23]\n" + "trn2 z1.d, z1.d, z24.d\n" + "trn1 z26.d, z3.d, z27.d\n" + "ld1b { z25.b }, p5/Z, [x10]\n" + "ld1b { z24.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45199b88 // smmla z8.s, z28.b, z25.b\n" + ".inst 0x45199b50 // smmla z16.s, z26.b, z25.b\n" + ".inst 0x45189b8c // smmla z12.s, z28.b, z24.b\n" + ".inst 0x45189b54 // smmla z20.s, z26.b, z24.b\n" + "ld1b { z25.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z24.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45199b89 // smmla z9.s, z28.b, z25.b\n" + ".inst 0x45199b51 // smmla z17.s, z26.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #4, MUL VL]\n" "subs x27, x27, #0x8\n" - ".inst 0x4506980d // smmla z13.s, z0.b, z6.b\n" - ".inst 0x45069855 // smmla z21.s, z2.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x4507980a // smmla z10.s, z0.b, z7.b\n" - ".inst 0x45079852 // smmla z18.s, z2.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x4506980e // smmla z14.s, z0.b, z6.b\n" - ".inst 0x45069856 // smmla z22.s, z2.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x45189b8d // smmla z13.s, z28.b, z24.b\n" + ".inst 0x45189b55 // smmla z21.s, z26.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #5, MUL VL]\n" + "trn2 z3.d, z3.d, z27.d\n" + ".inst 0x45199b8a // smmla z10.s, z28.b, z25.b\n" + ".inst 0x45199b52 // smmla z18.s, z26.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x45189b8e // smmla z14.s, z28.b, z24.b\n" + ".inst 0x45189b56 // smmla z22.s, z26.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" - ".inst 0x45079853 // smmla z19.s, z2.b, z7.b\n" - ".inst 0x4506980f // smmla z15.s, z0.b, z6.b\n" - ".inst 0x45069857 // smmla z23.s, z2.b, z6.b\n" + ".inst 0x45199b8b // smmla z11.s, z28.b, z25.b\n" + ".inst 0x45199b53 // smmla z19.s, z26.b, z25.b\n" + ".inst 0x45189b8f // smmla z15.s, z28.b, z24.b\n" + ".inst 0x45189b57 // smmla z23.s, z26.b, z24.b\n" "ble 43f\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45079828 // smmla z8.s, z1.b, z7.b\n" - ".inst 0x45079870 // smmla z16.s, z3.b, z7.b\n" - ".inst 0x4506982c // smmla z12.s, z1.b, z6.b\n" - ".inst 0x45069874 // smmla z20.s, z3.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45079829 // smmla z9.s, z1.b, z7.b\n" - ".inst 0x45079871 // smmla z17.s, z3.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x4506982d // smmla z13.s, z1.b, z6.b\n" - ".inst 0x45069875 // smmla z21.s, z3.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x4507982a // smmla z10.s, z1.b, z7.b\n" - ".inst 0x45079872 // smmla z18.s, z3.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - ".inst 0x45069876 // smmla z22.s, z3.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + "ld1b { z25.b }, p5/Z, [x10]\n" + "ld1b { z24.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45199828 // smmla z8.s, z1.b, z25.b\n" + ".inst 0x45199870 // smmla z16.s, z3.b, z25.b\n" + ".inst 0x4518982c // smmla z12.s, z1.b, z24.b\n" + ".inst 0x45189874 // smmla z20.s, z3.b, z24.b\n" + "ld1b { z25.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z24.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45199829 // smmla z9.s, z1.b, z25.b\n" + ".inst 0x45199871 // smmla z17.s, z3.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x4518982d // smmla z13.s, z1.b, z24.b\n" + ".inst 0x45189875 // smmla z21.s, z3.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x4519982a // smmla z10.s, z1.b, z25.b\n" + ".inst 0x45199872 // smmla z18.s, z3.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x4518982e // smmla z14.s, z1.b, z24.b\n" + ".inst 0x45189876 // smmla z22.s, z3.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - ".inst 0x4507982b // smmla z11.s, z1.b, z7.b\n" - ".inst 0x45079873 // smmla z19.s, z3.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" - ".inst 0x45069877 // smmla z23.s, z3.b, z6.b\n" + ".inst 0x4519982b // smmla z11.s, z1.b, z25.b\n" + ".inst 0x45199873 // smmla z19.s, z3.b, z25.b\n" + ".inst 0x4518982f // smmla z15.s, z1.b, z24.b\n" + ".inst 0x45189877 // smmla z23.s, z3.b, z24.b\n" "43:" // Height 4: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 38b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "uzp1 z7.d, z8.d, z12.d\n" - "add x22, x23, x20, LSL #2\n" + "add x22, x9, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" + "uzp1 z25.d, z8.d, z12.d\n" + "add x20, x21, x20, LSL #2\n" "uzp2 z8.d, z8.d, z12.d\n" - "uzp1 z12.d, z9.d, z13.d\n" - "st1w { z7.s }, p4, [x9]\n" + "uzp1 z24.d, z9.d, z13.d\n" + "st1w { z25.s }, p4, [x9]\n" "uzp2 z9.d, z9.d, z13.d\n" - "uzp1 z13.d, z10.d, z14.d\n" - "st1w { z12.s }, p3, [x9, #1, MUL VL]\n" + "uzp1 z25.d, z10.d, z14.d\n" + "st1w { z24.s }, p3, [x9, #1, MUL VL]\n" "uzp2 z10.d, z10.d, z14.d\n" - "uzp1 z14.d, z11.d, z15.d\n" - "st1w { z13.s }, p2, [x9, #2, MUL VL]\n" + "uzp1 z24.d, z11.d, z15.d\n" + "st1w { z25.s }, p2, [x9, #2, MUL VL]\n" "uzp2 z11.d, z11.d, z15.d\n" - "uzp1 z15.d, z16.d, z20.d\n" - "st1w { z14.s }, p1, [x9, #3, MUL VL]\n" + "uzp1 z25.d, z16.d, z20.d\n" + "st1w { z24.s }, p1, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" "uzp2 z16.d, z16.d, z20.d\n" - "uzp1 z20.d, z17.d, z21.d\n" - "st1w { z8.s }, p4, [x24]\n" + "uzp1 z24.d, z17.d, z21.d\n" + "st1w { z8.s }, p4, [x22]\n" "uzp2 z17.d, z17.d, z21.d\n" "uzp1 z21.d, z18.d, z22.d\n" - "st1w { z9.s }, p3, [x24, #1, MUL VL]\n" + "st1w { z9.s }, p3, [x22, #1, MUL VL]\n" "uzp2 z18.d, z18.d, z22.d\n" - "uzp1 z22.d, z19.d, z23.d\n" - "st1w { z10.s }, p2, [x24, #2, MUL VL]\n" + "uzp1 z20.d, z19.d, z23.d\n" + "st1w { z10.s }, p2, [x22, #2, MUL VL]\n" "uzp2 z19.d, z19.d, z23.d\n" - "st1w { z11.s }, p1, [x24, #3, MUL VL]\n" - "st1w { z15.s }, p4, [x23]\n" - "st1w { z20.s }, p3, [x23, #1, MUL VL]\n" - "st1w { z21.s }, p2, [x23, #2, MUL VL]\n" - "st1w { z22.s }, p1, [x23, #3, MUL VL]\n" - "st1w { z16.s }, p4, [x22]\n" - "st1w { z17.s }, p3, [x22, #1, MUL VL]\n" - "st1w { z18.s }, p2, [x22, #2, MUL VL]\n" - "st1w { z19.s }, p1, [x22, #3, MUL VL]\n" + "st1w { z11.s }, p1, [x22, #3, MUL VL]\n" + "st1w { z25.s }, p4, [x21]\n" + "st1w { z24.s }, p3, [x21, #1, MUL VL]\n" + "st1w { z21.s }, p2, [x21, #2, MUL VL]\n" + "st1w { z20.s }, p1, [x21, #3, MUL VL]\n" + "st1w { z16.s }, p4, [x20]\n" + "st1w { z17.s }, p3, [x20, #1, MUL VL]\n" + "st1w { z18.s }, p2, [x20, #2, MUL VL]\n" + "st1w { z19.s }, p1, [x20, #3, MUL VL]\n" "44:" // Height 4: Writeback done "decw x11, ALL, MUL #4\n" "cmp x11, XZR\n" @@ -959,54 +959,54 @@ void sve_hybrid_s8s32_mmla_6x4VL ( "whilelt p1.s, x20, x11\n" "tbz %x[flags], #0, 47f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x9]\n" + "add x23, x9, x20, LSL #2\n" "add x22, x23, x20, LSL #2\n" + "ld1w { z19.s }, p4/Z, [x9]\n" "add x21, x22, x20, LSL #2\n" - "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n" + "add x20, x21, x20, LSL #2\n" + "ld1w { z17.s }, p3/Z, [x9, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x24]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z17.s }, p4/Z, [x23]\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" - "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x23]\n" + "zip1 z8.d, z19.d, z12.d\n" + "zip2 z12.d, z19.d, z12.d\n" + "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n" + "zip1 z9.d, z17.d, z13.d\n" + "zip2 z13.d, z17.d, z13.d\n" + "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z17.s }, p4/Z, [x22]\n" + "zip1 z10.d, z18.d, z14.d\n" + "zip2 z14.d, z18.d, z14.d\n" + "ld1w { z18.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z19.s }, p2/Z, [x22, #2, MUL VL]\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" - "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x22]\n" + "ld1w { z24.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x21]\n" "zip1 z16.d, z17.d, z20.d\n" "zip2 z20.d, z17.d, z20.d\n" - "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n" + "ld1w { z21.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x21, #2, MUL VL]\n" "zip1 z17.d, z18.d, z21.d\n" "zip2 z21.d, z18.d, z21.d\n" - "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n" - "ld1w { z25.s }, p4/Z, [x21]\n" + "ld1w { z23.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z25.s }, p4/Z, [x20]\n" "zip1 z18.d, z19.d, z22.d\n" "zip2 z22.d, z19.d, z22.d\n" - "ld1w { z26.s }, p3/Z, [x21, #1, MUL VL]\n" - "ld1w { z27.s }, p2/Z, [x21, #2, MUL VL]\n" + "ld1w { z26.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z27.s }, p2/Z, [x20, #2, MUL VL]\n" "zip1 z19.d, z24.d, z23.d\n" "zip2 z23.d, z24.d, z23.d\n" - "ld1w { z6.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z0.s }, p1/Z, [x20, #3, MUL VL]\n" "zip1 z24.d, z25.d, z28.d\n" "zip2 z28.d, z25.d, z28.d\n" "zip1 z25.d, z26.d, z29.d\n" "zip2 z29.d, z26.d, z29.d\n" "zip1 z26.d, z27.d, z30.d\n" "zip2 z30.d, z27.d, z30.d\n" - "zip1 z27.d, z6.d, z31.d\n" - "zip2 z31.d, z6.d, z31.d\n" + "zip1 z27.d, z0.d, z31.d\n" + "zip2 z31.d, z0.d, z31.d\n" "b 48f\n" "47:" // Height 5: no accumulate "mov z8.s, #0x0\n" @@ -1038,15 +1038,15 @@ void sve_hybrid_s8s32_mmla_6x4VL ( "49:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 50f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 51f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -1057,231 +1057,231 @@ void sve_hybrid_s8s32_mmla_6x4VL ( "b 51f\n" "50:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" "51:" // Height 5: input setup done "cmp x27, #0x10\n" "ble 53f\n" "52:" // Height 5: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" - "ld1rqb { z3.b }, p0/Z, [x24]\n" - "ld1rqb { z4.b }, p0/Z, [x23]\n" - "trn1 z0.d, z1.d, z2.d\n" - "trn2 z1.d, z1.d, z2.d\n" - "ld1rqb { z5.b }, p0/Z, [x22]\n" - "trn1 z2.d, z3.d, z4.d\n" - "trn2 z3.d, z3.d, z4.d\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "trn1 z4.d, z5.d, z6.d\n" - "trn2 z5.d, z5.d, z6.d\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45079808 // smmla z8.s, z0.b, z7.b\n" - ".inst 0x45079850 // smmla z16.s, z2.b, z7.b\n" - ".inst 0x45079898 // smmla z24.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1rqb { z6.b }, p0/Z, [x26]\n" + "ld1rqb { z1.b }, p0/Z, [x25]\n" + "ld1rqb { z7.b }, p0/Z, [x24]\n" + "ld1rqb { z2.b }, p0/Z, [x23]\n" + "trn1 z5.d, z6.d, z1.d\n" + "trn2 z6.d, z6.d, z1.d\n" + "ld1rqb { z4.b }, p0/Z, [x22]\n" + "trn1 z3.d, z7.d, z2.d\n" + "trn2 z7.d, z7.d, z2.d\n" + "ld1b { z1.b }, p5/Z, [x10]\n" + "trn1 z2.d, z4.d, z0.d\n" + "trn2 z4.d, z4.d, z0.d\n" + "ld1b { z0.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x450198a8 // smmla z8.s, z5.b, z1.b\n" + ".inst 0x45019870 // smmla z16.s, z3.b, z1.b\n" + ".inst 0x45019858 // smmla z24.s, z2.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #2, MUL VL]\n" "sub x27, x27, #0x10\n" - ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n" - ".inst 0x45069854 // smmla z20.s, z2.b, z6.b\n" + ".inst 0x450098ac // smmla z12.s, z5.b, z0.b\n" + ".inst 0x45009874 // smmla z20.s, z3.b, z0.b\n" "cmp x27, #0x10\n" "add x26, x26, #0x10\n" - ".inst 0x4506989c // smmla z28.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" + ".inst 0x4500985c // smmla z28.s, z2.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x450198a9 // smmla z9.s, z5.b, z1.b\n" "add x25, x25, #0x10\n" - ".inst 0x45079851 // smmla z17.s, z2.b, z7.b\n" - ".inst 0x45079899 // smmla z25.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x45019871 // smmla z17.s, z3.b, z1.b\n" + ".inst 0x45019859 // smmla z25.s, z2.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #4, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x4506980d // smmla z13.s, z0.b, z6.b\n" - ".inst 0x45069855 // smmla z21.s, z2.b, z6.b\n" + ".inst 0x450098ad // smmla z13.s, z5.b, z0.b\n" + ".inst 0x45009875 // smmla z21.s, z3.b, z0.b\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - ".inst 0x4506989d // smmla z29.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x4507980a // smmla z10.s, z0.b, z7.b\n" - ".inst 0x45079852 // smmla z18.s, z2.b, z7.b\n" - ".inst 0x4507989a // smmla z26.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x4506980e // smmla z14.s, z0.b, z6.b\n" - ".inst 0x45069856 // smmla z22.s, z2.b, z6.b\n" - ".inst 0x4506989e // smmla z30.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x4500985d // smmla z29.s, z2.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x450198aa // smmla z10.s, z5.b, z1.b\n" + ".inst 0x45019872 // smmla z18.s, z3.b, z1.b\n" + ".inst 0x4501985a // smmla z26.s, z2.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x450098ae // smmla z14.s, z5.b, z0.b\n" + ".inst 0x45009876 // smmla z22.s, z3.b, z0.b\n" + ".inst 0x4500985e // smmla z30.s, z2.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" - ".inst 0x45079853 // smmla z19.s, z2.b, z7.b\n" - ".inst 0x4507989b // smmla z27.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-8, MUL VL]\n" - ".inst 0x4506980f // smmla z15.s, z0.b, z6.b\n" - ".inst 0x45069857 // smmla z23.s, z2.b, z6.b\n" - ".inst 0x4506989f // smmla z31.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-7, MUL VL]\n" - ".inst 0x45079828 // smmla z8.s, z1.b, z7.b\n" - ".inst 0x45079870 // smmla z16.s, z3.b, z7.b\n" - ".inst 0x450798b8 // smmla z24.s, z5.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-6, MUL VL]\n" - ".inst 0x4506982c // smmla z12.s, z1.b, z6.b\n" - ".inst 0x45069874 // smmla z20.s, z3.b, z6.b\n" - ".inst 0x450698bc // smmla z28.s, z5.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-5, MUL VL]\n" - ".inst 0x45079829 // smmla z9.s, z1.b, z7.b\n" - ".inst 0x45079871 // smmla z17.s, z3.b, z7.b\n" - ".inst 0x450798b9 // smmla z25.s, z5.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-4, MUL VL]\n" - ".inst 0x4506982d // smmla z13.s, z1.b, z6.b\n" - ".inst 0x45069875 // smmla z21.s, z3.b, z6.b\n" - ".inst 0x450698bd // smmla z29.s, z5.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-3, MUL VL]\n" - ".inst 0x4507982a // smmla z10.s, z1.b, z7.b\n" - ".inst 0x45079872 // smmla z18.s, z3.b, z7.b\n" - ".inst 0x450798ba // smmla z26.s, z5.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-2, MUL VL]\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - ".inst 0x45069876 // smmla z22.s, z3.b, z6.b\n" - ".inst 0x450698be // smmla z30.s, z5.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-1, MUL VL]\n" - ".inst 0x4507982b // smmla z11.s, z1.b, z7.b\n" - ".inst 0x45079873 // smmla z19.s, z3.b, z7.b\n" - ".inst 0x450798bb // smmla z27.s, z5.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" - ".inst 0x45069877 // smmla z23.s, z3.b, z6.b\n" - ".inst 0x450698bf // smmla z31.s, z5.b, z6.b\n" + ".inst 0x450198ab // smmla z11.s, z5.b, z1.b\n" + ".inst 0x45019873 // smmla z19.s, z3.b, z1.b\n" + ".inst 0x4501985b // smmla z27.s, z2.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #-8, MUL VL]\n" + ".inst 0x450098af // smmla z15.s, z5.b, z0.b\n" + ".inst 0x45009877 // smmla z23.s, z3.b, z0.b\n" + ".inst 0x4500985f // smmla z31.s, z2.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #-7, MUL VL]\n" + ".inst 0x450198c8 // smmla z8.s, z6.b, z1.b\n" + ".inst 0x450198f0 // smmla z16.s, z7.b, z1.b\n" + ".inst 0x45019898 // smmla z24.s, z4.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #-6, MUL VL]\n" + ".inst 0x450098cc // smmla z12.s, z6.b, z0.b\n" + ".inst 0x450098f4 // smmla z20.s, z7.b, z0.b\n" + ".inst 0x4500989c // smmla z28.s, z4.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #-5, MUL VL]\n" + ".inst 0x450198c9 // smmla z9.s, z6.b, z1.b\n" + ".inst 0x450198f1 // smmla z17.s, z7.b, z1.b\n" + ".inst 0x45019899 // smmla z25.s, z4.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #-4, MUL VL]\n" + ".inst 0x450098cd // smmla z13.s, z6.b, z0.b\n" + ".inst 0x450098f5 // smmla z21.s, z7.b, z0.b\n" + ".inst 0x4500989d // smmla z29.s, z4.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #-3, MUL VL]\n" + ".inst 0x450198ca // smmla z10.s, z6.b, z1.b\n" + ".inst 0x450198f2 // smmla z18.s, z7.b, z1.b\n" + ".inst 0x4501989a // smmla z26.s, z4.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #-2, MUL VL]\n" + ".inst 0x450098ce // smmla z14.s, z6.b, z0.b\n" + ".inst 0x450098f6 // smmla z22.s, z7.b, z0.b\n" + ".inst 0x4500989e // smmla z30.s, z4.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #-1, MUL VL]\n" + ".inst 0x450198cb // smmla z11.s, z6.b, z1.b\n" + ".inst 0x450198f3 // smmla z19.s, z7.b, z1.b\n" + ".inst 0x4501989b // smmla z27.s, z4.b, z1.b\n" + ".inst 0x450098cf // smmla z15.s, z6.b, z0.b\n" + ".inst 0x450098f7 // smmla z23.s, z7.b, z0.b\n" + ".inst 0x4500989f // smmla z31.s, z4.b, z0.b\n" "bgt 52b\n" "53:" // Height 5: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" + "ld1rqb { z4.b }, p0/Z, [x25]\n" "ld1rqb { z3.b }, p0/Z, [x24]\n" - "ld1rqb { z4.b }, p0/Z, [x23]\n" - "trn1 z0.d, z1.d, z2.d\n" - "trn2 z1.d, z1.d, z2.d\n" + "ld1rqb { z2.b }, p0/Z, [x23]\n" + "trn1 z7.d, z1.d, z4.d\n" + "trn2 z1.d, z1.d, z4.d\n" "ld1rqb { z5.b }, p0/Z, [x22]\n" - "trn1 z2.d, z3.d, z4.d\n" - "trn2 z3.d, z3.d, z4.d\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "trn1 z4.d, z5.d, z6.d\n" - "trn2 z5.d, z5.d, z6.d\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45079808 // smmla z8.s, z0.b, z7.b\n" - ".inst 0x45079850 // smmla z16.s, z2.b, z7.b\n" - ".inst 0x45079898 // smmla z24.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" + "trn1 z6.d, z3.d, z2.d\n" + "trn2 z3.d, z3.d, z2.d\n" + "ld1b { z2.b }, p5/Z, [x10]\n" + "trn1 z4.d, z5.d, z0.d\n" + "trn2 z5.d, z5.d, z0.d\n" + "ld1b { z0.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x450298e8 // smmla z8.s, z7.b, z2.b\n" + ".inst 0x450298d0 // smmla z16.s, z6.b, z2.b\n" + ".inst 0x45029898 // smmla z24.s, z4.b, z2.b\n" + "ld1b { z2.b }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x8\n" - ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n" - ".inst 0x45069854 // smmla z20.s, z2.b, z6.b\n" - ".inst 0x4506989c // smmla z28.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" - ".inst 0x45079851 // smmla z17.s, z2.b, z7.b\n" - ".inst 0x45079899 // smmla z25.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x4506980d // smmla z13.s, z0.b, z6.b\n" - ".inst 0x45069855 // smmla z21.s, z2.b, z6.b\n" - ".inst 0x4506989d // smmla z29.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x4507980a // smmla z10.s, z0.b, z7.b\n" - ".inst 0x45079852 // smmla z18.s, z2.b, z7.b\n" - ".inst 0x4507989a // smmla z26.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x4506980e // smmla z14.s, z0.b, z6.b\n" - ".inst 0x45069856 // smmla z22.s, z2.b, z6.b\n" - ".inst 0x4506989e // smmla z30.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" + ".inst 0x450098ec // smmla z12.s, z7.b, z0.b\n" + ".inst 0x450098d4 // smmla z20.s, z6.b, z0.b\n" + ".inst 0x4500989c // smmla z28.s, z4.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x450298e9 // smmla z9.s, z7.b, z2.b\n" + ".inst 0x450298d1 // smmla z17.s, z6.b, z2.b\n" + ".inst 0x45029899 // smmla z25.s, z4.b, z2.b\n" + "ld1b { z2.b }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x450098ed // smmla z13.s, z7.b, z0.b\n" + ".inst 0x450098d5 // smmla z21.s, z6.b, z0.b\n" + ".inst 0x4500989d // smmla z29.s, z4.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x450298ea // smmla z10.s, z7.b, z2.b\n" + ".inst 0x450298d2 // smmla z18.s, z6.b, z2.b\n" + ".inst 0x4502989a // smmla z26.s, z4.b, z2.b\n" + "ld1b { z2.b }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x450098ee // smmla z14.s, z7.b, z0.b\n" + ".inst 0x450098d6 // smmla z22.s, z6.b, z0.b\n" + ".inst 0x4500989e // smmla z30.s, z4.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x450298eb // smmla z11.s, z7.b, z2.b\n" "addvl x10, x10, #8\n" - ".inst 0x45079853 // smmla z19.s, z2.b, z7.b\n" - ".inst 0x4507989b // smmla z27.s, z4.b, z7.b\n" - ".inst 0x4506980f // smmla z15.s, z0.b, z6.b\n" - ".inst 0x45069857 // smmla z23.s, z2.b, z6.b\n" - ".inst 0x4506989f // smmla z31.s, z4.b, z6.b\n" + ".inst 0x450298d3 // smmla z19.s, z6.b, z2.b\n" + ".inst 0x4502989b // smmla z27.s, z4.b, z2.b\n" + ".inst 0x450098ef // smmla z15.s, z7.b, z0.b\n" + ".inst 0x450098d7 // smmla z23.s, z6.b, z0.b\n" + ".inst 0x4500989f // smmla z31.s, z4.b, z0.b\n" "ble 54f\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45079828 // smmla z8.s, z1.b, z7.b\n" - ".inst 0x45079870 // smmla z16.s, z3.b, z7.b\n" - ".inst 0x450798b8 // smmla z24.s, z5.b, z7.b\n" - ".inst 0x4506982c // smmla z12.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x45069874 // smmla z20.s, z3.b, z6.b\n" - ".inst 0x450698bc // smmla z28.s, z5.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45079829 // smmla z9.s, z1.b, z7.b\n" - ".inst 0x45079871 // smmla z17.s, z3.b, z7.b\n" - ".inst 0x450798b9 // smmla z25.s, z5.b, z7.b\n" - ".inst 0x4506982d // smmla z13.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x45069875 // smmla z21.s, z3.b, z6.b\n" - ".inst 0x450698bd // smmla z29.s, z5.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x4507982a // smmla z10.s, z1.b, z7.b\n" - ".inst 0x45079872 // smmla z18.s, z3.b, z7.b\n" - ".inst 0x450798ba // smmla z26.s, z5.b, z7.b\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x45069876 // smmla z22.s, z3.b, z6.b\n" - ".inst 0x450698be // smmla z30.s, z5.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + "ld1b { z2.b }, p5/Z, [x10]\n" + "ld1b { z0.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45029828 // smmla z8.s, z1.b, z2.b\n" + ".inst 0x45029870 // smmla z16.s, z3.b, z2.b\n" + ".inst 0x450298b8 // smmla z24.s, z5.b, z2.b\n" + ".inst 0x4500982c // smmla z12.s, z1.b, z0.b\n" + "ld1b { z2.b }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x45009874 // smmla z20.s, z3.b, z0.b\n" + ".inst 0x450098bc // smmla z28.s, z5.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45029829 // smmla z9.s, z1.b, z2.b\n" + ".inst 0x45029871 // smmla z17.s, z3.b, z2.b\n" + ".inst 0x450298b9 // smmla z25.s, z5.b, z2.b\n" + ".inst 0x4500982d // smmla z13.s, z1.b, z0.b\n" + "ld1b { z2.b }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x45009875 // smmla z21.s, z3.b, z0.b\n" + ".inst 0x450098bd // smmla z29.s, z5.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x4502982a // smmla z10.s, z1.b, z2.b\n" + ".inst 0x45029872 // smmla z18.s, z3.b, z2.b\n" + ".inst 0x450298ba // smmla z26.s, z5.b, z2.b\n" + ".inst 0x4500982e // smmla z14.s, z1.b, z0.b\n" + "ld1b { z2.b }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x45009876 // smmla z22.s, z3.b, z0.b\n" + ".inst 0x450098be // smmla z30.s, z5.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - ".inst 0x4507982b // smmla z11.s, z1.b, z7.b\n" - ".inst 0x45079873 // smmla z19.s, z3.b, z7.b\n" - ".inst 0x450798bb // smmla z27.s, z5.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" - ".inst 0x45069877 // smmla z23.s, z3.b, z6.b\n" - ".inst 0x450698bf // smmla z31.s, z5.b, z6.b\n" + ".inst 0x4502982b // smmla z11.s, z1.b, z2.b\n" + ".inst 0x45029873 // smmla z19.s, z3.b, z2.b\n" + ".inst 0x450298bb // smmla z27.s, z5.b, z2.b\n" + ".inst 0x4500982f // smmla z15.s, z1.b, z0.b\n" + ".inst 0x45009877 // smmla z23.s, z3.b, z0.b\n" + ".inst 0x450098bf // smmla z31.s, z5.b, z0.b\n" "54:" // Height 5: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 49b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "uzp1 z7.d, z8.d, z12.d\n" + "add x23, x9, x20, LSL #2\n" "add x22, x23, x20, LSL #2\n" + "uzp1 z2.d, z8.d, z12.d\n" "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "uzp2 z8.d, z8.d, z12.d\n" - "uzp1 z12.d, z9.d, z13.d\n" + "uzp1 z1.d, z9.d, z13.d\n" "uzp2 z9.d, z9.d, z13.d\n" - "uzp1 z13.d, z10.d, z14.d\n" - "st1w { z7.s }, p4, [x9]\n" + "uzp1 z0.d, z10.d, z14.d\n" + "st1w { z2.s }, p4, [x9]\n" "uzp2 z10.d, z10.d, z14.d\n" - "uzp1 z14.d, z11.d, z15.d\n" - "st1w { z12.s }, p3, [x9, #1, MUL VL]\n" + "uzp1 z2.d, z11.d, z15.d\n" + "st1w { z1.s }, p3, [x9, #1, MUL VL]\n" "uzp2 z11.d, z11.d, z15.d\n" - "uzp1 z15.d, z16.d, z20.d\n" - "st1w { z13.s }, p2, [x9, #2, MUL VL]\n" + "uzp1 z1.d, z16.d, z20.d\n" + "st1w { z0.s }, p2, [x9, #2, MUL VL]\n" "uzp2 z16.d, z16.d, z20.d\n" - "uzp1 z20.d, z17.d, z21.d\n" - "st1w { z14.s }, p1, [x9, #3, MUL VL]\n" + "uzp1 z0.d, z17.d, z21.d\n" + "st1w { z2.s }, p1, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" "uzp2 z17.d, z17.d, z21.d\n" "uzp1 z21.d, z18.d, z22.d\n" - "st1w { z8.s }, p4, [x24]\n" + "st1w { z8.s }, p4, [x23]\n" "uzp2 z18.d, z18.d, z22.d\n" - "uzp1 z22.d, z19.d, z23.d\n" - "st1w { z9.s }, p3, [x24, #1, MUL VL]\n" + "uzp1 z20.d, z19.d, z23.d\n" + "st1w { z9.s }, p3, [x23, #1, MUL VL]\n" "uzp2 z19.d, z19.d, z23.d\n" "uzp1 z24.d, z24.d, z28.d\n" - "st1w { z10.s }, p2, [x24, #2, MUL VL]\n" + "st1w { z10.s }, p2, [x23, #2, MUL VL]\n" "uzp1 z25.d, z25.d, z29.d\n" "uzp1 z26.d, z26.d, z30.d\n" - "st1w { z11.s }, p1, [x24, #3, MUL VL]\n" + "st1w { z11.s }, p1, [x23, #3, MUL VL]\n" "uzp1 z27.d, z27.d, z31.d\n" - "st1w { z15.s }, p4, [x23]\n" - "st1w { z20.s }, p3, [x23, #1, MUL VL]\n" - "st1w { z21.s }, p2, [x23, #2, MUL VL]\n" - "st1w { z22.s }, p1, [x23, #3, MUL VL]\n" - "st1w { z16.s }, p4, [x22]\n" - "st1w { z17.s }, p3, [x22, #1, MUL VL]\n" - "st1w { z18.s }, p2, [x22, #2, MUL VL]\n" - "st1w { z19.s }, p1, [x22, #3, MUL VL]\n" - "st1w { z24.s }, p4, [x21]\n" - "st1w { z25.s }, p3, [x21, #1, MUL VL]\n" - "st1w { z26.s }, p2, [x21, #2, MUL VL]\n" - "st1w { z27.s }, p1, [x21, #3, MUL VL]\n" + "st1w { z1.s }, p4, [x22]\n" + "st1w { z0.s }, p3, [x22, #1, MUL VL]\n" + "st1w { z21.s }, p2, [x22, #2, MUL VL]\n" + "st1w { z20.s }, p1, [x22, #3, MUL VL]\n" + "st1w { z16.s }, p4, [x21]\n" + "st1w { z17.s }, p3, [x21, #1, MUL VL]\n" + "st1w { z18.s }, p2, [x21, #2, MUL VL]\n" + "st1w { z19.s }, p1, [x21, #3, MUL VL]\n" + "st1w { z24.s }, p4, [x20]\n" + "st1w { z25.s }, p3, [x20, #1, MUL VL]\n" + "st1w { z26.s }, p2, [x20, #2, MUL VL]\n" + "st1w { z27.s }, p1, [x20, #3, MUL VL]\n" "55:" // Height 5: Writeback done "decw x11, ALL, MUL #4\n" "cmp x11, XZR\n" @@ -1307,26 +1307,26 @@ void sve_hybrid_s8s32_mmla_6x4VL ( "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "add x24, x9, x20, LSL #2\n" "add x23, x24, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x9]\n" + "ld1w { z17.s }, p4/Z, [x9]\n" "add x22, x23, x20, LSL #2\n" "add x21, x22, x20, LSL #2\n" - "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n" + "ld1w { z18.s }, p3/Z, [x9, #1, MUL VL]\n" + "ld1w { z20.s }, p2/Z, [x9, #2, MUL VL]\n" "add x20, x21, x20, LSL #2\n" "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n" "ld1w { z12.s }, p4/Z, [x24]\n" - "zip1 z8.d, z9.d, z12.d\n" + "zip1 z8.d, z17.d, z12.d\n" "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n" "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n" - "zip2 z12.d, z9.d, z12.d\n" - "zip1 z9.d, z10.d, z13.d\n" + "zip2 z12.d, z17.d, z12.d\n" + "zip1 z9.d, z18.d, z13.d\n" "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n" "ld1w { z17.s }, p4/Z, [x23]\n" - "zip2 z13.d, z10.d, z13.d\n" - "zip1 z10.d, z11.d, z14.d\n" + "zip2 z13.d, z18.d, z13.d\n" + "zip1 z10.d, z20.d, z14.d\n" "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n" "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n" - "zip2 z14.d, z11.d, z14.d\n" + "zip2 z14.d, z20.d, z14.d\n" "zip1 z11.d, z16.d, z15.d\n" "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n" "ld1w { z20.s }, p4/Z, [x22]\n" @@ -1344,7 +1344,7 @@ void sve_hybrid_s8s32_mmla_6x4VL ( "ld1w { z27.s }, p2/Z, [x21, #2, MUL VL]\n" "zip2 z22.d, z19.d, z22.d\n" "zip1 z19.d, z24.d, z23.d\n" - "ld1w { z6.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z0.s }, p1/Z, [x21, #3, MUL VL]\n" "ld1w { z28.s }, p4/Z, [x20]\n" "zip2 z23.d, z24.d, z23.d\n" "zip1 z24.d, z25.d, z28.d\n" @@ -1356,8 +1356,8 @@ void sve_hybrid_s8s32_mmla_6x4VL ( "zip2 z29.d, z26.d, z29.d\n" "zip1 z26.d, z27.d, z30.d\n" "zip2 z30.d, z27.d, z30.d\n" - "zip1 z27.d, z6.d, z31.d\n" - "zip2 z31.d, z6.d, z31.d\n" + "zip1 z27.d, z0.d, z31.d\n" + "zip2 z31.d, z0.d, z31.d\n" "b 59f\n" "58:" // Height 6: no accumulate "mov z8.s, #0x0\n" @@ -1389,16 +1389,16 @@ void sve_hybrid_s8s32_mmla_6x4VL ( "60:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 61f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 62f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -1410,184 +1410,184 @@ void sve_hybrid_s8s32_mmla_6x4VL ( "b 62f\n" "61:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" + "add x21, x22, x21\n" "62:" // Height 6: input setup done "cmp x27, #0x10\n" "ble 64f\n" "63:" // Height 6: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1rqb { z3.b }, p0/Z, [x24]\n" - "ld1rqb { z4.b }, p0/Z, [x23]\n" - "trn2 z1.d, z1.d, z2.d\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1rqb { z5.b }, p0/Z, [x22]\n" - "ld1rqb { z6.b }, p0/Z, [x21]\n" - "trn2 z3.d, z3.d, z4.d\n" - "trn1 z4.d, z5.d, z6.d\n" - "trn2 z5.d, z5.d, z6.d\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45079808 // smmla z8.s, z0.b, z7.b\n" - ".inst 0x45079850 // smmla z16.s, z2.b, z7.b\n" - ".inst 0x45079898 // smmla z24.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1rqb { z7.b }, p0/Z, [x26]\n" + "ld1rqb { z0.b }, p0/Z, [x25]\n" + "trn1 z6.d, z7.d, z0.d\n" + "ld1rqb { z5.b }, p0/Z, [x24]\n" + "ld1rqb { z1.b }, p0/Z, [x23]\n" + "trn2 z7.d, z7.d, z0.d\n" + "trn1 z4.d, z5.d, z1.d\n" + "ld1rqb { z3.b }, p0/Z, [x22]\n" + "ld1rqb { z0.b }, p0/Z, [x21]\n" + "trn2 z5.d, z5.d, z1.d\n" + "trn1 z2.d, z3.d, z0.d\n" + "trn2 z3.d, z3.d, z0.d\n" + "ld1b { z1.b }, p5/Z, [x10]\n" + "ld1b { z0.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x450198c8 // smmla z8.s, z6.b, z1.b\n" + ".inst 0x45019890 // smmla z16.s, z4.b, z1.b\n" + ".inst 0x45019858 // smmla z24.s, z2.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #2, MUL VL]\n" "sub x27, x27, #0x10\n" - ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n" - ".inst 0x45069854 // smmla z20.s, z2.b, z6.b\n" + ".inst 0x450098cc // smmla z12.s, z6.b, z0.b\n" + ".inst 0x45009894 // smmla z20.s, z4.b, z0.b\n" "cmp x27, #0x10\n" "add x26, x26, #0x10\n" - ".inst 0x4506989c // smmla z28.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" + ".inst 0x4500985c // smmla z28.s, z2.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x450198c9 // smmla z9.s, z6.b, z1.b\n" "add x25, x25, #0x10\n" - ".inst 0x45079851 // smmla z17.s, z2.b, z7.b\n" - ".inst 0x45079899 // smmla z25.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x45019891 // smmla z17.s, z4.b, z1.b\n" + ".inst 0x45019859 // smmla z25.s, z2.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #4, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x4506980d // smmla z13.s, z0.b, z6.b\n" - ".inst 0x45069855 // smmla z21.s, z2.b, z6.b\n" + ".inst 0x450098cd // smmla z13.s, z6.b, z0.b\n" + ".inst 0x45009895 // smmla z21.s, z4.b, z0.b\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - ".inst 0x4506989d // smmla z29.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x4507980a // smmla z10.s, z0.b, z7.b\n" + ".inst 0x4500985d // smmla z29.s, z2.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x450198ca // smmla z10.s, z6.b, z1.b\n" "add x21, x21, #0x10\n" - ".inst 0x45079852 // smmla z18.s, z2.b, z7.b\n" - ".inst 0x4507989a // smmla z26.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x4506980e // smmla z14.s, z0.b, z6.b\n" - ".inst 0x45069856 // smmla z22.s, z2.b, z6.b\n" - ".inst 0x4506989e // smmla z30.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x45019892 // smmla z18.s, z4.b, z1.b\n" + ".inst 0x4501985a // smmla z26.s, z2.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x450098ce // smmla z14.s, z6.b, z0.b\n" + ".inst 0x45009896 // smmla z22.s, z4.b, z0.b\n" + ".inst 0x4500985e // smmla z30.s, z2.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" - ".inst 0x45079853 // smmla z19.s, z2.b, z7.b\n" - ".inst 0x4507989b // smmla z27.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-8, MUL VL]\n" - ".inst 0x4506980f // smmla z15.s, z0.b, z6.b\n" - ".inst 0x45069857 // smmla z23.s, z2.b, z6.b\n" - ".inst 0x4506989f // smmla z31.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-7, MUL VL]\n" - ".inst 0x45079828 // smmla z8.s, z1.b, z7.b\n" - ".inst 0x45079870 // smmla z16.s, z3.b, z7.b\n" - ".inst 0x450798b8 // smmla z24.s, z5.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-6, MUL VL]\n" - ".inst 0x4506982c // smmla z12.s, z1.b, z6.b\n" - ".inst 0x45069874 // smmla z20.s, z3.b, z6.b\n" - ".inst 0x450698bc // smmla z28.s, z5.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-5, MUL VL]\n" - ".inst 0x45079829 // smmla z9.s, z1.b, z7.b\n" - ".inst 0x45079871 // smmla z17.s, z3.b, z7.b\n" - ".inst 0x450798b9 // smmla z25.s, z5.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-4, MUL VL]\n" - ".inst 0x4506982d // smmla z13.s, z1.b, z6.b\n" - ".inst 0x45069875 // smmla z21.s, z3.b, z6.b\n" - ".inst 0x450698bd // smmla z29.s, z5.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-3, MUL VL]\n" - ".inst 0x4507982a // smmla z10.s, z1.b, z7.b\n" - ".inst 0x45079872 // smmla z18.s, z3.b, z7.b\n" - ".inst 0x450798ba // smmla z26.s, z5.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-2, MUL VL]\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - ".inst 0x45069876 // smmla z22.s, z3.b, z6.b\n" - ".inst 0x450698be // smmla z30.s, z5.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-1, MUL VL]\n" - ".inst 0x4507982b // smmla z11.s, z1.b, z7.b\n" - ".inst 0x45079873 // smmla z19.s, z3.b, z7.b\n" - ".inst 0x450798bb // smmla z27.s, z5.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" - ".inst 0x45069877 // smmla z23.s, z3.b, z6.b\n" - ".inst 0x450698bf // smmla z31.s, z5.b, z6.b\n" + ".inst 0x450198cb // smmla z11.s, z6.b, z1.b\n" + ".inst 0x45019893 // smmla z19.s, z4.b, z1.b\n" + ".inst 0x4501985b // smmla z27.s, z2.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #-8, MUL VL]\n" + ".inst 0x450098cf // smmla z15.s, z6.b, z0.b\n" + ".inst 0x45009897 // smmla z23.s, z4.b, z0.b\n" + ".inst 0x4500985f // smmla z31.s, z2.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #-7, MUL VL]\n" + ".inst 0x450198e8 // smmla z8.s, z7.b, z1.b\n" + ".inst 0x450198b0 // smmla z16.s, z5.b, z1.b\n" + ".inst 0x45019878 // smmla z24.s, z3.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #-6, MUL VL]\n" + ".inst 0x450098ec // smmla z12.s, z7.b, z0.b\n" + ".inst 0x450098b4 // smmla z20.s, z5.b, z0.b\n" + ".inst 0x4500987c // smmla z28.s, z3.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #-5, MUL VL]\n" + ".inst 0x450198e9 // smmla z9.s, z7.b, z1.b\n" + ".inst 0x450198b1 // smmla z17.s, z5.b, z1.b\n" + ".inst 0x45019879 // smmla z25.s, z3.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #-4, MUL VL]\n" + ".inst 0x450098ed // smmla z13.s, z7.b, z0.b\n" + ".inst 0x450098b5 // smmla z21.s, z5.b, z0.b\n" + ".inst 0x4500987d // smmla z29.s, z3.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #-3, MUL VL]\n" + ".inst 0x450198ea // smmla z10.s, z7.b, z1.b\n" + ".inst 0x450198b2 // smmla z18.s, z5.b, z1.b\n" + ".inst 0x4501987a // smmla z26.s, z3.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #-2, MUL VL]\n" + ".inst 0x450098ee // smmla z14.s, z7.b, z0.b\n" + ".inst 0x450098b6 // smmla z22.s, z5.b, z0.b\n" + ".inst 0x4500987e // smmla z30.s, z3.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #-1, MUL VL]\n" + ".inst 0x450198eb // smmla z11.s, z7.b, z1.b\n" + ".inst 0x450198b3 // smmla z19.s, z5.b, z1.b\n" + ".inst 0x4501987b // smmla z27.s, z3.b, z1.b\n" + ".inst 0x450098ef // smmla z15.s, z7.b, z0.b\n" + ".inst 0x450098b7 // smmla z23.s, z5.b, z0.b\n" + ".inst 0x4500987f // smmla z31.s, z3.b, z0.b\n" "bgt 63b\n" "64:" // Height 6: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" + "ld1rqb { z0.b }, p0/Z, [x25]\n" + "trn1 z7.d, z1.d, z0.d\n" "ld1rqb { z3.b }, p0/Z, [x24]\n" - "ld1rqb { z4.b }, p0/Z, [x23]\n" - "trn2 z1.d, z1.d, z2.d\n" - "trn1 z2.d, z3.d, z4.d\n" + "ld1rqb { z2.b }, p0/Z, [x23]\n" + "trn2 z1.d, z1.d, z0.d\n" + "trn1 z6.d, z3.d, z2.d\n" "ld1rqb { z5.b }, p0/Z, [x22]\n" - "ld1rqb { z6.b }, p0/Z, [x21]\n" - "trn2 z3.d, z3.d, z4.d\n" - "trn1 z4.d, z5.d, z6.d\n" - "trn2 z5.d, z5.d, z6.d\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45079808 // smmla z8.s, z0.b, z7.b\n" - ".inst 0x45079850 // smmla z16.s, z2.b, z7.b\n" - ".inst 0x45079898 // smmla z24.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1rqb { z0.b }, p0/Z, [x21]\n" + "trn2 z3.d, z3.d, z2.d\n" + "trn1 z4.d, z5.d, z0.d\n" + "trn2 z5.d, z5.d, z0.d\n" + "ld1b { z2.b }, p5/Z, [x10]\n" + "ld1b { z0.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x450298e8 // smmla z8.s, z7.b, z2.b\n" + ".inst 0x450298d0 // smmla z16.s, z6.b, z2.b\n" + ".inst 0x45029898 // smmla z24.s, z4.b, z2.b\n" + "ld1b { z2.b }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x8\n" - ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n" - ".inst 0x45069854 // smmla z20.s, z2.b, z6.b\n" - ".inst 0x4506989c // smmla z28.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" - ".inst 0x45079851 // smmla z17.s, z2.b, z7.b\n" - ".inst 0x45079899 // smmla z25.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x4506980d // smmla z13.s, z0.b, z6.b\n" - ".inst 0x45069855 // smmla z21.s, z2.b, z6.b\n" - ".inst 0x4506989d // smmla z29.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x4507980a // smmla z10.s, z0.b, z7.b\n" - ".inst 0x45079852 // smmla z18.s, z2.b, z7.b\n" - ".inst 0x4507989a // smmla z26.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x4506980e // smmla z14.s, z0.b, z6.b\n" - ".inst 0x45069856 // smmla z22.s, z2.b, z6.b\n" - ".inst 0x4506989e // smmla z30.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" + ".inst 0x450098ec // smmla z12.s, z7.b, z0.b\n" + ".inst 0x450098d4 // smmla z20.s, z6.b, z0.b\n" + ".inst 0x4500989c // smmla z28.s, z4.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x450298e9 // smmla z9.s, z7.b, z2.b\n" + ".inst 0x450298d1 // smmla z17.s, z6.b, z2.b\n" + ".inst 0x45029899 // smmla z25.s, z4.b, z2.b\n" + "ld1b { z2.b }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x450098ed // smmla z13.s, z7.b, z0.b\n" + ".inst 0x450098d5 // smmla z21.s, z6.b, z0.b\n" + ".inst 0x4500989d // smmla z29.s, z4.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x450298ea // smmla z10.s, z7.b, z2.b\n" + ".inst 0x450298d2 // smmla z18.s, z6.b, z2.b\n" + ".inst 0x4502989a // smmla z26.s, z4.b, z2.b\n" + "ld1b { z2.b }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x450098ee // smmla z14.s, z7.b, z0.b\n" + ".inst 0x450098d6 // smmla z22.s, z6.b, z0.b\n" + ".inst 0x4500989e // smmla z30.s, z4.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x450298eb // smmla z11.s, z7.b, z2.b\n" "addvl x10, x10, #8\n" - ".inst 0x45079853 // smmla z19.s, z2.b, z7.b\n" - ".inst 0x4507989b // smmla z27.s, z4.b, z7.b\n" - ".inst 0x4506980f // smmla z15.s, z0.b, z6.b\n" - ".inst 0x45069857 // smmla z23.s, z2.b, z6.b\n" - ".inst 0x4506989f // smmla z31.s, z4.b, z6.b\n" + ".inst 0x450298d3 // smmla z19.s, z6.b, z2.b\n" + ".inst 0x4502989b // smmla z27.s, z4.b, z2.b\n" + ".inst 0x450098ef // smmla z15.s, z7.b, z0.b\n" + ".inst 0x450098d7 // smmla z23.s, z6.b, z0.b\n" + ".inst 0x4500989f // smmla z31.s, z4.b, z0.b\n" "ble 65f\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45079828 // smmla z8.s, z1.b, z7.b\n" - ".inst 0x45079870 // smmla z16.s, z3.b, z7.b\n" - ".inst 0x450798b8 // smmla z24.s, z5.b, z7.b\n" - ".inst 0x4506982c // smmla z12.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x45069874 // smmla z20.s, z3.b, z6.b\n" - ".inst 0x450698bc // smmla z28.s, z5.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45079829 // smmla z9.s, z1.b, z7.b\n" - ".inst 0x45079871 // smmla z17.s, z3.b, z7.b\n" - ".inst 0x450798b9 // smmla z25.s, z5.b, z7.b\n" - ".inst 0x4506982d // smmla z13.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x45069875 // smmla z21.s, z3.b, z6.b\n" - ".inst 0x450698bd // smmla z29.s, z5.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x4507982a // smmla z10.s, z1.b, z7.b\n" - ".inst 0x45079872 // smmla z18.s, z3.b, z7.b\n" - ".inst 0x450798ba // smmla z26.s, z5.b, z7.b\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x45069876 // smmla z22.s, z3.b, z6.b\n" - ".inst 0x450698be // smmla z30.s, z5.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + "ld1b { z2.b }, p5/Z, [x10]\n" + "ld1b { z0.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45029828 // smmla z8.s, z1.b, z2.b\n" + ".inst 0x45029870 // smmla z16.s, z3.b, z2.b\n" + ".inst 0x450298b8 // smmla z24.s, z5.b, z2.b\n" + ".inst 0x4500982c // smmla z12.s, z1.b, z0.b\n" + "ld1b { z2.b }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x45009874 // smmla z20.s, z3.b, z0.b\n" + ".inst 0x450098bc // smmla z28.s, z5.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45029829 // smmla z9.s, z1.b, z2.b\n" + ".inst 0x45029871 // smmla z17.s, z3.b, z2.b\n" + ".inst 0x450298b9 // smmla z25.s, z5.b, z2.b\n" + ".inst 0x4500982d // smmla z13.s, z1.b, z0.b\n" + "ld1b { z2.b }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x45009875 // smmla z21.s, z3.b, z0.b\n" + ".inst 0x450098bd // smmla z29.s, z5.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x4502982a // smmla z10.s, z1.b, z2.b\n" + ".inst 0x45029872 // smmla z18.s, z3.b, z2.b\n" + ".inst 0x450298ba // smmla z26.s, z5.b, z2.b\n" + ".inst 0x4500982e // smmla z14.s, z1.b, z0.b\n" + "ld1b { z2.b }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x45009876 // smmla z22.s, z3.b, z0.b\n" + ".inst 0x450098be // smmla z30.s, z5.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - ".inst 0x4507982b // smmla z11.s, z1.b, z7.b\n" - ".inst 0x45079873 // smmla z19.s, z3.b, z7.b\n" - ".inst 0x450798bb // smmla z27.s, z5.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" - ".inst 0x45069877 // smmla z23.s, z3.b, z6.b\n" - ".inst 0x450698bf // smmla z31.s, z5.b, z6.b\n" + ".inst 0x4502982b // smmla z11.s, z1.b, z2.b\n" + ".inst 0x45029873 // smmla z19.s, z3.b, z2.b\n" + ".inst 0x450298bb // smmla z27.s, z5.b, z2.b\n" + ".inst 0x4500982f // smmla z15.s, z1.b, z0.b\n" + ".inst 0x45009877 // smmla z23.s, z3.b, z0.b\n" + ".inst 0x450098bf // smmla z31.s, z5.b, z0.b\n" "65:" // Height 6: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1596,7 +1596,7 @@ void sve_hybrid_s8s32_mmla_6x4VL ( "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "add x24, x9, x20, LSL #2\n" "add x23, x24, x20, LSL #2\n" - "uzp1 z7.d, z8.d, z12.d\n" + "uzp1 z0.d, z8.d, z12.d\n" "add x22, x23, x20, LSL #2\n" "add x21, x22, x20, LSL #2\n" "uzp2 z8.d, z8.d, z12.d\n" @@ -1604,7 +1604,7 @@ void sve_hybrid_s8s32_mmla_6x4VL ( "add x20, x21, x20, LSL #2\n" "uzp2 z9.d, z9.d, z13.d\n" "uzp1 z13.d, z10.d, z14.d\n" - "st1w { z7.s }, p4, [x9]\n" + "st1w { z0.s }, p4, [x9]\n" "uzp2 z10.d, z10.d, z14.d\n" "uzp1 z14.d, z11.d, z15.d\n" "st1w { z12.s }, p3, [x9, #1, MUL VL]\n" @@ -1664,7 +1664,6 @@ void sve_hybrid_s8s32_mmla_6x4VL ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "68:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -1672,4 +1671,4 @@ void sve_hybrid_s8s32_mmla_6x4VL ( } } // namespace arm_gemm -#endif // __ARM_FEATURE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_dot_4x4VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_dot_4x4VL.hpp index c66ebedc4d37052b657a328ae0b8d7f6282e4e24..11fe5ce7e36f757ada15410cb417ea707e79dd5c 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_dot_4x4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_dot_4x4VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef ARM_COMPUTE_ENABLE_SVE + #include "../std_transforms_sve.hpp" #include "../performance_parameters.hpp" @@ -74,7 +74,6 @@ public: template static inline PerformanceParameters get_performance_parameters(const CPUInfo *ci) { - if (std::is_same::value) { switch (ci->get_cpu_model()) { default: @@ -97,5 +96,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_dot_4x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_dot_4x4VL/generic.cpp index 79bd563a4b954c7ecd8de5360485a58553ab50ce..e74b424888bd73b6e4f4e42d6c5e6277ac2a80fb 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_dot_4x4VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_dot_4x4VL/generic.cpp @@ -104,11 +104,11 @@ void sve_hybrid_u8qa_dot_4x4VL ( "4:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 5f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" "cbnz x26, 6f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -121,39 +121,39 @@ void sve_hybrid_u8qa_dot_4x4VL ( "7:" // Height 1: Multiply loop: Main loop head "whilelt p0.b, XZR, x25\n" "ld1rqb { z0.b }, p0/Z, [x24]\n" - "ld1b { z4.b }, p2/Z, [x28]\n" - "udot z16.s, z4.b, z0.b[0]\n" - "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n" - "udot z17.s, z5.b, z0.b[0]\n" - "udot z18.s, z6.b, z0.b[0]\n" - "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n" - "udot z19.s, z7.b, z0.b[0]\n" - "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n" - "udot z16.s, z8.b, z0.b[1]\n" - "ld1b { z9.b }, p2/Z, [x28, #5, MUL VL]\n" - "ld1b { z10.b }, p2/Z, [x28, #6, MUL VL]\n" - "udot z17.s, z9.b, z0.b[1]\n" - "udot z18.s, z10.b, z0.b[1]\n" - "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n" + "ld1b { z20.b }, p2/Z, [x28]\n" + "udot z16.s, z20.b, z0.b[0]\n" + "ld1b { z21.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z20.b }, p2/Z, [x28, #2, MUL VL]\n" + "udot z17.s, z21.b, z0.b[0]\n" + "udot z18.s, z20.b, z0.b[0]\n" + "ld1b { z20.b }, p2/Z, [x28, #3, MUL VL]\n" + "udot z19.s, z20.b, z0.b[0]\n" + "ld1b { z20.b }, p2/Z, [x28, #4, MUL VL]\n" + "udot z16.s, z20.b, z0.b[1]\n" + "ld1b { z21.b }, p2/Z, [x28, #5, MUL VL]\n" + "ld1b { z20.b }, p2/Z, [x28, #6, MUL VL]\n" + "udot z17.s, z21.b, z0.b[1]\n" + "udot z18.s, z20.b, z0.b[1]\n" + "ld1b { z20.b }, p2/Z, [x28, #7, MUL VL]\n" "addvl x28, x28, #16\n" - "udot z19.s, z4.b, z0.b[1]\n" - "ld1b { z5.b }, p2/Z, [x28, #-8, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x28, #-6, MUL VL]\n" - "udot z16.s, z5.b, z0.b[2]\n" - "udot z17.s, z6.b, z0.b[2]\n" - "ld1b { z8.b }, p2/Z, [x28, #-5, MUL VL]\n" - "udot z18.s, z7.b, z0.b[2]\n" - "udot z19.s, z8.b, z0.b[2]\n" - "ld1b { z9.b }, p2/Z, [x28, #-4, MUL VL]\n" - "ld1b { z10.b }, p2/Z, [x28, #-3, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #-2, MUL VL]\n" - "udot z16.s, z9.b, z0.b[3]\n" - "udot z17.s, z10.b, z0.b[3]\n" - "ld1b { z5.b }, p2/Z, [x28, #-1, MUL VL]\n" - "udot z18.s, z4.b, z0.b[3]\n" - "udot z19.s, z5.b, z0.b[3]\n" + "udot z19.s, z20.b, z0.b[1]\n" + "ld1b { z22.b }, p2/Z, [x28, #-8, MUL VL]\n" + "ld1b { z20.b }, p2/Z, [x28, #-7, MUL VL]\n" + "ld1b { z21.b }, p2/Z, [x28, #-6, MUL VL]\n" + "udot z16.s, z22.b, z0.b[2]\n" + "udot z17.s, z20.b, z0.b[2]\n" + "ld1b { z20.b }, p2/Z, [x28, #-5, MUL VL]\n" + "udot z18.s, z21.b, z0.b[2]\n" + "udot z19.s, z20.b, z0.b[2]\n" + "ld1b { z22.b }, p2/Z, [x28, #-4, MUL VL]\n" + "ld1b { z20.b }, p2/Z, [x28, #-3, MUL VL]\n" + "ld1b { z21.b }, p2/Z, [x28, #-2, MUL VL]\n" + "udot z16.s, z22.b, z0.b[3]\n" + "udot z17.s, z20.b, z0.b[3]\n" + "ld1b { z20.b }, p2/Z, [x28, #-1, MUL VL]\n" + "udot z18.s, z21.b, z0.b[3]\n" + "udot z19.s, z20.b, z0.b[3]\n" "add x24, x24, #0x10\n" "tbnz %x[flags], #31, 8f\n" "udot z11.s, z0.b, z15.b\n" @@ -164,47 +164,47 @@ void sve_hybrid_u8qa_dot_4x4VL ( "9:" // Height 1: Multiply loop: Single iteration only "whilelt p0.b, XZR, x25\n" "ld1rqb { z0.b }, p0/Z, [x24]\n" - "ld1b { z4.b }, p2/Z, [x28]\n" + "ld1b { z22.b }, p2/Z, [x28]\n" "subs x25, x25, #0x4\n" - "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n" - "udot z16.s, z4.b, z0.b[0]\n" - "udot z17.s, z5.b, z0.b[0]\n" - "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n" - "udot z18.s, z6.b, z0.b[0]\n" - "udot z19.s, z7.b, z0.b[0]\n" + "ld1b { z20.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z21.b }, p2/Z, [x28, #2, MUL VL]\n" + "udot z16.s, z22.b, z0.b[0]\n" + "udot z17.s, z20.b, z0.b[0]\n" + "ld1b { z20.b }, p2/Z, [x28, #3, MUL VL]\n" + "udot z18.s, z21.b, z0.b[0]\n" + "udot z19.s, z20.b, z0.b[0]\n" "addvl x28, x28, #4\n" "ble 10f\n" - "ld1b { z8.b }, p2/Z, [x28]\n" - "ld1b { z9.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z20.b }, p2/Z, [x28]\n" + "ld1b { z22.b }, p2/Z, [x28, #1, MUL VL]\n" "subs x25, x25, #0x4\n" - "udot z16.s, z8.b, z0.b[1]\n" - "ld1b { z10.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #3, MUL VL]\n" - "udot z17.s, z9.b, z0.b[1]\n" - "udot z18.s, z10.b, z0.b[1]\n" - "udot z19.s, z4.b, z0.b[1]\n" + "udot z16.s, z20.b, z0.b[1]\n" + "ld1b { z21.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z20.b }, p2/Z, [x28, #3, MUL VL]\n" + "udot z17.s, z22.b, z0.b[1]\n" + "udot z18.s, z21.b, z0.b[1]\n" + "udot z19.s, z20.b, z0.b[1]\n" "addvl x28, x28, #4\n" "ble 10f\n" - "ld1b { z5.b }, p2/Z, [x28]\n" - "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z20.b }, p2/Z, [x28]\n" + "ld1b { z22.b }, p2/Z, [x28, #1, MUL VL]\n" "subs x25, x25, #0x4\n" - "udot z16.s, z5.b, z0.b[2]\n" - "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" - "udot z17.s, z6.b, z0.b[2]\n" - "udot z18.s, z7.b, z0.b[2]\n" - "udot z19.s, z8.b, z0.b[2]\n" + "udot z16.s, z20.b, z0.b[2]\n" + "ld1b { z21.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z20.b }, p2/Z, [x28, #3, MUL VL]\n" + "udot z17.s, z22.b, z0.b[2]\n" + "udot z18.s, z21.b, z0.b[2]\n" + "udot z19.s, z20.b, z0.b[2]\n" "addvl x28, x28, #4\n" "ble 10f\n" - "ld1b { z9.b }, p2/Z, [x28]\n" - "ld1b { z10.b }, p2/Z, [x28, #1, MUL VL]\n" - "udot z16.s, z9.b, z0.b[3]\n" - "udot z17.s, z10.b, z0.b[3]\n" - "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z5.b }, p2/Z, [x28, #3, MUL VL]\n" - "udot z18.s, z4.b, z0.b[3]\n" - "udot z19.s, z5.b, z0.b[3]\n" + "ld1b { z21.b }, p2/Z, [x28]\n" + "ld1b { z20.b }, p2/Z, [x28, #1, MUL VL]\n" + "udot z16.s, z21.b, z0.b[3]\n" + "udot z17.s, z20.b, z0.b[3]\n" + "ld1b { z21.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z20.b }, p2/Z, [x28, #3, MUL VL]\n" + "udot z18.s, z21.b, z0.b[3]\n" + "udot z19.s, z20.b, z0.b[3]\n" "addvl x28, x28, #4\n" "10:" // Height 1: Multiply loop: multiply skip "tbnz %x[flags], #31, 11f\n" @@ -218,71 +218,71 @@ void sve_hybrid_u8qa_dot_4x4VL ( "mov x20, #0x4\n" "whilelt p0.s, XZR, x20\n" "add x20, %x[qp], %[b_offset]\n" - "ld1rw { z1.s }, p2/Z, [x20]\n" + "ld1rw { z20.s }, p2/Z, [x20]\n" "uaddv d11, p0, z11.s\n" "mov z11.s, z11.s[0]\n" - "neg z1.s, p2/M, z1.s\n" - "mul z11.s, p2/M, z11.s, z1.s\n" + "neg z20.s, p2/M, z20.s\n" + "mul z11.s, p2/M, z11.s, z20.s\n" "12:" // Height 1: skip row sum fixup "add z16.s, z16.s, z11.s\n" "add z17.s, z17.s, z11.s\n" - "ld1w { z0.s }, p2/Z, [x10]\n" - "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n" + "ld1w { z23.s }, p2/Z, [x10]\n" + "ld1w { z22.s }, p2/Z, [x10, #1, MUL VL]\n" "add z18.s, z18.s, z11.s\n" "add z19.s, z19.s, z11.s\n" - "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n" + "ld1w { z21.s }, p2/Z, [x10, #2, MUL VL]\n" + "ld1w { z20.s }, p2/Z, [x10, #3, MUL VL]\n" "add x20, %x[qp], %[per_layer_mul]\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add z16.s, z16.s, z0.s\n" - "add z17.s, z17.s, z1.s\n" - "add z18.s, z18.s, z2.s\n" - "add z19.s, z19.s, z3.s\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + "add z16.s, z16.s, z23.s\n" + "add z17.s, z17.s, z22.s\n" + "add z18.s, z18.s, z21.s\n" + "add z19.s, z19.s, z20.s\n" + "ld1rw { z20.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[per_layer_right_shift]\n" "ld1rw { z0.s }, p2/Z, [x20]\n" - ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n" - ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n" + ".inst 0x04b47610 // sqrdmulh z16.s, z16.s, z20.s\n" + ".inst 0x04b47631 // sqrdmulh z17.s, z17.s, z20.s\n" "addvl x10, x10, #4\n" - ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n" - ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n" + ".inst 0x04b47652 // sqrdmulh z18.s, z18.s, z20.s\n" + ".inst 0x04b47673 // sqrdmulh z19.s, z19.s, z20.s\n" "tbz %x[flags], #5, 13f\n" - "and z4.d, z16.d, z0.d\n" - "and z5.d, z17.d, z0.d\n" - "and z6.d, z18.d, z0.d\n" - "and z7.d, z19.d, z0.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z16.s, z16.s, z4.s\n" - "sqadd z17.s, z17.s, z5.s\n" - "sqadd z18.s, z18.s, z6.s\n" - "sqadd z19.s, z19.s, z7.s\n" + "and z23.d, z16.d, z0.d\n" + "and z22.d, z17.d, z0.d\n" + "and z21.d, z18.d, z0.d\n" + "and z20.d, z19.d, z0.d\n" + "asr z23.s, z23.s, #0x1f\n" + "asr z22.s, z22.s, #0x1f\n" + "asr z21.s, z21.s, #0x1f\n" + "asr z20.s, z20.s, #0x1f\n" + "sqadd z16.s, z16.s, z23.s\n" + "sqadd z17.s, z17.s, z22.s\n" + "sqadd z18.s, z18.s, z21.s\n" + "sqadd z19.s, z19.s, z20.s\n" "13:" // Height 1: no shift correction "add x20, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + "ld1rw { z20.s }, p2/Z, [x20]\n" ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n" - "add z16.s, z16.s, z4.s\n" + "add z16.s, z16.s, z20.s\n" ".inst 0x44828811 // srshl z17.s, p2/M, z17.s, z0.s\n" ".inst 0x44828812 // srshl z18.s, p2/M, z18.s, z0.s\n" - "add z17.s, z17.s, z4.s\n" - "add z18.s, z18.s, z4.s\n" + "add z17.s, z17.s, z20.s\n" + "add z18.s, z18.s, z20.s\n" ".inst 0x44828813 // srshl z19.s, p2/M, z19.s, z0.s\n" "add x20, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x20]\n" - "add z19.s, z19.s, z4.s\n" + "ld1rw { z21.s }, p2/Z, [x20]\n" + "add z19.s, z19.s, z20.s\n" "add x20, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x20]\n" - "smin z16.s, p2/M, z16.s, z6.s\n" - "smin z17.s, p2/M, z17.s, z6.s\n" - "smin z18.s, p2/M, z18.s, z6.s\n" - "smin z19.s, p2/M, z19.s, z6.s\n" - "smax z16.s, p2/M, z16.s, z5.s\n" - "smax z17.s, p2/M, z17.s, z5.s\n" - "smax z18.s, p2/M, z18.s, z5.s\n" + "ld1rw { z20.s }, p2/Z, [x20]\n" + "smin z16.s, p2/M, z16.s, z21.s\n" + "smin z17.s, p2/M, z17.s, z21.s\n" + "smin z18.s, p2/M, z18.s, z21.s\n" + "smin z19.s, p2/M, z19.s, z21.s\n" + "smax z16.s, p2/M, z16.s, z20.s\n" + "smax z17.s, p2/M, z17.s, z20.s\n" + "smax z18.s, p2/M, z18.s, z20.s\n" "uzp1 z16.h, z16.h, z17.h\n" - "smax z19.s, p2/M, z19.s, z5.s\n" + "smax z19.s, p2/M, z19.s, z20.s\n" "uzp1 z17.h, z18.h, z19.h\n" "uzp1 z16.b, z16.b, z17.b\n" "st1b { z16.b }, p1, [x27]\n" @@ -317,12 +317,12 @@ void sve_hybrid_u8qa_dot_4x4VL ( "18:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 19f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" "cbnz x26, 20f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -330,7 +330,7 @@ void sve_hybrid_u8qa_dot_4x4VL ( "b 20f\n" "19:" // Height 2: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20\n" + "add x23, x24, x21\n" "20:" // Height 2: input setup done "cmp x25, #0x10\n" "ble 23f\n" @@ -339,56 +339,56 @@ void sve_hybrid_u8qa_dot_4x4VL ( "ld1rqb { z0.b }, p0/Z, [x24]\n" "ld1rqb { z1.b }, p0/Z, [x23]\n" "add x24, x24, #0x10\n" - "ld1b { z4.b }, p2/Z, [x28]\n" - "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n" - "udot z16.s, z4.b, z0.b[0]\n" - "udot z20.s, z4.b, z1.b[0]\n" - "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n" - "udot z17.s, z5.b, z0.b[0]\n" - "udot z21.s, z5.b, z1.b[0]\n" - "udot z18.s, z6.b, z0.b[0]\n" - "udot z22.s, z6.b, z1.b[0]\n" - "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n" - "ld1b { z9.b }, p2/Z, [x28, #5, MUL VL]\n" - "udot z19.s, z7.b, z0.b[0]\n" - "udot z23.s, z7.b, z1.b[0]\n" - "ld1b { z10.b }, p2/Z, [x28, #6, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28]\n" + "ld1b { z26.b }, p2/Z, [x28, #1, MUL VL]\n" + "udot z16.s, z24.b, z0.b[0]\n" + "udot z20.s, z24.b, z1.b[0]\n" + "ld1b { z24.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z25.b }, p2/Z, [x28, #3, MUL VL]\n" + "udot z17.s, z26.b, z0.b[0]\n" + "udot z21.s, z26.b, z1.b[0]\n" + "udot z18.s, z24.b, z0.b[0]\n" + "udot z22.s, z24.b, z1.b[0]\n" + "ld1b { z24.b }, p2/Z, [x28, #4, MUL VL]\n" + "ld1b { z27.b }, p2/Z, [x28, #5, MUL VL]\n" + "udot z19.s, z25.b, z0.b[0]\n" + "udot z23.s, z25.b, z1.b[0]\n" + "ld1b { z26.b }, p2/Z, [x28, #6, MUL VL]\n" + "ld1b { z25.b }, p2/Z, [x28, #7, MUL VL]\n" "addvl x28, x28, #16\n" - "udot z16.s, z8.b, z0.b[1]\n" - "udot z20.s, z8.b, z1.b[1]\n" - "ld1b { z5.b }, p2/Z, [x28, #-8, MUL VL]\n" - "udot z17.s, z9.b, z0.b[1]\n" - "udot z21.s, z9.b, z1.b[1]\n" - "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x28, #-6, MUL VL]\n" - "udot z18.s, z10.b, z0.b[1]\n" - "udot z22.s, z10.b, z1.b[1]\n" - "ld1b { z8.b }, p2/Z, [x28, #-5, MUL VL]\n" - "ld1b { z9.b }, p2/Z, [x28, #-4, MUL VL]\n" - "udot z19.s, z4.b, z0.b[1]\n" - "udot z23.s, z4.b, z1.b[1]\n" - "ld1b { z10.b }, p2/Z, [x28, #-3, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #-2, MUL VL]\n" - "udot z16.s, z5.b, z0.b[2]\n" - "udot z20.s, z5.b, z1.b[2]\n" - "ld1b { z5.b }, p2/Z, [x28, #-1, MUL VL]\n" + "udot z16.s, z24.b, z0.b[1]\n" + "udot z20.s, z24.b, z1.b[1]\n" + "ld1b { z24.b }, p2/Z, [x28, #-8, MUL VL]\n" + "udot z17.s, z27.b, z0.b[1]\n" + "udot z21.s, z27.b, z1.b[1]\n" + "ld1b { z30.b }, p2/Z, [x28, #-7, MUL VL]\n" + "ld1b { z29.b }, p2/Z, [x28, #-6, MUL VL]\n" + "udot z18.s, z26.b, z0.b[1]\n" + "udot z22.s, z26.b, z1.b[1]\n" + "ld1b { z28.b }, p2/Z, [x28, #-5, MUL VL]\n" + "ld1b { z27.b }, p2/Z, [x28, #-4, MUL VL]\n" + "udot z19.s, z25.b, z0.b[1]\n" + "udot z23.s, z25.b, z1.b[1]\n" + "ld1b { z26.b }, p2/Z, [x28, #-3, MUL VL]\n" + "ld1b { z25.b }, p2/Z, [x28, #-2, MUL VL]\n" + "udot z16.s, z24.b, z0.b[2]\n" + "udot z20.s, z24.b, z1.b[2]\n" + "ld1b { z24.b }, p2/Z, [x28, #-1, MUL VL]\n" "add x23, x23, #0x10\n" - "udot z17.s, z6.b, z0.b[2]\n" - "udot z21.s, z6.b, z1.b[2]\n" - "udot z18.s, z7.b, z0.b[2]\n" - "udot z22.s, z7.b, z1.b[2]\n" - "udot z19.s, z8.b, z0.b[2]\n" - "udot z23.s, z8.b, z1.b[2]\n" - "udot z16.s, z9.b, z0.b[3]\n" - "udot z20.s, z9.b, z1.b[3]\n" - "udot z17.s, z10.b, z0.b[3]\n" - "udot z21.s, z10.b, z1.b[3]\n" - "udot z18.s, z4.b, z0.b[3]\n" - "udot z22.s, z4.b, z1.b[3]\n" - "udot z19.s, z5.b, z0.b[3]\n" - "udot z23.s, z5.b, z1.b[3]\n" + "udot z17.s, z30.b, z0.b[2]\n" + "udot z21.s, z30.b, z1.b[2]\n" + "udot z18.s, z29.b, z0.b[2]\n" + "udot z22.s, z29.b, z1.b[2]\n" + "udot z19.s, z28.b, z0.b[2]\n" + "udot z23.s, z28.b, z1.b[2]\n" + "udot z16.s, z27.b, z0.b[3]\n" + "udot z20.s, z27.b, z1.b[3]\n" + "udot z17.s, z26.b, z0.b[3]\n" + "udot z21.s, z26.b, z1.b[3]\n" + "udot z18.s, z25.b, z0.b[3]\n" + "udot z22.s, z25.b, z1.b[3]\n" + "udot z19.s, z24.b, z0.b[3]\n" + "udot z23.s, z24.b, z1.b[3]\n" "tbnz %x[flags], #31, 22f\n" "udot z11.s, z0.b, z15.b\n" "udot z12.s, z1.b, z15.b\n" @@ -401,63 +401,63 @@ void sve_hybrid_u8qa_dot_4x4VL ( "ld1rqb { z0.b }, p0/Z, [x24]\n" "ld1rqb { z1.b }, p0/Z, [x23]\n" "subs x25, x25, #0x4\n" - "ld1b { z4.b }, p2/Z, [x28]\n" - "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n" - "udot z16.s, z4.b, z0.b[0]\n" - "udot z20.s, z4.b, z1.b[0]\n" - "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n" - "udot z17.s, z5.b, z0.b[0]\n" - "udot z21.s, z5.b, z1.b[0]\n" - "udot z18.s, z6.b, z0.b[0]\n" - "udot z22.s, z6.b, z1.b[0]\n" + "ld1b { z24.b }, p2/Z, [x28]\n" + "ld1b { z26.b }, p2/Z, [x28, #1, MUL VL]\n" + "udot z16.s, z24.b, z0.b[0]\n" + "udot z20.s, z24.b, z1.b[0]\n" + "ld1b { z25.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #3, MUL VL]\n" + "udot z17.s, z26.b, z0.b[0]\n" + "udot z21.s, z26.b, z1.b[0]\n" + "udot z18.s, z25.b, z0.b[0]\n" + "udot z22.s, z25.b, z1.b[0]\n" "addvl x28, x28, #4\n" - "udot z19.s, z7.b, z0.b[0]\n" - "udot z23.s, z7.b, z1.b[0]\n" + "udot z19.s, z24.b, z0.b[0]\n" + "udot z23.s, z24.b, z1.b[0]\n" "ble 24f\n" - "ld1b { z8.b }, p2/Z, [x28]\n" - "ld1b { z9.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z27.b }, p2/Z, [x28]\n" + "ld1b { z26.b }, p2/Z, [x28, #1, MUL VL]\n" "subs x25, x25, #0x4\n" - "udot z16.s, z8.b, z0.b[1]\n" - "ld1b { z10.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #3, MUL VL]\n" - "udot z20.s, z8.b, z1.b[1]\n" - "udot z17.s, z9.b, z0.b[1]\n" - "udot z21.s, z9.b, z1.b[1]\n" - "udot z18.s, z10.b, z0.b[1]\n" + "udot z16.s, z27.b, z0.b[1]\n" + "ld1b { z25.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #3, MUL VL]\n" + "udot z20.s, z27.b, z1.b[1]\n" + "udot z17.s, z26.b, z0.b[1]\n" + "udot z21.s, z26.b, z1.b[1]\n" + "udot z18.s, z25.b, z0.b[1]\n" "addvl x28, x28, #4\n" - "udot z22.s, z10.b, z1.b[1]\n" - "udot z19.s, z4.b, z0.b[1]\n" - "udot z23.s, z4.b, z1.b[1]\n" + "udot z22.s, z25.b, z1.b[1]\n" + "udot z19.s, z24.b, z0.b[1]\n" + "udot z23.s, z24.b, z1.b[1]\n" "ble 24f\n" - "ld1b { z5.b }, p2/Z, [x28]\n" - "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z27.b }, p2/Z, [x28]\n" + "ld1b { z26.b }, p2/Z, [x28, #1, MUL VL]\n" "subs x25, x25, #0x4\n" - "udot z16.s, z5.b, z0.b[2]\n" - "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" - "udot z20.s, z5.b, z1.b[2]\n" - "udot z17.s, z6.b, z0.b[2]\n" - "udot z21.s, z6.b, z1.b[2]\n" - "udot z18.s, z7.b, z0.b[2]\n" + "udot z16.s, z27.b, z0.b[2]\n" + "ld1b { z25.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #3, MUL VL]\n" + "udot z20.s, z27.b, z1.b[2]\n" + "udot z17.s, z26.b, z0.b[2]\n" + "udot z21.s, z26.b, z1.b[2]\n" + "udot z18.s, z25.b, z0.b[2]\n" "addvl x28, x28, #4\n" - "udot z22.s, z7.b, z1.b[2]\n" - "udot z19.s, z8.b, z0.b[2]\n" - "udot z23.s, z8.b, z1.b[2]\n" + "udot z22.s, z25.b, z1.b[2]\n" + "udot z19.s, z24.b, z0.b[2]\n" + "udot z23.s, z24.b, z1.b[2]\n" "ble 24f\n" - "ld1b { z9.b }, p2/Z, [x28]\n" - "ld1b { z10.b }, p2/Z, [x28, #1, MUL VL]\n" - "udot z16.s, z9.b, z0.b[3]\n" - "udot z20.s, z9.b, z1.b[3]\n" - "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z5.b }, p2/Z, [x28, #3, MUL VL]\n" - "udot z17.s, z10.b, z0.b[3]\n" - "udot z21.s, z10.b, z1.b[3]\n" - "udot z18.s, z4.b, z0.b[3]\n" - "udot z22.s, z4.b, z1.b[3]\n" + "ld1b { z24.b }, p2/Z, [x28]\n" + "ld1b { z26.b }, p2/Z, [x28, #1, MUL VL]\n" + "udot z16.s, z24.b, z0.b[3]\n" + "udot z20.s, z24.b, z1.b[3]\n" + "ld1b { z25.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #3, MUL VL]\n" + "udot z17.s, z26.b, z0.b[3]\n" + "udot z21.s, z26.b, z1.b[3]\n" + "udot z18.s, z25.b, z0.b[3]\n" + "udot z22.s, z25.b, z1.b[3]\n" "addvl x28, x28, #4\n" - "udot z19.s, z5.b, z0.b[3]\n" - "udot z23.s, z5.b, z1.b[3]\n" + "udot z19.s, z24.b, z0.b[3]\n" + "udot z23.s, z24.b, z1.b[3]\n" "24:" // Height 2: Multiply loop: multiply skip "tbnz %x[flags], #31, 25f\n" "udot z11.s, z0.b, z15.b\n" @@ -473,120 +473,120 @@ void sve_hybrid_u8qa_dot_4x4VL ( "mov x20, #0x4\n" "whilelt p0.s, XZR, x20\n" "add x20, %x[qp], %[b_offset]\n" - "ld1rw { z2.s }, p2/Z, [x20]\n" + "ld1rw { z24.s }, p2/Z, [x20]\n" "uaddv d11, p0, z11.s\n" "mov z11.s, z11.s[0]\n" "uaddv d12, p0, z12.s\n" - "neg z2.s, p2/M, z2.s\n" + "neg z24.s, p2/M, z24.s\n" "mov z12.s, z12.s[0]\n" - "mul z11.s, p2/M, z11.s, z2.s\n" - "mul z12.s, p2/M, z12.s, z2.s\n" + "mul z11.s, p2/M, z11.s, z24.s\n" + "mul z12.s, p2/M, z12.s, z24.s\n" "26:" // Height 2: skip row sum fixup "add z16.s, z16.s, z11.s\n" "add z17.s, z17.s, z11.s\n" - "ld1w { z0.s }, p2/Z, [x10]\n" - "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n" + "ld1w { z28.s }, p2/Z, [x10]\n" + "ld1w { z27.s }, p2/Z, [x10, #1, MUL VL]\n" "add z18.s, z18.s, z11.s\n" "add z19.s, z19.s, z11.s\n" - "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n" + "ld1w { z26.s }, p2/Z, [x10, #2, MUL VL]\n" + "ld1w { z25.s }, p2/Z, [x10, #3, MUL VL]\n" "add z20.s, z20.s, z12.s\n" "add z21.s, z21.s, z12.s\n" "add x20, %x[qp], %[per_layer_mul]\n" "orr %x[flags], %x[flags], #0x80000000\n" "add z22.s, z22.s, z12.s\n" "add z23.s, z23.s, z12.s\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + "ld1rw { z24.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[per_layer_right_shift]\n" - "add z16.s, z16.s, z0.s\n" - "add z17.s, z17.s, z1.s\n" + "add z16.s, z16.s, z28.s\n" + "add z17.s, z17.s, z27.s\n" "addvl x10, x10, #4\n" - "add z18.s, z18.s, z2.s\n" - "add z19.s, z19.s, z3.s\n" - "add z20.s, z20.s, z0.s\n" - "add z21.s, z21.s, z1.s\n" + "add z18.s, z18.s, z26.s\n" + "add z19.s, z19.s, z25.s\n" + "add z20.s, z20.s, z28.s\n" + "add z21.s, z21.s, z27.s\n" "ld1rw { z0.s }, p2/Z, [x20]\n" - "add z22.s, z22.s, z2.s\n" - "add z23.s, z23.s, z3.s\n" - ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n" - ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n" - ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n" - ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n" - ".inst 0x04a47694 // sqrdmulh z20.s, z20.s, z4.s\n" - ".inst 0x04a476b5 // sqrdmulh z21.s, z21.s, z4.s\n" - ".inst 0x04a476d6 // sqrdmulh z22.s, z22.s, z4.s\n" - ".inst 0x04a476f7 // sqrdmulh z23.s, z23.s, z4.s\n" + "add z22.s, z22.s, z26.s\n" + "add z23.s, z23.s, z25.s\n" + ".inst 0x04b87610 // sqrdmulh z16.s, z16.s, z24.s\n" + ".inst 0x04b87631 // sqrdmulh z17.s, z17.s, z24.s\n" + ".inst 0x04b87652 // sqrdmulh z18.s, z18.s, z24.s\n" + ".inst 0x04b87673 // sqrdmulh z19.s, z19.s, z24.s\n" + ".inst 0x04b87694 // sqrdmulh z20.s, z20.s, z24.s\n" + ".inst 0x04b876b5 // sqrdmulh z21.s, z21.s, z24.s\n" + ".inst 0x04b876d6 // sqrdmulh z22.s, z22.s, z24.s\n" + ".inst 0x04b876f7 // sqrdmulh z23.s, z23.s, z24.s\n" "tbz %x[flags], #5, 27f\n" - "and z4.d, z16.d, z0.d\n" - "asr z4.s, z4.s, #0x1f\n" - "sqadd z16.s, z16.s, z4.s\n" - "and z5.d, z17.d, z0.d\n" - "and z6.d, z18.d, z0.d\n" - "and z7.d, z19.d, z0.d\n" - "and z8.d, z20.d, z0.d\n" - "and z9.d, z21.d, z0.d\n" - "and z10.d, z22.d, z0.d\n" - "and z4.d, z23.d, z0.d\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "asr z8.s, z8.s, #0x1f\n" - "asr z9.s, z9.s, #0x1f\n" - "asr z10.s, z10.s, #0x1f\n" - "asr z4.s, z4.s, #0x1f\n" - "sqadd z17.s, z17.s, z5.s\n" - "sqadd z18.s, z18.s, z6.s\n" - "sqadd z19.s, z19.s, z7.s\n" - "sqadd z20.s, z20.s, z8.s\n" - "sqadd z21.s, z21.s, z9.s\n" - "sqadd z22.s, z22.s, z10.s\n" - "sqadd z23.s, z23.s, z4.s\n" + "and z24.d, z16.d, z0.d\n" + "asr z24.s, z24.s, #0x1f\n" + "sqadd z16.s, z16.s, z24.s\n" + "and z30.d, z17.d, z0.d\n" + "and z29.d, z18.d, z0.d\n" + "and z28.d, z19.d, z0.d\n" + "and z27.d, z20.d, z0.d\n" + "and z26.d, z21.d, z0.d\n" + "and z25.d, z22.d, z0.d\n" + "and z24.d, z23.d, z0.d\n" + "asr z30.s, z30.s, #0x1f\n" + "asr z29.s, z29.s, #0x1f\n" + "asr z28.s, z28.s, #0x1f\n" + "asr z27.s, z27.s, #0x1f\n" + "asr z26.s, z26.s, #0x1f\n" + "asr z25.s, z25.s, #0x1f\n" + "asr z24.s, z24.s, #0x1f\n" + "sqadd z17.s, z17.s, z30.s\n" + "sqadd z18.s, z18.s, z29.s\n" + "sqadd z19.s, z19.s, z28.s\n" + "sqadd z20.s, z20.s, z27.s\n" + "sqadd z21.s, z21.s, z26.s\n" + "sqadd z22.s, z22.s, z25.s\n" + "sqadd z23.s, z23.s, z24.s\n" "27:" // Height 2: no shift correction "add x20, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + "ld1rw { z24.s }, p2/Z, [x20]\n" ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n" - "add z16.s, z16.s, z4.s\n" + "add z16.s, z16.s, z24.s\n" ".inst 0x44828811 // srshl z17.s, p2/M, z17.s, z0.s\n" ".inst 0x44828812 // srshl z18.s, p2/M, z18.s, z0.s\n" - "add z17.s, z17.s, z4.s\n" - "add z18.s, z18.s, z4.s\n" + "add z17.s, z17.s, z24.s\n" + "add z18.s, z18.s, z24.s\n" ".inst 0x44828813 // srshl z19.s, p2/M, z19.s, z0.s\n" ".inst 0x44828814 // srshl z20.s, p2/M, z20.s, z0.s\n" - "add z19.s, z19.s, z4.s\n" - "add z20.s, z20.s, z4.s\n" + "add z19.s, z19.s, z24.s\n" + "add z20.s, z20.s, z24.s\n" ".inst 0x44828815 // srshl z21.s, p2/M, z21.s, z0.s\n" ".inst 0x44828816 // srshl z22.s, p2/M, z22.s, z0.s\n" - "add z21.s, z21.s, z4.s\n" - "add z22.s, z22.s, z4.s\n" + "add z21.s, z21.s, z24.s\n" + "add z22.s, z22.s, z24.s\n" ".inst 0x44828817 // srshl z23.s, p2/M, z23.s, z0.s\n" "add x20, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x20]\n" - "add z23.s, z23.s, z4.s\n" + "ld1rw { z25.s }, p2/Z, [x20]\n" + "add z23.s, z23.s, z24.s\n" "add x20, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x20]\n" - "smin z16.s, p2/M, z16.s, z6.s\n" - "smin z17.s, p2/M, z17.s, z6.s\n" - "smin z18.s, p2/M, z18.s, z6.s\n" - "smin z19.s, p2/M, z19.s, z6.s\n" - "smin z20.s, p2/M, z20.s, z6.s\n" - "smin z21.s, p2/M, z21.s, z6.s\n" - "smin z22.s, p2/M, z22.s, z6.s\n" - "smin z23.s, p2/M, z23.s, z6.s\n" - "smax z16.s, p2/M, z16.s, z5.s\n" - "smax z17.s, p2/M, z17.s, z5.s\n" - "smax z18.s, p2/M, z18.s, z5.s\n" + "ld1rw { z24.s }, p2/Z, [x20]\n" + "smin z16.s, p2/M, z16.s, z25.s\n" + "smin z17.s, p2/M, z17.s, z25.s\n" + "smin z18.s, p2/M, z18.s, z25.s\n" + "smin z19.s, p2/M, z19.s, z25.s\n" + "smin z20.s, p2/M, z20.s, z25.s\n" + "smin z21.s, p2/M, z21.s, z25.s\n" + "smin z22.s, p2/M, z22.s, z25.s\n" + "smin z23.s, p2/M, z23.s, z25.s\n" + "smax z16.s, p2/M, z16.s, z24.s\n" + "smax z17.s, p2/M, z17.s, z24.s\n" + "smax z18.s, p2/M, z18.s, z24.s\n" "uzp1 z16.h, z16.h, z17.h\n" - "smax z19.s, p2/M, z19.s, z5.s\n" - "smax z20.s, p2/M, z20.s, z5.s\n" + "smax z19.s, p2/M, z19.s, z24.s\n" + "smax z20.s, p2/M, z20.s, z24.s\n" "uzp1 z17.h, z18.h, z19.h\n" "uzp1 z16.b, z16.b, z17.b\n" - "smax z21.s, p2/M, z21.s, z5.s\n" - "smax z22.s, p2/M, z22.s, z5.s\n" + "smax z21.s, p2/M, z21.s, z24.s\n" + "smax z22.s, p2/M, z22.s, z24.s\n" "uzp1 z20.h, z20.h, z21.h\n" "st1b { z16.b }, p1, [x27]\n" - "smax z23.s, p2/M, z23.s, z5.s\n" - "uzp1 z21.h, z22.h, z23.h\n" - "uzp1 z20.b, z20.b, z21.b\n" + "smax z23.s, p2/M, z23.s, z24.s\n" + "uzp1 z16.h, z22.h, z23.h\n" + "uzp1 z20.b, z20.b, z16.b\n" "st1b { z20.b }, p1, [x23]\n" "addvl x27, x27, #1\n" "28:" // Height 2: Writeback done @@ -624,13 +624,13 @@ void sve_hybrid_u8qa_dot_4x4VL ( "32:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 33f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" "cbnz x26, 34f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -639,8 +639,8 @@ void sve_hybrid_u8qa_dot_4x4VL ( "b 34f\n" "33:" // Height 3: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" "34:" // Height 3: input setup done "cmp x25, #0x10\n" "ble 37f\n" @@ -650,73 +650,73 @@ void sve_hybrid_u8qa_dot_4x4VL ( "ld1rqb { z1.b }, p0/Z, [x23]\n" "add x24, x24, #0x10\n" "ld1rqb { z2.b }, p0/Z, [x22]\n" - "ld1b { z4.b }, p2/Z, [x28]\n" - "udot z16.s, z4.b, z0.b[0]\n" - "udot z20.s, z4.b, z1.b[0]\n" - "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n" - "udot z24.s, z4.b, z2.b[0]\n" - "udot z17.s, z5.b, z0.b[0]\n" - "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n" - "udot z21.s, z5.b, z1.b[0]\n" - "udot z25.s, z5.b, z2.b[0]\n" - "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n" - "udot z18.s, z6.b, z0.b[0]\n" - "udot z22.s, z6.b, z1.b[0]\n" - "ld1b { z9.b }, p2/Z, [x28, #5, MUL VL]\n" - "ld1b { z10.b }, p2/Z, [x28, #6, MUL VL]\n" - "udot z26.s, z6.b, z2.b[0]\n" - "udot z19.s, z7.b, z0.b[0]\n" - "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n" + "ld1b { z28.b }, p2/Z, [x28]\n" + "udot z16.s, z28.b, z0.b[0]\n" + "udot z20.s, z28.b, z1.b[0]\n" + "ld1b { z30.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z29.b }, p2/Z, [x28, #2, MUL VL]\n" + "udot z24.s, z28.b, z2.b[0]\n" + "udot z17.s, z30.b, z0.b[0]\n" + "ld1b { z28.b }, p2/Z, [x28, #3, MUL VL]\n" + "udot z21.s, z30.b, z1.b[0]\n" + "udot z25.s, z30.b, z2.b[0]\n" + "ld1b { z3.b }, p2/Z, [x28, #4, MUL VL]\n" + "udot z18.s, z29.b, z0.b[0]\n" + "udot z22.s, z29.b, z1.b[0]\n" + "ld1b { z31.b }, p2/Z, [x28, #5, MUL VL]\n" + "ld1b { z30.b }, p2/Z, [x28, #6, MUL VL]\n" + "udot z26.s, z29.b, z2.b[0]\n" + "udot z19.s, z28.b, z0.b[0]\n" + "ld1b { z29.b }, p2/Z, [x28, #7, MUL VL]\n" "addvl x28, x28, #16\n" - "udot z23.s, z7.b, z1.b[0]\n" - "udot z27.s, z7.b, z2.b[0]\n" - "ld1b { z5.b }, p2/Z, [x28, #-8, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n" - "udot z16.s, z8.b, z0.b[1]\n" - "udot z20.s, z8.b, z1.b[1]\n" - "ld1b { z7.b }, p2/Z, [x28, #-6, MUL VL]\n" + "udot z23.s, z28.b, z1.b[0]\n" + "udot z27.s, z28.b, z2.b[0]\n" + "ld1b { z28.b }, p2/Z, [x28, #-8, MUL VL]\n" + "ld1b { z5.b }, p2/Z, [x28, #-7, MUL VL]\n" + "udot z16.s, z3.b, z0.b[1]\n" + "udot z20.s, z3.b, z1.b[1]\n" + "ld1b { z4.b }, p2/Z, [x28, #-6, MUL VL]\n" "add x23, x23, #0x10\n" - "udot z24.s, z8.b, z2.b[1]\n" - "udot z17.s, z9.b, z0.b[1]\n" - "ld1b { z8.b }, p2/Z, [x28, #-5, MUL VL]\n" + "udot z24.s, z3.b, z2.b[1]\n" + "udot z17.s, z31.b, z0.b[1]\n" + "ld1b { z3.b }, p2/Z, [x28, #-5, MUL VL]\n" "add x22, x22, #0x10\n" - "udot z21.s, z9.b, z1.b[1]\n" - "udot z25.s, z9.b, z2.b[1]\n" - "ld1b { z9.b }, p2/Z, [x28, #-4, MUL VL]\n" - "udot z18.s, z10.b, z0.b[1]\n" - "udot z22.s, z10.b, z1.b[1]\n" - "udot z26.s, z10.b, z2.b[1]\n" - "udot z19.s, z4.b, z0.b[1]\n" - "ld1b { z10.b }, p2/Z, [x28, #-3, MUL VL]\n" - "udot z23.s, z4.b, z1.b[1]\n" - "udot z27.s, z4.b, z2.b[1]\n" - "ld1b { z4.b }, p2/Z, [x28, #-2, MUL VL]\n" - "udot z16.s, z5.b, z0.b[2]\n" - "udot z20.s, z5.b, z1.b[2]\n" - "udot z24.s, z5.b, z2.b[2]\n" - "udot z17.s, z6.b, z0.b[2]\n" - "ld1b { z5.b }, p2/Z, [x28, #-1, MUL VL]\n" - "udot z21.s, z6.b, z1.b[2]\n" - "udot z25.s, z6.b, z2.b[2]\n" - "udot z18.s, z7.b, z0.b[2]\n" - "udot z22.s, z7.b, z1.b[2]\n" - "udot z26.s, z7.b, z2.b[2]\n" - "udot z19.s, z8.b, z0.b[2]\n" - "udot z23.s, z8.b, z1.b[2]\n" - "udot z27.s, z8.b, z2.b[2]\n" - "udot z16.s, z9.b, z0.b[3]\n" - "udot z20.s, z9.b, z1.b[3]\n" - "udot z24.s, z9.b, z2.b[3]\n" - "udot z17.s, z10.b, z0.b[3]\n" - "udot z21.s, z10.b, z1.b[3]\n" - "udot z25.s, z10.b, z2.b[3]\n" - "udot z18.s, z4.b, z0.b[3]\n" - "udot z22.s, z4.b, z1.b[3]\n" - "udot z26.s, z4.b, z2.b[3]\n" - "udot z19.s, z5.b, z0.b[3]\n" - "udot z23.s, z5.b, z1.b[3]\n" - "udot z27.s, z5.b, z2.b[3]\n" + "udot z21.s, z31.b, z1.b[1]\n" + "udot z25.s, z31.b, z2.b[1]\n" + "ld1b { z31.b }, p2/Z, [x28, #-4, MUL VL]\n" + "udot z18.s, z30.b, z0.b[1]\n" + "udot z22.s, z30.b, z1.b[1]\n" + "udot z26.s, z30.b, z2.b[1]\n" + "udot z19.s, z29.b, z0.b[1]\n" + "ld1b { z30.b }, p2/Z, [x28, #-3, MUL VL]\n" + "udot z23.s, z29.b, z1.b[1]\n" + "udot z27.s, z29.b, z2.b[1]\n" + "ld1b { z29.b }, p2/Z, [x28, #-2, MUL VL]\n" + "udot z16.s, z28.b, z0.b[2]\n" + "udot z20.s, z28.b, z1.b[2]\n" + "udot z24.s, z28.b, z2.b[2]\n" + "udot z17.s, z5.b, z0.b[2]\n" + "ld1b { z28.b }, p2/Z, [x28, #-1, MUL VL]\n" + "udot z21.s, z5.b, z1.b[2]\n" + "udot z25.s, z5.b, z2.b[2]\n" + "udot z18.s, z4.b, z0.b[2]\n" + "udot z22.s, z4.b, z1.b[2]\n" + "udot z26.s, z4.b, z2.b[2]\n" + "udot z19.s, z3.b, z0.b[2]\n" + "udot z23.s, z3.b, z1.b[2]\n" + "udot z27.s, z3.b, z2.b[2]\n" + "udot z16.s, z31.b, z0.b[3]\n" + "udot z20.s, z31.b, z1.b[3]\n" + "udot z24.s, z31.b, z2.b[3]\n" + "udot z17.s, z30.b, z0.b[3]\n" + "udot z21.s, z30.b, z1.b[3]\n" + "udot z25.s, z30.b, z2.b[3]\n" + "udot z18.s, z29.b, z0.b[3]\n" + "udot z22.s, z29.b, z1.b[3]\n" + "udot z26.s, z29.b, z2.b[3]\n" + "udot z19.s, z28.b, z0.b[3]\n" + "udot z23.s, z28.b, z1.b[3]\n" + "udot z27.s, z28.b, z2.b[3]\n" "tbnz %x[flags], #31, 36f\n" "udot z11.s, z0.b, z15.b\n" "udot z12.s, z1.b, z15.b\n" @@ -731,79 +731,79 @@ void sve_hybrid_u8qa_dot_4x4VL ( "ld1rqb { z1.b }, p0/Z, [x23]\n" "subs x25, x25, #0x4\n" "ld1rqb { z2.b }, p0/Z, [x22]\n" - "ld1b { z4.b }, p2/Z, [x28]\n" - "udot z16.s, z4.b, z0.b[0]\n" - "udot z20.s, z4.b, z1.b[0]\n" - "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n" - "udot z24.s, z4.b, z2.b[0]\n" - "udot z17.s, z5.b, z0.b[0]\n" - "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n" - "udot z21.s, z5.b, z1.b[0]\n" - "udot z25.s, z5.b, z2.b[0]\n" + "ld1b { z28.b }, p2/Z, [x28]\n" + "udot z16.s, z28.b, z0.b[0]\n" + "udot z20.s, z28.b, z1.b[0]\n" + "ld1b { z30.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z29.b }, p2/Z, [x28, #2, MUL VL]\n" + "udot z24.s, z28.b, z2.b[0]\n" + "udot z17.s, z30.b, z0.b[0]\n" + "ld1b { z28.b }, p2/Z, [x28, #3, MUL VL]\n" + "udot z21.s, z30.b, z1.b[0]\n" + "udot z25.s, z30.b, z2.b[0]\n" "addvl x28, x28, #4\n" - "udot z18.s, z6.b, z0.b[0]\n" - "udot z22.s, z6.b, z1.b[0]\n" - "udot z26.s, z6.b, z2.b[0]\n" - "udot z19.s, z7.b, z0.b[0]\n" - "udot z23.s, z7.b, z1.b[0]\n" - "udot z27.s, z7.b, z2.b[0]\n" + "udot z18.s, z29.b, z0.b[0]\n" + "udot z22.s, z29.b, z1.b[0]\n" + "udot z26.s, z29.b, z2.b[0]\n" + "udot z19.s, z28.b, z0.b[0]\n" + "udot z23.s, z28.b, z1.b[0]\n" + "udot z27.s, z28.b, z2.b[0]\n" "ble 38f\n" - "ld1b { z8.b }, p2/Z, [x28]\n" - "ld1b { z9.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z31.b }, p2/Z, [x28]\n" + "ld1b { z30.b }, p2/Z, [x28, #1, MUL VL]\n" "subs x25, x25, #0x4\n" - "udot z16.s, z8.b, z0.b[1]\n" - "ld1b { z10.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #3, MUL VL]\n" - "udot z20.s, z8.b, z1.b[1]\n" - "udot z24.s, z8.b, z2.b[1]\n" - "udot z17.s, z9.b, z0.b[1]\n" - "udot z21.s, z9.b, z1.b[1]\n" + "udot z16.s, z31.b, z0.b[1]\n" + "ld1b { z29.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z28.b }, p2/Z, [x28, #3, MUL VL]\n" + "udot z20.s, z31.b, z1.b[1]\n" + "udot z24.s, z31.b, z2.b[1]\n" + "udot z17.s, z30.b, z0.b[1]\n" + "udot z21.s, z30.b, z1.b[1]\n" "addvl x28, x28, #4\n" - "udot z25.s, z9.b, z2.b[1]\n" - "udot z18.s, z10.b, z0.b[1]\n" - "udot z22.s, z10.b, z1.b[1]\n" - "udot z26.s, z10.b, z2.b[1]\n" - "udot z19.s, z4.b, z0.b[1]\n" - "udot z23.s, z4.b, z1.b[1]\n" - "udot z27.s, z4.b, z2.b[1]\n" + "udot z25.s, z30.b, z2.b[1]\n" + "udot z18.s, z29.b, z0.b[1]\n" + "udot z22.s, z29.b, z1.b[1]\n" + "udot z26.s, z29.b, z2.b[1]\n" + "udot z19.s, z28.b, z0.b[1]\n" + "udot z23.s, z28.b, z1.b[1]\n" + "udot z27.s, z28.b, z2.b[1]\n" "ble 38f\n" - "ld1b { z5.b }, p2/Z, [x28]\n" - "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z31.b }, p2/Z, [x28]\n" + "ld1b { z30.b }, p2/Z, [x28, #1, MUL VL]\n" "subs x25, x25, #0x4\n" - "udot z16.s, z5.b, z0.b[2]\n" - "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" - "udot z20.s, z5.b, z1.b[2]\n" - "udot z24.s, z5.b, z2.b[2]\n" - "udot z17.s, z6.b, z0.b[2]\n" - "udot z21.s, z6.b, z1.b[2]\n" + "udot z16.s, z31.b, z0.b[2]\n" + "ld1b { z29.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z28.b }, p2/Z, [x28, #3, MUL VL]\n" + "udot z20.s, z31.b, z1.b[2]\n" + "udot z24.s, z31.b, z2.b[2]\n" + "udot z17.s, z30.b, z0.b[2]\n" + "udot z21.s, z30.b, z1.b[2]\n" "addvl x28, x28, #4\n" - "udot z25.s, z6.b, z2.b[2]\n" - "udot z18.s, z7.b, z0.b[2]\n" - "udot z22.s, z7.b, z1.b[2]\n" - "udot z26.s, z7.b, z2.b[2]\n" - "udot z19.s, z8.b, z0.b[2]\n" - "udot z23.s, z8.b, z1.b[2]\n" - "udot z27.s, z8.b, z2.b[2]\n" + "udot z25.s, z30.b, z2.b[2]\n" + "udot z18.s, z29.b, z0.b[2]\n" + "udot z22.s, z29.b, z1.b[2]\n" + "udot z26.s, z29.b, z2.b[2]\n" + "udot z19.s, z28.b, z0.b[2]\n" + "udot z23.s, z28.b, z1.b[2]\n" + "udot z27.s, z28.b, z2.b[2]\n" "ble 38f\n" - "ld1b { z9.b }, p2/Z, [x28]\n" - "ld1b { z10.b }, p2/Z, [x28, #1, MUL VL]\n" - "udot z16.s, z9.b, z0.b[3]\n" - "udot z20.s, z9.b, z1.b[3]\n" - "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z5.b }, p2/Z, [x28, #3, MUL VL]\n" - "udot z24.s, z9.b, z2.b[3]\n" - "udot z17.s, z10.b, z0.b[3]\n" - "udot z21.s, z10.b, z1.b[3]\n" - "udot z25.s, z10.b, z2.b[3]\n" + "ld1b { z31.b }, p2/Z, [x28]\n" + "ld1b { z30.b }, p2/Z, [x28, #1, MUL VL]\n" + "udot z16.s, z31.b, z0.b[3]\n" + "udot z20.s, z31.b, z1.b[3]\n" + "ld1b { z29.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z28.b }, p2/Z, [x28, #3, MUL VL]\n" + "udot z24.s, z31.b, z2.b[3]\n" + "udot z17.s, z30.b, z0.b[3]\n" + "udot z21.s, z30.b, z1.b[3]\n" + "udot z25.s, z30.b, z2.b[3]\n" "addvl x28, x28, #4\n" - "udot z18.s, z4.b, z0.b[3]\n" - "udot z22.s, z4.b, z1.b[3]\n" - "udot z26.s, z4.b, z2.b[3]\n" - "udot z19.s, z5.b, z0.b[3]\n" - "udot z23.s, z5.b, z1.b[3]\n" - "udot z27.s, z5.b, z2.b[3]\n" + "udot z18.s, z29.b, z0.b[3]\n" + "udot z22.s, z29.b, z1.b[3]\n" + "udot z26.s, z29.b, z2.b[3]\n" + "udot z19.s, z28.b, z0.b[3]\n" + "udot z23.s, z28.b, z1.b[3]\n" + "udot z27.s, z28.b, z2.b[3]\n" "38:" // Height 3: Multiply loop: multiply skip "tbnz %x[flags], #31, 39f\n" "udot z11.s, z0.b, z15.b\n" @@ -821,33 +821,33 @@ void sve_hybrid_u8qa_dot_4x4VL ( "mov x20, #0x4\n" "whilelt p0.s, XZR, x20\n" "add x20, %x[qp], %[b_offset]\n" - "ld1rw { z3.s }, p2/Z, [x20]\n" + "ld1rw { z28.s }, p2/Z, [x20]\n" "uaddv d11, p0, z11.s\n" "mov z11.s, z11.s[0]\n" "uaddv d12, p0, z12.s\n" "uaddv d13, p0, z13.s\n" "mov z12.s, z12.s[0]\n" "mov z13.s, z13.s[0]\n" - "neg z3.s, p2/M, z3.s\n" - "mul z11.s, p2/M, z11.s, z3.s\n" - "mul z12.s, p2/M, z12.s, z3.s\n" - "mul z13.s, p2/M, z13.s, z3.s\n" + "neg z28.s, p2/M, z28.s\n" + "mul z11.s, p2/M, z11.s, z28.s\n" + "mul z12.s, p2/M, z12.s, z28.s\n" + "mul z13.s, p2/M, z13.s, z28.s\n" "40:" // Height 3: skip row sum fixup "add z16.s, z16.s, z11.s\n" "add z17.s, z17.s, z11.s\n" "ld1w { z0.s }, p2/Z, [x10]\n" - "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n" + "ld1w { z31.s }, p2/Z, [x10, #1, MUL VL]\n" "add z18.s, z18.s, z11.s\n" "add z19.s, z19.s, z11.s\n" - "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n" + "ld1w { z30.s }, p2/Z, [x10, #2, MUL VL]\n" + "ld1w { z29.s }, p2/Z, [x10, #3, MUL VL]\n" "add z20.s, z20.s, z12.s\n" "add z21.s, z21.s, z12.s\n" "add x20, %x[qp], %[per_layer_mul]\n" "orr %x[flags], %x[flags], #0x80000000\n" "add z22.s, z22.s, z12.s\n" "add z23.s, z23.s, z12.s\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + "ld1rw { z28.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[per_layer_right_shift]\n" "add z24.s, z24.s, z13.s\n" "add z25.s, z25.s, z13.s\n" @@ -855,133 +855,133 @@ void sve_hybrid_u8qa_dot_4x4VL ( "add z26.s, z26.s, z13.s\n" "add z27.s, z27.s, z13.s\n" "add z16.s, z16.s, z0.s\n" - "add z17.s, z17.s, z1.s\n" - "add z18.s, z18.s, z2.s\n" - "add z19.s, z19.s, z3.s\n" + "add z17.s, z17.s, z31.s\n" + "add z18.s, z18.s, z30.s\n" + "add z19.s, z19.s, z29.s\n" "add z20.s, z20.s, z0.s\n" - "add z21.s, z21.s, z1.s\n" - "add z22.s, z22.s, z2.s\n" - "add z23.s, z23.s, z3.s\n" + "add z21.s, z21.s, z31.s\n" + "add z22.s, z22.s, z30.s\n" + "add z23.s, z23.s, z29.s\n" "add z24.s, z24.s, z0.s\n" - "add z25.s, z25.s, z1.s\n" + "add z25.s, z25.s, z31.s\n" "ld1rw { z0.s }, p2/Z, [x20]\n" - "add z26.s, z26.s, z2.s\n" - "add z27.s, z27.s, z3.s\n" - ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n" - ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n" - ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n" - ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n" - ".inst 0x04a47694 // sqrdmulh z20.s, z20.s, z4.s\n" - ".inst 0x04a476b5 // sqrdmulh z21.s, z21.s, z4.s\n" - ".inst 0x04a476d6 // sqrdmulh z22.s, z22.s, z4.s\n" - ".inst 0x04a476f7 // sqrdmulh z23.s, z23.s, z4.s\n" - ".inst 0x04a47718 // sqrdmulh z24.s, z24.s, z4.s\n" - ".inst 0x04a47739 // sqrdmulh z25.s, z25.s, z4.s\n" - ".inst 0x04a4775a // sqrdmulh z26.s, z26.s, z4.s\n" - ".inst 0x04a4777b // sqrdmulh z27.s, z27.s, z4.s\n" + "add z26.s, z26.s, z30.s\n" + "add z27.s, z27.s, z29.s\n" + ".inst 0x04bc7610 // sqrdmulh z16.s, z16.s, z28.s\n" + ".inst 0x04bc7631 // sqrdmulh z17.s, z17.s, z28.s\n" + ".inst 0x04bc7652 // sqrdmulh z18.s, z18.s, z28.s\n" + ".inst 0x04bc7673 // sqrdmulh z19.s, z19.s, z28.s\n" + ".inst 0x04bc7694 // sqrdmulh z20.s, z20.s, z28.s\n" + ".inst 0x04bc76b5 // sqrdmulh z21.s, z21.s, z28.s\n" + ".inst 0x04bc76d6 // sqrdmulh z22.s, z22.s, z28.s\n" + ".inst 0x04bc76f7 // sqrdmulh z23.s, z23.s, z28.s\n" + ".inst 0x04bc7718 // sqrdmulh z24.s, z24.s, z28.s\n" + ".inst 0x04bc7739 // sqrdmulh z25.s, z25.s, z28.s\n" + ".inst 0x04bc775a // sqrdmulh z26.s, z26.s, z28.s\n" + ".inst 0x04bc777b // sqrdmulh z27.s, z27.s, z28.s\n" "tbz %x[flags], #5, 41f\n" - "and z4.d, z16.d, z0.d\n" - "and z5.d, z17.d, z0.d\n" - "and z6.d, z18.d, z0.d\n" - "and z7.d, z19.d, z0.d\n" - "and z8.d, z20.d, z0.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "asr z8.s, z8.s, #0x1f\n" - "sqadd z16.s, z16.s, z4.s\n" - "sqadd z17.s, z17.s, z5.s\n" - "sqadd z18.s, z18.s, z6.s\n" - "sqadd z19.s, z19.s, z7.s\n" - "sqadd z20.s, z20.s, z8.s\n" - "and z9.d, z21.d, z0.d\n" - "and z10.d, z22.d, z0.d\n" - "and z4.d, z23.d, z0.d\n" - "and z5.d, z24.d, z0.d\n" - "and z6.d, z25.d, z0.d\n" - "and z7.d, z26.d, z0.d\n" - "and z8.d, z27.d, z0.d\n" - "asr z9.s, z9.s, #0x1f\n" - "asr z10.s, z10.s, #0x1f\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "asr z8.s, z8.s, #0x1f\n" - "sqadd z21.s, z21.s, z9.s\n" - "sqadd z22.s, z22.s, z10.s\n" - "sqadd z23.s, z23.s, z4.s\n" - "sqadd z24.s, z24.s, z5.s\n" - "sqadd z25.s, z25.s, z6.s\n" - "sqadd z26.s, z26.s, z7.s\n" - "sqadd z27.s, z27.s, z8.s\n" + "and z1.d, z16.d, z0.d\n" + "and z31.d, z17.d, z0.d\n" + "and z30.d, z18.d, z0.d\n" + "and z29.d, z19.d, z0.d\n" + "and z28.d, z20.d, z0.d\n" + "asr z1.s, z1.s, #0x1f\n" + "asr z31.s, z31.s, #0x1f\n" + "asr z30.s, z30.s, #0x1f\n" + "asr z29.s, z29.s, #0x1f\n" + "asr z28.s, z28.s, #0x1f\n" + "sqadd z16.s, z16.s, z1.s\n" + "sqadd z17.s, z17.s, z31.s\n" + "sqadd z18.s, z18.s, z30.s\n" + "sqadd z19.s, z19.s, z29.s\n" + "sqadd z20.s, z20.s, z28.s\n" + "and z3.d, z21.d, z0.d\n" + "and z2.d, z22.d, z0.d\n" + "and z1.d, z23.d, z0.d\n" + "and z31.d, z24.d, z0.d\n" + "and z30.d, z25.d, z0.d\n" + "and z29.d, z26.d, z0.d\n" + "and z28.d, z27.d, z0.d\n" + "asr z3.s, z3.s, #0x1f\n" + "asr z2.s, z2.s, #0x1f\n" + "asr z1.s, z1.s, #0x1f\n" + "asr z31.s, z31.s, #0x1f\n" + "asr z30.s, z30.s, #0x1f\n" + "asr z29.s, z29.s, #0x1f\n" + "asr z28.s, z28.s, #0x1f\n" + "sqadd z21.s, z21.s, z3.s\n" + "sqadd z22.s, z22.s, z2.s\n" + "sqadd z23.s, z23.s, z1.s\n" + "sqadd z24.s, z24.s, z31.s\n" + "sqadd z25.s, z25.s, z30.s\n" + "sqadd z26.s, z26.s, z29.s\n" + "sqadd z27.s, z27.s, z28.s\n" "41:" // Height 3: no shift correction "add x20, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + "ld1rw { z28.s }, p2/Z, [x20]\n" ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n" - "add z16.s, z16.s, z4.s\n" + "add z16.s, z16.s, z28.s\n" ".inst 0x44828811 // srshl z17.s, p2/M, z17.s, z0.s\n" ".inst 0x44828812 // srshl z18.s, p2/M, z18.s, z0.s\n" - "add z17.s, z17.s, z4.s\n" - "add z18.s, z18.s, z4.s\n" + "add z17.s, z17.s, z28.s\n" + "add z18.s, z18.s, z28.s\n" ".inst 0x44828813 // srshl z19.s, p2/M, z19.s, z0.s\n" ".inst 0x44828814 // srshl z20.s, p2/M, z20.s, z0.s\n" - "add z19.s, z19.s, z4.s\n" - "add z20.s, z20.s, z4.s\n" + "add z19.s, z19.s, z28.s\n" + "add z20.s, z20.s, z28.s\n" ".inst 0x44828815 // srshl z21.s, p2/M, z21.s, z0.s\n" ".inst 0x44828816 // srshl z22.s, p2/M, z22.s, z0.s\n" - "add z21.s, z21.s, z4.s\n" - "add z22.s, z22.s, z4.s\n" + "add z21.s, z21.s, z28.s\n" + "add z22.s, z22.s, z28.s\n" ".inst 0x44828817 // srshl z23.s, p2/M, z23.s, z0.s\n" ".inst 0x44828818 // srshl z24.s, p2/M, z24.s, z0.s\n" - "add z23.s, z23.s, z4.s\n" - "add z24.s, z24.s, z4.s\n" + "add z23.s, z23.s, z28.s\n" + "add z24.s, z24.s, z28.s\n" ".inst 0x44828819 // srshl z25.s, p2/M, z25.s, z0.s\n" ".inst 0x4482881a // srshl z26.s, p2/M, z26.s, z0.s\n" - "add z25.s, z25.s, z4.s\n" - "add z26.s, z26.s, z4.s\n" + "add z25.s, z25.s, z28.s\n" + "add z26.s, z26.s, z28.s\n" ".inst 0x4482881b // srshl z27.s, p2/M, z27.s, z0.s\n" "add x20, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x20]\n" - "add z27.s, z27.s, z4.s\n" + "ld1rw { z29.s }, p2/Z, [x20]\n" + "add z27.s, z27.s, z28.s\n" "add x20, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x20]\n" - "smin z16.s, p2/M, z16.s, z6.s\n" - "smin z17.s, p2/M, z17.s, z6.s\n" - "smin z18.s, p2/M, z18.s, z6.s\n" - "smin z19.s, p2/M, z19.s, z6.s\n" - "smin z20.s, p2/M, z20.s, z6.s\n" - "smin z21.s, p2/M, z21.s, z6.s\n" - "smin z22.s, p2/M, z22.s, z6.s\n" - "smin z23.s, p2/M, z23.s, z6.s\n" - "smin z24.s, p2/M, z24.s, z6.s\n" - "smin z25.s, p2/M, z25.s, z6.s\n" - "smin z26.s, p2/M, z26.s, z6.s\n" - "smin z27.s, p2/M, z27.s, z6.s\n" - "smax z16.s, p2/M, z16.s, z5.s\n" - "smax z17.s, p2/M, z17.s, z5.s\n" - "smax z18.s, p2/M, z18.s, z5.s\n" + "ld1rw { z28.s }, p2/Z, [x20]\n" + "smin z16.s, p2/M, z16.s, z29.s\n" + "smin z17.s, p2/M, z17.s, z29.s\n" + "smin z18.s, p2/M, z18.s, z29.s\n" + "smin z19.s, p2/M, z19.s, z29.s\n" + "smin z20.s, p2/M, z20.s, z29.s\n" + "smin z21.s, p2/M, z21.s, z29.s\n" + "smin z22.s, p2/M, z22.s, z29.s\n" + "smin z23.s, p2/M, z23.s, z29.s\n" + "smin z24.s, p2/M, z24.s, z29.s\n" + "smin z25.s, p2/M, z25.s, z29.s\n" + "smin z26.s, p2/M, z26.s, z29.s\n" + "smin z27.s, p2/M, z27.s, z29.s\n" + "smax z16.s, p2/M, z16.s, z28.s\n" + "smax z17.s, p2/M, z17.s, z28.s\n" + "smax z18.s, p2/M, z18.s, z28.s\n" "uzp1 z16.h, z16.h, z17.h\n" - "smax z19.s, p2/M, z19.s, z5.s\n" - "smax z20.s, p2/M, z20.s, z5.s\n" + "smax z19.s, p2/M, z19.s, z28.s\n" + "smax z20.s, p2/M, z20.s, z28.s\n" "uzp1 z17.h, z18.h, z19.h\n" "uzp1 z16.b, z16.b, z17.b\n" - "smax z21.s, p2/M, z21.s, z5.s\n" - "smax z22.s, p2/M, z22.s, z5.s\n" + "smax z21.s, p2/M, z21.s, z28.s\n" + "smax z22.s, p2/M, z22.s, z28.s\n" "uzp1 z20.h, z20.h, z21.h\n" "st1b { z16.b }, p1, [x27]\n" - "smax z23.s, p2/M, z23.s, z5.s\n" - "smax z24.s, p2/M, z24.s, z5.s\n" - "uzp1 z21.h, z22.h, z23.h\n" - "uzp1 z20.b, z20.b, z21.b\n" - "smax z25.s, p2/M, z25.s, z5.s\n" - "smax z26.s, p2/M, z26.s, z5.s\n" + "smax z23.s, p2/M, z23.s, z28.s\n" + "smax z24.s, p2/M, z24.s, z28.s\n" + "uzp1 z16.h, z22.h, z23.h\n" + "uzp1 z20.b, z20.b, z16.b\n" + "smax z25.s, p2/M, z25.s, z28.s\n" + "smax z26.s, p2/M, z26.s, z28.s\n" "uzp1 z24.h, z24.h, z25.h\n" "st1b { z20.b }, p1, [x23]\n" - "smax z27.s, p2/M, z27.s, z5.s\n" - "uzp1 z25.h, z26.h, z27.h\n" - "uzp1 z24.b, z24.b, z25.b\n" + "smax z27.s, p2/M, z27.s, z28.s\n" + "uzp1 z16.h, z26.h, z27.h\n" + "uzp1 z24.b, z24.b, z16.b\n" "st1b { z24.b }, p1, [x22]\n" "addvl x27, x27, #1\n" "42:" // Height 3: Writeback done @@ -1027,14 +1027,14 @@ void sve_hybrid_u8qa_dot_4x4VL ( "46:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 47f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" - "ldr x21, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" + "ldr x21, [x20, #0x18]\n" "cbnz x26, 48f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -1044,9 +1044,9 @@ void sve_hybrid_u8qa_dot_4x4VL ( "b 48f\n" "47:" // Height 4: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" + "add x21, x22, x21\n" "48:" // Height 4: input setup done "cmp x25, #0x10\n" "ble 51f\n" @@ -1059,88 +1059,88 @@ void sve_hybrid_u8qa_dot_4x4VL ( "ld1rqb { z3.b }, p0/Z, [x21]\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - "ld1b { z4.b }, p2/Z, [x28]\n" - "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n" - "udot z16.s, z4.b, z0.b[0]\n" - "udot z20.s, z4.b, z1.b[0]\n" - "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n" - "udot z24.s, z4.b, z2.b[0]\n" - "udot z28.s, z4.b, z3.b[0]\n" - "udot z17.s, z5.b, z0.b[0]\n" - "udot z21.s, z5.b, z1.b[0]\n" + "ld1b { z5.b }, p2/Z, [x28]\n" + "ld1b { z4.b }, p2/Z, [x28, #1, MUL VL]\n" + "udot z16.s, z5.b, z0.b[0]\n" + "udot z20.s, z5.b, z1.b[0]\n" + "ld1b { z10.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z9.b }, p2/Z, [x28, #3, MUL VL]\n" + "udot z24.s, z5.b, z2.b[0]\n" + "udot z28.s, z5.b, z3.b[0]\n" + "udot z17.s, z4.b, z0.b[0]\n" + "udot z21.s, z4.b, z1.b[0]\n" "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n" - "ld1b { z9.b }, p2/Z, [x28, #5, MUL VL]\n" - "udot z25.s, z5.b, z2.b[0]\n" - "udot z29.s, z5.b, z3.b[0]\n" - "ld1b { z10.b }, p2/Z, [x28, #6, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n" - "udot z18.s, z6.b, z0.b[0]\n" - "udot z22.s, z6.b, z1.b[0]\n" + "ld1b { z7.b }, p2/Z, [x28, #5, MUL VL]\n" + "udot z25.s, z4.b, z2.b[0]\n" + "udot z29.s, z4.b, z3.b[0]\n" + "ld1b { z6.b }, p2/Z, [x28, #6, MUL VL]\n" + "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n" + "udot z18.s, z10.b, z0.b[0]\n" + "udot z22.s, z10.b, z1.b[0]\n" "addvl x28, x28, #16\n" - "ld1b { z5.b }, p2/Z, [x28, #-8, MUL VL]\n" - "udot z26.s, z6.b, z2.b[0]\n" - "udot z30.s, z6.b, z3.b[0]\n" - "ld1b { z6.b }, p2/Z, [x28, #-7, MUL VL]\n" + "ld1b { z4.b }, p2/Z, [x28, #-8, MUL VL]\n" + "udot z26.s, z10.b, z2.b[0]\n" + "udot z30.s, z10.b, z3.b[0]\n" + "ld1b { z10.b }, p2/Z, [x28, #-7, MUL VL]\n" "add x21, x21, #0x10\n" - "udot z19.s, z7.b, z0.b[0]\n" - "udot z23.s, z7.b, z1.b[0]\n" - "udot z27.s, z7.b, z2.b[0]\n" - "udot z31.s, z7.b, z3.b[0]\n" - "ld1b { z7.b }, p2/Z, [x28, #-6, MUL VL]\n" + "udot z19.s, z9.b, z0.b[0]\n" + "udot z23.s, z9.b, z1.b[0]\n" + "udot z27.s, z9.b, z2.b[0]\n" + "udot z31.s, z9.b, z3.b[0]\n" + "ld1b { z9.b }, p2/Z, [x28, #-6, MUL VL]\n" "udot z16.s, z8.b, z0.b[1]\n" "udot z20.s, z8.b, z1.b[1]\n" "udot z24.s, z8.b, z2.b[1]\n" "udot z28.s, z8.b, z3.b[1]\n" "ld1b { z8.b }, p2/Z, [x28, #-5, MUL VL]\n" - "udot z17.s, z9.b, z0.b[1]\n" - "udot z21.s, z9.b, z1.b[1]\n" - "udot z25.s, z9.b, z2.b[1]\n" - "udot z29.s, z9.b, z3.b[1]\n" - "ld1b { z9.b }, p2/Z, [x28, #-4, MUL VL]\n" - "udot z18.s, z10.b, z0.b[1]\n" - "udot z22.s, z10.b, z1.b[1]\n" - "udot z26.s, z10.b, z2.b[1]\n" - "udot z30.s, z10.b, z3.b[1]\n" - "ld1b { z10.b }, p2/Z, [x28, #-3, MUL VL]\n" - "udot z19.s, z4.b, z0.b[1]\n" - "udot z23.s, z4.b, z1.b[1]\n" - "udot z27.s, z4.b, z2.b[1]\n" - "udot z31.s, z4.b, z3.b[1]\n" - "ld1b { z4.b }, p2/Z, [x28, #-2, MUL VL]\n" - "udot z16.s, z5.b, z0.b[2]\n" - "udot z20.s, z5.b, z1.b[2]\n" - "udot z24.s, z5.b, z2.b[2]\n" - "udot z28.s, z5.b, z3.b[2]\n" - "ld1b { z5.b }, p2/Z, [x28, #-1, MUL VL]\n" - "udot z17.s, z6.b, z0.b[2]\n" - "udot z21.s, z6.b, z1.b[2]\n" - "udot z25.s, z6.b, z2.b[2]\n" - "udot z29.s, z6.b, z3.b[2]\n" - "udot z18.s, z7.b, z0.b[2]\n" - "udot z22.s, z7.b, z1.b[2]\n" - "udot z26.s, z7.b, z2.b[2]\n" - "udot z30.s, z7.b, z3.b[2]\n" + "udot z17.s, z7.b, z0.b[1]\n" + "udot z21.s, z7.b, z1.b[1]\n" + "udot z25.s, z7.b, z2.b[1]\n" + "udot z29.s, z7.b, z3.b[1]\n" + "ld1b { z7.b }, p2/Z, [x28, #-4, MUL VL]\n" + "udot z18.s, z6.b, z0.b[1]\n" + "udot z22.s, z6.b, z1.b[1]\n" + "udot z26.s, z6.b, z2.b[1]\n" + "udot z30.s, z6.b, z3.b[1]\n" + "ld1b { z6.b }, p2/Z, [x28, #-3, MUL VL]\n" + "udot z19.s, z5.b, z0.b[1]\n" + "udot z23.s, z5.b, z1.b[1]\n" + "udot z27.s, z5.b, z2.b[1]\n" + "udot z31.s, z5.b, z3.b[1]\n" + "ld1b { z5.b }, p2/Z, [x28, #-2, MUL VL]\n" + "udot z16.s, z4.b, z0.b[2]\n" + "udot z20.s, z4.b, z1.b[2]\n" + "udot z24.s, z4.b, z2.b[2]\n" + "udot z28.s, z4.b, z3.b[2]\n" + "ld1b { z4.b }, p2/Z, [x28, #-1, MUL VL]\n" + "udot z17.s, z10.b, z0.b[2]\n" + "udot z21.s, z10.b, z1.b[2]\n" + "udot z25.s, z10.b, z2.b[2]\n" + "udot z29.s, z10.b, z3.b[2]\n" + "udot z18.s, z9.b, z0.b[2]\n" + "udot z22.s, z9.b, z1.b[2]\n" + "udot z26.s, z9.b, z2.b[2]\n" + "udot z30.s, z9.b, z3.b[2]\n" "udot z19.s, z8.b, z0.b[2]\n" "udot z23.s, z8.b, z1.b[2]\n" "udot z27.s, z8.b, z2.b[2]\n" "udot z31.s, z8.b, z3.b[2]\n" - "udot z16.s, z9.b, z0.b[3]\n" - "udot z20.s, z9.b, z1.b[3]\n" - "udot z24.s, z9.b, z2.b[3]\n" - "udot z28.s, z9.b, z3.b[3]\n" - "udot z17.s, z10.b, z0.b[3]\n" - "udot z21.s, z10.b, z1.b[3]\n" - "udot z25.s, z10.b, z2.b[3]\n" - "udot z29.s, z10.b, z3.b[3]\n" - "udot z18.s, z4.b, z0.b[3]\n" - "udot z22.s, z4.b, z1.b[3]\n" - "udot z26.s, z4.b, z2.b[3]\n" - "udot z30.s, z4.b, z3.b[3]\n" - "udot z19.s, z5.b, z0.b[3]\n" - "udot z23.s, z5.b, z1.b[3]\n" - "udot z27.s, z5.b, z2.b[3]\n" - "udot z31.s, z5.b, z3.b[3]\n" + "udot z16.s, z7.b, z0.b[3]\n" + "udot z20.s, z7.b, z1.b[3]\n" + "udot z24.s, z7.b, z2.b[3]\n" + "udot z28.s, z7.b, z3.b[3]\n" + "udot z17.s, z6.b, z0.b[3]\n" + "udot z21.s, z6.b, z1.b[3]\n" + "udot z25.s, z6.b, z2.b[3]\n" + "udot z29.s, z6.b, z3.b[3]\n" + "udot z18.s, z5.b, z0.b[3]\n" + "udot z22.s, z5.b, z1.b[3]\n" + "udot z26.s, z5.b, z2.b[3]\n" + "udot z30.s, z5.b, z3.b[3]\n" + "udot z19.s, z4.b, z0.b[3]\n" + "udot z23.s, z4.b, z1.b[3]\n" + "udot z27.s, z4.b, z2.b[3]\n" + "udot z31.s, z4.b, z3.b[3]\n" "tbnz %x[flags], #31, 50f\n" "udot z11.s, z0.b, z15.b\n" "udot z12.s, z1.b, z15.b\n" @@ -1157,95 +1157,95 @@ void sve_hybrid_u8qa_dot_4x4VL ( "subs x25, x25, #0x4\n" "ld1rqb { z2.b }, p0/Z, [x22]\n" "ld1rqb { z3.b }, p0/Z, [x21]\n" - "ld1b { z4.b }, p2/Z, [x28]\n" - "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n" - "udot z16.s, z4.b, z0.b[0]\n" - "udot z20.s, z4.b, z1.b[0]\n" - "ld1b { z6.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x28, #3, MUL VL]\n" - "udot z24.s, z4.b, z2.b[0]\n" - "udot z28.s, z4.b, z3.b[0]\n" - "udot z17.s, z5.b, z0.b[0]\n" - "udot z21.s, z5.b, z1.b[0]\n" + "ld1b { z7.b }, p2/Z, [x28]\n" + "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" + "udot z16.s, z7.b, z0.b[0]\n" + "udot z20.s, z7.b, z1.b[0]\n" + "ld1b { z5.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z4.b }, p2/Z, [x28, #3, MUL VL]\n" + "udot z24.s, z7.b, z2.b[0]\n" + "udot z28.s, z7.b, z3.b[0]\n" + "udot z17.s, z6.b, z0.b[0]\n" + "udot z21.s, z6.b, z1.b[0]\n" "addvl x28, x28, #4\n" - "udot z25.s, z5.b, z2.b[0]\n" - "udot z29.s, z5.b, z3.b[0]\n" - "udot z18.s, z6.b, z0.b[0]\n" - "udot z22.s, z6.b, z1.b[0]\n" - "udot z26.s, z6.b, z2.b[0]\n" - "udot z30.s, z6.b, z3.b[0]\n" - "udot z19.s, z7.b, z0.b[0]\n" - "udot z23.s, z7.b, z1.b[0]\n" - "udot z27.s, z7.b, z2.b[0]\n" - "udot z31.s, z7.b, z3.b[0]\n" + "udot z25.s, z6.b, z2.b[0]\n" + "udot z29.s, z6.b, z3.b[0]\n" + "udot z18.s, z5.b, z0.b[0]\n" + "udot z22.s, z5.b, z1.b[0]\n" + "udot z26.s, z5.b, z2.b[0]\n" + "udot z30.s, z5.b, z3.b[0]\n" + "udot z19.s, z4.b, z0.b[0]\n" + "udot z23.s, z4.b, z1.b[0]\n" + "udot z27.s, z4.b, z2.b[0]\n" + "udot z31.s, z4.b, z3.b[0]\n" "ble 52f\n" - "ld1b { z8.b }, p2/Z, [x28]\n" - "ld1b { z9.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z7.b }, p2/Z, [x28]\n" + "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" "subs x25, x25, #0x4\n" - "udot z16.s, z8.b, z0.b[1]\n" - "ld1b { z10.b }, p2/Z, [x28, #2, MUL VL]\n" + "udot z16.s, z7.b, z0.b[1]\n" + "ld1b { z5.b }, p2/Z, [x28, #2, MUL VL]\n" "ld1b { z4.b }, p2/Z, [x28, #3, MUL VL]\n" - "udot z20.s, z8.b, z1.b[1]\n" - "udot z24.s, z8.b, z2.b[1]\n" - "udot z28.s, z8.b, z3.b[1]\n" - "udot z17.s, z9.b, z0.b[1]\n" + "udot z20.s, z7.b, z1.b[1]\n" + "udot z24.s, z7.b, z2.b[1]\n" + "udot z28.s, z7.b, z3.b[1]\n" + "udot z17.s, z6.b, z0.b[1]\n" "addvl x28, x28, #4\n" - "udot z21.s, z9.b, z1.b[1]\n" - "udot z25.s, z9.b, z2.b[1]\n" - "udot z29.s, z9.b, z3.b[1]\n" - "udot z18.s, z10.b, z0.b[1]\n" - "udot z22.s, z10.b, z1.b[1]\n" - "udot z26.s, z10.b, z2.b[1]\n" - "udot z30.s, z10.b, z3.b[1]\n" + "udot z21.s, z6.b, z1.b[1]\n" + "udot z25.s, z6.b, z2.b[1]\n" + "udot z29.s, z6.b, z3.b[1]\n" + "udot z18.s, z5.b, z0.b[1]\n" + "udot z22.s, z5.b, z1.b[1]\n" + "udot z26.s, z5.b, z2.b[1]\n" + "udot z30.s, z5.b, z3.b[1]\n" "udot z19.s, z4.b, z0.b[1]\n" "udot z23.s, z4.b, z1.b[1]\n" "udot z27.s, z4.b, z2.b[1]\n" "udot z31.s, z4.b, z3.b[1]\n" "ble 52f\n" - "ld1b { z5.b }, p2/Z, [x28]\n" + "ld1b { z7.b }, p2/Z, [x28]\n" "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" "subs x25, x25, #0x4\n" - "udot z16.s, z5.b, z0.b[2]\n" - "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" - "udot z20.s, z5.b, z1.b[2]\n" - "udot z24.s, z5.b, z2.b[2]\n" - "udot z28.s, z5.b, z3.b[2]\n" + "udot z16.s, z7.b, z0.b[2]\n" + "ld1b { z5.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z4.b }, p2/Z, [x28, #3, MUL VL]\n" + "udot z20.s, z7.b, z1.b[2]\n" + "udot z24.s, z7.b, z2.b[2]\n" + "udot z28.s, z7.b, z3.b[2]\n" "udot z17.s, z6.b, z0.b[2]\n" "addvl x28, x28, #4\n" "udot z21.s, z6.b, z1.b[2]\n" "udot z25.s, z6.b, z2.b[2]\n" "udot z29.s, z6.b, z3.b[2]\n" - "udot z18.s, z7.b, z0.b[2]\n" - "udot z22.s, z7.b, z1.b[2]\n" - "udot z26.s, z7.b, z2.b[2]\n" - "udot z30.s, z7.b, z3.b[2]\n" - "udot z19.s, z8.b, z0.b[2]\n" - "udot z23.s, z8.b, z1.b[2]\n" - "udot z27.s, z8.b, z2.b[2]\n" - "udot z31.s, z8.b, z3.b[2]\n" + "udot z18.s, z5.b, z0.b[2]\n" + "udot z22.s, z5.b, z1.b[2]\n" + "udot z26.s, z5.b, z2.b[2]\n" + "udot z30.s, z5.b, z3.b[2]\n" + "udot z19.s, z4.b, z0.b[2]\n" + "udot z23.s, z4.b, z1.b[2]\n" + "udot z27.s, z4.b, z2.b[2]\n" + "udot z31.s, z4.b, z3.b[2]\n" "ble 52f\n" - "ld1b { z9.b }, p2/Z, [x28]\n" - "ld1b { z10.b }, p2/Z, [x28, #1, MUL VL]\n" - "udot z16.s, z9.b, z0.b[3]\n" - "udot z20.s, z9.b, z1.b[3]\n" - "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z5.b }, p2/Z, [x28, #3, MUL VL]\n" - "udot z24.s, z9.b, z2.b[3]\n" - "udot z28.s, z9.b, z3.b[3]\n" - "udot z17.s, z10.b, z0.b[3]\n" - "udot z21.s, z10.b, z1.b[3]\n" + "ld1b { z7.b }, p2/Z, [x28]\n" + "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" + "udot z16.s, z7.b, z0.b[3]\n" + "udot z20.s, z7.b, z1.b[3]\n" + "ld1b { z5.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z4.b }, p2/Z, [x28, #3, MUL VL]\n" + "udot z24.s, z7.b, z2.b[3]\n" + "udot z28.s, z7.b, z3.b[3]\n" + "udot z17.s, z6.b, z0.b[3]\n" + "udot z21.s, z6.b, z1.b[3]\n" "addvl x28, x28, #4\n" - "udot z25.s, z10.b, z2.b[3]\n" - "udot z29.s, z10.b, z3.b[3]\n" - "udot z18.s, z4.b, z0.b[3]\n" - "udot z22.s, z4.b, z1.b[3]\n" - "udot z26.s, z4.b, z2.b[3]\n" - "udot z30.s, z4.b, z3.b[3]\n" - "udot z19.s, z5.b, z0.b[3]\n" - "udot z23.s, z5.b, z1.b[3]\n" - "udot z27.s, z5.b, z2.b[3]\n" - "udot z31.s, z5.b, z3.b[3]\n" + "udot z25.s, z6.b, z2.b[3]\n" + "udot z29.s, z6.b, z3.b[3]\n" + "udot z18.s, z5.b, z0.b[3]\n" + "udot z22.s, z5.b, z1.b[3]\n" + "udot z26.s, z5.b, z2.b[3]\n" + "udot z30.s, z5.b, z3.b[3]\n" + "udot z19.s, z4.b, z0.b[3]\n" + "udot z23.s, z4.b, z1.b[3]\n" + "udot z27.s, z4.b, z2.b[3]\n" + "udot z31.s, z4.b, z3.b[3]\n" "52:" // Height 4: Multiply loop: multiply skip "tbnz %x[flags], #31, 53f\n" "udot z11.s, z0.b, z15.b\n" @@ -1265,7 +1265,7 @@ void sve_hybrid_u8qa_dot_4x4VL ( "mov x20, #0x4\n" "whilelt p0.s, XZR, x20\n" "add x20, %x[qp], %[b_offset]\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + "ld1rw { z0.s }, p2/Z, [x20]\n" "uaddv d11, p0, z11.s\n" "mov z11.s, z11.s[0]\n" "uaddv d12, p0, z12.s\n" @@ -1273,28 +1273,28 @@ void sve_hybrid_u8qa_dot_4x4VL ( "mov z12.s, z12.s[0]\n" "mov z13.s, z13.s[0]\n" "uaddv d14, p0, z14.s\n" - "neg z4.s, p2/M, z4.s\n" + "neg z0.s, p2/M, z0.s\n" "mov z14.s, z14.s[0]\n" - "mul z11.s, p2/M, z11.s, z4.s\n" - "mul z12.s, p2/M, z12.s, z4.s\n" - "mul z13.s, p2/M, z13.s, z4.s\n" - "mul z14.s, p2/M, z14.s, z4.s\n" + "mul z11.s, p2/M, z11.s, z0.s\n" + "mul z12.s, p2/M, z12.s, z0.s\n" + "mul z13.s, p2/M, z13.s, z0.s\n" + "mul z14.s, p2/M, z14.s, z0.s\n" "54:" // Height 4: skip row sum fixup "add z16.s, z16.s, z11.s\n" "add z17.s, z17.s, z11.s\n" - "ld1w { z0.s }, p2/Z, [x10]\n" - "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n" + "ld1w { z4.s }, p2/Z, [x10]\n" + "ld1w { z0.s }, p2/Z, [x10, #1, MUL VL]\n" "add z18.s, z18.s, z11.s\n" "add z19.s, z19.s, z11.s\n" - "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n" + "ld1w { z3.s }, p2/Z, [x10, #2, MUL VL]\n" + "ld1w { z2.s }, p2/Z, [x10, #3, MUL VL]\n" "add z20.s, z20.s, z12.s\n" "add z21.s, z21.s, z12.s\n" "add x20, %x[qp], %[per_layer_mul]\n" "orr %x[flags], %x[flags], #0x80000000\n" "add z22.s, z22.s, z12.s\n" "add z23.s, z23.s, z12.s\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + "ld1rw { z1.s }, p2/Z, [x20]\n" "add x20, %x[qp], %[per_layer_right_shift]\n" "add z24.s, z24.s, z13.s\n" "add z25.s, z25.s, z13.s\n" @@ -1305,174 +1305,174 @@ void sve_hybrid_u8qa_dot_4x4VL ( "add z29.s, z29.s, z14.s\n" "add z30.s, z30.s, z14.s\n" "add z31.s, z31.s, z14.s\n" - "add z16.s, z16.s, z0.s\n" - "add z17.s, z17.s, z1.s\n" - "add z18.s, z18.s, z2.s\n" - "add z19.s, z19.s, z3.s\n" - "add z20.s, z20.s, z0.s\n" - "add z21.s, z21.s, z1.s\n" - "add z22.s, z22.s, z2.s\n" - "add z23.s, z23.s, z3.s\n" - "add z24.s, z24.s, z0.s\n" - "add z25.s, z25.s, z1.s\n" - "add z26.s, z26.s, z2.s\n" - "add z27.s, z27.s, z3.s\n" - "add z28.s, z28.s, z0.s\n" - "add z29.s, z29.s, z1.s\n" + "add z16.s, z16.s, z4.s\n" + "add z17.s, z17.s, z0.s\n" + "add z18.s, z18.s, z3.s\n" + "add z19.s, z19.s, z2.s\n" + "add z20.s, z20.s, z4.s\n" + "add z21.s, z21.s, z0.s\n" + "add z22.s, z22.s, z3.s\n" + "add z23.s, z23.s, z2.s\n" + "add z24.s, z24.s, z4.s\n" + "add z25.s, z25.s, z0.s\n" + "add z26.s, z26.s, z3.s\n" + "add z27.s, z27.s, z2.s\n" + "add z28.s, z28.s, z4.s\n" + "add z29.s, z29.s, z0.s\n" "ld1rw { z0.s }, p2/Z, [x20]\n" - "add z30.s, z30.s, z2.s\n" - "add z31.s, z31.s, z3.s\n" - ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n" - ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n" - ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n" - ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n" - ".inst 0x04a47694 // sqrdmulh z20.s, z20.s, z4.s\n" - ".inst 0x04a476b5 // sqrdmulh z21.s, z21.s, z4.s\n" - ".inst 0x04a476d6 // sqrdmulh z22.s, z22.s, z4.s\n" - ".inst 0x04a476f7 // sqrdmulh z23.s, z23.s, z4.s\n" - ".inst 0x04a47718 // sqrdmulh z24.s, z24.s, z4.s\n" - ".inst 0x04a47739 // sqrdmulh z25.s, z25.s, z4.s\n" - ".inst 0x04a4775a // sqrdmulh z26.s, z26.s, z4.s\n" - ".inst 0x04a4777b // sqrdmulh z27.s, z27.s, z4.s\n" - ".inst 0x04a4779c // sqrdmulh z28.s, z28.s, z4.s\n" - ".inst 0x04a477bd // sqrdmulh z29.s, z29.s, z4.s\n" - ".inst 0x04a477de // sqrdmulh z30.s, z30.s, z4.s\n" - ".inst 0x04a477ff // sqrdmulh z31.s, z31.s, z4.s\n" + "add z30.s, z30.s, z3.s\n" + "add z31.s, z31.s, z2.s\n" + ".inst 0x04a17610 // sqrdmulh z16.s, z16.s, z1.s\n" + ".inst 0x04a17631 // sqrdmulh z17.s, z17.s, z1.s\n" + ".inst 0x04a17652 // sqrdmulh z18.s, z18.s, z1.s\n" + ".inst 0x04a17673 // sqrdmulh z19.s, z19.s, z1.s\n" + ".inst 0x04a17694 // sqrdmulh z20.s, z20.s, z1.s\n" + ".inst 0x04a176b5 // sqrdmulh z21.s, z21.s, z1.s\n" + ".inst 0x04a176d6 // sqrdmulh z22.s, z22.s, z1.s\n" + ".inst 0x04a176f7 // sqrdmulh z23.s, z23.s, z1.s\n" + ".inst 0x04a17718 // sqrdmulh z24.s, z24.s, z1.s\n" + ".inst 0x04a17739 // sqrdmulh z25.s, z25.s, z1.s\n" + ".inst 0x04a1775a // sqrdmulh z26.s, z26.s, z1.s\n" + ".inst 0x04a1777b // sqrdmulh z27.s, z27.s, z1.s\n" + ".inst 0x04a1779c // sqrdmulh z28.s, z28.s, z1.s\n" + ".inst 0x04a177bd // sqrdmulh z29.s, z29.s, z1.s\n" + ".inst 0x04a177de // sqrdmulh z30.s, z30.s, z1.s\n" + ".inst 0x04a177ff // sqrdmulh z31.s, z31.s, z1.s\n" "tbz %x[flags], #5, 55f\n" - "and z4.d, z16.d, z0.d\n" - "and z5.d, z17.d, z0.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "sqadd z16.s, z16.s, z4.s\n" - "sqadd z17.s, z17.s, z5.s\n" - "and z6.d, z18.d, z0.d\n" - "and z7.d, z19.d, z0.d\n" - "and z8.d, z20.d, z0.d\n" - "and z9.d, z21.d, z0.d\n" - "and z10.d, z22.d, z0.d\n" - "and z4.d, z23.d, z0.d\n" - "and z5.d, z24.d, z0.d\n" - "asr z6.s, z6.s, #0x1f\n" + "and z2.d, z16.d, z0.d\n" + "and z1.d, z17.d, z0.d\n" + "asr z2.s, z2.s, #0x1f\n" + "asr z1.s, z1.s, #0x1f\n" + "sqadd z16.s, z16.s, z2.s\n" + "sqadd z17.s, z17.s, z1.s\n" + "and z7.d, z18.d, z0.d\n" + "and z6.d, z19.d, z0.d\n" + "and z5.d, z20.d, z0.d\n" + "and z4.d, z21.d, z0.d\n" + "and z3.d, z22.d, z0.d\n" + "and z2.d, z23.d, z0.d\n" + "and z1.d, z24.d, z0.d\n" "asr z7.s, z7.s, #0x1f\n" - "asr z8.s, z8.s, #0x1f\n" - "asr z9.s, z9.s, #0x1f\n" - "asr z10.s, z10.s, #0x1f\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "sqadd z18.s, z18.s, z6.s\n" - "sqadd z19.s, z19.s, z7.s\n" - "sqadd z20.s, z20.s, z8.s\n" - "sqadd z21.s, z21.s, z9.s\n" - "sqadd z22.s, z22.s, z10.s\n" - "sqadd z23.s, z23.s, z4.s\n" - "sqadd z24.s, z24.s, z5.s\n" - "and z6.d, z25.d, z0.d\n" - "and z7.d, z26.d, z0.d\n" - "and z8.d, z27.d, z0.d\n" - "and z9.d, z28.d, z0.d\n" - "and z10.d, z29.d, z0.d\n" - "and z4.d, z30.d, z0.d\n" - "and z5.d, z31.d, z0.d\n" "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "asr z8.s, z8.s, #0x1f\n" - "asr z9.s, z9.s, #0x1f\n" - "asr z10.s, z10.s, #0x1f\n" + "asr z5.s, z5.s, #0x1f\n" "asr z4.s, z4.s, #0x1f\n" + "asr z3.s, z3.s, #0x1f\n" + "asr z2.s, z2.s, #0x1f\n" + "asr z1.s, z1.s, #0x1f\n" + "sqadd z18.s, z18.s, z7.s\n" + "sqadd z19.s, z19.s, z6.s\n" + "sqadd z20.s, z20.s, z5.s\n" + "sqadd z21.s, z21.s, z4.s\n" + "sqadd z22.s, z22.s, z3.s\n" + "sqadd z23.s, z23.s, z2.s\n" + "sqadd z24.s, z24.s, z1.s\n" + "and z7.d, z25.d, z0.d\n" + "and z6.d, z26.d, z0.d\n" + "and z5.d, z27.d, z0.d\n" + "and z4.d, z28.d, z0.d\n" + "and z3.d, z29.d, z0.d\n" + "and z2.d, z30.d, z0.d\n" + "and z1.d, z31.d, z0.d\n" + "asr z7.s, z7.s, #0x1f\n" + "asr z6.s, z6.s, #0x1f\n" "asr z5.s, z5.s, #0x1f\n" - "sqadd z25.s, z25.s, z6.s\n" - "sqadd z26.s, z26.s, z7.s\n" - "sqadd z27.s, z27.s, z8.s\n" - "sqadd z28.s, z28.s, z9.s\n" - "sqadd z29.s, z29.s, z10.s\n" - "sqadd z30.s, z30.s, z4.s\n" - "sqadd z31.s, z31.s, z5.s\n" + "asr z4.s, z4.s, #0x1f\n" + "asr z3.s, z3.s, #0x1f\n" + "asr z2.s, z2.s, #0x1f\n" + "asr z1.s, z1.s, #0x1f\n" + "sqadd z25.s, z25.s, z7.s\n" + "sqadd z26.s, z26.s, z6.s\n" + "sqadd z27.s, z27.s, z5.s\n" + "sqadd z28.s, z28.s, z4.s\n" + "sqadd z29.s, z29.s, z3.s\n" + "sqadd z30.s, z30.s, z2.s\n" + "sqadd z31.s, z31.s, z1.s\n" "55:" // Height 4: no shift correction "add x20, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x20]\n" + "ld1rw { z2.s }, p2/Z, [x20]\n" ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n" - "add z16.s, z16.s, z4.s\n" + "add z16.s, z16.s, z2.s\n" ".inst 0x44828811 // srshl z17.s, p2/M, z17.s, z0.s\n" ".inst 0x44828812 // srshl z18.s, p2/M, z18.s, z0.s\n" - "add z17.s, z17.s, z4.s\n" - "add z18.s, z18.s, z4.s\n" + "add z17.s, z17.s, z2.s\n" + "add z18.s, z18.s, z2.s\n" ".inst 0x44828813 // srshl z19.s, p2/M, z19.s, z0.s\n" ".inst 0x44828814 // srshl z20.s, p2/M, z20.s, z0.s\n" - "add z19.s, z19.s, z4.s\n" - "add z20.s, z20.s, z4.s\n" + "add z19.s, z19.s, z2.s\n" + "add z20.s, z20.s, z2.s\n" ".inst 0x44828815 // srshl z21.s, p2/M, z21.s, z0.s\n" ".inst 0x44828816 // srshl z22.s, p2/M, z22.s, z0.s\n" - "add z21.s, z21.s, z4.s\n" - "add z22.s, z22.s, z4.s\n" + "add z21.s, z21.s, z2.s\n" + "add z22.s, z22.s, z2.s\n" ".inst 0x44828817 // srshl z23.s, p2/M, z23.s, z0.s\n" ".inst 0x44828818 // srshl z24.s, p2/M, z24.s, z0.s\n" - "add z23.s, z23.s, z4.s\n" - "add z24.s, z24.s, z4.s\n" + "add z23.s, z23.s, z2.s\n" + "add z24.s, z24.s, z2.s\n" ".inst 0x44828819 // srshl z25.s, p2/M, z25.s, z0.s\n" ".inst 0x4482881a // srshl z26.s, p2/M, z26.s, z0.s\n" - "add z25.s, z25.s, z4.s\n" - "add z26.s, z26.s, z4.s\n" + "add z25.s, z25.s, z2.s\n" + "add z26.s, z26.s, z2.s\n" ".inst 0x4482881b // srshl z27.s, p2/M, z27.s, z0.s\n" ".inst 0x4482881c // srshl z28.s, p2/M, z28.s, z0.s\n" - "add z27.s, z27.s, z4.s\n" - "add z28.s, z28.s, z4.s\n" + "add z27.s, z27.s, z2.s\n" + "add z28.s, z28.s, z2.s\n" ".inst 0x4482881d // srshl z29.s, p2/M, z29.s, z0.s\n" ".inst 0x4482881e // srshl z30.s, p2/M, z30.s, z0.s\n" - "add z29.s, z29.s, z4.s\n" - "add z30.s, z30.s, z4.s\n" + "add z29.s, z29.s, z2.s\n" + "add z30.s, z30.s, z2.s\n" ".inst 0x4482881f // srshl z31.s, p2/M, z31.s, z0.s\n" "add x20, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x20]\n" - "add z31.s, z31.s, z4.s\n" + "ld1rw { z1.s }, p2/Z, [x20]\n" + "add z31.s, z31.s, z2.s\n" "add x20, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x20]\n" - "smin z16.s, p2/M, z16.s, z6.s\n" - "smin z17.s, p2/M, z17.s, z6.s\n" - "smin z18.s, p2/M, z18.s, z6.s\n" - "smin z19.s, p2/M, z19.s, z6.s\n" - "smin z20.s, p2/M, z20.s, z6.s\n" - "smin z21.s, p2/M, z21.s, z6.s\n" - "smin z22.s, p2/M, z22.s, z6.s\n" - "smin z23.s, p2/M, z23.s, z6.s\n" - "smin z24.s, p2/M, z24.s, z6.s\n" - "smin z25.s, p2/M, z25.s, z6.s\n" - "smin z26.s, p2/M, z26.s, z6.s\n" - "smin z27.s, p2/M, z27.s, z6.s\n" - "smin z28.s, p2/M, z28.s, z6.s\n" - "smin z29.s, p2/M, z29.s, z6.s\n" - "smin z30.s, p2/M, z30.s, z6.s\n" - "smin z31.s, p2/M, z31.s, z6.s\n" - "smax z16.s, p2/M, z16.s, z5.s\n" - "smax z17.s, p2/M, z17.s, z5.s\n" - "smax z18.s, p2/M, z18.s, z5.s\n" + "ld1rw { z0.s }, p2/Z, [x20]\n" + "smin z16.s, p2/M, z16.s, z1.s\n" + "smin z17.s, p2/M, z17.s, z1.s\n" + "smin z18.s, p2/M, z18.s, z1.s\n" + "smin z19.s, p2/M, z19.s, z1.s\n" + "smin z20.s, p2/M, z20.s, z1.s\n" + "smin z21.s, p2/M, z21.s, z1.s\n" + "smin z22.s, p2/M, z22.s, z1.s\n" + "smin z23.s, p2/M, z23.s, z1.s\n" + "smin z24.s, p2/M, z24.s, z1.s\n" + "smin z25.s, p2/M, z25.s, z1.s\n" + "smin z26.s, p2/M, z26.s, z1.s\n" + "smin z27.s, p2/M, z27.s, z1.s\n" + "smin z28.s, p2/M, z28.s, z1.s\n" + "smin z29.s, p2/M, z29.s, z1.s\n" + "smin z30.s, p2/M, z30.s, z1.s\n" + "smin z31.s, p2/M, z31.s, z1.s\n" + "smax z16.s, p2/M, z16.s, z0.s\n" + "smax z17.s, p2/M, z17.s, z0.s\n" + "smax z18.s, p2/M, z18.s, z0.s\n" "uzp1 z16.h, z16.h, z17.h\n" - "smax z19.s, p2/M, z19.s, z5.s\n" - "smax z20.s, p2/M, z20.s, z5.s\n" + "smax z19.s, p2/M, z19.s, z0.s\n" + "smax z20.s, p2/M, z20.s, z0.s\n" "uzp1 z17.h, z18.h, z19.h\n" "uzp1 z16.b, z16.b, z17.b\n" - "smax z21.s, p2/M, z21.s, z5.s\n" - "smax z22.s, p2/M, z22.s, z5.s\n" + "smax z21.s, p2/M, z21.s, z0.s\n" + "smax z22.s, p2/M, z22.s, z0.s\n" "uzp1 z20.h, z20.h, z21.h\n" "st1b { z16.b }, p1, [x27]\n" - "smax z23.s, p2/M, z23.s, z5.s\n" - "smax z24.s, p2/M, z24.s, z5.s\n" - "uzp1 z21.h, z22.h, z23.h\n" - "uzp1 z20.b, z20.b, z21.b\n" - "smax z25.s, p2/M, z25.s, z5.s\n" - "smax z26.s, p2/M, z26.s, z5.s\n" + "smax z23.s, p2/M, z23.s, z0.s\n" + "smax z24.s, p2/M, z24.s, z0.s\n" + "uzp1 z16.h, z22.h, z23.h\n" + "uzp1 z20.b, z20.b, z16.b\n" + "smax z25.s, p2/M, z25.s, z0.s\n" + "smax z26.s, p2/M, z26.s, z0.s\n" "uzp1 z24.h, z24.h, z25.h\n" "st1b { z20.b }, p1, [x23]\n" - "smax z27.s, p2/M, z27.s, z5.s\n" - "smax z28.s, p2/M, z28.s, z5.s\n" - "uzp1 z25.h, z26.h, z27.h\n" - "uzp1 z24.b, z24.b, z25.b\n" - "smax z29.s, p2/M, z29.s, z5.s\n" - "smax z30.s, p2/M, z30.s, z5.s\n" + "smax z27.s, p2/M, z27.s, z0.s\n" + "smax z28.s, p2/M, z28.s, z0.s\n" + "uzp1 z16.h, z26.h, z27.h\n" + "uzp1 z24.b, z24.b, z16.b\n" + "smax z29.s, p2/M, z29.s, z0.s\n" + "smax z30.s, p2/M, z30.s, z0.s\n" "uzp1 z28.h, z28.h, z29.h\n" "st1b { z24.b }, p1, [x22]\n" - "smax z31.s, p2/M, z31.s, z5.s\n" - "uzp1 z29.h, z30.h, z31.h\n" - "uzp1 z28.b, z28.b, z29.b\n" + "smax z31.s, p2/M, z31.s, z0.s\n" + "uzp1 z16.h, z30.h, z31.h\n" + "uzp1 z28.b, z28.b, z16.b\n" "st1b { z28.b }, p1, [x21]\n" "addvl x27, x27, #1\n" "56:" // Height 4: Writeback done @@ -1491,7 +1491,6 @@ void sve_hybrid_u8qa_dot_4x4VL ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "58:" // Exit - : [M] "+&r" (M), [flags] "+&r" (flags), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp) : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -1499,4 +1498,4 @@ void sve_hybrid_u8qa_dot_4x4VL ( } } // namespace arm_gemm -#endif // __ARM_FEATURE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_mmla_4x4VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_mmla_4x4VL.hpp index da27554a0f76abfc843d406a64fe2eeda0649c6e..5de68cc73804e352e26e6651c480ab576100cdd0 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_mmla_4x4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_mmla_4x4VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef ARM_COMPUTE_ENABLE_SVE + #include "../std_transforms_sve.hpp" #include "../performance_parameters.hpp" @@ -74,7 +74,6 @@ public: template static inline PerformanceParameters get_performance_parameters(const CPUInfo *ci) { - if (std::is_same::value) { switch (ci->get_cpu_model()) { default: @@ -97,5 +96,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_mmla_4x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_mmla_4x4VL/generic.cpp index f9d38c292569c343020bbc237472f2e6640a02f6..69894bec410a0475d14439365bd565f77daf79c9 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_mmla_4x4VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8qa_mmla_4x4VL/generic.cpp @@ -108,11 +108,11 @@ void sve_hybrid_u8qa_mmla_4x4VL ( "4:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 5f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" "cbnz x26, 6f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -125,41 +125,41 @@ void sve_hybrid_u8qa_mmla_4x4VL ( "7:" // Height 1: Multiply loop: Main loop head "whilelt p0.b, XZR, x25\n" "ld1rqb { z1.b }, p0/Z, [x24]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1b { z5.b }, p2/Z, [x28]\n" - ".inst 0x45c59810 // ummla z16.s, z0.b, z5.b\n" - "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" - "trn2 z1.d, z1.d, z2.d\n" - "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" - ".inst 0x45c69814 // ummla z20.s, z0.b, z6.b\n" - ".inst 0x45c79811 // ummla z17.s, z0.b, z7.b\n" - "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n" - "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n" - ".inst 0x45c89815 // ummla z21.s, z0.b, z8.b\n" - ".inst 0x45c99812 // ummla z18.s, z0.b, z9.b\n" - "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n" - "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n" + "trn1 z0.d, z1.d, z26.d\n" + "ld1b { z24.b }, p2/Z, [x28]\n" + ".inst 0x45d89810 // ummla z16.s, z0.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #1, MUL VL]\n" + "trn2 z1.d, z1.d, z26.d\n" + "ld1b { z24.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z26.b }, p2/Z, [x28, #3, MUL VL]\n" + ".inst 0x45d99814 // ummla z20.s, z0.b, z25.b\n" + ".inst 0x45d89811 // ummla z17.s, z0.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #4, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #5, MUL VL]\n" + ".inst 0x45da9815 // ummla z21.s, z0.b, z26.b\n" + ".inst 0x45d99812 // ummla z18.s, z0.b, z25.b\n" + "ld1b { z26.b }, p2/Z, [x28, #6, MUL VL]\n" + "ld1b { z25.b }, p2/Z, [x28, #7, MUL VL]\n" "addvl x28, x28, #16\n" - ".inst 0x45ca9816 // ummla z22.s, z0.b, z10.b\n" - "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n" - ".inst 0x45c49813 // ummla z19.s, z0.b, z4.b\n" - ".inst 0x45c59817 // ummla z23.s, z0.b, z5.b\n" - "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n" - ".inst 0x45c69830 // ummla z16.s, z1.b, z6.b\n" - "ld1b { z8.b }, p2/Z, [x28, #-6, MUL VL]\n" - "ld1b { z9.b }, p2/Z, [x28, #-5, MUL VL]\n" - ".inst 0x45c79834 // ummla z20.s, z1.b, z7.b\n" - "ld1b { z10.b }, p2/Z, [x28, #-4, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #-3, MUL VL]\n" - ".inst 0x45c89831 // ummla z17.s, z1.b, z8.b\n" - ".inst 0x45c99835 // ummla z21.s, z1.b, z9.b\n" - "ld1b { z5.b }, p2/Z, [x28, #-2, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x28, #-1, MUL VL]\n" - ".inst 0x45ca9832 // ummla z18.s, z1.b, z10.b\n" - ".inst 0x45c49836 // ummla z22.s, z1.b, z4.b\n" - ".inst 0x45c59833 // ummla z19.s, z1.b, z5.b\n" - ".inst 0x45c69837 // ummla z23.s, z1.b, z6.b\n" + ".inst 0x45d89816 // ummla z22.s, z0.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x28, #-8, MUL VL]\n" + ".inst 0x45da9813 // ummla z19.s, z0.b, z26.b\n" + ".inst 0x45d99817 // ummla z23.s, z0.b, z25.b\n" + "ld1b { z26.b }, p2/Z, [x28, #-7, MUL VL]\n" + ".inst 0x45d89830 // ummla z16.s, z1.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #-6, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #-5, MUL VL]\n" + ".inst 0x45da9834 // ummla z20.s, z1.b, z26.b\n" + "ld1b { z27.b }, p2/Z, [x28, #-4, MUL VL]\n" + "ld1b { z26.b }, p2/Z, [x28, #-3, MUL VL]\n" + ".inst 0x45d99831 // ummla z17.s, z1.b, z25.b\n" + ".inst 0x45d89835 // ummla z21.s, z1.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #-2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #-1, MUL VL]\n" + ".inst 0x45db9832 // ummla z18.s, z1.b, z27.b\n" + ".inst 0x45da9836 // ummla z22.s, z1.b, z26.b\n" + ".inst 0x45d99833 // ummla z19.s, z1.b, z25.b\n" + ".inst 0x45d89837 // ummla z23.s, z1.b, z24.b\n" "add x24, x24, #0x10\n" "tbnz %x[flags], #31, 8f\n" "udot z11.s, z0.b, z15.b\n" @@ -171,43 +171,43 @@ void sve_hybrid_u8qa_mmla_4x4VL ( "9:" // Height 1: Multiply loop: Single iteration only "whilelt p0.b, XZR, x25\n" "ld1rqb { z1.b }, p0/Z, [x24]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1b { z5.b }, p2/Z, [x28]\n" - ".inst 0x45c59810 // ummla z16.s, z0.b, z5.b\n" - "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" + "trn1 z0.d, z1.d, z27.d\n" + "ld1b { z24.b }, p2/Z, [x28]\n" + ".inst 0x45d89810 // ummla z16.s, z0.b, z24.b\n" + "ld1b { z26.b }, p2/Z, [x28, #1, MUL VL]\n" "subs x25, x25, #0x8\n" - "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" - "trn2 z1.d, z1.d, z2.d\n" - ".inst 0x45c69814 // ummla z20.s, z0.b, z6.b\n" - "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n" - "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n" - ".inst 0x45c79811 // ummla z17.s, z0.b, z7.b\n" - ".inst 0x45c89815 // ummla z21.s, z0.b, z8.b\n" - "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n" - "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n" - ".inst 0x45c99812 // ummla z18.s, z0.b, z9.b\n" - ".inst 0x45ca9816 // ummla z22.s, z0.b, z10.b\n" - ".inst 0x45c49813 // ummla z19.s, z0.b, z4.b\n" - ".inst 0x45c59817 // ummla z23.s, z0.b, z5.b\n" + "ld1b { z25.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #3, MUL VL]\n" + "trn2 z1.d, z1.d, z27.d\n" + ".inst 0x45da9814 // ummla z20.s, z0.b, z26.b\n" + "ld1b { z27.b }, p2/Z, [x28, #4, MUL VL]\n" + "ld1b { z26.b }, p2/Z, [x28, #5, MUL VL]\n" + ".inst 0x45d99811 // ummla z17.s, z0.b, z25.b\n" + ".inst 0x45d89815 // ummla z21.s, z0.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #6, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #7, MUL VL]\n" + ".inst 0x45db9812 // ummla z18.s, z0.b, z27.b\n" + ".inst 0x45da9816 // ummla z22.s, z0.b, z26.b\n" + ".inst 0x45d99813 // ummla z19.s, z0.b, z25.b\n" + ".inst 0x45d89817 // ummla z23.s, z0.b, z24.b\n" "addvl x28, x28, #8\n" "ble 10f\n" - "ld1b { z6.b }, p2/Z, [x28]\n" - ".inst 0x45c69830 // ummla z16.s, z1.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n" - ".inst 0x45c79834 // ummla z20.s, z1.b, z7.b\n" - "ld1b { z8.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z9.b }, p2/Z, [x28, #3, MUL VL]\n" - ".inst 0x45c89831 // ummla z17.s, z1.b, z8.b\n" - ".inst 0x45c99835 // ummla z21.s, z1.b, z9.b\n" - "ld1b { z10.b }, p2/Z, [x28, #4, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #5, MUL VL]\n" - ".inst 0x45ca9832 // ummla z18.s, z1.b, z10.b\n" - ".inst 0x45c49836 // ummla z22.s, z1.b, z4.b\n" - "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n" - ".inst 0x45c59833 // ummla z19.s, z1.b, z5.b\n" - ".inst 0x45c69837 // ummla z23.s, z1.b, z6.b\n" + "ld1b { z24.b }, p2/Z, [x28]\n" + ".inst 0x45d89830 // ummla z16.s, z1.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x28, #1, MUL VL]\n" + ".inst 0x45d89834 // ummla z20.s, z1.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #3, MUL VL]\n" + ".inst 0x45d99831 // ummla z17.s, z1.b, z25.b\n" + ".inst 0x45d89835 // ummla z21.s, z1.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #4, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #5, MUL VL]\n" + ".inst 0x45d99832 // ummla z18.s, z1.b, z25.b\n" + ".inst 0x45d89836 // ummla z22.s, z1.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #6, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #7, MUL VL]\n" + ".inst 0x45d99833 // ummla z19.s, z1.b, z25.b\n" + ".inst 0x45d89837 // ummla z23.s, z1.b, z24.b\n" "addvl x28, x28, #8\n" "10:" // Height 1: Multiply loop: multiply skip "tbnz %x[flags], #31, 11f\n" @@ -224,74 +224,74 @@ void sve_hybrid_u8qa_mmla_4x4VL ( "uzp1 z19.d, z19.d, z23.d\n" "mov z23.d, z16.d\n" "tbnz %x[flags], #31, 12f\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1rw { z1.s }, p2/Z, [x23]\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1rw { z16.s }, p2/Z, [x20]\n" ".inst 0x4491a96b // addp z11.s, p2/m, z11.s, z11.s\n" - "neg z1.s, p2/M, z1.s\n" + "neg z16.s, p2/M, z16.s\n" "mov z11.s, z11.s[0]\n" - "mul z11.s, p2/M, z11.s, z1.s\n" + "mul z11.s, p2/M, z11.s, z16.s\n" "12:" // Height 1: skip row sum fixup "add z23.s, z23.s, z11.s\n" "add z17.s, z17.s, z11.s\n" - "ld1w { z0.s }, p2/Z, [x10]\n" - "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x10]\n" + "ld1w { z21.s }, p2/Z, [x10, #1, MUL VL]\n" "add z18.s, z18.s, z11.s\n" "add z19.s, z19.s, z11.s\n" - "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n" - "add x23, %x[qp], %[per_layer_mul]\n" + "ld1w { z20.s }, p2/Z, [x10, #2, MUL VL]\n" + "ld1w { z16.s }, p2/Z, [x10, #3, MUL VL]\n" + "add x20, %x[qp], %[per_layer_mul]\n" "orr %x[flags], %x[flags], #0x80000000\n" - "add z23.s, z23.s, z0.s\n" - "add z17.s, z17.s, z1.s\n" - "add z18.s, z18.s, z2.s\n" - "add z19.s, z19.s, z3.s\n" - "ld1rw { z4.s }, p2/Z, [x23]\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" - "ld1rw { z0.s }, p2/Z, [x23]\n" - ".inst 0x04a476f7 // sqrdmulh z23.s, z23.s, z4.s\n" - ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n" + "add z23.s, z23.s, z22.s\n" + "add z17.s, z17.s, z21.s\n" + "add z18.s, z18.s, z20.s\n" + "add z19.s, z19.s, z16.s\n" + "ld1rw { z16.s }, p2/Z, [x20]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "ld1rw { z0.s }, p2/Z, [x20]\n" + ".inst 0x04b076f7 // sqrdmulh z23.s, z23.s, z16.s\n" + ".inst 0x04b07631 // sqrdmulh z17.s, z17.s, z16.s\n" "addvl x10, x10, #4\n" - ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n" - ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n" + ".inst 0x04b07652 // sqrdmulh z18.s, z18.s, z16.s\n" + ".inst 0x04b07673 // sqrdmulh z19.s, z19.s, z16.s\n" "tbz %x[flags], #5, 13f\n" - "and z4.d, z23.d, z0.d\n" - "and z5.d, z17.d, z0.d\n" - "and z6.d, z18.d, z0.d\n" - "and z7.d, z19.d, z0.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "sqadd z23.s, z23.s, z4.s\n" - "sqadd z17.s, z17.s, z5.s\n" - "sqadd z18.s, z18.s, z6.s\n" - "sqadd z19.s, z19.s, z7.s\n" + "and z22.d, z23.d, z0.d\n" + "and z21.d, z17.d, z0.d\n" + "and z20.d, z18.d, z0.d\n" + "and z16.d, z19.d, z0.d\n" + "asr z22.s, z22.s, #0x1f\n" + "asr z21.s, z21.s, #0x1f\n" + "asr z20.s, z20.s, #0x1f\n" + "asr z16.s, z16.s, #0x1f\n" + "sqadd z23.s, z23.s, z22.s\n" + "sqadd z17.s, z17.s, z21.s\n" + "sqadd z18.s, z18.s, z20.s\n" + "sqadd z19.s, z19.s, z16.s\n" "13:" // Height 1: no shift correction - "add x23, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x23]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1rw { z16.s }, p2/Z, [x20]\n" ".inst 0x44828817 // srshl z23.s, p2/M, z23.s, z0.s\n" - "add z23.s, z23.s, z4.s\n" + "add z23.s, z23.s, z16.s\n" ".inst 0x44828811 // srshl z17.s, p2/M, z17.s, z0.s\n" ".inst 0x44828812 // srshl z18.s, p2/M, z18.s, z0.s\n" - "add z17.s, z17.s, z4.s\n" - "add z18.s, z18.s, z4.s\n" + "add z17.s, z17.s, z16.s\n" + "add z18.s, z18.s, z16.s\n" ".inst 0x44828813 // srshl z19.s, p2/M, z19.s, z0.s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x23]\n" - "add z19.s, z19.s, z4.s\n" - "add x23, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x23]\n" - "smin z23.s, p2/M, z23.s, z6.s\n" - "smin z17.s, p2/M, z17.s, z6.s\n" - "smin z18.s, p2/M, z18.s, z6.s\n" - "smin z19.s, p2/M, z19.s, z6.s\n" - "smax z23.s, p2/M, z23.s, z5.s\n" - "smax z17.s, p2/M, z17.s, z5.s\n" - "smax z18.s, p2/M, z18.s, z5.s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1rw { z20.s }, p2/Z, [x20]\n" + "add z19.s, z19.s, z16.s\n" + "add x20, %x[qp], %[minval]\n" + "ld1rw { z16.s }, p2/Z, [x20]\n" + "smin z23.s, p2/M, z23.s, z20.s\n" + "smin z17.s, p2/M, z17.s, z20.s\n" + "smin z18.s, p2/M, z18.s, z20.s\n" + "smin z19.s, p2/M, z19.s, z20.s\n" + "smax z23.s, p2/M, z23.s, z16.s\n" + "smax z17.s, p2/M, z17.s, z16.s\n" + "smax z18.s, p2/M, z18.s, z16.s\n" "uzp1 z23.h, z23.h, z17.h\n" - "smax z19.s, p2/M, z19.s, z5.s\n" - "uzp1 z17.h, z18.h, z19.h\n" - "uzp1 z23.b, z23.b, z17.b\n" + "smax z19.s, p2/M, z19.s, z16.s\n" + "uzp1 z16.h, z18.h, z19.h\n" + "uzp1 z23.b, z23.b, z16.b\n" "st1b { z23.b }, p1, [x27]\n" "addvl x27, x27, #1\n" "14:" // Height 1: Writeback done @@ -324,12 +324,12 @@ void sve_hybrid_u8qa_mmla_4x4VL ( "18:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 19f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" "cbnz x26, 20f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -337,49 +337,49 @@ void sve_hybrid_u8qa_mmla_4x4VL ( "b 20f\n" "19:" // Height 2: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20\n" + "add x23, x24, x21\n" "20:" // Height 2: input setup done "cmp x25, #0x10\n" "ble 23f\n" "21:" // Height 2: Multiply loop: Main loop head "whilelt p0.b, XZR, x25\n" "ld1rqb { z1.b }, p0/Z, [x24]\n" - "ld1rqb { z2.b }, p0/Z, [x23]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1b { z5.b }, p2/Z, [x28]\n" - ".inst 0x45c59810 // ummla z16.s, z0.b, z5.b\n" - "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" - "trn2 z1.d, z1.d, z2.d\n" - "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" - ".inst 0x45c69814 // ummla z20.s, z0.b, z6.b\n" - ".inst 0x45c79811 // ummla z17.s, z0.b, z7.b\n" - "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n" - "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n" - ".inst 0x45c89815 // ummla z21.s, z0.b, z8.b\n" - ".inst 0x45c99812 // ummla z18.s, z0.b, z9.b\n" - "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n" - "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n" + "ld1rqb { z26.b }, p0/Z, [x23]\n" + "trn1 z0.d, z1.d, z26.d\n" + "ld1b { z24.b }, p2/Z, [x28]\n" + ".inst 0x45d89810 // ummla z16.s, z0.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #1, MUL VL]\n" + "trn2 z1.d, z1.d, z26.d\n" + "ld1b { z24.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z26.b }, p2/Z, [x28, #3, MUL VL]\n" + ".inst 0x45d99814 // ummla z20.s, z0.b, z25.b\n" + ".inst 0x45d89811 // ummla z17.s, z0.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #4, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #5, MUL VL]\n" + ".inst 0x45da9815 // ummla z21.s, z0.b, z26.b\n" + ".inst 0x45d99812 // ummla z18.s, z0.b, z25.b\n" + "ld1b { z26.b }, p2/Z, [x28, #6, MUL VL]\n" + "ld1b { z25.b }, p2/Z, [x28, #7, MUL VL]\n" "addvl x28, x28, #16\n" - ".inst 0x45ca9816 // ummla z22.s, z0.b, z10.b\n" - "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n" - ".inst 0x45c49813 // ummla z19.s, z0.b, z4.b\n" - ".inst 0x45c59817 // ummla z23.s, z0.b, z5.b\n" - "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n" - ".inst 0x45c69830 // ummla z16.s, z1.b, z6.b\n" - "ld1b { z8.b }, p2/Z, [x28, #-6, MUL VL]\n" - "ld1b { z9.b }, p2/Z, [x28, #-5, MUL VL]\n" - ".inst 0x45c79834 // ummla z20.s, z1.b, z7.b\n" - "ld1b { z10.b }, p2/Z, [x28, #-4, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #-3, MUL VL]\n" - ".inst 0x45c89831 // ummla z17.s, z1.b, z8.b\n" - ".inst 0x45c99835 // ummla z21.s, z1.b, z9.b\n" - "ld1b { z5.b }, p2/Z, [x28, #-2, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x28, #-1, MUL VL]\n" - ".inst 0x45ca9832 // ummla z18.s, z1.b, z10.b\n" - ".inst 0x45c49836 // ummla z22.s, z1.b, z4.b\n" - ".inst 0x45c59833 // ummla z19.s, z1.b, z5.b\n" - ".inst 0x45c69837 // ummla z23.s, z1.b, z6.b\n" + ".inst 0x45d89816 // ummla z22.s, z0.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x28, #-8, MUL VL]\n" + ".inst 0x45da9813 // ummla z19.s, z0.b, z26.b\n" + ".inst 0x45d99817 // ummla z23.s, z0.b, z25.b\n" + "ld1b { z26.b }, p2/Z, [x28, #-7, MUL VL]\n" + ".inst 0x45d89830 // ummla z16.s, z1.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #-6, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #-5, MUL VL]\n" + ".inst 0x45da9834 // ummla z20.s, z1.b, z26.b\n" + "ld1b { z27.b }, p2/Z, [x28, #-4, MUL VL]\n" + "ld1b { z26.b }, p2/Z, [x28, #-3, MUL VL]\n" + ".inst 0x45d99831 // ummla z17.s, z1.b, z25.b\n" + ".inst 0x45d89835 // ummla z21.s, z1.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #-2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #-1, MUL VL]\n" + ".inst 0x45db9832 // ummla z18.s, z1.b, z27.b\n" + ".inst 0x45da9836 // ummla z22.s, z1.b, z26.b\n" + ".inst 0x45d99833 // ummla z19.s, z1.b, z25.b\n" + ".inst 0x45d89837 // ummla z23.s, z1.b, z24.b\n" "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" "tbnz %x[flags], #31, 22f\n" @@ -392,44 +392,44 @@ void sve_hybrid_u8qa_mmla_4x4VL ( "23:" // Height 2: Multiply loop: Single iteration only "whilelt p0.b, XZR, x25\n" "ld1rqb { z1.b }, p0/Z, [x24]\n" - "ld1rqb { z2.b }, p0/Z, [x23]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1b { z5.b }, p2/Z, [x28]\n" - ".inst 0x45c59810 // ummla z16.s, z0.b, z5.b\n" - "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1rqb { z27.b }, p0/Z, [x23]\n" + "trn1 z0.d, z1.d, z27.d\n" + "ld1b { z24.b }, p2/Z, [x28]\n" + ".inst 0x45d89810 // ummla z16.s, z0.b, z24.b\n" + "ld1b { z26.b }, p2/Z, [x28, #1, MUL VL]\n" "subs x25, x25, #0x8\n" - "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" - "trn2 z1.d, z1.d, z2.d\n" - ".inst 0x45c69814 // ummla z20.s, z0.b, z6.b\n" - "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n" - "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n" - ".inst 0x45c79811 // ummla z17.s, z0.b, z7.b\n" - ".inst 0x45c89815 // ummla z21.s, z0.b, z8.b\n" - "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n" - "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n" - ".inst 0x45c99812 // ummla z18.s, z0.b, z9.b\n" - ".inst 0x45ca9816 // ummla z22.s, z0.b, z10.b\n" - ".inst 0x45c49813 // ummla z19.s, z0.b, z4.b\n" - ".inst 0x45c59817 // ummla z23.s, z0.b, z5.b\n" + "ld1b { z25.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #3, MUL VL]\n" + "trn2 z1.d, z1.d, z27.d\n" + ".inst 0x45da9814 // ummla z20.s, z0.b, z26.b\n" + "ld1b { z27.b }, p2/Z, [x28, #4, MUL VL]\n" + "ld1b { z26.b }, p2/Z, [x28, #5, MUL VL]\n" + ".inst 0x45d99811 // ummla z17.s, z0.b, z25.b\n" + ".inst 0x45d89815 // ummla z21.s, z0.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #6, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #7, MUL VL]\n" + ".inst 0x45db9812 // ummla z18.s, z0.b, z27.b\n" + ".inst 0x45da9816 // ummla z22.s, z0.b, z26.b\n" + ".inst 0x45d99813 // ummla z19.s, z0.b, z25.b\n" + ".inst 0x45d89817 // ummla z23.s, z0.b, z24.b\n" "addvl x28, x28, #8\n" "ble 24f\n" - "ld1b { z6.b }, p2/Z, [x28]\n" - ".inst 0x45c69830 // ummla z16.s, z1.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n" - ".inst 0x45c79834 // ummla z20.s, z1.b, z7.b\n" - "ld1b { z8.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z9.b }, p2/Z, [x28, #3, MUL VL]\n" - ".inst 0x45c89831 // ummla z17.s, z1.b, z8.b\n" - ".inst 0x45c99835 // ummla z21.s, z1.b, z9.b\n" - "ld1b { z10.b }, p2/Z, [x28, #4, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #5, MUL VL]\n" - ".inst 0x45ca9832 // ummla z18.s, z1.b, z10.b\n" - ".inst 0x45c49836 // ummla z22.s, z1.b, z4.b\n" - "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n" - ".inst 0x45c59833 // ummla z19.s, z1.b, z5.b\n" - ".inst 0x45c69837 // ummla z23.s, z1.b, z6.b\n" + "ld1b { z24.b }, p2/Z, [x28]\n" + ".inst 0x45d89830 // ummla z16.s, z1.b, z24.b\n" + "ld1b { z24.b }, p2/Z, [x28, #1, MUL VL]\n" + ".inst 0x45d89834 // ummla z20.s, z1.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #3, MUL VL]\n" + ".inst 0x45d99831 // ummla z17.s, z1.b, z25.b\n" + ".inst 0x45d89835 // ummla z21.s, z1.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #4, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #5, MUL VL]\n" + ".inst 0x45d99832 // ummla z18.s, z1.b, z25.b\n" + ".inst 0x45d89836 // ummla z22.s, z1.b, z24.b\n" + "ld1b { z25.b }, p2/Z, [x28, #6, MUL VL]\n" + "ld1b { z24.b }, p2/Z, [x28, #7, MUL VL]\n" + ".inst 0x45d99833 // ummla z19.s, z1.b, z25.b\n" + ".inst 0x45d89837 // ummla z23.s, z1.b, z24.b\n" "addvl x28, x28, #8\n" "24:" // Height 2: Multiply loop: multiply skip "tbnz %x[flags], #31, 25f\n" @@ -440,133 +440,133 @@ void sve_hybrid_u8qa_mmla_4x4VL ( "add x26, x26, #0x1\n" "cmp x26, x20\n" "bne 18b\n" - "uzp1 z7.d, z16.d, z20.d\n" + "uzp1 z24.d, z16.d, z20.d\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "uzp2 z16.d, z16.d, z20.d\n" - "add x22, x27, x20\n" + "add x23, x27, x20\n" "uzp1 z20.d, z17.d, z21.d\n" "uzp2 z17.d, z17.d, z21.d\n" "uzp1 z21.d, z18.d, z22.d\n" "uzp2 z18.d, z18.d, z22.d\n" "uzp1 z22.d, z19.d, z23.d\n" "uzp2 z19.d, z19.d, z23.d\n" - "mov z23.d, z7.d\n" + "mov z23.d, z24.d\n" "tbnz %x[flags], #31, 26f\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1rw { z2.s }, p2/Z, [x23]\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1rw { z24.s }, p2/Z, [x20]\n" ".inst 0x4491a96b // addp z11.s, p2/m, z11.s, z11.s\n" - "neg z2.s, p2/M, z2.s\n" + "neg z24.s, p2/M, z24.s\n" "mov z12.s, z11.s[3]\n" "mov z11.s, z11.s[0]\n" - "mul z11.s, p2/M, z11.s, z2.s\n" - "mul z12.s, p2/M, z12.s, z2.s\n" + "mul z11.s, p2/M, z11.s, z24.s\n" + "mul z12.s, p2/M, z12.s, z24.s\n" "26:" // Height 2: skip row sum fixup "add z23.s, z23.s, z11.s\n" "add z20.s, z20.s, z11.s\n" - "ld1w { z0.s }, p2/Z, [x10]\n" - "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n" + "ld1w { z28.s }, p2/Z, [x10]\n" + "ld1w { z27.s }, p2/Z, [x10, #1, MUL VL]\n" "add z21.s, z21.s, z11.s\n" "add z22.s, z22.s, z11.s\n" - "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n" + "ld1w { z26.s }, p2/Z, [x10, #2, MUL VL]\n" + "ld1w { z25.s }, p2/Z, [x10, #3, MUL VL]\n" "add z16.s, z16.s, z12.s\n" "add z17.s, z17.s, z12.s\n" - "add x23, %x[qp], %[per_layer_mul]\n" + "add x20, %x[qp], %[per_layer_mul]\n" "orr %x[flags], %x[flags], #0x80000000\n" "add z18.s, z18.s, z12.s\n" "add z19.s, z19.s, z12.s\n" - "ld1rw { z4.s }, p2/Z, [x23]\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" - "add z23.s, z23.s, z0.s\n" - "add z20.s, z20.s, z1.s\n" + "ld1rw { z24.s }, p2/Z, [x20]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" + "add z23.s, z23.s, z28.s\n" + "add z20.s, z20.s, z27.s\n" "addvl x10, x10, #4\n" - "add z21.s, z21.s, z2.s\n" - "add z22.s, z22.s, z3.s\n" - "add z16.s, z16.s, z0.s\n" - "add z17.s, z17.s, z1.s\n" - "ld1rw { z0.s }, p2/Z, [x23]\n" - "add z18.s, z18.s, z2.s\n" - "add z19.s, z19.s, z3.s\n" - ".inst 0x04a476f7 // sqrdmulh z23.s, z23.s, z4.s\n" - ".inst 0x04a47694 // sqrdmulh z20.s, z20.s, z4.s\n" - ".inst 0x04a476b5 // sqrdmulh z21.s, z21.s, z4.s\n" - ".inst 0x04a476d6 // sqrdmulh z22.s, z22.s, z4.s\n" - ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n" - ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n" - ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n" - ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n" + "add z21.s, z21.s, z26.s\n" + "add z22.s, z22.s, z25.s\n" + "add z16.s, z16.s, z28.s\n" + "add z17.s, z17.s, z27.s\n" + "ld1rw { z0.s }, p2/Z, [x20]\n" + "add z18.s, z18.s, z26.s\n" + "add z19.s, z19.s, z25.s\n" + ".inst 0x04b876f7 // sqrdmulh z23.s, z23.s, z24.s\n" + ".inst 0x04b87694 // sqrdmulh z20.s, z20.s, z24.s\n" + ".inst 0x04b876b5 // sqrdmulh z21.s, z21.s, z24.s\n" + ".inst 0x04b876d6 // sqrdmulh z22.s, z22.s, z24.s\n" + ".inst 0x04b87610 // sqrdmulh z16.s, z16.s, z24.s\n" + ".inst 0x04b87631 // sqrdmulh z17.s, z17.s, z24.s\n" + ".inst 0x04b87652 // sqrdmulh z18.s, z18.s, z24.s\n" + ".inst 0x04b87673 // sqrdmulh z19.s, z19.s, z24.s\n" "tbz %x[flags], #5, 27f\n" - "and z4.d, z23.d, z0.d\n" - "asr z4.s, z4.s, #0x1f\n" - "sqadd z23.s, z23.s, z4.s\n" - "and z5.d, z20.d, z0.d\n" - "and z6.d, z21.d, z0.d\n" - "and z7.d, z22.d, z0.d\n" - "and z8.d, z16.d, z0.d\n" - "and z9.d, z17.d, z0.d\n" - "and z10.d, z18.d, z0.d\n" - "and z4.d, z19.d, z0.d\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "asr z8.s, z8.s, #0x1f\n" - "asr z9.s, z9.s, #0x1f\n" - "asr z10.s, z10.s, #0x1f\n" - "asr z4.s, z4.s, #0x1f\n" - "sqadd z20.s, z20.s, z5.s\n" - "sqadd z21.s, z21.s, z6.s\n" - "sqadd z22.s, z22.s, z7.s\n" - "sqadd z16.s, z16.s, z8.s\n" - "sqadd z17.s, z17.s, z9.s\n" - "sqadd z18.s, z18.s, z10.s\n" - "sqadd z19.s, z19.s, z4.s\n" + "and z24.d, z23.d, z0.d\n" + "asr z24.s, z24.s, #0x1f\n" + "sqadd z23.s, z23.s, z24.s\n" + "and z30.d, z20.d, z0.d\n" + "and z29.d, z21.d, z0.d\n" + "and z28.d, z22.d, z0.d\n" + "and z27.d, z16.d, z0.d\n" + "and z26.d, z17.d, z0.d\n" + "and z25.d, z18.d, z0.d\n" + "and z24.d, z19.d, z0.d\n" + "asr z30.s, z30.s, #0x1f\n" + "asr z29.s, z29.s, #0x1f\n" + "asr z28.s, z28.s, #0x1f\n" + "asr z27.s, z27.s, #0x1f\n" + "asr z26.s, z26.s, #0x1f\n" + "asr z25.s, z25.s, #0x1f\n" + "asr z24.s, z24.s, #0x1f\n" + "sqadd z20.s, z20.s, z30.s\n" + "sqadd z21.s, z21.s, z29.s\n" + "sqadd z22.s, z22.s, z28.s\n" + "sqadd z16.s, z16.s, z27.s\n" + "sqadd z17.s, z17.s, z26.s\n" + "sqadd z18.s, z18.s, z25.s\n" + "sqadd z19.s, z19.s, z24.s\n" "27:" // Height 2: no shift correction - "add x23, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x23]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1rw { z24.s }, p2/Z, [x20]\n" ".inst 0x44828817 // srshl z23.s, p2/M, z23.s, z0.s\n" - "add z23.s, z23.s, z4.s\n" + "add z23.s, z23.s, z24.s\n" ".inst 0x44828814 // srshl z20.s, p2/M, z20.s, z0.s\n" ".inst 0x44828815 // srshl z21.s, p2/M, z21.s, z0.s\n" - "add z20.s, z20.s, z4.s\n" - "add z21.s, z21.s, z4.s\n" + "add z20.s, z20.s, z24.s\n" + "add z21.s, z21.s, z24.s\n" ".inst 0x44828816 // srshl z22.s, p2/M, z22.s, z0.s\n" ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n" - "add z22.s, z22.s, z4.s\n" - "add z16.s, z16.s, z4.s\n" + "add z22.s, z22.s, z24.s\n" + "add z16.s, z16.s, z24.s\n" ".inst 0x44828811 // srshl z17.s, p2/M, z17.s, z0.s\n" ".inst 0x44828812 // srshl z18.s, p2/M, z18.s, z0.s\n" - "add z17.s, z17.s, z4.s\n" - "add z18.s, z18.s, z4.s\n" + "add z17.s, z17.s, z24.s\n" + "add z18.s, z18.s, z24.s\n" ".inst 0x44828813 // srshl z19.s, p2/M, z19.s, z0.s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x23]\n" - "add z19.s, z19.s, z4.s\n" - "add x23, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x23]\n" - "smin z23.s, p2/M, z23.s, z6.s\n" - "smin z20.s, p2/M, z20.s, z6.s\n" - "smin z21.s, p2/M, z21.s, z6.s\n" - "smin z22.s, p2/M, z22.s, z6.s\n" - "smin z16.s, p2/M, z16.s, z6.s\n" - "smin z17.s, p2/M, z17.s, z6.s\n" - "smin z18.s, p2/M, z18.s, z6.s\n" - "smin z19.s, p2/M, z19.s, z6.s\n" - "smax z23.s, p2/M, z23.s, z5.s\n" - "smax z20.s, p2/M, z20.s, z5.s\n" - "smax z21.s, p2/M, z21.s, z5.s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1rw { z25.s }, p2/Z, [x20]\n" + "add z19.s, z19.s, z24.s\n" + "add x20, %x[qp], %[minval]\n" + "ld1rw { z24.s }, p2/Z, [x20]\n" + "smin z23.s, p2/M, z23.s, z25.s\n" + "smin z20.s, p2/M, z20.s, z25.s\n" + "smin z21.s, p2/M, z21.s, z25.s\n" + "smin z22.s, p2/M, z22.s, z25.s\n" + "smin z16.s, p2/M, z16.s, z25.s\n" + "smin z17.s, p2/M, z17.s, z25.s\n" + "smin z18.s, p2/M, z18.s, z25.s\n" + "smin z19.s, p2/M, z19.s, z25.s\n" + "smax z23.s, p2/M, z23.s, z24.s\n" + "smax z20.s, p2/M, z20.s, z24.s\n" + "smax z21.s, p2/M, z21.s, z24.s\n" "uzp1 z23.h, z23.h, z20.h\n" - "smax z22.s, p2/M, z22.s, z5.s\n" - "smax z16.s, p2/M, z16.s, z5.s\n" + "smax z22.s, p2/M, z22.s, z24.s\n" + "smax z16.s, p2/M, z16.s, z24.s\n" "uzp1 z20.h, z21.h, z22.h\n" "uzp1 z23.b, z23.b, z20.b\n" - "smax z17.s, p2/M, z17.s, z5.s\n" - "smax z18.s, p2/M, z18.s, z5.s\n" + "smax z17.s, p2/M, z17.s, z24.s\n" + "smax z18.s, p2/M, z18.s, z24.s\n" "uzp1 z16.h, z16.h, z17.h\n" "st1b { z23.b }, p1, [x27]\n" - "smax z19.s, p2/M, z19.s, z5.s\n" + "smax z19.s, p2/M, z19.s, z24.s\n" "uzp1 z17.h, z18.h, z19.h\n" "uzp1 z16.b, z16.b, z17.b\n" - "st1b { z16.b }, p1, [x22]\n" + "st1b { z16.b }, p1, [x23]\n" "addvl x27, x27, #1\n" "28:" // Height 2: Writeback done "decw x9, ALL, MUL #4\n" @@ -607,13 +607,13 @@ void sve_hybrid_u8qa_mmla_4x4VL ( "32:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 33f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" "cbnz x26, 34f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -622,8 +622,8 @@ void sve_hybrid_u8qa_mmla_4x4VL ( "b 34f\n" "33:" // Height 3: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" "34:" // Height 3: input setup done "cmp x25, #0x10\n" "ble 37f\n" @@ -634,60 +634,60 @@ void sve_hybrid_u8qa_mmla_4x4VL ( "ld1rqb { z3.b }, p0/Z, [x22]\n" "trn1 z0.d, z1.d, z2.d\n" "trn2 z1.d, z1.d, z2.d\n" - "ld1b { z5.b }, p2/Z, [x28]\n" - "trn1 z2.d, z3.d, z4.d\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x45c59810 // ummla z16.s, z0.b, z5.b\n" - "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" - ".inst 0x45c59858 // ummla z24.s, z2.b, z5.b\n" - "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" - ".inst 0x45c69814 // ummla z20.s, z0.b, z6.b\n" - "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n" - "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n" - ".inst 0x45c6985c // ummla z28.s, z2.b, z6.b\n" - ".inst 0x45c79811 // ummla z17.s, z0.b, z7.b\n" - "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n" + "ld1b { z4.b }, p2/Z, [x28]\n" + "trn1 z2.d, z3.d, z5.d\n" + "trn2 z3.d, z3.d, z5.d\n" + ".inst 0x45c49810 // ummla z16.s, z0.b, z4.b\n" + "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n" + ".inst 0x45c49858 // ummla z24.s, z2.b, z4.b\n" + "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z9.b }, p2/Z, [x28, #3, MUL VL]\n" + ".inst 0x45c59814 // ummla z20.s, z0.b, z5.b\n" + "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n" + "ld1b { z7.b }, p2/Z, [x28, #5, MUL VL]\n" + ".inst 0x45c5985c // ummla z28.s, z2.b, z5.b\n" + ".inst 0x45c49811 // ummla z17.s, z0.b, z4.b\n" + "ld1b { z6.b }, p2/Z, [x28, #6, MUL VL]\n" "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n" "addvl x28, x28, #16\n" - ".inst 0x45c79859 // ummla z25.s, z2.b, z7.b\n" - "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n" - ".inst 0x45c89815 // ummla z21.s, z0.b, z8.b\n" - ".inst 0x45c8985d // ummla z29.s, z2.b, z8.b\n" - "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n" - ".inst 0x45c99812 // ummla z18.s, z0.b, z9.b\n" - ".inst 0x45c9985a // ummla z26.s, z2.b, z9.b\n" - "ld1b { z8.b }, p2/Z, [x28, #-6, MUL VL]\n" - "ld1b { z9.b }, p2/Z, [x28, #-5, MUL VL]\n" - ".inst 0x45ca9816 // ummla z22.s, z0.b, z10.b\n" - ".inst 0x45ca985e // ummla z30.s, z2.b, z10.b\n" - "ld1b { z10.b }, p2/Z, [x28, #-4, MUL VL]\n" + ".inst 0x45c49859 // ummla z25.s, z2.b, z4.b\n" + "ld1b { z4.b }, p2/Z, [x28, #-8, MUL VL]\n" + ".inst 0x45c99815 // ummla z21.s, z0.b, z9.b\n" + ".inst 0x45c9985d // ummla z29.s, z2.b, z9.b\n" + "ld1b { z10.b }, p2/Z, [x28, #-7, MUL VL]\n" + ".inst 0x45c89812 // ummla z18.s, z0.b, z8.b\n" + ".inst 0x45c8985a // ummla z26.s, z2.b, z8.b\n" + "ld1b { z9.b }, p2/Z, [x28, #-6, MUL VL]\n" + "ld1b { z8.b }, p2/Z, [x28, #-5, MUL VL]\n" + ".inst 0x45c79816 // ummla z22.s, z0.b, z7.b\n" + ".inst 0x45c7985e // ummla z30.s, z2.b, z7.b\n" + "ld1b { z7.b }, p2/Z, [x28, #-4, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x45c49813 // ummla z19.s, z0.b, z4.b\n" - ".inst 0x45c4985b // ummla z27.s, z2.b, z4.b\n" - "ld1b { z4.b }, p2/Z, [x28, #-3, MUL VL]\n" + ".inst 0x45c69813 // ummla z19.s, z0.b, z6.b\n" + ".inst 0x45c6985b // ummla z27.s, z2.b, z6.b\n" + "ld1b { z6.b }, p2/Z, [x28, #-3, MUL VL]\n" "add x23, x23, #0x10\n" ".inst 0x45c59817 // ummla z23.s, z0.b, z5.b\n" ".inst 0x45c5985f // ummla z31.s, z2.b, z5.b\n" "ld1b { z5.b }, p2/Z, [x28, #-2, MUL VL]\n" "add x22, x22, #0x10\n" - ".inst 0x45c69830 // ummla z16.s, z1.b, z6.b\n" - ".inst 0x45c69878 // ummla z24.s, z3.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x28, #-1, MUL VL]\n" - ".inst 0x45c79834 // ummla z20.s, z1.b, z7.b\n" - ".inst 0x45c7987c // ummla z28.s, z3.b, z7.b\n" - ".inst 0x45c89831 // ummla z17.s, z1.b, z8.b\n" - ".inst 0x45c89879 // ummla z25.s, z3.b, z8.b\n" - ".inst 0x45c99835 // ummla z21.s, z1.b, z9.b\n" - ".inst 0x45c9987d // ummla z29.s, z3.b, z9.b\n" - ".inst 0x45ca9832 // ummla z18.s, z1.b, z10.b\n" - ".inst 0x45ca987a // ummla z26.s, z3.b, z10.b\n" - ".inst 0x45c49836 // ummla z22.s, z1.b, z4.b\n" - ".inst 0x45c4987e // ummla z30.s, z3.b, z4.b\n" + ".inst 0x45c49830 // ummla z16.s, z1.b, z4.b\n" + ".inst 0x45c49878 // ummla z24.s, z3.b, z4.b\n" + "ld1b { z4.b }, p2/Z, [x28, #-1, MUL VL]\n" + ".inst 0x45ca9834 // ummla z20.s, z1.b, z10.b\n" + ".inst 0x45ca987c // ummla z28.s, z3.b, z10.b\n" + ".inst 0x45c99831 // ummla z17.s, z1.b, z9.b\n" + ".inst 0x45c99879 // ummla z25.s, z3.b, z9.b\n" + ".inst 0x45c89835 // ummla z21.s, z1.b, z8.b\n" + ".inst 0x45c8987d // ummla z29.s, z3.b, z8.b\n" + ".inst 0x45c79832 // ummla z18.s, z1.b, z7.b\n" + ".inst 0x45c7987a // ummla z26.s, z3.b, z7.b\n" + ".inst 0x45c69836 // ummla z22.s, z1.b, z6.b\n" + ".inst 0x45c6987e // ummla z30.s, z3.b, z6.b\n" ".inst 0x45c59833 // ummla z19.s, z1.b, z5.b\n" ".inst 0x45c5987b // ummla z27.s, z3.b, z5.b\n" - ".inst 0x45c69837 // ummla z23.s, z1.b, z6.b\n" - ".inst 0x45c6987f // ummla z31.s, z3.b, z6.b\n" + ".inst 0x45c49837 // ummla z23.s, z1.b, z4.b\n" + ".inst 0x45c4987f // ummla z31.s, z3.b, z4.b\n" "tbnz %x[flags], #31, 36f\n" "udot z11.s, z0.b, z15.b\n" "udot z13.s, z2.b, z15.b\n" @@ -708,56 +708,56 @@ void sve_hybrid_u8qa_mmla_4x4VL ( "trn1 z2.d, z3.d, z4.d\n" "trn2 z3.d, z3.d, z4.d\n" ".inst 0x45c59810 // ummla z16.s, z0.b, z5.b\n" - "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z4.b }, p2/Z, [x28, #1, MUL VL]\n" ".inst 0x45c59858 // ummla z24.s, z2.b, z5.b\n" - "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z9.b }, p2/Z, [x28, #2, MUL VL]\n" "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" "subs x25, x25, #0x8\n" - "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n" - "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n" - ".inst 0x45c69814 // ummla z20.s, z0.b, z6.b\n" - ".inst 0x45c6985c // ummla z28.s, z2.b, z6.b\n" - "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n" - "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n" - ".inst 0x45c79811 // ummla z17.s, z0.b, z7.b\n" - ".inst 0x45c79859 // ummla z25.s, z2.b, z7.b\n" + "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n" + "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n" + ".inst 0x45c49814 // ummla z20.s, z0.b, z4.b\n" + ".inst 0x45c4985c // ummla z28.s, z2.b, z4.b\n" + "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n" + "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n" + ".inst 0x45c99811 // ummla z17.s, z0.b, z9.b\n" + ".inst 0x45c99859 // ummla z25.s, z2.b, z9.b\n" ".inst 0x45c89815 // ummla z21.s, z0.b, z8.b\n" ".inst 0x45c8985d // ummla z29.s, z2.b, z8.b\n" "addvl x28, x28, #8\n" - ".inst 0x45c99812 // ummla z18.s, z0.b, z9.b\n" - ".inst 0x45c9985a // ummla z26.s, z2.b, z9.b\n" - ".inst 0x45ca9816 // ummla z22.s, z0.b, z10.b\n" - ".inst 0x45ca985e // ummla z30.s, z2.b, z10.b\n" - ".inst 0x45c49813 // ummla z19.s, z0.b, z4.b\n" - ".inst 0x45c4985b // ummla z27.s, z2.b, z4.b\n" - ".inst 0x45c59817 // ummla z23.s, z0.b, z5.b\n" - ".inst 0x45c5985f // ummla z31.s, z2.b, z5.b\n" + ".inst 0x45c79812 // ummla z18.s, z0.b, z7.b\n" + ".inst 0x45c7985a // ummla z26.s, z2.b, z7.b\n" + ".inst 0x45c69816 // ummla z22.s, z0.b, z6.b\n" + ".inst 0x45c6985e // ummla z30.s, z2.b, z6.b\n" + ".inst 0x45c59813 // ummla z19.s, z0.b, z5.b\n" + ".inst 0x45c5985b // ummla z27.s, z2.b, z5.b\n" + ".inst 0x45c49817 // ummla z23.s, z0.b, z4.b\n" + ".inst 0x45c4985f // ummla z31.s, z2.b, z4.b\n" "ble 38f\n" - "ld1b { z6.b }, p2/Z, [x28]\n" - ".inst 0x45c69830 // ummla z16.s, z1.b, z6.b\n" - ".inst 0x45c69878 // ummla z24.s, z3.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n" - "ld1b { z8.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z9.b }, p2/Z, [x28, #3, MUL VL]\n" - ".inst 0x45c79834 // ummla z20.s, z1.b, z7.b\n" - ".inst 0x45c7987c // ummla z28.s, z3.b, z7.b\n" - "ld1b { z10.b }, p2/Z, [x28, #4, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #5, MUL VL]\n" - ".inst 0x45c89831 // ummla z17.s, z1.b, z8.b\n" - ".inst 0x45c89879 // ummla z25.s, z3.b, z8.b\n" + "ld1b { z4.b }, p2/Z, [x28]\n" + ".inst 0x45c49830 // ummla z16.s, z1.b, z4.b\n" + ".inst 0x45c49878 // ummla z24.s, z3.b, z4.b\n" + "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" + ".inst 0x45c59834 // ummla z20.s, z1.b, z5.b\n" + ".inst 0x45c5987c // ummla z28.s, z3.b, z5.b\n" + "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n" + "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n" + ".inst 0x45c49831 // ummla z17.s, z1.b, z4.b\n" + ".inst 0x45c49879 // ummla z25.s, z3.b, z4.b\n" "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n" - ".inst 0x45c99835 // ummla z21.s, z1.b, z9.b\n" - ".inst 0x45c9987d // ummla z29.s, z3.b, z9.b\n" - ".inst 0x45ca9832 // ummla z18.s, z1.b, z10.b\n" - ".inst 0x45ca987a // ummla z26.s, z3.b, z10.b\n" + "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n" + ".inst 0x45c89835 // ummla z21.s, z1.b, z8.b\n" + ".inst 0x45c8987d // ummla z29.s, z3.b, z8.b\n" + ".inst 0x45c79832 // ummla z18.s, z1.b, z7.b\n" + ".inst 0x45c7987a // ummla z26.s, z3.b, z7.b\n" "addvl x28, x28, #8\n" - ".inst 0x45c49836 // ummla z22.s, z1.b, z4.b\n" - ".inst 0x45c4987e // ummla z30.s, z3.b, z4.b\n" + ".inst 0x45c69836 // ummla z22.s, z1.b, z6.b\n" + ".inst 0x45c6987e // ummla z30.s, z3.b, z6.b\n" ".inst 0x45c59833 // ummla z19.s, z1.b, z5.b\n" ".inst 0x45c5987b // ummla z27.s, z3.b, z5.b\n" - ".inst 0x45c69837 // ummla z23.s, z1.b, z6.b\n" - ".inst 0x45c6987f // ummla z31.s, z3.b, z6.b\n" + ".inst 0x45c49837 // ummla z23.s, z1.b, z4.b\n" + ".inst 0x45c4987f // ummla z31.s, z3.b, z4.b\n" "38:" // Height 3: Multiply loop: multiply skip "tbnz %x[flags], #31, 39f\n" "udot z11.s, z0.b, z15.b\n" @@ -770,12 +770,12 @@ void sve_hybrid_u8qa_mmla_4x4VL ( "cmp x26, x20\n" "bne 32b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "uzp1 z7.d, z16.d, z20.d\n" - "add x22, x27, x20\n" + "uzp1 z0.d, z16.d, z20.d\n" + "add x23, x27, x20\n" "uzp2 z16.d, z16.d, z20.d\n" "uzp1 z20.d, z17.d, z21.d\n" "uzp2 z17.d, z17.d, z21.d\n" - "add x21, x22, x20\n" + "add x22, x23, x20\n" "uzp1 z21.d, z18.d, z22.d\n" "uzp2 z18.d, z18.d, z22.d\n" "uzp1 z22.d, z19.d, z23.d\n" @@ -784,170 +784,170 @@ void sve_hybrid_u8qa_mmla_4x4VL ( "uzp1 z25.d, z25.d, z29.d\n" "uzp1 z26.d, z26.d, z30.d\n" "uzp1 z27.d, z27.d, z31.d\n" - "mov z31.d, z7.d\n" + "mov z31.d, z0.d\n" "tbnz %x[flags], #31, 40f\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1rw { z3.s }, p2/Z, [x23]\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1rw { z23.s }, p2/Z, [x20]\n" ".inst 0x4491a96b // addp z11.s, p2/m, z11.s, z11.s\n" ".inst 0x4491a9ad // addp z13.s, p2/m, z13.s, z13.s\n" - "neg z3.s, p2/M, z3.s\n" + "neg z23.s, p2/M, z23.s\n" "mov z12.s, z11.s[3]\n" "mov z11.s, z11.s[0]\n" - "mul z11.s, p2/M, z11.s, z3.s\n" + "mul z11.s, p2/M, z11.s, z23.s\n" "mov z13.s, z13.s[0]\n" - "mul z12.s, p2/M, z12.s, z3.s\n" - "mul z13.s, p2/M, z13.s, z3.s\n" + "mul z12.s, p2/M, z12.s, z23.s\n" + "mul z13.s, p2/M, z13.s, z23.s\n" "40:" // Height 3: skip row sum fixup "add z31.s, z31.s, z11.s\n" "add z20.s, z20.s, z11.s\n" "ld1w { z0.s }, p2/Z, [x10]\n" - "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n" + "ld1w { z30.s }, p2/Z, [x10, #1, MUL VL]\n" "add z21.s, z21.s, z11.s\n" "add z22.s, z22.s, z11.s\n" - "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n" + "ld1w { z29.s }, p2/Z, [x10, #2, MUL VL]\n" + "ld1w { z28.s }, p2/Z, [x10, #3, MUL VL]\n" "add z16.s, z16.s, z12.s\n" "add z17.s, z17.s, z12.s\n" - "add x23, %x[qp], %[per_layer_mul]\n" + "add x20, %x[qp], %[per_layer_mul]\n" "orr %x[flags], %x[flags], #0x80000000\n" "add z18.s, z18.s, z12.s\n" "add z19.s, z19.s, z12.s\n" - "ld1rw { z4.s }, p2/Z, [x23]\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" + "ld1rw { z23.s }, p2/Z, [x20]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" "add z24.s, z24.s, z13.s\n" "add z25.s, z25.s, z13.s\n" "addvl x10, x10, #4\n" "add z26.s, z26.s, z13.s\n" "add z27.s, z27.s, z13.s\n" "add z31.s, z31.s, z0.s\n" - "add z20.s, z20.s, z1.s\n" - "add z21.s, z21.s, z2.s\n" - "add z22.s, z22.s, z3.s\n" + "add z20.s, z20.s, z30.s\n" + "add z21.s, z21.s, z29.s\n" + "add z22.s, z22.s, z28.s\n" "add z16.s, z16.s, z0.s\n" - "add z17.s, z17.s, z1.s\n" - "add z18.s, z18.s, z2.s\n" - "add z19.s, z19.s, z3.s\n" + "add z17.s, z17.s, z30.s\n" + "add z18.s, z18.s, z29.s\n" + "add z19.s, z19.s, z28.s\n" "add z24.s, z24.s, z0.s\n" - "add z25.s, z25.s, z1.s\n" - "ld1rw { z0.s }, p2/Z, [x23]\n" - "add z26.s, z26.s, z2.s\n" - "add z27.s, z27.s, z3.s\n" - ".inst 0x04a477ff // sqrdmulh z31.s, z31.s, z4.s\n" - ".inst 0x04a47694 // sqrdmulh z20.s, z20.s, z4.s\n" - ".inst 0x04a476b5 // sqrdmulh z21.s, z21.s, z4.s\n" - ".inst 0x04a476d6 // sqrdmulh z22.s, z22.s, z4.s\n" - ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n" - ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n" - ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n" - ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n" - ".inst 0x04a47718 // sqrdmulh z24.s, z24.s, z4.s\n" - ".inst 0x04a47739 // sqrdmulh z25.s, z25.s, z4.s\n" - ".inst 0x04a4775a // sqrdmulh z26.s, z26.s, z4.s\n" - ".inst 0x04a4777b // sqrdmulh z27.s, z27.s, z4.s\n" + "add z25.s, z25.s, z30.s\n" + "ld1rw { z0.s }, p2/Z, [x20]\n" + "add z26.s, z26.s, z29.s\n" + "add z27.s, z27.s, z28.s\n" + ".inst 0x04b777ff // sqrdmulh z31.s, z31.s, z23.s\n" + ".inst 0x04b77694 // sqrdmulh z20.s, z20.s, z23.s\n" + ".inst 0x04b776b5 // sqrdmulh z21.s, z21.s, z23.s\n" + ".inst 0x04b776d6 // sqrdmulh z22.s, z22.s, z23.s\n" + ".inst 0x04b77610 // sqrdmulh z16.s, z16.s, z23.s\n" + ".inst 0x04b77631 // sqrdmulh z17.s, z17.s, z23.s\n" + ".inst 0x04b77652 // sqrdmulh z18.s, z18.s, z23.s\n" + ".inst 0x04b77673 // sqrdmulh z19.s, z19.s, z23.s\n" + ".inst 0x04b77718 // sqrdmulh z24.s, z24.s, z23.s\n" + ".inst 0x04b77739 // sqrdmulh z25.s, z25.s, z23.s\n" + ".inst 0x04b7775a // sqrdmulh z26.s, z26.s, z23.s\n" + ".inst 0x04b7777b // sqrdmulh z27.s, z27.s, z23.s\n" "tbz %x[flags], #5, 41f\n" - "and z4.d, z31.d, z0.d\n" - "and z5.d, z20.d, z0.d\n" - "and z6.d, z21.d, z0.d\n" - "and z7.d, z22.d, z0.d\n" - "and z8.d, z16.d, z0.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "asr z8.s, z8.s, #0x1f\n" - "sqadd z31.s, z31.s, z4.s\n" - "sqadd z20.s, z20.s, z5.s\n" - "sqadd z21.s, z21.s, z6.s\n" - "sqadd z22.s, z22.s, z7.s\n" - "sqadd z16.s, z16.s, z8.s\n" - "and z9.d, z17.d, z0.d\n" - "and z10.d, z18.d, z0.d\n" - "and z4.d, z19.d, z0.d\n" - "and z5.d, z24.d, z0.d\n" - "and z6.d, z25.d, z0.d\n" - "and z7.d, z26.d, z0.d\n" - "and z8.d, z27.d, z0.d\n" - "asr z9.s, z9.s, #0x1f\n" - "asr z10.s, z10.s, #0x1f\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "asr z8.s, z8.s, #0x1f\n" - "sqadd z17.s, z17.s, z9.s\n" - "sqadd z18.s, z18.s, z10.s\n" - "sqadd z19.s, z19.s, z4.s\n" - "sqadd z24.s, z24.s, z5.s\n" - "sqadd z25.s, z25.s, z6.s\n" - "sqadd z26.s, z26.s, z7.s\n" - "sqadd z27.s, z27.s, z8.s\n" + "and z1.d, z31.d, z0.d\n" + "and z30.d, z20.d, z0.d\n" + "and z29.d, z21.d, z0.d\n" + "and z28.d, z22.d, z0.d\n" + "and z23.d, z16.d, z0.d\n" + "asr z1.s, z1.s, #0x1f\n" + "asr z30.s, z30.s, #0x1f\n" + "asr z29.s, z29.s, #0x1f\n" + "asr z28.s, z28.s, #0x1f\n" + "asr z23.s, z23.s, #0x1f\n" + "sqadd z31.s, z31.s, z1.s\n" + "sqadd z20.s, z20.s, z30.s\n" + "sqadd z21.s, z21.s, z29.s\n" + "sqadd z22.s, z22.s, z28.s\n" + "sqadd z16.s, z16.s, z23.s\n" + "and z3.d, z17.d, z0.d\n" + "and z2.d, z18.d, z0.d\n" + "and z1.d, z19.d, z0.d\n" + "and z30.d, z24.d, z0.d\n" + "and z29.d, z25.d, z0.d\n" + "and z28.d, z26.d, z0.d\n" + "and z23.d, z27.d, z0.d\n" + "asr z3.s, z3.s, #0x1f\n" + "asr z2.s, z2.s, #0x1f\n" + "asr z1.s, z1.s, #0x1f\n" + "asr z30.s, z30.s, #0x1f\n" + "asr z29.s, z29.s, #0x1f\n" + "asr z28.s, z28.s, #0x1f\n" + "asr z23.s, z23.s, #0x1f\n" + "sqadd z17.s, z17.s, z3.s\n" + "sqadd z18.s, z18.s, z2.s\n" + "sqadd z19.s, z19.s, z1.s\n" + "sqadd z24.s, z24.s, z30.s\n" + "sqadd z25.s, z25.s, z29.s\n" + "sqadd z26.s, z26.s, z28.s\n" + "sqadd z27.s, z27.s, z23.s\n" "41:" // Height 3: no shift correction - "add x23, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x23]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1rw { z23.s }, p2/Z, [x20]\n" ".inst 0x4482881f // srshl z31.s, p2/M, z31.s, z0.s\n" - "add z31.s, z31.s, z4.s\n" + "add z31.s, z31.s, z23.s\n" ".inst 0x44828814 // srshl z20.s, p2/M, z20.s, z0.s\n" ".inst 0x44828815 // srshl z21.s, p2/M, z21.s, z0.s\n" - "add z20.s, z20.s, z4.s\n" - "add z21.s, z21.s, z4.s\n" + "add z20.s, z20.s, z23.s\n" + "add z21.s, z21.s, z23.s\n" ".inst 0x44828816 // srshl z22.s, p2/M, z22.s, z0.s\n" ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n" - "add z22.s, z22.s, z4.s\n" - "add z16.s, z16.s, z4.s\n" + "add z22.s, z22.s, z23.s\n" + "add z16.s, z16.s, z23.s\n" ".inst 0x44828811 // srshl z17.s, p2/M, z17.s, z0.s\n" ".inst 0x44828812 // srshl z18.s, p2/M, z18.s, z0.s\n" - "add z17.s, z17.s, z4.s\n" - "add z18.s, z18.s, z4.s\n" + "add z17.s, z17.s, z23.s\n" + "add z18.s, z18.s, z23.s\n" ".inst 0x44828813 // srshl z19.s, p2/M, z19.s, z0.s\n" ".inst 0x44828818 // srshl z24.s, p2/M, z24.s, z0.s\n" - "add z19.s, z19.s, z4.s\n" - "add z24.s, z24.s, z4.s\n" + "add z19.s, z19.s, z23.s\n" + "add z24.s, z24.s, z23.s\n" ".inst 0x44828819 // srshl z25.s, p2/M, z25.s, z0.s\n" ".inst 0x4482881a // srshl z26.s, p2/M, z26.s, z0.s\n" - "add z25.s, z25.s, z4.s\n" - "add z26.s, z26.s, z4.s\n" + "add z25.s, z25.s, z23.s\n" + "add z26.s, z26.s, z23.s\n" ".inst 0x4482881b // srshl z27.s, p2/M, z27.s, z0.s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x23]\n" - "add z27.s, z27.s, z4.s\n" - "add x23, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x23]\n" - "smin z31.s, p2/M, z31.s, z6.s\n" - "smin z20.s, p2/M, z20.s, z6.s\n" - "smin z21.s, p2/M, z21.s, z6.s\n" - "smin z22.s, p2/M, z22.s, z6.s\n" - "smin z16.s, p2/M, z16.s, z6.s\n" - "smin z17.s, p2/M, z17.s, z6.s\n" - "smin z18.s, p2/M, z18.s, z6.s\n" - "smin z19.s, p2/M, z19.s, z6.s\n" - "smin z24.s, p2/M, z24.s, z6.s\n" - "smin z25.s, p2/M, z25.s, z6.s\n" - "smin z26.s, p2/M, z26.s, z6.s\n" - "smin z27.s, p2/M, z27.s, z6.s\n" - "smax z31.s, p2/M, z31.s, z5.s\n" - "smax z20.s, p2/M, z20.s, z5.s\n" - "smax z21.s, p2/M, z21.s, z5.s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1rw { z28.s }, p2/Z, [x20]\n" + "add z27.s, z27.s, z23.s\n" + "add x20, %x[qp], %[minval]\n" + "ld1rw { z23.s }, p2/Z, [x20]\n" + "smin z31.s, p2/M, z31.s, z28.s\n" + "smin z20.s, p2/M, z20.s, z28.s\n" + "smin z21.s, p2/M, z21.s, z28.s\n" + "smin z22.s, p2/M, z22.s, z28.s\n" + "smin z16.s, p2/M, z16.s, z28.s\n" + "smin z17.s, p2/M, z17.s, z28.s\n" + "smin z18.s, p2/M, z18.s, z28.s\n" + "smin z19.s, p2/M, z19.s, z28.s\n" + "smin z24.s, p2/M, z24.s, z28.s\n" + "smin z25.s, p2/M, z25.s, z28.s\n" + "smin z26.s, p2/M, z26.s, z28.s\n" + "smin z27.s, p2/M, z27.s, z28.s\n" + "smax z31.s, p2/M, z31.s, z23.s\n" + "smax z20.s, p2/M, z20.s, z23.s\n" + "smax z21.s, p2/M, z21.s, z23.s\n" "uzp1 z31.h, z31.h, z20.h\n" - "smax z22.s, p2/M, z22.s, z5.s\n" - "smax z16.s, p2/M, z16.s, z5.s\n" + "smax z22.s, p2/M, z22.s, z23.s\n" + "smax z16.s, p2/M, z16.s, z23.s\n" "uzp1 z20.h, z21.h, z22.h\n" "uzp1 z31.b, z31.b, z20.b\n" - "smax z17.s, p2/M, z17.s, z5.s\n" - "smax z18.s, p2/M, z18.s, z5.s\n" + "smax z17.s, p2/M, z17.s, z23.s\n" + "smax z18.s, p2/M, z18.s, z23.s\n" "uzp1 z16.h, z16.h, z17.h\n" "st1b { z31.b }, p1, [x27]\n" - "smax z19.s, p2/M, z19.s, z5.s\n" - "smax z24.s, p2/M, z24.s, z5.s\n" + "smax z19.s, p2/M, z19.s, z23.s\n" + "smax z24.s, p2/M, z24.s, z23.s\n" "uzp1 z17.h, z18.h, z19.h\n" "uzp1 z16.b, z16.b, z17.b\n" - "smax z25.s, p2/M, z25.s, z5.s\n" - "smax z26.s, p2/M, z26.s, z5.s\n" + "smax z25.s, p2/M, z25.s, z23.s\n" + "smax z26.s, p2/M, z26.s, z23.s\n" "uzp1 z24.h, z24.h, z25.h\n" - "st1b { z16.b }, p1, [x22]\n" - "smax z27.s, p2/M, z27.s, z5.s\n" - "uzp1 z25.h, z26.h, z27.h\n" - "uzp1 z24.b, z24.b, z25.b\n" - "st1b { z24.b }, p1, [x21]\n" + "st1b { z16.b }, p1, [x23]\n" + "smax z27.s, p2/M, z27.s, z23.s\n" + "uzp1 z16.h, z26.h, z27.h\n" + "uzp1 z24.b, z24.b, z16.b\n" + "st1b { z24.b }, p1, [x22]\n" "addvl x27, x27, #1\n" "42:" // Height 3: Writeback done "decw x9, ALL, MUL #4\n" @@ -992,14 +992,14 @@ void sve_hybrid_u8qa_mmla_4x4VL ( "46:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w25, [x20, x26, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 47f\n" - "ldr x21, [%x[input_ptr], x26, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x24, [x21, #0x0]\n" - "ldr x23, [x21, #0x8]\n" - "ldr x22, [x21, #0x10]\n" - "ldr x21, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x26, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x24, [x20, #0x0]\n" + "ldr x23, [x20, #0x8]\n" + "ldr x22, [x20, #0x10]\n" + "ldr x21, [x20, #0x18]\n" "cbnz x26, 48f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x24, x24, x20\n" @@ -1009,9 +1009,9 @@ void sve_hybrid_u8qa_mmla_4x4VL ( "b 48f\n" "47:" // Height 4: setup direct input "mov x24, %x[input_ptr]\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" + "add x21, x22, x21\n" "48:" // Height 4: input setup done "cmp x25, #0x10\n" "ble 51f\n" @@ -1021,63 +1021,63 @@ void sve_hybrid_u8qa_mmla_4x4VL ( "ld1rqb { z2.b }, p0/Z, [x23]\n" "trn1 z0.d, z1.d, z2.d\n" "ld1rqb { z3.b }, p0/Z, [x22]\n" - "ld1rqb { z4.b }, p0/Z, [x21]\n" + "ld1rqb { z5.b }, p0/Z, [x21]\n" "trn2 z1.d, z1.d, z2.d\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1b { z5.b }, p2/Z, [x28]\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x45c59810 // ummla z16.s, z0.b, z5.b\n" - ".inst 0x45c59858 // ummla z24.s, z2.b, z5.b\n" - "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n" - ".inst 0x45c69814 // ummla z20.s, z0.b, z6.b\n" - ".inst 0x45c6985c // ummla z28.s, z2.b, z6.b\n" - "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" - "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n" - ".inst 0x45c79811 // ummla z17.s, z0.b, z7.b\n" - ".inst 0x45c79859 // ummla z25.s, z2.b, z7.b\n" - "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n" - ".inst 0x45c89815 // ummla z21.s, z0.b, z8.b\n" - ".inst 0x45c8985d // ummla z29.s, z2.b, z8.b\n" + "trn1 z2.d, z3.d, z5.d\n" + "ld1b { z4.b }, p2/Z, [x28]\n" + "trn2 z3.d, z3.d, z5.d\n" + ".inst 0x45c49810 // ummla z16.s, z0.b, z4.b\n" + ".inst 0x45c49858 // ummla z24.s, z2.b, z4.b\n" + "ld1b { z4.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z5.b }, p2/Z, [x28, #2, MUL VL]\n" + ".inst 0x45c49814 // ummla z20.s, z0.b, z4.b\n" + ".inst 0x45c4985c // ummla z28.s, z2.b, z4.b\n" + "ld1b { z4.b }, p2/Z, [x28, #3, MUL VL]\n" + "ld1b { z8.b }, p2/Z, [x28, #4, MUL VL]\n" + ".inst 0x45c59811 // ummla z17.s, z0.b, z5.b\n" + ".inst 0x45c59859 // ummla z25.s, z2.b, z5.b\n" + "ld1b { z7.b }, p2/Z, [x28, #5, MUL VL]\n" + "ld1b { z6.b }, p2/Z, [x28, #6, MUL VL]\n" + ".inst 0x45c49815 // ummla z21.s, z0.b, z4.b\n" + ".inst 0x45c4985d // ummla z29.s, z2.b, z4.b\n" "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n" "addvl x28, x28, #16\n" - "ld1b { z6.b }, p2/Z, [x28, #-8, MUL VL]\n" - ".inst 0x45c99812 // ummla z18.s, z0.b, z9.b\n" - ".inst 0x45c9985a // ummla z26.s, z2.b, z9.b\n" - ".inst 0x45ca9816 // ummla z22.s, z0.b, z10.b\n" - "ld1b { z7.b }, p2/Z, [x28, #-7, MUL VL]\n" - "ld1b { z8.b }, p2/Z, [x28, #-6, MUL VL]\n" - ".inst 0x45ca985e // ummla z30.s, z2.b, z10.b\n" - ".inst 0x45c49813 // ummla z19.s, z0.b, z4.b\n" - "ld1b { z9.b }, p2/Z, [x28, #-5, MUL VL]\n" - "ld1b { z10.b }, p2/Z, [x28, #-4, MUL VL]\n" - ".inst 0x45c4985b // ummla z27.s, z2.b, z4.b\n" + "ld1b { z4.b }, p2/Z, [x28, #-8, MUL VL]\n" + ".inst 0x45c89812 // ummla z18.s, z0.b, z8.b\n" + ".inst 0x45c8985a // ummla z26.s, z2.b, z8.b\n" + ".inst 0x45c79816 // ummla z22.s, z0.b, z7.b\n" + "ld1b { z10.b }, p2/Z, [x28, #-7, MUL VL]\n" + "ld1b { z9.b }, p2/Z, [x28, #-6, MUL VL]\n" + ".inst 0x45c7985e // ummla z30.s, z2.b, z7.b\n" + ".inst 0x45c69813 // ummla z19.s, z0.b, z6.b\n" + "ld1b { z8.b }, p2/Z, [x28, #-5, MUL VL]\n" + "ld1b { z7.b }, p2/Z, [x28, #-4, MUL VL]\n" + ".inst 0x45c6985b // ummla z27.s, z2.b, z6.b\n" ".inst 0x45c59817 // ummla z23.s, z0.b, z5.b\n" - "ld1b { z4.b }, p2/Z, [x28, #-3, MUL VL]\n" + "ld1b { z6.b }, p2/Z, [x28, #-3, MUL VL]\n" "add x24, x24, #0x10\n" ".inst 0x45c5985f // ummla z31.s, z2.b, z5.b\n" - ".inst 0x45c69830 // ummla z16.s, z1.b, z6.b\n" + ".inst 0x45c49830 // ummla z16.s, z1.b, z4.b\n" "ld1b { z5.b }, p2/Z, [x28, #-2, MUL VL]\n" "add x23, x23, #0x10\n" - ".inst 0x45c69878 // ummla z24.s, z3.b, z6.b\n" - "ld1b { z6.b }, p2/Z, [x28, #-1, MUL VL]\n" - ".inst 0x45c79834 // ummla z20.s, z1.b, z7.b\n" + ".inst 0x45c49878 // ummla z24.s, z3.b, z4.b\n" + "ld1b { z4.b }, p2/Z, [x28, #-1, MUL VL]\n" + ".inst 0x45ca9834 // ummla z20.s, z1.b, z10.b\n" "add x22, x22, #0x10\n" - ".inst 0x45c7987c // ummla z28.s, z3.b, z7.b\n" - ".inst 0x45c89831 // ummla z17.s, z1.b, z8.b\n" + ".inst 0x45ca987c // ummla z28.s, z3.b, z10.b\n" + ".inst 0x45c99831 // ummla z17.s, z1.b, z9.b\n" "add x21, x21, #0x10\n" - ".inst 0x45c89879 // ummla z25.s, z3.b, z8.b\n" - ".inst 0x45c99835 // ummla z21.s, z1.b, z9.b\n" - ".inst 0x45c9987d // ummla z29.s, z3.b, z9.b\n" - ".inst 0x45ca9832 // ummla z18.s, z1.b, z10.b\n" - ".inst 0x45ca987a // ummla z26.s, z3.b, z10.b\n" - ".inst 0x45c49836 // ummla z22.s, z1.b, z4.b\n" - ".inst 0x45c4987e // ummla z30.s, z3.b, z4.b\n" + ".inst 0x45c99879 // ummla z25.s, z3.b, z9.b\n" + ".inst 0x45c89835 // ummla z21.s, z1.b, z8.b\n" + ".inst 0x45c8987d // ummla z29.s, z3.b, z8.b\n" + ".inst 0x45c79832 // ummla z18.s, z1.b, z7.b\n" + ".inst 0x45c7987a // ummla z26.s, z3.b, z7.b\n" + ".inst 0x45c69836 // ummla z22.s, z1.b, z6.b\n" + ".inst 0x45c6987e // ummla z30.s, z3.b, z6.b\n" ".inst 0x45c59833 // ummla z19.s, z1.b, z5.b\n" ".inst 0x45c5987b // ummla z27.s, z3.b, z5.b\n" - ".inst 0x45c69837 // ummla z23.s, z1.b, z6.b\n" - ".inst 0x45c6987f // ummla z31.s, z3.b, z6.b\n" + ".inst 0x45c49837 // ummla z23.s, z1.b, z4.b\n" + ".inst 0x45c4987f // ummla z31.s, z3.b, z4.b\n" "tbnz %x[flags], #31, 50f\n" "udot z11.s, z0.b, z15.b\n" "udot z13.s, z2.b, z15.b\n" @@ -1093,62 +1093,62 @@ void sve_hybrid_u8qa_mmla_4x4VL ( "ld1rqb { z2.b }, p0/Z, [x23]\n" "trn1 z0.d, z1.d, z2.d\n" "ld1rqb { z3.b }, p0/Z, [x22]\n" - "ld1rqb { z4.b }, p0/Z, [x21]\n" + "ld1rqb { z5.b }, p0/Z, [x21]\n" "trn2 z1.d, z1.d, z2.d\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1b { z5.b }, p2/Z, [x28]\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x45c59810 // ummla z16.s, z0.b, z5.b\n" - ".inst 0x45c59858 // ummla z24.s, z2.b, z5.b\n" - "ld1b { z6.b }, p2/Z, [x28, #1, MUL VL]\n" - "ld1b { z7.b }, p2/Z, [x28, #2, MUL VL]\n" + "trn1 z2.d, z3.d, z5.d\n" + "ld1b { z4.b }, p2/Z, [x28]\n" + "trn2 z3.d, z3.d, z5.d\n" + ".inst 0x45c49810 // ummla z16.s, z0.b, z4.b\n" + ".inst 0x45c49858 // ummla z24.s, z2.b, z4.b\n" + "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n" "subs x25, x25, #0x8\n" - ".inst 0x45c69814 // ummla z20.s, z0.b, z6.b\n" + ".inst 0x45c59814 // ummla z20.s, z0.b, z5.b\n" "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" - "ld1b { z9.b }, p2/Z, [x28, #4, MUL VL]\n" - ".inst 0x45c6985c // ummla z28.s, z2.b, z6.b\n" - ".inst 0x45c79811 // ummla z17.s, z0.b, z7.b\n" - "ld1b { z10.b }, p2/Z, [x28, #5, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #6, MUL VL]\n" - ".inst 0x45c79859 // ummla z25.s, z2.b, z7.b\n" + "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n" + ".inst 0x45c5985c // ummla z28.s, z2.b, z5.b\n" + ".inst 0x45c49811 // ummla z17.s, z0.b, z4.b\n" + "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n" + "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n" + ".inst 0x45c49859 // ummla z25.s, z2.b, z4.b\n" ".inst 0x45c89815 // ummla z21.s, z0.b, z8.b\n" - "ld1b { z5.b }, p2/Z, [x28, #7, MUL VL]\n" + "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n" ".inst 0x45c8985d // ummla z29.s, z2.b, z8.b\n" - ".inst 0x45c99812 // ummla z18.s, z0.b, z9.b\n" + ".inst 0x45c79812 // ummla z18.s, z0.b, z7.b\n" "addvl x28, x28, #8\n" - ".inst 0x45c9985a // ummla z26.s, z2.b, z9.b\n" - ".inst 0x45ca9816 // ummla z22.s, z0.b, z10.b\n" - ".inst 0x45ca985e // ummla z30.s, z2.b, z10.b\n" - ".inst 0x45c49813 // ummla z19.s, z0.b, z4.b\n" - ".inst 0x45c4985b // ummla z27.s, z2.b, z4.b\n" - ".inst 0x45c59817 // ummla z23.s, z0.b, z5.b\n" - ".inst 0x45c5985f // ummla z31.s, z2.b, z5.b\n" + ".inst 0x45c7985a // ummla z26.s, z2.b, z7.b\n" + ".inst 0x45c69816 // ummla z22.s, z0.b, z6.b\n" + ".inst 0x45c6985e // ummla z30.s, z2.b, z6.b\n" + ".inst 0x45c59813 // ummla z19.s, z0.b, z5.b\n" + ".inst 0x45c5985b // ummla z27.s, z2.b, z5.b\n" + ".inst 0x45c49817 // ummla z23.s, z0.b, z4.b\n" + ".inst 0x45c4985f // ummla z31.s, z2.b, z4.b\n" "ble 52f\n" - "ld1b { z6.b }, p2/Z, [x28]\n" - ".inst 0x45c69830 // ummla z16.s, z1.b, z6.b\n" - ".inst 0x45c69878 // ummla z24.s, z3.b, z6.b\n" - "ld1b { z7.b }, p2/Z, [x28, #1, MUL VL]\n" - "ld1b { z8.b }, p2/Z, [x28, #2, MUL VL]\n" - "ld1b { z9.b }, p2/Z, [x28, #3, MUL VL]\n" - ".inst 0x45c79834 // ummla z20.s, z1.b, z7.b\n" - ".inst 0x45c7987c // ummla z28.s, z3.b, z7.b\n" - "ld1b { z10.b }, p2/Z, [x28, #4, MUL VL]\n" - "ld1b { z4.b }, p2/Z, [x28, #5, MUL VL]\n" - ".inst 0x45c89831 // ummla z17.s, z1.b, z8.b\n" - ".inst 0x45c89879 // ummla z25.s, z3.b, z8.b\n" + "ld1b { z4.b }, p2/Z, [x28]\n" + ".inst 0x45c49830 // ummla z16.s, z1.b, z4.b\n" + ".inst 0x45c49878 // ummla z24.s, z3.b, z4.b\n" + "ld1b { z5.b }, p2/Z, [x28, #1, MUL VL]\n" + "ld1b { z4.b }, p2/Z, [x28, #2, MUL VL]\n" + "ld1b { z8.b }, p2/Z, [x28, #3, MUL VL]\n" + ".inst 0x45c59834 // ummla z20.s, z1.b, z5.b\n" + ".inst 0x45c5987c // ummla z28.s, z3.b, z5.b\n" + "ld1b { z7.b }, p2/Z, [x28, #4, MUL VL]\n" + "ld1b { z6.b }, p2/Z, [x28, #5, MUL VL]\n" + ".inst 0x45c49831 // ummla z17.s, z1.b, z4.b\n" + ".inst 0x45c49879 // ummla z25.s, z3.b, z4.b\n" "ld1b { z5.b }, p2/Z, [x28, #6, MUL VL]\n" - "ld1b { z6.b }, p2/Z, [x28, #7, MUL VL]\n" - ".inst 0x45c99835 // ummla z21.s, z1.b, z9.b\n" - ".inst 0x45c9987d // ummla z29.s, z3.b, z9.b\n" - ".inst 0x45ca9832 // ummla z18.s, z1.b, z10.b\n" - ".inst 0x45ca987a // ummla z26.s, z3.b, z10.b\n" + "ld1b { z4.b }, p2/Z, [x28, #7, MUL VL]\n" + ".inst 0x45c89835 // ummla z21.s, z1.b, z8.b\n" + ".inst 0x45c8987d // ummla z29.s, z3.b, z8.b\n" + ".inst 0x45c79832 // ummla z18.s, z1.b, z7.b\n" + ".inst 0x45c7987a // ummla z26.s, z3.b, z7.b\n" "addvl x28, x28, #8\n" - ".inst 0x45c49836 // ummla z22.s, z1.b, z4.b\n" - ".inst 0x45c4987e // ummla z30.s, z3.b, z4.b\n" + ".inst 0x45c69836 // ummla z22.s, z1.b, z6.b\n" + ".inst 0x45c6987e // ummla z30.s, z3.b, z6.b\n" ".inst 0x45c59833 // ummla z19.s, z1.b, z5.b\n" ".inst 0x45c5987b // ummla z27.s, z3.b, z5.b\n" - ".inst 0x45c69837 // ummla z23.s, z1.b, z6.b\n" - ".inst 0x45c6987f // ummla z31.s, z3.b, z6.b\n" + ".inst 0x45c49837 // ummla z23.s, z1.b, z4.b\n" + ".inst 0x45c4987f // ummla z31.s, z3.b, z4.b\n" "52:" // Height 4: Multiply loop: multiply skip "tbnz %x[flags], #31, 53f\n" "udot z11.s, z0.b, z15.b\n" @@ -1161,12 +1161,12 @@ void sve_hybrid_u8qa_mmla_4x4VL ( "cmp x26, x20\n" "bne 46b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "uzp1 z7.d, z16.d, z20.d\n" - "add x22, x27, x20\n" - "add x21, x22, x20\n" + "uzp1 z0.d, z16.d, z20.d\n" + "add x23, x27, x20\n" + "add x22, x23, x20\n" "uzp2 z16.d, z16.d, z20.d\n" "uzp1 z20.d, z17.d, z21.d\n" - "add x20, x21, x20\n" + "add x21, x22, x20\n" "uzp2 z17.d, z17.d, z21.d\n" "uzp1 z21.d, z18.d, z22.d\n" "uzp2 z18.d, z18.d, z22.d\n" @@ -1180,38 +1180,38 @@ void sve_hybrid_u8qa_mmla_4x4VL ( "uzp2 z26.d, z26.d, z30.d\n" "uzp1 z30.d, z27.d, z31.d\n" "uzp2 z27.d, z27.d, z31.d\n" - "mov z31.d, z7.d\n" + "mov z31.d, z0.d\n" "tbnz %x[flags], #31, 54f\n" - "add x23, %x[qp], %[b_offset]\n" - "ld1rw { z4.s }, p2/Z, [x23]\n" + "add x20, %x[qp], %[b_offset]\n" + "ld1rw { z0.s }, p2/Z, [x20]\n" ".inst 0x4491a96b // addp z11.s, p2/m, z11.s, z11.s\n" ".inst 0x4491a9ad // addp z13.s, p2/m, z13.s, z13.s\n" - "neg z4.s, p2/M, z4.s\n" + "neg z0.s, p2/M, z0.s\n" "mov z12.s, z11.s[3]\n" "mov z11.s, z11.s[0]\n" - "mul z11.s, p2/M, z11.s, z4.s\n" + "mul z11.s, p2/M, z11.s, z0.s\n" "mov z14.s, z13.s[3]\n" "mov z13.s, z13.s[0]\n" - "mul z12.s, p2/M, z12.s, z4.s\n" - "mul z13.s, p2/M, z13.s, z4.s\n" - "mul z14.s, p2/M, z14.s, z4.s\n" + "mul z12.s, p2/M, z12.s, z0.s\n" + "mul z13.s, p2/M, z13.s, z0.s\n" + "mul z14.s, p2/M, z14.s, z0.s\n" "54:" // Height 4: skip row sum fixup "add z31.s, z31.s, z11.s\n" "add z20.s, z20.s, z11.s\n" - "ld1w { z0.s }, p2/Z, [x10]\n" - "ld1w { z1.s }, p2/Z, [x10, #1, MUL VL]\n" + "ld1w { z4.s }, p2/Z, [x10]\n" + "ld1w { z0.s }, p2/Z, [x10, #1, MUL VL]\n" "add z21.s, z21.s, z11.s\n" "add z22.s, z22.s, z11.s\n" - "ld1w { z2.s }, p2/Z, [x10, #2, MUL VL]\n" - "ld1w { z3.s }, p2/Z, [x10, #3, MUL VL]\n" + "ld1w { z3.s }, p2/Z, [x10, #2, MUL VL]\n" + "ld1w { z2.s }, p2/Z, [x10, #3, MUL VL]\n" "add z16.s, z16.s, z12.s\n" "add z17.s, z17.s, z12.s\n" - "add x23, %x[qp], %[per_layer_mul]\n" + "add x20, %x[qp], %[per_layer_mul]\n" "orr %x[flags], %x[flags], #0x80000000\n" "add z18.s, z18.s, z12.s\n" "add z19.s, z19.s, z12.s\n" - "ld1rw { z4.s }, p2/Z, [x23]\n" - "add x23, %x[qp], %[per_layer_right_shift]\n" + "ld1rw { z1.s }, p2/Z, [x20]\n" + "add x20, %x[qp], %[per_layer_right_shift]\n" "add z23.s, z23.s, z13.s\n" "add z28.s, z28.s, z13.s\n" "addvl x10, x10, #4\n" @@ -1221,175 +1221,175 @@ void sve_hybrid_u8qa_mmla_4x4VL ( "add z25.s, z25.s, z14.s\n" "add z26.s, z26.s, z14.s\n" "add z27.s, z27.s, z14.s\n" - "add z31.s, z31.s, z0.s\n" - "add z20.s, z20.s, z1.s\n" - "add z21.s, z21.s, z2.s\n" - "add z22.s, z22.s, z3.s\n" - "add z16.s, z16.s, z0.s\n" - "add z17.s, z17.s, z1.s\n" - "add z18.s, z18.s, z2.s\n" - "add z19.s, z19.s, z3.s\n" - "add z23.s, z23.s, z0.s\n" - "add z28.s, z28.s, z1.s\n" - "add z29.s, z29.s, z2.s\n" - "add z30.s, z30.s, z3.s\n" - "add z24.s, z24.s, z0.s\n" - "add z25.s, z25.s, z1.s\n" - "ld1rw { z0.s }, p2/Z, [x23]\n" - "add z26.s, z26.s, z2.s\n" - "add z27.s, z27.s, z3.s\n" - ".inst 0x04a477ff // sqrdmulh z31.s, z31.s, z4.s\n" - ".inst 0x04a47694 // sqrdmulh z20.s, z20.s, z4.s\n" - ".inst 0x04a476b5 // sqrdmulh z21.s, z21.s, z4.s\n" - ".inst 0x04a476d6 // sqrdmulh z22.s, z22.s, z4.s\n" - ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n" - ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n" - ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n" - ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n" - ".inst 0x04a476f7 // sqrdmulh z23.s, z23.s, z4.s\n" - ".inst 0x04a4779c // sqrdmulh z28.s, z28.s, z4.s\n" - ".inst 0x04a477bd // sqrdmulh z29.s, z29.s, z4.s\n" - ".inst 0x04a477de // sqrdmulh z30.s, z30.s, z4.s\n" - ".inst 0x04a47718 // sqrdmulh z24.s, z24.s, z4.s\n" - ".inst 0x04a47739 // sqrdmulh z25.s, z25.s, z4.s\n" - ".inst 0x04a4775a // sqrdmulh z26.s, z26.s, z4.s\n" - ".inst 0x04a4777b // sqrdmulh z27.s, z27.s, z4.s\n" + "add z31.s, z31.s, z4.s\n" + "add z20.s, z20.s, z0.s\n" + "add z21.s, z21.s, z3.s\n" + "add z22.s, z22.s, z2.s\n" + "add z16.s, z16.s, z4.s\n" + "add z17.s, z17.s, z0.s\n" + "add z18.s, z18.s, z3.s\n" + "add z19.s, z19.s, z2.s\n" + "add z23.s, z23.s, z4.s\n" + "add z28.s, z28.s, z0.s\n" + "add z29.s, z29.s, z3.s\n" + "add z30.s, z30.s, z2.s\n" + "add z24.s, z24.s, z4.s\n" + "add z25.s, z25.s, z0.s\n" + "ld1rw { z0.s }, p2/Z, [x20]\n" + "add z26.s, z26.s, z3.s\n" + "add z27.s, z27.s, z2.s\n" + ".inst 0x04a177ff // sqrdmulh z31.s, z31.s, z1.s\n" + ".inst 0x04a17694 // sqrdmulh z20.s, z20.s, z1.s\n" + ".inst 0x04a176b5 // sqrdmulh z21.s, z21.s, z1.s\n" + ".inst 0x04a176d6 // sqrdmulh z22.s, z22.s, z1.s\n" + ".inst 0x04a17610 // sqrdmulh z16.s, z16.s, z1.s\n" + ".inst 0x04a17631 // sqrdmulh z17.s, z17.s, z1.s\n" + ".inst 0x04a17652 // sqrdmulh z18.s, z18.s, z1.s\n" + ".inst 0x04a17673 // sqrdmulh z19.s, z19.s, z1.s\n" + ".inst 0x04a176f7 // sqrdmulh z23.s, z23.s, z1.s\n" + ".inst 0x04a1779c // sqrdmulh z28.s, z28.s, z1.s\n" + ".inst 0x04a177bd // sqrdmulh z29.s, z29.s, z1.s\n" + ".inst 0x04a177de // sqrdmulh z30.s, z30.s, z1.s\n" + ".inst 0x04a17718 // sqrdmulh z24.s, z24.s, z1.s\n" + ".inst 0x04a17739 // sqrdmulh z25.s, z25.s, z1.s\n" + ".inst 0x04a1775a // sqrdmulh z26.s, z26.s, z1.s\n" + ".inst 0x04a1777b // sqrdmulh z27.s, z27.s, z1.s\n" "tbz %x[flags], #5, 55f\n" - "and z4.d, z31.d, z0.d\n" - "and z5.d, z20.d, z0.d\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "sqadd z31.s, z31.s, z4.s\n" - "sqadd z20.s, z20.s, z5.s\n" - "and z6.d, z21.d, z0.d\n" - "and z7.d, z22.d, z0.d\n" - "and z8.d, z16.d, z0.d\n" - "and z9.d, z17.d, z0.d\n" - "and z10.d, z18.d, z0.d\n" - "and z4.d, z19.d, z0.d\n" - "and z5.d, z23.d, z0.d\n" - "asr z6.s, z6.s, #0x1f\n" + "and z2.d, z31.d, z0.d\n" + "and z1.d, z20.d, z0.d\n" + "asr z2.s, z2.s, #0x1f\n" + "asr z1.s, z1.s, #0x1f\n" + "sqadd z31.s, z31.s, z2.s\n" + "sqadd z20.s, z20.s, z1.s\n" + "and z7.d, z21.d, z0.d\n" + "and z6.d, z22.d, z0.d\n" + "and z5.d, z16.d, z0.d\n" + "and z4.d, z17.d, z0.d\n" + "and z3.d, z18.d, z0.d\n" + "and z2.d, z19.d, z0.d\n" + "and z1.d, z23.d, z0.d\n" "asr z7.s, z7.s, #0x1f\n" - "asr z8.s, z8.s, #0x1f\n" - "asr z9.s, z9.s, #0x1f\n" - "asr z10.s, z10.s, #0x1f\n" - "asr z4.s, z4.s, #0x1f\n" - "asr z5.s, z5.s, #0x1f\n" - "sqadd z21.s, z21.s, z6.s\n" - "sqadd z22.s, z22.s, z7.s\n" - "sqadd z16.s, z16.s, z8.s\n" - "sqadd z17.s, z17.s, z9.s\n" - "sqadd z18.s, z18.s, z10.s\n" - "sqadd z19.s, z19.s, z4.s\n" - "sqadd z23.s, z23.s, z5.s\n" - "and z6.d, z28.d, z0.d\n" - "and z7.d, z29.d, z0.d\n" - "and z8.d, z30.d, z0.d\n" - "and z9.d, z24.d, z0.d\n" - "and z10.d, z25.d, z0.d\n" - "and z4.d, z26.d, z0.d\n" - "and z5.d, z27.d, z0.d\n" "asr z6.s, z6.s, #0x1f\n" - "asr z7.s, z7.s, #0x1f\n" - "asr z8.s, z8.s, #0x1f\n" - "asr z9.s, z9.s, #0x1f\n" - "asr z10.s, z10.s, #0x1f\n" + "asr z5.s, z5.s, #0x1f\n" "asr z4.s, z4.s, #0x1f\n" + "asr z3.s, z3.s, #0x1f\n" + "asr z2.s, z2.s, #0x1f\n" + "asr z1.s, z1.s, #0x1f\n" + "sqadd z21.s, z21.s, z7.s\n" + "sqadd z22.s, z22.s, z6.s\n" + "sqadd z16.s, z16.s, z5.s\n" + "sqadd z17.s, z17.s, z4.s\n" + "sqadd z18.s, z18.s, z3.s\n" + "sqadd z19.s, z19.s, z2.s\n" + "sqadd z23.s, z23.s, z1.s\n" + "and z7.d, z28.d, z0.d\n" + "and z6.d, z29.d, z0.d\n" + "and z5.d, z30.d, z0.d\n" + "and z4.d, z24.d, z0.d\n" + "and z3.d, z25.d, z0.d\n" + "and z2.d, z26.d, z0.d\n" + "and z1.d, z27.d, z0.d\n" + "asr z7.s, z7.s, #0x1f\n" + "asr z6.s, z6.s, #0x1f\n" "asr z5.s, z5.s, #0x1f\n" - "sqadd z28.s, z28.s, z6.s\n" - "sqadd z29.s, z29.s, z7.s\n" - "sqadd z30.s, z30.s, z8.s\n" - "sqadd z24.s, z24.s, z9.s\n" - "sqadd z25.s, z25.s, z10.s\n" - "sqadd z26.s, z26.s, z4.s\n" - "sqadd z27.s, z27.s, z5.s\n" + "asr z4.s, z4.s, #0x1f\n" + "asr z3.s, z3.s, #0x1f\n" + "asr z2.s, z2.s, #0x1f\n" + "asr z1.s, z1.s, #0x1f\n" + "sqadd z28.s, z28.s, z7.s\n" + "sqadd z29.s, z29.s, z6.s\n" + "sqadd z30.s, z30.s, z5.s\n" + "sqadd z24.s, z24.s, z4.s\n" + "sqadd z25.s, z25.s, z3.s\n" + "sqadd z26.s, z26.s, z2.s\n" + "sqadd z27.s, z27.s, z1.s\n" "55:" // Height 4: no shift correction - "add x23, %x[qp], %[c_offset]\n" - "ld1rw { z4.s }, p2/Z, [x23]\n" + "add x20, %x[qp], %[c_offset]\n" + "ld1rw { z2.s }, p2/Z, [x20]\n" ".inst 0x4482881f // srshl z31.s, p2/M, z31.s, z0.s\n" - "add z31.s, z31.s, z4.s\n" + "add z31.s, z31.s, z2.s\n" ".inst 0x44828814 // srshl z20.s, p2/M, z20.s, z0.s\n" ".inst 0x44828815 // srshl z21.s, p2/M, z21.s, z0.s\n" - "add z20.s, z20.s, z4.s\n" - "add z21.s, z21.s, z4.s\n" + "add z20.s, z20.s, z2.s\n" + "add z21.s, z21.s, z2.s\n" ".inst 0x44828816 // srshl z22.s, p2/M, z22.s, z0.s\n" ".inst 0x44828810 // srshl z16.s, p2/M, z16.s, z0.s\n" - "add z22.s, z22.s, z4.s\n" - "add z16.s, z16.s, z4.s\n" + "add z22.s, z22.s, z2.s\n" + "add z16.s, z16.s, z2.s\n" ".inst 0x44828811 // srshl z17.s, p2/M, z17.s, z0.s\n" ".inst 0x44828812 // srshl z18.s, p2/M, z18.s, z0.s\n" - "add z17.s, z17.s, z4.s\n" - "add z18.s, z18.s, z4.s\n" + "add z17.s, z17.s, z2.s\n" + "add z18.s, z18.s, z2.s\n" ".inst 0x44828813 // srshl z19.s, p2/M, z19.s, z0.s\n" ".inst 0x44828817 // srshl z23.s, p2/M, z23.s, z0.s\n" - "add z19.s, z19.s, z4.s\n" - "add z23.s, z23.s, z4.s\n" + "add z19.s, z19.s, z2.s\n" + "add z23.s, z23.s, z2.s\n" ".inst 0x4482881c // srshl z28.s, p2/M, z28.s, z0.s\n" ".inst 0x4482881d // srshl z29.s, p2/M, z29.s, z0.s\n" - "add z28.s, z28.s, z4.s\n" - "add z29.s, z29.s, z4.s\n" + "add z28.s, z28.s, z2.s\n" + "add z29.s, z29.s, z2.s\n" ".inst 0x4482881e // srshl z30.s, p2/M, z30.s, z0.s\n" ".inst 0x44828818 // srshl z24.s, p2/M, z24.s, z0.s\n" - "add z30.s, z30.s, z4.s\n" - "add z24.s, z24.s, z4.s\n" + "add z30.s, z30.s, z2.s\n" + "add z24.s, z24.s, z2.s\n" ".inst 0x44828819 // srshl z25.s, p2/M, z25.s, z0.s\n" ".inst 0x4482881a // srshl z26.s, p2/M, z26.s, z0.s\n" - "add z25.s, z25.s, z4.s\n" - "add z26.s, z26.s, z4.s\n" + "add z25.s, z25.s, z2.s\n" + "add z26.s, z26.s, z2.s\n" ".inst 0x4482881b // srshl z27.s, p2/M, z27.s, z0.s\n" - "add x23, %x[qp], %[maxval]\n" - "ld1rw { z6.s }, p2/Z, [x23]\n" - "add z27.s, z27.s, z4.s\n" - "add x23, %x[qp], %[minval]\n" - "ld1rw { z5.s }, p2/Z, [x23]\n" - "smin z31.s, p2/M, z31.s, z6.s\n" - "smin z20.s, p2/M, z20.s, z6.s\n" - "smin z21.s, p2/M, z21.s, z6.s\n" - "smin z22.s, p2/M, z22.s, z6.s\n" - "smin z16.s, p2/M, z16.s, z6.s\n" - "smin z17.s, p2/M, z17.s, z6.s\n" - "smin z18.s, p2/M, z18.s, z6.s\n" - "smin z19.s, p2/M, z19.s, z6.s\n" - "smin z23.s, p2/M, z23.s, z6.s\n" - "smin z28.s, p2/M, z28.s, z6.s\n" - "smin z29.s, p2/M, z29.s, z6.s\n" - "smin z30.s, p2/M, z30.s, z6.s\n" - "smin z24.s, p2/M, z24.s, z6.s\n" - "smin z25.s, p2/M, z25.s, z6.s\n" - "smin z26.s, p2/M, z26.s, z6.s\n" - "smin z27.s, p2/M, z27.s, z6.s\n" - "smax z31.s, p2/M, z31.s, z5.s\n" - "smax z20.s, p2/M, z20.s, z5.s\n" - "smax z21.s, p2/M, z21.s, z5.s\n" + "add x20, %x[qp], %[maxval]\n" + "ld1rw { z1.s }, p2/Z, [x20]\n" + "add z27.s, z27.s, z2.s\n" + "add x20, %x[qp], %[minval]\n" + "ld1rw { z0.s }, p2/Z, [x20]\n" + "smin z31.s, p2/M, z31.s, z1.s\n" + "smin z20.s, p2/M, z20.s, z1.s\n" + "smin z21.s, p2/M, z21.s, z1.s\n" + "smin z22.s, p2/M, z22.s, z1.s\n" + "smin z16.s, p2/M, z16.s, z1.s\n" + "smin z17.s, p2/M, z17.s, z1.s\n" + "smin z18.s, p2/M, z18.s, z1.s\n" + "smin z19.s, p2/M, z19.s, z1.s\n" + "smin z23.s, p2/M, z23.s, z1.s\n" + "smin z28.s, p2/M, z28.s, z1.s\n" + "smin z29.s, p2/M, z29.s, z1.s\n" + "smin z30.s, p2/M, z30.s, z1.s\n" + "smin z24.s, p2/M, z24.s, z1.s\n" + "smin z25.s, p2/M, z25.s, z1.s\n" + "smin z26.s, p2/M, z26.s, z1.s\n" + "smin z27.s, p2/M, z27.s, z1.s\n" + "smax z31.s, p2/M, z31.s, z0.s\n" + "smax z20.s, p2/M, z20.s, z0.s\n" + "smax z21.s, p2/M, z21.s, z0.s\n" "uzp1 z31.h, z31.h, z20.h\n" - "smax z22.s, p2/M, z22.s, z5.s\n" - "smax z16.s, p2/M, z16.s, z5.s\n" + "smax z22.s, p2/M, z22.s, z0.s\n" + "smax z16.s, p2/M, z16.s, z0.s\n" "uzp1 z20.h, z21.h, z22.h\n" "uzp1 z31.b, z31.b, z20.b\n" - "smax z17.s, p2/M, z17.s, z5.s\n" - "smax z18.s, p2/M, z18.s, z5.s\n" + "smax z17.s, p2/M, z17.s, z0.s\n" + "smax z18.s, p2/M, z18.s, z0.s\n" "uzp1 z16.h, z16.h, z17.h\n" "st1b { z31.b }, p1, [x27]\n" - "smax z19.s, p2/M, z19.s, z5.s\n" - "smax z23.s, p2/M, z23.s, z5.s\n" + "smax z19.s, p2/M, z19.s, z0.s\n" + "smax z23.s, p2/M, z23.s, z0.s\n" "uzp1 z17.h, z18.h, z19.h\n" "uzp1 z16.b, z16.b, z17.b\n" - "smax z28.s, p2/M, z28.s, z5.s\n" - "smax z29.s, p2/M, z29.s, z5.s\n" + "smax z28.s, p2/M, z28.s, z0.s\n" + "smax z29.s, p2/M, z29.s, z0.s\n" "uzp1 z23.h, z23.h, z28.h\n" - "st1b { z16.b }, p1, [x22]\n" - "smax z30.s, p2/M, z30.s, z5.s\n" - "smax z24.s, p2/M, z24.s, z5.s\n" - "uzp1 z28.h, z29.h, z30.h\n" - "uzp1 z23.b, z23.b, z28.b\n" - "smax z25.s, p2/M, z25.s, z5.s\n" - "smax z26.s, p2/M, z26.s, z5.s\n" + "st1b { z16.b }, p1, [x23]\n" + "smax z30.s, p2/M, z30.s, z0.s\n" + "smax z24.s, p2/M, z24.s, z0.s\n" + "uzp1 z16.h, z29.h, z30.h\n" + "uzp1 z23.b, z23.b, z16.b\n" + "smax z25.s, p2/M, z25.s, z0.s\n" + "smax z26.s, p2/M, z26.s, z0.s\n" "uzp1 z24.h, z24.h, z25.h\n" - "st1b { z23.b }, p1, [x21]\n" - "smax z27.s, p2/M, z27.s, z5.s\n" - "uzp1 z25.h, z26.h, z27.h\n" - "uzp1 z24.b, z24.b, z25.b\n" - "st1b { z24.b }, p1, [x20]\n" + "st1b { z23.b }, p1, [x22]\n" + "smax z27.s, p2/M, z27.s, z0.s\n" + "uzp1 z16.h, z26.h, z27.h\n" + "uzp1 z24.b, z24.b, z16.b\n" + "st1b { z24.b }, p1, [x21]\n" "addvl x27, x27, #1\n" "56:" // Height 4: Writeback done "decw x9, ALL, MUL #4\n" @@ -1407,7 +1407,6 @@ void sve_hybrid_u8qa_mmla_4x4VL ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "58:" // Exit - : [M] "+&r" (M), [flags] "+&r" (flags), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [b_offset] "I" (offsetof(Requantize32, b_offset)), [c_offset] "I" (offsetof(Requantize32, c_offset)), [col_bias] "r" (col_bias), [maxval] "I" (offsetof(Requantize32, maxval)), [minval] "I" (offsetof(Requantize32, minval)), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)), [per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [qp] "r" (qp) : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -1415,4 +1414,4 @@ void sve_hybrid_u8qa_mmla_4x4VL ( } } // namespace arm_gemm -#endif // __ARM_FEATURE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL.hpp index 901cc6d63e5446cb69b47ae0d28b58c13006ada5..e9197e8ec53d1269a043bf72dd29d4a4fffd0693 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef ARM_COMPUTE_ENABLE_SVE + #include "../std_transforms_sve.hpp" #include "../performance_parameters.hpp" @@ -39,6 +39,7 @@ namespace arm_gemm { // Actual kernel implementations void sve_hybrid_u8u32_dot_6x4VL( ARGLIST ); +void sve_hybrid_u8u32_dot_6x4VL_a64fx( ARGLIST ); class cls_sve_hybrid_u8u32_dot_6x4VL { @@ -74,7 +75,6 @@ public: template static inline PerformanceParameters get_performance_parameters(const CPUInfo *ci) { - if (std::is_same::value) { switch (ci->get_cpu_model()) { default: @@ -83,10 +83,11 @@ public: return { 20.98 }; case CPUModel::V1: return { 62.19 }; + case CPUModel::A64FX: + return { 91.23 }; } } - if (std::is_same::value) { switch (ci->get_cpu_model()) { default: @@ -95,6 +96,8 @@ public: return { 22.75, 3.90, 0.47 }; case CPUModel::V1: return { 48.09, 16.24, 0.83 }; + case CPUModel::A64FX: + return { 101.62, 3.15, 0.42 }; } } @@ -103,13 +106,19 @@ public: // Default to the generic kernel kern_type kernel=sve_hybrid_u8u32_dot_6x4VL; - cls_sve_hybrid_u8u32_dot_6x4VL(const CPUInfo *) + cls_sve_hybrid_u8u32_dot_6x4VL(const CPUInfo *ci) { + switch(ci->get_cpu_model()) { + default: + break; + case CPUModel::A64FX: + kernel=sve_hybrid_u8u32_dot_6x4VL_a64fx; + break; + } } }; } // namespace arm_gemm #undef ARGLIST - #endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL/a64fx.cpp index a7dbef329eece876226eb274766f4ad6716043ad..4d0f44982a8281af41d7d5a373dbafa2242b0387 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL/a64fx.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL/a64fx.cpp @@ -115,11 +115,11 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "5:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 6f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 7f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -135,12 +135,12 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "8:" // Height 1: Multiply loop: Main loop "udot z8.s, z6.b, z0.b\n" "udot z9.s, z7.b, z0.b\n" - "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n" - "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n" + "ld1b { z17.b }, p4/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" "add x26, x26, #0x4\n" - "udot z10.s, z6.b, z0.b\n" - "udot z11.s, z7.b, z0.b\n" + "udot z10.s, z17.b, z0.b\n" + "udot z11.s, z16.b, z0.b\n" "subs x27, x27, #0x4\n" "ld1rw { z0.s }, p4/Z, [x26]\n" "ld1b { z6.b }, p4/Z, [x10]\n" @@ -150,12 +150,12 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "udot z8.s, z6.b, z0.b\n" "udot z9.s, z7.b, z0.b\n" - "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n" - "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n" + "ld1b { z17.b }, p4/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p4/Z, [x10, #3, MUL VL]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" - "udot z10.s, z6.b, z0.b\n" - "udot z11.s, z7.b, z0.b\n" + "udot z10.s, z17.b, z0.b\n" + "udot z11.s, z16.b, z0.b\n" "addvl x10, x10, #4\n" "bne 5b\n" "st1w { z8.s }, p3, [x9]\n" @@ -183,15 +183,15 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "whilelt p0.s, x20, x11\n" "tbz %x[flags], #0, 13f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" + "add x20, x9, x20, LSL #2\n" "ld1w { z8.s }, p3/Z, [x9]\n" "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x24]\n" - "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n" - "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n" - "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n" + "ld1w { z12.s }, p3/Z, [x20]\n" + "ld1w { z13.s }, p2/Z, [x20, #1, MUL VL]\n" + "ld1w { z14.s }, p1/Z, [x20, #2, MUL VL]\n" + "ld1w { z15.s }, p0/Z, [x20, #3, MUL VL]\n" "b 14f\n" "13:" // Height 2: no accumulate "mov z8.s, #0x0\n" @@ -207,12 +207,12 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "15:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 16f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 17f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -220,7 +220,7 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "b 17f\n" "16:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" + "add x25, x26, x21\n" "17:" // Height 2: input setup done "subs x27, x27, #0x4\n" "ld1rw { z0.s }, p4/Z, [x26]\n" @@ -231,18 +231,18 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "18:" // Height 2: Multiply loop: Main loop "udot z8.s, z6.b, z0.b\n" "udot z12.s, z6.b, z1.b\n" - "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n" + "ld1b { z17.b }, p4/Z, [x10, #2, MUL VL]\n" "add x26, x26, #0x4\n" "udot z9.s, z7.b, z0.b\n" "udot z13.s, z7.b, z1.b\n" - "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n" + "ld1b { z16.b }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" "subs x27, x27, #0x4\n" "add x25, x25, #0x4\n" - "udot z10.s, z6.b, z0.b\n" - "udot z14.s, z6.b, z1.b\n" - "udot z11.s, z7.b, z0.b\n" - "udot z15.s, z7.b, z1.b\n" + "udot z10.s, z17.b, z0.b\n" + "udot z14.s, z17.b, z1.b\n" + "udot z11.s, z16.b, z0.b\n" + "udot z15.s, z16.b, z1.b\n" "ld1rw { z0.s }, p4/Z, [x26]\n" "ld1rw { z1.s }, p4/Z, [x25]\n" "ld1b { z6.b }, p4/Z, [x10]\n" @@ -252,29 +252,29 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "udot z8.s, z6.b, z0.b\n" "udot z12.s, z6.b, z1.b\n" - "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n" + "ld1b { z17.b }, p4/Z, [x10, #2, MUL VL]\n" "udot z9.s, z7.b, z0.b\n" "udot z13.s, z7.b, z1.b\n" - "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n" + "ld1b { z16.b }, p4/Z, [x10, #3, MUL VL]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" - "udot z10.s, z6.b, z0.b\n" - "udot z14.s, z6.b, z1.b\n" + "udot z10.s, z17.b, z0.b\n" + "udot z14.s, z17.b, z1.b\n" "addvl x10, x10, #4\n" - "udot z11.s, z7.b, z0.b\n" - "udot z15.s, z7.b, z1.b\n" + "udot z11.s, z16.b, z0.b\n" + "udot z15.s, z16.b, z1.b\n" "bne 15b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" + "add x20, x9, x20, LSL #2\n" "st1w { z8.s }, p3, [x9]\n" "st1w { z9.s }, p2, [x9, #1, MUL VL]\n" "st1w { z10.s }, p1, [x9, #2, MUL VL]\n" "st1w { z11.s }, p0, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "st1w { z12.s }, p3, [x24]\n" - "st1w { z13.s }, p2, [x24, #1, MUL VL]\n" - "st1w { z14.s }, p1, [x24, #2, MUL VL]\n" - "st1w { z15.s }, p0, [x24, #3, MUL VL]\n" + "st1w { z12.s }, p3, [x20]\n" + "st1w { z13.s }, p2, [x20, #1, MUL VL]\n" + "st1w { z14.s }, p1, [x20, #2, MUL VL]\n" + "st1w { z15.s }, p0, [x20, #3, MUL VL]\n" "20:" // Height 2: Writeback done "decw x11, ALL, MUL #4\n" "cmp x11, XZR\n" @@ -295,20 +295,20 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "whilelt p0.s, x20, x11\n" "tbz %x[flags], #0, 23f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x21, x9, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z8.s }, p3/Z, [x9]\n" "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x24]\n" - "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n" - "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n" - "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n" - "ld1w { z16.s }, p3/Z, [x23]\n" - "ld1w { z17.s }, p2/Z, [x23, #1, MUL VL]\n" - "ld1w { z18.s }, p1/Z, [x23, #2, MUL VL]\n" - "ld1w { z19.s }, p0/Z, [x23, #3, MUL VL]\n" + "ld1w { z12.s }, p3/Z, [x21]\n" + "ld1w { z13.s }, p2/Z, [x21, #1, MUL VL]\n" + "ld1w { z14.s }, p1/Z, [x21, #2, MUL VL]\n" + "ld1w { z15.s }, p0/Z, [x21, #3, MUL VL]\n" + "ld1w { z16.s }, p3/Z, [x20]\n" + "ld1w { z17.s }, p2/Z, [x20, #1, MUL VL]\n" + "ld1w { z18.s }, p1/Z, [x20, #2, MUL VL]\n" + "ld1w { z19.s }, p0/Z, [x20, #3, MUL VL]\n" "b 24f\n" "23:" // Height 3: no accumulate "mov z8.s, #0x0\n" @@ -328,13 +328,13 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "25:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 26f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 27f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -343,8 +343,8 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "b 27f\n" "26:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" "27:" // Height 3: input setup done "subs x27, x27, #0x4\n" "ld1rw { z0.s }, p4/Z, [x26]\n" @@ -360,21 +360,21 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "subs x27, x27, #0x4\n" "udot z16.s, z6.b, z2.b\n" "udot z9.s, z7.b, z0.b\n" - "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n" + "ld1b { z21.b }, p4/Z, [x10, #2, MUL VL]\n" "add x25, x25, #0x4\n" "udot z13.s, z7.b, z1.b\n" "udot z17.s, z7.b, z2.b\n" - "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n" + "ld1b { z20.b }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" "add x24, x24, #0x4\n" - "udot z10.s, z6.b, z0.b\n" - "udot z14.s, z6.b, z1.b\n" - "udot z18.s, z6.b, z2.b\n" - "udot z11.s, z7.b, z0.b\n" + "udot z10.s, z21.b, z0.b\n" + "udot z14.s, z21.b, z1.b\n" + "udot z18.s, z21.b, z2.b\n" + "udot z11.s, z20.b, z0.b\n" "ld1rw { z0.s }, p4/Z, [x26]\n" "ld1b { z6.b }, p4/Z, [x10]\n" - "udot z15.s, z7.b, z1.b\n" - "udot z19.s, z7.b, z2.b\n" + "udot z15.s, z20.b, z1.b\n" + "udot z19.s, z20.b, z2.b\n" "ld1rw { z1.s }, p4/Z, [x25]\n" "ld1rw { z2.s }, p4/Z, [x24]\n" "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n" @@ -386,35 +386,35 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "add x28, x28, #0x1\n" "udot z16.s, z6.b, z2.b\n" "udot z9.s, z7.b, z0.b\n" - "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n" + "ld1b { z21.b }, p4/Z, [x10, #2, MUL VL]\n" "cmp x28, x20\n" "udot z13.s, z7.b, z1.b\n" "udot z17.s, z7.b, z2.b\n" - "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n" + "ld1b { z20.b }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "udot z10.s, z6.b, z0.b\n" - "udot z14.s, z6.b, z1.b\n" - "udot z18.s, z6.b, z2.b\n" - "udot z11.s, z7.b, z0.b\n" - "udot z15.s, z7.b, z1.b\n" - "udot z19.s, z7.b, z2.b\n" + "udot z10.s, z21.b, z0.b\n" + "udot z14.s, z21.b, z1.b\n" + "udot z18.s, z21.b, z2.b\n" + "udot z11.s, z20.b, z0.b\n" + "udot z15.s, z20.b, z1.b\n" + "udot z19.s, z20.b, z2.b\n" "bne 25b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x21, x9, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "st1w { z8.s }, p3, [x9]\n" "st1w { z9.s }, p2, [x9, #1, MUL VL]\n" "st1w { z10.s }, p1, [x9, #2, MUL VL]\n" "st1w { z11.s }, p0, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "st1w { z12.s }, p3, [x24]\n" - "st1w { z13.s }, p2, [x24, #1, MUL VL]\n" - "st1w { z14.s }, p1, [x24, #2, MUL VL]\n" - "st1w { z15.s }, p0, [x24, #3, MUL VL]\n" - "st1w { z16.s }, p3, [x23]\n" - "st1w { z17.s }, p2, [x23, #1, MUL VL]\n" - "st1w { z18.s }, p1, [x23, #2, MUL VL]\n" - "st1w { z19.s }, p0, [x23, #3, MUL VL]\n" + "st1w { z12.s }, p3, [x21]\n" + "st1w { z13.s }, p2, [x21, #1, MUL VL]\n" + "st1w { z14.s }, p1, [x21, #2, MUL VL]\n" + "st1w { z15.s }, p0, [x21, #3, MUL VL]\n" + "st1w { z16.s }, p3, [x20]\n" + "st1w { z17.s }, p2, [x20, #1, MUL VL]\n" + "st1w { z18.s }, p1, [x20, #2, MUL VL]\n" + "st1w { z19.s }, p0, [x20, #3, MUL VL]\n" "30:" // Height 3: Writeback done "decw x11, ALL, MUL #4\n" "cmp x11, XZR\n" @@ -435,25 +435,25 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "whilelt p0.s, x20, x11\n" "tbz %x[flags], #0, 33f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x22, x9, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" "ld1w { z8.s }, p3/Z, [x9]\n" - "add x22, x23, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x24]\n" - "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n" - "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n" - "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n" - "ld1w { z16.s }, p3/Z, [x23]\n" - "ld1w { z17.s }, p2/Z, [x23, #1, MUL VL]\n" - "ld1w { z18.s }, p1/Z, [x23, #2, MUL VL]\n" - "ld1w { z19.s }, p0/Z, [x23, #3, MUL VL]\n" - "ld1w { z20.s }, p3/Z, [x22]\n" - "ld1w { z21.s }, p2/Z, [x22, #1, MUL VL]\n" - "ld1w { z22.s }, p1/Z, [x22, #2, MUL VL]\n" - "ld1w { z23.s }, p0/Z, [x22, #3, MUL VL]\n" + "ld1w { z12.s }, p3/Z, [x22]\n" + "ld1w { z13.s }, p2/Z, [x22, #1, MUL VL]\n" + "ld1w { z14.s }, p1/Z, [x22, #2, MUL VL]\n" + "ld1w { z15.s }, p0/Z, [x22, #3, MUL VL]\n" + "ld1w { z16.s }, p3/Z, [x21]\n" + "ld1w { z17.s }, p2/Z, [x21, #1, MUL VL]\n" + "ld1w { z18.s }, p1/Z, [x21, #2, MUL VL]\n" + "ld1w { z19.s }, p0/Z, [x21, #3, MUL VL]\n" + "ld1w { z20.s }, p3/Z, [x20]\n" + "ld1w { z21.s }, p2/Z, [x20, #1, MUL VL]\n" + "ld1w { z22.s }, p1/Z, [x20, #2, MUL VL]\n" + "ld1w { z23.s }, p0/Z, [x20, #3, MUL VL]\n" "b 34f\n" "33:" // Height 4: no accumulate "mov z8.s, #0x0\n" @@ -477,14 +477,14 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "35:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 36f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 37f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -494,9 +494,9 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "b 37f\n" "36:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" "37:" // Height 4: input setup done "subs x27, x27, #0x4\n" "ld1rw { z0.s }, p4/Z, [x26]\n" @@ -513,7 +513,7 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "subs x27, x27, #0x4\n" "udot z16.s, z6.b, z2.b\n" "udot z20.s, z6.b, z3.b\n" - "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n" + "ld1b { z25.b }, p4/Z, [x10, #2, MUL VL]\n" "add x25, x25, #0x4\n" "udot z9.s, z7.b, z0.b\n" "udot z13.s, z7.b, z1.b\n" @@ -521,19 +521,19 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "add x23, x23, #0x4\n" "udot z17.s, z7.b, z2.b\n" "udot z21.s, z7.b, z3.b\n" - "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n" + "ld1b { z24.b }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "udot z10.s, z6.b, z0.b\n" - "udot z14.s, z6.b, z1.b\n" - "udot z18.s, z6.b, z2.b\n" - "udot z22.s, z6.b, z3.b\n" + "udot z10.s, z25.b, z0.b\n" + "udot z14.s, z25.b, z1.b\n" + "udot z18.s, z25.b, z2.b\n" + "udot z22.s, z25.b, z3.b\n" "ld1b { z6.b }, p4/Z, [x10]\n" - "udot z11.s, z7.b, z0.b\n" - "udot z15.s, z7.b, z1.b\n" + "udot z11.s, z24.b, z0.b\n" + "udot z15.s, z24.b, z1.b\n" "ld1rw { z0.s }, p4/Z, [x26]\n" "ld1rw { z1.s }, p4/Z, [x25]\n" - "udot z19.s, z7.b, z2.b\n" - "udot z23.s, z7.b, z3.b\n" + "udot z19.s, z24.b, z2.b\n" + "udot z23.s, z24.b, z3.b\n" "ld1rw { z2.s }, p4/Z, [x24]\n" "ld1rw { z3.s }, p4/Z, [x23]\n" "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n" @@ -545,44 +545,44 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "add x28, x28, #0x1\n" "udot z16.s, z6.b, z2.b\n" "udot z20.s, z6.b, z3.b\n" - "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n" + "ld1b { z25.b }, p4/Z, [x10, #2, MUL VL]\n" "cmp x28, x20\n" "udot z9.s, z7.b, z0.b\n" "udot z13.s, z7.b, z1.b\n" "udot z17.s, z7.b, z2.b\n" "udot z21.s, z7.b, z3.b\n" - "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n" + "ld1b { z24.b }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "udot z10.s, z6.b, z0.b\n" - "udot z14.s, z6.b, z1.b\n" - "udot z18.s, z6.b, z2.b\n" - "udot z22.s, z6.b, z3.b\n" - "udot z11.s, z7.b, z0.b\n" - "udot z15.s, z7.b, z1.b\n" - "udot z19.s, z7.b, z2.b\n" - "udot z23.s, z7.b, z3.b\n" + "udot z10.s, z25.b, z0.b\n" + "udot z14.s, z25.b, z1.b\n" + "udot z18.s, z25.b, z2.b\n" + "udot z22.s, z25.b, z3.b\n" + "udot z11.s, z24.b, z0.b\n" + "udot z15.s, z24.b, z1.b\n" + "udot z19.s, z24.b, z2.b\n" + "udot z23.s, z24.b, z3.b\n" "bne 35b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x22, x9, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" "st1w { z8.s }, p3, [x9]\n" - "add x22, x23, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "st1w { z9.s }, p2, [x9, #1, MUL VL]\n" "st1w { z10.s }, p1, [x9, #2, MUL VL]\n" "st1w { z11.s }, p0, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "st1w { z12.s }, p3, [x24]\n" - "st1w { z13.s }, p2, [x24, #1, MUL VL]\n" - "st1w { z14.s }, p1, [x24, #2, MUL VL]\n" - "st1w { z15.s }, p0, [x24, #3, MUL VL]\n" - "st1w { z16.s }, p3, [x23]\n" - "st1w { z17.s }, p2, [x23, #1, MUL VL]\n" - "st1w { z18.s }, p1, [x23, #2, MUL VL]\n" - "st1w { z19.s }, p0, [x23, #3, MUL VL]\n" - "st1w { z20.s }, p3, [x22]\n" - "st1w { z21.s }, p2, [x22, #1, MUL VL]\n" - "st1w { z22.s }, p1, [x22, #2, MUL VL]\n" - "st1w { z23.s }, p0, [x22, #3, MUL VL]\n" + "st1w { z12.s }, p3, [x22]\n" + "st1w { z13.s }, p2, [x22, #1, MUL VL]\n" + "st1w { z14.s }, p1, [x22, #2, MUL VL]\n" + "st1w { z15.s }, p0, [x22, #3, MUL VL]\n" + "st1w { z16.s }, p3, [x21]\n" + "st1w { z17.s }, p2, [x21, #1, MUL VL]\n" + "st1w { z18.s }, p1, [x21, #2, MUL VL]\n" + "st1w { z19.s }, p0, [x21, #3, MUL VL]\n" + "st1w { z20.s }, p3, [x20]\n" + "st1w { z21.s }, p2, [x20, #1, MUL VL]\n" + "st1w { z22.s }, p1, [x20, #2, MUL VL]\n" + "st1w { z23.s }, p0, [x20, #3, MUL VL]\n" "40:" // Height 4: Writeback done "decw x11, ALL, MUL #4\n" "cmp x11, XZR\n" @@ -603,30 +603,30 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "whilelt p0.s, x20, x11\n" "tbz %x[flags], #0, 43f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "ld1w { z8.s }, p3/Z, [x9]\n" + "add x23, x9, x20, LSL #2\n" "add x22, x23, x20, LSL #2\n" + "ld1w { z8.s }, p3/Z, [x9]\n" "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z9.s }, p2/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p1/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p0/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p3/Z, [x24]\n" - "ld1w { z13.s }, p2/Z, [x24, #1, MUL VL]\n" - "ld1w { z14.s }, p1/Z, [x24, #2, MUL VL]\n" - "ld1w { z15.s }, p0/Z, [x24, #3, MUL VL]\n" - "ld1w { z16.s }, p3/Z, [x23]\n" - "ld1w { z17.s }, p2/Z, [x23, #1, MUL VL]\n" - "ld1w { z18.s }, p1/Z, [x23, #2, MUL VL]\n" - "ld1w { z19.s }, p0/Z, [x23, #3, MUL VL]\n" - "ld1w { z20.s }, p3/Z, [x22]\n" - "ld1w { z21.s }, p2/Z, [x22, #1, MUL VL]\n" - "ld1w { z22.s }, p1/Z, [x22, #2, MUL VL]\n" - "ld1w { z23.s }, p0/Z, [x22, #3, MUL VL]\n" - "ld1w { z24.s }, p3/Z, [x21]\n" - "ld1w { z25.s }, p2/Z, [x21, #1, MUL VL]\n" - "ld1w { z26.s }, p1/Z, [x21, #2, MUL VL]\n" - "ld1w { z27.s }, p0/Z, [x21, #3, MUL VL]\n" + "ld1w { z12.s }, p3/Z, [x23]\n" + "ld1w { z13.s }, p2/Z, [x23, #1, MUL VL]\n" + "ld1w { z14.s }, p1/Z, [x23, #2, MUL VL]\n" + "ld1w { z15.s }, p0/Z, [x23, #3, MUL VL]\n" + "ld1w { z16.s }, p3/Z, [x22]\n" + "ld1w { z17.s }, p2/Z, [x22, #1, MUL VL]\n" + "ld1w { z18.s }, p1/Z, [x22, #2, MUL VL]\n" + "ld1w { z19.s }, p0/Z, [x22, #3, MUL VL]\n" + "ld1w { z20.s }, p3/Z, [x21]\n" + "ld1w { z21.s }, p2/Z, [x21, #1, MUL VL]\n" + "ld1w { z22.s }, p1/Z, [x21, #2, MUL VL]\n" + "ld1w { z23.s }, p0/Z, [x21, #3, MUL VL]\n" + "ld1w { z24.s }, p3/Z, [x20]\n" + "ld1w { z25.s }, p2/Z, [x20, #1, MUL VL]\n" + "ld1w { z26.s }, p1/Z, [x20, #2, MUL VL]\n" + "ld1w { z27.s }, p0/Z, [x20, #3, MUL VL]\n" "b 44f\n" "43:" // Height 5: no accumulate "mov z8.s, #0x0\n" @@ -654,15 +654,15 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "45:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 46f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 47f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -673,10 +673,10 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "b 47f\n" "46:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" "47:" // Height 5: input setup done "subs x27, x27, #0x4\n" "ld1rw { z0.s }, p4/Z, [x26]\n" @@ -698,29 +698,29 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "add x24, x24, #0x4\n" "udot z24.s, z6.b, z4.b\n" "udot z9.s, z7.b, z0.b\n" - "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n" + "ld1b { z29.b }, p4/Z, [x10, #2, MUL VL]\n" "add x23, x23, #0x4\n" "udot z13.s, z7.b, z1.b\n" "udot z17.s, z7.b, z2.b\n" "add x22, x22, #0x4\n" "udot z21.s, z7.b, z3.b\n" "udot z25.s, z7.b, z4.b\n" - "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n" + "ld1b { z28.b }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "udot z10.s, z6.b, z0.b\n" - "udot z14.s, z6.b, z1.b\n" - "udot z18.s, z6.b, z2.b\n" - "udot z22.s, z6.b, z3.b\n" - "udot z26.s, z6.b, z4.b\n" - "udot z11.s, z7.b, z0.b\n" + "udot z10.s, z29.b, z0.b\n" + "udot z14.s, z29.b, z1.b\n" + "udot z18.s, z29.b, z2.b\n" + "udot z22.s, z29.b, z3.b\n" + "udot z26.s, z29.b, z4.b\n" + "udot z11.s, z28.b, z0.b\n" "ld1rw { z0.s }, p4/Z, [x26]\n" "ld1b { z6.b }, p4/Z, [x10]\n" - "udot z15.s, z7.b, z1.b\n" - "udot z19.s, z7.b, z2.b\n" + "udot z15.s, z28.b, z1.b\n" + "udot z19.s, z28.b, z2.b\n" "ld1rw { z1.s }, p4/Z, [x25]\n" "ld1rw { z2.s }, p4/Z, [x24]\n" - "udot z23.s, z7.b, z3.b\n" - "udot z27.s, z7.b, z4.b\n" + "udot z23.s, z28.b, z3.b\n" + "udot z27.s, z28.b, z4.b\n" "ld1rw { z3.s }, p4/Z, [x23]\n" "ld1rw { z4.s }, p4/Z, [x22]\n" "ld1b { z7.b }, p4/Z, [x10, #1, MUL VL]\n" @@ -735,50 +735,50 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "cmp x28, x20\n" "udot z24.s, z6.b, z4.b\n" "udot z9.s, z7.b, z0.b\n" - "ld1b { z6.b }, p4/Z, [x10, #2, MUL VL]\n" + "ld1b { z29.b }, p4/Z, [x10, #2, MUL VL]\n" "udot z13.s, z7.b, z1.b\n" "udot z17.s, z7.b, z2.b\n" "udot z21.s, z7.b, z3.b\n" "udot z25.s, z7.b, z4.b\n" - "ld1b { z7.b }, p4/Z, [x10, #3, MUL VL]\n" + "ld1b { z28.b }, p4/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "udot z10.s, z6.b, z0.b\n" - "udot z14.s, z6.b, z1.b\n" - "udot z18.s, z6.b, z2.b\n" - "udot z22.s, z6.b, z3.b\n" - "udot z26.s, z6.b, z4.b\n" - "udot z11.s, z7.b, z0.b\n" - "udot z15.s, z7.b, z1.b\n" - "udot z19.s, z7.b, z2.b\n" - "udot z23.s, z7.b, z3.b\n" - "udot z27.s, z7.b, z4.b\n" + "udot z10.s, z29.b, z0.b\n" + "udot z14.s, z29.b, z1.b\n" + "udot z18.s, z29.b, z2.b\n" + "udot z22.s, z29.b, z3.b\n" + "udot z26.s, z29.b, z4.b\n" + "udot z11.s, z28.b, z0.b\n" + "udot z15.s, z28.b, z1.b\n" + "udot z19.s, z28.b, z2.b\n" + "udot z23.s, z28.b, z3.b\n" + "udot z27.s, z28.b, z4.b\n" "bne 45b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "st1w { z8.s }, p3, [x9]\n" + "add x23, x9, x20, LSL #2\n" "add x22, x23, x20, LSL #2\n" + "st1w { z8.s }, p3, [x9]\n" "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "st1w { z9.s }, p2, [x9, #1, MUL VL]\n" "st1w { z10.s }, p1, [x9, #2, MUL VL]\n" "st1w { z11.s }, p0, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "st1w { z12.s }, p3, [x24]\n" - "st1w { z13.s }, p2, [x24, #1, MUL VL]\n" - "st1w { z14.s }, p1, [x24, #2, MUL VL]\n" - "st1w { z15.s }, p0, [x24, #3, MUL VL]\n" - "st1w { z16.s }, p3, [x23]\n" - "st1w { z17.s }, p2, [x23, #1, MUL VL]\n" - "st1w { z18.s }, p1, [x23, #2, MUL VL]\n" - "st1w { z19.s }, p0, [x23, #3, MUL VL]\n" - "st1w { z20.s }, p3, [x22]\n" - "st1w { z21.s }, p2, [x22, #1, MUL VL]\n" - "st1w { z22.s }, p1, [x22, #2, MUL VL]\n" - "st1w { z23.s }, p0, [x22, #3, MUL VL]\n" - "st1w { z24.s }, p3, [x21]\n" - "st1w { z25.s }, p2, [x21, #1, MUL VL]\n" - "st1w { z26.s }, p1, [x21, #2, MUL VL]\n" - "st1w { z27.s }, p0, [x21, #3, MUL VL]\n" + "st1w { z12.s }, p3, [x23]\n" + "st1w { z13.s }, p2, [x23, #1, MUL VL]\n" + "st1w { z14.s }, p1, [x23, #2, MUL VL]\n" + "st1w { z15.s }, p0, [x23, #3, MUL VL]\n" + "st1w { z16.s }, p3, [x22]\n" + "st1w { z17.s }, p2, [x22, #1, MUL VL]\n" + "st1w { z18.s }, p1, [x22, #2, MUL VL]\n" + "st1w { z19.s }, p0, [x22, #3, MUL VL]\n" + "st1w { z20.s }, p3, [x21]\n" + "st1w { z21.s }, p2, [x21, #1, MUL VL]\n" + "st1w { z22.s }, p1, [x21, #2, MUL VL]\n" + "st1w { z23.s }, p0, [x21, #3, MUL VL]\n" + "st1w { z24.s }, p3, [x20]\n" + "st1w { z25.s }, p2, [x20, #1, MUL VL]\n" + "st1w { z26.s }, p1, [x20, #2, MUL VL]\n" + "st1w { z27.s }, p0, [x20, #3, MUL VL]\n" "50:" // Height 5: Writeback done "decw x11, ALL, MUL #4\n" "cmp x11, XZR\n" @@ -862,16 +862,16 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "55:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 56f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 57f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -883,11 +883,11 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "b 57f\n" "56:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" + "add x21, x22, x21\n" "57:" // Height 6: input setup done "subs x27, x27, #0x4\n" "ld1rw { z0.s }, p4/Z, [x26]\n" @@ -1022,7 +1022,6 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "62:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -1030,4 +1029,4 @@ void sve_hybrid_u8u32_dot_6x4VL_a64fx ( } } // namespace arm_gemm -#endif // ARM_COMPUTE_ENABLE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL/generic.cpp index 30a108af7ee8644e7419ae77ba9ca819949652ba..7871c0b0034192e0ca97f84ba68e3a3e030c27a7 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_dot_6x4VL/generic.cpp @@ -115,11 +115,11 @@ void sve_hybrid_u8u32_dot_6x4VL ( "5:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 6f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 7f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -132,87 +132,87 @@ void sve_hybrid_u8u32_dot_6x4VL ( "8:" // Height 1: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" "ld1rqb { z0.b }, p0/Z, [x26]\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "udot z8.s, z6.b, z0.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z9.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "udot z10.s, z6.b, z0.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" - "udot z11.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #4, MUL VL]\n" - "udot z8.s, z6.b, z0.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #5, MUL VL]\n" - "udot z9.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #6, MUL VL]\n" - "udot z10.s, z6.b, z0.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #7, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10]\n" + "udot z8.s, z16.b, z0.b[0]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z9.s, z16.b, z0.b[0]\n" + "ld1b { z16.b }, p5/Z, [x10, #2, MUL VL]\n" + "udot z10.s, z16.b, z0.b[0]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" + "udot z11.s, z16.b, z0.b[0]\n" + "ld1b { z16.b }, p5/Z, [x10, #4, MUL VL]\n" + "udot z8.s, z16.b, z0.b[1]\n" + "ld1b { z16.b }, p5/Z, [x10, #5, MUL VL]\n" + "udot z9.s, z16.b, z0.b[1]\n" + "ld1b { z16.b }, p5/Z, [x10, #6, MUL VL]\n" + "udot z10.s, z16.b, z0.b[1]\n" + "ld1b { z16.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "udot z11.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #-8, MUL VL]\n" - "ld1b { z7.b }, p5/Z, [x10, #-7, MUL VL]\n" - "udot z8.s, z6.b, z0.b[2]\n" - "udot z9.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #-6, MUL VL]\n" - "ld1b { z7.b }, p5/Z, [x10, #-5, MUL VL]\n" - "udot z10.s, z6.b, z0.b[2]\n" - "udot z11.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #-4, MUL VL]\n" - "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n" - "udot z8.s, z6.b, z0.b[3]\n" - "udot z9.s, z7.b, z0.b[3]\n" - "ld1b { z6.b }, p5/Z, [x10, #-2, MUL VL]\n" - "ld1b { z7.b }, p5/Z, [x10, #-1, MUL VL]\n" + "udot z11.s, z16.b, z0.b[1]\n" + "ld1b { z17.b }, p5/Z, [x10, #-8, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #-7, MUL VL]\n" + "udot z8.s, z17.b, z0.b[2]\n" + "udot z9.s, z16.b, z0.b[2]\n" + "ld1b { z17.b }, p5/Z, [x10, #-6, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #-5, MUL VL]\n" + "udot z10.s, z17.b, z0.b[2]\n" + "udot z11.s, z16.b, z0.b[2]\n" + "ld1b { z17.b }, p5/Z, [x10, #-4, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #-3, MUL VL]\n" + "udot z8.s, z17.b, z0.b[3]\n" + "udot z9.s, z16.b, z0.b[3]\n" + "ld1b { z17.b }, p5/Z, [x10, #-2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #-1, MUL VL]\n" "sub x27, x27, #0x10\n" "cmp x27, #0x10\n" - "udot z10.s, z6.b, z0.b[3]\n" - "udot z11.s, z7.b, z0.b[3]\n" + "udot z10.s, z17.b, z0.b[3]\n" + "udot z11.s, z16.b, z0.b[3]\n" "add x26, x26, #0x10\n" "bgt 8b\n" "9:" // Height 1: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" "ld1rqb { z0.b }, p0/Z, [x26]\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "udot z8.s, z6.b, z0.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z9.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10]\n" + "udot z8.s, z16.b, z0.b[0]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z9.s, z16.b, z0.b[0]\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x4\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" - "udot z10.s, z6.b, z0.b[0]\n" - "udot z11.s, z7.b, z0.b[0]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" + "udot z10.s, z17.b, z0.b[0]\n" + "udot z11.s, z16.b, z0.b[0]\n" "addvl x10, x10, #4\n" "ble 10f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z8.s, z6.b, z0.b[1]\n" - "udot z9.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z8.s, z17.b, z0.b[1]\n" + "udot z9.s, z16.b, z0.b[1]\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x4\n" - "udot z10.s, z6.b, z0.b[1]\n" - "udot z11.s, z7.b, z0.b[1]\n" + "udot z10.s, z17.b, z0.b[1]\n" + "udot z11.s, z16.b, z0.b[1]\n" "addvl x10, x10, #4\n" "ble 10f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z8.s, z6.b, z0.b[2]\n" - "udot z9.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z8.s, z17.b, z0.b[2]\n" + "udot z9.s, z16.b, z0.b[2]\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x4\n" - "udot z10.s, z6.b, z0.b[2]\n" - "udot z11.s, z7.b, z0.b[2]\n" + "udot z10.s, z17.b, z0.b[2]\n" + "udot z11.s, z16.b, z0.b[2]\n" "addvl x10, x10, #4\n" "ble 10f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z8.s, z6.b, z0.b[3]\n" - "udot z9.s, z7.b, z0.b[3]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" - "udot z10.s, z6.b, z0.b[3]\n" - "udot z11.s, z7.b, z0.b[3]\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z8.s, z17.b, z0.b[3]\n" + "udot z9.s, z16.b, z0.b[3]\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" + "udot z10.s, z17.b, z0.b[3]\n" + "udot z11.s, z16.b, z0.b[3]\n" "addvl x10, x10, #4\n" "10:" // Height 1: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -244,15 +244,15 @@ void sve_hybrid_u8u32_dot_6x4VL ( "whilelt p1.s, x20, x11\n" "tbz %x[flags], #0, 14f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" + "add x20, x9, x20, LSL #2\n" "ld1w { z8.s }, p4/Z, [x9]\n" "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x24]\n" - "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n" - "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x20]\n" + "ld1w { z13.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z15.s }, p1/Z, [x20, #3, MUL VL]\n" "b 15f\n" "14:" // Height 2: no accumulate "mov z8.s, #0x0\n" @@ -268,12 +268,12 @@ void sve_hybrid_u8u32_dot_6x4VL ( "16:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 17f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 18f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -281,146 +281,146 @@ void sve_hybrid_u8u32_dot_6x4VL ( "b 18f\n" "17:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" + "add x25, x26, x21\n" "18:" // Height 2: input setup done "cmp x27, #0x10\n" "ble 20f\n" "19:" // Height 2: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z0.b }, p0/Z, [x26]\n" - "ld1rqb { z1.b }, p0/Z, [x25]\n" + "ld1rqb { z1.b }, p0/Z, [x26]\n" + "ld1rqb { z0.b }, p0/Z, [x25]\n" "sub x27, x27, #0x10\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z8.s, z6.b, z0.b[0]\n" - "udot z12.s, z6.b, z1.b[0]\n" - "udot z9.s, z7.b, z0.b[0]\n" - "udot z13.s, z7.b, z1.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" - "udot z10.s, z6.b, z0.b[0]\n" - "udot z14.s, z6.b, z1.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #4, MUL VL]\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z8.s, z17.b, z1.b[0]\n" + "udot z12.s, z17.b, z0.b[0]\n" + "udot z9.s, z16.b, z1.b[0]\n" + "udot z13.s, z16.b, z0.b[0]\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" + "udot z10.s, z17.b, z1.b[0]\n" + "udot z14.s, z17.b, z0.b[0]\n" + "ld1b { z17.b }, p5/Z, [x10, #4, MUL VL]\n" "cmp x27, #0x10\n" - "udot z11.s, z7.b, z0.b[0]\n" - "udot z15.s, z7.b, z1.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #5, MUL VL]\n" + "udot z11.s, z16.b, z1.b[0]\n" + "udot z15.s, z16.b, z0.b[0]\n" + "ld1b { z16.b }, p5/Z, [x10, #5, MUL VL]\n" "add x26, x26, #0x10\n" - "udot z8.s, z6.b, z0.b[1]\n" - "udot z12.s, z6.b, z1.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #6, MUL VL]\n" + "udot z8.s, z17.b, z1.b[1]\n" + "udot z12.s, z17.b, z0.b[1]\n" + "ld1b { z17.b }, p5/Z, [x10, #6, MUL VL]\n" "add x25, x25, #0x10\n" - "udot z9.s, z7.b, z0.b[1]\n" - "udot z13.s, z7.b, z1.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #7, MUL VL]\n" + "udot z9.s, z16.b, z1.b[1]\n" + "udot z13.s, z16.b, z0.b[1]\n" + "ld1b { z16.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "udot z10.s, z6.b, z0.b[1]\n" - "udot z14.s, z6.b, z1.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #-8, MUL VL]\n" - "udot z11.s, z7.b, z0.b[1]\n" - "udot z15.s, z7.b, z1.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #-7, MUL VL]\n" - "udot z8.s, z6.b, z0.b[2]\n" - "udot z12.s, z6.b, z1.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #-6, MUL VL]\n" - "udot z9.s, z7.b, z0.b[2]\n" - "udot z13.s, z7.b, z1.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #-5, MUL VL]\n" - "udot z10.s, z6.b, z0.b[2]\n" - "udot z14.s, z6.b, z1.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #-4, MUL VL]\n" - "udot z11.s, z7.b, z0.b[2]\n" - "udot z15.s, z7.b, z1.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n" - "udot z8.s, z6.b, z0.b[3]\n" - "udot z12.s, z6.b, z1.b[3]\n" - "ld1b { z6.b }, p5/Z, [x10, #-2, MUL VL]\n" - "udot z9.s, z7.b, z0.b[3]\n" - "udot z13.s, z7.b, z1.b[3]\n" - "ld1b { z7.b }, p5/Z, [x10, #-1, MUL VL]\n" - "udot z10.s, z6.b, z0.b[3]\n" - "udot z14.s, z6.b, z1.b[3]\n" - "udot z11.s, z7.b, z0.b[3]\n" - "udot z15.s, z7.b, z1.b[3]\n" + "udot z10.s, z17.b, z1.b[1]\n" + "udot z14.s, z17.b, z0.b[1]\n" + "ld1b { z17.b }, p5/Z, [x10, #-8, MUL VL]\n" + "udot z11.s, z16.b, z1.b[1]\n" + "udot z15.s, z16.b, z0.b[1]\n" + "ld1b { z16.b }, p5/Z, [x10, #-7, MUL VL]\n" + "udot z8.s, z17.b, z1.b[2]\n" + "udot z12.s, z17.b, z0.b[2]\n" + "ld1b { z17.b }, p5/Z, [x10, #-6, MUL VL]\n" + "udot z9.s, z16.b, z1.b[2]\n" + "udot z13.s, z16.b, z0.b[2]\n" + "ld1b { z16.b }, p5/Z, [x10, #-5, MUL VL]\n" + "udot z10.s, z17.b, z1.b[2]\n" + "udot z14.s, z17.b, z0.b[2]\n" + "ld1b { z17.b }, p5/Z, [x10, #-4, MUL VL]\n" + "udot z11.s, z16.b, z1.b[2]\n" + "udot z15.s, z16.b, z0.b[2]\n" + "ld1b { z16.b }, p5/Z, [x10, #-3, MUL VL]\n" + "udot z8.s, z17.b, z1.b[3]\n" + "udot z12.s, z17.b, z0.b[3]\n" + "ld1b { z17.b }, p5/Z, [x10, #-2, MUL VL]\n" + "udot z9.s, z16.b, z1.b[3]\n" + "udot z13.s, z16.b, z0.b[3]\n" + "ld1b { z16.b }, p5/Z, [x10, #-1, MUL VL]\n" + "udot z10.s, z17.b, z1.b[3]\n" + "udot z14.s, z17.b, z0.b[3]\n" + "udot z11.s, z16.b, z1.b[3]\n" + "udot z15.s, z16.b, z0.b[3]\n" "bgt 19b\n" "20:" // Height 2: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" "ld1rqb { z0.b }, p0/Z, [x26]\n" "ld1rqb { z1.b }, p0/Z, [x25]\n" "subs x27, x27, #0x4\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z8.s, z6.b, z0.b[0]\n" - "udot z12.s, z6.b, z1.b[0]\n" - "udot z9.s, z7.b, z0.b[0]\n" - "udot z13.s, z7.b, z1.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" - "udot z10.s, z6.b, z0.b[0]\n" - "udot z14.s, z6.b, z1.b[0]\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z8.s, z17.b, z0.b[0]\n" + "udot z12.s, z17.b, z1.b[0]\n" + "udot z9.s, z16.b, z0.b[0]\n" + "udot z13.s, z16.b, z1.b[0]\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" + "udot z10.s, z17.b, z0.b[0]\n" + "udot z14.s, z17.b, z1.b[0]\n" "addvl x10, x10, #4\n" - "udot z11.s, z7.b, z0.b[0]\n" - "udot z15.s, z7.b, z1.b[0]\n" + "udot z11.s, z16.b, z0.b[0]\n" + "udot z15.s, z16.b, z1.b[0]\n" "ble 21f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z8.s, z6.b, z0.b[1]\n" - "udot z12.s, z6.b, z1.b[1]\n" - "udot z9.s, z7.b, z0.b[1]\n" - "udot z13.s, z7.b, z1.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z8.s, z17.b, z0.b[1]\n" + "udot z12.s, z17.b, z1.b[1]\n" + "udot z9.s, z16.b, z0.b[1]\n" + "udot z13.s, z16.b, z1.b[1]\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x4\n" - "udot z10.s, z6.b, z0.b[1]\n" - "udot z14.s, z6.b, z1.b[1]\n" + "udot z10.s, z17.b, z0.b[1]\n" + "udot z14.s, z17.b, z1.b[1]\n" "addvl x10, x10, #4\n" - "udot z11.s, z7.b, z0.b[1]\n" - "udot z15.s, z7.b, z1.b[1]\n" + "udot z11.s, z16.b, z0.b[1]\n" + "udot z15.s, z16.b, z1.b[1]\n" "ble 21f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z8.s, z6.b, z0.b[2]\n" - "udot z12.s, z6.b, z1.b[2]\n" - "udot z9.s, z7.b, z0.b[2]\n" - "udot z13.s, z7.b, z1.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z8.s, z17.b, z0.b[2]\n" + "udot z12.s, z17.b, z1.b[2]\n" + "udot z9.s, z16.b, z0.b[2]\n" + "udot z13.s, z16.b, z1.b[2]\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" "subs x27, x27, #0x4\n" - "udot z10.s, z6.b, z0.b[2]\n" - "udot z14.s, z6.b, z1.b[2]\n" + "udot z10.s, z17.b, z0.b[2]\n" + "udot z14.s, z17.b, z1.b[2]\n" "addvl x10, x10, #4\n" - "udot z11.s, z7.b, z0.b[2]\n" - "udot z15.s, z7.b, z1.b[2]\n" + "udot z11.s, z16.b, z0.b[2]\n" + "udot z15.s, z16.b, z1.b[2]\n" "ble 21f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z8.s, z6.b, z0.b[3]\n" - "udot z12.s, z6.b, z1.b[3]\n" - "udot z9.s, z7.b, z0.b[3]\n" - "udot z13.s, z7.b, z1.b[3]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" - "udot z10.s, z6.b, z0.b[3]\n" - "udot z14.s, z6.b, z1.b[3]\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z8.s, z17.b, z0.b[3]\n" + "udot z12.s, z17.b, z1.b[3]\n" + "udot z9.s, z16.b, z0.b[3]\n" + "udot z13.s, z16.b, z1.b[3]\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" + "udot z10.s, z17.b, z0.b[3]\n" + "udot z14.s, z17.b, z1.b[3]\n" "addvl x10, x10, #4\n" - "udot z11.s, z7.b, z0.b[3]\n" - "udot z15.s, z7.b, z1.b[3]\n" + "udot z11.s, z16.b, z0.b[3]\n" + "udot z15.s, z16.b, z1.b[3]\n" "21:" // Height 2: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 16b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" + "add x20, x9, x20, LSL #2\n" "st1w { z8.s }, p4, [x9]\n" "st1w { z9.s }, p3, [x9, #1, MUL VL]\n" "st1w { z10.s }, p2, [x9, #2, MUL VL]\n" "st1w { z11.s }, p1, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "st1w { z12.s }, p4, [x24]\n" - "st1w { z13.s }, p3, [x24, #1, MUL VL]\n" - "st1w { z14.s }, p2, [x24, #2, MUL VL]\n" - "st1w { z15.s }, p1, [x24, #3, MUL VL]\n" + "st1w { z12.s }, p4, [x20]\n" + "st1w { z13.s }, p3, [x20, #1, MUL VL]\n" + "st1w { z14.s }, p2, [x20, #2, MUL VL]\n" + "st1w { z15.s }, p1, [x20, #3, MUL VL]\n" "22:" // Height 2: Writeback done "decw x11, ALL, MUL #4\n" "cmp x11, XZR\n" @@ -441,20 +441,20 @@ void sve_hybrid_u8u32_dot_6x4VL ( "whilelt p1.s, x20, x11\n" "tbz %x[flags], #0, 25f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x21, x9, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z8.s }, p4/Z, [x9]\n" "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x24]\n" - "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n" - "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z16.s }, p4/Z, [x23]\n" - "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n" - "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x21]\n" + "ld1w { z13.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x21, #2, MUL VL]\n" + "ld1w { z15.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x20]\n" + "ld1w { z17.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z19.s }, p1/Z, [x20, #3, MUL VL]\n" "b 26f\n" "25:" // Height 3: no accumulate "mov z8.s, #0x0\n" @@ -474,13 +474,13 @@ void sve_hybrid_u8u32_dot_6x4VL ( "27:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 28f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 29f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -489,86 +489,86 @@ void sve_hybrid_u8u32_dot_6x4VL ( "b 29f\n" "28:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" "29:" // Height 3: input setup done "cmp x27, #0x10\n" "ble 31f\n" "30:" // Height 3: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z0.b }, p0/Z, [x26]\n" + "ld1rqb { z2.b }, p0/Z, [x26]\n" "ld1rqb { z1.b }, p0/Z, [x25]\n" "sub x27, x27, #0x10\n" - "ld1rqb { z2.b }, p0/Z, [x24]\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "udot z8.s, z6.b, z0.b[0]\n" - "udot z12.s, z6.b, z1.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z16.s, z6.b, z2.b[0]\n" - "udot z9.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "udot z13.s, z7.b, z1.b[0]\n" - "udot z17.s, z7.b, z2.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1rqb { z0.b }, p0/Z, [x24]\n" + "ld1b { z21.b }, p5/Z, [x10]\n" + "udot z8.s, z21.b, z2.b[0]\n" + "udot z12.s, z21.b, z1.b[0]\n" + "ld1b { z20.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z16.s, z21.b, z0.b[0]\n" + "udot z9.s, z20.b, z2.b[0]\n" + "ld1b { z21.b }, p5/Z, [x10, #2, MUL VL]\n" + "udot z13.s, z20.b, z1.b[0]\n" + "udot z17.s, z20.b, z0.b[0]\n" + "ld1b { z20.b }, p5/Z, [x10, #3, MUL VL]\n" "cmp x27, #0x10\n" - "udot z10.s, z6.b, z0.b[0]\n" - "udot z14.s, z6.b, z1.b[0]\n" + "udot z10.s, z21.b, z2.b[0]\n" + "udot z14.s, z21.b, z1.b[0]\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" - "udot z18.s, z6.b, z2.b[0]\n" - "udot z11.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #4, MUL VL]\n" + "udot z18.s, z21.b, z0.b[0]\n" + "udot z11.s, z20.b, z2.b[0]\n" + "ld1b { z21.b }, p5/Z, [x10, #4, MUL VL]\n" "add x24, x24, #0x10\n" - "udot z15.s, z7.b, z1.b[0]\n" - "udot z19.s, z7.b, z2.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #5, MUL VL]\n" - "udot z8.s, z6.b, z0.b[1]\n" - "udot z12.s, z6.b, z1.b[1]\n" - "udot z16.s, z6.b, z2.b[1]\n" - "udot z9.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #6, MUL VL]\n" - "udot z13.s, z7.b, z1.b[1]\n" - "udot z17.s, z7.b, z2.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #7, MUL VL]\n" + "udot z15.s, z20.b, z1.b[0]\n" + "udot z19.s, z20.b, z0.b[0]\n" + "ld1b { z20.b }, p5/Z, [x10, #5, MUL VL]\n" + "udot z8.s, z21.b, z2.b[1]\n" + "udot z12.s, z21.b, z1.b[1]\n" + "udot z16.s, z21.b, z0.b[1]\n" + "udot z9.s, z20.b, z2.b[1]\n" + "ld1b { z21.b }, p5/Z, [x10, #6, MUL VL]\n" + "udot z13.s, z20.b, z1.b[1]\n" + "udot z17.s, z20.b, z0.b[1]\n" + "ld1b { z20.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "udot z10.s, z6.b, z0.b[1]\n" - "udot z14.s, z6.b, z1.b[1]\n" - "udot z18.s, z6.b, z2.b[1]\n" - "udot z11.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #-8, MUL VL]\n" - "udot z15.s, z7.b, z1.b[1]\n" - "udot z19.s, z7.b, z2.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #-7, MUL VL]\n" - "udot z8.s, z6.b, z0.b[2]\n" - "udot z12.s, z6.b, z1.b[2]\n" - "udot z16.s, z6.b, z2.b[2]\n" - "udot z9.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #-6, MUL VL]\n" - "udot z13.s, z7.b, z1.b[2]\n" - "udot z17.s, z7.b, z2.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #-5, MUL VL]\n" - "udot z10.s, z6.b, z0.b[2]\n" - "udot z14.s, z6.b, z1.b[2]\n" - "udot z18.s, z6.b, z2.b[2]\n" - "udot z11.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #-4, MUL VL]\n" - "udot z15.s, z7.b, z1.b[2]\n" - "udot z19.s, z7.b, z2.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n" - "udot z8.s, z6.b, z0.b[3]\n" - "udot z12.s, z6.b, z1.b[3]\n" - "udot z16.s, z6.b, z2.b[3]\n" - "udot z9.s, z7.b, z0.b[3]\n" - "ld1b { z6.b }, p5/Z, [x10, #-2, MUL VL]\n" - "udot z13.s, z7.b, z1.b[3]\n" - "udot z17.s, z7.b, z2.b[3]\n" - "ld1b { z7.b }, p5/Z, [x10, #-1, MUL VL]\n" - "udot z10.s, z6.b, z0.b[3]\n" - "udot z14.s, z6.b, z1.b[3]\n" - "udot z18.s, z6.b, z2.b[3]\n" - "udot z11.s, z7.b, z0.b[3]\n" - "udot z15.s, z7.b, z1.b[3]\n" - "udot z19.s, z7.b, z2.b[3]\n" + "udot z10.s, z21.b, z2.b[1]\n" + "udot z14.s, z21.b, z1.b[1]\n" + "udot z18.s, z21.b, z0.b[1]\n" + "udot z11.s, z20.b, z2.b[1]\n" + "ld1b { z21.b }, p5/Z, [x10, #-8, MUL VL]\n" + "udot z15.s, z20.b, z1.b[1]\n" + "udot z19.s, z20.b, z0.b[1]\n" + "ld1b { z20.b }, p5/Z, [x10, #-7, MUL VL]\n" + "udot z8.s, z21.b, z2.b[2]\n" + "udot z12.s, z21.b, z1.b[2]\n" + "udot z16.s, z21.b, z0.b[2]\n" + "udot z9.s, z20.b, z2.b[2]\n" + "ld1b { z21.b }, p5/Z, [x10, #-6, MUL VL]\n" + "udot z13.s, z20.b, z1.b[2]\n" + "udot z17.s, z20.b, z0.b[2]\n" + "ld1b { z20.b }, p5/Z, [x10, #-5, MUL VL]\n" + "udot z10.s, z21.b, z2.b[2]\n" + "udot z14.s, z21.b, z1.b[2]\n" + "udot z18.s, z21.b, z0.b[2]\n" + "udot z11.s, z20.b, z2.b[2]\n" + "ld1b { z21.b }, p5/Z, [x10, #-4, MUL VL]\n" + "udot z15.s, z20.b, z1.b[2]\n" + "udot z19.s, z20.b, z0.b[2]\n" + "ld1b { z20.b }, p5/Z, [x10, #-3, MUL VL]\n" + "udot z8.s, z21.b, z2.b[3]\n" + "udot z12.s, z21.b, z1.b[3]\n" + "udot z16.s, z21.b, z0.b[3]\n" + "udot z9.s, z20.b, z2.b[3]\n" + "ld1b { z21.b }, p5/Z, [x10, #-2, MUL VL]\n" + "udot z13.s, z20.b, z1.b[3]\n" + "udot z17.s, z20.b, z0.b[3]\n" + "ld1b { z20.b }, p5/Z, [x10, #-1, MUL VL]\n" + "udot z10.s, z21.b, z2.b[3]\n" + "udot z14.s, z21.b, z1.b[3]\n" + "udot z18.s, z21.b, z0.b[3]\n" + "udot z11.s, z20.b, z2.b[3]\n" + "udot z15.s, z20.b, z1.b[3]\n" + "udot z19.s, z20.b, z0.b[3]\n" "bgt 30b\n" "31:" // Height 3: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" @@ -576,100 +576,100 @@ void sve_hybrid_u8u32_dot_6x4VL ( "ld1rqb { z1.b }, p0/Z, [x25]\n" "subs x27, x27, #0x4\n" "ld1rqb { z2.b }, p0/Z, [x24]\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "udot z8.s, z6.b, z0.b[0]\n" - "udot z12.s, z6.b, z1.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z16.s, z6.b, z2.b[0]\n" - "udot z9.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "udot z13.s, z7.b, z1.b[0]\n" - "udot z17.s, z7.b, z2.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1b { z21.b }, p5/Z, [x10]\n" + "udot z8.s, z21.b, z0.b[0]\n" + "udot z12.s, z21.b, z1.b[0]\n" + "ld1b { z20.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z16.s, z21.b, z2.b[0]\n" + "udot z9.s, z20.b, z0.b[0]\n" + "ld1b { z21.b }, p5/Z, [x10, #2, MUL VL]\n" + "udot z13.s, z20.b, z1.b[0]\n" + "udot z17.s, z20.b, z2.b[0]\n" + "ld1b { z20.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "udot z10.s, z6.b, z0.b[0]\n" - "udot z14.s, z6.b, z1.b[0]\n" - "udot z18.s, z6.b, z2.b[0]\n" - "udot z11.s, z7.b, z0.b[0]\n" - "udot z15.s, z7.b, z1.b[0]\n" - "udot z19.s, z7.b, z2.b[0]\n" + "udot z10.s, z21.b, z0.b[0]\n" + "udot z14.s, z21.b, z1.b[0]\n" + "udot z18.s, z21.b, z2.b[0]\n" + "udot z11.s, z20.b, z0.b[0]\n" + "udot z15.s, z20.b, z1.b[0]\n" + "udot z19.s, z20.b, z2.b[0]\n" "ble 32f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z8.s, z6.b, z0.b[1]\n" - "udot z12.s, z6.b, z1.b[1]\n" - "udot z16.s, z6.b, z2.b[1]\n" - "udot z9.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z21.b }, p5/Z, [x10]\n" + "ld1b { z20.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z8.s, z21.b, z0.b[1]\n" + "udot z12.s, z21.b, z1.b[1]\n" + "udot z16.s, z21.b, z2.b[1]\n" + "udot z9.s, z20.b, z0.b[1]\n" + "ld1b { z21.b }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x4\n" - "udot z13.s, z7.b, z1.b[1]\n" - "udot z17.s, z7.b, z2.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "udot z13.s, z20.b, z1.b[1]\n" + "udot z17.s, z20.b, z2.b[1]\n" + "ld1b { z20.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "udot z10.s, z6.b, z0.b[1]\n" - "udot z14.s, z6.b, z1.b[1]\n" - "udot z18.s, z6.b, z2.b[1]\n" - "udot z11.s, z7.b, z0.b[1]\n" - "udot z15.s, z7.b, z1.b[1]\n" - "udot z19.s, z7.b, z2.b[1]\n" + "udot z10.s, z21.b, z0.b[1]\n" + "udot z14.s, z21.b, z1.b[1]\n" + "udot z18.s, z21.b, z2.b[1]\n" + "udot z11.s, z20.b, z0.b[1]\n" + "udot z15.s, z20.b, z1.b[1]\n" + "udot z19.s, z20.b, z2.b[1]\n" "ble 32f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z8.s, z6.b, z0.b[2]\n" - "udot z12.s, z6.b, z1.b[2]\n" - "udot z16.s, z6.b, z2.b[2]\n" - "udot z9.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z21.b }, p5/Z, [x10]\n" + "ld1b { z20.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z8.s, z21.b, z0.b[2]\n" + "udot z12.s, z21.b, z1.b[2]\n" + "udot z16.s, z21.b, z2.b[2]\n" + "udot z9.s, z20.b, z0.b[2]\n" + "ld1b { z21.b }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x4\n" - "udot z13.s, z7.b, z1.b[2]\n" - "udot z17.s, z7.b, z2.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "udot z13.s, z20.b, z1.b[2]\n" + "udot z17.s, z20.b, z2.b[2]\n" + "ld1b { z20.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "udot z10.s, z6.b, z0.b[2]\n" - "udot z14.s, z6.b, z1.b[2]\n" - "udot z18.s, z6.b, z2.b[2]\n" - "udot z11.s, z7.b, z0.b[2]\n" - "udot z15.s, z7.b, z1.b[2]\n" - "udot z19.s, z7.b, z2.b[2]\n" + "udot z10.s, z21.b, z0.b[2]\n" + "udot z14.s, z21.b, z1.b[2]\n" + "udot z18.s, z21.b, z2.b[2]\n" + "udot z11.s, z20.b, z0.b[2]\n" + "udot z15.s, z20.b, z1.b[2]\n" + "udot z19.s, z20.b, z2.b[2]\n" "ble 32f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z8.s, z6.b, z0.b[3]\n" - "udot z12.s, z6.b, z1.b[3]\n" - "udot z16.s, z6.b, z2.b[3]\n" - "udot z9.s, z7.b, z0.b[3]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "udot z13.s, z7.b, z1.b[3]\n" - "udot z17.s, z7.b, z2.b[3]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1b { z21.b }, p5/Z, [x10]\n" + "ld1b { z20.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z8.s, z21.b, z0.b[3]\n" + "udot z12.s, z21.b, z1.b[3]\n" + "udot z16.s, z21.b, z2.b[3]\n" + "udot z9.s, z20.b, z0.b[3]\n" + "ld1b { z21.b }, p5/Z, [x10, #2, MUL VL]\n" + "udot z13.s, z20.b, z1.b[3]\n" + "udot z17.s, z20.b, z2.b[3]\n" + "ld1b { z20.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "udot z10.s, z6.b, z0.b[3]\n" - "udot z14.s, z6.b, z1.b[3]\n" - "udot z18.s, z6.b, z2.b[3]\n" - "udot z11.s, z7.b, z0.b[3]\n" - "udot z15.s, z7.b, z1.b[3]\n" - "udot z19.s, z7.b, z2.b[3]\n" + "udot z10.s, z21.b, z0.b[3]\n" + "udot z14.s, z21.b, z1.b[3]\n" + "udot z18.s, z21.b, z2.b[3]\n" + "udot z11.s, z20.b, z0.b[3]\n" + "udot z15.s, z20.b, z1.b[3]\n" + "udot z19.s, z20.b, z2.b[3]\n" "32:" // Height 3: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 27b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x21, x9, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "st1w { z8.s }, p4, [x9]\n" "st1w { z9.s }, p3, [x9, #1, MUL VL]\n" "st1w { z10.s }, p2, [x9, #2, MUL VL]\n" "st1w { z11.s }, p1, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "st1w { z12.s }, p4, [x24]\n" - "st1w { z13.s }, p3, [x24, #1, MUL VL]\n" - "st1w { z14.s }, p2, [x24, #2, MUL VL]\n" - "st1w { z15.s }, p1, [x24, #3, MUL VL]\n" - "st1w { z16.s }, p4, [x23]\n" - "st1w { z17.s }, p3, [x23, #1, MUL VL]\n" - "st1w { z18.s }, p2, [x23, #2, MUL VL]\n" - "st1w { z19.s }, p1, [x23, #3, MUL VL]\n" + "st1w { z12.s }, p4, [x21]\n" + "st1w { z13.s }, p3, [x21, #1, MUL VL]\n" + "st1w { z14.s }, p2, [x21, #2, MUL VL]\n" + "st1w { z15.s }, p1, [x21, #3, MUL VL]\n" + "st1w { z16.s }, p4, [x20]\n" + "st1w { z17.s }, p3, [x20, #1, MUL VL]\n" + "st1w { z18.s }, p2, [x20, #2, MUL VL]\n" + "st1w { z19.s }, p1, [x20, #3, MUL VL]\n" "33:" // Height 3: Writeback done "decw x11, ALL, MUL #4\n" "cmp x11, XZR\n" @@ -690,25 +690,25 @@ void sve_hybrid_u8u32_dot_6x4VL ( "whilelt p1.s, x20, x11\n" "tbz %x[flags], #0, 36f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x22, x9, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" "ld1w { z8.s }, p4/Z, [x9]\n" - "add x22, x23, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x24]\n" - "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n" - "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z16.s }, p4/Z, [x23]\n" - "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n" - "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x22]\n" - "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n" - "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x22]\n" + "ld1w { z13.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x22, #2, MUL VL]\n" + "ld1w { z15.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x21]\n" + "ld1w { z17.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x21, #2, MUL VL]\n" + "ld1w { z19.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x20]\n" + "ld1w { z21.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z23.s }, p1/Z, [x20, #3, MUL VL]\n" "b 37f\n" "36:" // Height 4: no accumulate "mov z8.s, #0x0\n" @@ -732,14 +732,14 @@ void sve_hybrid_u8u32_dot_6x4VL ( "38:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 39f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 40f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -749,105 +749,105 @@ void sve_hybrid_u8u32_dot_6x4VL ( "b 40f\n" "39:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" "40:" // Height 4: input setup done "cmp x27, #0x10\n" "ble 42f\n" "41:" // Height 4: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z0.b }, p0/Z, [x26]\n" - "ld1rqb { z1.b }, p0/Z, [x25]\n" + "ld1rqb { z3.b }, p0/Z, [x26]\n" + "ld1rqb { z2.b }, p0/Z, [x25]\n" "sub x27, x27, #0x10\n" - "ld1rqb { z2.b }, p0/Z, [x24]\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" + "ld1rqb { z1.b }, p0/Z, [x24]\n" + "ld1rqb { z0.b }, p0/Z, [x23]\n" "cmp x27, #0x10\n" "add x26, x26, #0x10\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z8.s, z6.b, z0.b[0]\n" - "udot z12.s, z6.b, z1.b[0]\n" - "udot z16.s, z6.b, z2.b[0]\n" - "udot z20.s, z6.b, z3.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z25.b }, p5/Z, [x10]\n" + "ld1b { z24.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z8.s, z25.b, z3.b[0]\n" + "udot z12.s, z25.b, z2.b[0]\n" + "udot z16.s, z25.b, z1.b[0]\n" + "udot z20.s, z25.b, z0.b[0]\n" + "ld1b { z25.b }, p5/Z, [x10, #2, MUL VL]\n" "add x25, x25, #0x10\n" - "udot z9.s, z7.b, z0.b[0]\n" - "udot z13.s, z7.b, z1.b[0]\n" + "udot z9.s, z24.b, z3.b[0]\n" + "udot z13.s, z24.b, z2.b[0]\n" "add x24, x24, #0x10\n" "add x23, x23, #0x10\n" - "udot z17.s, z7.b, z2.b[0]\n" - "udot z21.s, z7.b, z3.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" - "udot z10.s, z6.b, z0.b[0]\n" - "udot z14.s, z6.b, z1.b[0]\n" - "udot z18.s, z6.b, z2.b[0]\n" - "udot z22.s, z6.b, z3.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #4, MUL VL]\n" - "udot z11.s, z7.b, z0.b[0]\n" - "udot z15.s, z7.b, z1.b[0]\n" - "udot z19.s, z7.b, z2.b[0]\n" - "udot z23.s, z7.b, z3.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #5, MUL VL]\n" - "udot z8.s, z6.b, z0.b[1]\n" - "udot z12.s, z6.b, z1.b[1]\n" - "udot z16.s, z6.b, z2.b[1]\n" - "udot z20.s, z6.b, z3.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #6, MUL VL]\n" - "udot z9.s, z7.b, z0.b[1]\n" - "udot z13.s, z7.b, z1.b[1]\n" - "udot z17.s, z7.b, z2.b[1]\n" - "udot z21.s, z7.b, z3.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #7, MUL VL]\n" + "udot z17.s, z24.b, z1.b[0]\n" + "udot z21.s, z24.b, z0.b[0]\n" + "ld1b { z24.b }, p5/Z, [x10, #3, MUL VL]\n" + "udot z10.s, z25.b, z3.b[0]\n" + "udot z14.s, z25.b, z2.b[0]\n" + "udot z18.s, z25.b, z1.b[0]\n" + "udot z22.s, z25.b, z0.b[0]\n" + "ld1b { z25.b }, p5/Z, [x10, #4, MUL VL]\n" + "udot z11.s, z24.b, z3.b[0]\n" + "udot z15.s, z24.b, z2.b[0]\n" + "udot z19.s, z24.b, z1.b[0]\n" + "udot z23.s, z24.b, z0.b[0]\n" + "ld1b { z24.b }, p5/Z, [x10, #5, MUL VL]\n" + "udot z8.s, z25.b, z3.b[1]\n" + "udot z12.s, z25.b, z2.b[1]\n" + "udot z16.s, z25.b, z1.b[1]\n" + "udot z20.s, z25.b, z0.b[1]\n" + "ld1b { z25.b }, p5/Z, [x10, #6, MUL VL]\n" + "udot z9.s, z24.b, z3.b[1]\n" + "udot z13.s, z24.b, z2.b[1]\n" + "udot z17.s, z24.b, z1.b[1]\n" + "udot z21.s, z24.b, z0.b[1]\n" + "ld1b { z24.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "udot z10.s, z6.b, z0.b[1]\n" - "udot z14.s, z6.b, z1.b[1]\n" - "udot z18.s, z6.b, z2.b[1]\n" - "udot z22.s, z6.b, z3.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #-8, MUL VL]\n" - "udot z11.s, z7.b, z0.b[1]\n" - "udot z15.s, z7.b, z1.b[1]\n" - "udot z19.s, z7.b, z2.b[1]\n" - "udot z23.s, z7.b, z3.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #-7, MUL VL]\n" - "udot z8.s, z6.b, z0.b[2]\n" - "udot z12.s, z6.b, z1.b[2]\n" - "udot z16.s, z6.b, z2.b[2]\n" - "udot z20.s, z6.b, z3.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #-6, MUL VL]\n" - "udot z9.s, z7.b, z0.b[2]\n" - "udot z13.s, z7.b, z1.b[2]\n" - "udot z17.s, z7.b, z2.b[2]\n" - "udot z21.s, z7.b, z3.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #-5, MUL VL]\n" - "udot z10.s, z6.b, z0.b[2]\n" - "udot z14.s, z6.b, z1.b[2]\n" - "udot z18.s, z6.b, z2.b[2]\n" - "udot z22.s, z6.b, z3.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #-4, MUL VL]\n" - "udot z11.s, z7.b, z0.b[2]\n" - "udot z15.s, z7.b, z1.b[2]\n" - "udot z19.s, z7.b, z2.b[2]\n" - "udot z23.s, z7.b, z3.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n" - "udot z8.s, z6.b, z0.b[3]\n" - "udot z12.s, z6.b, z1.b[3]\n" - "udot z16.s, z6.b, z2.b[3]\n" - "udot z20.s, z6.b, z3.b[3]\n" - "ld1b { z6.b }, p5/Z, [x10, #-2, MUL VL]\n" - "udot z9.s, z7.b, z0.b[3]\n" - "udot z13.s, z7.b, z1.b[3]\n" - "udot z17.s, z7.b, z2.b[3]\n" - "udot z21.s, z7.b, z3.b[3]\n" - "ld1b { z7.b }, p5/Z, [x10, #-1, MUL VL]\n" - "udot z10.s, z6.b, z0.b[3]\n" - "udot z14.s, z6.b, z1.b[3]\n" - "udot z18.s, z6.b, z2.b[3]\n" - "udot z22.s, z6.b, z3.b[3]\n" - "udot z11.s, z7.b, z0.b[3]\n" - "udot z15.s, z7.b, z1.b[3]\n" - "udot z19.s, z7.b, z2.b[3]\n" - "udot z23.s, z7.b, z3.b[3]\n" + "udot z10.s, z25.b, z3.b[1]\n" + "udot z14.s, z25.b, z2.b[1]\n" + "udot z18.s, z25.b, z1.b[1]\n" + "udot z22.s, z25.b, z0.b[1]\n" + "ld1b { z25.b }, p5/Z, [x10, #-8, MUL VL]\n" + "udot z11.s, z24.b, z3.b[1]\n" + "udot z15.s, z24.b, z2.b[1]\n" + "udot z19.s, z24.b, z1.b[1]\n" + "udot z23.s, z24.b, z0.b[1]\n" + "ld1b { z24.b }, p5/Z, [x10, #-7, MUL VL]\n" + "udot z8.s, z25.b, z3.b[2]\n" + "udot z12.s, z25.b, z2.b[2]\n" + "udot z16.s, z25.b, z1.b[2]\n" + "udot z20.s, z25.b, z0.b[2]\n" + "ld1b { z25.b }, p5/Z, [x10, #-6, MUL VL]\n" + "udot z9.s, z24.b, z3.b[2]\n" + "udot z13.s, z24.b, z2.b[2]\n" + "udot z17.s, z24.b, z1.b[2]\n" + "udot z21.s, z24.b, z0.b[2]\n" + "ld1b { z24.b }, p5/Z, [x10, #-5, MUL VL]\n" + "udot z10.s, z25.b, z3.b[2]\n" + "udot z14.s, z25.b, z2.b[2]\n" + "udot z18.s, z25.b, z1.b[2]\n" + "udot z22.s, z25.b, z0.b[2]\n" + "ld1b { z25.b }, p5/Z, [x10, #-4, MUL VL]\n" + "udot z11.s, z24.b, z3.b[2]\n" + "udot z15.s, z24.b, z2.b[2]\n" + "udot z19.s, z24.b, z1.b[2]\n" + "udot z23.s, z24.b, z0.b[2]\n" + "ld1b { z24.b }, p5/Z, [x10, #-3, MUL VL]\n" + "udot z8.s, z25.b, z3.b[3]\n" + "udot z12.s, z25.b, z2.b[3]\n" + "udot z16.s, z25.b, z1.b[3]\n" + "udot z20.s, z25.b, z0.b[3]\n" + "ld1b { z25.b }, p5/Z, [x10, #-2, MUL VL]\n" + "udot z9.s, z24.b, z3.b[3]\n" + "udot z13.s, z24.b, z2.b[3]\n" + "udot z17.s, z24.b, z1.b[3]\n" + "udot z21.s, z24.b, z0.b[3]\n" + "ld1b { z24.b }, p5/Z, [x10, #-1, MUL VL]\n" + "udot z10.s, z25.b, z3.b[3]\n" + "udot z14.s, z25.b, z2.b[3]\n" + "udot z18.s, z25.b, z1.b[3]\n" + "udot z22.s, z25.b, z0.b[3]\n" + "udot z11.s, z24.b, z3.b[3]\n" + "udot z15.s, z24.b, z2.b[3]\n" + "udot z19.s, z24.b, z1.b[3]\n" + "udot z23.s, z24.b, z0.b[3]\n" "bgt 41b\n" "42:" // Height 4: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" @@ -856,121 +856,121 @@ void sve_hybrid_u8u32_dot_6x4VL ( "subs x27, x27, #0x4\n" "ld1rqb { z2.b }, p0/Z, [x24]\n" "ld1rqb { z3.b }, p0/Z, [x23]\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z8.s, z6.b, z0.b[0]\n" - "udot z12.s, z6.b, z1.b[0]\n" - "udot z16.s, z6.b, z2.b[0]\n" - "udot z20.s, z6.b, z3.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "udot z9.s, z7.b, z0.b[0]\n" - "udot z13.s, z7.b, z1.b[0]\n" - "udot z17.s, z7.b, z2.b[0]\n" - "udot z21.s, z7.b, z3.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1b { z25.b }, p5/Z, [x10]\n" + "ld1b { z24.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z8.s, z25.b, z0.b[0]\n" + "udot z12.s, z25.b, z1.b[0]\n" + "udot z16.s, z25.b, z2.b[0]\n" + "udot z20.s, z25.b, z3.b[0]\n" + "ld1b { z25.b }, p5/Z, [x10, #2, MUL VL]\n" + "udot z9.s, z24.b, z0.b[0]\n" + "udot z13.s, z24.b, z1.b[0]\n" + "udot z17.s, z24.b, z2.b[0]\n" + "udot z21.s, z24.b, z3.b[0]\n" + "ld1b { z24.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "udot z10.s, z6.b, z0.b[0]\n" - "udot z14.s, z6.b, z1.b[0]\n" - "udot z18.s, z6.b, z2.b[0]\n" - "udot z22.s, z6.b, z3.b[0]\n" - "udot z11.s, z7.b, z0.b[0]\n" - "udot z15.s, z7.b, z1.b[0]\n" - "udot z19.s, z7.b, z2.b[0]\n" - "udot z23.s, z7.b, z3.b[0]\n" + "udot z10.s, z25.b, z0.b[0]\n" + "udot z14.s, z25.b, z1.b[0]\n" + "udot z18.s, z25.b, z2.b[0]\n" + "udot z22.s, z25.b, z3.b[0]\n" + "udot z11.s, z24.b, z0.b[0]\n" + "udot z15.s, z24.b, z1.b[0]\n" + "udot z19.s, z24.b, z2.b[0]\n" + "udot z23.s, z24.b, z3.b[0]\n" "ble 43f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z8.s, z6.b, z0.b[1]\n" - "udot z12.s, z6.b, z1.b[1]\n" - "udot z16.s, z6.b, z2.b[1]\n" - "udot z20.s, z6.b, z3.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z25.b }, p5/Z, [x10]\n" + "ld1b { z24.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z8.s, z25.b, z0.b[1]\n" + "udot z12.s, z25.b, z1.b[1]\n" + "udot z16.s, z25.b, z2.b[1]\n" + "udot z20.s, z25.b, z3.b[1]\n" + "ld1b { z25.b }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x4\n" - "udot z9.s, z7.b, z0.b[1]\n" - "udot z13.s, z7.b, z1.b[1]\n" - "udot z17.s, z7.b, z2.b[1]\n" - "udot z21.s, z7.b, z3.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "udot z9.s, z24.b, z0.b[1]\n" + "udot z13.s, z24.b, z1.b[1]\n" + "udot z17.s, z24.b, z2.b[1]\n" + "udot z21.s, z24.b, z3.b[1]\n" + "ld1b { z24.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "udot z10.s, z6.b, z0.b[1]\n" - "udot z14.s, z6.b, z1.b[1]\n" - "udot z18.s, z6.b, z2.b[1]\n" - "udot z22.s, z6.b, z3.b[1]\n" - "udot z11.s, z7.b, z0.b[1]\n" - "udot z15.s, z7.b, z1.b[1]\n" - "udot z19.s, z7.b, z2.b[1]\n" - "udot z23.s, z7.b, z3.b[1]\n" + "udot z10.s, z25.b, z0.b[1]\n" + "udot z14.s, z25.b, z1.b[1]\n" + "udot z18.s, z25.b, z2.b[1]\n" + "udot z22.s, z25.b, z3.b[1]\n" + "udot z11.s, z24.b, z0.b[1]\n" + "udot z15.s, z24.b, z1.b[1]\n" + "udot z19.s, z24.b, z2.b[1]\n" + "udot z23.s, z24.b, z3.b[1]\n" "ble 43f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z8.s, z6.b, z0.b[2]\n" - "udot z12.s, z6.b, z1.b[2]\n" - "udot z16.s, z6.b, z2.b[2]\n" - "udot z20.s, z6.b, z3.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z25.b }, p5/Z, [x10]\n" + "ld1b { z24.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z8.s, z25.b, z0.b[2]\n" + "udot z12.s, z25.b, z1.b[2]\n" + "udot z16.s, z25.b, z2.b[2]\n" + "udot z20.s, z25.b, z3.b[2]\n" + "ld1b { z25.b }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x4\n" - "udot z9.s, z7.b, z0.b[2]\n" - "udot z13.s, z7.b, z1.b[2]\n" - "udot z17.s, z7.b, z2.b[2]\n" - "udot z21.s, z7.b, z3.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "udot z9.s, z24.b, z0.b[2]\n" + "udot z13.s, z24.b, z1.b[2]\n" + "udot z17.s, z24.b, z2.b[2]\n" + "udot z21.s, z24.b, z3.b[2]\n" + "ld1b { z24.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "udot z10.s, z6.b, z0.b[2]\n" - "udot z14.s, z6.b, z1.b[2]\n" - "udot z18.s, z6.b, z2.b[2]\n" - "udot z22.s, z6.b, z3.b[2]\n" - "udot z11.s, z7.b, z0.b[2]\n" - "udot z15.s, z7.b, z1.b[2]\n" - "udot z19.s, z7.b, z2.b[2]\n" - "udot z23.s, z7.b, z3.b[2]\n" + "udot z10.s, z25.b, z0.b[2]\n" + "udot z14.s, z25.b, z1.b[2]\n" + "udot z18.s, z25.b, z2.b[2]\n" + "udot z22.s, z25.b, z3.b[2]\n" + "udot z11.s, z24.b, z0.b[2]\n" + "udot z15.s, z24.b, z1.b[2]\n" + "udot z19.s, z24.b, z2.b[2]\n" + "udot z23.s, z24.b, z3.b[2]\n" "ble 43f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z8.s, z6.b, z0.b[3]\n" - "udot z12.s, z6.b, z1.b[3]\n" - "udot z16.s, z6.b, z2.b[3]\n" - "udot z20.s, z6.b, z3.b[3]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "udot z9.s, z7.b, z0.b[3]\n" - "udot z13.s, z7.b, z1.b[3]\n" - "udot z17.s, z7.b, z2.b[3]\n" - "udot z21.s, z7.b, z3.b[3]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1b { z25.b }, p5/Z, [x10]\n" + "ld1b { z24.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z8.s, z25.b, z0.b[3]\n" + "udot z12.s, z25.b, z1.b[3]\n" + "udot z16.s, z25.b, z2.b[3]\n" + "udot z20.s, z25.b, z3.b[3]\n" + "ld1b { z25.b }, p5/Z, [x10, #2, MUL VL]\n" + "udot z9.s, z24.b, z0.b[3]\n" + "udot z13.s, z24.b, z1.b[3]\n" + "udot z17.s, z24.b, z2.b[3]\n" + "udot z21.s, z24.b, z3.b[3]\n" + "ld1b { z24.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "udot z10.s, z6.b, z0.b[3]\n" - "udot z14.s, z6.b, z1.b[3]\n" - "udot z18.s, z6.b, z2.b[3]\n" - "udot z22.s, z6.b, z3.b[3]\n" - "udot z11.s, z7.b, z0.b[3]\n" - "udot z15.s, z7.b, z1.b[3]\n" - "udot z19.s, z7.b, z2.b[3]\n" - "udot z23.s, z7.b, z3.b[3]\n" + "udot z10.s, z25.b, z0.b[3]\n" + "udot z14.s, z25.b, z1.b[3]\n" + "udot z18.s, z25.b, z2.b[3]\n" + "udot z22.s, z25.b, z3.b[3]\n" + "udot z11.s, z24.b, z0.b[3]\n" + "udot z15.s, z24.b, z1.b[3]\n" + "udot z19.s, z24.b, z2.b[3]\n" + "udot z23.s, z24.b, z3.b[3]\n" "43:" // Height 4: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 38b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" + "add x22, x9, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" "st1w { z8.s }, p4, [x9]\n" - "add x22, x23, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "st1w { z9.s }, p3, [x9, #1, MUL VL]\n" "st1w { z10.s }, p2, [x9, #2, MUL VL]\n" "st1w { z11.s }, p1, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "st1w { z12.s }, p4, [x24]\n" - "st1w { z13.s }, p3, [x24, #1, MUL VL]\n" - "st1w { z14.s }, p2, [x24, #2, MUL VL]\n" - "st1w { z15.s }, p1, [x24, #3, MUL VL]\n" - "st1w { z16.s }, p4, [x23]\n" - "st1w { z17.s }, p3, [x23, #1, MUL VL]\n" - "st1w { z18.s }, p2, [x23, #2, MUL VL]\n" - "st1w { z19.s }, p1, [x23, #3, MUL VL]\n" - "st1w { z20.s }, p4, [x22]\n" - "st1w { z21.s }, p3, [x22, #1, MUL VL]\n" - "st1w { z22.s }, p2, [x22, #2, MUL VL]\n" - "st1w { z23.s }, p1, [x22, #3, MUL VL]\n" + "st1w { z12.s }, p4, [x22]\n" + "st1w { z13.s }, p3, [x22, #1, MUL VL]\n" + "st1w { z14.s }, p2, [x22, #2, MUL VL]\n" + "st1w { z15.s }, p1, [x22, #3, MUL VL]\n" + "st1w { z16.s }, p4, [x21]\n" + "st1w { z17.s }, p3, [x21, #1, MUL VL]\n" + "st1w { z18.s }, p2, [x21, #2, MUL VL]\n" + "st1w { z19.s }, p1, [x21, #3, MUL VL]\n" + "st1w { z20.s }, p4, [x20]\n" + "st1w { z21.s }, p3, [x20, #1, MUL VL]\n" + "st1w { z22.s }, p2, [x20, #2, MUL VL]\n" + "st1w { z23.s }, p1, [x20, #3, MUL VL]\n" "44:" // Height 4: Writeback done "decw x11, ALL, MUL #4\n" "cmp x11, XZR\n" @@ -991,30 +991,30 @@ void sve_hybrid_u8u32_dot_6x4VL ( "whilelt p1.s, x20, x11\n" "tbz %x[flags], #0, 47f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "ld1w { z8.s }, p4/Z, [x9]\n" + "add x23, x9, x20, LSL #2\n" "add x22, x23, x20, LSL #2\n" + "ld1w { z8.s }, p4/Z, [x9]\n" "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "ld1w { z9.s }, p3/Z, [x9, #1, MUL VL]\n" "ld1w { z10.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z11.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x24]\n" - "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n" - "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z16.s }, p4/Z, [x23]\n" - "ld1w { z17.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z18.s }, p2/Z, [x23, #2, MUL VL]\n" - "ld1w { z19.s }, p1/Z, [x23, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x22]\n" - "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n" - "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n" - "ld1w { z24.s }, p4/Z, [x21]\n" - "ld1w { z25.s }, p3/Z, [x21, #1, MUL VL]\n" - "ld1w { z26.s }, p2/Z, [x21, #2, MUL VL]\n" - "ld1w { z27.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x23]\n" + "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n" + "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z16.s }, p4/Z, [x22]\n" + "ld1w { z17.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x22, #2, MUL VL]\n" + "ld1w { z19.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x21]\n" + "ld1w { z21.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x21, #2, MUL VL]\n" + "ld1w { z23.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z24.s }, p4/Z, [x20]\n" + "ld1w { z25.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z26.s }, p2/Z, [x20, #2, MUL VL]\n" + "ld1w { z27.s }, p1/Z, [x20, #3, MUL VL]\n" "b 48f\n" "47:" // Height 5: no accumulate "mov z8.s, #0x0\n" @@ -1042,15 +1042,15 @@ void sve_hybrid_u8u32_dot_6x4VL ( "49:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 50f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 51f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -1061,124 +1061,124 @@ void sve_hybrid_u8u32_dot_6x4VL ( "b 51f\n" "50:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" "51:" // Height 5: input setup done "cmp x27, #0x10\n" "ble 53f\n" "52:" // Height 5: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z0.b }, p0/Z, [x26]\n" - "ld1rqb { z1.b }, p0/Z, [x25]\n" + "ld1rqb { z4.b }, p0/Z, [x26]\n" + "ld1rqb { z3.b }, p0/Z, [x25]\n" "sub x27, x27, #0x10\n" "ld1rqb { z2.b }, p0/Z, [x24]\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" + "ld1rqb { z1.b }, p0/Z, [x23]\n" "cmp x27, #0x10\n" "add x26, x26, #0x10\n" - "ld1rqb { z4.b }, p0/Z, [x22]\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "udot z8.s, z6.b, z0.b[0]\n" - "udot z12.s, z6.b, z1.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z16.s, z6.b, z2.b[0]\n" - "udot z20.s, z6.b, z3.b[0]\n" + "ld1rqb { z0.b }, p0/Z, [x22]\n" + "ld1b { z29.b }, p5/Z, [x10]\n" + "udot z8.s, z29.b, z4.b[0]\n" + "udot z12.s, z29.b, z3.b[0]\n" + "ld1b { z28.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z16.s, z29.b, z2.b[0]\n" + "udot z20.s, z29.b, z1.b[0]\n" "add x25, x25, #0x10\n" - "udot z24.s, z6.b, z4.b[0]\n" - "udot z9.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" + "udot z24.s, z29.b, z0.b[0]\n" + "udot z9.s, z28.b, z4.b[0]\n" + "ld1b { z29.b }, p5/Z, [x10, #2, MUL VL]\n" "add x24, x24, #0x10\n" - "udot z13.s, z7.b, z1.b[0]\n" - "udot z17.s, z7.b, z2.b[0]\n" + "udot z13.s, z28.b, z3.b[0]\n" + "udot z17.s, z28.b, z2.b[0]\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - "udot z21.s, z7.b, z3.b[0]\n" - "udot z25.s, z7.b, z4.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" - "udot z10.s, z6.b, z0.b[0]\n" - "udot z14.s, z6.b, z1.b[0]\n" - "udot z18.s, z6.b, z2.b[0]\n" - "udot z22.s, z6.b, z3.b[0]\n" - "udot z26.s, z6.b, z4.b[0]\n" - "udot z11.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #4, MUL VL]\n" - "udot z15.s, z7.b, z1.b[0]\n" - "udot z19.s, z7.b, z2.b[0]\n" - "udot z23.s, z7.b, z3.b[0]\n" - "udot z27.s, z7.b, z4.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #5, MUL VL]\n" - "udot z8.s, z6.b, z0.b[1]\n" - "udot z12.s, z6.b, z1.b[1]\n" - "udot z16.s, z6.b, z2.b[1]\n" - "udot z20.s, z6.b, z3.b[1]\n" - "udot z24.s, z6.b, z4.b[1]\n" - "udot z9.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #6, MUL VL]\n" - "udot z13.s, z7.b, z1.b[1]\n" - "udot z17.s, z7.b, z2.b[1]\n" - "udot z21.s, z7.b, z3.b[1]\n" - "udot z25.s, z7.b, z4.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #7, MUL VL]\n" + "udot z21.s, z28.b, z1.b[0]\n" + "udot z25.s, z28.b, z0.b[0]\n" + "ld1b { z28.b }, p5/Z, [x10, #3, MUL VL]\n" + "udot z10.s, z29.b, z4.b[0]\n" + "udot z14.s, z29.b, z3.b[0]\n" + "udot z18.s, z29.b, z2.b[0]\n" + "udot z22.s, z29.b, z1.b[0]\n" + "udot z26.s, z29.b, z0.b[0]\n" + "udot z11.s, z28.b, z4.b[0]\n" + "ld1b { z29.b }, p5/Z, [x10, #4, MUL VL]\n" + "udot z15.s, z28.b, z3.b[0]\n" + "udot z19.s, z28.b, z2.b[0]\n" + "udot z23.s, z28.b, z1.b[0]\n" + "udot z27.s, z28.b, z0.b[0]\n" + "ld1b { z28.b }, p5/Z, [x10, #5, MUL VL]\n" + "udot z8.s, z29.b, z4.b[1]\n" + "udot z12.s, z29.b, z3.b[1]\n" + "udot z16.s, z29.b, z2.b[1]\n" + "udot z20.s, z29.b, z1.b[1]\n" + "udot z24.s, z29.b, z0.b[1]\n" + "udot z9.s, z28.b, z4.b[1]\n" + "ld1b { z29.b }, p5/Z, [x10, #6, MUL VL]\n" + "udot z13.s, z28.b, z3.b[1]\n" + "udot z17.s, z28.b, z2.b[1]\n" + "udot z21.s, z28.b, z1.b[1]\n" + "udot z25.s, z28.b, z0.b[1]\n" + "ld1b { z28.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "udot z10.s, z6.b, z0.b[1]\n" - "udot z14.s, z6.b, z1.b[1]\n" - "udot z18.s, z6.b, z2.b[1]\n" - "udot z22.s, z6.b, z3.b[1]\n" - "udot z26.s, z6.b, z4.b[1]\n" - "udot z11.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #-8, MUL VL]\n" - "udot z15.s, z7.b, z1.b[1]\n" - "udot z19.s, z7.b, z2.b[1]\n" - "udot z23.s, z7.b, z3.b[1]\n" - "udot z27.s, z7.b, z4.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #-7, MUL VL]\n" - "udot z8.s, z6.b, z0.b[2]\n" - "udot z12.s, z6.b, z1.b[2]\n" - "udot z16.s, z6.b, z2.b[2]\n" - "udot z20.s, z6.b, z3.b[2]\n" - "udot z24.s, z6.b, z4.b[2]\n" - "udot z9.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #-6, MUL VL]\n" - "udot z13.s, z7.b, z1.b[2]\n" - "udot z17.s, z7.b, z2.b[2]\n" - "udot z21.s, z7.b, z3.b[2]\n" - "udot z25.s, z7.b, z4.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #-5, MUL VL]\n" - "udot z10.s, z6.b, z0.b[2]\n" - "udot z14.s, z6.b, z1.b[2]\n" - "udot z18.s, z6.b, z2.b[2]\n" - "udot z22.s, z6.b, z3.b[2]\n" - "udot z26.s, z6.b, z4.b[2]\n" - "udot z11.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #-4, MUL VL]\n" - "udot z15.s, z7.b, z1.b[2]\n" - "udot z19.s, z7.b, z2.b[2]\n" - "udot z23.s, z7.b, z3.b[2]\n" - "udot z27.s, z7.b, z4.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n" - "udot z8.s, z6.b, z0.b[3]\n" - "udot z12.s, z6.b, z1.b[3]\n" - "udot z16.s, z6.b, z2.b[3]\n" - "udot z20.s, z6.b, z3.b[3]\n" - "udot z24.s, z6.b, z4.b[3]\n" - "udot z9.s, z7.b, z0.b[3]\n" - "ld1b { z6.b }, p5/Z, [x10, #-2, MUL VL]\n" - "udot z13.s, z7.b, z1.b[3]\n" - "udot z17.s, z7.b, z2.b[3]\n" - "udot z21.s, z7.b, z3.b[3]\n" - "udot z25.s, z7.b, z4.b[3]\n" - "ld1b { z7.b }, p5/Z, [x10, #-1, MUL VL]\n" - "udot z10.s, z6.b, z0.b[3]\n" - "udot z14.s, z6.b, z1.b[3]\n" - "udot z18.s, z6.b, z2.b[3]\n" - "udot z22.s, z6.b, z3.b[3]\n" - "udot z26.s, z6.b, z4.b[3]\n" - "udot z11.s, z7.b, z0.b[3]\n" - "udot z15.s, z7.b, z1.b[3]\n" - "udot z19.s, z7.b, z2.b[3]\n" - "udot z23.s, z7.b, z3.b[3]\n" - "udot z27.s, z7.b, z4.b[3]\n" + "udot z10.s, z29.b, z4.b[1]\n" + "udot z14.s, z29.b, z3.b[1]\n" + "udot z18.s, z29.b, z2.b[1]\n" + "udot z22.s, z29.b, z1.b[1]\n" + "udot z26.s, z29.b, z0.b[1]\n" + "udot z11.s, z28.b, z4.b[1]\n" + "ld1b { z29.b }, p5/Z, [x10, #-8, MUL VL]\n" + "udot z15.s, z28.b, z3.b[1]\n" + "udot z19.s, z28.b, z2.b[1]\n" + "udot z23.s, z28.b, z1.b[1]\n" + "udot z27.s, z28.b, z0.b[1]\n" + "ld1b { z28.b }, p5/Z, [x10, #-7, MUL VL]\n" + "udot z8.s, z29.b, z4.b[2]\n" + "udot z12.s, z29.b, z3.b[2]\n" + "udot z16.s, z29.b, z2.b[2]\n" + "udot z20.s, z29.b, z1.b[2]\n" + "udot z24.s, z29.b, z0.b[2]\n" + "udot z9.s, z28.b, z4.b[2]\n" + "ld1b { z29.b }, p5/Z, [x10, #-6, MUL VL]\n" + "udot z13.s, z28.b, z3.b[2]\n" + "udot z17.s, z28.b, z2.b[2]\n" + "udot z21.s, z28.b, z1.b[2]\n" + "udot z25.s, z28.b, z0.b[2]\n" + "ld1b { z28.b }, p5/Z, [x10, #-5, MUL VL]\n" + "udot z10.s, z29.b, z4.b[2]\n" + "udot z14.s, z29.b, z3.b[2]\n" + "udot z18.s, z29.b, z2.b[2]\n" + "udot z22.s, z29.b, z1.b[2]\n" + "udot z26.s, z29.b, z0.b[2]\n" + "udot z11.s, z28.b, z4.b[2]\n" + "ld1b { z29.b }, p5/Z, [x10, #-4, MUL VL]\n" + "udot z15.s, z28.b, z3.b[2]\n" + "udot z19.s, z28.b, z2.b[2]\n" + "udot z23.s, z28.b, z1.b[2]\n" + "udot z27.s, z28.b, z0.b[2]\n" + "ld1b { z28.b }, p5/Z, [x10, #-3, MUL VL]\n" + "udot z8.s, z29.b, z4.b[3]\n" + "udot z12.s, z29.b, z3.b[3]\n" + "udot z16.s, z29.b, z2.b[3]\n" + "udot z20.s, z29.b, z1.b[3]\n" + "udot z24.s, z29.b, z0.b[3]\n" + "udot z9.s, z28.b, z4.b[3]\n" + "ld1b { z29.b }, p5/Z, [x10, #-2, MUL VL]\n" + "udot z13.s, z28.b, z3.b[3]\n" + "udot z17.s, z28.b, z2.b[3]\n" + "udot z21.s, z28.b, z1.b[3]\n" + "udot z25.s, z28.b, z0.b[3]\n" + "ld1b { z28.b }, p5/Z, [x10, #-1, MUL VL]\n" + "udot z10.s, z29.b, z4.b[3]\n" + "udot z14.s, z29.b, z3.b[3]\n" + "udot z18.s, z29.b, z2.b[3]\n" + "udot z22.s, z29.b, z1.b[3]\n" + "udot z26.s, z29.b, z0.b[3]\n" + "udot z11.s, z28.b, z4.b[3]\n" + "udot z15.s, z28.b, z3.b[3]\n" + "udot z19.s, z28.b, z2.b[3]\n" + "udot z23.s, z28.b, z1.b[3]\n" + "udot z27.s, z28.b, z0.b[3]\n" "bgt 52b\n" "53:" // Height 5: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" @@ -1188,142 +1188,142 @@ void sve_hybrid_u8u32_dot_6x4VL ( "ld1rqb { z2.b }, p0/Z, [x24]\n" "ld1rqb { z3.b }, p0/Z, [x23]\n" "ld1rqb { z4.b }, p0/Z, [x22]\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "udot z8.s, z6.b, z0.b[0]\n" - "udot z12.s, z6.b, z1.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z16.s, z6.b, z2.b[0]\n" - "udot z20.s, z6.b, z3.b[0]\n" - "udot z24.s, z6.b, z4.b[0]\n" - "udot z9.s, z7.b, z0.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "udot z13.s, z7.b, z1.b[0]\n" - "udot z17.s, z7.b, z2.b[0]\n" - "udot z21.s, z7.b, z3.b[0]\n" - "udot z25.s, z7.b, z4.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1b { z29.b }, p5/Z, [x10]\n" + "udot z8.s, z29.b, z0.b[0]\n" + "udot z12.s, z29.b, z1.b[0]\n" + "ld1b { z28.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z16.s, z29.b, z2.b[0]\n" + "udot z20.s, z29.b, z3.b[0]\n" + "udot z24.s, z29.b, z4.b[0]\n" + "udot z9.s, z28.b, z0.b[0]\n" + "ld1b { z29.b }, p5/Z, [x10, #2, MUL VL]\n" + "udot z13.s, z28.b, z1.b[0]\n" + "udot z17.s, z28.b, z2.b[0]\n" + "udot z21.s, z28.b, z3.b[0]\n" + "udot z25.s, z28.b, z4.b[0]\n" + "ld1b { z28.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "udot z10.s, z6.b, z0.b[0]\n" - "udot z14.s, z6.b, z1.b[0]\n" - "udot z18.s, z6.b, z2.b[0]\n" - "udot z22.s, z6.b, z3.b[0]\n" - "udot z26.s, z6.b, z4.b[0]\n" - "udot z11.s, z7.b, z0.b[0]\n" - "udot z15.s, z7.b, z1.b[0]\n" - "udot z19.s, z7.b, z2.b[0]\n" - "udot z23.s, z7.b, z3.b[0]\n" - "udot z27.s, z7.b, z4.b[0]\n" + "udot z10.s, z29.b, z0.b[0]\n" + "udot z14.s, z29.b, z1.b[0]\n" + "udot z18.s, z29.b, z2.b[0]\n" + "udot z22.s, z29.b, z3.b[0]\n" + "udot z26.s, z29.b, z4.b[0]\n" + "udot z11.s, z28.b, z0.b[0]\n" + "udot z15.s, z28.b, z1.b[0]\n" + "udot z19.s, z28.b, z2.b[0]\n" + "udot z23.s, z28.b, z3.b[0]\n" + "udot z27.s, z28.b, z4.b[0]\n" "ble 54f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z8.s, z6.b, z0.b[1]\n" - "udot z12.s, z6.b, z1.b[1]\n" - "udot z16.s, z6.b, z2.b[1]\n" - "udot z20.s, z6.b, z3.b[1]\n" + "ld1b { z29.b }, p5/Z, [x10]\n" + "ld1b { z28.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z8.s, z29.b, z0.b[1]\n" + "udot z12.s, z29.b, z1.b[1]\n" + "udot z16.s, z29.b, z2.b[1]\n" + "udot z20.s, z29.b, z3.b[1]\n" "subs x27, x27, #0x4\n" - "udot z24.s, z6.b, z4.b[1]\n" - "udot z9.s, z7.b, z0.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "udot z13.s, z7.b, z1.b[1]\n" - "udot z17.s, z7.b, z2.b[1]\n" - "udot z21.s, z7.b, z3.b[1]\n" - "udot z25.s, z7.b, z4.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "udot z24.s, z29.b, z4.b[1]\n" + "udot z9.s, z28.b, z0.b[1]\n" + "ld1b { z29.b }, p5/Z, [x10, #2, MUL VL]\n" + "udot z13.s, z28.b, z1.b[1]\n" + "udot z17.s, z28.b, z2.b[1]\n" + "udot z21.s, z28.b, z3.b[1]\n" + "udot z25.s, z28.b, z4.b[1]\n" + "ld1b { z28.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "udot z10.s, z6.b, z0.b[1]\n" - "udot z14.s, z6.b, z1.b[1]\n" - "udot z18.s, z6.b, z2.b[1]\n" - "udot z22.s, z6.b, z3.b[1]\n" - "udot z26.s, z6.b, z4.b[1]\n" - "udot z11.s, z7.b, z0.b[1]\n" - "udot z15.s, z7.b, z1.b[1]\n" - "udot z19.s, z7.b, z2.b[1]\n" - "udot z23.s, z7.b, z3.b[1]\n" - "udot z27.s, z7.b, z4.b[1]\n" + "udot z10.s, z29.b, z0.b[1]\n" + "udot z14.s, z29.b, z1.b[1]\n" + "udot z18.s, z29.b, z2.b[1]\n" + "udot z22.s, z29.b, z3.b[1]\n" + "udot z26.s, z29.b, z4.b[1]\n" + "udot z11.s, z28.b, z0.b[1]\n" + "udot z15.s, z28.b, z1.b[1]\n" + "udot z19.s, z28.b, z2.b[1]\n" + "udot z23.s, z28.b, z3.b[1]\n" + "udot z27.s, z28.b, z4.b[1]\n" "ble 54f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z8.s, z6.b, z0.b[2]\n" - "udot z12.s, z6.b, z1.b[2]\n" - "udot z16.s, z6.b, z2.b[2]\n" - "udot z20.s, z6.b, z3.b[2]\n" + "ld1b { z29.b }, p5/Z, [x10]\n" + "ld1b { z28.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z8.s, z29.b, z0.b[2]\n" + "udot z12.s, z29.b, z1.b[2]\n" + "udot z16.s, z29.b, z2.b[2]\n" + "udot z20.s, z29.b, z3.b[2]\n" "subs x27, x27, #0x4\n" - "udot z24.s, z6.b, z4.b[2]\n" - "udot z9.s, z7.b, z0.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "udot z13.s, z7.b, z1.b[2]\n" - "udot z17.s, z7.b, z2.b[2]\n" - "udot z21.s, z7.b, z3.b[2]\n" - "udot z25.s, z7.b, z4.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "udot z24.s, z29.b, z4.b[2]\n" + "udot z9.s, z28.b, z0.b[2]\n" + "ld1b { z29.b }, p5/Z, [x10, #2, MUL VL]\n" + "udot z13.s, z28.b, z1.b[2]\n" + "udot z17.s, z28.b, z2.b[2]\n" + "udot z21.s, z28.b, z3.b[2]\n" + "udot z25.s, z28.b, z4.b[2]\n" + "ld1b { z28.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "udot z10.s, z6.b, z0.b[2]\n" - "udot z14.s, z6.b, z1.b[2]\n" - "udot z18.s, z6.b, z2.b[2]\n" - "udot z22.s, z6.b, z3.b[2]\n" - "udot z26.s, z6.b, z4.b[2]\n" - "udot z11.s, z7.b, z0.b[2]\n" - "udot z15.s, z7.b, z1.b[2]\n" - "udot z19.s, z7.b, z2.b[2]\n" - "udot z23.s, z7.b, z3.b[2]\n" - "udot z27.s, z7.b, z4.b[2]\n" + "udot z10.s, z29.b, z0.b[2]\n" + "udot z14.s, z29.b, z1.b[2]\n" + "udot z18.s, z29.b, z2.b[2]\n" + "udot z22.s, z29.b, z3.b[2]\n" + "udot z26.s, z29.b, z4.b[2]\n" + "udot z11.s, z28.b, z0.b[2]\n" + "udot z15.s, z28.b, z1.b[2]\n" + "udot z19.s, z28.b, z2.b[2]\n" + "udot z23.s, z28.b, z3.b[2]\n" + "udot z27.s, z28.b, z4.b[2]\n" "ble 54f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z8.s, z6.b, z0.b[3]\n" - "udot z12.s, z6.b, z1.b[3]\n" - "udot z16.s, z6.b, z2.b[3]\n" - "udot z20.s, z6.b, z3.b[3]\n" - "udot z24.s, z6.b, z4.b[3]\n" - "udot z9.s, z7.b, z0.b[3]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "udot z13.s, z7.b, z1.b[3]\n" - "udot z17.s, z7.b, z2.b[3]\n" - "udot z21.s, z7.b, z3.b[3]\n" - "udot z25.s, z7.b, z4.b[3]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1b { z29.b }, p5/Z, [x10]\n" + "ld1b { z28.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z8.s, z29.b, z0.b[3]\n" + "udot z12.s, z29.b, z1.b[3]\n" + "udot z16.s, z29.b, z2.b[3]\n" + "udot z20.s, z29.b, z3.b[3]\n" + "udot z24.s, z29.b, z4.b[3]\n" + "udot z9.s, z28.b, z0.b[3]\n" + "ld1b { z29.b }, p5/Z, [x10, #2, MUL VL]\n" + "udot z13.s, z28.b, z1.b[3]\n" + "udot z17.s, z28.b, z2.b[3]\n" + "udot z21.s, z28.b, z3.b[3]\n" + "udot z25.s, z28.b, z4.b[3]\n" + "ld1b { z28.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "udot z10.s, z6.b, z0.b[3]\n" - "udot z14.s, z6.b, z1.b[3]\n" - "udot z18.s, z6.b, z2.b[3]\n" - "udot z22.s, z6.b, z3.b[3]\n" - "udot z26.s, z6.b, z4.b[3]\n" - "udot z11.s, z7.b, z0.b[3]\n" - "udot z15.s, z7.b, z1.b[3]\n" - "udot z19.s, z7.b, z2.b[3]\n" - "udot z23.s, z7.b, z3.b[3]\n" - "udot z27.s, z7.b, z4.b[3]\n" + "udot z10.s, z29.b, z0.b[3]\n" + "udot z14.s, z29.b, z1.b[3]\n" + "udot z18.s, z29.b, z2.b[3]\n" + "udot z22.s, z29.b, z3.b[3]\n" + "udot z26.s, z29.b, z4.b[3]\n" + "udot z11.s, z28.b, z0.b[3]\n" + "udot z15.s, z28.b, z1.b[3]\n" + "udot z19.s, z28.b, z2.b[3]\n" + "udot z23.s, z28.b, z3.b[3]\n" + "udot z27.s, z28.b, z4.b[3]\n" "54:" // Height 5: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 49b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "st1w { z8.s }, p4, [x9]\n" + "add x23, x9, x20, LSL #2\n" "add x22, x23, x20, LSL #2\n" + "st1w { z8.s }, p4, [x9]\n" "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "st1w { z9.s }, p3, [x9, #1, MUL VL]\n" "st1w { z10.s }, p2, [x9, #2, MUL VL]\n" "st1w { z11.s }, p1, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "st1w { z12.s }, p4, [x24]\n" - "st1w { z13.s }, p3, [x24, #1, MUL VL]\n" - "st1w { z14.s }, p2, [x24, #2, MUL VL]\n" - "st1w { z15.s }, p1, [x24, #3, MUL VL]\n" - "st1w { z16.s }, p4, [x23]\n" - "st1w { z17.s }, p3, [x23, #1, MUL VL]\n" - "st1w { z18.s }, p2, [x23, #2, MUL VL]\n" - "st1w { z19.s }, p1, [x23, #3, MUL VL]\n" - "st1w { z20.s }, p4, [x22]\n" - "st1w { z21.s }, p3, [x22, #1, MUL VL]\n" - "st1w { z22.s }, p2, [x22, #2, MUL VL]\n" - "st1w { z23.s }, p1, [x22, #3, MUL VL]\n" - "st1w { z24.s }, p4, [x21]\n" - "st1w { z25.s }, p3, [x21, #1, MUL VL]\n" - "st1w { z26.s }, p2, [x21, #2, MUL VL]\n" - "st1w { z27.s }, p1, [x21, #3, MUL VL]\n" + "st1w { z12.s }, p4, [x23]\n" + "st1w { z13.s }, p3, [x23, #1, MUL VL]\n" + "st1w { z14.s }, p2, [x23, #2, MUL VL]\n" + "st1w { z15.s }, p1, [x23, #3, MUL VL]\n" + "st1w { z16.s }, p4, [x22]\n" + "st1w { z17.s }, p3, [x22, #1, MUL VL]\n" + "st1w { z18.s }, p2, [x22, #2, MUL VL]\n" + "st1w { z19.s }, p1, [x22, #3, MUL VL]\n" + "st1w { z20.s }, p4, [x21]\n" + "st1w { z21.s }, p3, [x21, #1, MUL VL]\n" + "st1w { z22.s }, p2, [x21, #2, MUL VL]\n" + "st1w { z23.s }, p1, [x21, #3, MUL VL]\n" + "st1w { z24.s }, p4, [x20]\n" + "st1w { z25.s }, p3, [x20, #1, MUL VL]\n" + "st1w { z26.s }, p2, [x20, #2, MUL VL]\n" + "st1w { z27.s }, p1, [x20, #3, MUL VL]\n" "55:" // Height 5: Writeback done "decw x11, ALL, MUL #4\n" "cmp x11, XZR\n" @@ -1407,16 +1407,16 @@ void sve_hybrid_u8u32_dot_6x4VL ( "60:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 61f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 62f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -1428,143 +1428,143 @@ void sve_hybrid_u8u32_dot_6x4VL ( "b 62f\n" "61:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" + "add x21, x22, x21\n" "62:" // Height 6: input setup done "cmp x27, #0x10\n" "ble 64f\n" "63:" // Height 6: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z0.b }, p0/Z, [x26]\n" - "ld1rqb { z1.b }, p0/Z, [x25]\n" + "ld1rqb { z7.b }, p0/Z, [x26]\n" + "ld1rqb { z6.b }, p0/Z, [x25]\n" "sub x27, x27, #0x10\n" - "ld1rqb { z2.b }, p0/Z, [x24]\n" - "ld1rqb { z3.b }, p0/Z, [x23]\n" + "ld1rqb { z5.b }, p0/Z, [x24]\n" + "ld1rqb { z4.b }, p0/Z, [x23]\n" "cmp x27, #0x10\n" "add x26, x26, #0x10\n" - "ld1rqb { z4.b }, p0/Z, [x22]\n" - "ld1rqb { z5.b }, p0/Z, [x21]\n" + "ld1rqb { z3.b }, p0/Z, [x22]\n" + "ld1rqb { z2.b }, p0/Z, [x21]\n" "add x25, x25, #0x10\n" "add x24, x24, #0x10\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z8.s, z6.b, z0.b[0]\n" - "udot z12.s, z6.b, z1.b[0]\n" - "udot z16.s, z6.b, z2.b[0]\n" - "udot z20.s, z6.b, z3.b[0]\n" + "ld1b { z1.b }, p5/Z, [x10]\n" + "ld1b { z0.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z8.s, z1.b, z7.b[0]\n" + "udot z12.s, z1.b, z6.b[0]\n" + "udot z16.s, z1.b, z5.b[0]\n" + "udot z20.s, z1.b, z4.b[0]\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - "udot z24.s, z6.b, z4.b[0]\n" - "udot z28.s, z6.b, z5.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" + "udot z24.s, z1.b, z3.b[0]\n" + "udot z28.s, z1.b, z2.b[0]\n" + "ld1b { z1.b }, p5/Z, [x10, #2, MUL VL]\n" "add x21, x21, #0x10\n" - "udot z9.s, z7.b, z0.b[0]\n" - "udot z13.s, z7.b, z1.b[0]\n" - "udot z17.s, z7.b, z2.b[0]\n" - "udot z21.s, z7.b, z3.b[0]\n" - "udot z25.s, z7.b, z4.b[0]\n" - "udot z29.s, z7.b, z5.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" - "udot z10.s, z6.b, z0.b[0]\n" - "udot z14.s, z6.b, z1.b[0]\n" - "udot z18.s, z6.b, z2.b[0]\n" - "udot z22.s, z6.b, z3.b[0]\n" - "udot z26.s, z6.b, z4.b[0]\n" - "udot z30.s, z6.b, z5.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #4, MUL VL]\n" - "udot z11.s, z7.b, z0.b[0]\n" - "udot z15.s, z7.b, z1.b[0]\n" - "udot z19.s, z7.b, z2.b[0]\n" - "udot z23.s, z7.b, z3.b[0]\n" - "udot z27.s, z7.b, z4.b[0]\n" - "udot z31.s, z7.b, z5.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #5, MUL VL]\n" - "udot z8.s, z6.b, z0.b[1]\n" - "udot z12.s, z6.b, z1.b[1]\n" - "udot z16.s, z6.b, z2.b[1]\n" - "udot z20.s, z6.b, z3.b[1]\n" - "udot z24.s, z6.b, z4.b[1]\n" - "udot z28.s, z6.b, z5.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #6, MUL VL]\n" - "udot z9.s, z7.b, z0.b[1]\n" - "udot z13.s, z7.b, z1.b[1]\n" - "udot z17.s, z7.b, z2.b[1]\n" - "udot z21.s, z7.b, z3.b[1]\n" - "udot z25.s, z7.b, z4.b[1]\n" - "udot z29.s, z7.b, z5.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #7, MUL VL]\n" + "udot z9.s, z0.b, z7.b[0]\n" + "udot z13.s, z0.b, z6.b[0]\n" + "udot z17.s, z0.b, z5.b[0]\n" + "udot z21.s, z0.b, z4.b[0]\n" + "udot z25.s, z0.b, z3.b[0]\n" + "udot z29.s, z0.b, z2.b[0]\n" + "ld1b { z0.b }, p5/Z, [x10, #3, MUL VL]\n" + "udot z10.s, z1.b, z7.b[0]\n" + "udot z14.s, z1.b, z6.b[0]\n" + "udot z18.s, z1.b, z5.b[0]\n" + "udot z22.s, z1.b, z4.b[0]\n" + "udot z26.s, z1.b, z3.b[0]\n" + "udot z30.s, z1.b, z2.b[0]\n" + "ld1b { z1.b }, p5/Z, [x10, #4, MUL VL]\n" + "udot z11.s, z0.b, z7.b[0]\n" + "udot z15.s, z0.b, z6.b[0]\n" + "udot z19.s, z0.b, z5.b[0]\n" + "udot z23.s, z0.b, z4.b[0]\n" + "udot z27.s, z0.b, z3.b[0]\n" + "udot z31.s, z0.b, z2.b[0]\n" + "ld1b { z0.b }, p5/Z, [x10, #5, MUL VL]\n" + "udot z8.s, z1.b, z7.b[1]\n" + "udot z12.s, z1.b, z6.b[1]\n" + "udot z16.s, z1.b, z5.b[1]\n" + "udot z20.s, z1.b, z4.b[1]\n" + "udot z24.s, z1.b, z3.b[1]\n" + "udot z28.s, z1.b, z2.b[1]\n" + "ld1b { z1.b }, p5/Z, [x10, #6, MUL VL]\n" + "udot z9.s, z0.b, z7.b[1]\n" + "udot z13.s, z0.b, z6.b[1]\n" + "udot z17.s, z0.b, z5.b[1]\n" + "udot z21.s, z0.b, z4.b[1]\n" + "udot z25.s, z0.b, z3.b[1]\n" + "udot z29.s, z0.b, z2.b[1]\n" + "ld1b { z0.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "udot z10.s, z6.b, z0.b[1]\n" - "udot z14.s, z6.b, z1.b[1]\n" - "udot z18.s, z6.b, z2.b[1]\n" - "udot z22.s, z6.b, z3.b[1]\n" - "udot z26.s, z6.b, z4.b[1]\n" - "udot z30.s, z6.b, z5.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #-8, MUL VL]\n" - "udot z11.s, z7.b, z0.b[1]\n" - "udot z15.s, z7.b, z1.b[1]\n" - "udot z19.s, z7.b, z2.b[1]\n" - "udot z23.s, z7.b, z3.b[1]\n" - "udot z27.s, z7.b, z4.b[1]\n" - "udot z31.s, z7.b, z5.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #-7, MUL VL]\n" - "udot z8.s, z6.b, z0.b[2]\n" - "udot z12.s, z6.b, z1.b[2]\n" - "udot z16.s, z6.b, z2.b[2]\n" - "udot z20.s, z6.b, z3.b[2]\n" - "udot z24.s, z6.b, z4.b[2]\n" - "udot z28.s, z6.b, z5.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #-6, MUL VL]\n" - "udot z9.s, z7.b, z0.b[2]\n" - "udot z13.s, z7.b, z1.b[2]\n" - "udot z17.s, z7.b, z2.b[2]\n" - "udot z21.s, z7.b, z3.b[2]\n" - "udot z25.s, z7.b, z4.b[2]\n" - "udot z29.s, z7.b, z5.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #-5, MUL VL]\n" - "udot z10.s, z6.b, z0.b[2]\n" - "udot z14.s, z6.b, z1.b[2]\n" - "udot z18.s, z6.b, z2.b[2]\n" - "udot z22.s, z6.b, z3.b[2]\n" - "udot z26.s, z6.b, z4.b[2]\n" - "udot z30.s, z6.b, z5.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #-4, MUL VL]\n" - "udot z11.s, z7.b, z0.b[2]\n" - "udot z15.s, z7.b, z1.b[2]\n" - "udot z19.s, z7.b, z2.b[2]\n" - "udot z23.s, z7.b, z3.b[2]\n" - "udot z27.s, z7.b, z4.b[2]\n" - "udot z31.s, z7.b, z5.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n" - "udot z8.s, z6.b, z0.b[3]\n" - "udot z12.s, z6.b, z1.b[3]\n" - "udot z16.s, z6.b, z2.b[3]\n" - "udot z20.s, z6.b, z3.b[3]\n" - "udot z24.s, z6.b, z4.b[3]\n" - "udot z28.s, z6.b, z5.b[3]\n" - "ld1b { z6.b }, p5/Z, [x10, #-2, MUL VL]\n" - "udot z9.s, z7.b, z0.b[3]\n" - "udot z13.s, z7.b, z1.b[3]\n" - "udot z17.s, z7.b, z2.b[3]\n" - "udot z21.s, z7.b, z3.b[3]\n" - "udot z25.s, z7.b, z4.b[3]\n" - "udot z29.s, z7.b, z5.b[3]\n" - "ld1b { z7.b }, p5/Z, [x10, #-1, MUL VL]\n" - "udot z10.s, z6.b, z0.b[3]\n" - "udot z14.s, z6.b, z1.b[3]\n" - "udot z18.s, z6.b, z2.b[3]\n" - "udot z22.s, z6.b, z3.b[3]\n" - "udot z26.s, z6.b, z4.b[3]\n" - "udot z30.s, z6.b, z5.b[3]\n" - "udot z11.s, z7.b, z0.b[3]\n" - "udot z15.s, z7.b, z1.b[3]\n" - "udot z19.s, z7.b, z2.b[3]\n" - "udot z23.s, z7.b, z3.b[3]\n" - "udot z27.s, z7.b, z4.b[3]\n" - "udot z31.s, z7.b, z5.b[3]\n" + "udot z10.s, z1.b, z7.b[1]\n" + "udot z14.s, z1.b, z6.b[1]\n" + "udot z18.s, z1.b, z5.b[1]\n" + "udot z22.s, z1.b, z4.b[1]\n" + "udot z26.s, z1.b, z3.b[1]\n" + "udot z30.s, z1.b, z2.b[1]\n" + "ld1b { z1.b }, p5/Z, [x10, #-8, MUL VL]\n" + "udot z11.s, z0.b, z7.b[1]\n" + "udot z15.s, z0.b, z6.b[1]\n" + "udot z19.s, z0.b, z5.b[1]\n" + "udot z23.s, z0.b, z4.b[1]\n" + "udot z27.s, z0.b, z3.b[1]\n" + "udot z31.s, z0.b, z2.b[1]\n" + "ld1b { z0.b }, p5/Z, [x10, #-7, MUL VL]\n" + "udot z8.s, z1.b, z7.b[2]\n" + "udot z12.s, z1.b, z6.b[2]\n" + "udot z16.s, z1.b, z5.b[2]\n" + "udot z20.s, z1.b, z4.b[2]\n" + "udot z24.s, z1.b, z3.b[2]\n" + "udot z28.s, z1.b, z2.b[2]\n" + "ld1b { z1.b }, p5/Z, [x10, #-6, MUL VL]\n" + "udot z9.s, z0.b, z7.b[2]\n" + "udot z13.s, z0.b, z6.b[2]\n" + "udot z17.s, z0.b, z5.b[2]\n" + "udot z21.s, z0.b, z4.b[2]\n" + "udot z25.s, z0.b, z3.b[2]\n" + "udot z29.s, z0.b, z2.b[2]\n" + "ld1b { z0.b }, p5/Z, [x10, #-5, MUL VL]\n" + "udot z10.s, z1.b, z7.b[2]\n" + "udot z14.s, z1.b, z6.b[2]\n" + "udot z18.s, z1.b, z5.b[2]\n" + "udot z22.s, z1.b, z4.b[2]\n" + "udot z26.s, z1.b, z3.b[2]\n" + "udot z30.s, z1.b, z2.b[2]\n" + "ld1b { z1.b }, p5/Z, [x10, #-4, MUL VL]\n" + "udot z11.s, z0.b, z7.b[2]\n" + "udot z15.s, z0.b, z6.b[2]\n" + "udot z19.s, z0.b, z5.b[2]\n" + "udot z23.s, z0.b, z4.b[2]\n" + "udot z27.s, z0.b, z3.b[2]\n" + "udot z31.s, z0.b, z2.b[2]\n" + "ld1b { z0.b }, p5/Z, [x10, #-3, MUL VL]\n" + "udot z8.s, z1.b, z7.b[3]\n" + "udot z12.s, z1.b, z6.b[3]\n" + "udot z16.s, z1.b, z5.b[3]\n" + "udot z20.s, z1.b, z4.b[3]\n" + "udot z24.s, z1.b, z3.b[3]\n" + "udot z28.s, z1.b, z2.b[3]\n" + "ld1b { z1.b }, p5/Z, [x10, #-2, MUL VL]\n" + "udot z9.s, z0.b, z7.b[3]\n" + "udot z13.s, z0.b, z6.b[3]\n" + "udot z17.s, z0.b, z5.b[3]\n" + "udot z21.s, z0.b, z4.b[3]\n" + "udot z25.s, z0.b, z3.b[3]\n" + "udot z29.s, z0.b, z2.b[3]\n" + "ld1b { z0.b }, p5/Z, [x10, #-1, MUL VL]\n" + "udot z10.s, z1.b, z7.b[3]\n" + "udot z14.s, z1.b, z6.b[3]\n" + "udot z18.s, z1.b, z5.b[3]\n" + "udot z22.s, z1.b, z4.b[3]\n" + "udot z26.s, z1.b, z3.b[3]\n" + "udot z30.s, z1.b, z2.b[3]\n" + "udot z11.s, z0.b, z7.b[3]\n" + "udot z15.s, z0.b, z6.b[3]\n" + "udot z19.s, z0.b, z5.b[3]\n" + "udot z23.s, z0.b, z4.b[3]\n" + "udot z27.s, z0.b, z3.b[3]\n" + "udot z31.s, z0.b, z2.b[3]\n" "bgt 63b\n" "64:" // Height 6: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" @@ -1575,127 +1575,127 @@ void sve_hybrid_u8u32_dot_6x4VL ( "ld1rqb { z3.b }, p0/Z, [x23]\n" "ld1rqb { z4.b }, p0/Z, [x22]\n" "ld1rqb { z5.b }, p0/Z, [x21]\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z8.s, z6.b, z0.b[0]\n" - "udot z12.s, z6.b, z1.b[0]\n" - "udot z16.s, z6.b, z2.b[0]\n" - "udot z20.s, z6.b, z3.b[0]\n" - "udot z24.s, z6.b, z4.b[0]\n" - "udot z28.s, z6.b, z5.b[0]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "udot z9.s, z7.b, z0.b[0]\n" - "udot z13.s, z7.b, z1.b[0]\n" - "udot z17.s, z7.b, z2.b[0]\n" - "udot z21.s, z7.b, z3.b[0]\n" - "udot z25.s, z7.b, z4.b[0]\n" - "udot z29.s, z7.b, z5.b[0]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1b { z7.b }, p5/Z, [x10]\n" + "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z8.s, z7.b, z0.b[0]\n" + "udot z12.s, z7.b, z1.b[0]\n" + "udot z16.s, z7.b, z2.b[0]\n" + "udot z20.s, z7.b, z3.b[0]\n" + "udot z24.s, z7.b, z4.b[0]\n" + "udot z28.s, z7.b, z5.b[0]\n" + "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" + "udot z9.s, z6.b, z0.b[0]\n" + "udot z13.s, z6.b, z1.b[0]\n" + "udot z17.s, z6.b, z2.b[0]\n" + "udot z21.s, z6.b, z3.b[0]\n" + "udot z25.s, z6.b, z4.b[0]\n" + "udot z29.s, z6.b, z5.b[0]\n" + "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "udot z10.s, z6.b, z0.b[0]\n" - "udot z14.s, z6.b, z1.b[0]\n" - "udot z18.s, z6.b, z2.b[0]\n" - "udot z22.s, z6.b, z3.b[0]\n" - "udot z26.s, z6.b, z4.b[0]\n" - "udot z30.s, z6.b, z5.b[0]\n" - "udot z11.s, z7.b, z0.b[0]\n" - "udot z15.s, z7.b, z1.b[0]\n" - "udot z19.s, z7.b, z2.b[0]\n" - "udot z23.s, z7.b, z3.b[0]\n" - "udot z27.s, z7.b, z4.b[0]\n" - "udot z31.s, z7.b, z5.b[0]\n" + "udot z10.s, z7.b, z0.b[0]\n" + "udot z14.s, z7.b, z1.b[0]\n" + "udot z18.s, z7.b, z2.b[0]\n" + "udot z22.s, z7.b, z3.b[0]\n" + "udot z26.s, z7.b, z4.b[0]\n" + "udot z30.s, z7.b, z5.b[0]\n" + "udot z11.s, z6.b, z0.b[0]\n" + "udot z15.s, z6.b, z1.b[0]\n" + "udot z19.s, z6.b, z2.b[0]\n" + "udot z23.s, z6.b, z3.b[0]\n" + "udot z27.s, z6.b, z4.b[0]\n" + "udot z31.s, z6.b, z5.b[0]\n" "ble 65f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z8.s, z6.b, z0.b[1]\n" - "udot z12.s, z6.b, z1.b[1]\n" - "udot z16.s, z6.b, z2.b[1]\n" - "udot z20.s, z6.b, z3.b[1]\n" + "ld1b { z7.b }, p5/Z, [x10]\n" + "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z8.s, z7.b, z0.b[1]\n" + "udot z12.s, z7.b, z1.b[1]\n" + "udot z16.s, z7.b, z2.b[1]\n" + "udot z20.s, z7.b, z3.b[1]\n" "subs x27, x27, #0x4\n" - "udot z24.s, z6.b, z4.b[1]\n" - "udot z28.s, z6.b, z5.b[1]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "udot z9.s, z7.b, z0.b[1]\n" - "udot z13.s, z7.b, z1.b[1]\n" - "udot z17.s, z7.b, z2.b[1]\n" - "udot z21.s, z7.b, z3.b[1]\n" - "udot z25.s, z7.b, z4.b[1]\n" - "udot z29.s, z7.b, z5.b[1]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "udot z24.s, z7.b, z4.b[1]\n" + "udot z28.s, z7.b, z5.b[1]\n" + "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" + "udot z9.s, z6.b, z0.b[1]\n" + "udot z13.s, z6.b, z1.b[1]\n" + "udot z17.s, z6.b, z2.b[1]\n" + "udot z21.s, z6.b, z3.b[1]\n" + "udot z25.s, z6.b, z4.b[1]\n" + "udot z29.s, z6.b, z5.b[1]\n" + "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "udot z10.s, z6.b, z0.b[1]\n" - "udot z14.s, z6.b, z1.b[1]\n" - "udot z18.s, z6.b, z2.b[1]\n" - "udot z22.s, z6.b, z3.b[1]\n" - "udot z26.s, z6.b, z4.b[1]\n" - "udot z30.s, z6.b, z5.b[1]\n" - "udot z11.s, z7.b, z0.b[1]\n" - "udot z15.s, z7.b, z1.b[1]\n" - "udot z19.s, z7.b, z2.b[1]\n" - "udot z23.s, z7.b, z3.b[1]\n" - "udot z27.s, z7.b, z4.b[1]\n" - "udot z31.s, z7.b, z5.b[1]\n" + "udot z10.s, z7.b, z0.b[1]\n" + "udot z14.s, z7.b, z1.b[1]\n" + "udot z18.s, z7.b, z2.b[1]\n" + "udot z22.s, z7.b, z3.b[1]\n" + "udot z26.s, z7.b, z4.b[1]\n" + "udot z30.s, z7.b, z5.b[1]\n" + "udot z11.s, z6.b, z0.b[1]\n" + "udot z15.s, z6.b, z1.b[1]\n" + "udot z19.s, z6.b, z2.b[1]\n" + "udot z23.s, z6.b, z3.b[1]\n" + "udot z27.s, z6.b, z4.b[1]\n" + "udot z31.s, z6.b, z5.b[1]\n" "ble 65f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z8.s, z6.b, z0.b[2]\n" - "udot z12.s, z6.b, z1.b[2]\n" - "udot z16.s, z6.b, z2.b[2]\n" - "udot z20.s, z6.b, z3.b[2]\n" + "ld1b { z7.b }, p5/Z, [x10]\n" + "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z8.s, z7.b, z0.b[2]\n" + "udot z12.s, z7.b, z1.b[2]\n" + "udot z16.s, z7.b, z2.b[2]\n" + "udot z20.s, z7.b, z3.b[2]\n" "subs x27, x27, #0x4\n" - "udot z24.s, z6.b, z4.b[2]\n" - "udot z28.s, z6.b, z5.b[2]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "udot z9.s, z7.b, z0.b[2]\n" - "udot z13.s, z7.b, z1.b[2]\n" - "udot z17.s, z7.b, z2.b[2]\n" - "udot z21.s, z7.b, z3.b[2]\n" - "udot z25.s, z7.b, z4.b[2]\n" - "udot z29.s, z7.b, z5.b[2]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "udot z24.s, z7.b, z4.b[2]\n" + "udot z28.s, z7.b, z5.b[2]\n" + "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" + "udot z9.s, z6.b, z0.b[2]\n" + "udot z13.s, z6.b, z1.b[2]\n" + "udot z17.s, z6.b, z2.b[2]\n" + "udot z21.s, z6.b, z3.b[2]\n" + "udot z25.s, z6.b, z4.b[2]\n" + "udot z29.s, z6.b, z5.b[2]\n" + "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "udot z10.s, z6.b, z0.b[2]\n" - "udot z14.s, z6.b, z1.b[2]\n" - "udot z18.s, z6.b, z2.b[2]\n" - "udot z22.s, z6.b, z3.b[2]\n" - "udot z26.s, z6.b, z4.b[2]\n" - "udot z30.s, z6.b, z5.b[2]\n" - "udot z11.s, z7.b, z0.b[2]\n" - "udot z15.s, z7.b, z1.b[2]\n" - "udot z19.s, z7.b, z2.b[2]\n" - "udot z23.s, z7.b, z3.b[2]\n" - "udot z27.s, z7.b, z4.b[2]\n" - "udot z31.s, z7.b, z5.b[2]\n" + "udot z10.s, z7.b, z0.b[2]\n" + "udot z14.s, z7.b, z1.b[2]\n" + "udot z18.s, z7.b, z2.b[2]\n" + "udot z22.s, z7.b, z3.b[2]\n" + "udot z26.s, z7.b, z4.b[2]\n" + "udot z30.s, z7.b, z5.b[2]\n" + "udot z11.s, z6.b, z0.b[2]\n" + "udot z15.s, z6.b, z1.b[2]\n" + "udot z19.s, z6.b, z2.b[2]\n" + "udot z23.s, z6.b, z3.b[2]\n" + "udot z27.s, z6.b, z4.b[2]\n" + "udot z31.s, z6.b, z5.b[2]\n" "ble 65f\n" - "ld1b { z6.b }, p5/Z, [x10]\n" - "ld1b { z7.b }, p5/Z, [x10, #1, MUL VL]\n" - "udot z8.s, z6.b, z0.b[3]\n" - "udot z12.s, z6.b, z1.b[3]\n" - "udot z16.s, z6.b, z2.b[3]\n" - "udot z20.s, z6.b, z3.b[3]\n" - "udot z24.s, z6.b, z4.b[3]\n" - "udot z28.s, z6.b, z5.b[3]\n" - "ld1b { z6.b }, p5/Z, [x10, #2, MUL VL]\n" - "udot z9.s, z7.b, z0.b[3]\n" - "udot z13.s, z7.b, z1.b[3]\n" - "udot z17.s, z7.b, z2.b[3]\n" - "udot z21.s, z7.b, z3.b[3]\n" - "udot z25.s, z7.b, z4.b[3]\n" - "udot z29.s, z7.b, z5.b[3]\n" - "ld1b { z7.b }, p5/Z, [x10, #3, MUL VL]\n" + "ld1b { z7.b }, p5/Z, [x10]\n" + "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" + "udot z8.s, z7.b, z0.b[3]\n" + "udot z12.s, z7.b, z1.b[3]\n" + "udot z16.s, z7.b, z2.b[3]\n" + "udot z20.s, z7.b, z3.b[3]\n" + "udot z24.s, z7.b, z4.b[3]\n" + "udot z28.s, z7.b, z5.b[3]\n" + "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" + "udot z9.s, z6.b, z0.b[3]\n" + "udot z13.s, z6.b, z1.b[3]\n" + "udot z17.s, z6.b, z2.b[3]\n" + "udot z21.s, z6.b, z3.b[3]\n" + "udot z25.s, z6.b, z4.b[3]\n" + "udot z29.s, z6.b, z5.b[3]\n" + "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" "addvl x10, x10, #4\n" - "udot z10.s, z6.b, z0.b[3]\n" - "udot z14.s, z6.b, z1.b[3]\n" - "udot z18.s, z6.b, z2.b[3]\n" - "udot z22.s, z6.b, z3.b[3]\n" - "udot z26.s, z6.b, z4.b[3]\n" - "udot z30.s, z6.b, z5.b[3]\n" - "udot z11.s, z7.b, z0.b[3]\n" - "udot z15.s, z7.b, z1.b[3]\n" - "udot z19.s, z7.b, z2.b[3]\n" - "udot z23.s, z7.b, z3.b[3]\n" - "udot z27.s, z7.b, z4.b[3]\n" - "udot z31.s, z7.b, z5.b[3]\n" + "udot z10.s, z7.b, z0.b[3]\n" + "udot z14.s, z7.b, z1.b[3]\n" + "udot z18.s, z7.b, z2.b[3]\n" + "udot z22.s, z7.b, z3.b[3]\n" + "udot z26.s, z7.b, z4.b[3]\n" + "udot z30.s, z7.b, z5.b[3]\n" + "udot z11.s, z6.b, z0.b[3]\n" + "udot z15.s, z6.b, z1.b[3]\n" + "udot z19.s, z6.b, z2.b[3]\n" + "udot z23.s, z6.b, z3.b[3]\n" + "udot z27.s, z6.b, z4.b[3]\n" + "udot z31.s, z6.b, z5.b[3]\n" "65:" // Height 6: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1748,7 +1748,6 @@ void sve_hybrid_u8u32_dot_6x4VL ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "68:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -1756,4 +1755,4 @@ void sve_hybrid_u8u32_dot_6x4VL ( } } // namespace arm_gemm -#endif // ARM_COMPUTE_ENABLE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_mmla_6x4VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_mmla_6x4VL.hpp index c0d089278e394717b1f244ac4d7b41c1ada0ed2d..8c6a3dba7dce7729c07b59a5b179bdb8d1fda34d 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_mmla_6x4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_mmla_6x4VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef ARM_COMPUTE_ENABLE_SVE + #include "../std_transforms_sve.hpp" #include "../performance_parameters.hpp" @@ -74,7 +74,6 @@ public: template static inline PerformanceParameters get_performance_parameters(const CPUInfo *ci) { - if (std::is_same::value) { switch (ci->get_cpu_model()) { default: @@ -86,7 +85,6 @@ public: } } - if (std::is_same::value) { switch (ci->get_cpu_model()) { default: @@ -111,5 +109,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_mmla_6x4VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_mmla_6x4VL/generic.cpp index 59f33289b4c4084182851937d85b818d1b3b192f..9269576d9009bb4b573051e0dd3dc66bd2c5d0b1 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_mmla_6x4VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_hybrid_u8u32_mmla_6x4VL/generic.cpp @@ -100,16 +100,16 @@ void sve_hybrid_u8u32_mmla_6x4VL ( "incw x20\n" "whilelt p1.s, x20, x11\n" "tbz %x[flags], #0, 3f\n" - "ld1w { z9.s }, p4/Z, [x9]\n" - "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n" + "ld1w { z19.s }, p4/Z, [x9]\n" + "ld1w { z18.s }, p3/Z, [x9, #1, MUL VL]\n" + "zip1 z8.d, z19.d, z12.d\n" + "zip2 z12.d, z19.d, z12.d\n" + "ld1w { z17.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" + "zip1 z9.d, z18.d, z13.d\n" + "zip2 z13.d, z18.d, z13.d\n" + "zip1 z10.d, z17.d, z14.d\n" + "zip2 z14.d, z17.d, z14.d\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" "b 4f\n" @@ -127,11 +127,11 @@ void sve_hybrid_u8u32_mmla_6x4VL ( "5:" // Height 1: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 6f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" "cbnz x28, 7f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -143,86 +143,86 @@ void sve_hybrid_u8u32_mmla_6x4VL ( "ble 9f\n" "8:" // Height 1: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z1.b }, p0/Z, [x26]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45c79808 // ummla z8.s, z0.b, z7.b\n" - ".inst 0x45c6980c // ummla z12.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45c79809 // ummla z9.s, z0.b, z7.b\n" - ".inst 0x45c6980d // ummla z13.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x45c7980a // ummla z10.s, z0.b, z7.b\n" - ".inst 0x45c6980e // ummla z14.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + "ld1rqb { z20.b }, p0/Z, [x26]\n" + "trn1 z18.d, z20.d, z19.d\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45d19a48 // ummla z8.s, z18.b, z17.b\n" + ".inst 0x45d09a4c // ummla z12.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45d19a49 // ummla z9.s, z18.b, z17.b\n" + ".inst 0x45d09a4d // ummla z13.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #4, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x45d19a4a // ummla z10.s, z18.b, z17.b\n" + ".inst 0x45d09a4e // ummla z14.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #6, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "trn2 z1.d, z1.d, z2.d\n" - ".inst 0x45c7980b // ummla z11.s, z0.b, z7.b\n" - ".inst 0x45c6980f // ummla z15.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-8, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #-7, MUL VL]\n" - ".inst 0x45c79828 // ummla z8.s, z1.b, z7.b\n" - ".inst 0x45c6982c // ummla z12.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-6, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #-5, MUL VL]\n" - ".inst 0x45c79829 // ummla z9.s, z1.b, z7.b\n" - ".inst 0x45c6982d // ummla z13.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-4, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #-3, MUL VL]\n" - ".inst 0x45c7982a // ummla z10.s, z1.b, z7.b\n" - ".inst 0x45c6982e // ummla z14.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #-1, MUL VL]\n" + "trn2 z20.d, z20.d, z19.d\n" + ".inst 0x45d19a4b // ummla z11.s, z18.b, z17.b\n" + ".inst 0x45d09a4f // ummla z15.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #-8, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #-7, MUL VL]\n" + ".inst 0x45d19a88 // ummla z8.s, z20.b, z17.b\n" + ".inst 0x45d09a8c // ummla z12.s, z20.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #-6, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #-5, MUL VL]\n" + ".inst 0x45d19a89 // ummla z9.s, z20.b, z17.b\n" + ".inst 0x45d09a8d // ummla z13.s, z20.b, z16.b\n" + "ld1b { z16.b }, p5/Z, [x10, #-4, MUL VL]\n" + "ld1b { z7.b }, p5/Z, [x10, #-3, MUL VL]\n" + ".inst 0x45d09a8a // ummla z10.s, z20.b, z16.b\n" + ".inst 0x45c79a8e // ummla z14.s, z20.b, z7.b\n" + "ld1b { z17.b }, p5/Z, [x10, #-2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #-1, MUL VL]\n" "sub x27, x27, #0x10\n" "cmp x27, #0x10\n" - ".inst 0x45c7982b // ummla z11.s, z1.b, z7.b\n" - ".inst 0x45c6982f // ummla z15.s, z1.b, z6.b\n" + ".inst 0x45d19a8b // ummla z11.s, z20.b, z17.b\n" + ".inst 0x45d09a8f // ummla z15.s, z20.b, z16.b\n" "add x26, x26, #0x10\n" "bgt 8b\n" "9:" // Height 1: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" "ld1rqb { z1.b }, p0/Z, [x26]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45c79808 // ummla z8.s, z0.b, z7.b\n" - ".inst 0x45c6980c // ummla z12.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45c79809 // ummla z9.s, z0.b, z7.b\n" - ".inst 0x45c6980d // ummla z13.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x45c7980a // ummla z10.s, z0.b, z7.b\n" - ".inst 0x45c6980e // ummla z14.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + "trn1 z18.d, z1.d, z19.d\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45d19a48 // ummla z8.s, z18.b, z17.b\n" + ".inst 0x45d09a4c // ummla z12.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45d19a49 // ummla z9.s, z18.b, z17.b\n" + ".inst 0x45d09a4d // ummla z13.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #4, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x45d19a4a // ummla z10.s, z18.b, z17.b\n" + ".inst 0x45d09a4e // ummla z14.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #6, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #7, MUL VL]\n" "subs x27, x27, #0x8\n" - "trn2 z1.d, z1.d, z2.d\n" - ".inst 0x45c7980b // ummla z11.s, z0.b, z7.b\n" - ".inst 0x45c6980f // ummla z15.s, z0.b, z6.b\n" + "trn2 z1.d, z1.d, z19.d\n" + ".inst 0x45d19a4b // ummla z11.s, z18.b, z17.b\n" + ".inst 0x45d09a4f // ummla z15.s, z18.b, z16.b\n" "addvl x10, x10, #8\n" "ble 10f\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45c79828 // ummla z8.s, z1.b, z7.b\n" - ".inst 0x45c6982c // ummla z12.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45c79829 // ummla z9.s, z1.b, z7.b\n" - ".inst 0x45c6982d // ummla z13.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x45c7982a // ummla z10.s, z1.b, z7.b\n" - ".inst 0x45c6982e // ummla z14.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" - ".inst 0x45c7982b // ummla z11.s, z1.b, z7.b\n" - ".inst 0x45c6982f // ummla z15.s, z1.b, z6.b\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45d19828 // ummla z8.s, z1.b, z17.b\n" + ".inst 0x45d0982c // ummla z12.s, z1.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45d19829 // ummla z9.s, z1.b, z17.b\n" + ".inst 0x45d0982d // ummla z13.s, z1.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #4, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x45d1982a // ummla z10.s, z1.b, z17.b\n" + ".inst 0x45d0982e // ummla z14.s, z1.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #6, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x45d1982b // ummla z11.s, z1.b, z17.b\n" + ".inst 0x45d0982f // ummla z15.s, z1.b, z16.b\n" "addvl x10, x10, #8\n" "10:" // Height 1: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -258,21 +258,21 @@ void sve_hybrid_u8u32_mmla_6x4VL ( "whilelt p1.s, x20, x11\n" "tbz %x[flags], #0, 14f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x9]\n" - "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n" + "add x20, x9, x20, LSL #2\n" + "ld1w { z18.s }, p4/Z, [x9]\n" + "ld1w { z2.s }, p3/Z, [x9, #1, MUL VL]\n" + "ld1w { z17.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x24]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" + "ld1w { z12.s }, p4/Z, [x20]\n" + "zip1 z8.d, z18.d, z12.d\n" + "zip2 z12.d, z18.d, z12.d\n" + "ld1w { z13.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x20, #2, MUL VL]\n" + "zip1 z9.d, z2.d, z13.d\n" + "zip2 z13.d, z2.d, z13.d\n" + "ld1w { z15.s }, p1/Z, [x20, #3, MUL VL]\n" + "zip1 z10.d, z17.d, z14.d\n" + "zip2 z14.d, z17.d, z14.d\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" "b 15f\n" @@ -290,12 +290,12 @@ void sve_hybrid_u8u32_mmla_6x4VL ( "16:" // Height 2: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 17f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" "cbnz x28, 18f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -303,95 +303,95 @@ void sve_hybrid_u8u32_mmla_6x4VL ( "b 18f\n" "17:" // Height 2: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" + "add x25, x26, x21\n" "18:" // Height 2: input setup done "cmp x27, #0x10\n" "ble 20f\n" "19:" // Height 2: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45c79808 // ummla z8.s, z0.b, z7.b\n" - ".inst 0x45c6980c // ummla z12.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45c79809 // ummla z9.s, z0.b, z7.b\n" - ".inst 0x45c6980d // ummla z13.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x45c7980a // ummla z10.s, z0.b, z7.b\n" - ".inst 0x45c6980e // ummla z14.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + "ld1rqb { z20.b }, p0/Z, [x26]\n" + "ld1rqb { z19.b }, p0/Z, [x25]\n" + "trn1 z18.d, z20.d, z19.d\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45d19a48 // ummla z8.s, z18.b, z17.b\n" + ".inst 0x45d09a4c // ummla z12.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45d19a49 // ummla z9.s, z18.b, z17.b\n" + ".inst 0x45d09a4d // ummla z13.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #4, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x45d19a4a // ummla z10.s, z18.b, z17.b\n" + ".inst 0x45d09a4e // ummla z14.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #6, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - "trn2 z1.d, z1.d, z2.d\n" - ".inst 0x45c7980b // ummla z11.s, z0.b, z7.b\n" - ".inst 0x45c6980f // ummla z15.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-8, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #-7, MUL VL]\n" - ".inst 0x45c79828 // ummla z8.s, z1.b, z7.b\n" - ".inst 0x45c6982c // ummla z12.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-6, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #-5, MUL VL]\n" - ".inst 0x45c79829 // ummla z9.s, z1.b, z7.b\n" - ".inst 0x45c6982d // ummla z13.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-4, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #-3, MUL VL]\n" - ".inst 0x45c7982a // ummla z10.s, z1.b, z7.b\n" - ".inst 0x45c6982e // ummla z14.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #-1, MUL VL]\n" + "trn2 z20.d, z20.d, z19.d\n" + ".inst 0x45d19a4b // ummla z11.s, z18.b, z17.b\n" + ".inst 0x45d09a4f // ummla z15.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #-8, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #-7, MUL VL]\n" + ".inst 0x45d19a88 // ummla z8.s, z20.b, z17.b\n" + ".inst 0x45d09a8c // ummla z12.s, z20.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #-6, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #-5, MUL VL]\n" + ".inst 0x45d19a89 // ummla z9.s, z20.b, z17.b\n" + ".inst 0x45d09a8d // ummla z13.s, z20.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #-4, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #-3, MUL VL]\n" + ".inst 0x45d19a8a // ummla z10.s, z20.b, z17.b\n" + ".inst 0x45d09a8e // ummla z14.s, z20.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #-2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #-1, MUL VL]\n" "sub x27, x27, #0x10\n" "cmp x27, #0x10\n" - ".inst 0x45c7982b // ummla z11.s, z1.b, z7.b\n" - ".inst 0x45c6982f // ummla z15.s, z1.b, z6.b\n" + ".inst 0x45d19a8b // ummla z11.s, z20.b, z17.b\n" + ".inst 0x45d09a8f // ummla z15.s, z20.b, z16.b\n" "add x26, x26, #0x10\n" "add x25, x25, #0x10\n" "bgt 19b\n" "20:" // Height 2: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45c79808 // ummla z8.s, z0.b, z7.b\n" - ".inst 0x45c6980c // ummla z12.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45c79809 // ummla z9.s, z0.b, z7.b\n" - ".inst 0x45c6980d // ummla z13.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x45c7980a // ummla z10.s, z0.b, z7.b\n" - ".inst 0x45c6980e // ummla z14.s, z0.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + "ld1rqb { z19.b }, p0/Z, [x25]\n" + "trn1 z18.d, z1.d, z19.d\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45d19a48 // ummla z8.s, z18.b, z17.b\n" + ".inst 0x45d09a4c // ummla z12.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45d19a49 // ummla z9.s, z18.b, z17.b\n" + ".inst 0x45d09a4d // ummla z13.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #4, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x45d19a4a // ummla z10.s, z18.b, z17.b\n" + ".inst 0x45d09a4e // ummla z14.s, z18.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #6, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #7, MUL VL]\n" "subs x27, x27, #0x8\n" - "trn2 z1.d, z1.d, z2.d\n" - ".inst 0x45c7980b // ummla z11.s, z0.b, z7.b\n" - ".inst 0x45c6980f // ummla z15.s, z0.b, z6.b\n" + "trn2 z1.d, z1.d, z19.d\n" + ".inst 0x45d19a4b // ummla z11.s, z18.b, z17.b\n" + ".inst 0x45d09a4f // ummla z15.s, z18.b, z16.b\n" "addvl x10, x10, #8\n" "ble 21f\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45c79828 // ummla z8.s, z1.b, z7.b\n" - ".inst 0x45c6982c // ummla z12.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45c79829 // ummla z9.s, z1.b, z7.b\n" - ".inst 0x45c6982d // ummla z13.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x45c7982a // ummla z10.s, z1.b, z7.b\n" - ".inst 0x45c6982e // ummla z14.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" - ".inst 0x45c7982b // ummla z11.s, z1.b, z7.b\n" - ".inst 0x45c6982f // ummla z15.s, z1.b, z6.b\n" + "ld1b { z17.b }, p5/Z, [x10]\n" + "ld1b { z16.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45d19828 // ummla z8.s, z1.b, z17.b\n" + ".inst 0x45d0982c // ummla z12.s, z1.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45d19829 // ummla z9.s, z1.b, z17.b\n" + ".inst 0x45d0982d // ummla z13.s, z1.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #4, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x45d1982a // ummla z10.s, z1.b, z17.b\n" + ".inst 0x45d0982e // ummla z14.s, z1.b, z16.b\n" + "ld1b { z17.b }, p5/Z, [x10, #6, MUL VL]\n" + "ld1b { z16.b }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x45d1982b // ummla z11.s, z1.b, z17.b\n" + ".inst 0x45d0982f // ummla z15.s, z1.b, z16.b\n" "addvl x10, x10, #8\n" "21:" // Height 2: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" @@ -399,24 +399,24 @@ void sve_hybrid_u8u32_mmla_6x4VL ( "cmp x28, x20\n" "bne 16b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "uzp1 z7.d, z8.d, z12.d\n" + "add x20, x9, x20, LSL #2\n" + "uzp1 z16.d, z8.d, z12.d\n" "uzp2 z8.d, z8.d, z12.d\n" - "uzp1 z12.d, z9.d, z13.d\n" + "uzp1 z17.d, z9.d, z13.d\n" "uzp2 z9.d, z9.d, z13.d\n" - "st1w { z7.s }, p4, [x9]\n" - "uzp1 z13.d, z10.d, z14.d\n" + "st1w { z16.s }, p4, [x9]\n" + "uzp1 z16.d, z10.d, z14.d\n" "uzp2 z10.d, z10.d, z14.d\n" - "st1w { z12.s }, p3, [x9, #1, MUL VL]\n" - "uzp1 z14.d, z11.d, z15.d\n" + "st1w { z17.s }, p3, [x9, #1, MUL VL]\n" + "uzp1 z2.d, z11.d, z15.d\n" "uzp2 z11.d, z11.d, z15.d\n" - "st1w { z13.s }, p2, [x9, #2, MUL VL]\n" - "st1w { z14.s }, p1, [x9, #3, MUL VL]\n" + "st1w { z16.s }, p2, [x9, #2, MUL VL]\n" + "st1w { z2.s }, p1, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" - "st1w { z8.s }, p4, [x24]\n" - "st1w { z9.s }, p3, [x24, #1, MUL VL]\n" - "st1w { z10.s }, p2, [x24, #2, MUL VL]\n" - "st1w { z11.s }, p1, [x24, #3, MUL VL]\n" + "st1w { z8.s }, p4, [x20]\n" + "st1w { z9.s }, p3, [x20, #1, MUL VL]\n" + "st1w { z10.s }, p2, [x20, #2, MUL VL]\n" + "st1w { z11.s }, p1, [x20, #3, MUL VL]\n" "22:" // Height 2: Writeback done "decw x11, ALL, MUL #4\n" "cmp x11, XZR\n" @@ -437,28 +437,28 @@ void sve_hybrid_u8u32_mmla_6x4VL ( "whilelt p1.s, x20, x11\n" "tbz %x[flags], #0, 25f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x9]\n" - "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n" + "add x21, x9, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" + "ld1w { z19.s }, p4/Z, [x9]\n" + "ld1w { z17.s }, p3/Z, [x9, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x24]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z17.s }, p4/Z, [x23]\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" - "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x21]\n" + "zip1 z8.d, z19.d, z12.d\n" + "zip2 z12.d, z19.d, z12.d\n" + "ld1w { z13.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x21, #2, MUL VL]\n" + "zip1 z9.d, z17.d, z13.d\n" + "zip2 z13.d, z17.d, z13.d\n" + "ld1w { z15.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z17.s }, p4/Z, [x20]\n" + "zip1 z10.d, z18.d, z14.d\n" + "zip2 z14.d, z18.d, z14.d\n" + "ld1w { z18.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z19.s }, p2/Z, [x20, #2, MUL VL]\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" - "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z24.s }, p1/Z, [x20, #3, MUL VL]\n" "zip1 z16.d, z17.d, z20.d\n" "zip2 z20.d, z17.d, z20.d\n" "zip1 z17.d, z18.d, z21.d\n" @@ -490,13 +490,13 @@ void sve_hybrid_u8u32_mmla_6x4VL ( "27:" // Height 3: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 28f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" "cbnz x28, 29f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -505,169 +505,169 @@ void sve_hybrid_u8u32_mmla_6x4VL ( "b 29f\n" "28:" // Height 3: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" "29:" // Height 3: input setup done "cmp x27, #0x10\n" "ble 31f\n" "30:" // Height 3: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" - "ld1rqb { z3.b }, p0/Z, [x24]\n" - "trn1 z0.d, z1.d, z2.d\n" - "trn2 z1.d, z1.d, z2.d\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45c79808 // ummla z8.s, z0.b, z7.b\n" - ".inst 0x45c79850 // ummla z16.s, z2.b, z7.b\n" - ".inst 0x45c6980c // ummla z12.s, z0.b, z6.b\n" - ".inst 0x45c69854 // ummla z20.s, z2.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45c79809 // ummla z9.s, z0.b, z7.b\n" - ".inst 0x45c79851 // ummla z17.s, z2.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x45c6980d // ummla z13.s, z0.b, z6.b\n" - ".inst 0x45c69855 // ummla z21.s, z2.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" + "ld1rqb { z30.b }, p0/Z, [x26]\n" + "ld1rqb { z24.b }, p0/Z, [x25]\n" + "ld1rqb { z28.b }, p0/Z, [x24]\n" + "trn1 z27.d, z30.d, z24.d\n" + "trn2 z30.d, z30.d, z24.d\n" + "ld1b { z25.b }, p5/Z, [x10]\n" + "trn1 z26.d, z28.d, z29.d\n" + "ld1b { z24.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45d99b68 // ummla z8.s, z27.b, z25.b\n" + ".inst 0x45d99b50 // ummla z16.s, z26.b, z25.b\n" + ".inst 0x45d89b6c // ummla z12.s, z27.b, z24.b\n" + ".inst 0x45d89b54 // ummla z20.s, z26.b, z24.b\n" + "ld1b { z25.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z24.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45d99b69 // ummla z9.s, z27.b, z25.b\n" + ".inst 0x45d99b51 // ummla z17.s, z26.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #4, MUL VL]\n" + "trn2 z28.d, z28.d, z29.d\n" + ".inst 0x45d89b6d // ummla z13.s, z27.b, z24.b\n" + ".inst 0x45d89b55 // ummla z21.s, z26.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #5, MUL VL]\n" "sub x27, x27, #0x10\n" - ".inst 0x45c7980a // ummla z10.s, z0.b, z7.b\n" - ".inst 0x45c79852 // ummla z18.s, z2.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x45d99b6a // ummla z10.s, z27.b, z25.b\n" + ".inst 0x45d99b52 // ummla z18.s, z26.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #6, MUL VL]\n" "cmp x27, #0x10\n" - ".inst 0x45c6980e // ummla z14.s, z0.b, z6.b\n" - ".inst 0x45c69856 // ummla z22.s, z2.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x45d89b6e // ummla z14.s, z27.b, z24.b\n" + ".inst 0x45d89b56 // ummla z22.s, z26.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - ".inst 0x45c7980b // ummla z11.s, z0.b, z7.b\n" - ".inst 0x45c79853 // ummla z19.s, z2.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-8, MUL VL]\n" + ".inst 0x45d99b6b // ummla z11.s, z27.b, z25.b\n" + ".inst 0x45d99b53 // ummla z19.s, z26.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #-8, MUL VL]\n" "add x26, x26, #0x10\n" - ".inst 0x45c6980f // ummla z15.s, z0.b, z6.b\n" - ".inst 0x45c69857 // ummla z23.s, z2.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-7, MUL VL]\n" + ".inst 0x45d89b6f // ummla z15.s, z27.b, z24.b\n" + ".inst 0x45d89b57 // ummla z23.s, z26.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #-7, MUL VL]\n" "add x25, x25, #0x10\n" - ".inst 0x45c79828 // ummla z8.s, z1.b, z7.b\n" - ".inst 0x45c79870 // ummla z16.s, z3.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-6, MUL VL]\n" + ".inst 0x45d99bc8 // ummla z8.s, z30.b, z25.b\n" + ".inst 0x45d99b90 // ummla z16.s, z28.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #-6, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x45c6982c // ummla z12.s, z1.b, z6.b\n" - ".inst 0x45c69874 // ummla z20.s, z3.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-5, MUL VL]\n" - ".inst 0x45c79829 // ummla z9.s, z1.b, z7.b\n" - ".inst 0x45c79871 // ummla z17.s, z3.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-4, MUL VL]\n" - ".inst 0x45c6982d // ummla z13.s, z1.b, z6.b\n" - ".inst 0x45c69875 // ummla z21.s, z3.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-3, MUL VL]\n" - ".inst 0x45c7982a // ummla z10.s, z1.b, z7.b\n" - ".inst 0x45c79872 // ummla z18.s, z3.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-2, MUL VL]\n" - ".inst 0x45c6982e // ummla z14.s, z1.b, z6.b\n" - ".inst 0x45c69876 // ummla z22.s, z3.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-1, MUL VL]\n" - ".inst 0x45c7982b // ummla z11.s, z1.b, z7.b\n" - ".inst 0x45c79873 // ummla z19.s, z3.b, z7.b\n" - ".inst 0x45c6982f // ummla z15.s, z1.b, z6.b\n" - ".inst 0x45c69877 // ummla z23.s, z3.b, z6.b\n" + ".inst 0x45d89bcc // ummla z12.s, z30.b, z24.b\n" + ".inst 0x45d89b94 // ummla z20.s, z28.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #-5, MUL VL]\n" + ".inst 0x45d99bc9 // ummla z9.s, z30.b, z25.b\n" + ".inst 0x45d99b91 // ummla z17.s, z28.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #-4, MUL VL]\n" + ".inst 0x45d89bcd // ummla z13.s, z30.b, z24.b\n" + ".inst 0x45d89b95 // ummla z21.s, z28.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #-3, MUL VL]\n" + ".inst 0x45d99bca // ummla z10.s, z30.b, z25.b\n" + ".inst 0x45d99b92 // ummla z18.s, z28.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #-2, MUL VL]\n" + ".inst 0x45d89bce // ummla z14.s, z30.b, z24.b\n" + ".inst 0x45d89b96 // ummla z22.s, z28.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #-1, MUL VL]\n" + ".inst 0x45d99bcb // ummla z11.s, z30.b, z25.b\n" + ".inst 0x45d99b93 // ummla z19.s, z28.b, z25.b\n" + ".inst 0x45d89bcf // ummla z15.s, z30.b, z24.b\n" + ".inst 0x45d89b97 // ummla z23.s, z28.b, z24.b\n" "bgt 30b\n" "31:" // Height 3: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" + "ld1rqb { z24.b }, p0/Z, [x25]\n" "ld1rqb { z3.b }, p0/Z, [x24]\n" - "trn1 z0.d, z1.d, z2.d\n" - "trn2 z1.d, z1.d, z2.d\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45c79808 // ummla z8.s, z0.b, z7.b\n" - ".inst 0x45c79850 // ummla z16.s, z2.b, z7.b\n" - ".inst 0x45c6980c // ummla z12.s, z0.b, z6.b\n" - ".inst 0x45c69854 // ummla z20.s, z2.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45c79809 // ummla z9.s, z0.b, z7.b\n" - ".inst 0x45c79851 // ummla z17.s, z2.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" + "trn1 z27.d, z1.d, z24.d\n" + "trn2 z1.d, z1.d, z24.d\n" + "ld1b { z25.b }, p5/Z, [x10]\n" + "trn1 z26.d, z3.d, z28.d\n" + "ld1b { z24.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45d99b68 // ummla z8.s, z27.b, z25.b\n" + ".inst 0x45d99b50 // ummla z16.s, z26.b, z25.b\n" + ".inst 0x45d89b6c // ummla z12.s, z27.b, z24.b\n" + ".inst 0x45d89b54 // ummla z20.s, z26.b, z24.b\n" + "ld1b { z25.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z24.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45d99b69 // ummla z9.s, z27.b, z25.b\n" + ".inst 0x45d99b51 // ummla z17.s, z26.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #4, MUL VL]\n" "subs x27, x27, #0x8\n" - ".inst 0x45c6980d // ummla z13.s, z0.b, z6.b\n" - ".inst 0x45c69855 // ummla z21.s, z2.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x45c7980a // ummla z10.s, z0.b, z7.b\n" - ".inst 0x45c79852 // ummla z18.s, z2.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x45c6980e // ummla z14.s, z0.b, z6.b\n" - ".inst 0x45c69856 // ummla z22.s, z2.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x45d89b6d // ummla z13.s, z27.b, z24.b\n" + ".inst 0x45d89b55 // ummla z21.s, z26.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #5, MUL VL]\n" + "trn2 z3.d, z3.d, z28.d\n" + ".inst 0x45d99b6a // ummla z10.s, z27.b, z25.b\n" + ".inst 0x45d99b52 // ummla z18.s, z26.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x45d89b6e // ummla z14.s, z27.b, z24.b\n" + ".inst 0x45d89b56 // ummla z22.s, z26.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - ".inst 0x45c7980b // ummla z11.s, z0.b, z7.b\n" - ".inst 0x45c79853 // ummla z19.s, z2.b, z7.b\n" - ".inst 0x45c6980f // ummla z15.s, z0.b, z6.b\n" - ".inst 0x45c69857 // ummla z23.s, z2.b, z6.b\n" + ".inst 0x45d99b6b // ummla z11.s, z27.b, z25.b\n" + ".inst 0x45d99b53 // ummla z19.s, z26.b, z25.b\n" + ".inst 0x45d89b6f // ummla z15.s, z27.b, z24.b\n" + ".inst 0x45d89b57 // ummla z23.s, z26.b, z24.b\n" "ble 32f\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45c79828 // ummla z8.s, z1.b, z7.b\n" - ".inst 0x45c79870 // ummla z16.s, z3.b, z7.b\n" - ".inst 0x45c6982c // ummla z12.s, z1.b, z6.b\n" - ".inst 0x45c69874 // ummla z20.s, z3.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45c79829 // ummla z9.s, z1.b, z7.b\n" - ".inst 0x45c79871 // ummla z17.s, z3.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x45c6982d // ummla z13.s, z1.b, z6.b\n" - ".inst 0x45c69875 // ummla z21.s, z3.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x45c7982a // ummla z10.s, z1.b, z7.b\n" - ".inst 0x45c79872 // ummla z18.s, z3.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x45c6982e // ummla z14.s, z1.b, z6.b\n" - ".inst 0x45c69876 // ummla z22.s, z3.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + "ld1b { z25.b }, p5/Z, [x10]\n" + "ld1b { z24.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45d99828 // ummla z8.s, z1.b, z25.b\n" + ".inst 0x45d99870 // ummla z16.s, z3.b, z25.b\n" + ".inst 0x45d8982c // ummla z12.s, z1.b, z24.b\n" + ".inst 0x45d89874 // ummla z20.s, z3.b, z24.b\n" + "ld1b { z25.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z24.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45d99829 // ummla z9.s, z1.b, z25.b\n" + ".inst 0x45d99871 // ummla z17.s, z3.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x45d8982d // ummla z13.s, z1.b, z24.b\n" + ".inst 0x45d89875 // ummla z21.s, z3.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x45d9982a // ummla z10.s, z1.b, z25.b\n" + ".inst 0x45d99872 // ummla z18.s, z3.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x45d8982e // ummla z14.s, z1.b, z24.b\n" + ".inst 0x45d89876 // ummla z22.s, z3.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - ".inst 0x45c7982b // ummla z11.s, z1.b, z7.b\n" - ".inst 0x45c79873 // ummla z19.s, z3.b, z7.b\n" - ".inst 0x45c6982f // ummla z15.s, z1.b, z6.b\n" - ".inst 0x45c69877 // ummla z23.s, z3.b, z6.b\n" + ".inst 0x45d9982b // ummla z11.s, z1.b, z25.b\n" + ".inst 0x45d99873 // ummla z19.s, z3.b, z25.b\n" + ".inst 0x45d8982f // ummla z15.s, z1.b, z24.b\n" + ".inst 0x45d89877 // ummla z23.s, z3.b, z24.b\n" "32:" // Height 3: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 27b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "uzp1 z7.d, z8.d, z12.d\n" + "add x21, x9, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" + "uzp1 z25.d, z8.d, z12.d\n" "uzp2 z8.d, z8.d, z12.d\n" - "uzp1 z12.d, z9.d, z13.d\n" - "st1w { z7.s }, p4, [x9]\n" + "uzp1 z24.d, z9.d, z13.d\n" + "st1w { z25.s }, p4, [x9]\n" "uzp2 z9.d, z9.d, z13.d\n" - "uzp1 z13.d, z10.d, z14.d\n" - "st1w { z12.s }, p3, [x9, #1, MUL VL]\n" + "uzp1 z25.d, z10.d, z14.d\n" + "st1w { z24.s }, p3, [x9, #1, MUL VL]\n" "uzp2 z10.d, z10.d, z14.d\n" - "uzp1 z14.d, z11.d, z15.d\n" - "st1w { z13.s }, p2, [x9, #2, MUL VL]\n" + "uzp1 z24.d, z11.d, z15.d\n" + "st1w { z25.s }, p2, [x9, #2, MUL VL]\n" "uzp2 z11.d, z11.d, z15.d\n" "uzp1 z16.d, z16.d, z20.d\n" - "st1w { z14.s }, p1, [x9, #3, MUL VL]\n" + "st1w { z24.s }, p1, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" "uzp1 z17.d, z17.d, z21.d\n" "uzp1 z18.d, z18.d, z22.d\n" - "st1w { z8.s }, p4, [x24]\n" + "st1w { z8.s }, p4, [x21]\n" "uzp1 z19.d, z19.d, z23.d\n" - "st1w { z9.s }, p3, [x24, #1, MUL VL]\n" - "st1w { z10.s }, p2, [x24, #2, MUL VL]\n" - "st1w { z11.s }, p1, [x24, #3, MUL VL]\n" - "st1w { z16.s }, p4, [x23]\n" - "st1w { z17.s }, p3, [x23, #1, MUL VL]\n" - "st1w { z18.s }, p2, [x23, #2, MUL VL]\n" - "st1w { z19.s }, p1, [x23, #3, MUL VL]\n" + "st1w { z9.s }, p3, [x21, #1, MUL VL]\n" + "st1w { z10.s }, p2, [x21, #2, MUL VL]\n" + "st1w { z11.s }, p1, [x21, #3, MUL VL]\n" + "st1w { z16.s }, p4, [x20]\n" + "st1w { z17.s }, p3, [x20, #1, MUL VL]\n" + "st1w { z18.s }, p2, [x20, #2, MUL VL]\n" + "st1w { z19.s }, p1, [x20, #3, MUL VL]\n" "33:" // Height 3: Writeback done "decw x11, ALL, MUL #4\n" "cmp x11, XZR\n" @@ -688,37 +688,37 @@ void sve_hybrid_u8u32_mmla_6x4VL ( "whilelt p1.s, x20, x11\n" "tbz %x[flags], #0, 36f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x9]\n" - "add x22, x23, x20, LSL #2\n" - "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n" + "add x22, x9, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" + "ld1w { z19.s }, p4/Z, [x9]\n" + "add x20, x21, x20, LSL #2\n" + "ld1w { z17.s }, p3/Z, [x9, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x24]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z17.s }, p4/Z, [x23]\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" - "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x22]\n" + "zip1 z8.d, z19.d, z12.d\n" + "zip2 z12.d, z19.d, z12.d\n" + "ld1w { z13.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x22, #2, MUL VL]\n" + "zip1 z9.d, z17.d, z13.d\n" + "zip2 z13.d, z17.d, z13.d\n" + "ld1w { z15.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z17.s }, p4/Z, [x21]\n" + "zip1 z10.d, z18.d, z14.d\n" + "zip2 z14.d, z18.d, z14.d\n" + "ld1w { z18.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z19.s }, p2/Z, [x21, #2, MUL VL]\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" - "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x22]\n" + "ld1w { z24.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x20]\n" "zip1 z16.d, z17.d, z20.d\n" "zip2 z20.d, z17.d, z20.d\n" - "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n" + "ld1w { z21.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x20, #2, MUL VL]\n" "zip1 z17.d, z18.d, z21.d\n" "zip2 z21.d, z18.d, z21.d\n" - "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z23.s }, p1/Z, [x20, #3, MUL VL]\n" "zip1 z18.d, z19.d, z22.d\n" "zip2 z22.d, z19.d, z22.d\n" "zip1 z19.d, z24.d, z23.d\n" @@ -746,14 +746,14 @@ void sve_hybrid_u8u32_mmla_6x4VL ( "38:" // Height 4: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 39f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" "cbnz x28, 40f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -763,182 +763,182 @@ void sve_hybrid_u8u32_mmla_6x4VL ( "b 40f\n" "39:" // Height 4: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" "40:" // Height 4: input setup done "cmp x27, #0x10\n" "ble 42f\n" "41:" // Height 4: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1rqb { z3.b }, p0/Z, [x24]\n" - "ld1rqb { z4.b }, p0/Z, [x23]\n" - "trn2 z1.d, z1.d, z2.d\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45c79808 // ummla z8.s, z0.b, z7.b\n" - ".inst 0x45c79850 // ummla z16.s, z2.b, z7.b\n" - ".inst 0x45c6980c // ummla z12.s, z0.b, z6.b\n" - ".inst 0x45c69854 // ummla z20.s, z2.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45c79809 // ummla z9.s, z0.b, z7.b\n" - ".inst 0x45c79851 // ummla z17.s, z2.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x45c6980d // ummla z13.s, z0.b, z6.b\n" - ".inst 0x45c69855 // ummla z21.s, z2.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" + "ld1rqb { z30.b }, p0/Z, [x26]\n" + "ld1rqb { z24.b }, p0/Z, [x25]\n" + "trn1 z29.d, z30.d, z24.d\n" + "ld1rqb { z28.b }, p0/Z, [x24]\n" + "ld1rqb { z27.b }, p0/Z, [x23]\n" + "trn2 z30.d, z30.d, z24.d\n" + "trn1 z26.d, z28.d, z27.d\n" + "ld1b { z25.b }, p5/Z, [x10]\n" + "ld1b { z24.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45d99ba8 // ummla z8.s, z29.b, z25.b\n" + ".inst 0x45d99b50 // ummla z16.s, z26.b, z25.b\n" + ".inst 0x45d89bac // ummla z12.s, z29.b, z24.b\n" + ".inst 0x45d89b54 // ummla z20.s, z26.b, z24.b\n" + "ld1b { z25.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z24.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45d99ba9 // ummla z9.s, z29.b, z25.b\n" + ".inst 0x45d99b51 // ummla z17.s, z26.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #4, MUL VL]\n" + "trn2 z28.d, z28.d, z27.d\n" + ".inst 0x45d89bad // ummla z13.s, z29.b, z24.b\n" + ".inst 0x45d89b55 // ummla z21.s, z26.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #5, MUL VL]\n" "sub x27, x27, #0x10\n" - ".inst 0x45c7980a // ummla z10.s, z0.b, z7.b\n" - ".inst 0x45c79852 // ummla z18.s, z2.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x45d99baa // ummla z10.s, z29.b, z25.b\n" + ".inst 0x45d99b52 // ummla z18.s, z26.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #6, MUL VL]\n" "cmp x27, #0x10\n" - ".inst 0x45c6980e // ummla z14.s, z0.b, z6.b\n" - ".inst 0x45c69856 // ummla z22.s, z2.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x45d89bae // ummla z14.s, z29.b, z24.b\n" + ".inst 0x45d89b56 // ummla z22.s, z26.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - ".inst 0x45c7980b // ummla z11.s, z0.b, z7.b\n" - ".inst 0x45c79853 // ummla z19.s, z2.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-8, MUL VL]\n" + ".inst 0x45d99bab // ummla z11.s, z29.b, z25.b\n" + ".inst 0x45d99b53 // ummla z19.s, z26.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #-8, MUL VL]\n" "add x26, x26, #0x10\n" - ".inst 0x45c6980f // ummla z15.s, z0.b, z6.b\n" - ".inst 0x45c69857 // ummla z23.s, z2.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-7, MUL VL]\n" + ".inst 0x45d89baf // ummla z15.s, z29.b, z24.b\n" + ".inst 0x45d89b57 // ummla z23.s, z26.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #-7, MUL VL]\n" "add x25, x25, #0x10\n" - ".inst 0x45c79828 // ummla z8.s, z1.b, z7.b\n" - ".inst 0x45c79870 // ummla z16.s, z3.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-6, MUL VL]\n" + ".inst 0x45d99bc8 // ummla z8.s, z30.b, z25.b\n" + ".inst 0x45d99b90 // ummla z16.s, z28.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #-6, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x45c6982c // ummla z12.s, z1.b, z6.b\n" - ".inst 0x45c69874 // ummla z20.s, z3.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-5, MUL VL]\n" + ".inst 0x45d89bcc // ummla z12.s, z30.b, z24.b\n" + ".inst 0x45d89b94 // ummla z20.s, z28.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #-5, MUL VL]\n" "add x23, x23, #0x10\n" - ".inst 0x45c79829 // ummla z9.s, z1.b, z7.b\n" - ".inst 0x45c79871 // ummla z17.s, z3.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-4, MUL VL]\n" - ".inst 0x45c6982d // ummla z13.s, z1.b, z6.b\n" - ".inst 0x45c69875 // ummla z21.s, z3.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-3, MUL VL]\n" - ".inst 0x45c7982a // ummla z10.s, z1.b, z7.b\n" - ".inst 0x45c79872 // ummla z18.s, z3.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-2, MUL VL]\n" - ".inst 0x45c6982e // ummla z14.s, z1.b, z6.b\n" - ".inst 0x45c69876 // ummla z22.s, z3.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-1, MUL VL]\n" - ".inst 0x45c7982b // ummla z11.s, z1.b, z7.b\n" - ".inst 0x45c79873 // ummla z19.s, z3.b, z7.b\n" - ".inst 0x45c6982f // ummla z15.s, z1.b, z6.b\n" - ".inst 0x45c69877 // ummla z23.s, z3.b, z6.b\n" + ".inst 0x45d99bc9 // ummla z9.s, z30.b, z25.b\n" + ".inst 0x45d99b91 // ummla z17.s, z28.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #-4, MUL VL]\n" + ".inst 0x45d89bcd // ummla z13.s, z30.b, z24.b\n" + ".inst 0x45d89b95 // ummla z21.s, z28.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #-3, MUL VL]\n" + ".inst 0x45d99bca // ummla z10.s, z30.b, z25.b\n" + ".inst 0x45d99b92 // ummla z18.s, z28.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #-2, MUL VL]\n" + ".inst 0x45d89bce // ummla z14.s, z30.b, z24.b\n" + ".inst 0x45d89b96 // ummla z22.s, z28.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #-1, MUL VL]\n" + ".inst 0x45d99bcb // ummla z11.s, z30.b, z25.b\n" + ".inst 0x45d99b93 // ummla z19.s, z28.b, z25.b\n" + ".inst 0x45d89bcf // ummla z15.s, z30.b, z24.b\n" + ".inst 0x45d89b97 // ummla z23.s, z28.b, z24.b\n" "bgt 41b\n" "42:" // Height 4: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" + "ld1rqb { z24.b }, p0/Z, [x25]\n" + "trn1 z28.d, z1.d, z24.d\n" "ld1rqb { z3.b }, p0/Z, [x24]\n" - "ld1rqb { z4.b }, p0/Z, [x23]\n" - "trn2 z1.d, z1.d, z2.d\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45c79808 // ummla z8.s, z0.b, z7.b\n" - ".inst 0x45c79850 // ummla z16.s, z2.b, z7.b\n" - ".inst 0x45c6980c // ummla z12.s, z0.b, z6.b\n" - ".inst 0x45c69854 // ummla z20.s, z2.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45c79809 // ummla z9.s, z0.b, z7.b\n" - ".inst 0x45c79851 // ummla z17.s, z2.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" + "ld1rqb { z27.b }, p0/Z, [x23]\n" + "trn2 z1.d, z1.d, z24.d\n" + "trn1 z26.d, z3.d, z27.d\n" + "ld1b { z25.b }, p5/Z, [x10]\n" + "ld1b { z24.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45d99b88 // ummla z8.s, z28.b, z25.b\n" + ".inst 0x45d99b50 // ummla z16.s, z26.b, z25.b\n" + ".inst 0x45d89b8c // ummla z12.s, z28.b, z24.b\n" + ".inst 0x45d89b54 // ummla z20.s, z26.b, z24.b\n" + "ld1b { z25.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z24.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45d99b89 // ummla z9.s, z28.b, z25.b\n" + ".inst 0x45d99b51 // ummla z17.s, z26.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #4, MUL VL]\n" "subs x27, x27, #0x8\n" - ".inst 0x45c6980d // ummla z13.s, z0.b, z6.b\n" - ".inst 0x45c69855 // ummla z21.s, z2.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - "trn2 z3.d, z3.d, z4.d\n" - ".inst 0x45c7980a // ummla z10.s, z0.b, z7.b\n" - ".inst 0x45c79852 // ummla z18.s, z2.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x45c6980e // ummla z14.s, z0.b, z6.b\n" - ".inst 0x45c69856 // ummla z22.s, z2.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x45d89b8d // ummla z13.s, z28.b, z24.b\n" + ".inst 0x45d89b55 // ummla z21.s, z26.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #5, MUL VL]\n" + "trn2 z3.d, z3.d, z27.d\n" + ".inst 0x45d99b8a // ummla z10.s, z28.b, z25.b\n" + ".inst 0x45d99b52 // ummla z18.s, z26.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x45d89b8e // ummla z14.s, z28.b, z24.b\n" + ".inst 0x45d89b56 // ummla z22.s, z26.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - ".inst 0x45c7980b // ummla z11.s, z0.b, z7.b\n" - ".inst 0x45c79853 // ummla z19.s, z2.b, z7.b\n" - ".inst 0x45c6980f // ummla z15.s, z0.b, z6.b\n" - ".inst 0x45c69857 // ummla z23.s, z2.b, z6.b\n" + ".inst 0x45d99b8b // ummla z11.s, z28.b, z25.b\n" + ".inst 0x45d99b53 // ummla z19.s, z26.b, z25.b\n" + ".inst 0x45d89b8f // ummla z15.s, z28.b, z24.b\n" + ".inst 0x45d89b57 // ummla z23.s, z26.b, z24.b\n" "ble 43f\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45c79828 // ummla z8.s, z1.b, z7.b\n" - ".inst 0x45c79870 // ummla z16.s, z3.b, z7.b\n" - ".inst 0x45c6982c // ummla z12.s, z1.b, z6.b\n" - ".inst 0x45c69874 // ummla z20.s, z3.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45c79829 // ummla z9.s, z1.b, z7.b\n" - ".inst 0x45c79871 // ummla z17.s, z3.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x45c6982d // ummla z13.s, z1.b, z6.b\n" - ".inst 0x45c69875 // ummla z21.s, z3.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x45c7982a // ummla z10.s, z1.b, z7.b\n" - ".inst 0x45c79872 // ummla z18.s, z3.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x45c6982e // ummla z14.s, z1.b, z6.b\n" - ".inst 0x45c69876 // ummla z22.s, z3.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + "ld1b { z25.b }, p5/Z, [x10]\n" + "ld1b { z24.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45d99828 // ummla z8.s, z1.b, z25.b\n" + ".inst 0x45d99870 // ummla z16.s, z3.b, z25.b\n" + ".inst 0x45d8982c // ummla z12.s, z1.b, z24.b\n" + ".inst 0x45d89874 // ummla z20.s, z3.b, z24.b\n" + "ld1b { z25.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1b { z24.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45d99829 // ummla z9.s, z1.b, z25.b\n" + ".inst 0x45d99871 // ummla z17.s, z3.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x45d8982d // ummla z13.s, z1.b, z24.b\n" + ".inst 0x45d89875 // ummla z21.s, z3.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x45d9982a // ummla z10.s, z1.b, z25.b\n" + ".inst 0x45d99872 // ummla z18.s, z3.b, z25.b\n" + "ld1b { z25.b }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x45d8982e // ummla z14.s, z1.b, z24.b\n" + ".inst 0x45d89876 // ummla z22.s, z3.b, z24.b\n" + "ld1b { z24.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - ".inst 0x45c7982b // ummla z11.s, z1.b, z7.b\n" - ".inst 0x45c79873 // ummla z19.s, z3.b, z7.b\n" - ".inst 0x45c6982f // ummla z15.s, z1.b, z6.b\n" - ".inst 0x45c69877 // ummla z23.s, z3.b, z6.b\n" + ".inst 0x45d9982b // ummla z11.s, z1.b, z25.b\n" + ".inst 0x45d99873 // ummla z19.s, z3.b, z25.b\n" + ".inst 0x45d8982f // ummla z15.s, z1.b, z24.b\n" + ".inst 0x45d89877 // ummla z23.s, z3.b, z24.b\n" "43:" // Height 4: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 38b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "uzp1 z7.d, z8.d, z12.d\n" - "add x22, x23, x20, LSL #2\n" + "add x22, x9, x20, LSL #2\n" + "add x21, x22, x20, LSL #2\n" + "uzp1 z25.d, z8.d, z12.d\n" + "add x20, x21, x20, LSL #2\n" "uzp2 z8.d, z8.d, z12.d\n" - "uzp1 z12.d, z9.d, z13.d\n" - "st1w { z7.s }, p4, [x9]\n" + "uzp1 z24.d, z9.d, z13.d\n" + "st1w { z25.s }, p4, [x9]\n" "uzp2 z9.d, z9.d, z13.d\n" - "uzp1 z13.d, z10.d, z14.d\n" - "st1w { z12.s }, p3, [x9, #1, MUL VL]\n" + "uzp1 z25.d, z10.d, z14.d\n" + "st1w { z24.s }, p3, [x9, #1, MUL VL]\n" "uzp2 z10.d, z10.d, z14.d\n" - "uzp1 z14.d, z11.d, z15.d\n" - "st1w { z13.s }, p2, [x9, #2, MUL VL]\n" + "uzp1 z24.d, z11.d, z15.d\n" + "st1w { z25.s }, p2, [x9, #2, MUL VL]\n" "uzp2 z11.d, z11.d, z15.d\n" - "uzp1 z15.d, z16.d, z20.d\n" - "st1w { z14.s }, p1, [x9, #3, MUL VL]\n" + "uzp1 z25.d, z16.d, z20.d\n" + "st1w { z24.s }, p1, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" "uzp2 z16.d, z16.d, z20.d\n" - "uzp1 z20.d, z17.d, z21.d\n" - "st1w { z8.s }, p4, [x24]\n" + "uzp1 z24.d, z17.d, z21.d\n" + "st1w { z8.s }, p4, [x22]\n" "uzp2 z17.d, z17.d, z21.d\n" "uzp1 z21.d, z18.d, z22.d\n" - "st1w { z9.s }, p3, [x24, #1, MUL VL]\n" + "st1w { z9.s }, p3, [x22, #1, MUL VL]\n" "uzp2 z18.d, z18.d, z22.d\n" - "uzp1 z22.d, z19.d, z23.d\n" - "st1w { z10.s }, p2, [x24, #2, MUL VL]\n" + "uzp1 z20.d, z19.d, z23.d\n" + "st1w { z10.s }, p2, [x22, #2, MUL VL]\n" "uzp2 z19.d, z19.d, z23.d\n" - "st1w { z11.s }, p1, [x24, #3, MUL VL]\n" - "st1w { z15.s }, p4, [x23]\n" - "st1w { z20.s }, p3, [x23, #1, MUL VL]\n" - "st1w { z21.s }, p2, [x23, #2, MUL VL]\n" - "st1w { z22.s }, p1, [x23, #3, MUL VL]\n" - "st1w { z16.s }, p4, [x22]\n" - "st1w { z17.s }, p3, [x22, #1, MUL VL]\n" - "st1w { z18.s }, p2, [x22, #2, MUL VL]\n" - "st1w { z19.s }, p1, [x22, #3, MUL VL]\n" + "st1w { z11.s }, p1, [x22, #3, MUL VL]\n" + "st1w { z25.s }, p4, [x21]\n" + "st1w { z24.s }, p3, [x21, #1, MUL VL]\n" + "st1w { z21.s }, p2, [x21, #2, MUL VL]\n" + "st1w { z20.s }, p1, [x21, #3, MUL VL]\n" + "st1w { z16.s }, p4, [x20]\n" + "st1w { z17.s }, p3, [x20, #1, MUL VL]\n" + "st1w { z18.s }, p2, [x20, #2, MUL VL]\n" + "st1w { z19.s }, p1, [x20, #3, MUL VL]\n" "44:" // Height 4: Writeback done "decw x11, ALL, MUL #4\n" "cmp x11, XZR\n" @@ -959,54 +959,54 @@ void sve_hybrid_u8u32_mmla_6x4VL ( "whilelt p1.s, x20, x11\n" "tbz %x[flags], #0, 47f\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x9]\n" + "add x23, x9, x20, LSL #2\n" "add x22, x23, x20, LSL #2\n" + "ld1w { z19.s }, p4/Z, [x9]\n" "add x21, x22, x20, LSL #2\n" - "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n" + "add x20, x21, x20, LSL #2\n" + "ld1w { z17.s }, p3/Z, [x9, #1, MUL VL]\n" + "ld1w { z18.s }, p2/Z, [x9, #2, MUL VL]\n" "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n" - "ld1w { z12.s }, p4/Z, [x24]\n" - "zip1 z8.d, z9.d, z12.d\n" - "zip2 z12.d, z9.d, z12.d\n" - "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n" - "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n" - "zip1 z9.d, z10.d, z13.d\n" - "zip2 z13.d, z10.d, z13.d\n" - "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n" - "ld1w { z17.s }, p4/Z, [x23]\n" - "zip1 z10.d, z11.d, z14.d\n" - "zip2 z14.d, z11.d, z14.d\n" - "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n" - "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n" + "ld1w { z12.s }, p4/Z, [x23]\n" + "zip1 z8.d, z19.d, z12.d\n" + "zip2 z12.d, z19.d, z12.d\n" + "ld1w { z13.s }, p3/Z, [x23, #1, MUL VL]\n" + "ld1w { z14.s }, p2/Z, [x23, #2, MUL VL]\n" + "zip1 z9.d, z17.d, z13.d\n" + "zip2 z13.d, z17.d, z13.d\n" + "ld1w { z15.s }, p1/Z, [x23, #3, MUL VL]\n" + "ld1w { z17.s }, p4/Z, [x22]\n" + "zip1 z10.d, z18.d, z14.d\n" + "zip2 z14.d, z18.d, z14.d\n" + "ld1w { z18.s }, p3/Z, [x22, #1, MUL VL]\n" + "ld1w { z19.s }, p2/Z, [x22, #2, MUL VL]\n" "zip1 z11.d, z16.d, z15.d\n" "zip2 z15.d, z16.d, z15.d\n" - "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n" - "ld1w { z20.s }, p4/Z, [x22]\n" + "ld1w { z24.s }, p1/Z, [x22, #3, MUL VL]\n" + "ld1w { z20.s }, p4/Z, [x21]\n" "zip1 z16.d, z17.d, z20.d\n" "zip2 z20.d, z17.d, z20.d\n" - "ld1w { z21.s }, p3/Z, [x22, #1, MUL VL]\n" - "ld1w { z22.s }, p2/Z, [x22, #2, MUL VL]\n" + "ld1w { z21.s }, p3/Z, [x21, #1, MUL VL]\n" + "ld1w { z22.s }, p2/Z, [x21, #2, MUL VL]\n" "zip1 z17.d, z18.d, z21.d\n" "zip2 z21.d, z18.d, z21.d\n" - "ld1w { z23.s }, p1/Z, [x22, #3, MUL VL]\n" - "ld1w { z25.s }, p4/Z, [x21]\n" + "ld1w { z23.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z25.s }, p4/Z, [x20]\n" "zip1 z18.d, z19.d, z22.d\n" "zip2 z22.d, z19.d, z22.d\n" - "ld1w { z26.s }, p3/Z, [x21, #1, MUL VL]\n" - "ld1w { z27.s }, p2/Z, [x21, #2, MUL VL]\n" + "ld1w { z26.s }, p3/Z, [x20, #1, MUL VL]\n" + "ld1w { z27.s }, p2/Z, [x20, #2, MUL VL]\n" "zip1 z19.d, z24.d, z23.d\n" "zip2 z23.d, z24.d, z23.d\n" - "ld1w { z6.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z0.s }, p1/Z, [x20, #3, MUL VL]\n" "zip1 z24.d, z25.d, z28.d\n" "zip2 z28.d, z25.d, z28.d\n" "zip1 z25.d, z26.d, z29.d\n" "zip2 z29.d, z26.d, z29.d\n" "zip1 z26.d, z27.d, z30.d\n" "zip2 z30.d, z27.d, z30.d\n" - "zip1 z27.d, z6.d, z31.d\n" - "zip2 z31.d, z6.d, z31.d\n" + "zip1 z27.d, z0.d, z31.d\n" + "zip2 z31.d, z0.d, z31.d\n" "b 48f\n" "47:" // Height 5: no accumulate "mov z8.s, #0x0\n" @@ -1038,15 +1038,15 @@ void sve_hybrid_u8u32_mmla_6x4VL ( "49:" // Height 5: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 50f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" "cbnz x28, 51f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -1057,231 +1057,231 @@ void sve_hybrid_u8u32_mmla_6x4VL ( "b 51f\n" "50:" // Height 5: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" "51:" // Height 5: input setup done "cmp x27, #0x10\n" "ble 53f\n" "52:" // Height 5: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" - "ld1rqb { z3.b }, p0/Z, [x24]\n" - "ld1rqb { z4.b }, p0/Z, [x23]\n" - "trn1 z0.d, z1.d, z2.d\n" - "trn2 z1.d, z1.d, z2.d\n" - "ld1rqb { z5.b }, p0/Z, [x22]\n" - "trn1 z2.d, z3.d, z4.d\n" - "trn2 z3.d, z3.d, z4.d\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "trn1 z4.d, z5.d, z6.d\n" - "trn2 z5.d, z5.d, z6.d\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45c79808 // ummla z8.s, z0.b, z7.b\n" - ".inst 0x45c79850 // ummla z16.s, z2.b, z7.b\n" - ".inst 0x45c79898 // ummla z24.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1rqb { z6.b }, p0/Z, [x26]\n" + "ld1rqb { z1.b }, p0/Z, [x25]\n" + "ld1rqb { z7.b }, p0/Z, [x24]\n" + "ld1rqb { z2.b }, p0/Z, [x23]\n" + "trn1 z5.d, z6.d, z1.d\n" + "trn2 z6.d, z6.d, z1.d\n" + "ld1rqb { z4.b }, p0/Z, [x22]\n" + "trn1 z3.d, z7.d, z2.d\n" + "trn2 z7.d, z7.d, z2.d\n" + "ld1b { z1.b }, p5/Z, [x10]\n" + "trn1 z2.d, z4.d, z0.d\n" + "trn2 z4.d, z4.d, z0.d\n" + "ld1b { z0.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45c198a8 // ummla z8.s, z5.b, z1.b\n" + ".inst 0x45c19870 // ummla z16.s, z3.b, z1.b\n" + ".inst 0x45c19858 // ummla z24.s, z2.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #2, MUL VL]\n" "sub x27, x27, #0x10\n" - ".inst 0x45c6980c // ummla z12.s, z0.b, z6.b\n" - ".inst 0x45c69854 // ummla z20.s, z2.b, z6.b\n" + ".inst 0x45c098ac // ummla z12.s, z5.b, z0.b\n" + ".inst 0x45c09874 // ummla z20.s, z3.b, z0.b\n" "cmp x27, #0x10\n" "add x26, x26, #0x10\n" - ".inst 0x45c6989c // ummla z28.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45c79809 // ummla z9.s, z0.b, z7.b\n" + ".inst 0x45c0985c // ummla z28.s, z2.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45c198a9 // ummla z9.s, z5.b, z1.b\n" "add x25, x25, #0x10\n" - ".inst 0x45c79851 // ummla z17.s, z2.b, z7.b\n" - ".inst 0x45c79899 // ummla z25.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x45c19871 // ummla z17.s, z3.b, z1.b\n" + ".inst 0x45c19859 // ummla z25.s, z2.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #4, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x45c6980d // ummla z13.s, z0.b, z6.b\n" - ".inst 0x45c69855 // ummla z21.s, z2.b, z6.b\n" + ".inst 0x45c098ad // ummla z13.s, z5.b, z0.b\n" + ".inst 0x45c09875 // ummla z21.s, z3.b, z0.b\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - ".inst 0x45c6989d // ummla z29.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x45c7980a // ummla z10.s, z0.b, z7.b\n" - ".inst 0x45c79852 // ummla z18.s, z2.b, z7.b\n" - ".inst 0x45c7989a // ummla z26.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x45c6980e // ummla z14.s, z0.b, z6.b\n" - ".inst 0x45c69856 // ummla z22.s, z2.b, z6.b\n" - ".inst 0x45c6989e // ummla z30.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x45c0985d // ummla z29.s, z2.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x45c198aa // ummla z10.s, z5.b, z1.b\n" + ".inst 0x45c19872 // ummla z18.s, z3.b, z1.b\n" + ".inst 0x45c1985a // ummla z26.s, z2.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x45c098ae // ummla z14.s, z5.b, z0.b\n" + ".inst 0x45c09876 // ummla z22.s, z3.b, z0.b\n" + ".inst 0x45c0985e // ummla z30.s, z2.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - ".inst 0x45c7980b // ummla z11.s, z0.b, z7.b\n" - ".inst 0x45c79853 // ummla z19.s, z2.b, z7.b\n" - ".inst 0x45c7989b // ummla z27.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-8, MUL VL]\n" - ".inst 0x45c6980f // ummla z15.s, z0.b, z6.b\n" - ".inst 0x45c69857 // ummla z23.s, z2.b, z6.b\n" - ".inst 0x45c6989f // ummla z31.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-7, MUL VL]\n" - ".inst 0x45c79828 // ummla z8.s, z1.b, z7.b\n" - ".inst 0x45c79870 // ummla z16.s, z3.b, z7.b\n" - ".inst 0x45c798b8 // ummla z24.s, z5.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-6, MUL VL]\n" - ".inst 0x45c6982c // ummla z12.s, z1.b, z6.b\n" - ".inst 0x45c69874 // ummla z20.s, z3.b, z6.b\n" - ".inst 0x45c698bc // ummla z28.s, z5.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-5, MUL VL]\n" - ".inst 0x45c79829 // ummla z9.s, z1.b, z7.b\n" - ".inst 0x45c79871 // ummla z17.s, z3.b, z7.b\n" - ".inst 0x45c798b9 // ummla z25.s, z5.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-4, MUL VL]\n" - ".inst 0x45c6982d // ummla z13.s, z1.b, z6.b\n" - ".inst 0x45c69875 // ummla z21.s, z3.b, z6.b\n" - ".inst 0x45c698bd // ummla z29.s, z5.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-3, MUL VL]\n" - ".inst 0x45c7982a // ummla z10.s, z1.b, z7.b\n" - ".inst 0x45c79872 // ummla z18.s, z3.b, z7.b\n" - ".inst 0x45c798ba // ummla z26.s, z5.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-2, MUL VL]\n" - ".inst 0x45c6982e // ummla z14.s, z1.b, z6.b\n" - ".inst 0x45c69876 // ummla z22.s, z3.b, z6.b\n" - ".inst 0x45c698be // ummla z30.s, z5.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-1, MUL VL]\n" - ".inst 0x45c7982b // ummla z11.s, z1.b, z7.b\n" - ".inst 0x45c79873 // ummla z19.s, z3.b, z7.b\n" - ".inst 0x45c798bb // ummla z27.s, z5.b, z7.b\n" - ".inst 0x45c6982f // ummla z15.s, z1.b, z6.b\n" - ".inst 0x45c69877 // ummla z23.s, z3.b, z6.b\n" - ".inst 0x45c698bf // ummla z31.s, z5.b, z6.b\n" + ".inst 0x45c198ab // ummla z11.s, z5.b, z1.b\n" + ".inst 0x45c19873 // ummla z19.s, z3.b, z1.b\n" + ".inst 0x45c1985b // ummla z27.s, z2.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #-8, MUL VL]\n" + ".inst 0x45c098af // ummla z15.s, z5.b, z0.b\n" + ".inst 0x45c09877 // ummla z23.s, z3.b, z0.b\n" + ".inst 0x45c0985f // ummla z31.s, z2.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #-7, MUL VL]\n" + ".inst 0x45c198c8 // ummla z8.s, z6.b, z1.b\n" + ".inst 0x45c198f0 // ummla z16.s, z7.b, z1.b\n" + ".inst 0x45c19898 // ummla z24.s, z4.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #-6, MUL VL]\n" + ".inst 0x45c098cc // ummla z12.s, z6.b, z0.b\n" + ".inst 0x45c098f4 // ummla z20.s, z7.b, z0.b\n" + ".inst 0x45c0989c // ummla z28.s, z4.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #-5, MUL VL]\n" + ".inst 0x45c198c9 // ummla z9.s, z6.b, z1.b\n" + ".inst 0x45c198f1 // ummla z17.s, z7.b, z1.b\n" + ".inst 0x45c19899 // ummla z25.s, z4.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #-4, MUL VL]\n" + ".inst 0x45c098cd // ummla z13.s, z6.b, z0.b\n" + ".inst 0x45c098f5 // ummla z21.s, z7.b, z0.b\n" + ".inst 0x45c0989d // ummla z29.s, z4.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #-3, MUL VL]\n" + ".inst 0x45c198ca // ummla z10.s, z6.b, z1.b\n" + ".inst 0x45c198f2 // ummla z18.s, z7.b, z1.b\n" + ".inst 0x45c1989a // ummla z26.s, z4.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #-2, MUL VL]\n" + ".inst 0x45c098ce // ummla z14.s, z6.b, z0.b\n" + ".inst 0x45c098f6 // ummla z22.s, z7.b, z0.b\n" + ".inst 0x45c0989e // ummla z30.s, z4.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #-1, MUL VL]\n" + ".inst 0x45c198cb // ummla z11.s, z6.b, z1.b\n" + ".inst 0x45c198f3 // ummla z19.s, z7.b, z1.b\n" + ".inst 0x45c1989b // ummla z27.s, z4.b, z1.b\n" + ".inst 0x45c098cf // ummla z15.s, z6.b, z0.b\n" + ".inst 0x45c098f7 // ummla z23.s, z7.b, z0.b\n" + ".inst 0x45c0989f // ummla z31.s, z4.b, z0.b\n" "bgt 52b\n" "53:" // Height 5: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" + "ld1rqb { z4.b }, p0/Z, [x25]\n" "ld1rqb { z3.b }, p0/Z, [x24]\n" - "ld1rqb { z4.b }, p0/Z, [x23]\n" - "trn1 z0.d, z1.d, z2.d\n" - "trn2 z1.d, z1.d, z2.d\n" + "ld1rqb { z2.b }, p0/Z, [x23]\n" + "trn1 z7.d, z1.d, z4.d\n" + "trn2 z1.d, z1.d, z4.d\n" "ld1rqb { z5.b }, p0/Z, [x22]\n" - "trn1 z2.d, z3.d, z4.d\n" - "trn2 z3.d, z3.d, z4.d\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "trn1 z4.d, z5.d, z6.d\n" - "trn2 z5.d, z5.d, z6.d\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45c79808 // ummla z8.s, z0.b, z7.b\n" - ".inst 0x45c79850 // ummla z16.s, z2.b, z7.b\n" - ".inst 0x45c79898 // ummla z24.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" + "trn1 z6.d, z3.d, z2.d\n" + "trn2 z3.d, z3.d, z2.d\n" + "ld1b { z2.b }, p5/Z, [x10]\n" + "trn1 z4.d, z5.d, z0.d\n" + "trn2 z5.d, z5.d, z0.d\n" + "ld1b { z0.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45c298e8 // ummla z8.s, z7.b, z2.b\n" + ".inst 0x45c298d0 // ummla z16.s, z6.b, z2.b\n" + ".inst 0x45c29898 // ummla z24.s, z4.b, z2.b\n" + "ld1b { z2.b }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x8\n" - ".inst 0x45c6980c // ummla z12.s, z0.b, z6.b\n" - ".inst 0x45c69854 // ummla z20.s, z2.b, z6.b\n" - ".inst 0x45c6989c // ummla z28.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45c79809 // ummla z9.s, z0.b, z7.b\n" - ".inst 0x45c79851 // ummla z17.s, z2.b, z7.b\n" - ".inst 0x45c79899 // ummla z25.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x45c6980d // ummla z13.s, z0.b, z6.b\n" - ".inst 0x45c69855 // ummla z21.s, z2.b, z6.b\n" - ".inst 0x45c6989d // ummla z29.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x45c7980a // ummla z10.s, z0.b, z7.b\n" - ".inst 0x45c79852 // ummla z18.s, z2.b, z7.b\n" - ".inst 0x45c7989a // ummla z26.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x45c6980e // ummla z14.s, z0.b, z6.b\n" - ".inst 0x45c69856 // ummla z22.s, z2.b, z6.b\n" - ".inst 0x45c6989e // ummla z30.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" - ".inst 0x45c7980b // ummla z11.s, z0.b, z7.b\n" + ".inst 0x45c098ec // ummla z12.s, z7.b, z0.b\n" + ".inst 0x45c098d4 // ummla z20.s, z6.b, z0.b\n" + ".inst 0x45c0989c // ummla z28.s, z4.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45c298e9 // ummla z9.s, z7.b, z2.b\n" + ".inst 0x45c298d1 // ummla z17.s, z6.b, z2.b\n" + ".inst 0x45c29899 // ummla z25.s, z4.b, z2.b\n" + "ld1b { z2.b }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x45c098ed // ummla z13.s, z7.b, z0.b\n" + ".inst 0x45c098d5 // ummla z21.s, z6.b, z0.b\n" + ".inst 0x45c0989d // ummla z29.s, z4.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x45c298ea // ummla z10.s, z7.b, z2.b\n" + ".inst 0x45c298d2 // ummla z18.s, z6.b, z2.b\n" + ".inst 0x45c2989a // ummla z26.s, z4.b, z2.b\n" + "ld1b { z2.b }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x45c098ee // ummla z14.s, z7.b, z0.b\n" + ".inst 0x45c098d6 // ummla z22.s, z6.b, z0.b\n" + ".inst 0x45c0989e // ummla z30.s, z4.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x45c298eb // ummla z11.s, z7.b, z2.b\n" "addvl x10, x10, #8\n" - ".inst 0x45c79853 // ummla z19.s, z2.b, z7.b\n" - ".inst 0x45c7989b // ummla z27.s, z4.b, z7.b\n" - ".inst 0x45c6980f // ummla z15.s, z0.b, z6.b\n" - ".inst 0x45c69857 // ummla z23.s, z2.b, z6.b\n" - ".inst 0x45c6989f // ummla z31.s, z4.b, z6.b\n" + ".inst 0x45c298d3 // ummla z19.s, z6.b, z2.b\n" + ".inst 0x45c2989b // ummla z27.s, z4.b, z2.b\n" + ".inst 0x45c098ef // ummla z15.s, z7.b, z0.b\n" + ".inst 0x45c098d7 // ummla z23.s, z6.b, z0.b\n" + ".inst 0x45c0989f // ummla z31.s, z4.b, z0.b\n" "ble 54f\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45c79828 // ummla z8.s, z1.b, z7.b\n" - ".inst 0x45c79870 // ummla z16.s, z3.b, z7.b\n" - ".inst 0x45c798b8 // ummla z24.s, z5.b, z7.b\n" - ".inst 0x45c6982c // ummla z12.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x45c69874 // ummla z20.s, z3.b, z6.b\n" - ".inst 0x45c698bc // ummla z28.s, z5.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45c79829 // ummla z9.s, z1.b, z7.b\n" - ".inst 0x45c79871 // ummla z17.s, z3.b, z7.b\n" - ".inst 0x45c798b9 // ummla z25.s, z5.b, z7.b\n" - ".inst 0x45c6982d // ummla z13.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x45c69875 // ummla z21.s, z3.b, z6.b\n" - ".inst 0x45c698bd // ummla z29.s, z5.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x45c7982a // ummla z10.s, z1.b, z7.b\n" - ".inst 0x45c79872 // ummla z18.s, z3.b, z7.b\n" - ".inst 0x45c798ba // ummla z26.s, z5.b, z7.b\n" - ".inst 0x45c6982e // ummla z14.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x45c69876 // ummla z22.s, z3.b, z6.b\n" - ".inst 0x45c698be // ummla z30.s, z5.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + "ld1b { z2.b }, p5/Z, [x10]\n" + "ld1b { z0.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45c29828 // ummla z8.s, z1.b, z2.b\n" + ".inst 0x45c29870 // ummla z16.s, z3.b, z2.b\n" + ".inst 0x45c298b8 // ummla z24.s, z5.b, z2.b\n" + ".inst 0x45c0982c // ummla z12.s, z1.b, z0.b\n" + "ld1b { z2.b }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x45c09874 // ummla z20.s, z3.b, z0.b\n" + ".inst 0x45c098bc // ummla z28.s, z5.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45c29829 // ummla z9.s, z1.b, z2.b\n" + ".inst 0x45c29871 // ummla z17.s, z3.b, z2.b\n" + ".inst 0x45c298b9 // ummla z25.s, z5.b, z2.b\n" + ".inst 0x45c0982d // ummla z13.s, z1.b, z0.b\n" + "ld1b { z2.b }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x45c09875 // ummla z21.s, z3.b, z0.b\n" + ".inst 0x45c098bd // ummla z29.s, z5.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x45c2982a // ummla z10.s, z1.b, z2.b\n" + ".inst 0x45c29872 // ummla z18.s, z3.b, z2.b\n" + ".inst 0x45c298ba // ummla z26.s, z5.b, z2.b\n" + ".inst 0x45c0982e // ummla z14.s, z1.b, z0.b\n" + "ld1b { z2.b }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x45c09876 // ummla z22.s, z3.b, z0.b\n" + ".inst 0x45c098be // ummla z30.s, z5.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - ".inst 0x45c7982b // ummla z11.s, z1.b, z7.b\n" - ".inst 0x45c79873 // ummla z19.s, z3.b, z7.b\n" - ".inst 0x45c798bb // ummla z27.s, z5.b, z7.b\n" - ".inst 0x45c6982f // ummla z15.s, z1.b, z6.b\n" - ".inst 0x45c69877 // ummla z23.s, z3.b, z6.b\n" - ".inst 0x45c698bf // ummla z31.s, z5.b, z6.b\n" + ".inst 0x45c2982b // ummla z11.s, z1.b, z2.b\n" + ".inst 0x45c29873 // ummla z19.s, z3.b, z2.b\n" + ".inst 0x45c298bb // ummla z27.s, z5.b, z2.b\n" + ".inst 0x45c0982f // ummla z15.s, z1.b, z0.b\n" + ".inst 0x45c09877 // ummla z23.s, z3.b, z0.b\n" + ".inst 0x45c098bf // ummla z31.s, z5.b, z0.b\n" "54:" // Height 5: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" "cmp x28, x20\n" "bne 49b\n" "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" - "add x24, x9, x20, LSL #2\n" - "add x23, x24, x20, LSL #2\n" - "uzp1 z7.d, z8.d, z12.d\n" + "add x23, x9, x20, LSL #2\n" "add x22, x23, x20, LSL #2\n" + "uzp1 z2.d, z8.d, z12.d\n" "add x21, x22, x20, LSL #2\n" + "add x20, x21, x20, LSL #2\n" "uzp2 z8.d, z8.d, z12.d\n" - "uzp1 z12.d, z9.d, z13.d\n" + "uzp1 z1.d, z9.d, z13.d\n" "uzp2 z9.d, z9.d, z13.d\n" - "uzp1 z13.d, z10.d, z14.d\n" - "st1w { z7.s }, p4, [x9]\n" + "uzp1 z0.d, z10.d, z14.d\n" + "st1w { z2.s }, p4, [x9]\n" "uzp2 z10.d, z10.d, z14.d\n" - "uzp1 z14.d, z11.d, z15.d\n" - "st1w { z12.s }, p3, [x9, #1, MUL VL]\n" + "uzp1 z2.d, z11.d, z15.d\n" + "st1w { z1.s }, p3, [x9, #1, MUL VL]\n" "uzp2 z11.d, z11.d, z15.d\n" - "uzp1 z15.d, z16.d, z20.d\n" - "st1w { z13.s }, p2, [x9, #2, MUL VL]\n" + "uzp1 z1.d, z16.d, z20.d\n" + "st1w { z0.s }, p2, [x9, #2, MUL VL]\n" "uzp2 z16.d, z16.d, z20.d\n" - "uzp1 z20.d, z17.d, z21.d\n" - "st1w { z14.s }, p1, [x9, #3, MUL VL]\n" + "uzp1 z0.d, z17.d, z21.d\n" + "st1w { z2.s }, p1, [x9, #3, MUL VL]\n" "addvl x9, x9, #4\n" "uzp2 z17.d, z17.d, z21.d\n" "uzp1 z21.d, z18.d, z22.d\n" - "st1w { z8.s }, p4, [x24]\n" + "st1w { z8.s }, p4, [x23]\n" "uzp2 z18.d, z18.d, z22.d\n" - "uzp1 z22.d, z19.d, z23.d\n" - "st1w { z9.s }, p3, [x24, #1, MUL VL]\n" + "uzp1 z20.d, z19.d, z23.d\n" + "st1w { z9.s }, p3, [x23, #1, MUL VL]\n" "uzp2 z19.d, z19.d, z23.d\n" "uzp1 z24.d, z24.d, z28.d\n" - "st1w { z10.s }, p2, [x24, #2, MUL VL]\n" + "st1w { z10.s }, p2, [x23, #2, MUL VL]\n" "uzp1 z25.d, z25.d, z29.d\n" "uzp1 z26.d, z26.d, z30.d\n" - "st1w { z11.s }, p1, [x24, #3, MUL VL]\n" + "st1w { z11.s }, p1, [x23, #3, MUL VL]\n" "uzp1 z27.d, z27.d, z31.d\n" - "st1w { z15.s }, p4, [x23]\n" - "st1w { z20.s }, p3, [x23, #1, MUL VL]\n" - "st1w { z21.s }, p2, [x23, #2, MUL VL]\n" - "st1w { z22.s }, p1, [x23, #3, MUL VL]\n" - "st1w { z16.s }, p4, [x22]\n" - "st1w { z17.s }, p3, [x22, #1, MUL VL]\n" - "st1w { z18.s }, p2, [x22, #2, MUL VL]\n" - "st1w { z19.s }, p1, [x22, #3, MUL VL]\n" - "st1w { z24.s }, p4, [x21]\n" - "st1w { z25.s }, p3, [x21, #1, MUL VL]\n" - "st1w { z26.s }, p2, [x21, #2, MUL VL]\n" - "st1w { z27.s }, p1, [x21, #3, MUL VL]\n" + "st1w { z1.s }, p4, [x22]\n" + "st1w { z0.s }, p3, [x22, #1, MUL VL]\n" + "st1w { z21.s }, p2, [x22, #2, MUL VL]\n" + "st1w { z20.s }, p1, [x22, #3, MUL VL]\n" + "st1w { z16.s }, p4, [x21]\n" + "st1w { z17.s }, p3, [x21, #1, MUL VL]\n" + "st1w { z18.s }, p2, [x21, #2, MUL VL]\n" + "st1w { z19.s }, p1, [x21, #3, MUL VL]\n" + "st1w { z24.s }, p4, [x20]\n" + "st1w { z25.s }, p3, [x20, #1, MUL VL]\n" + "st1w { z26.s }, p2, [x20, #2, MUL VL]\n" + "st1w { z27.s }, p1, [x20, #3, MUL VL]\n" "55:" // Height 5: Writeback done "decw x11, ALL, MUL #4\n" "cmp x11, XZR\n" @@ -1307,26 +1307,26 @@ void sve_hybrid_u8u32_mmla_6x4VL ( "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "add x24, x9, x20, LSL #2\n" "add x23, x24, x20, LSL #2\n" - "ld1w { z9.s }, p4/Z, [x9]\n" + "ld1w { z17.s }, p4/Z, [x9]\n" "add x22, x23, x20, LSL #2\n" "add x21, x22, x20, LSL #2\n" - "ld1w { z10.s }, p3/Z, [x9, #1, MUL VL]\n" - "ld1w { z11.s }, p2/Z, [x9, #2, MUL VL]\n" + "ld1w { z18.s }, p3/Z, [x9, #1, MUL VL]\n" + "ld1w { z20.s }, p2/Z, [x9, #2, MUL VL]\n" "add x20, x21, x20, LSL #2\n" "ld1w { z16.s }, p1/Z, [x9, #3, MUL VL]\n" "ld1w { z12.s }, p4/Z, [x24]\n" - "zip1 z8.d, z9.d, z12.d\n" + "zip1 z8.d, z17.d, z12.d\n" "ld1w { z13.s }, p3/Z, [x24, #1, MUL VL]\n" "ld1w { z14.s }, p2/Z, [x24, #2, MUL VL]\n" - "zip2 z12.d, z9.d, z12.d\n" - "zip1 z9.d, z10.d, z13.d\n" + "zip2 z12.d, z17.d, z12.d\n" + "zip1 z9.d, z18.d, z13.d\n" "ld1w { z15.s }, p1/Z, [x24, #3, MUL VL]\n" "ld1w { z17.s }, p4/Z, [x23]\n" - "zip2 z13.d, z10.d, z13.d\n" - "zip1 z10.d, z11.d, z14.d\n" + "zip2 z13.d, z18.d, z13.d\n" + "zip1 z10.d, z20.d, z14.d\n" "ld1w { z18.s }, p3/Z, [x23, #1, MUL VL]\n" "ld1w { z19.s }, p2/Z, [x23, #2, MUL VL]\n" - "zip2 z14.d, z11.d, z14.d\n" + "zip2 z14.d, z20.d, z14.d\n" "zip1 z11.d, z16.d, z15.d\n" "ld1w { z24.s }, p1/Z, [x23, #3, MUL VL]\n" "ld1w { z20.s }, p4/Z, [x22]\n" @@ -1344,7 +1344,7 @@ void sve_hybrid_u8u32_mmla_6x4VL ( "ld1w { z27.s }, p2/Z, [x21, #2, MUL VL]\n" "zip2 z22.d, z19.d, z22.d\n" "zip1 z19.d, z24.d, z23.d\n" - "ld1w { z6.s }, p1/Z, [x21, #3, MUL VL]\n" + "ld1w { z0.s }, p1/Z, [x21, #3, MUL VL]\n" "ld1w { z28.s }, p4/Z, [x20]\n" "zip2 z23.d, z24.d, z23.d\n" "zip1 z24.d, z25.d, z28.d\n" @@ -1356,8 +1356,8 @@ void sve_hybrid_u8u32_mmla_6x4VL ( "zip2 z29.d, z26.d, z29.d\n" "zip1 z26.d, z27.d, z30.d\n" "zip2 z30.d, z27.d, z30.d\n" - "zip1 z27.d, z6.d, z31.d\n" - "zip2 z31.d, z6.d, z31.d\n" + "zip1 z27.d, z0.d, z31.d\n" + "zip2 z31.d, z0.d, z31.d\n" "b 59f\n" "58:" // Height 6: no accumulate "mov z8.s, #0x0\n" @@ -1389,16 +1389,16 @@ void sve_hybrid_u8u32_mmla_6x4VL ( "60:" // Height 6: String loop "ldr x20, [%x[args_ptr], %[offsetof_string_lengths]]\n" "ldr w27, [x20, x28, LSL #0x2]\n" - "ldr x20, [%x[args_ptr], %[offsetof_input_offset]]\n" + "ldr x21, [%x[args_ptr], %[offsetof_input_offset]]\n" "tbz %x[flags], #3, 61f\n" - "ldr x21, [%x[input_ptr], x28, LSL #0x3]\n" - "add x21, x21, x20, LSL #3\n" - "ldr x26, [x21, #0x0]\n" - "ldr x25, [x21, #0x8]\n" - "ldr x24, [x21, #0x10]\n" - "ldr x23, [x21, #0x18]\n" - "ldr x22, [x21, #0x20]\n" - "ldr x21, [x21, #0x28]\n" + "ldr x20, [%x[input_ptr], x28, LSL #0x3]\n" + "add x20, x20, x21, LSL #3\n" + "ldr x26, [x20, #0x0]\n" + "ldr x25, [x20, #0x8]\n" + "ldr x24, [x20, #0x10]\n" + "ldr x23, [x20, #0x18]\n" + "ldr x22, [x20, #0x20]\n" + "ldr x21, [x20, #0x28]\n" "cbnz x28, 62f\n" "ldr x20, [%x[args_ptr], %[offsetof_input_initial_col]]\n" "add x26, x26, x20\n" @@ -1410,184 +1410,184 @@ void sve_hybrid_u8u32_mmla_6x4VL ( "b 62f\n" "61:" // Height 6: setup direct input "mov x26, %x[input_ptr]\n" - "add x25, x26, x20\n" - "add x24, x25, x20\n" - "add x23, x24, x20\n" - "add x22, x23, x20\n" - "add x21, x22, x20\n" + "add x25, x26, x21\n" + "add x24, x25, x21\n" + "add x23, x24, x21\n" + "add x22, x23, x21\n" + "add x21, x22, x21\n" "62:" // Height 6: input setup done "cmp x27, #0x10\n" "ble 64f\n" "63:" // Height 6: Multiply loop: Main loop head "whilelt p0.b, XZR, x27\n" - "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" - "ld1rqb { z3.b }, p0/Z, [x24]\n" - "ld1rqb { z4.b }, p0/Z, [x23]\n" - "trn2 z1.d, z1.d, z2.d\n" - "trn1 z2.d, z3.d, z4.d\n" - "ld1rqb { z5.b }, p0/Z, [x22]\n" - "ld1rqb { z6.b }, p0/Z, [x21]\n" - "trn2 z3.d, z3.d, z4.d\n" - "trn1 z4.d, z5.d, z6.d\n" - "trn2 z5.d, z5.d, z6.d\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45c79808 // ummla z8.s, z0.b, z7.b\n" - ".inst 0x45c79850 // ummla z16.s, z2.b, z7.b\n" - ".inst 0x45c79898 // ummla z24.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1rqb { z7.b }, p0/Z, [x26]\n" + "ld1rqb { z0.b }, p0/Z, [x25]\n" + "trn1 z6.d, z7.d, z0.d\n" + "ld1rqb { z5.b }, p0/Z, [x24]\n" + "ld1rqb { z1.b }, p0/Z, [x23]\n" + "trn2 z7.d, z7.d, z0.d\n" + "trn1 z4.d, z5.d, z1.d\n" + "ld1rqb { z3.b }, p0/Z, [x22]\n" + "ld1rqb { z0.b }, p0/Z, [x21]\n" + "trn2 z5.d, z5.d, z1.d\n" + "trn1 z2.d, z3.d, z0.d\n" + "trn2 z3.d, z3.d, z0.d\n" + "ld1b { z1.b }, p5/Z, [x10]\n" + "ld1b { z0.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45c198c8 // ummla z8.s, z6.b, z1.b\n" + ".inst 0x45c19890 // ummla z16.s, z4.b, z1.b\n" + ".inst 0x45c19858 // ummla z24.s, z2.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #2, MUL VL]\n" "sub x27, x27, #0x10\n" - ".inst 0x45c6980c // ummla z12.s, z0.b, z6.b\n" - ".inst 0x45c69854 // ummla z20.s, z2.b, z6.b\n" + ".inst 0x45c098cc // ummla z12.s, z6.b, z0.b\n" + ".inst 0x45c09894 // ummla z20.s, z4.b, z0.b\n" "cmp x27, #0x10\n" "add x26, x26, #0x10\n" - ".inst 0x45c6989c // ummla z28.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45c79809 // ummla z9.s, z0.b, z7.b\n" + ".inst 0x45c0985c // ummla z28.s, z2.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45c198c9 // ummla z9.s, z6.b, z1.b\n" "add x25, x25, #0x10\n" - ".inst 0x45c79851 // ummla z17.s, z2.b, z7.b\n" - ".inst 0x45c79899 // ummla z25.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x45c19891 // ummla z17.s, z4.b, z1.b\n" + ".inst 0x45c19859 // ummla z25.s, z2.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #4, MUL VL]\n" "add x24, x24, #0x10\n" - ".inst 0x45c6980d // ummla z13.s, z0.b, z6.b\n" - ".inst 0x45c69855 // ummla z21.s, z2.b, z6.b\n" + ".inst 0x45c098cd // ummla z13.s, z6.b, z0.b\n" + ".inst 0x45c09895 // ummla z21.s, z4.b, z0.b\n" "add x23, x23, #0x10\n" "add x22, x22, #0x10\n" - ".inst 0x45c6989d // ummla z29.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x45c7980a // ummla z10.s, z0.b, z7.b\n" + ".inst 0x45c0985d // ummla z29.s, z2.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x45c198ca // ummla z10.s, z6.b, z1.b\n" "add x21, x21, #0x10\n" - ".inst 0x45c79852 // ummla z18.s, z2.b, z7.b\n" - ".inst 0x45c7989a // ummla z26.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x45c6980e // ummla z14.s, z0.b, z6.b\n" - ".inst 0x45c69856 // ummla z22.s, z2.b, z6.b\n" - ".inst 0x45c6989e // ummla z30.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x45c19892 // ummla z18.s, z4.b, z1.b\n" + ".inst 0x45c1985a // ummla z26.s, z2.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x45c098ce // ummla z14.s, z6.b, z0.b\n" + ".inst 0x45c09896 // ummla z22.s, z4.b, z0.b\n" + ".inst 0x45c0985e // ummla z30.s, z2.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #16\n" - ".inst 0x45c7980b // ummla z11.s, z0.b, z7.b\n" - ".inst 0x45c79853 // ummla z19.s, z2.b, z7.b\n" - ".inst 0x45c7989b // ummla z27.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-8, MUL VL]\n" - ".inst 0x45c6980f // ummla z15.s, z0.b, z6.b\n" - ".inst 0x45c69857 // ummla z23.s, z2.b, z6.b\n" - ".inst 0x45c6989f // ummla z31.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-7, MUL VL]\n" - ".inst 0x45c79828 // ummla z8.s, z1.b, z7.b\n" - ".inst 0x45c79870 // ummla z16.s, z3.b, z7.b\n" - ".inst 0x45c798b8 // ummla z24.s, z5.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-6, MUL VL]\n" - ".inst 0x45c6982c // ummla z12.s, z1.b, z6.b\n" - ".inst 0x45c69874 // ummla z20.s, z3.b, z6.b\n" - ".inst 0x45c698bc // ummla z28.s, z5.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-5, MUL VL]\n" - ".inst 0x45c79829 // ummla z9.s, z1.b, z7.b\n" - ".inst 0x45c79871 // ummla z17.s, z3.b, z7.b\n" - ".inst 0x45c798b9 // ummla z25.s, z5.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-4, MUL VL]\n" - ".inst 0x45c6982d // ummla z13.s, z1.b, z6.b\n" - ".inst 0x45c69875 // ummla z21.s, z3.b, z6.b\n" - ".inst 0x45c698bd // ummla z29.s, z5.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-3, MUL VL]\n" - ".inst 0x45c7982a // ummla z10.s, z1.b, z7.b\n" - ".inst 0x45c79872 // ummla z18.s, z3.b, z7.b\n" - ".inst 0x45c798ba // ummla z26.s, z5.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #-2, MUL VL]\n" - ".inst 0x45c6982e // ummla z14.s, z1.b, z6.b\n" - ".inst 0x45c69876 // ummla z22.s, z3.b, z6.b\n" - ".inst 0x45c698be // ummla z30.s, z5.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #-1, MUL VL]\n" - ".inst 0x45c7982b // ummla z11.s, z1.b, z7.b\n" - ".inst 0x45c79873 // ummla z19.s, z3.b, z7.b\n" - ".inst 0x45c798bb // ummla z27.s, z5.b, z7.b\n" - ".inst 0x45c6982f // ummla z15.s, z1.b, z6.b\n" - ".inst 0x45c69877 // ummla z23.s, z3.b, z6.b\n" - ".inst 0x45c698bf // ummla z31.s, z5.b, z6.b\n" + ".inst 0x45c198cb // ummla z11.s, z6.b, z1.b\n" + ".inst 0x45c19893 // ummla z19.s, z4.b, z1.b\n" + ".inst 0x45c1985b // ummla z27.s, z2.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #-8, MUL VL]\n" + ".inst 0x45c098cf // ummla z15.s, z6.b, z0.b\n" + ".inst 0x45c09897 // ummla z23.s, z4.b, z0.b\n" + ".inst 0x45c0985f // ummla z31.s, z2.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #-7, MUL VL]\n" + ".inst 0x45c198e8 // ummla z8.s, z7.b, z1.b\n" + ".inst 0x45c198b0 // ummla z16.s, z5.b, z1.b\n" + ".inst 0x45c19878 // ummla z24.s, z3.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #-6, MUL VL]\n" + ".inst 0x45c098ec // ummla z12.s, z7.b, z0.b\n" + ".inst 0x45c098b4 // ummla z20.s, z5.b, z0.b\n" + ".inst 0x45c0987c // ummla z28.s, z3.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #-5, MUL VL]\n" + ".inst 0x45c198e9 // ummla z9.s, z7.b, z1.b\n" + ".inst 0x45c198b1 // ummla z17.s, z5.b, z1.b\n" + ".inst 0x45c19879 // ummla z25.s, z3.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #-4, MUL VL]\n" + ".inst 0x45c098ed // ummla z13.s, z7.b, z0.b\n" + ".inst 0x45c098b5 // ummla z21.s, z5.b, z0.b\n" + ".inst 0x45c0987d // ummla z29.s, z3.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #-3, MUL VL]\n" + ".inst 0x45c198ea // ummla z10.s, z7.b, z1.b\n" + ".inst 0x45c198b2 // ummla z18.s, z5.b, z1.b\n" + ".inst 0x45c1987a // ummla z26.s, z3.b, z1.b\n" + "ld1b { z1.b }, p5/Z, [x10, #-2, MUL VL]\n" + ".inst 0x45c098ee // ummla z14.s, z7.b, z0.b\n" + ".inst 0x45c098b6 // ummla z22.s, z5.b, z0.b\n" + ".inst 0x45c0987e // ummla z30.s, z3.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #-1, MUL VL]\n" + ".inst 0x45c198eb // ummla z11.s, z7.b, z1.b\n" + ".inst 0x45c198b3 // ummla z19.s, z5.b, z1.b\n" + ".inst 0x45c1987b // ummla z27.s, z3.b, z1.b\n" + ".inst 0x45c098ef // ummla z15.s, z7.b, z0.b\n" + ".inst 0x45c098b7 // ummla z23.s, z5.b, z0.b\n" + ".inst 0x45c0987f // ummla z31.s, z3.b, z0.b\n" "bgt 63b\n" "64:" // Height 6: Multiply loop: Single iteration only "whilelt p0.b, XZR, x27\n" "ld1rqb { z1.b }, p0/Z, [x26]\n" - "ld1rqb { z2.b }, p0/Z, [x25]\n" - "trn1 z0.d, z1.d, z2.d\n" + "ld1rqb { z0.b }, p0/Z, [x25]\n" + "trn1 z7.d, z1.d, z0.d\n" "ld1rqb { z3.b }, p0/Z, [x24]\n" - "ld1rqb { z4.b }, p0/Z, [x23]\n" - "trn2 z1.d, z1.d, z2.d\n" - "trn1 z2.d, z3.d, z4.d\n" + "ld1rqb { z2.b }, p0/Z, [x23]\n" + "trn2 z1.d, z1.d, z0.d\n" + "trn1 z6.d, z3.d, z2.d\n" "ld1rqb { z5.b }, p0/Z, [x22]\n" - "ld1rqb { z6.b }, p0/Z, [x21]\n" - "trn2 z3.d, z3.d, z4.d\n" - "trn1 z4.d, z5.d, z6.d\n" - "trn2 z5.d, z5.d, z6.d\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45c79808 // ummla z8.s, z0.b, z7.b\n" - ".inst 0x45c79850 // ummla z16.s, z2.b, z7.b\n" - ".inst 0x45c79898 // ummla z24.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" + "ld1rqb { z0.b }, p0/Z, [x21]\n" + "trn2 z3.d, z3.d, z2.d\n" + "trn1 z4.d, z5.d, z0.d\n" + "trn2 z5.d, z5.d, z0.d\n" + "ld1b { z2.b }, p5/Z, [x10]\n" + "ld1b { z0.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45c298e8 // ummla z8.s, z7.b, z2.b\n" + ".inst 0x45c298d0 // ummla z16.s, z6.b, z2.b\n" + ".inst 0x45c29898 // ummla z24.s, z4.b, z2.b\n" + "ld1b { z2.b }, p5/Z, [x10, #2, MUL VL]\n" "subs x27, x27, #0x8\n" - ".inst 0x45c6980c // ummla z12.s, z0.b, z6.b\n" - ".inst 0x45c69854 // ummla z20.s, z2.b, z6.b\n" - ".inst 0x45c6989c // ummla z28.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45c79809 // ummla z9.s, z0.b, z7.b\n" - ".inst 0x45c79851 // ummla z17.s, z2.b, z7.b\n" - ".inst 0x45c79899 // ummla z25.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x45c6980d // ummla z13.s, z0.b, z6.b\n" - ".inst 0x45c69855 // ummla z21.s, z2.b, z6.b\n" - ".inst 0x45c6989d // ummla z29.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x45c7980a // ummla z10.s, z0.b, z7.b\n" - ".inst 0x45c79852 // ummla z18.s, z2.b, z7.b\n" - ".inst 0x45c7989a // ummla z26.s, z4.b, z7.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x45c6980e // ummla z14.s, z0.b, z6.b\n" - ".inst 0x45c69856 // ummla z22.s, z2.b, z6.b\n" - ".inst 0x45c6989e // ummla z30.s, z4.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" - ".inst 0x45c7980b // ummla z11.s, z0.b, z7.b\n" + ".inst 0x45c098ec // ummla z12.s, z7.b, z0.b\n" + ".inst 0x45c098d4 // ummla z20.s, z6.b, z0.b\n" + ".inst 0x45c0989c // ummla z28.s, z4.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45c298e9 // ummla z9.s, z7.b, z2.b\n" + ".inst 0x45c298d1 // ummla z17.s, z6.b, z2.b\n" + ".inst 0x45c29899 // ummla z25.s, z4.b, z2.b\n" + "ld1b { z2.b }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x45c098ed // ummla z13.s, z7.b, z0.b\n" + ".inst 0x45c098d5 // ummla z21.s, z6.b, z0.b\n" + ".inst 0x45c0989d // ummla z29.s, z4.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x45c298ea // ummla z10.s, z7.b, z2.b\n" + ".inst 0x45c298d2 // ummla z18.s, z6.b, z2.b\n" + ".inst 0x45c2989a // ummla z26.s, z4.b, z2.b\n" + "ld1b { z2.b }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x45c098ee // ummla z14.s, z7.b, z0.b\n" + ".inst 0x45c098d6 // ummla z22.s, z6.b, z0.b\n" + ".inst 0x45c0989e // ummla z30.s, z4.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #7, MUL VL]\n" + ".inst 0x45c298eb // ummla z11.s, z7.b, z2.b\n" "addvl x10, x10, #8\n" - ".inst 0x45c79853 // ummla z19.s, z2.b, z7.b\n" - ".inst 0x45c7989b // ummla z27.s, z4.b, z7.b\n" - ".inst 0x45c6980f // ummla z15.s, z0.b, z6.b\n" - ".inst 0x45c69857 // ummla z23.s, z2.b, z6.b\n" - ".inst 0x45c6989f // ummla z31.s, z4.b, z6.b\n" + ".inst 0x45c298d3 // ummla z19.s, z6.b, z2.b\n" + ".inst 0x45c2989b // ummla z27.s, z4.b, z2.b\n" + ".inst 0x45c098ef // ummla z15.s, z7.b, z0.b\n" + ".inst 0x45c098d7 // ummla z23.s, z6.b, z0.b\n" + ".inst 0x45c0989f // ummla z31.s, z4.b, z0.b\n" "ble 65f\n" - "ld1b { z7.b }, p5/Z, [x10]\n" - "ld1b { z6.b }, p5/Z, [x10, #1, MUL VL]\n" - ".inst 0x45c79828 // ummla z8.s, z1.b, z7.b\n" - ".inst 0x45c79870 // ummla z16.s, z3.b, z7.b\n" - ".inst 0x45c798b8 // ummla z24.s, z5.b, z7.b\n" - ".inst 0x45c6982c // ummla z12.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #2, MUL VL]\n" - ".inst 0x45c69874 // ummla z20.s, z3.b, z6.b\n" - ".inst 0x45c698bc // ummla z28.s, z5.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #3, MUL VL]\n" - ".inst 0x45c79829 // ummla z9.s, z1.b, z7.b\n" - ".inst 0x45c79871 // ummla z17.s, z3.b, z7.b\n" - ".inst 0x45c798b9 // ummla z25.s, z5.b, z7.b\n" - ".inst 0x45c6982d // ummla z13.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #4, MUL VL]\n" - ".inst 0x45c69875 // ummla z21.s, z3.b, z6.b\n" - ".inst 0x45c698bd // ummla z29.s, z5.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #5, MUL VL]\n" - ".inst 0x45c7982a // ummla z10.s, z1.b, z7.b\n" - ".inst 0x45c79872 // ummla z18.s, z3.b, z7.b\n" - ".inst 0x45c798ba // ummla z26.s, z5.b, z7.b\n" - ".inst 0x45c6982e // ummla z14.s, z1.b, z6.b\n" - "ld1b { z7.b }, p5/Z, [x10, #6, MUL VL]\n" - ".inst 0x45c69876 // ummla z22.s, z3.b, z6.b\n" - ".inst 0x45c698be // ummla z30.s, z5.b, z6.b\n" - "ld1b { z6.b }, p5/Z, [x10, #7, MUL VL]\n" + "ld1b { z2.b }, p5/Z, [x10]\n" + "ld1b { z0.b }, p5/Z, [x10, #1, MUL VL]\n" + ".inst 0x45c29828 // ummla z8.s, z1.b, z2.b\n" + ".inst 0x45c29870 // ummla z16.s, z3.b, z2.b\n" + ".inst 0x45c298b8 // ummla z24.s, z5.b, z2.b\n" + ".inst 0x45c0982c // ummla z12.s, z1.b, z0.b\n" + "ld1b { z2.b }, p5/Z, [x10, #2, MUL VL]\n" + ".inst 0x45c09874 // ummla z20.s, z3.b, z0.b\n" + ".inst 0x45c098bc // ummla z28.s, z5.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #3, MUL VL]\n" + ".inst 0x45c29829 // ummla z9.s, z1.b, z2.b\n" + ".inst 0x45c29871 // ummla z17.s, z3.b, z2.b\n" + ".inst 0x45c298b9 // ummla z25.s, z5.b, z2.b\n" + ".inst 0x45c0982d // ummla z13.s, z1.b, z0.b\n" + "ld1b { z2.b }, p5/Z, [x10, #4, MUL VL]\n" + ".inst 0x45c09875 // ummla z21.s, z3.b, z0.b\n" + ".inst 0x45c098bd // ummla z29.s, z5.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #5, MUL VL]\n" + ".inst 0x45c2982a // ummla z10.s, z1.b, z2.b\n" + ".inst 0x45c29872 // ummla z18.s, z3.b, z2.b\n" + ".inst 0x45c298ba // ummla z26.s, z5.b, z2.b\n" + ".inst 0x45c0982e // ummla z14.s, z1.b, z0.b\n" + "ld1b { z2.b }, p5/Z, [x10, #6, MUL VL]\n" + ".inst 0x45c09876 // ummla z22.s, z3.b, z0.b\n" + ".inst 0x45c098be // ummla z30.s, z5.b, z0.b\n" + "ld1b { z0.b }, p5/Z, [x10, #7, MUL VL]\n" "addvl x10, x10, #8\n" - ".inst 0x45c7982b // ummla z11.s, z1.b, z7.b\n" - ".inst 0x45c79873 // ummla z19.s, z3.b, z7.b\n" - ".inst 0x45c798bb // ummla z27.s, z5.b, z7.b\n" - ".inst 0x45c6982f // ummla z15.s, z1.b, z6.b\n" - ".inst 0x45c69877 // ummla z23.s, z3.b, z6.b\n" - ".inst 0x45c698bf // ummla z31.s, z5.b, z6.b\n" + ".inst 0x45c2982b // ummla z11.s, z1.b, z2.b\n" + ".inst 0x45c29873 // ummla z19.s, z3.b, z2.b\n" + ".inst 0x45c298bb // ummla z27.s, z5.b, z2.b\n" + ".inst 0x45c0982f // ummla z15.s, z1.b, z0.b\n" + ".inst 0x45c09877 // ummla z23.s, z3.b, z0.b\n" + ".inst 0x45c098bf // ummla z31.s, z5.b, z0.b\n" "65:" // Height 6: Multiply loop: multiply skip "ldr w20, [%x[args_ptr], %[offsetof_num_strings]]\n" "add x28, x28, #0x1\n" @@ -1596,7 +1596,7 @@ void sve_hybrid_u8u32_mmla_6x4VL ( "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "add x24, x9, x20, LSL #2\n" "add x23, x24, x20, LSL #2\n" - "uzp1 z7.d, z8.d, z12.d\n" + "uzp1 z0.d, z8.d, z12.d\n" "add x22, x23, x20, LSL #2\n" "add x21, x22, x20, LSL #2\n" "uzp2 z8.d, z8.d, z12.d\n" @@ -1604,7 +1604,7 @@ void sve_hybrid_u8u32_mmla_6x4VL ( "add x20, x21, x20, LSL #2\n" "uzp2 z9.d, z9.d, z13.d\n" "uzp1 z13.d, z10.d, z14.d\n" - "st1w { z7.s }, p4, [x9]\n" + "st1w { z0.s }, p4, [x9]\n" "uzp2 z10.d, z10.d, z14.d\n" "uzp1 z14.d, z11.d, z15.d\n" "st1w { z12.s }, p3, [x9, #1, MUL VL]\n" @@ -1664,7 +1664,6 @@ void sve_hybrid_u8u32_mmla_6x4VL ( "madd %x[input_ptr], x20, x21, %x[input_ptr]\n" "b 1b\n" "68:" // Exit - : [M] "+&r" (M), [input_ptr] "+&r" (input_ptr), [output_ptr] "+&r" (output_ptr) : [args_ptr] "r" (&ka), [flags] "r" (flags), [offsetof_B_ptr] "I" (offsetof(KernelArgs, B_ptr)), [offsetof_N] "I" (offsetof(KernelArgs, N)), [offsetof_input_initial_col] "I" (offsetof(KernelArgs, input_initial_col)), [offsetof_input_offset] "I" (offsetof(KernelArgs, input_offset)), [offsetof_num_strings] "I" (offsetof(KernelArgs, num_strings)), [offsetof_output_offset] "I" (offsetof(KernelArgs, output_offset)), [offsetof_string_lengths] "I" (offsetof(KernelArgs, string_lengths)) : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "x9", "x10", "x11", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -1672,4 +1671,4 @@ void sve_hybrid_u8u32_mmla_6x4VL ( } } // namespace arm_gemm -#endif // __ARM_FEATURE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_dot_8x3VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_dot_8x3VL.hpp index f5fdf993aae9daa11865464a3a9e40f28014415b..1ae035c614461d08fd8897be99a380ae665c25d1 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_dot_8x3VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_dot_8x3VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef ARM_COMPUTE_ENABLE_SVE + #include "../std_transforms_sve.hpp" #include "../bfloat.hpp" #include "../performance_parameters.hpp" @@ -56,11 +56,6 @@ public: return get_vector_length() * 3; } - static unsigned int stripe_width() - { - return get_vector_length(); - } - static constexpr unsigned int k_unroll() { return 2; @@ -97,5 +92,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_dot_8x3VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_dot_8x3VL/generic.cpp index 94452929c6d58805d9a6934c6d30bc5652238cb2..e507bc5551a7a922207a1dbbc82964965d5611ff 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_dot_8x3VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_dot_8x3VL/generic.cpp @@ -29,8 +29,12 @@ namespace arm_gemm { void sve_interleaved_bf16fp32_dot_8x3VL( - const bfloat16 *Apanel, const bfloat16 *Bpanel, - float *Cpanel, int ablocks, int bblocks, int K) { + const bfloat16 *Apanel, + const bfloat16 *Bpanel, + float *Cpanel, + int ablocks, + int bblocks, + int K) { struct KernelArgs { size_t K = {}; @@ -85,10 +89,10 @@ void sve_interleaved_bf16fp32_dot_8x3VL( "3:" // main loop head ".inst 0x64604088 // bfdot z8.s, z4.h, z0.h[0]\n" ".inst 0x6468408b // bfdot z11.s, z4.h, z0.h[1]\n" - "ld1rqh { z2.h }, p0/Z, [%x[Apanel], #32]\n" + "ld1rqh { z3.h }, p0/Z, [%x[Apanel], #32]\n" ".inst 0x6470408e // bfdot z14.s, z4.h, z0.h[2]\n" ".inst 0x64784091 // bfdot z17.s, z4.h, z0.h[3]\n" - "ld1rqh { z3.h }, p0/Z, [%x[Apanel], #48]\n" + "ld1rqh { z7.h }, p0/Z, [%x[Apanel], #48]\n" ".inst 0x64614094 // bfdot z20.s, z4.h, z1.h[0]\n" ".inst 0x64694097 // bfdot z23.s, z4.h, z1.h[1]\n" "sub x20, x20, #0x2\n" @@ -115,35 +119,35 @@ void sve_interleaved_bf16fp32_dot_8x3VL( ".inst 0x646940d9 // bfdot z25.s, z6.h, z1.h[1]\n" ".inst 0x647140dc // bfdot z28.s, z6.h, z1.h[2]\n" ".inst 0x647940df // bfdot z31.s, z6.h, z1.h[3]\n" - "ld1h { z6.h }, p0/Z, [x22, #5, MUL VL]\n" + "ld1h { z2.h }, p0/Z, [x22, #5, MUL VL]\n" "addvl x22, x22, #6\n" - ".inst 0x64624088 // bfdot z8.s, z4.h, z2.h[0]\n" - ".inst 0x646a408b // bfdot z11.s, z4.h, z2.h[1]\n" + ".inst 0x64634088 // bfdot z8.s, z4.h, z3.h[0]\n" + ".inst 0x646b408b // bfdot z11.s, z4.h, z3.h[1]\n" "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #16]\n" - ".inst 0x6472408e // bfdot z14.s, z4.h, z2.h[2]\n" - ".inst 0x647a4091 // bfdot z17.s, z4.h, z2.h[3]\n" - ".inst 0x64634094 // bfdot z20.s, z4.h, z3.h[0]\n" - ".inst 0x646b4097 // bfdot z23.s, z4.h, z3.h[1]\n" - ".inst 0x6473409a // bfdot z26.s, z4.h, z3.h[2]\n" - ".inst 0x647b409d // bfdot z29.s, z4.h, z3.h[3]\n" + ".inst 0x6473408e // bfdot z14.s, z4.h, z3.h[2]\n" + ".inst 0x647b4091 // bfdot z17.s, z4.h, z3.h[3]\n" + ".inst 0x64674094 // bfdot z20.s, z4.h, z7.h[0]\n" + ".inst 0x646f4097 // bfdot z23.s, z4.h, z7.h[1]\n" + ".inst 0x6477409a // bfdot z26.s, z4.h, z7.h[2]\n" + ".inst 0x647f409d // bfdot z29.s, z4.h, z7.h[3]\n" "ld1h { z4.h }, p0/Z, [x22]\n" - ".inst 0x646240a9 // bfdot z9.s, z5.h, z2.h[0]\n" - ".inst 0x646a40ac // bfdot z12.s, z5.h, z2.h[1]\n" - ".inst 0x647240af // bfdot z15.s, z5.h, z2.h[2]\n" - ".inst 0x647a40b2 // bfdot z18.s, z5.h, z2.h[3]\n" - ".inst 0x646340b5 // bfdot z21.s, z5.h, z3.h[0]\n" - ".inst 0x646b40b8 // bfdot z24.s, z5.h, z3.h[1]\n" - ".inst 0x647340bb // bfdot z27.s, z5.h, z3.h[2]\n" - ".inst 0x647b40be // bfdot z30.s, z5.h, z3.h[3]\n" + ".inst 0x646340a9 // bfdot z9.s, z5.h, z3.h[0]\n" + ".inst 0x646b40ac // bfdot z12.s, z5.h, z3.h[1]\n" + ".inst 0x647340af // bfdot z15.s, z5.h, z3.h[2]\n" + ".inst 0x647b40b2 // bfdot z18.s, z5.h, z3.h[3]\n" + ".inst 0x646740b5 // bfdot z21.s, z5.h, z7.h[0]\n" + ".inst 0x646f40b8 // bfdot z24.s, z5.h, z7.h[1]\n" + ".inst 0x647740bb // bfdot z27.s, z5.h, z7.h[2]\n" + ".inst 0x647f40be // bfdot z30.s, z5.h, z7.h[3]\n" "ld1h { z5.h }, p0/Z, [x22, #1, MUL VL]\n" - ".inst 0x646240ca // bfdot z10.s, z6.h, z2.h[0]\n" - ".inst 0x646a40cd // bfdot z13.s, z6.h, z2.h[1]\n" - ".inst 0x647240d0 // bfdot z16.s, z6.h, z2.h[2]\n" - ".inst 0x647a40d3 // bfdot z19.s, z6.h, z2.h[3]\n" - ".inst 0x646340d6 // bfdot z22.s, z6.h, z3.h[0]\n" - ".inst 0x646b40d9 // bfdot z25.s, z6.h, z3.h[1]\n" - ".inst 0x647340dc // bfdot z28.s, z6.h, z3.h[2]\n" - ".inst 0x647b40df // bfdot z31.s, z6.h, z3.h[3]\n" + ".inst 0x6463404a // bfdot z10.s, z2.h, z3.h[0]\n" + ".inst 0x646b404d // bfdot z13.s, z2.h, z3.h[1]\n" + ".inst 0x64734050 // bfdot z16.s, z2.h, z3.h[2]\n" + ".inst 0x647b4053 // bfdot z19.s, z2.h, z3.h[3]\n" + ".inst 0x64674056 // bfdot z22.s, z2.h, z7.h[0]\n" + ".inst 0x646f4059 // bfdot z25.s, z2.h, z7.h[1]\n" + ".inst 0x6477405c // bfdot z28.s, z2.h, z7.h[2]\n" + ".inst 0x647f405f // bfdot z31.s, z2.h, z7.h[3]\n" "ld1h { z6.h }, p0/Z, [x22, #2, MUL VL]\n" "bge 3b\n" "4:" // main loop skip @@ -174,37 +178,37 @@ void sve_interleaved_bf16fp32_dot_8x3VL( ".inst 0x647140dc // bfdot z28.s, z6.h, z1.h[2]\n" ".inst 0x647940df // bfdot z31.s, z6.h, z1.h[3]\n" "cbz x20, 5f\n" - "ld1rqh { z0.h }, p0/Z, [%x[Apanel]]\n" - "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #16]\n" + "ld1rqh { z4.h }, p0/Z, [%x[Apanel]]\n" + "ld1rqh { z3.h }, p0/Z, [%x[Apanel], #16]\n" "add %x[Apanel], %x[Apanel], #0x20\n" - "ld1h { z7.h }, p0/Z, [x22]\n" - "ld1h { z4.h }, p0/Z, [x22, #1, MUL VL]\n" - ".inst 0x646040e8 // bfdot z8.s, z7.h, z0.h[0]\n" - "ld1h { z5.h }, p0/Z, [x22, #2, MUL VL]\n" - ".inst 0x646840eb // bfdot z11.s, z7.h, z0.h[1]\n" - ".inst 0x647040ee // bfdot z14.s, z7.h, z0.h[2]\n" - ".inst 0x647840f1 // bfdot z17.s, z7.h, z0.h[3]\n" - ".inst 0x646140f4 // bfdot z20.s, z7.h, z1.h[0]\n" + "ld1h { z2.h }, p0/Z, [x22]\n" + "ld1h { z1.h }, p0/Z, [x22, #1, MUL VL]\n" + ".inst 0x64644048 // bfdot z8.s, z2.h, z4.h[0]\n" + "ld1h { z0.h }, p0/Z, [x22, #2, MUL VL]\n" + ".inst 0x646c404b // bfdot z11.s, z2.h, z4.h[1]\n" + ".inst 0x6474404e // bfdot z14.s, z2.h, z4.h[2]\n" + ".inst 0x647c4051 // bfdot z17.s, z2.h, z4.h[3]\n" + ".inst 0x64634054 // bfdot z20.s, z2.h, z3.h[0]\n" "addvl x22, x22, #3\n" - ".inst 0x646940f7 // bfdot z23.s, z7.h, z1.h[1]\n" - ".inst 0x647140fa // bfdot z26.s, z7.h, z1.h[2]\n" - ".inst 0x647940fd // bfdot z29.s, z7.h, z1.h[3]\n" - ".inst 0x64604089 // bfdot z9.s, z4.h, z0.h[0]\n" - ".inst 0x6468408c // bfdot z12.s, z4.h, z0.h[1]\n" - ".inst 0x6470408f // bfdot z15.s, z4.h, z0.h[2]\n" - ".inst 0x64784092 // bfdot z18.s, z4.h, z0.h[3]\n" - ".inst 0x64614095 // bfdot z21.s, z4.h, z1.h[0]\n" - ".inst 0x64694098 // bfdot z24.s, z4.h, z1.h[1]\n" - ".inst 0x6471409b // bfdot z27.s, z4.h, z1.h[2]\n" - ".inst 0x6479409e // bfdot z30.s, z4.h, z1.h[3]\n" - ".inst 0x646040aa // bfdot z10.s, z5.h, z0.h[0]\n" - ".inst 0x646840ad // bfdot z13.s, z5.h, z0.h[1]\n" - ".inst 0x647040b0 // bfdot z16.s, z5.h, z0.h[2]\n" - ".inst 0x647840b3 // bfdot z19.s, z5.h, z0.h[3]\n" - ".inst 0x646140b6 // bfdot z22.s, z5.h, z1.h[0]\n" - ".inst 0x646940b9 // bfdot z25.s, z5.h, z1.h[1]\n" - ".inst 0x647140bc // bfdot z28.s, z5.h, z1.h[2]\n" - ".inst 0x647940bf // bfdot z31.s, z5.h, z1.h[3]\n" + ".inst 0x646b4057 // bfdot z23.s, z2.h, z3.h[1]\n" + ".inst 0x6473405a // bfdot z26.s, z2.h, z3.h[2]\n" + ".inst 0x647b405d // bfdot z29.s, z2.h, z3.h[3]\n" + ".inst 0x64644029 // bfdot z9.s, z1.h, z4.h[0]\n" + ".inst 0x646c402c // bfdot z12.s, z1.h, z4.h[1]\n" + ".inst 0x6474402f // bfdot z15.s, z1.h, z4.h[2]\n" + ".inst 0x647c4032 // bfdot z18.s, z1.h, z4.h[3]\n" + ".inst 0x64634035 // bfdot z21.s, z1.h, z3.h[0]\n" + ".inst 0x646b4038 // bfdot z24.s, z1.h, z3.h[1]\n" + ".inst 0x6473403b // bfdot z27.s, z1.h, z3.h[2]\n" + ".inst 0x647b403e // bfdot z30.s, z1.h, z3.h[3]\n" + ".inst 0x6464400a // bfdot z10.s, z0.h, z4.h[0]\n" + ".inst 0x646c400d // bfdot z13.s, z0.h, z4.h[1]\n" + ".inst 0x64744010 // bfdot z16.s, z0.h, z4.h[2]\n" + ".inst 0x647c4013 // bfdot z19.s, z0.h, z4.h[3]\n" + ".inst 0x64634016 // bfdot z22.s, z0.h, z3.h[0]\n" + ".inst 0x646b4019 // bfdot z25.s, z0.h, z3.h[1]\n" + ".inst 0x6473401c // bfdot z28.s, z0.h, z3.h[2]\n" + ".inst 0x647b401f // bfdot z31.s, z0.h, z3.h[3]\n" "5:" // multiply loop done "st1w { z8.s }, p0, [%x[Cpanel]]\n" "subs x23, x23, #0x1\n" @@ -243,4 +247,4 @@ void sve_interleaved_bf16fp32_dot_8x3VL( } } // namespace arm_gemm -#endif // __ARM_FEATURE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_mmla_8x3VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_mmla_8x3VL.hpp index 1de8c68494ade6bedcdd57bb28024c330ad888e6..c5096ff4bac8f630f302e0f3ca746d8014cdab43 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_mmla_8x3VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_mmla_8x3VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022 Arm Limited. + * Copyright (c) 2019-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef ARM_COMPUTE_ENABLE_SVE + #include "../std_transforms_sve.hpp" #include "../bfloat.hpp" #include "../performance_parameters.hpp" @@ -56,11 +56,6 @@ public: return get_vector_length() * 3; } - static unsigned int stripe_width() - { - return get_vector_length(); - } - static constexpr unsigned int k_unroll() { return 4; @@ -109,5 +104,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_mmla_8x3VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_mmla_8x3VL/generic.cpp index fe5382db05e19474537fea70b9c93fde57601a22..ba7185752a5d68e48e515053f68c89d2655ad2c9 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_mmla_8x3VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_bf16fp32_mmla_8x3VL/generic.cpp @@ -29,8 +29,12 @@ namespace arm_gemm { void sve_interleaved_bf16fp32_mmla_8x3VL( - const bfloat16 *Apanel, const bfloat16 *Bpanel, - float *Cpanel, int ablocks, int bblocks, int K) { + const bfloat16 *Apanel, + const bfloat16 *Bpanel, + float *Cpanel, + int ablocks, + int bblocks, + int K) { struct KernelArgs { size_t K = {}; @@ -85,82 +89,82 @@ void sve_interleaved_bf16fp32_mmla_8x3VL( "mov z31.b, #0x0\n" "blt 4f\n" "3:" // main loop head - "ld1rqh { z3.h }, p0/Z, [%x[Apanel]]\n" + "ld1rqh { z6.h }, p0/Z, [%x[Apanel]]\n" ".inst 0x6464e408 // bfmmla z8.s, z0.h, z4.h\n" ".inst 0x6465e40b // bfmmla z11.s, z0.h, z5.h\n" ".inst 0x6464e42e // bfmmla z14.s, z1.h, z4.h\n" ".inst 0x6465e431 // bfmmla z17.s, z1.h, z5.h\n" - "ld1h { z6.h }, p0/Z, [x22]\n" + "ld1h { z7.h }, p0/Z, [x22]\n" ".inst 0x6464e454 // bfmmla z20.s, z2.h, z4.h\n" ".inst 0x6465e457 // bfmmla z23.s, z2.h, z5.h\n" - "ld1h { z7.h }, p0/Z, [x22, #1, MUL VL]\n" - ".inst 0x6464e47a // bfmmla z26.s, z3.h, z4.h\n" - ".inst 0x6465e47d // bfmmla z29.s, z3.h, z5.h\n" - "ld1h { z4.h }, p0/Z, [x22, #2, MUL VL]\n" - "ld1h { z5.h }, p0/Z, [x22, #3, MUL VL]\n" - ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - ".inst 0x6467e40c // bfmmla z12.s, z0.h, z7.h\n" - ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" - ".inst 0x6467e432 // bfmmla z18.s, z1.h, z7.h\n" + "ld1h { z3.h }, p0/Z, [x22, #1, MUL VL]\n" + ".inst 0x6464e4da // bfmmla z26.s, z6.h, z4.h\n" + ".inst 0x6465e4dd // bfmmla z29.s, z6.h, z5.h\n" + "ld1h { z5.h }, p0/Z, [x22, #2, MUL VL]\n" + "ld1h { z4.h }, p0/Z, [x22, #3, MUL VL]\n" + ".inst 0x6467e409 // bfmmla z9.s, z0.h, z7.h\n" + ".inst 0x6463e40c // bfmmla z12.s, z0.h, z3.h\n" + ".inst 0x6467e42f // bfmmla z15.s, z1.h, z7.h\n" + ".inst 0x6463e432 // bfmmla z18.s, z1.h, z3.h\n" "sub x20, x20, #0x2\n" - ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" - ".inst 0x6467e458 // bfmmla z24.s, z2.h, z7.h\n" + ".inst 0x6467e455 // bfmmla z21.s, z2.h, z7.h\n" + ".inst 0x6463e458 // bfmmla z24.s, z2.h, z3.h\n" "cmp x20, #0x2\n" - ".inst 0x6466e47b // bfmmla z27.s, z3.h, z6.h\n" - ".inst 0x6467e47e // bfmmla z30.s, z3.h, z7.h\n" - "ld1h { z6.h }, p0/Z, [x22, #4, MUL VL]\n" - ".inst 0x6464e40a // bfmmla z10.s, z0.h, z4.h\n" - ".inst 0x6465e40d // bfmmla z13.s, z0.h, z5.h\n" + ".inst 0x6467e4db // bfmmla z27.s, z6.h, z7.h\n" + ".inst 0x6463e4de // bfmmla z30.s, z6.h, z3.h\n" + "ld1h { z3.h }, p0/Z, [x22, #4, MUL VL]\n" + ".inst 0x6465e40a // bfmmla z10.s, z0.h, z5.h\n" + ".inst 0x6464e40d // bfmmla z13.s, z0.h, z4.h\n" "ld1rqh { z0.h }, p0/Z, [%x[Apanel], #16]\n" - ".inst 0x6464e430 // bfmmla z16.s, z1.h, z4.h\n" - ".inst 0x6465e433 // bfmmla z19.s, z1.h, z5.h\n" + ".inst 0x6465e430 // bfmmla z16.s, z1.h, z5.h\n" + ".inst 0x6464e433 // bfmmla z19.s, z1.h, z4.h\n" "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #32]\n" - ".inst 0x6464e456 // bfmmla z22.s, z2.h, z4.h\n" - ".inst 0x6465e459 // bfmmla z25.s, z2.h, z5.h\n" + ".inst 0x6465e456 // bfmmla z22.s, z2.h, z5.h\n" + ".inst 0x6464e459 // bfmmla z25.s, z2.h, z4.h\n" "ld1h { z7.h }, p0/Z, [x22, #5, MUL VL]\n" - ".inst 0x6464e47c // bfmmla z28.s, z3.h, z4.h\n" - ".inst 0x6465e47f // bfmmla z31.s, z3.h, z5.h\n" - "ld1rqh { z2.h }, p0/Z, [%x[Apanel], #48]\n" - "ld1rqh { z3.h }, p0/Z, [%x[Apanel], #64]\n" - "ld1h { z4.h }, p0/Z, [x22, #6, MUL VL]\n" - ".inst 0x6466e408 // bfmmla z8.s, z0.h, z6.h\n" - "ld1h { z5.h }, p0/Z, [x22, #7, MUL VL]\n" + ".inst 0x6465e4dc // bfmmla z28.s, z6.h, z5.h\n" + ".inst 0x6464e4df // bfmmla z31.s, z6.h, z4.h\n" + "ld1rqh { z5.h }, p0/Z, [%x[Apanel], #48]\n" + "ld1rqh { z6.h }, p0/Z, [%x[Apanel], #64]\n" + "ld1h { z2.h }, p0/Z, [x22, #6, MUL VL]\n" + ".inst 0x6463e408 // bfmmla z8.s, z0.h, z3.h\n" + "ld1h { z4.h }, p0/Z, [x22, #7, MUL VL]\n" "addvl x22, x22, #16\n" ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" + ".inst 0x6463e42e // bfmmla z14.s, z1.h, z3.h\n" ".inst 0x6467e431 // bfmmla z17.s, z1.h, z7.h\n" - ".inst 0x6466e454 // bfmmla z20.s, z2.h, z6.h\n" - ".inst 0x6467e457 // bfmmla z23.s, z2.h, z7.h\n" - ".inst 0x6466e47a // bfmmla z26.s, z3.h, z6.h\n" - ".inst 0x6467e47d // bfmmla z29.s, z3.h, z7.h\n" - "ld1h { z6.h }, p0/Z, [x22, #-8, MUL VL]\n" + ".inst 0x6463e4b4 // bfmmla z20.s, z5.h, z3.h\n" + ".inst 0x6467e4b7 // bfmmla z23.s, z5.h, z7.h\n" + ".inst 0x6463e4da // bfmmla z26.s, z6.h, z3.h\n" + ".inst 0x6467e4dd // bfmmla z29.s, z6.h, z7.h\n" + "ld1h { z3.h }, p0/Z, [x22, #-8, MUL VL]\n" "ld1h { z7.h }, p0/Z, [x22, #-7, MUL VL]\n" - ".inst 0x6464e409 // bfmmla z9.s, z0.h, z4.h\n" - ".inst 0x6465e40c // bfmmla z12.s, z0.h, z5.h\n" - ".inst 0x6464e42f // bfmmla z15.s, z1.h, z4.h\n" - ".inst 0x6465e432 // bfmmla z18.s, z1.h, z5.h\n" - ".inst 0x6464e455 // bfmmla z21.s, z2.h, z4.h\n" - ".inst 0x6465e458 // bfmmla z24.s, z2.h, z5.h\n" - ".inst 0x6464e47b // bfmmla z27.s, z3.h, z4.h\n" - ".inst 0x6465e47e // bfmmla z30.s, z3.h, z5.h\n" + ".inst 0x6462e409 // bfmmla z9.s, z0.h, z2.h\n" + ".inst 0x6464e40c // bfmmla z12.s, z0.h, z4.h\n" + ".inst 0x6462e42f // bfmmla z15.s, z1.h, z2.h\n" + ".inst 0x6464e432 // bfmmla z18.s, z1.h, z4.h\n" + ".inst 0x6462e4b5 // bfmmla z21.s, z5.h, z2.h\n" + ".inst 0x6464e4b8 // bfmmla z24.s, z5.h, z4.h\n" + ".inst 0x6462e4db // bfmmla z27.s, z6.h, z2.h\n" + ".inst 0x6464e4de // bfmmla z30.s, z6.h, z4.h\n" "ld1h { z4.h }, p0/Z, [x22, #-6, MUL VL]\n" - ".inst 0x6466e40a // bfmmla z10.s, z0.h, z6.h\n" + ".inst 0x6463e40a // bfmmla z10.s, z0.h, z3.h\n" ".inst 0x6467e40d // bfmmla z13.s, z0.h, z7.h\n" "ld1rqh { z0.h }, p0/Z, [%x[Apanel], #80]\n" - ".inst 0x6466e430 // bfmmla z16.s, z1.h, z6.h\n" + ".inst 0x6463e430 // bfmmla z16.s, z1.h, z3.h\n" ".inst 0x6467e433 // bfmmla z19.s, z1.h, z7.h\n" "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #96]\n" - ".inst 0x6466e456 // bfmmla z22.s, z2.h, z6.h\n" - ".inst 0x6467e459 // bfmmla z25.s, z2.h, z7.h\n" + ".inst 0x6463e4b6 // bfmmla z22.s, z5.h, z3.h\n" + ".inst 0x6467e4b9 // bfmmla z25.s, z5.h, z7.h\n" "ld1h { z5.h }, p0/Z, [x22, #-5, MUL VL]\n" - ".inst 0x6466e47c // bfmmla z28.s, z3.h, z6.h\n" - ".inst 0x6467e47f // bfmmla z31.s, z3.h, z7.h\n" + ".inst 0x6463e4dc // bfmmla z28.s, z6.h, z3.h\n" + ".inst 0x6467e4df // bfmmla z31.s, z6.h, z7.h\n" "ld1rqh { z2.h }, p0/Z, [%x[Apanel], #112]\n" "add %x[Apanel], %x[Apanel], #0x80\n" "addvl x22, x22, #-4\n" "bge 3b\n" "4:" // main loop skip - "ld1rqh { z3.h }, p0/Z, [%x[Apanel]]\n" + "ld1rqh { z7.h }, p0/Z, [%x[Apanel]]\n" ".inst 0x6464e408 // bfmmla z8.s, z0.h, z4.h\n" ".inst 0x6465e40b // bfmmla z11.s, z0.h, z5.h\n" ".inst 0x6464e42e // bfmmla z14.s, z1.h, z4.h\n" @@ -168,114 +172,114 @@ void sve_interleaved_bf16fp32_mmla_8x3VL( "ld1h { z6.h }, p0/Z, [x22]\n" ".inst 0x6464e454 // bfmmla z20.s, z2.h, z4.h\n" ".inst 0x6465e457 // bfmmla z23.s, z2.h, z5.h\n" - "ld1h { z7.h }, p0/Z, [x22, #1, MUL VL]\n" - ".inst 0x6464e47a // bfmmla z26.s, z3.h, z4.h\n" - ".inst 0x6465e47d // bfmmla z29.s, z3.h, z5.h\n" - "ld1h { z4.h }, p0/Z, [x22, #2, MUL VL]\n" - "ld1h { z5.h }, p0/Z, [x22, #3, MUL VL]\n" + "ld1h { z3.h }, p0/Z, [x22, #1, MUL VL]\n" + ".inst 0x6464e4fa // bfmmla z26.s, z7.h, z4.h\n" + ".inst 0x6465e4fd // bfmmla z29.s, z7.h, z5.h\n" + "ld1h { z5.h }, p0/Z, [x22, #2, MUL VL]\n" + "ld1h { z4.h }, p0/Z, [x22, #3, MUL VL]\n" ".inst 0x6466e409 // bfmmla z9.s, z0.h, z6.h\n" - ".inst 0x6467e40c // bfmmla z12.s, z0.h, z7.h\n" + ".inst 0x6463e40c // bfmmla z12.s, z0.h, z3.h\n" ".inst 0x6466e42f // bfmmla z15.s, z1.h, z6.h\n" - ".inst 0x6467e432 // bfmmla z18.s, z1.h, z7.h\n" + ".inst 0x6463e432 // bfmmla z18.s, z1.h, z3.h\n" "add %x[Apanel], %x[Apanel], #0x10\n" ".inst 0x6466e455 // bfmmla z21.s, z2.h, z6.h\n" - ".inst 0x6467e458 // bfmmla z24.s, z2.h, z7.h\n" + ".inst 0x6463e458 // bfmmla z24.s, z2.h, z3.h\n" "addvl x22, x22, #4\n" - ".inst 0x6466e47b // bfmmla z27.s, z3.h, z6.h\n" - ".inst 0x6467e47e // bfmmla z30.s, z3.h, z7.h\n" - ".inst 0x6464e40a // bfmmla z10.s, z0.h, z4.h\n" - ".inst 0x6465e40d // bfmmla z13.s, z0.h, z5.h\n" - ".inst 0x6464e430 // bfmmla z16.s, z1.h, z4.h\n" - ".inst 0x6465e433 // bfmmla z19.s, z1.h, z5.h\n" - ".inst 0x6464e456 // bfmmla z22.s, z2.h, z4.h\n" - ".inst 0x6465e459 // bfmmla z25.s, z2.h, z5.h\n" - ".inst 0x6464e47c // bfmmla z28.s, z3.h, z4.h\n" - ".inst 0x6465e47f // bfmmla z31.s, z3.h, z5.h\n" + ".inst 0x6466e4fb // bfmmla z27.s, z7.h, z6.h\n" + ".inst 0x6463e4fe // bfmmla z30.s, z7.h, z3.h\n" + ".inst 0x6465e40a // bfmmla z10.s, z0.h, z5.h\n" + ".inst 0x6464e40d // bfmmla z13.s, z0.h, z4.h\n" + ".inst 0x6465e430 // bfmmla z16.s, z1.h, z5.h\n" + ".inst 0x6464e433 // bfmmla z19.s, z1.h, z4.h\n" + ".inst 0x6465e456 // bfmmla z22.s, z2.h, z5.h\n" + ".inst 0x6464e459 // bfmmla z25.s, z2.h, z4.h\n" + ".inst 0x6465e4fc // bfmmla z28.s, z7.h, z5.h\n" + ".inst 0x6464e4ff // bfmmla z31.s, z7.h, z4.h\n" "cbz x20, 5f\n" - "ld1h { z6.h }, p0/Z, [x22]\n" - "ld1rqh { z0.h }, p0/Z, [%x[Apanel]]\n" - ".inst 0x6466e408 // bfmmla z8.s, z0.h, z6.h\n" - "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #16]\n" - "ld1h { z7.h }, p0/Z, [x22, #1, MUL VL]\n" - ".inst 0x6467e40b // bfmmla z11.s, z0.h, z7.h\n" - "ld1rqh { z2.h }, p0/Z, [%x[Apanel], #32]\n" - "ld1rqh { z3.h }, p0/Z, [%x[Apanel], #48]\n" - ".inst 0x6466e42e // bfmmla z14.s, z1.h, z6.h\n" - ".inst 0x6467e431 // bfmmla z17.s, z1.h, z7.h\n" - ".inst 0x6466e454 // bfmmla z20.s, z2.h, z6.h\n" - "ld1h { z4.h }, p0/Z, [x22, #2, MUL VL]\n" - ".inst 0x6467e457 // bfmmla z23.s, z2.h, z7.h\n" - ".inst 0x6466e47a // bfmmla z26.s, z3.h, z6.h\n" - "ld1h { z5.h }, p0/Z, [x22, #3, MUL VL]\n" - ".inst 0x6467e47d // bfmmla z29.s, z3.h, z7.h\n" - "ld1h { z6.h }, p0/Z, [x22, #4, MUL VL]\n" - "ld1h { z7.h }, p0/Z, [x22, #5, MUL VL]\n" - ".inst 0x6464e409 // bfmmla z9.s, z0.h, z4.h\n" - ".inst 0x6465e40c // bfmmla z12.s, z0.h, z5.h\n" + "ld1h { z1.h }, p0/Z, [x22]\n" + "ld1rqh { z7.h }, p0/Z, [%x[Apanel]]\n" + ".inst 0x6461e4e8 // bfmmla z8.s, z7.h, z1.h\n" + "ld1rqh { z6.h }, p0/Z, [%x[Apanel], #16]\n" + "ld1h { z0.h }, p0/Z, [x22, #1, MUL VL]\n" + ".inst 0x6460e4eb // bfmmla z11.s, z7.h, z0.h\n" + "ld1rqh { z5.h }, p0/Z, [%x[Apanel], #32]\n" + "ld1rqh { z4.h }, p0/Z, [%x[Apanel], #48]\n" + ".inst 0x6461e4ce // bfmmla z14.s, z6.h, z1.h\n" + ".inst 0x6460e4d1 // bfmmla z17.s, z6.h, z0.h\n" + ".inst 0x6461e4b4 // bfmmla z20.s, z5.h, z1.h\n" + "ld1h { z3.h }, p0/Z, [x22, #2, MUL VL]\n" + ".inst 0x6460e4b7 // bfmmla z23.s, z5.h, z0.h\n" + ".inst 0x6461e49a // bfmmla z26.s, z4.h, z1.h\n" + "ld1h { z2.h }, p0/Z, [x22, #3, MUL VL]\n" + ".inst 0x6460e49d // bfmmla z29.s, z4.h, z0.h\n" + "ld1h { z1.h }, p0/Z, [x22, #4, MUL VL]\n" + "ld1h { z0.h }, p0/Z, [x22, #5, MUL VL]\n" + ".inst 0x6463e4e9 // bfmmla z9.s, z7.h, z3.h\n" + ".inst 0x6462e4ec // bfmmla z12.s, z7.h, z2.h\n" "addvl x22, x22, #6\n" - ".inst 0x6464e42f // bfmmla z15.s, z1.h, z4.h\n" - ".inst 0x6465e432 // bfmmla z18.s, z1.h, z5.h\n" + ".inst 0x6463e4cf // bfmmla z15.s, z6.h, z3.h\n" + ".inst 0x6462e4d2 // bfmmla z18.s, z6.h, z2.h\n" "add %x[Apanel], %x[Apanel], #0x40\n" - ".inst 0x6464e455 // bfmmla z21.s, z2.h, z4.h\n" - ".inst 0x6465e458 // bfmmla z24.s, z2.h, z5.h\n" - ".inst 0x6464e47b // bfmmla z27.s, z3.h, z4.h\n" - ".inst 0x6465e47e // bfmmla z30.s, z3.h, z5.h\n" - ".inst 0x6466e40a // bfmmla z10.s, z0.h, z6.h\n" - ".inst 0x6467e40d // bfmmla z13.s, z0.h, z7.h\n" - ".inst 0x6466e430 // bfmmla z16.s, z1.h, z6.h\n" - ".inst 0x6467e433 // bfmmla z19.s, z1.h, z7.h\n" - ".inst 0x6466e456 // bfmmla z22.s, z2.h, z6.h\n" - ".inst 0x6467e459 // bfmmla z25.s, z2.h, z7.h\n" - ".inst 0x6466e47c // bfmmla z28.s, z3.h, z6.h\n" - ".inst 0x6467e47f // bfmmla z31.s, z3.h, z7.h\n" + ".inst 0x6463e4b5 // bfmmla z21.s, z5.h, z3.h\n" + ".inst 0x6462e4b8 // bfmmla z24.s, z5.h, z2.h\n" + ".inst 0x6463e49b // bfmmla z27.s, z4.h, z3.h\n" + ".inst 0x6462e49e // bfmmla z30.s, z4.h, z2.h\n" + ".inst 0x6461e4ea // bfmmla z10.s, z7.h, z1.h\n" + ".inst 0x6460e4ed // bfmmla z13.s, z7.h, z0.h\n" + ".inst 0x6461e4d0 // bfmmla z16.s, z6.h, z1.h\n" + ".inst 0x6460e4d3 // bfmmla z19.s, z6.h, z0.h\n" + ".inst 0x6461e4b6 // bfmmla z22.s, z5.h, z1.h\n" + ".inst 0x6460e4b9 // bfmmla z25.s, z5.h, z0.h\n" + ".inst 0x6461e49c // bfmmla z28.s, z4.h, z1.h\n" + ".inst 0x6460e49f // bfmmla z31.s, z4.h, z0.h\n" "5:" // multiply loop done - "uzp1 z4.d, z8.d, z11.d\n" + "uzp1 z0.d, z8.d, z11.d\n" "uzp2 z8.d, z8.d, z11.d\n" - "st1w { z4.s }, p0, [%x[Cpanel]]\n" - "uzp1 z11.d, z9.d, z12.d\n" + "st1w { z0.s }, p0, [%x[Cpanel]]\n" + "uzp1 z0.d, z9.d, z12.d\n" "uzp2 z9.d, z9.d, z12.d\n" - "st1w { z11.s }, p0, [%x[Cpanel], #1, MUL VL]\n" - "uzp1 z12.d, z10.d, z13.d\n" + "st1w { z0.s }, p0, [%x[Cpanel], #1, MUL VL]\n" + "uzp1 z0.d, z10.d, z13.d\n" "uzp2 z10.d, z10.d, z13.d\n" - "st1w { z12.s }, p0, [%x[Cpanel], #2, MUL VL]\n" + "st1w { z0.s }, p0, [%x[Cpanel], #2, MUL VL]\n" "st1w { z8.s }, p0, [%x[Cpanel], #3, MUL VL]\n" - "uzp1 z13.d, z14.d, z17.d\n" + "uzp1 z0.d, z14.d, z17.d\n" "uzp2 z14.d, z14.d, z17.d\n" "st1w { z9.s }, p0, [%x[Cpanel], #4, MUL VL]\n" - "uzp1 z17.d, z15.d, z18.d\n" + "uzp1 z1.d, z15.d, z18.d\n" "subs x23, x23, #0x1\n" "st1w { z10.s }, p0, [%x[Cpanel], #5, MUL VL]\n" "uzp2 z15.d, z15.d, z18.d\n" - "uzp1 z18.d, z16.d, z19.d\n" - "st1w { z13.s }, p0, [%x[Cpanel], #6, MUL VL]\n" + "uzp1 z17.d, z16.d, z19.d\n" + "st1w { z0.s }, p0, [%x[Cpanel], #6, MUL VL]\n" "uzp2 z16.d, z16.d, z19.d\n" - "uzp1 z19.d, z20.d, z23.d\n" - "st1w { z17.s }, p0, [%x[Cpanel], #7, MUL VL]\n" + "uzp1 z0.d, z20.d, z23.d\n" + "st1w { z1.s }, p0, [%x[Cpanel], #7, MUL VL]\n" "addvl %x[Cpanel], %x[Cpanel], #16\n" "uzp2 z20.d, z20.d, z23.d\n" - "st1w { z18.s }, p0, [%x[Cpanel], #-8, MUL VL]\n" + "st1w { z17.s }, p0, [%x[Cpanel], #-8, MUL VL]\n" "uzp1 z23.d, z21.d, z24.d\n" "uzp2 z21.d, z21.d, z24.d\n" "st1w { z14.s }, p0, [%x[Cpanel], #-7, MUL VL]\n" - "uzp1 z24.d, z22.d, z25.d\n" + "uzp1 z19.d, z22.d, z25.d\n" "uzp2 z22.d, z22.d, z25.d\n" "st1w { z15.s }, p0, [%x[Cpanel], #-6, MUL VL]\n" - "uzp1 z25.d, z26.d, z29.d\n" + "uzp1 z18.d, z26.d, z29.d\n" "uzp2 z26.d, z26.d, z29.d\n" "st1w { z16.s }, p0, [%x[Cpanel], #-5, MUL VL]\n" - "uzp1 z29.d, z27.d, z30.d\n" + "uzp1 z17.d, z27.d, z30.d\n" "uzp2 z27.d, z27.d, z30.d\n" - "st1w { z19.s }, p0, [%x[Cpanel], #-4, MUL VL]\n" - "uzp1 z30.d, z28.d, z31.d\n" + "st1w { z0.s }, p0, [%x[Cpanel], #-4, MUL VL]\n" + "uzp1 z16.d, z28.d, z31.d\n" "uzp2 z28.d, z28.d, z31.d\n" "st1w { z23.s }, p0, [%x[Cpanel], #-3, MUL VL]\n" - "st1w { z24.s }, p0, [%x[Cpanel], #-2, MUL VL]\n" + "st1w { z19.s }, p0, [%x[Cpanel], #-2, MUL VL]\n" "st1w { z20.s }, p0, [%x[Cpanel], #-1, MUL VL]\n" "st1w { z21.s }, p0, [%x[Cpanel]]\n" "st1w { z22.s }, p0, [%x[Cpanel], #1, MUL VL]\n" - "st1w { z25.s }, p0, [%x[Cpanel], #2, MUL VL]\n" - "st1w { z29.s }, p0, [%x[Cpanel], #3, MUL VL]\n" - "st1w { z30.s }, p0, [%x[Cpanel], #4, MUL VL]\n" + "st1w { z18.s }, p0, [%x[Cpanel], #2, MUL VL]\n" + "st1w { z17.s }, p0, [%x[Cpanel], #3, MUL VL]\n" + "st1w { z16.s }, p0, [%x[Cpanel], #4, MUL VL]\n" "st1w { z26.s }, p0, [%x[Cpanel], #5, MUL VL]\n" "st1w { z27.s }, p0, [%x[Cpanel], #6, MUL VL]\n" "st1w { z28.s }, p0, [%x[Cpanel], #7, MUL VL]\n" @@ -290,4 +294,4 @@ void sve_interleaved_bf16fp32_mmla_8x3VL( } } // namespace arm_gemm -#endif // __ARM_FEATURE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL.hpp index 6f1089d517175b2cc3d4127b4fc11853c621ed16..6c5416776330cb67a2dac131e5704bb05464a9dd 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef ARM_COMPUTE_ENABLE_SVE + #include "../std_transforms_sve.hpp" #include "../performance_parameters.hpp" @@ -56,11 +56,6 @@ public: return get_vector_length<__fp16>() * 3; } - static unsigned int stripe_width() - { - return get_vector_length<__fp16>(); - } - static constexpr unsigned int k_unroll() { return 1; @@ -81,6 +76,8 @@ public: return { 13.84, 2.07, 2.52 }; case CPUModel::V1: return { 31.90, 5.15, 10.34 }; + case CPUModel::A64FX: + return { 44.34, 3.23, 7.06 }; } } @@ -104,5 +101,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL/a64fx.cpp index 928750988953ea06830d4929e67044c0adcfb8ac..609277d889554dbd24d069c1ae27138d2f314185 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL/a64fx.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL/a64fx.cpp @@ -28,8 +28,12 @@ namespace arm_gemm { void sve_interleaved_fp16_mla_8x3VL_a64fx( - const __fp16 *Apanel, const __fp16 *Bpanel, - __fp16 *Cpanel, int ablocks, int bblocks, int K) { + const __fp16 *Apanel, + const __fp16 *Bpanel, + __fp16 *Cpanel, + int ablocks, + int bblocks, + int K) { struct KernelArgs { size_t K = {}; @@ -88,7 +92,7 @@ void sve_interleaved_fp16_mla_8x3VL_a64fx( "fmla z9.h, p0/M, z1.h, z3.h\n" "sub x20, x20, #0x2\n" "fmla z10.h, p0/M, z2.h, z3.h\n" - "ld1rh { z3.h }, p0/Z, [%x[Apanel], #8]\n" + "ld1rh { z7.h }, p0/Z, [%x[Apanel], #8]\n" "fmla z11.h, p0/M, z0.h, z4.h\n" "fmla z12.h, p0/M, z1.h, z4.h\n" "fmla z13.h, p0/M, z2.h, z4.h\n" @@ -97,63 +101,63 @@ void sve_interleaved_fp16_mla_8x3VL_a64fx( "fmla z15.h, p0/M, z1.h, z5.h\n" "cmp x20, #0x2\n" "fmla z16.h, p0/M, z2.h, z5.h\n" - "ld1rh { z5.h }, p0/Z, [%x[Apanel], #12]\n" + "ld1rh { z3.h }, p0/Z, [%x[Apanel], #12]\n" "fmla z17.h, p0/M, z0.h, z6.h\n" "fmla z18.h, p0/M, z1.h, z6.h\n" "fmla z19.h, p0/M, z2.h, z6.h\n" - "ld1rh { z6.h }, p0/Z, [%x[Apanel], #14]\n" - "fmla z20.h, p0/M, z0.h, z3.h\n" - "fmla z21.h, p0/M, z1.h, z3.h\n" - "fmla z22.h, p0/M, z2.h, z3.h\n" - "ld1rh { z3.h }, p0/Z, [%x[Apanel], #16]\n" + "ld1rh { z5.h }, p0/Z, [%x[Apanel], #14]\n" + "fmla z20.h, p0/M, z0.h, z7.h\n" + "fmla z21.h, p0/M, z1.h, z7.h\n" + "fmla z22.h, p0/M, z2.h, z7.h\n" + "ld1rh { z7.h }, p0/Z, [%x[Apanel], #16]\n" "fmla z23.h, p0/M, z0.h, z4.h\n" "fmla z24.h, p0/M, z1.h, z4.h\n" "fmla z25.h, p0/M, z2.h, z4.h\n" "ld1rh { z4.h }, p0/Z, [%x[Apanel], #18]\n" - "fmla z26.h, p0/M, z0.h, z5.h\n" - "fmla z27.h, p0/M, z1.h, z5.h\n" - "fmla z28.h, p0/M, z2.h, z5.h\n" - "ld1rh { z5.h }, p0/Z, [%x[Apanel], #20]\n" - "fmla z29.h, p0/M, z0.h, z6.h\n" - "ld1h { z0.h }, p0/Z, [x22, #3, MUL VL]\n" - "fmla z30.h, p0/M, z1.h, z6.h\n" - "fmla z31.h, p0/M, z2.h, z6.h\n" - "ld1h { z1.h }, p0/Z, [x22, #4, MUL VL]\n" - "ld1h { z2.h }, p0/Z, [x22, #5, MUL VL]\n" - "fmla z8.h, p0/M, z0.h, z3.h\n" - "ld1rh { z6.h }, p0/Z, [%x[Apanel], #22]\n" - "fmla z9.h, p0/M, z1.h, z3.h\n" - "fmla z10.h, p0/M, z2.h, z3.h\n" - "fmla z11.h, p0/M, z0.h, z4.h\n" - "ld1rh { z3.h }, p0/Z, [%x[Apanel], #24]\n" - "fmla z12.h, p0/M, z1.h, z4.h\n" - "fmla z13.h, p0/M, z2.h, z4.h\n" + "fmla z26.h, p0/M, z0.h, z3.h\n" + "fmla z27.h, p0/M, z1.h, z3.h\n" + "fmla z28.h, p0/M, z2.h, z3.h\n" + "ld1rh { z3.h }, p0/Z, [%x[Apanel], #20]\n" + "fmla z29.h, p0/M, z0.h, z5.h\n" + "ld1h { z6.h }, p0/Z, [x22, #3, MUL VL]\n" + "fmla z30.h, p0/M, z1.h, z5.h\n" + "fmla z31.h, p0/M, z2.h, z5.h\n" + "ld1h { z2.h }, p0/Z, [x22, #4, MUL VL]\n" + "ld1h { z5.h }, p0/Z, [x22, #5, MUL VL]\n" + "fmla z8.h, p0/M, z6.h, z7.h\n" + "ld1rh { z1.h }, p0/Z, [%x[Apanel], #22]\n" + "fmla z9.h, p0/M, z2.h, z7.h\n" + "fmla z10.h, p0/M, z5.h, z7.h\n" + "fmla z11.h, p0/M, z6.h, z4.h\n" + "ld1rh { z7.h }, p0/Z, [%x[Apanel], #24]\n" + "fmla z12.h, p0/M, z2.h, z4.h\n" + "fmla z13.h, p0/M, z5.h, z4.h\n" "ld1rh { z4.h }, p0/Z, [%x[Apanel], #26]\n" - "fmla z14.h, p0/M, z0.h, z5.h\n" - "fmla z15.h, p0/M, z1.h, z5.h\n" + "fmla z14.h, p0/M, z6.h, z3.h\n" + "fmla z15.h, p0/M, z2.h, z3.h\n" "addvl x22, x22, #6\n" - "fmla z16.h, p0/M, z2.h, z5.h\n" - "ld1rh { z5.h }, p0/Z, [%x[Apanel], #28]\n" - "fmla z17.h, p0/M, z0.h, z6.h\n" - "fmla z18.h, p0/M, z1.h, z6.h\n" - "fmla z19.h, p0/M, z2.h, z6.h\n" - "ld1rh { z6.h }, p0/Z, [%x[Apanel], #30]\n" + "fmla z16.h, p0/M, z5.h, z3.h\n" + "ld1rh { z0.h }, p0/Z, [%x[Apanel], #28]\n" + "fmla z17.h, p0/M, z6.h, z1.h\n" + "fmla z18.h, p0/M, z2.h, z1.h\n" + "fmla z19.h, p0/M, z5.h, z1.h\n" + "ld1rh { z1.h }, p0/Z, [%x[Apanel], #30]\n" "add %x[Apanel], %x[Apanel], #0x20\n" - "fmla z20.h, p0/M, z0.h, z3.h\n" - "fmla z21.h, p0/M, z1.h, z3.h\n" - "fmla z22.h, p0/M, z2.h, z3.h\n" - "fmla z23.h, p0/M, z0.h, z4.h\n" + "fmla z20.h, p0/M, z6.h, z7.h\n" + "fmla z21.h, p0/M, z2.h, z7.h\n" + "fmla z22.h, p0/M, z5.h, z7.h\n" + "fmla z23.h, p0/M, z6.h, z4.h\n" "ld1rh { z3.h }, p0/Z, [%x[Apanel]]\n" - "fmla z24.h, p0/M, z1.h, z4.h\n" - "fmla z25.h, p0/M, z2.h, z4.h\n" + "fmla z24.h, p0/M, z2.h, z4.h\n" + "fmla z25.h, p0/M, z5.h, z4.h\n" "ld1rh { z4.h }, p0/Z, [%x[Apanel], #2]\n" - "fmla z26.h, p0/M, z0.h, z5.h\n" - "fmla z27.h, p0/M, z1.h, z5.h\n" - "fmla z28.h, p0/M, z2.h, z5.h\n" - "fmla z29.h, p0/M, z0.h, z6.h\n" + "fmla z26.h, p0/M, z6.h, z0.h\n" + "fmla z27.h, p0/M, z2.h, z0.h\n" + "fmla z28.h, p0/M, z5.h, z0.h\n" + "fmla z29.h, p0/M, z6.h, z1.h\n" "ld1h { z0.h }, p0/Z, [x22]\n" - "fmla z30.h, p0/M, z1.h, z6.h\n" - "fmla z31.h, p0/M, z2.h, z6.h\n" + "fmla z30.h, p0/M, z2.h, z1.h\n" + "fmla z31.h, p0/M, z5.h, z1.h\n" "ld1h { z1.h }, p0/Z, [x22, #1, MUL VL]\n" "ld1h { z2.h }, p0/Z, [x22, #2, MUL VL]\n" "ld1rh { z5.h }, p0/Z, [%x[Apanel], #4]\n" @@ -164,7 +168,7 @@ void sve_interleaved_fp16_mla_8x3VL_a64fx( "fmla z9.h, p0/M, z1.h, z3.h\n" "addvl x22, x22, #3\n" "fmla z10.h, p0/M, z2.h, z3.h\n" - "ld1rh { z3.h }, p0/Z, [%x[Apanel], #8]\n" + "ld1rh { z7.h }, p0/Z, [%x[Apanel], #8]\n" "fmla z11.h, p0/M, z0.h, z4.h\n" "fmla z12.h, p0/M, z1.h, z4.h\n" "fmla z13.h, p0/M, z2.h, z4.h\n" @@ -176,58 +180,58 @@ void sve_interleaved_fp16_mla_8x3VL_a64fx( "fmla z17.h, p0/M, z0.h, z6.h\n" "fmla z18.h, p0/M, z1.h, z6.h\n" "fmla z19.h, p0/M, z2.h, z6.h\n" - "ld1rh { z6.h }, p0/Z, [%x[Apanel], #14]\n" - "fmla z20.h, p0/M, z0.h, z3.h\n" - "fmla z21.h, p0/M, z1.h, z3.h\n" + "ld1rh { z3.h }, p0/Z, [%x[Apanel], #14]\n" + "fmla z20.h, p0/M, z0.h, z7.h\n" + "fmla z21.h, p0/M, z1.h, z7.h\n" "add %x[Apanel], %x[Apanel], #0x10\n" - "fmla z22.h, p0/M, z2.h, z3.h\n" + "fmla z22.h, p0/M, z2.h, z7.h\n" "fmla z23.h, p0/M, z0.h, z4.h\n" "fmla z24.h, p0/M, z1.h, z4.h\n" "fmla z25.h, p0/M, z2.h, z4.h\n" "fmla z26.h, p0/M, z0.h, z5.h\n" "fmla z27.h, p0/M, z1.h, z5.h\n" "fmla z28.h, p0/M, z2.h, z5.h\n" - "fmla z29.h, p0/M, z0.h, z6.h\n" - "fmla z30.h, p0/M, z1.h, z6.h\n" - "fmla z31.h, p0/M, z2.h, z6.h\n" + "fmla z29.h, p0/M, z0.h, z3.h\n" + "fmla z30.h, p0/M, z1.h, z3.h\n" + "fmla z31.h, p0/M, z2.h, z3.h\n" "cbz x20, 5f\n" - "ld1h { z0.h }, p0/Z, [x22]\n" - "ld1h { z1.h }, p0/Z, [x22, #1, MUL VL]\n" - "ld1h { z2.h }, p0/Z, [x22, #2, MUL VL]\n" + "ld1h { z6.h }, p0/Z, [x22]\n" + "ld1h { z5.h }, p0/Z, [x22, #1, MUL VL]\n" + "ld1h { z4.h }, p0/Z, [x22, #2, MUL VL]\n" "ld1rh { z3.h }, p0/Z, [%x[Apanel]]\n" - "fmla z8.h, p0/M, z0.h, z3.h\n" - "ld1rh { z4.h }, p0/Z, [%x[Apanel], #2]\n" - "ld1rh { z5.h }, p0/Z, [%x[Apanel], #4]\n" - "fmla z9.h, p0/M, z1.h, z3.h\n" - "ld1rh { z6.h }, p0/Z, [%x[Apanel], #6]\n" - "fmla z10.h, p0/M, z2.h, z3.h\n" - "fmla z11.h, p0/M, z0.h, z4.h\n" - "fmla z12.h, p0/M, z1.h, z4.h\n" - "fmla z13.h, p0/M, z2.h, z4.h\n" + "fmla z8.h, p0/M, z6.h, z3.h\n" + "ld1rh { z2.h }, p0/Z, [%x[Apanel], #2]\n" + "ld1rh { z1.h }, p0/Z, [%x[Apanel], #4]\n" + "fmla z9.h, p0/M, z5.h, z3.h\n" + "ld1rh { z0.h }, p0/Z, [%x[Apanel], #6]\n" + "fmla z10.h, p0/M, z4.h, z3.h\n" + "fmla z11.h, p0/M, z6.h, z2.h\n" + "fmla z12.h, p0/M, z5.h, z2.h\n" + "fmla z13.h, p0/M, z4.h, z2.h\n" "ld1rh { z3.h }, p0/Z, [%x[Apanel], #8]\n" - "fmla z14.h, p0/M, z0.h, z5.h\n" - "fmla z15.h, p0/M, z1.h, z5.h\n" - "ld1rh { z4.h }, p0/Z, [%x[Apanel], #10]\n" - "fmla z16.h, p0/M, z2.h, z5.h\n" - "fmla z17.h, p0/M, z0.h, z6.h\n" - "ld1rh { z5.h }, p0/Z, [%x[Apanel], #12]\n" - "fmla z18.h, p0/M, z1.h, z6.h\n" - "fmla z19.h, p0/M, z2.h, z6.h\n" - "ld1rh { z6.h }, p0/Z, [%x[Apanel], #14]\n" - "fmla z20.h, p0/M, z0.h, z3.h\n" - "fmla z21.h, p0/M, z1.h, z3.h\n" + "fmla z14.h, p0/M, z6.h, z1.h\n" + "fmla z15.h, p0/M, z5.h, z1.h\n" + "ld1rh { z2.h }, p0/Z, [%x[Apanel], #10]\n" + "fmla z16.h, p0/M, z4.h, z1.h\n" + "fmla z17.h, p0/M, z6.h, z0.h\n" + "ld1rh { z1.h }, p0/Z, [%x[Apanel], #12]\n" + "fmla z18.h, p0/M, z5.h, z0.h\n" + "fmla z19.h, p0/M, z4.h, z0.h\n" + "ld1rh { z0.h }, p0/Z, [%x[Apanel], #14]\n" + "fmla z20.h, p0/M, z6.h, z3.h\n" + "fmla z21.h, p0/M, z5.h, z3.h\n" "addvl x22, x22, #3\n" - "fmla z22.h, p0/M, z2.h, z3.h\n" - "fmla z23.h, p0/M, z0.h, z4.h\n" + "fmla z22.h, p0/M, z4.h, z3.h\n" + "fmla z23.h, p0/M, z6.h, z2.h\n" "add %x[Apanel], %x[Apanel], #0x10\n" - "fmla z24.h, p0/M, z1.h, z4.h\n" - "fmla z25.h, p0/M, z2.h, z4.h\n" - "fmla z26.h, p0/M, z0.h, z5.h\n" - "fmla z27.h, p0/M, z1.h, z5.h\n" - "fmla z28.h, p0/M, z2.h, z5.h\n" - "fmla z29.h, p0/M, z0.h, z6.h\n" - "fmla z30.h, p0/M, z1.h, z6.h\n" - "fmla z31.h, p0/M, z2.h, z6.h\n" + "fmla z24.h, p0/M, z5.h, z2.h\n" + "fmla z25.h, p0/M, z4.h, z2.h\n" + "fmla z26.h, p0/M, z6.h, z1.h\n" + "fmla z27.h, p0/M, z5.h, z1.h\n" + "fmla z28.h, p0/M, z4.h, z1.h\n" + "fmla z29.h, p0/M, z6.h, z0.h\n" + "fmla z30.h, p0/M, z5.h, z0.h\n" + "fmla z31.h, p0/M, z4.h, z0.h\n" "5:" // multiply loop done "st1h { z8.h }, p0, [%x[Cpanel]]\n" "subs x23, x23, #0x1\n" @@ -261,7 +265,7 @@ void sve_interleaved_fp16_mla_8x3VL_a64fx( "bne 1b\n" : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks) : [args_ptr] "r" (&ka), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_bblocks] "I" (offsetof(KernelArgs, bblocks)) - : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL/generic.cpp index 1ac2ac075e94cc6c63030bba5a0edfed16f9447f..3b16c97e2c1c853d20e54e377dbc36e567869788 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_8x3VL/generic.cpp @@ -28,8 +28,12 @@ namespace arm_gemm { void sve_interleaved_fp16_mla_8x3VL( - const __fp16 *Apanel, const __fp16 *Bpanel, - __fp16 *Cpanel, int ablocks, int bblocks, int K) { + const __fp16 *Apanel, + const __fp16 *Bpanel, + __fp16 *Cpanel, + int ablocks, + int bblocks, + int K) { struct KernelArgs { size_t K = {}; @@ -83,16 +87,16 @@ void sve_interleaved_fp16_mla_8x3VL( "3:" // main loop head "fmla z8.h, z2.h, z0.h[0]\n" "fmla z11.h, z2.h, z0.h[1]\n" - "ld1rqh { z1.h }, p0/Z, [%x[Apanel], #16]\n" + "ld1rqh { z7.h }, p0/Z, [%x[Apanel], #16]\n" "fmla z14.h, z2.h, z0.h[2]\n" "fmla z17.h, z2.h, z0.h[3]\n" - "ld1h { z5.h }, p0/Z, [x22, #3, MUL VL]\n" + "ld1h { z6.h }, p0/Z, [x22, #3, MUL VL]\n" "fmla z20.h, z2.h, z0.h[4]\n" "fmla z23.h, z2.h, z0.h[5]\n" - "ld1h { z6.h }, p0/Z, [x22, #4, MUL VL]\n" + "ld1h { z5.h }, p0/Z, [x22, #4, MUL VL]\n" "fmla z26.h, z2.h, z0.h[6]\n" "fmla z29.h, z2.h, z0.h[7]\n" - "ld1h { z7.h }, p0/Z, [x22, #5, MUL VL]\n" + "ld1h { z1.h }, p0/Z, [x22, #5, MUL VL]\n" "fmla z9.h, z3.h, z0.h[0]\n" "fmla z12.h, z3.h, z0.h[1]\n" "addvl x22, x22, #6\n" @@ -116,31 +120,31 @@ void sve_interleaved_fp16_mla_8x3VL( "fmla z28.h, z4.h, z0.h[6]\n" "fmla z31.h, z4.h, z0.h[7]\n" "ld1rqh { z0.h }, p0/Z, [%x[Apanel]]\n" - "fmla z8.h, z5.h, z1.h[0]\n" - "fmla z11.h, z5.h, z1.h[1]\n" + "fmla z8.h, z6.h, z7.h[0]\n" + "fmla z11.h, z6.h, z7.h[1]\n" "ld1h { z4.h }, p0/Z, [x22, #2, MUL VL]\n" - "fmla z14.h, z5.h, z1.h[2]\n" - "fmla z17.h, z5.h, z1.h[3]\n" - "fmla z20.h, z5.h, z1.h[4]\n" - "fmla z23.h, z5.h, z1.h[5]\n" - "fmla z26.h, z5.h, z1.h[6]\n" - "fmla z29.h, z5.h, z1.h[7]\n" - "fmla z9.h, z6.h, z1.h[0]\n" - "fmla z12.h, z6.h, z1.h[1]\n" - "fmla z15.h, z6.h, z1.h[2]\n" - "fmla z18.h, z6.h, z1.h[3]\n" - "fmla z21.h, z6.h, z1.h[4]\n" - "fmla z24.h, z6.h, z1.h[5]\n" - "fmla z27.h, z6.h, z1.h[6]\n" - "fmla z30.h, z6.h, z1.h[7]\n" - "fmla z10.h, z7.h, z1.h[0]\n" - "fmla z13.h, z7.h, z1.h[1]\n" - "fmla z16.h, z7.h, z1.h[2]\n" - "fmla z19.h, z7.h, z1.h[3]\n" - "fmla z22.h, z7.h, z1.h[4]\n" - "fmla z25.h, z7.h, z1.h[5]\n" - "fmla z28.h, z7.h, z1.h[6]\n" - "fmla z31.h, z7.h, z1.h[7]\n" + "fmla z14.h, z6.h, z7.h[2]\n" + "fmla z17.h, z6.h, z7.h[3]\n" + "fmla z20.h, z6.h, z7.h[4]\n" + "fmla z23.h, z6.h, z7.h[5]\n" + "fmla z26.h, z6.h, z7.h[6]\n" + "fmla z29.h, z6.h, z7.h[7]\n" + "fmla z9.h, z5.h, z7.h[0]\n" + "fmla z12.h, z5.h, z7.h[1]\n" + "fmla z15.h, z5.h, z7.h[2]\n" + "fmla z18.h, z5.h, z7.h[3]\n" + "fmla z21.h, z5.h, z7.h[4]\n" + "fmla z24.h, z5.h, z7.h[5]\n" + "fmla z27.h, z5.h, z7.h[6]\n" + "fmla z30.h, z5.h, z7.h[7]\n" + "fmla z10.h, z1.h, z7.h[0]\n" + "fmla z13.h, z1.h, z7.h[1]\n" + "fmla z16.h, z1.h, z7.h[2]\n" + "fmla z19.h, z1.h, z7.h[3]\n" + "fmla z22.h, z1.h, z7.h[4]\n" + "fmla z25.h, z1.h, z7.h[5]\n" + "fmla z28.h, z1.h, z7.h[6]\n" + "fmla z31.h, z1.h, z7.h[7]\n" "bge 3b\n" "4:" // main loop skip "fmla z8.h, z2.h, z0.h[0]\n" @@ -170,36 +174,36 @@ void sve_interleaved_fp16_mla_8x3VL( "fmla z28.h, z4.h, z0.h[6]\n" "fmla z31.h, z4.h, z0.h[7]\n" "cbz x20, 5f\n" - "ld1rqh { z0.h }, p0/Z, [%x[Apanel]]\n" - "ld1h { z5.h }, p0/Z, [x22]\n" - "fmla z8.h, z5.h, z0.h[0]\n" - "ld1h { z6.h }, p0/Z, [x22, #1, MUL VL]\n" - "ld1h { z7.h }, p0/Z, [x22, #2, MUL VL]\n" - "fmla z11.h, z5.h, z0.h[1]\n" - "fmla z14.h, z5.h, z0.h[2]\n" - "fmla z17.h, z5.h, z0.h[3]\n" + "ld1rqh { z3.h }, p0/Z, [%x[Apanel]]\n" + "ld1h { z2.h }, p0/Z, [x22]\n" + "fmla z8.h, z2.h, z3.h[0]\n" + "ld1h { z1.h }, p0/Z, [x22, #1, MUL VL]\n" + "ld1h { z0.h }, p0/Z, [x22, #2, MUL VL]\n" + "fmla z11.h, z2.h, z3.h[1]\n" + "fmla z14.h, z2.h, z3.h[2]\n" + "fmla z17.h, z2.h, z3.h[3]\n" "add %x[Apanel], %x[Apanel], #0x10\n" - "fmla z20.h, z5.h, z0.h[4]\n" - "fmla z23.h, z5.h, z0.h[5]\n" + "fmla z20.h, z2.h, z3.h[4]\n" + "fmla z23.h, z2.h, z3.h[5]\n" "addvl x22, x22, #3\n" - "fmla z26.h, z5.h, z0.h[6]\n" - "fmla z29.h, z5.h, z0.h[7]\n" - "fmla z9.h, z6.h, z0.h[0]\n" - "fmla z12.h, z6.h, z0.h[1]\n" - "fmla z15.h, z6.h, z0.h[2]\n" - "fmla z18.h, z6.h, z0.h[3]\n" - "fmla z21.h, z6.h, z0.h[4]\n" - "fmla z24.h, z6.h, z0.h[5]\n" - "fmla z27.h, z6.h, z0.h[6]\n" - "fmla z30.h, z6.h, z0.h[7]\n" - "fmla z10.h, z7.h, z0.h[0]\n" - "fmla z13.h, z7.h, z0.h[1]\n" - "fmla z16.h, z7.h, z0.h[2]\n" - "fmla z19.h, z7.h, z0.h[3]\n" - "fmla z22.h, z7.h, z0.h[4]\n" - "fmla z25.h, z7.h, z0.h[5]\n" - "fmla z28.h, z7.h, z0.h[6]\n" - "fmla z31.h, z7.h, z0.h[7]\n" + "fmla z26.h, z2.h, z3.h[6]\n" + "fmla z29.h, z2.h, z3.h[7]\n" + "fmla z9.h, z1.h, z3.h[0]\n" + "fmla z12.h, z1.h, z3.h[1]\n" + "fmla z15.h, z1.h, z3.h[2]\n" + "fmla z18.h, z1.h, z3.h[3]\n" + "fmla z21.h, z1.h, z3.h[4]\n" + "fmla z24.h, z1.h, z3.h[5]\n" + "fmla z27.h, z1.h, z3.h[6]\n" + "fmla z30.h, z1.h, z3.h[7]\n" + "fmla z10.h, z0.h, z3.h[0]\n" + "fmla z13.h, z0.h, z3.h[1]\n" + "fmla z16.h, z0.h, z3.h[2]\n" + "fmla z19.h, z0.h, z3.h[3]\n" + "fmla z22.h, z0.h, z3.h[4]\n" + "fmla z25.h, z0.h, z3.h[5]\n" + "fmla z28.h, z0.h, z3.h[6]\n" + "fmla z31.h, z0.h, z3.h[7]\n" "5:" // multiply loop done "st1h { z8.h }, p0, [%x[Cpanel]]\n" "subs x23, x23, #0x1\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL.hpp index 29b928ee3bf8d30b62ef6ed550748d2050a15ad4..23ab7ce10ac75b40e0ae6105f5de5b129ab3709e 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef ARM_COMPUTE_ENABLE_SVE + #include "../std_transforms_sve.hpp" #include "../performance_parameters.hpp" @@ -56,11 +56,6 @@ public: return get_vector_length() * 3; } - static unsigned int stripe_width() - { - return get_vector_length(); - } - static constexpr unsigned int k_unroll() { return 1; @@ -75,10 +70,14 @@ public: if (std::is_same::value) { switch (ci->get_cpu_model()) { - case CPUModel::V1: - return { 15.15, 9.24, 6.42 }; default: return { 7.2307, 3.876, 2.932 }; + case CPUModel::A64FX: + return { 26.52, 3.42, 4.59 }; + case CPUModel::A510: + return { 6.25, 3.84, 2.47 }; + case CPUModel::V1: + return { 15.15, 9.24, 6.42 }; } } @@ -102,5 +101,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL/a64fx.cpp index 3141a258a8adc69f3944aa0e81506ca1ecadda56..0b139137173d41de5a32829df9b40089bc1ff361 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL/a64fx.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL/a64fx.cpp @@ -28,8 +28,12 @@ namespace arm_gemm { void sve_interleaved_fp32_mla_8x3VL_a64fx( - const float *Apanel, const float *Bpanel, - float *Cpanel, int ablocks, int bblocks, int K) { + const float *Apanel, + const float *Bpanel, + float *Cpanel, + int ablocks, + int bblocks, + int K) { struct KernelArgs { size_t K = {}; @@ -88,7 +92,7 @@ void sve_interleaved_fp32_mla_8x3VL_a64fx( "fmla z9.s, p0/M, z1.s, z3.s\n" "sub x20, x20, #0x2\n" "fmla z10.s, p0/M, z2.s, z3.s\n" - "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n" + "ld1rw { z7.s }, p0/Z, [%x[Apanel], #16]\n" "fmla z11.s, p0/M, z0.s, z4.s\n" "fmla z12.s, p0/M, z1.s, z4.s\n" "fmla z13.s, p0/M, z2.s, z4.s\n" @@ -97,63 +101,63 @@ void sve_interleaved_fp32_mla_8x3VL_a64fx( "fmla z15.s, p0/M, z1.s, z5.s\n" "cmp x20, #0x2\n" "fmla z16.s, p0/M, z2.s, z5.s\n" - "ld1rw { z5.s }, p0/Z, [%x[Apanel], #24]\n" + "ld1rw { z3.s }, p0/Z, [%x[Apanel], #24]\n" "fmla z17.s, p0/M, z0.s, z6.s\n" "fmla z18.s, p0/M, z1.s, z6.s\n" "fmla z19.s, p0/M, z2.s, z6.s\n" - "ld1rw { z6.s }, p0/Z, [%x[Apanel], #28]\n" - "fmla z20.s, p0/M, z0.s, z3.s\n" - "fmla z21.s, p0/M, z1.s, z3.s\n" - "fmla z22.s, p0/M, z2.s, z3.s\n" - "ld1rw { z3.s }, p0/Z, [%x[Apanel], #32]\n" + "ld1rw { z5.s }, p0/Z, [%x[Apanel], #28]\n" + "fmla z20.s, p0/M, z0.s, z7.s\n" + "fmla z21.s, p0/M, z1.s, z7.s\n" + "fmla z22.s, p0/M, z2.s, z7.s\n" + "ld1rw { z7.s }, p0/Z, [%x[Apanel], #32]\n" "fmla z23.s, p0/M, z0.s, z4.s\n" "fmla z24.s, p0/M, z1.s, z4.s\n" "fmla z25.s, p0/M, z2.s, z4.s\n" "ld1rw { z4.s }, p0/Z, [%x[Apanel], #36]\n" - "fmla z26.s, p0/M, z0.s, z5.s\n" - "fmla z27.s, p0/M, z1.s, z5.s\n" - "fmla z28.s, p0/M, z2.s, z5.s\n" - "ld1rw { z5.s }, p0/Z, [%x[Apanel], #40]\n" - "fmla z29.s, p0/M, z0.s, z6.s\n" - "ld1w { z0.s }, p0/Z, [x22, #3, MUL VL]\n" - "fmla z30.s, p0/M, z1.s, z6.s\n" - "fmla z31.s, p0/M, z2.s, z6.s\n" - "ld1w { z1.s }, p0/Z, [x22, #4, MUL VL]\n" - "ld1w { z2.s }, p0/Z, [x22, #5, MUL VL]\n" - "fmla z8.s, p0/M, z0.s, z3.s\n" - "ld1rw { z6.s }, p0/Z, [%x[Apanel], #44]\n" - "fmla z9.s, p0/M, z1.s, z3.s\n" - "fmla z10.s, p0/M, z2.s, z3.s\n" - "fmla z11.s, p0/M, z0.s, z4.s\n" - "ld1rw { z3.s }, p0/Z, [%x[Apanel], #48]\n" - "fmla z12.s, p0/M, z1.s, z4.s\n" - "fmla z13.s, p0/M, z2.s, z4.s\n" + "fmla z26.s, p0/M, z0.s, z3.s\n" + "fmla z27.s, p0/M, z1.s, z3.s\n" + "fmla z28.s, p0/M, z2.s, z3.s\n" + "ld1rw { z3.s }, p0/Z, [%x[Apanel], #40]\n" + "fmla z29.s, p0/M, z0.s, z5.s\n" + "ld1w { z6.s }, p0/Z, [x22, #3, MUL VL]\n" + "fmla z30.s, p0/M, z1.s, z5.s\n" + "fmla z31.s, p0/M, z2.s, z5.s\n" + "ld1w { z2.s }, p0/Z, [x22, #4, MUL VL]\n" + "ld1w { z5.s }, p0/Z, [x22, #5, MUL VL]\n" + "fmla z8.s, p0/M, z6.s, z7.s\n" + "ld1rw { z1.s }, p0/Z, [%x[Apanel], #44]\n" + "fmla z9.s, p0/M, z2.s, z7.s\n" + "fmla z10.s, p0/M, z5.s, z7.s\n" + "fmla z11.s, p0/M, z6.s, z4.s\n" + "ld1rw { z7.s }, p0/Z, [%x[Apanel], #48]\n" + "fmla z12.s, p0/M, z2.s, z4.s\n" + "fmla z13.s, p0/M, z5.s, z4.s\n" "ld1rw { z4.s }, p0/Z, [%x[Apanel], #52]\n" - "fmla z14.s, p0/M, z0.s, z5.s\n" - "fmla z15.s, p0/M, z1.s, z5.s\n" + "fmla z14.s, p0/M, z6.s, z3.s\n" + "fmla z15.s, p0/M, z2.s, z3.s\n" "addvl x22, x22, #6\n" - "fmla z16.s, p0/M, z2.s, z5.s\n" - "ld1rw { z5.s }, p0/Z, [%x[Apanel], #56]\n" - "fmla z17.s, p0/M, z0.s, z6.s\n" - "fmla z18.s, p0/M, z1.s, z6.s\n" - "fmla z19.s, p0/M, z2.s, z6.s\n" - "ld1rw { z6.s }, p0/Z, [%x[Apanel], #60]\n" + "fmla z16.s, p0/M, z5.s, z3.s\n" + "ld1rw { z0.s }, p0/Z, [%x[Apanel], #56]\n" + "fmla z17.s, p0/M, z6.s, z1.s\n" + "fmla z18.s, p0/M, z2.s, z1.s\n" + "fmla z19.s, p0/M, z5.s, z1.s\n" + "ld1rw { z1.s }, p0/Z, [%x[Apanel], #60]\n" "add %x[Apanel], %x[Apanel], #0x40\n" - "fmla z20.s, p0/M, z0.s, z3.s\n" - "fmla z21.s, p0/M, z1.s, z3.s\n" - "fmla z22.s, p0/M, z2.s, z3.s\n" - "fmla z23.s, p0/M, z0.s, z4.s\n" + "fmla z20.s, p0/M, z6.s, z7.s\n" + "fmla z21.s, p0/M, z2.s, z7.s\n" + "fmla z22.s, p0/M, z5.s, z7.s\n" + "fmla z23.s, p0/M, z6.s, z4.s\n" "ld1rw { z3.s }, p0/Z, [%x[Apanel]]\n" - "fmla z24.s, p0/M, z1.s, z4.s\n" - "fmla z25.s, p0/M, z2.s, z4.s\n" + "fmla z24.s, p0/M, z2.s, z4.s\n" + "fmla z25.s, p0/M, z5.s, z4.s\n" "ld1rw { z4.s }, p0/Z, [%x[Apanel], #4]\n" - "fmla z26.s, p0/M, z0.s, z5.s\n" - "fmla z27.s, p0/M, z1.s, z5.s\n" - "fmla z28.s, p0/M, z2.s, z5.s\n" - "fmla z29.s, p0/M, z0.s, z6.s\n" + "fmla z26.s, p0/M, z6.s, z0.s\n" + "fmla z27.s, p0/M, z2.s, z0.s\n" + "fmla z28.s, p0/M, z5.s, z0.s\n" + "fmla z29.s, p0/M, z6.s, z1.s\n" "ld1w { z0.s }, p0/Z, [x22]\n" - "fmla z30.s, p0/M, z1.s, z6.s\n" - "fmla z31.s, p0/M, z2.s, z6.s\n" + "fmla z30.s, p0/M, z2.s, z1.s\n" + "fmla z31.s, p0/M, z5.s, z1.s\n" "ld1w { z1.s }, p0/Z, [x22, #1, MUL VL]\n" "ld1w { z2.s }, p0/Z, [x22, #2, MUL VL]\n" "ld1rw { z5.s }, p0/Z, [%x[Apanel], #8]\n" @@ -164,7 +168,7 @@ void sve_interleaved_fp32_mla_8x3VL_a64fx( "fmla z9.s, p0/M, z1.s, z3.s\n" "addvl x22, x22, #3\n" "fmla z10.s, p0/M, z2.s, z3.s\n" - "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n" + "ld1rw { z7.s }, p0/Z, [%x[Apanel], #16]\n" "fmla z11.s, p0/M, z0.s, z4.s\n" "fmla z12.s, p0/M, z1.s, z4.s\n" "fmla z13.s, p0/M, z2.s, z4.s\n" @@ -176,58 +180,58 @@ void sve_interleaved_fp32_mla_8x3VL_a64fx( "fmla z17.s, p0/M, z0.s, z6.s\n" "fmla z18.s, p0/M, z1.s, z6.s\n" "fmla z19.s, p0/M, z2.s, z6.s\n" - "ld1rw { z6.s }, p0/Z, [%x[Apanel], #28]\n" - "fmla z20.s, p0/M, z0.s, z3.s\n" - "fmla z21.s, p0/M, z1.s, z3.s\n" + "ld1rw { z3.s }, p0/Z, [%x[Apanel], #28]\n" + "fmla z20.s, p0/M, z0.s, z7.s\n" + "fmla z21.s, p0/M, z1.s, z7.s\n" "add %x[Apanel], %x[Apanel], #0x20\n" - "fmla z22.s, p0/M, z2.s, z3.s\n" + "fmla z22.s, p0/M, z2.s, z7.s\n" "fmla z23.s, p0/M, z0.s, z4.s\n" "fmla z24.s, p0/M, z1.s, z4.s\n" "fmla z25.s, p0/M, z2.s, z4.s\n" "fmla z26.s, p0/M, z0.s, z5.s\n" "fmla z27.s, p0/M, z1.s, z5.s\n" "fmla z28.s, p0/M, z2.s, z5.s\n" - "fmla z29.s, p0/M, z0.s, z6.s\n" - "fmla z30.s, p0/M, z1.s, z6.s\n" - "fmla z31.s, p0/M, z2.s, z6.s\n" + "fmla z29.s, p0/M, z0.s, z3.s\n" + "fmla z30.s, p0/M, z1.s, z3.s\n" + "fmla z31.s, p0/M, z2.s, z3.s\n" "cbz x20, 5f\n" - "ld1w { z0.s }, p0/Z, [x22]\n" - "ld1w { z1.s }, p0/Z, [x22, #1, MUL VL]\n" - "ld1w { z2.s }, p0/Z, [x22, #2, MUL VL]\n" + "ld1w { z6.s }, p0/Z, [x22]\n" + "ld1w { z5.s }, p0/Z, [x22, #1, MUL VL]\n" + "ld1w { z4.s }, p0/Z, [x22, #2, MUL VL]\n" "ld1rw { z3.s }, p0/Z, [%x[Apanel]]\n" - "fmla z8.s, p0/M, z0.s, z3.s\n" - "ld1rw { z4.s }, p0/Z, [%x[Apanel], #4]\n" - "ld1rw { z5.s }, p0/Z, [%x[Apanel], #8]\n" - "fmla z9.s, p0/M, z1.s, z3.s\n" - "ld1rw { z6.s }, p0/Z, [%x[Apanel], #12]\n" - "fmla z10.s, p0/M, z2.s, z3.s\n" - "fmla z11.s, p0/M, z0.s, z4.s\n" - "fmla z12.s, p0/M, z1.s, z4.s\n" - "fmla z13.s, p0/M, z2.s, z4.s\n" + "fmla z8.s, p0/M, z6.s, z3.s\n" + "ld1rw { z2.s }, p0/Z, [%x[Apanel], #4]\n" + "ld1rw { z1.s }, p0/Z, [%x[Apanel], #8]\n" + "fmla z9.s, p0/M, z5.s, z3.s\n" + "ld1rw { z0.s }, p0/Z, [%x[Apanel], #12]\n" + "fmla z10.s, p0/M, z4.s, z3.s\n" + "fmla z11.s, p0/M, z6.s, z2.s\n" + "fmla z12.s, p0/M, z5.s, z2.s\n" + "fmla z13.s, p0/M, z4.s, z2.s\n" "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n" - "fmla z14.s, p0/M, z0.s, z5.s\n" - "fmla z15.s, p0/M, z1.s, z5.s\n" - "ld1rw { z4.s }, p0/Z, [%x[Apanel], #20]\n" - "fmla z16.s, p0/M, z2.s, z5.s\n" - "fmla z17.s, p0/M, z0.s, z6.s\n" - "ld1rw { z5.s }, p0/Z, [%x[Apanel], #24]\n" - "fmla z18.s, p0/M, z1.s, z6.s\n" - "fmla z19.s, p0/M, z2.s, z6.s\n" - "ld1rw { z6.s }, p0/Z, [%x[Apanel], #28]\n" - "fmla z20.s, p0/M, z0.s, z3.s\n" - "fmla z21.s, p0/M, z1.s, z3.s\n" + "fmla z14.s, p0/M, z6.s, z1.s\n" + "fmla z15.s, p0/M, z5.s, z1.s\n" + "ld1rw { z2.s }, p0/Z, [%x[Apanel], #20]\n" + "fmla z16.s, p0/M, z4.s, z1.s\n" + "fmla z17.s, p0/M, z6.s, z0.s\n" + "ld1rw { z1.s }, p0/Z, [%x[Apanel], #24]\n" + "fmla z18.s, p0/M, z5.s, z0.s\n" + "fmla z19.s, p0/M, z4.s, z0.s\n" + "ld1rw { z0.s }, p0/Z, [%x[Apanel], #28]\n" + "fmla z20.s, p0/M, z6.s, z3.s\n" + "fmla z21.s, p0/M, z5.s, z3.s\n" "addvl x22, x22, #3\n" - "fmla z22.s, p0/M, z2.s, z3.s\n" - "fmla z23.s, p0/M, z0.s, z4.s\n" + "fmla z22.s, p0/M, z4.s, z3.s\n" + "fmla z23.s, p0/M, z6.s, z2.s\n" "add %x[Apanel], %x[Apanel], #0x20\n" - "fmla z24.s, p0/M, z1.s, z4.s\n" - "fmla z25.s, p0/M, z2.s, z4.s\n" - "fmla z26.s, p0/M, z0.s, z5.s\n" - "fmla z27.s, p0/M, z1.s, z5.s\n" - "fmla z28.s, p0/M, z2.s, z5.s\n" - "fmla z29.s, p0/M, z0.s, z6.s\n" - "fmla z30.s, p0/M, z1.s, z6.s\n" - "fmla z31.s, p0/M, z2.s, z6.s\n" + "fmla z24.s, p0/M, z5.s, z2.s\n" + "fmla z25.s, p0/M, z4.s, z2.s\n" + "fmla z26.s, p0/M, z6.s, z1.s\n" + "fmla z27.s, p0/M, z5.s, z1.s\n" + "fmla z28.s, p0/M, z4.s, z1.s\n" + "fmla z29.s, p0/M, z6.s, z0.s\n" + "fmla z30.s, p0/M, z5.s, z0.s\n" + "fmla z31.s, p0/M, z4.s, z0.s\n" "5:" // multiply loop done "st1w { z8.s }, p0, [%x[Cpanel]]\n" "subs x23, x23, #0x1\n" @@ -261,7 +265,7 @@ void sve_interleaved_fp32_mla_8x3VL_a64fx( "bne 1b\n" : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks) : [args_ptr] "r" (&ka), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_bblocks] "I" (offsetof(KernelArgs, bblocks)) - : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL/generic.cpp index 9d1c0c3728504131e9dc89e5f82e6f2af96cad68..c7f32ff7a99d4478eb7ce39837d4f6fc14003948 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_8x3VL/generic.cpp @@ -28,8 +28,12 @@ namespace arm_gemm { void sve_interleaved_fp32_mla_8x3VL( - const float *Apanel, const float *Bpanel, - float *Cpanel, int ablocks, int bblocks, int K) { + const float *Apanel, + const float *Bpanel, + float *Cpanel, + int ablocks, + int bblocks, + int K) { struct KernelArgs { size_t K = {}; @@ -84,10 +88,10 @@ void sve_interleaved_fp32_mla_8x3VL( "3:" // main loop head "fmla z8.s, z4.s, z0.s[0]\n" "fmla z11.s, z4.s, z0.s[1]\n" - "ld1rqw { z2.s }, p0/Z, [%x[Apanel], #32]\n" + "ld1rqw { z3.s }, p0/Z, [%x[Apanel], #32]\n" "fmla z14.s, z4.s, z0.s[2]\n" "fmla z17.s, z4.s, z0.s[3]\n" - "ld1rqw { z3.s }, p0/Z, [%x[Apanel], #48]\n" + "ld1rqw { z7.s }, p0/Z, [%x[Apanel], #48]\n" "fmla z20.s, z4.s, z1.s[0]\n" "fmla z23.s, z4.s, z1.s[1]\n" "sub x20, x20, #0x2\n" @@ -114,35 +118,35 @@ void sve_interleaved_fp32_mla_8x3VL( "fmla z25.s, z6.s, z1.s[1]\n" "fmla z28.s, z6.s, z1.s[2]\n" "fmla z31.s, z6.s, z1.s[3]\n" - "ld1w { z6.s }, p0/Z, [x22, #5, MUL VL]\n" + "ld1w { z2.s }, p0/Z, [x22, #5, MUL VL]\n" "addvl x22, x22, #6\n" - "fmla z8.s, z4.s, z2.s[0]\n" - "fmla z11.s, z4.s, z2.s[1]\n" + "fmla z8.s, z4.s, z3.s[0]\n" + "fmla z11.s, z4.s, z3.s[1]\n" "ld1rqw { z1.s }, p0/Z, [%x[Apanel], #16]\n" - "fmla z14.s, z4.s, z2.s[2]\n" - "fmla z17.s, z4.s, z2.s[3]\n" - "fmla z20.s, z4.s, z3.s[0]\n" - "fmla z23.s, z4.s, z3.s[1]\n" - "fmla z26.s, z4.s, z3.s[2]\n" - "fmla z29.s, z4.s, z3.s[3]\n" + "fmla z14.s, z4.s, z3.s[2]\n" + "fmla z17.s, z4.s, z3.s[3]\n" + "fmla z20.s, z4.s, z7.s[0]\n" + "fmla z23.s, z4.s, z7.s[1]\n" + "fmla z26.s, z4.s, z7.s[2]\n" + "fmla z29.s, z4.s, z7.s[3]\n" "ld1w { z4.s }, p0/Z, [x22]\n" - "fmla z9.s, z5.s, z2.s[0]\n" - "fmla z12.s, z5.s, z2.s[1]\n" - "fmla z15.s, z5.s, z2.s[2]\n" - "fmla z18.s, z5.s, z2.s[3]\n" - "fmla z21.s, z5.s, z3.s[0]\n" - "fmla z24.s, z5.s, z3.s[1]\n" - "fmla z27.s, z5.s, z3.s[2]\n" - "fmla z30.s, z5.s, z3.s[3]\n" + "fmla z9.s, z5.s, z3.s[0]\n" + "fmla z12.s, z5.s, z3.s[1]\n" + "fmla z15.s, z5.s, z3.s[2]\n" + "fmla z18.s, z5.s, z3.s[3]\n" + "fmla z21.s, z5.s, z7.s[0]\n" + "fmla z24.s, z5.s, z7.s[1]\n" + "fmla z27.s, z5.s, z7.s[2]\n" + "fmla z30.s, z5.s, z7.s[3]\n" "ld1w { z5.s }, p0/Z, [x22, #1, MUL VL]\n" - "fmla z10.s, z6.s, z2.s[0]\n" - "fmla z13.s, z6.s, z2.s[1]\n" - "fmla z16.s, z6.s, z2.s[2]\n" - "fmla z19.s, z6.s, z2.s[3]\n" - "fmla z22.s, z6.s, z3.s[0]\n" - "fmla z25.s, z6.s, z3.s[1]\n" - "fmla z28.s, z6.s, z3.s[2]\n" - "fmla z31.s, z6.s, z3.s[3]\n" + "fmla z10.s, z2.s, z3.s[0]\n" + "fmla z13.s, z2.s, z3.s[1]\n" + "fmla z16.s, z2.s, z3.s[2]\n" + "fmla z19.s, z2.s, z3.s[3]\n" + "fmla z22.s, z2.s, z7.s[0]\n" + "fmla z25.s, z2.s, z7.s[1]\n" + "fmla z28.s, z2.s, z7.s[2]\n" + "fmla z31.s, z2.s, z7.s[3]\n" "ld1w { z6.s }, p0/Z, [x22, #2, MUL VL]\n" "bge 3b\n" "4:" // main loop skip @@ -173,37 +177,37 @@ void sve_interleaved_fp32_mla_8x3VL( "fmla z28.s, z6.s, z1.s[2]\n" "fmla z31.s, z6.s, z1.s[3]\n" "cbz x20, 5f\n" - "ld1rqw { z0.s }, p0/Z, [%x[Apanel]]\n" - "ld1rqw { z1.s }, p0/Z, [%x[Apanel], #16]\n" + "ld1rqw { z4.s }, p0/Z, [%x[Apanel]]\n" + "ld1rqw { z3.s }, p0/Z, [%x[Apanel], #16]\n" "add %x[Apanel], %x[Apanel], #0x20\n" - "ld1w { z7.s }, p0/Z, [x22]\n" - "ld1w { z4.s }, p0/Z, [x22, #1, MUL VL]\n" - "fmla z8.s, z7.s, z0.s[0]\n" - "ld1w { z5.s }, p0/Z, [x22, #2, MUL VL]\n" - "fmla z11.s, z7.s, z0.s[1]\n" - "fmla z14.s, z7.s, z0.s[2]\n" - "fmla z17.s, z7.s, z0.s[3]\n" - "fmla z20.s, z7.s, z1.s[0]\n" + "ld1w { z2.s }, p0/Z, [x22]\n" + "ld1w { z1.s }, p0/Z, [x22, #1, MUL VL]\n" + "fmla z8.s, z2.s, z4.s[0]\n" + "ld1w { z0.s }, p0/Z, [x22, #2, MUL VL]\n" + "fmla z11.s, z2.s, z4.s[1]\n" + "fmla z14.s, z2.s, z4.s[2]\n" + "fmla z17.s, z2.s, z4.s[3]\n" + "fmla z20.s, z2.s, z3.s[0]\n" "addvl x22, x22, #3\n" - "fmla z23.s, z7.s, z1.s[1]\n" - "fmla z26.s, z7.s, z1.s[2]\n" - "fmla z29.s, z7.s, z1.s[3]\n" - "fmla z9.s, z4.s, z0.s[0]\n" - "fmla z12.s, z4.s, z0.s[1]\n" - "fmla z15.s, z4.s, z0.s[2]\n" - "fmla z18.s, z4.s, z0.s[3]\n" - "fmla z21.s, z4.s, z1.s[0]\n" - "fmla z24.s, z4.s, z1.s[1]\n" - "fmla z27.s, z4.s, z1.s[2]\n" - "fmla z30.s, z4.s, z1.s[3]\n" - "fmla z10.s, z5.s, z0.s[0]\n" - "fmla z13.s, z5.s, z0.s[1]\n" - "fmla z16.s, z5.s, z0.s[2]\n" - "fmla z19.s, z5.s, z0.s[3]\n" - "fmla z22.s, z5.s, z1.s[0]\n" - "fmla z25.s, z5.s, z1.s[1]\n" - "fmla z28.s, z5.s, z1.s[2]\n" - "fmla z31.s, z5.s, z1.s[3]\n" + "fmla z23.s, z2.s, z3.s[1]\n" + "fmla z26.s, z2.s, z3.s[2]\n" + "fmla z29.s, z2.s, z3.s[3]\n" + "fmla z9.s, z1.s, z4.s[0]\n" + "fmla z12.s, z1.s, z4.s[1]\n" + "fmla z15.s, z1.s, z4.s[2]\n" + "fmla z18.s, z1.s, z4.s[3]\n" + "fmla z21.s, z1.s, z3.s[0]\n" + "fmla z24.s, z1.s, z3.s[1]\n" + "fmla z27.s, z1.s, z3.s[2]\n" + "fmla z30.s, z1.s, z3.s[3]\n" + "fmla z10.s, z0.s, z4.s[0]\n" + "fmla z13.s, z0.s, z4.s[1]\n" + "fmla z16.s, z0.s, z4.s[2]\n" + "fmla z19.s, z0.s, z4.s[3]\n" + "fmla z22.s, z0.s, z3.s[0]\n" + "fmla z25.s, z0.s, z3.s[1]\n" + "fmla z28.s, z0.s, z3.s[2]\n" + "fmla z31.s, z0.s, z3.s[3]\n" "5:" // multiply loop done "st1w { z8.s }, p0, [%x[Cpanel]]\n" "subs x23, x23, #0x1\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL.hpp index 0d707b0391a88e1dc4c37c5927d3c1129c2998f0..cf3069f828b01b0f6b7d77219fdff0afdea2ddd8 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef ARM_COMPUTE_ENABLE_SVE + #include "../std_transforms_sve.hpp" #include "../performance_parameters.hpp" @@ -35,6 +35,7 @@ namespace arm_gemm { // Actual kernel implementations void sve_interleaved_s8s32_dot_8x3VL( ARGLIST ); +void sve_interleaved_s8s32_dot_8x3VL_a64fx( ARGLIST ); class cls_sve_interleaved_s8s32_dot_8x3VL { @@ -55,11 +56,6 @@ public: return get_vector_length() * 3; } - static unsigned int stripe_width() - { - return get_vector_length(); - } - static constexpr unsigned int k_unroll() { return 4; @@ -80,6 +76,8 @@ public: return { 63.30, 4.97, 11.35 }; case CPUModel::A510: return { 27.42, 3.47, 2.88 }; + case CPUModel::A64FX: + return { 109.18, 3.88, 7.85 }; } } @@ -92,6 +90,8 @@ public: return { 52.24, 7.49, 0.80 }; case CPUModel::A510: return { 27.47, 1.70, 0.28 }; + case CPUModel::A64FX: + return { 109.92, 2.36, 0.41 }; } } @@ -100,13 +100,19 @@ public: // Default to the generic kernel kern_type kernel=sve_interleaved_s8s32_dot_8x3VL; - cls_sve_interleaved_s8s32_dot_8x3VL(const CPUInfo *) + cls_sve_interleaved_s8s32_dot_8x3VL(const CPUInfo *ci) { + switch(ci->get_cpu_model()) { + default: + break; + case CPUModel::A64FX: + kernel=sve_interleaved_s8s32_dot_8x3VL_a64fx; + break; + } } }; } // namespace arm_gemm #undef ARGLIST - #endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL/a64fx.cpp index a7ca48d87aa6434be2fedeb74c54a63b833a6c15..c668a7b746fd5a6f5ea733bc05dc38b29d88f962 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL/a64fx.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL/a64fx.cpp @@ -29,8 +29,12 @@ namespace arm_gemm { void sve_interleaved_s8s32_dot_8x3VL_a64fx( - const int8_t *Apanel, const int8_t *Bpanel, - int32_t *Cpanel, int ablocks, int bblocks, int K) { + const int8_t *Apanel, + const int8_t *Bpanel, + int32_t *Cpanel, + int ablocks, + int bblocks, + int K) { struct KernelArgs { size_t K = {}; @@ -89,7 +93,7 @@ void sve_interleaved_s8s32_dot_8x3VL_a64fx( "sdot z9.s, z1.b, z3.b\n" "sub x20, x20, #0x2\n" "sdot z10.s, z2.b, z3.b\n" - "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n" + "ld1rw { z7.s }, p0/Z, [%x[Apanel], #16]\n" "sdot z11.s, z0.b, z4.b\n" "sdot z12.s, z1.b, z4.b\n" "sdot z13.s, z2.b, z4.b\n" @@ -98,63 +102,63 @@ void sve_interleaved_s8s32_dot_8x3VL_a64fx( "sdot z15.s, z1.b, z5.b\n" "cmp x20, #0x2\n" "sdot z16.s, z2.b, z5.b\n" - "ld1rw { z5.s }, p0/Z, [%x[Apanel], #24]\n" + "ld1rw { z3.s }, p0/Z, [%x[Apanel], #24]\n" "sdot z17.s, z0.b, z6.b\n" "sdot z18.s, z1.b, z6.b\n" "sdot z19.s, z2.b, z6.b\n" - "ld1rw { z6.s }, p0/Z, [%x[Apanel], #28]\n" - "sdot z20.s, z0.b, z3.b\n" - "sdot z21.s, z1.b, z3.b\n" - "sdot z22.s, z2.b, z3.b\n" - "ld1rw { z3.s }, p0/Z, [%x[Apanel], #32]\n" + "ld1rw { z5.s }, p0/Z, [%x[Apanel], #28]\n" + "sdot z20.s, z0.b, z7.b\n" + "sdot z21.s, z1.b, z7.b\n" + "sdot z22.s, z2.b, z7.b\n" + "ld1rw { z7.s }, p0/Z, [%x[Apanel], #32]\n" "sdot z23.s, z0.b, z4.b\n" "sdot z24.s, z1.b, z4.b\n" "sdot z25.s, z2.b, z4.b\n" "ld1rw { z4.s }, p0/Z, [%x[Apanel], #36]\n" - "sdot z26.s, z0.b, z5.b\n" - "sdot z27.s, z1.b, z5.b\n" - "sdot z28.s, z2.b, z5.b\n" - "ld1rw { z5.s }, p0/Z, [%x[Apanel], #40]\n" - "sdot z29.s, z0.b, z6.b\n" - "ld1b { z0.b }, p0/Z, [x22, #3, MUL VL]\n" - "sdot z30.s, z1.b, z6.b\n" - "sdot z31.s, z2.b, z6.b\n" - "ld1b { z1.b }, p0/Z, [x22, #4, MUL VL]\n" - "ld1b { z2.b }, p0/Z, [x22, #5, MUL VL]\n" - "sdot z8.s, z0.b, z3.b\n" - "ld1rw { z6.s }, p0/Z, [%x[Apanel], #44]\n" - "sdot z9.s, z1.b, z3.b\n" - "sdot z10.s, z2.b, z3.b\n" - "sdot z11.s, z0.b, z4.b\n" - "ld1rw { z3.s }, p0/Z, [%x[Apanel], #48]\n" - "sdot z12.s, z1.b, z4.b\n" - "sdot z13.s, z2.b, z4.b\n" + "sdot z26.s, z0.b, z3.b\n" + "sdot z27.s, z1.b, z3.b\n" + "sdot z28.s, z2.b, z3.b\n" + "ld1rw { z3.s }, p0/Z, [%x[Apanel], #40]\n" + "sdot z29.s, z0.b, z5.b\n" + "ld1b { z6.b }, p0/Z, [x22, #3, MUL VL]\n" + "sdot z30.s, z1.b, z5.b\n" + "sdot z31.s, z2.b, z5.b\n" + "ld1b { z2.b }, p0/Z, [x22, #4, MUL VL]\n" + "ld1b { z5.b }, p0/Z, [x22, #5, MUL VL]\n" + "sdot z8.s, z6.b, z7.b\n" + "ld1rw { z1.s }, p0/Z, [%x[Apanel], #44]\n" + "sdot z9.s, z2.b, z7.b\n" + "sdot z10.s, z5.b, z7.b\n" + "sdot z11.s, z6.b, z4.b\n" + "ld1rw { z7.s }, p0/Z, [%x[Apanel], #48]\n" + "sdot z12.s, z2.b, z4.b\n" + "sdot z13.s, z5.b, z4.b\n" "ld1rw { z4.s }, p0/Z, [%x[Apanel], #52]\n" - "sdot z14.s, z0.b, z5.b\n" - "sdot z15.s, z1.b, z5.b\n" + "sdot z14.s, z6.b, z3.b\n" + "sdot z15.s, z2.b, z3.b\n" "addvl x22, x22, #6\n" - "sdot z16.s, z2.b, z5.b\n" - "ld1rw { z5.s }, p0/Z, [%x[Apanel], #56]\n" - "sdot z17.s, z0.b, z6.b\n" - "sdot z18.s, z1.b, z6.b\n" - "sdot z19.s, z2.b, z6.b\n" - "ld1rw { z6.s }, p0/Z, [%x[Apanel], #60]\n" + "sdot z16.s, z5.b, z3.b\n" + "ld1rw { z0.s }, p0/Z, [%x[Apanel], #56]\n" + "sdot z17.s, z6.b, z1.b\n" + "sdot z18.s, z2.b, z1.b\n" + "sdot z19.s, z5.b, z1.b\n" + "ld1rw { z1.s }, p0/Z, [%x[Apanel], #60]\n" "add %x[Apanel], %x[Apanel], #0x40\n" - "sdot z20.s, z0.b, z3.b\n" - "sdot z21.s, z1.b, z3.b\n" - "sdot z22.s, z2.b, z3.b\n" - "sdot z23.s, z0.b, z4.b\n" + "sdot z20.s, z6.b, z7.b\n" + "sdot z21.s, z2.b, z7.b\n" + "sdot z22.s, z5.b, z7.b\n" + "sdot z23.s, z6.b, z4.b\n" "ld1rw { z3.s }, p0/Z, [%x[Apanel]]\n" - "sdot z24.s, z1.b, z4.b\n" - "sdot z25.s, z2.b, z4.b\n" + "sdot z24.s, z2.b, z4.b\n" + "sdot z25.s, z5.b, z4.b\n" "ld1rw { z4.s }, p0/Z, [%x[Apanel], #4]\n" - "sdot z26.s, z0.b, z5.b\n" - "sdot z27.s, z1.b, z5.b\n" - "sdot z28.s, z2.b, z5.b\n" - "sdot z29.s, z0.b, z6.b\n" + "sdot z26.s, z6.b, z0.b\n" + "sdot z27.s, z2.b, z0.b\n" + "sdot z28.s, z5.b, z0.b\n" + "sdot z29.s, z6.b, z1.b\n" "ld1b { z0.b }, p0/Z, [x22]\n" - "sdot z30.s, z1.b, z6.b\n" - "sdot z31.s, z2.b, z6.b\n" + "sdot z30.s, z2.b, z1.b\n" + "sdot z31.s, z5.b, z1.b\n" "ld1b { z1.b }, p0/Z, [x22, #1, MUL VL]\n" "ld1b { z2.b }, p0/Z, [x22, #2, MUL VL]\n" "ld1rw { z5.s }, p0/Z, [%x[Apanel], #8]\n" @@ -165,7 +169,7 @@ void sve_interleaved_s8s32_dot_8x3VL_a64fx( "sdot z9.s, z1.b, z3.b\n" "addvl x22, x22, #3\n" "sdot z10.s, z2.b, z3.b\n" - "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n" + "ld1rw { z7.s }, p0/Z, [%x[Apanel], #16]\n" "sdot z11.s, z0.b, z4.b\n" "sdot z12.s, z1.b, z4.b\n" "sdot z13.s, z2.b, z4.b\n" @@ -177,58 +181,58 @@ void sve_interleaved_s8s32_dot_8x3VL_a64fx( "sdot z17.s, z0.b, z6.b\n" "sdot z18.s, z1.b, z6.b\n" "sdot z19.s, z2.b, z6.b\n" - "ld1rw { z6.s }, p0/Z, [%x[Apanel], #28]\n" - "sdot z20.s, z0.b, z3.b\n" - "sdot z21.s, z1.b, z3.b\n" + "ld1rw { z3.s }, p0/Z, [%x[Apanel], #28]\n" + "sdot z20.s, z0.b, z7.b\n" + "sdot z21.s, z1.b, z7.b\n" "add %x[Apanel], %x[Apanel], #0x20\n" - "sdot z22.s, z2.b, z3.b\n" + "sdot z22.s, z2.b, z7.b\n" "sdot z23.s, z0.b, z4.b\n" "sdot z24.s, z1.b, z4.b\n" "sdot z25.s, z2.b, z4.b\n" "sdot z26.s, z0.b, z5.b\n" "sdot z27.s, z1.b, z5.b\n" "sdot z28.s, z2.b, z5.b\n" - "sdot z29.s, z0.b, z6.b\n" - "sdot z30.s, z1.b, z6.b\n" - "sdot z31.s, z2.b, z6.b\n" + "sdot z29.s, z0.b, z3.b\n" + "sdot z30.s, z1.b, z3.b\n" + "sdot z31.s, z2.b, z3.b\n" "cbz x20, 5f\n" - "ld1b { z0.b }, p0/Z, [x22]\n" - "ld1b { z1.b }, p0/Z, [x22, #1, MUL VL]\n" - "ld1b { z2.b }, p0/Z, [x22, #2, MUL VL]\n" + "ld1b { z6.b }, p0/Z, [x22]\n" + "ld1b { z5.b }, p0/Z, [x22, #1, MUL VL]\n" + "ld1b { z4.b }, p0/Z, [x22, #2, MUL VL]\n" "ld1rw { z3.s }, p0/Z, [%x[Apanel]]\n" - "sdot z8.s, z0.b, z3.b\n" - "ld1rw { z4.s }, p0/Z, [%x[Apanel], #4]\n" - "ld1rw { z5.s }, p0/Z, [%x[Apanel], #8]\n" - "sdot z9.s, z1.b, z3.b\n" - "ld1rw { z6.s }, p0/Z, [%x[Apanel], #12]\n" - "sdot z10.s, z2.b, z3.b\n" - "sdot z11.s, z0.b, z4.b\n" - "sdot z12.s, z1.b, z4.b\n" - "sdot z13.s, z2.b, z4.b\n" + "sdot z8.s, z6.b, z3.b\n" + "ld1rw { z2.s }, p0/Z, [%x[Apanel], #4]\n" + "ld1rw { z1.s }, p0/Z, [%x[Apanel], #8]\n" + "sdot z9.s, z5.b, z3.b\n" + "ld1rw { z0.s }, p0/Z, [%x[Apanel], #12]\n" + "sdot z10.s, z4.b, z3.b\n" + "sdot z11.s, z6.b, z2.b\n" + "sdot z12.s, z5.b, z2.b\n" + "sdot z13.s, z4.b, z2.b\n" "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n" - "sdot z14.s, z0.b, z5.b\n" - "sdot z15.s, z1.b, z5.b\n" - "ld1rw { z4.s }, p0/Z, [%x[Apanel], #20]\n" - "sdot z16.s, z2.b, z5.b\n" - "sdot z17.s, z0.b, z6.b\n" - "ld1rw { z5.s }, p0/Z, [%x[Apanel], #24]\n" - "sdot z18.s, z1.b, z6.b\n" - "sdot z19.s, z2.b, z6.b\n" - "ld1rw { z6.s }, p0/Z, [%x[Apanel], #28]\n" - "sdot z20.s, z0.b, z3.b\n" - "sdot z21.s, z1.b, z3.b\n" + "sdot z14.s, z6.b, z1.b\n" + "sdot z15.s, z5.b, z1.b\n" + "ld1rw { z2.s }, p0/Z, [%x[Apanel], #20]\n" + "sdot z16.s, z4.b, z1.b\n" + "sdot z17.s, z6.b, z0.b\n" + "ld1rw { z1.s }, p0/Z, [%x[Apanel], #24]\n" + "sdot z18.s, z5.b, z0.b\n" + "sdot z19.s, z4.b, z0.b\n" + "ld1rw { z0.s }, p0/Z, [%x[Apanel], #28]\n" + "sdot z20.s, z6.b, z3.b\n" + "sdot z21.s, z5.b, z3.b\n" "addvl x22, x22, #3\n" - "sdot z22.s, z2.b, z3.b\n" - "sdot z23.s, z0.b, z4.b\n" + "sdot z22.s, z4.b, z3.b\n" + "sdot z23.s, z6.b, z2.b\n" "add %x[Apanel], %x[Apanel], #0x20\n" - "sdot z24.s, z1.b, z4.b\n" - "sdot z25.s, z2.b, z4.b\n" - "sdot z26.s, z0.b, z5.b\n" - "sdot z27.s, z1.b, z5.b\n" - "sdot z28.s, z2.b, z5.b\n" - "sdot z29.s, z0.b, z6.b\n" - "sdot z30.s, z1.b, z6.b\n" - "sdot z31.s, z2.b, z6.b\n" + "sdot z24.s, z5.b, z2.b\n" + "sdot z25.s, z4.b, z2.b\n" + "sdot z26.s, z6.b, z1.b\n" + "sdot z27.s, z5.b, z1.b\n" + "sdot z28.s, z4.b, z1.b\n" + "sdot z29.s, z6.b, z0.b\n" + "sdot z30.s, z5.b, z0.b\n" + "sdot z31.s, z4.b, z0.b\n" "5:" // multiply loop done "st1w { z8.s }, p0, [%x[Cpanel]]\n" "subs x23, x23, #0x1\n" @@ -262,7 +266,7 @@ void sve_interleaved_s8s32_dot_8x3VL_a64fx( "bne 1b\n" : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks) : [args_ptr] "r" (&ka), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_bblocks] "I" (offsetof(KernelArgs, bblocks)) - : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL/generic.cpp index e5f59d220b8cf916f68be1a1ebf1af8bb0344afd..f6e1a75c15c6cfa3e5d8e81bd8fcb8dc582cd283 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_8x3VL/generic.cpp @@ -29,8 +29,12 @@ namespace arm_gemm { void sve_interleaved_s8s32_dot_8x3VL( - const int8_t *Apanel, const int8_t *Bpanel, - int32_t *Cpanel, int ablocks, int bblocks, int K) { + const int8_t *Apanel, + const int8_t *Bpanel, + int32_t *Cpanel, + int ablocks, + int bblocks, + int K) { struct KernelArgs { size_t K = {}; @@ -85,10 +89,10 @@ void sve_interleaved_s8s32_dot_8x3VL( "3:" // main loop head "sdot z8.s, z4.b, z0.b[0]\n" "sdot z11.s, z4.b, z0.b[1]\n" - "ld1rqb { z2.b }, p0/Z, [%x[Apanel], #32]\n" + "ld1rqb { z3.b }, p0/Z, [%x[Apanel], #32]\n" "sdot z14.s, z4.b, z0.b[2]\n" "sdot z17.s, z4.b, z0.b[3]\n" - "ld1rqb { z3.b }, p0/Z, [%x[Apanel], #48]\n" + "ld1rqb { z7.b }, p0/Z, [%x[Apanel], #48]\n" "sdot z20.s, z4.b, z1.b[0]\n" "sdot z23.s, z4.b, z1.b[1]\n" "sub x20, x20, #0x2\n" @@ -115,35 +119,35 @@ void sve_interleaved_s8s32_dot_8x3VL( "sdot z25.s, z6.b, z1.b[1]\n" "sdot z28.s, z6.b, z1.b[2]\n" "sdot z31.s, z6.b, z1.b[3]\n" - "ld1b { z6.b }, p0/Z, [x22, #5, MUL VL]\n" + "ld1b { z2.b }, p0/Z, [x22, #5, MUL VL]\n" "addvl x22, x22, #6\n" - "sdot z8.s, z4.b, z2.b[0]\n" - "sdot z11.s, z4.b, z2.b[1]\n" + "sdot z8.s, z4.b, z3.b[0]\n" + "sdot z11.s, z4.b, z3.b[1]\n" "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #16]\n" - "sdot z14.s, z4.b, z2.b[2]\n" - "sdot z17.s, z4.b, z2.b[3]\n" - "sdot z20.s, z4.b, z3.b[0]\n" - "sdot z23.s, z4.b, z3.b[1]\n" - "sdot z26.s, z4.b, z3.b[2]\n" - "sdot z29.s, z4.b, z3.b[3]\n" + "sdot z14.s, z4.b, z3.b[2]\n" + "sdot z17.s, z4.b, z3.b[3]\n" + "sdot z20.s, z4.b, z7.b[0]\n" + "sdot z23.s, z4.b, z7.b[1]\n" + "sdot z26.s, z4.b, z7.b[2]\n" + "sdot z29.s, z4.b, z7.b[3]\n" "ld1b { z4.b }, p0/Z, [x22]\n" - "sdot z9.s, z5.b, z2.b[0]\n" - "sdot z12.s, z5.b, z2.b[1]\n" - "sdot z15.s, z5.b, z2.b[2]\n" - "sdot z18.s, z5.b, z2.b[3]\n" - "sdot z21.s, z5.b, z3.b[0]\n" - "sdot z24.s, z5.b, z3.b[1]\n" - "sdot z27.s, z5.b, z3.b[2]\n" - "sdot z30.s, z5.b, z3.b[3]\n" + "sdot z9.s, z5.b, z3.b[0]\n" + "sdot z12.s, z5.b, z3.b[1]\n" + "sdot z15.s, z5.b, z3.b[2]\n" + "sdot z18.s, z5.b, z3.b[3]\n" + "sdot z21.s, z5.b, z7.b[0]\n" + "sdot z24.s, z5.b, z7.b[1]\n" + "sdot z27.s, z5.b, z7.b[2]\n" + "sdot z30.s, z5.b, z7.b[3]\n" "ld1b { z5.b }, p0/Z, [x22, #1, MUL VL]\n" - "sdot z10.s, z6.b, z2.b[0]\n" - "sdot z13.s, z6.b, z2.b[1]\n" - "sdot z16.s, z6.b, z2.b[2]\n" - "sdot z19.s, z6.b, z2.b[3]\n" - "sdot z22.s, z6.b, z3.b[0]\n" - "sdot z25.s, z6.b, z3.b[1]\n" - "sdot z28.s, z6.b, z3.b[2]\n" - "sdot z31.s, z6.b, z3.b[3]\n" + "sdot z10.s, z2.b, z3.b[0]\n" + "sdot z13.s, z2.b, z3.b[1]\n" + "sdot z16.s, z2.b, z3.b[2]\n" + "sdot z19.s, z2.b, z3.b[3]\n" + "sdot z22.s, z2.b, z7.b[0]\n" + "sdot z25.s, z2.b, z7.b[1]\n" + "sdot z28.s, z2.b, z7.b[2]\n" + "sdot z31.s, z2.b, z7.b[3]\n" "ld1b { z6.b }, p0/Z, [x22, #2, MUL VL]\n" "bge 3b\n" "4:" // main loop skip @@ -174,37 +178,37 @@ void sve_interleaved_s8s32_dot_8x3VL( "sdot z28.s, z6.b, z1.b[2]\n" "sdot z31.s, z6.b, z1.b[3]\n" "cbz x20, 5f\n" - "ld1rqb { z0.b }, p0/Z, [%x[Apanel]]\n" - "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #16]\n" + "ld1rqb { z4.b }, p0/Z, [%x[Apanel]]\n" + "ld1rqb { z3.b }, p0/Z, [%x[Apanel], #16]\n" "add %x[Apanel], %x[Apanel], #0x20\n" - "ld1b { z7.b }, p0/Z, [x22]\n" - "ld1b { z4.b }, p0/Z, [x22, #1, MUL VL]\n" - "sdot z8.s, z7.b, z0.b[0]\n" - "ld1b { z5.b }, p0/Z, [x22, #2, MUL VL]\n" - "sdot z11.s, z7.b, z0.b[1]\n" - "sdot z14.s, z7.b, z0.b[2]\n" - "sdot z17.s, z7.b, z0.b[3]\n" - "sdot z20.s, z7.b, z1.b[0]\n" + "ld1b { z2.b }, p0/Z, [x22]\n" + "ld1b { z1.b }, p0/Z, [x22, #1, MUL VL]\n" + "sdot z8.s, z2.b, z4.b[0]\n" + "ld1b { z0.b }, p0/Z, [x22, #2, MUL VL]\n" + "sdot z11.s, z2.b, z4.b[1]\n" + "sdot z14.s, z2.b, z4.b[2]\n" + "sdot z17.s, z2.b, z4.b[3]\n" + "sdot z20.s, z2.b, z3.b[0]\n" "addvl x22, x22, #3\n" - "sdot z23.s, z7.b, z1.b[1]\n" - "sdot z26.s, z7.b, z1.b[2]\n" - "sdot z29.s, z7.b, z1.b[3]\n" - "sdot z9.s, z4.b, z0.b[0]\n" - "sdot z12.s, z4.b, z0.b[1]\n" - "sdot z15.s, z4.b, z0.b[2]\n" - "sdot z18.s, z4.b, z0.b[3]\n" - "sdot z21.s, z4.b, z1.b[0]\n" - "sdot z24.s, z4.b, z1.b[1]\n" - "sdot z27.s, z4.b, z1.b[2]\n" - "sdot z30.s, z4.b, z1.b[3]\n" - "sdot z10.s, z5.b, z0.b[0]\n" - "sdot z13.s, z5.b, z0.b[1]\n" - "sdot z16.s, z5.b, z0.b[2]\n" - "sdot z19.s, z5.b, z0.b[3]\n" - "sdot z22.s, z5.b, z1.b[0]\n" - "sdot z25.s, z5.b, z1.b[1]\n" - "sdot z28.s, z5.b, z1.b[2]\n" - "sdot z31.s, z5.b, z1.b[3]\n" + "sdot z23.s, z2.b, z3.b[1]\n" + "sdot z26.s, z2.b, z3.b[2]\n" + "sdot z29.s, z2.b, z3.b[3]\n" + "sdot z9.s, z1.b, z4.b[0]\n" + "sdot z12.s, z1.b, z4.b[1]\n" + "sdot z15.s, z1.b, z4.b[2]\n" + "sdot z18.s, z1.b, z4.b[3]\n" + "sdot z21.s, z1.b, z3.b[0]\n" + "sdot z24.s, z1.b, z3.b[1]\n" + "sdot z27.s, z1.b, z3.b[2]\n" + "sdot z30.s, z1.b, z3.b[3]\n" + "sdot z10.s, z0.b, z4.b[0]\n" + "sdot z13.s, z0.b, z4.b[1]\n" + "sdot z16.s, z0.b, z4.b[2]\n" + "sdot z19.s, z0.b, z4.b[3]\n" + "sdot z22.s, z0.b, z3.b[0]\n" + "sdot z25.s, z0.b, z3.b[1]\n" + "sdot z28.s, z0.b, z3.b[2]\n" + "sdot z31.s, z0.b, z3.b[3]\n" "5:" // multiply loop done "st1w { z8.s }, p0, [%x[Cpanel]]\n" "subs x23, x23, #0x1\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_8x3VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_8x3VL.hpp index 4e65296f8b114fccd9280d7b65809773da9a3efe..82734abfbedf4d75954fbd8bc66183e00fc17b14 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_8x3VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_8x3VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef ARM_COMPUTE_ENABLE_SVE + #include "../std_transforms_sve.hpp" #include "../performance_parameters.hpp" @@ -55,11 +55,6 @@ public: return get_vector_length() * 3; } - static unsigned int stripe_width() - { - return get_vector_length(); - } - static constexpr unsigned int k_unroll() { return 8; @@ -89,7 +84,7 @@ public: default: return { 61.97, 3.64, 0.50 }; case CPUModel::V1: - return { 95.28, 7.99, 0.79 }; + return { 95.28, 7.99, 0.79 }; case CPUModel::A510: return { 43.36, 1.86, 0.28 }; } @@ -108,5 +103,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_8x3VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_8x3VL/generic.cpp index 104d5f918ea0501dad47b65b67ec976dd4cc7561..bfed5000fc6b7ba2887b8bdc2031cd264a607aca 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_8x3VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_8x3VL/generic.cpp @@ -29,8 +29,12 @@ namespace arm_gemm { void sve_interleaved_s8s32_mmla_8x3VL( - const int8_t *Apanel, const int8_t *Bpanel, - int32_t *Cpanel, int ablocks, int bblocks, int K) { + const int8_t *Apanel, + const int8_t *Bpanel, + int32_t *Cpanel, + int ablocks, + int bblocks, + int K) { struct KernelArgs { size_t K = {}; @@ -85,82 +89,82 @@ void sve_interleaved_s8s32_mmla_8x3VL( "mov z31.s, #0x0\n" "blt 4f\n" "3:" // main loop head - "ld1rqb { z3.b }, p0/Z, [%x[Apanel]]\n" + "ld1rqb { z6.b }, p0/Z, [%x[Apanel]]\n" ".inst 0x45049808 // smmla z8.s, z0.b, z4.b\n" ".inst 0x4505980b // smmla z11.s, z0.b, z5.b\n" ".inst 0x4504982e // smmla z14.s, z1.b, z4.b\n" ".inst 0x45059831 // smmla z17.s, z1.b, z5.b\n" - "ld1b { z6.b }, p0/Z, [x22]\n" + "ld1b { z7.b }, p0/Z, [x22]\n" ".inst 0x45049854 // smmla z20.s, z2.b, z4.b\n" ".inst 0x45059857 // smmla z23.s, z2.b, z5.b\n" - "ld1b { z7.b }, p0/Z, [x22, #1, MUL VL]\n" - ".inst 0x4504987a // smmla z26.s, z3.b, z4.b\n" - ".inst 0x4505987d // smmla z29.s, z3.b, z5.b\n" - "ld1b { z4.b }, p0/Z, [x22, #2, MUL VL]\n" - "ld1b { z5.b }, p0/Z, [x22, #3, MUL VL]\n" - ".inst 0x45069809 // smmla z9.s, z0.b, z6.b\n" - ".inst 0x4507980c // smmla z12.s, z0.b, z7.b\n" - ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" - ".inst 0x45079832 // smmla z18.s, z1.b, z7.b\n" + "ld1b { z3.b }, p0/Z, [x22, #1, MUL VL]\n" + ".inst 0x450498da // smmla z26.s, z6.b, z4.b\n" + ".inst 0x450598dd // smmla z29.s, z6.b, z5.b\n" + "ld1b { z5.b }, p0/Z, [x22, #2, MUL VL]\n" + "ld1b { z4.b }, p0/Z, [x22, #3, MUL VL]\n" + ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n" + ".inst 0x4503980c // smmla z12.s, z0.b, z3.b\n" + ".inst 0x4507982f // smmla z15.s, z1.b, z7.b\n" + ".inst 0x45039832 // smmla z18.s, z1.b, z3.b\n" "sub x20, x20, #0x2\n" - ".inst 0x45069855 // smmla z21.s, z2.b, z6.b\n" - ".inst 0x45079858 // smmla z24.s, z2.b, z7.b\n" + ".inst 0x45079855 // smmla z21.s, z2.b, z7.b\n" + ".inst 0x45039858 // smmla z24.s, z2.b, z3.b\n" "cmp x20, #0x2\n" - ".inst 0x4506987b // smmla z27.s, z3.b, z6.b\n" - ".inst 0x4507987e // smmla z30.s, z3.b, z7.b\n" - "ld1b { z6.b }, p0/Z, [x22, #4, MUL VL]\n" - ".inst 0x4504980a // smmla z10.s, z0.b, z4.b\n" - ".inst 0x4505980d // smmla z13.s, z0.b, z5.b\n" + ".inst 0x450798db // smmla z27.s, z6.b, z7.b\n" + ".inst 0x450398de // smmla z30.s, z6.b, z3.b\n" + "ld1b { z3.b }, p0/Z, [x22, #4, MUL VL]\n" + ".inst 0x4505980a // smmla z10.s, z0.b, z5.b\n" + ".inst 0x4504980d // smmla z13.s, z0.b, z4.b\n" "ld1rqb { z0.b }, p0/Z, [%x[Apanel], #16]\n" - ".inst 0x45049830 // smmla z16.s, z1.b, z4.b\n" - ".inst 0x45059833 // smmla z19.s, z1.b, z5.b\n" + ".inst 0x45059830 // smmla z16.s, z1.b, z5.b\n" + ".inst 0x45049833 // smmla z19.s, z1.b, z4.b\n" "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #32]\n" - ".inst 0x45049856 // smmla z22.s, z2.b, z4.b\n" - ".inst 0x45059859 // smmla z25.s, z2.b, z5.b\n" + ".inst 0x45059856 // smmla z22.s, z2.b, z5.b\n" + ".inst 0x45049859 // smmla z25.s, z2.b, z4.b\n" "ld1b { z7.b }, p0/Z, [x22, #5, MUL VL]\n" - ".inst 0x4504987c // smmla z28.s, z3.b, z4.b\n" - ".inst 0x4505987f // smmla z31.s, z3.b, z5.b\n" - "ld1rqb { z2.b }, p0/Z, [%x[Apanel], #48]\n" - "ld1rqb { z3.b }, p0/Z, [%x[Apanel], #64]\n" - "ld1b { z4.b }, p0/Z, [x22, #6, MUL VL]\n" - ".inst 0x45069808 // smmla z8.s, z0.b, z6.b\n" - "ld1b { z5.b }, p0/Z, [x22, #7, MUL VL]\n" + ".inst 0x450598dc // smmla z28.s, z6.b, z5.b\n" + ".inst 0x450498df // smmla z31.s, z6.b, z4.b\n" + "ld1rqb { z5.b }, p0/Z, [%x[Apanel], #48]\n" + "ld1rqb { z6.b }, p0/Z, [%x[Apanel], #64]\n" + "ld1b { z2.b }, p0/Z, [x22, #6, MUL VL]\n" + ".inst 0x45039808 // smmla z8.s, z0.b, z3.b\n" + "ld1b { z4.b }, p0/Z, [x22, #7, MUL VL]\n" "addvl x22, x22, #16\n" ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" + ".inst 0x4503982e // smmla z14.s, z1.b, z3.b\n" ".inst 0x45079831 // smmla z17.s, z1.b, z7.b\n" - ".inst 0x45069854 // smmla z20.s, z2.b, z6.b\n" - ".inst 0x45079857 // smmla z23.s, z2.b, z7.b\n" - ".inst 0x4506987a // smmla z26.s, z3.b, z6.b\n" - ".inst 0x4507987d // smmla z29.s, z3.b, z7.b\n" - "ld1b { z6.b }, p0/Z, [x22, #-8, MUL VL]\n" + ".inst 0x450398b4 // smmla z20.s, z5.b, z3.b\n" + ".inst 0x450798b7 // smmla z23.s, z5.b, z7.b\n" + ".inst 0x450398da // smmla z26.s, z6.b, z3.b\n" + ".inst 0x450798dd // smmla z29.s, z6.b, z7.b\n" + "ld1b { z3.b }, p0/Z, [x22, #-8, MUL VL]\n" "ld1b { z7.b }, p0/Z, [x22, #-7, MUL VL]\n" - ".inst 0x45049809 // smmla z9.s, z0.b, z4.b\n" - ".inst 0x4505980c // smmla z12.s, z0.b, z5.b\n" - ".inst 0x4504982f // smmla z15.s, z1.b, z4.b\n" - ".inst 0x45059832 // smmla z18.s, z1.b, z5.b\n" - ".inst 0x45049855 // smmla z21.s, z2.b, z4.b\n" - ".inst 0x45059858 // smmla z24.s, z2.b, z5.b\n" - ".inst 0x4504987b // smmla z27.s, z3.b, z4.b\n" - ".inst 0x4505987e // smmla z30.s, z3.b, z5.b\n" + ".inst 0x45029809 // smmla z9.s, z0.b, z2.b\n" + ".inst 0x4504980c // smmla z12.s, z0.b, z4.b\n" + ".inst 0x4502982f // smmla z15.s, z1.b, z2.b\n" + ".inst 0x45049832 // smmla z18.s, z1.b, z4.b\n" + ".inst 0x450298b5 // smmla z21.s, z5.b, z2.b\n" + ".inst 0x450498b8 // smmla z24.s, z5.b, z4.b\n" + ".inst 0x450298db // smmla z27.s, z6.b, z2.b\n" + ".inst 0x450498de // smmla z30.s, z6.b, z4.b\n" "ld1b { z4.b }, p0/Z, [x22, #-6, MUL VL]\n" - ".inst 0x4506980a // smmla z10.s, z0.b, z6.b\n" + ".inst 0x4503980a // smmla z10.s, z0.b, z3.b\n" ".inst 0x4507980d // smmla z13.s, z0.b, z7.b\n" "ld1rqb { z0.b }, p0/Z, [%x[Apanel], #80]\n" - ".inst 0x45069830 // smmla z16.s, z1.b, z6.b\n" + ".inst 0x45039830 // smmla z16.s, z1.b, z3.b\n" ".inst 0x45079833 // smmla z19.s, z1.b, z7.b\n" "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #96]\n" - ".inst 0x45069856 // smmla z22.s, z2.b, z6.b\n" - ".inst 0x45079859 // smmla z25.s, z2.b, z7.b\n" + ".inst 0x450398b6 // smmla z22.s, z5.b, z3.b\n" + ".inst 0x450798b9 // smmla z25.s, z5.b, z7.b\n" "ld1b { z5.b }, p0/Z, [x22, #-5, MUL VL]\n" - ".inst 0x4506987c // smmla z28.s, z3.b, z6.b\n" - ".inst 0x4507987f // smmla z31.s, z3.b, z7.b\n" + ".inst 0x450398dc // smmla z28.s, z6.b, z3.b\n" + ".inst 0x450798df // smmla z31.s, z6.b, z7.b\n" "ld1rqb { z2.b }, p0/Z, [%x[Apanel], #112]\n" "add %x[Apanel], %x[Apanel], #0x80\n" "addvl x22, x22, #-4\n" "bge 3b\n" "4:" // main loop skip - "ld1rqb { z3.b }, p0/Z, [%x[Apanel]]\n" + "ld1rqb { z7.b }, p0/Z, [%x[Apanel]]\n" ".inst 0x45049808 // smmla z8.s, z0.b, z4.b\n" ".inst 0x4505980b // smmla z11.s, z0.b, z5.b\n" ".inst 0x4504982e // smmla z14.s, z1.b, z4.b\n" @@ -168,114 +172,114 @@ void sve_interleaved_s8s32_mmla_8x3VL( "ld1b { z6.b }, p0/Z, [x22]\n" ".inst 0x45049854 // smmla z20.s, z2.b, z4.b\n" ".inst 0x45059857 // smmla z23.s, z2.b, z5.b\n" - "ld1b { z7.b }, p0/Z, [x22, #1, MUL VL]\n" - ".inst 0x4504987a // smmla z26.s, z3.b, z4.b\n" - ".inst 0x4505987d // smmla z29.s, z3.b, z5.b\n" - "ld1b { z4.b }, p0/Z, [x22, #2, MUL VL]\n" - "ld1b { z5.b }, p0/Z, [x22, #3, MUL VL]\n" + "ld1b { z3.b }, p0/Z, [x22, #1, MUL VL]\n" + ".inst 0x450498fa // smmla z26.s, z7.b, z4.b\n" + ".inst 0x450598fd // smmla z29.s, z7.b, z5.b\n" + "ld1b { z5.b }, p0/Z, [x22, #2, MUL VL]\n" + "ld1b { z4.b }, p0/Z, [x22, #3, MUL VL]\n" ".inst 0x45069809 // smmla z9.s, z0.b, z6.b\n" - ".inst 0x4507980c // smmla z12.s, z0.b, z7.b\n" + ".inst 0x4503980c // smmla z12.s, z0.b, z3.b\n" ".inst 0x4506982f // smmla z15.s, z1.b, z6.b\n" - ".inst 0x45079832 // smmla z18.s, z1.b, z7.b\n" + ".inst 0x45039832 // smmla z18.s, z1.b, z3.b\n" "add %x[Apanel], %x[Apanel], #0x10\n" ".inst 0x45069855 // smmla z21.s, z2.b, z6.b\n" - ".inst 0x45079858 // smmla z24.s, z2.b, z7.b\n" + ".inst 0x45039858 // smmla z24.s, z2.b, z3.b\n" "addvl x22, x22, #4\n" - ".inst 0x4506987b // smmla z27.s, z3.b, z6.b\n" - ".inst 0x4507987e // smmla z30.s, z3.b, z7.b\n" - ".inst 0x4504980a // smmla z10.s, z0.b, z4.b\n" - ".inst 0x4505980d // smmla z13.s, z0.b, z5.b\n" - ".inst 0x45049830 // smmla z16.s, z1.b, z4.b\n" - ".inst 0x45059833 // smmla z19.s, z1.b, z5.b\n" - ".inst 0x45049856 // smmla z22.s, z2.b, z4.b\n" - ".inst 0x45059859 // smmla z25.s, z2.b, z5.b\n" - ".inst 0x4504987c // smmla z28.s, z3.b, z4.b\n" - ".inst 0x4505987f // smmla z31.s, z3.b, z5.b\n" + ".inst 0x450698fb // smmla z27.s, z7.b, z6.b\n" + ".inst 0x450398fe // smmla z30.s, z7.b, z3.b\n" + ".inst 0x4505980a // smmla z10.s, z0.b, z5.b\n" + ".inst 0x4504980d // smmla z13.s, z0.b, z4.b\n" + ".inst 0x45059830 // smmla z16.s, z1.b, z5.b\n" + ".inst 0x45049833 // smmla z19.s, z1.b, z4.b\n" + ".inst 0x45059856 // smmla z22.s, z2.b, z5.b\n" + ".inst 0x45049859 // smmla z25.s, z2.b, z4.b\n" + ".inst 0x450598fc // smmla z28.s, z7.b, z5.b\n" + ".inst 0x450498ff // smmla z31.s, z7.b, z4.b\n" "cbz x20, 5f\n" - "ld1b { z6.b }, p0/Z, [x22]\n" - "ld1rqb { z0.b }, p0/Z, [%x[Apanel]]\n" - ".inst 0x45069808 // smmla z8.s, z0.b, z6.b\n" - "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #16]\n" - "ld1b { z7.b }, p0/Z, [x22, #1, MUL VL]\n" - ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n" - "ld1rqb { z2.b }, p0/Z, [%x[Apanel], #32]\n" - "ld1rqb { z3.b }, p0/Z, [%x[Apanel], #48]\n" - ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n" - ".inst 0x45079831 // smmla z17.s, z1.b, z7.b\n" - ".inst 0x45069854 // smmla z20.s, z2.b, z6.b\n" - "ld1b { z4.b }, p0/Z, [x22, #2, MUL VL]\n" - ".inst 0x45079857 // smmla z23.s, z2.b, z7.b\n" - ".inst 0x4506987a // smmla z26.s, z3.b, z6.b\n" - "ld1b { z5.b }, p0/Z, [x22, #3, MUL VL]\n" - ".inst 0x4507987d // smmla z29.s, z3.b, z7.b\n" - "ld1b { z6.b }, p0/Z, [x22, #4, MUL VL]\n" - "ld1b { z7.b }, p0/Z, [x22, #5, MUL VL]\n" - ".inst 0x45049809 // smmla z9.s, z0.b, z4.b\n" - ".inst 0x4505980c // smmla z12.s, z0.b, z5.b\n" + "ld1b { z1.b }, p0/Z, [x22]\n" + "ld1rqb { z7.b }, p0/Z, [%x[Apanel]]\n" + ".inst 0x450198e8 // smmla z8.s, z7.b, z1.b\n" + "ld1rqb { z6.b }, p0/Z, [%x[Apanel], #16]\n" + "ld1b { z0.b }, p0/Z, [x22, #1, MUL VL]\n" + ".inst 0x450098eb // smmla z11.s, z7.b, z0.b\n" + "ld1rqb { z5.b }, p0/Z, [%x[Apanel], #32]\n" + "ld1rqb { z4.b }, p0/Z, [%x[Apanel], #48]\n" + ".inst 0x450198ce // smmla z14.s, z6.b, z1.b\n" + ".inst 0x450098d1 // smmla z17.s, z6.b, z0.b\n" + ".inst 0x450198b4 // smmla z20.s, z5.b, z1.b\n" + "ld1b { z3.b }, p0/Z, [x22, #2, MUL VL]\n" + ".inst 0x450098b7 // smmla z23.s, z5.b, z0.b\n" + ".inst 0x4501989a // smmla z26.s, z4.b, z1.b\n" + "ld1b { z2.b }, p0/Z, [x22, #3, MUL VL]\n" + ".inst 0x4500989d // smmla z29.s, z4.b, z0.b\n" + "ld1b { z1.b }, p0/Z, [x22, #4, MUL VL]\n" + "ld1b { z0.b }, p0/Z, [x22, #5, MUL VL]\n" + ".inst 0x450398e9 // smmla z9.s, z7.b, z3.b\n" + ".inst 0x450298ec // smmla z12.s, z7.b, z2.b\n" "addvl x22, x22, #6\n" - ".inst 0x4504982f // smmla z15.s, z1.b, z4.b\n" - ".inst 0x45059832 // smmla z18.s, z1.b, z5.b\n" + ".inst 0x450398cf // smmla z15.s, z6.b, z3.b\n" + ".inst 0x450298d2 // smmla z18.s, z6.b, z2.b\n" "add %x[Apanel], %x[Apanel], #0x40\n" - ".inst 0x45049855 // smmla z21.s, z2.b, z4.b\n" - ".inst 0x45059858 // smmla z24.s, z2.b, z5.b\n" - ".inst 0x4504987b // smmla z27.s, z3.b, z4.b\n" - ".inst 0x4505987e // smmla z30.s, z3.b, z5.b\n" - ".inst 0x4506980a // smmla z10.s, z0.b, z6.b\n" - ".inst 0x4507980d // smmla z13.s, z0.b, z7.b\n" - ".inst 0x45069830 // smmla z16.s, z1.b, z6.b\n" - ".inst 0x45079833 // smmla z19.s, z1.b, z7.b\n" - ".inst 0x45069856 // smmla z22.s, z2.b, z6.b\n" - ".inst 0x45079859 // smmla z25.s, z2.b, z7.b\n" - ".inst 0x4506987c // smmla z28.s, z3.b, z6.b\n" - ".inst 0x4507987f // smmla z31.s, z3.b, z7.b\n" + ".inst 0x450398b5 // smmla z21.s, z5.b, z3.b\n" + ".inst 0x450298b8 // smmla z24.s, z5.b, z2.b\n" + ".inst 0x4503989b // smmla z27.s, z4.b, z3.b\n" + ".inst 0x4502989e // smmla z30.s, z4.b, z2.b\n" + ".inst 0x450198ea // smmla z10.s, z7.b, z1.b\n" + ".inst 0x450098ed // smmla z13.s, z7.b, z0.b\n" + ".inst 0x450198d0 // smmla z16.s, z6.b, z1.b\n" + ".inst 0x450098d3 // smmla z19.s, z6.b, z0.b\n" + ".inst 0x450198b6 // smmla z22.s, z5.b, z1.b\n" + ".inst 0x450098b9 // smmla z25.s, z5.b, z0.b\n" + ".inst 0x4501989c // smmla z28.s, z4.b, z1.b\n" + ".inst 0x4500989f // smmla z31.s, z4.b, z0.b\n" "5:" // multiply loop done - "uzp1 z4.d, z8.d, z11.d\n" + "uzp1 z0.d, z8.d, z11.d\n" "uzp2 z8.d, z8.d, z11.d\n" - "st1w { z4.s }, p0, [%x[Cpanel]]\n" - "uzp1 z11.d, z9.d, z12.d\n" + "st1w { z0.s }, p0, [%x[Cpanel]]\n" + "uzp1 z0.d, z9.d, z12.d\n" "uzp2 z9.d, z9.d, z12.d\n" - "st1w { z11.s }, p0, [%x[Cpanel], #1, MUL VL]\n" - "uzp1 z12.d, z10.d, z13.d\n" + "st1w { z0.s }, p0, [%x[Cpanel], #1, MUL VL]\n" + "uzp1 z0.d, z10.d, z13.d\n" "uzp2 z10.d, z10.d, z13.d\n" - "st1w { z12.s }, p0, [%x[Cpanel], #2, MUL VL]\n" + "st1w { z0.s }, p0, [%x[Cpanel], #2, MUL VL]\n" "st1w { z8.s }, p0, [%x[Cpanel], #3, MUL VL]\n" - "uzp1 z13.d, z14.d, z17.d\n" + "uzp1 z0.d, z14.d, z17.d\n" "uzp2 z14.d, z14.d, z17.d\n" "st1w { z9.s }, p0, [%x[Cpanel], #4, MUL VL]\n" - "uzp1 z17.d, z15.d, z18.d\n" + "uzp1 z1.d, z15.d, z18.d\n" "subs x23, x23, #0x1\n" "st1w { z10.s }, p0, [%x[Cpanel], #5, MUL VL]\n" "uzp2 z15.d, z15.d, z18.d\n" - "uzp1 z18.d, z16.d, z19.d\n" - "st1w { z13.s }, p0, [%x[Cpanel], #6, MUL VL]\n" + "uzp1 z17.d, z16.d, z19.d\n" + "st1w { z0.s }, p0, [%x[Cpanel], #6, MUL VL]\n" "uzp2 z16.d, z16.d, z19.d\n" - "uzp1 z19.d, z20.d, z23.d\n" - "st1w { z17.s }, p0, [%x[Cpanel], #7, MUL VL]\n" + "uzp1 z0.d, z20.d, z23.d\n" + "st1w { z1.s }, p0, [%x[Cpanel], #7, MUL VL]\n" "addvl %x[Cpanel], %x[Cpanel], #16\n" "uzp2 z20.d, z20.d, z23.d\n" - "st1w { z18.s }, p0, [%x[Cpanel], #-8, MUL VL]\n" + "st1w { z17.s }, p0, [%x[Cpanel], #-8, MUL VL]\n" "uzp1 z23.d, z21.d, z24.d\n" "uzp2 z21.d, z21.d, z24.d\n" "st1w { z14.s }, p0, [%x[Cpanel], #-7, MUL VL]\n" - "uzp1 z24.d, z22.d, z25.d\n" + "uzp1 z19.d, z22.d, z25.d\n" "uzp2 z22.d, z22.d, z25.d\n" "st1w { z15.s }, p0, [%x[Cpanel], #-6, MUL VL]\n" - "uzp1 z25.d, z26.d, z29.d\n" + "uzp1 z18.d, z26.d, z29.d\n" "uzp2 z26.d, z26.d, z29.d\n" "st1w { z16.s }, p0, [%x[Cpanel], #-5, MUL VL]\n" - "uzp1 z29.d, z27.d, z30.d\n" + "uzp1 z17.d, z27.d, z30.d\n" "uzp2 z27.d, z27.d, z30.d\n" - "st1w { z19.s }, p0, [%x[Cpanel], #-4, MUL VL]\n" - "uzp1 z30.d, z28.d, z31.d\n" + "st1w { z0.s }, p0, [%x[Cpanel], #-4, MUL VL]\n" + "uzp1 z16.d, z28.d, z31.d\n" "uzp2 z28.d, z28.d, z31.d\n" "st1w { z23.s }, p0, [%x[Cpanel], #-3, MUL VL]\n" - "st1w { z24.s }, p0, [%x[Cpanel], #-2, MUL VL]\n" + "st1w { z19.s }, p0, [%x[Cpanel], #-2, MUL VL]\n" "st1w { z20.s }, p0, [%x[Cpanel], #-1, MUL VL]\n" "st1w { z21.s }, p0, [%x[Cpanel]]\n" "st1w { z22.s }, p0, [%x[Cpanel], #1, MUL VL]\n" - "st1w { z25.s }, p0, [%x[Cpanel], #2, MUL VL]\n" - "st1w { z29.s }, p0, [%x[Cpanel], #3, MUL VL]\n" - "st1w { z30.s }, p0, [%x[Cpanel], #4, MUL VL]\n" + "st1w { z18.s }, p0, [%x[Cpanel], #2, MUL VL]\n" + "st1w { z17.s }, p0, [%x[Cpanel], #3, MUL VL]\n" + "st1w { z16.s }, p0, [%x[Cpanel], #4, MUL VL]\n" "st1w { z26.s }, p0, [%x[Cpanel], #5, MUL VL]\n" "st1w { z27.s }, p0, [%x[Cpanel], #6, MUL VL]\n" "st1w { z28.s }, p0, [%x[Cpanel], #7, MUL VL]\n" @@ -290,4 +294,4 @@ void sve_interleaved_s8s32_mmla_8x3VL( } } // namespace arm_gemm -#endif // __ARM_FEATURE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL.hpp index 0afcdd2ce49dfcdaeaf5b5ec523bc7e70d43ea51..c0b215ccb420f336dd98a8dfef8e503948d46a58 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef ARM_COMPUTE_ENABLE_SVE + #include "../std_transforms_sve.hpp" #include "../performance_parameters.hpp" @@ -35,6 +35,7 @@ namespace arm_gemm { // Actual kernel implementations void sve_interleaved_u8u32_dot_8x3VL( ARGLIST ); +void sve_interleaved_u8u32_dot_8x3VL_a64fx( ARGLIST ); class cls_sve_interleaved_u8u32_dot_8x3VL { @@ -55,11 +56,6 @@ public: return get_vector_length() * 3; } - static unsigned int stripe_width() - { - return get_vector_length(); - } - static constexpr unsigned int k_unroll() { return 4; @@ -80,6 +76,8 @@ public: return { 27.44, 3.41, 2.90 }; case CPUModel::V1: return { 63.30, 4.97, 11.52 }; + case CPUModel::A64FX: + return { 109.76, 3.88, 6.76 }; } } @@ -92,6 +90,8 @@ public: return { 27.45, 1.65, 0.28 }; case CPUModel::V1: return { 52.24, 7.49, 0.80 }; + case CPUModel::A64FX: + return { 110.18, 2.34, 0.40 }; } } @@ -100,13 +100,19 @@ public: // Default to the generic kernel kern_type kernel=sve_interleaved_u8u32_dot_8x3VL; - cls_sve_interleaved_u8u32_dot_8x3VL(const CPUInfo *) + cls_sve_interleaved_u8u32_dot_8x3VL(const CPUInfo *ci) { + switch(ci->get_cpu_model()) { + default: + break; + case CPUModel::A64FX: + kernel=sve_interleaved_u8u32_dot_8x3VL_a64fx; + break; + } } }; } // namespace arm_gemm #undef ARGLIST - #endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL/a64fx.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL/a64fx.cpp index 2bfec8f35021255c0045113eb9ea5246e28899bc..79e794a834bc0f5463d1ce8f3b4d04f3a7289cc8 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL/a64fx.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL/a64fx.cpp @@ -29,8 +29,12 @@ namespace arm_gemm { void sve_interleaved_u8u32_dot_8x3VL_a64fx( - const uint8_t *Apanel, const uint8_t *Bpanel, - uint32_t *Cpanel, int ablocks, int bblocks, int K) { + const uint8_t *Apanel, + const uint8_t *Bpanel, + uint32_t *Cpanel, + int ablocks, + int bblocks, + int K) { struct KernelArgs { size_t K = {}; @@ -89,7 +93,7 @@ void sve_interleaved_u8u32_dot_8x3VL_a64fx( "udot z9.s, z1.b, z3.b\n" "sub x20, x20, #0x2\n" "udot z10.s, z2.b, z3.b\n" - "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n" + "ld1rw { z7.s }, p0/Z, [%x[Apanel], #16]\n" "udot z11.s, z0.b, z4.b\n" "udot z12.s, z1.b, z4.b\n" "udot z13.s, z2.b, z4.b\n" @@ -98,63 +102,63 @@ void sve_interleaved_u8u32_dot_8x3VL_a64fx( "udot z15.s, z1.b, z5.b\n" "cmp x20, #0x2\n" "udot z16.s, z2.b, z5.b\n" - "ld1rw { z5.s }, p0/Z, [%x[Apanel], #24]\n" + "ld1rw { z3.s }, p0/Z, [%x[Apanel], #24]\n" "udot z17.s, z0.b, z6.b\n" "udot z18.s, z1.b, z6.b\n" "udot z19.s, z2.b, z6.b\n" - "ld1rw { z6.s }, p0/Z, [%x[Apanel], #28]\n" - "udot z20.s, z0.b, z3.b\n" - "udot z21.s, z1.b, z3.b\n" - "udot z22.s, z2.b, z3.b\n" - "ld1rw { z3.s }, p0/Z, [%x[Apanel], #32]\n" + "ld1rw { z5.s }, p0/Z, [%x[Apanel], #28]\n" + "udot z20.s, z0.b, z7.b\n" + "udot z21.s, z1.b, z7.b\n" + "udot z22.s, z2.b, z7.b\n" + "ld1rw { z7.s }, p0/Z, [%x[Apanel], #32]\n" "udot z23.s, z0.b, z4.b\n" "udot z24.s, z1.b, z4.b\n" "udot z25.s, z2.b, z4.b\n" "ld1rw { z4.s }, p0/Z, [%x[Apanel], #36]\n" - "udot z26.s, z0.b, z5.b\n" - "udot z27.s, z1.b, z5.b\n" - "udot z28.s, z2.b, z5.b\n" - "ld1rw { z5.s }, p0/Z, [%x[Apanel], #40]\n" - "udot z29.s, z0.b, z6.b\n" - "ld1b { z0.b }, p0/Z, [x22, #3, MUL VL]\n" - "udot z30.s, z1.b, z6.b\n" - "udot z31.s, z2.b, z6.b\n" - "ld1b { z1.b }, p0/Z, [x22, #4, MUL VL]\n" - "ld1b { z2.b }, p0/Z, [x22, #5, MUL VL]\n" - "udot z8.s, z0.b, z3.b\n" - "ld1rw { z6.s }, p0/Z, [%x[Apanel], #44]\n" - "udot z9.s, z1.b, z3.b\n" - "udot z10.s, z2.b, z3.b\n" - "udot z11.s, z0.b, z4.b\n" - "ld1rw { z3.s }, p0/Z, [%x[Apanel], #48]\n" - "udot z12.s, z1.b, z4.b\n" - "udot z13.s, z2.b, z4.b\n" + "udot z26.s, z0.b, z3.b\n" + "udot z27.s, z1.b, z3.b\n" + "udot z28.s, z2.b, z3.b\n" + "ld1rw { z3.s }, p0/Z, [%x[Apanel], #40]\n" + "udot z29.s, z0.b, z5.b\n" + "ld1b { z6.b }, p0/Z, [x22, #3, MUL VL]\n" + "udot z30.s, z1.b, z5.b\n" + "udot z31.s, z2.b, z5.b\n" + "ld1b { z2.b }, p0/Z, [x22, #4, MUL VL]\n" + "ld1b { z5.b }, p0/Z, [x22, #5, MUL VL]\n" + "udot z8.s, z6.b, z7.b\n" + "ld1rw { z1.s }, p0/Z, [%x[Apanel], #44]\n" + "udot z9.s, z2.b, z7.b\n" + "udot z10.s, z5.b, z7.b\n" + "udot z11.s, z6.b, z4.b\n" + "ld1rw { z7.s }, p0/Z, [%x[Apanel], #48]\n" + "udot z12.s, z2.b, z4.b\n" + "udot z13.s, z5.b, z4.b\n" "ld1rw { z4.s }, p0/Z, [%x[Apanel], #52]\n" - "udot z14.s, z0.b, z5.b\n" - "udot z15.s, z1.b, z5.b\n" + "udot z14.s, z6.b, z3.b\n" + "udot z15.s, z2.b, z3.b\n" "addvl x22, x22, #6\n" - "udot z16.s, z2.b, z5.b\n" - "ld1rw { z5.s }, p0/Z, [%x[Apanel], #56]\n" - "udot z17.s, z0.b, z6.b\n" - "udot z18.s, z1.b, z6.b\n" - "udot z19.s, z2.b, z6.b\n" - "ld1rw { z6.s }, p0/Z, [%x[Apanel], #60]\n" + "udot z16.s, z5.b, z3.b\n" + "ld1rw { z0.s }, p0/Z, [%x[Apanel], #56]\n" + "udot z17.s, z6.b, z1.b\n" + "udot z18.s, z2.b, z1.b\n" + "udot z19.s, z5.b, z1.b\n" + "ld1rw { z1.s }, p0/Z, [%x[Apanel], #60]\n" "add %x[Apanel], %x[Apanel], #0x40\n" - "udot z20.s, z0.b, z3.b\n" - "udot z21.s, z1.b, z3.b\n" - "udot z22.s, z2.b, z3.b\n" - "udot z23.s, z0.b, z4.b\n" + "udot z20.s, z6.b, z7.b\n" + "udot z21.s, z2.b, z7.b\n" + "udot z22.s, z5.b, z7.b\n" + "udot z23.s, z6.b, z4.b\n" "ld1rw { z3.s }, p0/Z, [%x[Apanel]]\n" - "udot z24.s, z1.b, z4.b\n" - "udot z25.s, z2.b, z4.b\n" + "udot z24.s, z2.b, z4.b\n" + "udot z25.s, z5.b, z4.b\n" "ld1rw { z4.s }, p0/Z, [%x[Apanel], #4]\n" - "udot z26.s, z0.b, z5.b\n" - "udot z27.s, z1.b, z5.b\n" - "udot z28.s, z2.b, z5.b\n" - "udot z29.s, z0.b, z6.b\n" + "udot z26.s, z6.b, z0.b\n" + "udot z27.s, z2.b, z0.b\n" + "udot z28.s, z5.b, z0.b\n" + "udot z29.s, z6.b, z1.b\n" "ld1b { z0.b }, p0/Z, [x22]\n" - "udot z30.s, z1.b, z6.b\n" - "udot z31.s, z2.b, z6.b\n" + "udot z30.s, z2.b, z1.b\n" + "udot z31.s, z5.b, z1.b\n" "ld1b { z1.b }, p0/Z, [x22, #1, MUL VL]\n" "ld1b { z2.b }, p0/Z, [x22, #2, MUL VL]\n" "ld1rw { z5.s }, p0/Z, [%x[Apanel], #8]\n" @@ -165,7 +169,7 @@ void sve_interleaved_u8u32_dot_8x3VL_a64fx( "udot z9.s, z1.b, z3.b\n" "addvl x22, x22, #3\n" "udot z10.s, z2.b, z3.b\n" - "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n" + "ld1rw { z7.s }, p0/Z, [%x[Apanel], #16]\n" "udot z11.s, z0.b, z4.b\n" "udot z12.s, z1.b, z4.b\n" "udot z13.s, z2.b, z4.b\n" @@ -177,58 +181,58 @@ void sve_interleaved_u8u32_dot_8x3VL_a64fx( "udot z17.s, z0.b, z6.b\n" "udot z18.s, z1.b, z6.b\n" "udot z19.s, z2.b, z6.b\n" - "ld1rw { z6.s }, p0/Z, [%x[Apanel], #28]\n" - "udot z20.s, z0.b, z3.b\n" - "udot z21.s, z1.b, z3.b\n" + "ld1rw { z3.s }, p0/Z, [%x[Apanel], #28]\n" + "udot z20.s, z0.b, z7.b\n" + "udot z21.s, z1.b, z7.b\n" "add %x[Apanel], %x[Apanel], #0x20\n" - "udot z22.s, z2.b, z3.b\n" + "udot z22.s, z2.b, z7.b\n" "udot z23.s, z0.b, z4.b\n" "udot z24.s, z1.b, z4.b\n" "udot z25.s, z2.b, z4.b\n" "udot z26.s, z0.b, z5.b\n" "udot z27.s, z1.b, z5.b\n" "udot z28.s, z2.b, z5.b\n" - "udot z29.s, z0.b, z6.b\n" - "udot z30.s, z1.b, z6.b\n" - "udot z31.s, z2.b, z6.b\n" + "udot z29.s, z0.b, z3.b\n" + "udot z30.s, z1.b, z3.b\n" + "udot z31.s, z2.b, z3.b\n" "cbz x20, 5f\n" - "ld1b { z0.b }, p0/Z, [x22]\n" - "ld1b { z1.b }, p0/Z, [x22, #1, MUL VL]\n" - "ld1b { z2.b }, p0/Z, [x22, #2, MUL VL]\n" + "ld1b { z6.b }, p0/Z, [x22]\n" + "ld1b { z5.b }, p0/Z, [x22, #1, MUL VL]\n" + "ld1b { z4.b }, p0/Z, [x22, #2, MUL VL]\n" "ld1rw { z3.s }, p0/Z, [%x[Apanel]]\n" - "udot z8.s, z0.b, z3.b\n" - "ld1rw { z4.s }, p0/Z, [%x[Apanel], #4]\n" - "ld1rw { z5.s }, p0/Z, [%x[Apanel], #8]\n" - "udot z9.s, z1.b, z3.b\n" - "ld1rw { z6.s }, p0/Z, [%x[Apanel], #12]\n" - "udot z10.s, z2.b, z3.b\n" - "udot z11.s, z0.b, z4.b\n" - "udot z12.s, z1.b, z4.b\n" - "udot z13.s, z2.b, z4.b\n" + "udot z8.s, z6.b, z3.b\n" + "ld1rw { z2.s }, p0/Z, [%x[Apanel], #4]\n" + "ld1rw { z1.s }, p0/Z, [%x[Apanel], #8]\n" + "udot z9.s, z5.b, z3.b\n" + "ld1rw { z0.s }, p0/Z, [%x[Apanel], #12]\n" + "udot z10.s, z4.b, z3.b\n" + "udot z11.s, z6.b, z2.b\n" + "udot z12.s, z5.b, z2.b\n" + "udot z13.s, z4.b, z2.b\n" "ld1rw { z3.s }, p0/Z, [%x[Apanel], #16]\n" - "udot z14.s, z0.b, z5.b\n" - "udot z15.s, z1.b, z5.b\n" - "ld1rw { z4.s }, p0/Z, [%x[Apanel], #20]\n" - "udot z16.s, z2.b, z5.b\n" - "udot z17.s, z0.b, z6.b\n" - "ld1rw { z5.s }, p0/Z, [%x[Apanel], #24]\n" - "udot z18.s, z1.b, z6.b\n" - "udot z19.s, z2.b, z6.b\n" - "ld1rw { z6.s }, p0/Z, [%x[Apanel], #28]\n" - "udot z20.s, z0.b, z3.b\n" - "udot z21.s, z1.b, z3.b\n" + "udot z14.s, z6.b, z1.b\n" + "udot z15.s, z5.b, z1.b\n" + "ld1rw { z2.s }, p0/Z, [%x[Apanel], #20]\n" + "udot z16.s, z4.b, z1.b\n" + "udot z17.s, z6.b, z0.b\n" + "ld1rw { z1.s }, p0/Z, [%x[Apanel], #24]\n" + "udot z18.s, z5.b, z0.b\n" + "udot z19.s, z4.b, z0.b\n" + "ld1rw { z0.s }, p0/Z, [%x[Apanel], #28]\n" + "udot z20.s, z6.b, z3.b\n" + "udot z21.s, z5.b, z3.b\n" "addvl x22, x22, #3\n" - "udot z22.s, z2.b, z3.b\n" - "udot z23.s, z0.b, z4.b\n" + "udot z22.s, z4.b, z3.b\n" + "udot z23.s, z6.b, z2.b\n" "add %x[Apanel], %x[Apanel], #0x20\n" - "udot z24.s, z1.b, z4.b\n" - "udot z25.s, z2.b, z4.b\n" - "udot z26.s, z0.b, z5.b\n" - "udot z27.s, z1.b, z5.b\n" - "udot z28.s, z2.b, z5.b\n" - "udot z29.s, z0.b, z6.b\n" - "udot z30.s, z1.b, z6.b\n" - "udot z31.s, z2.b, z6.b\n" + "udot z24.s, z5.b, z2.b\n" + "udot z25.s, z4.b, z2.b\n" + "udot z26.s, z6.b, z1.b\n" + "udot z27.s, z5.b, z1.b\n" + "udot z28.s, z4.b, z1.b\n" + "udot z29.s, z6.b, z0.b\n" + "udot z30.s, z5.b, z0.b\n" + "udot z31.s, z4.b, z0.b\n" "5:" // multiply loop done "st1w { z8.s }, p0, [%x[Cpanel]]\n" "subs x23, x23, #0x1\n" @@ -262,7 +266,7 @@ void sve_interleaved_u8u32_dot_8x3VL_a64fx( "bne 1b\n" : [Apanel] "+&r" (Apanel), [Cpanel] "+&r" (Cpanel), [ablocks] "+&r" (ablocks) : [args_ptr] "r" (&ka), [offsetof_Bpanel] "I" (offsetof(KernelArgs, Bpanel)), [offsetof_K] "I" (offsetof(KernelArgs, K)), [offsetof_bblocks] "I" (offsetof(KernelArgs, bblocks)) - : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "x20", "x21", "x22", "x23", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL/generic.cpp index 99fff4e83dd1a1124d994bbf2dceec0f03f6fd5b..1c88336c2d9d1ba18a6819a2e67bc9a80936d5e7 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL/generic.cpp @@ -29,8 +29,12 @@ namespace arm_gemm { void sve_interleaved_u8u32_dot_8x3VL( - const uint8_t *Apanel, const uint8_t *Bpanel, - uint32_t *Cpanel, int ablocks, int bblocks, int K) { + const uint8_t *Apanel, + const uint8_t *Bpanel, + uint32_t *Cpanel, + int ablocks, + int bblocks, + int K) { struct KernelArgs { size_t K = {}; @@ -85,10 +89,10 @@ void sve_interleaved_u8u32_dot_8x3VL( "3:" // main loop head "udot z8.s, z4.b, z0.b[0]\n" "udot z11.s, z4.b, z0.b[1]\n" - "ld1rqb { z2.b }, p0/Z, [%x[Apanel], #32]\n" + "ld1rqb { z3.b }, p0/Z, [%x[Apanel], #32]\n" "udot z14.s, z4.b, z0.b[2]\n" "udot z17.s, z4.b, z0.b[3]\n" - "ld1rqb { z3.b }, p0/Z, [%x[Apanel], #48]\n" + "ld1rqb { z7.b }, p0/Z, [%x[Apanel], #48]\n" "udot z20.s, z4.b, z1.b[0]\n" "udot z23.s, z4.b, z1.b[1]\n" "sub x20, x20, #0x2\n" @@ -115,35 +119,35 @@ void sve_interleaved_u8u32_dot_8x3VL( "udot z25.s, z6.b, z1.b[1]\n" "udot z28.s, z6.b, z1.b[2]\n" "udot z31.s, z6.b, z1.b[3]\n" - "ld1b { z6.b }, p0/Z, [x22, #5, MUL VL]\n" + "ld1b { z2.b }, p0/Z, [x22, #5, MUL VL]\n" "addvl x22, x22, #6\n" - "udot z8.s, z4.b, z2.b[0]\n" - "udot z11.s, z4.b, z2.b[1]\n" + "udot z8.s, z4.b, z3.b[0]\n" + "udot z11.s, z4.b, z3.b[1]\n" "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #16]\n" - "udot z14.s, z4.b, z2.b[2]\n" - "udot z17.s, z4.b, z2.b[3]\n" - "udot z20.s, z4.b, z3.b[0]\n" - "udot z23.s, z4.b, z3.b[1]\n" - "udot z26.s, z4.b, z3.b[2]\n" - "udot z29.s, z4.b, z3.b[3]\n" + "udot z14.s, z4.b, z3.b[2]\n" + "udot z17.s, z4.b, z3.b[3]\n" + "udot z20.s, z4.b, z7.b[0]\n" + "udot z23.s, z4.b, z7.b[1]\n" + "udot z26.s, z4.b, z7.b[2]\n" + "udot z29.s, z4.b, z7.b[3]\n" "ld1b { z4.b }, p0/Z, [x22]\n" - "udot z9.s, z5.b, z2.b[0]\n" - "udot z12.s, z5.b, z2.b[1]\n" - "udot z15.s, z5.b, z2.b[2]\n" - "udot z18.s, z5.b, z2.b[3]\n" - "udot z21.s, z5.b, z3.b[0]\n" - "udot z24.s, z5.b, z3.b[1]\n" - "udot z27.s, z5.b, z3.b[2]\n" - "udot z30.s, z5.b, z3.b[3]\n" + "udot z9.s, z5.b, z3.b[0]\n" + "udot z12.s, z5.b, z3.b[1]\n" + "udot z15.s, z5.b, z3.b[2]\n" + "udot z18.s, z5.b, z3.b[3]\n" + "udot z21.s, z5.b, z7.b[0]\n" + "udot z24.s, z5.b, z7.b[1]\n" + "udot z27.s, z5.b, z7.b[2]\n" + "udot z30.s, z5.b, z7.b[3]\n" "ld1b { z5.b }, p0/Z, [x22, #1, MUL VL]\n" - "udot z10.s, z6.b, z2.b[0]\n" - "udot z13.s, z6.b, z2.b[1]\n" - "udot z16.s, z6.b, z2.b[2]\n" - "udot z19.s, z6.b, z2.b[3]\n" - "udot z22.s, z6.b, z3.b[0]\n" - "udot z25.s, z6.b, z3.b[1]\n" - "udot z28.s, z6.b, z3.b[2]\n" - "udot z31.s, z6.b, z3.b[3]\n" + "udot z10.s, z2.b, z3.b[0]\n" + "udot z13.s, z2.b, z3.b[1]\n" + "udot z16.s, z2.b, z3.b[2]\n" + "udot z19.s, z2.b, z3.b[3]\n" + "udot z22.s, z2.b, z7.b[0]\n" + "udot z25.s, z2.b, z7.b[1]\n" + "udot z28.s, z2.b, z7.b[2]\n" + "udot z31.s, z2.b, z7.b[3]\n" "ld1b { z6.b }, p0/Z, [x22, #2, MUL VL]\n" "bge 3b\n" "4:" // main loop skip @@ -174,37 +178,37 @@ void sve_interleaved_u8u32_dot_8x3VL( "udot z28.s, z6.b, z1.b[2]\n" "udot z31.s, z6.b, z1.b[3]\n" "cbz x20, 5f\n" - "ld1rqb { z0.b }, p0/Z, [%x[Apanel]]\n" - "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #16]\n" + "ld1rqb { z4.b }, p0/Z, [%x[Apanel]]\n" + "ld1rqb { z3.b }, p0/Z, [%x[Apanel], #16]\n" "add %x[Apanel], %x[Apanel], #0x20\n" - "ld1b { z7.b }, p0/Z, [x22]\n" - "ld1b { z4.b }, p0/Z, [x22, #1, MUL VL]\n" - "udot z8.s, z7.b, z0.b[0]\n" - "ld1b { z5.b }, p0/Z, [x22, #2, MUL VL]\n" - "udot z11.s, z7.b, z0.b[1]\n" - "udot z14.s, z7.b, z0.b[2]\n" - "udot z17.s, z7.b, z0.b[3]\n" - "udot z20.s, z7.b, z1.b[0]\n" + "ld1b { z2.b }, p0/Z, [x22]\n" + "ld1b { z1.b }, p0/Z, [x22, #1, MUL VL]\n" + "udot z8.s, z2.b, z4.b[0]\n" + "ld1b { z0.b }, p0/Z, [x22, #2, MUL VL]\n" + "udot z11.s, z2.b, z4.b[1]\n" + "udot z14.s, z2.b, z4.b[2]\n" + "udot z17.s, z2.b, z4.b[3]\n" + "udot z20.s, z2.b, z3.b[0]\n" "addvl x22, x22, #3\n" - "udot z23.s, z7.b, z1.b[1]\n" - "udot z26.s, z7.b, z1.b[2]\n" - "udot z29.s, z7.b, z1.b[3]\n" - "udot z9.s, z4.b, z0.b[0]\n" - "udot z12.s, z4.b, z0.b[1]\n" - "udot z15.s, z4.b, z0.b[2]\n" - "udot z18.s, z4.b, z0.b[3]\n" - "udot z21.s, z4.b, z1.b[0]\n" - "udot z24.s, z4.b, z1.b[1]\n" - "udot z27.s, z4.b, z1.b[2]\n" - "udot z30.s, z4.b, z1.b[3]\n" - "udot z10.s, z5.b, z0.b[0]\n" - "udot z13.s, z5.b, z0.b[1]\n" - "udot z16.s, z5.b, z0.b[2]\n" - "udot z19.s, z5.b, z0.b[3]\n" - "udot z22.s, z5.b, z1.b[0]\n" - "udot z25.s, z5.b, z1.b[1]\n" - "udot z28.s, z5.b, z1.b[2]\n" - "udot z31.s, z5.b, z1.b[3]\n" + "udot z23.s, z2.b, z3.b[1]\n" + "udot z26.s, z2.b, z3.b[2]\n" + "udot z29.s, z2.b, z3.b[3]\n" + "udot z9.s, z1.b, z4.b[0]\n" + "udot z12.s, z1.b, z4.b[1]\n" + "udot z15.s, z1.b, z4.b[2]\n" + "udot z18.s, z1.b, z4.b[3]\n" + "udot z21.s, z1.b, z3.b[0]\n" + "udot z24.s, z1.b, z3.b[1]\n" + "udot z27.s, z1.b, z3.b[2]\n" + "udot z30.s, z1.b, z3.b[3]\n" + "udot z10.s, z0.b, z4.b[0]\n" + "udot z13.s, z0.b, z4.b[1]\n" + "udot z16.s, z0.b, z4.b[2]\n" + "udot z19.s, z0.b, z4.b[3]\n" + "udot z22.s, z0.b, z3.b[0]\n" + "udot z25.s, z0.b, z3.b[1]\n" + "udot z28.s, z0.b, z3.b[2]\n" + "udot z31.s, z0.b, z3.b[3]\n" "5:" // multiply loop done "st1w { z8.s }, p0, [%x[Cpanel]]\n" "subs x23, x23, #0x1\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_8x3VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_8x3VL.hpp index 58d21d6c407fc90aab28780547be9caacd799fe2..067d0bf2581a1a20e753b1c4dfdff4fcdb139006 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_8x3VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_8x3VL.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,20 +10,20 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ #pragma once - #ifdef ARM_COMPUTE_ENABLE_SVE + #include "../std_transforms_sve.hpp" #include "../performance_parameters.hpp" @@ -55,11 +55,6 @@ public: return get_vector_length() * 3; } - static unsigned int stripe_width() - { - return get_vector_length(); - } - static constexpr unsigned int k_unroll() { return 8; @@ -108,5 +103,4 @@ public: } // namespace arm_gemm #undef ARGLIST - #endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_8x3VL/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_8x3VL/generic.cpp index 0b70d034dd4b3bc9ad3ee52efd6efeb60fcf5d8a..28449ea99b5533cf78deffcb61c13c17c8195ac5 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_8x3VL/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_8x3VL/generic.cpp @@ -29,8 +29,12 @@ namespace arm_gemm { void sve_interleaved_u8u32_mmla_8x3VL( - const uint8_t *Apanel, const uint8_t *Bpanel, - uint32_t *Cpanel, int ablocks, int bblocks, int K) { + const uint8_t *Apanel, + const uint8_t *Bpanel, + uint32_t *Cpanel, + int ablocks, + int bblocks, + int K) { struct KernelArgs { size_t K = {}; @@ -85,82 +89,82 @@ void sve_interleaved_u8u32_mmla_8x3VL( "mov z31.s, #0x0\n" "blt 4f\n" "3:" // main loop head - "ld1rqb { z3.b }, p0/Z, [%x[Apanel]]\n" + "ld1rqb { z6.b }, p0/Z, [%x[Apanel]]\n" ".inst 0x45c49808 // ummla z8.s, z0.b, z4.b\n" ".inst 0x45c5980b // ummla z11.s, z0.b, z5.b\n" ".inst 0x45c4982e // ummla z14.s, z1.b, z4.b\n" ".inst 0x45c59831 // ummla z17.s, z1.b, z5.b\n" - "ld1b { z6.b }, p0/Z, [x22]\n" + "ld1b { z7.b }, p0/Z, [x22]\n" ".inst 0x45c49854 // ummla z20.s, z2.b, z4.b\n" ".inst 0x45c59857 // ummla z23.s, z2.b, z5.b\n" - "ld1b { z7.b }, p0/Z, [x22, #1, MUL VL]\n" - ".inst 0x45c4987a // ummla z26.s, z3.b, z4.b\n" - ".inst 0x45c5987d // ummla z29.s, z3.b, z5.b\n" - "ld1b { z4.b }, p0/Z, [x22, #2, MUL VL]\n" - "ld1b { z5.b }, p0/Z, [x22, #3, MUL VL]\n" - ".inst 0x45c69809 // ummla z9.s, z0.b, z6.b\n" - ".inst 0x45c7980c // ummla z12.s, z0.b, z7.b\n" - ".inst 0x45c6982f // ummla z15.s, z1.b, z6.b\n" - ".inst 0x45c79832 // ummla z18.s, z1.b, z7.b\n" + "ld1b { z3.b }, p0/Z, [x22, #1, MUL VL]\n" + ".inst 0x45c498da // ummla z26.s, z6.b, z4.b\n" + ".inst 0x45c598dd // ummla z29.s, z6.b, z5.b\n" + "ld1b { z5.b }, p0/Z, [x22, #2, MUL VL]\n" + "ld1b { z4.b }, p0/Z, [x22, #3, MUL VL]\n" + ".inst 0x45c79809 // ummla z9.s, z0.b, z7.b\n" + ".inst 0x45c3980c // ummla z12.s, z0.b, z3.b\n" + ".inst 0x45c7982f // ummla z15.s, z1.b, z7.b\n" + ".inst 0x45c39832 // ummla z18.s, z1.b, z3.b\n" "sub x20, x20, #0x2\n" - ".inst 0x45c69855 // ummla z21.s, z2.b, z6.b\n" - ".inst 0x45c79858 // ummla z24.s, z2.b, z7.b\n" + ".inst 0x45c79855 // ummla z21.s, z2.b, z7.b\n" + ".inst 0x45c39858 // ummla z24.s, z2.b, z3.b\n" "cmp x20, #0x2\n" - ".inst 0x45c6987b // ummla z27.s, z3.b, z6.b\n" - ".inst 0x45c7987e // ummla z30.s, z3.b, z7.b\n" - "ld1b { z6.b }, p0/Z, [x22, #4, MUL VL]\n" - ".inst 0x45c4980a // ummla z10.s, z0.b, z4.b\n" - ".inst 0x45c5980d // ummla z13.s, z0.b, z5.b\n" + ".inst 0x45c798db // ummla z27.s, z6.b, z7.b\n" + ".inst 0x45c398de // ummla z30.s, z6.b, z3.b\n" + "ld1b { z3.b }, p0/Z, [x22, #4, MUL VL]\n" + ".inst 0x45c5980a // ummla z10.s, z0.b, z5.b\n" + ".inst 0x45c4980d // ummla z13.s, z0.b, z4.b\n" "ld1rqb { z0.b }, p0/Z, [%x[Apanel], #16]\n" - ".inst 0x45c49830 // ummla z16.s, z1.b, z4.b\n" - ".inst 0x45c59833 // ummla z19.s, z1.b, z5.b\n" + ".inst 0x45c59830 // ummla z16.s, z1.b, z5.b\n" + ".inst 0x45c49833 // ummla z19.s, z1.b, z4.b\n" "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #32]\n" - ".inst 0x45c49856 // ummla z22.s, z2.b, z4.b\n" - ".inst 0x45c59859 // ummla z25.s, z2.b, z5.b\n" + ".inst 0x45c59856 // ummla z22.s, z2.b, z5.b\n" + ".inst 0x45c49859 // ummla z25.s, z2.b, z4.b\n" "ld1b { z7.b }, p0/Z, [x22, #5, MUL VL]\n" - ".inst 0x45c4987c // ummla z28.s, z3.b, z4.b\n" - ".inst 0x45c5987f // ummla z31.s, z3.b, z5.b\n" - "ld1rqb { z2.b }, p0/Z, [%x[Apanel], #48]\n" - "ld1rqb { z3.b }, p0/Z, [%x[Apanel], #64]\n" - "ld1b { z4.b }, p0/Z, [x22, #6, MUL VL]\n" - ".inst 0x45c69808 // ummla z8.s, z0.b, z6.b\n" - "ld1b { z5.b }, p0/Z, [x22, #7, MUL VL]\n" + ".inst 0x45c598dc // ummla z28.s, z6.b, z5.b\n" + ".inst 0x45c498df // ummla z31.s, z6.b, z4.b\n" + "ld1rqb { z5.b }, p0/Z, [%x[Apanel], #48]\n" + "ld1rqb { z6.b }, p0/Z, [%x[Apanel], #64]\n" + "ld1b { z2.b }, p0/Z, [x22, #6, MUL VL]\n" + ".inst 0x45c39808 // ummla z8.s, z0.b, z3.b\n" + "ld1b { z4.b }, p0/Z, [x22, #7, MUL VL]\n" "addvl x22, x22, #16\n" ".inst 0x45c7980b // ummla z11.s, z0.b, z7.b\n" - ".inst 0x45c6982e // ummla z14.s, z1.b, z6.b\n" + ".inst 0x45c3982e // ummla z14.s, z1.b, z3.b\n" ".inst 0x45c79831 // ummla z17.s, z1.b, z7.b\n" - ".inst 0x45c69854 // ummla z20.s, z2.b, z6.b\n" - ".inst 0x45c79857 // ummla z23.s, z2.b, z7.b\n" - ".inst 0x45c6987a // ummla z26.s, z3.b, z6.b\n" - ".inst 0x45c7987d // ummla z29.s, z3.b, z7.b\n" - "ld1b { z6.b }, p0/Z, [x22, #-8, MUL VL]\n" + ".inst 0x45c398b4 // ummla z20.s, z5.b, z3.b\n" + ".inst 0x45c798b7 // ummla z23.s, z5.b, z7.b\n" + ".inst 0x45c398da // ummla z26.s, z6.b, z3.b\n" + ".inst 0x45c798dd // ummla z29.s, z6.b, z7.b\n" + "ld1b { z3.b }, p0/Z, [x22, #-8, MUL VL]\n" "ld1b { z7.b }, p0/Z, [x22, #-7, MUL VL]\n" - ".inst 0x45c49809 // ummla z9.s, z0.b, z4.b\n" - ".inst 0x45c5980c // ummla z12.s, z0.b, z5.b\n" - ".inst 0x45c4982f // ummla z15.s, z1.b, z4.b\n" - ".inst 0x45c59832 // ummla z18.s, z1.b, z5.b\n" - ".inst 0x45c49855 // ummla z21.s, z2.b, z4.b\n" - ".inst 0x45c59858 // ummla z24.s, z2.b, z5.b\n" - ".inst 0x45c4987b // ummla z27.s, z3.b, z4.b\n" - ".inst 0x45c5987e // ummla z30.s, z3.b, z5.b\n" + ".inst 0x45c29809 // ummla z9.s, z0.b, z2.b\n" + ".inst 0x45c4980c // ummla z12.s, z0.b, z4.b\n" + ".inst 0x45c2982f // ummla z15.s, z1.b, z2.b\n" + ".inst 0x45c49832 // ummla z18.s, z1.b, z4.b\n" + ".inst 0x45c298b5 // ummla z21.s, z5.b, z2.b\n" + ".inst 0x45c498b8 // ummla z24.s, z5.b, z4.b\n" + ".inst 0x45c298db // ummla z27.s, z6.b, z2.b\n" + ".inst 0x45c498de // ummla z30.s, z6.b, z4.b\n" "ld1b { z4.b }, p0/Z, [x22, #-6, MUL VL]\n" - ".inst 0x45c6980a // ummla z10.s, z0.b, z6.b\n" + ".inst 0x45c3980a // ummla z10.s, z0.b, z3.b\n" ".inst 0x45c7980d // ummla z13.s, z0.b, z7.b\n" "ld1rqb { z0.b }, p0/Z, [%x[Apanel], #80]\n" - ".inst 0x45c69830 // ummla z16.s, z1.b, z6.b\n" + ".inst 0x45c39830 // ummla z16.s, z1.b, z3.b\n" ".inst 0x45c79833 // ummla z19.s, z1.b, z7.b\n" "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #96]\n" - ".inst 0x45c69856 // ummla z22.s, z2.b, z6.b\n" - ".inst 0x45c79859 // ummla z25.s, z2.b, z7.b\n" + ".inst 0x45c398b6 // ummla z22.s, z5.b, z3.b\n" + ".inst 0x45c798b9 // ummla z25.s, z5.b, z7.b\n" "ld1b { z5.b }, p0/Z, [x22, #-5, MUL VL]\n" - ".inst 0x45c6987c // ummla z28.s, z3.b, z6.b\n" - ".inst 0x45c7987f // ummla z31.s, z3.b, z7.b\n" + ".inst 0x45c398dc // ummla z28.s, z6.b, z3.b\n" + ".inst 0x45c798df // ummla z31.s, z6.b, z7.b\n" "ld1rqb { z2.b }, p0/Z, [%x[Apanel], #112]\n" "add %x[Apanel], %x[Apanel], #0x80\n" "addvl x22, x22, #-4\n" "bge 3b\n" "4:" // main loop skip - "ld1rqb { z3.b }, p0/Z, [%x[Apanel]]\n" + "ld1rqb { z7.b }, p0/Z, [%x[Apanel]]\n" ".inst 0x45c49808 // ummla z8.s, z0.b, z4.b\n" ".inst 0x45c5980b // ummla z11.s, z0.b, z5.b\n" ".inst 0x45c4982e // ummla z14.s, z1.b, z4.b\n" @@ -168,114 +172,114 @@ void sve_interleaved_u8u32_mmla_8x3VL( "ld1b { z6.b }, p0/Z, [x22]\n" ".inst 0x45c49854 // ummla z20.s, z2.b, z4.b\n" ".inst 0x45c59857 // ummla z23.s, z2.b, z5.b\n" - "ld1b { z7.b }, p0/Z, [x22, #1, MUL VL]\n" - ".inst 0x45c4987a // ummla z26.s, z3.b, z4.b\n" - ".inst 0x45c5987d // ummla z29.s, z3.b, z5.b\n" - "ld1b { z4.b }, p0/Z, [x22, #2, MUL VL]\n" - "ld1b { z5.b }, p0/Z, [x22, #3, MUL VL]\n" + "ld1b { z3.b }, p0/Z, [x22, #1, MUL VL]\n" + ".inst 0x45c498fa // ummla z26.s, z7.b, z4.b\n" + ".inst 0x45c598fd // ummla z29.s, z7.b, z5.b\n" + "ld1b { z5.b }, p0/Z, [x22, #2, MUL VL]\n" + "ld1b { z4.b }, p0/Z, [x22, #3, MUL VL]\n" ".inst 0x45c69809 // ummla z9.s, z0.b, z6.b\n" - ".inst 0x45c7980c // ummla z12.s, z0.b, z7.b\n" + ".inst 0x45c3980c // ummla z12.s, z0.b, z3.b\n" ".inst 0x45c6982f // ummla z15.s, z1.b, z6.b\n" - ".inst 0x45c79832 // ummla z18.s, z1.b, z7.b\n" + ".inst 0x45c39832 // ummla z18.s, z1.b, z3.b\n" "add %x[Apanel], %x[Apanel], #0x10\n" ".inst 0x45c69855 // ummla z21.s, z2.b, z6.b\n" - ".inst 0x45c79858 // ummla z24.s, z2.b, z7.b\n" + ".inst 0x45c39858 // ummla z24.s, z2.b, z3.b\n" "addvl x22, x22, #4\n" - ".inst 0x45c6987b // ummla z27.s, z3.b, z6.b\n" - ".inst 0x45c7987e // ummla z30.s, z3.b, z7.b\n" - ".inst 0x45c4980a // ummla z10.s, z0.b, z4.b\n" - ".inst 0x45c5980d // ummla z13.s, z0.b, z5.b\n" - ".inst 0x45c49830 // ummla z16.s, z1.b, z4.b\n" - ".inst 0x45c59833 // ummla z19.s, z1.b, z5.b\n" - ".inst 0x45c49856 // ummla z22.s, z2.b, z4.b\n" - ".inst 0x45c59859 // ummla z25.s, z2.b, z5.b\n" - ".inst 0x45c4987c // ummla z28.s, z3.b, z4.b\n" - ".inst 0x45c5987f // ummla z31.s, z3.b, z5.b\n" + ".inst 0x45c698fb // ummla z27.s, z7.b, z6.b\n" + ".inst 0x45c398fe // ummla z30.s, z7.b, z3.b\n" + ".inst 0x45c5980a // ummla z10.s, z0.b, z5.b\n" + ".inst 0x45c4980d // ummla z13.s, z0.b, z4.b\n" + ".inst 0x45c59830 // ummla z16.s, z1.b, z5.b\n" + ".inst 0x45c49833 // ummla z19.s, z1.b, z4.b\n" + ".inst 0x45c59856 // ummla z22.s, z2.b, z5.b\n" + ".inst 0x45c49859 // ummla z25.s, z2.b, z4.b\n" + ".inst 0x45c598fc // ummla z28.s, z7.b, z5.b\n" + ".inst 0x45c498ff // ummla z31.s, z7.b, z4.b\n" "cbz x20, 5f\n" - "ld1b { z6.b }, p0/Z, [x22]\n" - "ld1rqb { z0.b }, p0/Z, [%x[Apanel]]\n" - ".inst 0x45c69808 // ummla z8.s, z0.b, z6.b\n" - "ld1rqb { z1.b }, p0/Z, [%x[Apanel], #16]\n" - "ld1b { z7.b }, p0/Z, [x22, #1, MUL VL]\n" - ".inst 0x45c7980b // ummla z11.s, z0.b, z7.b\n" - "ld1rqb { z2.b }, p0/Z, [%x[Apanel], #32]\n" - "ld1rqb { z3.b }, p0/Z, [%x[Apanel], #48]\n" - ".inst 0x45c6982e // ummla z14.s, z1.b, z6.b\n" - ".inst 0x45c79831 // ummla z17.s, z1.b, z7.b\n" - ".inst 0x45c69854 // ummla z20.s, z2.b, z6.b\n" - "ld1b { z4.b }, p0/Z, [x22, #2, MUL VL]\n" - ".inst 0x45c79857 // ummla z23.s, z2.b, z7.b\n" - ".inst 0x45c6987a // ummla z26.s, z3.b, z6.b\n" - "ld1b { z5.b }, p0/Z, [x22, #3, MUL VL]\n" - ".inst 0x45c7987d // ummla z29.s, z3.b, z7.b\n" - "ld1b { z6.b }, p0/Z, [x22, #4, MUL VL]\n" - "ld1b { z7.b }, p0/Z, [x22, #5, MUL VL]\n" - ".inst 0x45c49809 // ummla z9.s, z0.b, z4.b\n" - ".inst 0x45c5980c // ummla z12.s, z0.b, z5.b\n" + "ld1b { z1.b }, p0/Z, [x22]\n" + "ld1rqb { z7.b }, p0/Z, [%x[Apanel]]\n" + ".inst 0x45c198e8 // ummla z8.s, z7.b, z1.b\n" + "ld1rqb { z6.b }, p0/Z, [%x[Apanel], #16]\n" + "ld1b { z0.b }, p0/Z, [x22, #1, MUL VL]\n" + ".inst 0x45c098eb // ummla z11.s, z7.b, z0.b\n" + "ld1rqb { z5.b }, p0/Z, [%x[Apanel], #32]\n" + "ld1rqb { z4.b }, p0/Z, [%x[Apanel], #48]\n" + ".inst 0x45c198ce // ummla z14.s, z6.b, z1.b\n" + ".inst 0x45c098d1 // ummla z17.s, z6.b, z0.b\n" + ".inst 0x45c198b4 // ummla z20.s, z5.b, z1.b\n" + "ld1b { z3.b }, p0/Z, [x22, #2, MUL VL]\n" + ".inst 0x45c098b7 // ummla z23.s, z5.b, z0.b\n" + ".inst 0x45c1989a // ummla z26.s, z4.b, z1.b\n" + "ld1b { z2.b }, p0/Z, [x22, #3, MUL VL]\n" + ".inst 0x45c0989d // ummla z29.s, z4.b, z0.b\n" + "ld1b { z1.b }, p0/Z, [x22, #4, MUL VL]\n" + "ld1b { z0.b }, p0/Z, [x22, #5, MUL VL]\n" + ".inst 0x45c398e9 // ummla z9.s, z7.b, z3.b\n" + ".inst 0x45c298ec // ummla z12.s, z7.b, z2.b\n" "addvl x22, x22, #6\n" - ".inst 0x45c4982f // ummla z15.s, z1.b, z4.b\n" - ".inst 0x45c59832 // ummla z18.s, z1.b, z5.b\n" + ".inst 0x45c398cf // ummla z15.s, z6.b, z3.b\n" + ".inst 0x45c298d2 // ummla z18.s, z6.b, z2.b\n" "add %x[Apanel], %x[Apanel], #0x40\n" - ".inst 0x45c49855 // ummla z21.s, z2.b, z4.b\n" - ".inst 0x45c59858 // ummla z24.s, z2.b, z5.b\n" - ".inst 0x45c4987b // ummla z27.s, z3.b, z4.b\n" - ".inst 0x45c5987e // ummla z30.s, z3.b, z5.b\n" - ".inst 0x45c6980a // ummla z10.s, z0.b, z6.b\n" - ".inst 0x45c7980d // ummla z13.s, z0.b, z7.b\n" - ".inst 0x45c69830 // ummla z16.s, z1.b, z6.b\n" - ".inst 0x45c79833 // ummla z19.s, z1.b, z7.b\n" - ".inst 0x45c69856 // ummla z22.s, z2.b, z6.b\n" - ".inst 0x45c79859 // ummla z25.s, z2.b, z7.b\n" - ".inst 0x45c6987c // ummla z28.s, z3.b, z6.b\n" - ".inst 0x45c7987f // ummla z31.s, z3.b, z7.b\n" + ".inst 0x45c398b5 // ummla z21.s, z5.b, z3.b\n" + ".inst 0x45c298b8 // ummla z24.s, z5.b, z2.b\n" + ".inst 0x45c3989b // ummla z27.s, z4.b, z3.b\n" + ".inst 0x45c2989e // ummla z30.s, z4.b, z2.b\n" + ".inst 0x45c198ea // ummla z10.s, z7.b, z1.b\n" + ".inst 0x45c098ed // ummla z13.s, z7.b, z0.b\n" + ".inst 0x45c198d0 // ummla z16.s, z6.b, z1.b\n" + ".inst 0x45c098d3 // ummla z19.s, z6.b, z0.b\n" + ".inst 0x45c198b6 // ummla z22.s, z5.b, z1.b\n" + ".inst 0x45c098b9 // ummla z25.s, z5.b, z0.b\n" + ".inst 0x45c1989c // ummla z28.s, z4.b, z1.b\n" + ".inst 0x45c0989f // ummla z31.s, z4.b, z0.b\n" "5:" // multiply loop done - "uzp1 z4.d, z8.d, z11.d\n" + "uzp1 z0.d, z8.d, z11.d\n" "uzp2 z8.d, z8.d, z11.d\n" - "st1w { z4.s }, p0, [%x[Cpanel]]\n" - "uzp1 z11.d, z9.d, z12.d\n" + "st1w { z0.s }, p0, [%x[Cpanel]]\n" + "uzp1 z0.d, z9.d, z12.d\n" "uzp2 z9.d, z9.d, z12.d\n" - "st1w { z11.s }, p0, [%x[Cpanel], #1, MUL VL]\n" - "uzp1 z12.d, z10.d, z13.d\n" + "st1w { z0.s }, p0, [%x[Cpanel], #1, MUL VL]\n" + "uzp1 z0.d, z10.d, z13.d\n" "uzp2 z10.d, z10.d, z13.d\n" - "st1w { z12.s }, p0, [%x[Cpanel], #2, MUL VL]\n" + "st1w { z0.s }, p0, [%x[Cpanel], #2, MUL VL]\n" "st1w { z8.s }, p0, [%x[Cpanel], #3, MUL VL]\n" - "uzp1 z13.d, z14.d, z17.d\n" + "uzp1 z0.d, z14.d, z17.d\n" "uzp2 z14.d, z14.d, z17.d\n" "st1w { z9.s }, p0, [%x[Cpanel], #4, MUL VL]\n" - "uzp1 z17.d, z15.d, z18.d\n" + "uzp1 z1.d, z15.d, z18.d\n" "subs x23, x23, #0x1\n" "st1w { z10.s }, p0, [%x[Cpanel], #5, MUL VL]\n" "uzp2 z15.d, z15.d, z18.d\n" - "uzp1 z18.d, z16.d, z19.d\n" - "st1w { z13.s }, p0, [%x[Cpanel], #6, MUL VL]\n" + "uzp1 z17.d, z16.d, z19.d\n" + "st1w { z0.s }, p0, [%x[Cpanel], #6, MUL VL]\n" "uzp2 z16.d, z16.d, z19.d\n" - "uzp1 z19.d, z20.d, z23.d\n" - "st1w { z17.s }, p0, [%x[Cpanel], #7, MUL VL]\n" + "uzp1 z0.d, z20.d, z23.d\n" + "st1w { z1.s }, p0, [%x[Cpanel], #7, MUL VL]\n" "addvl %x[Cpanel], %x[Cpanel], #16\n" "uzp2 z20.d, z20.d, z23.d\n" - "st1w { z18.s }, p0, [%x[Cpanel], #-8, MUL VL]\n" + "st1w { z17.s }, p0, [%x[Cpanel], #-8, MUL VL]\n" "uzp1 z23.d, z21.d, z24.d\n" "uzp2 z21.d, z21.d, z24.d\n" "st1w { z14.s }, p0, [%x[Cpanel], #-7, MUL VL]\n" - "uzp1 z24.d, z22.d, z25.d\n" + "uzp1 z19.d, z22.d, z25.d\n" "uzp2 z22.d, z22.d, z25.d\n" "st1w { z15.s }, p0, [%x[Cpanel], #-6, MUL VL]\n" - "uzp1 z25.d, z26.d, z29.d\n" + "uzp1 z18.d, z26.d, z29.d\n" "uzp2 z26.d, z26.d, z29.d\n" "st1w { z16.s }, p0, [%x[Cpanel], #-5, MUL VL]\n" - "uzp1 z29.d, z27.d, z30.d\n" + "uzp1 z17.d, z27.d, z30.d\n" "uzp2 z27.d, z27.d, z30.d\n" - "st1w { z19.s }, p0, [%x[Cpanel], #-4, MUL VL]\n" - "uzp1 z30.d, z28.d, z31.d\n" + "st1w { z0.s }, p0, [%x[Cpanel], #-4, MUL VL]\n" + "uzp1 z16.d, z28.d, z31.d\n" "uzp2 z28.d, z28.d, z31.d\n" "st1w { z23.s }, p0, [%x[Cpanel], #-3, MUL VL]\n" - "st1w { z24.s }, p0, [%x[Cpanel], #-2, MUL VL]\n" + "st1w { z19.s }, p0, [%x[Cpanel], #-2, MUL VL]\n" "st1w { z20.s }, p0, [%x[Cpanel], #-1, MUL VL]\n" "st1w { z21.s }, p0, [%x[Cpanel]]\n" "st1w { z22.s }, p0, [%x[Cpanel], #1, MUL VL]\n" - "st1w { z25.s }, p0, [%x[Cpanel], #2, MUL VL]\n" - "st1w { z29.s }, p0, [%x[Cpanel], #3, MUL VL]\n" - "st1w { z30.s }, p0, [%x[Cpanel], #4, MUL VL]\n" + "st1w { z18.s }, p0, [%x[Cpanel], #2, MUL VL]\n" + "st1w { z17.s }, p0, [%x[Cpanel], #3, MUL VL]\n" + "st1w { z16.s }, p0, [%x[Cpanel], #4, MUL VL]\n" "st1w { z26.s }, p0, [%x[Cpanel], #5, MUL VL]\n" "st1w { z27.s }, p0, [%x[Cpanel], #6, MUL VL]\n" "st1w { z28.s }, p0, [%x[Cpanel], #7, MUL VL]\n" @@ -290,4 +294,4 @@ void sve_interleaved_u8u32_mmla_8x3VL( } } // namespace arm_gemm -#endif // __ARM_FEATURE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/misc.cpp b/src/core/NEON/kernels/arm_gemm/misc.cpp index cf99bbdb462942b0fecedc203599ab86c1203816..87310d996d63e395e95a1c98cc031aae405a1266 100644 --- a/src/core/NEON/kernels/arm_gemm/misc.cpp +++ b/src/core/NEON/kernels/arm_gemm/misc.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2018, 2022 Arm Limited. + * Copyright (c) 2017-2018, 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -56,10 +56,14 @@ WeightFormat get_weight_format(const KernelWeightFormat kwf, size_t element_size wf_i |= 0x10; } +#ifdef ARM_COMPUTE_ENABLE_SVE // Get total bytes in vector output if (kwf_i & 0x1) { vector_bytes = vector_count * get_vector_length(); } else { +#else + if (1) { +#endif vector_bytes = vector_count * 16; } diff --git a/src/core/NEON/kernels/arm_gemm/quantized.hpp b/src/core/NEON/kernels/arm_gemm/quantized.hpp index 3f3443025cb5b71006321fda509467bc173e18df..31dd65b397739fe2476eaa45bc2e532cfaa2eb01 100644 --- a/src/core/NEON/kernels/arm_gemm/quantized.hpp +++ b/src/core/NEON/kernels/arm_gemm/quantized.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2019, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -42,7 +42,7 @@ void compute_col_sums(const Requantize32 &qp, unsigned int width, unsigned int h unsigned int multi, unsigned int first_col); template -void row_sums_indirect(unsigned int num_strings, const unsigned int *string_lengths, IndirectInputArg A_arg, +void row_sums_indirect(size_t num_strings, const unsigned int *string_lengths, IndirectInputArg A_arg, size_t M, int32_t *output_ptr, const Requantize32 *qp); } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/rowsum_indirect_s8.cpp b/src/core/NEON/kernels/arm_gemm/rowsum_indirect_s8.cpp index 7345793f938b19bce5d296854538ce2dd1a6cc2b..94cd7ddfebf394350da4c6f63f0b71c9fbe555eb 100644 --- a/src/core/NEON/kernels/arm_gemm/rowsum_indirect_s8.cpp +++ b/src/core/NEON/kernels/arm_gemm/rowsum_indirect_s8.cpp @@ -34,7 +34,7 @@ namespace arm_gemm { template<> void row_sums_indirect( - unsigned int num_strings, const unsigned int *string_lengths, IndirectInputArg A_arg, + size_t num_strings, const unsigned int *string_lengths, IndirectInputArg A_arg, size_t M, int32_t *out_ptr, const Requantize32 *qp ) { diff --git a/src/core/NEON/kernels/arm_gemm/rowsum_indirect_u8.cpp b/src/core/NEON/kernels/arm_gemm/rowsum_indirect_u8.cpp index ff95507d79ccdda9effbdfdf53dfe763b396cdc9..2ab0397fdaccaa951567c96be5bf582f09930450 100644 --- a/src/core/NEON/kernels/arm_gemm/rowsum_indirect_u8.cpp +++ b/src/core/NEON/kernels/arm_gemm/rowsum_indirect_u8.cpp @@ -34,7 +34,7 @@ namespace arm_gemm { template<> void row_sums_indirect( - unsigned int num_strings, const unsigned int *string_lengths, IndirectInputArg A_arg, + size_t num_strings, const unsigned int *string_lengths, IndirectInputArg A_arg, size_t M, int32_t *out_ptr, const Requantize32 *qp ) { diff --git a/src/core/NEON/kernels/arm_gemm/std_transforms_sme.hpp b/src/core/NEON/kernels/arm_gemm/std_transforms_sme.hpp index ae452e118425a7a1c44ef506108c49f73febb787..afe24e7ce062d5825dca6b20f22fa310ac606167 100644 --- a/src/core/NEON/kernels/arm_gemm/std_transforms_sme.hpp +++ b/src/core/NEON/kernels/arm_gemm/std_transforms_sme.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -67,9 +67,8 @@ public: } template - void Merge(TOut *out, const TResult *in, int stride, int y0, int ymax, int x0, int xmax, const TOut *bias, const Activation act, bool accumulate) { + void Merge(TOut *, const TResult *, int, int, int, int, int, const TOut *, const Activation, bool) { // Separate merge not supported for SME. - ARM_COMPUTE_UNUSED(out, in, stride, y0, ymax, x0, xmax, bias, act, accumulate); } }; diff --git a/src/core/NEON/kernels/arm_gemm/transform.cpp b/src/core/NEON/kernels/arm_gemm/transform.cpp index ef5a01a578496b9a702aea846e36f35b4bd6d52f..5aa62f0fe4eec033b09deeaec4d50c804227655a 100644 --- a/src/core/NEON/kernels/arm_gemm/transform.cpp +++ b/src/core/NEON/kernels/arm_gemm/transform.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -132,7 +132,9 @@ template void Transform<8, 1, true, VLType::None>(float *, const float *, int, i #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) template void Transform<8, 1, true, VLType::None>(float *, const __fp16 *, int, int, int, int, int); #endif // defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#ifdef ARM_COMPUTE_ENABLE_BF16 template void Transform<8, 1, true, VLType::None>(float *, const bfloat16 *, int, int, int, int, int); +#endif #endif // AArch32 } // namespace arm_gemm diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_128.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_128.hpp index e6186984e88a7269be83157eaed54e7457ddbdb5..8574d892266f03729e18f1419b7a5bfa6f3ea29e 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_128.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_128.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -193,7 +193,6 @@ void a64_transpose_interleave_128(uint32_t *out, const uint32_t *in, size_t widt "bge 1b\n" "cbz %x[height], 20f\n" "10:" // Main loop skip - "11:" // Tail row loop: Head "mov x20, %x[width]\n" "mov x25, %x[in]\n" @@ -264,7 +263,6 @@ void a64_transpose_interleave_128(uint32_t *out, const uint32_t *in, size_t widt "add %x[out], %x[out], #0x80\n" "bge 11b\n" "20:" // Done - : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [width] "r" (width) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25" @@ -286,4 +284,5 @@ void Transform<32, 1, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_1x4.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_1x4.hpp index 6d97f71c7db402fbd42672168dbaba4fa5d8e956..cdf1f98608e49e6babcf34b2c67e7c06f585864e 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_1x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_1x4.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -427,4 +427,5 @@ void Transform<12, 4, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_1x8.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_1x8.hpp index 96d132b74f82d0a18046d22fa83826b4163e60f4..da0809d4d6de45ad0ead8a29b199b60e3ed5f970 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_1x8.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_1x8.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -39,7 +39,6 @@ void a64_transpose_interleave_12_1x8(uint8_t *out, const uint8_t *in, size_t wid size_t out_stride = 12 * roundup(height, 8) * sizeof(uint8_t); __asm__ __volatile__( - "1:" // Main row loop: Head "mov x9, %x[in]\n" "add x28, x9, %x[in_stride]\n" @@ -332,4 +331,5 @@ void Transform<12, 8, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_2x2.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_2x2.hpp index 04af6fd71334902a7f14c96bde251190a27f7137..cef468e9cc71bbcb175f530c50ab3efe4f040688 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_2x2.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_2x2.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -236,7 +236,6 @@ void a64_transpose_interleave_12_2x2(uint16_t *out, const uint16_t *in, size_t w "bge 1b\n" "cbz %x[height], 20f\n" "10:" // Main loop skip - "11:" // Tail row loop: Head "mov x9, %x[in]\n" "mov x20, %x[width]\n" @@ -319,7 +318,6 @@ void a64_transpose_interleave_12_2x2(uint16_t *out, const uint16_t *in, size_t w "add %x[out], %x[out], #0x30\n" "bge 11b\n" "20:" // Done - : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [pad_row] "r" (pad_row), [width] "r" (width) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" @@ -341,4 +339,5 @@ void Transform<12, 2, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_2x4.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_2x4.hpp index e6ddc10e04e77f98e72b75cf6f44a878766265c1..4c02d0534d36acb4abdd01130cd292b4933264fa 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_2x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_2x4.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -276,7 +276,6 @@ void a64_transpose_interleave_12_2x4(uint16_t *out, const uint16_t *in, size_t w "bge 1b\n" "cbz %x[height], 20f\n" "10:" // Main loop skip - "11:" // Tail row loop: Head "mov x9, %x[in]\n" "add x28, x9, %x[in_stride]\n" @@ -420,7 +419,6 @@ void a64_transpose_interleave_12_2x4(uint16_t *out, const uint16_t *in, size_t w "add %x[out], %x[out], #0x60\n" "bge 11b\n" "20:" // Done - : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [pad_row] "r" (pad_row), [width] "r" (width) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" @@ -442,4 +440,5 @@ void Transform<12, 4, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_2x4_fp32bf16.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_2x4_fp32bf16.hpp index e487d4d839aece6b031edeee5fc7f248000b4aaf..2a3208d18d89afeb7b609450dad1d36a8984ab01 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_2x4_fp32bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_2x4_fp32bf16.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -710,7 +710,6 @@ void a64_transpose_interleave_12_2x4_fp32bf16(bfloat16 *out, const float *in, si "add %x[out], %x[out], #0x60\n" "bge 11b\n" "20:" // Done - : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [pad_row] "r" (pad_row), [width] "r" (width) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" @@ -731,4 +730,5 @@ void Transform<12, 4, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_s8s16.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_s8s16.hpp index 7938325fa403ed75fec0ba4068545b55375a9794..4d9d5e7f43581a8387001f1b1a0453a614efd06e 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_s8s16.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_s8s16.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -182,7 +182,6 @@ void a64_transpose_interleave_12_s8s16(int16_t *out, const int8_t *in, size_t wi "bge 1b\n" "cbz %x[height], 20f\n" "10:" // Main loop skip - "11:" // Tail row loop: Head "mov x20, %x[width]\n" "mov x25, %x[in]\n" @@ -251,7 +250,6 @@ void a64_transpose_interleave_12_s8s16(int16_t *out, const int8_t *in, size_t wi "add %x[out], %x[out], #0x18\n" "bge 11b\n" "20:" // Done - : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [width] "r" (width) : "cc", "memory", "v0", "v1", "v2", "v3", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25" @@ -272,4 +270,5 @@ void Transform<12, 1, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_u8u16.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_u8u16.hpp index 4c66fb2c2fb54f7b96c996a3c9222187206ba3e3..b0cd7e4ef7d46dab5161db94a8c5889b2e11d929 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_u8u16.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12_u8u16.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -182,7 +182,6 @@ void a64_transpose_interleave_12_u8u16(uint16_t *out, const uint8_t *in, size_t "bge 1b\n" "cbz %x[height], 20f\n" "10:" // Main loop skip - "11:" // Tail row loop: Head "mov x20, %x[width]\n" "mov x25, %x[in]\n" @@ -251,7 +250,6 @@ void a64_transpose_interleave_12_u8u16(uint16_t *out, const uint8_t *in, size_t "add %x[out], %x[out], #0x18\n" "bge 11b\n" "20:" // Done - : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [width] "r" (width) : "cc", "memory", "v0", "v1", "v2", "v3", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25" @@ -272,4 +270,5 @@ void Transform<12, 1, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_16.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_16.hpp index f06c167361dae262c4c697f0ac67fa8e04f614a2..0399f8becc60226b84f4198321483cf5c3a621aa 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_16.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_16.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -137,4 +137,5 @@ void Transform<4, 1, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_16_1x4.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_16_1x4.hpp index e0ccb368c29323c7f1bcbc54359dec411644d73e..f3a1dde73ff60c500581e9a414a16122325b3a84 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_16_1x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_16_1x4.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -327,4 +327,5 @@ void Transform<16, 4, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_16_1x8.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_16_1x8.hpp index fa45f4fd4de36b8a28947c94397f6a5f7df8f7fe..7c7e91e666e648456aba25612d2fe620ccbe9945 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_16_1x8.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_16_1x8.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -39,7 +39,6 @@ void a64_transpose_interleave_16_1x8(uint8_t *out, const uint8_t *in, size_t wid size_t out_stride = 16 * roundup(height, 8) * sizeof(uint8_t); __asm__ __volatile__( - "1:" // Main row loop: Head "mov x9, %x[in]\n" "add x28, x9, %x[in_stride]\n" @@ -288,4 +287,5 @@ void Transform<16, 8, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_16_2x2.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_16_2x2.hpp index 06efa9781eea3cc4942635ba964bf44bf4ce677a..b4515cbfd46c6b807d16b910d579df41a01e8f20 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_16_2x2.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_16_2x2.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -163,7 +163,6 @@ void a64_transpose_interleave_16_2x2(uint16_t *out, const uint16_t *in, size_t w "bge 1b\n" "cbz %x[height], 16f\n" "8:" // Main loop skip - "9:" // Tail row loop: Head "mov x9, %x[in]\n" "mov x20, %x[width]\n" @@ -221,7 +220,6 @@ void a64_transpose_interleave_16_2x2(uint16_t *out, const uint16_t *in, size_t w "add %x[out], %x[out], #0x40\n" "bge 9b\n" "16:" // Done - : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [pad_row] "r" (pad_row), [width] "r" (width) : "cc", "memory", "v0", "v1", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" @@ -243,4 +241,5 @@ void Transform<16, 2, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_16_2x4.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_16_2x4.hpp index dafa53eec3cd807223e0fd25abd16d2dc6a8657d..ac6746724072e9b8565cdb5c8edabe7343d0c855 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_16_2x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_16_2x4.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -320,7 +320,6 @@ void a64_transpose_interleave_16_2x4(uint16_t *out, const uint16_t *in, size_t w "bge 1b\n" "cbz %x[height], 20f\n" "10:" // Main loop skip - "11:" // Tail row loop: Head "mov x9, %x[in]\n" "add x28, x9, %x[in_stride]\n" @@ -486,7 +485,6 @@ void a64_transpose_interleave_16_2x4(uint16_t *out, const uint16_t *in, size_t w "add %x[out], %x[out], #0x80\n" "bge 11b\n" "20:" // Done - : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [pad_row] "r" (pad_row), [width] "r" (width) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" @@ -508,4 +506,5 @@ void Transform<16, 4, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_16_2x4_fp32bf16.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_16_2x4_fp32bf16.hpp index e012d0920f9f7f0e760b686d86947fbb44f2db38..b9fe8b126a854b9b66ec9f80729146ecca2b8fec 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_16_2x4_fp32bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_16_2x4_fp32bf16.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -281,7 +281,6 @@ void a64_transpose_interleave_16_2x4_fp32bf16(bfloat16 *out, const float *in, si "bge 1b\n" "cbz %x[height], 16f\n" "8:" // Main loop skip - "9:" // Tail row loop: Head "mov x9, %x[in]\n" "add x28, x9, %x[in_stride]\n" @@ -423,7 +422,6 @@ void a64_transpose_interleave_16_2x4_fp32bf16(bfloat16 *out, const float *in, si "add %x[out], %x[out], #0x80\n" "bge 9b\n" "16:" // Done - : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [pad_row] "r" (pad_row), [width] "r" (width) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" @@ -444,4 +442,5 @@ void Transform<16, 4, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_24.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_24.hpp index 20f9d39f4eb3bc7348d9861969763dc9b397edd4..46211ad4e4d811120ffe17dc65ac72ee1e86e996 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_24.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_24.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -158,7 +158,6 @@ void a64_transpose_interleave_24(uint16_t *out, const uint16_t *in, size_t width "bge 1b\n" "cbz %x[height], 20f\n" "10:" // Main loop skip - "11:" // Tail row loop: Head "mov x20, %x[width]\n" "mov x25, %x[in]\n" @@ -268,4 +267,5 @@ void Transform<12, 1, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_24_2x4_fp32bf16.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_24_2x4_fp32bf16.hpp index 22d68acd51b8a2b686382b2546d41e604541aa30..1cb7bc4445f12f0e4489441d4b9f5b81615717d8 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_24_2x4_fp32bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_24_2x4_fp32bf16.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -762,7 +762,6 @@ void a64_transpose_interleave_24_2x4_fp32bf16(bfloat16 *out, const float *in, si "add %x[out], %x[out], #0xc0\n" "bge 11b\n" "20:" // Done - : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [pad_row] "r" (pad_row), [width] "r" (width) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" @@ -783,4 +782,5 @@ void Transform<24, 4, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_24_bf16fp32.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_24_bf16fp32.hpp index 799a9cd91d156135240f73a19b8d3cf9222461e4..dcaf69d2a895a1498a22451904fdced6067923f5 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_24_bf16fp32.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_24_bf16fp32.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -198,7 +198,6 @@ void a64_transpose_interleave_24_bf16fp32(float *out, const bfloat16 *in, size_t "bge 1b\n" "cbz %x[height], 20f\n" "10:" // Main loop skip - "11:" // Tail row loop: Head "mov x20, %x[width]\n" "mov x25, %x[in]\n" @@ -271,7 +270,6 @@ void a64_transpose_interleave_24_bf16fp32(float *out, const bfloat16 *in, size_t "add %x[out], %x[out], #0x30\n" "bge 11b\n" "20:" // Done - : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [width] "r" (width) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25" @@ -292,4 +290,5 @@ void Transform<12, 1, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_24_fp16fp32.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_24_fp16fp32.hpp index 621c5f99ff116b93138e712bc047cc30eedd7532..966b75664e146af26c5befa0937654731ab95c0a 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_24_fp16fp32.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_24_fp16fp32.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -270,7 +270,6 @@ void a64_transpose_interleave_24_fp16fp32(float *out, const __fp16 *in, size_t w "add %x[out], %x[out], #0x30\n" "bge 11b\n" "20:" // Done - : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [width] "r" (width) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25" @@ -291,4 +290,5 @@ void Transform<12, 1, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_32_1x4.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_32_1x4.hpp index 5cd7bd0512ea7b25d256dde9ccc9f023f5055ef1..4a226750284de536a8423f7c61c4edd9bfbf16bd 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_32_1x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_32_1x4.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -503,4 +503,5 @@ void Transform<32, 4, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_32_2x2.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_32_2x2.hpp index 706d7cd3591029c91a4a539ec868f0aa38a94178..237536697c730a1dd19fb7f94970a6e15c3b31a5 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_32_2x2.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_32_2x2.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -280,7 +280,6 @@ void a64_transpose_interleave_32_2x2(uint16_t *out, const uint16_t *in, size_t w "bge 1b\n" "cbz %x[height], 24f\n" "12:" // Main loop skip - "13:" // Tail row loop: Head "mov x25, %x[in]\n" "mov x20, %x[width]\n" @@ -427,7 +426,6 @@ void a64_transpose_interleave_32_2x2(uint16_t *out, const uint16_t *in, size_t w "add %x[out], %x[out], #0x80\n" "bge 13b\n" "24:" // Done - : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [pad_row] "r" (pad_row), [width] "r" (width) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23", "x24", "x25" @@ -449,4 +447,5 @@ void Transform<32, 2, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_48.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_48.hpp index b4827525cd14861a6eeb950060aa3c5641984704..f35752d5a808a15bc683d27b4521038f7b0ee34f 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_48.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_48.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -137,7 +137,6 @@ void a64_transpose_interleave_48(uint16_t *out, const uint16_t *in, size_t width "bge 1b\n" "cbz %x[height], 20f\n" "10:" // Main loop skip - "11:" // Tail row loop: Head "mov x20, %x[width]\n" "mov x25, %x[in]\n" @@ -241,4 +240,5 @@ void Transform<6, 1, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_4_1x16.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_4_1x16.hpp index e1ab14e594c472493b328aa2a1714f41ec9e8e34..6ef02ac0445ad5b6d3e56d0716890e9e00340723 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_4_1x16.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_4_1x16.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -39,7 +39,6 @@ void a64_transpose_interleave_4_1x16(uint8_t *out, const uint8_t *in, size_t wid size_t out_stride = 4 * roundup(height, 16) * sizeof(uint8_t); __asm__ __volatile__( - "1:" // Main row loop: Head "mov x17, %x[in]\n" "add x16, x17, %x[in_stride]\n" @@ -316,4 +315,5 @@ void Transform<4, 16, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_4_1x4.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_4_1x4.hpp index 8adc69e8b3fe922f46102aa4208cb79d56e64b61..5667820865d2a27f100e1fd6a995d46d336b8ef7 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_4_1x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_4_1x4.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -333,4 +333,5 @@ void Transform<4, 4, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_64.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_64.hpp index 07602bdc8d24919e65c3e607f3d3b4aefda5fff7..328274a4887bfe5d98513281e0bfb16deb667fb5 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_64.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_64.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -145,7 +145,6 @@ void a64_transpose_interleave_64(uint16_t *out, const uint16_t *in, size_t width "bge 1b\n" "cbz %x[height], 20f\n" "10:" // Main loop skip - "11:" // Tail row loop: Head "mov x20, %x[width]\n" "mov x25, %x[in]\n" @@ -251,4 +250,5 @@ void Transform<32, 1, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_96.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_96.hpp index a048fbb1098b35a5201566144fd9e58dcdbf0da1..feb469ab0ebf2a3ec01689383e22bea43bbbe72f 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_96.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_96.hpp @@ -24,7 +24,7 @@ #pragma once -#ifdef __aarch64__ +#if defined(__aarch64__) namespace { @@ -177,7 +177,6 @@ void a64_transpose_interleave_96(uint32_t *out, const uint32_t *in, size_t width "bge 1b\n" "cbz %x[height], 20f\n" "10:" // Main loop skip - "11:" // Tail row loop: Head "mov x20, %x[width]\n" "mov x25, %x[in]\n" @@ -265,4 +264,5 @@ void Transform<24, 1, true, VLType::None>( ); } -#endif + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL.hpp index 01921c5ad9edcb50db37781821acca5295fc6963..a4d480c405589f1a23047479bd55fce6709b2c36 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SME) namespace { @@ -140,4 +139,5 @@ void Transform<16, 1, true, VLType::SME>( ); } -#endif // __ARM_FEATURE_SVE + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL_1x4.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL_1x4.hpp index 6b9b471fdcadba74b8c02ae50c0d7884360ce1e6..552abfc1c67752cc4036e4eaa4d0b8113e43d147 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL_1x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL_1x4.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SME) namespace { @@ -178,4 +177,5 @@ void Transform<16, 4, true, VLType::SME>( ); } -#endif // __ARM_FEATURE_SVE + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL_2x2.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL_2x2.hpp index 96128cf9c2a4e1b458753d359c49435a1d2ec09e..9c6f5c83a1cc00cff29bc6e4a2655a9b3e743416 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL_2x2.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL_2x2.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SME) namespace { @@ -150,4 +149,5 @@ void Transform<16, 2, true, VLType::SME>( ); } -#endif // __ARM_FEATURE_SVE + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL_2x2_fp32bf16.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL_2x2_fp32bf16.hpp index 080db1c5c1f69945eece53a4ce7e301303a7fe07..2756327815349541b4bc33a5ab42b717976f5338 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL_2x2_fp32bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_16VL_2x2_fp32bf16.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SME) namespace { @@ -197,4 +196,5 @@ void Transform<16, 2, true, VLType::SME>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_1VL.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_1VL.hpp index 7e496095f46b7c37d83618de03b4eeee99cf35cc..a6ddb8fec09046feb423ab2b20a908c4526beec0 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_1VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_1VL.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SME) namespace { @@ -210,4 +209,5 @@ void Transform<1, 1, true, VLType::SME>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_1VL_1x4.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_1VL_1x4.hpp index 45d3c0729eb2ac4a604c720a2de0d7e6f85a105f..399a52e233703fdd547cba73c180f67456d145f0 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_1VL_1x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_1VL_1x4.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SME) namespace { @@ -146,4 +145,5 @@ void Transform<1, 4, true, VLType::SME>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_1VL_2x2.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_1VL_2x2.hpp index 7120d1d33e3eb573446f69d902f6a6f82b1d2fad..6318e29a7974ebceb81e29afab5bc9720b5e7fcb 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_1VL_2x2.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_1VL_2x2.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SME) namespace { @@ -206,4 +205,4 @@ void Transform<1, 2, true, VLType::SME>( ); } -#endif +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_1VL_2x2_fp32bf16.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_1VL_2x2_fp32bf16.hpp index 72e7b0c99a9b65f693e8d2d4d2ab4b57689aa54f..b90063028d7ed85c447acfaf39838dd1c6f19925 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_1VL_2x2_fp32bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_1VL_2x2_fp32bf16.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SME) namespace { @@ -219,4 +218,5 @@ void Transform<1, 2, true, VLType::SME>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_2VL.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_2VL.hpp index a057fd514e496ac16958879b73adddbd6b673107..f827197ab70ebb98abedb9f3129a82c45b52c720 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_2VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_2VL.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SME) namespace { @@ -222,4 +221,5 @@ void Transform<2, 1, true, VLType::SME>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_2VL_1x4.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_2VL_1x4.hpp index 9eb4075677b0637049ab2c8ad2486efa6f17921e..c471d66e17d333139f0ecd8b57c0579ac46bd8e9 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_2VL_1x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_2VL_1x4.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SME) namespace { @@ -146,4 +145,5 @@ void Transform<2, 4, true, VLType::SME>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_2VL_2x2.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_2VL_2x2.hpp index 3fc3920500f4b7e98572a0e0545fadf24d6f7bf8..5f967fa6155c0332735683a08d1143d7f63de81d 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_2VL_2x2.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_2VL_2x2.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SME) namespace { @@ -208,4 +207,5 @@ void Transform<2, 2, true, VLType::SME>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_2VL_2x2_fp32bf16.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_2VL_2x2_fp32bf16.hpp index 9d402a2d58621f7d9c0dfaefba9c904b7ae2973b..f22b833821c76fcb6ef0cd44c642659794eb1991 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_2VL_2x2_fp32bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_2VL_2x2_fp32bf16.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SME) namespace { @@ -236,4 +235,5 @@ void Transform<2, 2, true, VLType::SME>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_4VL.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_4VL.hpp index 362bebbea0c60f7845bfba0093bd693428154c39..14636e321840d7c4e66e0ac00a79e77cdde3b43a 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_4VL.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SME) namespace { @@ -185,4 +184,5 @@ void Transform<4, 1, true, VLType::SME>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_4VL_1x4.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_4VL_1x4.hpp index cbcc0b4c8ba08bbba4bfd7b8eb09c4c950ccd1ed..2d46a481f31ba82aaeb417ea5f20d26364dcec0c 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_4VL_1x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_4VL_1x4.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SME) namespace { @@ -121,4 +120,5 @@ void Transform<4, 4, true, VLType::SME>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_4VL_2x2.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_4VL_2x2.hpp index 9b28578217561f70882300357df609a206361d76..002a12479a20f6edcccc8753c382bdba8bfb4e5f 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_4VL_2x2.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_4VL_2x2.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SME) namespace { @@ -168,4 +167,5 @@ void Transform<4, 2, true, VLType::SME>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_4VL_2x2_fp32bf16.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_4VL_2x2_fp32bf16.hpp index 887307001976a7b33f1c81186ba25c2d1c41dc48..2a43f34f71cf765c47d97c47f24710430afc7893 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_4VL_2x2_fp32bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_4VL_2x2_fp32bf16.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SME) namespace { @@ -186,4 +185,5 @@ void Transform<4, 2, true, VLType::SME>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_8VL.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_8VL.hpp new file mode 100644 index 0000000000000000000000000000000000000000..be9ad666a980c73365b181dee472a3b0be776536 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_8VL.hpp @@ -0,0 +1,208 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#pragma once + +#if defined(ARM_COMPUTE_ENABLE_SME) + +namespace { + +void sme_transpose_interleave_8VL(uint16_t *out, const uint16_t *in, size_t width, size_t in_stride, size_t height) +{ + size_t out_stride = 8 * height * sme::get_vector_length(); + + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "cmp %x[height], #0x2\n" + "ptrue p7.b\n" + "blt 4f\n" + "1:" // Main row loop: Head + "mov x25, %x[in]\n" + "add x24, x25, %x[in_stride]\n" + "add %x[in], x24, %x[in_stride]\n" + "mov x23, %x[out]\n" + "sub %x[height], %x[height], #0x2\n" + "mov x22, %x[width]\n" + "2:" // Main row loop: Column loop + "mov x21, x22\n" + "whilelt p0.h, XZR, x21\n" + "ld1h { z31.h }, p0/Z, [x25]\n" + "dech x21\n" + "whilelt p6.h, XZR, x21\n" + "ld1h { z30.h }, p6/Z, [x25, #1, MUL VL]\n" + "dech x21\n" + "whilelt p5.h, XZR, x21\n" + "ld1h { z29.h }, p5/Z, [x25, #2, MUL VL]\n" + "dech x21\n" + "whilelt p4.h, XZR, x21\n" + "ld1h { z28.h }, p4/Z, [x25, #3, MUL VL]\n" + "dech x21\n" + "whilelt p3.h, XZR, x21\n" + "ld1h { z27.h }, p3/Z, [x25, #4, MUL VL]\n" + "dech x21\n" + "whilelt p2.h, XZR, x21\n" + "ld1h { z26.h }, p2/Z, [x25, #5, MUL VL]\n" + "dech x21\n" + "whilelt p1.h, XZR, x21\n" + "ld1h { z25.h }, p1/Z, [x25, #6, MUL VL]\n" + "dech x21\n" + "mov x20, x23\n" + "ld1h { z24.h }, p0/Z, [x24]\n" + "whilelt p0.h, XZR, x21\n" + "dech x22, ALL, MUL #8\n" + "ld1h { z23.h }, p0/Z, [x25, #7, MUL VL]\n" + "ld1h { z22.h }, p6/Z, [x24, #1, MUL VL]\n" + "cmp x22, #0x0\n" + "addvl x25, x25, #8\n" + "ld1h { z21.h }, p5/Z, [x24, #2, MUL VL]\n" + "add x23, x23, %x[out_stride]\n" + "ld1h { z20.h }, p4/Z, [x24, #3, MUL VL]\n" + "ld1h { z19.h }, p3/Z, [x24, #4, MUL VL]\n" + "ld1h { z18.h }, p2/Z, [x24, #5, MUL VL]\n" + "ld1h { z17.h }, p1/Z, [x24, #6, MUL VL]\n" + "ld1h { z16.h }, p0/Z, [x24, #7, MUL VL]\n" + "st1h { z31.h }, p7, [x20]\n" + "addvl x24, x24, #8\n" + "st1h { z30.h }, p7, [x20, #1, MUL VL]\n" + "st1h { z29.h }, p7, [x20, #2, MUL VL]\n" + "st1h { z28.h }, p7, [x20, #3, MUL VL]\n" + "st1h { z27.h }, p7, [x20, #4, MUL VL]\n" + "st1h { z26.h }, p7, [x20, #5, MUL VL]\n" + "st1h { z25.h }, p7, [x20, #6, MUL VL]\n" + "st1h { z23.h }, p7, [x20, #7, MUL VL]\n" + "addvl x20, x20, #16\n" + "st1h { z24.h }, p7, [x20, #-8, MUL VL]\n" + "st1h { z22.h }, p7, [x20, #-7, MUL VL]\n" + "st1h { z21.h }, p7, [x20, #-6, MUL VL]\n" + "st1h { z20.h }, p7, [x20, #-5, MUL VL]\n" + "st1h { z19.h }, p7, [x20, #-4, MUL VL]\n" + "st1h { z18.h }, p7, [x20, #-3, MUL VL]\n" + "st1h { z17.h }, p7, [x20, #-2, MUL VL]\n" + "st1h { z16.h }, p7, [x20, #-1, MUL VL]\n" + "bgt 2b\n" + "3:" // Main row loop: Column loop skip + "cmp %x[height], #0x2\n" + "addvl %x[out], %x[out], #16\n" + "bge 1b\n" + "cbz %x[height], 8f\n" + "4:" // Main loop skip + "5:" // Tail row loop: Head + "mov x25, %x[in]\n" + "add %x[in], x25, %x[in_stride]\n" + "mov x23, %x[out]\n" + "sub %x[height], %x[height], #0x1\n" + "mov x21, %x[width]\n" + "6:" // Tail row loop: Column loop + "mov x20, x21\n" + "whilelt p0.h, XZR, x20\n" + "ld1h { z23.h }, p0/Z, [x25]\n" + "dech x20\n" + "whilelt p0.h, XZR, x20\n" + "ld1h { z22.h }, p0/Z, [x25, #1, MUL VL]\n" + "dech x20\n" + "whilelt p0.h, XZR, x20\n" + "ld1h { z21.h }, p0/Z, [x25, #2, MUL VL]\n" + "dech x20\n" + "whilelt p0.h, XZR, x20\n" + "ld1h { z20.h }, p0/Z, [x25, #3, MUL VL]\n" + "dech x20\n" + "whilelt p0.h, XZR, x20\n" + "ld1h { z19.h }, p0/Z, [x25, #4, MUL VL]\n" + "dech x20\n" + "whilelt p0.h, XZR, x20\n" + "ld1h { z18.h }, p0/Z, [x25, #5, MUL VL]\n" + "dech x20\n" + "whilelt p0.h, XZR, x20\n" + "ld1h { z17.h }, p0/Z, [x25, #6, MUL VL]\n" + "dech x20\n" + "dech x21, ALL, MUL #8\n" + "whilelt p0.h, XZR, x20\n" + "cmp x21, #0x0\n" + "ld1h { z16.h }, p0/Z, [x25, #7, MUL VL]\n" + "st1h { z23.h }, p7, [x23]\n" + "addvl x25, x25, #8\n" + "st1h { z22.h }, p7, [x23, #1, MUL VL]\n" + "st1h { z21.h }, p7, [x23, #2, MUL VL]\n" + "st1h { z20.h }, p7, [x23, #3, MUL VL]\n" + "st1h { z19.h }, p7, [x23, #4, MUL VL]\n" + "st1h { z18.h }, p7, [x23, #5, MUL VL]\n" + "st1h { z17.h }, p7, [x23, #6, MUL VL]\n" + "st1h { z16.h }, p7, [x23, #7, MUL VL]\n" + "add x23, x23, %x[out_stride]\n" + "bgt 6b\n" + "7:" // Tail row loop: Column loop skip + "cmp %x[height], #0x1\n" + "addvl %x[out], %x[out], #8\n" + "bge 5b\n" + "8:" // Done + ".inst 0xd503467f // SMSTOP\n" + : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) + : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x20", "x21", "x22", "x23", "x24", "x25", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +} // anonymous namespace + +template<> +void Transform<8, 1, true, VLType::SME>( + float *out, const float *in, int stride, int x0, int xmax, int k0, int kmax) +{ + sme_transpose_interleave_8VL( + reinterpret_cast(out), + reinterpret_cast(in + k0 * stride + x0), + (xmax-x0) * sizeof(float) / 2, + stride * sizeof(float), + (kmax-k0) + ); +} + +template<> +void Transform<8, 1, true, VLType::SME>( + bfloat16 *out, const bfloat16 *in, int stride, int x0, int xmax, int k0, int kmax) +{ + sme_transpose_interleave_8VL( + reinterpret_cast(out), + reinterpret_cast(in + k0 * stride + x0), + (xmax-x0) * sizeof(bfloat16) / 2, + stride * sizeof(bfloat16), + (kmax-k0) + ); +} + +template<> +void Transform<8, 1, true, VLType::SME>( + __fp16 *out, const __fp16 *in, int stride, int x0, int xmax, int k0, int kmax) +{ + sme_transpose_interleave_8VL( + reinterpret_cast(out), + reinterpret_cast(in + k0 * stride + x0), + (xmax-x0) * sizeof(__fp16) / 2, + stride * sizeof(__fp16), + (kmax-k0) + ); +} + + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_8VL_1x4.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_8VL_1x4.hpp new file mode 100644 index 0000000000000000000000000000000000000000..45d2e24258d9e1f3b62b2398dbccdd0158292e4e --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_8VL_1x4.hpp @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#pragma once + +#if defined(ARM_COMPUTE_ENABLE_SME) + +namespace { + +void sme_transpose_interleave_8VL_1x4(uint8_t *out, const uint8_t *in, size_t width, size_t in_stride, size_t height) +{ + uint8_t *pad_row = reinterpret_cast(alloca(width * sizeof(uint8_t))); + + if (height % 4) { + memset(pad_row, 0, width * sizeof(uint8_t)); + } + + size_t out_stride = 8 * roundup(height, 4) * sme::get_vector_length(); + + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "ptrue p2.b\n" + "1:" // Main row loop: Head + "mov x26, %x[in]\n" + "add x25, x26, %x[in_stride]\n" + "add x24, x25, %x[in_stride]\n" + "add x23, x24, %x[in_stride]\n" + "cmp %x[height], #0x3\n" + "add %x[in], x23, %x[in_stride]\n" + "csel x23, x23, %x[pad_row], GT\n" + "csel x24, x24, %x[pad_row], GE\n" + "cmp %x[height], #0x1\n" + "mov x22, %x[out]\n" + "csel x25, x25, %x[pad_row], GT\n" + "sub %x[height], %x[height], #0x4\n" + "mov x21, %x[width]\n" + "2:" // Main row loop: Column loop + "mov x20, x21\n" + "whilelt p1.b, XZR, x20\n" + "ld1b { z19.b }, p1/Z, [x26]\n" + "decb x20\n" + "whilelt p0.b, XZR, x20\n" + "ld1b { z17.b }, p0/Z, [x26, #1, MUL VL]\n" + "ld1b { z18.b }, p1/Z, [x25]\n" + "decw x21, ALL, MUL #8\n" + "cmp x21, #0x0\n" + "ld1b { z21.b }, p0/Z, [x25, #1, MUL VL]\n" + "addvl x26, x26, #2\n" + "addvl x25, x25, #2\n" + "ld1b { z16.b }, p1/Z, [x24]\n" + "zip1 z24.b, z19.b, z16.b\n" + "zip2 z20.b, z19.b, z16.b\n" + "ld1b { z16.b }, p0/Z, [x24, #1, MUL VL]\n" + "zip1 z23.b, z17.b, z16.b\n" + "zip2 z22.b, z17.b, z16.b\n" + "addvl x24, x24, #2\n" + "ld1b { z16.b }, p1/Z, [x23]\n" + "zip1 z17.b, z18.b, z16.b\n" + "zip2 z19.b, z18.b, z16.b\n" + "ld1b { z16.b }, p0/Z, [x23, #1, MUL VL]\n" + "zip1 z18.b, z21.b, z16.b\n" + "zip2 z21.b, z21.b, z16.b\n" + "addvl x23, x23, #2\n" + "zip1 z16.b, z24.b, z17.b\n" + "zip2 z17.b, z24.b, z17.b\n" + "st1b { z16.b }, p2, [x22]\n" + "zip1 z16.b, z20.b, z19.b\n" + "zip2 z20.b, z20.b, z19.b\n" + "st1b { z17.b }, p2, [x22, #1, MUL VL]\n" + "zip1 z19.b, z23.b, z18.b\n" + "zip2 z18.b, z23.b, z18.b\n" + "st1b { z16.b }, p2, [x22, #2, MUL VL]\n" + "zip1 z17.b, z22.b, z21.b\n" + "zip2 z16.b, z22.b, z21.b\n" + "st1b { z20.b }, p2, [x22, #3, MUL VL]\n" + "st1b { z19.b }, p2, [x22, #4, MUL VL]\n" + "st1b { z18.b }, p2, [x22, #5, MUL VL]\n" + "st1b { z17.b }, p2, [x22, #6, MUL VL]\n" + "st1b { z16.b }, p2, [x22, #7, MUL VL]\n" + "add x22, x22, %x[out_stride]\n" + "bgt 2b\n" + "3:" // Main row loop: Column loop skip + "cmp %x[height], #0x1\n" + "addvl %x[out], %x[out], #8\n" + "bge 1b\n" + ".inst 0xd503467f // SMSTOP\n" + : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) + : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [pad_row] "r" (pad_row), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +} // anonymous namespace + +template<> +void Transform<8, 4, true, VLType::SME>( + uint8_t *out, const uint8_t *in, int stride, int x0, int xmax, int k0, int kmax) +{ + sme_transpose_interleave_8VL_1x4( + reinterpret_cast(out), + reinterpret_cast(in + k0 * stride + x0), + (xmax-x0) * sizeof(uint8_t) / 1, + stride * sizeof(uint8_t), + (kmax-k0) + ); +} + +template<> +void Transform<8, 4, true, VLType::SME>( + int8_t *out, const int8_t *in, int stride, int x0, int xmax, int k0, int kmax) +{ + sme_transpose_interleave_8VL_1x4( + reinterpret_cast(out), + reinterpret_cast(in + k0 * stride + x0), + (xmax-x0) * sizeof(int8_t) / 1, + stride * sizeof(int8_t), + (kmax-k0) + ); +} + + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_8VL_2x2.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_8VL_2x2.hpp new file mode 100644 index 0000000000000000000000000000000000000000..ec7c415e273c5b5c9f0778f9bbe47c3f2860d5a6 --- /dev/null +++ b/src/core/NEON/kernels/arm_gemm/transforms/sme_transpose_interleave_8VL_2x2.hpp @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#pragma once + +#if defined(ARM_COMPUTE_ENABLE_SME) + +namespace { + +void sme_transpose_interleave_8VL_2x2(uint16_t *out, const uint16_t *in, size_t width, size_t in_stride, size_t height) +{ + uint16_t *pad_row = reinterpret_cast(alloca(width * sizeof(uint16_t))); + + if (height % 2) { + memset(pad_row, 0, width * sizeof(uint16_t)); + } + + size_t out_stride = 8 * roundup(height, 2) * sme::get_vector_length(); + + __asm__ __volatile__( + ".inst 0xd503477f // SMSTART ZA\n" + "ptrue p4.b\n" + "1:" // Main row loop: Head + "mov x24, %x[in]\n" + "add x23, x24, %x[in_stride]\n" + "cmp %x[height], #0x1\n" + "add %x[in], x23, %x[in_stride]\n" + "mov x22, %x[out]\n" + "csel x23, x23, %x[pad_row], GT\n" + "sub %x[height], %x[height], #0x2\n" + "mov x21, %x[width]\n" + "2:" // Main row loop: Column loop + "mov x20, x21\n" + "whilelt p3.h, XZR, x20\n" + "ld1h { z20.h }, p3/Z, [x24]\n" + "dech x20\n" + "whilelt p2.h, XZR, x20\n" + "ld1h { z19.h }, p2/Z, [x24, #1, MUL VL]\n" + "dech x20\n" + "whilelt p1.h, XZR, x20\n" + "ld1h { z18.h }, p1/Z, [x24, #2, MUL VL]\n" + "dech x20\n" + "whilelt p0.h, XZR, x20\n" + "ld1h { z24.h }, p0/Z, [x24, #3, MUL VL]\n" + "ld1h { z17.h }, p3/Z, [x23]\n" + "decw x21, ALL, MUL #8\n" + "cmp x21, #0x0\n" + "zip1 z23.h, z20.h, z17.h\n" + "ld1h { z16.h }, p2/Z, [x23, #1, MUL VL]\n" + "addvl x24, x24, #4\n" + "zip2 z22.h, z20.h, z17.h\n" + "zip1 z21.h, z19.h, z16.h\n" + "ld1h { z17.h }, p1/Z, [x23, #2, MUL VL]\n" + "zip2 z20.h, z19.h, z16.h\n" + "zip1 z19.h, z18.h, z17.h\n" + "ld1h { z16.h }, p0/Z, [x23, #3, MUL VL]\n" + "addvl x23, x23, #4\n" + "zip2 z18.h, z18.h, z17.h\n" + "zip1 z17.h, z24.h, z16.h\n" + "zip2 z16.h, z24.h, z16.h\n" + "st1h { z23.h }, p4, [x22]\n" + "st1h { z22.h }, p4, [x22, #1, MUL VL]\n" + "st1h { z21.h }, p4, [x22, #2, MUL VL]\n" + "st1h { z20.h }, p4, [x22, #3, MUL VL]\n" + "st1h { z19.h }, p4, [x22, #4, MUL VL]\n" + "st1h { z18.h }, p4, [x22, #5, MUL VL]\n" + "st1h { z17.h }, p4, [x22, #6, MUL VL]\n" + "st1h { z16.h }, p4, [x22, #7, MUL VL]\n" + "add x22, x22, %x[out_stride]\n" + "bgt 2b\n" + "3:" // Main row loop: Column loop skip + "cmp %x[height], #0x1\n" + "addvl %x[out], %x[out], #8\n" + "bge 1b\n" + ".inst 0xd503467f // SMSTOP\n" + : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) + : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [pad_row] "r" (pad_row), [width] "r" (width) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x20", "x21", "x22", "x23", "x24", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +} // anonymous namespace + +template<> +void Transform<8, 2, true, VLType::SME>( + bfloat16 *out, const bfloat16 *in, int stride, int x0, int xmax, int k0, int kmax) +{ + sme_transpose_interleave_8VL_2x2( + reinterpret_cast(out), + reinterpret_cast(in + k0 * stride + x0), + (xmax-x0) * sizeof(bfloat16) / 2, + stride * sizeof(bfloat16), + (kmax-k0) + ); +} + +template<> +void Transform<8, 2, true, VLType::SME>( + __fp16 *out, const __fp16 *in, int stride, int x0, int xmax, int k0, int kmax) +{ + sme_transpose_interleave_8VL_2x2( + reinterpret_cast(out), + reinterpret_cast(in + k0 * stride + x0), + (xmax-x0) * sizeof(__fp16) / 2, + stride * sizeof(__fp16), + (kmax-k0) + ); +} + + +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_12VL_2x4_fp32bf16.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_12VL_2x4_fp32bf16.hpp index 847718992a148f450153dc1813fc5615fce67bf0..f627fe575f5cb07883d8521c33428e3ccfac33fe 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_12VL_2x4_fp32bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_12VL_2x4_fp32bf16.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace { @@ -373,4 +372,5 @@ void Transform<12, 4, true, VLType::SVE>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_1VL.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_1VL.hpp index 74fce4ddf960c32e36020088e693f7997a711b4c..b33c4f6c2d97450e382946931dd450cb9e5a9913 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_1VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_1VL.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace { @@ -101,7 +100,6 @@ void sve_transpose_interleave_1VL(uint32_t *out, const uint32_t *in, size_t widt "bge 1b\n" "cbz %x[height], 12f\n" "6:" // Main loop skip - "7:" // Tail row loop: Head "mov x21, %x[width]\n" "cntw x20, ALL, MUL #2\n" @@ -138,7 +136,6 @@ void sve_transpose_interleave_1VL(uint32_t *out, const uint32_t *in, size_t widt "addvl %x[out], %x[out], #1\n" "bge 7b\n" "12:" // Done - : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [width] "r" (width) : "cc", "memory", "p0", "p1", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23" @@ -160,4 +157,5 @@ void Transform<1, 1, true, VLType::SVE>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_1VL_1x4.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_1VL_1x4.hpp index a034be5e748c6da3541642342b197e1c86ff8ff9..e468787815aeb2a7ca43f6deb42dc513a6c64b86 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_1VL_1x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_1VL_1x4.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace { @@ -305,4 +304,5 @@ void Transform<1, 4, true, VLType::SVE>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_3VL.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_3VL.hpp index 82d41840612dcecb3fd46557119ebe936151cd1b..546800fa6915f459be0e3b6c98327d8c89d31e81 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_3VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_3VL.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace { @@ -93,7 +92,6 @@ void sve_transpose_interleave_3VL(uint16_t *out, const uint16_t *in, size_t widt "bge 1b\n" "cbz %x[height], 8f\n" "4:" // Main loop skip - "5:" // Tail row loop: Head "mov x26, %x[in]\n" "add %x[in], x26, %x[in_stride]\n" @@ -123,7 +121,6 @@ void sve_transpose_interleave_3VL(uint16_t *out, const uint16_t *in, size_t widt "addvl %x[out], %x[out], #3\n" "bge 5b\n" "8:" // Done - : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [width] "r" (width) : "cc", "memory", "p0", "p1", "p2", "p3", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27" @@ -171,4 +168,5 @@ void Transform<3, 1, true, VLType::SVE>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_3VL_1x4.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_3VL_1x4.hpp index ec7095db7b6c51946e51e98476cd47511251de6e..a44141c109138b4c9cfbbb6852714cf6cc900865 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_3VL_1x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_3VL_1x4.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace { @@ -329,7 +328,6 @@ void sve_transpose_interleave_3VL_1x4(uint8_t *out, const uint8_t *in, size_t wi "addvl %x[out], %x[out], #3\n" "bge 7b\n" "12:" // Done - : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [pad_row] "r" (pad_row), [width] "r" (width) : "cc", "memory", "p0", "p1", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -364,4 +362,5 @@ void Transform<3, 4, true, VLType::SVE>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_3VL_2x2.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_3VL_2x2.hpp index 3d14383a6490fee9d62876cfea90bb214343c11f..36a15a16b38ce3e7e57ba3b277de15bee5ac4176 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_3VL_2x2.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_3VL_2x2.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace { @@ -292,7 +291,6 @@ void sve_transpose_interleave_3VL_2x2(uint16_t *out, const uint16_t *in, size_t "addvl %x[out], %x[out], #3\n" "bge 7b\n" "12:" // Done - : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [pad_row] "r" (pad_row), [width] "r" (width) : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -314,4 +312,5 @@ void Transform<3, 2, true, VLType::SVE>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_4VL.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_4VL.hpp index a39235187fa7de9565a8d8253c4d2d72f8ffb8dc..e661e2698adbd71954fbb1fd9bd6e3ffe264ab9e 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_4VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_4VL.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace { @@ -103,7 +102,6 @@ void sve_transpose_interleave_4VL(uint16_t *out, const uint16_t *in, size_t widt "bge 1b\n" "cbz %x[height], 8f\n" "4:" // Main loop skip - "5:" // Tail row loop: Head "mov x26, %x[in]\n" "add %x[in], x26, %x[in_stride]\n" @@ -137,7 +135,6 @@ void sve_transpose_interleave_4VL(uint16_t *out, const uint16_t *in, size_t widt "addvl %x[out], %x[out], #4\n" "bge 5b\n" "8:" // Done - : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [width] "r" (width) : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -185,4 +182,5 @@ void Transform<4, 1, true, VLType::SVE>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_4VL_1x4.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_4VL_1x4.hpp index e3489398d4b6c787d6c4fda6d4aeb842780dfe4a..03a78f72f18a77b3339716715e97dfc23e7ab004 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_4VL_1x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_4VL_1x4.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace { @@ -317,4 +316,5 @@ void Transform<4, 4, true, VLType::SVE>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_4VL_2x2.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_4VL_2x2.hpp index 9505dc5e6d962a946c5e00812b62de2dfb3d9d90..b196799cfe2daccbf7e94c12bbf5c367cd6a6038 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_4VL_2x2.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_4VL_2x2.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace { @@ -248,7 +247,6 @@ void sve_transpose_interleave_4VL_2x2(uint16_t *out, const uint16_t *in, size_t "bge 1b\n" "cbz %x[height], 12f\n" "6:" // Main loop skip - "7:" // Tail row loop: Head "mov x12, %x[in]\n" "mov x21, %x[width]\n" @@ -323,7 +321,6 @@ void sve_transpose_interleave_4VL_2x2(uint16_t *out, const uint16_t *in, size_t "addvl %x[out], %x[out], #4\n" "bge 7b\n" "12:" // Done - : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [pad_row] "r" (pad_row), [width] "r" (width) : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -345,4 +342,5 @@ void Transform<4, 2, true, VLType::SVE>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_6VL_1x8.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_6VL_1x8.hpp index 982c0545ed202be274dca8f9a01d2adbae1a3758..68fe2d0cbeb73f2c64823244e76ac60eff0021c9 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_6VL_1x8.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_6VL_1x8.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace { @@ -292,4 +291,5 @@ void Transform<6, 8, true, VLType::SVE>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_6VL_2x4.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_6VL_2x4.hpp index 2b5741a49cfc66472ce02357060228e920213698..910fc6cb02d18b557925e83276c24d003cefc7f2 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_6VL_2x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_6VL_2x4.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace { @@ -260,7 +259,6 @@ void sve_transpose_interleave_6VL_2x4(uint16_t *out, const uint16_t *in, size_t "bge 1b\n" "cbz %x[height], 12f\n" "6:" // Main loop skip - "7:" // Tail row loop: Head "mov x12, %x[in]\n" "add x11, x12, %x[in_stride]\n" @@ -386,7 +384,6 @@ void sve_transpose_interleave_6VL_2x4(uint16_t *out, const uint16_t *in, size_t "addvl %x[out], %x[out], #6\n" "bge 7b\n" "12:" // Done - : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [pad_row] "r" (pad_row), [width] "r" (width) : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -408,4 +405,5 @@ void Transform<6, 4, true, VLType::SVE>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_6VL_2x4_fp32bf16.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_6VL_2x4_fp32bf16.hpp index 146da3386902370df7205a92fdacd01a2422051f..f0f10d2f438394d81e8f5a79c7b9c6bfd6b4731f 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_6VL_2x4_fp32bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_6VL_2x4_fp32bf16.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace { @@ -235,4 +234,5 @@ void Transform<6, 4, true, VLType::SVE>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_6VL_4x2.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_6VL_4x2.hpp index f6fc5e8b84b7cd9b41b9f193a2ee2ab7423e957b..c638eaacded6105d6b138da21aecdff01f6a5271 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_6VL_4x2.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_6VL_4x2.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace { @@ -296,7 +295,6 @@ void sve_transpose_interleave_6VL_4x2(uint32_t *out, const uint32_t *in, size_t "addvl %x[out], %x[out], #6\n" "bge 7b\n" "12:" // Done - : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [pad_row] "r" (pad_row), [width] "r" (width) : "cc", "memory", "p0", "p1", "p2", "p3", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -318,4 +316,5 @@ void Transform<6, 2, true, VLType::SVE>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_8VL.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_8VL.hpp index 07147acd8e65ea8d4c84d587b2691f33245f8b77..0526bd05961f1064734608cb70840357182479f1 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_8VL.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_8VL.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace { @@ -281,7 +280,6 @@ void sve_transpose_interleave_8VL(uint32_t *out, const uint32_t *in, size_t widt "addvl %x[out], %x[out], #8\n" "bge 7b\n" "12:" // Done - : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [width] "r" (width) : "cc", "memory", "p0", "p1", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -303,4 +301,5 @@ void Transform<8, 1, true, VLType::SVE>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_8VL_1x4.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_8VL_1x4.hpp index 3ba50fee605fb14fa9e9d494f3148cf11aea2c41..98f0770d7711b858120ef9715e0fc86d8f2136bd 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_8VL_1x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_8VL_1x4.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace { @@ -283,4 +282,5 @@ void Transform<8, 4, true, VLType::SVE>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_8VL_1x8.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_8VL_1x8.hpp index 6b5ca38ab12839a26f4747c718df543b91d2d318..3fa529214396a89c3129e1da1fbf517eaf61d865 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_8VL_1x8.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_8VL_1x8.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace { @@ -256,4 +255,5 @@ void Transform<8, 8, true, VLType::SVE>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_8VL_2x2.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_8VL_2x2.hpp index 237e9b684f038f128593ed61ef6cc4cea6e0a890..02977ecf1e557d19458be3fdaddc8b213e3d7762 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_8VL_2x2.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_8VL_2x2.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace { @@ -354,7 +353,6 @@ void sve_transpose_interleave_8VL_2x2(uint16_t *out, const uint16_t *in, size_t "addvl %x[out], %x[out], #8\n" "bge 7b\n" "12:" // Done - : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [pad_row] "r" (pad_row), [width] "r" (width) : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -376,4 +374,5 @@ void Transform<8, 2, true, VLType::SVE>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_8VL_2x4.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_8VL_2x4.hpp index 51cae7dd5af9acbbb82571ac356cf6e4f8ee461a..34799c60a6c3a5fe1db2f3e8545e2108930fc836 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_8VL_2x4.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_8VL_2x4.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace { @@ -439,7 +438,6 @@ void sve_transpose_interleave_8VL_2x4(uint16_t *out, const uint16_t *in, size_t "addvl %x[out], %x[out], #8\n" "bge 7b\n" "12:" // Done - : [height] "+&r" (height), [in] "+&r" (in), [out] "+&r" (out) : [in_stride] "r" (in_stride), [out_stride] "r" (out_stride), [pad_row] "r" (pad_row), [width] "r" (width) : "cc", "memory", "p0", "p1", "p2", "x9", "x10", "x11", "x12", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" @@ -461,4 +459,5 @@ void Transform<8, 4, true, VLType::SVE>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_8VL_2x4_fp32bf16.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_8VL_2x4_fp32bf16.hpp index 4ad882870ee0c9333db413f983fc7d37e407b303..5a48e579aeedfe0bac7cfe475a03def62cca938b 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_8VL_2x4_fp32bf16.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_transpose_interleave_8VL_2x4_fp32bf16.hpp @@ -24,8 +24,7 @@ #pragma once -#ifdef __ARM_FEATURE_SVE - +#if defined(ARM_COMPUTE_ENABLE_SVE) namespace { @@ -279,4 +278,5 @@ void Transform<8, 4, true, VLType::SVE>( ); } -#endif + +#endif // defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_gemm/utils.hpp b/src/core/NEON/kernels/arm_gemm/utils.hpp index a28ddadc682c74722d738440d657b317d49eb3fb..11b1bd3e0529e4ad63c02c66510ceb8c91659aa8 100644 --- a/src/core/NEON/kernels/arm_gemm/utils.hpp +++ b/src/core/NEON/kernels/arm_gemm/utils.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2022 Arm Limited. + * Copyright (c) 2017-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -80,7 +80,8 @@ inline T roundup(const T a, const T b) { enum class VLType { None, SVE, - SME + SME, + SME2 }; template diff --git a/src/core/NEON/kernels/assembly/depthwise.hpp b/src/core/NEON/kernels/assembly/depthwise.hpp index 8eb278c22e1f2169f897ef26271e6bca4d225ae0..dbd47ccfa98886ea6d4bf09aaaae4a9c2aa241a8 100644 --- a/src/core/NEON/kernels/assembly/depthwise.hpp +++ b/src/core/NEON/kernels/assembly/depthwise.hpp @@ -27,6 +27,7 @@ #include "arm_gemm.hpp" #include "arm_gemm_local.hpp" #include "depthwise_common.hpp" +#include "premultiply.hpp" namespace arm_conv { @@ -38,8 +39,8 @@ struct DepthwiseConfig std::string filter = ""; DepthwiseConfig(DepthwiseMethod method) - : method(method){}; - DepthwiseConfig(){}; + : method(method) {}; + DepthwiseConfig() {}; }; struct DepthwiseArgs @@ -112,17 +113,64 @@ struct DepthwiseArgs } }; +template +struct Tile +{ + TInput *array; + + unsigned int tile_rows = 0; + unsigned int tile_cols = 0; + unsigned int tile_channels = 0; + + Tile(TInput *array, unsigned int tile_rows, unsigned int tile_cols, unsigned int tile_channels) + : array(array), tile_rows(tile_rows), tile_cols(tile_cols), tile_channels(tile_channels) + { + } + + Tile() + : Tile(nullptr, 0, 0, 0) + { + } + + void load_from( + const TInput *input, + const unsigned int ld_row, const unsigned int ld_col, + const unsigned int n_rows, const unsigned int n_cols, + const int input_i, const int input_j, + const unsigned int channel_multiplier) const + { + const auto pad_top = input_i < 0 ? -input_i : 0; + const auto pad_left = input_j < 0 ? -input_j : 0; + + const auto padded_rows = std::min(n_rows - input_i, tile_rows) - pad_top; + const auto padded_cols = std::min(n_cols - input_j, tile_cols) - pad_left; + + if(padded_rows < tile_rows || padded_cols < tile_cols) + { + memset(array, 0, tile_rows * tile_cols * tile_channels * sizeof(TInput)); + } + + do_premultiply( + (TInput *)input + std::max(input_i, 0) * ld_row + std::max(input_j, 0) * ld_col, + ld_row, ld_col, + array + pad_top * tile_cols * tile_channels + pad_left * tile_channels, + tile_cols * tile_channels, tile_channels, + padded_rows, padded_cols, tile_channels / channel_multiplier, + channel_multiplier); + } +}; + template class DepthwiseCommon : public IDepthwiseCommon { - protected: +protected: const DepthwiseArgs m_args; // Copy of arguments std::string m_name{}; - public: +public: DepthwiseCommon(const DepthwiseArgs &args) - : m_args(args){}; - DepthwiseCommon(DepthwiseCommon &) = delete; + : m_args(args) {}; + DepthwiseCommon(DepthwiseCommon &) = delete; DepthwiseCommon &operator=(DepthwiseCommon &) = delete; std::string name() const override @@ -133,7 +181,7 @@ class DepthwiseCommon : public IDepthwiseCommon void set_name(std::string name) { // Only allow the name to be set once - if (m_name.empty()) + if(m_name.empty()) { m_name = name; } @@ -209,47 +257,47 @@ class DepthwiseCommon : public IDepthwiseCommon // passed different input/output tensors. Dilation is handled at this // level; so we set the dilation in the arguments to zero. DepthwiseArgs args(this->m_args); - args.n_batches = batches; - args.input_rows = input_height; - args.input_cols = input_width; + args.n_batches = batches; + args.input_rows = input_height; + args.input_cols = input_width; args.input_channels = channels; - args.output_rows = output_height; - args.output_cols = output_width; - args.padding = padding; + args.output_rows = output_height; + args.output_cols = output_width; + args.padding = padding; args.dilation_rows = args.dilation_cols = 1; - auto ld_input_col_d = ld_input_col * m_args.dilation_cols; - auto ld_input_row_d = ld_input_row * m_args.dilation_rows; + auto ld_input_col_d = ld_input_col * m_args.dilation_cols; + auto ld_input_row_d = ld_input_row * m_args.dilation_rows; auto ld_output_col_d = ld_output_col * m_args.dilation_cols; auto ld_output_row_d = ld_output_row * m_args.dilation_rows; - for (size_t drow = 0; drow < m_args.dilation_rows; drow++) + for(size_t drow = 0; drow < m_args.dilation_rows; drow++) { size_t start_i; std::tie(args.output_rows, args.input_rows, start_i, args.padding.top, args.padding.bottom) = - get_reduced_view_for_dilation( - output_height, input_height, drow, m_args.dilation_rows, - m_args.kernel_rows, m_args.stride_rows, padding.top); + get_reduced_view_for_dilation( + output_height, input_height, drow, m_args.dilation_rows, + m_args.kernel_rows, m_args.stride_rows, padding.top); - auto input_row = static_cast(input) + start_i * ld_input_row; + auto input_row = static_cast(input) + start_i * ld_input_row; auto output_row = static_cast(output) + drow * ld_output_row; - if (args.output_rows) + if(args.output_rows) { - for (size_t dcol = 0; dcol < m_args.dilation_cols; dcol++) + for(size_t dcol = 0; dcol < m_args.dilation_cols; dcol++) { size_t start_j; std::tie(args.output_cols, args.input_cols, start_j, args.padding.left, args.padding.right) = - get_reduced_view_for_dilation( - output_width, input_width, dcol, m_args.dilation_cols, - m_args.kernel_cols, m_args.stride_cols, padding.left); + get_reduced_view_for_dilation( + output_width, input_width, dcol, m_args.dilation_cols, + m_args.kernel_cols, m_args.stride_cols, padding.left); - const TInput *input_col = input_row + start_j * ld_input_col; - TOutput *output_col = output_row + dcol * ld_output_col; + const TInput *input_col = input_row + start_j * ld_input_col; + TOutput *output_col = output_row + dcol * ld_output_col; - if (args.output_cols) + if(args.output_cols) { this->execute_internal( args, input_col, ld_input_col_d, ld_input_row_d, ld_input_batch, parameters, @@ -261,7 +309,7 @@ class DepthwiseCommon : public IDepthwiseCommon } } - protected: +protected: virtual void execute_internal( const DepthwiseArgs &instance_args, const void *input, @@ -276,6 +324,11 @@ class DepthwiseCommon : public IDepthwiseCommon void *working_space, unsigned int thread_id, unsigned int n_threads) const = 0; + + virtual bool uses_premultiply() const + { + return true; + } }; template diff --git a/src/core/NEON/kernels/assembly/depthwise_common.hpp b/src/core/NEON/kernels/assembly/depthwise_common.hpp index fea6326897db1ef437f0d15c033cb47e7f36be35..a5db793b3df54fef0e876e06b43105e3b6e8a3a1 100644 --- a/src/core/NEON/kernels/assembly/depthwise_common.hpp +++ b/src/core/NEON/kernels/assembly/depthwise_common.hpp @@ -85,7 +85,7 @@ public: size_t ld_weight_row = 0) = 0; // Determine the amount of working space required - virtual size_t get_working_size(unsigned int n_threads, unsigned int n_input_channels) const = 0; + virtual size_t get_working_size(unsigned int n_threads) const = 0; // Execute the convolution over the specified area of memory. virtual void execute( diff --git a/src/core/NEON/kernels/assembly/pool_common.hpp b/src/core/NEON/kernels/assembly/pool_common.hpp index 599e18ac592b42a91a93b762b7c5d406830e2198..f1f70cf1d65fdda9b98b8cad0aad54c101fe90c6 100644 --- a/src/core/NEON/kernels/assembly/pool_common.hpp +++ b/src/core/NEON/kernels/assembly/pool_common.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -66,7 +66,6 @@ public: // Determine the amount of working space required. virtual size_t get_working_size(unsigned int num_threads) const = 0; - virtual size_t get_working_size(unsigned int num_threads, unsigned int n_channels) const = 0; // Execute pooling over the specified area of memory. virtual void execute( diff --git a/src/core/NEON/kernels/assembly/pooling.hpp b/src/core/NEON/kernels/assembly/pooling.hpp index 1b47853eaf8c9032bb94a6cd11c18ad5dfe048fa..e8db35c5930d8654837c637abf5aa26a6b36d469 100644 --- a/src/core/NEON/kernels/assembly/pooling.hpp +++ b/src/core/NEON/kernels/assembly/pooling.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -122,11 +122,7 @@ public: PoolingCommon(PoolingCommon &) = delete; PoolingCommon &operator=(PoolingCommon &) = delete; - size_t get_working_size(unsigned int, unsigned int) const override = 0; - size_t get_working_size(unsigned int n_threads) const override - { - return this->get_working_size(n_threads, m_args.n_channels); - } + size_t get_working_size(unsigned int) const override = 0; // Execute pooling over the specified area of memory. void execute( diff --git a/src/core/NEON/kernels/assembly/premultiply.hpp b/src/core/NEON/kernels/assembly/premultiply.hpp new file mode 100644 index 0000000000000000000000000000000000000000..16f26de38abd74dfa0945a17a662cd31db2a3c59 --- /dev/null +++ b/src/core/NEON/kernels/assembly/premultiply.hpp @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +void do_premultiply_float_6(const float *in_ptr, + const unsigned int ld_row, + const unsigned int ld_col, + float *out_ptr, + const unsigned int out_ld_row, + const unsigned int out_ld_col, + const unsigned int tile_rows, + const unsigned int tile_cols, + const unsigned input_channels); + +template +void do_premultiply(const T *in_ptr, + const unsigned int ld_row, + const unsigned int ld_col, + T *out_ptr, + const unsigned int out_ld_row, + const unsigned int out_ld_col, + const unsigned int tile_rows, + const unsigned int tile_cols, + const unsigned input_channels, + const unsigned int channel_multiplier) +{ + if(sizeof(T) == 4 && channel_multiplier == 6) + { + do_premultiply_float_6( + (const float *)in_ptr, ld_row, ld_col, + (float *)out_ptr, out_ld_row, out_ld_col, + tile_rows, tile_cols, + input_channels); + } + else + { + for(unsigned int i = 0; i < tile_rows; i++) + { + const T *ip2 = in_ptr + i * ld_row; + T *op2 = out_ptr + i * out_ld_row; + for(unsigned int j = 0; j < tile_cols; j++) + { + const T *ip = ip2; + T *op = op2; + for(unsigned int c = 0; c < input_channels; c++) + { + T val = *ip; + ip++; + + for(unsigned int r = 0; r < channel_multiplier; r++) + { + op[r] = val; + } + op += channel_multiplier; + } + ip2 += ld_col; + op2 += out_ld_col; + } + } + } +} diff --git a/src/core/NEON/kernels/assembly/winograd.hpp b/src/core/NEON/kernels/assembly/winograd.hpp index 836402e83d62bd949ffdac2efdd2799c221096f4..50290757ecffe823fc46ca032a1c143e6cb4fa1d 100644 --- a/src/core/NEON/kernels/assembly/winograd.hpp +++ b/src/core/NEON/kernels/assembly/winograd.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,7 +24,7 @@ #pragma once -#include "src/cpu/kernels/assembly/arm_gemm.hpp" +#include "arm_gemm.hpp" #include namespace arm_conv diff --git a/src/core/NEON/kernels/batchnormalization/impl/SVE/fp16.cpp b/src/core/NEON/kernels/batchnormalization/impl/SVE/fp16.cpp index c7cfd7457d8c344eaa40b22f3d2dcf7f0590bd92..98cd9aa7fe57517356b61cc4336e95bbc1fdbfe5 100644 --- a/src/core/NEON/kernels/batchnormalization/impl/SVE/fp16.cpp +++ b/src/core/NEON/kernels/batchnormalization/impl/SVE/fp16.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021 Arm Limited. + * Copyright (c) 2020-2021,2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/ITensorPack.h" #include "arm_compute/core/Window.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/core/NEON/SVEMath.h" #include diff --git a/src/core/NEON/kernels/batchnormalization/impl/SVE/fp32.cpp b/src/core/NEON/kernels/batchnormalization/impl/SVE/fp32.cpp index b8a540158b4040f97d24c392d254734c4c265e6d..952ab320bfab9f71a67387e357bd2af53c0cc751 100644 --- a/src/core/NEON/kernels/batchnormalization/impl/SVE/fp32.cpp +++ b/src/core/NEON/kernels/batchnormalization/impl/SVE/fp32.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021 Arm Limited. + * Copyright (c) 2020-2021,2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/ITensorPack.h" #include "arm_compute/core/Window.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/core/NEON/SVEMath.h" #include diff --git a/src/core/NEON/kernels/convolution/winograd/input_transform.hpp b/src/core/NEON/kernels/convolution/winograd/input_transform.hpp index 113b7ea9287db3187ad66108b587b7bf26a759d9..265551288d4c152eb335e97fcc373dfbe1c761f3 100644 --- a/src/core/NEON/kernels/convolution/winograd/input_transform.hpp +++ b/src/core/NEON/kernels/convolution/winograd/input_transform.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,9 +24,7 @@ #pragma once -#include "arm_compute/core/Error.h" - -#include "src/core/NEON/kernels/assembly/winograd.hpp" +#include "winograd.hpp" #include "src/core/NEON/kernels/arm_conv/addressing.hpp" #include @@ -191,10 +189,9 @@ class TransformDirect : public TransformBase TOut *outptr, size_t ld_out_matrix, unsigned int pad_top, unsigned int valid_rows, unsigned int pad_left, unsigned int valid_cols, - void *working_space + void * ) const override { - ARM_COMPUTE_UNUSED(working_space); const auto end_i = this->get_input_rows() - pad_top; const auto pad_bottom = end_i < valid_rows ? 0 : end_i - valid_rows; const auto end_j = this->get_input_cols() - pad_left; diff --git a/src/core/NEON/kernels/convolution/winograd/input_transforms/arm_fp32_1x8.cpp b/src/core/NEON/kernels/convolution/winograd/input_transforms/arm_fp32_1x8.cpp index 44f8752a0ce825db097ee49d66507275092da230..ae589f9772d10f913a93695f58c377994d7de831 100644 --- a/src/core/NEON/kernels/convolution/winograd/input_transforms/arm_fp32_1x8.cpp +++ b/src/core/NEON/kernels/convolution/winograd/input_transforms/arm_fp32_1x8.cpp @@ -30,12 +30,12 @@ namespace winograd { namespace input_transform { void arm_fp32_1x8( - unsigned int n_channels, - const float * input_base, + const unsigned int n_channels, + const float *const input_base, size_t, // We don't need to stride over rows - size_t input_col_stride, + const size_t input_col_stride, float *outptr, - size_t matrix_stride + const size_t matrix_stride ) { constexpr int inner_tile_cols = 8; diff --git a/src/core/NEON/kernels/convolution/winograd/output_transform.hpp b/src/core/NEON/kernels/convolution/winograd/output_transform.hpp index 5148495608ff952a011ab78738a3bd7393408640..971cc99cd2c696064eb46c5f27b0f20b4e84afbd 100644 --- a/src/core/NEON/kernels/convolution/winograd/output_transform.hpp +++ b/src/core/NEON/kernels/convolution/winograd/output_transform.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,7 +24,7 @@ #pragma once -#include "src/core/NEON/kernels/assembly/winograd.hpp" +#include "winograd.hpp" #include "src/core/NEON/kernels/arm_conv/addressing.hpp" diff --git a/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x2_1x7.cpp b/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x2_1x7.cpp index bbf0ce58b4fb744de447b6f222b30c8aded3b733..cce3745c77e728468cd2a33703b9c5672ef64b00 100644 --- a/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x2_1x7.cpp +++ b/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x2_1x7.cpp @@ -33,13 +33,13 @@ namespace output_transform { void arm_fp32_1x2_1x7( unsigned int n_channels, const float* inptr, - size_t matrix_stride, + const size_t matrix_stride, const float* bptr, float *outptr, size_t, // No need to stride across rows - size_t output_col_stride, - float output_min, - float output_max + const size_t output_col_stride, + const float output_min, + const float output_max ) { constexpr auto inner_tile_cols = 8u, output_tile_cols = 2u; diff --git a/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x4_1x5.cpp b/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x4_1x5.cpp index feb2a5a2c1f4c8e9c5b07fe5384cabdeb11fcb0c..10f25a4aab73ab0699f9a2f98c2b737c29c05033 100644 --- a/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x4_1x5.cpp +++ b/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x4_1x5.cpp @@ -33,13 +33,13 @@ namespace output_transform { void arm_fp32_1x4_1x5( unsigned int n_channels, const float* inptr, - size_t matrix_stride, + const size_t matrix_stride, const float* bptr, float *outptr, size_t, // No need to stride across rows - size_t output_col_stride, - float output_min, - float output_max + const size_t output_col_stride, + const float output_min, + const float output_max ) { constexpr auto inner_tile_cols = 8u, output_tile_cols = 4u; diff --git a/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x6_1x3.cpp b/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x6_1x3.cpp index ffe60e700d6ea8e8c88e548cdd88d241f4bd13a4..c45ff8cf2c8060681825bc8f744c3b48cdb691d7 100644 --- a/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x6_1x3.cpp +++ b/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x6_1x3.cpp @@ -34,13 +34,13 @@ namespace output_transform { void arm_fp32_1x6_1x3( unsigned int n_channels, const float* inptr, - size_t matrix_stride, + const size_t matrix_stride, const float* bptr, float *outptr, size_t, // No need to stride across rows - size_t output_col_stride, - float output_min, - float output_max + const size_t output_col_stride, + const float output_min, + const float output_max ) { constexpr unsigned int inner_tile_cols = 8, output_tile_cols = 6; diff --git a/src/core/NEON/kernels/convolution/winograd/output_transforms/sme_fp32_mopa_4x4_3x3.cpp b/src/core/NEON/kernels/convolution/winograd/output_transforms/sme_fp32_mopa_4x4_3x3.cpp index 54749e6f28d449c8a35e3afa7f0e332b847f5dc5..55ed24cd7496f5f70872c698cc05c9dff4819a34 100644 --- a/src/core/NEON/kernels/convolution/winograd/output_transforms/sme_fp32_mopa_4x4_3x3.cpp +++ b/src/core/NEON/kernels/convolution/winograd/output_transforms/sme_fp32_mopa_4x4_3x3.cpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) && defined(ARM_COMPUTE_ENABLE_SME) +#if defined(ARM_COMPUTE_ENABLE_SME) #include @@ -888,5 +888,4 @@ void sme_fp32_mopa_4x4_3x3( } // namespace winograd } // namespace arm_conv -#endif //defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) && defined(ARM_COMPUTE_ENABLE_SME) - +#endif // defined(ARM_COMPUTE_ENABLE_SME) diff --git a/src/core/NEON/kernels/convolution/winograd/output_transforms_fp32.cpp b/src/core/NEON/kernels/convolution/winograd/output_transforms_fp32.cpp index a221aee5d8b36b9286df93707e2c2d045902a803..0a7030324ecb8c79de8d41513abd281f729eb339 100644 --- a/src/core/NEON/kernels/convolution/winograd/output_transforms_fp32.cpp +++ b/src/core/NEON/kernels/convolution/winograd/output_transforms_fp32.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -30,9 +30,9 @@ namespace winograd { namespace output_transform { #if defined(__aarch64__) -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(ARM_COMPUTE_ENABLE_SME) +#if defined(ARM_COMPUTE_ENABLE_SME) void sme_fp32_mopa_4x4_3x3(unsigned int, const float *, size_t, const float *, float *, size_t, size_t, float, float); -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(ARM_COMPUTE_ENABLE_SME) +#endif // defined(ARM_COMPUTE_ENABLE_SME) #endif // defined(__aarch64__) void arm_fp32_4x4_3x3(unsigned int, const float *, size_t, const float *, float *, size_t, size_t, float, float); void arm_fp32_2x2_3x3(unsigned int, const float *, size_t, const float *, float *, size_t, size_t, float, float); @@ -49,9 +49,9 @@ void arm_fp32_1x2_1x7(unsigned int, const float *, size_t, const float *, float static const TransformImplementation transforms_fp32[] = { #if defined(__aarch64__) -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(ARM_COMPUTE_ENABLE_SME) +#if defined(ARM_COMPUTE_ENABLE_SME) { IMPL(4, 4, 3, 3, sme_fp32_mopa_4x4_3x3, Unpadded), MethodConstraints::RequiresSME }, -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(ARM_COMPUTE_ENABLE_SME) +#endif // defined(ARM_COMPUTE_ENABLE_SME) #endif // defined(__aarch64__) { IMPL(4, 4, 3, 3, arm_fp32_4x4_3x3, Unpadded), MethodConstraints::LargerShape }, { IMPL(2, 2, 3, 3, arm_fp32_2x2_3x3, Unpadded) }, diff --git a/src/core/NEON/kernels/convolution/winograd/padding.cpp b/src/core/NEON/kernels/convolution/winograd/padding.cpp deleted file mode 100644 index aca8448658b3090e4b30e5f2c18e93a283c3a989..0000000000000000000000000000000000000000 --- a/src/core/NEON/kernels/convolution/winograd/padding.cpp +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Copyright (c) 2019 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#include -#include - -#include "padding.hpp" - -namespace padding -{ -template -void copy_and_pad_tile( - unsigned int tile_rows, - unsigned int tile_cols, - unsigned int n_channels, - const T *inptr, - unsigned int in_row_stride, - unsigned int in_col_stride, - T* outptr, - unsigned int out_row_stride, - unsigned int out_col_stride, - unsigned int pad_top, - unsigned int pad_left, - unsigned int pad_bottom, - unsigned int pad_right, - T pad_value -) -{ - for (unsigned int out_i = 0; out_i < tile_rows; out_i++) - { - for (unsigned int out_j = 0; out_j < tile_cols; out_j++) - { - T* const output = outptr + out_i*out_row_stride + out_j*out_col_stride; - - if (out_i < pad_top || tile_rows - pad_bottom <= out_i || - out_j < pad_left || tile_cols - pad_right <= out_j) - { - for (unsigned int n = 0; n < n_channels; n++) - { - output[n] = pad_value; - } - } - else - { - const auto in_i = out_i - pad_top, in_j = out_j - pad_left; - const T* const input = inptr + in_i*in_row_stride + in_j*in_col_stride; - std::memcpy(output, input, n_channels * sizeof(T)); - } - } - } -} - -template void copy_and_pad_tile( - unsigned int, unsigned int, unsigned int, - const uint8_t *, unsigned int, unsigned int, - uint8_t *, unsigned int, unsigned int, - unsigned int, unsigned int, unsigned int, unsigned int, uint8_t -); - -template void copy_and_pad_tile( - unsigned int, unsigned int, unsigned int, - const float *, unsigned int, unsigned int, - float *, unsigned int, unsigned int, - unsigned int, unsigned int, unsigned int, unsigned int, float -); - -#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC -template void copy_and_pad_tile( - unsigned int, unsigned int, unsigned int, - const __fp16 *, unsigned int, unsigned int, - __fp16 *, unsigned int, unsigned int, - unsigned int, unsigned int, unsigned int, unsigned int, __fp16 -); -#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC - -template -void CopyCropped::execute( - const size_t size, - const void * const inptr, - const size_t in_row_stride, - const size_t in_col_stride, - void * const outptr, - const size_t out_row_stride, - const size_t out_col_stride, - const unsigned int pad_top, - const unsigned int pad_left, - const unsigned int pad_bottom, - const unsigned int pad_right -) -{ - for (unsigned int out_i = 0, in_i = pad_top; in_i < TileRows - pad_bottom; out_i++, in_i++) - { - for (unsigned int out_j = 0, in_j = pad_left; in_j < TileCols - pad_right; out_j++, in_j++) - { - std::memcpy( - static_cast(outptr) + out_i*out_row_stride + out_j*out_col_stride, - static_cast(inptr) + in_i*in_row_stride + in_j*in_col_stride, - size - ); - } - } -} - -template class CopyCropped<2, 2>; -template class CopyCropped<3, 3>; -template class CopyCropped<4, 4>; - -template -void crop_and_copy_tile( - unsigned int tile_rows, - unsigned int tile_cols, - unsigned int n_channels, - const T *inptr, - unsigned int in_row_stride, - unsigned int in_col_stride, - T *outptr, - unsigned int out_row_stride, - unsigned int out_col_stride, - unsigned int crop_top, - unsigned int crop_left, - unsigned int crop_bottom, - unsigned int crop_right -) -{ - for (unsigned int out_i = 0, in_i = crop_top; in_i < tile_rows - crop_bottom; out_i++, in_i++) - { - for (unsigned int out_j = 0, in_j = crop_left; in_j < tile_cols - crop_right; out_j++, in_j++) - { - std::memcpy( - outptr + out_i*out_row_stride + out_j*out_col_stride, - inptr + in_i*in_row_stride + in_j*in_col_stride, - sizeof(T) * n_channels - ); - } - } -} - -template void crop_and_copy_tile( - unsigned int tile_rows, - unsigned int tile_cols, - unsigned int n_channels, - const float *inptr, - unsigned int in_row_stride, - unsigned int in_col_stride, - float *outptr, - unsigned int out_row_stride, - unsigned int out_col_stride, - unsigned int crop_top, - unsigned int crop_left, - unsigned int crop_bottom, - unsigned int crop_right -); - -#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC -template void crop_and_copy_tile( - unsigned int tile_rows, - unsigned int tile_cols, - unsigned int n_channels, - const __fp16 *inptr, - unsigned int in_row_stride, - unsigned int in_col_stride, - __fp16 *outptr, - unsigned int out_row_stride, - unsigned int out_col_stride, - unsigned int crop_top, - unsigned int crop_left, - unsigned int crop_bottom, - unsigned int crop_right -); -#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC -} // namespace padding diff --git a/src/core/NEON/kernels/convolution/winograd/weight_transform.hpp b/src/core/NEON/kernels/convolution/winograd/weight_transform.hpp index db0f53df1b251e6a035dd336a980182ba3e906e2..5569bc1b894813a9200521d5328d5d82d177a022 100644 --- a/src/core/NEON/kernels/convolution/winograd/weight_transform.hpp +++ b/src/core/NEON/kernels/convolution/winograd/weight_transform.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,7 +24,7 @@ #pragma once -#include "src/core/NEON/kernels/assembly/winograd.hpp" +#include "winograd.hpp" #include #include diff --git a/src/core/NEON/kernels/convolution/winograd/winograd_implementations.hpp b/src/core/NEON/kernels/convolution/winograd/winograd_implementations.hpp index 510f69baaa8812c5fca96e4db206360eb82140c8..af0dd0429899cada637854640e73f8106042314a 100644 --- a/src/core/NEON/kernels/convolution/winograd/winograd_implementations.hpp +++ b/src/core/NEON/kernels/convolution/winograd/winograd_implementations.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,7 +24,7 @@ #pragma once -#include "src/core/NEON/kernels/assembly/winograd.hpp" +#include "winograd.hpp" #include #include @@ -314,6 +314,7 @@ bool get_implementation( false, // Indirect input {}, // No activation max_threads, + false, // Not fixed format fast_mode, gemm_cfg )); diff --git a/src/core/NEON/kernels/detail/NEActivationFunctionDetail.h b/src/core/NEON/kernels/detail/NEActivationFunctionDetail.h index 25d682d8aec476a2b0a97080ca6bf4471172e52b..3900ea62cdddbf5e5dca98f4a70fb4fb19389dd0 100644 --- a/src/core/NEON/kernels/detail/NEActivationFunctionDetail.h +++ b/src/core/NEON/kernels/detail/NEActivationFunctionDetail.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_DETAIL_NEACTIVATION_FUNCTION_DETAIL_H #define ARM_COMPUTE_DETAIL_NEACTIVATION_FUNCTION_DETAIL_H +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/core/NEON/wrapper/wrapper.h" namespace arm_compute diff --git a/src/core/Utils.cpp b/src/core/Utils.cpp index d05a044fc82958491b53e32c9e53dd648944f5fe..0701ee7c9076b3e91ec283e8d1792753e0bf6896 100644 --- a/src/core/Utils.cpp +++ b/src/core/Utils.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2022 Arm Limited. + * Copyright (c) 2016-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,10 +22,12 @@ * SOFTWARE. */ -#include "arm_compute/core/Helpers.h" - #include "arm_compute/core/Utils.h" +#include "arm_compute/core/Helpers.h" +#include "arm_compute/core/utils/StringUtils.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" + #include #include #include @@ -73,32 +75,6 @@ std::string read_file(const std::string &filename, bool binary) return out; } -const std::string &string_from_format(Format format) -{ - static std::map formats_map = - { - { Format::UNKNOWN, "UNKNOWN" }, - { Format::U8, "U8" }, - { Format::S16, "S16" }, - { Format::U16, "U16" }, - { Format::S32, "S32" }, - { Format::U32, "U32" }, - { Format::F16, "F16" }, - { Format::F32, "F32" }, - { Format::UV88, "UV88" }, - { Format::RGB888, "RGB888" }, - { Format::RGBA8888, "RGBA8888" }, - { Format::YUV444, "YUV444" }, - { Format::YUYV422, "YUYV422" }, - { Format::NV12, "NV12" }, - { Format::NV21, "NV21" }, - { Format::IYUV, "IYUV" }, - { Format::UYVY422, "UYVY422" } - }; - - return formats_map[format]; -} - const std::string &string_from_channel(Channel channel) { static std::map channels_map = @@ -120,84 +96,6 @@ const std::string &string_from_channel(Channel channel) return channels_map[channel]; } -const std::string &string_from_data_layout(DataLayout dl) -{ - static std::map dl_map = - { - { DataLayout::UNKNOWN, "UNKNOWN" }, - { DataLayout::NCHW, "NCHW" }, - { DataLayout::NHWC, "NHWC" }, - }; - - return dl_map[dl]; -} - -const std::string &string_from_data_type(DataType dt) -{ - static std::map dt_map = - { - { DataType::UNKNOWN, "UNKNOWN" }, - { DataType::S8, "S8" }, - { DataType::U8, "U8" }, - { DataType::S16, "S16" }, - { DataType::U16, "U16" }, - { DataType::S32, "S32" }, - { DataType::U32, "U32" }, - { DataType::S64, "S64" }, - { DataType::U64, "U64" }, - { DataType::F16, "F16" }, - { DataType::F32, "F32" }, - { DataType::F64, "F64" }, - { DataType::SIZET, "SIZET" }, - { DataType::QSYMM8, "QSYMM8" }, - { DataType::QSYMM8_PER_CHANNEL, "QSYMM8_PER_CHANNEL" }, - { DataType::QASYMM8, "QASYMM8" }, - { DataType::QASYMM8_SIGNED, "QASYMM8_SIGNED" }, - { DataType::QSYMM16, "QSYMM16" }, - { DataType::QASYMM16, "QASYMM16" }, - }; - - return dt_map[dt]; -} - -const std::string &string_from_activation_func(ActivationLayerInfo::ActivationFunction act) -{ - static std::map act_map = - { - { ActivationLayerInfo::ActivationFunction::ABS, "ABS" }, - { ActivationLayerInfo::ActivationFunction::LINEAR, "LINEAR" }, - { ActivationLayerInfo::ActivationFunction::LOGISTIC, "LOGISTIC" }, - { ActivationLayerInfo::ActivationFunction::RELU, "RELU" }, - { ActivationLayerInfo::ActivationFunction::BOUNDED_RELU, "BRELU" }, - { ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU, "LU_BRELU" }, - { ActivationLayerInfo::ActivationFunction::LEAKY_RELU, "LRELU" }, - { ActivationLayerInfo::ActivationFunction::SOFT_RELU, "SRELU" }, - { ActivationLayerInfo::ActivationFunction::ELU, "ELU" }, - { ActivationLayerInfo::ActivationFunction::SQRT, "SQRT" }, - { ActivationLayerInfo::ActivationFunction::SQUARE, "SQUARE" }, - { ActivationLayerInfo::ActivationFunction::TANH, "TANH" }, - { ActivationLayerInfo::ActivationFunction::IDENTITY, "IDENTITY" }, - { ActivationLayerInfo::ActivationFunction::HARD_SWISH, "HARD_SWISH" }, - { ActivationLayerInfo::ActivationFunction::SWISH, "SWISH" }, - { ActivationLayerInfo::ActivationFunction::GELU, "GELU" } - - }; - - return act_map[act]; -} - -const std::string &string_from_interpolation_policy(InterpolationPolicy policy) -{ - static std::map interpolation_policy_map = - { - { InterpolationPolicy::AREA, "AREA" }, - { InterpolationPolicy::BILINEAR, "BILINEAR" }, - { InterpolationPolicy::NEAREST_NEIGHBOR, "NEAREST_NEIGHBOUR" }, - }; - - return interpolation_policy_map[policy]; -} - const std::string &string_from_border_mode(BorderMode border_mode) { static std::map border_mode_map = @@ -325,45 +223,6 @@ std::string string_from_pixel_value(const PixelValue &value, const DataType data return converted_string; } -DataType data_type_from_name(const std::string &name) -{ - static const std::map data_types = - { - { "f16", DataType::F16 }, - { "f32", DataType::F32 }, - { "qasymm8", DataType::QASYMM8 }, - { "qasymm8_signed", DataType::QASYMM8_SIGNED }, - }; - -#ifndef ARM_COMPUTE_EXCEPTIONS_DISABLED - try - { -#endif /* ARM_COMPUTE_EXCEPTIONS_DISABLED */ - return data_types.at(utility::tolower(name)); - -#ifndef ARM_COMPUTE_EXCEPTIONS_DISABLED - } - catch(const std::out_of_range &) - { - ARM_COMPUTE_ERROR_VAR("Invalid data type name: %s", name.c_str()); - } -#endif /* ARM_COMPUTE_EXCEPTIONS_DISABLED */ -} - -std::string lower_string(const std::string &val) -{ - std::string res = val; - std::transform(res.begin(), res.end(), res.begin(), ::tolower); - return res; -} - -std::string upper_string(const std::string &val) -{ - std::string res = val; - std::transform(res.begin(), res.end(), res.begin(), ::toupper); - return res; -} - PadStrideInfo calculate_same_pad(TensorShape input_shape, TensorShape weights_shape, PadStrideInfo conv_info, DataLayout data_layout, const Size2D &dilation, const DimensionRoundingType &rounding_type) { @@ -555,7 +414,7 @@ QuantizationInfo get_softmax_output_quantization_info(DataType input_type, bool return QuantizationInfo(1.f / 256, 0); } -std::pair get_quantized_activation_min_max(ActivationLayerInfo act_info, DataType data_type, UniformQuantizationInfo oq_info) +std::pair get_quantized_activation_min_max(const ActivationLayerInfo &act_info, DataType data_type, UniformQuantizationInfo oq_info) { const bool is_qasymm8_signed = is_data_type_quantized_asymmetric_signed(data_type); const auto a = act_info.a(); @@ -638,6 +497,12 @@ void print_consecutive_elements(std::ostream &s, DataType dt, const uint8_t *ptr case DataType::S32: print_consecutive_elements_impl(s, reinterpret_cast(ptr), n, stream_width, element_delim); break; + case DataType::U64: + print_consecutive_elements_impl(s, reinterpret_cast(ptr), n, stream_width, element_delim); + break; + case DataType::S64: + print_consecutive_elements_impl(s, reinterpret_cast(ptr), n, stream_width, element_delim); + break; case DataType::BFLOAT16: print_consecutive_elements_impl(s, reinterpret_cast(ptr), n, stream_width, element_delim); break; @@ -674,6 +539,10 @@ int max_consecutive_elements_display_width(std::ostream &s, DataType dt, const u return max_consecutive_elements_display_width_impl(s, reinterpret_cast(ptr), n); case DataType::S32: return max_consecutive_elements_display_width_impl(s, reinterpret_cast(ptr), n); + case DataType::U64: + return max_consecutive_elements_display_width_impl(s, reinterpret_cast(ptr), n); + case DataType::S64: + return max_consecutive_elements_display_width_impl(s, reinterpret_cast(ptr), n); case DataType::BFLOAT16: return max_consecutive_elements_display_width_impl(s, reinterpret_cast(ptr), n); case DataType::F16: @@ -687,4 +556,4 @@ int max_consecutive_elements_display_width(std::ostream &s, DataType dt, const u } #endif /* ARM_COMPUTE_ASSERTS_ENABLED */ -} // namespace arm_compute \ No newline at end of file +} // namespace arm_compute diff --git a/src/core/helpers/AutoConfiguration.h b/src/core/helpers/AutoConfiguration.h index 18ffbd6295b1912b13a931a2bf5715f7b73b7831..8715dcd74b4d365fb62417c85752253420e0b398 100644 --- a/src/core/helpers/AutoConfiguration.h +++ b/src/core/helpers/AutoConfiguration.h @@ -24,6 +24,7 @@ #ifndef SRC_CORE_HELPERS_AUTOCONFIGURATION_H #define SRC_CORE_HELPERS_AUTOCONFIGURATION_H +#include "arm_compute/core/utils/DataTypeUtils.h" #include "arm_compute/core/ITensorInfo.h" #include "arm_compute/core/Types.h" diff --git a/src/core/utils/ActivationFunctionUtils.cpp b/src/core/utils/ActivationFunctionUtils.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4854b8eb0bead6c5857c052f969b862ba795e3bb --- /dev/null +++ b/src/core/utils/ActivationFunctionUtils.cpp @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2016-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "arm_compute/core/utils/ActivationFunctionUtils.h" + +#include + +namespace arm_compute +{ +const std::string &string_from_activation_func(const ActivationFunction& act) +{ + static std::map act_map = + { + { ActivationFunction::ABS, "ABS" }, + { ActivationFunction::LINEAR, "LINEAR" }, + { ActivationFunction::LOGISTIC, "LOGISTIC" }, + { ActivationFunction::RELU, "RELU" }, + { ActivationFunction::BOUNDED_RELU, "BRELU" }, + { ActivationFunction::LU_BOUNDED_RELU, "LU_BRELU" }, + { ActivationFunction::LEAKY_RELU, "LRELU" }, + { ActivationFunction::SOFT_RELU, "SRELU" }, + { ActivationFunction::ELU, "ELU" }, + { ActivationFunction::SQRT, "SQRT" }, + { ActivationFunction::SQUARE, "SQUARE" }, + { ActivationFunction::TANH, "TANH" }, + { ActivationFunction::IDENTITY, "IDENTITY" }, + { ActivationFunction::HARD_SWISH, "HARD_SWISH" }, + { ActivationFunction::SWISH, "SWISH" }, + { ActivationFunction::GELU, "GELU" } + + }; + + return act_map[act]; +} + +} // namespace arm_compute diff --git a/src/core/utils/AssemblyUtils.cpp b/src/core/utils/AssemblyUtils.cpp index 45e7ff78be05aa4abb9709713669d99dee0638f2..6d483adc7fff3fc9e0f4d9085c89a9b33e0e1aec 100644 --- a/src/core/utils/AssemblyUtils.cpp +++ b/src/core/utils/AssemblyUtils.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,6 +23,8 @@ */ #include "src/core/utils/AssemblyUtils.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" + namespace arm_compute { namespace assembly_utils diff --git a/src/core/utils/AssemblyUtils.h b/src/core/utils/AssemblyUtils.h index 7514175ed6424a36ec9759afb58ca441d059b83d..60bad3b61836c24e799a6204833bd9065a06c564 100644 --- a/src/core/utils/AssemblyUtils.h +++ b/src/core/utils/AssemblyUtils.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -30,6 +30,8 @@ namespace arm_compute { +class ActivationLayerInfo; + namespace assembly_utils { /** Performs a mapping between Compute Library ActivationLayerInfo and the assembly Activation structure. diff --git a/src/core/utils/DataLayoutUtils.cpp b/src/core/utils/DataLayoutUtils.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4919b79a42320d6b8c60a9807b573d0c3487dc92 --- /dev/null +++ b/src/core/utils/DataLayoutUtils.cpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2016-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "arm_compute/core/utils/DataLayoutUtils.h" + +namespace arm_compute +{ + +const std::string &string_from_data_layout(DataLayout dl) +{ + static std::map dl_map = + { + { DataLayout::UNKNOWN, "UNKNOWN" }, + { DataLayout::NCHW, "NCHW" }, + { DataLayout::NHWC, "NHWC" }, + }; + + return dl_map[dl]; +} + +} // namespace arm_compute diff --git a/src/core/utils/DataTypeUtils.cpp b/src/core/utils/DataTypeUtils.cpp new file mode 100644 index 0000000000000000000000000000000000000000..07999354d9716a02abfb04d03c33b7553ab3ed38 --- /dev/null +++ b/src/core/utils/DataTypeUtils.cpp @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2016-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "arm_compute/core/utils/DataTypeUtils.h" + +#include + +namespace arm_compute +{ +const std::string &string_from_data_type(DataType dt) +{ + static std::map dt_map = + { + { DataType::UNKNOWN, "UNKNOWN" }, + { DataType::S8, "S8" }, + { DataType::U8, "U8" }, + { DataType::S16, "S16" }, + { DataType::U16, "U16" }, + { DataType::S32, "S32" }, + { DataType::U32, "U32" }, + { DataType::S64, "S64" }, + { DataType::U64, "U64" }, + { DataType::F16, "F16" }, + { DataType::F32, "F32" }, + { DataType::F64, "F64" }, + { DataType::SIZET, "SIZET" }, + { DataType::QSYMM8, "QSYMM8" }, + { DataType::QSYMM8_PER_CHANNEL, "QSYMM8_PER_CHANNEL" }, + { DataType::QASYMM8, "QASYMM8" }, + { DataType::QASYMM8_SIGNED, "QASYMM8_SIGNED" }, + { DataType::QSYMM16, "QSYMM16" }, + { DataType::QASYMM16, "QASYMM16" }, + }; + + return dt_map[dt]; +} + +DataType data_type_from_name(const std::string &name) +{ + static const std::map data_types = + { + { "f16", DataType::F16 }, + { "f32", DataType::F32 }, + { "qasymm8", DataType::QASYMM8 }, + { "qasymm8_signed", DataType::QASYMM8_SIGNED }, + }; + +#ifndef ARM_COMPUTE_EXCEPTIONS_DISABLED + try + { +#endif /* ARM_COMPUTE_EXCEPTIONS_DISABLED */ + return data_types.at(utility::tolower(name)); + +#ifndef ARM_COMPUTE_EXCEPTIONS_DISABLED + } + catch(const std::out_of_range &) + { + ARM_COMPUTE_ERROR_VAR("Invalid data type name: %s", name.c_str()); + } +#endif /* ARM_COMPUTE_EXCEPTIONS_DISABLED */ +} + +} // namespace arm_compute diff --git a/src/core/utils/FormatUtils.cpp b/src/core/utils/FormatUtils.cpp new file mode 100644 index 0000000000000000000000000000000000000000..05b649ded2029c530b99a03bae96d539a211440a --- /dev/null +++ b/src/core/utils/FormatUtils.cpp @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2016-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "arm_compute/core/utils/FormatUtils.h" + +#include + +namespace arm_compute +{ +const std::string &string_from_format(Format format) +{ + static std::map formats_map = + { + { Format::UNKNOWN, "UNKNOWN" }, + { Format::U8, "U8" }, + { Format::S16, "S16" }, + { Format::U16, "U16" }, + { Format::S32, "S32" }, + { Format::U32, "U32" }, + { Format::F16, "F16" }, + { Format::F32, "F32" }, + { Format::UV88, "UV88" }, + { Format::RGB888, "RGB888" }, + { Format::RGBA8888, "RGBA8888" }, + { Format::YUV444, "YUV444" }, + { Format::YUYV422, "YUYV422" }, + { Format::NV12, "NV12" }, + { Format::NV21, "NV21" }, + { Format::IYUV, "IYUV" }, + { Format::UYVY422, "UYVY422" } + }; + + return formats_map[format]; +} +} // namespace arm_compute diff --git a/src/core/utils/InterpolationPolicyUtils.cpp b/src/core/utils/InterpolationPolicyUtils.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2d6cabe85efe2dee73b5b342aa5d8743a943cd5f --- /dev/null +++ b/src/core/utils/InterpolationPolicyUtils.cpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2016-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "arm_compute/core/utils/InterpolationPolicyUtils.h" + +namespace arm_compute +{ + +const std::string &string_from_interpolation_policy(InterpolationPolicy policy) +{ + static std::map interpolation_policy_map = + { + { InterpolationPolicy::AREA, "AREA" }, + { InterpolationPolicy::BILINEAR, "BILINEAR" }, + { InterpolationPolicy::NEAREST_NEIGHBOR, "NEAREST_NEIGHBOUR" }, + }; + + return interpolation_policy_map[policy]; +} + +} // namespace arm_compute diff --git a/src/core/utils/StringUtils.cpp b/src/core/utils/StringUtils.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6d05c9b64e164e323a0d447a6726099f5c835884 --- /dev/null +++ b/src/core/utils/StringUtils.cpp @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2016-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "arm_compute/core/utils/StringUtils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace arm_compute +{ +std::string lower_string(const std::string &val) +{ + std::string res = val; + std::transform(res.begin(), res.end(), res.begin(), ::tolower); + return res; +} + +std::string upper_string(const std::string &val) +{ + std::string res = val; + std::transform(res.begin(), res.end(), res.begin(), ::toupper); + return res; +} + +std::string float_to_string_with_full_precision(float val) +{ + std::stringstream ss; + ss.precision(std::numeric_limits::max_digits10); + ss << val; + + if(val != static_cast(val)) + { + ss << "f"; + } + + return ss.str(); +} + +std::string join(const std::vector strings, const std::string &sep) +{ + if(strings.empty()) + { + return ""; + } + return std::accumulate( + std::next(strings.begin()), + strings.end(), + strings.at(0), + [&sep](const std::string & a, const std::string & b) + { + return a + sep + b; + }); +} +} diff --git a/src/core/utils/quantization/AsymmHelpers.cpp b/src/core/utils/quantization/AsymmHelpers.cpp index f5b69c7a444a2197ced4362209b4c8d6a36c526c..086d63b968086278b3c241269d41ca0a546f79c6 100644 --- a/src/core/utils/quantization/AsymmHelpers.cpp +++ b/src/core/utils/quantization/AsymmHelpers.cpp @@ -23,8 +23,9 @@ */ #include "arm_compute/core/utils/quantization/AsymmHelpers.h" #include "arm_compute/core/Helpers.h" -#include "support/ToolchainSupport.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/core/utils/quantization/AsymmHelpers.h" +#include "support/ToolchainSupport.h" #include #include @@ -239,15 +240,14 @@ void compute_quantized_multipliers_and_shifts(const ITensorInfo *input, int32_t saturating_rounding_doubling_highmul(int32_t a, int32_t b) { - bool overflow = a == b && a == std::numeric_limits::min(); - int64_t a_64(a); - int64_t b_64(b); - int64_t ab_64 = a_64 * b_64; - const bool is_positive_or_zero = - a == 0 || b == 0 || - (std::signbit(static_cast(a)) == std::signbit(static_cast(b))); - int32_t nudge = is_positive_or_zero ? (1 << 30) : (1 - (1 << 30)); - int32_t ab_x2_high32 = static_cast((ab_64 + nudge) / (1ll << 31)); + bool overflow = a == b && a == std::numeric_limits::min(); + int64_t a_64(a); + int64_t b_64(b); + int64_t ab_64 = a_64 * b_64; + const bool is_positive_or_zero = + a == 0 || b == 0 || (std::signbit(static_cast(a)) == std::signbit(static_cast(b))); + int32_t nudge = is_positive_or_zero ? (1 << 30) : (1 - (1 << 30)); + int32_t ab_x2_high32 = static_cast((ab_64 + nudge) / (1ll << 31)); return overflow ? std::numeric_limits::max() : ab_x2_high32; } diff --git a/src/cpu/CpuContext.cpp b/src/cpu/CpuContext.cpp index d91f9179634fe2a51734062c4535d8e382cf0557..7c14891ef801ffc86dca8d7acb38ea8f232d7e71 100644 --- a/src/cpu/CpuContext.cpp +++ b/src/cpu/CpuContext.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -37,6 +37,10 @@ #endif // defined(_WIN64) #endif // !defined(__APPLE__) && !defined(__OpenBSD__) +#ifndef BARE_METAL +#include +#endif /* BARE_METAL */ + namespace arm_compute { namespace cpu diff --git a/src/cpu/CpuTypes.h b/src/cpu/CpuTypes.h new file mode 100644 index 0000000000000000000000000000000000000000..0f7b9b655278dca9abcb7f083d32c57bea916e7d --- /dev/null +++ b/src/cpu/CpuTypes.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_CPUTYPES +#define ARM_COMPUTE_CPUTYPES + +namespace arm_compute +{ +/* Type definitions compatible with arm_neon.h and arm_sve.h */ +#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC +typedef __fp16 float16_t; +#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC +typedef float float32_t; +} + +#endif /* ARM_COMPUTE_CPUTYPES */ diff --git a/src/cpu/kernels/CpuActivationKernel.cpp b/src/cpu/kernels/CpuActivationKernel.cpp index 20a8489cdd5deae7585672de1cf833cf1d7d419a..f4bd4e6cadc59aa5934792f71e31ded7532ff69c 100644 --- a/src/cpu/kernels/CpuActivationKernel.cpp +++ b/src/cpu/kernels/CpuActivationKernel.cpp @@ -48,7 +48,7 @@ static const std::vector available_kernel #ifdef ARM_COMPUTE_ENABLE_SVE { "sve2_q8_activation_lut", - [](const ActivationDataTypeISASelectorData & data) { return ActivationLayerInfo::is_lut_supported(data.f, data.dt) && data.cpumodel == CPUModel::A510 && data.isa.sve2; }, + [](const ActivationDataTypeISASelectorData & data) { return (data.dt == DataType::QASYMM8 || data.dt == DataType::QASYMM8_SIGNED) && data.cpumodel == CPUModel::A510 && data.isa.sve2; }, REGISTER_QASYMM8_SVE2(arm_compute::cpu::sve2_q8_activation_lut) }, #endif // ARM_COMPUTE_ENABLE_SVE @@ -56,7 +56,7 @@ static const std::vector available_kernel { // Neon LUT implementantion takes precedence "neon_q8_activation_lut", - [](const ActivationDataTypeISASelectorData & data) { return ActivationLayerInfo::is_lut_supported(data.f, data.dt); }, + [](const ActivationDataTypeISASelectorData & data) { return data.dt == DataType::QASYMM8 || data.dt == DataType::QASYMM8_SIGNED; }, REGISTER_Q8_NEON(arm_compute::cpu::neon_q8_activation_lut) }, #endif // __aarch64__ @@ -184,6 +184,72 @@ std::pair validate_and_configure_window(const ITensorInfo *src, return std::make_pair(Status{}, win); } +#ifdef __aarch64__ +void init_lut(ActivationLayerInfo::ActivationFunction act_func, DataType data_type, + const UniformQuantizationInfo &qi_in, const UniformQuantizationInfo &qi_out, + ActivationLayerInfo::LookupTable256 &lut, float a, float b) +{ + for(size_t i = 0; i < lut.size(); ++i) + { + float tmp_f = (data_type == DataType::QASYMM8) ? dequantize_qasymm8(i, qi_in) : dequantize_qasymm8_signed(i, qi_in); + switch(act_func) + { + case ActivationLayerInfo::ActivationFunction::HARD_SWISH: + tmp_f = tmp_f * ((std::min(std::max((tmp_f + 3), 0.0f), 6.0f)) * 0.166666667f); + break; + case ActivationLayerInfo::ActivationFunction::LEAKY_RELU: + tmp_f = tmp_f > 0 ? tmp_f : tmp_f * a; + break; + case ActivationLayerInfo::ActivationFunction::LOGISTIC: + tmp_f = 1.f / (1.f + std::exp(-tmp_f)); + break; + case ActivationLayerInfo::ActivationFunction::ABS: + tmp_f = std::abs(tmp_f); + break; + case ActivationLayerInfo::ActivationFunction::LINEAR: + tmp_f = a * tmp_f + b; + break; + case ActivationLayerInfo::ActivationFunction::RELU: + tmp_f = std::max<>(0.f, tmp_f); + break; + case ActivationLayerInfo::ActivationFunction::BOUNDED_RELU: + tmp_f = std::min<>(a, std::max(0.f, tmp_f)); + break; + case ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU: + tmp_f = std::min<>(a, std::max<>(b, tmp_f)); + break; + case ActivationLayerInfo::ActivationFunction::SOFT_RELU: + tmp_f = (tmp_f > 12.f) ? tmp_f : std::log(1.f + std::exp(tmp_f)); + break; + case ActivationLayerInfo::ActivationFunction::ELU: + tmp_f = (tmp_f >= 0) ? tmp_f : a * (std::exp(tmp_f) - 1); + break; + case ActivationLayerInfo::ActivationFunction::SQRT: + tmp_f = std::sqrt(tmp_f); + break; + case ActivationLayerInfo::ActivationFunction::SQUARE: + tmp_f = tmp_f * tmp_f; + break; + case ActivationLayerInfo::ActivationFunction::TANH: + tmp_f = a * std::tanh(b * tmp_f); + break; + case ActivationLayerInfo::ActivationFunction::IDENTITY: + break; + case ActivationLayerInfo::ActivationFunction::SWISH: + tmp_f = tmp_f / (1.f + std::exp(-a * tmp_f)); + break; + case ActivationLayerInfo::ActivationFunction::GELU: + tmp_f = tmp_f * (0.5f * (1.0f + erff(tmp_f / 1.41421356237f))); + break; + default: + ARM_COMPUTE_ERROR("Not supported"); + tmp_f = 0; + break; + } + lut[i] = (data_type == DataType::QASYMM8) ? quantize_qasymm8(tmp_f, qi_out) : quantize_qasymm8_signed(tmp_f, qi_out); + } +} +#endif // __aarch64__ } // namespace void CpuActivationKernel::configure(const ITensorInfo *src, ITensorInfo *dst, ActivationLayerInfo activation_info) @@ -205,9 +271,12 @@ void CpuActivationKernel::configure(const ITensorInfo *src, ITensorInfo *dst, Ac _name = std::string("CpuActivationKernel").append("/").append(uk->name); #ifdef __aarch64__ - if(ActivationLayerInfo::is_lut_supported(activation_info.activation(), src->data_type())) + if(src->data_type() == DataType::QASYMM8 || src->data_type() == DataType::QASYMM8_SIGNED) { - activation_info.init_lut(src->data_type(), src->quantization_info().uniform(), (dst) ? dst->quantization_info().uniform() : src->quantization_info().uniform()); + ActivationLayerInfo::LookupTable256 tmp_lut; + init_lut(activation_info.activation(), src->data_type(), src->quantization_info().uniform(), (dst) ? dst->quantization_info().uniform() : src->quantization_info().uniform(), + tmp_lut, activation_info.a(), activation_info.b()); + activation_info.setLookupTable256(tmp_lut); } #endif // __aarch64__ _act_info = activation_info; diff --git a/src/cpu/kernels/CpuActivationKernel.h b/src/cpu/kernels/CpuActivationKernel.h index fe2d783059af2dd5c24eec5a7afa50a653c15b13..804407653f36a5de1a6744387b5e60d1666b89a5 100644 --- a/src/cpu/kernels/CpuActivationKernel.h +++ b/src/cpu/kernels/CpuActivationKernel.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2022 Arm Limited. + * Copyright (c) 2017-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CPU_ACTIVATION_KERNEL_H #define ARM_COMPUTE_CPU_ACTIVATION_KERNEL_H +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/core/common/Macros.h" #include "src/cpu/ICpuKernel.h" diff --git a/src/cpu/kernels/CpuAddMulAddKernel.h b/src/cpu/kernels/CpuAddMulAddKernel.h index 595b580060962b6bf86c7e9b097a8515969323bd..67ce6f029afef00aac5d8fa36f299d9d8de1ab6a 100644 --- a/src/cpu/kernels/CpuAddMulAddKernel.h +++ b/src/cpu/kernels/CpuAddMulAddKernel.h @@ -25,6 +25,7 @@ #ifndef SRC_CPU_KERNELS_CPUADDMULADDKERNEL #define SRC_CPU_KERNELS_CPUADDMULADDKERNEL +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/core/common/Macros.h" #include "src/cpu/ICpuKernel.h" diff --git a/src/cpu/kernels/CpuCastKernel.cpp b/src/cpu/kernels/CpuCastKernel.cpp index 15a9ddcab4082853911f24d01cd25a5cb7efc378..641dea40dcc82f67c52b960dc3839fbb1c83f4ac 100644 --- a/src/cpu/kernels/CpuCastKernel.cpp +++ b/src/cpu/kernels/CpuCastKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2022 Arm Limited. + * Copyright (c) 2016-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -99,9 +99,16 @@ Status validate_arguments(const ITensorInfo *src, const ITensorInfo *dst, Conver ARM_COMPUTE_RETURN_ERROR_ON_CPU_BF16_UNSUPPORTED(dst); ARM_COMPUTE_UNUSED(policy); ARM_COMPUTE_RETURN_ERROR_ON(src == dst); +#ifdef __aarch64__ + ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(src, 1, DataType::QASYMM8_SIGNED, DataType::QASYMM8, DataType::U8, + DataType::S16, DataType::U16, DataType::BFLOAT16, DataType::F16, + DataType::F32, DataType::S32, DataType::S64, DataType::U64); +#else // __aarch64__ ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(src, 1, DataType::QASYMM8_SIGNED, DataType::QASYMM8, DataType::U8, DataType::S16, DataType::U16, DataType::BFLOAT16, DataType::F16, DataType::F32, DataType::S32); +#endif // __aarch64__ + ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(dst, 1, DataType::QASYMM8_SIGNED, DataType::QASYMM8, DataType::U8, DataType::S16, DataType::U16, DataType::BFLOAT16, DataType::F16, DataType::U32, DataType::S32, DataType::F32); @@ -141,6 +148,13 @@ Status validate_arguments(const ITensorInfo *src, const ITensorInfo *dst, Conver && dst->data_type() != DataType::F16 && dst->data_type() != DataType::F32 && dst->data_type() != DataType::U8), "Only data_types supported [in] S32 -> [out] QASYMM8, F16, F32, U8"); +#ifdef __aarch64__ + ARM_COMPUTE_RETURN_ERROR_ON_MSG(src->data_type() == DataType::S64 && dst->data_type() != DataType::F32, + "Only data_types supported [in] S64 -> [out] F32"); + + ARM_COMPUTE_RETURN_ERROR_ON_MSG(src->data_type() == DataType::U64 && dst->data_type() != DataType::F32, + "Only data_types supported [in] U64 -> [out] F32"); +#endif // __aarch64__ // Validate in case of configured dst if(dst->total_size() > 0) @@ -174,6 +188,111 @@ Status CpuCastKernel::validate(const ITensorInfo *src, const ITensorInfo *dst, C ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(src, dst, policy)); return Status{}; } +#ifdef __aarch64__ +namespace +{ +template +inline void internal_neon_convert(const T1 *src_ptr, T2 *dst_ptr) +{ + ARM_COMPUTE_UNUSED(src_ptr); + ARM_COMPUTE_UNUSED(dst_ptr); +} + +template <> +inline void internal_neon_convert(const int64_t *src_ptr, float *dst_ptr) +{ + const float64x2x4_t texels0 = + { + { + vcvtq_f64_s64(vld1q_s64(src_ptr)), + vcvtq_f64_s64(vld1q_s64(src_ptr + 2)), + vcvtq_f64_s64(vld1q_s64(src_ptr + 4)), + vcvtq_f64_s64(vld1q_s64(src_ptr + 6)) + } + }; + const float64x2x4_t texels1 = + { + { + vcvtq_f64_s64(vld1q_s64(src_ptr + 8)), + vcvtq_f64_s64(vld1q_s64(src_ptr + 10)), + vcvtq_f64_s64(vld1q_s64(src_ptr + 12)), + vcvtq_f64_s64(vld1q_s64(src_ptr + 14)) + } + }; + const float32x4x4_t texels = + { + { + vcombine_f32(vcvt_f32_f64(texels0.val[0]), vcvt_f32_f64(texels0.val[1])), + vcombine_f32(vcvt_f32_f64(texels0.val[2]), vcvt_f32_f64(texels0.val[3])), + vcombine_f32(vcvt_f32_f64(texels1.val[0]), vcvt_f32_f64(texels1.val[1])), + vcombine_f32(vcvt_f32_f64(texels1.val[2]), vcvt_f32_f64(texels1.val[3])) + } + }; + vst1q_f32(dst_ptr, texels.val[0]); + vst1q_f32(dst_ptr + 4, texels.val[1]); + vst1q_f32(dst_ptr + 8, texels.val[2]); + vst1q_f32(dst_ptr + 12, texels.val[3]); +} + +template <> +inline void internal_neon_convert(const uint64_t *src_ptr, float *dst_ptr) +{ + const float64x2x4_t texels0 = + { + { + vcvtq_f64_u64(vld1q_u64(src_ptr)), + vcvtq_f64_u64(vld1q_u64(src_ptr + 2)), + vcvtq_f64_u64(vld1q_u64(src_ptr + 4)), + vcvtq_f64_u64(vld1q_u64(src_ptr + 6)) + } + }; + const float64x2x4_t texels1 = + { + { + vcvtq_f64_u64(vld1q_u64(src_ptr + 8)), + vcvtq_f64_u64(vld1q_u64(src_ptr + 10)), + vcvtq_f64_u64(vld1q_u64(src_ptr + 12)), + vcvtq_f64_u64(vld1q_u64(src_ptr + 14)) + } + }; + + const float32x4x4_t texels = + { + { + vcombine_f32(vcvt_f32_f64(texels0.val[0]), vcvt_f32_f64(texels0.val[1])), + vcombine_f32(vcvt_f32_f64(texels0.val[2]), vcvt_f32_f64(texels0.val[3])), + vcombine_f32(vcvt_f32_f64(texels1.val[0]), vcvt_f32_f64(texels1.val[1])), + vcombine_f32(vcvt_f32_f64(texels1.val[2]), vcvt_f32_f64(texels1.val[3])) + } + }; + + vst1q_f32(dst_ptr, texels.val[0]); + vst1q_f32(dst_ptr + 4, texels.val[1]); + vst1q_f32(dst_ptr + 8, texels.val[2]); + vst1q_f32(dst_ptr + 12, texels.val[3]); +} + +template +inline void convert64(Iterator &src, Iterator &dst, const Window &win, int window_start_x, int window_end_x, int window_step_x) +{ + execute_window_loop(win, [&](const Coordinates &) + { + const auto src_ptr = reinterpret_cast(src.ptr()); + const auto dst_ptr = reinterpret_cast(dst.ptr()); + int x = window_start_x; + for(; x <= (window_end_x - window_step_x); x += window_step_x) + { + internal_neon_convert(src_ptr + x, dst_ptr + x); + } + for(; x < window_end_x; ++x) + { + *(dst_ptr + x) = static_cast(*(src_ptr + x)); + } + }, + src, dst); +} +} // namespace +#endif // __aarch64__ void CpuCastKernel::run_op(ITensorPack &tensors, const Window &window, const ThreadInfo &info) { @@ -203,6 +322,37 @@ void CpuCastKernel::run_op(ITensorPack &tensors, const Window &window, const Thr switch(_src->info()->data_type()) { +#ifdef __aarch64__ + case DataType::U64: + { + switch(_dst->info()->data_type()) + { + case DataType::F32: + { + convert64(src, dst, win, window_start_x, window_end_x, window_step_x); + break; + } + default: + ARM_COMPUTE_ERROR("dst data type not supported"); + } + break; + } + case DataType::S64: + { + switch(_dst->info()->data_type()) + { + case DataType::F32: + { + convert64(src, dst, win, window_start_x, window_end_x, window_step_x); + break; + } + default: + ARM_COMPUTE_ERROR("dst data type not supported"); + } + break; + } +#endif // __aarch64__ + case DataType::QASYMM8_SIGNED: { switch(_dst->info()->data_type()) @@ -909,7 +1059,6 @@ void CpuCastKernel::run_op(ITensorPack &tensors, const Window &window, const Thr ARM_COMPUTE_ERROR("dst data type not supported"); } break; - case DataType::S32: switch(_dst->info()->data_type()) { diff --git a/src/cpu/kernels/CpuCastKernel.h b/src/cpu/kernels/CpuCastKernel.h index de4ace21409f2b26bfc634090d6f2d0d44f62389..76237368d87d195ffec154683aeb561b48f8a479 100644 --- a/src/cpu/kernels/CpuCastKernel.h +++ b/src/cpu/kernels/CpuCastKernel.h @@ -57,9 +57,10 @@ public: * - BFLOAT16 -> F32 * - F16 -> QASYMM8_SIGNED, QASYMM8, F32, S32, U8 * - S32 -> QASYMM8_SIGNED, QASYMM8, F16, F32, U8 + * - S64 -> F32 * - F32 -> QASYMM8_SIGNED, QASYMM8, BFLOAT16, F16, S32, U8 * - * @param[in] src The src tensor to convert. Data types supported: QASYMM8_SIGNED/QASYMM8/U8/U16/S16/BFLOAT16/F16/F32. + * @param[in] src The src tensor to convert. Data types supported: QASYMM8_SIGNED/QASYMM8/U8/U16/S16/S32/S64/BFLOAT16/F16/F32. * @param[out] dst The dst tensor. Data types supported: QASYMM8_SIGNED/QASYMM8/U8/U16/S16/U32/S32/BFLOAT16/F16/F32. * @param[in] policy Conversion policy. * diff --git a/src/cpu/kernels/CpuConcatenateWidthKernel.cpp b/src/cpu/kernels/CpuConcatenateWidthKernel.cpp index 90813ff7b48994e193a480aac45bd6380123e62d..f00b37a01bfd6ce7ed72666ad08dc6213452efa2 100644 --- a/src/cpu/kernels/CpuConcatenateWidthKernel.cpp +++ b/src/cpu/kernels/CpuConcatenateWidthKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,18 +24,12 @@ #include "src/cpu/kernels/CpuConcatenateWidthKernel.h" #include "arm_compute/core/Error.h" +#include "arm_compute/core/Steps.h" #include "arm_compute/core/Helpers.h" -#include "arm_compute/core/ITensor.h" -#include "arm_compute/core/TensorInfo.h" -#include "arm_compute/core/Utils.h" #include "arm_compute/core/Validate.h" -#include "arm_compute/core/Window.h" #include "src/core/NEON/NEAsymm.h" -#include "src/core/NEON/wrapper/wrapper.h" -#include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" -#include namespace arm_compute { diff --git a/src/cpu/kernels/CpuDepthwiseConv2dNativeKernel.cpp b/src/cpu/kernels/CpuDepthwiseConv2dNativeKernel.cpp index f47df1e405e304761f33831e45b199a8665fced6..d6c56d2012ab5fe1cf1f95ab21c668f8908a192b 100644 --- a/src/cpu/kernels/CpuDepthwiseConv2dNativeKernel.cpp +++ b/src/cpu/kernels/CpuDepthwiseConv2dNativeKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022 Arm Limited. + * Copyright (c) 2019-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,7 +32,6 @@ #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" #include "src/cpu/kernels/depthwiseconv2d/list.h" -#include "support/ToolchainSupport.h" namespace arm_compute { diff --git a/src/cpu/kernels/CpuDepthwiseConv2dNativeKernel.h b/src/cpu/kernels/CpuDepthwiseConv2dNativeKernel.h index 8bc4d83c20568e282c50898d56dff8e2719651cc..9fabd0b01cf24a353830da9b8fcc5494df8d62b9 100644 --- a/src/cpu/kernels/CpuDepthwiseConv2dNativeKernel.h +++ b/src/cpu/kernels/CpuDepthwiseConv2dNativeKernel.h @@ -25,6 +25,7 @@ #define ARM_COMPUTE_CPU_DEPTHWISE_CONV2D_NATIVE_KERNEL_H #include "arm_compute/core/utils/misc/Traits.h" +#include "arm_compute/function_info/ConvolutionInfo.h" #include "src/core/common/Macros.h" #include "src/cpu/ICpuKernel.h" #include "support/AclRequires.h" diff --git a/src/cpu/kernels/CpuKernelSelectionTypes.h b/src/cpu/kernels/CpuKernelSelectionTypes.h index edcbff07420a9ad85ed263297825ec9b5118e183..39adc9af7cb01907d0cfdd8cb3091ed0dc4b9ceb 100644 --- a/src/cpu/kernels/CpuKernelSelectionTypes.h +++ b/src/cpu/kernels/CpuKernelSelectionTypes.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -81,7 +81,7 @@ struct ActivationDataTypeISASelectorData DataType dt; const CPUModel &cpumodel; const cpuinfo::CpuIsaInfo &isa; - ActivationLayerInfo::ActivationFunction f; + const ActivationFunction f; }; struct CpuAddKernelDataTypeISASelectorData diff --git a/src/cpu/kernels/CpuMaxUnpoolingLayerKernel.cpp b/src/cpu/kernels/CpuMaxUnpoolingLayerKernel.cpp index 604f22f6cc1b42a9b148cd746f706e099f3f3632..7d077c75bf857179a37b9bca7800a001a43836ef 100644 --- a/src/cpu/kernels/CpuMaxUnpoolingLayerKernel.cpp +++ b/src/cpu/kernels/CpuMaxUnpoolingLayerKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022 Arm Limited. + * Copyright (c) 2020-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,7 +32,6 @@ #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" #include "src/cpu/kernels/maxunpool/list.h" -#include "support/ToolchainSupport.h" namespace arm_compute { @@ -145,4 +144,4 @@ const std::vector &CpuMaxUnpooli } } // namespace kernels } // namespace cpu -} // namespace arm_compute \ No newline at end of file +} // namespace arm_compute diff --git a/src/cpu/kernels/CpuMulKernel.h b/src/cpu/kernels/CpuMulKernel.h index 0d6f58611710802762892de5e5aa564957cc56ab..9e4a37110b5f7e76c75ce4aa6158c0b3036f5410 100644 --- a/src/cpu/kernels/CpuMulKernel.h +++ b/src/cpu/kernels/CpuMulKernel.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2022 Arm Limited. + * Copyright (c) 2016-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CPU_MUL_KERNEL_H #define ARM_COMPUTE_CPU_MUL_KERNEL_H +#include "arm_compute/core/Rounding.h" #include "src/core/common/Macros.h" #include "src/cpu/ICpuKernel.h" diff --git a/src/cpu/kernels/CpuScaleKernel.cpp b/src/cpu/kernels/CpuScaleKernel.cpp index b8bb5ad18a5fd69e5e2a8cedfe9cdf7d6ca712e9..332304599f628eb65b27e07e24fd7c8985b02f7c 100644 --- a/src/cpu/kernels/CpuScaleKernel.cpp +++ b/src/cpu/kernels/CpuScaleKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2022 Arm Limited. + * Copyright (c) 2016-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #include "src/cpu/kernels/CpuScaleKernel.h" #include "arm_compute/core/Helpers.h" +#include "arm_compute/core/utils/InterpolationPolicyUtils.h" #include "arm_compute/core/Window.h" #include "src/core/common/Registrars.h" #include "src/core/helpers/ScaleHelpers.h" @@ -136,10 +137,10 @@ Status validate_arguments(const ITensorInfo *src, const ITensorInfo *dx, const I const auto *uk = CpuScaleKernel::get_implementation(ScaleKernelDataTypeISASelectorData{ src->data_type(), CPUInfo::get().get_isa(), info.interpolation_policy }); ARM_COMPUTE_RETURN_ERROR_ON(uk == nullptr || uk->ukernel == nullptr); - ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(dst); ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(src, dst); ARM_COMPUTE_RETURN_ERROR_ON(dst == src); + ARM_COMPUTE_RETURN_ERROR_ON(src->num_channels()!=1); ARM_COMPUTE_RETURN_ERROR_ON(info.sampling_policy != SamplingPolicy::CENTER && info.sampling_policy != SamplingPolicy::TOP_LEFT); ARM_COMPUTE_UNUSED(info.constant_border_value); ARM_COMPUTE_RETURN_ERROR_ON_MSG(info.use_padding, "Padding is not supported"); diff --git a/src/cpu/kernels/CpuSoftmaxKernel.cpp b/src/cpu/kernels/CpuSoftmaxKernel.cpp index 93cce785bd319df8f1635b0704c05f72233c7d86..e06ab9917c8567f0d685f29ba2b72da38f16d981 100644 --- a/src/cpu/kernels/CpuSoftmaxKernel.cpp +++ b/src/cpu/kernels/CpuSoftmaxKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2022 Arm Limited. + * Copyright (c) 2017-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,6 +27,7 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/ITensor.h" #include "arm_compute/core/TensorInfo.h" +#include "arm_compute/core/Utils.h" #include "arm_compute/core/Validate.h" #include "arm_compute/core/Window.h" #include "src/core/CPP/Validate.h" diff --git a/src/cpu/kernels/activation/generic/neon/impl.h b/src/cpu/kernels/activation/generic/neon/impl.h index 4f392866b55e69088e9400715eed6d409658a0e3..05885d84769e997119aa125b3a7fba0bd4ea68a4 100644 --- a/src/cpu/kernels/activation/generic/neon/impl.h +++ b/src/cpu/kernels/activation/generic/neon/impl.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022 Arm Limited. + * Copyright (c) 2020-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,6 +23,7 @@ */ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/Window.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/core/NEON/wrapper/wrapper.h" namespace arm_compute { @@ -212,7 +213,7 @@ void fp_neon_activation_impl(const ITensor *src, ITensor *dst, const ActivationL tmp = in * ((std::min(std::max((in + 3), 0.0f), 6.0f)) * 0.166666667f); break; case ActivationLayerInfo::ActivationFunction::SWISH: - tmp = in / (static_cast(1) + std::exp(-a*in)); + tmp = in / (static_cast(1) + std::exp(-a * in)); break; case ActivationLayerInfo::ActivationFunction::GELU: tmp = in * static_cast(0.5f * (1.0f + erff(static_cast(in) / 1.41421356237f))); diff --git a/src/cpu/kernels/activation/generic/neon/lut.cpp b/src/cpu/kernels/activation/generic/neon/lut.cpp index 90690ffcaa88a0a28e86b7f9932b40c6f1034df1..c973e964e4f93664c218a20acab190d2848f346a 100644 --- a/src/cpu/kernels/activation/generic/neon/lut.cpp +++ b/src/cpu/kernels/activation/generic/neon/lut.cpp @@ -23,6 +23,7 @@ */ #include "arm_compute/core/Helpers.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/cpu/kernels/lut/list.h" namespace arm_compute @@ -32,7 +33,7 @@ namespace cpu #ifdef __aarch64__ void neon_q8_activation_lut(const ITensor *src, ITensor *dst, const ActivationLayerInfo &act_info, const Window &window) { - ARM_COMPUTE_ERROR_ON(!ActivationLayerInfo::is_lut_supported(act_info.activation(), src->info()->data_type())); + ARM_COMPUTE_ERROR_ON(src->info()->data_type() != DataType::QASYMM8 && src->info()->data_type() != DataType::QASYMM8_SIGNED); const auto window_end_x = window.x().end(); Window win_collapsed = window.collapse_if_possible(window, Window::DimZ); win_collapsed.set(Window::DimX, Window::Dimension(0, 1, 1)); diff --git a/src/cpu/kernels/activation/generic/neon/qasymm8.cpp b/src/cpu/kernels/activation/generic/neon/qasymm8.cpp index f5555574cb5df58c4c3017f2822de67efecaaebd..e7c146e46f1be02a68bac3f038427a1125446d4e 100644 --- a/src/cpu/kernels/activation/generic/neon/qasymm8.cpp +++ b/src/cpu/kernels/activation/generic/neon/qasymm8.cpp @@ -24,6 +24,7 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/Window.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/core/NEON/NEAsymm.h" #include "src/core/NEON/NEMath.h" #include "src/core/NEON/wrapper/wrapper.h" diff --git a/src/cpu/kernels/activation/generic/neon/qasymm8_signed.cpp b/src/cpu/kernels/activation/generic/neon/qasymm8_signed.cpp index d75d0071a2d2c9277a4e89a283aae08b52ed5e73..52c396459b116eae34a4f6747779a22b0121535a 100644 --- a/src/cpu/kernels/activation/generic/neon/qasymm8_signed.cpp +++ b/src/cpu/kernels/activation/generic/neon/qasymm8_signed.cpp @@ -23,6 +23,7 @@ */ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/Window.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/core/NEON/NEAsymm.h" #include "src/core/NEON/NEMath.h" #include "src/core/NEON/wrapper/wrapper.h" diff --git a/src/cpu/kernels/activation/generic/neon/qsymm16.cpp b/src/cpu/kernels/activation/generic/neon/qsymm16.cpp index ba147459388f416f83c161589f9c122525ea606d..2aea6cba3c27e9a268e5259ccb6545f70be7f329 100644 --- a/src/cpu/kernels/activation/generic/neon/qsymm16.cpp +++ b/src/cpu/kernels/activation/generic/neon/qsymm16.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022 Arm Limited. + * Copyright (c) 2020-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #include "arm_compute/core/ITensorPack.h" #include "arm_compute/core/Window.h" #include "arm_compute/core/experimental/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/core/NEON/NEMath.h" #include "src/core/NEON/NESymm.h" #include "src/core/NEON/wrapper/wrapper.h" diff --git a/src/cpu/kernels/activation/generic/sve/fp16.cpp b/src/cpu/kernels/activation/generic/sve/fp16.cpp index 6e9648b5bfdcffd88708fa19f017e1ba605ab737..4757c60d8f80cd62bec73620a9e5462e1c1317f9 100644 --- a/src/cpu/kernels/activation/generic/sve/fp16.cpp +++ b/src/cpu/kernels/activation/generic/sve/fp16.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022 Arm Limited. + * Copyright (c) 2020-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,6 +27,7 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/ITensorPack.h" #include "arm_compute/core/Window.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include #include diff --git a/src/cpu/kernels/activation/generic/sve/fp32.cpp b/src/cpu/kernels/activation/generic/sve/fp32.cpp index 8d5d627d709394fc78fc2568388b1a4201c3d01f..87f04c255ac8205812d8b20ddcd2799c1dbe2e6c 100644 --- a/src/cpu/kernels/activation/generic/sve/fp32.cpp +++ b/src/cpu/kernels/activation/generic/sve/fp32.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022 Arm Limited. + * Copyright (c) 2020-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/ITensorPack.h" #include "arm_compute/core/Window.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/core/NEON/SVEMath.h" #include diff --git a/src/cpu/kernels/activation/generic/sve2/lut.cpp b/src/cpu/kernels/activation/generic/sve2/lut.cpp index 2e5975744b83c9892d2fea41aad92dae13730b9b..d65de8d64988798e720bbfbdb00d06da62677f20 100644 --- a/src/cpu/kernels/activation/generic/sve2/lut.cpp +++ b/src/cpu/kernels/activation/generic/sve2/lut.cpp @@ -23,6 +23,7 @@ */ #include "arm_compute/core/Helpers.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/cpu/kernels/lut/list.h" namespace arm_compute @@ -32,7 +33,7 @@ namespace cpu #ifdef __aarch64__ void sve2_q8_activation_lut(const ITensor *src, ITensor *dst, const ActivationLayerInfo &act_info, const Window &window) { - ARM_COMPUTE_ERROR_ON(!ActivationLayerInfo::is_lut_supported(act_info.activation(), src->info()->data_type())); + ARM_COMPUTE_ERROR_ON(src->info()->data_type() != DataType::QASYMM8 && src->info()->data_type() != DataType::QASYMM8_SIGNED); const auto window_end_x = window.x().end(); Window win_collapsed = window.collapse_if_possible(window, Window::DimZ); win_collapsed.set(Window::DimX, Window::Dimension(0, 1, 1)); diff --git a/src/cpu/kernels/activation/generic/sve2/qasymm8.cpp b/src/cpu/kernels/activation/generic/sve2/qasymm8.cpp index 928a414fb0e7ffd9f76d06a14d2fca9af3b3a2ac..bc9bc7aa3cb352fe3d884cca81ade63fc423c32c 100644 --- a/src/cpu/kernels/activation/generic/sve2/qasymm8.cpp +++ b/src/cpu/kernels/activation/generic/sve2/qasymm8.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022 Arm Limited. + * Copyright (c) 2020-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/Window.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include #include @@ -48,16 +49,15 @@ void sve2_qasymm8_activation(const ITensor *src, ITensor *dst, const ActivationL Iterator input(src, win_collapsed); Iterator output(dst, win_collapsed); - const UniformQuantizationInfo qi_in = src->info()->quantization_info().uniform(); - const UniformQuantizationInfo qi_out = dst->info()->quantization_info().uniform(); - const auto va = svdup_n_u8(quantize_qasymm8(act_info.a(), qi_in)); - const auto vb = svdup_n_u8(quantize_qasymm8(act_info.b(), qi_in)); - const auto const_0 = quantize_qasymm8(0.f, qi_in); - const auto vconst_0 = svdup_n_u8(const_0); - const auto vconst_1 = svdup_n_f32(1.f); - const auto va_f32 = svdup_n_f32(act_info.a()); - const auto vb_f32 = svdup_n_f32(act_info.b()); - + const UniformQuantizationInfo qi_in = src->info()->quantization_info().uniform(); + const UniformQuantizationInfo qi_out = dst->info()->quantization_info().uniform(); + const auto va = svdup_n_u8(quantize_qasymm8(act_info.a(), qi_in)); + const auto vb = svdup_n_u8(quantize_qasymm8(act_info.b(), qi_in)); + const auto const_0 = quantize_qasymm8(0.f, qi_in); + const auto vconst_0 = svdup_n_u8(const_0); + const auto vconst_1 = svdup_n_f32(1.f); + const auto va_f32 = svdup_n_f32(act_info.a()); + const auto vb_f32 = svdup_n_f32(act_info.b()); // Initialise scale/offset for re-quantization bool requant = true; diff --git a/src/cpu/kernels/activation/generic/sve2/qasymm8_signed.cpp b/src/cpu/kernels/activation/generic/sve2/qasymm8_signed.cpp index da4f5e222c985f718be7253ea26371805248cefd..d20684f54dc0662d827393866a9a242893f8c253 100644 --- a/src/cpu/kernels/activation/generic/sve2/qasymm8_signed.cpp +++ b/src/cpu/kernels/activation/generic/sve2/qasymm8_signed.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022 Arm Limited. + * Copyright (c) 2020-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,6 +23,7 @@ */ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/Window.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/core/NEON/wrapper/wrapper.h" #include diff --git a/src/cpu/kernels/activation/generic/sve2/qsymm16.cpp b/src/cpu/kernels/activation/generic/sve2/qsymm16.cpp index 1d6f68273a3d7d612dd114a288a0b4a7986eae5f..5154fac8a7ef2d03e9a91aa8651322602b8932ea 100644 --- a/src/cpu/kernels/activation/generic/sve2/qsymm16.cpp +++ b/src/cpu/kernels/activation/generic/sve2/qsymm16.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022 Arm Limited. + * Copyright (c) 2020-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #include "arm_compute/core/ITensorPack.h" #include "arm_compute/core/Window.h" #include "arm_compute/core/experimental/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include #include diff --git a/src/cpu/kernels/add/generic/neon/fp16.cpp b/src/cpu/kernels/add/generic/neon/fp16.cpp index 1e3bc3c986304d240802e55fac48971759a43597..fca7b2cd9fdaf2187e2a0e7d88ba928cdf45a106 100644 --- a/src/cpu/kernels/add/generic/neon/fp16.cpp +++ b/src/cpu/kernels/add/generic/neon/fp16.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,6 +23,7 @@ */ #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) +#include "src/cpu/CpuTypes.h" #include "src/cpu/kernels/add/generic/neon/impl.h" namespace arm_compute diff --git a/src/cpu/kernels/add/generic/sve/fp16.cpp b/src/cpu/kernels/add/generic/sve/fp16.cpp index 65a1369bfcb052a037cb3db8b8f6371c63a1da4b..581f3abded3001663f5e6fd6f822a8e664d130f1 100644 --- a/src/cpu/kernels/add/generic/sve/fp16.cpp +++ b/src/cpu/kernels/add/generic/sve/fp16.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) +#include "src/cpu/CpuTypes.h" #include "src/cpu/kernels/add/generic/sve/impl.h" namespace arm_compute diff --git a/src/cpu/kernels/addmuladd/generic/neon/fp16.cpp b/src/cpu/kernels/addmuladd/generic/neon/fp16.cpp index 2a7e602b793741df62f55495459fe4208369a554..d8e5f694a844e5fb5f192511bb77bbeb93a76f9c 100644 --- a/src/cpu/kernels/addmuladd/generic/neon/fp16.cpp +++ b/src/cpu/kernels/addmuladd/generic/neon/fp16.cpp @@ -26,6 +26,8 @@ #include "arm_compute/core/ITensor.h" #include "arm_compute/core/Types.h" #include "arm_compute/core/Window.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" +#include "src/cpu/CpuTypes.h" #include #include @@ -34,6 +36,8 @@ #if defined(__aarch64__) && defined(ENABLE_FP16_KERNELS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace { +using arm_compute::float16_t; + void a64_add_bn_clamp_direct_fp16_2x32( float16_t *out, size_t out_stride, float16_t *out_direct, size_t out_direct_stride, diff --git a/src/cpu/kernels/addmuladd/generic/neon/fp32.cpp b/src/cpu/kernels/addmuladd/generic/neon/fp32.cpp index a102682f8ba822f2d454b48645ca27b26c88d580..b0c487ec56a5e6b34495beaa7934ed9f6cf8fdf5 100644 --- a/src/cpu/kernels/addmuladd/generic/neon/fp32.cpp +++ b/src/cpu/kernels/addmuladd/generic/neon/fp32.cpp @@ -26,6 +26,7 @@ #include "arm_compute/core/ITensor.h" #include "arm_compute/core/Types.h" #include "arm_compute/core/Window.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include #include diff --git a/src/cpu/kernels/addmuladd/generic/neon/qasymm8.cpp b/src/cpu/kernels/addmuladd/generic/neon/qasymm8.cpp index 2c0ad70f8f609b07569bafa4099fc8db9d4cff4d..f7448a6717e426de855d41b4eb22290c873ce52d 100644 --- a/src/cpu/kernels/addmuladd/generic/neon/qasymm8.cpp +++ b/src/cpu/kernels/addmuladd/generic/neon/qasymm8.cpp @@ -27,6 +27,7 @@ #include "arm_compute/core/QuantizationInfo.h" #include "arm_compute/core/Types.h" #include "arm_compute/core/Window.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include #include diff --git a/src/cpu/kernels/addmuladd/generic/neon/qasymm8_signed.cpp b/src/cpu/kernels/addmuladd/generic/neon/qasymm8_signed.cpp index 3bde83cc267cc3a3a379444942bc788609483756..1ae2cb76a99f0632d2a5763ba04527b82ab4a441 100644 --- a/src/cpu/kernels/addmuladd/generic/neon/qasymm8_signed.cpp +++ b/src/cpu/kernels/addmuladd/generic/neon/qasymm8_signed.cpp @@ -27,6 +27,7 @@ #include "arm_compute/core/QuantizationInfo.h" #include "arm_compute/core/Types.h" #include "arm_compute/core/Window.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include #include diff --git a/src/cpu/kernels/boundingboxtransform/generic/neon/fp16.cpp b/src/cpu/kernels/boundingboxtransform/generic/neon/fp16.cpp index 6826ff6691e62697982111e6eeb2d684942e192c..56614790591c2a8a00122b4bd76d5c0e740f7019 100644 --- a/src/cpu/kernels/boundingboxtransform/generic/neon/fp16.cpp +++ b/src/cpu/kernels/boundingboxtransform/generic/neon/fp16.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,9 @@ * SOFTWARE. */ #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) +#include "src/cpu/CpuTypes.h" #include "src/cpu/kernels/boundingboxtransform/generic/neon/impl.h" + namespace arm_compute { namespace cpu diff --git a/src/cpu/kernels/boundingboxtransform/generic/neon/impl.cpp b/src/cpu/kernels/boundingboxtransform/generic/neon/impl.cpp index 2d08c879cc3ed5ac3262288a637392408652d6b2..d74a8a712d8f9afee19823ee3eaa8fab689d9d7d 100644 --- a/src/cpu/kernels/boundingboxtransform/generic/neon/impl.cpp +++ b/src/cpu/kernels/boundingboxtransform/generic/neon/impl.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022 Arm Limited. + * Copyright (c) 2019-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,9 @@ * SOFTWARE. */ #include "src/cpu/kernels/boundingboxtransform/generic/neon/impl.h" + +#include "src/cpu/CpuTypes.h" + namespace arm_compute { namespace cpu @@ -143,4 +146,4 @@ template void bounding_box_transform(const ITensor *boxes, ITensor *pred_ template void bounding_box_transform(const ITensor *boxes, ITensor *pred_boxes, const ITensor *deltas, BoundingBoxTransformInfo bbinfo, const Window &window); #endif //defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) } // namespace cpu -} // namespace arm_compute \ No newline at end of file +} // namespace arm_compute diff --git a/src/cpu/kernels/cast/generic/neon/fp16.cpp b/src/cpu/kernels/cast/generic/neon/fp16.cpp index d2c66923ccbe3679543270ecbc980ce630d12b89..385ca1898d3e372abd8e7540ffec7c75d21cada1 100644 --- a/src/cpu/kernels/cast/generic/neon/fp16.cpp +++ b/src/cpu/kernels/cast/generic/neon/fp16.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2022 Arm Limited. + * Copyright (c) 2016-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -28,6 +28,8 @@ #include "src/cpu/kernels/cast/list.h" #include "support/SaturateCast.h" +#include "arm_neon.h" + namespace arm_compute { namespace cpu diff --git a/src/cpu/kernels/depthwiseconv2d/generic/neon/fp16.cpp b/src/cpu/kernels/depthwiseconv2d/generic/neon/fp16.cpp index 553d816ff0f77a0de1a1633cb11fa2f895ce3a84..e85a1664ea7b2e3c4742b55587ab807d907265ea 100644 --- a/src/cpu/kernels/depthwiseconv2d/generic/neon/fp16.cpp +++ b/src/cpu/kernels/depthwiseconv2d/generic/neon/fp16.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,9 @@ * SOFTWARE. */ #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC +#include "src/cpu/CpuTypes.h" #include "src/cpu/kernels/depthwiseconv2d/generic/neon/impl.h" + namespace arm_compute { namespace cpu diff --git a/src/cpu/kernels/depthwiseconv2d/generic/neon/impl.cpp b/src/cpu/kernels/depthwiseconv2d/generic/neon/impl.cpp index 350e25ed9439ad8a8b684727b48441c1f338c7d3..f12825477140ab74fd7a001c2b999f38b6197747 100644 --- a/src/cpu/kernels/depthwiseconv2d/generic/neon/impl.cpp +++ b/src/cpu/kernels/depthwiseconv2d/generic/neon/impl.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022 Arm Limited. + * Copyright (c) 2019-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,6 +23,7 @@ */ #include "src/cpu/kernels/depthwiseconv2d/generic/neon/impl.h" #include "arm_compute/core/utils/quantization/AsymmHelpers.h" +#include "arm_compute/function_info/ConvolutionInfo.h" #include "src/core/NEON/wrapper/wrapper.h" namespace arm_compute diff --git a/src/cpu/kernels/depthwiseconv2d/generic/neon/impl.h b/src/cpu/kernels/depthwiseconv2d/generic/neon/impl.h index a7ba28654131655c920d7b99f17fc9ec8965f7a5..1f01ce43d9fbdd9e8db3c93c3e8cb4891741ea58 100644 --- a/src/cpu/kernels/depthwiseconv2d/generic/neon/impl.h +++ b/src/cpu/kernels/depthwiseconv2d/generic/neon/impl.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,8 @@ #include "arm_compute/core/Helpers.h" namespace arm_compute { +struct ConvolutionInfo; + namespace cpu { template diff --git a/src/cpu/kernels/directconv2d/nhwc/neon/impl.h b/src/cpu/kernels/directconv2d/nhwc/neon/impl.h index 88a151fba47418e96a5d327ea8e497f0baf16626..3b26fcdf2954be4677b2be14f2dc8bd77790b78a 100644 --- a/src/cpu/kernels/directconv2d/nhwc/neon/impl.h +++ b/src/cpu/kernels/directconv2d/nhwc/neon/impl.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #ifndef SRC_CORE_NEON_KERNELS_CONV2D_IMPL_H #define SRC_CORE_NEON_KERNELS_CONV2D_IMPL_H +#include "arm_compute/core/ITensor.h" #include "src/core/helpers/WindowHelpers.h" namespace arm_compute diff --git a/src/cpu/kernels/elementwise_unary/generic/sve/fp16.cpp b/src/cpu/kernels/elementwise_unary/generic/sve/fp16.cpp index 01567a7852309cee790033f36587fafc5beaeb4f..a883309b2ecb2ee55d945388b45b68c8808b2228 100644 --- a/src/cpu/kernels/elementwise_unary/generic/sve/fp16.cpp +++ b/src/cpu/kernels/elementwise_unary/generic/sve/fp16.cpp @@ -23,6 +23,7 @@ */ #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) #include "arm_compute/core/Helpers.h" +#include "src/cpu/CpuTypes.h" #include "src/cpu/kernels/elementwise_unary/generic/sve/impl.h" namespace arm_compute diff --git a/src/cpu/kernels/elementwise_unary/generic/sve/fp32.cpp b/src/cpu/kernels/elementwise_unary/generic/sve/fp32.cpp index 47645ff80f5476122f74c16c9ed8135eb254d6ff..b21ed8ddbc9d5b99852206b457c793bcd97e1fe5 100644 --- a/src/cpu/kernels/elementwise_unary/generic/sve/fp32.cpp +++ b/src/cpu/kernels/elementwise_unary/generic/sve/fp32.cpp @@ -23,6 +23,7 @@ */ #include "arm_compute/core/Helpers.h" +#include "src/cpu/CpuTypes.h" #include "src/cpu/kernels/elementwise_unary/generic/sve/impl.h" namespace arm_compute diff --git a/src/cpu/kernels/instancenorm/generic/neon/impl.h b/src/cpu/kernels/instancenorm/generic/neon/impl.h index fa4b4b656c5667e4af45bd7437a953f5fc75cb5a..1d413a9bcd8240f9708357f9caeec933b126d0ca 100644 --- a/src/cpu/kernels/instancenorm/generic/neon/impl.h +++ b/src/cpu/kernels/instancenorm/generic/neon/impl.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,9 @@ #ifndef SRC_CORE_SVE_KERNELS_INSTANCENORM_IMPL_H #define SRC_CORE_SVE_KERNELS_INSTANCENORM_IMPL_H #include "arm_compute/core/Helpers.h" + +#include "arm_neon.h" + namespace arm_compute { namespace cpu diff --git a/src/cpu/kernels/internal/CpuDepthwiseConv2dAssemblyWrapperKernel.cpp b/src/cpu/kernels/internal/CpuDepthwiseConv2dAssemblyWrapperKernel.cpp index 5360abf5ac1ee3d1341d680b038a812af75b3b76..b503a8b734856b8aaf808b8a3b08dfeaa270b962 100644 --- a/src/cpu/kernels/internal/CpuDepthwiseConv2dAssemblyWrapperKernel.cpp +++ b/src/cpu/kernels/internal/CpuDepthwiseConv2dAssemblyWrapperKernel.cpp @@ -303,6 +303,20 @@ Status CpuDepthwiseConv2dAssemblyWrapperKernel::validate(const ITensorInfo *src, ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DIMENSIONS(dst->tensor_shape(), dst_shape); ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(src, dst); } + + // Assembly kernels cannot work with padding greater than the kernel. + const auto &padding = info.pad_stride_info; + const auto &dilation = info.dilation; + const auto &wei_shape = weights->tensor_shape(); + + const auto dilated_wei_w = wei_shape[1] + (wei_shape[1] - 1) * (dilation.x() - 1); + const auto dilated_wei_h = wei_shape[2] + (wei_shape[2] - 1) * (dilation.y() - 1); + + ARM_COMPUTE_RETURN_ERROR_ON( + padding.pad_left() >= dilated_wei_w || padding.pad_right() >= dilated_wei_w || + padding.pad_top() >= dilated_wei_h || padding.pad_bottom() >= dilated_wei_h + ); + return Status{}; } @@ -353,9 +367,9 @@ size_t CpuDepthwiseConv2dAssemblyWrapperKernel::get_storage_size() const return _kernel_asm->get_storage_size(); } -size_t CpuDepthwiseConv2dAssemblyWrapperKernel::get_working_size(unsigned int num_threads, unsigned int num_input_channels) const +size_t CpuDepthwiseConv2dAssemblyWrapperKernel::get_working_size(unsigned int num_threads) const { - return _kernel_asm->get_working_size(num_threads, num_input_channels); + return _kernel_asm->get_working_size(num_threads); } bool CpuDepthwiseConv2dAssemblyWrapperKernel::is_configured() const diff --git a/src/cpu/kernels/internal/CpuDepthwiseConv2dAssemblyWrapperKernel.h b/src/cpu/kernels/internal/CpuDepthwiseConv2dAssemblyWrapperKernel.h index a32a7a3ec867daa5231129024bd4d4fdd6b4da55..f61cb1b09c4572eeffb35630ebf1ecad6f8339b4 100644 --- a/src/cpu/kernels/internal/CpuDepthwiseConv2dAssemblyWrapperKernel.h +++ b/src/cpu/kernels/internal/CpuDepthwiseConv2dAssemblyWrapperKernel.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022 Arm Limited. + * Copyright (c) 2019-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -40,6 +40,8 @@ class IDepthwiseCommon; namespace arm_compute { +struct ConvolutionInfo; + namespace cpu { namespace kernels @@ -96,12 +98,11 @@ public: /** Get size of the workspace needed by the assembly kernel. * - * @param[in] num_threads Maximum number of threads that are going to be spawned. - * @param[in] num_input_channels Number of channels of the input tensor. + * @param[in] num_threads Maximum number of threads that are going to be spawned. * * @return size of workspace */ - size_t get_working_size(unsigned int num_threads, unsigned int num_input_channels) const; + size_t get_working_size(unsigned int num_threads) const; /** Was the asm kernel successfully configured? * diff --git a/src/cpu/kernels/l2normlayer/generic/neon/fp16.cpp b/src/cpu/kernels/l2normlayer/generic/neon/fp16.cpp index ed84c10d7250dc6367d421c05f520271d3f8144c..661c3d7f46717d825618f38f9c20fdac578a0a8a 100644 --- a/src/cpu/kernels/l2normlayer/generic/neon/fp16.cpp +++ b/src/cpu/kernels/l2normlayer/generic/neon/fp16.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,9 +23,11 @@ */ #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) +#include "arm_compute/core/Helpers.h" + +#include "src/cpu/CpuTypes.h" #include "src/cpu/kernels/l2normlayer/generic/neon/impl.h" -#include "arm_compute/core/Helpers.h" namespace arm_compute { namespace cpu diff --git a/src/cpu/kernels/meanstddevnorm/generic/neon/fp16.cpp b/src/cpu/kernels/meanstddevnorm/generic/neon/fp16.cpp index 3e712b5593642318b34cb757e0df7a0df48a9223..47bf64ae578c75c2aa1cc764b50000190cfc5221 100644 --- a/src/cpu/kernels/meanstddevnorm/generic/neon/fp16.cpp +++ b/src/cpu/kernels/meanstddevnorm/generic/neon/fp16.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,7 @@ * SOFTWARE. */ #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) +#include "src/cpu/CpuTypes.h" #include "src/cpu/kernels/meanstddevnorm/generic/neon/impl.h" namespace arm_compute diff --git a/src/cpu/kernels/pool3d/neon/fp16.cpp b/src/cpu/kernels/pool3d/neon/fp16.cpp index b79bcd93b5cef1bc169b1a3c567191c48f1053b3..0130a960982929d106b767bdbc6e3f1c1af7b787 100644 --- a/src/cpu/kernels/pool3d/neon/fp16.cpp +++ b/src/cpu/kernels/pool3d/neon/fp16.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,9 @@ * SOFTWARE. */ #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) +#include "src/cpu/CpuTypes.h" #include "src/cpu/kernels/pool3d/neon/impl.h" + namespace arm_compute { namespace cpu @@ -33,4 +35,4 @@ void neon_fp16_pool3d(const ITensor *src, ITensor *dst0, Pooling3dLayerInfo &poo } } // namespace cpu } // namespace arm_compute -#endif /* defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) */ \ No newline at end of file +#endif /* defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) */ diff --git a/src/cpu/kernels/roialign/generic/neon/fp16.cpp b/src/cpu/kernels/roialign/generic/neon/fp16.cpp index 6e585a4dfae2a183560813b635282ac08394e9dc..c265d5d4eb410c45f9d4e4a0c4fc9cf3438b3051 100644 --- a/src/cpu/kernels/roialign/generic/neon/fp16.cpp +++ b/src/cpu/kernels/roialign/generic/neon/fp16.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,9 @@ * SOFTWARE. */ #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) +#include "src/cpu/CpuTypes.h" #include "src/cpu/kernels/roialign/generic/neon/impl.h" + namespace arm_compute { namespace cpu diff --git a/src/cpu/kernels/roialign/generic/neon/impl.cpp b/src/cpu/kernels/roialign/generic/neon/impl.cpp index a4502e703d4c7c83fff629e1daee02487a9fba0c..630d649abaef12dd6f8dfea74bc00fe9f90703a1 100644 --- a/src/cpu/kernels/roialign/generic/neon/impl.cpp +++ b/src/cpu/kernels/roialign/generic/neon/impl.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022 Arm Limited. + * Copyright (c) 2019-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,10 @@ * SOFTWARE. */ #include "src/cpu/kernels/roialign/generic/neon/impl.h" + #include "src/core/NEON/INEKernel.h" +#include "src/cpu/CpuTypes.h" + namespace arm_compute { namespace cpu diff --git a/src/cpu/kernels/softmax/generic/neon/fp16.cpp b/src/cpu/kernels/softmax/generic/neon/fp16.cpp index 3cb1cd683eca7276afd0a4e34945270ee61f3f32..f6556696b06ca8ba7776c7130c691066001cce24 100644 --- a/src/cpu/kernels/softmax/generic/neon/fp16.cpp +++ b/src/cpu/kernels/softmax/generic/neon/fp16.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,6 +23,7 @@ */ #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) #include "arm_compute/core/Helpers.h" +#include "src/cpu/CpuTypes.h" #include "src/cpu/kernels/softmax/generic/neon/impl.h" namespace arm_compute diff --git a/src/cpu/kernels/softmax/generic/sve/fp16.cpp b/src/cpu/kernels/softmax/generic/sve/fp16.cpp index 7aefcb1441af85f29ee5b711be11e8720dc09381..15a523bfc9b1fbe7b10326fec76c359b96ac6c4d 100644 --- a/src/cpu/kernels/softmax/generic/sve/fp16.cpp +++ b/src/cpu/kernels/softmax/generic/sve/fp16.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,6 +23,7 @@ */ #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) #include "arm_compute/core/Helpers.h" +#include "src/cpu/CpuTypes.h" #include "src/cpu/kernels/softmax/generic/sve/impl.h" namespace arm_compute { diff --git a/src/cpu/operators/CpuActivation.h b/src/cpu/operators/CpuActivation.h index f1807d5e47824801e8a222309b941cb41b3fd076..e21fc7d32c16e6862e14e11548479cf15cd596c3 100644 --- a/src/cpu/operators/CpuActivation.h +++ b/src/cpu/operators/CpuActivation.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CPU_ACTIVATION_H #define ARM_COMPUTE_CPU_ACTIVATION_H +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/cpu/ICpuOperator.h" namespace arm_compute diff --git a/src/cpu/operators/CpuAdd.h b/src/cpu/operators/CpuAdd.h index 4ad6d7fe65e63502e512cb84e5c53796dd66114e..db05c100cccd03a63c0123e006be367ba588e6c5 100644 --- a/src/cpu/operators/CpuAdd.h +++ b/src/cpu/operators/CpuAdd.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CPU_ADD_H #define ARM_COMPUTE_CPU_ADD_H +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/cpu/ICpuOperator.h" namespace arm_compute diff --git a/src/cpu/operators/CpuCast.h b/src/cpu/operators/CpuCast.h index a8342581cba4d16592c535461c2716460f2f1dcc..356b033dbd37805ff7e885c57b71a19dc16d6141 100644 --- a/src/cpu/operators/CpuCast.h +++ b/src/cpu/operators/CpuCast.h @@ -52,8 +52,9 @@ public: * |F16 | QASYMM8_SIGNED, QASYMM8, F32, S32, U8 | * |S32 | QASYMM8_SIGNED, QASYMM8, F16, F32, U8 | * |F32 | QASYMM8_SIGNED, QASYMM8, BFLOAT16, F16, S32, U8| + * |S64 | F32 | * - * @param[in] src The source tensor to convert. Data types supported: U8/S8/U16/S16/U32/S32/F16/F32. + * @param[in] src The source tensor to convert. Data types supported: U8/S8/U16/S16/U32/S32/S64/F16/F32. * @param[out] dst The destination tensor. Data types supported: U8/S8/U16/S16/U32/S32/F16/F32. * @param[in] policy Conversion policy. * diff --git a/src/cpu/operators/CpuConv2d.cpp b/src/cpu/operators/CpuConv2d.cpp index fa8a7a185ce5b6603df08b7fab0b86ede46839a8..447b7409892a8f499d271a9d20b65ed9cdfac3d6 100644 --- a/src/cpu/operators/CpuConv2d.cpp +++ b/src/cpu/operators/CpuConv2d.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -178,10 +178,6 @@ ConvolutionMethod CpuConv2d::get_convolution_method(const ITensorInfo *input, co { return ConvolutionMethod::DIRECT; } - if((weights->dimension(idx_h) > 7) && (input->dimension(idx_c) > output->dimension(idx_c)) && (NEFFTConvolutionLayer::validate(input, weights, nullptr, output, conv_info, act_info))) - { - return ConvolutionMethod::FFT; - } if(input->dimension(idx_c) < 16) { return ConvolutionMethod::GEMM; diff --git a/src/cpu/operators/CpuConv2d.h b/src/cpu/operators/CpuConv2d.h index daf068f361fcf53d6d7d968c69b1240f34a55e2b..0908ac0cbbde8266ed3e1e845d9a458e3f5bf11a 100644 --- a/src/cpu/operators/CpuConv2d.h +++ b/src/cpu/operators/CpuConv2d.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -21,6 +21,7 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/core/common/Macros.h" #include "src/cpu/ICpuOperator.h" diff --git a/src/cpu/operators/CpuDepthwiseConv2dAssemblyDispatch.cpp b/src/cpu/operators/CpuDepthwiseConv2dAssemblyDispatch.cpp index a5b9eca56e023da2f3f633f73207507a80393210..d078155155ce84c4becba6b16209c63e38e3152f 100644 --- a/src/cpu/operators/CpuDepthwiseConv2dAssemblyDispatch.cpp +++ b/src/cpu/operators/CpuDepthwiseConv2dAssemblyDispatch.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022 Arm Limited. + * Copyright (c) 2019-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -77,7 +77,7 @@ void CpuDepthwiseConv2dAssemblyDispatch::configure(const ITensorInfo *src, // Compute memory requirements for assembly kernels constexpr size_t alignment = 4096; - _pImpl->mem_req.push_back({ TensorType::ACL_INT_0, dwc_wrapper->get_working_size(num_threads, src->dimension(0)), alignment }); + _pImpl->mem_req.push_back({ TensorType::ACL_INT_0, dwc_wrapper->get_working_size(num_threads), alignment }); _pImpl->mem_req.push_back({ TensorType::ACL_INT_1, dwc_wrapper->get_storage_size(), alignment }); _pImpl->asm_kernel = std::move(dwc_wrapper); } diff --git a/src/cpu/operators/CpuDepthwiseConv2dAssemblyDispatch.h b/src/cpu/operators/CpuDepthwiseConv2dAssemblyDispatch.h index af202ced5b517c9435ae4dcf7919909f89695c73..f222ab9cf946e1159b9d486a87749dffca299c9f 100644 --- a/src/cpu/operators/CpuDepthwiseConv2dAssemblyDispatch.h +++ b/src/cpu/operators/CpuDepthwiseConv2dAssemblyDispatch.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,11 +24,14 @@ #ifndef ARM_COMPUTE_CPU_DEPTHWISE_CONV2D_ASSEMBLY_DISPATCH_H #define ARM_COMPUTE_CPU_DEPTHWISE_CONV2D_ASSEMBLY_DISPATCH_H +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/core/common/Macros.h" #include "src/cpu/ICpuOperator.h" namespace arm_compute { +struct ConvolutionInfo; + namespace cpu { /** Depthwise convolution assembly kernel glue */ diff --git a/src/cpu/operators/CpuFill.h b/src/cpu/operators/CpuFill.h index 1cb99f5662ca9968a281d3d7f7ba9b93acac1b35..41d9a9fa8aa33a856d826caec6dbb167a384759e 100644 --- a/src/cpu/operators/CpuFill.h +++ b/src/cpu/operators/CpuFill.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021,2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CPU_FILL_H #define ARM_COMPUTE_CPU_FILL_H +#include "arm_compute/core/PixelValue.h" #include "src/cpu/ICpuOperator.h" namespace arm_compute diff --git a/src/cpu/operators/CpuFullyConnected.h b/src/cpu/operators/CpuFullyConnected.h index a5a464f67a9cba239083a2e644673b1508d63d10..1e8c6478d09283ae863078cc3f0b083afcbef55c 100644 --- a/src/cpu/operators/CpuFullyConnected.h +++ b/src/cpu/operators/CpuFullyConnected.h @@ -27,6 +27,7 @@ #include "src/cpu/ICpuOperator.h" #include "arm_compute/core/TensorInfo.h" +#include "arm_compute/function_info/FullyConnectedLayerInfo.h" #include @@ -107,8 +108,8 @@ public: FullyConnectedLayerInfo fc_info, WeightsInfo weights_info); //Inherited methods override - void run(ITensorPack &tensors) override; - void prepare(ITensorPack &tensors) override; + void run(ITensorPack &tensors) override; + void prepare(ITensorPack &tensors) override; experimental::MemoryRequirements workspace() const override; private: @@ -158,8 +159,8 @@ private: bool _dynamic_weights; #ifdef ARM_COMPUTE_ASSERTS_ENABLED - int _asrt_run_count{}; - int _asrt_prepare_count{}; + int _asrt_run_count{}; + int _asrt_prepare_count{}; #endif // ARM_COMPUTE_ASSERTS_ENABLED }; } // namespace cpu diff --git a/src/cpu/operators/CpuGemm.h b/src/cpu/operators/CpuGemm.h index bc8adaec32276a1c9d871db270797a6cb203f826..9b08e5d0f69c3f484ad6089cc373a7664f9001da 100644 --- a/src/cpu/operators/CpuGemm.h +++ b/src/cpu/operators/CpuGemm.h @@ -29,6 +29,7 @@ #include "arm_compute/core/ITensorPack.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Types.h" +#include "arm_compute/function_info/GEMMInfo.h" #include "src/cpu/kernels/CpuGemmInterleave4x4Kernel.h" #include "src/cpu/kernels/CpuGemmMatrixAdditionKernel.h" #include "src/cpu/kernels/CpuGemmMatrixMultiplyKernel.h" diff --git a/src/cpu/operators/CpuGemmConv2d.h b/src/cpu/operators/CpuGemmConv2d.h index 08b76a6c46b9d47a442f4c324ee4e198472504c8..81d34ae93de7f1d00d0b202738b951eebbfa4dc9 100644 --- a/src/cpu/operators/CpuGemmConv2d.h +++ b/src/cpu/operators/CpuGemmConv2d.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,7 @@ #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/cpu/ICpuOperator.h" #include diff --git a/src/cpu/operators/CpuGemmLowpMatrixMultiplyCore.h b/src/cpu/operators/CpuGemmLowpMatrixMultiplyCore.h index a7f62aeaa90fc391bcc2346cc81eac945caac0c6..a1b34291d06fb367f5193e278ae937ef84a4b232 100644 --- a/src/cpu/operators/CpuGemmLowpMatrixMultiplyCore.h +++ b/src/cpu/operators/CpuGemmLowpMatrixMultiplyCore.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #define ARM_COMPUTE_CPU_GEMMLOWP_MATRIXMULTIPLY_CORE_H #include "arm_compute/core/TensorInfo.h" +#include "arm_compute/function_info/GEMMInfo.h" #include "src/core/common/Macros.h" #include "src/cpu/ICpuOperator.h" diff --git a/src/cpu/operators/CpuMatMul.cpp b/src/cpu/operators/CpuMatMul.cpp index 87cb6c6b5485b2e2c2fcf2a5104bee7830b211eb..8811a7ea6b850ed7b232a7324123220579c99774 100644 --- a/src/cpu/operators/CpuMatMul.cpp +++ b/src/cpu/operators/CpuMatMul.cpp @@ -25,9 +25,10 @@ #include "src/cpu/operators/CpuMatMul.h" #include "arm_compute/core/Types.h" #include "arm_compute/core/Validate.h" -#include "arm_compute/core/utils/quantization/AsymmHelpers.h" #include "arm_compute/core/experimental/Types.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/quantization/AsymmHelpers.h" +#include "arm_compute/function_info/MatMulInfo.h" #include "arm_compute/runtime/NEON/NEScheduler.h" #include "arm_compute/runtime/NEON/functions/NEMatMul.h" #include "src/common/utils/Log.h" @@ -45,7 +46,6 @@ namespace cpu { namespace { - Status get_gemmlowp_output_stage_info(const ITensorInfo *src, const ITensorInfo *weights, const ITensorInfo *dst, const ActivationLayerInfo &act, GEMMLowpOutputStageInfo &gemmlowp_output_stage_info) { @@ -74,15 +74,14 @@ Status get_gemmlowp_output_stage_info(const ITensorInfo *src, const ITensorInfo return Status{}; } - -} +} // namespace CpuMatMul::CpuMatMul() : _transpose_kernel_lhs(), _transpose_kernel_rhs(), _asm_glue(), _lhs_transposed(), _rhs_transposed(), _original_lhs_shape(), _original_rhs_shape(), _original_dst_shape() { } -Status CpuMatMul::validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *dst, const MatMulInfo &info, const CpuMatMulSettings &settings) +Status CpuMatMul::validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *dst, const MatMulInfo &info, const CpuMatMulSettings &settings, const ActivationLayerInfo &act_info) { ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(lhs, rhs, dst); ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(lhs, 1, DataType::F32, DataType::F16, DataType::QASYMM8, DataType::QASYMM8_SIGNED); @@ -100,7 +99,7 @@ Status CpuMatMul::validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const TensorInfo rhs_transposed{}; auto gemm_info = AsmGemmInfo(); - gemm_info.activation_info = info.fused_activation(); + gemm_info.activation_info = act_info; gemm_info.fast_mode = settings.fast_math(); // Validate and then permute a/b @@ -139,7 +138,7 @@ Status CpuMatMul::validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const return Status{}; } -void CpuMatMul::configure(ITensorInfo *lhs, ITensorInfo *rhs, ITensorInfo *dst, const MatMulInfo &info, const CpuMatMulSettings &settings) +void CpuMatMul::configure(ITensorInfo *lhs, ITensorInfo *rhs, ITensorInfo *dst, const MatMulInfo &info, const CpuMatMulSettings &settings, const ActivationLayerInfo &act_info) { ARM_COMPUTE_ERROR_ON_NULLPTR(lhs, rhs, dst); ARM_COMPUTE_LOG_PARAMS(lhs, rhs, dst, info, settings); @@ -189,7 +188,7 @@ void CpuMatMul::configure(ITensorInfo *lhs, ITensorInfo *rhs, ITensorInfo *dst, // ----------------------------------------------------- // Use transposed tensors if the corresponding transpose flags are set // Fill AsmGemmInfo class object before configuration - _gemm_info.activation_info = info.fused_activation(); + _gemm_info.activation_info = act_info; _gemm_info.fast_mode = settings.fast_math(); _gemm_info.negated_offsets = false; diff --git a/src/cpu/operators/CpuMatMul.h b/src/cpu/operators/CpuMatMul.h index 9f5833b24fd3c86029f3316a28c372ce5eaeb189..475c019fd0fd99c915527fdeff8b37cd8962159d 100644 --- a/src/cpu/operators/CpuMatMul.h +++ b/src/cpu/operators/CpuMatMul.h @@ -64,15 +64,17 @@ public: * @param[out] dst Output tensor to store the result of the batched matrix multiplication. Data types supported: same as @p lhs / @p rhs. * @param[in] info Contains MatMul operation information described in @ref MatMulInfo. * @param[in] settings The settings for matmul operation (i.e fast math) + * @param[in] act_info Class containing information about fused activation function. */ - void configure(ITensorInfo *lhs, ITensorInfo *rhs, ITensorInfo *dst, const MatMulInfo &info, const CpuMatMulSettings &settings); + void configure(ITensorInfo *lhs, ITensorInfo *rhs, ITensorInfo *dst, const MatMulInfo &info, const CpuMatMulSettings &settings, const ActivationLayerInfo &act_info = ActivationLayerInfo()); /** Static function to check if given info will lead to a valid configuration * * Similar to CpuMatMul::configure() * * @return a status */ - static Status validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *dst, const MatMulInfo &info, const CpuMatMulSettings &settings); + static Status validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *dst, const MatMulInfo &info, const CpuMatMulSettings &settings, + const ActivationLayerInfo &act_info = ActivationLayerInfo()); // Inherited methods overridden: void run(ITensorPack &tensors) override; diff --git a/src/cpu/operators/CpuMul.h b/src/cpu/operators/CpuMul.h index 576a357d42fc9354de669d85c4c233ac90a4c1f8..3e0edbf0508f8805905619876be274393cb1ecb5 100644 --- a/src/cpu/operators/CpuMul.h +++ b/src/cpu/operators/CpuMul.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021 Arm Limited. + * Copyright (c) 2016-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #define ARM_COMPUTE_CPU_MUL_H #include "arm_compute/core/ITensorInfo.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/cpu/ICpuOperator.h" namespace arm_compute @@ -102,4 +103,4 @@ public: }; } // namespace cpu } // namespace arm_compute -#endif /* ARM_COMPUTE_CPU_MUL_H */ \ No newline at end of file +#endif /* ARM_COMPUTE_CPU_MUL_H */ diff --git a/src/cpu/operators/CpuScale.cpp b/src/cpu/operators/CpuScale.cpp index a13a0f56a25dca9839582605bff86f301bdacdcd..8a712bf088a28489c7a93852643952ef41954639 100644 --- a/src/cpu/operators/CpuScale.cpp +++ b/src/cpu/operators/CpuScale.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,7 +23,9 @@ */ #include "src/cpu/operators/CpuScale.h" +#include "arm_compute/core/Helpers.h" #include "arm_compute/runtime/NEON/NEScheduler.h" +#include "arm_compute/core/TensorInfo.h" #include "src/common/utils/Log.h" #include "src/core/utils/ScaleUtils.h" #include "src/cpu/kernels/CpuScaleKernel.h" diff --git a/src/cpu/operators/CpuSub.h b/src/cpu/operators/CpuSub.h index d463d1e06308e7a17a418aa9a383269ec1ec76f1..88908637aa1dcec46a54a4b8315649947db48e6b 100644 --- a/src/cpu/operators/CpuSub.h +++ b/src/cpu/operators/CpuSub.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CPU_SUB_H #define ARM_COMPUTE_CPU_SUB_H +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/cpu/ICpuOperator.h" namespace arm_compute @@ -66,4 +67,4 @@ public: }; } // namespace cpu } // namespace arm_compute -#endif /* ARM_COMPUTE_CPU_SUB_H */ \ No newline at end of file +#endif /* ARM_COMPUTE_CPU_SUB_H */ diff --git a/src/cpu/operators/internal/CpuGemmAssemblyDispatch.cpp b/src/cpu/operators/internal/CpuGemmAssemblyDispatch.cpp index 9c856314060d680b01c01415ade26e5e05bc462f..3069d6b541250f3ca953ccf4688c5576c3b7098f 100644 --- a/src/cpu/operators/internal/CpuGemmAssemblyDispatch.cpp +++ b/src/cpu/operators/internal/CpuGemmAssemblyDispatch.cpp @@ -794,7 +794,7 @@ Status CpuGemmAssemblyDispatch::validate(const ITensorInfo *a, const ITensorInfo ARM_COMPUTE_RETURN_ERROR_ON_MSG(a->data_type() == DataType::S8 && d->data_type() != DataType::S32, "Only S32 output supported for S8 input"); ARM_COMPUTE_RETURN_ERROR_ON_MSG(a->data_type() == DataType::QASYMM8 && (d->data_type() != DataType::QASYMM8 && d->data_type() != DataType::S32), "Only QASYMM8/S32 output supported for QASYMM8 input"); - arm_compute::WeightFormat expected_weight_format; + arm_compute::WeightFormat expected_weight_format = arm_compute::WeightFormat::UNSPECIFIED; const Status ret = CpuGemmAssemblyDispatch::has_opt_impl(expected_weight_format, a, b, c, d, info); if((bool)ret && expected_weight_format != arm_compute::WeightFormat::ANY) { diff --git a/src/cpu/operators/internal/CpuGemmAssemblyDispatch.h b/src/cpu/operators/internal/CpuGemmAssemblyDispatch.h index 588c45294a4f2e6408633adde38db6dffa2b7833..ceb7a3f7756ac7a8248a49841bd2b43823a7cc7c 100644 --- a/src/cpu/operators/internal/CpuGemmAssemblyDispatch.h +++ b/src/cpu/operators/internal/CpuGemmAssemblyDispatch.h @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CPU_INTERNAL_CPU_GEMM_ASSEMBLY_DISPATCH_H #define ARM_COMPUTE_CPU_INTERNAL_CPU_GEMM_ASSEMBLY_DISPATCH_H +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/core/common/Macros.h" #include "src/cpu/ICpuOperator.h" diff --git a/src/dynamic_fusion/runtime/gpu/cl/ClKernelRuntime.cpp b/src/dynamic_fusion/runtime/gpu/cl/ClKernelRuntime.cpp index b3ec39362c46e35f21a03d6415bc3a97144306a5..15a5632d0b1ceba7ba5d427d344ca538de7ee870 100644 --- a/src/dynamic_fusion/runtime/gpu/cl/ClKernelRuntime.cpp +++ b/src/dynamic_fusion/runtime/gpu/cl/ClKernelRuntime.cpp @@ -24,6 +24,9 @@ #include "ClKernelRuntime.h" #include "arm_compute/core/CL/ICLTensor.h" #include "src/core/CL/CLUtils.h" +#ifdef ACL_INTERNAL_TEST_CKW_IN_DF +#include "src/dynamic_fusion/runtime/gpu/cl/ckw_driver/GpuCkwKernelArgumentsHelpers.h" +#endif // ACL_INTERNAL_TEST_CKW_IN_DF #include "src/dynamic_fusion/sketch/gpu/GpuKernelSourceCode.h" #include "src/gpu/cl/ClKernelLibrary.h" @@ -42,6 +45,7 @@ void ClKernelRuntime::configure(const ClCompileContext &compile_ctx, const GpuKe opencl::ClKernelLibrary &klib = opencl::ClKernelLibrary::get(); _kernel = static_cast(compile_ctx.create_kernel(code.name(), code.name(), // program name has to be provided to differentiate between different unfusable components' kernels. + // Each program contains exactly one kernel code.code(), klib.kernel_path() /* Kernel path: Used in cases of embedded kernels */, code.build_options().options(), @@ -57,8 +61,12 @@ void ClKernelRuntime::configure(const ClCompileContext &compile_ctx, const GpuKe _arguments = code.arguments(); } +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF + inline void ClKernelRuntime::add_tensor_argument(unsigned int &idx, const GpuKernelArgumentInfo &arg, const ICLTensor *tensor, const Window &arg_slice, std::vector &cl_images) { + ARM_COMPUTE_ERROR_ON_NULLPTR(tensor); + switch(arg.type) { case GpuKernelArgumentInfo::Type::Scalar: @@ -142,6 +150,18 @@ inline void ClKernelRuntime::add_tensor_argument(unsigned int &idx, const GpuKer add_4d_tensor_nhwc_argument(idx, tensor); break; } + case GpuKernelArgumentInfo::Type::Tensor_Special_0: + { + const ITensorInfo *info = tensor->info(); + const Strides &strides = info->strides_in_bytes(); + + _kernel.setArg(idx++, tensor->cl_buffer()); + const size_t dim1xdim2 = info->tensor_shape()[1] * info->tensor_shape()[2]; + _kernel.setArg(idx++, static_cast(dim1xdim2)); + const size_t stride1 = strides[1]; + _kernel.setArg(idx++, static_cast(stride1)); + break; + } default: { ARM_COMPUTE_ERROR("Unsupported"); @@ -149,21 +169,65 @@ inline void ClKernelRuntime::add_tensor_argument(unsigned int &idx, const GpuKer } } +#else // ACL_INTERNAL_TEST_CKW_IN_DF +inline void ClKernelRuntime::add_kernel_argument(unsigned int &idx, const GpuKernelArgumentBinding &arg, const ICLTensor *tensor, std::vector &cl_images) +{ + switch(arg.type()) + { + case GpuKernelArgumentBinding::Type::TensorStorage: + { + switch(arg.tensor_storage_type()) + { + case TensorStorageType::ClBufferUint8Ptr: + { + cl_add_buffer_argument(_kernel, idx, tensor->cl_buffer()); + break; + } + case TensorStorageType::ClImage2dReadOnly: + { + cl::Image2D tensor_image2d = create_image2d_from_tensor(tensor, CLImage2DType::ReadOnly); + cl_images.push_back(tensor_image2d); + cl_add_texture_argument(_kernel, idx, tensor_image2d); + break; + } + case TensorStorageType::ClImage2dWriteOnly: + { + cl::Image2D tensor_image2d = create_image2d_from_tensor(tensor, CLImage2DType::WriteOnly); + cl_images.push_back(tensor_image2d); + cl_add_texture_argument(_kernel, idx, tensor_image2d); + break; + } + default: + { + ARM_COMPUTE_ERROR("Do not accept other TensorStorageType"); + break; + } + } + break; + } + case GpuKernelArgumentBinding::Type::TensorComponent: + { + cl_add_tensor_component_argument(_kernel, idx, tensor, arg.tensor_component_type()); + break; + } + default: + { + ARM_COMPUTE_ERROR("Do not accept other types of kernel arguments"); + break; + } + } +} + +#endif // ACL_INTERNAL_TEST_CKW_IN_DF void ClKernelRuntime::run_op(ITensorPack &tensors, const Window &window, cl::CommandQueue &queue) { ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this); ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(ICLKernel::window(), window); Window slice = window.first_slice_window_3D(); - // Don't slice matrix along the z dimension if matrix has just 2 dimensions and matrix A more than 2 - // This scenario can happen when the matrix multiplication is used to perform a convolution operation - Window slice_fixed_z = slice; - slice_fixed_z.set(Window::DimX, Window::Dimension(0, 1, 1)); - slice_fixed_z.set(Window::DimY, Window::Dimension(0, 1, 1)); /// NOTE: Parameters extracted from old kernels. So far they seem to be constant /// but we may need to make them into another configuration passed from GpuWorkloadSourceCode if needed in the future - constexpr bool slide_along_dimz = true; constexpr bool skip_sliding_window = false; constexpr bool use_dummy_work_items = false; @@ -171,23 +235,27 @@ void ClKernelRuntime::run_op(ITensorPack &tensors, const Window &window, cl::Com do { // Set kernel arguments - Window arg_slice = slice; // CLImages created from tensor arguments. Need to be retained until enqueue std::vector cl_images; +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF for(auto id_arg : _arguments) { const auto arg = id_arg.second; auto tensor = utils::cast::polymorphic_downcast(tensors.get_tensor(id_arg.first)); ARM_COMPUTE_ERROR_ON_NULLPTR(tensor); ARM_COMPUTE_ERROR_ON_NULLPTR(tensor->info()); - if(!slide_along_dimz) - { - // The stride_z for matrix must be zero if we do not slice - ARM_COMPUTE_ERROR_ON(tensor->info()->strides_in_bytes()[3] != 0); - arg_slice = slice_fixed_z; - } - add_tensor_argument(idx, *arg.kernel_argument_info(), tensor, arg_slice, cl_images); + add_tensor_argument(idx, *arg.kernel_argument_info(), tensor, slice, cl_images); + } + +#else // ACL_INTERNAL_TEST_CKW_IN_DF + for(const auto &arg : _arguments) + { + auto tensor = utils::cast::polymorphic_downcast(tensors.get_tensor(arg.id())); + ARM_COMPUTE_ERROR_ON_NULLPTR(tensor); + ARM_COMPUTE_ERROR_ON_NULLPTR(tensor->info()); + add_kernel_argument(idx, arg, tensor, cl_images); } +#endif // ACL_INTERNAL_TEST_CKW_IN_DF // Dispatch kernel enqueue(queue, *this, slice, lws_hint(), use_dummy_work_items); diff --git a/src/dynamic_fusion/runtime/gpu/cl/ClKernelRuntime.h b/src/dynamic_fusion/runtime/gpu/cl/ClKernelRuntime.h index 4787acabcd05c67b118b85a1ffa8ac9bb5a5abe3..92e73503cecee2d332257ded4294af597da9e401 100644 --- a/src/dynamic_fusion/runtime/gpu/cl/ClKernelRuntime.h +++ b/src/dynamic_fusion/runtime/gpu/cl/ClKernelRuntime.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -29,6 +29,8 @@ #include "src/gpu/cl/ClCompileContext.h" #include "src/gpu/cl/IClKernel.h" +#include + namespace arm_compute { namespace experimental @@ -57,6 +59,7 @@ public: virtual void run_op(ITensorPack &tensors, const Window &window, cl::CommandQueue &queue) override; private: +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF /** Set a kernel tensor argument * * @param[in,out] idx Index at which to start adding the tensor's arguments. Will be incremented by the number of kernel arguments set. @@ -66,9 +69,19 @@ private: * @param[out] cl_images Extra cl images created from the tensor (will need to be retained until the kernel is enqueued) */ inline void add_tensor_argument(unsigned int &idx, const GpuKernelArgumentInfo &arg, const ICLTensor *tensor, const Window &arg_slice, std::vector &cl_images); +#else // ACL_INTERNAL_TEST_CKW_IN_DF + /** Set a kernel argument as part of a tensor + * + * @param[in,out] idx Index at which to start adding the tensor's arguments. Will be incremented by the number of kernel arguments set. + * @param[in] arg Kernel argument binding, as part of @p tensor + * @param[in] tensor Tensor of which the kernel argument @p arg is a part of + * @param[out] cl_images Extra cl images created from the tensor (will need to be retained until the kernel is enqueued) + */ + inline void add_kernel_argument(unsigned int &idx, const GpuKernelArgumentBinding &arg, const ICLTensor *tensor, std::vector &cl_images); +#endif // ACL_INTERNAL_TEST_CKW_IN_DF private: - GpuKernelArgumentList _arguments{}; /** All kernel arguments required by the runtime */ + GpuKernelArgumentList _arguments{}; }; } // namespace dynamic_fusion diff --git a/src/dynamic_fusion/runtime/gpu/cl/ckw_driver/GpuCkwKernelArgumentsHelpers.cpp b/src/dynamic_fusion/runtime/gpu/cl/ckw_driver/GpuCkwKernelArgumentsHelpers.cpp new file mode 100644 index 0000000000000000000000000000000000000000..84fb279237a1351b8c68f95f7f02b47429c1a20e --- /dev/null +++ b/src/dynamic_fusion/runtime/gpu/cl/ckw_driver/GpuCkwKernelArgumentsHelpers.cpp @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "GpuCkwKernelArgumentsHelpers.h" + +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ +void cl_add_tensor_component_argument(cl::Kernel &kernel, unsigned int &idx, const ICLTensor *tensor, TensorComponentType component) +{ + ARM_COMPUTE_ERROR_ON(tensor == nullptr); + + const auto *info = tensor->info(); + const auto &strides = info->strides_in_bytes(); + + switch(component) + { + case TensorComponentType::OffsetFirstElement: + kernel.setArg(idx++, info->offset_first_element_in_bytes()); + break; + case TensorComponentType::Stride0: + kernel.setArg(idx++, strides[0]); + break; + case TensorComponentType::Stride1: + kernel.setArg(idx++, strides[1]); + break; + case TensorComponentType::Stride2: + kernel.setArg(idx++, strides[2]); + break; + case TensorComponentType::Stride3: + kernel.setArg(idx++, strides[3]); + break; + case TensorComponentType::Stride4: + kernel.setArg(idx++, strides[4]); + break; + case TensorComponentType::Dim0: + kernel.setArg(idx++, info->dimension(0)); + break; + case TensorComponentType::Dim1: + kernel.setArg(idx++, info->dimension(1)); + break; + case TensorComponentType::Dim2: + kernel.setArg(idx++, info->dimension(2)); + break; + case TensorComponentType::Dim3: + kernel.setArg(idx++, info->dimension(3)); + break; + case TensorComponentType::Dim4: + kernel.setArg(idx++, info->dimension(4)); + break; + case TensorComponentType::Dim1xDim2: + kernel.setArg(idx++, info->dimension(1) * info->dimension(2)); + break; + case TensorComponentType::Dim2xDim3: + kernel.setArg(idx++, info->dimension(2) * info->dimension(3)); + break; + case TensorComponentType::Dim1xDim2xDim3: + kernel.setArg(idx++, info->dimension(1) * info->dimension(2) * info->dimension(3)); + break; + case TensorComponentType::Unknown: + default: + ARM_COMPUTE_ERROR("Unknown tensor component"); + } +} + +void cl_add_buffer_argument(cl::Kernel &kernel, unsigned int &idx, const cl::Buffer &buffer) +{ + kernel.setArg(idx++, buffer); +} + +void cl_add_texture_argument(cl::Kernel &kernel, unsigned int &idx, const cl::Image &image) +{ + kernel.setArg(idx++, image); +} + +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute diff --git a/src/dynamic_fusion/runtime/gpu/cl/ckw_driver/GpuCkwKernelArgumentsHelpers.h b/src/dynamic_fusion/runtime/gpu/cl/ckw_driver/GpuCkwKernelArgumentsHelpers.h new file mode 100644 index 0000000000000000000000000000000000000000..4cbb157a48e05d9096b5016bb9a8fa1760b32675 --- /dev/null +++ b/src/dynamic_fusion/runtime/gpu/cl/ckw_driver/GpuCkwKernelArgumentsHelpers.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef ACL_SRC_DYNAMIC_FUSION_RUNTIME_GPU_CL_CKW_DRIVER_GPUCKWKERNELARGUMENTSHELPERS +#define ACL_SRC_DYNAMIC_FUSION_RUNTIME_GPU_CL_CKW_DRIVER_GPUCKWKERNELARGUMENTSHELPERS + +#include "arm_compute/core/CL/ICLTensor.h" + +#include "src/dynamic_fusion/sketch/gpu/GpuKernelArgument.h" + +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ +/** Select a Compute Kernel Writer tensor component from a tensor and add to the kernel's arguments at the specified index idx. + * + * @param[in,out] kernel OpenCL kernel to configure with the provided argument. + * @param[in,out] idx Index at which to add the argument. + * @param[in] tensor Tensor from which to access the tensor component. + * @param[in] component Tensor component to select such as tensor dimensions, strides, etc. + */ +void cl_add_tensor_component_argument(cl::Kernel &kernel, unsigned int &idx, const ICLTensor *tensor, TensorComponentType component); + +/** Add an OpenCL buffer object to the kernel's arguments at the specified index @p idx. + * + * @param[in,out] kernel OpenCL kernel to configure with the provided argument. + * @param[in,out] idx Index at which to add the argument. + * @param[in] buffer OpenCL buffer containing the tensor's data. + */ +void cl_add_buffer_argument(cl::Kernel &kernel, unsigned int &idx, const cl::Buffer &buffer); + +/** Add an OpenCL image object to the kernel's arguments at the specified index @p idx. + * + * @param[in,out] kernel OpenCL kernel to configure with the provided argument. + * @param[in,out] idx Index at which to add the argument. + * @param[in] image OpenCL image containing the image's data. + */ +void cl_add_texture_argument(cl::Kernel &kernel, unsigned int &idx, const cl::Image &image); + +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute + +#endif /* ACL_SRC_DYNAMIC_FUSION_RUNTIME_GPU_CL_CKW_DRIVER_GPUCKWKERNELARGUMENTSHELPERS */ diff --git a/src/dynamic_fusion/sketch/gpu/GpuKernelArgument.h b/src/dynamic_fusion/sketch/gpu/GpuKernelArgument.h index eb36e91d4818c5c77276036ccefec703a75546e7..226e1a2df3a8293f62c9300472f574e764f901c9 100644 --- a/src/dynamic_fusion/sketch/gpu/GpuKernelArgument.h +++ b/src/dynamic_fusion/sketch/gpu/GpuKernelArgument.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -33,6 +33,7 @@ namespace experimental namespace dynamic_fusion { /** Contain information required to set up a kernel argument at run time + * @deprecated To be removed along with ClTemplateWriter */ struct GpuKernelArgumentInfo { @@ -53,7 +54,9 @@ struct GpuKernelArgumentInfo Tensor_3D, Tensor_4D, Tensor_4D_t_Buffer, - Tensor_4D_t_Image + Tensor_4D_t_Image, + + Tensor_Special_0, }; /** Default constructor */ GpuKernelArgumentInfo() = default; @@ -64,10 +67,9 @@ struct GpuKernelArgumentInfo } Type type{ Type::Tensor_4D_t_Buffer }; }; - bool operator==(const GpuKernelArgumentInfo &info0, const GpuKernelArgumentInfo &info1); - /** Kernel argument information linked with its corresponding @ref ITensorInfo + * @deprecated To be removed along with ClTemplateWriter */ class GpuKernelArgument { @@ -122,6 +124,130 @@ private: TensorInfo _tensor_info{}; GpuKernelArgumentInfo _kernel_arg_info{}; }; +#ifdef ACL_INTERNAL_TEST_CKW_IN_DF +/** Describe how the tensor runtime memory can be accessed + * + * Please see documentation under @ref GpuKernelArgumentBinding + */ +enum class TensorStorageType +{ + Unknown, + ClBufferUint8Ptr, + ClImage2dReadOnly, + ClImage2dWriteOnly, +}; + +/** Describe additional runtime information about the tensor + * + * Please see documentation under @ref GpuKernelArgumentBinding + */ +enum class TensorComponentType +{ + Unknown, + OffsetFirstElement, + Stride0, + Stride1, + Stride2, + Stride3, + Stride4, + Dim0, + Dim1, + Dim2, + Dim3, + Dim4, + Dim1xDim2, + Dim2xDim3, + Dim1xDim2xDim3, +}; + +/** Describe how to extract information from a runtime Gpu tensor, and set it as an argument to a gpu kernel at runtime + * + * A kernel argument is just an argument to the gpu kernel as shown in the argument list below. This contrasts with a "workload argument" which is a tensor (@ref GpuWorkloadArgument) + * void kernel(arg0, arg1, ... argN) + * + * In a kernel generated using dynamic fusion (@ref GpuKernelSourceCode), every kernel argument describes part of a tensor. + * A tensor is described as: **storages** followed by **components** + * + * A storage (@ref TensorStorageType) describes how the tensor runtime memory can be accessed (e.g. via a global uint8 pointer to a CL buffer) + * A component (@ref TensorComponentType) describes additional runtime information about the tensor (e.g. the dimensions of the tensor) + * + * The arguments are arranged in the order of use in the generated kernel code: + * + * arg0 , arg1 , arg2 , ..., , argN + * storage, component0, component1, ..., componentX, storage, component0, component1, ..., componentY + * | tensor0 | tensor1 | + * + * An example argument list: + * + * void kernel( + * image2d_t t0_image, // TensorStorageType::ClImage2dReadOnly + * uint8_t* t0_ptr, // TensorStorageType::ClBufferUint8Ptr + * uint t0_dim0, // TensorComponentType::Dim0 + * uint t0_stride1, // TensorComponentType::Stride1 + * image2d_t t1_ptr, // TensorStorageType::ClImage2dReadOnly + * uint t1_dim1xdim2, // TensorComponentType::Dim1xDim2 + * uint t1_stride1, // TensorComponentType::Stride1 + * uint t1_stride2, // TensorComponentType:Stride2 + * ) + * + */ +class GpuKernelArgumentBinding +{ +public: + enum class Type : int32_t + { + TensorStorage, /** @ref TensorStorageType */ + TensorComponent /** @ref TensorComponentType */ + }; + GpuKernelArgumentBinding(ITensorInfo::Id id, TensorStorageType storage) + : _type{ Type::TensorStorage }, _id{ id }, _value{} + { + _value.tensor_storage_type = storage; + } + GpuKernelArgumentBinding(ITensorInfo::Id id, TensorComponentType component) + : _type{ Type::TensorComponent }, _id{ id }, _value{} + { + _value.tensor_component_type = component; + } + /** Storage type of the tensor + */ + TensorStorageType tensor_storage_type() const + { + ARM_COMPUTE_ERROR_ON(_type != Type::TensorStorage); + return _value.tensor_storage_type; + } + /** Component of the tensor + */ + TensorComponentType tensor_component_type() const + { + ARM_COMPUTE_ERROR_ON(_type != Type::TensorComponent); + return _value.tensor_component_type; + } + /** Id of the tensor this kernel argument belongs to + */ + ITensorInfo::Id id() const + { + return _id; + } + /** Type of the kernel argument + */ + Type type() const + { + return _type; + } + +private: + Type _type; + ITensorInfo::Id _id; + union Value + { + TensorStorageType tensor_storage_type; + TensorComponentType tensor_component_type; + }; + Value _value; +}; +#endif // ACL_INTERNAL_TEST_CKW_IN_DF + } // namespace dynamic_fusion } // namespace experimental } // namespace arm_compute diff --git a/src/dynamic_fusion/sketch/gpu/GpuKernelComponentGraph.cpp b/src/dynamic_fusion/sketch/gpu/GpuKernelComponentGraph.cpp index b70a192775ccc2cddd1871478c8685752a0a145f..5a65ede38b77103e2def9ca64684d3471bab3844 100644 --- a/src/dynamic_fusion/sketch/gpu/GpuKernelComponentGraph.cpp +++ b/src/dynamic_fusion/sketch/gpu/GpuKernelComponentGraph.cpp @@ -44,14 +44,14 @@ std::vector GpuKernelComponentGraph::get_tensor_ids(c return tensor_ids; } -GpuKernelComponentGraph::GpuKernelComponentGraph(GpuComponentServices *services) - : _services{ services }, _components{}, _tensors{}, _dependency_graph{} +GpuKernelComponentGraph::GpuKernelComponentGraph(GpuWorkloadContext *context, GpuComponentServices *services) + : _context{ context }, _services{ services }, _components{}, _tensors{}, _dependency_graph{} { } GpuKernelComponentStream GpuKernelComponentGraph::fuse(const MemoryDescriptorMap &mem_map) const { - GpuKernelComponentStream stream{ _services, mem_map }; + GpuKernelComponentStream stream{ _context, _services, mem_map }; const auto op_seq = _dependency_graph.build_operators_sequence(); stream.new_component_group(); diff --git a/src/dynamic_fusion/sketch/gpu/GpuKernelComponentGraph.h b/src/dynamic_fusion/sketch/gpu/GpuKernelComponentGraph.h index 8314ea0a508a883c8944148601c581fcee2fc18b..85c9b4584004c349748015a5d2843b855a2fc5f6 100644 --- a/src/dynamic_fusion/sketch/gpu/GpuKernelComponentGraph.h +++ b/src/dynamic_fusion/sketch/gpu/GpuKernelComponentGraph.h @@ -21,8 +21,8 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifndef SRC_DYNAMIC_FUSION_SKETCH_GPU_GPUKERNELCOMPONENTGRAPH -#define SRC_DYNAMIC_FUSION_SKETCH_GPU_GPUKERNELCOMPONENTGRAPH +#ifndef ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_GPUKERNELCOMPONENTGRAPH +#define ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_GPUKERNELCOMPONENTGRAPH #include "src/dynamic_fusion/sketch/ArgumentPack.h" #include "src/dynamic_fusion/sketch/gpu/GpuComponentServices.h" @@ -49,9 +49,10 @@ class GpuKernelComponentGraph public: /** Constructor * + * @param[in] context @ref GpuWorkloadContext to be used by the graph * @param[in] services @ref GpuComponentServices to be used by the graph */ - GpuKernelComponentGraph(GpuComponentServices *services); + GpuKernelComponentGraph(GpuWorkloadContext *context, GpuComponentServices *services); /** Prevent instances of this class from being copy constructed */ GpuKernelComponentGraph(const GpuKernelComponentGraph &graph) = delete; /** Prevent instances of this class from being copied */ @@ -98,6 +99,7 @@ public: private: static std::vector get_tensor_ids(const std::vector tensors); + GpuWorkloadContext *_context; GpuComponentServices *_services; std::map> _components; std::map _tensors; @@ -106,4 +108,4 @@ private: } // namespace dynamic_fusion } // namespace experimental } // namespace arm_compute -#endif /* SRC_DYNAMIC_FUSION_SKETCH_GPU_GPUKERNELCOMPONENTGRAPH */ +#endif /* ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_GPUKERNELCOMPONENTGRAPH */ diff --git a/src/dynamic_fusion/sketch/gpu/GpuKernelComponentStream.cpp b/src/dynamic_fusion/sketch/gpu/GpuKernelComponentStream.cpp index 8f4eadc477120d216dd1f158e44e103cafa715f6..a2b6623370afc24657368c8ddf273fc51bb20b60 100644 --- a/src/dynamic_fusion/sketch/gpu/GpuKernelComponentStream.cpp +++ b/src/dynamic_fusion/sketch/gpu/GpuKernelComponentStream.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -33,8 +33,8 @@ namespace experimental { namespace dynamic_fusion { -GpuKernelComponentStream::GpuKernelComponentStream(GpuComponentServices *services, const MemoryDescriptorMap &mem_map) - : _services{ services }, _component_groups{}, _mem_map{ mem_map } +GpuKernelComponentStream::GpuKernelComponentStream(GpuWorkloadContext *context, GpuComponentServices *services, const MemoryDescriptorMap &mem_map) + : _context{ context }, _services{ services }, _component_groups{}, _mem_map{ mem_map } { } @@ -51,7 +51,7 @@ GpuWorkloadSourceCode GpuKernelComponentStream::write_workload_code() const GpuKernelSourceCode kernel_code = logical_kernel.write_kernel_code(); // The whole unit workload stage is determined by the root component const auto unit_workload_stage = group.get_root_component()->properties().stage(); - source_code.add_unit_workload(kernel_code, unit_workload_stage, _mem_map); + source_code.add_unit_workload(kernel_code, unit_workload_stage, _mem_map, _context); } return source_code; } diff --git a/src/dynamic_fusion/sketch/gpu/GpuKernelComponentStream.h b/src/dynamic_fusion/sketch/gpu/GpuKernelComponentStream.h index cbaa7c297bf39cbeca65463be62d5fe9cc075070..ba2503a9381036b6cd17b4f76caa77e19796099f 100644 --- a/src/dynamic_fusion/sketch/gpu/GpuKernelComponentStream.h +++ b/src/dynamic_fusion/sketch/gpu/GpuKernelComponentStream.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -49,10 +49,11 @@ class GpuKernelComponentStream public: /** Constructor * + * @param[in] context @ref GpuWorkloadContext to be used throughout the stream * @param[in] services @ref GpuComponentServices to be used throughout the stream * @param[in] mem_map @ref MemoryDescriptor map used to assemble the @ref GpuWorkloadSourceCode */ - GpuKernelComponentStream(GpuComponentServices *services, const MemoryDescriptorMap &mem_map); + GpuKernelComponentStream(GpuWorkloadContext *context, GpuComponentServices *services, const MemoryDescriptorMap &mem_map); /** Allow instances of this class to be copy constructed */ GpuKernelComponentStream(const GpuKernelComponentStream &stream) = default; /** Allow instances of this class to be copied */ @@ -78,6 +79,7 @@ public: bool add_component(IGpuKernelComponent *component); private: + GpuWorkloadContext *_context; GpuComponentServices *_services; std::vector _component_groups{}; MemoryDescriptorMap _mem_map{}; diff --git a/src/dynamic_fusion/sketch/gpu/GpuKernelSourceCode.h b/src/dynamic_fusion/sketch/gpu/GpuKernelSourceCode.h index 7479328d7bee3c39b49de84f8566099ad16d8d7b..64e1cdc3bc5b5f4bce08947fc60e4fd72fcbfc45 100644 --- a/src/dynamic_fusion/sketch/gpu/GpuKernelSourceCode.h +++ b/src/dynamic_fusion/sketch/gpu/GpuKernelSourceCode.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -21,14 +21,18 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifndef SRC_DYNAMIC_FUSION_SKETCH_GPU_GPUKERNELSOURCECODE -#define SRC_DYNAMIC_FUSION_SKETCH_GPU_GPUKERNELSOURCECODE +#ifndef ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_GPUKERNELSOURCECODE +#define ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_GPUKERNELSOURCECODE #include "arm_compute/core/CL/CLCompileContext.h" #include "arm_compute/core/Window.h" #include "src/dynamic_fusion/sketch/gpu/GpuKernelArgument.h" +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF #include +#else // ACL_INTERNAL_TEST_CKW_IN_DF +#include +#endif // ACL_INTERNAL_TEST_CKW_IN_DF #include namespace arm_compute @@ -38,7 +42,11 @@ namespace experimental namespace dynamic_fusion { /** The argument list of a @ref GpuKernelSourceCode */ +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF using GpuKernelArgumentList = std::map; +#else // ACL_INTERNAL_TEST_CKW_IN_DF +using GpuKernelArgumentList = std::deque; +#endif // ACL_INTERNAL_TEST_CKW_IN_DF /** Container of kernel code to be compiled and run in a @ref GpuUnitWorkload */ @@ -123,4 +131,4 @@ private: } // namespace dynamic_fusion } // namespace experimental } // namespace arm_compute -#endif /* SRC_DYNAMIC_FUSION_SKETCH_GPU_GPUKERNELSOURCECODE */ +#endif /* ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_GPUKERNELSOURCECODE */ diff --git a/src/dynamic_fusion/sketch/gpu/GpuLogicalKernel.cpp b/src/dynamic_fusion/sketch/gpu/GpuLogicalKernel.cpp index 00f625de28fb703ee91b378bf0b82e9e2af37040..c99984fc0e1d9294634cfb669fe6b71354fec6e8 100644 --- a/src/dynamic_fusion/sketch/gpu/GpuLogicalKernel.cpp +++ b/src/dynamic_fusion/sketch/gpu/GpuLogicalKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -29,7 +29,11 @@ #include "src/dynamic_fusion/sketch/gpu/GpuComponentServices.h" #include "src/dynamic_fusion/sketch/gpu/components/IGpuKernelComponent.h" #include "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentStore.h" +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF #include "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateWriter.h" +#else // ACL_INTERNAL_TEST_CKW_IN_DF +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwDriver.h" +#endif // ACL_INTERNAL_TEST_CKW_IN_DF namespace arm_compute { @@ -46,11 +50,19 @@ GpuLogicalKernel::GpuLogicalKernel(GpuComponentServices *services, const GpuKern GpuKernelSourceCode GpuLogicalKernel::write_kernel_code() { GpuKernelSourceCode code; - ClTemplateWriter writer{ _comp_group }; +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF + ClTemplateWriter writer { _comp_group }; +#else // ACL_INTERNAL_TEST_CKW_IN_DF + GpuCkwDriver writer { _comp_group }; +#endif // ACL_INTERNAL_TEST_CKW_IN_DF code.name(writer.get_name()); code.code(writer.get_code()); +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF code.arguments(writer.get_tensors()); +#else // ACL_INTERNAL_TEST_CKW_IN_DF + code.arguments(writer.get_kernel_arguments()); +#endif // ACL_INTERNAL_TEST_CKW_IN_DF code.build_options(writer.get_build_options()); code.config_id(writer.get_config_id()); code.window(writer.get_window()); diff --git a/src/dynamic_fusion/sketch/gpu/GpuWorkloadContext.cpp b/src/dynamic_fusion/sketch/gpu/GpuWorkloadContext.cpp index 623bf351f809e4c45555af5f2eae7b7f7a53efd9..c2bd01270395198341f94d267471ebba3620f149 100644 --- a/src/dynamic_fusion/sketch/gpu/GpuWorkloadContext.cpp +++ b/src/dynamic_fusion/sketch/gpu/GpuWorkloadContext.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -21,8 +21,10 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ + #include "arm_compute/dynamic_fusion/sketch/gpu/GpuWorkloadContext.h" #include "arm_compute/core/CL/CLCompileContext.h" +#include "src/dynamic_fusion/sketch/gpu/GpuWorkloadContextImpl.h" namespace arm_compute { @@ -31,25 +33,114 @@ namespace experimental namespace dynamic_fusion { GpuWorkloadContext::GpuWorkloadContext(CLCompileContext *cl_compile_ctx) - : _gpu_language{ GpuLanguage::OpenCL }, _cl_compile_ctx{ cl_compile_ctx } + : _impl{ std::make_unique(GpuLanguage::OpenCL, cl_compile_ctx) } { } +GpuWorkloadContext::~GpuWorkloadContext() = default; + +GpuWorkloadContext::GpuWorkloadContext(GpuWorkloadContext &&other) = default; + +GpuWorkloadContext &GpuWorkloadContext::operator=(GpuWorkloadContext &&other) = default; + GpuTarget GpuWorkloadContext::gpu_target() const { - return _cl_compile_ctx->get_gpu_target(); + return _impl->cl_compile_context()->get_gpu_target(); } GpuLanguage GpuWorkloadContext::gpu_language() const { - return _gpu_language; + return _impl->gpu_language(); } const CLCompileContext *GpuWorkloadContext::cl_compile_context() const +{ + return _impl->cl_compile_context(); +} + +void GpuWorkloadContext::register_user_tensor(ITensorInfo &tensor_info) +{ + _impl->register_user_tensor(tensor_info); +} + +GpuWorkloadContext::Impl &GpuWorkloadContext::implementation() +{ + return *_impl; +} + +const GpuWorkloadContext::Impl &GpuWorkloadContext::implementation() const +{ + return *_impl; +} + +GpuWorkloadContext::Impl::Impl(GpuLanguage gpu_language, CLCompileContext *cl_compile_ctx) + : _gpu_language(gpu_language), _cl_compile_ctx(cl_compile_ctx), _next_tensor_id(1), _mem_map(), _managed_tensor_info() +{ +} + +GpuLanguage GpuWorkloadContext::Impl::gpu_language() const +{ + return _gpu_language; +} + +const CLCompileContext *GpuWorkloadContext::Impl::cl_compile_context() const { return _cl_compile_ctx; } +const MemoryDescriptorMap &GpuWorkloadContext::Impl::mem_map() const +{ + return _mem_map; +} + +void GpuWorkloadContext::Impl::register_user_tensor(ITensorInfo &tensor_info) +{ + ARM_COMPUTE_ERROR_ON(tensor_info.has_valid_id()); + + const auto tensor_id = next_tensor_id(); + + tensor_info.set_id(tensor_id); + _mem_map[tensor_id] = MemoryDescriptor{ MemoryType::User }; + // Save a *copy* of the user tensor info in workload context for future reference + // Note that this means if the user modifies the @p tensor_info, the change will not be reflected in the context + _managed_tensor_info.emplace(tensor_info.id(), std::make_unique(tensor_info)); +} + +ITensorInfo *GpuWorkloadContext::Impl::create_virtual_tensor() +{ + auto tensor_info = std::make_unique(); + const auto tensor_id = -next_tensor_id(); + tensor_info->set_id(tensor_id); + _mem_map[tensor_id] = MemoryDescriptor{ MemoryType::Virtual }; + auto inserted = _managed_tensor_info.emplace(tensor_info->id(), std::move(tensor_info)); + return inserted.first->second.get(); +} + +ITensorInfo *GpuWorkloadContext::Impl::create_auxiliary_tensor(const ITensorInfo &itensor_info) +{ + auto tensor_info = std::make_unique(itensor_info); + const auto tensor_id = next_tensor_id(); + tensor_info->set_id(tensor_id); + _mem_map[tensor_id] = MemoryDescriptor{ MemoryType::Auxiliary, AuxMemoryInfo{ tensor_info->total_size() } }; + auto inserted = _managed_tensor_info.emplace(tensor_info->id(), std::move(tensor_info)); + return inserted.first->second.get(); +} + +ITensorInfo *GpuWorkloadContext::Impl::get_tensor_info(ITensorInfo::Id id) +{ + return _managed_tensor_info.at(id).get(); +} + +const ITensorInfo *GpuWorkloadContext::Impl::get_tensor_info(ITensorInfo::Id id) const +{ + return _managed_tensor_info.at(id).get(); +} + +ITensorInfo::Id GpuWorkloadContext::Impl::next_tensor_id() +{ + return _next_tensor_id++; +} + } // namespace dynamic_fusion } // namespace experimental } // namespace arm_compute diff --git a/src/dynamic_fusion/sketch/gpu/GpuWorkloadContextImpl.h b/src/dynamic_fusion/sketch/gpu/GpuWorkloadContextImpl.h new file mode 100644 index 0000000000000000000000000000000000000000..c169476a70c01d82c00f403abb5b68f36853df41 --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/GpuWorkloadContextImpl.h @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_GPUWORKLOADCONTEXTIMPL_H +#define ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_GPUWORKLOADCONTEXTIMPL_H + +#include "arm_compute/core/CL/CLCompileContext.h" +#include "arm_compute/core/ITensorInfo.h" +#include "arm_compute/dynamic_fusion/sketch/MemoryDescriptor.h" +#include "arm_compute/dynamic_fusion/sketch/gpu/GpuWorkloadContext.h" + +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ +/** Internal implementation of workload context. */ +class GpuWorkloadContext::Impl +{ +public: + /** Constructor + * + * @param[in] gpu_language Target GPU language. + * @param[in] cl_compile_ctx CL compile context. + */ + Impl(GpuLanguage gpu_language, CLCompileContext *cl_compile_ctx); + + /** Copy constructor */ + Impl(Impl &) = default; + + /** Assignment */ + Impl &operator=(Impl &) = default; + + /** Get target GPU language. */ + GpuLanguage gpu_language() const; + + /** Get CL compile context. */ + const CLCompileContext *cl_compile_context() const; + + /** Get memory descriptor registry. */ + const MemoryDescriptorMap &mem_map() const; + + /** Set a new ID and register the user tensor info. + * + * @param[in, out] tensor_info The tensor info to be registered. + */ + void register_user_tensor(ITensorInfo &tensor_info); + + /** Create a virtual (see @ref MemoryType) tensor info and save it + * + * @return ITensorInfo* The created virtual tensor info object pointer + */ + ITensorInfo *create_virtual_tensor(); + /** Create an auxiliary (see @ref MemoryType) tensor info and save it + * + * @param[in] tensor_info @ref ITensorInfo to copy from + * + * @return ITensorInfo* The created auxiliary tensor info object pointer + */ + ITensorInfo *create_auxiliary_tensor(const ITensorInfo &tensor_info); + + /** Get tensor info created by this context, from id */ + ITensorInfo *get_tensor_info(ITensorInfo::Id id); + + /** Get tensor info created by this context, from id */ + const ITensorInfo *get_tensor_info(ITensorInfo::Id id) const; + +private: + ITensorInfo::Id next_tensor_id(); + + GpuLanguage _gpu_language; + CLCompileContext *_cl_compile_ctx; + + ITensorInfo::Id _next_tensor_id; + MemoryDescriptorMap _mem_map; + std::map> _managed_tensor_info; +}; + +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute + +#endif // ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_GPUWORKLOADCONTEXTIMPL_H diff --git a/src/dynamic_fusion/sketch/gpu/GpuWorkloadSketch.cpp b/src/dynamic_fusion/sketch/gpu/GpuWorkloadSketch.cpp index 33f672071d8b06ad9716ad5b9c74a8187df46002..d3a20c0dfe4ce9119510b0c6793035130ecd8910 100644 --- a/src/dynamic_fusion/sketch/gpu/GpuWorkloadSketch.cpp +++ b/src/dynamic_fusion/sketch/gpu/GpuWorkloadSketch.cpp @@ -43,20 +43,6 @@ const GpuWorkloadSketch::Context *GpuWorkloadSketch::gpu_context() const return _impl->context(); } -void GpuWorkloadSketch::register_new_tensor(ITensorInfo &tensor_info) -{ - tensor_info.set_id(_impl->allocate_new_tensor_id()); - // All input output tensors are User tensors that need real backing memory - _impl->register_memory_descriptor(tensor_info, MemoryDescriptor{ MemoryType::User }); -} - -TensorInfo GpuWorkloadSketch::create_tensor_info() -{ - TensorInfo tensor_info{}; - register_new_tensor(tensor_info); - return tensor_info; -} - GpuWorkloadSketch::Implementation &GpuWorkloadSketch::implementation() { return *_impl; diff --git a/src/dynamic_fusion/sketch/gpu/GpuWorkloadSketchImpl.h b/src/dynamic_fusion/sketch/gpu/GpuWorkloadSketchImpl.h index d5075d5c9446bb3331328218d525d4f5fd74b737..d3033898e9a722c82e4c1105eb9e444a17a50215 100644 --- a/src/dynamic_fusion/sketch/gpu/GpuWorkloadSketchImpl.h +++ b/src/dynamic_fusion/sketch/gpu/GpuWorkloadSketchImpl.h @@ -29,9 +29,7 @@ #include "src/dynamic_fusion/sketch/gpu/GpuComponentServices.h" #include "src/dynamic_fusion/sketch/gpu/GpuKernelComponentGraph.h" #include "src/dynamic_fusion/sketch/gpu/GpuOperatorGroup.h" - -#include -#include +#include "src/dynamic_fusion/sketch/gpu/GpuWorkloadContextImpl.h" namespace arm_compute { @@ -51,10 +49,8 @@ public: Context *context) : _context{ context }, _comp_services{}, - _component_graph{ &_comp_services }, - _operator_group{}, - _managed_tensor_info_list{ std::vector>() }, - _mem_map{} + _component_graph{ _context, &_comp_services }, + _operator_group{} { } /** Prevent instances of this class from being copy constructed */ @@ -90,10 +86,6 @@ public: { return _operator_group; } - ITensorInfo::Id allocate_new_tensor_id() - { - return ++_next_id; - } /** Generate @ref GpuWorkloadSourceCode from the workload sketch * @note The sketch must be valid. Any error encountered during the building of the code will be thrown. * @@ -101,7 +93,8 @@ public: */ GpuWorkloadSourceCode generate_source_code() const { - return component_graph().fuse(_mem_map).write_workload_code(); + const auto mem_map = _context->implementation().mem_map(); + return component_graph().fuse(mem_map).write_workload_code(); } /** Create a virtual (see @ref MemoryType) tensor info and save it * @@ -109,17 +102,8 @@ public: */ ITensorInfo *create_virtual_tensor() { - auto uptr = std::make_unique(); - uptr->set_id(-allocate_new_tensor_id()); // virtual tensors must have negative id - register_memory_descriptor(*uptr, MemoryDescriptor{ MemoryType::Virtual }); - _managed_tensor_info_list.emplace_back(std::move(uptr)); - return _managed_tensor_info_list.back().get(); + return _context->implementation().create_virtual_tensor(); } - /** Create an auxiliary (see @ref MemoryType) tensor info and save it - * - * @return ITensorInfo* The created auxiliary tensor info object pointer - */ - /** Create an auxiliary (see @ref MemoryType) tensor info and save it * * @param[in] tensor_info @ref ITensorInfo to copy from @@ -128,30 +112,19 @@ public: */ ITensorInfo *create_auxiliary_tensor(const ITensorInfo &tensor_info) { - auto uptr = std::make_unique(tensor_info); - uptr->set_id(allocate_new_tensor_id()); - register_memory_descriptor(*uptr, MemoryDescriptor{ MemoryType::Auxiliary, AuxMemoryInfo{ uptr->total_size() } }); - _managed_tensor_info_list.emplace_back(std::move(uptr)); - return _managed_tensor_info_list.back().get(); + return _context->implementation().create_auxiliary_tensor(tensor_info); } - /** Register memory descriptor of a tensor info - * - * @param[in] info @ref ITensorInfo to be registered - * @param[in] mem_desc @ref MemoryDescriptor to be registered with @p info - */ - void register_memory_descriptor(const ITensorInfo &info, const MemoryDescriptor &mem_desc) + + ITensorInfo *get_tensor_info(ITensorInfo::Id id) { - _mem_map[info.id()] = mem_desc; + return _context->implementation().get_tensor_info(id); } private: - Context *_context; - GpuComponentServices _comp_services; - GpuKernelComponentGraph _component_graph; - GpuOperatorGroup _operator_group; - ITensorInfo::Id _next_id{ ITensorInfo::invalid_tensor_id }; - std::vector> _managed_tensor_info_list; - MemoryDescriptorMap _mem_map; + Context *_context; + GpuComponentServices _comp_services; + GpuKernelComponentGraph _component_graph; + GpuOperatorGroup _operator_group; }; } // namespace dynamic_fusion } // namespace experimental diff --git a/src/dynamic_fusion/sketch/gpu/GpuWorkloadSourceCode.h b/src/dynamic_fusion/sketch/gpu/GpuWorkloadSourceCode.h index d1d0bdf77f426b28334720d7689dab8a7ab9337e..578366daaf5b600ce9f0a1f1e6345eb3ceb015f9 100644 --- a/src/dynamic_fusion/sketch/gpu/GpuWorkloadSourceCode.h +++ b/src/dynamic_fusion/sketch/gpu/GpuWorkloadSourceCode.h @@ -27,6 +27,7 @@ #include "arm_compute/core/experimental/Types.h" #include "arm_compute/dynamic_fusion/sketch/MemoryDescriptor.h" #include "src/dynamic_fusion/sketch/gpu/GpuKernelSourceCode.h" +#include "src/dynamic_fusion/sketch/gpu/GpuWorkloadContextImpl.h" namespace arm_compute { @@ -34,10 +35,45 @@ namespace experimental { namespace dynamic_fusion { +#ifdef ACL_INTERNAL_TEST_CKW_IN_DF +namespace +{ +/** Extract kernel arguments of one tensor from a flat list of kernel arguments. + * + * @param[in] flat_kernel_args + * @return GpuKernelArgumentList + */ +GpuKernelArgumentList extract_kernel_args_for_one_tensor(GpuKernelArgumentList &flat_kernel_args) +{ + if(flat_kernel_args.empty()) + { + return {}; + } + GpuKernelArgumentList tensor_kargs{}; + + const GpuKernelArgumentBinding &karg_head = flat_kernel_args.front(); + tensor_kargs.push_back(karg_head); + flat_kernel_args.pop_front(); + const auto tensor_id = karg_head.id(); + + while(!flat_kernel_args.empty()) + { + const GpuKernelArgumentBinding &karg = flat_kernel_args.front(); + if(karg.id() != tensor_id) // Encounter the next tensor, return the current tensor's kernel arguments + { + return tensor_kargs; + } + tensor_kargs.push_back(karg); + flat_kernel_args.pop_front(); + } + return tensor_kargs; +} +} +#endif // ACL_INTERNAL_TEST_CKW_IN_DF /** Uniquely identifies a @ref GpuUnitWorkload within a @ref GpuWorkloadSourceCode */ using UnitWorkloadId = int32_t; -/** Describes all the info related to a kernel in order to: +/** Describes all the info related to a **workload argument** (tensor) in order to: * - be used by runtime to configure gpu kernel argument * - be used by memory managers to allocate required memory */ @@ -46,6 +82,7 @@ class GpuWorkloadArgument public: /** Default constructor */ GpuWorkloadArgument() = default; +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF /** Constructor * * @param[in] tensor_info @ref ITensorInfo of the workload argument @@ -60,6 +97,22 @@ public: _kernel_arg_info{ kernel_arg_info } { } +#else // ACL_INTERNAL_TEST_CKW_IN_DF + /** Constructor + * + * @param[in] tensor_info @ref ITensorInfo of the workload argument + * @param[in] mem_desc @ref MemoryDescriptor of the workload argument + * @param[in] kernel_arg_list @ref GpuKernelArgumentList of the workload argument + */ + GpuWorkloadArgument(const ITensorInfo &tensor_info, + const MemoryDescriptor &mem_desc, + const GpuKernelArgumentList &kernel_args) + : _tensor_info{ tensor_info }, + _mem_desc{ mem_desc }, + _kernel_args{ kernel_args } + { + } +#endif // ACL_INTERNAL_TEST_CKW_IN_DF /** Get tensor id within workload */ ITensorInfo::Id id() const { @@ -85,6 +138,7 @@ public: { return &_mem_desc; } +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF /** Get @ref GpuKernelArgumentInfo of the argument */ GpuKernelArgumentInfo *kernel_argument_info() { @@ -95,6 +149,18 @@ public: { return &_kernel_arg_info; } +#else // ACL_INTERNAL_TEST_CKW_IN_DF + /** Get @ref GpuKernelArgumentList of the workload tensor */ + GpuKernelArgumentList *kernel_argument_list() + { + return &_kernel_args; + } + /** Get @ref GpuKernelArgumentList of the workload tensor */ + const GpuKernelArgumentList *kernel_argument_list() const + { + return &_kernel_args; + } +#endif // ACL_INTERNAL_TEST_CKW_IN_DF /** Check if the workload argument has valid id * * @return true If has valid id @@ -106,9 +172,13 @@ public: } private: - TensorInfo _tensor_info{}; - MemoryDescriptor _mem_desc{}; - GpuKernelArgumentInfo _kernel_arg_info{}; + TensorInfo _tensor_info{}; + MemoryDescriptor _mem_desc{}; +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF + GpuKernelArgumentInfo _kernel_arg_info {}; +#else // ACL_INTERNAL_TEST_CKW_IN_DF + GpuKernelArgumentList _kernel_args {}; +#endif // ACL_INTERNAL_TEST_CKW_IN_DF }; /** Describes when a unit workload is run. @@ -179,15 +249,18 @@ public: * @param[in] kernel_code @ref GpuKernelSourceCode to be contained within the unit workload * @param[in] stage Stage of the unit workload * @param[in] mem_map @ref MemoryDescriptor map for all tensors within the unit workload + * @param[in] context @ref GpuWorkloadContext associated with the unit workload * * @return UnitWorkloadId Allocated unit workload id */ - UnitWorkloadId add_unit_workload(const GpuKernelSourceCode &kernel_code, const UnitWorkloadStage &stage, const MemoryDescriptorMap &mem_map) + UnitWorkloadId add_unit_workload(const GpuKernelSourceCode &kernel_code, const UnitWorkloadStage &stage, const MemoryDescriptorMap &mem_map, const GpuWorkloadContext *context) { // Use the size of the kernel codes as Id const auto uwk_id = static_cast(_unit_workloads.size()); const auto unit_work = GpuUnitWorkload(uwk_id, kernel_code, stage); _unit_workloads.push_back(unit_work); +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF + ARM_COMPUTE_UNUSED(context); // Assemble kernel argument with memory descriptor to form workload argument for(const auto &id_arg : kernel_code.arguments()) { @@ -200,6 +273,28 @@ public: } _tensor_uwork_map[arg_id].insert(uwk_id); } +#else // ACL_INTERNAL_TEST_CKW_IN_DF + GpuKernelArgumentList flat_kernel_args = kernel_code.arguments(); + GpuKernelArgumentList tensor_kargs{}; + while(true) + { + tensor_kargs = extract_kernel_args_for_one_tensor(flat_kernel_args); + if(tensor_kargs.empty()) + { + break; + } + else + { + const auto tensor_id = tensor_kargs.at(0).id(); + _workload_arguments[tensor_id] = GpuWorkloadArgument{ *context->implementation().get_tensor_info(tensor_id), mem_map.at(tensor_id), tensor_kargs }; + if(_tensor_uwork_map.find(tensor_id) == _tensor_uwork_map.end()) + { + _tensor_uwork_map[tensor_id] = std::set(); + } + _tensor_uwork_map[tensor_id].insert(uwk_id); + } + } +#endif // ACL_INTERNAL_TEST_CKW_IN_DF return uwk_id; } /** Get a unit workload from its id */ diff --git a/src/dynamic_fusion/sketch/gpu/IGpuKernelWriter.h b/src/dynamic_fusion/sketch/gpu/IGpuKernelWriter.h index ae67790b4b188b2b68ecdcc3a153f129832c694a..1d8b231efdab759548a92738c3f42925c73d665c 100644 --- a/src/dynamic_fusion/sketch/gpu/IGpuKernelWriter.h +++ b/src/dynamic_fusion/sketch/gpu/IGpuKernelWriter.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,9 +27,11 @@ #include "arm_compute/core/CL/CLCompileContext.h" #include "arm_compute/core/Window.h" #include "src/dynamic_fusion/sketch/gpu/GpuKernelArgument.h" +#include "src/dynamic_fusion/sketch/gpu/GpuKernelSourceCode.h" #include #include +#include namespace arm_compute { @@ -51,13 +53,28 @@ public: /** Generate kernel code */ virtual std::string get_code() = 0; /** Generate build options */ - virtual CLBuildOptions get_build_options() = 0; + virtual CLBuildOptions get_build_options() + { + return {}; + } /** Generate config id string of the entire kernel. This is used for tuning */ virtual std::string get_config_id() = 0; /** Generate execution window */ virtual Window get_window() const = 0; - /** Get the kernel argument lists of the kernel*/ - virtual std::map get_tensors() = 0; + /** Get the kernel argument lists of the kernel + * @deprecated To be removed along with ClTemplateWriter + */ + virtual std::map get_tensors() + { + return {}; + } +#ifdef ACL_INTERNAL_TEST_CKW_IN_DF + /** Get the flat list of arguments of the kernel*/ + virtual GpuKernelArgumentList get_kernel_arguments() + { + return {}; + } +#endif // ACL_INTERNAL_TEST_CKW_IN_DF }; } // namespace dynamic_fusion diff --git a/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwComponentArgument.cpp b/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwComponentArgument.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4b4c22fa1d23779c0aeadad606f0f16b32914caf --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwComponentArgument.cpp @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwComponentArgument.h" +#include "ckw/Error.h" + +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ + +GpuCkwComponentArgument::GpuCkwComponentArgument() +{ +} + +GpuCkwComponentArgument::GpuCkwComponentArgument(ckw::TensorOperand &tensor) + : _tensor(&tensor) +{ +} + +GpuCkwComponentArgument &GpuCkwComponentArgument::init_virtual_tensor(ckw::TileOperand &tile, const ckw::TensorTileSampler &tile_sampler) +{ + CKW_ASSERT(_tile == nullptr); + + _tile = &tile; + _tile_sampler = tile_sampler; + + return *this; +} + +bool GpuCkwComponentArgument::has_tensor() const +{ + return _tensor != nullptr; +} + +ckw::TensorOperand &GpuCkwComponentArgument::tensor() +{ + CKW_ASSERT(_tensor != nullptr); + + return *_tensor; +} + +const ckw::TensorOperand &GpuCkwComponentArgument::tensor() const +{ + CKW_ASSERT(_tensor != nullptr); + + return *_tensor; +} + +bool GpuCkwComponentArgument::has_tile() const +{ + return _tile != nullptr; +} + +ckw::TileOperand &GpuCkwComponentArgument::tile() +{ + CKW_ASSERT(_tile != nullptr); + + return *_tile; +} + +const ckw::TileOperand &GpuCkwComponentArgument::tile() const +{ + CKW_ASSERT(_tile != nullptr); + + return *_tile; +} + +ckw::TensorTileSampler &GpuCkwComponentArgument::tile_sampler() +{ + CKW_ASSERT(_tile != nullptr); + + return _tile_sampler; +} + +const ckw::TensorTileSampler &GpuCkwComponentArgument::tile_sampler() const +{ + CKW_ASSERT(_tile != nullptr); + + return _tile_sampler; +} + +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute diff --git a/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwComponentArgument.h b/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwComponentArgument.h new file mode 100644 index 0000000000000000000000000000000000000000..80f91389a039e84810429c9d11ecf96508cc7401 --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwComponentArgument.h @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_GPUCKWCOMPONENTARGUMENT_H +#define ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_GPUCKWCOMPONENTARGUMENT_H + +#include "ckw/TensorTileSampler.h" + +namespace ckw +{ +class TensorOperand; +class TileOperand; +} // namespace ckw + +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ + +/** The argument of a dynamic fusion component which can be either user tensor or virtual tensor. */ +class GpuCkwComponentArgument +{ +public: + /** Initialize a new instance of @ref GpuCkwComponentArgument class for empty virtual tensor. */ + GpuCkwComponentArgument(); + + /** Initialize a new instance of @ref GpuCkwComponentArgument class for user tensor. + * + * @param[in] tensor The user tensor. + */ + explicit GpuCkwComponentArgument(ckw::TensorOperand &tensor); + + /** Set virtual tensor information (tile, sampler) for the argument. + * + * If the component is a user tensor, it can be treated as virtual tensor as well + * and won't be loaded again using @ref GpuCkwKernelWriter::op_load_once method. + * + * @param[in] tile The tile that has been loaded. + * @param[in] sampler The tensor sampling information that has been used to load the tile. + */ + GpuCkwComponentArgument &init_virtual_tensor(ckw::TileOperand &tile, const ckw::TensorTileSampler &sampler); + + /** Get whether the argument is a user tensor. */ + bool has_tensor() const; + + /** Get the tensor operand. + * + * If the tensor is not available, throw an error. + */ + ckw::TensorOperand &tensor(); + + /** Get the tensor operand. + * + * If the tensor is not available, throw an error. + */ + const ckw::TensorOperand &tensor() const; + + /** Get whether the argument contains a tile. + * + * The argument can be either a user tensor that has been loaded, + * or a virtual tensor (i.e. a tile with tensor sampling information). + */ + bool has_tile() const; + + /** Get the tile operand. + * + * If the tile is not available, throw an error. + */ + ckw::TileOperand &tile(); + + /** Get the tile operand. + * + * If the tile is not available, throw an error. + */ + const ckw::TileOperand &tile() const; + + /** Get the tensor sampling information for the tile. + * + * If the tile is not available, throw an error. + */ + ckw::TensorTileSampler &tile_sampler(); + + /** Get the tensor sampling information for the tile. + * + * If the tile is not available, throw an error. + */ + const ckw::TensorTileSampler &tile_sampler() const; + +private: + ckw::TensorOperand *_tensor{ nullptr }; + ckw::TileOperand *_tile{ nullptr }; + ckw::TensorTileSampler _tile_sampler{}; +}; + +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute + +#endif // ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_GPUCKWCOMPONENTARGUMENT_H diff --git a/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwDriver.cpp b/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwDriver.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a24a172d773157bd0d2bbea046c8077bcc1c5f6a --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwDriver.cpp @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwDriver.h" + +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/IGpuCkwComponentDriver.h" +#include "src/dynamic_fusion/sketch/gpu/components/IGpuKernelComponent.h" + +#include "arm_compute/core/Error.h" +#include "arm_compute/core/Window.h" +#include "src/common/utils/Log.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwVariableTable.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/components/utils/type_converter/Common.h" + +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwKernelWriter.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwScopedKernelWriter.h" + +using namespace ckw; +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ +GpuCkwDriver::GpuCkwDriver(const GpuKernelComponentGroup &components) + : _components{ components }, _kernel{ GpuTargetLanguage::OpenCL }, _code{} +{ + // Generate kernel name + std::string name = ""; + for(auto &comp : _components) + { + auto ckw_driver = comp->ckw_component_driver(); + ARM_COMPUTE_ERROR_ON(ckw_driver == nullptr); + name += ckw_driver->get_name(_components) + "__"; + } + + // Generate kernel code + _kernel.name(name); + GpuCkwKernelWriter root_writer(_kernel); + GpuCkwScopedKernelWriter writer(&root_writer); + GpuCkwVariableTable vtable{}; + + for(auto &comp : _components) + { + auto ckw_driver = comp->ckw_component_driver(); + ARM_COMPUTE_ERROR_ON(ckw_driver == nullptr); + ckw_driver->write_component_code(_components, vtable, writer); + } + _code = root_writer.generate_code(); +} + +std::string GpuCkwDriver::get_name() +{ + return _kernel.name(); +} + +std::string GpuCkwDriver::get_code() +{ + return _code; +} + +std::string GpuCkwDriver::get_config_id() +{ + std::string id = ""; + for(auto &comp : _components) + { + auto ckw_driver = comp->ckw_component_driver(); + ARM_COMPUTE_ERROR_ON(ckw_driver == nullptr); + id = ckw_driver->get_tuner_id(_components) + "__"; + } + return id; +} + +Window GpuCkwDriver::get_window() const +{ + const auto root_comp = _components.get_root_component(); + ARM_COMPUTE_ERROR_ON_MSG(root_comp == nullptr, "No root component found"); + return root_comp->ckw_component_driver()->get_window(); +} + +GpuKernelArgumentList GpuCkwDriver::get_kernel_arguments() +{ + GpuKernelArgumentList args{}; + for(const auto &arg : _kernel.arguments()) + { + switch(arg.type()) + { + case KernelArgument::Type::TensorStorage: + { + args.emplace_back(static_cast(arg.id()), from_ckw(arg.tensor_storage_type())); + break; + } + case KernelArgument::Type::TensorComponent: + { + args.emplace_back(static_cast(arg.id()), from_ckw(arg.tensor_component_type())); + break; + } + default: + { + ARM_COMPUTE_ERROR("Unsupported KernelArgument Type"); + break; + } + } + } + return args; +} + +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute diff --git a/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwDriver.h b/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwDriver.h new file mode 100644 index 0000000000000000000000000000000000000000..19db575fea7870535981890cc88b50ca3c0be24f --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwDriver.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_GPUCKWDRIVER +#define ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_GPUCKWDRIVER + +#include "src/dynamic_fusion/sketch/gpu/GpuKernelArgument.h" +#include "src/dynamic_fusion/sketch/gpu/GpuKernelComponentGroup.h" +#include "src/dynamic_fusion/sketch/gpu/IGpuKernelWriter.h" + +#include "ckw/Kernel.h" + +#include +#include + +namespace arm_compute +{ +/** Forward declarations */ +class Window; + +namespace experimental +{ +namespace dynamic_fusion +{ +/** Use Kernel Writer to write kernel code + * Used by dynamic_fusion module + */ +class GpuCkwDriver : public IGpuKernelWriter +{ +public: + /** Default constructor */ + GpuCkwDriver() = default; + /** Constructor + * + * @param[in] components Kernel component group from which the kernel will be generated + */ + GpuCkwDriver(const GpuKernelComponentGroup &components); + /** Destructor */ + ~GpuCkwDriver() override = default; + /** Generate kernel name */ + std::string get_name() override; + /** Generate kernel code */ + std::string get_code() override; + /** Generate config id string of the entire kernel. This is used for tuning */ + std::string get_config_id() override; + /** Generate execution window */ + Window get_window() const override; + /** Get the flat list of arguments of the kernel*/ + GpuKernelArgumentList get_kernel_arguments() override; + +private: + GpuKernelComponentGroup _components{}; + ckw::Kernel _kernel; + std::string _code; +}; + +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute + +#endif /* ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_GPUCKWDRIVER */ diff --git a/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwKernelWriter.cpp b/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwKernelWriter.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ca4f121566b546184a9bb342250376e4b77751f2 --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwKernelWriter.cpp @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwKernelWriter.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwComponentArgument.h" +#include "ckw/Error.h" +#include "ckw/TileInfo.h" + +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ + +GpuCkwKernelWriter::GpuCkwKernelWriter(ckw::Kernel &kernel) + : KernelWriter(kernel) +{ +} + +void GpuCkwKernelWriter::op_load_once(GpuCkwComponentArgument *tensor_or_tile, const ckw::TensorTileSampler &sampler) +{ + if(!tensor_or_tile->has_tile()) + { + CKW_ASSERT(tensor_or_tile->has_tensor()); + + auto &tensor = tensor_or_tile->tensor(); + + const auto tile_name = tensor.name() + "_tile"; + auto &tile = declare_tile(tile_name.c_str(), ckw::TileInfo(tensor.data_type(), sampler.height(), sampler.width())); + + op_load(tile, tensor, sampler); + + tensor_or_tile->init_virtual_tensor(tile, sampler); + } +} + +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute diff --git a/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwKernelWriter.h b/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwKernelWriter.h new file mode 100644 index 0000000000000000000000000000000000000000..b916e6b28bb225bffb0bf3b23a91e1402719fec4 --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwKernelWriter.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_GPUCKWKERNELWRITER_H +#define ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_GPUCKWKERNELWRITER_H + +#include "ckw/KernelWriter.h" +#include "ckw/TensorTileSampler.h" + +namespace ckw +{ +class Kernel; +} // namespace ckw + +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ + +class GpuCkwComponentArgument; + +/** Extended implementation of kernel writer for dynamic fusion. */ +class GpuCkwKernelWriter : public ckw::KernelWriter +{ +public: + /** Initialize a new instance of @ref GpuCkwKernelWriter class. + * + * @param[in] kernel The kernel to be generated. + */ + explicit GpuCkwKernelWriter(ckw::Kernel &kernel); + + /** Load the user tensor to the tile in the same component argument if it hasn't been loaded. + * + * @param[in] tensor_or_tile The component argument that is either a user tensor or a virtual tensor. + * @param[in] sampler The tensor sampling information to load the tile. + */ + void op_load_once(GpuCkwComponentArgument *tensor_or_tile, const ckw::TensorTileSampler &sampler); +}; + +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute + +#endif // ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_GPUCKWKERNELWRITER_H diff --git a/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwScopedKernelWriter.cpp b/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwScopedKernelWriter.cpp new file mode 100644 index 0000000000000000000000000000000000000000..043fda9e6f13d807ae90f3e231ce36051c3d173c --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwScopedKernelWriter.cpp @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwScopedKernelWriter.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwKernelWriter.h" + +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ + +GpuCkwScopedKernelWriter::GpuCkwScopedKernelWriter(GpuCkwKernelWriter *writer) + : _writer(writer), _parent_id_space(writer->id_space()) +{ + _writer->next_id_space(); +} + +GpuCkwScopedKernelWriter::GpuCkwScopedKernelWriter(const GpuCkwScopedKernelWriter &other) + : _writer(other._writer), _parent_id_space(other._writer->id_space()) +{ + _writer->next_id_space(); +} + +GpuCkwKernelWriter *GpuCkwScopedKernelWriter::operator->() +{ + return _writer; +} + +const GpuCkwKernelWriter *GpuCkwScopedKernelWriter::operator->() const +{ + return _writer; +} + +GpuCkwKernelWriter *GpuCkwScopedKernelWriter::writer() +{ + return _writer; +} + +const GpuCkwKernelWriter *GpuCkwScopedKernelWriter::writer() const +{ + return _writer; +} + +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute diff --git a/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwScopedKernelWriter.h b/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwScopedKernelWriter.h new file mode 100644 index 0000000000000000000000000000000000000000..4d11b5e3e41212a357ce29f6e275c1012be4bd28 --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwScopedKernelWriter.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_GPUCKWSCOPEDKERNELWRITER_H +#define ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_GPUCKWSCOPEDKERNELWRITER_H + +#include + +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ + +class GpuCkwKernelWriter; + +/** Helper to automatically manage kernel writer ID space. */ +class GpuCkwScopedKernelWriter +{ +public: + /** Initialize a new instance of @ref GpuCkwScopedKernelWriter class. */ + explicit GpuCkwScopedKernelWriter(GpuCkwKernelWriter *writer); + + /** Create a new scope from the specified scoped kernel writer. */ + GpuCkwScopedKernelWriter(const GpuCkwScopedKernelWriter &other); + + /** Assignment is disallowed. */ + GpuCkwScopedKernelWriter &operator=(const GpuCkwScopedKernelWriter &) = delete; + + /** Access the underlying kernel writer. */ + GpuCkwKernelWriter *operator->(); + + /** Access the underlying kernel writer. */ + const GpuCkwKernelWriter *operator->() const; + + /** Get the kernel writer. */ + GpuCkwKernelWriter *writer(); + + /** Get the kernel writer. */ + const GpuCkwKernelWriter *writer() const; + +private: + GpuCkwKernelWriter *_writer; + int32_t _parent_id_space; +}; + +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute + +#endif // ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_GPUCKWSCOPEDKERNELWRITER_H diff --git a/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwVariableTable.cpp b/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwVariableTable.cpp new file mode 100644 index 0000000000000000000000000000000000000000..37c27cd116c0661f7a36302583d5518fea8caef1 --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwVariableTable.cpp @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwVariableTable.h" + +#include "src/dynamic_fusion/sketch/gpu/GpuKernelArgument.h" +#include "src/dynamic_fusion/sketch/gpu/GpuKernelComponentGroup.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwKernelWriter.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwScopedKernelWriter.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/components/utils/type_converter/Common.h" +#include + +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ +GpuCkwComponentArgument *GpuCkwVariableTable::declare_variable(const GpuKernelComponentGroup &comp_group, GpuCkwScopedKernelWriter &writer, const ITensorInfo *tensor, TensorStorageType storage, + const std::string &alias) +{ + ARM_COMPUTE_ERROR_ON_MSG(!tensor->has_valid_id(), "Tensor info with valid id expected"); + + // Do not re-declare if the variable associated with the tensor has already been declared + auto it = _vars.find(tensor->id()); + + if(it != _vars.end()) + { + return &it->second; + } + if(comp_group.is_intermediate_tensor(tensor)) + { + // Create a virtual tensor variable + GpuCkwComponentArgument var; + auto &&inserted = _vars.emplace(tensor->id(), var); + return &(inserted.first->second); + } + else + { + // Create a user tensor variable + std::stringstream ss; + ss << alias << "_t" << abs(tensor->id()); + const auto uniq_name = ss.str(); + GpuCkwComponentArgument var{ writer->declare_tensor_argument(uniq_name, to_ckw(*tensor), to_ckw(storage)) }; + auto &&inserted = _vars.emplace(tensor->id(), var); + return &(inserted.first->second); + } +} +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute diff --git a/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwVariableTable.h b/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwVariableTable.h new file mode 100644 index 0000000000000000000000000000000000000000..0649dcba9df0f137bd5e335a95ba6197d7b89d00 --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwVariableTable.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_GPUCKWVARIABLETABLE +#define ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_GPUCKWVARIABLETABLE + +#include "arm_compute/core/ITensorInfo.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwComponentArgument.h" + +#include + +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ +class GpuKernelComponentGroup; +class GpuCkwScopedKernelWriter; +enum class TensorStorageType; + +/** A table of all the variables used in the kernel. + * + * It determines whether we create an virtual tensor var or a user tensor var + * It avoids duplicating variables for the same tensors (Tensors with the same id) + * Each kernel has exactly one variable table. + */ +class GpuCkwVariableTable +{ +public: + /** Declare a kernel component variable(argument) for the corresponding tensor info. + * + * @param[in] comp_group Component group the tensor belongs to + * @param[in] writer Compute Kernel Writer + * @param[in] tensor Tensor info with which the new variable is associated + * @param[in] storage Tensor storage type associated with the tensor + * @param[in] alias Alias for the variable. Will be used as part of the variable name + * + * @return GpuCkwComponentArgument* + */ + GpuCkwComponentArgument *declare_variable(const GpuKernelComponentGroup &comp_group, GpuCkwScopedKernelWriter &writer, const ITensorInfo *tensor, TensorStorageType storage, + const std::string &alias = "unnamed"); + +private: + std::map _vars{}; +}; + +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute +#endif /* ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_GPUCKWVARIABLETABLE */ diff --git a/src/dynamic_fusion/sketch/gpu/ckw_driver/IGpuCkwComponentDriver.h b/src/dynamic_fusion/sketch/gpu/ckw_driver/IGpuCkwComponentDriver.h new file mode 100644 index 0000000000000000000000000000000000000000..14086f785e0b79b5fb4b9d0d1b06c5ff73e5f801 --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/ckw_driver/IGpuCkwComponentDriver.h @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_IGPUCKWCOMPONENTDRIVER +#define ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_IGPUCKWCOMPONENTDRIVER + +#include "arm_compute/core/Window.h" +#include "src/dynamic_fusion/sketch/ArgumentPack.h" +#include "src/dynamic_fusion/sketch/gpu/components/Types.h" + +namespace arm_compute +{ +class ITensorInfo; +namespace experimental +{ +namespace dynamic_fusion +{ +/** Forward declaration */ +class GpuKernelComponentGroup; +class GpuCkwVariableTable; +class GpuCkwScopedKernelWriter; + +/** An interface used by @ref GpuCkwDriver to write source code for a kernel component + * + * There are 3 main architecture layers for using Compute Kernel Writer (Ckw) inside ACL's dynamic fusion module + * From top level to bottom level: + * | Layer | Library + * =========================== + * | dynamic_fusion | acl + * | ckw_driver | acl + * | ckw | ckw + * + * ckw_driver is a glue layer that directs how fused code is produced using the ckw library + * + * There are two main groups within ckw_driver: + * - @ref GpuCkwDriver is a global driver that coordinates how the final fused code along with all the info necessary + * for run time execution is produced using ckw + * - Various classes implementing @ref IGpuCkwComponentDriver is a component driver that directs ckw to generate kernel component code (e.g. activation, store etc.) + * + * The overall flow goes like this: + * In dynamic_fusion module, @ref GpuLogicalKernel instantiates a @ref GpuCkwDriver from a @ref GpuKernelComponentGroup + * The logical kernel then uses the global driver's various interfaces to generate the code info. + * In particular, the @ref GpuCkwDriver::get_code() interface will call into each @ref IGpuCkwComponentDriver::write_component_code() + */ +class IGpuCkwComponentDriver +{ +public: + using ComponentGroup = GpuKernelComponentGroup; + +public: + /** Constructor + * + * @param[in] id Component id + * @param[in] tensors Tensor arguments to the components + */ + IGpuCkwComponentDriver(ComponentId id, const ArgumentPack &tensors) + : _id{ id }, _tensors{ tensors } + { + } + /** Destructor */ + virtual ~IGpuCkwComponentDriver() + { + } + /** Generate kernel component code + * + * @param[in] comp_group Component group of which the component is a part of + * @param[in, out] vtable Table of variables declared by each component + * @param[in, out] writer CKW writer that writes code scoped to this kernel component. + * + * @note @p writer can only be passed via value since the new scope is created in the copy constructor + */ + virtual void write_component_code(const ComponentGroup &comp_group, GpuCkwVariableTable &vtable, GpuCkwScopedKernelWriter writer) const = 0; + /** Get tensor arguments */ + ArgumentPack tensors() const + { + return _tensors; + } + /** Generate the execution window for the component */ + virtual Window get_window() const + { + return Window{}; + } + /** Generate the name of the component + * + * This will be concatenated with other components' names to form the name of the kernel + */ + virtual std::string get_name(const ComponentGroup &comp_group) const + { + ARM_COMPUTE_UNUSED(comp_group); + return "unnamed"; + } + /** Generate the tuner id of the component + * This id should capture all the parameters that distinguish one kernel's lws tuning from another. + * e.g. two components that are identical in every other way, but have output tensor dimensions should + * have different tuner ids, because the lws of one may not be optimal on the other. + * + * This will be concatenated with other components' tuner id to form the tuner id of the kernel + */ + virtual std::string get_tuner_id(const ComponentGroup &comp_group) const + { + ARM_COMPUTE_UNUSED(comp_group); + return ""; + } + /** Get component id */ + ComponentId id() const + { + return _id; + } + +private: + ComponentId _id{ -1 }; + ArgumentPack _tensors{}; +}; +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute + +#endif /* ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_IGPUCKWCOMPONENTDRIVER */ diff --git a/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwActivation.cpp b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwActivation.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c07fac0e0d997ac125ab91c232f19d307f5d5cce --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwActivation.cpp @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "GpuCkwActivation.h" + +#include "arm_compute/core/Error.h" +#include "arm_compute/core/Validate.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "ckw/TensorTileSampler.h" +#include "src/core/helpers/WindowHelpers.h" +#include "src/dynamic_fusion/sketch/gpu/GpuKernelArgument.h" +#include "src/dynamic_fusion/sketch/gpu/GpuKernelComponentGroup.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwKernelWriter.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwScopedKernelWriter.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwVariableTable.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/components/utils/WriterHelper.h" +#include + +using namespace ckw; +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ +namespace +{ +/** Create a simple sampler from tile of dimension [m0, n0] + */ +inline TensorTileSampler create_sampler(GpuCkwScopedKernelWriter &writer, int32_t m0, int32_t n0) +{ + TensorTileSampler sampler; + + auto &gid_0 = writer->declare_tile("gid_0", ckw::DataType::Int32); + auto &gid_1 = writer->declare_tile("gid_1", ckw::DataType::Int32); + auto &gid_2 = writer->declare_tile("gid_2", ckw::DataType::Int32); + + auto &const_0 = writer->declare_tile("0", 0); + writer->op_get_global_id(gid_0, 0); + writer->op_get_global_id(gid_1, 1); + writer->op_get_global_id(gid_2, 2); + + auto &x_coord = writer->declare_tile("x_coord", ckw::DataType::Int32); + auto &y_coord = writer->declare_tile("y_coord", ckw::DataType::Int32); + auto &m0_t = writer->declare_tile("m0", m0); + auto &n0_t = writer->declare_tile("n0", n0); + writer->op_binary_expression(x_coord, gid_0, BinaryOp::Mul, n0_t); + writer->op_binary_expression(y_coord, gid_1, BinaryOp::Mul, m0_t); + + sampler.x(x_coord); + sampler.y(y_coord); + sampler.z(const_0); // 3rd dimension collapsed with 2nd dimension + sampler.b(gid_2); + + sampler.width(n0); + sampler.height(m0); + + sampler.format(TensorSamplerFormat::C_WH_1); // 3rd dimension collapsed with 2nd dimension + sampler.address_mode_x(TensorSamplerAddressModeX::None); + sampler.address_mode_y(TensorSamplerAddressModeY::ClampToBorder); + sampler.address_mode_z(TensorSamplerAddressModeZ::Skip); // Dimensions higher than 3 not supported yet + + return sampler; +} +} // namespace + +GpuCkwActivation::GpuCkwActivation(ComponentId id, + const ArgumentPack &tensors, + const Attributes &attributes) + : IGpuCkwComponentDriver{ id, tensors }, + _src{}, + _dst{}, + _attributes{ attributes } +{ + _src = this->tensors().get_const_tensor(TensorType::ACL_SRC_0); + _dst = this->tensors().get_const_tensor(TensorType::ACL_DST_0); + ARM_COMPUTE_ERROR_ON_NULLPTR(_src, _dst); +} + +void GpuCkwActivation::write_component_code(const ComponentGroup &comp_group, GpuCkwVariableTable &vtable, GpuCkwScopedKernelWriter writer) const +{ + const auto root_window = comp_group.get_root_component()->ckw_component_driver()->get_window(); + const unsigned int n0 = root_window.x().step(); + const unsigned int m0 = root_window.y().step(); + + GpuCkwComponentArgument *src = vtable.declare_variable(comp_group, writer, _src, TensorStorageType::ClBufferUint8Ptr, "src"); + GpuCkwComponentArgument *dst = vtable.declare_variable(comp_group, writer, _dst, TensorStorageType::ClBufferUint8Ptr, "dst"); + + load_src_dst_tiles_and_prepare_sampler(writer, src, dst, m0, n0, create_sampler); + + auto &src_tile = src->tile(); + auto &dst_tile = dst->tile(); + + // Constants + const auto &constant_minus_1 = writer->declare_tile("minus_1", -1); + const auto &constant_pos_1 = writer->declare_tile("one", 1); + const auto &constant_zero = writer->declare_tile("zero", 0); + const auto &constant_A = writer->declare_tile("A_VAL", _attributes.a()); + const auto &constant_B = writer->declare_tile("B_VAL", _attributes.b()); + + // Perform the operation. + switch(_attributes.activation()) + { + case ActivationLayerInfo::ActivationFunction::LOGISTIC: + { + // dst = src * -1 + writer->op_binary_expression(dst_tile, src_tile, BinaryOp::Mul, constant_minus_1); + // dst = exp(src * -1) + writer->op_unary_elementwise_function(dst_tile, UnaryFunction::Exp, dst_tile); + // dst = 1 + (exp(src * -1)) + writer->op_binary_expression(dst_tile, dst_tile, BinaryOp::Add, constant_pos_1); + // dst = 1 / 1 + (exp(src * -1)) + writer->op_binary_expression(dst_tile, constant_pos_1, BinaryOp::Div, dst_tile); + break; + } + case ActivationLayerInfo::ActivationFunction::TANH: + { + // dst = B_VAL * src + writer->op_binary_expression(dst_tile, src_tile, BinaryOp::Mul, constant_B); + // dst = tanh(B_VAL * src) + writer->op_unary_elementwise_function(dst_tile, UnaryFunction::Tanh, dst_tile); + // dst = A_VAL * tanh(B_VAL * src) + writer->op_binary_expression(dst_tile, dst_tile, BinaryOp::Mul, constant_A); + break; + } + case ActivationLayerInfo::ActivationFunction::RELU: + { + // dst = max(src, 0) + writer->op_binary_elementwise_function(dst_tile, ckw::BinaryFunction::Max, src_tile, constant_zero); + break; + } + case ActivationLayerInfo::ActivationFunction::BOUNDED_RELU: + { + //dst = max(src, 0) + writer->op_binary_elementwise_function(dst_tile, ckw::BinaryFunction::Max, src_tile, constant_zero); + //dst = min(max(src, 0), A_VAL) + writer->op_binary_elementwise_function(dst_tile, ckw::BinaryFunction::Min, dst_tile, constant_A); + break; + } + case ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU: + { + //dst = max(src, B_VAL) + writer->op_binary_elementwise_function(dst_tile, ckw::BinaryFunction::Max, src_tile, constant_B); + //dst = min(max(src, B_VAL), A_VAL) + writer->op_binary_elementwise_function(dst_tile, ckw::BinaryFunction::Min, dst_tile, constant_A); + break; + } + default: + CKW_ASSERT(false); + break; + } +} + +Window GpuCkwActivation::get_window() const +{ + ARM_COMPUTE_ERROR_ON_MSG(_dst->tensor_shape().total_size() == 0U, "Destination tensor is not initialized"); + + TensorShape output_shape = _dst->tensor_shape(); + // Collapse Dim 1 (W) and Dim 2 (H) together, leave Dim 0 (C) unchanged + // This is in line with the collapsing convention used by operators like Conv2d + output_shape.collapse(2U, 1U); + constexpr unsigned int vector_size_byte_opencl = 16; + const unsigned int num_elems_processed_per_iteration = adjust_vec_size(vector_size_byte_opencl / _dst->element_size(), _dst->dimension(0)); + Window win = calculate_max_window(output_shape, Steps(num_elems_processed_per_iteration)); + + return win; +} + +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute diff --git a/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwActivation.h b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwActivation.h new file mode 100644 index 0000000000000000000000000000000000000000..e157e36cbf298455acbace56e01c1011e251d20b --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwActivation.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_COMPONENTS_GPUCKWACTIVATION +#define ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_COMPONENTS_GPUCKWACTIVATION + +#include "src/core/common/Macros.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/IGpuCkwComponentDriver.h" +#include "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentActivation.h" + +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ +class GpuCkwActivation : public IGpuCkwComponentDriver +{ +public: + using Attributes = ClComponentActivation::Attributes; + /** Constructor + * + * For supported configurations please refer to @ref GpuCkwActivation::validate() + * + * @param[in] id Component id + * @param[in] tensors Tensor arguments to the component + * @param[in] attributes Component attributes + */ + GpuCkwActivation(ComponentId id, + const ArgumentPack &tensors, + const Attributes &attributes); + ARM_COMPUTE_DISALLOW_COPY_ALLOW_MOVE(GpuCkwActivation); + /** Destructor */ + ~GpuCkwActivation() override = default; + // Inherited methods overriden: + virtual void write_component_code(const ComponentGroup &comp_group, GpuCkwVariableTable &vtable, GpuCkwScopedKernelWriter writer) const override; + Window get_window() const override; + +private: + const ITensorInfo *_src; + const ITensorInfo *_dst; + Attributes _attributes; +}; +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute + +#endif /* ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_COMPONENTS_GPUCKWACTIVATION */ diff --git a/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwCast.cpp b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwCast.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6ecf2bac44a0a0b4f2c84d738fc395ccd10b827a --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwCast.cpp @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "GpuCkwCast.h" + +#include "arm_compute/core/Error.h" +#include "arm_compute/core/Validate.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "ckw/TensorTileSampler.h" +#include "src/core/helpers/WindowHelpers.h" +#include "src/dynamic_fusion/sketch/gpu/GpuKernelArgument.h" +#include "src/dynamic_fusion/sketch/gpu/GpuKernelComponentGroup.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwKernelWriter.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwScopedKernelWriter.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwVariableTable.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/components/utils/type_converter/Common.h" +#include + +using namespace ckw; +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ +namespace +{ +/** Create a simple sampler from tile of dimension [m0, n0] + */ +inline TensorTileSampler create_sampler(GpuCkwScopedKernelWriter &writer, int32_t m0, int32_t n0) +{ + TensorTileSampler sampler; + + auto &gid_0 = writer->declare_tile("gid_0", ckw::DataType::Int32); + auto &gid_1 = writer->declare_tile("gid_1", ckw::DataType::Int32); + auto &gid_2 = writer->declare_tile("gid_2", ckw::DataType::Int32); + + auto &const_0 = writer->declare_tile("0", 0); + writer->op_get_global_id(gid_0, 0); + writer->op_get_global_id(gid_1, 1); + writer->op_get_global_id(gid_2, 2); + + auto &x_coord = writer->declare_tile("x_coord", ckw::DataType::Int32); + auto &y_coord = writer->declare_tile("y_coord", ckw::DataType::Int32); + auto &m0_t = writer->declare_tile("m0", m0); + auto &n0_t = writer->declare_tile("n0", n0); + writer->op_binary_expression(x_coord, gid_0, BinaryOp::Mul, n0_t); + writer->op_binary_expression(y_coord, gid_1, BinaryOp::Mul, m0_t); + + sampler.x(x_coord); + sampler.y(y_coord); + sampler.z(const_0); // 3rd dimension collapsed with 2nd dimension + sampler.b(gid_2); + + sampler.width(n0); + sampler.height(m0); + + sampler.format(TensorSamplerFormat::C_WH_1); // 3rd dimension collapsed with 2nd dimension + sampler.address_mode_x(TensorSamplerAddressModeX::None); + sampler.address_mode_y(TensorSamplerAddressModeY::ClampToBorder); + sampler.address_mode_z(TensorSamplerAddressModeZ::Skip); // Dimensions higher than 3 not supported yet + + return sampler; +} +} // namespace + +GpuCkwCast::GpuCkwCast(ComponentId id, + const ArgumentPack &tensors, + const Attributes &attributes) + : IGpuCkwComponentDriver{ id, tensors }, + _src{}, + _dst{}, + _attributes{ attributes } +{ + _src = this->tensors().get_const_tensor(TensorType::ACL_SRC_0); + _dst = this->tensors().get_const_tensor(TensorType::ACL_DST_0); + ARM_COMPUTE_ERROR_ON_NULLPTR(_src, _dst); +} + +void GpuCkwCast::write_component_code(const ComponentGroup &comp_group, GpuCkwVariableTable &vtable, GpuCkwScopedKernelWriter writer) const +{ + const auto root_window = comp_group.get_root_component()->ckw_component_driver()->get_window(); + const unsigned int n0 = root_window.x().step(); + const unsigned int m0 = root_window.y().step(); + + GpuCkwComponentArgument *src = vtable.declare_variable(comp_group, writer, _src, TensorStorageType::ClBufferUint8Ptr, "src"); + GpuCkwComponentArgument *dst = vtable.declare_variable(comp_group, writer, _dst, TensorStorageType::ClBufferUint8Ptr, "dst"); + + // Load the source tile and prepare the sampler. + if(!src->has_tile()) + { + const auto sampler = create_sampler(writer, m0, n0); + writer->op_load_once(src, sampler); + } + else + { + const auto &sampler = src->tile_sampler(); + writer->op_load_once(src, sampler); + } + + const auto &src_tile = src->tile(); + const auto &sampler = src->tile_sampler(); + + // Prepare the output tile. + if(!dst->has_tile()) + { + // Get Target datatype and convert it to ckw::DataType. + ckw::DataType target_dt = dynamic_fusion::to_ckw(_attributes.data_type()); + + // Create dst_tile based on src_tile dimensions and with target DataType. + const TileInfo src_tile_info = src_tile.tile_info(); + const TileInfo dst_tile_info = TileInfo(target_dt, src_tile_info.height(), src_tile_info.width()); + + // Declare dst_tile + auto &tile = writer->declare_tile("dst_tile", dst_tile_info); + dst->init_virtual_tensor(tile, sampler); + } + + const auto &dst_tile = dst->tile(); + + // Check if this op is cast-down or cast-up + const size_t src_size = data_size_from_type(_src->data_type()); + const size_t dst_size = data_size_from_type(_dst->data_type()); + const bool cast_down = (src_size >= dst_size); + + if(cast_down && is_data_type_quantized(_src->data_type())) + { + const auto &constant_x80 = writer->declare_tile("0x80", 0x80); + writer->op_binary_expression(src_tile, src_tile, BinaryOp::BitwiseXOR, constant_x80); + } + + ckw::ConvertPolicy convert_policy = ckw::ConvertPolicy::None; + + if(cast_down && (is_data_type_float(_src->data_type()) || _attributes.convert_policy() == ConvertPolicy::SATURATE)) + { + convert_policy = ckw::ConvertPolicy::Saturate; + } + + writer->op_cast_expression(dst_tile, src_tile, convert_policy); +} + +Window GpuCkwCast::get_window() const +{ + ARM_COMPUTE_ERROR_ON_MSG(_dst->tensor_shape().total_size() == 0U, "Destination tensor is not initialized"); + + TensorShape output_shape = _dst->tensor_shape(); + // Collapse Dim 1 (W) and Dim 2 (H) together, leave Dim 0 (C) unchanged + // This is in line with the collapsing convention used by operators like Conv2d + output_shape.collapse(2U, 1U); + constexpr unsigned int vector_size_byte_opencl = 16; + const unsigned int num_elems_processed_per_iteration = adjust_vec_size(vector_size_byte_opencl / _dst->element_size(), _dst->dimension(0)); + Window win = calculate_max_window(output_shape, Steps(num_elems_processed_per_iteration)); + + return win; +} + +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute diff --git a/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwCast.h b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwCast.h new file mode 100644 index 0000000000000000000000000000000000000000..821cec1e19b97e8ae05bbf2d8def591e6884f1ba --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwCast.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_COMPONENTS_GPUCKWCAST +#define ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_COMPONENTS_GPUCKWCAST + +#include "src/core/common/Macros.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/IGpuCkwComponentDriver.h" +#include "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentCast.h" + +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ +class GpuCkwCast : public IGpuCkwComponentDriver +{ +public: + using Attributes = ClComponentCast::Attributes; + /** Constructor + * + * For supported configurations please refer to @ref ClComponentCast::validate() + * + * @param[in] id Component id + * @param[in] tensors Tensor arguments to the component + * @param[in] attributes Component attributes + */ + GpuCkwCast(ComponentId id, + const ArgumentPack &tensors, + const Attributes &attributes); + ARM_COMPUTE_DISALLOW_COPY_ALLOW_MOVE(GpuCkwCast); + /** Destructor */ + ~GpuCkwCast() override = default; + // Inherited methods overriden: + virtual void write_component_code(const ComponentGroup &comp_group, GpuCkwVariableTable &vtable, GpuCkwScopedKernelWriter writer) const override; + Window get_window() const override; + +private: + const ITensorInfo *_src; + const ITensorInfo *_dst; + Attributes _attributes; +}; +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute + +#endif /* ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_COMPONENTS_GPUCKWCAST */ diff --git a/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwDirectConv2d.cpp b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwDirectConv2d.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3c906646a67d8451ad0f13b116b3e655322d8573 --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwDirectConv2d.cpp @@ -0,0 +1,333 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwDirectConv2d.h" + +#include "arm_compute/core/Error.h" +#include "arm_compute/core/Validate.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/StringUtils.h" + +#include "ckw/TensorTileSampler.h" +#include "ckw/TileInfo.h" + +#include "src/core/helpers/WindowHelpers.h" +#include "src/dynamic_fusion/sketch/gpu/GpuKernelArgument.h" +#include "src/dynamic_fusion/sketch/gpu/GpuKernelComponentGroup.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwKernelWriter.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwScopedKernelWriter.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwVariableTable.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/components/utils/WriterHelper.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/components/utils/type_converter/Common.h" + +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ + +using TileContainer = std::vector>; + +GpuCkwDirectConv2d::GpuCkwDirectConv2d(ComponentId id, + const ArgumentPack &tensors, + const Attributes &attributes, + const Settings &settings) + : IGpuCkwComponentDriver{ id, tensors }, + _src{}, + _wei{}, + _bia{}, + _dst{}, + _attributes{ attributes }, + _settings{ settings } +{ + _src = this->tensors().get_const_tensor(TensorType::ACL_SRC_0); + _wei = this->tensors().get_const_tensor(TensorType::ACL_SRC_1); + _bia = this->tensors().get_const_tensor(TensorType::ACL_SRC_2); + _dst = this->tensors().get_const_tensor(TensorType::ACL_DST_0); + ARM_COMPUTE_ERROR_ON_NULLPTR(_src, _wei, _dst); // Bias can be null +} + +void GpuCkwDirectConv2d::write_component_code(const ComponentGroup &comp_group, GpuCkwVariableTable &vtable, GpuCkwScopedKernelWriter writer) const +{ + const auto desc = _settings.direct_conv_descriptor(); + ARM_COMPUTE_ERROR_ON_MSG(desc.export_input_to_cl_image || desc.export_output_to_cl_image, + "Only the weights tensor can be exported to cl_image"); + + const unsigned int channel_idx = get_data_layout_dimension_index(_src->data_layout(), DataLayoutDimension::CHANNEL); + const unsigned int width_idx = get_data_layout_dimension_index(_wei->data_layout(), DataLayoutDimension::WIDTH); + const unsigned int height_idx = get_data_layout_dimension_index(_wei->data_layout(), DataLayoutDimension::HEIGHT); + + const auto root_window = comp_group.get_root_component()->ckw_component_driver()->get_window(); + + // Tunable parameters + const int32_t m0 = root_window.y().step(); + const int32_t n0 = root_window.x().step(); + const int32_t k0 = adjust_vec_size(_settings.direct_conv_descriptor().k0, _src->dimension(channel_idx)); + const int32_t partial_n0 = _dst->dimension(0) % n0; + + const int32_t K = _src->dimension(channel_idx); + + // Exporting the weights tensor to an OpenCL image object is currently only supported when: + // a) k0 is equal to 4 + // The current implementation expects to read a vector of 4 float values into the OpenCL image object. + // b) K is a multiple of 4 + // This is a limitation in the current interface due to the variable table being responsible for maintaining + // information about the TensorStorageType rather than the TensorTileSampler. As a result, TensorStorageType cannot + // be reassigned, and we cannot use a texture object for the weights tensor in cases where we expect to have an + // extra loop to compute the left-over elements. + const bool use_cl_image_for_weights = desc.export_weights_to_cl_image && (k0 == 4) && (K % 4 == 0); + + GpuCkwComponentArgument *src = vtable.declare_variable(comp_group, writer, _src, TensorStorageType::ClBufferUint8Ptr, "src"); + GpuCkwComponentArgument *wei = vtable.declare_variable( + comp_group, writer, _wei, use_cl_image_for_weights ? TensorStorageType::ClImage2dReadOnly : TensorStorageType::ClBufferUint8Ptr, "wei"); + GpuCkwComponentArgument *dst = vtable.declare_variable(comp_group, writer, _dst, TensorStorageType::ClBufferUint8Ptr, "dst"); + GpuCkwComponentArgument *bia = nullptr; + + const bool using_bias = _bia != nullptr; + + if(using_bias) + { + bia = vtable.declare_variable(comp_group, writer, _bia, TensorStorageType::ClBufferUint8Ptr, "bia"); + } + + // Constants + const auto kernel_height = static_cast(_wei->dimension(height_idx)); + const auto kernel_width = static_cast(_wei->dimension(width_idx)); + const auto src_channels = static_cast(_src->dimension(channel_idx)); + auto &tile_kernel_w = writer->declare_tile("kernel_w", kernel_width); + auto &tile_kernel_size = writer->declare_tile("kernel_size", kernel_width * kernel_height); + auto &tile_src_c = writer->declare_tile("src_c", static_cast(_src->dimension(channel_idx))); + auto &tile_dst_w = writer->declare_tile("dst_w", static_cast(_dst->dimension(width_idx))); + auto &tile_stride_x = writer->declare_tile("stride_x", static_cast(_attributes.stride().x())); + auto &tile_stride_y = writer->declare_tile("stride_y", static_cast(_attributes.stride().y())); + auto &tile_pad_x = writer->declare_tile("pad_x", static_cast(_attributes.pad().left)); + auto &tile_pad_y = writer->declare_tile("pad_y", static_cast(_attributes.pad().top)); + auto &tile_k0 = writer->declare_tile("k0", k0); + auto &tile_0 = writer->declare_tile("0", 0); + auto &tile_1 = writer->declare_tile("1", 1); + + auto &tile_gid_0 = writer->declare_tile("gid_0", ckw::DataType::Int32); + auto &tile_gid_1 = writer->declare_tile("gid_1", ckw::DataType::Int32); + auto &tile_gid_2 = writer->declare_tile("gid_2", ckw::DataType::Int32); + + writer->op_get_global_id(tile_gid_0, 0); + writer->op_get_global_id(tile_gid_1, 1); + writer->op_get_global_id(tile_gid_2, 2); + + auto &tile_cout = writer->declare_tile("cout", ckw::DataType::Int32); // OFM + auto &tile_mout = writer->declare_tile("mout", ckw::DataType::Int32); // WIDTH x HEIGHT + auto &tile_bout = writer->declare_tile("bout", ckw::DataType::Int32); // BATCH SIZE IDX + + // Get the boundary aware coordinates at each global dimension index + get_coord(writer, tile_cout, tile_gid_0, n0, partial_n0, tile_cout.name() + "_dim0_", tile_0); + get_coord(writer, tile_mout, tile_gid_1, m0, 0, tile_mout.name() + "_dim1_", tile_0); + get_coord(writer, tile_bout, tile_gid_2, 1, 0, tile_bout.name() + "_dim2_", tile_0); + + TensorTileSampler src_sampler; + src_sampler.width(k0); + src_sampler.height(m0); + src_sampler.format(TensorSamplerFormat::C_WH_1); + // We cannot have out-of-bounds reads in the X dimension (mapped to the IFMs) as we have an extra loop to + // compute left-over elements + src_sampler.address_mode_x(TensorSamplerAddressModeX::None); + // We cannot have out-of-bounds reads when the kernel height is equal to 1. Otherwise, we need to ensure the + // indirection buffer mi does not contain negative values representing out-of-bounds reads. + src_sampler.address_mode_y(kernel_height == 1 ? TensorSamplerAddressModeY::None : TensorSamplerAddressModeY::SkipMinEdgeOnly); + src_sampler.address_mode_z(TensorSamplerAddressModeZ::None); + + TensorTileSampler wei_sampler; + wei_sampler.width(k0); + wei_sampler.height(n0); + wei_sampler.format(TensorSamplerFormat::C_WH_1); + // We cannot have out-of-bounds accesses for the weights + wei_sampler.address_mode_x(TensorSamplerAddressModeX::None); + wei_sampler.address_mode_y(TensorSamplerAddressModeY::None); + wei_sampler.address_mode_z(TensorSamplerAddressModeZ::None); + + TensorTileSampler dst_sampler; + dst_sampler.width(n0); + dst_sampler.height(m0); + dst_sampler.format(TensorSamplerFormat::C_WH_1); + dst_sampler.address_mode_x(TensorSamplerAddressModeX::OverlappingMin); + dst_sampler.address_mode_y(TensorSamplerAddressModeY::ClampToMaxEdgeOnly); + dst_sampler.address_mode_z(TensorSamplerAddressModeZ::None); + dst_sampler.x(tile_cout); + dst_sampler.y(tile_mout); + dst_sampler.z(tile_0); + dst_sampler.b(tile_bout); + + if(!dst->has_tile()) + { + auto &tile = writer->declare_tile("dst", TileInfo(to_ckw(_dst->data_type()), m0, n0)); + dst->init_virtual_tensor(tile, dst_sampler); + } + auto &tile_dst = dst->tile(); + + writer->op_assign(tile_dst, tile_0); + + // We create a 2d container of size (M0, 1) to store the indices for iteration + TileContainer it; + for(int m = 0; m < m0; ++m) + { + std::vector idx { std::to_string(m) }; + it.push_back({ idx }); + } + const auto &tile_it = writer->declare_tile("it", it, ckw::DataType::Int32); + + auto &tile_xi = writer->declare_tile("xi", TileInfo(ckw::DataType::Int32, m0, 1)); + auto &tile_yi = writer->declare_tile("yi", TileInfo(ckw::DataType::Int32, m0, 1)); + + // Convert the linear index to coordinate + // xi = ((mout + i) % dst_w) * stride_x - pad_x + // yi = ((mout + i) / dst_w) * stride_y - pad_y + writer->op_binary_expression(tile_xi, tile_mout, BinaryOp::Add, tile_it); + writer->op_binary_expression(tile_yi, tile_mout, BinaryOp::Add, tile_it); + writer->op_binary_expression(tile_xi, tile_xi, BinaryOp::Mod, tile_dst_w); + writer->op_binary_expression(tile_yi, tile_yi, BinaryOp::Div, tile_dst_w); + writer->op_binary_expression(tile_xi, tile_xi, BinaryOp::Mul, tile_stride_x); + writer->op_binary_expression(tile_yi, tile_yi, BinaryOp::Mul, tile_stride_y); + writer->op_binary_expression(tile_xi, tile_xi, BinaryOp::Sub, tile_pad_x); + writer->op_binary_expression(tile_yi, tile_yi, BinaryOp::Sub, tile_pad_y); + + auto &tile_y_b = writer->declare_tile("y_b", ckw::DataType::Int32); + writer->op_binary_expression(tile_y_b, tile_cout, BinaryOp::Mul, tile_kernel_size); + + auto &tile_i = writer->declare_tile("i", ckw::DataType::Int32); + writer->op_assign(tile_i, tile_0); + + // clang-format off + writer->op_for_loop(tile_i, BinaryOp::Less, tile_kernel_size, tile_i, AssignmentOp::Increment, tile_1, [&]() + { + auto &tile_x_k = writer->declare_tile("x_k", ckw::DataType::Int32); + auto &tile_y_k = writer->declare_tile("y_k", ckw::DataType::Int32); + + writer->op_binary_expression(tile_x_k, tile_i, BinaryOp::Mod, tile_kernel_w); + writer->op_binary_expression(tile_y_k, tile_i, BinaryOp::Div, tile_kernel_w); + + auto &tile_ck = writer->declare_tile("ck", ckw::DataType::Int32); + writer->op_assign(tile_ck, tile_0); + + auto &tile_mi = writer->declare_tile("mi", TileInfo(ckw::DataType::Int32, m0, 1)); + // Construct an indirection buffer containing the precalculated addresses of elements in the source tensor + // x_s = xi + x_k + // y_s = yi + y_k + // mi = x_s + y_s * width; + // mi = select(-1, mi, x_s >= 0); + // mi = select(-1, mi, x_s < width); + // mi = select(-1, mi, y_s >= 0); + // mi = select(-1, mi, y_s < height); + writer->util_get_indirect_buffer(tile_mi, src->tensor(), src_sampler, tile_xi, tile_yi, tile_x_k, tile_y_k); + + src_sampler.x(tile_ck); + src_sampler.y(tile_mi); + src_sampler.z(tile_0); + src_sampler.b(tile_bout); + + wei_sampler.x(tile_ck); + wei_sampler.y(tile_y_b); + wei_sampler.z(tile_0); + wei_sampler.b(tile_0); + + auto &tile_src_c_minus_k0 = writer->declare_tile("src_c_minus_k0", src_channels - k0); + + writer->op_for_loop(tile_ck, BinaryOp::LessEqual, tile_src_c_minus_k0, tile_ck, AssignmentOp::Increment, tile_k0, [&]() + { + auto &tile_lhs = writer->declare_tile("lhs", TileInfo(to_ckw(_src->data_type()), m0, k0)); + auto &tile_rhs = writer->declare_tile("rhs", TileInfo(to_ckw(_wei->data_type()), n0, k0)); + writer->op_assign(tile_lhs, tile_0); + writer->op_assign(tile_rhs, tile_0); + + writer->op_load_indirect(tile_lhs, src->tensor(), src_sampler); + writer->op_load(tile_rhs, wei->tensor(), wei_sampler, tile_kernel_size); + + writer->op_binary_expression(tile_dst, tile_lhs, BinaryOp::MatMul_Nt_T, tile_rhs); + }); + + // Left-over accumulations for when K is not a multiple of k0 + if(!(K % k0 == 0)) + { + writer->op_for_loop(tile_ck, BinaryOp::Less, tile_src_c, tile_ck, AssignmentOp::Increment, tile_1, [&]() + { + auto &tile_lhs = writer->declare_tile("lhs_leftover", TileInfo(to_ckw(_src->data_type()), m0, 1)); + auto &tile_rhs = writer->declare_tile("rhs_leftover", TileInfo(to_ckw(_wei->data_type()), n0, 1)); + writer->op_assign(tile_lhs, tile_0); + writer->op_assign(tile_rhs, tile_0); + + writer->op_load_indirect(tile_lhs, src->tensor(), src_sampler); + writer->op_load(tile_rhs, wei->tensor(), wei_sampler, tile_kernel_size); + + writer->op_binary_expression(tile_dst, tile_lhs, BinaryOp::MatMul_Nt_T, tile_rhs); + }); + } + + writer->op_binary_expression(tile_y_b, tile_y_b, BinaryOp::Add, tile_1); + }); + // clang-format on + + // Bias addition + // NOTE: This operation will be removed from this kernel as the interface is standardized. The intended way of + // performing bias addition is to fuse this convolution kernel with a following elementwise addition kernel. + if(using_bias) + { + if(!bia->has_tile()) + { + // Reuse the destination sampler for the bias + writer->op_load_once(bia, dst_sampler); + } + auto &tile_bia = bia->tile(); + + writer->op_binary_expression(tile_dst, tile_dst, BinaryOp::Add, tile_bia); + } +} + +Window GpuCkwDirectConv2d::get_window() const +{ + ARM_COMPUTE_ERROR_ON_MSG(_dst->tensor_shape().total_size() == 0U, "Destination tensor is not initialized"); + + const auto dst_shape = _dst->tensor_shape(); + const auto desc = _settings.direct_conv_descriptor(); + + const unsigned int n0 = adjust_vec_size(desc.n0, dst_shape[0]); + const unsigned int m0 = adjust_vec_size(desc.m0, dst_shape[1] * dst_shape[2]); + + Window win = calculate_max_window(dst_shape, Steps(n0, m0)); + + const size_t dim_y_collapsed = ceil_to_multiple(dst_shape[1] * dst_shape[2], m0); + win.set(Window::DimY, Window::Dimension(0, dim_y_collapsed, m0)); + win.set(Window::DimZ, Window::Dimension(0, dst_shape.total_size_upper(3), 1)); + + return win; +} + +std::string GpuCkwDirectConv2d::get_name(const ComponentGroup &comp_group) const +{ + ARM_COMPUTE_UNUSED(comp_group); + + return "direct_conv2d"; +} + +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute diff --git a/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwDirectConv2d.h b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwDirectConv2d.h new file mode 100644 index 0000000000000000000000000000000000000000..ac32d2df32a6c7cd3a3006021b908b9e3a5b0b17 --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwDirectConv2d.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_COMPONENTS_GPUCKWDIRECTCONV2D +#define ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_COMPONENTS_GPUCKWDIRECTCONV2D + +#include "arm_compute/dynamic_fusion/sketch/attributes/Conv2dAttributes.h" + +#include "src/core/common/Macros.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/IGpuCkwComponentDriver.h" +#include "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentDirectConv2d.h" + +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ +class GpuCkwDirectConv2d final : public IGpuCkwComponentDriver +{ +public: + using Attributes = ClComponentDirectConv2d::Attributes; + using Settings = ClComponentDirectConv2d::Settings; + +public: + /** Constructor + * + * For supported configurations please refer to @ref ClComponentDirectConv2d::validate() + * + * @param[in] id Component id + * @param[in] tensors Tensor arguments to the component + * @param[in] attributes Component attributes. Attributes are a set of parameters that define what a component does + * @param[in] settings Component settings. Settings are a set of parameters that influence the implementation of a component + */ + GpuCkwDirectConv2d(ComponentId id, + const ArgumentPack &tensors, + const Attributes &attributes, + const Settings &settings); + + ARM_COMPUTE_DISALLOW_COPY_ALLOW_MOVE(GpuCkwDirectConv2d); + + /** Destructor */ + ~GpuCkwDirectConv2d() override = default; + + // Inherited methods overriden + virtual void write_component_code(const ComponentGroup &comp_group, + GpuCkwVariableTable &vtable, + GpuCkwScopedKernelWriter writer) const override; + Window get_window() const override; + std::string get_name(const ComponentGroup &comp_group) const override; + +private: + const ITensorInfo *_src; + const ITensorInfo *_wei; + const ITensorInfo *_bia; + const ITensorInfo *_dst; + + const Attributes _attributes; + const Settings _settings; +}; +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute + +#endif /* ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_COMPONENTS_GPUCKWDIRECTCONV2D */ diff --git a/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwElementwiseBinary.cpp b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwElementwiseBinary.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c8bf999261c26577a0615585e28d32e83b677491 --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwElementwiseBinary.cpp @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "GpuCkwElementwiseBinary.h" + +#include "arm_compute/core/Error.h" +#include "arm_compute/core/Validate.h" +#include "arm_compute/core/utils/StringUtils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "ckw/TensorTileSampler.h" +#include "ckw/types/TensorSamplerTypes.h" +#include "src/core/helpers/WindowHelpers.h" +#include "src/dynamic_fusion/sketch/gpu/GpuKernelArgument.h" +#include "src/dynamic_fusion/sketch/gpu/GpuKernelComponentGroup.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwKernelWriter.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwScopedKernelWriter.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwVariableTable.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/components/utils/WriterHelper.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/components/utils/type_converter/Common.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/components/utils/type_converter/ElementwiseBinary.h" +#include "src/dynamic_fusion/sketch/gpu/components/utils/type_printer/ElementwiseBinary.h" +#include "support/StringSupport.h" +#include +#include + +using namespace ckw; +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ +GpuCkwElementwiseBinary::GpuCkwElementwiseBinary(ComponentId id, + const ArgumentPack &tensors, + const Attributes &attributes) + : IGpuCkwComponentDriver{ id, tensors }, + _lhs{}, + _rhs{}, + _dst{}, + _attributes{ attributes } +{ + _lhs = this->tensors().get_const_tensor(TensorType::ACL_SRC_0); + _rhs = this->tensors().get_const_tensor(TensorType::ACL_SRC_1); + _dst = this->tensors().get_const_tensor(TensorType::ACL_DST_0); + ARM_COMPUTE_ERROR_ON_NULLPTR(_lhs, _rhs, _dst); +} + +void GpuCkwElementwiseBinary::write_component_code(const ComponentGroup &comp_group, GpuCkwVariableTable &vtable, GpuCkwScopedKernelWriter writer) const +{ + const auto root_window = comp_group.get_root_component()->ckw_component_driver()->get_window(); + const auto n0 = static_cast(root_window.x().step()); + const auto m0 = static_cast(root_window.y().step()); + + GpuCkwComponentArgument *lhs = vtable.declare_variable(comp_group, writer, _lhs, TensorStorageType::ClBufferUint8Ptr, "lhs"); + GpuCkwComponentArgument *rhs = vtable.declare_variable(comp_group, writer, _rhs, TensorStorageType::ClBufferUint8Ptr, "rhs"); + GpuCkwComponentArgument *dst = vtable.declare_variable(comp_group, writer, _dst, TensorStorageType::ClBufferUint8Ptr, "dst"); + + auto &gid_0 = writer->declare_tile("gid_0", ckw::DataType::Int32); + auto &gid_1 = writer->declare_tile("gid_1", ckw::DataType::Int32); + auto &gid_2 = writer->declare_tile("gid_2", ckw::DataType::Int32); + + writer->op_get_global_id(gid_0, 0); + writer->op_get_global_id(gid_1, 1); + writer->op_get_global_id(gid_2, 2); + + auto &const_0 = writer->declare_tile("0", 0); + + // Load the LHS and RHS tiles + if(!lhs->has_tile()) + { + auto sampler = create_boundary_aware_2d_sampler(writer, gid_0, gid_1, _lhs->dimension(0), _lhs->dimension(1), n0, m0, "lhs_", const_0); + sampler.format(TensorSamplerFormat::C_WH_1); // 3rd dimension collapsed with 2nd dimension + sampler.z(const_0); + sampler.b(gid_2); + writer->op_load_once(lhs, sampler); + } + if(!rhs->has_tile()) + { + auto sampler = create_boundary_aware_2d_sampler(writer, gid_0, gid_1, _rhs->dimension(0), _rhs->dimension(1), n0, m0, "rhs_", const_0); + sampler.format(TensorSamplerFormat::C_WH_1); // 3rd dimension collapsed with 2nd dimension + sampler.z(const_0); + sampler.b(gid_2); + writer->op_load_once(rhs, sampler); + } + + auto dst_sampler = create_boundary_aware_2d_sampler(writer, gid_0, gid_1, _dst->dimension(0), _dst->dimension(1), n0, m0, "dst_", const_0); + dst_sampler.format(TensorSamplerFormat::C_WH_1); // 3rd dimension collapsed with 2nd dimension + dst_sampler.z(const_0); + dst_sampler.b(gid_2); + + // Prepare the output tile. + if(!dst->has_tile()) + { + auto &tile = writer->declare_tile("dst_tile", ckw::TileInfo(to_ckw(_dst->data_type()), dst_sampler.height(), dst_sampler.width())); + dst->init_virtual_tensor(tile, dst_sampler); + } + + auto &lhs_tile = lhs->tile(); + auto &rhs_tile = rhs->tile(); + auto &dst_tile = dst->tile(); + + // Perform the operation. + writer->op_binary_expression(dst_tile, lhs_tile, to_ckw(_attributes), rhs_tile); +} + +Window GpuCkwElementwiseBinary::get_window() const +{ + ARM_COMPUTE_ERROR_ON_MSG(_dst->tensor_shape().total_size() == 0U, "Destination tensor is not initialized"); + + TensorShape output_shape = _dst->tensor_shape(); + // Collapse Dim 1 (W) and Dim 2 (H) together, leave Dim 0 (C) unchanged + // This is in line with the collapsing convention used by operators like Conv2d + output_shape.collapse(2U, 1U); + constexpr unsigned int vector_size_byte_opencl = 16; + const unsigned int num_elems_processed_per_iteration = adjust_vec_size(vector_size_byte_opencl / _dst->element_size(), _dst->dimension(0)); + Window win = calculate_max_window(output_shape, Steps(num_elems_processed_per_iteration)); + + return win; +} + +std::string GpuCkwElementwiseBinary::get_name(const ComponentGroup &comp_group) const +{ + ARM_COMPUTE_UNUSED(comp_group); + const std::vector build_params = + { + "elementwise_binary", + "op", to_string(_attributes.operation()), + "dt", lower_string(string_from_data_type(_dst->data_type())), + }; + return join(build_params, "_"); +} + +std::string GpuCkwElementwiseBinary::get_tuner_id(const ComponentGroup &comp_group) const +{ + ARM_COMPUTE_UNUSED(comp_group); + /// NOTE: Hardcoded for now, the parameters should ideally be exported by ckw (a selection of constant tiles) + std::vector build_params = + { + "elementwise_binary", + "op", to_string(_attributes.operation()), + "dt", lower_string(string_from_data_type(_dst->data_type())), + "dst_dim0", support::cpp11::to_string(_dst->dimension(0)), + "dst_dim1", support::cpp11::to_string(_dst->dimension(1)), + }; + return join(build_params, "_"); +} +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute diff --git a/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwElementwiseBinary.h b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwElementwiseBinary.h new file mode 100644 index 0000000000000000000000000000000000000000..e9c41530f8703c4b9fd7e2c364ac519e9a695935 --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwElementwiseBinary.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_COMPONENTS_GPUCKWELEMENTWISEBINARY +#define ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_COMPONENTS_GPUCKWELEMENTWISEBINARY + +#include "src/core/common/Macros.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/IGpuCkwComponentDriver.h" +#include "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentElementwiseBinary.h" + +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ +class GpuCkwElementwiseBinary : public IGpuCkwComponentDriver +{ +public: + using Attributes = ClComponentElementwiseBinary::Attributes; + /** Constructor + * + * For supported configurations please refer to @ref ClComponentElementwiseBinary::validate() + * + * @param[in] id Component id + * @param[in] tensors Tensor arguments to the component + * @param[in] attributes Component attributes + */ + GpuCkwElementwiseBinary(ComponentId id, + const ArgumentPack &tensors, + const Attributes &attributes); + ARM_COMPUTE_DISALLOW_COPY_ALLOW_MOVE(GpuCkwElementwiseBinary); + /** Destructor */ + ~GpuCkwElementwiseBinary() override = default; + // Inherited methods overriden: + virtual void write_component_code(const ComponentGroup &comp_group, GpuCkwVariableTable &vtable, GpuCkwScopedKernelWriter writer) const override; + Window get_window() const override; + std::string get_name(const ComponentGroup &comp_group) const override; + std::string get_tuner_id(const ComponentGroup &comp_group) const override; + +private: + const ITensorInfo *_lhs; + const ITensorInfo *_rhs; + const ITensorInfo *_dst; + Attributes _attributes; +}; +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute + +#endif /* ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_COMPONENTS_GPUCKWELEMENTWISEBINARY */ diff --git a/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwStore.cpp b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwStore.cpp new file mode 100644 index 0000000000000000000000000000000000000000..891739153783e7145e6c279edf4b663b73e743a8 --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwStore.cpp @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "GpuCkwStore.h" + +#include "arm_compute/core/Error.h" +#include "src/dynamic_fusion/sketch/gpu/GpuKernelArgument.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwKernelWriter.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwScopedKernelWriter.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwVariableTable.h" +#include + +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ +GpuCkwStore::GpuCkwStore(ComponentId id, const ArgumentPack &tensors) + : IGpuCkwComponentDriver{ id, tensors }, _src{}, _dst{} +{ + _src = this->tensors().get_const_tensor(TensorType::ACL_SRC_0); + _dst = this->tensors().get_const_tensor(TensorType::ACL_DST_0); +} +void GpuCkwStore::write_component_code(const ComponentGroup &comp_group, GpuCkwVariableTable &vtable, GpuCkwScopedKernelWriter writer) const +{ + auto src = vtable.declare_variable(comp_group, writer, _src, TensorStorageType::ClBufferUint8Ptr, "src"); + auto dst = vtable.declare_variable(comp_group, writer, _dst, TensorStorageType::ClBufferUint8Ptr, "dst"); + + auto &src_tile = src->tile(); + const auto &sampler = src->tile_sampler(); + auto &dst_tensor = dst->tensor(); + + writer->op_store(dst_tensor, src_tile, sampler); +} + +std::string GpuCkwStore::get_name(const ComponentGroup &comp_group) const +{ + ARM_COMPUTE_UNUSED(comp_group); + return "store"; +} +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute diff --git a/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwStore.h b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwStore.h new file mode 100644 index 0000000000000000000000000000000000000000..8e35651caf8ff21136eb7df9e9bad6a5c89191fd --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwStore.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_COMPONENTS_GPUCKWSTORE +#define ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_COMPONENTS_GPUCKWSTORE + +#include "src/core/common/Macros.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/IGpuCkwComponentDriver.h" + +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ +/** An interface used by @ref ClTemplateWriter to write source code for a kernel component + */ +class GpuCkwStore : public IGpuCkwComponentDriver +{ +public: + /** Constructor + * + * @param[in] id Component id + * @param[in] tensors Tensor arguments to the component + */ + GpuCkwStore(ComponentId id, const ArgumentPack &tensors); + ARM_COMPUTE_DISALLOW_COPY_ALLOW_MOVE(GpuCkwStore); + /** Destructor */ + ~GpuCkwStore() override = default; + // Inherited methods overriden: + virtual void write_component_code(const ComponentGroup &comp_group, GpuCkwVariableTable &vtable, GpuCkwScopedKernelWriter writer) const override; + std::string get_name(const ComponentGroup &comp_group) const override; + +private: + const ITensorInfo *_src; + const ITensorInfo *_dst; +}; +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute + +#endif /* ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_COMPONENTS_GPUCKWSTORE */ diff --git a/src/dynamic_fusion/sketch/gpu/ckw_driver/components/utils/WriterHelper.h b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/utils/WriterHelper.h new file mode 100644 index 0000000000000000000000000000000000000000..f4a056c5a084d2e1102eb233b6a7ef92a6c836e5 --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/utils/WriterHelper.h @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_COMPONENTS_UTILS_WRITERHELPER +#define ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_COMPONENTS_UTILS_WRITERHELPER + +#include "arm_compute/core/utils/misc/Utility.h" +#include "ckw/TensorTileSampler.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwComponentArgument.h" +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/GpuCkwScopedKernelWriter.h" + +#include +#include + +using namespace ckw; +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ +using SamplerCreator = std::function; + +/** Load src and dst tiles of dimension [m0, n0] only when not loaded and prepare the sampler + */ +inline void load_src_dst_tiles_and_prepare_sampler(GpuCkwScopedKernelWriter &writer, GpuCkwComponentArgument *src, GpuCkwComponentArgument *dst, int32_t m0, int32_t n0, SamplerCreator create_sampler) +{ + if(!src->has_tile()) + { + const auto sampler = create_sampler(writer, m0, n0); + writer->op_load_once(src, sampler); + } + else + { + const auto &sampler = src->tile_sampler(); + writer->op_load_once(src, sampler); + } + + auto &src_tile = src->tile(); + const auto &sampler = src->tile_sampler(); + + // Prepare the output tile. + if(!dst->has_tile()) + { + auto &tile = writer->declare_tile("dst_tile", src_tile.tile_info()); + dst->init_virtual_tensor(tile, sampler); + } +} + +/** Get boundary aware coordinate along one axis. Load and store of size step_v at the coordinate will not be out of bound + * + * @param[in,out] writer Writer + * @param[out] coord Resultant coordinate + * @param[in] gid Global work item id + * @param[in] step_v Step size / vector size + * @param[in] leftover_step_v Leftover step size at the boundary + * @param[in] prefix Prefix to all the tiles declared within this function + * @param[in] const_0 Constant tile of value 0 + */ +inline void get_coord(GpuCkwScopedKernelWriter writer, TileOperand &coord, TileOperand &gid, int32_t step_v, int32_t leftover_step_v, const std::string &prefix, TileOperand &const_0) +{ + auto &step = writer->declare_tile(prefix + "step", step_v); + auto &leftover_step = writer->declare_tile(prefix + "leftover_step", leftover_step_v); + + // step - leftover_step + auto &step_minus_leftover = writer->declare_tile(prefix + "step_minus_leftover", ckw::DataType::Int32); + writer->op_binary_expression(step_minus_leftover, step, ckw::BinaryOp::Sub, leftover_step); + + // (step - leftover_step) % step + auto &coord_correction = writer->declare_tile(prefix + "coord_correction", ckw::DataType::Int32); + writer->op_binary_expression(coord_correction, step_minus_leftover, ckw::BinaryOp::Mod, step); + + // (gid * step) + auto &raw_coord = writer->declare_tile(prefix + "raw_coord", ckw::DataType::Int32); + writer->op_binary_expression(raw_coord, gid, ckw::BinaryOp::Mul, step); + + // (gid * step) - (step - leftover_step) % step + auto &corrected_coord = writer->declare_tile(prefix + "corrected_coord", ckw::DataType::Int32); + writer->op_binary_expression(corrected_coord, raw_coord, ckw::BinaryOp::Sub, coord_correction); + + // max((gid * step) - (step - leftover_step) % step, 0) + writer->op_binary_elementwise_function(coord, ckw::BinaryFunction::Max, corrected_coord, const_0); +} + +/** Declare coordinate tiles "{prefix}_dim0_coord" and "{prefix}_dim1_coord", and create a boundary-aware sampler from tile of size [n0, m0], against the overall dimensions [dim0, dim1] + * The load and store of tile [n0, m0] will never be out of bound of [dim0, dim1] + */ + +/** Declare coordinate tiles "{prefix}_dim0_coord" and "{prefix}_dim1_coord", and create a boundary-aware sampler from tile of size [n0, m0], against the overall dimensions [dim0, dim1] + * The load and store of tile [n0, m0] will never be out of bound of [dim0, dim1] + * + * @param[in,out] writer Writer + * @param[in] gid_0 Global work item id 0 + * @param[in] gid_1 Global work item id 1 + * @param[in] dim0_v Dimension 0 + * @param[in] dim1_v Dimension 1 + * @param[in] n0_v Tile size dimension 0 + * @param[in] m0_v Tile size dimension 1 + * @param[in] prefix Prefix to all the tiles declared within this function + * @param[in] const_0 Constant tile of value 0 + * + * @return TensorTileSampler + */ +inline TensorTileSampler create_boundary_aware_2d_sampler(GpuCkwScopedKernelWriter writer, TileOperand &gid_0, TileOperand &gid_1, int32_t dim0_v, int32_t dim1_v, int32_t n0_v, int32_t m0_v, + const std::string prefix, TileOperand &const_0) +{ + // Clamp tile size [n0, m0] against dimension [dim0, dim1] + // This is needed to: + // * Guard against tile sizes are bigger than the tensor dimensions + // * Handle broadcasting tiles (e.g. src tensor is of size 1 in one of the dimensions) + n0_v = utility::clamp(n0_v, 1, dim0_v); + m0_v = utility::clamp(m0_v, 1, dim1_v); + const int32_t partial_n0_v = dim0_v % n0_v; + const int32_t partial_m0_v = dim1_v % m0_v; + + // Declare #prefix_dim0_coord and #prefix_dim1_coord + auto &dim0_coord = writer->declare_tile(prefix + "dim0_coord", ckw::DataType::Int32); + get_coord(writer, dim0_coord, gid_0, n0_v, partial_n0_v, prefix + "dim0_", const_0); + auto &dim1_coord = writer->declare_tile(prefix + "dim1_coord", ckw::DataType::Int32); + get_coord(writer, dim1_coord, gid_1, m0_v, partial_m0_v, prefix + "dim1_", const_0); + + // Set sampler + // Only set fields related to boundary aware loading/storing. Other info (e.g. format) is not responsibility of this function + TensorTileSampler sampler; + + sampler.x(dim0_coord); + sampler.y(dim1_coord); + + sampler.width(n0_v); + sampler.height(m0_v); + + sampler.address_mode_x(TensorSamplerAddressModeX::None); + sampler.address_mode_y(TensorSamplerAddressModeY::None); + sampler.address_mode_z(TensorSamplerAddressModeZ::None); + + return sampler; +} +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute +#endif /* ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_COMPONENTS_UTILS_WRITERHELPER */ diff --git a/src/dynamic_fusion/sketch/gpu/ckw_driver/components/utils/type_converter/Common.h b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/utils/type_converter/Common.h new file mode 100644 index 0000000000000000000000000000000000000000..34b1283add66beb1a59ae4e4abc9f6c3426cb25a --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/utils/type_converter/Common.h @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_COMPONENTS_UTILS_TYPE_CONVERTER_COMMON +#define ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_COMPONENTS_UTILS_TYPE_CONVERTER_COMMON + +#include "arm_compute/core/ITensorInfo.h" +#include "arm_compute/core/TensorShape.h" +#include "arm_compute/core/Types.h" +#include "ckw/TensorInfo.h" +#include "src/dynamic_fusion/sketch/gpu/GpuKernelArgument.h" + +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ +inline ckw::DataType to_ckw(DataType dt) +{ + switch(dt) + { + case DataType::F32: + return ckw::DataType::Fp32; + case DataType::F16: + return ckw::DataType::Fp16; + case DataType::S32: + return ckw::DataType::Int32; + case DataType::S16: + return ckw::DataType::Int16; + case DataType::S8: + case DataType::QASYMM8_SIGNED: + return ckw::DataType::Int8; + case DataType::U32: + return ckw::DataType::Uint32; + case DataType::U16: + return ckw::DataType::Uint16; + case DataType::U8: + case DataType::QASYMM8: + return ckw::DataType::Uint8; + default: + return ckw::DataType::Unknown; + } +} + +inline ckw::TensorShape to_ckw(const TensorShape &shape) +{ + ARM_COMPUTE_ERROR_ON(shape.num_max_dimensions < std::tuple_size {}); + ARM_COMPUTE_ERROR_ON(std::tuple_size {} != 5); + /// NOTE: Overflow danger. Use size_t? + return ckw::TensorShape + { + static_cast(shape[0]), + static_cast(shape[1]), + static_cast(shape[2]), + static_cast(shape[3]), + static_cast(shape[4]) + }; +} +inline ckw::TensorDataLayout to_ckw(DataLayout dl) +{ + switch(dl) + { + case DataLayout::NHWC: + return ckw::TensorDataLayout::Nhwc; + case DataLayout::NDHWC: + return ckw::TensorDataLayout::Ndhwc; + default: + return ckw::TensorDataLayout::Unknown; + } +} +inline ckw::TensorInfo to_ckw(const ITensorInfo &tensor_info) +{ + return ckw::TensorInfo + { + to_ckw(tensor_info.data_type()), + to_ckw(tensor_info.tensor_shape()), + to_ckw(tensor_info.data_layout()), + tensor_info.id() + }; +} + +inline TensorComponentType from_ckw(const ckw::TensorComponentType &component) +{ + switch(component) + { + case ckw::TensorComponentType::OffsetFirstElement: + return TensorComponentType::OffsetFirstElement; + case ckw::TensorComponentType::Stride0: + return TensorComponentType::Stride0; + case ckw::TensorComponentType::Stride1: + return TensorComponentType::Stride1; + case ckw::TensorComponentType::Stride2: + return TensorComponentType::Stride2; + case ckw::TensorComponentType::Stride3: + return TensorComponentType::Stride3; + case ckw::TensorComponentType::Stride4: + return TensorComponentType::Stride4; + case ckw::TensorComponentType::Dim0: + return TensorComponentType::Dim0; + case ckw::TensorComponentType::Dim1: + return TensorComponentType::Dim1; + case ckw::TensorComponentType::Dim2: + return TensorComponentType::Dim2; + case ckw::TensorComponentType::Dim3: + return TensorComponentType::Dim3; + case ckw::TensorComponentType::Dim4: + return TensorComponentType::Dim4; + case ckw::TensorComponentType::Dim1xDim2: + return TensorComponentType::Dim1xDim2; + case ckw::TensorComponentType::Dim2xDim3: + return TensorComponentType::Dim2xDim3; + case ckw::TensorComponentType::Dim1xDim2xDim3: + return TensorComponentType::Dim1xDim2xDim3; + case ckw::TensorComponentType::Unknown: + return TensorComponentType::Unknown; + default: + ARM_COMPUTE_ERROR("Unknown CKW tensor component"); + return TensorComponentType::Unknown; + } +} + +inline ckw::TensorStorageType to_ckw(const TensorStorageType &storage) +{ + switch(storage) + { + case TensorStorageType::ClBufferUint8Ptr: + return ckw::TensorStorageType::BufferUint8Ptr; + case TensorStorageType::ClImage2dReadOnly: + return ckw::TensorStorageType::Texture2dReadOnly; + case TensorStorageType::ClImage2dWriteOnly: + return ckw::TensorStorageType::Texture2dWriteOnly; + case TensorStorageType::Unknown: + return ckw::TensorStorageType::Unknown; + default: + ARM_COMPUTE_ERROR("Unknown tensor storage type"); + return ckw::TensorStorageType::Unknown; + } +} +inline TensorStorageType from_ckw(const ckw::TensorStorageType &storage) +{ + switch(storage) + { + case ckw::TensorStorageType::BufferUint8Ptr: + return TensorStorageType::ClBufferUint8Ptr; + case ckw::TensorStorageType::Texture2dReadOnly: + return TensorStorageType::ClImage2dReadOnly; + case ckw::TensorStorageType::Texture2dWriteOnly: + return TensorStorageType::ClImage2dWriteOnly; + case ckw::TensorStorageType::Unknown: + return TensorStorageType::Unknown; + default: + ARM_COMPUTE_ERROR("Unknown CKW tensor storage type"); + return TensorStorageType::Unknown; + } +} +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute +#endif /* ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_COMPONENTS_UTILS_TYPE_CONVERTER_COMMON */ diff --git a/src/dynamic_fusion/sketch/gpu/ckw_driver/components/utils/type_converter/ElementwiseBinary.h b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/utils/type_converter/ElementwiseBinary.h new file mode 100644 index 0000000000000000000000000000000000000000..9cb022fc10e0d626141b092616f05897feaba6e2 --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/ckw_driver/components/utils/type_converter/ElementwiseBinary.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_COMPONENTS_UTILS_TYPE_CONVERTER_ELEMENTWISEBINARY +#define ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_COMPONENTS_UTILS_TYPE_CONVERTER_ELEMENTWISEBINARY + +#include "ckw/types/Operators.h" +#include "src/dynamic_fusion/sketch/gpu/operators/internal/GpuElementwiseBinaryCommon.h" + +namespace arm_compute +{ +namespace experimental +{ +namespace dynamic_fusion +{ +inline ckw::BinaryOp to_ckw(const ElementwiseBinaryCommonAttributes &attributes) +{ + switch(attributes.operation()) + { + case ElementwiseBinaryCommonAttributes::ElementwiseOp::Add: + return ckw::BinaryOp::Add; + case ElementwiseBinaryCommonAttributes::ElementwiseOp::Sub: + return ckw::BinaryOp::Sub; + case ElementwiseBinaryCommonAttributes::ElementwiseOp::Div: + return ckw::BinaryOp::Div; + case ElementwiseBinaryCommonAttributes::ElementwiseOp::Mul: + return ckw::BinaryOp::Mul; + case ElementwiseBinaryCommonAttributes::ElementwiseOp::Min: + case ElementwiseBinaryCommonAttributes::ElementwiseOp::Max: + case ElementwiseBinaryCommonAttributes::ElementwiseOp::Power: + case ElementwiseBinaryCommonAttributes::ElementwiseOp::Prelu: + case ElementwiseBinaryCommonAttributes::ElementwiseOp::SquaredDiff: + default: + ARM_COMPUTE_ERROR("Cannot convert ElementwiseBinaryCommonAttributes to corresponding ckw::BinaryOp"); + } +} +} // namespace dynamic_fusion +} // namespace experimental +} // namespace arm_compute + +#endif /* ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_CKW_DRIVER_COMPONENTS_UTILS_TYPE_CONVERTER_ELEMENTWISEBINARY */ diff --git a/src/dynamic_fusion/sketch/gpu/components/IGpuKernelComponent.h b/src/dynamic_fusion/sketch/gpu/components/IGpuKernelComponent.h index 8bb19155a29f0a33bdadd728b355a21804b8181d..af766a7eceb7eb6850015e7525c869c45b376a69 100644 --- a/src/dynamic_fusion/sketch/gpu/components/IGpuKernelComponent.h +++ b/src/dynamic_fusion/sketch/gpu/components/IGpuKernelComponent.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -21,8 +21,8 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifndef SRC_DYNAMIC_FUSION_SKETCH_GPU_COMPONENTS_IGPUKERNELCOMPONENT -#define SRC_DYNAMIC_FUSION_SKETCH_GPU_COMPONENTS_IGPUKERNELCOMPONENT +#ifndef ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_COMPONENTS_IGPUKERNELCOMPONENT +#define ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_COMPONENTS_IGPUKERNELCOMPONENT #include "Types.h" @@ -60,6 +60,7 @@ inline bool operator==(const KernelProperties &config0, const KernelProperties & /** Forward declaration */ class IGpuTemplateComponentWriter; +class IGpuCkwComponentDriver; /** An abstract interface of a component. It enables manipulation by the component graph for purposes like fusion */ @@ -103,8 +104,15 @@ public: { return _properties; } - /** Get template writer for the component */ - virtual const IGpuTemplateComponentWriter *template_writer() const = 0; + /** Get writer for the component */ + virtual const IGpuTemplateComponentWriter *template_writer() const + { + return nullptr; + } + virtual const IGpuCkwComponentDriver *ckw_component_driver() const + { + return nullptr; + } /** Get component type */ virtual GpuComponentType type() const = 0; @@ -116,4 +124,4 @@ private: } // namespace dynamic_fusion } // namespace experimental } // namespace arm_compute -#endif /* SRC_DYNAMIC_FUSION_SKETCH_GPU_COMPONENTS_IGPUKERNELCOMPONENT */ +#endif /* ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_COMPONENTS_IGPUKERNELCOMPONENT */ diff --git a/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentActivation.cpp b/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentActivation.cpp index 6eaa45c25d19c60091d52b466cd0ddd5aa3d32b3..c41257d18c328ad18fb8d70f9a0105892e67f34c 100644 --- a/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentActivation.cpp +++ b/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentActivation.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,7 +24,11 @@ #include "ClComponentActivation.h" #include "src/core/CL/CLValidate.h" +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF #include "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateActivation.h" +#else //ACL_INTERNAL_TEST_CKW_IN_DF +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwActivation.h" +#endif //ACL_INTERNAL_TEST_CKW_IN_DF namespace arm_compute { @@ -32,7 +36,6 @@ namespace experimental { namespace dynamic_fusion { - Status ClComponentActivation::validate(const Properties &properties, const ArgumentPack &tensors, const Attributes &attributes) @@ -66,14 +69,33 @@ ClComponentActivation::ClComponentActivation(ComponentId const ArgumentPack &tensors, const Attributes &attributes) : IGpuKernelComponent{ id, properties, tensors }, - _component_writer{ std::make_unique(id, tensors, attributes) } +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF + _component_writer +{ + std::make_unique(id, tensors, attributes) +} +#else //ACL_INTERNAL_TEST_CKW_IN_DF + _component_writer +{ + std::make_unique(id, tensors, attributes) +} +#endif //ACL_INTERNAL_TEST_CKW_IN_DF { } +ClComponentActivation::~ClComponentActivation() +{ +} + +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF const IGpuTemplateComponentWriter *ClComponentActivation::template_writer() const +#else //ACL_INTERNAL_TEST_CKW_IN_DF +const IGpuCkwComponentDriver *ClComponentActivation::ckw_component_driver() const +#endif //ACL_INTERNAL_TEST_CKW_IN_DF { return _component_writer.get(); } + } // namespace dynamic_fusion } // namespace experimental } // namespace arm_compute diff --git a/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentActivation.h b/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentActivation.h index d1b849ec7357dcbccdd4b5677b3b7f08a2a3e75a..9b090af988a107b845c711a89b74144c045bd9f9 100644 --- a/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentActivation.h +++ b/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentActivation.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,8 +24,11 @@ #ifndef SRC_DYNAMIC_FUSION_SKETCH_GPU_COMPONENTS_CL_CLCOMPONENTACTIVATION #define SRC_DYNAMIC_FUSION_SKETCH_GPU_COMPONENTS_CL_CLCOMPONENTACTIVATION +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/dynamic_fusion/sketch/gpu/components/IGpuKernelComponent.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" + namespace arm_compute { /** Forward declaration */ @@ -39,7 +42,11 @@ template class ArgumentPack; /** Forward declaration */ +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF class ClTemplateActivation; +#else //ACL_INTERNAL_TEST_CKW_IN_DF +class GpuCkwActivation; +#endif //ACL_INTERNAL_TEST_CKW_IN_DF class ClComponentActivation final : public IGpuKernelComponent { @@ -88,7 +95,7 @@ public: const Attributes &attributes); /** Destructor */ - ~ClComponentActivation() override = default; + ~ClComponentActivation() override; /** Prevent instances of this class from being copy constructed */ ClComponentActivation(const ClComponentActivation &component) = delete; @@ -102,8 +109,12 @@ public: /** Allow instances of this class to be moved */ ClComponentActivation &operator=(ClComponentActivation &&component) = default; - /** Get template writer for the component */ + /** Get writer for the component */ +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF const IGpuTemplateComponentWriter *template_writer() const override; +#else //ACL_INTERNAL_TEST_CKW_IN_DF + const IGpuCkwComponentDriver *ckw_component_driver() const override; +#endif //ACL_INTERNAL_TEST_CKW_IN_DF /** Get component type */ GpuComponentType type() const override @@ -112,7 +123,11 @@ public: } private: +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF std::unique_ptr _component_writer; +#else //ACL_INTERNAL_TEST_CKW_IN_DF + std::unique_ptr _component_writer; +#endif //ACL_INTERNAL_TEST_CKW_IN_DF }; } // namespace dynamic_fusion } // namespace experimental diff --git a/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentCast.cpp b/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentCast.cpp index 007ba6380c47bd390e92faef04218fdc555d6c31..635869f817a32e875de2ef0b03be4bb903b3e406 100644 --- a/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentCast.cpp +++ b/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentCast.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,7 +26,11 @@ #include "arm_compute/core/Error.h" #include "src/core/CL/CLValidate.h" #include "src/dynamic_fusion/sketch/ArgumentPack.h" +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF #include "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateCast.h" +#else //ACL_INTERNAL_TEST_CKW_IN_DF +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwCast.h" +#endif //ACL_INTERNAL_TEST_CKW_IN_DF namespace arm_compute { @@ -66,17 +70,32 @@ ClComponentCast::ClComponentCast(ComponentId id, const Attributes &attributes, const Settings &settings) : IGpuKernelComponent{ id, properties, tensors }, - _component_writer{ std::make_unique(id, tensors, attributes) } +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF + _component_writer +{ + std::make_unique(id, tensors, attributes) +} +#else //ACL_INTERNAL_TEST_CKW_IN_DF + _component_writer +{ + std::make_unique(id, tensors, attributes) +} +#endif //ACL_INTERNAL_TEST_CKW_IN_DF { ARM_COMPUTE_UNUSED(attributes, settings); } ClComponentCast::~ClComponentCast() { } +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF const IGpuTemplateComponentWriter *ClComponentCast::template_writer() const +#else //ACL_INTERNAL_TEST_CKW_IN_DF +const IGpuCkwComponentDriver *ClComponentCast::ckw_component_driver() const +#endif //ACL_INTERNAL_TEST_CKW_IN_DF { return _component_writer.get(); } + } // namespace dynamic_fusion } // namespace experimental } // namespace arm_compute diff --git a/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentCast.h b/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentCast.h index 84d6f07f16fe388de7c3a448a687d350c36d522d..37b8cbb6c91698e749f570d5718b27c397ae8397 100644 --- a/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentCast.h +++ b/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentCast.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -48,7 +48,11 @@ private: }; /** Forward declaration */ +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF class ClTemplateCast; +#else //ACL_INTERNAL_TEST_CKW_IN_DF +class GpuCkwCast; +#endif //ACL_INTERNAL_TEST_CKW_IN_DF class ClComponentCast final : public IGpuKernelComponent { @@ -115,8 +119,12 @@ public: ClComponentCast(ClComponentCast &&component) = default; /** Allow instances of this class to be moved */ ClComponentCast &operator=(ClComponentCast &&component) = default; - /** Get template writer for the component */ + /** Get writer for the component */ +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF const IGpuTemplateComponentWriter *template_writer() const override; +#else //ACL_INTERNAL_TEST_CKW_IN_DF + const IGpuCkwComponentDriver *ckw_component_driver() const override; +#endif //ACL_INTERNAL_TEST_CKW_IN_DF /** Get component type */ GpuComponentType type() const override { @@ -124,7 +132,11 @@ public: } private: +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF std::unique_ptr _component_writer; +#else //ACL_INTERNAL_TEST_CKW_IN_DF + std::unique_ptr _component_writer; +#endif //ACL_INTERNAL_TEST_CKW_IN_DF }; } // namespace dynamic_fusion } // namespace experimental diff --git a/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentDirectConv2d.cpp b/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentDirectConv2d.cpp index 3965deced13f8d005a420536ab63307bd758b7a9..a713c8200352fd1754b16bda57210cfef54eb459 100644 --- a/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentDirectConv2d.cpp +++ b/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentDirectConv2d.cpp @@ -26,8 +26,14 @@ #include "arm_compute/core/Validate.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" #include "arm_compute/dynamic_fusion/sketch/attributes/Conv2dAttributes.h" + #include "src/core/CL/CLValidate.h" + +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF #include "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateDirectConv2d.h" +#else // ACL_INTERNAL_TEST_CKW_IN_DF +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwDirectConv2d.h" +#endif // ACL_INTERNAL_TEST_CKW_IN_DF namespace arm_compute { @@ -145,16 +151,27 @@ ClComponentDirectConv2d::ClComponentDirectConv2d( const Attributes &attributes, const Settings &settings) : IGpuKernelComponent{ id, properties, tensors }, +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF _component_writer{ std::make_unique(id, tensors, attributes, settings) } +#else // ACL_INTERNAL_TEST_CKW_IN_DF + _component_writer{ std::make_unique(id, tensors, attributes, settings) } +#endif // ACL_INTERNAL_TEST_CKW_IN_DF { } + ClComponentDirectConv2d::~ClComponentDirectConv2d() { } + +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF const IGpuTemplateComponentWriter *ClComponentDirectConv2d::template_writer() const +#else // ACL_INTERNAL_TEST_CKW_IN_DF +const IGpuCkwComponentDriver *ClComponentDirectConv2d::ckw_component_driver() const +#endif // ACL_INTERNAL_TEST_CKW_IN_DF { return _component_writer.get(); } + } // namespace dynamic_fusion } // namespace experimental } // namespace arm_compute diff --git a/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentDirectConv2d.h b/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentDirectConv2d.h index 8e555dce57d0d6d13300cf655d5595b4bee80a88..24acb1b2c155ae91e2d5655e4fb7ea86845d3045 100644 --- a/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentDirectConv2d.h +++ b/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentDirectConv2d.h @@ -66,7 +66,11 @@ private: }; /** Forward declaration */ +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF class ClTemplateDirectConv2d; +#else // ACL_INTERNAL_TEST_CKW_IN_DF +class GpuCkwDirectConv2d; +#endif // ACL_INTERNAL_TEST_CKW_IN_DF class ClComponentDirectConv2d final : public IGpuKernelComponent { @@ -134,8 +138,12 @@ public: ClComponentDirectConv2d(ClComponentDirectConv2d &&component) = default; /** Allow instances of this class to be moved */ ClComponentDirectConv2d &operator=(ClComponentDirectConv2d &&component) = default; - /** Get template writer for the component */ + /** Get writer for the component */ +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF const IGpuTemplateComponentWriter *template_writer() const override; +#else // ACL_INTERNAL_TEST_CKW_IN_DF + const IGpuCkwComponentDriver *ckw_component_driver() const override; +#endif // ACL_INTERNAL_TEST_CKW_IN_DF /** Get component type */ GpuComponentType type() const override { @@ -143,7 +151,11 @@ public: } private: +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF std::unique_ptr _component_writer; +#else // ACL_INTERNAL_TEST_CKW_IN_DF + std::unique_ptr _component_writer; +#endif // ACL_INTERNAL_TEST_CKW_IN_DF }; } // namespace dynamic_fusion } // namespace experimental diff --git a/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentElementwiseBinary.cpp b/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentElementwiseBinary.cpp index b21c7c382f6e583ca21a9b046b7e4851175a681d..88d729170c7f713b2fb7580da8d4828a75b6099c 100644 --- a/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentElementwiseBinary.cpp +++ b/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentElementwiseBinary.cpp @@ -25,7 +25,11 @@ #include "arm_compute/core/Validate.h" #include "src/core/CL/CLValidate.h" +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF #include "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateElementwiseBinary.h" +#else //ACL_INTERNAL_TEST_CKW_IN_DF +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwElementwiseBinary.h" +#endif //ACL_INTERNAL_TEST_CKW_IN_DF namespace arm_compute { @@ -105,22 +109,38 @@ Status ClComponentElementwiseBinary::validate(const ArgumentPack &t return Status{}; } +ClComponentElementwiseBinary::~ClComponentElementwiseBinary() +{ +} ClComponentElementwiseBinary::ClComponentElementwiseBinary( ComponentId id, const Properties &properties, const ArgumentPack &tensors, const Attributes &attributes) : IGpuKernelComponent{ id, properties, tensors }, - _component_writer{ std::make_unique(id, tensors, attributes) } +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF + _component_writer { + std::make_unique(id, tensors, attributes) } -ClComponentElementwiseBinary::~ClComponentElementwiseBinary() +#else //ACL_INTERNAL_TEST_CKW_IN_DF + _component_writer { + std::make_unique(id, tensors, attributes) } +#endif //ACL_INTERNAL_TEST_CKW_IN_DF +{ +} + +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF const IGpuTemplateComponentWriter *ClComponentElementwiseBinary::template_writer() const +#else //ACL_INTERNAL_TEST_CKW_IN_DF +const IGpuCkwComponentDriver *ClComponentElementwiseBinary::ckw_component_driver() const +#endif //ACL_INTERNAL_TEST_CKW_IN_DF { return _component_writer.get(); } + } // namespace dynamic_fusion } // namespace experimental } // namespace arm_compute diff --git a/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentElementwiseBinary.h b/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentElementwiseBinary.h index 02e61019f49756f406cc57fb2f8ab5281f5fb7f8..f7175903d0153bf04f2bfdfe5d4ecc4e8f694572 100644 --- a/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentElementwiseBinary.h +++ b/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentElementwiseBinary.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -21,10 +21,9 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifndef SRC_DYNAMIC_FUSION_SKETCH_GPU_COMPONENTS_CL_CLCOMPONENTELEMENTWISEBINARY -#define SRC_DYNAMIC_FUSION_SKETCH_GPU_COMPONENTS_CL_CLCOMPONENTELEMENTWISEBINARY +#ifndef ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_COMPONENTS_CL_CLCOMPONENTELEMENTWISEBINARY +#define ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_COMPONENTS_CL_CLCOMPONENTELEMENTWISEBINARY -#include "arm_compute/core/Error.h" #include "src/dynamic_fusion/sketch/gpu/components/IGpuKernelComponent.h" #include "src/dynamic_fusion/sketch/gpu/operators/internal/GpuElementwiseBinaryCommon.h" @@ -41,7 +40,11 @@ template class ArgumentPack; /** Forward declaration */ +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF class ClTemplateElementwiseBinary; +#else //ACL_INTERNAL_TEST_CKW_IN_DF +class GpuCkwElementwiseBinary; +#endif //ACL_INTERNAL_TEST_CKW_IN_DF class ClComponentElementwiseBinary final : public IGpuKernelComponent { @@ -101,8 +104,13 @@ public: ClComponentElementwiseBinary(ClComponentElementwiseBinary &&component) = default; /** Allow instances of this class to be moved */ ClComponentElementwiseBinary &operator=(ClComponentElementwiseBinary &&component) = default; - /** Get template writer for the component */ + /** Get writer for the component */ +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF const IGpuTemplateComponentWriter *template_writer() const override; +#else //ACL_INTERNAL_TEST_CKW_IN_DF + const IGpuCkwComponentDriver *ckw_component_driver() const override; +#endif //ACL_INTERNAL_TEST_CKW_IN_DF + /** Get component type */ GpuComponentType type() const override { @@ -110,9 +118,13 @@ public: } private: +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF std::unique_ptr _component_writer; +#else //ACL_INTERNAL_TEST_CKW_IN_DF + std::unique_ptr _component_writer; +#endif //ACL_INTERNAL_TEST_CKW_IN_DF }; } // namespace dynamic_fusion } // namespace experimental } // namespace arm_compute -#endif /* SRC_DYNAMIC_FUSION_SKETCH_GPU_COMPONENTS_CL_CLCOMPONENTELEMENTWISEBINARY */ +#endif /* ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_COMPONENTS_CL_CLCOMPONENTELEMENTWISEBINARY */ diff --git a/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentStore.cpp b/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentStore.cpp index f49f397ec1e21bc170cb46a469cc2d12b286a425..12b81c3d568c0b4167cec46ec93322dc8fbd8e24 100644 --- a/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentStore.cpp +++ b/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentStore.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,7 +24,11 @@ #include "ClComponentStore.h" #include "src/dynamic_fusion/sketch/ArgumentPack.h" +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF #include "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateStore.h" +#else //ACL_INTERNAL_TEST_CKW_IN_DF +#include "src/dynamic_fusion/sketch/gpu/ckw_driver/components/GpuCkwStore.h" +#endif //ACL_INTERNAL_TEST_CKW_IN_DF #include @@ -42,13 +46,28 @@ Status ClComponentStore::validate( return Status{}; } ClComponentStore::ClComponentStore(ComponentId id, const Properties &properties, const ArgumentPack &tensors) - : IGpuKernelComponent{ id, properties, tensors }, _component_writer{ std::make_unique(id, tensors) } + : IGpuKernelComponent{ id, properties, tensors }, +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF + _component_writer +{ + std::make_unique(id, tensors) +} +#else //ACL_INTERNAL_TEST_CKW_IN_DF + _component_writer +{ + std::make_unique(id, tensors) +} +#endif //ACL_INTERNAL_TEST_CKW_IN_DF { } ClComponentStore::~ClComponentStore() { } +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF const IGpuTemplateComponentWriter *ClComponentStore::template_writer() const +#else //ACL_INTERNAL_TEST_CKW_IN_DF +const IGpuCkwComponentDriver *ClComponentStore::ckw_component_driver() const +#endif //ACL_INTERNAL_TEST_CKW_IN_DF { return _component_writer.get(); } diff --git a/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentStore.h b/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentStore.h index bf8c9f031e6bf478059f0ee5ad2b96035f04cda0..853ee39012bcbc00ce12b683c4943eb8f3b6768d 100644 --- a/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentStore.h +++ b/src/dynamic_fusion/sketch/gpu/components/cl/ClComponentStore.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,9 +24,7 @@ #ifndef SRC_DYNAMIC_FUSION_SKETCH_GPU_COMPONENTS_CL_CLCOMPONENTSTORE #define SRC_DYNAMIC_FUSION_SKETCH_GPU_COMPONENTS_CL_CLCOMPONENTSTORE -#include "arm_compute/core/Error.h" #include "src/dynamic_fusion/sketch/gpu/components/IGpuKernelComponent.h" -#include "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateStore.h" #include namespace arm_compute @@ -40,6 +38,11 @@ namespace dynamic_fusion /** Forward declaration */ template class ArgumentPack; +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF +class ClTemplateStore; +#else //ACL_INTERNAL_TEST_CKW_IN_DF +class GpuCkwStore; +#endif //ACL_INTERNAL_TEST_CKW_IN_DF class ClComponentStore final : public IGpuKernelComponent { @@ -85,8 +88,12 @@ public: ClComponentStore(ClComponentStore &&component) = default; /** Allow instances of this class to be moved */ ClComponentStore &operator=(ClComponentStore &&component) = default; - /** Get template writer for the component */ + /** Get writer for the component */ +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF const IGpuTemplateComponentWriter *template_writer() const override; +#else //ACL_INTERNAL_TEST_CKW_IN_DF + const IGpuCkwComponentDriver *ckw_component_driver() const override; +#endif //ACL_INTERNAL_TEST_CKW_IN_DF /** Get component type */ GpuComponentType type() const override { @@ -94,7 +101,11 @@ public: } private: +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF std::unique_ptr _component_writer; +#else //ACL_INTERNAL_TEST_CKW_IN_DF + std::unique_ptr _component_writer; +#endif //ACL_INTERNAL_TEST_CKW_IN_DF }; } // namespace dynamic_fusion } // namespace experimental diff --git a/src/dynamic_fusion/sketch/gpu/components/utils/type_printer/ElementwiseBinary.h b/src/dynamic_fusion/sketch/gpu/components/utils/type_printer/ElementwiseBinary.h new file mode 100644 index 0000000000000000000000000000000000000000..bc7133f4df8fac9d7e0811ad9cfad68bd75924a8 --- /dev/null +++ b/src/dynamic_fusion/sketch/gpu/components/utils/type_printer/ElementwiseBinary.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_COMPONENTS_UTILS_TYPE_PRINTER_ELEMENTWISEBINARY +#define ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_COMPONENTS_UTILS_TYPE_PRINTER_ELEMENTWISEBINARY + +#include "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentElementwiseBinary.h" + +#include +#include +#include + +namespace arm_compute +{ +/** Type printers for all types related to the component @ref ClComponentElementwiseBinary + */ + +using namespace experimental::dynamic_fusion; + +/** Formatted output of the pute::experimental::dynamic_fusion::ClComponentElementwiseBinary::Attributes::ElementwiseOp type. + * + * @param[out] os Output stream. + * @param[in] op arm_compute::experimental::dynamic_fusion::ClComponentElementwiseBinary::Attributes::ElementwiseOp type to output. + * + * @return Modified output stream. + */ +inline ::std::ostream &operator<<(::std::ostream &os, const ClComponentElementwiseBinary::Attributes::ElementwiseOp &op) +{ + const std::map op_name = + { + { ClComponentElementwiseBinary::Attributes::ElementwiseOp::Add, "add" }, + { ClComponentElementwiseBinary::Attributes::ElementwiseOp::Div, "div" }, + { ClComponentElementwiseBinary::Attributes::ElementwiseOp::Max, "max" }, + { ClComponentElementwiseBinary::Attributes::ElementwiseOp::Min, "min" }, + { ClComponentElementwiseBinary::Attributes::ElementwiseOp::Mul, "mul" }, + { ClComponentElementwiseBinary::Attributes::ElementwiseOp::Power, "power" }, + { ClComponentElementwiseBinary::Attributes::ElementwiseOp::Prelu, "prelu" }, + { ClComponentElementwiseBinary::Attributes::ElementwiseOp::SquaredDiff, "squareddiff" }, + { ClComponentElementwiseBinary::Attributes::ElementwiseOp::Sub, "sub" } + }; + os << op_name.at(op); + return os; +} +/** Formatted output of the arm_compute::experimental::dynamic_fusion::ClComponentElementwiseBinary::Attributes::ElementwiseOp type. + * + * @param[in] op arm_compute::experimental::dynamic_fusion::ClComponentElementwiseBinary::Attributes::ElementwiseOp type to output. + * + * @return Formatted string. + */ +inline std::string to_string(const ClComponentElementwiseBinary::Attributes::ElementwiseOp &op) +{ + std::stringstream str; + str << op; + return str.str(); +} +} // namespace arm_compute +#endif /* ACL_SRC_DYNAMIC_FUSION_SKETCH_GPU_COMPONENTS_UTILS_TYPE_PRINTER_ELEMENTWISEBINARY */ diff --git a/src/dynamic_fusion/sketch/gpu/operators/GpuSigmoid.cpp b/src/dynamic_fusion/sketch/gpu/operators/GpuSigmoid.cpp index 75b726bfda5ebf9150958ed9048ea089cff9c048..09debad9692468ec7b6debb706210ec3083d50ce 100644 --- a/src/dynamic_fusion/sketch/gpu/operators/GpuSigmoid.cpp +++ b/src/dynamic_fusion/sketch/gpu/operators/GpuSigmoid.cpp @@ -24,13 +24,13 @@ #include "arm_compute/dynamic_fusion/sketch/gpu/operators/GpuSigmoid.h" #include "arm_compute/core/experimental/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" +#include "src/common/utils/Log.h" +#include "src/core/helpers/AutoConfiguration.h" #include "src/dynamic_fusion/sketch/ArgumentPack.h" #include "src/dynamic_fusion/sketch/gpu/GpuWorkloadSketchImpl.h" #include "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentActivation.h" -#include "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateActivation.h" -#include "src/core/helpers/AutoConfiguration.h" -#include "src/common/utils/Log.h" namespace arm_compute { @@ -81,7 +81,7 @@ constexpr GpuOperatorType operator_type = GpuOperatorType::Simple; } // namespace Status GpuSigmoid::is_supported_op(const GpuWorkloadContext &context, - const ITensorInfo *src) + const ITensorInfo *src) { return is_supported_op_helper(context, src, nullptr); } @@ -112,8 +112,8 @@ Status GpuSigmoid::validate_op(const GpuWorkloadSketch &sketch, return is_supported_op_helper(*sketch.gpu_context(), src, &dst_info_to_validate); } -ITensorInfo *GpuSigmoid::create_op(GpuWorkloadSketch &sketch, - ITensorInfo *src) +ITensorInfo *GpuSigmoid::create_op(GpuWorkloadSketch &sketch, + ITensorInfo *src) { ARM_COMPUTE_ERROR_ON_NULLPTR(src); ARM_COMPUTE_LOG_PARAMS(src); diff --git a/src/dynamic_fusion/sketch/gpu/operators/GpuSoftmax.cpp b/src/dynamic_fusion/sketch/gpu/operators/GpuSoftmax.cpp index 291a1e5bda09905e9c3f58761afc52c01e246441..ffc4553a7db5433bcce0f4a8076bbb654ec7afda 100644 --- a/src/dynamic_fusion/sketch/gpu/operators/GpuSoftmax.cpp +++ b/src/dynamic_fusion/sketch/gpu/operators/GpuSoftmax.cpp @@ -136,9 +136,9 @@ void GpuSoftmax::create_op(GpuWorkloadSketch &sketch, ARM_COMPUTE_ERROR_ON_NULLPTR(src, dst); ARM_COMPUTE_LOG_PARAMS(src, dst, attributes); TensorShape logits_sum_shape = src->tensor_shape(); - ITensorInfo *logits = sketch.implementation().create_auxiliary_tensor(src->clone()->set_tensor_shape(logits_sum_shape)); + ITensorInfo *logits = sketch.implementation().create_auxiliary_tensor(src->clone()->set_id(ITensorInfo::invalid_tensor_id).set_tensor_shape(logits_sum_shape)); logits_sum_shape.set(0, 1); - ITensorInfo *sum = sketch.implementation().create_auxiliary_tensor(src->clone()->set_tensor_shape(logits_sum_shape)); + ITensorInfo *sum = sketch.implementation().create_auxiliary_tensor(src->clone()->set_id(ITensorInfo::invalid_tensor_id).set_tensor_shape(logits_sum_shape)); // Auto initialize dst tensor info and the auxiliary tensor infos as well auto_init_if_empty(*dst, *src->clone()); diff --git a/src/dynamic_fusion/sketch/gpu/template_writer/GpuKernelVariableTable.h b/src/dynamic_fusion/sketch/gpu/template_writer/GpuKernelVariableTable.h index 14f1c36a1cabc30175a5b7c213fd8f2e7f916a83..a49d38e10c832d6aab16378eac2678ecbbcf894c 100644 --- a/src/dynamic_fusion/sketch/gpu/template_writer/GpuKernelVariableTable.h +++ b/src/dynamic_fusion/sketch/gpu/template_writer/GpuKernelVariableTable.h @@ -32,6 +32,7 @@ #include #include #include +#include namespace arm_compute { diff --git a/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateActivation.cpp b/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateActivation.cpp index 036d7060b4302f8d468cb9f2714fc66f008ad10f..3c7c843dd872f650d6ac1d8a8e23420fbc1de334 100644 --- a/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateActivation.cpp +++ b/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateActivation.cpp @@ -23,7 +23,9 @@ */ #include "ClTemplateActivation.h" -#include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/ActivationFunctionUtils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/helpers/WindowHelpers.h" #include "src/dynamic_fusion/sketch/gpu/GpuKernelComponentGroup.h" #include "support/StringSupport.h" diff --git a/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateActivation.h b/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateActivation.h index 22f8d428abef0d5f8c3bb4cbc492e8e7dcae1424..ec78cf6ce52520bf6bc80c49ce9cb04ccd58e524 100644 --- a/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateActivation.h +++ b/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateActivation.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #define SRC_DYNAMIC_FUSION_SKETCH_GPU_TEMPLATE_WRITER_CL_CLTEMPLATEACTIVATION #include "arm_compute/core/experimental/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentActivation.h" #include "src/dynamic_fusion/sketch/gpu/template_writer/GpuKernelVariableTable.h" #include "src/dynamic_fusion/sketch/gpu/template_writer/IGpuTemplateComponentWriter.h" diff --git a/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateCast.cpp b/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateCast.cpp index e850fee313d6703ff74efe16441f9e103df71596..4956879ad3fe6722bfbf7044965c4505f7868f04 100644 --- a/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateCast.cpp +++ b/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateCast.cpp @@ -23,6 +23,8 @@ */ #include "ClTemplateCast.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/helpers/WindowHelpers.h" #include "src/dynamic_fusion/sketch/gpu/GpuKernelComponentGroup.h" diff --git a/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateDirectConv2d.cpp b/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateDirectConv2d.cpp index ca531fe28edf79f6e88bc84d4e31e96b4a566570..3322487910e7f82bf084ce7203f0df33bc3f665f 100644 --- a/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateDirectConv2d.cpp +++ b/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateDirectConv2d.cpp @@ -26,7 +26,9 @@ #include "src/dynamic_fusion/sketch/gpu/GpuKernelComponentGroup.h" #include "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentDirectConv2d.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/helpers/WindowHelpers.h" #include "support/StringSupport.h" diff --git a/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateElementwiseBinary.cpp b/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateElementwiseBinary.cpp index 52164ba41dcf270ca52cb51f775ec98eb5a28390..c0481ae190e3577f191386382f207107c6c5dd16 100644 --- a/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateElementwiseBinary.cpp +++ b/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateElementwiseBinary.cpp @@ -26,7 +26,9 @@ #include "src/dynamic_fusion/sketch/gpu/GpuKernelComponentGroup.h" #include "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentElementwiseBinary.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/helpers/WindowHelpers.h" #include "support/StringSupport.h" diff --git a/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateLogits1DMaxShiftExpSum.cpp b/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateLogits1DMaxShiftExpSum.cpp index 6de0ba7617a06815a9a86d553c854a06ce9e3ca3..a8d8d32b122da92d5ffef14aaa049132d195e935 100644 --- a/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateLogits1DMaxShiftExpSum.cpp +++ b/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateLogits1DMaxShiftExpSum.cpp @@ -24,6 +24,8 @@ #include "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateLogits1DMaxShiftExpSum.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/helpers/WindowHelpers.h" #include "src/dynamic_fusion/sketch/gpu/GpuKernelComponentGroup.h" #include "support/StringSupport.h" diff --git a/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateLogits1DNorm.cpp b/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateLogits1DNorm.cpp index 0e1c9ef28fce3e293e3c85ebad21087881e9eea6..056e570a25c502000b32b8c5fe8d5193c9a786aa 100644 --- a/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateLogits1DNorm.cpp +++ b/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateLogits1DNorm.cpp @@ -24,6 +24,7 @@ #include "src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateLogits1DNorm.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "src/core/helpers/WindowHelpers.h" #include "src/dynamic_fusion/sketch/gpu/GpuKernelComponentGroup.h" #include "support/StringSupport.h" diff --git a/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplatePool2d.cpp b/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplatePool2d.cpp index fb86e163f204bdcce0af7a571b499bb6a64af9a0..34840c21007e6491f1f13c56c7c8e06dca0d969b 100644 --- a/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplatePool2d.cpp +++ b/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplatePool2d.cpp @@ -27,6 +27,8 @@ #include "src/dynamic_fusion/sketch/gpu/components/cl/ClComponentDirectConv2d.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/helpers/WindowHelpers.h" #include "support/StringSupport.h" diff --git a/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateReshape.cpp b/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateReshape.cpp index 9a989456a7fb5233fb129b28c8cca90e0a167a5a..8b50f1e209d8fc018c802de142b5ac51a47bc4de 100644 --- a/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateReshape.cpp +++ b/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateReshape.cpp @@ -22,6 +22,9 @@ * SOFTWARE. */ #include "ClTemplateReshape.h" + +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/helpers/WindowHelpers.h" #include "src/dynamic_fusion/sketch/gpu/GpuKernelComponentGroup.h" diff --git a/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateResize.cpp b/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateResize.cpp index bcb6492b43857b525794c5f5808feb3092ee91b5..aaed1d990d5b25fa1131c54c269bf6ae00f1281a 100644 --- a/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateResize.cpp +++ b/src/dynamic_fusion/sketch/gpu/template_writer/cl/ClTemplateResize.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,9 @@ #include "ClTemplateResize.h" +#include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/helpers/WindowHelpers.h" #include "src/core/utils/ScaleUtils.h" #include "src/dynamic_fusion/sketch/gpu/GpuKernelComponentGroup.h" diff --git a/src/gpu/cl/ClKernelLibrary.cpp b/src/gpu/cl/ClKernelLibrary.cpp index a9080049b53e5ef5bfcada8cdd691426b9c0f38f..de2e9f9742f2bfbd91f56b5cdf6f25477bca18aa 100644 --- a/src/gpu/cl/ClKernelLibrary.cpp +++ b/src/gpu/cl/ClKernelLibrary.cpp @@ -319,6 +319,10 @@ const std::map ClKernelLibrary::_kernel_program_map = { "l2_normalize_x", "common/l2_normalize.cl" }, { "l2_normalize_y", "common/l2_normalize.cl" }, { "l2_normalize_z", "common/l2_normalize.cl" }, + { "mat_mul_native_mmul_nt_nt", "common/mat_mul_mmul.cl" }, + { "mat_mul_native_mmul_t_nt", "common/mat_mul_mmul.cl" }, + { "mat_mul_native_mmul_nt_t", "common/mat_mul_mmul.cl" }, + { "mat_mul_native_mmul_t_t", "common/mat_mul_mmul.cl" }, { "mat_mul_native_nt_nt", "common/mat_mul.cl" }, { "mat_mul_native_nt_t", "common/mat_mul.cl" }, { "mat_mul_native_t_nt", "common/mat_mul.cl" }, @@ -797,6 +801,10 @@ const std::map ClKernelLibrary::_program_source_map = { "common/mat_mul.cl", #include "./cl_kernels/common/mat_mul.clembed" + }, + { + "common/mat_mul_mmul.cl", +#include "./cl_kernels/common/mat_mul_mmul.clembed" }, { "common/mat_mul_quantized.cl", diff --git a/src/gpu/cl/kernels/ClActivationKernel.cpp b/src/gpu/cl/kernels/ClActivationKernel.cpp index 13d55b3f5ac88efeaeaef0fe3ba290d172fe7e8d..ab1543729f9f1d810a576d73d9ec94ef5a5c079e 100644 --- a/src/gpu/cl/kernels/ClActivationKernel.cpp +++ b/src/gpu/cl/kernels/ClActivationKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021 Arm Limited. + * Copyright (c) 2016-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,6 +27,10 @@ #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/ActivationFunctionUtils.h" +#include "arm_compute/core/utils/StringUtils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/gpu/cl/kernels/ClActivationKernel.h b/src/gpu/cl/kernels/ClActivationKernel.h index 95e010889eeddbf12d5ac76be75c7acbdab9f01b..82e35b61044fc848a2b521b3ea8e3c3efe912d28 100644 --- a/src/gpu/cl/kernels/ClActivationKernel.h +++ b/src/gpu/cl/kernels/ClActivationKernel.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021 Arm Limited. + * Copyright (c) 2016-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CL_ACTIVATION_KERNEL_H #define ARM_COMPUTE_CL_ACTIVATION_KERNEL_H +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/core/common/Macros.h" #include "src/gpu/cl/ClCompileContext.h" #include "src/gpu/cl/IClKernel.h" diff --git a/src/gpu/cl/kernels/ClBatchConcatenateKernel.cpp b/src/gpu/cl/kernels/ClBatchConcatenateKernel.cpp index 62040df63ef22f945d4cdafccfc757324848dd7e..3d8ecf1fcc3a3ba362eacd8a44fff5d4250d1334 100644 --- a/src/gpu/cl/kernels/ClBatchConcatenateKernel.cpp +++ b/src/gpu/cl/kernels/ClBatchConcatenateKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022 Arm Limited. + * Copyright (c) 2019-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -28,6 +28,8 @@ #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/WindowHelpers.h" #include "support/Cast.h" diff --git a/src/gpu/cl/kernels/ClCastKernel.cpp b/src/gpu/cl/kernels/ClCastKernel.cpp index 6baa31e710b4476e747629ab45891df2e0434153..f621ad62d79155e828224370e47c3a2afc1aaf88 100644 --- a/src/gpu/cl/kernels/ClCastKernel.cpp +++ b/src/gpu/cl/kernels/ClCastKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2022 Arm Limited. + * Copyright (c) 2016-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -29,6 +29,8 @@ #include "arm_compute/core/CL/OpenCL.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/StringUtils.h" #include "arm_compute/core/Validate.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" @@ -54,7 +56,7 @@ Status validate_arguments(const ITensorInfo *src, const ITensorInfo *dst, Conver 1, DataType::U8, DataType::S8, DataType::QASYMM8, DataType::QASYMM8_SIGNED, DataType::QSYMM8_PER_CHANNEL, DataType::S16, DataType::U16, DataType::U32, DataType::S32, DataType::F16, - DataType::F32); + DataType::F32, DataType::S64, DataType::U64); ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(dst, 1, DataType::U8, DataType::S8, DataType::QASYMM8, DataType::S16, diff --git a/src/gpu/cl/kernels/ClCastKernel.h b/src/gpu/cl/kernels/ClCastKernel.h index 7fadfa73d0590ab7f61d40804102cc42ea0dfe2a..a021b3c78c74bdbe7af95efebd141a35f25bfa50 100644 --- a/src/gpu/cl/kernels/ClCastKernel.h +++ b/src/gpu/cl/kernels/ClCastKernel.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2022 Arm Limited. + * Copyright (c) 2016-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -53,12 +53,14 @@ public: * - U16 -> U8, S8, S16, U32, S32, F16, F32 * - S16 -> U8, S8, U16, U32, S32, F16, F32 * - U32 -> U8, S8, U16, S16, S32, F16, F32 + * - S64 -> U8, S8, U16, S16, U32, S32, F16, F32 + * - U64 -> U8, S8, U16, S16, U32, S32, F16, F32 * - S32 -> U8, S8, U16, S16, U32, F16, F32 - * - F16 -> U8, S8, U16, S16, U32, F32 - * - F32 -> U8, S8, U16, S16, U32, F16 + * - F16 -> U8, S8, U16, S16, U32, S32, F32 + * - F32 -> U8, S8, U16, S16, U32, S32, F16 * * @param[in] compile_context The compile context to be used. - * @param[in] src The source tensor to convert. Data types supported: U8/S8/QSYMM8_PER_CHANNEL/U16/S16/U32/S32/F16/F32. + * @param[in] src The source tensor to convert. Data types supported: U8/S8/QSYMM8_PER_CHANNEL/U16/S16/U32/S32/U64/S64/F16/F32. * @param[out] dst The destination tensor. Data types supported: U8/S8/QASYMM8/U16/S16/U32/S32/F16/F32. * @param[in] policy Conversion policy */ diff --git a/src/gpu/cl/kernels/ClCol2ImKernel.cpp b/src/gpu/cl/kernels/ClCol2ImKernel.cpp index ac7c4a43f9f24cfc24c9cf96f535dd5b2517d360..3316742912c1e8bcea2eac1d4c980711be859c7c 100644 --- a/src/gpu/cl/kernels/ClCol2ImKernel.cpp +++ b/src/gpu/cl/kernels/ClCol2ImKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -29,6 +29,7 @@ #include "arm_compute/core/CL/OpenCL.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/gpu/cl/kernels/ClCopyKernel.cpp b/src/gpu/cl/kernels/ClCopyKernel.cpp index eefdc925a46c5a2d8ded76caff9e59731f38010b..47194488194c38a092e8e8d4c1dfa821e0abb63d 100644 --- a/src/gpu/cl/kernels/ClCopyKernel.cpp +++ b/src/gpu/cl/kernels/ClCopyKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -29,6 +29,7 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/Validate.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" diff --git a/src/gpu/cl/kernels/ClDepthConcatenateKernel.cpp b/src/gpu/cl/kernels/ClDepthConcatenateKernel.cpp index 9704294d62b91d026578f3a4c6217cd1c91f1adb..a05cd1321e562f3b64abc0a412244320010d8e0e 100644 --- a/src/gpu/cl/kernels/ClDepthConcatenateKernel.cpp +++ b/src/gpu/cl/kernels/ClDepthConcatenateKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2022 Arm Limited. + * Copyright (c) 2017-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -28,6 +28,8 @@ #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/WindowHelpers.h" #include "support/Cast.h" diff --git a/src/gpu/cl/kernels/ClDequantizeKernel.cpp b/src/gpu/cl/kernels/ClDequantizeKernel.cpp index c2e8f2f91fd90c8e7cd19d23dbeca6fc9b480a9e..756cd56a8b93c0898ab5fbbba289f048152b9686 100644 --- a/src/gpu/cl/kernels/ClDequantizeKernel.cpp +++ b/src/gpu/cl/kernels/ClDequantizeKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -28,6 +28,7 @@ #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/StringUtils.h" #include "arm_compute/core/Validate.h" #include "src/core/CL/CLValidate.h" diff --git a/src/gpu/cl/kernels/ClDirectConv2dKernel.cpp b/src/gpu/cl/kernels/ClDirectConv2dKernel.cpp index 68d7e30c9b4e1eb21fdda081420f9ccd3ca9d129..7ad398412a0bf4fe0cd6869d1b40d8ecb1d6f4db 100644 --- a/src/gpu/cl/kernels/ClDirectConv2dKernel.cpp +++ b/src/gpu/cl/kernels/ClDirectConv2dKernel.cpp @@ -23,15 +23,17 @@ */ #include "src/gpu/cl/kernels/ClDirectConv2dKernel.h" +#include "arm_compute/core/utils/ActivationFunctionUtils.h" #include "arm_compute/core/CL/CLKernelLibrary.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/ITensor.h" #include "arm_compute/core/KernelDescriptors.h" #include "arm_compute/core/PixelValue.h" -#include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" #include "arm_compute/core/utils/quantization/AsymmHelpers.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/AccessWindowStatic.h" #include "src/core/CL/CLUtils.h" #include "src/core/CL/CLValidate.h" @@ -431,35 +433,20 @@ void ClDirectConv2dKernel::run_op(ITensorPack &tensors, const Window &window, cl if(_export_weights_to_cl_image) { - const size_t image_w = weights->info()->dimension(0) / 4; - const size_t image_h = weights->info()->dimension(1) * weights->info()->dimension(2) * weights->info()->dimension(3); - const TensorShape shape2d(image_w, image_h); - const size_t image_row_pitch = weights->info()->strides_in_bytes()[1]; - - // Export cl_buffer to cl_image - weights_cl_image = create_image2d_from_buffer(CLKernelLibrary::get().context(), weights->cl_buffer(), shape2d, weights->info()->data_type(), image_row_pitch, CLImage2DType::ReadOnly); + // Export tensor to cl_image + weights_cl_image = create_image2d_from_tensor(weights, CLImage2DType::ReadOnly); } if(_export_output_to_cl_image) { - const size_t image_w = dst->info()->dimension(0) / 4; - const size_t image_h = dst->info()->dimension(1) * dst->info()->dimension(2) * dst->info()->dimension(3); - const TensorShape shape2d(image_w, image_h); - const size_t image_row_pitch = dst->info()->strides_in_bytes()[1]; - - // Export cl_buffer to cl_image - output_cl_image = create_image2d_from_buffer(CLKernelLibrary::get().context(), dst->cl_buffer(), shape2d, dst->info()->data_type(), image_row_pitch, CLImage2DType::WriteOnly); + // Export tensor to cl_image + output_cl_image = create_image2d_from_tensor(dst, CLImage2DType::WriteOnly); } if(_export_input_to_cl_image) { - const size_t image_w = src->info()->dimension(0) / 4; - const size_t image_h = src->info()->dimension(1) * src->info()->dimension(2) * src->info()->dimension(3); - const TensorShape shape2d(image_w, image_h); - const size_t image_row_pitch = src->info()->strides_in_bytes()[1]; - - // Export cl_buffer to cl_image - input_cl_image = create_image2d_from_buffer(CLKernelLibrary::get().context(), src->cl_buffer(), shape2d, src->info()->data_type(), image_row_pitch, CLImage2DType::ReadOnly); + // Export tensor to cl_image + input_cl_image = create_image2d_from_tensor(src, CLImage2DType::ReadOnly); } unsigned int idx = 0; diff --git a/src/gpu/cl/kernels/ClDirectConv2dKernel.h b/src/gpu/cl/kernels/ClDirectConv2dKernel.h index 25171a0536298fb73d683cdb605f20263339d5f8..7132762b35aa161298d60fc9148b38057406aa67 100644 --- a/src/gpu/cl/kernels/ClDirectConv2dKernel.h +++ b/src/gpu/cl/kernels/ClDirectConv2dKernel.h @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CL_DIRECT_CONV2D_KERNEL_H #define ARM_COMPUTE_CL_DIRECT_CONV2D_KERNEL_H +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/core/common/Macros.h" #include "src/gpu/cl/ClCompileContext.h" #include "src/gpu/cl/IClKernel.h" diff --git a/src/gpu/cl/kernels/ClDirectConv3dKernel.cpp b/src/gpu/cl/kernels/ClDirectConv3dKernel.cpp index 79f425189a4bcf7eb6eebfbc23292ded40bbf79c..6191178911ef6f2a235cebc82258506b3bee2e1d 100644 --- a/src/gpu/cl/kernels/ClDirectConv3dKernel.cpp +++ b/src/gpu/cl/kernels/ClDirectConv3dKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,8 +24,10 @@ #include "src/gpu/cl/kernels/ClDirectConv3dKernel.h" #include "arm_compute/core/CL/ICLTensor.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" #include "arm_compute/core/utils/quantization/AsymmHelpers.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/WindowHelpers.h" #include "support/Cast.h" diff --git a/src/gpu/cl/kernels/ClElementwiseKernel.cpp b/src/gpu/cl/kernels/ClElementwiseKernel.cpp index 65cd05299533ca861ac2598d82470dbccdafbe54..6beee576b5d21cdd3fc583f71c4156ed27489290 100644 --- a/src/gpu/cl/kernels/ClElementwiseKernel.cpp +++ b/src/gpu/cl/kernels/ClElementwiseKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,8 +23,12 @@ */ #include "src/gpu/cl/kernels/ClElementwiseKernel.h" +#include "arm_compute/core/utils/ActivationFunctionUtils.h" #include "arm_compute/core/CL/CLHelpers.h" #include "arm_compute/core/CL/ICLTensor.h" +#include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/common/utils/Validate.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" diff --git a/src/gpu/cl/kernels/ClElementwiseKernel.h b/src/gpu/cl/kernels/ClElementwiseKernel.h index 3783e1571c9e66e681b3f28272826aa8d1f9ba09..ea3ddb2124c14a3cb08730a307b0b33bc623de20 100644 --- a/src/gpu/cl/kernels/ClElementwiseKernel.h +++ b/src/gpu/cl/kernels/ClElementwiseKernel.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CL_ELEMENTWISE_KERNEL_H #define ARM_COMPUTE_CL_ELEMENTWISE_KERNEL_H +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/core/KernelTypes.h" #include "src/core/common/Macros.h" #include "src/gpu/cl/ClCompileContext.h" diff --git a/src/gpu/cl/kernels/ClElementwiseUnaryKernel.cpp b/src/gpu/cl/kernels/ClElementwiseUnaryKernel.cpp index 40b1eaca1f3ed3bc2ed46dc60bade3360749bf17..744a3a40c70c3f35ada7556aff2c265ff5d2a558 100644 --- a/src/gpu/cl/kernels/ClElementwiseUnaryKernel.cpp +++ b/src/gpu/cl/kernels/ClElementwiseUnaryKernel.cpp @@ -23,8 +23,11 @@ */ #include "src/gpu/cl/kernels/ClElementwiseUnaryKernel.h" +#include "arm_compute/core/Utils.h" #include "arm_compute/core/CL/CLHelpers.h" #include "arm_compute/core/CL/ICLTensor.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/WindowHelpers.h" #include "support/Cast.h" diff --git a/src/gpu/cl/kernels/ClFloorKernel.cpp b/src/gpu/cl/kernels/ClFloorKernel.cpp index 1cb78242cb685a073a9050d9209b24d1a52b74c1..f9f834875ad2d7f1f28551fd96d0a280421aa198 100644 --- a/src/gpu/cl/kernels/ClFloorKernel.cpp +++ b/src/gpu/cl/kernels/ClFloorKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -29,6 +29,7 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/Validate.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" diff --git a/src/gpu/cl/kernels/ClGemmLowpMatrixMultiplyNativeKernel.cpp b/src/gpu/cl/kernels/ClGemmLowpMatrixMultiplyNativeKernel.cpp index bad3d25d22babd1c9edf33b470d24553ef4f1d71..accafeecc267ac03ea25b4ec51226f79174838bf 100644 --- a/src/gpu/cl/kernels/ClGemmLowpMatrixMultiplyNativeKernel.cpp +++ b/src/gpu/cl/kernels/ClGemmLowpMatrixMultiplyNativeKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022 Arm Limited. + * Copyright (c) 2019-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -29,7 +29,7 @@ #include "arm_compute/core/CL/OpenCL.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" -#include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/StringUtils.h" #include "arm_compute/core/Validate.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" diff --git a/src/gpu/cl/kernels/ClGemmLowpMatrixMultiplyReshapedKernel.cpp b/src/gpu/cl/kernels/ClGemmLowpMatrixMultiplyReshapedKernel.cpp index 0325c00a5c62aa2972b1a8a908a0021f3444da38..15493f7ddcf7082aad42f55572b020290eeb6f7b 100644 --- a/src/gpu/cl/kernels/ClGemmLowpMatrixMultiplyReshapedKernel.cpp +++ b/src/gpu/cl/kernels/ClGemmLowpMatrixMultiplyReshapedKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022 Arm Limited. + * Copyright (c) 2019-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -29,9 +29,9 @@ #include "arm_compute/core/CL/OpenCL.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" -#include "arm_compute/core/Utils.h" #include "arm_compute/core/Validate.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/gpu/cl/kernels/ClGemmLowpMatrixMultiplyReshapedOnlyRhsKernel.cpp b/src/gpu/cl/kernels/ClGemmLowpMatrixMultiplyReshapedOnlyRhsKernel.cpp index 90084ea97fa2d37a1de4f7f159b963b268f931b2..5d552b8d63befc6f5bc808b74f3ab5a97392c2b6 100644 --- a/src/gpu/cl/kernels/ClGemmLowpMatrixMultiplyReshapedOnlyRhsKernel.cpp +++ b/src/gpu/cl/kernels/ClGemmLowpMatrixMultiplyReshapedOnlyRhsKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022 Arm Limited. + * Copyright (c) 2019-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -29,9 +29,9 @@ #include "arm_compute/core/CL/OpenCL.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" -#include "arm_compute/core/Utils.h" #include "arm_compute/core/Validate.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/AccessWindowStatic.h" #include "src/core/helpers/AutoConfiguration.h" diff --git a/src/gpu/cl/kernels/ClGemmLowpMatrixMultiplyReshapedOnlyRhsMMULKernel.cpp b/src/gpu/cl/kernels/ClGemmLowpMatrixMultiplyReshapedOnlyRhsMMULKernel.cpp index cdd047cb28beefa01e97827095a528f211d34226..792c71da76eb2455bf0102f0a721221a4ca464d8 100644 --- a/src/gpu/cl/kernels/ClGemmLowpMatrixMultiplyReshapedOnlyRhsMMULKernel.cpp +++ b/src/gpu/cl/kernels/ClGemmLowpMatrixMultiplyReshapedOnlyRhsMMULKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,10 +23,12 @@ */ #include "src/gpu/cl/kernels/ClGemmLowpMatrixMultiplyReshapedOnlyRhsMMULKernel.h" +#include "arm_compute/core/utils/ActivationFunctionUtils.h" #include "arm_compute/core/CL/CLHelpers.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/gpu/cl/kernels/ClGemmLowpOffsetContributionKernel.cpp b/src/gpu/cl/kernels/ClGemmLowpOffsetContributionKernel.cpp index a8efd0610b35ec70050320cf5cd4a8a453e74702..9ec0b5182f98ddf839e8fe5f6c34f7027c8d6f44 100644 --- a/src/gpu/cl/kernels/ClGemmLowpOffsetContributionKernel.cpp +++ b/src/gpu/cl/kernels/ClGemmLowpOffsetContributionKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2022 Arm Limited. + * Copyright (c) 2017-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,7 +27,9 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/Validate.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/gpu/cl/kernels/ClGemmLowpOffsetContributionOutputStageKernel.cpp b/src/gpu/cl/kernels/ClGemmLowpOffsetContributionOutputStageKernel.cpp index a1697254cc1b2e418a5816b306f8d8f20d1bbf69..c5fb54f524772e63ac24f7e36d94039ca7b77a4c 100644 --- a/src/gpu/cl/kernels/ClGemmLowpOffsetContributionOutputStageKernel.cpp +++ b/src/gpu/cl/kernels/ClGemmLowpOffsetContributionOutputStageKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022 Arm Limited. + * Copyright (c) 2018-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -28,6 +28,8 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/StringUtils.h" #include "arm_compute/core/Validate.h" #include "src/core/helpers/AutoConfiguration.h" diff --git a/src/gpu/cl/kernels/ClGemmLowpQuantizeDownInt32ScaleByFixedPointKernel.cpp b/src/gpu/cl/kernels/ClGemmLowpQuantizeDownInt32ScaleByFixedPointKernel.cpp index 795f3174a2030c0563f4fb8efd9c7debeead4320..39754385a13fd0cda2ddac7e736a36488c8d729b 100644 --- a/src/gpu/cl/kernels/ClGemmLowpQuantizeDownInt32ScaleByFixedPointKernel.cpp +++ b/src/gpu/cl/kernels/ClGemmLowpQuantizeDownInt32ScaleByFixedPointKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022 Arm Limited. + * Copyright (c) 2020-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -28,7 +28,9 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Validate.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "arm_compute/core/utils/quantization/AsymmHelpers.h" #include "src/core/helpers/AutoConfiguration.h" diff --git a/src/gpu/cl/kernels/ClGemmLowpQuantizeDownInt32ScaleByFloatKernel.cpp b/src/gpu/cl/kernels/ClGemmLowpQuantizeDownInt32ScaleByFloatKernel.cpp index 8d4cb923d6a94e9174d2d39ef7de3da99d47d644..f3796983266f50f8575eb46912d5e3da344acef9 100644 --- a/src/gpu/cl/kernels/ClGemmLowpQuantizeDownInt32ScaleByFloatKernel.cpp +++ b/src/gpu/cl/kernels/ClGemmLowpQuantizeDownInt32ScaleByFloatKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022 Arm Limited. + * Copyright (c) 2018-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -28,8 +28,10 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Validate.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" #include "arm_compute/core/utils/quantization/AsymmHelpers.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/gpu/cl/kernels/ClGemmLowpQuantizeDownInt32ScaleKernel.cpp b/src/gpu/cl/kernels/ClGemmLowpQuantizeDownInt32ScaleKernel.cpp index bad9d961b863a5440f73f66483b7c44a7399a4a5..5d54db214a1c88643b4a4e9ee8cadb9a047c82a2 100644 --- a/src/gpu/cl/kernels/ClGemmLowpQuantizeDownInt32ScaleKernel.cpp +++ b/src/gpu/cl/kernels/ClGemmLowpQuantizeDownInt32ScaleKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022 Arm Limited. + * Copyright (c) 2020-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,7 +27,10 @@ #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/Validate.h" +#include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/utils/quantization/AsymmHelpers.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" @@ -159,4 +162,4 @@ void ClGemmLowpQuantizeDownInt32ScaleKernel::run_op(ITensorPack &tensors, const } } // namespace kernels } // namespace opencl -} // namespace arm_compute \ No newline at end of file +} // namespace arm_compute diff --git a/src/gpu/cl/kernels/ClGemmLowpReductionKernel.cpp b/src/gpu/cl/kernels/ClGemmLowpReductionKernel.cpp index 6ab547cfd527b0457d2d8edbd2e84cf9f5bf30cc..ea88b485a045fc2dfc918044dee64995f0f5a42c 100644 --- a/src/gpu/cl/kernels/ClGemmLowpReductionKernel.cpp +++ b/src/gpu/cl/kernels/ClGemmLowpReductionKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2022 Arm Limited. + * Copyright (c) 2017-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,9 @@ #include "arm_compute/core/CL/CLHelpers.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/KernelDescriptors.h" +#include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/gpu/cl/kernels/ClGemmMatrixMultiplyNativeKernel.cpp b/src/gpu/cl/kernels/ClGemmMatrixMultiplyNativeKernel.cpp index 05988997e7f3cfd096633b0344471e133aaa6071..5fea097ae38aed476c9bce81b4ce418583a66aa5 100644 --- a/src/gpu/cl/kernels/ClGemmMatrixMultiplyNativeKernel.cpp +++ b/src/gpu/cl/kernels/ClGemmMatrixMultiplyNativeKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,15 +23,16 @@ */ #include "src/gpu/cl/kernels/ClGemmMatrixMultiplyNativeKernel.h" +#include "arm_compute/core/utils/ActivationFunctionUtils.h" #include "arm_compute/core/CL/CLHelpers.h" #include "arm_compute/core/CL/CLKernelLibrary.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/CL/OpenCL.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" -#include "arm_compute/core/Utils.h" #include "arm_compute/core/Validate.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/AccessWindowStatic.h" #include "src/core/CL/CLUtils.h" #include "src/core/experimental/PostOpUtils.h" diff --git a/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedKernel.cpp b/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedKernel.cpp index f74a5d87affa9bd79392faf555e91a94055f7745..f14a6f1900399d5731c6fbdbc2733794add4164e 100644 --- a/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedKernel.cpp +++ b/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedKernel.cpp @@ -23,15 +23,16 @@ */ #include "src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedKernel.h" +#include "arm_compute/core/utils/ActivationFunctionUtils.h" #include "arm_compute/core/CL/CLHelpers.h" #include "arm_compute/core/CL/CLKernelLibrary.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/CL/OpenCL.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" -#include "arm_compute/core/Utils.h" #include "arm_compute/core/Validate.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/experimental/PostOpUtils.h" diff --git a/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedOnlyRhsKernel.cpp b/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedOnlyRhsKernel.cpp index efd0a95eabac9db8c841f3f8c3bb80f0f2baf477..f780538f53a3c12964b8990cf55d99021b7359c1 100644 --- a/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedOnlyRhsKernel.cpp +++ b/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedOnlyRhsKernel.cpp @@ -23,8 +23,10 @@ */ #include "src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedOnlyRhsKernel.h" +#include "arm_compute/core/utils/ActivationFunctionUtils.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/experimental/PostOpUtils.h" diff --git a/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedOnlyRhsMMULKernel.cpp b/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedOnlyRhsMMULKernel.cpp index f252afb06a142e9c7ece0bb738ca9e9fcca59994..734f8f9b4c934dc213ca5ab19ff295b6fa0f7afa 100644 --- a/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedOnlyRhsMMULKernel.cpp +++ b/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedOnlyRhsMMULKernel.cpp @@ -23,13 +23,14 @@ */ #include "src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedOnlyRhsMMULKernel.h" +#include "arm_compute/core/utils/ActivationFunctionUtils.h" #include "arm_compute/core/CL/CLHelpers.h" #include "arm_compute/core/CL/CLKernelLibrary.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/CL/OpenCL.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" -#include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/StringUtils.h" #include "arm_compute/core/Validate.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" #include "src/core/CL/CLUtils.h" diff --git a/src/gpu/cl/kernels/ClGemmReshapeLhsMatrixKernel.cpp b/src/gpu/cl/kernels/ClGemmReshapeLhsMatrixKernel.cpp index 413c70ae1b19244d3d43568b7432db4b3757bf97..bf4b664b6e71279191e6c21ac65c62746966c3ba 100644 --- a/src/gpu/cl/kernels/ClGemmReshapeLhsMatrixKernel.cpp +++ b/src/gpu/cl/kernels/ClGemmReshapeLhsMatrixKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -29,8 +29,8 @@ #include "arm_compute/core/CL/OpenCL.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" -#include "arm_compute/core/Utils.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/AccessWindowStatic.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" diff --git a/src/gpu/cl/kernels/ClHeightConcatenateKernel.cpp b/src/gpu/cl/kernels/ClHeightConcatenateKernel.cpp index 95eca69bde70fa4731319ad1623676bd3825a545..719201d1fe239c4ab65ba0832d11e6cb1e05b4e4 100644 --- a/src/gpu/cl/kernels/ClHeightConcatenateKernel.cpp +++ b/src/gpu/cl/kernels/ClHeightConcatenateKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022 Arm Limited. + * Copyright (c) 2019-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,8 +27,9 @@ #include "arm_compute/core/CL/CLKernelLibrary.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/Helpers.h" -#include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/WindowHelpers.h" #include "support/Cast.h" diff --git a/src/gpu/cl/kernels/ClIm2ColKernel.cpp b/src/gpu/cl/kernels/ClIm2ColKernel.cpp index 6d1271d2488b367527eea9fb2f46153b56653cd6..e89084719929214573a4751b38316288e400234c 100644 --- a/src/gpu/cl/kernels/ClIm2ColKernel.cpp +++ b/src/gpu/cl/kernels/ClIm2ColKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -31,6 +31,7 @@ #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Validate.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/AccessWindowStatic.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" diff --git a/src/gpu/cl/kernels/ClIndirectConv2dAddressPrecalculationKernel.cpp b/src/gpu/cl/kernels/ClIndirectConv2dAddressPrecalculationKernel.cpp index 8e12f23fa6e9310128c7b5cf16c1aa6dc3a9be15..d291fad76c655fd5644b965842b592ddc6d8c79a 100644 --- a/src/gpu/cl/kernels/ClIndirectConv2dAddressPrecalculationKernel.cpp +++ b/src/gpu/cl/kernels/ClIndirectConv2dAddressPrecalculationKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -28,6 +28,7 @@ #include "arm_compute/core/ITensor.h" #include "arm_compute/core/KernelDescriptors.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/gpu/cl/kernels/ClIndirectConv2dKernel.cpp b/src/gpu/cl/kernels/ClIndirectConv2dKernel.cpp index b8915cc406752ab26e06990632cac15f1e3b3958..a337eb50fda55e586b47b9106ffee1809cfbe511 100644 --- a/src/gpu/cl/kernels/ClIndirectConv2dKernel.cpp +++ b/src/gpu/cl/kernels/ClIndirectConv2dKernel.cpp @@ -23,11 +23,13 @@ */ #include "src/gpu/cl/kernels/ClIndirectConv2dKernel.h" +#include "arm_compute/core/utils/ActivationFunctionUtils.h" #include "arm_compute/core/CL/CLKernelLibrary.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/KernelDescriptors.h" -#include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" diff --git a/src/gpu/cl/kernels/ClIndirectConv2dKernel.h b/src/gpu/cl/kernels/ClIndirectConv2dKernel.h index d86029c5c9eaee908000a78630133b93b705f75f..b6c7b35fa48c4849887323369174751ceb57ff0d 100644 --- a/src/gpu/cl/kernels/ClIndirectConv2dKernel.h +++ b/src/gpu/cl/kernels/ClIndirectConv2dKernel.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CL_DIRECT_CONV2D_KERNEL_H #define ARM_COMPUTE_CL_DIRECT_CONV2D_KERNEL_H +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/core/common/Macros.h" #include "src/gpu/cl/ClCompileContext.h" #include "src/gpu/cl/IClKernel.h" diff --git a/src/gpu/cl/kernels/ClMatMulLowpNativeKernel.cpp b/src/gpu/cl/kernels/ClMatMulLowpNativeKernel.cpp index d5ecdf7dd27338ce3e5c7efc364968c44a9cdd04..f7fdbe2c238bc100ed48c29aaba171dcf5fb6372 100644 --- a/src/gpu/cl/kernels/ClMatMulLowpNativeKernel.cpp +++ b/src/gpu/cl/kernels/ClMatMulLowpNativeKernel.cpp @@ -23,18 +23,23 @@ */ #include "src/gpu/cl/kernels/ClMatMulLowpNativeKernel.h" +#include "arm_compute/core/utils/ActivationFunctionUtils.h" #include "arm_compute/core/CL/CLHelpers.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/ITensorPack.h" #include "arm_compute/core/TensorInfo.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" #include "arm_compute/core/utils/quantization/AsymmHelpers.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/common/utils/Log.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" #include "src/gpu/cl/ClCompileContext.h" +#include "arm_compute/core/QuantizationInfo.h" + #include "support/Cast.h" #include "support/StringSupport.h" @@ -98,34 +103,48 @@ ClMatMulLowpNativeKernel::ClMatMulLowpNativeKernel() { _type = CLKernelType::GEMM; } -Status ClMatMulLowpNativeKernel::validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *output, const MatMulKernelInfo &matmul_kernel_info) +Status ClMatMulLowpNativeKernel::validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *bias, const ITensorInfo *dst, const MatMulKernelInfo &matmul_kernel_info, + const ActivationLayerInfo &act_info) { - ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(lhs, rhs, output); + ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(lhs, rhs, dst); ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(lhs, 1, DataType::QASYMM8, DataType::QASYMM8_SIGNED); ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(lhs, rhs); ARM_COMPUTE_RETURN_ON_ERROR(validate_matmul_kernel_info(matmul_kernel_info)); ARM_COMPUTE_RETURN_ON_ERROR(validate_input_shapes(lhs->tensor_shape(), rhs->tensor_shape(), matmul_kernel_info)); - if(output->total_size() != 0) + ARM_COMPUTE_RETURN_ERROR_ON_MSG((act_info.activation() != ActivationFunction::IDENTITY && act_info.activation() != ActivationFunction::RELU + && act_info.activation() != ActivationFunction::LU_BOUNDED_RELU && act_info.activation() != ActivationFunction::BOUNDED_RELU), + "Activation Function specified is unsupported."); + const TensorShape expected_output_shape = misc::shape_calculator::compute_matmul_shape(lhs->tensor_shape(), rhs->tensor_shape(), matmul_kernel_info); + + if(dst->total_size() != 0) + { + const TensorInfo tensor_info_output = dst->clone()->set_tensor_shape(expected_output_shape); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(dst, &tensor_info_output); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(lhs, dst); + } + + if(bias != nullptr) { - const TensorInfo tensor_info_output = output->clone()->set_tensor_shape(misc::shape_calculator::compute_matmul_shape(lhs->tensor_shape(), rhs->tensor_shape(), matmul_kernel_info)); - ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(output, &tensor_info_output); - ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(lhs, output); + ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(bias, 1, DataType::S32); + ARM_COMPUTE_RETURN_ERROR_ON(bias->num_dimensions() > 1); + ARM_COMPUTE_RETURN_ERROR_ON(expected_output_shape[0] != bias->dimension(0)); } return Status{}; } -void ClMatMulLowpNativeKernel::configure(const ClCompileContext &compile_context, ITensorInfo *lhs, ITensorInfo *rhs, ITensorInfo *output, const MatMulKernelInfo &matmul_kernel_info) +void ClMatMulLowpNativeKernel::configure(const ClCompileContext &compile_context, ITensorInfo *lhs, ITensorInfo *rhs, ITensorInfo *bias, ITensorInfo *dst, const MatMulKernelInfo &matmul_kernel_info, + const ActivationLayerInfo &act_info) { - ARM_COMPUTE_ERROR_ON_NULLPTR(lhs, rhs, output, &compile_context, &matmul_kernel_info); - ARM_COMPUTE_LOG_PARAMS(lhs, rhs, output, matmul_kernel_info); - ARM_COMPUTE_ERROR_THROW_ON(validate(lhs, rhs, output, matmul_kernel_info)); + ARM_COMPUTE_ERROR_ON_NULLPTR(lhs, rhs, dst, &compile_context, &matmul_kernel_info); + ARM_COMPUTE_LOG_PARAMS(lhs, rhs, bias, dst, matmul_kernel_info); + ARM_COMPUTE_ERROR_THROW_ON(validate(lhs, rhs, bias, dst, matmul_kernel_info)); - // output tensor auto initialization if not yet initialized - auto_init_if_empty(*output, lhs->clone()->set_tensor_shape(misc::shape_calculator::compute_matmul_shape(lhs->tensor_shape(), rhs->tensor_shape(), matmul_kernel_info))); + // dst tensor auto initialization if not yet initialized + auto_init_if_empty(*dst, lhs->clone()->set_tensor_shape(misc::shape_calculator::compute_matmul_shape(lhs->tensor_shape(), rhs->tensor_shape(), matmul_kernel_info))); - const int m = output->dimension(1); - const int n = output->dimension(0); + const int m = dst->dimension(1); + const int n = dst->dimension(0); const int k = matmul_kernel_info.adj_lhs ? lhs->tensor_shape().y() : lhs->tensor_shape().x(); const bool adj_lhs = matmul_kernel_info.adj_lhs; @@ -133,7 +152,7 @@ void ClMatMulLowpNativeKernel::configure(const ClCompileContext &compile_context int n0 = adjust_vec_size(matmul_kernel_info.n0, n); // Configure kernel window - Window win = calculate_max_window(*output, Steps(n0, m0)); + Window win = calculate_max_window(*dst, Steps(n0, m0)); win = win.collapse(win, Window::DimZ); IClKernel::configure_internal(win); @@ -152,7 +171,7 @@ void ClMatMulLowpNativeKernel::configure(const ClCompileContext &compile_context const UniformQuantizationInfo lqinfo = lhs->quantization_info().uniform(); const UniformQuantizationInfo rqinfo = rhs->quantization_info().uniform(); - const UniformQuantizationInfo dqinfo = output->quantization_info().uniform(); + const UniformQuantizationInfo dqinfo = dst->quantization_info().uniform(); float multiplier = lqinfo.scale * rqinfo.scale / dqinfo.scale; int output_multiplier = 0; @@ -162,9 +181,22 @@ void ClMatMulLowpNativeKernel::configure(const ClCompileContext &compile_context build_opts.add_option("-DDST_MULTIPLIER=" + support::cpp11::to_string(output_multiplier)); build_opts.add_option("-DDST_SHIFT=" + support::cpp11::to_string(output_shift)); - build_opts.add_option("-DLHS_OFFSET=" + support::cpp11::to_string(-lqinfo.offset)); // Note this is passed as negative to maintain similarity with CLDirectConv2D - build_opts.add_option("-DRHS_OFFSET=" + support::cpp11::to_string(-rqinfo.offset)); // Note this is passed as negative to maintain similarity with CLDirectConv2D - build_opts.add_option("-DDST_OFFSET=" + support::cpp11::to_string(dqinfo.offset)); // Passed as positive (unlike the above two) + // Note : Offset is not negated, unlike gemmlowp kernels + build_opts.add_option("-DLHS_OFFSET=" + support::cpp11::to_string(lqinfo.offset)); + build_opts.add_option("-DRHS_OFFSET=" + support::cpp11::to_string(rqinfo.offset)); + build_opts.add_option("-DDST_OFFSET=" + support::cpp11::to_string(dqinfo.offset)); + build_opts.add_option_if(bias != nullptr, "-DBIAS"); + + // Floating point boundaries are quantized prior to being passed as arguments. + // Note: We expect the input and output tensors to always adopt a per-tensor quantization approach + int a_val{}; + int b_val{}; + std::tie(b_val, a_val) = get_quantized_activation_min_max(act_info, dst->data_type(), dqinfo); + + build_opts.add_option("-DA_VAL=" + support::cpp11::to_string(a_val)); + build_opts.add_option("-DB_VAL=" + support::cpp11::to_string(b_val)); + build_opts.add_option("-DACTIVATION_TYPE=" + lower_string(string_from_activation_func(act_info.activation()))); + build_opts.add_option("-DZERO_POINT=" + support::cpp11::to_string(dqinfo.offset)); std::string kernel_name("mat_mul_native_quantized"); kernel_name += matmul_kernel_info.adj_lhs ? "_t" : "_nt"; @@ -177,7 +209,7 @@ void ClMatMulLowpNativeKernel::configure(const ClCompileContext &compile_context _kernel = create_kernel(compile_context, kernel_name, build_opts.options()); // Set config_id for enabling LWS tuning - const size_t number_of_batches = output->tensor_shape().total_size() / (m * n); + const size_t number_of_batches = dst->tensor_shape().total_size() / (m * n); _config_id = kernel_name; _config_id += "_"; @@ -203,18 +235,23 @@ void ClMatMulLowpNativeKernel::run_op(ITensorPack &tensors, const Window &window ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this); ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(ICLKernel::window(), window); - const ICLTensor *lhs = utils::cast::polymorphic_downcast(tensors.get_const_tensor(TensorType::ACL_SRC_0)); - const ICLTensor *rhs = utils::cast::polymorphic_downcast(tensors.get_const_tensor(TensorType::ACL_SRC_1)); - ICLTensor *output = utils::cast::polymorphic_downcast(tensors.get_tensor(TensorType::ACL_DST)); - ARM_COMPUTE_ERROR_ON_NULLPTR(lhs, rhs, output); - ARM_COMPUTE_LOG_PARAMS(lhs, rhs, output); + const ICLTensor *lhs = utils::cast::polymorphic_downcast(tensors.get_const_tensor(TensorType::ACL_SRC_0)); + const ICLTensor *rhs = utils::cast::polymorphic_downcast(tensors.get_const_tensor(TensorType::ACL_SRC_1)); + const ICLTensor *bias = utils::cast::polymorphic_downcast(tensors.get_const_tensor(TensorType::ACL_SRC_2)); + ICLTensor *dst = utils::cast::polymorphic_downcast(tensors.get_tensor(TensorType::ACL_DST)); + ARM_COMPUTE_ERROR_ON_NULLPTR(lhs, rhs, dst); + ARM_COMPUTE_LOG_PARAMS(lhs, rhs, bias, dst); unsigned int idx = 0; Window window_collapsed = window.collapse(ICLKernel::window(), Window::DimZ); add_3d_tensor_nhw_argument(idx, lhs); add_3d_tensor_nhw_argument(idx, rhs); - add_3d_tensor_nhw_argument(idx, output); + if(bias != nullptr) + { + add_3d_tensor_nhw_argument(idx, bias); + } + add_3d_tensor_nhw_argument(idx, dst); enqueue(queue, *this, window_collapsed, lws_hint()); } diff --git a/src/gpu/cl/kernels/ClMatMulLowpNativeKernel.h b/src/gpu/cl/kernels/ClMatMulLowpNativeKernel.h index d70ff30b91b6ff9577ab0111b3fe1f7321352b32..64415f42f7bd345693be8e674c68e0725a682075 100644 --- a/src/gpu/cl/kernels/ClMatMulLowpNativeKernel.h +++ b/src/gpu/cl/kernels/ClMatMulLowpNativeKernel.h @@ -24,6 +24,7 @@ #ifndef ACL_SRC_GPU_CL_KERNELS_CLMATMULLOWPNATIVEKERNEL #define ACL_SRC_GPU_CL_KERNELS_CLMATMULLOWPNATIVEKERNEL +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/core/common/Macros.h" #include "src/gpu/cl/ClCompileContext.h" #include "src/gpu/cl/IClKernel.h" @@ -43,22 +44,26 @@ public: ARM_COMPUTE_DISALLOW_COPY_ALLOW_MOVE(ClMatMulLowpNativeKernel); /** Initialise the kernel's input and output. * - * @param[in] compile_context The compile context to be used. - * @param[in] lhs Input tensor for the LHS matrix. Data type supported: QASYMM8_SIGNED/QASYMM8. - * Dimensions above 2 are collapsed onto dimension 2 and represent the batch. - * @param[in] rhs Input tensor for the RHS matrix. Data type supported: same as @p lhs. - * Dimensions above 2 are collapsed onto dimension 2 and represent the batch. - * @param[out] dst Output tensor info. Data type supported: same as @p lhs - * @param[in] matmul_info Attributes for Batch MatMul Kernel + * @param[in] compile_context The compile context to be used. + * @param[in] lhs Input tensor info for the LHS matrix. Data type supported: QASYMM8_SIGNED/QASYMM8. + * Dimensions above 2 are collapsed onto dimension 2 and represent the batch. + * @param[in] rhs Input tensor info for the RHS matrix. Data type supported: same as @p lhs. + * Dimensions above 2 are collapsed onto dimension 2 and represent the batch. + * @param[in] bias Bias tensor info. Can be nullptr. Data type supported: S32. + * @param[out] dst Output tensor info. Data type supported: same as @p lhs + * @param[in] matmul_kernel_info Attributes for Batch MatMul Kernel + * @param[in] act_info (Optional) Class containing information about fused activation function. */ - void configure(const ClCompileContext &compile_context, ITensorInfo *lhs, ITensorInfo *rhs, ITensorInfo *dst, const MatMulKernelInfo &matmul_info); + void configure(const ClCompileContext &compile_context, ITensorInfo *lhs, ITensorInfo *rhs, ITensorInfo *bias, ITensorInfo *dst, const MatMulKernelInfo &matmul_kernel_info, + const ActivationLayerInfo &act_info = ActivationLayerInfo()); /** Static function to check if given info will lead to a valid configuration * * Similar to @ref ClMatMulLowpNativeKernel::configure() * * @return a status */ - static Status validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *dst, const MatMulKernelInfo &matmul_info); + static Status validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *bias, const ITensorInfo *dst, const MatMulKernelInfo &matmul_kernel_info, + const ActivationLayerInfo &act_info = ActivationLayerInfo()); // Inherited methods overridden: void run_op(ITensorPack &tensors, const Window &window, cl::CommandQueue &queue) override; diff --git a/src/gpu/cl/kernels/ClMatMulNativeKernel.cpp b/src/gpu/cl/kernels/ClMatMulNativeKernel.cpp index 8f53c1998f949c3c049474e715bf0a81aca34cf3..8f8ccfc41f39ca5858278f50fae8245e6ac2c4c3 100644 --- a/src/gpu/cl/kernels/ClMatMulNativeKernel.cpp +++ b/src/gpu/cl/kernels/ClMatMulNativeKernel.cpp @@ -23,11 +23,14 @@ */ #include "src/gpu/cl/kernels/ClMatMulNativeKernel.h" +#include "arm_compute/core/utils/ActivationFunctionUtils.h" #include "arm_compute/core/CL/CLHelpers.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/ITensorPack.h" #include "arm_compute/core/TensorInfo.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/common/utils/Log.h" #include "src/core/CL/CLUtils.h" @@ -112,7 +115,7 @@ Status validate_export_to_cl_image(const ITensorInfo *rhs, const MatMulKernelInf ARM_COMPUTE_RETURN_ERROR_ON_MSG(!export_to_cl_image(rhs), "Export to CLImage is not supported for this device/configuration"); } - return Status {}; + return Status{}; } } ClMatMulNativeKernel::ClMatMulNativeKernel() @@ -120,8 +123,10 @@ ClMatMulNativeKernel::ClMatMulNativeKernel() _type = CLKernelType::GEMM; } -Status ClMatMulNativeKernel::validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *dst, const MatMulKernelInfo &matmul_kernel_info) +Status ClMatMulNativeKernel::validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *bias, const ITensorInfo *dst, const MatMulKernelInfo &matmul_kernel_info, + const ActivationLayerInfo &act_info) { + ARM_COMPUTE_UNUSED(act_info); ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(lhs, rhs, dst); ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(lhs, 1, DataType::F32, DataType::F16); ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(lhs, rhs); @@ -129,20 +134,30 @@ Status ClMatMulNativeKernel::validate(const ITensorInfo *lhs, const ITensorInfo ARM_COMPUTE_RETURN_ON_ERROR(validate_input_shapes(lhs->tensor_shape(), rhs->tensor_shape(), matmul_kernel_info)); ARM_COMPUTE_RETURN_ON_ERROR(validate_export_to_cl_image(rhs, matmul_kernel_info)); + const TensorShape expected_output_shape = misc::shape_calculator::compute_matmul_shape(lhs->tensor_shape(), rhs->tensor_shape(), matmul_kernel_info); + if(dst->total_size() != 0) { - const TensorInfo tensor_info_dst = dst->clone()->set_tensor_shape(misc::shape_calculator::compute_matmul_shape(lhs->tensor_shape(), rhs->tensor_shape(), matmul_kernel_info)); + const TensorInfo tensor_info_dst = dst->clone()->set_tensor_shape(expected_output_shape); ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(dst, &tensor_info_dst); ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(lhs, dst); } + if(bias != nullptr) + { + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(bias, lhs); + ARM_COMPUTE_RETURN_ERROR_ON_MSG((bias->num_dimensions() > 1), "Multi dimensional bias is unsupported."); + ARM_COMPUTE_RETURN_ERROR_ON_MSG(bias->dimension(0) != expected_output_shape[0], "First dimension of bias and output tensors must match."); + } + return Status{}; } -void ClMatMulNativeKernel::configure(const ClCompileContext &compile_context, ITensorInfo *lhs, ITensorInfo *rhs, ITensorInfo *dst, const MatMulKernelInfo &matmul_kernel_info) +void ClMatMulNativeKernel::configure(const ClCompileContext &compile_context, ITensorInfo *lhs, ITensorInfo *rhs, ITensorInfo *bias, ITensorInfo *dst, const MatMulKernelInfo &matmul_kernel_info, + const ActivationLayerInfo &act_info) { ARM_COMPUTE_ERROR_ON_NULLPTR(lhs, rhs, dst, &compile_context, &matmul_kernel_info); - ARM_COMPUTE_LOG_PARAMS(lhs, rhs, dst, matmul_kernel_info); - ARM_COMPUTE_ERROR_THROW_ON(validate(lhs, rhs, dst, matmul_kernel_info)); + ARM_COMPUTE_LOG_PARAMS(lhs, rhs, bias, dst, matmul_kernel_info); + ARM_COMPUTE_ERROR_THROW_ON(validate(lhs, rhs, bias, dst, matmul_kernel_info)); // dst tensor auto initialization if not yet initialized auto_init_if_empty(*dst, lhs->clone()->set_tensor_shape(misc::shape_calculator::compute_matmul_shape(lhs->tensor_shape(), rhs->tensor_shape(), matmul_kernel_info))); @@ -174,8 +189,14 @@ void ClMatMulNativeKernel::configure(const ClCompileContext &compile_context, IT build_opts.add_option("-DPARTIAL_STORE_M0=" + support::cpp11::to_string(partial_store_m0)); build_opts.add_option("-DPARTIAL_STORE_N0=" + support::cpp11::to_string(partial_store_n0)); build_opts.add_option("-DK=" + support::cpp11::to_string(k)); + build_opts.add_option_if(bias != nullptr, "-DBIAS"); build_opts.add_option_if_else(_export_rhs_to_cl_image, "-DRHS_TENSOR_TYPE=IMAGE", "-DRHS_TENSOR_TYPE=BUFFER"); + // Define values for activation function + build_opts.add_option(("-DA_VAL=" + float_to_string_with_full_precision(act_info.a()))); + build_opts.add_option(("-DB_VAL=" + float_to_string_with_full_precision(act_info.b()))); + build_opts.add_option("-DACTIVATION_TYPE=" + lower_string(string_from_activation_func(act_info.activation()))); + std::string kernel_name("mat_mul_native"); kernel_name += matmul_kernel_info.adj_lhs ? "_t" : "_nt"; kernel_name += matmul_kernel_info.adj_rhs ? "_t" : "_nt"; @@ -218,11 +239,12 @@ void ClMatMulNativeKernel::run_op(ITensorPack &tensors, const Window &window, cl ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this); ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(ICLKernel::window(), window); - const ICLTensor *lhs = utils::cast::polymorphic_downcast(tensors.get_const_tensor(TensorType::ACL_SRC_0)); - const ICLTensor *rhs = utils::cast::polymorphic_downcast(tensors.get_const_tensor(TensorType::ACL_SRC_1)); - ICLTensor *dst = utils::cast::polymorphic_downcast(tensors.get_tensor(TensorType::ACL_DST)); + const ICLTensor *lhs = utils::cast::polymorphic_downcast(tensors.get_const_tensor(TensorType::ACL_SRC_0)); + const ICLTensor *rhs = utils::cast::polymorphic_downcast(tensors.get_const_tensor(TensorType::ACL_SRC_1)); + const ICLTensor *bias = utils::cast::polymorphic_downcast(tensors.get_const_tensor(TensorType::ACL_SRC_2)); // nullptr if bias is not present + ICLTensor *dst = utils::cast::polymorphic_downcast(tensors.get_tensor(TensorType::ACL_DST)); ARM_COMPUTE_ERROR_ON_NULLPTR(lhs, rhs, dst); - ARM_COMPUTE_LOG_PARAMS(lhs, rhs, dst); + ARM_COMPUTE_LOG_PARAMS(lhs, rhs, bias, dst); unsigned int idx = 0; Window window_collapsed = window.collapse(ICLKernel::window(), Window::DimZ); @@ -243,6 +265,10 @@ void ClMatMulNativeKernel::run_op(ITensorPack &tensors, const Window &window, cl } add_3d_tensor_nhw_argument(idx, rhs); + if(bias != nullptr) + { + add_3d_tensor_nhw_argument(idx, bias); + } add_3d_tensor_nhw_argument(idx, dst); enqueue(queue, *this, window_collapsed, lws_hint()); diff --git a/src/gpu/cl/kernels/ClMatMulNativeKernel.h b/src/gpu/cl/kernels/ClMatMulNativeKernel.h index f706256e31aa792c9b3ac58c4dd8defb9414a6ed..fe2b787c1208b383f2b576257e9d7fc45386cdbe 100644 --- a/src/gpu/cl/kernels/ClMatMulNativeKernel.h +++ b/src/gpu/cl/kernels/ClMatMulNativeKernel.h @@ -42,22 +42,26 @@ public: ARM_COMPUTE_DISALLOW_COPY_ALLOW_MOVE(ClMatMulNativeKernel); /** Initialise the kernel's input and output. * - * @param[in] compile_context The compile context to be used. - * @param[in] lhs Input tensor for the LHS matrix. Data type supported: F32/F16. - * Dimensions above 2 are collapsed onto dimension 2 and represent the batch. - * @param[in] rhs Input tensor for the RHS matrix. Data type supported: same as @p lhs. - * Dimensions above 2 are collapsed onto dimension 2 and represent the batch. - * @param[out] dst Output tensor info. Data type supported: same as @p lhs - * @param[in] matmul_info Attributes for Batch MatMul Kernel + * @param[in] compile_context The compile context to be used. + * @param[in] lhs Input tensor info for the LHS matrix. Data type supported: F32/F16. + * Dimensions above 2 are collapsed onto dimension 2 and represent the batch. + * @param[in] rhs Input tensor info for the RHS matrix. Data type supported: same as @p lhs. + * Dimensions above 2 are collapsed onto dimension 2 and represent the batch. + * @param[in] bias Bias tensor info for bias matrix. Can be nullptr. Data type supported: same as @p lhs. + * @param[out] dst Output tensor info. Data type supported: same as @p lhs + * @param[in] matmul_kernel_info Attributes for Batch MatMul Kernel + * @param[in] act_info (Optional) Specifies activation function to use after Matrix multiplication. Default is Identity function. */ - void configure(const ClCompileContext &compile_context, ITensorInfo *lhs, ITensorInfo *rhs, ITensorInfo *dst, const MatMulKernelInfo &matmul_info); + void configure(const ClCompileContext &compile_context, ITensorInfo *lhs, ITensorInfo *rhs, ITensorInfo *bias, ITensorInfo *dst, const MatMulKernelInfo &matmul_kernel_info, + const ActivationLayerInfo &act_info = ActivationLayerInfo()); /** Static function to check if given info will lead to a valid configuration * * Similar to @ref ClMatMulNativeKernel::configure() * * @return a status */ - static Status validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *dst, const MatMulKernelInfo &matmul_info); + static Status validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *bias, const ITensorInfo *dst, const MatMulKernelInfo &matmul_kernel_info, + const ActivationLayerInfo &act_info = ActivationLayerInfo()); // Inherited methods overridden: void run_op(ITensorPack &tensors, const Window &window, cl::CommandQueue &queue) override; diff --git a/src/gpu/cl/kernels/ClMatMulNativeMMULKernel.cpp b/src/gpu/cl/kernels/ClMatMulNativeMMULKernel.cpp new file mode 100644 index 0000000000000000000000000000000000000000..44c720a40c8015af2f95f1862ee77c14730d7b4c --- /dev/null +++ b/src/gpu/cl/kernels/ClMatMulNativeMMULKernel.cpp @@ -0,0 +1,282 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "src/gpu/cl/kernels/ClMatMulNativeMMULKernel.h" + +#include "arm_compute/core/CL/CLHelpers.h" +#include "arm_compute/core/CL/ICLTensor.h" +#include "arm_compute/core/ITensorPack.h" +#include "arm_compute/core/KernelDescriptors.h" +#include "arm_compute/core/TensorInfo.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" + +#include "src/common/utils/Log.h" +#include "src/core/helpers/AutoConfiguration.h" +#include "src/core/helpers/WindowHelpers.h" + +#include "support/Cast.h" +#include "support/StringSupport.h" + +namespace arm_compute +{ +namespace opencl +{ +namespace kernels +{ +namespace +{ +// Block size dimensions for the MMUL extension +constexpr int mmul_m0 = 4; +constexpr int mmul_n0 = 4; +constexpr int mmul_k0 = 4; + +inline std::pair adjust_m0_n0(int m0, int n0, int m, int n) +{ + m0 = std::min(m0, m); + n0 = adjust_vec_size(n0, n); + return { m0, n0 }; +} + +Status validate_matmul_kernel_info(const MatMulKernelInfo &matmul_kernel_info) +{ + const bool adj_lhs = matmul_kernel_info.adj_lhs; + const int m0 = matmul_kernel_info.m0; + const int n0 = matmul_kernel_info.n0; + const int k0 = matmul_kernel_info.k0; + + // Validate M0 + ARM_COMPUTE_RETURN_ERROR_ON_MSG(m0 < 1, "Only positive integers are supported for M0"); + + if(adj_lhs) + { + ARM_COMPUTE_RETURN_ERROR_ON_MSG((m0 != 1) && (m0 != 2) && (m0 != 3) && (m0 != 4) && (m0 != 8) && (m0 != 16), "Only 1,2,3,4,8,16 are supported for M0 for Lhs transposed"); + } + + // Validate N0 + ARM_COMPUTE_RETURN_ERROR_ON_MSG(n0 < 1, "Only positive integers are supported for N0"); + ARM_COMPUTE_RETURN_ERROR_ON_MSG((n0 != 1) && (n0 != 2) && (n0 != 3) && (n0 != 4) && (n0 != 8) && (n0 != 16), "Only 1,2,3,4,8,16 are supported for N0"); + + // Validate K0 + ARM_COMPUTE_RETURN_ERROR_ON_MSG((k0 != 1), "Only 1 is supported for k0"); + + return Status{}; +} + +Status validate_input_shapes(const TensorShape &lhs_shape, const TensorShape &rhs_shape, const MatMulKernelInfo &matmul_kernel_info) +{ + const size_t lhs_k = matmul_kernel_info.adj_lhs ? lhs_shape.y() : lhs_shape.x(); + const size_t rhs_k = matmul_kernel_info.adj_rhs ? rhs_shape.x() : rhs_shape.y(); + + ARM_COMPUTE_RETURN_ERROR_ON_MSG(lhs_k != rhs_k, "K dimension in Lhs and Rhs matrices must match."); + ARM_COMPUTE_RETURN_ERROR_ON_MSG_VAR((lhs_k % mmul_k0) != 0, "K dimension must be a multiple of %d", mmul_k0); + ARM_COMPUTE_RETURN_ERROR_ON_MSG(lhs_shape.total_size() == 0, "Lhs tensor can't be empty"); + ARM_COMPUTE_RETURN_ERROR_ON_MSG(rhs_shape.total_size() == 0, "Rhs tensor can't be empty"); + + constexpr size_t batch_dim_start = 2; + for(size_t i = batch_dim_start; i < Coordinates::num_max_dimensions; ++i) + { + ARM_COMPUTE_RETURN_ERROR_ON_MSG(lhs_shape[i] != rhs_shape[i], "Batch dimension broadcasting is not supported"); + } + + return Status{}; +} + +std::pair validate_and_configure_window(ITensorInfo *lhs, ITensorInfo *rhs, ITensorInfo *dst, const MatMulKernelInfo &matmul_kernel_info) +{ + ARM_COMPUTE_UNUSED(lhs, rhs); + + const Window win = calculate_max_window(*dst, Steps(1, 1)); + + // Collapse along the Z direction + // This collapse needs to be here in order to tune the Z dimension of LWS + Window collapsed = win.collapse(win, Window::DimZ); + + // Reconfigure window size, one arm_matrix_multiply call needs 16 threads to finish. + Window::Dimension x_dimension = collapsed.x(); + Window::Dimension y_dimension = collapsed.y(); + + const int m = dst->dimension(1); + const int n = dst->dimension(0); + + int m0{}; + int n0{}; + std::tie(m0, n0) = adjust_m0_n0(matmul_kernel_info.m0, matmul_kernel_info.n0, m, n); + + // Make M and N multiple of M0 and N0 respectively + const unsigned int ceil_to_multiple_n_n0 = ceil_to_multiple(n, n0); + const unsigned int ceil_to_multiple_m_m0 = ceil_to_multiple(m, m0); + + // Divide M and N by M0 and N0 respectively + const unsigned int n_div_n0 = ceil_to_multiple_n_n0 / n0; + const unsigned int m_div_m0 = ceil_to_multiple_m_m0 / m0; + + // Make n_div_n0 and m_div_m0 multiple of mmul_n0 and mmul_m0 respectively + const unsigned int ceil_to_multiple_n_div_n0_mmul_n0 = ceil_to_multiple(n_div_n0, mmul_n0); + const unsigned int ceil_to_multiple_m_div_m0_mmul_m0 = ceil_to_multiple(m_div_m0, mmul_m0); + + // Ensure x_dimension is multiple of MMUL block size (mmul_m0 * mmul_n0) + x_dimension.set_end(ceil_to_multiple_n_div_n0_mmul_n0 * mmul_m0); + y_dimension.set_end(ceil_to_multiple_m_div_m0_mmul_m0 / mmul_m0); + + collapsed.set(Window::DimX, x_dimension); + collapsed.set(Window::DimY, y_dimension); + + return std::make_pair(Status{}, collapsed); +} +} +ClMatMulNativeMMULKernel::ClMatMulNativeMMULKernel() +{ + _type = CLKernelType::GEMM; +} + +Status ClMatMulNativeMMULKernel::validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *bias, const ITensorInfo *dst, const MatMulKernelInfo &matmul_kernel_info) +{ + ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(lhs, rhs, dst); + ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(lhs, 1, DataType::F32, DataType::F16); + ARM_COMPUTE_RETURN_ERROR_ON_MSG(!arm_matrix_multiply_supported(CLKernelLibrary::get().get_device()), "The extension cl_arm_matrix_multiply is not supported on the target platform"); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(lhs, rhs); + ARM_COMPUTE_RETURN_ON_ERROR(validate_matmul_kernel_info(matmul_kernel_info)); + ARM_COMPUTE_RETURN_ON_ERROR(validate_input_shapes(lhs->tensor_shape(), rhs->tensor_shape(), matmul_kernel_info)); + + const TensorShape expected_output_shape = misc::shape_calculator::compute_matmul_shape(lhs->tensor_shape(), rhs->tensor_shape(), matmul_kernel_info); + + if(dst->total_size() != 0) + { + const TensorInfo tensor_info_dst = dst->clone()->set_tensor_shape(expected_output_shape); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(dst, &tensor_info_dst); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(lhs, dst); + } + + if(bias != nullptr) + { + ARM_COMPUTE_RETURN_ERROR_ON_MSG((bias->num_dimensions() > 1), "Multi dimensional bias is unsupported."); + ARM_COMPUTE_RETURN_ERROR_ON_MSG(bias->dimension(0) != expected_output_shape[0], "First dimension of bias and output tensors must match."); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(lhs, bias); + } + + return Status{}; +} +void ClMatMulNativeMMULKernel::configure(const ClCompileContext &compile_context, ITensorInfo *lhs, ITensorInfo *rhs, ITensorInfo *bias, ITensorInfo *dst, const MatMulKernelInfo &matmul_kernel_info) +{ + ARM_COMPUTE_ERROR_ON_NULLPTR(lhs, rhs, dst); + ARM_COMPUTE_LOG_PARAMS(lhs, rhs, bias, dst, matmul_kernel_info); + ARM_COMPUTE_ERROR_THROW_ON(validate(lhs, rhs, bias, dst, matmul_kernel_info)); + + // dst tensor auto initialization if not yet initialized + auto_init_if_empty(*dst, lhs->clone()->set_tensor_shape(misc::shape_calculator::compute_matmul_shape(lhs->tensor_shape(), rhs->tensor_shape(), matmul_kernel_info))); + + const int m = dst->dimension(1); + const int n = dst->dimension(0); + const int k = matmul_kernel_info.adj_lhs ? lhs->tensor_shape().y() : lhs->tensor_shape().x(); + + _m = m; + _n = n; + _k = k; + + int m0{}; + int n0{}; + std::tie(m0, n0) = adjust_m0_n0(matmul_kernel_info.m0, matmul_kernel_info.n0, m, n); + + // Configure kernel window + const auto win_config = validate_and_configure_window(lhs, rhs, dst, matmul_kernel_info); + ARM_COMPUTE_ERROR_THROW_ON(win_config.first); + IClKernel::configure_internal(win_config.second); + + // Calculate partial (store instead of load) M0 and partial N0 for the partial blocks at the end of a row/column if any. This is to avoid padding. + const unsigned int m0_leftover = m % m0; + const unsigned int n0_leftover = n % n0; + + CLBuildOptions build_opts; + build_opts.add_option("-DDATA_TYPE=" + get_cl_type_from_data_type(lhs->data_type())); + build_opts.add_option_if(lhs->data_type() == DataType::F16, "-DHALF_PRECISION"); + build_opts.add_option("-DM0=" + support::cpp11::to_string(m0)); + build_opts.add_option("-DN0=" + support::cpp11::to_string(n0)); + build_opts.add_option("-DM0_LEFTOVER=" + support::cpp11::to_string(m0_leftover)); + build_opts.add_option("-DN0_LEFTOVER=" + support::cpp11::to_string(n0_leftover)); + build_opts.add_option("-DMMUL_M0=" + support::cpp11::to_string(mmul_m0)); + build_opts.add_option("-DMMUL_N0=" + support::cpp11::to_string(mmul_n0)); + build_opts.add_option("-DMMUL_K0=" + support::cpp11::to_string(mmul_k0)); + build_opts.add_option_if(bias != nullptr, "-DBIAS"); + + std::string kernel_name("mat_mul_native_mmul"); + kernel_name += matmul_kernel_info.adj_lhs ? "_t" : "_nt"; + kernel_name += matmul_kernel_info.adj_rhs ? "_t" : "_nt"; + + // A macro guard to compile ONLY the kernel of interest + build_opts.add_option("-D" + upper_string(kernel_name)); + + // Create kernel + _kernel = create_kernel(compile_context, kernel_name, build_opts.options()); + + // Set config_id for enabling LWS tuning + _config_id = kernel_name; + _config_id += "_"; + _config_id += lower_string(string_from_data_type(lhs->data_type())); + _config_id += "_"; + _config_id += support::cpp11::to_string(k); + _config_id += "_"; + _config_id += support::cpp11::to_string(dst->dimension(2)); + _config_id += "_"; + _config_id += support::cpp11::to_string(m0); + _config_id += "_"; + _config_id += support::cpp11::to_string(n0); + _config_id += "_"; + _config_id += support::cpp11::to_string(matmul_kernel_info.k0); +} + +void ClMatMulNativeMMULKernel::run_op(ITensorPack &tensors, const Window &window, cl::CommandQueue &queue) +{ + ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this); + ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(ICLKernel::window(), window); + + const ICLTensor *lhs = utils::cast::polymorphic_downcast(tensors.get_const_tensor(TensorType::ACL_SRC_0)); + const ICLTensor *rhs = utils::cast::polymorphic_downcast(tensors.get_const_tensor(TensorType::ACL_SRC_1)); + const ICLTensor *bias = utils::cast::polymorphic_downcast(tensors.get_const_tensor(TensorType::ACL_SRC_2)); // nullptr if bias is not present + ICLTensor *dst = utils::cast::polymorphic_downcast(tensors.get_tensor(TensorType::ACL_DST)); + ARM_COMPUTE_ERROR_ON_NULLPTR(lhs, rhs, dst); + ARM_COMPUTE_LOG_PARAMS(lhs, rhs, bias, dst); + unsigned int idx = 0; + + add_3d_tensor_nhw_argument(idx, lhs); + add_3d_tensor_nhw_argument(idx, rhs); + if(bias != nullptr) + { + add_3d_tensor_nhw_argument(idx, bias); + } + add_3d_tensor_nhw_argument(idx, dst); + + // Pass m and n at runtime as signed ints, to ensure results of any subtractions they could be operand in, would still be signed. + _kernel.setArg(idx++, _m); + _kernel.setArg(idx++, _n); + _kernel.setArg(idx++, _k); + + // LWS_x should be multiple of 16 at least. (32, 2) has been chosen to have more work-items on a single core + // LWS also enforces the order of execution of the work items which improves cache utilization + enqueue(queue, *this, window, cl::NDRange(32, 2), false); +} + +} // namespace kernels +} // namespace opencl +} // namespace arm_compute diff --git a/src/gpu/cl/kernels/ClMatMulNativeMMULKernel.h b/src/gpu/cl/kernels/ClMatMulNativeMMULKernel.h new file mode 100644 index 0000000000000000000000000000000000000000..80448974c413cbba0da65f86e230c0f732cb1ce2 --- /dev/null +++ b/src/gpu/cl/kernels/ClMatMulNativeMMULKernel.h @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_SRC_GPU_CL_KERNELS_CLMATMULNATIVEMMULKERNEL +#define ACL_SRC_GPU_CL_KERNELS_CLMATMULNATIVEMMULKERNEL + +#include "src/core/common/Macros.h" +#include "src/gpu/cl/ClCompileContext.h" +#include "src/gpu/cl/IClKernel.h" + +namespace arm_compute +{ +struct MatMulKernelInfo; +namespace opencl +{ +namespace kernels +{ +class ClMatMulNativeMMULKernel : public IClKernel +{ +public: + ClMatMulNativeMMULKernel(); + ARM_COMPUTE_DISALLOW_COPY_ALLOW_MOVE(ClMatMulNativeMMULKernel); + /** Initialize the kernel's input and output. + * + * This kernel performs matrix multiplication of lhs and rhs: + * + * dst = matmul(lhs, rhs) + * + * Valid data layouts: + * - All + * + * Valid data type configurations: + * |lhs |rhs |dst | + * |:--------------|:--------------|:--------------| + * |F32 |F32 |F32 | + * |F16 |F16 |F16 | + * + * Shape definitions: + * Dim0, Dim1, Dim2... + * lhs: [ K, M, Batch dims...] + * rhs: [ N, K, Batch dims...] + * dst: [ N, M, Batch dims...] + * + * Valid shape configurations: + * - K must be a multiple of 4 (MMUL_K0). + * - No broadcasting in batch dimensions. I.e. batch dims must be the same across lhs, rhs and dst + * + * @param[in] compile_context The compile context to be used. + * @param[in] lhs Input tensor info for the LHS matrix. + * @param[in] rhs Input tensor info for the RHS matrix. + * @param[in] bias Bias tensor info. Can be nullptr. Data type supported: Same as @p lhs. + * @param[out] dst Output tensor info. + * @param[in] matmul_info Attributes for Batch MatMul Kernel + */ + void configure(const ClCompileContext &compile_context, ITensorInfo *lhs, ITensorInfo *rhs, ITensorInfo *bias, ITensorInfo *dst, const MatMulKernelInfo &matmul_info); + /** Static function to check if given info will lead to a valid configuration + * + * Similar to @ref ClMatMulNativeMMULKernel::configure() + * + * @return a status + */ + static Status validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *bias, const ITensorInfo *dst, const MatMulKernelInfo &matmul_info); + + // Inherited methods overridden: + void run_op(ITensorPack &tensors, const Window &window, cl::CommandQueue &queue) override; + +private: + int _m{ 1 }; + int _n{ 1 }; + int _k{ 1 }; +}; +} // namespace kernels +} // namespace opencl +} // namespace arm_compute +#endif /* ACL_SRC_GPU_CL_KERNELS_CLMATMULNATIVEMMULKERNEL */ diff --git a/src/gpu/cl/kernels/ClMulKernel.cpp b/src/gpu/cl/kernels/ClMulKernel.cpp index 0bf1932085f5a453c65d0856ab9d058fa029e5fd..5ca0639852f0aa68504abb81dbe91ea0d2a9d972 100644 --- a/src/gpu/cl/kernels/ClMulKernel.cpp +++ b/src/gpu/cl/kernels/ClMulKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021 Arm Limited. + * Copyright (c) 2016-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,11 +23,15 @@ */ #include "src/gpu/cl/kernels/ClMulKernel.h" +#include "arm_compute/core/utils/ActivationFunctionUtils.h" #include "arm_compute/core/CL/CLHelpers.h" #include "arm_compute/core/CL/CLKernelLibrary.h" #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/CL/OpenCL.h" #include "arm_compute/core/TensorInfo.h" +#include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/gpu/cl/kernels/ClMulKernel.h b/src/gpu/cl/kernels/ClMulKernel.h index 41c862eb0382ff98fb4e6b4167daceb1253c9eeb..4e62a6d67a61795e0c29c6185b61b41e6d1e87ee 100644 --- a/src/gpu/cl/kernels/ClMulKernel.h +++ b/src/gpu/cl/kernels/ClMulKernel.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021 Arm Limited. + * Copyright (c) 2016-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CL_MUL_KERNEL_H #define ARM_COMPUTE_CL_MUL_KERNEL_H +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/core/common/Macros.h" #include "src/gpu/cl/ClCompileContext.h" #include "src/gpu/cl/IClKernel.h" diff --git a/src/gpu/cl/kernels/ClPool2dKernel.cpp b/src/gpu/cl/kernels/ClPool2dKernel.cpp index 83bc6bb442ca1389613a2156b9cb5c5596234b88..a1afc585e0d952e6ccf2219a87e95020dc8602d5 100644 --- a/src/gpu/cl/kernels/ClPool2dKernel.cpp +++ b/src/gpu/cl/kernels/ClPool2dKernel.cpp @@ -25,7 +25,9 @@ #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/TensorInfo.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/gpu/cl/kernels/ClPool3dKernel.cpp b/src/gpu/cl/kernels/ClPool3dKernel.cpp index a090ac57748fb4f48476eb5df5ff2968eb66e272..d068832fedd9a890dbec6d0ea693e4ece1bd80c5 100644 --- a/src/gpu/cl/kernels/ClPool3dKernel.cpp +++ b/src/gpu/cl/kernels/ClPool3dKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,7 +25,9 @@ #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/TensorInfo.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/gpu/cl/kernels/ClQuantizeKernel.cpp b/src/gpu/cl/kernels/ClQuantizeKernel.cpp index de222a99b28a8fb6ce3761c06cb8623c1b4a06cd..5c8bf97f0f00ed1e6ff95acfad6834e7e6f88b7f 100644 --- a/src/gpu/cl/kernels/ClQuantizeKernel.cpp +++ b/src/gpu/cl/kernels/ClQuantizeKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -29,6 +29,7 @@ #include "arm_compute/core/Error.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/StringUtils.h" #include "arm_compute/core/Validate.h" #include "arm_compute/core/utils/quantization/AsymmHelpers.h" diff --git a/src/gpu/cl/kernels/ClScaleKernel.cpp b/src/gpu/cl/kernels/ClScaleKernel.cpp index 910287194ec6d11d8d947e1deabdd3a72b0f0530..4c4373a215381d88616d3364e0676995609b4628 100644 --- a/src/gpu/cl/kernels/ClScaleKernel.cpp +++ b/src/gpu/cl/kernels/ClScaleKernel.cpp @@ -25,6 +25,10 @@ #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/TensorInfo.h" +#include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/StringUtils.h" +#include "arm_compute/core/utils/InterpolationPolicyUtils.h" #include "src/core/AccessWindowStatic.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/WindowHelpers.h" @@ -64,6 +68,7 @@ Status validate_arguments(const ITensorInfo *src, const ITensorInfo *dst, const ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(src, dst); ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_QUANTIZATION_INFO(src, dst); ARM_COMPUTE_RETURN_ERROR_ON(dst == src); + ARM_COMPUTE_RETURN_ERROR_ON(src->num_channels()!=1); ARM_COMPUTE_RETURN_ERROR_ON(info.align_corners && !arm_compute::scale_utils::is_align_corners_allowed_sampling_policy(info.sampling_policy)); ARM_COMPUTE_RETURN_ERROR_ON(is_data_type_quantized(src->data_type()) && !is_data_type_quantized_asymmetric(src->data_type())); diff --git a/src/gpu/cl/kernels/ClSoftmaxKernel.cpp b/src/gpu/cl/kernels/ClSoftmaxKernel.cpp index 467bbe46ce59898ad249457ecb08fc962cf10f2d..59299fa4415c5e53859988fd680bb45c77fae958 100644 --- a/src/gpu/cl/kernels/ClSoftmaxKernel.cpp +++ b/src/gpu/cl/kernels/ClSoftmaxKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,7 +25,9 @@ #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/Utils.h" #include "arm_compute/core/experimental/Types.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/utils/quantization/AsymmHelpers.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/gpu/cl/kernels/ClTransposeKernel.cpp b/src/gpu/cl/kernels/ClTransposeKernel.cpp index bdce2a2f5cdbbdeacfc2e609b105dd4d807be075..6450ffb5b22a39923b100b81e649f78db928bb68 100644 --- a/src/gpu/cl/kernels/ClTransposeKernel.cpp +++ b/src/gpu/cl/kernels/ClTransposeKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -30,6 +30,7 @@ #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Utils.h" #include "arm_compute/core/Validate.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" diff --git a/src/gpu/cl/kernels/ClTransposedConvolutionKernel.cpp b/src/gpu/cl/kernels/ClTransposedConvolutionKernel.cpp index 1390451ed84f69da6a3318adcd28c4d46564b304..ae825694c5dcfa6fc1c52fc201662c23c024e5a0 100644 --- a/src/gpu/cl/kernels/ClTransposedConvolutionKernel.cpp +++ b/src/gpu/cl/kernels/ClTransposedConvolutionKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,7 +24,9 @@ #include "src/gpu/cl/kernels/ClTransposedConvolutionKernel.h" #include "arm_compute/core/CL/ICLTensor.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" diff --git a/src/gpu/cl/kernels/ClWidthConcatenate2TensorsKernel.cpp b/src/gpu/cl/kernels/ClWidthConcatenate2TensorsKernel.cpp index b04a80a1e9d48df9f801df1e8aaf341b728065fd..0a9a3f021f5e1f3254732b0d7d02c9de0b4cd85c 100644 --- a/src/gpu/cl/kernels/ClWidthConcatenate2TensorsKernel.cpp +++ b/src/gpu/cl/kernels/ClWidthConcatenate2TensorsKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022 Arm Limited. + * Copyright (c) 2018-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,8 +26,9 @@ #include "arm_compute/core/CL/CLHelpers.h" #include "arm_compute/core/CL/CLKernelLibrary.h" #include "arm_compute/core/CL/ICLTensor.h" -#include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/WindowHelpers.h" #include "src/core/utils/helpers/tensor_info.h" diff --git a/src/gpu/cl/kernels/ClWidthConcatenate4TensorsKernel.cpp b/src/gpu/cl/kernels/ClWidthConcatenate4TensorsKernel.cpp index 741637795a8dbd96e32b9b236d04c6e6d2ab35da..54f7ad344ac91fb43e623dc2d9d24ce8f4e7fc99 100644 --- a/src/gpu/cl/kernels/ClWidthConcatenate4TensorsKernel.cpp +++ b/src/gpu/cl/kernels/ClWidthConcatenate4TensorsKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022 Arm Limited. + * Copyright (c) 2018-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -28,6 +28,8 @@ #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/WindowHelpers.h" #include "src/core/utils/helpers/tensor_info.h" diff --git a/src/gpu/cl/kernels/ClWidthConcatenateKernel.cpp b/src/gpu/cl/kernels/ClWidthConcatenateKernel.cpp index 7ed609f08dc7ae3e6d96cffb0ba018e12ec62d75..2dfe7fce5235454d592670b4f3488678f80a6f69 100644 --- a/src/gpu/cl/kernels/ClWidthConcatenateKernel.cpp +++ b/src/gpu/cl/kernels/ClWidthConcatenateKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022 Arm Limited. + * Copyright (c) 2018-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -28,6 +28,8 @@ #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/WindowHelpers.h" #include "support/Cast.h" diff --git a/src/gpu/cl/kernels/ClWinogradFilterTransformKernel.cpp b/src/gpu/cl/kernels/ClWinogradFilterTransformKernel.cpp index 136376a39f39e8f60c70bf4621f0b489bc0c8cfc..7148a4c85cb87d814e348e150ad7b32af9635a37 100644 --- a/src/gpu/cl/kernels/ClWinogradFilterTransformKernel.cpp +++ b/src/gpu/cl/kernels/ClWinogradFilterTransformKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022 Arm Limited. + * Copyright (c) 2018-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -29,7 +29,7 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Types.h" -#include "arm_compute/core/Utils.h" +#include "arm_compute/core/utils/StringUtils.h" #include "arm_compute/core/Validate.h" #include "arm_compute/core/Window.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" diff --git a/src/gpu/cl/kernels/ClWinogradInputTransformKernel.cpp b/src/gpu/cl/kernels/ClWinogradInputTransformKernel.cpp index 48d806dc7cce854c89e04b62416ecf794f84e8aa..fab6c36032144fa05044be5c6cd0fa49ed85e665 100644 --- a/src/gpu/cl/kernels/ClWinogradInputTransformKernel.cpp +++ b/src/gpu/cl/kernels/ClWinogradInputTransformKernel.cpp @@ -30,8 +30,8 @@ #include "arm_compute/core/Error.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/Types.h" -#include "arm_compute/core/Utils.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/AccessWindowStatic.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" diff --git a/src/gpu/cl/kernels/ClWinogradOutputTransformKernel.cpp b/src/gpu/cl/kernels/ClWinogradOutputTransformKernel.cpp index c5c24886bd6d1a63a1c299413379aec890434e42..bf974d30d8649e6b64661a8038be25ca3fabfc98 100644 --- a/src/gpu/cl/kernels/ClWinogradOutputTransformKernel.cpp +++ b/src/gpu/cl/kernels/ClWinogradOutputTransformKernel.cpp @@ -23,6 +23,7 @@ */ #include "src/gpu/cl/kernels/ClWinogradOutputTransformKernel.h" +#include "arm_compute/core/utils/ActivationFunctionUtils.h" #include "arm_compute/core/CL/CLHelpers.h" #include "arm_compute/core/CL/CLKernelLibrary.h" #include "arm_compute/core/CL/ICLTensor.h" @@ -30,10 +31,10 @@ #include "arm_compute/core/IAccessWindow.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Types.h" -#include "arm_compute/core/Utils.h" #include "arm_compute/core/Validate.h" #include "arm_compute/core/Window.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/core/AccessWindowStatic.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" diff --git a/src/gpu/cl/operators/ClActivation.h b/src/gpu/cl/operators/ClActivation.h index 75b38e8a0061fea63876ce3baba9dd38c6958f6c..348dc279297798191fcf3f2387aa7c1bbac868fd 100644 --- a/src/gpu/cl/operators/ClActivation.h +++ b/src/gpu/cl/operators/ClActivation.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CL_ACTIVATION_H #define ARM_COMPUTE_CL_ACTIVATION_H +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/gpu/cl/ClCompileContext.h" #include "src/gpu/cl/IClOperator.h" diff --git a/src/gpu/cl/operators/ClAdd.h b/src/gpu/cl/operators/ClAdd.h index d99f983ed0c0fb270de1bbdcec5e2ab000173bc8..a17ce7b5d6a84f2828f2a5df68b61b09a5a84d40 100644 --- a/src/gpu/cl/operators/ClAdd.h +++ b/src/gpu/cl/operators/ClAdd.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CL_ADD_H #define ARM_COMPUTE_CL_ADD_H +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/gpu/cl/ClCompileContext.h" #include "src/gpu/cl/IClOperator.h" diff --git a/src/gpu/cl/operators/ClDirectConv2d.h b/src/gpu/cl/operators/ClDirectConv2d.h index 85365b76ff2152c708d8f3747c0522ddda4ffa85..fedb9e971e325766f8337cc20453721465c1c759 100644 --- a/src/gpu/cl/operators/ClDirectConv2d.h +++ b/src/gpu/cl/operators/ClDirectConv2d.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CL_DIRECT_CONV2D_H #define ARM_COMPUTE_CL_DIRECT_CONV2D_H +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/gpu/cl/ClCompileContext.h" #include "src/gpu/cl/IClKernel.h" #include "src/gpu/cl/IClOperator.h" @@ -79,4 +80,4 @@ private: }; } // namespace opencl } // namespace arm_compute -#endif /* ARM_COMPUTE_CL_DIRECT_CONV2D_H */ \ No newline at end of file +#endif /* ARM_COMPUTE_CL_DIRECT_CONV2D_H */ diff --git a/src/gpu/cl/operators/ClElementwiseOperations.h b/src/gpu/cl/operators/ClElementwiseOperations.h index 304b250d668d32057a1fb8fc3305822f73536b05..120049cb7fb9154a52cebccd2b8307158ccff020 100644 --- a/src/gpu/cl/operators/ClElementwiseOperations.h +++ b/src/gpu/cl/operators/ClElementwiseOperations.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CL_ELEMENTWISE_OPERATIONS_H #define ARM_COMPUTE_CL_ELEMENTWISE_OPERATIONS_H +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/gpu/cl/ClCompileContext.h" #include "src/gpu/cl/IClOperator.h" diff --git a/src/gpu/cl/operators/ClFill.h b/src/gpu/cl/operators/ClFill.h index c9289b2b95fa2918f3a6b22e941dca321a5df793..3bbe27ef71e42fcf4b09e075d6947eb91b1208b8 100644 --- a/src/gpu/cl/operators/ClFill.h +++ b/src/gpu/cl/operators/ClFill.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CL_FILL_H #define ARM_COMPUTE_CL_FILL_H +#include "arm_compute/core/PixelValue.h" #include "arm_compute/core/Window.h" #include "src/gpu/cl/ClCompileContext.h" #include "src/gpu/cl/IClOperator.h" diff --git a/src/gpu/cl/operators/ClFullyConnected.cpp b/src/gpu/cl/operators/ClFullyConnected.cpp index b289cc0104c43854b29eb71f343c4081a78dc5a4..5845bbc69e9d41dbb280390f5a9da4270f2b3c7e 100644 --- a/src/gpu/cl/operators/ClFullyConnected.cpp +++ b/src/gpu/cl/operators/ClFullyConnected.cpp @@ -35,9 +35,13 @@ #include "src/gpu/cl/operators/ClFlatten.h" #include "src/gpu/cl/operators/ClGemm.h" #include "src/gpu/cl/operators/ClGemmLowpMatrixMultiplyCore.h" +#include "src/gpu/cl/operators/ClMatMul.h" #include "src/gpu/cl/operators/ClTranspose.h" #include "src/gpu/cl/utils/ClAuxTensorHandler.h" +#include "src/runtime/heuristics/matmul_native/ClMatMulNativeKernelConfig.h" +#include "src/runtime/heuristics/matmul_native/IClMatMulNativeKernelConfig.h" + #include "src/common/utils/Log.h" #include "support/Cast.h" @@ -52,6 +56,12 @@ using namespace arm_compute::misc::shape_calculator; namespace { +// Function to calculate batched tensor shape in format [M, 1, B0, B1 ..] which is the format matmul expects +inline TensorShape get_reshaped_matmul_tensor(const TensorShape &src) +{ + return TensorShape(src.x(), 1, src.y(), src.collapsed_from(2).z()); // Return value optimisation +} + Status construct_gemmlowp_output_stage(const ITensorInfo &src, const ITensorInfo &weights, const ITensorInfo &dst, GEMMLowpOutputStageInfo &gemmlowp_output_stage, ActivationLayerInfo activation_info) { @@ -99,43 +109,65 @@ Status construct_gemmlowp_output_stage(const ITensorInfo &src, const ITensorInfo return Status{}; } -Status validate_mm(const ITensorInfo &src, const ITensorInfo &weights, const ITensorInfo *bias, const ITensorInfo &dst, const FullyConnectedLayerInfo &fc_info) +Status validate_mm(const ITensorInfo &src, const ITensorInfo &weights, const ITensorInfo *bias, const ITensorInfo &dst, const FullyConnectedLayerInfo &fc_info, bool use_matmul) { - GEMMLowpOutputStageInfo gemmlowp_output_stage; - ARM_COMPUTE_RETURN_ON_ERROR(construct_gemmlowp_output_stage(src, weights, dst, gemmlowp_output_stage, fc_info.activation_info)); - - const GEMMInfo &gemm_info = GEMMInfo(false, // is_a_reshaped - false, // is_b_reshaped - true, // reshape_b_only_on_first_run - 0, // depth_output_gemm3d - false, // reinterpret_input_as_3d - fc_info.retain_internal_weights, // retain_internal_weights - gemmlowp_output_stage, // gemmlowp_output_stage - fc_info.fp_mixed_precision, // fp_mixed_precision - false, // fast_math - true, // broadcast_bias - ActivationLayerInfo()); // activation_info - - if(is_data_type_quantized_asymmetric(src.data_type())) + // Note : If input is dynamic and data is not batched, use matmul, else use gemm + const bool transpose_weights = fc_info.transpose_weights ? !fc_info.are_weights_reshaped : false; + const bool use_dynamic_gemm = !use_matmul && !weights.are_values_constant() && transpose_weights; // use dynamic gemm as fallback for matmul + const bool is_quantized = is_data_type_quantized_asymmetric(src.data_type()); + + if(use_matmul) { - const UniformQuantizationInfo iq_info = src.quantization_info().uniform(); - const UniformQuantizationInfo wq_info = weights.quantization_info().uniform(); - - // Since we need negative offsets for computing convolution, we need to change QuantizationInfo() - // Extract and negate src and weights offset - const QuantizationInfo src_quantization_info(iq_info.scale, -iq_info.offset); - const QuantizationInfo weights_quantization_info(wq_info.scale, -wq_info.offset); - - // Validate gemmlowp function - ARM_COMPUTE_RETURN_ON_ERROR(ClGemmLowpMatrixMultiplyCore::validate(&src.clone()->set_quantization_info(src_quantization_info), - &weights.clone()->set_quantization_info(weights_quantization_info), - bias, - &dst, - gemm_info)); + const MatMulInfo m_info = MatMulInfo().adj_rhs(transpose_weights); + + // Note: LHS is reshaped here to match ClMatMul expectations of batch index - From [M, B0, B1] to [M, 1, B0, B1] + TensorInfo lhs_to_use = src.clone()->set_tensor_shape(get_reshaped_matmul_tensor(src.tensor_shape())); + + const GPUTarget gpu_target = CLScheduler::get().target(); + std::unique_ptr t = cl_matmul::ClMatMulNativeKernelConfigurationFactory::create(gpu_target); + const MatMulKernelInfo kernel_info = t->configure(&lhs_to_use, &weights, m_info); + + return is_quantized ? kernels::ClMatMulLowpNativeKernel::validate(&lhs_to_use, &weights, bias, &dst, kernel_info, fc_info.activation_info) : + kernels::ClMatMulNativeKernel::validate(&lhs_to_use, &weights, bias, &dst, kernel_info, fc_info.activation_info); } else { - ARM_COMPUTE_RETURN_ON_ERROR(ClGemm::validate(&src, &weights, bias, &dst, 1.f, 1.f, gemm_info)); + GEMMLowpOutputStageInfo gemmlowp_output_stage; + ARM_COMPUTE_RETURN_ON_ERROR(construct_gemmlowp_output_stage(src, weights, dst, gemmlowp_output_stage, fc_info.activation_info)); + + const GEMMInfo &gemm_info = GEMMInfo(false, // is_a_reshaped + false, // is_b_reshaped + !use_dynamic_gemm, // reshape_b_only_on_first_run + 0, // depth_output_gemm3d + false, // reinterpret_input_as_3d + fc_info.retain_internal_weights, // retain_internal_weights + gemmlowp_output_stage, // gemmlowp_output_stage + fc_info.fp_mixed_precision, // fp_mixed_precision + false, // fast_math + true, // broadcast_bias + ActivationLayerInfo()); // activation_info + + if(is_quantized) + { + const UniformQuantizationInfo iq_info = src.quantization_info().uniform(); + const UniformQuantizationInfo wq_info = weights.quantization_info().uniform(); + + // Since we need negative offsets for computing convolution, we need to change QuantizationInfo() + // Extract and negate src and weights offset + const QuantizationInfo src_quantization_info(iq_info.scale, -iq_info.offset); + const QuantizationInfo weights_quantization_info(wq_info.scale, -wq_info.offset); + + // Validate gemmlowp function + ARM_COMPUTE_RETURN_ON_ERROR(ClGemmLowpMatrixMultiplyCore::validate(&src.clone()->set_quantization_info(src_quantization_info), + &weights.clone()->set_quantization_info(weights_quantization_info), + bias, + &dst, + gemm_info)); + } + else + { + ARM_COMPUTE_RETURN_ON_ERROR(ClGemm::validate(&src, &weights, bias, &dst, 1.f, 1.f, gemm_info)); + } } return Status{}; @@ -148,6 +180,8 @@ ClFullyConnected::ClFullyConnected() _reshape_weights(nullptr), _mm_gemm(nullptr), _mm_gemmlowp(nullptr), + _matmul_native_kernel(nullptr), + _matmul_lowp_native_kernel(nullptr), _aux_mem(Count) { } @@ -157,50 +191,84 @@ ClFullyConnected::~ClFullyConnected() = default; void ClFullyConnected::configure_mm(const CLCompileContext &compile_context, ITensorInfo *src, ITensorInfo *weights, ITensorInfo *bias, ITensorInfo *dst, const FullyConnectedLayerInfo &fc_info) { - GEMMLowpOutputStageInfo gemmlowp_output_stage; - construct_gemmlowp_output_stage(*src, *weights, *dst, gemmlowp_output_stage, fc_info.activation_info); - - const GEMMInfo &gemm_info = GEMMInfo(false, // is_a_reshaped - false, // is_b_reshaped - !_dynamic_weights, // reshape_b_only_on_first_run - 0, // depth_output_gemm3d - false, // reinterpret_input_as_3d - fc_info.retain_internal_weights, // retain_internal_weights - gemmlowp_output_stage, // gemmlowp_output_stage - fc_info.fp_mixed_precision, // fp_mixed_precision - false, // fast_math - true, // broadcast_bias - fc_info.activation_info); // activation_info - - if(_is_quantized) + // If weights are dynamic and matmul is supported use matmul, else use gemm + if(_use_matmul) { - // Since we need negative offsets for computing convolution, we need to change QuantizationInfo() - // Extract and negate input and weights offset - const QuantizationInfo src_quantization_info = src->quantization_info(); - const QuantizationInfo weights_quantization_info = weights->quantization_info(); + // Specify whether transpose weights is necessary in matmul info + const MatMulInfo mat_info = MatMulInfo().adj_rhs(_transpose_weights); - TensorInfo src_info = src->clone()->set_quantization_info(src_quantization_info); - TensorInfo weights_info = weights->clone()->set_quantization_info(weights_quantization_info); + // Note: MatMul does not need offset negation unlike gemm + // 1. Change shape when calling matmul to fit batch expectations. + _lhs_to_use = src->clone()->set_tensor_shape(get_reshaped_matmul_tensor(_lhs_to_use.tensor_shape())); - src_info.set_quantization_info(QuantizationInfo(src_quantization_info.uniform().scale, -src_quantization_info.uniform().offset)); - weights_info.set_quantization_info(QuantizationInfo(weights_quantization_info.uniform().scale, -weights_quantization_info.uniform().offset)); + // 2. Use heuristics to get kernel info object + const GPUTarget gpu_target = CLScheduler::get().target(); + std::unique_ptr kernel_config = cl_matmul::ClMatMulNativeKernelConfigurationFactory::create(gpu_target); + MatMulKernelInfo kernel_info = kernel_config->configure(src, weights, mat_info); - // Configure gemmlowp function - _mm_gemmlowp = std::make_unique(); - _mm_gemmlowp->configure(compile_context, &src_info, &weights_info, bias, dst, gemm_info); + // 3. Configure relevant matmul kernel + if(_is_quantized) + { + _matmul_lowp_native_kernel = std::make_unique(); + _matmul_lowp_native_kernel->set_target(gpu_target); + _matmul_lowp_native_kernel->configure(compile_context, src, weights, bias, dst, kernel_info, fc_info.activation_info); + } + else + { + _matmul_native_kernel = std::make_unique(); + _matmul_native_kernel->set_target(gpu_target); + _matmul_native_kernel->configure(compile_context, src, weights, bias, dst, kernel_info, fc_info.activation_info); + } } else { - // Configure matrix multiply kernel - _mm_gemm = std::make_unique(); - _mm_gemm->configure(compile_context, src, weights, bias, dst, 1.f, 1.f, gemm_info); + // Configure GEMM + GEMMLowpOutputStageInfo gemmlowp_output_stage; + construct_gemmlowp_output_stage(*src, *weights, *dst, gemmlowp_output_stage, fc_info.activation_info); + + const GEMMInfo &gemm_info = GEMMInfo(false, // is_a_reshaped + false, // is_b_reshaped + !_dynamic_gemm, // reshape_b_only_on_first_run + 0, // depth_output_gemm3d + false, // reinterpret_input_as_3d + fc_info.retain_internal_weights, // retain_internal_weights + gemmlowp_output_stage, // gemmlowp_output_stage + fc_info.fp_mixed_precision, // fp_mixed_precision + false, // fast_math + true, // broadcast_bias + fc_info.activation_info); // activation_info + + if(_is_quantized) + { + // Since we need negative offsets for computing convolution, we need to change QuantizationInfo() + // Extract and negate input and weights offset + const QuantizationInfo src_quantization_info = src->quantization_info(); + const QuantizationInfo weights_quantization_info = weights->quantization_info(); + + TensorInfo src_info = src->clone()->set_quantization_info(src_quantization_info); + TensorInfo weights_info = weights->clone()->set_quantization_info(weights_quantization_info); + + src_info.set_quantization_info(QuantizationInfo(src_quantization_info.uniform().scale, -src_quantization_info.uniform().offset)); + weights_info.set_quantization_info(QuantizationInfo(weights_quantization_info.uniform().scale, -weights_quantization_info.uniform().offset)); + + // Configure gemmlowp function + _mm_gemmlowp = std::make_unique(); + _mm_gemmlowp->configure(compile_context, &src_info, &weights_info, bias, dst, gemm_info); + } + else + { + // Configure matrix multiply kernel + _mm_gemm = std::make_unique(); + _mm_gemm->configure(compile_context, src, weights, bias, dst, 1.f, 1.f, gemm_info); + } } } void ClFullyConnected::configure_conv_fc(const CLCompileContext &compile_context, ITensorInfo *src, ITensorInfo *weights, ITensorInfo *bias, ITensorInfo *dst, const FullyConnectedLayerInfo &fc_info) { - ARM_COMPUTE_ERROR_ON((weights->dimension(1) != (src->dimension(0) * src->dimension(1) * src->dimension(2)))); + // MatMul fuses transpose operation, so we use the first dimension for comparison where appropriate. + ARM_COMPUTE_ERROR_ON((weights->dimension((_use_matmul && _transpose_weights) ? 0 : 1) != (src->dimension(0) * src->dimension(1) * src->dimension(2)))); // If the fully connected layer is called after a convolution layer, the input tensor must be linearized @@ -211,6 +279,7 @@ void ClFullyConnected::configure_conv_fc(const CLCompileContext &compile_context _flatten = std::make_unique(); _flatten->configure(compile_context, src, &_flattened_src); + // Note: if flatten has > 1 dimensions after, these dimensions are batch // Configure matrix multiply kernel configure_mm(compile_context, &_flattened_src, weights, bias, dst, fc_info); } @@ -218,7 +287,8 @@ void ClFullyConnected::configure_conv_fc(const CLCompileContext &compile_context void ClFullyConnected::configure_fc_fc(const CLCompileContext &compile_context, ITensorInfo *src, ITensorInfo *weights, ITensorInfo *bias, ITensorInfo *dst, const FullyConnectedLayerInfo &fc_info) { - ARM_COMPUTE_ERROR_ON(src->dimension(0) != weights->dimension(1)); + // MatMul fuses transpose operation, so we use the first dimension for comparison where appropriate. + ARM_COMPUTE_ERROR_ON(src->dimension(0) != weights->dimension((_use_matmul && _transpose_weights) ? 0 : 1)); // Configure matrix multiply kernel configure_mm(compile_context, src, weights, bias, dst, fc_info); @@ -228,19 +298,27 @@ void ClFullyConnected::configure(const CLCompileContext &compile_context, ITenso FullyConnectedLayerInfo fc_info) { ARM_COMPUTE_ERROR_ON_NULLPTR(src, weights, dst); + const GPUTarget gpu_target = get_arch_from_target(CLScheduler::get().target()); // Perform validate step ARM_COMPUTE_ERROR_THROW_ON(ClFullyConnected::validate(src, weights, biases, dst, fc_info)); ARM_COMPUTE_LOG_PARAMS(src, weights, biases, dst, fc_info); - _are_weights_converted = true; - _are_weights_reshaped = fc_info.transpose_weights ? fc_info.are_weights_reshaped : true; - _is_fc_after_conv = true; - _is_quantized = is_data_type_quantized_asymmetric(src->data_type()); - _is_prepared = fc_info.retain_internal_weights; - _weights_to_use = TensorInfo(*weights); - _weights_to_use_idx = ACL_SRC_1; - _dynamic_weights = !weights->are_values_constant() && !_are_weights_reshaped; + _transpose_weights = fc_info.transpose_weights ? !fc_info.are_weights_reshaped : false; + _is_fc_after_conv = true; + _is_quantized = is_data_type_quantized_asymmetric(src->data_type()); + _is_prepared = fc_info.retain_internal_weights; + _weights_to_use = TensorInfo(*weights); + _weights_to_use_idx = ACL_SRC_1; + + // When using dynamic weights - use matmul kernels. + // Note: MatMul is not used in the following cases (Gemm is used as fallback) : + // 1. When the weights tensor is not dynamic + // 2. MatMul does not support broadcasting batch dimension, and therefore is disabled if fc is batched. + // 3. When FC is after convolution and src tensor data layout does not match weights trained data layout (weights conversion kernel is required) + const bool is_batched_fc_layer = dst->dimension(1) > 1; + _use_matmul = gpu_target != GPUTarget::MIDGARD && !weights->are_values_constant() && !is_batched_fc_layer && !(src->num_dimensions() > 1 && (src->data_layout() != fc_info.weights_trained_layout)); + _dynamic_gemm = !weights->are_values_constant() && _transpose_weights && !_use_matmul; // With the Fully Connected layer we can have 4 different cases: // 1) Convolution layer -> Fully Connected layer without batches @@ -249,7 +327,6 @@ void ClFullyConnected::configure(const CLCompileContext &compile_context, ITenso // 4) Fully Connected layer -> Fully Connected layer with batches // Check if we have a fully connected layer with batches - const bool is_batched_fc_layer = dst->dimension(1) > 1; if(is_batched_fc_layer) { _is_fc_after_conv = (TensorShape::num_max_dimensions >= 4) && (std::equal(src->tensor_shape().cbegin() + 3, @@ -263,8 +340,8 @@ void ClFullyConnected::configure(const CLCompileContext &compile_context, ITenso ITensorInfo *weights_used = weights; - // Reshape weights if needed - if(!_are_weights_reshaped) + // Reshape weights if needed - Not needed when matmul is in use as matmul fuses transpose op. + if(_transpose_weights && !_use_matmul) { // Reshape the weights _reshape_weights = std::make_unique(); @@ -284,9 +361,9 @@ void ClFullyConnected::configure(const CLCompileContext &compile_context, ITenso src->tensor_shape(), fc_info.weights_trained_layout); - weights_used = &_converted_weights; - _weights_to_use_idx = offset_int_vec(ConvertedWeights); - _are_weights_converted = false; + weights_used = &_converted_weights; + _weights_to_use_idx = offset_int_vec(ConvertedWeights); + _run_convert_weights = true; } if(_is_fc_after_conv) @@ -302,39 +379,47 @@ void ClFullyConnected::configure(const CLCompileContext &compile_context, ITenso // Update TensorInfo of final weights used (Need to be done in the end due to padding expansion) _weights_to_use = *weights_used; - // Set auxiliary memory requirements - auto gemm_mem_req = (_is_quantized) ? _mm_gemmlowp->workspace() : _mm_gemm->workspace(); - for(unsigned int i = 0; i < gemm_mem_req.size(); ++i) + if(_use_matmul) { - _aux_mem[i] = gemm_mem_req[i]; - } - if(_aux_mem[1].size > 0 || _aux_mem[2].size > 0) // Persistent weights memory on GEMMs - { - // Release permuted weights at the of prepare as they are further transposed by the assembly dispatch - // Keep all the auxiliary tensors in case of dynamic weights as they are recalculated every time - _aux_mem[TransposedWeights] = MemoryInfo( - offset_int_vec(TransposedWeights), - _dynamic_weights ? MemoryLifetime::Temporary : MemoryLifetime::Prepare, - _reshaped_weights.total_size()); - _aux_mem[ConvertedWeights] = MemoryInfo( - offset_int_vec(ConvertedWeights), - _dynamic_weights ? MemoryLifetime::Temporary : MemoryLifetime::Prepare, - _converted_weights.total_size()); + // Note : MatMul does not use transpose and does not need auxillary memory, so only converted weights are added to aux_mem + _aux_mem[ConvertedWeights] = MemoryInfo(offset_int_vec(ConvertedWeights), MemoryLifetime::Temporary, _converted_weights.total_size()); } else { - // Release permuted weights at the of prepare as they are further transposed by the assembly dispatch - const auto transposed_wei_lft = (_weights_to_use_idx == offset_int_vec(TransposedWeights)) ? MemoryLifetime::Persistent : MemoryLifetime::Prepare; - const auto converted_wei_lft = (_weights_to_use_idx == offset_int_vec(ConvertedWeights)) ? MemoryLifetime::Persistent : MemoryLifetime::Prepare; - - _aux_mem[TransposedWeights] = MemoryInfo( - offset_int_vec(TransposedWeights), - _dynamic_weights ? MemoryLifetime::Temporary : transposed_wei_lft, - _reshaped_weights.total_size()); - _aux_mem[ConvertedWeights] = MemoryInfo( - offset_int_vec(ConvertedWeights), - _dynamic_weights ? MemoryLifetime::Temporary : converted_wei_lft, - _converted_weights.total_size()); + // Set auxiliary memory requirements for gemm operators + auto gemm_mem_req = (_is_quantized) ? _mm_gemmlowp->workspace() : _mm_gemm->workspace(); + for(unsigned int i = 0; i < gemm_mem_req.size(); ++i) + { + _aux_mem[i] = gemm_mem_req[i]; + } + if(_aux_mem[1].size > 0 || _aux_mem[2].size > 0) // Persistent weights memory on GEMMs + { + // Release permuted weights at the of prepare as they are further transposed by the assembly dispatch + // Keep all the auxiliary tensors in case of dynamic weights as they are recalculated every time + _aux_mem[TransposedWeights] = MemoryInfo( + offset_int_vec(TransposedWeights), + _dynamic_gemm ? MemoryLifetime::Temporary : MemoryLifetime::Prepare, + _reshaped_weights.total_size()); + _aux_mem[ConvertedWeights] = MemoryInfo( + offset_int_vec(ConvertedWeights), + _dynamic_gemm ? MemoryLifetime::Temporary : MemoryLifetime::Prepare, + _converted_weights.total_size()); + } + else + { + // Release permuted weights at the of prepare as they are further transposed by the assembly dispatch + const auto transposed_wei_lft = (_weights_to_use_idx == offset_int_vec(TransposedWeights)) ? MemoryLifetime::Persistent : MemoryLifetime::Prepare; + const auto converted_wei_lft = (_weights_to_use_idx == offset_int_vec(ConvertedWeights)) ? MemoryLifetime::Persistent : MemoryLifetime::Prepare; + + _aux_mem[TransposedWeights] = MemoryInfo( + offset_int_vec(TransposedWeights), + _dynamic_gemm ? MemoryLifetime::Temporary : transposed_wei_lft, + _reshaped_weights.total_size()); + _aux_mem[ConvertedWeights] = MemoryInfo( + offset_int_vec(ConvertedWeights), + _dynamic_gemm ? MemoryLifetime::Temporary : converted_wei_lft, + _converted_weights.total_size()); + } } _aux_mem[FlattenedSrc] = MemoryInfo(offset_int_vec(FlattenedSrc), MemoryLifetime::Temporary, _flattened_src.total_size()); } @@ -348,13 +433,19 @@ Status ClFullyConnected::validate(const ITensorInfo *src, const ITensorInfo *wei ARM_COMPUTE_RETURN_ERROR_ON(weights->num_dimensions() > 2); ARM_COMPUTE_RETURN_ERROR_ON(fc_info.activation_info.enabled() && is_data_type_quantized(src->data_type()) && fc_info.activation_info.activation() != ActivationLayerInfo::ActivationFunction::RELU && fc_info.activation_info.activation() != ActivationLayerInfo::ActivationFunction::BOUNDED_RELU && fc_info.activation_info.activation() != ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU); + const GPUTarget gpu_target = get_arch_from_target(CLScheduler::get().target()); - bool weights_reshaped = fc_info.transpose_weights ? fc_info.are_weights_reshaped : true; - bool is_fc_after_conv = true; + const bool transpose_weights = fc_info.transpose_weights ? !fc_info.are_weights_reshaped : false; + bool is_fc_after_conv = true; + + // When using dynamic weights - use matmul kernels. + // Note: MatMul does not support broadcasting so fallback with batched cases. + const bool is_batched_fc_layer = dst->dimension(1) > 1; + const bool use_matmul = gpu_target != GPUTarget::MIDGARD && !weights->are_values_constant() && !is_batched_fc_layer && !(src->num_dimensions() > 1 && (src->data_layout() != fc_info.weights_trained_layout)); const ITensorInfo &flatten_src = TensorInfo(src->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(compute_flatten_shape(src)).set_data_layout(DataLayout::NCHW)); const ITensorInfo &reshaped_weights = TensorInfo(weights->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(compute_transposed_shape(*weights))); - const ITensorInfo &converted_weights = weights_reshaped ? TensorInfo(weights->clone()->set_is_resizable(true).reset_padding()) : TensorInfo(*reshaped_weights.clone()); + const ITensorInfo &converted_weights = (transpose_weights && !use_matmul) ? TensorInfo(*reshaped_weights.clone()) : TensorInfo(weights->clone()->set_is_resizable(true).reset_padding()); // With the Fully Connected layer we can have 4 different cases: // 1) Convolution layer -> Fully Connected layer without batches @@ -378,8 +469,7 @@ Status ClFullyConnected::validate(const ITensorInfo *src, const ITensorInfo *wei } } - // Check if we have a fully connected layer with batches - const bool is_batched_fc_layer = dst->dimension(1) > 1; + // Check if FC is after conv (flatten kernel is run in case where FC is after conv.) if(is_batched_fc_layer) { is_fc_after_conv = (TensorShape::num_max_dimensions >= 4) && (std::equal(src->tensor_shape().cbegin() + 3, @@ -391,7 +481,8 @@ Status ClFullyConnected::validate(const ITensorInfo *src, const ITensorInfo *wei is_fc_after_conv = src->num_dimensions() > 1; } - if(!weights_reshaped) + // Transpose kernel does not run when matmul is supported as matmul fuses transpose op. + if(transpose_weights && !use_matmul) { // Validate reshape weights kernel ARM_COMPUTE_RETURN_ON_ERROR(ClTranspose::validate(weights, &reshaped_weights)); @@ -411,7 +502,9 @@ Status ClFullyConnected::validate(const ITensorInfo *src, const ITensorInfo *wei if(is_fc_after_conv) { // Fully Connected layer after a Convolution Layer without batches - ARM_COMPUTE_RETURN_ERROR_ON((weights_to_use->dimension(1) != (src->dimension(0) * src->dimension(1) * src->dimension(2)))); + // K Index of matrix multiplication. MatMul performs transpose in kernel, so index is 0 when matmul and transpose enabled + const int weight_idx = (use_matmul && transpose_weights) ? 0 : 1; + ARM_COMPUTE_RETURN_ERROR_ON((weights_to_use->dimension(weight_idx) != (src->dimension(0) * src->dimension(1) * src->dimension(2)))); // Validate flatten kernel ARM_COMPUTE_RETURN_ON_ERROR(ClFlatten::validate(src, &flatten_src)); @@ -420,11 +513,13 @@ Status ClFullyConnected::validate(const ITensorInfo *src, const ITensorInfo *wei else { // Fully Connected layer after a Fully Connected Layer without batches - ARM_COMPUTE_RETURN_ERROR_ON(src->dimension(0) != weights_to_use->dimension(1)); + // K Index of matrix multiplication. MatMul performs transpose in kernel, so index is 0 when matmul and transpose enabled + const int weight_idx = (use_matmul && transpose_weights) ? 0 : 1; + ARM_COMPUTE_RETURN_ERROR_ON(src->dimension(0) != weights_to_use->dimension(weight_idx)); } // Validate matrix multiply kernel - ARM_COMPUTE_RETURN_ON_ERROR(validate_mm(*src_to_use, *weights_to_use, biases, *dst, fc_info)); + ARM_COMPUTE_RETURN_ON_ERROR(validate_mm(*src_to_use, *weights_to_use, biases, *dst, fc_info, use_matmul)); return Status{}; } @@ -435,7 +530,7 @@ void ClFullyConnected::run(ITensorPack &tensors) #ifdef ARM_COMPUTE_ASSERTS_ENABLED ++_asrt_run_count; - ARM_COMPUTE_ERROR_ON(_dynamic_weights && _asrt_prepare_count != _asrt_run_count); + ARM_COMPUTE_ERROR_ON(_dynamic_gemm && _asrt_prepare_count != _asrt_run_count); #endif // ARM_COMPUTE_ASSERTS_ENABLED auto src = tensors.get_const_tensor(ACL_SRC_0); @@ -457,24 +552,41 @@ void ClFullyConnected::run(ITensorPack &tensors) gemm_pack.add_const_tensor(ACL_SRC_1, weights.get()); } - // Run matrix multiply - if(_is_quantized) + // Run MatMul Op + if(_use_matmul) { - _mm_gemmlowp->run(gemm_pack); + // Run matmul kernels for matrix multiplication + if(_is_quantized) + { + CLScheduler::get().enqueue_op(*_matmul_lowp_native_kernel, gemm_pack, true); + } + else + { + CLScheduler::get().enqueue_op(*_matmul_native_kernel, gemm_pack, true); + } } else { - _mm_gemm->run(gemm_pack); + // Run matrix multiply + if(_is_quantized) + { + _mm_gemmlowp->run(gemm_pack); + } + else + { + _mm_gemm->run(gemm_pack); + } } } void ClFullyConnected::prepare(ITensorPack &tensors) { - if(!_is_prepared || _dynamic_weights) + // Note : Running prepare() each run when _use_matmul is true is unnecessary unless weights conversion is needed. + if(!_is_prepared || _dynamic_gemm) { #ifdef ARM_COMPUTE_ASSERTS_ENABLED ++_asrt_prepare_count; - ARM_COMPUTE_ERROR_ON(!_dynamic_weights && _asrt_prepare_count > 1); + ARM_COMPUTE_ERROR_ON(!_dynamic_gemm && !_use_matmul && _asrt_prepare_count > 1); #endif // ARM_COMPUTE_ASSERTS_ENABLED auto weights = tensors.get_const_tensor(ACL_SRC_1); @@ -485,8 +597,8 @@ void ClFullyConnected::prepare(ITensorPack &tensors) // Pointer to current weights const ITensor *cur_weights = weights; - // Reshape of the weights if needed - if(!_are_weights_reshaped) + // Reshape weights if needed. Disabled when matmul kernels are enabled as matmul fuses transpose. + if(_transpose_weights && !_use_matmul) { // Run reshape weights kernel and mark weights as unused ITensorPack transpose_pack{ { ACL_SRC, weights }, { ACL_DST, reshaped_weights.get() } }; @@ -497,7 +609,7 @@ void ClFullyConnected::prepare(ITensorPack &tensors) } // Convert weights if needed - if(!_are_weights_converted) + if(_run_convert_weights) { ITensorPack convert_pack{ { ACL_SRC, cur_weights }, { ACL_DST, converted_weights.get() } }; _convert_weights->run(convert_pack); @@ -510,14 +622,18 @@ void ClFullyConnected::prepare(ITensorPack &tensors) gemm_pack.add_const_tensor(ACL_SRC_1, cur_weights); // Prepare GEMM prepare and release unused weights - if(!_is_quantized) - { - _mm_gemm->prepare(gemm_pack); - } - else + if(_dynamic_gemm || !_use_matmul) { - _mm_gemmlowp->prepare(gemm_pack); + if(!_is_quantized) + { + _mm_gemm->prepare(gemm_pack); + } + else + { + _mm_gemmlowp->prepare(gemm_pack); + } } + _is_prepared = true; } } diff --git a/src/gpu/cl/operators/ClFullyConnected.h b/src/gpu/cl/operators/ClFullyConnected.h index d08d5db8a45b7cee2f7a27edf6fa42d5075a201a..d975859d87fa428ad5015278eb11e85610a2cc1d 100644 --- a/src/gpu/cl/operators/ClFullyConnected.h +++ b/src/gpu/cl/operators/ClFullyConnected.h @@ -25,6 +25,7 @@ #define ARM_COMPUTE_CL_FULLY_CONNECTED_H #include "arm_compute/core/TensorInfo.h" +#include "arm_compute/function_info/FullyConnectedLayerInfo.h" #include "src/gpu/cl/ClCompileContext.h" #include "src/gpu/cl/IClOperator.h" @@ -41,7 +42,12 @@ class ClFlatten; class ClGemm; class ClGemmLowpMatrixMultiplyCore; class ClTranspose; - +// Kernel Forward Declarations +namespace kernels +{ +class ClMatMulNativeKernel; +class ClMatMulLowpNativeKernel; +} /** Basic function to compute a Fully Connected layer on OpenCL. This function calls the following OpenCL kernels: * * -# @ref opencl::kernels::ClIm2ColKernel (called when the input comes from a convolutional layer) @@ -118,25 +124,30 @@ private: std::unique_ptr _mm_gemm; std::unique_ptr _mm_gemmlowp; + std::unique_ptr _matmul_native_kernel; + std::unique_ptr _matmul_lowp_native_kernel; + experimental::MemoryRequirements _aux_mem{}; TensorInfo _flattened_src{}; TensorInfo _converted_weights{}; TensorInfo _reshaped_weights{}; - + TensorInfo _lhs_to_use{}; TensorInfo _weights_to_use{}; int _weights_to_use_idx{ ACL_SRC_1 }; - bool _are_weights_converted{ true }; - bool _are_weights_reshaped{ true }; + bool _run_convert_weights{ false }; + bool _transpose_weights{ false }; + bool _dynamic_gemm{ false }; + bool _use_matmul{ false }; + bool _is_fc_after_conv{ true }; bool _is_quantized{ false }; bool _is_prepared{ false }; - bool _dynamic_weights{ false }; #ifdef ARM_COMPUTE_ASSERTS_ENABLED - int _asrt_run_count{}; - int _asrt_prepare_count{}; + int _asrt_run_count {}; + int _asrt_prepare_count{}; #endif // ARM_COMPUTE_ASSERTS_ENABLED }; } // namespace opencl diff --git a/src/gpu/cl/operators/ClGemm.h b/src/gpu/cl/operators/ClGemm.h index ea8a058fd5b86f31f40755db8ff441803e370c14..11f9f2b3d8ab30381a33d4690bab43b1f204fbf1 100644 --- a/src/gpu/cl/operators/ClGemm.h +++ b/src/gpu/cl/operators/ClGemm.h @@ -25,6 +25,7 @@ #define ARM_COMPUTE_CL_GEMM_H #include "arm_compute/core/TensorInfo.h" +#include "arm_compute/function_info/GEMMInfo.h" #include "arm_compute/runtime/CL/CLTensor.h" #include "arm_compute/runtime/CL/CLTypes.h" diff --git a/src/gpu/cl/operators/ClGemmLowpMatrixMultiplyCore.h b/src/gpu/cl/operators/ClGemmLowpMatrixMultiplyCore.h index 6fa4352bf887b2a922cb7c7c29ae5e4c17a8f500..6e32a90fc47978c8c23f7009c472c9051cf9c7ce 100644 --- a/src/gpu/cl/operators/ClGemmLowpMatrixMultiplyCore.h +++ b/src/gpu/cl/operators/ClGemmLowpMatrixMultiplyCore.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2022 Arm Limited. + * Copyright (c) 2017-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #define ARM_COMPUTE_CL_GEMMLOWP_MATRIXMULTIPLY_CORE_H #include "arm_compute/core/TensorInfo.h" +#include "arm_compute/function_info/GEMMInfo.h" #include "arm_compute/runtime/CL/CLTypes.h" #include "src/gpu/cl/ClCompileContext.h" @@ -154,4 +155,4 @@ private: }; } // namespace opencl } // namespace arm_compute -#endif /* ARM_COMPUTE_CL_GEMMLOWP_MATRIXMULTIPLY_CORE_H */ \ No newline at end of file +#endif /* ARM_COMPUTE_CL_GEMMLOWP_MATRIXMULTIPLY_CORE_H */ diff --git a/src/gpu/cl/operators/ClIndirectConv2d.h b/src/gpu/cl/operators/ClIndirectConv2d.h index 917a67f421dce8ee448c8ddb69c9672e1a811066..e50fa25069c63d82002a4ce50997eebdceaa1817 100644 --- a/src/gpu/cl/operators/ClIndirectConv2d.h +++ b/src/gpu/cl/operators/ClIndirectConv2d.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,6 +25,7 @@ #define ARM_COMPUTE_CL_INDIRECT_CONV2D_H #include "arm_compute/core/TensorInfo.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "arm_compute/runtime/CL/CLTensor.h" #include "arm_compute/runtime/CL/CLTypes.h" diff --git a/src/gpu/cl/operators/ClMatMul.cpp b/src/gpu/cl/operators/ClMatMul.cpp index 3822c16aa1231fd8b84231488434e9b5184620b4..49d14127cadb8a048346334c1f2ff090530e21e5 100644 --- a/src/gpu/cl/operators/ClMatMul.cpp +++ b/src/gpu/cl/operators/ClMatMul.cpp @@ -47,7 +47,7 @@ ClMatMul::ClMatMul() { } -Status ClMatMul::validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *dst, const MatMulInfo &matmul_info) +Status ClMatMul::validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *dst, const MatMulInfo &matmul_info, const ActivationLayerInfo &act_info) { ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(lhs, rhs, dst); ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(lhs, 1, DataType::QASYMM8, DataType::QASYMM8_SIGNED, DataType::F16, DataType::F32); @@ -57,15 +57,15 @@ Status ClMatMul::validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const std::unique_ptr t = ClMatMulNativeKernelConfigurationFactory::create(gpu_target); - MatMulKernelInfo kernel_info = t->configure(lhs, rhs, matmul_info); + const MatMulKernelInfo kernel_info = t->configure(lhs, rhs, matmul_info); - bool is_quantized = is_data_type_quantized_asymmetric(lhs->data_type()); + const bool is_quantized = is_data_type_quantized_asymmetric(lhs->data_type()); - return is_quantized ? ClMatMulLowpNativeKernel::validate(lhs, rhs, dst, kernel_info) : - ClMatMulNativeKernel::validate(lhs, rhs, dst, kernel_info); + return is_quantized ? ClMatMulLowpNativeKernel::validate(lhs, rhs, nullptr /* bias */, dst, kernel_info, act_info) : + ClMatMulNativeKernel::validate(lhs, rhs, nullptr /* bias */, dst, kernel_info, act_info); } -void ClMatMul::configure(const CLCompileContext &compile_context, ITensorInfo *lhs, ITensorInfo *rhs, ITensorInfo *dst, const MatMulInfo &matmul_info) +void ClMatMul::configure(const CLCompileContext &compile_context, ITensorInfo *lhs, ITensorInfo *rhs, ITensorInfo *dst, const MatMulInfo &matmul_info, const ActivationLayerInfo &act_info) { ARM_COMPUTE_ERROR_ON_NULLPTR(lhs, rhs, dst); ARM_COMPUTE_LOG_PARAMS(lhs, rhs, dst, matmul_info); @@ -86,14 +86,14 @@ void ClMatMul::configure(const CLCompileContext &compile_context, ITensorInfo *l _matmul_lowp_native_kernel->set_target(gpu_target); // Configure the low-precision native matrix multiply kernel - _matmul_lowp_native_kernel->configure(compile_context, lhs, rhs, dst, kernel_info); + _matmul_lowp_native_kernel->configure(compile_context, lhs, rhs, nullptr /* bias */, dst, kernel_info, act_info); } else { _matmul_native_kernel->set_target(gpu_target); // Configure the native matrix multiply kernel - _matmul_native_kernel->configure(compile_context, lhs, rhs, dst, kernel_info); + _matmul_native_kernel->configure(compile_context, lhs, rhs, nullptr /* bias */, dst, kernel_info, act_info); } } diff --git a/src/gpu/cl/operators/ClMatMul.h b/src/gpu/cl/operators/ClMatMul.h index 3d9863266efc3bc3877d3cc040743c5fe434c4b6..abbb75239a143323ae1d8119ad1e3f33f550090d 100644 --- a/src/gpu/cl/operators/ClMatMul.h +++ b/src/gpu/cl/operators/ClMatMul.h @@ -21,12 +21,14 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifndef ACL_ARM_COMPUTE_SRC_GPU_CL_OPERATORS_CLMATMUL -#define ACL_ARM_COMPUTE_SRC_GPU_CL_OPERATORS_CLMATMUL +#ifndef ACL_SRC_GPU_CL_OPERATORS_CLMATMUL +#define ACL_SRC_GPU_CL_OPERATORS_CLMATMUL +#include "arm_compute/function_info/ActivationLayerInfo.h" +#include "arm_compute/function_info/MatMulInfo.h" #include "src/gpu/cl/IClOperator.h" -#include "src/gpu/cl/kernels/ClMatMulNativeKernel.h" #include "src/gpu/cl/kernels/ClMatMulLowpNativeKernel.h" +#include "src/gpu/cl/kernels/ClMatMulNativeKernel.h" #include @@ -69,24 +71,26 @@ public: * @param[in] rhs Right-hand side tensor info. Data types supported: same as @p lhs. * @param[out] dst Output tensor to store the result of the batched matrix multiplication. Data types supported: same as @p lhs. * @param[in] matmul_info Contains MatMul operation information described in @ref MatMulInfo. + * @param[in] act_info Class containing information about fused activation function. */ - void configure(const CLCompileContext &compile_context, ITensorInfo *lhs, ITensorInfo *rhs, ITensorInfo *dst, const MatMulInfo &matmul_info); + void configure(const CLCompileContext &compile_context, ITensorInfo *lhs, ITensorInfo *rhs, ITensorInfo *dst, const MatMulInfo &matmul_info, + const ActivationLayerInfo &act_info = ActivationLayerInfo()); /** Static function to check if given info will lead to a valid configuration * * Similar to @ref ClMatMul::configure() * * @return a status */ - static Status validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *dst, const MatMulInfo &matmul_info); + static Status validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *dst, const MatMulInfo &matmul_info, const ActivationLayerInfo &act_info = ActivationLayerInfo()); // Inherited methods overridden: void run(ITensorPack &tensors) override; private: - std::unique_ptr _matmul_native_kernel{nullptr}; - std::unique_ptr _matmul_lowp_native_kernel{nullptr}; + std::unique_ptr _matmul_native_kernel{ nullptr }; + std::unique_ptr _matmul_lowp_native_kernel{ nullptr }; bool _is_quantized{ false }; }; } // namespace opencl } // namespace arm_compute -#endif /* ACL_ARM_COMPUTE_SRC_GPU_CL_OPERATORS_CLMATMUL */ +#endif /* ACL_SRC_GPU_CL_OPERATORS_CLMATMUL */ diff --git a/src/gpu/cl/operators/ClMul.h b/src/gpu/cl/operators/ClMul.h index 6a158c910d81c9d485a91aed7ca2466b9454dc9a..6086bc9d524638392db17cccc3eb3b9a016b281e 100644 --- a/src/gpu/cl/operators/ClMul.h +++ b/src/gpu/cl/operators/ClMul.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,9 @@ #ifndef ARM_COMPUTE_CL_MUL_H #define ARM_COMPUTE_CL_MUL_H +#include "arm_compute/core/Types.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" + #include "src/gpu/cl/ClCompileContext.h" #include "src/gpu/cl/IClOperator.h" @@ -100,4 +103,4 @@ public: }; } // namespace opencl } // namespace arm_compute -#endif /* ARM_COMPUTE_CL_MUL_H */ \ No newline at end of file +#endif /* ARM_COMPUTE_CL_MUL_H */ diff --git a/src/gpu/cl/operators/ClSub.h b/src/gpu/cl/operators/ClSub.h index 902adbf39d9540adb22079d4d209132e76ee812e..7eac4371434b34609c2d7886f847bd9518b8bed1 100644 --- a/src/gpu/cl/operators/ClSub.h +++ b/src/gpu/cl/operators/ClSub.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,6 +24,7 @@ #ifndef ARM_COMPUTE_CL_SUB_H #define ARM_COMPUTE_CL_SUB_H +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "src/gpu/cl/ClCompileContext.h" #include "src/gpu/cl/IClOperator.h" diff --git a/src/graph/GraphBuilder.cpp b/src/graph/GraphBuilder.cpp index 15abf3738a21fc5b95a2931d17a6e7540b80f8d6..7e5d3133d174caba58e4ff8effb54f3a304cbc97 100644 --- a/src/graph/GraphBuilder.cpp +++ b/src/graph/GraphBuilder.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,6 +23,7 @@ */ #include "arm_compute/graph/GraphBuilder.h" +#include "arm_compute/core/utils/DataTypeUtils.h" #include "arm_compute/graph/Graph.h" #include "arm_compute/graph/Utils.h" #include "arm_compute/graph/algorithms/TopologicalSort.h" diff --git a/src/graph/backends/NEON/NEDeviceBackend.cpp b/src/graph/backends/NEON/NEDeviceBackend.cpp index 47ef2c239443c0e60e228f8ff828735f9cd0c052..18456538da4de49a6c0224c43dda5dd297e0c27c 100644 --- a/src/graph/backends/NEON/NEDeviceBackend.cpp +++ b/src/graph/backends/NEON/NEDeviceBackend.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021,2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -44,8 +44,6 @@ #include "arm_compute/runtime/PoolManager.h" #include "arm_compute/runtime/Scheduler.h" -#include "support/ToolchainSupport.h" - namespace arm_compute { namespace graph diff --git a/src/graph/backends/NEON/NEFunctionFactory.cpp b/src/graph/backends/NEON/NEFunctionFactory.cpp index 0fc5291648d6f9e44ce272c8fb1d4022a172b857..57c64475ecea1f2bb3a33222838d4f553cf11b30 100644 --- a/src/graph/backends/NEON/NEFunctionFactory.cpp +++ b/src/graph/backends/NEON/NEFunctionFactory.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021,2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -32,8 +32,6 @@ #include "arm_compute/graph/nodes/Nodes.h" #include "arm_compute/runtime/CPP/CPPFunctions.h" #include "arm_compute/runtime/NEON/NEFunctions.h" -#include "support/Cast.h" -#include "support/ToolchainSupport.h" using namespace arm_compute::utils::cast; diff --git a/src/graph/mutators/NodeFusionMutator.cpp b/src/graph/mutators/NodeFusionMutator.cpp index 5284fce806f1b00d8d617d6c377d34f540fc2b5e..8eb3e4cb71bce3baee458f5cd845fcea12a510e4 100644 --- a/src/graph/mutators/NodeFusionMutator.cpp +++ b/src/graph/mutators/NodeFusionMutator.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,6 +23,7 @@ */ #include "arm_compute/graph/mutators/NodeFusionMutator.h" +#include "arm_compute/core/utils/DataTypeUtils.h" #include "arm_compute/graph/GraphBuilder.h" #include "arm_compute/graph/Logger.h" #include "arm_compute/graph/Utils.h" diff --git a/src/graph/nodes/QuantizationLayerNode.cpp b/src/graph/nodes/QuantizationLayerNode.cpp index 08e2a4d961439e8b7a063aedc0701aeb13275970..4906808dae3089b63b1725ea39fa25bb3dae7177 100644 --- a/src/graph/nodes/QuantizationLayerNode.cpp +++ b/src/graph/nodes/QuantizationLayerNode.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2020 Arm Limited. + * Copyright (c) 2019-2020,2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,6 +23,7 @@ */ #include "arm_compute/graph/nodes/QuantizationLayerNode.h" +#include "arm_compute/core/utils/DataTypeUtils.h" #include "arm_compute/graph/Graph.h" #include "arm_compute/graph/INodeVisitor.h" diff --git a/src/runtime/CL/CLMemoryRegion.cpp b/src/runtime/CL/CLMemoryRegion.cpp index 780a563d63e211665b9d26a670f8c2da5670a3c0..00f91a0ffb07fa2cc8aab64eec182def8386aee5 100644 --- a/src/runtime/CL/CLMemoryRegion.cpp +++ b/src/runtime/CL/CLMemoryRegion.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -30,7 +30,6 @@ namespace arm_compute { ICLMemoryRegion::ICLMemoryRegion(size_t size) : IMemoryRegion(size), - _queue(CLScheduler::get().queue()), _ctx(CLScheduler::get().context()), _mapping(nullptr), _mem() @@ -73,6 +72,14 @@ CLBufferMemoryRegion::CLBufferMemoryRegion(const cl::Buffer &buffer) _mem = buffer; } +CLBufferMemoryRegion::~CLBufferMemoryRegion() +{ + // Flush the command queue to ensure all commands that may use this memory buffer are scheduled to be finished before + // this buffer is freed + // Do not call finish as it is a blocking call which affects the performance + CLScheduler::get().queue().flush(); +} + void *CLBufferMemoryRegion::ptr() { return nullptr; @@ -111,7 +118,10 @@ ICLSVMMemoryRegion::~ICLSVMMemoryRegion() { try { - clFinish(_queue.get()); + // Can only use the blocking finish instead of the non-blocking flush here, because clSVMFree requires all + // commands that may use the svm pointer to finish beforehand + // https://registry.khronos.org/OpenCL/sdk/3.0/docs/man/html/clSVMFree.html + clFinish(CLScheduler::get().queue().get()); _mem = cl::Buffer(); clSVMFree(_ctx.get(), _ptr); } diff --git a/src/runtime/CL/functions/CLActivationLayer.cpp b/src/runtime/CL/functions/CLActivationLayer.cpp index bf6986866342c44baa7270d4b44122aadf8acb7f..f324b1a68ce5d8563c65c1ad4482a673d9a45476 100644 --- a/src/runtime/CL/functions/CLActivationLayer.cpp +++ b/src/runtime/CL/functions/CLActivationLayer.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021 Arm Limited. + * Copyright (c) 2016-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,7 @@ #include "arm_compute/core/CL/ICLTensor.h" #include "arm_compute/core/Types.h" #include "arm_compute/core/Validate.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "arm_compute/runtime/CL/CLRuntimeContext.h" #include "src/core/CL/ICLKernel.h" #include "src/gpu/cl/operators/ClActivation.h" diff --git a/src/runtime/CL/functions/CLArgMinMaxLayer.cpp b/src/runtime/CL/functions/CLArgMinMaxLayer.cpp index 1b0a86a8642066b0f017963e442c2b2dde845c7d..b30d73902537ced5df25b64844e0face7a5207e9 100644 --- a/src/runtime/CL/functions/CLArgMinMaxLayer.cpp +++ b/src/runtime/CL/functions/CLArgMinMaxLayer.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -39,7 +39,7 @@ namespace arm_compute { CLArgMinMaxLayer::CLArgMinMaxLayer(std::shared_ptr memory_manager) - : _memory_group(std::move(memory_manager)), _results_vector(), _not_reshaped_output(), _reduction_kernels_vector(), _reshape(), _num_of_stages(), _reduction_axis() + : _memory_group(std::move(memory_manager)), _not_reshaped_output(), _arg_min_max_kernel(), _reshape(), _reduction_axis() { } @@ -53,7 +53,6 @@ Status CLArgMinMaxLayer::validate(const ITensorInfo *input, int axis, const ITen ARM_COMPUTE_RETURN_ERROR_ON_MSG(op != ReductionOperation::ARG_IDX_MAX && op != ReductionOperation::ARG_IDX_MIN, "Invalid reduction operation"); ARM_COMPUTE_RETURN_ERROR_ON_MSG(axis >= static_cast(TensorShape::num_max_dimensions), "Reduction axis greater than max number of dimensions"); ARM_COMPUTE_RETURN_ERROR_ON_MSG(axis > 3, "Unsupported reduction axis"); - const unsigned int num_of_stages = utils::calculate_number_of_stages_only_x_axis(input->dimension(0), axis); DataType output_data_type = DataType::S32; TensorInfo not_reshaped_output; @@ -76,39 +75,7 @@ Status CLArgMinMaxLayer::validate(const ITensorInfo *input, int axis, const ITen initialize_tensorinfo(not_reshaped_output, shape_before_reshape, output_data_type, input_num_channles, input_qinfo); - if(num_of_stages == 1) - { - ARM_COMPUTE_RETURN_ON_ERROR(CLArgMinMaxLayerKernel::validate(input, nullptr, ¬_reshaped_output, axis, op)); - } - else - { - // Create temporary tensor infos - std::vector sums_vector(num_of_stages - 1); - - // Create intermediate tensor info - TensorShape shape{ input->tensor_shape() }; - - for(unsigned int i = 0; i < num_of_stages - 1; i++) - { - shape.set(0, ceil(shape.x() / 128.f)); - sums_vector[i].set_data_type(input->data_type()); - sums_vector[i].set_tensor_shape(shape); - sums_vector[i].set_num_channels(input->num_channels()); - } - - // Validate ReductionOperation only on first kernel - ARM_COMPUTE_RETURN_ON_ERROR(CLArgMinMaxLayerKernel::validate(input, nullptr, &sums_vector[0], axis, op)); - - // Validate ReductionOperation on intermediate stages - for(unsigned int i = 1; i < num_of_stages - 1; ++i) - { - ARM_COMPUTE_RETURN_ON_ERROR(CLArgMinMaxLayerKernel::validate(input, &sums_vector[i - 1], &sums_vector[i], axis, op)); - } - - // Validate ReductionOperation on the last stage - const unsigned int last_stage = num_of_stages - 1; - ARM_COMPUTE_RETURN_ON_ERROR(CLArgMinMaxLayerKernel::validate(input, &sums_vector[last_stage - 1], ¬_reshaped_output, axis, op)); - } + ARM_COMPUTE_RETURN_ON_ERROR(CLArgMinMaxLayerKernel::validate(input, ¬_reshaped_output, axis, op)); ARM_COMPUTE_RETURN_ON_ERROR(CLReshapeLayer::validate(¬_reshaped_output, output)); return Status{}; } @@ -123,55 +90,21 @@ void CLArgMinMaxLayer::configure(const CLCompileContext &compile_context, const ARM_COMPUTE_ERROR_ON_NULLPTR(input, output); ARM_COMPUTE_LOG_PARAMS(input, axis, output, op); - _num_of_stages = utils::calculate_number_of_stages_only_x_axis(input->info()->dimension(0), axis); _reduction_axis = axis; const TensorShape output_shape = arm_compute::misc::shape_calculator::compute_reduced_shape(input->info()->tensor_shape(), axis, false); DataType output_data_type = (output->info()->data_type() == DataType::UNKNOWN) ? DataType::S32 : output->info()->data_type(); auto_init_if_empty(*output->info(), input->info()->clone()->set_tensor_shape(output_shape).set_data_type(output_data_type).reset_padding().set_is_resizable(true)); - // Configure reduction operation kernels - _reduction_kernels_vector.reserve(_num_of_stages); + TensorShape not_reshaped_output_shape{ input->info()->tensor_shape() }; + not_reshaped_output_shape.set(axis, 1); + auto_init_if_empty(*_not_reshaped_output.info(), input->info()->clone()->set_tensor_shape(not_reshaped_output_shape).set_data_type(output_data_type).reset_padding().set_is_resizable(true)); - auto add_reduction_kernel = [this, &compile_context, axis, op](const ICLTensor * input, const ICLTensor * prev_output, ICLTensor * output) - { - _reduction_kernels_vector.emplace_back(std::make_unique()); - _reduction_kernels_vector.back()->configure(compile_context, input, prev_output, output, axis, op); - }; + _arg_min_max_kernel = std::make_unique(); + _arg_min_max_kernel->configure(compile_context, input, &_not_reshaped_output, axis, op); _memory_group.manage(&_not_reshaped_output); - // Create temporary tensors - if(_num_of_stages == 1) - { - add_reduction_kernel(input, nullptr, &_not_reshaped_output); - } - else - { - _results_vector.resize(_num_of_stages - 1); - TensorShape shape{ input->info()->tensor_shape() }; - for(unsigned int i = 0; i < _num_of_stages - 1; i++) - { - shape.set(0, ceil(shape.x() / 128.f)); - _results_vector[i].allocator()->init(input->info()->clone()->set_tensor_shape(shape).set_data_type(output_data_type)); - } - - // Apply ReductionOperation only on first kernel - _memory_group.manage(&_results_vector[0]); - add_reduction_kernel(input, nullptr, &_results_vector[0]); - - // Apply ReductionOperation on intermediate stages - for(unsigned int i = 1; i < _num_of_stages - 1; ++i) - { - _memory_group.manage(&_results_vector[i]); - add_reduction_kernel(input, &_results_vector[i - 1], &_results_vector[i]); - _results_vector[i - 1].allocator()->allocate(); - } - - // Apply ReductionOperation on the last stage - const unsigned int last_stage = _num_of_stages - 1; - add_reduction_kernel(input, &_results_vector[last_stage - 1], &_not_reshaped_output); - _results_vector[last_stage - 1].allocator()->allocate(); - } + _reshape.configure(compile_context, &_not_reshaped_output, output); _not_reshaped_output.allocator()->allocate(); } @@ -180,10 +113,7 @@ void CLArgMinMaxLayer::run() { MemoryGroupResourceScope scope_mg(_memory_group); - for(unsigned int i = 0; i < _num_of_stages; ++i) - { - CLScheduler::get().enqueue(*_reduction_kernels_vector[i], false); - } + CLScheduler::get().enqueue(*_arg_min_max_kernel, false); _reshape.run(); } } // namespace arm_compute diff --git a/src/runtime/CL/functions/CLDeconvolutionLayer.cpp b/src/runtime/CL/functions/CLDeconvolutionLayer.cpp index 5c25cbafaf02b54db36a7b4c43a0eb231c902810..4421a18f2a7f39f5ce7ea5fdbd520718a627894e 100644 --- a/src/runtime/CL/functions/CLDeconvolutionLayer.cpp +++ b/src/runtime/CL/functions/CLDeconvolutionLayer.cpp @@ -23,6 +23,7 @@ */ #include "arm_compute/runtime/CL/functions/CLDeconvolutionLayer.h" +#include "arm_compute/core/Types.h" #include "arm_compute/core/Utils.h" #include "arm_compute/core/Validate.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" @@ -155,7 +156,9 @@ DeconvolutionMethod CLDeconvolutionLayer::get_deconvolution_method(const ITensor if(weights->dimension(idx_w) != deconv_info.stride().first || weights->dimension(idx_h) != deconv_info.stride().second) { - if(input->data_layout() == DataLayout::NHWC && ofm <= 16) + // We observe better performance for FP32 types only when ofm <= 16. + // A better heuristic is required for selecting the method for FP16 data types. + if(input->data_layout() == DataLayout::NHWC && !((input->data_type() == DataType::F32) && (ofm > 16))) { return DeconvolutionMethod::DIRECT; } diff --git a/src/runtime/CL/functions/CLDirectDeconvolutionLayer.cpp b/src/runtime/CL/functions/CLDirectDeconvolutionLayer.cpp index b263d73e9e50518f96547152ef25d3e093457d4c..88c3c6193c3c5fe5943bdb2727f90ec602ab3833 100644 --- a/src/runtime/CL/functions/CLDirectDeconvolutionLayer.cpp +++ b/src/runtime/CL/functions/CLDirectDeconvolutionLayer.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -67,8 +67,8 @@ Status CLDirectDeconvolutionLayer::validate(const ITensorInfo *input, const ITen const size_t idx_h = get_data_layout_dimension_index(data_layout, DataLayoutDimension::HEIGHT); const size_t idx_c = get_data_layout_dimension_index(data_layout, DataLayoutDimension::CHANNEL); - ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(idx_w) != weights->dimension(idx_h)); ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(idx_w) < 1); + ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(idx_h) < 1); auto out_dims = deconvolution_output_dimensions(input->dimension(idx_w), input->dimension(idx_h), weights->dimension(idx_w), weights->dimension(idx_h), info); diff --git a/src/runtime/CL/functions/CLGEMMDeconvolutionLayer.cpp b/src/runtime/CL/functions/CLGEMMDeconvolutionLayer.cpp index 8af83e2280da3583b92eee6936581a02237ad9d2..9fc81c11dad818fd99563520bed3e964d6b0f1f7 100644 --- a/src/runtime/CL/functions/CLGEMMDeconvolutionLayer.cpp +++ b/src/runtime/CL/functions/CLGEMMDeconvolutionLayer.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -27,6 +27,7 @@ #include "arm_compute/core/Validate.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" #include "arm_compute/core/utils/quantization/AsymmHelpers.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "arm_compute/runtime/CL/CLScheduler.h" #include "src/core/CL/kernels/CLDeconvolutionReshapeOutputKernel.h" #include "src/core/CL/kernels/CLFillBorderKernel.h" diff --git a/src/runtime/CL/functions/CLMatMul.cpp b/src/runtime/CL/functions/CLMatMul.cpp index ae5a01f6796d95adc0161e54ae0f267327b2b7a6..bef422fca1e7bea7866e4476510a609d27e49d17 100644 --- a/src/runtime/CL/functions/CLMatMul.cpp +++ b/src/runtime/CL/functions/CLMatMul.cpp @@ -42,25 +42,26 @@ CLMatMul::CLMatMul() CLMatMul::~CLMatMul() = default; -void CLMatMul::configure(ICLTensor *lhs, ICLTensor *rhs, ICLTensor *output, const MatMulInfo &matmul_info, const GpuMatMulSettings &settings) +void CLMatMul::configure(ICLTensor *lhs, ICLTensor *rhs, ICLTensor *output, const MatMulInfo &matmul_info, const GpuMatMulSettings &settings, const ActivationLayerInfo &act_info) { ARM_COMPUTE_UNUSED(settings); - configure(CLKernelLibrary::get().get_compile_context(), lhs, rhs, output, matmul_info); + configure(CLKernelLibrary::get().get_compile_context(), lhs, rhs, output, matmul_info, settings, act_info); } -void CLMatMul::configure(const CLCompileContext &compile_context, ICLTensor *lhs, ICLTensor *rhs, ICLTensor *output, const MatMulInfo &matmul_info, const GpuMatMulSettings &settings) +void CLMatMul::configure(const CLCompileContext &compile_context, ICLTensor *lhs, ICLTensor *rhs, ICLTensor *output, const MatMulInfo &matmul_info, const GpuMatMulSettings &settings, + const ActivationLayerInfo &act_info) { ARM_COMPUTE_ERROR_ON_NULLPTR(lhs, rhs, output); ARM_COMPUTE_UNUSED(settings); _impl->op = std::make_unique(); - _impl->op->configure(compile_context, lhs->info(), rhs->info(), output->info(), matmul_info); + _impl->op->configure(compile_context, lhs->info(), rhs->info(), output->info(), matmul_info, act_info); _impl->run_pack = { { ACL_SRC_0, lhs }, { ACL_SRC_1, rhs }, { ACL_DST, output } }; } -Status CLMatMul::validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *output, const MatMulInfo &matmul_info) +Status CLMatMul::validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *output, const MatMulInfo &matmul_info, const ActivationLayerInfo &act_info) { - return OperatorType::validate(lhs, rhs, output, matmul_info); + return OperatorType::validate(lhs, rhs, output, matmul_info, act_info); } void CLMatMul::run() diff --git a/src/runtime/CL/functions/CLReduceMean.cpp b/src/runtime/CL/functions/CLReduceMean.cpp index 6a2fcc600a02dfe060951ec394137622dcfde5c7..cddbf77d7cf9130901d53ae335671fd0a9d3cd4c 100644 --- a/src/runtime/CL/functions/CLReduceMean.cpp +++ b/src/runtime/CL/functions/CLReduceMean.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -82,7 +82,7 @@ Status validate_config(const ITensorInfo *input, const Coordinates &reduction_ax ARM_COMPUTE_RETURN_ERROR_ON(i > static_cast(axis_local[i])); const unsigned int remove_index = axis_local[i] - i; ARM_COMPUTE_RETURN_ERROR_ON(remove_index >= out_shape.num_dimensions()); - out_shape.remove_dimension(remove_index); + out_shape.remove_dimension(remove_index, false); } } const TensorInfo out_info = input->clone()->set_tensor_shape(out_shape); @@ -182,7 +182,7 @@ void CLReduceMean::configure(const CLCompileContext &compile_context, ICLTensor std::sort(axis_local.begin(), axis_local.begin() + _reduction_ops); for(int i = 0; i < _reduction_ops; ++i) { - out_shape.remove_dimension(axis_local[i] - i); + out_shape.remove_dimension(axis_local[i] - i, false); } auto_init_if_empty(*tmp_output->info(), tmp_input->info()->clone()->set_tensor_shape(out_shape)); _reshape.configure(compile_context, &_reduced_outs[_reduction_ops - 1], tmp_output); diff --git a/src/runtime/CPP/CPPScheduler.cpp b/src/runtime/CPP/CPPScheduler.cpp index 39811ec156129ed7da45f6d7479d41ec9dc435f0..45e872428f832c82b85eb77a8e38c369e0288613 100644 --- a/src/runtime/CPP/CPPScheduler.cpp +++ b/src/runtime/CPP/CPPScheduler.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2022 Arm Limited. + * Copyright (c) 2016-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -172,7 +172,7 @@ public: void start(); /** Wait for the current kernel execution to complete. */ - void wait(); + std::exception_ptr wait(); /** Function ran by the worker thread. */ void worker_thread(); @@ -244,17 +244,13 @@ void Thread::start() _cv.notify_one(); } -void Thread::wait() +std::exception_ptr Thread::wait() { { std::unique_lock lock(_m); _cv.wait(lock, [&] { return _job_complete; }); } - - if(_current_exception) - { - std::rethrow_exception(_current_exception); - } + return _current_exception; } void Thread::worker_thread() @@ -509,15 +505,34 @@ void CPPScheduler::run_workloads(std::vector &workloads) thread_it->start(); } info.thread_id = t; // Set main thread's thread_id - process_workloads(workloads, feeder, info); // Main thread processes workloads + std::exception_ptr last_exception = nullptr; #ifndef ARM_COMPUTE_EXCEPTIONS_DISABLED + try + { +#endif /* ARM_COMPUTE_EXCEPTIONS_DISABLED */ + process_workloads(workloads, feeder, info); // Main thread processes workloads +#ifndef ARM_COMPUTE_EXCEPTIONS_DISABLED + } + catch (...) + { + last_exception = std::current_exception(); + } + try { #endif /* ARM_COMPUTE_EXCEPTIONS_DISABLED */ thread_it = _impl->_threads.begin(); for(unsigned int i = 0; i < num_threads_to_use - 1; ++i, ++thread_it) { - thread_it->wait(); + std::exception_ptr current_exception = thread_it->wait(); + if (current_exception) + { + last_exception = current_exception; + } + } + if (last_exception) + { + std::rethrow_exception(last_exception); } #ifndef ARM_COMPUTE_EXCEPTIONS_DISABLED } diff --git a/src/runtime/NEON/functions/NEAddMulAdd.cpp b/src/runtime/NEON/functions/NEAddMulAdd.cpp index 55008de5d66d6e9d6b9bf9b514ef6707a58370c4..cfeaefc4fd251f976ec2540308ff827f6058dffc 100644 --- a/src/runtime/NEON/functions/NEAddMulAdd.cpp +++ b/src/runtime/NEON/functions/NEAddMulAdd.cpp @@ -24,6 +24,7 @@ #include "arm_compute/runtime/NEON/functions/NEAddMulAdd.h" +#include "arm_compute/runtime/Tensor.h" #include "src/common/utils/Log.h" #include "src/core/helpers/MemoryHelpers.h" #include "src/cpu/operators/CpuAddMulAdd.h" diff --git a/src/runtime/NEON/functions/NEDeconvolutionLayer.cpp b/src/runtime/NEON/functions/NEDeconvolutionLayer.cpp index 8534d2a8f34173b28df41ae0774d369433ca31d4..439aff0840eb395e2726672afdc0c267bfa1bbaf 100644 --- a/src/runtime/NEON/functions/NEDeconvolutionLayer.cpp +++ b/src/runtime/NEON/functions/NEDeconvolutionLayer.cpp @@ -89,8 +89,8 @@ Status NEDeconvolutionLayer::validate(const ITensorInfo *input, const ITensorInf ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::F32, DataType::F16, DataType::QASYMM8, DataType::QASYMM8_SIGNED); const unsigned int width_idx = get_data_layout_dimension_index(weights->data_layout(), DataLayoutDimension::WIDTH); const unsigned int height_idx = get_data_layout_dimension_index(weights->data_layout(), DataLayoutDimension::HEIGHT); - ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(width_idx) != weights->dimension(height_idx)); ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(width_idx) < 1); + ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(height_idx) < 1); ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_LAYOUT(weights, input); if(is_data_type_quantized_per_channel(weights->data_type()) && is_data_type_quantized(input->data_type())) { diff --git a/src/runtime/NEON/functions/NEMatMul.cpp b/src/runtime/NEON/functions/NEMatMul.cpp index 0c46516f1e0a2749abb726bc2f0bb8b64e1f6618..58640f40eaf1449318f31945569b5232574ef393 100644 --- a/src/runtime/NEON/functions/NEMatMul.cpp +++ b/src/runtime/NEON/functions/NEMatMul.cpp @@ -25,9 +25,9 @@ #include "arm_compute/core/Validate.h" #include "arm_compute/runtime/MemoryGroup.h" +#include "arm_compute/runtime/Tensor.h" #include "src/core/helpers/MemoryHelpers.h" #include "src/cpu/operators/CpuMatMul.h" -#include "arm_compute/runtime/Tensor.h" namespace arm_compute { @@ -49,7 +49,7 @@ NEMatMul::NEMatMul() NEMatMul::~NEMatMul() = default; -void NEMatMul::configure(ITensor *lhs, ITensor *rhs, ITensor *output, const MatMulInfo &info, const CpuMatMulSettings &settings) +void NEMatMul::configure(ITensor *lhs, ITensor *rhs, ITensor *output, const MatMulInfo &info, const CpuMatMulSettings &settings, const ActivationLayerInfo &act_info) { _impl->lhs = lhs; _impl->rhs = rhs; @@ -57,14 +57,14 @@ void NEMatMul::configure(ITensor *lhs, ITensor *rhs, ITensor *output, const MatM ARM_COMPUTE_ERROR_ON_NULLPTR(_impl->lhs, _impl->rhs, _impl->output); _impl->op = std::make_unique(); - _impl->op->configure(lhs->info(), rhs->info(), output->info(), info, settings); + _impl->op->configure(lhs->info(), rhs->info(), output->info(), info, settings, act_info); _impl->run_pack = { { ACL_SRC_0, lhs }, { ACL_SRC_1, rhs }, { ACL_DST, output } }; _impl->workspace_tensors = manage_workspace(_impl->op->workspace(), _impl->memory_group, _impl->run_pack); } -Status NEMatMul::validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *output, const MatMulInfo &info, const CpuMatMulSettings &settings) +Status NEMatMul::validate(const ITensorInfo *lhs, const ITensorInfo *rhs, const ITensorInfo *output, const MatMulInfo &info, const CpuMatMulSettings &settings, const ActivationLayerInfo &act_info) { - return cpu::CpuMatMul::validate(lhs, rhs, output, info, settings); + return cpu::CpuMatMul::validate(lhs, rhs, output, info, settings, act_info); } void NEMatMul::run() diff --git a/src/runtime/NEON/functions/NEReduceMean.cpp b/src/runtime/NEON/functions/NEReduceMean.cpp index 9c9b79a1e51910bca514bcd146c900d33339eecd..9f964792951384a017d535bcc782c537bf6592dd 100644 --- a/src/runtime/NEON/functions/NEReduceMean.cpp +++ b/src/runtime/NEON/functions/NEReduceMean.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022 Arm Limited. + * Copyright (c) 2018-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -78,7 +78,7 @@ Status validate_config(const ITensorInfo *input, const Coordinates &reduction_ax ARM_COMPUTE_RETURN_ERROR_ON(i > static_cast(axis_local[i])); const unsigned int remove_index = axis_local[i] - i; ARM_COMPUTE_RETURN_ERROR_ON(remove_index >= out_shape.num_dimensions()); - out_shape.remove_dimension(remove_index); + out_shape.remove_dimension(remove_index, false); } } const TensorInfo out_info = input->clone()->set_tensor_shape(out_shape); @@ -156,7 +156,7 @@ void NEReduceMean::configure(ITensor *input, const Coordinates &reduction_axis, std::sort(axis_local.begin(), axis_local.begin() + _reduction_ops); for(int i = 0; i < _reduction_ops; ++i) { - out_shape.remove_dimension(axis_local[i] - i); + out_shape.remove_dimension(axis_local[i] - i, false); } auto_init_if_empty(*tmp_output->info(), tmp_input->info()->clone()->set_tensor_shape(out_shape)); _reshape.configure(&_reduced_outs[_reduction_ops - 1], tmp_output); diff --git a/src/runtime/NEON/functions/NEScale.cpp b/src/runtime/NEON/functions/NEScale.cpp index 686017f7c185d6d348fbfe9608c032663a1519c1..09f037334e9f04cdd3afb43833e337fe94f5cf8c 100644 --- a/src/runtime/NEON/functions/NEScale.cpp +++ b/src/runtime/NEON/functions/NEScale.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2022 Arm Limited. + * Copyright (c) 2016-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,6 +23,7 @@ */ #include "arm_compute/runtime/NEON/functions/NEScale.h" +#include "arm_compute/runtime/Tensor.h" #include "src/common/utils/Log.h" #include "src/core/utils/ScaleUtils.h" #include "src/cpu/operators/CpuScale.h" diff --git a/src/runtime/heuristics/dwc_native/ClDWCNativeDefaultConfigBifrost.cpp b/src/runtime/heuristics/dwc_native/ClDWCNativeDefaultConfigBifrost.cpp index f55685ee499712ece027e0fa9ca959c3c7228b52..5311fdcec3fc6e92b618e4a956532003bb6a7ff3 100644 --- a/src/runtime/heuristics/dwc_native/ClDWCNativeDefaultConfigBifrost.cpp +++ b/src/runtime/heuristics/dwc_native/ClDWCNativeDefaultConfigBifrost.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -28,6 +28,7 @@ #include "arm_compute/core/GPUTarget.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/TensorShape.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" namespace arm_compute { diff --git a/src/runtime/heuristics/dwc_native/ClDWCNativeDefaultConfigValhall.cpp b/src/runtime/heuristics/dwc_native/ClDWCNativeDefaultConfigValhall.cpp index 49485c83a961d52a2f78bb90e4d2dd93adac4789..51f3787875ff60bec3be6e330ae26236630ff0da 100644 --- a/src/runtime/heuristics/dwc_native/ClDWCNativeDefaultConfigValhall.cpp +++ b/src/runtime/heuristics/dwc_native/ClDWCNativeDefaultConfigValhall.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -28,6 +28,7 @@ #include "arm_compute/core/GPUTarget.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/TensorShape.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" namespace arm_compute { diff --git a/src/runtime/heuristics/matmul_native/ClMatMulNativeHelpers.cpp b/src/runtime/heuristics/matmul_native/ClMatMulNativeHelpers.cpp index b9e0d5adf86a940744ed58fcb7712165cc4ea19f..1e06e84d4d79914731bb254544bb9ba6649c0e5f 100644 --- a/src/runtime/heuristics/matmul_native/ClMatMulNativeHelpers.cpp +++ b/src/runtime/heuristics/matmul_native/ClMatMulNativeHelpers.cpp @@ -52,7 +52,7 @@ MatMulKernelInfo select_info(const MatMulKernelInfo &info0, if(rhs_lock_padding == false) { - if(bool(opencl::kernels::ClMatMulNativeKernel::validate(&lhs_info, &rhs_info, &dst_info, info0))) + if(bool(opencl::kernels::ClMatMulNativeKernel::validate(&lhs_info, &rhs_info, nullptr, &dst_info, info0))) { return info0; } diff --git a/src/runtime/heuristics/matmul_native/IClMatMulNativeKernelConfig.h b/src/runtime/heuristics/matmul_native/IClMatMulNativeKernelConfig.h index ee9b03e63b3c60743e8d87b88fcb5f821ff6e247..4f548bd01d6df6b0948e53b13e59561c9f1a2489 100644 --- a/src/runtime/heuristics/matmul_native/IClMatMulNativeKernelConfig.h +++ b/src/runtime/heuristics/matmul_native/IClMatMulNativeKernelConfig.h @@ -27,6 +27,7 @@ #include "arm_compute/core/GPUTarget.h" #include "arm_compute/core/KernelDescriptors.h" #include "arm_compute/core/Types.h" +#include "arm_compute/function_info/MatMulInfo.h" #include "src/core/common/Macros.h" namespace arm_compute @@ -110,6 +111,6 @@ public: protected: GPUTarget _target; }; -} // namespace opencl +} // namespace cl_matmul } // namespace arm_compute #endif /* SRC_RUNTIME_HEURISTICS_MATMUL_NATIVE_ICLMATMULNATIVEKERNELCONFIG */ diff --git a/support/ToolchainSupport.h b/support/ToolchainSupport.h index 0557d1d775eb5a1b2cee8306d3b46163ad3b95a6..96826dad5ec4d2f7ae6dedcd151f9ba9340e7475 100644 --- a/support/ToolchainSupport.h +++ b/support/ToolchainSupport.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2022 Arm Limited. + * Copyright (c) 2017-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -33,10 +33,6 @@ #include #include -#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC -#include -#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC - #include "support/Bfloat16.h" #include "support/Half.h" @@ -46,6 +42,10 @@ namespace arm_compute { +#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC +typedef __fp16 float16_t; +#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + namespace support { namespace cpp11 diff --git a/tests/IAccessor.h b/tests/IAccessor.h index c54c00e99e65cf6353ca1777c536ca6379dd8f27..75faee19ce1c237d6384dd821762db85fcc9de29 100644 --- a/tests/IAccessor.h +++ b/tests/IAccessor.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 Arm Limited. + * Copyright (c) 2017-2019, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,7 @@ #include "arm_compute/core/Coordinates.h" #include "arm_compute/core/Types.h" +#include "arm_compute/core/QuantizationInfo.h" namespace arm_compute { diff --git a/tests/SimpleTensor.h b/tests/SimpleTensor.h index 9ea171d492e94434db766bdb42361f97c96e3721..419621e8088456050acdf8091874c7c9057c2f2e 100644 --- a/tests/SimpleTensor.h +++ b/tests/SimpleTensor.h @@ -392,6 +392,8 @@ int SimpleTensor::num_channels() const case Format::S16: case Format::U32: case Format::S32: + case Format::U64: + case Format::S64: case Format::F16: case Format::F32: return 1; diff --git a/tests/benchmark/fixtures/ScaleFixture.h b/tests/benchmark/fixtures/ScaleFixture.h index 953872ea64077a51606efcbd636e66cb98896004..81f34bd538559fb7d5fd41ab137d135af4c13ea5 100644 --- a/tests/benchmark/fixtures/ScaleFixture.h +++ b/tests/benchmark/fixtures/ScaleFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2020 Arm Limited. + * Copyright (c) 2017-2020, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -40,7 +40,6 @@ template class ScaleFixture : public framework::Fixture { public: - template void setup(TensorShape shape, DataType data_type, DataLayout data_layout, InterpolationPolicy policy, BorderMode border_mode, SamplingPolicy sampling_policy) { constexpr float max_width = 8192.0f; diff --git a/tests/benchmark/fixtures/ScaleLayerFixture.h b/tests/benchmark/fixtures/ScaleLayerFixture.h index dbcdd336611aa54dc6af89ddd068b01e76386419..59cc12c77de2be71cbef27a29089386cff83a850 100644 --- a/tests/benchmark/fixtures/ScaleLayerFixture.h +++ b/tests/benchmark/fixtures/ScaleLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -40,7 +40,6 @@ template class ScaleLayerFixture : public framework::Fixture { public: - template void setup(TensorShape shape, InterpolationPolicy policy, BorderMode border_mode, SamplingPolicy sampling_policy, float sx, float sy, DataType data_type) { constexpr float max_width = 8192.0f; diff --git a/tests/benchmark_examples/RunExample.cpp b/tests/benchmark_examples/RunExample.cpp index a23d91e9deea4d840049a425ca2def55d74e856e..3e56ea2e646efd4942671f34145106aa890546af 100644 --- a/tests/benchmark_examples/RunExample.cpp +++ b/tests/benchmark_examples/RunExample.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021,2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,7 @@ #define BENCHMARK_EXAMPLES #include "utils/Utils.cpp" +#include "arm_compute/core/Version.h" #include "arm_compute/runtime/Scheduler.h" #include "tests/framework/Framework.h" #include "tests/framework/Macros.h" diff --git a/tests/datasets/DepthwiseConvolutionLayerDataset.h b/tests/datasets/DepthwiseConvolutionLayerDataset.h index d0e8efcde50586310399e4a9384589c0d341569e..f88cb887fc54c14767df587762b44816917f429f 100644 --- a/tests/datasets/DepthwiseConvolutionLayerDataset.h +++ b/tests/datasets/DepthwiseConvolutionLayerDataset.h @@ -152,6 +152,8 @@ public: add_config(TensorShape(33U, 27U, 7U), Size2D(5U, 7U), PadStrideInfo(3, 2, 1, 3, 0, 2, DimensionRoundingType::FLOOR)); add_config(TensorShape(33U, 27U, 7U), Size2D(5U, 7U), PadStrideInfo(3, 2, 1, 0, 1, 0, DimensionRoundingType::FLOOR)); add_config(TensorShape(33U, 27U, 7U), Size2D(5U, 7U), PadStrideInfo(3, 2, 0, 1, 0, 1, DimensionRoundingType::FLOOR)); + // Padding greater than kernel size. + add_config(TensorShape(128, 56, 56), Size2D(4, 4), PadStrideInfo(2, 2, 0, 10, 0, 10, DimensionRoundingType::FLOOR)); } }; diff --git a/tests/datasets/LargeMatMulMMULDataset.h b/tests/datasets/LargeMatMulMMULDataset.h new file mode 100644 index 0000000000000000000000000000000000000000..23e0b3e5c86f5b252ce2f5f923e6ba4d60606ace --- /dev/null +++ b/tests/datasets/LargeMatMulMMULDataset.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef ACL_TESTS_DATASETS_LARGEMATMULMMULDATASET +#define ACL_TESTS_DATASETS_LARGEMATMULMMULDATASET + +#include "arm_compute/core/TensorShape.h" +#include "arm_compute/core/Types.h" +#include "tests/datasets/MatMulDataset.h" + +namespace arm_compute +{ +namespace test +{ +namespace datasets +{ +/** MatMul MMUL shapes are similar to MatMul shapes except that K has to be a multiple of MMUL_K0 which is 4 (e.g. see src/gpu/cl/kernels/ClMatMulNativeMMULKernel.cpp for the definition) + */ +class LargeMatMulMMULDataset final : public MatMulDataset +{ +public: + LargeMatMulMMULDataset() + { + add_config(TensorShape(24U, 13U, 3U, 2U), TensorShape(33U, 24U, 3U, 2U), TensorShape(33U, 13U, 3U, 2U)); + add_config(TensorShape(36U, 12U, 1U, 5U), TensorShape(21U, 36U, 1U, 5U), TensorShape(21U, 12U, 1U, 5U)); + add_config(TensorShape(44U, 38U, 3U, 2U), TensorShape(21U, 44U, 3U, 2U), TensorShape(21U, 38U, 3U, 2U)); + } +}; + +class HighDimensionalMatMulMMULDataset final : public MatMulDataset +{ +public: + HighDimensionalMatMulMMULDataset() + { + add_config(TensorShape(4U, 5U, 2U, 2U, 2U, 2U), TensorShape(5U, 4U, 2U, 2U, 2U, 2U), TensorShape(5U, 5U, 2U, 2U, 2U, 2U)); // 6D tensor + } +}; + +} // namespace datasets +} // namespace test +} // namespace arm_compute + +#endif /* ACL_TESTS_DATASETS_LARGEMATMULMMULDATASET */ diff --git a/tests/datasets/SmallMatMulMMULDataset.h b/tests/datasets/SmallMatMulMMULDataset.h new file mode 100644 index 0000000000000000000000000000000000000000..9e517488af48086ed7f836c0f82e2464b40f5dda --- /dev/null +++ b/tests/datasets/SmallMatMulMMULDataset.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef ACL_TESTS_DATASETS_SMALLMATMULMMULDATASET +#define ACL_TESTS_DATASETS_SMALLMATMULMMULDATASET + +#include "arm_compute/core/TensorShape.h" +#include "arm_compute/core/Types.h" +#include "tests/datasets/MatMulDataset.h" + +namespace arm_compute +{ +namespace test +{ +namespace datasets +{ +/** MatMul MMUL shapes are similar to MatMul shapes except that K has to be a multiple of MMUL_K0 which is 4 (e.g. see src/gpu/cl/kernels/ClMatMulNativeMMULKernel.cpp for the definition) + */ +class SmallMatMulMMULDataset final : public MatMulDataset +{ +public: + SmallMatMulMMULDataset() + { + add_config(TensorShape(8U, 4U, 2U, 2U), TensorShape(2U, 8U, 2U, 2U), TensorShape(2U, 4U, 2U, 2U)); + add_config(TensorShape(28U, 1U), TensorShape(23U, 28U), TensorShape(23U, 1U)); + add_config(TensorShape(8U, 4U, 2U), TensorShape(16U, 8U, 2U), TensorShape(16U, 4U, 2U)); + add_config(TensorShape(32U, 2U), TensorShape(17U, 32U), TensorShape(17U, 2U)); + add_config(TensorShape(8U, 6U), TensorShape(7U, 8U), TensorShape(7U, 6U)); + } +}; + +class TinyMatMulMMULDataset final : public MatMulDataset +{ +public: + TinyMatMulMMULDataset() + { + add_config(TensorShape(4U, 4U), TensorShape(4U, 4U), TensorShape(4U, 4U)); + } +}; + +} // namespace datasets +} // namespace test +} // namespace arm_compute + +#endif /* ACL_TESTS_DATASETS_SMALLMATMULMMULDATASET */ diff --git a/tests/framework/Framework.cpp b/tests/framework/Framework.cpp index a3dee07862e3067b8af05be7202eb77e1114130e..bfb955c525061399493da0f9dd5a63acffab2da3 100644 --- a/tests/framework/Framework.cpp +++ b/tests/framework/Framework.cpp @@ -130,10 +130,12 @@ Framework &Framework::get() void Framework::init(const FrameworkConfig &config) { _test_filter.reset(new TestFilter(config.mode, config.name_filter, config.id_filter)); - _num_iterations = config.num_iterations; - _log_level = config.log_level; - _cooldown_sec = config.cooldown_sec; - _configure_only = config.configure_only; + _num_iterations = config.num_iterations; + _log_level = config.log_level; + _cooldown_sec = config.cooldown_sec; + _configure_only = config.configure_only; + _print_rerun_cmd = config.print_rerun_cmd; + _seed = config.seed; _instruments = std::set(std::begin(config.instruments), std::end(config.instruments)); } @@ -292,13 +294,13 @@ bool Framework::error_on_missing_assets() const return _error_on_missing_assets; } -void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory) +TestResult::Status Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory) { if(test_factory.status() == TestCaseFactory::Status::DISABLED) { log_test_skipped(info); set_test_result(info, TestResult(TestResult::Status::DISABLED)); - return; + return TestResult::Status::DISABLED; } log_test_start(info); @@ -538,6 +540,7 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory) set_test_result(info, result); log_test_end(info); + return result.status; } bool Framework::run() @@ -581,9 +584,11 @@ bool Framework::run() CLScheduler::get().set_queue(new_queue); } #endif // ARM_COMPUTE_CL - - run_test(test_info, *test_factory); - + TestResult::Status result = run_test(test_info, *test_factory); + if((_print_rerun_cmd) && (result == TestResult::Status::CRASHED || result == TestResult::Status::FAILED)) + { + std::cout << "Rerun command: ./arm_compute_validation --filter='^" << test_info.name << "$' --seed=" << _seed << std::endl; + } ++id_run_test; // Run test delay diff --git a/tests/framework/Framework.h b/tests/framework/Framework.h index 311f177f8d3ae403f44c1bf478041a5753f19441..2dded300384c609811518a39551b2eb5275fef90 100644 --- a/tests/framework/Framework.h +++ b/tests/framework/Framework.h @@ -64,6 +64,8 @@ struct FrameworkConfig float cooldown_sec{ -1.f }; /**< Delay between tests in seconds. */ LogLevel log_level{ LogLevel::NONE }; /**< Verbosity of the output. */ bool configure_only{ false }; /**< Only configure kernels */ + bool print_rerun_cmd{ false }; /**< Print the command to rerun the failed testcase */ + unsigned int seed{ 0 }; /**< The seed that is used to fill tensors with random values.*/ }; /** Information about a test case. @@ -328,7 +330,7 @@ private: Framework(const Framework &) = delete; Framework &operator=(const Framework &) = delete; - void run_test(const TestInfo &info, TestCaseFactory &test_factory); + TestResult::Status run_test(const TestInfo &info, TestCaseFactory &test_factory); std::map count_test_results() const; /** Returns the current test suite name. @@ -356,6 +358,8 @@ private: std::vector _printers{}; bool _configure_only{ false }; bool _new_fixture_call{ false }; + bool _print_rerun_cmd{ false }; + unsigned int _seed{ 0 }; using create_function = std::unique_ptr(); std::map _available_instruments{}; diff --git a/tests/framework/Macros.h b/tests/framework/Macros.h index ac03bb02b6267d4fdb90350b880bdcb47a221983..5ce0842864fc78a001213daf4781a0278ade656d 100644 --- a/tests/framework/Macros.h +++ b/tests/framework/Macros.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -127,13 +127,13 @@ void do_setup() override \ { \ framework::Framework::get().set_new_fixture_call(false); \ - apply(this, &FIXTURE::setup, _data); \ + apply(this, &FIXTURE::setup, _data); \ } #define FIXTURE_DATA_SETUP_NEW(FIXTURE) \ void do_setup() override \ { \ framework::Framework::get().set_new_fixture_call(true); \ - apply(this, &FIXTURE::setup, _data); \ + apply(this, &FIXTURE::setup, _data); \ configure_target(); \ if(!framework::Framework::get().configure_only()) \ { \ @@ -324,4 +324,4 @@ // // TEST CASE MACROS END // -#endif /* ARM_COMPUTE_TEST_FRAMEWORK_MACROS */ \ No newline at end of file +#endif /* ARM_COMPUTE_TEST_FRAMEWORK_MACROS */ diff --git a/tests/main.cpp b/tests/main.cpp index 58347cad427d7dff4cfffde3e6734b7cecf59c77..e862c7627e2083500bf14e8b91ad8f6d4481ae28 100644 --- a/tests/main.cpp +++ b/tests/main.cpp @@ -21,6 +21,7 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ +#include "arm_compute/core/Version.h" #include "support/StringSupport.h" #include "tests/AssetsLibrary.h" #include "tests/framework/DatasetModes.h" @@ -127,6 +128,8 @@ int main(int argc, char **argv) error_on_missing_assets->set_help("Mark a test as failed instead of skipping it when assets are missing"); auto assets = parser.add_positional_option>("assets"); assets->set_help("Path to the assets directory"); + auto print_rerun_command = parser.add_option("rerun-cmd"); + print_rerun_command->set_help("Print out the command to rerun the exact failed testcase"); #ifdef ARM_COMPUTE_CL auto enable_tuner = parser.add_option("enable-tuner"); enable_tuner->set_help("Enable OpenCL dynamic tuner"); @@ -270,14 +273,16 @@ int main(int argc, char **argv) // Initialize framework framework::FrameworkConfig fconfig; - fconfig.instruments = options.instruments->value(); - fconfig.name_filter = filter->value(); - fconfig.id_filter = filter_id->value(); - fconfig.num_iterations = options.iterations->value(); - fconfig.mode = dataset_mode->value(); - fconfig.log_level = options.log_level->value(); - fconfig.cooldown_sec = cooldown_sec->value(); - fconfig.configure_only = configure_only->value(); + fconfig.instruments = options.instruments->value(); + fconfig.name_filter = filter->value(); + fconfig.id_filter = filter_id->value(); + fconfig.num_iterations = options.iterations->value(); + fconfig.mode = dataset_mode->value(); + fconfig.log_level = options.log_level->value(); + fconfig.cooldown_sec = cooldown_sec->value(); + fconfig.configure_only = configure_only->value(); + fconfig.print_rerun_cmd = print_rerun_command->value(); + fconfig.seed = seed->value(); framework.init(fconfig); for(auto &p : printers) diff --git a/tests/validation/CL/ArgMinMax.cpp b/tests/validation/CL/ArgMinMax.cpp index 1d849ed0c7a7168ff9ad364da69d468953d5275f..8566972f819f6fdb3239b4f5261ea74736c73694 100644 --- a/tests/validation/CL/ArgMinMax.cpp +++ b/tests/validation/CL/ArgMinMax.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,15 +22,11 @@ * SOFTWARE. */ #include "arm_compute/core/Types.h" -#include "arm_compute/core/utils/misc/ShapeCalculator.h" #include "arm_compute/runtime/CL/CLTensor.h" #include "arm_compute/runtime/CL/CLTensorAllocator.h" #include "arm_compute/runtime/CL/functions/CLArgMinMaxLayer.h" -#include "arm_compute/runtime/CL/functions/CLReductionOperation.h" #include "tests/CL/CLAccessor.h" #include "tests/datasets/ShapeDatasets.h" -#include "tests/datasets/SplitDataset.h" -#include "tests/framework/Asserts.h" #include "tests/framework/Macros.h" #include "tests/validation/Validation.h" #include "tests/validation/fixtures/ArgMinMaxFixture.h" @@ -46,6 +42,8 @@ namespace const auto ArgMinMaxSmallDataset = framework::dataset::make("Shape", { TensorShape{ 1U, 7U, 1U, 3U }, + TensorShape{ 3U, 1U, 3U, 2U }, + TensorShape{ 2U, 1U, 3U, 2U }, TensorShape{ 149U, 5U, 1U, 2U }, TensorShape{ 166U, 5U, 1U, 2U }, TensorShape{ 322U, 5U, 1U, 2U }, @@ -53,6 +51,22 @@ const auto ArgMinMaxSmallDataset = framework::dataset::make("Shape", TensorShape{ 2560, 2U, 2U, 2U }, }); +const auto ArgMinMaxSmallDatasetAxis0 = framework::dataset::make("Shape", +{ + TensorShape{ 1U, 5U }, + TensorShape{ 2U, 3U }, + TensorShape{ 1U }, + TensorShape{ 3U }, + TensorShape{ 2U }, + TensorShape{ 5U }, + TensorShape{ 17U }, + TensorShape{ 15U, 2U }, +}); + +const auto OpsDataset = framework::dataset::make("Operation", { ReductionOperation::ARG_IDX_MIN, ReductionOperation::ARG_IDX_MAX }); +const auto AxisDataset = framework::dataset::make("Axis", { 0, 1, 2, 3 }); +const auto QInfoDataset = framework::dataset::make("QuantizationInfo", { QuantizationInfo(5.f / 255.f, 20) }); + const auto ArgMinMaxLargeDataset = framework::dataset::make("Shape", { TensorShape{ 517U, 123U, 13U, 2U } }); } // namespace @@ -85,47 +99,78 @@ DATA_TEST_CASE(Validate, framework::DatasetMode::ALL, zip(zip(zip(zip( // clang-format on // *INDENT-ON* -template -using CLArgMinMaxValidationFixture = ArgMinMaxValidationFixture; +template +using CLArgMinMaxValidationFixture = ArgMinMaxValidationFixture; + +using CLArgMinMaxValidationFixture_S32_S32 = CLArgMinMaxValidationFixture; +using CLArgMinMaxValidationFixture_F16_S32 = CLArgMinMaxValidationFixture; +using CLArgMinMaxValidationFixture_F32_S32 = CLArgMinMaxValidationFixture; +using CLArgMinMaxValidationFixture_F32_S64 = CLArgMinMaxValidationFixture; TEST_SUITE(S32) +FIXTURE_DATA_TEST_CASE(RunSmallAxis0, + CLArgMinMaxValidationFixture_S32_S32, + framework::DatasetMode::PRECOMMIT, + combine(combine(combine(combine(ArgMinMaxSmallDatasetAxis0, + framework::dataset::make("DataTypeIn", DataType::S32)), + framework::dataset::make("DataTypeOut", DataType::S32)), + framework::dataset::make("Axis", { 0 })), + OpsDataset)) +{ + // Validate output + validate(CLAccessor(_target), _reference); +} + FIXTURE_DATA_TEST_CASE(RunSmall, - CLArgMinMaxValidationFixture, + CLArgMinMaxValidationFixture_S32_S32, framework::DatasetMode::PRECOMMIT, - combine(combine(combine(ArgMinMaxSmallDataset, framework::dataset::make("DataType", DataType::S32)), framework::dataset::make("Axis", { 0, 1, 2, 3 })), - framework::dataset::make("Operation", { ReductionOperation::ARG_IDX_MIN, ReductionOperation::ARG_IDX_MAX }))) + combine(combine(combine(combine(ArgMinMaxSmallDataset, + framework::dataset::make("DataTypeIn", DataType::S32)), + framework::dataset::make("DataTypeOut", DataType::S32)), + AxisDataset), + OpsDataset)) { // Validate output validate(CLAccessor(_target), _reference); } FIXTURE_DATA_TEST_CASE(RunLarge, - CLArgMinMaxValidationFixture, + CLArgMinMaxValidationFixture_S32_S32, framework::DatasetMode::NIGHTLY, - combine(combine(combine(ArgMinMaxLargeDataset, framework::dataset::make("DataType", DataType::S32)), framework::dataset::make("Axis", { 0, 1, 2, 3 })), - framework::dataset::make("Operation", { ReductionOperation::ARG_IDX_MIN, ReductionOperation::ARG_IDX_MAX }))) + combine(combine(combine(combine(ArgMinMaxLargeDataset, + framework::dataset::make("DataTypeIn", DataType::S32)), + framework::dataset::make("DataTypeOut", DataType::S32)), + AxisDataset), + OpsDataset)) { // Validate output validate(CLAccessor(_target), _reference); } + TEST_SUITE_END() // S32 TEST_SUITE(Float) TEST_SUITE(FP16) FIXTURE_DATA_TEST_CASE(RunSmall, - CLArgMinMaxValidationFixture, + CLArgMinMaxValidationFixture_F16_S32, framework::DatasetMode::PRECOMMIT, - combine(combine(combine(ArgMinMaxSmallDataset, framework::dataset::make("DataType", DataType::F16)), framework::dataset::make("Axis", { 0, 1, 2, 3 })), - framework::dataset::make("Operation", { ReductionOperation::ARG_IDX_MIN, ReductionOperation::ARG_IDX_MAX }))) + combine(combine(combine(combine(ArgMinMaxSmallDataset, + framework::dataset::make("DataTypeIn", DataType::F16)), + framework::dataset::make("DataTypeOut", DataType::S32)), + AxisDataset), + OpsDataset)) { // Validate output validate(CLAccessor(_target), _reference); } FIXTURE_DATA_TEST_CASE(RunLarge, - CLArgMinMaxValidationFixture, + CLArgMinMaxValidationFixture_F16_S32, framework::DatasetMode::NIGHTLY, - combine(combine(combine(ArgMinMaxLargeDataset, framework::dataset::make("DataType", DataType::F16)), framework::dataset::make("Axis", { 0, 1, 2, 3 })), - framework::dataset::make("Operation", { ReductionOperation::ARG_IDX_MIN, ReductionOperation::ARG_IDX_MAX }))) + combine(combine(combine(combine(ArgMinMaxLargeDataset, + framework::dataset::make("DataTypeIn", DataType::F16)), + framework::dataset::make("DataTypeOut", DataType::S32)), + AxisDataset), + OpsDataset)) { // Validate output validate(CLAccessor(_target), _reference); @@ -134,49 +179,77 @@ TEST_SUITE_END() // FP16 TEST_SUITE(FP32) FIXTURE_DATA_TEST_CASE(RunSmall, - CLArgMinMaxValidationFixture, + CLArgMinMaxValidationFixture_F32_S32, framework::DatasetMode::PRECOMMIT, - combine(combine(combine(ArgMinMaxSmallDataset, framework::dataset::make("DataType", DataType::F32)), framework::dataset::make("Axis", { 0, 1, 2, 3 })), - framework::dataset::make("Operation", { ReductionOperation::ARG_IDX_MIN, ReductionOperation::ARG_IDX_MAX }))) + combine(combine(combine(combine(ArgMinMaxSmallDataset, + framework::dataset::make("DataTypeIn", DataType::F32)), + framework::dataset::make("DataTypeOut", DataType::S32)), + AxisDataset), + OpsDataset)) +{ + // Validate output + validate(CLAccessor(_target), _reference); +} + +FIXTURE_DATA_TEST_CASE(RunSmall_F32_S64, + CLArgMinMaxValidationFixture_F32_S64, + framework::DatasetMode::PRECOMMIT, + combine(combine(combine(combine(ArgMinMaxSmallDataset, + framework::dataset::make("DataTypeIn", DataType::F32)), + framework::dataset::make("DataTypeOut", DataType::S64)), + AxisDataset), + OpsDataset)) { // Validate output validate(CLAccessor(_target), _reference); } FIXTURE_DATA_TEST_CASE(RunLarge, - CLArgMinMaxValidationFixture, + CLArgMinMaxValidationFixture_F32_S32, framework::DatasetMode::NIGHTLY, - combine(combine(combine(ArgMinMaxLargeDataset, framework::dataset::make("DataType", DataType::F32)), framework::dataset::make("Axis", { 0, 1, 2, 3 })), - framework::dataset::make("Operation", { ReductionOperation::ARG_IDX_MIN, ReductionOperation::ARG_IDX_MAX }))) + combine(combine(combine(combine(ArgMinMaxLargeDataset, + framework::dataset::make("DataTypeIn", DataType::F32)), + framework::dataset::make("DataTypeOut", DataType::S32)), + AxisDataset), + OpsDataset)) { // Validate output validate(CLAccessor(_target), _reference); } + TEST_SUITE_END() // FP32 TEST_SUITE_END() // Float -template -using CLArgMinMaxQuantizedValidationFixture = ArgMinMaxValidationQuantizedFixture; +template +using CLArgMinMaxQuantizedValidationFixture = ArgMinMaxValidationQuantizedFixture; + +using CLArgMinMaxQuantizedValidationFixture_U8_S32 = CLArgMinMaxQuantizedValidationFixture; +using CLArgMinMaxQuantizedValidationFixture_S8_S32 = CLArgMinMaxQuantizedValidationFixture; TEST_SUITE(Quantized) TEST_SUITE(QASYMM8) FIXTURE_DATA_TEST_CASE(RunSmall, - CLArgMinMaxQuantizedValidationFixture, + CLArgMinMaxQuantizedValidationFixture_U8_S32, framework::DatasetMode::PRECOMMIT, - combine(combine(combine(combine(ArgMinMaxSmallDataset, framework::dataset::make("DataType", DataType::QASYMM8)), framework::dataset::make("Axis", { 0, 1, 2, 3 })), - framework::dataset::make("Operation", { ReductionOperation::ARG_IDX_MIN, ReductionOperation::ARG_IDX_MAX })), - framework::dataset::make("QuantizationInfo", { QuantizationInfo(5.f / 255.f, 20) }))) + combine(combine(combine(combine(combine(ArgMinMaxSmallDataset, + framework::dataset::make("DataTypeIn", DataType::QASYMM8)), + framework::dataset::make("DataTypeOut", DataType::S32)), + AxisDataset), + OpsDataset), + QInfoDataset)) { // Validate output validate(CLAccessor(_target), _reference); } - FIXTURE_DATA_TEST_CASE(RunLarge, - CLArgMinMaxQuantizedValidationFixture, + CLArgMinMaxQuantizedValidationFixture_U8_S32, framework::DatasetMode::NIGHTLY, - combine(combine(combine(combine(ArgMinMaxLargeDataset, framework::dataset::make("DataType", DataType::QASYMM8)), framework::dataset::make("Axis", { 0, 1, 2, 3 })), - framework::dataset::make("Operation", { ReductionOperation::ARG_IDX_MIN, ReductionOperation::ARG_IDX_MAX })), - framework::dataset::make("QuantizationInfo", { QuantizationInfo(5.f / 255.f, 20) }))) + combine(combine(combine(combine(combine(ArgMinMaxLargeDataset, + framework::dataset::make("DataTypeIn", DataType::QASYMM8)), + framework::dataset::make("DataTypeOut", DataType::S32)), + AxisDataset), + OpsDataset), + QInfoDataset)) { // Validate output validate(CLAccessor(_target), _reference); @@ -185,28 +258,32 @@ TEST_SUITE_END() // QASYMM8 TEST_SUITE(QASYMM8_SIGNED) FIXTURE_DATA_TEST_CASE(RunSmall, - CLArgMinMaxQuantizedValidationFixture, + CLArgMinMaxQuantizedValidationFixture_S8_S32, framework::DatasetMode::PRECOMMIT, - combine(combine(combine(combine(ArgMinMaxSmallDataset, framework::dataset::make("DataType", DataType::QASYMM8_SIGNED)), framework::dataset::make("Axis", { 0, 1, 2, 3 })), - framework::dataset::make("Operation", { ReductionOperation::ARG_IDX_MIN, ReductionOperation::ARG_IDX_MAX })), - framework::dataset::make("QuantizationInfo", { QuantizationInfo(5.f / 255.f, 20) }))) + combine(combine(combine(combine(combine(ArgMinMaxSmallDataset, + framework::dataset::make("DataTypeIn", DataType::QASYMM8_SIGNED)), + framework::dataset::make("DataTypeOut", DataType::S32)), + AxisDataset), + OpsDataset), + QInfoDataset)) { // Validate output validate(CLAccessor(_target), _reference); } - FIXTURE_DATA_TEST_CASE(RunLarge, - CLArgMinMaxQuantizedValidationFixture, + CLArgMinMaxQuantizedValidationFixture_S8_S32, framework::DatasetMode::NIGHTLY, - combine(combine(combine(combine(ArgMinMaxLargeDataset, framework::dataset::make("DataType", DataType::QASYMM8_SIGNED)), framework::dataset::make("Axis", { 0, 1, 2, 3 })), - framework::dataset::make("Operation", { ReductionOperation::ARG_IDX_MIN, ReductionOperation::ARG_IDX_MAX })), - framework::dataset::make("QuantizationInfo", { QuantizationInfo(5.f / 255.f, 20) }))) + combine(combine(combine(combine(combine(ArgMinMaxLargeDataset, + framework::dataset::make("DataTypeIn", DataType::QASYMM8_SIGNED)), + framework::dataset::make("DataTypeOut", DataType::S32)), + AxisDataset), + OpsDataset), + QInfoDataset)) { // Validate output validate(CLAccessor(_target), _reference); } TEST_SUITE_END() // QASYMM8_SIGNED - TEST_SUITE_END() // Quantized TEST_SUITE_END() // ArgMinMax TEST_SUITE_END() // CL diff --git a/tests/validation/CL/Cast.cpp b/tests/validation/CL/Cast.cpp index 84455ba941f3ac5213f5e8a4eeb046631c2e4ac2..ee1606d93c16532f112006f39154f2da299a9e19 100644 --- a/tests/validation/CL/Cast.cpp +++ b/tests/validation/CL/Cast.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2020, 2022 Arm Limited. + * Copyright (c) 2018-2020, 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -122,6 +122,26 @@ const auto CastF32toS16Dataset = combine(framework::dataset::make("DataType", Da const auto CastF32toU32Dataset = combine(framework::dataset::make("DataType", DataType::F32), framework::dataset::make("DataType", DataType::U32)); const auto CastF32toS32Dataset = combine(framework::dataset::make("DataType", DataType::F32), framework::dataset::make("DataType", DataType::S32)); const auto CastF32toF16Dataset = combine(framework::dataset::make("DataType", DataType::F32), framework::dataset::make("DataType", DataType::F16)); + +// U64 +const auto CastU64toU8Dataset = combine(framework::dataset::make("DataType", DataType::U64), framework::dataset::make("DataType", DataType::U8)); +const auto CastU64toS8Dataset = combine(framework::dataset::make("DataType", DataType::U64), framework::dataset::make("DataType", DataType::S8)); +const auto CastU64toU16Dataset = combine(framework::dataset::make("DataType", DataType::U64), framework::dataset::make("DataType", DataType::U16)); +const auto CastU64toS16Dataset = combine(framework::dataset::make("DataType", DataType::U64), framework::dataset::make("DataType", DataType::S16)); +const auto CastU64toU32Dataset = combine(framework::dataset::make("DataType", DataType::U64), framework::dataset::make("DataType", DataType::U32)); +const auto CastU64toS32Dataset = combine(framework::dataset::make("DataType", DataType::U64), framework::dataset::make("DataType", DataType::S32)); +const auto CastU64toF16Dataset = combine(framework::dataset::make("DataType", DataType::U64), framework::dataset::make("DataType", DataType::F16)); +const auto CastU64toF32Dataset = combine(framework::dataset::make("DataType", DataType::U64), framework::dataset::make("DataType", DataType::F32)); + +// S64 +const auto CastS64toU8Dataset = combine(framework::dataset::make("DataType", DataType::S64), framework::dataset::make("DataType", DataType::U8)); +const auto CastS64toS8Dataset = combine(framework::dataset::make("DataType", DataType::S64), framework::dataset::make("DataType", DataType::S8)); +const auto CastS64toU16Dataset = combine(framework::dataset::make("DataType", DataType::S64), framework::dataset::make("DataType", DataType::U16)); +const auto CastS64toS16Dataset = combine(framework::dataset::make("DataType", DataType::S64), framework::dataset::make("DataType", DataType::S16)); +const auto CastS64toU32Dataset = combine(framework::dataset::make("DataType", DataType::S64), framework::dataset::make("DataType", DataType::U32)); +const auto CastS64toS32Dataset = combine(framework::dataset::make("DataType", DataType::S64), framework::dataset::make("DataType", DataType::S32)); +const auto CastS64toF16Dataset = combine(framework::dataset::make("DataType", DataType::S64), framework::dataset::make("DataType", DataType::F16)); +const auto CastS64toF32Dataset = combine(framework::dataset::make("DataType", DataType::S64), framework::dataset::make("DataType", DataType::F32)); } // namespace TEST_SUITE(CL) @@ -227,6 +247,26 @@ CAST_SUITE(F32_to_U32, DataType::F32, DataType::U32, CLCastToU32Fixture, CAST_SUITE(F32_to_S32, DataType::F32, DataType::S32, CLCastToS32Fixture, CastF32toS32Dataset, one_tolerance) CAST_SUITE(F32_to_F16, DataType::F32, DataType::F16, CLCastToF16Fixture, CastF32toF16Dataset, zero_tolerance) +// S64 +CAST_SUITE(S64_to_U8, DataType::S64, DataType::U8, CLCastToU8Fixture, CastS64toU8Dataset, one_tolerance) +CAST_SUITE(S64_to_S8, DataType::S64, DataType::S8, CLCastToS8Fixture, CastS64toS8Dataset, one_tolerance) +CAST_SUITE(S64_to_U16, DataType::S64, DataType::U16, CLCastToU16Fixture, CastS64toU16Dataset, one_tolerance) +CAST_SUITE(S64_to_S16, DataType::S64, DataType::S16, CLCastToS16Fixture, CastS64toS16Dataset, one_tolerance) +CAST_SUITE(S64_to_U32, DataType::S64, DataType::U32, CLCastToU32Fixture, CastS64toU32Dataset, one_tolerance) +CAST_SUITE(S64_to_S32, DataType::S64, DataType::S32, CLCastToS32Fixture, CastS64toS32Dataset, one_tolerance) +CAST_SUITE(S64_to_F16, DataType::S64, DataType::F16, CLCastToF16Fixture, CastS64toF16Dataset, zero_tolerance) +CAST_SUITE(S64_to_F32, DataType::S64, DataType::F32, CLCastToF32Fixture, CastS64toF32Dataset, zero_tolerance) + +// U64 +CAST_SUITE(U64_to_U8, DataType::U64, DataType::U8, CLCastToU8Fixture, CastU64toU8Dataset, one_tolerance) +CAST_SUITE(U64_to_S8, DataType::U64, DataType::S8, CLCastToS8Fixture, CastU64toS8Dataset, one_tolerance) +CAST_SUITE(U64_to_U16, DataType::U64, DataType::U16, CLCastToU16Fixture, CastU64toU16Dataset, one_tolerance) +CAST_SUITE(U64_to_S16, DataType::U64, DataType::S16, CLCastToS16Fixture, CastU64toS16Dataset, one_tolerance) +CAST_SUITE(U64_to_U32, DataType::U64, DataType::U32, CLCastToU32Fixture, CastU64toU32Dataset, one_tolerance) +CAST_SUITE(U64_to_S32, DataType::U64, DataType::S32, CLCastToS32Fixture, CastU64toS32Dataset, one_tolerance) +CAST_SUITE(U64_to_F16, DataType::U64, DataType::F16, CLCastToF16Fixture, CastU64toF16Dataset, zero_tolerance) +CAST_SUITE(U64_to_F32, DataType::U64, DataType::F32, CLCastToF32Fixture, CastU64toF32Dataset, zero_tolerance) + TEST_SUITE_END() // Cast TEST_SUITE_END() // CL } // namespace validation diff --git a/tests/validation/CL/DeconvolutionLayer.cpp b/tests/validation/CL/DeconvolutionLayer.cpp index a04f273b60f53180bec6030a8cff41a6e1795467..6b12fc01a10666c7c0c37e1ab77ed24a7550751d 100644 --- a/tests/validation/CL/DeconvolutionLayer.cpp +++ b/tests/validation/CL/DeconvolutionLayer.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2022 Arm Limited. + * Copyright (c) 2017-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -81,6 +81,9 @@ const auto data2x2_precommit = datasets::SmallDeconvolutionShapes() * framework: const auto data1x1 = datasets::SmallDeconvolutionShapes() * framework::dataset::make("StrideX", 1, 4) * framework::dataset::make("StrideY", 1, 4) * framework::dataset::make("PadX", 0, 1) * framework::dataset::make("PadY", 0, 1) * framework::dataset::make("NumKernels", { 3 }); +const auto data5x1 = datasets::SmallDeconvolutionShapes() * framework::dataset::make("StrideX", 1, 4) * framework::dataset::make("StrideY", 1, 4) * framework::dataset::make("PadX", 0, 1) + * framework::dataset::make("PadY", 0, 1) * framework::dataset::make("NumKernels", { 3 }); + const auto data_layouts_dataset = framework::dataset::make("DataLayout", { DataLayout::NCHW, DataLayout::NHWC }); const auto add_bias_dataset = framework::dataset::make("AddBias", { true, false }); @@ -185,6 +188,9 @@ using CLDeconvolutionLayerFixture1x1 = DeconvolutionValidationFixture using CLDeconvolutionLayerAsymmFixture9x9 = DeconvolutionValidationAsymmFixture; +template +using CLDeconvolutionLayerFixture5x1 = DeconvolutionValidationFixture; + TEST_SUITE(Float) TEST_SUITE(FP32) @@ -265,6 +271,17 @@ FIXTURE_DATA_TEST_CASE(RunSmall, CLDeconvolutionLayerAsymmFixture9x9, fra validate(CLAccessor(_target), _reference, tolerance_fp32); } TEST_SUITE_END() // W9x9 + +TEST_SUITE(W5x1) +FIXTURE_DATA_TEST_CASE(Run, CLDeconvolutionLayerFixture5x1, framework::DatasetMode::NIGHTLY, combine(combine(combine(data5x1, framework::dataset::make("DataType", DataType::F32)), + data_layouts_dataset), + add_bias_dataset)) +{ + // Validate output + validate(CLAccessor(_target), _reference, tolerance_fp32); +} +TEST_SUITE_END() // W5x1 + TEST_SUITE_END() // FP32 TEST_SUITE(FP16) @@ -318,6 +335,16 @@ FIXTURE_DATA_TEST_CASE(Run, CLDeconvolutionLayerFixture1x1, framework::Dat } TEST_SUITE_END() // W1x1 +TEST_SUITE(W5x1) +FIXTURE_DATA_TEST_CASE(Run, CLDeconvolutionLayerFixture5x1, framework::DatasetMode::NIGHTLY, combine(combine(combine(data5x1, framework::dataset::make("DataType", DataType::F16)), + data_layouts_dataset), + add_bias_dataset)) +{ + // Validate output + validate(CLAccessor(_target), _reference, tolerance_f16, tolerance_num); +} +TEST_SUITE_END() // W5x1 + TEST_SUITE_END() // FP16 TEST_SUITE_END() // Float @@ -333,6 +360,9 @@ using CLDeconvolutionLayerQuantizedFixture2x2 = DeconvolutionValidationQuantized template using CLDeconvolutionLayerQuantizedFixture1x1 = DeconvolutionValidationQuantizedFixture; +template +using CLDeconvolutionLayerQuantizedFixture5x1 = DeconvolutionValidationQuantizedFixture; + template using CLDeconvolutionLayerQuantizedPerChannelFixture4x4 = DeconvolutionValidationQuantizedPerChannelFixture; @@ -345,6 +375,9 @@ using CLDeconvolutionLayerQuantizedPerChannelFixture2x2 = DeconvolutionValidatio template using CLDeconvolutionLayerQuantizedPerChannelFixture1x1 = DeconvolutionValidationQuantizedPerChannelFixture; +template +using CLDeconvolutionLayerQuantizedPerChannelFixture5x1 = DeconvolutionValidationQuantizedPerChannelFixture; + TEST_SUITE(Quantized) TEST_SUITE(QASYMM8) @@ -412,6 +445,19 @@ FIXTURE_DATA_TEST_CASE(Run, CLDeconvolutionLayerQuantizedFixture1x1, fr } TEST_SUITE_END() // W1x1 +TEST_SUITE(W5x1) +FIXTURE_DATA_TEST_CASE(Run, CLDeconvolutionLayerQuantizedFixture5x1, framework::DatasetMode::NIGHTLY, combine(combine(combine(combine(combine(data5x1, framework::dataset::make("DataType", + DataType::QASYMM8)), + data_layouts_dataset), + framework::dataset::make("InputQuantizationInfo", { QuantizationInfo(1.f / 255.f, 10), QuantizationInfo(2.f / 255.f, 5) })), + framework::dataset::make("OutputQuantizationInfo", { QuantizationInfo(3.f / 255.f, 5), QuantizationInfo(4.f / 255.f, 10) })), + add_bias_dataset)) +{ + // Validate output + validate(CLAccessor(_target), _reference, tolerance_qasymm8, tolerance_num); +} +TEST_SUITE_END() // W5x1 + TEST_SUITE_END() // QASYMM8 TEST_SUITE(QASYMM8_SIGNED) @@ -485,6 +531,19 @@ FIXTURE_DATA_TEST_CASE(Run, CLDeconvolutionLayerQuantizedFixture1x1, fra } TEST_SUITE_END() // W1x1 +TEST_SUITE(W5x1) +FIXTURE_DATA_TEST_CASE(Run, CLDeconvolutionLayerQuantizedFixture5x1, framework::DatasetMode::NIGHTLY, combine(combine(combine(combine(combine(data5x1, framework::dataset::make("DataType", + DataType::QASYMM8_SIGNED)), + data_layouts_dataset), + framework::dataset::make("InputQuantizationInfo", { QuantizationInfo(1.f / 255.f, 10), QuantizationInfo(2.f / 255.f, 5) })), + framework::dataset::make("OutputQuantizationInfo", { QuantizationInfo(3.f / 255.f, 5), QuantizationInfo(4.f / 255.f, 10) })), + add_bias_dataset)) +{ + // Validate output + validate(CLAccessor(_target), _reference, tolerance_qasymm8, tolerance_num); +} +TEST_SUITE_END() // W5x1 + TEST_SUITE_END() // QASYMM8_SIGNED const auto input_qinfo_dataset = framework::dataset::make("InputQuantizationInfo", { QuantizationInfo(1.f / 255.f, 10) }); @@ -619,6 +678,31 @@ FIXTURE_DATA_TEST_CASE(RunSmallSigned, CLDeconvolutionLayerQuantizedPerChannelFi } TEST_SUITE_END() // W1x1 +TEST_SUITE(W5x1) +FIXTURE_DATA_TEST_CASE(RunSmall, CLDeconvolutionLayerQuantizedPerChannelFixture5x1, framework::DatasetMode::NIGHTLY, combine(combine(combine(combine(combine(combine(data5x1, + framework::dataset::make("DataType", DataType::QASYMM8)), + data_layouts_dataset), + input_qinfo_dataset), + output_qinfo_dataset), + add_bias_dataset), + framework::dataset::make("WeightsDataType", { DataType::QSYMM8_PER_CHANNEL }))) +{ + // Validate output + validate(CLAccessor(_target), _reference, tolerance_qasymm8, tolerance_num); +} +FIXTURE_DATA_TEST_CASE(RunSmallSigned, CLDeconvolutionLayerQuantizedPerChannelFixture5x1, framework::DatasetMode::NIGHTLY, combine(combine(combine(combine(combine(combine(data5x1, + framework::dataset::make("DataType", DataType::QASYMM8_SIGNED)), + data_layouts_dataset), + input_signed_qinfo_dataset), + output_signed_qinfo_dataset), + add_bias_dataset), + framework::dataset::make("WeightsDataType", { DataType::QSYMM8_PER_CHANNEL }))) +{ + // Validate output + validate(CLAccessor(_target), _reference, tolerance_qasymm8, tolerance_num); +} +TEST_SUITE_END() // W5x1 + TEST_SUITE_END() // QSYMM8_PER_CHANNEL TEST_SUITE_END() // Quantized diff --git a/tests/validation/CL/FullyConnectedLayer.cpp b/tests/validation/CL/FullyConnectedLayer.cpp index 9213ab541d2de46e13a261cf0fde7c9c91f40e2e..474a87dd1cbfdd74a1a0c025f27b8d121e708e5e 100644 --- a/tests/validation/CL/FullyConnectedLayer.cpp +++ b/tests/validation/CL/FullyConnectedLayer.cpp @@ -131,6 +131,8 @@ template using CLFullyConnectedLayerMixedDataLayoutFixture = FullyConnectedLayerValidationFixture; template using CLFullyConnectedLayerDynamicWeightsFixture = FullyConnectedWithDynamicWeightsFixture; +template +using CLFullyConnectedNoBiasFixture = FullyConnectedDynamicNoBiasFixture; TEST_SUITE(Float) TEST_SUITE(FP16) @@ -151,9 +153,9 @@ FIXTURE_DATA_TEST_CASE(RunLarge, CLFullyConnectedLayerFixture, framework:: validate(CLAccessor(_target), _reference, tolerance_f16, tolerance_num); } FIXTURE_DATA_TEST_CASE(RunDynamicWeights, CLFullyConnectedLayerDynamicWeightsFixture, framework::DatasetMode::PRECOMMIT, combine(combine(combine(datasets::SmallFullyConnectedLayerDataset(), - framework::dataset::make("DataType", DataType::F16)), - framework::dataset::make("ActivationInfo", ActivationLayerInfo())), - framework::dataset::make("WeightsReshaped", { false, true }))) + framework::dataset::make("DataType", DataType::F16)), + framework::dataset::make("ActivationInfo", ActivationLayerInfo())), + framework::dataset::make("WeightsReshaped", { false, true }))) { } TEST_SUITE_END() @@ -179,9 +181,15 @@ FIXTURE_DATA_TEST_CASE(RunMixedDataLayout, CLFullyConnectedLayerMixedDataLayoutF validate(CLAccessor(_target), _reference, rel_tolerance_f32, 0, abs_tolerance_f32); } FIXTURE_DATA_TEST_CASE(RunDynamicWeights, CLFullyConnectedLayerDynamicWeightsFixture, framework::DatasetMode::PRECOMMIT, combine(combine(combine(datasets::SmallFullyConnectedLayerDataset(), - framework::dataset::make("DataType", DataType::F32)), - framework::dataset::make("ActivationInfo", ActivationLayerInfo())), - framework::dataset::make("WeightsReshaped", { false, true }))) + framework::dataset::make("DataType", DataType::F32)), + framework::dataset::make("ActivationInfo", ActivationLayerInfo())), + framework::dataset::make("WeightsReshaped", { false, true }))) +{ +} +FIXTURE_DATA_TEST_CASE(RunDynamicNoBias, CLFullyConnectedNoBiasFixture, framework::DatasetMode::PRECOMMIT, combine(combine(combine(datasets::SmallFullyConnectedLayerDataset(), + framework::dataset::make("DataType", DataType::F32)), + framework::dataset::make("ActivationInfo", { ActivationLayerInfo(), ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::RELU) })), + framework::dataset::make("WeightsReshaped", { false }))) { } FIXTURE_DATA_TEST_CASE(RunLarge, CLFullyConnectedLayerFixture, framework::DatasetMode::NIGHTLY, combine(combine(combine(datasets::LargeFullyConnectedLayerDataset(), FullyConnectedParameters), @@ -230,9 +238,9 @@ FIXTURE_DATA_TEST_CASE(RunLarge, CLFullyConnectedLayerQuantizedFixture, validate(CLAccessor(_target), _reference, tolerance_qasymm8); } FIXTURE_DATA_TEST_CASE(RunDynamicWeights, CLFullyConnectedLayerDynamicWeightsFixture, framework::DatasetMode::PRECOMMIT, combine(combine(combine(datasets::SmallFullyConnectedLayerDataset(), - framework::dataset::make("DataType", DataType::QASYMM8)), - framework::dataset::make("ActivationInfo", ActivationLayerInfo())), - framework::dataset::make("WeightsReshaped", { false /* COMPMID-6000: Support FullyConnected with quantized dynamic weights already reshaped */ }))) + framework::dataset::make("DataType", DataType::QASYMM8)), + framework::dataset::make("ActivationInfo", ActivationLayerInfo())), + framework::dataset::make("WeightsReshaped", { false /* COMPMID-6000: Support FullyConnected with quantized dynamic weights already reshaped */ }))) { } TEST_SUITE_END() /* QASYMM8 */ @@ -259,9 +267,15 @@ FIXTURE_DATA_TEST_CASE(RunMixedDataLayout, CLFullyConnectedLayerQuantizedMixedDa validate(CLAccessor(_target), _reference, tolerance_qasymm8); } FIXTURE_DATA_TEST_CASE(RunDynamicWeights, CLFullyConnectedLayerDynamicWeightsFixture, framework::DatasetMode::PRECOMMIT, combine(combine(combine(datasets::SmallFullyConnectedLayerDataset(), - framework::dataset::make("DataType", DataType::QASYMM8_SIGNED)), - framework::dataset::make("ActivationInfo", ActivationLayerInfo())), - framework::dataset::make("WeightsReshaped", { false /* COMPMID-6000: Support FullyConnected with quantized dynamic weights already reshaped */ }))) + framework::dataset::make("DataType", DataType::QASYMM8_SIGNED)), + framework::dataset::make("ActivationInfo", ActivationLayerInfo())), + framework::dataset::make("WeightsReshaped", { false /* COMPMID-6000: Support FullyConnected with quantized dynamic weights already reshaped */ }))) +{ +} +FIXTURE_DATA_TEST_CASE(RunDynamicNoBias, CLFullyConnectedNoBiasFixture, framework::DatasetMode::PRECOMMIT, combine(combine(combine(datasets::SmallFullyConnectedLayerDataset(), + framework::dataset::make("DataType", DataType::QASYMM8_SIGNED)), + framework::dataset::make("ActivationInfo", ActivationLayerInfo())), + framework::dataset::make("WeightsReshaped", { false /* COMPMID-6000: Support FullyConnected with quantized dynamic weights already reshaped */ }))) { } TEST_SUITE_END() // QASYMM8_SIGNED diff --git a/tests/validation/CL/MatMul.cpp b/tests/validation/CL/MatMul.cpp index 6364b1620067220a834c0240470c760c9b98b780..844597f3e982faf4577ba1283ba481fba46a1afd 100644 --- a/tests/validation/CL/MatMul.cpp +++ b/tests/validation/CL/MatMul.cpp @@ -26,6 +26,7 @@ #include "arm_compute/runtime/CL/functions/CLMatMul.h" #include "tests/CL/CLAccessor.h" +#include "tests/datasets/ActivationFunctionsDataset.h" #include "tests/framework/DatasetModes.h" #include "tests/framework/Macros.h" #include "tests/framework/TestCase.h" @@ -44,11 +45,13 @@ namespace validation { namespace { -RelativeTolerance tolerance_f32(0.001f); /**< Tolerance value for comparing reference's output against implementation's output for fp32 data type */ -constexpr float abs_tolerance_f32(0.0001f); /**< Absolute tolerance value for comparing reference's output against implementation's output for fp32 data type in case using relative tolerance fails because of small values */ -constexpr float abs_tolerance_f16(0.001f); /**< Absolute tolerance value for comparing reference's output against implementation's output for fp16 data type in case using relative tolerance fails because of small values */ -RelativeTolerance tolerance_f16(half(0.01)); /**< Tolerance value for comparing reference's output against implementation's output for fp16 data type */ -constexpr AbsoluteTolerance tolerance_quant(1); /**< Tolerance value for comparing reference's output against implementation's output for quantized data types */ +RelativeTolerance tolerance_f32(0.001f); /**< Tolerance value for comparing reference's output against implementation's output for fp32 data type */ +constexpr float abs_tolerance_f32( + 0.0001f); /**< Absolute tolerance value for comparing reference's output against implementation's output for fp32 data type in case using relative tolerance fails because of small values */ +constexpr float abs_tolerance_f16( + 0.001f); /**< Absolute tolerance value for comparing reference's output against implementation's output for fp16 data type in case using relative tolerance fails because of small values */ +RelativeTolerance tolerance_f16(half(0.01)); /**< Tolerance value for comparing reference's output against implementation's output for fp16 data type */ +constexpr AbsoluteTolerance tolerance_quant(1); /**< Tolerance value for comparing reference's output against implementation's output for quantized data types */ } // namespace template @@ -57,25 +60,68 @@ using CLMatMulFixture = MatMulValidationFixture using CLQuantizedMatMulFixture = QuantizedMatMulValidationFixture; +template +using CLMatMulActivationFixture = MatMulValidationWithActivationFixture; + +template +using CLMatMulActivationAlphaBetaFixture = MatMulValidationWithActivationAlphaBetaFixture; + +template +using CLQuantizedMatMulActivationFixture = QuantizedMatMulValidationWithActivationFixture; + +/* The main act functions matmul (float) is expected to support */ +const auto ActivationFunctionsDataset = framework::dataset::make("ActivationInfo", +{ + ActivationLayerInfo(), + ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::RELU), + ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::BOUNDED_RELU, 0.5f), + ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU, 0.75f, 0.25f), +}); + +/* (Float datatype only) Larger activation functions dataset, used during some nightly tests. */ +const auto AllActivationsDataset = combine(datasets::ActivationFunctions(), framework::dataset::make("AlphaBeta", { 0.5f, 1.f })); + +// Alpha beta values should be integer values +// This is for testing purposes with quantized datatypes and is not a limitation of the kernel. +// To properly remove this restriction, dst_qinfo should be auto-initialised with consideration for alpha beta values +// The main act functions quantized matmul kernels are expected to support +const auto ActivationFunctionsQuantizedDataset = concat(concat(concat( + framework::dataset::make("ActivationInfo", ActivationLayerInfo()), + framework::dataset::make("ActivationInfo", ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::RELU))), + framework::dataset::make("ActivationInfo", ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::BOUNDED_RELU, 1.f))), + framework::dataset::make("ActivationInfo", ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU, 2.f, 1.f))); + TEST_SUITE(CL) TEST_SUITE(MatMul) TEST_SUITE(Float) TEST_SUITE(FP32) -FIXTURE_DATA_TEST_CASE(RunSmall, CLMatMulFixture, framework::DatasetMode::ALL, combine(combine(combine(datasets::SmallMatMulDataset(), +FIXTURE_DATA_TEST_CASE(RunSmall, CLMatMulActivationFixture, framework::DatasetMode::ALL, combine(combine(combine(combine(datasets::SmallMatMulDataset(), + framework::dataset::make("TransposeA", { false, true })), + framework::dataset::make("TransposeB", { false, true })), + framework::dataset::make("DataType", DataType::F32)), + ActivationFunctionsDataset)) +{ + // Validate output + validate(CLAccessor(_target), _reference, tolerance_f32, 0.f, abs_tolerance_f32); +} + +FIXTURE_DATA_TEST_CASE(RunLarge, CLMatMulActivationFixture, framework::DatasetMode::NIGHTLY, combine(combine(combine(combine(datasets::LargeMatMulDataset(), framework::dataset::make("TransposeA", { false, true })), - framework::dataset::make("TransposeB", { false, true })), - framework::dataset::make("DataType", DataType::F32))) + framework::dataset::make("TransposeB", { false, true })), + framework::dataset::make("DataType", DataType::F32)), + ActivationFunctionsDataset)) { // Validate output validate(CLAccessor(_target), _reference, tolerance_f32, 0.f, abs_tolerance_f32); } -FIXTURE_DATA_TEST_CASE(RunLarge, CLMatMulFixture, framework::DatasetMode::NIGHTLY, combine(combine(combine(datasets::LargeMatMulDataset(), - framework::dataset::make("TransposeA", { false, true })), - framework::dataset::make("TransposeB", { false, true })), - framework::dataset::make("DataType", DataType::F32))) +FIXTURE_DATA_TEST_CASE(RunAllActivations, CLMatMulActivationAlphaBetaFixture, framework::DatasetMode::NIGHTLY, combine(combine(combine(combine(datasets::SmallerMatMulDataset(), + framework::dataset::make("TransposeA", { false })), + framework::dataset::make("TransposeB", { true })), + framework::dataset::make("DataType", DataType::F32)), + AllActivationsDataset)) { // Validate output validate(CLAccessor(_target), _reference, tolerance_f32, 0.f, abs_tolerance_f32); @@ -85,19 +131,21 @@ TEST_SUITE_END() // FP32 TEST_SUITE(FP16) -FIXTURE_DATA_TEST_CASE(RunSmall, CLMatMulFixture, framework::DatasetMode::ALL, combine(combine(combine(datasets::SmallMatMulDataset(), - framework::dataset::make("TransposeA", { false, true })), - framework::dataset::make("TransposeB", { false, true })), - framework::dataset::make("DataType", DataType::F16))) +FIXTURE_DATA_TEST_CASE(RunSmall, CLMatMulActivationFixture, framework::DatasetMode::ALL, combine(combine(combine(combine(datasets::SmallMatMulDataset(), + framework::dataset::make("TransposeA", { false, true })), + framework::dataset::make("TransposeB", { false, true })), + framework::dataset::make("DataType", DataType::F16)), + ActivationFunctionsDataset)) { // Validate output validate(CLAccessor(_target), _reference, tolerance_f16, 0.f, abs_tolerance_f16); } -FIXTURE_DATA_TEST_CASE(RunLarge, CLMatMulFixture, framework::DatasetMode::NIGHTLY, combine(combine(combine(datasets::LargeMatMulDataset(), - framework::dataset::make("TransposeA", { false, true })), - framework::dataset::make("TransposeB", { false, true })), - framework::dataset::make("DataType", DataType::F16))) +FIXTURE_DATA_TEST_CASE(RunLarge, CLMatMulActivationFixture, framework::DatasetMode::NIGHTLY, combine(combine(combine(combine(datasets::LargeMatMulDataset(), + framework::dataset::make("TransposeA", { false, true })), + framework::dataset::make("TransposeB", { false, true })), + framework::dataset::make("DataType", DataType::F16)), + ActivationFunctionsDataset)) { // Validate output validate(CLAccessor(_target), _reference, tolerance_f16, 0.f, abs_tolerance_f16); @@ -110,32 +158,30 @@ TEST_SUITE(Quantized) TEST_SUITE(QASYMM8) FIXTURE_DATA_TEST_CASE(RunSmall, CLQuantizedMatMulFixture, framework::DatasetMode::ALL, combine(combine(combine(combine(combine(combine(combine(combine( - datasets::SmallMatMulDataset(), - framework::dataset::make("TransposeA", { false, true })), - framework::dataset::make("TransposeB", { false, true })), - framework::dataset::make("DataType", DataType::QASYMM8)), - framework::dataset::make("ActivationInfo", { ActivationLayerInfo() })), - framework::dataset::make("NumberOfExtraRuns", { 0, 1 })), - framework::dataset::make("LhsQInfo", { QuantizationInfo(1.f / 50, 1) })), - framework::dataset::make("RhsQInfo", { QuantizationInfo(1.f / 30, -1) })), - framework::dataset::make("DstQInfo", { QuantizationInfo(1.f, 2) })) -) + datasets::SmallMatMulDataset(), + framework::dataset::make("TransposeA", { false, true })), + framework::dataset::make("TransposeB", { false, true })), + framework::dataset::make("DataType", DataType::QASYMM8)), + ActivationFunctionsQuantizedDataset), + framework::dataset::make("NumberOfExtraRuns", { 0, 1 })), + framework::dataset::make("LhsQInfo", { QuantizationInfo(1.f / 50, 1) })), + framework::dataset::make("RhsQInfo", { QuantizationInfo(1.f / 30, -1) })), + framework::dataset::make("DstQInfo", { QuantizationInfo(1.f, 2) }))) { // Validate output validate(CLAccessor(_target), _reference, tolerance_quant); } FIXTURE_DATA_TEST_CASE(RunLarge, CLQuantizedMatMulFixture, framework::DatasetMode::NIGHTLY, combine(combine(combine(combine(combine(combine(combine(combine( - datasets::LargeMatMulDataset(), - framework::dataset::make("TransposeA", { false, true })), - framework::dataset::make("TransposeB", { false, true })), - framework::dataset::make("DataType", DataType::QASYMM8)), - framework::dataset::make("ActivationInfo", { ActivationLayerInfo() })), - framework::dataset::make("NumberOfExtraRuns", { 0, 1 })), - framework::dataset::make("LhsQInfo", { QuantizationInfo(1.f / 100, 1) })), - framework::dataset::make("RhsQInfo", { QuantizationInfo(1.f / 200, -1) })), - framework::dataset::make("DstQInfo", { QuantizationInfo(1.f, 2) })) -) + datasets::LargeMatMulDataset(), + framework::dataset::make("TransposeA", { false, true })), + framework::dataset::make("TransposeB", { false, true })), + framework::dataset::make("DataType", DataType::QASYMM8)), + ActivationFunctionsQuantizedDataset), + framework::dataset::make("NumberOfExtraRuns", { 0, 1 })), + framework::dataset::make("LhsQInfo", { QuantizationInfo(1.f / 100, 1) })), + framework::dataset::make("RhsQInfo", { QuantizationInfo(1.f / 200, -1) })), + framework::dataset::make("DstQInfo", { QuantizationInfo(1.f, 2) }))) { // Validate output validate(CLAccessor(_target), _reference, tolerance_quant); @@ -146,32 +192,30 @@ TEST_SUITE_END() // QASYMM8 TEST_SUITE(QASYMM8_SIGNED) FIXTURE_DATA_TEST_CASE(RunSmall, CLQuantizedMatMulFixture, framework::DatasetMode::ALL, combine(combine(combine(combine(combine(combine(combine(combine( - datasets::SmallMatMulDataset(), - framework::dataset::make("TransposeA", { false, true })), - framework::dataset::make("TransposeB", { false, true })), - framework::dataset::make("DataType", DataType::QASYMM8_SIGNED)), - framework::dataset::make("ActivationInfo", { ActivationLayerInfo() })), - framework::dataset::make("NumberOfExtraRuns", { 0, 1 })), - framework::dataset::make("LhsQInfo", { QuantizationInfo(1.f / 50, 1) })), - framework::dataset::make("RhsQInfo", { QuantizationInfo(1.f / 30, -1) })), - framework::dataset::make("DstQInfo", { QuantizationInfo(1.f, 2) })) -) + datasets::SmallMatMulDataset(), + framework::dataset::make("TransposeA", { false, true })), + framework::dataset::make("TransposeB", { false, true })), + framework::dataset::make("DataType", DataType::QASYMM8_SIGNED)), + ActivationFunctionsQuantizedDataset), + framework::dataset::make("NumberOfExtraRuns", { 0, 1 })), + framework::dataset::make("LhsQInfo", { QuantizationInfo(1.f / 50, 1) })), + framework::dataset::make("RhsQInfo", { QuantizationInfo(1.f / 30, -1) })), + framework::dataset::make("DstQInfo", { QuantizationInfo(1.f, 2) }))) { // Validate output validate(CLAccessor(_target), _reference, tolerance_quant); } FIXTURE_DATA_TEST_CASE(RunLarge, CLQuantizedMatMulFixture, framework::DatasetMode::NIGHTLY, combine(combine(combine(combine(combine(combine(combine(combine( - datasets::LargeMatMulDataset(), - framework::dataset::make("TransposeA", { false, true })), - framework::dataset::make("TransposeB", { false, true })), - framework::dataset::make("DataType", DataType::QASYMM8_SIGNED)), - framework::dataset::make("ActivationInfo", { ActivationLayerInfo() })), - framework::dataset::make("NumberOfExtraRuns", { 0, 1 })), - framework::dataset::make("LhsQInfo", { QuantizationInfo(1.f / 100, 1) })), - framework::dataset::make("RhsQInfo", { QuantizationInfo(1.f / 200, -1) })), - framework::dataset::make("DstQInfo", { QuantizationInfo(1.f, 2) })) -) + datasets::LargeMatMulDataset(), + framework::dataset::make("TransposeA", { false, true })), + framework::dataset::make("TransposeB", { false, true })), + framework::dataset::make("DataType", DataType::QASYMM8_SIGNED)), + ActivationFunctionsQuantizedDataset), + framework::dataset::make("NumberOfExtraRuns", { 0, 1 })), + framework::dataset::make("LhsQInfo", { QuantizationInfo(1.f / 100, 1) })), + framework::dataset::make("RhsQInfo", { QuantizationInfo(1.f / 200, -1) })), + framework::dataset::make("DstQInfo", { QuantizationInfo(1.f, 50) }))) { // Validate output validate(CLAccessor(_target), _reference, tolerance_quant); diff --git a/tests/validation/CL/MatMulKernel.cpp b/tests/validation/CL/MatMulKernel.cpp index ff872aaa0a57da64bb43eaad30855c61e0bdad4f..b47f8bc9245478ffb0520ffdde08f1f22e59a08c 100644 --- a/tests/validation/CL/MatMulKernel.cpp +++ b/tests/validation/CL/MatMulKernel.cpp @@ -75,6 +75,9 @@ const auto k0_values_nightly_lhs_t_rhs_nt = framework::dataset::make("K0", { 1, template using CLMatMulKernelFixture = MatMulKernelValidationFixture; +template +using CLMatMulKernelBiasFixture = MatMulKernelWithBiasValidation; + TEST_SUITE(CL) TEST_SUITE(MatMulKernel) TEST_SUITE(Validate) @@ -162,7 +165,7 @@ TEST_CASE(SupportedBlockSizes, framework::DatasetMode::ALL) for(auto &pair : supported_block_sizes) { TensorInfo output_info; - Status status = ClMatMulNativeKernel::validate(&lhs_info, &rhs_info, &output_info, pair.first); + Status status = ClMatMulNativeKernel::validate(&lhs_info, &rhs_info, nullptr, &output_info, pair.first); if(!pair.first.export_rhs_to_cl_image || export_to_cl_image_supported) { @@ -222,7 +225,7 @@ TEST_CASE(ExportToCLImage, framework::DatasetMode::ALL) }; TensorInfo output_info; - Status status = ClMatMulNativeKernel::validate(&lhs_info, &rhs_info, &output_info, matmul_kernel_info); + Status status = ClMatMulNativeKernel::validate(&lhs_info, &rhs_info, nullptr, &output_info, matmul_kernel_info); const bool expected = std::get<4>(tuple); ARM_COMPUTE_EXPECT(bool(status) == expected, framework::LogLevel::ERRORS); @@ -233,22 +236,25 @@ TEST_CASE(ExportToCLImage, framework::DatasetMode::ALL) TEST_CASE(ValidateInputShapes, framework::DatasetMode::ALL) { // Configurations are assumed to be Nt/Nt, but will be transposed inside the test to test other configurations - using ShapeConfigurationTuple = std::tuple; + using ShapeConfigurationTuple = std::tuple; const std::vector shape_configurations = { - { TensorShape(5U, 1U), TensorShape(3U, 5U), true }, - { TensorShape(10U, 12U), TensorShape(3U, 10U), true }, - { TensorShape(8U, 4U), TensorShape(2U, 8U), true }, - { TensorShape(8U, 4U), TensorShape(2U, 5U), false }, // Mismatch in the K dimension - { TensorShape(5U, 0U), TensorShape(2U, 5U), false }, // Invalid dimension - { TensorShape(5U, 4U, 3U, 4U, 5U, 6U), TensorShape(2U, 5U, 3U, 4U, 5U, 6U), true }, - { TensorShape(5U, 4U, 3U, 4U, 5U, 1U), TensorShape(2U, 5U, 3U, 4U, 5U, 6U), false }, // no batch broadcasting - { TensorShape(5U, 4U, 3U, 4U, 9U, 6U), TensorShape(2U, 5U, 3U, 4U, 5U, 6U), false }, // mismatch in batch dimension + { TensorShape(5U, 1U), TensorShape(3U, 5U), TensorShape(3U), true }, + { TensorShape(10U, 12U), TensorShape(3U, 10U), TensorShape(3U), true }, + { TensorShape(8U, 4U), TensorShape(2U, 8U), TensorShape(2U), true }, + { TensorShape(8U, 4U), TensorShape(2U, 5U), TensorShape(2U), false }, // Mismatch in the K dimension + { TensorShape(5U, 0U), TensorShape(2U, 5U), TensorShape(2U), false }, // Invalid dimension + { TensorShape(5U, 4U, 3U, 4U, 5U, 6U), TensorShape(2U, 5U, 3U, 4U, 5U, 6U), TensorShape(2U), true }, + { TensorShape(5U, 4U, 3U, 4U, 5U, 1U), TensorShape(2U, 5U, 3U, 4U, 5U, 6U), TensorShape(2U), false }, // no batch broadcasting + { TensorShape(5U, 4U, 3U, 4U, 9U, 6U), TensorShape(2U, 5U, 3U, 4U, 5U, 6U), TensorShape(2U), false }, // mismatch in batch dimension + { TensorShape(5U, 1U), TensorShape(3U, 5U), TensorShape(1U), false }, // Unsupported bias broadcasting. + { TensorShape(5U, 1U), TensorShape(3U, 5U), TensorShape(3U, 3U), false }, // 2D bias is unsupported. + { TensorShape(5U, 1U), TensorShape(3U, 5U), TensorShape(6U), false }, // bias first dimension != dst first dimension }; for(auto &tuple : shape_configurations) { - const bool expected = std::get<2>(tuple); + const bool expected = std::get<3>(tuple); for(bool adj_lhs : { @@ -262,6 +268,7 @@ TEST_CASE(ValidateInputShapes, framework::DatasetMode::ALL) { TensorShape lhs_shape = std::get<0>(tuple); TensorShape rhs_shape = std::get<1>(tuple); + TensorShape bia_shape = std::get<2>(tuple); if(adj_lhs) { @@ -275,11 +282,12 @@ TEST_CASE(ValidateInputShapes, framework::DatasetMode::ALL) const TensorInfo lhs_info = TensorInfo(lhs_shape, 1, DataType::F32); const TensorInfo rhs_info = TensorInfo(rhs_shape, 1, DataType::F32); + const TensorInfo bia_info = TensorInfo(bia_shape, 1, DataType::F32); TensorInfo output_info; MatMulKernelInfo matmul_kernel_info{ adj_lhs, adj_rhs, 1, 1, 1, false /* export_rhs_to_cl_image */ }; - Status status = ClMatMulNativeKernel::validate(&lhs_info, &rhs_info, &output_info, matmul_kernel_info); + Status status = ClMatMulNativeKernel::validate(&lhs_info, &rhs_info, &bia_info, &output_info, matmul_kernel_info); ARM_COMPUTE_EXPECT(bool(status) == expected, framework::LogLevel::ERRORS); } } @@ -322,7 +330,7 @@ TEST_CASE(ValidateDataTypes, framework::DatasetMode::ALL) const TensorInfo rhs_info(shape, 1, std::get<1>(tuple)); TensorInfo output_info(shape, 1, std::get<2>(tuple)); - Status status = ClMatMulNativeKernel::validate(&lhs_info, &rhs_info, &output_info, matmul_kernel_info); + Status status = ClMatMulNativeKernel::validate(&lhs_info, &rhs_info, nullptr, &output_info, matmul_kernel_info); ARM_COMPUTE_EXPECT(bool(status) == expected, framework::LogLevel::ERRORS); } } @@ -356,6 +364,19 @@ FIXTURE_DATA_TEST_CASE(RunSmall, CLMatMulKernelFixture, framework::Datase // Validate output validate(CLAccessor(_target), _reference, tolerance_f32, 0.f, abs_tolerance_f32); } +FIXTURE_DATA_TEST_CASE(RunWithBias, CLMatMulKernelBiasFixture, framework::DatasetMode::ALL, combine(combine(combine(combine(combine(combine(combine(datasets::SmallMatMulDataset(), + framework::dataset::make("TransposeA", { false, true })), + framework::dataset::make("TransposeB", { false, true })), + m0_values_precommit), + n0_values_precommit), + k0_values_precommit), + framework::dataset::make("ExportRhsToCLImage", { false })), + framework::dataset::make("DataType", DataType::F32))) + +{ + // Validate output + validate(CLAccessor(_target), _reference, tolerance_f32, 0.f, abs_tolerance_f32); +} FIXTURE_DATA_TEST_CASE(RunLargeNoTranspose, CLMatMulKernelFixture, framework::DatasetMode::NIGHTLY, combine(combine(combine(combine(combine(combine(combine(datasets::LargeMatMulDataset(), framework::dataset::make("TransposeA", { false })), framework::dataset::make("TransposeB", { false })), diff --git a/tests/validation/CL/MatMulLowpNativeKernel.cpp b/tests/validation/CL/MatMulLowpNativeKernel.cpp index fd7a4cb156a4fe5e223c3cbce118cb49b63cef6b..90eee4fb82b4bb788cb0bb27ac3c0a01c0687c61 100644 --- a/tests/validation/CL/MatMulLowpNativeKernel.cpp +++ b/tests/validation/CL/MatMulLowpNativeKernel.cpp @@ -49,6 +49,9 @@ constexpr AbsoluteTolerance tolerance_quant(1); /**< Tolerance value for template using CLMatMulLowpNativeKernelFixture = MatMulKernelValidationFixture; +template +using CLMatMulLowpKernelWithBiasFixture = MatMulKernelWithBiasValidation; + /** M0 values to test --precommit*/ const auto m0_values_precommit = framework::dataset::make("M0", { 1, 3 }); @@ -103,7 +106,7 @@ TEST_CASE(SupportedKernelConfigurations, framework::DatasetMode::ALL) for(auto &pair : supported_block_sizes) { TensorInfo output_info; - Status status = ClMatMulLowpNativeKernel::validate(&lhs_info, &rhs_info, &output_info, pair.first); + Status status = ClMatMulLowpNativeKernel::validate(&lhs_info, &rhs_info, nullptr, &output_info, pair.first); ARM_COMPUTE_EXPECT(bool(status) == pair.second, framework::LogLevel::ERRORS); } @@ -112,22 +115,24 @@ TEST_CASE(SupportedKernelConfigurations, framework::DatasetMode::ALL) TEST_CASE(ValidateInputShapes, framework::DatasetMode::ALL) { // Configurations are assumed to be Nt/Nt, but will be transposed inside the test to test other configurations - using ShapeConfigurationTuple = std::tuple; + using ShapeConfigurationTuple = std::tuple; const std::vector shape_configurations = { - { TensorShape(5U, 1U), TensorShape(3U, 5U), true }, - { TensorShape(10U, 12U), TensorShape(3U, 10U), true }, - { TensorShape(8U, 4U), TensorShape(2U, 8U), true }, - { TensorShape(8U, 4U), TensorShape(2U, 5U), false }, // Mismatch in the K dimension - { TensorShape(5U, 0U), TensorShape(2U, 5U), false }, // Invalid dimension - { TensorShape(5U, 4U, 3U, 4U, 5U, 6U), TensorShape(2U, 5U, 3U, 4U, 5U, 6U), true }, - { TensorShape(5U, 4U, 3U, 4U, 5U, 1U), TensorShape(2U, 5U, 3U, 4U, 5U, 6U), false }, // no batch broadcasting - { TensorShape(5U, 4U, 3U, 4U, 9U, 6U), TensorShape(2U, 5U, 3U, 4U, 5U, 6U), false }, // mismatch in batch dimension + { TensorShape(5U, 1U), TensorShape(3U, 5U), TensorShape(3U), true }, + { TensorShape(10U, 12U), TensorShape(3U, 10U), TensorShape(3U), true }, + { TensorShape(8U, 4U), TensorShape(2U, 8U), TensorShape(2U), true }, + { TensorShape(8U, 4U), TensorShape(2U, 5U), TensorShape(2U), false }, // Mismatch in the K dimension + { TensorShape(5U, 0U), TensorShape(2U, 5U), TensorShape(2U), false }, // Invalid dimension + { TensorShape(5U, 4U, 3U, 4U, 5U, 6U), TensorShape(2U, 5U, 3U, 4U, 5U, 6U), TensorShape(2U), true }, + { TensorShape(5U, 4U, 3U, 4U, 5U, 1U), TensorShape(2U, 5U, 3U, 4U, 5U, 6U), TensorShape(2U), false }, // no batch broadcasting + { TensorShape(5U, 4U, 3U, 4U, 9U, 6U), TensorShape(2U, 5U, 3U, 4U, 5U, 6U), TensorShape(2U), false }, // mismatch in batch dimension + { TensorShape(5U, 1U), TensorShape(3U, 5U), TensorShape(1U), false }, // invalid broadcast of bias + { TensorShape(5U, 1U), TensorShape(3U, 5U), TensorShape(3U, 3U), false }, // 2d bias is invalid }; for(auto &tuple : shape_configurations) { - const bool expected = std::get<2>(tuple); + const bool expected = std::get<3>(tuple); for(bool adj_lhs : { @@ -141,6 +146,7 @@ TEST_CASE(ValidateInputShapes, framework::DatasetMode::ALL) { TensorShape lhs_shape = std::get<0>(tuple); TensorShape rhs_shape = std::get<1>(tuple); + TensorShape bia_shape = std::get<2>(tuple); if(adj_lhs) { @@ -154,11 +160,12 @@ TEST_CASE(ValidateInputShapes, framework::DatasetMode::ALL) const TensorInfo lhs_info = TensorInfo(lhs_shape, 1, DataType::QASYMM8_SIGNED); const TensorInfo rhs_info = TensorInfo(rhs_shape, 1, DataType::QASYMM8_SIGNED); + const TensorInfo bia_info = TensorInfo(bia_shape, 1, DataType::S32); TensorInfo output_info; MatMulKernelInfo matmul_kernel_info{ adj_lhs, adj_rhs, 1, 1, 1, false /* export_rhs_to_cl_image */ }; - Status status = ClMatMulLowpNativeKernel::validate(&lhs_info, &rhs_info, &output_info, matmul_kernel_info); + Status status = ClMatMulLowpNativeKernel::validate(&lhs_info, &rhs_info, &bia_info, &output_info, matmul_kernel_info); ARM_COMPUTE_EXPECT(bool(status) == expected, framework::LogLevel::ERRORS); } } @@ -167,41 +174,44 @@ TEST_CASE(ValidateInputShapes, framework::DatasetMode::ALL) TEST_CASE(ValidateDataTypes, framework::DatasetMode::ALL) { - using DataTypeConfigurationTuple = std::tuple; + using DataTypeConfigurationTuple = std::tuple; const std::vector data_type_configurations = { - { DataType::F32, DataType::F32, DataType::F32, false }, // no floating point types - { DataType::F16, DataType::F16, DataType::F16, false }, // no floating point types - { DataType::F64, DataType::F64, DataType::F64, false }, // no double precision - { DataType::QASYMM8, DataType::QASYMM8, DataType::QASYMM8, true }, - { DataType::QASYMM8_SIGNED, DataType::QASYMM8_SIGNED, DataType::QASYMM8_SIGNED, true }, - { DataType::QSYMM8_PER_CHANNEL, DataType::QSYMM8_PER_CHANNEL, DataType::QSYMM8_PER_CHANNEL, false }, // only qasymm8/qasymm8_signed is supported - { DataType::QASYMM16, DataType::QASYMM16, DataType::QASYMM16, false }, // only qasymm8/qasymm8_signed is supported - { DataType::QSYMM16, DataType::QSYMM16, DataType::QSYMM16, false }, // only qasymm8/qasymm8_signed is supported - { DataType::QSYMM8, DataType::QSYMM8, DataType::QSYMM8, false }, // only qasymm8/qasymm8_signed is supported - { DataType::QASYMM8, DataType::QASYMM8_SIGNED, DataType::QASYMM8, false }, // no mixed data types - { DataType::S64, DataType::S64, DataType::S64, false }, // no integral types - { DataType::S32, DataType::S32, DataType::S32, false }, // no integral types - { DataType::S16, DataType::S16, DataType::S16, false }, // no integral types - { DataType::S8, DataType::S8, DataType::S8, false }, // no integral types - { DataType::U64, DataType::U64, DataType::U64, false }, // no integral types - { DataType::U32, DataType::U32, DataType::U32, false }, // no integral types - { DataType::U16, DataType::U16, DataType::U16, false }, // no integral types - { DataType::U8, DataType::U8, DataType::U8, false }, // no integral types + { DataType::F32, DataType::F32, DataType::F32, DataType::F32, false }, // no floating point types + { DataType::F16, DataType::F16, DataType::F16, DataType::F16, false }, // no floating point types + { DataType::F64, DataType::F64, DataType::F64, DataType::F64, false }, // no double precision + { DataType::QASYMM8, DataType::QASYMM8, DataType::S32, DataType::QASYMM8, true }, + { DataType::QASYMM8_SIGNED, DataType::QASYMM8_SIGNED, DataType::S32, DataType::QASYMM8_SIGNED, true }, + { DataType::QSYMM8_PER_CHANNEL, DataType::QSYMM8_PER_CHANNEL, DataType::S32, DataType::QSYMM8_PER_CHANNEL, false }, // only qasymm8/qasymm8_signed is supported + { DataType::QASYMM16, DataType::QASYMM16, DataType::S32, DataType::QASYMM16, false }, // only qasymm8/qasymm8_signed is supported + { DataType::QSYMM16, DataType::QSYMM16, DataType::S32, DataType::QSYMM16, false }, // only qasymm8/qasymm8_signed is supported + { DataType::QSYMM8, DataType::QSYMM8, DataType::S32, DataType::QSYMM8, false }, // only qasymm8/qasymm8_signed is supported + { DataType::QASYMM8, DataType::QASYMM8_SIGNED, DataType::S32, DataType::QASYMM8, false }, // no mixed data types + { DataType::S64, DataType::S64, DataType::S64, DataType::S64, false }, // no integral types + { DataType::S32, DataType::S32, DataType::S32, DataType::S32, false }, // no integral types + { DataType::S16, DataType::S16, DataType::S16, DataType::S16, false }, // no integral types + { DataType::S8, DataType::S8, DataType::S8, DataType::S8, false }, // no integral types + { DataType::U64, DataType::U64, DataType::U64, DataType::U64, false }, // no integral types + { DataType::U32, DataType::U32, DataType::U32, DataType::U32, false }, // no integral types + { DataType::U16, DataType::U16, DataType::U16, DataType::U16, false }, // no integral types + { DataType::U8, DataType::U8, DataType::U8, DataType::U8, false }, // no integral types + { DataType::QASYMM8, DataType::QASYMM8, DataType::F32, DataType::QASYMM8, false } // Only S32 bias is supported }; // It's enough to test a single shape and block size configuration while checking data types - const TensorShape shape = TensorShape(10U, 10U); + const TensorShape shape = TensorShape(10U, 10U); + const TensorShape bia_shape = TensorShape(10U); const MatMulKernelInfo matmul_kernel_info{ false, false, 1, 1, 1, false }; for(auto &tuple : data_type_configurations) { - const bool expected = std::get<3>(tuple); + const bool expected = std::get<4>(tuple); const TensorInfo lhs_info(shape, 1, std::get<0>(tuple)); const TensorInfo rhs_info(shape, 1, std::get<1>(tuple)); - TensorInfo output_info(shape, 1, std::get<2>(tuple)); + const TensorInfo bia_info(bia_shape, 1, std::get<2>(tuple)); + TensorInfo output_info(shape, 1, std::get<3>(tuple)); - Status status = ClMatMulLowpNativeKernel::validate(&lhs_info, &rhs_info, &output_info, matmul_kernel_info); + Status status = ClMatMulLowpNativeKernel::validate(&lhs_info, &rhs_info, &bia_info, &output_info, matmul_kernel_info); ARM_COMPUTE_EXPECT(bool(status) == expected, framework::LogLevel::ERRORS); } } @@ -234,6 +244,18 @@ FIXTURE_DATA_TEST_CASE(RunSmall, CLMatMulLowpNativeKernelFixture, framew // Validate output validate(CLAccessor(_target), _reference, tolerance_quant); } +FIXTURE_DATA_TEST_CASE(RunWithBias, CLMatMulLowpKernelWithBiasFixture, framework::DatasetMode::ALL, combine(combine(combine(combine(combine(combine(combine(datasets::SmallMatMulDataset(), + framework::dataset::make("TransposeA", { true, false })), + framework::dataset::make("TransposeB", { true, false })), + m0_values_precommit), + n0_values_precommit), + k0_values_precommit), + framework::dataset::make("ExportRhsToCLImage", { false })), + framework::dataset::make("DataType", DataType::QASYMM8_SIGNED))) +{ + // Validate output + validate(CLAccessor(_target), _reference, tolerance_quant); +} FIXTURE_DATA_TEST_CASE(RunLargeNoTranspose, CLMatMulLowpNativeKernelFixture, framework::DatasetMode::NIGHTLY, combine(combine(combine(combine(combine(combine(combine(datasets::LargeMatMulDataset(), framework::dataset::make("TransposeA", { false })), diff --git a/tests/validation/CL/MatMulNativeMMULKernel.cpp b/tests/validation/CL/MatMulNativeMMULKernel.cpp new file mode 100644 index 0000000000000000000000000000000000000000..70c80985db1528be9d7ac9efb26274c5986be502 --- /dev/null +++ b/tests/validation/CL/MatMulNativeMMULKernel.cpp @@ -0,0 +1,501 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "arm_compute/runtime/CL/CLTensor.h" +#include "src/gpu/cl/kernels/ClMatMulNativeMMULKernel.h" +#include "tests/datasets/LargeMatMulMMULDataset.h" +#include "tests/datasets/SmallMatMulMMULDataset.h" +#include "tests/framework/Macros.h" +#include "tests/framework/datasets/Datasets.h" +#include "tests/validation/Validation.h" +#include "tests/validation/fixtures/MatMulKernelFixture.h" +#include "tests/validation/reference/Permute.h" + +#include + +namespace arm_compute +{ +namespace test +{ +namespace validation +{ +namespace +{ +RelativeTolerance tolerance_f32(0.001f); /**< Tolerance value for comparing reference's output against implementation's output for floating point data types */ +constexpr float abs_tolerance_f32( + 0.0001f); /**< Absolute tolerance value for comparing reference's output against implementation's output for floating point data types in case using relative tolerance fails because of small values */ +constexpr float abs_tolerance_f16( + 0.001f); /**< Absolute tolerance value for comparing reference's output against implementation's output for fp16 data types in case using relative tolerance fails because of small values */ +RelativeTolerance tolerance_f16(half(0.01)); /**< Tolerance value for comparing reference's output against implementation's output for floating point data types */ +} // namespace + +/** M0 values to test --precommit*/ +const auto m0_values_precommit = framework::dataset::make("M0", { 1, 3 }); + +/** N0 values to test --precommit*/ +const auto n0_values_precommit = framework::dataset::make("N0", { 2, 4 }); + +/** M0 values to test --nightly*/ +const auto m0_values_nightly_lhs_nt = framework::dataset::make("M0", { 1, 2, 3, 4, 5, 6, 7, 8 }); +const auto m0_values_nightly_lhs_t = framework::dataset::make("M0", { 1, 2, 3, 4, 8 }); + +/** N0 values to test --nightly*/ +const auto n0_values_nightly_rhs_nt = framework::dataset::make("N0", { 1, 2, 3, 4, 8, 16 }); +const auto n0_values_nightly_rhs_t = framework::dataset::make("N0", { 1, 2, 3, 4, 8 }); + +/** K0 value -- Fixed to 1 */ +const auto k0_value = framework::dataset::make("K0", { 1 }); + +template +using CLMatMulNativeMMULKernelFixture = MatMulKernelValidationFixture; + +template +using CLMatMulKernelBiasFixture = MatMulKernelWithBiasValidation; + +TEST_SUITE(CL) +TEST_SUITE(MatMulNativeMMULKernel) +TEST_SUITE(Validate) + +TEST_CASE(SupportedBlockSizes, framework::DatasetMode::ALL) +{ + if(arm_matrix_multiply_supported(CLKernelLibrary::get().get_device())) + { + using MatMulConfigurationPair = std::pair; + + const std::vector supported_block_sizes = + { + // MatMulKernelInfo(adj_lhs, adj_rhs, M0, N0, K0, export_rhs_to_cl_image = false) + // Lhs not-transposed, Rhs not-transposed + { MatMulKernelInfo(false, false, 0, 1, 1), false }, // M0 should be > 0 + { MatMulKernelInfo(false, false, 3, 5, 1), false }, // N0 not in {1, 2, 3, 4, 8, 16} + { MatMulKernelInfo(false, false, 3, 6, 1), false }, // N0 not in {1, 2, 3, 4, 8, 16} + { MatMulKernelInfo(false, false, 3, 3, 4), false }, // K0 not 1 + { MatMulKernelInfo(false, false, 9, 1, 1), true }, + { MatMulKernelInfo(false, false, 3, 16, 1), true }, + { MatMulKernelInfo(false, false, 7, 3, 1), true }, + + // Lhs transposed, Rhs not-transposed + { MatMulKernelInfo(true, false, 3, 11, 1), false }, // N0 not in {1, 2, 3, 4, 8, 16} + { MatMulKernelInfo(true, false, 3, 7, 1), false }, // N0 not in {1, 2, 3, 4, 8, 16} + { MatMulKernelInfo(true, false, 6, 3, 1), false }, // M0 not in {1, 2, 3, 4, 8, 16} + { MatMulKernelInfo(true, false, 5, 3, 1), false }, // M0 not in {1, 2, 3, 4, 8, 16} + { MatMulKernelInfo(true, false, 2, 2, 2), false }, // K0 is not 1 + { MatMulKernelInfo(true, false, 4, 1, 1), true }, + { MatMulKernelInfo(true, false, 3, 3, 1), true }, + { MatMulKernelInfo(true, false, 2, 4, 1), true }, + + // Lhs not-transposed, Rhs not-transposed + { MatMulKernelInfo(false, true, 3, 11, 1), false }, // N0 not in {1, 2, 3, 4, 8} + { MatMulKernelInfo(false, true, 2, 17, 1), false }, // N0 not in {1, 2, 3, 4, 8} + { MatMulKernelInfo(false, true, 4, 5, 1), false }, // N0 not in {1, 2, 3, 4, 8} + { MatMulKernelInfo(false, true, 4, 4, 7), false }, // K0 is not 1 + { MatMulKernelInfo(false, true, 4, 7, 1), false }, // N0 not in {1, 2, 3, 4, 8} + { MatMulKernelInfo(false, true, 3, 8, 1), true }, + { MatMulKernelInfo(false, true, 8, 16, 1), true }, + { MatMulKernelInfo(false, true, 2, 4, 1), true }, + + // Lhs transposed, Rhs transposed + { MatMulKernelInfo(true, true, 3, 11, 1), false }, // N0 not in {1, 2, 3, 4, 8, 16} + { MatMulKernelInfo(true, true, 3, 7, 1), false }, // N0 not in {1, 2, 3, 4, 8, 16} + { MatMulKernelInfo(true, true, 6, 3, 1), false }, // M0 not in {1, 2, 3, 4, 8, 16} + { MatMulKernelInfo(true, true, 5, 3, 1), false }, // M0 not in {1, 2, 3, 4, 8, 16} + { MatMulKernelInfo(true, true, 4, 8, 2), false }, // K0 is not 1 + { MatMulKernelInfo(true, true, 4, 8, 1), true }, + { MatMulKernelInfo(true, true, 3, 3, 1), true }, + { MatMulKernelInfo(true, true, 16, 4, 1), true }, + }; + + // Set big enough shapes so that block sizes are not truncated. Also, set all dimensions equal + // so that it doesn't fail for different NT/T configurations. We aim to test the block sizes here, + // not the shapes themselves. + const TensorInfo lhs_info = TensorInfo(TensorShape(100U, 100U), 1, DataType::F32); + const TensorInfo rhs_info = TensorInfo(TensorShape(100U, 100U), 1, DataType::F32); + + for(auto &pair : supported_block_sizes) + { + TensorInfo output_info; + Status status = ClMatMulNativeMMULKernel::validate(&lhs_info, &rhs_info, nullptr, &output_info, pair.first); + ARM_COMPUTE_EXPECT(bool(status) == pair.second, framework::LogLevel::ERRORS); + } + } + else + { + ARM_COMPUTE_TEST_INFO("cl_arm_matrix_multiply not supported. TEST skipped"); + framework::ARM_COMPUTE_PRINT_INFO(); + } +} + +TEST_CASE(ValidateInputShapes, framework::DatasetMode::ALL) +{ + if(arm_matrix_multiply_supported(CLKernelLibrary::get().get_device())) + { + // Configurations are assumed to be Nt/Nt, but will be transposed inside the test to test other configurations + using ShapeConfigurationTuple = std::tuple; // lhs, rhs, bias, result + const std::vector shape_configurations = + { + { TensorShape(4U, 1U), TensorShape(3U, 4U), TensorShape(3U), true }, + { TensorShape(12U, 12U), TensorShape(3U, 12U), TensorShape(3U), true }, + { TensorShape(8U, 4U), TensorShape(2U, 8U), TensorShape(2U), true }, + { TensorShape(8U, 4U), TensorShape(2U, 4U), TensorShape(2U), false }, // Mismatch in the K dimension + { TensorShape(5U, 0U), TensorShape(2U, 5U), TensorShape(2U), false }, // Invalid dimension + { TensorShape(5U, 7U), TensorShape(2U, 5U), TensorShape(2U), false }, // K not a multiple of 4 (MMUL_K0) + { TensorShape(8U, 4U, 3U, 4U, 5U, 6U), TensorShape(2U, 8U, 3U, 4U, 5U, 6U), TensorShape(2U), true }, + { TensorShape(5U, 4U, 3U, 4U, 5U, 1U), TensorShape(2U, 5U, 3U, 4U, 5U, 6U), TensorShape(2U), false }, // No batch broadcasting + { TensorShape(5U, 4U, 3U, 4U, 9U, 6U), TensorShape(2U, 5U, 3U, 4U, 5U, 6U), TensorShape(2U), false }, // Mismatch in batch dimension + { TensorShape(4U, 1U), TensorShape(3U, 4U), TensorShape(1U), false }, // Bias first dimensions != dst first dimension. + { TensorShape(4U, 1U), TensorShape(3U, 4U), TensorShape(5U, 6U), false }, // Bias is 2d which is invalid. + }; + + for(auto &tuple : shape_configurations) + { + const bool expected = std::get<3>(tuple); + + for(bool adj_lhs : + { + false, true + }) + { + for(bool adj_rhs : + { + false, true + }) + { + TensorShape lhs_shape = std::get<0>(tuple); + TensorShape rhs_shape = std::get<1>(tuple); + TensorShape bia_shape = std::get<2>(tuple); + + if(adj_lhs) + { + permute(lhs_shape, PermutationVector(1U, 0U)); + } + + if(adj_rhs) + { + permute(rhs_shape, PermutationVector(1U, 0U)); + } + + const TensorInfo lhs_info = TensorInfo(lhs_shape, 1, DataType::F32); + const TensorInfo rhs_info = TensorInfo(rhs_shape, 1, DataType::F32); + const TensorInfo bia_info = TensorInfo(bia_shape, 1, DataType::F32); + TensorInfo output_info; + + MatMulKernelInfo matmul_kernel_info{ adj_lhs, adj_rhs, 1, 1, 1, false /* export_rhs_to_cl_image */ }; + + Status status = ClMatMulNativeMMULKernel::validate(&lhs_info, &rhs_info, &bia_info, &output_info, matmul_kernel_info); + ARM_COMPUTE_EXPECT(bool(status) == expected, framework::LogLevel::ERRORS); + } + } + } + } + else + { + ARM_COMPUTE_TEST_INFO("cl_arm_matrix_multiply not supported. TEST skipped"); + framework::ARM_COMPUTE_PRINT_INFO(); + } +} + +TEST_CASE(ValidateDataTypes, framework::DatasetMode::ALL) +{ + if(arm_matrix_multiply_supported(CLKernelLibrary::get().get_device())) + { + // Configurations are assumed to be Nt/Nt, but will be transposed inside the test to test other configurations + using DataTypeConfigurationTuple = std::tuple; + const std::vector data_type_configurations = + { + { DataType::F32, DataType::F32, DataType::F32, DataType::F32, true }, + { DataType::F16, DataType::F16, DataType::F16, DataType::F16, true }, + { DataType::F32, DataType::F32, DataType::F32, DataType::F32, true }, + { DataType::F32, DataType::F32, DataType::F16, DataType::F32, false }, // incorrect bias type + { DataType::F16, DataType::F32, DataType::F32, DataType::F32, false }, // no mixed precision + { DataType::F64, DataType::F64, DataType::F64, DataType::F64, false }, // no double precision + { DataType::QASYMM8, DataType::QASYMM8, DataType::S32, DataType::QASYMM8, false }, // no quantized types + { DataType::QASYMM8_SIGNED, DataType::QASYMM8_SIGNED, DataType::S32, DataType::QASYMM8_SIGNED, false }, // no quantized types + { DataType::QSYMM8_PER_CHANNEL, DataType::QSYMM8_PER_CHANNEL, DataType::S32, DataType::QSYMM8_PER_CHANNEL, false }, // no quantized types + { DataType::QASYMM16, DataType::QASYMM16, DataType::S32, DataType::QASYMM16, false }, // no quantized types + { DataType::QSYMM16, DataType::QSYMM16, DataType::S32, DataType::QSYMM16, false }, // no quantized types + { DataType::QSYMM8, DataType::QSYMM8, DataType::S32, DataType::QSYMM8, false }, // no quantized types + { DataType::S64, DataType::S64, DataType::S64, DataType::S64, false }, // no integral types + { DataType::S32, DataType::S32, DataType::S32, DataType::S32, false }, // no integral types + { DataType::S16, DataType::S16, DataType::S16, DataType::S16, false }, // no integral types + { DataType::S8, DataType::S8, DataType::S8, DataType::S8, false }, // no integral types + { DataType::U64, DataType::U64, DataType::U64, DataType::U64, false }, // no integral types + { DataType::U32, DataType::U32, DataType::U32, DataType::U32, false }, // no integral types + { DataType::U16, DataType::U16, DataType::U16, DataType::U16, false }, // no integral types + { DataType::U8, DataType::U8, DataType::U8, DataType::U8, false }, // no integral types + }; + + const TensorShape shape = TensorShape(8U, 8U); + const TensorShape bia_shape = TensorShape(8U); + const MatMulKernelInfo matmul_kernel_info{ false, false, 1, 1, 1, false }; + for(auto &tuple : data_type_configurations) + { + const bool expected = std::get<4>(tuple); + + const TensorInfo lhs_info(shape, 1, std::get<0>(tuple)); + const TensorInfo rhs_info(shape, 1, std::get<1>(tuple)); + const TensorInfo bia_info(bia_shape, 1, std::get<2>(tuple)); + TensorInfo output_info(shape, 1, std::get<3>(tuple)); + + Status status = ClMatMulNativeMMULKernel::validate(&lhs_info, &rhs_info, &bia_info, &output_info, matmul_kernel_info); + ARM_COMPUTE_EXPECT(bool(status) == expected, framework::LogLevel::ERRORS); + } + } + else + { + ARM_COMPUTE_TEST_INFO("cl_arm_matrix_multiply not supported. TEST skipped"); + framework::ARM_COMPUTE_PRINT_INFO(); + } +} + +TEST_SUITE_END() // Validate + +TEST_SUITE(Float) +TEST_SUITE(FP32) +TEST_SUITE(Buffer) +FIXTURE_DATA_TEST_CASE(RunTiny, CLMatMulNativeMMULKernelFixture, framework::DatasetMode::ALL, combine(combine(combine(combine(combine(combine(combine(datasets::TinyMatMulMMULDataset(), + framework::dataset::make("TransposeA", { false, true })), + framework::dataset::make("TransposeB", { false, true })), + m0_values_precommit), + n0_values_precommit), + k0_value), + framework::dataset::make("ExportRhsToCLImage", { false })), + framework::dataset::make("DataType", DataType::F32))) +{ + // Validate output + if(_device_supports_mmul) + { + validate(CLAccessor(_target), _reference, tolerance_f32, 0.f, abs_tolerance_f32); + } +} +FIXTURE_DATA_TEST_CASE(RunSmall, CLMatMulNativeMMULKernelFixture, framework::DatasetMode::ALL, combine(combine(combine(combine(combine(combine(combine(datasets::SmallMatMulMMULDataset(), + framework::dataset::make("TransposeA", { false, true })), + framework::dataset::make("TransposeB", { false, true })), + m0_values_precommit), + n0_values_precommit), + k0_value), + framework::dataset::make("ExportRhsToCLImage", { false })), + framework::dataset::make("DataType", DataType::F32))) +{ + // Validate output + if(_device_supports_mmul) + { + validate(CLAccessor(_target), _reference, tolerance_f32, 0.f, abs_tolerance_f32); + } +} +FIXTURE_DATA_TEST_CASE(RunWithBias, CLMatMulKernelBiasFixture, framework::DatasetMode::ALL, combine(combine(combine(combine(combine(combine(combine(datasets::SmallMatMulMMULDataset(), + framework::dataset::make("TransposeA", { false, true })), + framework::dataset::make("TransposeB", { false, true })), + m0_values_precommit), + n0_values_precommit), + k0_value), + framework::dataset::make("ExportRhsToCLImage", { false })), + framework::dataset::make("DataType", DataType::F32))) +{ + // Validate output + if(_device_supports_mmul) + { + validate(CLAccessor(_target), _reference, tolerance_f32, 0.f, abs_tolerance_f32); + } +} +FIXTURE_DATA_TEST_CASE(RunLargeNoTranspose, CLMatMulNativeMMULKernelFixture, framework::DatasetMode::NIGHTLY, + combine(combine(combine(combine(combine(combine(combine(datasets::LargeMatMulMMULDataset(), + framework::dataset::make("TransposeA", { false })), + framework::dataset::make("TransposeB", { false })), + m0_values_nightly_lhs_nt), + n0_values_nightly_rhs_nt), + k0_value), + framework::dataset::make("ExportRhsToCLImage", { false })), + framework::dataset::make("DataType", DataType::F32))) +{ + // Validate output + if(_device_supports_mmul) + { + validate(CLAccessor(_target), _reference, tolerance_f32, 0.f, abs_tolerance_f32); + } +} + +FIXTURE_DATA_TEST_CASE(RunLargeRhsTranspose, CLMatMulNativeMMULKernelFixture, framework::DatasetMode::NIGHTLY, + combine(combine(combine(combine(combine(combine(combine(datasets::LargeMatMulMMULDataset(), + framework::dataset::make("TransposeA", { false })), + framework::dataset::make("TransposeB", { true })), + m0_values_nightly_lhs_nt), + n0_values_nightly_rhs_t), + k0_value), + framework::dataset::make("ExportRhsToCLImage", { false })), + framework::dataset::make("DataType", DataType::F32))) +{ + // Validate output + if(_device_supports_mmul) + { + validate(CLAccessor(_target), _reference, tolerance_f32, 0.f, abs_tolerance_f32); + } +} +FIXTURE_DATA_TEST_CASE(RunLargeLhsTransposed, CLMatMulNativeMMULKernelFixture, framework::DatasetMode::NIGHTLY, + combine(combine(combine(combine(combine(combine(combine(datasets::LargeMatMulMMULDataset(), + framework::dataset::make("TransposeA", { true })), + framework::dataset::make("TransposeB", { false })), + m0_values_nightly_lhs_t), + n0_values_nightly_rhs_nt), + k0_value), + framework::dataset::make("ExportRhsToCLImage", { false })), + framework::dataset::make("DataType", DataType::F32))) +{ + // Validate output + // Validate output + if(_device_supports_mmul) + { + validate(CLAccessor(_target), _reference, tolerance_f32, 0.f, abs_tolerance_f32); + } +} +FIXTURE_DATA_TEST_CASE(RunLargeLhsTransposedRhsTransposed, CLMatMulNativeMMULKernelFixture, framework::DatasetMode::NIGHTLY, + combine(combine(combine(combine(combine(combine(combine(datasets::LargeMatMulMMULDataset(), + framework::dataset::make("TransposeA", { true })), + framework::dataset::make("TransposeB", { true })), + m0_values_nightly_lhs_t), + n0_values_nightly_rhs_t), + k0_value), + framework::dataset::make("ExportRhsToCLImage", { false })), + framework::dataset::make("DataType", DataType::F32))) +{ + // Validate output + // Validate output + if(_device_supports_mmul) + { + validate(CLAccessor(_target), _reference, tolerance_f32, 0.f, abs_tolerance_f32); + } +} +// Running High Dimensional test is enough for FP32, because we're stressing the number of dimensions, not data type or M0/N0/K0 +// It's a good idea to test for each Lhs/Rhs T/NT combinations because they're different CL kernels +FIXTURE_DATA_TEST_CASE(RunHighDimensional, CLMatMulNativeMMULKernelFixture, framework::DatasetMode::ALL, + combine(combine(combine(combine(combine(combine(combine(datasets::HighDimensionalMatMulMMULDataset(), + framework::dataset::make("TransposeA", { false, true })), + framework::dataset::make("TransposeB", { false, true })), + framework::dataset::make("M0", { 2 })), + framework::dataset::make("N0", { 2 })), + framework::dataset::make("K0", { 1 })), + framework::dataset::make("ExportRhsToCLImage", { false })), + framework::dataset::make("DataType", DataType::F32))) +{ + // Validate output + if(_device_supports_mmul) + { + validate(CLAccessor(_target), _reference, tolerance_f32, 0.f, abs_tolerance_f32); + } +} +TEST_SUITE_END() // Buffer + +TEST_SUITE_END() // FP32 + +TEST_SUITE(FP16) +TEST_SUITE(Buffer) +FIXTURE_DATA_TEST_CASE(RunSmall, CLMatMulNativeMMULKernelFixture, framework::DatasetMode::ALL, combine(combine(combine(combine(combine(combine(combine(datasets::SmallMatMulMMULDataset(), + framework::dataset::make("TransposeA", { false, true })), + framework::dataset::make("TransposeB", { false, true })), + m0_values_precommit), + n0_values_precommit), + k0_value), + framework::dataset::make("ExportRhsToCLImage", { false })), + framework::dataset::make("DataType", DataType::F16))) +{ + // Validate output + if(_device_supports_mmul) + { + validate(CLAccessor(_target), _reference, tolerance_f16, 0.f, abs_tolerance_f16); + } +} +FIXTURE_DATA_TEST_CASE(RunLargeNoTranspose, CLMatMulNativeMMULKernelFixture, framework::DatasetMode::NIGHTLY, + combine(combine(combine(combine(combine(combine(combine(datasets::LargeMatMulMMULDataset(), + framework::dataset::make("TransposeA", { false })), + framework::dataset::make("TransposeB", { false })), + m0_values_nightly_lhs_nt), + n0_values_nightly_rhs_nt), + k0_value), + framework::dataset::make("ExportRhsToCLImage", { false })), + framework::dataset::make("DataType", DataType::F16))) +{ + // Validate output + if(_device_supports_mmul) + { + validate(CLAccessor(_target), _reference, tolerance_f16, 0.f, abs_tolerance_f16); + } +} +FIXTURE_DATA_TEST_CASE(RunLargeRhsTranspose, CLMatMulNativeMMULKernelFixture, framework::DatasetMode::NIGHTLY, + combine(combine(combine(combine(combine(combine(combine(datasets::LargeMatMulMMULDataset(), + framework::dataset::make("TransposeA", { false })), + framework::dataset::make("TransposeB", { true })), + m0_values_nightly_lhs_nt), + n0_values_nightly_rhs_t), + k0_value), + framework::dataset::make("ExportRhsToCLImage", { false })), + framework::dataset::make("DataType", DataType::F16))) +{ + // Validate output + if(_device_supports_mmul) + { + validate(CLAccessor(_target), _reference, tolerance_f16, 0.f, abs_tolerance_f16); + } +} +FIXTURE_DATA_TEST_CASE(RunLargeLhsTransposed, CLMatMulNativeMMULKernelFixture, framework::DatasetMode::NIGHTLY, + combine(combine(combine(combine(combine(combine(combine(datasets::LargeMatMulMMULDataset(), + framework::dataset::make("TransposeA", { true })), + framework::dataset::make("TransposeB", { false })), + m0_values_nightly_lhs_t), + n0_values_nightly_rhs_nt), + k0_value), + framework::dataset::make("ExportRhsToCLImage", { false })), + framework::dataset::make("DataType", DataType::F16))) +{ + // Validate output + // Validate output + if(_device_supports_mmul) + { + validate(CLAccessor(_target), _reference, tolerance_f16, 0.f, abs_tolerance_f16); + } +} +FIXTURE_DATA_TEST_CASE(RunLargeLhsTransposedRhsTransposed, CLMatMulNativeMMULKernelFixture, framework::DatasetMode::NIGHTLY, + combine(combine(combine(combine(combine(combine(combine(datasets::LargeMatMulMMULDataset(), + framework::dataset::make("TransposeA", { true })), + framework::dataset::make("TransposeB", { true })), + m0_values_nightly_lhs_t), + n0_values_nightly_rhs_t), + k0_value), + framework::dataset::make("ExportRhsToCLImage", { false })), + framework::dataset::make("DataType", DataType::F16))) +{ + // Validate output + // Validate output + if(_device_supports_mmul) + { + validate(CLAccessor(_target), _reference, tolerance_f16, 0.f, abs_tolerance_f16); + } +} +TEST_SUITE_END() // Buffer + +TEST_SUITE_END() // FP16 +TEST_SUITE_END() // Float +TEST_SUITE_END() // MatMulNativeMMULKernel +TEST_SUITE_END() // CL +} // namespace validation +} // namespace test +} // namespace arm_compute diff --git a/tests/validation/CL/ReduceMean.cpp b/tests/validation/CL/ReduceMean.cpp index 947f84af49d6c8fe69985721b032eea07384a8bf..8a8fa4aef0370d687193fe5c671a45bafdba6339 100644 --- a/tests/validation/CL/ReduceMean.cpp +++ b/tests/validation/CL/ReduceMean.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2020 Arm Limited. + * Copyright (c) 2018-2020, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -45,7 +45,7 @@ constexpr AbsoluteTolerance tolerance_f32(0.001f); /**< Tolerance value constexpr AbsoluteTolerance tolerance_f16(0.03f); /**< Tolerance value for comparing reference's output against implementation's output for 16-bit floating-point type */ constexpr AbsoluteTolerance tolerance_qasymm8(1); /**< Tolerance value for comparing reference's output against implementation's output for 8-bit asymmetric quantized type */ -const auto axis_keep = combine(framework::dataset::make("Axis", { Coordinates(0), Coordinates(1, 0), Coordinates(1, 2), Coordinates(0, 2), Coordinates(1, 3), Coordinates(0, 1, 2, 3) }), +const auto axis_keep = combine(framework::dataset::make("Axis", { Coordinates(0), Coordinates(1, 0), Coordinates(1, 2), Coordinates(0, 2), Coordinates(1, 3), Coordinates(2, 3), Coordinates(0, 1, 2, 3) }), framework::dataset::make("KeepDims", { true })); const auto axis_drop = combine(framework::dataset::make("Axis", { Coordinates(0), Coordinates(1), Coordinates(3), Coordinates(1, 2), Coordinates(2, 1) }), framework::dataset::make("KeepDims", { false })); } // namespace diff --git a/tests/validation/CL/Tile.cpp b/tests/validation/CL/Tile.cpp index a06c05744f8526288893332c7e00d509b1bd53af..f243780c00e5e205a07306d675655ce6ebbf67bd 100644 --- a/tests/validation/CL/Tile.cpp +++ b/tests/validation/CL/Tile.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2020 Arm Limited. + * Copyright (c) 2018-2020, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -42,6 +42,7 @@ namespace validation namespace { const auto MultiplesDataset = framework::dataset::make("Multiples", { Multiples{ 3 }, + Multiples{ 7 }, Multiples{ 2, 2 }, Multiples{ 1, 1, 3, 4 }, Multiples{ 2, 1, 2, 2 }, diff --git a/tests/validation/Helpers.h b/tests/validation/Helpers.h index 5cdba4606ee73ef097ef982edef97b0fa5c0891b..df3b08ba536b26744adc97b503b5257ee714a174 100644 --- a/tests/validation/Helpers.h +++ b/tests/validation/Helpers.h @@ -26,6 +26,7 @@ #include "arm_compute/core/Types.h" #include "arm_compute/core/Utils.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" #include "support/Half.h" #include "tests/Globals.h" #include "tests/SimpleTensor.h" diff --git a/tests/validation/NEON/ActivationLayer.cpp b/tests/validation/NEON/ActivationLayer.cpp index d64945ab31c7aad24829ad0034eb55ba6bc1e932..73f5de68ac74eaa3214ab6c45924ee144b9086bf 100644 --- a/tests/validation/NEON/ActivationLayer.cpp +++ b/tests/validation/NEON/ActivationLayer.cpp @@ -23,6 +23,7 @@ */ #include "arm_compute/core/Types.h" #include "arm_compute/core/utils/misc/Traits.h" +#include "arm_compute/core/utils/StringUtils.h" #include "arm_compute/runtime/NEON/functions/NEActivationLayer.h" #include "arm_compute/runtime/RuntimeContext.h" #include "arm_compute/runtime/Tensor.h" @@ -316,10 +317,16 @@ DATA_TEST_CASE(KernelSelection, framework::DatasetMode::ALL, concat(concat( const auto *selected_impl = CpuActivationKernel::get_implementation(ActivationDataTypeISASelectorData{data_type, CPUModel::GENERIC, cpu_isa,ActivationLayerInfo::ActivationFunction::BOUNDED_RELU}, cpu::KernelSelectionType::Preferred); ARM_COMPUTE_ERROR_ON_NULLPTR(selected_impl); - std::string expected = lower_string(cpu_ext) + "_" + cpu_impl_dt(data_type) + "_activation"; + if( data_type == DataType::QASYMM8 || data_type == DataType::QASYMM8_SIGNED) + { +#ifdef __aarch64__ + expected = "neon_q8_activation_lut"; +#else // __aarch64__ + expected = lower_string(cpu_ext) + "_" + cpu_impl_dt(data_type) + "_activation"; +#endif // __aarch64__ + } std::string actual = selected_impl->name; - ARM_COMPUTE_EXPECT_EQUAL(expected, actual, framework::LogLevel::ERRORS); } // clang-format on diff --git a/tests/validation/NEON/ArgMinMax.cpp b/tests/validation/NEON/ArgMinMax.cpp index 0a4071076a50b7cec6d02fc2dfd5bdc55ea396ad..2e21a7db7b20f6e07aef4fea693e6579bf1a2f01 100644 --- a/tests/validation/NEON/ArgMinMax.cpp +++ b/tests/validation/NEON/ArgMinMax.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -43,6 +43,27 @@ namespace test { namespace validation { +namespace +{ +const auto OpsDataset = framework::dataset::make("Operation", { ReductionOperation::ARG_IDX_MIN, ReductionOperation::ARG_IDX_MAX }); +const auto AxisDataset = framework::dataset::make("Axis", { 0, 1, 2, 3 }); +const auto QInfoDataset = framework::dataset::make("QuantizationInfo", { QuantizationInfo(5.f / 255.f, 20) }); + +const auto ArgMinMaxSmallDatasetAxis0 = framework::dataset::make("Shape", +{ + TensorShape{ 1U, 5U }, + TensorShape{ 2U, 3U }, + TensorShape{ 1U }, + TensorShape{ 3U }, + TensorShape{ 2U }, + TensorShape{ 5U }, + TensorShape{ 17U }, + TensorShape{ 15U, 2U }, +}); +using ArgMinMaxSmallDataset = datasets::Small4DShapes; +using ArgMinMaxLargeDataset = datasets::Large4DShapes; +} + TEST_SUITE(NEON) TEST_SUITE(ArgMinMax) @@ -70,23 +91,46 @@ DATA_TEST_CASE(Validate, framework::DatasetMode::ALL, zip(zip(zip(zip( // clang-format on // *INDENT-ON* -template -using NEArgMinMaxValidationFixture = ArgMinMaxValidationFixture; +template +using NEArgMinMaxValidationFixture = ArgMinMaxValidationFixture; +using NEArgMinMaxValidationFixture_S32_S32 = NEArgMinMaxValidationFixture; +using NEArgMinMaxValidationFixture_F16_S32 = NEArgMinMaxValidationFixture; +using NEArgMinMaxValidationFixture_F32_S32 = NEArgMinMaxValidationFixture; TEST_SUITE(S32) -FIXTURE_DATA_TEST_CASE(RunSmall, - NEArgMinMaxValidationFixture, +FIXTURE_DATA_TEST_CASE(RunSmallAxis0, + NEArgMinMaxValidationFixture_S32_S32, framework::DatasetMode::PRECOMMIT, - combine(combine(combine(datasets::Small4DShapes(), framework::dataset::make("DataType", DataType::S32)), framework::dataset::make("Axis", { 0, 1, 2, 3 })), framework::dataset::make("Operation", { ReductionOperation::ARG_IDX_MIN, ReductionOperation::ARG_IDX_MAX }))) + combine(combine(combine(combine(ArgMinMaxSmallDatasetAxis0, + framework::dataset::make("DataTypeIn", DataType::S32)), + framework::dataset::make("DataTypeOut", DataType::S32)), + framework::dataset::make("Axis", { 0 })), + OpsDataset)) { // Validate output validate(Accessor(_target), _reference); } +FIXTURE_DATA_TEST_CASE(RunSmall, + NEArgMinMaxValidationFixture_S32_S32, + framework::DatasetMode::PRECOMMIT, + combine(combine(combine(combine(ArgMinMaxSmallDataset(), + framework::dataset::make("DataTypeIn", DataType::S32)), + framework::dataset::make("DataTypeOut", DataType::S32)), + AxisDataset), + OpsDataset)) +{ + // Validate output + validate(Accessor(_target), _reference); +} FIXTURE_DATA_TEST_CASE(RunLarge, - NEArgMinMaxValidationFixture, + NEArgMinMaxValidationFixture_S32_S32, framework::DatasetMode::NIGHTLY, - combine(combine(combine(datasets::Large4DShapes(), framework::dataset::make("DataType", DataType::S32)), framework::dataset::make("Axis", { 0, 1, 2, 3 })), framework::dataset::make("Operation", { ReductionOperation::ARG_IDX_MIN, ReductionOperation::ARG_IDX_MAX }))) + combine(combine(combine(combine(ArgMinMaxLargeDataset(), + framework::dataset::make("DataTypeIn", DataType::S32)), + framework::dataset::make("DataTypeOut", DataType::S32)), + AxisDataset), + OpsDataset)) { // Validate output validate(Accessor(_target), _reference); @@ -97,18 +141,26 @@ TEST_SUITE(Float) #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC TEST_SUITE(FP16) FIXTURE_DATA_TEST_CASE(RunSmall, - NEArgMinMaxValidationFixture, + NEArgMinMaxValidationFixture_F16_S32, framework::DatasetMode::PRECOMMIT, - combine(combine(combine(datasets::Small4DShapes(), framework::dataset::make("DataType", DataType::F16)), framework::dataset::make("Axis", { 0, 1, 2, 3 })), framework::dataset::make("Operation", { ReductionOperation::ARG_IDX_MIN, ReductionOperation::ARG_IDX_MAX }))) + combine(combine(combine(combine(ArgMinMaxSmallDataset(), + framework::dataset::make("DataTypeIn", DataType::F16)), + framework::dataset::make("DataTypeOut", DataType::S32)), + AxisDataset), + OpsDataset)) { // Validate output validate(Accessor(_target), _reference); } FIXTURE_DATA_TEST_CASE(RunLarge, - NEArgMinMaxValidationFixture, + NEArgMinMaxValidationFixture_F16_S32, framework::DatasetMode::NIGHTLY, - combine(combine(combine(datasets::Large4DShapes(), framework::dataset::make("DataType", DataType::F16)), framework::dataset::make("Axis", { 0, 1, 2, 3 })), framework::dataset::make("Operation", { ReductionOperation::ARG_IDX_MIN, ReductionOperation::ARG_IDX_MAX }))) + combine(combine(combine(combine(ArgMinMaxLargeDataset(), + framework::dataset::make("DataTypeIn", DataType::F16)), + framework::dataset::make("DataTypeOut", DataType::S32)), + AxisDataset), + OpsDataset)) { // Validate output validate(Accessor(_target), _reference); @@ -118,18 +170,26 @@ TEST_SUITE_END() // FP16 TEST_SUITE(FP32) FIXTURE_DATA_TEST_CASE(RunSmall, - NEArgMinMaxValidationFixture, + NEArgMinMaxValidationFixture_F32_S32, framework::DatasetMode::PRECOMMIT, - combine(combine(combine(datasets::Small4DShapes(), framework::dataset::make("DataType", DataType::F32)), framework::dataset::make("Axis", { 0, 1, 2, 3 })), framework::dataset::make("Operation", { ReductionOperation::ARG_IDX_MIN, ReductionOperation::ARG_IDX_MAX }))) + combine(combine(combine(combine(ArgMinMaxSmallDataset(), + framework::dataset::make("DataTypeIn", DataType::F32)), + framework::dataset::make("DataTypeOut", DataType::S32)), + AxisDataset), + OpsDataset)) { // Validate output validate(Accessor(_target), _reference); } FIXTURE_DATA_TEST_CASE(RunLarge, - NEArgMinMaxValidationFixture, + NEArgMinMaxValidationFixture_F32_S32, framework::DatasetMode::NIGHTLY, - combine(combine(combine(datasets::Large4DShapes(), framework::dataset::make("DataType", DataType::F32)), framework::dataset::make("Axis", { 0, 1, 2, 3 })), framework::dataset::make("Operation", { ReductionOperation::ARG_IDX_MIN, ReductionOperation::ARG_IDX_MAX }))) + combine(combine(combine(combine(ArgMinMaxLargeDataset(), + framework::dataset::make("DataTypeIn", DataType::F32)), + framework::dataset::make("DataTypeOut", DataType::S32)), + AxisDataset), + OpsDataset)) { // Validate output validate(Accessor(_target), _reference); @@ -137,27 +197,35 @@ FIXTURE_DATA_TEST_CASE(RunLarge, TEST_SUITE_END() // FP32 TEST_SUITE_END() // Float -template -using NEArgMinMaxQuantizedValidationFixture = ArgMinMaxValidationQuantizedFixture; +template +using NEArgMinMaxQuantizedValidationFixture = ArgMinMaxValidationQuantizedFixture; + +using NEArgMinMaxQuantizedValidationFixture_U8_S32 = NEArgMinMaxQuantizedValidationFixture; +using NEArgMinMaxQuantizedValidationFixture_S8_S32 = NEArgMinMaxQuantizedValidationFixture; TEST_SUITE(QASYMM8) FIXTURE_DATA_TEST_CASE(RunSmall, - NEArgMinMaxQuantizedValidationFixture, + NEArgMinMaxQuantizedValidationFixture_U8_S32, framework::DatasetMode::PRECOMMIT, - combine(combine(combine(combine(datasets::Small4DShapes(), framework::dataset::make("DataType", DataType::QASYMM8)), framework::dataset::make("Axis", { 0, 1, 2, 3 })), - framework::dataset::make("Operation", { ReductionOperation::ARG_IDX_MIN, ReductionOperation::ARG_IDX_MAX })), - framework::dataset::make("QuantizationInfo", { QuantizationInfo(5.f / 255.f, 20) }))) + combine(combine(combine(combine(combine(ArgMinMaxSmallDataset(), + framework::dataset::make("DataTypeIn", DataType::QASYMM8)), + framework::dataset::make("DataTypeOut", DataType::S32)), + AxisDataset), + OpsDataset), + QInfoDataset)) { // Validate output validate(Accessor(_target), _reference); } - FIXTURE_DATA_TEST_CASE(RunLarge, - NEArgMinMaxQuantizedValidationFixture, + NEArgMinMaxQuantizedValidationFixture_U8_S32, framework::DatasetMode::NIGHTLY, - combine(combine(combine(combine(datasets::Large4DShapes(), framework::dataset::make("DataType", DataType::QASYMM8)), framework::dataset::make("Axis", { 0, 1, 2, 3 })), - framework::dataset::make("Operation", { ReductionOperation::ARG_IDX_MIN, ReductionOperation::ARG_IDX_MAX })), - framework::dataset::make("QuantizationInfo", { QuantizationInfo(5.f / 255.f, 20) }))) + combine(combine(combine(combine(combine(ArgMinMaxLargeDataset(), + framework::dataset::make("DataTypeIn", DataType::QASYMM8)), + framework::dataset::make("DataTypeOut", DataType::S32)), + AxisDataset), + OpsDataset), + QInfoDataset)) { // Validate output validate(Accessor(_target), _reference); @@ -166,22 +234,27 @@ TEST_SUITE_END() // QASYMM8 TEST_SUITE(QASYMM8_SIGNED) FIXTURE_DATA_TEST_CASE(RunSmall, - NEArgMinMaxQuantizedValidationFixture, + NEArgMinMaxQuantizedValidationFixture_S8_S32, framework::DatasetMode::PRECOMMIT, - combine(combine(combine(combine(datasets::Small4DShapes(), framework::dataset::make("DataType", DataType::QASYMM8_SIGNED)), framework::dataset::make("Axis", { 0, 1, 2, 3 })), - framework::dataset::make("Operation", { ReductionOperation::ARG_IDX_MIN, ReductionOperation::ARG_IDX_MAX })), - framework::dataset::make("QuantizationInfo", { QuantizationInfo(5.f / 127.f, 20) }))) + combine(combine(combine(combine(combine(ArgMinMaxSmallDataset(), + framework::dataset::make("DataTypeIn", DataType::QASYMM8_SIGNED)), + framework::dataset::make("DataTypeOut", DataType::S32)), + AxisDataset), + OpsDataset), + QInfoDataset)) { // Validate output validate(Accessor(_target), _reference); } - FIXTURE_DATA_TEST_CASE(RunLarge, - NEArgMinMaxQuantizedValidationFixture, + NEArgMinMaxQuantizedValidationFixture_S8_S32, framework::DatasetMode::NIGHTLY, - combine(combine(combine(combine(datasets::Large4DShapes(), framework::dataset::make("DataType", DataType::QASYMM8_SIGNED)), framework::dataset::make("Axis", { 0, 1, 2, 3 })), - framework::dataset::make("Operation", { ReductionOperation::ARG_IDX_MIN, ReductionOperation::ARG_IDX_MAX })), - framework::dataset::make("QuantizationInfo", { QuantizationInfo(5.f / 127.f, 20) }))) + combine(combine(combine(combine(combine(ArgMinMaxLargeDataset(), + framework::dataset::make("DataTypeIn", DataType::QASYMM8_SIGNED)), + framework::dataset::make("DataTypeOut", DataType::S32)), + AxisDataset), + OpsDataset), + QInfoDataset)) { // Validate output validate(Accessor(_target), _reference); diff --git a/tests/validation/NEON/ArithmeticAddition.cpp b/tests/validation/NEON/ArithmeticAddition.cpp index e18d61e4152b1af69a4bf47e3d37319bd0186c30..535c3e634ed1333b48482828648baa50399f63b2 100644 --- a/tests/validation/NEON/ArithmeticAddition.cpp +++ b/tests/validation/NEON/ArithmeticAddition.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2022 Arm Limited. + * Copyright (c) 2017-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,7 @@ * SOFTWARE. */ #include "arm_compute/core/Types.h" +#include "arm_compute/core/utils/StringUtils.h" #include "arm_compute/runtime/NEON/functions/NEArithmeticAddition.h" #include "arm_compute/runtime/Tensor.h" #include "arm_compute/runtime/TensorAllocator.h" diff --git a/tests/validation/NEON/Cast.cpp b/tests/validation/NEON/Cast.cpp index 3a77106a42f241c5120e3c12d27b5ec378d8ec91..a1ddcc9cad2dad3090d7d961a22efe58a0d87636 100644 --- a/tests/validation/NEON/Cast.cpp +++ b/tests/validation/NEON/Cast.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022 Arm Limited. + * Copyright (c) 2019-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,7 @@ * SOFTWARE. */ #include "arm_compute/core/Types.h" +#include "arm_compute/core/utils/StringUtils.h" #include "arm_compute/runtime/NEON/functions/NECast.h" #include "arm_compute/runtime/Tensor.h" #include "arm_compute/runtime/TensorAllocator.h" @@ -100,6 +101,11 @@ const auto CastF32toS32Dataset = combine(framework::dataset::make("Da const auto CastF32toQASYMM8Dataset = combine(framework::dataset::make("DataType", DataType::F32), framework::dataset::make("DataType", DataType::QASYMM8)); const auto CastF32toQASYMM8_SIGNEDDataset = combine(framework::dataset::make("DataType", DataType::F32), framework::dataset::make("DataType", DataType::QASYMM8_SIGNED)); +// U64 +const auto CastU64toF32Dataset = combine(framework::dataset::make("DataType", DataType::U64), framework::dataset::make("DataType", DataType::F32)); + +// S64 +const auto CastS64toF32Dataset = combine(framework::dataset::make("DataType", DataType::S64), framework::dataset::make("DataType", DataType::F32)); } // namespace TEST_SUITE(NEON) @@ -107,6 +113,8 @@ TEST_SUITE(Cast) template using NECastToU8Fixture = CastValidationFixture; template +using NECastToS8Fixture = CastValidationFixture; +template using NECastToU16Fixture = CastValidationFixture; template using NECastToS16Fixture = CastValidationFixture; @@ -115,6 +123,10 @@ using NECastToU32Fixture = CastValidationFixture using NECastToS32Fixture = CastValidationFixture; template +using NECastToU64Fixture = CastValidationFixture; +template +using NECastToS64Fixture = CastValidationFixture; +template using NECastToF16Fixture = CastValidationFixture; template using NECastToF32Fixture = CastValidationFixture; @@ -188,6 +200,14 @@ CAST_SUITE(F32_to_F16, DataType::F32, DataType::F16, NECastToF16Fixture, CAST_SUITE(F32_to_S32, DataType::F32, DataType::S32, NECastToS32Fixture, CastF32toS32Dataset, one_tolerance) CAST_SUITE(F32_to_U8, DataType::F32, DataType::S32, NECastToS32Fixture, CastF32toS32Dataset, one_tolerance) +#ifdef __aarch64__ +// S64 +CAST_SUITE(S64_to_F32, DataType::S64, DataType::F32, NECastToF32Fixture, CastS64toF32Dataset, zero_tolerance) + +// U64 +CAST_SUITE(U64_to_F32, DataType::U64, DataType::F32, NECastToF32Fixture, CastU64toF32Dataset, zero_tolerance) +#endif // __aarch64__ + DATA_TEST_CASE(KernelSelectionDstFP16, framework::DatasetMode::ALL, combine(framework::dataset::make("CpuExt", std::string("NEON")), framework::dataset::make("DataType", diff --git a/tests/validation/NEON/DeconvolutionLayer.cpp b/tests/validation/NEON/DeconvolutionLayer.cpp index af255431934ab3601177d5e30e4df82c3a0faedd..d26d26adf7e94d4bb40b3cc8d80dabc1feabf365 100644 --- a/tests/validation/NEON/DeconvolutionLayer.cpp +++ b/tests/validation/NEON/DeconvolutionLayer.cpp @@ -76,6 +76,9 @@ const auto data3x3_precommit = datasets::SmallDeconvolutionShapes() * framework: const auto data1x1 = datasets::SmallDeconvolutionShapes() * framework::dataset::make("StrideX", 1, 4) * framework::dataset::make("StrideY", 1, 4) * framework::dataset::make("PadX", 0, 1) * framework::dataset::make("PadY", 0, 1) * framework::dataset::make("NumKernels", { 3 }); +const auto data5x1 = datasets::SmallDeconvolutionShapes() * framework::dataset::make("StrideX", 1, 4) * framework::dataset::make("StrideY", 1, 4) * framework::dataset::make("PadX", 0, 1) + * framework::dataset::make("PadY", 0, 1) * framework::dataset::make("NumKernels", { 3 }); + const auto data_layouts_dataset = framework::dataset::make("DataLayout", { DataLayout::NCHW, DataLayout::NHWC }); const auto add_bias_dataset = framework::dataset::make("AddBias", { true, false }); @@ -159,6 +162,9 @@ using NEDeconvolutionLayerAsymmFixture9x9 = DeconvolutionValidationAsymmFixture< template using NEDeconvolutionLayerFixture1x1 = DeconvolutionValidationFixture; +template +using NEDeconvolutionLayerFixture5x1 = DeconvolutionValidationFixture; + TEST_SUITE(Float) TEST_SUITE(FP32) TEST_SUITE(W4x4) @@ -222,6 +228,15 @@ FIXTURE_DATA_TEST_CASE(RunLarge, NEDeconvolutionLayerAsymmFixture9x9, fra validate(Accessor(_target), _reference, tolerance_fp32); } TEST_SUITE_END() // W9x9 +TEST_SUITE(W5x1) +FIXTURE_DATA_TEST_CASE(Run, NEDeconvolutionLayerFixture5x1, framework::DatasetMode::NIGHTLY, combine(combine(combine(data5x1, framework::dataset::make("DataType", DataType::F32)), + data_layouts_dataset), + add_bias_dataset)) +{ + // Validate output + validate(Accessor(_target), _reference, tolerance_fp32); +} +TEST_SUITE_END() // W5x1 TEST_SUITE_END() // FP32 #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC @@ -261,6 +276,15 @@ FIXTURE_DATA_TEST_CASE(Run, NEDeconvolutionLayerFixture1x1, framework::Dat validate(Accessor(_target), _reference, tolerance_fp16, tolerance_num_fp16); } TEST_SUITE_END() // W1x1 +TEST_SUITE(W5x1) +FIXTURE_DATA_TEST_CASE(Run, NEDeconvolutionLayerFixture5x1, framework::DatasetMode::NIGHTLY, combine(combine(combine(data5x1, framework::dataset::make("DataType", DataType::F16)), + data_layouts_dataset), + add_bias_dataset)) +{ + // Validate output + validate(Accessor(_target), _reference, tolerance_fp16, tolerance_num_fp16); +} +TEST_SUITE_END() // W5x1 TEST_SUITE_END() // FP16 #endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ @@ -275,6 +299,9 @@ using NEDeconvolutionLayerQuantizedFixture3x3 = DeconvolutionValidationQuantized template using NEDeconvolutionLayerQuantizedFixture1x1 = DeconvolutionValidationQuantizedFixture; +template +using NEDeconvolutionLayerQuantizedFixture5x1 = DeconvolutionValidationQuantizedFixture; + template using NEDeconvolutionLayerQuantizedPerChannelFixture4x4 = DeconvolutionValidationQuantizedPerChannelFixture; @@ -284,6 +311,9 @@ using NEDeconvolutionLayerQuantizedPerChannelFixture3x3 = DeconvolutionValidatio template using NEDeconvolutionLayerQuantizedPerChannelFixture1x1 = DeconvolutionValidationQuantizedPerChannelFixture; +template +using NEDeconvolutionLayerQuantizedPerChannelFixture5x1 = DeconvolutionValidationQuantizedPerChannelFixture; + TEST_SUITE(Quantized) TEST_SUITE(QASYMM8) @@ -338,6 +368,19 @@ FIXTURE_DATA_TEST_CASE(Run, NEDeconvolutionLayerQuantizedFixture1x1, fr } TEST_SUITE_END() // W1x1 +TEST_SUITE(W5x1) +FIXTURE_DATA_TEST_CASE(Run, NEDeconvolutionLayerQuantizedFixture5x1, framework::DatasetMode::NIGHTLY, combine(combine(combine(combine(combine(data5x1, framework::dataset::make("DataType", + DataType::QASYMM8)), + data_layouts_dataset), + input_qinfo_dataset), + output_qinfo_dataset), + add_bias_dataset)) +{ + // Validate output + validate(Accessor(_target), _reference, tolerance_quantized, tolerance_num_quant); +} +TEST_SUITE_END() // W5x1 + TEST_SUITE_END() // QASYMM8 TEST_SUITE(QASYMM8_SIGNED) @@ -394,6 +437,19 @@ FIXTURE_DATA_TEST_CASE(Run, NEDeconvolutionLayerQuantizedFixture1x1, fra } TEST_SUITE_END() // W1x1 +TEST_SUITE(W5x1) +FIXTURE_DATA_TEST_CASE(Run, NEDeconvolutionLayerQuantizedFixture5x1, framework::DatasetMode::NIGHTLY, combine(combine(combine(combine(combine(data5x1, framework::dataset::make("DataType", + DataType::QASYMM8_SIGNED)), + data_layouts_dataset), + input_qinfo_dataset), + output_qinfo_dataset), + add_bias_dataset)) +{ + // Validate output + validate(Accessor(_target), _reference, tolerance_quantized, tolerance_num_quant); +} +TEST_SUITE_END() // W5x1 + TEST_SUITE_END() // QASYMM8_SIGNED const auto input_qinfo_per_channel_dataset = framework::dataset::make("InputQuantizationInfo", { QuantizationInfo(1.f / 255.f, 10) }); @@ -478,6 +534,31 @@ FIXTURE_DATA_TEST_CASE(RunSigned, NEDeconvolutionLayerQuantizedPerChannelFixture } TEST_SUITE_END() // W1x1 +TEST_SUITE(W5x1) +FIXTURE_DATA_TEST_CASE(Run, NEDeconvolutionLayerQuantizedPerChannelFixture5x1, framework::DatasetMode::ALL, combine(combine(combine(combine(combine(combine(data5x1, + framework::dataset::make("DataType", DataType::QASYMM8)), + data_layouts_dataset), + input_qinfo_per_channel_dataset), + output_qinfo_per_channel_dataset), + add_bias_dataset), + framework::dataset::make("WeightsDataType", { DataType::QSYMM8_PER_CHANNEL }))) +{ + // Validate output + validate(Accessor(_target), _reference, tolerance_quantized, tolerance_num_quant); +} +FIXTURE_DATA_TEST_CASE(RunSigned, NEDeconvolutionLayerQuantizedPerChannelFixture5x1, framework::DatasetMode::ALL, combine(combine(combine(combine(combine(combine(data5x1, + framework::dataset::make("DataType", DataType::QASYMM8_SIGNED)), + data_layouts_dataset), + input_signed_qinfo_per_channel_dataset), + output_signed_qinfo_per_channel_dataset), + add_bias_dataset), + framework::dataset::make("WeightsDataType", { DataType::QSYMM8_PER_CHANNEL }))) +{ + // Validate output + validate(Accessor(_target), _reference, tolerance_quantized, tolerance_num_quant); +} +TEST_SUITE_END() // W5x1 + TEST_SUITE_END() // QSYMM8_PER_CHANNEL TEST_SUITE_END() // Quantized diff --git a/tests/validation/NEON/DepthwiseConvolutionLayerNative.cpp b/tests/validation/NEON/DepthwiseConvolutionLayerNative.cpp index 89c7964bead3fa0405529bf9b85069d8e442f4d3..221fc5d249d56005ae712404524beb13236af9e8 100644 --- a/tests/validation/NEON/DepthwiseConvolutionLayerNative.cpp +++ b/tests/validation/NEON/DepthwiseConvolutionLayerNative.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022 Arm Limited. + * Copyright (c) 2019-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -21,6 +21,7 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ +#include "arm_compute/core/utils/StringUtils.h" #include "src/cpu/kernels/CpuDepthwiseConv2dNativeKernel.h" #include "tests/NEON/Accessor.h" #include "tests/NEON/Helper.h" diff --git a/tests/validation/NEON/DirectConvolutionLayer.cpp b/tests/validation/NEON/DirectConvolutionLayer.cpp index 73c1a5c3e3e03f334961dfd54e7e9b6c85f41efb..0779c9d388abd630b076efd245ce565cfdafc895 100644 --- a/tests/validation/NEON/DirectConvolutionLayer.cpp +++ b/tests/validation/NEON/DirectConvolutionLayer.cpp @@ -23,6 +23,7 @@ */ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/Types.h" +#include "arm_compute/core/utils/StringUtils.h" #include "arm_compute/runtime/NEON/functions/NEDirectConvolutionLayer.h" #include "arm_compute/runtime/Tensor.h" #include "arm_compute/runtime/TensorAllocator.h" diff --git a/tests/validation/NEON/ElementwiseKernelSelection.cpp b/tests/validation/NEON/ElementwiseKernelSelection.cpp index 78adc747fd28f571a5d82c791667046ef5a4ea79..7990a51936cb8d046eabc62bd5440ec91646c567 100644 --- a/tests/validation/NEON/ElementwiseKernelSelection.cpp +++ b/tests/validation/NEON/ElementwiseKernelSelection.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,7 @@ * SOFTWARE. */ #include "arm_compute/core/Types.h" +#include "arm_compute/core/utils/StringUtils.h" #include "src/common/cpuinfo/CpuIsaInfo.h" #include "src/cpu/kernels/CpuElementwiseKernel.h" #include "src/cpu/kernels/CpuElementwiseUnaryKernel.h" diff --git a/tests/validation/NEON/Floor.cpp b/tests/validation/NEON/Floor.cpp index d3bd3e0b1cd8053fb214de529f4fbf5a66d4b966..3cd1033ef97dbb1647f5ffbc93ed067327304bf7 100644 --- a/tests/validation/NEON/Floor.cpp +++ b/tests/validation/NEON/Floor.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2020, 2022 Arm Limited. + * Copyright (c) 2017-2020, 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,7 @@ * SOFTWARE. */ #include "arm_compute/core/Types.h" +#include "arm_compute/core/utils/StringUtils.h" #include "arm_compute/runtime/NEON/functions/NEFloor.h" #include "arm_compute/runtime/Tensor.h" #include "arm_compute/runtime/TensorAllocator.h" diff --git a/tests/validation/NEON/GEMM.cpp b/tests/validation/NEON/GEMM.cpp index 36ccc1a4f7aa4a6a6790222ef3a142f2a294d2d9..f956cdfeda85a3f0e8ceb1dbda59011b864a499c 100644 --- a/tests/validation/NEON/GEMM.cpp +++ b/tests/validation/NEON/GEMM.cpp @@ -22,6 +22,7 @@ * SOFTWARE. */ #include "arm_compute/core/Types.h" +#include "arm_compute/core/utils/StringUtils.h" #include "arm_compute/runtime/NEON/functions/NEGEMM.h" #include "arm_compute/runtime/Tensor.h" #include "arm_compute/runtime/TensorAllocator.h" diff --git a/tests/validation/NEON/MaxUnpoolingLayer.cpp b/tests/validation/NEON/MaxUnpoolingLayer.cpp index 8904993afef3d71c5eb4ac2cd10d4a438cea5718..0eb021fe716541a85e3b557ef38d7c143354d1c5 100644 --- a/tests/validation/NEON/MaxUnpoolingLayer.cpp +++ b/tests/validation/NEON/MaxUnpoolingLayer.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022 Arm Limited. + * Copyright (c) 2020-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,7 @@ * SOFTWARE. */ #include "arm_compute/core/Types.h" +#include "arm_compute/core/utils/StringUtils.h" #include "arm_compute/runtime/NEON/functions/NEMaxUnpoolingLayer.h" #include "arm_compute/runtime/NEON/functions/NEPoolingLayer.h" #include "arm_compute/runtime/Tensor.h" diff --git a/tests/validation/NEON/ReduceMean.cpp b/tests/validation/NEON/ReduceMean.cpp index b4a3f0d39930de35704075bbb61974efb958eb35..49a38cd49ca73a7865cb96a3eba649c13fc9f56a 100644 --- a/tests/validation/NEON/ReduceMean.cpp +++ b/tests/validation/NEON/ReduceMean.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -49,7 +49,7 @@ constexpr AbsoluteTolerance tolerance_f16(0.03f); /**< Tolerance value fo constexpr AbsoluteTolerance tolerance_u8(1); /**< Tolerance value for comparing reference's output against implementation's output for unsigned 8-bit asymmetric quantized type */ constexpr AbsoluteTolerance tolerance_s8(2); /**< Tolerance value for comparing reference's output against implementation's output for signed 8-bit asymmetric quantized type */ -const auto axis_keep = combine(framework::dataset::make("Axis", { Coordinates(0), Coordinates(1, 0), Coordinates(1, 2), Coordinates(0, 2), Coordinates(1, 3), Coordinates(0, 1, 2, 3) }), +const auto axis_keep = combine(framework::dataset::make("Axis", { Coordinates(0), Coordinates(1, 0), Coordinates(1, 2), Coordinates(0, 2), Coordinates(1, 3), Coordinates(2, 3), Coordinates(0, 1, 2, 3) }), framework::dataset::make("KeepDims", { true })); const auto axis_drop = combine(framework::dataset::make("Axis", { Coordinates(0), Coordinates(1), Coordinates(3) }), framework::dataset::make("KeepDims", { false })); } // namespace diff --git a/tests/validation/NEON/SoftmaxLayer.cpp b/tests/validation/NEON/SoftmaxLayer.cpp index 90843537439b0608ac2008a8cf1196224573899e..b372bdf3facf0b55e4fa3b94392850a8277bb658 100644 --- a/tests/validation/NEON/SoftmaxLayer.cpp +++ b/tests/validation/NEON/SoftmaxLayer.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2020, 2022 Arm Limited. + * Copyright (c) 2017-2020, 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,6 +22,7 @@ * SOFTWARE. */ #include "arm_compute/core/Types.h" +#include "arm_compute/core/utils/StringUtils.h" #include "arm_compute/runtime/NEON/functions/NESoftmaxLayer.h" #include "arm_compute/runtime/Tensor.h" #include "arm_compute/runtime/TensorAllocator.h" diff --git a/tests/validation/UNIT/CPPScheduler.cpp b/tests/validation/UNIT/CPPScheduler.cpp new file mode 100644 index 0000000000000000000000000000000000000000..52431653b56b67e15a376b550e9b5c0e0cc4d92b --- /dev/null +++ b/tests/validation/UNIT/CPPScheduler.cpp @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "arm_compute/runtime/CPP/CPPScheduler.h" + +#include "arm_compute/core/CPP/ICPPKernel.h" +#include "tests/framework/Asserts.h" +#include "tests/framework/Macros.h" + +#include + +using namespace arm_compute; +using namespace arm_compute::test; + +namespace +{ +class TestException: public std::exception +{ +public: + const char* what() const noexcept override + { + return "Expected test exception"; + } +}; + +class TestKernel: public ICPPKernel +{ +public: + TestKernel() + { + Window window; + window.set(0, Window::Dimension(0, 2)); + configure(window); + } + + const char* name() const override + { + return "TestKernel"; + } + + void run(const Window &, const ThreadInfo &) override + { + throw TestException(); + } + +}; +} + +TEST_SUITE(UNIT) +TEST_SUITE(CPPScheduler) + +#if !defined(BARE_METAL) +TEST_CASE(RethrowException, framework::DatasetMode::ALL) +{ + CPPScheduler scheduler; + CPPScheduler::Hints hints(0); + TestKernel kernel; + + scheduler.set_num_threads(2); + try + { + scheduler.schedule(&kernel, hints); + } + catch(const TestException&) + { + return; + } + ARM_COMPUTE_EXPECT_FAIL("Expected exception not caught", framework::LogLevel::ERRORS); +} +#endif // !defined(BARE_METAL) + +TEST_SUITE_END() +TEST_SUITE_END() diff --git a/tests/validation/dynamic_fusion/gpu/Integration.cpp b/tests/validation/dynamic_fusion/gpu/Integration.cpp index 6a283f8082d4dfd0b8714289e6a8cd80f331e046..89cca5cd669f9439879b1b2c7647af5aa456e0fc 100644 --- a/tests/validation/dynamic_fusion/gpu/Integration.cpp +++ b/tests/validation/dynamic_fusion/gpu/Integration.cpp @@ -23,24 +23,33 @@ */ #include "arm_compute/core/CL/CLKernelLibrary.h" +#include "arm_compute/core/QuantizationInfo.h" #include "arm_compute/core/TensorInfo.h" +#include "arm_compute/core/Types.h" #include "arm_compute/dynamic_fusion/runtime/gpu/cl/ClWorkloadRuntime.h" #include "arm_compute/dynamic_fusion/sketch/attributes/CastAttributes.h" #include "arm_compute/dynamic_fusion/sketch/attributes/Conv2dAttributes.h" +#include "arm_compute/dynamic_fusion/sketch/attributes/DepthwiseConv2dAttributes.h" #include "arm_compute/dynamic_fusion/sketch/gpu/GpuWorkloadSketch.h" #include "arm_compute/dynamic_fusion/sketch/gpu/operators/GpuAdd.h" #include "arm_compute/dynamic_fusion/sketch/gpu/operators/GpuCast.h" #include "arm_compute/dynamic_fusion/sketch/gpu/operators/GpuConv2d.h" +#include "arm_compute/dynamic_fusion/sketch/gpu/operators/GpuDepthwiseConv2d.h" +#include "arm_compute/dynamic_fusion/sketch/gpu/operators/GpuMul.h" #include "arm_compute/dynamic_fusion/sketch/gpu/operators/GpuOutput.h" +#include "arm_compute/dynamic_fusion/sketch/gpu/operators/GpuSigmoid.h" #include "tests/CL/CLAccessor.h" #include "tests/framework/Macros.h" #include "tests/validation/Validation.h" #include "tests/validation/dynamic_fusion/Utils.h" +#include "tests/validation/reference/ActivationLayer.h" #include "tests/validation/reference/ConvolutionLayer.h" #include "tests/validation/reference/DepthConvertLayer.h" +#include "tests/validation/reference/DepthwiseConvolutionLayer.h" #include "tests/validation/reference/ElementwiseOperations.h" #include "tests/validation/reference/Permute.h" +#include "tests/validation/reference/PixelWiseMultiplication.h" using namespace arm_compute::experimental::dynamic_fusion; using namespace arm_compute::test::validation::utils; @@ -54,6 +63,7 @@ namespace validation TEST_SUITE(CL) TEST_SUITE(INTEGRATION) TEST_SUITE(DYNAMIC_FUSION) +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF // Conv2d is not ported to ckw yet. COMPMID-6259 TEST_CASE(Conv2d, framework::DatasetMode::ALL) { /* Computation: @@ -69,17 +79,17 @@ TEST_CASE(Conv2d, framework::DatasetMode::ALL) // Create a new workload sketch auto cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - auto gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + auto context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; // Fuse conv2d Conv2dAttributes conv2d_attr{}; - TensorInfo input_info = sketch.create_tensor_info(t_input_shape, 1, data_type, data_layout); - TensorInfo weight_info = sketch.create_tensor_info(TensorInfo(t_weight_shape, 1, data_type, data_layout)); + TensorInfo input_info = context.create_tensor_info(t_input_shape, 1, data_type, data_layout); + TensorInfo weight_info = context.create_tensor_info(TensorInfo(t_weight_shape, 1, data_type, data_layout)); ITensorInfo *conv_out_info = GpuConv2d::create_op(sketch, &input_info, &weight_info, nullptr, conv2d_attr); - TensorInfo dst_info = sketch.create_tensor_info(); + TensorInfo dst_info = context.create_tensor_info(); GpuOutput::create_op(sketch, conv_out_info, &dst_info); // Configure runtime @@ -143,6 +153,7 @@ TEST_CASE(Conv2d, framework::DatasetMode::ALL) RelativeTolerance tolerance_f32(0.001f); /**< Tolerance value for comparing reference's output against implementation's output for floating point data types */ validate(CLAccessor(t_dst), ref_t_dst_nchw, tolerance_f32); } +#endif // ACL_INTERNAL_TEST_CKW_IN_DF TEST_CASE(Add_Output_Add_Output, framework::DatasetMode::ALL) { /* Computation: @@ -156,15 +167,15 @@ TEST_CASE(Add_Output_Add_Output, framework::DatasetMode::ALL) // Create a new workload sketch auto cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - auto gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + auto context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; - TensorInfo in_0_info = sketch.create_tensor_info(t_input_shape, 1, data_type); - TensorInfo in_1_info = sketch.create_tensor_info(t_input_shape, 1, data_type); - TensorInfo in_2_info = sketch.create_tensor_info(t_input_shape, 1, data_type); + TensorInfo in_0_info = context.create_tensor_info(t_input_shape, 1, data_type); + TensorInfo in_1_info = context.create_tensor_info(t_input_shape, 1, data_type); + TensorInfo in_2_info = context.create_tensor_info(t_input_shape, 1, data_type); - TensorInfo out_0_info = sketch.create_tensor_info(); - TensorInfo out_1_info = sketch.create_tensor_info(); + TensorInfo out_0_info = context.create_tensor_info(); + TensorInfo out_1_info = context.create_tensor_info(); ITensorInfo *ans_0_info = GpuAdd::create_op(sketch, &in_0_info, &in_1_info); GpuOutput::create_op(sketch, ans_0_info, &out_0_info); @@ -253,15 +264,15 @@ TEST_CASE(Add_Output_Add_Cast_Cast_Output, framework::DatasetMode::ALL) // Create a new workload sketch auto cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - auto gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + auto context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; - TensorInfo in_0_info = sketch.create_tensor_info(t_input_shape, 1, data_type); - TensorInfo in_1_info = sketch.create_tensor_info(t_input_shape, 1, data_type); - TensorInfo in_2_info = sketch.create_tensor_info(t_input_shape, 1, data_type); + TensorInfo in_0_info = context.create_tensor_info(t_input_shape, 1, data_type); + TensorInfo in_1_info = context.create_tensor_info(t_input_shape, 1, data_type); + TensorInfo in_2_info = context.create_tensor_info(t_input_shape, 1, data_type); - TensorInfo out_0_info = sketch.create_tensor_info(); - TensorInfo out_1_info = sketch.create_tensor_info(); + TensorInfo out_0_info = context.create_tensor_info(); + TensorInfo out_1_info = context.create_tensor_info(); CastAttributes cast_0_attr; cast_0_attr.data_type(DataType::S32).convert_policy(ConvertPolicy::SATURATE); @@ -348,6 +359,214 @@ TEST_CASE(Add_Output_Add_Cast_Cast_Output, framework::DatasetMode::ALL) validate(CLAccessor(t_out_0), ref_t_out_0, tolerance_add_f32); validate(CLAccessor(t_out_1), ref_t_out_1, tolerance_cast_f32); } + +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF // Conv2d is not ported to ckw yet. COMPMID-6259 +TEST_CASE(Conv2d_Sigmoid_DepthwiseConv2d_Mul, framework::DatasetMode::ALL) +{ + // (tensor0) + // | + // ======|============================================== Sketch 0 + // | (tensor1) +---- (tensor2) + // | | | | + // +-- input -- weights -- biases --+ | + // | | | + // | Conv2d | | + // | | | + // +----------- output -------------+ | + // | | + // +-- input ---+ | + // | | | + // | Sigmoid | | + // | | | + // +-- output --+ | + // | | + // +-- input ---+ | + // | | | + // | Output | | + // | | | + // +-- output --+ | + // | | + // (tensor5) | + // | | + // +--------+ | + // ======|=============================|================ Sketch 1 + // | (tensor3) (tensor4) | + // | | | | + // +-- input -- weights -- biases --+ | + // | | | + // | DepthwiseConv2d | | + // | | | + // +----------- output -------------+ | + // | | + // +--+ +----------------+ + // | | + // +-- lhs -- rhs --+ + // | | + // | Multiply | + // | | + // +---- output ----+ + // | + // +-- input ---+ + // | | + // | Output | + // | | + // +-- output --+ + // | + // (tensor6) + + TensorShape conv2d_src_shape(10, 20, 30); + TensorShape conv2d_wei_shape(10, 3, 3, 5); + TensorShape conv2d_bia_shape(5); + TensorShape conv2d_dst_shape(5, 18, 28); + TensorShape dwc_wei_shape(5, 3, 3); + TensorShape dwc_bia_shape(5); + TensorShape dwc_dst_shape(5, 16, 26); + + // Initialize the context. + CLScheduler::get().default_reinit(); + + auto cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); + GpuWorkloadContext context(&cl_compile_ctx); + + auto tensor0_info = context.create_tensor_info(conv2d_src_shape, 1, DataType::F32, DataLayout::NHWC); + + // Create the first sketch: conv2d + cast + output. + GpuWorkloadSketch sketch0(&context); + + Conv2dAttributes conv2d_attr; + auto tensor1_info = context.create_tensor_info(conv2d_wei_shape, 1, DataType::F32, DataLayout::NHWC); + auto tensor2_info = context.create_tensor_info(conv2d_bia_shape, 1, DataType::F32, DataLayout::NHWC); + ARM_COMPUTE_EXPECT(GpuConv2d::validate_op(sketch0, &tensor0_info, &tensor1_info, &tensor2_info, conv2d_attr), framework::LogLevel::ERRORS); + auto ans_info = GpuConv2d::create_op(sketch0, &tensor0_info, &tensor1_info, &tensor2_info, conv2d_attr); + + ARM_COMPUTE_EXPECT(GpuSigmoid::validate_op(sketch0, ans_info), framework::LogLevel::ERRORS); + ans_info = GpuSigmoid::create_op(sketch0, ans_info); + + DepthwiseConv2dAttributes dwc_attr; + auto tensor3_info = context.create_tensor_info(dwc_wei_shape, 1, DataType::F32, DataLayout::NHWC); + auto tensor4_info = context.create_tensor_info(dwc_bia_shape, 1, DataType::F32, DataLayout::NHWC); + ARM_COMPUTE_EXPECT(!GpuDepthwiseConv2d::validate_op(sketch0, ans_info, &tensor3_info, &tensor4_info, dwc_attr), framework::LogLevel::ERRORS); + + auto tensor5_info = context.create_tensor_info(); + ARM_COMPUTE_EXPECT(GpuOutput::validate_op(sketch0, ans_info, &tensor5_info), framework::LogLevel::ERRORS); + GpuOutput::create_op(sketch0, ans_info, &tensor5_info); + + // Create the first workload runtime. + ClWorkloadRuntime runtime0; + runtime0.configure(sketch0); + + // Create the second sketch: dwc + sigmoid + output. + GpuWorkloadSketch sketch1(&context); + + ARM_COMPUTE_EXPECT(GpuDepthwiseConv2d::validate_op(sketch1, &tensor5_info, &tensor3_info, &tensor4_info, dwc_attr), framework::LogLevel::ERRORS); + ans_info = GpuDepthwiseConv2d::create_op(sketch1, &tensor5_info, &tensor3_info, &tensor4_info, dwc_attr); + + ARM_COMPUTE_EXPECT(GpuMul::validate_op(sketch1, ans_info, &tensor2_info), framework::LogLevel::ERRORS); + ans_info = GpuMul::create_op(sketch1, ans_info, &tensor2_info); + + auto tensor6_info = context.create_tensor_info(); + ARM_COMPUTE_EXPECT(GpuOutput::validate_op(sketch1, ans_info, &tensor6_info), framework::LogLevel::ERRORS); + GpuOutput::create_op(sketch1, ans_info, &tensor6_info); + + // Create the second workload runtime. + ClWorkloadRuntime runtime1; + runtime1.configure(sketch1); + + // Create the user tensors. + CLTensor tensor0; + CLTensor tensor1; + CLTensor tensor2; + CLTensor tensor3; + CLTensor tensor4; + CLTensor tensor5; + CLTensor tensor6; + + tensor0.allocator()->init(tensor0_info); + tensor1.allocator()->init(tensor1_info); + tensor2.allocator()->init(tensor2_info); + tensor3.allocator()->init(tensor3_info); + tensor4.allocator()->init(tensor4_info); + tensor5.allocator()->init(tensor5_info); + tensor6.allocator()->init(tensor6_info); + + tensor0.allocator()->allocate(); + tensor1.allocator()->allocate(); + tensor2.allocator()->allocate(); + tensor3.allocator()->allocate(); + tensor4.allocator()->allocate(); + tensor5.allocator()->allocate(); + tensor6.allocator()->allocate(); + + // Allocate the auxiliary tensors. + for(auto &data : runtime0.get_auxiliary_tensors()) + { + auto tensor = std::get<0>(data); + auto &tensor_info = std::get<1>(data); + auto mem_req = std::get<2>(data); + + tensor->allocator()->init(tensor_info, mem_req.alignment); + tensor->allocator()->allocate(); + } + + for(auto &data : runtime1.get_auxiliary_tensors()) + { + auto tensor = std::get<0>(data); + auto &tensor_info = std::get<1>(data); + auto mem_req = std::get<2>(data); + + tensor->allocator()->init(tensor_info, mem_req.alignment); + tensor->allocator()->allocate(); + } + + // Fill the input tensors with random data. + fill(CLAccessor(tensor0), 0, library.get()); + fill(CLAccessor(tensor1), 1, library.get()); + fill(CLAccessor(tensor2), 2, library.get()); + fill(CLAccessor(tensor3), 3, library.get()); + fill(CLAccessor(tensor4), 4, library.get()); + + // Run each runtime. + runtime0.run({ &tensor0, &tensor1, &tensor2, &tensor5 }); + runtime1.run({ &tensor5, &tensor3, &tensor4, &tensor2, &tensor6 }); + + // Compute the reference result. + SimpleTensor ref_conv2d_src(conv2d_src_shape, DataType::F32, 1, QuantizationInfo(), DataLayout::NHWC); + SimpleTensor ref_conv2d_wei(conv2d_wei_shape, DataType::F32, 1, QuantizationInfo(), DataLayout::NHWC); + SimpleTensor ref_conv2d_bia(conv2d_bia_shape, DataType::F32, 1, QuantizationInfo(), DataLayout::NHWC); + SimpleTensor ref_dwc_wei(dwc_wei_shape, DataType::F32, 1, QuantizationInfo(), DataLayout::NHWC); + SimpleTensor ref_dwc_bia(dwc_bia_shape, DataType::F32, 1, QuantizationInfo(), DataLayout::NHWC); + + fill(ref_conv2d_src, 0, library.get()); + fill(ref_conv2d_wei, 1, library.get()); + fill(ref_conv2d_bia, 2, library.get()); + fill(ref_dwc_wei, 3, library.get()); + fill(ref_dwc_bia, 4, library.get()); + + PermutationVector nhwc_to_nchw(1, 2, 0); + + auto conv2d_dst_shape_nchw = conv2d_dst_shape; + permute(conv2d_dst_shape_nchw, nhwc_to_nchw); + const auto ref_conv2d_src_nchw = reference::permute(ref_conv2d_src, nhwc_to_nchw); + const auto ref_conv2d_wei_nchw = reference::permute(ref_conv2d_wei, nhwc_to_nchw); + const auto ref_conv2d_bia_nchw = reference::permute(ref_conv2d_bia, nhwc_to_nchw); + const auto ref_conv2d_dst_nchw = reference::convolution_layer(ref_conv2d_src_nchw, ref_conv2d_wei_nchw, ref_conv2d_bia_nchw, conv2d_dst_shape_nchw, PadStrideInfo()); + + const auto ref_sigmoid_dst_nchw = reference::activation_layer(ref_conv2d_dst_nchw, ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::LOGISTIC)); + + auto dwc_dst_shape_nchw = dwc_dst_shape; + permute(dwc_dst_shape_nchw, nhwc_to_nchw); + const auto ref_dwc_wei_nchw = reference::permute(ref_dwc_wei, nhwc_to_nchw); + const auto ref_dwc_bia_nchw = reference::permute(ref_dwc_bia, nhwc_to_nchw); + const auto ref_dwc_dst_nchw = reference::depthwise_convolution(ref_sigmoid_dst_nchw, ref_dwc_wei_nchw, ref_dwc_bia_nchw, dwc_dst_shape_nchw, PadStrideInfo(), 1); + + const auto ref_mul_dst_nchw = reference::pixel_wise_multiplication(ref_dwc_dst_nchw, ref_conv2d_bia_nchw, 1.0, ConvertPolicy::SATURATE, RoundingPolicy::TO_NEAREST_UP, + DataType::F32); + + constexpr RelativeTolerance tolerance(0.001f); + validate(CLAccessor(tensor6), ref_mul_dst_nchw, tolerance); +} +#endif // ACL_INTERNAL_TEST_CKW_IN_DF + TEST_SUITE(Invalid_Fusion_Should_Fail) TEST_CASE(Multiple_Complex_Ops_0, framework::DatasetMode::ALL) { @@ -368,12 +587,12 @@ TEST_CASE(Multiple_Complex_Ops_0, framework::DatasetMode::ALL) // Create a new workload sketch auto cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - auto gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + auto context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; // Create tensor infos - TensorInfo input_info = sketch.create_tensor_info(t_input_shape, 1, data_type, data_layout); - TensorInfo weight_info = sketch.create_tensor_info(TensorInfo(t_weight_shape, 1, data_type, data_layout)); + TensorInfo input_info = context.create_tensor_info(t_input_shape, 1, data_type, data_layout); + TensorInfo weight_info = context.create_tensor_info(TensorInfo(t_weight_shape, 1, data_type, data_layout)); ITensorInfo *dst_info; // Fuse conv2d into the workload @@ -386,7 +605,7 @@ TEST_CASE(Multiple_Complex_Ops_0, framework::DatasetMode::ALL) } // Create tensor infos - TensorInfo weight_info_2 = sketch.create_tensor_info(t_weight_info); + TensorInfo weight_info_2 = context.create_tensor_info(t_weight_info); // Fuse conv2d into the workload { diff --git a/tests/validation/dynamic_fusion/gpu/cl/Add.cpp b/tests/validation/dynamic_fusion/gpu/cl/Add.cpp index 0034b0f07f7d768a9490867d2d5818bbbcb8e468..d9a3d9533ce286da2434c594f01bf7a899141891 100644 --- a/tests/validation/dynamic_fusion/gpu/cl/Add.cpp +++ b/tests/validation/dynamic_fusion/gpu/cl/Add.cpp @@ -87,12 +87,12 @@ DATA_TEST_CASE(Validate, framework::DatasetMode::ALL, zip(zip( { // Create a new workload sketch auto cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - auto gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + auto context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; // Validate Elementwise Add - auto lhs_info = sketch.create_tensor_info(input1_info); - auto rhs_info = sketch.create_tensor_info(input2_info); + auto lhs_info = context.create_tensor_info(input1_info); + auto rhs_info = context.create_tensor_info(input2_info); bool res = bool(GpuAdd::validate_op(sketch, &lhs_info, &rhs_info)); ARM_COMPUTE_EXPECT(res == expected, framework::LogLevel::ERRORS); diff --git a/tests/validation/dynamic_fusion/gpu/cl/Clamp.cpp b/tests/validation/dynamic_fusion/gpu/cl/Clamp.cpp index 177c02c2c73d7b8cdb08b370d240558f599fb7f4..285c0d6608adcc838d4f6e8cec0fb1e720e8a857 100644 --- a/tests/validation/dynamic_fusion/gpu/cl/Clamp.cpp +++ b/tests/validation/dynamic_fusion/gpu/cl/Clamp.cpp @@ -69,11 +69,11 @@ DATA_TEST_CASE(Validate, framework::DatasetMode::ALL, zip(zip(zip( { // Create a new workload sketch CLCompileContext cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - GpuWorkloadContext gpu_ctx{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + GpuWorkloadContext context{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; // Fuse Clamp - const TensorInfo src_info = sketch.create_tensor_info(input_info); + const TensorInfo src_info = context.create_tensor_info(input_info); ClampAttributes attributes {}; attributes.min_val(min_val) @@ -111,7 +111,8 @@ FIXTURE_DATA_TEST_CASE(RunSmall5dOneOp, framework::dataset::make("DataType", DataType::F16))) { // Validate output - validate(CLAccessor(_target), _reference, tolerance); + ARM_COMPUTE_TEST_INFO("Currently 5D+ tensors are unsupported for this operation."); + framework::ARM_COMPUTE_PRINT_INFO(); } FIXTURE_DATA_TEST_CASE(RunSmallTwoOps, @@ -150,7 +151,8 @@ FIXTURE_DATA_TEST_CASE(RunSmall5dOneOp, framework::dataset::make("DataType", DataType::F32))) { // Validate output - validate(CLAccessor(_target), _reference, tolerance); + ARM_COMPUTE_TEST_INFO("Currently 5D+ tensors are unsupported for this operation."); + framework::ARM_COMPUTE_PRINT_INFO(); } FIXTURE_DATA_TEST_CASE(RunSmallTwoOps, diff --git a/tests/validation/dynamic_fusion/gpu/cl/DepthwiseConv2d.cpp b/tests/validation/dynamic_fusion/gpu/cl/DepthwiseConv2d.cpp index b6331d70c801c1729136dd0fb6efbbe17fa55c4a..71b0114225197873233734d656d0f1314a2f46ae 100644 --- a/tests/validation/dynamic_fusion/gpu/cl/DepthwiseConv2d.cpp +++ b/tests/validation/dynamic_fusion/gpu/cl/DepthwiseConv2d.cpp @@ -22,6 +22,7 @@ * SOFTWARE. */ +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF // Do not include this test if ACL_INTERNAL_TEST_CKW_IN_DF and the op has not been ported to ckw #include "arm_compute/dynamic_fusion/sketch/gpu/operators/GpuDepthwiseConv2d.h" #include "tests/CL/CLAccessor.h" @@ -242,12 +243,12 @@ DATA_TEST_CASE(Validate, framework::DatasetMode::ALL, zip(zip(zip(zip(zip(zip(zi input_info, weights_info, biases_info, padding, stride, depth_multiplier, dilation, expected) { CLCompileContext cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - GpuWorkloadContext gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + GpuWorkloadContext context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; - const TensorInfo sketch_input_info = sketch.create_tensor_info(input_info); - const TensorInfo sketch_weights_info = sketch.create_tensor_info(weights_info); - const TensorInfo sketch_biases_info = sketch.create_tensor_info(biases_info); + const TensorInfo sketch_input_info = context.create_tensor_info(input_info); + const TensorInfo sketch_weights_info = context.create_tensor_info(weights_info); + const TensorInfo sketch_biases_info = context.create_tensor_info(biases_info); DepthwiseConv2dAttributes attributes {}; attributes.pad(padding) @@ -432,3 +433,5 @@ TEST_SUITE_END() // CL } // namespace validation } // namespace test } // namespace arm_compute + +#endif // ACL_INTERNAL_TEST_CKW_IN_DF diff --git a/tests/validation/dynamic_fusion/gpu/cl/DirectConv2d.cpp b/tests/validation/dynamic_fusion/gpu/cl/DirectConv2d.cpp index cccad182ca3d252a528de4c1ad73a1939b2a9ff4..f27a1796c98ecf26931ecbe592494bb7bc913680 100644 --- a/tests/validation/dynamic_fusion/gpu/cl/DirectConv2d.cpp +++ b/tests/validation/dynamic_fusion/gpu/cl/DirectConv2d.cpp @@ -157,12 +157,12 @@ DATA_TEST_CASE(Validate, framework::DatasetMode::ALL, zip(zip(zip(zip( input_info, weights_info, biases_info, conv2d_attrs, expected) { auto cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - auto gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + auto context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; - const TensorInfo sketch_input_info = sketch.create_tensor_info(input_info); - const TensorInfo sketch_weights_info = sketch.create_tensor_info(weights_info); - const TensorInfo sketch_biases_info = sketch.create_tensor_info(biases_info); + const TensorInfo sketch_input_info = context.create_tensor_info(input_info); + const TensorInfo sketch_weights_info = context.create_tensor_info(weights_info); + const TensorInfo sketch_biases_info = context.create_tensor_info(biases_info); bool is_valid = bool(GpuConv2d::validate_op(sketch, &sketch_input_info, &sketch_weights_info, &sketch_biases_info, conv2d_attrs)); ARM_COMPUTE_EXPECT(is_valid == expected, framework::LogLevel::ERRORS); } diff --git a/tests/validation/dynamic_fusion/gpu/cl/Mul.cpp b/tests/validation/dynamic_fusion/gpu/cl/Mul.cpp index a9e8f9c15fb8f239839e7cf21a9bd62df82546a2..2da2b9eabd4bfd5e61aff2d33bcf8bd676a0d246 100644 --- a/tests/validation/dynamic_fusion/gpu/cl/Mul.cpp +++ b/tests/validation/dynamic_fusion/gpu/cl/Mul.cpp @@ -102,12 +102,12 @@ DATA_TEST_CASE(Validate, framework::DatasetMode::ALL, zip(zip( { // Create a new workload sketch auto cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - auto gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + auto context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; // Validate Elementwise Mul - auto lhs_info = sketch.create_tensor_info(input1_info); - auto rhs_info = sketch.create_tensor_info(input2_info); + auto lhs_info = context.create_tensor_info(input1_info); + auto rhs_info = context.create_tensor_info(input2_info); bool res = bool(GpuMul::validate_op(sketch, &lhs_info, &rhs_info)); ARM_COMPUTE_EXPECT(res == expected, framework::LogLevel::ERRORS); diff --git a/tests/validation/dynamic_fusion/gpu/cl/Pool2d.cpp b/tests/validation/dynamic_fusion/gpu/cl/Pool2d.cpp index a7772aef4d9fdacc1728ef6c5f27027164b9dda9..7f5efd662ab08dc77d4b8912f6d5067af8a3f3d0 100644 --- a/tests/validation/dynamic_fusion/gpu/cl/Pool2d.cpp +++ b/tests/validation/dynamic_fusion/gpu/cl/Pool2d.cpp @@ -22,6 +22,7 @@ * SOFTWARE. */ +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF // Do not include this test if ACL_INTERNAL_TEST_CKW_IN_DF and the op has not been ported to ckw #include "arm_compute/dynamic_fusion/sketch/gpu/operators/GpuPool2d.h" #include "tests/CL/CLAccessor.h" @@ -101,15 +102,15 @@ DATA_TEST_CASE(Validate, framework::DatasetMode::ALL, zip(zip(zip( { // Create a new workload sketch auto cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - auto gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + auto context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; // Declare GpuPool2d settings const GpuPool2dSettings &settings = GpuPool2dSettings().mixed_precision(false); // Validate Pool2d Configuration - auto src_info = sketch.create_tensor_info(input_info); - auto dst_info = sketch.create_tensor_info(output_info); + auto src_info = context.create_tensor_info(input_info); + auto dst_info = context.create_tensor_info(output_info); bool res = bool(GpuPool2d::validate_op(sketch, &src_info, &dst_info, pool2d_attr, settings)); ARM_COMPUTE_EXPECT(res == expected, framework::LogLevel::ERRORS); } @@ -132,7 +133,7 @@ FIXTURE_DATA_TEST_CASE(RunLarge, DynamicFusionGpuPool2dFixture, framework validate(CLAccessor(_target), _reference, tolerance_f32); } FIXTURE_DATA_TEST_CASE(RunSpecial, DFSpecialGpuPool2dFixture, framework::DatasetMode::ALL, combine(datasets::PoolingLayerDatasetSpecialDynamicFusion(), - framework::dataset::make("DataType", DataType::F32))) + framework::dataset::make("DataType", DataType::F32))) { // Validate output validate(CLAccessor(_target), _reference, tolerance_f32); @@ -231,3 +232,5 @@ TEST_SUITE_END() // CL } } } + +#endif // ACL_INTERNAL_TEST_CKW_IN_DF diff --git a/tests/validation/dynamic_fusion/gpu/cl/Reshape.cpp b/tests/validation/dynamic_fusion/gpu/cl/Reshape.cpp index 6d88be448ec0d637ffaf987ba7ff5ea6501548fa..4d038b2780dfc4736c2e646c44aeb8e2d23b9d14 100644 --- a/tests/validation/dynamic_fusion/gpu/cl/Reshape.cpp +++ b/tests/validation/dynamic_fusion/gpu/cl/Reshape.cpp @@ -21,6 +21,7 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF // Do not include this test if ACL_INTERNAL_TEST_CKW_IN_DF and the op has not been ported to ckw #include "tests/CL/CLAccessor.h" #include "tests/datasets/ReshapeLayerDataset.h" #include "tests/framework/Macros.h" @@ -53,13 +54,13 @@ input_info, output_shape, expected) { // Create a new workload sketch auto cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - auto gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + auto context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; // Create sketch tensors TensorShape input_shape = input_info.tensor_shape(); ARM_COMPUTE_UNUSED(input_shape); - TensorInfo src_info = sketch.create_tensor_info(input_info); + TensorInfo src_info = context.create_tensor_info(input_info); ReshapeAttributes attributes; attributes.shape(output_shape); @@ -121,3 +122,5 @@ TEST_SUITE_END() // CL } // namespace validation } // namespace test } // namespace arm_compute + +#endif // ACL_INTERNAL_TEST_CKW_IN_DF diff --git a/tests/validation/dynamic_fusion/gpu/cl/Resize.cpp b/tests/validation/dynamic_fusion/gpu/cl/Resize.cpp index 696be54c92b41e9f9695b1cf229b16188ef3495d..9ca1c5f0da8206f12190b9c3d4ebe983ae89e60f 100644 --- a/tests/validation/dynamic_fusion/gpu/cl/Resize.cpp +++ b/tests/validation/dynamic_fusion/gpu/cl/Resize.cpp @@ -22,6 +22,7 @@ * SOFTWARE. */ +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF // Do not include this test if ACL_INTERNAL_TEST_CKW_IN_DF and the op has not been ported to ckw #include "arm_compute/dynamic_fusion/sketch/gpu/operators/GpuResize.h" #include "tests/CL/CLAccessor.h" @@ -95,10 +96,10 @@ TEST_CASE(NullPtr, framework::DatasetMode::ALL) const TensorInfo output_info = TensorInfo{ default_output_shape, 1, default_data_type, default_data_layout }; CLCompileContext cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - GpuWorkloadContext gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + GpuWorkloadContext context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; - const TensorInfo sketch_input_info = sketch.create_tensor_info(input_info); + const TensorInfo sketch_input_info = context.create_tensor_info(input_info); // nullptr is given as input Status status = GpuResize::validate_op(sketch, nullptr, ResizeAttributes()); @@ -135,10 +136,10 @@ TEST_CASE(SupportDataType, framework::DatasetMode::ALL) const TensorInfo input_info = TensorInfo{ default_input_shape, 1, kv.first, default_data_layout }; CLCompileContext cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - GpuWorkloadContext gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + GpuWorkloadContext context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; - const TensorInfo sketch_input_info = sketch.create_tensor_info(input_info); + const TensorInfo sketch_input_info = context.create_tensor_info(input_info); ResizeAttributes attributes; attributes.output_width(default_output_shape[0]); // shape is not important unless it's empty @@ -157,10 +158,10 @@ TEST_CASE(MismatchingDataType, framework::DatasetMode::ALL) const TensorInfo output_info = TensorInfo{ default_output_shape, 1, non_default_data_type, default_data_layout }; CLCompileContext cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - GpuWorkloadContext gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + GpuWorkloadContext context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; - const TensorInfo sketch_input_info = sketch.create_tensor_info(input_info); + const TensorInfo sketch_input_info = context.create_tensor_info(input_info); Status status = GpuResize::validate_op(sketch, &sketch_input_info, ResizeAttributes()); ARM_COMPUTE_EXPECT(bool(status) == false, framework::LogLevel::ERRORS); @@ -177,10 +178,10 @@ TEST_CASE(AlignedCornerNotSupported, framework::DatasetMode::ALL) const TensorInfo output_info = TensorInfo{ default_output_shape, 1, default_data_type, default_data_layout }; CLCompileContext cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - GpuWorkloadContext gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + GpuWorkloadContext context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; - const TensorInfo sketch_input_info = sketch.create_tensor_info(input_info); + const TensorInfo sketch_input_info = context.create_tensor_info(input_info); ResizeAttributes attributes{}; attributes.interpolation_policy(interpolation_policy) @@ -198,10 +199,10 @@ TEST_CASE(UnsupportedInterpolationPolicy, framework::DatasetMode::ALL) constexpr auto interpolation_policy = InterpolationPolicy::AREA; CLCompileContext cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - GpuWorkloadContext gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + GpuWorkloadContext context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; - const TensorInfo sketch_input_info = sketch.create_tensor_info(input_info); + const TensorInfo sketch_input_info = context.create_tensor_info(input_info); ResizeAttributes attributes{}; attributes.interpolation_policy(interpolation_policy); @@ -217,10 +218,10 @@ TEST_CASE(UnsupportedLayout, framework::DatasetMode::ALL) constexpr auto interpolation_policy = InterpolationPolicy::BILINEAR; CLCompileContext cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - GpuWorkloadContext gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + GpuWorkloadContext context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; - const TensorInfo sketch_input_info = sketch.create_tensor_info(input_info); + const TensorInfo sketch_input_info = context.create_tensor_info(input_info); ResizeAttributes attributes{}; attributes.interpolation_policy(interpolation_policy); @@ -517,3 +518,5 @@ TEST_SUITE_END() // CL } // namespace validation } // namespace test } // namespace arm_compute + +#endif // ACL_INTERNAL_TEST_CKW_IN_DF diff --git a/tests/validation/dynamic_fusion/gpu/cl/Sigmoid.cpp b/tests/validation/dynamic_fusion/gpu/cl/Sigmoid.cpp index aace23eff49d4516e9d413239109a1700a0fdd0a..e99551117103a8a3af3017aef0f79656b42a6c0c 100644 --- a/tests/validation/dynamic_fusion/gpu/cl/Sigmoid.cpp +++ b/tests/validation/dynamic_fusion/gpu/cl/Sigmoid.cpp @@ -61,11 +61,11 @@ DATA_TEST_CASE(Validate, framework::DatasetMode::ALL, zip( { // Create a new workload sketch CLCompileContext cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - GpuWorkloadContext gpu_ctx{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + GpuWorkloadContext context{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; // Fuse sigmoid - const TensorInfo src_info = sketch.create_tensor_info(input_info); + const TensorInfo src_info = context.create_tensor_info(input_info); const bool res = static_cast(GpuSigmoid::validate_op(sketch, &src_info)); ARM_COMPUTE_EXPECT(res == expected, framework::LogLevel::ERRORS); @@ -97,7 +97,8 @@ FIXTURE_DATA_TEST_CASE(RunSmall5dOneOp, framework::dataset::make("DataType", DataType::F16))) { // Validate output - validate(CLAccessor(_target), _reference, tolerance_f16); + ARM_COMPUTE_TEST_INFO("Currently 5D+ tensors are unsupported for this operation."); + framework::ARM_COMPUTE_PRINT_INFO(); } FIXTURE_DATA_TEST_CASE(RunSmallTwoOps, @@ -133,7 +134,8 @@ FIXTURE_DATA_TEST_CASE(RunSmall5dOneOp, framework::dataset::make("DataType", DataType::F32))) { // Validate output - validate(CLAccessor(_target), _reference, tolerance_f32); + ARM_COMPUTE_TEST_INFO("Currently 5D+ tensors are unsupported for this operation."); + framework::ARM_COMPUTE_PRINT_INFO(); } FIXTURE_DATA_TEST_CASE(RunSmallTwoOps, diff --git a/tests/validation/dynamic_fusion/gpu/cl/Softmax.cpp b/tests/validation/dynamic_fusion/gpu/cl/Softmax.cpp index d09454e05be89be4feb68a3a66d829aebc45205a..340f5dc2a3e1df3fbdb3397ea1d34498500b3c50 100644 --- a/tests/validation/dynamic_fusion/gpu/cl/Softmax.cpp +++ b/tests/validation/dynamic_fusion/gpu/cl/Softmax.cpp @@ -21,6 +21,7 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ +#ifndef ACL_INTERNAL_TEST_CKW_IN_DF // Do not include this test if ACL_INTERNAL_TEST_CKW_IN_DF and the op has not been ported to ckw #include "arm_compute/core/Types.h" #include "arm_compute/dynamic_fusion/sketch/gpu/operators/GpuSoftmax.h" @@ -104,13 +105,13 @@ DATA_TEST_CASE(Validate, framework::DatasetMode::ALL, zip(zip(zip(zip( { // Create a new workload sketch CLCompileContext cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - GpuWorkloadContext gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + GpuWorkloadContext context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; SoftmaxAttributes softmax_attr{}; softmax_attr.axis(axis).beta(beta).is_log_softmax(false); - TensorInfo src_info = sketch.create_tensor_info(input_info); - TensorInfo dst_info = sketch.create_tensor_info(output_info); + TensorInfo src_info = context.create_tensor_info(input_info); + TensorInfo dst_info = context.create_tensor_info(output_info); const bool res = static_cast(GpuSoftmax::validate_op(sketch, &src_info, &dst_info, softmax_attr)); ARM_COMPUTE_EXPECT(res == expected, framework::LogLevel::ERRORS); } @@ -196,3 +197,5 @@ TEST_SUITE_END() // CL } // namespace validation } // namespace test } // namespace arm_compute + +#endif // ACL_INTERNAL_TEST_CKW_IN_DF diff --git a/tests/validation/dynamic_fusion/gpu/cl/Sub.cpp b/tests/validation/dynamic_fusion/gpu/cl/Sub.cpp index 977e0110da35f00eba336e1c25bf9575e2c4d468..0bb05c296164095edd12424979b9fed95cbb3e2c 100644 --- a/tests/validation/dynamic_fusion/gpu/cl/Sub.cpp +++ b/tests/validation/dynamic_fusion/gpu/cl/Sub.cpp @@ -89,12 +89,12 @@ DATA_TEST_CASE(Validate, framework::DatasetMode::ALL, zip(zip( { // Create a new workload sketch auto cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - auto gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + auto context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; // Validate Elementwise Sub - auto lhs_info = sketch.create_tensor_info(input1_info); - auto rhs_info = sketch.create_tensor_info(input2_info); + auto lhs_info = context.create_tensor_info(input1_info); + auto rhs_info = context.create_tensor_info(input2_info); bool res = bool(GpuSub::validate_op(sketch, &lhs_info, &rhs_info)); ARM_COMPUTE_EXPECT(res == expected, framework::LogLevel::ERRORS); diff --git a/tests/validation/dynamic_fusion/gpu/cl/Tanh.cpp b/tests/validation/dynamic_fusion/gpu/cl/Tanh.cpp index 183cd079a3d16b084f52880108eb519c474676df..12f3677abf5e7176a75d4790a48f5202f462090b 100644 --- a/tests/validation/dynamic_fusion/gpu/cl/Tanh.cpp +++ b/tests/validation/dynamic_fusion/gpu/cl/Tanh.cpp @@ -61,11 +61,11 @@ DATA_TEST_CASE(Validate, framework::DatasetMode::ALL, zip( { // Create a new workload sketch CLCompileContext cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - GpuWorkloadContext gpu_ctx{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + GpuWorkloadContext context{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; // Fuse tanh - const TensorInfo src_info = sketch.create_tensor_info(input_info); + const TensorInfo src_info = context.create_tensor_info(input_info); const bool res = static_cast(GpuTanh::validate_op(sketch, &src_info)); ARM_COMPUTE_EXPECT(res == expected, framework::LogLevel::ERRORS); @@ -97,7 +97,8 @@ FIXTURE_DATA_TEST_CASE(RunSmall5dOneOp, framework::dataset::make("DataType", DataType::F16))) { // Validate output - validate(CLAccessor(_target), _reference, tolerance_f16); + ARM_COMPUTE_TEST_INFO("Currently 5D+ tensors are unsupported for this operation."); + framework::ARM_COMPUTE_PRINT_INFO(); } FIXTURE_DATA_TEST_CASE(RunSmallTwoOps, @@ -133,7 +134,8 @@ FIXTURE_DATA_TEST_CASE(RunSmall5dOneOp, framework::dataset::make("DataType", DataType::F32))) { // Validate output - validate(CLAccessor(_target), _reference, tolerance_f32); + ARM_COMPUTE_TEST_INFO("Currently 5D+ tensors are unsupported for this operation."); + framework::ARM_COMPUTE_PRINT_INFO(); } FIXTURE_DATA_TEST_CASE(RunSmallTwoOps, diff --git a/tests/validation/fixtures/ActivationLayerFixture.h b/tests/validation/fixtures/ActivationLayerFixture.h index 531b3abdf992eb9cb512cca26a72f414a101d2e8..849c649b4c65075fb720f9ba099a0b48160119d5 100644 --- a/tests/validation/fixtures/ActivationLayerFixture.h +++ b/tests/validation/fixtures/ActivationLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -52,7 +52,6 @@ public: { } - template void setup(TensorShape shape, bool in_place, ActivationLayerInfo::ActivationFunction function, float alpha_beta, DataType data_type, QuantizationInfo quantization_info) { ActivationLayerInfo info(function, alpha_beta, alpha_beta); @@ -234,7 +233,6 @@ template { public: - template void setup(TensorShape shape, bool in_place, ActivationLayerInfo::ActivationFunction function, float alpha_beta, DataType data_type) { ActivationValidationGenericFixture::setup(shape, in_place, function, alpha_beta, data_type, QuantizationInfo()); @@ -245,7 +243,6 @@ template { public: - template void setup(TensorShape shape, bool in_place, ActivationLayerInfo::ActivationFunction function, float alpha_beta, DataType data_type, QuantizationInfo quantization_info) { ActivationValidationGenericFixture::setup(shape, in_place, function, alpha_beta, data_type, quantization_info); diff --git a/tests/validation/fixtures/AddMulAddFixture.h b/tests/validation/fixtures/AddMulAddFixture.h index fac2bfe528a22c9480c3bc044f482e320de59d81..a3a761bff1d95e52baf3137f545567a1aa1e12e9 100644 --- a/tests/validation/fixtures/AddMulAddFixture.h +++ b/tests/validation/fixtures/AddMulAddFixture.h @@ -49,7 +49,6 @@ template void setup(const TensorShape &shape, DataType data_type, ActivationLayerInfo &act_info, bool interm_out) { compute_target(shape, data_type, act_info, interm_out); @@ -138,7 +137,6 @@ class AddMulAddFloatValidationFixture : public AddMulAddGenericFixture; - template void setup(const TensorShape &shape, DataType data_type, ActivationLayerInfo act_info) { Parent::setup(shape, data_type, act_info, interm_out); @@ -195,7 +193,6 @@ class AddMulAddQuantizedValidationFixture : public AddMulAddGenericFixture; - template void setup(const TensorShape &shape, DataType data_type, ActivationLayerInfo act_info, QuantizationInfo input1_qinfo, QuantizationInfo input2_qinfo, QuantizationInfo bn_mul_qinfo, QuantizationInfo bn_add_qinfo, QuantizationInfo add_output_qinfo, QuantizationInfo final_output_qinfo) diff --git a/tests/validation/fixtures/ArgMinMaxFixture.h b/tests/validation/fixtures/ArgMinMaxFixture.h index 2bbce4077ebbac4d5835fd0230cfc314f6488017..7a823568a8e826e7d96a2006222139db1341b02d 100644 --- a/tests/validation/fixtures/ArgMinMaxFixture.h +++ b/tests/validation/fixtures/ArgMinMaxFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022 Arm Limited. + * Copyright (c) 2018-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -42,15 +42,14 @@ namespace test { namespace validation { -template +template class ArgMinMaxValidationBaseFixture : public framework::Fixture { public: - template - void setup(TensorShape shape, DataType data_type, int axis, ReductionOperation op, QuantizationInfo q_info) + void setup(TensorShape shape, DataType input_type, DataType output_type, int axis, ReductionOperation op, QuantizationInfo q_info) { - _target = compute_target(shape, data_type, axis, op, q_info); - _reference = compute_reference(shape, data_type, axis, op, q_info); + _target = compute_target(shape, input_type, output_type, axis, op, q_info); + _reference = compute_reference(shape, input_type, output_type, axis, op, q_info); } protected: @@ -98,11 +97,11 @@ protected: } } - TensorType compute_target(TensorShape &src_shape, DataType data_type, int axis, ReductionOperation op, QuantizationInfo q_info) + TensorType compute_target(TensorShape &src_shape, DataType input_type, DataType output_type, int axis, ReductionOperation op, QuantizationInfo q_info) { // Create tensors - TensorType src = create_tensor(src_shape, data_type, 1, q_info); - TensorType dst; + TensorType src = create_tensor(src_shape, input_type, 1, q_info); + TensorType dst = create_tensor(compute_output_shape(src_shape, axis), output_type, 1, q_info); // Create and configure function FunctionType arg_min_max_layer; @@ -127,41 +126,43 @@ protected: return dst; } - SimpleTensor compute_reference(TensorShape &src_shape, DataType data_type, int axis, ReductionOperation op, QuantizationInfo q_info) + TensorShape compute_output_shape(const TensorShape &src_shape, int axis) + { + return arm_compute::misc::shape_calculator::compute_reduced_shape(src_shape, axis, false); + } + + SimpleTensor compute_reference(TensorShape &src_shape, DataType input_type, DataType output_type, int axis, ReductionOperation op, QuantizationInfo q_info) { // Create reference - SimpleTensor src{ src_shape, data_type, 1, q_info }; + SimpleTensor src{ src_shape, input_type, 1, q_info }; // Fill reference fill(src); - TensorShape output_shape = arm_compute::misc::shape_calculator::compute_reduced_shape(src_shape, axis, false); - return reference::reduction_operation(src, output_shape, axis, op); + return reference::reduction_operation(src, compute_output_shape(src_shape, axis), axis, op, output_type); } - TensorType _target{}; - SimpleTensor _reference{}; + TensorType _target{}; + SimpleTensor _reference{}; }; -template -class ArgMinMaxValidationQuantizedFixture : public ArgMinMaxValidationBaseFixture +template +class ArgMinMaxValidationQuantizedFixture : public ArgMinMaxValidationBaseFixture { public: - template - void setup(const TensorShape &shape, DataType data_type, int axis, ReductionOperation op, QuantizationInfo quantization_info) + void setup(const TensorShape &shape, DataType input_type, DataType output_type, int axis, ReductionOperation op, QuantizationInfo quantization_info) { - ArgMinMaxValidationBaseFixture::setup(shape, data_type, axis, op, quantization_info); + ArgMinMaxValidationBaseFixture::setup(shape, input_type, output_type, axis, op, quantization_info); } }; -template -class ArgMinMaxValidationFixture : public ArgMinMaxValidationBaseFixture +template +class ArgMinMaxValidationFixture : public ArgMinMaxValidationBaseFixture { public: - template - void setup(const TensorShape &shape, DataType data_type, int axis, ReductionOperation op) + void setup(const TensorShape &shape, DataType input_type, DataType output_type, int axis, ReductionOperation op) { - ArgMinMaxValidationBaseFixture::setup(shape, data_type, axis, op, QuantizationInfo()); + ArgMinMaxValidationBaseFixture::setup(shape, input_type, output_type, axis, op, QuantizationInfo()); } }; } // namespace validation diff --git a/tests/validation/fixtures/ArithmeticDivisionFixture.h b/tests/validation/fixtures/ArithmeticDivisionFixture.h index 2c2c01c31c4c4fd80e23f5f31e30e72e48d9ba47..e11a386130635e9327bee4e391b3f7099dadec81 100644 --- a/tests/validation/fixtures/ArithmeticDivisionFixture.h +++ b/tests/validation/fixtures/ArithmeticDivisionFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -44,7 +44,6 @@ template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType data_type) { _target = compute_target(shape0, shape1, data_type); @@ -117,7 +116,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type) { ArithmeticDivisionBroadcastValidationFixture::setup(shape, shape, data_type); diff --git a/tests/validation/fixtures/ArithmeticOperationsFixture.h b/tests/validation/fixtures/ArithmeticOperationsFixture.h index 0f7e44e58852ffc1213f90b0f905d0ae11805c50..0785af1151c7e67393c12c1cb9bbbfa7715bdc25 100644 --- a/tests/validation/fixtures/ArithmeticOperationsFixture.h +++ b/tests/validation/fixtures/ArithmeticOperationsFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -45,7 +45,6 @@ template void setup(reference::ArithmeticOperation op, const TensorShape &shape0, const TensorShape &shape1, DataType data_type, ConvertPolicy convert_policy, QuantizationInfo qinfo0, QuantizationInfo qinfo1, QuantizationInfo qinfo_out, ActivationLayerInfo act_info, bool is_inplace) { @@ -150,7 +149,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType data_type, ConvertPolicy convert_policy, bool is_inplace) { ArithmeticOperationGenericFixture::setup(reference::ArithmeticOperation::ADD, shape0, shape1, data_type, convert_policy, @@ -162,7 +160,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type, ConvertPolicy convert_policy, bool is_inplace) { ArithmeticOperationGenericFixture::setup(reference::ArithmeticOperation::ADD, shape, shape, data_type, convert_policy, @@ -174,7 +171,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType data_type, ConvertPolicy convert_policy, ActivationLayerInfo act_info, bool is_inplace) { ArithmeticOperationGenericFixture::setup(reference::ArithmeticOperation::ADD, shape0, shape1, data_type, convert_policy, @@ -186,7 +182,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type, ConvertPolicy convert_policy, ActivationLayerInfo act_info, bool is_inplace) { ArithmeticOperationGenericFixture::setup(reference::ArithmeticOperation::ADD, shape, shape, data_type, convert_policy, @@ -198,7 +193,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type, ConvertPolicy convert_policy, QuantizationInfo qinfo0, QuantizationInfo qinfo1, QuantizationInfo qinfo_out, bool is_inplace) { @@ -211,7 +205,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType data_type, ConvertPolicy convert_policy, QuantizationInfo qinfo0, QuantizationInfo qinfo1, QuantizationInfo qinfo_out, bool is_inplace) { @@ -224,7 +217,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType data_type, ConvertPolicy convert_policy, bool is_inplace) { ArithmeticOperationGenericFixture::setup(reference::ArithmeticOperation::SUB, shape0, shape1, data_type, convert_policy, @@ -236,7 +228,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType data_type, ConvertPolicy convert_policy, ActivationLayerInfo act_info, bool is_inplace) { @@ -249,7 +240,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type, ConvertPolicy convert_policy, bool is_inplace) { ArithmeticOperationGenericFixture::setup(reference::ArithmeticOperation::SUB, shape, shape, data_type, convert_policy, @@ -261,7 +251,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type, ConvertPolicy convert_policy, ActivationLayerInfo act_info, bool is_inplace) { ArithmeticOperationGenericFixture::setup(reference::ArithmeticOperation::SUB, shape, shape, data_type, convert_policy, @@ -273,7 +262,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type, ConvertPolicy convert_policy, QuantizationInfo qinfo0, QuantizationInfo qinfo1, QuantizationInfo qinfo_out, bool is_inplace) { @@ -286,7 +274,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType data_type, ConvertPolicy convert_policy, QuantizationInfo qinfo0, QuantizationInfo qinfo1, QuantizationInfo qinfo_out, bool is_inplace) { diff --git a/tests/validation/fixtures/BatchNormalizationLayerFixture.h b/tests/validation/fixtures/BatchNormalizationLayerFixture.h index 38b4d4375abea4f57608fb3d39434333be19c3ee..54a0ed9e09de14fb44c94526e6af6c7d9168a608 100644 --- a/tests/validation/fixtures/BatchNormalizationLayerFixture.h +++ b/tests/validation/fixtures/BatchNormalizationLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -44,7 +44,6 @@ template void setup(TensorShape shape0, TensorShape shape1, float epsilon, bool use_beta, bool use_gamma, ActivationLayerInfo act_info, DataType dt, DataLayout data_layout) { _data_type = dt; diff --git a/tests/validation/fixtures/BatchNormalizationLayerFusionFixture.h b/tests/validation/fixtures/BatchNormalizationLayerFusionFixture.h index 1ffb8f38f3d35e9c5ab07da8c630c632ac913e62..161eeb0ef4605fb0948323e9da0118926747881a 100644 --- a/tests/validation/fixtures/BatchNormalizationLayerFusionFixture.h +++ b/tests/validation/fixtures/BatchNormalizationLayerFusionFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -45,7 +45,6 @@ template void setup(TensorShape src_shape, TensorShape w_shape, TensorShape b_shape, TensorShape dst_shape, PadStrideInfo info, Size2D dilation, bool use_conv_b, bool use_beta, bool use_gamma, float epsilon, DataType dt, DataLayout data_layout) { diff --git a/tests/validation/fixtures/BatchToSpaceLayerFixture.h b/tests/validation/fixtures/BatchToSpaceLayerFixture.h index 19fc82a87bd643a40abf4d74af82f6cb218b500e..56a6109dbc1a1de96e5f4104ad0cf82d6a1c25c2 100644 --- a/tests/validation/fixtures/BatchToSpaceLayerFixture.h +++ b/tests/validation/fixtures/BatchToSpaceLayerFixture.h @@ -40,7 +40,6 @@ template void setup(const TensorShape &input_shape, const std::vector &block_shape, const CropInfo &crop_info, const TensorShape &output_shape, DataType data_type, DataLayout data_layout) { _target = compute_target(input_shape, block_shape, crop_info, output_shape, data_type, data_layout); diff --git a/tests/validation/fixtures/BitwiseAndFixture.h b/tests/validation/fixtures/BitwiseAndFixture.h index 0550532e4b24aa4064d9a67724a87e0d9fe287a0..745a34058e983550734399f312c7340749462df5 100644 --- a/tests/validation/fixtures/BitwiseAndFixture.h +++ b/tests/validation/fixtures/BitwiseAndFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -43,7 +43,6 @@ template void setup(TensorShape shape, DataType data_type) { _target = compute_target(shape, data_type); diff --git a/tests/validation/fixtures/BitwiseNotFixture.h b/tests/validation/fixtures/BitwiseNotFixture.h index 9cf19383816adb92089e3f4e86b666373cb9d62a..bdfd255156556065913d8480f55592e7a637d24f 100644 --- a/tests/validation/fixtures/BitwiseNotFixture.h +++ b/tests/validation/fixtures/BitwiseNotFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -43,7 +43,6 @@ template void setup(TensorShape shape, DataType data_type) { _target = compute_target(shape, data_type); diff --git a/tests/validation/fixtures/BitwiseOrFixture.h b/tests/validation/fixtures/BitwiseOrFixture.h index c4fc7d7c22b346903627c62633ca0f1cbe02eaaa..03560e01710f4837b9c150c7bcf2a37f8467e0c0 100644 --- a/tests/validation/fixtures/BitwiseOrFixture.h +++ b/tests/validation/fixtures/BitwiseOrFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -43,7 +43,6 @@ template void setup(TensorShape shape, DataType data_type) { _target = compute_target(shape, data_type); diff --git a/tests/validation/fixtures/BitwiseXorFixture.h b/tests/validation/fixtures/BitwiseXorFixture.h index 479694c78ef14cd5985a6b66b74404f2bb654c68..4872b231a5227ed485da8fb0ba983dbc6450837e 100644 --- a/tests/validation/fixtures/BitwiseXorFixture.h +++ b/tests/validation/fixtures/BitwiseXorFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -43,7 +43,6 @@ template void setup(TensorShape shape, DataType data_type) { _target = compute_target(shape, data_type); diff --git a/tests/validation/fixtures/BoundingBoxTransformFixture.h b/tests/validation/fixtures/BoundingBoxTransformFixture.h index cd6ce99796a17476bd9494438c7a5df439ec5272..03edaeab16a57a460d788f30e825bb80daeca942 100644 --- a/tests/validation/fixtures/BoundingBoxTransformFixture.h +++ b/tests/validation/fixtures/BoundingBoxTransformFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -102,7 +102,6 @@ class BoundingBoxTransformGenericFixture : public framework::Fixture public: using TDeltas = typename std::conditional::type, uint16_t>::value, uint8_t, T>::type; - template void setup(TensorShape deltas_shape, const BoundingBoxTransformInfo &info, DataType data_type, QuantizationInfo deltas_qinfo) { const bool is_qasymm16 = data_type == DataType::QASYMM16; @@ -215,7 +214,6 @@ template { public: - template void setup(TensorShape deltas_shape, const BoundingBoxTransformInfo &info, DataType data_type) { BoundingBoxTransformGenericFixture::setup(deltas_shape, info, data_type, QuantizationInfo()); @@ -228,7 +226,6 @@ template { public: - template void setup(TensorShape deltas_shape, const BoundingBoxTransformInfo &info, DataType data_type, QuantizationInfo deltas_qinfo) { BoundingBoxTransformGenericFixture::setup(deltas_shape, info, data_type, deltas_qinfo); diff --git a/tests/validation/fixtures/CastFixture.h b/tests/validation/fixtures/CastFixture.h index 056f6642612442b26cef3972906057bad821fc00..e9d624e6f344e4d7281875ffa246c06154b31fe0 100644 --- a/tests/validation/fixtures/CastFixture.h +++ b/tests/validation/fixtures/CastFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -36,7 +36,6 @@ template void setup(TensorShape shape, DataType dt_in, DataType dt_out, ConvertPolicy policy) { _target = compute_target(shape, dt_in, dt_out, policy); @@ -86,6 +85,16 @@ protected: library->fill_tensor_uniform(tensor, i, static_cast(signed_min), static_cast(signed_max)); break; } + case DataType::U64: + { + library->fill_tensor_uniform(tensor, i, static_cast(unsigned_min), static_cast(unsigned_max)); + break; + } + case DataType::S64: + { + library->fill_tensor_uniform(tensor, i, static_cast(signed_min), static_cast(signed_max)); + break; + } default: ARM_COMPUTE_ERROR("NOT SUPPORTED!"); } diff --git a/tests/validation/fixtures/ChannelShuffleLayerFixture.h b/tests/validation/fixtures/ChannelShuffleLayerFixture.h index 9199d9296d5c67b7f84f9c19840f414b978bb6b5..530dba3893d3b4436a83e7414389f9408d941707 100644 --- a/tests/validation/fixtures/ChannelShuffleLayerFixture.h +++ b/tests/validation/fixtures/ChannelShuffleLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -45,7 +45,6 @@ template void setup(TensorShape shape, unsigned int num_groups, DataType data_type, DataLayout data_layout) { _target = compute_target(shape, data_type, num_groups, data_layout); diff --git a/tests/validation/fixtures/Col2ImFixture.h b/tests/validation/fixtures/Col2ImFixture.h index ee7a14f844e553d92a68c6145c0808c8068925be..4d56d607b7b03dcd9dd062fb6a9ea4e64e8d4206 100644 --- a/tests/validation/fixtures/Col2ImFixture.h +++ b/tests/validation/fixtures/Col2ImFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -48,7 +48,6 @@ template void setup(TensorShape input_shape, const unsigned int convolved_width, unsigned int convolved_height, unsigned int num_groups, DataType data_type) { const Size2D convolved_dims(convolved_width, convolved_height); diff --git a/tests/validation/fixtures/ComparisonFixture.h b/tests/validation/fixtures/ComparisonFixture.h index 6b38a376a4195a10db38d3381ea52ffffa083386..f25d5abb737859bdedc79e10442d4bdf8673e88b 100644 --- a/tests/validation/fixtures/ComparisonFixture.h +++ b/tests/validation/fixtures/ComparisonFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -44,7 +44,6 @@ template void setup(ComparisonOperation op, const TensorShape &shape0, const TensorShape &shape1, DataType data_type, QuantizationInfo qinfo0, QuantizationInfo qinfo1) { _target = compute_target(op, shape0, shape1, data_type, qinfo0, qinfo1); @@ -117,7 +116,6 @@ template { public: - template void setup(ComparisonOperation op, const TensorShape &shape0, const TensorShape &shape1, DataType data_type) { ComparisonValidationGenericFixture::setup(op, shape0, shape1, data_type, QuantizationInfo(), QuantizationInfo()); @@ -128,7 +126,6 @@ template { public: - template void setup(ComparisonOperation op, const TensorShape &shape, DataType data_type) { ComparisonValidationGenericFixture::setup(op, shape, shape, data_type, QuantizationInfo(), QuantizationInfo()); @@ -139,7 +136,6 @@ template { public: - template void setup(ComparisonOperation op, const TensorShape &shape, DataType data_type, QuantizationInfo qinfo0, QuantizationInfo qinfo1) { @@ -151,7 +147,6 @@ template { public: - template void setup(ComparisonOperation op, const TensorShape &shape0, const TensorShape &shape1, DataType data_type, QuantizationInfo qinfo0, QuantizationInfo qinfo1) { ComparisonValidationGenericFixture::setup(op, shape0, shape1, data_type, qinfo0, qinfo1); diff --git a/tests/validation/fixtures/ComputeAllAnchorsFixture.h b/tests/validation/fixtures/ComputeAllAnchorsFixture.h index e0fe35ba6fef84823302aa8c1308364dfbdf8e9d..620f1b53faa910eb3c58d99ed8a3e2608fab55f5 100644 --- a/tests/validation/fixtures/ComputeAllAnchorsFixture.h +++ b/tests/validation/fixtures/ComputeAllAnchorsFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -44,7 +44,6 @@ template void setup(size_t num_anchors, const ComputeAnchorsInfo &info, DataType data_type, QuantizationInfo qinfo) { _target = compute_target(num_anchors, data_type, info, qinfo); @@ -107,7 +106,6 @@ template { public: - template void setup(size_t num_anchors, const ComputeAnchorsInfo &info, DataType data_type) { ComputeAllAnchorsGenericFixture::setup(num_anchors, info, data_type, QuantizationInfo()); @@ -118,7 +116,6 @@ template { public: - template void setup(size_t num_anchors, const ComputeAnchorsInfo &info, DataType data_type, QuantizationInfo qinfo) { ComputeAllAnchorsGenericFixture::setup(num_anchors, info, data_type, qinfo); diff --git a/tests/validation/fixtures/ConcatenateLayerFixture.h b/tests/validation/fixtures/ConcatenateLayerFixture.h index bafb8b2392249ae07513102361aaf1549e521da5..3a021661aca4faf3e1b63b20f61b3c357330b95b 100644 --- a/tests/validation/fixtures/ConcatenateLayerFixture.h +++ b/tests/validation/fixtures/ConcatenateLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -50,7 +50,6 @@ private: using SrcITensorType = typename std::conditional::type; public: - template void setup(TensorShape shape, DataType data_type, unsigned int axis) { // Create input shapes diff --git a/tests/validation/fixtures/ConvertFullyConnectedWeightsFixture.h b/tests/validation/fixtures/ConvertFullyConnectedWeightsFixture.h index 38088b40005c2ebf18e806bd5ee69740a81fa8ea..7ad14e1b40bad9737156ee703ed1d10363557e51 100644 --- a/tests/validation/fixtures/ConvertFullyConnectedWeightsFixture.h +++ b/tests/validation/fixtures/ConvertFullyConnectedWeightsFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022 Arm Limited. + * Copyright (c) 2018-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -43,7 +43,6 @@ template void setup(TensorShape input_shape, unsigned int weights_w, DataLayout training_data_layout, DataType data_type) { const unsigned int height = input_shape.x() * input_shape.y() * input_shape.z(); diff --git a/tests/validation/fixtures/ConvolutionLayerFixture.h b/tests/validation/fixtures/ConvolutionLayerFixture.h index e4c2e4bfea02e999fc2bc80ba1f0fea00a82f394..8562955b79925488312c695ca01e0e4f7d055bd2 100644 --- a/tests/validation/fixtures/ConvolutionLayerFixture.h +++ b/tests/validation/fixtures/ConvolutionLayerFixture.h @@ -92,7 +92,6 @@ public: int32_t, T >::type; public: - template void setup(TensorShape input_shape, TensorShape weights_shape, TensorShape bias_shape, TensorShape output_shape, PadStrideInfo info, Size2D dilation, bool reshape_weights, DataType data_type, DataType weights_data_type, DataLayout data_layout, QuantizationInfo quantization_info, QuantizationInfo weight_quantization_info, ActivationLayerInfo act_info, bool mixed_layout = false, PaddingList pre_pad_layer = PaddingList({})) @@ -362,7 +361,6 @@ template { public: - template void setup(TensorShape input_shape, TensorShape weights_shape, TensorShape bias_shape, TensorShape output_shape, PadStrideInfo info, Size2D dilation, bool reshape_weights, DataType data_type, DataLayout data_layout, ActivationLayerInfo act_info) { @@ -376,7 +374,6 @@ template { public: - template void setup(TensorShape input_shape, TensorShape weights_shape, TensorShape bias_shape, TensorShape output_shape, PadStrideInfo info, Size2D dilation, bool reshape_weights, DataType data_type, DataLayout data_layout, ActivationLayerInfo act_info, PaddingList pre_pad_layer = PaddingList({})) { @@ -390,7 +387,6 @@ template { public: - template void setup(TensorShape input_shape, TensorShape weights_shape, TensorShape bias_shape, TensorShape output_shape, PadStrideInfo info, Size2D dilation, bool reshape_weights, DataType data_type, DataLayout data_layout, QuantizationInfo quantization_info, ActivationLayerInfo act_info) { @@ -403,7 +399,6 @@ template { public: - template void setup(TensorShape input_shape, TensorShape weights_shape, TensorShape bias_shape, TensorShape output_shape, PadStrideInfo info, Size2D dilation, bool reshape_weights, DataType data_type, DataLayout data_layout, QuantizationInfo quantization_info, ActivationLayerInfo act_info, DataType weights_data_type) { @@ -514,7 +509,6 @@ template void setup(TensorShape input_shape, TensorShape weights_shape, TensorShape bias_shape, TensorShape output_shape, PadStrideInfo info, Size2D dilation, DataLayout data_layout, const DataType data_type) { @@ -702,7 +696,6 @@ template class HasOptImplFixture : public framework::Fixture { public: - template void setup(DataType data_type, arm_compute::WeightFormat query_weight_format) { auto conv = std::make_unique(); diff --git a/tests/validation/fixtures/CopyFixture.h b/tests/validation/fixtures/CopyFixture.h index eca75e7b69d4c1b6c4714324a2a391ce0fa3c4aa..f5e711a50021aa80d8c492eef0bf99ec895260a9 100644 --- a/tests/validation/fixtures/CopyFixture.h +++ b/tests/validation/fixtures/CopyFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -43,7 +43,6 @@ template void setup(TensorShape input_shape, TensorShape output_shape, DataType data_type) { _target = compute_target(input_shape, output_shape, data_type); diff --git a/tests/validation/fixtures/CropResizeFixture.h b/tests/validation/fixtures/CropResizeFixture.h index 5aa63c4ef8a1dee30f45c7dcee2fb770fb1a343d..30a3fd85690078f677e2ea552cec01b42a1bbf86 100644 --- a/tests/validation/fixtures/CropResizeFixture.h +++ b/tests/validation/fixtures/CropResizeFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -46,7 +46,6 @@ template void setup(TensorShape src_shape, TensorShape boxes_shape, Coordinates2D crop_size, InterpolationPolicy method, float extrapolation_value, bool is_outside_bounds, DataType data_type) { diff --git a/tests/validation/fixtures/DeconvolutionLayerFixture.h b/tests/validation/fixtures/DeconvolutionLayerFixture.h index d13eab2f5497e003844413c4aa54caaf543e2994..83170c413c5afee2ff0d1d803c2c639fd3e4e7c7 100644 --- a/tests/validation/fixtures/DeconvolutionLayerFixture.h +++ b/tests/validation/fixtures/DeconvolutionLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2022 Arm Limited. + * Copyright (c) 2017-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -49,7 +49,6 @@ public: using TBias = typename std::conditional < std::is_same::type, uint8_t>::value || std::is_same::type, int8_t>::value, int32_t, T >::type; public: - template void setup(TensorShape input_shape, TensorShape weights_shape, TensorShape bias_shape, TensorShape output_shape, PadStrideInfo info, DataType data_type, DataType weights_data_type, DataLayout data_layout, QuantizationInfo input_quantization_info, QuantizationInfo output_quantization_info, QuantizationInfo weights_quantization_info, bool add_bias) @@ -246,11 +245,9 @@ template { public: - template void setup(TensorShape input_shape, unsigned int sx, unsigned int sy, unsigned int padx, unsigned int pady, unsigned int num_kernels, DataType data_type, DataLayout data_layout, bool add_bias) { - ARM_COMPUTE_ERROR_ON_MSG(kernel_size_x != kernel_size_y, "Only square kernels supported"); const TensorShape weights_shape(kernel_size_x, kernel_size_y, input_shape.z(), num_kernels); const TensorShape bias_shape(num_kernels); const PadStrideInfo info(sx, sy, padx, pady, DimensionRoundingType::CEIL); @@ -267,11 +264,9 @@ template { public: - template void setup(TensorShape input_shape, unsigned int sx, unsigned int sy, unsigned int pad_left, unsigned int pad_right, unsigned int pad_top, unsigned int pad_bottom, unsigned int num_kernels, DataType data_type, DataLayout data_layout, bool add_bias) { - ARM_COMPUTE_ERROR_ON_MSG(kernel_size_x != kernel_size_y, "Only square kernels supported"); const TensorShape weights_shape(kernel_size_x, kernel_size_y, input_shape.z(), num_kernels); const TensorShape bias_shape(num_kernels); const PadStrideInfo info(sx, sy, pad_left, pad_right, pad_top, pad_bottom, DimensionRoundingType::CEIL); @@ -288,11 +283,9 @@ template { public: - template void setup(TensorShape input_shape, unsigned int sx, unsigned int sy, unsigned int padx, unsigned int pady, unsigned int num_kernels, DataType data_type, DataLayout data_layout, QuantizationInfo input_quantization_info, QuantizationInfo output_quantization_info, bool add_bias) { - ARM_COMPUTE_ERROR_ON_MSG(kernel_size_x != kernel_size_y, "Only square kernels supported"); const TensorShape weights_shape(kernel_size_x, kernel_size_y, input_shape.z(), num_kernels); const TensorShape bias_shape(num_kernels); const PadStrideInfo info(sx, sy, padx, pady, DimensionRoundingType::CEIL); @@ -310,12 +303,10 @@ template { public: - template void setup(TensorShape input_shape, unsigned int sx, unsigned int sy, unsigned int padx, unsigned int pady, unsigned int num_kernels, DataType data_type, DataLayout data_layout, QuantizationInfo input_quantization_info, QuantizationInfo output_quantization_info, bool add_bias, DataType weights_data_type) { - ARM_COMPUTE_ERROR_ON_MSG(kernel_size_x != kernel_size_y, "Only square kernels supported"); const TensorShape weights_shape(kernel_size_x, kernel_size_y, input_shape.z(), num_kernels); const TensorShape bias_shape(num_kernels); const PadStrideInfo info(sx, sy, padx, pady, DimensionRoundingType::CEIL); diff --git a/tests/validation/fixtures/DepthConvertLayerFixture.h b/tests/validation/fixtures/DepthConvertLayerFixture.h index 53d29b44bacc51933ba4ad0351f4df25ef97c04a..f55d20bf3eff022d23ae392287b79775009e7fde 100644 --- a/tests/validation/fixtures/DepthConvertLayerFixture.h +++ b/tests/validation/fixtures/DepthConvertLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2022 Arm Limited. + * Copyright (c) 2017-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -45,7 +45,6 @@ template void setup(TensorShape shape, DataType dt_in, DataType dt_out, ConvertPolicy policy, uint32_t shift, QuantizationInfo quantization_info) { _shift = shift; @@ -130,7 +129,6 @@ template { public: - template void setup(TensorShape shape, DataType dt_in, DataType dt_out, ConvertPolicy policy, uint32_t shift) { DepthConvertLayerValidationBaseFixture::setup(shape, dt_in, dt_out, policy, @@ -142,7 +140,6 @@ template { public: - template void setup(TensorShape shape, DataType dt_in, DataType dt_out, ConvertPolicy policy, uint32_t shift, QuantizationInfo quantization_info) { DepthConvertLayerValidationBaseFixture::setup(shape, dt_in, dt_out, policy, diff --git a/tests/validation/fixtures/DepthToSpaceLayerFixture.h b/tests/validation/fixtures/DepthToSpaceLayerFixture.h index b7f555247491dc07b080c4d2f2fe41892c12a2c1..abe3d8b22f0d8b5accb245eaea306ebbaf22f08a 100644 --- a/tests/validation/fixtures/DepthToSpaceLayerFixture.h +++ b/tests/validation/fixtures/DepthToSpaceLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -39,7 +39,6 @@ template void setup(TensorShape input_shape, int32_t block_shape, TensorShape output_shape, DataType data_type, DataLayout data_layout) { _target = compute_target(input_shape, block_shape, output_shape, data_type, data_layout); diff --git a/tests/validation/fixtures/DepthwiseConvolutionLayerFixture.h b/tests/validation/fixtures/DepthwiseConvolutionLayerFixture.h index a08080a665d8807e56455579927c21bcc28d8cec..5a00c7c2d6bd42978735d9edf6418d05e2c4b0a7 100644 --- a/tests/validation/fixtures/DepthwiseConvolutionLayerFixture.h +++ b/tests/validation/fixtures/DepthwiseConvolutionLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2022 Arm Limited. + * Copyright (c) 2017-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -55,7 +55,6 @@ public: using TBias = typename std::conditional < std::is_same::value || std::is_same::value, int32_t, T >::type; public: - template void setup(TensorShape in_shape, Size2D kernel_size, PadStrideInfo pad_stride_info, Size2D dilation, unsigned int depth_multiplier, DataType input_data_type, DataType weights_data_type, QuantizationInfo input_quantization_info, QuantizationInfo weights_quantization_info, QuantizationInfo output_quantization_info, @@ -289,7 +288,6 @@ template { public: - template void setup(TensorShape in_shape, Size2D kernel_size, PadStrideInfo pad_stride_info, Size2D dilation, unsigned int depth_multiplier, DataType data_type, DataLayout data_layout, ActivationLayerInfo act_info) { @@ -303,7 +301,6 @@ template { public: - template void setup(size_t width, size_t height, size_t channel, size_t batch, Size2D kernel_size, size_t depth_multiplier, Size2D dilation, Size2D stride, bool padding_valid, DataType data_type, DataLayout data_layout) { @@ -442,7 +439,6 @@ template { public: - template void setup(size_t width, size_t height, size_t channel, size_t batch, Size2D kernel_size, size_t depth_multiplier, Size2D dilation, Size2D stride, bool padding_valid, DataType data_type, DataLayout data_layout, const ActivationLayerInfo &act_info, unsigned int n0, bool export_to_cl_image) { @@ -640,7 +636,6 @@ template { public: - template void setup(TensorShape in_shape, Size2D kernel_size, PadStrideInfo pad_stride_info, Size2D dilation, unsigned int depth_multiplier, DataType data_type, QuantizationInfo input_quantization_info, QuantizationInfo output_quantization_info, DataLayout data_layout, ActivationLayerInfo act_info) { @@ -654,7 +649,6 @@ template { public: - template void setup(TensorShape in_shape, Size2D kernel_size, PadStrideInfo pad_stride_info, Size2D dilation, unsigned int depth_multiplier, DataType input_data_type, DataType weights_data_type, QuantizationInfo input_quantization_info, QuantizationInfo output_quantization_info, DataLayout data_layout, ActivationLayerInfo act_info) { diff --git a/tests/validation/fixtures/DequantizationLayerFixture.h b/tests/validation/fixtures/DequantizationLayerFixture.h index af998bb7408ca2ff0308bbdb5a892ee4eaee810f..4eb25a5bc518f12cb23b06b175190de10374765c 100644 --- a/tests/validation/fixtures/DequantizationLayerFixture.h +++ b/tests/validation/fixtures/DequantizationLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -47,7 +47,6 @@ template void setup(TensorShape shape, DataType src_data_type, DataType dst_datatype, DataLayout data_layout) { _quantization_info = generate_quantization_info(src_data_type, shape.z()); diff --git a/tests/validation/fixtures/DirectConvolution3DFixture.h b/tests/validation/fixtures/DirectConvolution3DFixture.h index 2250dcaeb0fdc63a577f3b92363124e895d1f4b7..6aad5490d90b6fabe5250f85fe5d8fe4d4b01023 100644 --- a/tests/validation/fixtures/DirectConvolution3DFixture.h +++ b/tests/validation/fixtures/DirectConvolution3DFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -42,7 +42,6 @@ class DirectConvolution3DValidationGenericFixture : public framework::Fixture public: using TBias = typename std::conditional < std::is_same::value || std::is_same::value, int32_t, T >::type; - template void setup(const TensorShape &input_shape, int stride_x, int stride_y, int stride_z, int pad_x, int pad_y, int pad_z, unsigned int kernel_width, int kernel_height, int kernel_depth, unsigned int num_kernels, bool has_bias, const ActivationLayerInfo &act_info, const DataType &data_type, const DataLayout &data_layout, const QuantizationInfo &src_qinfo = QuantizationInfo(), const QuantizationInfo &weights_qinfo = QuantizationInfo(), const QuantizationInfo &dst_qinfo = QuantizationInfo()) @@ -157,7 +156,6 @@ template { public: - template void setup(TensorShape input_shape, int stride_x, int stride_y, int stride_z, int pad_x, int pad_y, int pad_z, unsigned int kernel_width, int kernel_height, int kernel_depth, unsigned int num_kernels, bool has_bias, ActivationLayerInfo act_info, DataType data_type, DataLayout data_layout) { @@ -170,7 +168,6 @@ template { public: - template void setup(TensorShape input_shape, int stride_x, int stride_y, int stride_z, int pad_x, int pad_y, int pad_z, unsigned int kernel_width, int kernel_height, int kernel_depth, unsigned int num_kernels, bool has_bias, ActivationLayerInfo act_info, DataType data_type, DataLayout data_layout, QuantizationInfo src_qinfo, QuantizationInfo weights_qinfo, QuantizationInfo dst_qinfo) diff --git a/tests/validation/fixtures/DirectConvolutionLayerFixture.h b/tests/validation/fixtures/DirectConvolutionLayerFixture.h index 31186e2b1dfa042c1491327ecf5185297fae71b2..a666ff96a5da3010f575f1ecfefd2327d79153cc 100644 --- a/tests/validation/fixtures/DirectConvolutionLayerFixture.h +++ b/tests/validation/fixtures/DirectConvolutionLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2022 Arm Limited. + * Copyright (c) 2017-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -51,7 +51,6 @@ class DirectConvolutionValidationGenericFixture : public framework::Fixture public: using TBias = typename std::conditional < std::is_same::value || std::is_same::value, int32_t, T >::type; - template void setup(TensorShape input_shape, int stride_x, int stride_y, int pad_x, int pad_y, unsigned int kernel_size, unsigned int num_kernels, DataType data_type, QuantizationInfo quantization_info, ActivationLayerInfo act_info, DataLayout data_layout, bool mixed_layout = false) { @@ -73,7 +72,6 @@ public: _reference = compute_reference(input_shape, weights_shape, bias_shape, output_shape, info, data_type, bias_data_type, quantization_info, act_info); } - template void setup(TensorShape input_shape, TensorShape weights_shape, TensorShape bias_shape, TensorShape output_shape, PadStrideInfo info, Size2D dilation, DataType data_type, QuantizationInfo quantization_info, ActivationLayerInfo act_info, DataLayout data_layout) { @@ -230,7 +228,6 @@ template { public: - template void setup(TensorShape input_shape, int stride_x, int stride_y, int pad_x, int pad_y, unsigned int kernel_size, unsigned int num_kernels, DataType data_type, ActivationLayerInfo act_info, DataLayout data_layout) { @@ -243,7 +240,6 @@ template { public: - template void setup(TensorShape input_shape, int stride_x, int stride_y, int pad_x, int pad_y, unsigned int kernel_size, unsigned int num_kernels, DataType data_type, QuantizationInfo quantization_info, ActivationLayerInfo act_info, DataLayout data_layout) { @@ -256,7 +252,6 @@ template { public: - template void setup(TensorShape input_shape, TensorShape weights_shape, TensorShape bias_shape, TensorShape output_shape, PadStrideInfo info, Size2D dilation, DataType data_type, QuantizationInfo quantization_info, ActivationLayerInfo act_info, DataLayout data_layout) { @@ -269,7 +264,6 @@ template { public: - template void setup(TensorShape input_shape, TensorShape weights_shape, TensorShape bias_shape, TensorShape output_shape, PadStrideInfo info, Size2D dilation, DataType data_type, ActivationLayerInfo act_info) { diff --git a/tests/validation/fixtures/DropoutLayerFixture.h b/tests/validation/fixtures/DropoutLayerFixture.h index 2a91911410b4a50b54c6193d7b8256b0a4be2f5f..a84f2a6407dad91247f109fcbbec2a6e89743fff 100644 --- a/tests/validation/fixtures/DropoutLayerFixture.h +++ b/tests/validation/fixtures/DropoutLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -46,7 +46,6 @@ template void setup(TensorShape shape, float ratio, bool forward, DataType data_type) { _target = compute_target(shape, ratio, forward, data_type); diff --git a/tests/validation/fixtures/ElementwiseOperationsFixture.h b/tests/validation/fixtures/ElementwiseOperationsFixture.h index 6f31a730ae288a7534e6a9099902f6f90697c2e5..b5547b67628b7d1f3075839ba4a031da22c18958 100644 --- a/tests/validation/fixtures/ElementwiseOperationsFixture.h +++ b/tests/validation/fixtures/ElementwiseOperationsFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -46,7 +46,6 @@ template void setup(ArithmeticOperation op, const TensorShape &shape0, const TensorShape &shape1, DataType data_type0, DataType data_type1, DataType output_data_type, QuantizationInfo qinfo0, QuantizationInfo qinfo1, QuantizationInfo qinfo_out, bool is_inplace = false, bool use_dynamic_shape = false) @@ -187,7 +186,6 @@ template { public: - template void setup(ArithmeticOperation op, const TensorShape &shape0, const TensorShape &shape1, DataType data_type0, DataType data_type1, DataType output_data_type, QuantizationInfo qinfo0, QuantizationInfo qinfo1, QuantizationInfo qinfo_out, ActivationLayerInfo act_info, bool is_inplace = true) @@ -277,7 +275,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType data_type0, DataType data_type1, DataType output_data_type, bool is_inplace) { ArithmeticOperationsGenericFixture::setup(ArithmeticOperation::DIV, shape0, shape1, @@ -290,7 +287,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type0, DataType data_type1, DataType output_data_type, bool is_inplace) { ArithmeticOperationsGenericFixture::setup(ArithmeticOperation::DIV, shape, shape, @@ -303,7 +299,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType data_type0, DataType data_type1, DataType output_data_type, bool is_inplace) { ArithmeticOperationsGenericFixture::setup(ArithmeticOperation::DIV, shape0, shape1, @@ -316,7 +311,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type0, DataType data_type1, DataType output_data_type, bool is_inplace) { ArithmeticOperationsGenericFixture::setup(ArithmeticOperation::DIV, shape, shape, @@ -329,7 +323,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType data_type0, DataType data_type1, DataType output_data_type, ActivationLayerInfo act_info, bool is_inplace) { ArithmeticOperationsFuseActivationFixture::setup(ArithmeticOperation::DIV, shape0, shape1, @@ -342,7 +335,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type0, DataType data_type1, DataType output_data_type, ActivationLayerInfo act_info, bool is_inplace) { ArithmeticOperationsFuseActivationFixture::setup(ArithmeticOperation::DIV, shape, shape, @@ -355,7 +347,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type0, DataType data_type1, DataType output_data_type, ActivationLayerInfo act_info, bool is_inplace) { ArithmeticOperationsFuseActivationFixture::setup(ArithmeticOperation::DIV, shape, shape, @@ -368,7 +359,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type0, DataType data_type1, DataType output_data_type, QuantizationInfo qinfo0, QuantizationInfo qinfo1, QuantizationInfo qinfo_out, bool is_inplace) @@ -383,7 +373,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType data_type0, DataType data_type1, DataType output_data_type, bool is_inplace) { ArithmeticOperationsGenericFixture::setup(ArithmeticOperation::MAX, shape0, shape1, @@ -396,7 +385,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type0, DataType data_type1, DataType output_data_type, bool is_inplace) { ArithmeticOperationsGenericFixture::setup(ArithmeticOperation::MAX, shape, shape, @@ -409,7 +397,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType data_type0, DataType data_type1, DataType output_data_type, ActivationLayerInfo act_info, bool is_inplace) { ArithmeticOperationsFuseActivationFixture::setup(ArithmeticOperation::MAX, shape0, shape1, @@ -422,7 +409,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type0, DataType data_type1, DataType output_data_type, ActivationLayerInfo act_info, bool is_inplace) { ArithmeticOperationsFuseActivationFixture::setup(ArithmeticOperation::MAX, shape, shape, @@ -435,7 +421,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type0, DataType data_type1, DataType output_data_type, QuantizationInfo qinfo0, QuantizationInfo qinfo1, QuantizationInfo qinfo_out, bool is_inplace) @@ -450,7 +435,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType data_type0, DataType data_type1, DataType output_data_type, QuantizationInfo qinfo0, QuantizationInfo qinfo1, QuantizationInfo qinfo_out, bool is_inplace) @@ -465,7 +449,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType data_type0, DataType data_type1, DataType output_data_type, bool is_inplace) { ArithmeticOperationsGenericFixture::setup(ArithmeticOperation::MIN, shape0, shape1, @@ -478,7 +461,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type0, DataType data_type1, DataType output_data_type, bool is_inplace) { ArithmeticOperationsGenericFixture::setup(ArithmeticOperation::MIN, shape, shape, @@ -491,7 +473,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType data_type0, DataType data_type1, DataType output_data_type, ActivationLayerInfo act_info, bool is_inplace) { ArithmeticOperationsFuseActivationFixture::setup(ArithmeticOperation::MIN, shape0, shape1, @@ -504,7 +485,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type0, DataType data_type1, DataType output_data_type, ActivationLayerInfo act_info, bool is_inplace) { ArithmeticOperationsFuseActivationFixture::setup(ArithmeticOperation::MIN, shape, shape, @@ -517,7 +497,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type0, DataType data_type1, DataType output_data_type, QuantizationInfo qinfo0, QuantizationInfo qinfo1, QuantizationInfo qinfo_out, bool is_inplace) @@ -532,7 +511,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType data_type0, DataType data_type1, DataType output_data_type, QuantizationInfo qinfo0, QuantizationInfo qinfo1, QuantizationInfo qinfo_out, bool is_inplace) @@ -547,7 +525,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType data_type0, DataType data_type1, DataType output_data_type, bool is_inplace) { ArithmeticOperationsGenericFixture::setup(ArithmeticOperation::SQUARED_DIFF, shape0, shape1, @@ -560,7 +537,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type0, DataType data_type1, DataType output_data_type, bool is_inplace) { ArithmeticOperationsGenericFixture::setup(ArithmeticOperation::SQUARED_DIFF, shape, shape, @@ -573,7 +549,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType data_type0, DataType data_type1, DataType output_data_type, ActivationLayerInfo act_info, bool is_inplace) { ArithmeticOperationsFuseActivationFixture::setup(ArithmeticOperation::SQUARED_DIFF, shape0, shape1, @@ -586,7 +561,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type0, DataType data_type1, DataType output_data_type, ActivationLayerInfo act_info, bool is_inplace) { ArithmeticOperationsFuseActivationFixture::setup(ArithmeticOperation::SQUARED_DIFF, shape, shape, @@ -599,7 +573,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type0, DataType data_type1, DataType output_data_type, QuantizationInfo qinfo0, QuantizationInfo qinfo1, QuantizationInfo qinfo_out, bool is_inplace) @@ -614,7 +587,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType data_type0, DataType data_type1, DataType output_data_type, QuantizationInfo qinfo0, QuantizationInfo qinfo1, QuantizationInfo qinfo_out, bool is_inplace) @@ -629,7 +601,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType data_type0, DataType data_type1, DataType output_data_type) { ArithmeticOperationsGenericFixture::setup(ArithmeticOperation::PRELU, shape0, shape1, @@ -642,7 +613,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type0, DataType data_type1, DataType output_data_type) { ArithmeticOperationsGenericFixture::setup(ArithmeticOperation::PRELU, shape, shape, @@ -655,7 +625,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type0, DataType data_type1, DataType output_data_type, QuantizationInfo qinfo0, QuantizationInfo qinfo1, QuantizationInfo qinfo_out) @@ -670,7 +639,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType data_type0, DataType data_type1, DataType output_data_type, QuantizationInfo qinfo0, QuantizationInfo qinfo1, QuantizationInfo qinfo_out) @@ -685,7 +653,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType data_type0, DataType data_type1, DataType output_data_type, bool is_inplace) { ArithmeticOperationsGenericFixture::setup(ArithmeticOperation::POWER, shape0, shape1, @@ -698,7 +665,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type0, DataType data_type1, DataType output_data_type, bool is_inplace) { ArithmeticOperationsGenericFixture::setup(ArithmeticOperation::POWER, shape, shape, @@ -711,7 +677,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType data_type0, DataType data_type1, DataType output_data_type, ActivationLayerInfo act_info, bool is_inplace) { ArithmeticOperationsFuseActivationFixture::setup(ArithmeticOperation::POWER, shape0, shape1, @@ -724,7 +689,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type0, DataType data_type1, DataType output_data_type, ActivationLayerInfo act_info, bool is_inplace) { ArithmeticOperationsFuseActivationFixture::setup(ArithmeticOperation::POWER, shape, shape, diff --git a/tests/validation/fixtures/ElementwiseUnaryFixture.h b/tests/validation/fixtures/ElementwiseUnaryFixture.h index 9b40d34d2b6f6d56c0e0397448a4ccfb98543477..15344288db6c6435254243a952324d9e6a3cdbab 100644 --- a/tests/validation/fixtures/ElementwiseUnaryFixture.h +++ b/tests/validation/fixtures/ElementwiseUnaryFixture.h @@ -50,7 +50,6 @@ template void setup(TensorShape input_shape, DataType input_data_type, bool in_place, ElementWiseUnary op, bool use_dynamic_shape = false, QuantizationInfo qinfo = QuantizationInfo(), QuantizationInfo qinfo_out = QuantizationInfo()) { @@ -277,7 +276,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type, QuantizationInfo qinfo, QuantizationInfo qinfo_out) { ElementWiseUnaryValidationFixture::setup(shape, data_type, false, ElementWiseUnary::RSQRT, false, qinfo, qinfo_out); @@ -288,7 +286,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type) { ElementWiseUnaryValidationFixture::setup(shape, data_type, false, ElementWiseUnary::RSQRT); @@ -299,7 +296,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type) { ElementWiseUnaryValidationFixture::setup(shape, data_type, false, ElementWiseUnary::RSQRT, true); @@ -310,7 +306,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type) { ElementWiseUnaryValidationFixture::setup(shape, data_type, false, ElementWiseUnary::EXP); @@ -321,7 +316,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type, QuantizationInfo iq, QuantizationInfo oq) { ElementWiseUnaryValidationFixture::setup(shape, data_type, false, ElementWiseUnary::EXP, false, iq, oq); @@ -332,7 +326,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type) { ElementWiseUnaryValidationFixture::setup(shape, data_type, false, ElementWiseUnary::NEG); @@ -343,7 +336,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type, QuantizationInfo iq, QuantizationInfo oq) { ElementWiseUnaryValidationFixture::setup(shape, data_type, false, ElementWiseUnary::NEG, false, iq, oq); @@ -354,7 +346,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type, bool in_place) { ElementWiseUnaryValidationFixture::setup(shape, data_type, in_place, ElementWiseUnary::NEG); @@ -365,7 +356,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type, bool in_place, QuantizationInfo iq, QuantizationInfo oq) { ElementWiseUnaryValidationFixture::setup(shape, data_type, in_place, ElementWiseUnary::NEG, false, iq, oq); @@ -376,7 +366,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type) { ElementWiseUnaryValidationFixture::setup(shape, data_type, false, ElementWiseUnary::LOG); @@ -387,7 +376,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type, QuantizationInfo iq, QuantizationInfo oq) { ElementWiseUnaryValidationFixture::setup(shape, data_type, false, ElementWiseUnary::LOG, false, iq, oq); @@ -398,7 +386,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type) { ElementWiseUnaryValidationFixture::setup(shape, data_type, false, ElementWiseUnary::ABS); @@ -409,7 +396,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type, QuantizationInfo iq, QuantizationInfo oq) { ElementWiseUnaryValidationFixture::setup(shape, data_type, false, ElementWiseUnary::ABS, false, iq, oq); @@ -420,7 +406,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type) { ElementWiseUnaryValidationFixture::setup(shape, data_type, false, ElementWiseUnary::SIN); @@ -431,7 +416,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type, QuantizationInfo iq, QuantizationInfo oq) { ElementWiseUnaryValidationFixture::setup(shape, data_type, false, ElementWiseUnary::SIN, false, iq, oq); @@ -442,7 +426,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type) { ElementWiseUnaryValidationFixture::setup(shape, data_type, false, ElementWiseUnary::ROUND); @@ -453,7 +436,6 @@ template { public: - template void setup(const TensorShape &shape, DataType data_type, QuantizationInfo iq, QuantizationInfo oq) { ElementWiseUnaryValidationFixture::setup(shape, data_type, false, ElementWiseUnary::ROUND, false, iq, oq); diff --git a/tests/validation/fixtures/FFTFixture.h b/tests/validation/fixtures/FFTFixture.h index 16ac212fccc1eaf88ed683a6d9869b989a5b57f6..024227b22ab397fd229faba53c0597d39a6dce8b 100644 --- a/tests/validation/fixtures/FFTFixture.h +++ b/tests/validation/fixtures/FFTFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022 Arm Limited. + * Copyright (c) 2019-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -47,7 +47,6 @@ template void setup(TensorShape shape, DataType data_type) { _target = compute_target(shape, data_type); @@ -134,7 +133,6 @@ template void setup(TensorShape input_shape, TensorShape weights_shape, TensorShape bias_shape, TensorShape output_shape, PadStrideInfo info, Size2D dilation, DataType data_type, DataLayout data_layout, ActivationLayerInfo act_info, bool mixed_layout = false) { @@ -271,7 +269,6 @@ template { public: - template void setup(TensorShape input_shape, TensorShape weights_shape, TensorShape bias_shape, TensorShape output_shape, PadStrideInfo info, Size2D dilation, DataType data_type, DataLayout data_layout, ActivationLayerInfo act_info) { diff --git a/tests/validation/fixtures/FillFixture.h b/tests/validation/fixtures/FillFixture.h index 706c13565d4b158824aa28208fbf712a882c6ab3..0239a6890384dcfb3a69500276b908340054114e 100644 --- a/tests/validation/fixtures/FillFixture.h +++ b/tests/validation/fixtures/FillFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Arm Limited. + * Copyright (c) 2019, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -42,7 +42,6 @@ template void setup(TensorShape input_shape, DataType data_type) { _target = compute_target(input_shape, data_type); diff --git a/tests/validation/fixtures/FlattenLayerFixture.h b/tests/validation/fixtures/FlattenLayerFixture.h index b6c0dde28ac262179d9d994de6de69ab9a74c024..e72487c7cf9491dd925c3343c3a8ca985335e470 100644 --- a/tests/validation/fixtures/FlattenLayerFixture.h +++ b/tests/validation/fixtures/FlattenLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -50,7 +50,6 @@ template void setup(TensorShape shape, DataType data_type) { TensorShape shape_flatten; diff --git a/tests/validation/fixtures/FloorFixture.h b/tests/validation/fixtures/FloorFixture.h index d6e19444dbee44550a30ff95f0d15081e51c051d..7d38666f475542621cc5b3723913aa2d0c065cbb 100644 --- a/tests/validation/fixtures/FloorFixture.h +++ b/tests/validation/fixtures/FloorFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -44,7 +44,6 @@ template void setup(TensorShape shape, DataType data_type) { _target = compute_target(shape, data_type); diff --git a/tests/validation/fixtures/FullyConnectedLayerFixture.h b/tests/validation/fixtures/FullyConnectedLayerFixture.h index 75bef144ad20d068d0c0dc48b2a87f69a5d6f3d0..fb1cb4dcb62f9cbbd892d69b30a5e7f57725dc7e 100644 --- a/tests/validation/fixtures/FullyConnectedLayerFixture.h +++ b/tests/validation/fixtures/FullyConnectedLayerFixture.h @@ -54,7 +54,6 @@ public: using TBias = typename std::conditional < (std::is_same::value || std::is_same::value), int32_t, T >::type; public: - template void setup(TensorShape input_shape, TensorShape weights_shape, TensorShape bias_shape, TensorShape output_shape, bool transpose_weights, bool reshape_weights, DataType data_type, QuantizationInfo quantization_info, ActivationLayerInfo activation_info, bool mixed_layout = false) { @@ -248,7 +247,6 @@ template { public: - template void setup(TensorShape input_shape, TensorShape weights_shape, TensorShape bias_shape, TensorShape output_shape, bool transpose_weights, bool reshape_weights, DataType data_type, ActivationLayerInfo activation_info) { @@ -262,7 +260,6 @@ template { public: - template void setup(TensorShape input_shape, TensorShape weights_shape, TensorShape bias_shape, TensorShape output_shape, bool transpose_weights, bool reshape_weights, DataType data_type, QuantizationInfo quantization_info, ActivationLayerInfo activation_info) { @@ -335,9 +332,9 @@ private: void validate_with_tolerance(TensorType &target, SimpleTensor &ref) { - constexpr AbsoluteTolerance abs_tolerance_f16(0.3f); + constexpr AbsoluteTolerance abs_tolerance_f16(0.3f); const RelativeTolerance rel_tolerance_f16(half_float::half(0.2f)); - constexpr float tolerance_num_f16 = 0.07f; + constexpr float tolerance_num_f16 = 0.07f; validate(AccessorType(target), ref, rel_tolerance_f16, tolerance_num_f16, abs_tolerance_f16); } @@ -358,38 +355,37 @@ public: using TDecay = typename std::decay::type; using TBias = typename std::conditional < (std::is_same::value || std::is_same::value), int32_t, T >::type; - template void setup(TensorShape src_shape, TensorShape weights_shape, TensorShape bias_shape, TensorShape dst_shape, - DataType data_type, ActivationLayerInfo activation_info, bool constant_weights, bool constant_bias, bool weights_reshaped) + DataType data_type, ActivationLayerInfo activation_info, bool constant_weights, bool constant_bias, bool weights_reshaped, bool remove_bias = false) { _data_type = data_type; - const bool is_quantized = is_data_type_quantized(data_type); - + const bool is_quantized = is_data_type_quantized(data_type); const DataType bias_data_type = (is_quantized) ? DataType::S32 : data_type; const QuantizationInfo src_qinfo = is_quantized ? QuantizationInfo(0.1f, 10) : QuantizationInfo(); const QuantizationInfo weights_qinfo = is_quantized ? QuantizationInfo(0.3f, 20) : QuantizationInfo(); const QuantizationInfo dst_qinfo = is_quantized ? QuantizationInfo(0.2f, 5) : QuantizationInfo(); - // Setup tensor meta-data + // Configure TensorInfo Objects const TensorInfo src_info(src_shape, 1, data_type, src_qinfo); - _src.allocator()->init(src_info); + const TensorInfo dst_info(dst_shape, 1, data_type, dst_qinfo); + TensorInfo bias_info(bias_shape, 1, bias_data_type); + TensorInfo wei_info(weights_shape, 1, data_type, weights_qinfo); - TensorInfo wei_info(weights_shape, 1, data_type, weights_qinfo); if(!constant_weights && weights_reshaped) { const TensorShape tr_weights_shape{ weights_shape[1], weights_shape[0] }; wei_info.set_tensor_shape(tr_weights_shape); } wei_info.set_are_values_constant(constant_weights); - _weights.allocator()->init(wei_info); - - TensorInfo bias_info(bias_shape, 1, bias_data_type); bias_info.set_are_values_constant(constant_bias); - _bias.allocator()->init(bias_info); - const TensorInfo dst_info(dst_shape, 1, data_type, dst_qinfo); + // Initialise Tensors + _src.allocator()->init(src_info); + _weights.allocator()->init(wei_info); + if(!remove_bias) + _bias.allocator()->init(bias_info); _dst.allocator()->init(dst_info); // Configure FC layer and mark the weights as non constant @@ -401,12 +397,13 @@ public: fc_info.transpose_weights = !weights_reshaped; } FunctionType fc; - fc.configure(&_src, &_weights, &_bias, &_dst, fc_info); + fc.configure(&_src, &_weights, (remove_bias) ? nullptr : &_bias, &_dst, fc_info); // Allocate all the tensors _src.allocator()->allocate(); _weights.allocator()->allocate(); - _bias.allocator()->allocate(); + if(!remove_bias) + _bias.allocator()->allocate(); _dst.allocator()->allocate(); // Run multiple iterations with different inputs @@ -424,11 +421,20 @@ public: fill(AccessorType(_weights), 1); fill(weights, 1); } - if(constant_bias) + if(constant_bias && !remove_bias) { fill(AccessorType(_bias), 2); fill(bias, 2); } + // To remove bias, fill with 0 + if(remove_bias && is_quantized) + { + library->fill_tensor_value(bias, 0); + } + else if(remove_bias) + { + library->fill_tensor_value(bias, (float)0.0); + } for(int i = 0; i < num_iterations; ++i) { @@ -446,7 +452,7 @@ public: fill(AccessorType(_weights), randomizer_offset + 1); } } - if(!constant_bias) + if(!constant_bias && !remove_bias) { fill(AccessorType(_bias), randomizer_offset + 2); } @@ -462,7 +468,7 @@ public: { fill(weights, randomizer_offset + 1); } - if(!constant_bias) + if(!constant_bias && !remove_bias) { fill(bias, randomizer_offset + 2); } @@ -486,12 +492,23 @@ template { public: - template void setup(TensorShape src_shape, TensorShape weights_shape, TensorShape bias_shape, TensorShape dst_shape, DataType data_type, ActivationLayerInfo activation_info, bool weights_reshaped) { FullyConnectedWithDynamicTensorsFixture::setup(src_shape, weights_shape, bias_shape, - dst_shape, data_type, activation_info, false, true, weights_reshaped); + dst_shape, data_type, activation_info, false, true, weights_reshaped, false); + } +}; + +template +class FullyConnectedDynamicNoBiasFixture : public FullyConnectedWithDynamicTensorsFixture +{ +public: + void setup(TensorShape src_shape, TensorShape weights_shape, TensorShape bias_shape, TensorShape dst_shape, + DataType data_type, ActivationLayerInfo activation_info, bool weights_reshaped) + { + FullyConnectedWithDynamicTensorsFixture::setup(src_shape, weights_shape, bias_shape, + dst_shape, data_type, activation_info, false, true, weights_reshaped, true); } }; @@ -499,12 +516,11 @@ template { public: - template void setup(TensorShape src_shape, TensorShape weights_shape, TensorShape bias_shape, TensorShape dst_shape, DataType data_type, ActivationLayerInfo activation_info) { FullyConnectedWithDynamicTensorsFixture::setup(src_shape, weights_shape, bias_shape, - dst_shape, data_type, activation_info, true, false, false /* weights_reshaped (not used) */); + dst_shape, data_type, activation_info, true, false, false, false /* weights_reshaped (not used) */); } }; } // namespace validation diff --git a/tests/validation/fixtures/FuseBatchNormalizationFixture.h b/tests/validation/fixtures/FuseBatchNormalizationFixture.h index 6fbabeee56b98ec1bbf927c3c3982387a147fb36..a05e4169a7ae297187627dd5225199f9ad598035 100644 --- a/tests/validation/fixtures/FuseBatchNormalizationFixture.h +++ b/tests/validation/fixtures/FuseBatchNormalizationFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -47,7 +47,6 @@ template void setup(TensorShape shape_w, DataType data_type, DataLayout data_layout, bool in_place, bool with_bias, bool with_gamma, bool with_beta) { std::tie(_target_w, _target_b) = compute_target(shape_w, data_type, data_layout, in_place, with_bias, with_gamma, with_beta); diff --git a/tests/validation/fixtures/GEMMFixture.h b/tests/validation/fixtures/GEMMFixture.h index 5dc27117532b1590b2facf446d18e7e714bd54bd..f1e0ee9150f5fd43737b191dc04cf479bba43955 100644 --- a/tests/validation/fixtures/GEMMFixture.h +++ b/tests/validation/fixtures/GEMMFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2022 Arm Limited. + * Copyright (c) 2017-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -52,7 +52,6 @@ template void setup(TensorShape shape_a, TensorShape shape_b, TensorShape shape_c, TensorShape output_shape, float alpha, float beta, bool pretranspose, DataType data_type) { ARM_COMPUTE_UNUSED(pretranspose); @@ -230,7 +229,6 @@ template void setup(unsigned int m, unsigned int n, unsigned int k, unsigned int batch_size, float alpha, float beta, bool broadcast_bias, bool fp16_mixed_precision, const ActivationLayerInfo &act_info, DataType data_type, GPUTarget gpu_arch) { @@ -354,7 +352,6 @@ template void setup(unsigned int m_w, unsigned int m_h, unsigned int n, unsigned int k, unsigned int batch_size, float alpha, float beta, bool broadcast_bias, bool fp16_mixed_precision, const ActivationLayerInfo &act_info, DataType data_type, GPUTarget gpu_arch) { @@ -476,7 +473,6 @@ template void setup(unsigned int m, unsigned int n, unsigned int k, unsigned int batch_size, float alpha, float beta, unsigned int v0, unsigned int h0, bool broadcast_bias, bool fp16_mixed_precision, const ActivationLayerInfo &act_info, DataType data_type, GPUTarget gpu_arch) { @@ -632,7 +628,6 @@ template void setup(unsigned int m_w, unsigned int m_h, unsigned int n, unsigned int k, unsigned int batch_size, float alpha, float beta, unsigned int v0, unsigned int h0, bool broadcast_bias, bool fp16_mixed_precision, const ActivationLayerInfo &act_info, DataType data_type, GPUTarget gpu_arch) { @@ -786,7 +781,6 @@ template void setup(unsigned int m, unsigned int n, unsigned int k, unsigned int batch_size, unsigned int m0, unsigned int n0, unsigned int k0, unsigned int v0, unsigned int h0, bool interleave_lhs, bool interleave_rhs, bool export_to_cl_image, DataType data_type, float alpha, float beta, bool broadcast_bias, bool lhs_transpose, const ActivationLayerInfo &act_info) { @@ -973,7 +967,6 @@ class GEMMMatrixMultiplyReshapedWithPostOpsValidationFixture : public framework: public: using PostOpArgBroadcast = std::tuple; // Instruct fixture if we need broadcasting in dimension 0, 1, 2 of each PostOp argument public: - template void setup(unsigned int m, unsigned int n, unsigned int k, unsigned int batch_size, unsigned int m0, unsigned int n0, unsigned int k0, unsigned int v0, unsigned int h0, bool interleave_lhs, bool interleave_rhs, bool export_to_cl_image, DataType data_type, float alpha, float beta, bool broadcast_bias, bool lhs_transpose, const ActivationLayerInfo &act_info, const experimental::PostOpList &post_ops) @@ -1227,7 +1220,6 @@ template void setup(unsigned int m_w, unsigned int m_h, unsigned int n, unsigned int k, unsigned int batch_size, unsigned int m0, unsigned int n0, unsigned int k0, unsigned int v0, unsigned int h0, bool interleave_lhs, bool interleave_rhs, bool export_to_cl_image, DataType data_type, float alpha, float beta, bool lhs_transpose, const ActivationLayerInfo &act_info) { @@ -1407,7 +1399,6 @@ template void setup(unsigned int m, unsigned int n, unsigned int k, unsigned int batch_size, unsigned int m0, unsigned int n0, unsigned int k0, unsigned int h0, bool interleave_rhs, bool transpose_rhs, bool export_to_cl_image, DataType data_type, float alpha, float beta, bool broadcast_bias, const ActivationLayerInfo &act_info) { @@ -1575,7 +1566,6 @@ class GEMMMatrixMultiplyReshapedOnlyRHSWithPostOpsValidationFixture : public fra { public: using PostOpArgBroadcast = std::tuple; // Instruct fixture if we need broadcasting in dimension 0, 1, 2 of each PostOp argument - template void setup(unsigned int m, unsigned int n, unsigned int k, unsigned int batch_size, unsigned int m0, unsigned int n0, unsigned int k0, unsigned int h0, bool interleave_rhs, bool transpose_rhs, bool export_to_cl_image, DataType data_type, float alpha, float beta, bool broadcast_bias, const ActivationLayerInfo &act_info, const experimental::PostOpList &post_ops) @@ -1810,7 +1800,6 @@ template void setup(unsigned int m_w, unsigned int m_h, unsigned int n, unsigned int k, unsigned int batch_size, unsigned int m0, unsigned int n0, unsigned int k0, unsigned int h0, bool interleave_rhs, bool transpose_rhs, bool export_to_cl_image, bool has_pad_y, DataType data_type, float alpha, float beta, const ActivationLayerInfo &act_info) { @@ -1980,7 +1969,6 @@ template void setup(unsigned int m, unsigned int n, unsigned int k, unsigned int batch_size, unsigned int m0, unsigned int n0, unsigned int k0, DataType data_type, float alpha, float beta, bool broadcast_bias, const ActivationLayerInfo &act_info) { @@ -2119,7 +2107,6 @@ class GEMMMatrixMultiplyNativeWithPostOpsValidationFixture : public framework::F public: using PostOpArgBroadcast = std::tuple; // Instruct fixture if we need broadcasting in dimension 0, 1, 2 of each PostOp argument public: - template void setup(unsigned int m, unsigned int n, unsigned int k, unsigned int batch_size, unsigned int m0, unsigned int n0, unsigned int k0, DataType data_type, float alpha, float beta, bool broadcast_bias, const ActivationLayerInfo &act_info, const experimental::PostOpList &post_ops) { @@ -2324,7 +2311,6 @@ template void setup(unsigned int m_w, unsigned int m_h, unsigned int n, unsigned int k, unsigned int batch_size, unsigned int m0, unsigned int n0, unsigned int k0, DataType data_type, float alpha, float beta, const ActivationLayerInfo &act_info) { @@ -2459,7 +2445,6 @@ template void setup(unsigned int m, unsigned int n, unsigned int k, unsigned int batch_size, unsigned int m0, unsigned int n0, unsigned int k0, bool export_to_cl_image, DataType data_type, float alpha, float beta, bool broadcast_bias, const ActivationLayerInfo &act_info) diff --git a/tests/validation/fixtures/GEMMInterleave4x4Fixture.h b/tests/validation/fixtures/GEMMInterleave4x4Fixture.h index 72cea8c27d4c43b2893860d53b10ac00646014ad..59fc4608694685b6a47eb52849d75db05b64ff84 100644 --- a/tests/validation/fixtures/GEMMInterleave4x4Fixture.h +++ b/tests/validation/fixtures/GEMMInterleave4x4Fixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -46,7 +46,6 @@ template void setup(size_t x, size_t y, DataType data_type) { _data_type = data_type; diff --git a/tests/validation/fixtures/GEMMLowpFixture.h b/tests/validation/fixtures/GEMMLowpFixture.h index 043fc9cb3cd5aa6a95d93b39b95c993ee4743906..1492ac694595ed6baad2633777aa6df0b65ef25d 100644 --- a/tests/validation/fixtures/GEMMLowpFixture.h +++ b/tests/validation/fixtures/GEMMLowpFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2022 Arm Limited. + * Copyright (c) 2017-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -231,7 +231,6 @@ template void setup(TensorShape shape_a, TensorShape shape_b, TensorShape shape_output, int32_t a_offset, int32_t b_offset) { _target = compute_target(shape_a, shape_b, shape_output, a_offset, b_offset); @@ -258,7 +257,6 @@ template void setup(TensorShape shape_a, TensorShape shape_b, TensorShape shape_output, int32_t a_offset, int32_t b_offset, GEMMLowpOutputStageInfo output_stage, DataType data_type_b, bool reshape_b_only_on_first_run) { @@ -332,7 +330,6 @@ class GEMMLowpMatrixMultiplyCoreFusedOffsetOutputValidationFixture : public GEMMLowpMatrixMultiplyCoreFusedOffsetOutputGenericValidationFixture { public: - template void setup(TensorShape shape_a, TensorShape shape_b, TensorShape shape_output, int32_t a_offset, int32_t b_offset, GEMMLowpOutputStageInfo output_stage, DataType data_type_b) { GEMMLowpMatrixMultiplyCoreFusedOffsetOutputGenericValidationFixture::setup(shape_a, shape_b, @@ -344,7 +341,6 @@ template class GEMMLowpQuantizeDownInt32ToUint8ScaleValidationFixture : public framework::Fixture { public: - template void setup(TensorShape shape, int32_t result_offset, int32_t result_mult_int, int32_t result_shift, int32_t min, int32_t max, bool add_bias) { _target = compute_target(shape, result_offset, result_mult_int, result_shift, min, max, add_bias); @@ -446,7 +442,6 @@ template class GEMMLowpQuantizeDownInt32ToInt8ScaleValidationFixture : public framework::Fixture { public: - template void setup(TensorShape shape, int32_t result_offset, int32_t result_mult_int, int32_t result_shift, int32_t min, int32_t max, bool add_bias) { _target = compute_target(shape, result_offset, result_mult_int, result_shift, min, max, add_bias); @@ -548,7 +543,6 @@ template class GEMMLowpQuantizeDownInt32ToInt8ScaleByFixedPointValidationFixture : public framework::Fixture { public: - template void setup(TensorShape shape, int32_t result_fixedpoint_multiplier, int32_t result_shift, int32_t result_offset_after_shift, int32_t min, int32_t max, bool add_bias) { _target = compute_target(shape, result_fixedpoint_multiplier, result_shift, result_offset_after_shift, min, max, add_bias); @@ -643,7 +637,6 @@ template class GEMMLowpQuantizeDownInt32ToUint8ScaleByFixedPointValidationFixture : public framework::Fixture { public: - template void setup(TensorShape shape, int32_t result_fixedpoint_multiplier, int32_t result_shift, int32_t result_offset_after_shift, int32_t min, int32_t max, bool add_bias) { _target = compute_target(shape, result_fixedpoint_multiplier, result_shift, result_offset_after_shift, min, max, add_bias); @@ -738,7 +731,6 @@ template void setup(DataType data_type, TensorShape shape, float result_real_multiplier, int32_t result_offset, int32_t min, int32_t max, bool add_bias) { _target = compute_target(data_type, shape, result_real_multiplier, result_offset, min, max, add_bias); @@ -841,7 +833,6 @@ template class GEMMLowpQuantizeDownInt32ToInt16ScaleByFixedPointValidationFixture : public framework::Fixture { public: - template void setup(TensorShape shape, int32_t result_fixedpoint_multiplier, int32_t result_shift, int32_t min, int32_t max, bool add_bias) { _target = compute_target(shape, result_fixedpoint_multiplier, result_shift, min, max, add_bias); @@ -936,7 +927,6 @@ template void setup(unsigned int m, unsigned int n, unsigned int k, unsigned int batch_size, unsigned int m0, unsigned int n0, unsigned int k0, unsigned int v0, unsigned int h0, bool interleave_lhs, bool interleave_rhs, DataType data_type) { @@ -1087,7 +1077,6 @@ template void setup(unsigned int m_w, unsigned int m_h, unsigned int n, unsigned int k, unsigned int batch_size, unsigned int m0, unsigned int n0, unsigned int k0, unsigned int v0, unsigned int h0, bool interleave_lhs, bool interleave_rhs, DataType data_type) { @@ -1244,7 +1233,6 @@ template void setup(unsigned int m, unsigned int n, unsigned int k, unsigned int batch_size, unsigned int m0, unsigned int n0, unsigned int k0, unsigned int h0, bool interleave_rhs, bool transpose_rhs, DataType data_type) { @@ -1387,7 +1375,6 @@ template void setup(unsigned int m, unsigned int n, unsigned int k, unsigned int batch_size, unsigned int m0, unsigned int n0, unsigned int k0, unsigned int h0, bool interleave_rhs, bool transpose_rhs, bool broadcast_bias, DataType data_type) { @@ -1595,7 +1582,6 @@ template void setup(unsigned int m, unsigned int n, unsigned int k, unsigned int batch_size, unsigned int m0, unsigned int n0, unsigned int k0, unsigned int h0, bool interleave_rhs, bool transpose_rhs, DataType data_type) { @@ -1751,7 +1737,6 @@ template void setup(unsigned int m_w, unsigned int m_h, unsigned int n, unsigned int k, unsigned int batch_size, unsigned int m0, unsigned int n0, unsigned int k0, unsigned int h0, bool interleave_rhs, bool transpose_rhs, DataType data_type) { @@ -1900,7 +1885,6 @@ template class GEMMLowpMatrixMultiplyNativeValidationFixture : public framework::Fixture { public: - template void setup(unsigned int m, unsigned int n, unsigned int k, unsigned int batch_size, unsigned int m0, unsigned int n0, unsigned int k0) { GEMMLHSMatrixInfo lhs_info; @@ -1995,7 +1979,6 @@ template class GEMMLowpMatrixMultiplyNative3DValidationFixture : public framework::Fixture { public: - template void setup(unsigned int m_w, unsigned int m_h, unsigned int n, unsigned int k, unsigned int batch_size, unsigned int m0, unsigned int n0, unsigned int k0) { GEMMLHSMatrixInfo lhs_info; diff --git a/tests/validation/fixtures/GEMMReshapeLHSMatrixFixture.h b/tests/validation/fixtures/GEMMReshapeLHSMatrixFixture.h index a9d6c9b6aae4ae468509a6ec41668b466fd1e619..d88029f93e72dcab036f884d95f83af3e09accab 100644 --- a/tests/validation/fixtures/GEMMReshapeLHSMatrixFixture.h +++ b/tests/validation/fixtures/GEMMReshapeLHSMatrixFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -50,7 +50,6 @@ template void setup(TensorShape shape_in, unsigned int batch_size, DataType data_type, unsigned int m0, unsigned int k0, unsigned int v0, bool interleave, bool transpose) { GEMMLHSMatrixInfo lhs_info; diff --git a/tests/validation/fixtures/GEMMReshapeRHSMatrixFixture.h b/tests/validation/fixtures/GEMMReshapeRHSMatrixFixture.h index cdb3ec3944ac79ee60b8b2bca3f0a1690831a69a..0929faf04a7a2a20e1fdfa7e68c7a461749dc4d3 100644 --- a/tests/validation/fixtures/GEMMReshapeRHSMatrixFixture.h +++ b/tests/validation/fixtures/GEMMReshapeRHSMatrixFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -50,7 +50,6 @@ template void setup(TensorShape shape_in, unsigned int batch_size, DataType data_type, unsigned int n0, unsigned int k0, unsigned int h0, bool interleave, bool transpose) { GEMMRHSMatrixInfo rhs_info; diff --git a/tests/validation/fixtures/GEMMTranspose1xWFixture.h b/tests/validation/fixtures/GEMMTranspose1xWFixture.h index 08f478fe2be81acfcd26daa0e7e6e92be1998912..3765515b570e04e708841b7a23e43895444bbc07 100644 --- a/tests/validation/fixtures/GEMMTranspose1xWFixture.h +++ b/tests/validation/fixtures/GEMMTranspose1xWFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -46,7 +46,6 @@ template void setup(size_t x, size_t y, DataType data_type) { _data_type = data_type; diff --git a/tests/validation/fixtures/GatherFixture.h b/tests/validation/fixtures/GatherFixture.h index b28f93d8508aac3b26b34fda6c38b717e55c8b94..857b0387b7fee28b2f8b5937a644691032cdd3ea 100644 --- a/tests/validation/fixtures/GatherFixture.h +++ b/tests/validation/fixtures/GatherFixture.h @@ -47,7 +47,6 @@ template void setup(TensorShape input_shape, TensorShape indices_shape, int axis, DataType data_type) { _target = compute_target(input_shape, data_type, axis, indices_shape); diff --git a/tests/validation/fixtures/Im2ColFixture.h b/tests/validation/fixtures/Im2ColFixture.h index a0732c3eb381783d19d4fb22525e2f068172053e..5c7978f4abe924ea7bb2b9533162bcb524ac07c6 100644 --- a/tests/validation/fixtures/Im2ColFixture.h +++ b/tests/validation/fixtures/Im2ColFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -48,7 +48,6 @@ template void setup(TensorShape input_shape, DataType data_type, const Size2D &kernel_dims, const PadStrideInfo &conv_info, const QuantizationInfo &quant_info, const DataLayout &data_layout, unsigned int num_groups) { diff --git a/tests/validation/fixtures/IndirectConv2dAddressPrecalculationFixture.h b/tests/validation/fixtures/IndirectConv2dAddressPrecalculationFixture.h index f595a8663d8a5c1537aecafa03e05d5bca20543a..7374093f514ac59e842ea2141dde51d9437210cb 100644 --- a/tests/validation/fixtures/IndirectConv2dAddressPrecalculationFixture.h +++ b/tests/validation/fixtures/IndirectConv2dAddressPrecalculationFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -44,7 +44,6 @@ template class IndirectConv2dAddressPrecalculationValidationFixture : public framework::Fixture { public: - template void setup(unsigned int src_w, unsigned int src_h, unsigned int src_b, diff --git a/tests/validation/fixtures/InstanceNormalizationLayerFixture.h b/tests/validation/fixtures/InstanceNormalizationLayerFixture.h index 0f8c15527738f1a28753f32cbe03b833aea1faad..c26dd99f028fded87f5c4b108cde4f955b2d2c5c 100644 --- a/tests/validation/fixtures/InstanceNormalizationLayerFixture.h +++ b/tests/validation/fixtures/InstanceNormalizationLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -44,7 +44,6 @@ template void setup(TensorShape shape, DataType data_type, DataLayout data_layout, bool in_place) { _target = compute_target(shape, data_type, data_layout, in_place); diff --git a/tests/validation/fixtures/L2NormalizeLayerFixture.h b/tests/validation/fixtures/L2NormalizeLayerFixture.h index 2dea66c98b37a928a49ded0d9677cc53beb737e1..b8f4b1eaf3469f884563772ab82fa5be2b3973d8 100644 --- a/tests/validation/fixtures/L2NormalizeLayerFixture.h +++ b/tests/validation/fixtures/L2NormalizeLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -48,7 +48,6 @@ template void setup(TensorShape shape, DataType data_type, DataLayout data_layout, int axis, float epsilon) { _target = compute_target(shape, data_type, data_layout, axis, epsilon); diff --git a/tests/validation/fixtures/LSTMLayerFixture.h b/tests/validation/fixtures/LSTMLayerFixture.h index c3a54726dee3703a297888ddba308eb6780807b8..a32e9adfe5a1d92f53cadb3bbec7a7d2b5c787e2 100644 --- a/tests/validation/fixtures/LSTMLayerFixture.h +++ b/tests/validation/fixtures/LSTMLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022 Arm Limited. + * Copyright (c) 2018-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -46,7 +46,6 @@ template void setup(TensorShape input_shape, TensorShape input_weights_shape, TensorShape recurrent_weights_shape, TensorShape cell_bias_shape, TensorShape output_cell_shape, TensorShape output_shape, TensorShape scratch_shape, ActivationLayerInfo info, float cell_threshold, float projection_threshold, DataType data_type, bool projection_opt, bool peephole_opt, bool use_layer_norm) diff --git a/tests/validation/fixtures/LogicalFixture.h b/tests/validation/fixtures/LogicalFixture.h index c7dd1f6510c8acc2cb133bbb117a06c9eb392b5c..60dc963ba7f3c078180b7d5603731cd7723f759c 100644 --- a/tests/validation/fixtures/LogicalFixture.h +++ b/tests/validation/fixtures/LogicalFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021 Arm Limited. + * Copyright (c) 2020-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -79,7 +79,6 @@ class LogicalBinaryOperationValidationFixture : public LogicalOperationValidatio using Parent = LogicalOperationValidationFixtureBase; public: - template void setup(TensorShape shape0, TensorShape shape1) { Parent::_target = compute_target(shape0, shape1); @@ -135,7 +134,6 @@ class LogicalNotValidationFixture : public LogicalOperationValidationFixtureBase using Parent = LogicalOperationValidationFixtureBase; public: - template void setup(TensorShape shape, DataType data_type) { Parent::_target = compute_target(shape, data_type); diff --git a/tests/validation/fixtures/MatMulFixture.h b/tests/validation/fixtures/MatMulFixture.h index 2f94c1f9d2ace2788762a5a33615f5e8610a6982..e6b28f5766fe0cdd894d53a7602c6e635340beeb 100644 --- a/tests/validation/fixtures/MatMulFixture.h +++ b/tests/validation/fixtures/MatMulFixture.h @@ -48,7 +48,6 @@ template void setup(TensorShape shape_a, TensorShape shape_b, TensorShape output_shape, bool transpose_a, bool transpose_b, DataType data_type, ActivationLayerInfo act_info, int num_extra_runs, Settings settings, QuantizationInfo a_qinfo = QuantizationInfo(), QuantizationInfo b_qinfo = QuantizationInfo(), QuantizationInfo o_qinfo = QuantizationInfo()) { @@ -112,14 +111,14 @@ protected: // Configure MatMulInfo class MatMulInfo mm_info; - mm_info.adj_lhs(transpose_a).adj_rhs(transpose_b).fused_activation(act_info); + mm_info.adj_lhs(transpose_a).adj_rhs(transpose_b); // Ensure values are dynamic a.info()->set_are_values_constant(false); b.info()->set_are_values_constant(false); // Configure operator - matmul.configure(&a, &b, &dst, mm_info, settings); + matmul.configure(&a, &b, &dst, mm_info, settings, act_info); // Assertions ARM_COMPUTE_ASSERT(a.info()->is_resizable()); @@ -162,8 +161,8 @@ protected: } template - typename std::enable_if::value, SimpleTensor>::type - compute_reference_gemm(const SimpleTensor &a, const SimpleTensor &b, const SimpleTensor &c, float alpha, float beta, const QuantizationInfo &o_qinfo) + typename std::enable_if < !std::is_integral::value, SimpleTensor>::type + compute_reference_gemm(const SimpleTensor &a, const SimpleTensor &b, const SimpleTensor &c, float alpha, float beta, const QuantizationInfo &o_qinfo) { ARM_COMPUTE_UNUSED(o_qinfo); @@ -172,7 +171,7 @@ protected: template typename std::enable_if::value, SimpleTensor>::type - compute_reference_gemm(const SimpleTensor &a, const SimpleTensor &b, const SimpleTensor &c, float alpha, float beta, const QuantizationInfo &o_qinfo) + compute_reference_gemm(const SimpleTensor &a, const SimpleTensor &b, const SimpleTensor &c, float alpha, float beta, const QuantizationInfo &o_qinfo) { ARM_COMPUTE_UNUSED(alpha, beta); @@ -183,18 +182,18 @@ protected: const auto multiplier = aq.scale * bq.scale / oq.scale; int32_t output_multiplier = 0; - int32_t output_shift = 0; + int32_t output_shift = 0; quantization::calculate_quantized_multiplier(multiplier, &output_multiplier, &output_shift); std::vector output_multipliers{ output_multiplier }; std::vector output_shifts{ output_shift }; //The lhs and rhs offsets are negated here to keep the reference aligned with the function implementation where the lhs and rhs offsets are also negated. const auto tmp = reference::gemmlowp_matrix_multiply_core( - a, b, c.shape(), -aq.offset, -bq.offset); + a, b, c.shape(), -aq.offset, -bq.offset); auto output = reference::gemmlowp_quantize_down_scale_by_fixedpoint( - tmp, output_multipliers, output_shifts, oq.offset, - std::numeric_limits::lowest(), std::numeric_limits::max()); + tmp, output_multipliers, output_shifts, oq.offset, + std::numeric_limits::lowest(), std::numeric_limits::max()); output.quantization_info(o_qinfo); return output; @@ -271,7 +270,6 @@ template { public: - template void setup(TensorShape shape_a, TensorShape shape_b, TensorShape output_shape, bool transpose_a, bool transpose_b, DataType data_type) { MatMulGenericValidationFixture::setup(shape_a, shape_b, output_shape, transpose_a, transpose_b, data_type, ActivationLayerInfo(), 0, @@ -279,11 +277,32 @@ public: } }; +template +class MatMulValidationWithDynamicTensorsFixture : public MatMulGenericValidationFixture +{ +public: + void setup(TensorShape shape_a, TensorShape shape_b, TensorShape output_shape, bool transpose_a, bool transpose_b, DataType data_type, ActivationLayerInfo act_info, int num_extra_runs) + { + MatMulGenericValidationFixture::setup(shape_a, shape_b, output_shape, transpose_a, transpose_b, data_type, act_info, num_extra_runs, Settings()); + } +}; + +template +class QuantizedMatMulValidationFixture : public MatMulGenericValidationFixture +{ +public: + void setup(TensorShape shape_a, TensorShape shape_b, TensorShape output_shape, bool transpose_a, bool transpose_b, DataType data_type, ActivationLayerInfo act_info, int num_extra_runs, + QuantizationInfo a_qinfo, QuantizationInfo b_qinfo, QuantizationInfo o_qinfo) + { + MatMulGenericValidationFixture::setup(shape_a, shape_b, output_shape, transpose_a, transpose_b, data_type, act_info, num_extra_runs, Settings(), + a_qinfo, b_qinfo, o_qinfo); + } +}; + template class MatMulValidationWithActivationFixture : public MatMulGenericValidationFixture { public: - template void setup(TensorShape shape_a, TensorShape shape_b, TensorShape output_shape, bool transpose_a, bool transpose_b, DataType data_type, ActivationLayerInfo act_info) { MatMulGenericValidationFixture::setup(shape_a, shape_b, output_shape, transpose_a, transpose_b, data_type, act_info, 0, Settings()); @@ -291,24 +310,28 @@ public: }; template -class MatMulValidationWithDynamicTensorsFixture : public MatMulGenericValidationFixture +class MatMulValidationWithActivationAlphaBetaFixture : public MatMulGenericValidationFixture { public: - template - void setup(TensorShape shape_a, TensorShape shape_b, TensorShape output_shape, bool transpose_a, bool transpose_b, DataType data_type, ActivationLayerInfo act_info, int num_extra_runs) + void setup(TensorShape shape_a, TensorShape shape_b, TensorShape output_shape, bool transpose_a, bool transpose_b, DataType data_type, ActivationLayerInfo::ActivationFunction function, + float alpha_beta) { - MatMulGenericValidationFixture::setup(shape_a, shape_b, output_shape, transpose_a, transpose_b, data_type, act_info, num_extra_runs, Settings()); + ActivationLayerInfo act_info(function, alpha_beta, alpha_beta); + MatMulGenericValidationFixture::setup(shape_a, shape_b, output_shape, transpose_a, transpose_b, data_type, act_info, 0, Settings()); } }; template -class QuantizedMatMulValidationFixture : public MatMulGenericValidationFixture +class QuantizedMatMulValidationWithActivationFixture : public MatMulGenericValidationFixture { public: - template - void setup(TensorShape shape_a, TensorShape shape_b, TensorShape output_shape, bool transpose_a, bool transpose_b, DataType data_type, ActivationLayerInfo act_info, int num_extra_runs, QuantizationInfo a_qinfo, QuantizationInfo b_qinfo, QuantizationInfo o_qinfo) + void setup(TensorShape shape_a, TensorShape shape_b, TensorShape output_shape, bool transpose_a, bool transpose_b, DataType data_type, ActivationLayerInfo::ActivationFunction function, + float alpha_beta, int num_extra_runs, + QuantizationInfo a_qinfo, QuantizationInfo b_qinfo, QuantizationInfo o_qinfo) { - MatMulGenericValidationFixture::setup(shape_a, shape_b, output_shape, transpose_a, transpose_b, data_type, act_info, num_extra_runs, Settings(), a_qinfo, b_qinfo, o_qinfo); + ActivationLayerInfo act_info(function, alpha_beta, alpha_beta); + MatMulGenericValidationFixture::setup(shape_a, shape_b, output_shape, transpose_a, transpose_b, data_type, act_info, num_extra_runs, Settings(), + a_qinfo, b_qinfo, o_qinfo); } }; diff --git a/tests/validation/fixtures/MatMulKernelFixture.h b/tests/validation/fixtures/MatMulKernelFixture.h index 7d0b1a40a9b826cd3856b1450d3c2f1d77cac653..91ac77d5af0487a58e07a51b9851a31973a61678 100644 --- a/tests/validation/fixtures/MatMulKernelFixture.h +++ b/tests/validation/fixtures/MatMulKernelFixture.h @@ -36,7 +36,7 @@ #include "tests/validation/reference/GEMMLowp.h" #include "tests/validation/reference/Permute.h" #include "tests/validation/reference/ReshapeLayer.h" - +#include #include namespace arm_compute @@ -47,13 +47,16 @@ namespace validation { using namespace arm_compute::opencl::kernels; -template -class MatMulKernelValidationFixture : public framework::Fixture +template +class MatMulKernelGenericValidationFixture : public framework::Fixture { public: - template - void setup(TensorShape shape_a, TensorShape shape_b, TensorShape output_shape, bool pretranspose_a, bool pretranspose_b, int M0, int N0, int K0, bool export_rhs_to_cl_image, DataType data_type) + void setup(TensorShape shape_a, TensorShape shape_b, TensorShape output_shape, bool pretranspose_a, bool pretranspose_b, int M0, int N0, int K0, bool export_rhs_to_cl_image, DataType data_type, + bool enable_bias) { + // Flag to create a bias + _enable_bias = enable_bias; + // For brevity, the input shapes are assumed to be not-transposed for both Lhs and Rhs matrices. QuantizationInfo lhs_q_info; QuantizationInfo rhs_q_info; @@ -94,13 +97,25 @@ public: permute(shape_b, PermutationVector(1U, 0U)); } + // Skip configurations unsupported by the device. _device_supports_export_to_cl_image = image2d_from_buffer_supported(CLKernelLibrary::get().get_device()); + if(!_device_supports_export_to_cl_image && export_rhs_to_cl_image) + { + ARM_COMPUTE_TEST_INFO("cl_khr_image2d_from_buffer not supported. TEST skipped"); + framework::ARM_COMPUTE_PRINT_INFO(); + return; // Note: Also need to skip the validate in corresponding FIXTURE_DATA_TEST_CASEs. + } - if(!export_rhs_to_cl_image || _device_supports_export_to_cl_image) + _device_supports_mmul = arm_matrix_multiply_supported(CLKernelLibrary::get().get_device()); + if(!_device_supports_mmul && use_mmul) { - _target = compute_target(shape_a, shape_b, output_shape, pretranspose_a, pretranspose_b, M0, N0, K0, export_rhs_to_cl_image, data_type, lhs_q_info, rhs_q_info, dst_q_info); - _reference = compute_reference(shape_a, shape_b, output_shape, pretranspose_a, pretranspose_b, data_type, lhs_q_info, rhs_q_info, dst_q_info); + ARM_COMPUTE_TEST_INFO("cl_arm_matrix_multiply not supported. TEST skipped"); + framework::ARM_COMPUTE_PRINT_INFO(); + return; // Note: Also need to skip the validate in corresponding FIXTURE_DATA_TEST_CASEs. } + + _target = compute_target(shape_a, shape_b, output_shape, pretranspose_a, pretranspose_b, M0, N0, K0, export_rhs_to_cl_image, data_type, lhs_q_info, rhs_q_info, dst_q_info); + _reference = compute_reference(shape_a, shape_b, output_shape, pretranspose_a, pretranspose_b, data_type, lhs_q_info, rhs_q_info, dst_q_info); } protected: @@ -126,6 +141,16 @@ protected: } } + template + void fill_bias_s32(U &&tensor, int i, const UniformQuantizationInfo &q_info) + { + // For quantized cases, fill the S32 bias according to the following to avoid saturation of test cases. + // The following code limits size of bias values to within expected range of output quantization. + const unsigned int bound = std::abs(q_info.scale * 256); // 256 is size of 8 bit datatype + std::uniform_int_distribution distribution(-(bound / 10), (bound / 10)); + library->fill(tensor, distribution, i); + } + template void fill_constant(U &&tensor, D value) { @@ -144,12 +169,15 @@ protected: matmul_info.k0 = K0; matmul_info.export_rhs_to_cl_image = export_rhs_to_cl_image; + bool is_quantized = is_data_type_quantized(data_type); + // Create tensors - CLTensor a = create_tensor(shape_a, data_type, 1, lhs_q_info); - CLTensor b = create_tensor(shape_b, data_type, 1, rhs_q_info); - CLTensor dst = create_tensor(output_shape, data_type, 1, dst_q_info); + CLTensor a = create_tensor(shape_a, data_type, 1, lhs_q_info); + CLTensor b = create_tensor(shape_b, data_type, 1, rhs_q_info); + CLTensor bias = create_tensor(output_shape[0], (is_quantized) ? DataType::S32 : data_type, 1, dst_q_info); + CLTensor dst = create_tensor(output_shape, data_type, 1, dst_q_info); - matMul.configure(a.info(), b.info(), dst.info(), matmul_info); + matMul.configure(a.info(), b.info(), (_enable_bias) ? bias.info() : nullptr, dst.info(), matmul_info); ARM_COMPUTE_ASSERT(a.info()->is_resizable()); ARM_COMPUTE_ASSERT(b.info()->is_resizable()); ARM_COMPUTE_ASSERT(dst.info()->is_resizable()); @@ -172,6 +200,22 @@ protected: { ACL_SRC_1, &b }, { ACL_DST, &dst } }); + + if(_enable_bias) + { + // Allocate, fill and add bias to TensorPack obj + bias.allocator()->allocate(); + if(is_quantized) + { + fill_bias_s32(CLAccessor(bias), 2, dst_q_info.uniform()); + } + else + { + fill(CLAccessor(bias), 2); + } + tensors_pack.add_tensor(ACL_SRC_2, &bias); + } + matMul.run(tensors_pack); return dst; @@ -240,9 +284,21 @@ protected: template typename std::enable_if < std::is_same::value || std::is_same::value, SimpleTensor>::type gemm_reference(SimpleTensor &a, SimpleTensor &b, SimpleTensor &c) { + // Fill bias, then copy first dimension into subsequent dimensions to mimic broadcast + // of bias tensor from shape [dst.dimension(0)] to [dst.tensor_shape()] in target kernel + if(_enable_bias) + { + fill(c, 2); + const int n = c.shape().x(); + const int other_dims = c.shape().collapsed_from(1)[1]; + for(int i = 1; i < other_dims; ++i) // For all data, copy first n elements into remaining batches + { + memcpy(c.data() + i * n, c.data(), n * sizeof(T)); + } + } // Setting beta to 0 will effectively disable C for the // computation of the reference: alpha * A * B + 0 * C - return reference::gemm(a, b, c, 1.0f, 0.f); + return reference::gemm(a, b, c, 1.0f, (_enable_bias) ? 1.0f : 0.f); } template @@ -264,18 +320,57 @@ protected: constexpr int32_t gemmlowp_max_bound = std::numeric_limits::max(); SimpleTensor bias{ c.shape(), DataType::S32 }; - fill_constant(bias, static_cast(0)); + if(_enable_bias) + { + // Identical to float implementation, fill and copy values of bias first dimension + fill_bias_s32(bias, 2, cq); + const int n = bias.shape().x(); + const int other_dims = bias.shape().collapsed_from(1)[1]; + const unsigned int dt_size = sizeof(int32_t); + for(int i = 1; i < other_dims; ++i) + { + memcpy(bias.data() + i * n, bias.data(), n * dt_size); + } + } + else + { + fill_constant(bias, static_cast(0)); // effectively disable bias + } const SimpleTensor final_result = reference::gemmlowp_quantize_down_scale_by_fixedpoint(result, bias, gemmlowp_multipliers, gemmlowp_shifts, gemmlowp_offset, gemmlowp_min_bound, gemmlowp_max_bound); + return final_result; } CLTensor _target{}; SimpleTensor _reference{}; + bool _enable_bias{ false }; bool _device_supports_export_to_cl_image{ true }; + bool _device_supports_mmul{ true }; }; +template +class MatMulKernelValidationFixture : public MatMulKernelGenericValidationFixture +{ +public: + void setup(TensorShape shape_a, TensorShape shape_b, TensorShape output_shape, bool pretranspose_a, bool pretranspose_b, int M0, int N0, int K0, bool export_rhs_to_cl_image, DataType data_type) + { + MatMulKernelGenericValidationFixture::setup(shape_a, shape_b, output_shape, pretranspose_a, pretranspose_b, M0, N0, K0, export_rhs_to_cl_image, data_type, + false /* enable bias */); + } +}; + +template +class MatMulKernelWithBiasValidation : public MatMulKernelGenericValidationFixture +{ +public: + void setup(TensorShape shape_a, TensorShape shape_b, TensorShape output_shape, bool pretranspose_a, bool pretranspose_b, int M0, int N0, int K0, bool export_rhs_to_cl_image, DataType data_type) + { + MatMulKernelGenericValidationFixture::setup(shape_a, shape_b, output_shape, pretranspose_a, pretranspose_b, M0, N0, K0, export_rhs_to_cl_image, data_type, + true /* enable bias */); + } +}; } // namespace validation } // namespace test } // namespace arm_compute diff --git a/tests/validation/fixtures/MaxUnpoolingLayerFixture.h b/tests/validation/fixtures/MaxUnpoolingLayerFixture.h index b2575aa26b4b64494234444c76a123e91b362732..808e3ffabdfd2ef7f3327e5795fce6d3e4d9bfe1 100644 --- a/tests/validation/fixtures/MaxUnpoolingLayerFixture.h +++ b/tests/validation/fixtures/MaxUnpoolingLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021 Arm Limited. + * Copyright (c) 2020-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -46,7 +46,6 @@ template void setup(TensorShape shape, PoolingLayerInfo pool_info, DataType data_type, DataLayout data_layout) { std::mt19937 gen(library->seed()); @@ -150,7 +149,6 @@ template { public: - template void setup(TensorShape shape, PoolingType pool_type, Size2D pool_size, PadStrideInfo pad_stride_info, DataType data_type, DataLayout data_layout) { MaxUnpoolingLayerValidationGenericFixture::setup(shape, PoolingLayerInfo(pool_type, pool_size, data_layout, pad_stride_info, true), diff --git a/tests/validation/fixtures/MeanStdDevNormalizationLayerFixture.h b/tests/validation/fixtures/MeanStdDevNormalizationLayerFixture.h index f3c108e6da89bf9c8a022e3a8778454ab4bfe6b8..bf5d20790cd61aafce927a87dbc7df1def2553fc 100644 --- a/tests/validation/fixtures/MeanStdDevNormalizationLayerFixture.h +++ b/tests/validation/fixtures/MeanStdDevNormalizationLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022 Arm Limited. + * Copyright (c) 2019-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -44,7 +44,6 @@ template void setup(TensorShape shape, DataType dt, bool in_place, float epsilon = 1e-8) { QuantizationInfo qi = QuantizationInfo(0.5f, 10); diff --git a/tests/validation/fixtures/NonMaxSuppressionFixture.h b/tests/validation/fixtures/NonMaxSuppressionFixture.h index abfe909fc4187628d22ffc7090db4f58df579cfd..043b4731aaa5888e558acf04a7e422556939d7e1 100644 --- a/tests/validation/fixtures/NonMaxSuppressionFixture.h +++ b/tests/validation/fixtures/NonMaxSuppressionFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -46,7 +46,6 @@ template class NMSValidationFixture : public framework::Fixture { public: - template void setup(TensorShape input_shape, unsigned int max_output_size, float score_threshold, float nms_threshold) { ARM_COMPUTE_ERROR_ON(max_output_size == 0); diff --git a/tests/validation/fixtures/NormalizationLayerFixture.h b/tests/validation/fixtures/NormalizationLayerFixture.h index 8a29e6482682459d01d8cbeb9841a77da56869f5..ddaa3533f53344f200020cd20f1d153438995476 100644 --- a/tests/validation/fixtures/NormalizationLayerFixture.h +++ b/tests/validation/fixtures/NormalizationLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -46,7 +46,6 @@ template void setup(TensorShape shape, NormType norm_type, int norm_size, float beta, bool is_scaled, DataType data_type, DataLayout data_layout) { NormalizationLayerInfo info(norm_type, norm_size, 5, beta, 1.f, is_scaled); @@ -119,7 +118,6 @@ template { public: - template void setup(TensorShape shape, NormType norm_type, int norm_size, float beta, bool is_scaled, DataType data_type, DataLayout data_layout) { NormalizationValidationGenericFixture::setup(shape, norm_type, norm_size, beta, is_scaled, data_type, data_layout); diff --git a/tests/validation/fixtures/NormalizePlanarYUVLayerFixture.h b/tests/validation/fixtures/NormalizePlanarYUVLayerFixture.h index 4586afbb5d3e6e8f3f82ff2048f67e3e2cda1b78..5f2c8659505c62ddd444e2d45388a53faf7022ff 100644 --- a/tests/validation/fixtures/NormalizePlanarYUVLayerFixture.h +++ b/tests/validation/fixtures/NormalizePlanarYUVLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -44,7 +44,6 @@ template void setup(TensorShape shape0, TensorShape shape1, DataType dt, DataLayout data_layout, QuantizationInfo quantization_info) { _data_type = dt; @@ -144,7 +143,6 @@ template { public: - template void setup(TensorShape shape0, TensorShape shape1, DataType dt, DataLayout data_layout) { NormalizePlanarYUVLayerValidationGenericFixture::setup(shape0, shape1, dt, data_layout, QuantizationInfo()); @@ -155,7 +153,6 @@ template { public: - template void setup(TensorShape shape0, TensorShape shape1, DataType dt, DataLayout data_layout, QuantizationInfo quantization_info) { NormalizePlanarYUVLayerValidationGenericFixture::setup(shape0, shape1, dt, data_layout, quantization_info); diff --git a/tests/validation/fixtures/PadLayerFixture.h b/tests/validation/fixtures/PadLayerFixture.h index 1951813df82f16accf99ad64b4bcb6768e29c262..93b43616ff0bd71a969447769e6d5cc4b987adee 100644 --- a/tests/validation/fixtures/PadLayerFixture.h +++ b/tests/validation/fixtures/PadLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -44,7 +44,6 @@ template void setup(TensorShape shape, DataType data_type, const PaddingList &padding, const PaddingMode mode) { PaddingList clamped_padding = padding; diff --git a/tests/validation/fixtures/PermuteFixture.h b/tests/validation/fixtures/PermuteFixture.h index 118971394a4947c7748e06cbcd16895ad6f17cb7..b1b3845a8d12600b63b63d0e6296bd907c85dc83 100644 --- a/tests/validation/fixtures/PermuteFixture.h +++ b/tests/validation/fixtures/PermuteFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -45,7 +45,6 @@ template void setup(TensorShape input_shape, PermutationVector perm, DataType data_type) { _target = compute_target(input_shape, data_type, perm); diff --git a/tests/validation/fixtures/PixelWiseMultiplicationFixture.h b/tests/validation/fixtures/PixelWiseMultiplicationFixture.h index 8dc517910981d309526c0d6f0cc3a45e1f7a798d..4345d8a13f0adae7b926547dfd62c51894703ab9 100644 --- a/tests/validation/fixtures/PixelWiseMultiplicationFixture.h +++ b/tests/validation/fixtures/PixelWiseMultiplicationFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -44,7 +44,6 @@ template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType dt_in1, @@ -155,7 +154,6 @@ template { public: - template void setup(const TensorShape &shape, DataType dt_in1, DataType dt_in2, DataType dt_out, float scale, ConvertPolicy convert_policy, RoundingPolicy rounding_policy, bool is_inplace) { PixelWiseMultiplicationGenericValidationFixture::setup(shape, shape, dt_in1, dt_in2, dt_out, scale, convert_policy, rounding_policy, @@ -167,7 +165,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType dt_in1, DataType dt_in2, DataType dt_out, float scale, ConvertPolicy convert_policy, RoundingPolicy rounding_policy, bool is_inplace) { @@ -180,7 +177,6 @@ template { public: - template void setup(const TensorShape &shape, DataType dt_in1, DataType dt_in2, float scale, ConvertPolicy convert_policy, RoundingPolicy rounding_policy, ActivationLayerInfo act_info, bool is_inplace) { PixelWiseMultiplicationGenericValidationFixture::setup(shape, shape, dt_in1, dt_in2, dt_in2, scale, convert_policy, rounding_policy, @@ -192,7 +188,6 @@ template { public: - template void setup(const TensorShape &shape, DataType dt_in1, DataType dt_in2, float scale, ConvertPolicy convert_policy, RoundingPolicy rounding_policy, ActivationLayerInfo act_info, bool is_inplace) { PixelWiseMultiplicationGenericValidationFixture::setup(shape, shape, dt_in1, dt_in2, dt_in2, scale, convert_policy, rounding_policy, @@ -204,7 +199,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType dt_in1, DataType dt_in2, float scale, ConvertPolicy convert_policy, RoundingPolicy rounding_policy, ActivationLayerInfo act_info, bool is_inplace) { @@ -217,7 +211,6 @@ template { public: - template void setup(const TensorShape &shape, DataType dt_in1, DataType dt_in2, DataType dt_out, float scale, ConvertPolicy convert_policy, RoundingPolicy rounding_policy, QuantizationInfo qinfo0, QuantizationInfo qinfo1, QuantizationInfo qinfo_out, bool is_inplace) { @@ -230,7 +223,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType dt_in1, DataType dt_in2, DataType dt_out, float scale, ConvertPolicy convert_policy, RoundingPolicy rounding_policy, QuantizationInfo qinfo0, QuantizationInfo qinfo1, QuantizationInfo qinfo_out, bool is_inplace) { diff --git a/tests/validation/fixtures/Pooling3dLayerFixture.h b/tests/validation/fixtures/Pooling3dLayerFixture.h index 563f1dcced18ab1e70acc12c0136931d58e89a84..1bdf615fb1aca2808ba188daf256b0313a2a3d4f 100644 --- a/tests/validation/fixtures/Pooling3dLayerFixture.h +++ b/tests/validation/fixtures/Pooling3dLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -45,7 +45,6 @@ template void setup(TensorShape shape, Pooling3dLayerInfo pool_info, DataType data_type, QuantizationInfo input_qinfo = QuantizationInfo(), QuantizationInfo output_qinfo = QuantizationInfo()) { _target = compute_target(shape, pool_info, data_type, input_qinfo, output_qinfo); @@ -120,7 +119,6 @@ template { public: - template void setup(TensorShape shape, PoolingType pool_type, Size3D pool_size, Size3D stride, Padding3D padding, bool exclude_padding, DataType data_type) { Pooling3dLayerValidationGenericFixture::setup(shape, Pooling3dLayerInfo(pool_type, pool_size, stride, padding, exclude_padding), @@ -132,7 +130,6 @@ template { public: - template void setup(TensorShape shape, PoolingType pool_type, Size3D pool_size, Size3D stride, Padding3D padding, bool exclude_padding, DataType data_type, QuantizationInfo input_qinfo = QuantizationInfo(), QuantizationInfo output_qinfo = QuantizationInfo()) { @@ -145,7 +142,6 @@ template { public: - template void setup(TensorShape shape, PoolingType pool_type, DataType data_type) { Pooling3dLayerValidationGenericFixture::setup(shape, Pooling3dLayerInfo(pool_type), data_type); @@ -156,7 +152,6 @@ template { public: - template void setup(TensorShape src_shape, Pooling3dLayerInfo pool_info, DataType data_type) { Pooling3dLayerValidationGenericFixture::setup(src_shape, pool_info, data_type); diff --git a/tests/validation/fixtures/PoolingLayerFixture.h b/tests/validation/fixtures/PoolingLayerFixture.h index f34aaa8bfae9c436f865dbb15c7ccef70766d6da..59c920868b13ccf00eeaa4727e1515cff474ff1d 100644 --- a/tests/validation/fixtures/PoolingLayerFixture.h +++ b/tests/validation/fixtures/PoolingLayerFixture.h @@ -45,7 +45,6 @@ template void setup(TensorShape shape, PoolingLayerInfo pool_info, DataType data_type, DataLayout data_layout, bool indices = false, QuantizationInfo input_qinfo = QuantizationInfo(), QuantizationInfo output_qinfo = QuantizationInfo(), bool mixed_layout = false) { @@ -161,7 +160,6 @@ template { public: - template void setup(TensorShape shape, PoolingType pool_type, Size2D pool_size, PadStrideInfo pad_stride_info, bool exclude_padding, DataType data_type, DataLayout data_layout, bool use_kernel_indices) { PoolingLayerValidationGenericFixture::setup(shape, PoolingLayerInfo(pool_type, pool_size, data_layout, pad_stride_info, exclude_padding, false, @@ -174,7 +172,6 @@ template { public: - template void setup(TensorShape shape, PoolingType pool_type, Size2D pool_size, PadStrideInfo pad_stride_info, bool exclude_padding, DataType data_type, DataLayout data_layout) { PoolingLayerValidationGenericFixture::setup(shape, PoolingLayerInfo(pool_type, pool_size, data_layout, pad_stride_info, exclude_padding), @@ -186,7 +183,6 @@ template { public: - template void setup(TensorShape shape, PoolingType pool_type, Size2D pool_size, PadStrideInfo pad_stride_info, bool exclude_padding, DataType data_type, DataLayout data_layout, bool fp_mixed_precision = false) { PoolingLayerValidationGenericFixture::setup(shape, PoolingLayerInfo(pool_type, pool_size, data_layout, pad_stride_info, exclude_padding, fp_mixed_precision), @@ -198,7 +194,6 @@ template { public: - template void setup(TensorShape shape, PoolingType pool_type, Size2D pool_size, PadStrideInfo pad_stride_info, bool exclude_padding, DataType data_type, DataLayout data_layout = DataLayout::NCHW, QuantizationInfo input_qinfo = QuantizationInfo(), QuantizationInfo output_qinfo = QuantizationInfo()) { @@ -211,7 +206,6 @@ template { public: - template void setup(TensorShape src_shape, PoolingLayerInfo pool_info, DataType data_type) { PoolingLayerValidationGenericFixture::setup(src_shape, pool_info, data_type, pool_info.data_layout); @@ -222,7 +216,6 @@ template { public: - template void setup(TensorShape shape, PoolingType pool_type, DataType data_type, DataLayout data_layout = DataLayout::NCHW) { PoolingLayerValidationGenericFixture::setup(shape, PoolingLayerInfo(pool_type, data_layout), data_type, data_layout); diff --git a/tests/validation/fixtures/PriorBoxLayerFixture.h b/tests/validation/fixtures/PriorBoxLayerFixture.h index 40a9f5d97653aa0a56de11e0f160f808d31e7a00..0a76cfd155f41f009abaf2432f5cfcc3edfd9fcd 100644 --- a/tests/validation/fixtures/PriorBoxLayerFixture.h +++ b/tests/validation/fixtures/PriorBoxLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -45,7 +45,6 @@ template void setup(TensorShape input_shape, PriorBoxLayerInfo info, DataType data_type, DataLayout data_layout) { TensorInfo input_info(input_shape, 1, data_type); diff --git a/tests/validation/fixtures/QLSTMLayerNormalizationFixture.h b/tests/validation/fixtures/QLSTMLayerNormalizationFixture.h index b496e4abad6d72fbcfac098b18d1159044deff60..e864b4affe53ca6172bcc0d28877f899684a3d7a 100644 --- a/tests/validation/fixtures/QLSTMLayerNormalizationFixture.h +++ b/tests/validation/fixtures/QLSTMLayerNormalizationFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021 Arm Limited. + * Copyright (c) 2020-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -44,7 +44,6 @@ template void setup(TensorShape input_shape, TensorShape weight_shape, TensorShape bias_shape, DataType data_type, QuantizationInfo weight_qinfo) { ARM_COMPUTE_ERROR_ON(data_type != DataType::QSYMM16); diff --git a/tests/validation/fixtures/QuantizationLayerFixture.h b/tests/validation/fixtures/QuantizationLayerFixture.h index 8e8b920a9a1eec530936c3541b5e7f5b4ee21cd2..1b21967bda258a752fad8c396102b180530ce0d4 100644 --- a/tests/validation/fixtures/QuantizationLayerFixture.h +++ b/tests/validation/fixtures/QuantizationLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -46,7 +46,6 @@ template void setup(TensorShape shape, DataType data_type_in, DataType data_type_out, QuantizationInfo qinfo, QuantizationInfo qinfo_in) { _target = compute_target(shape, data_type_in, data_type_out, qinfo, qinfo_in); @@ -108,7 +107,6 @@ template { public: - template void setup(TensorShape shape, DataType data_type_in, DataType data_type_out, QuantizationInfo qinfo) { QuantizationValidationGenericFixture::setup(shape, data_type_in, data_type_out, qinfo, QuantizationInfo()); diff --git a/tests/validation/fixtures/RNNLayerFixture.h b/tests/validation/fixtures/RNNLayerFixture.h index 983cd43398a9dd711df683062f4eafc46e08cf4c..e9a05e7838b4309f3dd52db7ee0d32ee7ae394b1 100644 --- a/tests/validation/fixtures/RNNLayerFixture.h +++ b/tests/validation/fixtures/RNNLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -42,7 +42,6 @@ template void setup(TensorShape input_shape, TensorShape weights_shape, TensorShape recurrent_weights_shape, TensorShape bias_shape, TensorShape output_shape, ActivationLayerInfo info, DataType data_type) { diff --git a/tests/validation/fixtures/ROIAlignLayerFixture.h b/tests/validation/fixtures/ROIAlignLayerFixture.h index 4c349bcd47f805a54d2b2280274635b29c2cb3c2..ad76dcbbd9c2d0c45826249ba4fee452963b6885 100644 --- a/tests/validation/fixtures/ROIAlignLayerFixture.h +++ b/tests/validation/fixtures/ROIAlignLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -45,7 +45,6 @@ template void setup(TensorShape input_shape, const ROIPoolingLayerInfo pool_info, TensorShape rois_shape, DataType data_type, DataLayout data_layout, QuantizationInfo qinfo, QuantizationInfo output_qinfo) { _rois_data_type = is_data_type_quantized_asymmetric(data_type) ? DataType::QASYMM16 : data_type; @@ -189,7 +188,6 @@ template { public: - template void setup(TensorShape input_shape, const ROIPoolingLayerInfo pool_info, TensorShape rois_shape, DataType data_type, DataLayout data_layout) { ROIAlignLayerGenericFixture::setup(input_shape, pool_info, rois_shape, data_type, data_layout, @@ -201,7 +199,6 @@ template { public: - template void setup(TensorShape input_shape, const ROIPoolingLayerInfo pool_info, TensorShape rois_shape, DataType data_type, DataLayout data_layout, QuantizationInfo qinfo, QuantizationInfo output_qinfo) { diff --git a/tests/validation/fixtures/ROIPoolingLayerFixture.h b/tests/validation/fixtures/ROIPoolingLayerFixture.h index 763c5a7d76bccd5e0041be2bac626131da100db0..4b46a6176d7c1360f9a9c384c52c5a1d5639d685 100644 --- a/tests/validation/fixtures/ROIPoolingLayerFixture.h +++ b/tests/validation/fixtures/ROIPoolingLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -45,7 +45,6 @@ template void setup(TensorShape input_shape, const ROIPoolingLayerInfo pool_info, TensorShape rois_shape, DataType data_type, DataLayout data_layout, QuantizationInfo qinfo, QuantizationInfo output_qinfo) { _target = compute_target(input_shape, data_type, data_layout, pool_info, rois_shape, qinfo, output_qinfo); @@ -174,7 +173,6 @@ template { public: - template void setup(TensorShape input_shape, const ROIPoolingLayerInfo pool_info, TensorShape rois_shape, DataType data_type, DataLayout data_layout, QuantizationInfo qinfo, QuantizationInfo output_qinfo) { @@ -187,7 +185,6 @@ template { public: - template void setup(TensorShape input_shape, const ROIPoolingLayerInfo pool_info, TensorShape rois_shape, DataType data_type, DataLayout data_layout) { ROIPoolingLayerGenericFixture::setup(input_shape, pool_info, rois_shape, data_type, data_layout, diff --git a/tests/validation/fixtures/RangeFixture.h b/tests/validation/fixtures/RangeFixture.h index 9265e299b69704f174b1a53cdb0064b5e30d1980..166613a3188096f380a7059fd72f5f54c55ced1d 100644 --- a/tests/validation/fixtures/RangeFixture.h +++ b/tests/validation/fixtures/RangeFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -55,7 +55,6 @@ template void setup(const DataType data_type0, float start, float step, const QuantizationInfo qinfo0 = QuantizationInfo()) { _target = compute_target(data_type0, qinfo0, start, step); diff --git a/tests/validation/fixtures/ReduceMeanFixture.h b/tests/validation/fixtures/ReduceMeanFixture.h index 13354eedc18cc058413f8e6da5dfda3d63222246..e61941435cc032d772a99a8fa1f94387ee8da045 100644 --- a/tests/validation/fixtures/ReduceMeanFixture.h +++ b/tests/validation/fixtures/ReduceMeanFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -47,7 +47,6 @@ template void setup(TensorShape shape, DataType data_type, Coordinates axis, bool keep_dims, QuantizationInfo quantization_info_input, QuantizationInfo quantization_info_output) { _target = compute_target(shape, data_type, axis, keep_dims, quantization_info_input, quantization_info_output); @@ -128,9 +127,9 @@ protected: #ifdef ARM_COMPUTE_OPENCL_ENABLED is_opencl = std::is_same::value; // Round down to zero on opencl to match kernel -#endif /* ARM_COMPUTE_OPENCL_ENABLED */ - out = reference::reduction_operation(i == 0 ? src : out, output_shape, axis[i], ReductionOperation::MEAN_SUM, quantization_info_output, is_opencl ? RoundingPolicy::TO_ZERO : RoundingPolicy::TO_NEAREST_UP); - +#endif /* ARM_COMPUTE_OPENCL_ENABLED */ + out = reference::reduction_operation(i == 0 ? src : out, output_shape, axis[i], ReductionOperation::MEAN_SUM, data_type, quantization_info_output, + is_opencl ? RoundingPolicy::TO_ZERO : RoundingPolicy::TO_NEAREST_UP); } if(!keep_dims) @@ -139,7 +138,7 @@ protected: std::sort(axis.begin(), axis.begin() + axis.num_dimensions()); for(unsigned int i = 0; i < axis.num_dimensions(); ++i) { - output_shape.remove_dimension(axis[i] - i); + output_shape.remove_dimension(axis[i] - i, false); } out = reference::reshape_layer(out, output_shape); @@ -155,7 +154,6 @@ template { public: - template void setup(TensorShape shape, DataType data_type, Coordinates axis, bool keep_dims, QuantizationInfo quantization_info_input, QuantizationInfo quantization_info_output) { ReduceMeanValidationFixture::setup(shape, data_type, axis, keep_dims, quantization_info_input, quantization_info_output); @@ -166,7 +164,6 @@ template { public: - template void setup(TensorShape shape, DataType data_type, Coordinates axis, bool keep_dims) { ReduceMeanValidationFixture::setup(shape, data_type, axis, keep_dims, QuantizationInfo(), QuantizationInfo()); diff --git a/tests/validation/fixtures/ReductionOperationFixture.h b/tests/validation/fixtures/ReductionOperationFixture.h index c333f1391f7547f273d27d6b713b47628366e856..b44f299486dcac6d2ef3ade98329412589116eae 100644 --- a/tests/validation/fixtures/ReductionOperationFixture.h +++ b/tests/validation/fixtures/ReductionOperationFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2022 Arm Limited. + * Copyright (c) 2017-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -45,7 +45,6 @@ template void setup(TensorShape shape, DataType data_type, unsigned int axis, ReductionOperation op, QuantizationInfo quantization_info, bool keep_dims = false) { const bool is_arg_min_max = (op == ReductionOperation::ARG_IDX_MAX) || (op == ReductionOperation::ARG_IDX_MIN); @@ -135,7 +134,7 @@ protected: // Fill reference fill(src); - return reference::reduction_operation(src, dst_shape, axis, op, quantization_info); + return reference::reduction_operation(src, dst_shape, axis, op, data_type, quantization_info); } TensorType _target{}; @@ -149,7 +148,6 @@ template { public: - template void setup(TensorShape shape, DataType data_type, unsigned int axis, ReductionOperation op, QuantizationInfo quantization_info = QuantizationInfo(), bool keep_dims = false) { ReductionOperationValidationFixture::setup(shape, data_type, axis, op, quantization_info, keep_dims); @@ -160,7 +158,6 @@ template { public: - template void setup(TensorShape shape, DataType data_type, unsigned int axis, ReductionOperation op, bool keep_dims = false) { ReductionOperationValidationFixture::setup(shape, data_type, axis, op, QuantizationInfo(), keep_dims); diff --git a/tests/validation/fixtures/ReorderFixture.h b/tests/validation/fixtures/ReorderFixture.h index bb3e3730df3c14dc44789b61eeca74eb0530bbec..36e62696bcdff8a2d93ab55fbb87a3aab78cf6de 100644 --- a/tests/validation/fixtures/ReorderFixture.h +++ b/tests/validation/fixtures/ReorderFixture.h @@ -44,7 +44,6 @@ template void setup(TensorShape input_shape, TensorShape output_shape, WeightFormat input_wf, WeightFormat output_wf, DataType data_type) { _target = compute_target(input_shape, output_shape, input_wf, output_wf, data_type); diff --git a/tests/validation/fixtures/ReorgLayerFixture.h b/tests/validation/fixtures/ReorgLayerFixture.h index 76a486e3d726d665236fb9ff7ac9942890efb10c..f87017190e29d93dc95d9a81fa3c70892300b2c7 100644 --- a/tests/validation/fixtures/ReorgLayerFixture.h +++ b/tests/validation/fixtures/ReorgLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -43,7 +43,6 @@ template void setup(TensorShape input_shape, int32_t stride, DataType data_type, DataLayout data_layout) { _target = compute_target(input_shape, stride, data_type, data_layout); diff --git a/tests/validation/fixtures/ReshapeLayerFixture.h b/tests/validation/fixtures/ReshapeLayerFixture.h index 8a98379ef2d0a2d31d56f1c01385a8612d5cf951..b4c3a9fa1b82ea748ab591022d876d7757e7f350 100644 --- a/tests/validation/fixtures/ReshapeLayerFixture.h +++ b/tests/validation/fixtures/ReshapeLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -44,7 +44,6 @@ template void setup(TensorShape input_shape, TensorShape output_shape, DataType data_type) { _target = compute_target(input_shape, output_shape, data_type); diff --git a/tests/validation/fixtures/ReverseFixture.h b/tests/validation/fixtures/ReverseFixture.h index 9d047a006714e61837ecac522193f8a9357071a9..509fd93abf3a164790d9b6e64be5734381f601ae 100644 --- a/tests/validation/fixtures/ReverseFixture.h +++ b/tests/validation/fixtures/ReverseFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -45,7 +45,6 @@ template void setup(TensorShape shape, TensorShape axis_shape, DataType data_type) { _target = compute_target(shape, axis_shape, data_type); diff --git a/tests/validation/fixtures/ScaleFixture.h b/tests/validation/fixtures/ScaleFixture.h index 72feb62016caeda6318136db20f4b97333604216..b00a86f6dce60405dcd3cc23efc1ee3ae2efa82a 100644 --- a/tests/validation/fixtures/ScaleFixture.h +++ b/tests/validation/fixtures/ScaleFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2022 Arm Limited. + * Copyright (c) 2017-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -38,7 +38,6 @@ template void setup(TensorShape shape, DataType data_type, QuantizationInfo quantization_info, DataLayout data_layout, InterpolationPolicy policy, BorderMode border_mode, SamplingPolicy sampling_policy, bool align_corners, bool mixed_layout, QuantizationInfo output_quantization_info) { @@ -211,7 +210,6 @@ template { public: - template void setup(TensorShape shape, DataType data_type, QuantizationInfo quantization_info, DataLayout data_layout, InterpolationPolicy policy, BorderMode border_mode, SamplingPolicy sampling_policy, bool align_corners) { @@ -231,7 +229,6 @@ template { public: - template void setup(TensorShape shape, DataType data_type, QuantizationInfo input_quantization_info, QuantizationInfo output_quantization_info, DataLayout data_layout, InterpolationPolicy policy, BorderMode border_mode, SamplingPolicy sampling_policy, bool align_corners) @@ -252,7 +249,6 @@ template { public: - template void setup(TensorShape shape, DataType data_type, DataLayout data_layout, InterpolationPolicy policy, BorderMode border_mode, SamplingPolicy sampling_policy, bool align_corners) { ScaleValidationGenericFixture::setup(shape, diff --git a/tests/validation/fixtures/ScharrFixture.h b/tests/validation/fixtures/ScharrFixture.h index e67d0178b012b75f2720cf1064bc53d70a8e3b62..b54a9d29e6c308bac5b4ff149def1ceaa54643a3 100644 --- a/tests/validation/fixtures/ScharrFixture.h +++ b/tests/validation/fixtures/ScharrFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -66,7 +66,6 @@ template void setup(TensorShape shape, BorderMode border_mode, Format format, GradientDimension gradient_dimension) { // Generate a random constant value diff --git a/tests/validation/fixtures/SelectFixture.h b/tests/validation/fixtures/SelectFixture.h index 7b035574f77bdfb50a53ec3cc1f6e9c2b13427ff..8cb6f062f932627413aff552f95e2966d4208de2 100644 --- a/tests/validation/fixtures/SelectFixture.h +++ b/tests/validation/fixtures/SelectFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -63,7 +63,6 @@ template void setup(TensorShape shape, bool has_same_same_rank, DataType data_type) { TensorShape condition_shape = detail::select_condition_shape(shape, has_same_same_rank); diff --git a/tests/validation/fixtures/SliceOperationsFixtures.h b/tests/validation/fixtures/SliceOperationsFixtures.h index 267cdd5123dd7963ba4fce565cd8fc4f264d2d32..b1f91ea2e0bb001b73a333e4742afadbf28f5e64 100644 --- a/tests/validation/fixtures/SliceOperationsFixtures.h +++ b/tests/validation/fixtures/SliceOperationsFixtures.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -45,7 +45,6 @@ template void setup(TensorShape shape, Coordinates starts, Coordinates ends, DataType data_type) { _target = compute_target(shape, starts, ends, data_type); @@ -108,7 +107,6 @@ template void setup(TensorShape shape, Coordinates starts, Coordinates ends, BiStrides strides, int32_t begin_mask, int32_t end_mask, int32_t shrink_mask, diff --git a/tests/validation/fixtures/SoftmaxLayerFixture.h b/tests/validation/fixtures/SoftmaxLayerFixture.h index 352e215dee220eee0fa2fde136da0ec71cbd1d9f..f4bf8df9c0c378e0847c8fb97bec2d46322eac04 100644 --- a/tests/validation/fixtures/SoftmaxLayerFixture.h +++ b/tests/validation/fixtures/SoftmaxLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -46,7 +46,6 @@ template void setup(TensorShape shape, DataType data_type, QuantizationInfo quantization_info, float beta, size_t axis) { _quantization_info = quantization_info; @@ -131,7 +130,6 @@ template { public: - template void setup(TensorShape shape, DataType data_type, float beta, size_t axis) { SoftmaxValidationGenericFixture::setup(shape, @@ -146,7 +144,6 @@ template { public: - template void setup(TensorShape shape, DataType data_type, QuantizationInfo quantization_info, float beta, size_t axis) { SoftmaxValidationGenericFixture::setup(shape, diff --git a/tests/validation/fixtures/SpaceToBatchFixture.h b/tests/validation/fixtures/SpaceToBatchFixture.h index cfa7775c2105b16b9bbd16454073dc689e5b47b2..964e5113016bc64d72a2dcd1be4c050acb1c621e 100644 --- a/tests/validation/fixtures/SpaceToBatchFixture.h +++ b/tests/validation/fixtures/SpaceToBatchFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -39,7 +39,6 @@ template void setup(TensorShape input_shape, TensorShape block_shape_shape, TensorShape paddings_shape, TensorShape output_shape, DataType data_type, DataLayout data_layout, QuantizationInfo quantization_info) { @@ -140,7 +139,6 @@ template { public: - template void setup(TensorShape input_shape, TensorShape block_shape_shape, TensorShape paddings_shape, TensorShape output_shape, DataType data_type, DataLayout data_layout) { @@ -152,7 +150,6 @@ template { public: - template void setup(TensorShape input_shape, TensorShape block_shape_shape, TensorShape paddings_shape, TensorShape output_shape, DataType data_type, DataLayout data_layout, QuantizationInfo quantization_info) { diff --git a/tests/validation/fixtures/SpaceToDepthFixture.h b/tests/validation/fixtures/SpaceToDepthFixture.h index 0776e495eb7bfc2e5b753bf793ef2d5c042b8766..2d2e9fad7d926a89b673f7fea6095f865b8623d0 100644 --- a/tests/validation/fixtures/SpaceToDepthFixture.h +++ b/tests/validation/fixtures/SpaceToDepthFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -40,7 +40,6 @@ template void setup(TensorShape input_shape, TensorShape output_shape, const int block_shape, DataType data_type, DataLayout data_layout) { _target = compute_target(input_shape, output_shape, block_shape, data_type, data_layout); diff --git a/tests/validation/fixtures/SplitFixture.h b/tests/validation/fixtures/SplitFixture.h index 3006f214679cee19208b14e1ac3df6810b074431..203925329c5e723c22246395083af28c14d7f9e1 100644 --- a/tests/validation/fixtures/SplitFixture.h +++ b/tests/validation/fixtures/SplitFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -47,7 +47,6 @@ template void setup(TensorShape shape, unsigned int axis, unsigned int splits, DataType data_type) { _target = compute_target(shape, axis, splits, data_type); @@ -149,7 +148,6 @@ template void setup(TensorShape shape, unsigned int axis, std::vector split_shapes, DataType data_type) { _target = compute_target(shape, axis, split_shapes, data_type); diff --git a/tests/validation/fixtures/StackLayerFixture.h b/tests/validation/fixtures/StackLayerFixture.h index f729e059b002a46a152cb0d0b08656453d93332d..7320a032bdfb3f22b0285f506074d2bae3578d96 100644 --- a/tests/validation/fixtures/StackLayerFixture.h +++ b/tests/validation/fixtures/StackLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -52,7 +52,6 @@ template void setup(TensorShape shape_src, int axis, DataType data_type, int num_tensors) { _target = compute_target(shape_src, axis, data_type, num_tensors); diff --git a/tests/validation/fixtures/TileFixture.h b/tests/validation/fixtures/TileFixture.h index 35773d9a83c31e962b3beffc9d39e3bd1e717f2d..979eee5ab18667af6a85c0df9ddaff18b5bb40e2 100644 --- a/tests/validation/fixtures/TileFixture.h +++ b/tests/validation/fixtures/TileFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -44,7 +44,6 @@ template void setup(TensorShape shape, DataType data_type, const Multiples &multiples) { _target = compute_target(shape, data_type, multiples); diff --git a/tests/validation/fixtures/TransposeFixture.h b/tests/validation/fixtures/TransposeFixture.h index 6f032d4889f885d4285c2702ff3b918b66674496..92eb9af0c1618d89a5c8b914b38c649baf977eb2 100644 --- a/tests/validation/fixtures/TransposeFixture.h +++ b/tests/validation/fixtures/TransposeFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -44,7 +44,6 @@ template void setup(TensorShape shape, DataType data_type) { _target = compute_target(shape, data_type); diff --git a/tests/validation/fixtures/UNIT/DynamicTensorFixture.h b/tests/validation/fixtures/UNIT/DynamicTensorFixture.h index bdf43050e6fba44ce55aeb322ba5bcd6cd999b85..3e96dcbf2dd066d8c379cba50e10544af9da773e 100644 --- a/tests/validation/fixtures/UNIT/DynamicTensorFixture.h +++ b/tests/validation/fixtures/UNIT/DynamicTensorFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -127,7 +127,6 @@ class DynamicTensorType3SingleFunction : public framework::Fixture using T = float; public: - template void setup(TensorShape input_level0, TensorShape input_level1) { input_l0 = input_level0; @@ -251,7 +250,6 @@ class DynamicTensorType3ComplexFunction : public framework::Fixture using T = float; public: - template void setup(std::vector input_shapes, TensorShape weights_shape, TensorShape bias_shape, std::vector output_shapes, PadStrideInfo info) { num_iterations = input_shapes.size(); @@ -390,7 +388,6 @@ class DynamicTensorType2PipelineFunction : public framework::Fixture using T = float; public: - template void setup(std::vector input_shapes) { _data_type = DataType::F32; diff --git a/tests/validation/fixtures/UnstackFixture.h b/tests/validation/fixtures/UnstackFixture.h index 4f8b28073500352c3a30f00cf022ad4245098208..30b7dd55390c8666e62629f1f017cad0dc42bbe8 100644 --- a/tests/validation/fixtures/UnstackFixture.h +++ b/tests/validation/fixtures/UnstackFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -47,7 +47,6 @@ template void setup(TensorShape input_shape, int axis, int num, DataType data_type) { _target = compute_target(input_shape, axis, num, data_type); diff --git a/tests/validation/fixtures/WeightsReshapeFixture.h b/tests/validation/fixtures/WeightsReshapeFixture.h index 7c7214acacb7ef3b314efa195b3e2ae58e4ef709..68bd8b689d81c999c29fc34e6255db266b9eda63 100644 --- a/tests/validation/fixtures/WeightsReshapeFixture.h +++ b/tests/validation/fixtures/WeightsReshapeFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -48,7 +48,6 @@ template void setup(TensorShape input_shape, DataType data_type, bool has_bias, unsigned int num_groups) { const TensorShape output_shape = compute_weights_reshaped_shape(TensorInfo(input_shape, 1, data_type), has_bias, num_groups); diff --git a/tests/validation/fixtures/WinogradConvolutionLayerFixture.h b/tests/validation/fixtures/WinogradConvolutionLayerFixture.h index e99eb910e88568ef2aad80e6c26848cb9e8e4809..1e8820492ab3d55c8f0e5ca0e7b8d3aae941dcb1 100644 --- a/tests/validation/fixtures/WinogradConvolutionLayerFixture.h +++ b/tests/validation/fixtures/WinogradConvolutionLayerFixture.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -55,7 +55,6 @@ template void setup(TensorShape input_shape, TensorShape weights_shape, TensorShape bias_shape, TensorShape output_shape, PadStrideInfo info, Size2D dilation, DataType data_type, ActivationLayerInfo act_info, const DataLayout &data_layout) @@ -243,7 +242,6 @@ template void setup(TensorShape input_shape, WinogradInfo winograd_info, DataLayout data_layout, DataType data_type) { TensorShape output_shape = compute_winograd_input_transform_shape(TensorInfo(input_shape, 1, data_type), winograd_info); @@ -355,7 +353,6 @@ template void setup(TensorShape input_shape, Size2D output_tile, DataLayout data_layout, DataType data_type) { WinogradInfo winograd_info(output_tile, Size2D(input_shape[0], input_shape[1]), Size2D() /* Not needed */, PadStrideInfo() /* Not needed */, DataLayout::NCHW /* Not needed */); @@ -470,7 +467,6 @@ template void setup(TensorShape input_shape, WinogradInfo winograd_info, DataType data_type, ActivationLayerInfo act_info = ActivationLayerInfo()) { _target = compute_target(input_shape, winograd_info, data_type, act_info); diff --git a/tests/validation/fixtures/dynamic_fusion/gpu/cl/DepthwiseConv2dFixture.h b/tests/validation/fixtures/dynamic_fusion/gpu/cl/DepthwiseConv2dFixture.h index b15de71707582d4403c364d834b068200259dea5..6498a06e0354adc150502c750946d6952890d32c 100644 --- a/tests/validation/fixtures/dynamic_fusion/gpu/cl/DepthwiseConv2dFixture.h +++ b/tests/validation/fixtures/dynamic_fusion/gpu/cl/DepthwiseConv2dFixture.h @@ -60,7 +60,6 @@ public: || std::is_same::type, int8_t>::value, int32_t, T >::type; // If T: uint8_t or int8_t then TBias: int32_t, otherwise TBias: T - template void setup(TensorShape input_shape, Size2D kernel_size, const PadStrideInfo &pad_stride, const Size2D &dilation, const unsigned int depth_multiplier, const DataType data_type, const DataLayout data_layout) { @@ -126,14 +125,14 @@ protected: // Create a new workload sketch auto cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - auto gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + auto context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; // Create sketch tensors - TensorInfo input_info = sketch.create_tensor_info(TensorInfo(input_shape, 1, _data_type, _data_layout)); - TensorInfo weight_info = sketch.create_tensor_info(TensorInfo(weights_shape, 1, _data_type, _data_layout)); - TensorInfo bias_info = sketch.create_tensor_info(TensorInfo(bias_shape, 1, _data_type, _data_layout)); - TensorInfo dst_info = sketch.create_tensor_info(); + TensorInfo input_info = context.create_tensor_info(TensorInfo(input_shape, 1, _data_type, _data_layout)); + TensorInfo weight_info = context.create_tensor_info(TensorInfo(weights_shape, 1, _data_type, _data_layout)); + TensorInfo bias_info = context.create_tensor_info(TensorInfo(bias_shape, 1, _data_type, _data_layout)); + TensorInfo dst_info = context.create_tensor_info(); ITensorInfo *ans_info = FunctionType::create_op(sketch, &input_info, &weight_info, &bias_info, dwc_conv2d_attr); GpuOutput::create_op(sketch, ans_info, &dst_info); @@ -213,7 +212,6 @@ template { public: - template void setup(TensorShape input_shape, Size2D kernel_size, const PadStrideInfo &info, const Size2D &dilation, const unsigned int depth_multiplier, DataType data_type, DataLayout data_layout) { DynamicFusionGpuDepthwiseConv2dValidationGenericFixture::setup(input_shape, kernel_size, info, dilation, diff --git a/tests/validation/fixtures/dynamic_fusion/gpu/cl/DirectConv2dFixture.h b/tests/validation/fixtures/dynamic_fusion/gpu/cl/DirectConv2dFixture.h index d9ce4dff1891d994493a504dfcd4f87701a1bfb7..e30a564930edf67dccc4685c5f18bea28c58c135 100644 --- a/tests/validation/fixtures/dynamic_fusion/gpu/cl/DirectConv2dFixture.h +++ b/tests/validation/fixtures/dynamic_fusion/gpu/cl/DirectConv2dFixture.h @@ -88,7 +88,6 @@ public: || std::is_same::type, int8_t>::value, int32_t, T >::type; // If T: uint8_t or int8_t then TBias: int32_t, otherwise TBias: T - template void setup(TensorShape input_shape, TensorShape weights_shape, TensorShape bias_shape, TensorShape output_shape, const PadStrideInfo &info, const Size2D &dilation, DataType data_type, DataLayout data_layout, QuantizationInfo quantization_info, QuantizationInfo weight_quantization_info) { @@ -115,14 +114,14 @@ protected: // Create a new workload sketch auto cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - auto gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + auto context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; // Create sketch tensors - TensorInfo input_info = sketch.create_tensor_info(TensorInfo(input_shape, 1, _data_type, _data_layout)); - TensorInfo weight_info = sketch.create_tensor_info(TensorInfo(weights_shape, 1, _data_type, _data_layout)); - TensorInfo bias_info = sketch.create_tensor_info(TensorInfo(bias_shape, 1, _data_type, _data_layout)); - TensorInfo dst_info = sketch.create_tensor_info(); + TensorInfo input_info = context.create_tensor_info(TensorInfo(input_shape, 1, _data_type, _data_layout)); + TensorInfo weight_info = context.create_tensor_info(TensorInfo(weights_shape, 1, _data_type, _data_layout)); + TensorInfo bias_info = context.create_tensor_info(TensorInfo(bias_shape, 1, _data_type, _data_layout)); + TensorInfo dst_info = context.create_tensor_info(); ITensorInfo *ans_info = FunctionType::create_op(sketch, &input_info, &weight_info, &bias_info, conv2d_attr); GpuOutput::create_op(sketch, ans_info, &dst_info); @@ -203,7 +202,6 @@ template { public: - template void setup(TensorShape input_shape, TensorShape weights_shape, TensorShape output_shape, TensorShape bias_shape, const PadStrideInfo &info, const Size2D &dialation, DataType data_type, DataLayout data_layout, QuantizationInfo quantization_info) { @@ -222,7 +220,6 @@ class DynamicFusionDirectConv2dValidationGenericFixture : public framework::Fixt public: using TBias = typename std::conditional < std::is_same::value || std::is_same::value, int32_t, T >::type; - template void setup(TensorShape input_shape, int stride_x, int stride_y, int pad_x, int pad_y, unsigned int kernel_size, unsigned int num_kernels, DataType data_type, QuantizationInfo quantization_info, DataLayout data_layout) { @@ -256,14 +253,14 @@ protected: permute(output_shape, PermutationVector(2U, 0U, 1U)); auto cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - auto gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + auto context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; // Create sketch tensors - auto input_info = sketch.create_tensor_info(TensorInfo(input_shape, 1, data_type, data_layout)); - auto weight_info = sketch.create_tensor_info(TensorInfo(weights_shape, 1, data_type, data_layout)); - auto bias_info = sketch.create_tensor_info(TensorInfo(bias_shape, 1, bias_data_type, data_layout)); - auto dst_info = sketch.create_tensor_info(); + auto input_info = context.create_tensor_info(TensorInfo(input_shape, 1, data_type, data_layout)); + auto weight_info = context.create_tensor_info(TensorInfo(weights_shape, 1, data_type, data_layout)); + auto bias_info = context.create_tensor_info(TensorInfo(bias_shape, 1, bias_data_type, data_layout)); + auto dst_info = context.create_tensor_info(); ITensorInfo *ans_info = FunctionType::create_op(sketch, &input_info, &weight_info, &bias_info, conv2d_attr); GpuOutput::create_op(sketch, ans_info, &dst_info); @@ -341,7 +338,6 @@ template { public: - template void setup(TensorShape input_shape, int stride_x, int stride_y, int pad_x, int pad_y, unsigned int kernel_size, unsigned int num_kernels, DataType data_type, DataLayout data_layout) { diff --git a/tests/validation/fixtures/dynamic_fusion/gpu/cl/ElementwiseBinaryFixture.h b/tests/validation/fixtures/dynamic_fusion/gpu/cl/ElementwiseBinaryFixture.h index b0680c0e4ab5603dee93b3f8c6eff4a0eb4486ed..567322f18167f6fa0a7c3bdba8ab66690a9d4f2d 100644 --- a/tests/validation/fixtures/dynamic_fusion/gpu/cl/ElementwiseBinaryFixture.h +++ b/tests/validation/fixtures/dynamic_fusion/gpu/cl/ElementwiseBinaryFixture.h @@ -47,7 +47,6 @@ template void setup(ArithmeticOperation ref_op, const TensorShape &shape0, const TensorShape &shape1, const TensorShape &shape2, DataType data_type, bool is_inplace, bool fuse_two_ops = false) { _ref_op = ref_op; @@ -99,13 +98,13 @@ protected: { // Create a new workload sketch auto cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - auto gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + auto context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; // Fuse first element wise binary Op - TensorInfo lhs_info = sketch.create_tensor_info(TensorInfo(shape0, 1, _data_type)); - TensorInfo rhs_info = sketch.create_tensor_info(TensorInfo(shape1, 1, _data_type)); - TensorInfo dst_info = sketch.create_tensor_info(); + TensorInfo lhs_info = context.create_tensor_info(TensorInfo(shape0, 1, _data_type)); + TensorInfo rhs_info = context.create_tensor_info(TensorInfo(shape1, 1, _data_type)); + TensorInfo dst_info = context.create_tensor_info(); TensorInfo rhs_info_fuse; @@ -113,7 +112,7 @@ protected: if(_fuse) { - rhs_info_fuse = sketch.create_tensor_info(TensorInfo(shape2, 1, _data_type)); + rhs_info_fuse = context.create_tensor_info(TensorInfo(shape2, 1, _data_type)); ITensorInfo *ans2_info = FunctionType::create_op(sketch, ans_info, &rhs_info_fuse); GpuOutput::create_op(sketch, ans2_info, &dst_info); } @@ -220,7 +219,6 @@ template { public: - template void setup(ArithmeticOperation ref_op, const TensorShape &shape0, DataType data_type, bool is_inplace) { DynamicFusionGpuElementwiseBinaryValidationGenericFixture::setup(ref_op, shape0, shape0, TensorShape(), data_type, is_inplace); @@ -231,7 +229,6 @@ template { public: - template void setup(ArithmeticOperation ref_op, const TensorShape &shape0, const TensorShape &shape1, DataType data_type, bool is_inplace) { DynamicFusionGpuElementwiseBinaryValidationGenericFixture::setup(ref_op, shape0, shape1, TensorShape(), data_type, is_inplace); @@ -242,7 +239,6 @@ template { public: - template void setup(ArithmeticOperation ref_op, const TensorShape &shape0, const TensorShape &shape1, const TensorShape &shape2, DataType data_type, bool is_inplace, bool fuse_two_ops) { DynamicFusionGpuElementwiseBinaryValidationGenericFixture::setup(ref_op, shape0, shape1, shape2, data_type, is_inplace, fuse_two_ops); diff --git a/tests/validation/fixtures/dynamic_fusion/gpu/cl/Pool2dFixture.h b/tests/validation/fixtures/dynamic_fusion/gpu/cl/Pool2dFixture.h index efb5cf1e74b41426f19375decdc4d183e54158e1..0efb761967b251d59b7fd268a3c22e7f4094cee9 100644 --- a/tests/validation/fixtures/dynamic_fusion/gpu/cl/Pool2dFixture.h +++ b/tests/validation/fixtures/dynamic_fusion/gpu/cl/Pool2dFixture.h @@ -51,7 +51,6 @@ template void setup(TensorShape input_shape, const Pool2dAttributes &pool_attr, DataType data_type, bool mixed_precision) { _target = compute_target(input_shape, pool_attr, data_type, mixed_precision); @@ -91,12 +90,12 @@ protected: // Create a new workload sketch auto cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - auto gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + auto context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; // Create sketch tensors - auto input_info = sketch.create_tensor_info(TensorInfo(input_shape, 1, data_type, DataLayout::NHWC)); - auto dst_info = sketch.create_tensor_info(); + auto input_info = context.create_tensor_info(TensorInfo(input_shape, 1, data_type, DataLayout::NHWC)); + auto dst_info = context.create_tensor_info(); // Create Pool2dSettings GpuPool2dSettings pool_settings = GpuPool2dSettings().mixed_precision(mixed_precision); @@ -151,7 +150,6 @@ template { public: - template void setup(TensorShape input_shape, PoolingType pool_type, Size2D pool_size, Padding2D pad, Size2D stride, bool exclude_padding, DataType data_type) { DynamicFusionGpuPool2dValidationGenericFixture::setup(input_shape, @@ -164,7 +162,6 @@ template { public: - template void setup(TensorShape input_shape, PoolingType pool_type, Size2D pool_size, Padding2D pad, Size2D stride, bool exclude_padding, DataType data_type, bool mixed_precision) { DynamicFusionGpuPool2dValidationGenericFixture::setup(input_shape, @@ -177,7 +174,6 @@ template { public: - template void setup(TensorShape input_shape, Pool2dAttributes pool_attr, DataType data_type) { DynamicFusionGpuPool2dValidationGenericFixture::setup(input_shape, pool_attr, data_type, false); diff --git a/tests/validation/fixtures/dynamic_fusion/operators/ActivationFixture.h b/tests/validation/fixtures/dynamic_fusion/operators/ActivationFixture.h index 9656c497ea81e34eb4ab3327362fe18c8ee2c066..18c3b6bfbb7d12fcb14bcee194999defadd8fc6a 100644 --- a/tests/validation/fixtures/dynamic_fusion/operators/ActivationFixture.h +++ b/tests/validation/fixtures/dynamic_fusion/operators/ActivationFixture.h @@ -47,7 +47,6 @@ template void setup(TensorShape shape, bool fuse, DataType data_type, ActivationLayerInfo act_info, TArgs... args) { _fuse = fuse; @@ -102,12 +101,12 @@ protected: { // Create a new workload sketch CLCompileContext cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - GpuWorkloadContext gpu_ctx{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + GpuWorkloadContext context{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; // Create sketch tensors - TensorInfo src_info = sketch.create_tensor_info(TensorInfo(shape, 1, _data_type)); - TensorInfo dst_info = sketch.create_tensor_info(TensorInfo(shape, 1, _data_type)); + TensorInfo src_info = context.create_tensor_info(TensorInfo(shape, 1, _data_type)); + TensorInfo dst_info = context.create_tensor_info(TensorInfo(shape, 1, _data_type)); ITensorInfo *ans_0_info = FunctionType::create_op(sketch, &src_info, args...); if(_fuse) @@ -177,7 +176,6 @@ template { public: - template void setup(TensorShape shape, bool fuse, DataType data_type) { ActivationLayerInfo act_info{ ActivationLayerInfo::ActivationFunction::LOGISTIC }; @@ -189,7 +187,6 @@ template { public: - template void setup(TensorShape shape, bool fuse, DataType data_type) { ActivationLayerInfo act_info{ ActivationLayerInfo::ActivationFunction::TANH }; diff --git a/tests/validation/fixtures/dynamic_fusion/operators/CastFixture.h b/tests/validation/fixtures/dynamic_fusion/operators/CastFixture.h index cd39ec0a06f35481abd13f7602b35abcafa1fa95..d8e250cb3607ee9edc76765d933bd1f8c54e4edc 100644 --- a/tests/validation/fixtures/dynamic_fusion/operators/CastFixture.h +++ b/tests/validation/fixtures/dynamic_fusion/operators/CastFixture.h @@ -47,7 +47,6 @@ template void setup(TensorShape shape, DataType dt_in, DataType dt_out, ConvertPolicy policy) { _target = compute_target(shape, dt_in, dt_out, policy); @@ -112,12 +111,12 @@ protected: { // Create a new workload sketch auto cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - auto gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + auto context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; // Create sketch tensors - TensorInfo src_info = sketch.create_tensor_info(TensorInfo(shape, 1, dt_in, DataLayout::NCHW)); // layout is not important - TensorInfo dst_info = sketch.create_tensor_info(); + TensorInfo src_info = context.create_tensor_info(TensorInfo(shape, 1, dt_in, DataLayout::NCHW)); // layout is not important + TensorInfo dst_info = context.create_tensor_info(); CastAttributes attributes; attributes.convert_policy(policy).data_type(dt_out); diff --git a/tests/validation/fixtures/dynamic_fusion/operators/ClampFixture.h b/tests/validation/fixtures/dynamic_fusion/operators/ClampFixture.h index a1fd22582f641c06b5d52ee97571fdf16d068c58..3c325d739c00f4fd90aa631480cbaf544cebadd5 100644 --- a/tests/validation/fixtures/dynamic_fusion/operators/ClampFixture.h +++ b/tests/validation/fixtures/dynamic_fusion/operators/ClampFixture.h @@ -46,7 +46,6 @@ template void setup(TensorShape shape, ClampAttributes attributes, bool fuse, DataType data_type) { // CLAMP is implemented as LU_BOUNDED_RELU with the alpha and beta variables swapped. @@ -104,12 +103,12 @@ protected: { // Create a new workload sketch CLCompileContext cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - GpuWorkloadContext gpu_ctx{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + GpuWorkloadContext context{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; // Create sketch tensors - TensorInfo src_info = sketch.create_tensor_info(TensorInfo(shape, 1, _data_type)); - TensorInfo dst_info = sketch.create_tensor_info(TensorInfo(shape, 1, _data_type)); + TensorInfo src_info = context.create_tensor_info(TensorInfo(shape, 1, _data_type)); + TensorInfo dst_info = context.create_tensor_info(TensorInfo(shape, 1, _data_type)); ITensorInfo *ans_0_info = FunctionType::create_op(sketch, &src_info, attributes); if(_fuse) diff --git a/tests/validation/fixtures/dynamic_fusion/operators/MulFixture.h b/tests/validation/fixtures/dynamic_fusion/operators/MulFixture.h index 0530707c3867035aee811ac441c86152ca8150b3..02dc996ffa63351102136e39f3350904ee22daa5 100644 --- a/tests/validation/fixtures/dynamic_fusion/operators/MulFixture.h +++ b/tests/validation/fixtures/dynamic_fusion/operators/MulFixture.h @@ -1,26 +1,26 @@ /* -* Copyright (c) 2023 Arm Limited. -* -* SPDX-License-Identifier: MIT -* -* Permission is hereby granted, free of charge, to any person obtaining a copy -* of this software and associated documentation files (the "Software"), to -* deal in the Software without restriction, including without limitation the -* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -* sell copies of the Software, and to permit persons to whom the Software is -* furnished to do so, subject to the following conditions: -* -* The above copyright notice and this permission notice shall be included in all -* copies or substantial portions of the Software. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ + * Copyright (c) 2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ #ifndef TESTS_VALIDATION_FIXTURES_DYNAMIC_FUSION_OPERATORS_MULFIXTURE #define TESTS_VALIDATION_FIXTURES_DYNAMIC_FUSION_OPERATORS_MULFIXTURE @@ -52,7 +52,6 @@ template void setup(const TensorShape &shape0, const TensorShape &shape1, const TensorShape &shape2, DataType data_type, bool is_inplace, bool fuse_two_ops = false) { _data_type = data_type; @@ -75,13 +74,13 @@ protected: { // Create a new workload sketch auto cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - auto gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + auto context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; // Fuse first multiplication op - TensorInfo lhs_info = sketch.create_tensor_info(TensorInfo(shape0, 1, _data_type)); - TensorInfo rhs_info = sketch.create_tensor_info(TensorInfo(shape1, 1, _data_type)); - TensorInfo dst_info = sketch.create_tensor_info(); + TensorInfo lhs_info = context.create_tensor_info(TensorInfo(shape0, 1, _data_type)); + TensorInfo rhs_info = context.create_tensor_info(TensorInfo(shape1, 1, _data_type)); + TensorInfo dst_info = context.create_tensor_info(); TensorInfo rhs_info_fuse; @@ -89,7 +88,7 @@ protected: if(_fuse) { - rhs_info_fuse = sketch.create_tensor_info(TensorInfo(shape2, 1, _data_type)); + rhs_info_fuse = context.create_tensor_info(TensorInfo(shape2, 1, _data_type)); ITensorInfo *ans2_info = FunctionType::create_op(sketch, ans_info, &rhs_info_fuse); GpuOutput::create_op(sketch, ans2_info, &dst_info); } @@ -200,7 +199,6 @@ template { public: - template void setup(const TensorShape &shape0, DataType data_type, bool is_inplace) { DynamicFusionMulValidationFixture::setup(shape0, shape0, TensorShape(), data_type, is_inplace); @@ -211,7 +209,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, DataType data_type, bool is_inplace) { DynamicFusionMulValidationFixture::setup(shape0, shape1, TensorShape(), data_type, is_inplace); @@ -222,7 +219,6 @@ template { public: - template void setup(const TensorShape &shape0, const TensorShape &shape1, const TensorShape &shape2, DataType data_type, bool is_inplace, bool fuse_two_ops) { DynamicFusionMulValidationFixture::setup(shape0, shape1, shape2, data_type, is_inplace, fuse_two_ops); diff --git a/tests/validation/fixtures/dynamic_fusion/operators/ReshapeFixture.h b/tests/validation/fixtures/dynamic_fusion/operators/ReshapeFixture.h index e0b62d093f72aa8391a5409c37975332c8faac41..abfc6459d62d7e21a1c893277010eca0d450f70c 100644 --- a/tests/validation/fixtures/dynamic_fusion/operators/ReshapeFixture.h +++ b/tests/validation/fixtures/dynamic_fusion/operators/ReshapeFixture.h @@ -50,7 +50,6 @@ template void setup(TensorShape input_shape, TensorShape output_shape, DataType data_type) { _target = compute_target(input_shape, output_shape, data_type); @@ -71,12 +70,12 @@ protected: // Create a new workload sketch auto cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - auto gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + auto context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; // Create sketch tensors - TensorInfo src_info = sketch.create_tensor_info(TensorInfo(input_shape, 1, data_type)); - TensorInfo dst_info = sketch.create_tensor_info(TensorInfo(output_shape, 1, data_type)); + TensorInfo src_info = context.create_tensor_info(TensorInfo(input_shape, 1, data_type)); + TensorInfo dst_info = context.create_tensor_info(TensorInfo(output_shape, 1, data_type)); ReshapeAttributes attributes; attributes.shape(output_shape); diff --git a/tests/validation/fixtures/dynamic_fusion/operators/ResizeFixture.h b/tests/validation/fixtures/dynamic_fusion/operators/ResizeFixture.h index 581a3e894730b22be1311a4316e9b918c051140e..c44f0371d09efeb6a569e6e6a0061d518548298a 100644 --- a/tests/validation/fixtures/dynamic_fusion/operators/ResizeFixture.h +++ b/tests/validation/fixtures/dynamic_fusion/operators/ResizeFixture.h @@ -52,7 +52,6 @@ template void setup(TensorShape shape, DataType data_type, QuantizationInfo quantization_info, DataLayout data_layout, InterpolationPolicy interpolation_policy, SamplingPolicy sampling_policy, bool align_corners, QuantizationInfo output_quantization_info) @@ -137,13 +136,13 @@ protected: // Create a new workload sketch CLCompileContext cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - GpuWorkloadContext gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + GpuWorkloadContext context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; // Create sketch tensors - TensorInfo src_info = sketch.create_tensor_info(TensorInfo(shape, 1, _data_type, _data_layout)); + TensorInfo src_info = context.create_tensor_info(TensorInfo(shape, 1, _data_type, _data_layout)); src_info.set_quantization_info(_input_quantization_info); - TensorInfo dst_info = sketch.create_tensor_info(); + TensorInfo dst_info = context.create_tensor_info(); ResizeAttributes attributes; attributes.align_corners(_align_corners).sampling_policy(_sampling_policy).interpolation_policy(_interpolation_policy).output_width(_output_width).output_height(_output_height); @@ -223,7 +222,6 @@ template { public: - template void setup(TensorShape shape, DataType data_type, DataLayout data_layout, InterpolationPolicy policy, SamplingPolicy sampling_policy, bool align_corners) { DynamicFusionResizeGenericValidationFixture::setup(shape, @@ -241,7 +239,6 @@ template { public: - template void setup(TensorShape shape, DataType data_type, QuantizationInfo quantization_info, DataLayout data_layout, InterpolationPolicy policy, SamplingPolicy sampling_policy, bool align_corners) { diff --git a/tests/validation/fixtures/dynamic_fusion/operators/SoftmaxFixture.h b/tests/validation/fixtures/dynamic_fusion/operators/SoftmaxFixture.h index 38177114e68cb1fadd46841083f3da5e186458bd..1ed133d2efd2dca5a4f702d4b172ec06f9c9ed67 100644 --- a/tests/validation/fixtures/dynamic_fusion/operators/SoftmaxFixture.h +++ b/tests/validation/fixtures/dynamic_fusion/operators/SoftmaxFixture.h @@ -46,7 +46,6 @@ template void setup(TensorShape shape, DataType data_type, float beta, size_t axis, bool is_log) { _reference = compute_reference(shape, data_type, beta, axis, is_log); @@ -82,13 +81,13 @@ protected: { // Create a new workload sketch CLCompileContext cl_compile_ctx = CLKernelLibrary::get().get_compile_context(); - GpuWorkloadContext gpu_ctx = GpuWorkloadContext{ &cl_compile_ctx }; - GpuWorkloadSketch sketch{ &gpu_ctx }; + GpuWorkloadContext context = GpuWorkloadContext{ &cl_compile_ctx }; + GpuWorkloadSketch sketch{ &context }; SoftmaxAttributes softmax_attr{}; softmax_attr.axis(axis).beta(beta).is_log_softmax(is_log); - TensorInfo src_info = sketch.create_tensor_info(shape, 1, data_type); - TensorInfo dst_info = sketch.create_tensor_info(shape, 1, data_type); + TensorInfo src_info = context.create_tensor_info(shape, 1, data_type); + TensorInfo dst_info = context.create_tensor_info(shape, 1, data_type); FunctionType::create_op(sketch, &src_info, &dst_info, softmax_attr); // Configure runtime @@ -143,7 +142,6 @@ template { public: - template void setup(TensorShape shape, DataType data_type, float beta, size_t axis, bool is_log) { DynamicFusionSoftmaxValidationGenericFixture::setup(shape, diff --git a/tests/validation/reference/DepthConvertLayer.cpp b/tests/validation/reference/DepthConvertLayer.cpp index 94c719ade7de969b7dab1716379ae52a146dca91..1e4939129e5ce7425baf9450df5a4c779d8c9560 100644 --- a/tests/validation/reference/DepthConvertLayer.cpp +++ b/tests/validation/reference/DepthConvertLayer.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2020 Arm Limited. + * Copyright (c) 2017-2020, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -186,6 +186,25 @@ template SimpleTensor depth_convert(const SimpleTensor &src, Dat template SimpleTensor depth_convert(const SimpleTensor &src, DataType dt_out, ConvertPolicy policy, uint32_t shift); template SimpleTensor depth_convert(const SimpleTensor &src, DataType dt_out, ConvertPolicy policy, uint32_t shift); +// S64 +template SimpleTensor depth_convert(const SimpleTensor &src, DataType dt_out, ConvertPolicy policy, uint32_t shift); +template SimpleTensor depth_convert(const SimpleTensor &src, DataType dt_out, ConvertPolicy policy, uint32_t shift); +template SimpleTensor depth_convert(const SimpleTensor &src, DataType dt_out, ConvertPolicy policy, uint32_t shift); +template SimpleTensor depth_convert(const SimpleTensor &src, DataType dt_out, ConvertPolicy policy, uint32_t shift); +template SimpleTensor depth_convert(const SimpleTensor &src, DataType dt_out, ConvertPolicy policy, uint32_t shift); +template SimpleTensor depth_convert(const SimpleTensor &src, DataType dt_out, ConvertPolicy policy, uint32_t shift); +template SimpleTensor depth_convert(const SimpleTensor &src, DataType dt_out, ConvertPolicy policy, uint32_t shift); +template SimpleTensor depth_convert(const SimpleTensor &src, DataType dt_out, ConvertPolicy policy, uint32_t shift); + +// U64 +template SimpleTensor depth_convert(const SimpleTensor &src, DataType dt_out, ConvertPolicy policy, uint32_t shift); +template SimpleTensor depth_convert(const SimpleTensor &src, DataType dt_out, ConvertPolicy policy, uint32_t shift); +template SimpleTensor depth_convert(const SimpleTensor &src, DataType dt_out, ConvertPolicy policy, uint32_t shift); +template SimpleTensor depth_convert(const SimpleTensor &src, DataType dt_out, ConvertPolicy policy, uint32_t shift); +template SimpleTensor depth_convert(const SimpleTensor &src, DataType dt_out, ConvertPolicy policy, uint32_t shift); +template SimpleTensor depth_convert(const SimpleTensor &src, DataType dt_out, ConvertPolicy policy, uint32_t shift); +template SimpleTensor depth_convert(const SimpleTensor &src, DataType dt_out, ConvertPolicy policy, uint32_t shift); +template SimpleTensor depth_convert(const SimpleTensor &src, DataType dt_out, ConvertPolicy policy, uint32_t shift); } // namespace reference } // namespace validation } // namespace test diff --git a/tests/validation/reference/ReductionOperation.cpp b/tests/validation/reference/ReductionOperation.cpp index e2890afb9f2f491aeff753e6c4089d1e78316ccc..c189bc2d475f5ea7378aaea333c68237da7928ae 100644 --- a/tests/validation/reference/ReductionOperation.cpp +++ b/tests/validation/reference/ReductionOperation.cpp @@ -181,12 +181,12 @@ OT reduce_operation_arg_min_max(const T *ptr, int reduce_elements, ReductionOper } // namespace template -SimpleTensor compute_reduction_operation(const SimpleTensor &src, const TensorShape &dst_shape, unsigned int axis, ReductionOperation op, RoundingPolicy policy) +SimpleTensor compute_reduction_operation(const SimpleTensor &src, const TensorShape &dst_shape, unsigned int axis, ReductionOperation op, + DataType output_type, RoundingPolicy policy) { // Create reference - const bool is_arg_min_max = (op == ReductionOperation::ARG_IDX_MIN || op == ReductionOperation::ARG_IDX_MAX); - DataType output_data_type = is_arg_min_max ? DataType::S32 : src.data_type(); - SimpleTensor dst{ dst_shape, output_data_type, 1, src.quantization_info() }; + const bool is_arg_min_max = (op == ReductionOperation::ARG_IDX_MIN || op == ReductionOperation::ARG_IDX_MAX); + SimpleTensor dst{ dst_shape, output_type, 1, src.quantization_info() }; const unsigned int src_width = src.shape().x(); const unsigned int src_height = src.shape().y(); const unsigned int src_depth = src.shape().z(); @@ -275,74 +275,89 @@ SimpleTensor compute_reduction_operation(const SimpleTensor &src, const T } template -SimpleTensor reduction_operation(const SimpleTensor &src, const TensorShape &dst_shape, unsigned int axis, ReductionOperation op, QuantizationInfo quantization_info_output, RoundingPolicy policy) +SimpleTensor reduction_operation(const SimpleTensor &src, const TensorShape &dst_shape, unsigned int axis, ReductionOperation op, + DataType output_type, QuantizationInfo quantization_info_output, RoundingPolicy policy) { ARM_COMPUTE_UNUSED(quantization_info_output); - return compute_reduction_operation(src, dst_shape, axis, op, policy); + return compute_reduction_operation(src, dst_shape, axis, op, output_type, policy); } template <> -SimpleTensor reduction_operation(const SimpleTensor &src, const TensorShape &dst_shape, unsigned int axis, ReductionOperation op, QuantizationInfo quantization_info_output, RoundingPolicy policy) +SimpleTensor reduction_operation(const SimpleTensor &src, const TensorShape &dst_shape, unsigned int axis, ReductionOperation op, + DataType output_type, QuantizationInfo quantization_info_output, RoundingPolicy policy) { if(src.data_type() == DataType::QASYMM8) { // If the operation is MEAN_SUM, we can directly use the uint8 implementation without taking into account scale and offset if(op == ReductionOperation::MEAN_SUM && src.quantization_info() == quantization_info_output) { - return compute_reduction_operation(src, dst_shape, axis, op, policy); + return compute_reduction_operation(src, dst_shape, axis, op, output_type, policy); } else { SimpleTensor src_f = convert_from_asymmetric(src); - SimpleTensor dst_f = reference::reduction_operation(src_f, dst_shape, axis, op); + SimpleTensor dst_f = reference::reduction_operation(src_f, dst_shape, axis, op, output_type); return convert_to_asymmetric(dst_f, quantization_info_output); } } else { - return compute_reduction_operation(src, dst_shape, axis, op, policy); + return compute_reduction_operation(src, dst_shape, axis, op, output_type, policy); } } template <> -SimpleTensor reduction_operation(const SimpleTensor &src, const TensorShape &dst_shape, unsigned int axis, ReductionOperation op, QuantizationInfo quantization_info_output, RoundingPolicy policy) +SimpleTensor reduction_operation(const SimpleTensor &src, const TensorShape &dst_shape, unsigned int axis, + ReductionOperation op, DataType output_type, QuantizationInfo quantization_info_output, RoundingPolicy policy) { if(src.data_type() == DataType::QASYMM8_SIGNED) { // If the operation is MEAN_SUM, we can directly use the int8 implementation without taking into account scale and offset if(op == ReductionOperation::MEAN_SUM && src.quantization_info() == quantization_info_output) { - return compute_reduction_operation(src, dst_shape, axis, op, policy); + return compute_reduction_operation(src, dst_shape, axis, op, output_type, policy); } else { SimpleTensor src_f = convert_from_asymmetric(src); - SimpleTensor dst_f = reference::reduction_operation(src_f, dst_shape, axis, op); + SimpleTensor dst_f = reference::reduction_operation(src_f, dst_shape, axis, op, output_type); return convert_to_asymmetric(dst_f, quantization_info_output); } } else { - return compute_reduction_operation(src, dst_shape, axis, op, policy); + return compute_reduction_operation(src, dst_shape, axis, op, output_type, policy); } } template SimpleTensor reduction_operation(const SimpleTensor &src, const TensorShape &dst_shape, unsigned int axis, ReductionOperation op, - QuantizationInfo quantization_info_output = QuantizationInfo(), RoundingPolicy policy = RoundingPolicy::TO_ZERO); + DataType output_type = DataType::S32, QuantizationInfo quantization_info_output = QuantizationInfo(), + RoundingPolicy policy = RoundingPolicy::TO_ZERO); + template SimpleTensor reduction_operation(const SimpleTensor &src, const TensorShape &dst_shape, unsigned int axis, ReductionOperation op, + DataType output_type = DataType::S32, QuantizationInfo quantization_info_output = QuantizationInfo(), RoundingPolicy policy = RoundingPolicy::TO_ZERO); template SimpleTensor reduction_operation(const SimpleTensor &src, const TensorShape &dst_shape, unsigned int axis, ReductionOperation op, + DataType output_type = DataType::S32, QuantizationInfo quantization_info_output = QuantizationInfo(), RoundingPolicy policy = RoundingPolicy::TO_ZERO); + template SimpleTensor reduction_operation(const SimpleTensor &src, const TensorShape &dst_shape, unsigned int axis, ReductionOperation op, + DataType output_type = DataType::S32, QuantizationInfo quantization_info_output = QuantizationInfo(), RoundingPolicy policy = RoundingPolicy::TO_ZERO); template SimpleTensor reduction_operation(const SimpleTensor &src, const TensorShape &dst_shape, unsigned int axis, ReductionOperation op, + DataType output_type = DataType::S32, QuantizationInfo quantization_info_output = QuantizationInfo(), RoundingPolicy policy = RoundingPolicy::TO_ZERO); template SimpleTensor reduction_operation(const SimpleTensor &src, const TensorShape &dst_shape, unsigned int axis, ReductionOperation op, + DataType output_type = DataType::S32, QuantizationInfo quantization_info_output = QuantizationInfo(), RoundingPolicy policy = RoundingPolicy::TO_ZERO); template SimpleTensor reduction_operation(const SimpleTensor &src, const TensorShape &dst_shape, unsigned int axis, ReductionOperation op, + DataType output_type = DataType::S32, QuantizationInfo quantization_info_output = QuantizationInfo(), RoundingPolicy policy = RoundingPolicy::TO_ZERO); +template SimpleTensor reduction_operation(const SimpleTensor &src, const TensorShape &dst_shape, unsigned int axis, ReductionOperation op, + DataType output_type = DataType::S32, QuantizationInfo quantization_info_output = QuantizationInfo(), + RoundingPolicy policy = RoundingPolicy::TO_ZERO); } // namespace reference } // namespace validation } // namespace test diff --git a/tests/validation/reference/ReductionOperation.h b/tests/validation/reference/ReductionOperation.h index dd97778834fff96d5f80025b30c9239fdb09fd1a..fb2e7a70939afbc60620842bbf7917bdf03e3c49 100644 --- a/tests/validation/reference/ReductionOperation.h +++ b/tests/validation/reference/ReductionOperation.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2020 Arm Limited. + * Copyright (c) 2017-2020, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,8 +24,8 @@ #ifndef ARM_COMPUTE_TEST_REDUCTION_OPERATION_H #define ARM_COMPUTE_TEST_REDUCTION_OPERATION_H -#include "tests/SimpleTensor.h" #include "arm_compute/core/Rounding.h" +#include "tests/SimpleTensor.h" #include "tests/validation/Helpers.h" namespace arm_compute @@ -37,7 +37,7 @@ namespace validation namespace reference { template -SimpleTensor reduction_operation(const SimpleTensor &src, const TensorShape &dst_shape, unsigned int axis, ReductionOperation op, +SimpleTensor reduction_operation(const SimpleTensor &src, const TensorShape &dst_shape, unsigned int axis, ReductionOperation op, DataType output_type = DataType::S32, QuantizationInfo quantization_info_output = QuantizationInfo(), RoundingPolicy policy = RoundingPolicy::TO_ZERO); } // namespace reference } // namespace validation diff --git a/utils/TypePrinter.h b/utils/TypePrinter.h index 0327cd1b358f90b35c0a40ef97c4609e597006c3..4bc326b574978da6faae956a54960fb65d97ff78 100644 --- a/utils/TypePrinter.h +++ b/utils/TypePrinter.h @@ -46,6 +46,10 @@ #include "arm_compute/dynamic_fusion/sketch/attributes/ResizeAttributes.h" #include "arm_compute/dynamic_fusion/sketch/attributes/SoftmaxAttributes.h" #include "arm_compute/dynamic_fusion/sketch/gpu/operators/GpuPool2d.h" +#include "arm_compute/function_info/ConvolutionInfo.h" +#include "arm_compute/function_info/FullyConnectedLayerInfo.h" +#include "arm_compute/function_info/GEMMInfo.h" +#include "arm_compute/function_info/MatMulInfo.h" #include "arm_compute/runtime/CL/CLTunerTypes.h" #include "arm_compute/runtime/CL/CLTypes.h" #include "arm_compute/runtime/FunctionDescriptors.h" @@ -484,10 +488,7 @@ inline ::std::ostream &operator<<(::std::ostream &os, const BoundingBoxTransform #if defined(ARM_COMPUTE_ENABLE_BF16) inline ::std::ostream &operator<<(::std::ostream &os, const bfloat16 &v) { - std::stringstream str; - str << v; - os << str.str(); - return os; + return os << float(v); } #endif /* defined(ARM_COMPUTE_ENABLE_BF16) */ @@ -3690,9 +3691,7 @@ inline ::std::ostream &operator<<(::std::ostream &os, const arm_compute::MatMulI os << "MatMulKernelInfo=" << "[" << "adj_lhs=" << matmul_info.adj_lhs() << ", " - << "adj_rhs=" << matmul_info.adj_rhs() << ", " - << "fused_activation=" << matmul_info.fused_activation().activation() << "]"; - + << "adj_rhs=" << matmul_info.adj_rhs() << "] "; return os; } /** Formatted output of the arm_compute::MatMulInfo type. diff --git a/utils/Utils.h b/utils/Utils.h index e3a5bb2c3c22ddd8fbea2605f6fdda4a36d9e72b..d181022ffe0cfba1d868f0734829b43c30776669 100644 --- a/utils/Utils.h +++ b/utils/Utils.h @@ -383,7 +383,22 @@ public: // Check if the typestring matches the given one std::string expect_typestr = get_typestring(tensor.info()->data_type()); - ARM_COMPUTE_ERROR_ON_MSG(_typestring != expect_typestr, "Typestrings mismatch"); + + bool enable_f32_to_f16_conversion = false; + if(_typestring != expect_typestr) + { + const std::string f32_typestring = "data_layout()); // Correct dimensions (Needs to match TensorShape dimension corrections) @@ -427,7 +442,7 @@ public: case arm_compute::DataType::F16: { // Read data - if(!are_layouts_different && !_fortran_order && tensor.info()->padding().empty()) + if(!are_layouts_different && !_fortran_order && tensor.info()->padding().empty() && !enable_f32_to_f16_conversion) { // If tensor has no padding read directly from stream. _fs.read(reinterpret_cast(tensor.buffer()), tensor.info()->total_size()); @@ -466,7 +481,17 @@ public: { Coordinates dst(id); arm_compute::permute(dst, perm); - _fs.read(reinterpret_cast(tensor.ptr_to_element(dst)), tensor.info()->element_size()); + if(enable_f32_to_f16_conversion) + { + float f32_val = 0; + _fs.read(reinterpret_cast(&f32_val), 4u); + half f16_val = half_float::half_cast(f32_val); + *(reinterpret_cast(tensor.ptr_to_element(dst))) = f16_val; + } + else + { + _fs.read(reinterpret_cast(tensor.ptr_to_element(dst)), tensor.info()->element_size()); + } }); }